diff --git a/.gitignore b/.gitignore
index eff6268c1973c..1c8d4b15925de 100644
--- a/.gitignore
+++ b/.gitignore
@@ -53,5 +53,11 @@ autoconf/autom4te.cache
 # VS2017 and VSCode config files.
 .vscode
 .vs
-# clangd index
-.clangd
+# clangd index. (".clangd" is a config file now, thus trailing slash)
+.clangd/
+.cache
+# static analyzer regression testing project files
+/clang/utils/analyzer/projects/*/CachedSource
+/clang/utils/analyzer/projects/*/PatchedSource
+/clang/utils/analyzer/projects/*/ScanBuildResults
+/clang/utils/analyzer/projects/*/RefScanBuildResults
diff --git a/clang-tools-extra/clang-change-namespace/CMakeLists.txt b/clang-tools-extra/clang-change-namespace/CMakeLists.txt
index bfce9869dde84..4cd9e7520b97b 100644
--- a/clang-tools-extra/clang-change-namespace/CMakeLists.txt
+++ b/clang-tools-extra/clang-change-namespace/CMakeLists.txt
@@ -5,6 +5,9 @@ set(LLVM_LINK_COMPONENTS
 
 add_clang_library(clangChangeNamespace
   ChangeNamespace.cpp
+
+  DEPENDS
+  omp_gen
 )
 
 clang_target_link_libraries(clangChangeNamespace
diff --git a/clang-tools-extra/clang-change-namespace/ChangeNamespace.cpp b/clang-tools-extra/clang-change-namespace/ChangeNamespace.cpp
index e2a70db4102b2..61ae7c4cc7036 100644
--- a/clang-tools-extra/clang-change-namespace/ChangeNamespace.cpp
+++ b/clang-tools-extra/clang-change-namespace/ChangeNamespace.cpp
@@ -19,14 +19,8 @@ namespace change_namespace {
 
 namespace {
 
-inline std::string
-joinNamespaces(const llvm::SmallVectorImpl<StringRef> &Namespaces) {
-  if (Namespaces.empty())
-    return "";
-  std::string Result(Namespaces.front());
-  for (auto I = Namespaces.begin() + 1, E = Namespaces.end(); I != E; ++I)
-    Result += ("::" + *I).str();
-  return Result;
+inline std::string joinNamespaces(ArrayRef<StringRef> Namespaces) {
+  return llvm::join(Namespaces, "::");
 }
 
 // Given "a::b::c", returns {"a", "b", "c"}.
diff --git a/clang-tools-extra/clang-doc/CMakeLists.txt b/clang-tools-extra/clang-doc/CMakeLists.txt
index 56ec9eb6e94e2..a43660c488495 100644
--- a/clang-tools-extra/clang-doc/CMakeLists.txt
+++ b/clang-tools-extra/clang-doc/CMakeLists.txt
@@ -15,6 +15,9 @@ add_clang_library(clangDoc
   Representation.cpp
   Serialize.cpp
   YAMLGenerator.cpp
+
+  DEPENDS
+  omp_gen
 )
 
 clang_target_link_libraries(clangDoc
diff --git a/clang-tools-extra/clang-include-fixer/CMakeLists.txt b/clang-tools-extra/clang-include-fixer/CMakeLists.txt
index d8685cb20758c..5581d436f2fe4 100644
--- a/clang-tools-extra/clang-include-fixer/CMakeLists.txt
+++ b/clang-tools-extra/clang-include-fixer/CMakeLists.txt
@@ -12,6 +12,9 @@ add_clang_library(clangIncludeFixer
 
   LINK_LIBS
   findAllSymbols
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangIncludeFixer
diff --git a/clang-tools-extra/clang-include-fixer/find-all-symbols/CMakeLists.txt b/clang-tools-extra/clang-include-fixer/find-all-symbols/CMakeLists.txt
index c0760e8b69c6c..0d9f686dfe80a 100644
--- a/clang-tools-extra/clang-include-fixer/find-all-symbols/CMakeLists.txt
+++ b/clang-tools-extra/clang-include-fixer/find-all-symbols/CMakeLists.txt
@@ -12,6 +12,9 @@ add_clang_library(findAllSymbols
   PragmaCommentHandler.cpp
   STLPostfixHeaderMap.cpp
   SymbolInfo.cpp
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(findAllSymbols
diff --git a/clang-tools-extra/clang-include-fixer/plugin/CMakeLists.txt b/clang-tools-extra/clang-include-fixer/plugin/CMakeLists.txt
index df792ea1bbe43..6d0328ed83124 100644
--- a/clang-tools-extra/clang-include-fixer/plugin/CMakeLists.txt
+++ b/clang-tools-extra/clang-include-fixer/plugin/CMakeLists.txt
@@ -10,4 +10,7 @@ add_clang_library(clangIncludeFixerPlugin
   clangSema
   clangTooling
   ${LLVM_PTHREAD_LIB}
+
+  DEPENDS
+  omp_gen
   )
diff --git a/clang-tools-extra/clang-move/CMakeLists.txt b/clang-tools-extra/clang-move/CMakeLists.txt
index e3bef84baa003..0b748ce6cd99a 100644
--- a/clang-tools-extra/clang-move/CMakeLists.txt
+++ b/clang-tools-extra/clang-move/CMakeLists.txt
@@ -6,6 +6,9 @@ set(LLVM_LINK_COMPONENTS
 add_clang_library(clangMove
   Move.cpp
   HelperDeclRefGraph.cpp
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangMove
diff --git a/clang-tools-extra/clang-move/Move.cpp b/clang-tools-extra/clang-move/Move.cpp
index 28184a0dce0c6..3f09f68a8046f 100644
--- a/clang-tools-extra/clang-move/Move.cpp
+++ b/clang-tools-extra/clang-move/Move.cpp
@@ -552,20 +552,22 @@ void ClangMoveTool::registerMatchers(ast_matchers::MatchFinder *Finder) {
 
   // Match static functions/variable definitions which are defined in named
   // namespaces.
-  Optional<ast_matchers::internal::Matcher<NamedDecl>> HasAnySymbolNames;
+  SmallVector<std::string, 4> QualNames;
+  QualNames.reserve(Context->Spec.Names.size());
   for (StringRef SymbolName : Context->Spec.Names) {
-    llvm::StringRef GlobalSymbolName = SymbolName.trim().ltrim(':');
-    const auto HasName = hasName(("::" + GlobalSymbolName).str());
-    HasAnySymbolNames =
-        HasAnySymbolNames ? anyOf(*HasAnySymbolNames, HasName) : HasName;
+    QualNames.push_back(("::" + SymbolName.trim().ltrim(':')).str());
   }
 
-  if (!HasAnySymbolNames) {
+  if (QualNames.empty()) {
     llvm::errs() << "No symbols being moved.\n";
     return;
   }
+
+  ast_matchers::internal::Matcher<NamedDecl> HasAnySymbolNames =
+      hasAnyName(SmallVector<StringRef, 4>(QualNames.begin(), QualNames.end()));
+
   auto InMovedClass =
-      hasOutermostEnclosingClass(cxxRecordDecl(*HasAnySymbolNames));
+      hasOutermostEnclosingClass(cxxRecordDecl(HasAnySymbolNames));
 
   // Matchers for helper declarations in old.cc.
   auto InAnonymousNS = hasParent(namespaceDecl(isAnonymous()));
@@ -612,17 +614,17 @@ void ClangMoveTool::registerMatchers(ast_matchers::MatchFinder *Finder) {
   // Create a MatchCallback for class declarations.
   MatchCallbacks.push_back(std::make_unique<ClassDeclarationMatch>(this));
   // Match moved class declarations.
-  auto MovedClass = cxxRecordDecl(InOldFiles, *HasAnySymbolNames,
-                                  isDefinition(), TopLevelDecl)
-                        .bind("moved_class");
+  auto MovedClass =
+      cxxRecordDecl(InOldFiles, HasAnySymbolNames, isDefinition(), TopLevelDecl)
+          .bind("moved_class");
   Finder->addMatcher(MovedClass, MatchCallbacks.back().get());
   // Match moved class methods (static methods included) which are defined
   // outside moved class declaration.
-  Finder->addMatcher(
-      cxxMethodDecl(InOldFiles, ofOutermostEnclosingClass(*HasAnySymbolNames),
-                    isDefinition())
-          .bind("class_method"),
-      MatchCallbacks.back().get());
+  Finder->addMatcher(cxxMethodDecl(InOldFiles,
+                                   ofOutermostEnclosingClass(HasAnySymbolNames),
+                                   isDefinition())
+                         .bind("class_method"),
+                     MatchCallbacks.back().get());
   // Match static member variable definition of the moved class.
   Finder->addMatcher(
       varDecl(InMovedClass, InOldFiles, isDefinition(), isStaticDataMember())
@@ -630,20 +632,20 @@ void ClangMoveTool::registerMatchers(ast_matchers::MatchFinder *Finder) {
       MatchCallbacks.back().get());
 
   MatchCallbacks.push_back(std::make_unique<FunctionDeclarationMatch>(this));
-  Finder->addMatcher(functionDecl(InOldFiles, *HasAnySymbolNames, TopLevelDecl)
+  Finder->addMatcher(functionDecl(InOldFiles, HasAnySymbolNames, TopLevelDecl)
                          .bind("function"),
                      MatchCallbacks.back().get());
 
   MatchCallbacks.push_back(std::make_unique<VarDeclarationMatch>(this));
   Finder->addMatcher(
-      varDecl(InOldFiles, *HasAnySymbolNames, TopLevelDecl).bind("var"),
+      varDecl(InOldFiles, HasAnySymbolNames, TopLevelDecl).bind("var"),
       MatchCallbacks.back().get());
 
   // Match enum definition in old.h. Enum helpers (which are defined in old.cc)
   // will not be moved for now no matter whether they are used or not.
   MatchCallbacks.push_back(std::make_unique<EnumDeclarationMatch>(this));
   Finder->addMatcher(
-      enumDecl(InOldHeader, *HasAnySymbolNames, isDefinition(), TopLevelDecl)
+      enumDecl(InOldHeader, HasAnySymbolNames, isDefinition(), TopLevelDecl)
           .bind("enum"),
       MatchCallbacks.back().get());
 
@@ -653,7 +655,7 @@ void ClangMoveTool::registerMatchers(ast_matchers::MatchFinder *Finder) {
   MatchCallbacks.push_back(std::make_unique<TypeAliasMatch>(this));
   Finder->addMatcher(namedDecl(anyOf(typedefDecl().bind("typedef"),
                                      typeAliasDecl().bind("type_alias")),
-                               InOldHeader, *HasAnySymbolNames, TopLevelDecl),
+                               InOldHeader, HasAnySymbolNames, TopLevelDecl),
                      MatchCallbacks.back().get());
 }
 
diff --git a/clang-tools-extra/clang-query/CMakeLists.txt b/clang-tools-extra/clang-query/CMakeLists.txt
index 8192ec4770ebc..043d26147a7f1 100644
--- a/clang-tools-extra/clang-query/CMakeLists.txt
+++ b/clang-tools-extra/clang-query/CMakeLists.txt
@@ -7,6 +7,9 @@ set(LLVM_LINK_COMPONENTS
 add_clang_library(clangQuery
   Query.cpp
   QueryParser.cpp
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangQuery
diff --git a/clang-tools-extra/clang-query/Query.cpp b/clang-tools-extra/clang-query/Query.cpp
index 2fc7af6a56e14..ca2a285e9eb7a 100644
--- a/clang-tools-extra/clang-query/Query.cpp
+++ b/clang-tools-extra/clang-query/Query.cpp
@@ -46,12 +46,12 @@ bool HelpQuery::run(llvm::raw_ostream &OS, QuerySession &QS) const {
         "  set traversal <kind>              "
         "Set traversal kind of clang-query session. Available kinds are:\n"
         "    AsIs                            "
-        "Print and match the AST as clang sees it.\n"
+        "Print and match the AST as clang sees it.  This mode is the "
+        "default.\n"
         "    IgnoreImplicitCastsAndParentheses  "
         "Omit implicit casts and parens in matching and dumping.\n"
         "    IgnoreUnlessSpelledInSource     "
-        "Omit AST nodes unless spelled in the source.  This mode is the "
-        "default.\n"
+        "Omit AST nodes unless spelled in the source.\n"
         "  set output <feature>              "
         "Set whether to output only <feature> content.\n"
         "  enable output <feature>           "
@@ -157,8 +157,7 @@ bool MatchQuery::run(llvm::raw_ostream &OS, QuerySession &QS) const {
           OS << "Binding for \"" << BI->first << "\":\n";
           const ASTContext &Ctx = AST->getASTContext();
           const SourceManager &SM = Ctx.getSourceManager();
-          ASTDumper Dumper(OS, &Ctx.getCommentCommandTraits(), &SM,
-                SM.getDiagnostics().getShowColors(), Ctx.getPrintingPolicy());
+          ASTDumper Dumper(OS, Ctx, SM.getDiagnostics().getShowColors());
           Dumper.SetTraversalKind(QS.TK);
           Dumper.Visit(BI->second);
           OS << "\n";
diff --git a/clang-tools-extra/clang-query/QuerySession.h b/clang-tools-extra/clang-query/QuerySession.h
index 1660e4039f613..20c788b206a0a 100644
--- a/clang-tools-extra/clang-query/QuerySession.h
+++ b/clang-tools-extra/clang-query/QuerySession.h
@@ -26,7 +26,7 @@ class QuerySession {
   QuerySession(llvm::ArrayRef<std::unique_ptr<ASTUnit>> ASTs)
       : ASTs(ASTs), PrintOutput(false), DiagOutput(true),
         DetailedASTOutput(false), BindRoot(true), PrintMatcher(false),
-        Terminate(false), TK(ast_type_traits::TK_IgnoreUnlessSpelledInSource) {}
+        Terminate(false), TK(ast_type_traits::TK_AsIs) {}
 
   llvm::ArrayRef<std::unique_ptr<ASTUnit>> ASTs;
 
diff --git a/clang-tools-extra/clang-reorder-fields/CMakeLists.txt b/clang-tools-extra/clang-reorder-fields/CMakeLists.txt
index 995efbb2f83a3..4013c46433e65 100644
--- a/clang-tools-extra/clang-reorder-fields/CMakeLists.txt
+++ b/clang-tools-extra/clang-reorder-fields/CMakeLists.txt
@@ -5,6 +5,9 @@ set(LLVM_LINK_COMPONENTS
 
 add_clang_library(clangReorderFields
   ReorderFieldsAction.cpp
+
+  DEPENDS
+  omp_gen
 )
 
 clang_target_link_libraries(clangReorderFields
diff --git a/clang-tools-extra/clang-tidy/CMakeLists.txt b/clang-tools-extra/clang-tidy/CMakeLists.txt
index f552cb3ad835a..02573534ccaef 100644
--- a/clang-tools-extra/clang-tidy/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/CMakeLists.txt
@@ -15,6 +15,7 @@ add_clang_library(clangTidy
 
   DEPENDS
   ClangSACheckers
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidy
diff --git a/clang-tools-extra/clang-tidy/ClangTidy.cpp b/clang-tools-extra/clang-tidy/ClangTidy.cpp
index 163aad2ec3782..d6913dfd3c078 100644
--- a/clang-tools-extra/clang-tidy/ClangTidy.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidy.cpp
@@ -15,6 +15,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "ClangTidy.h"
+#include "ClangTidyCheck.h"
 #include "ClangTidyDiagnosticConsumer.h"
 #include "ClangTidyModuleRegistry.h"
 #include "ClangTidyProfiling.h"
diff --git a/clang-tools-extra/clang-tidy/ClangTidy.h b/clang-tools-extra/clang-tidy/ClangTidy.h
index 0f3c3ac8ab96c..99dcbb6a269a8 100644
--- a/clang-tools-extra/clang-tidy/ClangTidy.h
+++ b/clang-tools-extra/clang-tidy/ClangTidy.h
@@ -9,19 +9,22 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CLANGTIDY_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CLANGTIDY_H
 
-#include "ClangTidyCheck.h"
 #include "ClangTidyDiagnosticConsumer.h"
 #include "ClangTidyOptions.h"
-#include "llvm/Support/raw_ostream.h"
 #include <memory>
 #include <vector>
 
+namespace llvm {
+class raw_ostream;
+} // namespace llvm
+
 namespace clang {
 
+class ASTConsumer;
 class CompilerInstance;
 namespace tooling {
 class CompilationDatabase;
-}
+} // namespace tooling
 
 namespace tidy {
 
diff --git a/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp b/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
index 7ddf054a21a94..e149978bcdeaf 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
@@ -76,16 +76,25 @@ ClangTidyCheck::OptionsView::get(StringRef LocalName) const {
   return llvm::make_error<MissingOptionError>((NamePrefix + LocalName).str());
 }
 
+static ClangTidyOptions::OptionMap::const_iterator
+findPriorityOption(const ClangTidyOptions::OptionMap &Options, StringRef NamePrefix,
+          StringRef LocalName) {
+  auto IterLocal = Options.find((NamePrefix + LocalName).str());
+  auto IterGlobal = Options.find(LocalName.str());
+  if (IterLocal == Options.end())
+    return IterGlobal;
+  if (IterGlobal == Options.end())
+    return IterLocal;
+  if (IterLocal->second.Priority >= IterGlobal->second.Priority)
+    return IterLocal;
+  return IterGlobal;
+}
+
 llvm::Expected<std::string>
 ClangTidyCheck::OptionsView::getLocalOrGlobal(StringRef LocalName) const {
-  auto IterLocal = CheckOptions.find(NamePrefix + LocalName.str());
-  auto IterGlobal = CheckOptions.find(LocalName.str());
-  if (IterLocal != CheckOptions.end() &&
-      (IterGlobal == CheckOptions.end() ||
-       IterLocal->second.Priority >= IterGlobal->second.Priority))
-    return IterLocal->second.Value;
-  if (IterGlobal != CheckOptions.end())
-    return IterGlobal->second.Value;
+  auto Iter = findPriorityOption(CheckOptions, NamePrefix, LocalName);
+  if (Iter != CheckOptions.end())
+    return Iter->second.Value;
   return llvm::make_error<MissingOptionError>((NamePrefix + LocalName).str());
 }
 
@@ -124,14 +133,9 @@ bool ClangTidyCheck::OptionsView::get<bool>(StringRef LocalName,
 template <>
 llvm::Expected<bool>
 ClangTidyCheck::OptionsView::getLocalOrGlobal<bool>(StringRef LocalName) const {
-  auto IterLocal = CheckOptions.find(NamePrefix + LocalName.str());
-  auto IterGlobal = CheckOptions.find(LocalName.str());
-  if (IterLocal != CheckOptions.end() &&
-      (IterGlobal == CheckOptions.end() ||
-       IterLocal->second.Priority >= IterGlobal->second.Priority))
-    return getAsBool(IterLocal->second.Value, NamePrefix + LocalName);
-  if (IterGlobal != CheckOptions.end())
-    return getAsBool(IterGlobal->second.Value, llvm::Twine(LocalName));
+  auto Iter = findPriorityOption(CheckOptions, NamePrefix, LocalName);
+  if (Iter != CheckOptions.end())
+    return getAsBool(Iter->second.Value, Iter->first);
   return llvm::make_error<MissingOptionError>((NamePrefix + LocalName).str());
 }
 
@@ -157,12 +161,13 @@ void ClangTidyCheck::OptionsView::store(ClangTidyOptions::OptionMap &Options,
   store(Options, LocalName, llvm::itostr(Value));
 }
 
-llvm::Expected<int64_t> ClangTidyCheck::OptionsView::getEnumInt(
-    StringRef LocalName, ArrayRef<std::pair<StringRef, int64_t>> Mapping,
-    bool CheckGlobal, bool IgnoreCase) {
-  auto Iter = CheckOptions.find((NamePrefix + LocalName).str());
-  if (CheckGlobal && Iter == CheckOptions.end())
-    Iter = CheckOptions.find(LocalName.str());
+llvm::Expected<int64_t>
+ClangTidyCheck::OptionsView::getEnumInt(StringRef LocalName,
+                                        ArrayRef<NameAndValue> Mapping,
+                                        bool CheckGlobal, bool IgnoreCase) {
+  auto Iter = CheckGlobal
+                  ? findPriorityOption(CheckOptions, NamePrefix, LocalName)
+                  : CheckOptions.find((NamePrefix + LocalName).str());
   if (Iter == CheckOptions.end())
     return llvm::make_error<MissingOptionError>((NamePrefix + LocalName).str());
 
@@ -171,19 +176,19 @@ llvm::Expected<int64_t> ClangTidyCheck::OptionsView::getEnumInt(
   unsigned EditDistance = -1;
   for (const auto &NameAndEnum : Mapping) {
     if (IgnoreCase) {
-      if (Value.equals_lower(NameAndEnum.first))
-        return NameAndEnum.second;
-    } else if (Value.equals(NameAndEnum.first)) {
-      return NameAndEnum.second;
-    } else if (Value.equals_lower(NameAndEnum.first)) {
-      Closest = NameAndEnum.first;
+      if (Value.equals_lower(NameAndEnum.second))
+        return NameAndEnum.first;
+    } else if (Value.equals(NameAndEnum.second)) {
+      return NameAndEnum.first;
+    } else if (Value.equals_lower(NameAndEnum.second)) {
+      Closest = NameAndEnum.second;
       EditDistance = 0;
       continue;
     }
-    unsigned Distance = Value.edit_distance(NameAndEnum.first);
+    unsigned Distance = Value.edit_distance(NameAndEnum.second);
     if (Distance < EditDistance) {
       EditDistance = Distance;
-      Closest = NameAndEnum.first;
+      Closest = NameAndEnum.second;
     }
   }
   if (EditDistance < 3)
diff --git a/clang-tools-extra/clang-tidy/ClangTidyCheck.h b/clang-tools-extra/clang-tidy/ClangTidyCheck.h
index 84438c21a30b8..3c625ee0cb796 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyCheck.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyCheck.h
@@ -13,20 +13,26 @@
 #include "ClangTidyOptions.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/Basic/Diagnostic.h"
-#include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/Optional.h"
-#include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Error.h"
-#include <memory>
 #include <type_traits>
+#include <utility>
 #include <vector>
 
 namespace clang {
 
 class CompilerInstance;
+class SourceManager;
 
 namespace tidy {
 
+/// This class should be specialized by any enum type that needs to be converted
+/// to and from an \ref llvm::StringRef.
+template <class T> struct OptionEnumMapping {
+  // Specializations of this struct must implement this function.
+  static ArrayRef<std::pair<T, StringRef>> getEnumMapping() = delete;
+};
+
 template <typename T> class OptionError : public llvm::ErrorInfo<T> {
   std::error_code convertToErrorCode() const override {
     return llvm::inconvertibleErrorCode();
@@ -313,36 +319,38 @@ class ClangTidyCheck : public ast_matchers::MatchFinder::MatchCallback {
     }
 
     /// Read a named option from the ``Context`` and parse it as an
-    /// enum type ``T`` using the \p Mapping provided. If \p IgnoreCase is set,
-    /// it will search the mapping ignoring the case.
+    /// enum type ``T``.
     ///
     /// Reads the option with the check-local name \p LocalName from the
     /// ``CheckOptions``. If the corresponding key is not present, returns a
     /// ``MissingOptionError``. If the key can't be parsed as a ``T`` returns a
     /// ``UnparseableEnumOptionError``.
+    ///
+    /// \ref clang::tidy::OptionEnumMapping must be specialized for ``T`` to
+    /// supply the mapping required to convert between ``T`` and a string.
     template <typename T>
     std::enable_if_t<std::is_enum<T>::value, llvm::Expected<T>>
-    get(StringRef LocalName, ArrayRef<std::pair<StringRef, T>> Mapping,
-        bool IgnoreCase = false) {
-      if (llvm::Expected<int64_t> ValueOr = getEnumInt(
-              LocalName, typeEraseMapping(Mapping), false, IgnoreCase))
+    get(StringRef LocalName, bool IgnoreCase = false) {
+      if (llvm::Expected<int64_t> ValueOr =
+              getEnumInt(LocalName, typeEraseMapping<T>(), false, IgnoreCase))
         return static_cast<T>(*ValueOr);
       else
         return std::move(ValueOr.takeError());
     }
 
     /// Read a named option from the ``Context`` and parse it as an
-    /// enum type ``T`` using the \p Mapping provided. If \p IgnoreCase is set,
-    /// it will search the mapping ignoring the case.
+    /// enum type ``T``.
     ///
     /// Reads the option with the check-local name \p LocalName from the
     /// ``CheckOptions``. If the corresponding key is not present or it can't be
     /// parsed as a ``T``, returns \p Default.
+    ///
+    /// \ref clang::tidy::OptionEnumMapping must be specialized for ``T`` to
+    /// supply the mapping required to convert between ``T`` and a string.
     template <typename T>
     std::enable_if_t<std::is_enum<T>::value, T>
-    get(StringRef LocalName, ArrayRef<std::pair<StringRef, T>> Mapping,
-        T Default, bool IgnoreCase = false) {
-      if (auto ValueOr = get(LocalName, Mapping, IgnoreCase))
+    get(StringRef LocalName, T Default, bool IgnoreCase = false) {
+      if (auto ValueOr = get<T>(LocalName, IgnoreCase))
         return *ValueOr;
       else
         logErrToStdErr(ValueOr.takeError());
@@ -350,40 +358,41 @@ class ClangTidyCheck : public ast_matchers::MatchFinder::MatchCallback {
     }
 
     /// Read a named option from the ``Context`` and parse it as an
-    /// enum type ``T`` using the \p Mapping provided. If \p IgnoreCase is set,
-    /// it will search the mapping ignoring the case.
+    /// enum type ``T``.
     ///
     /// Reads the option with the check-local name \p LocalName from local or
     /// global ``CheckOptions``. Gets local option first. If local is not
     /// present, falls back to get global option. If global option is not
     /// present either, returns a ``MissingOptionError``. If the key can't be
     /// parsed as a ``T`` returns a ``UnparseableEnumOptionError``.
+    ///
+    /// \ref clang::tidy::OptionEnumMapping must be specialized for ``T`` to
+    /// supply the mapping required to convert between ``T`` and a string.
     template <typename T>
     std::enable_if_t<std::is_enum<T>::value, llvm::Expected<T>>
     getLocalOrGlobal(StringRef LocalName,
-                     ArrayRef<std::pair<StringRef, T>> Mapping,
                      bool IgnoreCase = false) {
-      if (llvm::Expected<int64_t> ValueOr = getEnumInt(
-              LocalName, typeEraseMapping(Mapping), true, IgnoreCase))
+      if (llvm::Expected<int64_t> ValueOr =
+              getEnumInt(LocalName, typeEraseMapping<T>(), true, IgnoreCase))
         return static_cast<T>(*ValueOr);
       else
         return std::move(ValueOr.takeError());
     }
 
     /// Read a named option from the ``Context`` and parse it as an
-    /// enum type ``T`` using the \p Mapping provided. If \p IgnoreCase is set,
-    /// it will search the mapping ignoring the case.
+    /// enum type ``T``.
     ///
     /// Reads the option with the check-local name \p LocalName from local or
     /// global ``CheckOptions``. Gets local option first. If local is not
     /// present, falls back to get global option. If global option is not
     /// present either or it can't be parsed as a ``T``, returns \p Default.
+    ///
+    /// \ref clang::tidy::OptionEnumMapping must be specialized for ``T`` to
+    /// supply the mapping required to convert between ``T`` and a string.
     template <typename T>
     std::enable_if_t<std::is_enum<T>::value, T>
-    getLocalOrGlobal(StringRef LocalName,
-                     ArrayRef<std::pair<StringRef, T>> Mapping, T Default,
-                     bool IgnoreCase = false) {
-      if (auto ValueOr = getLocalOrGlobal(LocalName, Mapping, IgnoreCase))
+    getLocalOrGlobal(StringRef LocalName, T Default, bool IgnoreCase = false) {
+      if (auto ValueOr = getLocalOrGlobal<T>(LocalName, IgnoreCase))
         return *ValueOr;
       else
         logErrToStdErr(ValueOr.takeError());
@@ -401,21 +410,25 @@ class ClangTidyCheck : public ast_matchers::MatchFinder::MatchCallback {
                int64_t Value) const;
 
     /// Stores an option with the check-local name \p LocalName as the string
-    /// representation of the Enum \p Value using the \p Mapping to \p Options.
+    /// representation of the Enum \p Value to \p Options.
+    ///
+    /// \ref clang::tidy::OptionEnumMapping must be specialized for ``T`` to
+    /// supply the mapping required to convert between ``T`` and a string.
     template <typename T>
     std::enable_if_t<std::is_enum<T>::value>
-    store(ClangTidyOptions::OptionMap &Options, StringRef LocalName, T Value,
-          ArrayRef<std::pair<StringRef, T>> Mapping) {
+    store(ClangTidyOptions::OptionMap &Options, StringRef LocalName, T Value) {
+      ArrayRef<std::pair<T, StringRef>> Mapping =
+          OptionEnumMapping<T>::getEnumMapping();
       auto Iter = llvm::find_if(
-          Mapping, [&](const std::pair<StringRef, T> &NameAndEnum) {
-            return NameAndEnum.second == Value;
+          Mapping, [&](const std::pair<T, StringRef> &NameAndEnum) {
+            return NameAndEnum.first == Value;
           });
       assert(Iter != Mapping.end() && "Unknown Case Value");
-      store(Options, LocalName, Iter->first);
+      store(Options, LocalName, Iter->second);
     }
 
   private:
-    using NameAndValue = std::pair<StringRef, int64_t>;
+    using NameAndValue = std::pair<int64_t, StringRef>;
 
     llvm::Expected<int64_t> getEnumInt(StringRef LocalName,
                                        ArrayRef<NameAndValue> Mapping,
@@ -423,12 +436,14 @@ class ClangTidyCheck : public ast_matchers::MatchFinder::MatchCallback {
 
     template <typename T>
     std::enable_if_t<std::is_enum<T>::value, std::vector<NameAndValue>>
-    typeEraseMapping(ArrayRef<std::pair<StringRef, T>> Mapping) {
+    typeEraseMapping() {
+      ArrayRef<std::pair<T, StringRef>> Mapping =
+          OptionEnumMapping<T>::getEnumMapping();
       std::vector<NameAndValue> Result;
       Result.reserve(Mapping.size());
       for (auto &MappedItem : Mapping) {
-        Result.emplace_back(MappedItem.first,
-                            static_cast<int64_t>(MappedItem.second));
+        Result.emplace_back(static_cast<int64_t>(MappedItem.first),
+                            MappedItem.second);
       }
       return Result;
     }
diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
index 5c23323b03f0d..521e6ef549b96 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
@@ -18,16 +18,21 @@
 #include "ClangTidyDiagnosticConsumer.h"
 #include "ClangTidyOptions.h"
 #include "GlobList.h"
+#include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTDiagnostic.h"
 #include "clang/AST/Attr.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticOptions.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/SourceManager.h"
 #include "clang/Frontend/DiagnosticRenderer.h"
 #include "clang/Tooling/Core/Diagnostic.h"
+#include "clang/Tooling/Core/Replacement.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
-#include "llvm/Support/Regex.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/Regex.h"
 #include <tuple>
 #include <vector>
 using namespace clang;
diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
index 90cc4da3021f1..eaa7f1851ce32 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
@@ -12,21 +12,15 @@
 #include "ClangTidyOptions.h"
 #include "ClangTidyProfiling.h"
 #include "clang/Basic/Diagnostic.h"
-#include "clang/Basic/SourceManager.h"
 #include "clang/Tooling/Core/Diagnostic.h"
-#include "clang/Tooling/Refactoring.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/StringMap.h"
-#include "llvm/Support/Timer.h"
-
-namespace llvm {
-class Regex;
-}
+#include "llvm/Support/Regex.h"
 
 namespace clang {
 
 class ASTContext;
 class CompilerInstance;
+class SourceManager;
 namespace ast_matchers {
 class MatchFinder;
 }
diff --git a/clang-tools-extra/clang-tidy/ClangTidyModule.cpp b/clang-tools-extra/clang-tidy/ClangTidyModule.cpp
index 6e44f72079877..e9031d498eeff 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyModule.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "ClangTidyModule.h"
+#include "ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/ClangTidyModule.h b/clang-tools-extra/clang-tidy/ClangTidyModule.h
index 5c484f1a68ddb..31cf4774a885a 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyModule.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyModule.h
@@ -9,17 +9,20 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CLANGTIDYMODULE_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CLANGTIDYMODULE_H
 
-#include "ClangTidy.h"
+#include "ClangTidyOptions.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include <functional>
 #include <map>
 #include <memory>
 #include <string>
-#include <utility>
 
 namespace clang {
 namespace tidy {
 
+class ClangTidyCheck;
+class ClangTidyContext;
+
 /// A collection of \c ClangTidyCheckFactory instances.
 ///
 /// All clang-tidy modules register their check factories with an instance of
@@ -27,12 +30,12 @@ namespace tidy {
 class ClangTidyCheckFactories {
 public:
   using CheckFactory = std::function<std::unique_ptr<ClangTidyCheck>(
-      StringRef Name, ClangTidyContext *Context)>;
+      llvm::StringRef Name, ClangTidyContext *Context)>;
 
   /// Registers check \p Factory with name \p Name.
   ///
   /// For all checks that have default constructors, use \c registerCheck.
-  void registerCheckFactory(StringRef Name, CheckFactory Factory);
+  void registerCheckFactory(llvm::StringRef Name, CheckFactory Factory);
 
   /// Registers the \c CheckType with the name \p Name.
   ///
@@ -55,9 +58,9 @@ class ClangTidyCheckFactories {
   ///   }
   /// };
   /// \endcode
-  template <typename CheckType> void registerCheck(StringRef CheckName) {
+  template <typename CheckType> void registerCheck(llvm::StringRef CheckName) {
     registerCheckFactory(CheckName,
-                         [](StringRef Name, ClangTidyContext *Context) {
+                         [](llvm::StringRef Name, ClangTidyContext *Context) {
                            return std::make_unique<CheckType>(Name, Context);
                          });
   }
diff --git a/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp b/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
index 4fa78e2803f13..4c1f286737817 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
@@ -287,7 +287,7 @@ FileOptionsProvider::FileOptionsProvider(
     const ClangTidyOptions &OverrideOptions,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS)
     : FileOptionsBaseProvider(GlobalOptions, DefaultOptions, OverrideOptions,
-                              VFS){};
+                              VFS){}
 
 FileOptionsProvider::FileOptionsProvider(
     const ClangTidyGlobalOptions &GlobalOptions,
diff --git a/clang-tools-extra/clang-tidy/ClangTidyProfiling.h b/clang-tools-extra/clang-tidy/ClangTidyProfiling.h
index a266e388bbe3a..9487a34620e0a 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyProfiling.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyProfiling.h
@@ -13,10 +13,11 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/Support/Chrono.h"
 #include "llvm/Support/Timer.h"
-#include "llvm/Support/raw_ostream.h"
 #include <string>
-#include <utility>
-#include <vector>
+
+namespace llvm {
+class raw_ostream;
+} // namespace llvm
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/CMakeLists.txt b/clang-tools-extra/clang-tidy/abseil/CMakeLists.txt
index 7d8601ad60e45..b6ea21879dafa 100644
--- a/clang-tools-extra/clang-tidy/abseil/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/abseil/CMakeLists.txt
@@ -28,6 +28,9 @@ add_clang_library(clangTidyAbseilModule
   LINK_LIBS
   clangTidy
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyAbseilModule
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationAdditionCheck.h b/clang-tools-extra/clang-tidy/abseil/DurationAdditionCheck.h
index 64b4d89e834f6..4f93be6abaa10 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationAdditionCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/DurationAdditionCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_TIMEADDITIONCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_TIMEADDITIONCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationComparisonCheck.h b/clang-tools-extra/clang-tidy/abseil/DurationComparisonCheck.h
index 4c7085fce7731..b3d6add1dd5bf 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationComparisonCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/DurationComparisonCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_DURATIONCOMPARISONCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_DURATIONCOMPARISONCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationConversionCastCheck.h b/clang-tools-extra/clang-tidy/abseil/DurationConversionCastCheck.h
index bc1d6203c1c62..0aa95fffa3bbf 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationConversionCastCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/DurationConversionCastCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_DURATIONCONVERSIONCASTCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_DURATIONCONVERSIONCASTCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.cpp b/clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.cpp
index 01e01469acee8..ab55dd7f9019e 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.cpp
@@ -9,6 +9,7 @@
 #include "DurationDivisionCheck.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.h b/clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.h
index 42f6cd3701ba0..a4e4dd57a4d80 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_DURATIONDIVISIONCHECK_H_
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_DURATIONDIVISIONCHECK_H_
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.cpp b/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.cpp
index d46265d18ae80..ed7e9fd0dfade 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.cpp
+++ b/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.cpp
@@ -10,6 +10,7 @@
 #include "DurationRewriter.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
 #include "clang/Tooling/FixIt.h"
 
 using namespace clang::ast_matchers;
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.h b/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.h
index cc773153d5118..f804ca296f6bd 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_DURATIONFACTORYFLOATCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_DURATIONFACTORYFLOATCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationFactoryScaleCheck.h b/clang-tools-extra/clang-tidy/abseil/DurationFactoryScaleCheck.h
index 3b79b91d0eeaa..15a9f3fca3c43 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationFactoryScaleCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/DurationFactoryScaleCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_DURATIONFACTORYSCALECHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_DURATIONFACTORYSCALECHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationSubtractionCheck.h b/clang-tools-extra/clang-tidy/abseil/DurationSubtractionCheck.h
index 89deb375785f1..19a684a995cde 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationSubtractionCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/DurationSubtractionCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_DURATIONSUBTRACTIONCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_DURATIONSUBTRACTIONCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.h b/clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.h
index 180613388918e..b3eb5677f4f7b 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_TIMEDOUBLECONVERSIONCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_TIMEDOUBLECONVERSIONCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.h b/clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.h
index 4b7e18bc43987..439e7db9982e5 100644
--- a/clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_FASTERSTRSPLITDELIMITERCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_FASTERSTRSPLITDELIMITERCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/NoInternalDependenciesCheck.h b/clang-tools-extra/clang-tidy/abseil/NoInternalDependenciesCheck.h
index c71a81fe29361..9b0f2323e6b74 100644
--- a/clang-tools-extra/clang-tidy/abseil/NoInternalDependenciesCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/NoInternalDependenciesCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_NOINTERNALDEPSCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_NOINTERNALDEPSCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/NoNamespaceCheck.h b/clang-tools-extra/clang-tidy/abseil/NoNamespaceCheck.h
index e19f53b972bcf..9460a39f274cc 100644
--- a/clang-tools-extra/clang-tidy/abseil/NoNamespaceCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/NoNamespaceCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_NONAMESPACECHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_NONAMESPACECHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/RedundantStrcatCallsCheck.h b/clang-tools-extra/clang-tidy/abseil/RedundantStrcatCallsCheck.h
index 850e354ef8672..7146497733091 100644
--- a/clang-tools-extra/clang-tidy/abseil/RedundantStrcatCallsCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/RedundantStrcatCallsCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_REDUNDANTSTRCATCALLSCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_REDUNDANTSTRCATCALLSCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/StrCatAppendCheck.h b/clang-tools-extra/clang-tidy/abseil/StrCatAppendCheck.h
index b5b36813d5f2d..3b41b2be13108 100644
--- a/clang-tools-extra/clang-tidy/abseil/StrCatAppendCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/StrCatAppendCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_STRCATAPPENDCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_STRCATAPPENDCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.cpp b/clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.cpp
index 9539e28cd7909..11bbcbcb527f5 100644
--- a/clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.cpp
+++ b/clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.cpp
@@ -12,8 +12,8 @@
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Frontend/CompilerInstance.h"
-
-#include <cassert>
+#include "clang/Lex/Lexer.h"
+#include "clang/Lex/Preprocessor.h"
 
 using namespace clang::ast_matchers;
 
@@ -27,7 +27,6 @@ StringFindStartswithCheck::StringFindStartswithCheck(StringRef Name,
       StringLikeClasses(utils::options::parseStringList(
           Options.get("StringLikeClasses", "::std::basic_string"))),
       IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::getMapping(),
                                             utils::IncludeSorter::IS_LLVM)),
       AbseilStringsMatchHeader(
           Options.get("AbseilStringsMatchHeader", "absl/strings/match.h")) {}
@@ -122,8 +121,7 @@ void StringFindStartswithCheck::storeOptions(
     ClangTidyOptions::OptionMap &Opts) {
   Options.store(Opts, "StringLikeClasses",
                 utils::options::serializeStringList(StringLikeClasses));
-  Options.store(Opts, "IncludeStyle", IncludeStyle,
-                utils::IncludeSorter::getMapping());
+  Options.store(Opts, "IncludeStyle", IncludeStyle);
   Options.store(Opts, "AbseilStringsMatchHeader", AbseilStringsMatchHeader);
 }
 
diff --git a/clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.h b/clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.h
index 8a702f73cdad7..d232d3b3efb61 100644
--- a/clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_STRINGFINDSTARTSWITHCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_STRINGFINDSTARTSWITHCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 #include "../utils/IncludeInserter.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 
diff --git a/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp b/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp
index f60ce20007661..87af9ea6cd5ac 100644
--- a/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp
@@ -15,6 +15,9 @@
 #include "clang/Tooling/Transformer/RewriteRule.h"
 #include "clang/Tooling/Transformer/Stencil.h"
 
+// FixItHint - Hint to check documentation script to mark this check as
+// providing a FixIt.
+
 using namespace clang::ast_matchers;
 
 namespace clang {
diff --git a/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.h b/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.h
index 351cc3784a96e..a021dcdb74a92 100644
--- a/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_STRINGFINDSTRCONTAINSCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_STRINGFINDSTRCONTAINSCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 #include "../utils/TransformerClangTidyCheck.h"
 
 namespace clang {
diff --git a/clang-tools-extra/clang-tidy/abseil/TimeComparisonCheck.h b/clang-tools-extra/clang-tidy/abseil/TimeComparisonCheck.h
index 55984338544b2..f855c31d52825 100644
--- a/clang-tools-extra/clang-tidy/abseil/TimeComparisonCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/TimeComparisonCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_TIMECOMPARECHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_TIMECOMPARECHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp b/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp
index 368aa576c5f43..92319715fbc56 100644
--- a/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.cpp
@@ -10,6 +10,7 @@
 #include "DurationRewriter.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
 #include "clang/Tooling/FixIt.h"
 
 using namespace clang::ast_matchers;
diff --git a/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.h b/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.h
index 4e3cd3b5bee5c..9fe58b7d9303d 100644
--- a/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_TIMESUBTRACTIONCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_TIMESUBTRACTIONCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp b/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp
index a660cbdebda5a..4d28d1242f25c 100644
--- a/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp
@@ -10,6 +10,7 @@
 #include "DurationRewriter.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
 
 using namespace clang::ast_matchers;
 
diff --git a/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.h b/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.h
index 55978615faf91..7a450a8e92496 100644
--- a/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_UPGRADEDURATIONCONVERSIONSCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ABSEIL_UPGRADEDURATIONCONVERSIONSCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 #include <unordered_set>
 
diff --git a/clang-tools-extra/clang-tidy/add_new_check.py b/clang-tools-extra/clang-tidy/add_new_check.py
index 454d128a5dcb3..231f43c0b8f3d 100755
--- a/clang-tools-extra/clang-tidy/add_new_check.py
+++ b/clang-tools-extra/clang-tidy/add_new_check.py
@@ -289,6 +289,19 @@ def write_test(module_path, module, check_name, test_extension):
 """ % {'check_name_dashes': check_name_dashes})
 
 
+def get_actual_filename(dirname, filename):
+  if not os.path.isdir(dirname): 
+    return ""
+  name = os.path.join(dirname, filename)
+  if (os.path.isfile(name)):
+    return name
+  caselessname = filename.lower()
+  for file in os.listdir(dirname):
+    if (file.lower() == caselessname):
+      return os.path.join(dirname, file)
+  return ""
+
+
 # Recreates the list of checks in the docs/clang-tidy/checks directory.
 def update_checks_list(clang_tidy_path):
   docs_dir = os.path.join(clang_tidy_path, '../docs/clang-tidy/checks')
@@ -304,7 +317,8 @@ def update_checks_list(clang_tidy_path):
   def has_auto_fix(check_name):
     dirname, _, check_name = check_name.partition("-")
 
-    checkerCode = os.path.join(dirname, get_camel_name(check_name)) + ".cpp"
+    checkerCode = get_actual_filename(dirname,
+                                      get_camel_name(check_name) + '.cpp')
 
     if not os.path.isfile(checkerCode):
       return ""
diff --git a/clang-tools-extra/clang-tidy/android/CMakeLists.txt b/clang-tools-extra/clang-tidy/android/CMakeLists.txt
index 72ed16061f6c0..d00e1a7fa1dbf 100644
--- a/clang-tools-extra/clang-tidy/android/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/android/CMakeLists.txt
@@ -25,6 +25,9 @@ add_clang_library(clangTidyAndroidModule
   LINK_LIBS
   clangTidy
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyAndroidModule
diff --git a/clang-tools-extra/clang-tidy/boost/CMakeLists.txt b/clang-tools-extra/clang-tidy/boost/CMakeLists.txt
index 2bddb707dd0e1..8bf959d632749 100644
--- a/clang-tools-extra/clang-tidy/boost/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/boost/CMakeLists.txt
@@ -10,6 +10,9 @@ add_clang_library(clangTidyBoostModule
   LINK_LIBS
   clangTidy
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyBoostModule
diff --git a/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.cpp
index 1f3dec497c076..3591a7da8eb77 100644
--- a/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/BadSignalToKillThreadCheck.cpp
@@ -9,6 +9,7 @@
 #include "BadSignalToKillThreadCheck.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Preprocessor.h"
 
 using namespace clang::ast_matchers;
 
diff --git a/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp
index e5f9ebd486265..8c89350c6d07c 100644
--- a/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp
@@ -11,6 +11,7 @@
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/Analysis/CloneDetection.h"
+#include "clang/Lex/Lexer.h"
 #include "llvm/Support/Casting.h"
 
 using namespace clang;
diff --git a/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.h b/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.h
index 4a3eadf68f348..1f8ebbe2d3a5e 100644
--- a/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.h
+++ b/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.h
@@ -10,7 +10,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_BRANCHCLONECHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_BRANCHCLONECHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp b/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
index d010c3ce7e522..3f735a8484d84 100644
--- a/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
@@ -34,6 +34,7 @@
 #include "MisplacedWideningCastCheck.h"
 #include "MoveForwardingReferenceCheck.h"
 #include "MultipleStatementMacroCheck.h"
+#include "NoEscapeCheck.h"
 #include "NotNullTerminatedResultCheck.h"
 #include "ParentVirtualCallCheck.h"
 #include "PosixReturnCheck.h"
@@ -120,6 +121,7 @@ class BugproneModule : public ClangTidyModule {
         "bugprone-multiple-statement-macro");
     CheckFactories.registerCheck<cppcoreguidelines::NarrowingConversionsCheck>(
         "bugprone-narrowing-conversions");
+    CheckFactories.registerCheck<NoEscapeCheck>("bugprone-no-escape");
     CheckFactories.registerCheck<NotNullTerminatedResultCheck>(
         "bugprone-not-null-terminated-result");
     CheckFactories.registerCheck<ParentVirtualCallCheck>(
diff --git a/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt b/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
index e220fd0e67fbb..6e7a94928a5af 100644
--- a/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
@@ -29,6 +29,7 @@ add_clang_library(clangTidyBugproneModule
   MisplacedWideningCastCheck.cpp
   MoveForwardingReferenceCheck.cpp
   MultipleStatementMacroCheck.cpp
+  NoEscapeCheck.cpp
   NotNullTerminatedResultCheck.cpp
   ParentVirtualCallCheck.cpp
   PosixReturnCheck.cpp
@@ -62,6 +63,9 @@ add_clang_library(clangTidyBugproneModule
   clangTidy
   clangTidyCppCoreGuidelinesModule
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyBugproneModule
diff --git a/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
index c9dd47ef0cf92..310fbec72a500 100644
--- a/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
@@ -10,8 +10,10 @@
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/Analysis/Analyses/ExprMutationAnalyzer.h"
+#include "../utils/Aliasing.h"
 
 using namespace clang::ast_matchers;
+using clang::tidy::utils::hasPtrOrReferenceInFunc;
 
 namespace clang {
 namespace tidy {
@@ -24,54 +26,6 @@ loopEndingStmt(internal::Matcher<Stmt> Internal) {
                     callExpr(Internal, callee(functionDecl(isNoReturn())))));
 }
 
-/// Return whether `S` is a reference to the declaration of `Var`.
-static bool isAccessForVar(const Stmt *S, const VarDecl *Var) {
-  if (const auto *DRE = dyn_cast<DeclRefExpr>(S))
-    return DRE->getDecl() == Var;
-
-  return false;
-}
-
-/// Return whether `Var` has a pointer or reference in `S`.
-static bool isPtrOrReferenceForVar(const Stmt *S, const VarDecl *Var) {
-  if (const auto *DS = dyn_cast<DeclStmt>(S)) {
-    for (const Decl *D : DS->getDeclGroup()) {
-      if (const auto *LeftVar = dyn_cast<VarDecl>(D)) {
-        if (LeftVar->hasInit() && LeftVar->getType()->isReferenceType()) {
-          return isAccessForVar(LeftVar->getInit(), Var);
-        }
-      }
-    }
-  } else if (const auto *UnOp = dyn_cast<UnaryOperator>(S)) {
-    if (UnOp->getOpcode() == UO_AddrOf)
-      return isAccessForVar(UnOp->getSubExpr(), Var);
-  }
-
-  return false;
-}
-
-/// Return whether `Var` has a pointer or reference in `S`.
-static bool hasPtrOrReferenceInStmt(const Stmt *S, const VarDecl *Var) {
-  if (isPtrOrReferenceForVar(S, Var))
-    return true;
-
-  for (const Stmt *Child : S->children()) {
-    if (!Child)
-      continue;
-
-    if (hasPtrOrReferenceInStmt(Child, Var))
-      return true;
-  }
-
-  return false;
-}
-
-/// Return whether `Var` has a pointer or reference in `Func`.
-static bool hasPtrOrReferenceInFunc(const FunctionDecl *Func,
-                                    const VarDecl *Var) {
-  return hasPtrOrReferenceInStmt(Func->getBody(), Var);
-}
-
 /// Return whether `Var` was changed in `LoopStmt`.
 static bool isChanged(const Stmt *LoopStmt, const VarDecl *Var,
                       ASTContext *Context) {
diff --git a/clang-tools-extra/clang-tidy/bugprone/MisplacedOperatorInStrlenInAllocCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/MisplacedOperatorInStrlenInAllocCheck.cpp
index ed2c17607169d..7c8c26370118f 100644
--- a/clang-tools-extra/clang-tidy/bugprone/MisplacedOperatorInStrlenInAllocCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/MisplacedOperatorInStrlenInAllocCheck.cpp
@@ -9,6 +9,7 @@
 #include "MisplacedOperatorInStrlenInAllocCheck.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
 
 using namespace clang::ast_matchers;
 
@@ -18,12 +19,10 @@ namespace bugprone {
 
 void MisplacedOperatorInStrlenInAllocCheck::registerMatchers(
     MatchFinder *Finder) {
-  const auto StrLenFunc = functionDecl(anyOf(
-      hasName("::strlen"), hasName("::std::strlen"), hasName("::strnlen"),
-      hasName("::std::strnlen"), hasName("::strnlen_s"),
-      hasName("::std::strnlen_s"), hasName("::wcslen"),
-      hasName("::std::wcslen"), hasName("::wcsnlen"), hasName("::std::wcsnlen"),
-      hasName("::wcsnlen_s"), hasName("std::wcsnlen_s")));
+  const auto StrLenFunc = functionDecl(hasAnyName(
+      "::strlen", "::std::strlen", "::strnlen", "::std::strnlen", "::strnlen_s",
+      "::std::strnlen_s", "::wcslen", "::std::wcslen", "::wcsnlen",
+      "::std::wcsnlen", "::wcsnlen_s", "std::wcsnlen_s"));
 
   const auto BadUse =
       callExpr(callee(StrLenFunc),
@@ -41,12 +40,10 @@ void MisplacedOperatorInStrlenInAllocCheck::registerMatchers(
             hasDescendant(BadUse)),
       BadUse);
 
-  const auto Alloc0Func =
-      functionDecl(anyOf(hasName("::malloc"), hasName("std::malloc"),
-                         hasName("::alloca"), hasName("std::alloca")));
-  const auto Alloc1Func =
-      functionDecl(anyOf(hasName("::calloc"), hasName("std::calloc"),
-                         hasName("::realloc"), hasName("std::realloc")));
+  const auto Alloc0Func = functionDecl(
+      hasAnyName("::malloc", "std::malloc", "::alloca", "std::alloca"));
+  const auto Alloc1Func = functionDecl(
+      hasAnyName("::calloc", "std::calloc", "::realloc", "std::realloc"));
 
   const auto Alloc0FuncPtr =
       varDecl(hasType(isConstQualified()),
diff --git a/clang-tools-extra/clang-tidy/bugprone/MisplacedPointerArithmeticInAllocCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/MisplacedPointerArithmeticInAllocCheck.cpp
index ca5209448b00c..2a6a0ae53a4f3 100644
--- a/clang-tools-extra/clang-tidy/bugprone/MisplacedPointerArithmeticInAllocCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/MisplacedPointerArithmeticInAllocCheck.cpp
@@ -9,6 +9,7 @@
 #include "MisplacedPointerArithmeticInAllocCheck.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
 
 using namespace clang::ast_matchers;
 
@@ -18,10 +19,9 @@ namespace bugprone {
 
 void MisplacedPointerArithmeticInAllocCheck::registerMatchers(
     MatchFinder *Finder) {
-  const auto AllocFunc = functionDecl(
-      anyOf(hasName("::malloc"), hasName("std::malloc"), hasName("::alloca"),
-            hasName("::calloc"), hasName("std::calloc"), hasName("::realloc"),
-            hasName("std::realloc")));
+  const auto AllocFunc =
+      functionDecl(hasAnyName("::malloc", "std::malloc", "::alloca", "::calloc",
+                              "std::calloc", "::realloc", "std::realloc"));
 
   const auto AllocFuncPtr =
       varDecl(hasType(isConstQualified()),
diff --git a/clang-tools-extra/clang-tidy/bugprone/NoEscapeCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/NoEscapeCheck.cpp
new file mode 100644
index 0000000000000..654e80d9a9c6c
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/bugprone/NoEscapeCheck.cpp
@@ -0,0 +1,51 @@
+//===--- NoEscapeCheck.cpp - clang-tidy -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "NoEscapeCheck.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang {
+namespace tidy {
+namespace bugprone {
+
+void NoEscapeCheck::registerMatchers(MatchFinder *Finder) {
+  Finder->addMatcher(callExpr(callee(functionDecl(hasName("::dispatch_async"))),
+                              argumentCountIs(2),
+                              hasArgument(1, blockExpr().bind("arg-block"))),
+                     this);
+  Finder->addMatcher(callExpr(callee(functionDecl(hasName("::dispatch_after"))),
+                              argumentCountIs(3),
+                              hasArgument(2, blockExpr().bind("arg-block"))),
+                     this);
+}
+
+void NoEscapeCheck::check(const MatchFinder::MatchResult &Result) {
+  const auto *MatchedEscapingBlock =
+      Result.Nodes.getNodeAs<BlockExpr>("arg-block");
+  const BlockDecl *EscapingBlockDecl = MatchedEscapingBlock->getBlockDecl();
+  for (const BlockDecl::Capture &CapturedVar : EscapingBlockDecl->captures()) {
+    const VarDecl *Var = CapturedVar.getVariable();
+    if (Var && Var->hasAttr<NoEscapeAttr>()) {
+      // FIXME: Add a method to get the location of the use of a CapturedVar so
+      // that we can diagnose the use of the pointer instead of the block.
+      diag(MatchedEscapingBlock->getBeginLoc(),
+           "pointer %0 with attribute 'noescape' is captured by an "
+           "asynchronously-executed block")
+          << Var;
+      diag(Var->getBeginLoc(), "the 'noescape' attribute is declared here.",
+           DiagnosticIDs::Note);
+    }
+  }
+}
+
+} // namespace bugprone
+} // namespace tidy
+} // namespace clang
diff --git a/clang-tools-extra/clang-tidy/bugprone/NoEscapeCheck.h b/clang-tools-extra/clang-tidy/bugprone/NoEscapeCheck.h
new file mode 100644
index 0000000000000..126c37c9df0a1
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/bugprone/NoEscapeCheck.h
@@ -0,0 +1,39 @@
+//===--- NoEscapeCheck.h - clang-tidy ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_NOESCAPECHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_NOESCAPECHECK_H
+
+#include "../ClangTidyCheck.h"
+
+namespace clang {
+namespace tidy {
+namespace bugprone {
+
+/// Block arguments in `dispatch_async()` and `dispatch_after()` are guaranteed
+/// to escape. If those blocks capture any pointers with the `noescape`
+/// attribute, then we warn the user of their error.
+///
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/bugprone-no-escape.html
+class NoEscapeCheck : public ClangTidyCheck {
+public:
+  NoEscapeCheck(StringRef Name, ClangTidyContext *Context)
+      : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.Blocks;
+  }
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+};
+
+} // namespace bugprone
+} // namespace tidy
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_NOESCAPECHECK_H
diff --git a/clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.cpp
index 87c1c85e70976..269d69c1c124e 100644
--- a/clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.cpp
@@ -12,6 +12,7 @@
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Lex/Lexer.h"
 #include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
 
 using namespace clang::ast_matchers;
 
diff --git a/clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.h b/clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.h
index f2a7cd3998f74..18bd2335adfe3 100644
--- a/clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.h
+++ b/clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_NOT_NULL_TERMINATED_RESULT_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_NOT_NULL_TERMINATED_RESULT_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.cpp
index 076e239826cb6..586940ffd795a 100644
--- a/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.cpp
@@ -10,6 +10,7 @@
 #include "../utils/Matchers.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
 
 using namespace clang::ast_matchers;
 
diff --git a/clang-tools-extra/clang-tidy/bugprone/ReservedIdentifierCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ReservedIdentifierCheck.cpp
index 9619524a0c038..47a84c225711e 100644
--- a/clang-tools-extra/clang-tidy/bugprone/ReservedIdentifierCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/ReservedIdentifierCheck.cpp
@@ -11,6 +11,7 @@
 #include "../utils/OptionsUtils.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Token.h"
 #include <algorithm>
 #include <cctype>
 
diff --git a/clang-tools-extra/clang-tidy/bugprone/SpuriouslyWakeUpFunctionsCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SpuriouslyWakeUpFunctionsCheck.cpp
index 844d672f121fb..ee46818839644 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SpuriouslyWakeUpFunctionsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/SpuriouslyWakeUpFunctionsCheck.cpp
@@ -54,8 +54,7 @@ void SpuriouslyWakeUpFunctionsCheck::registerMatchers(MatchFinder *Finder) {
           .bind("wait"));
 
   auto hasWaitDescendantC = hasDescendant(
-      callExpr(callee(functionDecl(
-                   anyOf(hasName("cnd_wait"), hasName("cnd_timedwait")))))
+      callExpr(callee(functionDecl(hasAnyName("cnd_wait", "cnd_timedwait"))))
           .bind("wait"));
   if (getLangOpts().CPlusPlus) {
     // Check for `CON54-CPP`
diff --git a/clang-tools-extra/clang-tidy/bugprone/SuspiciousIncludeCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SuspiciousIncludeCheck.cpp
index 87ff284408dca..80004f91c0692 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SuspiciousIncludeCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/SuspiciousIncludeCheck.cpp
@@ -8,6 +8,7 @@
 
 #include "SuspiciousIncludeCheck.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/Lex/Preprocessor.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/bugprone/SuspiciousMemsetUsageCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SuspiciousMemsetUsageCheck.cpp
index 9f98316984ed8..37748d9fa8ccf 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SuspiciousMemsetUsageCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/SuspiciousMemsetUsageCheck.cpp
@@ -20,11 +20,19 @@ namespace tidy {
 namespace bugprone {
 
 void SuspiciousMemsetUsageCheck::registerMatchers(MatchFinder *Finder) {
-  // Note: void *memset(void *buffer, int fill_char, size_t byte_count);
+  // Match the standard memset:
+  // void *memset(void *buffer, int fill_char, size_t byte_count);
+  auto MemsetDecl =
+      functionDecl(hasName("::memset"),
+                   parameterCountIs(3),
+                   hasParameter(0, hasType(pointerType(pointee(voidType())))),
+                   hasParameter(1, hasType(isInteger())),
+                   hasParameter(2, hasType(isInteger())));
+
   // Look for memset(x, '0', z). Probably memset(x, 0, z) was intended.
   Finder->addMatcher(
       callExpr(
-          callee(functionDecl(hasName("::memset"))),
+          callee(MemsetDecl),
           hasArgument(1, characterLiteral(equals(static_cast<unsigned>('0')))
                              .bind("char-zero-fill")),
           unless(
@@ -36,14 +44,14 @@ void SuspiciousMemsetUsageCheck::registerMatchers(MatchFinder *Finder) {
 
   // Look for memset with an integer literal in its fill_char argument.
   // Will check if it gets truncated.
-  Finder->addMatcher(callExpr(callee(functionDecl(hasName("::memset"))),
+  Finder->addMatcher(callExpr(callee(MemsetDecl),
                               hasArgument(1, integerLiteral().bind("num-fill")),
                               unless(isInTemplateInstantiation())),
                      this);
 
   // Look for memset(x, y, 0) as that is most likely an argument swap.
   Finder->addMatcher(
-      callExpr(callee(functionDecl(hasName("::memset"))),
+      callExpr(callee(MemsetDecl),
                unless(hasArgument(1, anyOf(characterLiteral(equals(
                                                static_cast<unsigned>('0'))),
                                            integerLiteral()))),
diff --git a/clang-tools-extra/clang-tidy/bugprone/UnusedRaiiCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UnusedRaiiCheck.cpp
index 34a489e324e4b..70ce413d20ff1 100644
--- a/clang-tools-extra/clang-tidy/bugprone/UnusedRaiiCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/UnusedRaiiCheck.cpp
@@ -27,9 +27,10 @@ void UnusedRaiiCheck::registerMatchers(MatchFinder *Finder) {
   // Look for temporaries that are constructed in-place and immediately
   // destroyed. Look for temporaries created by a functional cast but not for
   // those returned from a call.
-  auto BindTemp =
-      cxxBindTemporaryExpr(unless(has(ignoringParenImpCasts(callExpr()))))
-          .bind("temp");
+  auto BindTemp = cxxBindTemporaryExpr(
+                      unless(has(ignoringParenImpCasts(callExpr()))),
+                      unless(has(ignoringParenImpCasts(objcMessageExpr()))))
+                      .bind("temp");
   Finder->addMatcher(
       traverse(ast_type_traits::TK_AsIs,
                exprWithCleanups(
@@ -79,6 +80,7 @@ void UnusedRaiiCheck::check(const MatchFinder::MatchResult &Result) {
   auto Matches =
       match(expr(hasDescendant(typeLoc().bind("t"))), *E, *Result.Context);
   const auto *TL = selectFirst<TypeLoc>("t", Matches);
+  assert(TL);
   D << FixItHint::CreateInsertion(
       Lexer::getLocForEndOfToken(TL->getEndLoc(), 0, *Result.SourceManager,
                                  getLangOpts()),
diff --git a/clang-tools-extra/clang-tidy/cert/CMakeLists.txt b/clang-tools-extra/clang-tidy/cert/CMakeLists.txt
index 2aa172718a383..e56de01b4780c 100644
--- a/clang-tools-extra/clang-tidy/cert/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/cert/CMakeLists.txt
@@ -28,6 +28,9 @@ add_clang_library(clangTidyCERTModule
   clangTidyPerformanceModule
   clangTidyReadabilityModule
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyCERTModule
diff --git a/clang-tools-extra/clang-tidy/cert/CommandProcessorCheck.cpp b/clang-tools-extra/clang-tidy/cert/CommandProcessorCheck.cpp
index 72b39d18f9f18..131ae9b3ed55f 100644
--- a/clang-tools-extra/clang-tidy/cert/CommandProcessorCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cert/CommandProcessorCheck.cpp
@@ -19,8 +19,7 @@ namespace cert {
 void CommandProcessorCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       callExpr(
-          callee(functionDecl(anyOf(hasName("::system"), hasName("::popen"),
-                                    hasName("::_popen")))
+          callee(functionDecl(hasAnyName("::system", "::popen", "::_popen"))
                      .bind("func")),
           // Do not diagnose when the call expression passes a null pointer
           // constant to system(); that only checks for the presence of a
diff --git a/clang-tools-extra/clang-tidy/cert/DontModifyStdNamespaceCheck.cpp b/clang-tools-extra/clang-tidy/cert/DontModifyStdNamespaceCheck.cpp
index 2b30933385644..f4051c34aad9c 100644
--- a/clang-tools-extra/clang-tidy/cert/DontModifyStdNamespaceCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cert/DontModifyStdNamespaceCheck.cpp
@@ -19,7 +19,7 @@ namespace cert {
 void DontModifyStdNamespaceCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       namespaceDecl(unless(isExpansionInSystemHeader()),
-                    anyOf(hasName("std"), hasName("posix")),
+                    hasAnyName("std", "posix"),
                     has(decl(unless(anyOf(
                         functionDecl(isExplicitTemplateSpecialization()),
                         cxxRecordDecl(isExplicitTemplateSpecialization()))))))
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/AvoidGotoCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/AvoidGotoCheck.h
index a00fe9cb4887a..9913d9caf6b28 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/AvoidGotoCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/AvoidGotoCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_AVOIDGOTOCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_AVOIDGOTOCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/CMakeLists.txt b/clang-tools-extra/clang-tidy/cppcoreguidelines/CMakeLists.txt
index 797799da8b087..39c2c552eb73e 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/CMakeLists.txt
@@ -32,6 +32,9 @@ add_clang_library(clangTidyCppCoreGuidelinesModule
   clangTidyModernizeModule
   clangTidyReadabilityModule
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyCppCoreGuidelinesModule
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.cpp
index 608df8a0ab441..f1755d3f9b855 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.cpp
@@ -10,6 +10,8 @@
 
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
 
 using namespace clang::ast_matchers;
 
@@ -25,13 +27,11 @@ InitVariablesCheck::InitVariablesCheck(StringRef Name,
                                        ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
       IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::getMapping(),
                                             utils::IncludeSorter::IS_LLVM)),
       MathHeader(Options.get("MathHeader", "math.h")) {}
 
 void InitVariablesCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle,
-                utils::IncludeSorter::getMapping());
+  Options.store(Opts, "IncludeStyle", IncludeStyle);
   Options.store(Opts, "MathHeader", MathHeader);
 }
 
@@ -40,6 +40,7 @@ void InitVariablesCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       varDecl(unless(hasInitializer(anything())), unless(isInstantiated()),
               isLocalVarDecl(), unless(isStaticLocal()), isDefinition(),
+              unless(hasParent(cxxCatchStmt())),
               optionally(hasParent(declStmt(hasParent(
                   cxxForRangeStmt(hasLoopVariable(varDecl().bind(BadDecl))))))),
               unless(equalsBoundNode(BadDecl)))
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/InterfacesGlobalInitCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/InterfacesGlobalInitCheck.h
index ecaaaa3360946..ddca98d062a11 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/InterfacesGlobalInitCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/InterfacesGlobalInitCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_INTERFACES_GLOBAL_INIT_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_INTERFACES_GLOBAL_INIT_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/MacroUsageCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/MacroUsageCheck.cpp
index 9c8652770963d..febc295d78e6a 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/MacroUsageCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/MacroUsageCheck.cpp
@@ -9,6 +9,7 @@
 #include "MacroUsageCheck.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Regex.h"
 #include <algorithm>
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/MacroUsageCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/MacroUsageCheck.h
index 1938eb6727181..81450802c86d6 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/MacroUsageCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/MacroUsageCheck.h
@@ -9,11 +9,12 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_MACROUSAGECHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_MACROUSAGECHECK_H
 
-#include "../ClangTidy.h"
-#include "clang/Lex/Preprocessor.h"
+#include "../ClangTidyCheck.h"
+#include "clang/Lex/MacroInfo.h"
 #include <string>
 
 namespace clang {
+class MacroDirective;
 namespace tidy {
 namespace cppcoreguidelines {
 
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/NarrowingConversionsCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/NarrowingConversionsCheck.h
index 6a9f38bb5d509..177cdabaa2527 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/NarrowingConversionsCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/NarrowingConversionsCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_NARROWING_CONVERSIONS_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_NARROWING_CONVERSIONS_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/NoMallocCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/NoMallocCheck.h
index cf676421bc53e..6be01923ce851 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/NoMallocCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/NoMallocCheck.h
@@ -9,9 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_NO_MALLOC_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_NO_MALLOC_H
 
-#include "../ClangTidy.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/OwningMemoryCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/OwningMemoryCheck.h
index aaa3511d2f2e8..bac16bb6e1b09 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/OwningMemoryCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/OwningMemoryCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_OWNING_MEMORY_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_OWNING_MEMORY_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsArrayToPointerDecayCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsArrayToPointerDecayCheck.h
index a219d03b1c12d..b81a46f0d02a1 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsArrayToPointerDecayCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsArrayToPointerDecayCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_BOUNDS_ARRAY_TO_POINTER_DECAY_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_BOUNDS_ARRAY_TO_POINTER_DECAY_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.cpp
index b48511287f883..dd0bedd742a40 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.cpp
@@ -22,7 +22,6 @@ ProBoundsConstantArrayIndexCheck::ProBoundsConstantArrayIndexCheck(
     StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context), GslHeader(Options.get("GslHeader", "")),
       IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::getMapping(),
                                             utils::IncludeSorter::IS_LLVM)) {}
 
 void ProBoundsConstantArrayIndexCheck::storeOptions(
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.h
index 1b5d7f1428db2..ac7475b4372db 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_BOUNDS_CONSTANT_ARRAY_INDEX_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_BOUNDS_CONSTANT_ARRAY_INDEX_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 #include "../utils/IncludeInserter.h"
 
 namespace clang {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h
index ab43c789e0d32..a3aef8e6899fe 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_BOUNDS_POINTER_ARITHMETIC_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_BOUNDS_POINTER_ARITHMETIC_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeConstCastCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeConstCastCheck.h
index 969e8092e2138..3689e2e963b09 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeConstCastCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeConstCastCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_TYPE_CONST_CAST_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_TYPE_CONST_CAST_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeCstyleCastCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeCstyleCastCheck.h
index 144b4c7b6da96..76d2b67107fe5 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeCstyleCastCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeCstyleCastCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_TYPE_CSTYLE_CAST_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_TYPE_CSTYLE_CAST_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.h
index 338fc14023883..5b4144396eab4 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_TYPE_MEMBER_INIT_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_TYPE_MEMBER_INIT_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeReinterpretCastCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeReinterpretCastCheck.h
index c33bbd66fae12..8be18003d2310 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeReinterpretCastCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeReinterpretCastCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_TYPE_REINTERPRETCAST_CHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_TYPE_REINTERPRETCAST_CHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeStaticCastDowncastCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeStaticCastDowncastCheck.h
index 5eacb42a613da..e3d7685319a35 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeStaticCastDowncastCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeStaticCastDowncastCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_TYPE_STATIC_CAST_DOWNCAST_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_TYPE_STATIC_CAST_DOWNCAST_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeUnionAccessCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeUnionAccessCheck.h
index 5d36e05225d1c..de4f8bb3fcafa 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeUnionAccessCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeUnionAccessCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_TYPE_UNION_ACCESS_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_TYPE_UNION_ACCESS_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeVarargCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeVarargCheck.h
index 11348d768e871..06de06e3e8361 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeVarargCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeVarargCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_TYPE_VARARG_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_TYPE_VARARG_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.h
index a2322a9f0031c..002c724f86b8c 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_SLICING_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_SLICING_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.h
index 78468cc5c31ef..83fca3d5799b0 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_SPECIAL_MEMBER_FUNCTIONS_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_SPECIAL_MEMBER_FUNCTIONS_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 #include "llvm/ADT/DenseMapInfo.h"
 
diff --git a/clang-tools-extra/clang-tidy/darwin/CMakeLists.txt b/clang-tools-extra/clang-tidy/darwin/CMakeLists.txt
index 28245b7bce91f..aeb9ac7071426 100644
--- a/clang-tools-extra/clang-tidy/darwin/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/darwin/CMakeLists.txt
@@ -11,6 +11,9 @@ add_clang_library(clangTidyDarwinModule
   LINK_LIBS
   clangTidy
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyDarwinModule
diff --git a/clang-tools-extra/clang-tidy/fuchsia/CMakeLists.txt b/clang-tools-extra/clang-tidy/fuchsia/CMakeLists.txt
index daf092eb49230..dd07813cded09 100644
--- a/clang-tools-extra/clang-tidy/fuchsia/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/fuchsia/CMakeLists.txt
@@ -17,6 +17,9 @@ add_clang_library(clangTidyFuchsiaModule
   clangTidy
   clangTidyGoogleModule
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyFuchsiaModule
diff --git a/clang-tools-extra/clang-tidy/fuchsia/DefaultArgumentsDeclarationsCheck.cpp b/clang-tools-extra/clang-tidy/fuchsia/DefaultArgumentsDeclarationsCheck.cpp
index 926c8b426dfe5..273c322edb8b4 100644
--- a/clang-tools-extra/clang-tidy/fuchsia/DefaultArgumentsDeclarationsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/fuchsia/DefaultArgumentsDeclarationsCheck.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "DefaultArgumentsDeclarationsCheck.h"
+#include "clang/Lex/Lexer.h"
 
 using namespace clang::ast_matchers;
 
diff --git a/clang-tools-extra/clang-tidy/google/AvoidNSObjectNewCheck.cpp b/clang-tools-extra/clang-tidy/google/AvoidNSObjectNewCheck.cpp
index b7388fd31bfc2..aab07bad81220 100644
--- a/clang-tools-extra/clang-tidy/google/AvoidNSObjectNewCheck.cpp
+++ b/clang-tools-extra/clang-tidy/google/AvoidNSObjectNewCheck.cpp
@@ -12,6 +12,7 @@
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
+#include "clang/Lex/Lexer.h"
 #include "llvm/Support/FormatVariadic.h"
 #include <map>
 #include <string>
diff --git a/clang-tools-extra/clang-tidy/google/AvoidUnderscoreInGoogletestNameCheck.cpp b/clang-tools-extra/clang-tidy/google/AvoidUnderscoreInGoogletestNameCheck.cpp
index a811d37af77f1..3a5aaa1e18328 100644
--- a/clang-tools-extra/clang-tidy/google/AvoidUnderscoreInGoogletestNameCheck.cpp
+++ b/clang-tools-extra/clang-tidy/google/AvoidUnderscoreInGoogletestNameCheck.cpp
@@ -13,6 +13,8 @@
 #include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Lex/MacroArgs.h"
+#include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/google/CMakeLists.txt b/clang-tools-extra/clang-tidy/google/CMakeLists.txt
index 68076c1d3bef0..e38ba8abb78d3 100644
--- a/clang-tools-extra/clang-tidy/google/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/google/CMakeLists.txt
@@ -27,6 +27,9 @@ add_clang_library(clangTidyGoogleModule
   clangTidy
   clangTidyReadabilityModule
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyGoogleModule
diff --git a/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp b/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp
index 16fed8156e81a..b2261ca7f5075 100644
--- a/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp
+++ b/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp
@@ -9,6 +9,8 @@
 #include "UpgradeGoogletestCaseCheck.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
 
 using namespace clang::ast_matchers;
 
diff --git a/clang-tools-extra/clang-tidy/hicpp/CMakeLists.txt b/clang-tools-extra/clang-tidy/hicpp/CMakeLists.txt
index 53edd21fa61bc..a25c21db4425a 100644
--- a/clang-tools-extra/clang-tidy/hicpp/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/hicpp/CMakeLists.txt
@@ -20,6 +20,9 @@ add_clang_library(clangTidyHICPPModule
   clangTidyPerformanceModule
   clangTidyReadabilityModule
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyHICPPModule
diff --git a/clang-tools-extra/clang-tidy/hicpp/ExceptionBaseclassCheck.h b/clang-tools-extra/clang-tidy/hicpp/ExceptionBaseclassCheck.h
index ed9fcb6f7eadf..ab5e95506ab09 100644
--- a/clang-tools-extra/clang-tidy/hicpp/ExceptionBaseclassCheck.h
+++ b/clang-tools-extra/clang-tidy/hicpp/ExceptionBaseclassCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_HICPP_EXCEPTION_BASECLASS_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_HICPP_EXCEPTION_BASECLASS_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp b/clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp
index ad367dcfed388..253c3893f0d42 100644
--- a/clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp
@@ -9,6 +9,7 @@
 #include "../ClangTidy.h"
 #include "../ClangTidyModule.h"
 #include "../ClangTidyModuleRegistry.h"
+#include "../bugprone/UndelegatedConstructorCheck.h"
 #include "../bugprone/UseAfterMoveCheck.h"
 #include "../cppcoreguidelines/AvoidGotoCheck.h"
 #include "../cppcoreguidelines/NoMallocCheck.h"
@@ -20,7 +21,6 @@
 #include "../google/ExplicitConstructorCheck.h"
 #include "../misc/NewDeleteOverloadsCheck.h"
 #include "../misc/StaticAssertCheck.h"
-#include "../bugprone/UndelegatedConstructorCheck.h"
 #include "../modernize/AvoidCArraysCheck.h"
 #include "../modernize/DeprecatedHeadersCheck.h"
 #include "../modernize/UseAutoCheck.h"
@@ -34,7 +34,7 @@
 #include "../performance/NoexceptMoveConstructorCheck.h"
 #include "../readability/BracesAroundStatementsCheck.h"
 #include "../readability/FunctionSizeCheck.h"
-#include "../readability/IdentifierNamingCheck.h"
+#include "../readability/NamedParameterCheck.h"
 #include "../readability/UppercaseLiteralSuffixCheck.h"
 #include "ExceptionBaseclassCheck.h"
 #include "MultiwayPathsCoveredCheck.h"
@@ -65,7 +65,7 @@ class HICPPModule : public ClangTidyModule {
         "hicpp-explicit-conversions");
     CheckFactories.registerCheck<readability::FunctionSizeCheck>(
         "hicpp-function-size");
-    CheckFactories.registerCheck<readability::IdentifierNamingCheck>(
+    CheckFactories.registerCheck<readability::NamedParameterCheck>(
         "hicpp-named-parameter");
     CheckFactories.registerCheck<bugprone::UseAfterMoveCheck>(
         "hicpp-invalid-access-moved");
diff --git a/clang-tools-extra/clang-tidy/hicpp/MultiwayPathsCoveredCheck.h b/clang-tools-extra/clang-tidy/hicpp/MultiwayPathsCoveredCheck.h
index 6c2a45f716341..c9f9b9f1110f1 100644
--- a/clang-tools-extra/clang-tidy/hicpp/MultiwayPathsCoveredCheck.h
+++ b/clang-tools-extra/clang-tidy/hicpp/MultiwayPathsCoveredCheck.h
@@ -9,9 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_HICPP_MULTIWAY_PATHS_COVERED_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_HICPP_MULTIWAY_PATHS_COVERED_H
 
-#include "../ClangTidy.h"
-#include "clang/ASTMatchers/ASTMatchFinder.h"
-#include <iostream>
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/hicpp/NoAssemblerCheck.h b/clang-tools-extra/clang-tidy/hicpp/NoAssemblerCheck.h
index d91c5de2fba0d..a613bc45a4dc9 100644
--- a/clang-tools-extra/clang-tidy/hicpp/NoAssemblerCheck.h
+++ b/clang-tools-extra/clang-tidy/hicpp/NoAssemblerCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_HICPP_NO_ASSEMBLER_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_HICPP_NO_ASSEMBLER_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/hicpp/SignedBitwiseCheck.h b/clang-tools-extra/clang-tidy/hicpp/SignedBitwiseCheck.h
index 74b3f0eb235de..3510fbdc09adc 100644
--- a/clang-tools-extra/clang-tidy/hicpp/SignedBitwiseCheck.h
+++ b/clang-tools-extra/clang-tidy/hicpp/SignedBitwiseCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_HICPP_SIGNED_BITWISE_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_HICPP_SIGNED_BITWISE_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/linuxkernel/CMakeLists.txt b/clang-tools-extra/clang-tidy/linuxkernel/CMakeLists.txt
index 5edb652182bec..7a86cfb32cb84 100644
--- a/clang-tools-extra/clang-tidy/linuxkernel/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/linuxkernel/CMakeLists.txt
@@ -10,6 +10,9 @@ add_clang_library(clangTidyLinuxKernelModule
   LINK_LIBS
   clangTidy
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyLinuxKernelModule
diff --git a/clang-tools-extra/clang-tidy/llvm/CMakeLists.txt b/clang-tools-extra/clang-tidy/llvm/CMakeLists.txt
index 74c92eeaa3a9e..5b5cbaffcc9e7 100644
--- a/clang-tools-extra/clang-tidy/llvm/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/llvm/CMakeLists.txt
@@ -15,6 +15,9 @@ add_clang_library(clangTidyLLVMModule
   clangTidy
   clangTidyReadabilityModule
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyLLVMModule
diff --git a/clang-tools-extra/clang-tidy/llvm/HeaderGuardCheck.cpp b/clang-tools-extra/clang-tidy/llvm/HeaderGuardCheck.cpp
index 27e29de54f076..3e4c39d941938 100644
--- a/clang-tools-extra/clang-tidy/llvm/HeaderGuardCheck.cpp
+++ b/clang-tools-extra/clang-tidy/llvm/HeaderGuardCheck.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "HeaderGuardCheck.h"
+#include "clang/Tooling/Tooling.h"
 
 namespace clang {
 namespace tidy {
@@ -53,6 +54,10 @@ std::string LLVMHeaderGuardCheck::getHeaderGuard(StringRef Filename,
   if (StringRef(Guard).startswith("clang"))
     Guard = "LLVM_" + Guard;
 
+  // The prevalent style in flang is FORTRAN_FOO_BAR_H
+  if (StringRef(Guard).startswith("flang"))
+    Guard = "FORTRAN" + Guard.substr(sizeof("flang") - 1);
+
   return StringRef(Guard).upper();
 }
 
diff --git a/clang-tools-extra/clang-tidy/llvm/LLVMTidyModule.cpp b/clang-tools-extra/clang-tidy/llvm/LLVMTidyModule.cpp
index 2aaf07639267d..bf871e21f501f 100644
--- a/clang-tools-extra/clang-tidy/llvm/LLVMTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/llvm/LLVMTidyModule.cpp
@@ -9,6 +9,7 @@
 #include "../ClangTidy.h"
 #include "../ClangTidyModule.h"
 #include "../ClangTidyModuleRegistry.h"
+#include "../readability/ElseAfterReturnCheck.h"
 #include "../readability/NamespaceCommentCheck.h"
 #include "../readability/QualifiedAutoCheck.h"
 #include "HeaderGuardCheck.h"
@@ -24,6 +25,8 @@ namespace llvm_check {
 class LLVMModule : public ClangTidyModule {
 public:
   void addCheckFactories(ClangTidyCheckFactories &CheckFactories) override {
+    CheckFactories.registerCheck<readability::ElseAfterReturnCheck>(
+        "llvm-else-after-return");
     CheckFactories.registerCheck<LLVMHeaderGuardCheck>("llvm-header-guard");
     CheckFactories.registerCheck<IncludeOrderCheck>("llvm-include-order");
     CheckFactories.registerCheck<readability::NamespaceCommentCheck>(
@@ -40,6 +43,9 @@ class LLVMModule : public ClangTidyModule {
   ClangTidyOptions getModuleOptions() override {
     ClangTidyOptions Options;
     Options.CheckOptions["llvm-qualified-auto.AddConstToQualified"] = "0";
+    Options.CheckOptions["llvm-else-after-return.WarnOnUnfixable"] = "0";
+    Options.CheckOptions["llvm-else-after-return.WarnOnConditionVariables"] =
+        "0";
     return Options;
   }
 };
diff --git a/clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.cpp b/clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.cpp
index 2ca976fd4bc95..13646ccbafa6d 100644
--- a/clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.cpp
@@ -10,6 +10,7 @@
 #include "PreferIsaOrDynCastInConditionalsCheck.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
 
 using namespace clang::ast_matchers;
 
@@ -41,14 +42,13 @@ void PreferIsaOrDynCastInConditionalsCheck::registerMatchers(
 
   auto CallExpression =
       callExpr(
-          allOf(unless(isMacroID()), unless(cxxMemberCallExpr()),
-                allOf(callee(namedDecl(anyOf(hasName("isa"), hasName("cast"),
-                                             hasName("cast_or_null"),
-                                             hasName("dyn_cast"),
-                                             hasName("dyn_cast_or_null")))
-                                 .bind("func")),
-                      hasArgument(0, anyOf(declRefExpr().bind("arg"),
-                                           cxxMemberCallExpr().bind("arg"))))))
+          allOf(
+              unless(isMacroID()), unless(cxxMemberCallExpr()),
+              allOf(callee(namedDecl(hasAnyName("isa", "cast", "cast_or_null",
+                                                "dyn_cast", "dyn_cast_or_null"))
+                               .bind("func")),
+                    hasArgument(0, anyOf(declRefExpr().bind("arg"),
+                                         cxxMemberCallExpr().bind("arg"))))))
           .bind("rhs");
 
   Finder->addMatcher(
diff --git a/clang-tools-extra/clang-tidy/llvm/TwineLocalCheck.cpp b/clang-tools-extra/clang-tidy/llvm/TwineLocalCheck.cpp
index 63a9314a49e5e..907bdcb264bf3 100644
--- a/clang-tools-extra/clang-tidy/llvm/TwineLocalCheck.cpp
+++ b/clang-tools-extra/clang-tidy/llvm/TwineLocalCheck.cpp
@@ -19,8 +19,10 @@ namespace llvm_check {
 
 void TwineLocalCheck::registerMatchers(MatchFinder *Finder) {
   auto TwineType =
-      qualType(hasDeclaration(recordDecl(hasName("::llvm::Twine"))));
-  Finder->addMatcher(varDecl(hasType(TwineType)).bind("variable"), this);
+      qualType(hasDeclaration(cxxRecordDecl(hasName("::llvm::Twine"))));
+  Finder->addMatcher(
+      varDecl(unless(parmVarDecl()), hasType(TwineType)).bind("variable"),
+      this);
 }
 
 void TwineLocalCheck::check(const MatchFinder::MatchResult &Result) {
diff --git a/clang-tools-extra/clang-tidy/llvmlibc/CMakeLists.txt b/clang-tools-extra/clang-tidy/llvmlibc/CMakeLists.txt
index 4a4e654f72c61..85b22d157a0ec 100644
--- a/clang-tools-extra/clang-tidy/llvmlibc/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/llvmlibc/CMakeLists.txt
@@ -13,6 +13,9 @@ add_clang_library(clangTidyLLVMLibcModule
   clangTidy
   clangTidyPortabilityModule
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyLLVMLibcModule
diff --git a/clang-tools-extra/clang-tidy/llvmlibc/RestrictSystemLibcHeadersCheck.cpp b/clang-tools-extra/clang-tidy/llvmlibc/RestrictSystemLibcHeadersCheck.cpp
index 4a19b5359d4f9..e64338756a25b 100644
--- a/clang-tools-extra/clang-tidy/llvmlibc/RestrictSystemLibcHeadersCheck.cpp
+++ b/clang-tools-extra/clang-tidy/llvmlibc/RestrictSystemLibcHeadersCheck.cpp
@@ -11,6 +11,10 @@
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/HeaderSearchOptions.h"
+#include "clang/Lex/Preprocessor.h"
+
+// FixItHint - Hint to check documentation script to mark this check as
+// providing a FixIt.
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/misc/CMakeLists.txt b/clang-tools-extra/clang-tidy/misc/CMakeLists.txt
index 236a3a8875bdb..880b7aacdc65a 100644
--- a/clang-tools-extra/clang-tidy/misc/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/misc/CMakeLists.txt
@@ -23,6 +23,9 @@ add_clang_library(clangTidyMiscModule
   LINK_LIBS
   clangTidy
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyMiscModule
diff --git a/clang-tools-extra/clang-tidy/misc/UnusedParametersCheck.cpp b/clang-tools-extra/clang-tidy/misc/UnusedParametersCheck.cpp
index 91d88747527e3..535ac78b3f78a 100644
--- a/clang-tools-extra/clang-tidy/misc/UnusedParametersCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/UnusedParametersCheck.cpp
@@ -13,6 +13,7 @@
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/Lex/Lexer.h"
 #include "llvm/ADT/STLExtras.h"
+#include <unordered_map>
 #include <unordered_set>
 
 using namespace clang::ast_matchers;
diff --git a/clang-tools-extra/clang-tidy/modernize/AvoidBindCheck.cpp b/clang-tools-extra/clang-tidy/modernize/AvoidBindCheck.cpp
index 37f9eb6955d3b..63f9f066f1943 100644
--- a/clang-tools-extra/clang-tidy/modernize/AvoidBindCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/AvoidBindCheck.cpp
@@ -18,6 +18,7 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSet.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/Regex.h"
@@ -35,7 +36,8 @@ namespace modernize {
 namespace {
 
 enum BindArgumentKind { BK_Temporary, BK_Placeholder, BK_CallExpr, BK_Other };
-enum CaptureMode { CM_None, CM_ByRef, CM_ByValue, CM_InitExpression };
+enum CaptureMode { CM_None, CM_ByRef, CM_ByValue };
+enum CaptureExpr { CE_None, CE_Var, CE_InitExpression };
 
 enum CallableType {
   CT_Other,          // unknown
@@ -60,6 +62,10 @@ struct BindArgument {
   // captured.
   CaptureMode CM = CM_None;
 
+  // Whether the argument is a simple variable (we can capture it directly),
+  // or an expression (we must introduce a capture variable).
+  CaptureExpr CE = CE_None;
+
   // The exact spelling of this argument in the source code.
   StringRef SourceTokens;
 
@@ -86,6 +92,7 @@ struct CallableInfo {
   CallableType Type = CT_Other;
   CallableMaterializationKind Materialization = CMK_Other;
   CaptureMode CM = CM_None;
+  CaptureExpr CE = CE_None;
   StringRef SourceTokens;
   std::string CaptureIdentifier;
   std::string UsageIdentifier;
@@ -102,6 +109,12 @@ struct LambdaProperties {
 
 } // end namespace
 
+static bool tryCaptureAsLocalVariable(const MatchFinder::MatchResult &Result,
+                                      BindArgument &B, const Expr *E);
+
+static bool tryCaptureAsMemberVariable(const MatchFinder::MatchResult &Result,
+                                       BindArgument &B, const Expr *E);
+
 static const Expr *ignoreTemporariesAndPointers(const Expr *E) {
   if (const auto *T = dyn_cast<UnaryOperator>(E))
     return ignoreTemporariesAndPointers(T->getSubExpr());
@@ -148,12 +161,22 @@ initializeBindArgumentForCallExpr(const MatchFinder::MatchResult &Result,
   // std::ref(x) means to capture x by reference.
   if (isCallExprNamed(CE, "boost::ref") || isCallExprNamed(CE, "std::ref")) {
     B.Kind = BK_Other;
+    if (tryCaptureAsLocalVariable(Result, B, CE->getArg(0)) ||
+        tryCaptureAsMemberVariable(Result, B, CE->getArg(0))) {
+      B.CE = CE_Var;
+    } else {
+      // The argument to std::ref is an expression that produces a reference.
+      // Create a capture reference to hold it.
+      B.CE = CE_InitExpression;
+      B.UsageIdentifier = "capture" + llvm::utostr(CaptureIndex++);
+    }
+    // Strip off the reference wrapper.
+    B.SourceTokens = getSourceTextForExpr(Result, CE->getArg(0));
     B.CM = CM_ByRef;
-    B.UsageIdentifier =
-        std::string(getSourceTextForExpr(Result, CE->getArg(0)));
   } else {
     B.Kind = BK_CallExpr;
-    B.CM = CM_InitExpression;
+    B.CM = CM_ByValue;
+    B.CE = CE_InitExpression;
     B.UsageIdentifier = "capture" + llvm::utostr(CaptureIndex++);
   }
   B.CaptureIdentifier = B.UsageIdentifier;
@@ -204,6 +227,7 @@ static bool tryCaptureAsMemberVariable(const MatchFinder::MatchResult &Result,
 
   E = E->IgnoreImplicit();
   if (isa<CXXThisExpr>(E)) {
+    // E is a direct use of "this".
     B.CM = CM_ByValue;
     B.UsageIdentifier = std::string(getSourceTextForExpr(Result, E));
     B.CaptureIdentifier = "this";
@@ -217,17 +241,22 @@ static bool tryCaptureAsMemberVariable(const MatchFinder::MatchResult &Result,
   if (!ME->isLValue() || !isa<FieldDecl>(ME->getMemberDecl()))
     return false;
 
-  B.CM = CM_ByValue;
-  B.UsageIdentifier = std::string(getSourceTextForExpr(Result, E));
-  B.CaptureIdentifier = "this";
-  return true;
+  if (isa<CXXThisExpr>(ME->getBase())) {
+    // E refers to a data member without an explicit "this".
+    B.CM = CM_ByValue;
+    B.UsageIdentifier = std::string(getSourceTextForExpr(Result, E));
+    B.CaptureIdentifier = "this";
+    return true;
+  }
+
+  return false;
 }
 
 static SmallVector<BindArgument, 4>
 buildBindArguments(const MatchFinder::MatchResult &Result,
                    const CallableInfo &Callable) {
   SmallVector<BindArgument, 4> BindArguments;
-  llvm::Regex MatchPlaceholder("^_([0-9]+)$");
+  static llvm::Regex MatchPlaceholder("^_([0-9]+)$");
 
   const auto *BindCall = Result.Nodes.getNodeAs<CallExpr>("bind");
 
@@ -251,7 +280,10 @@ buildBindArguments(const MatchFinder::MatchResult &Result,
       B.IsUsed = true;
 
     SmallVector<StringRef, 2> Matches;
-    if (MatchPlaceholder.match(B.SourceTokens, &Matches)) {
+    const auto *DRE = dyn_cast<DeclRefExpr>(E);
+    if (MatchPlaceholder.match(B.SourceTokens, &Matches) ||
+        // Check for match with qualifiers removed.
+        (DRE && MatchPlaceholder.match(DRE->getDecl()->getName(), &Matches))) {
       B.Kind = BK_Placeholder;
       B.PlaceHolderIndex = std::stoi(std::string(Matches[1]));
       B.UsageIdentifier = "PH" + llvm::utostr(B.PlaceHolderIndex);
@@ -273,11 +305,13 @@ buildBindArguments(const MatchFinder::MatchResult &Result,
     // safe.
     B.Kind = BK_Other;
     if (IsObjectPtr) {
-      B.CM = CM_InitExpression;
+      B.CE = CE_InitExpression;
+      B.CM = CM_ByValue;
       B.UsageIdentifier = "ObjectPtr";
       B.CaptureIdentifier = B.UsageIdentifier;
     } else if (anyDescendantIsLocal(B.E)) {
-      B.CM = CM_InitExpression;
+      B.CE = CE_InitExpression;
+      B.CM = CM_ByValue;
       B.CaptureIdentifier = "capture" + llvm::utostr(CaptureIndex++);
       B.UsageIdentifier = B.CaptureIdentifier;
     }
@@ -337,9 +371,12 @@ static void addFunctionCallArgs(ArrayRef<BindArgument> Args,
 
     Stream << Delimiter;
 
-    if (B.Kind == BK_Placeholder || B.CM != CM_None)
+    if (B.Kind == BK_Placeholder) {
+      Stream << "std::forward<decltype(" << B.UsageIdentifier << ")>";
+      Stream << "(" << B.UsageIdentifier << ")";
+    } else if (B.CM != CM_None)
       Stream << B.UsageIdentifier;
-    else if (B.CM == CM_None)
+    else
       Stream << B.SourceTokens;
 
     Delimiter = ", ";
@@ -357,9 +394,9 @@ static bool isPlaceHolderIndexRepeated(const ArrayRef<BindArgument> Args) {
   return false;
 }
 
-static std::vector<const CXXMethodDecl *>
+static std::vector<const FunctionDecl *>
 findCandidateCallOperators(const CXXRecordDecl *RecordDecl, size_t NumArgs) {
-  std::vector<const CXXMethodDecl *> Candidates;
+  std::vector<const FunctionDecl *> Candidates;
 
   for (const clang::CXXMethodDecl *Method : RecordDecl->methods()) {
     OverloadedOperatorKind OOK = Method->getOverloadedOperator();
@@ -373,6 +410,23 @@ findCandidateCallOperators(const CXXRecordDecl *RecordDecl, size_t NumArgs) {
     Candidates.push_back(Method);
   }
 
+  // Find templated operator(), if any.
+  for (const clang::Decl *D : RecordDecl->decls()) {
+    const auto *FTD = dyn_cast<FunctionTemplateDecl>(D);
+    if (!FTD)
+      continue;
+    const FunctionDecl *FD = FTD->getTemplatedDecl();
+
+    OverloadedOperatorKind OOK = FD->getOverloadedOperator();
+    if (OOK != OverloadedOperatorKind::OO_Call)
+      continue;
+
+    if (FD->getNumParams() > NumArgs)
+      continue;
+
+    Candidates.push_back(FD);
+  }
+
   return Candidates;
 }
 
@@ -407,7 +461,7 @@ static bool isFixitSupported(const CallableInfo &Callee,
 
 const FunctionDecl *getCallOperator(const CXXRecordDecl *Callable,
                                     size_t NumArgs) {
-  std::vector<const CXXMethodDecl *> Candidates =
+  std::vector<const FunctionDecl *> Candidates =
       findCandidateCallOperators(Callable, NumArgs);
   if (Candidates.size() != 1)
     return nullptr;
@@ -466,11 +520,15 @@ getCallableMaterialization(const MatchFinder::MatchResult &Result) {
 
   const auto *NoTemporaries = ignoreTemporariesAndPointers(CallableExpr);
 
-  if (isa<CallExpr>(NoTemporaries))
+  const auto *CE = dyn_cast<CXXConstructExpr>(NoTemporaries);
+  const auto *FC = dyn_cast<CXXFunctionalCastExpr>(NoTemporaries);
+  if ((isa<CallExpr>(NoTemporaries)) || (CE && (CE->getNumArgs() > 0)) ||
+      (FC && (FC->getCastKind() == CK_ConstructorConversion)))
+    // CE is something that looks like a call, with arguments - either
+    // a function call or a constructor invocation.
     return CMK_CallExpression;
 
-  if (isa<CXXFunctionalCastExpr>(NoTemporaries) ||
-      isa<CXXConstructExpr>(NoTemporaries))
+  if (isa<CXXFunctionalCastExpr>(NoTemporaries) || CE)
     return CMK_Function;
 
   if (const auto *DRE = dyn_cast<DeclRefExpr>(NoTemporaries)) {
@@ -503,13 +561,15 @@ getLambdaProperties(const MatchFinder::MatchResult &Result) {
       getCallMethodDecl(Result, LP.Callable.Type, LP.Callable.Materialization);
   LP.Callable.SourceTokens = getSourceTextForExpr(Result, CalleeExpr);
   if (LP.Callable.Materialization == CMK_VariableRef) {
+    LP.Callable.CE = CE_Var;
     LP.Callable.CM = CM_ByValue;
     LP.Callable.UsageIdentifier =
         std::string(getSourceTextForExpr(Result, CalleeExpr));
     LP.Callable.CaptureIdentifier = std::string(
         getSourceTextForExpr(Result, ignoreTemporariesAndPointers(CalleeExpr)));
   } else if (LP.Callable.Materialization == CMK_CallExpression) {
-    LP.Callable.CM = CM_InitExpression;
+    LP.Callable.CE = CE_InitExpression;
+    LP.Callable.CM = CM_ByValue;
     LP.Callable.UsageIdentifier = "Func";
     LP.Callable.CaptureIdentifier = "Func";
     LP.Callable.CaptureInitializer = getSourceTextForExpr(Result, CalleeExpr);
@@ -523,7 +583,7 @@ getLambdaProperties(const MatchFinder::MatchResult &Result) {
 }
 
 static bool emitCapture(llvm::StringSet<> &CaptureSet, StringRef Delimiter,
-                        CaptureMode CM, StringRef Identifier,
+                        CaptureMode CM, CaptureExpr CE, StringRef Identifier,
                         StringRef InitExpression, raw_ostream &Stream) {
   if (CM == CM_None)
     return false;
@@ -537,7 +597,7 @@ static bool emitCapture(llvm::StringSet<> &CaptureSet, StringRef Delimiter,
   if (CM == CM_ByRef)
     Stream << "&";
   Stream << Identifier;
-  if (CM == CM_InitExpression)
+  if (CE == CE_InitExpression)
     Stream << " = " << InitExpression;
 
   CaptureSet.insert(Identifier);
@@ -550,9 +610,9 @@ static void emitCaptureList(const LambdaProperties &LP,
   llvm::StringSet<> CaptureSet;
   bool AnyCapturesEmitted = false;
 
-  AnyCapturesEmitted = emitCapture(CaptureSet, "", LP.Callable.CM,
-                                   LP.Callable.CaptureIdentifier,
-                                   LP.Callable.CaptureInitializer, Stream);
+  AnyCapturesEmitted = emitCapture(
+      CaptureSet, "", LP.Callable.CM, LP.Callable.CE,
+      LP.Callable.CaptureIdentifier, LP.Callable.CaptureInitializer, Stream);
 
   for (const BindArgument &B : LP.BindArguments) {
     if (B.CM == CM_None || !B.IsUsed)
@@ -560,7 +620,7 @@ static void emitCaptureList(const LambdaProperties &LP,
 
     StringRef Delimiter = AnyCapturesEmitted ? ", " : "";
 
-    if (emitCapture(CaptureSet, Delimiter, B.CM, B.CaptureIdentifier,
+    if (emitCapture(CaptureSet, Delimiter, B.CM, B.CE, B.CaptureIdentifier,
                     B.SourceTokens, Stream))
       AnyCapturesEmitted = true;
   }
@@ -585,8 +645,7 @@ void AvoidBindCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
 void AvoidBindCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       callExpr(
-          callee(namedDecl(
-              anyOf(hasName("::boost::bind"), hasName("::std::bind")))),
+          callee(namedDecl(hasAnyName("::boost::bind", "::std::bind"))),
           hasArgument(
               0, anyOf(expr(hasType(memberPointerType())).bind("ref"),
                        expr(hasParent(materializeTemporaryExpr().bind("ref"))),
@@ -635,19 +694,18 @@ void AvoidBindCheck::check(const MatchFinder::MatchResult &Result) {
     Stream << MethodDecl->getName();
   } else {
     Stream << " { return ";
-    switch (LP.Callable.CM) {
-    case CM_ByValue:
-    case CM_ByRef:
+    switch (LP.Callable.CE) {
+    case CE_Var:
       if (LP.Callable.UsageIdentifier != LP.Callable.CaptureIdentifier) {
         Stream << "(" << LP.Callable.UsageIdentifier << ")";
         break;
       }
       LLVM_FALLTHROUGH;
-    case CM_InitExpression:
+    case CE_InitExpression:
       Stream << LP.Callable.UsageIdentifier;
       break;
     default:
-      Ref->printPretty(Stream, nullptr, Result.Context->getPrintingPolicy());
+      Stream << getSourceTextForExpr(Result, Ref);
     }
   }
 
diff --git a/clang-tools-extra/clang-tidy/modernize/CMakeLists.txt b/clang-tools-extra/clang-tidy/modernize/CMakeLists.txt
index ee170279c8347..147c2741d59e4 100644
--- a/clang-tools-extra/clang-tidy/modernize/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/modernize/CMakeLists.txt
@@ -43,6 +43,9 @@ add_clang_library(clangTidyModernizeModule
   clangTidy
   clangTidyReadabilityModule
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyModernizeModule
diff --git a/clang-tools-extra/clang-tidy/modernize/ConcatNestedNamespacesCheck.cpp b/clang-tools-extra/clang-tidy/modernize/ConcatNestedNamespacesCheck.cpp
index 48afcdc5c14be..83c3fac949baa 100644
--- a/clang-tools-extra/clang-tidy/modernize/ConcatNestedNamespacesCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/ConcatNestedNamespacesCheck.cpp
@@ -9,8 +9,8 @@
 #include "ConcatNestedNamespacesCheck.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
 #include <algorithm>
-#include <iterator>
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp b/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp
index dd18d866e5084..b90af1521baf5 100644
--- a/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp
@@ -28,6 +28,31 @@ using namespace llvm;
 
 namespace clang {
 namespace tidy {
+
+template <> struct OptionEnumMapping<modernize::Confidence::Level> {
+  static llvm::ArrayRef<std::pair<modernize::Confidence::Level, StringRef>>
+  getEnumMapping() {
+    static constexpr std::pair<modernize::Confidence::Level, StringRef>
+        Mapping[] = {{modernize::Confidence::CL_Reasonable, "reasonable"},
+                     {modernize::Confidence::CL_Safe, "safe"},
+                     {modernize::Confidence::CL_Risky, "risky"}};
+    return makeArrayRef(Mapping);
+  }
+};
+
+template <> struct OptionEnumMapping<modernize::VariableNamer::NamingStyle> {
+  static llvm::ArrayRef<
+      std::pair<modernize::VariableNamer::NamingStyle, StringRef>>
+  getEnumMapping() {
+    static constexpr std::pair<modernize::VariableNamer::NamingStyle, StringRef>
+        Mapping[] = {{modernize::VariableNamer::NS_CamelCase, "CamelCase"},
+                     {modernize::VariableNamer::NS_CamelBack, "camelBack"},
+                     {modernize::VariableNamer::NS_LowerCase, "lower_case"},
+                     {modernize::VariableNamer::NS_UpperCase, "UPPER_CASE"}};
+    return makeArrayRef(Mapping);
+  }
+};
+
 namespace modernize {
 
 static const char LoopNameArray[] = "forLoopArray";
@@ -44,25 +69,6 @@ static const char EndVarName[] = "endVar";
 static const char DerefByValueResultName[] = "derefByValueResult";
 static const char DerefByRefResultName[] = "derefByRefResult";
 
-static ArrayRef<std::pair<StringRef, Confidence::Level>>
-getConfidenceMapping() {
-  static constexpr std::pair<StringRef, Confidence::Level> Mapping[] = {
-      {"reasonable", Confidence::CL_Reasonable},
-      {"safe", Confidence::CL_Safe},
-      {"risky", Confidence::CL_Risky}};
-  return makeArrayRef(Mapping);
-}
-
-static ArrayRef<std::pair<StringRef, VariableNamer::NamingStyle>>
-getStyleMapping() {
-  static constexpr std::pair<StringRef, VariableNamer::NamingStyle> Mapping[] =
-      {{"CamelCase", VariableNamer::NS_CamelCase},
-       {"camelBack", VariableNamer::NS_CamelBack},
-       {"lower_case", VariableNamer::NS_LowerCase},
-       {"UPPER_CASE", VariableNamer::NS_UpperCase}};
-  return makeArrayRef(Mapping);
-}
-
 // shared matchers
 static const TypeMatcher AnyType() { return anything(); }
 
@@ -146,9 +152,8 @@ StatementMatcher makeArrayLoopMatcher() {
 ///   - If the end iterator variable 'g' is defined, it is the same as 'f'.
 StatementMatcher makeIteratorLoopMatcher() {
   StatementMatcher BeginCallMatcher =
-      cxxMemberCallExpr(
-          argumentCountIs(0),
-          callee(cxxMethodDecl(anyOf(hasName("begin"), hasName("cbegin")))))
+      cxxMemberCallExpr(argumentCountIs(0),
+                        callee(cxxMethodDecl(hasAnyName("begin", "cbegin"))))
           .bind(BeginCallName);
 
   DeclarationMatcher InitDeclMatcher =
@@ -162,8 +167,7 @@ StatementMatcher makeIteratorLoopMatcher() {
       varDecl(hasInitializer(anything())).bind(EndVarName);
 
   StatementMatcher EndCallMatcher = cxxMemberCallExpr(
-      argumentCountIs(0),
-      callee(cxxMethodDecl(anyOf(hasName("end"), hasName("cend")))));
+      argumentCountIs(0), callee(cxxMethodDecl(hasAnyName("end", "cend"))));
 
   StatementMatcher IteratorBoundMatcher =
       expr(anyOf(ignoringParenImpCasts(
@@ -280,8 +284,7 @@ StatementMatcher makePseudoArrayLoopMatcher() {
       ));
 
   StatementMatcher SizeCallMatcher = cxxMemberCallExpr(
-      argumentCountIs(0),
-      callee(cxxMethodDecl(anyOf(hasName("size"), hasName("length")))),
+      argumentCountIs(0), callee(cxxMethodDecl(hasAnyName("size", "length"))),
       on(anyOf(hasType(pointsTo(RecordWithBeginEnd)),
                hasType(RecordWithBeginEnd))));
 
@@ -477,15 +480,13 @@ LoopConvertCheck::RangeDescriptor::RangeDescriptor()
 LoopConvertCheck::LoopConvertCheck(StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context), TUInfo(new TUTrackingInfo),
       MaxCopySize(Options.get("MaxCopySize", 16ULL)),
-      MinConfidence(Options.get("MinConfidence", getConfidenceMapping(),
-                                Confidence::CL_Reasonable)),
-      NamingStyle(Options.get("NamingStyle", getStyleMapping(),
-                              VariableNamer::NS_CamelCase)) {}
+      MinConfidence(Options.get("MinConfidence", Confidence::CL_Reasonable)),
+      NamingStyle(Options.get("NamingStyle", VariableNamer::NS_CamelCase)) {}
 
 void LoopConvertCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
   Options.store(Opts, "MaxCopySize", std::to_string(MaxCopySize));
-  Options.store(Opts, "MinConfidence", MinConfidence, getConfidenceMapping());
-  Options.store(Opts, "NamingStyle", NamingStyle, getStyleMapping());
+  Options.store(Opts, "MinConfidence", MinConfidence);
+  Options.store(Opts, "NamingStyle", NamingStyle);
 }
 
 void LoopConvertCheck::registerMatchers(MatchFinder *Finder) {
diff --git a/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp b/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp
index 67f281b3ed1f7..56d4cceb6002a 100644
--- a/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp
@@ -500,7 +500,7 @@ void ForLoopIndexUseVisitor::addUsage(const Usage &U) {
 ///     int k = *i + 2;
 ///   }
 /// \endcode
-bool ForLoopIndexUseVisitor::TraverseUnaryDeref(UnaryOperator *Uop) {
+bool ForLoopIndexUseVisitor::TraverseUnaryOperator(UnaryOperator *Uop) {
   // If we dereference an iterator that's actually a pointer, count the
   // occurrence.
   if (isDereferenceOfUop(Uop, IndexVar)) {
diff --git a/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.h b/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.h
index b2cba8c0172ae..4f75c9c7522f4 100644
--- a/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.h
+++ b/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.h
@@ -346,7 +346,7 @@ class ForLoopIndexUseVisitor
   bool TraverseLambdaCapture(LambdaExpr *LE, const LambdaCapture *C,
                              Expr *Init);
   bool TraverseMemberExpr(MemberExpr *Member);
-  bool TraverseUnaryDeref(UnaryOperator *Uop);
+  bool TraverseUnaryOperator(UnaryOperator *Uop);
   bool VisitDeclRefExpr(DeclRefExpr *E);
   bool VisitDeclStmt(DeclStmt *S);
   bool TraverseStmt(Stmt *S);
diff --git a/clang-tools-extra/clang-tidy/modernize/MakeSharedCheck.cpp b/clang-tools-extra/clang-tidy/modernize/MakeSharedCheck.cpp
index 11d24432edafc..5ab3f3e26be2c 100644
--- a/clang-tools-extra/clang-tidy/modernize/MakeSharedCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/MakeSharedCheck.cpp
@@ -8,6 +8,9 @@
 
 #include "MakeSharedCheck.h"
 
+// FixItHint - Hint to check documentation script to mark this check as
+// providing a FixIt.
+
 using namespace clang::ast_matchers;
 
 namespace clang {
diff --git a/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp b/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp
index e34fd7038bb86..c677043946f7f 100644
--- a/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp
@@ -45,7 +45,6 @@ MakeSmartPtrCheck::MakeSmartPtrCheck(StringRef Name, ClangTidyContext *Context,
                                      StringRef MakeSmartPtrFunctionName)
     : ClangTidyCheck(Name, Context),
       IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::getMapping(),
                                             utils::IncludeSorter::IS_LLVM)),
       MakeSmartPtrFunctionHeader(
           Options.get("MakeSmartPtrFunctionHeader", StdMemoryHeader)),
@@ -54,8 +53,7 @@ MakeSmartPtrCheck::MakeSmartPtrCheck(StringRef Name, ClangTidyContext *Context,
       IgnoreMacros(Options.getLocalOrGlobal("IgnoreMacros", true)) {}
 
 void MakeSmartPtrCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle,
-                utils::IncludeSorter::getMapping());
+  Options.store(Opts, "IncludeStyle", IncludeStyle);
   Options.store(Opts, "MakeSmartPtrFunctionHeader", MakeSmartPtrFunctionHeader);
   Options.store(Opts, "MakeSmartPtrFunction", MakeSmartPtrFunctionName);
   Options.store(Opts, "IgnoreMacros", IgnoreMacros);
diff --git a/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp b/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp
index ed1a1a26bb62b..b6dedfbc2b6eb 100644
--- a/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp
@@ -121,13 +121,11 @@ collectParamDecls(const CXXConstructorDecl *Ctor,
 PassByValueCheck::PassByValueCheck(StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
       IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::getMapping(),
                                             utils::IncludeSorter::IS_LLVM)),
       ValuesOnly(Options.get("ValuesOnly", false)) {}
 
 void PassByValueCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle,
-                utils::IncludeSorter::getMapping());
+  Options.store(Opts, "IncludeStyle", IncludeStyle);
   Options.store(Opts, "ValuesOnly", ValuesOnly);
 }
 
diff --git a/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp b/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp
index 295be200bca6f..f98254dbf7c83 100644
--- a/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp
@@ -75,12 +75,10 @@ ReplaceAutoPtrCheck::ReplaceAutoPtrCheck(StringRef Name,
                                          ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
       IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::getMapping(),
                                             utils::IncludeSorter::IS_LLVM)) {}
 
 void ReplaceAutoPtrCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle,
-                utils::IncludeSorter::getMapping());
+  Options.store(Opts, "IncludeStyle", IncludeStyle);
 }
 
 void ReplaceAutoPtrCheck::registerMatchers(MatchFinder *Finder) {
diff --git a/clang-tools-extra/clang-tidy/modernize/ReplaceDisallowCopyAndAssignMacroCheck.cpp b/clang-tools-extra/clang-tidy/modernize/ReplaceDisallowCopyAndAssignMacroCheck.cpp
index 2219a3c477b36..9752fd0a9e955 100644
--- a/clang-tools-extra/clang-tidy/modernize/ReplaceDisallowCopyAndAssignMacroCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/ReplaceDisallowCopyAndAssignMacroCheck.cpp
@@ -9,6 +9,8 @@
 #include "ReplaceDisallowCopyAndAssignMacroCheck.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Lex/MacroArgs.h"
+#include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
 #include "llvm/Support/FormatVariadic.h"
 
 namespace clang {
diff --git a/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp b/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp
index 9cfbd87239dce..66917df3e91d2 100644
--- a/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp
@@ -24,7 +24,6 @@ ReplaceRandomShuffleCheck::ReplaceRandomShuffleCheck(StringRef Name,
                                                      ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
       IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::getMapping(),
                                             utils::IncludeSorter::IS_LLVM)) {}
 
 void ReplaceRandomShuffleCheck::registerMatchers(MatchFinder *Finder) {
@@ -52,8 +51,7 @@ void ReplaceRandomShuffleCheck::registerPPCallbacks(
 
 void ReplaceRandomShuffleCheck::storeOptions(
     ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle,
-                utils::IncludeSorter::getMapping());
+  Options.store(Opts, "IncludeStyle", IncludeStyle);
 }
 
 void ReplaceRandomShuffleCheck::check(const MatchFinder::MatchResult &Result) {
diff --git a/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.cpp
index e3953c5d84041..8e31b4b5f75fc 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.cpp
@@ -11,6 +11,7 @@
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Preprocessor.h"
 #include "clang/Tooling/FixIt.h"
 #include "llvm/ADT/StringExtras.h"
 
diff --git a/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.h b/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.h
index 4da9b7a9aa92f..0cd4e7741c3e5 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.h
+++ b/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.h
@@ -10,7 +10,8 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MODERNIZE_USETRAILINGRETURNTYPECHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MODERNIZE_USETRAILINGRETURNTYPECHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
+#include "clang/Lex/Token.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/modernize/UseUncaughtExceptionsCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseUncaughtExceptionsCheck.cpp
index 82308f75c1330..b9d68cea8ced0 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseUncaughtExceptionsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseUncaughtExceptionsCheck.cpp
@@ -9,6 +9,7 @@
 #include "UseUncaughtExceptionsCheck.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
 
 using namespace clang::ast_matchers;
 
diff --git a/clang-tools-extra/clang-tidy/mpi/CMakeLists.txt b/clang-tools-extra/clang-tidy/mpi/CMakeLists.txt
index 1ac973648eff4..113740cc56b12 100644
--- a/clang-tools-extra/clang-tidy/mpi/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/mpi/CMakeLists.txt
@@ -11,6 +11,9 @@ add_clang_library(clangTidyMPIModule
   LINK_LIBS
   clangTidy
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyMPIModule
diff --git a/clang-tools-extra/clang-tidy/objc/CMakeLists.txt b/clang-tools-extra/clang-tidy/objc/CMakeLists.txt
index 297471b54ea11..9a553f5a785b3 100644
--- a/clang-tools-extra/clang-tidy/objc/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/objc/CMakeLists.txt
@@ -16,6 +16,9 @@ add_clang_library(clangTidyObjCModule
   LINK_LIBS
   clangTidy
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyObjCModule
diff --git a/clang-tools-extra/clang-tidy/objc/NSInvocationArgumentLifetimeCheck.cpp b/clang-tools-extra/clang-tidy/objc/NSInvocationArgumentLifetimeCheck.cpp
index ecbf0953ec6d5..2b9eee2f0f33b 100644
--- a/clang-tools-extra/clang-tidy/objc/NSInvocationArgumentLifetimeCheck.cpp
+++ b/clang-tools-extra/clang-tidy/objc/NSInvocationArgumentLifetimeCheck.cpp
@@ -8,7 +8,6 @@
 
 #include "NSInvocationArgumentLifetimeCheck.h"
 #include "clang/AST/ASTContext.h"
-#include "clang/AST/Attrs.inc"
 #include "clang/AST/ComputeDependence.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/Expr.h"
@@ -23,10 +22,10 @@
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
+#include "clang/Lex/Lexer.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/raw_ostream.h"
 
 using namespace clang::ast_matchers;
 
diff --git a/clang-tools-extra/clang-tidy/objc/SuperSelfCheck.h b/clang-tools-extra/clang-tidy/objc/SuperSelfCheck.h
index 03f0fc1bb1f9b..4dcc0c1d7064e 100644
--- a/clang-tools-extra/clang-tidy/objc/SuperSelfCheck.h
+++ b/clang-tools-extra/clang-tidy/objc/SuperSelfCheck.h
@@ -10,7 +10,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OBJC_SUPERSELFCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OBJC_SUPERSELFCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/openmp/CMakeLists.txt b/clang-tools-extra/clang-tidy/openmp/CMakeLists.txt
index a23c719456aac..4ef61f88c0509 100644
--- a/clang-tools-extra/clang-tidy/openmp/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/openmp/CMakeLists.txt
@@ -11,6 +11,9 @@ add_clang_library(clangTidyOpenMPModule
   LINK_LIBS
   clangTidy
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyOpenMPModule
diff --git a/clang-tools-extra/clang-tidy/openmp/ExceptionEscapeCheck.h b/clang-tools-extra/clang-tidy/openmp/ExceptionEscapeCheck.h
index d7a80feaec099..1fb78a95c77f5 100644
--- a/clang-tools-extra/clang-tidy/openmp/ExceptionEscapeCheck.h
+++ b/clang-tools-extra/clang-tidy/openmp/ExceptionEscapeCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OPENMP_EXCEPTIONESCAPECHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OPENMP_EXCEPTIONESCAPECHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 #include "../utils/ExceptionAnalyzer.h"
 
 namespace clang {
diff --git a/clang-tools-extra/clang-tidy/openmp/UseDefaultNoneCheck.h b/clang-tools-extra/clang-tidy/openmp/UseDefaultNoneCheck.h
index 09744c01a7cc4..8901fa7c4a78b 100644
--- a/clang-tools-extra/clang-tidy/openmp/UseDefaultNoneCheck.h
+++ b/clang-tools-extra/clang-tidy/openmp/UseDefaultNoneCheck.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OPENMP_USEDEFAULTNONECHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OPENMP_USEDEFAULTNONECHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/performance/CMakeLists.txt b/clang-tools-extra/clang-tidy/performance/CMakeLists.txt
index 5aa8c24c21cad..0e38e9fc5c0c9 100644
--- a/clang-tools-extra/clang-tidy/performance/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/performance/CMakeLists.txt
@@ -23,6 +23,9 @@ add_clang_library(clangTidyPerformanceModule
   LINK_LIBS
   clangTidy
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyPerformanceModule
diff --git a/clang-tools-extra/clang-tidy/performance/FasterStringFindCheck.cpp b/clang-tools-extra/clang-tidy/performance/FasterStringFindCheck.cpp
index bb0a02f06fc63..0d2efb31b7744 100644
--- a/clang-tools-extra/clang-tidy/performance/FasterStringFindCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/FasterStringFindCheck.cpp
@@ -51,7 +51,8 @@ FasterStringFindCheck::FasterStringFindCheck(StringRef Name,
                                              ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
       StringLikeClasses(utils::options::parseStringList(
-          Options.get("StringLikeClasses", "std::basic_string"))) {}
+          Options.get("StringLikeClasses",
+                      "::std::basic_string;::std::basic_string_view"))) {}
 
 void FasterStringFindCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
   Options.store(Opts, "StringLikeClasses",
@@ -59,9 +60,6 @@ void FasterStringFindCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
 }
 
 void FasterStringFindCheck::registerMatchers(MatchFinder *Finder) {
-  if (!getLangOpts().CPlusPlus)
-    return;
-
   const auto SingleChar =
       expr(ignoringParenCasts(stringLiteral(hasSize(1)).bind("literal")));
   const auto StringFindFunctions =
diff --git a/clang-tools-extra/clang-tidy/performance/FasterStringFindCheck.h b/clang-tools-extra/clang-tidy/performance/FasterStringFindCheck.h
index 7d46d06c375cd..71a45705f1e81 100644
--- a/clang-tools-extra/clang-tidy/performance/FasterStringFindCheck.h
+++ b/clang-tools-extra/clang-tidy/performance/FasterStringFindCheck.h
@@ -27,6 +27,9 @@ namespace performance {
 class FasterStringFindCheck : public ClangTidyCheck {
 public:
   FasterStringFindCheck(StringRef Name, ClangTidyContext *Context);
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override{
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
diff --git a/clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.h b/clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.h
index 3745c7bb46811..71f662ba55445 100644
--- a/clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.h
+++ b/clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.h
@@ -22,6 +22,9 @@ namespace performance {
 class ForRangeCopyCheck : public ClangTidyCheck {
 public:
   ForRangeCopyCheck(StringRef Name, ClangTidyContext *Context);
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override{
+    return LangOpts.CPlusPlus11;
+  }
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
diff --git a/clang-tools-extra/clang-tidy/performance/ImplicitConversionInLoopCheck.h b/clang-tools-extra/clang-tidy/performance/ImplicitConversionInLoopCheck.h
index b372003a80024..4b271366f0f66 100644
--- a/clang-tools-extra/clang-tidy/performance/ImplicitConversionInLoopCheck.h
+++ b/clang-tools-extra/clang-tidy/performance/ImplicitConversionInLoopCheck.h
@@ -22,6 +22,9 @@ class ImplicitConversionInLoopCheck : public ClangTidyCheck {
 public:
   ImplicitConversionInLoopCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+      bool isLanguageVersionSupported(const LangOptions &LangOpts) const override{
+        return LangOpts.CPlusPlus11;
+      }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
diff --git a/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.h b/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.h
index e224dcc90685f..533b30dc32b0e 100644
--- a/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.h
+++ b/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.h
@@ -26,6 +26,9 @@ namespace performance {
 class InefficientVectorOperationCheck : public ClangTidyCheck {
 public:
   InefficientVectorOperationCheck(StringRef Name, ClangTidyContext *Context);
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override{
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
diff --git a/clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.cpp b/clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.cpp
index d09673fa7f23f..4cbb014867c4d 100644
--- a/clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.cpp
@@ -24,7 +24,6 @@ MoveConstructorInitCheck::MoveConstructorInitCheck(StringRef Name,
                                                    ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
       IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::getMapping(),
                                             utils::IncludeSorter::IS_LLVM)) {}
 
 void MoveConstructorInitCheck::registerMatchers(MatchFinder *Finder) {
@@ -97,8 +96,7 @@ void MoveConstructorInitCheck::registerPPCallbacks(
 }
 
 void MoveConstructorInitCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle,
-                utils::IncludeSorter::getMapping());
+  Options.store(Opts, "IncludeStyle", IncludeStyle);
 }
 
 } // namespace performance
diff --git a/clang-tools-extra/clang-tidy/performance/NoexceptMoveConstructorCheck.cpp b/clang-tools-extra/clang-tidy/performance/NoexceptMoveConstructorCheck.cpp
index a54b295b2e0c7..d23f0ab0cf44d 100644
--- a/clang-tools-extra/clang-tidy/performance/NoexceptMoveConstructorCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/NoexceptMoveConstructorCheck.cpp
@@ -9,6 +9,7 @@
 #include "NoexceptMoveConstructorCheck.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
 #include "clang/Tooling/FixIt.h"
 
 using namespace clang::ast_matchers;
diff --git a/clang-tools-extra/clang-tidy/performance/TypePromotionInMathFnCheck.cpp b/clang-tools-extra/clang-tidy/performance/TypePromotionInMathFnCheck.cpp
index d08cec1a2c3ca..597445d0fc266 100644
--- a/clang-tools-extra/clang-tidy/performance/TypePromotionInMathFnCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/TypePromotionInMathFnCheck.cpp
@@ -32,7 +32,6 @@ TypePromotionInMathFnCheck::TypePromotionInMathFnCheck(
     StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
       IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::getMapping(),
                                             utils::IncludeSorter::IS_LLVM)) {}
 
 void TypePromotionInMathFnCheck::registerPPCallbacks(
@@ -44,8 +43,7 @@ void TypePromotionInMathFnCheck::registerPPCallbacks(
 
 void TypePromotionInMathFnCheck::storeOptions(
     ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle,
-                utils::IncludeSorter::getMapping());
+  Options.store(Opts, "IncludeStyle", IncludeStyle);
 }
 
 void TypePromotionInMathFnCheck::registerMatchers(MatchFinder *Finder) {
diff --git a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.h b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.h
index 0a530e3732962..eeb9138d8532d 100644
--- a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.h
+++ b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.h
@@ -26,6 +26,9 @@ namespace performance {
 class UnnecessaryCopyInitialization : public ClangTidyCheck {
 public:
   UnnecessaryCopyInitialization(StringRef Name, ClangTidyContext *Context);
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override{
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
diff --git a/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp b/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp
index 5b5f2ff994783..5de53b1840f12 100644
--- a/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp
@@ -69,7 +69,6 @@ UnnecessaryValueParamCheck::UnnecessaryValueParamCheck(
     StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
       IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::getMapping(),
                                             utils::IncludeSorter::IS_LLVM)),
       AllowedTypes(
           utils::options::parseStringList(Options.get("AllowedTypes", ""))) {}
@@ -181,8 +180,7 @@ void UnnecessaryValueParamCheck::registerPPCallbacks(
 
 void UnnecessaryValueParamCheck::storeOptions(
     ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle,
-                utils::IncludeSorter::getMapping());
+  Options.store(Opts, "IncludeStyle", IncludeStyle);
   Options.store(Opts, "AllowedTypes",
                 utils::options::serializeStringList(AllowedTypes));
 }
diff --git a/clang-tools-extra/clang-tidy/plugin/CMakeLists.txt b/clang-tools-extra/clang-tidy/plugin/CMakeLists.txt
index 0bfe122314eb4..0b281e0799cf3 100644
--- a/clang-tools-extra/clang-tidy/plugin/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/plugin/CMakeLists.txt
@@ -4,6 +4,9 @@ add_clang_library(clangTidyPlugin
   LINK_LIBS
   clangTidy
   ${ALL_CLANG_TIDY_CHECKS}
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyPlugin
diff --git a/clang-tools-extra/clang-tidy/portability/CMakeLists.txt b/clang-tools-extra/clang-tidy/portability/CMakeLists.txt
index b4fcf28a99034..a0de5871e036a 100644
--- a/clang-tools-extra/clang-tidy/portability/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/portability/CMakeLists.txt
@@ -11,6 +11,9 @@ add_clang_library(clangTidyPortabilityModule
   LINK_LIBS
   clangTidy
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyPortabilityModule
diff --git a/clang-tools-extra/clang-tidy/portability/RestrictSystemIncludesCheck.h b/clang-tools-extra/clang-tidy/portability/RestrictSystemIncludesCheck.h
index c34f054fba2e0..80eddf7bcee79 100644
--- a/clang-tools-extra/clang-tidy/portability/RestrictSystemIncludesCheck.h
+++ b/clang-tools-extra/clang-tidy/portability/RestrictSystemIncludesCheck.h
@@ -11,6 +11,7 @@
 
 #include "../ClangTidyCheck.h"
 #include "../GlobList.h"
+#include "clang/Lex/PPCallbacks.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/readability/CMakeLists.txt b/clang-tools-extra/clang-tidy/readability/CMakeLists.txt
index ca2f69b8deb87..4539ab177ced1 100644
--- a/clang-tools-extra/clang-tidy/readability/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/readability/CMakeLists.txt
@@ -47,6 +47,9 @@ add_clang_library(clangTidyReadabilityModule
   LINK_LIBS
   clangTidy
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyReadabilityModule
diff --git a/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp b/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp
index ba054fb3aaf34..b2ad1bcf39463 100644
--- a/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp
+++ b/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp
@@ -13,6 +13,7 @@
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/Basic/SourceLocation.h"
+#include "clang/Lex/Lexer.h"
 
 using namespace clang::ast_matchers;
 
diff --git a/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.h b/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.h
index d9947650ab539..333fe94479dda 100644
--- a/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.h
+++ b/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.h
@@ -10,7 +10,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_CONVERTMEMFUNCTOSTATIC_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_CONVERTMEMFUNCTOSTATIC_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.cpp b/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.cpp
index a13c172c7f769..9ad6fb737ec9c 100644
--- a/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.cpp
@@ -9,6 +9,7 @@
 #include "ElseAfterReturnCheck.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
 #include "clang/Tooling/FixIt.h"
 #include "llvm/ADT/SmallVector.h"
 
@@ -18,52 +19,48 @@ namespace clang {
 namespace tidy {
 namespace readability {
 
-namespace {
 static const char ReturnStr[] = "return";
 static const char ContinueStr[] = "continue";
 static const char BreakStr[] = "break";
 static const char ThrowStr[] = "throw";
 static const char WarningMessage[] = "do not use 'else' after '%0'";
 static const char WarnOnUnfixableStr[] = "WarnOnUnfixable";
+static const char WarnOnConditionVariablesStr[] = "WarnOnConditionVariables";
 
-const DeclRefExpr *findUsage(const Stmt *Node, int64_t DeclIdentifier) {
+static const DeclRefExpr *findUsage(const Stmt *Node, int64_t DeclIdentifier) {
   if (!Node)
     return nullptr;
   if (const auto *DeclRef = dyn_cast<DeclRefExpr>(Node)) {
-    if (DeclRef->getDecl()->getID() == DeclIdentifier) {
+    if (DeclRef->getDecl()->getID() == DeclIdentifier)
       return DeclRef;
-    }
   } else {
     for (const Stmt *ChildNode : Node->children()) {
-      if (const DeclRefExpr *Result = findUsage(ChildNode, DeclIdentifier)) {
+      if (const DeclRefExpr *Result = findUsage(ChildNode, DeclIdentifier))
         return Result;
-      }
     }
   }
   return nullptr;
 }
 
-const DeclRefExpr *
+static const DeclRefExpr *
 findUsageRange(const Stmt *Node,
-               const llvm::iterator_range<int64_t *> &DeclIdentifiers) {
+               const llvm::ArrayRef<int64_t> &DeclIdentifiers) {
   if (!Node)
     return nullptr;
   if (const auto *DeclRef = dyn_cast<DeclRefExpr>(Node)) {
-    if (llvm::is_contained(DeclIdentifiers, DeclRef->getDecl()->getID())) {
+    if (llvm::is_contained(DeclIdentifiers, DeclRef->getDecl()->getID()))
       return DeclRef;
-    }
   } else {
     for (const Stmt *ChildNode : Node->children()) {
       if (const DeclRefExpr *Result =
-              findUsageRange(ChildNode, DeclIdentifiers)) {
+              findUsageRange(ChildNode, DeclIdentifiers))
         return Result;
-      }
     }
   }
   return nullptr;
 }
 
-const DeclRefExpr *checkInitDeclUsageInElse(const IfStmt *If) {
+static const DeclRefExpr *checkInitDeclUsageInElse(const IfStmt *If) {
   const auto *InitDeclStmt = dyn_cast_or_null<DeclStmt>(If->getInit());
   if (!InitDeclStmt)
     return nullptr;
@@ -80,25 +77,23 @@ const DeclRefExpr *checkInitDeclUsageInElse(const IfStmt *If) {
   return findUsageRange(If->getElse(), DeclIdentifiers);
 }
 
-const DeclRefExpr *checkConditionVarUsageInElse(const IfStmt *If) {
-  const VarDecl *CondVar = If->getConditionVariable();
-  return CondVar != nullptr ? findUsage(If->getElse(), CondVar->getID())
-                            : nullptr;
+static const DeclRefExpr *checkConditionVarUsageInElse(const IfStmt *If) {
+  if (const VarDecl *CondVar = If->getConditionVariable())
+    return findUsage(If->getElse(), CondVar->getID());
+  return nullptr;
 }
 
-bool containsDeclInScope(const Stmt *Node) {
-  if (isa<DeclStmt>(Node)) {
+static bool containsDeclInScope(const Stmt *Node) {
+  if (isa<DeclStmt>(Node))
     return true;
-  }
-  if (const auto *Compound = dyn_cast<CompoundStmt>(Node)) {
+  if (const auto *Compound = dyn_cast<CompoundStmt>(Node))
     return llvm::any_of(Compound->body(), [](const Stmt *SubNode) {
       return isa<DeclStmt>(SubNode);
     });
-  }
   return false;
 }
 
-void removeElseAndBrackets(DiagnosticBuilder &Diag, ASTContext &Context,
+static void removeElseAndBrackets(DiagnosticBuilder &Diag, ASTContext &Context,
                            const Stmt *Else, SourceLocation ElseLoc) {
   auto Remap = [&](SourceLocation Loc) {
     return Context.getSourceManager().getExpansionLoc(Loc);
@@ -132,15 +127,17 @@ void removeElseAndBrackets(DiagnosticBuilder &Diag, ASTContext &Context,
         SourceRange(ElseExpandedLoc, EndLoc), Repl);
   }
 }
-} // namespace
 
 ElseAfterReturnCheck::ElseAfterReturnCheck(StringRef Name,
                                            ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
-      WarnOnUnfixable(Options.get(WarnOnUnfixableStr, true)) {}
+      WarnOnUnfixable(Options.get(WarnOnUnfixableStr, true)),
+      WarnOnConditionVariables(Options.get(WarnOnConditionVariablesStr, true)) {
+}
 
 void ElseAfterReturnCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
   Options.store(Opts, WarnOnUnfixableStr, WarnOnUnfixable);
+  Options.store(Opts, WarnOnConditionVariablesStr, WarnOnConditionVariables);
 }
 
 void ElseAfterReturnCheck::registerMatchers(MatchFinder *Finder) {
@@ -185,6 +182,8 @@ void ElseAfterReturnCheck::check(const MatchFinder::MatchResult &Result) {
   }
 
   if (checkConditionVarUsageInElse(If) != nullptr) {
+    if (!WarnOnConditionVariables)
+      return;
     if (IsLastInScope) {
       // If the if statement is the last statement its enclosing statements
       // scope, we can pull the decl out of the if statement.
@@ -218,6 +217,8 @@ void ElseAfterReturnCheck::check(const MatchFinder::MatchResult &Result) {
   }
 
   if (checkInitDeclUsageInElse(If) != nullptr) {
+    if (!WarnOnConditionVariables)
+      return;
     if (IsLastInScope) {
       // If the if statement is the last statement its enclosing statements
       // scope, we can pull the decl out of the if statement.
diff --git a/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.h b/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.h
index 990d4b4c3f6b7..07d6314640a8b 100644
--- a/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.h
@@ -28,6 +28,7 @@ class ElseAfterReturnCheck : public ClangTidyCheck {
 
 private:
   const bool WarnOnUnfixable;
+  const bool WarnOnConditionVariables;
 };
 
 } // namespace readability
diff --git a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
index 6e7fcaa4345a5..c885aac89072a 100644
--- a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
@@ -26,6 +26,26 @@ using namespace clang::ast_matchers;
 
 namespace clang {
 namespace tidy {
+
+llvm::ArrayRef<
+    std::pair<readability::IdentifierNamingCheck::CaseType, StringRef>>
+OptionEnumMapping<
+    readability::IdentifierNamingCheck::CaseType>::getEnumMapping() {
+  static constexpr std::pair<readability::IdentifierNamingCheck::CaseType,
+                             StringRef>
+      Mapping[] = {
+          {readability::IdentifierNamingCheck::CT_AnyCase, "aNy_CasE"},
+          {readability::IdentifierNamingCheck::CT_LowerCase, "lower_case"},
+          {readability::IdentifierNamingCheck::CT_UpperCase, "UPPER_CASE"},
+          {readability::IdentifierNamingCheck::CT_CamelBack, "camelBack"},
+          {readability::IdentifierNamingCheck::CT_CamelCase, "CamelCase"},
+          {readability::IdentifierNamingCheck::CT_CamelSnakeCase,
+           "Camel_Snake_Case"},
+          {readability::IdentifierNamingCheck::CT_CamelSnakeBack,
+           "camel_Snake_Back"}};
+  return llvm::makeArrayRef(Mapping);
+}
+
 namespace readability {
 
 // clang-format off
@@ -99,16 +119,6 @@ static StringRef const StyleNames[] = {
 #undef NAMING_KEYS
 // clang-format on
 
-static constexpr std::pair<StringRef, IdentifierNamingCheck::CaseType>
-    Mapping[] = {
-        {"aNy_CasE", IdentifierNamingCheck::CT_AnyCase},
-        {"lower_case", IdentifierNamingCheck::CT_LowerCase},
-        {"UPPER_CASE", IdentifierNamingCheck::CT_UpperCase},
-        {"camelBack", IdentifierNamingCheck::CT_CamelBack},
-        {"CamelCase", IdentifierNamingCheck::CT_CamelCase},
-        {"Camel_Snake_Case", IdentifierNamingCheck::CT_CamelSnakeCase},
-        {"camel_Snake_Back", IdentifierNamingCheck::CT_CamelSnakeBack}};
-
 IdentifierNamingCheck::IdentifierNamingCheck(StringRef Name,
                                              ClangTidyContext *Context)
     : RenamerClangTidyCheck(Name, Context),
@@ -117,7 +127,7 @@ IdentifierNamingCheck::IdentifierNamingCheck(StringRef Name,
 
   for (auto const &Name : StyleNames) {
     auto CaseOptional = [&]() -> llvm::Optional<CaseType> {
-      auto ValueOr = Options.get((Name + "Case").str(), makeArrayRef(Mapping));
+      auto ValueOr = Options.get<CaseType>((Name + "Case").str());
       if (ValueOr)
         return *ValueOr;
       llvm::logAllUnhandledErrors(
@@ -148,7 +158,7 @@ void IdentifierNamingCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
     if (NamingStyles[i]) {
       if (NamingStyles[i]->Case) {
         Options.store(Opts, (StyleNames[i] + "Case").str(),
-                      *NamingStyles[i]->Case, llvm::makeArrayRef(Mapping));
+                      *NamingStyles[i]->Case);
       }
       Options.store(Opts, (StyleNames[i] + "Prefix").str(),
                     NamingStyles[i]->Prefix);
diff --git a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.h b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.h
index 04bf53fe16b56..0f6c77b2c9a86 100644
--- a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.h
@@ -75,6 +75,12 @@ class IdentifierNamingCheck final : public RenamerClangTidyCheck {
 };
 
 } // namespace readability
+template <>
+struct OptionEnumMapping<readability::IdentifierNamingCheck::CaseType> {
+  static llvm::ArrayRef<
+      std::pair<readability::IdentifierNamingCheck::CaseType, StringRef>>
+  getEnumMapping();
+};
 } // namespace tidy
 } // namespace clang
 
diff --git a/clang-tools-extra/clang-tidy/readability/MagicNumbersCheck.h b/clang-tools-extra/clang-tidy/readability/MagicNumbersCheck.h
index a53fa917066ee..9612fd7ec6aab 100644
--- a/clang-tools-extra/clang-tidy/readability/MagicNumbersCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/MagicNumbersCheck.h
@@ -10,9 +10,9 @@
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_MAGICNUMBERSCHECK_H
 
 #include "../ClangTidyCheck.h"
+#include "clang/Lex/Lexer.h"
 #include <llvm/ADT/APFloat.h>
 #include <llvm/ADT/SmallVector.h>
-#include <vector>
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp b/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp
index ee087e3b3133c..7a8ad2c3aec62 100644
--- a/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp
@@ -11,6 +11,7 @@
 #include "clang/AST/ParentMapContext.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
 
 using namespace clang::ast_matchers;
 
diff --git a/clang-tools-extra/clang-tidy/readability/RedundantPreprocessorCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantPreprocessorCheck.cpp
index 0870f2db2e6db..822c4cd14ddc1 100644
--- a/clang-tools-extra/clang-tidy/readability/RedundantPreprocessorCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/RedundantPreprocessorCheck.cpp
@@ -8,6 +8,9 @@
 
 #include "RedundantPreprocessorCheck.h"
 #include "clang/Frontend/CompilerInstance.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/readability/UppercaseLiteralSuffixCheck.cpp b/clang-tools-extra/clang-tidy/readability/UppercaseLiteralSuffixCheck.cpp
index aab45faa6cb3a..eef6253f6a1ff 100644
--- a/clang-tools-extra/clang-tidy/readability/UppercaseLiteralSuffixCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/UppercaseLiteralSuffixCheck.cpp
@@ -8,9 +8,9 @@
 
 #include "UppercaseLiteralSuffixCheck.h"
 #include "../utils/ASTUtils.h"
-#include "../utils/OptionsUtils.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallString.h"
 
diff --git a/clang-tools-extra/clang-tidy/readability/UseAnyOfAllOfCheck.h b/clang-tools-extra/clang-tidy/readability/UseAnyOfAllOfCheck.h
index c44ef2e70ae98..768fb5254e288 100644
--- a/clang-tools-extra/clang-tidy/readability/UseAnyOfAllOfCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/UseAnyOfAllOfCheck.h
@@ -10,7 +10,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_USEALGORITHMCHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_USEALGORITHMCHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 #include "../utils/IncludeInserter.h"
 
 namespace clang {
diff --git a/clang-tools-extra/clang-tidy/tool/CMakeLists.txt b/clang-tools-extra/clang-tidy/tool/CMakeLists.txt
index 00554e43ad2fb..f12cff1b1303a 100644
--- a/clang-tools-extra/clang-tidy/tool/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/tool/CMakeLists.txt
@@ -15,6 +15,9 @@ add_clang_library(clangTidyMain
   LINK_LIBS
   clangTidy
   ${ALL_CLANG_TIDY_CHECKS}
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyMain
diff --git a/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp b/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
index 82939faf1ea88..e248c04ea5709 100644
--- a/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
+++ b/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
@@ -25,8 +25,6 @@
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/WithColor.h"
 
-using namespace clang::ast_matchers;
-using namespace clang::driver;
 using namespace clang::tooling;
 using namespace llvm;
 
diff --git a/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py b/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
index 2b5e78b38f016..4272ae0957fe2 100755
--- a/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
+++ b/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
@@ -277,7 +277,6 @@ def main():
     tmpdir = tempfile.mkdtemp()
 
   # Build up a big regexy filter from all command line arguments.
-  args.files = [re.escape(f) for f in args.files]
   file_name_re = re.compile('|'.join(args.files))
 
   return_code = 0
diff --git a/clang-tools-extra/clang-tidy/utils/Aliasing.cpp b/clang-tools-extra/clang-tidy/utils/Aliasing.cpp
new file mode 100644
index 0000000000000..3a88126a9ee6a
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/utils/Aliasing.cpp
@@ -0,0 +1,65 @@
+//===------------- Aliasing.cpp - clang-tidy ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Aliasing.h"
+
+#include "clang/AST/Expr.h"
+
+namespace clang {
+namespace tidy {
+namespace utils {
+
+/// Return whether \p S is a reference to the declaration of \p Var.
+static bool isAccessForVar(const Stmt *S, const VarDecl *Var) {
+  if (const auto *DRE = dyn_cast<DeclRefExpr>(S))
+    return DRE->getDecl() == Var;
+
+  return false;
+}
+
+/// Return whether \p Var has a pointer or reference in \p S.
+static bool isPtrOrReferenceForVar(const Stmt *S, const VarDecl *Var) {
+  if (const auto *DS = dyn_cast<DeclStmt>(S)) {
+    for (const Decl *D : DS->getDeclGroup()) {
+      if (const auto *LeftVar = dyn_cast<VarDecl>(D)) {
+        if (LeftVar->hasInit() && LeftVar->getType()->isReferenceType()) {
+          return isAccessForVar(LeftVar->getInit(), Var);
+        }
+      }
+    }
+  } else if (const auto *UnOp = dyn_cast<UnaryOperator>(S)) {
+    if (UnOp->getOpcode() == UO_AddrOf)
+      return isAccessForVar(UnOp->getSubExpr(), Var);
+  }
+
+  return false;
+}
+
+/// Return whether \p Var has a pointer or reference in \p S.
+static bool hasPtrOrReferenceInStmt(const Stmt *S, const VarDecl *Var) {
+  if (isPtrOrReferenceForVar(S, Var))
+    return true;
+
+  for (const Stmt *Child : S->children()) {
+    if (!Child)
+      continue;
+
+    if (hasPtrOrReferenceInStmt(Child, Var))
+      return true;
+  }
+
+  return false;
+}
+
+bool hasPtrOrReferenceInFunc(const FunctionDecl *Func, const VarDecl *Var) {
+  return hasPtrOrReferenceInStmt(Func->getBody(), Var);
+}
+
+} // namespace utils
+} // namespace tidy
+} // namespace clang
diff --git a/clang-tools-extra/clang-tidy/utils/Aliasing.h b/clang-tools-extra/clang-tidy/utils/Aliasing.h
new file mode 100644
index 0000000000000..e43995a6714ad
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/utils/Aliasing.h
@@ -0,0 +1,36 @@
+//===------------- Aliasing.h - clang-tidy --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_UTILS_ALIASING_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_UTILS_ALIASING_H
+
+#include "clang/AST/Decl.h"
+
+namespace clang {
+namespace tidy {
+namespace utils {
+
+/// Returns whether \p Var has a pointer or reference in \p Func.
+///
+/// Example:
+/// void f() {
+///   int n;
+///   ...
+///   int *p = &n;
+/// }
+///
+/// For `f()` and `n` the function returns ``true`` because `p` is a
+/// pointer to `n` created in `f()`.
+
+bool hasPtrOrReferenceInFunc(const FunctionDecl *Func, const VarDecl *Var);
+
+} // namespace utils
+} // namespace tidy
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_UTILS_ALIASING_H
diff --git a/clang-tools-extra/clang-tidy/utils/CMakeLists.txt b/clang-tools-extra/clang-tidy/utils/CMakeLists.txt
index 26032f21c7016..70edb9c33d74b 100644
--- a/clang-tools-extra/clang-tidy/utils/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/utils/CMakeLists.txt
@@ -4,6 +4,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 add_clang_library(clangTidyUtils
+  Aliasing.cpp
   ASTUtils.cpp
   DeclRefExprUtils.cpp
   ExceptionAnalyzer.cpp
@@ -23,6 +24,9 @@ add_clang_library(clangTidyUtils
 
   LINK_LIBS
   clangTidy
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyUtils
diff --git a/clang-tools-extra/clang-tidy/utils/ExprSequence.cpp b/clang-tools-extra/clang-tidy/utils/ExprSequence.cpp
index 8fddf4d4996aa..15c4375a64be7 100644
--- a/clang-tools-extra/clang-tidy/utils/ExprSequence.cpp
+++ b/clang-tools-extra/clang-tidy/utils/ExprSequence.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "ExprSequence.h"
+#include "clang/AST/ParentMapContext.h"
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/utils/HeaderGuard.h b/clang-tools-extra/clang-tidy/utils/HeaderGuard.h
index d3e09a32b6878..86c0f3dc8fda2 100644
--- a/clang-tools-extra/clang-tidy/utils/HeaderGuard.h
+++ b/clang-tools-extra/clang-tidy/utils/HeaderGuard.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_UTILS_HEADERGUARD_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_UTILS_HEADERGUARD_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 #include "../utils/FileExtensionsUtils.h"
 
 namespace clang {
diff --git a/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp b/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp
index 6013412a41385..c9d018f076e76 100644
--- a/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp
+++ b/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "IncludeSorter.h"
+#include "clang/Basic/SourceManager.h"
 #include "clang/Lex/Lexer.h"
 
 namespace clang {
@@ -174,13 +175,14 @@ Optional<FixItHint> IncludeSorter::CreateIncludeInsertion(StringRef FileName,
                                     IncludeStmt);
 }
 
-llvm::ArrayRef<std::pair<StringRef, IncludeSorter::IncludeStyle>>
-IncludeSorter::getMapping() {
-  static constexpr std::pair<StringRef, IncludeSorter::IncludeStyle> Mapping[] =
-      {{"llvm", IS_LLVM}, {"google", IS_Google}};
+} // namespace utils
+
+llvm::ArrayRef<std::pair<utils::IncludeSorter::IncludeStyle, StringRef>>
+OptionEnumMapping<utils::IncludeSorter::IncludeStyle>::getEnumMapping() {
+  static constexpr std::pair<utils::IncludeSorter::IncludeStyle, StringRef>
+      Mapping[] = {{utils::IncludeSorter::IS_LLVM, "llvm"},
+                   {utils::IncludeSorter::IS_Google, "google"}};
   return makeArrayRef(Mapping);
 }
-
-} // namespace utils
 } // namespace tidy
 } // namespace clang
diff --git a/clang-tools-extra/clang-tidy/utils/IncludeSorter.h b/clang-tools-extra/clang-tidy/utils/IncludeSorter.h
index 7dab2cc536a48..1d8997364e5ce 100644
--- a/clang-tools-extra/clang-tidy/utils/IncludeSorter.h
+++ b/clang-tools-extra/clang-tidy/utils/IncludeSorter.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_INCLUDESORTER_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_INCLUDESORTER_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 #include <string>
 
 namespace clang {
@@ -25,8 +25,6 @@ class IncludeSorter {
   /// Supported include styles.
   enum IncludeStyle { IS_LLVM = 0, IS_Google = 1 };
 
-  static ArrayRef<std::pair<StringRef, IncludeStyle>> getMapping();
-
   /// The classifications of inclusions, in the order they should be sorted.
   enum IncludeKinds {
     IK_MainTUInclude = 0,    ///< e.g. ``#include "foo.h"`` when editing foo.cc
@@ -66,6 +64,11 @@ class IncludeSorter {
 };
 
 } // namespace utils
+
+template <> struct OptionEnumMapping<utils::IncludeSorter::IncludeStyle> {
+  static ArrayRef<std::pair<utils::IncludeSorter::IncludeStyle, StringRef>>
+  getEnumMapping();
+};
 } // namespace tidy
 } // namespace clang
 #endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_INCLUDESORTER_H
diff --git a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
index e90ab1782b4b4..040378d980f1a 100644
--- a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
@@ -156,14 +156,17 @@ void RenamerClangTidyCheck::addUsage(
   // is already in there
   RenamerClangTidyCheck::NamingCheckFailure &Failure =
       NamingCheckFailures[Decl];
-  if (!Failure.RawUsageLocs.insert(FixLocation.getRawEncoding()).second)
-    return;
 
   if (!Failure.ShouldFix())
     return;
 
+  if (SourceMgr && SourceMgr->isWrittenInScratchSpace(FixLocation))
+    Failure.FixStatus = RenamerClangTidyCheck::ShouldFixStatus::InsideMacro;
+
   if (!utils::rangeCanBeFixed(Range, SourceMgr))
     Failure.FixStatus = RenamerClangTidyCheck::ShouldFixStatus::InsideMacro;
+
+  Failure.RawUsageLocs.insert(FixLocation.getRawEncoding());
 }
 
 void RenamerClangTidyCheck::addUsage(const NamedDecl *Decl, SourceRange Range,
@@ -248,13 +251,15 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) {
   if (const auto *Decl =
           Result.Nodes.getNodeAs<CXXConstructorDecl>("classRef")) {
 
-    addUsage(Decl->getParent(), Decl->getNameInfo().getSourceRange());
+    addUsage(Decl->getParent(), Decl->getNameInfo().getSourceRange(),
+             Result.SourceManager);
 
     for (const auto *Init : Decl->inits()) {
       if (!Init->isWritten() || Init->isInClassMemberInitializer())
         continue;
       if (const FieldDecl *FD = Init->getAnyMember())
-        addUsage(FD, SourceRange(Init->getMemberLocation()));
+        addUsage(FD, SourceRange(Init->getMemberLocation()),
+                 Result.SourceManager);
       // Note: delegating constructors and base class initializers are handled
       // via the "typeLoc" matcher.
     }
@@ -271,7 +276,7 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) {
     // we want instead to replace the next token, that will be the identifier.
     Range.setBegin(CharSourceRange::getTokenRange(Range).getEnd());
 
-    addUsage(Decl->getParent(), Range);
+    addUsage(Decl->getParent(), Range, Result.SourceManager);
     return;
   }
 
@@ -289,7 +294,7 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) {
     // further TypeLocs handled below
 
     if (Decl) {
-      addUsage(Decl, Loc->getSourceRange());
+      addUsage(Decl, Loc->getSourceRange(), Result.SourceManager);
       return;
     }
 
@@ -300,7 +305,7 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) {
       SourceRange Range(Ref.getTemplateNameLoc(), Ref.getTemplateNameLoc());
       if (const auto *ClassDecl = dyn_cast<TemplateDecl>(Decl)) {
         if (const NamedDecl *TemplDecl = ClassDecl->getTemplatedDecl())
-          addUsage(TemplDecl, Range);
+          addUsage(TemplDecl, Range, Result.SourceManager);
         return;
       }
     }
@@ -308,7 +313,7 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) {
     if (const auto &Ref =
             Loc->getAs<DependentTemplateSpecializationTypeLoc>()) {
       if (const TagDecl *Decl = Ref.getTypePtr()->getAsTagDecl())
-        addUsage(Decl, Loc->getSourceRange());
+        addUsage(Decl, Loc->getSourceRange(), Result.SourceManager);
       return;
     }
   }
@@ -317,7 +322,7 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) {
           Result.Nodes.getNodeAs<NestedNameSpecifierLoc>("nestedNameLoc")) {
     if (const NestedNameSpecifier *Spec = Loc->getNestedNameSpecifier()) {
       if (const NamespaceDecl *Decl = Spec->getAsNamespace()) {
-        addUsage(Decl, Loc->getLocalSourceRange());
+        addUsage(Decl, Loc->getLocalSourceRange(), Result.SourceManager);
         return;
       }
     }
@@ -325,7 +330,8 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) {
 
   if (const auto *Decl = Result.Nodes.getNodeAs<UsingDecl>("using")) {
     for (const auto *Shadow : Decl->shadows())
-      addUsage(Shadow->getTargetDecl(), Decl->getNameInfo().getSourceRange());
+      addUsage(Shadow->getTargetDecl(), Decl->getNameInfo().getSourceRange(),
+               Result.SourceManager);
     return;
   }
 
@@ -371,14 +377,14 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) {
     // Fix using namespace declarations.
     if (const auto *UsingNS = dyn_cast<UsingDirectiveDecl>(Decl))
       addUsage(UsingNS->getNominatedNamespaceAsWritten(),
-               UsingNS->getIdentLocation());
+               UsingNS->getIdentLocation(), Result.SourceManager);
 
     if (!Decl->getIdentifier() || Decl->getName().empty() || Decl->isImplicit())
       return;
 
     const auto *Canonical = cast<NamedDecl>(Decl->getCanonicalDecl());
     if (Canonical != Decl) {
-      addUsage(Canonical, Decl->getLocation());
+      addUsage(Canonical, Decl->getLocation(), Result.SourceManager);
       return;
     }
 
@@ -386,7 +392,8 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) {
     if (const auto *Value = Result.Nodes.getNodeAs<ValueDecl>("decl")) {
       if (const Type *TypePtr = Value->getType().getTypePtrOrNull()) {
         if (const auto *Typedef = TypePtr->getAs<TypedefType>())
-          addUsage(Typedef->getDecl(), Value->getSourceRange());
+          addUsage(Typedef->getDecl(), Value->getSourceRange(),
+                   Result.SourceManager);
       }
     }
 
@@ -394,11 +401,13 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) {
     if (const auto *Value = Result.Nodes.getNodeAs<FunctionDecl>("decl")) {
       if (const auto *Typedef =
               Value->getReturnType().getTypePtr()->getAs<TypedefType>())
-        addUsage(Typedef->getDecl(), Value->getSourceRange());
+        addUsage(Typedef->getDecl(), Value->getSourceRange(),
+                 Result.SourceManager);
       for (const ParmVarDecl *Param : Value->parameters()) {
         if (const TypedefType *Typedef =
                 Param->getType().getTypePtr()->getAs<TypedefType>())
-          addUsage(Typedef->getDecl(), Value->getSourceRange());
+          addUsage(Typedef->getDecl(), Value->getSourceRange(),
+                   Result.SourceManager);
       }
     }
 
diff --git a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.h b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.h
index d20f27ce0466f..a352b6a5584ae 100644
--- a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.h
+++ b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.h
@@ -12,6 +12,7 @@
 #include "../ClangTidyCheck.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/FunctionExtras.h"
 #include "llvm/ADT/Optional.h"
 #include <string>
 #include <utility>
diff --git a/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.cpp b/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.cpp
index 0a57856b79722..03af5dd1565f8 100644
--- a/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "TransformerClangTidyCheck.h"
+#include "clang/Lex/Preprocessor.h"
 #include "llvm/ADT/STLExtras.h"
 
 namespace clang {
@@ -32,7 +33,6 @@ TransformerClangTidyCheck::TransformerClangTidyCheck(
     StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context), Rule(MakeRule(getLangOpts(), Options)),
       IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            IncludeSorter::getMapping(),
                                             IncludeSorter::IS_LLVM)) {
   if (Rule)
     assert(llvm::all_of(Rule->Cases, hasExplanation) &&
@@ -45,7 +45,6 @@ TransformerClangTidyCheck::TransformerClangTidyCheck(RewriteRule R,
                                                      ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context), Rule(std::move(R)),
       IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            IncludeSorter::getMapping(),
                                             IncludeSorter::IS_LLVM)) {
   assert(llvm::all_of(Rule->Cases, hasExplanation) &&
          "clang-tidy checks must have an explanation by default;"
@@ -111,8 +110,7 @@ void TransformerClangTidyCheck::check(
 
 void TransformerClangTidyCheck::storeOptions(
     ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle,
-                IncludeSorter::getMapping());
+  Options.store(Opts, "IncludeStyle", IncludeStyle);
 }
 
 } // namespace utils
diff --git a/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.h b/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.h
index d99f927a79732..829a22fe8e2cc 100644
--- a/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.h
+++ b/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.h
@@ -9,14 +9,11 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_TRANSFORMER_CLANG_TIDY_CHECK_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_TRANSFORMER_CLANG_TIDY_CHECK_H
 
-#include "../ClangTidy.h"
+#include "../ClangTidyCheck.h"
 #include "IncludeInserter.h"
 #include "IncludeSorter.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
-#include "clang/Frontend/CompilerInstance.h"
 #include "clang/Tooling/Transformer/Transformer.h"
-#include <deque>
-#include <vector>
 
 namespace clang {
 namespace tidy {
diff --git a/clang-tools-extra/clang-tidy/zircon/CMakeLists.txt b/clang-tools-extra/clang-tidy/zircon/CMakeLists.txt
index 29c99e92e0974..89c0d5737cec1 100644
--- a/clang-tools-extra/clang-tidy/zircon/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/zircon/CMakeLists.txt
@@ -10,6 +10,9 @@ add_clang_library(clangTidyZirconModule
   LINK_LIBS
   clangTidy
   clangTidyUtils
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangTidyZirconModule
diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt
index fa4f8b41ff8d4..9eb06941e4dd3 100644
--- a/clang-tools-extra/clangd/CMakeLists.txt
+++ b/clang-tools-extra/clangd/CMakeLists.txt
@@ -36,6 +36,10 @@ add_clang_library(clangDaemon
   CollectMacros.cpp
   CompileCommands.cpp
   Compiler.cpp
+  Config.cpp
+  ConfigCompile.cpp
+  ConfigProvider.cpp
+  ConfigYAML.cpp
   Diagnostics.cpp
   DraftStore.cpp
   ExpectedTypes.cpp
@@ -100,6 +104,9 @@ add_clang_library(clangDaemon
   clangTidy
   ${LLVM_PTHREAD_LIB}
   ${ALL_CLANG_TIDY_CHECKS}
+
+  DEPENDS
+  omp_gen
   )
 
 clang_target_link_libraries(clangDaemon
diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp
index 93f2746bb6f0c..b0aba886edbe4 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.cpp
+++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp
@@ -599,8 +599,8 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
              }},
             {"semanticTokensProvider",
              llvm::json::Object{
-                 {"documentProvider", llvm::json::Object{{"edits", true}}},
-                 {"rangeProvider", false},
+                 {"full", llvm::json::Object{{"delta", true}}},
+                 {"range", false},
                  {"legend",
                   llvm::json::Object{{"tokenTypes", semanticTokenTypes()},
                                      {"tokenModifiers", llvm::json::Array()}}},
@@ -879,7 +879,8 @@ void ClangdLSPServer::onDocumentOnTypeFormatting(
         "onDocumentOnTypeFormatting called for non-added file",
         ErrorCode::InvalidParams));
 
-  Reply(Server->formatOnType(Code->Contents, File, Params.position, Params.ch));
+  Server->formatOnType(File, Code->Contents, Params.position, Params.ch,
+                       std::move(Reply));
 }
 
 void ClangdLSPServer::onDocumentRangeFormatting(
@@ -892,12 +893,15 @@ void ClangdLSPServer::onDocumentRangeFormatting(
         "onDocumentRangeFormatting called for non-added file",
         ErrorCode::InvalidParams));
 
-  auto ReplacementsOrError =
-      Server->formatRange(Code->Contents, File, Params.range);
-  if (ReplacementsOrError)
-    Reply(replacementsToEdits(Code->Contents, ReplacementsOrError.get()));
-  else
-    Reply(ReplacementsOrError.takeError());
+  Server->formatRange(
+      File, Code->Contents, Params.range,
+      [Code = Code->Contents, Reply = std::move(Reply)](
+          llvm::Expected<tooling::Replacements> Result) mutable {
+        if (Result)
+          Reply(replacementsToEdits(Code, Result.get()));
+        else
+          Reply(Result.takeError());
+      });
 }
 
 void ClangdLSPServer::onDocumentFormatting(
@@ -910,11 +914,14 @@ void ClangdLSPServer::onDocumentFormatting(
         "onDocumentFormatting called for non-added file",
         ErrorCode::InvalidParams));
 
-  auto ReplacementsOrError = Server->formatFile(Code->Contents, File);
-  if (ReplacementsOrError)
-    Reply(replacementsToEdits(Code->Contents, ReplacementsOrError.get()));
-  else
-    Reply(ReplacementsOrError.takeError());
+  Server->formatFile(File, Code->Contents,
+                     [Code = Code->Contents, Reply = std::move(Reply)](
+                         llvm::Expected<tooling::Replacements> Result) mutable {
+                       if (Result)
+                         Reply(replacementsToEdits(Code, Result.get()));
+                       else
+                         Reply(Result.takeError());
+                     });
 }
 
 /// The functions constructs a flattened view of the DocumentSymbol hierarchy.
@@ -1304,9 +1311,9 @@ void ClangdLSPServer::onSemanticTokens(const SemanticTokensParams &Params,
       });
 }
 
-void ClangdLSPServer::onSemanticTokensEdits(
-    const SemanticTokensEditsParams &Params,
-    Callback<SemanticTokensOrEdits> CB) {
+void ClangdLSPServer::onSemanticTokensDelta(
+    const SemanticTokensDeltaParams &Params,
+    Callback<SemanticTokensOrDelta> CB) {
   Server->semanticHighlights(
       Params.textDocument.uri.file(),
       [this, PrevResultID(Params.previousResultId),
@@ -1316,7 +1323,7 @@ void ClangdLSPServer::onSemanticTokensEdits(
           return CB(HT.takeError());
         std::vector<SemanticToken> Toks = toSemanticTokens(*HT);
 
-        SemanticTokensOrEdits Result;
+        SemanticTokensOrDelta Result;
         {
           std::lock_guard<std::mutex> Lock(SemanticTokensMutex);
           auto &Last = LastSemanticTokens[File];
@@ -1324,8 +1331,8 @@ void ClangdLSPServer::onSemanticTokensEdits(
           if (PrevResultID == Last.resultId) {
             Result.edits = diffTokens(Last.tokens, Toks);
           } else {
-            vlog("semanticTokens/edits: wanted edits vs {0} but last result "
-                 "had ID {1}. Returning full token list.",
+            vlog("semanticTokens/full/delta: wanted edits vs {0} but last "
+                 "result had ID {1}. Returning full token list.",
                  PrevResultID, Last.resultId);
             Result.tokens = Toks;
           }
@@ -1386,8 +1393,8 @@ ClangdLSPServer::ClangdLSPServer(
   MsgHandler->bind("typeHierarchy/resolve", &ClangdLSPServer::onResolveTypeHierarchy);
   MsgHandler->bind("textDocument/selectionRange", &ClangdLSPServer::onSelectionRange);
   MsgHandler->bind("textDocument/documentLink", &ClangdLSPServer::onDocumentLink);
-  MsgHandler->bind("textDocument/semanticTokens", &ClangdLSPServer::onSemanticTokens);
-  MsgHandler->bind("textDocument/semanticTokens/edits", &ClangdLSPServer::onSemanticTokensEdits);
+  MsgHandler->bind("textDocument/semanticTokens/full", &ClangdLSPServer::onSemanticTokens);
+  MsgHandler->bind("textDocument/semanticTokens/full/delta", &ClangdLSPServer::onSemanticTokensDelta);
   // clang-format on
 }
 
diff --git a/clang-tools-extra/clangd/ClangdLSPServer.h b/clang-tools-extra/clangd/ClangdLSPServer.h
index 6ccef9500679c..a779e9036c4a8 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.h
+++ b/clang-tools-extra/clangd/ClangdLSPServer.h
@@ -121,8 +121,8 @@ class ClangdLSPServer : private ClangdServer::Callbacks {
   void onDocumentLink(const DocumentLinkParams &,
                       Callback<std::vector<DocumentLink>>);
   void onSemanticTokens(const SemanticTokensParams &, Callback<SemanticTokens>);
-  void onSemanticTokensEdits(const SemanticTokensEditsParams &,
-                             Callback<SemanticTokensOrEdits>);
+  void onSemanticTokensDelta(const SemanticTokensDeltaParams &,
+                             Callback<SemanticTokensOrDelta>);
 
   std::vector<Fix> getFixes(StringRef File, const clangd::Diagnostic &D);
 
diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp
index 6c7255c96963d..5d99104dadaf6 100644
--- a/clang-tools-extra/clangd/ClangdServer.cpp
+++ b/clang-tools-extra/clangd/ClangdServer.cpp
@@ -8,6 +8,7 @@
 
 #include "ClangdServer.h"
 #include "CodeComplete.h"
+#include "Config.h"
 #include "FindSymbols.h"
 #include "Format.h"
 #include "HeaderSourceSwitch.h"
@@ -132,7 +133,7 @@ ClangdServer::Options::operator TUScheduler::Options() const {
 ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
                            const ThreadsafeFS &TFS, const Options &Opts,
                            Callbacks *Callbacks)
-    : TFS(TFS),
+    : ConfigProvider(Opts.ConfigProvider), TFS(TFS),
       DynamicIdx(Opts.BuildDynamicSymbolIndex
                      ? new FileIndex(Opts.HeavyweightDynamicSymbolIndex)
                      : nullptr),
@@ -147,7 +148,14 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
       // FIXME(ioeric): this can be slow and we may be able to index on less
       // critical paths.
       WorkScheduler(
-          CDB, TUScheduler::Options(Opts),
+          CDB,
+          [&, this] {
+            TUScheduler::Options O(Opts);
+            O.ContextProvider = [this](PathRef P) {
+              return createProcessingContext(P);
+            };
+            return O;
+          }(),
           std::make_unique<UpdateIndexCallbacks>(
               DynamicIdx.get(), Callbacks, Opts.TheiaSemanticHighlighting)) {
   // Adds an index to the stack, at higher priority than existing indexes.
@@ -170,7 +178,8 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
         [Callbacks](BackgroundQueue::Stats S) {
           if (Callbacks)
             Callbacks->onBackgroundIndexProgress(S);
-        });
+        },
+        [this](PathRef P) { return createProcessingContext(P); });
     AddIndex(BackgroundIdx.get());
   }
   if (DynamicIdx)
@@ -296,40 +305,46 @@ void ClangdServer::signatureHelp(PathRef File, Position Pos,
                                 std::move(Action));
 }
 
-llvm::Expected<tooling::Replacements>
-ClangdServer::formatRange(llvm::StringRef Code, PathRef File, Range Rng) {
+void ClangdServer::formatRange(PathRef File, llvm::StringRef Code, Range Rng,
+                               Callback<tooling::Replacements> CB) {
   llvm::Expected<size_t> Begin = positionToOffset(Code, Rng.start);
   if (!Begin)
-    return Begin.takeError();
+    return CB(Begin.takeError());
   llvm::Expected<size_t> End = positionToOffset(Code, Rng.end);
   if (!End)
-    return End.takeError();
-  return formatCode(Code, File, {tooling::Range(*Begin, *End - *Begin)});
+    return CB(End.takeError());
+  formatCode(File, Code, {tooling::Range(*Begin, *End - *Begin)},
+             std::move(CB));
 }
 
-llvm::Expected<tooling::Replacements>
-ClangdServer::formatFile(llvm::StringRef Code, PathRef File) {
+void ClangdServer::formatFile(PathRef File, llvm::StringRef Code,
+                              Callback<tooling::Replacements> CB) {
   // Format everything.
-  return formatCode(Code, File, {tooling::Range(0, Code.size())});
+  formatCode(File, Code, {tooling::Range(0, Code.size())}, std::move(CB));
 }
 
-llvm::Expected<std::vector<TextEdit>>
-ClangdServer::formatOnType(llvm::StringRef Code, PathRef File, Position Pos,
-                           StringRef TriggerText) {
+void ClangdServer::formatOnType(PathRef File, llvm::StringRef Code,
+                                Position Pos, StringRef TriggerText,
+                                Callback<std::vector<TextEdit>> CB) {
   llvm::Expected<size_t> CursorPos = positionToOffset(Code, Pos);
   if (!CursorPos)
-    return CursorPos.takeError();
-  auto Style = format::getStyle(format::DefaultFormatStyle, File,
-                                format::DefaultFallbackStyle, Code,
-                                TFS.view(/*CWD=*/llvm::None).get());
-  if (!Style)
-    return Style.takeError();
-
-  std::vector<TextEdit> Result;
-  for (const tooling::Replacement &R :
-       formatIncremental(Code, *CursorPos, TriggerText, *Style))
-    Result.push_back(replacementToEdit(Code, R));
-  return Result;
+    return CB(CursorPos.takeError());
+  auto Action = [File = File.str(), Code = Code.str(),
+                 TriggerText = TriggerText.str(), CursorPos = *CursorPos,
+                 CB = std::move(CB), this]() mutable {
+    auto Style = format::getStyle(format::DefaultFormatStyle, File,
+                                  format::DefaultFallbackStyle, Code,
+                                  TFS.view(/*CWD=*/llvm::None).get());
+    if (!Style)
+      return CB(Style.takeError());
+
+    std::vector<TextEdit> Result;
+    for (const tooling::Replacement &R :
+         formatIncremental(Code, CursorPos, TriggerText, *Style))
+      Result.push_back(replacementToEdit(Code, R));
+    return CB(Result);
+  };
+  WorkScheduler.run("FormatOnType", File, std::move(Action));
 }
 
 void ClangdServer::prepareRename(PathRef File, Position Pos,
@@ -561,21 +576,25 @@ void ClangdServer::switchSourceHeader(
   WorkScheduler.runWithAST("SwitchHeaderSource", Path, std::move(Action));
 }
 
-llvm::Expected<tooling::Replacements>
-ClangdServer::formatCode(llvm::StringRef Code, PathRef File,
-                         llvm::ArrayRef<tooling::Range> Ranges) {
+void ClangdServer::formatCode(PathRef File, llvm::StringRef Code,
+                              llvm::ArrayRef<tooling::Range> Ranges,
+                              Callback<tooling::Replacements> CB) {
   // Call clang-format.
-  format::FormatStyle Style = getFormatStyleForFile(File, Code, TFS);
-  tooling::Replacements IncludeReplaces =
-      format::sortIncludes(Style, Code, Ranges, File);
-  auto Changed = tooling::applyAllReplacements(Code, IncludeReplaces);
-  if (!Changed)
-    return Changed.takeError();
-
-  return IncludeReplaces.merge(format::reformat(
-      Style, *Changed,
-      tooling::calculateRangesAfterReplacements(IncludeReplaces, Ranges),
-      File));
+  auto Action = [File = File.str(), Code = Code.str(), Ranges = Ranges.vec(),
+                 CB = std::move(CB), this]() mutable {
+    format::FormatStyle Style = getFormatStyleForFile(File, Code, TFS);
+    tooling::Replacements IncludeReplaces =
+        format::sortIncludes(Style, Code, Ranges, File);
+    auto Changed = tooling::applyAllReplacements(Code, IncludeReplaces);
+    if (!Changed)
+      return CB(Changed.takeError());
+
+    CB(IncludeReplaces.merge(format::reformat(
+        Style, *Changed,
+        tooling::calculateRangesAfterReplacements(IncludeReplaces, Ranges),
+        File)));
+  };
+  WorkScheduler.run("Format", File, std::move(Action));
 }
 
 void ClangdServer::findDocumentHighlights(
@@ -636,7 +655,7 @@ void ClangdServer::workspaceSymbols(
     llvm::StringRef Query, int Limit,
     Callback<std::vector<SymbolInformation>> CB) {
   WorkScheduler.run(
-      "getWorkspaceSymbols",
+      "getWorkspaceSymbols", /*Path=*/"",
       [Query = Query.str(), Limit, CB = std::move(CB), this]() mutable {
         CB(clangd::getWorkspaceSymbols(Query, Limit, Index,
                                        WorkspaceRoot.getValueOr("")));
@@ -726,6 +745,31 @@ llvm::StringMap<TUScheduler::FileStats> ClangdServer::fileStats() const {
   return WorkScheduler.fileStats();
 }
 
+Context ClangdServer::createProcessingContext(PathRef File) const {
+  if (!ConfigProvider)
+    return Context::current().clone();
+
+  config::Params Params;
+  llvm::SmallString<256> PosixPath;
+  if (!File.empty()) {
+    assert(llvm::sys::path::is_absolute(File));
+    llvm::sys::path::native(File, PosixPath, llvm::sys::path::Style::posix);
+    Params.Path = PosixPath.str();
+  }
+
+  auto DiagnosticHandler = [](const llvm::SMDiagnostic &Diag) {
+    if (Diag.getKind() == llvm::SourceMgr::DK_Error) {
+      elog("config error at {0}:{1}:{2}: {3}", Diag.getFilename(),
+           Diag.getLineNo(), Diag.getColumnNo(), Diag.getMessage());
+    } else {
+      log("config warning at {0}:{1}:{2}: {3}", Diag.getFilename(),
+          Diag.getLineNo(), Diag.getColumnNo(), Diag.getMessage());
+    }
+  };
+  Config C = ConfigProvider->getConfig(Params, DiagnosticHandler);
+  return Context::current().derive(Config::Key, std::move(C));
+}
+
 LLVM_NODISCARD bool
 ClangdServer::blockUntilIdleForTest(llvm::Optional<double> TimeoutSeconds) {
   return WorkScheduler.blockUntilIdle(timeoutSeconds(TimeoutSeconds)) &&
diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h
index 021540df66f36..ea82081f24405 100644
--- a/clang-tools-extra/clangd/ClangdServer.h
+++ b/clang-tools-extra/clangd/ClangdServer.h
@@ -11,6 +11,7 @@
 
 #include "../clang-tidy/ClangTidyOptions.h"
 #include "CodeComplete.h"
+#include "ConfigProvider.h"
 #include "GlobalCompilationDatabase.h"
 #include "Hover.h"
 #include "Protocol.h"
@@ -99,7 +100,7 @@ class ClangdServer {
     bool StorePreamblesInMemory = true;
     /// Reuse even stale preambles, and rebuild them in the background.
     /// This improves latency at the cost of accuracy.
-    bool AsyncPreambleBuilds = false;
+    bool AsyncPreambleBuilds = true;
 
     /// If true, ClangdServer builds a dynamic in-memory index for symbols in
     /// opened files and uses the index to augment code completion results.
@@ -113,6 +114,9 @@ class ClangdServer {
     /// If set, use this index to augment code completion results.
     SymbolIndex *StaticIndex = nullptr;
 
+    /// If set, queried to obtain the configuration to handle each request.
+    config::Provider *ConfigProvider = nullptr;
+
     /// If set, enable clang-tidy in clangd and use to it get clang-tidy
     /// configurations for a particular file.
     /// Clangd supports only a small subset of ClangTidyOptions, these options
@@ -247,18 +251,17 @@ class ClangdServer {
                       Callback<ReferencesResult> CB);
 
   /// Run formatting for \p Rng inside \p File with content \p Code.
-  llvm::Expected<tooling::Replacements> formatRange(StringRef Code,
-                                                    PathRef File, Range Rng);
+  void formatRange(PathRef File, StringRef Code, Range Rng,
+                   Callback<tooling::Replacements> CB);
 
   /// Run formatting for the whole \p File with content \p Code.
-  llvm::Expected<tooling::Replacements> formatFile(StringRef Code,
-                                                   PathRef File);
+  void formatFile(PathRef File, StringRef Code,
+                  Callback<tooling::Replacements> CB);
 
   /// Run formatting after \p TriggerText was typed at \p Pos in \p File with
   /// content \p Code.
-  llvm::Expected<std::vector<TextEdit>> formatOnType(StringRef Code,
-                                                     PathRef File, Position Pos,
-                                                     StringRef TriggerText);
+  void formatOnType(PathRef File, StringRef Code, Position Pos,
+                    StringRef TriggerText, Callback<std::vector<TextEdit>> CB);
 
   /// Test the validity of a rename operation.
   void prepareRename(PathRef File, Position Pos,
@@ -323,11 +326,18 @@ class ClangdServer {
   blockUntilIdleForTest(llvm::Optional<double> TimeoutSeconds = 10);
 
 private:
-  /// FIXME: This stats several files to find a .clang-format file. I/O can be
-  /// slow. Think of a way to cache this.
-  llvm::Expected<tooling::Replacements>
-  formatCode(llvm::StringRef Code, PathRef File,
-             ArrayRef<tooling::Range> Ranges);
+  void formatCode(PathRef File, llvm::StringRef Code,
+                  ArrayRef<tooling::Range> Ranges,
+                  Callback<tooling::Replacements> CB);
+
+  /// Derives a context for a task processing the specified source file.
+  /// This includes the current configuration (see Options::ConfigProvider).
+  /// The empty string means no particular file is the target.
+  /// Rather than called by each feature, this is exposed to the components
+  /// that control worker threads, like TUScheduler and BackgroundIndex.
+  /// This means it's OK to do some IO here, and it cuts across all features.
+  Context createProcessingContext(PathRef) const;
+  config::Provider *ConfigProvider = nullptr;
 
   const ThreadsafeFS &TFS;
 
diff --git a/clang-tools-extra/clangd/CompileCommands.cpp b/clang-tools-extra/clangd/CompileCommands.cpp
index a2d704f57bda6..0b27e0e3e8284 100644
--- a/clang-tools-extra/clangd/CompileCommands.cpp
+++ b/clang-tools-extra/clangd/CompileCommands.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CompileCommands.h"
+#include "Config.h"
 #include "support/Logger.h"
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Tooling/ArgumentsAdjusters.h"
@@ -182,6 +183,10 @@ CommandMangler CommandMangler::forTests() {
 }
 
 void CommandMangler::adjust(std::vector<std::string> &Cmd) const {
+  // FIXME: remove const_cast once unique_function is const-compatible.
+  for (auto &Edit : const_cast<Config &>(Config::current()).CompileFlags.Edits)
+    Edit(Cmd);
+
   // Check whether the flag exists, either as -flag or -flag=*
   auto Has = [&](llvm::StringRef Flag) {
     for (llvm::StringRef Arg : Cmd) {
diff --git a/clang-tools-extra/clangd/Config.cpp b/clang-tools-extra/clangd/Config.cpp
new file mode 100644
index 0000000000000..3b9b9bf71e24d
--- /dev/null
+++ b/clang-tools-extra/clangd/Config.cpp
@@ -0,0 +1,25 @@
+//===--- Config.cpp - User configuration of clangd behavior ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Config.h"
+#include "support/Context.h"
+
+namespace clang {
+namespace clangd {
+
+Key<Config> Config::Key;
+
+const Config &Config::current() {
+  if (const Config *C = Context::current().get(Key))
+    return *C;
+  static Config Default;
+  return Default;
+}
+
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/Config.h b/clang-tools-extra/clangd/Config.h
new file mode 100644
index 0000000000000..878c9e8549b52
--- /dev/null
+++ b/clang-tools-extra/clangd/Config.h
@@ -0,0 +1,63 @@
+//===--- Config.h - User configuration of clangd behavior --------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Various clangd features have configurable behaviour (or can be disabled).
+// This file defines "resolved" configuration seen by features within clangd.
+// For example, settings may vary per-file, the resolved Config only contains
+// settings that apply to the current file.
+//
+// This is distinct from how the config is specified by the user (Fragment)
+// interpreted (CompiledFragment), and combined (Provider).
+// ConfigFragment.h describes the steps to add a new configuration option.
+//
+// Because this structure is shared throughout clangd, it's a potential source
+// of layering problems. Config should be expressed in terms of simple
+// vocubulary types where possible.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIG_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIG_H
+
+#include "support/Context.h"
+#include "llvm/ADT/FunctionExtras.h"
+#include <string>
+#include <vector>
+
+namespace clang {
+namespace clangd {
+
+/// Settings that express user/project preferences and control clangd behavior.
+///
+/// Generally, features should consume Config::current() and the caller is
+/// responsible for setting it appropriately. In practice these callers are
+/// ClangdServer, TUScheduler, and BackgroundQueue.
+struct Config {
+  /// Returns the Config of the current Context, or an empty configuration.
+  static const Config &current();
+  /// Context key which can be used to set the current Config.
+  static clangd::Key<Config> Key;
+
+  Config() = default;
+  Config(const Config &) = delete;
+  Config &operator=(const Config &) = delete;
+  Config(Config &&) = default;
+  Config &operator=(Config &&) = default;
+
+  /// Controls how the compile command for the current file is determined.
+  struct {
+    // Edits to apply to the compile command, in sequence.
+    // FIXME: these functions need to be const-callable. For now, const_cast.
+    std::vector<llvm::unique_function<void(std::vector<std::string> &)>> Edits;
+  } CompileFlags;
+};
+
+} // namespace clangd
+} // namespace clang
+
+#endif
diff --git a/clang-tools-extra/clangd/ConfigCompile.cpp b/clang-tools-extra/clangd/ConfigCompile.cpp
new file mode 100644
index 0000000000000..04c0df88bbf76
--- /dev/null
+++ b/clang-tools-extra/clangd/ConfigCompile.cpp
@@ -0,0 +1,172 @@
+//===--- ConfigCompile.cpp - Translating Fragments into Config ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Fragments are applied to Configs in two steps:
+//
+// 1. (When the fragment is first loaded)
+//    FragmentCompiler::compile() traverses the Fragment and creates
+//    function objects that know how to apply the configuration.
+// 2. (Every time a config is required)
+//    CompiledFragment() executes these functions to populate the Config.
+//
+// Work could be split between these steps in different ways. We try to
+// do as much work as possible in the first step. For example, regexes are
+// compiled in stage 1 and captured by the apply function. This is because:
+//
+//  - it's more efficient, as the work done in stage 1 must only be done once
+//  - problems can be reported in stage 1, in stage 2 we must silently recover
+//
+//===----------------------------------------------------------------------===//
+
+#include "Config.h"
+#include "ConfigFragment.h"
+#include "support/Logger.h"
+#include "support/Trace.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/SMLoc.h"
+#include "llvm/Support/SourceMgr.h"
+
+namespace clang {
+namespace clangd {
+namespace config {
+namespace {
+
+struct CompiledFragmentImpl {
+  // The independent conditions to check before using settings from this config.
+  // The following fragment has *two* conditions:
+  //   If: { Platform: [mac, linux], PathMatch: foo/.* }
+  // All of them must be satisfied: the platform and path conditions are ANDed.
+  // The OR logic for the platform condition is implemented inside the function.
+  std::vector<llvm::unique_function<bool(const Params &) const>> Conditions;
+  // Mutations that this fragment will apply to the configuration.
+  // These are invoked only if the conditions are satisfied.
+  std::vector<llvm::unique_function<void(Config &) const>> Apply;
+
+  bool operator()(const Params &P, Config &C) const {
+    for (const auto &C : Conditions) {
+      if (!C(P)) {
+        dlog("Config fragment {0}: condition not met", this);
+        return false;
+      }
+    }
+    dlog("Config fragment {0}: applying {1} rules", this, Apply.size());
+    for (const auto &A : Apply)
+      A(C);
+    return true;
+  }
+};
+
+// Wrapper around condition compile() functions to reduce arg-passing.
+struct FragmentCompiler {
+  CompiledFragmentImpl &Out;
+  DiagnosticCallback Diagnostic;
+  llvm::SourceMgr *SourceMgr;
+
+  llvm::Optional<llvm::Regex> compileRegex(const Located<std::string> &Text) {
+    std::string Anchored = "^(" + *Text + ")$";
+    llvm::Regex Result(Anchored);
+    std::string RegexError;
+    if (!Result.isValid(RegexError)) {
+      diag(Error, "Invalid regex " + Anchored + ": " + RegexError, Text.Range);
+      return llvm::None;
+    }
+    return Result;
+  }
+
+  void compile(Fragment &&F) {
+    compile(std::move(F.If));
+    compile(std::move(F.CompileFlags));
+  }
+
+  void compile(Fragment::IfBlock &&F) {
+    if (F.HasUnrecognizedCondition)
+      Out.Conditions.push_back([&](const Params &) { return false; });
+
+    auto PathMatch = std::make_unique<std::vector<llvm::Regex>>();
+    for (auto &Entry : F.PathMatch) {
+      if (auto RE = compileRegex(Entry))
+        PathMatch->push_back(std::move(*RE));
+    }
+    if (!PathMatch->empty()) {
+      Out.Conditions.push_back(
+          [PathMatch(std::move(PathMatch))](const Params &P) {
+            if (P.Path.empty())
+              return false;
+            return llvm::any_of(*PathMatch, [&](const llvm::Regex &RE) {
+              return RE.match(P.Path);
+            });
+          });
+    }
+
+    auto PathExclude = std::make_unique<std::vector<llvm::Regex>>();
+    for (auto &Entry : F.PathExclude) {
+      if (auto RE = compileRegex(Entry))
+        PathExclude->push_back(std::move(*RE));
+    }
+    if (!PathExclude->empty()) {
+      Out.Conditions.push_back(
+          [PathExclude(std::move(PathExclude))](const Params &P) {
+            if (P.Path.empty())
+              return false;
+            return llvm::none_of(*PathExclude, [&](const llvm::Regex &RE) {
+              return RE.match(P.Path);
+            });
+          });
+    }
+  }
+
+  void compile(Fragment::CompileFlagsBlock &&F) {
+    if (!F.Add.empty()) {
+      std::vector<std::string> Add;
+      for (auto &A : F.Add)
+        Add.push_back(std::move(*A));
+      Out.Apply.push_back([Add(std::move(Add))](Config &C) {
+        C.CompileFlags.Edits.push_back([Add](std::vector<std::string> &Args) {
+          Args.insert(Args.end(), Add.begin(), Add.end());
+        });
+      });
+    }
+  }
+
+  constexpr static llvm::SourceMgr::DiagKind Error = llvm::SourceMgr::DK_Error;
+  void diag(llvm::SourceMgr::DiagKind Kind, llvm::StringRef Message,
+            llvm::SMRange Range) {
+    if (Range.isValid() && SourceMgr != nullptr)
+      Diagnostic(SourceMgr->GetMessage(Range.Start, Kind, Message, Range));
+    else
+      Diagnostic(llvm::SMDiagnostic("", Kind, Message));
+  }
+};
+
+} // namespace
+
+CompiledFragment Fragment::compile(DiagnosticCallback D) && {
+  llvm::StringRef ConfigFile = "<unknown>";
+  std::pair<unsigned, unsigned> LineCol = {0, 0};
+  if (auto *SM = Source.Manager.get()) {
+    unsigned BufID = SM->getMainFileID();
+    LineCol = SM->getLineAndColumn(Source.Location, BufID);
+    ConfigFile = SM->getBufferInfo(BufID).Buffer->getBufferIdentifier();
+  }
+  trace::Span Tracer("ConfigCompile");
+  SPAN_ATTACH(Tracer, "ConfigFile", ConfigFile);
+  auto Result = std::make_shared<CompiledFragmentImpl>();
+  vlog("Config fragment: compiling {0}:{1} -> {2}", ConfigFile, LineCol.first,
+       Result.get());
+
+  FragmentCompiler{*Result, D, Source.Manager.get()}.compile(std::move(*this));
+  // Return as cheaply-copyable wrapper.
+  return [Result(std::move(Result))](const Params &P, Config &C) {
+    return (*Result)(P, C);
+  };
+}
+
+} // namespace config
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/ConfigFragment.h b/clang-tools-extra/clangd/ConfigFragment.h
new file mode 100644
index 0000000000000..42f9ec2edc724
--- /dev/null
+++ b/clang-tools-extra/clangd/ConfigFragment.h
@@ -0,0 +1,129 @@
+//===--- ConfigFragment.h - Unit of user-specified configuration -*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Various clangd features have configurable behaviour (or can be disabled).
+// The configuration system allows users to control this:
+//  - in a user config file, a project config file, via LSP, or via flags
+//  - specifying different settings for different files
+//
+// This file defines the config::Fragment structure which models one piece of
+// configuration as obtained from a source like a file.
+//
+// This is distinct from how the config is interpreted (CompiledFragment),
+// combined (Provider) and exposed to the rest of clangd (Config).
+//
+//===----------------------------------------------------------------------===//
+//
+// To add a new configuration option, you must:
+//  - add its syntactic form to Fragment
+//  - update ConfigYAML.cpp to parse it
+//  - add its semantic form to Config (in Config.h)
+//  - update ConfigCompile.cpp to map Fragment -> Config
+//  - make use of the option inside clangd
+//  - document the new option (config.md in the llvm/clangd-www repository)
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIGFRAGMENT_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIGFRAGMENT_H
+
+#include "ConfigProvider.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/SMLoc.h"
+#include "llvm/Support/SourceMgr.h"
+#include <string>
+#include <vector>
+
+namespace clang {
+namespace clangd {
+namespace config {
+
+/// An entity written in config along, with its optional location in the file.
+template <typename T> struct Located {
+  Located(T Value, llvm::SMRange Range = {})
+      : Range(Range), Value(std::move(Value)) {}
+
+  llvm::SMRange Range;
+  T &operator->() { return Value; }
+  const T &operator->() const { return Value; }
+  T &operator*() { return Value; }
+  const T &operator*() const { return Value; }
+
+private:
+  T Value;
+};
+
+/// A chunk of configuration obtained from a config file, LSP, or elsewhere.
+struct Fragment {
+  /// Parses fragments from a YAML file (one from each --- delimited document).
+  /// Documents that contained fatal errors are omitted from the results.
+  /// BufferName is used for the SourceMgr and diagnostics.
+  static std::vector<Fragment> parseYAML(llvm::StringRef YAML,
+                                         llvm::StringRef BufferName,
+                                         DiagnosticCallback);
+
+  /// Analyzes and consumes this fragment, possibly yielding more diagnostics.
+  /// This always produces a usable result (errors are recovered).
+  ///
+  /// Typically, providers will compile a Fragment once when it's first loaded,
+  /// caching the result for reuse.
+  /// Like a compiled program, this is good for performance and also encourages
+  /// errors to be reported early and only once.
+  ///
+  /// The returned function is a cheap-copyable wrapper of refcounted internals.
+  CompiledFragment compile(DiagnosticCallback) &&;
+
+  /// These fields are not part of the user-specified configuration, but
+  /// instead are populated by the parser to describe the configuration source.
+  struct SourceInfo {
+    /// Retains a buffer of the original source this fragment was parsed from.
+    /// Locations within Located<T> objects point into this SourceMgr.
+    /// Shared because multiple fragments are often parsed from one (YAML) file.
+    /// May be null, then all locations should be ignored.
+    std::shared_ptr<llvm::SourceMgr> Manager;
+    /// The start of the original source for this fragment.
+    /// Only valid if SourceManager is set.
+    llvm::SMLoc Location;
+  };
+  SourceInfo Source;
+
+  /// Conditions in the If block restrict when a Fragment applies.
+  ///
+  /// Each separate condition must match (combined with AND).
+  /// When one condition has multiple values, any may match (combined with OR).
+  /// e.g. `PathMatch: [foo/.*, bar/.*]` matches files in either directory.
+  ///
+  /// Conditions based on a file's path use the following form:
+  /// - if the fragment came from a project directory, the path is relative
+  /// - if the fragment is global (e.g. user config), the path is absolute
+  /// - paths always use forward-slashes (UNIX-style)
+  /// If no file is being processed, these conditions will not match.
+  struct IfBlock {
+    /// The file being processed must fully match a regular expression.
+    std::vector<Located<std::string>> PathMatch;
+    /// The file being processed must *not* fully match a regular expression.
+    std::vector<Located<std::string>> PathExclude;
+
+    /// An unrecognized key was found while parsing the condition.
+    /// The condition will evaluate to false.
+    bool HasUnrecognizedCondition = false;
+  };
+  IfBlock If;
+
+  struct CompileFlagsBlock {
+    std::vector<Located<std::string>> Add;
+  } CompileFlags;
+};
+
+} // namespace config
+} // namespace clangd
+} // namespace clang
+
+#endif
diff --git a/clang-tools-extra/clangd/ConfigProvider.cpp b/clang-tools-extra/clangd/ConfigProvider.cpp
new file mode 100644
index 0000000000000..4b466d53e2930
--- /dev/null
+++ b/clang-tools-extra/clangd/ConfigProvider.cpp
@@ -0,0 +1,207 @@
+//===--- ConfigProvider.cpp - Loading of user configuration ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ConfigProvider.h"
+#include "Config.h"
+#include "ConfigFragment.h"
+#include "support/ThreadsafeFS.h"
+#include "support/Trace.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/Path.h"
+#include <mutex>
+
+namespace clang {
+namespace clangd {
+namespace config {
+
+// Threadsafe cache around reading a YAML config file from disk.
+class FileConfigCache {
+  std::mutex Mu;
+  llvm::SmallVector<CompiledFragment, 1> CachedValue;
+  llvm::sys::TimePoint<> MTime = {};
+  unsigned Size = -1;
+
+  void updateCacheLocked(const llvm::vfs::Status &Stat,
+                         llvm::vfs::FileSystem &FS, DiagnosticCallback DC) {
+    if (Size == Stat.getSize() && MTime == Stat.getLastModificationTime())
+      return; // Already valid.
+
+    Size = Stat.getSize();
+    MTime = Stat.getLastModificationTime();
+    CachedValue.clear();
+
+    auto Buf = FS.getBufferForFile(Path);
+    // If stat() succeeds but we failed to read, don't cache failure.
+    if (!Buf) {
+      Size = -1;
+      MTime = {};
+      return;
+    }
+
+    // If file changed between stat and open, we don't know its mtime.
+    // For simplicity, don't cache the value in this case (use a bad key).
+    if (Buf->get()->getBufferSize() != Size) {
+      Size = -1;
+      MTime = {};
+    }
+
+    // Finally parse and compile the actual fragments.
+    for (auto &Fragment :
+         Fragment::parseYAML(Buf->get()->getBuffer(), Path, DC))
+      CachedValue.push_back(std::move(Fragment).compile(DC));
+  }
+
+public:
+  // Must be set before the cache is used. Not a constructor param to allow
+  // computing ancestor-relative paths to be deferred.
+  std::string Path;
+
+  // Retrieves up-to-date config fragments from disk.
+  // A cached result may be reused if the mtime and size are unchanged.
+  // (But several concurrent read()s can miss the cache after a single change).
+  // Future performance ideas:
+  // - allow caches to be reused based on short elapsed walltime
+  // - allow latency-sensitive operations to skip revalidating the cache
+  void read(const ThreadsafeFS &TFS, DiagnosticCallback DC,
+            std::vector<CompiledFragment> &Out) {
+    assert(llvm::sys::path::is_absolute(Path));
+    auto FS = TFS.view(/*CWD=*/llvm::None);
+    auto Stat = FS->status(Path);
+    if (!Stat || !Stat->isRegularFile()) {
+      // No point taking the lock to clear the cache. We know what to return.
+      // If the file comes back we'll invalidate the cache at that point.
+      return;
+    }
+
+    std::lock_guard<std::mutex> Lock(Mu);
+    updateCacheLocked(*Stat, *FS, DC);
+    llvm::copy(CachedValue, std::back_inserter(Out));
+  }
+};
+
+std::unique_ptr<Provider> Provider::fromYAMLFile(llvm::StringRef AbsPath,
+                                                 const ThreadsafeFS &FS) {
+  class AbsFileProvider : public Provider {
+    mutable FileConfigCache Cache; // threadsafe
+    const ThreadsafeFS &FS;
+
+    std::vector<CompiledFragment>
+    getFragments(const Params &P, DiagnosticCallback DC) const override {
+      std::vector<CompiledFragment> Result;
+      Cache.read(FS, DC, Result);
+      return Result;
+    };
+
+  public:
+    AbsFileProvider(llvm::StringRef Path, const ThreadsafeFS &FS) : FS(FS) {
+      assert(llvm::sys::path::is_absolute(Path));
+      Cache.Path = Path.str();
+    }
+  };
+
+  return std::make_unique<AbsFileProvider>(AbsPath, FS);
+}
+
+std::unique_ptr<Provider>
+Provider::fromAncestorRelativeYAMLFiles(llvm::StringRef RelPath,
+                                        const ThreadsafeFS &FS) {
+  class RelFileProvider : public Provider {
+    std::string RelPath;
+    const ThreadsafeFS &FS;
+
+    mutable std::mutex Mu;
+    // Keys are the ancestor directory, not the actual config path within it.
+    // We only insert into this map, so pointers to values are stable forever.
+    // Mutex guards the map itself, not the values (which are threadsafe).
+    mutable llvm::StringMap<FileConfigCache> Cache;
+
+    std::vector<CompiledFragment>
+    getFragments(const Params &P, DiagnosticCallback DC) const override {
+      namespace path = llvm::sys::path;
+
+      if (P.Path.empty())
+        return {};
+
+      // Compute absolute paths to all ancestors (substrings of P.Path).
+      llvm::StringRef Parent = path::parent_path(P.Path);
+      llvm::SmallVector<llvm::StringRef, 8> Ancestors;
+      for (auto I = path::begin(Parent, path::Style::posix),
+                E = path::end(Parent);
+           I != E; ++I) {
+        // Avoid weird non-substring cases like phantom "." components.
+        // In practice, Component is a substring for all "normal" ancestors.
+        if (I->end() < Parent.begin() && I->end() > Parent.end())
+          continue;
+        Ancestors.emplace_back(Parent.begin(), I->end() - Parent.begin());
+      }
+      // Ensure corresponding cache entries exist in the map.
+      llvm::SmallVector<FileConfigCache *, 8> Caches;
+      {
+        std::lock_guard<std::mutex> Lock(Mu);
+        for (llvm::StringRef Ancestor : Ancestors) {
+          auto R = Cache.try_emplace(Ancestor);
+          // Assemble the actual config file path only once.
+          if (R.second) {
+            llvm::SmallString<256> ConfigPath = Ancestor;
+            path::append(ConfigPath, RelPath);
+            R.first->second.Path = ConfigPath.str().str();
+          }
+          Caches.push_back(&R.first->second);
+        }
+      }
+      // Finally query each individual file.
+      // This will take a (per-file) lock for each file that actually exists.
+      std::vector<CompiledFragment> Result;
+      for (FileConfigCache *Cache : Caches)
+        Cache->read(FS, DC, Result);
+      return Result;
+    };
+
+  public:
+    RelFileProvider(llvm::StringRef RelPath, const ThreadsafeFS &FS)
+        : RelPath(RelPath), FS(FS) {
+      assert(llvm::sys::path::is_relative(RelPath));
+    }
+  };
+
+  return std::make_unique<RelFileProvider>(RelPath, FS);
+}
+
+std::unique_ptr<Provider>
+Provider::combine(std::vector<std::unique_ptr<Provider>> Providers) {
+  struct CombinedProvider : Provider {
+    std::vector<std::unique_ptr<Provider>> Providers;
+
+    std::vector<CompiledFragment>
+    getFragments(const Params &P, DiagnosticCallback DC) const override {
+      std::vector<CompiledFragment> Result;
+      for (const auto &Provider : Providers) {
+        for (auto &Fragment : Provider->getFragments(P, DC))
+          Result.push_back(std::move(Fragment));
+      }
+      return Result;
+    }
+  };
+  auto Result = std::make_unique<CombinedProvider>();
+  Result->Providers = std::move(Providers);
+  return Result;
+}
+
+Config Provider::getConfig(const Params &P, DiagnosticCallback DC) const {
+  trace::Span Tracer("getConfig");
+  if (!P.Path.empty())
+    SPAN_ATTACH(Tracer, "path", P.Path);
+  Config C;
+  for (const auto &Fragment : getFragments(P, DC))
+    Fragment(P, C);
+  return C;
+}
+
+} // namespace config
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/ConfigProvider.h b/clang-tools-extra/clangd/ConfigProvider.h
new file mode 100644
index 0000000000000..a773e56b3bd72
--- /dev/null
+++ b/clang-tools-extra/clangd/ConfigProvider.h
@@ -0,0 +1,97 @@
+//===--- ConfigProvider.h - Loading of user configuration --------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Various clangd features have configurable behaviour (or can be disabled).
+// The configuration system allows users to control this:
+//  - in a user config file, a project config file, via LSP, or via flags
+//  - specifying different settings for different files
+// This file defines the structures used for this, that produce a Config.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIGPROVIDER_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIGPROVIDER_H
+
+#include "llvm/ADT/FunctionExtras.h"
+#include "llvm/Support/SMLoc.h"
+#include "llvm/Support/SourceMgr.h"
+#include <string>
+#include <vector>
+
+namespace clang {
+namespace clangd {
+struct Config;
+class ThreadsafeFS;
+namespace config {
+
+/// Describes the context used to evaluate configuration fragments.
+struct Params {
+  /// Absolute path to a source file we're applying the config to. Unix slashes.
+  /// Empty if not configuring a particular file.
+  llvm::StringRef Path;
+};
+
+/// Used to report problems in parsing or interpreting a config.
+/// Errors reflect structurally invalid config that should be user-visible.
+/// Warnings reflect e.g. unknown properties that are recoverable.
+using DiagnosticCallback = llvm::function_ref<void(const llvm::SMDiagnostic &)>;
+
+/// A chunk of configuration that has been fully analyzed and is ready to apply.
+/// Typically this is obtained from a Fragment by calling Fragment::compile().
+///
+/// Calling it updates the configuration to reflect settings from the fragment.
+/// Returns true if the condition was met and the settings were used.
+using CompiledFragment = std::function<bool(const Params &, Config &)>;
+
+/// A source of configuration fragments.
+/// Generally these providers reflect a fixed policy for obtaining config,
+/// but return different concrete configuration over time.
+/// e.g. a provider that reads config from files is responsive to file changes.
+class Provider {
+public:
+  virtual ~Provider() = default;
+
+  // Reads fragments from a single YAML file with a fixed path.
+  static std::unique_ptr<Provider> fromYAMLFile(llvm::StringRef AbsPathPath,
+                                                const ThreadsafeFS &);
+  // Reads fragments from YAML files found relative to ancestors of Params.Path.
+  //
+  // All fragments that exist are returned, starting from distant ancestors.
+  // For instance, given RelPath of ".clangd", then for source file /foo/bar.cc,
+  // the searched fragments are [/.clangd, /foo/.clangd].
+  //
+  // If Params does not specify a path, no fragments are returned.
+  static std::unique_ptr<Provider>
+  fromAncestorRelativeYAMLFiles(llvm::StringRef RelPath, const ThreadsafeFS &);
+
+  /// A provider that includes fragments from all the supplied providers.
+  /// Order is preserved; later providers take precedence over earlier ones.
+  static std::unique_ptr<Provider>
+      combine(std::vector<std::unique_ptr<Provider>>);
+
+  /// Build a config based on this provider.
+  Config getConfig(const Params &, DiagnosticCallback) const;
+
+private:
+  /// Provide fragments that may be relevant to the file.
+  /// The configuration provider is not responsible for testing conditions.
+  ///
+  /// Providers are expected to cache compiled fragments, and only
+  /// reparse/recompile when the source data has changed.
+  /// Despite the need for caching, this function must be threadsafe.
+  ///
+  /// When parsing/compiling, the DiagnosticCallback is used to report errors.
+  virtual std::vector<CompiledFragment>
+  getFragments(const Params &, DiagnosticCallback) const = 0;
+};
+
+} // namespace config
+} // namespace clangd
+} // namespace clang
+
+#endif
diff --git a/clang-tools-extra/clangd/ConfigYAML.cpp b/clang-tools-extra/clangd/ConfigYAML.cpp
new file mode 100644
index 0000000000000..ef6003b024392
--- /dev/null
+++ b/clang-tools-extra/clangd/ConfigYAML.cpp
@@ -0,0 +1,218 @@
+//===--- ConfigYAML.cpp - Loading configuration fragments from YAML files -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ConfigFragment.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/YAMLParser.h"
+#include <system_error>
+
+namespace clang {
+namespace clangd {
+namespace config {
+namespace {
+using llvm::yaml::BlockScalarNode;
+using llvm::yaml::MappingNode;
+using llvm::yaml::Node;
+using llvm::yaml::ScalarNode;
+using llvm::yaml::SequenceNode;
+
+class Parser {
+  llvm::SourceMgr &SM;
+  bool HadError = false;
+
+public:
+  Parser(llvm::SourceMgr &SM) : SM(SM) {}
+
+  // Tries to parse N into F, returning false if it failed and we couldn't
+  // meaningfully recover (YAML syntax error, or hard semantic error).
+  bool parse(Fragment &F, Node &N) {
+    DictParser Dict("Config", this);
+    Dict.handle("If", [&](Node &N) { parse(F.If, N); });
+    Dict.handle("CompileFlags", [&](Node &N) { parse(F.CompileFlags, N); });
+    Dict.parse(N);
+    return !(N.failed() || HadError);
+  }
+
+private:
+  void parse(Fragment::IfBlock &F, Node &N) {
+    DictParser Dict("If", this);
+    Dict.unrecognized(
+        [&](llvm::StringRef) { F.HasUnrecognizedCondition = true; });
+    Dict.handle("PathMatch", [&](Node &N) {
+      if (auto Values = scalarValues(N))
+        F.PathMatch = std::move(*Values);
+    });
+    Dict.handle("PathExclude", [&](Node &N) {
+      if (auto Values = scalarValues(N))
+        F.PathExclude = std::move(*Values);
+    });
+    Dict.parse(N);
+  }
+
+  void parse(Fragment::CompileFlagsBlock &F, Node &N) {
+    DictParser Dict("CompileFlags", this);
+    Dict.handle("Add", [&](Node &N) {
+      if (auto Values = scalarValues(N))
+        F.Add = std::move(*Values);
+    });
+    Dict.parse(N);
+  }
+
+  // Helper for parsing mapping nodes (dictionaries).
+  // We don't use YamlIO as we want to control over unknown keys.
+  class DictParser {
+    llvm::StringRef Description;
+    std::vector<std::pair<llvm::StringRef, std::function<void(Node &)>>> Keys;
+    std::function<void(llvm::StringRef)> Unknown;
+    Parser *Outer;
+
+  public:
+    DictParser(llvm::StringRef Description, Parser *Outer)
+        : Description(Description), Outer(Outer) {}
+
+    // Parse is called when Key is encountered, and passed the associated value.
+    // It should emit diagnostics if the value is invalid (e.g. wrong type).
+    // If Key is seen twice, Parse runs only once and an error is reported.
+    void handle(llvm::StringLiteral Key, std::function<void(Node &)> Parse) {
+      for (const auto &Entry : Keys) {
+        (void) Entry;
+        assert(Entry.first != Key && "duplicate key handler");
+      }
+      Keys.emplace_back(Key, std::move(Parse));
+    }
+
+    // Fallback is called when a Key is not matched by any handle().
+    // A warning is also automatically emitted.
+    void unrecognized(std::function<void(llvm::StringRef)> Fallback) {
+      Unknown = std::move(Fallback);
+    }
+
+    // Process a mapping node and call handlers for each key/value pair.
+    void parse(Node &N) const {
+      if (N.getType() != Node::NK_Mapping) {
+        Outer->error(Description + " should be a dictionary", N);
+        return;
+      }
+      llvm::SmallSet<std::string, 8> Seen;
+      // We *must* consume all items, even on error, or the parser will assert.
+      for (auto &KV : llvm::cast<MappingNode>(N)) {
+        auto *K = KV.getKey();
+        if (!K) // YAMLParser emitted an error.
+          continue;
+        auto Key = Outer->scalarValue(*K, "Dictionary key");
+        if (!Key)
+          continue;
+        if (!Seen.insert(**Key).second) {
+          Outer->warning("Duplicate key " + **Key + " is ignored", *K);
+          continue;
+        }
+        auto *Value = KV.getValue();
+        if (!Value) // YAMLParser emitted an error.
+          continue;
+        bool Matched = false;
+        for (const auto &Handler : Keys) {
+          if (Handler.first == **Key) {
+            Matched = true;
+            Handler.second(*Value);
+            break;
+          }
+        }
+        if (!Matched) {
+          Outer->warning("Unknown " + Description + " key " + **Key, *K);
+          if (Unknown)
+            Unknown(**Key);
+        }
+      }
+    }
+  };
+
+  // Try to parse a single scalar value from the node, warn on failure.
+  llvm::Optional<Located<std::string>> scalarValue(Node &N,
+                                                   llvm::StringRef Desc) {
+    llvm::SmallString<256> Buf;
+    if (auto *S = llvm::dyn_cast<ScalarNode>(&N))
+      return Located<std::string>(S->getValue(Buf).str(), N.getSourceRange());
+    if (auto *BS = llvm::dyn_cast<BlockScalarNode>(&N))
+      return Located<std::string>(BS->getValue().str(), N.getSourceRange());
+    warning(Desc + " should be scalar", N);
+    return llvm::None;
+  }
+
+  // Try to parse a list of single scalar values, or just a single value.
+  llvm::Optional<std::vector<Located<std::string>>> scalarValues(Node &N) {
+    std::vector<Located<std::string>> Result;
+    if (auto *S = llvm::dyn_cast<ScalarNode>(&N)) {
+      llvm::SmallString<256> Buf;
+      Result.emplace_back(S->getValue(Buf).str(), N.getSourceRange());
+    } else if (auto *S = llvm::dyn_cast<BlockScalarNode>(&N)) {
+      Result.emplace_back(S->getValue().str(), N.getSourceRange());
+    } else if (auto *S = llvm::dyn_cast<SequenceNode>(&N)) {
+      // We *must* consume all items, even on error, or the parser will assert.
+      for (auto &Child : *S) {
+        if (auto Value = scalarValue(Child, "List item"))
+          Result.push_back(std::move(*Value));
+      }
+    } else {
+      warning("Expected scalar or list of scalars", N);
+      return llvm::None;
+    }
+    return Result;
+  }
+
+  // Report a "hard" error, reflecting a config file that can never be valid.
+  void error(const llvm::Twine &Msg, const Node &N) {
+    HadError = true;
+    SM.PrintMessage(N.getSourceRange().Start, llvm::SourceMgr::DK_Error, Msg,
+                    N.getSourceRange());
+  }
+
+  // Report a "soft" error that could be caused by e.g. version skew.
+  void warning(const llvm::Twine &Msg, const Node &N) {
+    SM.PrintMessage(N.getSourceRange().Start, llvm::SourceMgr::DK_Warning, Msg,
+                    N.getSourceRange());
+  }
+};
+
+} // namespace
+
+std::vector<Fragment> Fragment::parseYAML(llvm::StringRef YAML,
+                                          llvm::StringRef BufferName,
+                                          DiagnosticCallback Diags) {
+  // The YAML document may contain multiple conditional fragments.
+  // The SourceManager is shared for all of them.
+  auto SM = std::make_shared<llvm::SourceMgr>();
+  auto Buf = llvm::MemoryBuffer::getMemBufferCopy(YAML, BufferName);
+  // Adapt DiagnosticCallback to function-pointer interface.
+  // Callback receives both errors we emit and those from the YAML parser.
+  SM->setDiagHandler(
+      [](const llvm::SMDiagnostic &Diag, void *Ctx) {
+        (*reinterpret_cast<DiagnosticCallback *>(Ctx))(Diag);
+      },
+      &Diags);
+  std::vector<Fragment> Result;
+  for (auto &Doc : llvm::yaml::Stream(*Buf, *SM)) {
+    if (Node *N = Doc.getRoot()) {
+      Fragment Fragment;
+      Fragment.Source.Manager = SM;
+      Fragment.Source.Location = N->getSourceRange().Start;
+      if (Parser(*SM).parse(Fragment, *N))
+        Result.push_back(std::move(Fragment));
+    }
+  }
+  // Hack: stash the buffer in the SourceMgr to keep it alive.
+  // SM has two entries: "main" non-owning buffer, and ignored owning buffer.
+  SM->AddNewSourceBuffer(std::move(Buf), llvm::SMLoc());
+  return Result;
+}
+
+} // namespace config
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/FindTarget.cpp b/clang-tools-extra/clangd/FindTarget.cpp
index 1d09e8408c837..627f40c854368 100644
--- a/clang-tools-extra/clangd/FindTarget.cpp
+++ b/clang-tools-extra/clangd/FindTarget.cpp
@@ -58,6 +58,24 @@ nodeToString(const ast_type_traits::DynTypedNode &N) {
   return S;
 }
 
+// Helper function for getMembersReferencedViaDependentName()
+// which takes a dependent type `T` and heuristically
+// resolves it to a CXXRecordDecl in which we can try name lookup.
+CXXRecordDecl *resolveTypeToRecordDecl(const Type *T) {
+  assert(T);
+  if (const auto *ICNT = T->getAs<InjectedClassNameType>()) {
+    T = ICNT->getInjectedSpecializationType().getTypePtrOrNull();
+  }
+  const auto *TST = T->getAs<TemplateSpecializationType>();
+  if (!TST)
+    return nullptr;
+  const ClassTemplateDecl *TD = dyn_cast_or_null<ClassTemplateDecl>(
+      TST->getTemplateName().getAsTemplateDecl());
+  if (!TD)
+    return nullptr;
+  return TD->getTemplatedDecl();
+}
+
 // Given a dependent type and a member name, heuristically resolve the
 // name to one or more declarations.
 // The current heuristic is simply to look up the name in the primary
@@ -82,25 +100,17 @@ std::vector<const NamedDecl *> getMembersReferencedViaDependentName(
         ET->getDecl()->lookup(NameFactory(ET->getDecl()->getASTContext()));
     return {Result.begin(), Result.end()};
   }
-  if (auto *ICNT = T->getAs<InjectedClassNameType>()) {
-    T = ICNT->getInjectedSpecializationType().getTypePtrOrNull();
+  if (auto *RD = resolveTypeToRecordDecl(T)) {
+    if (!RD->hasDefinition())
+      return {};
+    RD = RD->getDefinition();
+    DeclarationName Name = NameFactory(RD->getASTContext());
+    return RD->lookupDependentName(Name, [=](const NamedDecl *D) {
+      return IsNonstaticMember ? D->isCXXInstanceMember()
+                               : !D->isCXXInstanceMember();
+    });
   }
-  auto *TST = T->getAs<TemplateSpecializationType>();
-  if (!TST)
-    return {};
-  const ClassTemplateDecl *TD = dyn_cast_or_null<ClassTemplateDecl>(
-      TST->getTemplateName().getAsTemplateDecl());
-  if (!TD)
-    return {};
-  CXXRecordDecl *RD = TD->getTemplatedDecl();
-  if (!RD->hasDefinition())
-    return {};
-  RD = RD->getDefinition();
-  DeclarationName Name = NameFactory(RD->getASTContext());
-  return RD->lookupDependentName(Name, [=](const NamedDecl *D) {
-    return IsNonstaticMember ? D->isCXXInstanceMember()
-                             : !D->isCXXInstanceMember();
-  });
+  return {};
 }
 
 // Given the type T of a dependent expression that appears of the LHS of a "->",
@@ -144,6 +154,28 @@ const Type *getPointeeType(const Type *T) {
   return FirstArg.getAsType().getTypePtrOrNull();
 }
 
+// Try to heuristically resolve a dependent expression `E` to one
+// or more declarations that it likely references.
+std::vector<const NamedDecl *> resolveDependentExprToDecls(const Expr *E) {
+  assert(E->isTypeDependent());
+  if (const auto *ME = dyn_cast<CXXDependentScopeMemberExpr>(E)) {
+    const Type *BaseType = ME->getBaseType().getTypePtrOrNull();
+    if (ME->isArrow()) {
+      BaseType = getPointeeType(BaseType);
+    }
+    return getMembersReferencedViaDependentName(
+        BaseType, [ME](ASTContext &) { return ME->getMember(); },
+        /*IsNonstaticMember=*/true);
+  }
+  if (const auto *RE = dyn_cast<DependentScopeDeclRefExpr>(E)) {
+    return getMembersReferencedViaDependentName(
+        RE->getQualifier()->getAsType(),
+        [RE](ASTContext &) { return RE->getDeclName(); },
+        /*IsNonstaticMember=*/false);
+  }
+  return {};
+}
+
 const NamedDecl *getTemplatePattern(const NamedDecl *D) {
   if (const CXXRecordDecl *CRD = dyn_cast<CXXRecordDecl>(D)) {
     if (const auto *Result = CRD->getTemplateInstantiationPattern())
@@ -341,21 +373,12 @@ struct TargetFinder {
       }
       void
       VisitCXXDependentScopeMemberExpr(const CXXDependentScopeMemberExpr *E) {
-        const Type *BaseType = E->getBaseType().getTypePtrOrNull();
-        if (E->isArrow()) {
-          BaseType = getPointeeType(BaseType);
-        }
-        for (const NamedDecl *D : getMembersReferencedViaDependentName(
-                 BaseType, [E](ASTContext &) { return E->getMember(); },
-                 /*IsNonstaticMember=*/true)) {
+        for (const NamedDecl *D : resolveDependentExprToDecls(E)) {
           Outer.add(D, Flags);
         }
       }
       void VisitDependentScopeDeclRefExpr(const DependentScopeDeclRefExpr *E) {
-        for (const NamedDecl *D : getMembersReferencedViaDependentName(
-                 E->getQualifier()->getAsType(),
-                 [E](ASTContext &) { return E->getDeclName(); },
-                 /*IsNonstaticMember=*/false)) {
+        for (const NamedDecl *D : resolveDependentExprToDecls(E)) {
           Outer.add(D, Flags);
         }
       }
diff --git a/clang-tools-extra/clangd/Hover.cpp b/clang-tools-extra/clangd/Hover.cpp
index 2861d63d4d008..8305a47240356 100644
--- a/clang-tools-extra/clangd/Hover.cpp
+++ b/clang-tools-extra/clangd/Hover.cpp
@@ -124,8 +124,8 @@ std::string printType(QualType QT, const PrintingPolicy &Policy) {
   // TypePrinter doesn't resolve decltypes, so resolve them here.
   // FIXME: This doesn't handle composite types that contain a decltype in them.
   // We should rather have a printing policy for that.
-  while (const auto *DT = QT->getAs<DecltypeType>())
-    QT = DT->getUnderlyingType();
+  while (!QT.isNull() && QT->isDecltypeType())
+    QT = QT->getAs<DecltypeType>()->getUnderlyingType();
   return QT.getAsString(Policy);
 }
 
@@ -289,31 +289,27 @@ const Expr *getDefaultArg(const ParmVarDecl *PVD) {
                                             : PVD->getDefaultArg();
 }
 
+HoverInfo::Param toHoverInfoParam(const ParmVarDecl *PVD,
+                                  const PrintingPolicy &Policy) {
+  HoverInfo::Param Out;
+  Out.Type = printType(PVD->getType(), Policy);
+  if (!PVD->getName().empty())
+    Out.Name = PVD->getNameAsString();
+  if (const Expr *DefArg = getDefaultArg(PVD)) {
+    Out.Default.emplace();
+    llvm::raw_string_ostream OS(*Out.Default);
+    DefArg->printPretty(OS, nullptr, Policy);
+  }
+  return Out;
+}
+
 // Populates Type, ReturnType, and Parameters for function-like decls.
 void fillFunctionTypeAndParams(HoverInfo &HI, const Decl *D,
                                const FunctionDecl *FD,
                                const PrintingPolicy &Policy) {
   HI.Parameters.emplace();
-  for (const ParmVarDecl *PVD : FD->parameters()) {
-    HI.Parameters->emplace_back();
-    auto &P = HI.Parameters->back();
-    if (!PVD->getType().isNull()) {
-      P.Type = printType(PVD->getType(), Policy);
-    } else {
-      std::string Param;
-      llvm::raw_string_ostream OS(Param);
-      PVD->dump(OS);
-      OS.flush();
-      elog("Got param with null type: {0}", Param);
-    }
-    if (!PVD->getName().empty())
-      P.Name = PVD->getNameAsString();
-    if (const Expr *DefArg = getDefaultArg(PVD)) {
-      P.Default.emplace();
-      llvm::raw_string_ostream Out(*P.Default);
-      DefArg->printPretty(Out, nullptr, Policy);
-    }
-  }
+  for (const ParmVarDecl *PVD : FD->parameters())
+    HI.Parameters->emplace_back(toHoverInfoParam(PVD, Policy));
 
   // We don't want any type info, if name already contains it. This is true for
   // constructors/destructors and conversion operators.
@@ -333,15 +329,28 @@ void fillFunctionTypeAndParams(HoverInfo &HI, const Decl *D,
 
 llvm::Optional<std::string> printExprValue(const Expr *E,
                                            const ASTContext &Ctx) {
-  Expr::EvalResult Constant;
+  // InitListExpr has two forms, syntactic and semantic. They are the same thing
+  // (refer to a same AST node) in most cases.
+  // When they are different, RAV returns the syntactic form, and we should feed
+  // the semantic form to EvaluateAsRValue.
+  if (const auto *ILE = llvm::dyn_cast<InitListExpr>(E)) {
+    if (!ILE->isSemanticForm())
+      E = ILE->getSemanticForm();
+  }
+
   // Evaluating [[foo]]() as "&foo" isn't useful, and prevents us walking up
   // to the enclosing call.
   QualType T = E->getType();
   if (T.isNull() || T->isFunctionType() || T->isFunctionPointerType() ||
       T->isFunctionReferenceType())
     return llvm::None;
+
+  Expr::EvalResult Constant;
   // Attempt to evaluate. If expr is dependent, evaluation crashes!
-  if (E->isValueDependent() || !E->EvaluateAsRValue(Constant, Ctx))
+  if (E->isValueDependent() || !E->EvaluateAsRValue(Constant, Ctx) ||
+      // Disable printing for record-types, as they are usually confusing and
+      // might make clang crash while printing the expressions.
+      Constant.Val.isStruct() || Constant.Val.isUnion())
     return llvm::None;
 
   // Show enums symbolically, not numerically like APValue::printPretty().
@@ -353,7 +362,7 @@ llvm::Optional<std::string> printExprValue(const Expr *E,
       if (ECD->getInitVal() == Val)
         return llvm::formatv("{0} ({1})", ECD->getNameAsString(), Val).str();
   }
-  return Constant.Val.getAsString(Ctx, E->getType());
+  return Constant.Val.getAsString(Ctx, T);
 }
 
 llvm::Optional<std::string> printExprValue(const SelectionTree::Node *N,
@@ -660,8 +669,10 @@ bool isHardLineBreakAfter(llvm::StringRef Line, llvm::StringRef Rest) {
 }
 
 void addLayoutInfo(const NamedDecl &ND, HoverInfo &HI) {
-  const auto &Ctx = ND.getASTContext();
+  if (ND.isInvalidDecl())
+    return;
 
+  const auto &Ctx = ND.getASTContext();
   if (auto *RD = llvm::dyn_cast<RecordDecl>(&ND)) {
     if (auto Size = Ctx.getTypeSizeInCharsIfKnown(RD->getTypeForDecl()))
       HI.Size = Size->getQuantity();
@@ -672,17 +683,101 @@ void addLayoutInfo(const NamedDecl &ND, HoverInfo &HI) {
     const auto *Record = FD->getParent();
     if (Record)
       Record = Record->getDefinition();
-    if (Record && !Record->isDependentType()) {
-      uint64_t OffsetBits = Ctx.getFieldOffset(FD);
-      if (auto Size = Ctx.getTypeSizeInCharsIfKnown(FD->getType())) {
+    if (Record && !Record->isInvalidDecl() && !Record->isDependentType()) {
+      HI.Offset = Ctx.getFieldOffset(FD) / 8;
+      if (auto Size = Ctx.getTypeSizeInCharsIfKnown(FD->getType()))
         HI.Size = Size->getQuantity();
-        HI.Offset = OffsetBits / 8;
-      }
     }
     return;
   }
 }
 
+// If N is passed as argument to a function, fill HI.CalleeArgInfo with
+// information about that argument.
+void maybeAddCalleeArgInfo(const SelectionTree::Node *N, HoverInfo &HI,
+                           const PrintingPolicy &Policy) {
+  const auto &OuterNode = N->outerImplicit();
+  if (!OuterNode.Parent)
+    return;
+  const auto *CE = OuterNode.Parent->ASTNode.get<CallExpr>();
+  if (!CE)
+    return;
+  const FunctionDecl *FD = CE->getDirectCallee();
+  // For non-function-call-like operatators (e.g. operator+, operator<<) it's
+  // not immediattely obvious what the "passed as" would refer to and, given
+  // fixed function signature, the value would be very low anyway, so we choose
+  // to not support that.
+  // Both variadic functions and operator() (especially relevant for lambdas)
+  // should be supported in the future.
+  if (!FD || FD->isOverloadedOperator() || FD->isVariadic())
+    return;
+
+  // Find argument index for N.
+  for (unsigned I = 0; I < CE->getNumArgs() && I < FD->getNumParams(); ++I) {
+    if (CE->getArg(I) != OuterNode.ASTNode.get<Expr>())
+      continue;
+
+    // Extract matching argument from function declaration.
+    if (const ParmVarDecl *PVD = FD->getParamDecl(I))
+      HI.CalleeArgInfo.emplace(toHoverInfoParam(PVD, Policy));
+    break;
+  }
+  if (!HI.CalleeArgInfo)
+    return;
+
+  // If we found a matching argument, also figure out if it's a
+  // [const-]reference. For this we need to walk up the AST from the arg itself
+  // to CallExpr and check all implicit casts, constructor calls, etc.
+  HoverInfo::PassType PassType;
+  if (const auto *E = N->ASTNode.get<Expr>()) {
+    if (E->getType().isConstQualified())
+      PassType.PassBy = HoverInfo::PassType::ConstRef;
+  }
+
+  for (auto *CastNode = N->Parent;
+       CastNode != OuterNode.Parent && !PassType.Converted;
+       CastNode = CastNode->Parent) {
+    if (const auto *ImplicitCast = CastNode->ASTNode.get<ImplicitCastExpr>()) {
+      switch (ImplicitCast->getCastKind()) {
+      case CK_NoOp:
+      case CK_DerivedToBase:
+      case CK_UncheckedDerivedToBase:
+        // If it was a reference before, it's still a reference.
+        if (PassType.PassBy != HoverInfo::PassType::Value)
+          PassType.PassBy = ImplicitCast->getType().isConstQualified()
+                                ? HoverInfo::PassType::ConstRef
+                                : HoverInfo::PassType::Ref;
+        break;
+      case CK_LValueToRValue:
+      case CK_ArrayToPointerDecay:
+      case CK_FunctionToPointerDecay:
+      case CK_NullToPointer:
+      case CK_NullToMemberPointer:
+        // No longer a reference, but we do not show this as type conversion.
+        PassType.PassBy = HoverInfo::PassType::Value;
+        break;
+      default:
+        PassType.PassBy = HoverInfo::PassType::Value;
+        PassType.Converted = true;
+        break;
+      }
+    } else if (const auto *CtorCall =
+                   CastNode->ASTNode.get<CXXConstructExpr>()) {
+      // We want to be smart about copy constructors. They should not show up as
+      // type conversion, but instead as passing by value.
+      if (CtorCall->getConstructor()->isCopyConstructor())
+        PassType.PassBy = HoverInfo::PassType::Value;
+      else
+        PassType.Converted = true;
+    } else { // Unknown implicit node, assume type conversion.
+      PassType.PassBy = HoverInfo::PassType::Value;
+      PassType.Converted = true;
+    }
+  }
+
+  HI.CallPassType.emplace(PassType);
+}
+
 } // namespace
 
 llvm::Optional<HoverInfo> getHover(ParsedAST &AST, Position Pos,
@@ -745,6 +840,7 @@ llvm::Optional<HoverInfo> getHover(ParsedAST &AST, Position Pos,
         // Look for a close enclosing expression to show the value of.
         if (!HI->Value)
           HI->Value = printExprValue(N, AST.getASTContext());
+        maybeAddCalleeArgInfo(N, *HI, AST.getASTContext().getPrintingPolicy());
       } else if (const Expr *E = N->ASTNode.get<Expr>()) {
         HI = getHoverContents(E, AST);
       }
@@ -825,6 +921,24 @@ markup::Document HoverInfo::present() const {
     Output.addParagraph().appendText(
         llvm::formatv("Size: {0} byte{1}", *Size, *Size == 1 ? "" : "s").str());
 
+  if (CalleeArgInfo) {
+    assert(CallPassType);
+    std::string Buffer;
+    llvm::raw_string_ostream OS(Buffer);
+    OS << "Passed ";
+    if (CallPassType->PassBy != HoverInfo::PassType::Value) {
+      OS << "by ";
+      if (CallPassType->PassBy == HoverInfo::PassType::ConstRef)
+        OS << "const ";
+      OS << "reference ";
+    }
+    if (CalleeArgInfo->Name)
+      OS << "as " << CalleeArgInfo->Name;
+    if (CallPassType->Converted && CalleeArgInfo->Type)
+      OS << " (converted to " << CalleeArgInfo->Type << ")";
+    Output.addParagraph().appendText(OS.str());
+  }
+
   if (!Documentation.empty())
     parseDocumentation(Documentation, Output);
 
@@ -849,6 +963,7 @@ markup::Document HoverInfo::present() const {
     // non-c++ projects or projects that are not making use of namespaces.
     Output.addCodeBlock(ScopeComment + DefinitionWithAccess);
   }
+
   return Output;
 }
 
diff --git a/clang-tools-extra/clangd/Hover.h b/clang-tools-extra/clangd/Hover.h
index b712d844e33d0..2f2afbf6723bf 100644
--- a/clang-tools-extra/clangd/Hover.h
+++ b/clang-tools-extra/clangd/Hover.h
@@ -77,11 +77,31 @@ struct HoverInfo {
   llvm::Optional<uint64_t> Size;
   /// Contains the offset of fields within the enclosing class.
   llvm::Optional<uint64_t> Offset;
+  // Set when symbol is inside function call. Contains information extracted
+  // from the callee definition about the argument this is passed as.
+  llvm::Optional<Param> CalleeArgInfo;
+  struct PassType {
+    // How the variable is passed to callee.
+    enum PassMode { Ref, ConstRef, Value };
+    PassMode PassBy = Ref;
+    // True if type conversion happened. This includes calls to implicit
+    // constructor, as well as built-in type conversions. Casting to base class
+    // is not considered conversion.
+    bool Converted = false;
+  };
+  // Set only if CalleeArgInfo is set.
+  llvm::Optional<PassType> CallPassType;
 
   /// Produce a user-readable information.
   markup::Document present() const;
 };
 
+inline bool operator==(const HoverInfo::PassType &LHS,
+                       const HoverInfo::PassType &RHS) {
+  return std::tie(LHS.PassBy, LHS.Converted) ==
+         std::tie(RHS.PassBy, RHS.Converted);
+}
+
 // Try to infer structure of a documentation comment (e.g. line breaks).
 // FIXME: move to another file so CodeComplete doesn't depend on Hover.
 void parseDocumentation(llvm::StringRef Input, markup::Document &Output);
diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp
index 048d076a977c4..7a7853a6245c8 100644
--- a/clang-tools-extra/clangd/ParsedAST.cpp
+++ b/clang-tools-extra/clangd/ParsedAST.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "ParsedAST.h"
+#include "../clang-tidy/ClangTidyCheck.h"
 #include "../clang-tidy/ClangTidyDiagnosticConsumer.h"
 #include "../clang-tidy/ClangTidyModuleRegistry.h"
 #include "AST.h"
diff --git a/clang-tools-extra/clangd/Preamble.cpp b/clang-tools-extra/clangd/Preamble.cpp
index ca0a76db78f4b..b71afa0b16191 100644
--- a/clang-tools-extra/clangd/Preamble.cpp
+++ b/clang-tools-extra/clangd/Preamble.cpp
@@ -229,9 +229,8 @@ struct ScannedPreamble {
 llvm::Expected<ScannedPreamble>
 scanPreamble(llvm::StringRef Contents, const tooling::CompileCommand &Cmd) {
   class EmptyFS : public ThreadsafeFS {
-  public:
-    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
-    view(llvm::NoneType) const override {
+  private:
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> viewImpl() const override {
       return new llvm::vfs::InMemoryFileSystem;
     }
   };
diff --git a/clang-tools-extra/clangd/Protocol.cpp b/clang-tools-extra/clangd/Protocol.cpp
index ecae65336e5b1..2396037157853 100644
--- a/clang-tools-extra/clangd/Protocol.cpp
+++ b/clang-tools-extra/clangd/Protocol.cpp
@@ -1029,7 +1029,7 @@ llvm::json::Value toJSON(const SemanticTokensEdit &Edit) {
       {"data", encodeTokens(Edit.tokens)}};
 }
 
-llvm::json::Value toJSON(const SemanticTokensOrEdits &TE) {
+llvm::json::Value toJSON(const SemanticTokensOrDelta &TE) {
   llvm::json::Object Result{{"resultId", TE.resultId}};
   if (TE.edits)
     Result["edits"] = *TE.edits;
@@ -1043,7 +1043,7 @@ bool fromJSON(const llvm::json::Value &Params, SemanticTokensParams &R) {
   return O && O.map("textDocument", R.textDocument);
 }
 
-bool fromJSON(const llvm::json::Value &Params, SemanticTokensEditsParams &R) {
+bool fromJSON(const llvm::json::Value &Params, SemanticTokensDeltaParams &R) {
   llvm::json::ObjectMapper O(Params);
   return O && O.map("textDocument", R.textDocument) &&
          O.map("previousResultId", R.previousResultId);
diff --git a/clang-tools-extra/clangd/Protocol.h b/clang-tools-extra/clangd/Protocol.h
index 0177ee262ae34..77d402a6a9ba1 100644
--- a/clang-tools-extra/clangd/Protocol.h
+++ b/clang-tools-extra/clangd/Protocol.h
@@ -1384,27 +1384,27 @@ struct SemanticTokens {
   // send a delta.
   std::string resultId;
 
-  /// The actual tokens. For a detailed description about how the data is
-  /// structured pls see
-  /// https://github.com/microsoft/vscode-extension-samples/blob/5ae1f7787122812dcc84e37427ca90af5ee09f14/semantic-tokens-sample/vscode.proposed.d.ts#L71
-  std::vector<SemanticToken> tokens;
+  /// The actual tokens.
+  std::vector<SemanticToken> tokens; // encoded as a flat integer array.
 };
 llvm::json::Value toJSON(const SemanticTokens &);
 
+/// Body of textDocument/semanticTokens/full request.
 struct SemanticTokensParams {
   /// The text document.
   TextDocumentIdentifier textDocument;
 };
 bool fromJSON(const llvm::json::Value &, SemanticTokensParams &);
 
+/// Body of textDocument/semanticTokens/full/delta request.
 /// Requests the changes in semantic tokens since a previous response.
-struct SemanticTokensEditsParams {
+struct SemanticTokensDeltaParams {
   /// The text document.
   TextDocumentIdentifier textDocument;
   /// The previous result id.
   std::string previousResultId;
 };
-bool fromJSON(const llvm::json::Value &Params, SemanticTokensEditsParams &R);
+bool fromJSON(const llvm::json::Value &Params, SemanticTokensDeltaParams &R);
 
 /// Describes a a replacement of a contiguous range of semanticTokens.
 struct SemanticTokensEdit {
@@ -1413,20 +1413,20 @@ struct SemanticTokensEdit {
   // We use token counts instead, and translate when serializing this struct.
   unsigned startToken = 0;
   unsigned deleteTokens = 0;
-  std::vector<SemanticToken> tokens;
+  std::vector<SemanticToken> tokens; // encoded as a flat integer array
 };
 llvm::json::Value toJSON(const SemanticTokensEdit &);
 
-/// This models LSP SemanticTokensEdits | SemanticTokens, which is the result of
-/// textDocument/semanticTokens/edits.
-struct SemanticTokensOrEdits {
+/// This models LSP SemanticTokensDelta | SemanticTokens, which is the result of
+/// textDocument/semanticTokens/full/delta.
+struct SemanticTokensOrDelta {
   std::string resultId;
   /// Set if we computed edits relative to a previous set of tokens.
   llvm::Optional<std::vector<SemanticTokensEdit>> edits;
   /// Set if we computed a fresh set of tokens.
-  llvm::Optional<std::vector<SemanticToken>> tokens;
+  llvm::Optional<std::vector<SemanticToken>> tokens; // encoded as integer array
 };
-llvm::json::Value toJSON(const SemanticTokensOrEdits &);
+llvm::json::Value toJSON(const SemanticTokensOrDelta &);
 
 /// Represents a semantic highlighting information that has to be applied on a
 /// specific line of the text document.
diff --git a/clang-tools-extra/clangd/RIFF.cpp b/clang-tools-extra/clangd/RIFF.cpp
index cdbae4f727394..b87c2d56af0c5 100644
--- a/clang-tools-extra/clangd/RIFF.cpp
+++ b/clang-tools-extra/clangd/RIFF.cpp
@@ -8,25 +8,29 @@
 
 #include "RIFF.h"
 #include "llvm/Support/Endian.h"
+#include "llvm/Support/Error.h"
 
 namespace clang {
 namespace clangd {
 namespace riff {
 
-static llvm::Error makeError(const char *Msg) {
-  return llvm::createStringError(llvm::inconvertibleErrorCode(), Msg);
+static llvm::Error makeError(const llvm::Twine &Msg) {
+  return llvm::make_error<llvm::StringError>(Msg,
+                                             llvm::inconvertibleErrorCode());
 }
 
 llvm::Expected<Chunk> readChunk(llvm::StringRef &Stream) {
   if (Stream.size() < 8)
-    return makeError("incomplete chunk header");
+    return makeError("incomplete chunk header: " + llvm::Twine(Stream.size()) +
+                     " bytes available");
   Chunk C;
   std::copy(Stream.begin(), Stream.begin() + 4, C.ID.begin());
   Stream = Stream.drop_front(4);
   uint32_t Len = llvm::support::endian::read32le(Stream.take_front(4).begin());
   Stream = Stream.drop_front(4);
   if (Stream.size() < Len)
-    return makeError("truncated chunk");
+    return makeError("truncated chunk: want " + llvm::Twine(Len) + ", got " +
+                     llvm::Twine(Stream.size()));
   C.Data = Stream.take_front(Len);
   Stream = Stream.drop_front(Len);
   if (Len % 2 & !Stream.empty()) { // Skip padding byte.
@@ -53,7 +57,7 @@ llvm::Expected<File> readFile(llvm::StringRef Stream) {
   if (!RIFF)
     return RIFF.takeError();
   if (RIFF->ID != fourCC("RIFF"))
-    return makeError("not a RIFF container");
+    return makeError("not a RIFF container: root is " + fourCCStr(RIFF->ID));
   if (RIFF->Data.size() < 4)
     return makeError("RIFF chunk too short");
   File F;
diff --git a/clang-tools-extra/clangd/RIFF.h b/clang-tools-extra/clangd/RIFF.h
index d827a90f2bd7c..96d8ab5463d36 100644
--- a/clang-tools-extra/clangd/RIFF.h
+++ b/clang-tools-extra/clangd/RIFF.h
@@ -44,6 +44,9 @@ using FourCC = std::array<char, 4>;
 inline constexpr FourCC fourCC(const char (&Literal)[5]) {
   return FourCC{{Literal[0], Literal[1], Literal[2], Literal[3]}};
 }
+inline constexpr llvm::StringRef fourCCStr(const FourCC &Data) {
+  return llvm::StringRef(&Data[0], Data.size());
+}
 // A chunk is a section in a RIFF container.
 struct Chunk {
   FourCC ID;
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp
index d2470da601408..ed75ce80999cd 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -224,7 +224,7 @@ class HighlightingsBuilder {
       // Create one token for each line in the skipped range, so it works
       // with line-based diffing.
       assert(R.start.line <= R.end.line);
-      for (int Line = R.start.line; Line < R.end.line; ++Line) {
+      for (int Line = R.start.line; Line <= R.end.line; ++Line) {
         // Don't bother computing the offset for the end of the line, just use
         // zero. The client will treat this highlighting kind specially, and
         // highlight the entire line visually (i.e. not just to where the text
diff --git a/clang-tools-extra/clangd/TUScheduler.cpp b/clang-tools-extra/clangd/TUScheduler.cpp
index 7f15d44134a05..5454b1c92c8a8 100644
--- a/clang-tools-extra/clangd/TUScheduler.cpp
+++ b/clang-tools-extra/clangd/TUScheduler.cpp
@@ -270,6 +270,10 @@ class PreambleThread {
 
       {
         WithContext Guard(std::move(CurrentReq->Ctx));
+        // Note that we don't make use of the ContextProvider here.
+        // Preamble tasks are always scheduled by ASTWorker tasks, and we
+        // reuse the context/config that was created at that level.
+
         // Build the preamble and let the waiters know about it.
         build(std::move(*CurrentReq));
       }
@@ -456,6 +460,8 @@ class ASTWorker {
   const DebouncePolicy UpdateDebounce;
   /// File that ASTWorker is responsible for.
   const Path FileName;
+  /// Callback to create processing contexts for tasks.
+  const std::function<Context(llvm::StringRef)> ContextProvider;
   const GlobalCompilationDatabase &CDB;
   /// Callback invoked when preamble or main file AST is built.
   ParsingCallbacks &Callbacks;
@@ -569,8 +575,9 @@ ASTWorker::ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
                      bool RunSync, const TUScheduler::Options &Opts,
                      ParsingCallbacks &Callbacks)
     : IdleASTs(LRUCache), RunSync(RunSync), UpdateDebounce(Opts.UpdateDebounce),
-      FileName(FileName), CDB(CDB), Callbacks(Callbacks), Barrier(Barrier),
-      Done(false), Status(FileName, Callbacks),
+      FileName(FileName), ContextProvider(Opts.ContextProvider), CDB(CDB),
+      Callbacks(Callbacks), Barrier(Barrier), Done(false),
+      Status(FileName, Callbacks),
       PreamblePeer(FileName, Callbacks, Opts.StorePreamblesInMemory,
                    RunSync || !Opts.AsyncPreambleBuilds, Status, *this) {
   // Set a fallback command because compile command can be accessed before
@@ -1055,6 +1062,9 @@ void ASTWorker::run() {
         Status.ASTActivity.K = ASTAction::RunningAction;
         Status.ASTActivity.Name = CurrentRequest->Name;
       });
+      llvm::Optional<WithContext> WithProvidedContext;
+      if (ContextProvider)
+        WithProvidedContext.emplace(ContextProvider(FileName));
       CurrentRequest->Action();
     }
 
@@ -1288,14 +1298,18 @@ llvm::StringMap<std::string> TUScheduler::getAllFileContents() const {
   return Results;
 }
 
-void TUScheduler::run(llvm::StringRef Name,
+void TUScheduler::run(llvm::StringRef Name, llvm::StringRef Path,
                       llvm::unique_function<void()> Action) {
   if (!PreambleTasks)
     return Action();
   PreambleTasks->runAsync(Name, [this, Ctx = Context::current().clone(),
+                                 Path(Path.str()),
                                  Action = std::move(Action)]() mutable {
     std::lock_guard<Semaphore> BarrierLock(Barrier);
     WithContext WC(std::move(Ctx));
+    llvm::Optional<WithContext> WithProvidedContext;
+    if (Opts.ContextProvider)
+      WithProvidedContext.emplace(Opts.ContextProvider(Path));
     Action();
   });
 }
@@ -1356,6 +1370,9 @@ void TUScheduler::runWithPreamble(llvm::StringRef Name, PathRef File,
         WithContext Guard(std::move(Ctx));
         trace::Span Tracer(Name);
         SPAN_ATTACH(Tracer, "file", File);
+        llvm::Optional<WithContext> WithProvidedContext;
+        if (Opts.ContextProvider)
+          WithProvidedContext.emplace(Opts.ContextProvider(File));
         Action(InputsAndPreamble{Contents, Command, Preamble.get()});
       };
 
diff --git a/clang-tools-extra/clangd/TUScheduler.h b/clang-tools-extra/clangd/TUScheduler.h
index fb32c3b2edff4..05c06da13380d 100644
--- a/clang-tools-extra/clangd/TUScheduler.h
+++ b/clang-tools-extra/clangd/TUScheduler.h
@@ -194,7 +194,12 @@ class TUScheduler {
 
     /// Whether to run PreamblePeer asynchronously.
     /// No-op if AsyncThreadsCount is 0.
-    bool AsyncPreambleBuilds = false;
+    bool AsyncPreambleBuilds = true;
+
+    /// Used to create a context that wraps each single operation.
+    /// Typically to inject per-file configuration.
+    /// If the path is empty, context sholud be "generic".
+    std::function<Context(PathRef)> ContextProvider;
   };
 
   TUScheduler(const GlobalCompilationDatabase &CDB, const Options &Opts,
@@ -233,7 +238,9 @@ class TUScheduler {
   llvm::StringMap<std::string> getAllFileContents() const;
 
   /// Schedule an async task with no dependencies.
-  void run(llvm::StringRef Name, llvm::unique_function<void()> Action);
+  /// Path may be empty (it is used only to set the Context).
+  void run(llvm::StringRef Name, llvm::StringRef Path,
+           llvm::unique_function<void()> Action);
 
   /// Defines how a runWithAST action is implicitly cancelled by other actions.
   enum ASTActionInvalidation {
@@ -301,6 +308,7 @@ class TUScheduler {
   // this inside clangd.
   // FIXME: remove this when there is proper index support via build system
   // integration.
+  // FIXME: move to ClangdServer via createProcessingContext.
   static llvm::Optional<llvm::StringRef> getFileBeingProcessedInContext();
 
 private:
diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp
index a7bbfd4f967c5..c208e953f2ab7 100644
--- a/clang-tools-extra/clangd/XRefs.cpp
+++ b/clang-tools-extra/clangd/XRefs.cpp
@@ -993,7 +993,6 @@ std::vector<DocumentHighlight> findDocumentHighlights(ParsedAST &AST,
           DeclRelation::TemplatePattern | DeclRelation::Alias;
       auto Decls = targetDecl(N->ASTNode, Relations);
       if (!Decls.empty()) {
-        auto Refs = findRefs({Decls.begin(), Decls.end()}, AST);
         // FIXME: we may get multiple DocumentHighlights with the same location
         // and different kinds, deduplicate them.
         for (const auto &Ref : findRefs({Decls.begin(), Decls.end()}, AST))
@@ -1050,7 +1049,7 @@ ReferencesResult findReferences(ParsedAST &AST, Position Pos, uint32_t Limit,
       const auto &IDToRefs = AST.getMacros().MacroRefs;
       auto Refs = IDToRefs.find(*MacroSID);
       if (Refs != IDToRefs.end()) {
-        for (const auto Ref : Refs->second) {
+        for (const auto &Ref : Refs->second) {
           Location Result;
           Result.range = Ref;
           Result.uri = URIMainFile;
diff --git a/clang-tools-extra/clangd/index/Background.cpp b/clang-tools-extra/clangd/index/Background.cpp
index 3e68d9e9c62a3..5024ace66b7cc 100644
--- a/clang-tools-extra/clangd/index/Background.cpp
+++ b/clang-tools-extra/clangd/index/Background.cpp
@@ -93,9 +93,11 @@ BackgroundIndex::BackgroundIndex(
     Context BackgroundContext, const ThreadsafeFS &TFS,
     const GlobalCompilationDatabase &CDB,
     BackgroundIndexStorage::Factory IndexStorageFactory, size_t ThreadPoolSize,
-    std::function<void(BackgroundQueue::Stats)> OnProgress)
+    std::function<void(BackgroundQueue::Stats)> OnProgress,
+    std::function<Context(PathRef)> ContextProvider)
     : SwapIndex(std::make_unique<MemIndex>()), TFS(TFS), CDB(CDB),
       BackgroundContext(std::move(BackgroundContext)),
+      ContextProvider(std::move(ContextProvider)),
       Rebuilder(this, &IndexedSymbols, ThreadPoolSize),
       IndexStorageFactory(std::move(IndexStorageFactory)),
       Queue(std::move(OnProgress)),
@@ -103,10 +105,9 @@ BackgroundIndex::BackgroundIndex(
           CDB.watch([&](const std::vector<std::string> &ChangedFiles) {
             enqueue(ChangedFiles);
           })) {
-  assert(Rebuilder.TUsBeforeFirstBuild > 0 &&
-         "Thread pool size can't be zero.");
+  assert(ThreadPoolSize > 0 && "Thread pool size can't be zero.");
   assert(this->IndexStorageFactory && "Storage factory can not be null!");
-  for (unsigned I = 0; I < Rebuilder.TUsBeforeFirstBuild; ++I) {
+  for (unsigned I = 0; I < ThreadPoolSize; ++I) {
     ThreadPool.runAsync("background-worker-" + llvm::Twine(I + 1), [this] {
       WithContext Ctx(this->BackgroundContext.clone());
       Queue.work([&] { Rebuilder.idle(); });
@@ -123,6 +124,11 @@ BackgroundQueue::Task BackgroundIndex::changedFilesTask(
     const std::vector<std::string> &ChangedFiles) {
   BackgroundQueue::Task T([this, ChangedFiles] {
     trace::Span Tracer("BackgroundIndexEnqueue");
+
+    llvm::Optional<WithContext> WithProvidedContext;
+    if (ContextProvider)
+      WithProvidedContext.emplace(ContextProvider(/*Path=*/""));
+
     // We're doing this asynchronously, because we'll read shards here too.
     log("Enqueueing {0} commands for indexing", ChangedFiles.size());
     SPAN_ATTACH(Tracer, "files", int64_t(ChangedFiles.size()));
@@ -148,17 +154,20 @@ static llvm::StringRef filenameWithoutExtension(llvm::StringRef Path) {
   return Path.drop_back(llvm::sys::path::extension(Path).size());
 }
 
-BackgroundQueue::Task
-BackgroundIndex::indexFileTask(tooling::CompileCommand Cmd) {
-  BackgroundQueue::Task T([this, Cmd] {
-    // We can't use llvm::StringRef here since we are going to
-    // move from Cmd during the call below.
-    const std::string FileName = Cmd.Filename;
-    if (auto Error = index(std::move(Cmd)))
-      elog("Indexing {0} failed: {1}", FileName, std::move(Error));
+BackgroundQueue::Task BackgroundIndex::indexFileTask(std::string Path) {
+  std::string Tag = filenameWithoutExtension(Path).str();
+  BackgroundQueue::Task T([this, Path(std::move(Path))] {
+    llvm::Optional<WithContext> WithProvidedContext;
+    if (ContextProvider)
+      WithProvidedContext.emplace(ContextProvider(Path));
+    auto Cmd = CDB.getCompileCommand(Path);
+    if (!Cmd)
+      return;
+    if (auto Error = index(std::move(*Cmd)))
+      elog("Indexing {0} failed: {1}", Path, std::move(Error));
   });
   T.QueuePri = IndexFile;
-  T.Tag = std::string(filenameWithoutExtension(Cmd.Filename));
+  T.Tag = std::move(Tag);
   return T;
 }
 
@@ -343,10 +352,8 @@ llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd) {
 // Restores shards for \p MainFiles from index storage. Then checks staleness of
 // those shards and returns a list of TUs that needs to be indexed to update
 // staleness.
-std::vector<tooling::CompileCommand>
+std::vector<std::string>
 BackgroundIndex::loadProject(std::vector<std::string> MainFiles) {
-  std::vector<tooling::CompileCommand> NeedsReIndexing;
-
   Rebuilder.startLoading();
   // Load shards for all of the mainfiles.
   const std::vector<LoadedShard> Result =
@@ -399,14 +406,7 @@ BackgroundIndex::loadProject(std::vector<std::string> MainFiles) {
     TUsToIndex.insert(TUForFile);
   }
 
-  for (PathRef TU : TUsToIndex) {
-    auto Cmd = CDB.getCompileCommand(TU);
-    if (!Cmd)
-      continue;
-    NeedsReIndexing.emplace_back(std::move(*Cmd));
-  }
-
-  return NeedsReIndexing;
+  return {TUsToIndex.begin(), TUsToIndex.end()};
 }
 
 } // namespace clangd
diff --git a/clang-tools-extra/clangd/index/Background.h b/clang-tools-extra/clangd/index/Background.h
index 9ae63828cc6bd..9adad17376863 100644
--- a/clang-tools-extra/clangd/index/Background.h
+++ b/clang-tools-extra/clangd/index/Background.h
@@ -56,9 +56,9 @@ class BackgroundIndexStorage {
   using Factory = llvm::unique_function<BackgroundIndexStorage *(PathRef)>;
 
   // Creates an Index Storage that saves shards into disk. Index storage uses
-  // CDBDirectory + ".clangd/index/" as the folder to save shards. CDBDirectory
-  // is the first directory containing a CDB in parent directories of a file, or
-  // user's home directory if none was found, e.g. standard library headers.
+  // CDBDirectory + ".cache/clangd/index/" as the folder to save shards.
+  // CDBDirectory is the first directory containing a CDB in parent directories
+  // of a file, or user cache directory if none was found, e.g. stdlib headers.
   static Factory createDiskBackedStorageFactory(
       std::function<llvm::Optional<ProjectInfo>(PathRef)> GetProjectInfo);
 };
@@ -135,8 +135,11 @@ class BackgroundIndex : public SwapIndex {
       Context BackgroundContext, const ThreadsafeFS &,
       const GlobalCompilationDatabase &CDB,
       BackgroundIndexStorage::Factory IndexStorageFactory,
-      size_t ThreadPoolSize = 0, // 0 = use all hardware threads
-      std::function<void(BackgroundQueue::Stats)> OnProgress = nullptr);
+      // Arbitrary value to ensure some concurrency in tests.
+      // In production an explicit value is passed.
+      size_t ThreadPoolSize = 4,
+      std::function<void(BackgroundQueue::Stats)> OnProgress = nullptr,
+      std::function<Context(PathRef)> ContextProvider = nullptr);
   ~BackgroundIndex(); // Blocks while the current task finishes.
 
   // Enqueue translation units for indexing.
@@ -181,6 +184,7 @@ class BackgroundIndex : public SwapIndex {
   const ThreadsafeFS &TFS;
   const GlobalCompilationDatabase &CDB;
   Context BackgroundContext;
+  std::function<Context(PathRef)> ContextProvider;
 
   llvm::Error index(tooling::CompileCommand);
 
@@ -191,12 +195,11 @@ class BackgroundIndex : public SwapIndex {
 
   BackgroundIndexStorage::Factory IndexStorageFactory;
   // Tries to load shards for the MainFiles and their dependencies.
-  std::vector<tooling::CompileCommand>
-  loadProject(std::vector<std::string> MainFiles);
+  std::vector<std::string> loadProject(std::vector<std::string> MainFiles);
 
   BackgroundQueue::Task
   changedFilesTask(const std::vector<std::string> &ChangedFiles);
-  BackgroundQueue::Task indexFileTask(tooling::CompileCommand Cmd);
+  BackgroundQueue::Task indexFileTask(std::string Path);
 
   // from lowest to highest priority
   enum QueuePriority {
diff --git a/clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp b/clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp
index eee050b0db9ad..ed1662ff59881 100644
--- a/clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp
+++ b/clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp
@@ -95,8 +95,8 @@ class NullStorage : public BackgroundIndexStorage {
 };
 
 // Creates and owns IndexStorages for multiple CDBs.
-// When a CDB root is found, shards are stored in $ROOT/.clangd/index.
-// When no root is found, the fallback path is ~/.cache/clangd/index.
+// When a CDB root is found, shards are stored in $ROOT/.cache/clangd/index/.
+// When no root is found, the fallback path is ~/.cache/clangd/index/.
 class DiskBackedIndexStorageManager {
 public:
   DiskBackedIndexStorageManager(
@@ -115,7 +115,7 @@ class DiskBackedIndexStorageManager {
     llvm::SmallString<128> StorageDir(FallbackDir);
     if (auto PI = GetProjectInfo(File)) {
       StorageDir = PI->SourceRoot;
-      llvm::sys::path::append(StorageDir, ".clangd", "index");
+      llvm::sys::path::append(StorageDir, ".cache", "clangd", "index");
     }
     auto &IndexStorage = IndexStorageMap[StorageDir];
     if (!IndexStorage)
diff --git a/clang-tools-extra/clangd/index/BackgroundRebuild.h b/clang-tools-extra/clangd/index/BackgroundRebuild.h
index 295f705c98e8f..bc68536c847f8 100644
--- a/clang-tools-extra/clangd/index/BackgroundRebuild.h
+++ b/clang-tools-extra/clangd/index/BackgroundRebuild.h
@@ -49,9 +49,7 @@ class BackgroundIndexRebuilder {
 public:
   BackgroundIndexRebuilder(SwapIndex *Target, FileSymbols *Source,
                            unsigned Threads)
-      : TUsBeforeFirstBuild(llvm::heavyweight_hardware_concurrency(Threads)
-                                .compute_thread_count()),
-        Target(Target), Source(Source) {}
+      : TUsBeforeFirstBuild(Threads), Target(Target), Source(Source) {}
 
   // Called to indicate a TU has been indexed.
   // May rebuild, if enough TUs have been indexed.
@@ -72,7 +70,7 @@ class BackgroundIndexRebuilder {
   // Ensures we won't start any more rebuilds.
   void shutdown();
 
-  // Thresholds for rebuilding as TUs get indexed.
+  // Thresholds for rebuilding as TUs get indexed. Exposed for testing.
   const unsigned TUsBeforeFirstBuild; // Typically one per worker thread.
   const unsigned TUsBeforeRebuild = 100;
 
diff --git a/clang-tools-extra/clangd/index/FileIndex.cpp b/clang-tools-extra/clangd/index/FileIndex.cpp
index 1a18af1303dd8..5f84545d7c73d 100644
--- a/clang-tools-extra/clangd/index/FileIndex.cpp
+++ b/clang-tools-extra/clangd/index/FileIndex.cpp
@@ -229,6 +229,7 @@ void FileSymbols::update(llvm::StringRef Key,
                          std::unique_ptr<RelationSlab> Relations,
                          bool CountReferences) {
   std::lock_guard<std::mutex> Lock(Mutex);
+  ++Version;
   if (!Symbols)
     SymbolsSnapshot.erase(Key);
   else
@@ -248,7 +249,8 @@ void FileSymbols::update(llvm::StringRef Key,
 }
 
 std::unique_ptr<SymbolIndex>
-FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle) {
+FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle,
+                        size_t *Version) {
   std::vector<std::shared_ptr<SymbolSlab>> SymbolSlabs;
   std::vector<std::shared_ptr<RefSlab>> RefSlabs;
   std::vector<std::shared_ptr<RelationSlab>> RelationSlabs;
@@ -264,6 +266,9 @@ FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle) {
     }
     for (const auto &FileAndRelations : RelatiosSnapshot)
       RelationSlabs.push_back(FileAndRelations.second);
+
+    if (Version)
+      *Version = this->Version;
   }
   std::vector<const Symbol *> AllSymbols;
   std::vector<Symbol> SymsStorage;
@@ -390,12 +395,23 @@ void FileIndex::updatePreamble(PathRef Path, llvm::StringRef Version,
         std::make_unique<RelationSlab>(std::move(*IF->Relations)),
         /*CountReferences=*/false);
   }
-  PreambleIndex.reset(
+  size_t IndexVersion = 0;
+  auto NewIndex =
       PreambleSymbols.buildIndex(UseDex ? IndexType::Heavy : IndexType::Light,
-                                 DuplicateHandling::PickOne));
-  vlog("Build dynamic index for header symbols with estimated memory usage of "
-       "{0} bytes",
-       PreambleIndex.estimateMemoryUsage());
+                                 DuplicateHandling::PickOne, &IndexVersion);
+  {
+    std::lock_guard<std::mutex> Lock(UpdateIndexMu);
+    if (IndexVersion <= PreambleIndexVersion) {
+      // We lost the race, some other thread built a later version.
+      return;
+    }
+    PreambleIndexVersion = IndexVersion;
+    PreambleIndex.reset(std::move(NewIndex));
+    vlog(
+        "Build dynamic index for header symbols with estimated memory usage of "
+        "{0} bytes",
+        PreambleIndex.estimateMemoryUsage());
+  }
 }
 
 void FileIndex::updateMain(PathRef Path, ParsedAST &AST) {
@@ -405,11 +421,22 @@ void FileIndex::updateMain(PathRef Path, ParsedAST &AST) {
       std::make_unique<RefSlab>(std::move(std::get<1>(Contents))),
       std::make_unique<RelationSlab>(std::move(std::get<2>(Contents))),
       /*CountReferences=*/true);
-  MainFileIndex.reset(
-      MainFileSymbols.buildIndex(IndexType::Light, DuplicateHandling::Merge));
-  vlog("Build dynamic index for main-file symbols with estimated memory usage "
-       "of {0} bytes",
-       MainFileIndex.estimateMemoryUsage());
+  size_t IndexVersion = 0;
+  auto NewIndex = MainFileSymbols.buildIndex(
+      IndexType::Light, DuplicateHandling::Merge, &IndexVersion);
+  {
+    std::lock_guard<std::mutex> Lock(UpdateIndexMu);
+    if (IndexVersion <= MainIndexVersion) {
+      // We lost the race, some other thread built a later version.
+      return;
+    }
+    MainIndexVersion = IndexVersion;
+    MainFileIndex.reset(std::move(NewIndex));
+    vlog(
+        "Build dynamic index for main-file symbols with estimated memory usage "
+        "of {0} bytes",
+        MainFileIndex.estimateMemoryUsage());
+  }
 }
 
 } // namespace clangd
diff --git a/clang-tools-extra/clangd/index/FileIndex.h b/clang-tools-extra/clangd/index/FileIndex.h
index 65a2c305dca55..e6f8d1ef9e3d7 100644
--- a/clang-tools-extra/clangd/index/FileIndex.h
+++ b/clang-tools-extra/clangd/index/FileIndex.h
@@ -81,9 +81,11 @@ class FileSymbols {
   /// The index keeps the slabs alive.
   /// Will count Symbol::References based on number of references in the main
   /// files, while building the index with DuplicateHandling::Merge option.
+  /// Version is populated with an increasing sequence counter.
   std::unique_ptr<SymbolIndex>
   buildIndex(IndexType,
-             DuplicateHandling DuplicateHandle = DuplicateHandling::PickOne);
+             DuplicateHandling DuplicateHandle = DuplicateHandling::PickOne,
+             size_t *Version = nullptr);
 
 private:
   struct RefSlabAndCountReferences {
@@ -92,6 +94,7 @@ class FileSymbols {
   };
   mutable std::mutex Mutex;
 
+  size_t Version = 0;
   llvm::StringMap<std::shared_ptr<SymbolSlab>> SymbolsSnapshot;
   llvm::StringMap<RefSlabAndCountReferences> RefsSnapshot;
   llvm::StringMap<std::shared_ptr<RelationSlab>> RelatiosSnapshot;
@@ -136,6 +139,12 @@ class FileIndex : public MergedIndex {
   // (Note that symbols *only* in the main file are not indexed).
   FileSymbols MainFileSymbols;
   SwapIndex MainFileIndex;
+
+  // While both the FileIndex and SwapIndex are threadsafe, we need to track
+  // versions to ensure that we don't overwrite newer indexes with older ones.
+  std::mutex UpdateIndexMu;
+  unsigned MainIndexVersion = 0;
+  unsigned PreambleIndexVersion = 0;
 };
 
 using SlabTuple = std::tuple<SymbolSlab, RefSlab, RelationSlab>;
diff --git a/clang-tools-extra/clangd/index/Serialization.cpp b/clang-tools-extra/clangd/index/Serialization.cpp
index 06527a615c202..11d70b550642b 100644
--- a/clang-tools-extra/clangd/index/Serialization.cpp
+++ b/clang-tools-extra/clangd/index/Serialization.cpp
@@ -426,20 +426,25 @@ llvm::Expected<IndexFileIn> readRIFF(llvm::StringRef Data) {
   if (!RIFF)
     return RIFF.takeError();
   if (RIFF->Type != riff::fourCC("CdIx"))
-    return makeError("wrong RIFF type");
+    return makeError("wrong RIFF filetype: " + riff::fourCCStr(RIFF->Type));
   llvm::StringMap<llvm::StringRef> Chunks;
   for (const auto &Chunk : RIFF->Chunks)
     Chunks.try_emplace(llvm::StringRef(Chunk.ID.data(), Chunk.ID.size()),
                        Chunk.Data);
 
-  for (llvm::StringRef RequiredChunk : {"meta", "stri"})
+  if (!Chunks.count("meta"))
+    return makeError("missing meta chunk");
+  Reader Meta(Chunks.lookup("meta"));
+  auto SeenVersion = Meta.consume32();
+  if (SeenVersion != Version)
+    return makeError("wrong version: want " + llvm::Twine(Version) + ", got " +
+                     llvm::Twine(SeenVersion));
+
+  // meta chunk is checked above, as we prefer the "version mismatch" error.
+  for (llvm::StringRef RequiredChunk : {"stri"})
     if (!Chunks.count(RequiredChunk))
       return makeError("missing required chunk " + RequiredChunk);
 
-  Reader Meta(Chunks.lookup("meta"));
-  if (Meta.consume32() != Version)
-    return makeError("wrong version");
-
   auto Strings = readStringTable(Chunks.lookup("stri"));
   if (!Strings)
     return Strings.takeError();
@@ -665,7 +670,7 @@ std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFilename,
   trace::Span OverallTracer("LoadIndex");
   auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename);
   if (!Buffer) {
-    elog("Can't open {0}", SymbolFilename);
+    elog("Can't open {0}: {1}", SymbolFilename, Buffer.getError().message());
     return nullptr;
   }
 
@@ -682,7 +687,7 @@ std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFilename,
       if (I->Relations)
         Relations = std::move(*I->Relations);
     } else {
-      elog("Bad Index: {0}", I.takeError());
+      elog("Bad index file: {0}", I.takeError());
       return nullptr;
     }
   }
diff --git a/clang-tools-extra/clangd/index/Symbol.h b/clang-tools-extra/clangd/index/Symbol.h
index 00cf496b6d53e..2d2644e31a03d 100644
--- a/clang-tools-extra/clangd/index/Symbol.h
+++ b/clang-tools-extra/clangd/index/Symbol.h
@@ -186,7 +186,8 @@ class SymbolSlab {
   const_iterator end() const { return Symbols.end(); }
   const_iterator find(const SymbolID &SymID) const;
 
-  size_t size() const { return Symbols.size(); }
+  using size_type = size_t;
+  size_type size() const { return Symbols.size(); }
   bool empty() const { return Symbols.empty(); }
   // Estimates the total memory usage.
   size_t bytes() const {
diff --git a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
index 3e25da385d7ab..6fc844c189315 100644
--- a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
+++ b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
@@ -32,6 +32,9 @@ llvm::cl::opt<std::string> IndexLocation(
 llvm::cl::opt<std::string>
     ExecCommand("c", llvm::cl::desc("Command to execute and then exit"));
 
+llvm::cl::opt<std::string> ProjectRoot("project-root",
+                                       llvm::cl::desc("Path to the project"));
+
 static constexpr char Overview[] = R"(
 This is an **experimental** interactive tool to process user-provided search
 queries over given symbol collection obtained via clangd-indexer. The
@@ -326,7 +329,8 @@ struct {
 
 std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) {
   return Index.startswith("remote:")
-             ? remote::getClient(Index.drop_front(strlen("remote:")))
+             ? remote::getClient(Index.drop_front(strlen("remote:")),
+                                 ProjectRoot)
              : loadIndex(Index, /*UseDex=*/true);
 }
 
diff --git a/clang-tools-extra/clangd/index/remote/Client.cpp b/clang-tools-extra/clangd/index/remote/Client.cpp
index 366a37a135a47..b468831930765 100644
--- a/clang-tools-extra/clangd/index/remote/Client.cpp
+++ b/clang-tools-extra/clangd/index/remote/Client.cpp
@@ -10,10 +10,14 @@
 
 #include "Client.h"
 #include "Index.grpc.pb.h"
+#include "index/Index.h"
 #include "index/Serialization.h"
 #include "marshalling/Marshalling.h"
 #include "support/Logger.h"
 #include "support/Trace.h"
+#include "llvm/ADT/StringRef.h"
+
+#include <chrono>
 
 namespace clang {
 namespace clangd {
@@ -25,7 +29,16 @@ class IndexClient : public clangd::SymbolIndex {
   using StreamingCall = std::unique_ptr<grpc::ClientReader<ReplyT>> (
       remote::SymbolIndex::Stub::*)(grpc::ClientContext *, const RequestT &);
 
-  // FIXME(kirillbobyrev): Set deadlines for requests.
+  template <typename ClangdRequestT, typename RequestT>
+  RequestT serializeRequest(ClangdRequestT Request) const {
+    return toProtobuf(Request);
+  }
+
+  template <>
+  FuzzyFindRequest serializeRequest(clangd::FuzzyFindRequest Request) const {
+    return toProtobuf(Request, ProjectRoot);
+  }
+
   template <typename RequestT, typename ReplyT, typename ClangdRequestT,
             typename CallbackT>
   bool streamRPC(ClangdRequestT Request,
@@ -33,8 +46,11 @@ class IndexClient : public clangd::SymbolIndex {
                  CallbackT Callback) const {
     bool FinalResult = false;
     trace::Span Tracer(RequestT::descriptor()->name());
-    const auto RPCRequest = toProtobuf(Request);
+    const auto RPCRequest = serializeRequest<ClangdRequestT, RequestT>(Request);
     grpc::ClientContext Context;
+    std::chrono::system_clock::time_point Deadline =
+        std::chrono::system_clock::now() + DeadlineWaitingTime;
+    Context.set_deadline(Deadline);
     auto Reader = (Stub.get()->*RPCCall)(&Context, RPCRequest);
     llvm::BumpPtrAllocator Arena;
     llvm::UniqueStringSaver Strings(Arena);
@@ -44,18 +60,22 @@ class IndexClient : public clangd::SymbolIndex {
         FinalResult = Reply.final_result();
         continue;
       }
-      auto Sym = fromProtobuf(Reply.stream_result(), &Strings);
-      if (!Sym)
+      auto Response =
+          fromProtobuf(Reply.stream_result(), &Strings, ProjectRoot);
+      if (!Response)
         elog("Received invalid {0}", ReplyT::descriptor()->name());
-      Callback(*Sym);
+      Callback(*Response);
     }
     SPAN_ATTACH(Tracer, "status", Reader->Finish().ok());
     return FinalResult;
   }
 
 public:
-  IndexClient(std::shared_ptr<grpc::Channel> Channel)
-      : Stub(remote::SymbolIndex::NewStub(Channel)) {}
+  IndexClient(
+      std::shared_ptr<grpc::Channel> Channel, llvm::StringRef ProjectRoot,
+      std::chrono::milliseconds DeadlineTime = std::chrono::milliseconds(1000))
+      : Stub(remote::SymbolIndex::NewStub(Channel)), ProjectRoot(ProjectRoot),
+        DeadlineWaitingTime(DeadlineTime) {}
 
   void lookup(const clangd::LookupRequest &Request,
               llvm::function_ref<void(const clangd::Symbol &)> Callback) const {
@@ -79,20 +99,26 @@ class IndexClient : public clangd::SymbolIndex {
             llvm::function_ref<void(const SymbolID &, const clangd::Symbol &)>)
       const {}
 
-  // IndexClient does not take any space since the data is stored on the server.
+  // IndexClient does not take any space since the data is stored on the
+  // server.
   size_t estimateMemoryUsage() const { return 0; }
 
 private:
   std::unique_ptr<remote::SymbolIndex::Stub> Stub;
+  std::string ProjectRoot;
+  // Each request will be terminated if it takes too long.
+  std::chrono::milliseconds DeadlineWaitingTime;
 };
 
 } // namespace
 
-std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address) {
+std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address,
+                                               llvm::StringRef ProjectRoot) {
   const auto Channel =
       grpc::CreateChannel(Address.str(), grpc::InsecureChannelCredentials());
   Channel->GetState(true);
-  return std::unique_ptr<clangd::SymbolIndex>(new IndexClient(Channel));
+  return std::unique_ptr<clangd::SymbolIndex>(
+      new IndexClient(Channel, ProjectRoot));
 }
 
 } // namespace remote
diff --git a/clang-tools-extra/clangd/index/remote/Client.h b/clang-tools-extra/clangd/index/remote/Client.h
index 9708fdbe851ae..7b4173af6cfad 100644
--- a/clang-tools-extra/clangd/index/remote/Client.h
+++ b/clang-tools-extra/clangd/index/remote/Client.h
@@ -10,6 +10,7 @@
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_REMOTE_INDEX_H
 
 #include "index/Index.h"
+#include "llvm/ADT/StringRef.h"
 
 namespace clang {
 namespace clangd {
@@ -17,12 +18,16 @@ namespace remote {
 
 /// Returns an SymbolIndex client that passes requests to remote index located
 /// at \p Address. The client allows synchronous RPC calls.
+/// \p IndexRoot is an absolute path on the local machine to the source tree
+/// described by the remote index. Paths returned by the index will be treated
+/// as relative to this directory.
 ///
 /// This method attempts to resolve the address and establish the connection.
 ///
 /// \returns nullptr if the address is not resolved during the function call or
 /// if the project was compiled without Remote Index support.
-std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address);
+std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address,
+                                               llvm::StringRef IndexRoot);
 
 } // namespace remote
 } // namespace clangd
diff --git a/clang-tools-extra/clangd/index/remote/Index.proto b/clang-tools-extra/clangd/index/remote/Index.proto
index f3af29dcd588b..99c4e3329d670 100644
--- a/clang-tools-extra/clangd/index/remote/Index.proto
+++ b/clang-tools-extra/clangd/index/remote/Index.proto
@@ -71,7 +71,7 @@ message Symbol {
   string name = 3;
   SymbolLocation definition = 4;
   string scope = 5;
-  SymbolLocation canonical_declarattion = 6;
+  SymbolLocation canonical_declaration = 6;
   int32 references = 7;
   uint32 origin = 8;
   string signature = 9;
@@ -99,7 +99,10 @@ message SymbolInfo {
 message SymbolLocation {
   Position start = 1;
   Position end = 2;
-  string file_uri = 3;
+  // clangd::SymbolLocation stores FileURI, but the protocol transmits a the
+  // relative path. Because paths are different on the remote and local machines
+  // they will be translated in the marshalling layer.
+  string file_path = 3;
 }
 
 message Position {
diff --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
index e95132d000ece..db895bf039bab 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "Marshalling.h"
+#include "Headers.h"
 #include "Index.pb.h"
 #include "Protocol.h"
 #include "index/Serialization.h"
@@ -16,7 +17,13 @@
 #include "index/SymbolOrigin.h"
 #include "support/Logger.h"
 #include "clang/Index/IndexSymbol.h"
+#include "llvm/ADT/None.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/StringSaver.h"
 
 namespace clang {
@@ -58,40 +65,66 @@ SymbolInfo toProtobuf(const clang::index::SymbolInfo &Info) {
   return Result;
 }
 
-clangd::SymbolLocation fromProtobuf(const SymbolLocation &Message,
-                                    llvm::UniqueStringSaver *Strings) {
+llvm::Optional<clangd::SymbolLocation>
+fromProtobuf(const SymbolLocation &Message, llvm::UniqueStringSaver *Strings,
+             llvm::StringRef IndexRoot) {
   clangd::SymbolLocation Location;
+  auto URIString = relativePathToURI(Message.file_path(), IndexRoot);
+  if (!URIString)
+    return llvm::None;
+  Location.FileURI = Strings->save(*URIString).begin();
   Location.Start = fromProtobuf(Message.start());
   Location.End = fromProtobuf(Message.end());
-  Location.FileURI = Strings->save(Message.file_uri()).begin();
   return Location;
 }
 
-SymbolLocation toProtobuf(const clangd::SymbolLocation &Location) {
+llvm::Optional<SymbolLocation>
+toProtobuf(const clangd::SymbolLocation &Location, llvm::StringRef IndexRoot) {
   remote::SymbolLocation Result;
+  auto RelativePath = uriToRelativePath(Location.FileURI, IndexRoot);
+  if (!RelativePath)
+    return llvm::None;
+  *Result.mutable_file_path() = *RelativePath;
   *Result.mutable_start() = toProtobuf(Location.Start);
   *Result.mutable_end() = toProtobuf(Location.End);
-  *Result.mutable_file_uri() = Location.FileURI;
   return Result;
 }
 
-HeaderWithReferences
-toProtobuf(const clangd::Symbol::IncludeHeaderWithReferences &IncludeHeader) {
+llvm::Optional<HeaderWithReferences>
+toProtobuf(const clangd::Symbol::IncludeHeaderWithReferences &IncludeHeader,
+           llvm::StringRef IndexRoot) {
   HeaderWithReferences Result;
-  Result.set_header(IncludeHeader.IncludeHeader.str());
   Result.set_references(IncludeHeader.References);
+  const std::string Header = IncludeHeader.IncludeHeader.str();
+  if (isLiteralInclude(Header)) {
+    Result.set_header(Header);
+    return Result;
+  }
+  auto RelativePath = uriToRelativePath(Header, IndexRoot);
+  if (!RelativePath)
+    return llvm::None;
+  Result.set_header(*RelativePath);
   return Result;
 }
 
-clangd::Symbol::IncludeHeaderWithReferences
-fromProtobuf(const HeaderWithReferences &Message) {
-  return clangd::Symbol::IncludeHeaderWithReferences{Message.header(),
+llvm::Optional<clangd::Symbol::IncludeHeaderWithReferences>
+fromProtobuf(const HeaderWithReferences &Message,
+             llvm::UniqueStringSaver *Strings, llvm::StringRef IndexRoot) {
+  std::string Header = Message.header();
+  if (Header.front() != '<' && Header.front() != '"') {
+    auto URIString = relativePathToURI(Header, IndexRoot);
+    if (!URIString)
+      return llvm::None;
+    Header = *URIString;
+  }
+  return clangd::Symbol::IncludeHeaderWithReferences{Strings->save(Header),
                                                      Message.references()};
 }
 
 } // namespace
 
-clangd::FuzzyFindRequest fromProtobuf(const FuzzyFindRequest *Request) {
+clangd::FuzzyFindRequest fromProtobuf(const FuzzyFindRequest *Request,
+                                      llvm::StringRef IndexRoot) {
   clangd::FuzzyFindRequest Result;
   Result.Query = Request->query();
   for (const auto &Scope : Request->scopes())
@@ -100,24 +133,29 @@ clangd::FuzzyFindRequest fromProtobuf(const FuzzyFindRequest *Request) {
   if (Request->limit())
     Result.Limit = Request->limit();
   Result.RestrictForCodeCompletion = Request->restricted_for_code_completion();
-  for (const auto &Path : Request->proximity_paths())
-    Result.ProximityPaths.push_back(Path);
+  for (const auto &Path : Request->proximity_paths()) {
+    llvm::SmallString<256> LocalPath = llvm::StringRef(IndexRoot);
+    llvm::sys::path::append(LocalPath, Path);
+    Result.ProximityPaths.push_back(std::string(LocalPath));
+  }
   for (const auto &Type : Request->preferred_types())
     Result.ProximityPaths.push_back(Type);
   return Result;
 }
 
 llvm::Optional<clangd::Symbol> fromProtobuf(const Symbol &Message,
-                                            llvm::UniqueStringSaver *Strings) {
+                                            llvm::UniqueStringSaver *Strings,
+                                            llvm::StringRef IndexRoot) {
   if (!Message.has_info() || !Message.has_definition() ||
-      !Message.has_canonical_declarattion()) {
-    elog("Cannot convert Symbol from Protobuf: {}", Message.ShortDebugString());
+      !Message.has_canonical_declaration()) {
+    elog("Cannot convert Symbol from Protobuf: {0}",
+         Message.ShortDebugString());
     return llvm::None;
   }
   clangd::Symbol Result;
   auto ID = SymbolID::fromStr(Message.id());
   if (!ID) {
-    elog("Cannot convert parse SymbolID {} from Protobuf: {}", ID.takeError(),
+    elog("Cannot parse SymbolID {0} given Protobuf: {1}", ID.takeError(),
          Message.ShortDebugString());
     return llvm::None;
   }
@@ -125,9 +163,15 @@ llvm::Optional<clangd::Symbol> fromProtobuf(const Symbol &Message,
   Result.SymInfo = fromProtobuf(Message.info());
   Result.Name = Message.name();
   Result.Scope = Message.scope();
-  Result.Definition = fromProtobuf(Message.definition(), Strings);
-  Result.CanonicalDeclaration =
-      fromProtobuf(Message.canonical_declarattion(), Strings);
+  auto Definition = fromProtobuf(Message.definition(), Strings, IndexRoot);
+  if (!Definition)
+    return llvm::None;
+  Result.Definition = *Definition;
+  auto Declaration =
+      fromProtobuf(Message.canonical_declaration(), Strings, IndexRoot);
+  if (!Declaration)
+    return llvm::None;
+  Result.CanonicalDeclaration = *Declaration;
   Result.References = Message.references();
   Result.Origin = static_cast<clangd::SymbolOrigin>(Message.origin());
   Result.Signature = Message.signature();
@@ -137,20 +181,26 @@ llvm::Optional<clangd::Symbol> fromProtobuf(const Symbol &Message,
   Result.ReturnType = Message.return_type();
   Result.Type = Message.type();
   for (const auto &Header : Message.headers()) {
-    Result.IncludeHeaders.push_back(fromProtobuf(Header));
+    auto SerializedHeader = fromProtobuf(Header, Strings, IndexRoot);
+    if (SerializedHeader)
+      Result.IncludeHeaders.push_back(*SerializedHeader);
   }
   Result.Flags = static_cast<clangd::Symbol::SymbolFlag>(Message.flags());
   return Result;
 }
 
 llvm::Optional<clangd::Ref> fromProtobuf(const Ref &Message,
-                                         llvm::UniqueStringSaver *Strings) {
+                                         llvm::UniqueStringSaver *Strings,
+                                         llvm::StringRef IndexRoot) {
   if (!Message.has_location()) {
     elog("Cannot convert Ref from Protobuf: {}", Message.ShortDebugString());
     return llvm::None;
   }
   clangd::Ref Result;
-  Result.Location = fromProtobuf(Message.location(), Strings);
+  auto Location = fromProtobuf(Message.location(), Strings, IndexRoot);
+  if (!Location)
+    return llvm::None;
+  Result.Location = *Location;
   Result.Kind = static_cast<clangd::RefKind>(Message.kind());
   return Result;
 }
@@ -162,7 +212,8 @@ LookupRequest toProtobuf(const clangd::LookupRequest &From) {
   return RPCRequest;
 }
 
-FuzzyFindRequest toProtobuf(const clangd::FuzzyFindRequest &From) {
+FuzzyFindRequest toProtobuf(const clangd::FuzzyFindRequest &From,
+                            llvm::StringRef IndexRoot) {
   FuzzyFindRequest RPCRequest;
   RPCRequest.set_query(From.Query);
   for (const auto &Scope : From.Scopes)
@@ -171,8 +222,12 @@ FuzzyFindRequest toProtobuf(const clangd::FuzzyFindRequest &From) {
   if (From.Limit)
     RPCRequest.set_limit(*From.Limit);
   RPCRequest.set_restricted_for_code_completion(From.RestrictForCodeCompletion);
-  for (const auto &Path : From.ProximityPaths)
-    RPCRequest.add_proximity_paths(Path);
+  for (const auto &Path : From.ProximityPaths) {
+    llvm::SmallString<256> RelativePath = llvm::StringRef(Path);
+    if (llvm::sys::path::replace_path_prefix(RelativePath, IndexRoot, ""))
+      RPCRequest.add_proximity_paths(llvm::sys::path::convert_to_slash(
+          RelativePath, llvm::sys::path::Style::posix));
+  }
   for (const auto &Type : From.PreferredTypes)
     RPCRequest.add_preferred_types(Type);
   return RPCRequest;
@@ -188,15 +243,18 @@ RefsRequest toProtobuf(const clangd::RefsRequest &From) {
   return RPCRequest;
 }
 
-Symbol toProtobuf(const clangd::Symbol &From) {
+Symbol toProtobuf(const clangd::Symbol &From, llvm::StringRef IndexRoot) {
   Symbol Result;
   Result.set_id(From.ID.str());
   *Result.mutable_info() = toProtobuf(From.SymInfo);
   Result.set_name(From.Name.str());
-  *Result.mutable_definition() = toProtobuf(From.Definition);
+  auto Definition = toProtobuf(From.Definition, IndexRoot);
+  if (Definition)
+    *Result.mutable_definition() = *Definition;
   Result.set_scope(From.Scope.str());
-  *Result.mutable_canonical_declarattion() =
-      toProtobuf(From.CanonicalDeclaration);
+  auto Declaration = toProtobuf(From.CanonicalDeclaration, IndexRoot);
+  if (Declaration)
+    *Result.mutable_canonical_declaration() = *Declaration;
   Result.set_references(From.References);
   Result.set_origin(static_cast<uint32_t>(From.Origin));
   Result.set_signature(From.Signature.str());
@@ -207,20 +265,82 @@ Symbol toProtobuf(const clangd::Symbol &From) {
   Result.set_return_type(From.ReturnType.str());
   Result.set_type(From.Type.str());
   for (const auto &Header : From.IncludeHeaders) {
+    auto Serialized = toProtobuf(Header, IndexRoot);
+    if (!Serialized)
+      continue;
     auto *NextHeader = Result.add_headers();
-    *NextHeader = toProtobuf(Header);
+    *NextHeader = *Serialized;
   }
   Result.set_flags(static_cast<uint32_t>(From.Flags));
   return Result;
 }
 
-Ref toProtobuf(const clangd::Ref &From) {
+// FIXME(kirillbobyrev): A reference without location is invalid.
+// llvm::Optional<Ref> here and on the server side?
+Ref toProtobuf(const clangd::Ref &From, llvm::StringRef IndexRoot) {
   Ref Result;
   Result.set_kind(static_cast<uint32_t>(From.Kind));
-  *Result.mutable_location() = toProtobuf(From.Location);
+  auto Location = toProtobuf(From.Location, IndexRoot);
+  if (Location)
+    *Result.mutable_location() = *Location;
   return Result;
 }
 
+llvm::Optional<std::string> relativePathToURI(llvm::StringRef RelativePath,
+                                              llvm::StringRef IndexRoot) {
+  assert(RelativePath == llvm::sys::path::convert_to_slash(
+                             RelativePath, llvm::sys::path::Style::posix));
+  assert(IndexRoot == llvm::sys::path::convert_to_slash(IndexRoot));
+  assert(IndexRoot.endswith(llvm::sys::path::get_separator()));
+  if (RelativePath.empty())
+    return std::string();
+  if (llvm::sys::path::is_absolute(RelativePath)) {
+    elog("Remote index client got absolute path from server: {0}",
+         RelativePath);
+    return llvm::None;
+  }
+  if (llvm::sys::path::is_relative(IndexRoot)) {
+    elog("Remote index client got a relative path as index root: {0}",
+         IndexRoot);
+    return llvm::None;
+  }
+  llvm::SmallString<256> FullPath = IndexRoot;
+  llvm::sys::path::append(FullPath, RelativePath);
+  auto Result = URI::createFile(FullPath);
+  return Result.toString();
+}
+
+llvm::Optional<std::string> uriToRelativePath(llvm::StringRef URI,
+                                              llvm::StringRef IndexRoot) {
+  assert(IndexRoot.endswith(llvm::sys::path::get_separator()));
+  assert(IndexRoot == llvm::sys::path::convert_to_slash(IndexRoot));
+  assert(!IndexRoot.empty());
+  if (llvm::sys::path::is_relative(IndexRoot)) {
+    elog("Index root {0} is not absolute path", IndexRoot);
+    return llvm::None;
+  }
+  auto ParsedURI = URI::parse(URI);
+  if (!ParsedURI) {
+    elog("Remote index got bad URI from client {0}: {1}", URI,
+         ParsedURI.takeError());
+    return llvm::None;
+  }
+  if (ParsedURI->scheme() != "file") {
+    elog("Remote index got URI with scheme other than \"file\" {0}: {1}", URI,
+         ParsedURI->scheme());
+    return llvm::None;
+  }
+  llvm::SmallString<256> Result = ParsedURI->body();
+  if (!llvm::sys::path::replace_path_prefix(Result, IndexRoot, "")) {
+    elog("Can not get relative path from the URI {0} given the index root {1}",
+         URI, IndexRoot);
+    return llvm::None;
+  }
+  // Make sure the result has UNIX slashes.
+  return llvm::sys::path::convert_to_slash(Result,
+                                           llvm::sys::path::Style::posix);
+}
+
 } // namespace remote
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
index 1bc25bf5a9de5..86cc8fa272fc5 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
@@ -6,7 +6,28 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// Transformations between native Clangd types and Protobuf-generated classes.
+// Marshalling provides translation between native clangd types into the
+// Protobuf-generated classes. Most translations are 1-to-1 and wrap variables
+// into appropriate Protobuf types.
+//
+// A notable exception is URI translation. Because paths to files are different
+// on indexing machine and client machine
+// ("/remote/machine/projects/llvm-project/llvm/include/HelloWorld.h" versus
+// "/usr/local/username/llvm-project/llvm/include/HelloWorld.h"), they need to
+// be converted appropriately. Remote machine strips the prefix from the
+// absolute path and passes paths relative to the project root over the wire
+// ("include/HelloWorld.h" in this example). The indexed project root is passed
+// to the remote server. Client receives this relative path and constructs a URI
+// that points to the relevant file in the filesystem. The relative path is
+// appended to IndexRoot to construct the full path and build the final URI.
+//
+// Index root is the absolute path to the project and includes a trailing slash.
+// The relative path passed over the wire has unix slashes.
+//
+// toProtobuf() functions serialize native clangd types and strip IndexRoot from
+// the file paths specific to indexing machine. fromProtobuf() functions
+// deserialize clangd types and translate relative paths into machine-native
+// URIs.
 //
 //===----------------------------------------------------------------------===//
 
@@ -15,24 +36,40 @@
 
 #include "Index.pb.h"
 #include "index/Index.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/StringSaver.h"
 
 namespace clang {
 namespace clangd {
 namespace remote {
 
-clangd::FuzzyFindRequest fromProtobuf(const FuzzyFindRequest *Request);
+clangd::FuzzyFindRequest fromProtobuf(const FuzzyFindRequest *Request,
+                                      llvm::StringRef IndexRoot);
 llvm::Optional<clangd::Symbol> fromProtobuf(const Symbol &Message,
-                                            llvm::UniqueStringSaver *Strings);
+                                            llvm::UniqueStringSaver *Strings,
+                                            llvm::StringRef IndexRoot);
 llvm::Optional<clangd::Ref> fromProtobuf(const Ref &Message,
-                                         llvm::UniqueStringSaver *Strings);
+                                         llvm::UniqueStringSaver *Strings,
+                                         llvm::StringRef IndexRoot);
 
 LookupRequest toProtobuf(const clangd::LookupRequest &From);
-FuzzyFindRequest toProtobuf(const clangd::FuzzyFindRequest &From);
+FuzzyFindRequest toProtobuf(const clangd::FuzzyFindRequest &From,
+                            llvm::StringRef IndexRoot);
 RefsRequest toProtobuf(const clangd::RefsRequest &From);
 
-Ref toProtobuf(const clangd::Ref &From);
-Symbol toProtobuf(const clangd::Symbol &From);
+Ref toProtobuf(const clangd::Ref &From, llvm::StringRef IndexRoot);
+Symbol toProtobuf(const clangd::Symbol &From, llvm::StringRef IndexRoot);
+
+/// Translates \p RelativePath into the absolute path and builds URI for the
+/// user machine. This translation happens on the client side with the
+/// \p RelativePath received from remote index server and \p IndexRoot is
+/// provided by the client.
+llvm::Optional<std::string> relativePathToURI(llvm::StringRef RelativePath,
+                                              llvm::StringRef IndexRoot);
+/// Translates a URI from the server's backing index to a relative path suitable
+/// to send over the wire to the client.
+llvm::Optional<std::string> uriToRelativePath(llvm::StringRef URI,
+                                              llvm::StringRef IndexRoot);
 
 } // namespace remote
 } // namespace clangd
diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp
index 4d84eb17210ea..718d623a48456 100644
--- a/clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -11,6 +11,7 @@
 #include "index/remote/marshalling/Marshalling.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/Signals.h"
 
 #include <grpc++/grpc++.h>
@@ -31,6 +32,9 @@ awaits gRPC lookup requests from the client.
 llvm::cl::opt<std::string> IndexPath(llvm::cl::desc("<INDEX FILE>"),
                                      llvm::cl::Positional, llvm::cl::Required);
 
+llvm::cl::opt<std::string> IndexRoot(llvm::cl::desc("<PROJECT ROOT>"),
+                                     llvm::cl::Positional, llvm::cl::Required);
+
 llvm::cl::opt<std::string> ServerAddress(
     "server-address", llvm::cl::init("0.0.0.0:50051"),
     llvm::cl::desc("Address of the invoked server. Defaults to 0.0.0.0:50051"));
@@ -41,8 +45,13 @@ std::unique_ptr<clangd::SymbolIndex> openIndex(llvm::StringRef Index) {
 
 class RemoteIndexServer final : public SymbolIndex::Service {
 public:
-  RemoteIndexServer(std::unique_ptr<clangd::SymbolIndex> Index)
-      : Index(std::move(Index)) {}
+  RemoteIndexServer(std::unique_ptr<clangd::SymbolIndex> Index,
+                    llvm::StringRef IndexRoot)
+      : Index(std::move(Index)) {
+    llvm::SmallString<256> NativePath = IndexRoot;
+    llvm::sys::path::native(NativePath);
+    IndexedProjectRoot = std::string(NativePath);
+  }
 
 private:
   grpc::Status Lookup(grpc::ServerContext *Context,
@@ -57,7 +66,8 @@ class RemoteIndexServer final : public SymbolIndex::Service {
     }
     Index->lookup(Req, [&](const clangd::Symbol &Sym) {
       LookupReply NextMessage;
-      *NextMessage.mutable_stream_result() = toProtobuf(Sym);
+      *NextMessage.mutable_stream_result() =
+          toProtobuf(Sym, IndexedProjectRoot);
       Reply->Write(NextMessage);
     });
     LookupReply LastMessage;
@@ -69,10 +79,11 @@ class RemoteIndexServer final : public SymbolIndex::Service {
   grpc::Status FuzzyFind(grpc::ServerContext *Context,
                          const FuzzyFindRequest *Request,
                          grpc::ServerWriter<FuzzyFindReply> *Reply) override {
-    const auto Req = fromProtobuf(Request);
+    const auto Req = fromProtobuf(Request, IndexedProjectRoot);
     bool HasMore = Index->fuzzyFind(Req, [&](const clangd::Symbol &Sym) {
       FuzzyFindReply NextMessage;
-      *NextMessage.mutable_stream_result() = toProtobuf(Sym);
+      *NextMessage.mutable_stream_result() =
+          toProtobuf(Sym, IndexedProjectRoot);
       Reply->Write(NextMessage);
     });
     FuzzyFindReply LastMessage;
@@ -92,7 +103,7 @@ class RemoteIndexServer final : public SymbolIndex::Service {
     }
     bool HasMore = Index->refs(Req, [&](const clangd::Ref &Reference) {
       RefsReply NextMessage;
-      *NextMessage.mutable_stream_result() = toProtobuf(Reference);
+      *NextMessage.mutable_stream_result() = toProtobuf(Reference, IndexRoot);
       Reply->Write(NextMessage);
     });
     RefsReply LastMessage;
@@ -102,11 +113,12 @@ class RemoteIndexServer final : public SymbolIndex::Service {
   }
 
   std::unique_ptr<clangd::SymbolIndex> Index;
+  std::string IndexedProjectRoot;
 };
 
 void runServer(std::unique_ptr<clangd::SymbolIndex> Index,
                const std::string &ServerAddress) {
-  RemoteIndexServer Service(std::move(Index));
+  RemoteIndexServer Service(std::move(Index), IndexRoot);
 
   grpc::EnableDefaultHealthCheckService(true);
   grpc::ServerBuilder Builder;
@@ -128,6 +140,11 @@ int main(int argc, char *argv[]) {
   llvm::cl::ParseCommandLineOptions(argc, argv, Overview);
   llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
 
+  if (!llvm::sys::path::is_absolute(IndexRoot)) {
+    llvm::outs() << "Index root should be an absolute path.\n";
+    return -1;
+  }
+
   std::unique_ptr<clang::clangd::SymbolIndex> Index = openIndex(IndexPath);
 
   if (!Index) {
diff --git a/clang-tools-extra/clangd/index/remote/unimplemented/UnimplementedClient.cpp b/clang-tools-extra/clangd/index/remote/unimplemented/UnimplementedClient.cpp
index f0fb612f6332f..a4d3127e04553 100644
--- a/clang-tools-extra/clangd/index/remote/unimplemented/UnimplementedClient.cpp
+++ b/clang-tools-extra/clangd/index/remote/unimplemented/UnimplementedClient.cpp
@@ -8,12 +8,14 @@
 
 #include "index/remote/Client.h"
 #include "support/Logger.h"
+#include "llvm/ADT/StringRef.h"
 
 namespace clang {
 namespace clangd {
 namespace remote {
 
-std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address) {
+std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address,
+                                               llvm::StringRef IndexRoot) {
   elog("Can't create SymbolIndex client without Remote Index support.");
   return nullptr;
 }
diff --git a/clang-tools-extra/clangd/refactor/tweaks/CMakeLists.txt b/clang-tools-extra/clangd/refactor/tweaks/CMakeLists.txt
index c5bff2f34a9a2..8f708cacfdf88 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/CMakeLists.txt
+++ b/clang-tools-extra/clangd/refactor/tweaks/CMakeLists.txt
@@ -36,4 +36,7 @@ add_clang_library(clangDaemonTweaks OBJECT
   clangToolingCore
   clangToolingRefactoring
   clangToolingSyntax
+
+  DEPENDS
+  omp_gen
   )
diff --git a/clang-tools-extra/clangd/refactor/tweaks/DumpAST.cpp b/clang-tools-extra/clangd/refactor/tweaks/DumpAST.cpp
index 616e100adf084..b2b883d645679 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/DumpAST.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/DumpAST.cpp
@@ -62,7 +62,7 @@ REGISTER_TWEAK(DumpAST)
 llvm::Expected<Tweak::Effect> DumpAST::apply(const Selection &Inputs) {
   std::string Str;
   llvm::raw_string_ostream OS(Str);
-  Node->dump(OS, Inputs.AST->getSourceManager());
+  Node->dump(OS, Inputs.AST->getASTContext());
   return Effect::showMessage(std::move(OS.str()));
 }
 
diff --git a/clang-tools-extra/clangd/support/ThreadsafeFS.cpp b/clang-tools-extra/clangd/support/ThreadsafeFS.cpp
index a25ac773bfa27..cadda8efa0952 100644
--- a/clang-tools-extra/clangd/support/ThreadsafeFS.cpp
+++ b/clang-tools-extra/clangd/support/ThreadsafeFS.cpp
@@ -83,7 +83,7 @@ ThreadsafeFS::view(PathRef CWD) const {
 }
 
 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
-RealThreadsafeFS::view(llvm::NoneType) const {
+RealThreadsafeFS::viewImpl() const {
   // Avoid using memory-mapped files.
   // FIXME: Try to use a similar approach in Sema instead of relying on
   //        propagation of the 'isVolatile' flag through all layers.
diff --git a/clang-tools-extra/clangd/support/ThreadsafeFS.h b/clang-tools-extra/clangd/support/ThreadsafeFS.h
index aa6825fb39990..ad0dd64258112 100644
--- a/clang-tools-extra/clangd/support/ThreadsafeFS.h
+++ b/clang-tools-extra/clangd/support/ThreadsafeFS.h
@@ -30,20 +30,25 @@ class ThreadsafeFS {
   virtual ~ThreadsafeFS() = default;
 
   /// Obtain a vfs::FileSystem with an arbitrary initial working directory.
-  virtual llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
-  view(llvm::NoneType CWD) const = 0;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
+  view(llvm::NoneType CWD) const {
+    return viewImpl();
+  }
 
   /// Obtain a vfs::FileSystem with a specified working directory.
   /// If the working directory can't be set (e.g. doesn't exist), logs and
   /// returns the FS anyway.
-  virtual llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
-  view(PathRef CWD) const;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> view(PathRef CWD) const;
+
+private:
+  /// Overridden by implementations to provide a vfs::FileSystem.
+  /// This is distinct from view(NoneType) to avoid GCC's -Woverloaded-virtual.
+  virtual llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> viewImpl() const = 0;
 };
 
 class RealThreadsafeFS : public ThreadsafeFS {
-public:
-  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
-      view(llvm::NoneType) const override;
+private:
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> viewImpl() const override;
 };
 
 } // namespace clangd
diff --git a/clang-tools-extra/clangd/test/background-index.test b/clang-tools-extra/clangd/test/background-index.test
index 821ac7174f59d..41184443e947a 100644
--- a/clang-tools-extra/clangd/test/background-index.test
+++ b/clang-tools-extra/clangd/test/background-index.test
@@ -15,8 +15,8 @@
 # RUN: clangd -background-index -lit-test < %t/definition.jsonrpc | FileCheck %t/definition.jsonrpc --check-prefixes=CHECK,BUILD
 
 # Test that the index is writing files in the expected location.
-# RUN: ls %t/.clangd/index/foo.cpp.*.idx
-# RUN: ls %t/sub_dir/.clangd/index/foo.h.*.idx
+# RUN: ls %t/.cache/clangd/index/foo.cpp.*.idx
+# RUN: ls %t/sub_dir/.cache/clangd/index/foo.h.*.idx
 
 # Test the index is read from disk: delete code and restart clangd.
 # RUN: rm %t/foo.cpp
diff --git a/clang-tools-extra/clangd/test/initialize-params.test b/clang-tools-extra/clangd/test/initialize-params.test
index f1f5312d10096..f0a0f791c2f68 100644
--- a/clang-tools-extra/clangd/test/initialize-params.test
+++ b/clang-tools-extra/clangd/test/initialize-params.test
@@ -42,8 +42,8 @@
 # CHECK-NEXT:      "renameProvider": true,
 # CHECK-NEXT:      "selectionRangeProvider": true,
 # CHECK-NEXT:      "semanticTokensProvider": {
-# CHECK-NEXT:        "documentProvider": {
-# CHECK-NEXT:          "edits": true
+# CHECK-NEXT:        "full": {
+# CHECK-NEXT:          "delta": true
 # CHECK-NEXT:        },
 # CHECK-NEXT:        "legend": {
 # CHECK-NEXT:          "tokenModifiers": [],
@@ -51,7 +51,7 @@
 # CHECK-NEXT:            "variable",
 # CHECK:               ]
 # CHECK-NEXT:        },
-# CHECK-NEXT:        "rangeProvider": false
+# CHECK-NEXT:        "range": false
 # CHECK-NEXT:      },
 # CHECK-NEXT:      "signatureHelpProvider": {
 # CHECK-NEXT:        "triggerCharacters": [
diff --git a/clang-tools-extra/clangd/test/semantic-tokens.test b/clang-tools-extra/clangd/test/semantic-tokens.test
index 6fab10362b285..ed0c0d09a13ed 100644
--- a/clang-tools-extra/clangd/test/semantic-tokens.test
+++ b/clang-tools-extra/clangd/test/semantic-tokens.test
@@ -13,7 +13,7 @@
 }}}
 ---
 # Non-incremental token request.
-{"jsonrpc":"2.0","id":1,"method":"textDocument/semanticTokens","params":{"textDocument":{"uri":"test:///foo.cpp"}}}
+{"jsonrpc":"2.0","id":1,"method":"textDocument/semanticTokens/full","params":{"textDocument":{"uri":"test:///foo.cpp"}}}
 # CHECK:       "id": 1,
 # CHECK-NEXT:  "jsonrpc": "2.0",
 # CHECK-NEXT:  "result": {
@@ -34,7 +34,7 @@
 }}
 ---
 # Incremental token request, based on previous response.
-{"jsonrpc":"2.0","id":2,"method":"textDocument/semanticTokens/edits","params":{
+{"jsonrpc":"2.0","id":2,"method":"textDocument/semanticTokens/full/delta","params":{
   "textDocument": {"uri":"test:///foo.cpp"},
   "previousResultId": "1"
 }}
@@ -60,7 +60,7 @@
 # CHECK-NEXT:  }
 ---
 # Incremental token request with incorrect baseline => full tokens list.
-{"jsonrpc":"2.0","id":2,"method":"textDocument/semanticTokens/edits","params":{
+{"jsonrpc":"2.0","id":2,"method":"textDocument/semanticTokens/full/delta","params":{
   "textDocument": {"uri":"test:///foo.cpp"},
   "previousResultId": "bogus"
 }}
diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp
index c77f92d3f183e..6e3d6a231da1e 100644
--- a/clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -320,7 +320,7 @@ opt<bool> Test{
     "lit-test",
     cat(Misc),
     desc("Abbreviation for -input-style=delimited -pretty -sync "
-         "-enable-test-scheme -log=verbose. "
+         "-enable-test-scheme -enable-config=0 -log=verbose. "
          "Intended to simplify lit tests"),
     init(false),
     Hidden,
@@ -427,6 +427,20 @@ opt<bool> AsyncPreamble{
     Hidden,
 };
 
+opt<bool> EnableConfig{
+    "enable-config",
+    cat(Misc),
+    desc(
+        "Read user and project configuration from YAML files.\n"
+        "Project config is from a .clangd file in the project directory.\n"
+        "User config is from clangd/config.yaml in the following directories:\n"
+        "\tWindows: %USERPROFILE%\\AppData\\Local\n"
+        "\tMac OS: ~/Library/Preferences/\n"
+        "\tOthers: $XDG_CONFIG_HOME, usually ~/.config\n"
+        "Configuration is documented at https://clangd.llvm.org/config.html"),
+    init(false),
+};
+
 /// Supports a test URI scheme with relaxed constraints for lit tests.
 /// The path in a test URI will be combined with a platform-specific fake
 /// directory to form an absolute path. For example, test:///a.cpp is resolved
@@ -510,6 +524,9 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var
     InputStyle = JSONStreamStyle::Delimited;
     LogLevel = Logger::Verbose;
     PrettyPrint = true;
+    // Disable config system by default to avoid external reads.
+    if (!EnableConfig.getNumOccurrences())
+      EnableConfig = false;
     // Disable background index on lit tests by default to prevent disk writes.
     if (!EnableBackgroundIndex.getNumOccurrences())
       EnableBackgroundIndex = false;
@@ -677,6 +694,23 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var
   CCOpts.RunParser = CodeCompletionParse;
 
   RealThreadsafeFS TFS;
+  std::unique_ptr<config::Provider> Config;
+  if (EnableConfig) {
+    std::vector<std::unique_ptr<config::Provider>> ProviderStack;
+    ProviderStack.push_back(
+        config::Provider::fromAncestorRelativeYAMLFiles(".clangd", TFS));
+    llvm::SmallString<256> UserConfig;
+    if (llvm::sys::path::user_config_directory(UserConfig)) {
+      llvm::sys::path::append(UserConfig, "clangd", "config.yaml");
+      vlog("User config file is {0}", UserConfig);
+      ProviderStack.push_back(config::Provider::fromYAMLFile(UserConfig, TFS));
+    } else {
+      elog("Couldn't determine user config file, not loading");
+    }
+    Config = config::Provider::combine(std::move(ProviderStack));
+    Opts.ConfigProvider = Config.get();
+  }
+
   // Initialize and run ClangdLSPServer.
   // Change stdin to binary to not lose \r\n on windows.
   llvm::sys::ChangeStdinToBinary();
diff --git a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
index 24322e1575abd..cb4d23e0be347 100644
--- a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
+++ b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
@@ -1,3 +1,5 @@
+#include "CompileCommands.h"
+#include "Config.h"
 #include "Headers.h"
 #include "SyncAPI.h"
 #include "TestFS.h"
@@ -5,6 +7,7 @@
 #include "TestTU.h"
 #include "index/Background.h"
 #include "index/BackgroundRebuild.h"
+#include "clang/Tooling/ArgumentsAdjusters.h"
 #include "clang/Tooling/CompilationDatabase.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/Threading.h"
@@ -24,6 +27,7 @@ namespace clang {
 namespace clangd {
 
 MATCHER_P(Named, N, "") { return arg.Name == N; }
+MATCHER_P(QName, N, "") { return (arg.Scope + arg.Name).str() == N; }
 MATCHER(Declared, "") {
   return !StringRef(arg.CanonicalDeclaration.FileURI).empty();
 }
@@ -104,6 +108,53 @@ TEST_F(BackgroundIndexTest, NoCrashOnErrorFile) {
   ASSERT_TRUE(Idx.blockUntilIdleForTest());
 }
 
+TEST_F(BackgroundIndexTest, Config) {
+  MockFS FS;
+  // Set up two identical TUs, foo and bar.
+  // They define foo::one and bar::one.
+  std::vector<tooling::CompileCommand> Cmds;
+  for (std::string Name : {"foo", "bar"}) {
+    std::string Filename = Name + ".cpp";
+    std::string Header = Name + ".h";
+    FS.Files[Filename] = "#include \"" + Header + "\"";
+    FS.Files[Header] = "namespace " + Name + " { int one; }";
+    tooling::CompileCommand Cmd;
+    Cmd.Filename = Filename;
+    Cmd.Directory = testRoot();
+    Cmd.CommandLine = {"clang++", Filename};
+    Cmds.push_back(std::move(Cmd));
+  }
+  // Context provider that installs a configuration mutating foo's command.
+  // This causes it to define foo::two instead of foo::one.
+  auto ContextProvider = [](PathRef P) {
+    Config C;
+    if (P.endswith("foo.cpp"))
+      C.CompileFlags.Edits.push_back(
+          [](std::vector<std::string> &Argv) { Argv.push_back("-Done=two"); });
+    return Context::current().derive(Config::Key, std::move(C));
+  };
+  // Create the background index.
+  llvm::StringMap<std::string> Storage;
+  size_t CacheHits = 0;
+  MemoryShardStorage MSS(Storage, CacheHits);
+  // We need the CommandMangler, because that applies the config we're testing.
+  OverlayCDB CDB(/*Base=*/nullptr, /*FallbackFlags=*/{},
+                 tooling::ArgumentsAdjuster(CommandMangler::forTests()));
+  BackgroundIndex Idx(
+      Context::empty(), FS, CDB, [&](llvm::StringRef) { return &MSS; },
+      /*ThreadPoolSize=*/4, /*OnProgress=*/nullptr, std::move(ContextProvider));
+  // Index the two files.
+  for (auto &Cmd : Cmds) {
+    std::string FullPath = testPath(Cmd.Filename);
+    CDB.setCompileCommand(FullPath, std::move(Cmd));
+  }
+  // Wait for both files to be indexed.
+  ASSERT_TRUE(Idx.blockUntilIdleForTest());
+  EXPECT_THAT(runFuzzyFind(Idx, ""),
+              UnorderedElementsAre(QName("foo"), QName("foo::two"),
+                                   QName("bar"), QName("bar::one")));
+}
+
 TEST_F(BackgroundIndexTest, IndexTwoFiles) {
   MockFS FS;
   // a.h yields different symbols when included by A.cc vs B.cc.
diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt
index 03e09669b13f5..c25e2b7f81037 100644
--- a/clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -41,6 +41,9 @@ add_unittest(ClangdUnitTests ClangdTests
   CollectMacrosTests.cpp
   CompileCommandsTests.cpp
   CompilerTests.cpp
+  ConfigCompileTests.cpp
+  ConfigProviderTests.cpp
+  ConfigYAMLTests.cpp
   DexTests.cpp
   DiagnosticsTests.cpp
   DraftStoreTests.cpp
diff --git a/clang-tools-extra/clangd/unittests/ClangdTests.cpp b/clang-tools-extra/clangd/unittests/ClangdTests.cpp
index e1f02c897fe37..c21719d58bd6a 100644
--- a/clang-tools-extra/clangd/unittests/ClangdTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ClangdTests.cpp
@@ -10,6 +10,7 @@
 #include "ClangdLSPServer.h"
 #include "ClangdServer.h"
 #include "CodeComplete.h"
+#include "ConfigFragment.h"
 #include "GlobalCompilationDatabase.h"
 #include "Matchers.h"
 #include "SyncAPI.h"
@@ -19,6 +20,7 @@
 #include "support/Threading.h"
 #include "clang/Config/config.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
+#include "clang/Tooling/ArgumentsAdjusters.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallVector.h"
@@ -47,6 +49,7 @@ using ::testing::Field;
 using ::testing::Gt;
 using ::testing::IsEmpty;
 using ::testing::Pair;
+using ::testing::SizeIs;
 using ::testing::UnorderedElementsAre;
 
 MATCHER_P2(DeclAt, File, Range, "") {
@@ -273,12 +276,13 @@ int b = a;
 TEST_F(ClangdVFSTest, PropagatesContexts) {
   static Key<int> Secret;
   struct ContextReadingFS : public ThreadsafeFS {
-    IntrusiveRefCntPtr<llvm::vfs::FileSystem>
-    view(llvm::NoneType) const override {
+    mutable int Got;
+
+  private:
+    IntrusiveRefCntPtr<llvm::vfs::FileSystem> viewImpl() const override {
       Got = Context::current().getExisting(Secret);
       return buildTestFS({});
     }
-    mutable int Got;
   } FS;
   struct Callbacks : public ClangdServer::Callbacks {
     void onDiagnosticsReady(PathRef File, llvm::StringRef Version,
@@ -300,6 +304,48 @@ TEST_F(ClangdVFSTest, PropagatesContexts) {
   EXPECT_EQ(Callbacks.Got, 42);
 }
 
+TEST(ClangdServerTest, RespectsConfig) {
+  // Go-to-definition will resolve as marked if FOO is defined.
+  Annotations Example(R"cpp(
+  #ifdef FOO
+  int [[x]];
+  #else
+  int x;
+  #endif
+  int y = ^x;
+  )cpp");
+  // Provide conditional config that defines FOO for foo.cc.
+  class ConfigProvider : public config::Provider {
+    std::vector<config::CompiledFragment>
+    getFragments(const config::Params &,
+                 config::DiagnosticCallback DC) const override {
+      config::Fragment F;
+      F.If.PathMatch.emplace_back(".*foo.cc");
+      F.CompileFlags.Add.emplace_back("-DFOO=1");
+      return {std::move(F).compile(DC)};
+    }
+  } CfgProvider;
+
+  auto Opts = ClangdServer::optsForTest();
+  Opts.ConfigProvider = &CfgProvider;
+  OverlayCDB CDB(/*Base=*/nullptr, /*FallbackFlags=*/{},
+                 tooling::ArgumentsAdjuster(CommandMangler::forTests()));
+  MockFS FS;
+  ClangdServer Server(CDB, FS, Opts);
+  // foo.cc sees the expected definition, as FOO is defined.
+  Server.addDocument(testPath("foo.cc"), Example.code());
+  auto Result = runLocateSymbolAt(Server, testPath("foo.cc"), Example.point());
+  ASSERT_TRUE(bool(Result)) << Result.takeError();
+  ASSERT_THAT(*Result, SizeIs(1));
+  EXPECT_EQ(Result->front().PreferredDeclaration.range, Example.range());
+  // bar.cc gets a different result, as FOO is not defined.
+  Server.addDocument(testPath("bar.cc"), Example.code());
+  Result = runLocateSymbolAt(Server, testPath("bar.cc"), Example.point());
+  ASSERT_TRUE(bool(Result)) << Result.takeError();
+  ASSERT_THAT(*Result, SizeIs(1));
+  EXPECT_NE(Result->front().PreferredDeclaration.range, Example.range());
+}
+
 TEST_F(ClangdVFSTest, PropagatesVersion) {
   MockCompilationDatabase CDB;
   MockFS FS;
@@ -877,7 +923,7 @@ void f() {}
   FS.Files[Path] = Code;
   runAddDocument(Server, Path, Code);
 
-  auto Replaces = Server.formatFile(Code, Path);
+  auto Replaces = runFormatFile(Server, Path, Code);
   EXPECT_TRUE(static_cast<bool>(Replaces));
   auto Changed = tooling::applyAllReplacements(Code, *Replaces);
   EXPECT_TRUE(static_cast<bool>(Changed));
@@ -925,12 +971,17 @@ TEST_F(ClangdVFSTest, ChangedHeaderFromISystem) {
 // preamble again. (They should be using the preamble's stat-cache)
 TEST(ClangdTests, PreambleVFSStatCache) {
   class StatRecordingFS : public ThreadsafeFS {
+    llvm::StringMap<unsigned> &CountStats;
+
   public:
+    // If relative paths are used, they are resolved with testPath().
+    llvm::StringMap<std::string> Files;
+
     StatRecordingFS(llvm::StringMap<unsigned> &CountStats)
         : CountStats(CountStats) {}
 
-    IntrusiveRefCntPtr<llvm::vfs::FileSystem>
-    view(llvm::NoneType) const override {
+  private:
+    IntrusiveRefCntPtr<llvm::vfs::FileSystem> viewImpl() const override {
       class StatRecordingVFS : public llvm::vfs::ProxyFileSystem {
       public:
         StatRecordingVFS(IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
@@ -954,10 +1005,6 @@ TEST(ClangdTests, PreambleVFSStatCache) {
       return IntrusiveRefCntPtr<StatRecordingVFS>(
           new StatRecordingVFS(buildTestFS(Files), CountStats));
     }
-
-    // If relative paths are used, they are resolved with testPath().
-    llvm::StringMap<std::string> Files;
-    llvm::StringMap<unsigned> &CountStats;
   };
 
   llvm::StringMap<unsigned> CountStats;
diff --git a/clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp b/clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp
index d064c941a5908..d86296b84e3f9 100644
--- a/clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp
@@ -7,7 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "CompileCommands.h"
+#include "Config.h"
 #include "TestFS.h"
+#include "support/Context.h"
 
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/StringExtras.h"
@@ -22,6 +24,7 @@ namespace clang {
 namespace clangd {
 namespace {
 
+using ::testing::_;
 using ::testing::Contains;
 using ::testing::ElementsAre;
 using ::testing::HasSubstr;
@@ -185,6 +188,25 @@ TEST(CommandMangler, ClangPathResolve) {
 }
 #endif
 
+TEST(CommandMangler, ConfigEdits) {
+  auto Mangler = CommandMangler::forTests();
+  std::vector<std::string> Cmd = {"clang++", "foo.cc"};
+  {
+    Config Cfg;
+    Cfg.CompileFlags.Edits.push_back([](std::vector<std::string> &Argv) {
+      for (auto &Arg : Argv)
+        for (char &C : Arg)
+          C = llvm::toUpper(C);
+    });
+    Cfg.CompileFlags.Edits.push_back(
+        [](std::vector<std::string> &Argv) { Argv.push_back("--hello"); });
+    WithContextValue WithConfig(Config::Key, std::move(Cfg));
+    Mangler.adjust(Cmd);
+  }
+  // Edits are applied in given order and before other mangling.
+  EXPECT_THAT(Cmd, ElementsAre(_, "FOO.CC", "--hello", "-fsyntax-only"));
+}
+
 } // namespace
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp b/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp
new file mode 100644
index 0000000000000..825d6878727d9
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp
@@ -0,0 +1,104 @@
+//===-- ConfigCompileTests.cpp --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Config.h"
+#include "ConfigFragment.h"
+#include "ConfigTesting.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace config {
+namespace {
+using ::testing::ElementsAre;
+using ::testing::IsEmpty;
+using ::testing::SizeIs;
+using ::testing::StartsWith;
+
+class ConfigCompileTests : public ::testing::Test {
+protected:
+  CapturedDiags Diags;
+  Config Conf;
+  Fragment Frag;
+  Params Parm;
+
+  bool compileAndApply() {
+    Conf = Config();
+    Diags.Diagnostics.clear();
+    auto Compiled = std::move(Frag).compile(Diags.callback());
+    return Compiled(Parm, Conf);
+  }
+};
+
+TEST_F(ConfigCompileTests, Condition) {
+  // No condition.
+  Frag = {};
+  Frag.CompileFlags.Add.emplace_back("X");
+  EXPECT_TRUE(compileAndApply()) << "Empty config";
+  EXPECT_THAT(Diags.Diagnostics, IsEmpty());
+  EXPECT_THAT(Conf.CompileFlags.Edits, SizeIs(1));
+
+  // Regex with no file.
+  Frag = {};
+  Frag.If.PathMatch.emplace_back("fo*");
+  EXPECT_FALSE(compileAndApply());
+  EXPECT_THAT(Diags.Diagnostics, IsEmpty());
+  EXPECT_THAT(Conf.CompileFlags.Edits, SizeIs(0));
+
+  // Following tests have a file path set.
+  Parm.Path = "bar";
+
+  // Non-matching regex.
+  Frag = {};
+  Frag.If.PathMatch.emplace_back("fo*");
+  EXPECT_FALSE(compileAndApply());
+  EXPECT_THAT(Diags.Diagnostics, IsEmpty());
+
+  // Matching regex.
+  Frag = {};
+  Frag.If.PathMatch.emplace_back("fo*");
+  Frag.If.PathMatch.emplace_back("ba*r");
+  EXPECT_TRUE(compileAndApply());
+  EXPECT_THAT(Diags.Diagnostics, IsEmpty());
+
+  // Excluded regex.
+  Frag = {};
+  Frag.If.PathMatch.emplace_back("b.*");
+  Frag.If.PathExclude.emplace_back(".*r");
+  EXPECT_FALSE(compileAndApply()) << "Included but also excluded";
+  EXPECT_THAT(Diags.Diagnostics, IsEmpty());
+
+  // Invalid regex.
+  Frag = {};
+  Frag.If.PathMatch.emplace_back("**]@theu");
+  EXPECT_TRUE(compileAndApply());
+  EXPECT_THAT(Diags.Diagnostics, SizeIs(1));
+  EXPECT_THAT(Diags.Diagnostics.front().Message, StartsWith("Invalid regex"));
+
+  // Valid regex and unknown key.
+  Frag = {};
+  Frag.If.HasUnrecognizedCondition = true;
+  Frag.If.PathMatch.emplace_back("ba*r");
+  EXPECT_FALSE(compileAndApply());
+  EXPECT_THAT(Diags.Diagnostics, IsEmpty());
+}
+
+TEST_F(ConfigCompileTests, CompileCommands) {
+  Frag.CompileFlags.Add.emplace_back("-foo");
+  std::vector<std::string> Argv = {"clang", "a.cc"};
+  EXPECT_TRUE(compileAndApply());
+  EXPECT_THAT(Conf.CompileFlags.Edits, SizeIs(1));
+  Conf.CompileFlags.Edits.front()(Argv);
+  EXPECT_THAT(Argv, ElementsAre("clang", "a.cc", "-foo"));
+}
+
+} // namespace
+} // namespace config
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/ConfigProviderTests.cpp b/clang-tools-extra/clangd/unittests/ConfigProviderTests.cpp
new file mode 100644
index 0000000000000..122b55cf64e01
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/ConfigProviderTests.cpp
@@ -0,0 +1,156 @@
+//===-- ConfigProviderTests.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Config.h"
+#include "ConfigProvider.h"
+#include "ConfigTesting.h"
+#include "TestFS.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "llvm/Support/SourceMgr.h"
+#include <atomic>
+
+namespace clang {
+namespace clangd {
+namespace config {
+namespace {
+using ::testing::ElementsAre;
+using ::testing::IsEmpty;
+
+// Provider that appends an arg to compile flags.
+// The arg is prefix<N>, where N is the times getFragments() was called.
+// It also yields a diagnostic each time it's called.
+class FakeProvider : public Provider {
+  std::string Prefix;
+  mutable std::atomic<unsigned> Index = {0};
+
+  std::vector<CompiledFragment>
+  getFragments(const Params &, DiagnosticCallback DC) const override {
+    DC(llvm::SMDiagnostic("", llvm::SourceMgr::DK_Error, Prefix));
+    CompiledFragment F =
+        [Arg(Prefix + std::to_string(++Index))](const Params &P, Config &C) {
+          C.CompileFlags.Edits.push_back(
+              [Arg](std::vector<std::string> &Argv) { Argv.push_back(Arg); });
+          return true;
+        };
+    return {F};
+  }
+
+public:
+  FakeProvider(llvm::StringRef Prefix) : Prefix(Prefix) {}
+};
+
+std::vector<std::string> getAddedArgs(Config &C) {
+  std::vector<std::string> Argv;
+  for (auto &Edit : C.CompileFlags.Edits)
+    Edit(Argv);
+  return Argv;
+}
+
+// The provider from combine() should invoke its providers in order, and not
+// cache their results.
+TEST(ProviderTest, Combine) {
+  CapturedDiags Diags;
+  std::vector<std::unique_ptr<Provider>> Providers;
+  Providers.push_back(std::make_unique<FakeProvider>("foo"));
+  Providers.push_back(std::make_unique<FakeProvider>("bar"));
+  auto Combined = Provider::combine(std::move(Providers));
+  Config Cfg = Combined->getConfig(Params(), Diags.callback());
+  EXPECT_THAT(Diags.Diagnostics,
+              ElementsAre(DiagMessage("foo"), DiagMessage("bar")));
+  EXPECT_THAT(getAddedArgs(Cfg), ElementsAre("foo1", "bar1"));
+  Diags.Diagnostics.clear();
+
+  Cfg = Combined->getConfig(Params(), Diags.callback());
+  EXPECT_THAT(Diags.Diagnostics,
+              ElementsAre(DiagMessage("foo"), DiagMessage("bar")));
+  EXPECT_THAT(getAddedArgs(Cfg), ElementsAre("foo2", "bar2"));
+}
+
+const char *AddFooWithErr = R"yaml(
+CompileFlags:
+  Add: foo
+  Unknown: 42
+)yaml";
+
+const char *AddBarBaz = R"yaml(
+CompileFlags:
+  Add: bar
+---
+CompileFlags:
+  Add: baz
+)yaml";
+
+TEST(ProviderTest, FromYAMLFile) {
+  MockFS FS;
+  FS.Files["foo.yaml"] = AddFooWithErr;
+
+  CapturedDiags Diags;
+  auto P = Provider::fromYAMLFile(testPath("foo.yaml"), FS);
+  auto Cfg = P->getConfig(Params(), Diags.callback());
+  EXPECT_THAT(Diags.Diagnostics,
+              ElementsAre(DiagMessage("Unknown CompileFlags key Unknown")));
+  EXPECT_THAT(getAddedArgs(Cfg), ElementsAre("foo"));
+  Diags.Diagnostics.clear();
+
+  Cfg = P->getConfig(Params(), Diags.callback());
+  EXPECT_THAT(Diags.Diagnostics, IsEmpty()) << "Cached, not re-parsed";
+  EXPECT_THAT(getAddedArgs(Cfg), ElementsAre("foo"));
+
+  FS.Files["foo.yaml"] = AddBarBaz;
+  Cfg = P->getConfig(Params(), Diags.callback());
+  EXPECT_THAT(Diags.Diagnostics, IsEmpty()) << "New config, no errors";
+  EXPECT_THAT(getAddedArgs(Cfg), ElementsAre("bar", "baz"));
+
+  FS.Files.erase("foo.yaml");
+  Cfg = P->getConfig(Params(), Diags.callback());
+  EXPECT_THAT(Diags.Diagnostics, IsEmpty()) << "Missing file is not an error";
+  EXPECT_THAT(getAddedArgs(Cfg), IsEmpty());
+}
+
+TEST(ProviderTest, FromAncestorRelativeYAMLFiles) {
+  MockFS FS;
+  FS.Files["a/b/c/foo.yaml"] = AddBarBaz;
+  FS.Files["a/foo.yaml"] = AddFooWithErr;
+
+  std::string ABCPath =
+      testPath("a/b/c/d/test.cc", llvm::sys::path::Style::posix);
+  Params ABCParams;
+  ABCParams.Path = ABCPath;
+  std::string APath =
+      testPath("a/b/e/f/test.cc", llvm::sys::path::Style::posix);
+  Params AParams;
+  AParams.Path = APath;
+
+  CapturedDiags Diags;
+  auto P = Provider::fromAncestorRelativeYAMLFiles("foo.yaml", FS);
+
+  auto Cfg = P->getConfig(Params(), Diags.callback());
+  EXPECT_THAT(Diags.Diagnostics, IsEmpty());
+  EXPECT_THAT(getAddedArgs(Cfg), IsEmpty());
+
+  Cfg = P->getConfig(ABCParams, Diags.callback());
+  EXPECT_THAT(Diags.Diagnostics,
+              ElementsAre(DiagMessage("Unknown CompileFlags key Unknown")));
+  EXPECT_THAT(getAddedArgs(Cfg), ElementsAre("foo", "bar", "baz"));
+  Diags.Diagnostics.clear();
+
+  Cfg = P->getConfig(AParams, Diags.callback());
+  EXPECT_THAT(Diags.Diagnostics, IsEmpty()) << "Cached config";
+  EXPECT_THAT(getAddedArgs(Cfg), ElementsAre("foo"));
+
+  FS.Files.erase("a/foo.yaml");
+  Cfg = P->getConfig(ABCParams, Diags.callback());
+  EXPECT_THAT(Diags.Diagnostics, IsEmpty());
+  EXPECT_THAT(getAddedArgs(Cfg), ElementsAre("bar", "baz"));
+}
+
+} // namespace
+} // namespace config
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/ConfigTesting.h b/clang-tools-extra/clangd/unittests/ConfigTesting.h
new file mode 100644
index 0000000000000..5541c6fffe596
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/ConfigTesting.h
@@ -0,0 +1,77 @@
+//===-- ConfigTesting.h - Helpers for configuration tests -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_UNITTESTS_CONFIGTESTING_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_UNITTESTS_CONFIGTESTING_H
+
+#include "Protocol.h"
+#include "llvm/Support/ScopedPrinter.h"
+#include "llvm/Support/SourceMgr.h"
+#include "gmock/gmock.h"
+#include <functional>
+
+namespace clang {
+namespace clangd {
+namespace config {
+
+// Provides a DiagnosticsCallback that records diganostics.
+// Unlike just pushing them into a vector, underlying storage need not survive.
+struct CapturedDiags {
+  std::function<void(const llvm::SMDiagnostic &)> callback() {
+    return [this](const llvm::SMDiagnostic &D) {
+      Diagnostics.emplace_back();
+      Diag &Out = Diagnostics.back();
+      Out.Message = D.getMessage().str();
+      Out.Kind = D.getKind();
+      Out.Pos.line = D.getLineNo() - 1;
+      Out.Pos.character = D.getColumnNo(); // Zero-based - bug in SourceMgr?
+      if (!D.getRanges().empty()) {
+        const auto &R = D.getRanges().front();
+        Out.Rng.emplace();
+        Out.Rng->start.line = Out.Rng->end.line = Out.Pos.line;
+        Out.Rng->start.character = R.first;
+        Out.Rng->end.character = R.second;
+      }
+    };
+  }
+  struct Diag {
+    std::string Message;
+    llvm::SourceMgr::DiagKind Kind;
+    Position Pos;
+    llvm::Optional<Range> Rng;
+
+    friend void PrintTo(const Diag &D, std::ostream *OS) {
+      *OS << (D.Kind == llvm::SourceMgr::DK_Error ? "error: " : "warning: ")
+          << D.Message << "@" << llvm::to_string(D.Pos);
+    }
+  };
+  std::vector<Diag> Diagnostics;
+};
+
+MATCHER_P(DiagMessage, M, "") { return arg.Message == M; }
+MATCHER_P(DiagKind, K, "") { return arg.Kind == K; }
+MATCHER_P(DiagPos, P, "") { return arg.Pos == P; }
+MATCHER_P(DiagRange, R, "") { return arg.Rng == R; }
+
+inline Position toPosition(llvm::SMLoc L, const llvm::SourceMgr &SM) {
+  auto LineCol = SM.getLineAndColumn(L);
+  Position P;
+  P.line = LineCol.first - 1;
+  P.character = LineCol.second - 1;
+  return P;
+}
+
+inline Range toRange(llvm::SMRange R, const llvm::SourceMgr &SM) {
+  return Range{toPosition(R.Start, SM), toPosition(R.End, SM)};
+}
+
+} // namespace config
+} // namespace clangd
+} // namespace clang
+
+#endif
diff --git a/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp b/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp
new file mode 100644
index 0000000000000..a9526ce2367c4
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp
@@ -0,0 +1,125 @@
+//===-- ConfigYAMLTests.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Annotations.h"
+#include "ConfigFragment.h"
+#include "ConfigTesting.h"
+#include "Protocol.h"
+#include "llvm/Support/SMLoc.h"
+#include "llvm/Support/ScopedPrinter.h"
+#include "llvm/Support/SourceMgr.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace config {
+template <typename T> void PrintTo(const Located<T> &V, std::ostream *OS) {
+  *OS << ::testing::PrintToString(*V);
+}
+
+namespace {
+using ::testing::AllOf;
+using ::testing::ElementsAre;
+using ::testing::IsEmpty;
+
+MATCHER_P(Val, Value, "") {
+  if (*arg == Value)
+    return true;
+  *result_listener << "value is " << *arg;
+  return false;
+}
+
+TEST(ParseYAML, SyntacticForms) {
+  CapturedDiags Diags;
+  const char *YAML = R"yaml(
+If:
+  PathMatch:
+    - 'abc'
+CompileFlags: { Add: [foo, bar] }
+---
+CompileFlags:
+  Add: |
+    b
+    az
+  )yaml";
+  auto Results = Fragment::parseYAML(YAML, "config.yaml", Diags.callback());
+  EXPECT_THAT(Diags.Diagnostics, IsEmpty());
+  ASSERT_EQ(Results.size(), 2u);
+  EXPECT_FALSE(Results.front().If.HasUnrecognizedCondition);
+  EXPECT_THAT(Results.front().If.PathMatch, ElementsAre(Val("abc")));
+  EXPECT_THAT(Results.front().CompileFlags.Add,
+              ElementsAre(Val("foo"), Val("bar")));
+
+  EXPECT_THAT(Results.back().CompileFlags.Add, ElementsAre(Val("b\naz\n")));
+}
+
+TEST(ParseYAML, Locations) {
+  CapturedDiags Diags;
+  Annotations YAML(R"yaml(
+If:
+  PathMatch: [['???bad***regex(((']]
+  )yaml");
+  auto Results =
+      Fragment::parseYAML(YAML.code(), "config.yaml", Diags.callback());
+  EXPECT_THAT(Diags.Diagnostics, IsEmpty());
+  ASSERT_EQ(Results.size(), 1u);
+  ASSERT_NE(Results.front().Source.Manager, nullptr);
+  EXPECT_EQ(toRange(Results.front().If.PathMatch.front().Range,
+                    *Results.front().Source.Manager),
+            YAML.range());
+}
+
+TEST(ParseYAML, Diagnostics) {
+  CapturedDiags Diags;
+  Annotations YAML(R"yaml(
+If:
+  [[UnknownCondition]]: "foo"
+CompileFlags:
+  Add: 'first'
+---
+CompileFlags: {^
+)yaml");
+  auto Results =
+      Fragment::parseYAML(YAML.code(), "config.yaml", Diags.callback());
+
+  ASSERT_THAT(
+      Diags.Diagnostics,
+      ElementsAre(AllOf(DiagMessage("Unknown If key UnknownCondition"),
+                        DiagKind(llvm::SourceMgr::DK_Warning),
+                        DiagPos(YAML.range().start), DiagRange(YAML.range())),
+                  AllOf(DiagMessage("Unexpected token. Expected Key, Flow "
+                                    "Entry, or Flow Mapping End."),
+                        DiagKind(llvm::SourceMgr::DK_Error),
+                        DiagPos(YAML.point()), DiagRange(llvm::None))));
+
+  ASSERT_EQ(Results.size(), 1u); // invalid fragment discarded.
+  EXPECT_THAT(Results.front().CompileFlags.Add, ElementsAre(Val("first")));
+  EXPECT_TRUE(Results.front().If.HasUnrecognizedCondition);
+}
+
+TEST(ParseYAML, Invalid) {
+  CapturedDiags Diags;
+  const char *YAML = R"yaml(
+If:
+
+horrible
+---
+- 1
+  )yaml";
+  auto Results = Fragment::parseYAML(YAML, "config.yaml", Diags.callback());
+  EXPECT_THAT(Diags.Diagnostics,
+              ElementsAre(DiagMessage("If should be a dictionary"),
+                          DiagMessage("Config should be a dictionary")));
+  ASSERT_THAT(Results, IsEmpty());
+}
+
+} // namespace
+} // namespace config
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/FileIndexTests.cpp b/clang-tools-extra/clangd/unittests/FileIndexTests.cpp
index 1026055c5c290..955ee50be8955 100644
--- a/clang-tools-extra/clangd/unittests/FileIndexTests.cpp
+++ b/clang-tools-extra/clangd/unittests/FileIndexTests.cpp
@@ -22,6 +22,7 @@
 #include "index/Relation.h"
 #include "index/Serialization.h"
 #include "index/Symbol.h"
+#include "support/Threading.h"
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Frontend/Utils.h"
 #include "clang/Index/IndexSymbol.h"
@@ -509,6 +510,31 @@ TEST(FileIndexTest, StalePreambleSymbolsDeleted) {
   EXPECT_THAT(runFuzzyFind(M, ""), UnorderedElementsAre(QName("b")));
 }
 
+// Verifies that concurrent calls to updateMain don't "lose" any updates.
+TEST(FileIndexTest, Threadsafety) {
+  FileIndex M;
+  Notification Go;
+
+  constexpr int Count = 10;
+  {
+    // Set up workers to concurrently call updateMain() with separate files.
+    AsyncTaskRunner Pool;
+    for (unsigned I = 0; I < Count; ++I) {
+      auto TU = TestTU::withCode(llvm::formatv("int xxx{0};", I).str());
+      TU.Filename = llvm::formatv("x{0}.c", I).str();
+      Pool.runAsync(TU.Filename, [&, Filename(testPath(TU.Filename)),
+                                  AST(TU.build())]() mutable {
+        Go.wait();
+        M.updateMain(Filename, AST);
+      });
+    }
+    // On your marks, get set...
+    Go.notify();
+  }
+
+  EXPECT_THAT(runFuzzyFind(M, "xxx"), ::testing::SizeIs(Count));
+}
+
 TEST(FileShardedIndexTest, Sharding) {
   auto AHeaderUri = URI::create(testPath("a.h")).toString();
   auto BHeaderUri = URI::create(testPath("b.h")).toString();
diff --git a/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp b/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp
index b888ba22f5bbf..31879e356ce0e 100644
--- a/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp
@@ -10,6 +10,8 @@
 #include "FindSymbols.h"
 #include "SyncAPI.h"
 #include "TestFS.h"
+#include "TestTU.h"
+#include "llvm/ADT/StringRef.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
@@ -42,160 +44,143 @@ ::testing::Matcher<DocumentSymbol> Children(ChildMatchers... ChildrenM) {
   return Field(&DocumentSymbol::children, ElementsAre(ChildrenM...));
 }
 
-ClangdServer::Options optsForTests() {
-  auto ServerOpts = ClangdServer::optsForTest();
-  ServerOpts.WorkspaceRoot = testRoot();
-  ServerOpts.BuildDynamicSymbolIndex = true;
-  return ServerOpts;
+std::vector<SymbolInformation> getSymbols(TestTU &TU, llvm::StringRef Query,
+                                          int Limit = 0) {
+  auto SymbolInfos = getWorkspaceSymbols(Query, Limit, TU.index().get(),
+                                         testPath(TU.Filename));
+  EXPECT_TRUE(bool(SymbolInfos)) << "workspaceSymbols returned an error";
+  return *SymbolInfos;
 }
 
-class WorkspaceSymbolsTest : public ::testing::Test {
-public:
-  WorkspaceSymbolsTest() : Server(CDB, FS, optsForTests()) {
-    // Make sure the test root directory is created.
-    FS.Files[testPath("unused")] = "";
-    CDB.ExtraClangFlags = {"-xc++"};
-  }
-
-protected:
-  MockFS FS;
-  MockCompilationDatabase CDB;
-  ClangdServer Server;
-  int Limit = 0;
-
-  std::vector<SymbolInformation> getSymbols(llvm::StringRef Query) {
-    EXPECT_TRUE(Server.blockUntilIdleForTest()) << "Waiting for preamble";
-    auto SymbolInfos = runWorkspaceSymbols(Server, Query, Limit);
-    EXPECT_TRUE(bool(SymbolInfos)) << "workspaceSymbols returned an error";
-    return *SymbolInfos;
-  }
-
-  void addFile(llvm::StringRef FileName, llvm::StringRef Contents) {
-    Server.addDocument(testPath(FileName), Contents);
-  }
-};
-
-} // namespace
-
-TEST_F(WorkspaceSymbolsTest, Macros) {
-  addFile("foo.cpp", R"cpp(
+TEST(WorkspaceSymbols, Macros) {
+  TestTU TU;
+  TU.Code = R"cpp(
        #define MACRO X
-       )cpp");
+       )cpp";
 
   // LSP's SymbolKind doesn't have a "Macro" kind, and
   // indexSymbolKindToSymbolKind() currently maps macros
   // to SymbolKind::String.
-  EXPECT_THAT(getSymbols("macro"),
+  EXPECT_THAT(getSymbols(TU, "macro"),
               ElementsAre(AllOf(QName("MACRO"), WithKind(SymbolKind::String))));
 }
 
-TEST_F(WorkspaceSymbolsTest, NoLocals) {
-  addFile("foo.cpp", R"cpp(
+TEST(WorkspaceSymbols, NoLocals) {
+  TestTU TU;
+  TU.Code = R"cpp(
       void test(int FirstParam, int SecondParam) {
         struct LocalClass {};
         int local_var;
-      })cpp");
-  EXPECT_THAT(getSymbols("l"), IsEmpty());
-  EXPECT_THAT(getSymbols("p"), IsEmpty());
+      })cpp";
+  EXPECT_THAT(getSymbols(TU, "l"), IsEmpty());
+  EXPECT_THAT(getSymbols(TU, "p"), IsEmpty());
 }
 
-TEST_F(WorkspaceSymbolsTest, Globals) {
-  addFile("foo.h", R"cpp(
+TEST(WorkspaceSymbols, Globals) {
+  TestTU TU;
+  TU.AdditionalFiles["foo.h"] = R"cpp(
       int global_var;
 
       int global_func();
 
-      struct GlobalStruct {};)cpp");
-  addFile("foo.cpp", R"cpp(
+      struct GlobalStruct {};)cpp";
+  TU.Code = R"cpp(
       #include "foo.h"
-      )cpp");
-  EXPECT_THAT(getSymbols("global"),
+      )cpp";
+  EXPECT_THAT(getSymbols(TU, "global"),
               UnorderedElementsAre(
                   AllOf(QName("GlobalStruct"), WithKind(SymbolKind::Struct)),
                   AllOf(QName("global_func"), WithKind(SymbolKind::Function)),
                   AllOf(QName("global_var"), WithKind(SymbolKind::Variable))));
 }
 
-TEST_F(WorkspaceSymbolsTest, Unnamed) {
-  addFile("foo.h", R"cpp(
+TEST(WorkspaceSymbols, Unnamed) {
+  TestTU TU;
+  TU.AdditionalFiles["foo.h"] = R"cpp(
       struct {
         int InUnnamed;
-      } UnnamedStruct;)cpp");
-  addFile("foo.cpp", R"cpp(
+      } UnnamedStruct;)cpp";
+  TU.Code = R"cpp(
       #include "foo.h"
-      )cpp");
-  EXPECT_THAT(getSymbols("UnnamedStruct"),
+      )cpp";
+  EXPECT_THAT(getSymbols(TU, "UnnamedStruct"),
               ElementsAre(AllOf(QName("UnnamedStruct"),
                                 WithKind(SymbolKind::Variable))));
-  EXPECT_THAT(getSymbols("InUnnamed"),
+  EXPECT_THAT(getSymbols(TU, "InUnnamed"),
               ElementsAre(AllOf(QName("(anonymous struct)::InUnnamed"),
                                 WithKind(SymbolKind::Field))));
 }
 
-TEST_F(WorkspaceSymbolsTest, InMainFile) {
-  addFile("foo.cpp", R"cpp(
+TEST(WorkspaceSymbols, InMainFile) {
+  TestTU TU;
+  TU.Code = R"cpp(
       int test() {}
-      static test2() {}
-      )cpp");
-  EXPECT_THAT(getSymbols("test"), ElementsAre(QName("test"), QName("test2")));
+      static void test2() {}
+      )cpp";
+  EXPECT_THAT(getSymbols(TU, "test"),
+              ElementsAre(QName("test"), QName("test2")));
 }
 
-TEST_F(WorkspaceSymbolsTest, Namespaces) {
-  addFile("foo.h", R"cpp(
+TEST(WorkspaceSymbols, Namespaces) {
+  TestTU TU;
+  TU.AdditionalFiles["foo.h"] = R"cpp(
       namespace ans1 {
         int ai1;
       namespace ans2 {
         int ai2;
       }
       }
-      )cpp");
-  addFile("foo.cpp", R"cpp(
+      )cpp";
+  TU.Code = R"cpp(
       #include "foo.h"
-      )cpp");
-  EXPECT_THAT(getSymbols("a"),
+      )cpp";
+  EXPECT_THAT(getSymbols(TU, "a"),
               UnorderedElementsAre(QName("ans1"), QName("ans1::ai1"),
                                    QName("ans1::ans2"),
                                    QName("ans1::ans2::ai2")));
-  EXPECT_THAT(getSymbols("::"), ElementsAre(QName("ans1")));
-  EXPECT_THAT(getSymbols("::a"), ElementsAre(QName("ans1")));
-  EXPECT_THAT(getSymbols("ans1::"),
+  EXPECT_THAT(getSymbols(TU, "::"), ElementsAre(QName("ans1")));
+  EXPECT_THAT(getSymbols(TU, "::a"), ElementsAre(QName("ans1")));
+  EXPECT_THAT(getSymbols(TU, "ans1::"),
               UnorderedElementsAre(QName("ans1::ai1"), QName("ans1::ans2")));
-  EXPECT_THAT(getSymbols("::ans1"), ElementsAre(QName("ans1")));
-  EXPECT_THAT(getSymbols("::ans1::"),
+  EXPECT_THAT(getSymbols(TU, "::ans1"), ElementsAre(QName("ans1")));
+  EXPECT_THAT(getSymbols(TU, "::ans1::"),
               UnorderedElementsAre(QName("ans1::ai1"), QName("ans1::ans2")));
-  EXPECT_THAT(getSymbols("::ans1::ans2"), ElementsAre(QName("ans1::ans2")));
-  EXPECT_THAT(getSymbols("::ans1::ans2::"),
+  EXPECT_THAT(getSymbols(TU, "::ans1::ans2"), ElementsAre(QName("ans1::ans2")));
+  EXPECT_THAT(getSymbols(TU, "::ans1::ans2::"),
               ElementsAre(QName("ans1::ans2::ai2")));
 }
 
-TEST_F(WorkspaceSymbolsTest, AnonymousNamespace) {
-  addFile("foo.cpp", R"cpp(
+TEST(WorkspaceSymbols, AnonymousNamespace) {
+  TestTU TU;
+  TU.Code = R"cpp(
       namespace {
       void test() {}
       }
-      )cpp");
-  EXPECT_THAT(getSymbols("test"), ElementsAre(QName("test")));
+      )cpp";
+  EXPECT_THAT(getSymbols(TU, "test"), ElementsAre(QName("test")));
 }
 
-TEST_F(WorkspaceSymbolsTest, MultiFile) {
-  addFile("foo.h", R"cpp(
+TEST(WorkspaceSymbols, MultiFile) {
+  TestTU TU;
+  TU.AdditionalFiles["foo.h"] = R"cpp(
       int foo() {
       }
-      )cpp");
-  addFile("foo2.h", R"cpp(
+      )cpp";
+  TU.AdditionalFiles["foo2.h"] = R"cpp(
       int foo2() {
       }
-      )cpp");
-  addFile("foo.cpp", R"cpp(
+      )cpp";
+  TU.Code = R"cpp(
       #include "foo.h"
       #include "foo2.h"
-      )cpp");
-  EXPECT_THAT(getSymbols("foo"),
+      )cpp";
+  EXPECT_THAT(getSymbols(TU, "foo"),
               UnorderedElementsAre(QName("foo"), QName("foo2")));
 }
 
-TEST_F(WorkspaceSymbolsTest, GlobalNamespaceQueries) {
-  addFile("foo.h", R"cpp(
+TEST(WorkspaceSymbols, GlobalNamespaceQueries) {
+  TestTU TU;
+  TU.AdditionalFiles["foo.h"] = R"cpp(
       int foo() {
       }
       class Foo {
@@ -205,21 +190,22 @@ TEST_F(WorkspaceSymbolsTest, GlobalNamespaceQueries) {
       int foo2() {
       }
       }
-      )cpp");
-  addFile("foo.cpp", R"cpp(
+      )cpp";
+  TU.Code = R"cpp(
       #include "foo.h"
-      )cpp");
-  EXPECT_THAT(getSymbols("::"),
+      )cpp";
+  EXPECT_THAT(getSymbols(TU, "::"),
               UnorderedElementsAre(
                   AllOf(QName("Foo"), WithKind(SymbolKind::Class)),
                   AllOf(QName("foo"), WithKind(SymbolKind::Function)),
                   AllOf(QName("ns"), WithKind(SymbolKind::Namespace))));
-  EXPECT_THAT(getSymbols(":"), IsEmpty());
-  EXPECT_THAT(getSymbols(""), IsEmpty());
+  EXPECT_THAT(getSymbols(TU, ":"), IsEmpty());
+  EXPECT_THAT(getSymbols(TU, ""), IsEmpty());
 }
 
-TEST_F(WorkspaceSymbolsTest, Enums) {
-  addFile("foo.h", R"cpp(
+TEST(WorkspaceSymbols, Enums) {
+  TestTU TU;
+  TU.AdditionalFiles["foo.h"] = R"cpp(
     enum {
       Red
     };
@@ -240,63 +226,66 @@ TEST_F(WorkspaceSymbolsTest, Enums) {
         White
       };
     }
-      )cpp");
-  addFile("foo.cpp", R"cpp(
+      )cpp";
+  TU.Code = R"cpp(
       #include "foo.h"
-      )cpp");
-  EXPECT_THAT(getSymbols("Red"), ElementsAre(QName("Red")));
-  EXPECT_THAT(getSymbols("::Red"), ElementsAre(QName("Red")));
-  EXPECT_THAT(getSymbols("Green"), ElementsAre(QName("Green")));
-  EXPECT_THAT(getSymbols("Green"), ElementsAre(QName("Green")));
-  EXPECT_THAT(getSymbols("Color2::Yellow"),
+      )cpp";
+  EXPECT_THAT(getSymbols(TU, "Red"), ElementsAre(QName("Red")));
+  EXPECT_THAT(getSymbols(TU, "::Red"), ElementsAre(QName("Red")));
+  EXPECT_THAT(getSymbols(TU, "Green"), ElementsAre(QName("Green")));
+  EXPECT_THAT(getSymbols(TU, "Green"), ElementsAre(QName("Green")));
+  EXPECT_THAT(getSymbols(TU, "Color2::Yellow"),
               ElementsAre(QName("Color2::Yellow")));
-  EXPECT_THAT(getSymbols("Yellow"), ElementsAre(QName("Color2::Yellow")));
+  EXPECT_THAT(getSymbols(TU, "Yellow"), ElementsAre(QName("Color2::Yellow")));
 
-  EXPECT_THAT(getSymbols("ns::Black"), ElementsAre(QName("ns::Black")));
-  EXPECT_THAT(getSymbols("ns::Blue"), ElementsAre(QName("ns::Blue")));
-  EXPECT_THAT(getSymbols("ns::Color4::White"),
+  EXPECT_THAT(getSymbols(TU, "ns::Black"), ElementsAre(QName("ns::Black")));
+  EXPECT_THAT(getSymbols(TU, "ns::Blue"), ElementsAre(QName("ns::Blue")));
+  EXPECT_THAT(getSymbols(TU, "ns::Color4::White"),
               ElementsAre(QName("ns::Color4::White")));
 }
 
-TEST_F(WorkspaceSymbolsTest, Ranking) {
-  addFile("foo.h", R"cpp(
+TEST(WorkspaceSymbols, Ranking) {
+  TestTU TU;
+  TU.AdditionalFiles["foo.h"] = R"cpp(
       namespace ns{}
       void func();
-      )cpp");
-  addFile("foo.cpp", R"cpp(
+      )cpp";
+  TU.Code = R"cpp(
       #include "foo.h"
-      )cpp");
-  EXPECT_THAT(getSymbols("::"), ElementsAre(QName("func"), QName("ns")));
+      )cpp";
+  EXPECT_THAT(getSymbols(TU, "::"), ElementsAre(QName("func"), QName("ns")));
 }
 
-TEST_F(WorkspaceSymbolsTest, WithLimit) {
-  addFile("foo.h", R"cpp(
+TEST(WorkspaceSymbols, WithLimit) {
+  TestTU TU;
+  TU.AdditionalFiles["foo.h"] = R"cpp(
       int foo;
       int foo2;
-      )cpp");
-  addFile("foo.cpp", R"cpp(
+      )cpp";
+  TU.Code = R"cpp(
       #include "foo.h"
-      )cpp");
+      )cpp";
   // Foo is higher ranked because of exact name match.
-  EXPECT_THAT(getSymbols("foo"),
+  EXPECT_THAT(getSymbols(TU, "foo"),
               UnorderedElementsAre(
                   AllOf(QName("foo"), WithKind(SymbolKind::Variable)),
                   AllOf(QName("foo2"), WithKind(SymbolKind::Variable))));
 
-  Limit = 1;
-  EXPECT_THAT(getSymbols("foo"), ElementsAre(QName("foo")));
+  EXPECT_THAT(getSymbols(TU, "foo", 1), ElementsAre(QName("foo")));
 }
 
-TEST_F(WorkspaceSymbolsTest, TempSpecs) {
-  addFile("foo.h", R"cpp(
+TEST(WorkspaceSymbols, TempSpecs) {
+  TestTU TU;
+  TU.ExtraArgs = {"-xc++"};
+  TU.Code = R"cpp(
       template <typename T, typename U, int X = 5> class Foo {};
       template <typename T> class Foo<int, T> {};
       template <> class Foo<bool, int> {};
       template <> class Foo<bool, int, 3> {};
-      )cpp");
+      )cpp";
   // Foo is higher ranked because of exact name match.
   EXPECT_THAT(
-      getSymbols("Foo"),
+      getSymbols(TU, "Foo"),
       UnorderedElementsAre(
           AllOf(QName("Foo"), WithKind(SymbolKind::Class)),
           AllOf(QName("Foo<int, T>"), WithKind(SymbolKind::Class)),
@@ -304,31 +293,14 @@ TEST_F(WorkspaceSymbolsTest, TempSpecs) {
           AllOf(QName("Foo<bool, int, 3>"), WithKind(SymbolKind::Class))));
 }
 
-namespace {
-class DocumentSymbolsTest : public ::testing::Test {
-public:
-  DocumentSymbolsTest() : Server(CDB, FS, optsForTests()) {}
-
-protected:
-  MockFS FS;
-  MockCompilationDatabase CDB;
-  ClangdServer Server;
-
-  std::vector<DocumentSymbol> getSymbols(PathRef File) {
-    EXPECT_TRUE(Server.blockUntilIdleForTest()) << "Waiting for preamble";
-    auto SymbolInfos = runDocumentSymbols(Server, File);
-    EXPECT_TRUE(bool(SymbolInfos)) << "documentSymbols returned an error";
-    return *SymbolInfos;
-  }
-
-  void addFile(llvm::StringRef FilePath, llvm::StringRef Contents) {
-    Server.addDocument(FilePath, Contents);
-  }
-};
-} // namespace
+std::vector<DocumentSymbol> getSymbols(ParsedAST AST) {
+  auto SymbolInfos = getDocumentSymbols(AST);
+  EXPECT_TRUE(bool(SymbolInfos)) << "documentSymbols returned an error";
+  return *SymbolInfos;
+}
 
-TEST_F(DocumentSymbolsTest, BasicSymbols) {
-  std::string FilePath = testPath("foo.cpp");
+TEST(DocumentSymbols, BasicSymbols) {
+  TestTU TU;
   Annotations Main(R"(
       class Foo;
       class Foo {
@@ -372,9 +344,9 @@ TEST_F(DocumentSymbolsTest, BasicSymbols) {
       } // namespace foo
     )");
 
-  addFile(FilePath, Main.code());
+  TU.Code = Main.code().str();
   EXPECT_THAT(
-      getSymbols(FilePath),
+      getSymbols(TU.build()),
       ElementsAreArray(
           {AllOf(WithName("Foo"), WithKind(SymbolKind::Class), Children()),
            AllOf(WithName("Foo"), WithKind(SymbolKind::Class),
@@ -416,8 +388,8 @@ TEST_F(DocumentSymbolsTest, BasicSymbols) {
                    AllOf(WithName("v2"), WithKind(SymbolKind::Namespace))))}));
 }
 
-TEST_F(DocumentSymbolsTest, DeclarationDefinition) {
-  std::string FilePath = testPath("foo.cpp");
+TEST(DocumentSymbols, DeclarationDefinition) {
+  TestTU TU;
   Annotations Main(R"(
       class Foo {
         void $decl[[f]]();
@@ -426,9 +398,9 @@ TEST_F(DocumentSymbolsTest, DeclarationDefinition) {
       }
     )");
 
-  addFile(FilePath, Main.code());
+  TU.Code = Main.code().str();
   EXPECT_THAT(
-      getSymbols(FilePath),
+      getSymbols(TU.build()),
       ElementsAre(
           AllOf(WithName("Foo"), WithKind(SymbolKind::Class),
                 Children(AllOf(WithName("f"), WithKind(SymbolKind::Method),
@@ -437,48 +409,45 @@ TEST_F(DocumentSymbolsTest, DeclarationDefinition) {
                 SymNameRange(Main.range("def")))));
 }
 
-TEST_F(DocumentSymbolsTest, Concepts) {
-  CDB.ExtraClangFlags = {"-std=c++20"};
-  std::string FilePath = testPath("foo.cpp");
-  addFile(FilePath,
-          "template <typename T> concept C = requires(T t) { t.foo(); };");
+TEST(DocumentSymbols, Concepts) {
+  TestTU TU;
+  TU.ExtraArgs = {"-std=c++20"};
+  TU.Code = "template <typename T> concept C = requires(T t) { t.foo(); };";
 
-  EXPECT_THAT(getSymbols(FilePath), ElementsAre(WithName("C")));
+  EXPECT_THAT(getSymbols(TU.build()), ElementsAre(WithName("C")));
 }
 
-TEST_F(DocumentSymbolsTest, ExternSymbol) {
-  std::string FilePath = testPath("foo.cpp");
-  addFile(testPath("foo.h"), R"cpp(
+TEST(DocumentSymbols, ExternSymbol) {
+  TestTU TU;
+  TU.AdditionalFiles["foo.h"] = R"cpp(
       extern int var;
-      )cpp");
-  addFile(FilePath, R"cpp(
+      )cpp";
+  TU.Code = R"cpp(
       #include "foo.h"
-      )cpp");
+      )cpp";
 
-  EXPECT_THAT(getSymbols(FilePath), IsEmpty());
+  EXPECT_THAT(getSymbols(TU.build()), IsEmpty());
 }
 
-TEST_F(DocumentSymbolsTest, NoLocals) {
-  std::string FilePath = testPath("foo.cpp");
-  addFile(FilePath,
-          R"cpp(
+TEST(DocumentSymbols, NoLocals) {
+  TestTU TU;
+  TU.Code = R"cpp(
       void test(int FirstParam, int SecondParam) {
         struct LocalClass {};
         int local_var;
-      })cpp");
-  EXPECT_THAT(getSymbols(FilePath), ElementsAre(WithName("test")));
+      })cpp";
+  EXPECT_THAT(getSymbols(TU.build()), ElementsAre(WithName("test")));
 }
 
-TEST_F(DocumentSymbolsTest, Unnamed) {
-  std::string FilePath = testPath("foo.h");
-  addFile(FilePath,
-          R"cpp(
+TEST(DocumentSymbols, Unnamed) {
+  TestTU TU;
+  TU.Code = R"cpp(
       struct {
         int InUnnamed;
       } UnnamedStruct;
-      )cpp");
+      )cpp";
   EXPECT_THAT(
-      getSymbols(FilePath),
+      getSymbols(TU.build()),
       ElementsAre(
           AllOf(WithName("(anonymous struct)"), WithKind(SymbolKind::Struct),
                 Children(AllOf(WithName("InUnnamed"),
@@ -487,26 +456,23 @@ TEST_F(DocumentSymbolsTest, Unnamed) {
                 Children())));
 }
 
-TEST_F(DocumentSymbolsTest, InHeaderFile) {
-  addFile(testPath("bar.h"), R"cpp(
+TEST(DocumentSymbols, InHeaderFile) {
+  TestTU TU;
+  TU.AdditionalFiles["bar.h"] = R"cpp(
       int foo() {
       }
-      )cpp");
-  std::string FilePath = testPath("foo.h");
-  addFile(FilePath, R"cpp(
+      )cpp";
+  TU.Code = R"cpp(
       #include "bar.h"
       int test() {
       }
-      )cpp");
-  addFile(testPath("foo.cpp"), R"cpp(
-      #include "foo.h"
-      )cpp");
-  EXPECT_THAT(getSymbols(FilePath), ElementsAre(WithName("test")));
+      )cpp";
+  EXPECT_THAT(getSymbols(TU.build()), ElementsAre(WithName("test")));
 }
 
-TEST_F(DocumentSymbolsTest, Template) {
-  std::string FilePath = testPath("foo.cpp");
-  addFile(FilePath, R"(
+TEST(DocumentSymbols, Template) {
+  TestTU TU;
+  TU.Code = R"(
     template <class T> struct Tmpl {T x = 0;};
     template <> struct Tmpl<int> {
       int y = 0;
@@ -523,9 +489,9 @@ TEST_F(DocumentSymbolsTest, Template) {
     int varTmpl = T();
     template <>
     double varTmpl<int> = 10.0;
-  )");
+  )";
   EXPECT_THAT(
-      getSymbols(FilePath),
+      getSymbols(TU.build()),
       ElementsAre(
           AllOf(WithName("Tmpl"), WithKind(SymbolKind::Struct),
                 Children(AllOf(WithName("x"), WithKind(SymbolKind::Field)))),
@@ -541,9 +507,9 @@ TEST_F(DocumentSymbolsTest, Template) {
           AllOf(WithName("varTmpl<int>"), Children())));
 }
 
-TEST_F(DocumentSymbolsTest, Namespaces) {
-  std::string FilePath = testPath("foo.cpp");
-  addFile(FilePath, R"cpp(
+TEST(DocumentSymbols, Namespaces) {
+  TestTU TU;
+  TU.Code = R"cpp(
       namespace ans1 {
         int ai1;
       namespace ans2 {
@@ -565,9 +531,9 @@ TEST_F(DocumentSymbolsTest, Namespaces) {
       class Bar {};
       }
       }
-      )cpp");
+      )cpp";
   EXPECT_THAT(
-      getSymbols(FilePath),
+      getSymbols(TU.build()),
       ElementsAreArray<::testing::Matcher<DocumentSymbol>>(
           {AllOf(WithName("ans1"),
                  Children(AllOf(WithName("ai1"), Children()),
@@ -579,9 +545,9 @@ TEST_F(DocumentSymbolsTest, Namespaces) {
                  Children(AllOf(WithName("nb"), Children(WithName("Bar")))))}));
 }
 
-TEST_F(DocumentSymbolsTest, Enums) {
-  std::string FilePath = testPath("foo.cpp");
-  addFile(FilePath, R"(
+TEST(DocumentSymbols, Enums) {
+  TestTU TU;
+  TU.Code = R"(
       enum {
         Red
       };
@@ -596,9 +562,9 @@ TEST_F(DocumentSymbolsTest, Enums) {
         Black
       };
       }
-    )");
+    )";
   EXPECT_THAT(
-      getSymbols(FilePath),
+      getSymbols(TU.build()),
       ElementsAre(
           AllOf(WithName("(anonymous enum)"), Children(WithName("Red"))),
           AllOf(WithName("Color"), Children(WithName("Green"))),
@@ -607,8 +573,8 @@ TEST_F(DocumentSymbolsTest, Enums) {
                                                Children(WithName("Black")))))));
 }
 
-TEST_F(DocumentSymbolsTest, FromMacro) {
-  std::string FilePath = testPath("foo.cpp");
+TEST(DocumentSymbols, FromMacro) {
+  TestTU TU;
   Annotations Main(R"(
     #define FF(name) \
       class name##_Test {};
@@ -620,31 +586,31 @@ TEST_F(DocumentSymbolsTest, FromMacro) {
 
     FF2();
   )");
-  addFile(FilePath, Main.code());
+  TU.Code = Main.code().str();
   EXPECT_THAT(
-      getSymbols(FilePath),
+      getSymbols(TU.build()),
       ElementsAre(
           AllOf(WithName("abc_Test"), SymNameRange(Main.range("expansion"))),
           AllOf(WithName("Test"), SymNameRange(Main.range("spelling")))));
 }
 
-TEST_F(DocumentSymbolsTest, FuncTemplates) {
-  std::string FilePath = testPath("foo.cpp");
+TEST(DocumentSymbols, FuncTemplates) {
+  TestTU TU;
   Annotations Source(R"cpp(
     template <class T>
     T foo() {}
 
     auto x = foo<int>();
-    auto y = foo<double>()
+    auto y = foo<double>();
   )cpp");
-  addFile(FilePath, Source.code());
+  TU.Code = Source.code().str();
   // Make sure we only see the template declaration, not instantiations.
-  EXPECT_THAT(getSymbols(FilePath),
+  EXPECT_THAT(getSymbols(TU.build()),
               ElementsAre(WithName("foo"), WithName("x"), WithName("y")));
 }
 
-TEST_F(DocumentSymbolsTest, UsingDirectives) {
-  std::string FilePath = testPath("foo.cpp");
+TEST(DocumentSymbols, UsingDirectives) {
+  TestTU TU;
   Annotations Source(R"cpp(
     namespace ns {
       int foo;
@@ -655,24 +621,24 @@ TEST_F(DocumentSymbolsTest, UsingDirectives) {
     using namespace ::ns;     // check we don't loose qualifiers.
     using namespace ns_alias; // and namespace aliases.
   )cpp");
-  addFile(FilePath, Source.code());
-  EXPECT_THAT(getSymbols(FilePath),
+  TU.Code = Source.code().str();
+  EXPECT_THAT(getSymbols(TU.build()),
               ElementsAre(WithName("ns"), WithName("ns_alias"),
                           WithName("using namespace ::ns"),
                           WithName("using namespace ns_alias")));
 }
 
-TEST_F(DocumentSymbolsTest, TempSpecs) {
-  std::string FilePath = testPath("foo.cpp");
-  addFile(FilePath, R"cpp(
+TEST(DocumentSymbols, TempSpecs) {
+  TestTU TU;
+  TU.Code = R"cpp(
       template <typename T, typename U, int X = 5> class Foo {};
       template <typename T> class Foo<int, T> {};
       template <> class Foo<bool, int> {};
       template <> class Foo<bool, int, 3> {};
-      )cpp");
+      )cpp";
   // Foo is higher ranked because of exact name match.
   EXPECT_THAT(
-      getSymbols(FilePath),
+      getSymbols(TU.build()),
       UnorderedElementsAre(
           AllOf(WithName("Foo"), WithKind(SymbolKind::Class)),
           AllOf(WithName("Foo<int, T>"), WithKind(SymbolKind::Class)),
@@ -680,9 +646,9 @@ TEST_F(DocumentSymbolsTest, TempSpecs) {
           AllOf(WithName("Foo<bool, int, 3>"), WithKind(SymbolKind::Class))));
 }
 
-TEST_F(DocumentSymbolsTest, Qualifiers) {
-  std::string FilePath = testPath("foo.cpp");
-  addFile(FilePath, R"cpp(
+TEST(DocumentSymbols, Qualifiers) {
+  TestTU TU;
+  TU.Code = R"cpp(
     namespace foo { namespace bar {
       struct Cls;
 
@@ -702,10 +668,10 @@ TEST_F(DocumentSymbolsTest, Qualifiers) {
 
     namespace alias = foo::bar;
     int ::alias::func4() { return 40; }
-  )cpp");
+  )cpp";
 
   // All the qualifiers should be preserved exactly as written.
-  EXPECT_THAT(getSymbols(FilePath),
+  EXPECT_THAT(getSymbols(TU.build()),
               UnorderedElementsAre(
                   WithName("foo"), WithName("foo::bar::Cls"),
                   WithName("foo::bar::func1"), WithName("::foo::bar::func2"),
@@ -713,9 +679,9 @@ TEST_F(DocumentSymbolsTest, Qualifiers) {
                   WithName("alias"), WithName("::alias::func4")));
 }
 
-TEST_F(DocumentSymbolsTest, QualifiersWithTemplateArgs) {
-  std::string FilePath = testPath("foo.cpp");
-  addFile(FilePath, R"cpp(
+TEST(DocumentSymbols, QualifiersWithTemplateArgs) {
+  TestTU TU;
+  TU.Code = R"cpp(
       template <typename T, typename U = double> class Foo;
 
       template <>
@@ -736,9 +702,9 @@ TEST_F(DocumentSymbolsTest, QualifiersWithTemplateArgs) {
       using Foo_type = Foo<int>;
       // If the whole type is aliased, this should be preserved too!
       int Foo_type::method3() { return 30; }
-      )cpp");
+      )cpp";
   EXPECT_THAT(
-      getSymbols(FilePath),
+      getSymbols(TU.build()),
       UnorderedElementsAre(WithName("Foo"), WithName("Foo<int, double>"),
                            WithName("int_type"),
                            WithName("Foo<int_type, double>::method1"),
@@ -746,5 +712,6 @@ TEST_F(DocumentSymbolsTest, QualifiersWithTemplateArgs) {
                            WithName("Foo_type::method3")));
 }
 
+} // namespace
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/HoverTests.cpp b/clang-tools-extra/clangd/unittests/HoverTests.cpp
index dc818ea661938..19ab6d63947b7 100644
--- a/clang-tools-extra/clangd/unittests/HoverTests.cpp
+++ b/clang-tools-extra/clangd/unittests/HoverTests.cpp
@@ -26,6 +26,8 @@ namespace clang {
 namespace clangd {
 namespace {
 
+using PassMode = HoverInfo::PassType::PassMode;
+
 TEST(Hover, Structured) {
   struct {
     const char *const Code;
@@ -696,7 +698,125 @@ class Foo {})cpp";
          HI.Parameters->back().Name = "v";
          HI.AccessSpecifier = "public";
        }},
-  };
+      {// Field type initializer.
+       R"cpp(
+          struct X { int x = 2; };
+          X ^[[x]];
+          )cpp",
+       [](HoverInfo &HI) {
+         HI.Name = "x";
+         HI.Kind = index::SymbolKind::Variable;
+         HI.NamespaceScope = "";
+         HI.Definition = "X x";
+         HI.Type = "struct X";
+       }},
+      {// Don't crash on null types.
+       R"cpp(auto [^[[x]]] = 1; /*error-ok*/)cpp",
+       [](HoverInfo &HI) {
+         HI.Name = "x";
+         HI.Kind = index::SymbolKind::Variable;
+         HI.NamespaceScope = "";
+         HI.Definition = "";
+         HI.Type = "NULL TYPE";
+         // Bindings are in theory public members of an anonymous struct.
+         HI.AccessSpecifier = "public";
+       }},
+      {// Extra info for function call.
+       R"cpp(
+          void fun(int arg_a, int &arg_b) {};
+          void code() {
+            int a = 1, b = 2;
+            fun(a, [[^b]]);
+          }
+          )cpp",
+       [](HoverInfo &HI) {
+         HI.Name = "b";
+         HI.Kind = index::SymbolKind::Variable;
+         HI.NamespaceScope = "";
+         HI.Definition = "int b = 2";
+         HI.LocalScope = "code::";
+         HI.Value = "2";
+         HI.Type = "int";
+         HI.CalleeArgInfo.emplace();
+         HI.CalleeArgInfo->Name = "arg_b";
+         HI.CalleeArgInfo->Type = "int &";
+         HI.CallPassType.emplace();
+         HI.CallPassType->PassBy = PassMode::Ref;
+         HI.CallPassType->Converted = false;
+       }},
+      {// Extra info for method call.
+       R"cpp(
+          class C {
+           public:
+            void fun(int arg_a = 3, int arg_b = 4) {}
+          };
+          void code() {
+            int a = 1, b = 2;
+            C c;
+            c.fun([[^a]], b);
+          }
+          )cpp",
+       [](HoverInfo &HI) {
+         HI.Name = "a";
+         HI.Kind = index::SymbolKind::Variable;
+         HI.NamespaceScope = "";
+         HI.Definition = "int a = 1";
+         HI.LocalScope = "code::";
+         HI.Value = "1";
+         HI.Type = "int";
+         HI.CalleeArgInfo.emplace();
+         HI.CalleeArgInfo->Name = "arg_a";
+         HI.CalleeArgInfo->Type = "int";
+         HI.CalleeArgInfo->Default = "3";
+         HI.CallPassType.emplace();
+         HI.CallPassType->PassBy = PassMode::Value;
+         HI.CallPassType->Converted = false;
+       }},
+      {// Dont crash on invalid decl
+       R"cpp(
+        // error-ok
+        struct Foo {
+          Bar [[x^x]];
+        };)cpp",
+       [](HoverInfo &HI) {
+         HI.Name = "xx";
+         HI.Kind = index::SymbolKind::Field;
+         HI.NamespaceScope = "";
+         HI.Definition = "int xx";
+         HI.LocalScope = "Foo::";
+         HI.Type = "int";
+         HI.AccessSpecifier = "public";
+       }},
+      {R"cpp(
+        // error-ok
+        struct Foo {
+          Bar xx;
+          int [[y^y]];
+        };)cpp",
+       [](HoverInfo &HI) {
+         HI.Name = "yy";
+         HI.Kind = index::SymbolKind::Field;
+         HI.NamespaceScope = "";
+         HI.Definition = "int yy";
+         HI.LocalScope = "Foo::";
+         HI.Type = "int";
+         HI.AccessSpecifier = "public";
+       }},
+       {// No crash on InitListExpr.
+        R"cpp(
+          struct Foo {
+            int a[10];
+          };
+          constexpr Foo k2 = {
+            ^[[{]]1} // FIXME: why the hover range is 1 character?
+          };
+         )cpp",
+       [](HoverInfo &HI) {
+         HI.Name = "expression";
+         HI.Kind = index::SymbolKind::Unknown;
+         HI.Type = "int [10]";
+         HI.Value = "{1}";
+       }}};
   for (const auto &Case : Cases) {
     SCOPED_TRACE(Case.Code);
 
@@ -729,6 +849,85 @@ class Foo {})cpp";
     EXPECT_EQ(H->Size, Expected.Size);
     EXPECT_EQ(H->Offset, Expected.Offset);
     EXPECT_EQ(H->AccessSpecifier, Expected.AccessSpecifier);
+    EXPECT_EQ(H->CalleeArgInfo, Expected.CalleeArgInfo);
+    EXPECT_EQ(H->CallPassType, Expected.CallPassType);
+  }
+}
+
+TEST(Hover, CallPassType) {
+  const llvm::StringRef CodePrefix = R"cpp(
+class Base {};
+class Derived : public Base {};
+class CustomClass {
+ public:
+  CustomClass() {}
+  CustomClass(const Base &x) {}
+  CustomClass(int &x) {}
+  CustomClass(float x) {}
+};
+
+void int_by_ref(int &x) {}
+void int_by_const_ref(const int &x) {}
+void int_by_value(int x) {}
+void base_by_ref(Base &x) {}
+void base_by_const_ref(const Base &x) {}
+void base_by_value(Base x) {}
+void float_by_value(float x) {}
+void custom_by_value(CustomClass x) {}
+
+void fun() {
+  int int_x;
+  int &int_ref = int_x;
+  const int &int_const_ref = int_x;
+  Base base;
+  const Base &base_const_ref = base;
+  Derived derived;
+  float float_x;
+)cpp";
+  const llvm::StringRef CodeSuffix = "}";
+
+  struct {
+    const char *const Code;
+    HoverInfo::PassType::PassMode PassBy;
+    bool Converted;
+  } Tests[] = {
+      // Integer tests
+      {"int_by_value([[^int_x]]);", PassMode::Value, false},
+      {"int_by_ref([[^int_x]]);", PassMode::Ref, false},
+      {"int_by_const_ref([[^int_x]]);", PassMode::ConstRef, false},
+      {"int_by_value([[^int_ref]]);", PassMode::Value, false},
+      {"int_by_const_ref([[^int_ref]]);", PassMode::ConstRef, false},
+      {"int_by_const_ref([[^int_ref]]);", PassMode::ConstRef, false},
+      {"int_by_const_ref([[^int_const_ref]]);", PassMode::ConstRef, false},
+      // Custom class tests
+      {"base_by_ref([[^base]]);", PassMode::Ref, false},
+      {"base_by_const_ref([[^base]]);", PassMode::ConstRef, false},
+      {"base_by_const_ref([[^base_const_ref]]);", PassMode::ConstRef, false},
+      {"base_by_value([[^base]]);", PassMode::Value, false},
+      {"base_by_value([[^base_const_ref]]);", PassMode::Value, false},
+      {"base_by_ref([[^derived]]);", PassMode::Ref, false},
+      {"base_by_const_ref([[^derived]]);", PassMode::ConstRef, false},
+      {"base_by_value([[^derived]]);", PassMode::Value, false},
+      // Converted tests
+      {"float_by_value([[^int_x]]);", PassMode::Value, true},
+      {"float_by_value([[^int_ref]]);", PassMode::Value, true},
+      {"float_by_value([[^int_const_ref]]);", PassMode::Value, true},
+      {"custom_by_value([[^int_x]]);", PassMode::Ref, true},
+      {"custom_by_value([[^float_x]]);", PassMode::Value, true},
+      {"custom_by_value([[^base]]);", PassMode::ConstRef, true},
+  };
+  for (const auto &Test : Tests) {
+    SCOPED_TRACE(Test.Code);
+
+    const auto Code = (CodePrefix + Test.Code + CodeSuffix).str();
+    Annotations T(Code);
+    TestTU TU = TestTU::withCode(T.code());
+    TU.ExtraArgs.push_back("-std=c++17");
+    auto AST = TU.build();
+    auto H = getHover(AST, T.point(), format::getLLVMStyle(), nullptr);
+    ASSERT_TRUE(H);
+    EXPECT_EQ(H->CallPassType->PassBy, Test.PassBy);
+    EXPECT_EQ(H->CallPassType->Converted, Test.Converted);
   }
 }
 
@@ -978,13 +1177,14 @@ TEST(Hover, All) {
             HI.LocalScope = "Foo::";
             HI.Type = "int";
             HI.Definition = "int x";
-            HI.Value = "{1}";
+            // FIXME: Initializer for x is a DesignatedInitListExpr, hence it is
+            // of struct type and omitted.
           }},
       {
           R"cpp(// Field, field designator
-            struct Foo { int x; };
+            struct Foo { int x; int y; };
             int main() {
-              Foo bar = { .^[[x]] = 2 };
+              Foo bar = { .^[[x]] = 2, .y = 2 };
             }
           )cpp",
           [](HoverInfo &HI) {
@@ -994,7 +1194,6 @@ TEST(Hover, All) {
             HI.LocalScope = "Foo::";
             HI.Type = "int";
             HI.Definition = "int x";
-            HI.Value = "{2}";
           }},
       {
           R"cpp(// Method call
@@ -1592,7 +1791,6 @@ TEST(Hover, All) {
             HI.LocalScope = "test::";
             HI.Type = "struct Test &&";
             HI.Definition = "Test &&test = {}";
-            HI.Value = "{}";
           }},
       {
           R"cpp(// auto on alias
@@ -1651,7 +1849,6 @@ TEST(Hover, All) {
             HI.NamespaceScope = "";
             HI.Name = "foo";
             HI.Type = "cls<cls<cls<int>>>";
-            HI.Value = "{}";
           }},
       {
           R"cpp(// type of nested templates.
@@ -2022,6 +2219,106 @@ protected: int method())",
 
 // In namespace ns1
 private: union foo {})",
+      },
+      {
+          [](HoverInfo &HI) {
+            HI.Kind = index::SymbolKind::Variable;
+            HI.Name = "foo";
+            HI.Definition = "int foo = 3";
+            HI.LocalScope = "test::Bar::";
+            HI.Value = "3";
+            HI.Type = "int";
+            HI.CalleeArgInfo.emplace();
+            HI.CalleeArgInfo->Name = "arg_a";
+            HI.CalleeArgInfo->Type = "int";
+            HI.CalleeArgInfo->Default = "7";
+            HI.CallPassType.emplace();
+            HI.CallPassType->PassBy = PassMode::Value;
+            HI.CallPassType->Converted = false;
+          },
+          R"(variable foo
+
+Type: int
+Value = 3
+Passed as arg_a
+
+// In test::Bar
+int foo = 3)",
+      },
+      {
+          [](HoverInfo &HI) {
+            HI.Kind = index::SymbolKind::Variable;
+            HI.Name = "foo";
+            HI.Definition = "int foo = 3";
+            HI.LocalScope = "test::Bar::";
+            HI.Value = "3";
+            HI.Type = "int";
+            HI.CalleeArgInfo.emplace();
+            HI.CalleeArgInfo->Name = "arg_a";
+            HI.CalleeArgInfo->Type = "int";
+            HI.CalleeArgInfo->Default = "7";
+            HI.CallPassType.emplace();
+            HI.CallPassType->PassBy = PassMode::Ref;
+            HI.CallPassType->Converted = false;
+          },
+          R"(variable foo
+
+Type: int
+Value = 3
+Passed by reference as arg_a
+
+// In test::Bar
+int foo = 3)",
+      },
+      {
+          [](HoverInfo &HI) {
+            HI.Kind = index::SymbolKind::Variable;
+            HI.Name = "foo";
+            HI.Definition = "int foo = 3";
+            HI.LocalScope = "test::Bar::";
+            HI.Value = "3";
+            HI.Type = "int";
+            HI.CalleeArgInfo.emplace();
+            HI.CalleeArgInfo->Name = "arg_a";
+            HI.CalleeArgInfo->Type = "int";
+            HI.CalleeArgInfo->Default = "7";
+            HI.CallPassType.emplace();
+            HI.CallPassType->PassBy = PassMode::Value;
+            HI.CallPassType->Converted = true;
+          },
+          R"(variable foo
+
+Type: int
+Value = 3
+Passed as arg_a (converted to int)
+
+// In test::Bar
+int foo = 3)",
+      },
+      {
+          [](HoverInfo &HI) {
+            HI.Kind = index::SymbolKind::Variable;
+            HI.Name = "foo";
+            HI.Definition = "int foo = 3";
+            HI.LocalScope = "test::Bar::";
+            HI.Value = "3";
+            HI.Type = "int";
+            HI.CalleeArgInfo.emplace();
+            HI.CalleeArgInfo->Name = "arg_a";
+            HI.CalleeArgInfo->Type = "int";
+            HI.CalleeArgInfo->Default = "7";
+            HI.CallPassType.emplace();
+            HI.CallPassType->PassBy = PassMode::ConstRef;
+            HI.CallPassType->Converted = true;
+          },
+          R"(variable foo
+
+Type: int
+Value = 3
+Passed by const reference as arg_a (converted to int)
+
+// In test::Bar
+int foo = 3)",
       }};
 
   for (const auto &C : Cases) {
diff --git a/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp b/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp
index 2f3fbc2a029ca..65d9cffeedc73 100644
--- a/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp
@@ -11,6 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "../../clang-tidy/ClangTidyCheck.h"
 #include "../../clang-tidy/ClangTidyModule.h"
 #include "../../clang-tidy/ClangTidyModuleRegistry.h"
 #include "AST.h"
diff --git a/clang-tools-extra/clangd/unittests/PreambleTests.cpp b/clang-tools-extra/clangd/unittests/PreambleTests.cpp
index 5bbcb292610e4..8482a1cc8237c 100644
--- a/clang-tools-extra/clangd/unittests/PreambleTests.cpp
+++ b/clang-tools-extra/clangd/unittests/PreambleTests.cpp
@@ -125,7 +125,7 @@ TEST(PreamblePatchTest, IncludeParsing) {
         #/**/include <b.h>)cpp",
   };
 
-  for (const auto Case : Cases) {
+  for (const auto &Case : Cases) {
     Annotations Test(Case);
     const auto Code = Test.code();
     SCOPED_TRACE(Code);
@@ -230,8 +230,8 @@ std::string getPreamblePatch(llvm::StringRef Baseline,
 TEST(PreamblePatchTest, Define) {
   // BAR should be defined while parsing the AST.
   struct {
-    llvm::StringLiteral Contents;
-    llvm::StringLiteral ExpectedPatch;
+    const char *const Contents;
+    const char *const ExpectedPatch;
   } Cases[] = {
       {
           R"cpp(
@@ -270,7 +270,7 @@ TEST(PreamblePatchTest, Define) {
     SCOPED_TRACE(Case.Contents);
     Annotations Modified(Case.Contents);
     EXPECT_THAT(getPreamblePatch("", Modified.code()),
-                MatchesRegex(Case.ExpectedPatch.str()));
+                MatchesRegex(Case.ExpectedPatch));
 
     auto AST = createPatchedAST("", Modified.code());
     ASSERT_TRUE(AST);
@@ -304,8 +304,8 @@ TEST(PreamblePatchTest, OrderingPreserved) {
 
 TEST(PreamblePatchTest, LocateMacroAtWorks) {
   struct {
-    llvm::StringLiteral Baseline;
-    llvm::StringLiteral Modified;
+    const char *const Baseline;
+    const char *const Modified;
   } Cases[] = {
       // Addition of new directive
       {
@@ -417,8 +417,8 @@ TEST(PreamblePatchTest, LocateMacroAtDeletion) {
 
 TEST(PreamblePatchTest, RefsToMacros) {
   struct {
-    llvm::StringLiteral Baseline;
-    llvm::StringLiteral Modified;
+    const char *const Baseline;
+    const char *const Modified;
   } Cases[] = {
       // Newly added
       {
@@ -491,8 +491,8 @@ TEST(TranslatePreamblePatchLocation, Simple) {
 
 TEST(PreamblePatch, ModifiedBounds) {
   struct {
-    llvm::StringLiteral Baseline;
-    llvm::StringLiteral Modified;
+    const char *const Baseline;
+    const char *const Modified;
   } Cases[] = {
       // Size increased
       {
diff --git a/clang-tools-extra/clangd/unittests/SyncAPI.cpp b/clang-tools-extra/clangd/unittests/SyncAPI.cpp
index e976b5ab93894..36b028bd4d878 100644
--- a/clang-tools-extra/clangd/unittests/SyncAPI.cpp
+++ b/clang-tools-extra/clangd/unittests/SyncAPI.cpp
@@ -105,23 +105,16 @@ llvm::Expected<FileEdits> runRename(ClangdServer &Server, PathRef File,
   return std::move(*Result);
 }
 
-std::string runDumpAST(ClangdServer &Server, PathRef File) {
-  llvm::Optional<std::string> Result;
-  Server.dumpAST(File, capture(Result));
+llvm::Expected<tooling::Replacements>
+runFormatFile(ClangdServer &Server, PathRef File, StringRef Code) {
+  llvm::Optional<llvm::Expected<tooling::Replacements>> Result;
+  Server.formatFile(File, Code, capture(Result));
   return std::move(*Result);
 }
 
-llvm::Expected<std::vector<SymbolInformation>>
-runWorkspaceSymbols(ClangdServer &Server, llvm::StringRef Query, int Limit) {
-  llvm::Optional<llvm::Expected<std::vector<SymbolInformation>>> Result;
-  Server.workspaceSymbols(Query, Limit, capture(Result));
-  return std::move(*Result);
-}
-
-llvm::Expected<std::vector<DocumentSymbol>>
-runDocumentSymbols(ClangdServer &Server, PathRef File) {
-  llvm::Optional<llvm::Expected<std::vector<DocumentSymbol>>> Result;
-  Server.documentSymbols(File, capture(Result));
+std::string runDumpAST(ClangdServer &Server, PathRef File) {
+  llvm::Optional<std::string> Result;
+  Server.dumpAST(File, capture(Result));
   return std::move(*Result);
 }
 
diff --git a/clang-tools-extra/clangd/unittests/SyncAPI.h b/clang-tools-extra/clangd/unittests/SyncAPI.h
index 22b03c6f1fa5e..e16b9d8ff19b0 100644
--- a/clang-tools-extra/clangd/unittests/SyncAPI.h
+++ b/clang-tools-extra/clangd/unittests/SyncAPI.h
@@ -44,13 +44,10 @@ llvm::Expected<FileEdits> runRename(ClangdServer &Server, PathRef File,
                                     Position Pos, StringRef NewName,
                                     const clangd::RenameOptions &RenameOpts);
 
-std::string runDumpAST(ClangdServer &Server, PathRef File);
-
-llvm::Expected<std::vector<SymbolInformation>>
-runWorkspaceSymbols(ClangdServer &Server, StringRef Query, int Limit);
+llvm::Expected<tooling::Replacements>
+runFormatFile(ClangdServer &Server, PathRef File, StringRef Code);
 
-Expected<std::vector<DocumentSymbol>> runDocumentSymbols(ClangdServer &Server,
-                                                         PathRef File);
+std::string runDumpAST(ClangdServer &Server, PathRef File);
 
 SymbolSlab runFuzzyFind(const SymbolIndex &Index, StringRef Query);
 SymbolSlab runFuzzyFind(const SymbolIndex &Index, const FuzzyFindRequest &Req);
diff --git a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp
index 53a5979c73188..f40377fd5d85b 100644
--- a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp
+++ b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp
@@ -65,8 +65,20 @@ MATCHER_P2(TUState, PreambleActivity, ASTActivity, "") {
   return true;
 }
 
+// Dummy ContextProvider to verify the provider is invoked & contexts are used.
+static Key<std::string> BoundPath;
+Context bindPath(PathRef F) {
+  return Context::current().derive(BoundPath, F.str());
+}
+llvm::StringRef boundPath() {
+  const std::string *V = Context::current().get(BoundPath);
+  return V ? *V : llvm::StringRef("");
+}
+
 TUScheduler::Options optsForTest() {
-  return TUScheduler::Options(ClangdServer::optsForTest());
+  TUScheduler::Options Opts(ClangdServer::optsForTest());
+  Opts.ContextProvider = bindPath;
+  return Opts;
 }
 
 class TUSchedulerTests : public ::testing::Test {
@@ -454,6 +466,7 @@ TEST_F(TUSchedulerTests, ManyUpdates) {
               [File, Nonce, Version(Inputs.Version), &Mut, &TotalUpdates,
                &LatestDiagVersion](std::vector<Diag>) {
                 EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));
+                EXPECT_EQ(File, boundPath());
 
                 std::lock_guard<std::mutex> Lock(Mut);
                 ++TotalUpdates;
@@ -474,6 +487,7 @@ TEST_F(TUSchedulerTests, ManyUpdates) {
               [File, Inputs, Nonce, &Mut,
                &TotalASTReads](Expected<InputsAndAST> AST) {
                 EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));
+                EXPECT_EQ(File, boundPath());
 
                 ASSERT_TRUE((bool)AST);
                 EXPECT_EQ(AST->Inputs.Contents, Inputs.Contents);
@@ -493,6 +507,7 @@ TEST_F(TUSchedulerTests, ManyUpdates) {
               [File, Inputs, Nonce, &Mut,
                &TotalPreambleReads](Expected<InputsAndPreamble> Preamble) {
                 EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));
+                EXPECT_EQ(File, boundPath());
 
                 ASSERT_TRUE((bool)Preamble);
                 EXPECT_EQ(Preamble->Contents, Inputs.Contents);
@@ -849,18 +864,22 @@ TEST_F(TUSchedulerTests, NoChangeDiags) {
 }
 
 TEST_F(TUSchedulerTests, Run) {
-  TUScheduler S(CDB, optsForTest());
+  auto Opts = optsForTest();
+  Opts.ContextProvider = bindPath;
+  TUScheduler S(CDB, Opts);
   std::atomic<int> Counter(0);
-  S.run("add 1", [&] { ++Counter; });
-  S.run("add 2", [&] { Counter += 2; });
+  S.run("add 1", /*Path=*/"", [&] { ++Counter; });
+  S.run("add 2", /*Path=*/"", [&] { Counter += 2; });
   ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
   EXPECT_EQ(Counter.load(), 3);
 
   Notification TaskRun;
   Key<int> TestKey;
   WithContextValue CtxWithKey(TestKey, 10);
-  S.run("props context", [&] {
+  const char *Path = "somepath";
+  S.run("props context", Path, [&] {
     EXPECT_EQ(Context::current().getExisting(TestKey), 10);
+    EXPECT_EQ(Path, boundPath());
     TaskRun.notify();
   });
   TaskRun.wait();
diff --git a/clang-tools-extra/clangd/unittests/TestFS.cpp b/clang-tools-extra/clangd/unittests/TestFS.cpp
index c436e9a7a9510..3b2fbc142a28f 100644
--- a/clang-tools-extra/clangd/unittests/TestFS.cpp
+++ b/clang-tools-extra/clangd/unittests/TestFS.cpp
@@ -79,13 +79,13 @@ const char *testRoot() {
 #endif
 }
 
-std::string testPath(PathRef File) {
+std::string testPath(PathRef File, llvm::sys::path::Style Style) {
   assert(llvm::sys::path::is_relative(File) && "FileName should be relative");
 
   llvm::SmallString<32> NativeFile = File;
-  llvm::sys::path::native(NativeFile);
+  llvm::sys::path::native(NativeFile, Style);
   llvm::SmallString<32> Path;
-  llvm::sys::path::append(Path, testRoot(), NativeFile);
+  llvm::sys::path::append(Path, Style, testRoot(), NativeFile);
   return std::string(Path.str());
 }
 
diff --git a/clang-tools-extra/clangd/unittests/TestFS.h b/clang-tools-extra/clangd/unittests/TestFS.h
index a487a361bfe1c..7972fe37c7fed 100644
--- a/clang-tools-extra/clangd/unittests/TestFS.h
+++ b/clang-tools-extra/clangd/unittests/TestFS.h
@@ -33,8 +33,7 @@ buildTestFS(llvm::StringMap<std::string> const &Files,
 // A VFS provider that returns TestFSes containing a provided set of files.
 class MockFS : public ThreadsafeFS {
 public:
-  IntrusiveRefCntPtr<llvm::vfs::FileSystem>
-  view(llvm::NoneType) const override {
+  IntrusiveRefCntPtr<llvm::vfs::FileSystem> viewImpl() const override {
     return buildTestFS(Files, Timestamps);
   }
 
@@ -70,7 +69,8 @@ class MockCompilationDatabase : public GlobalCompilationDatabase {
 const char *testRoot();
 
 // Returns a suitable absolute path for this OS.
-std::string testPath(PathRef File);
+std::string testPath(PathRef File,
+                     llvm::sys::path::Style = llvm::sys::path::Style::native);
 
 // unittest: is a scheme that refers to files relative to testRoot()
 // This anchor is used to force the linker to link in the generated object file
diff --git a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
index 8e68cfbdbeeea..a14ff13150d37 100644
--- a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
+++ b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
@@ -7,85 +7,286 @@
 //===----------------------------------------------------------------------===//
 
 #include "../TestTU.h"
+#include "TestFS.h"
+#include "index/Index.h"
+#include "index/Ref.h"
 #include "index/Serialization.h"
+#include "index/Symbol.h"
+#include "index/SymbolID.h"
 #include "index/remote/marshalling/Marshalling.h"
+#include "clang/Index/IndexSymbol.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/StringSaver.h"
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
+#include <cstring>
 
 namespace clang {
 namespace clangd {
 namespace remote {
 namespace {
 
+using llvm::sys::path::convert_to_slash;
+
+const char *testPathURI(llvm::StringRef Path,
+                        llvm::UniqueStringSaver &Strings) {
+  auto URI = URI::createFile(testPath(Path));
+  return Strings.save(URI.toString()).begin();
+}
+
+TEST(RemoteMarshallingTest, URITranslation) {
+  llvm::BumpPtrAllocator Arena;
+  llvm::UniqueStringSaver Strings(Arena);
+  clangd::Ref Original;
+  Original.Location.FileURI =
+      testPathURI("remote/machine/projects/llvm-project/clang-tools-extra/"
+                  "clangd/unittests/remote/MarshallingTests.cpp",
+                  Strings);
+  auto Serialized =
+      toProtobuf(Original, testPath("remote/machine/projects/llvm-project/"));
+  EXPECT_EQ(Serialized.location().file_path(),
+            "clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp");
+  const std::string LocalIndexPrefix = testPath("local/machine/project/");
+  auto Deserialized = fromProtobuf(Serialized, &Strings,
+                                   testPath("home/my-projects/llvm-project/"));
+  EXPECT_TRUE(Deserialized);
+  EXPECT_EQ(Deserialized->Location.FileURI,
+            testPathURI("home/my-projects/llvm-project/clang-tools-extra/"
+                        "clangd/unittests/remote/MarshallingTests.cpp",
+                        Strings));
+
+  clangd::Ref WithInvalidURI;
+  // Invalid URI results in empty path.
+  WithInvalidURI.Location.FileURI = "This is not a URI";
+  Serialized = toProtobuf(WithInvalidURI, testPath("home/"));
+  EXPECT_EQ(Serialized.location().file_path(), "");
+
+  // Can not use URIs with scheme different from "file".
+  auto UnittestURI =
+      URI::create(testPath("project/lib/HelloWorld.cpp"), "unittest");
+  EXPECT_TRUE(bool(UnittestURI));
+  WithInvalidURI.Location.FileURI =
+      Strings.save(UnittestURI->toString()).begin();
+  Serialized = toProtobuf(WithInvalidURI, testPath("project/lib/"));
+  EXPECT_EQ(Serialized.location().file_path(), "");
+
+  Ref WithAbsolutePath;
+  *WithAbsolutePath.mutable_location()->mutable_file_path() =
+      "/usr/local/user/home/HelloWorld.cpp";
+  Deserialized = fromProtobuf(WithAbsolutePath, &Strings, LocalIndexPrefix);
+  // Paths transmitted over the wire can not be absolute, they have to be
+  // relative.
+  EXPECT_FALSE(Deserialized);
+}
+
 TEST(RemoteMarshallingTest, SymbolSerialization) {
-  const auto *Header = R"(
-  // This is a class.
-  class Foo {
-  public:
-    Foo();
-
-    int Bar;
-  private:
-    double Number;
-  };
-  /// This is a function.
-  char baz();
-  template <typename T>
-  T getT ();
-  )";
-  const auto TU = TestTU::withHeaderCode(Header);
-  const auto Symbols = TU.headerSymbols();
-  // Sanity check: there are more than 5 symbols available.
-  EXPECT_GE(Symbols.size(), 5UL);
+  clangd::Symbol Sym;
+
+  auto ID = SymbolID::fromStr("057557CEBF6E6B2D");
+  EXPECT_TRUE(bool(ID));
+  Sym.ID = *ID;
+
+  index::SymbolInfo Info;
+  Info.Kind = index::SymbolKind::Function;
+  Info.SubKind = index::SymbolSubKind::AccessorGetter;
+  Info.Lang = index::SymbolLanguage::CXX;
+  Info.Properties = static_cast<index::SymbolPropertySet>(
+      index::SymbolProperty::TemplateSpecialization);
+  Sym.SymInfo = Info;
+
   llvm::BumpPtrAllocator Arena;
   llvm::UniqueStringSaver Strings(Arena);
-  for (auto &Sym : Symbols) {
-    const auto ProtobufMeessage = toProtobuf(Sym);
-    const auto SymToProtobufAndBack = fromProtobuf(ProtobufMeessage, &Strings);
-    EXPECT_TRUE(SymToProtobufAndBack.hasValue());
-    EXPECT_EQ(toYAML(Sym), toYAML(*SymToProtobufAndBack));
-  }
+
+  Sym.Name = Strings.save("Foo");
+  Sym.Scope = Strings.save("llvm::foo::bar::");
+
+  clangd::SymbolLocation Location;
+  Location.Start.setLine(1);
+  Location.Start.setColumn(15);
+  Location.End.setLine(3);
+  Location.End.setColumn(121);
+  Location.FileURI = testPathURI("home/Definition.cpp", Strings);
+  Sym.Definition = Location;
+
+  Location.Start.setLine(42);
+  Location.Start.setColumn(31);
+  Location.End.setLine(20);
+  Location.End.setColumn(400);
+  Location.FileURI = testPathURI("home/Declaration.h", Strings);
+  Sym.CanonicalDeclaration = Location;
+
+  Sym.References = 9000;
+  Sym.Origin = clangd::SymbolOrigin::Static;
+  Sym.Signature = Strings.save("(int X, char Y, Type T)");
+  Sym.TemplateSpecializationArgs = Strings.save("<int, char, bool, Type>");
+  Sym.CompletionSnippetSuffix =
+      Strings.save("({1: int X}, {2: char Y}, {3: Type T})");
+  Sym.Documentation = Strings.save("This is my amazing Foo constructor!");
+  Sym.ReturnType = Strings.save("Foo");
+
+  Sym.Flags = clangd::Symbol::SymbolFlag::IndexedForCodeCompletion;
+
+  // Check that symbols are exactly the same if the path to indexed project is
+  // the same on indexing machine and the client.
+  auto Serialized = toProtobuf(Sym, testPath("home/"));
+  auto Deserialized = fromProtobuf(Serialized, &Strings, testPath("home/"));
+  EXPECT_TRUE(Deserialized);
+  EXPECT_EQ(toYAML(Sym), toYAML(*Deserialized));
+  // Serialized paths are relative and have UNIX slashes.
+  EXPECT_EQ(convert_to_slash(Serialized.definition().file_path(),
+                             llvm::sys::path::Style::posix),
+            Serialized.definition().file_path());
+  EXPECT_TRUE(
+      llvm::sys::path::is_relative(Serialized.definition().file_path()));
+
+  // Fail with an invalid URI.
+  Location.FileURI = "Not A URI";
+  Sym.Definition = Location;
+  Serialized = toProtobuf(Sym, testPath("home/"));
+  Deserialized = fromProtobuf(Serialized, &Strings, testPath("home/"));
+  EXPECT_FALSE(Deserialized);
+
+  // Schemes other than "file" can not be used.
+  auto UnittestURI = URI::create(testPath("home/SomePath.h"), "unittest");
+  EXPECT_TRUE(bool(UnittestURI));
+  Location.FileURI = Strings.save(UnittestURI->toString()).begin();
+  Sym.Definition = Location;
+  Serialized = toProtobuf(Sym, testPath("home/"));
+  Deserialized = fromProtobuf(Serialized, &Strings, testPath("home/"));
+  EXPECT_FALSE(Deserialized);
+
+  // Passing root that is not prefix of the original file path.
+  Location.FileURI = testPathURI("home/File.h", Strings);
+  Sym.Definition = Location;
+  // Check that the symbol is valid and passing the correct path works.
+  Serialized = toProtobuf(Sym, testPath("home/"));
+  Deserialized = fromProtobuf(Serialized, &Strings, testPath("home/"));
+  EXPECT_TRUE(Deserialized);
+  EXPECT_EQ(Deserialized->Definition.FileURI,
+            testPathURI("home/File.h", Strings));
+  // Fail with a wrong root.
+  Serialized = toProtobuf(Sym, testPath("nothome/"));
+  Deserialized = fromProtobuf(Serialized, &Strings, testPath("home/"));
+  EXPECT_FALSE(Deserialized);
 }
 
-TEST(RemoteMarshallingTest, ReferenceSerialization) {
-  TestTU TU;
-  TU.HeaderCode = R"(
-  int foo();
-  int GlobalVariable = 42;
-  class Foo {
-  public:
-    Foo();
-
-    char Symbol = 'S';
-  };
-  template <typename T>
-  T getT() { return T(); }
-  )";
-  TU.Code = R"(
-  int foo() {
-    ++GlobalVariable;
-
-    Foo foo = Foo();
-    if (foo.Symbol - 'a' == 42) {
-      foo.Symbol = 'b';
-    }
-
-    const auto bar = getT<Foo>();
-  }
-  )";
-  const auto References = TU.headerRefs();
+TEST(RemoteMarshallingTest, RefSerialization) {
+  clangd::Ref Ref;
+  Ref.Kind = clangd::RefKind::Spelled | clangd::RefKind::Declaration;
+
   llvm::BumpPtrAllocator Arena;
   llvm::UniqueStringSaver Strings(Arena);
-  // Sanity check: there are more than 5 references available.
-  EXPECT_GE(References.numRefs(), 5UL);
-  for (const auto &SymbolWithRefs : References) {
-    for (const auto &Ref : SymbolWithRefs.second) {
-      const auto RefToProtobufAndBack = fromProtobuf(toProtobuf(Ref), &Strings);
-      EXPECT_TRUE(RefToProtobufAndBack.hasValue());
-      EXPECT_EQ(toYAML(Ref), toYAML(*RefToProtobufAndBack));
-    }
-  }
-} // namespace
+
+  clangd::SymbolLocation Location;
+  Location.Start.setLine(124);
+  Location.Start.setColumn(21);
+  Location.End.setLine(3213);
+  Location.End.setColumn(541);
+  Location.FileURI = testPathURI(
+      "llvm-project/llvm/clang-tools-extra/clangd/Protocol.h", Strings);
+  Ref.Location = Location;
+
+  auto Serialized = toProtobuf(Ref, testPath("llvm-project/"));
+  auto Deserialized =
+      fromProtobuf(Serialized, &Strings, testPath("llvm-project/"));
+  EXPECT_TRUE(Deserialized);
+  EXPECT_EQ(toYAML(Ref), toYAML(*Deserialized));
+}
+
+TEST(RemoteMarshallingTest, IncludeHeaderURIs) {
+  llvm::BumpPtrAllocator Arena;
+  llvm::UniqueStringSaver Strings(Arena);
+
+  llvm::SmallVector<clangd::Symbol::IncludeHeaderWithReferences, 1>
+      ValidHeaders;
+  clangd::Symbol::IncludeHeaderWithReferences Header;
+  Header.IncludeHeader = Strings.save(
+      URI::createFile("/usr/local/user/home/project/Header.h").toString());
+  Header.References = 21;
+  ValidHeaders.push_back(Header);
+  Header.IncludeHeader = Strings.save("<iostream>");
+  Header.References = 100;
+  ValidHeaders.push_back(Header);
+  Header.IncludeHeader = Strings.save("\"cstdio\"");
+  Header.References = 200;
+  ValidHeaders.push_back(Header);
+
+  llvm::SmallVector<clangd::Symbol::IncludeHeaderWithReferences, 1>
+      InvalidHeaders;
+  // This is an absolute path to a header: can not be transmitted over the wire.
+  Header.IncludeHeader = Strings.save(testPath("project/include/Common.h"));
+  Header.References = 42;
+  InvalidHeaders.push_back(Header);
+  // This is not a valid header: can not be transmitted over the wire;
+  Header.IncludeHeader = Strings.save("NotAHeader");
+  Header.References = 5;
+  InvalidHeaders.push_back(Header);
+
+  clangd::Symbol Sym;
+  // Fill in definition and declaration, Symbool will be invalid otherwise.
+  clangd::SymbolLocation Location;
+  Location.Start.setLine(1);
+  Location.Start.setColumn(2);
+  Location.End.setLine(3);
+  Location.End.setColumn(4);
+  Location.FileURI = testPathURI("File.h", Strings);
+  Sym.Definition = Location;
+  Sym.CanonicalDeclaration = Location;
+
+  // Try to serialize all headers but only valid ones will end up in Protobuf
+  // message.
+  auto AllHeaders = ValidHeaders;
+  AllHeaders.insert(AllHeaders.end(), InvalidHeaders.begin(),
+                    InvalidHeaders.end());
+  Sym.IncludeHeaders = AllHeaders;
+
+  auto Serialized = toProtobuf(Sym, convert_to_slash("/"));
+  EXPECT_EQ(static_cast<size_t>(Serialized.headers_size()),
+            ValidHeaders.size());
+  auto Deserialized = fromProtobuf(Serialized, &Strings, convert_to_slash("/"));
+  EXPECT_TRUE(Deserialized);
+
+  Sym.IncludeHeaders = ValidHeaders;
+  EXPECT_EQ(toYAML(Sym), toYAML(*Deserialized));
+}
+
+TEST(RemoteMarshallingTest, FuzzyFindRequestSerialization) {
+  clangd::FuzzyFindRequest Request;
+  Request.ProximityPaths = {testPath("remote/Header.h"),
+                            testPath("remote/subdir/OtherHeader.h"),
+                            testPath("notremote/File.h"), "Not a Path."};
+  auto Serialized = toProtobuf(Request, testPath("remote/"));
+  EXPECT_EQ(Serialized.proximity_paths_size(), 2);
+  auto Deserialized = fromProtobuf(&Serialized, testPath("home/"));
+  EXPECT_THAT(Deserialized.ProximityPaths,
+              testing::ElementsAre(testPath("home/Header.h"),
+                                   testPath("home/subdir/OtherHeader.h")));
+}
+
+TEST(RemoteMarshallingTest, RelativePathToURITranslation) {
+  EXPECT_TRUE(relativePathToURI("lib/File.cpp", testPath("home/project/")));
+  // RelativePath can not be absolute.
+  EXPECT_FALSE(relativePathToURI("/lib/File.cpp", testPath("home/project/")));
+  // IndexRoot has to be absolute path.
+  EXPECT_FALSE(relativePathToURI("lib/File.cpp", "home/project/"));
+}
+
+TEST(RemoteMarshallingTest, URIToRelativePathTranslation) {
+  llvm::BumpPtrAllocator Arena;
+  llvm::UniqueStringSaver Strings(Arena);
+  EXPECT_TRUE(
+      uriToRelativePath(testPathURI("home/project/lib/File.cpp", Strings),
+                        testPath("home/project/")));
+  // IndexRoot has to be absolute path.
+  EXPECT_FALSE(uriToRelativePath(
+      testPathURI("home/project/lib/File.cpp", Strings), "home/project/"));
+  // IndexRoot has to be be a prefix of the file path.
+  EXPECT_FALSE(
+      uriToRelativePath(testPathURI("home/project/lib/File.cpp", Strings),
+                        testPath("home/other/project/")));
+}
 
 } // namespace
 } // namespace remote
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index bd898de446b9d..c08fd45c2f967 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -94,6 +94,12 @@ New checks
   result of a memory allocation function (``malloc()``, ``calloc()``,
   ``realloc()``, ``alloca()``) instead of its argument.
 
+- New :doc:`bugprone-no-escape
+  <clang-tidy/checks/bugprone-no-escape>` check.
+
+  Finds pointers with the ``noescape`` attribute that are captured by an
+  asynchronously-executed block.
+
 - New :doc:`bugprone-spuriously-wake-up-functions
   <clang-tidy/checks/bugprone-spuriously-wake-up-functions>` check.
 
@@ -192,13 +198,28 @@ New check aliases
   :doc:`bugprone-signed-char-misuse
   <clang-tidy/checks/bugprone-signed-char-misuse>` was added.
 
+- New alias :doc:`llvm-else-after-return
+  <clang-tidy/checks/llvm-else-after-return>` to
+  :doc:`readability-else-after-return
+  <clang-tidy/checks/readability-else-after-return>` was added.
+
 Changes in existing checks
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-- Improved :doc:'readability-identifier-naming
+- Improved :doc:`performance-faster-string-find
+  <clang-tidy/checks/performance-faster-string-find>` check.
+
+  Now checks ``std::basic_string_view`` by default.
+
+- Improved :doc:`readability-else-after-return
+  <clang-tidy/checks/readability-else-after-return>` check now supports a
+  `WarnOnConditionVariables` option to control whether to refactor condition
+  variables where possible.
+
+- Improved :doc:`readability-identifier-naming
   <clang-tidy/checks/readability-identifier-naming>` check.
 
-  Now able to rename member references in class template definitions with 
+  Now able to rename member references in class template definitions with
   explicit access.
 
 - Improved :doc:`readability-qualified-auto
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone-no-escape.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone-no-escape.rst
new file mode 100644
index 0000000000000..770a71cc04255
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone-no-escape.rst
@@ -0,0 +1,19 @@
+.. title:: clang-tidy - bugprone-no-escape
+
+bugprone-no-escape
+==================
+
+Finds pointers with the ``noescape`` attribute that are captured by an
+asynchronously-executed block. The block arguments in ``dispatch_async()`` and
+``dispatch_after()`` are guaranteed to escape, so it is an error if a pointer with the
+``noescape`` attribute is captured by one of these blocks.
+
+The following is an example of an invalid use of the ``noescape`` attribute.
+
+  .. code-block:: objc
+
+    void foo(__attribute__((noescape)) int *p) {
+      dispatch_async(queue, ^{
+        *p = 123;
+      });
+    });
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index 17331605aa64e..78fed1a8ed9cd 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -70,6 +70,7 @@ Clang-Tidy Checks
    `bugprone-misplaced-widening-cast <bugprone-misplaced-widening-cast.html>`_,
    `bugprone-move-forwarding-reference <bugprone-move-forwarding-reference.html>`_, "Yes"
    `bugprone-multiple-statement-macro <bugprone-multiple-statement-macro.html>`_,
+   `bugprone-no-escape <bugprone-no-escape.html>`_, "Yes"
    `bugprone-not-null-terminated-result <bugprone-not-null-terminated-result.html>`_, "Yes"
    `bugprone-parent-virtual-call <bugprone-parent-virtual-call.html>`_, "Yes"
    `bugprone-posix-return <bugprone-posix-return.html>`_, "Yes"
@@ -133,6 +134,7 @@ Clang-Tidy Checks
    `clang-analyzer-valist.Uninitialized <clang-analyzer-valist.Uninitialized.html>`_,
    `clang-analyzer-valist.Unterminated <clang-analyzer-valist.Unterminated.html>`_,
    `cppcoreguidelines-avoid-goto <cppcoreguidelines-avoid-goto.html>`_,
+   `cppcoreguidelines-avoid-non-const-global-variables <cppcoreguidelines-avoid-non-const-global-variables.html>`_,
    `cppcoreguidelines-init-variables <cppcoreguidelines-init-variables.html>`_, "Yes"
    `cppcoreguidelines-interfaces-global-init <cppcoreguidelines-interfaces-global-init.html>`_,
    `cppcoreguidelines-macro-usage <cppcoreguidelines-macro-usage.html>`_,
@@ -189,7 +191,7 @@ Clang-Tidy Checks
    `llvm-prefer-isa-or-dyn-cast-in-conditionals <llvm-prefer-isa-or-dyn-cast-in-conditionals.html>`_, "Yes"
    `llvm-prefer-register-over-unsigned <llvm-prefer-register-over-unsigned.html>`_, "Yes"
    `llvm-twine-local <llvm-twine-local.html>`_, "Yes"
-   `llvmlibc-callee-namespace <llvmlibc-calle-namespace.html>`_,
+   `llvmlibc-callee-namespace <llvmlibc-callee-namespace.html>`_,
    `llvmlibc-implementation-in-namespace <llvmlibc-implementation-in-namespace.html>`_,
    `llvmlibc-restrict-system-libc-headers <llvmlibc-restrict-system-libc-headers.html>`_, "Yes"
    `misc-definitions-in-headers <misc-definitions-in-headers.html>`_, "Yes"
@@ -291,7 +293,7 @@ Clang-Tidy Checks
    `readability-redundant-member-init <readability-redundant-member-init.html>`_, "Yes"
    `readability-redundant-preprocessor <readability-redundant-preprocessor.html>`_,
    `readability-redundant-smartptr-get <readability-redundant-smartptr-get.html>`_, "Yes"
-   `readability-redundant-string-cstr <readability-redundant-string-cstr.html>`_,
+   `readability-redundant-string-cstr <readability-redundant-string-cstr.html>`_, "Yes"
    `readability-redundant-string-init <readability-redundant-string-init.html>`_, "Yes"
    `readability-simplify-boolean-expr <readability-simplify-boolean-expr.html>`_, "Yes"
    `readability-simplify-subscript-expr <readability-simplify-subscript-expr.html>`_, "Yes"
@@ -300,7 +302,7 @@ Clang-Tidy Checks
    `readability-string-compare <readability-string-compare.html>`_, "Yes"
    `readability-uniqueptr-delete-release <readability-uniqueptr-delete-release.html>`_, "Yes"
    `readability-uppercase-literal-suffix <readability-uppercase-literal-suffix.html>`_, "Yes"
-   `readability-use-anyofallof`_, "No"
+   `readability-use-anyofallof <readability-use-anyofallof.html>`_,
    `zircon-temporary-objects <zircon-temporary-objects.html>`_,
 
 
@@ -390,7 +392,6 @@ Clang-Tidy Checks
    `clang-analyzer-unix.cstring.NullArg <clang-analyzer-unix.cstring.NullArg.html>`_, `Clang Static Analyzer <https://clang.llvm.org/docs/analyzer/checkers.html>`_,
    `cppcoreguidelines-avoid-c-arrays <cppcoreguidelines-avoid-c-arrays.html>`_, `modernize-avoid-c-arrays <modernize-avoid-c-arrays.html>`_,
    `cppcoreguidelines-avoid-magic-numbers <cppcoreguidelines-avoid-magic-numbers.html>`_, `readability-magic-numbers <readability-magic-numbers.html>`_,
-   `cppcoreguidelines-avoid-non-const-global-variables <cppcoreguidelines-avoid-non-const-global-variables.html>`_, , , ""
    `cppcoreguidelines-c-copy-assignment-signature <cppcoreguidelines-c-copy-assignment-signature.html>`_, `misc-unconventional-assign-operator <misc-unconventional-assign-operator.html>`_,
    `cppcoreguidelines-explicit-virtual-functions <cppcoreguidelines-explicit-virtual-functions.html>`_, `modernize-use-override <modernize-use-override.html>`_, "Yes"
    `cppcoreguidelines-non-private-member-variables-in-classes <cppcoreguidelines-non-private-member-variables-in-classes.html>`_, `misc-non-private-member-variables-in-classes <misc-non-private-member-variables-in-classes.html>`_,
@@ -410,7 +411,7 @@ Clang-Tidy Checks
    `hicpp-new-delete-operators <hicpp-new-delete-operators.html>`_, `misc-new-delete-overloads <misc-new-delete-overloads.html>`_,
    `hicpp-no-array-decay <hicpp-no-array-decay.html>`_, `cppcoreguidelines-pro-bounds-array-to-pointer-decay <cppcoreguidelines-pro-bounds-array-to-pointer-decay.html>`_,
    `hicpp-no-malloc <hicpp-no-malloc.html>`_, `cppcoreguidelines-no-malloc <cppcoreguidelines-no-malloc.html>`_,
-   `hicpp-noexcept-move <hicpp-noexcept-move.html>`_, `performance-noexcept-move-constructor <performance-noexcept-move-constructor.html>`_,
+   `hicpp-noexcept-move <hicpp-noexcept-move.html>`_, `performance-noexcept-move-constructor <performance-noexcept-move-constructor.html>`_, "Yes"
    `hicpp-special-member-functions <hicpp-special-member-functions.html>`_, `cppcoreguidelines-special-member-functions <cppcoreguidelines-special-member-functions.html>`_,
    `hicpp-static-assert <hicpp-static-assert.html>`_, `misc-static-assert <misc-static-assert.html>`_, "Yes"
    `hicpp-undelegated-constructor <hicpp-undelegated-constructor.html>`_, `bugprone-undelegated-constructor <bugprone-undelegated-constructor.html>`_,
@@ -423,5 +424,5 @@ Clang-Tidy Checks
    `hicpp-use-nullptr <hicpp-use-nullptr.html>`_, `modernize-use-nullptr <modernize-use-nullptr.html>`_, "Yes"
    `hicpp-use-override <hicpp-use-override.html>`_, `modernize-use-override <modernize-use-override.html>`_, "Yes"
    `hicpp-vararg <hicpp-vararg.html>`_, `cppcoreguidelines-pro-type-vararg <cppcoreguidelines-pro-type-vararg.html>`_,
+   `llvm-else-after-return <llvm-else-after-return.html>`_, `readability-else-after-return <readability-else-after-return.html>`_, "Yes"
    `llvm-qualified-auto <llvm-qualified-auto.html>`_, `readability-qualified-auto <readability-qualified-auto.html>`_, "Yes"
-
diff --git a/clang-tools-extra/docs/clang-tidy/checks/llvm-else-after-return.rst b/clang-tools-extra/docs/clang-tidy/checks/llvm-else-after-return.rst
new file mode 100644
index 0000000000000..f9af610fe50db
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/llvm-else-after-return.rst
@@ -0,0 +1,11 @@
+.. title:: clang-tidy - llvm-else-after-return
+.. meta::
+   :http-equiv=refresh: 5;URL=readability-else-after-return.html
+
+llvm-else-after-return
+======================
+
+The llvm-else-after-return check is an alias, please see
+`readability-else-after-return <readability-else-after-return.html>`_
+for more information.
+
diff --git a/clang-tools-extra/docs/clang-tidy/checks/openmp-use-default-none.rst b/clang-tools-extra/docs/clang-tidy/checks/openmp-use-default-none.rst
index 4223a10bd6e9b..77114100ba1cb 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/openmp-use-default-none.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/openmp-use-default-none.rst
@@ -51,3 +51,12 @@ Example
     // WARNING: OpenMP directive ``parallel`` specifies ``default(shared)``
     //          clause. Consider using ``default(none)`` clause instead.
   }
+
+  // ``parallel`` directive can have ``default`` clause, and said clause is
+  // specified, but with ``firstprivate`` kind, which is not ``none``, diagnose.
+  void p0_3() {
+  #pragma omp parallel default(firstprivate)
+    ;
+    // WARNING: OpenMP directive ``parallel`` specifies ``default(firstprivate)``
+    //          clause. Consider using ``default(none)`` clause instead.
+  }
diff --git a/clang-tools-extra/docs/clang-tidy/checks/performance-faster-string-find.rst b/clang-tools-extra/docs/clang-tidy/checks/performance-faster-string-find.rst
index 8d7265e68beed..443da597506a7 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/performance-faster-string-find.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/performance-faster-string-find.rst
@@ -23,6 +23,8 @@ Options
 .. option:: StringLikeClasses
 
    Semicolon-separated list of names of string-like classes. By default only
-   ``std::basic_string`` is considered. The list of methods to consired is
-   fixed.
+   ``::std::basic_string`` and ``::std::basic_string_view`` are considered. 
+   The check will only consider member functions named ``find``, ``rfind``, 
+   ``find_first_of``, ``find_first_not_of``, ``find_last_of``, or 
+   ``find_last_not_of`` within these classes.
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability-else-after-return.rst b/clang-tools-extra/docs/clang-tidy/checks/readability-else-after-return.rst
index c178a6a68ec90..4adcd7bbfa26c 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability-else-after-return.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability-else-after-return.rst
@@ -59,6 +59,31 @@ Would be transformed into:
       }
     }
 
+Options
+-------
+
+.. option:: WarnOnUnfixable
+
+   When `true`, emit a warning for cases where the check can't output a 
+   Fix-It. These can occur with declarations inside the ``else`` branch that
+   would have an extended lifetime if the ``else`` branch was removed.
+   Default value is `true`.
+
+.. option:: WarnOnConditionVariables
+
+   When `true`, the check will attempt to refactor a variable defined inside
+   the condition of the ``if`` statement that is used in the ``else`` branch
+   defining them just before the ``if`` statement. This can only be done if 
+   the ``if`` statement is the last statement in its parents scope.
+   Default value is `true`.
+
+
+LLVM alias
+----------
+
+There is an alias of this check called llvm-else-after-return.
+In that version the options :option:`WarnOnUnfixable` and 
+:option:`WarnOnConditionVariables` are both set to `false` by default.
 
 This check helps to enforce this `LLVM Coding Standards recommendation
 <https://llvm.org/docs/CodingStandards.html#don-t-use-else-after-a-return>`_.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone-no-escape.m b/clang-tools-extra/test/clang-tidy/checkers/bugprone-no-escape.m
new file mode 100644
index 0000000000000..11dba3345fa1a
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone-no-escape.m
@@ -0,0 +1,28 @@
+// RUN: %check_clang_tidy %s bugprone-no-escape %t
+// RUN: %check_clang_tidy %s -assume-filename=bugprone-no-escape.c bugprone-no-escape %t -- -- -fblocks
+
+typedef struct dispatch_queue_s *dispatch_queue_t;
+typedef struct dispatch_time_s *dispatch_time_t;
+typedef void (^dispatch_block_t)(void);
+void dispatch_async(dispatch_queue_t queue, dispatch_block_t block);
+void dispatch_after(dispatch_time_t when, dispatch_queue_t queue, dispatch_block_t block);
+
+extern dispatch_queue_t queue;
+
+void test_noescape_attribute(__attribute__((noescape)) int *p, int *q) {
+  dispatch_async(queue, ^{
+    *p = 123;
+    // CHECK-MESSAGES: :[[@LINE-2]]:25: warning: pointer 'p' with attribute 'noescape' is captured by an asynchronously-executed block [bugprone-no-escape]
+    // CHECK-MESSAGES: :[[@LINE-4]]:30: note: the 'noescape' attribute is declared here.
+  });
+
+  dispatch_after(456, queue, ^{
+    *p = 789;
+    // CHECK-MESSAGES: :[[@LINE-2]]:30: warning: pointer 'p' with attribute 'noescape' is captured by an asynchronously-executed block [bugprone-no-escape]
+  });
+
+  dispatch_async(queue, ^{
+    *q = 0;
+    // CHECK-MESSAGES-NOT: :[[@LINE-2]]:25: warning: pointer 'q' with attribute 'noescape' is captured by an asynchronously-executed block
+  });
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone-suspicious-memset-usage.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone-suspicious-memset-usage.cpp
index f33ae5ae10a84..9a7e423f40122 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone-suspicious-memset-usage.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone-suspicious-memset-usage.cpp
@@ -75,3 +75,8 @@ void foo(int xsize, int ysize) {
   // despite v == 0.
   memset(p, -1, v);
 }
+
+void *memset(int);
+void NoCrash() {
+  memset(1);
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone-unused-raii-crash.mm b/clang-tools-extra/test/clang-tidy/checkers/bugprone-unused-raii-crash.mm
new file mode 100644
index 0000000000000..432fd5329c566
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone-unused-raii-crash.mm
@@ -0,0 +1,15 @@
+// RUN: clang-tidy %s -checks=-*,bugprone-unused-raii -- | count 0
+
+struct CxxClass {
+  ~CxxClass() {}
+};
+
+@interface ObjcClass {
+}
+- (CxxClass)set:(int)p;
+@end
+
+void test(ObjcClass *s) {
+  [s set:1]; // ok, no crash, no diagnostic emitted.
+  return;
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines-init-variables.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines-init-variables.cpp
index 4c92e1976f917..9a06850b3f669 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines-init-variables.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines-init-variables.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy %s cppcoreguidelines-init-variables %t -- -- -fno-delayed-template-parsing
+// RUN: %check_clang_tidy %s cppcoreguidelines-init-variables %t -- -- -fno-delayed-template-parsing -fexceptions
 
 // Ensure that function declarations are not changed.
 void some_func(int x, double d, bool b, const char *p);
@@ -84,3 +84,10 @@ void f(RANGE r) {
   for (char c : r) {
   }
 }
+
+void catch_variable_decl() {
+  // Expect no warning given here.
+  try {
+  } catch (int X) {
+  }
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/llvm-twine-local.cpp b/clang-tools-extra/test/clang-tidy/checkers/llvm-twine-local.cpp
index 06eb7613d439d..3dcf6abe0c22e 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/llvm-twine-local.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/llvm-twine-local.cpp
@@ -13,6 +13,7 @@ class Twine {
 using namespace llvm;
 
 void foo(const Twine &x);
+void bar(Twine x);
 
 static Twine Moo = Twine("bark") + "bah";
 // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: twine variables are prone to use-after-free bugs
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize-avoid-bind-permissive-parameter-list.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize-avoid-bind-permissive-parameter-list.cpp
index 6c81a6e9ab97d..e055fd3232306 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize-avoid-bind-permissive-parameter-list.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize-avoid-bind-permissive-parameter-list.cpp
@@ -54,5 +54,5 @@ void testLiteralParameters() {
 
   auto BBB = std::bind(add, _1, 2);
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind [modernize-avoid-bind]
-  // CHECK-FIXES: auto BBB = [](auto && PH1, auto && ...) { return add(PH1, 2); };
+  // CHECK-FIXES: auto BBB = [](auto && PH1, auto && ...) { return add(std::forward<decltype(PH1)>(PH1), 2); };
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize-avoid-bind.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize-avoid-bind.cpp
index 7e00858c1acce..3334eab2d407e 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize-avoid-bind.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize-avoid-bind.cpp
@@ -7,11 +7,11 @@ class bind_rt {};
 
 template <class Fp, class... Arguments>
 bind_rt<Fp, Arguments...> bind(Fp &&, Arguments &&...);
-}
+} // namespace impl
 
 template <typename T>
 T ref(T &t);
-}
+} // namespace std
 
 namespace boost {
 template <class Fp, class... Arguments>
@@ -58,12 +58,33 @@ struct F {
 
 void UseF(F);
 
+struct G {
+  G() : _member(0) {}
+  G(int m) : _member(m) {}
+
+  template <typename T>
+  void operator()(T) const {}
+
+  int _member;
+};
+
+template <typename T>
+struct H {
+  void operator()(T) const {};
+};
+
 struct placeholder {};
 placeholder _1;
 placeholder _2;
 
+namespace placeholders {
+using ::_1;
+using ::_2;
+} // namespace placeholders
+
 int add(int x, int y) { return x + y; }
 int addThree(int x, int y, int z) { return x + y + z; }
+void sub(int &x, int y) { x += y; }
 
 // Let's fake a minimal std::function-like facility.
 namespace std {
@@ -107,6 +128,7 @@ struct TestCaptureByValueStruct {
   int MemberVariable;
   static int StaticMemberVariable;
   F MemberStruct;
+  G MemberStructWithData;
 
   void testCaptureByValue(int Param, F f) {
     int x = 3;
@@ -145,6 +167,11 @@ struct TestCaptureByValueStruct {
     auto GGG = boost::bind(UseF, MemberStruct);
     // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: prefer a lambda to boost::bind [modernize-avoid-bind]
     // CHECK-FIXES: auto GGG = [this] { return UseF(MemberStruct); };
+
+    auto HHH = std::bind(add, MemberStructWithData._member, 1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: prefer a lambda to std::bind
+    // Correctly distinguish data members of other classes
+    // CHECK-FIXES: auto HHH = [capture0 = MemberStructWithData._member] { return add(capture0, 1); };
   }
 };
 
@@ -217,17 +244,38 @@ void testFunctionObjects() {
   auto EEE = std::bind(*D::create(), 1, 2);
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
   // CHECK-FIXES: auto EEE = [Func = *D::create()] { return Func(1, 2); };
+
+  auto FFF = std::bind(G(), 1);
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
+  // Templated function call operators may be used
+  // CHECK-FIXES: auto FFF = [] { return G()(1); };
+
+  int CTorArg = 42;
+  auto GGG = std::bind(G(CTorArg), 1);
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
+  // Function objects with constructor arguments should be captured
+  // CHECK-FIXES: auto GGG = [Func = G(CTorArg)] { return Func(1); };
 }
 
+template <typename T>
+void testMemberFnOfClassTemplate(T) {
+  auto HHH = std::bind(H<T>(), 42);
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
+  // Ensure function class template arguments are preserved
+  // CHECK-FIXES: auto HHH = [] { return H<T>()(42); };
+}
+
+template void testMemberFnOfClassTemplate(int);
+
 void testPlaceholders() {
   int x = 2;
   auto AAA = std::bind(add, x, _1);
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
-  // CHECK-FIXES: auto AAA = [x](auto && PH1) { return add(x, PH1); };
+  // CHECK-FIXES: auto AAA = [x](auto && PH1) { return add(x, std::forward<decltype(PH1)>(PH1)); };
 
   auto BBB = std::bind(add, _2, _1);
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
-  // CHECK-FIXES: auto BBB = [](auto && PH1, auto && PH2) { return add(PH2, PH1); };
+  // CHECK-FIXES: auto BBB = [](auto && PH1, auto && PH2) { return add(std::forward<decltype(PH2)>(PH2), std::forward<decltype(PH1)>(PH1)); };
 
   // No fix is applied for reused placeholders.
   auto CCC = std::bind(add, _1, _1);
@@ -238,7 +286,12 @@ void testPlaceholders() {
   // unnamed parameters.
   auto DDD = std::bind(add, _2, 1);
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
-  // CHECK-FIXES: auto DDD = [](auto &&, auto && PH2) { return add(PH2, 1); };
+  // CHECK-FIXES: auto DDD = [](auto &&, auto && PH2) { return add(std::forward<decltype(PH2)>(PH2), 1); };
+
+  // Namespace-qualified placeholders are valid too
+  auto EEE = std::bind(add, placeholders::_2, 1);
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
+  // CHECK-FIXES: auto EEE = [](auto &&, auto && PH2) { return add(std::forward<decltype(PH2)>(PH2), 1); };
 }
 
 void testGlobalFunctions() {
@@ -267,6 +320,7 @@ void testGlobalFunctions() {
 void testCapturedSubexpressions() {
   int x = 3;
   int y = 3;
+  int *p = &x;
 
   auto AAA = std::bind(add, 1, add(2, 5));
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
@@ -277,6 +331,11 @@ void testCapturedSubexpressions() {
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
   // Results of nested calls are captured by value.
   // CHECK-FIXES: auto BBB = [x, capture0 = add(y, 5)] { return add(x, capture0); };
+
+  auto CCC = std::bind(sub, std::ref(*p), _1);
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
+  // Expressions returning references are captured
+  // CHECK-FIXES: auto CCC = [&capture0 = *p](auto && PH1) { return sub(capture0, std::forward<decltype(PH1)>(PH1)); };
 }
 
 struct E {
diff --git a/clang-tools-extra/test/clang-tidy/checkers/openmp-use-default-none.cpp b/clang-tools-extra/test/clang-tidy/checkers/openmp-use-default-none.cpp
index 35d2d17b1e0e8..d1d3b0e441f3f 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/openmp-use-default-none.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/openmp-use-default-none.cpp
@@ -1,5 +1,5 @@
-// RUN: %check_clang_tidy %s openmp-use-default-none %t -- -- -fopenmp=libomp -fopenmp-version=40
-// RUN: %check_clang_tidy -std=c11 %s openmp-use-default-none %t -- -- -x c -fopenmp=libomp -fopenmp-version=40
+// RUN: %check_clang_tidy %s openmp-use-default-none %t -- -- -fopenmp=libomp -fopenmp-version=51
+// RUN: %check_clang_tidy -std=c11 %s openmp-use-default-none %t -- -- -x c -fopenmp=libomp -fopenmp-version=51
 
 //----------------------------------------------------------------------------//
 // Null cases.
@@ -42,6 +42,15 @@ void p0_2() {
   // CHECK-NOTES: :[[@LINE-3]]:22: note: existing 'default' clause specified here
 }
 
+// 'parallel' directive can have 'default' clause, and said clause specified,
+// but with 'firstprivate' kind, which is not 'none', diagnose.
+void p0_3() {
+#pragma omp parallel default(firstprivate)
+  ;
+  // CHECK-NOTES: :[[@LINE-2]]:1: warning: OpenMP directive 'parallel' specifies 'default(firstprivate)' clause, consider using 'default(none)' clause instead
+  // CHECK-NOTES: :[[@LINE-3]]:22: note: existing 'default' clause specified here
+}
+
 // 'task' directive.
 
 // 'task' directive can have 'default' clause, but said clause is not
@@ -68,6 +77,15 @@ void p1_2() {
   // CHECK-NOTES: :[[@LINE-3]]:18: note: existing 'default' clause specified here
 }
 
+// 'task' directive can have 'default' clause, and said clause specified,
+// but with 'firstprivate' kind, which is not 'none', diagnose.
+void p1_3() {
+#pragma omp task default(firstprivate)
+  ;
+  // CHECK-NOTES: :[[@LINE-2]]:1: warning: OpenMP directive 'task' specifies 'default(firstprivate)' clause, consider using 'default(none)' clause instead
+  // CHECK-NOTES: :[[@LINE-3]]:18: note: existing 'default' clause specified here
+}
+
 // 'teams' directive. (has to be inside of 'target' directive)
 
 // 'teams' directive can have 'default' clause, but said clause is not
@@ -97,6 +115,16 @@ void p2_2() {
   // CHECK-NOTES: :[[@LINE-3]]:19: note: existing 'default' clause specified here
 }
 
+// 'teams' directive can have 'default' clause, and said clause specified,
+// but with 'firstprivate' kind, which is not 'none', diagnose.
+void p2_3() {
+#pragma omp target
+#pragma omp teams default(firstprivate)
+  ;
+  // CHECK-NOTES: :[[@LINE-2]]:1: warning: OpenMP directive 'teams' specifies 'default(firstprivate)' clause, consider using 'default(none)' clause instead
+  // CHECK-NOTES: :[[@LINE-3]]:19: note: existing 'default' clause specified here
+}
+
 // 'taskloop' directive.
 
 // 'taskloop' directive can have 'default' clause, but said clause is not
@@ -126,6 +154,16 @@ void p3_2(const int a) {
   // CHECK-NOTES: :[[@LINE-4]]:22: note: existing 'default' clause specified here
 }
 
+// 'taskloop' directive can have 'default' clause, and said clause specified,
+// but with 'firstprivate' kind, which is not 'none', diagnose.
+void p3_3(const int a) {
+#pragma omp taskloop default(firstprivate)
+  for (int b = 0; b < a; b++)
+    ;
+  // CHECK-NOTES: :[[@LINE-3]]:1: warning: OpenMP directive 'taskloop' specifies 'default(firstprivate)' clause, consider using 'default(none)' clause instead
+  // CHECK-NOTES: :[[@LINE-4]]:22: note: existing 'default' clause specified here
+}
+
 //----------------------------------------------------------------------------//
 // Combined directives.
 // Let's not test every single possible permutation/combination of directives,
@@ -158,3 +196,13 @@ void p4_2(const int a) {
   // CHECK-NOTES: :[[@LINE-3]]:1: warning: OpenMP directive 'parallel for' specifies 'default(shared)' clause, consider using 'default(none)' clause instead
   // CHECK-NOTES: :[[@LINE-4]]:26: note: existing 'default' clause specified here
 }
+
+// 'parallel' directive can have 'default' clause, and said clause specified,
+// but with 'firstprivate' kind, which is not 'none', diagnose.
+void p4_3(const int a) {
+#pragma omp parallel for default(firstprivate)
+  for (int b = 0; b < a; b++)
+    ;
+  // CHECK-NOTES: :[[@LINE-3]]:1: warning: OpenMP directive 'parallel for' specifies 'default(firstprivate)' clause, consider using 'default(none)' clause instead
+  // CHECK-NOTES: :[[@LINE-4]]:26: note: existing 'default' clause specified here
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/performance-faster-string-find.cpp b/clang-tools-extra/test/clang-tidy/checkers/performance-faster-string-find.cpp
index b36e32301f5db..d26cd2c8ca30c 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/performance-faster-string-find.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/performance-faster-string-find.cpp
@@ -1,7 +1,8 @@
-// RUN: %check_clang_tidy %s performance-faster-string-find %t -- \
+// RUN: %check_clang_tidy %s performance-faster-string-find %t
+// RUN: %check_clang_tidy -check-suffix=CUSTOM %s performance-faster-string-find %t -- \
 // RUN:   -config="{CheckOptions: \
 // RUN:             [{key: performance-faster-string-find.StringLikeClasses, \
-// RUN:               value: 'std::basic_string; ::llvm::StringRef;'}]}" --
+// RUN:               value: '::llvm::StringRef;'}]}"
 
 namespace std {
 template <typename Char>
@@ -17,6 +18,20 @@ struct basic_string {
 
 typedef basic_string<char> string;
 typedef basic_string<wchar_t> wstring;
+
+template <typename Char>
+struct basic_string_view {
+  int find(const Char *, int = 0) const;
+  int find(const Char *, int, int) const;
+  int rfind(const Char *) const;
+  int find_first_of(const Char *) const;
+  int find_first_not_of(const Char *) const;
+  int find_last_of(const Char *) const;
+  int find_last_not_of(const Char *) const;
+};
+
+typedef basic_string_view<char> string_view;
+typedef basic_string_view<wchar_t> wstring_view;
 }  // namespace std
 
 namespace llvm {
@@ -75,11 +90,25 @@ void StringFind() {
   // CHECK-MESSAGES: [[@LINE-1]]:13: warning: 'find' called with a string literal
   // CHECK-FIXES: Str.find(L'\x3A9');
 
+  // std::string_view and std::wstring_view should work.
+  std::string_view StrView;
+  StrView.find("n");
+  // CHECK-MESSAGES: [[@LINE-1]]:16: warning: 'find' called with a string literal
+  // CHECK-FIXES: StrView.find('n');
+  std::wstring_view WStrView;
+
+  WStrView.find(L"n");
+  // CHECK-MESSAGES: [[@LINE-1]]:17: warning: 'find' called with a string literal
+  // CHECK-FIXES: WStrView.find(L'n');
+  WStrView.find(L"\x3A9");
+  // CHECK-MESSAGES: [[@LINE-1]]:17: warning: 'find' called with a string literal
+  // CHECK-FIXES: WStrView.find(L'\x3A9');
+
   // Also with other types, but only if it was specified in the options.
   llvm::StringRef sr;
   sr.find("x");
-  // CHECK-MESSAGES: [[@LINE-1]]:11: warning: 'find' called with a string literal
-  // CHECK-FIXES: sr.find('x');
+  // CHECK-MESSAGES-CUSTOM: [[@LINE-1]]:11: warning: 'find' called with a string literal
+  // CHECK-FIXES-CUSTOM: sr.find('x');
   NotStringRef nsr;
   nsr.find("x");
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability-else-after-return-no-cond-var-refactor.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability-else-after-return-no-cond-var-refactor.cpp
new file mode 100644
index 0000000000000..0d43240f1c6a6
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability-else-after-return-no-cond-var-refactor.cpp
@@ -0,0 +1,42 @@
+// RUN: %check_clang_tidy %s readability-else-after-return %t -- \
+// RUN:     -config='{CheckOptions: [ \
+// RUN:         {key: readability-else-after-return.WarnOnConditionVariables, value: false}, \
+// RUN:     ]}'
+
+bool foo(int Y) {
+  // Excess scopes are here so that the check would have to opportunity to
+  // refactor the variable out of the condition.
+
+  // Expect warnings here as we don't need to move declaration of 'X' out of the
+  // if condition as its not used in the else.
+  {
+    if (int X = Y)
+      return X < 0;
+    else
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: do not use 'else' after 'return'
+      return false;
+  }
+  {
+    if (int X = Y; X)
+      return X < 0;
+    else
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: do not use 'else' after 'return'
+      return false;
+  }
+
+  // Expect no warnings for these cases, as even though its safe to move
+  // declaration of 'X' out of the if condition, that has been disabled
+  // by the options.
+  {
+    if (int X = Y)
+      return false;
+    else
+      return X < 0;
+  }
+  {
+    if (int X = Y; X)
+      return false;
+    else
+      return X < 0;
+  }
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming.cpp
index 3a3bfecc01fcc..24c1c4270dec8 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming.cpp
@@ -562,3 +562,19 @@ void ReferenceBadNamedFunction() {
 }
 
 } // namespace redecls
+
+namespace scratchspace {
+#define DUP(Tok) Tok
+#define M1(Tok) DUP(badName##Tok())
+
+// We don't want a warning here as the call to this in Foo is in a scratch
+// buffer so its fix-it wouldn't be applied, resulting in invalid code.
+void badNameWarn();
+
+void Foo() {
+  M1(Warn);
+}
+
+#undef M1
+#undef DUP
+} // namespace scratchspace
diff --git a/clang-tools-extra/unittests/clang-tidy/ClangTidyOptionsTest.cpp b/clang-tools-extra/unittests/clang-tidy/ClangTidyOptionsTest.cpp
index a089281bf16c4..63f9a06e91bec 100644
--- a/clang-tools-extra/unittests/clang-tidy/ClangTidyOptionsTest.cpp
+++ b/clang-tools-extra/unittests/clang-tidy/ClangTidyOptionsTest.cpp
@@ -6,6 +6,20 @@
 
 namespace clang {
 namespace tidy {
+
+enum class Colours { Red, Orange, Yellow, Green, Blue, Indigo, Violet };
+
+template <> struct OptionEnumMapping<Colours> {
+  static llvm::ArrayRef<std::pair<Colours, StringRef>> getEnumMapping() {
+    static constexpr std::pair<Colours, StringRef> Mapping[] = {
+        {Colours::Red, "Red"},       {Colours::Orange, "Orange"},
+        {Colours::Yellow, "Yellow"}, {Colours::Green, "Green"},
+        {Colours::Blue, "Blue"},     {Colours::Indigo, "Indigo"},
+        {Colours::Violet, "Violet"}};
+    return makeArrayRef(Mapping);
+  }
+};
+
 namespace test {
 
 TEST(ParseLineFilter, EmptyFilter) {
@@ -208,16 +222,10 @@ TEST(CheckOptionsValidation, ValidIntOptions) {
 #undef CHECK_ERROR_INT
 }
 
+// FIXME: Figure out why this test causes crashes on mac os.
+#ifndef __APPLE__
 TEST(ValidConfiguration, ValidEnumOptions) {
 
-  enum class Colours { Red, Orange, Yellow, Green, Blue, Indigo, Violet };
-  static constexpr std::pair<StringRef, Colours> Mapping[] = {
-      {"Red", Colours::Red},       {"Orange", Colours::Orange},
-      {"Yellow", Colours::Yellow}, {"Green", Colours::Green},
-      {"Blue", Colours::Blue},     {"Indigo", Colours::Indigo},
-      {"Violet", Colours::Violet}};
-  static const auto Map = makeArrayRef(Mapping);
-
   ClangTidyOptions Options;
   auto &CheckOptions = Options.CheckOptions;
 
@@ -237,34 +245,37 @@ TEST(ValidConfiguration, ValidEnumOptions) {
 #define CHECK_ERROR_ENUM(Name, Expected)                                       \
   CHECK_ERROR(Name, UnparseableEnumOptionError, Expected)
 
-  CHECK_VAL(TestCheck.getLocal("Valid", Map), Colours::Red);
-  CHECK_VAL(TestCheck.getGlobal("GlobalValid", Map), Colours::Violet);
-  CHECK_VAL(TestCheck.getLocal("ValidWrongCase", Map, /*IgnoreCase*/ true),
-            Colours::Red);
+  CHECK_VAL(TestCheck.getIntLocal<Colours>("Valid"), Colours::Red);
+  CHECK_VAL(TestCheck.getIntGlobal<Colours>("GlobalValid"), Colours::Violet);
+
   CHECK_VAL(
-      TestCheck.getGlobal("GlobalValidWrongCase", Map, /*IgnoreCase*/ true),
-      Colours::Violet);
-  CHECK_ERROR_ENUM(TestCheck.getLocal("Invalid", Map),
+      TestCheck.getIntLocal<Colours>("ValidWrongCase", /*IgnoreCase*/ true),
+      Colours::Red);
+  CHECK_VAL(TestCheck.getIntGlobal<Colours>("GlobalValidWrongCase",
+                                            /*IgnoreCase*/ true),
+            Colours::Violet);
+  CHECK_ERROR_ENUM(TestCheck.getIntLocal<Colours>("Invalid"),
                    "invalid configuration value "
                    "'Scarlet' for option 'test.Invalid'");
-  CHECK_ERROR_ENUM(TestCheck.getLocal("ValidWrongCase", Map),
+  CHECK_ERROR_ENUM(TestCheck.getIntLocal<Colours>("ValidWrongCase"),
                    "invalid configuration value 'rED' for option "
                    "'test.ValidWrongCase'; did you mean 'Red'?");
-  CHECK_ERROR_ENUM(TestCheck.getLocal("NearMiss", Map),
+  CHECK_ERROR_ENUM(TestCheck.getIntLocal<Colours>("NearMiss"),
                    "invalid configuration value 'Oragne' for option "
                    "'test.NearMiss'; did you mean 'Orange'?");
-  CHECK_ERROR_ENUM(TestCheck.getGlobal("GlobalInvalid", Map),
+  CHECK_ERROR_ENUM(TestCheck.getIntGlobal<Colours>("GlobalInvalid"),
                    "invalid configuration value "
                    "'Purple' for option 'GlobalInvalid'");
-  CHECK_ERROR_ENUM(TestCheck.getGlobal("GlobalValidWrongCase", Map),
+  CHECK_ERROR_ENUM(TestCheck.getIntGlobal<Colours>("GlobalValidWrongCase"),
                    "invalid configuration value 'vIOLET' for option "
                    "'GlobalValidWrongCase'; did you mean 'Violet'?");
-  CHECK_ERROR_ENUM(TestCheck.getGlobal("GlobalNearMiss", Map),
+  CHECK_ERROR_ENUM(TestCheck.getIntGlobal<Colours>("GlobalNearMiss"),
                    "invalid configuration value 'Yelow' for option "
                    "'GlobalNearMiss'; did you mean 'Yellow'?");
 
 #undef CHECK_ERROR_ENUM
 }
+#endif
 
 #undef CHECK_VAL
 #undef CHECK_ERROR
diff --git a/clang-tools-extra/unittests/clang-tidy/ClangTidyTest.h b/clang-tools-extra/unittests/clang-tidy/ClangTidyTest.h
index fee2d94a4d572..b8ee9972f6f86 100644
--- a/clang-tools-extra/unittests/clang-tidy/ClangTidyTest.h
+++ b/clang-tools-extra/unittests/clang-tidy/ClangTidyTest.h
@@ -10,6 +10,7 @@
 #define LLVM_CLANG_TOOLS_EXTRA_UNITTESTS_CLANG_TIDY_CLANGTIDYTEST_H
 
 #include "ClangTidy.h"
+#include "ClangTidyCheck.h"
 #include "ClangTidyDiagnosticConsumer.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/Frontend/CompilerInstance.h"
@@ -18,7 +19,6 @@
 #include "clang/Tooling/Core/Replacement.h"
 #include "clang/Tooling/Refactoring.h"
 #include "clang/Tooling/Tooling.h"
-#include "llvm/ADT/Optional.h"
 #include "llvm/Support/Path.h"
 #include <map>
 #include <memory>
@@ -66,7 +66,9 @@ class TestClangTidyAction : public ASTFrontendAction {
     // that check constructors can access the context (for example, through
     // `getLangOpts()`).
     CheckFactory<CheckTypes...>::createChecks(&Context, Checks);
+    assert(!Checks.empty() && "No checks created");
     for (auto &Check : Checks) {
+      assert(Check.get() && "Checks can't be null");
       if (!Check->isLanguageVersionSupported(Context.getLangOpts()))
         continue;
       Check->registerMatchers(&Finder);
@@ -88,6 +90,7 @@ runCheckOnCode(StringRef Code, std::vector<ClangTidyError> *Errors = nullptr,
                const ClangTidyOptions &ExtraOptions = ClangTidyOptions(),
                std::map<StringRef, StringRef> PathsToContent =
                    std::map<StringRef, StringRef>()) {
+  static_assert(sizeof...(CheckTypes) > 0, "No checks specified");
   ClangTidyOptions Options = ExtraOptions;
   Options.Checks = "*";
   ClangTidyContext Context(std::make_unique<DefaultOptionsProvider>(
@@ -119,7 +122,7 @@ runCheckOnCode(StringRef Code, std::vector<ClangTidyError> *Errors = nullptr,
   llvm::IntrusiveRefCntPtr<FileManager> Files(
       new FileManager(FileSystemOptions(), InMemoryFileSystem));
 
-  SmallVector<std::unique_ptr<ClangTidyCheck>, 1> Checks;
+  SmallVector<std::unique_ptr<ClangTidyCheck>, sizeof...(CheckTypes)> Checks;
   tooling::ToolInvocation Invocation(
       Args,
       std::make_unique<TestClangTidyAction<CheckTypes...>>(Checks, Finder,
diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index 5a5e34aacbebb..7f8e0718c2ebc 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -9,6 +9,10 @@ endif()
 if( CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR )
   project(Clang)
 
+  set(CMAKE_CXX_STANDARD 14 CACHE STRING "C++ standard to conform to")
+  set(CMAKE_CXX_STANDARD_REQUIRED YES)
+  set(CMAKE_CXX_EXTENSIONS NO)
+
   # Rely on llvm-config.
   set(CONFIG_OUTPUT)
   if(LLVM_CONFIG)
@@ -517,7 +521,10 @@ add_subdirectory(include)
 
 # All targets below may depend on all tablegen'd files.
 get_property(CLANG_TABLEGEN_TARGETS GLOBAL PROPERTY CLANG_TABLEGEN_TARGETS)
-add_custom_target(clang-tablegen-targets DEPENDS ${CLANG_TABLEGEN_TARGETS})
+add_custom_target(clang-tablegen-targets
+  DEPENDS
+  omp_gen
+  ${CLANG_TABLEGEN_TARGETS})
 set_target_properties(clang-tablegen-targets PROPERTIES FOLDER "Misc")
 list(APPEND LLVM_COMMON_DEPENDS clang-tablegen-targets)
 
diff --git a/clang/cmake/caches/Fuchsia-stage2.cmake b/clang/cmake/caches/Fuchsia-stage2.cmake
index 4ef404b8aaef0..259684ff2b0d9 100644
--- a/clang/cmake/caches/Fuchsia-stage2.cmake
+++ b/clang/cmake/caches/Fuchsia-stage2.cmake
@@ -4,6 +4,9 @@ set(LLVM_TARGETS_TO_BUILD X86;ARM;AArch64;RISCV CACHE STRING "")
 
 set(PACKAGE_VENDOR Fuchsia CACHE STRING "")
 
+set(LLVM_ENABLE_PROJECTS "clang;clang-tools-extra;lld;llvm" CACHE STRING "")
+set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")
+
 set(LLVM_ENABLE_BACKTRACES OFF CACHE BOOL "")
 if(NOT APPLE)
   set(LLVM_ENABLE_LLD ON CACHE BOOL "")
@@ -19,6 +22,10 @@ set(LLVM_INCLUDE_GO_TESTS OFF CACHE BOOL "")
 set(LLVM_USE_RELATIVE_PATHS_IN_FILES ON CACHE BOOL "")
 set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "")
 
+if(MSVC)
+  set(LLVM_USE_CRT_RELEASE "MT" CACHE STRING "")
+endif()
+
 set(CLANG_DEFAULT_CXX_STDLIB libc++ CACHE STRING "")
 if(NOT APPLE)
   set(CLANG_DEFAULT_LINKER lld CACHE STRING "")
@@ -36,6 +43,8 @@ set(ENABLE_X86_RELAX_RELOCATIONS ON CACHE BOOL "")
 set(CMAKE_BUILD_TYPE Release CACHE STRING "")
 if (APPLE)
   set(MACOSX_DEPLOYMENT_TARGET 10.7 CACHE STRING "")
+elseif(MSVC)
+  set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded" CACHE STRING "")
 endif()
 
 if(APPLE)
@@ -44,6 +53,7 @@ if(APPLE)
 
   set(COMPILER_RT_ENABLE_TVOS OFF CACHE BOOL "")
   set(COMPILER_RT_ENABLE_WATCHOS OFF CACHE BOOL "")
+  set(COMPILER_RT_USE_BUILTINS_LIBRARY ON CACHE BOOL "")
 
   set(LIBUNWIND_ENABLE_SHARED OFF CACHE BOOL "")
   set(LIBUNWIND_INSTALL_LIBRARY OFF CACHE BOOL "")
@@ -63,12 +73,34 @@ if(APPLE)
   set(SANITIZER_MIN_OSX_VERSION 10.7 CACHE STRING "")
 endif()
 
+if(WIN32)
+  set(target "x86_64-pc-windows-msvc")
+
+  list(APPEND BUILTIN_TARGETS "${target}")
+  set(BUILTINS_${target}_CMAKE_SYSTEM_NAME Windows CACHE STRING "")
+  set(BUILTINS_${target}_CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "")
+
+  list(APPEND RUNTIME_TARGETS "${target}")
+  set(RUNTIMES_${target}_CMAKE_SYSTEM_NAME Windows CACHE STRING "")
+  set(RUNTIMES_${target}_CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "")
+  set(RUNTIMES_${target}_LIBCXX_ABI_VERSION 2 CACHE STRING "")
+  set(RUNTIMES_${target}_LIBCXX_HAS_WIN32_THREAD_API ON CACHE BOOL "")
+  set(RUNTIMES_${target}_LIBCXX_ENABLE_EXPERIMENTAL_LIBRARY OFF CACHE BOOL "")
+  set(RUNTIMES_${target}_LIBCXX_ENABLE_FILESYSTEM OFF CACHE BOOL "")
+  set(RUNTIMES_${target}_LIBCXX_ENABLE_ABI_LINKER_SCRIPT OFF CACHE BOOL "")
+  set(RUNTIMES_${target}_LIBCXX_ENABLE_SHARED OFF CACHE BOOL "")
+  set(RUNTIMES_${target}_LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx" CACHE STRING "")
+endif()
+
 foreach(target aarch64-unknown-linux-gnu;armv7-unknown-linux-gnueabihf;i386-unknown-linux-gnu;x86_64-unknown-linux-gnu)
   if(LINUX_${target}_SYSROOT)
     # Set the per-target builtins options.
     list(APPEND BUILTIN_TARGETS "${target}")
     set(BUILTINS_${target}_CMAKE_SYSTEM_NAME Linux CACHE STRING "")
     set(BUILTINS_${target}_CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "")
+    set(BUILTINS_${target}_CMAKE_C_FLAGS "--target=${target}" CACHE STRING "")
+    set(BUILTINS_${target}_CMAKE_CXX_FLAGS "--target=${target}" CACHE STRING "")
+    set(BUILTINS_${target}_CMAKE_ASM_FLAGS "--target=${target}" CACHE STRING "")
     set(BUILTINS_${target}_CMAKE_SYSROOT ${LINUX_${target}_SYSROOT} CACHE STRING "")
     set(BUILTINS_${target}_CMAKE_SHARED_LINKER_FLAGS "-fuse-ld=lld" CACHE STRING "")
     set(BUILTINS_${target}_CMAKE_MODULE_LINKER_FLAGS "-fuse-ld=lld" CACHE STRING "")
@@ -78,11 +110,14 @@ foreach(target aarch64-unknown-linux-gnu;armv7-unknown-linux-gnueabihf;i386-unkn
     list(APPEND RUNTIME_TARGETS "${target}")
     set(RUNTIMES_${target}_CMAKE_SYSTEM_NAME Linux CACHE STRING "")
     set(RUNTIMES_${target}_CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "")
+    set(RUNTIMES_${target}_CMAKE_C_FLAGS "--target=${target}" CACHE STRING "")
+    set(RUNTIMES_${target}_CMAKE_CXX_FLAGS "--target=${target}" CACHE STRING "")
+    set(RUNTIMES_${target}_CMAKE_ASM_FLAGS "--target=${target}" CACHE STRING "")
     set(RUNTIMES_${target}_CMAKE_SYSROOT ${LINUX_${target}_SYSROOT} CACHE STRING "")
     set(RUNTIMES_${target}_CMAKE_SHARED_LINKER_FLAGS "-fuse-ld=lld" CACHE STRING "")
     set(RUNTIMES_${target}_CMAKE_MODULE_LINKER_FLAGS "-fuse-ld=lld" CACHE STRING "")
     set(RUNTIMES_${target}_CMAKE_EXE_LINKER_FLAGS "-fuse-ld=lld" CACHE STRING "")
-    set(RUNTIMES_${target}_LLVM_ENABLE_ASSERTIONS ON CACHE BOOL "")
+    set(RUNTIMES_${target}_COMPILER_RT_USE_BUILTINS_LIBRARY ON CACHE BOOL "")
     set(RUNTIMES_${target}_LIBUNWIND_ENABLE_SHARED OFF CACHE BOOL "")
     set(RUNTIMES_${target}_LIBUNWIND_USE_COMPILER_RT ON CACHE BOOL "")
     set(RUNTIMES_${target}_LIBUNWIND_INSTALL_LIBRARY OFF CACHE BOOL "")
@@ -95,9 +130,10 @@ foreach(target aarch64-unknown-linux-gnu;armv7-unknown-linux-gnueabihf;i386-unkn
     set(RUNTIMES_${target}_LIBCXX_ENABLE_SHARED OFF CACHE BOOL "")
     set(RUNTIMES_${target}_LIBCXX_ENABLE_STATIC_ABI_LIBRARY ON CACHE BOOL "")
     set(RUNTIMES_${target}_LIBCXX_ABI_VERSION 2 CACHE STRING "")
+    set(RUNTIMES_${target}_LLVM_ENABLE_ASSERTIONS ON CACHE BOOL "")
     set(RUNTIMES_${target}_SANITIZER_CXX_ABI "libc++" CACHE STRING "")
     set(RUNTIMES_${target}_SANITIZER_CXX_ABI_INTREE ON CACHE BOOL "")
-    set(RUNTIMES_${target}_COMPILER_RT_USE_BUILTINS_LIBRARY ON CACHE BOOL "")
+    set(RUNTIMES_${target}_LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")
 
     # Use .build-id link.
     list(APPEND RUNTIME_BUILD_ID_LINK "${target}")
@@ -110,7 +146,7 @@ if(FUCHSIA_SDK)
   set(FUCHSIA_x86_64_NAME x64)
   set(FUCHSIA_riscv64_NAME riscv64)
   foreach(target i386;x86_64;aarch64;riscv64)
-    set(FUCHSIA_${target}_COMPILER_FLAGS "-I${FUCHSIA_SDK}/pkg/fdio/include")
+    set(FUCHSIA_${target}_COMPILER_FLAGS "--target=${target}-unknown-fuchsia -I${FUCHSIA_SDK}/pkg/fdio/include")
     set(FUCHSIA_${target}_LINKER_FLAGS "-L${FUCHSIA_SDK}/arch/${FUCHSIA_${target}_NAME}/lib")
     set(FUCHSIA_${target}_SYSROOT "${FUCHSIA_SDK}/arch/${FUCHSIA_${target}_NAME}/sysroot")
   endforeach()
@@ -142,7 +178,7 @@ if(FUCHSIA_SDK)
     set(RUNTIMES_${target}-unknown-fuchsia_CMAKE_MODULE_LINKER_FLAGS ${FUCHSIA_${target}_LINKER_FLAGS} CACHE STRING "")
     set(RUNTIMES_${target}-unknown-fuchsia_CMAKE_EXE_LINKER_FLAGS ${FUCHSIA_${target}_LINKER_FLAGS} CACHE STRING "")
     set(RUNTIMES_${target}-unknown-fuchsia_CMAKE_SYSROOT ${FUCHSIA_${target}_SYSROOT} CACHE PATH "")
-    set(RUNTIMES_${target}-unknown-fuchsia_LLVM_ENABLE_ASSERTIONS ON CACHE BOOL "")
+    set(RUNTIMES_${target}-unknown-fuchsia_COMPILER_RT_USE_BUILTINS_LIBRARY ON CACHE BOOL "")
     set(RUNTIMES_${target}-unknown-fuchsia_LIBUNWIND_USE_COMPILER_RT ON CACHE BOOL "")
     set(RUNTIMES_${target}-unknown-fuchsia_LIBUNWIND_HERMETIC_STATIC_LIBRARY ON CACHE BOOL "")
     set(RUNTIMES_${target}-unknown-fuchsia_LIBUNWIND_INSTALL_STATIC_LIBRARY OFF CACHE BOOL "")
@@ -157,6 +193,8 @@ if(FUCHSIA_SDK)
     set(RUNTIMES_${target}-unknown-fuchsia_LIBCXX_HERMETIC_STATIC_LIBRARY ON CACHE BOOL "")
     set(RUNTIMES_${target}-unknown-fuchsia_LIBCXX_STATICALLY_LINK_ABI_IN_SHARED_LIBRARY OFF CACHE BOOL "")
     set(RUNTIMES_${target}-unknown-fuchsia_LIBCXX_ABI_VERSION 2 CACHE STRING "")
+    set(RUNTIMES_${target}-unknown-fuchsia_LLVM_ENABLE_ASSERTIONS ON CACHE BOOL "")
+    set(RUNTIMES_${target}-unknown-fuchsia_LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")
 
     set(RUNTIMES_${target}-unknown-fuchsia+asan_LLVM_BUILD_COMPILER_RT OFF CACHE BOOL "")
     set(RUNTIMES_${target}-unknown-fuchsia+asan_LLVM_USE_SANITIZER "Address" CACHE STRING "")
diff --git a/clang/cmake/caches/Fuchsia.cmake b/clang/cmake/caches/Fuchsia.cmake
index 2a65de2039c2e..8688b71ecc753 100644
--- a/clang/cmake/caches/Fuchsia.cmake
+++ b/clang/cmake/caches/Fuchsia.cmake
@@ -4,6 +4,8 @@ set(LLVM_TARGETS_TO_BUILD X86;ARM;AArch64;RISCV CACHE STRING "")
 
 set(PACKAGE_VENDOR Fuchsia CACHE STRING "")
 
+set(LLVM_ENABLE_PROJECTS "clang;clang-tools-extra;lld;llvm" CACHE STRING "")
+
 set(LLVM_ENABLE_BACKTRACES OFF CACHE BOOL "")
 set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR ON CACHE BOOL "")
 set(LLVM_ENABLE_TERMINFO OFF CACHE BOOL "")
@@ -13,6 +15,10 @@ set(LLVM_INCLUDE_DOCS OFF CACHE BOOL "")
 set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "")
 set(LLVM_INCLUDE_GO_TESTS OFF CACHE BOOL "")
 
+if(MSVC)
+  set(LLVM_USE_CRT_RELEASE "MT" CACHE STRING "")
+endif()
+
 set(CLANG_DEFAULT_CXX_STDLIB libc++ CACHE STRING "")
 if(NOT APPLE)
   set(CLANG_DEFAULT_LINKER lld CACHE STRING "")
@@ -29,8 +35,10 @@ set(ENABLE_X86_RELAX_RELOCATIONS ON CACHE BOOL "")
 
 set(LLVM_ENABLE_ASSERTIONS ON CACHE BOOL "")
 set(CMAKE_BUILD_TYPE Release CACHE STRING "")
-if (APPLE)
+if(APPLE)
   set(MACOSX_DEPLOYMENT_TARGET 10.7 CACHE STRING "")
+elseif(MSVC)
+  set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded" CACHE STRING "")
 endif()
 
 if(APPLE)
@@ -49,20 +57,33 @@ set(LIBCXXABI_USE_COMPILER_RT ON CACHE BOOL "")
 set(LIBCXXABI_USE_LLVM_UNWINDER ON CACHE BOOL "")
 set(LIBCXX_ABI_VERSION 2 CACHE STRING "")
 set(LIBCXX_ENABLE_SHARED OFF CACHE BOOL "")
-set(LIBCXX_ENABLE_STATIC_ABI_LIBRARY ON CACHE BOOL "")
-set(LIBCXX_USE_COMPILER_RT ON CACHE BOOL "")
+if(WIN32)
+  set(LIBCXX_HAS_WIN32_THREAD_API ON CACHE BOOL "")
+  set(LIBCXX_ENABLE_EXPERIMENTAL_LIBRARY OFF CACHE BOOL "")
+  set(LIBCXX_ENABLE_FILESYSTEM OFF CACHE BOOL "")
+  set(LIBCXX_ENABLE_ABI_LINKER_SCRIPT OFF CACHE BOOL "")
+  set(LIBCXX_ENABLE_STATIC_ABI_LIBRARY OFF CACHE BOOL "")
+  set(BUILTINS_CMAKE_ARGS -DCMAKE_SYSTEM_NAME=Windows CACHE STRING "")
+  set(RUNTIMES_CMAKE_ARGS -DCMAKE_SYSTEM_NAME=Windows CACHE STRING "")
+  set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx" CACHE STRING "")
+else()
+  set(LIBCXX_ENABLE_STATIC_ABI_LIBRARY ON CACHE BOOL "")
+  set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")
+endif()
 
 if(BOOTSTRAP_CMAKE_SYSTEM_NAME)
   set(target "${BOOTSTRAP_CMAKE_CXX_COMPILER_TARGET}")
   if(STAGE2_LINUX_${target}_SYSROOT)
+    set(LLVM_BUILTIN_TARGETS "${target}" CACHE STRING "")
     set(BUILTINS_${target}_CMAKE_SYSTEM_NAME Linux CACHE STRING "")
     set(BUILTINS_${target}_CMAKE_BUILD_TYPE Release CACHE STRING "")
     set(BUILTINS_${target}_CMAKE_SYSROOT ${STAGE2_LINUX_${target}_SYSROOT} CACHE STRING "")
-    set(LLVM_BUILTIN_TARGETS "${target}" CACHE STRING "")
 
+    set(LLVM_RUNTIME_TARGETS "${target}" CACHE STRING "")
     set(RUNTIMES_${target}_CMAKE_SYSTEM_NAME Linux CACHE STRING "")
     set(RUNTIMES_${target}_CMAKE_BUILD_TYPE Release CACHE STRING "")
     set(RUNTIMES_${target}_CMAKE_SYSROOT ${STAGE2_LINUX_${target}_SYSROOT} CACHE STRING "")
+    set(RUNTIMES_${target}_COMPILER_RT_USE_BUILTINS_LIBRARY ON CACHE BOOL "")
     set(RUNTIMES_${target}_LLVM_ENABLE_ASSERTIONS ON CACHE BOOL "")
     set(RUNTIMES_${target}_LIBUNWIND_ENABLE_SHARED OFF CACHE BOOL "")
     set(RUNTIMES_${target}_LIBUNWIND_USE_COMPILER_RT ON CACHE BOOL "")
@@ -76,10 +97,9 @@ if(BOOTSTRAP_CMAKE_SYSTEM_NAME)
     set(RUNTIMES_${target}_LIBCXX_ENABLE_SHARED OFF CACHE BOOL "")
     set(RUNTIMES_${target}_LIBCXX_ENABLE_STATIC_ABI_LIBRARY ON CACHE BOOL "")
     set(RUNTIMES_${target}_LIBCXX_ABI_VERSION 2 CACHE STRING "")
+    set(RUNTIMES_${target}_LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")
     set(RUNTIMES_${target}_SANITIZER_CXX_ABI "libc++" CACHE STRING "")
     set(RUNTIMES_${target}_SANITIZER_CXX_ABI_INTREE ON CACHE BOOL "")
-    set(RUNTIMES_${target}_COMPILER_RT_USE_BUILTINS_LIBRARY ON CACHE BOOL "")
-    set(LLVM_RUNTIME_TARGETS "${target}" CACHE STRING "")
   endif()
 endif()
 
diff --git a/clang/cmake/modules/ClangConfig.cmake.in b/clang/cmake/modules/ClangConfig.cmake.in
index a5a7eae503282..c5ce56274b198 100644
--- a/clang/cmake/modules/ClangConfig.cmake.in
+++ b/clang/cmake/modules/ClangConfig.cmake.in
@@ -8,6 +8,7 @@ find_package(LLVM REQUIRED CONFIG
 set(CLANG_EXPORTED_TARGETS "@CLANG_EXPORTS@")
 set(CLANG_CMAKE_DIR "@CLANG_CONFIG_CMAKE_DIR@")
 set(CLANG_INCLUDE_DIRS "@CLANG_CONFIG_INCLUDE_DIRS@")
+set(CLANG_LINK_CLANG_DYLIB "@CLANG_LINK_CLANG_DYLIB@")
 
 # Provide all our library targets to users.
 include("@CLANG_CONFIG_EXPORTS_FILE@")
diff --git a/clang/docs/ClangCommandLineReference.rst b/clang/docs/ClangCommandLineReference.rst
index 67c341feffbb6..1613c8e453184 100644
--- a/clang/docs/ClangCommandLineReference.rst
+++ b/clang/docs/ClangCommandLineReference.rst
@@ -818,6 +818,10 @@ Discard value names in LLVM IR
 
 Enables an experimental new pass manager in LLVM.
 
+.. option:: -fexperimental-strict-floating-point
+
+Enables the use of non-default rounding modes and non-default exception handling on targets that are not currently ready.
+
 .. option:: -ffine-grained-bitfield-accesses, -fno-fine-grained-bitfield-accesses
 
 Use separate accesses for consecutive bitfield runs with legal widths and alignments.
@@ -2755,6 +2759,10 @@ Generate a \_\_mcount\_loc section entry for each \_\_fentry\_\_ call.
 
 Make StdCall calling convention the default
 
+.. option:: -mseses, -mno-seses
+
+Enable speculative execution side effect suppression (SESES). Includes LVI control flow integrity mitigations
+
 .. option:: -msign-return-address=<arg>
 
 Select return address signing scope
@@ -3127,6 +3135,12 @@ X86
 
 .. option:: -maes, -mno-aes
 
+.. option:: -mamx-bf16, -mno-amx-bf16
+
+.. option:: -mamx-int8, -mno-amx-int8
+
+.. option:: -mamx-tile, -mno-amx-tile
+
 .. option:: -mavx, -mno-avx
 
 .. option:: -mavx2, -mno-avx2
diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index 496e4c651921a..e84676760c300 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -2694,6 +2694,23 @@ the configuration (without a prefix: ``Auto``).
     Use tabs whenever we need to fill whitespace that spans at least from
     one tab stop to the next one.
 
+**WhitespaceSensitiveMacros** (``std::vector<std::string>``)
+  A vector of macros which are whitespace-sensitive and should not be touched.
+
+  These are expected to be macros of the form:
+
+  .. code-block:: c++
+
+    STRINGIZE(...)
+
+  In the .clang-format configuration file, this can be configured like:
+
+  .. code-block:: yaml
+
+    WhitespaceSensitiveMacros: ['STRINGIZE', 'PP_STRINGIZE']
+
+  For example: BOOST_PP_STRINGIZE.
+
 
 
 .. END_FORMAT_STYLE_OPTIONS
diff --git a/clang/docs/ClangFormattedStatus.rst b/clang/docs/ClangFormattedStatus.rst
index 3e140c40070c6..3e348eadd4ffa 100644
--- a/clang/docs/ClangFormattedStatus.rst
+++ b/clang/docs/ClangFormattedStatus.rst
@@ -17,7 +17,7 @@ Clang Formatted Status
 ======================
 
 :doc:`ClangFormattedStatus` describes the state of LLVM source
-tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52 (`4b776a98f1a <https://github.com/llvm/llvm-project/commit/4b776a98f1a>`_).
+tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24 (`eb50838ba08 <https://github.com/llvm/llvm-project/commit/eb50838ba08>`_).
 
 
 .. list-table:: LLVM Clang-Format Status
@@ -91,9 +91,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :none:`0%`
    * - clang/include/clang/AST
      - `113`
-     - `21`
-     - `92`
-     - :part:`18%`
+     - `20`
+     - `93`
+     - :part:`17%`
    * - clang/include/clang/ASTMatchers
      - `5`
      - `1`
@@ -260,10 +260,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `0`
      - :good:`100%`
    * - clang/include/clang/Tooling/Transformer
+     - `8`
      - `7`
-     - `6`
      - `1`
-     - :part:`85%`
+     - :part:`87%`
    * - clang/include/clang-c
      - `9`
      - `3`
@@ -325,10 +325,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `26`
      - :part:`23%`
    * - clang/lib/Basic/Targets
-     - `46`
-     - `21`
+     - `48`
+     - `23`
      - `25`
-     - :part:`45%`
+     - :part:`47%`
    * - clang/lib/CodeGen
      - `87`
      - `8`
@@ -370,15 +370,15 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `14`
      - :part:`12%`
    * - clang/lib/Driver/ToolChains
-     - `81`
-     - `25`
+     - `83`
+     - `27`
      - `56`
-     - :part:`30%`
+     - :part:`32%`
    * - clang/lib/Driver/ToolChains/Arch
-     - `16`
-     - `2`
+     - `18`
+     - `4`
      - `14`
-     - :part:`12%`
+     - :part:`22%`
    * - clang/lib/Edit
      - `3`
      - `0`
@@ -545,10 +545,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `1`
      - :part:`85%`
    * - clang/lib/Tooling/Transformer
-     - `6`
-     - `3`
+     - `7`
+     - `4`
      - `3`
-     - :part:`50%`
+     - :part:`57%`
    * - clang/tools/arcmt-test
      - `1`
      - `0`
@@ -616,9 +616,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :none:`0%`
    * - clang/tools/clang-import-test
      - `1`
-     - `0`
      - `1`
-     - :none:`0%`
+     - `0`
+     - :good:`100%`
    * - clang/tools/clang-offload-bundler
      - `1`
      - `0`
@@ -720,10 +720,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `1`
      - :part:`94%`
    * - clang/unittests/Frontend
-     - `8`
-     - `4`
+     - `9`
+     - `5`
      - `4`
-     - :part:`50%`
+     - :part:`55%`
    * - clang/unittests/Index
      - `1`
      - `1`
@@ -886,9 +886,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :none:`0%`
    * - clang-tools-extra/clang-tidy
      - `18`
-     - `12`
-     - `6`
-     - :part:`66%`
+     - `10`
+     - `8`
+     - :part:`55%`
    * - clang-tools-extra/clang-tidy/abseil
      - `40`
      - `28`
@@ -906,8 +906,8 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :good:`100%`
    * - clang-tools-extra/clang-tidy/bugprone
      - `105`
-     - `84`
-     - `21`
+     - `85`
+     - `20`
      - :part:`80%`
    * - clang-tools-extra/clang-tidy/cert
      - `29`
@@ -961,9 +961,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :part:`86%`
    * - clang-tools-extra/clang-tidy/modernize
      - `67`
-     - `45`
-     - `22`
-     - :part:`67%`
+     - `46`
+     - `21`
+     - :part:`68%`
    * - clang-tools-extra/clang-tidy/mpi
      - `5`
      - `4`
@@ -1015,10 +1015,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `0`
      - :good:`100%`
    * - clang-tools-extra/clangd
-     - `73`
-     - `57`
-     - `16`
-     - :part:`78%`
+     - `75`
+     - `58`
+     - `17`
+     - :part:`77%`
    * - clang-tools-extra/clangd/benchmarks
      - `1`
      - `1`
@@ -1090,10 +1090,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `0`
      - :good:`100%`
    * - clang-tools-extra/clangd/unittests
-     - `63`
-     - `50`
-     - `13`
-     - :part:`79%`
+     - `64`
+     - `52`
+     - `12`
+     - :part:`81%`
    * - clang-tools-extra/clangd/unittests/remote
      - `1`
      - `1`
@@ -1331,9 +1331,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :part:`14%`
    * - compiler-rt/lib/sanitizer_common/symbolizer
      - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
+     - `2`
+     - `0`
+     - :good:`100%`
    * - compiler-rt/lib/sanitizer_common/tests
      - `38`
      - `1`
@@ -1495,15 +1495,20 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `0`
      - :good:`100%`
    * - flang/include/flang/Evaluate
-     - `21`
-     - `21`
+     - `23`
+     - `23`
      - `0`
      - :good:`100%`
    * - flang/include/flang/Lower
-     - `9`
-     - `8`
+     - `13`
+     - `12`
      - `1`
-     - :part:`88%`
+     - :part:`92%`
+   * - flang/include/flang/Lower/Support
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - flang/include/flang/Optimizer/CodeGen
      - `1`
      - `1`
@@ -1540,15 +1545,15 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `0`
      - :good:`100%`
    * - flang/lib/Evaluate
-     - `30`
-     - `30`
+     - `32`
+     - `32`
      - `0`
      - :good:`100%`
    * - flang/lib/Lower
-     - `6`
-     - `5`
+     - `10`
+     - `9`
      - `1`
-     - :part:`83%`
+     - :part:`90%`
    * - flang/lib/Optimizer/Dialect
      - `4`
      - `4`
@@ -1566,9 +1571,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :good:`100%`
    * - flang/lib/Semantics
      - `68`
-     - `67`
-     - `1`
-     - :part:`98%`
+     - `65`
+     - `3`
+     - :part:`95%`
    * - flang/module
      - `1`
      - `0`
@@ -1650,10 +1655,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `0`
      - :good:`100%`
    * - libc/fuzzing/string
+     - `2`
      - `1`
-     - `0`
      - `1`
-     - :none:`0%`
+     - :part:`50%`
    * - libc/include
      - `4`
      - `4`
@@ -1675,8 +1680,8 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `0`
      - :good:`100%`
    * - libc/src/math
-     - `52`
-     - `52`
+     - `70`
+     - `70`
      - `0`
      - :good:`100%`
    * - libc/src/signal
@@ -1705,10 +1710,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `0`
      - :good:`100%`
    * - libc/src/string
-     - `12`
-     - `11`
+     - `14`
+     - `13`
      - `1`
-     - :part:`91%`
+     - :part:`92%`
    * - libc/src/string/memory_utils
      - `3`
      - `3`
@@ -1755,10 +1760,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `0`
      - :good:`100%`
    * - libc/utils/FPUtil
-     - `7`
-     - `6`
-     - `1`
-     - :part:`85%`
+     - `9`
+     - `9`
+     - `0`
+     - :good:`100%`
    * - libc/utils/HdrGen
      - `9`
      - `9`
@@ -2651,9 +2656,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :part:`50%`
    * - lldb/source/Plugins/Platform/MacOSX
      - `24`
-     - `8`
-     - `16`
-     - :part:`33%`
+     - `10`
+     - `14`
+     - :part:`41%`
    * - lldb/source/Plugins/Platform/MacOSX/objcxx
      - `1`
      - `1`
@@ -2786,9 +2791,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :part:`53%`
    * - lldb/source/Plugins/SymbolFile/NativePDB
      - `20`
-     - `12`
-     - `8`
-     - :part:`60%`
+     - `11`
+     - `9`
+     - :part:`55%`
    * - lldb/source/Plugins/SymbolFile/PDB
      - `6`
      - `4`
@@ -2999,6 +3004,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `0`
      - `2`
      - :none:`0%`
+   * - lldb/unittests/Language/CLanguages
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - lldb/unittests/Language/CPlusPlus
      - `1`
      - `0`
@@ -3390,10 +3400,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `60`
      - :part:`28%`
    * - llvm/include/llvm/Analysis
-     - `109`
-     - `30`
+     - `113`
+     - `34`
      - `79`
-     - :part:`27%`
+     - :part:`30%`
    * - llvm/include/llvm/Analysis/Utils
      - `1`
      - `0`
@@ -3421,9 +3431,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :none:`0%`
    * - llvm/include/llvm/CodeGen
      - `141`
-     - `33`
-     - `108`
-     - :part:`23%`
+     - `32`
+     - `109`
+     - :part:`22%`
    * - llvm/include/llvm/CodeGen/GlobalISel
      - `27`
      - `10`
@@ -3740,10 +3750,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `6`
      - :part:`25%`
    * - llvm/lib/Analysis
-     - `106`
-     - `30`
+     - `109`
+     - `33`
      - `76`
-     - :part:`28%`
+     - :part:`30%`
    * - llvm/lib/AsmParser
      - `6`
      - `2`
@@ -3906,9 +3916,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :good:`100%`
    * - llvm/lib/Frontend/OpenMP
      - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
+     - `2`
+     - `1`
+     - :part:`66%`
    * - llvm/lib/FuzzMutate
      - `5`
      - `2`
@@ -3916,9 +3926,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :part:`40%`
    * - llvm/lib/IR
      - `61`
-     - `9`
-     - `52`
-     - :part:`14%`
+     - `11`
+     - `50`
+     - :part:`18%`
    * - llvm/lib/IRReader
      - `1`
      - `0`
@@ -3976,9 +3986,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :part:`34%`
    * - llvm/lib/ObjectYAML
      - `22`
-     - `12`
-     - `10`
-     - :part:`54%`
+     - `13`
+     - `9`
+     - :part:`59%`
    * - llvm/lib/Option
      - `4`
      - `0`
@@ -4061,9 +4071,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :none:`0%`
    * - llvm/lib/Target/AMDGPU
      - `146`
-     - `11`
-     - `135`
-     - :part:`7%`
+     - `12`
+     - `134`
+     - :part:`8%`
    * - llvm/lib/Target/AMDGPU/AsmParser
      - `1`
      - `0`
@@ -4386,9 +4396,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :good:`100%`
    * - llvm/lib/Target/SystemZ
      - `40`
-     - `3`
-     - `37`
-     - :part:`7%`
+     - `4`
+     - `36`
+     - :part:`10%`
    * - llvm/lib/Target/SystemZ/AsmParser
      - `1`
      - `0`
@@ -4436,9 +4446,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :part:`50%`
    * - llvm/lib/Target/WebAssembly
      - `58`
-     - `42`
-     - `16`
-     - :part:`72%`
+     - `40`
+     - `18`
+     - :part:`68%`
    * - llvm/lib/Target/WebAssembly/AsmParser
      - `1`
      - `0`
@@ -4461,9 +4471,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :good:`100%`
    * - llvm/lib/Target/X86
      - `75`
-     - `12`
-     - `63`
-     - :part:`16%`
+     - `11`
+     - `64`
+     - :part:`14%`
    * - llvm/lib/Target/X86/AsmParser
      - `3`
      - `0`
@@ -4556,14 +4566,14 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :part:`6%`
    * - llvm/lib/Transforms/Instrumentation
      - `21`
-     - `2`
-     - `19`
-     - :part:`9%`
+     - `3`
+     - `18`
+     - :part:`14%`
    * - llvm/lib/Transforms/IPO
      - `39`
-     - `4`
-     - `35`
-     - :part:`10%`
+     - `5`
+     - `34`
+     - :part:`12%`
    * - llvm/lib/Transforms/ObjCARC
      - `15`
      - `3`
@@ -4736,9 +4746,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :good:`100%`
    * - llvm/tools/llvm-exegesis/lib
      - `44`
-     - `34`
-     - `10`
-     - :part:`77%`
+     - `35`
+     - `9`
+     - :part:`79%`
    * - llvm/tools/llvm-exegesis/lib/AArch64
      - `1`
      - `1`
@@ -5070,10 +5080,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `8`
      - :part:`20%`
    * - llvm/unittests/CodeGen/GlobalISel
-     - `10`
+     - `11`
      - `1`
-     - `9`
-     - :part:`10%`
+     - `10`
+     - :part:`9%`
    * - llvm/unittests/DebugInfo/CodeView
      - `3`
      - `1`
@@ -5690,10 +5700,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `0`
      - :good:`100%`
    * - llvm/utils/TableGen
-     - `74`
-     - `8`
+     - `75`
+     - `9`
      - `66`
-     - :part:`10%`
+     - :part:`12%`
    * - llvm/utils/TableGen/GlobalISel
      - `17`
      - `8`
@@ -6051,9 +6061,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :good:`100%`
    * - mlir/include/mlir/Dialect/Linalg/EDSC
      - `3`
-     - `2`
-     - `1`
-     - :part:`66%`
+     - `3`
+     - `0`
+     - :good:`100%`
    * - mlir/include/mlir/Dialect/Linalg/IR
      - `3`
      - `2`
@@ -6175,10 +6185,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `6`
      - :none:`0%`
    * - mlir/include/mlir/Support
-     - `9`
-     - `5`
+     - `10`
      - `4`
-     - :part:`55%`
+     - `6`
+     - :part:`40%`
    * - mlir/include/mlir/TableGen
      - `18`
      - `17`
@@ -6286,9 +6296,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :good:`100%`
    * - mlir/lib/Conversion/SPIRVToLLVM
      - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
+     - `1`
+     - `1`
+     - :part:`50%`
    * - mlir/lib/Conversion/StandardToLLVM
      - `1`
      - `1`
@@ -6435,23 +6445,23 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `0`
      - :good:`100%`
    * - mlir/lib/Dialect/Shape/Transforms
-     - `2`
-     - `2`
+     - `3`
+     - `3`
      - `0`
      - :good:`100%`
    * - mlir/lib/Dialect/SPIRV
      - `8`
-     - `5`
-     - `3`
-     - :part:`62%`
+     - `6`
+     - `2`
+     - :part:`75%`
    * - mlir/lib/Dialect/SPIRV/Serialization
      - `4`
      - `2`
      - `2`
      - :part:`50%`
    * - mlir/lib/Dialect/SPIRV/Transforms
-     - `4`
-     - `4`
+     - `5`
+     - `5`
      - `0`
      - :good:`100%`
    * - mlir/lib/Dialect/StandardOps/EDSC
@@ -6465,8 +6475,8 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `0`
      - :good:`100%`
    * - mlir/lib/Dialect/StandardOps/Transforms
-     - `3`
-     - `3`
+     - `4`
+     - `4`
      - `0`
      - :good:`100%`
    * - mlir/lib/Dialect/Vector
@@ -6481,9 +6491,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :good:`100%`
    * - mlir/lib/EDSC
      - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
+     - `1`
+     - `1`
+     - :part:`50%`
    * - mlir/lib/ExecutionEngine
      - `5`
      - `5`
@@ -6526,14 +6536,14 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :good:`100%`
    * - mlir/lib/Transforms
      - `19`
-     - `17`
-     - `2`
-     - :part:`89%`
+     - `16`
+     - `3`
+     - :part:`84%`
    * - mlir/lib/Transforms/Utils
      - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
+     - `6`
+     - `1`
+     - :part:`85%`
    * - mlir/lib/Translation
      - `1`
      - `1`
@@ -6571,9 +6581,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - :good:`100%`
    * - mlir/tools/mlir-tblgen
      - `16`
-     - `14`
-     - `2`
-     - :part:`87%`
+     - `16`
+     - `0`
+     - :good:`100%`
    * - mlir/tools/mlir-translate
      - `1`
      - `1`
@@ -6850,7 +6860,7 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 17, 2020 22:15:52
      - `4`
      - :part:`81%`
    * - Total
-     - :total:`13959`
-     - :total:`6696`
-     - :total:`7263`
-     - :total:`47%`
+     - :total:`14021`
+     - :total:`6759`
+     - :total:`7262`
+     - :total:`48%`
diff --git a/clang/docs/CommandGuide/clang.rst b/clang/docs/CommandGuide/clang.rst
index de0e0eda90974..5978650c32888 100644
--- a/clang/docs/CommandGuide/clang.rst
+++ b/clang/docs/CommandGuide/clang.rst
@@ -474,6 +474,16 @@ Code Generation Options
   optimization. With "thin", :doc:`ThinLTO <../ThinLTO>`
   compilation is invoked instead.
 
+  .. note::
+
+     On Darwin, when using :option:`-flto` along with :option:`-g` and
+     compiling and linking in separate steps, you also need to pass
+     ``-Wl,-object_path_lto,<lto-filename>.o`` at the linking step to instruct the
+     ld64 linker not to delete the temporary object file generated during Link
+     Time Optimization (this flag is automatically passed to the linker by Clang
+     if compilation and linking are done in a single step). This allows debugging
+     the executable as well as generating the ``.dSYM`` bundle using :manpage:`dsymutil(1)`.
+
 Driver Options
 ~~~~~~~~~~~~~~
 
diff --git a/clang/docs/LibASTMatchersReference.html b/clang/docs/LibASTMatchersReference.html
index bb5e4984fcdf4..60ff6ffe60567 100644
--- a/clang/docs/LibASTMatchersReference.html
+++ b/clang/docs/LibASTMatchersReference.html
@@ -676,9 +676,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
   #pragma omp parallel default(none)
   #pragma omp parallel default(shared)
+  #pragma omp parallel default(firstprivate)
   #pragma omp parallel
 
-``ompDefaultClause()`` matches ``default(none)`` and ``default(shared)``.
+``ompDefaultClause()`` matches ``default(none)``, ``default(shared)``, and ``default(firstprivate)``.
 </pre></td></tr>
 
 
@@ -1226,6 +1227,11 @@ <h2 id="decl-matchers">Node Matchers</h2>
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('fixedPointLiteral0')"><a name="fixedPointLiteral0Anchor">fixedPointLiteral</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FixedPointLiteral.html">FixedPointLiteral</a>&gt;...</td></tr>
+<tr><td colspan="4" class="doc" id="fixedPointLiteral0"><pre>Matches fixed point literals
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('floatLiteral0')"><a name="floatLiteral0Anchor">floatLiteral</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FloatingLiteral.html">FloatingLiteral</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="floatLiteral0"><pre>Matches float literals of all sizes / encodings, e.g.
 1.0, 1.0f, 1.0L and 1e10.
@@ -3002,7 +3008,7 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('isExpansionInFileMatching0')"><a name="isExpansionInFileMatching0Anchor">isExpansionInFileMatching</a></td><td>std::string RegExp</td></tr>
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('isExpansionInFileMatching0')"><a name="isExpansionInFileMatching0Anchor">isExpansionInFileMatching</a></td><td>StringRef RegExp, Regex::RegexFlags Flags = NoFlags</td></tr>
 <tr><td colspan="4" class="doc" id="isExpansionInFileMatching0"><pre>Matches AST nodes that were expanded within files whose name is
 partially matching a given regex.
 
@@ -3014,6 +3020,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   class Y {};
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;
+
+If the matcher is used in clang-query, RegexFlags parameter
+should be passed as a quoted string. e.g: "NoFlags".
+Flags can be combined with '|' example "IgnoreCase | BasicRegex"
 </pre></td></tr>
 
 
@@ -3720,7 +3730,7 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1NamedDecl.html">NamedDecl</a>&gt;</td><td class="name" onclick="toggle('matchesName0')"><a name="matchesName0Anchor">matchesName</a></td><td>std::string RegExp</td></tr>
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1NamedDecl.html">NamedDecl</a>&gt;</td><td class="name" onclick="toggle('matchesName0')"><a name="matchesName0Anchor">matchesName</a></td><td>StringRef RegExp, Regex::RegexFlags Flags = NoFlags</td></tr>
 <tr><td colspan="4" class="doc" id="matchesName0"><pre>Matches NamedDecl nodes whose fully qualified names contain
 a substring matched by the given RegExp.
 
@@ -3733,6 +3743,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 
 Example matches X (regexp is one of "::X", "^foo::.*X", among others)
   namespace foo { namespace bar { class X; } }
+
+If the matcher is used in clang-query, RegexFlags parameter
+should be passed as a quoted string. e.g: "NoFlags".
+Flags can be combined with '|' example "IgnoreCase | BasicRegex"
 </pre></td></tr>
 
 
@@ -3770,6 +3784,7 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   #pragma omp parallel
   #pragma omp parallel default(none)
   #pragma omp parallel default(shared)
+  #pragma omp parallel default(firstprivate)
 
 ``ompDefaultClause(isNoneKind())`` matches only ``default(none)``.
 </pre></td></tr>
@@ -3783,11 +3798,26 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   #pragma omp parallel
   #pragma omp parallel default(none)
   #pragma omp parallel default(shared)
+  #pragma omp parallel default(firstprivate)
 
 ``ompDefaultClause(isSharedKind())`` matches only ``default(shared)``.
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1OMPDefaultClause.html">OMPDefaultClause</a>&gt;</td><td class="name" onclick="toggle('isFirstPrivateKind0')"><a name="isFirstPrivateKind0Anchor">isSharedKind</a></td><td></td></tr>
+<tr><td colspan="4" class="doc" id="isFirstPrivateKind0"><pre>Matches if the OpenMP ``default`` clause has ``firstprivate`` kind specified.
+
+Given
+
+  #pragma omp parallel
+  #pragma omp parallel default(none)
+  #pragma omp parallel default(shared)
+  #pragma omp parallel default(firstprivate)
+
+``ompDefaultClause(isFirstPrivateKind())`` matches only ``default(firstprivate)``.
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1OMPExecutableDirective.html">OMPExecutableDirective</a>&gt;</td><td class="name" onclick="toggle('isAllowedToContainClauseKind0')"><a name="isAllowedToContainClauseKind0Anchor">isAllowedToContainClauseKind</a></td><td>OpenMPClauseKind CKind</td></tr>
 <tr><td colspan="4" class="doc" id="isAllowedToContainClauseKind0"><pre>Matches if the OpenMP directive is allowed to contain the specified OpenMP
 clause kind.
@@ -3927,12 +3957,16 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCMessageExpr.html">ObjCMessageExpr</a>&gt;</td><td class="name" onclick="toggle('matchesSelector0')"><a name="matchesSelector0Anchor">matchesSelector</a></td><td>std::string RegExp</td></tr>
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCMessageExpr.html">ObjCMessageExpr</a>&gt;</td><td class="name" onclick="toggle('matchesSelector0')"><a name="matchesSelector0Anchor">matchesSelector</a></td><td>StringRef RegExp, Regex::RegexFlags Flags = NoFlags</td></tr>
 <tr><td colspan="4" class="doc" id="matchesSelector0"><pre>Matches ObjC selectors whose name contains
 a substring matched by the given RegExp.
  matcher = objCMessageExpr(matchesSelector("loadHTMLStringmatches the outer message expr in the code below, but NOT the message
  invocation for self.bodyView.
     [self.bodyView loadHTMLString:html baseURL:NULL];
+
+If the matcher is used in clang-query, RegexFlags parameter
+should be passed as a quoted string. e.g: "NoFlags".
+Flags can be combined with '|' example "IgnoreCase | BasicRegex"
 </pre></td></tr>
 
 
@@ -4016,6 +4050,23 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ParmVarDecl.html">ParmVarDecl</a>&gt;</td><td class="name" onclick="toggle('isAtPosition0')"><a name="isAtPosition0Anchor">isAtPosition</a></td><td>unsigned N</td></tr>
+<tr><td colspan="4" class="doc" id="isAtPosition0"><pre>Matches the ParmVarDecl nodes that are at the N'th position in the parameter
+list. The parameter list could be that of either a block, function, or
+objc-method.
+
+
+Given
+
+void f(int a, int b, int c) {
+}
+
+``parmVarDecl(isAtPosition(0))`` matches ``int a``.
+
+``parmVarDecl(isAtPosition(1))`` matches ``int b``.
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt;</td><td class="name" onclick="toggle('asString0')"><a name="asString0Anchor">asString</a></td><td>std::string Name</td></tr>
 <tr><td colspan="4" class="doc" id="asString0"><pre>Matches if the matched type is represented by the given string.
 
@@ -4206,7 +4257,7 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('isExpansionInFileMatching1')"><a name="isExpansionInFileMatching1Anchor">isExpansionInFileMatching</a></td><td>std::string RegExp</td></tr>
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('isExpansionInFileMatching1')"><a name="isExpansionInFileMatching1Anchor">isExpansionInFileMatching</a></td><td>StringRef RegExp, Regex::RegexFlags Flags = NoFlags</td></tr>
 <tr><td colspan="4" class="doc" id="isExpansionInFileMatching1"><pre>Matches AST nodes that were expanded within files whose name is
 partially matching a given regex.
 
@@ -4218,6 +4269,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   class Y {};
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;
+
+If the matcher is used in clang-query, RegexFlags parameter
+should be passed as a quoted string. e.g: "NoFlags".
+Flags can be combined with '|' example "IgnoreCase | BasicRegex"
 </pre></td></tr>
 
 
@@ -4388,7 +4443,7 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;</td><td class="name" onclick="toggle('isExpansionInFileMatching2')"><a name="isExpansionInFileMatching2Anchor">isExpansionInFileMatching</a></td><td>std::string RegExp</td></tr>
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;</td><td class="name" onclick="toggle('isExpansionInFileMatching2')"><a name="isExpansionInFileMatching2Anchor">isExpansionInFileMatching</a></td><td>StringRef RegExp, Regex::RegexFlags Flags = NoFlags</td></tr>
 <tr><td colspan="4" class="doc" id="isExpansionInFileMatching2"><pre>Matches AST nodes that were expanded within files whose name is
 partially matching a given regex.
 
@@ -4400,6 +4455,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   class Y {};
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;
+
+If the matcher is used in clang-query, RegexFlags parameter
+should be passed as a quoted string. e.g: "NoFlags".
+Flags can be combined with '|' example "IgnoreCase | BasicRegex"
 </pre></td></tr>
 
 
@@ -4755,23 +4814,6 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;
 </pre></td></tr>
 
-
-<tr><td>Matcher&lt;clang::ParmVarDecl&gt;</td><td class="name" onclick="toggle('isAtPosition0')"><a name="isAtPosition0Anchor">isAtPosition</a></td><td>unsigned N</td></tr>
-<tr><td colspan="4" class="doc" id="isAtPosition0"><pre>Matches the ParmVarDecl nodes that are at the N'th position in the parameter
-list. The parameter list could be that of either a block, function, or
-objc-method.
-
-
-Given
-
-void f(int a, int b, int c) {
-}
-
-``parmVarDecl(isAtPosition(0))`` matches ``int a``.
-
-``parmVarDecl(isAtPosition(1))`` matches ``int b``.
-</pre></td></tr>
-
 <!--END_NARROWING_MATCHERS -->
 </table>
 
@@ -5632,18 +5674,33 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyBase0')"><a name="hasAnyBase0Anchor">hasAnyBase</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt; BaseSpecMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasAnyBase0"><pre>Matches C++ classes that have a direct or indirect base matching BaseSpecMatcher.
 
-Example matches DirectlyDerived, IndirectlyDerived (BaseSpecMatcher ==
-hasType(cxxRecordDecl(hasName("SpecialBase")))) class Foo;
+Example:
+matcher hasAnyBase(hasType(cxxRecordDecl(hasName("SpecialBase"))))
+  class Foo;
   class Bar : Foo {};
   class Baz : Bar {};
   class SpecialBase;
-  class DirectlyDerived : SpecialBase {};  // directly derived
-  class IndirectlyDerived : DirectlyDerived {};  // indirectly derived
+  class Proxy : SpecialBase {};  // matches Proxy
+  class IndirectlyDerived : Proxy {};  //matches IndirectlyDerived
 
 FIXME: Refactor this and isDerivedFrom to reuse implementation.
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;</td><td class="name" onclick="toggle('hasDirectBase0')"><a name="hasDirectBase0Anchor">hasDirectBase</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt; BaseSpecMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasDirectBase0"><pre>Matches C++ classes that have a direct base matching BaseSpecMatcher.
+
+Example:
+matcher hasDirectBase(hasType(cxxRecordDecl(hasName("SpecialBase"))))
+  class Foo;
+  class Bar : Foo {};
+  class Baz : Bar {};
+  class SpecialBase;
+  class Proxy : SpecialBase {};  // matches Proxy
+  class IndirectlyDerived : Proxy {};  // doesn't match
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;</td><td class="name" onclick="toggle('hasMethod0')"><a name="hasMethod0Anchor">hasMethod</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasMethod0"><pre>Matches the first method of a class or struct that satisfies InnerMatcher.
 
diff --git a/clang/docs/MatrixTypes.rst b/clang/docs/MatrixTypes.rst
index 54099e5aae930..5d022af441210 100644
--- a/clang/docs/MatrixTypes.rst
+++ b/clang/docs/MatrixTypes.rst
@@ -245,8 +245,8 @@ and if omitted ``row`` is used as ``columnStride``.
 **Preconditions**: ``columnStride`` is greater than or equal to the number of rows in ``M``.
 
 **Remarks**: The type ``T`` is the const-unqualified version of the matrix
-argument’s element type. The paramter ``columnStride`` is optional and if
-ommitted, the number of rows of ``M`` is used as ``columnStride``.
+argument’s element type. The parameter ``columnStride`` is optional and if
+omitted, the number of rows of ``M`` is used as ``columnStride``.
 
 **Effects**: Equivalent to:
 
diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index f56980faaee8a..000f23141af30 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -131,7 +131,7 @@ implementation.
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | loop extension               | clause: if for SIMD directives                               | :good:`done`             |                                                                       |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
-| loop extension               | inclusive scan extension (matching C++17 PSTL)               | :none:`claimed`          |                                                                       |
+| loop extension               | inclusive scan extension (matching C++17 PSTL)               | :good:`done`             |                                                                       |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | memory mangagement           | memory allocators                                            | :good:`done`             | r341687,r357929                                                       |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
@@ -268,3 +268,5 @@ want to help with the implementation.
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | loop extension               | Loop tiling transformation                                   | :part:`claimed`          |                                                                       |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
+| device extension             | 'present' map type modifier                                  | :part:`claimed`          |                                                                       |
++------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 15e6d35117b42..8a9a58aa01f8f 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -94,8 +94,8 @@ New Compiler Flags
 ------------------
 
 - -fstack-clash-protection will provide a protection against the stack clash
-  attack for x86 and s390x architectures through automatic probing of each page
-  of allocated stack.
+  attack for x86, s390x and ppc64 architectures through automatic probing of
+  each page of allocated stack.
 
 - -ffp-exception-behavior={ignore,maytrap,strict} allows the user to specify
   the floating-point exception behavior. The default setting is ``ignore``.
@@ -104,6 +104,10 @@ New Compiler Flags
   simplify access to the many single purpose floating point options. The default
   setting is ``precise``.
 
+- The default module cache has moved from /tmp to a per-user cache directory.
+  By default, this is ~/.cache but on some platforms or installations, this
+  might be elsewhere. The -fmodules-cache-path=... flag continues to work.
+
 Deprecated Compiler Flags
 -------------------------
 
@@ -239,6 +243,34 @@ These are major API changes that have happened since the 10.0.0 release of
 Clang. If upgrading an external codebase that uses Clang as a library,
 this section should help get you past the largest hurdles of upgrading.
 
+- ``RecursiveASTVisitor`` no longer calls separate methods to visit specific
+  operator kinds. Previously, ``RecursiveASTVisitor`` treated unary, binary,
+  and compound assignment operators as if they were subclasses of the
+  corresponding AST node. For example, the binary operator plus was treated as
+  if it was a ``BinAdd`` subclass of the ``BinaryOperator`` class: during AST
+  traversal of a ``BinaryOperator`` AST node that had a ``BO_Add`` opcode,
+  ``RecursiveASTVisitor`` was calling the ``TraverseBinAdd`` method instead of
+  ``TraverseBinaryOperator``. This feature was contributing a non-trivial
+  amount of complexity to the implementation of ``RecursiveASTVisitor``, it was
+  used only in a minor way in Clang, was not tested, and as a result it was
+  buggy. Furthermore, this feature was creating a non-uniformity in the API.
+  Since this feature was not documented, it was quite difficult to figure out
+  how to use ``RecursiveASTVisitor`` to visit operators.
+
+  To update your code to the new uniform API, move the code from separate
+  visitation methods into methods that correspond to the actual AST node and
+  perform case analysis based on the operator opcode as needed:
+
+  * ``TraverseUnary*() => TraverseUnaryOperator()``
+  * ``WalkUpFromUnary*() => WalkUpFromUnaryOperator()``
+  * ``VisitUnary*() => VisiUnaryOperator()``
+  * ``TraverseBin*() => TraverseBinaryOperator()``
+  * ``WalkUpFromBin*() => WalkUpFromBinaryOperator()``
+  * ``VisitBin*() => VisiBinaryOperator()``
+  * ``TraverseBin*Assign() => TraverseCompoundAssignOperator()``
+  * ``WalkUpFromBin*Assign() => WalkUpFromCompoundAssignOperator()``
+  * ``VisitBin*Assign() => VisiCompoundAssignOperator()``
+
 Build System Changes
 --------------------
 
@@ -254,44 +286,7 @@ release of Clang. Users of the build system should adjust accordingly.
 AST Matchers
 ------------
 
-- Traversal in AST Matchers was simplified to use the
-  ``TK_IgnoreUnlessSpelledInSource`` mode by default, instead of ``TK_AsIs``.
-  This means that many uses of the ``ignoringImplicit()`` and similar matchers
-  is no longer necessary.  Clients of AST Matchers which wish to match on
-  implicit AST nodes can wrap their matcher in ``traverse(TK_AsIs, ...)`` or
-  use ``TraversalKindScope`` if appropriate.  The ``clang-query`` tool also
-  uses ``IgnoreUnlessSpelledInSource`` by default.  The mode can be changed
-  using ``set traversal AsIs`` in the ``clang-query`` environment.
-
-  As this change requires downstream tools which use AST Matchers to adapt
-  to the new default, a porting guide may be useful for downstream tools
-  needing to adapt.
-
-  Note that although there are many steps below, only the first is
-  non-optional. The steps are intentionally extemely granular to facilitate
-  understanding of the guide itself. It is reasonable to do some of the
-  steps at the same time if you understand the guide:
-
-  1. Use ``(your ASTContext instance).getParentMapContext().setTraversalKind(TK_AsIs)``
-     to restore the previous behavior for your tool.  All further steps in
-     this porting guide are optional.
-  2. Wrap your existing matcher expressions with ``traverse(TK_AsIs, ...)``
-     before passing them to ``ASTMatchFinder::addMatcher``.
-  3. Remove ``(your ASTContext instance).getParentMapContext().setTraversalKind(TK_AsIs)``
-     from your tool so that the default behavior of your tool matches the
-     default behavior of upstream clang. This is made possible by wrapping
-     your matchers in ``traverse(TK_AsIs, ...)`` from step (2).
-  4. Audit your matcher expressions and remove ``traverse(TK_AsIs, ...)``
-     where not needed.
-  5. Audit your matcher expressions and remove calls to ``ignoring*()``
-     matchers where not needed.
-  6. Audit your matcher expressions and consider whether the matcher is
-     better using the ``TK_AsIs`` mode or if it can be better expressed in
-     the default mode. For example, some matchers explicitly match
-     ``has(implicitCastExpr(has(...)))``. Such matchers are sometimes
-     written by author who were unaware of the existence of the
-     ``ignoring*()`` matchers.
-
+- ...
 
 clang-format
 ------------
diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst
index d13d4180a1128..8615a77596b4f 100644
--- a/clang/docs/UsersManual.rst
+++ b/clang/docs/UsersManual.rst
@@ -1220,7 +1220,7 @@ are listed below.
 
    * ``-fno-math-errno``
 
-   * ``-ffinite-math``
+   * ``-ffinite-math-only``
 
    * ``-fassociative-math``
 
@@ -1306,14 +1306,14 @@ are listed below.
 **-f[no-]honor-infinities**
 
    If both ``-fno-honor-infinities`` and ``-fno-honor-nans`` are used,
-   has the same effect as specifying ``-ffinite-math``.
+   has the same effect as specifying ``-ffinite-math-only``.
 
 .. _opt_fhonor-nans:
 
 **-f[no-]honor-nans**
 
    If both ``-fno-honor-infinities`` and ``-fno-honor-nans`` are used,
-   has the same effect as specifying ``-ffinite-math``.
+   has the same effect as specifying ``-ffinite-math-only``.
 
 .. _opt_fsigned-zeros:
 
@@ -1351,9 +1351,9 @@ are listed below.
 
    Defaults to ``-fno-unsafe-math-optimizations``.
 
-.. _opt_ffinite-math:
+.. _opt_ffinite-math-only:
 
-**-f[no-]finite-math**
+**-f[no-]finite-math-only**
 
    Allow floating-point optimizations that assume arguments and results are
    not NaNs or +-Inf.  This defines the ``__FINITE_MATH_ONLY__`` preprocessor macro.
@@ -1362,7 +1362,7 @@ are listed below.
    * ``-fno-honor-infinities``
    * ``-fno-honor-nans``
 
-   Defaults to ``-fno-finite-math``.
+   Defaults to ``-fno-finite-math-only``.
 
 .. _opt_frounding-math:
 
@@ -3132,7 +3132,7 @@ Global objects must be constructed before the first kernel using the global obje
 is executed and destroyed just after the last kernel using the program objects is
 executed. In OpenCL v2.0 drivers there is no specific API for invoking global
 constructors. However, an easy workaround would be to enqueue a constructor
-initialization kernel that has a name ``@_GLOBAL__sub_I_<compiled file name>``.
+initialization kernel that has a name ``_GLOBAL__sub_I_<compiled file name>``.
 This kernel is only present if there are any global objects to be initialized in
 the compiled binary. One way to check this is by passing ``CL_PROGRAM_KERNEL_NAMES``
 to ``clGetProgramInfo`` (OpenCL v2.0 s5.8.7).
@@ -3148,7 +3148,7 @@ before running any kernels in which the objects are used.
      clang -cl-std=clc++ test.cl
 
 If there are any global objects to be initialized, the final binary will contain
-the ``@_GLOBAL__sub_I_test.cl`` kernel to be enqueued.
+the ``_GLOBAL__sub_I_test.cl`` kernel to be enqueued.
 
 Global destructors can not be invoked in OpenCL v2.0 drivers. However, all memory used
 for program scope objects is released on ``clReleaseProgram``.
diff --git a/clang/docs/analyzer/checkers.rst b/clang/docs/analyzer/checkers.rst
index d14bd2d68af9a..1583da7aff098 100644
--- a/clang/docs/analyzer/checkers.rst
+++ b/clang/docs/analyzer/checkers.rst
@@ -1836,6 +1836,19 @@ Check unreachable code.
    [x retain]; // warn
  }
 
+.. _alpha-cplusplus-SmartPtr:
+
+alpha.cplusplus.SmartPtr (C++)
+""""""""""""""""""""""""""""""
+Check for dereference of null smart pointers.
+
+.. code-block:: cpp
+
+ void deref_smart_ptr() {
+   std::unique_ptr<int> P;
+   *P; // warn: dereference of a default constructed smart unique_ptr
+ }
+
 alpha.llvm
 ^^^^^^^^^^
 
diff --git a/clang/docs/tools/dump_ast_matchers.py b/clang/docs/tools/dump_ast_matchers.py
index baba4cf7a232c..045833be76739 100755
--- a/clang/docs/tools/dump_ast_matchers.py
+++ b/clang/docs/tools/dump_ast_matchers.py
@@ -230,6 +230,28 @@ def act_on_decl(declaration, comment, allowed_types):
         add_matcher(result_type, name, args, comment)
       return
 
+    m = re.match(r"""^\s*AST_POLYMORPHIC_MATCHER_REGEX(?:_OVERLOAD)?\(
+                          \s*([^\s,]+)\s*,
+                          \s*AST_POLYMORPHIC_SUPPORTED_TYPES\(([^)]*)\),
+                          \s*([^\s,]+)\s*
+                       (?:,\s*\d+\s*)?
+                      \)\s*{\s*$""", declaration, flags=re.X)
+
+    if m:
+      name, results, arg_name = m.groups()[0:3]
+      result_types = [r.strip() for r in results.split(',')]
+      if allowed_types and allowed_types != result_types:
+        raise Exception('Inconsistent documentation for: %s' % name)
+      arg = "StringRef %s, Regex::RegexFlags Flags = NoFlags" % arg_name
+      comment += """
+If the matcher is used in clang-query, RegexFlags parameter
+should be passed as a quoted string. e.g: "NoFlags".
+Flags can be combined with '|' example \"IgnoreCase | BasicRegex\"
+"""
+      for result_type in result_types:
+        add_matcher(result_type, name, arg, comment)
+      return
+
     m = re.match(r"""^\s*AST_MATCHER_FUNCTION(_P)?(.?)(?:_OVERLOAD)?\(
                        (?:\s*([^\s,]+)\s*,)?
                           \s*([^\s,]+)\s*
@@ -275,6 +297,31 @@ def act_on_decl(declaration, comment, allowed_types):
         add_matcher(result_type, name, args, comment)
       return
 
+    m = re.match(r"""^\s*AST_MATCHER_REGEX(?:_OVERLOAD)?\(
+                       \s*([^\s,]+)\s*,
+                       \s*([^\s,]+)\s*,
+                       \s*([^\s,]+)\s*
+                       (?:,\s*\d+\s*)?
+                      \)\s*{""", declaration, flags=re.X)
+    if m:
+      result, name, arg_name = m.groups()[0:3]
+      if not result:
+        if not allowed_types:
+          raise Exception('Did not find allowed result types for: %s' % name)
+        result_types = allowed_types
+      else:
+        result_types = [result]
+      arg = "StringRef %s, Regex::RegexFlags Flags = NoFlags" % arg_name
+      comment += """
+If the matcher is used in clang-query, RegexFlags parameter
+should be passed as a quoted string. e.g: "NoFlags".
+Flags can be combined with '|' example \"IgnoreCase | BasicRegex\"
+"""
+
+      for result_type in result_types:
+        add_matcher(result_type, name, arg, comment)
+      return
+
     # Parse ArgumentAdapting matchers.
     m = re.match(
         r"""^.*ArgumentAdaptingMatcherFunc<.*>\s*
diff --git a/clang/docs/tools/generate_formatted_state.py b/clang/docs/tools/generate_formatted_state.py
index 1b620a84ac0b4..41d5cd9d40043 100755
--- a/clang/docs/tools/generate_formatted_state.py
+++ b/clang/docs/tools/generate_formatted_state.py
@@ -72,6 +72,8 @@ def get_style(count, passed):
      - {style2}`{percent}%`
 """
 
+FNULL = open(os.devnull, 'w')
+
 with open(DOC_FILE, 'wb') as output:
     sha = get_git_revision_short_hash()
     today = datetime.now().strftime("%B %d, %Y %H:%M:%S")
@@ -85,14 +87,22 @@ def get_style(count, passed):
         for subdir in subdirs:
             if any(sd == subdir for sd in skipped_dirs):
                 subdirs.remove(subdir)
+            else:
+                act_sub_dir = os.path.join(root, subdir)
+                # Check the git index to see if the directory contains tracked
+                # files. Reditect the output to a null descriptor as we aren't
+                # interested in it, just the return code.
+                git_check = subprocess.Popen(
+                    ["git", "ls-files", "--error-unmatch", act_sub_dir],
+                    stdout=FNULL,
+                    stderr=FNULL)
+                if git_check.wait() != 0:
+                    print("Skipping directory: ", act_sub_dir)
+                    subdirs.remove(subdir)
 
         path = os.path.relpath(root, TOP_DIR)
         path = path.replace('\\', '/')
 
-        head, _ = os.path.split(root)
-        while head:
-            head, _ = os.path.split(head)
-
         file_count = 0
         file_pass = 0
         file_fail = 0
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index 19471af4a675c..5fa728d6d66c8 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2167,7 +2167,7 @@ enum CXCursorKind {
    */
   CXCursor_ObjCSelfExpr = 147,
 
-  /** OpenMP 4.0 [2.4, Array Section].
+  /** OpenMP 5.0 [2.1.5, Array Section].
    */
   CXCursor_OMPArraySectionExpr = 148,
 
@@ -5934,6 +5934,7 @@ typedef void *CXEvalResult;
  * If cursor is a statement declaration tries to evaluate the
  * statement and if its variable, tries to evaluate its initializer,
  * into its corresponding type.
+ * If it's an expression, tries to evaluate the expression.
  */
 CINDEX_LINKAGE CXEvalResult clang_Cursor_Evaluate(CXCursor C);
 
diff --git a/clang/include/clang/AST/APValue.h b/clang/include/clang/AST/APValue.h
index 63359294ef632..cca92b5f82352 100644
--- a/clang/include/clang/AST/APValue.h
+++ b/clang/include/clang/AST/APValue.h
@@ -372,7 +372,7 @@ class APValue {
   bool isAddrLabelDiff() const { return Kind == AddrLabelDiff; }
 
   void dump() const;
-  void dump(raw_ostream &OS) const;
+  void dump(raw_ostream &OS, const ASTContext &Context) const;
 
   void printPretty(raw_ostream &OS, const ASTContext &Ctx, QualType Ty) const;
   std::string getAsString(const ASTContext &Ctx, QualType Ty) const;
diff --git a/clang/include/clang/AST/ASTDumper.h b/clang/include/clang/AST/ASTDumper.h
index f46ffb960db6a..a154bc2db3a7e 100644
--- a/clang/include/clang/AST/ASTDumper.h
+++ b/clang/include/clang/AST/ASTDumper.h
@@ -24,18 +24,11 @@ class ASTDumper : public ASTNodeTraverser<ASTDumper, TextNodeDumper> {
   const bool ShowColors;
 
 public:
-  ASTDumper(raw_ostream &OS, const comments::CommandTraits *Traits,
-            const SourceManager *SM)
-      : ASTDumper(OS, Traits, SM, SM && SM->getDiagnostics().getShowColors()) {}
-
-  ASTDumper(raw_ostream &OS, const comments::CommandTraits *Traits,
-            const SourceManager *SM, bool ShowColors)
-      : ASTDumper(OS, Traits, SM, ShowColors, LangOptions()) {}
-  ASTDumper(raw_ostream &OS, const comments::CommandTraits *Traits,
-            const SourceManager *SM, bool ShowColors,
-            const PrintingPolicy &PrintPolicy)
-      : NodeDumper(OS, ShowColors, SM, PrintPolicy, Traits), OS(OS),
-        ShowColors(ShowColors) {}
+  ASTDumper(raw_ostream &OS, const ASTContext &Context, bool ShowColors)
+      : NodeDumper(OS, Context, ShowColors), OS(OS), ShowColors(ShowColors) {}
+
+  ASTDumper(raw_ostream &OS, bool ShowColors)
+      : NodeDumper(OS, ShowColors), OS(OS), ShowColors(ShowColors) {}
 
   TextNodeDumper &doGetNodeDelegate() { return NodeDumper; }
 
diff --git a/clang/include/clang/AST/ASTNodeTraverser.h b/clang/include/clang/AST/ASTNodeTraverser.h
index c146c24a3d36f..26656b7162b61 100644
--- a/clang/include/clang/AST/ASTNodeTraverser.h
+++ b/clang/include/clang/AST/ASTNodeTraverser.h
@@ -22,10 +22,13 @@
 #include "clang/AST/LocInfoType.h"
 #include "clang/AST/StmtVisitor.h"
 #include "clang/AST/TemplateArgumentVisitor.h"
+#include "clang/AST/Type.h"
 #include "clang/AST/TypeVisitor.h"
 
 namespace clang {
 
+class APValue;
+
 /**
 
 ASTNodeTraverser traverses the Clang AST for dumping purposes.
@@ -50,6 +53,7 @@ struct {
   void Visit(const OMPClause *C);
   void Visit(const BlockDecl::Capture &C);
   void Visit(const GenericSelectionExpr::ConstAssociation &A);
+  void Visit(const APValue &Value, QualType Ty);
 };
 */
 template <typename Derived, typename NodeDelegateType>
@@ -211,6 +215,10 @@ class ASTNodeTraverser
     });
   }
 
+  void Visit(const APValue &Value, QualType Ty) {
+    getNodeDelegate().AddChild([=] { getNodeDelegate().Visit(Value, Ty); });
+  }
+
   void Visit(const comments::Comment *C, const comments::FullComment *FC) {
     getNodeDelegate().AddChild([=] {
       getNodeDelegate().Visit(C, FC);
@@ -352,8 +360,6 @@ class ASTNodeTraverser
   void VisitTemplateSpecializationType(const TemplateSpecializationType *T) {
     for (const auto &Arg : *T)
       Visit(Arg);
-    if (T->isTypeAlias())
-      Visit(T->getAliasedType());
   }
   void VisitObjCObjectPointerType(const ObjCObjectPointerType *T) {
     Visit(T->getPointeeType());
@@ -682,6 +688,15 @@ class ASTNodeTraverser
         Visit(A);
   }
 
+  void VisitSubstNonTypeTemplateParmExpr(const SubstNonTypeTemplateParmExpr *E) {
+    Visit(E->getParameter());
+  }
+  void VisitSubstNonTypeTemplateParmPackExpr(
+      const SubstNonTypeTemplateParmPackExpr *E) {
+    Visit(E->getParameterPack());
+    Visit(E->getArgumentPack());
+  }
+
   void VisitObjCAtCatchStmt(const ObjCAtCatchStmt *Node) {
     if (const VarDecl *CatchParam = Node->getCatchParamDecl())
       Visit(CatchParam);
@@ -690,6 +705,11 @@ class ASTNodeTraverser
   void VisitExpressionTemplateArgument(const TemplateArgument &TA) {
     Visit(TA.getAsExpr());
   }
+
+  void VisitTypeTemplateArgument(const TemplateArgument &TA) {
+    Visit(TA.getAsType());
+  }
+
   void VisitPackTemplateArgument(const TemplateArgument &TA) {
     for (const auto &TArg : TA.pack_elements())
       Visit(TArg);
diff --git a/clang/include/clang/AST/ASTTypeTraits.h b/clang/include/clang/AST/ASTTypeTraits.h
index 67fa4ab1b6a4f..328b7bce6ba50 100644
--- a/clang/include/clang/AST/ASTTypeTraits.h
+++ b/clang/include/clang/AST/ASTTypeTraits.h
@@ -278,7 +278,7 @@ class DynTypedNode {
   void print(llvm::raw_ostream &OS, const PrintingPolicy &PP) const;
 
   /// Dumps the node to the given output stream.
-  void dump(llvm::raw_ostream &OS, SourceManager &SM) const;
+  void dump(llvm::raw_ostream &OS, const ASTContext &Context) const;
 
   /// For nodes which represent textual entities in the source code,
   /// return their SourceRange.  For all other nodes, return SourceRange().
diff --git a/clang/include/clang/AST/Comment.h b/clang/include/clang/AST/Comment.h
index cd9c1ce2bce0c..54a4b0a9cfe66 100644
--- a/clang/include/clang/AST/Comment.h
+++ b/clang/include/clang/AST/Comment.h
@@ -209,9 +209,7 @@ class Comment {
 
   void dump() const;
   void dumpColor() const;
-  void dump(const ASTContext &Context) const;
-  void dump(raw_ostream &OS, const CommandTraits *Traits,
-            const SourceManager *SM) const;
+  void dump(raw_ostream &OS, const ASTContext &Context) const;
 
   SourceRange getSourceRange() const LLVM_READONLY { return Range; }
 
diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h
index 6c39f6aab1b9c..28faa2c1fc780 100644
--- a/clang/include/clang/AST/Decl.h
+++ b/clang/include/clang/AST/Decl.h
@@ -2618,7 +2618,13 @@ class FunctionDecl : public DeclaratorDecl,
   /// Retrieve the function declaration from which this function could
   /// be instantiated, if it is an instantiation (rather than a non-template
   /// or a specialization, for example).
-  FunctionDecl *getTemplateInstantiationPattern() const;
+  ///
+  /// If \p ForDefinition is \c false, explicit specializations will be treated
+  /// as if they were implicit instantiations. This will then find the pattern
+  /// corresponding to non-definition portions of the declaration, such as
+  /// default arguments and the exception specification.
+  FunctionDecl *
+  getTemplateInstantiationPattern(bool ForDefinition = true) const;
 
   /// Retrieve the primary template that this function template
   /// specialization either specializes or was instantiated from.
diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h
index ba84f27617f5f..4f33ff104ffd6 100644
--- a/clang/include/clang/AST/DeclBase.h
+++ b/clang/include/clang/AST/DeclBase.h
@@ -66,6 +66,7 @@ class SourceManager;
 class Stmt;
 class StoredDeclsMap;
 class TemplateDecl;
+class TemplateParameterList;
 class TranslationUnitDecl;
 class UsingDirectiveDecl;
 
@@ -862,6 +863,10 @@ class alignas(8) Decl {
   // within the scope of a template parameter).
   bool isTemplated() const;
 
+  /// Determine the number of levels of template parameter surrounding this
+  /// declaration.
+  unsigned getTemplateDepth() const;
+
   /// isDefinedOutsideFunctionOrMethod - This predicate returns true if this
   /// scoped decl is defined outside the current function or method.  This is
   /// roughly global variables and functions, but also handles enums (which
@@ -870,15 +875,19 @@ class alignas(8) Decl {
     return getParentFunctionOrMethod() == nullptr;
   }
 
-  /// Returns true if this declaration is lexically inside a function or inside
-  /// a variable initializer. It recognizes non-defining declarations as well
-  /// as members of local classes and lambdas:
+  /// Determine whether a substitution into this declaration would occur as
+  /// part of a substitution into a dependent local scope. Such a substitution
+  /// transitively substitutes into all constructs nested within this
+  /// declaration.
+  ///
+  /// This recognizes non-defining declarations as well as members of local
+  /// classes and lambdas:
   /// \code
-  ///     void foo() { void bar(); }
-  ///     void foo2() { class ABC { void bar(); }; }
-  ///     inline int x = [](){ return 0; }();
+  ///     template<typename T> void foo() { void bar(); }
+  ///     template<typename T> void foo2() { class ABC { void bar(); }; }
+  ///     template<typename T> inline int x = [](){ return 0; }();
   /// \endcode
-  bool isInLocalScope() const;
+  bool isInLocalScopeForInstantiation() const;
 
   /// If this decl is defined inside a function/method/block it returns
   /// the corresponding DeclContext, otherwise it returns null.
@@ -1038,8 +1047,16 @@ class alignas(8) Decl {
 
   /// If this is a declaration that describes some template, this
   /// method returns that template declaration.
+  ///
+  /// Note that this returns nullptr for partial specializations, because they
+  /// are not modeled as TemplateDecls. Use getDescribedTemplateParams to handle
+  /// those cases.
   TemplateDecl *getDescribedTemplate() const;
 
+  /// If this is a declaration that describes some template or partial
+  /// specialization, this returns the corresponding template parameter list.
+  const TemplateParameterList *getDescribedTemplateParams() const;
+
   /// Returns the function itself, or the templated function if this is a
   /// function template.
   FunctionDecl *getAsFunction() LLVM_READONLY;
diff --git a/clang/include/clang/AST/DependenceFlags.h b/clang/include/clang/AST/DependenceFlags.h
index 0b24bae6df9b1..3601cb90bb765 100644
--- a/clang/include/clang/AST/DependenceFlags.h
+++ b/clang/include/clang/AST/DependenceFlags.h
@@ -70,12 +70,13 @@ using TypeDependence = TypeDependenceScope::TypeDependence;
       UnexpandedPack = 1,                                                      \
       Instantiation = 2,                                                       \
       Dependent = 4,                                                           \
+      Error = 8,                                                               \
                                                                                \
       None = 0,                                                                \
       DependentInstantiation = Dependent | Instantiation,                      \
-      All = 7,                                                                 \
+      All = 15,                                                                \
                                                                                \
-      LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/Dependent)                    \
+      LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/Error)                        \
     };                                                                         \
   };                                                                           \
   using NAME = NAME##Scope::NAME;
@@ -118,10 +119,10 @@ class Dependence {
 
   Dependence(TypeDependence D)
       : V(translate(D, TypeDependence::UnexpandedPack, UnexpandedPack) |
-             translate(D, TypeDependence::Instantiation, Instantiation) |
-             translate(D, TypeDependence::Dependent, Dependent) |
-             translate(D, TypeDependence::VariablyModified, VariablyModified)) {
-  }
+          translate(D, TypeDependence::Instantiation, Instantiation) |
+          translate(D, TypeDependence::Dependent, Dependent) |
+          translate(D, TypeDependence::Error, Error) |
+          translate(D, TypeDependence::VariablyModified, VariablyModified)) {}
 
   Dependence(ExprDependence D)
       : V(translate(D, ExprDependence::UnexpandedPack, UnexpandedPack) |
@@ -133,17 +134,20 @@ class Dependence {
   Dependence(NestedNameSpecifierDependence D) :
     V ( translate(D, NNSDependence::UnexpandedPack, UnexpandedPack) |
             translate(D, NNSDependence::Instantiation, Instantiation) |
-            translate(D, NNSDependence::Dependent, Dependent)){}
+            translate(D, NNSDependence::Dependent, Dependent) |
+            translate(D, NNSDependence::Error, Error)) {}
 
   Dependence(TemplateArgumentDependence D)
       : V(translate(D, TADependence::UnexpandedPack, UnexpandedPack) |
-             translate(D, TADependence::Instantiation, Instantiation) |
-             translate(D, TADependence::Dependent, Dependent)) {}
+          translate(D, TADependence::Instantiation, Instantiation) |
+          translate(D, TADependence::Dependent, Dependent) |
+          translate(D, TADependence::Error, Error)) {}
 
   Dependence(TemplateNameDependence D)
       : V(translate(D, TNDependence::UnexpandedPack, UnexpandedPack) |
              translate(D, TNDependence::Instantiation, Instantiation) |
-             translate(D, TNDependence::Dependent, Dependent)) {}
+             translate(D, TNDependence::Dependent, Dependent) |
+             translate(D, TNDependence::Error, Error)) {}
 
   TypeDependence type() const {
     return translate(V, UnexpandedPack, TypeDependence::UnexpandedPack) |
@@ -164,19 +168,22 @@ class Dependence {
   NestedNameSpecifierDependence nestedNameSpecifier() const {
     return translate(V, UnexpandedPack, NNSDependence::UnexpandedPack) |
            translate(V, Instantiation, NNSDependence::Instantiation) |
-           translate(V, Dependent, NNSDependence::Dependent);
+           translate(V, Dependent, NNSDependence::Dependent) |
+           translate(V, Error, NNSDependence::Error);
   }
 
   TemplateArgumentDependence templateArgument() const {
     return translate(V, UnexpandedPack, TADependence::UnexpandedPack) |
            translate(V, Instantiation, TADependence::Instantiation) |
-           translate(V, Dependent, TADependence::Dependent);
+           translate(V, Dependent, TADependence::Dependent) |
+           translate(V, Error, TADependence::Error);
   }
 
   TemplateNameDependence templateName() const {
     return translate(V, UnexpandedPack, TNDependence::UnexpandedPack) |
            translate(V, Instantiation, TNDependence::Instantiation) |
-           translate(V, Dependent, TNDependence::Dependent);
+           translate(V, Dependent, TNDependence::Dependent) |
+           translate(V, Error, TNDependence::Error);
   }
 
 private:
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index 8ee8aa96ae638..66eafaaab715e 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -2118,21 +2118,21 @@ class ParenExpr : public Expr {
 ///
 class UnaryOperator final
     : public Expr,
-      private llvm::TrailingObjects<UnaryOperator, FPOptions> {
+      private llvm::TrailingObjects<UnaryOperator, FPOptionsOverride> {
   Stmt *Val;
 
-  size_t numTrailingObjects(OverloadToken<FPOptions>) const {
+  size_t numTrailingObjects(OverloadToken<FPOptionsOverride>) const {
     return UnaryOperatorBits.HasFPFeatures ? 1 : 0;
   }
 
-  FPOptions &getTrailingFPFeatures() {
+  FPOptionsOverride &getTrailingFPFeatures() {
     assert(UnaryOperatorBits.HasFPFeatures);
-    return *getTrailingObjects<FPOptions>();
+    return *getTrailingObjects<FPOptionsOverride>();
   }
 
-  const FPOptions &getTrailingFPFeatures() const {
+  const FPOptionsOverride &getTrailingFPFeatures() const {
     assert(UnaryOperatorBits.HasFPFeatures);
-    return *getTrailingObjects<FPOptions>();
+    return *getTrailingObjects<FPOptionsOverride>();
   }
 
 public:
@@ -2141,7 +2141,7 @@ class UnaryOperator final
 protected:
   UnaryOperator(const ASTContext &Ctx, Expr *input, Opcode opc, QualType type,
                 ExprValueKind VK, ExprObjectKind OK, SourceLocation l,
-                bool CanOverflow, FPOptions FPFeatures);
+                bool CanOverflow, FPOptionsOverride FPFeatures);
 
   /// Build an empty unary operator.
   explicit UnaryOperator(bool HasFPFeatures, EmptyShell Empty)
@@ -2156,7 +2156,7 @@ class UnaryOperator final
   static UnaryOperator *Create(const ASTContext &C, Expr *input, Opcode opc,
                                QualType type, ExprValueKind VK,
                                ExprObjectKind OK, SourceLocation l,
-                               bool CanOverflow, FPOptions FPFeatures);
+                               bool CanOverflow, FPOptionsOverride FPFeatures);
 
   Opcode getOpcode() const {
     return static_cast<Opcode>(UnaryOperatorBits.Opc);
@@ -2182,13 +2182,13 @@ class UnaryOperator final
   // Get the FP contractability status of this operator. Only meaningful for
   // operations on floating point types.
   bool isFPContractableWithinStatement(const LangOptions &LO) const {
-    return getFPFeatures(LO).allowFPContractWithinStatement();
+    return getFPFeaturesInEffect(LO).allowFPContractWithinStatement();
   }
 
   // Get the FENV_ACCESS status of this operator. Only meaningful for
   // operations on floating point types.
   bool isFEnvAccessOn(const LangOptions &LO) const {
-    return getFPFeatures(LO).allowFEnvAccess();
+    return getFPFeaturesInEffect(LO).getAllowFEnvAccess();
   }
 
   /// isPostfix - Return true if this is a postfix operation, like x++.
@@ -2263,19 +2263,26 @@ class UnaryOperator final
 
 protected:
   /// Get FPFeatures from trailing storage
-  FPOptions getStoredFPFeatures() const { return getTrailingFPFeatures(); }
+  FPOptionsOverride getStoredFPFeatures() const {
+    return getTrailingFPFeatures();
+  }
 
   /// Set FPFeatures in trailing storage, used only by Serialization
-  void setStoredFPFeatures(FPOptions F) { getTrailingFPFeatures() = F; }
+  void setStoredFPFeatures(FPOptionsOverride F) { getTrailingFPFeatures() = F; }
 
 public:
   // Get the FP features status of this operator. Only meaningful for
   // operations on floating point types.
-  FPOptions getFPFeatures(const LangOptions &LO) const {
+  FPOptions getFPFeaturesInEffect(const LangOptions &LO) const {
     if (UnaryOperatorBits.HasFPFeatures)
-      return getStoredFPFeatures();
+      return getStoredFPFeatures().applyOverrides(LO);
     return FPOptions::defaultWithoutTrailingStorage(LO);
   }
+  FPOptionsOverride getFPOptionsOverride() const {
+    if (UnaryOperatorBits.HasFPFeatures)
+      return getStoredFPFeatures();
+    return FPOptionsOverride();
+  }
 
   friend TrailingObjects;
   friend class ASTReader;
@@ -3633,14 +3640,14 @@ class BinaryOperator : public Expr {
   size_t offsetOfTrailingStorage() const;
 
   /// Return a pointer to the trailing FPOptions
-  FPOptions *getTrailingFPFeatures() {
+  FPOptionsOverride *getTrailingFPFeatures() {
     assert(BinaryOperatorBits.HasFPFeatures);
-    return reinterpret_cast<FPOptions *>(reinterpret_cast<char *>(this) +
-                                         offsetOfTrailingStorage());
+    return reinterpret_cast<FPOptionsOverride *>(
+        reinterpret_cast<char *>(this) + offsetOfTrailingStorage());
   }
-  const FPOptions *getTrailingFPFeatures() const {
+  const FPOptionsOverride *getTrailingFPFeatures() const {
     assert(BinaryOperatorBits.HasFPFeatures);
-    return reinterpret_cast<const FPOptions *>(
+    return reinterpret_cast<const FPOptionsOverride *>(
         reinterpret_cast<const char *>(this) + offsetOfTrailingStorage());
   }
 
@@ -3648,7 +3655,7 @@ class BinaryOperator : public Expr {
   /// allocated for the trailing objects when needed.
   BinaryOperator(const ASTContext &Ctx, Expr *lhs, Expr *rhs, Opcode opc,
                  QualType ResTy, ExprValueKind VK, ExprObjectKind OK,
-                 SourceLocation opLoc, FPOptions FPFeatures);
+                 SourceLocation opLoc, FPOptionsOverride FPFeatures);
 
   /// Construct an empty binary operator.
   explicit BinaryOperator(EmptyShell Empty) : Expr(BinaryOperatorClass, Empty) {
@@ -3661,7 +3668,7 @@ class BinaryOperator : public Expr {
   static BinaryOperator *Create(const ASTContext &C, Expr *lhs, Expr *rhs,
                                 Opcode opc, QualType ResTy, ExprValueKind VK,
                                 ExprObjectKind OK, SourceLocation opLoc,
-                                FPOptions FPFeatures);
+                                FPOptionsOverride FPFeatures);
   SourceLocation getExprLoc() const { return getOperatorLoc(); }
   SourceLocation getOperatorLoc() const { return BinaryOperatorBits.OpLoc; }
   void setOperatorLoc(SourceLocation L) { BinaryOperatorBits.OpLoc = L; }
@@ -3808,40 +3815,48 @@ class BinaryOperator : public Expr {
   bool hasStoredFPFeatures() const { return BinaryOperatorBits.HasFPFeatures; }
 
   /// Get FPFeatures from trailing storage
-  FPOptions getStoredFPFeatures() const {
+  FPOptionsOverride getStoredFPFeatures() const {
     assert(hasStoredFPFeatures());
     return *getTrailingFPFeatures();
   }
   /// Set FPFeatures in trailing storage, used only by Serialization
-  void setStoredFPFeatures(FPOptions F) {
+  void setStoredFPFeatures(FPOptionsOverride F) {
     assert(BinaryOperatorBits.HasFPFeatures);
     *getTrailingFPFeatures() = F;
   }
 
   // Get the FP features status of this operator. Only meaningful for
   // operations on floating point types.
-  FPOptions getFPFeatures(const LangOptions &LO) const {
+  FPOptions getFPFeaturesInEffect(const LangOptions &LO) const {
     if (BinaryOperatorBits.HasFPFeatures)
-      return getStoredFPFeatures();
+      return getStoredFPFeatures().applyOverrides(LO);
     return FPOptions::defaultWithoutTrailingStorage(LO);
   }
 
+  // This is used in ASTImporter
+  FPOptionsOverride getFPFeatures(const LangOptions &LO) const {
+    if (BinaryOperatorBits.HasFPFeatures)
+      return getStoredFPFeatures();
+    return FPOptionsOverride();
+  }
+
   // Get the FP contractability status of this operator. Only meaningful for
   // operations on floating point types.
   bool isFPContractableWithinStatement(const LangOptions &LO) const {
-    return getFPFeatures(LO).allowFPContractWithinStatement();
+    return getFPFeaturesInEffect(LO).allowFPContractWithinStatement();
   }
 
   // Get the FENV_ACCESS status of this operator. Only meaningful for
   // operations on floating point types.
   bool isFEnvAccessOn(const LangOptions &LO) const {
-    return getFPFeatures(LO).allowFEnvAccess();
+    return getFPFeaturesInEffect(LO).getAllowFEnvAccess();
   }
 
 protected:
   BinaryOperator(const ASTContext &Ctx, Expr *lhs, Expr *rhs, Opcode opc,
                  QualType ResTy, ExprValueKind VK, ExprObjectKind OK,
-                 SourceLocation opLoc, FPOptions FPFeatures, bool dead2);
+                 SourceLocation opLoc, FPOptionsOverride FPFeatures,
+                 bool dead2);
 
   /// Construct an empty BinaryOperator, SC is CompoundAssignOperator.
   BinaryOperator(StmtClass SC, EmptyShell Empty) : Expr(SC, Empty) {
@@ -3851,7 +3866,7 @@ class BinaryOperator : public Expr {
   /// Return the size in bytes needed for the trailing objects.
   /// Used to allocate the right amount of storage.
   static unsigned sizeOfTrailingObjects(bool HasFPFeatures) {
-    return HasFPFeatures * sizeof(FPOptions);
+    return HasFPFeatures * sizeof(FPOptionsOverride);
   }
 };
 
@@ -3873,7 +3888,7 @@ class CompoundAssignOperator : public BinaryOperator {
 protected:
   CompoundAssignOperator(const ASTContext &C, Expr *lhs, Expr *rhs, Opcode opc,
                          QualType ResType, ExprValueKind VK, ExprObjectKind OK,
-                         SourceLocation OpLoc, FPOptions FPFeatures,
+                         SourceLocation OpLoc, FPOptionsOverride FPFeatures,
                          QualType CompLHSType, QualType CompResultType)
       : BinaryOperator(C, lhs, rhs, opc, ResType, VK, OK, OpLoc, FPFeatures,
                        true),
@@ -3889,7 +3904,7 @@ class CompoundAssignOperator : public BinaryOperator {
   static CompoundAssignOperator *
   Create(const ASTContext &C, Expr *lhs, Expr *rhs, Opcode opc, QualType ResTy,
          ExprValueKind VK, ExprObjectKind OK, SourceLocation opLoc,
-         FPOptions FPFeatures, QualType CompLHSType = QualType(),
+         FPOptionsOverride FPFeatures, QualType CompLHSType = QualType(),
          QualType CompResultType = QualType());
 
   // The two computation types are the type the LHS is converted
diff --git a/clang/include/clang/AST/ExprCXX.h b/clang/include/clang/AST/ExprCXX.h
index 4c65daaa63fc3..178f4db770618 100644
--- a/clang/include/clang/AST/ExprCXX.h
+++ b/clang/include/clang/AST/ExprCXX.h
@@ -84,6 +84,7 @@ class CXXOperatorCallExpr final : public CallExpr {
   friend class ASTStmtWriter;
 
   SourceRange Range;
+  FPOptionsOverride Overrides;
 
   // CXXOperatorCallExpr has some trailing objects belonging
   // to CallExpr. See CallExpr for the details.
@@ -92,7 +93,7 @@ class CXXOperatorCallExpr final : public CallExpr {
 
   CXXOperatorCallExpr(OverloadedOperatorKind OpKind, Expr *Fn,
                       ArrayRef<Expr *> Args, QualType Ty, ExprValueKind VK,
-                      SourceLocation OperatorLoc, FPOptions FPFeatures,
+                      SourceLocation OperatorLoc, FPOptionsOverride FPFeatures,
                       ADLCallKind UsesADL);
 
   CXXOperatorCallExpr(unsigned NumArgs, EmptyShell Empty);
@@ -101,7 +102,7 @@ class CXXOperatorCallExpr final : public CallExpr {
   static CXXOperatorCallExpr *
   Create(const ASTContext &Ctx, OverloadedOperatorKind OpKind, Expr *Fn,
          ArrayRef<Expr *> Args, QualType Ty, ExprValueKind VK,
-         SourceLocation OperatorLoc, FPOptions FPFeatures,
+         SourceLocation OperatorLoc, FPOptionsOverride FPFeatures,
          ADLCallKind UsesADL = NotADL);
 
   static CXXOperatorCallExpr *CreateEmpty(const ASTContext &Ctx,
@@ -164,20 +165,10 @@ class CXXOperatorCallExpr final : public CallExpr {
     return T->getStmtClass() == CXXOperatorCallExprClass;
   }
 
-  // Set the FP contractability status of this operator. Only meaningful for
+  // Set the FPFeatures status of this operator. Only meaningful for
   // operations on floating point types.
-  void setFPFeatures(FPOptions F) {
-    CXXOperatorCallExprBits.FPFeatures = F.getAsOpaqueInt();
-  }
-  FPOptions getFPFeatures() const {
-    return FPOptions(CXXOperatorCallExprBits.FPFeatures);
-  }
-
-  // Get the FP contractability status of this operator. Only meaningful for
-  // operations on floating point types.
-  bool isFPContractableWithinStatement() const {
-    return getFPFeatures().allowFPContractWithinStatement();
-  }
+  void setFPFeatures(FPOptionsOverride F) { Overrides = F; }
+  FPOptionsOverride getFPFeatures() const { return Overrides; }
 };
 
 /// Represents a call to a member function that
@@ -1861,6 +1852,8 @@ class LambdaExpr final : public Expr,
   Stmt **getStoredStmts() { return getTrailingObjects<Stmt *>(); }
   Stmt *const *getStoredStmts() const { return getTrailingObjects<Stmt *>(); }
 
+  void initBodyIfNeeded() const;
+
 public:
   friend class ASTStmtReader;
   friend class ASTStmtWriter;
@@ -2009,17 +2002,12 @@ class LambdaExpr final : public Expr,
   /// a \p CompoundStmt, but can also be \p CoroutineBodyStmt wrapping
   /// a \p CompoundStmt. Note that unlike functions, lambda-expressions
   /// cannot have a function-try-block.
-  Stmt *getBody() const { return getStoredStmts()[capture_size()]; }
+  Stmt *getBody() const;
 
   /// Retrieve the \p CompoundStmt representing the body of the lambda.
   /// This is a convenience function for callers who do not need
   /// to handle node(s) which may wrap a \p CompoundStmt.
-  const CompoundStmt *getCompoundStmtBody() const {
-    Stmt *Body = getBody();
-    if (const auto *CoroBody = dyn_cast<CoroutineBodyStmt>(Body))
-      return cast<CompoundStmt>(CoroBody->getBody());
-    return cast<CompoundStmt>(Body);
-  }
+  const CompoundStmt *getCompoundStmtBody() const;
   CompoundStmt *getCompoundStmtBody() {
     const auto *ConstThis = this;
     return const_cast<CompoundStmt *>(ConstThis->getCompoundStmtBody());
@@ -2048,15 +2036,9 @@ class LambdaExpr final : public Expr,
 
   SourceLocation getEndLoc() const LLVM_READONLY { return ClosingBrace; }
 
-  child_range children() {
-    // Includes initialization exprs plus body stmt
-    return child_range(getStoredStmts(), getStoredStmts() + capture_size() + 1);
-  }
-
-  const_child_range children() const {
-    return const_child_range(getStoredStmts(),
-                             getStoredStmts() + capture_size() + 1);
-  }
+  /// Includes the captures and the body of the lambda.
+  child_range children();
+  const_child_range children() const;
 };
 
 /// An expression "T()" which creates a value-initialized rvalue of type
diff --git a/clang/include/clang/AST/ExprOpenMP.h b/clang/include/clang/AST/ExprOpenMP.h
index a2b59a6f26962..be5dda9923346 100644
--- a/clang/include/clang/AST/ExprOpenMP.h
+++ b/clang/include/clang/AST/ExprOpenMP.h
@@ -17,46 +17,61 @@
 #include "clang/AST/Expr.h"
 
 namespace clang {
-/// OpenMP 4.0 [2.4, Array Sections].
+/// OpenMP 5.0 [2.1.5, Array Sections].
 /// To specify an array section in an OpenMP construct, array subscript
 /// expressions are extended with the following syntax:
 /// \code
+/// [ lower-bound : length : stride ]
+/// [ lower-bound : length : ]
 /// [ lower-bound : length ]
+/// [ lower-bound : : stride ]
+/// [ lower-bound : : ]
 /// [ lower-bound : ]
+/// [ : length : stride ]
+/// [ : length : ]
 /// [ : length ]
+/// [ : : stride ]
+/// [ : : ]
 /// [ : ]
 /// \endcode
 /// The array section must be a subset of the original array.
 /// Array sections are allowed on multidimensional arrays. Base language array
 /// subscript expressions can be used to specify length-one dimensions of
 /// multidimensional array sections.
-/// The lower-bound and length are integral type expressions. When evaluated
+/// Each of the lower-bound, length, and stride expressions if specified must be
+/// an integral type expressions of the base language. When evaluated
 /// they represent a set of integer values as follows:
 /// \code
-/// { lower-bound, lower-bound + 1, lower-bound + 2,... , lower-bound + length -
-/// 1 }
+/// { lower-bound, lower-bound + stride, lower-bound + 2 * stride,... ,
+/// lower-bound + ((length - 1) * stride) }
 /// \endcode
 /// The lower-bound and length must evaluate to non-negative integers.
+/// The stride must evaluate to a positive integer.
 /// When the size of the array dimension is not known, the length must be
 /// specified explicitly.
-/// When the length is absent, it defaults to the size of the array dimension
-/// minus the lower-bound.
-/// When the lower-bound is absent it defaults to 0.
+/// When the stride is absent it defaults to 1.
+/// When the length is absent it defaults to ⌈(size − lower-bound)/stride⌉,
+/// where size is the size of the array dimension. When the lower-bound is
+/// absent it defaults to 0.
 class OMPArraySectionExpr : public Expr {
-  enum { BASE, LOWER_BOUND, LENGTH, END_EXPR };
+  enum { BASE, LOWER_BOUND, LENGTH, STRIDE, END_EXPR };
   Stmt *SubExprs[END_EXPR];
-  SourceLocation ColonLoc;
+  SourceLocation ColonLocFirst;
+  SourceLocation ColonLocSecond;
   SourceLocation RBracketLoc;
 
 public:
-  OMPArraySectionExpr(Expr *Base, Expr *LowerBound, Expr *Length, QualType Type,
-                      ExprValueKind VK, ExprObjectKind OK,
-                      SourceLocation ColonLoc, SourceLocation RBracketLoc)
-      : Expr(OMPArraySectionExprClass, Type, VK, OK), ColonLoc(ColonLoc),
+  OMPArraySectionExpr(Expr *Base, Expr *LowerBound, Expr *Length, Expr *Stride,
+                      QualType Type, ExprValueKind VK, ExprObjectKind OK,
+                      SourceLocation ColonLocFirst,
+                      SourceLocation ColonLocSecond, SourceLocation RBracketLoc)
+      : Expr(OMPArraySectionExprClass, Type, VK, OK),
+        ColonLocFirst(ColonLocFirst), ColonLocSecond(ColonLocSecond),
         RBracketLoc(RBracketLoc) {
     SubExprs[BASE] = Base;
     SubExprs[LOWER_BOUND] = LowerBound;
     SubExprs[LENGTH] = Length;
+    SubExprs[STRIDE] = Stride;
     setDependence(computeDependence(this));
   }
 
@@ -89,13 +104,22 @@ class OMPArraySectionExpr : public Expr {
   /// Set length of the array section.
   void setLength(Expr *E) { SubExprs[LENGTH] = E; }
 
+  /// Get stride of array section.
+  Expr *getStride() { return cast_or_null<Expr>(SubExprs[STRIDE]); }
+  const Expr *getStride() const { return cast_or_null<Expr>(SubExprs[STRIDE]); }
+  /// Set length of the array section.
+  void setStride(Expr *E) { SubExprs[STRIDE] = E; }
+
   SourceLocation getBeginLoc() const LLVM_READONLY {
     return getBase()->getBeginLoc();
   }
   SourceLocation getEndLoc() const LLVM_READONLY { return RBracketLoc; }
 
-  SourceLocation getColonLoc() const { return ColonLoc; }
-  void setColonLoc(SourceLocation L) { ColonLoc = L; }
+  SourceLocation getColonLocFirst() const { return ColonLocFirst; }
+  void setColonLocFirst(SourceLocation L) { ColonLocFirst = L; }
+
+  SourceLocation getColonLocSecond() const { return ColonLocSecond; }
+  void setColonLocSecond(SourceLocation L) { ColonLocSecond = L; }
 
   SourceLocation getRBracketLoc() const { return RBracketLoc; }
   void setRBracketLoc(SourceLocation L) { RBracketLoc = L; }
diff --git a/clang/include/clang/AST/JSONNodeDumper.h b/clang/include/clang/AST/JSONNodeDumper.h
index ae7b4c9a17ea7..4e7162992418a 100644
--- a/clang/include/clang/AST/JSONNodeDumper.h
+++ b/clang/include/clang/AST/JSONNodeDumper.h
@@ -23,10 +23,13 @@
 #include "clang/AST/CommentVisitor.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/Mangle.h"
+#include "clang/AST/Type.h"
 #include "llvm/Support/JSON.h"
 
 namespace clang {
 
+class APValue;
+
 class NodeStreamer {
   bool FirstChild = true;
   bool TopLevel = true;
@@ -201,6 +204,7 @@ class JSONNodeDumper
   void Visit(const OMPClause *C);
   void Visit(const BlockDecl::Capture &C);
   void Visit(const GenericSelectionExpr::ConstAssociation &A);
+  void Visit(const APValue &Value, QualType Ty);
 
   void VisitTypedefType(const TypedefType *TT);
   void VisitFunctionType(const FunctionType *T);
diff --git a/clang/include/clang/AST/NestedNameSpecifier.h b/clang/include/clang/AST/NestedNameSpecifier.h
index 47f5268c938b5..540ac3df48fe0 100644
--- a/clang/include/clang/AST/NestedNameSpecifier.h
+++ b/clang/include/clang/AST/NestedNameSpecifier.h
@@ -214,6 +214,9 @@ class NestedNameSpecifier : public llvm::FoldingSetNode {
   /// parameter pack (for C++11 variadic templates).
   bool containsUnexpandedParameterPack() const;
 
+  /// Whether this nested name specifier contains an error.
+  bool containsErrors() const;
+
   /// Print this nested name specifier to the given output stream. If
   /// `ResolveTemplateArguments` is true, we'll print actual types, e.g.
   /// `ns::SomeTemplate<int, MyClass>` instead of
diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 6c2ca1cb10088..6de7b6deb5149 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -7556,6 +7556,8 @@ class OMPClauseVisitorBase {
   case llvm::omp::Clause::Enum:                                                \
     break;
 #include "llvm/Frontend/OpenMP/OMPKinds.def"
+    default:
+      break;
     }
   }
   // Base case, ignore it. :)
diff --git a/clang/include/clang/AST/ParentMapContext.h b/clang/include/clang/AST/ParentMapContext.h
index 5f9936b28e8fb..be4d75df7b999 100644
--- a/clang/include/clang/AST/ParentMapContext.h
+++ b/clang/include/clang/AST/ParentMapContext.h
@@ -67,7 +67,7 @@ class ParentMapContext {
 private:
   ASTContext &ASTCtx;
   class ParentMap;
-  TraversalKind Traversal = TK_IgnoreUnlessSpelledInSource;
+  TraversalKind Traversal = TK_AsIs;
   std::unique_ptr<ParentMap> Parents;
 };
 
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 9bbd390f61cde..3dcfc9fee629a 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -48,29 +48,6 @@
 #include <cstddef>
 #include <type_traits>
 
-// The following three macros are used for meta programming.  The code
-// using them is responsible for defining macro OPERATOR().
-
-// All unary operators.
-#define UNARYOP_LIST()                                                         \
-  OPERATOR(PostInc) OPERATOR(PostDec) OPERATOR(PreInc) OPERATOR(PreDec)        \
-      OPERATOR(AddrOf) OPERATOR(Deref) OPERATOR(Plus) OPERATOR(Minus)          \
-      OPERATOR(Not) OPERATOR(LNot) OPERATOR(Real) OPERATOR(Imag)               \
-      OPERATOR(Extension) OPERATOR(Coawait)
-
-// All binary operators (excluding compound assign operators).
-#define BINOP_LIST()                                                           \
-  OPERATOR(PtrMemD) OPERATOR(PtrMemI) OPERATOR(Mul) OPERATOR(Div)              \
-      OPERATOR(Rem) OPERATOR(Add) OPERATOR(Sub) OPERATOR(Shl) OPERATOR(Shr)    \
-      OPERATOR(LT) OPERATOR(GT) OPERATOR(LE) OPERATOR(GE) OPERATOR(EQ)         \
-      OPERATOR(NE) OPERATOR(Cmp) OPERATOR(And) OPERATOR(Xor) OPERATOR(Or)      \
-      OPERATOR(LAnd) OPERATOR(LOr) OPERATOR(Assign) OPERATOR(Comma)
-
-// All compound assign operators.
-#define CAO_LIST()                                                             \
-  OPERATOR(Mul) OPERATOR(Div) OPERATOR(Rem) OPERATOR(Add) OPERATOR(Sub)        \
-      OPERATOR(Shl) OPERATOR(Shr) OPERATOR(And) OPERATOR(Or) OPERATOR(Xor)
-
 namespace clang {
 
 // A helper macro to implement short-circuiting when recursing.  It
@@ -83,6 +60,42 @@ namespace clang {
       return false;                                                            \
   } while (false)
 
+namespace detail {
+
+template <typename T, typename U>
+struct has_same_member_pointer_type : std::false_type {};
+template <typename T, typename U, typename R, typename... P>
+struct has_same_member_pointer_type<R (T::*)(P...), R (U::*)(P...)>
+    : std::true_type {};
+
+template <bool has_same_type> struct is_same_method_impl {
+  template <typename FirstMethodPtrTy, typename SecondMethodPtrTy>
+  static bool isSameMethod(FirstMethodPtrTy FirstMethodPtr,
+                           SecondMethodPtrTy SecondMethodPtr) {
+    return false;
+  }
+};
+
+template <> struct is_same_method_impl<true> {
+  template <typename FirstMethodPtrTy, typename SecondMethodPtrTy>
+  static bool isSameMethod(FirstMethodPtrTy FirstMethodPtr,
+                           SecondMethodPtrTy SecondMethodPtr) {
+    return FirstMethodPtr == SecondMethodPtr;
+  }
+};
+
+/// Returns true if and only if \p FirstMethodPtr and \p SecondMethodPtr
+/// are pointers to the same non-static member function.
+template <typename FirstMethodPtrTy, typename SecondMethodPtrTy>
+bool isSameMethod(FirstMethodPtrTy FirstMethodPtr,
+                  SecondMethodPtrTy SecondMethodPtr) {
+  return is_same_method_impl<has_same_member_pointer_type<
+      FirstMethodPtrTy,
+      SecondMethodPtrTy>::value>::isSameMethod(FirstMethodPtr, SecondMethodPtr);
+}
+
+} // end namespace detail
+
 /// A class that does preorder or postorder
 /// depth-first traversal on the entire Clang AST and visits each node.
 ///
@@ -325,23 +338,17 @@ template <typename Derived> class RecursiveASTVisitor {
   Stmt::child_range getStmtChildren(Stmt *S) { return S->children(); }
 
 private:
-  template<typename T, typename U>
-  struct has_same_member_pointer_type : std::false_type {};
-  template<typename T, typename U, typename R, typename... P>
-  struct has_same_member_pointer_type<R (T::*)(P...), R (U::*)(P...)>
-      : std::true_type {};
-
   // Traverse the given statement. If the most-derived traverse function takes a
   // data recursion queue, pass it on; otherwise, discard it. Note that the
   // first branch of this conditional must compile whether or not the derived
   // class can take a queue, so if we're taking the second arm, make the first
   // arm call our function rather than the derived class version.
 #define TRAVERSE_STMT_BASE(NAME, CLASS, VAR, QUEUE)                            \
-  (has_same_member_pointer_type<decltype(                                      \
-                                    &RecursiveASTVisitor::Traverse##NAME),     \
-                                decltype(&Derived::Traverse##NAME)>::value     \
+  (::clang::detail::has_same_member_pointer_type<                              \
+       decltype(&RecursiveASTVisitor::Traverse##NAME),                         \
+       decltype(&Derived::Traverse##NAME)>::value                              \
        ? static_cast<std::conditional_t<                                       \
-             has_same_member_pointer_type<                                     \
+             ::clang::detail::has_same_member_pointer_type<                    \
                  decltype(&RecursiveASTVisitor::Traverse##NAME),               \
                  decltype(&Derived::Traverse##NAME)>::value,                   \
              Derived &, RecursiveASTVisitor &>>(*this)                         \
@@ -377,60 +384,6 @@ template <typename Derived> class RecursiveASTVisitor {
   bool Visit##CLASS(CLASS *S) { return true; }
 #include "clang/AST/StmtNodes.inc"
 
-// Define Traverse*(), WalkUpFrom*(), and Visit*() for unary
-// operator methods.  Unary operators are not classes in themselves
-// (they're all opcodes in UnaryOperator) but do have visitors.
-#define OPERATOR(NAME)                                                         \
-  bool TraverseUnary##NAME(UnaryOperator *S,                                   \
-                           DataRecursionQueue *Queue = nullptr) {              \
-    if (!getDerived().shouldTraversePostOrder())                               \
-      TRY_TO(WalkUpFromUnary##NAME(S));                                        \
-    TRY_TO_TRAVERSE_OR_ENQUEUE_STMT(S->getSubExpr());                          \
-    return true;                                                               \
-  }                                                                            \
-  bool WalkUpFromUnary##NAME(UnaryOperator *S) {                               \
-    TRY_TO(WalkUpFromUnaryOperator(S));                                        \
-    TRY_TO(VisitUnary##NAME(S));                                               \
-    return true;                                                               \
-  }                                                                            \
-  bool VisitUnary##NAME(UnaryOperator *S) { return true; }
-
-  UNARYOP_LIST()
-#undef OPERATOR
-
-// Define Traverse*(), WalkUpFrom*(), and Visit*() for binary
-// operator methods.  Binary operators are not classes in themselves
-// (they're all opcodes in BinaryOperator) but do have visitors.
-#define GENERAL_BINOP_FALLBACK(NAME, BINOP_TYPE)                               \
-  bool TraverseBin##NAME(BINOP_TYPE *S, DataRecursionQueue *Queue = nullptr) { \
-    if (!getDerived().shouldTraversePostOrder())                               \
-      TRY_TO(WalkUpFromBin##NAME(S));                                          \
-    TRY_TO_TRAVERSE_OR_ENQUEUE_STMT(S->getLHS());                              \
-    TRY_TO_TRAVERSE_OR_ENQUEUE_STMT(S->getRHS());                              \
-    return true;                                                               \
-  }                                                                            \
-  bool WalkUpFromBin##NAME(BINOP_TYPE *S) {                                    \
-    TRY_TO(WalkUpFrom##BINOP_TYPE(S));                                         \
-    TRY_TO(VisitBin##NAME(S));                                                 \
-    return true;                                                               \
-  }                                                                            \
-  bool VisitBin##NAME(BINOP_TYPE *S) { return true; }
-
-#define OPERATOR(NAME) GENERAL_BINOP_FALLBACK(NAME, BinaryOperator)
-  BINOP_LIST()
-#undef OPERATOR
-
-// Define Traverse*(), WalkUpFrom*(), and Visit*() for compound
-// assignment methods.  Compound assignment operators are not
-// classes in themselves (they're all opcodes in
-// CompoundAssignOperator) but do have visitors.
-#define OPERATOR(NAME)                                                         \
-  GENERAL_BINOP_FALLBACK(NAME##Assign, CompoundAssignOperator)
-
-  CAO_LIST()
-#undef OPERATOR
-#undef GENERAL_BINOP_FALLBACK
-
 // ---- Methods on Types ----
 // FIXME: revamp to take TypeLoc's rather than Types.
 
@@ -549,42 +502,6 @@ template <typename Derived> class RecursiveASTVisitor {
 template <typename Derived>
 bool RecursiveASTVisitor<Derived>::dataTraverseNode(Stmt *S,
                                                     DataRecursionQueue *Queue) {
-#define DISPATCH_STMT(NAME, CLASS, VAR)                                        \
-  return TRAVERSE_STMT_BASE(NAME, CLASS, VAR, Queue);
-
-  // If we have a binary expr, dispatch to the subcode of the binop.  A smart
-  // optimizer (e.g. LLVM) will fold this comparison into the switch stmt
-  // below.
-  if (BinaryOperator *BinOp = dyn_cast<BinaryOperator>(S)) {
-    switch (BinOp->getOpcode()) {
-#define OPERATOR(NAME)                                                         \
-  case BO_##NAME:                                                              \
-    DISPATCH_STMT(Bin##NAME, BinaryOperator, S);
-
-      BINOP_LIST()
-#undef OPERATOR
-#undef BINOP_LIST
-
-#define OPERATOR(NAME)                                                         \
-  case BO_##NAME##Assign:                                                      \
-    DISPATCH_STMT(Bin##NAME##Assign, CompoundAssignOperator, S);
-
-      CAO_LIST()
-#undef OPERATOR
-#undef CAO_LIST
-    }
-  } else if (UnaryOperator *UnOp = dyn_cast<UnaryOperator>(S)) {
-    switch (UnOp->getOpcode()) {
-#define OPERATOR(NAME)                                                         \
-  case UO_##NAME:                                                              \
-    DISPATCH_STMT(Unary##NAME, UnaryOperator, S);
-
-      UNARYOP_LIST()
-#undef OPERATOR
-#undef UNARYOP_LIST
-    }
-  }
-
   // Top switch stmt: dispatch to TraverseFooStmt for each concrete FooStmt.
   switch (S->getStmtClass()) {
   case Stmt::NoStmtClass:
@@ -592,7 +509,7 @@ bool RecursiveASTVisitor<Derived>::dataTraverseNode(Stmt *S,
 #define ABSTRACT_STMT(STMT)
 #define STMT(CLASS, PARENT)                                                    \
   case Stmt::CLASS##Class:                                                     \
-    DISPATCH_STMT(CLASS, CLASS, S);
+    return TRAVERSE_STMT_BASE(CLASS, CLASS, S, Queue);
 #include "clang/AST/StmtNodes.inc"
   }
 
@@ -603,23 +520,44 @@ bool RecursiveASTVisitor<Derived>::dataTraverseNode(Stmt *S,
 
 template <typename Derived>
 bool RecursiveASTVisitor<Derived>::PostVisitStmt(Stmt *S) {
+  // In pre-order traversal mode, each Traverse##STMT method is responsible for
+  // calling WalkUpFrom. Therefore, if the user overrides Traverse##STMT and
+  // does not call the default implementation, the WalkUpFrom callback is not
+  // called. Post-order traversal mode should provide the same behavior
+  // regarding method overrides.
+  //
+  // In post-order traversal mode the Traverse##STMT method, when it receives a
+  // DataRecursionQueue, can't call WalkUpFrom after traversing children because
+  // it only enqueues the children and does not traverse them. TraverseStmt
+  // traverses the enqueued children, and we call WalkUpFrom here.
+  //
+  // However, to make pre-order and post-order modes identical with regards to
+  // whether they call WalkUpFrom at all, we call WalkUpFrom if and only if the
+  // user did not override the Traverse##STMT method. We implement the override
+  // check with isSameMethod calls below.
+
   switch (S->getStmtClass()) {
   case Stmt::NoStmtClass:
     break;
 #define ABSTRACT_STMT(STMT)
 #define STMT(CLASS, PARENT)                                                    \
   case Stmt::CLASS##Class:                                                     \
-    TRY_TO(WalkUpFrom##CLASS(static_cast<CLASS *>(S))); break;
+    if (::clang::detail::isSameMethod(&RecursiveASTVisitor::Traverse##CLASS,   \
+                                      &Derived::Traverse##CLASS)) {            \
+      TRY_TO(WalkUpFrom##CLASS(static_cast<CLASS *>(S)));                      \
+    }                                                                          \
+    break;
 #define INITLISTEXPR(CLASS, PARENT)                                            \
   case Stmt::CLASS##Class:                                                     \
-    {                                                                          \
+    if (::clang::detail::isSameMethod(&RecursiveASTVisitor::Traverse##CLASS,   \
+                                      &Derived::Traverse##CLASS)) {            \
       auto ILE = static_cast<CLASS *>(S);                                      \
       if (auto Syn = ILE->isSemanticForm() ? ILE->getSyntacticForm() : ILE)    \
         TRY_TO(WalkUpFrom##CLASS(Syn));                                        \
       if (auto Sem = ILE->isSemanticForm() ? ILE : ILE->getSemanticForm())     \
         TRY_TO(WalkUpFrom##CLASS(Sem));                                        \
-      break;                                                                   \
-    }
+    }                                                                          \
+    break;
 #include "clang/AST/StmtNodes.inc"
   }
 
@@ -669,9 +607,6 @@ bool RecursiveASTVisitor<Derived>::TraverseStmt(Stmt *S,
   return true;
 }
 
-#define DISPATCH(NAME, CLASS, VAR)                                             \
-  return getDerived().Traverse##NAME(static_cast<CLASS *>(VAR))
-
 template <typename Derived>
 bool RecursiveASTVisitor<Derived>::TraverseType(QualType T) {
   if (T.isNull())
@@ -681,7 +616,8 @@ bool RecursiveASTVisitor<Derived>::TraverseType(QualType T) {
 #define ABSTRACT_TYPE(CLASS, BASE)
 #define TYPE(CLASS, BASE)                                                      \
   case Type::CLASS:                                                            \
-    DISPATCH(CLASS##Type, CLASS##Type, const_cast<Type *>(T.getTypePtr()));
+    return getDerived().Traverse##CLASS##Type(                                 \
+        static_cast<CLASS##Type *>(const_cast<Type *>(T.getTypePtr())));
 #include "clang/AST/TypeNodes.inc"
   }
 
@@ -731,8 +667,6 @@ bool RecursiveASTVisitor<Derived>::TraverseDecl(Decl *D) {
   return true;
 }
 
-#undef DISPATCH
-
 template <typename Derived>
 bool RecursiveASTVisitor<Derived>::TraverseNestedNameSpecifier(
     NestedNameSpecifier *NNS) {
@@ -2220,8 +2154,13 @@ DEF_TRAVERSE_DECL(RequiresExprBodyDecl, {})
         TRY_TO_TRAVERSE_OR_ENQUEUE_STMT(SubStmt);                              \
       }                                                                        \
     }                                                                          \
-    if (!Queue && ReturnValue && getDerived().shouldTraversePostOrder())       \
+    /* Call WalkUpFrom if TRY_TO_TRAVERSE_OR_ENQUEUE_STMT has traversed the    \
+     * children already. If TRY_TO_TRAVERSE_OR_ENQUEUE_STMT only enqueued the  \
+     * children, PostVisitStmt will call WalkUpFrom after we are done visiting \
+     * children. */                                                            \
+    if (!Queue && ReturnValue && getDerived().shouldTraversePostOrder()) {     \
       TRY_TO(WalkUpFrom##STMT(S));                                             \
+    }                                                                          \
     return ReturnValue;                                                        \
   }
 
@@ -2392,6 +2331,9 @@ bool RecursiveASTVisitor<Derived>::TraverseSynOrSemInitListExpr(
     for (Stmt *SubStmt : S->children()) {
       TRY_TO_TRAVERSE_OR_ENQUEUE_STMT(SubStmt);
     }
+
+    if (!Queue && getDerived().shouldTraversePostOrder())
+      TRY_TO(WalkUpFromInitListExpr(S));
   }
   return true;
 }
@@ -3004,6 +2946,8 @@ bool RecursiveASTVisitor<Derived>::TraverseOMPClause(OMPClause *C) {
   case llvm::omp::Clause::Enum:                                                \
     break;
 #include "llvm/Frontend/OpenMP/OMPKinds.def"
+  default:
+    break;
   }
   return true;
 }
diff --git a/clang/include/clang/AST/Stmt.h b/clang/include/clang/AST/Stmt.h
index a226790aa76cb..d3fad58fcf596 100644
--- a/clang/include/clang/AST/Stmt.h
+++ b/clang/include/clang/AST/Stmt.h
@@ -614,9 +614,6 @@ class alignas(void *) Stmt {
     /// The kind of this overloaded operator. One of the enumerator
     /// value of OverloadedOperatorKind.
     unsigned OperatorKind : 6;
-
-    // Only meaningful for floating point types.
-    unsigned FPFeatures : 14;
   };
 
   class CXXRewrittenBinaryOperatorBitfields {
@@ -1169,9 +1166,7 @@ class alignas(void *) Stmt {
   /// Dumps the specified AST fragment and all subtrees to
   /// \c llvm::errs().
   void dump() const;
-  void dump(SourceManager &SM) const;
-  void dump(raw_ostream &OS, SourceManager &SM) const;
-  void dump(raw_ostream &OS) const;
+  void dump(raw_ostream &OS, const ASTContext &Context) const;
 
   /// \return Unique reproducible object identifier
   int64_t getID(const ASTContext &Context) const;
@@ -2282,6 +2277,8 @@ class WhileStmt final : public Stmt,
   enum { VarOffset = 0, BodyOffsetFromCond = 1 };
   enum { NumMandatoryStmtPtr = 2 };
 
+  SourceLocation LParenLoc, RParenLoc;
+
   unsigned varOffset() const { return VarOffset; }
   unsigned condOffset() const { return VarOffset + hasVarStorage(); }
   unsigned bodyOffset() const { return condOffset() + BodyOffsetFromCond; }
@@ -2292,7 +2289,8 @@ class WhileStmt final : public Stmt,
 
   /// Build a while statement.
   WhileStmt(const ASTContext &Ctx, VarDecl *Var, Expr *Cond, Stmt *Body,
-            SourceLocation WL);
+            SourceLocation WL, SourceLocation LParenLoc,
+            SourceLocation RParenLoc);
 
   /// Build an empty while statement.
   explicit WhileStmt(EmptyShell Empty, bool HasVar);
@@ -2300,7 +2298,8 @@ class WhileStmt final : public Stmt,
 public:
   /// Create a while statement.
   static WhileStmt *Create(const ASTContext &Ctx, VarDecl *Var, Expr *Cond,
-                           Stmt *Body, SourceLocation WL);
+                           Stmt *Body, SourceLocation WL,
+                           SourceLocation LParenLoc, SourceLocation RParenLoc);
 
   /// Create an empty while statement optionally with storage for
   /// a condition variable.
@@ -2364,6 +2363,11 @@ class WhileStmt final : public Stmt,
   SourceLocation getWhileLoc() const { return WhileStmtBits.WhileLoc; }
   void setWhileLoc(SourceLocation L) { WhileStmtBits.WhileLoc = L; }
 
+  SourceLocation getLParenLoc() const { return LParenLoc; }
+  void setLParenLoc(SourceLocation L) { LParenLoc = L; }
+  SourceLocation getRParenLoc() const { return RParenLoc; }
+  void setRParenLoc(SourceLocation L) { RParenLoc = L; }
+
   SourceLocation getBeginLoc() const { return getWhileLoc(); }
   SourceLocation getEndLoc() const LLVM_READONLY {
     return getBody()->getEndLoc();
diff --git a/clang/include/clang/AST/TextNodeDumper.h b/clang/include/clang/AST/TextNodeDumper.h
index b069bd09287a8..b4cfb5a380d17 100644
--- a/clang/include/clang/AST/TextNodeDumper.h
+++ b/clang/include/clang/AST/TextNodeDumper.h
@@ -22,10 +22,13 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/StmtVisitor.h"
 #include "clang/AST/TemplateArgumentVisitor.h"
+#include "clang/AST/Type.h"
 #include "clang/AST/TypeVisitor.h"
 
 namespace clang {
 
+class APValue;
+
 class TextTreeStructure {
   raw_ostream &OS;
   const bool ShowColors;
@@ -139,19 +142,29 @@ class TextNodeDumper
   const char *LastLocFilename = "";
   unsigned LastLocLine = ~0U;
 
-  const SourceManager *SM;
+  /// \p Context, \p SM, and \p Traits can be null. This is because we want
+  /// to be able to call \p dump() in a debugger without having to pass the
+  /// \p ASTContext to \p dump. Not all parts of the AST dump output will be
+  /// available without the \p ASTContext.
+  const ASTContext *Context = nullptr;
+  const SourceManager *SM = nullptr;
 
   /// The policy to use for printing; can be defaulted.
-  PrintingPolicy PrintPolicy;
+  PrintingPolicy PrintPolicy = LangOptions();
 
-  const comments::CommandTraits *Traits;
+  const comments::CommandTraits *Traits = nullptr;
 
   const char *getCommandName(unsigned CommandID);
 
+  void dumpAPValueChildren(const APValue &Value, QualType Ty,
+                           const APValue &(*IdxToChildFun)(const APValue &,
+                                                           unsigned),
+                           unsigned NumChildren, StringRef LabelSingular,
+                           StringRef LabelPlurial);
+
 public:
-  TextNodeDumper(raw_ostream &OS, bool ShowColors, const SourceManager *SM,
-                 const PrintingPolicy &PrintPolicy,
-                 const comments::CommandTraits *Traits);
+  TextNodeDumper(raw_ostream &OS, const ASTContext &Context, bool ShowColors);
+  TextNodeDumper(raw_ostream &OS, bool ShowColors);
 
   void Visit(const comments::Comment *C, const comments::FullComment *FC);
 
@@ -176,6 +189,8 @@ class TextNodeDumper
 
   void Visit(const GenericSelectionExpr::ConstAssociation &A);
 
+  void Visit(const APValue &Value, QualType Ty);
+
   void dumpPointer(const void *Ptr);
   void dumpLocation(SourceLocation Loc);
   void dumpSourceRange(SourceRange R);
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 10b8b41efeeb0..0fc50e0e799ff 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -1058,7 +1058,7 @@ class QualType {
 
   void dump(const char *s) const;
   void dump() const;
-  void dump(llvm::raw_ostream &OS) const;
+  void dump(llvm::raw_ostream &OS, const ASTContext &Context) const;
 
   void Profile(llvm::FoldingSetNodeID &ID) const {
     ID.AddPointer(getAsOpaquePtr());
@@ -2471,7 +2471,7 @@ class alignas(8) Type : public ExtQualsTypeCommonBase {
 
   CanQualType getCanonicalTypeUnqualified() const; // in CanonicalType.h
   void dump() const;
-  void dump(llvm::raw_ostream &OS) const;
+  void dump(llvm::raw_ostream &OS, const ASTContext &Context) const;
 };
 
 /// This will check for a TypedefType by removing any existing sugar
diff --git a/clang/include/clang/AST/VTableBuilder.h b/clang/include/clang/AST/VTableBuilder.h
index e6557e4389196..241dd13f903e5 100644
--- a/clang/include/clang/AST/VTableBuilder.h
+++ b/clang/include/clang/AST/VTableBuilder.h
@@ -354,6 +354,9 @@ class VTableContextBase {
   }
 
   bool IsMicrosoftABI;
+
+  /// Determine whether this function should be assigned a vtable slot.
+  static bool hasVtableSlot(const CXXMethodDecl *MD);
 };
 
 class ItaniumVTableContext : public VTableContextBase {
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index 6dbcc01442ed6..643419743a119 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -283,9 +283,10 @@ AST_POLYMORPHIC_MATCHER(isExpansionInSystemHeader,
 /// \endcode
 ///
 /// Usable as: Matcher<Decl>, Matcher<Stmt>, Matcher<TypeLoc>
-AST_POLYMORPHIC_MATCHER_P(isExpansionInFileMatching,
-                          AST_POLYMORPHIC_SUPPORTED_TYPES(Decl, Stmt, TypeLoc),
-                          std::string, RegExp) {
+AST_POLYMORPHIC_MATCHER_REGEX(isExpansionInFileMatching,
+                              AST_POLYMORPHIC_SUPPORTED_TYPES(Decl, Stmt,
+                                                              TypeLoc),
+                              RegExp) {
   auto &SourceManager = Finder->getASTContext().getSourceManager();
   auto ExpansionLoc = SourceManager.getExpansionLoc(Node.getBeginLoc());
   if (ExpansionLoc.isInvalid()) {
@@ -298,8 +299,7 @@ AST_POLYMORPHIC_MATCHER_P(isExpansionInFileMatching,
   }
 
   auto Filename = FileEntry->getName();
-  llvm::Regex RE(RegExp);
-  return RE.match(Filename);
+  return RegExp->match(Filename);
 }
 
 /// Matches statements that are (transitively) expanded from the named macro.
@@ -2748,11 +2748,9 @@ extern const internal::VariadicFunction<internal::Matcher<NamedDecl>, StringRef,
 /// \code
 ///   namespace foo { namespace bar { class X; } }
 /// \endcode
-AST_MATCHER_P(NamedDecl, matchesName, std::string, RegExp) {
-  assert(!RegExp.empty());
+AST_MATCHER_REGEX(NamedDecl, matchesName, RegExp) {
   std::string FullNameString = "::" + Node.getQualifiedNameAsString();
-  llvm::Regex RE(RegExp);
-  return RE.match(FullNameString);
+  return RegExp->match(FullNameString);
 }
 
 /// Matches overloaded operator names.
@@ -2864,7 +2862,7 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
 /// BaseSpecMatcher.
 ///
 /// Example:
-/// matcher hasAnyBase(hasType(cxxRecordDecl(hasName("SpecialBase")))))
+/// matcher hasAnyBase(hasType(cxxRecordDecl(hasName("SpecialBase"))))
 /// \code
 ///   class Foo;
 ///   class Bar : Foo {};
@@ -2880,6 +2878,26 @@ AST_MATCHER_P(CXXRecordDecl, hasAnyBase, internal::Matcher<CXXBaseSpecifier>,
   return internal::matchesAnyBase(Node, BaseSpecMatcher, Finder, Builder);
 }
 
+/// Matches C++ classes that have a direct base matching \p BaseSpecMatcher.
+///
+/// Example:
+/// matcher hasDirectBase(hasType(cxxRecordDecl(hasName("SpecialBase"))))
+/// \code
+///   class Foo;
+///   class Bar : Foo {};
+///   class Baz : Bar {};
+///   class SpecialBase;
+///   class Proxy : SpecialBase {};  // matches Proxy
+///   class IndirectlyDerived : Proxy {};  // doesn't match
+/// \endcode
+AST_MATCHER_P(CXXRecordDecl, hasDirectBase, internal::Matcher<CXXBaseSpecifier>,
+              BaseSpecMatcher) {
+  return Node.hasDefinition() &&
+         llvm::any_of(Node.bases(), [&](const CXXBaseSpecifier &Base) {
+           return BaseSpecMatcher.matches(Base, Finder, Builder);
+         });
+}
+
 /// Similar to \c isDerivedFrom(), but also matches classes that directly
 /// match \c Base.
 AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
@@ -3373,11 +3391,9 @@ extern const internal::VariadicFunction<internal::Matcher<ObjCMessageExpr>,
 /// \code
 ///     [self.bodyView loadHTMLString:html baseURL:NULL];
 /// \endcode
-AST_MATCHER_P(ObjCMessageExpr, matchesSelector, std::string, RegExp) {
-  assert(!RegExp.empty());
+AST_MATCHER_REGEX(ObjCMessageExpr, matchesSelector, RegExp) {
   std::string SelectorString = Node.getSelector().getAsString();
-  llvm::Regex RE(RegExp);
-  return RE.match(SelectorString);
+  return RegExp->match(SelectorString);
 }
 
 /// Matches when the selector is the empty selector
@@ -4323,7 +4339,7 @@ AST_POLYMORPHIC_MATCHER_P2(forEachArgumentWithParam,
 /// ``parmVarDecl(isAtPosition(0))`` matches ``int a``.
 ///
 /// ``parmVarDecl(isAtPosition(1))`` matches ``int b``.
-AST_MATCHER_P(clang::ParmVarDecl, isAtPosition, unsigned, N) {
+AST_MATCHER_P(ParmVarDecl, isAtPosition, unsigned, N) {
   const clang::DeclContext *Context = Node.getParentFunctionOrMethod();
 
   if (const auto *Decl = dyn_cast_or_null<FunctionDecl>(Context))
@@ -7174,10 +7190,12 @@ AST_MATCHER_P(OMPExecutableDirective, hasAnyClause,
 /// \code
 ///   #pragma omp parallel default(none)
 ///   #pragma omp parallel default(shared)
+///   #pragma omp parallel default(firstprivate)
 ///   #pragma omp parallel
 /// \endcode
 ///
-/// ``ompDefaultClause()`` matches ``default(none)`` and ``default(shared)``.
+/// ``ompDefaultClause()`` matches ``default(none)``, ``default(shared)``, and
+/// ``default(firstprivate)``
 extern const internal::VariadicDynCastAllOfMatcher<OMPClause, OMPDefaultClause>
     ompDefaultClause;
 
@@ -7189,6 +7207,7 @@ extern const internal::VariadicDynCastAllOfMatcher<OMPClause, OMPDefaultClause>
 ///   #pragma omp parallel
 ///   #pragma omp parallel default(none)
 ///   #pragma omp parallel default(shared)
+///   #pragma omp parallel default(firstprivate)
 /// \endcode
 ///
 /// ``ompDefaultClause(isNoneKind())`` matches only ``default(none)``.
@@ -7204,6 +7223,7 @@ AST_MATCHER(OMPDefaultClause, isNoneKind) {
 ///   #pragma omp parallel
 ///   #pragma omp parallel default(none)
 ///   #pragma omp parallel default(shared)
+///   #pragma omp parallel default(firstprivate)
 /// \endcode
 ///
 /// ``ompDefaultClause(isSharedKind())`` matches only ``default(shared)``.
@@ -7211,6 +7231,24 @@ AST_MATCHER(OMPDefaultClause, isSharedKind) {
   return Node.getDefaultKind() == llvm::omp::OMP_DEFAULT_shared;
 }
 
+/// Matches if the OpenMP ``default`` clause has ``firstprivate`` kind
+/// specified.
+///
+/// Given
+///
+/// \code
+///   #pragma omp parallel
+///   #pragma omp parallel default(none)
+///   #pragma omp parallel default(shared)
+///   #pragma omp parallel default(firstprivate)
+/// \endcode
+///
+/// ``ompDefaultClause(isFirstPrivateKind())`` matches only
+/// ``default(firstprivate)``.
+AST_MATCHER(OMPDefaultClause, isFirstPrivateKind) {
+  return Node.getDefaultKind() == llvm::omp::OMP_DEFAULT_firstprivate;
+}
+
 /// Matches if the OpenMP directive is allowed to contain the specified OpenMP
 /// clause kind.
 ///
diff --git a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
index fc41407ba2961..3992850c992de 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
@@ -40,7 +40,6 @@
 #include "clang/AST/DeclFriend.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
-#include "clang/AST/ExprObjC.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/NestedNameSpecifier.h"
@@ -61,11 +60,13 @@
 #include "llvm/ADT/iterator.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/Regex.h"
 #include <algorithm>
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
 #include <map>
+#include <memory>
 #include <string>
 #include <tuple>
 #include <type_traits>
@@ -1948,6 +1949,10 @@ bool matchesAnyBase(const CXXRecordDecl &Node,
                     const Matcher<CXXBaseSpecifier> &BaseSpecMatcher,
                     ASTMatchFinder *Finder, BoundNodesTreeBuilder *Builder);
 
+std::shared_ptr<llvm::Regex> createAndVerifyRegex(StringRef Regex,
+                                                  llvm::Regex::RegexFlags Flags,
+                                                  StringRef MatcherID);
+
 } // namespace internal
 
 } // namespace ast_matchers
diff --git a/clang/include/clang/ASTMatchers/ASTMatchersMacros.h b/clang/include/clang/ASTMatchers/ASTMatchersMacros.h
index 4977bf32e9f6e..45e8b1a88b81a 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchersMacros.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchersMacros.h
@@ -438,4 +438,122 @@
       ReturnTypesF>::Func MatcherName##Loc;                                    \
   AST_TYPE_TRAVERSE_MATCHER(MatcherName, FunctionName##Type, ReturnTypesF)
 
+/// AST_MATCHER_REGEX(Type, DefineMatcher, Param) { ... }
+/// defines a function named DefineMatcher() that takes a regular expression
+/// string paramater and an optional RegexFlags parameter and returns a
+/// Matcher<Type> object.
+///
+/// The code between the curly braces has access to the following variables:
+///
+///   Node:                  the AST node being matched; its type is Type.
+///   Param:                 a pointer to an \ref llvm::Regex object
+///   Finder:                an ASTMatchFinder*.
+///   Builder:               a BoundNodesTreeBuilder*.
+///
+/// The code should return true if 'Node' matches.
+#define AST_MATCHER_REGEX(Type, DefineMatcher, Param)                          \
+  AST_MATCHER_REGEX_OVERLOAD(Type, DefineMatcher, Param, 0)
+
+#define AST_MATCHER_REGEX_OVERLOAD(Type, DefineMatcher, Param, OverloadId)     \
+  namespace internal {                                                         \
+  class matcher_##DefineMatcher##OverloadId##Matcher                           \
+      : public ::clang::ast_matchers::internal::MatcherInterface<Type> {       \
+  public:                                                                      \
+    explicit matcher_##DefineMatcher##OverloadId##Matcher(                     \
+        std::shared_ptr<llvm::Regex> RE)                                       \
+        : Param(std::move(RE)) {}                                              \
+    bool matches(const Type &Node,                                             \
+                 ::clang::ast_matchers::internal::ASTMatchFinder *Finder,      \
+                 ::clang::ast_matchers::internal::BoundNodesTreeBuilder        \
+                     *Builder) const override;                                 \
+                                                                               \
+  private:                                                                     \
+    std::shared_ptr<llvm::Regex> const Param;                                  \
+  };                                                                           \
+  }                                                                            \
+  inline ::clang::ast_matchers::internal::Matcher<Type> DefineMatcher(         \
+      llvm::StringRef Param, llvm::Regex::RegexFlags RegexFlags) {             \
+    return ::clang::ast_matchers::internal::makeMatcher(                       \
+        new internal::matcher_##DefineMatcher##OverloadId##Matcher(            \
+            ::clang::ast_matchers::internal::createAndVerifyRegex(             \
+                Param, RegexFlags, #DefineMatcher)));                          \
+  }                                                                            \
+  inline ::clang::ast_matchers::internal::Matcher<Type> DefineMatcher(         \
+      llvm::StringRef Param) {                                                 \
+    return DefineMatcher(Param, llvm::Regex::NoFlags);                         \
+  }                                                                            \
+                                                                               \
+  typedef ::clang::ast_matchers::internal::Matcher<Type> (                     \
+      &DefineMatcher##_Type##OverloadId##Flags)(llvm::StringRef,               \
+                                                llvm::Regex::RegexFlags);      \
+  typedef ::clang::ast_matchers::internal::Matcher<Type> (                     \
+      &DefineMatcher##_Type##OverloadId)(llvm::StringRef);                     \
+  inline bool internal::matcher_##DefineMatcher##OverloadId##Matcher::matches( \
+      const Type &Node,                                                        \
+      ::clang::ast_matchers::internal::ASTMatchFinder *Finder,                 \
+      ::clang::ast_matchers::internal::BoundNodesTreeBuilder *Builder) const
+
+/// AST_POLYMORPHIC_MATCHER_REGEX(DefineMatcher, ReturnTypesF, Param) { ... }
+/// defines a function named DefineMatcher() that takes a regular expression
+/// string paramater and an optional RegexFlags parameter that is polymorphic in
+/// the return type.
+///
+/// The variables are the same as for
+/// AST_MATCHER_REGEX, with the addition of NodeType, which specifies the node
+/// type of the matcher Matcher<NodeType> returned by the function matcher().
+#define AST_POLYMORPHIC_MATCHER_REGEX(DefineMatcher, ReturnTypesF, Param)      \
+  AST_POLYMORPHIC_MATCHER_REGEX_OVERLOAD(DefineMatcher, ReturnTypesF, Param, 0)
+
+#define AST_POLYMORPHIC_MATCHER_REGEX_OVERLOAD(DefineMatcher, ReturnTypesF,    \
+                                               Param, OverloadId)              \
+  namespace internal {                                                         \
+  template <typename NodeType, typename ParamT>                                \
+  class matcher_##DefineMatcher##OverloadId##Matcher                           \
+      : public ::clang::ast_matchers::internal::MatcherInterface<NodeType> {   \
+  public:                                                                      \
+    explicit matcher_##DefineMatcher##OverloadId##Matcher(                     \
+        std::shared_ptr<llvm::Regex> RE)                                       \
+        : Param(std::move(RE)) {}                                              \
+    bool matches(const NodeType &Node,                                         \
+                 ::clang::ast_matchers::internal::ASTMatchFinder *Finder,      \
+                 ::clang::ast_matchers::internal::BoundNodesTreeBuilder        \
+                     *Builder) const override;                                 \
+                                                                               \
+  private:                                                                     \
+    std::shared_ptr<llvm::Regex> const Param;                                  \
+  };                                                                           \
+  }                                                                            \
+  inline ::clang::ast_matchers::internal::PolymorphicMatcherWithParam1<        \
+      internal::matcher_##DefineMatcher##OverloadId##Matcher,                  \
+      std::shared_ptr<llvm::Regex>, ReturnTypesF>                              \
+  DefineMatcher(llvm::StringRef Param, llvm::Regex::RegexFlags RegexFlags) {   \
+    return ::clang::ast_matchers::internal::PolymorphicMatcherWithParam1<      \
+        internal::matcher_##DefineMatcher##OverloadId##Matcher,                \
+        std::shared_ptr<llvm::Regex>, ReturnTypesF>(                           \
+        ::clang::ast_matchers::internal::createAndVerifyRegex(                 \
+            Param, RegexFlags, #DefineMatcher));                               \
+  }                                                                            \
+  inline ::clang::ast_matchers::internal::PolymorphicMatcherWithParam1<        \
+      internal::matcher_##DefineMatcher##OverloadId##Matcher,                  \
+      std::shared_ptr<llvm::Regex>, ReturnTypesF>                              \
+  DefineMatcher(llvm::StringRef Param) {                                       \
+    return DefineMatcher(Param, llvm::Regex::NoFlags);                         \
+  }                                                                            \
+  typedef ::clang::ast_matchers::internal::PolymorphicMatcherWithParam1<       \
+      internal::matcher_##DefineMatcher##OverloadId##Matcher,                  \
+      std::shared_ptr<llvm::Regex>, ReturnTypesF> (                            \
+      &DefineMatcher##_Type##OverloadId##Flags)(                               \
+      llvm::StringRef Param, llvm::Regex::RegexFlags RegexFlags);              \
+  typedef ::clang::ast_matchers::internal::PolymorphicMatcherWithParam1<       \
+      internal::matcher_##DefineMatcher##OverloadId##Matcher,                  \
+      std::shared_ptr<llvm::Regex>, ReturnTypesF> (                            \
+      &DefineMatcher##_Type##OverloadId)(llvm::StringRef Param);               \
+  template <typename NodeType, typename ParamT>                                \
+  bool internal::                                                              \
+      matcher_##DefineMatcher##OverloadId##Matcher<NodeType, ParamT>::matches( \
+          const NodeType &Node,                                                \
+          ::clang::ast_matchers::internal::ASTMatchFinder *Finder,             \
+          ::clang::ast_matchers::internal::BoundNodesTreeBuilder *Builder)     \
+          const
+
 #endif // LLVM_CLANG_ASTMATCHERS_ASTMATCHERSMACROS_H
diff --git a/clang/include/clang/Analysis/Analyses/Dominators.h b/clang/include/clang/Analysis/Analyses/Dominators.h
index 061c98137da29..95a661138df4d 100644
--- a/clang/include/clang/Analysis/Analyses/Dominators.h
+++ b/clang/include/clang/Analysis/Analyses/Dominators.h
@@ -167,9 +167,7 @@ class CFGDominatorTreeImpl : public ManagedAnalysis {
   }
 
   /// Releases the memory held by the dominator tree.
-  virtual void releaseMemory() {
-    DT.releaseMemory();
-  }
+  virtual void releaseMemory() { DT.reset(); }
 
   /// Converts the dominator tree to human readable form.
   virtual void print(raw_ostream &OS, const llvm::Module* M= nullptr) const {
@@ -351,7 +349,7 @@ ClangCFGPostDomReverseChildrenGetter::Get(
 ///
 template <> struct GraphTraits<clang::DomTreeNode *> {
   using NodeRef = ::clang::DomTreeNode *;
-  using ChildIteratorType = ::clang::DomTreeNode::iterator;
+  using ChildIteratorType = ::clang::DomTreeNode::const_iterator;
 
   static NodeRef getEntryNode(NodeRef N) { return N; }
   static ChildIteratorType child_begin(NodeRef N) { return N->begin(); }
diff --git a/clang/include/clang/Basic/AArch64SVEACLETypes.def b/clang/include/clang/Basic/AArch64SVEACLETypes.def
index 0640da83ebb3f..b98a07436e941 100644
--- a/clang/include/clang/Basic/AArch64SVEACLETypes.def
+++ b/clang/include/clang/Basic/AArch64SVEACLETypes.def
@@ -66,7 +66,7 @@ SVE_VECTOR_TYPE("__SVFloat16_t", "__SVFloat16_t", SveFloat16, SveFloat16Ty, 8, 1
 SVE_VECTOR_TYPE("__SVFloat32_t", "__SVFloat32_t", SveFloat32, SveFloat32Ty, 4, 32, true, true, false)
 SVE_VECTOR_TYPE("__SVFloat64_t", "__SVFloat64_t", SveFloat64, SveFloat64Ty, 2, 64, true, true, false)
 
-SVE_VECTOR_TYPE("__SVBFloat16_t", "__SVBFloat16_t", SveBFloat16, SveBFloat16Ty, 8, 16, false, false, true)
+SVE_VECTOR_TYPE("__SVBFloat16_t", "__SVBFloat16_t", SveBFloat16, SveBFloat16Ty, 8, 16, true, false, true)
 
 //
 // x2
@@ -85,6 +85,7 @@ SVE_VECTOR_TYPE("__clang_svfloat16x2_t", "svfloat16x2_t", SveFloat16x2, SveFloat
 SVE_VECTOR_TYPE("__clang_svfloat32x2_t", "svfloat32x2_t", SveFloat32x2, SveFloat32x2Ty, 8, 32, true, true, false)
 SVE_VECTOR_TYPE("__clang_svfloat64x2_t", "svfloat64x2_t", SveFloat64x2, SveFloat64x2Ty, 4, 64, true, true, false)
 
+SVE_VECTOR_TYPE("__clang_svbfloat16x2_t", "svbfloat16x2_t", SveBFloat16x2, SveBFloat16x2Ty, 16, 16, true, false, true)
 //
 // x3
 //
@@ -102,6 +103,7 @@ SVE_VECTOR_TYPE("__clang_svfloat16x3_t", "svfloat16x3_t", SveFloat16x3, SveFloat
 SVE_VECTOR_TYPE("__clang_svfloat32x3_t", "svfloat32x3_t", SveFloat32x3, SveFloat32x3Ty, 12, 32, true, true, false)
 SVE_VECTOR_TYPE("__clang_svfloat64x3_t", "svfloat64x3_t", SveFloat64x3, SveFloat64x3Ty, 6, 64, true, true, false)
 
+SVE_VECTOR_TYPE("__clang_svbfloat16x3_t", "svbfloat16x3_t", SveBFloat16x3, SveBFloat16x3Ty, 24, 16, true, false, true)
 //
 // x4
 //
@@ -119,6 +121,8 @@ SVE_VECTOR_TYPE("__clang_svfloat16x4_t", "svfloat16x4_t", SveFloat16x4, SveFloat
 SVE_VECTOR_TYPE("__clang_svfloat32x4_t", "svfloat32x4_t", SveFloat32x4, SveFloat32x4Ty, 16, 32, true, true, false)
 SVE_VECTOR_TYPE("__clang_svfloat64x4_t", "svfloat64x4_t", SveFloat64x4, SveFloat64x4Ty, 8, 64, true, true, false)
 
+SVE_VECTOR_TYPE("__clang_svbfloat16x4_t", "svbfloat16x4_t", SveBFloat16x4, SveBFloat16x4Ty, 32, 16, true, false, true)
+
 SVE_PREDICATE_TYPE("__SVBool_t", "__SVBool_t", SveBool, SveBoolTy, 16)
 
 #undef SVE_VECTOR_TYPE
diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def
index ac9af60288675..1416a64543a41 100644
--- a/clang/include/clang/Basic/Builtins.def
+++ b/clang/include/clang/Basic/Builtins.def
@@ -65,6 +65,7 @@
 // & -> reference (optionally followed by an address space number)
 // C -> const
 // D -> volatile
+// R -> restrict
 
 // The third value provided to the macro specifies information about attributes
 // of the function.  These must be kept in sync with the predicates in the
@@ -566,6 +567,7 @@ BUILTIN(__builtin___vprintf_chk, "iicC*a", "FP:1:")
 
 BUILTIN(__builtin_unpredictable, "LiLi"   , "nc")
 BUILTIN(__builtin_expect, "LiLiLi"   , "nc")
+BUILTIN(__builtin_expect_with_probability, "LiLiLid", "nc")
 BUILTIN(__builtin_prefetch, "vvC*.", "nc")
 BUILTIN(__builtin_readcyclecounter, "ULLi", "n")
 BUILTIN(__builtin_trap, "v", "nr")
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index 9add10c64962a..042a86368559a 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -60,11 +60,11 @@ BUILTIN(__builtin_amdgcn_ds_gws_sema_br, "vUiUi", "n")
 BUILTIN(__builtin_amdgcn_ds_gws_sema_p, "vUi", "n")
 BUILTIN(__builtin_amdgcn_fence, "vUicC*", "n")
 
-BUILTIN(__builtin_amdgcn_atomic_inc32, "ZiZiD*ZiUicC*", "n")
-BUILTIN(__builtin_amdgcn_atomic_inc64, "WiWiD*WiUicC*", "n")
+BUILTIN(__builtin_amdgcn_atomic_inc32, "UZiUZiD*UZiUicC*", "n")
+BUILTIN(__builtin_amdgcn_atomic_inc64, "UWiUWiD*UWiUicC*", "n")
 
-BUILTIN(__builtin_amdgcn_atomic_dec32, "ZiZiD*ZiUicC*", "n")
-BUILTIN(__builtin_amdgcn_atomic_dec64, "WiWiD*WiUicC*", "n")
+BUILTIN(__builtin_amdgcn_atomic_dec32, "UZiUZiD*UZiUicC*", "n")
+BUILTIN(__builtin_amdgcn_atomic_dec64, "UWiUWiD*UWiUicC*", "n")
 
 // FIXME: Need to disallow constant address space.
 BUILTIN(__builtin_amdgcn_div_scale, "dddbb*", "n")
@@ -77,6 +77,8 @@ BUILTIN(__builtin_amdgcn_trig_preop, "ddi", "nc")
 BUILTIN(__builtin_amdgcn_trig_preopf, "ffi", "nc")
 BUILTIN(__builtin_amdgcn_rcp, "dd", "nc")
 BUILTIN(__builtin_amdgcn_rcpf, "ff", "nc")
+BUILTIN(__builtin_amdgcn_sqrt, "dd", "nc")
+BUILTIN(__builtin_amdgcn_sqrtf, "ff", "nc")
 BUILTIN(__builtin_amdgcn_rsq, "dd", "nc")
 BUILTIN(__builtin_amdgcn_rsqf, "ff", "nc")
 BUILTIN(__builtin_amdgcn_rsq_clamp, "dd", "nc")
@@ -162,6 +164,7 @@ BUILTIN(__builtin_amdgcn_interp_mov, "fUiUiUiUi", "nc")
 
 TARGET_BUILTIN(__builtin_amdgcn_div_fixuph, "hhhh", "nc", "16-bit-insts")
 TARGET_BUILTIN(__builtin_amdgcn_rcph, "hh", "nc", "16-bit-insts")
+TARGET_BUILTIN(__builtin_amdgcn_sqrth, "hh", "nc", "16-bit-insts")
 TARGET_BUILTIN(__builtin_amdgcn_rsqh, "hh", "nc", "16-bit-insts")
 TARGET_BUILTIN(__builtin_amdgcn_sinh, "hh", "nc", "16-bit-insts")
 TARGET_BUILTIN(__builtin_amdgcn_cosh, "hh", "nc", "16-bit-insts")
diff --git a/clang/include/clang/Basic/BuiltinsPPC.def b/clang/include/clang/Basic/BuiltinsPPC.def
index 238e56b6dfce0..d0df5fcd15523 100644
--- a/clang/include/clang/Basic/BuiltinsPPC.def
+++ b/clang/include/clang/Basic/BuiltinsPPC.def
@@ -302,10 +302,40 @@ BUILTIN(__builtin_altivec_vrldnm, "V2ULLiV2ULLiV2ULLi", "")
 BUILTIN(__builtin_altivec_vpdepd, "V2ULLiV2ULLiV2ULLi", "")
 BUILTIN(__builtin_altivec_vpextd, "V2ULLiV2ULLiV2ULLi", "")
 
+// P10 Vector Centrifuge built-in.
+BUILTIN(__builtin_altivec_vcfuged, "V2ULLiV2ULLiV2ULLi", "")
+
+// P10 Vector Gather Every N-th Bit built-in.
+BUILTIN(__builtin_altivec_vgnb, "ULLiV1ULLLiIi", "")
+
 // P10 Vector Clear Bytes built-ins.
 BUILTIN(__builtin_altivec_vclrlb, "V16cV16cUi", "")
 BUILTIN(__builtin_altivec_vclrrb, "V16cV16cUi", "")
 
+// P10 Vector Count Leading / Trailing Zeroes under bit Mask built-ins.
+BUILTIN(__builtin_altivec_vclzdm, "V2ULLiV2ULLiV2ULLi", "")
+BUILTIN(__builtin_altivec_vctzdm, "V2ULLiV2ULLiV2ULLi", "")
+
+// P10 Vector Shift built-ins.
+BUILTIN(__builtin_altivec_vsldbi, "V16UcV16UcV16UcIi", "")
+BUILTIN(__builtin_altivec_vsrdbi, "V16UcV16UcV16UcIi", "")
+
+// P10 Vector Insert built-ins.
+BUILTIN(__builtin_altivec_vinsblx, "V16UcV16UcULLiULLi", "")
+BUILTIN(__builtin_altivec_vinsbrx, "V16UcV16UcULLiULLi", "")
+BUILTIN(__builtin_altivec_vinshlx, "V8UsV8UsULLiULLi", "")
+BUILTIN(__builtin_altivec_vinshrx, "V8UsV8UsULLiULLi", "")
+BUILTIN(__builtin_altivec_vinswlx, "V4UiV4UiULLiULLi", "")
+BUILTIN(__builtin_altivec_vinswrx, "V4UiV4UiULLiULLi", "")
+BUILTIN(__builtin_altivec_vinsdlx, "V2ULLiV2ULLiULLiULLi", "")
+BUILTIN(__builtin_altivec_vinsdrx, "V2ULLiV2ULLiULLiULLi", "")
+BUILTIN(__builtin_altivec_vinsbvlx, "V16UcV16UcULLiV16Uc", "")
+BUILTIN(__builtin_altivec_vinsbvrx, "V16UcV16UcULLiV16Uc", "")
+BUILTIN(__builtin_altivec_vinshvlx, "V8UsV8UsULLiV8Us", "")
+BUILTIN(__builtin_altivec_vinshvrx, "V8UsV8UsULLiV8Us", "")
+BUILTIN(__builtin_altivec_vinswvlx, "V4UiV4UiULLiV4Ui", "")
+BUILTIN(__builtin_altivec_vinswvrx, "V4UiV4UiULLiV4Ui", "")
+
 // VSX built-ins.
 
 BUILTIN(__builtin_vsx_lxvd2x, "V2divC*", "")
@@ -399,6 +429,11 @@ BUILTIN(__builtin_vsx_xvcpsgnsp, "V4fV4fV4f", "")
 BUILTIN(__builtin_vsx_xvabssp, "V4fV4f", "")
 BUILTIN(__builtin_vsx_xvabsdp, "V2dV2d", "")
 
+BUILTIN(__builtin_vsx_xxgenpcvbm, "V16UcV16Uci", "")
+BUILTIN(__builtin_vsx_xxgenpcvhm, "V8UsV8Usi", "")
+BUILTIN(__builtin_vsx_xxgenpcvwm, "V4UiV4Uii", "")
+BUILTIN(__builtin_vsx_xxgenpcvdm, "V2ULLiV2ULLii", "")
+
 // vector Insert/Extract exponent/significand builtins
 BUILTIN(__builtin_vsx_xviexpdp, "V2dV2ULLiV2ULLi", "")
 BUILTIN(__builtin_vsx_xviexpsp, "V4fV4UiV4Ui", "")
@@ -430,6 +465,17 @@ BUILTIN(__builtin_vsx_extractuword, "V2ULLiV16UcIi", "")
 BUILTIN(__builtin_vsx_xxpermdi, "v.", "t")
 BUILTIN(__builtin_vsx_xxsldwi, "v.", "t")
 
+BUILTIN(__builtin_vsx_xxeval, "V2ULLiV2ULLiV2ULLiV2ULLiIi", "")
+
+// P10 Vector Permute Extended built-in.
+BUILTIN(__builtin_vsx_xxpermx, "V16UcV16UcV16UcV16UcIi", "")
+
+// P10 Vector Blend built-ins.
+BUILTIN(__builtin_vsx_xxblendvb, "V16UcV16UcV16UcV16Uc", "")
+BUILTIN(__builtin_vsx_xxblendvh, "V8UsV8UsV8UsV8Us", "")
+BUILTIN(__builtin_vsx_xxblendvw, "V4UiV4UiV4UiV4Ui", "")
+BUILTIN(__builtin_vsx_xxblendvd, "V2ULLiV2ULLiV2ULLiV2ULLi", "")
+
 // Float 128 built-ins
 BUILTIN(__builtin_sqrtf128_round_to_odd, "LLdLLd", "")
 BUILTIN(__builtin_addf128_round_to_odd, "LLdLLdLLd", "")
@@ -480,6 +526,9 @@ BUILTIN(__builtin_divdeu, "ULLiULLiULLi", "")
 BUILTIN(__builtin_bpermd, "SLLiSLLiSLLi", "")
 BUILTIN(__builtin_pdepd, "ULLiULLiULLi", "")
 BUILTIN(__builtin_pextd, "ULLiULLiULLi", "")
+BUILTIN(__builtin_cfuged, "ULLiULLiULLi", "")
+BUILTIN(__builtin_cntlzdm, "ULLiULLiULLi", "")
+BUILTIN(__builtin_cnttzdm, "ULLiULLiULLi", "")
 
 // Vector int128 (un)pack
 BUILTIN(__builtin_unpack_vector_int128, "ULLiV1LLLii", "")
diff --git a/clang/include/clang/Basic/BuiltinsWebAssembly.def b/clang/include/clang/Basic/BuiltinsWebAssembly.def
index 5e6f0d90ab46f..ecee7782920f6 100644
--- a/clang/include/clang/Basic/BuiltinsWebAssembly.def
+++ b/clang/include/clang/Basic/BuiltinsWebAssembly.def
@@ -25,10 +25,6 @@
 BUILTIN(__builtin_wasm_memory_size, "zIi", "n")
 BUILTIN(__builtin_wasm_memory_grow, "zIiz", "n")
 
-// Bulk memory builtins
-TARGET_BUILTIN(__builtin_wasm_memory_init, "vIUiIUiv*UiUi", "", "bulk-memory")
-TARGET_BUILTIN(__builtin_wasm_data_drop, "vIUi", "", "bulk-memory")
-
 // Thread-local storage
 TARGET_BUILTIN(__builtin_wasm_tls_size, "z", "nc", "bulk-memory")
 TARGET_BUILTIN(__builtin_wasm_tls_align, "z", "nc", "bulk-memory")
diff --git a/clang/include/clang/Basic/BuiltinsX86.def b/clang/include/clang/Basic/BuiltinsX86.def
index 7dcbcf086ede4..35fb98352ec2b 100644
--- a/clang/include/clang/Basic/BuiltinsX86.def
+++ b/clang/include/clang/Basic/BuiltinsX86.def
@@ -1934,6 +1934,15 @@ TARGET_HEADER_BUILTIN(__readgsword,  "UsUNi", "nh", "intrin.h", ALL_MS_LANGUAGES
 TARGET_HEADER_BUILTIN(__readgsdword, "UNiUNi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(__readgsqword, "ULLiUNi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 
+TARGET_HEADER_BUILTIN(_InterlockedAnd64,         "WiWiD*Wi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedDecrement64,   "WiWiD*",   "nh", "intrin.h", ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchange64,    "WiWiD*Wi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangeAdd64, "WiWiD*Wi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangeSub64, "WiWiD*Wi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedIncrement64,   "WiWiD*",   "nh", "intrin.h", ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedOr64,          "WiWiD*Wi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedXor64,         "WiWiD*Wi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
+
 #undef BUILTIN
 #undef TARGET_BUILTIN
 #undef TARGET_HEADER_BUILTIN
diff --git a/clang/include/clang/Basic/BuiltinsX86_64.def b/clang/include/clang/Basic/BuiltinsX86_64.def
index c535f43203e56..f66ae78f7e81f 100644
--- a/clang/include/clang/Basic/BuiltinsX86_64.def
+++ b/clang/include/clang/Basic/BuiltinsX86_64.def
@@ -33,14 +33,6 @@ TARGET_HEADER_BUILTIN(__faststorefence, "v", "nh", "intrin.h", ALL_MS_LANGUAGES,
 TARGET_HEADER_BUILTIN(__shiftleft128, "ULLiULLiULLiUc", "nch", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(__shiftright128, "ULLiULLiULLiUc", "nch", "intrin.h", ALL_MS_LANGUAGES, "")
 
-TARGET_HEADER_BUILTIN(_InterlockedAnd64,         "LLiLLiD*LLi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
-TARGET_HEADER_BUILTIN(_InterlockedDecrement64,   "LLiLLiD*",    "nh", "intrin.h", ALL_MS_LANGUAGES, "")
-TARGET_HEADER_BUILTIN(_InterlockedExchange64,    "LLiLLiD*LLi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
-TARGET_HEADER_BUILTIN(_InterlockedExchangeAdd64, "LLiLLiD*LLi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
-TARGET_HEADER_BUILTIN(_InterlockedExchangeSub64, "LLiLLiD*LLi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
-TARGET_HEADER_BUILTIN(_InterlockedIncrement64,   "LLiLLiD*",    "nh", "intrin.h", ALL_MS_LANGUAGES, "")
-TARGET_HEADER_BUILTIN(_InterlockedOr64,          "LLiLLiD*LLi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
-TARGET_HEADER_BUILTIN(_InterlockedXor64,         "LLiLLiD*LLi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedCompareExchange128, "UcLLiD*LLiLLiLLi*", "nh", "intrin.h", ALL_MS_LANGUAGES, "cx16")
 
 TARGET_BUILTIN(__builtin_ia32_readeflags_u64, "UOi", "n", "")
@@ -101,6 +93,22 @@ TARGET_BUILTIN(__builtin_ia32_cvtsi2ss64, "V4fV4fOiIi", "ncV:128:", "avx512f")
 TARGET_BUILTIN(__builtin_ia32_cvtusi2sd64, "V2dV2dUOiIi", "ncV:128:", "avx512f")
 TARGET_BUILTIN(__builtin_ia32_cvtusi2ss64, "V4fV4fUOiIi", "ncV:128:", "avx512f")
 TARGET_BUILTIN(__builtin_ia32_directstore_u64, "vULi*ULi", "n", "movdiri")
+
+// AMX
+TARGET_BUILTIN(__builtin_ia32_tile_loadconfig, "vvC*", "n", "amx-tile")
+TARGET_BUILTIN(__builtin_ia32_tile_storeconfig, "vvC*", "n", "amx-tile")
+TARGET_BUILTIN(__builtin_ia32_tilerelease, "v", "n", "amx-tile")
+TARGET_BUILTIN(__builtin_ia32_tilezero, "vUc", "n", "amx-tile")
+
+TARGET_BUILTIN(__builtin_ia32_tileloadd64, "vIUcvC*z", "n", "amx-tile")
+TARGET_BUILTIN(__builtin_ia32_tileloaddt164, "vIUcvC*z", "n", "amx-tile")
+TARGET_BUILTIN(__builtin_ia32_tilestored64, "vIUcv*z", "n", "amx-tile")
+
+TARGET_BUILTIN(__builtin_ia32_tdpbssd, "vIUcIUcIUc", "n", "amx-int8")
+TARGET_BUILTIN(__builtin_ia32_tdpbsud, "vIUcIUcIUc", "n", "amx-int8")
+TARGET_BUILTIN(__builtin_ia32_tdpbusd, "vIUcIUcIUc", "n", "amx-int8")
+TARGET_BUILTIN(__builtin_ia32_tdpbuud, "vIUcIUcIUc", "n", "amx-int8")
+TARGET_BUILTIN(__builtin_ia32_tdpbf16ps, "vIUcIUcIUc", "n", "amx-bf16")
 TARGET_BUILTIN(__builtin_ia32_ptwrite64, "vUOi", "n", "ptwrite")
 
 #undef BUILTIN
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index d465e00d4c70f..c7e01eb128518 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -58,6 +58,8 @@ CODEGENOPT(DisableLLVMPasses , 1, 0) ///< Don't run any LLVM IR passes to get
                                      ///< frontend.
 CODEGENOPT(DisableLifetimeMarkers, 1, 0) ///< Don't emit any lifetime markers
 CODEGENOPT(DisableO0ImplyOptNone , 1, 0) ///< Don't annonate function with optnone at O0
+CODEGENOPT(ExperimentalStrictFloatingPoint, 1, 0) ///< Enables the new, experimental
+                                                  ///< strict floating point.
 CODEGENOPT(ExperimentalNewPassManager, 1, 0) ///< Enables the new, experimental
                                              ///< pass manager.
 CODEGENOPT(DebugPassManager, 1, 0) ///< Prints debug information for the new
@@ -252,7 +254,6 @@ CODEGENOPT(UnwindTables      , 1, 0) ///< Emit unwind tables.
 CODEGENOPT(VectorizeLoop     , 1, 0) ///< Run loop vectorizer.
 CODEGENOPT(VectorizeSLP      , 1, 0) ///< Run SLP vectorizer.
 CODEGENOPT(ProfileSampleAccurate, 1, 0) ///< Sample profile is accurate.
-CODEGENOPT(CallGraphProfile  , 1, 0) ///< Run call graph profile.
 
   /// Attempt to use register sized accesses to bit-fields in structures, when
   /// possible.
diff --git a/clang/include/clang/Basic/DiagnosticASTKinds.td b/clang/include/clang/Basic/DiagnosticASTKinds.td
index 75c4d0e0a139e..10bedaaf7aba7 100644
--- a/clang/include/clang/Basic/DiagnosticASTKinds.td
+++ b/clang/include/clang/Basic/DiagnosticASTKinds.td
@@ -101,8 +101,16 @@ def note_constexpr_null_subobject : Note<
   "access array element of|perform pointer arithmetic on|"
   "access real component of|"
   "access imaginary component of}0 null pointer">;
+def note_constexpr_function_param_value_unknown : Note<
+  "function parameter %0 with unknown value cannot be used in a constant "
+  "expression">;
+def note_constexpr_var_init_unknown : Note<
+  "initializer of %0 is unknown">;
 def note_constexpr_var_init_non_constant : Note<
   "initializer of %0 is not a constant expression">;
+def note_constexpr_var_init_weak : Note<
+  "initializer of weak variable %0 is not considered constant because "
+  "it may be different at runtime">;
 def note_constexpr_typeid_polymorphic : Note<
   "typeid applied to expression of polymorphic type %0 is "
   "not allowed in a constant expression in C++ standards before C++20">;
@@ -159,6 +167,9 @@ def note_constexpr_access_mutable : Note<
   "mutable member %1 is not allowed in a constant expression">;
 def note_constexpr_ltor_non_const_int : Note<
   "read of non-const variable %0 is not allowed in a constant expression">;
+def note_constexpr_ltor_non_integral : Note<
+  "read of variable %0 of non-integral, non-enumeration type %1 "
+  "is not allowed in a constant expression">;
 def note_constexpr_ltor_non_constexpr : Note<
   "read of non-constexpr variable %0 is not allowed in a constant expression">;
 def note_constexpr_ltor_incomplete_type : Note<
@@ -346,6 +357,9 @@ def err_experimental_clang_interp_failed : Error<
 def warn_integer_constant_overflow : Warning<
   "overflow in expression; result is %0 with type %1">,
   InGroup<DiagGroup<"integer-overflow">>;
+def warn_fixedpoint_constant_overflow : Warning<
+  "overflow in expression; result is %0 with type %1">,
+  InGroup<DiagGroup<"fixed-point-overflow">>;
 
 // This is a temporary diagnostic, and shall be removed once our
 // implementation is complete, and like the preceding constexpr notes belongs
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index dcb3b96cd057e..558639ecad6aa 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -56,12 +56,12 @@ def err_drv_no_cuda_libdevice : Error<
   "cannot find libdevice for %0. Provide path to different CUDA installation "
   "via --cuda-path, or pass -nocudalib to build without linking with libdevice.">;
 
-def err_drv_no_rocm_installation : Error<
-  "cannot find ROCm installation.  Provide its path via --rocm-path, or pass "
-  "-nogpulib and -nogpuinc to build without ROCm device library and HIP includes.">;
 def err_drv_no_rocm_device_lib : Error<
-  "cannot find device library for %0. Provide path to different ROCm installation "
-  "via --rocm-path, or pass -nogpulib to build without linking default libraries.">;
+  "cannot find ROCm device library%select{| for %1}0. Provide its path via --rocm-path or "
+  "--rocm-device-lib-path, or pass -nogpulib to build without ROCm device library.">;
+def err_drv_no_hip_runtime : Error<
+  "cannot find HIP runtime. Provide its path via --rocm-path, or pass "
+  "-nogpuinc to build without HIP runtime.">;
 
 def err_drv_cuda_version_unsupported : Error<
   "GPU arch %0 is supported by CUDA versions between %1 and %2 (inclusive), "
@@ -509,4 +509,6 @@ def warn_drv_libstdcxx_not_found : Warning<
   InGroup<DiagGroup<"stdlibcxx-not-found">>;
 
 def err_drv_cannot_mix_options : Error<"cannot specify '%1' along with '%0'">;
+
+def err_drv_invalid_object_mode : Error<"OBJECT_MODE setting %0 is not recognized and is not a valid setting.">;
 }
diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
index 687c60c9c5367..b202d2abffa00 100644
--- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td
+++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
@@ -37,6 +37,12 @@ def note_fe_backend_plugin: Note<"%0">, BackendInfo;
 def warn_fe_override_module : Warning<
     "overriding the module target triple with %0">,
     InGroup<DiagGroup<"override-module">>;
+def warn_fe_backend_unsupported_fp_rounding : Warning<
+    "overriding currently unsupported rounding mode on this target">,
+    InGroup<UnsupportedFPOpt>;
+def warn_fe_backend_unsupported_fp_exceptions : Warning<
+    "overriding currently unsupported use of floating point exceptions "
+    "on this target">, InGroup<UnsupportedFPOpt>;
 
 def remark_fe_backend_optimization_remark : Remark<"%0">, BackendInfo,
     InGroup<BackendOptimizationRemark>;
@@ -119,6 +125,9 @@ def warn_fe_serialized_diag_merge_failure : Warning<
 def warn_fe_serialized_diag_failure : Warning<
     "unable to open file %0 for serializing diagnostics (%1)">,
     InGroup<SerializedDiagnostics>;
+def warn_fe_serialized_diag_failure_during_finalisation : Warning<
+    "Received warning after diagnostic serialization teardown was underway: %0">,
+    InGroup<SerializedDiagnostics>;
 
 def err_verify_missing_line : Error<
     "missing or invalid line number following '@' in expected %0">;
@@ -240,6 +249,12 @@ def err_function_needs_feature : Error<
   "always_inline function %1 requires target feature '%2', but would "
   "be inlined into function %0 that is compiled without support for '%2'">;
 
+def warn_avx_calling_convention
+    : Warning<"AVX vector %select{return|argument}0 of type %1 without '%2' "
+              "enabled changes the ABI">,
+      InGroup<DiagGroup<"psabi">>;
+def err_avx_calling_convention : Error<warn_avx_calling_convention.Text>;
+
 def err_alias_to_undefined : Error<
   "%select{alias|ifunc}0 must point to a defined "
   "%select{variable or |}1function">;
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 23f530dd4e1fc..1e829be4028e4 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -107,6 +107,7 @@ def DoublePromotion : DiagGroup<"double-promotion">;
 def EnumTooLarge : DiagGroup<"enum-too-large">;
 def UnsupportedNan : DiagGroup<"unsupported-nan">;
 def UnsupportedAbs : DiagGroup<"unsupported-abs">;
+def UnsupportedFPOpt : DiagGroup<"unsupported-floating-point-opt">;
 def UnsupportedCB : DiagGroup<"unsupported-cb">;
 def UnsupportedGPOpt : DiagGroup<"unsupported-gpopt">;
 def UnsupportedTargetOpt : DiagGroup<"unsupported-target-opt">;
@@ -279,9 +280,12 @@ def CXX98CompatPedantic : DiagGroup<"c++98-compat-pedantic",
 
 def CXX11Narrowing : DiagGroup<"c++11-narrowing">;
 
-def CXX11WarnOverrideDestructor :
+def CXX11WarnInconsistentOverrideDestructor :
   DiagGroup<"inconsistent-missing-destructor-override">;
-def CXX11WarnOverrideMethod : DiagGroup<"inconsistent-missing-override">;
+def CXX11WarnInconsistentOverrideMethod :
+  DiagGroup<"inconsistent-missing-override">;
+def CXX11WarnSuggestOverrideDestructor : DiagGroup<"suggest-destructor-override">;
+def CXX11WarnSuggestOverride : DiagGroup<"suggest-override">;
 
 // Original name of this warning in Clang
 def : DiagGroup<"c++0x-narrowing", [CXX11Narrowing]>;
@@ -382,6 +386,8 @@ def IncompleteModule : DiagGroup<"incomplete-module",
 def PrivateModule : DiagGroup<"private-module">;
 
 def CXX11InlineNamespace : DiagGroup<"c++11-inline-namespace">;
+def InlineNamespaceReopenedNoninline
+    : DiagGroup<"inline-namespace-reopened-noninline">;
 def InvalidNoreturn : DiagGroup<"invalid-noreturn">;
 def InvalidSourceEncoding : DiagGroup<"invalid-source-encoding">;
 def KNRPromotedParameter : DiagGroup<"knr-promoted-parameter">;
@@ -1097,6 +1103,11 @@ def ObjCSignedCharBool : DiagGroup<"objc-signed-char-bool",
    ObjCBoolConstantConversion,
    TautologicalObjCBoolCompare]>;
 
+def ObjCPotentiallyDirectSelector : DiagGroup<"potentially-direct-selector">;
+def ObjCStrictPotentiallyDirectSelector :
+  DiagGroup<"strict-potentially-direct-selector",
+            [ObjCPotentiallyDirectSelector]>;
+
 // Inline ASM warnings.
 def ASMOperandWidths : DiagGroup<"asm-operand-widths">;
 def ASM : DiagGroup<"asm", [
diff --git a/clang/include/clang/Basic/DiagnosticIDs.h b/clang/include/clang/Basic/DiagnosticIDs.h
index 46f0fa423a391..00c939650e549 100644
--- a/clang/include/clang/Basic/DiagnosticIDs.h
+++ b/clang/include/clang/Basic/DiagnosticIDs.h
@@ -33,7 +33,7 @@ namespace clang {
       DIAG_SIZE_SERIALIZATION =  120,
       DIAG_SIZE_LEX           =  400,
       DIAG_SIZE_PARSE         =  600,
-      DIAG_SIZE_AST           =  200,
+      DIAG_SIZE_AST           =  250,
       DIAG_SIZE_COMMENT       =  100,
       DIAG_SIZE_CROSSTU       =  100,
       DIAG_SIZE_SEMA          = 4000,
diff --git a/clang/include/clang/Basic/DiagnosticLexKinds.td b/clang/include/clang/Basic/DiagnosticLexKinds.td
index fa07e9ae76c85..9cb06cf5b5e11 100644
--- a/clang/include/clang/Basic/DiagnosticLexKinds.td
+++ b/clang/include/clang/Basic/DiagnosticLexKinds.td
@@ -312,6 +312,9 @@ def pp_macro_not_used : Warning<"macro is not used">, DefaultIgnore,
 def warn_pp_undef_identifier : Warning<
   "%0 is not defined, evaluates to 0">,
   InGroup<DiagGroup<"undef">>, DefaultIgnore;
+def warn_pp_undef_prefix : Warning<
+  "%0 is not defined, evaluates to 0">,
+  InGroup<DiagGroup<"undef-prefix">>, DefaultIgnore;
 def warn_pp_ambiguous_macro : Warning<
   "ambiguous expansion of macro %0">, InGroup<AmbiguousMacro>;
 def note_pp_ambiguous_macro_chosen : Note<
diff --git a/clang/include/clang/Basic/DiagnosticOptions.def b/clang/include/clang/Basic/DiagnosticOptions.def
index 6d1a1af92821b..a946b5c6be8ef 100644
--- a/clang/include/clang/Basic/DiagnosticOptions.def
+++ b/clang/include/clang/Basic/DiagnosticOptions.def
@@ -65,6 +65,7 @@ VALUE_DIAGOPT(ShowCategories, 2, 0) /// Show categories: 0 -> none, 1 -> Number,
 ENUM_DIAGOPT(Format, TextDiagnosticFormat, 2, Clang) /// Format for diagnostics:
 
 DIAGOPT(ShowColors, 1, 0)       /// Show diagnostics with ANSI color sequences.
+DIAGOPT(UseANSIEscapeCodes, 1, 0)
 ENUM_DIAGOPT(ShowOverloads, OverloadsShown, 1,
              Ovl_All)    /// Overload candidates to show.
 DIAGOPT(VerifyDiagnostics, 1, 0) /// Check that diagnostics match the expected
diff --git a/clang/include/clang/Basic/DiagnosticOptions.h b/clang/include/clang/Basic/DiagnosticOptions.h
index 3e3c4e50a9e0f..7fbe534c5994b 100644
--- a/clang/include/clang/Basic/DiagnosticOptions.h
+++ b/clang/include/clang/Basic/DiagnosticOptions.h
@@ -98,6 +98,10 @@ class DiagnosticOptions : public RefCountedBase<DiagnosticOptions>{
   /// prefixes removed.
   std::vector<std::string> Warnings;
 
+  /// The list of prefixes from -Wundef-prefix=... used to generate warnings
+  /// for undefined macros.
+  std::vector<std::string> UndefPrefixes;
+
   /// The list of -R... options used to alter the diagnostic mappings, with the
   /// prefixes removed.
   std::vector<std::string> Remarks;
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 5d57cfd6e71d1..1038a4119d4cb 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -194,6 +194,8 @@ def err_function_declared_typedef : Error<
 def err_at_defs_cxx : Error<"@defs is not supported in Objective-C++">;
 def err_at_in_class : Error<"unexpected '@' in member specification">;
 def err_unexpected_semi : Error<"unexpected ';' before %0">;
+def err_postfix_after_unary_requires_parens : Error<
+  "expression cannot be followed by a postfix %0 operator; add parentheses">;
 def err_unparenthesized_non_primary_expr_in_requires_clause : Error<
   "parentheses are required around this expression in a requires clause">;
 def note_unparenthesized_non_primary_expr_in_requires_clause : Note<
@@ -1332,6 +1334,8 @@ def warn_omp_more_one_device_type_clause
       InGroup<OpenMPClauses>;
 def err_omp_variant_ctx_second_match_extension : Error<
   "only a single match extension allowed per OpenMP context selector">;
+def err_omp_invalid_dsa: Error<
+  "data-sharing attribute '%0' in '%1' clause requires OpenMP version %2 or above">;
 
 // Pragma loop support.
 def err_pragma_loop_missing_argument : Error<
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 7c72eba8c2c11..71517edd6659b 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -259,6 +259,9 @@ def err_invalid_vector_float_decl_spec : Error<
 def err_invalid_vector_double_decl_spec : Error <
   "use of 'double' with '__vector' requires VSX support to be enabled "
   "(available on POWER7 or later)">;
+def err_invalid_vector_bool_int128_decl_spec : Error <
+  "use of '__int128' with '__vector bool' requires VSX support enabled (on "
+  "POWER10 or later)">;
 def err_invalid_vector_long_long_decl_spec : Error <
   "use of 'long long' with '__vector bool' requires VSX support (available on "
   "POWER7 or later) or extended Altivec support (available on POWER8 or later) "
@@ -827,8 +830,8 @@ def err_opencl_half_load_store : Error<
 def err_opencl_cast_to_half : Error<"casting to type %0 is not allowed">;
 def err_opencl_half_declaration : Error<
   "declaring variable of type %0 is not allowed">;
-def err_opencl_half_param : Error<
-  "declaring function parameter of type %0 is not allowed; did you forget * ?">;
+def err_opencl_invalid_param : Error<
+  "declaring function parameter of type %0 is not allowed%select{; did you forget * ?|}1">;
 def err_opencl_invalid_return : Error<
   "declaring function return value of type %0 is not allowed %select{; did you forget * ?|}1">;
 def warn_enum_value_overflow : Warning<"overflow in enumeration value">;
@@ -1375,8 +1378,14 @@ def warn_multiple_selectors: Warning<
   "several methods with selector %0 of mismatched types are found "
   "for the @selector expression">,
   InGroup<SelectorTypeMismatch>, DefaultIgnore;
-def err_direct_selector_expression: Error<
+def err_direct_selector_expression : Error<
   "@selector expression formed with direct selector %0">;
+def warn_potentially_direct_selector_expression : Warning<
+  "@selector expression formed with potentially direct selector %0">,
+  InGroup<ObjCPotentiallyDirectSelector>;
+def warn_strict_potentially_direct_selector_expression : Warning<
+  warn_potentially_direct_selector_expression.Text>,
+  InGroup<ObjCStrictPotentiallyDirectSelector>, DefaultIgnore;
 
 def err_objc_kindof_nonobject : Error<
   "'__kindof' specifier cannot be applied to non-object type %0">;
@@ -1411,7 +1420,8 @@ def warn_cxx14_compat_inline_variable : Warning<
   DefaultIgnore, InGroup<CXXPre17Compat>;
 
 def warn_inline_namespace_reopened_noninline : Warning<
-  "inline namespace reopened as a non-inline namespace">;
+  "inline namespace reopened as a non-inline namespace">,
+  InGroup<InlineNamespaceReopenedNoninline>;
 def err_inline_namespace_mismatch : Error<
   "non-inline namespace cannot be reopened as inline">;
 
@@ -2357,12 +2367,22 @@ def override_keyword_hides_virtual_member_function : Error<
   "%select{function|functions}1">;
 def err_function_marked_override_not_overriding : Error<
   "%0 marked 'override' but does not override any member functions">;
-def warn_destructor_marked_not_override_overriding : Warning <
-  "%0 overrides a destructor but is not marked 'override'">,
-  InGroup<CXX11WarnOverrideDestructor>, DefaultIgnore;
-def warn_function_marked_not_override_overriding : Warning <
-  "%0 overrides a member function but is not marked 'override'">,
-  InGroup<CXX11WarnOverrideMethod>;
+def warn_destructor_marked_not_override_overriding : TextSubstitution <
+  "%0 overrides a destructor but is not marked 'override'">;
+def warn_function_marked_not_override_overriding : TextSubstitution <
+  "%0 overrides a member function but is not marked 'override'">;
+def warn_inconsistent_destructor_marked_not_override_overriding : Warning <
+  "%sub{warn_destructor_marked_not_override_overriding}0">,
+  InGroup<CXX11WarnInconsistentOverrideDestructor>, DefaultIgnore;
+def warn_inconsistent_function_marked_not_override_overriding : Warning <
+  "%sub{warn_function_marked_not_override_overriding}0">,
+  InGroup<CXX11WarnInconsistentOverrideMethod>;
+def warn_suggest_destructor_marked_not_override_overriding : Warning <
+  "%sub{warn_destructor_marked_not_override_overriding}0">,
+  InGroup<CXX11WarnSuggestOverrideDestructor>, DefaultIgnore;
+def warn_suggest_function_marked_not_override_overriding : Warning <
+  "%sub{warn_function_marked_not_override_overriding}0">,
+  InGroup<CXX11WarnSuggestOverride>, DefaultIgnore;
 def err_class_marked_final_used_as_base : Error<
   "base %0 is marked '%select{final|sealed}1'">;
 def warn_abstract_final_class : Warning<
@@ -5385,9 +5405,6 @@ def ext_typecheck_zero_array_size : Extension<
   "zero size arrays are an extension">, InGroup<ZeroLengthArray>;
 def err_typecheck_zero_array_size : Error<
   "zero-length arrays are not permitted in C++">;
-def warn_typecheck_zero_static_array_size : Warning<
-  "'static' has no effect on zero-length arrays">,
-  InGroup<ArrayBounds>;
 def err_array_size_non_int : Error<"size of array has non-integer type %0">;
 def err_init_element_not_constant : Error<
   "initializer element is not a compile-time constant">;
@@ -7989,6 +8006,10 @@ def err_ref_bad_target : Error<
 def err_ref_bad_target_global_initializer : Error<
   "reference to %select{__device__|__global__|__host__|__host__ __device__}0 "
   "function %1 in global initializer">;
+def err_capture_bad_target : Error<
+  "capture host variable %0 by reference in device or host device lambda function">;
+def err_capture_bad_target_this_ptr : Error<
+  "capture host side class data member by this pointer in device or host device lambda function">;
 def warn_kern_is_method : Extension<
   "kernel function %0 is a member function; this may not be accepted by nvcc">,
   InGroup<CudaCompat>;
@@ -9338,6 +9359,8 @@ def err_x86_builtin_invalid_rounding : Error<
   "invalid rounding argument">;
 def err_x86_builtin_invalid_scale : Error<
   "scale argument must be 1, 2, 4, or 8">;
+def err_x86_builtin_tile_arg_duplicate : Error<
+  "tile arguments must refer to different tiles">;
 
 def err_builtin_target_unsupported : Error<
   "builtin is not supported on this target">;
@@ -10048,6 +10071,8 @@ def err_omp_section_not_subset_of_array : Error<
   "array section must be a subset of the original array">;
 def err_omp_section_length_negative : Error<
   "section length is evaluated to a negative value %0">;
+def err_omp_section_stride_non_positive : Error<
+  "section stride is evaluated to a non-positive value %0">;
 def err_omp_section_length_undefined : Error<
   "section length is unspecified and cannot be inferred because subscripted value is %select{not an array|an array of unknown bound}0">;
 def err_omp_wrong_linear_modifier : Error<
@@ -10523,7 +10548,16 @@ def err_await_suspend_invalid_return_type : Error<
 def note_await_ready_no_bool_conversion : Note<
   "return type of 'await_ready' is required to be contextually convertible to 'bool'"
 >;
-}
+def warn_coroutine_handle_address_invalid_return_type : Warning <
+  "return type of 'coroutine_handle<>::address should be 'void*' (have %0) in order to get capability with existing async C API.">,
+  InGroup<Coroutine>;
+def err_coroutine_promise_final_suspend_requires_nothrow : Error<
+  "the expression 'co_await __promise.final_suspend()' is required to be non-throwing"
+>;
+def note_coroutine_function_declare_noexcept : Note<
+  "must be declared with 'noexcept'"
+>;
+} // end of coroutines issue category
 
 let CategoryName = "Documentation Issue" in {
 def warn_not_a_doxygen_trailing_member_comment : Warning<
@@ -10780,35 +10814,28 @@ def err_matrix_separate_incomplete_index: Error<
   "matrix row and column subscripts cannot be separated by any expression">;
 def err_matrix_subscript_comma: Error<
   "comma expressions are not allowed as indices in matrix subscript expressions">;
-
-def warn_mismatched_import : Warning<
-  "import %select{module|name}0 (%1) does not match the import %select{module|name}0 (%2) of the "
-  "previous declaration">,
-  InGroup<IgnoredAttributes>;
-def warn_import_on_definition : Warning<
-  "import %select{module|name}0 cannot be applied to a function with a definition">,
-  InGroup<IgnoredAttributes>;
-
-def err_builtin_matrix_arg: Error<"first argument must be a matrix">;
-
+def err_builtin_matrix_arg: Error<"1st argument must be a matrix">;
 def err_builtin_matrix_scalar_unsigned_arg: Error<
   "%0 argument must be a constant unsigned integer expression">;
-
 def err_builtin_matrix_pointer_arg: Error<
-  "%0 argument must be a pointer to a valid matrix element type">;
-
+  "%ordinal0 argument must be a pointer to a valid matrix element type">;
 def err_builtin_matrix_pointer_arg_mismatch: Error<
-  "the pointee of the second argument must match the element type of the first argument (%0 != %1)">;
-
+  "the pointee of the 2nd argument must match the element type of the 1st argument (%0 != %1)">;
 def err_builtin_matrix_store_to_const: Error<
   "cannot store matrix to read-only pointer">;
-
 def err_builtin_matrix_stride_too_small: Error<
   "stride must be greater or equal to the number of rows">;
-
 def err_builtin_matrix_invalid_dimension: Error<
   "%0 dimension is outside the allowed range [1, %1]">;
 
+def warn_mismatched_import : Warning<
+  "import %select{module|name}0 (%1) does not match the import %select{module|name}0 (%2) of the "
+  "previous declaration">,
+  InGroup<IgnoredAttributes>;
+def warn_import_on_definition : Warning<
+  "import %select{module|name}0 cannot be applied to a function with a definition">,
+  InGroup<IgnoredAttributes>;
+
 def err_preserve_field_info_not_field : Error<
   "__builtin_preserve_field_info argument %0 not a field access">;
 def err_preserve_field_info_not_const: Error<
@@ -10839,4 +10866,12 @@ def err_ext_int_bad_size : Error<"%select{signed|unsigned}0 _ExtInt must "
                                  "have a bit size of at least %select{2|1}0">;
 def err_ext_int_max_size : Error<"%select{signed|unsigned}0 _ExtInt of bit "
                                  "sizes greater than %1 not supported">;
+
+// errors of expect.with.probability
+def err_probability_not_constant_float : Error<
+   "probability argument to __builtin_expect_with_probability must be constant "
+   "floating-point expression">;
+def err_probability_out_of_range : Error<
+   "probability argument to __builtin_expect_with_probability is outside the "
+   "range [0.0, 1.0]">;
 } // end of sema component.
diff --git a/clang/include/clang/Basic/FPOptions.def b/clang/include/clang/Basic/FPOptions.def
new file mode 100644
index 0000000000000..6b6789b8ecc87
--- /dev/null
+++ b/clang/include/clang/Basic/FPOptions.def
@@ -0,0 +1,26 @@
+//===--- FPOptions.def - Floating Point Options database --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// This file defines the Floating Point language options. Users of this file
+//  must define the FPOPT macro to make use of this information.
+#ifndef OPTION
+#  error Define the OPTION macro to handle floating point language options
+#endif
+
+// OPTION(name, type, width, previousName)
+OPTION(FPContractMode, LangOptions::FPModeKind, 2, First)
+OPTION(RoundingMode, RoundingMode, 3, FPContractMode)
+OPTION(FPExceptionMode, LangOptions::FPExceptionModeKind, 2, RoundingMode)
+OPTION(AllowFEnvAccess, bool, 1, FPExceptionMode)
+OPTION(AllowFPReassociate, bool, 1, AllowFEnvAccess)
+OPTION(NoHonorNaNs, bool, 1, AllowFPReassociate)
+OPTION(NoHonorInfs, bool, 1, NoHonorNaNs)
+OPTION(NoSignedZero, bool, 1, NoHonorInfs)
+OPTION(AllowReciprocal, bool, 1, NoSignedZero)
+OPTION(AllowApproxFunc, bool, 1, AllowReciprocal)
+#undef OPTION
diff --git a/clang/include/clang/Basic/FixedPoint.h b/clang/include/clang/Basic/FixedPoint.h
index 55465c604a7dc..0d181f30907fe 100644
--- a/clang/include/clang/Basic/FixedPoint.h
+++ b/clang/include/clang/Basic/FixedPoint.h
@@ -28,8 +28,8 @@ class QualType;
 /// The fixed point semantics work similarly to llvm::fltSemantics. The width
 /// specifies the whole bit width of the underlying scaled integer (with padding
 /// if any). The scale represents the number of fractional bits in this type.
-/// When HasUnsignedPadding is true and this type is signed, the first bit
-/// in the value this represents is treaded as padding.
+/// When HasUnsignedPadding is true and this type is unsigned, the first bit
+/// in the value this represents is treated as padding.
 class FixedPointSemantics {
 public:
   FixedPointSemantics(unsigned Width, unsigned Scale, bool IsSigned,
@@ -93,49 +93,52 @@ class FixedPointSemantics {
 /// point types and should eventually be moved to LLVM if fixed point types gain
 /// native IR support.
 class APFixedPoint {
- public:
-   APFixedPoint(const llvm::APInt &Val, const FixedPointSemantics &Sema)
-       : Val(Val, !Sema.isSigned()), Sema(Sema) {
-     assert(Val.getBitWidth() == Sema.getWidth() &&
-            "The value should have a bit width that matches the Sema width");
-   }
-
-   APFixedPoint(uint64_t Val, const FixedPointSemantics &Sema)
-       : APFixedPoint(llvm::APInt(Sema.getWidth(), Val, Sema.isSigned()),
-                      Sema) {}
-
-   // Zero initialization.
-   APFixedPoint(const FixedPointSemantics &Sema) : APFixedPoint(0, Sema) {}
-
-   llvm::APSInt getValue() const { return llvm::APSInt(Val, !Sema.isSigned()); }
-   inline unsigned getWidth() const { return Sema.getWidth(); }
-   inline unsigned getScale() const { return Sema.getScale(); }
-   inline bool isSaturated() const { return Sema.isSaturated(); }
-   inline bool isSigned() const { return Sema.isSigned(); }
-   inline bool hasPadding() const { return Sema.hasUnsignedPadding(); }
-   FixedPointSemantics getSemantics() const { return Sema; }
-
-   bool getBoolValue() const { return Val.getBoolValue(); }
-
-   // Convert this number to match the semantics provided. If the overflow
-   // parameter is provided, set this value to true or false to indicate if this
-   // operation results in an overflow.
-   APFixedPoint convert(const FixedPointSemantics &DstSema,
-                        bool *Overflow = nullptr) const;
-
-   // Perform binary operations on a fixed point type. The resulting fixed point
-   // value will be in the common, full precision semantics that can represent
-   // the precision and ranges os both input values. See convert() for an
-   // explanation of the Overflow parameter.
-   APFixedPoint add(const APFixedPoint &Other, bool *Overflow = nullptr) const;
-
-   /// Perform a unary negation (-X) on this fixed point type, taking into
-   /// account saturation if applicable.
-   APFixedPoint negate(bool *Overflow = nullptr) const;
-
-   APFixedPoint shr(unsigned Amt) const {
-     return APFixedPoint(Val >> Amt, Sema);
-   }
+public:
+  APFixedPoint(const llvm::APInt &Val, const FixedPointSemantics &Sema)
+      : Val(Val, !Sema.isSigned()), Sema(Sema) {
+    assert(Val.getBitWidth() == Sema.getWidth() &&
+           "The value should have a bit width that matches the Sema width");
+  }
+
+  APFixedPoint(uint64_t Val, const FixedPointSemantics &Sema)
+      : APFixedPoint(llvm::APInt(Sema.getWidth(), Val, Sema.isSigned()),
+                     Sema) {}
+
+  // Zero initialization.
+  APFixedPoint(const FixedPointSemantics &Sema) : APFixedPoint(0, Sema) {}
+
+  llvm::APSInt getValue() const { return llvm::APSInt(Val, !Sema.isSigned()); }
+  inline unsigned getWidth() const { return Sema.getWidth(); }
+  inline unsigned getScale() const { return Sema.getScale(); }
+  inline bool isSaturated() const { return Sema.isSaturated(); }
+  inline bool isSigned() const { return Sema.isSigned(); }
+  inline bool hasPadding() const { return Sema.hasUnsignedPadding(); }
+  FixedPointSemantics getSemantics() const { return Sema; }
+
+  bool getBoolValue() const { return Val.getBoolValue(); }
+
+  // Convert this number to match the semantics provided. If the overflow
+  // parameter is provided, set this value to true or false to indicate if this
+  // operation results in an overflow.
+  APFixedPoint convert(const FixedPointSemantics &DstSema,
+                       bool *Overflow = nullptr) const;
+
+  // Perform binary operations on a fixed point type. The resulting fixed point
+  // value will be in the common, full precision semantics that can represent
+  // the precision and ranges of both input values. See convert() for an
+  // explanation of the Overflow parameter.
+  APFixedPoint add(const APFixedPoint &Other, bool *Overflow = nullptr) const;
+  APFixedPoint sub(const APFixedPoint &Other, bool *Overflow = nullptr) const;
+  APFixedPoint mul(const APFixedPoint &Other, bool *Overflow = nullptr) const;
+  APFixedPoint div(const APFixedPoint &Other, bool *Overflow = nullptr) const;
+
+  /// Perform a unary negation (-X) on this fixed point type, taking into
+  /// account saturation if applicable.
+  APFixedPoint negate(bool *Overflow = nullptr) const;
+
+  APFixedPoint shr(unsigned Amt) const {
+    return APFixedPoint(Val >> Amt, Sema);
+  }
 
   APFixedPoint shl(unsigned Amt) const {
     return APFixedPoint(Val << Amt, Sema);
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index 31849bbdd5456..fc554a35e721b 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -48,7 +48,7 @@ using IdentifierLocPair = std::pair<IdentifierInfo *, SourceLocation>;
 /// of a pointer to one of these classes.
 enum { IdentifierInfoAlignment = 8 };
 
-static constexpr int ObjCOrBuiltinIDBits = 14;
+static constexpr int ObjCOrBuiltinIDBits = 15;
 
 /// One of these records is kept for each identifier that
 /// is lexed.  This contains information about whether the token was \#define'd,
diff --git a/clang/include/clang/Basic/JsonSupport.h b/clang/include/clang/Basic/JsonSupport.h
index 1b118b45a0498..8b02e440df447 100644
--- a/clang/include/clang/Basic/JsonSupport.h
+++ b/clang/include/clang/Basic/JsonSupport.h
@@ -13,7 +13,7 @@
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/raw_ostream.h"
-
+#include <iterator>
 
 namespace clang {
 
@@ -98,7 +98,16 @@ inline void printSourceLocationAsJson(raw_ostream &Out, SourceLocation Loc,
     if (AddBraces)
       Out << "{ ";
     std::string filename(PLoc.getFilename());
-#ifdef _WIN32 // Handle windows-specific path delimiters.
+#ifdef _WIN32
+    // Remove forbidden Windows path characters
+    auto RemoveIt =
+        std::remove_if(filename.begin(), filename.end(), [](auto Char) {
+          static const char ForbiddenChars[] = "<>*?\"|";
+          return std::find(std::begin(ForbiddenChars), std::end(ForbiddenChars),
+                           Char) != std::end(ForbiddenChars);
+        });
+    filename.erase(RemoveIt, filename.end());
+    // Handle windows-specific path delimiters.
     std::replace(filename.begin(), filename.end(), '\\', '/');
 #endif
     Out << "\"line\": " << PLoc.getLine()
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index bc0c17a147891..70f68d664bb71 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -193,12 +193,12 @@ COMPATIBLE_LANGOPT(Deprecated        , 1, 0, "__DEPRECATED predefined macro")
 COMPATIBLE_LANGOPT(FastMath          , 1, 0, "fast FP math optimizations, and __FAST_MATH__ predefined macro")
 COMPATIBLE_LANGOPT(FiniteMathOnly    , 1, 0, "__FINITE_MATH_ONLY__ predefined macro")
 COMPATIBLE_LANGOPT(UnsafeFPMath      , 1, 0, "Unsafe Floating Point Math")
-COMPATIBLE_LANGOPT(AllowFPReassoc    , 1, 0, "Permit Floating Point reassociation")
-COMPATIBLE_LANGOPT(NoHonorNaNs       , 1, 0, "Permit Floating Point optimization without regard to NaN")
-COMPATIBLE_LANGOPT(NoHonorInfs       , 1, 0, "Permit Floating Point optimization without regard to infinities")
-COMPATIBLE_LANGOPT(NoSignedZero      , 1, 0, "Permit Floating Point optimization without regard to signed zeros")
-COMPATIBLE_LANGOPT(AllowRecip        , 1, 0, "Permit Floating Point reciprocal")
-COMPATIBLE_LANGOPT(ApproxFunc        , 1, 0, "Permit Floating Point approximation")
+BENIGN_LANGOPT(AllowFPReassoc    , 1, 0, "Permit Floating Point reassociation")
+BENIGN_LANGOPT(NoHonorNaNs       , 1, 0, "Permit Floating Point optimization without regard to NaN")
+BENIGN_LANGOPT(NoHonorInfs       , 1, 0, "Permit Floating Point optimization without regard to infinities")
+BENIGN_LANGOPT(NoSignedZero      , 1, 0, "Permit Floating Point optimization without regard to signed zeros")
+BENIGN_LANGOPT(AllowRecip        , 1, 0, "Permit Floating Point reciprocal")
+BENIGN_LANGOPT(ApproxFunc        , 1, 0, "Permit Floating Point approximation")
 
 BENIGN_LANGOPT(ObjCGCBitmapPrint , 1, 0, "printing of GC's bitmap layout for __weak/__strong ivars")
 
@@ -231,6 +231,7 @@ LANGOPT(OpenMPCUDANumSMs  , 32, 0, "Number of SMs for CUDA devices.")
 LANGOPT(OpenMPCUDABlocksPerSM  , 32, 0, "Number of blocks per SM for CUDA devices.")
 LANGOPT(OpenMPCUDAReductionBufNum , 32, 1024, "Number of the reduction records in the intermediate reduction buffer used for the teams reductions.")
 LANGOPT(OpenMPOptimisticCollapse  , 1, 0, "Use at most 32 bits to represent the collapsed loop nest counter.")
+LANGOPT(OpenMPCUDATargetParallel, 1, 0, "Support parallel execution of target region on Cuda-based devices.")
 LANGOPT(RenderScript      , 1, 0, "RenderScript")
 
 LANGOPT(CUDAIsDevice      , 1, 0, "compiling for CUDA device")
@@ -270,9 +271,10 @@ BENIGN_LANGOPT(SpellChecking , 1, 1, "spell-checking")
 LANGOPT(SinglePrecisionConstants , 1, 0, "treating double-precision floating point constants as single precision constants")
 LANGOPT(FastRelaxedMath , 1, 0, "OpenCL fast relaxed math")
 /// FP_CONTRACT mode (on/off/fast).
-ENUM_LANGOPT(DefaultFPContractMode, FPModeKind, 2, FPM_Off, "FP contraction type")
-ENUM_LANGOPT(FPRoundingMode, RoundingMode, 3, RoundingMode::NearestTiesToEven, "FP Rounding Mode type")
-ENUM_LANGOPT(FPExceptionMode, FPExceptionModeKind, 2, FPE_Ignore, "FP Exception Behavior Mode type")
+BENIGN_ENUM_LANGOPT(DefaultFPContractMode, FPModeKind, 2, FPM_Off, "FP contraction type")
+COMPATIBLE_LANGOPT(ExpStrictFP, 1, false, "Enable experimental strict floating point")
+BENIGN_ENUM_LANGOPT(FPRoundingMode, RoundingMode, 3, RoundingMode::NearestTiesToEven, "FP Rounding Mode type")
+BENIGN_ENUM_LANGOPT(FPExceptionMode, FPExceptionModeKind, 2, FPE_Ignore, "FP Exception Behavior Mode type")
 LANGOPT(NoBitFieldTypeAlign , 1, 0, "bit-field type alignment")
 LANGOPT(HexagonQdsp6Compat , 1, 0, "hexagon-qdsp6 backward compatibility")
 LANGOPT(ObjCAutoRefCount , 1, 0, "Objective-C automated reference counting")
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index 7b92a8964862b..a9213b7d86680 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -366,175 +366,176 @@ class LangOptions : public LangOptionsBase {
 };
 
 /// Floating point control options
+class FPOptionsOverride;
 class FPOptions {
+public:
+  // We start by defining the layout.
+  using storage_type = uint16_t;
+
   using RoundingMode = llvm::RoundingMode;
 
-public:
-  FPOptions()
-      : fp_contract(LangOptions::FPM_Off), fenv_access(LangOptions::FPM_Off),
-        rounding(LangOptions::FPR_ToNearest),
-        exceptions(LangOptions::FPE_Ignore), allow_reassoc(0), no_nans(0),
-        no_infs(0), no_signed_zeros(0), allow_reciprocal(0), approx_func(0) {}
+  // Define a fake option named "First" so that we have a PREVIOUS even for the
+  // real first option.
+  static constexpr storage_type FirstShift = 0, FirstWidth = 0;
+#define OPTION(NAME, TYPE, WIDTH, PREVIOUS)                                    \
+  static constexpr storage_type NAME##Shift =                                  \
+      PREVIOUS##Shift + PREVIOUS##Width;                                       \
+  static constexpr storage_type NAME##Width = WIDTH;                           \
+  static constexpr storage_type NAME##Mask = ((1 << NAME##Width) - 1)          \
+                                             << NAME##Shift;
+#include "clang/Basic/FPOptions.def"
 
+private:
+  storage_type Value;
+
+public:
+  FPOptions() : Value(0) {
+    setFPContractMode(LangOptions::FPM_Off);
+    setRoundingMode(static_cast<RoundingMode>(LangOptions::FPR_ToNearest));
+    setFPExceptionMode(LangOptions::FPE_Ignore);
+  }
   // Used for serializing.
   explicit FPOptions(unsigned I) { getFromOpaqueInt(I); }
 
-  explicit FPOptions(const LangOptions &LangOpts)
-      : fp_contract(LangOpts.getDefaultFPContractMode()),
-        fenv_access(LangOptions::FPM_Off),
-        rounding(static_cast<unsigned>(LangOpts.getFPRoundingMode())),
-        exceptions(LangOpts.getFPExceptionMode()),
-        allow_reassoc(LangOpts.AllowFPReassoc), no_nans(LangOpts.NoHonorNaNs),
-        no_infs(LangOpts.NoHonorInfs), no_signed_zeros(LangOpts.NoSignedZero),
-        allow_reciprocal(LangOpts.AllowRecip),
-        approx_func(LangOpts.ApproxFunc) {}
-  // FIXME: Use getDefaultFEnvAccessMode() when available.
-
-  void setFastMath(bool B = true) {
-    allow_reassoc = no_nans = no_infs = no_signed_zeros = approx_func =
-        allow_reciprocal = B;
+  explicit FPOptions(const LangOptions &LO) {
+    Value = 0;
+    setFPContractMode(LO.getDefaultFPContractMode());
+    setRoundingMode(LO.getFPRoundingMode());
+    setFPExceptionMode(LO.getFPExceptionMode());
+    setAllowFEnvAccess(LangOptions::FPM_Off),
+        setAllowFPReassociate(LO.AllowFPReassoc);
+    setNoHonorNaNs(LO.NoHonorNaNs);
+    setNoHonorInfs(LO.NoHonorInfs);
+    setNoSignedZero(LO.NoSignedZero);
+    setAllowReciprocal(LO.AllowRecip);
+    setAllowApproxFunc(LO.ApproxFunc);
   }
 
-  /// Return the default value of FPOptions that's used when trailing
-  /// storage isn't required.
-  static FPOptions defaultWithoutTrailingStorage(const LangOptions &LO);
-
-  /// Does this FPOptions require trailing storage when stored in various
-  /// AST nodes, or can it be recreated using `defaultWithoutTrailingStorage`?
-  bool requiresTrailingStorage(const LangOptions &LO);
-
   bool allowFPContractWithinStatement() const {
-    return fp_contract == LangOptions::FPM_On;
+    return getFPContractMode() == LangOptions::FPM_On;
+  }
+  void setAllowFPContractWithinStatement() {
+    setFPContractMode(LangOptions::FPM_On);
   }
 
   bool allowFPContractAcrossStatement() const {
-    return fp_contract == LangOptions::FPM_Fast;
+    return getFPContractMode() == LangOptions::FPM_Fast;
   }
-
-  void setAllowFPContractWithinStatement() {
-    fp_contract = LangOptions::FPM_On;
+  void setAllowFPContractAcrossStatement() {
+    setFPContractMode(LangOptions::FPM_Fast);
   }
 
-  void setAllowFPContractAcrossStatement() {
-    fp_contract = LangOptions::FPM_Fast;
+  bool isFPConstrained() const {
+    return getRoundingMode() !=
+               static_cast<unsigned>(RoundingMode::NearestTiesToEven) ||
+           getFPExceptionMode() != LangOptions::FPE_Ignore ||
+           getAllowFEnvAccess();
   }
 
-  void setDisallowFPContract() { fp_contract = LangOptions::FPM_Off; }
+  bool operator==(FPOptions other) const { return Value == other.Value; }
 
-  bool allowFEnvAccess() const { return fenv_access == LangOptions::FPM_On; }
+  /// Return the default value of FPOptions that's used when trailing
+  /// storage isn't required.
+  static FPOptions defaultWithoutTrailingStorage(const LangOptions &LO);
 
-  void setAllowFEnvAccess() { fenv_access = LangOptions::FPM_On; }
+  storage_type getAsOpaqueInt() const { return Value; }
+  void getFromOpaqueInt(storage_type value) { Value = value; }
 
-  void setFPPreciseEnabled(bool Value) {
-    if (Value) {
-      /* Precise mode implies fp_contract=on and disables ffast-math */
-      setFastMath(false);
-      setAllowFPContractWithinStatement();
-    } else {
-      /* Precise mode implies fp_contract=fast and enables ffast-math */
-      setFastMath(true);
-      setAllowFPContractAcrossStatement();
-    }
+  // We can define most of the accessors automatically:
+#define OPTION(NAME, TYPE, WIDTH, PREVIOUS)                                    \
+  unsigned get##NAME() const {                                                 \
+    return static_cast<unsigned>(TYPE((Value & NAME##Mask) >> NAME##Shift));   \
+  }                                                                            \
+  void set##NAME(TYPE value) {                                                 \
+    Value = (Value & ~NAME##Mask) | (storage_type(value) << NAME##Shift);      \
   }
+#include "clang/Basic/FPOptions.def"
+  LLVM_DUMP_METHOD void dump();
+};
 
-  void setDisallowFEnvAccess() { fenv_access = LangOptions::FPM_Off; }
+/// The FPOptions override type is value of the new FPOptions
+///  plus a mask showing which fields are actually set in it:
+class FPOptionsOverride {
+  FPOptions Options;
+  FPOptions::storage_type OverrideMask = 0;
 
-  RoundingMode getRoundingMode() const {
-    return static_cast<RoundingMode>(rounding);
-  }
+public:
+  using RoundingMode = llvm::RoundingMode;
+  FPOptionsOverride() {}
 
-  void setRoundingMode(RoundingMode RM) {
-    rounding = static_cast<unsigned>(RM);
-  }
+  // Used for serializing.
+  explicit FPOptionsOverride(unsigned I) { getFromOpaqueInt(I); }
+
+  bool requiresTrailingStorage() const { return OverrideMask != 0; }
 
-  LangOptions::FPExceptionModeKind getExceptionMode() const {
-    return static_cast<LangOptions::FPExceptionModeKind>(exceptions);
+  void setAllowFPContractWithinStatement() {
+    setFPContractModeOverride(LangOptions::FPM_On);
   }
 
-  void setExceptionMode(LangOptions::FPExceptionModeKind EM) {
-    exceptions = EM;
+  void setAllowFPContractAcrossStatement() {
+    setFPContractModeOverride(LangOptions::FPM_Fast);
   }
 
-  /// FMF Flag queries
-  bool allowAssociativeMath() const { return allow_reassoc; }
-  bool noHonorNaNs() const { return no_nans; }
-  bool noHonorInfs() const { return no_infs; }
-  bool noSignedZeros() const { return no_signed_zeros; }
-  bool allowReciprocalMath() const { return allow_reciprocal; }
-  bool allowApproximateFunctions() const { return approx_func; }
-
-  /// Flag setters
-  void setAllowAssociativeMath(bool B = true) { allow_reassoc = B; }
-  void setNoHonorNaNs(bool B = true) { no_nans = B; }
-  void setNoHonorInfs(bool B = true) { no_infs = B; }
-  void setNoSignedZeros(bool B = true) { no_signed_zeros = B; }
-  void setAllowReciprocalMath(bool B = true) { allow_reciprocal = B; }
-  void setAllowApproximateFunctions(bool B = true) { approx_func = B; }
+  void setDisallowFPContract() {
+    setFPContractModeOverride(LangOptions::FPM_Off);
+  }
 
-  bool isFPConstrained() const {
-    return getRoundingMode() != RoundingMode::NearestTiesToEven ||
-           getExceptionMode() != LangOptions::FPE_Ignore ||
-           allowFEnvAccess();
+  void setFPPreciseEnabled(bool Value) {
+    setAllowFPReassociateOverride(!Value);
+    setNoHonorNaNsOverride(!Value);
+    setNoHonorInfsOverride(!Value);
+    setNoSignedZeroOverride(!Value);
+    setAllowReciprocalOverride(!Value);
+    setAllowApproxFuncOverride(!Value);
+    if (Value)
+      /* Precise mode implies fp_contract=on and disables ffast-math */
+      setAllowFPContractWithinStatement();
+    else
+      /* Precise mode disabled sets fp_contract=fast and enables ffast-math */
+      setAllowFPContractAcrossStatement();
   }
 
-  /// Used to serialize this.
   unsigned getAsOpaqueInt() const {
-    return fp_contract | (fenv_access << 2) | (rounding << 3) |
-           (exceptions << 6) | (allow_reassoc << 8) | (no_nans << 9) |
-           (no_infs << 10) | (no_signed_zeros << 11) |
-           (allow_reciprocal << 12) | (approx_func << 13);
+    return Options.getAsOpaqueInt() << 16 | OverrideMask;
   }
-
-  /// Used with getAsOpaqueInt() to manage the float_control pragma stack.
   void getFromOpaqueInt(unsigned I) {
-    fp_contract = (static_cast<LangOptions::FPModeKind>(I & 3));
-    fenv_access = ((I >> 2) & 1);
-    rounding = static_cast<unsigned>(static_cast<RoundingMode>((I >> 3) & 7));
-    exceptions = (static_cast<LangOptions::FPExceptionModeKind>((I >> 6) & 3));
-    allow_reassoc = ((I >> 8) & 1);
-    no_nans = ((I >> 9) & 1);
-    no_infs = ((I >> 10) & 1);
-    no_signed_zeros = ((I >> 11) & 1);
-    allow_reciprocal = ((I >> 12) & 1);
-    approx_func = ((I >> 13) & 1);
+    OverrideMask = I & 0xffff;
+    Options.getFromOpaqueInt(I >> 16);
   }
 
-private:
-  /// Adjust BinaryOperatorBitfields::FPFeatures and
-  /// CXXOperatorCallExprBitfields::FPFeatures to match the total bit-field size
-  /// of these fields.
-  unsigned fp_contract : 2;
-  unsigned fenv_access : 1;
-  unsigned rounding : 3;
-  unsigned exceptions : 2;
-  /// Allow reassociation transformations for floating-point instructions
-  /// across multiple statements.
-  unsigned allow_reassoc : 1;
-  /// No NaNs - Allow optimizations to assume the arguments and result
-  /// are not NaN. If an argument is a nan, or the result would be a nan,
-  /// it produces a :ref:`poison value <poisonvalues>` instead.
-  unsigned no_nans : 1;
-  /// No Infs - Allow optimizations to assume the arguments and result
-  /// are not +/-Inf. If an argument is +/-Inf, or the result would be +/-Inf,
-  /// it produces a :ref:`poison value <poisonvalues>` instead.
-  unsigned no_infs : 1;
-  /// No Signed Zeros - Allow optimizations to treat the sign of a zero
-  /// argument or result as insignificant.
-  unsigned no_signed_zeros : 1;
-  /// Allow Reciprocal - Allow optimizations to use the reciprocal
-  /// of an argument rather than perform division.
-  unsigned allow_reciprocal : 1;
-  /// Approximate functions - Allow substitution of approximate calculations
-  /// for functions (sin, log, sqrt, etc).
-  unsigned approx_func : 1;
-};
+  FPOptions applyOverrides(const LangOptions &LO) {
+    FPOptions Base(LO);
+    FPOptions result((Base.getAsOpaqueInt() & ~OverrideMask) |
+                     (Options.getAsOpaqueInt() & OverrideMask));
+    return result;
+  }
 
-inline bool operator==(FPOptions LHS, FPOptions RHS) {
-  return LHS.getAsOpaqueInt() == RHS.getAsOpaqueInt();
-}
-inline bool operator!=(FPOptions LHS, FPOptions RHS) {
-  return LHS.getAsOpaqueInt() != RHS.getAsOpaqueInt();
-}
+  bool operator==(FPOptionsOverride other) const {
+    return Options == other.Options && OverrideMask == other.OverrideMask;
+  }
+  bool operator!=(FPOptionsOverride other) const { return !(*this == other); }
+
+#define OPTION(NAME, TYPE, WIDTH, PREVIOUS)                                    \
+  bool has##NAME##Override() const {                                           \
+    return OverrideMask & FPOptions::NAME##Mask;                               \
+  }                                                                            \
+  unsigned get##NAME##Override() const {                                       \
+    assert(has##NAME##Override());                                             \
+    return Options.get##NAME();                                                \
+  }                                                                            \
+  void clear##NAME##Override() {                                               \
+    /* Clear the actual value so that we don't have spurious differences when  \
+     * testing equality. */                                                    \
+    Options.set##NAME(TYPE(0));                                                \
+    OverrideMask &= ~FPOptions::NAME##Mask;                                    \
+  }                                                                            \
+  void set##NAME##Override(TYPE value) {                                       \
+    Options.set##NAME(value);                                                  \
+    OverrideMask |= FPOptions::NAME##Mask;                                     \
+  }
+#include "clang/Basic/FPOptions.def"
+  LLVM_DUMP_METHOD void dump();
+};
 
 /// Describes the kind of translation unit being processed.
 enum TranslationUnitKind {
diff --git a/clang/include/clang/Basic/ObjCRuntime.h b/clang/include/clang/Basic/ObjCRuntime.h
index 1c4a69269deea..26403bfa98c9d 100644
--- a/clang/include/clang/Basic/ObjCRuntime.h
+++ b/clang/include/clang/Basic/ObjCRuntime.h
@@ -476,6 +476,10 @@ class ObjCRuntime {
   friend bool operator!=(const ObjCRuntime &left, const ObjCRuntime &right) {
     return !(left == right);
   }
+
+  friend llvm::hash_code hash_value(const ObjCRuntime &OCR) {
+    return llvm::hash_combine(OCR.getKind(), OCR.getVersion());
+  }
 };
 
 raw_ostream &operator<<(raw_ostream &out, const ObjCRuntime &value);
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index 693ae4f938e11..5c666c1760b47 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -1645,8 +1645,7 @@ class SourceManager : public RefCountedBase<SourceManager> {
   unsigned local_sloc_entry_size() const { return LocalSLocEntryTable.size(); }
 
   /// Get a local SLocEntry. This is exposed for indexing.
-  const SrcMgr::SLocEntry &getLocalSLocEntry(unsigned Index,
-                                             bool *Invalid = nullptr) const {
+  const SrcMgr::SLocEntry &getLocalSLocEntry(unsigned Index) const {
     assert(Index < LocalSLocEntryTable.size() && "Invalid index");
     return LocalSLocEntryTable[Index];
   }
@@ -1739,12 +1738,13 @@ class SourceManager : public RefCountedBase<SourceManager> {
   const SrcMgr::SLocEntry &loadSLocEntry(unsigned Index, bool *Invalid) const;
 
   /// Get the entry with the given unwrapped FileID.
+  /// Invalid will not be modified for Local IDs.
   const SrcMgr::SLocEntry &getSLocEntryByID(int ID,
                                             bool *Invalid = nullptr) const {
     assert(ID != -1 && "Using FileID sentinel value");
     if (ID < 0)
       return getLoadedSLocEntryByID(ID, Invalid);
-    return getLocalSLocEntry(static_cast<unsigned>(ID), Invalid);
+    return getLocalSLocEntry(static_cast<unsigned>(ID));
   }
 
   const SrcMgr::SLocEntry &
diff --git a/clang/include/clang/Basic/TargetBuiltins.h b/clang/include/clang/Basic/TargetBuiltins.h
index eba055c302a7f..b472547012f0d 100644
--- a/clang/include/clang/Basic/TargetBuiltins.h
+++ b/clang/include/clang/Basic/TargetBuiltins.h
@@ -119,6 +119,11 @@ namespace clang {
   };
   }
 
+  /// VE builtins
+  namespace VE {
+  enum { LastTIBuiltin = clang::Builtin::FirstTSBuiltin - 1, LastTSBuiltin };
+  }
+
   /// Flags to identify the types for overloaded Neon builtins.
   ///
   /// These must be kept in sync with the flags in utils/TableGen/NeonEmitter.h.
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index 140f55ff66b10..2ee3b16596302 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -192,6 +192,7 @@ class TargetInfo : public virtual TransferrableTargetInfo,
   bool HasFloat128;
   bool HasFloat16;
   bool HasBFloat16;
+  bool HasStrictFP;
 
   unsigned char MaxAtomicPromoteWidth, MaxAtomicInlineWidth;
   unsigned short SimdDefaultAlign;
@@ -577,6 +578,9 @@ class TargetInfo : public virtual TransferrableTargetInfo,
   /// Determine whether the _BFloat16 type is supported on this target.
   virtual bool hasBFloat16Type() const { return HasBFloat16; }
 
+  /// Determine whether constrained floating point is supported on this target.
+  virtual bool hasStrictFP() const { return HasStrictFP; }
+
   /// Return the alignment that is suitable for storing any
   /// object with a fundamental alignment requirement.
   unsigned getSuitableAlign() const { return SuitableAlign; }
diff --git a/clang/include/clang/Basic/arm_neon.td b/clang/include/clang/Basic/arm_neon.td
index 289f5ea47b926..d0269f31c32dd 100644
--- a/clang/include/clang/Basic/arm_neon.td
+++ b/clang/include/clang/Basic/arm_neon.td
@@ -190,28 +190,20 @@ def OP_SCALAR_QRDMLAH_LN : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1,
 def OP_SCALAR_QRDMLSH_LN : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1,
                               (call "vget_lane", $p2, $p3)))>;
 
-multiclass ScalarGetSetLaneOpsF16<string scalarTy,
-                                  string vectorTy4, string vectorTy8> {
-  def _GET_LN  : Op<(bitcast scalarTy,
-                        (call "vget_lane",
-                            (bitcast "int16x4_t", $p0), $p1))>;
-  def _GET_LNQ : Op<(bitcast scalarTy,
-                        (call "vget_lane",
-                            (bitcast "int16x8_t", $p0), $p1))>;
-  def _SET_LN  : Op<(bitcast vectorTy4,
-                        (call "vset_lane",
-                            (bitcast "int16_t", $p0),
-                            (bitcast "int16x4_t", $p1), $p2))>;
-  def _SET_LNQ : Op<(bitcast vectorTy8,
-                        (call "vset_lane",
-                            (bitcast "int16_t", $p0),
-                            (bitcast "int16x8_t", $p1), $p2))>;
-}
-
-defm OP_SCALAR_HALF: ScalarGetSetLaneOpsF16<"float16_t",
-                                            "float16x4_t", "float16x8_t">;
-defm OP_SCALAR_BF16: ScalarGetSetLaneOpsF16<"bfloat16_t",
-                                            "bfloat16x4_t", "bfloat16x8_t">;
+def OP_SCALAR_HALF_GET_LN : Op<(bitcast "float16_t",
+                                   (call "vget_lane",
+                                         (bitcast "int16x4_t", $p0), $p1))>;
+def OP_SCALAR_HALF_GET_LNQ : Op<(bitcast "float16_t",
+                                    (call "vget_lane",
+                                          (bitcast "int16x8_t", $p0), $p1))>;
+def OP_SCALAR_HALF_SET_LN : Op<(bitcast "float16x4_t",
+                                   (call "vset_lane",
+                                         (bitcast "int16_t", $p0),
+                                         (bitcast "int16x4_t", $p1), $p2))>;
+def OP_SCALAR_HALF_SET_LNQ : Op<(bitcast "float16x8_t",
+                                    (call "vset_lane",
+                                          (bitcast "int16_t", $p0),
+                                          (bitcast "int16x8_t", $p1), $p2))>;
 
 def OP_DOT_LN
     : Op<(call "vdot", $p0, $p1,
@@ -260,6 +252,34 @@ def OP_BFMLALT_LN
     : Op<(call "vbfmlalt", $p0, $p1,
           (dup_typed $p1, (call "vget_lane", $p2, $p3)))>;
 
+def OP_VCVT_F32_BF16
+    : Op<(bitcast "R",
+          (call "vshll_n", (bitcast "int16x4_t", $p0),
+                           (literal "int32_t", "16")))>;
+def OP_VCVT_F32_BF16_LO
+    : Op<(call "vcvt_f32_bf16", (call "vget_low", $p0))>;
+def OP_VCVT_F32_BF16_HI
+    : Op<(call "vcvt_f32_bf16", (call "vget_high", $p0))>;
+
+def OP_VCVT_BF16_F32_LO_A64
+    : Op<(call "__a64_vcvtq_low_bf16", $p0)>;
+def OP_VCVT_BF16_F32_A64
+    : Op<(call "vget_low", (call "__a64_vcvtq_low_bf16", $p0))>;
+
+def OP_VCVT_BF16_F32_A32
+    : Op<(call "__a32_vcvt_bf16", $p0)>;
+
+def OP_VCVT_BF16_F32_LO_A32
+    : Op<(call "vcombine", (cast "bfloat16x4_t", (literal "uint64_t", "0ULL")),
+                           (call "__a32_vcvt_bf16", $p0))>;
+def OP_VCVT_BF16_F32_HI_A32
+    : Op<(call "vcombine", (call "__a32_vcvt_bf16", $p1),
+                           (call "vget_low", $p0))>;
+
+def OP_CVT_F32_BF16
+    : Op<(bitcast "R", (op "<<", (bitcast "int32_t", $p0),
+                                 (literal "int32_t", "16")))>;
+
 //===----------------------------------------------------------------------===//
 // Auxiliary Instructions
 //===----------------------------------------------------------------------===//
@@ -1918,10 +1938,12 @@ let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)" in {
   def VGET_HIGH_BF : NoTestOpInst<"vget_high", ".Q", "b", OP_HI>;
   def VGET_LOW_BF  : NoTestOpInst<"vget_low", ".Q", "b", OP_LO>;
 
-  def VGET_LANE_BF : IOpInst<"vget_lane", "1.I", "b", OP_SCALAR_BF16_GET_LN>;
-  def VSET_LANE_BF : IOpInst<"vset_lane", ".1.I", "b", OP_SCALAR_BF16_SET_LN>;
-  def VGET_LANEQ_BF : IOpInst<"vget_lane", "1.I", "Qb", OP_SCALAR_BF16_GET_LNQ>;
-  def VSET_LANEQ_BF : IOpInst<"vset_lane", ".1.I", "Qb", OP_SCALAR_BF16_SET_LNQ>;
+  def VGET_LANE_BF : IInst<"vget_lane", "1.I", "bQb">;
+  def VSET_LANE_BF : IInst<"vset_lane", ".1.I", "bQb">;
+  def SCALAR_VDUP_LANE_BF : IInst<"vdup_lane", "1.I", "Sb">;
+  def SCALAR_VDUP_LANEQ_BF : IInst<"vdup_laneq", "1QI", "Sb"> {
+    let isLaneQ = 1;
+  }
 
   def VLD1_BF : WInst<"vld1", ".(c*!)", "bQb">;
   def VLD2_BF : WInst<"vld2", "2(c*!)", "bQb">;
@@ -1955,18 +1977,31 @@ let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)" in {
   def VLD3_DUP_BF : WInst<"vld3_dup", "3(c*!)", "bQb">;
   def VLD4_DUP_BF : WInst<"vld4_dup", "4(c*!)", "bQb">;
 
+  def VCVT_F32_BF16 : SOpInst<"vcvt_f32_bf16", "(F>)(Bq!)",  "Qb", OP_VCVT_F32_BF16>;
+  def VCVT_LOW_F32_BF16 : SOpInst<"vcvt_low_f32", "(F>)(BQ!)",  "Qb", OP_VCVT_F32_BF16_LO>;
+  def VCVT_HIGH_F32_BF16 : SOpInst<"vcvt_high_f32", "(F>)(BQ!)", "Qb", OP_VCVT_F32_BF16_HI>;
+
+  def SCALAR_CVT_BF16_F32 : SInst<"vcvth_bf16", "(1B)1", "f">;
+  def SCALAR_CVT_F32_BF16 : SOpInst<"vcvtah_f32", "(1F>)(1!)", "b", OP_CVT_F32_BF16>;
 }
 
 let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC) && !defined(__aarch64__)" in {
-  def SCALAR_VDUP_LANE_BF_A32 : IOpInst<"vduph_lane", "1.I", "b", OP_SCALAR_BF16_GET_LN>;
-  def SCALAR_VDUP_LANEQ_BF_A32 : IOpInst<"vduph_laneq", "1.I", "Hb", OP_SCALAR_BF16_GET_LNQ>;
+  def VCVT_BF16_F32_A32_INTERNAL : WInst<"__a32_vcvt_bf16", "BQ", "f">;
+  def VCVT_BF16_F32_A32 : SOpInst<"vcvt_bf16", "BQ", "f", OP_VCVT_BF16_F32_A32>;
+  def VCVT_LOW_BF16_F32_A32 : SOpInst<"vcvt_low_bf16",  "BQ", "Qf", OP_VCVT_BF16_F32_LO_A32>;
+  def VCVT_HIGH_BF16_F32_A32 : SOpInst<"vcvt_high_bf16", "BBQ", "Qf", OP_VCVT_BF16_F32_HI_A32>;
 }
 
 let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC) && defined(__aarch64__)" in {
-  def SCALAR_VDUP_LANE_BF_A64 : IInst<"vdup_lane", "1.I", "Sb">;
-  def SCALAR_VDUP_LANEQ_BF_A64 : IInst<"vdup_laneq", "1QI", "Sb"> {
-    let isLaneQ = 1;
-  }
+  def VCVT_LOW_BF16_F32_A64_INTERNAL : WInst<"__a64_vcvtq_low_bf16", "BQ", "Hf">;
+  def VCVT_LOW_BF16_F32_A64 : SOpInst<"vcvt_low_bf16", "BQ", "Qf", OP_VCVT_BF16_F32_LO_A64>;
+  def VCVT_HIGH_BF16_F32_A64 : SInst<"vcvt_high_bf16", "BBQ", "Qf">;
+  def VCVT_BF16_F32 : SOpInst<"vcvt_bf16",    "BQ", "f", OP_VCVT_BF16_F32_A64>;
+
+  def COPY_LANE_BF16 : IOpInst<"vcopy_lane", "..I.I", "b", OP_COPY_LN>;
+  def COPYQ_LANE_BF16 : IOpInst<"vcopy_lane", "..IqI", "Qb", OP_COPY_LN>;
+  def COPY_LANEQ_BF16 : IOpInst<"vcopy_laneq", "..IQI", "b", OP_COPY_LN>;
+  def COPYQ_LANEQ_BF16 : IOpInst<"vcopy_laneq", "..I.I", "Qb", OP_COPY_LN>;
 }
 
 let ArchGuard = "defined(__ARM_FEATURE_BF16) && !defined(__aarch64__)" in {
diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td
index 8b64db87190f9..19a42e79c36ab 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -271,6 +271,11 @@ def SVLD1UH : MInst<"svld1uh_{d}", "dPX", "ilUiUl",          [IsLoad, IsZExtRetu
 def SVLD1SW : MInst<"svld1sw_{d}", "dPU", "lUl",             [IsLoad],               MemEltTyInt32,   "aarch64_sve_ld1">;
 def SVLD1UW : MInst<"svld1uw_{d}", "dPY", "lUl",             [IsLoad, IsZExtReturn], MemEltTyInt32,   "aarch64_sve_ld1">;
 
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+  def SVLD1_BF      : MInst<"svld1[_{2}]",      "dPc",  "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ld1">;
+  def SVLD1_VNUM_BF : MInst<"svld1_vnum[_{2}]", "dPcl", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ld1">;
+}
+
 // Load one vector (scalar base, VL displacement)
 def SVLD1_VNUM   : MInst<"svld1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad],               MemEltTyDefault, "aarch64_sve_ld1">;
 def SVLD1SB_VNUM : MInst<"svld1sb_vnum_{d}", "dPSl", "silUsUiUl",       [IsLoad],               MemEltTyInt8,    "aarch64_sve_ld1">;
@@ -376,6 +381,11 @@ def SVLDFF1UH_VNUM : MInst<"svldff1uh_vnum_{d}", "dPXl", "ilUiUl",          [IsL
 def SVLDFF1SW_VNUM : MInst<"svldff1sw_vnum_{d}", "dPUl", "lUl",             [IsLoad],               MemEltTyInt32,   "aarch64_sve_ldff1">;
 def SVLDFF1UW_VNUM : MInst<"svldff1uw_vnum_{d}", "dPYl", "lUl",             [IsLoad, IsZExtReturn], MemEltTyInt32,   "aarch64_sve_ldff1">;
 
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+  def SVLDFF1_BF      : MInst<"svldff1[_{2}]",      "dPc",  "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldff1">;
+  def SVLDFF1_VNUM_BF : MInst<"svldff1_vnum[_{2}]", "dPcl", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldff1">;
+}
+
 // First-faulting load one vector (vector base)
 def SVLDFF1_GATHER_BASES_U   : MInst<"svldff1_gather[_{2}base]_{d}",   "dPu", "ilUiUlfd", [IsGatherLoad],               MemEltTyDefault, "aarch64_sve_ldff1_gather_scalar_offset">;
 def SVLDFF1SB_GATHER_BASES_U : MInst<"svldff1sb_gather[_{2}base]_{d}", "dPu", "ilUiUl",   [IsGatherLoad],               MemEltTyInt8,    "aarch64_sve_ldff1_gather_scalar_offset">;
@@ -471,32 +481,67 @@ def SVLDNF1UH_VNUM : MInst<"svldnf1uh_vnum_{d}", "dPXl", "ilUiUl",          [IsL
 def SVLDNF1SW_VNUM : MInst<"svldnf1sw_vnum_{d}", "dPUl", "lUl",             [IsLoad],               MemEltTyInt32,   "aarch64_sve_ldnf1">;
 def SVLDNF1UW_VNUM : MInst<"svldnf1uw_vnum_{d}", "dPYl", "lUl",             [IsLoad, IsZExtReturn], MemEltTyInt32,   "aarch64_sve_ldnf1">;
 
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+  def SVLDNF1_BF      : MInst<"svldnf1[_{2}]",      "dPc",  "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnf1">;
+  def SVLDNF1_VNUM_BF : MInst<"svldnf1_vnum[_{2}]", "dPcl", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnf1">;
+}
+
 // Load one vector, unextended load, non-temporal (scalar base)
 def SVLDNT1 : MInst<"svldnt1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnt1">;
 
 // Load one vector, unextended load, non-temporal (scalar base, VL displacement)
 def SVLDNT1_VNUM : MInst<"svldnt1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnt1">;
 
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+  def SVLDNT1_BF      : MInst<"svldnt1[_{2}]",      "dPc",  "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnt1">;
+  def SVLDNT1_VNUM_BF : MInst<"svldnt1_vnum[_{2}]", "dPcl", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnt1">;
+}
+
 // Load one quadword and replicate (scalar base)
 def SVLD1RQ : SInst<"svld1rq[_{2}]", "dPc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld1rq">;
 
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+  def SVLD1RQ_BF : SInst<"svld1rq[_{2}]", "dPc",  "b", MergeNone, "aarch64_sve_ld1rq">;
+}
+
+multiclass StructLoad<string name, string proto, string i> {
+  def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i, [IsStructLoad]>;
+  let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+    def: SInst<name, proto, "b", MergeNone, i, [IsStructLoad]>;
+  }
+}
+
 // Load N-element structure into N vectors (scalar base)
-def SVLD2 : SInst<"svld2[_{2}]", "2Pc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld2", [IsStructLoad]>;
-def SVLD3 : SInst<"svld3[_{2}]", "3Pc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld3", [IsStructLoad]>;
-def SVLD4 : SInst<"svld4[_{2}]", "4Pc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld4", [IsStructLoad]>;
+defm SVLD2 : StructLoad<"svld2[_{2}]", "2Pc", "aarch64_sve_ld2">;
+defm SVLD3 : StructLoad<"svld3[_{2}]", "3Pc", "aarch64_sve_ld3">;
+defm SVLD4 : StructLoad<"svld4[_{2}]", "4Pc", "aarch64_sve_ld4">;
 
 // Load N-element structure into N vectors (scalar base, VL displacement)
-def SVLD2_VNUM : SInst<"svld2_vnum[_{2}]", "2Pcl", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld2", [IsStructLoad]>;
-def SVLD3_VNUM : SInst<"svld3_vnum[_{2}]", "3Pcl", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld3", [IsStructLoad]>;
-def SVLD4_VNUM : SInst<"svld4_vnum[_{2}]", "4Pcl", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld4", [IsStructLoad]>;
+defm SVLD2_VNUM : StructLoad<"svld2_vnum[_{2}]", "2Pcl", "aarch64_sve_ld2">;
+defm SVLD3_VNUM : StructLoad<"svld3_vnum[_{2}]", "3Pcl", "aarch64_sve_ld3">;
+defm SVLD4_VNUM : StructLoad<"svld4_vnum[_{2}]", "4Pcl", "aarch64_sve_ld4">;
 
 // Load one octoword and replicate (scalar base)
 let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_FP64)" in {
   def SVLD1RO : SInst<"svld1ro[_{2}]", "dPc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld1ro">;
 }
-let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_FP64) && defined(__ARM_FEATURE_BF16_SCALAR_ARITHMETIC)" in {
-  def SVLD1RO_BF : SInst<"svld1ro[_{2}]", "dPc", "b", MergeNone, "aarch64_sve_ld1ro">;
+let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_FP64) && defined(__ARM_FEATURE_SVE_BF16)" in {
+  def SVLD1RO_BF16 : SInst<"svld1ro[_{2}]", "dPc", "b", MergeNone, "aarch64_sve_ld1ro">;
+}
+
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+  def SVBFDOT        : SInst<"svbfdot[_{0}]",        "MMdd",  "b", MergeNone, "aarch64_sve_bfdot",        [IsOverloadNone]>;
+  def SVBFMLALB      : SInst<"svbfmlalb[_{0}]",      "MMdd",  "b", MergeNone, "aarch64_sve_bfmlalb",      [IsOverloadNone]>;
+  def SVBFMLALT      : SInst<"svbfmlalt[_{0}]",      "MMdd",  "b", MergeNone, "aarch64_sve_bfmlalt",      [IsOverloadNone]>;
+  def SVBFMMLA       : SInst<"svbfmmla[_{0}]",       "MMdd",  "b", MergeNone, "aarch64_sve_bfmmla",       [IsOverloadNone]>;
+  def SVBFDOT_N      : SInst<"svbfdot[_n_{0}]",      "MMda",  "b", MergeNone, "aarch64_sve_bfdot",        [IsOverloadNone]>;
+  def SVBFMLAL_N     : SInst<"svbfmlalb[_n_{0}]",    "MMda",  "b", MergeNone, "aarch64_sve_bfmlalb",      [IsOverloadNone]>;
+  def SVBFMLALT_N    : SInst<"svbfmlalt[_n_{0}]",    "MMda",  "b", MergeNone, "aarch64_sve_bfmlalt",      [IsOverloadNone]>;
+  def SVBFDOT_LANE   : SInst<"svbfdot_lane[_{0}]",   "MMddn", "b", MergeNone, "aarch64_sve_bfdot_lane",   [IsOverloadNone], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVBFMLALB_LANE : SInst<"svbfmlalb_lane[_{0}]", "MMddn", "b", MergeNone, "aarch64_sve_bfmlalb_lane", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVBFMLALT_LANE : SInst<"svbfmlalt_lane[_{0}]", "MMddn", "b", MergeNone, "aarch64_sve_bfmlalt_lane", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>;
 }
+
 ////////////////////////////////////////////////////////////////////////////////
 // Stores
 
@@ -518,6 +563,11 @@ def SVST1H_VNUM_U : MInst<"svst1h_vnum[_{d}]", "vPFld", "UiUl",            [IsSt
 def SVST1W_VNUM_S : MInst<"svst1w_vnum[_{d}]", "vPCld", "l",               [IsStore], MemEltTyInt32,   "aarch64_sve_st1">;
 def SVST1W_VNUM_U : MInst<"svst1w_vnum[_{d}]", "vPGld", "Ul",              [IsStore], MemEltTyInt32,   "aarch64_sve_st1">;
 
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+  def SVST1_BF      : MInst<"svst1[_{d}]",      "vPpd",  "b", [IsStore], MemEltTyDefault, "aarch64_sve_st1">;
+  def SVST1_VNUM_BF : MInst<"svst1_vnum[_{d}]", "vPpld", "b", [IsStore], MemEltTyDefault, "aarch64_sve_st1">;
+}
+
 // Store one vector (vector base)
 def SVST1_SCATTER_BASES_U     : MInst<"svst1_scatter[_{2}base_{d}]",  "vPud",  "ilUiUlfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_scalar_offset">;
 def SVST1B_SCATTER_BASES_U    : MInst<"svst1b_scatter[_{2}base_{d}]", "vPud",  "ilUiUl",   [IsScatterStore], MemEltTyInt8,    "aarch64_sve_st1_scatter_scalar_offset">;
@@ -587,15 +637,21 @@ def SVST1_SCATTER_INDEX_S     : MInst<"svst1_scatter[_{2}base]_index[_{d}]",  "v
 def SVST1H_SCATTER_INDEX_S    : MInst<"svst1h_scatter[_{2}base]_index[_{d}]", "vPuld", "ilUiUl",   [IsScatterStore], MemEltTyInt16,   "aarch64_sve_st1_scatter_scalar_offset">;
 def SVST1W_SCATTER_INDEX_S    : MInst<"svst1w_scatter[_{2}base]_index[_{d}]", "vPuld", "lUl",      [IsScatterStore], MemEltTyInt32,   "aarch64_sve_st1_scatter_scalar_offset">;
 
+multiclass StructStore<string name, string proto, string i> {
+  def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i, [IsStructStore]>;
+  let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+    def: SInst<name, proto, "b", MergeNone, i, [IsStructStore]>;
+  }
+}
 // Store N vectors into N-element structure (scalar base)
-def SVST2 : SInst<"svst2[_{d}]", "vPp2", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_st2", [IsStructStore]>;
-def SVST3 : SInst<"svst3[_{d}]", "vPp3", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_st3", [IsStructStore]>;
-def SVST4 : SInst<"svst4[_{d}]", "vPp4", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_st4", [IsStructStore]>;
+defm SVST2 : StructStore<"svst2[_{d}]", "vPp2", "aarch64_sve_st2">;
+defm SVST3 : StructStore<"svst3[_{d}]", "vPp3", "aarch64_sve_st3">;
+defm SVST4 : StructStore<"svst4[_{d}]", "vPp4", "aarch64_sve_st4">;
 
 // Store N vectors into N-element structure (scalar base, VL displacement)
-def SVST2_VNUM : SInst<"svst2_vnum[_{d}]", "vPpl2", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_st2", [IsStructStore]>;
-def SVST3_VNUM : SInst<"svst3_vnum[_{d}]", "vPpl3", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_st3", [IsStructStore]>;
-def SVST4_VNUM : SInst<"svst4_vnum[_{d}]", "vPpl4", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_st4", [IsStructStore]>;
+defm SVST2_VNUM : StructStore<"svst2_vnum[_{d}]", "vPpl2", "aarch64_sve_st2">;
+defm SVST3_VNUM : StructStore<"svst3_vnum[_{d}]", "vPpl3", "aarch64_sve_st3">;
+defm SVST4_VNUM : StructStore<"svst4_vnum[_{d}]", "vPpl4", "aarch64_sve_st4">;
 
 // Store one vector, with no truncation, non-temporal (scalar base)
 def SVSTNT1 : MInst<"svstnt1[_{d}]", "vPpd", "csilUcUsUiUlhfd", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">;
@@ -603,6 +659,11 @@ def SVSTNT1 : MInst<"svstnt1[_{d}]", "vPpd", "csilUcUsUiUlhfd", [IsStore], MemEl
 // Store one vector, with no truncation, non-temporal (scalar base, VL displacement)
 def SVSTNT1_VNUM : MInst<"svstnt1_vnum[_{d}]", "vPpld", "csilUcUsUiUlhfd", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">;
 
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+  def SVSTNT1_BF      : MInst<"svstnt1[_{d}]",      "vPpd",  "b", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">;
+  def SVSTNT1_VNUM_BF : MInst<"svstnt1_vnum[_{d}]", "vPpld", "b", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // Prefetches
 
@@ -664,13 +725,23 @@ def SVADRD : SInst<"svadrd[_{0}base]_[{2}]index",  "uud", "ilUiUl", MergeNone, "
 
 def SVDUPQ_8  : SInst<"svdupq[_n]_{d}", "dssssssssssssssss",  "cUc", MergeNone>;
 def SVDUPQ_16 : SInst<"svdupq[_n]_{d}", "dssssssss",  "sUsh", MergeNone>;
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+  def SVDUPQ_BF16 : SInst<"svdupq[_n]_{d}", "dssssssss",  "b", MergeNone>;
+}
 def SVDUPQ_32 : SInst<"svdupq[_n]_{d}", "dssss",  "iUif", MergeNone>;
 def SVDUPQ_64 : SInst<"svdupq[_n]_{d}", "dss",  "lUld", MergeNone>;
 
-def SVDUP   : SInst<"svdup[_n]_{d}", "ds",   "csilUcUsUiUlhfd", MergeNone,    "aarch64_sve_dup_x">;
-def SVDUP_M : SInst<"svdup[_n]_{d}", "ddPs", "csilUcUsUiUlhfd", MergeOp1,     "aarch64_sve_dup">;
-def SVDUP_X : SInst<"svdup[_n]_{d}", "dPs",  "csilUcUsUiUlhfd", MergeAnyExp,  "aarch64_sve_dup">;
-def SVDUP_Z : SInst<"svdup[_n]_{d}", "dPs",  "csilUcUsUiUlhfd", MergeZeroExp, "aarch64_sve_dup">;
+multiclass svdup_base<string n, string p, MergeType mt, string i> {
+  def NAME : SInst<n, p, "csilUcUsUiUlhfd", mt, i>;
+  let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+    def _BF16: SInst<n, p, "b", mt, i>;
+  }
+}
+
+defm SVDUP   : svdup_base<"svdup[_n]_{d}", "ds",   MergeNone,    "aarch64_sve_dup_x">;
+defm SVDUP_M : svdup_base<"svdup[_n]_{d}", "ddPs", MergeOp1,     "aarch64_sve_dup">;
+defm SVDUP_X : svdup_base<"svdup[_n]_{d}", "dPs",  MergeAnyExp,  "aarch64_sve_dup">;
+defm SVDUP_Z : svdup_base<"svdup[_n]_{d}", "dPs",  MergeZeroExp, "aarch64_sve_dup">;
 
 def SVINDEX : SInst<"svindex_{d}",   "dss",  "csilUcUsUiUl",    MergeNone,    "aarch64_sve_index">;
 
@@ -789,8 +860,11 @@ defm SVLSR : SInst_SHIFT<"svlsr", "aarch64_sve_lsr", "UcUsUiUl", "UcUsUi">;
 def SVASRD_M : SInst<"svasrd[_n_{d}]", "dPdi", "csil",            MergeOp1,  "aarch64_sve_asrd", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
 def SVASRD_X : SInst<"svasrd[_n_{d}]", "dPdi", "csil",            MergeAny,  "aarch64_sve_asrd", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
 def SVASRD_Z : SInst<"svasrd[_n_{d}]", "dPdi", "csil",            MergeZero, "aarch64_sve_asrd", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
-def SVINSR   : SInst<"svinsr[_n_{d}]", "dds",  "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_insr">;
 
+def SVINSR : SInst<"svinsr[_n_{d}]", "dds", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_insr">;
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+  def SVINSR_BF16 : SInst<"svinsr[_n_{d}]", "dds",  "b", MergeNone, "aarch64_sve_insr">;
+}
 
 ////////////////////////////////////////////////////////////////////////////////
 // Integer reductions
@@ -877,6 +951,10 @@ defm SVCLS : SInstCLS<"svcls", "csil",            "aarch64_sve_cls">;
 defm SVCLZ : SInstCLS<"svclz", "csilUcUsUiUl",    "aarch64_sve_clz">;
 defm SVCNT : SInstCLS<"svcnt", "csilUcUsUiUlhfd", "aarch64_sve_cnt">;
 
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+  defm SVCNT_BF16 : SInstCLS<"svcnt", "b", "aarch64_sve_cnt">;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // Conversion
 
@@ -1030,6 +1108,11 @@ defm SVFCVTZS_S64_F16 : SInstCvtMXZ<"svcvt_s64[_f16]", "ddPO", "dPO", "l",  "aar
 defm SVFCVTZS_S32_F32 : SInstCvtMXZ<"svcvt_s32[_f32]", "ddPM", "dPM", "i",  "aarch64_sve_fcvtzs", [IsOverloadCvt]>;
 defm SVFCVTZS_S64_F32 : SInstCvtMXZ<"svcvt_s64[_f32]", "ddPM", "dPM", "l",  "aarch64_sve_fcvtzs_i64f32">;
 
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+  defm SVCVT_BF16_F32   : SInstCvtMXZ<"svcvt_bf16[_f32]",  "ddPM", "dPM", "b",  "aarch64_sve_fcvt_bf16f32">;
+  def SVCVTNT_BF16_F32 : SInst<"svcvtnt_bf16[_f32]", "ddPM", "b",  MergeOp1, "aarch64_sve_fcvtnt_bf16f32", [IsOverloadNone]>;
+}
+
 // svcvt_s##_f64
 defm SVFCVTZS_S32_F64 : SInstCvtMXZ<"svcvt_s32[_f64]", "ttPd", "tPd", "d",  "aarch64_sve_fcvtzs_i32f64">;
 defm SVFCVTZS_S64_F64 : SInstCvtMXZ<"svcvt_s64[_f64]", "ddPN", "dPN", "l",  "aarch64_sve_fcvtzs", [IsOverloadCvt]>;
@@ -1103,24 +1186,45 @@ def SVCVTXNT_F32    : SInst<"svcvtxnt_f32[_f64]", "MMPd", "d", MergeOp1, "aarch6
 ////////////////////////////////////////////////////////////////////////////////
 // Permutations and selection
 
-def SVCLASTA     : SInst<"svclasta[_{d}]",    "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_clasta">;
-def SVCLASTA_N   : SInst<"svclasta[_n_{d}]",  "sPsd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_clasta_n">;
-def SVCLASTB     : SInst<"svclastb[_{d}]",    "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_clastb">;
-def SVCLASTB_N   : SInst<"svclastb[_n_{d}]",  "sPsd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_clastb_n">;
+multiclass SVEPerm<string name, string proto, string i> {
+  def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i>;
+  let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+    def: SInst<name, proto, "b", MergeNone, i>;
+  }
+}
+
+defm SVCLASTA    : SVEPerm<"svclasta[_{d}]",   "dPdd", "aarch64_sve_clasta">;
+defm SVCLASTA_N  : SVEPerm<"svclasta[_n_{d}]", "sPsd", "aarch64_sve_clasta_n">;
+defm SVCLASTB    : SVEPerm<"svclastb[_{d}]",   "dPdd", "aarch64_sve_clastb">;
+defm SVCLASTB_N  : SVEPerm<"svclastb[_n_{d}]", "sPsd", "aarch64_sve_clastb_n">;
+
 def SVCOMPACT    : SInst<"svcompact[_{d}]",   "dPd",  "ilUiUlfd",        MergeNone, "aarch64_sve_compact">;
 // Note: svdup_lane is implemented using the intrinsic for TBL to represent a
 // splat of any possible lane. It is upto LLVM to pick a more efficient
 // instruction such as DUP (indexed) if the lane index fits the range of the
 // instruction's immediate.
 def SVDUP_LANE   : SInst<"svdup_lane[_{d}]",  "ddL",  "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbl">;
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+def SVDUP_LANE_BF16 :
+                   SInst<"svdup_lane[_{d}]",  "ddL",  "b",               MergeNone, "aarch64_sve_tbl">;
+}
+
 def SVDUPQ_LANE  : SInst<"svdupq_lane[_{d}]", "ddn",  "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_dupq_lane">;
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+  def SVDUPQ_LANE_BF16  : SInst<"svdupq_lane[_{d}]", "ddn",  "b", MergeNone, "aarch64_sve_dupq_lane">;
+}
 def SVEXT        : SInst<"svext[_{d}]",       "dddi", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ext", [], [ImmCheck<2, ImmCheckExtract, 1>]>;
-def SVLASTA      : SInst<"svlasta[_{d}]",     "sPd",  "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_lasta">;
-def SVLASTB      : SInst<"svlastb[_{d}]",     "sPd",  "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_lastb">;
+defm SVLASTA     : SVEPerm<"svlasta[_{d}]",   "sPd",  "aarch64_sve_lasta">;
+defm SVLASTB     : SVEPerm<"svlastb[_{d}]",   "sPd",  "aarch64_sve_lastb">;
 def SVREV        : SInst<"svrev[_{d}]",       "dd",   "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_rev">;
 def SVSEL        : SInst<"svsel[_{d}]",       "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_sel">;
 def SVSPLICE     : SInst<"svsplice[_{d}]",    "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_splice">;
 def SVTBL        : SInst<"svtbl[_{d}]",       "ddu",  "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbl">;
+
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+  def SVTBL_BF16 : SInst<"svtbl[_{d}]",       "ddu",  "b",               MergeNone, "aarch64_sve_tbl">;
+}
+
 def SVTRN1       : SInst<"svtrn1[_{d}]",      "ddd",  "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn1">;
 def SVTRN2       : SInst<"svtrn2[_{d}]",      "ddd",  "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn2">;
 def SVUNPKHI_S   : SInst<"svunpkhi[_{d}]",    "dh",   "sil",             MergeNone, "aarch64_sve_sunpkhi">;
@@ -1132,6 +1236,19 @@ def SVUZP2       : SInst<"svuzp2[_{d}]",      "ddd",  "csilUcUsUiUlhfd", MergeNo
 def SVZIP1       : SInst<"svzip1[_{d}]",      "ddd",  "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip1">;
 def SVZIP2       : SInst<"svzip2[_{d}]",      "ddd",  "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2">;
 
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+def SVEXT_BF16    : SInst<"svext[_{d}]",    "dddi", "b", MergeNone, "aarch64_sve_ext", [], [ImmCheck<2, ImmCheckExtract, 1>]>;
+def SVREV_BF16    : SInst<"svrev[_{d}]",    "dd",   "b", MergeNone, "aarch64_sve_rev">;
+def SVSEL_BF16    : SInst<"svsel[_{d}]",    "dPdd", "b", MergeNone, "aarch64_sve_sel">;
+def SVSPLICE_BF16 : SInst<"svsplice[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_splice">;
+def SVTRN1_BF16   : SInst<"svtrn1[_{d}]",   "ddd",  "b", MergeNone, "aarch64_sve_trn1">;
+def SVTRN2_BF16   : SInst<"svtrn2[_{d}]",   "ddd",  "b", MergeNone, "aarch64_sve_trn2">;
+def SVUZP1_BF16   : SInst<"svuzp1[_{d}]",   "ddd",  "b", MergeNone, "aarch64_sve_uzp1">;
+def SVUZP2_BF16   : SInst<"svuzp2[_{d}]",   "ddd",  "b", MergeNone, "aarch64_sve_uzp2">;
+def SVZIP1_BF16   : SInst<"svzip1[_{d}]",   "ddd",  "b", MergeNone, "aarch64_sve_zip1">;
+def SVZIP2_BF16   : SInst<"svzip2[_{d}]",   "ddd",  "b", MergeNone, "aarch64_sve_zip2">;
+}
+
 def SVREV_B   : SInst<"svrev_{d}",      "PP",   "PcPsPiPl", MergeNone, "aarch64_sve_rev">;
 def SVSEL_B   : SInst<"svsel[_b]",      "PPPP", "Pc",       MergeNone, "aarch64_sve_sel">;
 def SVTRN1_B  : SInst<"svtrn1_{d}",     "PPP",  "PcPsPiPl", MergeNone, "aarch64_sve_trn1">;
@@ -1213,6 +1330,10 @@ def SVCNTD : SInst<"svcntd", "n", "", MergeNone, "aarch64_sve_cntd", [IsAppendSV
 def SVCNTP : SInst<"svcntp_{d}",  "nPP", "PcPsPiPl",        MergeNone, "aarch64_sve_cntp">;
 def SVLEN  : SInst<"svlen[_{d}]", "nd",  "csilUcUsUiUlhfd", MergeNone>;
 
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+def SVLEN_BF16 : SInst<"svlen[_{d}]", "nd", "b", MergeNone>;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // Saturating scalar arithmetic
 
@@ -1303,6 +1424,15 @@ def SVZIP1Q      : SInst<"svzip1q[_{d}]",     "ddd",  "csilUcUsUiUlhfd", MergeNo
 def SVZIP2Q      : SInst<"svzip2q[_{d}]",     "ddd",  "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2q">;
 }
 
+let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_FP64) && defined(__ARM_FEATURE_SVE_BF16)" in {
+def SVTRN1Q_BF16      : SInst<"svtrn1q[_{d}]",     "ddd",  "b", MergeNone, "aarch64_sve_trn1q">;
+def SVTRN2Q_BF16      : SInst<"svtrn2q[_{d}]",     "ddd",  "b", MergeNone, "aarch64_sve_trn2q">;
+def SVUZP1Q_BF16      : SInst<"svuzp1q[_{d}]",     "ddd",  "b", MergeNone, "aarch64_sve_uzp1q">;
+def SVUZP2Q_BF16      : SInst<"svuzp2q[_{d}]",     "ddd",  "b", MergeNone, "aarch64_sve_uzp2q">;
+def SVZIP1Q_BF16      : SInst<"svzip1q[_{d}]",     "ddd",  "b", MergeNone, "aarch64_sve_zip1q">;
+def SVZIP2Q_BF16      : SInst<"svzip2q[_{d}]",     "ddd",  "b", MergeNone, "aarch64_sve_zip2q">;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // Vector creation
 def SVUNDEF_1 : SInst<"svundef_{d}",  "d", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef]>;
@@ -1314,6 +1444,16 @@ def SVCREATE_2 : SInst<"svcreate2[_{d}]", "2dd",   "csilUcUsUiUlhfd", MergeNone,
 def SVCREATE_3 : SInst<"svcreate3[_{d}]", "3ddd",  "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tuple_create3", [IsTupleCreate]>;
 def SVCREATE_4 : SInst<"svcreate4[_{d}]", "4dddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tuple_create4", [IsTupleCreate]>;
 
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+def SVUNDEF_1_BF16 : SInst<"svundef_{d}",  "d", "b", MergeNone, "", [IsUndef]>;
+def SVUNDEF_2_BF16 : SInst<"svundef2_{d}", "2", "b", MergeNone, "", [IsUndef]>;
+def SVUNDEF_3_BF16 : SInst<"svundef3_{d}", "3", "b", MergeNone, "", [IsUndef]>;
+def SVUNDEF_4_BF16 : SInst<"svundef4_{d}", "4", "b", MergeNone, "", [IsUndef]>;
+
+def SVCREATE_2_BF16 : SInst<"svcreate2[_{d}]", "2dd",   "b", MergeNone, "aarch64_sve_tuple_create2", [IsTupleCreate]>;
+def SVCREATE_3_BF16 : SInst<"svcreate3[_{d}]", "3ddd",  "b", MergeNone, "aarch64_sve_tuple_create3", [IsTupleCreate]>;
+def SVCREATE_4_BF16 : SInst<"svcreate4[_{d}]", "4dddd", "b", MergeNone, "aarch64_sve_tuple_create4", [IsTupleCreate]>;
+}
 
 ////////////////////////////////////////////////////////////////////////////////
 // Vector insertion and extraction
@@ -1325,6 +1465,16 @@ def SVSET_2 : SInst<"svset2[_{d}]", "22id", "csilUcUsUiUlhfd", MergeNone, "aarch
 def SVSET_3 : SInst<"svset3[_{d}]", "33id", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tuple_set", [IsTupleSet], [ImmCheck<1, ImmCheck0_2>]>;
 def SVSET_4 : SInst<"svset4[_{d}]", "44id", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tuple_set", [IsTupleSet], [ImmCheck<1, ImmCheck0_3>]>;
 
+let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
+def SVGET_2_BF16 : SInst<"svget2[_{d}]", "d2i", "b", MergeNone, "aarch64_sve_tuple_get", [IsTupleGet], [ImmCheck<1, ImmCheck0_1>]>;
+def SVGET_3_BF16 : SInst<"svget3[_{d}]", "d3i", "b", MergeNone, "aarch64_sve_tuple_get", [IsTupleGet], [ImmCheck<1, ImmCheck0_2>]>;
+def SVGET_4_BF16 : SInst<"svget4[_{d}]", "d4i", "b", MergeNone, "aarch64_sve_tuple_get", [IsTupleGet], [ImmCheck<1, ImmCheck0_3>]>;
+
+def SVSET_2_BF16 : SInst<"svset2[_{d}]", "22id", "b", MergeNone, "aarch64_sve_tuple_set", [IsTupleSet], [ImmCheck<1, ImmCheck0_1>]>;
+def SVSET_3_BF16 : SInst<"svset3[_{d}]", "33id", "b", MergeNone, "aarch64_sve_tuple_set", [IsTupleSet], [ImmCheck<1, ImmCheck0_2>]>;
+def SVSET_4_BF16 : SInst<"svset4[_{d}]", "44id", "b", MergeNone, "aarch64_sve_tuple_set", [IsTupleSet], [ImmCheck<1, ImmCheck0_3>]>;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 WhileGE/GT
 let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
@@ -1881,6 +2031,11 @@ def SVWHILEWR_S : SInst<"svwhilewr[_{1}]", "Pcc", "iUif", MergeNone, "aarch64_sv
 def SVWHILEWR_D : SInst<"svwhilewr[_{1}]", "Pcc", "lUld", MergeNone, "aarch64_sve_whilewr_d", [IsOverloadWhileRW]>;
 }
 
+let ArchGuard = "defined(__ARM_FEATURE_SVE2) && defined(__ARM_FEATURE_BF16_SCALAR_ARITHMETIC)" in {
+def SVWHILERW_H_BF16 : SInst<"svwhilerw[_{1}]", "Pcc", "b", MergeNone, "aarch64_sve_whilerw_h", [IsOverloadWhileRW]>;
+def SVWHILEWR_H_BF16 : SInst<"svwhilewr[_{1}]", "Pcc", "b", MergeNone, "aarch64_sve_whilewr_h", [IsOverloadWhileRW]>;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 - Extended table lookup/permute
 let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
@@ -1888,6 +2043,11 @@ def SVTBL2 : SInst<"svtbl2[_{d}]", "d2u",  "csilUcUsUiUlhfd", MergeNone>;
 def SVTBX  : SInst<"svtbx[_{d}]",  "dddu", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbx">;
 }
 
+let ArchGuard = "defined(__ARM_FEATURE_SVE2) && defined(__ARM_FEATURE_SVE_BF16)" in {
+def SVTBL2_BF16 : SInst<"svtbl2[_{d}]", "d2u",  "b", MergeNone>;
+def SVTBX_BF16  : SInst<"svtbx[_{d}]",  "dddu", "b", MergeNone, "aarch64_sve_tbx">;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 - Optional
 
diff --git a/clang/include/clang/CodeGen/CodeGenABITypes.h b/clang/include/clang/CodeGen/CodeGenABITypes.h
index 0201f92074ecb..3c745fadbe78f 100644
--- a/clang/include/clang/CodeGen/CodeGenABITypes.h
+++ b/clang/include/clang/CodeGen/CodeGenABITypes.h
@@ -25,7 +25,9 @@
 
 #include "clang/AST/CanonicalType.h"
 #include "clang/AST/Type.h"
+#include "clang/Basic/ABI.h"
 #include "clang/CodeGen/CGFunctionInfo.h"
+#include "llvm/IR/BasicBlock.h"
 
 namespace llvm {
 class AttrBuilder;
@@ -40,6 +42,7 @@ class Type;
 namespace clang {
 class ASTContext;
 class CXXConstructorDecl;
+class CXXDestructorDecl;
 class CXXRecordDecl;
 class CXXMethodDecl;
 class CodeGenOptions;
@@ -90,6 +93,12 @@ const CGFunctionInfo &arrangeFreeFunctionCall(CodeGenModule &CGM,
 ImplicitCXXConstructorArgs
 getImplicitCXXConstructorArgs(CodeGenModule &CGM, const CXXConstructorDecl *D);
 
+llvm::Value *
+getCXXDestructorImplicitParam(CodeGenModule &CGM, llvm::BasicBlock *InsertBlock,
+                              llvm::BasicBlock::iterator InsertPoint,
+                              const CXXDestructorDecl *D, CXXDtorType Type,
+                              bool ForVirtualBase, bool Delegating);
+
 /// Returns null if the function type is incomplete and can't be lowered.
 llvm::FunctionType *convertFreeFunctionType(CodeGenModule &CGM,
                                             const FunctionDecl *FD);
diff --git a/clang/include/clang/Driver/Action.h b/clang/include/clang/Driver/Action.h
index 8ccbb6c2bbfa2..27c95c6f89d41 100644
--- a/clang/include/clang/Driver/Action.h
+++ b/clang/include/clang/Driver/Action.h
@@ -73,9 +73,10 @@ class Action {
     OffloadBundlingJobClass,
     OffloadUnbundlingJobClass,
     OffloadWrapperJobClass,
+    StaticLibJobClass,
 
     JobClassFirst = PreprocessJobClass,
-    JobClassLast = OffloadWrapperJobClass
+    JobClassLast = StaticLibJobClass
   };
 
   // The offloading kind determines if this action is binded to a particular
@@ -637,6 +638,17 @@ class OffloadWrapperJobAction : public JobAction {
   }
 };
 
+class StaticLibJobAction : public JobAction {
+  void anchor() override;
+
+public:
+  StaticLibJobAction(ActionList &Inputs, types::ID Type);
+
+  static bool classof(const Action *A) {
+    return A->getKind() == StaticLibJobClass;
+  }
+};
+
 } // namespace driver
 } // namespace clang
 
diff --git a/clang/include/clang/Driver/CC1Options.td b/clang/include/clang/Driver/CC1Options.td
deleted file mode 100644
index 9d2bfbf949efd..0000000000000
--- a/clang/include/clang/Driver/CC1Options.td
+++ /dev/null
@@ -1,941 +0,0 @@
-//===--- CC1Options.td - Options for clang -cc1 ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines the options accepted by clang -cc1 and clang -cc1as.
-//
-//===----------------------------------------------------------------------===//
-
-let Flags = [CC1Option, NoDriverOption] in {
-
-//===----------------------------------------------------------------------===//
-// Target Options
-//===----------------------------------------------------------------------===//
-
-let Flags = [CC1Option, CC1AsOption, NoDriverOption] in {
-
-def target_cpu : Separate<["-"], "target-cpu">,
-  HelpText<"Target a specific cpu type">;
-def target_feature : Separate<["-"], "target-feature">,
-  HelpText<"Target specific attributes">;
-def triple : Separate<["-"], "triple">,
-  HelpText<"Specify target triple (e.g. i686-apple-darwin9)">;
-def target_abi : Separate<["-"], "target-abi">,
-  HelpText<"Target a particular ABI type">;
-def target_sdk_version_EQ : Joined<["-"], "target-sdk-version=">,
-  HelpText<"The version of target SDK used for compilation">;
-
-}
-
-def target_linker_version : Separate<["-"], "target-linker-version">,
-  HelpText<"Target linker version">;
-def triple_EQ : Joined<["-"], "triple=">, Alias<triple>;
-def mfpmath : Separate<["-"], "mfpmath">,
-  HelpText<"Which unit to use for fp math">;
-
-def fpadding_on_unsigned_fixed_point : Flag<["-"], "fpadding-on-unsigned-fixed-point">,
-  HelpText<"Force each unsigned fixed point type to have an extra bit of padding to align their scales with those of signed fixed point types">;
-def fno_padding_on_unsigned_fixed_point : Flag<["-"], "fno-padding-on-unsigned-fixed-point">;
-
-//===----------------------------------------------------------------------===//
-// Analyzer Options
-//===----------------------------------------------------------------------===//
-
-def analysis_UnoptimizedCFG : Flag<["-"], "unoptimized-cfg">,
-  HelpText<"Generate unoptimized CFGs for all analyses">;
-def analysis_CFGAddImplicitDtors : Flag<["-"], "cfg-add-implicit-dtors">,
-  HelpText<"Add C++ implicit destructors to CFGs for all analyses">;
-
-def analyzer_store : Separate<["-"], "analyzer-store">,
-  HelpText<"Source Code Analysis - Abstract Memory Store Models">;
-def analyzer_store_EQ : Joined<["-"], "analyzer-store=">, Alias<analyzer_store>;
-
-def analyzer_constraints : Separate<["-"], "analyzer-constraints">,
-  HelpText<"Source Code Analysis - Symbolic Constraint Engines">;
-def analyzer_constraints_EQ : Joined<["-"], "analyzer-constraints=">,
-  Alias<analyzer_constraints>;
-
-def analyzer_output : Separate<["-"], "analyzer-output">,
-  HelpText<"Source Code Analysis - Output Options">;
-def analyzer_output_EQ : Joined<["-"], "analyzer-output=">,
-  Alias<analyzer_output>;
-
-def analyzer_purge : Separate<["-"], "analyzer-purge">,
-  HelpText<"Source Code Analysis - Dead Symbol Removal Frequency">;
-def analyzer_purge_EQ : Joined<["-"], "analyzer-purge=">, Alias<analyzer_purge>;
-
-def analyzer_opt_analyze_headers : Flag<["-"], "analyzer-opt-analyze-headers">,
-  HelpText<"Force the static analyzer to analyze functions defined in header files">;
-def analyzer_opt_analyze_nested_blocks : Flag<["-"], "analyzer-opt-analyze-nested-blocks">,
-  HelpText<"Analyze the definitions of blocks in addition to functions">;
-def analyzer_display_progress : Flag<["-"], "analyzer-display-progress">,
-  HelpText<"Emit verbose output about the analyzer's progress">;
-def analyze_function : Separate<["-"], "analyze-function">,
-  HelpText<"Run analysis on specific function (for C++ include parameters in name)">;
-def analyze_function_EQ : Joined<["-"], "analyze-function=">, Alias<analyze_function>;
-def trim_egraph : Flag<["-"], "trim-egraph">,
-  HelpText<"Only show error-related paths in the analysis graph">;
-def analyzer_viz_egraph_graphviz : Flag<["-"], "analyzer-viz-egraph-graphviz">,
-  HelpText<"Display exploded graph using GraphViz">;
-def analyzer_dump_egraph : Separate<["-"], "analyzer-dump-egraph">,
-  HelpText<"Dump exploded graph to the specified file">;
-def analyzer_dump_egraph_EQ : Joined<["-"], "analyzer-dump-egraph=">, Alias<analyzer_dump_egraph>;
-
-def analyzer_inline_max_stack_depth : Separate<["-"], "analyzer-inline-max-stack-depth">,
-  HelpText<"Bound on stack depth while inlining (4 by default)">;
-def analyzer_inline_max_stack_depth_EQ : Joined<["-"], "analyzer-inline-max-stack-depth=">,
-  Alias<analyzer_inline_max_stack_depth>;
-
-def analyzer_inlining_mode : Separate<["-"], "analyzer-inlining-mode">,
-  HelpText<"Specify the function selection heuristic used during inlining">;
-def analyzer_inlining_mode_EQ : Joined<["-"], "analyzer-inlining-mode=">, Alias<analyzer_inlining_mode>;
-
-def analyzer_disable_retry_exhausted : Flag<["-"], "analyzer-disable-retry-exhausted">,
-  HelpText<"Do not re-analyze paths leading to exhausted nodes with a different strategy (may decrease code coverage)">;
-
-def analyzer_max_loop : Separate<["-"], "analyzer-max-loop">,
-  HelpText<"The maximum number of times the analyzer will go through a loop">;
-def analyzer_stats : Flag<["-"], "analyzer-stats">,
-  HelpText<"Print internal analyzer statistics.">;
-
-def analyzer_checker : Separate<["-"], "analyzer-checker">,
-  HelpText<"Choose analyzer checkers to enable">,
-  ValuesCode<[{
-    const char *Values =
-    #define GET_CHECKERS
-    #define CHECKER(FULLNAME, CLASS, HT, DOC_URI, IS_HIDDEN)  FULLNAME ","
-    #include "clang/StaticAnalyzer/Checkers/Checkers.inc"
-    #undef GET_CHECKERS
-    #define GET_PACKAGES
-    #define PACKAGE(FULLNAME)  FULLNAME ","
-    #include "clang/StaticAnalyzer/Checkers/Checkers.inc"
-    #undef GET_PACKAGES
-    ;
-  }]>;
-def analyzer_checker_EQ : Joined<["-"], "analyzer-checker=">,
-  Alias<analyzer_checker>;
-
-def analyzer_disable_checker : Separate<["-"], "analyzer-disable-checker">,
-  HelpText<"Choose analyzer checkers to disable">;
-def analyzer_disable_checker_EQ : Joined<["-"], "analyzer-disable-checker=">,
-  Alias<analyzer_disable_checker>;
-
-def analyzer_disable_all_checks : Flag<["-"], "analyzer-disable-all-checks">,
-  HelpText<"Disable all static analyzer checks">;
-
-def analyzer_checker_help : Flag<["-"], "analyzer-checker-help">,
-  HelpText<"Display the list of analyzer checkers that are available">;
-
-def analyzer_checker_help_alpha : Flag<["-"], "analyzer-checker-help-alpha">,
-  HelpText<"Display the list of in development analyzer checkers. These "
-           "are NOT considered safe, they are unstable and will emit incorrect "
-           "reports. Enable ONLY FOR DEVELOPMENT purposes">;
-
-def analyzer_checker_help_developer : Flag<["-"], "analyzer-checker-help-developer">,
-  HelpText<"Display the list of developer-only checkers such as modeling "
-           "and debug checkers">;
-
-def analyzer_config_help : Flag<["-"], "analyzer-config-help">,
-  HelpText<"Display the list of -analyzer-config options. These are meant for "
-           "development purposes only!">;
-
-def analyzer_list_enabled_checkers : Flag<["-"], "analyzer-list-enabled-checkers">,
-  HelpText<"Display the list of enabled analyzer checkers">;
-
-def analyzer_config : Separate<["-"], "analyzer-config">,
-  HelpText<"Choose analyzer options to enable">;
-
-def analyzer_checker_option_help : Flag<["-"], "analyzer-checker-option-help">,
-  HelpText<"Display the list of checker and package options">;
-
-def analyzer_checker_option_help_alpha : Flag<["-"], "analyzer-checker-option-help-alpha">,
-  HelpText<"Display the list of in development checker and package options. "
-           "These are NOT considered safe, they are unstable and will emit "
-           "incorrect reports. Enable ONLY FOR DEVELOPMENT purposes">;
-
-def analyzer_checker_option_help_developer : Flag<["-"], "analyzer-checker-option-help-developer">,
-  HelpText<"Display the list of checker and package options meant for "
-           "development purposes only">;
-
-def analyzer_config_compatibility_mode : Separate<["-"], "analyzer-config-compatibility-mode">,
-  HelpText<"Don't emit errors on invalid analyzer-config inputs">;
-
-def analyzer_config_compatibility_mode_EQ : Joined<["-"], "analyzer-config-compatibility-mode=">,
-  Alias<analyzer_config_compatibility_mode>;
-
-def analyzer_werror : Flag<["-"], "analyzer-werror">,
-  HelpText<"Emit analyzer results as errors rather than warnings">;
-
-//===----------------------------------------------------------------------===//
-// Migrator Options
-//===----------------------------------------------------------------------===//
-def migrator_no_nsalloc_error : Flag<["-"], "no-ns-alloc-error">,
-  HelpText<"Do not error on use of NSAllocateCollectable/NSReallocateCollectable">;
-
-def migrator_no_finalize_removal : Flag<["-"], "no-finalize-removal">,
-  HelpText<"Do not remove finalize method in gc mode">;
-
-//===----------------------------------------------------------------------===//
-// CodeGen Options
-//===----------------------------------------------------------------------===//
-
-let Flags = [CC1Option, CC1AsOption, NoDriverOption] in {
-def debug_info_kind_EQ : Joined<["-"], "debug-info-kind=">;
-def debug_info_macro : Flag<["-"], "debug-info-macro">,
-  HelpText<"Emit macro debug information">;
-def default_function_attr : Separate<["-"], "default-function-attr">,
-  HelpText<"Apply given attribute to all functions">;
-def dwarf_version_EQ : Joined<["-"], "dwarf-version=">;
-def debugger_tuning_EQ : Joined<["-"], "debugger-tuning=">;
-def dwarf_debug_flags : Separate<["-"], "dwarf-debug-flags">,
-  HelpText<"The string to embed in the Dwarf debug flags record.">;
-def record_command_line : Separate<["-"], "record-command-line">,
-  HelpText<"The string to embed in the .LLVM.command.line section.">;
-def compress_debug_sections : Flag<["-", "--"], "compress-debug-sections">,
-    HelpText<"DWARF debug sections compression">;
-def compress_debug_sections_EQ : Joined<["-", "--"], "compress-debug-sections=">,
-    HelpText<"DWARF debug sections compression type">;
-def mno_exec_stack : Flag<["-"], "mnoexecstack">,
-  HelpText<"Mark the file as not needing an executable stack">;
-def massembler_no_warn : Flag<["-"], "massembler-no-warn">,
-  HelpText<"Make assembler not emit warnings">;
-def massembler_fatal_warnings : Flag<["-"], "massembler-fatal-warnings">,
-  HelpText<"Make assembler warnings fatal">;
-def mrelax_relocations : Flag<["--"], "mrelax-relocations">,
-    HelpText<"Use relaxable elf relocations">;
-def msave_temp_labels : Flag<["-"], "msave-temp-labels">,
-  HelpText<"Save temporary labels in the symbol table. "
-           "Note this may change .s semantics and shouldn't generally be used "
-           "on compiler-generated code.">;
-def mrelocation_model : Separate<["-"], "mrelocation-model">,
-  HelpText<"The relocation model to use">, Values<"static,pic,ropi,rwpi,ropi-rwpi,dynamic-no-pic">;
-def fno_math_builtin : Flag<["-"], "fno-math-builtin">,
-  HelpText<"Disable implicit builtin knowledge of math functions">;
-}
-
-def disable_llvm_verifier : Flag<["-"], "disable-llvm-verifier">,
-  HelpText<"Don't run the LLVM IR verifier pass">;
-def disable_llvm_passes : Flag<["-"], "disable-llvm-passes">,
-  HelpText<"Use together with -emit-llvm to get pristine LLVM IR from the "
-           "frontend by not running any LLVM passes at all">;
-def disable_llvm_optzns : Flag<["-"], "disable-llvm-optzns">,
-  Alias<disable_llvm_passes>;
-def disable_lifetimemarkers : Flag<["-"], "disable-lifetime-markers">,
-  HelpText<"Disable lifetime-markers emission even when optimizations are "
-           "enabled">;
-def disable_O0_optnone : Flag<["-"], "disable-O0-optnone">,
-  HelpText<"Disable adding the optnone attribute to functions at O0">;
-def disable_red_zone : Flag<["-"], "disable-red-zone">,
-  HelpText<"Do not emit code that uses the red zone.">;
-def dwarf_column_info : Flag<["-"], "dwarf-column-info">,
-  HelpText<"Turn on column location information.">;
-def dwarf_ext_refs : Flag<["-"], "dwarf-ext-refs">,
-  HelpText<"Generate debug info with external references to clang modules"
-           " or precompiled headers">;
-def dwarf_explicit_import : Flag<["-"], "dwarf-explicit-import">,
-  HelpText<"Generate explicit import from anonymous namespace to containing"
-           " scope">;
-def debug_forward_template_params : Flag<["-"], "debug-forward-template-params">,
-  HelpText<"Emit complete descriptions of template parameters in forward"
-           " declarations">;
-def fforbid_guard_variables : Flag<["-"], "fforbid-guard-variables">,
-  HelpText<"Emit an error if a C++ static local initializer would need a guard variable">;
-def no_implicit_float : Flag<["-"], "no-implicit-float">,
-  HelpText<"Don't generate implicit floating point instructions">;
-def fdump_vtable_layouts : Flag<["-"], "fdump-vtable-layouts">,
-  HelpText<"Dump the layouts of all vtables that will be emitted in a translation unit">;
-def fmerge_functions : Flag<["-"], "fmerge-functions">,
-  HelpText<"Permit merging of identical functions when optimizing.">;
-def femit_coverage_notes : Flag<["-"], "femit-coverage-notes">,
-  HelpText<"Emit a gcov coverage notes file when compiling.">;
-def femit_coverage_data: Flag<["-"], "femit-coverage-data">,
-  HelpText<"Instrument the program to emit gcov coverage data when run.">;
-def coverage_data_file : Separate<["-"], "coverage-data-file">,
-  HelpText<"Emit coverage data to this filename.">;
-def coverage_data_file_EQ : Joined<["-"], "coverage-data-file=">,
-  Alias<coverage_data_file>;
-def coverage_notes_file : Separate<["-"], "coverage-notes-file">,
-  HelpText<"Emit coverage notes to this filename.">;
-def coverage_notes_file_EQ : Joined<["-"], "coverage-notes-file=">,
-  Alias<coverage_notes_file>;
-def coverage_version_EQ : Joined<["-"], "coverage-version=">,
-  HelpText<"Four-byte version string for gcov files.">;
-def test_coverage : Flag<["-"], "test-coverage">,
-  HelpText<"Do not generate coverage files or remove coverage changes from IR">;
-def dump_coverage_mapping : Flag<["-"], "dump-coverage-mapping">,
-  HelpText<"Dump the coverage mapping records, for testing">;
-def fuse_register_sized_bitfield_access: Flag<["-"], "fuse-register-sized-bitfield-access">,
-  HelpText<"Use register sized accesses to bit-fields, when possible.">;
-def relaxed_aliasing : Flag<["-"], "relaxed-aliasing">,
-  HelpText<"Turn off Type Based Alias Analysis">;
-def no_struct_path_tbaa : Flag<["-"], "no-struct-path-tbaa">,
-  HelpText<"Turn off struct-path aware Type Based Alias Analysis">;
-def new_struct_path_tbaa : Flag<["-"], "new-struct-path-tbaa">,
-  HelpText<"Enable enhanced struct-path aware Type Based Alias Analysis">;
-def mdebug_pass : Separate<["-"], "mdebug-pass">,
-  HelpText<"Enable additional debug output">;
-def mframe_pointer_EQ : Joined<["-"], "mframe-pointer=">,
-  HelpText<"Specify which frame pointers to retain (all, non-leaf, none).">, Values<"all,non-leaf,none">;
-def mdisable_tail_calls : Flag<["-"], "mdisable-tail-calls">,
-  HelpText<"Disable tail call optimization, keeping the call stack accurate">;
-def menable_no_infinities : Flag<["-"], "menable-no-infs">,
-  HelpText<"Allow optimization to assume there are no infinities.">;
-def menable_no_nans : Flag<["-"], "menable-no-nans">,
-  HelpText<"Allow optimization to assume there are no NaNs.">;
-def menable_unsafe_fp_math : Flag<["-"], "menable-unsafe-fp-math">,
-  HelpText<"Allow unsafe floating-point math optimizations which may decrease "
-           "precision">;
-def mreassociate : Flag<["-"], "mreassociate">,
-  HelpText<"Allow reassociation transformations for floating-point instructions">;
-def mabi_EQ_ieeelongdouble : Flag<["-"], "mabi=ieeelongdouble">,
-  HelpText<"Use IEEE 754 quadruple-precision for long double">;
-def mfloat_abi : Separate<["-"], "mfloat-abi">,
-  HelpText<"The float ABI to use">;
-def mtp : Separate<["-"], "mtp">,
-  HelpText<"Mode for reading thread pointer">;
-def mlimit_float_precision : Separate<["-"], "mlimit-float-precision">,
-  HelpText<"Limit float precision to the given value">;
-def split_stacks : Flag<["-"], "split-stacks">,
-  HelpText<"Try to use a split stack if possible.">;
-def mno_zero_initialized_in_bss : Flag<["-"], "mno-zero-initialized-in-bss">,
-  HelpText<"Do not put zero initialized data in the BSS">;
-def mregparm : Separate<["-"], "mregparm">,
-  HelpText<"Limit the number of registers available for integer arguments">;
-def msmall_data_limit : Separate<["-"], "msmall-data-limit">,
-  HelpText<"Put global and static data smaller than the limit into a special section">;
-def munwind_tables : Flag<["-"], "munwind-tables">,
-  HelpText<"Generate unwinding tables for all functions">;
-def mconstructor_aliases : Flag<["-"], "mconstructor-aliases">,
-  HelpText<"Emit complete constructors and destructors as aliases when possible">;
-def mlink_bitcode_file : Separate<["-"], "mlink-bitcode-file">,
-  HelpText<"Link the given bitcode file before performing optimizations.">;
-def mlink_builtin_bitcode : Separate<["-"], "mlink-builtin-bitcode">,
-  HelpText<"Link and internalize needed symbols from the given bitcode file "
-           "before performing optimizations.">;
-def mlink_cuda_bitcode : Separate<["-"], "mlink-cuda-bitcode">,
-  Alias<mlink_builtin_bitcode>;
-def vectorize_loops : Flag<["-"], "vectorize-loops">,
-  HelpText<"Run the Loop vectorization passes">;
-def vectorize_slp : Flag<["-"], "vectorize-slp">,
-  HelpText<"Run the SLP vectorization passes">;
-def dependent_lib : Joined<["--"], "dependent-lib=">,
-  HelpText<"Add dependent library">;
-def linker_option : Joined<["--"], "linker-option=">,
-  HelpText<"Add linker option">;
-def fsanitize_coverage_type : Joined<["-"], "fsanitize-coverage-type=">,
-                              HelpText<"Sanitizer coverage type">;
-def fsanitize_coverage_indirect_calls
-    : Flag<["-"], "fsanitize-coverage-indirect-calls">,
-      HelpText<"Enable sanitizer coverage for indirect calls">;
-def fsanitize_coverage_trace_bb
-    : Flag<["-"], "fsanitize-coverage-trace-bb">,
-      HelpText<"Enable basic block tracing in sanitizer coverage">;
-def fsanitize_coverage_trace_cmp
-    : Flag<["-"], "fsanitize-coverage-trace-cmp">,
-      HelpText<"Enable cmp instruction tracing in sanitizer coverage">;
-def fsanitize_coverage_trace_div
-    : Flag<["-"], "fsanitize-coverage-trace-div">,
-      HelpText<"Enable div instruction tracing in sanitizer coverage">;
-def fsanitize_coverage_trace_gep
-    : Flag<["-"], "fsanitize-coverage-trace-gep">,
-      HelpText<"Enable gep instruction tracing in sanitizer coverage">;
-def fsanitize_coverage_8bit_counters
-    : Flag<["-"], "fsanitize-coverage-8bit-counters">,
-      HelpText<"Enable frequency counters in sanitizer coverage">;
-def fsanitize_coverage_inline_8bit_counters
-    : Flag<["-"], "fsanitize-coverage-inline-8bit-counters">,
-      HelpText<"Enable inline 8-bit counters in sanitizer coverage">;
-def fsanitize_coverage_inline_bool_flag
-    : Flag<["-"], "fsanitize-coverage-inline-bool-flag">,
-      HelpText<"Enable inline bool flag in sanitizer coverage">;
-def fsanitize_coverage_pc_table
-    : Flag<["-"], "fsanitize-coverage-pc-table">,
-      HelpText<"Create a table of coverage-instrumented PCs">;
-def fsanitize_coverage_trace_pc
-    : Flag<["-"], "fsanitize-coverage-trace-pc">,
-      HelpText<"Enable PC tracing in sanitizer coverage">;
-def fsanitize_coverage_trace_pc_guard
-    : Flag<["-"], "fsanitize-coverage-trace-pc-guard">,
-      HelpText<"Enable PC tracing with guard in sanitizer coverage">;
-def fsanitize_coverage_no_prune
-    : Flag<["-"], "fsanitize-coverage-no-prune">,
-      HelpText<"Disable coverage pruning (i.e. instrument all blocks/edges)">;
-def fsanitize_coverage_stack_depth
-    : Flag<["-"], "fsanitize-coverage-stack-depth">,
-      HelpText<"Enable max stack depth tracing">;
-def fpatchable_function_entry_offset_EQ
-    : Joined<["-"], "fpatchable-function-entry-offset=">, MetaVarName<"<M>">,
-      HelpText<"Generate M NOPs before function entry">;
-def fprofile_instrument_EQ : Joined<["-"], "fprofile-instrument=">,
-    HelpText<"Enable PGO instrumentation. The accepted value is clang, llvm, "
-             "or none">, Values<"none,clang,llvm">;
-def fprofile_instrument_path_EQ : Joined<["-"], "fprofile-instrument-path=">,
-    HelpText<"Generate instrumented code to collect execution counts into "
-             "<file> (overridden by LLVM_PROFILE_FILE env var)">;
-def fprofile_instrument_use_path_EQ :
-    Joined<["-"], "fprofile-instrument-use-path=">,
-    HelpText<"Specify the profile path in PGO use compilation">;
-def flto_visibility_public_std:
-    Flag<["-"], "flto-visibility-public-std">,
-    HelpText<"Use public LTO visibility for classes in std and stdext namespaces">;
-def flto_unit: Flag<["-"], "flto-unit">,
-    HelpText<"Emit IR to support LTO unit features (CFI, whole program vtable opt)">;
-def fno_lto_unit: Flag<["-"], "fno-lto-unit">;
-def fdebug_pass_manager : Flag<["-"], "fdebug-pass-manager">,
-    HelpText<"Prints debug information for the new pass manager">;
-def fno_debug_pass_manager : Flag<["-"], "fno-debug-pass-manager">,
-    HelpText<"Disables debug printing for the new pass manager">;
-// The driver option takes the key as a parameter to the -msign-return-address=
-// and -mbranch-protection= options, but CC1 has a separate option so we
-// don't have to parse the parameter twice.
-def msign_return_address_key_EQ : Joined<["-"], "msign-return-address-key=">,
-    Values<"a_key,b_key">;
-def mbranch_target_enforce : Flag<["-"], "mbranch-target-enforce">;
-def fno_dllexport_inlines : Flag<["-"], "fno-dllexport-inlines">;
-def cfguard_no_checks : Flag<["-"], "cfguard-no-checks">,
-    HelpText<"Emit Windows Control Flow Guard tables only (no checks)">;
-def cfguard : Flag<["-"], "cfguard">,
-    HelpText<"Emit Windows Control Flow Guard tables and checks">;
-
-def fdenormal_fp_math_f32_EQ : Joined<["-"], "fdenormal-fp-math-f32=">,
-   Group<f_Group>;
-
-//===----------------------------------------------------------------------===//
-// Dependency Output Options
-//===----------------------------------------------------------------------===//
-
-def sys_header_deps : Flag<["-"], "sys-header-deps">,
-  HelpText<"Include system headers in dependency output">;
-def module_file_deps : Flag<["-"], "module-file-deps">,
-  HelpText<"Include module files in dependency output">;
-def header_include_file : Separate<["-"], "header-include-file">,
-  HelpText<"Filename (or -) to write header include output to">;
-def show_includes : Flag<["--"], "show-includes">,
-  HelpText<"Print cl.exe style /showIncludes to stdout">;
-
-//===----------------------------------------------------------------------===//
-// Diagnostic Options
-//===----------------------------------------------------------------------===//
-
-def diagnostic_log_file : Separate<["-"], "diagnostic-log-file">,
-  HelpText<"Filename (or -) to log diagnostics to">;
-def diagnostic_serialized_file : Separate<["-"], "serialize-diagnostic-file">,
-  MetaVarName<"<filename>">,
-  HelpText<"File for serializing diagnostics in a binary format">;
-
-def fdiagnostics_format : Separate<["-"], "fdiagnostics-format">,
-  HelpText<"Change diagnostic formatting to match IDE and command line tools">, Values<"clang,msvc,msvc-fallback,vi">;
-def fdiagnostics_show_category : Separate<["-"], "fdiagnostics-show-category">,
-  HelpText<"Print diagnostic category">, Values<"none,id,name">;
-def fno_diagnostics_use_presumed_location : Flag<["-"], "fno-diagnostics-use-presumed-location">,
-  HelpText<"Ignore #line directives when displaying diagnostic locations">;
-def ftabstop : Separate<["-"], "ftabstop">, MetaVarName<"<N>">,
-  HelpText<"Set the tab stop distance.">;
-def ferror_limit : Separate<["-"], "ferror-limit">, MetaVarName<"<N>">,
-  HelpText<"Set the maximum number of errors to emit before stopping (0 = no limit).">;
-def fmacro_backtrace_limit : Separate<["-"], "fmacro-backtrace-limit">, MetaVarName<"<N>">,
-  HelpText<"Set the maximum number of entries to print in a macro expansion backtrace (0 = no limit).">;
-def ftemplate_backtrace_limit : Separate<["-"], "ftemplate-backtrace-limit">, MetaVarName<"<N>">,
-  HelpText<"Set the maximum number of entries to print in a template instantiation backtrace (0 = no limit).">;
-def fconstexpr_backtrace_limit : Separate<["-"], "fconstexpr-backtrace-limit">, MetaVarName<"<N>">,
-  HelpText<"Set the maximum number of entries to print in a constexpr evaluation backtrace (0 = no limit).">;
-def fspell_checking_limit : Separate<["-"], "fspell-checking-limit">, MetaVarName<"<N>">,
-  HelpText<"Set the maximum number of times to perform spell checking on unrecognized identifiers (0 = no limit).">;
-def fcaret_diagnostics_max_lines :
-  Separate<["-"], "fcaret-diagnostics-max-lines">, MetaVarName<"<N>">,
-  HelpText<"Set the maximum number of source lines to show in a caret diagnostic">;
-def verify_EQ : CommaJoined<["-"], "verify=">,
-  MetaVarName<"<prefixes>">,
-  HelpText<"Verify diagnostic output using comment directives that start with"
-           " prefixes in the comma-separated sequence <prefixes>">;
-def verify : Flag<["-"], "verify">,
-  HelpText<"Equivalent to -verify=expected">;
-def verify_ignore_unexpected : Flag<["-"], "verify-ignore-unexpected">,
-  HelpText<"Ignore unexpected diagnostic messages">;
-def verify_ignore_unexpected_EQ : CommaJoined<["-"], "verify-ignore-unexpected=">,
-  HelpText<"Ignore unexpected diagnostic messages">;
-def Wno_rewrite_macros : Flag<["-"], "Wno-rewrite-macros">,
-  HelpText<"Silence ObjC rewriting warnings">;
-
-//===----------------------------------------------------------------------===//
-// Frontend Options
-//===----------------------------------------------------------------------===//
-
-// This isn't normally used, it is just here so we can parse a
-// CompilerInvocation out of a driver-derived argument vector.
-def cc1 : Flag<["-"], "cc1">;
-def cc1as : Flag<["-"], "cc1as">;
-
-def ast_merge : Separate<["-"], "ast-merge">,
-  MetaVarName<"<ast file>">,
-  HelpText<"Merge the given AST file into the translation unit being compiled.">;
-def aux_target_cpu : Separate<["-"], "aux-target-cpu">,
-  HelpText<"Target a specific auxiliary cpu type">;
-def aux_target_feature : Separate<["-"], "aux-target-feature">,
-  HelpText<"Target specific auxiliary attributes">;
-def aux_triple : Separate<["-"], "aux-triple">,
-  HelpText<"Auxiliary target triple.">;
-def code_completion_at : Separate<["-"], "code-completion-at">,
-  MetaVarName<"<file>:<line>:<column>">,
-  HelpText<"Dump code-completion information at a location">;
-def remap_file : Separate<["-"], "remap-file">,
-  MetaVarName<"<from>;<to>">,
-  HelpText<"Replace the contents of the <from> file with the contents of the <to> file">;
-def code_completion_at_EQ : Joined<["-"], "code-completion-at=">,
-  Alias<code_completion_at>;
-def code_completion_macros : Flag<["-"], "code-completion-macros">,
-  HelpText<"Include macros in code-completion results">;
-def code_completion_patterns : Flag<["-"], "code-completion-patterns">,
-  HelpText<"Include code patterns in code-completion results">;
-def no_code_completion_globals : Flag<["-"], "no-code-completion-globals">,
-  HelpText<"Do not include global declarations in code-completion results.">;
-def no_code_completion_ns_level_decls : Flag<["-"], "no-code-completion-ns-level-decls">,
-  HelpText<"Do not include declarations inside namespaces (incl. global namespace) in the code-completion results.">;
-def code_completion_brief_comments : Flag<["-"], "code-completion-brief-comments">,
-  HelpText<"Include brief documentation comments in code-completion results.">;
-def code_completion_with_fixits : Flag<["-"], "code-completion-with-fixits">,
-  HelpText<"Include code completion results which require small fix-its.">;
-def disable_free : Flag<["-"], "disable-free">,
-  HelpText<"Disable freeing of memory on exit">;
-def discard_value_names : Flag<["-"], "discard-value-names">,
-  HelpText<"Discard value names in LLVM IR">;
-def load : Separate<["-"], "load">, MetaVarName<"<dsopath>">,
-  HelpText<"Load the named plugin (dynamic shared object)">;
-def plugin : Separate<["-"], "plugin">, MetaVarName<"<name>">,
-  HelpText<"Use the named plugin action instead of the default action (use \"help\" to list available options)">;
-def plugin_arg : JoinedAndSeparate<["-"], "plugin-arg-">,
-    MetaVarName<"<name> <arg>">,
-    HelpText<"Pass <arg> to plugin <name>">;
-def add_plugin : Separate<["-"], "add-plugin">, MetaVarName<"<name>">,
-  HelpText<"Use the named plugin action in addition to the default action">;
-def ast_dump_filter : Separate<["-"], "ast-dump-filter">,
-  MetaVarName<"<dump_filter>">,
-  HelpText<"Use with -ast-dump or -ast-print to dump/print only AST declaration"
-           " nodes having a certain substring in a qualified name. Use"
-           " -ast-list to list all filterable declaration node names.">;
-def fno_modules_global_index : Flag<["-"], "fno-modules-global-index">,
-  HelpText<"Do not automatically generate or update the global module index">;
-def fno_modules_error_recovery : Flag<["-"], "fno-modules-error-recovery">,
-  HelpText<"Do not automatically import modules for error recovery">;
-def fmodule_map_file_home_is_cwd : Flag<["-"], "fmodule-map-file-home-is-cwd">,
-  HelpText<"Use the current working directory as the home directory of "
-           "module maps specified by -fmodule-map-file=<FILE>">;
-def fmodule_feature : Separate<["-"], "fmodule-feature">,
-  MetaVarName<"<feature>">,
-  HelpText<"Enable <feature> in module map requires declarations">;
-def fmodules_embed_file_EQ : Joined<["-"], "fmodules-embed-file=">,
-  MetaVarName<"<file>">,
-  HelpText<"Embed the contents of the specified file into the module file "
-           "being compiled.">;
-def fmodules_embed_all_files : Joined<["-"], "fmodules-embed-all-files">,
-  HelpText<"Embed the contents of all files read by this compilation into "
-           "the produced module file.">;
-def fmodules_local_submodule_visibility :
-  Flag<["-"], "fmodules-local-submodule-visibility">,
-  HelpText<"Enforce name visibility rules across submodules of the same "
-           "top-level module.">;
-def fmodules_codegen :
-  Flag<["-"], "fmodules-codegen">,
-  HelpText<"Generate code for uses of this module that assumes an explicit "
-           "object file will be built for the module">;
-def fmodules_debuginfo :
-  Flag<["-"], "fmodules-debuginfo">,
-  HelpText<"Generate debug info for types in an object file built from this "
-           "module and do not generate them elsewhere">;
-def fmodule_format_EQ : Joined<["-"], "fmodule-format=">,
-  HelpText<"Select the container format for clang modules and PCH. "
-           "Supported options are 'raw' and 'obj'.">;
-def ftest_module_file_extension_EQ :
-  Joined<["-"], "ftest-module-file-extension=">,
-  HelpText<"introduce a module file extension for testing purposes. "
-           "The argument is parsed as blockname:major:minor:hashed:user info">;
-def fconcepts_ts : Flag<["-"], "fconcepts-ts">,
-  HelpText<"Enable C++ Extensions for Concepts. (deprecated - use -std=c++2a)">;
-def fno_concept_satisfaction_caching : Flag<["-"],
-                                            "fno-concept-satisfaction-caching">,
-  HelpText<"Disable satisfaction caching for C++2a Concepts.">;
-
-def frecovery_ast : Flag<["-"], "frecovery-ast">,
-  HelpText<"Preserve expressions in AST rather than dropping them when "
-           "encountering semantic errors">;
-def fno_recovery_ast : Flag<["-"], "fno-recovery-ast">;
-def frecovery_ast_type : Flag<["-"], "frecovery-ast-type">,
-  HelpText<"Preserve the type for recovery expressions when possible "
-          "(experimental)">;
-def fno_recovery_ast_type : Flag<["-"], "fno-recovery-ast-type">;
-
-let Group = Action_Group in {
-
-def Eonly : Flag<["-"], "Eonly">,
-  HelpText<"Just run preprocessor, no output (for timings)">;
-def dump_raw_tokens : Flag<["-"], "dump-raw-tokens">,
-  HelpText<"Lex file in raw mode and dump raw tokens">;
-def analyze : Flag<["-"], "analyze">,
-  HelpText<"Run static analysis engine">;
-def dump_tokens : Flag<["-"], "dump-tokens">,
-  HelpText<"Run preprocessor, dump internal rep of tokens">;
-def init_only : Flag<["-"], "init-only">,
-  HelpText<"Only execute frontend initialization">;
-def fixit : Flag<["-"], "fixit">,
-  HelpText<"Apply fix-it advice to the input source">;
-def fixit_EQ : Joined<["-"], "fixit=">,
-  HelpText<"Apply fix-it advice creating a file with the given suffix">;
-def print_preamble : Flag<["-"], "print-preamble">,
-  HelpText<"Print the \"preamble\" of a file, which is a candidate for implicit"
-           " precompiled headers.">;
-def emit_html : Flag<["-"], "emit-html">,
-  HelpText<"Output input source as HTML">;
-def ast_print : Flag<["-"], "ast-print">,
-  HelpText<"Build ASTs and then pretty-print them">;
-def ast_list : Flag<["-"], "ast-list">,
-  HelpText<"Build ASTs and print the list of declaration node qualified names">;
-def ast_dump : Flag<["-"], "ast-dump">,
-  HelpText<"Build ASTs and then debug dump them">;
-def ast_dump_EQ : Joined<["-"], "ast-dump=">,
-  HelpText<"Build ASTs and then debug dump them in the specified format. "
-           "Supported formats include: default, json">;
-def ast_dump_all : Flag<["-"], "ast-dump-all">,
-  HelpText<"Build ASTs and then debug dump them, forcing deserialization">;
-def ast_dump_all_EQ : Joined<["-"], "ast-dump-all=">,
-  HelpText<"Build ASTs and then debug dump them in the specified format, "
-           "forcing deserialization. Supported formats include: default, json">;
-def templight_dump : Flag<["-"], "templight-dump">,
-  HelpText<"Dump templight information to stdout">;
-def ast_dump_lookups : Flag<["-"], "ast-dump-lookups">,
-  HelpText<"Build ASTs and then debug dump their name lookup tables">;
-def ast_view : Flag<["-"], "ast-view">,
-  HelpText<"Build ASTs and view them with GraphViz">;
-def emit_module : Flag<["-"], "emit-module">,
-  HelpText<"Generate pre-compiled module file from a module map">;
-def emit_module_interface : Flag<["-"], "emit-module-interface">,
-  HelpText<"Generate pre-compiled module file from a C++ module interface">;
-def emit_header_module : Flag<["-"], "emit-header-module">,
-  HelpText<"Generate pre-compiled module file from a set of header files">;
-def emit_pch : Flag<["-"], "emit-pch">,
-  HelpText<"Generate pre-compiled header file">;
-def emit_llvm_bc : Flag<["-"], "emit-llvm-bc">,
-  HelpText<"Build ASTs then convert to LLVM, emit .bc file">;
-def emit_llvm_only : Flag<["-"], "emit-llvm-only">,
-  HelpText<"Build ASTs and convert to LLVM, discarding output">;
-def emit_codegen_only : Flag<["-"], "emit-codegen-only">,
-  HelpText<"Generate machine code, but discard output">;
-def emit_obj : Flag<["-"], "emit-obj">,
-  HelpText<"Emit native object files">;
-def rewrite_test : Flag<["-"], "rewrite-test">,
-  HelpText<"Rewriter playground">;
-def rewrite_macros : Flag<["-"], "rewrite-macros">,
-  HelpText<"Expand macros without full preprocessing">;
-def migrate : Flag<["-"], "migrate">,
-  HelpText<"Migrate source code">;
-def compiler_options_dump : Flag<["-"], "compiler-options-dump">,
-  HelpText<"Dump the compiler configuration options">;
-def print_dependency_directives_minimized_source : Flag<["-"],
-  "print-dependency-directives-minimized-source">,
-  HelpText<"Print the output of the dependency directives source minimizer">;
-}
-
-def emit_llvm_uselists : Flag<["-"], "emit-llvm-uselists">,
-  HelpText<"Preserve order of LLVM use-lists when serializing">;
-def no_emit_llvm_uselists : Flag<["-"], "no-emit-llvm-uselists">,
-  HelpText<"Don't preserve order of LLVM use-lists when serializing">;
-
-def mt_migrate_directory : Separate<["-"], "mt-migrate-directory">,
-  HelpText<"Directory for temporary files produced during ARC or ObjC migration">;
-def arcmt_check : Flag<["-"], "arcmt-check">,
-  HelpText<"Check for ARC migration issues that need manual handling">;
-def arcmt_modify : Flag<["-"], "arcmt-modify">,
-  HelpText<"Apply modifications to files to conform to ARC">;
-def arcmt_migrate : Flag<["-"], "arcmt-migrate">,
-  HelpText<"Apply modifications and produces temporary files that conform to ARC">;
-
-def opt_record_file : Separate<["-"], "opt-record-file">,
-  HelpText<"File name to use for YAML optimization record output">;
-def opt_record_passes : Separate<["-"], "opt-record-passes">,
-  HelpText<"Only record remark information for passes whose names match the given regular expression">;
-def opt_record_format : Separate<["-"], "opt-record-format">,
-  HelpText<"The format used for serializing remarks (default: YAML)">;
-
-def print_stats : Flag<["-"], "print-stats">,
-  HelpText<"Print performance metrics and statistics">;
-def stats_file : Joined<["-"], "stats-file=">,
-  HelpText<"Filename to write statistics to">;
-def fdump_record_layouts : Flag<["-"], "fdump-record-layouts">,
-  HelpText<"Dump record layout information">;
-def fdump_record_layouts_simple : Flag<["-"], "fdump-record-layouts-simple">,
-  HelpText<"Dump record layout information in a simple form used for testing">;
-def fix_what_you_can : Flag<["-"], "fix-what-you-can">,
-  HelpText<"Apply fix-it advice even in the presence of unfixable errors">;
-def fix_only_warnings : Flag<["-"], "fix-only-warnings">,
-  HelpText<"Apply fix-it advice only for warnings, not errors">;
-def fixit_recompile : Flag<["-"], "fixit-recompile">,
-  HelpText<"Apply fix-it changes and recompile">;
-def fixit_to_temp : Flag<["-"], "fixit-to-temporary">,
-  HelpText<"Apply fix-it changes to temporary files">;
-
-def foverride_record_layout_EQ : Joined<["-"], "foverride-record-layout=">,
-  HelpText<"Override record layouts with those in the given file">;
-def pch_through_header_EQ : Joined<["-"], "pch-through-header=">,
-  HelpText<"Stop PCH generation after including this file.  When using a PCH, "
-           "skip tokens until after this file is included.">;
-def pch_through_hdrstop_create : Flag<["-"], "pch-through-hdrstop-create">,
-  HelpText<"When creating a PCH, stop PCH generation after #pragma hdrstop.">;
-def pch_through_hdrstop_use : Flag<["-"], "pch-through-hdrstop-use">,
-  HelpText<"When using a PCH, skip tokens until after a #pragma hdrstop.">;
-def fno_pch_timestamp : Flag<["-"], "fno-pch-timestamp">,
-  HelpText<"Disable inclusion of timestamp in precompiled headers">;
-def building_pch_with_obj : Flag<["-"], "building-pch-with-obj">,
-  HelpText<"This compilation is part of building a PCH with corresponding object file.">;
-
-def aligned_alloc_unavailable : Flag<["-"], "faligned-alloc-unavailable">,
-  HelpText<"Aligned allocation/deallocation functions are unavailable">;
-
-//===----------------------------------------------------------------------===//
-// Language Options
-//===----------------------------------------------------------------------===//
-
-let Flags = [CC1Option, CC1AsOption, NoDriverOption] in {
-
-def version : Flag<["-"], "version">,
-  HelpText<"Print the compiler version">;
-def main_file_name : Separate<["-"], "main-file-name">,
-  HelpText<"Main file name to use for debug info and source if missing">;
-def split_dwarf_output : Separate<["-"], "split-dwarf-output">,
-  HelpText<"File name to use for split dwarf debug info output">;
-
-}
-
-def fblocks_runtime_optional : Flag<["-"], "fblocks-runtime-optional">,
-  HelpText<"Weakly link in the blocks runtime">;
-def fexternc_nounwind : Flag<["-"], "fexternc-nounwind">,
-  HelpText<"Assume all functions with C linkage do not unwind">;
-def split_dwarf_file : Separate<["-"], "split-dwarf-file">,
-  HelpText<"Name of the split dwarf debug info file to encode in the object file">;
-def fno_wchar : Flag<["-"], "fno-wchar">,
-  HelpText<"Disable C++ builtin type wchar_t">;
-def fconstant_string_class : Separate<["-"], "fconstant-string-class">,
-  MetaVarName<"<class name>">,
-  HelpText<"Specify the class to use for constant Objective-C string objects.">;
-def fobjc_arc_cxxlib_EQ : Joined<["-"], "fobjc-arc-cxxlib=">,
-  HelpText<"Objective-C++ Automatic Reference Counting standard library kind">, Values<"libc++,libstdc++,none">;
-def fobjc_runtime_has_weak : Flag<["-"], "fobjc-runtime-has-weak">,
-  HelpText<"The target Objective-C runtime supports ARC weak operations">;
-def fobjc_dispatch_method_EQ : Joined<["-"], "fobjc-dispatch-method=">,
-  HelpText<"Objective-C dispatch method to use">, Values<"legacy,non-legacy,mixed">;
-def disable_objc_default_synthesize_properties : Flag<["-"], "disable-objc-default-synthesize-properties">,
-  HelpText<"disable the default synthesis of Objective-C properties">;
-def fencode_extended_block_signature : Flag<["-"], "fencode-extended-block-signature">,
-  HelpText<"enable extended encoding of block type signature">;
-def function_alignment : Separate<["-"], "function-alignment">,
-    HelpText<"default alignment for functions">;
-def pic_level : Separate<["-"], "pic-level">,
-  HelpText<"Value for __PIC__">;
-def pic_is_pie : Flag<["-"], "pic-is-pie">,
-  HelpText<"File is for a position independent executable">;
-def fno_validate_pch : Flag<["-"], "fno-validate-pch">,
-  HelpText<"Disable validation of precompiled headers">;
-def fallow_pch_with_errors : Flag<["-"], "fallow-pch-with-compiler-errors">,
-  HelpText<"Accept a PCH file that was created with compiler errors">;
-def dump_deserialized_pch_decls : Flag<["-"], "dump-deserialized-decls">,
-  HelpText<"Dump declarations that are deserialized from PCH, for testing">;
-def error_on_deserialized_pch_decl : Separate<["-"], "error-on-deserialized-decl">,
-  HelpText<"Emit error if a specific declaration is deserialized from PCH, for testing">;
-def error_on_deserialized_pch_decl_EQ : Joined<["-"], "error-on-deserialized-decl=">,
-  Alias<error_on_deserialized_pch_decl>;
-def static_define : Flag<["-"], "static-define">,
-  HelpText<"Should __STATIC__ be defined">;
-def stack_protector : Separate<["-"], "stack-protector">,
-  HelpText<"Enable stack protectors">;
-def stack_protector_buffer_size : Separate<["-"], "stack-protector-buffer-size">,
-  HelpText<"Lower bound for a buffer to be considered for stack protection">;
-def fvisibility : Separate<["-"], "fvisibility">,
-  HelpText<"Default type and symbol visibility">;
-def ftype_visibility : Separate<["-"], "ftype-visibility">,
-  HelpText<"Default type visibility">;
-def fapply_global_visibility_to_externs : Flag<["-"], "fapply-global-visibility-to-externs">,
-  HelpText<"Apply global symbol visibility to external declarations without an explicit visibility">;
-def ftemplate_depth : Separate<["-"], "ftemplate-depth">,
-  HelpText<"Maximum depth of recursive template instantiation">;
-def foperator_arrow_depth : Separate<["-"], "foperator-arrow-depth">,
-  HelpText<"Maximum number of 'operator->'s to call for a member access">;
-def fconstexpr_depth : Separate<["-"], "fconstexpr-depth">,
-  HelpText<"Maximum depth of recursive constexpr function calls">;
-def fconstexpr_steps : Separate<["-"], "fconstexpr-steps">,
-  HelpText<"Maximum number of steps in constexpr function evaluation">;
-def fbracket_depth : Separate<["-"], "fbracket-depth">,
-  HelpText<"Maximum nesting level for parentheses, brackets, and braces">;
-def fconst_strings : Flag<["-"], "fconst-strings">,
-  HelpText<"Use a const qualified type for string literals in C and ObjC">;
-def fno_const_strings : Flag<["-"], "fno-const-strings">,
-  HelpText<"Don't use a const qualified type for string literals in C and ObjC">;
-def fno_bitfield_type_align : Flag<["-"], "fno-bitfield-type-align">,
-  HelpText<"Ignore bit-field types when aligning structures">;
-def ffake_address_space_map : Flag<["-"], "ffake-address-space-map">,
-  HelpText<"Use a fake address space map; OpenCL testing purposes only">;
-def faddress_space_map_mangling_EQ : Joined<["-"], "faddress-space-map-mangling=">, MetaVarName<"<yes|no|target>">,
-  HelpText<"Set the mode for address space map based mangling; OpenCL testing purposes only">;
-def funknown_anytype : Flag<["-"], "funknown-anytype">,
-  HelpText<"Enable parser support for the __unknown_anytype type; for testing purposes only">;
-def fdebugger_support : Flag<["-"], "fdebugger-support">,
-  HelpText<"Enable special debugger support behavior">;
-def fdebugger_cast_result_to_id : Flag<["-"], "fdebugger-cast-result-to-id">,
-  HelpText<"Enable casting unknown expression results to id">;
-def fdebugger_objc_literal : Flag<["-"], "fdebugger-objc-literal">,
-  HelpText<"Enable special debugger support for Objective-C subscripting and literals">;
-def fdeprecated_macro : Flag<["-"], "fdeprecated-macro">,
-  HelpText<"Defines the __DEPRECATED macro">;
-def fno_deprecated_macro : Flag<["-"], "fno-deprecated-macro">,
-  HelpText<"Undefines the __DEPRECATED macro">;
-def fobjc_subscripting_legacy_runtime : Flag<["-"], "fobjc-subscripting-legacy-runtime">,
-  HelpText<"Allow Objective-C array and dictionary subscripting in legacy runtime">;
-def vtordisp_mode_EQ : Joined<["-"], "vtordisp-mode=">,
-  HelpText<"Control vtordisp placement on win32 targets">;
-def fnative_half_type: Flag<["-"], "fnative-half-type">,
-  HelpText<"Use the native half type for __fp16 instead of promoting to float">;
-def fnative_half_arguments_and_returns : Flag<["-"], "fnative-half-arguments-and-returns">,
-  HelpText<"Use the native __fp16 type for arguments and returns (and skip ABI-specific lowering)">;
-def fallow_half_arguments_and_returns : Flag<["-"], "fallow-half-arguments-and-returns">,
-  HelpText<"Allow function arguments and returns of type half">;
-def fdefault_calling_conv_EQ : Joined<["-"], "fdefault-calling-conv=">,
-  HelpText<"Set default calling convention">, Values<"cdecl,fastcall,stdcall,vectorcall,regcall">;
-def finclude_default_header : Flag<["-"], "finclude-default-header">,
-  HelpText<"Include default header file for OpenCL">;
-def fdeclare_opencl_builtins : Flag<["-"], "fdeclare-opencl-builtins">,
-  HelpText<"Add OpenCL builtin function declarations (experimental)">;
-def fpreserve_vec3_type : Flag<["-"], "fpreserve-vec3-type">,
-  HelpText<"Preserve 3-component vector type">;
-def fwchar_type_EQ : Joined<["-"], "fwchar-type=">,
-  HelpText<"Select underlying type for wchar_t">, Values<"char,short,int">;
-def fsigned_wchar : Flag<["-"], "fsigned-wchar">,
-  HelpText<"Use a signed type for wchar_t">;
-def fno_signed_wchar : Flag<["-"], "fno-signed-wchar">,
-  HelpText<"Use an unsigned type for wchar_t">;
-def fcompatibility_qualified_id_block_param_type_checking : Flag<["-"], "fcompatibility-qualified-id-block-type-checking">,
-  HelpText<"Allow using blocks with parameters of more specific type than "
-           "the type system guarantees when a parameter is qualified id">;
-
-// FIXME: Remove these entirely once functionality/tests have been excised.
-def fobjc_gc_only : Flag<["-"], "fobjc-gc-only">, Group<f_Group>,
-  HelpText<"Use GC exclusively for Objective-C related memory management">;
-def fobjc_gc : Flag<["-"], "fobjc-gc">, Group<f_Group>,
-  HelpText<"Enable Objective-C garbage collection">;
-
-//===----------------------------------------------------------------------===//
-// Header Search Options
-//===----------------------------------------------------------------------===//
-
-def nostdsysteminc : Flag<["-"], "nostdsysteminc">,
-  HelpText<"Disable standard system #include directories">;
-def fdisable_module_hash : Flag<["-"], "fdisable-module-hash">,
-  HelpText<"Disable the module hash">;
-def fmodules_hash_content : Flag<["-"], "fmodules-hash-content">,
-  HelpText<"Enable hashing the content of a module file">;
-def fmodules_strict_context_hash : Flag<["-"], "fmodules-strict-context-hash">,
-  HelpText<"Enable hashing of all compiler options that could impact the "
-           "semantics of a module in an implicit build">;
-def c_isystem : JoinedOrSeparate<["-"], "c-isystem">, MetaVarName<"<directory>">,
-  HelpText<"Add directory to the C SYSTEM include search path">;
-def objc_isystem : JoinedOrSeparate<["-"], "objc-isystem">,
-  MetaVarName<"<directory>">,
-  HelpText<"Add directory to the ObjC SYSTEM include search path">;
-def objcxx_isystem : JoinedOrSeparate<["-"], "objcxx-isystem">,
-  MetaVarName<"<directory>">,
-  HelpText<"Add directory to the ObjC++ SYSTEM include search path">;
-def internal_isystem : JoinedOrSeparate<["-"], "internal-isystem">,
-  MetaVarName<"<directory>">,
-  HelpText<"Add directory to the internal system include search path; these "
-           "are assumed to not be user-provided and are used to model system "
-           "and standard headers' paths.">;
-def internal_externc_isystem : JoinedOrSeparate<["-"], "internal-externc-isystem">,
-  MetaVarName<"<directory>">,
-  HelpText<"Add directory to the internal system include search path with "
-           "implicit extern \"C\" semantics; these are assumed to not be "
-           "user-provided and are used to model system and standard headers' "
-           "paths.">;
-
-//===----------------------------------------------------------------------===//
-// Preprocessor Options
-//===----------------------------------------------------------------------===//
-
-def chain_include : Separate<["-"], "chain-include">, MetaVarName<"<file>">,
-  HelpText<"Include and chain a header file after turning it into PCH">;
-def preamble_bytes_EQ : Joined<["-"], "preamble-bytes=">,
-  HelpText<"Assume that the precompiled header is a precompiled preamble "
-           "covering the first N bytes of the main file">;
-def detailed_preprocessing_record : Flag<["-"], "detailed-preprocessing-record">,
-  HelpText<"include a detailed record of preprocessing actions">;
-def setup_static_analyzer : Flag<["-"], "setup-static-analyzer">,
-  HelpText<"Set up preprocessor for static analyzer (done automatically when static analyzer is run).">;
-def disable_pragma_debug_crash : Flag<["-"], "disable-pragma-debug-crash">,
-  HelpText<"Disable any #pragma clang __debug that can lead to crashing behavior. This is meant for testing.">;
-
-//===----------------------------------------------------------------------===//
-// OpenCL Options
-//===----------------------------------------------------------------------===//
-
-def cl_ext_EQ : CommaJoined<["-"], "cl-ext=">,
-  HelpText<"OpenCL only. Enable or disable OpenCL extensions. The argument is a comma-separated sequence of one or more extension names, each prefixed by '+' or '-'.">;
-
-//===----------------------------------------------------------------------===//
-// CUDA Options
-//===----------------------------------------------------------------------===//
-
-def fcuda_is_device : Flag<["-"], "fcuda-is-device">,
-  HelpText<"Generate code for CUDA device">;
-def fcuda_include_gpubinary : Separate<["-"], "fcuda-include-gpubinary">,
-  HelpText<"Incorporate CUDA device-side binary into host object file.">;
-def fcuda_allow_variadic_functions : Flag<["-"], "fcuda-allow-variadic-functions">,
-  HelpText<"Allow variadic functions in CUDA device code.">;
-def fno_cuda_host_device_constexpr : Flag<["-"], "fno-cuda-host-device-constexpr">,
-  HelpText<"Don't treat unattributed constexpr functions as __host__ __device__.">;
-
-//===----------------------------------------------------------------------===//
-// OpenMP Options
-//===----------------------------------------------------------------------===//
-
-def fopenmp_is_device : Flag<["-"], "fopenmp-is-device">,
-  HelpText<"Generate code only for an OpenMP target device.">;
-def fopenmp_host_ir_file_path : Separate<["-"], "fopenmp-host-ir-file-path">,
-  HelpText<"Path to the IR file produced by the frontend for the host.">;
-
-//===----------------------------------------------------------------------===//
-// SYCL Options
-//===----------------------------------------------------------------------===//
-
-def fsycl_is_device : Flag<["-"], "fsycl-is-device">,
-  HelpText<"Generate code for SYCL device.">;
-
-} // let Flags = [CC1Option]
-
-//===----------------------------------------------------------------------===//
-// cc1as-only Options
-//===----------------------------------------------------------------------===//
-
-let Flags = [CC1AsOption, NoDriverOption] in {
-
-// Language Options
-def n : Flag<["-"], "n">,
-  HelpText<"Don't automatically start assembly file with a text section">;
-
-// Frontend Options
-def filetype : Separate<["-"], "filetype">,
-    HelpText<"Specify the output file type ('asm', 'null', or 'obj')">;
-
-// Transliterate Options
-def output_asm_variant : Separate<["-"], "output-asm-variant">,
-    HelpText<"Select the asm variant index to use for output">;
-def show_encoding : Flag<["-"], "show-encoding">,
-    HelpText<"Show instruction encoding information in transliterate mode">;
-def show_inst : Flag<["-"], "show-inst">,
-    HelpText<"Show internal instruction representation in transliterate mode">;
-
-// Assemble Options
-def dwarf_debug_producer : Separate<["-"], "dwarf-debug-producer">,
-  HelpText<"The string to embed in the Dwarf debug AT_producer record.">;
-
-def defsym : Separate<["-"], "defsym">,
-  HelpText<"Define a value for a symbol">;
-} // let Flags = [CC1AsOption]
diff --git a/clang/include/clang/Driver/CLCompatOptions.td b/clang/include/clang/Driver/CLCompatOptions.td
deleted file mode 100644
index 17d248a3c5ad9..0000000000000
--- a/clang/include/clang/Driver/CLCompatOptions.td
+++ /dev/null
@@ -1,470 +0,0 @@
-//===--- CLCompatOptions.td - Options for clang-cl ------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines the options accepted by clang-cl.
-//
-//===----------------------------------------------------------------------===//
-
-def cl_Group : OptionGroup<"<clang-cl options>">, Flags<[CLOption]>,
-  HelpText<"CL.EXE COMPATIBILITY OPTIONS">;
-
-def cl_compile_Group : OptionGroup<"<clang-cl compile-only options>">,
-  Group<cl_Group>;
-
-def cl_ignored_Group : OptionGroup<"<clang-cl ignored options>">,
-  Group<cl_Group>;
-
-class CLFlag<string name> : Option<["/", "-"], name, KIND_FLAG>,
-  Group<cl_Group>, Flags<[CLOption, DriverOption]>;
-
-class CLCompileFlag<string name> : Option<["/", "-"], name, KIND_FLAG>,
-  Group<cl_compile_Group>, Flags<[CLOption, DriverOption]>;
-
-class CLIgnoredFlag<string name> : Option<["/", "-"], name, KIND_FLAG>,
-  Group<cl_ignored_Group>, Flags<[CLOption, DriverOption]>;
-
-class CLJoined<string name> : Option<["/", "-"], name, KIND_JOINED>,
-  Group<cl_Group>, Flags<[CLOption, DriverOption]>;
-
-class CLCompileJoined<string name> : Option<["/", "-"], name, KIND_JOINED>,
-  Group<cl_compile_Group>, Flags<[CLOption, DriverOption]>;
-
-class CLIgnoredJoined<string name> : Option<["/", "-"], name, KIND_JOINED>,
-  Group<cl_ignored_Group>, Flags<[CLOption, DriverOption, HelpHidden]>;
-
-class CLJoinedOrSeparate<string name> : Option<["/", "-"], name,
-  KIND_JOINED_OR_SEPARATE>, Group<cl_Group>, Flags<[CLOption, DriverOption]>;
-
-class CLCompileJoinedOrSeparate<string name> : Option<["/", "-"], name,
-  KIND_JOINED_OR_SEPARATE>, Group<cl_compile_Group>,
-  Flags<[CLOption, DriverOption]>;
-
-class CLRemainingArgsJoined<string name> : Option<["/", "-"], name,
-  KIND_REMAINING_ARGS_JOINED>, Group<cl_Group>, Flags<[CLOption, DriverOption]>;
-
-// Aliases:
-// (We don't put any of these in cl_compile_Group as the options they alias are
-// already in the right group.)
-
-def _SLASH_Brepro : CLFlag<"Brepro">,
-  HelpText<"Do not write current time into COFF output (breaks link.exe /incremental)">,
-  Alias<mno_incremental_linker_compatible>;
-def _SLASH_Brepro_ : CLFlag<"Brepro-">,
-  HelpText<"Write current time into COFF output (default)">,
-  Alias<mincremental_linker_compatible>;
-def _SLASH_C : CLFlag<"C">,
-  HelpText<"Do not discard comments when preprocessing">, Alias<C>;
-def _SLASH_c : CLFlag<"c">, HelpText<"Compile only">, Alias<c>;
-def _SLASH_d1PP : CLFlag<"d1PP">,
-  HelpText<"Retain macro definitions in /E mode">, Alias<dD>;
-def _SLASH_d1reportAllClassLayout : CLFlag<"d1reportAllClassLayout">,
-  HelpText<"Dump record layout information">,
-  Alias<Xclang>, AliasArgs<["-fdump-record-layouts"]>;
-def _SLASH_diagnostics_caret : CLFlag<"diagnostics:caret">,
-  HelpText<"Enable caret and column diagnostics (default)">;
-def _SLASH_diagnostics_column : CLFlag<"diagnostics:column">,
-  HelpText<"Disable caret diagnostics but keep column info">;
-def _SLASH_diagnostics_classic : CLFlag<"diagnostics:classic">,
-  HelpText<"Disable column and caret diagnostics">;
-def _SLASH_D : CLJoinedOrSeparate<"D">, HelpText<"Define macro">,
-  MetaVarName<"<macro[=value]>">, Alias<D>;
-def _SLASH_E : CLFlag<"E">, HelpText<"Preprocess to stdout">, Alias<E>;
-def _SLASH_fp_except : CLFlag<"fp:except">, HelpText<"">, Alias<ftrapping_math>;
-def _SLASH_fp_except_ : CLFlag<"fp:except-">,
-  HelpText<"">, Alias<fno_trapping_math>;
-def _SLASH_fp_fast : CLFlag<"fp:fast">, HelpText<"">, Alias<ffast_math>;
-def _SLASH_fp_precise : CLFlag<"fp:precise">,
-  HelpText<"">, Alias<fno_fast_math>;
-def _SLASH_fp_strict : CLFlag<"fp:strict">, HelpText<"">, Alias<fno_fast_math>;
-def _SLASH_GA : CLFlag<"GA">, Alias<ftlsmodel_EQ>, AliasArgs<["local-exec"]>,
-  HelpText<"Assume thread-local variables are defined in the executable">;
-def _SLASH_GR : CLFlag<"GR">, HelpText<"Emit RTTI data (default)">;
-def _SLASH_GR_ : CLFlag<"GR-">, HelpText<"Do not emit RTTI data">;
-def _SLASH_GF : CLIgnoredFlag<"GF">,
-  HelpText<"Enable string pooling (default)">;
-def _SLASH_GF_ : CLFlag<"GF-">, HelpText<"Disable string pooling">,
-  Alias<fwritable_strings>;
-def _SLASH_GS : CLFlag<"GS">,
-  HelpText<"Enable buffer security check (default)">;
-def _SLASH_GS_ : CLFlag<"GS-">, HelpText<"Disable buffer security check">;
-def : CLFlag<"Gs">, HelpText<"Use stack probes (default)">,
-  Alias<mstack_probe_size>, AliasArgs<["4096"]>;
-def _SLASH_Gs : CLJoined<"Gs">,
-  HelpText<"Set stack probe size (default 4096)">, Alias<mstack_probe_size>;
-def _SLASH_Gy : CLFlag<"Gy">, HelpText<"Put each function in its own section">,
-  Alias<ffunction_sections>;
-def _SLASH_Gy_ : CLFlag<"Gy-">,
-  HelpText<"Do not put each function in its own section (default)">,
-  Alias<fno_function_sections>;
-def _SLASH_Gw : CLFlag<"Gw">, HelpText<"Put each data item in its own section">,
-  Alias<fdata_sections>;
-def _SLASH_Gw_ : CLFlag<"Gw-">,
-  HelpText<"Do not put each data item in its own section (default)">,
-  Alias<fno_data_sections>;
-def _SLASH_help : CLFlag<"help">, Alias<help>,
-  HelpText<"Display available options">;
-def _SLASH_HELP : CLFlag<"HELP">, Alias<help>;
-def _SLASH_I : CLJoinedOrSeparate<"I">,
-  HelpText<"Add directory to include search path">, MetaVarName<"<dir>">,
-  Alias<I>;
-def _SLASH_J : CLFlag<"J">, HelpText<"Make char type unsigned">,
-  Alias<funsigned_char>;
-
-// The _SLASH_O option handles all the /O flags, but we also provide separate
-// aliased options to provide separate help messages.
-def _SLASH_O : CLJoined<"O">,
-  HelpText<"Set multiple /O flags at once; e.g. '/O2y-' for '/O2 /Oy-'">,
-  MetaVarName<"<flags>">;
-def : CLFlag<"O1">, Alias<_SLASH_O>, AliasArgs<["1"]>,
-  HelpText<"Optimize for size  (like /Og     /Os /Oy /Ob2 /GF /Gy)">;
-def : CLFlag<"O2">, Alias<_SLASH_O>, AliasArgs<["2"]>,
-  HelpText<"Optimize for speed (like /Og /Oi /Ot /Oy /Ob2 /GF /Gy)">;
-def : CLFlag<"Ob0">, Alias<_SLASH_O>, AliasArgs<["b0"]>,
-  HelpText<"Disable function inlining">;
-def : CLFlag<"Ob1">, Alias<_SLASH_O>, AliasArgs<["b1"]>,
-  HelpText<"Only inline functions explicitly or implicitly marked inline">;
-def : CLFlag<"Ob2">, Alias<_SLASH_O>, AliasArgs<["b2"]>,
-  HelpText<"Inline functions as deemed beneficial by the compiler">;
-def : CLFlag<"Od">, Alias<_SLASH_O>, AliasArgs<["d"]>,
-  HelpText<"Disable optimization">;
-def : CLFlag<"Og">, Alias<_SLASH_O>, AliasArgs<["g"]>,
-  HelpText<"No effect">;
-def : CLFlag<"Oi">, Alias<_SLASH_O>, AliasArgs<["i"]>,
-  HelpText<"Enable use of builtin functions">;
-def : CLFlag<"Oi-">, Alias<_SLASH_O>, AliasArgs<["i-"]>,
-  HelpText<"Disable use of builtin functions">;
-def : CLFlag<"Os">, Alias<_SLASH_O>, AliasArgs<["s"]>,
-  HelpText<"Optimize for size">;
-def : CLFlag<"Ot">, Alias<_SLASH_O>, AliasArgs<["t"]>,
-  HelpText<"Optimize for speed">;
-def : CLFlag<"Ox">, Alias<_SLASH_O>, AliasArgs<["x"]>,
-  HelpText<"Deprecated (like /Og /Oi /Ot /Oy /Ob2); use /O2">;
-def : CLFlag<"Oy">, Alias<_SLASH_O>, AliasArgs<["y"]>,
-  HelpText<"Enable frame pointer omission (x86 only)">;
-def : CLFlag<"Oy-">, Alias<_SLASH_O>, AliasArgs<["y-"]>,
-  HelpText<"Disable frame pointer omission (x86 only, default)">;
-
-def _SLASH_QUESTION : CLFlag<"?">, Alias<help>,
-  HelpText<"Display available options">;
-def _SLASH_Qvec : CLFlag<"Qvec">,
-  HelpText<"Enable the loop vectorization passes">, Alias<fvectorize>;
-def _SLASH_Qvec_ : CLFlag<"Qvec-">,
-  HelpText<"Disable the loop vectorization passes">, Alias<fno_vectorize>;
-def _SLASH_showIncludes : CLFlag<"showIncludes">,
-  HelpText<"Print info about included files to stderr">;
-def _SLASH_showIncludes_user : CLFlag<"showIncludes:user">,
-  HelpText<"Like /showIncludes but omit system headers">;
-def _SLASH_showFilenames : CLFlag<"showFilenames">,
-  HelpText<"Print the name of each compiled file">;
-def _SLASH_showFilenames_ : CLFlag<"showFilenames-">,
-  HelpText<"Do not print the name of each compiled file (default)">;
-def _SLASH_source_charset : CLCompileJoined<"source-charset:">,
-  HelpText<"Set source encoding, supports only UTF-8">,
-  Alias<finput_charset_EQ>;
-def _SLASH_execution_charset : CLCompileJoined<"execution-charset:">,
-  HelpText<"Set runtime encoding, supports only UTF-8">,
-  Alias<fexec_charset_EQ>;
-def _SLASH_std : CLCompileJoined<"std:">,
-  HelpText<"Set C++ version (c++14,c++17,c++latest)">;
-def _SLASH_U : CLJoinedOrSeparate<"U">, HelpText<"Undefine macro">,
-  MetaVarName<"<macro>">, Alias<U>;
-def _SLASH_validate_charset : CLFlag<"validate-charset">,
-  Alias<W_Joined>, AliasArgs<["invalid-source-encoding"]>;
-def _SLASH_validate_charset_ : CLFlag<"validate-charset-">,
-  Alias<W_Joined>, AliasArgs<["no-invalid-source-encoding"]>;
-def _SLASH_W0 : CLFlag<"W0">, HelpText<"Disable all warnings">, Alias<w>;
-def _SLASH_W1 : CLFlag<"W1">, HelpText<"Enable -Wall">, Alias<Wall>;
-def _SLASH_W2 : CLFlag<"W2">, HelpText<"Enable -Wall">, Alias<Wall>;
-def _SLASH_W3 : CLFlag<"W3">, HelpText<"Enable -Wall">, Alias<Wall>;
-def _SLASH_W4 : CLFlag<"W4">, HelpText<"Enable -Wall and -Wextra">, Alias<WCL4>;
-def _SLASH_Wall : CLFlag<"Wall">, HelpText<"Enable -Weverything">,
-  Alias<W_Joined>, AliasArgs<["everything"]>;
-def _SLASH_WX : CLFlag<"WX">, HelpText<"Treat warnings as errors">,
-  Alias<W_Joined>, AliasArgs<["error"]>;
-def _SLASH_WX_ : CLFlag<"WX-">,
-  HelpText<"Do not treat warnings as errors (default)">,
-  Alias<W_Joined>, AliasArgs<["no-error"]>;
-def _SLASH_w_flag : CLFlag<"w">, HelpText<"Disable all warnings">, Alias<w>;
-def _SLASH_wd4005 : CLFlag<"wd4005">, Alias<W_Joined>,
-  AliasArgs<["no-macro-redefined"]>;
-def _SLASH_wd4018 : CLFlag<"wd4018">, Alias<W_Joined>,
-  AliasArgs<["no-sign-compare"]>;
-def _SLASH_wd4100 : CLFlag<"wd4100">, Alias<W_Joined>,
-  AliasArgs<["no-unused-parameter"]>;
-def _SLASH_wd4910 : CLFlag<"wd4910">, Alias<W_Joined>,
-  AliasArgs<["no-dllexport-explicit-instantiation-decl"]>;
-def _SLASH_wd4996 : CLFlag<"wd4996">, Alias<W_Joined>,
-  AliasArgs<["no-deprecated-declarations"]>;
-def _SLASH_vd : CLJoined<"vd">, HelpText<"Control vtordisp placement">,
-  Alias<vtordisp_mode_EQ>;
-def _SLASH_X : CLFlag<"X">,
-  HelpText<"Do not add %INCLUDE% to include search path">, Alias<nostdlibinc>;
-def _SLASH_Zc_sizedDealloc : CLFlag<"Zc:sizedDealloc">,
-  HelpText<"Enable C++14 sized global deallocation functions">,
-  Alias<fsized_deallocation>;
-def _SLASH_Zc_sizedDealloc_ : CLFlag<"Zc:sizedDealloc-">,
-  HelpText<"Disable C++14 sized global deallocation functions">,
-  Alias<fno_sized_deallocation>;
-def _SLASH_Zc_alignedNew : CLFlag<"Zc:alignedNew">,
-  HelpText<"Enable C++17 aligned allocation functions">,
-  Alias<faligned_allocation>;
-def _SLASH_Zc_alignedNew_ : CLFlag<"Zc:alignedNew-">,
-  HelpText<"Disable C++17 aligned allocation functions">,
-  Alias<fno_aligned_allocation>;
-def _SLASH_Zc_char8_t : CLFlag<"Zc:char8_t">,
-  HelpText<"Enable char8_t from C++2a">,
-  Alias<fchar8__t>;
-def _SLASH_Zc_char8_t_ : CLFlag<"Zc:char8_t-">,
-  HelpText<"Disable char8_t from c++2a">,
-  Alias<fno_char8__t>;
-def _SLASH_Zc_strictStrings : CLFlag<"Zc:strictStrings">,
-  HelpText<"Treat string literals as const">, Alias<W_Joined>,
-  AliasArgs<["error=c++11-compat-deprecated-writable-strings"]>;
-def _SLASH_Zc_threadSafeInit : CLFlag<"Zc:threadSafeInit">,
-  HelpText<"Enable thread-safe initialization of static variables">,
-  Alias<fthreadsafe_statics>;
-def _SLASH_Zc_threadSafeInit_ : CLFlag<"Zc:threadSafeInit-">,
-  HelpText<"Disable thread-safe initialization of static variables">,
-  Alias<fno_threadsafe_statics>;
-def _SLASH_Zc_trigraphs : CLFlag<"Zc:trigraphs">,
-  HelpText<"Enable trigraphs">, Alias<ftrigraphs>;
-def _SLASH_Zc_trigraphs_off : CLFlag<"Zc:trigraphs-">,
-  HelpText<"Disable trigraphs (default)">, Alias<fno_trigraphs>;
-def _SLASH_Zc_twoPhase : CLFlag<"Zc:twoPhase">,
-  HelpText<"Enable two-phase name lookup in templates">,
-  Alias<fno_delayed_template_parsing>;
-def _SLASH_Zc_twoPhase_ : CLFlag<"Zc:twoPhase-">,
-  HelpText<"Disable two-phase name lookup in templates (default)">,
-  Alias<fdelayed_template_parsing>;
-def _SLASH_Z7 : CLFlag<"Z7">,
-  HelpText<"Enable CodeView debug information in object files">;
-def _SLASH_Zd : CLFlag<"Zd">,
-  HelpText<"Emit debug line number tables only">;
-def _SLASH_Zi : CLFlag<"Zi">, Alias<_SLASH_Z7>,
-  HelpText<"Like /Z7">;
-def _SLASH_Zp : CLJoined<"Zp">,
-  HelpText<"Set default maximum struct packing alignment">,
-  Alias<fpack_struct_EQ>;
-def _SLASH_Zp_flag : CLFlag<"Zp">,
-  HelpText<"Set default maximum struct packing alignment to 1">,
-  Alias<fpack_struct_EQ>, AliasArgs<["1"]>;
-def _SLASH_Zs : CLFlag<"Zs">, HelpText<"Syntax-check only">,
-  Alias<fsyntax_only>;
-def _SLASH_openmp_ : CLFlag<"openmp-">,
-  HelpText<"Disable OpenMP support">, Alias<fno_openmp>;
-def _SLASH_openmp : CLFlag<"openmp">, HelpText<"Enable OpenMP support">,
-  Alias<fopenmp>;
-def _SLASH_openmp_experimental : CLFlag<"openmp:experimental">,
-  HelpText<"Enable OpenMP support with experimental SIMD support">,
-  Alias<fopenmp>;
-
-// Non-aliases:
-
-def _SLASH_arch : CLCompileJoined<"arch:">,
-  HelpText<"Set architecture for code generation">;
-
-def _SLASH_M_Group : OptionGroup<"</M group>">, Group<cl_compile_Group>;
-def _SLASH_volatile_Group : OptionGroup<"</volatile group>">,
-  Group<cl_compile_Group>;
-
-def _SLASH_EH : CLJoined<"EH">, HelpText<"Set exception handling model">;
-def _SLASH_EP : CLFlag<"EP">,
-  HelpText<"Disable linemarker output and preprocess to stdout">;
-def _SLASH_FA : CLFlag<"FA">,
-  HelpText<"Output assembly code file during compilation">;
-def _SLASH_Fa : CLJoined<"Fa">,
-  HelpText<"Set assembly output file name (with /FA)">,
-  MetaVarName<"<file or dir/>">;
-def _SLASH_fallback : CLCompileFlag<"fallback">,
-  HelpText<"Fall back to cl.exe if clang-cl fails to compile">;
-def _SLASH_FI : CLJoinedOrSeparate<"FI">,
-  HelpText<"Include file before parsing">, Alias<include_>;
-def _SLASH_Fe : CLJoined<"Fe">,
-  HelpText<"Set output executable file name">,
-  MetaVarName<"<file or dir/>">;
-def _SLASH_Fi : CLCompileJoined<"Fi">,
-  HelpText<"Set preprocess output file name (with /P)">,
-  MetaVarName<"<file>">;
-def _SLASH_Fo : CLCompileJoined<"Fo">,
-  HelpText<"Set output object file (with /c)">,
-  MetaVarName<"<file or dir/>">;
-def _SLASH_guard : CLJoined<"guard:">,
-  HelpText<"Enable Control Flow Guard with /guard:cf, or only the table with /guard:cf,nochecks">;
-def _SLASH_GX : CLFlag<"GX">,
-  HelpText<"Deprecated; use /EHsc">;
-def _SLASH_GX_ : CLFlag<"GX-">,
-  HelpText<"Deprecated (like not passing /EH)">;
-def _SLASH_imsvc : CLJoinedOrSeparate<"imsvc">,
-  HelpText<"Add <dir> to system include search path, as if in %INCLUDE%">,
-  MetaVarName<"<dir>">;
-def _SLASH_LD : CLFlag<"LD">, HelpText<"Create DLL">;
-def _SLASH_LDd : CLFlag<"LDd">, HelpText<"Create debug DLL">;
-def _SLASH_link : CLRemainingArgsJoined<"link">,
-  HelpText<"Forward options to the linker">, MetaVarName<"<options>">;
-def _SLASH_MD : Option<["/", "-"], "MD", KIND_FLAG>, Group<_SLASH_M_Group>,
-  Flags<[CLOption, DriverOption]>, HelpText<"Use DLL run-time">;
-def _SLASH_MDd : Option<["/", "-"], "MDd", KIND_FLAG>, Group<_SLASH_M_Group>,
-  Flags<[CLOption, DriverOption]>, HelpText<"Use DLL debug run-time">;
-def _SLASH_MT : Option<["/", "-"], "MT", KIND_FLAG>, Group<_SLASH_M_Group>,
-  Flags<[CLOption, DriverOption]>, HelpText<"Use static run-time">;
-def _SLASH_MTd : Option<["/", "-"], "MTd", KIND_FLAG>, Group<_SLASH_M_Group>,
-  Flags<[CLOption, DriverOption]>, HelpText<"Use static debug run-time">;
-def _SLASH_o : CLJoinedOrSeparate<"o">,
-  HelpText<"Deprecated (set output file name); use /Fe or /Fe">,
-  MetaVarName<"<file or dir/>">;
-def _SLASH_P : CLFlag<"P">, HelpText<"Preprocess to file">;
-def _SLASH_Tc : CLCompileJoinedOrSeparate<"Tc">,
-  HelpText<"Treat <file> as C source file">, MetaVarName<"<file>">;
-def _SLASH_TC : CLCompileFlag<"TC">, HelpText<"Treat all source files as C">;
-def _SLASH_Tp : CLCompileJoinedOrSeparate<"Tp">,
-  HelpText<"Treat <file> as C++ source file">, MetaVarName<"<file>">;
-def _SLASH_TP : CLCompileFlag<"TP">, HelpText<"Treat all source files as C++">;
-def _SLASH_volatile_iso : Option<["/", "-"], "volatile:iso", KIND_FLAG>,
-  Group<_SLASH_volatile_Group>, Flags<[CLOption, DriverOption]>,
-  HelpText<"Volatile loads and stores have standard semantics">;
-def _SLASH_vmb : CLFlag<"vmb">,
-  HelpText<"Use a best-case representation method for member pointers">;
-def _SLASH_vmg : CLFlag<"vmg">,
-  HelpText<"Use a most-general representation for member pointers">;
-def _SLASH_vms : CLFlag<"vms">,
-  HelpText<"Set the default most-general representation to single inheritance">;
-def _SLASH_vmm : CLFlag<"vmm">,
-  HelpText<"Set the default most-general representation to "
-           "multiple inheritance">;
-def _SLASH_vmv : CLFlag<"vmv">,
-  HelpText<"Set the default most-general representation to "
-           "virtual inheritance">;
-def _SLASH_volatile_ms  : Option<["/", "-"], "volatile:ms", KIND_FLAG>,
-  Group<_SLASH_volatile_Group>, Flags<[CLOption, DriverOption]>,
-  HelpText<"Volatile loads and stores have acquire and release semantics">;
-def _SLASH_clang : CLJoined<"clang:">,
-  HelpText<"Pass <arg> to the clang driver">, MetaVarName<"<arg>">;
-def _SLASH_Zl : CLFlag<"Zl">,
-  HelpText<"Do not let object file auto-link default libraries">;
-
-def _SLASH_Yc : CLJoined<"Yc">,
-  HelpText<"Generate a pch file for all code up to and including <filename>">,
-  MetaVarName<"<filename>">;
-def _SLASH_Yu : CLJoined<"Yu">,
-  HelpText<"Load a pch file and use it instead of all code up to "
-           "and including <filename>">,
-  MetaVarName<"<filename>">;
-def _SLASH_Y_ : CLFlag<"Y-">,
-  HelpText<"Disable precompiled headers, overrides /Yc and /Yu">;
-def _SLASH_Zc_dllexportInlines : CLFlag<"Zc:dllexportInlines">,
-  HelpText<"dllexport/dllimport inline member functions of dllexport/import classes (default)">;
-def _SLASH_Zc_dllexportInlines_ : CLFlag<"Zc:dllexportInlines-">,
-  HelpText<"Do not dllexport/dllimport inline member functions of dllexport/import classes">;
-def _SLASH_Fp : CLJoined<"Fp">,
-  HelpText<"Set pch file name (with /Yc and /Yu)">, MetaVarName<"<file>">;
-
-def _SLASH_Gd : CLFlag<"Gd">,
-  HelpText<"Set __cdecl as a default calling convention">;
-def _SLASH_Gr : CLFlag<"Gr">,
-  HelpText<"Set __fastcall as a default calling convention">;
-def _SLASH_Gz : CLFlag<"Gz">,
-  HelpText<"Set __stdcall as a default calling convention">;
-def _SLASH_Gv : CLFlag<"Gv">,
-  HelpText<"Set __vectorcall as a default calling convention">;
-def _SLASH_Gregcall : CLFlag<"Gregcall">,
-  HelpText<"Set __regcall as a default calling convention">;
-
-// Ignored:
-
-def _SLASH_analyze_ : CLIgnoredFlag<"analyze-">;
-def _SLASH_bigobj : CLIgnoredFlag<"bigobj">;
-def _SLASH_cgthreads : CLIgnoredJoined<"cgthreads">;
-def _SLASH_d2FastFail : CLIgnoredFlag<"d2FastFail">;
-def _SLASH_d2Zi_PLUS : CLIgnoredFlag<"d2Zi+">;
-def _SLASH_errorReport : CLIgnoredJoined<"errorReport">;
-def _SLASH_FC : CLIgnoredFlag<"FC">;
-def _SLASH_Fd : CLIgnoredJoined<"Fd">;
-def _SLASH_FS : CLIgnoredFlag<"FS">;
-def _SLASH_JMC : CLIgnoredFlag<"JMC">;
-def _SLASH_kernel_ : CLIgnoredFlag<"kernel-">;
-def _SLASH_nologo : CLIgnoredFlag<"nologo">;
-def _SLASH_permissive_ : CLIgnoredFlag<"permissive-">;
-def _SLASH_RTC : CLIgnoredJoined<"RTC">;
-def _SLASH_sdl : CLIgnoredFlag<"sdl">;
-def _SLASH_sdl_ : CLIgnoredFlag<"sdl-">;
-def _SLASH_utf8 : CLIgnoredFlag<"utf-8">,
-  HelpText<"Set source and runtime encoding to UTF-8 (default)">;
-def _SLASH_w : CLIgnoredJoined<"w">;
-def _SLASH_Zc___cplusplus : CLIgnoredFlag<"Zc:__cplusplus">;
-def _SLASH_Zc_auto : CLIgnoredFlag<"Zc:auto">;
-def _SLASH_Zc_forScope : CLIgnoredFlag<"Zc:forScope">;
-def _SLASH_Zc_inline : CLIgnoredFlag<"Zc:inline">;
-def _SLASH_Zc_rvalueCast : CLIgnoredFlag<"Zc:rvalueCast">;
-def _SLASH_Zc_ternary : CLIgnoredFlag<"Zc:ternary">;
-def _SLASH_Zc_wchar_t : CLIgnoredFlag<"Zc:wchar_t">;
-def _SLASH_ZH_MD5 : CLIgnoredFlag<"ZH:MD5">;
-def _SLASH_ZH_SHA1 : CLIgnoredFlag<"ZH:SHA1">;
-def _SLASH_ZH_SHA_256 : CLIgnoredFlag<"ZH:SHA_256">;
-def _SLASH_Zm : CLIgnoredJoined<"Zm">;
-def _SLASH_Zo : CLIgnoredFlag<"Zo">;
-def _SLASH_Zo_ : CLIgnoredFlag<"Zo-">;
-
-
-// Unsupported:
-
-def _SLASH_await : CLFlag<"await">;
-def _SLASH_constexpr : CLJoined<"constexpr:">;
-def _SLASH_AI : CLJoinedOrSeparate<"AI">;
-def _SLASH_Bt : CLFlag<"Bt">;
-def _SLASH_Bt_plus : CLFlag<"Bt+">;
-def _SLASH_clr : CLJoined<"clr">;
-def _SLASH_d2 : CLJoined<"d2">;
-def _SLASH_doc : CLJoined<"doc">;
-def _SLASH_FA_joined : CLJoined<"FA">;
-def _SLASH_favor : CLJoined<"favor">;
-def _SLASH_F : CLJoinedOrSeparate<"F">;
-def _SLASH_Fm : CLJoined<"Fm">;
-def _SLASH_Fr : CLJoined<"Fr">;
-def _SLASH_FR : CLJoined<"FR">;
-def _SLASH_FU : CLJoinedOrSeparate<"FU">;
-def _SLASH_Fx : CLFlag<"Fx">;
-def _SLASH_G1 : CLFlag<"G1">;
-def _SLASH_G2 : CLFlag<"G2">;
-def _SLASH_Ge : CLFlag<"Ge">;
-def _SLASH_Gh : CLFlag<"Gh">;
-def _SLASH_GH : CLFlag<"GH">;
-def _SLASH_GL : CLFlag<"GL">;
-def _SLASH_GL_ : CLFlag<"GL-">;
-def _SLASH_Gm : CLFlag<"Gm">;
-def _SLASH_Gm_ : CLFlag<"Gm-">;
-def _SLASH_GT : CLFlag<"GT">;
-def _SLASH_GZ : CLFlag<"GZ">;
-def _SLASH_H : CLFlag<"H">;
-def _SLASH_homeparams : CLFlag<"homeparams">;
-def _SLASH_hotpatch : CLFlag<"hotpatch">;
-def _SLASH_kernel : CLFlag<"kernel">;
-def _SLASH_LN : CLFlag<"LN">;
-def _SLASH_MP : CLJoined<"MP">;
-def _SLASH_Qfast_transcendentals : CLFlag<"Qfast_transcendentals">;
-def _SLASH_QIfist : CLFlag<"QIfist">;
-def _SLASH_QIntel_jcc_erratum : CLFlag<"QIntel-jcc-erratum">;
-def _SLASH_Qimprecise_fwaits : CLFlag<"Qimprecise_fwaits">;
-def _SLASH_Qpar : CLFlag<"Qpar">;
-def _SLASH_Qpar_report : CLJoined<"Qpar-report">;
-def _SLASH_Qsafe_fp_loads : CLFlag<"Qsafe_fp_loads">;
-def _SLASH_Qspectre : CLFlag<"Qspectre">;
-def _SLASH_Qspectre_load : CLFlag<"Qspectre-load">;
-def _SLASH_Qspectre_load_cf : CLFlag<"Qspectre-load-cf">;
-def _SLASH_Qvec_report : CLJoined<"Qvec-report">;
-def _SLASH_u : CLFlag<"u">;
-def _SLASH_V : CLFlag<"V">;
-def _SLASH_WL : CLFlag<"WL">;
-def _SLASH_Wp64 : CLFlag<"Wp64">;
-def _SLASH_Yd : CLFlag<"Yd">;
-def _SLASH_Yl : CLJoined<"Yl">;
-def _SLASH_Za : CLFlag<"Za">;
-def _SLASH_Zc : CLJoined<"Zc:">;
-def _SLASH_Ze : CLFlag<"Ze">;
-def _SLASH_Zg : CLFlag<"Zg">;
-def _SLASH_ZI : CLFlag<"ZI">;
-def _SLASH_ZW : CLJoined<"ZW">;
diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index 90a0c871140a5..dc18f1314f81e 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -548,6 +548,9 @@ class Driver {
   /// handle this action.
   bool ShouldUseFlangCompiler(const JobAction &JA) const;
 
+  /// ShouldEmitStaticLibrary - Should the linker emit a static library.
+  bool ShouldEmitStaticLibrary(const llvm::opt::ArgList &Args) const;
+
   /// Returns true if we are performing any kind of LTO.
   bool isUsingLTO() const { return LTOMode != LTOK_None; }
 
@@ -618,7 +621,8 @@ class Driver {
   static bool GetReleaseVersion(StringRef Str,
                                 MutableArrayRef<unsigned> Digits);
   /// Compute the default -fmodule-cache-path.
-  static void getDefaultModuleCachePath(SmallVectorImpl<char> &Result);
+  /// \return True if the system provides a default cache directory.
+  static bool getDefaultModuleCachePath(SmallVectorImpl<char> &Result);
 };
 
 /// \return True if the last defined optimization level is -Ofast.
diff --git a/clang/include/clang/Driver/Job.h b/clang/include/clang/Driver/Job.h
index 1c6ea6971919a..6173b9d314b4d 100644
--- a/clang/include/clang/Driver/Job.h
+++ b/clang/include/clang/Driver/Job.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/Option/Option.h"
+#include "llvm/Support/Program.h"
 #include <memory>
 #include <string>
 #include <utility>
@@ -36,6 +37,69 @@ struct CrashReportInfo {
       : Filename(Filename), VFSPath(VFSPath) {}
 };
 
+// Encodes the kind of response file supported for a command invocation.
+// Response files are necessary if the command line gets too large, requiring
+// the arguments to be transferred to a file.
+struct ResponseFileSupport {
+  enum ResponseFileKind {
+    // Provides full support for response files, which means we can transfer
+    // all tool input arguments to a file.
+    RF_Full,
+    // Input file names can live in a file, but flags can't. This is a special
+    // case for old versions of Apple's ld64.
+    RF_FileList,
+    // Does not support response files: all arguments must be passed via
+    // command line.
+    RF_None
+  };
+  /// The level of support for response files.
+  ResponseFileKind ResponseKind;
+
+  /// The encoding to use when writing response files on Windows. Ignored on
+  /// other host OSes.
+  ///
+  /// Windows use cases: - GCC and Binutils on mingw only accept ANSI response
+  /// files encoded with the system current code page.
+  /// - MSVC's CL.exe and LINK.exe accept UTF16 on Windows.
+  /// - Clang accepts both UTF8 and UTF16.
+  ///
+  /// FIXME: When GNU tools learn how to parse UTF16 on Windows, we should
+  /// always use UTF16 for Windows, which is the Windows official encoding for
+  /// international characters.
+  llvm::sys::WindowsEncodingMethod ResponseEncoding;
+
+  /// What prefix to use for the command-line argument when passing a response
+  /// file.
+  const char *ResponseFlag;
+
+  /// Returns a ResponseFileSupport indicating that response files are not
+  /// supported.
+  static constexpr ResponseFileSupport None() {
+    return {RF_None, llvm::sys::WEM_UTF8, nullptr};
+  }
+
+  /// Returns a ResponseFileSupport indicating that response files are
+  /// supported, using the @file syntax. On windows, the file is written in the
+  /// UTF8 encoding. On other OSes, no re-encoding occurs.
+  static constexpr ResponseFileSupport AtFileUTF8() {
+    return {RF_Full, llvm::sys::WEM_UTF8, "@"};
+  }
+
+  /// Returns a ResponseFileSupport indicating that response files are
+  /// supported, using the @file syntax. On windows, the file is written in the
+  /// current ANSI code-page encoding. On other OSes, no re-encoding occurs.
+  static constexpr ResponseFileSupport AtFileCurCP() {
+    return {RF_Full, llvm::sys::WEM_CurrentCodePage, "@"};
+  }
+
+  /// Returns a ResponseFileSupport indicating that response files are
+  /// supported, using the @file syntax. On windows, the file is written in the
+  /// UTF-16 encoding. On other OSes, no re-encoding occurs.
+  static constexpr ResponseFileSupport AtFileUTF16() {
+    return {RF_Full, llvm::sys::WEM_UTF16, "@"};
+  }
+};
+
 /// Command - An executable path/name and argument vector to
 /// execute.
 class Command {
@@ -45,6 +109,9 @@ class Command {
   /// Tool - The tool which caused the creation of this job.
   const Tool &Creator;
 
+  /// Whether and how to generate response files if the arguments are too long.
+  ResponseFileSupport ResponseSupport;
+
   /// The executable to run.
   const char *Executable;
 
@@ -89,7 +156,8 @@ class Command {
   /// Whether the command will be executed in this process or not.
   bool InProcess = false;
 
-  Command(const Action &Source, const Tool &Creator, const char *Executable,
+  Command(const Action &Source, const Tool &Creator,
+          ResponseFileSupport ResponseSupport, const char *Executable,
           const llvm::opt::ArgStringList &Arguments,
           ArrayRef<InputInfo> Inputs);
   // FIXME: This really shouldn't be copyable, but is currently copied in some
@@ -109,11 +177,16 @@ class Command {
   /// getCreator - Return the Tool which caused the creation of this job.
   const Tool &getCreator() const { return Creator; }
 
+  /// Returns the kind of response file supported by the current invocation.
+  const ResponseFileSupport &getResponseFileSupport() {
+    return ResponseSupport;
+  }
+
   /// Set to pass arguments via a response file when launching the command
   void setResponseFile(const char *FileName);
 
-  /// Set an input file list, necessary if we need to use a response file but
-  /// the tool being called only supports input files lists.
+  /// Set an input file list, necessary if you specified an RF_FileList response
+  /// file support.
   void setInputFileList(llvm::opt::ArgStringList List) {
     InputFileList = std::move(List);
   }
@@ -136,7 +209,8 @@ class Command {
 /// Use the CC1 tool callback when available, to avoid creating a new process
 class CC1Command : public Command {
 public:
-  CC1Command(const Action &Source, const Tool &Creator, const char *Executable,
+  CC1Command(const Action &Source, const Tool &Creator,
+             ResponseFileSupport ResponseSupport, const char *Executable,
              const llvm::opt::ArgStringList &Arguments,
              ArrayRef<InputInfo> Inputs);
 
@@ -154,7 +228,7 @@ class CC1Command : public Command {
 class FallbackCommand : public Command {
 public:
   FallbackCommand(const Action &Source_, const Tool &Creator_,
-                  const char *Executable_,
+                  ResponseFileSupport ResponseSupport, const char *Executable_,
                   const llvm::opt::ArgStringList &Arguments_,
                   ArrayRef<InputInfo> Inputs,
                   std::unique_ptr<Command> Fallback_);
@@ -173,6 +247,7 @@ class FallbackCommand : public Command {
 class ForceSuccessCommand : public Command {
 public:
   ForceSuccessCommand(const Action &Source_, const Tool &Creator_,
+                      ResponseFileSupport ResponseSupport,
                       const char *Executable_,
                       const llvm::opt::ArgStringList &Arguments_,
                       ArrayRef<InputInfo> Inputs);
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 8ad582c84c604..f4556c15d744c 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -483,6 +483,9 @@ def Wnonportable_cfstrings : Joined<["-"], "Wnonportable-cfstrings">, Group<W_Gr
 def Wp_COMMA : CommaJoined<["-"], "Wp,">,
   HelpText<"Pass the comma separated arguments in <arg> to the preprocessor">,
   MetaVarName<"<arg>">, Group<Preprocessor_Group>;
+def Wundef_prefix_EQ : CommaJoined<["-"], "Wundef-prefix=">, Group<W_value_Group>,
+  Flags<[CC1Option, CoreOption, HelpHidden]>, MetaVarName<"<arg>">,
+  HelpText<"Enable warnings for undefined macros with a prefix in the comma separated list <arg>">;
 def Wwrite_strings : Flag<["-"], "Wwrite-strings">, Group<W_Group>, Flags<[CC1Option, HelpHidden]>;
 def Wno_write_strings : Flag<["-"], "Wno-write-strings">, Group<W_Group>, Flags<[CC1Option, HelpHidden]>;
 def W_Joined : Joined<["-"], "W">, Group<W_Group>, Flags<[CC1Option, CoreOption]>,
@@ -605,6 +608,8 @@ def hip_link : Flag<["--"], "hip-link">,
 def no_offload_arch_EQ : Joined<["--"], "no-offload-arch=">, Flags<[DriverOption]>,
   HelpText<"Remove CUDA/HIP offloading device architecture (e.g. sm_35, gfx906) from the list of devices to compile for. "
            "'all' resets the list to its default value.">;
+def emit_static_lib : Flag<["--"], "emit-static-lib">,
+  HelpText<"Enable linker job to emit a static library.">;
 def no_cuda_gpu_arch_EQ : Joined<["--"], "no-cuda-gpu-arch=">, Flags<[DriverOption]>,
   Alias<no_offload_arch_EQ>;
 def cuda_noopt_device_debug : Flag<["--"], "cuda-noopt-device-debug">,
@@ -632,15 +637,19 @@ defm cuda_short_ptr : OptInFFlag<"cuda-short-ptr",
   "Use 32-bit pointers for accessing const/local/shared address spaces">;
 def rocm_path_EQ : Joined<["--"], "rocm-path=">, Group<i_Group>,
   HelpText<"ROCm installation path, used for finding and automatically linking required bitcode libraries.">;
-def hip_device_lib_path_EQ : Joined<["--"], "hip-device-lib-path=">, Group<Link_Group>,
-  HelpText<"HIP device library path. Alternative to rocm-path.">;
+def rocm_device_lib_path_EQ : Joined<["--"], "rocm-device-lib-path=">, Group<Link_Group>,
+  HelpText<"ROCm device library path. Alternative to rocm-path.">;
+def : Joined<["--"], "hip-device-lib-path=">, Alias<rocm_device_lib_path_EQ>;
 def hip_device_lib_EQ : Joined<["--"], "hip-device-lib=">, Group<Link_Group>,
   HelpText<"HIP device library">;
+def hip_version_EQ : Joined<["--"], "hip-version=">,
+  HelpText<"HIP version in the format of major.minor.patch">;
 def fhip_dump_offload_linker_script : Flag<["-"], "fhip-dump-offload-linker-script">,
   Group<f_Group>, Flags<[NoArgumentUnused, HelpHidden]>;
 defm hip_new_launch_api : OptInFFlag<"hip-new-launch-api",
-  "Use new kernel launching API for HIP">;
-defm gpu_allow_device_init : OptInFFlag<"gpu-allow-device-init", "Allow device side init function in HIP">;
+  "Use", "Don't use", " new kernel launching API for HIP">;
+defm gpu_allow_device_init : OptInFFlag<"gpu-allow-device-init",
+  "Allow", "Don't allow", " device side init function in HIP">;
 def gpu_max_threads_per_block_EQ : Joined<["--"], "gpu-max-threads-per-block=">,
   Flags<[CC1Option]>,
   HelpText<"Default max threads per block for kernel launch bounds for HIP">;
@@ -844,7 +853,8 @@ def fdiagnostics_color : Flag<["-"], "fdiagnostics-color">, Group<f_Group>,
   Flags<[CoreOption, DriverOption]>;
 def fdiagnostics_color_EQ : Joined<["-"], "fdiagnostics-color=">, Group<f_Group>;
 def fansi_escape_codes : Flag<["-"], "fansi-escape-codes">, Group<f_Group>,
-  Flags<[CoreOption, CC1Option]>, HelpText<"Use ANSI escape codes for diagnostics">;
+  Flags<[CoreOption, CC1Option]>, HelpText<"Use ANSI escape codes for diagnostics">,
+  MarshallingInfoFlag<"DiagnosticOpts->UseANSIEscapeCodes", "false">;
 def fcomment_block_commands : CommaJoined<["-"], "fcomment-block-commands=">, Group<f_clang_Group>, Flags<[CC1Option]>,
   HelpText<"Treat each comma separated argument in <arg> as a documentation comment block command">,
   MetaVarName<"<arg>">;
@@ -1237,6 +1247,9 @@ def fexperimental_isel : Flag<["-"], "fexperimental-isel">, Group<f_clang_Group>
 def fexperimental_new_pass_manager : Flag<["-"], "fexperimental-new-pass-manager">,
   Group<f_clang_Group>, Flags<[CC1Option]>,
   HelpText<"Enables an experimental new pass manager in LLVM.">;
+def fexperimental_strict_floating_point : Flag<["-"], "fexperimental-strict-floating-point">,
+  Group<f_clang_Group>, Flags<[CC1Option]>,
+  HelpText<"Enables experimental strict floating point in LLVM.">;
 def finput_charset_EQ : Joined<["-"], "finput-charset=">, Group<f_Group>;
 def fexec_charset_EQ : Joined<["-"], "fexec-charset=">, Group<f_Group>;
 def finstrument_functions : Flag<["-"], "finstrument-functions">, Group<f_Group>, Flags<[CC1Option]>,
@@ -1584,7 +1597,6 @@ def fno_unwind_tables : Flag<["-"], "fno-unwind-tables">, Group<f_Group>;
 def fno_verbose_asm : Flag<["-"], "fno-verbose-asm">, Group<f_Group>, Flags<[CC1Option]>;
 def fno_working_directory : Flag<["-"], "fno-working-directory">, Group<f_Group>;
 def fno_wrapv : Flag<["-"], "fno-wrapv">, Group<f_Group>;
-def fno_zero_initialized_in_bss : Flag<["-"], "fno-zero-initialized-in-bss">, Group<f_Group>;
 def fobjc_arc : Flag<["-"], "fobjc-arc">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Synthesize retain and release calls for Objective-C pointers">;
 def fno_objc_arc : Flag<["-"], "fno-objc-arc">, Group<f_Group>;
@@ -1685,6 +1697,12 @@ def fopenmp_optimistic_collapse : Flag<["-"], "fopenmp-optimistic-collapse">, Gr
   Flags<[CC1Option, NoArgumentUnused, HelpHidden]>;
 def fno_openmp_optimistic_collapse : Flag<["-"], "fno-openmp-optimistic-collapse">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>;
+def fopenmp_cuda_parallel_target_regions : Flag<["-"], "fopenmp-cuda-parallel-target-regions">, Group<f_Group>,
+  Flags<[CC1Option, NoArgumentUnused, HelpHidden]>,
+  HelpText<"Support parallel execution of target regions on Cuda-based devices.">;
+def fno_openmp_cuda_parallel_target_regions : Flag<["-"], "fno-openmp-cuda-parallel-target-regions">, Group<f_Group>,
+  Flags<[NoArgumentUnused, HelpHidden]>,
+  HelpText<"Support only serial execution of target regions on Cuda-based devices.">;
 def static_openmp: Flag<["-"], "static-openmp">,
   HelpText<"Use the static host OpenMP runtime while linking.">;
 def fno_optimize_sibling_calls : Flag<["-"], "fno-optimize-sibling-calls">, Group<f_Group>;
@@ -1915,7 +1933,7 @@ def fwrapv : Flag<["-"], "fwrapv">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Treat signed integer overflow as two's complement">;
 def fwritable_strings : Flag<["-"], "fwritable-strings">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Store string literals as writable data">;
-def fzero_initialized_in_bss : Flag<["-"], "fzero-initialized-in-bss">, Group<f_Group>;
+defm zero_initialized_in_bss : OptOutFFlag<"zero-initialized-in-bss", "", "Don't place zero initialized data in BSS">;
 defm function_sections : OptInFFlag<"function-sections", "Place each function in its own section">;
 def fbasic_block_sections_EQ : Joined<["-"], "fbasic-block-sections=">, Group<f_Group>,
   Flags<[CC1Option, CC1AsOption]>,
@@ -2020,7 +2038,7 @@ def : Flag<["-"], "gno-record-gcc-switches">, Alias<gno_record_command_line>;
 def gstrict_dwarf : Flag<["-"], "gstrict-dwarf">, Group<g_flags_Group>;
 def gno_strict_dwarf : Flag<["-"], "gno-strict-dwarf">, Group<g_flags_Group>;
 def gcolumn_info : Flag<["-"], "gcolumn-info">, Group<g_flags_Group>, Flags<[CoreOption]>;
-def gno_column_info : Flag<["-"], "gno-column-info">, Group<g_flags_Group>, Flags<[CoreOption]>;
+def gno_column_info : Flag<["-"], "gno-column-info">, Group<g_flags_Group>, Flags<[CoreOption, CC1Option]>;
 def gsplit_dwarf : Flag<["-"], "gsplit-dwarf">, Group<g_flags_Group>;
 def gsplit_dwarf_EQ : Joined<["-"], "gsplit-dwarf=">, Group<g_flags_Group>,
   HelpText<"Set DWARF fission mode to either 'split' or 'single'">,
@@ -2254,6 +2272,11 @@ def mlvi_cfi : Flag<["-"], "mlvi-cfi">, Group<m_Group>, Flags<[CoreOption,Driver
   HelpText<"Enable only control-flow mitigations for Load Value Injection (LVI)">;
 def mno_lvi_cfi : Flag<["-"], "mno-lvi-cfi">, Group<m_Group>, Flags<[CoreOption,DriverOption]>,
   HelpText<"Disable control-flow mitigations for Load Value Injection (LVI)">;
+def m_seses : Flag<["-"], "mseses">, Group<m_Group>, Flags<[CoreOption, DriverOption]>,
+  HelpText<"Enable speculative execution side effect suppression (SESES). "
+    "Includes LVI control flow integrity mitigations">;
+def mno_seses : Flag<["-"], "mno-seses">, Group<m_Group>, Flags<[CoreOption, DriverOption]>,
+  HelpText<"Disable speculative execution side effect suppression (SESES)">;
 
 def mrelax : Flag<["-"], "mrelax">, Group<m_riscv_Features_Group>,
   HelpText<"Enable linker relaxation">;
@@ -3055,6 +3078,12 @@ def m3dnow : Flag<["-"], "m3dnow">, Group<m_x86_Features_Group>;
 def mno_3dnow : Flag<["-"], "mno-3dnow">, Group<m_x86_Features_Group>;
 def m3dnowa : Flag<["-"], "m3dnowa">, Group<m_x86_Features_Group>;
 def mno_3dnowa : Flag<["-"], "mno-3dnowa">, Group<m_x86_Features_Group>;
+def mamx_bf16 : Flag<["-"], "mamx-bf16">, Group<m_x86_Features_Group>;
+def mno_amx_bf16 : Flag<["-"], "mno-amx-bf16">, Group<m_x86_Features_Group>;
+def mtamx_int8 : Flag<["-"], "mamx-int8">, Group<m_x86_Features_Group>;
+def mno_amx_int8 : Flag<["-"], "mno-amx-int8">, Group<m_x86_Features_Group>;
+def mamx_tile : Flag<["-"], "mamx-tile">, Group<m_x86_Features_Group>;
+def mno_amx_tile : Flag<["-"], "mno-amx-tile">, Group<m_x86_Features_Group>;
 def msse : Flag<["-"], "msse">, Group<m_x86_Features_Group>;
 def mno_sse : Flag<["-"], "mno-sse">, Group<m_x86_Features_Group>;
 def msse2 : Flag<["-"], "msse2">, Group<m_x86_Features_Group>;
@@ -3413,6 +3442,1405 @@ def fno_sycl : Flag<["-"], "fno-sycl">, Group<sycl_Group>, Flags<[CoreOption]>,
 def sycl_std_EQ : Joined<["-"], "sycl-std=">, Group<sycl_Group>, Flags<[CC1Option, NoArgumentUnused, CoreOption]>,
   HelpText<"SYCL language standard to compile for.">, Values<"2017, 121, 1.2.1, sycl-1.2.1">;
 
-include "CC1Options.td"
+//===----------------------------------------------------------------------===//
+// CC1 Options
+//===----------------------------------------------------------------------===//
+
+let Flags = [CC1Option, NoDriverOption] in {
+
+//===----------------------------------------------------------------------===//
+// Target Options
+//===----------------------------------------------------------------------===//
+
+let Flags = [CC1Option, CC1AsOption, NoDriverOption] in {
+
+def target_cpu : Separate<["-"], "target-cpu">,
+  HelpText<"Target a specific cpu type">;
+def target_feature : Separate<["-"], "target-feature">,
+  HelpText<"Target specific attributes">;
+def triple : Separate<["-"], "triple">,
+  HelpText<"Specify target triple (e.g. i686-apple-darwin9)">,
+  MarshallingInfoString<"TargetOpts->Triple", "llvm::Triple::normalize(llvm::sys::getDefaultTargetTriple())", "std::string">,
+  AlwaysEmit, Normalizer<"normalizeTriple">, DenormalizeString;
+def target_abi : Separate<["-"], "target-abi">,
+  HelpText<"Target a particular ABI type">;
+def target_sdk_version_EQ : Joined<["-"], "target-sdk-version=">,
+  HelpText<"The version of target SDK used for compilation">;
+
+}
+
+def target_linker_version : Separate<["-"], "target-linker-version">,
+  HelpText<"Target linker version">;
+def triple_EQ : Joined<["-"], "triple=">, Alias<triple>;
+def mfpmath : Separate<["-"], "mfpmath">,
+  HelpText<"Which unit to use for fp math">;
+
+def fpadding_on_unsigned_fixed_point : Flag<["-"], "fpadding-on-unsigned-fixed-point">,
+  HelpText<"Force each unsigned fixed point type to have an extra bit of padding to align their scales with those of signed fixed point types">;
+def fno_padding_on_unsigned_fixed_point : Flag<["-"], "fno-padding-on-unsigned-fixed-point">;
+
+//===----------------------------------------------------------------------===//
+// Analyzer Options
+//===----------------------------------------------------------------------===//
+
+def analysis_UnoptimizedCFG : Flag<["-"], "unoptimized-cfg">,
+  HelpText<"Generate unoptimized CFGs for all analyses">;
+def analysis_CFGAddImplicitDtors : Flag<["-"], "cfg-add-implicit-dtors">,
+  HelpText<"Add C++ implicit destructors to CFGs for all analyses">;
+
+def analyzer_store : Separate<["-"], "analyzer-store">,
+  HelpText<"Source Code Analysis - Abstract Memory Store Models">;
+def analyzer_store_EQ : Joined<["-"], "analyzer-store=">, Alias<analyzer_store>;
+
+def analyzer_constraints : Separate<["-"], "analyzer-constraints">,
+  HelpText<"Source Code Analysis - Symbolic Constraint Engines">;
+def analyzer_constraints_EQ : Joined<["-"], "analyzer-constraints=">,
+  Alias<analyzer_constraints>;
+
+def analyzer_output : Separate<["-"], "analyzer-output">,
+  HelpText<"Source Code Analysis - Output Options">;
+def analyzer_output_EQ : Joined<["-"], "analyzer-output=">,
+  Alias<analyzer_output>;
+
+def analyzer_purge : Separate<["-"], "analyzer-purge">,
+  HelpText<"Source Code Analysis - Dead Symbol Removal Frequency">;
+def analyzer_purge_EQ : Joined<["-"], "analyzer-purge=">, Alias<analyzer_purge>;
+
+def analyzer_opt_analyze_headers : Flag<["-"], "analyzer-opt-analyze-headers">,
+  HelpText<"Force the static analyzer to analyze functions defined in header files">;
+def analyzer_opt_analyze_nested_blocks : Flag<["-"], "analyzer-opt-analyze-nested-blocks">,
+  HelpText<"Analyze the definitions of blocks in addition to functions">;
+def analyzer_display_progress : Flag<["-"], "analyzer-display-progress">,
+  HelpText<"Emit verbose output about the analyzer's progress">;
+def analyze_function : Separate<["-"], "analyze-function">,
+  HelpText<"Run analysis on specific function (for C++ include parameters in name)">;
+def analyze_function_EQ : Joined<["-"], "analyze-function=">, Alias<analyze_function>;
+def trim_egraph : Flag<["-"], "trim-egraph">,
+  HelpText<"Only show error-related paths in the analysis graph">;
+def analyzer_viz_egraph_graphviz : Flag<["-"], "analyzer-viz-egraph-graphviz">,
+  HelpText<"Display exploded graph using GraphViz">;
+def analyzer_dump_egraph : Separate<["-"], "analyzer-dump-egraph">,
+  HelpText<"Dump exploded graph to the specified file">;
+def analyzer_dump_egraph_EQ : Joined<["-"], "analyzer-dump-egraph=">, Alias<analyzer_dump_egraph>;
+
+def analyzer_inline_max_stack_depth : Separate<["-"], "analyzer-inline-max-stack-depth">,
+  HelpText<"Bound on stack depth while inlining (4 by default)">;
+def analyzer_inline_max_stack_depth_EQ : Joined<["-"], "analyzer-inline-max-stack-depth=">,
+  Alias<analyzer_inline_max_stack_depth>;
+
+def analyzer_inlining_mode : Separate<["-"], "analyzer-inlining-mode">,
+  HelpText<"Specify the function selection heuristic used during inlining">;
+def analyzer_inlining_mode_EQ : Joined<["-"], "analyzer-inlining-mode=">, Alias<analyzer_inlining_mode>;
+
+def analyzer_disable_retry_exhausted : Flag<["-"], "analyzer-disable-retry-exhausted">,
+  HelpText<"Do not re-analyze paths leading to exhausted nodes with a different strategy (may decrease code coverage)">;
+
+def analyzer_max_loop : Separate<["-"], "analyzer-max-loop">,
+  HelpText<"The maximum number of times the analyzer will go through a loop">;
+def analyzer_stats : Flag<["-"], "analyzer-stats">,
+  HelpText<"Print internal analyzer statistics.">;
+
+def analyzer_checker : Separate<["-"], "analyzer-checker">,
+  HelpText<"Choose analyzer checkers to enable">,
+  ValuesCode<[{
+    const char *Values =
+    #define GET_CHECKERS
+    #define CHECKER(FULLNAME, CLASS, HT, DOC_URI, IS_HIDDEN)  FULLNAME ","
+    #include "clang/StaticAnalyzer/Checkers/Checkers.inc"
+    #undef GET_CHECKERS
+    #define GET_PACKAGES
+    #define PACKAGE(FULLNAME)  FULLNAME ","
+    #include "clang/StaticAnalyzer/Checkers/Checkers.inc"
+    #undef GET_PACKAGES
+    ;
+  }]>;
+def analyzer_checker_EQ : Joined<["-"], "analyzer-checker=">,
+  Alias<analyzer_checker>;
+
+def analyzer_disable_checker : Separate<["-"], "analyzer-disable-checker">,
+  HelpText<"Choose analyzer checkers to disable">;
+def analyzer_disable_checker_EQ : Joined<["-"], "analyzer-disable-checker=">,
+  Alias<analyzer_disable_checker>;
+
+def analyzer_disable_all_checks : Flag<["-"], "analyzer-disable-all-checks">,
+  HelpText<"Disable all static analyzer checks">;
+
+def analyzer_checker_help : Flag<["-"], "analyzer-checker-help">,
+  HelpText<"Display the list of analyzer checkers that are available">;
+
+def analyzer_checker_help_alpha : Flag<["-"], "analyzer-checker-help-alpha">,
+  HelpText<"Display the list of in development analyzer checkers. These "
+           "are NOT considered safe, they are unstable and will emit incorrect "
+           "reports. Enable ONLY FOR DEVELOPMENT purposes">;
+
+def analyzer_checker_help_developer : Flag<["-"], "analyzer-checker-help-developer">,
+  HelpText<"Display the list of developer-only checkers such as modeling "
+           "and debug checkers">;
+
+def analyzer_config_help : Flag<["-"], "analyzer-config-help">,
+  HelpText<"Display the list of -analyzer-config options. These are meant for "
+           "development purposes only!">;
+
+def analyzer_list_enabled_checkers : Flag<["-"], "analyzer-list-enabled-checkers">,
+  HelpText<"Display the list of enabled analyzer checkers">;
+
+def analyzer_config : Separate<["-"], "analyzer-config">,
+  HelpText<"Choose analyzer options to enable">;
+
+def analyzer_checker_option_help : Flag<["-"], "analyzer-checker-option-help">,
+  HelpText<"Display the list of checker and package options">;
+
+def analyzer_checker_option_help_alpha : Flag<["-"], "analyzer-checker-option-help-alpha">,
+  HelpText<"Display the list of in development checker and package options. "
+           "These are NOT considered safe, they are unstable and will emit "
+           "incorrect reports. Enable ONLY FOR DEVELOPMENT purposes">;
 
-include "CLCompatOptions.td"
+def analyzer_checker_option_help_developer : Flag<["-"], "analyzer-checker-option-help-developer">,
+  HelpText<"Display the list of checker and package options meant for "
+           "development purposes only">;
+
+def analyzer_config_compatibility_mode : Separate<["-"], "analyzer-config-compatibility-mode">,
+  HelpText<"Don't emit errors on invalid analyzer-config inputs">;
+
+def analyzer_config_compatibility_mode_EQ : Joined<["-"], "analyzer-config-compatibility-mode=">,
+  Alias<analyzer_config_compatibility_mode>;
+
+def analyzer_werror : Flag<["-"], "analyzer-werror">,
+  HelpText<"Emit analyzer results as errors rather than warnings">;
+
+//===----------------------------------------------------------------------===//
+// Migrator Options
+//===----------------------------------------------------------------------===//
+def migrator_no_nsalloc_error : Flag<["-"], "no-ns-alloc-error">,
+  HelpText<"Do not error on use of NSAllocateCollectable/NSReallocateCollectable">;
+
+def migrator_no_finalize_removal : Flag<["-"], "no-finalize-removal">,
+  HelpText<"Do not remove finalize method in gc mode">;
+
+//===----------------------------------------------------------------------===//
+// CodeGen Options
+//===----------------------------------------------------------------------===//
+
+let Flags = [CC1Option, CC1AsOption, NoDriverOption] in {
+def debug_info_kind_EQ : Joined<["-"], "debug-info-kind=">;
+def debug_info_macro : Flag<["-"], "debug-info-macro">,
+  HelpText<"Emit macro debug information">;
+def default_function_attr : Separate<["-"], "default-function-attr">,
+  HelpText<"Apply given attribute to all functions">;
+def dwarf_version_EQ : Joined<["-"], "dwarf-version=">;
+def debugger_tuning_EQ : Joined<["-"], "debugger-tuning=">;
+def dwarf_debug_flags : Separate<["-"], "dwarf-debug-flags">,
+  HelpText<"The string to embed in the Dwarf debug flags record.">;
+def record_command_line : Separate<["-"], "record-command-line">,
+  HelpText<"The string to embed in the .LLVM.command.line section.">;
+def compress_debug_sections : Flag<["-", "--"], "compress-debug-sections">,
+    HelpText<"DWARF debug sections compression">;
+def compress_debug_sections_EQ : Joined<["-", "--"], "compress-debug-sections=">,
+    HelpText<"DWARF debug sections compression type">;
+def mno_exec_stack : Flag<["-"], "mnoexecstack">,
+  HelpText<"Mark the file as not needing an executable stack">;
+def massembler_no_warn : Flag<["-"], "massembler-no-warn">,
+  HelpText<"Make assembler not emit warnings">;
+def massembler_fatal_warnings : Flag<["-"], "massembler-fatal-warnings">,
+  HelpText<"Make assembler warnings fatal">;
+def mrelax_relocations : Flag<["--"], "mrelax-relocations">,
+    HelpText<"Use relaxable elf relocations">;
+def msave_temp_labels : Flag<["-"], "msave-temp-labels">,
+  HelpText<"Save temporary labels in the symbol table. "
+           "Note this may change .s semantics and shouldn't generally be used "
+           "on compiler-generated code.">;
+def mrelocation_model : Separate<["-"], "mrelocation-model">,
+  HelpText<"The relocation model to use">, Values<"static,pic,ropi,rwpi,ropi-rwpi,dynamic-no-pic">,
+  NormalizedValuesScope<"llvm::Reloc">,
+  NormalizedValues<["Static", "PIC_", "ROPI", "RWPI", "ROPI_RWPI", "DynamicNoPIC"]>,
+  MarshallingInfoString<"CodeGenOpts.RelocationModel", "PIC_", "Model">,
+  AutoNormalizeEnum;
+def fno_math_builtin : Flag<["-"], "fno-math-builtin">,
+  HelpText<"Disable implicit builtin knowledge of math functions">;
+}
+
+def disable_llvm_verifier : Flag<["-"], "disable-llvm-verifier">,
+  HelpText<"Don't run the LLVM IR verifier pass">;
+def disable_llvm_passes : Flag<["-"], "disable-llvm-passes">,
+  HelpText<"Use together with -emit-llvm to get pristine LLVM IR from the "
+           "frontend by not running any LLVM passes at all">;
+def disable_llvm_optzns : Flag<["-"], "disable-llvm-optzns">,
+  Alias<disable_llvm_passes>;
+def disable_lifetimemarkers : Flag<["-"], "disable-lifetime-markers">,
+  HelpText<"Disable lifetime-markers emission even when optimizations are "
+           "enabled">;
+def disable_O0_optnone : Flag<["-"], "disable-O0-optnone">,
+  HelpText<"Disable adding the optnone attribute to functions at O0">;
+def disable_red_zone : Flag<["-"], "disable-red-zone">,
+  HelpText<"Do not emit code that uses the red zone.">;
+def dwarf_ext_refs : Flag<["-"], "dwarf-ext-refs">,
+  HelpText<"Generate debug info with external references to clang modules"
+           " or precompiled headers">;
+def dwarf_explicit_import : Flag<["-"], "dwarf-explicit-import">,
+  HelpText<"Generate explicit import from anonymous namespace to containing"
+           " scope">;
+def debug_forward_template_params : Flag<["-"], "debug-forward-template-params">,
+  HelpText<"Emit complete descriptions of template parameters in forward"
+           " declarations">;
+def fforbid_guard_variables : Flag<["-"], "fforbid-guard-variables">,
+  HelpText<"Emit an error if a C++ static local initializer would need a guard variable">;
+def no_implicit_float : Flag<["-"], "no-implicit-float">,
+  HelpText<"Don't generate implicit floating point instructions">;
+def fdump_vtable_layouts : Flag<["-"], "fdump-vtable-layouts">,
+  HelpText<"Dump the layouts of all vtables that will be emitted in a translation unit">;
+def fmerge_functions : Flag<["-"], "fmerge-functions">,
+  HelpText<"Permit merging of identical functions when optimizing.">;
+def femit_coverage_notes : Flag<["-"], "femit-coverage-notes">,
+  HelpText<"Emit a gcov coverage notes file when compiling.">;
+def femit_coverage_data: Flag<["-"], "femit-coverage-data">,
+  HelpText<"Instrument the program to emit gcov coverage data when run.">;
+def coverage_data_file : Separate<["-"], "coverage-data-file">,
+  HelpText<"Emit coverage data to this filename.">;
+def coverage_data_file_EQ : Joined<["-"], "coverage-data-file=">,
+  Alias<coverage_data_file>;
+def coverage_notes_file : Separate<["-"], "coverage-notes-file">,
+  HelpText<"Emit coverage notes to this filename.">;
+def coverage_notes_file_EQ : Joined<["-"], "coverage-notes-file=">,
+  Alias<coverage_notes_file>;
+def coverage_version_EQ : Joined<["-"], "coverage-version=">,
+  HelpText<"Four-byte version string for gcov files.">;
+def test_coverage : Flag<["-"], "test-coverage">,
+  HelpText<"Do not generate coverage files or remove coverage changes from IR">;
+def dump_coverage_mapping : Flag<["-"], "dump-coverage-mapping">,
+  HelpText<"Dump the coverage mapping records, for testing">;
+def fuse_register_sized_bitfield_access: Flag<["-"], "fuse-register-sized-bitfield-access">,
+  HelpText<"Use register sized accesses to bit-fields, when possible.">;
+def relaxed_aliasing : Flag<["-"], "relaxed-aliasing">,
+  HelpText<"Turn off Type Based Alias Analysis">;
+def no_struct_path_tbaa : Flag<["-"], "no-struct-path-tbaa">,
+  HelpText<"Turn off struct-path aware Type Based Alias Analysis">;
+def new_struct_path_tbaa : Flag<["-"], "new-struct-path-tbaa">,
+  HelpText<"Enable enhanced struct-path aware Type Based Alias Analysis">;
+def mdebug_pass : Separate<["-"], "mdebug-pass">,
+  HelpText<"Enable additional debug output">;
+def mframe_pointer_EQ : Joined<["-"], "mframe-pointer=">,
+  HelpText<"Specify which frame pointers to retain (all, non-leaf, none).">, Values<"all,non-leaf,none">;
+def mdisable_tail_calls : Flag<["-"], "mdisable-tail-calls">,
+  HelpText<"Disable tail call optimization, keeping the call stack accurate">;
+def menable_no_infinities : Flag<["-"], "menable-no-infs">,
+  HelpText<"Allow optimization to assume there are no infinities.">;
+def menable_no_nans : Flag<["-"], "menable-no-nans">,
+  HelpText<"Allow optimization to assume there are no NaNs.">;
+def menable_unsafe_fp_math : Flag<["-"], "menable-unsafe-fp-math">,
+  HelpText<"Allow unsafe floating-point math optimizations which may decrease "
+           "precision">;
+def mreassociate : Flag<["-"], "mreassociate">,
+  HelpText<"Allow reassociation transformations for floating-point instructions">;
+def mabi_EQ_ieeelongdouble : Flag<["-"], "mabi=ieeelongdouble">,
+  HelpText<"Use IEEE 754 quadruple-precision for long double">;
+def mfloat_abi : Separate<["-"], "mfloat-abi">,
+  HelpText<"The float ABI to use">;
+def mtp : Separate<["-"], "mtp">,
+  HelpText<"Mode for reading thread pointer">;
+def mlimit_float_precision : Separate<["-"], "mlimit-float-precision">,
+  HelpText<"Limit float precision to the given value">;
+def split_stacks : Flag<["-"], "split-stacks">,
+  HelpText<"Try to use a split stack if possible.">;
+def mregparm : Separate<["-"], "mregparm">,
+  HelpText<"Limit the number of registers available for integer arguments">;
+def msmall_data_limit : Separate<["-"], "msmall-data-limit">,
+  HelpText<"Put global and static data smaller than the limit into a special section">;
+def munwind_tables : Flag<["-"], "munwind-tables">,
+  HelpText<"Generate unwinding tables for all functions">;
+def mconstructor_aliases : Flag<["-"], "mconstructor-aliases">,
+  HelpText<"Emit complete constructors and destructors as aliases when possible">;
+def mlink_bitcode_file : Separate<["-"], "mlink-bitcode-file">,
+  HelpText<"Link the given bitcode file before performing optimizations.">;
+def mlink_builtin_bitcode : Separate<["-"], "mlink-builtin-bitcode">,
+  HelpText<"Link and internalize needed symbols from the given bitcode file "
+           "before performing optimizations.">;
+def mlink_cuda_bitcode : Separate<["-"], "mlink-cuda-bitcode">,
+  Alias<mlink_builtin_bitcode>;
+def vectorize_loops : Flag<["-"], "vectorize-loops">,
+  HelpText<"Run the Loop vectorization passes">;
+def vectorize_slp : Flag<["-"], "vectorize-slp">,
+  HelpText<"Run the SLP vectorization passes">;
+def dependent_lib : Joined<["--"], "dependent-lib=">,
+  HelpText<"Add dependent library">;
+def linker_option : Joined<["--"], "linker-option=">,
+  HelpText<"Add linker option">;
+def fsanitize_coverage_type : Joined<["-"], "fsanitize-coverage-type=">,
+                              HelpText<"Sanitizer coverage type">;
+def fsanitize_coverage_indirect_calls
+    : Flag<["-"], "fsanitize-coverage-indirect-calls">,
+      HelpText<"Enable sanitizer coverage for indirect calls">;
+def fsanitize_coverage_trace_bb
+    : Flag<["-"], "fsanitize-coverage-trace-bb">,
+      HelpText<"Enable basic block tracing in sanitizer coverage">;
+def fsanitize_coverage_trace_cmp
+    : Flag<["-"], "fsanitize-coverage-trace-cmp">,
+      HelpText<"Enable cmp instruction tracing in sanitizer coverage">;
+def fsanitize_coverage_trace_div
+    : Flag<["-"], "fsanitize-coverage-trace-div">,
+      HelpText<"Enable div instruction tracing in sanitizer coverage">;
+def fsanitize_coverage_trace_gep
+    : Flag<["-"], "fsanitize-coverage-trace-gep">,
+      HelpText<"Enable gep instruction tracing in sanitizer coverage">;
+def fsanitize_coverage_8bit_counters
+    : Flag<["-"], "fsanitize-coverage-8bit-counters">,
+      HelpText<"Enable frequency counters in sanitizer coverage">;
+def fsanitize_coverage_inline_8bit_counters
+    : Flag<["-"], "fsanitize-coverage-inline-8bit-counters">,
+      HelpText<"Enable inline 8-bit counters in sanitizer coverage">;
+def fsanitize_coverage_inline_bool_flag
+    : Flag<["-"], "fsanitize-coverage-inline-bool-flag">,
+      HelpText<"Enable inline bool flag in sanitizer coverage">;
+def fsanitize_coverage_pc_table
+    : Flag<["-"], "fsanitize-coverage-pc-table">,
+      HelpText<"Create a table of coverage-instrumented PCs">;
+def fsanitize_coverage_trace_pc
+    : Flag<["-"], "fsanitize-coverage-trace-pc">,
+      HelpText<"Enable PC tracing in sanitizer coverage">;
+def fsanitize_coverage_trace_pc_guard
+    : Flag<["-"], "fsanitize-coverage-trace-pc-guard">,
+      HelpText<"Enable PC tracing with guard in sanitizer coverage">;
+def fsanitize_coverage_no_prune
+    : Flag<["-"], "fsanitize-coverage-no-prune">,
+      HelpText<"Disable coverage pruning (i.e. instrument all blocks/edges)">;
+def fsanitize_coverage_stack_depth
+    : Flag<["-"], "fsanitize-coverage-stack-depth">,
+      HelpText<"Enable max stack depth tracing">;
+def fpatchable_function_entry_offset_EQ
+    : Joined<["-"], "fpatchable-function-entry-offset=">, MetaVarName<"<M>">,
+      HelpText<"Generate M NOPs before function entry">;
+def fprofile_instrument_EQ : Joined<["-"], "fprofile-instrument=">,
+    HelpText<"Enable PGO instrumentation. The accepted value is clang, llvm, "
+             "or none">, Values<"none,clang,llvm">;
+def fprofile_instrument_path_EQ : Joined<["-"], "fprofile-instrument-path=">,
+    HelpText<"Generate instrumented code to collect execution counts into "
+             "<file> (overridden by LLVM_PROFILE_FILE env var)">;
+def fprofile_instrument_use_path_EQ :
+    Joined<["-"], "fprofile-instrument-use-path=">,
+    HelpText<"Specify the profile path in PGO use compilation">;
+def flto_visibility_public_std:
+    Flag<["-"], "flto-visibility-public-std">,
+    HelpText<"Use public LTO visibility for classes in std and stdext namespaces">;
+def flto_unit: Flag<["-"], "flto-unit">,
+    HelpText<"Emit IR to support LTO unit features (CFI, whole program vtable opt)">;
+def fno_lto_unit: Flag<["-"], "fno-lto-unit">;
+def fdebug_pass_manager : Flag<["-"], "fdebug-pass-manager">,
+    HelpText<"Prints debug information for the new pass manager">;
+def fno_debug_pass_manager : Flag<["-"], "fno-debug-pass-manager">,
+    HelpText<"Disables debug printing for the new pass manager">;
+// The driver option takes the key as a parameter to the -msign-return-address=
+// and -mbranch-protection= options, but CC1 has a separate option so we
+// don't have to parse the parameter twice.
+def msign_return_address_key_EQ : Joined<["-"], "msign-return-address-key=">,
+    Values<"a_key,b_key">;
+def mbranch_target_enforce : Flag<["-"], "mbranch-target-enforce">;
+def fno_dllexport_inlines : Flag<["-"], "fno-dllexport-inlines">;
+def cfguard_no_checks : Flag<["-"], "cfguard-no-checks">,
+    HelpText<"Emit Windows Control Flow Guard tables only (no checks)">;
+def cfguard : Flag<["-"], "cfguard">,
+    HelpText<"Emit Windows Control Flow Guard tables and checks">;
+
+def fdenormal_fp_math_f32_EQ : Joined<["-"], "fdenormal-fp-math-f32=">,
+   Group<f_Group>;
+
+//===----------------------------------------------------------------------===//
+// Dependency Output Options
+//===----------------------------------------------------------------------===//
+
+def sys_header_deps : Flag<["-"], "sys-header-deps">,
+  HelpText<"Include system headers in dependency output">;
+def module_file_deps : Flag<["-"], "module-file-deps">,
+  HelpText<"Include module files in dependency output">;
+def header_include_file : Separate<["-"], "header-include-file">,
+  HelpText<"Filename (or -) to write header include output to">;
+def show_includes : Flag<["--"], "show-includes">,
+  HelpText<"Print cl.exe style /showIncludes to stdout">;
+
+//===----------------------------------------------------------------------===//
+// Diagnostic Options
+//===----------------------------------------------------------------------===//
+
+def diagnostic_log_file : Separate<["-"], "diagnostic-log-file">,
+  HelpText<"Filename (or -) to log diagnostics to">;
+def diagnostic_serialized_file : Separate<["-"], "serialize-diagnostic-file">,
+  MetaVarName<"<filename>">,
+  HelpText<"File for serializing diagnostics in a binary format">;
+
+def fdiagnostics_format : Separate<["-"], "fdiagnostics-format">,
+  HelpText<"Change diagnostic formatting to match IDE and command line tools">, Values<"clang,msvc,msvc-fallback,vi">;
+def fdiagnostics_show_category : Separate<["-"], "fdiagnostics-show-category">,
+  HelpText<"Print diagnostic category">, Values<"none,id,name">;
+def fno_diagnostics_use_presumed_location : Flag<["-"], "fno-diagnostics-use-presumed-location">,
+  HelpText<"Ignore #line directives when displaying diagnostic locations">;
+def ftabstop : Separate<["-"], "ftabstop">, MetaVarName<"<N>">,
+  HelpText<"Set the tab stop distance.">;
+def ferror_limit : Separate<["-"], "ferror-limit">, MetaVarName<"<N>">,
+  HelpText<"Set the maximum number of errors to emit before stopping (0 = no limit).">;
+def fmacro_backtrace_limit : Separate<["-"], "fmacro-backtrace-limit">, MetaVarName<"<N>">,
+  HelpText<"Set the maximum number of entries to print in a macro expansion backtrace (0 = no limit).">;
+def ftemplate_backtrace_limit : Separate<["-"], "ftemplate-backtrace-limit">, MetaVarName<"<N>">,
+  HelpText<"Set the maximum number of entries to print in a template instantiation backtrace (0 = no limit).">;
+def fconstexpr_backtrace_limit : Separate<["-"], "fconstexpr-backtrace-limit">, MetaVarName<"<N>">,
+  HelpText<"Set the maximum number of entries to print in a constexpr evaluation backtrace (0 = no limit).">;
+def fspell_checking_limit : Separate<["-"], "fspell-checking-limit">, MetaVarName<"<N>">,
+  HelpText<"Set the maximum number of times to perform spell checking on unrecognized identifiers (0 = no limit).">;
+def fcaret_diagnostics_max_lines :
+  Separate<["-"], "fcaret-diagnostics-max-lines">, MetaVarName<"<N>">,
+  HelpText<"Set the maximum number of source lines to show in a caret diagnostic">;
+def verify_EQ : CommaJoined<["-"], "verify=">,
+  MetaVarName<"<prefixes>">,
+  HelpText<"Verify diagnostic output using comment directives that start with"
+           " prefixes in the comma-separated sequence <prefixes>">;
+def verify : Flag<["-"], "verify">,
+  HelpText<"Equivalent to -verify=expected">;
+def verify_ignore_unexpected : Flag<["-"], "verify-ignore-unexpected">,
+  HelpText<"Ignore unexpected diagnostic messages">;
+def verify_ignore_unexpected_EQ : CommaJoined<["-"], "verify-ignore-unexpected=">,
+  HelpText<"Ignore unexpected diagnostic messages">;
+def Wno_rewrite_macros : Flag<["-"], "Wno-rewrite-macros">,
+  HelpText<"Silence ObjC rewriting warnings">;
+
+//===----------------------------------------------------------------------===//
+// Frontend Options
+//===----------------------------------------------------------------------===//
+
+// This isn't normally used, it is just here so we can parse a
+// CompilerInvocation out of a driver-derived argument vector.
+def cc1 : Flag<["-"], "cc1">;
+def cc1as : Flag<["-"], "cc1as">;
+
+def ast_merge : Separate<["-"], "ast-merge">,
+  MetaVarName<"<ast file>">,
+  HelpText<"Merge the given AST file into the translation unit being compiled.">;
+def aux_target_cpu : Separate<["-"], "aux-target-cpu">,
+  HelpText<"Target a specific auxiliary cpu type">;
+def aux_target_feature : Separate<["-"], "aux-target-feature">,
+  HelpText<"Target specific auxiliary attributes">;
+def aux_triple : Separate<["-"], "aux-triple">,
+  HelpText<"Auxiliary target triple.">;
+def code_completion_at : Separate<["-"], "code-completion-at">,
+  MetaVarName<"<file>:<line>:<column>">,
+  HelpText<"Dump code-completion information at a location">;
+def remap_file : Separate<["-"], "remap-file">,
+  MetaVarName<"<from>;<to>">,
+  HelpText<"Replace the contents of the <from> file with the contents of the <to> file">;
+def code_completion_at_EQ : Joined<["-"], "code-completion-at=">,
+  Alias<code_completion_at>;
+def code_completion_macros : Flag<["-"], "code-completion-macros">,
+  HelpText<"Include macros in code-completion results">;
+def code_completion_patterns : Flag<["-"], "code-completion-patterns">,
+  HelpText<"Include code patterns in code-completion results">;
+def no_code_completion_globals : Flag<["-"], "no-code-completion-globals">,
+  HelpText<"Do not include global declarations in code-completion results.">;
+def no_code_completion_ns_level_decls : Flag<["-"], "no-code-completion-ns-level-decls">,
+  HelpText<"Do not include declarations inside namespaces (incl. global namespace) in the code-completion results.">;
+def code_completion_brief_comments : Flag<["-"], "code-completion-brief-comments">,
+  HelpText<"Include brief documentation comments in code-completion results.">;
+def code_completion_with_fixits : Flag<["-"], "code-completion-with-fixits">,
+  HelpText<"Include code completion results which require small fix-its.">;
+def disable_free : Flag<["-"], "disable-free">,
+  HelpText<"Disable freeing of memory on exit">;
+def discard_value_names : Flag<["-"], "discard-value-names">,
+  HelpText<"Discard value names in LLVM IR">;
+def load : Separate<["-"], "load">, MetaVarName<"<dsopath>">,
+  HelpText<"Load the named plugin (dynamic shared object)">;
+def plugin : Separate<["-"], "plugin">, MetaVarName<"<name>">,
+  HelpText<"Use the named plugin action instead of the default action (use \"help\" to list available options)">;
+def plugin_arg : JoinedAndSeparate<["-"], "plugin-arg-">,
+    MetaVarName<"<name> <arg>">,
+    HelpText<"Pass <arg> to plugin <name>">;
+def add_plugin : Separate<["-"], "add-plugin">, MetaVarName<"<name>">,
+  HelpText<"Use the named plugin action in addition to the default action">;
+def ast_dump_filter : Separate<["-"], "ast-dump-filter">,
+  MetaVarName<"<dump_filter>">,
+  HelpText<"Use with -ast-dump or -ast-print to dump/print only AST declaration"
+           " nodes having a certain substring in a qualified name. Use"
+           " -ast-list to list all filterable declaration node names.">;
+def fno_modules_global_index : Flag<["-"], "fno-modules-global-index">,
+  HelpText<"Do not automatically generate or update the global module index">;
+def fno_modules_error_recovery : Flag<["-"], "fno-modules-error-recovery">,
+  HelpText<"Do not automatically import modules for error recovery">;
+def fmodule_map_file_home_is_cwd : Flag<["-"], "fmodule-map-file-home-is-cwd">,
+  HelpText<"Use the current working directory as the home directory of "
+           "module maps specified by -fmodule-map-file=<FILE>">;
+def fmodule_feature : Separate<["-"], "fmodule-feature">,
+  MetaVarName<"<feature>">,
+  HelpText<"Enable <feature> in module map requires declarations">;
+def fmodules_embed_file_EQ : Joined<["-"], "fmodules-embed-file=">,
+  MetaVarName<"<file>">,
+  HelpText<"Embed the contents of the specified file into the module file "
+           "being compiled.">;
+def fmodules_embed_all_files : Joined<["-"], "fmodules-embed-all-files">,
+  HelpText<"Embed the contents of all files read by this compilation into "
+           "the produced module file.">;
+def fmodules_local_submodule_visibility :
+  Flag<["-"], "fmodules-local-submodule-visibility">,
+  HelpText<"Enforce name visibility rules across submodules of the same "
+           "top-level module.">;
+def fmodules_codegen :
+  Flag<["-"], "fmodules-codegen">,
+  HelpText<"Generate code for uses of this module that assumes an explicit "
+           "object file will be built for the module">;
+def fmodules_debuginfo :
+  Flag<["-"], "fmodules-debuginfo">,
+  HelpText<"Generate debug info for types in an object file built from this "
+           "module and do not generate them elsewhere">;
+def fmodule_format_EQ : Joined<["-"], "fmodule-format=">,
+  HelpText<"Select the container format for clang modules and PCH. "
+           "Supported options are 'raw' and 'obj'.">;
+def ftest_module_file_extension_EQ :
+  Joined<["-"], "ftest-module-file-extension=">,
+  HelpText<"introduce a module file extension for testing purposes. "
+           "The argument is parsed as blockname:major:minor:hashed:user info">;
+def fconcepts_ts : Flag<["-"], "fconcepts-ts">,
+  HelpText<"Enable C++ Extensions for Concepts. (deprecated - use -std=c++2a)">;
+def fno_concept_satisfaction_caching : Flag<["-"],
+                                            "fno-concept-satisfaction-caching">,
+  HelpText<"Disable satisfaction caching for C++2a Concepts.">;
+
+def frecovery_ast : Flag<["-"], "frecovery-ast">,
+  HelpText<"Preserve expressions in AST rather than dropping them when "
+           "encountering semantic errors">;
+def fno_recovery_ast : Flag<["-"], "fno-recovery-ast">;
+def frecovery_ast_type : Flag<["-"], "frecovery-ast-type">,
+  HelpText<"Preserve the type for recovery expressions when possible "
+          "(experimental)">;
+def fno_recovery_ast_type : Flag<["-"], "fno-recovery-ast-type">;
+
+let Group = Action_Group in {
+
+def Eonly : Flag<["-"], "Eonly">,
+  HelpText<"Just run preprocessor, no output (for timings)">;
+def dump_raw_tokens : Flag<["-"], "dump-raw-tokens">,
+  HelpText<"Lex file in raw mode and dump raw tokens">;
+def analyze : Flag<["-"], "analyze">,
+  HelpText<"Run static analysis engine">;
+def dump_tokens : Flag<["-"], "dump-tokens">,
+  HelpText<"Run preprocessor, dump internal rep of tokens">;
+def init_only : Flag<["-"], "init-only">,
+  HelpText<"Only execute frontend initialization">;
+def fixit : Flag<["-"], "fixit">,
+  HelpText<"Apply fix-it advice to the input source">;
+def fixit_EQ : Joined<["-"], "fixit=">,
+  HelpText<"Apply fix-it advice creating a file with the given suffix">;
+def print_preamble : Flag<["-"], "print-preamble">,
+  HelpText<"Print the \"preamble\" of a file, which is a candidate for implicit"
+           " precompiled headers.">;
+def emit_html : Flag<["-"], "emit-html">,
+  HelpText<"Output input source as HTML">;
+def ast_print : Flag<["-"], "ast-print">,
+  HelpText<"Build ASTs and then pretty-print them">;
+def ast_list : Flag<["-"], "ast-list">,
+  HelpText<"Build ASTs and print the list of declaration node qualified names">;
+def ast_dump : Flag<["-"], "ast-dump">,
+  HelpText<"Build ASTs and then debug dump them">;
+def ast_dump_EQ : Joined<["-"], "ast-dump=">,
+  HelpText<"Build ASTs and then debug dump them in the specified format. "
+           "Supported formats include: default, json">;
+def ast_dump_all : Flag<["-"], "ast-dump-all">,
+  HelpText<"Build ASTs and then debug dump them, forcing deserialization">;
+def ast_dump_all_EQ : Joined<["-"], "ast-dump-all=">,
+  HelpText<"Build ASTs and then debug dump them in the specified format, "
+           "forcing deserialization. Supported formats include: default, json">;
+def ast_dump_decl_types : Flag<["-"], "ast-dump-decl-types">,
+  HelpText<"Include declaration types in AST dumps">;
+def templight_dump : Flag<["-"], "templight-dump">,
+  HelpText<"Dump templight information to stdout">;
+def ast_dump_lookups : Flag<["-"], "ast-dump-lookups">,
+  HelpText<"Build ASTs and then debug dump their name lookup tables">;
+def ast_view : Flag<["-"], "ast-view">,
+  HelpText<"Build ASTs and view them with GraphViz">;
+def emit_module : Flag<["-"], "emit-module">,
+  HelpText<"Generate pre-compiled module file from a module map">;
+def emit_module_interface : Flag<["-"], "emit-module-interface">,
+  HelpText<"Generate pre-compiled module file from a C++ module interface">;
+def emit_header_module : Flag<["-"], "emit-header-module">,
+  HelpText<"Generate pre-compiled module file from a set of header files">;
+def emit_pch : Flag<["-"], "emit-pch">,
+  HelpText<"Generate pre-compiled header file">;
+def emit_llvm_bc : Flag<["-"], "emit-llvm-bc">,
+  HelpText<"Build ASTs then convert to LLVM, emit .bc file">;
+def emit_llvm_only : Flag<["-"], "emit-llvm-only">,
+  HelpText<"Build ASTs and convert to LLVM, discarding output">;
+def emit_codegen_only : Flag<["-"], "emit-codegen-only">,
+  HelpText<"Generate machine code, but discard output">;
+def emit_obj : Flag<["-"], "emit-obj">,
+  HelpText<"Emit native object files">;
+def rewrite_test : Flag<["-"], "rewrite-test">,
+  HelpText<"Rewriter playground">;
+def rewrite_macros : Flag<["-"], "rewrite-macros">,
+  HelpText<"Expand macros without full preprocessing">;
+def migrate : Flag<["-"], "migrate">,
+  HelpText<"Migrate source code">;
+def compiler_options_dump : Flag<["-"], "compiler-options-dump">,
+  HelpText<"Dump the compiler configuration options">;
+def print_dependency_directives_minimized_source : Flag<["-"],
+  "print-dependency-directives-minimized-source">,
+  HelpText<"Print the output of the dependency directives source minimizer">;
+}
+
+def emit_llvm_uselists : Flag<["-"], "emit-llvm-uselists">,
+  HelpText<"Preserve order of LLVM use-lists when serializing">;
+def no_emit_llvm_uselists : Flag<["-"], "no-emit-llvm-uselists">,
+  HelpText<"Don't preserve order of LLVM use-lists when serializing">;
+
+def mt_migrate_directory : Separate<["-"], "mt-migrate-directory">,
+  HelpText<"Directory for temporary files produced during ARC or ObjC migration">;
+def arcmt_check : Flag<["-"], "arcmt-check">,
+  HelpText<"Check for ARC migration issues that need manual handling">;
+def arcmt_modify : Flag<["-"], "arcmt-modify">,
+  HelpText<"Apply modifications to files to conform to ARC">;
+def arcmt_migrate : Flag<["-"], "arcmt-migrate">,
+  HelpText<"Apply modifications and produces temporary files that conform to ARC">;
+
+def opt_record_file : Separate<["-"], "opt-record-file">,
+  HelpText<"File name to use for YAML optimization record output">;
+def opt_record_passes : Separate<["-"], "opt-record-passes">,
+  HelpText<"Only record remark information for passes whose names match the given regular expression">;
+def opt_record_format : Separate<["-"], "opt-record-format">,
+  HelpText<"The format used for serializing remarks (default: YAML)">;
+
+def print_stats : Flag<["-"], "print-stats">,
+  HelpText<"Print performance metrics and statistics">;
+def stats_file : Joined<["-"], "stats-file=">,
+  HelpText<"Filename to write statistics to">;
+def fdump_record_layouts : Flag<["-"], "fdump-record-layouts">,
+  HelpText<"Dump record layout information">;
+def fdump_record_layouts_simple : Flag<["-"], "fdump-record-layouts-simple">,
+  HelpText<"Dump record layout information in a simple form used for testing">;
+def fix_what_you_can : Flag<["-"], "fix-what-you-can">,
+  HelpText<"Apply fix-it advice even in the presence of unfixable errors">;
+def fix_only_warnings : Flag<["-"], "fix-only-warnings">,
+  HelpText<"Apply fix-it advice only for warnings, not errors">;
+def fixit_recompile : Flag<["-"], "fixit-recompile">,
+  HelpText<"Apply fix-it changes and recompile">;
+def fixit_to_temp : Flag<["-"], "fixit-to-temporary">,
+  HelpText<"Apply fix-it changes to temporary files">;
+
+def foverride_record_layout_EQ : Joined<["-"], "foverride-record-layout=">,
+  HelpText<"Override record layouts with those in the given file">;
+def pch_through_header_EQ : Joined<["-"], "pch-through-header=">,
+  HelpText<"Stop PCH generation after including this file.  When using a PCH, "
+           "skip tokens until after this file is included.">;
+def pch_through_hdrstop_create : Flag<["-"], "pch-through-hdrstop-create">,
+  HelpText<"When creating a PCH, stop PCH generation after #pragma hdrstop.">;
+def pch_through_hdrstop_use : Flag<["-"], "pch-through-hdrstop-use">,
+  HelpText<"When using a PCH, skip tokens until after a #pragma hdrstop.">;
+def fno_pch_timestamp : Flag<["-"], "fno-pch-timestamp">,
+  HelpText<"Disable inclusion of timestamp in precompiled headers">;
+def building_pch_with_obj : Flag<["-"], "building-pch-with-obj">,
+  HelpText<"This compilation is part of building a PCH with corresponding object file.">;
+
+def aligned_alloc_unavailable : Flag<["-"], "faligned-alloc-unavailable">,
+  HelpText<"Aligned allocation/deallocation functions are unavailable">;
+
+//===----------------------------------------------------------------------===//
+// Language Options
+//===----------------------------------------------------------------------===//
+
+let Flags = [CC1Option, CC1AsOption, NoDriverOption] in {
+
+def version : Flag<["-"], "version">,
+  HelpText<"Print the compiler version">;
+def main_file_name : Separate<["-"], "main-file-name">,
+  HelpText<"Main file name to use for debug info and source if missing">;
+def split_dwarf_output : Separate<["-"], "split-dwarf-output">,
+  HelpText<"File name to use for split dwarf debug info output">;
+
+}
+
+def fblocks_runtime_optional : Flag<["-"], "fblocks-runtime-optional">,
+  HelpText<"Weakly link in the blocks runtime">;
+def fexternc_nounwind : Flag<["-"], "fexternc-nounwind">,
+  HelpText<"Assume all functions with C linkage do not unwind">;
+def split_dwarf_file : Separate<["-"], "split-dwarf-file">,
+  HelpText<"Name of the split dwarf debug info file to encode in the object file">;
+def fno_wchar : Flag<["-"], "fno-wchar">,
+  HelpText<"Disable C++ builtin type wchar_t">;
+def fconstant_string_class : Separate<["-"], "fconstant-string-class">,
+  MetaVarName<"<class name>">,
+  HelpText<"Specify the class to use for constant Objective-C string objects.">;
+def fobjc_arc_cxxlib_EQ : Joined<["-"], "fobjc-arc-cxxlib=">,
+  HelpText<"Objective-C++ Automatic Reference Counting standard library kind">, Values<"libc++,libstdc++,none">;
+def fobjc_runtime_has_weak : Flag<["-"], "fobjc-runtime-has-weak">,
+  HelpText<"The target Objective-C runtime supports ARC weak operations">;
+def fobjc_dispatch_method_EQ : Joined<["-"], "fobjc-dispatch-method=">,
+  HelpText<"Objective-C dispatch method to use">, Values<"legacy,non-legacy,mixed">;
+def disable_objc_default_synthesize_properties : Flag<["-"], "disable-objc-default-synthesize-properties">,
+  HelpText<"disable the default synthesis of Objective-C properties">;
+def fencode_extended_block_signature : Flag<["-"], "fencode-extended-block-signature">,
+  HelpText<"enable extended encoding of block type signature">;
+def function_alignment : Separate<["-"], "function-alignment">,
+    HelpText<"default alignment for functions">;
+def pic_level : Separate<["-"], "pic-level">,
+  HelpText<"Value for __PIC__">;
+def pic_is_pie : Flag<["-"], "pic-is-pie">,
+  HelpText<"File is for a position independent executable">;
+def fno_validate_pch : Flag<["-"], "fno-validate-pch">,
+  HelpText<"Disable validation of precompiled headers">;
+def fallow_pch_with_errors : Flag<["-"], "fallow-pch-with-compiler-errors">,
+  HelpText<"Accept a PCH file that was created with compiler errors">;
+def dump_deserialized_pch_decls : Flag<["-"], "dump-deserialized-decls">,
+  HelpText<"Dump declarations that are deserialized from PCH, for testing">;
+def error_on_deserialized_pch_decl : Separate<["-"], "error-on-deserialized-decl">,
+  HelpText<"Emit error if a specific declaration is deserialized from PCH, for testing">;
+def error_on_deserialized_pch_decl_EQ : Joined<["-"], "error-on-deserialized-decl=">,
+  Alias<error_on_deserialized_pch_decl>;
+def static_define : Flag<["-"], "static-define">,
+  HelpText<"Should __STATIC__ be defined">;
+def stack_protector : Separate<["-"], "stack-protector">,
+  HelpText<"Enable stack protectors">;
+def stack_protector_buffer_size : Separate<["-"], "stack-protector-buffer-size">,
+  HelpText<"Lower bound for a buffer to be considered for stack protection">;
+def fvisibility : Separate<["-"], "fvisibility">,
+  HelpText<"Default type and symbol visibility">;
+def ftype_visibility : Separate<["-"], "ftype-visibility">,
+  HelpText<"Default type visibility">;
+def fapply_global_visibility_to_externs : Flag<["-"], "fapply-global-visibility-to-externs">,
+  HelpText<"Apply global symbol visibility to external declarations without an explicit visibility">;
+def ftemplate_depth : Separate<["-"], "ftemplate-depth">,
+  HelpText<"Maximum depth of recursive template instantiation">;
+def foperator_arrow_depth : Separate<["-"], "foperator-arrow-depth">,
+  HelpText<"Maximum number of 'operator->'s to call for a member access">;
+def fconstexpr_depth : Separate<["-"], "fconstexpr-depth">,
+  HelpText<"Maximum depth of recursive constexpr function calls">;
+def fconstexpr_steps : Separate<["-"], "fconstexpr-steps">,
+  HelpText<"Maximum number of steps in constexpr function evaluation">;
+def fbracket_depth : Separate<["-"], "fbracket-depth">,
+  HelpText<"Maximum nesting level for parentheses, brackets, and braces">;
+def fconst_strings : Flag<["-"], "fconst-strings">,
+  HelpText<"Use a const qualified type for string literals in C and ObjC">;
+def fno_const_strings : Flag<["-"], "fno-const-strings">,
+  HelpText<"Don't use a const qualified type for string literals in C and ObjC">;
+def fno_bitfield_type_align : Flag<["-"], "fno-bitfield-type-align">,
+  HelpText<"Ignore bit-field types when aligning structures">;
+def ffake_address_space_map : Flag<["-"], "ffake-address-space-map">,
+  HelpText<"Use a fake address space map; OpenCL testing purposes only">;
+def faddress_space_map_mangling_EQ : Joined<["-"], "faddress-space-map-mangling=">, MetaVarName<"<yes|no|target>">,
+  HelpText<"Set the mode for address space map based mangling; OpenCL testing purposes only">;
+def funknown_anytype : Flag<["-"], "funknown-anytype">,
+  HelpText<"Enable parser support for the __unknown_anytype type; for testing purposes only">;
+def fdebugger_support : Flag<["-"], "fdebugger-support">,
+  HelpText<"Enable special debugger support behavior">;
+def fdebugger_cast_result_to_id : Flag<["-"], "fdebugger-cast-result-to-id">,
+  HelpText<"Enable casting unknown expression results to id">;
+def fdebugger_objc_literal : Flag<["-"], "fdebugger-objc-literal">,
+  HelpText<"Enable special debugger support for Objective-C subscripting and literals">;
+def fdeprecated_macro : Flag<["-"], "fdeprecated-macro">,
+  HelpText<"Defines the __DEPRECATED macro">;
+def fno_deprecated_macro : Flag<["-"], "fno-deprecated-macro">,
+  HelpText<"Undefines the __DEPRECATED macro">;
+def fobjc_subscripting_legacy_runtime : Flag<["-"], "fobjc-subscripting-legacy-runtime">,
+  HelpText<"Allow Objective-C array and dictionary subscripting in legacy runtime">;
+def vtordisp_mode_EQ : Joined<["-"], "vtordisp-mode=">,
+  HelpText<"Control vtordisp placement on win32 targets">;
+def fnative_half_type: Flag<["-"], "fnative-half-type">,
+  HelpText<"Use the native half type for __fp16 instead of promoting to float">;
+def fnative_half_arguments_and_returns : Flag<["-"], "fnative-half-arguments-and-returns">,
+  HelpText<"Use the native __fp16 type for arguments and returns (and skip ABI-specific lowering)">;
+def fallow_half_arguments_and_returns : Flag<["-"], "fallow-half-arguments-and-returns">,
+  HelpText<"Allow function arguments and returns of type half">;
+def fdefault_calling_conv_EQ : Joined<["-"], "fdefault-calling-conv=">,
+  HelpText<"Set default calling convention">, Values<"cdecl,fastcall,stdcall,vectorcall,regcall">;
+def finclude_default_header : Flag<["-"], "finclude-default-header">,
+  HelpText<"Include default header file for OpenCL">;
+def fdeclare_opencl_builtins : Flag<["-"], "fdeclare-opencl-builtins">,
+  HelpText<"Add OpenCL builtin function declarations (experimental)">;
+def fpreserve_vec3_type : Flag<["-"], "fpreserve-vec3-type">,
+  HelpText<"Preserve 3-component vector type">;
+def fwchar_type_EQ : Joined<["-"], "fwchar-type=">,
+  HelpText<"Select underlying type for wchar_t">, Values<"char,short,int">;
+def fsigned_wchar : Flag<["-"], "fsigned-wchar">,
+  HelpText<"Use a signed type for wchar_t">;
+def fno_signed_wchar : Flag<["-"], "fno-signed-wchar">,
+  HelpText<"Use an unsigned type for wchar_t">;
+def fcompatibility_qualified_id_block_param_type_checking : Flag<["-"], "fcompatibility-qualified-id-block-type-checking">,
+  HelpText<"Allow using blocks with parameters of more specific type than "
+           "the type system guarantees when a parameter is qualified id">;
+
+// FIXME: Remove these entirely once functionality/tests have been excised.
+def fobjc_gc_only : Flag<["-"], "fobjc-gc-only">, Group<f_Group>,
+  HelpText<"Use GC exclusively for Objective-C related memory management">;
+def fobjc_gc : Flag<["-"], "fobjc-gc">, Group<f_Group>,
+  HelpText<"Enable Objective-C garbage collection">;
+
+//===----------------------------------------------------------------------===//
+// Header Search Options
+//===----------------------------------------------------------------------===//
+
+def nostdsysteminc : Flag<["-"], "nostdsysteminc">,
+  HelpText<"Disable standard system #include directories">;
+def fdisable_module_hash : Flag<["-"], "fdisable-module-hash">,
+  HelpText<"Disable the module hash">;
+def fmodules_hash_content : Flag<["-"], "fmodules-hash-content">,
+  HelpText<"Enable hashing the content of a module file">;
+def fmodules_strict_context_hash : Flag<["-"], "fmodules-strict-context-hash">,
+  HelpText<"Enable hashing of all compiler options that could impact the "
+           "semantics of a module in an implicit build">,
+  MarshallingInfoFlag<"HeaderSearchOpts->ModulesStrictContextHash", "false">;
+def c_isystem : JoinedOrSeparate<["-"], "c-isystem">, MetaVarName<"<directory>">,
+  HelpText<"Add directory to the C SYSTEM include search path">;
+def objc_isystem : JoinedOrSeparate<["-"], "objc-isystem">,
+  MetaVarName<"<directory>">,
+  HelpText<"Add directory to the ObjC SYSTEM include search path">;
+def objcxx_isystem : JoinedOrSeparate<["-"], "objcxx-isystem">,
+  MetaVarName<"<directory>">,
+  HelpText<"Add directory to the ObjC++ SYSTEM include search path">;
+def internal_isystem : JoinedOrSeparate<["-"], "internal-isystem">,
+  MetaVarName<"<directory>">,
+  HelpText<"Add directory to the internal system include search path; these "
+           "are assumed to not be user-provided and are used to model system "
+           "and standard headers' paths.">;
+def internal_externc_isystem : JoinedOrSeparate<["-"], "internal-externc-isystem">,
+  MetaVarName<"<directory>">,
+  HelpText<"Add directory to the internal system include search path with "
+           "implicit extern \"C\" semantics; these are assumed to not be "
+           "user-provided and are used to model system and standard headers' "
+           "paths.">;
+
+//===----------------------------------------------------------------------===//
+// Preprocessor Options
+//===----------------------------------------------------------------------===//
+
+def chain_include : Separate<["-"], "chain-include">, MetaVarName<"<file>">,
+  HelpText<"Include and chain a header file after turning it into PCH">;
+def preamble_bytes_EQ : Joined<["-"], "preamble-bytes=">,
+  HelpText<"Assume that the precompiled header is a precompiled preamble "
+           "covering the first N bytes of the main file">;
+def detailed_preprocessing_record : Flag<["-"], "detailed-preprocessing-record">,
+  HelpText<"include a detailed record of preprocessing actions">;
+def setup_static_analyzer : Flag<["-"], "setup-static-analyzer">,
+  HelpText<"Set up preprocessor for static analyzer (done automatically when static analyzer is run).">;
+def disable_pragma_debug_crash : Flag<["-"], "disable-pragma-debug-crash">,
+  HelpText<"Disable any #pragma clang __debug that can lead to crashing behavior. This is meant for testing.">;
+
+//===----------------------------------------------------------------------===//
+// OpenCL Options
+//===----------------------------------------------------------------------===//
+
+def cl_ext_EQ : CommaJoined<["-"], "cl-ext=">,
+  HelpText<"OpenCL only. Enable or disable OpenCL extensions. The argument is a comma-separated sequence of one or more extension names, each prefixed by '+' or '-'.">;
+
+//===----------------------------------------------------------------------===//
+// CUDA Options
+//===----------------------------------------------------------------------===//
+
+def fcuda_is_device : Flag<["-"], "fcuda-is-device">,
+  HelpText<"Generate code for CUDA device">;
+def fcuda_include_gpubinary : Separate<["-"], "fcuda-include-gpubinary">,
+  HelpText<"Incorporate CUDA device-side binary into host object file.">;
+def fcuda_allow_variadic_functions : Flag<["-"], "fcuda-allow-variadic-functions">,
+  HelpText<"Allow variadic functions in CUDA device code.">;
+def fno_cuda_host_device_constexpr : Flag<["-"], "fno-cuda-host-device-constexpr">,
+  HelpText<"Don't treat unattributed constexpr functions as __host__ __device__.">;
+
+//===----------------------------------------------------------------------===//
+// OpenMP Options
+//===----------------------------------------------------------------------===//
+
+def fopenmp_is_device : Flag<["-"], "fopenmp-is-device">,
+  HelpText<"Generate code only for an OpenMP target device.">;
+def fopenmp_host_ir_file_path : Separate<["-"], "fopenmp-host-ir-file-path">,
+  HelpText<"Path to the IR file produced by the frontend for the host.">;
+
+//===----------------------------------------------------------------------===//
+// SYCL Options
+//===----------------------------------------------------------------------===//
+
+def fsycl_is_device : Flag<["-"], "fsycl-is-device">,
+  HelpText<"Generate code for SYCL device.">;
+
+} // let Flags = [CC1Option]
+
+//===----------------------------------------------------------------------===//
+// cc1as-only Options
+//===----------------------------------------------------------------------===//
+
+let Flags = [CC1AsOption, NoDriverOption] in {
+
+// Language Options
+def n : Flag<["-"], "n">,
+  HelpText<"Don't automatically start assembly file with a text section">;
+
+// Frontend Options
+def filetype : Separate<["-"], "filetype">,
+    HelpText<"Specify the output file type ('asm', 'null', or 'obj')">;
+
+// Transliterate Options
+def output_asm_variant : Separate<["-"], "output-asm-variant">,
+    HelpText<"Select the asm variant index to use for output">;
+def show_encoding : Flag<["-"], "show-encoding">,
+    HelpText<"Show instruction encoding information in transliterate mode">;
+def show_inst : Flag<["-"], "show-inst">,
+    HelpText<"Show internal instruction representation in transliterate mode">;
+
+// Assemble Options
+def dwarf_debug_producer : Separate<["-"], "dwarf-debug-producer">,
+  HelpText<"The string to embed in the Dwarf debug AT_producer record.">;
+
+def defsym : Separate<["-"], "defsym">,
+  HelpText<"Define a value for a symbol">;
+
+} // let Flags = [CC1AsOption]
+
+//===----------------------------------------------------------------------===//
+// clang-cl Options
+//===----------------------------------------------------------------------===//
+
+def cl_Group : OptionGroup<"<clang-cl options>">, Flags<[CLOption]>,
+  HelpText<"CL.EXE COMPATIBILITY OPTIONS">;
+
+def cl_compile_Group : OptionGroup<"<clang-cl compile-only options>">,
+  Group<cl_Group>;
+
+def cl_ignored_Group : OptionGroup<"<clang-cl ignored options>">,
+  Group<cl_Group>;
+
+class CLFlag<string name> : Option<["/", "-"], name, KIND_FLAG>,
+  Group<cl_Group>, Flags<[CLOption, DriverOption]>;
+
+class CLCompileFlag<string name> : Option<["/", "-"], name, KIND_FLAG>,
+  Group<cl_compile_Group>, Flags<[CLOption, DriverOption]>;
+
+class CLIgnoredFlag<string name> : Option<["/", "-"], name, KIND_FLAG>,
+  Group<cl_ignored_Group>, Flags<[CLOption, DriverOption]>;
+
+class CLJoined<string name> : Option<["/", "-"], name, KIND_JOINED>,
+  Group<cl_Group>, Flags<[CLOption, DriverOption]>;
+
+class CLCompileJoined<string name> : Option<["/", "-"], name, KIND_JOINED>,
+  Group<cl_compile_Group>, Flags<[CLOption, DriverOption]>;
+
+class CLIgnoredJoined<string name> : Option<["/", "-"], name, KIND_JOINED>,
+  Group<cl_ignored_Group>, Flags<[CLOption, DriverOption, HelpHidden]>;
+
+class CLJoinedOrSeparate<string name> : Option<["/", "-"], name,
+  KIND_JOINED_OR_SEPARATE>, Group<cl_Group>, Flags<[CLOption, DriverOption]>;
+
+class CLCompileJoinedOrSeparate<string name> : Option<["/", "-"], name,
+  KIND_JOINED_OR_SEPARATE>, Group<cl_compile_Group>,
+  Flags<[CLOption, DriverOption]>;
+
+class CLRemainingArgsJoined<string name> : Option<["/", "-"], name,
+  KIND_REMAINING_ARGS_JOINED>, Group<cl_Group>, Flags<[CLOption, DriverOption]>;
+
+// Aliases:
+// (We don't put any of these in cl_compile_Group as the options they alias are
+// already in the right group.)
+
+def _SLASH_Brepro : CLFlag<"Brepro">,
+  HelpText<"Do not write current time into COFF output (breaks link.exe /incremental)">,
+  Alias<mno_incremental_linker_compatible>;
+def _SLASH_Brepro_ : CLFlag<"Brepro-">,
+  HelpText<"Write current time into COFF output (default)">,
+  Alias<mincremental_linker_compatible>;
+def _SLASH_C : CLFlag<"C">,
+  HelpText<"Do not discard comments when preprocessing">, Alias<C>;
+def _SLASH_c : CLFlag<"c">, HelpText<"Compile only">, Alias<c>;
+def _SLASH_d1PP : CLFlag<"d1PP">,
+  HelpText<"Retain macro definitions in /E mode">, Alias<dD>;
+def _SLASH_d1reportAllClassLayout : CLFlag<"d1reportAllClassLayout">,
+  HelpText<"Dump record layout information">,
+  Alias<Xclang>, AliasArgs<["-fdump-record-layouts"]>;
+def _SLASH_diagnostics_caret : CLFlag<"diagnostics:caret">,
+  HelpText<"Enable caret and column diagnostics (default)">;
+def _SLASH_diagnostics_column : CLFlag<"diagnostics:column">,
+  HelpText<"Disable caret diagnostics but keep column info">;
+def _SLASH_diagnostics_classic : CLFlag<"diagnostics:classic">,
+  HelpText<"Disable column and caret diagnostics">;
+def _SLASH_D : CLJoinedOrSeparate<"D">, HelpText<"Define macro">,
+  MetaVarName<"<macro[=value]>">, Alias<D>;
+def _SLASH_E : CLFlag<"E">, HelpText<"Preprocess to stdout">, Alias<E>;
+def _SLASH_fp_except : CLFlag<"fp:except">, HelpText<"">, Alias<ftrapping_math>;
+def _SLASH_fp_except_ : CLFlag<"fp:except-">,
+  HelpText<"">, Alias<fno_trapping_math>;
+def _SLASH_fp_fast : CLFlag<"fp:fast">, HelpText<"">, Alias<ffast_math>;
+def _SLASH_fp_precise : CLFlag<"fp:precise">,
+  HelpText<"">, Alias<fno_fast_math>;
+def _SLASH_fp_strict : CLFlag<"fp:strict">, HelpText<"">, Alias<fno_fast_math>;
+def _SLASH_GA : CLFlag<"GA">, Alias<ftlsmodel_EQ>, AliasArgs<["local-exec"]>,
+  HelpText<"Assume thread-local variables are defined in the executable">;
+def _SLASH_GR : CLFlag<"GR">, HelpText<"Emit RTTI data (default)">;
+def _SLASH_GR_ : CLFlag<"GR-">, HelpText<"Do not emit RTTI data">;
+def _SLASH_GF : CLIgnoredFlag<"GF">,
+  HelpText<"Enable string pooling (default)">;
+def _SLASH_GF_ : CLFlag<"GF-">, HelpText<"Disable string pooling">,
+  Alias<fwritable_strings>;
+def _SLASH_GS : CLFlag<"GS">,
+  HelpText<"Enable buffer security check (default)">;
+def _SLASH_GS_ : CLFlag<"GS-">, HelpText<"Disable buffer security check">;
+def : CLFlag<"Gs">, HelpText<"Use stack probes (default)">,
+  Alias<mstack_probe_size>, AliasArgs<["4096"]>;
+def _SLASH_Gs : CLJoined<"Gs">,
+  HelpText<"Set stack probe size (default 4096)">, Alias<mstack_probe_size>;
+def _SLASH_Gy : CLFlag<"Gy">, HelpText<"Put each function in its own section">,
+  Alias<ffunction_sections>;
+def _SLASH_Gy_ : CLFlag<"Gy-">,
+  HelpText<"Do not put each function in its own section (default)">,
+  Alias<fno_function_sections>;
+def _SLASH_Gw : CLFlag<"Gw">, HelpText<"Put each data item in its own section">,
+  Alias<fdata_sections>;
+def _SLASH_Gw_ : CLFlag<"Gw-">,
+  HelpText<"Do not put each data item in its own section (default)">,
+  Alias<fno_data_sections>;
+def _SLASH_help : CLFlag<"help">, Alias<help>,
+  HelpText<"Display available options">;
+def _SLASH_HELP : CLFlag<"HELP">, Alias<help>;
+def _SLASH_I : CLJoinedOrSeparate<"I">,
+  HelpText<"Add directory to include search path">, MetaVarName<"<dir>">,
+  Alias<I>;
+def _SLASH_J : CLFlag<"J">, HelpText<"Make char type unsigned">,
+  Alias<funsigned_char>;
+
+// The _SLASH_O option handles all the /O flags, but we also provide separate
+// aliased options to provide separate help messages.
+def _SLASH_O : CLJoined<"O">,
+  HelpText<"Set multiple /O flags at once; e.g. '/O2y-' for '/O2 /Oy-'">,
+  MetaVarName<"<flags>">;
+def : CLFlag<"O1">, Alias<_SLASH_O>, AliasArgs<["1"]>,
+  HelpText<"Optimize for size  (like /Og     /Os /Oy /Ob2 /GF /Gy)">;
+def : CLFlag<"O2">, Alias<_SLASH_O>, AliasArgs<["2"]>,
+  HelpText<"Optimize for speed (like /Og /Oi /Ot /Oy /Ob2 /GF /Gy)">;
+def : CLFlag<"Ob0">, Alias<_SLASH_O>, AliasArgs<["b0"]>,
+  HelpText<"Disable function inlining">;
+def : CLFlag<"Ob1">, Alias<_SLASH_O>, AliasArgs<["b1"]>,
+  HelpText<"Only inline functions explicitly or implicitly marked inline">;
+def : CLFlag<"Ob2">, Alias<_SLASH_O>, AliasArgs<["b2"]>,
+  HelpText<"Inline functions as deemed beneficial by the compiler">;
+def : CLFlag<"Od">, Alias<_SLASH_O>, AliasArgs<["d"]>,
+  HelpText<"Disable optimization">;
+def : CLFlag<"Og">, Alias<_SLASH_O>, AliasArgs<["g"]>,
+  HelpText<"No effect">;
+def : CLFlag<"Oi">, Alias<_SLASH_O>, AliasArgs<["i"]>,
+  HelpText<"Enable use of builtin functions">;
+def : CLFlag<"Oi-">, Alias<_SLASH_O>, AliasArgs<["i-"]>,
+  HelpText<"Disable use of builtin functions">;
+def : CLFlag<"Os">, Alias<_SLASH_O>, AliasArgs<["s"]>,
+  HelpText<"Optimize for size">;
+def : CLFlag<"Ot">, Alias<_SLASH_O>, AliasArgs<["t"]>,
+  HelpText<"Optimize for speed">;
+def : CLFlag<"Ox">, Alias<_SLASH_O>, AliasArgs<["x"]>,
+  HelpText<"Deprecated (like /Og /Oi /Ot /Oy /Ob2); use /O2">;
+def : CLFlag<"Oy">, Alias<_SLASH_O>, AliasArgs<["y"]>,
+  HelpText<"Enable frame pointer omission (x86 only)">;
+def : CLFlag<"Oy-">, Alias<_SLASH_O>, AliasArgs<["y-"]>,
+  HelpText<"Disable frame pointer omission (x86 only, default)">;
+
+def _SLASH_QUESTION : CLFlag<"?">, Alias<help>,
+  HelpText<"Display available options">;
+def _SLASH_Qvec : CLFlag<"Qvec">,
+  HelpText<"Enable the loop vectorization passes">, Alias<fvectorize>;
+def _SLASH_Qvec_ : CLFlag<"Qvec-">,
+  HelpText<"Disable the loop vectorization passes">, Alias<fno_vectorize>;
+def _SLASH_showIncludes : CLFlag<"showIncludes">,
+  HelpText<"Print info about included files to stderr">;
+def _SLASH_showIncludes_user : CLFlag<"showIncludes:user">,
+  HelpText<"Like /showIncludes but omit system headers">;
+def _SLASH_showFilenames : CLFlag<"showFilenames">,
+  HelpText<"Print the name of each compiled file">;
+def _SLASH_showFilenames_ : CLFlag<"showFilenames-">,
+  HelpText<"Do not print the name of each compiled file (default)">;
+def _SLASH_source_charset : CLCompileJoined<"source-charset:">,
+  HelpText<"Set source encoding, supports only UTF-8">,
+  Alias<finput_charset_EQ>;
+def _SLASH_execution_charset : CLCompileJoined<"execution-charset:">,
+  HelpText<"Set runtime encoding, supports only UTF-8">,
+  Alias<fexec_charset_EQ>;
+def _SLASH_std : CLCompileJoined<"std:">,
+  HelpText<"Set C++ version (c++14,c++17,c++latest)">;
+def _SLASH_U : CLJoinedOrSeparate<"U">, HelpText<"Undefine macro">,
+  MetaVarName<"<macro>">, Alias<U>;
+def _SLASH_validate_charset : CLFlag<"validate-charset">,
+  Alias<W_Joined>, AliasArgs<["invalid-source-encoding"]>;
+def _SLASH_validate_charset_ : CLFlag<"validate-charset-">,
+  Alias<W_Joined>, AliasArgs<["no-invalid-source-encoding"]>;
+def _SLASH_W0 : CLFlag<"W0">, HelpText<"Disable all warnings">, Alias<w>;
+def _SLASH_W1 : CLFlag<"W1">, HelpText<"Enable -Wall">, Alias<Wall>;
+def _SLASH_W2 : CLFlag<"W2">, HelpText<"Enable -Wall">, Alias<Wall>;
+def _SLASH_W3 : CLFlag<"W3">, HelpText<"Enable -Wall">, Alias<Wall>;
+def _SLASH_W4 : CLFlag<"W4">, HelpText<"Enable -Wall and -Wextra">, Alias<WCL4>;
+def _SLASH_Wall : CLFlag<"Wall">, HelpText<"Enable -Weverything">,
+  Alias<W_Joined>, AliasArgs<["everything"]>;
+def _SLASH_WX : CLFlag<"WX">, HelpText<"Treat warnings as errors">,
+  Alias<W_Joined>, AliasArgs<["error"]>;
+def _SLASH_WX_ : CLFlag<"WX-">,
+  HelpText<"Do not treat warnings as errors (default)">,
+  Alias<W_Joined>, AliasArgs<["no-error"]>;
+def _SLASH_w_flag : CLFlag<"w">, HelpText<"Disable all warnings">, Alias<w>;
+def _SLASH_wd4005 : CLFlag<"wd4005">, Alias<W_Joined>,
+  AliasArgs<["no-macro-redefined"]>;
+def _SLASH_wd4018 : CLFlag<"wd4018">, Alias<W_Joined>,
+  AliasArgs<["no-sign-compare"]>;
+def _SLASH_wd4100 : CLFlag<"wd4100">, Alias<W_Joined>,
+  AliasArgs<["no-unused-parameter"]>;
+def _SLASH_wd4910 : CLFlag<"wd4910">, Alias<W_Joined>,
+  AliasArgs<["no-dllexport-explicit-instantiation-decl"]>;
+def _SLASH_wd4996 : CLFlag<"wd4996">, Alias<W_Joined>,
+  AliasArgs<["no-deprecated-declarations"]>;
+def _SLASH_vd : CLJoined<"vd">, HelpText<"Control vtordisp placement">,
+  Alias<vtordisp_mode_EQ>;
+def _SLASH_X : CLFlag<"X">,
+  HelpText<"Do not add %INCLUDE% to include search path">, Alias<nostdlibinc>;
+def _SLASH_Zc_sizedDealloc : CLFlag<"Zc:sizedDealloc">,
+  HelpText<"Enable C++14 sized global deallocation functions">,
+  Alias<fsized_deallocation>;
+def _SLASH_Zc_sizedDealloc_ : CLFlag<"Zc:sizedDealloc-">,
+  HelpText<"Disable C++14 sized global deallocation functions">,
+  Alias<fno_sized_deallocation>;
+def _SLASH_Zc_alignedNew : CLFlag<"Zc:alignedNew">,
+  HelpText<"Enable C++17 aligned allocation functions">,
+  Alias<faligned_allocation>;
+def _SLASH_Zc_alignedNew_ : CLFlag<"Zc:alignedNew-">,
+  HelpText<"Disable C++17 aligned allocation functions">,
+  Alias<fno_aligned_allocation>;
+def _SLASH_Zc_char8_t : CLFlag<"Zc:char8_t">,
+  HelpText<"Enable char8_t from C++2a">,
+  Alias<fchar8__t>;
+def _SLASH_Zc_char8_t_ : CLFlag<"Zc:char8_t-">,
+  HelpText<"Disable char8_t from c++2a">,
+  Alias<fno_char8__t>;
+def _SLASH_Zc_strictStrings : CLFlag<"Zc:strictStrings">,
+  HelpText<"Treat string literals as const">, Alias<W_Joined>,
+  AliasArgs<["error=c++11-compat-deprecated-writable-strings"]>;
+def _SLASH_Zc_threadSafeInit : CLFlag<"Zc:threadSafeInit">,
+  HelpText<"Enable thread-safe initialization of static variables">,
+  Alias<fthreadsafe_statics>;
+def _SLASH_Zc_threadSafeInit_ : CLFlag<"Zc:threadSafeInit-">,
+  HelpText<"Disable thread-safe initialization of static variables">,
+  Alias<fno_threadsafe_statics>;
+def _SLASH_Zc_trigraphs : CLFlag<"Zc:trigraphs">,
+  HelpText<"Enable trigraphs">, Alias<ftrigraphs>;
+def _SLASH_Zc_trigraphs_off : CLFlag<"Zc:trigraphs-">,
+  HelpText<"Disable trigraphs (default)">, Alias<fno_trigraphs>;
+def _SLASH_Zc_twoPhase : CLFlag<"Zc:twoPhase">,
+  HelpText<"Enable two-phase name lookup in templates">,
+  Alias<fno_delayed_template_parsing>;
+def _SLASH_Zc_twoPhase_ : CLFlag<"Zc:twoPhase-">,
+  HelpText<"Disable two-phase name lookup in templates (default)">,
+  Alias<fdelayed_template_parsing>;
+def _SLASH_Z7 : CLFlag<"Z7">,
+  HelpText<"Enable CodeView debug information in object files">;
+def _SLASH_Zd : CLFlag<"Zd">,
+  HelpText<"Emit debug line number tables only">;
+def _SLASH_Zi : CLFlag<"Zi">, Alias<_SLASH_Z7>,
+  HelpText<"Like /Z7">;
+def _SLASH_Zp : CLJoined<"Zp">,
+  HelpText<"Set default maximum struct packing alignment">,
+  Alias<fpack_struct_EQ>;
+def _SLASH_Zp_flag : CLFlag<"Zp">,
+  HelpText<"Set default maximum struct packing alignment to 1">,
+  Alias<fpack_struct_EQ>, AliasArgs<["1"]>;
+def _SLASH_Zs : CLFlag<"Zs">, HelpText<"Syntax-check only">,
+  Alias<fsyntax_only>;
+def _SLASH_openmp_ : CLFlag<"openmp-">,
+  HelpText<"Disable OpenMP support">, Alias<fno_openmp>;
+def _SLASH_openmp : CLFlag<"openmp">, HelpText<"Enable OpenMP support">,
+  Alias<fopenmp>;
+def _SLASH_openmp_experimental : CLFlag<"openmp:experimental">,
+  HelpText<"Enable OpenMP support with experimental SIMD support">,
+  Alias<fopenmp>;
+
+// Non-aliases:
+
+def _SLASH_arch : CLCompileJoined<"arch:">,
+  HelpText<"Set architecture for code generation">;
+
+def _SLASH_M_Group : OptionGroup<"</M group>">, Group<cl_compile_Group>;
+def _SLASH_volatile_Group : OptionGroup<"</volatile group>">,
+  Group<cl_compile_Group>;
+
+def _SLASH_EH : CLJoined<"EH">, HelpText<"Set exception handling model">;
+def _SLASH_EP : CLFlag<"EP">,
+  HelpText<"Disable linemarker output and preprocess to stdout">;
+def _SLASH_FA : CLFlag<"FA">,
+  HelpText<"Output assembly code file during compilation">;
+def _SLASH_Fa : CLJoined<"Fa">,
+  HelpText<"Set assembly output file name (with /FA)">,
+  MetaVarName<"<file or dir/>">;
+def _SLASH_fallback : CLCompileFlag<"fallback">,
+  HelpText<"Fall back to cl.exe if clang-cl fails to compile">;
+def _SLASH_FI : CLJoinedOrSeparate<"FI">,
+  HelpText<"Include file before parsing">, Alias<include_>;
+def _SLASH_Fe : CLJoined<"Fe">,
+  HelpText<"Set output executable file name">,
+  MetaVarName<"<file or dir/>">;
+def _SLASH_Fi : CLCompileJoined<"Fi">,
+  HelpText<"Set preprocess output file name (with /P)">,
+  MetaVarName<"<file>">;
+def _SLASH_Fo : CLCompileJoined<"Fo">,
+  HelpText<"Set output object file (with /c)">,
+  MetaVarName<"<file or dir/>">;
+def _SLASH_guard : CLJoined<"guard:">,
+  HelpText<"Enable Control Flow Guard with /guard:cf, or only the table with /guard:cf,nochecks">;
+def _SLASH_GX : CLFlag<"GX">,
+  HelpText<"Deprecated; use /EHsc">;
+def _SLASH_GX_ : CLFlag<"GX-">,
+  HelpText<"Deprecated (like not passing /EH)">;
+def _SLASH_imsvc : CLJoinedOrSeparate<"imsvc">,
+  HelpText<"Add <dir> to system include search path, as if in %INCLUDE%">,
+  MetaVarName<"<dir>">;
+def _SLASH_LD : CLFlag<"LD">, HelpText<"Create DLL">;
+def _SLASH_LDd : CLFlag<"LDd">, HelpText<"Create debug DLL">;
+def _SLASH_link : CLRemainingArgsJoined<"link">,
+  HelpText<"Forward options to the linker">, MetaVarName<"<options>">;
+def _SLASH_MD : Option<["/", "-"], "MD", KIND_FLAG>, Group<_SLASH_M_Group>,
+  Flags<[CLOption, DriverOption]>, HelpText<"Use DLL run-time">;
+def _SLASH_MDd : Option<["/", "-"], "MDd", KIND_FLAG>, Group<_SLASH_M_Group>,
+  Flags<[CLOption, DriverOption]>, HelpText<"Use DLL debug run-time">;
+def _SLASH_MT : Option<["/", "-"], "MT", KIND_FLAG>, Group<_SLASH_M_Group>,
+  Flags<[CLOption, DriverOption]>, HelpText<"Use static run-time">;
+def _SLASH_MTd : Option<["/", "-"], "MTd", KIND_FLAG>, Group<_SLASH_M_Group>,
+  Flags<[CLOption, DriverOption]>, HelpText<"Use static debug run-time">;
+def _SLASH_o : CLJoinedOrSeparate<"o">,
+  HelpText<"Deprecated (set output file name); use /Fe or /Fe">,
+  MetaVarName<"<file or dir/>">;
+def _SLASH_P : CLFlag<"P">, HelpText<"Preprocess to file">;
+def _SLASH_Tc : CLCompileJoinedOrSeparate<"Tc">,
+  HelpText<"Treat <file> as C source file">, MetaVarName<"<file>">;
+def _SLASH_TC : CLCompileFlag<"TC">, HelpText<"Treat all source files as C">;
+def _SLASH_Tp : CLCompileJoinedOrSeparate<"Tp">,
+  HelpText<"Treat <file> as C++ source file">, MetaVarName<"<file>">;
+def _SLASH_TP : CLCompileFlag<"TP">, HelpText<"Treat all source files as C++">;
+def _SLASH_volatile_iso : Option<["/", "-"], "volatile:iso", KIND_FLAG>,
+  Group<_SLASH_volatile_Group>, Flags<[CLOption, DriverOption]>,
+  HelpText<"Volatile loads and stores have standard semantics">;
+def _SLASH_vmb : CLFlag<"vmb">,
+  HelpText<"Use a best-case representation method for member pointers">;
+def _SLASH_vmg : CLFlag<"vmg">,
+  HelpText<"Use a most-general representation for member pointers">;
+def _SLASH_vms : CLFlag<"vms">,
+  HelpText<"Set the default most-general representation to single inheritance">;
+def _SLASH_vmm : CLFlag<"vmm">,
+  HelpText<"Set the default most-general representation to "
+           "multiple inheritance">;
+def _SLASH_vmv : CLFlag<"vmv">,
+  HelpText<"Set the default most-general representation to "
+           "virtual inheritance">;
+def _SLASH_volatile_ms  : Option<["/", "-"], "volatile:ms", KIND_FLAG>,
+  Group<_SLASH_volatile_Group>, Flags<[CLOption, DriverOption]>,
+  HelpText<"Volatile loads and stores have acquire and release semantics">;
+def _SLASH_clang : CLJoined<"clang:">,
+  HelpText<"Pass <arg> to the clang driver">, MetaVarName<"<arg>">;
+def _SLASH_Zl : CLFlag<"Zl">,
+  HelpText<"Do not let object file auto-link default libraries">;
+
+def _SLASH_Yc : CLJoined<"Yc">,
+  HelpText<"Generate a pch file for all code up to and including <filename>">,
+  MetaVarName<"<filename>">;
+def _SLASH_Yu : CLJoined<"Yu">,
+  HelpText<"Load a pch file and use it instead of all code up to "
+           "and including <filename>">,
+  MetaVarName<"<filename>">;
+def _SLASH_Y_ : CLFlag<"Y-">,
+  HelpText<"Disable precompiled headers, overrides /Yc and /Yu">;
+def _SLASH_Zc_dllexportInlines : CLFlag<"Zc:dllexportInlines">,
+  HelpText<"dllexport/dllimport inline member functions of dllexport/import classes (default)">;
+def _SLASH_Zc_dllexportInlines_ : CLFlag<"Zc:dllexportInlines-">,
+  HelpText<"Do not dllexport/dllimport inline member functions of dllexport/import classes">;
+def _SLASH_Fp : CLJoined<"Fp">,
+  HelpText<"Set pch file name (with /Yc and /Yu)">, MetaVarName<"<file>">;
+
+def _SLASH_Gd : CLFlag<"Gd">,
+  HelpText<"Set __cdecl as a default calling convention">;
+def _SLASH_Gr : CLFlag<"Gr">,
+  HelpText<"Set __fastcall as a default calling convention">;
+def _SLASH_Gz : CLFlag<"Gz">,
+  HelpText<"Set __stdcall as a default calling convention">;
+def _SLASH_Gv : CLFlag<"Gv">,
+  HelpText<"Set __vectorcall as a default calling convention">;
+def _SLASH_Gregcall : CLFlag<"Gregcall">,
+  HelpText<"Set __regcall as a default calling convention">;
+
+// Ignored:
+
+def _SLASH_analyze_ : CLIgnoredFlag<"analyze-">;
+def _SLASH_bigobj : CLIgnoredFlag<"bigobj">;
+def _SLASH_cgthreads : CLIgnoredJoined<"cgthreads">;
+def _SLASH_d2FastFail : CLIgnoredFlag<"d2FastFail">;
+def _SLASH_d2Zi_PLUS : CLIgnoredFlag<"d2Zi+">;
+def _SLASH_errorReport : CLIgnoredJoined<"errorReport">;
+def _SLASH_FC : CLIgnoredFlag<"FC">;
+def _SLASH_Fd : CLIgnoredJoined<"Fd">;
+def _SLASH_FS : CLIgnoredFlag<"FS">;
+def _SLASH_JMC : CLIgnoredFlag<"JMC">;
+def _SLASH_kernel_ : CLIgnoredFlag<"kernel-">;
+def _SLASH_nologo : CLIgnoredFlag<"nologo">;
+def _SLASH_permissive_ : CLIgnoredFlag<"permissive-">;
+def _SLASH_RTC : CLIgnoredJoined<"RTC">;
+def _SLASH_sdl : CLIgnoredFlag<"sdl">;
+def _SLASH_sdl_ : CLIgnoredFlag<"sdl-">;
+def _SLASH_utf8 : CLIgnoredFlag<"utf-8">,
+  HelpText<"Set source and runtime encoding to UTF-8 (default)">;
+def _SLASH_w : CLIgnoredJoined<"w">;
+def _SLASH_Zc___cplusplus : CLIgnoredFlag<"Zc:__cplusplus">;
+def _SLASH_Zc_auto : CLIgnoredFlag<"Zc:auto">;
+def _SLASH_Zc_forScope : CLIgnoredFlag<"Zc:forScope">;
+def _SLASH_Zc_inline : CLIgnoredFlag<"Zc:inline">;
+def _SLASH_Zc_rvalueCast : CLIgnoredFlag<"Zc:rvalueCast">;
+def _SLASH_Zc_ternary : CLIgnoredFlag<"Zc:ternary">;
+def _SLASH_Zc_wchar_t : CLIgnoredFlag<"Zc:wchar_t">;
+def _SLASH_ZH_MD5 : CLIgnoredFlag<"ZH:MD5">;
+def _SLASH_ZH_SHA1 : CLIgnoredFlag<"ZH:SHA1">;
+def _SLASH_ZH_SHA_256 : CLIgnoredFlag<"ZH:SHA_256">;
+def _SLASH_Zm : CLIgnoredJoined<"Zm">;
+def _SLASH_Zo : CLIgnoredFlag<"Zo">;
+def _SLASH_Zo_ : CLIgnoredFlag<"Zo-">;
+
+
+// Unsupported:
+
+def _SLASH_await : CLFlag<"await">;
+def _SLASH_constexpr : CLJoined<"constexpr:">;
+def _SLASH_AI : CLJoinedOrSeparate<"AI">;
+def _SLASH_Bt : CLFlag<"Bt">;
+def _SLASH_Bt_plus : CLFlag<"Bt+">;
+def _SLASH_clr : CLJoined<"clr">;
+def _SLASH_d2 : CLJoined<"d2">;
+def _SLASH_doc : CLJoined<"doc">;
+def _SLASH_FA_joined : CLJoined<"FA">;
+def _SLASH_favor : CLJoined<"favor">;
+def _SLASH_F : CLJoinedOrSeparate<"F">;
+def _SLASH_Fm : CLJoined<"Fm">;
+def _SLASH_Fr : CLJoined<"Fr">;
+def _SLASH_FR : CLJoined<"FR">;
+def _SLASH_FU : CLJoinedOrSeparate<"FU">;
+def _SLASH_Fx : CLFlag<"Fx">;
+def _SLASH_G1 : CLFlag<"G1">;
+def _SLASH_G2 : CLFlag<"G2">;
+def _SLASH_Ge : CLFlag<"Ge">;
+def _SLASH_Gh : CLFlag<"Gh">;
+def _SLASH_GH : CLFlag<"GH">;
+def _SLASH_GL : CLFlag<"GL">;
+def _SLASH_GL_ : CLFlag<"GL-">;
+def _SLASH_Gm : CLFlag<"Gm">;
+def _SLASH_Gm_ : CLFlag<"Gm-">;
+def _SLASH_GT : CLFlag<"GT">;
+def _SLASH_GZ : CLFlag<"GZ">;
+def _SLASH_H : CLFlag<"H">;
+def _SLASH_homeparams : CLFlag<"homeparams">;
+def _SLASH_hotpatch : CLFlag<"hotpatch">;
+def _SLASH_kernel : CLFlag<"kernel">;
+def _SLASH_LN : CLFlag<"LN">;
+def _SLASH_MP : CLJoined<"MP">;
+def _SLASH_Qfast_transcendentals : CLFlag<"Qfast_transcendentals">;
+def _SLASH_QIfist : CLFlag<"QIfist">;
+def _SLASH_QIntel_jcc_erratum : CLFlag<"QIntel-jcc-erratum">;
+def _SLASH_Qimprecise_fwaits : CLFlag<"Qimprecise_fwaits">;
+def _SLASH_Qpar : CLFlag<"Qpar">;
+def _SLASH_Qpar_report : CLJoined<"Qpar-report">;
+def _SLASH_Qsafe_fp_loads : CLFlag<"Qsafe_fp_loads">;
+def _SLASH_Qspectre : CLFlag<"Qspectre">;
+def _SLASH_Qspectre_load : CLFlag<"Qspectre-load">;
+def _SLASH_Qspectre_load_cf : CLFlag<"Qspectre-load-cf">;
+def _SLASH_Qvec_report : CLJoined<"Qvec-report">;
+def _SLASH_u : CLFlag<"u">;
+def _SLASH_V : CLFlag<"V">;
+def _SLASH_WL : CLFlag<"WL">;
+def _SLASH_Wp64 : CLFlag<"Wp64">;
+def _SLASH_Yd : CLFlag<"Yd">;
+def _SLASH_Yl : CLJoined<"Yl">;
+def _SLASH_Za : CLFlag<"Za">;
+def _SLASH_Zc : CLJoined<"Zc:">;
+def _SLASH_Ze : CLFlag<"Ze">;
+def _SLASH_Zg : CLFlag<"Zg">;
+def _SLASH_ZI : CLFlag<"ZI">;
+def _SLASH_ZW : CLJoined<"ZW">;
diff --git a/clang/include/clang/Driver/Tool.h b/clang/include/clang/Driver/Tool.h
index 8d04916069785..cc0a09fb27472 100644
--- a/clang/include/clang/Driver/Tool.h
+++ b/clang/include/clang/Driver/Tool.h
@@ -10,7 +10,6 @@
 #define LLVM_CLANG_DRIVER_TOOL_H
 
 #include "clang/Basic/LLVM.h"
-#include "llvm/Support/Program.h"
 
 namespace llvm {
 namespace opt {
@@ -31,24 +30,6 @@ namespace driver {
 
 /// Tool - Information on a specific compilation tool.
 class Tool {
-public:
-  // Documents the level of support for response files in this tool.
-  // Response files are necessary if the command line gets too large,
-  // requiring the arguments to be transferred to a file.
-  enum ResponseFileSupport {
-    // Provides full support for response files, which means we can transfer
-    // all tool input arguments to a file. E.g.: clang, gcc, binutils and MSVC
-    // tools.
-    RF_Full,
-    // Input file names can live in a file, but flags can't. E.g.: ld64 (Mac
-    // OS X linker).
-    RF_FileList,
-    // Does not support response files: all arguments must be passed via
-    // command line.
-    RF_None
-  };
-
-private:
   /// The tool name (for debugging).
   const char *Name;
 
@@ -58,20 +39,8 @@ class Tool {
   /// The tool chain this tool is a part of.
   const ToolChain &TheToolChain;
 
-  /// The level of support for response files seen in this tool
-  const ResponseFileSupport ResponseSupport;
-
-  /// The encoding to use when writing response files for this tool on Windows
-  const llvm::sys::WindowsEncodingMethod ResponseEncoding;
-
-  /// The flag used to pass a response file via command line to this tool
-  const char *const ResponseFlag;
-
 public:
-  Tool(const char *Name, const char *ShortName, const ToolChain &TC,
-       ResponseFileSupport ResponseSupport = RF_None,
-       llvm::sys::WindowsEncodingMethod ResponseEncoding = llvm::sys::WEM_UTF8,
-       const char *ResponseFlag = "@");
+  Tool(const char *Name, const char *ShortName, const ToolChain &TC);
 
 public:
   virtual ~Tool();
@@ -87,29 +56,6 @@ class Tool {
   virtual bool hasIntegratedCPP() const = 0;
   virtual bool isLinkJob() const { return false; }
   virtual bool isDsymutilJob() const { return false; }
-  /// Returns the level of support for response files of this tool,
-  /// whether it accepts arguments to be passed via a file on disk.
-  ResponseFileSupport getResponseFilesSupport() const {
-    return ResponseSupport;
-  }
-  /// Returns which encoding the response file should use. This is only
-  /// relevant on Windows platforms where there are different encodings being
-  /// accepted for different tools. On UNIX, UTF8 is universal.
-  ///
-  /// Windows use cases: - GCC and Binutils on mingw only accept ANSI response
-  /// files encoded with the system current code page.
-  /// - MSVC's CL.exe and LINK.exe accept UTF16 on Windows.
-  /// - Clang accepts both UTF8 and UTF16.
-  ///
-  /// FIXME: When GNU tools learn how to parse UTF16 on Windows, we should
-  /// always use UTF16 for Windows, which is the Windows official encoding for
-  /// international characters.
-  llvm::sys::WindowsEncodingMethod getResponseFileEncoding() const {
-    return ResponseEncoding;
-  }
-  /// Returns which prefix to use when passing the name of a response
-  /// file as a parameter to this tool.
-  const char *getResponseFileFlag() const { return ResponseFlag; }
 
   /// Does this tool have "good" standardized diagnostics, or should the
   /// driver add an additional "command failed" diagnostic on failures.
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index f33ed991f493d..7495e08fe6e64 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -139,6 +139,7 @@ class ToolChain {
   mutable std::unique_ptr<Tool> Flang;
   mutable std::unique_ptr<Tool> Assemble;
   mutable std::unique_ptr<Tool> Link;
+  mutable std::unique_ptr<Tool> StaticLibTool;
   mutable std::unique_ptr<Tool> IfsMerge;
   mutable std::unique_ptr<Tool> OffloadBundler;
   mutable std::unique_ptr<Tool> OffloadWrapper;
@@ -147,6 +148,7 @@ class ToolChain {
   Tool *getFlang() const;
   Tool *getAssemble() const;
   Tool *getLink() const;
+  Tool *getStaticLibTool() const;
   Tool *getIfsMerge() const;
   Tool *getClangAs() const;
   Tool *getOffloadBundler() const;
@@ -174,6 +176,7 @@ class ToolChain {
 
   virtual Tool *buildAssembler() const;
   virtual Tool *buildLinker() const;
+  virtual Tool *buildStaticLibTool() const;
   virtual Tool *getTool(Action::ActionClass AC) const;
 
   /// \name Utilities for implementing subclasses.
@@ -326,6 +329,9 @@ class ToolChain {
   /// the linker suffix or name.
   std::string GetLinkerPath() const;
 
+  /// Returns the linker path for emitting a static library.
+  std::string GetStaticLibToolPath() const;
+
   /// Dispatch to the specific toolchain for verbose printing.
   ///
   /// This is used when handling the verbose option to print detailed,
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index 96b73211d9fff..3549ec9eee0e5 100755
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -1425,6 +1425,17 @@ struct FormatStyle {
   /// For example: TESTSUITE
   std::vector<std::string> NamespaceMacros;
 
+  /// A vector of macros which are whitespace-sensitive and shouldn't be
+  /// touched.
+  ///
+  /// These are expected to be macros of the form:
+  /// \code
+  ///   STRINGIZE(...)
+  /// \endcode
+  ///
+  /// For example: STRINGIZE
+  std::vector<std::string> WhitespaceSensitiveMacros;
+
   tooling::IncludeStyle IncludeStyle;
 
   /// Indent case labels one level from the switch statement.
diff --git a/clang/include/clang/Frontend/ASTConsumers.h b/clang/include/clang/Frontend/ASTConsumers.h
index af8c4a517dcde..98cfc7cadc0dd 100644
--- a/clang/include/clang/Frontend/ASTConsumers.h
+++ b/clang/include/clang/Frontend/ASTConsumers.h
@@ -39,7 +39,7 @@ std::unique_ptr<ASTConsumer> CreateASTPrinter(std::unique_ptr<raw_ostream> OS,
 std::unique_ptr<ASTConsumer>
 CreateASTDumper(std::unique_ptr<raw_ostream> OS, StringRef FilterString,
                 bool DumpDecls, bool Deserialize, bool DumpLookups,
-                ASTDumpOutputFormat Format);
+                bool DumpDeclTypes, ASTDumpOutputFormat Format);
 
 // AST Decl node lister: prints qualified names of all filterable AST Decl
 // nodes.
diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h
index ba9bd439c4a39..cb935becaef19 100644
--- a/clang/include/clang/Frontend/CompilerInstance.h
+++ b/clang/include/clang/Frontend/CompilerInstance.h
@@ -764,10 +764,7 @@ class CompilerInstance : public ModuleLoader {
   static bool InitializeSourceManager(const FrontendInputFile &Input,
                                       DiagnosticsEngine &Diags,
                                       FileManager &FileMgr,
-                                      SourceManager &SourceMgr,
-                                      HeaderSearch *HS,
-                                      DependencyOutputOptions &DepOpts,
-                                      const FrontendOptions &Opts);
+                                      SourceManager &SourceMgr);
 
   /// }
 
diff --git a/clang/include/clang/Frontend/CompilerInvocation.h b/clang/include/clang/Frontend/CompilerInvocation.h
index e63408da8f5ee..c723fc084c854 100644
--- a/clang/include/clang/Frontend/CompilerInvocation.h
+++ b/clang/include/clang/Frontend/CompilerInvocation.h
@@ -153,6 +153,8 @@ class CompilerInvocation : public CompilerInvocationBase {
   /// one of the vaild-to-access (albeit arbitrary) states.
   ///
   /// \param [out] Res - The resulting invocation.
+  /// \param [in] CommandLineArgs - Array of argument strings, this must not
+  /// contain "-cc1".
   static bool CreateFromArgs(CompilerInvocation &Res,
                              ArrayRef<const char *> CommandLineArgs,
                              DiagnosticsEngine &Diags,
@@ -184,6 +186,18 @@ class CompilerInvocation : public CompilerInvocationBase {
   /// identifying the conditions under which the module was built.
   std::string getModuleHash() const;
 
+  using StringAllocator = llvm::function_ref<const char *(const llvm::Twine &)>;
+  /// Generate a cc1-compatible command line arguments from this instance.
+  ///
+  /// \param [out] Args - The generated arguments. Note that the caller is
+  /// responsible for inserting the path to the clang executable and "-cc1" if
+  /// desired.
+  /// \param SA - A function that given a Twine can allocate storage for a given
+  /// command line argument and return a pointer to the newly allocated string.
+  /// The returned pointer is what gets appended to Args.
+  void generateCC1CommandLine(llvm::SmallVectorImpl<const char *> &Args,
+                              StringAllocator SA) const;
+
   /// @}
   /// @name Option Subgroups
   /// @{
@@ -222,6 +236,16 @@ class CompilerInvocation : public CompilerInvocationBase {
   }
 
   /// @}
+
+private:
+  /// Parse options for flags that expose marshalling information in their
+  /// table-gen definition
+  ///
+  /// \param Args - The argument list containing the arguments to parse
+  /// \param Diags - The DiagnosticsEngine associated with CreateFromArgs
+  /// \returns - True if parsing was successful, false otherwise
+  bool parseSimpleArgs(const llvm::opt::ArgList &Args,
+                       DiagnosticsEngine &Diags);
 };
 
 IntrusiveRefCntPtr<llvm::vfs::FileSystem>
diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h
index 6069b5eea2650..b2be33032c08d 100644
--- a/clang/include/clang/Frontend/FrontendOptions.h
+++ b/clang/include/clang/Frontend/FrontendOptions.h
@@ -285,6 +285,9 @@ class FrontendOptions {
   /// Whether we include lookup table dumps in AST dumps.
   unsigned ASTDumpLookups : 1;
 
+  /// Whether we include declaration type dumps in AST dumps.
+  unsigned ASTDumpDeclTypes : 1;
+
   /// Whether we are performing an implicit module build.
   unsigned BuildingImplicitModule : 1;
 
diff --git a/clang/include/clang/Lex/LiteralSupport.h b/clang/include/clang/Lex/LiteralSupport.h
index 6829771b28308..0c4f0fe277b7c 100644
--- a/clang/include/clang/Lex/LiteralSupport.h
+++ b/clang/include/clang/Lex/LiteralSupport.h
@@ -40,7 +40,9 @@ void expandUCNs(SmallVectorImpl<char> &Buf, StringRef Input);
 /// of a ppnumber, classifying it as either integer, floating, or erroneous,
 /// determines the radix of the value and can convert it to a useful value.
 class NumericLiteralParser {
-  Preprocessor &PP; // needed for diagnostics
+  const SourceManager &SM;
+  const LangOptions &LangOpts;
+  DiagnosticsEngine &Diags;
 
   const char *const ThisTokBegin;
   const char *const ThisTokEnd;
@@ -54,9 +56,9 @@ class NumericLiteralParser {
   SmallString<32> UDSuffixBuf;
 
 public:
-  NumericLiteralParser(StringRef TokSpelling,
-                       SourceLocation TokLoc,
-                       Preprocessor &PP);
+  NumericLiteralParser(StringRef TokSpelling, SourceLocation TokLoc,
+                       const SourceManager &SM, const LangOptions &LangOpts,
+                       const TargetInfo &Target, DiagnosticsEngine &Diags);
   bool hadError : 1;
   bool isUnsigned : 1;
   bool isLong : 1;          // This is *not* set for long long.
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 1ae219781c696..e809d87b59a0f 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -241,6 +241,9 @@ class Parser : public CodeCompletionHandler {
   /// The "depth" of the template parameters currently being parsed.
   unsigned TemplateParameterDepth;
 
+  /// Current kind of OpenMP clause
+  OpenMPClauseKind OMPClauseKind = llvm::omp::OMPC_unknown;
+
   /// RAII class that manages the template parameter depth.
   class TemplateParameterDepthRAII {
     unsigned &Depth;
@@ -1088,12 +1091,40 @@ class Parser : public CodeCompletionHandler {
     }
   };
 
+  /// Introduces zero or more scopes for parsing. The scopes will all be exited
+  /// when the object is destroyed.
+  class MultiParseScope {
+    Parser &Self;
+    unsigned NumScopes = 0;
+
+    MultiParseScope(const MultiParseScope&) = delete;
+
+  public:
+    MultiParseScope(Parser &Self) : Self(Self) {}
+    void Enter(unsigned ScopeFlags) {
+      Self.EnterScope(ScopeFlags);
+      ++NumScopes;
+    }
+    void Exit() {
+      while (NumScopes) {
+        Self.ExitScope();
+        --NumScopes;
+      }
+    }
+    ~MultiParseScope() {
+      Exit();
+    }
+  };
+
   /// EnterScope - Start a new scope.
   void EnterScope(unsigned ScopeFlags);
 
   /// ExitScope - Pop a scope off the scope stack.
   void ExitScope();
 
+  /// Re-enter the template scopes for a declaration that might be a template.
+  unsigned ReenterTemplateScopes(MultiParseScope &S, Decl *D);
+
 private:
   /// RAII object used to modify the scope flags for the current scope.
   class ParseScopeFlags {
@@ -1278,13 +1309,7 @@ class Parser : public CodeCompletionHandler {
     Decl *D;
     CachedTokens Toks;
 
-    /// Whether this member function had an associated template
-    /// scope. When true, D is a template declaration.
-    /// otherwise, it is a member function declaration.
-    bool TemplateScope;
-
-    explicit LexedMethod(Parser* P, Decl *MD)
-      : Self(P), D(MD), TemplateScope(false) {}
+    explicit LexedMethod(Parser *P, Decl *MD) : Self(P), D(MD) {}
 
     void ParseLexedMethodDefs() override;
   };
@@ -1314,8 +1339,7 @@ class Parser : public CodeCompletionHandler {
   /// argument (C++ [class.mem]p2).
   struct LateParsedMethodDeclaration : public LateParsedDeclaration {
     explicit LateParsedMethodDeclaration(Parser *P, Decl *M)
-      : Self(P), Method(M), TemplateScope(false),
-        ExceptionSpecTokens(nullptr) {}
+        : Self(P), Method(M), ExceptionSpecTokens(nullptr) {}
 
     void ParseLexedMethodDeclarations() override;
 
@@ -1324,11 +1348,6 @@ class Parser : public CodeCompletionHandler {
     /// Method - The method declaration.
     Decl *Method;
 
-    /// Whether this member function had an associated template
-    /// scope. When true, D is a template declaration.
-    /// otherwise, it is a member function declaration.
-    bool TemplateScope;
-
     /// DefaultArgs - Contains the parameters of the function and
     /// their default arguments. At least one of the parameters will
     /// have a default argument, but all of the parameters of the
@@ -1373,18 +1392,13 @@ class Parser : public CodeCompletionHandler {
   /// parsed after the corresponding top-level class is complete.
   struct ParsingClass {
     ParsingClass(Decl *TagOrTemplate, bool TopLevelClass, bool IsInterface)
-      : TopLevelClass(TopLevelClass), TemplateScope(false),
-        IsInterface(IsInterface), TagOrTemplate(TagOrTemplate) { }
+        : TopLevelClass(TopLevelClass), IsInterface(IsInterface),
+          TagOrTemplate(TagOrTemplate) {}
 
     /// Whether this is a "top-level" class, meaning that it is
     /// not nested within another class.
     bool TopLevelClass : 1;
 
-    /// Whether this class had an associated template
-    /// scope. When true, TagOrTemplate is a template declaration;
-    /// otherwise, it is a tag declaration.
-    bool TemplateScope : 1;
-
     /// Whether this class is an __interface.
     bool IsInterface : 1;
 
@@ -1483,6 +1497,10 @@ class Parser : public CodeCompletionHandler {
     SourceRange getSourceRange() const LLVM_READONLY;
   };
 
+  // In ParseCXXInlineMethods.cpp.
+  struct ReenterTemplateScopeRAII;
+  struct ReenterClassScopeRAII;
+
   void LexTemplateFunctionForLateParsing(CachedTokens &Toks);
   void ParseLateTemplatedFuncDef(LateParsedTemplate &LPT);
 
@@ -2053,8 +2071,9 @@ class Parser : public CodeCompletionHandler {
   StmtResult ParseCompoundStatementBody(bool isStmtExpr = false);
   bool ParseParenExprOrCondition(StmtResult *InitStmt,
                                  Sema::ConditionResult &CondResult,
-                                 SourceLocation Loc,
-                                 Sema::ConditionKind CK);
+                                 SourceLocation Loc, Sema::ConditionKind CK,
+                                 SourceLocation *LParenLoc = nullptr,
+                                 SourceLocation *RParenLoc = nullptr);
   StmtResult ParseIfStatement(SourceLocation *TrailingElseLoc);
   StmtResult ParseSwitchStatement(SourceLocation *TrailingElseLoc);
   StmtResult ParseWhileStatement(SourceLocation *TrailingElseLoc);
@@ -3240,7 +3259,7 @@ class Parser : public CodeCompletionHandler {
       DeclaratorContext Context, const ParsedTemplateInfo &TemplateInfo,
       ParsingDeclRAIIObject &DiagsFromParams, SourceLocation &DeclEnd,
       ParsedAttributes &AccessAttrs, AccessSpecifier AS = AS_none);
-  bool ParseTemplateParameters(unsigned Depth,
+  bool ParseTemplateParameters(MultiParseScope &TemplateScopes, unsigned Depth,
                                SmallVectorImpl<NamedDecl *> &TemplateParams,
                                SourceLocation &LAngleLoc,
                                SourceLocation &RAngleLoc);
diff --git a/clang/include/clang/Sema/Scope.h b/clang/include/clang/Sema/Scope.h
index 9b8f1415a1e37..b7260f15fe1b9 100644
--- a/clang/include/clang/Sema/Scope.h
+++ b/clang/include/clang/Sema/Scope.h
@@ -322,8 +322,21 @@ class Scope {
   /// declared in.
   bool isDeclScope(const Decl *D) const { return DeclsInScope.count(D) != 0; }
 
-  DeclContext *getEntity() const { return Entity; }
-  void setEntity(DeclContext *E) { Entity = E; }
+  /// Get the entity corresponding to this scope.
+  DeclContext *getEntity() const {
+    return isTemplateParamScope() ? nullptr : Entity;
+  }
+
+  /// Get the DeclContext in which to continue unqualified lookup after a
+  /// lookup in this scope.
+  DeclContext *getLookupEntity() const { return Entity; }
+
+  void setEntity(DeclContext *E) {
+    assert(!isTemplateParamScope() &&
+           "entity associated with template param scope");
+    Entity = E;
+  }
+  void setLookupEntity(DeclContext *E) { Entity = E; }
 
   /// Determine whether any unrecoverable errors have occurred within this
   /// scope. Note that this may return false even if the scope contains invalid
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 8c8e981e60652..6f7ad8076718d 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -490,10 +490,41 @@ class Sema final {
             PragmaLocation(PragmaLocation),
             PragmaPushLocation(PragmaPushLocation) {}
     };
-    void Act(SourceLocation PragmaLocation,
-             PragmaMsStackAction Action,
-             llvm::StringRef StackSlotLabel,
-             ValueType Value);
+
+    void Act(SourceLocation PragmaLocation, PragmaMsStackAction Action,
+             llvm::StringRef StackSlotLabel, ValueType Value) {
+      if (Action == PSK_Reset) {
+        CurrentValue = DefaultValue;
+        CurrentPragmaLocation = PragmaLocation;
+        return;
+      }
+      if (Action & PSK_Push)
+        Stack.emplace_back(StackSlotLabel, CurrentValue, CurrentPragmaLocation,
+                           PragmaLocation);
+      else if (Action & PSK_Pop) {
+        if (!StackSlotLabel.empty()) {
+          // If we've got a label, try to find it and jump there.
+          auto I = llvm::find_if(llvm::reverse(Stack), [&](const Slot &x) {
+            return x.StackSlotLabel == StackSlotLabel;
+          });
+          // If we found the label so pop from there.
+          if (I != Stack.rend()) {
+            CurrentValue = I->Value;
+            CurrentPragmaLocation = I->PragmaLocation;
+            Stack.erase(std::prev(I.base()), Stack.end());
+          }
+        } else if (!Stack.empty()) {
+          // We do not have a label, just pop the last entry.
+          CurrentValue = Stack.back().Value;
+          CurrentPragmaLocation = Stack.back().PragmaLocation;
+          Stack.pop_back();
+        }
+      }
+      if (Action & PSK_Set) {
+        CurrentValue = Value;
+        CurrentPragmaLocation = PragmaLocation;
+      }
+    }
 
     // MSVC seems to add artificial slots to #pragma stacks on entering a C++
     // method body to restore the stacks on exit, so it works like this:
@@ -557,6 +588,15 @@ class Sema final {
 
   // This stack tracks the current state of Sema.CurFPFeatures.
   PragmaStack<unsigned> FpPragmaStack;
+  FPOptionsOverride CurFPFeatureOverrides() {
+    FPOptionsOverride result;
+    if (!FpPragmaStack.hasValue()) {
+      result = FPOptionsOverride();
+    } else {
+      result = FPOptionsOverride(FpPragmaStack.CurrentValue);
+    }
+    return result;
+  }
 
   // RAII object to push / pop sentinel slots for all MS #pragma stacks.
   // Actions should be performed only if we enter / exit a C++ method body.
@@ -1358,12 +1398,19 @@ class Sema final {
   /// statements.
   class FPFeaturesStateRAII {
   public:
-    FPFeaturesStateRAII(Sema &S) : S(S), OldFPFeaturesState(S.CurFPFeatures) {}
-    ~FPFeaturesStateRAII() { S.CurFPFeatures = OldFPFeaturesState; }
+    FPFeaturesStateRAII(Sema &S) : S(S), OldFPFeaturesState(S.CurFPFeatures) {
+      OldOverrides = S.FpPragmaStack.CurrentValue;
+    }
+    ~FPFeaturesStateRAII() {
+      S.CurFPFeatures = OldFPFeaturesState;
+      S.FpPragmaStack.CurrentValue = OldOverrides;
+    }
+    unsigned getOverrides() { return OldOverrides; }
 
   private:
     Sema& S;
     FPOptions OldFPFeaturesState;
+    unsigned OldOverrides;
   };
 
   void addImplicitTypedef(StringRef Name, QualType T);
@@ -1697,6 +1744,10 @@ class Sema final {
   static QualType GetTypeFromParser(ParsedType Ty,
                                     TypeSourceInfo **TInfo = nullptr);
   CanThrowResult canThrow(const Stmt *E);
+  /// Determine whether the callee of a particular function call can throw.
+  /// E, D and Loc are all optional.
+  static CanThrowResult canCalleeThrow(Sema &S, const Expr *E, const Decl *D,
+                                       SourceLocation Loc = SourceLocation());
   const FunctionProtoType *ResolveExceptionSpec(SourceLocation Loc,
                                                 const FunctionProtoType *FPT);
   void UpdateExceptionSpec(FunctionDecl *FD,
@@ -2862,8 +2913,6 @@ class Sema final {
                      Decl *EnumDecl, ArrayRef<Decl *> Elements, Scope *S,
                      const ParsedAttributesView &Attr);
 
-  DeclContext *getContainingDC(DeclContext *DC);
-
   /// Set the current declaration context until it gets popped.
   void PushDeclContext(Scope *S, DeclContext *DC);
   void PopDeclContext();
@@ -2873,6 +2922,11 @@ class Sema final {
   void EnterDeclaratorContext(Scope *S, DeclContext *DC);
   void ExitDeclaratorContext(Scope *S);
 
+  /// Enter a template parameter scope, after it's been associated with a particular
+  /// DeclContext. Causes lookup within the scope to chain through enclosing contexts
+  /// in the correct order.
+  void EnterTemplatedContext(Scope *S, DeclContext *DC);
+
   /// Push the parameters of D, which must be a function, into scope.
   void ActOnReenterFunctionContext(Scope* S, Decl* D);
   void ActOnExitFunctionContext();
@@ -4323,7 +4377,8 @@ class Sema final {
                                     ConditionResult Cond);
   StmtResult ActOnFinishSwitchStmt(SourceLocation SwitchLoc,
                                            Stmt *Switch, Stmt *Body);
-  StmtResult ActOnWhileStmt(SourceLocation WhileLoc, ConditionResult Cond,
+  StmtResult ActOnWhileStmt(SourceLocation WhileLoc, SourceLocation LParenLoc,
+                            ConditionResult Cond, SourceLocation RParenLoc,
                             Stmt *Body);
   StmtResult ActOnDoStmt(SourceLocation DoLoc, Stmt *Body,
                          SourceLocation WhileLoc, SourceLocation CondLParen,
@@ -4917,8 +4972,11 @@ class Sema final {
                                               SourceLocation RBLoc);
 
   ExprResult ActOnOMPArraySectionExpr(Expr *Base, SourceLocation LBLoc,
-                                      Expr *LowerBound, SourceLocation ColonLoc,
-                                      Expr *Length, SourceLocation RBLoc);
+                                      Expr *LowerBound,
+                                      SourceLocation ColonLocFirst,
+                                      SourceLocation ColonLocSecond,
+                                      Expr *Length, Expr *Stride,
+                                      SourceLocation RBLoc);
   ExprResult ActOnOMPArrayShapingExpr(Expr *Base, SourceLocation LParenLoc,
                                       SourceLocation RParenLoc,
                                       ArrayRef<Expr *> Dims,
@@ -6785,7 +6843,8 @@ class Sema final {
   void ActOnFinishCXXNonNestedClass();
 
   void ActOnReenterCXXMethodParameter(Scope *S, ParmVarDecl *Param);
-  unsigned ActOnReenterTemplateScope(Scope *S, Decl *Template);
+  unsigned ActOnReenterTemplateScope(Decl *Template,
+                                     llvm::function_ref<Scope *()> EnterScope);
   void ActOnStartDelayedMemberDeclarations(Scope *S, Decl *Record);
   void ActOnStartDelayedCXXMethodDeclaration(Scope *S, Decl *Method);
   void ActOnDelayedCXXMethodParameter(Scope *S, Decl *Param);
@@ -6906,7 +6965,7 @@ class Sema final {
 
   /// DiagnoseAbsenceOfOverrideControl - Diagnose if 'override' keyword was
   /// not used in the declaration of an overriding method.
-  void DiagnoseAbsenceOfOverrideControl(NamedDecl *D);
+  void DiagnoseAbsenceOfOverrideControl(NamedDecl *D, bool Inconsistent);
 
   /// CheckForFunctionMarkedFinal - Checks whether a virtual member function
   /// overrides a virtual member function marked 'final', according to
@@ -9043,6 +9102,8 @@ class Sema final {
              TemplateArgumentListInfo &Result,
              const MultiLevelTemplateArgumentList &TemplateArgs);
 
+  bool InstantiateDefaultArgument(SourceLocation CallLoc, FunctionDecl *FD,
+                                  ParmVarDecl *Param);
   void InstantiateExceptionSpec(SourceLocation PointOfInstantiation,
                                 FunctionDecl *Function);
   bool CheckInstantiatedFunctionTemplateConstraints(
@@ -9598,10 +9659,10 @@ class Sema final {
 
   /// Are precise floating point semantics currently enabled?
   bool isPreciseFPEnabled() {
-    return !CurFPFeatures.allowAssociativeMath() &&
-           !CurFPFeatures.noSignedZeros() &&
-           !CurFPFeatures.allowReciprocalMath() &&
-           !CurFPFeatures.allowApproximateFunctions();
+    return !CurFPFeatures.getAllowFPReassociate() &&
+           !CurFPFeatures.getNoSignedZero() &&
+           !CurFPFeatures.getAllowReciprocal() &&
+           !CurFPFeatures.getAllowApproxFunc();
   }
 
   /// ActOnPragmaFloatControl - Call on well-formed \#pragma float_control
@@ -9644,21 +9705,21 @@ class Sema final {
   /// ActOnPragmaFPContract - Called on well formed
   /// \#pragma {STDC,OPENCL} FP_CONTRACT and
   /// \#pragma clang fp contract
-  void ActOnPragmaFPContract(LangOptions::FPModeKind FPC);
+  void ActOnPragmaFPContract(SourceLocation Loc, LangOptions::FPModeKind FPC);
 
   /// Called on well formed
   /// \#pragma clang fp reassociate
-  void ActOnPragmaFPReassociate(bool IsEnabled);
+  void ActOnPragmaFPReassociate(SourceLocation Loc, bool IsEnabled);
 
   /// ActOnPragmaFenvAccess - Called on well formed
   /// \#pragma STDC FENV_ACCESS
   void ActOnPragmaFEnvAccess(SourceLocation Loc, bool IsEnabled);
 
   /// Called to set rounding mode for floating point operations.
-  void setRoundingMode(llvm::RoundingMode);
+  void setRoundingMode(SourceLocation Loc, llvm::RoundingMode);
 
   /// Called to set exception behavior for floating point operations.
-  void setExceptionMode(LangOptions::FPExceptionModeKind);
+  void setExceptionMode(SourceLocation Loc, LangOptions::FPExceptionModeKind);
 
   /// AddAlignmentAttributesForRecord - Adds any needed alignment attributes to
   /// a the record decl, to handle '\#pragma pack' and '\#pragma options align'.
@@ -9796,6 +9857,9 @@ class Sema final {
   void CheckCompletedCoroutineBody(FunctionDecl *FD, Stmt *&Body);
   ClassTemplateDecl *lookupCoroutineTraits(SourceLocation KwLoc,
                                            SourceLocation FuncLoc);
+  /// Check that the expression co_await promise.final_suspend() shall not be
+  /// potentially-throwing.
+  bool checkFinalSuspendNoThrow(const Stmt *FinalSuspend);
 
   //===--------------------------------------------------------------------===//
   // OpenCL extensions.
@@ -11770,12 +11834,13 @@ class Sema final {
   /// - Otherwise, returns true without emitting any diagnostics.
   bool CheckCUDACall(SourceLocation Loc, FunctionDecl *Callee);
 
+  void CUDACheckLambdaCapture(CXXMethodDecl *D, const sema::Capture &Capture);
+
   /// Set __device__ or __host__ __device__ attributes on the given lambda
   /// operator() method.
   ///
-  /// CUDA lambdas declared inside __device__ or __global__ functions inherit
-  /// the __device__ attribute.  Similarly, lambdas inside __host__ __device__
-  /// functions become __host__ __device__ themselves.
+  /// CUDA lambdas by default is host device function unless it has explicit
+  /// host or device attribute.
   void CUDASetLambdaAttrs(CXXMethodDecl *Method);
 
   /// Finds a function in \p Matches with highest calling priority
@@ -11922,7 +11987,7 @@ class Sema final {
   void CodeCompleteDesignator(const QualType BaseType,
                               llvm::ArrayRef<Expr *> InitExprs,
                               const Designation &D);
-  void CodeCompleteAfterIf(Scope *S);
+  void CodeCompleteAfterIf(Scope *S, bool IsBracedThen);
 
   void CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS, bool EnteringContext,
                                bool IsUsingDeclaration, QualType BaseType,
@@ -11938,6 +12003,7 @@ class Sema final {
 
   void CodeCompleteLambdaIntroducer(Scope *S, LambdaIntroducer &Intro,
                                     bool AfterAmpersand);
+  void CodeCompleteAfterFunctionEquals(Declarator &D);
 
   void CodeCompleteObjCAtDirective(Scope *S);
   void CodeCompleteObjCAtVisibility(Scope *S);
@@ -12083,6 +12149,13 @@ class Sema final {
   bool CheckSystemZBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall);
   bool CheckX86BuiltinRoundingOrSAE(unsigned BuiltinID, CallExpr *TheCall);
   bool CheckX86BuiltinGatherScatterScale(unsigned BuiltinID, CallExpr *TheCall);
+  bool CheckX86BuiltinTileArguments(unsigned BuiltinID, CallExpr *TheCall);
+  bool CheckX86BuiltinTileArgumentsRange(CallExpr *TheCall,
+                                         ArrayRef<int> ArgNums);
+  bool CheckX86BuiltinTileArgumentsRange(CallExpr *TheCall, int ArgNum);
+  bool CheckX86BuiltinTileDuplicate(CallExpr *TheCall, ArrayRef<int> ArgNums);
+  bool CheckX86BuiltinTileRangeAndDuplicate(CallExpr *TheCall,
+                                            ArrayRef<int> ArgNums);
   bool CheckX86BuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID,
                                    CallExpr *TheCall);
   bool CheckPPCBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID,
diff --git a/clang/include/clang/Sema/Template.h b/clang/include/clang/Sema/Template.h
index 8166b6cf9b6f0..91d175fdd0506 100644
--- a/clang/include/clang/Sema/Template.h
+++ b/clang/include/clang/Sema/Template.h
@@ -42,6 +42,17 @@ class TypedefNameDecl;
 class TypeSourceInfo;
 class VarDecl;
 
+/// The kind of template substitution being performed.
+enum class TemplateSubstitutionKind : char {
+  /// We are substituting template parameters for template arguments in order
+  /// to form a template specialization.
+  Specialization,
+  /// We are substituting template parameters for (typically) other template
+  /// parameters in order to rewrite a declaration as a different declaration
+  /// (for example, when forming a deduction guide from a constructor).
+  Rewrite,
+};
+
   /// Data structure that captures multiple levels of template argument
   /// lists for use in template instantiation.
   ///
@@ -73,6 +84,9 @@ class VarDecl;
     /// being substituted.
     unsigned NumRetainedOuterLevels = 0;
 
+    /// The kind of substitution described by this argument list.
+    TemplateSubstitutionKind Kind = TemplateSubstitutionKind::Specialization;
+
   public:
     /// Construct an empty set of template argument lists.
     MultiLevelTemplateArgumentList() = default;
@@ -83,6 +97,18 @@ class VarDecl;
       addOuterTemplateArguments(&TemplateArgs);
     }
 
+    void setKind(TemplateSubstitutionKind K) { Kind = K; }
+
+    /// Determine the kind of template substitution being performed.
+    TemplateSubstitutionKind getKind() const { return Kind; }
+
+    /// Determine whether we are rewriting template parameters rather than
+    /// substituting for them. If so, we should not leave references to the
+    /// original template parameters behind.
+    bool isRewrite() const {
+      return Kind == TemplateSubstitutionKind::Rewrite;
+    }
+
     /// Determine the number of levels in this template argument
     /// list.
     unsigned getNumLevels() const {
@@ -95,6 +121,10 @@ class VarDecl;
       return TemplateArgumentLists.size();
     }
 
+    unsigned getNumRetainedOuterLevels() const {
+      return NumRetainedOuterLevels;
+    }
+
     /// Determine how many of the \p OldDepth outermost template parameter
     /// lists would be removed by substituting these arguments.
     unsigned getNewDepth(unsigned OldDepth) const {
@@ -159,6 +189,9 @@ class VarDecl;
     void addOuterRetainedLevel() {
       ++NumRetainedOuterLevels;
     }
+    void addOuterRetainedLevels(unsigned Num) {
+      NumRetainedOuterLevels += Num;
+    }
 
     /// Retrieve the innermost template argument list.
     const ArgList &getInnermost() const {
diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h
index 35f73e1dfd7ca..7a6664af65d89 100644
--- a/clang/include/clang/Serialization/ASTWriter.h
+++ b/clang/include/clang/Serialization/ASTWriter.h
@@ -62,6 +62,7 @@ class CXXRecordDecl;
 class CXXTemporary;
 class FileEntry;
 class FPOptions;
+class FPOptionsOverride;
 class FunctionDecl;
 class HeaderSearch;
 class HeaderSearchOptions;
@@ -506,7 +507,7 @@ class ASTWriter : public ASTDeserializationListener,
                             bool IsModule);
   void WriteDeclUpdatesBlocks(RecordDataImpl &OffsetsRecord);
   void WriteDeclContextVisibleUpdate(const DeclContext *DC);
-  void WriteFPPragmaOptions(const FPOptions &Opts);
+  void WriteFPPragmaOptions(const FPOptionsOverride &Opts);
   void WriteOpenCLExtensions(Sema &SemaRef);
   void WriteOpenCLExtensionTypes(Sema &SemaRef);
   void WriteOpenCLExtensionDecls(Sema &SemaRef);
diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
index 9de324506bf66..cbd9254003282 100644
--- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
+++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
@@ -130,7 +130,8 @@ def CallAndMessageModeling : Checker<"CallAndMessageModeling">,
            "function/method call. For instance, if we can't reason about the "
            "nullability of the implicit this parameter after a method call, "
            "this checker conservatively assumes it to be non-null">,
-  Documentation<HasDocumentation>;
+  Documentation<HasDocumentation>,
+  Hidden;
 
 def CallAndMessageChecker : Checker<"CallAndMessage">,
   HelpText<"Check for logical errors for function calls and Objective-C "
@@ -220,7 +221,8 @@ def StackAddrEscapeChecker : Checker<"StackAddressEscape">,
 
 def DynamicTypePropagation : Checker<"DynamicTypePropagation">,
   HelpText<"Generate dynamic type information">,
-  Documentation<NotDocumented>;
+  Documentation<NotDocumented>,
+  Hidden;
 
 def NonnullGlobalConstantsChecker: Checker<"NonnilStringConstants">,
   HelpText<"Assume that const string-like globals are non-null">,
@@ -354,9 +356,17 @@ def StdCLibraryFunctionsChecker : Checker<"StdCLibraryFunctions">,
                   "If set to true, the checker displays the found summaries "
                   "for the given translation unit.",
                   "false",
-                  Released>
+                  Released,
+                  Hide>,
+    CmdLineOption<Boolean,
+                  "ModelPOSIX",
+                  "If set to true, the checker models functions from the "
+                  "POSIX standard.",
+                  "false",
+                  InAlpha>
   ]>,
-  Documentation<NotDocumented>;
+  Documentation<NotDocumented>,
+  Hidden;
 
 def TrustNonnullChecker : Checker<"TrustNonnull">,
   HelpText<"Trust that returns from framework methods annotated with _Nonnull "
@@ -365,18 +375,6 @@ def TrustNonnullChecker : Checker<"TrustNonnull">,
 
 } // end "apiModeling"
 
-let ParentPackage = APIModelingAlpha in {
-
-def StdCLibraryFunctionArgsChecker : Checker<"StdCLibraryFunctionArgs">,
-  HelpText<"Check constraints of arguments of C standard library functions, "
-           "such as whether the parameter of isalpha is in the range [0, 255] "
-           "or is EOF.">,
-  Dependencies<[StdCLibraryFunctionsChecker]>,
-  WeakDependencies<[NonNullParamChecker]>,
-  Documentation<NotDocumented>;
-
-} // end "alpha.apiModeling"
-
 //===----------------------------------------------------------------------===//
 // Evaluate "builtin" functions.
 //===----------------------------------------------------------------------===//
@@ -535,6 +533,14 @@ def BlockInCriticalSectionChecker : Checker<"BlockInCriticalSection">,
   HelpText<"Check for calls to blocking functions inside a critical section">,
   Documentation<HasAlphaDocumentation>;
 
+def StdCLibraryFunctionArgsChecker : Checker<"StdCLibraryFunctionArgs">,
+  HelpText<"Check constraints of arguments of C standard library functions, "
+           "such as whether the parameter of isalpha is in the range [0, 255] "
+           "or is EOF.">,
+  Dependencies<[StdCLibraryFunctionsChecker]>,
+  WeakDependencies<[NonNullParamChecker]>,
+  Documentation<NotDocumented>;
+
 } // end "alpha.unix"
 
 //===----------------------------------------------------------------------===//
@@ -571,9 +577,17 @@ def CXXSelfAssignmentChecker : Checker<"SelfAssignment">,
   Documentation<NotDocumented>,
   Hidden;
 
-def SmartPtrModeling: Checker<"SmartPtr">,
+def SmartPtrModeling: Checker<"SmartPtrModeling">,
   HelpText<"Model behavior of C++ smart pointers">,
   Documentation<NotDocumented>,
+    CheckerOptions<[
+    CmdLineOption<Boolean,
+                  "ModelSmartPtrDereference",
+                  "Enable modeling for SmartPtr null dereferences",
+                  "false",
+                  InAlpha,
+                  Hide>,
+  ]>,
   Hidden;
 
 def MoveChecker: Checker<"Move">,
@@ -730,6 +744,11 @@ def MismatchedIteratorChecker : Checker<"MismatchedIterator">,
   Dependencies<[IteratorModeling]>,
   Documentation<HasAlphaDocumentation>;
 
+def SmartPtrChecker: Checker<"SmartPtr">,
+  HelpText<"Find the dereference of null SmrtPtr">,
+  Dependencies<[SmartPtrModeling]>,
+  Documentation<HasAlphaDocumentation>;
+
 } // end: "alpha.cplusplus"
 
 
@@ -1372,6 +1391,12 @@ def AnalysisOrderChecker : Checker<"AnalysisOrder">,
                   "false",
                   Released,
                   Hide>,
+    CmdLineOption<Boolean,
+                  "EvalCall",
+                  "",
+                  "false",
+                  Released,
+                  Hide>,
     CmdLineOption<Boolean,
                   "PreCall",
                   "",
diff --git a/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def b/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def
index 8944dfe0f749a..f0359d2dbb3c2 100644
--- a/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def
+++ b/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def
@@ -321,10 +321,17 @@ ANALYZER_OPTION(bool, ShouldDisplayCheckerNameForText, "display-checker-name",
 ANALYZER_OPTION(unsigned, CTUImportThreshold, "ctu-import-threshold",
                 "The maximal amount of translation units that is considered "
                 "for import when inlining functions during CTU analysis. "
-                "Lowering this threshold can alleviate the memory burder of "
+                "Lowering this threshold can alleviate the memory burden of "
                 "analysis with many interdependent definitions located in "
-                "various translation units.",
-                100u)
+                "various translation units. This is valid only for non C++ "
+                "source files.",
+                24u)
+
+ANALYZER_OPTION(unsigned, CTUImportCppThreshold, "ctu-import-cpp-threshold",
+                "The maximal amount of translation units that is considered "
+                "for import when inlining functions during CTU analysis of C++ "
+                "source files.",
+                8u)
 
 ANALYZER_OPTION(
     unsigned, AlwaysInlineSize, "ipa-always-inline-size",
diff --git a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
index 51565524db1e6..27bc0dda1f1ce 100644
--- a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
+++ b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
@@ -19,6 +19,7 @@
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
@@ -135,7 +136,7 @@ class BugReport {
   SmallVector<FixItHint, 4> Fixits;
 
   BugReport(Kind kind, const BugType &bt, StringRef desc)
-      : K(kind), BT(bt), Description(desc) {}
+      : BugReport(kind, bt, "", desc) {}
 
   BugReport(Kind K, const BugType &BT, StringRef ShortDescription,
             StringRef Description)
@@ -369,16 +370,13 @@ class PathSensitiveBugReport : public BugReport {
 public:
   PathSensitiveBugReport(const BugType &bt, StringRef desc,
                          const ExplodedNode *errorNode)
-      : BugReport(Kind::PathSensitive, bt, desc), ErrorNode(errorNode),
-        ErrorNodeRange(getStmt() ? getStmt()->getSourceRange()
-                                 : SourceRange()) {}
+      : PathSensitiveBugReport(bt, desc, desc, errorNode) {}
 
   PathSensitiveBugReport(const BugType &bt, StringRef shortDesc, StringRef desc,
                          const ExplodedNode *errorNode)
-      : BugReport(Kind::PathSensitive, bt, shortDesc, desc),
-        ErrorNode(errorNode),
-        ErrorNodeRange(getStmt() ? getStmt()->getSourceRange()
-                                 : SourceRange()) {}
+      : PathSensitiveBugReport(bt, shortDesc, desc, errorNode,
+                               /*LocationToUnique*/ {},
+                               /*DeclToUnique*/ nullptr) {}
 
   /// Create a PathSensitiveBugReport with a custom uniqueing location.
   ///
@@ -391,11 +389,13 @@ class PathSensitiveBugReport : public BugReport {
                          const ExplodedNode *errorNode,
                          PathDiagnosticLocation LocationToUnique,
                          const Decl *DeclToUnique)
-      : BugReport(Kind::PathSensitive, bt, desc), ErrorNode(errorNode),
-        ErrorNodeRange(getStmt() ? getStmt()->getSourceRange() : SourceRange()),
-        UniqueingLocation(LocationToUnique), UniqueingDecl(DeclToUnique) {
-    assert(errorNode);
-  }
+      : PathSensitiveBugReport(bt, desc, desc, errorNode, LocationToUnique,
+                               DeclToUnique) {}
+
+  PathSensitiveBugReport(const BugType &bt, StringRef shortDesc, StringRef desc,
+                         const ExplodedNode *errorNode,
+                         PathDiagnosticLocation LocationToUnique,
+                         const Decl *DeclToUnique);
 
   static bool classof(const BugReport *R) {
     return R->getKind() == Kind::PathSensitive;
diff --git a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h
index de0ee5de81b53..365b1ff1bfe3e 100644
--- a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h
+++ b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h
@@ -386,6 +386,8 @@ class FalsePositiveRefutationBRVisitor final : public BugReporterVisitor {
 
   void finalizeVisitor(BugReporterContext &BRC, const ExplodedNode *EndPathNode,
                        PathSensitiveBugReport &BR) override;
+  void addConstraints(const ExplodedNode *N,
+                      bool OverwriteConstraintsOnExistingSyms);
 };
 
 
diff --git a/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h b/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
index 820ccfceecf6a..d2f71baa56a46 100644
--- a/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
+++ b/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
@@ -43,10 +43,12 @@ class CallEvent;
 class CheckerBase;
 class CheckerContext;
 class CheckerRegistry;
+struct CheckerRegistryData;
 class ExplodedGraph;
 class ExplodedNode;
 class ExplodedNodeSet;
 class ExprEngine;
+struct EvalCallOptions;
 class MemRegion;
 struct NodeBuilderContext;
 class ObjCMethodCall;
@@ -129,7 +131,7 @@ class CheckerManager {
   const Preprocessor *PP = nullptr;
   CheckerNameRef CurrentCheckerName;
   DiagnosticsEngine &Diags;
-  std::unique_ptr<CheckerRegistry> Registry;
+  std::unique_ptr<CheckerRegistryData> RegistryData;
 
 public:
   // These constructors are defined in the Frontend library, because
@@ -151,8 +153,8 @@ class CheckerManager {
       : CheckerManager(Context, AOptions, PP, {}, {}) {}
 
   /// Constructs a CheckerManager without requiring an AST. No checker
-  /// registration will take place. Only useful for retrieving the
-  /// CheckerRegistry and print for help flags where the AST is unavalaible.
+  /// registration will take place. Only useful when one needs to print the
+  /// help flags through CheckerRegistryData, and the AST is unavalaible.
   CheckerManager(AnalyzerOptions &AOptions, const LangOptions &LangOpts,
                  DiagnosticsEngine &Diags, ArrayRef<std::string> plugins);
 
@@ -171,7 +173,9 @@ class CheckerManager {
     assert(PP);
     return *PP;
   }
-  const CheckerRegistry &getCheckerRegistry() const { return *Registry; }
+  const CheckerRegistryData &getCheckerRegistryData() const {
+    return *RegistryData;
+  }
   DiagnosticsEngine &getDiagnostics() const { return Diags; }
   ASTContext &getASTContext() const {
     assert(Context);
@@ -433,9 +437,9 @@ class CheckerManager {
   /// Run checkers for evaluating a call.
   ///
   /// Warning: Currently, the CallEvent MUST come from a CallExpr!
-  void runCheckersForEvalCall(ExplodedNodeSet &Dst,
-                              const ExplodedNodeSet &Src,
-                              const CallEvent &CE, ExprEngine &Eng);
+  void runCheckersForEvalCall(ExplodedNodeSet &Dst, const ExplodedNodeSet &Src,
+                              const CallEvent &CE, ExprEngine &Eng,
+                              const EvalCallOptions &CallOpts);
 
   /// Run checkers for the entire Translation Unit.
   void runCheckersOnEndOfTranslationUnit(const TranslationUnitDecl *TU,
diff --git a/clang/include/clang/StaticAnalyzer/Core/CheckerRegistryData.h b/clang/include/clang/StaticAnalyzer/Core/CheckerRegistryData.h
new file mode 100644
index 0000000000000..43248d8e6bb8d
--- /dev/null
+++ b/clang/include/clang/StaticAnalyzer/Core/CheckerRegistryData.h
@@ -0,0 +1,226 @@
+//===- CheckerRegistryData.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the data structures to which the TableGen file Checkers.td
+// maps to, as well as what was parsed from the the specific invocation (whether
+// a checker/package is enabled, their options values, etc).
+//
+// The parsing of the invocation is done by CheckerRegistry, which is found in
+// the Frontend library. This allows the Core and Checkers libraries to utilize
+// this information, such as enforcing rules on checker dependency bug emission,
+// ensuring all checker options were queried, etc.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_STATICANALYZER_CORE_CHECKERREGISTRYDATA_H
+#define LLVM_CLANG_STATICANALYZER_CORE_CHECKERREGISTRYDATA_H
+
+#include "clang/Basic/LLVM.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace clang {
+
+class AnalyzerOptions;
+
+namespace ento {
+
+class CheckerManager;
+
+/// Initialization functions perform any necessary setup for a checker.
+/// They should include a call to CheckerManager::registerChecker.
+using RegisterCheckerFn = void (*)(CheckerManager &);
+using ShouldRegisterFunction = bool (*)(const CheckerManager &);
+
+/// Specifies a command line option. It may either belong to a checker or a
+/// package.
+struct CmdLineOption {
+  StringRef OptionType;
+  StringRef OptionName;
+  StringRef DefaultValStr;
+  StringRef Description;
+  StringRef DevelopmentStatus;
+  bool IsHidden;
+
+  CmdLineOption(StringRef OptionType, StringRef OptionName,
+                StringRef DefaultValStr, StringRef Description,
+                StringRef DevelopmentStatus, bool IsHidden)
+      : OptionType(OptionType), OptionName(OptionName),
+        DefaultValStr(DefaultValStr), Description(Description),
+        DevelopmentStatus(DevelopmentStatus), IsHidden(IsHidden) {
+
+    assert((OptionType == "bool" || OptionType == "string" ||
+            OptionType == "int") &&
+           "Unknown command line option type!");
+
+    assert((OptionType != "bool" ||
+            (DefaultValStr == "true" || DefaultValStr == "false")) &&
+           "Invalid value for boolean command line option! Maybe incorrect "
+           "parameters to the addCheckerOption or addPackageOption method?");
+
+    int Tmp;
+    assert((OptionType != "int" || !DefaultValStr.getAsInteger(0, Tmp)) &&
+           "Invalid value for integer command line option! Maybe incorrect "
+           "parameters to the addCheckerOption or addPackageOption method?");
+    (void)Tmp;
+
+    assert((DevelopmentStatus == "alpha" || DevelopmentStatus == "beta" ||
+            DevelopmentStatus == "released") &&
+           "Invalid development status!");
+  }
+
+  LLVM_DUMP_METHOD void dump() const;
+  LLVM_DUMP_METHOD void dumpToStream(llvm::raw_ostream &Out) const;
+};
+
+using CmdLineOptionList = llvm::SmallVector<CmdLineOption, 0>;
+
+struct CheckerInfo;
+
+using CheckerInfoList = std::vector<CheckerInfo>;
+using CheckerInfoListRange = llvm::iterator_range<CheckerInfoList::iterator>;
+using ConstCheckerInfoList = llvm::SmallVector<const CheckerInfo *, 0>;
+using CheckerInfoSet = llvm::SetVector<const CheckerInfo *>;
+
+/// Specifies a checker. Note that this isn't what we call a checker object,
+/// it merely contains everything required to create one.
+struct CheckerInfo {
+  enum class StateFromCmdLine {
+    // This checker wasn't explicitly enabled or disabled.
+    State_Unspecified,
+    // This checker was explicitly disabled.
+    State_Disabled,
+    // This checker was explicitly enabled.
+    State_Enabled
+  };
+
+  RegisterCheckerFn Initialize = nullptr;
+  ShouldRegisterFunction ShouldRegister = nullptr;
+  StringRef FullName;
+  StringRef Desc;
+  StringRef DocumentationUri;
+  CmdLineOptionList CmdLineOptions;
+  bool IsHidden = false;
+  StateFromCmdLine State = StateFromCmdLine::State_Unspecified;
+
+  ConstCheckerInfoList Dependencies;
+  ConstCheckerInfoList WeakDependencies;
+
+  bool isEnabled(const CheckerManager &mgr) const {
+    return State == StateFromCmdLine::State_Enabled && ShouldRegister(mgr);
+  }
+
+  bool isDisabled(const CheckerManager &mgr) const {
+    return State == StateFromCmdLine::State_Disabled || !ShouldRegister(mgr);
+  }
+
+  // Since each checker must have a different full name, we can identify
+  // CheckerInfo objects by them.
+  bool operator==(const CheckerInfo &Rhs) const {
+    return FullName == Rhs.FullName;
+  }
+
+  CheckerInfo(RegisterCheckerFn Fn, ShouldRegisterFunction sfn, StringRef Name,
+              StringRef Desc, StringRef DocsUri, bool IsHidden)
+      : Initialize(Fn), ShouldRegister(sfn), FullName(Name), Desc(Desc),
+        DocumentationUri(DocsUri), IsHidden(IsHidden) {}
+
+  // Used for lower_bound.
+  explicit CheckerInfo(StringRef FullName) : FullName(FullName) {}
+
+  LLVM_DUMP_METHOD void dump() const;
+  LLVM_DUMP_METHOD void dumpToStream(llvm::raw_ostream &Out) const;
+};
+
+using StateFromCmdLine = CheckerInfo::StateFromCmdLine;
+
+/// Specifies a package. Each package option is implicitly an option for all
+/// checkers within the package.
+struct PackageInfo {
+  StringRef FullName;
+  CmdLineOptionList CmdLineOptions;
+
+  // Since each package must have a different full name, we can identify
+  // CheckerInfo objects by them.
+  bool operator==(const PackageInfo &Rhs) const {
+    return FullName == Rhs.FullName;
+  }
+
+  explicit PackageInfo(StringRef FullName) : FullName(FullName) {}
+
+  LLVM_DUMP_METHOD void dump() const;
+  LLVM_DUMP_METHOD void dumpToStream(llvm::raw_ostream &Out) const;
+};
+
+using PackageInfoList = llvm::SmallVector<PackageInfo, 0>;
+
+namespace checker_registry {
+
+template <class T> struct FullNameLT {
+  bool operator()(const T &Lhs, const T &Rhs) {
+    return Lhs.FullName < Rhs.FullName;
+  }
+};
+
+using PackageNameLT = FullNameLT<PackageInfo>;
+using CheckerNameLT = FullNameLT<CheckerInfo>;
+
+template <class CheckerOrPackageInfoList>
+std::conditional_t<std::is_const<CheckerOrPackageInfoList>::value,
+                   typename CheckerOrPackageInfoList::const_iterator,
+                   typename CheckerOrPackageInfoList::iterator>
+binaryFind(CheckerOrPackageInfoList &Collection, StringRef FullName) {
+
+  using CheckerOrPackage = typename CheckerOrPackageInfoList::value_type;
+  using CheckerOrPackageFullNameLT = FullNameLT<CheckerOrPackage>;
+
+  assert(llvm::is_sorted(Collection, CheckerOrPackageFullNameLT{}) &&
+         "In order to efficiently gather checkers/packages, this function "
+         "expects them to be already sorted!");
+
+  return llvm::lower_bound(Collection, CheckerOrPackage(FullName),
+                           CheckerOrPackageFullNameLT{});
+}
+} // namespace checker_registry
+
+struct CheckerRegistryData {
+public:
+  CheckerInfoSet EnabledCheckers;
+
+  CheckerInfoList Checkers;
+  PackageInfoList Packages;
+  /// Used for counting how many checkers belong to a certain package in the
+  /// \c Checkers field. For convenience purposes.
+  llvm::StringMap<size_t> PackageSizes;
+
+  /// Contains all (FullName, CmdLineOption) pairs. Similarly to dependencies,
+  /// we only modify the actual CheckerInfo and PackageInfo objects once all
+  /// of them have been added.
+  llvm::SmallVector<std::pair<StringRef, CmdLineOption>, 0> PackageOptions;
+  llvm::SmallVector<std::pair<StringRef, CmdLineOption>, 0> CheckerOptions;
+
+  llvm::SmallVector<std::pair<StringRef, StringRef>, 0> Dependencies;
+  llvm::SmallVector<std::pair<StringRef, StringRef>, 0> WeakDependencies;
+
+  CheckerInfoListRange getMutableCheckersForCmdLineArg(StringRef CmdLineArg);
+
+  /// Prints the name and description of all checkers in this registry.
+  /// This output is not intended to be machine-parseable.
+  void printCheckerWithDescList(const AnalyzerOptions &AnOpts, raw_ostream &Out,
+                                size_t MaxNameChars = 30) const;
+  void printEnabledCheckerList(raw_ostream &Out) const;
+  void printCheckerOptionList(const AnalyzerOptions &AnOpts,
+                              raw_ostream &Out) const;
+};
+
+} // namespace ento
+} // namespace clang
+
+#endif // LLVM_CLANG_STATICANALYZER_CORE_CHECKERREGISTRYDATA_H
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h
index a71469e90ea25..d75f9f63286db 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h
@@ -788,6 +788,10 @@ class CXXMemberOperatorCall : public CXXInstanceCall {
     // to implicit this-parameter on the declaration.
     return CallArgumentIndex + 1;
   }
+
+  OverloadedOperatorKind getOverloadedOperator() const {
+    return getOriginExpr()->getOperator();
+  }
 };
 
 /// Represents an implicit call to a C++ destructor.
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
index baa3a94adb640..cdfe986355c56 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
@@ -96,8 +96,37 @@ class RegionAndSymbolInvalidationTraits;
 class SymbolManager;
 class SwitchNodeBuilder;
 
+/// Hints for figuring out of a call should be inlined during evalCall().
+struct EvalCallOptions {
+  /// This call is a constructor or a destructor for which we do not currently
+  /// compute the this-region correctly.
+  bool IsCtorOrDtorWithImproperlyModeledTargetRegion = false;
+
+  /// This call is a constructor or a destructor for a single element within
+  /// an array, a part of array construction or destruction.
+  bool IsArrayCtorOrDtor = false;
+
+  /// This call is a constructor or a destructor of a temporary value.
+  bool IsTemporaryCtorOrDtor = false;
+
+  /// This call is a constructor for a temporary that is lifetime-extended
+  /// by binding it to a reference-type field within an aggregate,
+  /// for example 'A { const C &c; }; A a = { C() };'
+  bool IsTemporaryLifetimeExtendedViaAggregate = false;
+
+  /// This call is a pre-C++17 elidable constructor that we failed to elide
+  /// because we failed to compute the target region into which
+  /// this constructor would have been ultimately elided. Analysis that
+  /// we perform in this case is still correct but it behaves differently,
+  /// as if copy elision is disabled.
+  bool IsElidableCtorThatHasNotBeenElided = false;
+
+  EvalCallOptions() {}
+};
+
 class ExprEngine {
   void anchor();
+
 public:
   /// The modes of inlining, which override the default analysis-wide settings.
   enum InliningModes {
@@ -108,34 +137,6 @@ class ExprEngine {
     Inline_Minimal = 0x1
   };
 
-  /// Hints for figuring out of a call should be inlined during evalCall().
-  struct EvalCallOptions {
-    /// This call is a constructor or a destructor for which we do not currently
-    /// compute the this-region correctly.
-    bool IsCtorOrDtorWithImproperlyModeledTargetRegion = false;
-
-    /// This call is a constructor or a destructor for a single element within
-    /// an array, a part of array construction or destruction.
-    bool IsArrayCtorOrDtor = false;
-
-    /// This call is a constructor or a destructor of a temporary value.
-    bool IsTemporaryCtorOrDtor = false;
-
-    /// This call is a constructor for a temporary that is lifetime-extended
-    /// by binding it to a reference-type field within an aggregate,
-    /// for example 'A { const C &c; }; A a = { C() };'
-    bool IsTemporaryLifetimeExtendedViaAggregate = false;
-
-    /// This call is a pre-C++17 elidable constructor that we failed to elide
-    /// because we failed to compute the target region into which
-    /// this constructor would have been ultimately elided. Analysis that
-    /// we perform in this case is still correct but it behaves differently,
-    /// as if copy elision is disabled.
-    bool IsElidableCtorThatHasNotBeenElided = false;
-
-    EvalCallOptions() {}
-  };
-
 private:
   cross_tu::CrossTranslationUnitContext &CTU;
 
diff --git a/clang/include/clang/StaticAnalyzer/Frontend/CheckerRegistry.h b/clang/include/clang/StaticAnalyzer/Frontend/CheckerRegistry.h
index 7b72fae0fefed..43dbfb1585151 100644
--- a/clang/include/clang/StaticAnalyzer/Frontend/CheckerRegistry.h
+++ b/clang/include/clang/StaticAnalyzer/Frontend/CheckerRegistry.h
@@ -5,17 +5,22 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// Contains the logic for parsing the TableGen file Checkers.td, and parsing the
+// specific invocation of the analyzer (which checker/package is enabled, values
+// of their options, etc). This is in the frontend library because checker
+// registry functions are called from here but are defined in the dependent
+// library libStaticAnalyzerCheckers, but the actual data structure that holds
+// the parsed information is in the Core library.
+//
+//===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_STATICANALYZER_CORE_CHECKERREGISTRY_H
-#define LLVM_CLANG_STATICANALYZER_CORE_CHECKERREGISTRY_H
+#ifndef LLVM_CLANG_STATICANALYZER_FRONTEND_CHECKERREGISTRY_H
+#define LLVM_CLANG_STATICANALYZER_FRONTEND_CHECKERREGISTRY_H
 
 #include "clang/Basic/LLVM.h"
-#include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/StringMap.h"
+#include "clang/StaticAnalyzer/Core/CheckerRegistryData.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/raw_ostream.h"
-#include <cstddef>
-#include <vector>
 
 // FIXME: move this information to an HTML file in docs/.
 // At the very least, a checker plugin is a dynamic library that exports
@@ -83,143 +88,16 @@ class CheckerManager;
 /// "core.builtin", or the full name "core.builtin.NoReturnFunctionChecker".
 class CheckerRegistry {
 public:
-  CheckerRegistry(ArrayRef<std::string> plugins, DiagnosticsEngine &diags,
-                  AnalyzerOptions &AnOpts,
+  CheckerRegistry(CheckerRegistryData &Data, ArrayRef<std::string> Plugins,
+                  DiagnosticsEngine &Diags, AnalyzerOptions &AnOpts,
                   ArrayRef<std::function<void(CheckerRegistry &)>>
-                      checkerRegistrationFns = {});
+                      CheckerRegistrationFns = {});
 
   /// Collects all enabled checkers in the field EnabledCheckers. It preserves
   /// the order of insertion, as dependencies have to be enabled before the
   /// checkers that depend on them.
   void initializeRegistry(const CheckerManager &Mgr);
 
-  /// Initialization functions perform any necessary setup for a checker.
-  /// They should include a call to CheckerManager::registerChecker.
-  using InitializationFunction = void (*)(CheckerManager &);
-  using ShouldRegisterFunction = bool (*)(const CheckerManager &);
-
-  /// Specifies a command line option. It may either belong to a checker or a
-  /// package.
-  struct CmdLineOption {
-    StringRef OptionType;
-    StringRef OptionName;
-    StringRef DefaultValStr;
-    StringRef Description;
-    StringRef DevelopmentStatus;
-    bool IsHidden;
-
-    CmdLineOption(StringRef OptionType, StringRef OptionName,
-                  StringRef DefaultValStr, StringRef Description,
-                  StringRef DevelopmentStatus, bool IsHidden)
-        : OptionType(OptionType), OptionName(OptionName),
-          DefaultValStr(DefaultValStr), Description(Description),
-          DevelopmentStatus(DevelopmentStatus), IsHidden(IsHidden) {
-
-      assert((OptionType == "bool" || OptionType == "string" ||
-              OptionType == "int") &&
-             "Unknown command line option type!");
-
-      assert((OptionType != "bool" ||
-              (DefaultValStr == "true" || DefaultValStr == "false")) &&
-             "Invalid value for boolean command line option! Maybe incorrect "
-             "parameters to the addCheckerOption or addPackageOption method?");
-
-      int Tmp;
-      assert((OptionType != "int" || !DefaultValStr.getAsInteger(0, Tmp)) &&
-             "Invalid value for integer command line option! Maybe incorrect "
-             "parameters to the addCheckerOption or addPackageOption method?");
-      (void)Tmp;
-
-      assert((DevelopmentStatus == "alpha" || DevelopmentStatus == "beta" ||
-              DevelopmentStatus == "released") &&
-             "Invalid development status!");
-    }
-
-    LLVM_DUMP_METHOD void dump() const { dumpToStream(llvm::errs()); }
-    LLVM_DUMP_METHOD void dumpToStream(llvm::raw_ostream &Out) const;
-  };
-
-  using CmdLineOptionList = llvm::SmallVector<CmdLineOption, 0>;
-
-  struct CheckerInfo;
-
-  using CheckerInfoList = std::vector<CheckerInfo>;
-  using CheckerInfoListRange = llvm::iterator_range<CheckerInfoList::iterator>;
-  using ConstCheckerInfoList = llvm::SmallVector<const CheckerInfo *, 0>;
-  using CheckerInfoSet = llvm::SetVector<const CheckerInfo *>;
-
-  /// Specifies a checker. Note that this isn't what we call a checker object,
-  /// it merely contains everything required to create one.
-  struct CheckerInfo {
-    enum class StateFromCmdLine {
-      // This checker wasn't explicitly enabled or disabled.
-      State_Unspecified,
-      // This checker was explicitly disabled.
-      State_Disabled,
-      // This checker was explicitly enabled.
-      State_Enabled
-    };
-
-    InitializationFunction Initialize = nullptr;
-    ShouldRegisterFunction ShouldRegister = nullptr;
-    StringRef FullName;
-    StringRef Desc;
-    StringRef DocumentationUri;
-    CmdLineOptionList CmdLineOptions;
-    bool IsHidden = false;
-    StateFromCmdLine State = StateFromCmdLine::State_Unspecified;
-
-    ConstCheckerInfoList Dependencies;
-    ConstCheckerInfoList WeakDependencies;
-
-    bool isEnabled(const CheckerManager &mgr) const {
-      return State == StateFromCmdLine::State_Enabled && ShouldRegister(mgr);
-    }
-
-    bool isDisabled(const CheckerManager &mgr) const {
-      return State == StateFromCmdLine::State_Disabled || !ShouldRegister(mgr);
-    }
-
-    // Since each checker must have a different full name, we can identify
-    // CheckerInfo objects by them.
-    bool operator==(const CheckerInfo &Rhs) const {
-      return FullName == Rhs.FullName;
-    }
-
-    CheckerInfo(InitializationFunction Fn, ShouldRegisterFunction sfn,
-                StringRef Name, StringRef Desc, StringRef DocsUri,
-                bool IsHidden)
-        : Initialize(Fn), ShouldRegister(sfn), FullName(Name), Desc(Desc),
-          DocumentationUri(DocsUri), IsHidden(IsHidden) {}
-
-    // Used for lower_bound.
-    explicit CheckerInfo(StringRef FullName) : FullName(FullName) {}
-
-    LLVM_DUMP_METHOD void dump() const { dumpToStream(llvm::errs()); }
-    LLVM_DUMP_METHOD void dumpToStream(llvm::raw_ostream &Out) const;
-  };
-
-  using StateFromCmdLine = CheckerInfo::StateFromCmdLine;
-
-  /// Specifies a package. Each package option is implicitly an option for all
-  /// checkers within the package.
-  struct PackageInfo {
-    StringRef FullName;
-    CmdLineOptionList CmdLineOptions;
-
-    // Since each package must have a different full name, we can identify
-    // CheckerInfo objects by them.
-    bool operator==(const PackageInfo &Rhs) const {
-      return FullName == Rhs.FullName;
-    }
-
-    explicit PackageInfo(StringRef FullName) : FullName(FullName) {}
-
-    LLVM_DUMP_METHOD void dump() const { dumpToStream(llvm::errs()); }
-    LLVM_DUMP_METHOD void dumpToStream(llvm::raw_ostream &Out) const;
-  };
-
-  using PackageInfoList = llvm::SmallVector<PackageInfo, 0>;
 
 private:
   /// Default initialization function for checkers -- since CheckerManager
@@ -237,7 +115,7 @@ class CheckerRegistry {
 public:
   /// Adds a checker to the registry. Use this non-templated overload when your
   /// checker requires custom initialization.
-  void addChecker(InitializationFunction Fn, ShouldRegisterFunction sfn,
+  void addChecker(RegisterCheckerFn Fn, ShouldRegisterFunction sfn,
                   StringRef FullName, StringRef Desc, StringRef DocsUri,
                   bool IsHidden);
 
@@ -305,46 +183,17 @@ class CheckerRegistry {
   /// Check if every option corresponds to a specific checker or package.
   void validateCheckerOptions() const;
 
-  /// Prints the name and description of all checkers in this registry.
-  /// This output is not intended to be machine-parseable.
-  void printCheckerWithDescList(raw_ostream &Out,
-                                size_t MaxNameChars = 30) const;
-  void printEnabledCheckerList(raw_ostream &Out) const;
-  void printCheckerOptionList(raw_ostream &Out) const;
-
 private:
-  /// Return an iterator range of mutable CheckerInfos \p CmdLineArg applies to.
-  /// For example, it'll return the checkers for the core package, if
-  /// \p CmdLineArg is "core".
-  CheckerInfoListRange getMutableCheckersForCmdLineArg(StringRef CmdLineArg);
-
-  CheckerInfoList Checkers;
-  PackageInfoList Packages;
-  /// Used for couting how many checkers belong to a certain package in the
-  /// \c Checkers field. For convenience purposes.
-  llvm::StringMap<size_t> PackageSizes;
-
-  /// Contains all (Dependendent checker, Dependency) pairs. We need this, as
-  /// we'll resolve dependencies after all checkers were added first.
-  llvm::SmallVector<std::pair<StringRef, StringRef>, 0> Dependencies;
-  llvm::SmallVector<std::pair<StringRef, StringRef>, 0> WeakDependencies;
-
   template <bool IsWeak> void resolveDependencies();
-
-  /// Contains all (FullName, CmdLineOption) pairs. Similarly to dependencies,
-  /// we only modify the actual CheckerInfo and PackageInfo objects once all
-  /// of them have been added.
-  llvm::SmallVector<std::pair<StringRef, CmdLineOption>, 0> PackageOptions;
-  llvm::SmallVector<std::pair<StringRef, CmdLineOption>, 0> CheckerOptions;
-
   void resolveCheckerAndPackageOptions();
 
+  CheckerRegistryData &Data;
+
   DiagnosticsEngine &Diags;
   AnalyzerOptions &AnOpts;
-  CheckerInfoSet EnabledCheckers;
 };
 
 } // namespace ento
 } // namespace clang
 
-#endif // LLVM_CLANG_STATICANALYZER_CORE_CHECKERREGISTRY_H
+#endif // LLVM_CLANG_STATICANALYZER_FRONTEND_CHECKERREGISTRY_H
diff --git a/clang/include/clang/Testing/TestClangConfig.h b/clang/include/clang/Testing/TestClangConfig.h
new file mode 100644
index 0000000000000..eefa36dc2ebb9
--- /dev/null
+++ b/clang/include/clang/Testing/TestClangConfig.h
@@ -0,0 +1,85 @@
+//===--- TestClangConfig.h ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TESTING_TESTCLANGCONFIG_H
+#define LLVM_CLANG_TESTING_TESTCLANGCONFIG_H
+
+#include "clang/Testing/CommandLineArgs.h"
+#include "llvm/Support/raw_ostream.h"
+#include <string>
+#include <vector>
+
+namespace clang {
+
+/// A Clang configuration for end-to-end tests that can be converted to
+/// command line arguments for the driver.
+///
+/// The configuration is represented as typed, named values, making it easier
+/// and safer to work with compared to an array of string command line flags.
+struct TestClangConfig {
+  TestLanguage Language;
+
+  /// The argument of the `-target` command line flag.
+  std::string Target;
+
+  bool isC() const { return Language == Lang_C89 || Language == Lang_C99; }
+
+  bool isC99OrLater() const { return Language == Lang_C99; }
+
+  bool isCXX() const {
+    return Language == Lang_CXX03 || Language == Lang_CXX11 ||
+           Language == Lang_CXX14 || Language == Lang_CXX17 ||
+           Language == Lang_CXX20;
+  }
+
+  bool isCXX11OrLater() const {
+    return Language == Lang_CXX11 || Language == Lang_CXX14 ||
+           Language == Lang_CXX17 || Language == Lang_CXX20;
+  }
+
+  bool isCXX14OrLater() const {
+    return Language == Lang_CXX14 || Language == Lang_CXX17 ||
+           Language == Lang_CXX20;
+  }
+
+  bool isCXX17OrLater() const {
+    return Language == Lang_CXX17 || Language == Lang_CXX20;
+  }
+
+  bool supportsCXXDynamicExceptionSpecification() const {
+    return Language == Lang_CXX03 || Language == Lang_CXX11 ||
+           Language == Lang_CXX14;
+  }
+
+  bool hasDelayedTemplateParsing() const {
+    return Target == "x86_64-pc-win32-msvc";
+  }
+
+  std::vector<std::string> getCommandLineArgs() const {
+    std::vector<std::string> Result = getCommandLineArgsForTesting(Language);
+    Result.push_back("-target");
+    Result.push_back(Target);
+    return Result;
+  }
+
+  std::string toString() const {
+    std::string Result;
+    llvm::raw_string_ostream OS(Result);
+    OS << "{ Language=" << Language << ", Target=" << Target << " }";
+    return OS.str();
+  }
+
+  friend std::ostream &operator<<(std::ostream &OS,
+                                  const TestClangConfig &ClangConfig) {
+    return OS << ClangConfig.toString();
+  }
+};
+
+} // end namespace clang
+
+#endif
diff --git a/clang/include/clang/Tooling/Refactoring/AtomicChange.h b/clang/include/clang/Tooling/Refactoring/AtomicChange.h
index 7cb9987e80c7b..f1034a3d05794 100644
--- a/clang/include/clang/Tooling/Refactoring/AtomicChange.h
+++ b/clang/include/clang/Tooling/Refactoring/AtomicChange.h
@@ -17,6 +17,7 @@
 #include "clang/Basic/SourceManager.h"
 #include "clang/Format/Format.h"
 #include "clang/Tooling/Core/Replacement.h"
+#include "llvm/ADT/Any.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
 
@@ -41,6 +42,9 @@ class AtomicChange {
   /// is being changed, e.g. the call to a refactored method.
   AtomicChange(const SourceManager &SM, SourceLocation KeyPosition);
 
+  AtomicChange(const SourceManager &SM, SourceLocation KeyPosition,
+               llvm::Any Metadata);
+
   /// Creates an atomic change for \p FilePath with a customized key.
   AtomicChange(llvm::StringRef FilePath, llvm::StringRef Key)
       : Key(Key), FilePath(FilePath) {}
@@ -120,6 +124,8 @@ class AtomicChange {
     return RemovedHeaders;
   }
 
+  const llvm::Any &getMetadata() const { return Metadata; }
+
 private:
   AtomicChange() {}
 
@@ -135,6 +141,12 @@ class AtomicChange {
   std::vector<std::string> InsertedHeaders;
   std::vector<std::string> RemovedHeaders;
   tooling::Replacements Replaces;
+
+  // This field stores metadata which is ignored for the purposes of applying
+  // edits to source, but may be useful for other consumers of AtomicChanges. In
+  // particular, consumers can use this to direct how they want to consume each
+  // edit.
+  llvm::Any Metadata;
 };
 
 using AtomicChanges = std::vector<AtomicChange>;
diff --git a/clang/include/clang/Tooling/ReplacementsYaml.h b/clang/include/clang/Tooling/ReplacementsYaml.h
index 2e3e401652e22..83e35d6232555 100644
--- a/clang/include/clang/Tooling/ReplacementsYaml.h
+++ b/clang/include/clang/Tooling/ReplacementsYaml.h
@@ -35,13 +35,7 @@ template <> struct MappingTraits<clang::tooling::Replacement> {
 
     NormalizedReplacement(const IO &, const clang::tooling::Replacement &R)
         : FilePath(R.getFilePath()), Offset(R.getOffset()),
-          Length(R.getLength()), ReplacementText(R.getReplacementText()) {
-      size_t lineBreakPos = ReplacementText.find('\n');
-      while (lineBreakPos != std::string::npos) {
-        ReplacementText.replace(lineBreakPos, 1, "\n\n");
-        lineBreakPos = ReplacementText.find('\n', lineBreakPos + 2);
-      }
-    }
+          Length(R.getLength()), ReplacementText(R.getReplacementText()) {}
 
     clang::tooling::Replacement denormalize(const IO &) {
       return clang::tooling::Replacement(FilePath, Offset, Length,
diff --git a/clang/include/clang/Tooling/Syntax/Nodes.h b/clang/include/clang/Tooling/Syntax/Nodes.h
index 139ac9aa8eca2..d97b127638bba 100644
--- a/clang/include/clang/Tooling/Syntax/Nodes.h
+++ b/clang/include/clang/Tooling/Syntax/Nodes.h
@@ -43,8 +43,17 @@ enum class NodeKind : uint16_t {
   PrefixUnaryOperatorExpression,
   PostfixUnaryOperatorExpression,
   BinaryOperatorExpression,
-  CxxNullPtrExpression,
+  ParenExpression,
   IntegerLiteralExpression,
+  CharacterLiteralExpression,
+  FloatingLiteralExpression,
+  StringLiteralExpression,
+  BoolLiteralExpression,
+  CxxNullPtrExpression,
+  IntegerUserDefinedLiteralExpression,
+  FloatUserDefinedLiteralExpression,
+  CharUserDefinedLiteralExpression,
+  StringUserDefinedLiteralExpression,
   IdExpression,
 
   // Statements.
@@ -157,7 +166,8 @@ enum class NodeRole : uint8_t {
   ParametersAndQualifiers_trailingReturn,
   IdExpression_id,
   IdExpression_qualifier,
-  NestedNameSpecifier_specifier
+  NestedNameSpecifier_specifier,
+  ParenExpression_subExpression
 };
 /// For debugging purposes.
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, NodeRole R);
@@ -244,7 +254,72 @@ class UnknownExpression final : public Expression {
   }
 };
 
-/// C++11 'nullptr' expression.
+/// Models a parenthesized expression `(E)`. C++ [expr.prim.paren]
+/// e.g. `(3 + 2)` in `a = 1 + (3 + 2);`
+class ParenExpression final : public Expression {
+public:
+  ParenExpression() : Expression(NodeKind::ParenExpression) {}
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::ParenExpression;
+  }
+  syntax::Leaf *openParen();
+  syntax::Expression *subExpression();
+  syntax::Leaf *closeParen();
+};
+
+/// Expression for integer literals. C++ [lex.icon]
+class IntegerLiteralExpression final : public Expression {
+public:
+  IntegerLiteralExpression() : Expression(NodeKind::IntegerLiteralExpression) {}
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::IntegerLiteralExpression;
+  }
+  syntax::Leaf *literalToken();
+};
+
+/// Expression for character literals. C++ [lex.ccon]
+class CharacterLiteralExpression final : public Expression {
+public:
+  CharacterLiteralExpression()
+      : Expression(NodeKind::CharacterLiteralExpression) {}
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::CharacterLiteralExpression;
+  }
+  syntax::Leaf *literalToken();
+};
+
+/// Expression for floating-point literals. C++ [lex.fcon]
+class FloatingLiteralExpression final : public Expression {
+public:
+  FloatingLiteralExpression()
+      : Expression(NodeKind::FloatingLiteralExpression) {}
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::FloatingLiteralExpression;
+  }
+  syntax::Leaf *literalToken();
+};
+
+/// Expression for string-literals. C++ [lex.string]
+class StringLiteralExpression final : public Expression {
+public:
+  StringLiteralExpression() : Expression(NodeKind::StringLiteralExpression) {}
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::StringLiteralExpression;
+  }
+  syntax::Leaf *literalToken();
+};
+
+/// Expression for boolean literals. C++ [lex.bool]
+class BoolLiteralExpression final : public Expression {
+public:
+  BoolLiteralExpression() : Expression(NodeKind::BoolLiteralExpression) {}
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::BoolLiteralExpression;
+  }
+  syntax::Leaf *literalToken();
+};
+
+/// Expression for the `nullptr` literal. C++ [lex.nullptr]
 class CxxNullPtrExpression final : public Expression {
 public:
   CxxNullPtrExpression() : Expression(NodeKind::CxxNullPtrExpression) {}
@@ -254,16 +329,72 @@ class CxxNullPtrExpression final : public Expression {
   syntax::Leaf *nullPtrKeyword();
 };
 
-/// Expression for integer literals.
-class IntegerLiteralExpression final : public Expression {
+/// Expression for user-defined literal. C++ [lex.ext]
+/// user-defined-literal:
+///   user-defined-integer-literal
+///   user-defined-floating-point-literal
+///   user-defined-string-literal
+///   user-defined-character-literal
+class UserDefinedLiteralExpression : public Expression {
 public:
-  IntegerLiteralExpression() : Expression(NodeKind::IntegerLiteralExpression) {}
+  UserDefinedLiteralExpression(NodeKind K) : Expression(K) {}
   static bool classof(const Node *N) {
-    return N->kind() == NodeKind::IntegerLiteralExpression;
+    return N->kind() == NodeKind::IntegerUserDefinedLiteralExpression ||
+           N->kind() == NodeKind::FloatUserDefinedLiteralExpression ||
+           N->kind() == NodeKind::CharUserDefinedLiteralExpression ||
+           N->kind() == NodeKind::StringUserDefinedLiteralExpression;
   }
   syntax::Leaf *literalToken();
 };
 
+/// Expression for user-defined-integer-literal. C++ [lex.ext]
+class IntegerUserDefinedLiteralExpression final
+    : public UserDefinedLiteralExpression {
+public:
+  IntegerUserDefinedLiteralExpression()
+      : UserDefinedLiteralExpression(
+            NodeKind::IntegerUserDefinedLiteralExpression) {}
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::IntegerUserDefinedLiteralExpression;
+  }
+};
+
+/// Expression for user-defined-floating-point-literal. C++ [lex.ext]
+class FloatUserDefinedLiteralExpression final
+    : public UserDefinedLiteralExpression {
+public:
+  FloatUserDefinedLiteralExpression()
+      : UserDefinedLiteralExpression(
+            NodeKind::FloatUserDefinedLiteralExpression) {}
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::FloatUserDefinedLiteralExpression;
+  }
+};
+
+/// Expression for user-defined-character-literal. C++ [lex.ext]
+class CharUserDefinedLiteralExpression final
+    : public UserDefinedLiteralExpression {
+public:
+  CharUserDefinedLiteralExpression()
+      : UserDefinedLiteralExpression(
+            NodeKind::CharUserDefinedLiteralExpression) {}
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::CharUserDefinedLiteralExpression;
+  }
+};
+
+/// Expression for user-defined-string-literal. C++ [lex.ext]
+class StringUserDefinedLiteralExpression final
+    : public UserDefinedLiteralExpression {
+public:
+  StringUserDefinedLiteralExpression()
+      : UserDefinedLiteralExpression(
+            NodeKind::StringUserDefinedLiteralExpression) {}
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::StringUserDefinedLiteralExpression;
+  }
+};
+
 /// An abstract class for prefix and postfix unary operators.
 class UnaryOperatorExpression : public Expression {
 public:
diff --git a/clang/include/clang/Tooling/Transformer/RangeSelector.h b/clang/include/clang/Tooling/Transformer/RangeSelector.h
index eaab3fc4c0c78..2807037bc208f 100644
--- a/clang/include/clang/Tooling/Transformer/RangeSelector.h
+++ b/clang/include/clang/Tooling/Transformer/RangeSelector.h
@@ -32,10 +32,20 @@ inline RangeSelector charRange(CharSourceRange R) {
 }
 
 /// Selects from the start of \p Begin and to the end of \p End.
-RangeSelector range(RangeSelector Begin, RangeSelector End);
+RangeSelector enclose(RangeSelector Begin, RangeSelector End);
 
 /// Convenience version of \c range where end-points are bound nodes.
-RangeSelector range(std::string BeginID, std::string EndID);
+RangeSelector encloseNodes(std::string BeginID, std::string EndID);
+
+/// DEPRECATED. Use `enclose`.
+inline RangeSelector range(RangeSelector Begin, RangeSelector End) {
+  return enclose(std::move(Begin), std::move(End));
+}
+
+/// DEPRECATED. Use `encloseNodes`.
+inline RangeSelector range(std::string BeginID, std::string EndID) {
+  return encloseNodes(std::move(BeginID), std::move(EndID));
+}
 
 /// Selects the (empty) range [B,B) when \p Selector selects the range [B,E).
 RangeSelector before(RangeSelector Selector);
diff --git a/clang/include/clang/Tooling/Transformer/RewriteRule.h b/clang/include/clang/Tooling/Transformer/RewriteRule.h
index 0a961ccc475df..d9e68717d5c8f 100644
--- a/clang/include/clang/Tooling/Transformer/RewriteRule.h
+++ b/clang/include/clang/Tooling/Transformer/RewriteRule.h
@@ -21,6 +21,7 @@
 #include "clang/Tooling/Refactoring/AtomicChange.h"
 #include "clang/Tooling/Transformer/MatchConsumer.h"
 #include "clang/Tooling/Transformer/RangeSelector.h"
+#include "llvm/ADT/Any.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Error.h"
@@ -35,6 +36,7 @@ namespace transformer {
 struct Edit {
   CharSourceRange Range;
   std::string Replacement;
+  llvm::Any Metadata;
 };
 
 /// Maps a match result to a list of concrete edits (with possible
@@ -85,6 +87,7 @@ struct ASTEdit {
   RangeSelector TargetRange;
   TextGenerator Replacement;
   TextGenerator Note;
+  llvm::Any Metadata;
 };
 
 /// Lifts a list of `ASTEdit`s into an `EditGenerator`.
@@ -258,6 +261,11 @@ inline ASTEdit insertAfter(RangeSelector S, TextGenerator Replacement) {
 /// Removes the source selected by \p S.
 ASTEdit remove(RangeSelector S);
 
+inline ASTEdit withMetadata(ASTEdit edit, llvm::Any Metadata) {
+  edit.Metadata = std::move(Metadata);
+  return edit;
+}
+
 /// The following three functions are a low-level part of the RewriteRule
 /// API. We expose them for use in implementing the fixtures that interpret
 /// RewriteRule, like Transformer and TransfomerTidy, or for more advanced
diff --git a/clang/include/clang/Tooling/Transformer/Stencil.h b/clang/include/clang/Tooling/Transformer/Stencil.h
index 0363b689dc5b7..1b50a670f70b6 100644
--- a/clang/include/clang/Tooling/Transformer/Stencil.h
+++ b/clang/include/clang/Tooling/Transformer/Stencil.h
@@ -69,14 +69,6 @@ template <typename... Ts> Stencil cat(Ts &&... Parts) {
 // Functions for conveniently building stencils.
 //
 
-/// DEPRECATED: Use `cat` instead.
-/// \returns exactly the text provided.
-Stencil text(llvm::StringRef Text);
-
-/// DEPRECATED: Use `cat` instead.
-/// \returns the source corresponding to the selected range.
-Stencil selection(RangeSelector Selector);
-
 /// Generates the source of the expression bound to \p Id, wrapping it in
 /// parentheses if it may parse differently depending on context. For example, a
 /// binary operation is always wrapped, while a variable reference is never
@@ -112,7 +104,7 @@ Stencil maybeAddressOf(llvm::StringRef ExprId);
 /// Additionally, `e` is wrapped in parentheses, if needed.
 Stencil access(llvm::StringRef BaseId, Stencil Member);
 inline Stencil access(llvm::StringRef BaseId, llvm::StringRef Member) {
-  return access(BaseId, text(Member));
+  return access(BaseId, detail::makeStencil(Member));
 }
 
 /// Chooses between the two stencil parts, based on whether \p ID is bound in
@@ -123,7 +115,8 @@ Stencil ifBound(llvm::StringRef Id, Stencil TrueStencil, Stencil FalseStencil);
 /// match.
 inline Stencil ifBound(llvm::StringRef Id, llvm::StringRef TrueText,
                        llvm::StringRef FalseText) {
-  return ifBound(Id, text(TrueText), text(FalseText));
+  return ifBound(Id, detail::makeStencil(TrueText),
+                 detail::makeStencil(FalseText));
 }
 
 /// Wraps a \c MatchConsumer in a \c Stencil, so that it can be used in a \c
diff --git a/clang/include/clang/module.modulemap b/clang/include/clang/module.modulemap
index 7549ff2e3bcd6..13d4dbf9dc2e8 100644
--- a/clang/include/clang/module.modulemap
+++ b/clang/include/clang/module.modulemap
@@ -54,6 +54,7 @@ module Clang_Basic {
   textual header "Basic/CodeGenOptions.def"
   textual header "Basic/DiagnosticOptions.def"
   textual header "Basic/Features.def"
+  textual header "Basic/FPOptions.def"
   textual header "Basic/MSP430Target.def"
   textual header "Basic/LangOptions.def"
   textual header "Basic/OpenCLExtensions.def"
diff --git a/clang/lib/ARCMigrate/CMakeLists.txt b/clang/lib/ARCMigrate/CMakeLists.txt
index 619328ca5ca76..6f19bea476daa 100644
--- a/clang/lib/ARCMigrate/CMakeLists.txt
+++ b/clang/lib/ARCMigrate/CMakeLists.txt
@@ -34,4 +34,7 @@ add_clang_library(clangARCMigrate
   clangRewrite
   clangSema
   clangSerialization
+
+  DEPENDS
+  omp_gen
   )
diff --git a/clang/lib/ARCMigrate/TransProperties.cpp b/clang/lib/ARCMigrate/TransProperties.cpp
index adfb4bd77ac60..cba2256ef97b0 100644
--- a/clang/lib/ARCMigrate/TransProperties.cpp
+++ b/clang/lib/ARCMigrate/TransProperties.cpp
@@ -287,7 +287,10 @@ class PropertiesRewriter {
   public:
     PlusOneAssign(ObjCIvarDecl *D) : Ivar(D) {}
 
-    bool VisitBinAssign(BinaryOperator *E) {
+    bool VisitBinaryOperator(BinaryOperator *E) {
+      if (E->getOpcode() != BO_Assign)
+        return true;
+
       Expr *lhs = E->getLHS()->IgnoreParenImpCasts();
       if (ObjCIvarRefExpr *RE = dyn_cast<ObjCIvarRefExpr>(lhs)) {
         if (RE->getDecl() != Ivar)
diff --git a/clang/lib/AST/APValue.cpp b/clang/lib/AST/APValue.cpp
index 50f8d05dacb4f..f3828bb54c1d3 100644
--- a/clang/lib/AST/APValue.cpp
+++ b/clang/lib/AST/APValue.cpp
@@ -378,11 +378,6 @@ void APValue::swap(APValue &RHS) {
   memcpy(RHS.Data.buffer, TmpData, DataSize);
 }
 
-LLVM_DUMP_METHOD void APValue::dump() const {
-  dump(llvm::errs());
-  llvm::errs() << '\n';
-}
-
 static double GetApproxValue(const llvm::APFloat &F) {
   llvm::APFloat V = F;
   bool ignored;
@@ -391,85 +386,6 @@ static double GetApproxValue(const llvm::APFloat &F) {
   return V.convertToDouble();
 }
 
-void APValue::dump(raw_ostream &OS) const {
-  switch (getKind()) {
-  case None:
-    OS << "None";
-    return;
-  case Indeterminate:
-    OS << "Indeterminate";
-    return;
-  case Int:
-    OS << "Int: " << getInt();
-    return;
-  case Float:
-    OS << "Float: " << GetApproxValue(getFloat());
-    return;
-  case FixedPoint:
-    OS << "FixedPoint : " << getFixedPoint();
-    return;
-  case Vector:
-    OS << "Vector: ";
-    getVectorElt(0).dump(OS);
-    for (unsigned i = 1; i != getVectorLength(); ++i) {
-      OS << ", ";
-      getVectorElt(i).dump(OS);
-    }
-    return;
-  case ComplexInt:
-    OS << "ComplexInt: " << getComplexIntReal() << ", " << getComplexIntImag();
-    return;
-  case ComplexFloat:
-    OS << "ComplexFloat: " << GetApproxValue(getComplexFloatReal())
-       << ", " << GetApproxValue(getComplexFloatImag());
-    return;
-  case LValue:
-    OS << "LValue: <todo>";
-    return;
-  case Array:
-    OS << "Array: ";
-    for (unsigned I = 0, N = getArrayInitializedElts(); I != N; ++I) {
-      getArrayInitializedElt(I).dump(OS);
-      if (I != getArraySize() - 1) OS << ", ";
-    }
-    if (hasArrayFiller()) {
-      OS << getArraySize() - getArrayInitializedElts() << " x ";
-      getArrayFiller().dump(OS);
-    }
-    return;
-  case Struct:
-    OS << "Struct ";
-    if (unsigned N = getStructNumBases()) {
-      OS << " bases: ";
-      getStructBase(0).dump(OS);
-      for (unsigned I = 1; I != N; ++I) {
-        OS << ", ";
-        getStructBase(I).dump(OS);
-      }
-    }
-    if (unsigned N = getStructNumFields()) {
-      OS << " fields: ";
-      getStructField(0).dump(OS);
-      for (unsigned I = 1; I != N; ++I) {
-        OS << ", ";
-        getStructField(I).dump(OS);
-      }
-    }
-    return;
-  case Union:
-    OS << "Union: ";
-    getUnionValue().dump(OS);
-    return;
-  case MemberPointer:
-    OS << "MemberPointer: <todo>";
-    return;
-  case AddrLabelDiff:
-    OS << "AddrLabelDiff: <todo>";
-    return;
-  }
-  llvm_unreachable("Unknown APValue kind!");
-}
-
 void APValue::printPretty(raw_ostream &Out, const ASTContext &Ctx,
                           QualType Ty) const {
   switch (getKind()) {
diff --git a/clang/lib/AST/ASTDumper.cpp b/clang/lib/AST/ASTDumper.cpp
index d7a25cc827aa4..284e5bdbc6b0f 100644
--- a/clang/lib/AST/ASTDumper.cpp
+++ b/clang/lib/AST/ASTDumper.cpp
@@ -159,17 +159,22 @@ void QualType::dump(const char *msg) const {
   dump();
 }
 
-LLVM_DUMP_METHOD void QualType::dump() const { dump(llvm::errs()); }
+LLVM_DUMP_METHOD void QualType::dump() const {
+  ASTDumper Dumper(llvm::errs(), /*ShowColors=*/false);
+  Dumper.Visit(*this);
+}
 
-LLVM_DUMP_METHOD void QualType::dump(llvm::raw_ostream &OS) const {
-  ASTDumper Dumper(OS, nullptr, nullptr);
+LLVM_DUMP_METHOD void QualType::dump(llvm::raw_ostream &OS,
+                                     const ASTContext &Context) const {
+  ASTDumper Dumper(OS, Context, Context.getDiagnostics().getShowColors());
   Dumper.Visit(*this);
 }
 
-LLVM_DUMP_METHOD void Type::dump() const { dump(llvm::errs()); }
+LLVM_DUMP_METHOD void Type::dump() const { QualType(this, 0).dump(); }
 
-LLVM_DUMP_METHOD void Type::dump(llvm::raw_ostream &OS) const {
-  QualType(this, 0).dump(OS);
+LLVM_DUMP_METHOD void Type::dump(llvm::raw_ostream &OS,
+                                 const ASTContext &Context) const {
+  QualType(this, 0).dump(OS, Context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -189,8 +194,7 @@ LLVM_DUMP_METHOD void Decl::dump(raw_ostream &OS, bool Deserialize,
     (void)Deserialize; // FIXME?
     P.Visit(this);
   } else {
-    ASTDumper P(OS, &Ctx.getCommentCommandTraits(), &SM,
-                SM.getDiagnostics().getShowColors(), Ctx.getPrintingPolicy());
+    ASTDumper P(OS, Ctx, Ctx.getDiagnostics().getShowColors());
     P.setDeserialize(Deserialize);
     P.Visit(this);
   }
@@ -198,9 +202,7 @@ LLVM_DUMP_METHOD void Decl::dump(raw_ostream &OS, bool Deserialize,
 
 LLVM_DUMP_METHOD void Decl::dumpColor() const {
   const ASTContext &Ctx = getASTContext();
-  ASTDumper P(llvm::errs(), &Ctx.getCommentCommandTraits(),
-              &Ctx.getSourceManager(), /*ShowColors*/ true,
-              Ctx.getPrintingPolicy());
+  ASTDumper P(llvm::errs(), Ctx, /*ShowColors=*/true);
   P.Visit(this);
 }
 
@@ -214,10 +216,8 @@ LLVM_DUMP_METHOD void DeclContext::dumpLookups(raw_ostream &OS,
   const DeclContext *DC = this;
   while (!DC->isTranslationUnit())
     DC = DC->getParent();
-  ASTContext &Ctx = cast<TranslationUnitDecl>(DC)->getASTContext();
-  const SourceManager &SM = Ctx.getSourceManager();
-  ASTDumper P(OS, &Ctx.getCommentCommandTraits(), &Ctx.getSourceManager(),
-              SM.getDiagnostics().getShowColors(), Ctx.getPrintingPolicy());
+  const ASTContext &Ctx = cast<TranslationUnitDecl>(DC)->getASTContext();
+  ASTDumper P(OS, Ctx, Ctx.getDiagnostics().getShowColors());
   P.setDeserialize(Deserialize);
   P.dumpLookups(this, DumpDecls);
 }
@@ -226,27 +226,19 @@ LLVM_DUMP_METHOD void DeclContext::dumpLookups(raw_ostream &OS,
 // Stmt method implementations
 //===----------------------------------------------------------------------===//
 
-LLVM_DUMP_METHOD void Stmt::dump(SourceManager &SM) const {
-  dump(llvm::errs(), SM);
-}
-
-LLVM_DUMP_METHOD void Stmt::dump(raw_ostream &OS, SourceManager &SM) const {
-  ASTDumper P(OS, nullptr, &SM);
-  P.Visit(this);
-}
-
-LLVM_DUMP_METHOD void Stmt::dump(raw_ostream &OS) const {
-  ASTDumper P(OS, nullptr, nullptr);
+LLVM_DUMP_METHOD void Stmt::dump() const {
+  ASTDumper P(llvm::errs(), /*ShowColors=*/false);
   P.Visit(this);
 }
 
-LLVM_DUMP_METHOD void Stmt::dump() const {
-  ASTDumper P(llvm::errs(), nullptr, nullptr);
+LLVM_DUMP_METHOD void Stmt::dump(raw_ostream &OS,
+                                 const ASTContext &Context) const {
+  ASTDumper P(OS, Context, Context.getDiagnostics().getShowColors());
   P.Visit(this);
 }
 
 LLVM_DUMP_METHOD void Stmt::dumpColor() const {
-  ASTDumper P(llvm::errs(), nullptr, nullptr, /*ShowColors*/true);
+  ASTDumper P(llvm::errs(), /*ShowColors=*/true);
   P.Visit(this);
 }
 
@@ -255,27 +247,42 @@ LLVM_DUMP_METHOD void Stmt::dumpColor() const {
 //===----------------------------------------------------------------------===//
 
 LLVM_DUMP_METHOD void Comment::dump() const {
-  dump(llvm::errs(), nullptr, nullptr);
-}
-
-LLVM_DUMP_METHOD void Comment::dump(const ASTContext &Context) const {
-  dump(llvm::errs(), &Context.getCommentCommandTraits(),
-       &Context.getSourceManager());
+  const auto *FC = dyn_cast<FullComment>(this);
+  if (!FC)
+    return;
+  ASTDumper Dumper(llvm::errs(), /*ShowColors=*/false);
+  Dumper.Visit(FC, FC);
 }
 
-void Comment::dump(raw_ostream &OS, const CommandTraits *Traits,
-                   const SourceManager *SM) const {
-  const FullComment *FC = dyn_cast<FullComment>(this);
+LLVM_DUMP_METHOD void Comment::dump(raw_ostream &OS,
+                                    const ASTContext &Context) const {
+  const auto *FC = dyn_cast<FullComment>(this);
   if (!FC)
     return;
-  ASTDumper D(OS, Traits, SM);
-  D.Visit(FC, FC);
+  ASTDumper Dumper(OS, Context, Context.getDiagnostics().getShowColors());
+  Dumper.Visit(FC, FC);
 }
 
 LLVM_DUMP_METHOD void Comment::dumpColor() const {
-  const FullComment *FC = dyn_cast<FullComment>(this);
+  const auto *FC = dyn_cast<FullComment>(this);
   if (!FC)
     return;
-  ASTDumper D(llvm::errs(), nullptr, nullptr, /*ShowColors*/true);
-  D.Visit(FC, FC);
+  ASTDumper Dumper(llvm::errs(), /*ShowColors=*/true);
+  Dumper.Visit(FC, FC);
+}
+
+//===----------------------------------------------------------------------===//
+// APValue method implementations
+//===----------------------------------------------------------------------===//
+
+LLVM_DUMP_METHOD void APValue::dump() const {
+  ASTDumper Dumper(llvm::errs(), /*ShowColors=*/false);
+  Dumper.Visit(*this, /*Ty=*/QualType());
+}
+
+LLVM_DUMP_METHOD void APValue::dump(raw_ostream &OS,
+                                    const ASTContext &Context) const {
+  ASTDumper Dumper(llvm::errs(), Context,
+                   Context.getDiagnostics().getShowColors());
+  Dumper.Visit(*this, /*Ty=*/Context.getPointerType(Context.CharTy));
 }
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 8b96e20e374d3..3779e0cb872b5 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -3646,6 +3646,54 @@ ExpectedDecl ASTNodeImporter::VisitIndirectFieldDecl(IndirectFieldDecl *D) {
   return ToIndirectField;
 }
 
+/// Used as return type of getFriendCountAndPosition.
+struct FriendCountAndPosition {
+  /// Number of similar looking friends.
+  unsigned int TotalCount;
+  /// Index of the specific FriendDecl.
+  unsigned int IndexOfDecl;
+};
+
+template <class T>
+static FriendCountAndPosition getFriendCountAndPosition(
+    const FriendDecl *FD,
+    llvm::function_ref<T(const FriendDecl *)> GetCanTypeOrDecl) {
+  unsigned int FriendCount = 0;
+  llvm::Optional<unsigned int> FriendPosition;
+  const auto *RD = cast<CXXRecordDecl>(FD->getLexicalDeclContext());
+
+  T TypeOrDecl = GetCanTypeOrDecl(FD);
+
+  for (const FriendDecl *FoundFriend : RD->friends()) {
+    if (FoundFriend == FD) {
+      FriendPosition = FriendCount;
+      ++FriendCount;
+    } else if (!FoundFriend->getFriendDecl() == !FD->getFriendDecl() &&
+               GetCanTypeOrDecl(FoundFriend) == TypeOrDecl) {
+      ++FriendCount;
+    }
+  }
+
+  assert(FriendPosition && "Friend decl not found in own parent.");
+
+  return {FriendCount, *FriendPosition};
+}
+
+static FriendCountAndPosition getFriendCountAndPosition(const FriendDecl *FD) {
+  if (FD->getFriendType())
+    return getFriendCountAndPosition<QualType>(FD, [](const FriendDecl *F) {
+      if (TypeSourceInfo *TSI = F->getFriendType())
+        return TSI->getType().getCanonicalType();
+      llvm_unreachable("Wrong friend object type.");
+    });
+  else
+    return getFriendCountAndPosition<Decl *>(FD, [](const FriendDecl *F) {
+      if (Decl *D = F->getFriendDecl())
+        return D->getCanonicalDecl();
+      llvm_unreachable("Wrong friend object type.");
+    });
+}
+
 ExpectedDecl ASTNodeImporter::VisitFriendDecl(FriendDecl *D) {
   // Import the major distinguishing characteristics of a declaration.
   DeclContext *DC, *LexicalDC;
@@ -3654,25 +3702,37 @@ ExpectedDecl ASTNodeImporter::VisitFriendDecl(FriendDecl *D) {
 
   // Determine whether we've already imported this decl.
   // FriendDecl is not a NamedDecl so we cannot use lookup.
-  auto *RD = cast<CXXRecordDecl>(DC);
+  // We try to maintain order and count of redundant friend declarations.
+  const auto *RD = cast<CXXRecordDecl>(DC);
   FriendDecl *ImportedFriend = RD->getFirstFriend();
+  SmallVector<FriendDecl *, 2> ImportedEquivalentFriends;
 
   while (ImportedFriend) {
+    bool Match = false;
     if (D->getFriendDecl() && ImportedFriend->getFriendDecl()) {
-      if (IsStructuralMatch(D->getFriendDecl(), ImportedFriend->getFriendDecl(),
-                            /*Complain=*/false))
-        return Importer.MapImported(D, ImportedFriend);
-
+      Match =
+          IsStructuralMatch(D->getFriendDecl(), ImportedFriend->getFriendDecl(),
+                            /*Complain=*/false);
     } else if (D->getFriendType() && ImportedFriend->getFriendType()) {
-      if (Importer.IsStructurallyEquivalent(
-            D->getFriendType()->getType(),
-            ImportedFriend->getFriendType()->getType(), true))
-        return Importer.MapImported(D, ImportedFriend);
+      Match = Importer.IsStructurallyEquivalent(
+          D->getFriendType()->getType(),
+          ImportedFriend->getFriendType()->getType(), /*Complain=*/false);
     }
+    if (Match)
+      ImportedEquivalentFriends.push_back(ImportedFriend);
+
     ImportedFriend = ImportedFriend->getNextFriend();
   }
+  FriendCountAndPosition CountAndPosition = getFriendCountAndPosition(D);
+
+  assert(ImportedEquivalentFriends.size() <= CountAndPosition.TotalCount &&
+         "Class with non-matching friends is imported, ODR check wrong?");
+  if (ImportedEquivalentFriends.size() == CountAndPosition.TotalCount)
+    return Importer.MapImported(
+        D, ImportedEquivalentFriends[CountAndPosition.IndexOfDecl]);
 
   // Not found. Create it.
+  // The declarations will be put into order later by ImportDeclContext.
   FriendDecl::FriendUnion ToFU;
   if (NamedDecl *FriendD = D->getFriendDecl()) {
     NamedDecl *ToFriendD;
@@ -6057,11 +6117,13 @@ ExpectedStmt ASTNodeImporter::VisitWhileStmt(WhileStmt *S) {
   auto ToCond = importChecked(Err, S->getCond());
   auto ToBody = importChecked(Err, S->getBody());
   auto ToWhileLoc = importChecked(Err, S->getWhileLoc());
+  auto ToLParenLoc = importChecked(Err, S->getLParenLoc());
+  auto ToRParenLoc = importChecked(Err, S->getRParenLoc());
   if (Err)
     return std::move(Err);
 
   return WhileStmt::Create(Importer.getToContext(), ToConditionVariable, ToCond,
-                           ToBody, ToWhileLoc);
+                           ToBody, ToWhileLoc, ToLParenLoc, ToRParenLoc);
 }
 
 ExpectedStmt ASTNodeImporter::VisitDoStmt(DoStmt *S) {
@@ -6676,7 +6738,7 @@ ExpectedStmt ASTNodeImporter::VisitUnaryOperator(UnaryOperator *E) {
   return UnaryOperator::Create(
       Importer.getToContext(), ToSubExpr, E->getOpcode(), ToType,
       E->getValueKind(), E->getObjectKind(), ToOperatorLoc, E->canOverflow(),
-      E->getFPFeatures(Importer.getFromContext().getLangOpts()));
+      E->getFPOptionsOverride());
 }
 
 ExpectedStmt
diff --git a/clang/lib/AST/ASTImporterLookupTable.cpp b/clang/lib/AST/ASTImporterLookupTable.cpp
index 7390329d4ed8d..4d6fff8f34191 100644
--- a/clang/lib/AST/ASTImporterLookupTable.cpp
+++ b/clang/lib/AST/ASTImporterLookupTable.cpp
@@ -45,7 +45,11 @@ struct Builder : RecursiveASTVisitor<Builder> {
           LT.add(RTy->getAsCXXRecordDecl());
         else if (const auto *SpecTy = dyn_cast<TemplateSpecializationType>(Ty))
           LT.add(SpecTy->getAsCXXRecordDecl());
-        else if (isa<TypedefType>(Ty)) {
+        else if (const auto *SubstTy =
+                     dyn_cast<SubstTemplateTypeParmType>(Ty)) {
+          if (SubstTy->getAsCXXRecordDecl())
+            LT.add(SubstTy->getAsCXXRecordDecl());
+        } else if (isa<TypedefType>(Ty)) {
           // We do not put friend typedefs to the lookup table because
           // ASTImporter does not organize typedefs into redecl chains.
         } else {
diff --git a/clang/lib/AST/ASTTypeTraits.cpp b/clang/lib/AST/ASTTypeTraits.cpp
index 37e81e5813eca..34fc587694be9 100644
--- a/clang/lib/AST/ASTTypeTraits.cpp
+++ b/clang/lib/AST/ASTTypeTraits.cpp
@@ -118,6 +118,8 @@ ASTNodeKind ASTNodeKind::getFromNode(const OMPClause &C) {
 #define OMP_CLAUSE_NO_CLASS(Enum, Str)                                         \
   case llvm::omp::Clause::Enum:                                                \
     llvm_unreachable("unexpected OpenMP clause kind");
+  default:
+    break;
 #include "llvm/Frontend/OpenMP/OMPKinds.def"
   }
   llvm_unreachable("invalid stmt kind");
@@ -150,13 +152,14 @@ void DynTypedNode::print(llvm::raw_ostream &OS,
     OS << "Unable to print values of type " << NodeKind.asStringRef() << "\n";
 }
 
-void DynTypedNode::dump(llvm::raw_ostream &OS, SourceManager &SM) const {
+void DynTypedNode::dump(llvm::raw_ostream &OS,
+                        const ASTContext &Context) const {
   if (const Decl *D = get<Decl>())
     D->dump(OS);
   else if (const Stmt *S = get<Stmt>())
-    S->dump(OS, SM);
+    S->dump(OS, Context);
   else if (const Type *T = get<Type>())
-    T->dump(OS);
+    T->dump(OS, Context);
   else
     OS << "Unable to dump values of type " << NodeKind.asStringRef() << "\n";
 }
diff --git a/clang/lib/AST/CMakeLists.txt b/clang/lib/AST/CMakeLists.txt
index 6018d32af8f8f..35099fd0dacf8 100644
--- a/clang/lib/AST/CMakeLists.txt
+++ b/clang/lib/AST/CMakeLists.txt
@@ -119,4 +119,5 @@ add_clang_library(clangAST
 
   DEPENDS
   Opcodes
+  omp_gen
   )
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index 1676f319394dd..5c0a98815dd79 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -3623,7 +3623,8 @@ bool FunctionDecl::isTemplateInstantiation() const {
   return clang::isTemplateInstantiation(getTemplateSpecializationKind());
 }
 
-FunctionDecl *FunctionDecl::getTemplateInstantiationPattern() const {
+FunctionDecl *
+FunctionDecl::getTemplateInstantiationPattern(bool ForDefinition) const {
   // If this is a generic lambda call operator specialization, its
   // instantiation pattern is always its primary template's pattern
   // even if its primary template was instantiated from another
@@ -3640,18 +3641,20 @@ FunctionDecl *FunctionDecl::getTemplateInstantiationPattern() const {
   }
 
   if (MemberSpecializationInfo *Info = getMemberSpecializationInfo()) {
-    if (!clang::isTemplateInstantiation(Info->getTemplateSpecializationKind()))
+    if (ForDefinition &&
+        !clang::isTemplateInstantiation(Info->getTemplateSpecializationKind()))
       return nullptr;
     return getDefinitionOrSelf(cast<FunctionDecl>(Info->getInstantiatedFrom()));
   }
 
-  if (!clang::isTemplateInstantiation(getTemplateSpecializationKind()))
+  if (ForDefinition &&
+      !clang::isTemplateInstantiation(getTemplateSpecializationKind()))
     return nullptr;
 
   if (FunctionTemplateDecl *Primary = getPrimaryTemplate()) {
     // If we hit a point where the user provided a specialization of this
     // template, we're done looking.
-    while (!Primary->isMemberSpecialization()) {
+    while (!ForDefinition || !Primary->isMemberSpecialization()) {
       auto *NewPrimary = Primary->getInstantiatedFromMemberTemplate();
       if (!NewPrimary)
         break;
diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp
index 2aab53f4fa908..da1eadd9d931d 100644
--- a/clang/lib/AST/DeclBase.cpp
+++ b/clang/lib/AST/DeclBase.cpp
@@ -240,15 +240,47 @@ TemplateDecl *Decl::getDescribedTemplate() const {
   return nullptr;
 }
 
+const TemplateParameterList *Decl::getDescribedTemplateParams() const {
+  if (auto *TD = getDescribedTemplate())
+    return TD->getTemplateParameters();
+  if (auto *CTPSD = dyn_cast<ClassTemplatePartialSpecializationDecl>(this))
+    return CTPSD->getTemplateParameters();
+  if (auto *VTPSD = dyn_cast<VarTemplatePartialSpecializationDecl>(this))
+    return VTPSD->getTemplateParameters();
+  return nullptr;
+}
+
 bool Decl::isTemplated() const {
-  // A declaration is dependent if it is a template or a template pattern, or
+  // A declaration is templated if it is a template or a template pattern, or
   // is within (lexcially for a friend, semantically otherwise) a dependent
   // context.
   // FIXME: Should local extern declarations be treated like friends?
   if (auto *AsDC = dyn_cast<DeclContext>(this))
     return AsDC->isDependentContext();
   auto *DC = getFriendObjectKind() ? getLexicalDeclContext() : getDeclContext();
-  return DC->isDependentContext() || isTemplateDecl() || getDescribedTemplate();
+  return DC->isDependentContext() || isTemplateDecl() ||
+         getDescribedTemplateParams();
+}
+
+unsigned Decl::getTemplateDepth() const {
+  if (auto *DC = dyn_cast<DeclContext>(this))
+    if (DC->isFileContext())
+      return 0;
+
+  if (auto *TPL = getDescribedTemplateParams())
+    return TPL->getDepth() + 1;
+
+  // If this is a dependent lambda, there might be an enclosing variable
+  // template. In this case, the next step is not the parent DeclContext (or
+  // even a DeclContext at all).
+  auto *RD = dyn_cast<CXXRecordDecl>(this);
+  if (RD && RD->isDependentLambda())
+    if (Decl *Context = RD->getLambdaContextDecl())
+      return Context->getTemplateDepth();
+
+  const DeclContext *DC =
+      getFriendObjectKind() ? getLexicalDeclContext() : getDeclContext();
+  return cast<Decl>(DC)->getTemplateDepth();
 }
 
 const DeclContext *Decl::getParentFunctionOrMethod() const {
@@ -332,8 +364,10 @@ void Decl::setDeclContextsImpl(DeclContext *SemaDC, DeclContext *LexicalDC,
   }
 }
 
-bool Decl::isInLocalScope() const {
+bool Decl::isInLocalScopeForInstantiation() const {
   const DeclContext *LDC = getLexicalDeclContext();
+  if (!LDC->isDependentContext())
+    return false;
   while (true) {
     if (LDC->isFunctionOrMethod())
       return true;
diff --git a/clang/lib/AST/DeclPrinter.cpp b/clang/lib/AST/DeclPrinter.cpp
index 4df6512e6c76c..2e48b2b46c4da 100644
--- a/clang/lib/AST/DeclPrinter.cpp
+++ b/clang/lib/AST/DeclPrinter.cpp
@@ -1374,7 +1374,12 @@ void DeclPrinter::VisitObjCProtocolDecl(ObjCProtocolDecl *PID) {
 }
 
 void DeclPrinter::VisitObjCCategoryImplDecl(ObjCCategoryImplDecl *PID) {
-  Out << "@implementation " << *PID->getClassInterface() << '(' << *PID <<")\n";
+  Out << "@implementation ";
+  if (const auto *CID = PID->getClassInterface())
+    Out << *CID;
+  else
+    Out << "<<error-type>>";
+  Out << '(' << *PID << ")\n";
 
   VisitDeclContext(PID, false);
   Out << "@end";
@@ -1382,7 +1387,11 @@ void DeclPrinter::VisitObjCCategoryImplDecl(ObjCCategoryImplDecl *PID) {
 }
 
 void DeclPrinter::VisitObjCCategoryDecl(ObjCCategoryDecl *PID) {
-  Out << "@interface " << *PID->getClassInterface();
+  Out << "@interface ";
+  if (const auto *CID = PID->getClassInterface())
+    Out << *CID;
+  else
+    Out << "<<error-type>>";
   if (auto TypeParams = PID->getTypeParamList()) {
     PrintObjCTypeParams(TypeParams);
   }
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 5cbd66f11601c..343a271c33944 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -3812,6 +3812,11 @@ Expr::isNullPointerConstant(ASTContext &Ctx,
       return Source->isNullPointerConstant(Ctx, NPC);
   }
 
+  // If the expression has no type information, it cannot be a null pointer
+  // constant.
+  if (getType().isNull())
+    return NPCK_NotNull;
+
   // C++11 nullptr_t is always a null pointer constant.
   if (getType()->isNullPtrType())
     return NPCK_CXX11_nullptr;
@@ -4463,7 +4468,7 @@ ParenListExpr *ParenListExpr::CreateEmpty(const ASTContext &Ctx,
 BinaryOperator::BinaryOperator(const ASTContext &Ctx, Expr *lhs, Expr *rhs,
                                Opcode opc, QualType ResTy, ExprValueKind VK,
                                ExprObjectKind OK, SourceLocation opLoc,
-                               FPOptions FPFeatures)
+                               FPOptionsOverride FPFeatures)
     : Expr(BinaryOperatorClass, ResTy, VK, OK) {
   BinaryOperatorBits.Opc = opc;
   assert(!isCompoundAssignmentOp() &&
@@ -4471,8 +4476,7 @@ BinaryOperator::BinaryOperator(const ASTContext &Ctx, Expr *lhs, Expr *rhs,
   BinaryOperatorBits.OpLoc = opLoc;
   SubExprs[LHS] = lhs;
   SubExprs[RHS] = rhs;
-  BinaryOperatorBits.HasFPFeatures =
-      FPFeatures.requiresTrailingStorage(Ctx.getLangOpts());
+  BinaryOperatorBits.HasFPFeatures = FPFeatures.requiresTrailingStorage();
   if (BinaryOperatorBits.HasFPFeatures)
     *getTrailingFPFeatures() = FPFeatures;
   setDependence(computeDependence(this));
@@ -4481,7 +4485,7 @@ BinaryOperator::BinaryOperator(const ASTContext &Ctx, Expr *lhs, Expr *rhs,
 BinaryOperator::BinaryOperator(const ASTContext &Ctx, Expr *lhs, Expr *rhs,
                                Opcode opc, QualType ResTy, ExprValueKind VK,
                                ExprObjectKind OK, SourceLocation opLoc,
-                               FPOptions FPFeatures, bool dead2)
+                               FPOptionsOverride FPFeatures, bool dead2)
     : Expr(CompoundAssignOperatorClass, ResTy, VK, OK) {
   BinaryOperatorBits.Opc = opc;
   assert(isCompoundAssignmentOp() &&
@@ -4489,8 +4493,7 @@ BinaryOperator::BinaryOperator(const ASTContext &Ctx, Expr *lhs, Expr *rhs,
   BinaryOperatorBits.OpLoc = opLoc;
   SubExprs[LHS] = lhs;
   SubExprs[RHS] = rhs;
-  BinaryOperatorBits.HasFPFeatures =
-      FPFeatures.requiresTrailingStorage(Ctx.getLangOpts());
+  BinaryOperatorBits.HasFPFeatures = FPFeatures.requiresTrailingStorage();
   if (BinaryOperatorBits.HasFPFeatures)
     *getTrailingFPFeatures() = FPFeatures;
   setDependence(computeDependence(this));
@@ -4508,8 +4511,8 @@ BinaryOperator *BinaryOperator::Create(const ASTContext &C, Expr *lhs,
                                        Expr *rhs, Opcode opc, QualType ResTy,
                                        ExprValueKind VK, ExprObjectKind OK,
                                        SourceLocation opLoc,
-                                       FPOptions FPFeatures) {
-  bool HasFPFeatures = FPFeatures.requiresTrailingStorage(C.getLangOpts());
+                                       FPOptionsOverride FPFeatures) {
+  bool HasFPFeatures = FPFeatures.requiresTrailingStorage();
   unsigned Extra = sizeOfTrailingObjects(HasFPFeatures);
   void *Mem =
       C.Allocate(sizeof(BinaryOperator) + Extra, alignof(BinaryOperator));
@@ -4525,11 +4528,13 @@ CompoundAssignOperator::CreateEmpty(const ASTContext &C, bool HasFPFeatures) {
   return new (Mem) CompoundAssignOperator(C, EmptyShell(), HasFPFeatures);
 }
 
-CompoundAssignOperator *CompoundAssignOperator::Create(
-    const ASTContext &C, Expr *lhs, Expr *rhs, Opcode opc, QualType ResTy,
-    ExprValueKind VK, ExprObjectKind OK, SourceLocation opLoc,
-    FPOptions FPFeatures, QualType CompLHSType, QualType CompResultType) {
-  bool HasFPFeatures = FPFeatures.requiresTrailingStorage(C.getLangOpts());
+CompoundAssignOperator *
+CompoundAssignOperator::Create(const ASTContext &C, Expr *lhs, Expr *rhs,
+                               Opcode opc, QualType ResTy, ExprValueKind VK,
+                               ExprObjectKind OK, SourceLocation opLoc,
+                               FPOptionsOverride FPFeatures,
+                               QualType CompLHSType, QualType CompResultType) {
+  bool HasFPFeatures = FPFeatures.requiresTrailingStorage();
   unsigned Extra = sizeOfTrailingObjects(HasFPFeatures);
   void *Mem = C.Allocate(sizeof(CompoundAssignOperator) + Extra,
                          alignof(CompoundAssignOperator));
@@ -4540,7 +4545,7 @@ CompoundAssignOperator *CompoundAssignOperator::Create(
 
 UnaryOperator *UnaryOperator::CreateEmpty(const ASTContext &C,
                                           bool hasFPFeatures) {
-  void *Mem = C.Allocate(totalSizeToAlloc<FPOptions>(hasFPFeatures),
+  void *Mem = C.Allocate(totalSizeToAlloc<FPOptionsOverride>(hasFPFeatures),
                          alignof(UnaryOperator));
   return new (Mem) UnaryOperator(hasFPFeatures, EmptyShell());
 }
@@ -4548,13 +4553,12 @@ UnaryOperator *UnaryOperator::CreateEmpty(const ASTContext &C,
 UnaryOperator::UnaryOperator(const ASTContext &Ctx, Expr *input, Opcode opc,
                              QualType type, ExprValueKind VK, ExprObjectKind OK,
                              SourceLocation l, bool CanOverflow,
-                             FPOptions FPFeatures)
+                             FPOptionsOverride FPFeatures)
     : Expr(UnaryOperatorClass, type, VK, OK), Val(input) {
   UnaryOperatorBits.Opc = opc;
   UnaryOperatorBits.CanOverflow = CanOverflow;
   UnaryOperatorBits.Loc = l;
-  UnaryOperatorBits.HasFPFeatures =
-      FPFeatures.requiresTrailingStorage(Ctx.getLangOpts());
+  UnaryOperatorBits.HasFPFeatures = FPFeatures.requiresTrailingStorage();
   setDependence(computeDependence(this));
 }
 
@@ -4562,9 +4566,9 @@ UnaryOperator *UnaryOperator::Create(const ASTContext &C, Expr *input,
                                      Opcode opc, QualType type,
                                      ExprValueKind VK, ExprObjectKind OK,
                                      SourceLocation l, bool CanOverflow,
-                                     FPOptions FPFeatures) {
-  bool HasFPFeatures = FPFeatures.requiresTrailingStorage(C.getLangOpts());
-  unsigned Size = totalSizeToAlloc<FPOptions>(HasFPFeatures);
+                                     FPOptionsOverride FPFeatures) {
+  bool HasFPFeatures = FPFeatures.requiresTrailingStorage();
+  unsigned Size = totalSizeToAlloc<FPOptionsOverride>(HasFPFeatures);
   void *Mem = C.Allocate(Size, alignof(UnaryOperator));
   return new (Mem)
       UnaryOperator(C, input, opc, type, VK, OK, l, CanOverflow, FPFeatures);
@@ -4775,8 +4779,10 @@ QualType OMPArraySectionExpr::getBaseOriginalType(const Expr *Base) {
 
 RecoveryExpr::RecoveryExpr(ASTContext &Ctx, QualType T, SourceLocation BeginLoc,
                            SourceLocation EndLoc, ArrayRef<Expr *> SubExprs)
-    : Expr(RecoveryExprClass, T, VK_LValue, OK_Ordinary), BeginLoc(BeginLoc),
-      EndLoc(EndLoc), NumExprs(SubExprs.size()) {
+    : Expr(RecoveryExprClass, T.getNonReferenceType(),
+           T->isDependentType() ? VK_LValue : getValueKindForType(T),
+           OK_Ordinary),
+      BeginLoc(BeginLoc), EndLoc(EndLoc), NumExprs(SubExprs.size()) {
   assert(!T.isNull());
   assert(llvm::all_of(SubExprs, [](Expr* E) { return E != nullptr; }));
 
diff --git a/clang/lib/AST/ExprCXX.cpp b/clang/lib/AST/ExprCXX.cpp
index f9f825b29ed07..5d99f61c579ff 100644
--- a/clang/lib/AST/ExprCXX.cpp
+++ b/clang/lib/AST/ExprCXX.cpp
@@ -525,27 +525,27 @@ CXXOperatorCallExpr::CXXOperatorCallExpr(OverloadedOperatorKind OpKind,
                                          Expr *Fn, ArrayRef<Expr *> Args,
                                          QualType Ty, ExprValueKind VK,
                                          SourceLocation OperatorLoc,
-                                         FPOptions FPFeatures,
+                                         FPOptionsOverride FPFeatures,
                                          ADLCallKind UsesADL)
     : CallExpr(CXXOperatorCallExprClass, Fn, /*PreArgs=*/{}, Args, Ty, VK,
                OperatorLoc, /*MinNumArgs=*/0, UsesADL) {
   CXXOperatorCallExprBits.OperatorKind = OpKind;
-  CXXOperatorCallExprBits.FPFeatures = FPFeatures.getAsOpaqueInt();
   assert(
       (CXXOperatorCallExprBits.OperatorKind == static_cast<unsigned>(OpKind)) &&
       "OperatorKind overflow!");
-  assert((CXXOperatorCallExprBits.FPFeatures == FPFeatures.getAsOpaqueInt()) &&
-         "FPFeatures overflow!");
   Range = getSourceRangeImpl();
+  Overrides = FPFeatures;
 }
 
 CXXOperatorCallExpr::CXXOperatorCallExpr(unsigned NumArgs, EmptyShell Empty)
     : CallExpr(CXXOperatorCallExprClass, /*NumPreArgs=*/0, NumArgs, Empty) {}
 
-CXXOperatorCallExpr *CXXOperatorCallExpr::Create(
-    const ASTContext &Ctx, OverloadedOperatorKind OpKind, Expr *Fn,
-    ArrayRef<Expr *> Args, QualType Ty, ExprValueKind VK,
-    SourceLocation OperatorLoc, FPOptions FPFeatures, ADLCallKind UsesADL) {
+CXXOperatorCallExpr *
+CXXOperatorCallExpr::Create(const ASTContext &Ctx,
+                            OverloadedOperatorKind OpKind, Expr *Fn,
+                            ArrayRef<Expr *> Args, QualType Ty,
+                            ExprValueKind VK, SourceLocation OperatorLoc,
+                            FPOptionsOverride FPFeatures, ADLCallKind UsesADL) {
   // Allocate storage for the trailing objects of CallExpr.
   unsigned NumArgs = Args.size();
   unsigned SizeOfTrailingObjects =
@@ -1118,6 +1118,10 @@ LambdaExpr::LambdaExpr(QualType T, SourceRange IntroducerRange,
 LambdaExpr::LambdaExpr(EmptyShell Empty, unsigned NumCaptures)
     : Expr(LambdaExprClass, Empty) {
   LambdaExprBits.NumCaptures = NumCaptures;
+
+  // Initially don't initialize the body of the LambdaExpr. The body will
+  // be lazily deserialized when needed.
+  getStoredStmts()[NumCaptures] = nullptr; // Not one past the end.
 }
 
 LambdaExpr *LambdaExpr::Create(const ASTContext &Context, CXXRecordDecl *Class,
@@ -1147,6 +1151,25 @@ LambdaExpr *LambdaExpr::CreateDeserialized(const ASTContext &C,
   return new (Mem) LambdaExpr(EmptyShell(), NumCaptures);
 }
 
+void LambdaExpr::initBodyIfNeeded() const {
+  if (!getStoredStmts()[capture_size()]) {
+    auto *This = const_cast<LambdaExpr *>(this);
+    This->getStoredStmts()[capture_size()] = getCallOperator()->getBody();
+  }
+}
+
+Stmt *LambdaExpr::getBody() const {
+  initBodyIfNeeded();
+  return getStoredStmts()[capture_size()];
+}
+
+const CompoundStmt *LambdaExpr::getCompoundStmtBody() const {
+  Stmt *Body = getBody();
+  if (const auto *CoroBody = dyn_cast<CoroutineBodyStmt>(Body))
+    return cast<CompoundStmt>(CoroBody->getBody());
+  return cast<CompoundStmt>(Body);
+}
+
 bool LambdaExpr::isInitCapture(const LambdaCapture *C) const {
   return (C->capturesVariable() && C->getCapturedVar()->isInitCapture() &&
           (getCallOperator() == C->getCapturedVar()->getDeclContext()));
@@ -1216,6 +1239,17 @@ ArrayRef<NamedDecl *> LambdaExpr::getExplicitTemplateParameters() const {
 
 bool LambdaExpr::isMutable() const { return !getCallOperator()->isConst(); }
 
+LambdaExpr::child_range LambdaExpr::children() {
+  initBodyIfNeeded();
+  return child_range(getStoredStmts(), getStoredStmts() + capture_size() + 1);
+}
+
+LambdaExpr::const_child_range LambdaExpr::children() const {
+  initBodyIfNeeded();
+  return const_child_range(getStoredStmts(),
+                           getStoredStmts() + capture_size() + 1);
+}
+
 ExprWithCleanups::ExprWithCleanups(Expr *subexpr,
                                    bool CleanupsHaveSideEffects,
                                    ArrayRef<CleanupObject> objects)
diff --git a/clang/lib/AST/ExprClassification.cpp b/clang/lib/AST/ExprClassification.cpp
index 42873d090ffa6..31aa734ffedba 100644
--- a/clang/lib/AST/ExprClassification.cpp
+++ b/clang/lib/AST/ExprClassification.cpp
@@ -130,7 +130,6 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
   case Expr::UnresolvedLookupExprClass:
   case Expr::UnresolvedMemberExprClass:
   case Expr::TypoExprClass:
-  case Expr::RecoveryExprClass:
   case Expr::DependentCoawaitExprClass:
   case Expr::CXXDependentScopeMemberExprClass:
   case Expr::DependentScopeDeclRefExprClass:
@@ -276,6 +275,7 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
       return Cl::CL_PRValue;
     }
 
+  case Expr::RecoveryExprClass:
   case Expr::OpaqueValueExprClass:
     return ClassifyExprValueKind(Lang, E, E->getValueKind());
 
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 640bdd0d45e53..a4dc0ccad1e0f 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -2664,6 +2664,155 @@ static bool handleFloatFloatBinOp(EvalInfo &Info, const Expr *E,
   return true;
 }
 
+static bool handleLogicalOpForVector(const APInt &LHSValue,
+                                     BinaryOperatorKind Opcode,
+                                     const APInt &RHSValue, APInt &Result) {
+  bool LHS = (LHSValue != 0);
+  bool RHS = (RHSValue != 0);
+
+  if (Opcode == BO_LAnd)
+    Result = LHS && RHS;
+  else
+    Result = LHS || RHS;
+  return true;
+}
+static bool handleLogicalOpForVector(const APFloat &LHSValue,
+                                     BinaryOperatorKind Opcode,
+                                     const APFloat &RHSValue, APInt &Result) {
+  bool LHS = !LHSValue.isZero();
+  bool RHS = !RHSValue.isZero();
+
+  if (Opcode == BO_LAnd)
+    Result = LHS && RHS;
+  else
+    Result = LHS || RHS;
+  return true;
+}
+
+static bool handleLogicalOpForVector(const APValue &LHSValue,
+                                     BinaryOperatorKind Opcode,
+                                     const APValue &RHSValue, APInt &Result) {
+  // The result is always an int type, however operands match the first.
+  if (LHSValue.getKind() == APValue::Int)
+    return handleLogicalOpForVector(LHSValue.getInt(), Opcode,
+                                    RHSValue.getInt(), Result);
+  assert(LHSValue.getKind() == APValue::Float && "Should be no other options");
+  return handleLogicalOpForVector(LHSValue.getFloat(), Opcode,
+                                  RHSValue.getFloat(), Result);
+}
+
+template <typename APTy>
+static bool
+handleCompareOpForVectorHelper(const APTy &LHSValue, BinaryOperatorKind Opcode,
+                               const APTy &RHSValue, APInt &Result) {
+  switch (Opcode) {
+  default:
+    llvm_unreachable("unsupported binary operator");
+  case BO_EQ:
+    Result = (LHSValue == RHSValue);
+    break;
+  case BO_NE:
+    Result = (LHSValue != RHSValue);
+    break;
+  case BO_LT:
+    Result = (LHSValue < RHSValue);
+    break;
+  case BO_GT:
+    Result = (LHSValue > RHSValue);
+    break;
+  case BO_LE:
+    Result = (LHSValue <= RHSValue);
+    break;
+  case BO_GE:
+    Result = (LHSValue >= RHSValue);
+    break;
+  }
+
+  return true;
+}
+
+static bool handleCompareOpForVector(const APValue &LHSValue,
+                                     BinaryOperatorKind Opcode,
+                                     const APValue &RHSValue, APInt &Result) {
+  // The result is always an int type, however operands match the first.
+  if (LHSValue.getKind() == APValue::Int)
+    return handleCompareOpForVectorHelper(LHSValue.getInt(), Opcode,
+                                          RHSValue.getInt(), Result);
+  assert(LHSValue.getKind() == APValue::Float && "Should be no other options");
+  return handleCompareOpForVectorHelper(LHSValue.getFloat(), Opcode,
+                                        RHSValue.getFloat(), Result);
+}
+
+// Perform binary operations for vector types, in place on the LHS.
+static bool handleVectorVectorBinOp(EvalInfo &Info, const Expr *E,
+                                    BinaryOperatorKind Opcode,
+                                    APValue &LHSValue,
+                                    const APValue &RHSValue) {
+  assert(Opcode != BO_PtrMemD && Opcode != BO_PtrMemI &&
+         "Operation not supported on vector types");
+
+  const auto *VT = E->getType()->castAs<VectorType>();
+  unsigned NumElements = VT->getNumElements();
+  QualType EltTy = VT->getElementType();
+
+  // In the cases (typically C as I've observed) where we aren't evaluating
+  // constexpr but are checking for cases where the LHS isn't yet evaluatable,
+  // just give up.
+  if (!LHSValue.isVector()) {
+    assert(LHSValue.isLValue() &&
+           "A vector result that isn't a vector OR uncalculated LValue");
+    Info.FFDiag(E);
+    return false;
+  }
+
+  assert(LHSValue.getVectorLength() == NumElements &&
+         RHSValue.getVectorLength() == NumElements && "Different vector sizes");
+
+  SmallVector<APValue, 4> ResultElements;
+
+  for (unsigned EltNum = 0; EltNum < NumElements; ++EltNum) {
+    APValue LHSElt = LHSValue.getVectorElt(EltNum);
+    APValue RHSElt = RHSValue.getVectorElt(EltNum);
+
+    if (EltTy->isIntegerType()) {
+      APSInt EltResult{Info.Ctx.getIntWidth(EltTy),
+                       EltTy->isUnsignedIntegerType()};
+      bool Success = true;
+
+      if (BinaryOperator::isLogicalOp(Opcode))
+        Success = handleLogicalOpForVector(LHSElt, Opcode, RHSElt, EltResult);
+      else if (BinaryOperator::isComparisonOp(Opcode))
+        Success = handleCompareOpForVector(LHSElt, Opcode, RHSElt, EltResult);
+      else
+        Success = handleIntIntBinOp(Info, E, LHSElt.getInt(), Opcode,
+                                    RHSElt.getInt(), EltResult);
+
+      if (!Success) {
+        Info.FFDiag(E);
+        return false;
+      }
+      ResultElements.emplace_back(EltResult);
+
+    } else if (EltTy->isFloatingType()) {
+      assert(LHSElt.getKind() == APValue::Float &&
+             RHSElt.getKind() == APValue::Float &&
+             "Mismatched LHS/RHS/Result Type");
+      APFloat LHSFloat = LHSElt.getFloat();
+
+      if (!handleFloatFloatBinOp(Info, E, LHSFloat, Opcode,
+                                 RHSElt.getFloat())) {
+        Info.FFDiag(E);
+        return false;
+      }
+
+      ResultElements.emplace_back(LHSFloat);
+    }
+  }
+
+  LHSValue = APValue(ResultElements.data(), ResultElements.size());
+  return true;
+}
+
 /// Cast an lvalue referring to a base subobject to a derived class, by
 /// truncating the lvalue's path to the given length.
 static bool CastToDerivedClass(EvalInfo &Info, const Expr *E, LValue &Result,
@@ -2876,7 +3025,7 @@ static bool evaluateVarDeclInit(EvalInfo &Info, const Expr *E,
     if (Info.checkingPotentialConstantExpression())
       return false;
     if (!Frame || !Frame->Arguments) {
-      Info.FFDiag(E, diag::note_invalid_subexpr_in_const_expr);
+      Info.FFDiag(E, diag::note_constexpr_function_param_value_unknown) << VD;
       return false;
     }
     Result = &Frame->Arguments[PVD->getFunctionScopeIndex()];
@@ -2907,12 +3056,34 @@ static bool evaluateVarDeclInit(EvalInfo &Info, const Expr *E,
   }
 
   // Dig out the initializer, and use the declaration which it's attached to.
+  // FIXME: We should eventually check whether the variable has a reachable
+  // initializing declaration.
   const Expr *Init = VD->getAnyInitializer(VD);
-  if (!Init || Init->isValueDependent()) {
-    // If we're checking a potential constant expression, the variable could be
-    // initialized later.
-    if (!Info.checkingPotentialConstantExpression())
-      Info.FFDiag(E, diag::note_invalid_subexpr_in_const_expr);
+  if (!Init) {
+    // Don't diagnose during potential constant expression checking; an
+    // initializer might be added later.
+    if (!Info.checkingPotentialConstantExpression()) {
+      Info.FFDiag(E, diag::note_constexpr_var_init_unknown, 1)
+        << VD;
+      Info.Note(VD->getLocation(), diag::note_declared_at);
+    }
+    return false;
+  }
+
+  if (Init->isValueDependent()) {
+    // The DeclRefExpr is not value-dependent, but the variable it refers to
+    // has a value-dependent initializer. This should only happen in
+    // constant-folding cases, where the variable is not actually of a suitable
+    // type for use in a constant expression (otherwise the DeclRefExpr would
+    // have been value-dependent too), so diagnose that.
+    assert(!VD->mightBeUsableInConstantExpressions(Info.Ctx));
+    if (!Info.checkingPotentialConstantExpression()) {
+      Info.FFDiag(E, Info.getLangOpts().CPlusPlus11
+                         ? diag::note_constexpr_ltor_non_constexpr
+                         : diag::note_constexpr_ltor_non_integral, 1)
+          << VD << VD->getType();
+      Info.Note(VD->getLocation(), diag::note_declared_at);
+    }
     return false;
   }
 
@@ -2923,13 +3094,6 @@ static bool evaluateVarDeclInit(EvalInfo &Info, const Expr *E,
     return true;
   }
 
-  // Never evaluate the initializer of a weak variable. We can't be sure that
-  // this is the definition which will be used.
-  if (VD->isWeak()) {
-    Info.FFDiag(E, diag::note_invalid_subexpr_in_const_expr);
-    return false;
-  }
-
   // Check that we can fold the initializer. In C++, we will have already done
   // this in the cases where it matters for conformance.
   SmallVector<PartialDiagnosticAt, 8> Notes;
@@ -2939,13 +3103,24 @@ static bool evaluateVarDeclInit(EvalInfo &Info, const Expr *E,
     Info.Note(VD->getLocation(), diag::note_declared_at);
     Info.addNotes(Notes);
     return false;
-  } else if (!VD->checkInitIsICE()) {
+  }
+
+  // Check that the variable is actually usable in constant expressions.
+  if (!VD->checkInitIsICE()) {
     Info.CCEDiag(E, diag::note_constexpr_var_init_non_constant,
                  Notes.size() + 1) << VD;
     Info.Note(VD->getLocation(), diag::note_declared_at);
     Info.addNotes(Notes);
   }
 
+  // Never use the initializer of a weak variable, not even for constant
+  // folding. We can't be sure that this is the definition that will be used.
+  if (VD->isWeak()) {
+    Info.FFDiag(E, diag::note_constexpr_var_init_weak) << VD;
+    Info.Note(VD->getLocation(), diag::note_declared_at);
+    return false;
+  }
+
   Result = VD->getEvaluatedValue();
   return true;
 }
@@ -3648,6 +3823,11 @@ static CompleteObject findCompleteObject(EvalInfo &Info, const Expr *E,
       return CompleteObject();
     }
 
+    // In OpenCL if a variable is in constant address space it is a const value.
+    bool IsConstant = BaseType.isConstQualified() ||
+                      (Info.getLangOpts().OpenCL &&
+                       BaseType.getAddressSpace() == LangAS::opencl_constant);
+
     // Unless we're looking at a local variable or argument in a constexpr call,
     // the variable we're reading must be const.
     if (!Frame) {
@@ -3665,9 +3845,7 @@ static CompleteObject findCompleteObject(EvalInfo &Info, const Expr *E,
       } else if (BaseType->isIntegralOrEnumerationType()) {
         // In OpenCL if a variable is in constant address space it is a const
         // value.
-        if (!(BaseType.isConstQualified() ||
-              (Info.getLangOpts().OpenCL &&
-               BaseType.getAddressSpace() == LangAS::opencl_constant))) {
+        if (!IsConstant) {
           if (!IsAccess)
             return CompleteObject(LVal.getLValueBase(), nullptr, BaseType);
           if (Info.getLangOpts().CPlusPlus) {
@@ -3680,27 +3858,29 @@ static CompleteObject findCompleteObject(EvalInfo &Info, const Expr *E,
         }
       } else if (!IsAccess) {
         return CompleteObject(LVal.getLValueBase(), nullptr, BaseType);
-      } else if (BaseType->isFloatingType() && BaseType.isConstQualified()) {
-        // We support folding of const floating-point types, in order to make
-        // static const data members of such types (supported as an extension)
-        // more useful.
-        if (Info.getLangOpts().CPlusPlus11) {
-          Info.CCEDiag(E, diag::note_constexpr_ltor_non_constexpr, 1) << VD;
+      } else if (IsConstant && Info.checkingPotentialConstantExpression() &&
+                 BaseType->isLiteralType(Info.Ctx) && !VD->hasDefinition()) {
+        // This variable might end up being constexpr. Don't diagnose it yet.
+      } else if (IsConstant) {
+        // Keep evaluating to see what we can do. In particular, we support
+        // folding of const floating-point types, in order to make static const
+        // data members of such types (supported as an extension) more useful.
+        if (Info.getLangOpts().CPlusPlus) {
+          Info.CCEDiag(E, Info.getLangOpts().CPlusPlus11
+                              ? diag::note_constexpr_ltor_non_constexpr
+                              : diag::note_constexpr_ltor_non_integral, 1)
+              << VD << BaseType;
           Info.Note(VD->getLocation(), diag::note_declared_at);
         } else {
           Info.CCEDiag(E);
         }
-      } else if (BaseType.isConstQualified() && VD->hasDefinition(Info.Ctx)) {
-        Info.CCEDiag(E, diag::note_constexpr_ltor_non_constexpr) << VD;
-        // Keep evaluating to see what we can do.
       } else {
-        // FIXME: Allow folding of values of any literal type in all languages.
-        if (Info.checkingPotentialConstantExpression() &&
-            VD->getType().isConstQualified() && !VD->hasDefinition(Info.Ctx)) {
-          // The definition of this variable could be constexpr. We can't
-          // access it right now, but may be able to in future.
-        } else if (Info.getLangOpts().CPlusPlus11) {
-          Info.FFDiag(E, diag::note_constexpr_ltor_non_constexpr, 1) << VD;
+        // Never allow reading a non-const value.
+        if (Info.getLangOpts().CPlusPlus) {
+          Info.FFDiag(E, Info.getLangOpts().CPlusPlus11
+                             ? diag::note_constexpr_ltor_non_constexpr
+                             : diag::note_constexpr_ltor_non_integral, 1)
+              << VD << BaseType;
           Info.Note(VD->getLocation(), diag::note_declared_at);
         } else {
           Info.FFDiag(E);
@@ -3910,12 +4090,26 @@ struct CompoundAssignSubobjectHandler {
       return false;
     case APValue::LValue:
       return foundPointer(Subobj, SubobjType);
+    case APValue::Vector:
+      return foundVector(Subobj, SubobjType);
     default:
       // FIXME: can this happen?
       Info.FFDiag(E);
       return false;
     }
   }
+
+  bool foundVector(APValue &Value, QualType SubobjType) {
+    if (!checkConst(SubobjType))
+      return false;
+
+    if (!SubobjType->isVectorType()) {
+      Info.FFDiag(E);
+      return false;
+    }
+    return handleVectorVectorBinOp(Info, E, Opcode, Value, RHS);
+  }
+
   bool found(APSInt &Value, QualType SubobjType) {
     if (!checkConst(SubobjType))
       return false;
@@ -4312,37 +4506,48 @@ static bool HandleBaseToDerivedCast(EvalInfo &Info, const CastExpr *E,
 }
 
 /// Get the value to use for a default-initialized object of type T.
-static APValue getDefaultInitValue(QualType T) {
+/// Return false if it encounters something invalid.
+static bool getDefaultInitValue(QualType T, APValue &Result) {
+  bool Success = true;
   if (auto *RD = T->getAsCXXRecordDecl()) {
-    if (RD->isUnion())
-      return APValue((const FieldDecl*)nullptr);
-
-    APValue Struct(APValue::UninitStruct(), RD->getNumBases(),
-                   std::distance(RD->field_begin(), RD->field_end()));
+    if (RD->isInvalidDecl()) {
+      Result = APValue();
+      return false;
+    }
+    if (RD->isUnion()) {
+      Result = APValue((const FieldDecl *)nullptr);
+      return true;
+    }
+    Result = APValue(APValue::UninitStruct(), RD->getNumBases(),
+                     std::distance(RD->field_begin(), RD->field_end()));
 
     unsigned Index = 0;
     for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
-           End = RD->bases_end(); I != End; ++I, ++Index)
-      Struct.getStructBase(Index) = getDefaultInitValue(I->getType());
+                                                  End = RD->bases_end();
+         I != End; ++I, ++Index)
+      Success &= getDefaultInitValue(I->getType(), Result.getStructBase(Index));
 
     for (const auto *I : RD->fields()) {
       if (I->isUnnamedBitfield())
         continue;
-      Struct.getStructField(I->getFieldIndex()) =
-          getDefaultInitValue(I->getType());
+      Success &= getDefaultInitValue(I->getType(),
+                                     Result.getStructField(I->getFieldIndex()));
     }
-    return Struct;
+    return Success;
   }
 
   if (auto *AT =
           dyn_cast_or_null<ConstantArrayType>(T->getAsArrayTypeUnsafe())) {
-    APValue Array(APValue::UninitArray(), 0, AT->getSize().getZExtValue());
-    if (Array.hasArrayFiller())
-      Array.getArrayFiller() = getDefaultInitValue(AT->getElementType());
-    return Array;
+    Result = APValue(APValue::UninitArray(), 0, AT->getSize().getZExtValue());
+    if (Result.hasArrayFiller())
+      Success &=
+          getDefaultInitValue(AT->getElementType(), Result.getArrayFiller());
+
+    return Success;
   }
 
-  return APValue::IndeterminateValue();
+  Result = APValue::IndeterminateValue();
+  return true;
 }
 
 namespace {
@@ -4372,10 +4577,8 @@ static bool EvaluateVarDecl(EvalInfo &Info, const VarDecl *VD) {
       Info.CurrentCall->createTemporary(VD, VD->getType(), true, Result);
 
   const Expr *InitE = VD->getInit();
-  if (!InitE) {
-    Val = getDefaultInitValue(VD->getType());
-    return true;
-  }
+  if (!InitE)
+    return getDefaultInitValue(VD->getType(), Val);
 
   if (InitE->isValueDependent())
     return false;
@@ -5372,11 +5575,11 @@ struct StartLifetimeOfUnionMemberHandler {
   const Expr *LHSExpr;
   const FieldDecl *Field;
   bool DuringInit;
-
+  bool Failed = false;
   static const AccessKinds AccessKind = AK_Assign;
 
   typedef bool result_type;
-  bool failed() { return false; }
+  bool failed() { return Failed; }
   bool found(APValue &Subobj, QualType SubobjType) {
     // We are supposed to perform no initialization but begin the lifetime of
     // the object. We interpret that as meaning to do what default
@@ -5400,8 +5603,9 @@ struct StartLifetimeOfUnionMemberHandler {
                   diag::note_constexpr_union_member_change_during_init);
       return false;
     }
-
-    Subobj.setUnion(Field, getDefaultInitValue(Field->getType()));
+    APValue Result;
+    Failed = !getDefaultInitValue(Field->getType(), Result);
+    Subobj.setUnion(Field, Result);
     return true;
   }
   bool found(APSInt &Value, QualType SubobjType) {
@@ -5717,8 +5921,9 @@ static bool HandleConstructorCall(const Expr *E, const LValue &This,
     for (; !declaresSameEntity(*FieldIt, FD); ++FieldIt) {
       assert(FieldIt != RD->field_end() && "missing field?");
       if (!FieldIt->isUnnamedBitfield())
-        Result.getStructField(FieldIt->getFieldIndex()) =
-            getDefaultInitValue(FieldIt->getType());
+        Success &= getDefaultInitValue(
+            FieldIt->getType(),
+            Result.getStructField(FieldIt->getFieldIndex()));
     }
     ++FieldIt;
   };
@@ -5770,10 +5975,10 @@ static bool HandleConstructorCall(const Expr *E, const LValue &This,
           if (CD->isUnion())
             *Value = APValue(FD);
           else
-            // FIXME: This immediately starts the lifetime of all members of an
-            // anonymous struct. It would be preferable to strictly start member
-            // lifetime in initialization order.
-            *Value = getDefaultInitValue(Info.Ctx.getRecordType(CD));
+            // FIXME: This immediately starts the lifetime of all members of
+            // an anonymous struct. It would be preferable to strictly start
+            // member lifetime in initialization order.
+            Success &= getDefaultInitValue(Info.Ctx.getRecordType(CD), *Value);
         }
         // Store Subobject as its parent before updating it for the last element
         // in the chain.
@@ -5820,8 +6025,9 @@ static bool HandleConstructorCall(const Expr *E, const LValue &This,
   if (!RD->isUnion()) {
     for (; FieldIt != RD->field_end(); ++FieldIt) {
       if (!FieldIt->isUnnamedBitfield())
-        Result.getStructField(FieldIt->getFieldIndex()) =
-            getDefaultInitValue(FieldIt->getType());
+        Success &= getDefaultInitValue(
+            FieldIt->getType(),
+            Result.getStructField(FieldIt->getFieldIndex()));
     }
   }
 
@@ -8907,8 +9113,8 @@ bool PointerExprEvaluator::VisitCXXNewExpr(const CXXNewExpr *E) {
   } else if (Init) {
     if (!EvaluateInPlace(*Val, Info, Result, Init))
       return false;
-  } else {
-    *Val = getDefaultInitValue(AllocType);
+  } else if (!getDefaultInitValue(AllocType, *Val)) {
+    return false;
   }
 
   // Array new returns a pointer to the first element, not a pointer to the
@@ -9279,8 +9485,7 @@ bool RecordExprEvaluator::VisitCXXConstructExpr(const CXXConstructExpr *E,
     if (ZeroInit)
       return ZeroInitialization(E, T);
 
-    Result = getDefaultInitValue(T);
-    return true;
+    return getDefaultInitValue(T, Result);
   }
 
   const FunctionDecl *Definition = nullptr;
@@ -9516,10 +9721,9 @@ namespace {
     bool VisitCastExpr(const CastExpr* E);
     bool VisitInitListExpr(const InitListExpr *E);
     bool VisitUnaryImag(const UnaryOperator *E);
-    // FIXME: Missing: unary -, unary ~, binary add/sub/mul/div,
-    //                 binary comparisons, binary and/or/xor,
-    //                 conditional operator (for GNU conditional select),
-    //                 shufflevector, ExtVectorElementExpr
+    bool VisitBinaryOperator(const BinaryOperator *E);
+    // FIXME: Missing: unary -, unary ~, conditional operator (for GNU
+    //                 conditional select), shufflevector, ExtVectorElementExpr
   };
 } // end anonymous namespace
 
@@ -9667,6 +9871,41 @@ bool VectorExprEvaluator::VisitUnaryImag(const UnaryOperator *E) {
   return ZeroInitialization(E);
 }
 
+bool VectorExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
+  BinaryOperatorKind Op = E->getOpcode();
+  assert(Op != BO_PtrMemD && Op != BO_PtrMemI && Op != BO_Cmp &&
+         "Operation not supported on vector types");
+
+  if (Op == BO_Comma)
+    return ExprEvaluatorBaseTy::VisitBinaryOperator(E);
+
+  Expr *LHS = E->getLHS();
+  Expr *RHS = E->getRHS();
+
+  assert(LHS->getType()->isVectorType() && RHS->getType()->isVectorType() &&
+         "Must both be vector types");
+  // Checking JUST the types are the same would be fine, except shifts don't
+  // need to have their types be the same (since you always shift by an int).
+  assert(LHS->getType()->getAs<VectorType>()->getNumElements() ==
+             E->getType()->getAs<VectorType>()->getNumElements() &&
+         RHS->getType()->getAs<VectorType>()->getNumElements() ==
+             E->getType()->getAs<VectorType>()->getNumElements() &&
+         "All operands must be the same size.");
+
+  APValue LHSValue;
+  APValue RHSValue;
+  bool LHSOK = Evaluate(LHSValue, Info, LHS);
+  if (!LHSOK && !Info.noteFailure())
+    return false;
+  if (!Evaluate(RHSValue, Info, RHS) || !LHSOK)
+    return false;
+
+  if (!handleVectorVectorBinOp(Info, E, Op, LHSValue, RHSValue))
+    return false;
+
+  return Success(LHSValue, E);
+}
+
 //===----------------------------------------------------------------------===//
 // Array Evaluation
 //===----------------------------------------------------------------------===//
@@ -10992,6 +11231,7 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
   }
 
   case Builtin::BI__builtin_expect:
+  case Builtin::BI__builtin_expect_with_probability:
     return Visit(E->getArg(0));
 
   case Builtin::BI__builtin_ffs:
@@ -12673,8 +12913,14 @@ bool FixedPointExprEvaluator::VisitCastExpr(const CastExpr *E) {
       return false;
     bool Overflowed;
     APFixedPoint Result = Src.convert(DestFXSema, &Overflowed);
-    if (Overflowed && !HandleOverflow(Info, E, Result, DestType))
-      return false;
+    if (Overflowed) {
+      if (Info.checkingForUndefinedBehavior())
+        Info.Ctx.getDiagnostics().Report(E->getExprLoc(),
+                                         diag::warn_fixedpoint_constant_overflow)
+          << Result.toString() << E->getType();
+      else if (!HandleOverflow(Info, E, Result, E->getType()))
+        return false;
+    }
     return Success(Result, E);
   }
   case CK_IntegralToFixedPoint: {
@@ -12686,8 +12932,14 @@ bool FixedPointExprEvaluator::VisitCastExpr(const CastExpr *E) {
     APFixedPoint IntResult = APFixedPoint::getFromIntValue(
         Src, Info.Ctx.getFixedPointSemantics(DestType), &Overflowed);
 
-    if (Overflowed && !HandleOverflow(Info, E, IntResult, DestType))
-      return false;
+    if (Overflowed) {
+      if (Info.checkingForUndefinedBehavior())
+        Info.Ctx.getDiagnostics().Report(E->getExprLoc(),
+                                         diag::warn_fixedpoint_constant_overflow)
+          << IntResult.toString() << E->getType();
+      else if (!HandleOverflow(Info, E, IntResult, E->getType()))
+        return false;
+    }
 
     return Success(IntResult, E);
   }
@@ -12700,6 +12952,9 @@ bool FixedPointExprEvaluator::VisitCastExpr(const CastExpr *E) {
 }
 
 bool FixedPointExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
+  if (E->isPtrMemOp() || E->isAssignmentOp() || E->getOpcode() == BO_Comma)
+    return ExprEvaluatorBaseTy::VisitBinaryOperator(E);
+
   const Expr *LHS = E->getLHS();
   const Expr *RHS = E->getRHS();
   FixedPointSemantics ResultFXSema =
@@ -12712,20 +12967,45 @@ bool FixedPointExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
   if (!EvaluateFixedPointOrInteger(RHS, RHSFX, Info))
     return false;
 
+  bool OpOverflow = false, ConversionOverflow = false;
+  APFixedPoint Result(LHSFX.getSemantics());
   switch (E->getOpcode()) {
   case BO_Add: {
-    bool AddOverflow, ConversionOverflow;
-    APFixedPoint Result = LHSFX.add(RHSFX, &AddOverflow)
-                              .convert(ResultFXSema, &ConversionOverflow);
-    if ((AddOverflow || ConversionOverflow) &&
-        !HandleOverflow(Info, E, Result, E->getType()))
+    Result = LHSFX.add(RHSFX, &OpOverflow)
+                  .convert(ResultFXSema, &ConversionOverflow);
+    break;
+  }
+  case BO_Sub: {
+    Result = LHSFX.sub(RHSFX, &OpOverflow)
+                  .convert(ResultFXSema, &ConversionOverflow);
+    break;
+  }
+  case BO_Mul: {
+    Result = LHSFX.mul(RHSFX, &OpOverflow)
+                  .convert(ResultFXSema, &ConversionOverflow);
+    break;
+  }
+  case BO_Div: {
+    if (RHSFX.getValue() == 0) {
+      Info.FFDiag(E, diag::note_expr_divide_by_zero);
       return false;
-    return Success(Result, E);
+    }
+    Result = LHSFX.div(RHSFX, &OpOverflow)
+                  .convert(ResultFXSema, &ConversionOverflow);
+    break;
   }
   default:
     return false;
   }
-  llvm_unreachable("Should've exited before this");
+  if (OpOverflow || ConversionOverflow) {
+    if (Info.checkingForUndefinedBehavior())
+      Info.Ctx.getDiagnostics().Report(E->getExprLoc(),
+                                       diag::warn_fixedpoint_constant_overflow)
+        << Result.toString() << E->getType();
+    else if (!HandleOverflow(Info, E, Result, E->getType()))
+      return false;
+  }
+  return Success(Result, E);
 }
 
 //===----------------------------------------------------------------------===//
@@ -14012,10 +14292,11 @@ bool VarDecl::evaluateDestruction(
   // Make a copy of the value for the destructor to mutate, if we know it.
   // Otherwise, treat the value as default-initialized; if the destructor works
   // anyway, then the destruction is constant (and must be essentially empty).
-  APValue DestroyedValue =
-      (getEvaluatedValue() && !getEvaluatedValue()->isAbsent())
-          ? *getEvaluatedValue()
-          : getDefaultInitValue(getType());
+  APValue DestroyedValue;
+  if (getEvaluatedValue() && !getEvaluatedValue()->isAbsent())
+    DestroyedValue = *getEvaluatedValue();
+  else if (!getDefaultInitValue(getType(), DestroyedValue))
+    return false;
 
   EvalInfo Info(getASTContext(), EStatus, EvalInfo::EM_ConstantExpression);
   Info.setEvaluatingDecl(this, DestroyedValue,
diff --git a/clang/lib/AST/FormatString.cpp b/clang/lib/AST/FormatString.cpp
index e9f6b88631af5..83b952116a5e0 100644
--- a/clang/lib/AST/FormatString.cpp
+++ b/clang/lib/AST/FormatString.cpp
@@ -419,7 +419,6 @@ ArgType::matchesType(ASTContext &C, QualType argTy) const {
       QualType pointeeTy = PT->getPointeeType();
       if (const BuiltinType *BT = pointeeTy->getAs<BuiltinType>())
         switch (BT->getKind()) {
-          case BuiltinType::Void:
           case BuiltinType::Char_U:
           case BuiltinType::UChar:
           case BuiltinType::Char_S:
diff --git a/clang/lib/AST/JSONNodeDumper.cpp b/clang/lib/AST/JSONNodeDumper.cpp
index c9a84a2b7d57d..4bd00ece86ab8 100644
--- a/clang/lib/AST/JSONNodeDumper.cpp
+++ b/clang/lib/AST/JSONNodeDumper.cpp
@@ -183,6 +183,13 @@ void JSONNodeDumper::Visit(const GenericSelectionExpr::ConstAssociation &A) {
   attributeOnlyIfTrue("selected", A.isSelected());
 }
 
+void JSONNodeDumper::Visit(const APValue &Value, QualType Ty) {
+  std::string Str;
+  llvm::raw_string_ostream OS(Str);
+  Value.printPretty(OS, Ctx, Ty);
+  JOS.attribute("value", OS.str());
+}
+
 void JSONNodeDumper::writeIncludeStack(PresumedLoc Loc, bool JustFirst) {
   if (Loc.isInvalid())
     return;
@@ -1272,12 +1279,8 @@ void JSONNodeDumper::VisitCXXTypeidExpr(const CXXTypeidExpr *CTE) {
 }
 
 void JSONNodeDumper::VisitConstantExpr(const ConstantExpr *CE) {
-  if (CE->getResultAPValueKind() != APValue::None) {
-    std::string Str;
-    llvm::raw_string_ostream OS(Str);
-    CE->getAPValueResult().printPretty(OS, Ctx, CE->getType());
-    JOS.attribute("value", OS.str());
-  }
+  if (CE->getResultAPValueKind() != APValue::None)
+    Visit(CE->getAPValueResult(), CE->getType());
 }
 
 void JSONNodeDumper::VisitInitListExpr(const InitListExpr *ILE) {
diff --git a/clang/lib/AST/Mangle.cpp b/clang/lib/AST/Mangle.cpp
index 53f1a48d69a5e..a732325006c66 100644
--- a/clang/lib/AST/Mangle.cpp
+++ b/clang/lib/AST/Mangle.cpp
@@ -25,6 +25,7 @@
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Mangler.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
diff --git a/clang/lib/AST/NestedNameSpecifier.cpp b/clang/lib/AST/NestedNameSpecifier.cpp
index 87bf4e122ec84..08e8819a4d69d 100644
--- a/clang/lib/AST/NestedNameSpecifier.cpp
+++ b/clang/lib/AST/NestedNameSpecifier.cpp
@@ -243,6 +243,10 @@ bool NestedNameSpecifier::containsUnexpandedParameterPack() const {
   return getDependence() & NestedNameSpecifierDependence::UnexpandedPack;
 }
 
+bool NestedNameSpecifier::containsErrors() const {
+  return getDependence() & NestedNameSpecifierDependence::Error;
+}
+
 /// Print this nested name specifier to the given output
 /// stream.
 void NestedNameSpecifier::print(raw_ostream &OS, const PrintingPolicy &Policy,
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 6a8b3ce231f2f..a0b0dca553907 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -51,6 +51,8 @@ OMPClause::child_range OMPClause::used_children() {
   case OMPC_match:
   case OMPC_unknown:
     break;
+  default:
+    break;
   }
   llvm_unreachable("unknown OMPClause");
 }
@@ -154,6 +156,8 @@ const OMPClauseWithPreInit *OMPClauseWithPreInit::get(const OMPClause *C) {
   case OMPC_uses_allocators:
   case OMPC_affinity:
     break;
+  default:
+    break;
   }
 
   return nullptr;
@@ -246,6 +250,8 @@ const OMPClauseWithPostUpdate *OMPClauseWithPostUpdate::get(const OMPClause *C)
   case OMPC_uses_allocators:
   case OMPC_affinity:
     break;
+  default:
+    break;
   }
 
   return nullptr;
diff --git a/clang/lib/AST/RecordLayoutBuilder.cpp b/clang/lib/AST/RecordLayoutBuilder.cpp
index 028e82a5df4d7..d56c7e2ab8c0e 100644
--- a/clang/lib/AST/RecordLayoutBuilder.cpp
+++ b/clang/lib/AST/RecordLayoutBuilder.cpp
@@ -15,6 +15,7 @@
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/VTableBuilder.h"
 #include "clang/Basic/TargetInfo.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/Support/Format.h"
@@ -1186,11 +1187,10 @@ ItaniumRecordLayoutBuilder::LayoutBase(const BaseSubobjectInfo *Base) {
   // Query the external layout to see if it provides an offset.
   bool HasExternalLayout = false;
   if (UseExternalLayout) {
-    // FIXME: This appears to be reversed.
     if (Base->IsVirtual)
-      HasExternalLayout = External.getExternalNVBaseOffset(Base->Class, Offset);
-    else
       HasExternalLayout = External.getExternalVBaseOffset(Base->Class, Offset);
+    else
+      HasExternalLayout = External.getExternalNVBaseOffset(Base->Class, Offset);
   }
 
   // Clang <= 6 incorrectly applied the 'packed' attribute to base classes.
@@ -2107,7 +2107,7 @@ static const CXXMethodDecl *computeKeyFunction(ASTContext &Context,
     if (MD->isImplicit())
       continue;
 
-    if (MD->isInlineSpecified())
+    if (MD->isInlineSpecified() || MD->isConstexpr())
       continue;
 
     if (MD->hasInlineBody())
@@ -2568,9 +2568,11 @@ MicrosoftRecordLayoutBuilder::layoutNonVirtualBases(const CXXRecordDecl *RD) {
   // information about the bases, such as required alignment and the presence of
   // zero sized members.
   const ASTRecordLayout *PreviousBaseLayout = nullptr;
+  bool HasPolymorphicBaseClass = false;
   // Iterate through the bases and lay out the non-virtual ones.
   for (const CXXBaseSpecifier &Base : RD->bases()) {
     const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl();
+    HasPolymorphicBaseClass |= BaseDecl->isPolymorphic();
     const ASTRecordLayout &BaseLayout = Context.getASTRecordLayout(BaseDecl);
     // Mark and skip virtual bases.
     if (Base.isVirtual()) {
@@ -2594,11 +2596,23 @@ MicrosoftRecordLayoutBuilder::layoutNonVirtualBases(const CXXRecordDecl *RD) {
     layoutNonVirtualBase(RD, BaseDecl, BaseLayout, PreviousBaseLayout);
   }
   // Figure out if we need a fresh VFPtr for this class.
-  if (!PrimaryBase && RD->isDynamicClass())
-    for (CXXRecordDecl::method_iterator i = RD->method_begin(),
-                                        e = RD->method_end();
-         !HasOwnVFPtr && i != e; ++i)
-      HasOwnVFPtr = i->isVirtual() && i->size_overridden_methods() == 0;
+  if (RD->isPolymorphic()) {
+    if (!HasPolymorphicBaseClass)
+      // This class introduces polymorphism, so we need a vftable to store the
+      // RTTI information.
+      HasOwnVFPtr = true;
+    else if (!PrimaryBase) {
+      // We have a polymorphic base class but can't extend its vftable. Add a
+      // new vfptr if we would use any vftable slots.
+      for (CXXMethodDecl *M : RD->methods()) {
+        if (MicrosoftVTableContext::hasVtableSlot(M) &&
+            M->size_overridden_methods() == 0) {
+          HasOwnVFPtr = true;
+          break;
+        }
+      }
+    }
+  }
   // If we don't have a primary base then we have a leading object that could
   // itself lead with a zero-sized object, something we track.
   bool CheckLeadingLayout = !PrimaryBase;
@@ -2993,7 +3007,8 @@ void MicrosoftRecordLayoutBuilder::computeVtorDispSet(
   llvm::SmallPtrSet<const CXXRecordDecl *, 2> BasesWithOverriddenMethods;
   // Seed the working set with our non-destructor, non-pure virtual methods.
   for (const CXXMethodDecl *MD : RD->methods())
-    if (MD->isVirtual() && !isa<CXXDestructorDecl>(MD) && !MD->isPure())
+    if (MicrosoftVTableContext::hasVtableSlot(MD) &&
+        !isa<CXXDestructorDecl>(MD) && !MD->isPure())
       Work.insert(MD);
   while (!Work.empty()) {
     const CXXMethodDecl *MD = *Work.begin();
diff --git a/clang/lib/AST/Stmt.cpp b/clang/lib/AST/Stmt.cpp
index ce76d4941b32b..25e685be3e9b1 100644
--- a/clang/lib/AST/Stmt.cpp
+++ b/clang/lib/AST/Stmt.cpp
@@ -1012,7 +1012,8 @@ void SwitchStmt::setConditionVariable(const ASTContext &Ctx, VarDecl *V) {
 }
 
 WhileStmt::WhileStmt(const ASTContext &Ctx, VarDecl *Var, Expr *Cond,
-                     Stmt *Body, SourceLocation WL)
+                     Stmt *Body, SourceLocation WL, SourceLocation LParenLoc,
+                     SourceLocation RParenLoc)
     : Stmt(WhileStmtClass) {
   bool HasVar = Var != nullptr;
   WhileStmtBits.HasVar = HasVar;
@@ -1023,6 +1024,8 @@ WhileStmt::WhileStmt(const ASTContext &Ctx, VarDecl *Var, Expr *Cond,
     setConditionVariable(Ctx, Var);
 
   setWhileLoc(WL);
+  setLParenLoc(LParenLoc);
+  setRParenLoc(RParenLoc);
 }
 
 WhileStmt::WhileStmt(EmptyShell Empty, bool HasVar)
@@ -1031,12 +1034,14 @@ WhileStmt::WhileStmt(EmptyShell Empty, bool HasVar)
 }
 
 WhileStmt *WhileStmt::Create(const ASTContext &Ctx, VarDecl *Var, Expr *Cond,
-                             Stmt *Body, SourceLocation WL) {
+                             Stmt *Body, SourceLocation WL,
+                             SourceLocation LParenLoc,
+                             SourceLocation RParenLoc) {
   bool HasVar = Var != nullptr;
   void *Mem =
       Ctx.Allocate(totalSizeToAlloc<Stmt *>(NumMandatoryStmtPtr + HasVar),
                    alignof(WhileStmt));
-  return new (Mem) WhileStmt(Ctx, Var, Cond, Body, WL);
+  return new (Mem) WhileStmt(Ctx, Var, Cond, Body, WL, LParenLoc, RParenLoc);
 }
 
 WhileStmt *WhileStmt::CreateEmpty(const ASTContext &Ctx, bool HasVar) {
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index 11e222f95176d..f797f5fe8e6d1 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -1343,11 +1343,16 @@ void StmtPrinter::VisitOMPArraySectionExpr(OMPArraySectionExpr *Node) {
   OS << "[";
   if (Node->getLowerBound())
     PrintExpr(Node->getLowerBound());
-  if (Node->getColonLoc().isValid()) {
+  if (Node->getColonLocFirst().isValid()) {
     OS << ":";
     if (Node->getLength())
       PrintExpr(Node->getLength());
   }
+  if (Node->getColonLocSecond().isValid()) {
+    OS << ":";
+    if (Node->getStride())
+      PrintExpr(Node->getStride());
+  }
   OS << "]";
 }
 
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 0d49c443e03e2..5b0a0ac392c00 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -11,15 +11,20 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/TextNodeDumper.h"
+#include "clang/AST/APValue.h"
 #include "clang/AST/DeclFriend.h"
 #include "clang/AST/DeclOpenMP.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/LocInfoType.h"
+#include "clang/AST/Type.h"
 #include "clang/Basic/Module.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TypeTraits.h"
 
+#include <algorithm>
+#include <utility>
+
 using namespace clang;
 
 static void dumpPreviousDeclImpl(raw_ostream &OS, ...) {}
@@ -51,12 +56,15 @@ static void dumpPreviousDecl(raw_ostream &OS, const Decl *D) {
   llvm_unreachable("Decl that isn't part of DeclNodes.inc!");
 }
 
-TextNodeDumper::TextNodeDumper(raw_ostream &OS, bool ShowColors,
-                               const SourceManager *SM,
-                               const PrintingPolicy &PrintPolicy,
-                               const comments::CommandTraits *Traits)
-    : TextTreeStructure(OS, ShowColors), OS(OS), ShowColors(ShowColors), SM(SM),
-      PrintPolicy(PrintPolicy), Traits(Traits) {}
+TextNodeDumper::TextNodeDumper(raw_ostream &OS, const ASTContext &Context,
+                               bool ShowColors)
+    : TextTreeStructure(OS, ShowColors), OS(OS), ShowColors(ShowColors),
+      Context(&Context), SM(&Context.getSourceManager()),
+      PrintPolicy(Context.getPrintingPolicy()),
+      Traits(&Context.getCommentCommandTraits()) {}
+
+TextNodeDumper::TextNodeDumper(raw_ostream &OS, bool ShowColors)
+    : TextTreeStructure(OS, ShowColors), OS(OS), ShowColors(ShowColors) {}
 
 void TextNodeDumper::Visit(const comments::Comment *C,
                            const comments::FullComment *FC) {
@@ -347,6 +355,218 @@ void TextNodeDumper::Visit(const GenericSelectionExpr::ConstAssociation &A) {
     OS << " selected";
 }
 
+static double GetApproxValue(const llvm::APFloat &F) {
+  llvm::APFloat V = F;
+  bool ignored;
+  V.convert(llvm::APFloat::IEEEdouble(), llvm::APFloat::rmNearestTiesToEven,
+            &ignored);
+  return V.convertToDouble();
+}
+
+/// True if the \p APValue \p Value can be folded onto the current line.
+static bool isSimpleAPValue(const APValue &Value) {
+  switch (Value.getKind()) {
+  case APValue::None:
+  case APValue::Indeterminate:
+  case APValue::Int:
+  case APValue::Float:
+  case APValue::FixedPoint:
+  case APValue::ComplexInt:
+  case APValue::ComplexFloat:
+  case APValue::LValue:
+  case APValue::MemberPointer:
+  case APValue::AddrLabelDiff:
+    return true;
+  case APValue::Vector:
+  case APValue::Array:
+  case APValue::Struct:
+    return false;
+  case APValue::Union:
+    return isSimpleAPValue(Value.getUnionValue());
+  }
+  llvm_unreachable("unexpected APValue kind!");
+}
+
+/// Dump the children of the \p APValue \p Value.
+///
+/// \param[in] Value          The \p APValue to visit
+/// \param[in] Ty             The \p QualType passed to \p Visit
+///
+/// \param[in] IdxToChildFun  A function mapping an \p APValue and an index
+///                           to one of the child of the \p APValue
+///
+/// \param[in] NumChildren    \p IdxToChildFun will be called on \p Value with
+///                           the indices in the range \p [0,NumChildren(
+///
+/// \param[in] LabelSingular  The label to use on a line with a single child
+/// \param[in] LabelPlurial   The label to use on a line with multiple children
+void TextNodeDumper::dumpAPValueChildren(
+    const APValue &Value, QualType Ty,
+    const APValue &(*IdxToChildFun)(const APValue &, unsigned),
+    unsigned NumChildren, StringRef LabelSingular, StringRef LabelPlurial) {
+  // To save some vertical space we print up to MaxChildrenPerLine APValues
+  // considered to be simple (by isSimpleAPValue) on a single line.
+  constexpr unsigned MaxChildrenPerLine = 4;
+  unsigned I = 0;
+  while (I < NumChildren) {
+    unsigned J = I;
+    while (J < NumChildren) {
+      if (isSimpleAPValue(IdxToChildFun(Value, J)) &&
+          (J - I < MaxChildrenPerLine)) {
+        ++J;
+        continue;
+      }
+      break;
+    }
+
+    J = std::max(I + 1, J);
+
+    // Print [I,J) on a single line.
+    AddChild(J - I > 1 ? LabelPlurial : LabelSingular, [=]() {
+      for (unsigned X = I; X < J; ++X) {
+        Visit(IdxToChildFun(Value, X), Ty);
+        if (X + 1 != J)
+          OS << ", ";
+      }
+    });
+    I = J;
+  }
+}
+
+void TextNodeDumper::Visit(const APValue &Value, QualType Ty) {
+  ColorScope Color(OS, ShowColors, ValueKindColor);
+  switch (Value.getKind()) {
+  case APValue::None:
+    OS << "None";
+    return;
+  case APValue::Indeterminate:
+    OS << "Indeterminate";
+    return;
+  case APValue::Int:
+    OS << "Int ";
+    {
+      ColorScope Color(OS, ShowColors, ValueColor);
+      OS << Value.getInt();
+    }
+    return;
+  case APValue::Float:
+    OS << "Float ";
+    {
+      ColorScope Color(OS, ShowColors, ValueColor);
+      OS << GetApproxValue(Value.getFloat());
+    }
+    return;
+  case APValue::FixedPoint:
+    OS << "FixedPoint ";
+    {
+      ColorScope Color(OS, ShowColors, ValueColor);
+      OS << Value.getFixedPoint();
+    }
+    return;
+  case APValue::Vector: {
+    unsigned VectorLength = Value.getVectorLength();
+    OS << "Vector length=" << VectorLength;
+
+    dumpAPValueChildren(
+        Value, Ty,
+        [](const APValue &Value, unsigned Index) -> const APValue & {
+          return Value.getVectorElt(Index);
+        },
+        VectorLength, "element", "elements");
+    return;
+  }
+  case APValue::ComplexInt:
+    OS << "ComplexInt ";
+    {
+      ColorScope Color(OS, ShowColors, ValueColor);
+      OS << Value.getComplexIntReal() << " + " << Value.getComplexIntImag()
+         << 'i';
+    }
+    return;
+  case APValue::ComplexFloat:
+    OS << "ComplexFloat ";
+    {
+      ColorScope Color(OS, ShowColors, ValueColor);
+      OS << GetApproxValue(Value.getComplexFloatReal()) << " + "
+         << GetApproxValue(Value.getComplexFloatImag()) << 'i';
+    }
+    return;
+  case APValue::LValue:
+    (void)Context;
+    OS << "LValue <todo>";
+    return;
+  case APValue::Array: {
+    unsigned ArraySize = Value.getArraySize();
+    unsigned NumInitializedElements = Value.getArrayInitializedElts();
+    OS << "Array size=" << ArraySize;
+
+    dumpAPValueChildren(
+        Value, Ty,
+        [](const APValue &Value, unsigned Index) -> const APValue & {
+          return Value.getArrayInitializedElt(Index);
+        },
+        NumInitializedElements, "element", "elements");
+
+    if (Value.hasArrayFiller()) {
+      AddChild("filler", [=] {
+        {
+          ColorScope Color(OS, ShowColors, ValueColor);
+          OS << ArraySize - NumInitializedElements << " x ";
+        }
+        Visit(Value.getArrayFiller(), Ty);
+      });
+    }
+
+    return;
+  }
+  case APValue::Struct: {
+    OS << "Struct";
+
+    dumpAPValueChildren(
+        Value, Ty,
+        [](const APValue &Value, unsigned Index) -> const APValue & {
+          return Value.getStructBase(Index);
+        },
+        Value.getStructNumBases(), "base", "bases");
+
+    dumpAPValueChildren(
+        Value, Ty,
+        [](const APValue &Value, unsigned Index) -> const APValue & {
+          return Value.getStructField(Index);
+        },
+        Value.getStructNumFields(), "field", "fields");
+
+    return;
+  }
+  case APValue::Union: {
+    OS << "Union";
+    {
+      ColorScope Color(OS, ShowColors, ValueColor);
+      if (const FieldDecl *FD = Value.getUnionField())
+        OS << " ." << *cast<NamedDecl>(FD);
+    }
+    // If the union value is considered to be simple, fold it into the
+    // current line to save some vertical space.
+    const APValue &UnionValue = Value.getUnionValue();
+    if (isSimpleAPValue(UnionValue)) {
+      OS << ' ';
+      Visit(UnionValue, Ty);
+    } else {
+      AddChild([=] { Visit(UnionValue, Ty); });
+    }
+
+    return;
+  }
+  case APValue::MemberPointer:
+    OS << "MemberPointer <todo>";
+    return;
+  case APValue::AddrLabelDiff:
+    OS << "AddrLabelDiff <todo>";
+    return;
+  }
+  llvm_unreachable("Unknown APValue kind!");
+}
+
 void TextNodeDumper::dumpPointer(const void *Ptr) {
   ColorScope Color(OS, ShowColors, AddressColor);
   OS << ' ' << Ptr;
@@ -709,11 +929,9 @@ void TextNodeDumper::VisitCaseStmt(const CaseStmt *Node) {
 }
 
 void TextNodeDumper::VisitConstantExpr(const ConstantExpr *Node) {
-  if (Node->getResultAPValueKind() != APValue::None) {
-    ColorScope Color(OS, ShowColors, ValueColor);
-    OS << " ";
-    Node->getAPValueResult().dump(OS);
-  }
+  if (Node->hasAPValueResult())
+    AddChild("value",
+             [=] { Visit(Node->getAPValueResult(), Node->getType()); });
 }
 
 void TextNodeDumper::VisitCallExpr(const CallExpr *Node) {
@@ -1451,6 +1669,16 @@ void TextNodeDumper::VisitVarDecl(const VarDecl *D) {
     OS << " destroyed";
   if (D->isParameterPack())
     OS << " pack";
+
+  if (D->hasInit()) {
+    const Expr *E = D->getInit();
+    // Only dump the value of constexpr VarDecls for now.
+    if (E && !E->isValueDependent() && D->isConstexpr()) {
+      const APValue *Value = D->evaluateValue();
+      if (Value)
+        AddChild("value", [=] { Visit(*Value, E->getType()); });
+    }
+  }
 }
 
 void TextNodeDumper::VisitBindingDecl(const BindingDecl *D) {
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index ba0d86befe1b9..10a6a26101302 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -347,15 +347,7 @@ ExtIntType::ExtIntType(bool IsUnsigned, unsigned NumBits)
 DependentExtIntType::DependentExtIntType(const ASTContext &Context,
                                          bool IsUnsigned, Expr *NumBitsExpr)
     : Type(DependentExtInt, QualType{},
-           ((NumBitsExpr->isValueDependent() || NumBitsExpr->isTypeDependent())
-                ? TypeDependence::Dependent
-                : TypeDependence::None) |
-               (NumBitsExpr->isInstantiationDependent()
-                    ? TypeDependence::Instantiation
-                    : TypeDependence::None) |
-               (NumBitsExpr->containsUnexpandedParameterPack()
-                    ? TypeDependence::VariablyModified
-                    : TypeDependence::None)),
+           toTypeDependence(NumBitsExpr->getDependence())),
       Context(Context), ExprAndUnsigned(NumBitsExpr, IsUnsigned) {}
 
 bool DependentExtIntType::isUnsigned() const {
@@ -3591,7 +3583,7 @@ TemplateSpecializationType::TemplateSpecializationType(
 
   auto *TemplateArgs = reinterpret_cast<TemplateArgument *>(this + 1);
   for (const TemplateArgument &Arg : Args) {
-    // Update instantiation-dependent and variably-modified bits.
+    // Update instantiation-dependent, variably-modified, and error bits.
     // If the canonical type exists and is non-dependent, the template
     // specialization type can be non-dependent even if one of the type
     // arguments is. Given:
diff --git a/clang/lib/AST/VTableBuilder.cpp b/clang/lib/AST/VTableBuilder.cpp
index 013ebe23ec918..f5865ce96b647 100644
--- a/clang/lib/AST/VTableBuilder.cpp
+++ b/clang/lib/AST/VTableBuilder.cpp
@@ -408,7 +408,7 @@ void FinalOverriders::dump(raw_ostream &Out, BaseSubobject Base,
 
   // Now dump the overriders for this base subobject.
   for (const auto *MD : RD->methods()) {
-    if (!MD->isVirtual())
+    if (!VTableContextBase::hasVtableSlot(MD))
       continue;
     MD = MD->getCanonicalDecl();
 
@@ -486,8 +486,8 @@ static bool HasSameVirtualSignature(const CXXMethodDecl *LHS,
 
 bool VCallOffsetMap::MethodsCanShareVCallOffset(const CXXMethodDecl *LHS,
                                                 const CXXMethodDecl *RHS) {
-  assert(LHS->isVirtual() && "LHS must be virtual!");
-  assert(RHS->isVirtual() && "LHS must be virtual!");
+  assert(VTableContextBase::hasVtableSlot(LHS) && "LHS must be virtual!");
+  assert(VTableContextBase::hasVtableSlot(RHS) && "LHS must be virtual!");
 
   // A destructor can share a vcall offset with another destructor.
   if (isa<CXXDestructorDecl>(LHS))
@@ -697,7 +697,7 @@ void VCallAndVBaseOffsetBuilder::AddVCallOffsets(BaseSubobject Base,
 
   // Add the vcall offsets.
   for (const auto *MD : RD->methods()) {
-    if (!MD->isVirtual())
+    if (!VTableContextBase::hasVtableSlot(MD))
       continue;
     MD = MD->getCanonicalDecl();
 
@@ -1085,7 +1085,7 @@ typedef llvm::SmallPtrSet<const CXXMethodDecl *, 8> OverriddenMethodsSetTy;
 template <class VisitorTy>
 static void
 visitAllOverriddenMethods(const CXXMethodDecl *MD, VisitorTy &Visitor) {
-  assert(MD->isVirtual() && "Method is not virtual!");
+  assert(VTableContextBase::hasVtableSlot(MD) && "Method is not virtual!");
 
   for (const CXXMethodDecl *OverriddenMD : MD->overridden_methods()) {
     if (!Visitor(OverriddenMD))
@@ -1489,7 +1489,7 @@ void ItaniumVTableBuilder::AddMethods(
 
   // Now go through all virtual member functions and add them.
   for (const auto *MD : RD->methods()) {
-    if (!MD->isVirtual())
+    if (!ItaniumVTableContext::hasVtableSlot(MD))
       continue;
     MD = MD->getCanonicalDecl();
 
@@ -2169,7 +2169,7 @@ void ItaniumVTableBuilder::dumpLayout(raw_ostream &Out) {
 
   for (const auto *MD : MostDerivedClass->methods()) {
     // We only want virtual member functions.
-    if (!MD->isVirtual())
+    if (!ItaniumVTableContext::hasVtableSlot(MD))
       continue;
     MD = MD->getCanonicalDecl();
 
@@ -2257,6 +2257,10 @@ VTableLayout::VTableLayout(ArrayRef<size_t> VTableIndices,
 
 VTableLayout::~VTableLayout() { }
 
+bool VTableContextBase::hasVtableSlot(const CXXMethodDecl *MD) {
+  return MD->isVirtual() && !MD->isConsteval();
+}
+
 ItaniumVTableContext::ItaniumVTableContext(
     ASTContext &Context, VTableComponentLayout ComponentLayout)
     : VTableContextBase(/*MS=*/false), ComponentLayout(ComponentLayout) {}
@@ -2537,8 +2541,9 @@ class VFTableBuilder {
     BasesSetVectorTy VisitedBases;
     AddMethods(BaseSubobject(MostDerivedClass, CharUnits::Zero()), 0, nullptr,
                VisitedBases);
-    assert((HasRTTIComponent ? Components.size() - 1 : Components.size()) &&
-           "vftable can't be empty");
+    // Note that it is possible for the vftable to contain only an RTTI
+    // pointer, if all virtual functions are constewval.
+    assert(!Components.empty() && "vftable can't be empty");
 
     assert(MethodVFTableLocations.empty());
     for (const auto &I : MethodInfoMap) {
@@ -2917,7 +2922,7 @@ static void GroupNewVirtualOverloads(
     if (Inserted)
       Groups.push_back(MethodGroup());
     if (const auto *MD = dyn_cast<CXXMethodDecl>(ND))
-      if (MD->isVirtual())
+      if (MicrosoftVTableContext::hasVtableSlot(MD))
         Groups[J->second].push_back(MD->getCanonicalDecl());
   }
 
@@ -3513,7 +3518,7 @@ static const FullPathTy *selectBestPath(ASTContext &Context,
         getOffsetOfFullPath(Context, TopLevelRD, SpecificPath);
     FinalOverriders Overriders(TopLevelRD, CharUnits::Zero(), TopLevelRD);
     for (const CXXMethodDecl *MD : Info.IntroducingObject->methods()) {
-      if (!MD->isVirtual())
+      if (!MicrosoftVTableContext::hasVtableSlot(MD))
         continue;
       FinalOverriders::OverriderInfo OI =
           Overriders.getOverrider(MD->getCanonicalDecl(), BaseOffset);
@@ -3652,7 +3657,7 @@ void MicrosoftVTableContext::dumpMethodLocations(
 
   for (const auto &I : NewMethods) {
     const CXXMethodDecl *MD = cast<const CXXMethodDecl>(I.first.getDecl());
-    assert(MD->isVirtual());
+    assert(hasVtableSlot(MD));
 
     std::string MethodName = PredefinedExpr::ComputeName(
         PredefinedExpr::PrettyFunctionNoVirtual, MD);
@@ -3772,7 +3777,7 @@ MicrosoftVTableContext::getVFTableLayout(const CXXRecordDecl *RD,
 
 MethodVFTableLocation
 MicrosoftVTableContext::getMethodVFTableLocation(GlobalDecl GD) {
-  assert(cast<CXXMethodDecl>(GD.getDecl())->isVirtual() &&
+  assert(hasVtableSlot(cast<CXXMethodDecl>(GD.getDecl())) &&
          "Only use this method for virtual methods or dtors");
   if (isa<CXXDestructorDecl>(GD.getDecl()))
     assert(GD.getDtorType() == Dtor_Deleting);
diff --git a/clang/lib/ASTMatchers/ASTMatchFinder.cpp b/clang/lib/ASTMatchers/ASTMatchFinder.cpp
index 1b49067f0f9d7..e88da16dd3d40 100644
--- a/clang/lib/ASTMatchers/ASTMatchFinder.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchFinder.cpp
@@ -43,6 +43,13 @@ typedef MatchFinder::MatchCallback MatchCallback;
 // optimize this on.
 static const unsigned MaxMemoizationEntries = 10000;
 
+enum class MatchType {
+  Ancestors,
+
+  Descendants,
+  Child,
+};
+
 // We use memoization to avoid running the same matcher on the same
 // AST node twice.  This struct is the key for looking up match
 // result.  It consists of an ID of the MatcherInterface (for
@@ -60,10 +67,11 @@ struct MatchKey {
   DynTypedNode Node;
   BoundNodesTreeBuilder BoundNodes;
   TraversalKind Traversal = TK_AsIs;
+  MatchType Type;
 
   bool operator<(const MatchKey &Other) const {
-    return std::tie(Traversal, MatcherID, Node, BoundNodes) <
-           std::tie(Other.Traversal, Other.MatcherID, Other.Node,
+    return std::tie(Traversal, Type, MatcherID, Node, BoundNodes) <
+           std::tie(Other.Traversal, Other.Type, Other.MatcherID, Other.Node,
                     Other.BoundNodes);
   }
 };
@@ -456,7 +464,8 @@ class MatchASTVisitor : public RecursiveASTVisitor<MatchASTVisitor>,
     // Note that we key on the bindings *before* the match.
     Key.BoundNodes = *Builder;
     Key.Traversal = Ctx.getParentMapContext().getTraversalKind();
-
+    // Memoize result even doing a single-level match, it might be expensive.
+    Key.Type = MaxDepth == 1 ? MatchType::Child : MatchType::Descendants;
     MemoizationMap::iterator I = ResultCache.find(Key);
     if (I != ResultCache.end()) {
       *Builder = I->second.Nodes;
@@ -693,7 +702,10 @@ class MatchASTVisitor : public RecursiveASTVisitor<MatchASTVisitor>,
                                             BoundNodesTreeBuilder *Builder,
                                             AncestorMatchMode MatchMode) {
     // For AST-nodes that don't have an identity, we can't memoize.
-    if (!Builder->isComparable())
+    // When doing a single-level match, we don't need to memoize because
+    // ParentMap (in ASTContext) already memoizes the result.
+    if (!Builder->isComparable() ||
+        MatchMode == AncestorMatchMode::AMM_ParentOnly)
       return matchesAncestorOfRecursively(Node, Ctx, Matcher, Builder,
                                           MatchMode);
 
@@ -702,6 +714,7 @@ class MatchASTVisitor : public RecursiveASTVisitor<MatchASTVisitor>,
     Key.Node = Node;
     Key.BoundNodes = *Builder;
     Key.Traversal = Ctx.getParentMapContext().getTraversalKind();
+    Key.Type = MatchType::Ancestors;
 
     // Note that we cannot use insert and reuse the iterator, as recursive
     // calls to match might invalidate the result cache iterators.
@@ -742,7 +755,7 @@ class MatchASTVisitor : public RecursiveASTVisitor<MatchASTVisitor>,
             return D->getKind() == Decl::TranslationUnit;
           })) {
         llvm::errs() << "Tried to match orphan node:\n";
-        Node.dump(llvm::errs(), ActiveASTContext->getSourceManager());
+        Node.dump(llvm::errs(), *ActiveASTContext);
         llvm_unreachable("Parent map should be complete!");
       }
 #endif
diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index 9b69734d075d3..4b9baf7a0e751 100644
--- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -29,6 +29,8 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cassert>
@@ -682,6 +684,19 @@ getExpansionLocOfMacro(StringRef MacroName, SourceLocation Loc,
   return llvm::None;
 }
 
+std::shared_ptr<llvm::Regex> createAndVerifyRegex(StringRef Regex,
+                                                  llvm::Regex::RegexFlags Flags,
+                                                  StringRef MatcherID) {
+  assert(!Regex.empty() && "Empty regex string");
+  auto SharedRegex = std::make_shared<llvm::Regex>(Regex, Flags);
+  std::string Error;
+  if (!SharedRegex->isValid(Error)) {
+    llvm::WithColor::error()
+        << "building matcher '" << MatcherID << "': " << Error << "\n";
+    llvm::WithColor::note() << " input was '" << Regex << "'\n";
+  }
+  return SharedRegex;
+}
 } // end namespace internal
 
 const internal::VariadicDynCastAllOfMatcher<Stmt, ObjCAutoreleasePoolStmt>
diff --git a/clang/lib/ASTMatchers/CMakeLists.txt b/clang/lib/ASTMatchers/CMakeLists.txt
index cde871cd31cac..68f214b778401 100644
--- a/clang/lib/ASTMatchers/CMakeLists.txt
+++ b/clang/lib/ASTMatchers/CMakeLists.txt
@@ -14,4 +14,7 @@ add_clang_library(clangASTMatchers
   clangAST
   clangBasic
   clangLex
+
+  DEPENDS
+  omp_gen
   )
diff --git a/clang/lib/ASTMatchers/Dynamic/CMakeLists.txt b/clang/lib/ASTMatchers/Dynamic/CMakeLists.txt
index 1c6c1e68ce1e4..adb891575b308 100644
--- a/clang/lib/ASTMatchers/Dynamic/CMakeLists.txt
+++ b/clang/lib/ASTMatchers/Dynamic/CMakeLists.txt
@@ -23,4 +23,7 @@ add_clang_library(clangDynamicASTMatchers
   clangAST
   clangASTMatchers
   clangBasic
+
+  DEPENDS
+  omp_gen
   )
diff --git a/clang/lib/ASTMatchers/Dynamic/Marshallers.cpp b/clang/lib/ASTMatchers/Dynamic/Marshallers.cpp
index 78b6ca1aa227f..989ee0fa75cdd 100644
--- a/clang/lib/ASTMatchers/Dynamic/Marshallers.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Marshallers.cpp
@@ -10,6 +10,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Regex.h"
 #include <string>
 
 static llvm::Optional<std::string>
@@ -110,3 +111,62 @@ clang::ast_matchers::dynamic::internal::ArgTypeTraits<
                           "UETT_");
   return llvm::None;
 }
+
+static constexpr std::pair<llvm::StringRef, llvm::Regex::RegexFlags>
+    RegexMap[] = {
+        {"NoFlags", llvm::Regex::RegexFlags::NoFlags},
+        {"IgnoreCase", llvm::Regex::RegexFlags::IgnoreCase},
+        {"Newline", llvm::Regex::RegexFlags::Newline},
+        {"BasicRegex", llvm::Regex::RegexFlags::BasicRegex},
+};
+
+llvm::Optional<llvm::Regex::RegexFlags> getRegexFlag(llvm::StringRef Flag) {
+  for (const auto &StringFlag : RegexMap) {
+    if (Flag == StringFlag.first)
+      return StringFlag.second;
+  }
+  return llvm::None;
+}
+
+llvm::Optional<llvm::StringRef> getCloseRegexMatch(llvm::StringRef Flag) {
+  for (const auto &StringFlag : RegexMap) {
+    if (Flag.edit_distance(StringFlag.first) < 3)
+      return StringFlag.first;
+  }
+  return llvm::None;
+}
+
+llvm::Optional<llvm::Regex::RegexFlags>
+clang::ast_matchers::dynamic::internal::ArgTypeTraits<
+    llvm::Regex::RegexFlags>::getFlags(llvm::StringRef Flags) {
+  llvm::Optional<llvm::Regex::RegexFlags> Flag;
+  SmallVector<StringRef, 4> Split;
+  Flags.split(Split, '|', -1, false);
+  for (StringRef OrFlag : Split) {
+    if (llvm::Optional<llvm::Regex::RegexFlags> NextFlag =
+            getRegexFlag(OrFlag.trim()))
+      Flag = Flag.getValueOr(llvm::Regex::NoFlags) | *NextFlag;
+    else
+      return None;
+  }
+  return Flag;
+}
+
+llvm::Optional<std::string>
+clang::ast_matchers::dynamic::internal::ArgTypeTraits<
+    llvm::Regex::RegexFlags>::getBestGuess(const VariantValue &Value) {
+  if (!Value.isString())
+    return llvm::None;
+  SmallVector<StringRef, 4> Split;
+  llvm::StringRef(Value.getString()).split(Split, '|', -1, false);
+  for (llvm::StringRef &Flag : Split) {
+    if (llvm::Optional<llvm::StringRef> BestGuess =
+            getCloseRegexMatch(Flag.trim()))
+      Flag = *BestGuess;
+    else
+      return None;
+  }
+  if (Split.empty())
+    return None;
+  return llvm::join(Split, " | ");
+}
diff --git a/clang/lib/ASTMatchers/Dynamic/Marshallers.h b/clang/lib/ASTMatchers/Dynamic/Marshallers.h
index e07db59433b71..33f6d1e4155cb 100644
--- a/clang/lib/ASTMatchers/Dynamic/Marshallers.h
+++ b/clang/lib/ASTMatchers/Dynamic/Marshallers.h
@@ -35,6 +35,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Support/Regex.h"
 #include <cassert>
 #include <cstddef>
 #include <iterator>
@@ -192,6 +193,24 @@ template <> struct ArgTypeTraits<CastKind> {
   static llvm::Optional<std::string> getBestGuess(const VariantValue &Value);
 };
 
+template <> struct ArgTypeTraits<llvm::Regex::RegexFlags> {
+private:
+  static Optional<llvm::Regex::RegexFlags> getFlags(llvm::StringRef Flags);
+
+public:
+  static bool is(const VariantValue &Value) {
+    return Value.isString() && getFlags(Value.getString());
+  }
+
+  static llvm::Regex::RegexFlags get(const VariantValue &Value) {
+    return *getFlags(Value.getString());
+  }
+
+  static ArgKind getKind() { return ArgKind(ArgKind::AK_String); }
+
+  static llvm::Optional<std::string> getBestGuess(const VariantValue &Value);
+};
+
 template <> struct ArgTypeTraits<OpenMPClauseKind> {
 private:
   static Optional<OpenMPClauseKind> getClauseKind(llvm::StringRef ClauseKind) {
@@ -711,6 +730,71 @@ class OverloadedMatcherDescriptor : public MatcherDescriptor {
   std::vector<std::unique_ptr<MatcherDescriptor>> Overloads;
 };
 
+template <typename ReturnType>
+class RegexMatcherDescriptor : public MatcherDescriptor {
+public:
+  RegexMatcherDescriptor(ReturnType (*WithFlags)(StringRef,
+                                                 llvm::Regex::RegexFlags),
+                         ReturnType (*NoFlags)(StringRef),
+                         ArrayRef<ASTNodeKind> RetKinds)
+      : WithFlags(WithFlags), NoFlags(NoFlags),
+        RetKinds(RetKinds.begin(), RetKinds.end()) {}
+  bool isVariadic() const override { return true; }
+  unsigned getNumArgs() const override { return 0; }
+
+  void getArgKinds(ASTNodeKind ThisKind, unsigned ArgNo,
+                   std::vector<ArgKind> &Kinds) const override {
+    assert(ArgNo < 2);
+    Kinds.push_back(ArgKind::AK_String);
+  }
+
+  bool isConvertibleTo(ASTNodeKind Kind, unsigned *Specificity,
+                       ASTNodeKind *LeastDerivedKind) const override {
+    return isRetKindConvertibleTo(RetKinds, Kind, Specificity,
+                                  LeastDerivedKind);
+  }
+
+  VariantMatcher create(SourceRange NameRange, ArrayRef<ParserValue> Args,
+                        Diagnostics *Error) const override {
+    if (Args.size() < 1 || Args.size() > 2) {
+      Error->addError(NameRange, Diagnostics::ET_RegistryWrongArgCount)
+          << "1 or 2" << Args.size();
+      return VariantMatcher();
+    }
+    if (!ArgTypeTraits<StringRef>::is(Args[0].Value)) {
+      Error->addError(Args[0].Range, Error->ET_RegistryWrongArgType)
+          << 1 << ArgTypeTraits<StringRef>::getKind().asString()
+          << Args[0].Value.getTypeAsString();
+      return VariantMatcher();
+    }
+    if (Args.size() == 1) {
+      return outvalueToVariantMatcher(
+          NoFlags(ArgTypeTraits<StringRef>::get(Args[0].Value)));
+    }
+    if (!ArgTypeTraits<llvm::Regex::RegexFlags>::is(Args[1].Value)) {
+      if (llvm::Optional<std::string> BestGuess =
+              ArgTypeTraits<llvm::Regex::RegexFlags>::getBestGuess(
+                  Args[1].Value)) {
+        Error->addError(Args[1].Range, Error->ET_RegistryUnknownEnumWithReplace)
+            << 2 << Args[1].Value.getString() << *BestGuess;
+      } else {
+        Error->addError(Args[1].Range, Error->ET_RegistryWrongArgType)
+            << 2 << ArgTypeTraits<llvm::Regex::RegexFlags>::getKind().asString()
+            << Args[1].Value.getTypeAsString();
+      }
+      return VariantMatcher();
+    }
+    return outvalueToVariantMatcher(
+        WithFlags(ArgTypeTraits<StringRef>::get(Args[0].Value),
+                  ArgTypeTraits<llvm::Regex::RegexFlags>::get(Args[1].Value)));
+  }
+
+private:
+  ReturnType (*const WithFlags)(StringRef, llvm::Regex::RegexFlags);
+  ReturnType (*const NoFlags)(StringRef);
+  const std::vector<ASTNodeKind> RetKinds;
+};
+
 /// Variadic operator marshaller function.
 class VariadicOperatorMatcherDescriptor : public MatcherDescriptor {
 public:
@@ -814,6 +898,16 @@ makeMatcherAutoMarshall(ReturnType (*Func)(ArgType1, ArgType2),
       reinterpret_cast<void (*)()>(Func), MatcherName, RetTypes, AKs);
 }
 
+template <typename ReturnType>
+std::unique_ptr<MatcherDescriptor> makeMatcherRegexMarshall(
+    ReturnType (*FuncFlags)(llvm::StringRef, llvm::Regex::RegexFlags),
+    ReturnType (*Func)(llvm::StringRef)) {
+  std::vector<ASTNodeKind> RetTypes;
+  BuildReturnTypeVector<ReturnType>::build(RetTypes);
+  return std::make_unique<RegexMatcherDescriptor<ReturnType>>(FuncFlags, Func,
+                                                              RetTypes);
+}
+
 /// Variadic overload.
 template <typename ResultT, typename ArgT,
           ResultT (*Func)(ArrayRef<const ArgT *>)>
diff --git a/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
index 950a56f795516..ec2215804c098 100644
--- a/clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -90,6 +90,9 @@ void RegistryMaps::registerMatcher(
     REGISTER_MATCHER_OVERLOAD(name);                                           \
   } while (false)
 
+#define REGISTER_REGEX_MATCHER(name)                                           \
+  registerMatcher(#name, internal::makeMatcherRegexMarshall(name, name))
+
 /// Generate a registry map with all the known matchers.
 /// Please keep sorted alphabetically!
 RegistryMaps::RegistryMaps() {
@@ -121,6 +124,10 @@ RegistryMaps::RegistryMaps() {
   };
   REGISTER_MATCHER_OVERLOAD(equals);
 
+  REGISTER_REGEX_MATCHER(isExpansionInFileMatching);
+  REGISTER_REGEX_MATCHER(matchesName);
+  REGISTER_REGEX_MATCHER(matchesSelector);
+
   REGISTER_MATCHER(accessSpecDecl);
   REGISTER_MATCHER(addrLabelExpr);
   REGISTER_MATCHER(alignOfExpr);
@@ -151,6 +158,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(characterLiteral);
   REGISTER_MATCHER(chooseExpr);
   REGISTER_MATCHER(classTemplateDecl);
+  REGISTER_MATCHER(classTemplatePartialSpecializationDecl);
   REGISTER_MATCHER(classTemplateSpecializationDecl);
   REGISTER_MATCHER(complexType);
   REGISTER_MATCHER(compoundLiteralExpr);
@@ -219,7 +227,6 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(floatLiteral);
   REGISTER_MATCHER(forEach);
   REGISTER_MATCHER(forEachArgumentWithParam);
-  REGISTER_MATCHER(isAtPosition);
   REGISTER_MATCHER(forEachConstructorInitializer);
   REGISTER_MATCHER(forEachDescendant);
   REGISTER_MATCHER(forEachOverridden);
@@ -271,6 +278,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(hasDefinition);
   REGISTER_MATCHER(hasDescendant);
   REGISTER_MATCHER(hasDestinationType);
+  REGISTER_MATCHER(hasDirectBase);
   REGISTER_MATCHER(hasDynamicExceptionSpec);
   REGISTER_MATCHER(hasEitherOperand);
   REGISTER_MATCHER(hasElementType);
@@ -353,6 +361,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(isArray);
   REGISTER_MATCHER(isArrow);
   REGISTER_MATCHER(isAssignmentOperator);
+  REGISTER_MATCHER(isAtPosition);
   REGISTER_MATCHER(isBaseInitializer);
   REGISTER_MATCHER(isBitField);
   REGISTER_MATCHER(isCatchAll);
@@ -370,8 +379,9 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(isDefinition);
   REGISTER_MATCHER(isDelegatingConstructor);
   REGISTER_MATCHER(isDeleted);
+  REGISTER_MATCHER(isEnum);
   REGISTER_MATCHER(isExceptionVariable);
-  REGISTER_MATCHER(isExpansionInFileMatching);
+  REGISTER_MATCHER(isExpandedFromMacro);
   REGISTER_MATCHER(isExpansionInMainFile);
   REGISTER_MATCHER(isExpansionInSystemHeader);
   REGISTER_MATCHER(isExplicit);
@@ -379,6 +389,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(isExpr);
   REGISTER_MATCHER(isExternC);
   REGISTER_MATCHER(isFinal);
+  REGISTER_MATCHER(isFirstPrivateKind);
   REGISTER_MATCHER(isImplicit);
   REGISTER_MATCHER(isInStdNamespace);
   REGISTER_MATCHER(isInTemplateInstantiation);
@@ -426,8 +437,6 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(labelStmt);
   REGISTER_MATCHER(lambdaExpr);
   REGISTER_MATCHER(linkageSpecDecl);
-  REGISTER_MATCHER(matchesName);
-  REGISTER_MATCHER(matchesSelector);
   REGISTER_MATCHER(materializeTemporaryExpr);
   REGISTER_MATCHER(member);
   REGISTER_MATCHER(memberExpr);
diff --git a/clang/lib/Analysis/BodyFarm.cpp b/clang/lib/Analysis/BodyFarm.cpp
index e10fbce711465..f9f0553d28f05 100644
--- a/clang/lib/Analysis/BodyFarm.cpp
+++ b/clang/lib/Analysis/BodyFarm.cpp
@@ -116,18 +116,17 @@ BinaryOperator *ASTMaker::makeAssignment(const Expr *LHS, const Expr *RHS,
                                          QualType Ty) {
   return BinaryOperator::Create(
       C, const_cast<Expr *>(LHS), const_cast<Expr *>(RHS), BO_Assign, Ty,
-      VK_RValue, OK_Ordinary, SourceLocation(), FPOptions(C.getLangOpts()));
+      VK_RValue, OK_Ordinary, SourceLocation(), FPOptionsOverride());
 }
 
 BinaryOperator *ASTMaker::makeComparison(const Expr *LHS, const Expr *RHS,
                                          BinaryOperator::Opcode Op) {
   assert(BinaryOperator::isLogicalOp(Op) ||
          BinaryOperator::isComparisonOp(Op));
-  return BinaryOperator::Create(C, const_cast<Expr *>(LHS),
-                                const_cast<Expr *>(RHS), Op,
-                                C.getLogicalOperationType(), VK_RValue,
-                                OK_Ordinary, SourceLocation(),
-                                FPOptions(C.getLangOpts()));
+  return BinaryOperator::Create(
+      C, const_cast<Expr *>(LHS), const_cast<Expr *>(RHS), Op,
+      C.getLogicalOperationType(), VK_RValue, OK_Ordinary, SourceLocation(),
+      FPOptionsOverride());
 }
 
 CompoundStmt *ASTMaker::makeCompound(ArrayRef<Stmt *> Stmts) {
@@ -148,8 +147,7 @@ DeclRefExpr *ASTMaker::makeDeclRefExpr(
 UnaryOperator *ASTMaker::makeDereference(const Expr *Arg, QualType Ty) {
   return UnaryOperator::Create(C, const_cast<Expr *>(Arg), UO_Deref, Ty,
                                VK_LValue, OK_Ordinary, SourceLocation(),
-                               /*CanOverflow*/ false,
-                               FPOptions(C.getLangOpts()));
+                               /*CanOverflow*/ false, FPOptionsOverride());
 }
 
 ImplicitCastExpr *ASTMaker::makeLvalueToRvalue(const Expr *Arg, QualType Ty) {
@@ -297,7 +295,7 @@ static CallExpr *create_call_once_lambda_call(ASTContext &C, ASTMaker M,
       /*QualType=*/C.VoidTy,
       /*ExprValueType=*/VK_RValue,
       /*SourceLocation=*/SourceLocation(),
-      /*FPFeatures=*/FPOptions(C.getLangOpts()));
+      /*FPFeatures=*/FPOptionsOverride());
 }
 
 /// Create a fake body for std::call_once.
@@ -457,7 +455,7 @@ static Stmt *create_call_once(ASTContext &C, const FunctionDecl *D) {
       /* QualType=*/C.IntTy,
       /* ExprValueKind=*/VK_RValue,
       /* ExprObjectKind=*/OK_Ordinary, SourceLocation(),
-      /* CanOverflow*/ false, FPOptions(C.getLangOpts()));
+      /* CanOverflow*/ false, FPOptionsOverride());
 
   // Create assignment.
   BinaryOperator *FlagAssignment = M.makeAssignment(
@@ -522,7 +520,7 @@ static Stmt *create_dispatch_once(ASTContext &C, const FunctionDecl *D) {
   Expr *DoneValue =
       UnaryOperator::Create(C, M.makeIntegerLiteral(0, C.LongTy), UO_Not,
                             C.LongTy, VK_RValue, OK_Ordinary, SourceLocation(),
-                            /*CanOverflow*/ false, FPOptions(C.getLangOpts()));
+                            /*CanOverflow*/ false, FPOptionsOverride());
 
   BinaryOperator *B =
     M.makeAssignment(
diff --git a/clang/lib/Analysis/CMakeLists.txt b/clang/lib/Analysis/CMakeLists.txt
index 3333eab3b96d1..82cb00a8c3e0e 100644
--- a/clang/lib/Analysis/CMakeLists.txt
+++ b/clang/lib/Analysis/CMakeLists.txt
@@ -35,6 +35,9 @@ add_clang_library(clangAnalysis
   clangASTMatchers
   clangBasic
   clangLex
+
+  DEPENDS
+  omp_gen
   )
 
 add_subdirectory(plugins)
diff --git a/clang/lib/Analysis/ReachableCode.cpp b/clang/lib/Analysis/ReachableCode.cpp
index 369879ad65f50..221d137dadb87 100644
--- a/clang/lib/Analysis/ReachableCode.cpp
+++ b/clang/lib/Analysis/ReachableCode.cpp
@@ -138,10 +138,10 @@ static bool isDeadReturn(const CFGBlock *B, const Stmt *S) {
 static SourceLocation getTopMostMacro(SourceLocation Loc, SourceManager &SM) {
   assert(Loc.isMacroID());
   SourceLocation Last;
-  while (Loc.isMacroID()) {
+  do {
     Last = Loc;
     Loc = SM.getImmediateMacroCallerLoc(Loc);
-  }
+  } while (Loc.isMacroID());
   return Last;
 }
 
diff --git a/clang/lib/Analysis/UninitializedValues.cpp b/clang/lib/Analysis/UninitializedValues.cpp
index 3d44d1c070fea..67cd39728c350 100644
--- a/clang/lib/Analysis/UninitializedValues.cpp
+++ b/clang/lib/Analysis/UninitializedValues.cpp
@@ -405,6 +405,15 @@ static bool isPointerToConst(const QualType &QT) {
   return QT->isAnyPointerType() && QT->getPointeeType().isConstQualified();
 }
 
+static bool hasTrivialBody(CallExpr *CE) {
+  if (FunctionDecl *FD = CE->getDirectCallee()) {
+    if (FunctionTemplateDecl *FTD = FD->getPrimaryTemplate())
+      return FTD->getTemplatedDecl()->hasTrivialBody();
+    return FD->hasTrivialBody();
+  }
+  return false;
+}
+
 void ClassifyRefs::VisitCallExpr(CallExpr *CE) {
   // Classify arguments to std::move as used.
   if (CE->isCallToStdMove()) {
@@ -413,7 +422,7 @@ void ClassifyRefs::VisitCallExpr(CallExpr *CE) {
       classify(CE->getArg(0), Use);
     return;
   }
-
+  bool isTrivialBody = hasTrivialBody(CE);
   // If a value is passed by const pointer to a function,
   // we should not assume that it is initialized by the call, and we
   // conservatively do not assume that it is used.
@@ -423,7 +432,7 @@ void ClassifyRefs::VisitCallExpr(CallExpr *CE) {
        I != E; ++I) {
     if ((*I)->isGLValue()) {
       if ((*I)->getType().isConstQualified())
-        classify((*I), ConstRefUse);
+        classify((*I), isTrivialBody ? Ignore : ConstRefUse);
     } else if (isPointerToConst((*I)->getType())) {
       const Expr *Ex = stripCasts(DC->getParentASTContext(), *I);
       const auto *UO = dyn_cast<UnaryOperator>(Ex);
diff --git a/clang/lib/Basic/CMakeLists.txt b/clang/lib/Basic/CMakeLists.txt
index 65cb9c505dd02..1b55d84173771 100644
--- a/clang/lib/Basic/CMakeLists.txt
+++ b/clang/lib/Basic/CMakeLists.txt
@@ -84,6 +84,7 @@ add_clang_library(clangBasic
   Targets/Sparc.cpp
   Targets/SystemZ.cpp
   Targets/TCE.cpp
+  Targets/VE.cpp
   Targets/WebAssembly.cpp
   Targets/X86.cpp
   Targets/XCore.cpp
@@ -94,5 +95,8 @@ add_clang_library(clangBasic
   XRayInstr.cpp
   XRayLists.cpp
   ${version_inc}
+
+  DEPENDS
+  omp_gen
   )
 
diff --git a/clang/lib/Basic/DiagnosticIDs.cpp b/clang/lib/Basic/DiagnosticIDs.cpp
index 6f8b5440a4f3e..8c7e63e063019 100644
--- a/clang/lib/Basic/DiagnosticIDs.cpp
+++ b/clang/lib/Basic/DiagnosticIDs.cpp
@@ -85,6 +85,7 @@ VALIDATE_DIAG_SIZE(LEX)
 VALIDATE_DIAG_SIZE(PARSE)
 VALIDATE_DIAG_SIZE(AST)
 VALIDATE_DIAG_SIZE(COMMENT)
+VALIDATE_DIAG_SIZE(CROSSTU)
 VALIDATE_DIAG_SIZE(SEMA)
 VALIDATE_DIAG_SIZE(ANALYSIS)
 VALIDATE_DIAG_SIZE(REFACTORING)
diff --git a/clang/lib/Basic/FixedPoint.cpp b/clang/lib/Basic/FixedPoint.cpp
index 05600dfc6d212..ed8b92c98fdbb 100644
--- a/clang/lib/Basic/FixedPoint.cpp
+++ b/clang/lib/Basic/FixedPoint.cpp
@@ -173,6 +173,142 @@ APFixedPoint APFixedPoint::add(const APFixedPoint &Other,
   return APFixedPoint(Result, CommonFXSema);
 }
 
+APFixedPoint APFixedPoint::sub(const APFixedPoint &Other,
+                               bool *Overflow) const {
+  auto CommonFXSema = Sema.getCommonSemantics(Other.getSemantics());
+  APFixedPoint ConvertedThis = convert(CommonFXSema);
+  APFixedPoint ConvertedOther = Other.convert(CommonFXSema);
+  llvm::APSInt ThisVal = ConvertedThis.getValue();
+  llvm::APSInt OtherVal = ConvertedOther.getValue();
+  bool Overflowed = false;
+
+  llvm::APSInt Result;
+  if (CommonFXSema.isSaturated()) {
+    Result = CommonFXSema.isSigned() ? ThisVal.ssub_sat(OtherVal)
+                                     : ThisVal.usub_sat(OtherVal);
+  } else {
+    Result = ThisVal.isSigned() ? ThisVal.ssub_ov(OtherVal, Overflowed)
+                                : ThisVal.usub_ov(OtherVal, Overflowed);
+  }
+
+  if (Overflow)
+    *Overflow = Overflowed;
+
+  return APFixedPoint(Result, CommonFXSema);
+}
+
+APFixedPoint APFixedPoint::mul(const APFixedPoint &Other,
+                               bool *Overflow) const {
+  auto CommonFXSema = Sema.getCommonSemantics(Other.getSemantics());
+  APFixedPoint ConvertedThis = convert(CommonFXSema);
+  APFixedPoint ConvertedOther = Other.convert(CommonFXSema);
+  llvm::APSInt ThisVal = ConvertedThis.getValue();
+  llvm::APSInt OtherVal = ConvertedOther.getValue();
+  bool Overflowed = false;
+
+  // Widen the LHS and RHS so we can perform a full multiplication.
+  unsigned Wide = CommonFXSema.getWidth() * 2;
+  if (CommonFXSema.isSigned()) {
+    ThisVal = ThisVal.sextOrSelf(Wide);
+    OtherVal = OtherVal.sextOrSelf(Wide);
+  } else {
+    ThisVal = ThisVal.zextOrSelf(Wide);
+    OtherVal = OtherVal.zextOrSelf(Wide);
+  }
+
+  // Perform the full multiplication and downscale to get the same scale.
+  //
+  // Note that the right shifts here perform an implicit downwards rounding.
+  // This rounding could discard bits that would technically place the result
+  // outside the representable range. We interpret the spec as allowing us to
+  // perform the rounding step first, avoiding the overflow case that would
+  // arise.
+  llvm::APSInt Result;
+  if (CommonFXSema.isSigned())
+    Result = ThisVal.smul_ov(OtherVal, Overflowed)
+                    .ashr(CommonFXSema.getScale());
+  else
+    Result = ThisVal.umul_ov(OtherVal, Overflowed)
+                    .lshr(CommonFXSema.getScale());
+  assert(!Overflowed && "Full multiplication cannot overflow!");
+  Result.setIsSigned(CommonFXSema.isSigned());
+
+  // If our result lies outside of the representative range of the common
+  // semantic, we either have overflow or saturation.
+  llvm::APSInt Max = APFixedPoint::getMax(CommonFXSema).getValue()
+                                                       .extOrTrunc(Wide);
+  llvm::APSInt Min = APFixedPoint::getMin(CommonFXSema).getValue()
+                                                       .extOrTrunc(Wide);
+  if (CommonFXSema.isSaturated()) {
+    if (Result < Min)
+      Result = Min;
+    else if (Result > Max)
+      Result = Max;
+  } else
+    Overflowed = Result < Min || Result > Max;
+
+  if (Overflow)
+    *Overflow = Overflowed;
+
+  return APFixedPoint(Result.sextOrTrunc(CommonFXSema.getWidth()),
+                      CommonFXSema);
+}
+
+APFixedPoint APFixedPoint::div(const APFixedPoint &Other,
+                               bool *Overflow) const {
+  auto CommonFXSema = Sema.getCommonSemantics(Other.getSemantics());
+  APFixedPoint ConvertedThis = convert(CommonFXSema);
+  APFixedPoint ConvertedOther = Other.convert(CommonFXSema);
+  llvm::APSInt ThisVal = ConvertedThis.getValue();
+  llvm::APSInt OtherVal = ConvertedOther.getValue();
+  bool Overflowed = false;
+
+  // Widen the LHS and RHS so we can perform a full division.
+  unsigned Wide = CommonFXSema.getWidth() * 2;
+  if (CommonFXSema.isSigned()) {
+    ThisVal = ThisVal.sextOrSelf(Wide);
+    OtherVal = OtherVal.sextOrSelf(Wide);
+  } else {
+    ThisVal = ThisVal.zextOrSelf(Wide);
+    OtherVal = OtherVal.zextOrSelf(Wide);
+  }
+
+  // Upscale to compensate for the loss of precision from division, and
+  // perform the full division.
+  ThisVal = ThisVal.shl(CommonFXSema.getScale());
+  llvm::APSInt Result;
+  if (CommonFXSema.isSigned()) {
+    llvm::APInt Rem;
+    llvm::APInt::sdivrem(ThisVal, OtherVal, Result, Rem);
+    // If the quotient is negative and the remainder is nonzero, round
+    // towards negative infinity by subtracting epsilon from the result.
+    if (ThisVal.isNegative() != OtherVal.isNegative() && !Rem.isNullValue())
+      Result = Result - 1;
+  } else
+    Result = ThisVal.udiv(OtherVal);
+  Result.setIsSigned(CommonFXSema.isSigned());
+
+  // If our result lies outside of the representative range of the common
+  // semantic, we either have overflow or saturation.
+  llvm::APSInt Max = APFixedPoint::getMax(CommonFXSema).getValue()
+                                                       .extOrTrunc(Wide);
+  llvm::APSInt Min = APFixedPoint::getMin(CommonFXSema).getValue()
+                                                       .extOrTrunc(Wide);
+  if (CommonFXSema.isSaturated()) {
+    if (Result < Min)
+      Result = Min;
+    else if (Result > Max)
+      Result = Max;
+  } else
+    Overflowed = Result < Min || Result > Max;
+
+  if (Overflow)
+    *Overflow = Overflowed;
+
+  return APFixedPoint(Result.sextOrTrunc(CommonFXSema.getWidth()),
+                      CommonFXSema);
+}
+
 void APFixedPoint::toString(llvm::SmallVectorImpl<char> &Str) const {
   llvm::APSInt Val = getValue();
   unsigned Scale = getScale();
diff --git a/clang/lib/Basic/LangOptions.cpp b/clang/lib/Basic/LangOptions.cpp
index a74efdc460bf7..c08670c87fb69 100644
--- a/clang/lib/Basic/LangOptions.cpp
+++ b/clang/lib/Basic/LangOptions.cpp
@@ -24,7 +24,7 @@ void LangOptions::resetNonModularOptions() {
 #define LANGOPT(Name, Bits, Default, Description)
 #define BENIGN_LANGOPT(Name, Bits, Default, Description) Name = Default;
 #define BENIGN_ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
-  Name = Default;
+  Name = static_cast<unsigned>(Default);
 #include "clang/Basic/LangOptions.def"
 
   // These options do not affect AST generation.
@@ -53,6 +53,17 @@ FPOptions FPOptions::defaultWithoutTrailingStorage(const LangOptions &LO) {
   return result;
 }
 
-bool FPOptions::requiresTrailingStorage(const LangOptions &LO) {
-  return getAsOpaqueInt() != defaultWithoutTrailingStorage(LO).getAsOpaqueInt();
+LLVM_DUMP_METHOD void FPOptions::dump() {
+#define OPTION(NAME, TYPE, WIDTH, PREVIOUS)                                    \
+  llvm::errs() << "\n " #NAME " " << get##NAME();
+#include "clang/Basic/FPOptions.def"
+  llvm::errs() << "\n";
+}
+
+LLVM_DUMP_METHOD void FPOptionsOverride::dump() {
+#define OPTION(NAME, TYPE, WIDTH, PREVIOUS)                                    \
+  if (has##NAME##Override())                                                   \
+    llvm::errs() << "\n " #NAME " Override is " << get##NAME##Override();
+#include "clang/Basic/FPOptions.def"
+  llvm::errs() << "\n";
 }
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index a000e4dee3b85..cae61ad4f2e3a 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -178,6 +178,8 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind,
   case OMPC_uses_allocators:
   case OMPC_affinity:
     break;
+  default:
+    break;
   }
   llvm_unreachable("Invalid OpenMP simple clause kind");
 }
@@ -427,6 +429,8 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind,
   case OMPC_uses_allocators:
   case OMPC_affinity:
     break;
+  default:
+    break;
   }
   llvm_unreachable("Invalid OpenMP simple clause kind");
 }
@@ -580,7 +584,7 @@ bool clang::isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind) {
 void clang::getOpenMPCaptureRegions(
     SmallVectorImpl<OpenMPDirectiveKind> &CaptureRegions,
     OpenMPDirectiveKind DKind) {
-  assert(DKind <= OMPD_unknown);
+  assert(unsigned(DKind) < llvm::omp::Directive_enumSize);
   switch (DKind) {
   case OMPD_parallel:
   case OMPD_parallel_for:
@@ -681,6 +685,7 @@ void clang::getOpenMPCaptureRegions(
   case OMPD_end_declare_variant:
     llvm_unreachable("OpenMP Directive is not allowed");
   case OMPD_unknown:
+  default:
     llvm_unreachable("Unknown OpenMP directive");
   }
 }
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 6e4c8e8052bdc..0a76c78cd44fb 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -861,11 +861,8 @@ FileID SourceManager::getFileIDLocal(unsigned SLocOffset) const {
     --I;
     if (I->getOffset() <= SLocOffset) {
       FileID Res = FileID::get(int(I - LocalSLocEntryTable.begin()));
-
-      // If this isn't an expansion, remember it.  We have good locality across
-      // FileID lookups.
-      if (!I->isExpansion())
-        LastFileIDLookup = Res;
+      // Remember it.  We have good locality across FileID lookups.
+      LastFileIDLookup = Res;
       NumLinearScans += NumProbes+1;
       return Res;
     }
@@ -882,11 +879,8 @@ FileID SourceManager::getFileIDLocal(unsigned SLocOffset) const {
   unsigned LessIndex = 0;
   NumProbes = 0;
   while (true) {
-    bool Invalid = false;
     unsigned MiddleIndex = (GreaterIndex-LessIndex)/2+LessIndex;
-    unsigned MidOffset = getLocalSLocEntry(MiddleIndex, &Invalid).getOffset();
-    if (Invalid)
-      return FileID::get(0);
+    unsigned MidOffset = getLocalSLocEntry(MiddleIndex).getOffset();
 
     ++NumProbes;
 
@@ -898,15 +892,12 @@ FileID SourceManager::getFileIDLocal(unsigned SLocOffset) const {
     }
 
     // If the middle index contains the value, succeed and return.
-    // FIXME: This could be made faster by using a function that's aware of
-    // being in the local area.
-    if (isOffsetInFileID(FileID::get(MiddleIndex), SLocOffset)) {
+    if (MiddleIndex + 1 == LocalSLocEntryTable.size() ||
+        SLocOffset < getLocalSLocEntry(MiddleIndex + 1).getOffset()) {
       FileID Res = FileID::get(MiddleIndex);
 
-      // If this isn't a macro expansion, remember it.  We have good locality
-      // across FileID lookups.
-      if (!LocalSLocEntryTable[MiddleIndex].isExpansion())
-        LastFileIDLookup = Res;
+      // Remember it.  We have good locality across FileID lookups.
+      LastFileIDLookup = Res;
       NumBinaryProbes += NumProbes;
       return Res;
     }
@@ -944,9 +935,7 @@ FileID SourceManager::getFileIDLoaded(unsigned SLocOffset) const {
     const SrcMgr::SLocEntry &E = getLoadedSLocEntry(I);
     if (E.getOffset() <= SLocOffset) {
       FileID Res = FileID::get(-int(I) - 2);
-
-      if (!E.isExpansion())
-        LastFileIDLookup = Res;
+      LastFileIDLookup = Res;
       NumLinearScans += NumProbes + 1;
       return Res;
     }
@@ -979,8 +968,7 @@ FileID SourceManager::getFileIDLoaded(unsigned SLocOffset) const {
 
     if (isOffsetInFileID(FileID::get(-int(MiddleIndex) - 2), SLocOffset)) {
       FileID Res = FileID::get(-int(MiddleIndex) - 2);
-      if (!E.isExpansion())
-        LastFileIDLookup = Res;
+      LastFileIDLookup = Res;
       NumBinaryProbes += NumProbes;
       return Res;
     }
@@ -1695,11 +1683,7 @@ FileID SourceManager::translateFile(const FileEntry *SourceFile) const {
   // The location we're looking for isn't in the main file; look
   // through all of the local source locations.
   for (unsigned I = 0, N = local_sloc_entry_size(); I != N; ++I) {
-    bool Invalid = false;
-    const SLocEntry &SLoc = getLocalSLocEntry(I, &Invalid);
-    if (Invalid)
-      return FileID();
-
+    const SLocEntry &SLoc = getLocalSLocEntry(I);
     if (SLoc.isFile() && SLoc.getFile().getContentCache() &&
         SLoc.getFile().getContentCache()->OrigEntry == SourceFile)
       return FileID::get(I);
diff --git a/clang/lib/Basic/TargetInfo.cpp b/clang/lib/Basic/TargetInfo.cpp
index 7f360b715da90..eccdc21d724a1 100644
--- a/clang/lib/Basic/TargetInfo.cpp
+++ b/clang/lib/Basic/TargetInfo.cpp
@@ -37,6 +37,7 @@ TargetInfo::TargetInfo(const llvm::Triple &T) : TargetOpts(), Triple(T) {
   HasFloat128 = false;
   HasFloat16 = false;
   HasBFloat16 = false;
+  HasStrictFP = false;
   PointerWidth = PointerAlign = 32;
   BoolWidth = BoolAlign = 8;
   IntWidth = IntAlign = 32;
diff --git a/clang/lib/Basic/Targets.cpp b/clang/lib/Basic/Targets.cpp
index 69133ca31fec4..6bbcafa27dfe6 100644
--- a/clang/lib/Basic/Targets.cpp
+++ b/clang/lib/Basic/Targets.cpp
@@ -33,6 +33,7 @@
 #include "Targets/Sparc.h"
 #include "Targets/SystemZ.h"
 #include "Targets/TCE.h"
+#include "Targets/VE.h"
 #include "Targets/WebAssembly.h"
 #include "Targets/X86.h"
 #include "Targets/XCore.h"
@@ -613,6 +614,9 @@ TargetInfo *AllocateTarget(const llvm::Triple &Triple,
     return new LinuxTargetInfo<RenderScript32TargetInfo>(Triple, Opts);
   case llvm::Triple::renderscript64:
     return new LinuxTargetInfo<RenderScript64TargetInfo>(Triple, Opts);
+
+  case llvm::Triple::ve:
+    return new LinuxTargetInfo<VETargetInfo>(Triple, Opts);
   }
 }
 } // namespace targets
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index b474d1203deed..25c02cb888c1b 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -260,6 +260,24 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__ARM_NEON_FP", "0xE");
   }
 
+  if (FPU & SveMode)
+    Builder.defineMacro("__ARM_FEATURE_SVE", "1");
+
+  if (HasSVE2)
+    Builder.defineMacro("__ARM_FEATURE_SVE2", "1");
+
+  if (HasSVE2 && HasSVE2AES)
+    Builder.defineMacro("__ARM_FEATURE_SVE2_AES", "1");
+
+  if (HasSVE2 && HasSVE2BitPerm)
+    Builder.defineMacro("__ARM_FEATURE_SVE2_BITPERM", "1");
+
+  if (HasSVE2 && HasSVE2SHA3)
+    Builder.defineMacro("__ARM_FEATURE_SVE2_SHA3", "1");
+
+  if (HasSVE2 && HasSVE2SM4)
+    Builder.defineMacro("__ARM_FEATURE_SVE2_SM4", "1");
+
   if (HasCRC)
     Builder.defineMacro("__ARM_FEATURE_CRC32", "1");
 
@@ -290,8 +308,22 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__ARM_FEATURE_BF16", "1");
     Builder.defineMacro("__ARM_FEATURE_BF16_VECTOR_ARITHMETIC", "1");
     Builder.defineMacro("__ARM_BF16_FORMAT_ALTERNATIVE", "1");
+    Builder.defineMacro("__ARM_FEATURE_BF16_SCALAR_ARITHMETIC", "1");
+  }
+
+  if ((FPU & SveMode) && HasBFloat16) {
+    Builder.defineMacro("__ARM_FEATURE_SVE_BF16", "1");
   }
 
+  if ((FPU & SveMode) && HasMatmulFP64)
+    Builder.defineMacro("__ARM_FEATURE_SVE_MATMUL_FP64", "1");
+
+  if ((FPU & SveMode) && HasMatmulFP32)
+    Builder.defineMacro("__ARM_FEATURE_SVE_MATMUL_FP32", "1");
+
+  if ((FPU & SveMode) && HasMatMul)
+    Builder.defineMacro("__ARM_FEATURE_SVE_MATMUL_INT8", "1");
+
   if ((FPU & NeonMode) && HasFP16FML)
     Builder.defineMacro("__ARM_FEATURE_FP16FML", "1");
 
@@ -354,7 +386,11 @@ ArrayRef<Builtin::Info> AArch64TargetInfo::getTargetBuiltins() const {
 bool AArch64TargetInfo::hasFeature(StringRef Feature) const {
   return Feature == "aarch64" || Feature == "arm64" || Feature == "arm" ||
          (Feature == "neon" && (FPU & NeonMode)) ||
-         (Feature == "sve" && (FPU & SveMode));
+         ((Feature == "sve" || Feature == "sve2" || Feature == "sve2-bitperm" ||
+           Feature == "sve2-aes" || Feature == "sve2-sha3" ||
+           Feature == "sve2-sm4" || Feature == "f64mm" || Feature == "f32mm" ||
+           Feature == "i8mm" || Feature == "bf16") &&
+          (FPU & SveMode));
 }
 
 bool AArch64TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
@@ -370,13 +406,60 @@ bool AArch64TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
   HasTME = false;
   HasMatMul = false;
   HasBFloat16 = false;
+  HasSVE2 = false;
+  HasSVE2AES = false;
+  HasSVE2SHA3 = false;
+  HasSVE2SM4 = false;
+  HasSVE2BitPerm = false;
+  HasMatmulFP64 = false;
+  HasMatmulFP32 = false;
+
   ArchKind = llvm::AArch64::ArchKind::ARMV8A;
 
   for (const auto &Feature : Features) {
     if (Feature == "+neon")
       FPU |= NeonMode;
-    if (Feature == "+sve")
+    if (Feature == "+sve") {
+      FPU |= SveMode;
+      HasFullFP16 = 1;
+    }
+    if (Feature == "+sve2") {
       FPU |= SveMode;
+      HasFullFP16 = 1;
+      HasSVE2 = 1;
+    }
+    if (Feature == "+sve2-aes") {
+      FPU |= SveMode;
+      HasFullFP16 = 1;
+      HasSVE2 = 1;
+      HasSVE2AES = 1;
+    }
+    if (Feature == "+sve2-sha3") {
+      FPU |= SveMode;
+      HasFullFP16 = 1;
+      HasSVE2 = 1;
+      HasSVE2SHA3 = 1;
+    }
+    if (Feature == "+sve2-sm4") {
+      FPU |= SveMode;
+      HasFullFP16 = 1;
+      HasSVE2 = 1;
+      HasSVE2SM4 = 1;
+    }
+    if (Feature == "+sve2-bitperm") {
+      FPU |= SveMode;
+      HasFullFP16 = 1;
+      HasSVE2 = 1;
+      HasSVE2BitPerm = 1;
+    }
+    if (Feature == "+f32mm") {
+      FPU |= SveMode;
+      HasMatmulFP32 = true;
+    }
+    if (Feature == "+f64mm") {
+      FPU |= SveMode;
+      HasMatmulFP64 = true;
+    }
     if (Feature == "+crc")
       HasCRC = true;
     if (Feature == "+crypto")
diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h
index 4a58c6671f4e2..d1982897d84e2 100644
--- a/clang/lib/Basic/Targets/AArch64.h
+++ b/clang/lib/Basic/Targets/AArch64.h
@@ -37,6 +37,13 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
   bool HasMTE;
   bool HasTME;
   bool HasMatMul;
+  bool HasSVE2;
+  bool HasSVE2AES;
+  bool HasSVE2SHA3;
+  bool HasSVE2SM4;
+  bool HasSVE2BitPerm;
+  bool HasMatmulFP64;
+  bool HasMatmulFP32;
 
   llvm::AArch64::ArchKind ArchKind;
 
diff --git a/clang/lib/Basic/Targets/AMDGPU.h b/clang/lib/Basic/Targets/AMDGPU.h
index 387b91abb537d..d0394492cad6d 100644
--- a/clang/lib/Basic/Targets/AMDGPU.h
+++ b/clang/lib/Basic/Targets/AMDGPU.h
@@ -130,8 +130,26 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
         "exec_hi", "tma_lo", "tma_hi", "tba_lo", "tba_hi",
     });
 
+    switch (*Name) {
+    case 'I':
+      Info.setRequiresImmediate(-16, 64);
+      return true;
+    case 'J':
+      Info.setRequiresImmediate(-32768, 32767);
+      return true;
+    case 'A':
+    case 'B':
+    case 'C':
+      Info.setRequiresImmediate();
+      return true;
+    default:
+      break;
+    }
+
     StringRef S(Name);
-    if (S == "A") {
+
+    if (S == "DA" || S == "DB") {
+      Name++;
       Info.setRequiresImmediate();
       return true;
     }
@@ -203,6 +221,12 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
   // the constraint.  In practice, it won't be changed unless the
   // constraint is longer than one character.
   std::string convertConstraint(const char *&Constraint) const override {
+
+    StringRef S(Constraint);
+    if (S == "DA" || S == "DB") {
+      return std::string("^") + std::string(Constraint++, 2);
+    }
+
     const char *Begin = Constraint;
     TargetInfo::ConstraintInfo Info("", "");
     if (validateAsmConstraint(Constraint, Info))
diff --git a/clang/lib/Basic/Targets/MSP430.h b/clang/lib/Basic/Targets/MSP430.h
index 620f12d2b8e3a..9d42e4d4bb183 100644
--- a/clang/lib/Basic/Targets/MSP430.h
+++ b/clang/lib/Basic/Targets/MSP430.h
@@ -64,8 +64,14 @@ class LLVM_LIBRARY_VISIBILITY MSP430TargetInfo : public TargetInfo {
   ArrayRef<const char *> getGCCRegNames() const override;
 
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override {
-    // No aliases.
-    return None;
+    // Make r0 - r3 be recognized by llc (f.e., in clobber list)
+    static const TargetInfo::GCCRegAlias GCCRegAliases[] = {
+        {{"r0"}, "pc"},
+        {{"r1"}, "sp"},
+        {{"r2"}, "sr"},
+        {{"r3"}, "cg"},
+    };
+    return llvm::makeArrayRef(GCCRegAliases);
   }
 
   bool validateAsmConstraint(const char *&Name,
diff --git a/clang/lib/Basic/Targets/PPC.h b/clang/lib/Basic/Targets/PPC.h
index 22a367a5ad42f..858059bacb86b 100644
--- a/clang/lib/Basic/Targets/PPC.h
+++ b/clang/lib/Basic/Targets/PPC.h
@@ -343,13 +343,20 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo {
   const char *getFloat128Mangling() const override { return "u9__ieee128"; }
 
   bool hasExtIntType() const override { return true; }
+
+  bool isSPRegName(StringRef RegName) const override {
+    return RegName.equals("r1") || RegName.equals("x1");
+  }
 };
 
 class LLVM_LIBRARY_VISIBILITY PPC32TargetInfo : public PPCTargetInfo {
 public:
   PPC32TargetInfo(const llvm::Triple &Triple, const TargetOptions &Opts)
       : PPCTargetInfo(Triple, Opts) {
-    resetDataLayout("E-m:e-p:32:32-i64:64-n32");
+    if (Triple.isOSAIX())
+      resetDataLayout("E-m:a-p:32:32-i64:64-n32");
+    else
+      resetDataLayout("E-m:e-p:32:32-i64:64-n32");
 
     switch (getTriple().getOS()) {
     case llvm::Triple::Linux:
@@ -395,7 +402,11 @@ class LLVM_LIBRARY_VISIBILITY PPC64TargetInfo : public PPCTargetInfo {
     IntMaxType = SignedLong;
     Int64Type = SignedLong;
 
-    if ((Triple.getArch() == llvm::Triple::ppc64le)) {
+    if (Triple.isOSAIX()) {
+      // TODO: Set appropriate ABI for AIX platform.
+      resetDataLayout("E-m:a-i64:64-n32:64");
+      SuitableAlign = 64;
+    } else if ((Triple.getArch() == llvm::Triple::ppc64le)) {
       resetDataLayout("e-m:e-i64:64-n32:64");
       ABI = "elfv2";
     } else {
@@ -403,9 +414,6 @@ class LLVM_LIBRARY_VISIBILITY PPC64TargetInfo : public PPCTargetInfo {
       ABI = "elfv1";
     }
 
-    if (Triple.getOS() == llvm::Triple::AIX)
-      SuitableAlign = 64;
-
     if (Triple.isOSFreeBSD() || Triple.getOS() == llvm::Triple::AIX ||
         Triple.isMusl()) {
       LongDoubleWidth = LongDoubleAlign = 64;
diff --git a/clang/lib/Basic/Targets/SystemZ.h b/clang/lib/Basic/Targets/SystemZ.h
index 134b0313b86ab..d7869e3754a82 100644
--- a/clang/lib/Basic/Targets/SystemZ.h
+++ b/clang/lib/Basic/Targets/SystemZ.h
@@ -48,6 +48,7 @@ class LLVM_LIBRARY_VISIBILITY SystemZTargetInfo : public TargetInfo {
     MinGlobalAlign = 16;
     resetDataLayout("E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-a:8:16-n32:64");
     MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 64;
+    HasStrictFP = true;
   }
 
   void getTargetDefines(const LangOptions &Opts,
diff --git a/clang/lib/Basic/Targets/VE.cpp b/clang/lib/Basic/Targets/VE.cpp
new file mode 100644
index 0000000000000..22223654e8adc
--- /dev/null
+++ b/clang/lib/Basic/Targets/VE.cpp
@@ -0,0 +1,39 @@
+//===--- VE.cpp - Implement VE target feature support ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements VE TargetInfo objects.
+//
+//===----------------------------------------------------------------------===//
+
+#include "VE.h"
+#include "clang/Basic/Builtins.h"
+#include "clang/Basic/MacroBuilder.h"
+#include "clang/Basic/TargetBuiltins.h"
+
+using namespace clang;
+using namespace clang::targets;
+
+void VETargetInfo::getTargetDefines(const LangOptions &Opts,
+                                    MacroBuilder &Builder) const {
+  Builder.defineMacro("_LP64", "1");
+  Builder.defineMacro("unix", "1");
+  Builder.defineMacro("__unix__", "1");
+  Builder.defineMacro("__linux__", "1");
+  Builder.defineMacro("__ve", "1");
+  Builder.defineMacro("__ve__", "1");
+  Builder.defineMacro("__STDC_HOSTED__", "1");
+  Builder.defineMacro("__STDC__", "1");
+  Builder.defineMacro("__NEC__", "1");
+  // FIXME: define __FAST_MATH__ 1 if -ffast-math is enabled
+  // FIXME: define __OPTIMIZE__ n if -On is enabled
+  // FIXME: define __VECTOR__ n 1 if automatic vectorization is enabled
+}
+
+ArrayRef<Builtin::Info> VETargetInfo::getTargetBuiltins() const {
+  return ArrayRef<Builtin::Info>();
+}
diff --git a/clang/lib/Basic/Targets/VE.h b/clang/lib/Basic/Targets/VE.h
new file mode 100644
index 0000000000000..f863a0af0acb6
--- /dev/null
+++ b/clang/lib/Basic/Targets/VE.h
@@ -0,0 +1,170 @@
+//===--- VE.h - Declare VE target feature support ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares VE TargetInfo objects.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_BASIC_TARGETS_VE_H
+#define LLVM_CLANG_LIB_BASIC_TARGETS_VE_H
+
+#include "clang/Basic/TargetInfo.h"
+#include "clang/Basic/TargetOptions.h"
+#include "llvm/ADT/Triple.h"
+#include "llvm/Support/Compiler.h"
+
+namespace clang {
+namespace targets {
+
+class LLVM_LIBRARY_VISIBILITY VETargetInfo : public TargetInfo {
+  static const Builtin::Info BuiltinInfo[];
+
+public:
+  VETargetInfo(const llvm::Triple &Triple, const TargetOptions &)
+      : TargetInfo(Triple) {
+    NoAsmVariants = true;
+    LongDoubleWidth = 128;
+    LongDoubleAlign = 128;
+    LongDoubleFormat = &llvm::APFloat::IEEEquad();
+    DoubleAlign = LongLongAlign = 64;
+    SuitableAlign = 64;
+    LongWidth = LongAlign = PointerWidth = PointerAlign = 64;
+    SizeType = UnsignedLong;
+    PtrDiffType = SignedLong;
+    IntPtrType = SignedLong;
+    IntMaxType = SignedLong;
+    Int64Type = SignedLong;
+    RegParmMax = 8;
+    MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 64;
+
+    WCharType = UnsignedInt;
+    WIntType = UnsignedInt;
+    UseZeroLengthBitfieldAlignment = true;
+    resetDataLayout("e-m:e-i64:64-n32:64-S128");
+  }
+
+  void getTargetDefines(const LangOptions &Opts,
+                        MacroBuilder &Builder) const override;
+
+  bool hasSjLjLowering() const override {
+    // TODO
+    return false;
+  }
+
+  ArrayRef<Builtin::Info> getTargetBuiltins() const override;
+
+  BuiltinVaListKind getBuiltinVaListKind() const override {
+    return TargetInfo::VoidPtrBuiltinVaList;
+  }
+
+  CallingConvCheckResult checkCallingConvention(CallingConv CC) const override {
+    switch (CC) {
+    default:
+      return CCCR_Warning;
+    case CC_C:
+      return CCCR_OK;
+    }
+  }
+
+  const char *getClobbers() const override { return ""; }
+
+  ArrayRef<const char *> getGCCRegNames() const override {
+    static const char *const GCCRegNames[] = {
+        // Regular registers
+        "sx0",  "sx1",  "sx2",  "sx3",  "sx4",  "sx5",  "sx6",  "sx7",
+        "sx8",  "sx9",  "sx10", "sx11", "sx12", "sx13", "sx14", "sx15",
+        "sx16", "sx17", "sx18", "sx19", "sx20", "sx21", "sx22", "sx23",
+        "sx24", "sx25", "sx26", "sx27", "sx28", "sx29", "sx30", "sx31",
+        "sx32", "sx33", "sx34", "sx35", "sx36", "sx37", "sx38", "sx39",
+        "sx40", "sx41", "sx42", "sx43", "sx44", "sx45", "sx46", "sx47",
+        "sx48", "sx49", "sx50", "sx51", "sx52", "sx53", "sx54", "sx55",
+        "sx56", "sx57", "sx58", "sx59", "sx60", "sx61", "sx62", "sx63",
+    };
+    return llvm::makeArrayRef(GCCRegNames);
+  }
+
+  ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override {
+    static const TargetInfo::GCCRegAlias GCCRegAliases[] = {
+        {{"s0"}, "sx0"},
+        {{"s1"}, "sx1"},
+        {{"s2"}, "sx2"},
+        {{"s3"}, "sx3"},
+        {{"s4"}, "sx4"},
+        {{"s5"}, "sx5"},
+        {{"s6"}, "sx6"},
+        {{"s7"}, "sx7"},
+        {{"s8", "sl"}, "sx8"},
+        {{"s9", "fp"}, "sx9"},
+        {{"s10", "lr"}, "sx10"},
+        {{"s11", "sp"}, "sx11"},
+        {{"s12", "outer"}, "sx12"},
+        {{"s13"}, "sx13"},
+        {{"s14", "tp"}, "sx14"},
+        {{"s15", "got"}, "sx15"},
+        {{"s16", "plt"}, "sx16"},
+        {{"s17", "info"}, "sx17"},
+        {{"s18"}, "sx18"},
+        {{"s19"}, "sx19"},
+        {{"s20"}, "sx20"},
+        {{"s21"}, "sx21"},
+        {{"s22"}, "sx22"},
+        {{"s23"}, "sx23"},
+        {{"s24"}, "sx24"},
+        {{"s25"}, "sx25"},
+        {{"s26"}, "sx26"},
+        {{"s27"}, "sx27"},
+        {{"s28"}, "sx28"},
+        {{"s29"}, "sx29"},
+        {{"s30"}, "sx30"},
+        {{"s31"}, "sx31"},
+        {{"s32"}, "sx32"},
+        {{"s33"}, "sx33"},
+        {{"s34"}, "sx34"},
+        {{"s35"}, "sx35"},
+        {{"s36"}, "sx36"},
+        {{"s37"}, "sx37"},
+        {{"s38"}, "sx38"},
+        {{"s39"}, "sx39"},
+        {{"s40"}, "sx40"},
+        {{"s41"}, "sx41"},
+        {{"s42"}, "sx42"},
+        {{"s43"}, "sx43"},
+        {{"s44"}, "sx44"},
+        {{"s45"}, "sx45"},
+        {{"s46"}, "sx46"},
+        {{"s47"}, "sx47"},
+        {{"s48"}, "sx48"},
+        {{"s49"}, "sx49"},
+        {{"s50"}, "sx50"},
+        {{"s51"}, "sx51"},
+        {{"s52"}, "sx52"},
+        {{"s53"}, "sx53"},
+        {{"s54"}, "sx54"},
+        {{"s55"}, "sx55"},
+        {{"s56"}, "sx56"},
+        {{"s57"}, "sx57"},
+        {{"s58"}, "sx58"},
+        {{"s59"}, "sx59"},
+        {{"s60"}, "sx60"},
+        {{"s61"}, "sx61"},
+        {{"s62"}, "sx62"},
+        {{"s63"}, "sx63"},
+    };
+    return llvm::makeArrayRef(GCCRegAliases);
+  }
+
+  bool validateAsmConstraint(const char *&Name,
+                             TargetInfo::ConstraintInfo &Info) const override {
+    return false;
+  }
+
+  bool allowsLargerPreferedTypeAlignment() const override { return false; }
+};
+} // namespace targets
+} // namespace clang
+#endif // LLVM_CLANG_LIB_BASIC_TARGETS_VE_H
diff --git a/clang/lib/Basic/Targets/WebAssembly.h b/clang/lib/Basic/Targets/WebAssembly.h
index e09e21d908022..77a2fe9ae1173 100644
--- a/clang/lib/Basic/Targets/WebAssembly.h
+++ b/clang/lib/Basic/Targets/WebAssembly.h
@@ -133,11 +133,7 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyTargetInfo : public TargetInfo {
 
   bool hasExtIntType() const override { return true; }
 
-  bool hasProtectedVisibility() const override {
-    // TODO: For now, continue to advertise "protected" support for
-    // Emscripten targets.
-    return getTriple().isOSEmscripten();
-  }
+  bool hasProtectedVisibility() const override { return false; }
 };
 
 class LLVM_LIBRARY_VISIBILITY WebAssembly32TargetInfo
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index 0092c52b2b231..543f232d24591 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -62,6 +62,7 @@ static const char *const GCCRegNames[] = {
     "cr0",   "cr2",   "cr3",   "cr4",   "cr8",
     "dr0",   "dr1",   "dr2",   "dr3",   "dr6",     "dr7",
     "bnd0",  "bnd1",  "bnd2",  "bnd3",
+    "tmm0",  "tmm1",  "tmm2",  "tmm3",  "tmm4",    "tmm5",  "tmm6",  "tmm7",
 };
 
 const TargetInfo::AddlRegName AddlRegNames[] = {
@@ -107,337 +108,15 @@ bool X86TargetInfo::initFeatureMap(
   // FIXME: This *really* should not be here.
   // X86_64 always has SSE2.
   if (getTriple().getArch() == llvm::Triple::x86_64)
-    setFeatureEnabledImpl(Features, "sse2", true);
+    setFeatureEnabled(Features, "sse2", true);
 
   using namespace llvm::X86;
-  const enum CPUKind Kind = parseArchX86(CPU);
 
-  // Enable X87 for all X86 processors but Lakemont.
-  if (Kind != CK_Lakemont)
-    setFeatureEnabledImpl(Features, "x87", true);
+  SmallVector<StringRef, 16> CPUFeatures;
+  getFeaturesForCPU(CPU, CPUFeatures);
+  for (auto &F : CPUFeatures)
+    setFeatureEnabled(Features, F, true);
 
-  // Enable cmpxchg8 for i586 and greater CPUs. Include generic for backwards
-  // compatibility.
-  if (Kind >= CK_i586 || Kind == CK_None)
-    setFeatureEnabledImpl(Features, "cx8", true);
-
-  switch (Kind) {
-  case CK_None:
-  case CK_i386:
-  case CK_i486:
-  case CK_i586:
-  case CK_Pentium:
-  case CK_PentiumPro:
-  case CK_i686:
-  case CK_Lakemont:
-    break;
-
-  case CK_Cooperlake:
-    // CPX inherits all CLX features plus AVX512BF16
-    setFeatureEnabledImpl(Features, "avx512bf16", true);
-    LLVM_FALLTHROUGH;
-  case CK_Cascadelake:
-    // CLX inherits all SKX features plus AVX512VNNI
-    setFeatureEnabledImpl(Features, "avx512vnni", true);
-    LLVM_FALLTHROUGH;
-  case CK_SkylakeServer:
-    setFeatureEnabledImpl(Features, "avx512f", true);
-    setFeatureEnabledImpl(Features, "avx512cd", true);
-    setFeatureEnabledImpl(Features, "avx512dq", true);
-    setFeatureEnabledImpl(Features, "avx512bw", true);
-    setFeatureEnabledImpl(Features, "avx512vl", true);
-    setFeatureEnabledImpl(Features, "clwb", true);
-    setFeatureEnabledImpl(Features, "pku", true);
-    // SkylakeServer cores inherits all SKL features, except SGX
-    goto SkylakeCommon;
-
-  case CK_Tigerlake:
-    setFeatureEnabledImpl(Features, "avx512vp2intersect", true);
-    setFeatureEnabledImpl(Features, "movdiri", true);
-    setFeatureEnabledImpl(Features, "movdir64b", true);
-    setFeatureEnabledImpl(Features, "shstk", true);
-    // Tigerlake cores inherits IcelakeClient, except pconfig and wbnoinvd
-    goto IcelakeCommon;
-
-  case CK_IcelakeServer:
-    setFeatureEnabledImpl(Features, "pconfig", true);
-    setFeatureEnabledImpl(Features, "wbnoinvd", true);
-    LLVM_FALLTHROUGH;
-  case CK_IcelakeClient:
-IcelakeCommon:
-    setFeatureEnabledImpl(Features, "vaes", true);
-    setFeatureEnabledImpl(Features, "gfni", true);
-    setFeatureEnabledImpl(Features, "vpclmulqdq", true);
-    setFeatureEnabledImpl(Features, "avx512bitalg", true);
-    setFeatureEnabledImpl(Features, "avx512vbmi2", true);
-    setFeatureEnabledImpl(Features, "avx512vnni", true);
-    setFeatureEnabledImpl(Features, "avx512vpopcntdq", true);
-    setFeatureEnabledImpl(Features, "rdpid", true);
-    setFeatureEnabledImpl(Features, "clwb", true);
-    LLVM_FALLTHROUGH;
-  case CK_Cannonlake:
-    setFeatureEnabledImpl(Features, "avx512f", true);
-    setFeatureEnabledImpl(Features, "avx512cd", true);
-    setFeatureEnabledImpl(Features, "avx512dq", true);
-    setFeatureEnabledImpl(Features, "avx512bw", true);
-    setFeatureEnabledImpl(Features, "avx512vl", true);
-    setFeatureEnabledImpl(Features, "avx512ifma", true);
-    setFeatureEnabledImpl(Features, "avx512vbmi", true);
-    setFeatureEnabledImpl(Features, "pku", true);
-    setFeatureEnabledImpl(Features, "sha", true);
-    LLVM_FALLTHROUGH;
-  case CK_SkylakeClient:
-    setFeatureEnabledImpl(Features, "sgx", true);
-    // SkylakeServer cores inherits all SKL features, except SGX
-SkylakeCommon:
-    setFeatureEnabledImpl(Features, "xsavec", true);
-    setFeatureEnabledImpl(Features, "xsaves", true);
-    setFeatureEnabledImpl(Features, "clflushopt", true);
-    setFeatureEnabledImpl(Features, "aes", true);
-    LLVM_FALLTHROUGH;
-  case CK_Broadwell:
-    setFeatureEnabledImpl(Features, "rdseed", true);
-    setFeatureEnabledImpl(Features, "adx", true);
-    setFeatureEnabledImpl(Features, "prfchw", true);
-    LLVM_FALLTHROUGH;
-  case CK_Haswell:
-    setFeatureEnabledImpl(Features, "avx2", true);
-    setFeatureEnabledImpl(Features, "lzcnt", true);
-    setFeatureEnabledImpl(Features, "bmi", true);
-    setFeatureEnabledImpl(Features, "bmi2", true);
-    setFeatureEnabledImpl(Features, "fma", true);
-    setFeatureEnabledImpl(Features, "invpcid", true);
-    setFeatureEnabledImpl(Features, "movbe", true);
-    LLVM_FALLTHROUGH;
-  case CK_IvyBridge:
-    setFeatureEnabledImpl(Features, "rdrnd", true);
-    setFeatureEnabledImpl(Features, "f16c", true);
-    setFeatureEnabledImpl(Features, "fsgsbase", true);
-    LLVM_FALLTHROUGH;
-  case CK_SandyBridge:
-    setFeatureEnabledImpl(Features, "avx", true);
-    setFeatureEnabledImpl(Features, "xsave", true);
-    setFeatureEnabledImpl(Features, "xsaveopt", true);
-    LLVM_FALLTHROUGH;
-  case CK_Westmere:
-    setFeatureEnabledImpl(Features, "pclmul", true);
-    LLVM_FALLTHROUGH;
-  case CK_Nehalem:
-    setFeatureEnabledImpl(Features, "sse4.2", true);
-    LLVM_FALLTHROUGH;
-  case CK_Penryn:
-    setFeatureEnabledImpl(Features, "sse4.1", true);
-    LLVM_FALLTHROUGH;
-  case CK_Core2:
-    setFeatureEnabledImpl(Features, "ssse3", true);
-    setFeatureEnabledImpl(Features, "sahf", true);
-    LLVM_FALLTHROUGH;
-  case CK_Nocona:
-    setFeatureEnabledImpl(Features, "cx16", true);
-    LLVM_FALLTHROUGH;
-  case CK_Yonah:
-  case CK_Prescott:
-    setFeatureEnabledImpl(Features, "sse3", true);
-    LLVM_FALLTHROUGH;
-  case CK_PentiumM:
-  case CK_Pentium4:
-  case CK_x86_64:
-    setFeatureEnabledImpl(Features, "sse2", true);
-    LLVM_FALLTHROUGH;
-  case CK_Pentium3:
-  case CK_C3_2:
-    setFeatureEnabledImpl(Features, "sse", true);
-    LLVM_FALLTHROUGH;
-  case CK_Pentium2:
-    setFeatureEnabledImpl(Features, "fxsr", true);
-    LLVM_FALLTHROUGH;
-  case CK_PentiumMMX:
-  case CK_K6:
-  case CK_WinChipC6:
-    setFeatureEnabledImpl(Features, "mmx", true);
-    break;
-
-  case CK_Tremont:
-    setFeatureEnabledImpl(Features, "clwb", true);
-    setFeatureEnabledImpl(Features, "gfni", true);
-    LLVM_FALLTHROUGH;
-  case CK_GoldmontPlus:
-    setFeatureEnabledImpl(Features, "ptwrite", true);
-    setFeatureEnabledImpl(Features, "rdpid", true);
-    setFeatureEnabledImpl(Features, "sgx", true);
-    LLVM_FALLTHROUGH;
-  case CK_Goldmont:
-    setFeatureEnabledImpl(Features, "sha", true);
-    setFeatureEnabledImpl(Features, "rdseed", true);
-    setFeatureEnabledImpl(Features, "xsave", true);
-    setFeatureEnabledImpl(Features, "xsaveopt", true);
-    setFeatureEnabledImpl(Features, "xsavec", true);
-    setFeatureEnabledImpl(Features, "xsaves", true);
-    setFeatureEnabledImpl(Features, "clflushopt", true);
-    setFeatureEnabledImpl(Features, "fsgsbase", true);
-    setFeatureEnabledImpl(Features, "aes", true);
-    LLVM_FALLTHROUGH;
-  case CK_Silvermont:
-    setFeatureEnabledImpl(Features, "rdrnd", true);
-    setFeatureEnabledImpl(Features, "pclmul", true);
-    setFeatureEnabledImpl(Features, "sse4.2", true);
-    setFeatureEnabledImpl(Features, "prfchw", true);
-    LLVM_FALLTHROUGH;
-  case CK_Bonnell:
-    setFeatureEnabledImpl(Features, "movbe", true);
-    setFeatureEnabledImpl(Features, "ssse3", true);
-    setFeatureEnabledImpl(Features, "fxsr", true);
-    setFeatureEnabledImpl(Features, "cx16", true);
-    setFeatureEnabledImpl(Features, "sahf", true);
-    setFeatureEnabledImpl(Features, "mmx", true);
-    break;
-
-  case CK_KNM:
-    // TODO: Add avx5124fmaps/avx5124vnniw.
-    setFeatureEnabledImpl(Features, "avx512vpopcntdq", true);
-    LLVM_FALLTHROUGH;
-  case CK_KNL:
-    setFeatureEnabledImpl(Features, "avx512f", true);
-    setFeatureEnabledImpl(Features, "avx512cd", true);
-    setFeatureEnabledImpl(Features, "avx512er", true);
-    setFeatureEnabledImpl(Features, "avx512pf", true);
-    setFeatureEnabledImpl(Features, "prfchw", true);
-    setFeatureEnabledImpl(Features, "prefetchwt1", true);
-    setFeatureEnabledImpl(Features, "fxsr", true);
-    setFeatureEnabledImpl(Features, "rdseed", true);
-    setFeatureEnabledImpl(Features, "adx", true);
-    setFeatureEnabledImpl(Features, "lzcnt", true);
-    setFeatureEnabledImpl(Features, "bmi", true);
-    setFeatureEnabledImpl(Features, "bmi2", true);
-    setFeatureEnabledImpl(Features, "fma", true);
-    setFeatureEnabledImpl(Features, "rdrnd", true);
-    setFeatureEnabledImpl(Features, "f16c", true);
-    setFeatureEnabledImpl(Features, "fsgsbase", true);
-    setFeatureEnabledImpl(Features, "aes", true);
-    setFeatureEnabledImpl(Features, "pclmul", true);
-    setFeatureEnabledImpl(Features, "cx16", true);
-    setFeatureEnabledImpl(Features, "xsaveopt", true);
-    setFeatureEnabledImpl(Features, "xsave", true);
-    setFeatureEnabledImpl(Features, "movbe", true);
-    setFeatureEnabledImpl(Features, "sahf", true);
-    setFeatureEnabledImpl(Features, "mmx", true);
-    break;
-
-  case CK_K6_2:
-  case CK_K6_3:
-  case CK_WinChip2:
-  case CK_C3:
-    setFeatureEnabledImpl(Features, "3dnow", true);
-    break;
-
-  case CK_AMDFAM10:
-    setFeatureEnabledImpl(Features, "sse4a", true);
-    setFeatureEnabledImpl(Features, "lzcnt", true);
-    setFeatureEnabledImpl(Features, "popcnt", true);
-    setFeatureEnabledImpl(Features, "sahf", true);
-    LLVM_FALLTHROUGH;
-  case CK_K8SSE3:
-    setFeatureEnabledImpl(Features, "sse3", true);
-    LLVM_FALLTHROUGH;
-  case CK_K8:
-    setFeatureEnabledImpl(Features, "sse2", true);
-    LLVM_FALLTHROUGH;
-  case CK_AthlonXP:
-    setFeatureEnabledImpl(Features, "sse", true);
-    setFeatureEnabledImpl(Features, "fxsr", true);
-    LLVM_FALLTHROUGH;
-  case CK_Athlon:
-  case CK_Geode:
-    setFeatureEnabledImpl(Features, "3dnowa", true);
-    break;
-
-  case CK_BTVER2:
-    setFeatureEnabledImpl(Features, "avx", true);
-    setFeatureEnabledImpl(Features, "aes", true);
-    setFeatureEnabledImpl(Features, "pclmul", true);
-    setFeatureEnabledImpl(Features, "bmi", true);
-    setFeatureEnabledImpl(Features, "f16c", true);
-    setFeatureEnabledImpl(Features, "xsaveopt", true);
-    setFeatureEnabledImpl(Features, "movbe", true);
-    LLVM_FALLTHROUGH;
-  case CK_BTVER1:
-    setFeatureEnabledImpl(Features, "ssse3", true);
-    setFeatureEnabledImpl(Features, "sse4a", true);
-    setFeatureEnabledImpl(Features, "lzcnt", true);
-    setFeatureEnabledImpl(Features, "popcnt", true);
-    setFeatureEnabledImpl(Features, "prfchw", true);
-    setFeatureEnabledImpl(Features, "cx16", true);
-    setFeatureEnabledImpl(Features, "fxsr", true);
-    setFeatureEnabledImpl(Features, "sahf", true);
-    setFeatureEnabledImpl(Features, "mmx", true);
-    break;
-
-  case CK_ZNVER2:
-    setFeatureEnabledImpl(Features, "clwb", true);
-    setFeatureEnabledImpl(Features, "rdpid", true);
-    setFeatureEnabledImpl(Features, "wbnoinvd", true);
-    LLVM_FALLTHROUGH;
-  case CK_ZNVER1:
-    setFeatureEnabledImpl(Features, "adx", true);
-    setFeatureEnabledImpl(Features, "aes", true);
-    setFeatureEnabledImpl(Features, "avx2", true);
-    setFeatureEnabledImpl(Features, "bmi", true);
-    setFeatureEnabledImpl(Features, "bmi2", true);
-    setFeatureEnabledImpl(Features, "clflushopt", true);
-    setFeatureEnabledImpl(Features, "clzero", true);
-    setFeatureEnabledImpl(Features, "cx16", true);
-    setFeatureEnabledImpl(Features, "f16c", true);
-    setFeatureEnabledImpl(Features, "fma", true);
-    setFeatureEnabledImpl(Features, "fsgsbase", true);
-    setFeatureEnabledImpl(Features, "fxsr", true);
-    setFeatureEnabledImpl(Features, "lzcnt", true);
-    setFeatureEnabledImpl(Features, "mmx", true);
-    setFeatureEnabledImpl(Features, "mwaitx", true);
-    setFeatureEnabledImpl(Features, "movbe", true);
-    setFeatureEnabledImpl(Features, "pclmul", true);
-    setFeatureEnabledImpl(Features, "popcnt", true);
-    setFeatureEnabledImpl(Features, "prfchw", true);
-    setFeatureEnabledImpl(Features, "rdrnd", true);
-    setFeatureEnabledImpl(Features, "rdseed", true);
-    setFeatureEnabledImpl(Features, "sahf", true);
-    setFeatureEnabledImpl(Features, "sha", true);
-    setFeatureEnabledImpl(Features, "sse4a", true);
-    setFeatureEnabledImpl(Features, "xsave", true);
-    setFeatureEnabledImpl(Features, "xsavec", true);
-    setFeatureEnabledImpl(Features, "xsaveopt", true);
-    setFeatureEnabledImpl(Features, "xsaves", true);
-    break;
-
-  case CK_BDVER4:
-    setFeatureEnabledImpl(Features, "avx2", true);
-    setFeatureEnabledImpl(Features, "bmi2", true);
-    setFeatureEnabledImpl(Features, "mwaitx", true);
-    LLVM_FALLTHROUGH;
-  case CK_BDVER3:
-    setFeatureEnabledImpl(Features, "fsgsbase", true);
-    setFeatureEnabledImpl(Features, "xsaveopt", true);
-    LLVM_FALLTHROUGH;
-  case CK_BDVER2:
-    setFeatureEnabledImpl(Features, "bmi", true);
-    setFeatureEnabledImpl(Features, "fma", true);
-    setFeatureEnabledImpl(Features, "f16c", true);
-    setFeatureEnabledImpl(Features, "tbm", true);
-    LLVM_FALLTHROUGH;
-  case CK_BDVER1:
-    // xop implies avx, sse4a and fma4.
-    setFeatureEnabledImpl(Features, "xop", true);
-    setFeatureEnabledImpl(Features, "lwp", true);
-    setFeatureEnabledImpl(Features, "lzcnt", true);
-    setFeatureEnabledImpl(Features, "aes", true);
-    setFeatureEnabledImpl(Features, "pclmul", true);
-    setFeatureEnabledImpl(Features, "prfchw", true);
-    setFeatureEnabledImpl(Features, "cx16", true);
-    setFeatureEnabledImpl(Features, "fxsr", true);
-    setFeatureEnabledImpl(Features, "xsave", true);
-    setFeatureEnabledImpl(Features, "sahf", true);
-    setFeatureEnabledImpl(Features, "mmx", true);
-    break;
-  }
   if (!TargetInfo::initFeatureMap(Features, Diags, CPU, FeaturesVec))
     return false;
 
@@ -450,12 +129,6 @@ bool X86TargetInfo::initFeatureMap(
       llvm::find(FeaturesVec, "-popcnt") == FeaturesVec.end())
     Features["popcnt"] = true;
 
-  // Enable prfchw if 3DNow! is enabled and prfchw is not explicitly disabled.
-  I = Features.find("3dnow");
-  if (I != Features.end() && I->getValue() &&
-      llvm::find(FeaturesVec, "-prfchw") == FeaturesVec.end())
-    Features["prfchw"] = true;
-
   // Additionally, if SSE is enabled and mmx is not explicitly disabled,
   // then enable MMX.
   I = Features.find("sse");
@@ -463,264 +136,34 @@ bool X86TargetInfo::initFeatureMap(
       llvm::find(FeaturesVec, "-mmx") == FeaturesVec.end())
     Features["mmx"] = true;
 
-  return true;
-}
-
-void X86TargetInfo::setSSELevel(llvm::StringMap<bool> &Features,
-                                X86SSEEnum Level, bool Enabled) {
-  if (Enabled) {
-    switch (Level) {
-    case AVX512F:
-      Features["avx512f"] = true;
-      Features["fma"] = true;
-      Features["f16c"] = true;
-      LLVM_FALLTHROUGH;
-    case AVX2:
-      Features["avx2"] = true;
-      LLVM_FALLTHROUGH;
-    case AVX:
-      Features["avx"] = true;
-      Features["xsave"] = true;
-      LLVM_FALLTHROUGH;
-    case SSE42:
-      Features["sse4.2"] = true;
-      LLVM_FALLTHROUGH;
-    case SSE41:
-      Features["sse4.1"] = true;
-      LLVM_FALLTHROUGH;
-    case SSSE3:
-      Features["ssse3"] = true;
-      LLVM_FALLTHROUGH;
-    case SSE3:
-      Features["sse3"] = true;
-      LLVM_FALLTHROUGH;
-    case SSE2:
-      Features["sse2"] = true;
-      LLVM_FALLTHROUGH;
-    case SSE1:
-      Features["sse"] = true;
-      LLVM_FALLTHROUGH;
-    case NoSSE:
-      break;
-    }
-    return;
-  }
-
-  switch (Level) {
-  case NoSSE:
-  case SSE1:
-    Features["sse"] = false;
-    LLVM_FALLTHROUGH;
-  case SSE2:
-    Features["sse2"] = Features["pclmul"] = Features["aes"] = false;
-    Features["sha"] = Features["gfni"] = false;
-    LLVM_FALLTHROUGH;
-  case SSE3:
-    Features["sse3"] = false;
-    setXOPLevel(Features, NoXOP, false);
-    LLVM_FALLTHROUGH;
-  case SSSE3:
-    Features["ssse3"] = false;
-    LLVM_FALLTHROUGH;
-  case SSE41:
-    Features["sse4.1"] = false;
-    LLVM_FALLTHROUGH;
-  case SSE42:
-    Features["sse4.2"] = false;
-    LLVM_FALLTHROUGH;
-  case AVX:
-    Features["fma"] = Features["avx"] = Features["f16c"] = false;
-    Features["xsave"] = Features["xsaveopt"] = Features["vaes"] = false;
-    Features["vpclmulqdq"] = false;
-    setXOPLevel(Features, FMA4, false);
-    LLVM_FALLTHROUGH;
-  case AVX2:
-    Features["avx2"] = false;
-    LLVM_FALLTHROUGH;
-  case AVX512F:
-    Features["avx512f"] = Features["avx512cd"] = Features["avx512er"] = false;
-    Features["avx512pf"] = Features["avx512dq"] = Features["avx512bw"] = false;
-    Features["avx512vl"] = Features["avx512vbmi"] = false;
-    Features["avx512ifma"] = Features["avx512vpopcntdq"] = false;
-    Features["avx512bitalg"] = Features["avx512vnni"] = false;
-    Features["avx512vbmi2"] = Features["avx512bf16"] = false;
-    Features["avx512vp2intersect"] = false;
-    break;
-  }
-}
-
-void X86TargetInfo::setMMXLevel(llvm::StringMap<bool> &Features,
-                                MMX3DNowEnum Level, bool Enabled) {
-  if (Enabled) {
-    switch (Level) {
-    case AMD3DNowAthlon:
-      Features["3dnowa"] = true;
-      LLVM_FALLTHROUGH;
-    case AMD3DNow:
-      Features["3dnow"] = true;
-      LLVM_FALLTHROUGH;
-    case MMX:
-      Features["mmx"] = true;
-      LLVM_FALLTHROUGH;
-    case NoMMX3DNow:
-      break;
-    }
-    return;
-  }
-
-  switch (Level) {
-  case NoMMX3DNow:
-  case MMX:
-    Features["mmx"] = false;
-    LLVM_FALLTHROUGH;
-  case AMD3DNow:
-    Features["3dnow"] = false;
-    LLVM_FALLTHROUGH;
-  case AMD3DNowAthlon:
-    Features["3dnowa"] = false;
-    break;
-  }
-}
-
-void X86TargetInfo::setXOPLevel(llvm::StringMap<bool> &Features, XOPEnum Level,
-                                bool Enabled) {
-  if (Enabled) {
-    switch (Level) {
-    case XOP:
-      Features["xop"] = true;
-      LLVM_FALLTHROUGH;
-    case FMA4:
-      Features["fma4"] = true;
-      setSSELevel(Features, AVX, true);
-      LLVM_FALLTHROUGH;
-    case SSE4A:
-      Features["sse4a"] = true;
-      setSSELevel(Features, SSE3, true);
-      LLVM_FALLTHROUGH;
-    case NoXOP:
-      break;
-    }
-    return;
-  }
+  // Enable xsave if avx is enabled and xsave is not explicitly disabled.
+  I = Features.find("avx");
+  if (I != Features.end() && I->getValue() &&
+      llvm::find(FeaturesVec, "-xsave") == FeaturesVec.end())
+    Features["xsave"] = true;
 
-  switch (Level) {
-  case NoXOP:
-  case SSE4A:
-    Features["sse4a"] = false;
-    LLVM_FALLTHROUGH;
-  case FMA4:
-    Features["fma4"] = false;
-    LLVM_FALLTHROUGH;
-  case XOP:
-    Features["xop"] = false;
-    break;
-  }
+  return true;
 }
 
-void X86TargetInfo::setFeatureEnabledImpl(llvm::StringMap<bool> &Features,
-                                          StringRef Name, bool Enabled) {
-  // This is a bit of a hack to deal with the sse4 target feature when used
-  // as part of the target attribute. We handle sse4 correctly everywhere
-  // else. See below for more information on how we handle the sse4 options.
-  if (Name != "sse4")
-    Features[Name] = Enabled;
-
-  if (Name == "mmx") {
-    setMMXLevel(Features, MMX, Enabled);
-  } else if (Name == "sse") {
-    setSSELevel(Features, SSE1, Enabled);
-  } else if (Name == "sse2") {
-    setSSELevel(Features, SSE2, Enabled);
-  } else if (Name == "sse3") {
-    setSSELevel(Features, SSE3, Enabled);
-  } else if (Name == "ssse3") {
-    setSSELevel(Features, SSSE3, Enabled);
-  } else if (Name == "sse4.2") {
-    setSSELevel(Features, SSE42, Enabled);
-  } else if (Name == "sse4.1") {
-    setSSELevel(Features, SSE41, Enabled);
-  } else if (Name == "3dnow") {
-    setMMXLevel(Features, AMD3DNow, Enabled);
-  } else if (Name == "3dnowa") {
-    setMMXLevel(Features, AMD3DNowAthlon, Enabled);
-  } else if (Name == "aes") {
-    if (Enabled)
-      setSSELevel(Features, SSE2, Enabled);
-    else
-      Features["vaes"] = false;
-  } else if (Name == "vaes") {
-    if (Enabled) {
-      setSSELevel(Features, AVX, Enabled);
-      Features["aes"] = true;
-    }
-  } else if (Name == "pclmul") {
-    if (Enabled)
-      setSSELevel(Features, SSE2, Enabled);
-    else
-      Features["vpclmulqdq"] = false;
-  } else if (Name == "vpclmulqdq") {
-    if (Enabled) {
-      setSSELevel(Features, AVX, Enabled);
-      Features["pclmul"] = true;
-    }
-  } else if (Name == "gfni") {
-     if (Enabled)
-      setSSELevel(Features, SSE2, Enabled);
-  } else if (Name == "avx") {
-    setSSELevel(Features, AVX, Enabled);
-  } else if (Name == "avx2") {
-    setSSELevel(Features, AVX2, Enabled);
-  } else if (Name == "avx512f") {
-    setSSELevel(Features, AVX512F, Enabled);
-  } else if (Name.startswith("avx512")) {
-    if (Enabled)
-      setSSELevel(Features, AVX512F, Enabled);
-    // Enable BWI instruction if certain features are being enabled.
-    if ((Name == "avx512vbmi" || Name == "avx512vbmi2" ||
-         Name == "avx512bitalg" || Name == "avx512bf16") && Enabled)
-      Features["avx512bw"] = true;
-    // Also disable some features if BWI is being disabled.
-    if (Name == "avx512bw" && !Enabled) {
-      Features["avx512vbmi"] = false;
-      Features["avx512vbmi2"] = false;
-      Features["avx512bitalg"] = false;
-      Features["avx512bf16"] = false;
-    }
-  } else if (Name == "fma") {
-    if (Enabled)
-      setSSELevel(Features, AVX, Enabled);
-    else
-      setSSELevel(Features, AVX512F, Enabled);
-  } else if (Name == "fma4") {
-    setXOPLevel(Features, FMA4, Enabled);
-  } else if (Name == "xop") {
-    setXOPLevel(Features, XOP, Enabled);
-  } else if (Name == "sse4a") {
-    setXOPLevel(Features, SSE4A, Enabled);
-  } else if (Name == "f16c") {
-    if (Enabled)
-      setSSELevel(Features, AVX, Enabled);
-    else
-      setSSELevel(Features, AVX512F, Enabled);
-  } else if (Name == "sha") {
-    if (Enabled)
-      setSSELevel(Features, SSE2, Enabled);
-  } else if (Name == "sse4") {
+void X86TargetInfo::setFeatureEnabled(llvm::StringMap<bool> &Features,
+                                      StringRef Name, bool Enabled) const {
+  if (Name == "sse4") {
     // We can get here via the __target__ attribute since that's not controlled
     // via the -msse4/-mno-sse4 command line alias. Handle this the same way
     // here - turn on the sse4.2 if enabled, turn off the sse4.1 level if
     // disabled.
     if (Enabled)
-      setSSELevel(Features, SSE42, Enabled);
+      Name = "sse4.2";
     else
-      setSSELevel(Features, SSE41, Enabled);
-  } else if (Name == "xsave") {
-    if (!Enabled)
-      Features["xsaveopt"] = false;
-  } else if (Name == "xsaveopt" || Name == "xsavec" || Name == "xsaves") {
-    if (Enabled)
-      Features["xsave"] = true;
+      Name = "sse4.1";
   }
+
+  Features[Name] = Enabled;
+
+  SmallVector<StringRef, 8> ImpliedFeatures;
+  llvm::X86::getImpliedFeatures(Name, Enabled, ImpliedFeatures);
+  for (const auto &F : ImpliedFeatures)
+    Features[F] = Enabled;
 }
 
 /// handleTargetFeatures - Perform initialization based on the user
@@ -855,6 +298,12 @@ bool X86TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasINVPCID = true;
     } else if (Feature == "+enqcmd") {
       HasENQCMD = true;
+    } else if (Feature == "+amx-bf16") {
+      HasAMXBF16 = true;
+    } else if (Feature == "+amx-int8") {
+      HasAMXINT8 = true;
+    } else if (Feature == "+amx-tile") {
+      HasAMXTILE = true;
     } else if (Feature == "+serialize") {
       HasSERIALIZE = true;
     } else if (Feature == "+tsxldtrk") {
@@ -1250,6 +699,12 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__INVPCID__");
   if (HasENQCMD)
     Builder.defineMacro("__ENQCMD__");
+  if (HasAMXTILE)
+    Builder.defineMacro("__AMXTILE__");
+  if (HasAMXINT8)
+    Builder.defineMacro("__AMXINT8__");
+  if (HasAMXBF16)
+    Builder.defineMacro("__AMXBF16__");
   if (HasSERIALIZE)
     Builder.defineMacro("__SERIALIZE__");
   if (HasTSXLDTRK)
@@ -1346,6 +801,9 @@ bool X86TargetInfo::isValidFeatureName(StringRef Name) const {
       .Case("3dnowa", true)
       .Case("adx", true)
       .Case("aes", true)
+      .Case("amx-bf16", true)
+      .Case("amx-int8", true)
+      .Case("amx-tile", true)
       .Case("avx", true)
       .Case("avx2", true)
       .Case("avx512f", true)
@@ -1428,6 +886,9 @@ bool X86TargetInfo::hasFeature(StringRef Feature) const {
   return llvm::StringSwitch<bool>(Feature)
       .Case("adx", HasADX)
       .Case("aes", HasAES)
+      .Case("amx-bf16", HasAMXBF16)
+      .Case("amx-int8", HasAMXINT8)
+      .Case("amx-tile", HasAMXTILE)
       .Case("avx", SSELevel >= AVX)
       .Case("avx2", SSELevel >= AVX2)
       .Case("avx512f", SSELevel >= AVX512F)
@@ -1518,14 +979,14 @@ bool X86TargetInfo::hasFeature(StringRef Feature) const {
 // X86TargetInfo::hasFeature for a somewhat comprehensive list).
 bool X86TargetInfo::validateCpuSupports(StringRef FeatureStr) const {
   return llvm::StringSwitch<bool>(FeatureStr)
-#define X86_FEATURE_COMPAT(VAL, ENUM, STR) .Case(STR, true)
+#define X86_FEATURE_COMPAT(ENUM, STR) .Case(STR, true)
 #include "llvm/Support/X86TargetParser.def"
       .Default(false);
 }
 
 static llvm::X86::ProcessorFeatures getFeature(StringRef Name) {
   return llvm::StringSwitch<llvm::X86::ProcessorFeatures>(Name)
-#define X86_FEATURE_COMPAT(VAL, ENUM, STR) .Case(STR, llvm::X86::ENUM)
+#define X86_FEATURE_COMPAT(ENUM, STR) .Case(STR, llvm::X86::FEATURE_##ENUM)
 #include "llvm/Support/X86TargetParser.def"
       ;
   // Note, this function should only be used after ensuring the value is
@@ -1553,15 +1014,8 @@ unsigned X86TargetInfo::multiVersionSortPriority(StringRef Name) const {
   using namespace llvm::X86;
   CPUKind Kind = parseArchX86(Name);
   if (Kind != CK_None) {
-    switch (Kind) {
-    default:
-      llvm_unreachable(
-          "CPU Type without a key feature used in 'target' attribute");
-#define PROC_WITH_FEAT(ENUM, STR, IS64, KEY_FEAT)                              \
-  case CK_##ENUM:                                                              \
-    return (getFeaturePriority(llvm::X86::KEY_FEAT) << 1) + 1;
-#include "llvm/Support/X86TargetParser.def"
-    }
+    ProcessorFeatures KeyFeature = getKeyFeature(Kind);
+    return (getFeaturePriority(KeyFeature) << 1) + 1;
   }
 
   // Now we know we have a feature, so get its priority and shift it a few so
@@ -1608,9 +1062,9 @@ void X86TargetInfo::getCPUSpecificCPUDispatchFeatures(
 bool X86TargetInfo::validateCpuIs(StringRef FeatureStr) const {
   return llvm::StringSwitch<bool>(FeatureStr)
 #define X86_VENDOR(ENUM, STRING) .Case(STRING, true)
-#define X86_CPU_TYPE_COMPAT_ALIAS(ENUM, ALIAS) .Case(ALIAS, true)
-#define X86_CPU_TYPE_COMPAT(ARCHNAME, ENUM, STR) .Case(STR, true)
-#define X86_CPU_SUBTYPE_COMPAT(ARCHNAME, ENUM, STR) .Case(STR, true)
+#define X86_CPU_TYPE_ALIAS(ENUM, ALIAS) .Case(ALIAS, true)
+#define X86_CPU_TYPE(ENUM, STR) .Case(STR, true)
+#define X86_CPU_SUBTYPE(ENUM, STR) .Case(STR, true)
 #include "llvm/Support/X86TargetParser.def"
       .Default(false);
 }
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index c33c608e27c84..72a01d2514c23 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -125,6 +125,9 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
   bool HasPTWRITE = false;
   bool HasINVPCID = false;
   bool HasENQCMD = false;
+  bool HasAMXTILE = false;
+  bool HasAMXINT8 = false;
+  bool HasAMXBF16 = false;
   bool HasSERIALIZE = false;
   bool HasTSXLDTRK = false;
 
@@ -138,6 +141,7 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
       : TargetInfo(Triple) {
     LongDoubleFormat = &llvm::APFloat::x87DoubleExtended();
     AddrSpaceMap = &X86AddrSpaceMap;
+    HasStrictFP = true;
   }
 
   const char *getLongDoubleMangling() const override {
@@ -259,24 +263,8 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  static void setSSELevel(llvm::StringMap<bool> &Features, X86SSEEnum Level,
-                          bool Enabled);
-
-  static void setMMXLevel(llvm::StringMap<bool> &Features, MMX3DNowEnum Level,
-                          bool Enabled);
-
-  static void setXOPLevel(llvm::StringMap<bool> &Features, XOPEnum Level,
-                          bool Enabled);
-
   void setFeatureEnabled(llvm::StringMap<bool> &Features, StringRef Name,
-                         bool Enabled) const override {
-    setFeatureEnabledImpl(Features, Name, Enabled);
-  }
-
-  // This exists purely to cut down on the number of virtual calls in
-  // initFeatureMap which calls this repeatedly.
-  static void setFeatureEnabledImpl(llvm::StringMap<bool> &Features,
-                                    StringRef Name, bool Enabled);
+                         bool Enabled) const final;
 
   bool
   initFeatureMap(llvm::StringMap<bool> &Features, DiagnosticsEngine &Diags,
@@ -285,7 +273,7 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
 
   bool isValidFeatureName(StringRef Name) const override;
 
-  bool hasFeature(StringRef Feature) const override;
+  bool hasFeature(StringRef Feature) const final;
 
   bool handleTargetFeatures(std::vector<std::string> &Features,
                             DiagnosticsEngine &Diags) override;
diff --git a/clang/lib/Basic/Version.cpp b/clang/lib/Basic/Version.cpp
index c4b7d34ed1686..286107cab9d7b 100644
--- a/clang/lib/Basic/Version.cpp
+++ b/clang/lib/Basic/Version.cpp
@@ -97,8 +97,12 @@ std::string getClangToolFullVersion(StringRef ToolName) {
 #ifdef CLANG_VENDOR
   OS << CLANG_VENDOR;
 #endif
-  OS << ToolName << " version " CLANG_VERSION_STRING " "
-     << getClangFullRepositoryVersion();
+  OS << ToolName << " version " CLANG_VERSION_STRING;
+
+  std::string repo = getClangFullRepositoryVersion();
+  if (!repo.empty()) {
+    OS << " " << repo;
+  }
 
   return OS.str();
 }
@@ -111,7 +115,13 @@ std::string getClangFullCPPVersion() {
 #ifdef CLANG_VENDOR
   OS << CLANG_VENDOR;
 #endif
-  OS << "Clang " CLANG_VERSION_STRING " " << getClangFullRepositoryVersion();
+  OS << "Clang " CLANG_VERSION_STRING;
+
+  std::string repo = getClangFullRepositoryVersion();
+  if (!repo.empty()) {
+    OS << " " << repo;
+  }
+
   return OS.str();
 }
 
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 9e6d5e4593d3d..dce0940670a24 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -620,6 +620,9 @@ void EmitAssemblyHelper::CreatePasses(legacy::PassManager &MPM,
   PMBuilder.SizeLevel = CodeGenOpts.OptimizeSize;
   PMBuilder.SLPVectorize = CodeGenOpts.VectorizeSLP;
   PMBuilder.LoopVectorize = CodeGenOpts.VectorizeLoop;
+  // Only enable CGProfilePass when using integrated assembler, since
+  // non-integrated assemblers don't recognize .cgprofile section.
+  PMBuilder.CallGraphProfile = !CodeGenOpts.DisableIntegratedAS;
 
   PMBuilder.DisableUnrollLoops = !CodeGenOpts.UnrollLoops;
   // Loop interleaving in the loop vectorizer has historically been set to be
@@ -1144,7 +1147,9 @@ void EmitAssemblyHelper::EmitAssemblyWithNewPassManager(
   PTO.LoopInterleaving = CodeGenOpts.UnrollLoops;
   PTO.LoopVectorization = CodeGenOpts.VectorizeLoop;
   PTO.SLPVectorization = CodeGenOpts.VectorizeSLP;
-  PTO.CallGraphProfile = CodeGenOpts.CallGraphProfile;
+  // Only enable CGProfilePass when using integrated assembler, since
+  // non-integrated assemblers don't recognize .cgprofile section.
+  PTO.CallGraphProfile = !CodeGenOpts.DisableIntegratedAS;
   PTO.Coroutines = LangOpts.Coroutines;
 
   PassInstrumentationCallbacks PIC;
@@ -1562,7 +1567,9 @@ static void runThinLTOBackend(
   Conf.PTO.LoopInterleaving = CGOpts.UnrollLoops;
   Conf.PTO.LoopVectorization = CGOpts.VectorizeLoop;
   Conf.PTO.SLPVectorization = CGOpts.VectorizeSLP;
-  Conf.PTO.CallGraphProfile = CGOpts.CallGraphProfile;
+  // Only enable CGProfilePass when using integrated assembler, since
+  // non-integrated assemblers don't recognize .cgprofile section.
+  Conf.PTO.CallGraphProfile = !CGOpts.DisableIntegratedAS;
 
   // Context sensitive profile.
   if (CGOpts.hasProfileCSIRInstr()) {
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 7943c3f34504f..35a93a7889f40 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -219,8 +219,9 @@ static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
       Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
   Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]);
   if (Invert)
-    Result = CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result,
-                                     llvm::ConstantInt::get(IntType, -1));
+    Result =
+        CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result,
+                                llvm::ConstantInt::getAllOnesValue(IntType));
   Result = EmitFromInt(CGF, Result, T, ValueType);
   return RValue::get(Result);
 }
@@ -2194,6 +2195,33 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
         Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval");
     return RValue::get(Result);
   }
+  case Builtin::BI__builtin_expect_with_probability: {
+    Value *ArgValue = EmitScalarExpr(E->getArg(0));
+    llvm::Type *ArgType = ArgValue->getType();
+
+    Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
+    llvm::APFloat Probability(0.0);
+    const Expr *ProbArg = E->getArg(2);
+    bool EvalSucceed = ProbArg->EvaluateAsFloat(Probability, CGM.getContext());
+    assert(EvalSucceed && "probability should be able to evaluate as float");
+    (void)EvalSucceed;
+    bool LoseInfo = false;
+    Probability.convert(llvm::APFloat::IEEEdouble(),
+                        llvm::RoundingMode::Dynamic, &LoseInfo);
+    llvm::Type *Ty = ConvertType(ProbArg->getType());
+    Constant *Confidence = ConstantFP::get(Ty, Probability);
+    // Don't generate llvm.expect.with.probability on -O0 as the backend
+    // won't use it for anything.
+    // Note, we still IRGen ExpectedValue because it could have side-effects.
+    if (CGM.getCodeGenOpts().OptimizationLevel == 0)
+      return RValue::get(ArgValue);
+
+    Function *FnExpect =
+        CGM.getIntrinsic(Intrinsic::expect_with_probability, ArgType);
+    Value *Result = Builder.CreateCall(
+        FnExpect, {ArgValue, ExpectedValue, Confidence}, "expval");
+    return RValue::get(Result);
+  }
   case Builtin::BI__builtin_assume_aligned: {
     const Expr *Ptr = E->getArg(0);
     Value *PtrValue = EmitScalarExpr(Ptr);
@@ -4708,6 +4736,7 @@ struct ARMVectorIntrinsicInfo {
       TypeModifier }
 
 static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = {
+  NEONMAP1(__a32_vcvt_bf16_v, arm_neon_vcvtfp2bf, 0),
   NEONMAP0(splat_lane_v),
   NEONMAP0(splat_laneq_v),
   NEONMAP0(splatq_lane_v),
@@ -4721,6 +4750,11 @@ static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = {
   NEONMAP1(vaeseq_v, arm_neon_aese, 0),
   NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0),
   NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0),
+  NEONMAP1(vbfdot_v, arm_neon_bfdot, 0),
+  NEONMAP1(vbfdotq_v, arm_neon_bfdot, 0),
+  NEONMAP1(vbfmlalbq_v, arm_neon_bfmlalb, 0),
+  NEONMAP1(vbfmlaltq_v, arm_neon_bfmlalt, 0),
+  NEONMAP1(vbfmmlaq_v, arm_neon_bfmmla, 0),
   NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType),
   NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType),
   NEONMAP1(vcadd_rot270_v, arm_neon_vcadd_rot270, Add1ArgType),
@@ -4781,6 +4815,7 @@ static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = {
   NEONMAP1(vcvtaq_u16_v, arm_neon_vcvtau, 0),
   NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
   NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
+  NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
   NEONMAP1(vcvtm_s16_v, arm_neon_vcvtms, 0),
   NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
   NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
@@ -4997,6 +5032,7 @@ static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = {
 };
 
 static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[] = {
+  NEONMAP1(__a64_vcvtq_low_bf16_v, aarch64_neon_bfcvtn, 0),
   NEONMAP0(splat_lane_v),
   NEONMAP0(splat_laneq_v),
   NEONMAP0(splatq_lane_v),
@@ -5056,6 +5092,7 @@ static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[] = {
   NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
   NEONMAP0(vcvtq_f16_v),
   NEONMAP0(vcvtq_f32_v),
+  NEONMAP1(vcvtq_high_bf16_v, aarch64_neon_bfcvtn2, 0),
   NEONMAP2(vcvtq_n_f16_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
   NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
   NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
@@ -5211,6 +5248,7 @@ static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[] = {
   NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
   NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
   NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
+  NEONMAP1(vcvth_bf16_f32, aarch64_neon_bfcvt, 0),
   NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
   NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
   NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
@@ -6209,6 +6247,11 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
     llvm::Type *Tys[2] = { Ty, InputTy };
     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vbfmlalt");
   }
+  case NEON::BI__builtin_neon___a32_vcvt_bf16_v: {
+    llvm::Type *Tys[1] = { Ty };
+    Function *F = CGM.getIntrinsic(Int, Tys);
+    return EmitNeonCall(F, Ops, "vcvtfp2bf");
+  }
 
   }
 
@@ -6389,21 +6432,27 @@ static bool HasExtraNeonArgument(unsigned BuiltinID) {
   default: break;
   case NEON::BI__builtin_neon_vget_lane_i8:
   case NEON::BI__builtin_neon_vget_lane_i16:
+  case NEON::BI__builtin_neon_vget_lane_bf16:
   case NEON::BI__builtin_neon_vget_lane_i32:
   case NEON::BI__builtin_neon_vget_lane_i64:
   case NEON::BI__builtin_neon_vget_lane_f32:
   case NEON::BI__builtin_neon_vgetq_lane_i8:
   case NEON::BI__builtin_neon_vgetq_lane_i16:
+  case NEON::BI__builtin_neon_vgetq_lane_bf16:
   case NEON::BI__builtin_neon_vgetq_lane_i32:
   case NEON::BI__builtin_neon_vgetq_lane_i64:
   case NEON::BI__builtin_neon_vgetq_lane_f32:
+  case NEON::BI__builtin_neon_vduph_lane_bf16:
+  case NEON::BI__builtin_neon_vduph_laneq_bf16:
   case NEON::BI__builtin_neon_vset_lane_i8:
   case NEON::BI__builtin_neon_vset_lane_i16:
+  case NEON::BI__builtin_neon_vset_lane_bf16:
   case NEON::BI__builtin_neon_vset_lane_i32:
   case NEON::BI__builtin_neon_vset_lane_i64:
   case NEON::BI__builtin_neon_vset_lane_f32:
   case NEON::BI__builtin_neon_vsetq_lane_i8:
   case NEON::BI__builtin_neon_vsetq_lane_i16:
+  case NEON::BI__builtin_neon_vsetq_lane_bf16:
   case NEON::BI__builtin_neon_vsetq_lane_i32:
   case NEON::BI__builtin_neon_vsetq_lane_i64:
   case NEON::BI__builtin_neon_vsetq_lane_f32:
@@ -6411,6 +6460,7 @@ static bool HasExtraNeonArgument(unsigned BuiltinID) {
   case NEON::BI__builtin_neon_vsha1cq_u32:
   case NEON::BI__builtin_neon_vsha1pq_u32:
   case NEON::BI__builtin_neon_vsha1mq_u32:
+  case NEON::BI__builtin_neon_vcvth_bf16_f32:
   case clang::ARM::BI_MoveToCoprocessor:
   case clang::ARM::BI_MoveToCoprocessor2:
     return false;
@@ -6849,12 +6899,16 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case NEON::BI__builtin_neon_vget_lane_i16:
   case NEON::BI__builtin_neon_vget_lane_i32:
   case NEON::BI__builtin_neon_vget_lane_i64:
+  case NEON::BI__builtin_neon_vget_lane_bf16:
   case NEON::BI__builtin_neon_vget_lane_f32:
   case NEON::BI__builtin_neon_vgetq_lane_i8:
   case NEON::BI__builtin_neon_vgetq_lane_i16:
   case NEON::BI__builtin_neon_vgetq_lane_i32:
   case NEON::BI__builtin_neon_vgetq_lane_i64:
+  case NEON::BI__builtin_neon_vgetq_lane_bf16:
   case NEON::BI__builtin_neon_vgetq_lane_f32:
+  case NEON::BI__builtin_neon_vduph_lane_bf16:
+  case NEON::BI__builtin_neon_vduph_laneq_bf16:
     return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
 
   case NEON::BI__builtin_neon_vrndns_f32: {
@@ -6867,11 +6921,13 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case NEON::BI__builtin_neon_vset_lane_i16:
   case NEON::BI__builtin_neon_vset_lane_i32:
   case NEON::BI__builtin_neon_vset_lane_i64:
+  case NEON::BI__builtin_neon_vset_lane_bf16:
   case NEON::BI__builtin_neon_vset_lane_f32:
   case NEON::BI__builtin_neon_vsetq_lane_i8:
   case NEON::BI__builtin_neon_vsetq_lane_i16:
   case NEON::BI__builtin_neon_vsetq_lane_i32:
   case NEON::BI__builtin_neon_vsetq_lane_i64:
+  case NEON::BI__builtin_neon_vsetq_lane_bf16:
   case NEON::BI__builtin_neon_vsetq_lane_f32:
     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
 
@@ -6888,6 +6944,11 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops,
                         "vsha1h");
 
+  case NEON::BI__builtin_neon_vcvth_bf16_f32: {
+    return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vcvtbfp2bf), Ops,
+                        "vcvtbfp2bf");
+  }
+
   // The ARM _MoveToCoprocessor builtins put the input register value as
   // the first argument, but the LLVM intrinsic expects it as the third one.
   case ARM::BI_MoveToCoprocessor:
@@ -7666,6 +7727,8 @@ CodeGenFunction::getSVEPredType(SVETypeFlags TypeFlags) {
   case SVETypeFlags::EltTyInt64:
     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
 
+  case SVETypeFlags::EltTyBFloat16:
+    return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
   case SVETypeFlags::EltTyFloat16:
     return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
   case SVETypeFlags::EltTyFloat32:
@@ -8324,6 +8387,7 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
   case SVE::BI__builtin_sve_svdupq_n_s64:
   case SVE::BI__builtin_sve_svdupq_n_u16:
   case SVE::BI__builtin_sve_svdupq_n_f16:
+  case SVE::BI__builtin_sve_svdupq_n_bf16:
   case SVE::BI__builtin_sve_svdupq_n_s16:
   case SVE::BI__builtin_sve_svdupq_n_u32:
   case SVE::BI__builtin_sve_svdupq_n_f32:
@@ -8374,6 +8438,7 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
   case SVE::BI__builtin_sve_svpfalse_b:
     return ConstantInt::getFalse(Ty);
 
+  case SVE::BI__builtin_sve_svlen_bf16:
   case SVE::BI__builtin_sve_svlen_f16:
   case SVE::BI__builtin_sve_svlen_f32:
   case SVE::BI__builtin_sve_svlen_f64:
@@ -8402,6 +8467,7 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
   case SVE::BI__builtin_sve_svtbl2_u64:
   case SVE::BI__builtin_sve_svtbl2_s64:
   case SVE::BI__builtin_sve_svtbl2_f16:
+  case SVE::BI__builtin_sve_svtbl2_bf16:
   case SVE::BI__builtin_sve_svtbl2_f32:
   case SVE::BI__builtin_sve_svtbl2_f64: {
     SVETypeFlags TF(Builtin->TypeModifier);
@@ -9282,11 +9348,13 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
   case NEON::BI__builtin_neon_vset_lane_i16:
   case NEON::BI__builtin_neon_vset_lane_i32:
   case NEON::BI__builtin_neon_vset_lane_i64:
+  case NEON::BI__builtin_neon_vset_lane_bf16:
   case NEON::BI__builtin_neon_vset_lane_f32:
   case NEON::BI__builtin_neon_vsetq_lane_i8:
   case NEON::BI__builtin_neon_vsetq_lane_i16:
   case NEON::BI__builtin_neon_vsetq_lane_i32:
   case NEON::BI__builtin_neon_vsetq_lane_i64:
+  case NEON::BI__builtin_neon_vsetq_lane_bf16:
   case NEON::BI__builtin_neon_vsetq_lane_f32:
     Ops.push_back(EmitScalarExpr(E->getArg(2)));
     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
@@ -9565,11 +9633,13 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
                           : Intrinsic::aarch64_neon_sqsub;
     return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl");
   }
+  case NEON::BI__builtin_neon_vget_lane_bf16:
   case NEON::BI__builtin_neon_vduph_lane_bf16:
   case NEON::BI__builtin_neon_vduph_lane_f16: {
     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
                                         "vget_lane");
   }
+  case NEON::BI__builtin_neon_vgetq_lane_bf16:
   case NEON::BI__builtin_neon_vduph_laneq_bf16:
   case NEON::BI__builtin_neon_vduph_laneq_f16: {
     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
@@ -11585,11 +11655,11 @@ Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
   std::tie(Index, Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
 #define X86_VENDOR(ENUM, STRING)                                               \
   .Case(STRING, {0u, static_cast<unsigned>(llvm::X86::ENUM)})
-#define X86_CPU_TYPE_COMPAT_ALIAS(ENUM, ALIAS)             \
+#define X86_CPU_TYPE_ALIAS(ENUM, ALIAS)                                        \
   .Case(ALIAS, {1u, static_cast<unsigned>(llvm::X86::ENUM)})
-#define X86_CPU_TYPE_COMPAT(ARCHNAME, ENUM, STR)                               \
+#define X86_CPU_TYPE(ENUM, STR)                                                \
   .Case(STR, {1u, static_cast<unsigned>(llvm::X86::ENUM)})
-#define X86_CPU_SUBTYPE_COMPAT(ARCHNAME, ENUM, STR)                            \
+#define X86_CPU_SUBTYPE(ENUM, STR)                                             \
   .Case(STR, {2u, static_cast<unsigned>(llvm::X86::ENUM)})
 #include "llvm/Support/X86TargetParser.def"
                                .Default({0, 0});
@@ -11619,7 +11689,7 @@ CodeGenFunction::GetX86CpuSupportsMask(ArrayRef<StringRef> FeatureStrs) {
   for (const StringRef &FeatureStr : FeatureStrs) {
     unsigned Feature =
         StringSwitch<unsigned>(FeatureStr)
-#define X86_FEATURE_COMPAT(VAL, ENUM, STR) .Case(STR, VAL)
+#define X86_FEATURE_COMPAT(ENUM, STR) .Case(STR, llvm::X86::FEATURE_##ENUM)
 #include "llvm/Support/X86TargetParser.def"
         ;
     FeaturesMask |= (1ULL << Feature);
@@ -14168,9 +14238,14 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
   case PPC::BI__builtin_vsx_xvsqrtdp: {
     llvm::Type *ResultType = ConvertType(E->getType());
     Value *X = EmitScalarExpr(E->getArg(0));
-    ID = Intrinsic::sqrt;
-    llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
-    return Builder.CreateCall(F, X);
+    if (Builder.getIsFPConstrained()) {
+      llvm::Function *F = CGM.getIntrinsic(
+          Intrinsic::experimental_constrained_sqrt, ResultType);
+      return Builder.CreateConstrainedFPCall(F, X);
+    } else {
+      llvm::Function *F = CGM.getIntrinsic(Intrinsic::sqrt, ResultType);
+      return Builder.CreateCall(F, X);
+    }
   }
   // Count leading zeros
   case PPC::BI__builtin_altivec_vclzb:
@@ -14227,21 +14302,32 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
     Value *X = EmitScalarExpr(E->getArg(0));
     if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
         BuiltinID == PPC::BI__builtin_vsx_xvrspim)
-      ID = Intrinsic::floor;
+      ID = Builder.getIsFPConstrained()
+               ? Intrinsic::experimental_constrained_floor
+               : Intrinsic::floor;
     else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
              BuiltinID == PPC::BI__builtin_vsx_xvrspi)
-      ID = Intrinsic::round;
+      ID = Builder.getIsFPConstrained()
+               ? Intrinsic::experimental_constrained_round
+               : Intrinsic::round;
     else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
              BuiltinID == PPC::BI__builtin_vsx_xvrspic)
-      ID = Intrinsic::nearbyint;
+      ID = Builder.getIsFPConstrained()
+               ? Intrinsic::experimental_constrained_nearbyint
+               : Intrinsic::nearbyint;
     else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
              BuiltinID == PPC::BI__builtin_vsx_xvrspip)
-      ID = Intrinsic::ceil;
+      ID = Builder.getIsFPConstrained()
+               ? Intrinsic::experimental_constrained_ceil
+               : Intrinsic::ceil;
     else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
              BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
-      ID = Intrinsic::trunc;
+      ID = Builder.getIsFPConstrained()
+               ? Intrinsic::experimental_constrained_trunc
+               : Intrinsic::trunc;
     llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
-    return Builder.CreateCall(F, X);
+    return Builder.getIsFPConstrained() ? Builder.CreateConstrainedFPCall(F, X)
+                                        : Builder.CreateCall(F, X);
   }
 
   // Absolute value
@@ -14266,21 +14352,43 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
     Value *X = EmitScalarExpr(E->getArg(0));
     Value *Y = EmitScalarExpr(E->getArg(1));
     Value *Z = EmitScalarExpr(E->getArg(2));
-    llvm::Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
+    llvm::Function *F;
+    if (Builder.getIsFPConstrained())
+      F = CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
+    else
+      F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
     switch (BuiltinID) {
       case PPC::BI__builtin_vsx_xvmaddadp:
       case PPC::BI__builtin_vsx_xvmaddasp:
-        return Builder.CreateCall(F, {X, Y, Z});
+        if (Builder.getIsFPConstrained())
+          return Builder.CreateConstrainedFPCall(F, {X, Y, Z});
+        else
+          return Builder.CreateCall(F, {X, Y, Z});
       case PPC::BI__builtin_vsx_xvnmaddadp:
       case PPC::BI__builtin_vsx_xvnmaddasp:
-        return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, Z}), "neg");
+        if (Builder.getIsFPConstrained())
+          return Builder.CreateFNeg(
+              Builder.CreateConstrainedFPCall(F, {X, Y, Z}), "neg");
+        else
+          return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, Z}), "neg");
       case PPC::BI__builtin_vsx_xvmsubadp:
       case PPC::BI__builtin_vsx_xvmsubasp:
-        return Builder.CreateCall(F, {X, Y, Builder.CreateFNeg(Z, "neg")});
+        if (Builder.getIsFPConstrained())
+          return Builder.CreateConstrainedFPCall(
+              F, {X, Y, Builder.CreateFNeg(Z, "neg")});
+        else
+          return Builder.CreateCall(F, {X, Y, Builder.CreateFNeg(Z, "neg")});
       case PPC::BI__builtin_vsx_xvnmsubadp:
       case PPC::BI__builtin_vsx_xvnmsubasp:
-        return Builder.CreateFNeg(
-            Builder.CreateCall(F, {X, Y, Builder.CreateFNeg(Z, "neg")}), "neg");
+        if (Builder.getIsFPConstrained())
+          return Builder.CreateFNeg(
+              Builder.CreateConstrainedFPCall(
+                  F, {X, Y, Builder.CreateFNeg(Z, "neg")}),
+              "neg");
+        else
+          return Builder.CreateFNeg(
+              Builder.CreateCall(F, {X, Y, Builder.CreateFNeg(Z, "neg")}),
+              "neg");
     }
     llvm_unreachable("Unknown FMA operation");
     return nullptr; // Suppress no-return warning
@@ -14596,6 +14704,10 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
   case AMDGPU::BI__builtin_amdgcn_rcpf:
   case AMDGPU::BI__builtin_amdgcn_rcph:
     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rcp);
+  case AMDGPU::BI__builtin_amdgcn_sqrt:
+  case AMDGPU::BI__builtin_amdgcn_sqrtf:
+  case AMDGPU::BI__builtin_amdgcn_sqrth:
+    return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_sqrt);
   case AMDGPU::BI__builtin_amdgcn_rsq:
   case AMDGPU::BI__builtin_amdgcn_rsqf:
   case AMDGPU::BI__builtin_amdgcn_rsqh:
@@ -15990,30 +16102,6 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_grow, ResultType);
     return Builder.CreateCall(Callee, Args);
   }
-  case WebAssembly::BI__builtin_wasm_memory_init: {
-    llvm::APSInt SegConst;
-    if (!E->getArg(0)->isIntegerConstantExpr(SegConst, getContext()))
-      llvm_unreachable("Constant arg isn't actually constant?");
-    llvm::APSInt MemConst;
-    if (!E->getArg(1)->isIntegerConstantExpr(MemConst, getContext()))
-      llvm_unreachable("Constant arg isn't actually constant?");
-    if (!MemConst.isNullValue())
-      ErrorUnsupported(E, "non-zero memory index");
-    Value *Args[] = {llvm::ConstantInt::get(getLLVMContext(), SegConst),
-                     llvm::ConstantInt::get(getLLVMContext(), MemConst),
-                     EmitScalarExpr(E->getArg(2)), EmitScalarExpr(E->getArg(3)),
-                     EmitScalarExpr(E->getArg(4))};
-    Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_init);
-    return Builder.CreateCall(Callee, Args);
-  }
-  case WebAssembly::BI__builtin_wasm_data_drop: {
-    llvm::APSInt SegConst;
-    if (!E->getArg(0)->isIntegerConstantExpr(SegConst, getContext()))
-      llvm_unreachable("Constant arg isn't actually constant?");
-    Value *Arg = llvm::ConstantInt::get(getLLVMContext(), SegConst);
-    Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_data_drop);
-    return Builder.CreateCall(Callee, {Arg});
-  }
   case WebAssembly::BI__builtin_wasm_tls_size: {
     llvm::Type *ResultType = ConvertType(E->getType());
     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_tls_size, ResultType);
diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp
index 351c5058aa4c9..baf2c79cc2b66 100644
--- a/clang/lib/CodeGen/CGCUDANV.cpp
+++ b/clang/lib/CodeGen/CGCUDANV.cpp
@@ -242,7 +242,7 @@ void CGNVCUDARuntime::emitDeviceStub(CodeGenFunction &CGF,
   EmittedKernels.push_back({CGF.CurFn, CGF.CurFuncDecl});
   if (CudaFeatureEnabled(CGM.getTarget().getSDKVersion(),
                          CudaFeature::CUDA_USES_NEW_LAUNCH) ||
-      CGF.getLangOpts().HIPUseNewLaunchAPI)
+      (CGF.getLangOpts().HIP && CGF.getLangOpts().HIPUseNewLaunchAPI))
     emitDeviceStubBodyNew(CGF, Args);
   else
     emitDeviceStubBodyLegacy(CGF, Args);
diff --git a/clang/lib/CodeGen/CGCXXABI.cpp b/clang/lib/CodeGen/CGCXXABI.cpp
index 928fbaea82787..65327a2435b5b 100644
--- a/clang/lib/CodeGen/CGCXXABI.cpp
+++ b/clang/lib/CodeGen/CGCXXABI.cpp
@@ -156,6 +156,8 @@ void CGCXXABI::setCXXABIThisValue(CodeGenFunction &CGF, llvm::Value *ThisPtr) {
 
 void CGCXXABI::EmitReturnFromThunk(CodeGenFunction &CGF,
                                    RValue RV, QualType ResultType) {
+  assert(!CGF.hasAggregateEvaluationKind(ResultType) &&
+         "cannot handle aggregates");
   CGF.EmitReturnOfRValue(RV, ResultType);
 }
 
diff --git a/clang/lib/CodeGen/CGCXXABI.h b/clang/lib/CodeGen/CGCXXABI.h
index 2b7f45fcab988..f5b3fc13bbbd4 100644
--- a/clang/lib/CodeGen/CGCXXABI.h
+++ b/clang/lib/CodeGen/CGCXXABI.h
@@ -402,6 +402,13 @@ class CGCXXABI {
                              CXXCtorType Type, bool ForVirtualBase,
                              bool Delegating, CallArgList &Args);
 
+  /// Get the implicit (second) parameter that comes after the "this" pointer,
+  /// or nullptr if there is isn't one.
+  virtual llvm::Value *
+  getCXXDestructorImplicitParam(CodeGenFunction &CGF,
+                                const CXXDestructorDecl *DD, CXXDtorType Type,
+                                bool ForVirtualBase, bool Delegating) = 0;
+
   /// Emit the destructor call.
   virtual void EmitDestructorCall(CodeGenFunction &CGF,
                                   const CXXDestructorDecl *DD, CXXDtorType Type,
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 87242442a57f8..e8235c775d8f5 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -2504,9 +2504,11 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
                   ArrSize) {
                 llvm::AttrBuilder Attrs;
                 Attrs.addDereferenceableAttr(
-                  getContext().getTypeSizeInChars(ETy).getQuantity()*ArrSize);
+                    getContext().getTypeSizeInChars(ETy).getQuantity() *
+                    ArrSize);
                 AI->addAttrs(Attrs);
-              } else if (getContext().getTargetAddressSpace(ETy) == 0 &&
+              } else if (getContext().getTargetInfo().getNullPointerValue(
+                             ETy.getAddressSpace()) == 0 &&
                          !CGM.getCodeGenOpts().NullPointerIsValid) {
                 AI->addAttr(llvm::Attribute::NonNull);
               }
@@ -4245,7 +4247,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(CallInfo);
 
   const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
-  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl))
+  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
     // We can only guarantee that a function is called from the correct
     // context/function based on the appropriate target attributes,
     // so only check in the case where we have both always_inline and target
@@ -4256,6 +4258,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
         TargetDecl->hasAttr<TargetAttr>())
       checkTargetFeatures(Loc, FD);
 
+    // Some architectures (such as x86-64) have the ABI changed based on
+    // attribute-target/features. Give them a chance to diagnose.
+    CGM.getTargetCodeGenInfo().checkFunctionCallABI(
+        CGM, Loc, dyn_cast_or_null<FunctionDecl>(CurCodeDecl), FD, CallArgs);
+  }
+
 #ifndef NDEBUG
   if (!(CallInfo.isVariadic() && CallInfo.getArgStruct())) {
     // For an inalloca varargs function, we don't expect CallInfo to match the
diff --git a/clang/lib/CodeGen/CGCleanup.h b/clang/lib/CodeGen/CGCleanup.h
index a2b3f71a2f861..ef4f6b9ec1332 100644
--- a/clang/lib/CodeGen/CGCleanup.h
+++ b/clang/lib/CodeGen/CGCleanup.h
@@ -102,7 +102,7 @@ class EHScope {
   };
 
 public:
-  enum Kind { Cleanup, Catch, Terminate, Filter, PadEnd };
+  enum Kind { Cleanup, Catch, Terminate, Filter };
 
   EHScope(Kind kind, EHScopeStack::stable_iterator enclosingEHScope)
     : CachedLandingPad(nullptr), CachedEHDispatchBlock(nullptr),
@@ -487,17 +487,6 @@ class EHTerminateScope : public EHScope {
   }
 };
 
-class EHPadEndScope : public EHScope {
-public:
-  EHPadEndScope(EHScopeStack::stable_iterator enclosingEHScope)
-      : EHScope(PadEnd, enclosingEHScope) {}
-  static size_t getSize() { return sizeof(EHPadEndScope); }
-
-  static bool classof(const EHScope *scope) {
-    return scope->getKind() == PadEnd;
-  }
-};
-
 /// A non-stable pointer into the scope stack.
 class EHScopeStack::iterator {
   char *Ptr;
@@ -535,10 +524,6 @@ class EHScopeStack::iterator {
     case EHScope::Terminate:
       Size = EHTerminateScope::getSize();
       break;
-
-    case EHScope::PadEnd:
-      Size = EHPadEndScope::getSize();
-      break;
     }
     Ptr += llvm::alignTo(Size, ScopeStackAlignment);
     return *this;
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index f2549769080e4..1729c7ed3c310 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -1400,10 +1400,15 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
 
   Address address = Address::invalid();
   Address AllocaAddr = Address::invalid();
-  Address OpenMPLocalAddr =
-      getLangOpts().OpenMP
-          ? CGM.getOpenMPRuntime().getAddressOfLocalVariable(*this, &D)
-          : Address::invalid();
+  Address OpenMPLocalAddr = Address::invalid();
+  if (CGM.getLangOpts().OpenMPIRBuilder)
+    OpenMPLocalAddr = OMPBuilderCBHelpers::getAddressOfLocalVariable(*this, &D);
+  else
+    OpenMPLocalAddr =
+        getLangOpts().OpenMP
+            ? CGM.getOpenMPRuntime().getAddressOfLocalVariable(*this, &D)
+            : Address::invalid();
+
   bool NRVO = getLangOpts().ElideConstructors && D.isNRVOVariable();
 
   if (getLangOpts().OpenMP && OpenMPLocalAddr.isValid()) {
@@ -1875,9 +1880,7 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) {
 ///
 /// \param init the initializing expression
 /// \param D the object to act as if we're initializing
-/// \param loc the address to initialize; its type is a pointer
-///   to the LLVM mapping of the object's type
-/// \param alignment the alignment of the address
+/// \param lvalue the lvalue to initialize
 /// \param capturedByInit true if \p D is a __block variable
 ///   whose address is potentially changed by the initializer
 void CodeGenFunction::EmitExprAsInit(const Expr *init, const ValueDecl *D,
diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp
index de3d1b1291464..bdf70252b5ade 100644
--- a/clang/lib/CodeGen/CGException.cpp
+++ b/clang/lib/CodeGen/CGException.cpp
@@ -651,9 +651,6 @@ CodeGenFunction::getEHDispatchBlock(EHScopeStack::stable_iterator si) {
     case EHScope::Terminate:
       dispatchBlock = getTerminateHandler();
       break;
-
-    case EHScope::PadEnd:
-      llvm_unreachable("PadEnd unnecessary for Itanium!");
     }
     scope.setCachedEHDispatchBlock(dispatchBlock);
   }
@@ -695,9 +692,6 @@ CodeGenFunction::getFuncletEHDispatchBlock(EHScopeStack::stable_iterator SI) {
   case EHScope::Terminate:
     DispatchBlock->setName("terminate");
     break;
-
-  case EHScope::PadEnd:
-    llvm_unreachable("PadEnd dispatch block missing!");
   }
   EHS.setCachedEHDispatchBlock(DispatchBlock);
   return DispatchBlock;
@@ -713,7 +707,6 @@ static bool isNonEHScope(const EHScope &S) {
   case EHScope::Filter:
   case EHScope::Catch:
   case EHScope::Terminate:
-  case EHScope::PadEnd:
     return false;
   }
 
@@ -780,9 +773,6 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
   case EHScope::Terminate:
     return getTerminateLandingPad();
 
-  case EHScope::PadEnd:
-    llvm_unreachable("PadEnd unnecessary for Itanium!");
-
   case EHScope::Catch:
   case EHScope::Cleanup:
   case EHScope::Filter:
@@ -848,9 +838,6 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
 
     case EHScope::Catch:
       break;
-
-    case EHScope::PadEnd:
-      llvm_unreachable("PadEnd unnecessary for Itanium!");
     }
 
     EHCatchScope &catchScope = cast<EHCatchScope>(*I);
@@ -1828,6 +1815,48 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF,
     llvm::Constant *ParentI8Fn =
         llvm::ConstantExpr::getBitCast(ParentCGF.CurFn, Int8PtrTy);
     ParentFP = Builder.CreateCall(RecoverFPIntrin, {ParentI8Fn, EntryFP});
+
+    // if the parent is a _finally, the passed-in ParentFP is the FP
+    // of parent _finally, not Establisher's FP (FP of outermost function).
+    // Establkisher FP is 2nd paramenter passed into parent _finally.
+    // Fortunately, it's always saved in parent's frame. The following
+    // code retrieves it, and escapes it so that spill instruction won't be
+    // optimized away.
+    if (ParentCGF.ParentCGF != nullptr) {
+      // Locate and escape Parent's frame_pointer.addr alloca
+      // Depending on target, should be 1st/2nd one in LocalDeclMap.
+      // Let's just scan for ImplicitParamDecl with VoidPtrTy.
+      llvm::AllocaInst *FramePtrAddrAlloca = nullptr;
+      for (auto &I : ParentCGF.LocalDeclMap) {
+        const VarDecl *D = cast<VarDecl>(I.first);
+        if (isa<ImplicitParamDecl>(D) &&
+            D->getType() == getContext().VoidPtrTy) {
+          assert(D->getName().startswith("frame_pointer"));
+          FramePtrAddrAlloca = cast<llvm::AllocaInst>(I.second.getPointer());
+          break;
+        }
+      }
+      assert(FramePtrAddrAlloca);
+      auto InsertPair = ParentCGF.EscapedLocals.insert(
+          std::make_pair(FramePtrAddrAlloca, ParentCGF.EscapedLocals.size()));
+      int FrameEscapeIdx = InsertPair.first->second;
+
+      // an example of a filter's prolog::
+      // %0 = call i8* @llvm.eh.recoverfp(bitcast(@"?fin$0@0@main@@"),..)
+      // %1 = call i8* @llvm.localrecover(bitcast(@"?fin$0@0@main@@"),..)
+      // %2 = bitcast i8* %1 to i8**
+      // %3 = load i8*, i8* *%2, align 8
+      //   ==> %3 is the frame-pointer of outermost host function
+      llvm::Function *FrameRecoverFn = llvm::Intrinsic::getDeclaration(
+          &CGM.getModule(), llvm::Intrinsic::localrecover);
+      llvm::Constant *ParentI8Fn =
+          llvm::ConstantExpr::getBitCast(ParentCGF.CurFn, Int8PtrTy);
+      ParentFP = Builder.CreateCall(
+          FrameRecoverFn, {ParentI8Fn, ParentFP,
+                           llvm::ConstantInt::get(Int32Ty, FrameEscapeIdx)});
+      ParentFP = Builder.CreateBitCast(ParentFP, CGM.VoidPtrPtrTy);
+      ParentFP = Builder.CreateLoad(Address(ParentFP, getPointerAlign()));
+    }
   }
 
   // Create llvm.localrecover calls for all captures.
@@ -2026,6 +2055,7 @@ void CodeGenFunction::pushSEHCleanup(CleanupKind Kind,
 
 void CodeGenFunction::EnterSEHTryStmt(const SEHTryStmt &S) {
   CodeGenFunction HelperCGF(CGM, /*suppressNewContext=*/true);
+  HelperCGF.ParentCGF = this;
   if (const SEHFinallyStmt *Finally = S.getFinallyHandler()) {
     // Outline the finally block.
     llvm::Function *FinallyFunc =
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index bead7e8996c05..9e8770573d701 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -2398,7 +2398,13 @@ EmitBitCastOfLValueToProperType(CodeGenFunction &CGF,
 static LValue EmitThreadPrivateVarDeclLValue(
     CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr,
     llvm::Type *RealVarTy, SourceLocation Loc) {
-  Addr = CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, Addr, Loc);
+  if (CGF.CGM.getLangOpts().OpenMPIRBuilder)
+    Addr = CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
+        CGF, VD, Addr, Loc);
+  else
+    Addr =
+        CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, Addr, Loc);
+
   Addr = CGF.Builder.CreateElementBitCast(Addr, RealVarTy);
   return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
 }
@@ -3843,7 +3849,7 @@ LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E,
   else
     ResultExprTy = BaseTy->getPointeeType();
   llvm::Value *Idx = nullptr;
-  if (IsLowerBound || E->getColonLoc().isInvalid()) {
+  if (IsLowerBound || E->getColonLocFirst().isInvalid()) {
     // Requesting lower bound or upper bound, but without provided length and
     // without ':' symbol for the default length -> length = 1.
     // Idx = LowerBound ?: 0;
diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp
index 13a5713521374..a49817898ae39 100644
--- a/clang/lib/CodeGen/CGExprComplex.cpp
+++ b/clang/lib/CodeGen/CGExprComplex.cpp
@@ -98,7 +98,7 @@ class ComplexExprEmitter
   }
 
   ComplexPairTy VisitStmt(Stmt *S) {
-    S->dump(CGF.getContext().getSourceManager());
+    S->dump(llvm::errs(), CGF.getContext());
     llvm_unreachable("Stmt can't have complex result type!");
   }
   ComplexPairTy VisitExpr(Expr *S);
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index a01571633f6c0..c6b2930faece1 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -777,7 +777,7 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD,
 
   if (const CXXRecordDecl *CD = dyn_cast<CXXRecordDecl>(RD)) {
     // Add a vtable pointer, if we need one and it hasn't already been added.
-    if (CD->isDynamicClass() && !IsPrimaryBase) {
+    if (Layout.hasOwnVFPtr()) {
       llvm::Constant *VTableAddressPoint =
           CGM.getCXXABI().getVTableAddressPointForConstExpr(
               BaseSubobject(CD, Offset), VTableClass);
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index f48f7185b55f1..6131f97995dce 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -413,7 +413,7 @@ class ScalarExprEmitter
   }
 
   Value *VisitStmt(Stmt *S) {
-    S->dump(CGF.getContext().getSourceManager());
+    S->dump(llvm::errs(), CGF.getContext());
     llvm_unreachable("Stmt can't have complex result type!");
   }
   Value *VisitExpr(Expr *S);
@@ -2389,7 +2389,7 @@ llvm::Value *ScalarExprEmitter::EmitIncDecConsiderOverflowBehavior(
     if (!E->canOverflow())
       return Builder.CreateNSWAdd(InVal, Amount, Name);
     return EmitOverflowCheckedBinOp(createBinOpInfoFromIncDec(
-        E, InVal, IsInc, E->getFPFeatures(CGF.getLangOpts())));
+        E, InVal, IsInc, E->getFPFeaturesInEffect(CGF.getLangOpts())));
   }
   llvm_unreachable("Unknown SignedOverflowBehaviorTy");
 }
@@ -2536,7 +2536,7 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
     } else if (E->canOverflow() && type->isUnsignedIntegerType() &&
                CGF.SanOpts.has(SanitizerKind::UnsignedIntegerOverflow)) {
       value = EmitOverflowCheckedBinOp(createBinOpInfoFromIncDec(
-          E, value, isInc, E->getFPFeatures(CGF.getLangOpts())));
+          E, value, isInc, E->getFPFeaturesInEffect(CGF.getLangOpts())));
     } else {
       llvm::Value *amt = llvm::ConstantInt::get(value->getType(), amount, true);
       value = Builder.CreateAdd(value, amt, isInc ? "inc" : "dec");
@@ -2736,7 +2736,7 @@ Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E) {
   BinOp.LHS = llvm::Constant::getNullValue(BinOp.RHS->getType());
   BinOp.Ty = E->getType();
   BinOp.Opcode = BO_Sub;
-  BinOp.FPFeatures = E->getFPFeatures(CGF.getLangOpts());
+  BinOp.FPFeatures = E->getFPFeaturesInEffect(CGF.getLangOpts());
   BinOp.E = E;
   return EmitSub(BinOp);
 }
@@ -2757,7 +2757,7 @@ Value *ScalarExprEmitter::VisitUnaryLNot(const UnaryOperator *E) {
     Value *Result;
     if (Oper->getType()->isFPOrFPVectorTy()) {
       CodeGenFunction::CGFPOptionsRAII FPOptsRAII(
-          CGF, E->getFPFeatures(CGF.getLangOpts()));
+          CGF, E->getFPFeaturesInEffect(CGF.getLangOpts()));
       Result = Builder.CreateFCmp(llvm::CmpInst::FCMP_OEQ, Oper, Zero, "cmp");
     } else
       Result = Builder.CreateICmp(llvm::CmpInst::ICMP_EQ, Oper, Zero, "cmp");
@@ -2960,7 +2960,7 @@ BinOpInfo ScalarExprEmitter::EmitBinOps(const BinaryOperator *E) {
   Result.RHS = Visit(E->getRHS());
   Result.Ty  = E->getType();
   Result.Opcode = E->getOpcode();
-  Result.FPFeatures = E->getFPFeatures(CGF.getLangOpts());
+  Result.FPFeatures = E->getFPFeaturesInEffect(CGF.getLangOpts());
   Result.E = E;
   return Result;
 }
@@ -2980,7 +2980,7 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue(
   OpInfo.RHS = Visit(E->getRHS());
   OpInfo.Ty = E->getComputationResultType();
   OpInfo.Opcode = E->getOpcode();
-  OpInfo.FPFeatures = E->getFPFeatures(CGF.getLangOpts());
+  OpInfo.FPFeatures = E->getFPFeaturesInEffect(CGF.getLangOpts());
   OpInfo.E = E;
   // Load/convert the LHS.
   LValue LHSLV = EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store);
@@ -3608,8 +3608,8 @@ Value *ScalarExprEmitter::EmitFixedPointBinOp(const BinOpInfo &op) {
   switch (op.Opcode) {
   case BO_AddAssign:
   case BO_Add: {
-    if (ResultFixedSema.isSaturated()) {
-      llvm::Intrinsic::ID IID = ResultFixedSema.isSigned()
+    if (CommonFixedSema.isSaturated()) {
+      llvm::Intrinsic::ID IID = CommonFixedSema.isSigned()
                                     ? llvm::Intrinsic::sadd_sat
                                     : llvm::Intrinsic::uadd_sat;
       Result = Builder.CreateBinaryIntrinsic(IID, FullLHS, FullRHS);
@@ -3620,8 +3620,8 @@ Value *ScalarExprEmitter::EmitFixedPointBinOp(const BinOpInfo &op) {
   }
   case BO_SubAssign:
   case BO_Sub: {
-    if (ResultFixedSema.isSaturated()) {
-      llvm::Intrinsic::ID IID = ResultFixedSema.isSigned()
+    if (CommonFixedSema.isSaturated()) {
+      llvm::Intrinsic::ID IID = CommonFixedSema.isSigned()
                                     ? llvm::Intrinsic::ssub_sat
                                     : llvm::Intrinsic::usub_sat;
       Result = Builder.CreateBinaryIntrinsic(IID, FullLHS, FullRHS);
@@ -3633,14 +3633,12 @@ Value *ScalarExprEmitter::EmitFixedPointBinOp(const BinOpInfo &op) {
   case BO_MulAssign:
   case BO_Mul: {
     llvm::Intrinsic::ID IID;
-    if (ResultFixedSema.isSaturated())
-      IID = ResultFixedSema.isSigned()
-                ? llvm::Intrinsic::smul_fix_sat
-                : llvm::Intrinsic::umul_fix_sat;
+    if (CommonFixedSema.isSaturated())
+      IID = CommonFixedSema.isSigned() ? llvm::Intrinsic::smul_fix_sat
+                                       : llvm::Intrinsic::umul_fix_sat;
     else
-      IID = ResultFixedSema.isSigned()
-                ? llvm::Intrinsic::smul_fix
-                : llvm::Intrinsic::umul_fix;
+      IID = CommonFixedSema.isSigned() ? llvm::Intrinsic::smul_fix
+                                       : llvm::Intrinsic::umul_fix;
     Result = Builder.CreateIntrinsic(IID, {FullLHS->getType()},
         {FullLHS, FullRHS, Builder.getInt32(CommonFixedSema.getScale())});
     break;
@@ -3648,11 +3646,11 @@ Value *ScalarExprEmitter::EmitFixedPointBinOp(const BinOpInfo &op) {
   case BO_DivAssign:
   case BO_Div: {
     llvm::Intrinsic::ID IID;
-    if (ResultFixedSema.isSaturated())
-      IID = ResultFixedSema.isSigned() ? llvm::Intrinsic::sdiv_fix_sat
+    if (CommonFixedSema.isSaturated())
+      IID = CommonFixedSema.isSigned() ? llvm::Intrinsic::sdiv_fix_sat
                                        : llvm::Intrinsic::udiv_fix_sat;
     else
-      IID = ResultFixedSema.isSigned() ? llvm::Intrinsic::sdiv_fix
+      IID = CommonFixedSema.isSigned() ? llvm::Intrinsic::sdiv_fix
                                        : llvm::Intrinsic::udiv_fix;
     Result = Builder.CreateIntrinsic(IID, {FullLHS->getType()},
         {FullLHS, FullRHS, Builder.getInt32(CommonFixedSema.getScale())});
@@ -4214,7 +4212,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
     Value *Zero = llvm::ConstantAggregateZero::get(LHS->getType());
     if (LHS->getType()->isFPOrFPVectorTy()) {
       CodeGenFunction::CGFPOptionsRAII FPOptsRAII(
-          CGF, E->getFPFeatures(CGF.getLangOpts()));
+          CGF, E->getFPFeaturesInEffect(CGF.getLangOpts()));
       LHS = Builder.CreateFCmp(llvm::CmpInst::FCMP_UNE, LHS, Zero, "cmp");
       RHS = Builder.CreateFCmp(llvm::CmpInst::FCMP_UNE, RHS, Zero, "cmp");
     } else {
@@ -4300,7 +4298,7 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
     Value *Zero = llvm::ConstantAggregateZero::get(LHS->getType());
     if (LHS->getType()->isFPOrFPVectorTy()) {
       CodeGenFunction::CGFPOptionsRAII FPOptsRAII(
-          CGF, E->getFPFeatures(CGF.getLangOpts()));
+          CGF, E->getFPFeaturesInEffect(CGF.getLangOpts()));
       LHS = Builder.CreateFCmp(llvm::CmpInst::FCMP_UNE, LHS, Zero, "cmp");
       RHS = Builder.CreateFCmp(llvm::CmpInst::FCMP_UNE, RHS, Zero, "cmp");
     } else {
diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp
index 4c5311cf001dd..158a548e66c1e 100644
--- a/clang/lib/CodeGen/CGObjC.cpp
+++ b/clang/lib/CodeGen/CGObjC.cpp
@@ -1493,8 +1493,7 @@ CodeGenFunction::generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
 
   BinaryOperator *assign = BinaryOperator::Create(
       getContext(), &ivarRef, finalArg, BO_Assign, ivarRef.getType(), VK_RValue,
-      OK_Ordinary, SourceLocation(),
-      FPOptions(getContext().getLangOpts()));
+      OK_Ordinary, SourceLocation(), FPOptionsOverride());
   EmitStmt(assign);
 }
 
@@ -3556,18 +3555,18 @@ CodeGenFunction::GenerateObjCAtomicSetterCopyHelperFunction(
   DeclRefExpr DstExpr(C, &DstDecl, false, DestTy, VK_RValue, SourceLocation());
   UnaryOperator *DST = UnaryOperator::Create(
       C, &DstExpr, UO_Deref, DestTy->getPointeeType(), VK_LValue, OK_Ordinary,
-      SourceLocation(), false, FPOptions(C.getLangOpts()));
+      SourceLocation(), false, FPOptionsOverride());
 
   DeclRefExpr SrcExpr(C, &SrcDecl, false, SrcTy, VK_RValue, SourceLocation());
   UnaryOperator *SRC = UnaryOperator::Create(
       C, &SrcExpr, UO_Deref, SrcTy->getPointeeType(), VK_LValue, OK_Ordinary,
-      SourceLocation(), false, FPOptions(C.getLangOpts()));
+      SourceLocation(), false, FPOptionsOverride());
 
   Expr *Args[2] = {DST, SRC};
   CallExpr *CalleeExp = cast<CallExpr>(PID->getSetterCXXAssignment());
   CXXOperatorCallExpr *TheCall = CXXOperatorCallExpr::Create(
       C, OO_Equal, CalleeExp->getCallee(), Args, DestTy->getPointeeType(),
-      VK_LValue, SourceLocation(), FPOptions(C.getLangOpts()));
+      VK_LValue, SourceLocation(), FPOptionsOverride());
 
   EmitStmt(TheCall);
 
@@ -3641,7 +3640,7 @@ CodeGenFunction::GenerateObjCAtomicGetterCopyHelperFunction(
 
   UnaryOperator *SRC = UnaryOperator::Create(
       C, &SrcExpr, UO_Deref, SrcTy->getPointeeType(), VK_LValue, OK_Ordinary,
-      SourceLocation(), false, FPOptions(C.getLangOpts()));
+      SourceLocation(), false, FPOptionsOverride());
 
   CXXConstructExpr *CXXConstExpr =
     cast<CXXConstructExpr>(PID->getGetterCXXConstructor());
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 1ff62fd5894a5..43cbe9c720ea3 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -1060,7 +1060,7 @@ static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
 CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
                                  StringRef Separator)
     : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
-      OffloadEntriesInfoManager(CGM) {
+      OMPBuilder(CGM.getModule()), OffloadEntriesInfoManager(CGM) {
   ASTContext &C = CGM.getContext();
   RecordDecl *RD = C.buildImplicitRecord("ident_t");
   QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
@@ -1081,7 +1081,7 @@ CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
   KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
 
   // Initialize Types used in OpenMPIRBuilder from OMPKinds.def
-  llvm::omp::types::initializeTypes(CGM.getModule());
+  OMPBuilder.initialize();
   loadOffloadInfoMetadata();
 }
 
@@ -1278,8 +1278,8 @@ static llvm::Function *emitParallelOrTeamsOutlinedFunction(
 
   // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new
   //       parallel region to make cancellation barriers work properly.
-  llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder();
-  PushAndPopStackRAII PSR(OMPBuilder, CGF, HasCancel);
+  llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
+  PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel);
   CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
                                     HasCancel, OutlinedHelperName);
   CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
@@ -1316,7 +1316,7 @@ llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
         CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
                                     TaskTVar->getType()->castAs<PointerType>())
             .getPointer(CGF)};
-    CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+    CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___kmpc_omp_task),
                         TaskArgs);
   };
@@ -1563,8 +1563,8 @@ llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
   CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
   CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
   llvm::CallInst *Call = CGF.Builder.CreateCall(
-      llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
-          CGM.getModule(), OMPRTL___kmpc_global_thread_num),
+      OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
+                                            OMPRTL___kmpc_global_thread_num),
       emitUpdateLocation(CGF, Loc));
   Call->setCallingConv(CGF.getRuntimeCC());
   Elem.second.ThreadID = Call;
@@ -1783,7 +1783,7 @@ Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
                          CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
                          getOrCreateThreadPrivateCache(VD)};
   return Address(CGF.EmitRuntimeCall(
-                     llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+                     OMPBuilder.getOrCreateRuntimeFunction(
                          CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
                      Args),
                  VDAddr.getAlignment());
@@ -1795,7 +1795,7 @@ void CGOpenMPRuntime::emitThreadPrivateVarInit(
   // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
   // library.
   llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
-  CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+  CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                           CGM.getModule(), OMPRTL___kmpc_global_thread_num),
                       OMPLoc);
   // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
@@ -1804,7 +1804,7 @@ void CGOpenMPRuntime::emitThreadPrivateVarInit(
       OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
       Ctor, CopyCtor, Dtor};
   CGF.EmitRuntimeCall(
-      llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+      OMPBuilder.getOrCreateRuntimeFunction(
           CGM.getModule(), OMPRTL___kmpc_threadprivate_register),
       Args);
 }
@@ -2068,7 +2068,7 @@ Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
   return Address(
       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
           CGF.EmitRuntimeCall(
-              llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+              OMPBuilder.getOrCreateRuntimeFunction(
                   CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
               Args),
           VarLVType->getPointerTo(/*AddrSpace=*/0)),
@@ -2122,8 +2122,8 @@ void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
     return;
   llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
   auto &M = CGM.getModule();
-  auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF,
-                                                         PrePostActionTy &) {
+  auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
+                    this](CodeGenFunction &CGF, PrePostActionTy &) {
     // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
     CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
     llvm::Value *Args[] = {
@@ -2135,18 +2135,17 @@ void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
     RealArgs.append(CapturedVars.begin(), CapturedVars.end());
 
     llvm::FunctionCallee RTLFn =
-        llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
-            M, OMPRTL___kmpc_fork_call);
+        OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
     CGF.EmitRuntimeCall(RTLFn, RealArgs);
   };
-  auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc,
-                    Loc](CodeGenFunction &CGF, PrePostActionTy &) {
+  auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
+                    this](CodeGenFunction &CGF, PrePostActionTy &) {
     CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
     llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
     // Build calls:
     // __kmpc_serialized_parallel(&Loc, GTid);
     llvm::Value *Args[] = {RTLoc, ThreadID};
-    CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+    CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                             M, OMPRTL___kmpc_serialized_parallel),
                         Args);
 
@@ -2165,7 +2164,7 @@ void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
 
     // __kmpc_end_serialized_parallel(&Loc, GTid);
     llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
-    CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+    CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                             M, OMPRTL___kmpc_end_serialized_parallel),
                         EndArgs);
   };
@@ -2284,12 +2283,12 @@ void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
         CGF.EmitScalarExpr(Hint), CGM.Int32Ty, /*isSigned=*/false));
   }
   CommonActionTy Action(
-      llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+      OMPBuilder.getOrCreateRuntimeFunction(
           CGM.getModule(),
           Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical),
       EnterArgs,
-      llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
-          CGM.getModule(), OMPRTL___kmpc_end_critical),
+      OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
+                                            OMPRTL___kmpc_end_critical),
       Args);
   CriticalOpGen.setAction(Action);
   emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
@@ -2306,10 +2305,10 @@ void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
   // }
   // Prepare arguments and build a call to __kmpc_master
   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
-  CommonActionTy Action(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+  CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___kmpc_master),
                         Args,
-                        llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+                        OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___kmpc_end_master),
                         Args,
                         /*Conditional=*/true);
@@ -2322,15 +2321,14 @@ void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
                                         SourceLocation Loc) {
   if (!CGF.HaveInsertPoint())
     return;
-  llvm::OpenMPIRBuilder *OMPBuilder = CGF.CGM.getOpenMPIRBuilder();
-  if (OMPBuilder) {
-    OMPBuilder->CreateTaskyield(CGF.Builder);
+  if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
+    OMPBuilder.CreateTaskyield(CGF.Builder);
   } else {
     // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
     llvm::Value *Args[] = {
         emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
         llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
-    CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+    CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___kmpc_omp_taskyield),
                         Args);
   }
@@ -2349,10 +2347,10 @@ void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
   // __kmpc_end_taskgroup(ident_t *, gtid);
   // Prepare arguments and build a call to __kmpc_taskgroup
   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
-  CommonActionTy Action(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+  CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___kmpc_taskgroup),
                         Args,
-                        llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+                        OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___kmpc_end_taskgroup),
                         Args);
   TaskgroupOpGen.setAction(Action);
@@ -2459,10 +2457,10 @@ void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
   }
   // Prepare arguments and build a call to __kmpc_single
   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
-  CommonActionTy Action(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+  CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___kmpc_single),
                         Args,
-                        llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+                        OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___kmpc_end_single),
                         Args,
                         /*Conditional=*/true);
@@ -2509,7 +2507,7 @@ void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
         CpyFn,                        // void (*) (void *, void *) <copy_func>
         DidItVal                      // i32 did_it
     };
-    CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+    CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___kmpc_copyprivate),
                         Args);
   }
@@ -2526,10 +2524,10 @@ void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
   // Prepare arguments and build a call to __kmpc_ordered
   if (IsThreads) {
     llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
-    CommonActionTy Action(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+    CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
                               CGM.getModule(), OMPRTL___kmpc_ordered),
                           Args,
-                          llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+                          OMPBuilder.getOrCreateRuntimeFunction(
                               CGM.getModule(), OMPRTL___kmpc_end_ordered),
                           Args);
     OrderedOpGen.setAction(Action);
@@ -2578,9 +2576,8 @@ void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
   // Check if we should use the OMPBuilder
   auto *OMPRegionInfo =
       dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo);
-  llvm::OpenMPIRBuilder *OMPBuilder = CGF.CGM.getOpenMPIRBuilder();
-  if (OMPBuilder) {
-    CGF.Builder.restoreIP(OMPBuilder->CreateBarrier(
+  if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
+    CGF.Builder.restoreIP(OMPBuilder.CreateBarrier(
         CGF.Builder, Kind, ForceSimpleCall, EmitChecks));
     return;
   }
@@ -2597,8 +2594,8 @@ void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
   if (OMPRegionInfo) {
     if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
       llvm::Value *Result = CGF.EmitRuntimeCall(
-          llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
-              CGM.getModule(), OMPRTL___kmpc_cancel_barrier),
+          OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
+                                                OMPRTL___kmpc_cancel_barrier),
           Args);
       if (EmitChecks) {
         // if (__kmpc_cancel_barrier()) {
@@ -2618,7 +2615,7 @@ void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
       return;
     }
   }
-  CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+  CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                           CGM.getModule(), OMPRTL___kmpc_barrier),
                       Args);
 }
@@ -2870,7 +2867,7 @@ void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
                                    : OMP_IDENT_WORK_SECTIONS),
       getThreadID(CGF, Loc)};
   auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
-  CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+  CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                           CGM.getModule(), OMPRTL___kmpc_for_static_fini),
                       Args);
 }
@@ -2919,7 +2916,7 @@ void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
   llvm::Value *Args[] = {
       emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
       CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
-  CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+  CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                           CGM.getModule(), OMPRTL___kmpc_push_num_threads),
                       Args);
 }
@@ -2934,21 +2931,20 @@ void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
   llvm::Value *Args[] = {
       emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
       llvm::ConstantInt::get(CGM.IntTy, unsigned(ProcBind), /*isSigned=*/true)};
-  CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+  CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                           CGM.getModule(), OMPRTL___kmpc_push_proc_bind),
                       Args);
 }
 
 void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
                                 SourceLocation Loc, llvm::AtomicOrdering AO) {
-  llvm::OpenMPIRBuilder *OMPBuilder = CGF.CGM.getOpenMPIRBuilder();
-  if (OMPBuilder) {
-    OMPBuilder->CreateFlush(CGF.Builder);
+  if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
+    OMPBuilder.CreateFlush(CGF.Builder);
   } else {
     if (!CGF.HaveInsertPoint())
       return;
     // Build call void __kmpc_flush(ident_t *loc)
-    CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+    CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___kmpc_flush),
                         emitUpdateLocation(CGF, Loc));
   }
@@ -4302,12 +4298,12 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
       DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
     AllocArgs.push_back(DeviceID);
     NewTask = CGF.EmitRuntimeCall(
-        llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+        OMPBuilder.getOrCreateRuntimeFunction(
             CGM.getModule(), OMPRTL___kmpc_omp_target_task_alloc),
         AllocArgs);
   } else {
     NewTask =
-        CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+        CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                                 CGM.getModule(), OMPRTL___kmpc_omp_task_alloc),
                             AllocArgs);
   }
@@ -4324,7 +4320,7 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
     llvm::Value *Tid = getThreadID(CGF, DC->getBeginLoc());
     Tid = CGF.Builder.CreateIntCast(Tid, CGF.IntTy, /*isSigned=*/false);
     llvm::Value *EvtVal = CGF.EmitRuntimeCall(
-        llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+        OMPBuilder.getOrCreateRuntimeFunction(
             CGM.getModule(), OMPRTL___kmpc_task_allow_completion_event),
         {Loc, Tid, NewTask});
     EvtVal = CGF.EmitScalarConversion(EvtVal, C.VoidPtrTy, Evt->getType(),
@@ -4463,7 +4459,7 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
     // FIXME: Emit the function and ignore its result for now unless the
     // runtime function is properly implemented.
     (void)CGF.EmitRuntimeCall(
-        llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+        OMPBuilder.getOrCreateRuntimeFunction(
             CGM.getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
         {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
   }
@@ -4966,7 +4962,7 @@ Address CGOpenMPRuntime::emitDepobjDependClause(
   llvm::Value *Args[] = {ThreadID, Size, Allocator};
 
   llvm::Value *Addr =
-      CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+      CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                               CGM.getModule(), OMPRTL___kmpc_alloc),
                           Args, ".dep.arr.addr");
   Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
@@ -5019,7 +5015,7 @@ void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
   llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
 
   // _kmpc_free(gtid, addr, nullptr);
-  (void)CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+  (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                                 CGM.getModule(), OMPRTL___kmpc_free),
                             Args);
 }
@@ -5120,11 +5116,11 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
     }
     if (!Data.Dependences.empty()) {
       CGF.EmitRuntimeCall(
-          llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+          OMPBuilder.getOrCreateRuntimeFunction(
               CGM.getModule(), OMPRTL___kmpc_omp_task_with_deps),
           DepTaskArgs);
     } else {
-      CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+      CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                               CGM.getModule(), OMPRTL___kmpc_omp_task),
                           TaskArgs);
     }
@@ -5144,8 +5140,8 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
     DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
   }
   auto &M = CGM.getModule();
-  auto &&ElseCodeGen = [&M, &TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
-                        &Data, &DepWaitTaskArgs,
+  auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
+                        TaskEntry, &Data, &DepWaitTaskArgs,
                         Loc](CodeGenFunction &CGF, PrePostActionTy &) {
     CodeGenFunction::RunCleanupsScope LocalScope(CGF);
     // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
@@ -5153,9 +5149,9 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
     // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
     // is specified.
     if (!Data.Dependences.empty())
-      CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
-                              M, OMPRTL___kmpc_omp_wait_deps),
-                          DepWaitTaskArgs);
+      CGF.EmitRuntimeCall(
+          OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_wait_deps),
+          DepWaitTaskArgs);
     // Call proxy_task_entry(gtid, new_task);
     auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
                       Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
@@ -5170,10 +5166,10 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
     // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
     // kmp_task_t *new_task);
     RegionCodeGenTy RCG(CodeGen);
-    CommonActionTy Action(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+    CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
                               M, OMPRTL___kmpc_omp_task_begin_if0),
                           TaskArgs,
-                          llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+                          OMPBuilder.getOrCreateRuntimeFunction(
                               M, OMPRTL___kmpc_omp_task_complete_if0),
                           TaskArgs);
     RCG.setAction(Action);
@@ -5269,7 +5265,7 @@ void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
       Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
                              Result.TaskDupFn, CGF.VoidPtrTy)
                        : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
-  CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+  CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                           CGM.getModule(), OMPRTL___kmpc_taskloop),
                       TaskArgs);
 }
@@ -5613,7 +5609,7 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
       Lock         // kmp_critical_name *&<lock>
   };
   llvm::Value *Res = CGF.EmitRuntimeCall(
-      llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+      OMPBuilder.getOrCreateRuntimeFunction(
           CGM.getModule(),
           WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
       Args);
@@ -5656,7 +5652,7 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
   RegionCodeGenTy RCG(CodeGen);
   CommonActionTy Action(
       nullptr, llvm::None,
-      llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+      OMPBuilder.getOrCreateRuntimeFunction(
           CGM.getModule(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait
                                       : OMPRTL___kmpc_end_reduce),
       EndArgs);
@@ -5781,7 +5777,7 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
         Lock       // kmp_critical_name *&<lock>
     };
     CommonActionTy Action(nullptr, llvm::None,
-                          llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+                          OMPBuilder.getOrCreateRuntimeFunction(
                               CGM.getModule(), OMPRTL___kmpc_end_reduce),
                           EndArgs);
     AtomicRCG.setAction(Action);
@@ -6121,7 +6117,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
         CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
             TaskRedInput.getPointer(), CGM.VoidPtrTy)};
     return CGF.EmitRuntimeCall(
-        llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+        OMPBuilder.getOrCreateRuntimeFunction(
             CGM.getModule(), OMPRTL___kmpc_taskred_modifier_init),
         Args);
   }
@@ -6132,7 +6128,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
       llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true),
       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(),
                                                       CGM.VoidPtrTy)};
-  return CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+  return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                                  CGM.getModule(), OMPRTL___kmpc_taskred_init),
                              Args);
 }
@@ -6150,7 +6146,7 @@ void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
                                                 IsWorksharingReduction ? 1 : 0,
                                                 /*isSigned=*/true)};
   (void)CGF.EmitRuntimeCall(
-      llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+      OMPBuilder.getOrCreateRuntimeFunction(
           CGM.getModule(), OMPRTL___kmpc_task_reduction_modifier_fini),
       Args);
 }
@@ -6186,7 +6182,7 @@ Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,
                              SharedLVal.getPointer(CGF), CGM.VoidPtrTy)};
   return Address(
       CGF.EmitRuntimeCall(
-          llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+          OMPBuilder.getOrCreateRuntimeFunction(
               CGM.getModule(), OMPRTL___kmpc_task_reduction_get_th_data),
           Args),
       SharedLVal.getAlignment());
@@ -6197,15 +6193,14 @@ void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
   if (!CGF.HaveInsertPoint())
     return;
 
-  llvm::OpenMPIRBuilder *OMPBuilder = CGF.CGM.getOpenMPIRBuilder();
-  if (OMPBuilder) {
-    OMPBuilder->CreateTaskwait(CGF.Builder);
+  if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
+    OMPBuilder.CreateTaskwait(CGF.Builder);
   } else {
     // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
     // global_tid);
     llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
     // Ignore return result until untied tasks are supported.
-    CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+    CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___kmpc_omp_taskwait),
                         Args);
   }
@@ -6266,7 +6261,7 @@ void CGOpenMPRuntime::emitCancellationPointCall(
           CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
       // Ignore return result until untied tasks are supported.
       llvm::Value *Result = CGF.EmitRuntimeCall(
-          llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+          OMPBuilder.getOrCreateRuntimeFunction(
               CGM.getModule(), OMPRTL___kmpc_cancellationpoint),
           Args);
       // if (__kmpc_cancellationpoint()) {
@@ -6296,17 +6291,15 @@ void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
   auto &M = CGM.getModule();
   if (auto *OMPRegionInfo =
           dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
-    auto &&ThenGen = [&M, Loc, CancelRegion,
+    auto &&ThenGen = [this, &M, Loc, CancelRegion,
                       OMPRegionInfo](CodeGenFunction &CGF, PrePostActionTy &) {
       CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
       llvm::Value *Args[] = {
           RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
           CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
       // Ignore return result until untied tasks are supported.
-      llvm::Value *Result =
-          CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
-                                  M, OMPRTL___kmpc_cancel),
-                              Args);
+      llvm::Value *Result = CGF.EmitRuntimeCall(
+          OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_cancel), Args);
       // if (__kmpc_cancel()) {
       //   exit from construct;
       // }
@@ -6402,7 +6395,7 @@ void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF,
       CGF.EmitLoadOfScalar(AllocatorTraitsLVal, AllocatorTraits->getExprLoc());
 
   llvm::Value *AllocatorVal =
-      CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+      CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                               CGM.getModule(), OMPRTL___kmpc_init_allocator),
                           {ThreadId, MemSpaceHandle, NumTraits, Traits});
   // Store to allocator.
@@ -6426,8 +6419,8 @@ void CGOpenMPRuntime::emitUsesAllocatorsFini(CodeGenFunction &CGF,
                                           CGF.getContext().VoidPtrTy,
                                           Allocator->getExprLoc());
   (void)CGF.EmitRuntimeCall(
-      llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
-          CGM.getModule(), OMPRTL___kmpc_destroy_allocator),
+      OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
+                                            OMPRTL___kmpc_destroy_allocator),
       {ThreadId, AllocatorVal});
 }
 
@@ -6680,6 +6673,8 @@ emitNumTeamsForTargetDirective(CodeGenFunction &CGF,
   case OMPD_requires:
   case OMPD_unknown:
     break;
+  default:
+    break;
   }
   llvm_unreachable("Unexpected directive kind.");
 }
@@ -6995,6 +6990,8 @@ emitNumThreadsForTargetDirective(CodeGenFunction &CGF,
   case OMPD_requires:
   case OMPD_unknown:
     break;
+  default:
+    break;
   }
   llvm_unreachable("Unsupported directive kind.");
 }
@@ -7178,7 +7175,7 @@ class MappableExprsHandler {
       // If there is no length associated with the expression and lower bound is
       // not specified too, that means we are using the whole length of the
       // base.
-      if (!OAE->getLength() && OAE->getColonLoc().isValid() &&
+      if (!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
           !OAE->getLowerBound())
         return CGF.getTypeSize(BaseTy);
 
@@ -7193,7 +7190,7 @@ class MappableExprsHandler {
 
       // If we don't have a length at this point, that is because we have an
       // array section with a single element.
-      if (!OAE->getLength() && OAE->getColonLoc().isInvalid())
+      if (!OAE->getLength() && OAE->getColonLocFirst().isInvalid())
         return ElemSize;
 
       if (const Expr *LenExpr = OAE->getLength()) {
@@ -7203,7 +7200,7 @@ class MappableExprsHandler {
                                              LenExpr->getExprLoc());
         return CGF.Builder.CreateNUWMul(LengthVal, ElemSize);
       }
-      assert(!OAE->getLength() && OAE->getColonLoc().isValid() &&
+      assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
              OAE->getLowerBound() && "expected array_section[lb:].");
       // Size = sizetype - lb * elemtype;
       llvm::Value *LengthVal = CGF.getTypeSize(BaseTy);
@@ -7276,7 +7273,7 @@ class MappableExprsHandler {
       return false;
 
     // An array section with no colon always refer to a single element.
-    if (OASE->getColonLoc().isInvalid())
+    if (OASE->getColonLocFirst().isInvalid())
       return false;
 
     const Expr *Length = OASE->getLength();
@@ -8914,6 +8911,7 @@ getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) {
     case OMPD_parallel_master_taskloop_simd:
     case OMPD_requires:
     case OMPD_unknown:
+    default:
       llvm_unreachable("Unexpected directive.");
     }
   }
@@ -9063,8 +9061,8 @@ void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
   // pre-existing components.
   llvm::Value *OffloadingArgs[] = {Handle};
   llvm::Value *PreviousSize = MapperCGF.EmitRuntimeCall(
-      llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
-          CGM.getModule(), OMPRTL___tgt_mapper_num_components),
+      OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
+                                            OMPRTL___tgt_mapper_num_components),
       OffloadingArgs);
   llvm::Value *ShiftedPreviousSize = MapperCGF.Builder.CreateShl(
       PreviousSize,
@@ -9171,7 +9169,7 @@ void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
     llvm::Value *OffloadingArgs[] = {Handle, CurBaseArg, CurBeginArg,
                                      CurSizeArg, CurMapType};
     MapperCGF.EmitRuntimeCall(
-        llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+        OMPBuilder.getOrCreateRuntimeFunction(
             CGM.getModule(), OMPRTL___tgt_push_mapper_component),
         OffloadingArgs);
   }
@@ -9253,8 +9251,8 @@ void CGOpenMPRuntime::emitUDMapperArrayInitOrDel(
   // data structure.
   llvm::Value *OffloadingArgs[] = {Handle, Base, Begin, ArraySize, MapTypeArg};
   MapperCGF.EmitRuntimeCall(
-      llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
-          CGM.getModule(), OMPRTL___tgt_push_mapper_component),
+      OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
+                                            OMPRTL___tgt_push_mapper_component),
       OffloadingArgs);
 }
 
@@ -9277,7 +9275,7 @@ void CGOpenMPRuntime::emitTargetNumIterationsCall(
     if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD)) {
       llvm::Value *Args[] = {DeviceID, NumIterations};
       CGF.EmitRuntimeCall(
-          llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+          OMPBuilder.getOrCreateRuntimeFunction(
               CGM.getModule(), OMPRTL___kmpc_push_target_tripcount),
           Args);
     }
@@ -9406,7 +9404,7 @@ void CGOpenMPRuntime::emitTargetCall(
                                        NumTeams,
                                        NumThreads};
       Return = CGF.EmitRuntimeCall(
-          llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+          OMPBuilder.getOrCreateRuntimeFunction(
               CGM.getModule(), HasNowait ? OMPRTL___tgt_target_teams_nowait
                                          : OMPRTL___tgt_target_teams),
           OffloadingArgs);
@@ -9419,7 +9417,7 @@ void CGOpenMPRuntime::emitTargetCall(
                                        InputInfo.SizesArray.getPointer(),
                                        MapTypesArray};
       Return = CGF.EmitRuntimeCall(
-          llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+          OMPBuilder.getOrCreateRuntimeFunction(
               CGM.getModule(),
               HasNowait ? OMPRTL___tgt_target_nowait : OMPRTL___tgt_target),
           OffloadingArgs);
@@ -9704,6 +9702,7 @@ void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,
     case OMPD_parallel_master_taskloop_simd:
     case OMPD_requires:
     case OMPD_unknown:
+    default:
       llvm_unreachable("Unknown target directive for OpenMP device codegen.");
     }
     return;
@@ -10054,7 +10053,7 @@ llvm::Function *CGOpenMPRuntime::emitRequiresDirectiveRegFun() {
            "Target or declare target region expected.");
     if (HasRequiresUnifiedSharedMemory)
       Flags = OMP_REQ_UNIFIED_SHARED_MEMORY;
-    CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+    CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___tgt_register_requires),
                         llvm::ConstantInt::get(CGM.Int64Ty, Flags));
     CGF.FinishFunction();
@@ -10082,9 +10081,8 @@ void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF,
   RealArgs.append(std::begin(Args), std::end(Args));
   RealArgs.append(CapturedVars.begin(), CapturedVars.end());
 
-  llvm::FunctionCallee RTLFn =
-      llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
-          CGM.getModule(), OMPRTL___kmpc_fork_teams);
+  llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(
+      CGM.getModule(), OMPRTL___kmpc_fork_teams);
   CGF.EmitRuntimeCall(RTLFn, RealArgs);
 }
 
@@ -10112,7 +10110,7 @@ void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
   // Build call __kmpc_push_num_teamss(&loc, global_tid, num_teams, thread_limit)
   llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,
                                      ThreadLimitVal};
-  CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+  CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                           CGM.getModule(), OMPRTL___kmpc_push_num_teams),
                       PushNumTeamsArgs);
 }
@@ -10167,7 +10165,7 @@ void CGOpenMPRuntime::emitTargetDataCalls(
     llvm::Value *OffloadingArgs[] = {
         DeviceID,         PointerNum,    BasePointersArrayArg,
         PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
-    CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+    CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___tgt_target_data_begin),
                         OffloadingArgs);
 
@@ -10204,7 +10202,7 @@ void CGOpenMPRuntime::emitTargetDataCalls(
     llvm::Value *OffloadingArgs[] = {
         DeviceID,         PointerNum,    BasePointersArrayArg,
         PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
-    CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+    CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___tgt_target_data_end),
                         OffloadingArgs);
   };
@@ -10362,12 +10360,13 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
     case OMPD_target_parallel_for_simd:
     case OMPD_requires:
     case OMPD_unknown:
+    default:
       llvm_unreachable("Unexpected standalone target data directive.");
       break;
     }
-    CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
-                            CGM.getModule(), RTLFn),
-                        OffloadingArgs);
+    CGF.EmitRuntimeCall(
+        OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), RTLFn),
+        OffloadingArgs);
   };
 
   auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray](
@@ -11058,15 +11057,13 @@ void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
           CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).getPointer(),
           CGM.VoidPtrTy)};
 
-  llvm::FunctionCallee RTLFn =
-      llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
-          CGM.getModule(), OMPRTL___kmpc_doacross_init);
+  llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(
+      CGM.getModule(), OMPRTL___kmpc_doacross_init);
   CGF.EmitRuntimeCall(RTLFn, Args);
   llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
       emitUpdateLocation(CGF, D.getEndLoc()), getThreadID(CGF, D.getEndLoc())};
-  llvm::FunctionCallee FiniRTLFn =
-      llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
-          CGM.getModule(), OMPRTL___kmpc_doacross_fini);
+  llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction(
+      CGM.getModule(), OMPRTL___kmpc_doacross_fini);
   CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn,
                                              llvm::makeArrayRef(FiniArgs));
 }
@@ -11094,12 +11091,12 @@ void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
       CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()};
   llvm::FunctionCallee RTLFn;
   if (C->getDependencyKind() == OMPC_DEPEND_source) {
-    RTLFn = llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
-        CGM.getModule(), OMPRTL___kmpc_doacross_post);
+    RTLFn = OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
+                                                  OMPRTL___kmpc_doacross_post);
   } else {
     assert(C->getDependencyKind() == OMPC_DEPEND_sink);
-    RTLFn = llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
-        CGM.getModule(), OMPRTL___kmpc_doacross_wait);
+    RTLFn = OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
+                                                  OMPRTL___kmpc_doacross_wait);
   }
   CGF.EmitRuntimeCall(RTLFn, Args);
 }
@@ -11203,14 +11200,13 @@ Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
   llvm::Value *Args[] = {ThreadID, Size, Allocator};
 
   llvm::Value *Addr =
-      CGF.EmitRuntimeCall(llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(
+      CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                               CGM.getModule(), OMPRTL___kmpc_alloc),
                           Args, getName({CVD->getName(), ".void.addr"}));
   llvm::Value *FiniArgs[OMPAllocateCleanupTy::CleanupArgs] = {ThreadID, Addr,
                                                               Allocator};
-  llvm::FunctionCallee FiniRTLFn =
-      llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction(CGM.getModule(),
-                                                        OMPRTL___kmpc_free);
+  llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction(
+      CGM.getModule(), OMPRTL___kmpc_free);
 
   CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FiniRTLFn,
                                                 llvm::makeArrayRef(FiniArgs));
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index dea92e16e59f8..eb22f155f5ef4 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -25,6 +25,7 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
+#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Support/AtomicOrdering.h"
@@ -37,6 +38,7 @@ class GlobalVariable;
 class StructType;
 class Type;
 class Value;
+class OpenMPIRBuilder;
 } // namespace llvm
 
 namespace clang {
@@ -284,6 +286,8 @@ class CGOpenMPRuntime {
     ~LastprivateConditionalRAII();
   };
 
+  llvm::OpenMPIRBuilder &getOMPBuilder() { return OMPBuilder; }
+
 protected:
   CodeGenModule &CGM;
   StringRef FirstSeparator, Separator;
@@ -368,6 +372,8 @@ class CGOpenMPRuntime {
   llvm::Value *getCriticalRegionLock(StringRef CriticalName);
 
 private:
+  /// An OpenMP-IR-Builder instance.
+  llvm::OpenMPIRBuilder OMPBuilder;
   /// Default const ident_t object used for initialization of all other
   /// ident_t objects.
   llvm::Constant *DefaultOpenMPPSource = nullptr;
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
index 896442daa4012..cbd443134e7a8 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
@@ -38,11 +38,9 @@ enum OpenMPRTLFunctionNVPTX {
   /// Call to void __kmpc_spmd_kernel_deinit_v2(int16_t RequiresOMPRuntime);
   OMPRTL_NVPTX__kmpc_spmd_kernel_deinit_v2,
   /// Call to void __kmpc_kernel_prepare_parallel(void
-  /// *outlined_function, int16_t
-  /// IsOMPRuntimeInitialized);
+  /// *outlined_function);
   OMPRTL_NVPTX__kmpc_kernel_prepare_parallel,
-  /// Call to bool __kmpc_kernel_parallel(void **outlined_function,
-  /// int16_t IsOMPRuntimeInitialized);
+  /// Call to bool __kmpc_kernel_parallel(void **outlined_function);
   OMPRTL_NVPTX__kmpc_kernel_parallel,
   /// Call to void __kmpc_kernel_end_parallel();
   OMPRTL_NVPTX__kmpc_kernel_end_parallel,
@@ -85,6 +83,9 @@ enum OpenMPRTLFunctionNVPTX {
   /// Call to void* __kmpc_data_sharing_coalesced_push_stack(size_t size,
   /// int16_t UseSharedMemory);
   OMPRTL_NVPTX__kmpc_data_sharing_coalesced_push_stack,
+  /// Call to void* __kmpc_data_sharing_push_stack(size_t size, int16_t
+  /// UseSharedMemory);
+  OMPRTL_NVPTX__kmpc_data_sharing_push_stack,
   /// Call to void __kmpc_data_sharing_pop_stack(void *a);
   OMPRTL_NVPTX__kmpc_data_sharing_pop_stack,
   /// Call to void __kmpc_begin_sharing_variables(void ***args,
@@ -816,6 +817,7 @@ static bool hasNestedSPMDDirective(ASTContext &Ctx,
     case OMPD_parallel_master_taskloop_simd:
     case OMPD_requires:
     case OMPD_unknown:
+    default:
       llvm_unreachable("Unexpected directive.");
     }
   }
@@ -896,6 +898,7 @@ static bool supportsSPMDExecutionMode(ASTContext &Ctx,
   case OMPD_parallel_master_taskloop_simd:
   case OMPD_requires:
   case OMPD_unknown:
+  default:
     break;
   }
   llvm_unreachable(
@@ -1069,6 +1072,7 @@ static bool hasNestedLightweightDirective(ASTContext &Ctx,
     case OMPD_parallel_master_taskloop_simd:
     case OMPD_requires:
     case OMPD_unknown:
+    default:
       llvm_unreachable("Unexpected directive.");
     }
   }
@@ -1155,6 +1159,7 @@ static bool supportsLightweightRuntime(ASTContext &Ctx,
   case OMPD_parallel_master_taskloop_simd:
   case OMPD_requires:
   case OMPD_unknown:
+  default:
     break;
   }
   llvm_unreachable(
@@ -1459,8 +1464,7 @@ void CGOpenMPRuntimeNVPTX::emitWorkerLoop(CodeGenFunction &CGF,
   CGF.InitTempAlloca(WorkFn, llvm::Constant::getNullValue(CGF.Int8PtrTy));
 
   // TODO: Optimize runtime initialization and pass in correct value.
-  llvm::Value *Args[] = {WorkFn.getPointer(),
-                         /*RequiresOMPRuntime=*/Bld.getInt16(1)};
+  llvm::Value *Args[] = {WorkFn.getPointer()};
   llvm::Value *Ret = CGF.EmitRuntimeCall(
       createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_parallel), Args);
   Bld.CreateStore(Bld.CreateZExt(Ret, CGF.Int8Ty), ExecStatus);
@@ -1588,17 +1592,16 @@ CGOpenMPRuntimeNVPTX::createNVPTXRuntimeFunction(unsigned Function) {
   }
   case OMPRTL_NVPTX__kmpc_kernel_prepare_parallel: {
     /// Build void __kmpc_kernel_prepare_parallel(
-    /// void *outlined_function, int16_t IsOMPRuntimeInitialized);
-    llvm::Type *TypeParams[] = {CGM.Int8PtrTy, CGM.Int16Ty};
+    /// void *outlined_function);
+    llvm::Type *TypeParams[] = {CGM.Int8PtrTy};
     auto *FnTy =
         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_prepare_parallel");
     break;
   }
   case OMPRTL_NVPTX__kmpc_kernel_parallel: {
-    /// Build bool __kmpc_kernel_parallel(void **outlined_function,
-    /// int16_t IsOMPRuntimeInitialized);
-    llvm::Type *TypeParams[] = {CGM.Int8PtrPtrTy, CGM.Int16Ty};
+    /// Build bool __kmpc_kernel_parallel(void **outlined_function);
+    llvm::Type *TypeParams[] = {CGM.Int8PtrPtrTy};
     llvm::Type *RetTy = CGM.getTypes().ConvertType(CGM.getContext().BoolTy);
     auto *FnTy =
         llvm::FunctionType::get(RetTy, TypeParams, /*isVarArg*/ false);
@@ -1753,6 +1756,16 @@ CGOpenMPRuntimeNVPTX::createNVPTXRuntimeFunction(unsigned Function) {
         FnTy, /*Name=*/"__kmpc_data_sharing_coalesced_push_stack");
     break;
   }
+  case OMPRTL_NVPTX__kmpc_data_sharing_push_stack: {
+    // Build void *__kmpc_data_sharing_push_stack(size_t size, int16_t
+    // UseSharedMemory);
+    llvm::Type *TypeParams[] = {CGM.SizeTy, CGM.Int16Ty};
+    auto *FnTy =
+        llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
+    RTLFn = CGM.CreateRuntimeFunction(
+        FnTy, /*Name=*/"__kmpc_data_sharing_push_stack");
+    break;
+  }
   case OMPRTL_NVPTX__kmpc_data_sharing_pop_stack: {
     // Build void __kmpc_data_sharing_pop_stack(void *a);
     llvm::Type *TypeParams[] = {CGM.VoidPtrTy};
@@ -2210,7 +2223,7 @@ void CGOpenMPRuntimeNVPTX::emitGenericVarsProlog(CodeGenFunction &CGF,
       GlobalRecCastAddr = Phi;
       I->getSecond().GlobalRecordAddr = Phi;
       I->getSecond().IsInSPMDModeFlag = IsSPMD;
-    } else if (IsInTTDRegion) {
+    } else if (!CGM.getLangOpts().OpenMPCUDATargetParallel && IsInTTDRegion) {
       assert(GlobalizedRecords.back().Records.size() < 2 &&
              "Expected less than 2 globalized records: one for target and one "
              "for teams.");
@@ -2283,12 +2296,16 @@ void CGOpenMPRuntimeNVPTX::emitGenericVarsProlog(CodeGenFunction &CGF,
     } else {
       // TODO: allow the usage of shared memory to be controlled by
       // the user, for now, default to global.
+      bool UseSharedMemory =
+          IsInTTDRegion && GlobalRecordSize <= SharedMemorySize;
       llvm::Value *GlobalRecordSizeArg[] = {
           llvm::ConstantInt::get(CGM.SizeTy, GlobalRecordSize),
-          CGF.Builder.getInt16(/*UseSharedMemory=*/0)};
+          CGF.Builder.getInt16(UseSharedMemory ? 1 : 0)};
       llvm::Value *GlobalRecValue = CGF.EmitRuntimeCall(
           createNVPTXRuntimeFunction(
-              OMPRTL_NVPTX__kmpc_data_sharing_coalesced_push_stack),
+              IsInTTDRegion
+                  ? OMPRTL_NVPTX__kmpc_data_sharing_push_stack
+                  : OMPRTL_NVPTX__kmpc_data_sharing_coalesced_push_stack),
           GlobalRecordSizeArg);
       GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
           GlobalRecValue, GlobalRecPtrTy);
@@ -2435,7 +2452,7 @@ void CGOpenMPRuntimeNVPTX::emitGenericVarsEpilog(CodeGenFunction &CGF,
                 OMPRTL_NVPTX__kmpc_data_sharing_pop_stack),
             CGF.EmitCastToVoidPtr(I->getSecond().GlobalRecordAddr));
         CGF.EmitBlock(ExitBB);
-      } else if (IsInTTDRegion) {
+      } else if (!CGM.getLangOpts().OpenMPCUDATargetParallel && IsInTTDRegion) {
         assert(GlobalizedRecords.back().RegionCounter > 0 &&
                "region counter must be > 0.");
         --GlobalizedRecords.back().RegionCounter;
@@ -2548,7 +2565,7 @@ void CGOpenMPRuntimeNVPTX::emitNonSPMDParallelCall(
     llvm::Value *ID = Bld.CreateBitOrPointerCast(WFn, CGM.Int8PtrTy);
 
     // Prepare for parallel region. Indicate the outlined function.
-    llvm::Value *Args[] = {ID, /*RequiresOMPRuntime=*/Bld.getInt16(1)};
+    llvm::Value *Args[] = {ID};
     CGF.EmitRuntimeCall(
         createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_prepare_parallel),
         Args);
@@ -5085,7 +5102,8 @@ static std::pair<unsigned, unsigned> getSMsBlocksPerSM(CodeGenModule &CGM) {
 }
 
 void CGOpenMPRuntimeNVPTX::clear() {
-  if (!GlobalizedRecords.empty()) {
+  if (!GlobalizedRecords.empty() &&
+      !CGM.getLangOpts().OpenMPCUDATargetParallel) {
     ASTContext &C = CGM.getContext();
     llvm::SmallVector<const GlobalPtrSizeRecsTy *, 4> GlobalRecs;
     llvm::SmallVector<const GlobalPtrSizeRecsTy *, 4> SharedRecs;
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index a3677bf7910b3..7135135d2a410 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -23,6 +23,7 @@
 #include "clang/AST/StmtOpenMP.h"
 #include "clang/Basic/OpenMPKinds.h"
 #include "clang/Basic/PrettyStackTrace.h"
+#include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Instructions.h"
@@ -1376,6 +1377,7 @@ void CodeGenFunction::EmitOMPReductionClauseInit(
     case OMPD_begin_declare_variant:
     case OMPD_end_declare_variant:
     case OMPD_unknown:
+    default:
       llvm_unreachable("Enexpected directive with task reductions.");
     }
 
@@ -1564,8 +1566,97 @@ static void emitEmptyBoundParameters(CodeGenFunction &,
                                      const OMPExecutableDirective &,
                                      llvm::SmallVectorImpl<llvm::Value *> &) {}
 
+Address CodeGenFunction::OMPBuilderCBHelpers::getAddressOfLocalVariable(
+    CodeGenFunction &CGF, const VarDecl *VD) {
+  CodeGenModule &CGM = CGF.CGM;
+  auto &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
+
+  if (!VD)
+    return Address::invalid();
+  const VarDecl *CVD = VD->getCanonicalDecl();
+  if (!CVD->hasAttr<OMPAllocateDeclAttr>())
+    return Address::invalid();
+  const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
+  // Use the default allocation.
+  if (AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
+      !AA->getAllocator())
+    return Address::invalid();
+  llvm::Value *Size;
+  CharUnits Align = CGM.getContext().getDeclAlign(CVD);
+  if (CVD->getType()->isVariablyModifiedType()) {
+    Size = CGF.getTypeSize(CVD->getType());
+    // Align the size: ((size + align - 1) / align) * align
+    Size = CGF.Builder.CreateNUWAdd(
+        Size, CGM.getSize(Align - CharUnits::fromQuantity(1)));
+    Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));
+    Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));
+  } else {
+    CharUnits Sz = CGM.getContext().getTypeSizeInChars(CVD->getType());
+    Size = CGM.getSize(Sz.alignTo(Align));
+  }
+
+  assert(AA->getAllocator() &&
+         "Expected allocator expression for non-default allocator.");
+  llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator());
+  // According to the standard, the original allocator type is a enum (integer).
+  // Convert to pointer type, if required.
+  if (Allocator->getType()->isIntegerTy())
+    Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy);
+  else if (Allocator->getType()->isPointerTy())
+    Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator,
+                                                                CGM.VoidPtrTy);
+
+  llvm::Value *Addr = OMPBuilder.CreateOMPAlloc(
+      CGF.Builder, Size, Allocator,
+      getNameWithSeparators({CVD->getName(), ".void.addr"}, ".", "."));
+  llvm::CallInst *FreeCI =
+      OMPBuilder.CreateOMPFree(CGF.Builder, Addr, Allocator);
+
+  CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI);
+  Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+      Addr,
+      CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())),
+      getNameWithSeparators({CVD->getName(), ".addr"}, ".", "."));
+  return Address(Addr, Align);
+}
+
+Address CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
+    CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr,
+    SourceLocation Loc) {
+  CodeGenModule &CGM = CGF.CGM;
+  if (CGM.getLangOpts().OpenMPUseTLS &&
+      CGM.getContext().getTargetInfo().isTLSSupported())
+    return VDAddr;
+
+  llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
+
+  llvm::Type *VarTy = VDAddr.getElementType();
+  llvm::Value *Data =
+      CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy);
+  llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy));
+  std::string Suffix = getNameWithSeparators({"cache", ""});
+  llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix);
+
+  llvm::CallInst *ThreadPrivateCacheCall =
+      OMPBuilder.CreateCachedThreadPrivate(CGF.Builder, Data, Size, CacheName);
+
+  return Address(ThreadPrivateCacheCall, VDAddr.getAlignment());
+}
+
+std::string CodeGenFunction::OMPBuilderCBHelpers::getNameWithSeparators(
+    ArrayRef<StringRef> Parts, StringRef FirstSeparator, StringRef Separator) {
+  SmallString<128> Buffer;
+  llvm::raw_svector_ostream OS(Buffer);
+  StringRef Sep = FirstSeparator;
+  for (StringRef Part : Parts) {
+    OS << Sep << Part;
+    Sep = Separator;
+  }
+  return OS.str().str();
+}
 void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
-  if (llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder()) {
+  if (CGM.getLangOpts().OpenMPIRBuilder) {
+    llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
     // Check if we have any if clause associated with the directive.
     llvm::Value *IfCond = nullptr;
     if (const auto *C = S.getSingleClause<OMPIfClause>())
@@ -1616,9 +1707,9 @@ void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
 
     CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
     CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
-    Builder.restoreIP(OMPBuilder->CreateParallel(Builder, BodyGenCB, PrivCB,
-                                                 FiniCB, IfCond, NumThreads,
-                                                 ProcBind, S.hasCancel()));
+    Builder.restoreIP(OMPBuilder.CreateParallel(Builder, BodyGenCB, PrivCB,
+                                                FiniCB, IfCond, NumThreads,
+                                                ProcBind, S.hasCancel()));
     return;
   }
 
@@ -3328,11 +3419,11 @@ void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
     // Generate condition for loop.
     BinaryOperator *Cond = BinaryOperator::Create(
         C, &IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue, OK_Ordinary,
-        S.getBeginLoc(), FPOptions(C.getLangOpts()));
+        S.getBeginLoc(), FPOptionsOverride());
     // Increment for loop counter.
     UnaryOperator *Inc = UnaryOperator::Create(
         C, &IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
-        S.getBeginLoc(), true, FPOptions(C.getLangOpts()));
+        S.getBeginLoc(), true, FPOptionsOverride());
     auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
       // Iterate through all sections and emit a switch construct:
       // switch (IV) {
@@ -3523,7 +3614,8 @@ static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
 }
 
 void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
-  if (llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder()) {
+  if (CGM.getLangOpts().OpenMPIRBuilder) {
+    llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
     using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
 
     const CapturedStmt *CS = S.getInnermostCapturedStmt();
@@ -3543,7 +3635,7 @@ void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
 
     CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
     CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
-    Builder.restoreIP(OMPBuilder->CreateMaster(Builder, BodyGenCB, FiniCB));
+    Builder.restoreIP(OMPBuilder.CreateMaster(Builder, BodyGenCB, FiniCB));
 
     return;
   }
@@ -3552,7 +3644,8 @@ void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
 }
 
 void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
-  if (llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder()) {
+  if (CGM.getLangOpts().OpenMPIRBuilder) {
+    llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
     using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
 
     const CapturedStmt *CS = S.getInnermostCapturedStmt();
@@ -3583,7 +3676,7 @@ void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
 
     CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
     CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
-    Builder.restoreIP(OMPBuilder->CreateCritical(
+    Builder.restoreIP(OMPBuilder.CreateCritical(
         Builder, BodyGenCB, FiniCB, S.getDirectiveName().getAsString(),
         HintInst));
 
@@ -3627,13 +3720,12 @@ void CodeGenFunction::EmitOMPParallelForSimdDirective(
   // directives: 'parallel' with 'for' directive.
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     Action.Enter(CGF);
-    CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
-                               emitDispatchForLoopBounds);
+    (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
   };
   {
     auto LPCRegion =
         CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
-    emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
+    emitCommonOMPParallelDirective(*this, S, OMPD_for_simd, CodeGen,
                                    emitEmptyBoundParameters);
   }
   // Check for outer lastprivate conditional update.
@@ -5212,6 +5304,7 @@ static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
   case OMPC_exclusive:
   case OMPC_uses_allocators:
   case OMPC_affinity:
+  default:
     llvm_unreachable("Clause is not allowed in 'omp atomic'.");
   }
 }
@@ -5784,7 +5877,8 @@ void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
       break;
     }
   }
-  if (llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder()) {
+  if (CGM.getLangOpts().OpenMPIRBuilder) {
+    llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
     // TODO: This check is necessary as we only generate `omp parallel` through
     // the OpenMPIRBuilder for now.
     if (S.getCancelRegion() == OMPD_parallel) {
@@ -5793,7 +5887,7 @@ void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
         IfCondition = EmitScalarExpr(IfCond,
                                      /*IgnoreResultAssign=*/true);
       return Builder.restoreIP(
-          OMPBuilder->CreateCancel(Builder, IfCondition, S.getCancelRegion()));
+          OMPBuilder.CreateCancel(Builder, IfCondition, S.getCancelRegion()));
     }
   }
 
diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp
index 276993581117e..65b3b0c5f53d0 100644
--- a/clang/lib/CodeGen/CGVTables.cpp
+++ b/clang/lib/CodeGen/CGVTables.cpp
@@ -363,7 +363,8 @@ void CodeGenFunction::EmitCallAndReturnForThunk(llvm::FunctionCallee Callee,
                                   : FPT->getReturnType();
   ReturnValueSlot Slot;
   if (!ResultType->isVoidType() &&
-      CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect)
+      (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect ||
+       hasAggregateEvaluationKind(ResultType)))
     Slot = ReturnValueSlot(ReturnValue, ResultType.isVolatileQualified(),
                            /*IsUnused=*/false, /*IsExternallyDestructed=*/true);
 
diff --git a/clang/lib/CodeGen/CodeGenABITypes.cpp b/clang/lib/CodeGen/CodeGenABITypes.cpp
index 322cf45c47af8..d3a16a1d5accf 100644
--- a/clang/lib/CodeGen/CodeGenABITypes.cpp
+++ b/clang/lib/CodeGen/CodeGenABITypes.cpp
@@ -115,3 +115,16 @@ unsigned CodeGen::getLLVMFieldNumber(CodeGenModule &CGM,
                                      const FieldDecl *FD) {
   return CGM.getTypes().getCGRecordLayout(RD).getLLVMFieldNo(FD);
 }
+
+llvm::Value *CodeGen::getCXXDestructorImplicitParam(
+    CodeGenModule &CGM, llvm::BasicBlock *InsertBlock,
+    llvm::BasicBlock::iterator InsertPoint, const CXXDestructorDecl *D,
+    CXXDtorType Type, bool ForVirtualBase, bool Delegating) {
+  CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
+  CGF.CurCodeDecl = D;
+  CGF.CurFuncDecl = D;
+  CGF.CurFn = InsertBlock->getParent();
+  CGF.Builder.SetInsertPoint(InsertBlock, InsertPoint);
+  return CGM.getCXXABI().getCXXDestructorImplicitParam(
+      CGF, D, Type, ForVirtualBase, Delegating);
+}
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 613e7df9d364d..8ce488f35dd32 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -87,8 +87,8 @@ CodeGenFunction::~CodeGenFunction() {
   // seems to be a reasonable spot. We do it here, as opposed to the deletion
   // time of the CodeGenModule, because we have to ensure the IR has not yet
   // been "emitted" to the outside, thus, modifications are still sensible.
-  if (llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder())
-    OMPBuilder->finalize();
+  if (CGM.getLangOpts().OpenMPIRBuilder)
+    CGM.getOpenMPRuntime().getOMPBuilder().finalize();
 }
 
 // Map the LangOption for exception behavior into
@@ -117,12 +117,12 @@ void CodeGenFunction::SetFPModel() {
 
 void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) {
   llvm::FastMathFlags FMF;
-  FMF.setAllowReassoc(FPFeatures.allowAssociativeMath());
-  FMF.setNoNaNs(FPFeatures.noHonorNaNs());
-  FMF.setNoInfs(FPFeatures.noHonorInfs());
-  FMF.setNoSignedZeros(FPFeatures.noSignedZeros());
-  FMF.setAllowReciprocal(FPFeatures.allowReciprocalMath());
-  FMF.setApproxFunc(FPFeatures.allowApproximateFunctions());
+  FMF.setAllowReassoc(FPFeatures.getAllowFPReassociate());
+  FMF.setNoNaNs(FPFeatures.getNoHonorNaNs());
+  FMF.setNoInfs(FPFeatures.getNoHonorInfs());
+  FMF.setNoSignedZeros(FPFeatures.getNoSignedZero());
+  FMF.setAllowReciprocal(FPFeatures.getAllowReciprocal());
+  FMF.setApproxFunc(FPFeatures.getAllowApproxFunc());
   FMF.setAllowContract(FPFeatures.allowFPContractAcrossStatement());
   Builder.setFastMathFlags(FMF);
 }
@@ -137,10 +137,12 @@ CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
 
   FMFGuard.emplace(CGF.Builder);
 
-  auto NewRoundingBehavior = FPFeatures.getRoundingMode();
+  llvm::RoundingMode NewRoundingBehavior =
+      static_cast<llvm::RoundingMode>(FPFeatures.getRoundingMode());
   CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);
   auto NewExceptionBehavior =
-      ToConstrainedExceptMD(FPFeatures.getExceptionMode());
+      ToConstrainedExceptMD(static_cast<LangOptions::FPExceptionModeKind>(
+          FPFeatures.getFPExceptionMode()));
   CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);
 
   CGF.SetFastMathFlags(FPFeatures);
@@ -159,13 +161,13 @@ CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
     if (OldValue != NewValue)
       CGF.CurFn->addFnAttr(Name, llvm::toStringRef(NewValue));
   };
-  mergeFnAttrValue("no-infs-fp-math", FPFeatures.noHonorInfs());
-  mergeFnAttrValue("no-nans-fp-math", FPFeatures.noHonorNaNs());
-  mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.noSignedZeros());
-  mergeFnAttrValue(
-      "unsafe-fp-math",
-      FPFeatures.allowAssociativeMath() && FPFeatures.allowReciprocalMath() &&
-          FPFeatures.allowApproximateFunctions() && FPFeatures.noSignedZeros());
+  mergeFnAttrValue("no-infs-fp-math", FPFeatures.getNoHonorInfs());
+  mergeFnAttrValue("no-nans-fp-math", FPFeatures.getNoHonorNaNs());
+  mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.getNoSignedZero());
+  mergeFnAttrValue("unsafe-fp-math", FPFeatures.getAllowFPReassociate() &&
+                                         FPFeatures.getAllowReciprocal() &&
+                                         FPFeatures.getAllowApproxFunc() &&
+                                         FPFeatures.getNoSignedZero());
 }
 
 CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 37a322567f51b..1fc2ed76ca9e6 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -26,6 +26,7 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/ExprOpenMP.h"
+#include "clang/AST/StmtOpenMP.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/ABI.h"
 #include "clang/Basic/CapturedStmt.h"
@@ -80,6 +81,7 @@ class OMPUseDevicePtrClause;
 class OMPUseDeviceAddrClause;
 class ReturnsNonNullAttr;
 class SVETypeFlags;
+class OMPExecutableDirective;
 
 namespace analyze_os_log {
 class OSLogBufferLayout;
@@ -259,117 +261,12 @@ class CodeGenFunction : public CodeGenTypeCache {
     unsigned Index;
   };
 
-  // Helper class for the OpenMP IR Builder. Allows reusability of code used for
-  // region body, and finalization codegen callbacks. This will class will also
-  // contain privatization functions used by the privatization call backs
-  struct OMPBuilderCBHelpers {
-
-    using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
-
-    /// Emit the Finalization for an OMP region
-    /// \param CGF	The Codegen function this belongs to
-    /// \param IP	Insertion point for generating the finalization code.
-    static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP) {
-      CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
-      assert(IP.getBlock()->end() != IP.getPoint() &&
-             "OpenMP IR Builder should cause terminated block!");
-
-      llvm::BasicBlock *IPBB = IP.getBlock();
-      llvm::BasicBlock *DestBB = IPBB->getUniqueSuccessor();
-      assert(DestBB && "Finalization block should have one successor!");
-
-      // erase and replace with cleanup branch.
-      IPBB->getTerminator()->eraseFromParent();
-      CGF.Builder.SetInsertPoint(IPBB);
-      CodeGenFunction::JumpDest Dest = CGF.getJumpDestInCurrentScope(DestBB);
-      CGF.EmitBranchThroughCleanup(Dest);
-    }
-
-    /// Emit the body of an OMP region
-    /// \param CGF	The Codegen function this belongs to
-    /// \param RegionBodyStmt	The body statement for the OpenMP region being
-    /// 			 generated
-    /// \param CodeGenIP	Insertion point for generating the body code.
-    /// \param FiniBB	The finalization basic block
-    static void EmitOMPRegionBody(CodeGenFunction &CGF,
-                                  const Stmt *RegionBodyStmt,
-                                  InsertPointTy CodeGenIP,
-                                  llvm::BasicBlock &FiniBB) {
-      llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
-      if (llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminator())
-        CodeGenIPBBTI->eraseFromParent();
-
-      CGF.Builder.SetInsertPoint(CodeGenIPBB);
-
-      CGF.EmitStmt(RegionBodyStmt);
-
-      if (CGF.Builder.saveIP().isSet())
-        CGF.Builder.CreateBr(&FiniBB);
-    }
-
-    /// RAII for preserving necessary info during Outlined region body codegen.
-    class OutlinedRegionBodyRAII {
-
-      llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
-      CodeGenFunction::JumpDest OldReturnBlock;
-      CodeGenFunction &CGF;
-
-    public:
-      OutlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
-                             llvm::BasicBlock &RetBB)
-          : CGF(cgf) {
-        assert(AllocaIP.isSet() &&
-               "Must specify Insertion point for allocas of outlined function");
-        OldAllocaIP = CGF.AllocaInsertPt;
-        CGF.AllocaInsertPt = &*AllocaIP.getPoint();
-
-        OldReturnBlock = CGF.ReturnBlock;
-        CGF.ReturnBlock = CGF.getJumpDestInCurrentScope(&RetBB);
-      }
-
-      ~OutlinedRegionBodyRAII() {
-        CGF.AllocaInsertPt = OldAllocaIP;
-        CGF.ReturnBlock = OldReturnBlock;
-      }
-    };
-
-    /// RAII for preserving necessary info during inlined region body codegen.
-    class InlinedRegionBodyRAII {
-
-      llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
-      CodeGenFunction &CGF;
-
-    public:
-      InlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
-                            llvm::BasicBlock &FiniBB)
-          : CGF(cgf) {
-        // Alloca insertion block should be in the entry block of the containing
-        // function so it expects an empty AllocaIP in which case will reuse the
-        // old alloca insertion point, or a new AllocaIP in the same block as
-        // the old one
-        assert((!AllocaIP.isSet() ||
-                CGF.AllocaInsertPt->getParent() == AllocaIP.getBlock()) &&
-               "Insertion point should be in the entry block of containing "
-               "function!");
-        OldAllocaIP = CGF.AllocaInsertPt;
-        if (AllocaIP.isSet())
-          CGF.AllocaInsertPt = &*AllocaIP.getPoint();
-
-        // TODO: Remove the call, after making sure the counter is not used by
-        //       the EHStack.
-        // Since this is an inlined region, it should not modify the
-        // ReturnBlock, and should reuse the one for the enclosing outlined
-        // region. So, the JumpDest being return by the function is discarded
-        (void)CGF.getJumpDestInCurrentScope(&FiniBB);
-      }
-
-      ~InlinedRegionBodyRAII() { CGF.AllocaInsertPt = OldAllocaIP; }
-    };
-  };
-
   CodeGenModule &CGM;  // Per-module state.
   const TargetInfo &Target;
 
+  // For EH/SEH outlined funclets, this field points to parent's CGF
+  CodeGenFunction *ParentCGF = nullptr;
+
   typedef std::pair<llvm::Value *, llvm::Value *> ComplexPairTy;
   LoopInfoStack LoopStack;
   CGBuilderTy Builder;
@@ -1695,6 +1592,169 @@ class CodeGenFunction : public CodeGenTypeCache {
     CallArgList OldCXXInheritedCtorInitExprArgs;
   };
 
+  // Helper class for the OpenMP IR Builder. Allows reusability of code used for
+  // region body, and finalization codegen callbacks. This will class will also
+  // contain privatization functions used by the privatization call backs
+  //
+  // TODO: this is temporary class for things that are being moved out of
+  // CGOpenMPRuntime, new versions of current CodeGenFunction methods, or
+  // utility function for use with the OMPBuilder. Once that move to use the
+  // OMPBuilder is done, everything here will either become part of CodeGenFunc.
+  // directly, or a new helper class that will contain functions used by both
+  // this and the OMPBuilder
+
+  struct OMPBuilderCBHelpers {
+
+    OMPBuilderCBHelpers() = delete;
+    OMPBuilderCBHelpers(const OMPBuilderCBHelpers &) = delete;
+    OMPBuilderCBHelpers &operator=(const OMPBuilderCBHelpers &) = delete;
+
+    using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
+
+    /// Cleanup action for allocate support.
+    class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
+
+    private:
+      llvm::CallInst *RTLFnCI;
+
+    public:
+      OMPAllocateCleanupTy(llvm::CallInst *RLFnCI) : RTLFnCI(RLFnCI) {
+        RLFnCI->removeFromParent();
+      }
+
+      void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
+        if (!CGF.HaveInsertPoint())
+          return;
+        CGF.Builder.Insert(RTLFnCI);
+      }
+    };
+
+    /// Returns address of the threadprivate variable for the current
+    /// thread. This Also create any necessary OMP runtime calls.
+    ///
+    /// \param VD VarDecl for Threadprivate variable.
+    /// \param VDAddr Address of the Vardecl
+    /// \param Loc  The location where the barrier directive was encountered
+    static Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
+                                          const VarDecl *VD, Address VDAddr,
+                                          SourceLocation Loc);
+
+    /// Gets the OpenMP-specific address of the local variable /p VD.
+    static Address getAddressOfLocalVariable(CodeGenFunction &CGF,
+                                             const VarDecl *VD);
+    /// Get the platform-specific name separator.
+    /// \param Parts different parts of the final name that needs separation
+    /// \param FirstSeparator First separator used between the initial two
+    ///        parts of the name.
+    /// \param Separator separator used between all of the rest consecutinve
+    ///        parts of the name
+    static std::string getNameWithSeparators(ArrayRef<StringRef> Parts,
+                                             StringRef FirstSeparator = ".",
+                                             StringRef Separator = ".");
+    /// Emit the Finalization for an OMP region
+    /// \param CGF	The Codegen function this belongs to
+    /// \param IP	Insertion point for generating the finalization code.
+    static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP) {
+      CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
+      assert(IP.getBlock()->end() != IP.getPoint() &&
+             "OpenMP IR Builder should cause terminated block!");
+
+      llvm::BasicBlock *IPBB = IP.getBlock();
+      llvm::BasicBlock *DestBB = IPBB->getUniqueSuccessor();
+      assert(DestBB && "Finalization block should have one successor!");
+
+      // erase and replace with cleanup branch.
+      IPBB->getTerminator()->eraseFromParent();
+      CGF.Builder.SetInsertPoint(IPBB);
+      CodeGenFunction::JumpDest Dest = CGF.getJumpDestInCurrentScope(DestBB);
+      CGF.EmitBranchThroughCleanup(Dest);
+    }
+
+    /// Emit the body of an OMP region
+    /// \param CGF	The Codegen function this belongs to
+    /// \param RegionBodyStmt	The body statement for the OpenMP region being
+    /// 			 generated
+    /// \param CodeGenIP	Insertion point for generating the body code.
+    /// \param FiniBB	The finalization basic block
+    static void EmitOMPRegionBody(CodeGenFunction &CGF,
+                                  const Stmt *RegionBodyStmt,
+                                  InsertPointTy CodeGenIP,
+                                  llvm::BasicBlock &FiniBB) {
+      llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
+      if (llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminator())
+        CodeGenIPBBTI->eraseFromParent();
+
+      CGF.Builder.SetInsertPoint(CodeGenIPBB);
+
+      CGF.EmitStmt(RegionBodyStmt);
+
+      if (CGF.Builder.saveIP().isSet())
+        CGF.Builder.CreateBr(&FiniBB);
+    }
+
+    /// RAII for preserving necessary info during Outlined region body codegen.
+    class OutlinedRegionBodyRAII {
+
+      llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
+      CodeGenFunction::JumpDest OldReturnBlock;
+      CGBuilderTy::InsertPoint IP;
+      CodeGenFunction &CGF;
+
+    public:
+      OutlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
+                             llvm::BasicBlock &RetBB)
+          : CGF(cgf) {
+        assert(AllocaIP.isSet() &&
+               "Must specify Insertion point for allocas of outlined function");
+        OldAllocaIP = CGF.AllocaInsertPt;
+        CGF.AllocaInsertPt = &*AllocaIP.getPoint();
+        IP = CGF.Builder.saveIP();
+
+        OldReturnBlock = CGF.ReturnBlock;
+        CGF.ReturnBlock = CGF.getJumpDestInCurrentScope(&RetBB);
+      }
+
+      ~OutlinedRegionBodyRAII() {
+        CGF.AllocaInsertPt = OldAllocaIP;
+        CGF.ReturnBlock = OldReturnBlock;
+        CGF.Builder.restoreIP(IP);
+      }
+    };
+
+    /// RAII for preserving necessary info during inlined region body codegen.
+    class InlinedRegionBodyRAII {
+
+      llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
+      CodeGenFunction &CGF;
+
+    public:
+      InlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
+                            llvm::BasicBlock &FiniBB)
+          : CGF(cgf) {
+        // Alloca insertion block should be in the entry block of the containing
+        // function so it expects an empty AllocaIP in which case will reuse the
+        // old alloca insertion point, or a new AllocaIP in the same block as
+        // the old one
+        assert((!AllocaIP.isSet() ||
+                CGF.AllocaInsertPt->getParent() == AllocaIP.getBlock()) &&
+               "Insertion point should be in the entry block of containing "
+               "function!");
+        OldAllocaIP = CGF.AllocaInsertPt;
+        if (AllocaIP.isSet())
+          CGF.AllocaInsertPt = &*AllocaIP.getPoint();
+
+        // TODO: Remove the call, after making sure the counter is not used by
+        //       the EHStack.
+        // Since this is an inlined region, it should not modify the
+        // ReturnBlock, and should reuse the one for the enclosing outlined
+        // region. So, the JumpDest being return by the function is discarded
+        (void)CGF.getJumpDestInCurrentScope(&FiniBB);
+      }
+
+      ~InlinedRegionBodyRAII() { CGF.AllocaInsertPt = OldAllocaIP; }
+    };
+  };
+
 private:
   /// CXXThisDecl - When generating code for a C++ member function,
   /// this will hold the implicit 'this' declaration.
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 7a9df700581e9..4ae8ce7e5ccf1 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -222,14 +222,6 @@ void CodeGenModule::createOpenMPRuntime() {
       OpenMPRuntime.reset(new CGOpenMPRuntime(*this));
     break;
   }
-
-  // The OpenMP-IR-Builder should eventually replace the above runtime codegens
-  // but we are not there yet so they both reside in CGModule for now and the
-  // OpenMP-IR-Builder is opt-in only.
-  if (LangOpts.OpenMPIRBuilder) {
-    OMPBuilder.reset(new llvm::OpenMPIRBuilder(TheModule));
-    OMPBuilder->initialize();
-  }
 }
 
 void CodeGenModule::createCUDARuntime() {
@@ -663,7 +655,7 @@ void CodeGenModule::Release() {
   if (!getCodeGenOpts().RecordCommandLine.empty())
     EmitCommandLineMetadata();
 
-  EmitTargetMetadata();
+  getTargetCodeGenInfo().emitTargetMetadata(*this, MangledDeclNames);
 
   EmitBackendOptionsMetadata(getCodeGenOpts());
 }
@@ -3656,26 +3648,29 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName,
 }
 
 llvm::Constant *
-CodeGenModule::GetAddrOfGlobal(GlobalDecl GD,
-                               ForDefinition_t IsForDefinition) {
+CodeGenModule::GetAddrOfGlobal(GlobalDecl GD, ForDefinition_t IsForDefinition) {
   const Decl *D = GD.getDecl();
+
   if (isa<CXXConstructorDecl>(D) || isa<CXXDestructorDecl>(D))
     return getAddrOfCXXStructor(GD, /*FnInfo=*/nullptr, /*FnType=*/nullptr,
                                 /*DontDefer=*/false, IsForDefinition);
-  else if (isa<CXXMethodDecl>(D)) {
-    auto FInfo = &getTypes().arrangeCXXMethodDeclaration(
-        cast<CXXMethodDecl>(D));
+
+  if (isa<CXXMethodDecl>(D)) {
+    auto FInfo =
+        &getTypes().arrangeCXXMethodDeclaration(cast<CXXMethodDecl>(D));
     auto Ty = getTypes().GetFunctionType(*FInfo);
     return GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/false,
                              IsForDefinition);
-  } else if (isa<FunctionDecl>(D)) {
+  }
+
+  if (isa<FunctionDecl>(D)) {
     const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
     llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
     return GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/false,
                              IsForDefinition);
-  } else
-    return GetAddrOfGlobalVar(cast<VarDecl>(D), /*Ty=*/nullptr,
-                              IsForDefinition);
+  }
+
+  return GetAddrOfGlobalVar(cast<VarDecl>(D), /*Ty=*/nullptr, IsForDefinition);
 }
 
 llvm::GlobalVariable *CodeGenModule::CreateOrReplaceCXXRuntimeVariable(
@@ -5788,21 +5783,6 @@ void CodeGenModule::EmitCommandLineMetadata() {
   CommandLineMetadata->addOperand(llvm::MDNode::get(Ctx, CommandLineNode));
 }
 
-void CodeGenModule::EmitTargetMetadata() {
-  // Warning, new MangledDeclNames may be appended within this loop.
-  // We rely on MapVector insertions adding new elements to the end
-  // of the container.
-  // FIXME: Move this loop into the one target that needs it, and only
-  // loop over those declarations for which we couldn't emit the target
-  // metadata when we emitted the declaration.
-  for (unsigned I = 0; I != MangledDeclNames.size(); ++I) {
-    auto Val = *(MangledDeclNames.begin() + I);
-    const Decl *D = Val.first.getDecl()->getMostRecentDecl();
-    llvm::GlobalValue *GV = GetGlobalValue(Val.second);
-    getTargetCodeGenInfo().emitTargetMD(D, GV, *this);
-  }
-}
-
 void CodeGenModule::EmitCoverageFile() {
   if (getCodeGenOpts().CoverageDataFile.empty() &&
       getCodeGenOpts().CoverageNotesFile.empty())
@@ -5990,6 +5970,9 @@ CharUnits CodeGenModule::getNaturalTypeAlignment(QualType T,
   if (TBAAInfo)
     *TBAAInfo = getTBAAAccessInfo(T);
 
+  // FIXME: This duplicates logic in ASTContext::getTypeAlignIfKnown. But
+  // that doesn't return the information we need to compute BaseInfo.
+
   // Honor alignment typedef attributes even on incomplete types.
   // We also honor them straight for C++ class types, even as pointees;
   // there's an expressivity gap here.
@@ -6001,32 +5984,46 @@ CharUnits CodeGenModule::getNaturalTypeAlignment(QualType T,
     }
   }
 
+  bool AlignForArray = T->isArrayType();
+
+  // Analyze the base element type, so we don't get confused by incomplete
+  // array types.
+  T = getContext().getBaseElementType(T);
+
+  if (T->isIncompleteType()) {
+    // We could try to replicate the logic from
+    // ASTContext::getTypeAlignIfKnown, but nothing uses the alignment if the
+    // type is incomplete, so it's impossible to test. We could try to reuse
+    // getTypeAlignIfKnown, but that doesn't return the information we need
+    // to set BaseInfo.  So just ignore the possibility that the alignment is
+    // greater than one.
+    if (BaseInfo)
+      *BaseInfo = LValueBaseInfo(AlignmentSource::Type);
+    return CharUnits::One();
+  }
+
   if (BaseInfo)
     *BaseInfo = LValueBaseInfo(AlignmentSource::Type);
 
   CharUnits Alignment;
-  if (T->isIncompleteType()) {
-    Alignment = CharUnits::One(); // Shouldn't be used, but pessimistic is best.
+  // For C++ class pointees, we don't know whether we're pointing at a
+  // base or a complete object, so we generally need to use the
+  // non-virtual alignment.
+  const CXXRecordDecl *RD;
+  if (forPointeeType && !AlignForArray && (RD = T->getAsCXXRecordDecl())) {
+    Alignment = getClassPointerAlignment(RD);
   } else {
-    // For C++ class pointees, we don't know whether we're pointing at a
-    // base or a complete object, so we generally need to use the
-    // non-virtual alignment.
-    const CXXRecordDecl *RD;
-    if (forPointeeType && (RD = T->getAsCXXRecordDecl())) {
-      Alignment = getClassPointerAlignment(RD);
-    } else {
-      Alignment = getContext().getTypeAlignInChars(T);
-      if (T.getQualifiers().hasUnaligned())
-        Alignment = CharUnits::One();
-    }
+    Alignment = getContext().getTypeAlignInChars(T);
+    if (T.getQualifiers().hasUnaligned())
+      Alignment = CharUnits::One();
+  }
 
-    // Cap to the global maximum type alignment unless the alignment
-    // was somehow explicit on the type.
-    if (unsigned MaxAlign = getLangOpts().MaxTypeAlign) {
-      if (Alignment.getQuantity() > MaxAlign &&
-          !getContext().isAlignmentRequired(T))
-        Alignment = CharUnits::fromQuantity(MaxAlign);
-    }
+  // Cap to the global maximum type alignment unless the alignment
+  // was somehow explicit on the type.
+  if (unsigned MaxAlign = getLangOpts().MaxTypeAlign) {
+    if (Alignment.getQuantity() > MaxAlign &&
+        !getContext().isAlignmentRequired(T))
+      Alignment = CharUnits::fromQuantity(MaxAlign);
   }
   return Alignment;
 }
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index 59b27fa32a1b3..a6c4a1f7b278c 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -324,7 +324,6 @@ class CodeGenModule : public CodeGenTypeCache {
   std::unique_ptr<CGObjCRuntime> ObjCRuntime;
   std::unique_ptr<CGOpenCLRuntime> OpenCLRuntime;
   std::unique_ptr<CGOpenMPRuntime> OpenMPRuntime;
-  std::unique_ptr<llvm::OpenMPIRBuilder> OMPBuilder;
   std::unique_ptr<CGCUDARuntime> CUDARuntime;
   std::unique_ptr<CGDebugInfo> DebugInfo;
   std::unique_ptr<ObjCEntrypoints> ObjCData;
@@ -597,9 +596,6 @@ class CodeGenModule : public CodeGenTypeCache {
     return *OpenMPRuntime;
   }
 
-  /// Return a pointer to the configured OpenMPIRBuilder, if any.
-  llvm::OpenMPIRBuilder *getOpenMPIRBuilder() { return OMPBuilder.get(); }
-
   /// Return a reference to the configured CUDA runtime.
   CGCUDARuntime &getCUDARuntime() {
     assert(CUDARuntime != nullptr);
@@ -1532,9 +1528,6 @@ class CodeGenModule : public CodeGenTypeCache {
   /// Emit the Clang commandline as llvm.commandline metadata.
   void EmitCommandLineMetadata();
 
-  /// Emits target specific Metadata for global declarations.
-  void EmitTargetMetadata();
-
   /// Emit the module flag metadata used to pass options controlling the
   /// the backend to LLVM.
   void EmitBackendOptionsMetadata(const CodeGenOptions CodeGenOpts);
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index 4811e309abbe9..d431c0263666e 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -619,6 +619,12 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
       return GET_SVE_FP_VEC(DoubleTy, false, 8);
     case BuiltinType::SveBFloat16:
       return GET_SVE_FP_VEC(BFloat16Ty, false, 8);
+    case BuiltinType::SveBFloat16x2:
+      return GET_SVE_FP_VEC(BFloat16Ty, false, 16);
+    case BuiltinType::SveBFloat16x3:
+      return GET_SVE_FP_VEC(BFloat16Ty, false, 24);
+    case BuiltinType::SveBFloat16x4:
+      return GET_SVE_FP_VEC(BFloat16Ty, false, 32);
 #undef GET_SVE_FP_VEC
     case BuiltinType::Dependent:
 #define BUILTIN_TYPE(Id, SingletonId)
diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp
index cdbfc88e7b707..78b268f423cbf 100644
--- a/clang/lib/CodeGen/CoverageMappingGen.cpp
+++ b/clang/lib/CodeGen/CoverageMappingGen.cpp
@@ -908,6 +908,18 @@ struct CounterCoverageMappingBuilder
     terminateRegion(S);
   }
 
+  void VisitCoroutineBodyStmt(const CoroutineBodyStmt *S) {
+    extendRegion(S);
+    Visit(S->getBody());
+  }
+
+  void VisitCoreturnStmt(const CoreturnStmt *S) {
+    extendRegion(S);
+    if (S->getOperand())
+      Visit(S->getOperand());
+    terminateRegion(S);
+  }
+
   void VisitCXXThrowExpr(const CXXThrowExpr *E) {
     extendRegion(E);
     if (E->getSubExpr())
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index 2829877cfe5d9..80de2a6e39505 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -228,6 +228,12 @@ class ItaniumCXXABI : public CodeGen::CGCXXABI {
                                                bool ForVirtualBase,
                                                bool Delegating) override;
 
+  llvm::Value *getCXXDestructorImplicitParam(CodeGenFunction &CGF,
+                                             const CXXDestructorDecl *DD,
+                                             CXXDtorType Type,
+                                             bool ForVirtualBase,
+                                             bool Delegating) override;
+
   void EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD,
                           CXXDtorType Type, bool ForVirtualBase,
                           bool Delegating, Address This,
@@ -1693,13 +1699,21 @@ CGCXXABI::AddedStructorArgs ItaniumCXXABI::getImplicitConstructorArgs(
   return AddedStructorArgs::prefix({{VTT, VTTTy}});
 }
 
+llvm::Value *ItaniumCXXABI::getCXXDestructorImplicitParam(
+    CodeGenFunction &CGF, const CXXDestructorDecl *DD, CXXDtorType Type,
+    bool ForVirtualBase, bool Delegating) {
+  GlobalDecl GD(DD, Type);
+  return CGF.GetVTTParameter(GD, ForVirtualBase, Delegating);
+}
+
 void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
                                        const CXXDestructorDecl *DD,
                                        CXXDtorType Type, bool ForVirtualBase,
                                        bool Delegating, Address This,
                                        QualType ThisTy) {
   GlobalDecl GD(DD, Type);
-  llvm::Value *VTT = CGF.GetVTTParameter(GD, ForVirtualBase, Delegating);
+  llvm::Value *VTT =
+      getCXXDestructorImplicitParam(CGF, DD, Type, ForVirtualBase, Delegating);
   QualType VTTTy = getContext().getPointerType(getContext().VoidPtrTy);
 
   CGCallee Callee;
diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
index 08e30527af47c..45c6cb6b2e0d1 100644
--- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -259,6 +259,12 @@ class MicrosoftCXXABI : public CGCXXABI {
                                                bool ForVirtualBase,
                                                bool Delegating) override;
 
+  llvm::Value *getCXXDestructorImplicitParam(CodeGenFunction &CGF,
+                                             const CXXDestructorDecl *DD,
+                                             CXXDtorType Type,
+                                             bool ForVirtualBase,
+                                             bool Delegating) override;
+
   void EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD,
                           CXXDtorType Type, bool ForVirtualBase,
                           bool Delegating, Address This,
@@ -1577,6 +1583,12 @@ CGCXXABI::AddedStructorArgs MicrosoftCXXABI::getImplicitConstructorArgs(
   return AddedStructorArgs::suffix({{MostDerivedArg, getContext().IntTy}});
 }
 
+llvm::Value *MicrosoftCXXABI::getCXXDestructorImplicitParam(
+    CodeGenFunction &CGF, const CXXDestructorDecl *DD, CXXDtorType Type,
+    bool ForVirtualBase, bool Delegating) {
+  return nullptr;
+}
+
 void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
                                          const CXXDestructorDecl *DD,
                                          CXXDtorType Type, bool ForVirtualBase,
@@ -1603,8 +1615,11 @@ void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
     BaseDtorEndBB = EmitDtorCompleteObjectHandler(CGF);
   }
 
+  llvm::Value *Implicit =
+      getCXXDestructorImplicitParam(CGF, DD, Type, ForVirtualBase,
+                                    Delegating); // = nullptr
   CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy,
-                            /*ImplicitParam=*/nullptr,
+                            /*ImplicitParam=*/Implicit,
                             /*ImplicitParamTy=*/QualType(), nullptr);
   if (BaseDtorEndBB) {
     // Complete object handler should continue to be the remaining
diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp
index 9710e676e58f8..9cd63ebe29ee0 100644
--- a/clang/lib/CodeGen/TargetInfo.cpp
+++ b/clang/lib/CodeGen/TargetInfo.cpp
@@ -20,6 +20,7 @@
 #include "clang/AST/Attr.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/Basic/CodeGenOptions.h"
+#include "clang/Basic/DiagnosticFrontend.h"
 #include "clang/CodeGen/CGFunctionInfo.h"
 #include "clang/CodeGen/SwiftCallingConv.h"
 #include "llvm/ADT/SmallBitVector.h"
@@ -498,11 +499,15 @@ static bool isEmptyField(ASTContext &Context, const FieldDecl *FD,
 
   // Constant arrays of empty records count as empty, strip them off.
   // Constant arrays of zero length always count as empty.
+  bool WasArray = false;
   if (AllowArrays)
     while (const ConstantArrayType *AT = Context.getAsConstantArrayType(FT)) {
       if (AT->getSize() == 0)
         return true;
       FT = AT->getElementType();
+      // The [[no_unique_address]] special case below does not apply to
+      // arrays of C++ empty records, so we need to remember this fact.
+      WasArray = true;
     }
 
   const RecordType *RT = FT->getAs<RecordType>();
@@ -513,7 +518,14 @@ static bool isEmptyField(ASTContext &Context, const FieldDecl *FD,
   //
   // FIXME: We should use a predicate for whether this behavior is true in the
   // current ABI.
-  if (isa<CXXRecordDecl>(RT->getDecl()))
+  //
+  // The exception to the above rule are fields marked with the
+  // [[no_unique_address]] attribute (since C++20).  Those do count as empty
+  // according to the Itanium ABI.  The exception applies only to records,
+  // not arrays of records, so we must also check whether we stripped off an
+  // array type above.
+  if (isa<CXXRecordDecl>(RT->getDecl()) &&
+      (WasArray || !FD->hasAttr<NoUniqueAddressAttr>()))
     return false;
 
   return isEmptyRecord(Context, FT, AllowArrays);
@@ -2466,8 +2478,110 @@ class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
       }
     }
   }
+
+  void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
+                            const FunctionDecl *Caller,
+                            const FunctionDecl *Callee,
+                            const CallArgList &Args) const override;
 };
 
+static void initFeatureMaps(const ASTContext &Ctx,
+                            llvm::StringMap<bool> &CallerMap,
+                            const FunctionDecl *Caller,
+                            llvm::StringMap<bool> &CalleeMap,
+                            const FunctionDecl *Callee) {
+  if (CalleeMap.empty() && CallerMap.empty()) {
+    // The caller is potentially nullptr in the case where the call isn't in a
+    // function.  In this case, the getFunctionFeatureMap ensures we just get
+    // the TU level setting (since it cannot be modified by 'target'..
+    Ctx.getFunctionFeatureMap(CallerMap, Caller);
+    Ctx.getFunctionFeatureMap(CalleeMap, Callee);
+  }
+}
+
+static bool checkAVXParamFeature(DiagnosticsEngine &Diag,
+                                 SourceLocation CallLoc,
+                                 const llvm::StringMap<bool> &CallerMap,
+                                 const llvm::StringMap<bool> &CalleeMap,
+                                 QualType Ty, StringRef Feature,
+                                 bool IsArgument) {
+  bool CallerHasFeat = CallerMap.lookup(Feature);
+  bool CalleeHasFeat = CalleeMap.lookup(Feature);
+  if (!CallerHasFeat && !CalleeHasFeat)
+    return Diag.Report(CallLoc, diag::warn_avx_calling_convention)
+           << IsArgument << Ty << Feature;
+
+  // Mixing calling conventions here is very clearly an error.
+  if (!CallerHasFeat || !CalleeHasFeat)
+    return Diag.Report(CallLoc, diag::err_avx_calling_convention)
+           << IsArgument << Ty << Feature;
+
+  // Else, both caller and callee have the required feature, so there is no need
+  // to diagnose.
+  return false;
+}
+
+static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,
+                          SourceLocation CallLoc,
+                          const llvm::StringMap<bool> &CallerMap,
+                          const llvm::StringMap<bool> &CalleeMap, QualType Ty,
+                          bool IsArgument) {
+  uint64_t Size = Ctx.getTypeSize(Ty);
+  if (Size > 256)
+    return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty,
+                                "avx512f", IsArgument);
+
+  if (Size > 128)
+    return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty, "avx",
+                                IsArgument);
+
+  return false;
+}
+
+void X86_64TargetCodeGenInfo::checkFunctionCallABI(
+    CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
+    const FunctionDecl *Callee, const CallArgList &Args) const {
+  llvm::StringMap<bool> CallerMap;
+  llvm::StringMap<bool> CalleeMap;
+  unsigned ArgIndex = 0;
+
+  // We need to loop through the actual call arguments rather than the the
+  // function's parameters, in case this variadic.
+  for (const CallArg &Arg : Args) {
+    // The "avx" feature changes how vectors >128 in size are passed. "avx512f"
+    // additionally changes how vectors >256 in size are passed. Like GCC, we
+    // warn when a function is called with an argument where this will change.
+    // Unlike GCC, we also error when it is an obvious ABI mismatch, that is,
+    // the caller and callee features are mismatched.
+    // Unfortunately, we cannot do this diagnostic in SEMA, since the callee can
+    // change its ABI with attribute-target after this call.
+    if (Arg.getType()->isVectorType() &&
+        CGM.getContext().getTypeSize(Arg.getType()) > 128) {
+      initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
+      QualType Ty = Arg.getType();
+      // The CallArg seems to have desugared the type already, so for clearer
+      // diagnostics, replace it with the type in the FunctionDecl if possible.
+      if (ArgIndex < Callee->getNumParams())
+        Ty = Callee->getParamDecl(ArgIndex)->getType();
+
+      if (checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
+                        CalleeMap, Ty, /*IsArgument*/ true))
+        return;
+    }
+    ++ArgIndex;
+  }
+
+  // Check return always, as we don't have a good way of knowing in codegen
+  // whether this value is used, tail-called, etc.
+  if (Callee->getReturnType()->isVectorType() &&
+      CGM.getContext().getTypeSize(Callee->getReturnType()) > 128) {
+    initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
+    checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
+                  CalleeMap, Callee->getReturnType(),
+                  /*IsArgument*/ false);
+  }
+}
+
 static std::string qualifyWindowsLibrary(llvm::StringRef Lib) {
   // If the argument does not end in .lib, automatically add the suffix.
   // If the argument contains a space, enclose it in quotes.
@@ -7105,7 +7219,9 @@ bool SystemZABIInfo::isFPArgumentType(QualType Ty) const {
 }
 
 QualType SystemZABIInfo::GetSingleElementType(QualType Ty) const {
-  if (const RecordType *RT = Ty->getAsStructureType()) {
+  const RecordType *RT = Ty->getAs<RecordType>();
+
+  if (RT && RT->isStructureOrClassType()) {
     const RecordDecl *RD = RT->getDecl();
     QualType Found;
 
@@ -7131,6 +7247,10 @@ QualType SystemZABIInfo::GetSingleElementType(QualType Ty) const {
       if (getContext().getLangOpts().CPlusPlus &&
           FD->isZeroLengthBitField(getContext()))
         continue;
+      // Like isSingleElementStruct(), ignore C++20 empty data members.
+      if (FD->hasAttr<NoUniqueAddressAttr>() &&
+          isEmptyRecord(getContext(), FD->getType(), true))
+        continue;
 
       // Unlike isSingleElementStruct(), arrays do not count.
       // Nested structures still do though.
@@ -7370,10 +7490,49 @@ ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const {
 
 namespace {
 
+class MSP430ABIInfo : public DefaultABIInfo {
+  static ABIArgInfo complexArgInfo() {
+    ABIArgInfo Info = ABIArgInfo::getDirect();
+    Info.setCanBeFlattened(false);
+    return Info;
+  }
+
+public:
+  MSP430ABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
+
+  ABIArgInfo classifyReturnType(QualType RetTy) const {
+    if (RetTy->isAnyComplexType())
+      return complexArgInfo();
+
+    return DefaultABIInfo::classifyReturnType(RetTy);
+  }
+
+  ABIArgInfo classifyArgumentType(QualType RetTy) const {
+    if (RetTy->isAnyComplexType())
+      return complexArgInfo();
+
+    return DefaultABIInfo::classifyArgumentType(RetTy);
+  }
+
+  // Just copy the original implementations because
+  // DefaultABIInfo::classify{Return,Argument}Type() are not virtual
+  void computeInfo(CGFunctionInfo &FI) const override {
+    if (!getCXXABI().classifyReturnType(FI))
+      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
+    for (auto &I : FI.arguments())
+      I.info = classifyArgumentType(I.type);
+  }
+
+  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+                    QualType Ty) const override {
+    return EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty));
+  }
+};
+
 class MSP430TargetCodeGenInfo : public TargetCodeGenInfo {
 public:
   MSP430TargetCodeGenInfo(CodeGenTypes &CGT)
-      : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {}
+      : TargetCodeGenInfo(std::make_unique<MSP430ABIInfo>(CGT)) {}
   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
                            CodeGen::CodeGenModule &M) const override;
 };
@@ -9535,11 +9694,15 @@ class XCoreABIInfo : public DefaultABIInfo {
 
 class XCoreTargetCodeGenInfo : public TargetCodeGenInfo {
   mutable TypeStringCache TSC;
+  void emitTargetMD(const Decl *D, llvm::GlobalValue *GV,
+                    const CodeGen::CodeGenModule &M) const;
+
 public:
   XCoreTargetCodeGenInfo(CodeGenTypes &CGT)
       : TargetCodeGenInfo(std::make_unique<XCoreABIInfo>(CGT)) {}
-  void emitTargetMD(const Decl *D, llvm::GlobalValue *GV,
-                    CodeGen::CodeGenModule &M) const override;
+  void emitTargetMetadata(CodeGen::CodeGenModule &CGM,
+                          const llvm::MapVector<GlobalDecl, StringRef>
+                              &MangledDeclNames) const override;
 };
 
 } // End anonymous namespace.
@@ -9700,11 +9863,13 @@ StringRef TypeStringCache::lookupStr(const IdentifierInfo *ID) {
 /// The output is tested by test/CodeGen/xcore-stringtype.c.
 ///
 static bool getTypeString(SmallStringEnc &Enc, const Decl *D,
-                          CodeGen::CodeGenModule &CGM, TypeStringCache &TSC);
+                          const CodeGen::CodeGenModule &CGM,
+                          TypeStringCache &TSC);
 
 /// XCore uses emitTargetMD to emit TypeString metadata for global symbols.
-void XCoreTargetCodeGenInfo::emitTargetMD(const Decl *D, llvm::GlobalValue *GV,
-                                          CodeGen::CodeGenModule &CGM) const {
+void XCoreTargetCodeGenInfo::emitTargetMD(
+    const Decl *D, llvm::GlobalValue *GV,
+    const CodeGen::CodeGenModule &CGM) const {
   SmallStringEnc Enc;
   if (getTypeString(Enc, D, CGM, TSC)) {
     llvm::LLVMContext &Ctx = CGM.getModule().getContext();
@@ -9716,6 +9881,21 @@ void XCoreTargetCodeGenInfo::emitTargetMD(const Decl *D, llvm::GlobalValue *GV,
   }
 }
 
+void XCoreTargetCodeGenInfo::emitTargetMetadata(
+    CodeGen::CodeGenModule &CGM,
+    const llvm::MapVector<GlobalDecl, StringRef> &MangledDeclNames) const {
+  // Warning, new MangledDeclNames may be appended within this loop.
+  // We rely on MapVector insertions adding new elements to the end
+  // of the container.
+  for (unsigned I = 0; I != MangledDeclNames.size(); ++I) {
+    auto Val = *(MangledDeclNames.begin() + I);
+    llvm::GlobalValue *GV = CGM.GetGlobalValue(Val.second);
+    if (GV) {
+      const Decl *D = Val.first.getDecl()->getMostRecentDecl();
+      emitTargetMD(D, GV, CGM);
+    }
+  }
+}
 //===----------------------------------------------------------------------===//
 // SPIR ABI Implementation
 //===----------------------------------------------------------------------===//
@@ -10049,7 +10229,8 @@ static bool appendType(SmallStringEnc &Enc, QualType QType,
 }
 
 static bool getTypeString(SmallStringEnc &Enc, const Decl *D,
-                          CodeGen::CodeGenModule &CGM, TypeStringCache &TSC) {
+                          const CodeGen::CodeGenModule &CGM,
+                          TypeStringCache &TSC) {
   if (!D)
     return false;
 
@@ -10548,6 +10729,56 @@ class RISCVTargetCodeGenInfo : public TargetCodeGenInfo {
 };
 } // namespace
 
+//===----------------------------------------------------------------------===//
+// VE ABI Implementation.
+//
+namespace {
+class VEABIInfo : public DefaultABIInfo {
+public:
+  VEABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
+
+private:
+  ABIArgInfo classifyReturnType(QualType RetTy) const;
+  ABIArgInfo classifyArgumentType(QualType RetTy) const;
+  void computeInfo(CGFunctionInfo &FI) const override;
+};
+} // end anonymous namespace
+
+ABIArgInfo VEABIInfo::classifyReturnType(QualType Ty) const {
+  if (Ty->isAnyComplexType()) {
+    return ABIArgInfo::getDirect();
+  }
+  return DefaultABIInfo::classifyReturnType(Ty);
+}
+
+ABIArgInfo VEABIInfo::classifyArgumentType(QualType Ty) const {
+  if (Ty->isAnyComplexType()) {
+    return ABIArgInfo::getDirect();
+  }
+  return DefaultABIInfo::classifyArgumentType(Ty);
+}
+
+void VEABIInfo::computeInfo(CGFunctionInfo &FI) const {
+
+  FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
+  for (auto &Arg : FI.arguments())
+    Arg.info = classifyArgumentType(Arg.type);
+}
+
+namespace {
+class VETargetCodeGenInfo : public TargetCodeGenInfo {
+public:
+  VETargetCodeGenInfo(CodeGenTypes &CGT)
+      : TargetCodeGenInfo(std::make_unique<VEABIInfo>(CGT)) {}
+  // VE ABI requires the arguments of variadic and prototype-less functions
+  // are passed in both registers and memory.
+  bool isNoProtoCallVariadic(const CallArgList &args,
+                             const FunctionNoProtoType *fnType) const override {
+    return true;
+  }
+};
+} // end anonymous namespace
+
 //===----------------------------------------------------------------------===//
 // Driver code
 //===----------------------------------------------------------------------===//
@@ -10750,6 +10981,8 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
   case llvm::Triple::spir:
   case llvm::Triple::spir64:
     return SetCGInfo(new SPIRTargetCodeGenInfo(Types));
+  case llvm::Triple::ve:
+    return SetCGInfo(new VETargetCodeGenInfo(Types));
   }
 }
 
diff --git a/clang/lib/CodeGen/TargetInfo.h b/clang/lib/CodeGen/TargetInfo.h
index 85565475eee79..1152cabce4a0d 100644
--- a/clang/lib/CodeGen/TargetInfo.h
+++ b/clang/lib/CodeGen/TargetInfo.h
@@ -57,10 +57,18 @@ class TargetCodeGenInfo {
   virtual void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
                                    CodeGen::CodeGenModule &M) const {}
 
-  /// emitTargetMD - Provides a convenient hook to handle extra
-  /// target-specific metadata for the given global.
-  virtual void emitTargetMD(const Decl *D, llvm::GlobalValue *GV,
-                            CodeGen::CodeGenModule &M) const {}
+  /// emitTargetMetadata - Provides a convenient hook to handle extra
+  /// target-specific metadata for the given globals.
+  virtual void emitTargetMetadata(
+      CodeGen::CodeGenModule &CGM,
+      const llvm::MapVector<GlobalDecl, StringRef> &MangledDeclNames) const {}
+
+  /// Any further codegen related checks that need to be done on a function call
+  /// in a target specific manner.
+  virtual void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
+                                    const FunctionDecl *Caller,
+                                    const FunctionDecl *Callee,
+                                    const CallArgList &Args) const {}
 
   /// Determines the size of struct _Unwind_Exception on this platform,
   /// in 8-bit units.  The Itanium ABI defines this as:
diff --git a/clang/lib/CrossTU/CrossTranslationUnit.cpp b/clang/lib/CrossTU/CrossTranslationUnit.cpp
index 24c3143db4107..80465c41d1515 100644
--- a/clang/lib/CrossTU/CrossTranslationUnit.cpp
+++ b/clang/lib/CrossTU/CrossTranslationUnit.cpp
@@ -12,6 +12,7 @@
 #include "clang/CrossTU/CrossTranslationUnit.h"
 #include "clang/AST/ASTImporter.h"
 #include "clang/AST/Decl.h"
+#include "clang/AST/ParentMapContext.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/CrossTU/CrossTUDiagnostic.h"
 #include "clang/Frontend/ASTUnit.h"
@@ -366,7 +367,9 @@ CrossTranslationUnitContext::ASTUnitStorage::ASTUnitStorage(
     CompilerInstance &CI)
     : Loader(CI, CI.getAnalyzerOpts()->CTUDir,
              CI.getAnalyzerOpts()->CTUInvocationList),
-      LoadGuard(CI.getAnalyzerOpts()->CTUImportThreshold) {}
+      LoadGuard(CI.getASTContext().getLangOpts().CPlusPlus
+                    ? CI.getAnalyzerOpts()->CTUImportCppThreshold
+                    : CI.getAnalyzerOpts()->CTUImportThreshold) {}
 
 llvm::Expected<ASTUnit *>
 CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFile(
@@ -718,6 +721,9 @@ CrossTranslationUnitContext::importDefinitionImpl(const T *D, ASTUnit *Unit) {
   assert(hasBodyOrInit(ToDecl) && "Imported Decl should have body or init.");
   ++NumGetCTUSuccess;
 
+  // Parent map is invalidated after changing the AST.
+  ToDecl->getASTContext().getParentMapContext().clear();
+
   return ToDecl;
 }
 
diff --git a/clang/lib/Driver/Action.cpp b/clang/lib/Driver/Action.cpp
index 0eb4c7257e7a8..2ec063d873be7 100644
--- a/clang/lib/Driver/Action.cpp
+++ b/clang/lib/Driver/Action.cpp
@@ -43,6 +43,8 @@ const char *Action::getClassName(ActionClass AC) {
     return "clang-offload-unbundler";
   case OffloadWrapperJobClass:
     return "clang-offload-wrapper";
+  case StaticLibJobClass:
+    return "static-lib-linker";
   }
 
   llvm_unreachable("invalid class");
@@ -415,3 +417,8 @@ void OffloadWrapperJobAction::anchor() {}
 OffloadWrapperJobAction::OffloadWrapperJobAction(ActionList &Inputs,
                                                  types::ID Type)
   : JobAction(OffloadWrapperJobClass, Inputs, Type) {}
+
+void StaticLibJobAction::anchor() {}
+
+StaticLibJobAction::StaticLibJobAction(ActionList &Inputs, types::ID Type)
+    : JobAction(StaticLibJobClass, Inputs, Type) {}
diff --git a/clang/lib/Driver/CMakeLists.txt b/clang/lib/Driver/CMakeLists.txt
index 6f25d3588ebba..9463ca5c109d6 100644
--- a/clang/lib/Driver/CMakeLists.txt
+++ b/clang/lib/Driver/CMakeLists.txt
@@ -31,6 +31,7 @@ add_clang_library(clangDriver
   ToolChains/Arch/RISCV.cpp
   ToolChains/Arch/Sparc.cpp
   ToolChains/Arch/SystemZ.cpp
+  ToolChains/Arch/VE.cpp
   ToolChains/Arch/X86.cpp
   ToolChains/AIX.cpp
   ToolChains/Ananas.cpp
@@ -67,6 +68,7 @@ add_clang_library(clangDriver
   ToolChains/RISCVToolchain.cpp
   ToolChains/Solaris.cpp
   ToolChains/TCE.cpp
+  ToolChains/VEToolchain.cpp
   ToolChains/WebAssembly.cpp
   ToolChains/XCore.cpp
   ToolChains/PPCLinux.cpp
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index a48761af400fd..ece8222dcf24f 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -43,6 +43,7 @@
 #include "ToolChains/RISCVToolchain.h"
 #include "ToolChains/Solaris.h"
 #include "ToolChains/TCE.h"
+#include "ToolChains/VEToolchain.h"
 #include "ToolChains/WebAssembly.h"
 #include "ToolChains/XCore.h"
 #include "clang/Basic/Version.h"
@@ -474,6 +475,26 @@ static llvm::Triple computeTargetTriple(const Driver &D,
       Target.getOS() == llvm::Triple::Minix)
     return Target;
 
+  // On AIX, the env OBJECT_MODE may affect the resulting arch variant.
+  if (Target.isOSAIX()) {
+    if (Optional<std::string> ObjectModeValue =
+            llvm::sys::Process::GetEnv("OBJECT_MODE")) {
+      StringRef ObjectMode = *ObjectModeValue;
+      llvm::Triple::ArchType AT = llvm::Triple::UnknownArch;
+
+      if (ObjectMode.equals("64")) {
+        AT = Target.get64BitArchVariant().getArch();
+      } else if (ObjectMode.equals("32")) {
+        AT = Target.get32BitArchVariant().getArch();
+      } else {
+        D.Diag(diag::err_drv_invalid_object_mode) << ObjectMode;
+      }
+
+      if (AT != llvm::Triple::UnknownArch && AT != Target.getArch())
+        Target.setArch(AT);
+    }
+  }
+
   // Handle pseudo-target flags '-m64', '-mx32', '-m32' and '-m16'.
   Arg *A = Args.getLastArg(options::OPT_m64, options::OPT_mx32,
                            options::OPT_m32, options::OPT_m16);
@@ -1455,7 +1476,8 @@ void Driver::setUpResponseFiles(Compilation &C, Command &Cmd) {
   // capacity if the tool does not support response files, there is a chance/
   // that things will just work without a response file, so we silently just
   // skip it.
-  if (Cmd.getCreator().getResponseFilesSupport() == Tool::RF_None ||
+  if (Cmd.getResponseFileSupport().ResponseKind ==
+          ResponseFileSupport::RF_None ||
       llvm::sys::commandLineFitsWithinSystemLimits(Cmd.getExecutable(),
                                                    Cmd.getArguments()))
     return;
@@ -2325,8 +2347,11 @@ class OffloadingActionBuilder final {
     /// Append top level actions generated by the builder.
     virtual void appendTopLevelActions(ActionList &AL) {}
 
-    /// Append linker actions generated by the builder.
-    virtual void appendLinkActions(ActionList &AL) {}
+    /// Append linker device actions generated by the builder.
+    virtual void appendLinkDeviceActions(ActionList &AL) {}
+
+    /// Append linker host action generated by the builder.
+    virtual Action* appendLinkHostActions(ActionList &AL) { return nullptr; }
 
     /// Append linker actions generated by the builder.
     virtual void appendLinkDependences(OffloadAction::DeviceDependences &DA) {}
@@ -2796,17 +2821,45 @@ class OffloadingActionBuilder final {
                                                            : ABRT_Success;
     }
 
-    void appendLinkDependences(OffloadAction::DeviceDependences &DA) override {
+    void appendLinkDeviceActions(ActionList &AL) override {
+      if (DeviceLinkerInputs.size() == 0)
+        return;
+
+      assert(DeviceLinkerInputs.size() == GpuArchList.size() &&
+             "Linker inputs and GPU arch list sizes do not match.");
+
       // Append a new link action for each device.
       unsigned I = 0;
       for (auto &LI : DeviceLinkerInputs) {
+        // Each entry in DeviceLinkerInputs corresponds to a GPU arch.
         auto *DeviceLinkAction =
             C.MakeAction<LinkJobAction>(LI, types::TY_Image);
-        DA.add(*DeviceLinkAction, *ToolChains[0],
-               CudaArchToString(GpuArchList[I]), AssociatedOffloadKind);
+        // Linking all inputs for the current GPU arch.
+        // LI contains all the inputs for the linker.
+        OffloadAction::DeviceDependences DeviceLinkDeps;
+        DeviceLinkDeps.add(*DeviceLinkAction, *ToolChains[0],
+            CudaArchToString(GpuArchList[I]), AssociatedOffloadKind);
+        AL.push_back(C.MakeAction<OffloadAction>(DeviceLinkDeps,
+            DeviceLinkAction->getType()));
         ++I;
       }
+      DeviceLinkerInputs.clear();
+
+      // Create a host object from all the device images by embedding them
+      // in a fat binary.
+      OffloadAction::DeviceDependences DDeps;
+      auto *TopDeviceLinkAction =
+          C.MakeAction<LinkJobAction>(AL, types::TY_Object);
+      DDeps.add(*TopDeviceLinkAction, *ToolChains[0],
+          nullptr, AssociatedOffloadKind);
+
+      // Offload the host object to the host linker.
+      AL.push_back(C.MakeAction<OffloadAction>(DDeps, TopDeviceLinkAction->getType()));
     }
+
+    Action* appendLinkHostActions(ActionList &AL) override { return AL.back(); }
+
+    void appendLinkDependences(OffloadAction::DeviceDependences &DA) override {}
   };
 
   /// OpenMP action builder. The host bitcode is passed to the device frontend
@@ -2934,7 +2987,7 @@ class OffloadingActionBuilder final {
       OpenMPDeviceActions.clear();
     }
 
-    void appendLinkActions(ActionList &AL) override {
+    void appendLinkDeviceActions(ActionList &AL) override {
       assert(ToolChains.size() == DeviceLinkerInputs.size() &&
              "Toolchains and linker inputs sizes do not match.");
 
@@ -2953,6 +3006,14 @@ class OffloadingActionBuilder final {
       DeviceLinkerInputs.clear();
     }
 
+    Action* appendLinkHostActions(ActionList &AL) override {
+      // Create wrapper bitcode from the result of device link actions and compile
+      // it to an object which will be added to the host link command.
+      auto *BC = C.MakeAction<OffloadWrapperJobAction>(AL, types::TY_LLVM_BC);
+      auto *ASM = C.MakeAction<BackendJobAction>(BC, types::TY_PP_Asm);
+      return C.MakeAction<AssembleJobAction>(ASM, types::TY_Object);
+    }
+
     void appendLinkDependences(OffloadAction::DeviceDependences &DA) override {}
 
     bool initialize() override {
@@ -3185,17 +3246,20 @@ class OffloadingActionBuilder final {
     for (DeviceActionBuilder *SB : SpecializedBuilders) {
       if (!SB->isValid())
         continue;
-      SB->appendLinkActions(DeviceAL);
+      SB->appendLinkDeviceActions(DeviceAL);
     }
 
     if (DeviceAL.empty())
       return nullptr;
 
-    // Create wrapper bitcode from the result of device link actions and compile
-    // it to an object which will be added to the host link command.
-    auto *BC = C.MakeAction<OffloadWrapperJobAction>(DeviceAL, types::TY_LLVM_BC);
-    auto *ASM = C.MakeAction<BackendJobAction>(BC, types::TY_PP_Asm);
-    return C.MakeAction<AssembleJobAction>(ASM, types::TY_Object);
+    // Let builders add host linking actions.
+    Action* HA;
+    for (DeviceActionBuilder *SB : SpecializedBuilders) {
+      if (!SB->isValid())
+        continue;
+      HA = SB->appendLinkHostActions(DeviceAL);
+    }
+    return HA;
   }
 
   /// Processes the host linker action. This currently consists of replacing it
@@ -3500,7 +3564,13 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
   if (!LinkerInputs.empty()) {
     if (Action *Wrapper = OffloadBuilder.makeHostLinkAction())
       LinkerInputs.push_back(Wrapper);
-    Action *LA = C.MakeAction<LinkJobAction>(LinkerInputs, types::TY_Image);
+    Action *LA;
+    // Check if this Linker Job should emit a static library.
+    if (ShouldEmitStaticLibrary(Args)) {
+      LA = C.MakeAction<StaticLibJobAction>(LinkerInputs, types::TY_Image);
+    } else {
+      LA = C.MakeAction<LinkJobAction>(LinkerInputs, types::TY_Image);
+    }
     LA = OffloadBuilder.processHostLinkAction(LA);
     Actions.push_back(LA);
   }
@@ -4717,13 +4787,11 @@ void Driver::generatePrefixedToolNames(
 }
 
 static bool ScanDirForExecutable(SmallString<128> &Dir,
-                                 ArrayRef<std::string> Names) {
-  for (const auto &Name : Names) {
-    llvm::sys::path::append(Dir, Name);
-    if (llvm::sys::fs::can_execute(Twine(Dir)))
-      return true;
-    llvm::sys::path::remove_filename(Dir);
-  }
+                                 const std::string &Name) {
+  llvm::sys::path::append(Dir, Name);
+  if (llvm::sys::fs::can_execute(Twine(Dir)))
+    return true;
+  llvm::sys::path::remove_filename(Dir);
   return false;
 }
 
@@ -4736,8 +4804,9 @@ std::string Driver::GetProgramPath(StringRef Name, const ToolChain &TC) const {
   for (const auto &PrefixDir : PrefixDirs) {
     if (llvm::sys::fs::is_directory(PrefixDir)) {
       SmallString<128> P(PrefixDir);
-      if (ScanDirForExecutable(P, TargetSpecificExecutables))
-        return std::string(P.str());
+      for (const auto &TargetSpecificExecutable : TargetSpecificExecutables)
+        if (ScanDirForExecutable(P, TargetSpecificExecutable))
+          return std::string(P.str());
     } else {
       SmallString<128> P((PrefixDir + Name).str());
       if (llvm::sys::fs::can_execute(Twine(P)))
@@ -4746,17 +4815,25 @@ std::string Driver::GetProgramPath(StringRef Name, const ToolChain &TC) const {
   }
 
   const ToolChain::path_list &List = TC.getProgramPaths();
-  for (const auto &Path : List) {
-    SmallString<128> P(Path);
-    if (ScanDirForExecutable(P, TargetSpecificExecutables))
-      return std::string(P.str());
-  }
+  for (const auto &TargetSpecificExecutable : TargetSpecificExecutables) {
+    // For each possible name of the tool look for it in
+    // program paths first, then the path.
+    // Higher priority names will be first, meaning that
+    // a higher priority name in the path will be found
+    // instead of a lower priority name in the program path.
+    // E.g. <triple>-gcc on the path will be found instead
+    // of gcc in the program path
+    for (const auto &Path : List) {
+      SmallString<128> P(Path);
+      if (ScanDirForExecutable(P, TargetSpecificExecutable))
+        return std::string(P.str());
+    }
 
-  // If all else failed, search the path.
-  for (const auto &TargetSpecificExecutable : TargetSpecificExecutables)
+    // Fall back to the path
     if (llvm::ErrorOr<std::string> P =
             llvm::sys::findProgramByName(TargetSpecificExecutable))
       return *P;
+  }
 
   return std::string(Name);
 }
@@ -4859,6 +4936,9 @@ const ToolChain &Driver::getToolChain(const ArgList &Args,
                Target.getArch() == llvm::Triple::ppc64le)
         TC = std::make_unique<toolchains::PPCLinuxToolChain>(*this, Target,
                                                               Args);
+      else if (Target.getArch() == llvm::Triple::ve)
+        TC = std::make_unique<toolchains::VEToolChain>(*this, Target, Args);
+
       else
         TC = std::make_unique<toolchains::Linux>(*this, Target, Args);
       break;
@@ -4951,6 +5031,9 @@ const ToolChain &Driver::getToolChain(const ArgList &Args,
       case llvm::Triple::riscv64:
         TC = std::make_unique<toolchains::RISCVToolChain>(*this, Target, Args);
         break;
+      case llvm::Triple::ve:
+        TC = std::make_unique<toolchains::VEToolChain>(*this, Target, Args);
+        break;
       default:
         if (Target.getVendor() == llvm::Triple::Myriad)
           TC = std::make_unique<toolchains::MyriadToolChain>(*this, Target,
@@ -5002,6 +5085,13 @@ bool Driver::ShouldUseFlangCompiler(const JobAction &JA) const {
   return true;
 }
 
+bool Driver::ShouldEmitStaticLibrary(const ArgList &Args) const {
+  // Only emit static library if the flag is set explicitly.
+  if (Args.hasArg(options::OPT_emit_static_lib))
+    return true;
+  return false;
+}
+
 /// GetReleaseVersion - Parse (([0-9]+)(.([0-9]+)(.([0-9]+)?))?)? and return the
 /// grouped values as integers. Numbers which are not provided are set to 0.
 ///
diff --git a/clang/lib/Driver/Job.cpp b/clang/lib/Driver/Job.cpp
index 274bda9dc9f16..4808a9f4628d5 100644
--- a/clang/lib/Driver/Job.cpp
+++ b/clang/lib/Driver/Job.cpp
@@ -36,11 +36,11 @@ using namespace clang;
 using namespace driver;
 
 Command::Command(const Action &Source, const Tool &Creator,
-                 const char *Executable,
+                 ResponseFileSupport ResponseSupport, const char *Executable,
                  const llvm::opt::ArgStringList &Arguments,
                  ArrayRef<InputInfo> Inputs)
-    : Source(Source), Creator(Creator), Executable(Executable),
-      Arguments(Arguments) {
+    : Source(Source), Creator(Creator), ResponseSupport(ResponseSupport),
+      Executable(Executable), Arguments(Arguments) {
   for (const auto &II : Inputs)
     if (II.isFilename())
       InputFilenames.push_back(II.getFilename());
@@ -102,7 +102,7 @@ static bool skipArgs(const char *Flag, bool HaveCrashVFS, int &SkipNum,
 
 void Command::writeResponseFile(raw_ostream &OS) const {
   // In a file list, we only write the set of inputs to the response file
-  if (Creator.getResponseFilesSupport() == Tool::RF_FileList) {
+  if (ResponseSupport.ResponseKind == ResponseFileSupport::RF_FileList) {
     for (const auto *Arg : InputFileList) {
       OS << Arg << '\n';
     }
@@ -131,7 +131,7 @@ void Command::buildArgvForResponseFile(
   // When not a file list, all arguments are sent to the response file.
   // This leaves us to set the argv to a single parameter, requesting the tool
   // to read the response file.
-  if (Creator.getResponseFilesSupport() != Tool::RF_FileList) {
+  if (ResponseSupport.ResponseKind != ResponseFileSupport::RF_FileList) {
     Out.push_back(Executable);
     Out.push_back(ResponseFileFlag.c_str());
     return;
@@ -149,7 +149,7 @@ void Command::buildArgvForResponseFile(
       Out.push_back(Arg);
     } else if (FirstInput) {
       FirstInput = false;
-      Out.push_back(Creator.getResponseFileFlag());
+      Out.push_back(ResponseSupport.ResponseFlag);
       Out.push_back(ResponseFile);
     }
   }
@@ -277,7 +277,7 @@ void Command::Print(raw_ostream &OS, const char *Terminator, bool Quote,
     writeResponseFile(OS);
     // Avoiding duplicated newline terminator, since FileLists are
     // newline-separated.
-    if (Creator.getResponseFilesSupport() != Tool::RF_FileList)
+    if (ResponseSupport.ResponseKind != ResponseFileSupport::RF_FileList)
       OS << "\n";
     OS << " (end of response file)";
   }
@@ -287,7 +287,7 @@ void Command::Print(raw_ostream &OS, const char *Terminator, bool Quote,
 
 void Command::setResponseFile(const char *FileName) {
   ResponseFile = FileName;
-  ResponseFileFlag = Creator.getResponseFileFlag();
+  ResponseFileFlag = ResponseSupport.ResponseFlag;
   ResponseFileFlag += FileName;
 }
 
@@ -327,7 +327,7 @@ int Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects,
 
     // Save the response file in the appropriate encoding
     if (std::error_code EC = writeFileWithEncoding(
-            ResponseFile, RespContents, Creator.getResponseFileEncoding())) {
+            ResponseFile, RespContents, ResponseSupport.ResponseEncoding)) {
       if (ErrMsg)
         *ErrMsg = EC.message();
       if (ExecutionFailed)
@@ -354,10 +354,11 @@ int Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects,
 }
 
 CC1Command::CC1Command(const Action &Source, const Tool &Creator,
+                       ResponseFileSupport ResponseSupport,
                        const char *Executable,
                        const llvm::opt::ArgStringList &Arguments,
                        ArrayRef<InputInfo> Inputs)
-    : Command(Source, Creator, Executable, Arguments, Inputs) {
+    : Command(Source, Creator, ResponseSupport, Executable, Arguments, Inputs) {
   InProcess = true;
 }
 
@@ -410,11 +411,13 @@ void CC1Command::setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) {
 }
 
 FallbackCommand::FallbackCommand(const Action &Source_, const Tool &Creator_,
+                                 ResponseFileSupport ResponseSupport,
                                  const char *Executable_,
                                  const llvm::opt::ArgStringList &Arguments_,
                                  ArrayRef<InputInfo> Inputs,
                                  std::unique_ptr<Command> Fallback_)
-    : Command(Source_, Creator_, Executable_, Arguments_, Inputs),
+    : Command(Source_, Creator_, ResponseSupport, Executable_, Arguments_,
+              Inputs),
       Fallback(std::move(Fallback_)) {}
 
 void FallbackCommand::Print(raw_ostream &OS, const char *Terminator,
@@ -451,9 +454,11 @@ int FallbackCommand::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects,
 }
 
 ForceSuccessCommand::ForceSuccessCommand(
-    const Action &Source_, const Tool &Creator_, const char *Executable_,
+    const Action &Source_, const Tool &Creator_,
+    ResponseFileSupport ResponseSupport, const char *Executable_,
     const llvm::opt::ArgStringList &Arguments_, ArrayRef<InputInfo> Inputs)
-    : Command(Source_, Creator_, Executable_, Arguments_, Inputs) {}
+    : Command(Source_, Creator_, ResponseSupport, Executable_, Arguments_,
+              Inputs) {}
 
 void ForceSuccessCommand::Print(raw_ostream &OS, const char *Terminator,
                             bool Quote, CrashReportInfo *CrashInfo) const {
diff --git a/clang/lib/Driver/SanitizerArgs.cpp b/clang/lib/Driver/SanitizerArgs.cpp
index 9d307e8097cbc..0b81152d57f6b 100644
--- a/clang/lib/Driver/SanitizerArgs.cpp
+++ b/clang/lib/Driver/SanitizerArgs.cpp
@@ -1068,7 +1068,7 @@ void SanitizerArgs::addArgs(const ToolChain &TC, const llvm::opt::ArgList &Args,
     CmdArgs.push_back(Args.MakeArgString("hwasan-abi=" + HwasanAbi));
   }
 
-  if (Sanitizers.has(SanitizerKind::HWAddress)) {
+  if (Sanitizers.has(SanitizerKind::HWAddress) && TC.getTriple().isAArch64()) {
     CmdArgs.push_back("-target-feature");
     CmdArgs.push_back("+tagged-globals");
   }
diff --git a/clang/lib/Driver/Tool.cpp b/clang/lib/Driver/Tool.cpp
index 9ff6e863a1242..449f69cfcb357 100644
--- a/clang/lib/Driver/Tool.cpp
+++ b/clang/lib/Driver/Tool.cpp
@@ -11,13 +11,8 @@
 
 using namespace clang::driver;
 
-Tool::Tool(const char *_Name, const char *_ShortName, const ToolChain &TC,
-           ResponseFileSupport _ResponseSupport,
-           llvm::sys::WindowsEncodingMethod _ResponseEncoding,
-           const char *_ResponseFlag)
-    : Name(_Name), ShortName(_ShortName), TheToolChain(TC),
-      ResponseSupport(_ResponseSupport), ResponseEncoding(_ResponseEncoding),
-      ResponseFlag(_ResponseFlag) {}
+Tool::Tool(const char *_Name, const char *_ShortName, const ToolChain &TC)
+    : Name(_Name), ShortName(_ShortName), TheToolChain(TC) {}
 
 Tool::~Tool() {
 }
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index c21bc0dc2a799..b8c12fc9241a6 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -275,6 +275,10 @@ Tool *ToolChain::buildLinker() const {
   llvm_unreachable("Linking is not supported by this toolchain");
 }
 
+Tool *ToolChain::buildStaticLibTool() const {
+  llvm_unreachable("Creating static lib is not supported by this toolchain");
+}
+
 Tool *ToolChain::getAssemble() const {
   if (!Assemble)
     Assemble.reset(buildAssembler());
@@ -293,6 +297,12 @@ Tool *ToolChain::getLink() const {
   return Link.get();
 }
 
+Tool *ToolChain::getStaticLibTool() const {
+  if (!StaticLibTool)
+    StaticLibTool.reset(buildStaticLibTool());
+  return StaticLibTool.get();
+}
+
 Tool *ToolChain::getIfsMerge() const {
   if (!IfsMerge)
     IfsMerge.reset(new tools::ifstool::Merger(*this));
@@ -322,6 +332,9 @@ Tool *ToolChain::getTool(Action::ActionClass AC) const {
   case Action::LinkJobClass:
     return getLink();
 
+  case Action::StaticLibJobClass:
+    return getStaticLibTool();
+
   case Action::InputClass:
   case Action::BindArchClass:
   case Action::OffloadClass:
@@ -565,6 +578,11 @@ std::string ToolChain::GetLinkerPath() const {
   return GetProgramPath(getDefaultLinker());
 }
 
+std::string ToolChain::GetStaticLibToolPath() const {
+  // TODO: Add support for static lib archiving on Windows
+  return GetProgramPath("llvm-ar");
+}
+
 types::ID ToolChain::LookupTypeForExtension(StringRef Ext) const {
   types::ID id = types::lookupTypeForExtension(Ext);
 
diff --git a/clang/lib/Driver/ToolChains/AIX.cpp b/clang/lib/Driver/ToolChains/AIX.cpp
index df2e30da32a8c..ac5544eedb00b 100644
--- a/clang/lib/Driver/ToolChains/AIX.cpp
+++ b/clang/lib/Driver/ToolChains/AIX.cpp
@@ -13,12 +13,15 @@
 #include "clang/Driver/Options.h"
 #include "clang/Driver/SanitizerArgs.h"
 #include "llvm/Option/ArgList.h"
+#include "llvm/Support/Path.h"
 
 using AIX = clang::driver::toolchains::AIX;
 using namespace clang::driver;
 using namespace clang::driver::tools;
+using namespace clang::driver::toolchains;
 
 using namespace llvm::opt;
+using namespace llvm::sys;
 
 void aix::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
                                   const InputInfo &Output,
@@ -73,7 +76,8 @@ void aix::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(II.getFilename());
 
   const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("as"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Exec, CmdArgs, Inputs));
 }
 
 void aix::Linker::ConstructJob(Compilation &C, const JobAction &JA,
@@ -152,7 +156,8 @@ void aix::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   const char *Exec = Args.MakeArgString(ToolChain.GetLinkerPath());
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Exec, CmdArgs, Inputs));
 }
 
 /// AIX - AIX tool chain which can call as(1) and ld(1) directly.
@@ -161,6 +166,43 @@ AIX::AIX(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
   getFilePaths().push_back(getDriver().SysRoot + "/usr/lib");
 }
 
+// Returns the effective header sysroot path to use.
+// This comes from either -isysroot or --sysroot.
+llvm::StringRef
+AIX::GetHeaderSysroot(const llvm::opt::ArgList &DriverArgs) const {
+  if (DriverArgs.hasArg(options::OPT_isysroot))
+    return DriverArgs.getLastArgValue(options::OPT_isysroot);
+  if (!getDriver().SysRoot.empty())
+    return getDriver().SysRoot;
+  return "/";
+}
+
+void AIX::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
+                                    ArgStringList &CC1Args) const {
+  // Return if -nostdinc is specified as a driver option.
+  if (DriverArgs.hasArg(options::OPT_nostdinc))
+    return;
+
+  llvm::StringRef Sysroot = GetHeaderSysroot(DriverArgs);
+  const Driver &D = getDriver();
+
+  // Add the Clang builtin headers (<resource>/include).
+  if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) {
+    SmallString<128> P(D.ResourceDir);
+    path::append(P, "/include");
+    addSystemInclude(DriverArgs, CC1Args, P.str());
+  }
+
+  // Return if -nostdlibinc is specified as a driver option.
+  if (DriverArgs.hasArg(options::OPT_nostdlibinc))
+    return;
+
+  // Add <sysroot>/usr/include.
+  SmallString<128> UP(Sysroot);
+  path::append(UP, "/usr/include");
+  addSystemInclude(DriverArgs, CC1Args, UP.str());
+}
+
 auto AIX::buildAssembler() const -> Tool * { return new aix::Assembler(*this); }
 
 auto AIX::buildLinker() const -> Tool * { return new aix::Linker(*this); }
diff --git a/clang/lib/Driver/ToolChains/AIX.h b/clang/lib/Driver/ToolChains/AIX.h
index 69b948bc0ea82..942bb3cceb8a7 100644
--- a/clang/lib/Driver/ToolChains/AIX.h
+++ b/clang/lib/Driver/ToolChains/AIX.h
@@ -63,9 +63,16 @@ class LLVM_LIBRARY_VISIBILITY AIX : public ToolChain {
   bool isPIEDefault() const override { return false; }
   bool isPICDefaultForced() const override { return true; }
 
+  void
+  AddClangSystemIncludeArgs(const llvm::opt::ArgList &DriverArgs,
+                            llvm::opt::ArgStringList &CC1Args) const override;
+
 protected:
   Tool *buildAssembler() const override;
   Tool *buildLinker() const override;
+
+private:
+  llvm::StringRef GetHeaderSysroot(const llvm::opt::ArgList &DriverArgs) const;
 };
 
 } // end namespace toolchains
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp
index edc52b86e45a9..bc6d1fcd4a008 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -21,16 +21,14 @@ using namespace clang::driver::toolchains;
 using namespace clang;
 using namespace llvm::opt;
 
-void RocmInstallationDetector::scanLibDevicePath() {
-  assert(!LibDevicePath.empty());
+void RocmInstallationDetector::scanLibDevicePath(llvm::StringRef Path) {
+  assert(!Path.empty());
 
   const StringRef Suffix(".bc");
   const StringRef Suffix2(".amdgcn.bc");
 
   std::error_code EC;
-  for (llvm::vfs::directory_iterator
-           LI = D.getVFS().dir_begin(LibDevicePath, EC),
-           LE;
+  for (llvm::vfs::directory_iterator LI = D.getVFS().dir_begin(Path, EC), LE;
        !EC && LI != LE; LI = LI.increment(EC)) {
     StringRef FilePath = LI->path();
     StringRef FileName = llvm::sys::path::filename(FilePath);
@@ -89,60 +87,114 @@ void RocmInstallationDetector::scanLibDevicePath() {
   }
 }
 
-RocmInstallationDetector::RocmInstallationDetector(
-    const Driver &D, const llvm::Triple &HostTriple,
-    const llvm::opt::ArgList &Args)
-    : D(D) {
-  struct Candidate {
-    std::string Path;
-    bool StrictChecking;
-
-    Candidate(std::string Path, bool StrictChecking = false)
-        : Path(Path), StrictChecking(StrictChecking) {}
-  };
+void RocmInstallationDetector::ParseHIPVersionFile(llvm::StringRef V) {
+  SmallVector<StringRef, 4> VersionParts;
+  V.split(VersionParts, '\n');
+  unsigned Major;
+  unsigned Minor;
+  for (auto Part : VersionParts) {
+    auto Splits = Part.split('=');
+    if (Splits.first == "HIP_VERSION_MAJOR")
+      Splits.second.getAsInteger(0, Major);
+    else if (Splits.first == "HIP_VERSION_MINOR")
+      Splits.second.getAsInteger(0, Minor);
+    else if (Splits.first == "HIP_VERSION_PATCH")
+      VersionPatch = Splits.second.str();
+  }
+  VersionMajorMinor = llvm::VersionTuple(Major, Minor);
+  DetectedVersion =
+      (Twine(Major) + "." + Twine(Minor) + "." + VersionPatch).str();
+}
 
+// For candidate specified by --rocm-path we do not do strict check.
+SmallVector<RocmInstallationDetector::Candidate, 4>
+RocmInstallationDetector::getInstallationPathCandidates() {
   SmallVector<Candidate, 4> Candidates;
+  if (!RocmPathArg.empty()) {
+    Candidates.emplace_back(RocmPathArg.str());
+    return Candidates;
+  }
 
-  if (Args.hasArg(clang::driver::options::OPT_rocm_path_EQ)) {
-    Candidates.emplace_back(
-        Args.getLastArgValue(clang::driver::options::OPT_rocm_path_EQ).str());
-  } else {
-    // Try to find relative to the compiler binary.
-    const char *InstallDir = D.getInstalledDir();
+  // Try to find relative to the compiler binary.
+  const char *InstallDir = D.getInstalledDir();
 
-    // Check both a normal Unix prefix position of the clang binary, as well as
-    // the Windows-esque layout the ROCm packages use with the host architecture
-    // subdirectory of bin.
+  // Check both a normal Unix prefix position of the clang binary, as well as
+  // the Windows-esque layout the ROCm packages use with the host architecture
+  // subdirectory of bin.
 
-    // Strip off directory (usually bin)
-    StringRef ParentDir = llvm::sys::path::parent_path(InstallDir);
-    StringRef ParentName = llvm::sys::path::filename(ParentDir);
+  // Strip off directory (usually bin)
+  StringRef ParentDir = llvm::sys::path::parent_path(InstallDir);
+  StringRef ParentName = llvm::sys::path::filename(ParentDir);
 
-    // Some builds use bin/{host arch}, so go up again.
-    if (ParentName == "bin") {
-      ParentDir = llvm::sys::path::parent_path(ParentDir);
-      ParentName = llvm::sys::path::filename(ParentDir);
-    }
+  // Some builds use bin/{host arch}, so go up again.
+  if (ParentName == "bin") {
+    ParentDir = llvm::sys::path::parent_path(ParentDir);
+    ParentName = llvm::sys::path::filename(ParentDir);
+  }
 
-    if (ParentName == "llvm") {
-      // Some versions of the rocm llvm package install to /opt/rocm/llvm/bin
-      Candidates.emplace_back(llvm::sys::path::parent_path(ParentDir).str(),
-                              /*StrictChecking=*/true);
-    }
+  // Some versions of the rocm llvm package install to /opt/rocm/llvm/bin
+  if (ParentName == "llvm")
+    ParentDir = llvm::sys::path::parent_path(ParentDir);
+
+  Candidates.emplace_back(ParentDir.str(), /*StrictChecking=*/true);
+
+  // Device library may be installed in clang resource directory.
+  Candidates.emplace_back(D.ResourceDir, /*StrictChecking=*/true);
+
+  Candidates.emplace_back(D.SysRoot + "/opt/rocm", /*StrictChecking=*/true);
+  return Candidates;
+}
 
-    Candidates.emplace_back(D.SysRoot + "/opt/rocm");
+RocmInstallationDetector::RocmInstallationDetector(
+    const Driver &D, const llvm::Triple &HostTriple,
+    const llvm::opt::ArgList &Args, bool DetectHIPRuntime, bool DetectDeviceLib)
+    : D(D) {
+  RocmPathArg = Args.getLastArgValue(clang::driver::options::OPT_rocm_path_EQ);
+  RocmDeviceLibPathArg =
+      Args.getAllArgValues(clang::driver::options::OPT_rocm_device_lib_path_EQ);
+  if (auto *A = Args.getLastArg(clang::driver::options::OPT_hip_version_EQ)) {
+    HIPVersionArg = A->getValue();
+    unsigned Major = 0;
+    unsigned Minor = 0;
+    SmallVector<StringRef, 3> Parts;
+    HIPVersionArg.split(Parts, '.');
+    if (Parts.size())
+      Parts[0].getAsInteger(0, Major);
+    if (Parts.size() > 1)
+      Parts[1].getAsInteger(0, Minor);
+    if (Parts.size() > 2)
+      VersionPatch = Parts[2].str();
+    if (VersionPatch.empty())
+      VersionPatch = "0";
+    if (Major == 0 || Minor == 0)
+      D.Diag(diag::err_drv_invalid_value)
+          << A->getAsString(Args) << HIPVersionArg;
+
+    VersionMajorMinor = llvm::VersionTuple(Major, Minor);
+    DetectedVersion =
+        (Twine(Major) + "." + Twine(Minor) + "." + VersionPatch).str();
+  } else {
+    VersionPatch = DefaultVersionPatch;
+    VersionMajorMinor =
+        llvm::VersionTuple(DefaultVersionMajor, DefaultVersionMinor);
+    DetectedVersion = (Twine(DefaultVersionMajor) + "." +
+                       Twine(DefaultVersionMinor) + "." + VersionPatch)
+                          .str();
   }
 
-  bool NoBuiltinLibs = Args.hasArg(options::OPT_nogpulib);
+  if (DetectHIPRuntime)
+    detectHIPRuntime();
+  if (DetectDeviceLib)
+    detectDeviceLibrary();
+}
 
+void RocmInstallationDetector::detectDeviceLibrary() {
   assert(LibDevicePath.empty());
 
-  if (Args.hasArg(clang::driver::options::OPT_hip_device_lib_path_EQ)) {
-    LibDevicePath
-      = Args.getLastArgValue(clang::driver::options::OPT_hip_device_lib_path_EQ);
-  } else if (const char *LibPathEnv = ::getenv("HIP_DEVICE_LIB_PATH")) {
+  if (!RocmDeviceLibPathArg.empty())
+    LibDevicePath = RocmDeviceLibPathArg[RocmDeviceLibPathArg.size() - 1];
+  else if (const char *LibPathEnv = ::getenv("HIP_DEVICE_LIB_PATH"))
     LibDevicePath = LibPathEnv;
-  }
 
   auto &FS = D.getVFS();
   if (!LibDevicePath.empty()) {
@@ -152,61 +204,109 @@ RocmInstallationDetector::RocmInstallationDetector(
     if (!FS.exists(LibDevicePath))
       return;
 
-    scanLibDevicePath();
-    IsValid = allGenericLibsValid() && !LibDeviceMap.empty();
+    scanLibDevicePath(LibDevicePath);
+    HasDeviceLibrary = allGenericLibsValid() && !LibDeviceMap.empty();
     return;
   }
 
+  // The install path situation in old versions of ROCm is a real mess, and
+  // use a different install layout. Multiple copies of the device libraries
+  // exist for each frontend project, and differ depending on which build
+  // system produced the packages. Standalone OpenCL builds also have a
+  // different directory structure from the ROCm OpenCL package.
+  auto Candidates = getInstallationPathCandidates();
+  for (const auto &Candidate : Candidates) {
+    auto CandidatePath = Candidate.Path;
+
+    // Check device library exists at the given path.
+    auto CheckDeviceLib = [&](StringRef Path) {
+      bool CheckLibDevice = (!NoBuiltinLibs || Candidate.StrictChecking);
+      if (CheckLibDevice && !FS.exists(Path))
+        return false;
+
+      scanLibDevicePath(Path);
+
+      if (!NoBuiltinLibs) {
+        // Check that the required non-target libraries are all available.
+        if (!allGenericLibsValid())
+          return false;
+
+        // Check that we have found at least one libdevice that we can link in
+        // if -nobuiltinlib hasn't been specified.
+        if (LibDeviceMap.empty())
+          return false;
+      }
+      return true;
+    };
+
+    // The possible structures are:
+    // - ${ROCM_ROOT}/amdgcn/bitcode/*
+    // - ${ROCM_ROOT}/lib/*
+    // - ${ROCM_ROOT}/lib/bitcode/*
+    // so try to detect these layouts.
+    static llvm::SmallVector<const char *, 2> SubDirsList[] = {
+        {"amdgcn", "bitcode"},
+        {"lib"},
+        {"lib", "bitcode"},
+    };
+
+    // Make a path by appending sub-directories to InstallPath.
+    auto MakePath = [&](const llvm::ArrayRef<const char *> &SubDirs) {
+      auto Path = CandidatePath;
+      for (auto SubDir : SubDirs)
+        llvm::sys::path::append(Path, SubDir);
+      return Path;
+    };
+
+    for (auto SubDirs : SubDirsList) {
+      LibDevicePath = MakePath(SubDirs);
+      HasDeviceLibrary = CheckDeviceLib(LibDevicePath);
+      if (HasDeviceLibrary)
+        return;
+    }
+  }
+}
+
+void RocmInstallationDetector::detectHIPRuntime() {
+  auto Candidates = getInstallationPathCandidates();
+  auto &FS = D.getVFS();
+
   for (const auto &Candidate : Candidates) {
     InstallPath = Candidate.Path;
     if (InstallPath.empty() || !FS.exists(InstallPath))
       continue;
 
-    // The install path situation in old versions of ROCm is a real mess, and
-    // use a different install layout. Multiple copies of the device libraries
-    // exist for each frontend project, and differ depending on which build
-    // system produced the packages. Standalone OpenCL builds also have a
-    // different directory structure from the ROCm OpenCL package.
-    //
-    // The desired structure is (${ROCM_ROOT} or
-    // ${OPENCL_ROOT})/amdgcn/bitcode/*, so try to detect this layout.
-
-    // BinPath = InstallPath + "/bin";
-    llvm::sys::path::append(IncludePath, InstallPath, "include");
-    llvm::sys::path::append(LibDevicePath, InstallPath, "amdgcn", "bitcode");
+    BinPath = InstallPath;
+    llvm::sys::path::append(BinPath, "bin");
+    IncludePath = InstallPath;
+    llvm::sys::path::append(IncludePath, "include");
+    LibPath = InstallPath;
+    llvm::sys::path::append(LibPath, "lib");
 
-    // We don't need the include path for OpenCL, since clang already ships with
-    // the default header.
-
-    bool CheckLibDevice = (!NoBuiltinLibs || Candidate.StrictChecking);
-    if (CheckLibDevice && !FS.exists(LibDevicePath))
+    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> VersionFile =
+        FS.getBufferForFile(BinPath + "/.hipVersion");
+    if (!VersionFile && Candidate.StrictChecking)
       continue;
 
-    scanLibDevicePath();
-
-    if (!NoBuiltinLibs) {
-      // Check that the required non-target libraries are all available.
-      if (!allGenericLibsValid())
-        continue;
-
-      // Check that we have found at least one libdevice that we can link in if
-      // -nobuiltinlib hasn't been specified.
-      if (LibDeviceMap.empty())
-        continue;
-    }
+    if (HIPVersionArg.empty() && VersionFile)
+      ParseHIPVersionFile((*VersionFile)->getBuffer());
 
-    IsValid = true;
-    break;
+    HasHIPRuntime = true;
+    return;
   }
+  HasHIPRuntime = false;
 }
 
 void RocmInstallationDetector::print(raw_ostream &OS) const {
-  if (isValid())
-    OS << "Found ROCm installation: " << InstallPath << '\n';
+  if (hasHIPRuntime())
+    OS << "Found HIP installation: " << InstallPath << ", version "
+       << DetectedVersion << '\n';
 }
 
 void RocmInstallationDetector::AddHIPIncludeArgs(const ArgList &DriverArgs,
                                                  ArgStringList &CC1Args) const {
+  bool UsesRuntimeWrapper = VersionMajorMinor > llvm::VersionTuple(3, 5);
+
   if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) {
     // HIP header includes standard library wrapper headers under clang
     // cuda_wrappers directory. Since these wrapper headers include_next
@@ -218,25 +318,28 @@ void RocmInstallationDetector::AddHIPIncludeArgs(const ArgList &DriverArgs,
     // Since standard C++ and other clang include paths are added in other
     // places after this function, here we only need to make sure wrapper
     // include path is added.
+    //
+    // ROCm 3.5 does not fully support the wrapper headers. Therefore it needs
+    // a workaround.
     SmallString<128> P(D.ResourceDir);
-    llvm::sys::path::append(P, "include");
-    llvm::sys::path::append(P, "cuda_wrappers");
+    if (UsesRuntimeWrapper)
+      llvm::sys::path::append(P, "include", "cuda_wrappers");
     CC1Args.push_back("-internal-isystem");
     CC1Args.push_back(DriverArgs.MakeArgString(P));
-    CC1Args.push_back("-include");
-    CC1Args.push_back("__clang_hip_runtime_wrapper.h");
   }
 
   if (DriverArgs.hasArg(options::OPT_nogpuinc))
     return;
 
-  if (!isValid()) {
-    D.Diag(diag::err_drv_no_rocm_installation);
+  if (!hasHIPRuntime()) {
+    D.Diag(diag::err_drv_no_hip_runtime);
     return;
   }
 
   CC1Args.push_back("-internal-isystem");
   CC1Args.push_back(DriverArgs.MakeArgString(getIncludePath()));
+  if (UsesRuntimeWrapper)
+    CC1Args.append({"-include", "__clang_hip_runtime_wrapper.h"});
 }
 
 void amdgpu::Linker::ConstructJob(Compilation &C, const JobAction &JA,
@@ -251,8 +354,9 @@ void amdgpu::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-shared");
   CmdArgs.push_back("-o");
   CmdArgs.push_back(Output.getFilename());
-  C.addCommand(std::make_unique<Command>(JA, *this, Args.MakeArgString(Linker),
-                                          CmdArgs, Inputs));
+  C.addCommand(
+      std::make_unique<Command>(JA, *this, ResponseFileSupport::AtFileCurCP(),
+                                Args.MakeArgString(Linker), CmdArgs, Inputs));
 }
 
 void amdgpu::getAMDGPUTargetFeatures(const Driver &D,
@@ -385,8 +489,9 @@ bool AMDGPUToolChain::isWave64(const llvm::opt::ArgList &DriverArgs,
 /// ROCM Toolchain
 ROCMToolChain::ROCMToolChain(const Driver &D, const llvm::Triple &Triple,
                              const ArgList &Args)
-  : AMDGPUToolChain(D, Triple, Args),
-    RocmInstallation(D, Triple, Args) { }
+    : AMDGPUToolChain(D, Triple, Args) {
+  RocmInstallation.detectDeviceLibrary();
+}
 
 void AMDGPUToolChain::addClangTargetOptions(
     const llvm::opt::ArgList &DriverArgs,
@@ -417,8 +522,8 @@ void ROCMToolChain::addClangTargetOptions(
   if (DriverArgs.hasArg(options::OPT_nogpulib))
     return;
 
-  if (!RocmInstallation.isValid()) {
-    getDriver().Diag(diag::err_drv_no_rocm_installation);
+  if (!RocmInstallation.hasDeviceLibrary()) {
+    getDriver().Diag(diag::err_drv_no_rocm_device_lib) << 0;
     return;
   }
 
@@ -428,7 +533,7 @@ void ROCMToolChain::addClangTargetOptions(
   const StringRef CanonArch = llvm::AMDGPU::getArchNameAMDGCN(Kind);
   std::string LibDeviceFile = RocmInstallation.getLibDeviceFile(CanonArch);
   if (LibDeviceFile.empty()) {
-    getDriver().Diag(diag::err_drv_no_rocm_device_lib) << GpuArch;
+    getDriver().Diag(diag::err_drv_no_rocm_device_lib) << 1 << GpuArch;
     return;
   }
 
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.h b/clang/lib/Driver/ToolChains/AMDGPU.h
index 671a987bf9fc2..5d44faf28b053 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.h
+++ b/clang/lib/Driver/ToolChains/AMDGPU.h
@@ -25,9 +25,9 @@ namespace driver {
 namespace tools {
 namespace amdgpu {
 
-class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC) : GnuTool("amdgpu::Linker", "ld.lld", TC) {}
+  Linker(const ToolChain &TC) : Tool("amdgpu::Linker", "ld.lld", TC) {}
   bool isLinkJob() const override { return true; }
   bool hasIntegratedCPP() const override { return false; }
   void ConstructJob(Compilation &C, const JobAction &JA,
@@ -85,12 +85,11 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUToolChain : public Generic_ELF {
     return true;
   }
 
+  /// Needed for translating LTO options.
+  const char *getDefaultLinker() const override { return "ld.lld"; }
 };
 
 class LLVM_LIBRARY_VISIBILITY ROCMToolChain : public AMDGPUToolChain {
-protected:
-  RocmInstallationDetector RocmInstallation;
-
 public:
   ROCMToolChain(const Driver &D, const llvm::Triple &Triple,
                 const llvm::opt::ArgList &Args);
diff --git a/clang/lib/Driver/ToolChains/AVR.cpp b/clang/lib/Driver/ToolChains/AVR.cpp
index a86d6fa9f3572..092bade53c635 100644
--- a/clang/lib/Driver/ToolChains/AVR.cpp
+++ b/clang/lib/Driver/ToolChains/AVR.cpp
@@ -118,13 +118,6 @@ void AVR::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddAllArgs(CmdArgs, options::OPT_L);
   getToolChain().AddFilePathLibArgs(Args, CmdArgs);
 
-  //   "Not [sic] that addr must be offset by adding 0x800000 the to
-  //    real SRAM address so that the linker knows that the address
-  //    is in the SRAM memory space."
-  //
-  //      - https://www.nongnu.org/avr-libc/user-manual/mem_sections.html
-  CmdArgs.push_back("-Tdata=0x800100");
-
   // If the family name is known, we can link with the device-specific libgcc.
   // Without it, libgcc will simply not be linked. This matches avr-gcc
   // behavior.
@@ -149,8 +142,9 @@ void AVR::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Args.MakeArgString(std::string("-m") + *FamilyName));
   }
 
-  C.addCommand(std::make_unique<Command>(JA, *this, Args.MakeArgString(Linker),
-                                          CmdArgs, Inputs));
+  C.addCommand(
+      std::make_unique<Command>(JA, *this, ResponseFileSupport::AtFileCurCP(),
+                                Args.MakeArgString(Linker), CmdArgs, Inputs));
 }
 
 llvm::Optional<std::string> AVRToolChain::findAVRLibcInstallation() const {
diff --git a/clang/lib/Driver/ToolChains/AVR.h b/clang/lib/Driver/ToolChains/AVR.h
index d244fc4f90e90..a3198b2495803 100644
--- a/clang/lib/Driver/ToolChains/AVR.h
+++ b/clang/lib/Driver/ToolChains/AVR.h
@@ -40,10 +40,10 @@ class LLVM_LIBRARY_VISIBILITY AVRToolChain : public Generic_ELF {
 
 namespace tools {
 namespace AVR {
-class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
   Linker(const llvm::Triple &Triple, const ToolChain &TC, bool LinkStdlib)
-      : GnuTool("AVR::Linker", "avr-ld", TC), Triple(Triple),
+      : Tool("AVR::Linker", "avr-ld", TC), Triple(Triple),
         LinkStdlib(LinkStdlib) {}
 
   bool hasIntegratedCPP() const override { return false; }
diff --git a/clang/lib/Driver/ToolChains/Ananas.cpp b/clang/lib/Driver/ToolChains/Ananas.cpp
index 10e4ea70db41d..a4141a57acccb 100644
--- a/clang/lib/Driver/ToolChains/Ananas.cpp
+++ b/clang/lib/Driver/ToolChains/Ananas.cpp
@@ -39,7 +39,8 @@ void ananas::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(II.getFilename());
 
   const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("as"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 void ananas::Linker::ConstructJob(Compilation &C, const JobAction &JA,
@@ -123,7 +124,8 @@ void ananas::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   const char *Exec = Args.MakeArgString(ToolChain.GetLinkerPath());
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 // Ananas - Ananas tool chain which can call as(1) and ld(1) directly.
diff --git a/clang/lib/Driver/ToolChains/Ananas.h b/clang/lib/Driver/ToolChains/Ananas.h
index 5e45b47fc1083..72ad3edcf0567 100644
--- a/clang/lib/Driver/ToolChains/Ananas.h
+++ b/clang/lib/Driver/ToolChains/Ananas.h
@@ -19,10 +19,9 @@ namespace tools {
 
 /// ananas -- Directly call GNU Binutils assembler and linker
 namespace ananas {
-class LLVM_LIBRARY_VISIBILITY Assembler : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 public:
-  Assembler(const ToolChain &TC)
-      : GnuTool("ananas::Assembler", "assembler", TC) {}
+  Assembler(const ToolChain &TC) : Tool("ananas::Assembler", "assembler", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
 
@@ -32,9 +31,9 @@ class LLVM_LIBRARY_VISIBILITY Assembler : public GnuTool {
                     const char *LinkingOutput) const override;
 };
 
-class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC) : GnuTool("ananas::Linker", "linker", TC) {}
+  Linker(const ToolChain &TC) : Tool("ananas::Linker", "linker", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
diff --git a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
index f16827e00f1f8..8659ebf17a722 100644
--- a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
@@ -62,6 +62,8 @@ isExperimentalExtension(StringRef Ext) {
       Ext == "zbf" || Ext == "zbm" || Ext == "zbp" || Ext == "zbr" ||
       Ext == "zbs" || Ext == "zbt" || Ext == "zbproposedc")
     return RISCVExtensionVersion{"0", "92"};
+  if (Ext == "v")
+    return RISCVExtensionVersion{"0", "8"};
   return None;
 }
 
@@ -399,6 +401,9 @@ static bool getArchFeatures(const Driver &D, StringRef MArch,
     case 'b':
       Features.push_back("+experimental-b");
       break;
+    case 'v':
+      Features.push_back("+experimental-v");
+      break;
     }
 
     // Consume full extension name and version, including any optional '_'
diff --git a/clang/lib/Driver/ToolChains/Arch/VE.cpp b/clang/lib/Driver/ToolChains/Arch/VE.cpp
new file mode 100644
index 0000000000000..fa10e4810f1c9
--- /dev/null
+++ b/clang/lib/Driver/ToolChains/Arch/VE.cpp
@@ -0,0 +1,26 @@
+//===--- VE.cpp - Tools Implementations -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "VE.h"
+#include "clang/Driver/Driver.h"
+#include "clang/Driver/DriverDiagnostic.h"
+#include "clang/Driver/Options.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Option/ArgList.h"
+
+using namespace clang::driver;
+using namespace clang::driver::tools;
+using namespace clang;
+using namespace llvm::opt;
+
+const char *ve::getVEAsmModeForCPU(StringRef Name, const llvm::Triple &Triple) {
+  return "";
+}
+
+void ve::getVETargetFeatures(const Driver &D, const ArgList &Args,
+                             std::vector<StringRef> &Features) {}
diff --git a/clang/lib/Driver/ToolChains/Arch/VE.h b/clang/lib/Driver/ToolChains/Arch/VE.h
new file mode 100644
index 0000000000000..713e3e7d042f2
--- /dev/null
+++ b/clang/lib/Driver/ToolChains/Arch/VE.h
@@ -0,0 +1,33 @@
+//===--- VE.h - VE-specific Tool Helpers ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_ARCH_VE_H
+#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_ARCH_VE_H
+
+#include "clang/Driver/Driver.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Option/Option.h"
+#include <string>
+#include <vector>
+
+namespace clang {
+namespace driver {
+namespace tools {
+namespace ve {
+
+void getVETargetFeatures(const Driver &D, const llvm::opt::ArgList &Args,
+                         std::vector<llvm::StringRef> &Features);
+const char *getVEAsmModeForCPU(llvm::StringRef Name,
+                               const llvm::Triple &Triple);
+
+} // end namespace ve
+} // namespace tools
+} // end namespace driver
+} // end namespace clang
+
+#endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_ARCH_VE_H
diff --git a/clang/lib/Driver/ToolChains/Arch/X86.cpp b/clang/lib/Driver/ToolChains/Arch/X86.cpp
index dbbc025de38c0..2cc44c09917f5 100644
--- a/clang/lib/Driver/ToolChains/Arch/X86.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/X86.cpp
@@ -94,6 +94,7 @@ const char *x86::getX86TargetCPU(const ArgList &Args,
 
   switch (Triple.getOS()) {
   case llvm::Triple::FreeBSD:
+    return "i686";
   case llvm::Triple::NetBSD:
   case llvm::Triple::OpenBSD:
     return "i486";
@@ -184,6 +185,24 @@ void x86::getX86TargetFeatures(const Driver &D, const llvm::Triple &Triple,
     LVIOpt = options::OPT_mlvi_cfi;
   }
 
+  if (Args.hasFlag(options::OPT_m_seses, options::OPT_mno_seses, false)) {
+    if (LVIOpt == options::OPT_mlvi_hardening)
+      D.Diag(diag::err_drv_argument_not_allowed_with)
+          << D.getOpts().getOptionName(options::OPT_mlvi_hardening)
+          << D.getOpts().getOptionName(options::OPT_m_seses);
+
+    if (SpectreOpt != clang::driver::options::ID::OPT_INVALID)
+      D.Diag(diag::err_drv_argument_not_allowed_with)
+          << D.getOpts().getOptionName(SpectreOpt)
+          << D.getOpts().getOptionName(options::OPT_m_seses);
+
+    Features.push_back("+seses");
+    if (!Args.hasArg(options::OPT_mno_lvi_cfi)) {
+      Features.push_back("+lvi-cfi");
+      LVIOpt = options::OPT_mlvi_cfi;
+    }
+  }
+
   if (SpectreOpt != clang::driver::options::ID::OPT_INVALID &&
       LVIOpt != clang::driver::options::ID::OPT_INVALID) {
     D.Diag(diag::err_drv_argument_not_allowed_with)
diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp
index 95f46b489e469..97cfa7d0e1569 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.cpp
+++ b/clang/lib/Driver/ToolChains/BareMetal.cpp
@@ -191,7 +191,7 @@ void baremetal::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-o");
   CmdArgs.push_back(Output.getFilename());
 
-  C.addCommand(std::make_unique<Command>(JA, *this,
-                                          Args.MakeArgString(TC.GetLinkerPath()),
-                                          CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Args.MakeArgString(TC.GetLinkerPath()),
+                                         CmdArgs, Inputs));
 }
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index aa18727671be0..9d6333bb5f1dd 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -15,6 +15,7 @@
 #include "Arch/RISCV.h"
 #include "Arch/Sparc.h"
 #include "Arch/SystemZ.h"
+#include "Arch/VE.h"
 #include "Arch/X86.h"
 #include "CommonArgs.h"
 #include "Hexagon.h"
@@ -368,27 +369,14 @@ static void getTargetFeatures(const Driver &D, const llvm::Triple &Triple,
     break;
   case llvm::Triple::msp430:
     msp430::getMSP430TargetFeatures(D, Args, Features);
+    break;
+  case llvm::Triple::ve:
+    ve::getVETargetFeatures(D, Args, Features);
   }
 
-  // Find the last of each feature.
-  llvm::StringMap<unsigned> LastOpt;
-  for (unsigned I = 0, N = Features.size(); I < N; ++I) {
-    StringRef Name = Features[I];
-    assert(Name[0] == '-' || Name[0] == '+');
-    LastOpt[Name.drop_front(1)] = I;
-  }
-
-  for (unsigned I = 0, N = Features.size(); I < N; ++I) {
-    // If this feature was overridden, ignore it.
-    StringRef Name = Features[I];
-    llvm::StringMap<unsigned>::iterator LastI = LastOpt.find(Name.drop_front(1));
-    assert(LastI != LastOpt.end());
-    unsigned Last = LastI->second;
-    if (Last != I)
-      continue;
-
+  for (auto Feature : unifyTargetFeatures(Features)) {
     CmdArgs.push_back(IsAux ? "-aux-target-feature" : "-target-feature");
-    CmdArgs.push_back(Name.data());
+    CmdArgs.push_back(Feature.data());
   }
 }
 
@@ -510,7 +498,7 @@ static codegenoptions::DebugInfoKind DebugLevelToInfoKind(const Arg &A) {
     return codegenoptions::DebugLineTablesOnly;
   if (A.getOption().matches(options::OPT_gline_directives_only))
     return codegenoptions::DebugDirectivesOnly;
-  return codegenoptions::LimitedDebugInfo;
+  return codegenoptions::DebugInfoConstructor;
 }
 
 static bool mustUseNonLeafFramePointerForTarget(const llvm::Triple &Triple) {
@@ -733,38 +721,6 @@ static void addDashXForInput(const ArgList &Args, const InputInfo &Input,
   }
 }
 
-static void appendUserToPath(SmallVectorImpl<char> &Result) {
-#ifdef LLVM_ON_UNIX
-  const char *Username = getenv("LOGNAME");
-#else
-  const char *Username = getenv("USERNAME");
-#endif
-  if (Username) {
-    // Validate that LoginName can be used in a path, and get its length.
-    size_t Len = 0;
-    for (const char *P = Username; *P; ++P, ++Len) {
-      if (!clang::isAlphanumeric(*P) && *P != '_') {
-        Username = nullptr;
-        break;
-      }
-    }
-
-    if (Username && Len > 0) {
-      Result.append(Username, Username + Len);
-      return;
-    }
-  }
-
-// Fallback to user id.
-#ifdef LLVM_ON_UNIX
-  std::string UID = llvm::utostr(getuid());
-#else
-  // FIXME: Windows seems to have an 'SID' that might work.
-  std::string UID = "9999";
-#endif
-  Result.append(UID.begin(), UID.end());
-}
-
 static void addPGOAndCoverageFlags(const ToolChain &TC, Compilation &C,
                                    const Driver &D, const InputInfo &Output,
                                    const ArgList &Args,
@@ -1663,6 +1619,10 @@ void Clang::RenderTargetOptions(const llvm::Triple &EffectiveTriple,
   case llvm::Triple::wasm64:
     AddWebAssemblyTargetArgs(Args, CmdArgs);
     break;
+
+  case llvm::Triple::ve:
+    AddVETargetArgs(Args, CmdArgs);
+    break;
   }
 }
 
@@ -2160,6 +2120,12 @@ void Clang::AddWebAssemblyTargetArgs(const ArgList &Args,
   }
 }
 
+void Clang::AddVETargetArgs(const ArgList &Args, ArgStringList &CmdArgs) const {
+  // Floating point operations and argument passing are hard.
+  CmdArgs.push_back("-mfloat-abi");
+  CmdArgs.push_back("hard");
+}
+
 void Clang::DumpCompilationDatabase(Compilation &C, StringRef Filename,
                                     StringRef Target, const InputInfo &Output,
                                     const InputInfo &Input, const ArgList &Args) const {
@@ -2414,7 +2380,7 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
           CmdArgs.push_back(Value.data());
         } else {
           RenderDebugEnablingArgs(Args, CmdArgs,
-                                  codegenoptions::LimitedDebugInfo,
+                                  codegenoptions::DebugInfoConstructor,
                                   DwarfVersion, llvm::DebuggerKind::Default);
         }
       } else if (Value.startswith("-mcpu") || Value.startswith("-mfpu") ||
@@ -3000,7 +2966,8 @@ static void RenderSCPOptions(const ToolChain &TC, const ArgList &Args,
   if (!EffectiveTriple.isOSLinux())
     return;
 
-  if (!EffectiveTriple.isX86() && !EffectiveTriple.isSystemZ())
+  if (!EffectiveTriple.isX86() && !EffectiveTriple.isSystemZ() &&
+      !EffectiveTriple.isPPC64())
     return;
 
   if (Args.hasFlag(options::OPT_fstack_clash_protection,
@@ -3203,11 +3170,13 @@ static void RenderBuiltinOptions(const ToolChain &TC, const llvm::Triple &T,
     CmdArgs.push_back("-fno-math-builtin");
 }
 
-void Driver::getDefaultModuleCachePath(SmallVectorImpl<char> &Result) {
-  llvm::sys::path::system_temp_directory(/*erasedOnReboot=*/false, Result);
-  llvm::sys::path::append(Result, "org.llvm.clang.");
-  appendUserToPath(Result);
-  llvm::sys::path::append(Result, "ModuleCache");
+bool Driver::getDefaultModuleCachePath(SmallVectorImpl<char> &Result) {
+  if (llvm::sys::path::cache_directory(Result)) {
+    llvm::sys::path::append(Result, "clang");
+    llvm::sys::path::append(Result, "ModuleCache");
+    return true;
+  }
+  return false;
 }
 
 static void RenderModulesOptions(Compilation &C, const Driver &D,
@@ -3264,6 +3233,7 @@ static void RenderModulesOptions(Compilation &C, const Driver &D,
     if (Arg *A = Args.getLastArg(options::OPT_fmodules_cache_path))
       Path = A->getValue();
 
+    bool HasPath = true;
     if (C.isForDiagnostics()) {
       // When generating crash reports, we want to emit the modules along with
       // the reproduction sources, so we ignore any provided module path.
@@ -3272,12 +3242,16 @@ static void RenderModulesOptions(Compilation &C, const Driver &D,
       llvm::sys::path::append(Path, "modules");
     } else if (Path.empty()) {
       // No module path was provided: use the default.
-      Driver::getDefaultModuleCachePath(Path);
+      HasPath = Driver::getDefaultModuleCachePath(Path);
     }
 
-    const char Arg[] = "-fmodules-cache-path=";
-    Path.insert(Path.begin(), Arg, Arg + strlen(Arg));
-    CmdArgs.push_back(Args.MakeArgString(Path));
+    // `HasPath` will only be false if getDefaultModuleCachePath() fails.
+    // That being said, that failure is unlikely and not caching is harmless.
+    if (HasPath) {
+      const char Arg[] = "-fmodules-cache-path=";
+      Path.insert(Path.begin(), Arg, Arg + strlen(Arg));
+      CmdArgs.push_back(Args.MakeArgString(Path));
+    }
   }
 
   if (HaveModules) {
@@ -3679,7 +3653,7 @@ static void RenderDebugOptions(const ToolChain &TC, const Driver &D,
   if (const Arg *A =
           Args.getLastArg(options::OPT_g_Group, options::OPT_gsplit_dwarf,
                           options::OPT_gsplit_dwarf_EQ)) {
-    DebugInfoKind = codegenoptions::LimitedDebugInfo;
+    DebugInfoKind = codegenoptions::DebugInfoConstructor;
 
     // If the last option explicitly specified a debug-info level, use it.
     if (checkDebugInfoOption(A, Args, D, TC) &&
@@ -3774,10 +3748,9 @@ static void RenderDebugOptions(const ToolChain &TC, const Driver &D,
   // not to include any column info.
   if (const Arg *A = Args.getLastArg(options::OPT_gcolumn_info))
     (void)checkDebugInfoOption(A, Args, D, TC);
-  if (Args.hasFlag(options::OPT_gcolumn_info, options::OPT_gno_column_info,
-                   /*Default=*/!EmitCodeView &&
-                       DebuggerTuning != llvm::DebuggerKind::SCE))
-    CmdArgs.push_back("-dwarf-column-info");
+  if (!Args.hasFlag(options::OPT_gcolumn_info, options::OPT_gno_column_info,
+                    !EmitCodeView && DebuggerTuning != llvm::DebuggerKind::SCE))
+    CmdArgs.push_back("-gno-column-info");
 
   // FIXME: Move backend command line options to the module.
   // If -gline-tables-only or -gline-directives-only is the last option it wins.
@@ -3785,7 +3758,7 @@ static void RenderDebugOptions(const ToolChain &TC, const Driver &D,
     if (checkDebugInfoOption(A, Args, D, TC)) {
       if (DebugInfoKind != codegenoptions::DebugLineTablesOnly &&
           DebugInfoKind != codegenoptions::DebugDirectivesOnly) {
-        DebugInfoKind = codegenoptions::LimitedDebugInfo;
+        DebugInfoKind = codegenoptions::DebugInfoConstructor;
         CmdArgs.push_back("-dwarf-ext-refs");
         CmdArgs.push_back("-fmodule-format=obj");
       }
@@ -3805,7 +3778,9 @@ static void RenderDebugOptions(const ToolChain &TC, const Driver &D,
           TC.GetDefaultStandaloneDebug());
   if (const Arg *A = Args.getLastArg(options::OPT_fstandalone_debug))
     (void)checkDebugInfoOption(A, Args, D, TC);
-  if (DebugInfoKind == codegenoptions::LimitedDebugInfo && NeedFullDebug)
+  if ((DebugInfoKind == codegenoptions::LimitedDebugInfo ||
+       DebugInfoKind == codegenoptions::DebugInfoConstructor) &&
+      NeedFullDebug)
     DebugInfoKind = codegenoptions::FullDebugInfo;
 
   if (Args.hasFlag(options::OPT_gembed_source, options::OPT_gno_embed_source,
@@ -3978,7 +3953,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     }
   }
 
-  const llvm::Triple *AuxTriple = IsCuda ? TC.getAuxTriple() : nullptr;
+  const llvm::Triple *AuxTriple =
+      (IsCuda || IsHIP) ? TC.getAuxTriple() : nullptr;
   bool IsWindowsMSVC = RawTriple.isWindowsMSVCEnvironment();
   bool IsIAMCU = RawTriple.isOSIAMCU();
 
@@ -4068,9 +4044,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (Triple.isOSWindows() && (Triple.getArch() == llvm::Triple::arm ||
                                Triple.getArch() == llvm::Triple::thumb)) {
     unsigned Offset = Triple.getArch() == llvm::Triple::arm ? 4 : 6;
-    unsigned Version;
-    Triple.getArchName().substr(Offset).getAsInteger(10, Version);
-    if (Version < 7)
+    unsigned Version = 0;
+    bool Failure =
+        Triple.getArchName().substr(Offset).consumeInteger(10, Version);
+    if (Failure || Version < 7)
       D.Diag(diag::err_target_unsupported_arch) << Triple.getArchName()
                                                 << TripleStr;
   }
@@ -4323,8 +4300,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
         II.getInputArg().renderAsInput(Args, CmdArgs);
     }
 
-    C.addCommand(std::make_unique<Command>(JA, *this, D.getClangProgramPath(),
-                                            CmdArgs, Inputs));
+    C.addCommand(
+        std::make_unique<Command>(JA, *this, ResponseFileSupport::AtFileUTF8(),
+                                  D.getClangProgramPath(), CmdArgs, Inputs));
     return;
   }
 
@@ -4580,8 +4558,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back(FPKeepKindStr);
 
   if (!Args.hasFlag(options::OPT_fzero_initialized_in_bss,
-                    options::OPT_fno_zero_initialized_in_bss))
-    CmdArgs.push_back("-mno-zero-initialized-in-bss");
+                    options::OPT_fno_zero_initialized_in_bss, true))
+    CmdArgs.push_back("-fno-zero-initialized-in-bss");
 
   bool OFastEnabled = isOptimizationLevelFast(Args);
   // If -Ofast is the optimization level, then -fstrict-aliasing should be
@@ -4891,10 +4869,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                   options::OPT_finstrument_functions_after_inlining,
                   options::OPT_finstrument_function_entry_bare);
 
-  // NVPTX doesn't support PGO or coverage. There's no runtime support for
-  // sampling, overhead of call arc collection is way too high and there's no
-  // way to collect the output.
-  if (!Triple.isNVPTX())
+  // NVPTX/AMDGCN doesn't support PGO or coverage. There's no runtime support
+  // for sampling, overhead of call arc collection is way too high and there's
+  // no way to collect the output.
+  if (!Triple.isNVPTX() && !Triple.isAMDGCN())
     addPGOAndCoverageFlags(TC, C, D, Output, Args, CmdArgs);
 
   Args.AddLastArg(CmdArgs, options::OPT_fclang_abi_compat_EQ);
@@ -5013,6 +4991,11 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
     Args.AddLastArg(CmdArgs, options::OPT_ftrigraphs,
                     options::OPT_fno_trigraphs);
+
+    // HIP headers has minimum C++ standard requirements. Therefore set the
+    // default language standard.
+    if (IsHIP)
+      CmdArgs.push_back(IsWindowsMSVC ? "-std=c++14" : "-std=c++11");
   }
 
   // GCC's behavior for -Wwrite-strings is a bit strange:
@@ -5243,6 +5226,13 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                        options::OPT_fno_openmp_cuda_mode, /*Default=*/false))
         CmdArgs.push_back("-fopenmp-cuda-mode");
 
+      // When in OpenMP offloading mode with NVPTX target, forward
+      // cuda-parallel-target-regions flag
+      if (Args.hasFlag(options::OPT_fopenmp_cuda_parallel_target_regions,
+                       options::OPT_fno_openmp_cuda_parallel_target_regions,
+                       /*Default=*/true))
+        CmdArgs.push_back("-fopenmp-cuda-parallel-target-regions");
+
       // When in OpenMP offloading mode with NVPTX target, check if full runtime
       // is required.
       if (Args.hasFlag(options::OPT_fopenmp_cuda_force_full_runtime,
@@ -5414,8 +5404,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // Forward -cl options to -cc1
   RenderOpenCLOptions(Args, CmdArgs);
 
-  if (Args.hasFlag(options::OPT_fhip_new_launch_api,
-                   options::OPT_fno_hip_new_launch_api, false))
+  if (IsHIP && Args.hasFlag(options::OPT_fhip_new_launch_api,
+                            options::OPT_fno_hip_new_launch_api, true))
     CmdArgs.push_back("-fhip-new-launch-api");
 
   if (Arg *A = Args.getLastArg(options::OPT_fcf_protection_EQ)) {
@@ -6235,19 +6225,21 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     auto CLCommand =
         getCLFallback()->GetCommand(C, JA, Output, Inputs, Args, LinkingOutput);
     C.addCommand(std::make_unique<FallbackCommand>(
-        JA, *this, Exec, CmdArgs, Inputs, std::move(CLCommand)));
+        JA, *this, ResponseFileSupport::AtFileUTF8(), Exec, CmdArgs, Inputs,
+        std::move(CLCommand)));
   } else if (Args.hasArg(options::OPT__SLASH_fallback) &&
              isa<PrecompileJobAction>(JA)) {
     // In /fallback builds, run the main compilation even if the pch generation
     // fails, so that the main compilation's fallback to cl.exe runs.
-    C.addCommand(std::make_unique<ForceSuccessCommand>(JA, *this, Exec,
-                                                        CmdArgs, Inputs));
+    C.addCommand(std::make_unique<ForceSuccessCommand>(
+        JA, *this, ResponseFileSupport::AtFileUTF8(), Exec, CmdArgs, Inputs));
   } else if (D.CC1Main && !D.CCGenDiagnostics) {
     // Invoke the CC1 directly in this process
-    C.addCommand(
-        std::make_unique<CC1Command>(JA, *this, Exec, CmdArgs, Inputs));
+    C.addCommand(std::make_unique<CC1Command>(
+        JA, *this, ResponseFileSupport::AtFileUTF8(), Exec, CmdArgs, Inputs));
   } else {
-    C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+    C.addCommand(std::make_unique<Command>(
+        JA, *this, ResponseFileSupport::AtFileUTF8(), Exec, CmdArgs, Inputs));
   }
 
   // Make the compile command echo its inputs for /showFilenames.
@@ -6282,7 +6274,7 @@ Clang::Clang(const ToolChain &TC)
     // CAUTION! The first constructor argument ("clang") is not arbitrary,
     // as it is for other tools. Some operations on a Tool actually test
     // whether that tool is Clang based on the Tool's Name as a string.
-    : Tool("clang", "clang frontend", TC, RF_Full) {}
+    : Tool("clang", "clang frontend", TC) {}
 
 Clang::~Clang() {}
 
@@ -6568,7 +6560,7 @@ void Clang::AddClangCLArgs(const ArgList &Args, types::ID InputType,
                           options::OPT_gline_tables_only)) {
     *EmitCodeView = true;
     if (DebugInfoArg->getOption().matches(options::OPT__SLASH_Z7))
-      *DebugInfoKind = codegenoptions::LimitedDebugInfo;
+      *DebugInfoKind = codegenoptions::DebugInfoConstructor;
     else
       *DebugInfoKind = codegenoptions::DebugLineTablesOnly;
   } else {
@@ -6865,7 +6857,7 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
     // the guard for source type, however there is a test which asserts
     // that some assembler invocation receives no -debug-info-kind,
     // and it's not clear whether that test is just overly restrictive.
-    DebugInfoKind = (WantDebug ? codegenoptions::LimitedDebugInfo
+    DebugInfoKind = (WantDebug ? codegenoptions::DebugInfoConstructor
                                : codegenoptions::NoDebugInfo);
     // Add the -fdebug-compilation-dir flag if needed.
     addDebugCompDirArg(Args, CmdArgs, C.getDriver().getVFS());
@@ -6985,7 +6977,8 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back(Input.getFilename());
 
   const char *Exec = getToolChain().getDriver().getClangProgramPath();
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileUTF8(), Exec, CmdArgs, Inputs));
 }
 
 // Begin OffloadBundler
@@ -7069,7 +7062,7 @@ void OffloadBundler::ConstructJob(Compilation &C, const JobAction &JA,
 
   // All the inputs are encoded as commands.
   C.addCommand(std::make_unique<Command>(
-      JA, *this,
+      JA, *this, ResponseFileSupport::None(),
       TCArgs.MakeArgString(getToolChain().GetProgramPath(getShortName())),
       CmdArgs, None));
 }
@@ -7135,7 +7128,7 @@ void OffloadBundler::ConstructJobMultipleOutputs(
 
   // All the inputs are encoded as commands.
   C.addCommand(std::make_unique<Command>(
-      JA, *this,
+      JA, *this, ResponseFileSupport::None(),
       TCArgs.MakeArgString(getToolChain().GetProgramPath(getShortName())),
       CmdArgs, None));
 }
@@ -7165,7 +7158,7 @@ void OffloadWrapper::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   C.addCommand(std::make_unique<Command>(
-      JA, *this,
+      JA, *this, ResponseFileSupport::None(),
       Args.MakeArgString(getToolChain().GetProgramPath(getShortName())),
       CmdArgs, Inputs));
 }
diff --git a/clang/lib/Driver/ToolChains/Clang.h b/clang/lib/Driver/ToolChains/Clang.h
index 48100c2fc6ec3..a607e3c27de9f 100644
--- a/clang/lib/Driver/ToolChains/Clang.h
+++ b/clang/lib/Driver/ToolChains/Clang.h
@@ -73,6 +73,8 @@ class LLVM_LIBRARY_VISIBILITY Clang : public Tool {
                           llvm::opt::ArgStringList &CmdArgs) const;
   void AddWebAssemblyTargetArgs(const llvm::opt::ArgList &Args,
                                 llvm::opt::ArgStringList &CmdArgs) const;
+  void AddVETargetArgs(const llvm::opt::ArgList &Args,
+                       llvm::opt::ArgStringList &CmdArgs) const;
 
   enum RewriteKind { RK_None, RK_Fragile, RK_NonFragile };
 
@@ -119,7 +121,7 @@ class LLVM_LIBRARY_VISIBILITY Clang : public Tool {
 class LLVM_LIBRARY_VISIBILITY ClangAs : public Tool {
 public:
   ClangAs(const ToolChain &TC)
-      : Tool("clang::as", "clang integrated assembler", TC, RF_Full) {}
+      : Tool("clang::as", "clang integrated assembler", TC) {}
   void AddMIPSTargetArgs(const llvm::opt::ArgList &Args,
                          llvm::opt::ArgStringList &CmdArgs) const;
   void AddX86TargetArgs(const llvm::opt::ArgList &Args,
diff --git a/clang/lib/Driver/ToolChains/CloudABI.cpp b/clang/lib/Driver/ToolChains/CloudABI.cpp
index 0602e4f6d0b3d..8dcfd4951bbfe 100644
--- a/clang/lib/Driver/ToolChains/CloudABI.cpp
+++ b/clang/lib/Driver/ToolChains/CloudABI.cpp
@@ -92,7 +92,8 @@ void cloudabi::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crtend.o")));
 
   const char *Exec = Args.MakeArgString(ToolChain.GetLinkerPath());
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 // CloudABI - CloudABI tool chain which can call ld(1) directly.
diff --git a/clang/lib/Driver/ToolChains/CloudABI.h b/clang/lib/Driver/ToolChains/CloudABI.h
index cc381c2b1e1fe..98bf23127706e 100644
--- a/clang/lib/Driver/ToolChains/CloudABI.h
+++ b/clang/lib/Driver/ToolChains/CloudABI.h
@@ -19,9 +19,9 @@ namespace tools {
 
 /// cloudabi -- Directly call GNU Binutils linker
 namespace cloudabi {
-class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC) : GnuTool("cloudabi::Linker", "linker", TC) {}
+  Linker(const ToolChain &TC) : Tool("cloudabi::Linker", "linker", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 7fdb3fdf00094..1cac5a0822a4b 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -12,6 +12,7 @@
 #include "Arch/Mips.h"
 #include "Arch/PPC.h"
 #include "Arch/SystemZ.h"
+#include "Arch/VE.h"
 #include "Arch/X86.h"
 #include "HIP.h"
 #include "Hexagon.h"
@@ -83,6 +84,31 @@ void tools::handleTargetFeaturesGroup(const ArgList &Args,
   }
 }
 
+std::vector<StringRef>
+tools::unifyTargetFeatures(const std::vector<StringRef> &Features) {
+  std::vector<StringRef> UnifiedFeatures;
+  // Find the last of each feature.
+  llvm::StringMap<unsigned> LastOpt;
+  for (unsigned I = 0, N = Features.size(); I < N; ++I) {
+    StringRef Name = Features[I];
+    assert(Name[0] == '-' || Name[0] == '+');
+    LastOpt[Name.drop_front(1)] = I;
+  }
+
+  for (unsigned I = 0, N = Features.size(); I < N; ++I) {
+    // If this feature was overridden, ignore it.
+    StringRef Name = Features[I];
+    llvm::StringMap<unsigned>::iterator LastI = LastOpt.find(Name.drop_front(1));
+    assert(LastI != LastOpt.end());
+    unsigned Last = LastI->second;
+    if (Last != I)
+      continue;
+
+    UnifiedFeatures.push_back(Name);
+  }
+  return UnifiedFeatures;
+}
+
 void tools::addDirectoryList(const ArgList &Args, ArgStringList &CmdArgs,
                              const char *ArgName, const char *EnvVar) {
   const char *DirList = ::getenv(EnvVar);
@@ -152,14 +178,12 @@ void tools::AddLinkerInputs(const ToolChain &TC, const InputInfoList &Inputs,
     addDirectoryList(Args, CmdArgs, "-L", "LIBRARY_PATH");
 
   for (const auto &II : Inputs) {
-    // If the current tool chain refers to an OpenMP or HIP offloading host, we
-    // should ignore inputs that refer to OpenMP or HIP offloading devices -
+    // If the current tool chain refers to an OpenMP offloading host, we
+    // should ignore inputs that refer to OpenMP offloading devices -
     // they will be embedded according to a proper linker script.
     if (auto *IA = II.getAction())
       if ((JA.isHostOffloading(Action::OFK_OpenMP) &&
-           IA->isDeviceOffloading(Action::OFK_OpenMP)) ||
-          (JA.isHostOffloading(Action::OFK_HIP) &&
-           IA->isDeviceOffloading(Action::OFK_HIP)))
+           IA->isDeviceOffloading(Action::OFK_OpenMP)))
         continue;
 
     if (!TC.HasNativeLLVMSupport() && types::isLLVMIR(II.getType()))
@@ -506,8 +530,11 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
 
 void tools::addArchSpecificRPath(const ToolChain &TC, const ArgList &Args,
                                  ArgStringList &CmdArgs) {
+  // Enable -frtlib-add-rpath by default for the case of VE.
+  const bool IsVE = TC.getTriple().isVE();
+  bool DefaultValue = IsVE;
   if (!Args.hasFlag(options::OPT_frtlib_add_rpath,
-                    options::OPT_fno_rtlib_add_rpath, false))
+                    options::OPT_fno_rtlib_add_rpath, DefaultValue))
     return;
 
   std::string CandidateRPath = TC.getArchSpecificLibPath();
@@ -862,10 +889,12 @@ void tools::SplitDebugInfo(const ToolChain &TC, Compilation &C, const Tool &T,
   InputInfo II(types::TY_Object, Output.getFilename(), Output.getFilename());
 
   // First extract the dwo sections.
-  C.addCommand(std::make_unique<Command>(JA, T, Exec, ExtractArgs, II));
+  C.addCommand(std::make_unique<Command>(
+      JA, T, ResponseFileSupport::AtFileCurCP(), Exec, ExtractArgs, II));
 
   // Then remove them from the original .o file.
-  C.addCommand(std::make_unique<Command>(JA, T, Exec, StripArgs, II));
+  C.addCommand(std::make_unique<Command>(
+      JA, T, ResponseFileSupport::AtFileCurCP(), Exec, StripArgs, II));
 }
 
 // Claim options we don't want to warn if they are unused. We do this for
@@ -1298,115 +1327,6 @@ void tools::AddRunTimeLibs(const ToolChain &TC, const Driver &D,
   }
 }
 
-/// Add HIP linker script arguments at the end of the argument list so that
-/// the fat binary is built by embedding the device images into the host. The
-/// linker script also defines a symbol required by the code generation so that
-/// the image can be retrieved at runtime. This should be used only in tool
-/// chains that support linker scripts.
-void tools::AddHIPLinkerScript(const ToolChain &TC, Compilation &C,
-                               const InputInfo &Output,
-                               const InputInfoList &Inputs, const ArgList &Args,
-                               ArgStringList &CmdArgs, const JobAction &JA,
-                               const Tool &T) {
-
-  // If this is not a HIP host toolchain, we don't need to do anything.
-  if (!JA.isHostOffloading(Action::OFK_HIP))
-    return;
-
-  InputInfoList DeviceInputs;
-  for (const auto &II : Inputs) {
-    const Action *A = II.getAction();
-    // Is this a device linking action?
-    if (A && isa<LinkJobAction>(A) && A->isDeviceOffloading(Action::OFK_HIP)) {
-      DeviceInputs.push_back(II);
-    }
-  }
-
-  if (DeviceInputs.empty())
-    return;
-
-  // Create temporary linker script. Keep it if save-temps is enabled.
-  const char *LKS;
-  std::string Name =
-      std::string(llvm::sys::path::filename(Output.getFilename()));
-  if (C.getDriver().isSaveTempsEnabled()) {
-    LKS = C.getArgs().MakeArgString(Name + ".lk");
-  } else {
-    auto TmpName = C.getDriver().GetTemporaryPath(Name, "lk");
-    LKS = C.addTempFile(C.getArgs().MakeArgString(TmpName));
-  }
-
-  // Add linker script option to the command.
-  CmdArgs.push_back("-T");
-  CmdArgs.push_back(LKS);
-
-  // Create a buffer to write the contents of the linker script.
-  std::string LksBuffer;
-  llvm::raw_string_ostream LksStream(LksBuffer);
-
-  // Get the HIP offload tool chain.
-  auto *HIPTC = static_cast<const toolchains::HIPToolChain *>(
-      C.getSingleOffloadToolChain<Action::OFK_HIP>());
-  assert(HIPTC->getTriple().getArch() == llvm::Triple::amdgcn &&
-         "Wrong platform");
-  (void)HIPTC;
-
-  const char *BundleFile;
-  if (C.getDriver().isSaveTempsEnabled()) {
-    BundleFile = C.getArgs().MakeArgString(Name + ".hipfb");
-  } else {
-    auto TmpName = C.getDriver().GetTemporaryPath(Name, "hipfb");
-    BundleFile = C.addTempFile(C.getArgs().MakeArgString(TmpName));
-  }
-  AMDGCN::constructHIPFatbinCommand(C, JA, BundleFile, DeviceInputs, Args, T);
-
-  // Add commands to embed target binaries. We ensure that each section and
-  // image is 16-byte aligned. This is not mandatory, but increases the
-  // likelihood of data to be aligned with a cache block in several main host
-  // machines.
-  LksStream << "/*\n";
-  LksStream << "       HIP Offload Linker Script\n";
-  LksStream << " *** Automatically generated by Clang ***\n";
-  LksStream << "*/\n";
-  LksStream << "TARGET(binary)\n";
-  LksStream << "INPUT(" << BundleFile << ")\n";
-  LksStream << "SECTIONS\n";
-  LksStream << "{\n";
-  LksStream << "  .hip_fatbin :\n";
-  LksStream << "  ALIGN(0x10)\n";
-  LksStream << "  {\n";
-  LksStream << "    PROVIDE_HIDDEN(__hip_fatbin = .);\n";
-  LksStream << "    " << BundleFile << "\n";
-  LksStream << "  }\n";
-  LksStream << "  /DISCARD/ :\n";
-  LksStream << "  {\n";
-  LksStream << "    * ( __CLANG_OFFLOAD_BUNDLE__* )\n";
-  LksStream << "  }\n";
-  LksStream << "}\n";
-  LksStream << "INSERT BEFORE .data\n";
-  LksStream.flush();
-
-  // Dump the contents of the linker script if the user requested that. We
-  // support this option to enable testing of behavior with -###.
-  if (C.getArgs().hasArg(options::OPT_fhip_dump_offload_linker_script))
-    llvm::errs() << LksBuffer;
-
-  // If this is a dry run, do not create the linker script file.
-  if (C.getArgs().hasArg(options::OPT__HASH_HASH_HASH))
-    return;
-
-  // Open script file and write the contents.
-  std::error_code EC;
-  llvm::raw_fd_ostream Lksf(LKS, EC, llvm::sys::fs::OF_None);
-
-  if (EC) {
-    C.getDriver().Diag(clang::diag::err_unable_to_make_temp) << EC.message();
-    return;
-  }
-
-  Lksf << LksBuffer;
-}
-
 SmallString<128> tools::getStatsFileName(const llvm::opt::ArgList &Args,
                                          const InputInfo &Output,
                                          const InputInfo &Input,
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.h b/clang/lib/Driver/ToolChains/CommonArgs.h
index 58bc92c9b7569..29dedec9b09cd 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.h
+++ b/clang/lib/Driver/ToolChains/CommonArgs.h
@@ -45,12 +45,6 @@ void AddRunTimeLibs(const ToolChain &TC, const Driver &D,
                     llvm::opt::ArgStringList &CmdArgs,
                     const llvm::opt::ArgList &Args);
 
-void AddHIPLinkerScript(const ToolChain &TC, Compilation &C,
-                        const InputInfo &Output, const InputInfoList &Inputs,
-                        const llvm::opt::ArgList &Args,
-                        llvm::opt::ArgStringList &CmdArgs, const JobAction &JA,
-                        const Tool &T);
-
 const char *SplitDebugName(const llvm::opt::ArgList &Args,
                            const InputInfo &Input, const InputInfo &Output);
 
@@ -112,10 +106,20 @@ void AddTargetFeature(const llvm::opt::ArgList &Args,
 std::string getCPUName(const llvm::opt::ArgList &Args, const llvm::Triple &T,
                        bool FromAs = false);
 
+/// Iterate \p Args and convert -mxxx to +xxx and -mno-xxx to -xxx and
+/// append it to \p Features.
+///
+/// Note: Since \p Features may contain default values before calling
+/// this function, or may be appended with entries to override arguments,
+/// entries in \p Features are not unique.
 void handleTargetFeaturesGroup(const llvm::opt::ArgList &Args,
                                std::vector<StringRef> &Features,
                                llvm::opt::OptSpecifier Group);
 
+/// If there are multiple +xxx or -xxx features, keep the last one.
+std::vector<StringRef>
+unifyTargetFeatures(const std::vector<StringRef> &Features);
+
 /// Handles the -save-stats option and returns the filename to save statistics
 /// to.
 SmallString<128> getStatsFileName(const llvm::opt::ArgList &Args,
diff --git a/clang/lib/Driver/ToolChains/CrossWindows.cpp b/clang/lib/Driver/ToolChains/CrossWindows.cpp
index dbf6114eb2ecc..127a8a5f24cce 100644
--- a/clang/lib/Driver/ToolChains/CrossWindows.cpp
+++ b/clang/lib/Driver/ToolChains/CrossWindows.cpp
@@ -57,7 +57,8 @@ void tools::CrossWindows::Assembler::ConstructJob(
   const std::string Assembler = TC.GetProgramPath("as");
   Exec = Args.MakeArgString(Assembler);
 
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Exec, CmdArgs, Inputs));
 }
 
 void tools::CrossWindows::Linker::ConstructJob(
@@ -202,7 +203,8 @@ void tools::CrossWindows::Linker::ConstructJob(
 
   Exec = Args.MakeArgString(TC.GetLinkerPath());
 
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileUTF8(), Exec, CmdArgs, Inputs));
 }
 
 CrossWindowsToolChain::CrossWindowsToolChain(const Driver &D,
diff --git a/clang/lib/Driver/ToolChains/CrossWindows.h b/clang/lib/Driver/ToolChains/CrossWindows.h
index 7267a35d48b94..df9a7f71bf9ff 100644
--- a/clang/lib/Driver/ToolChains/CrossWindows.h
+++ b/clang/lib/Driver/ToolChains/CrossWindows.h
@@ -33,8 +33,7 @@ class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 
 class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC)
-      : Tool("CrossWindows::Linker", "ld", TC, RF_Full) {}
+  Linker(const ToolChain &TC) : Tool("CrossWindows::Linker", "ld", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index a25541cf31e90..110a0bca9bc18 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -423,7 +423,11 @@ void NVPTX::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
     Exec = A->getValue();
   else
     Exec = Args.MakeArgString(TC.GetProgramPath("ptxas"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this,
+      ResponseFileSupport{ResponseFileSupport::RF_Full, llvm::sys::WEM_UTF8,
+                          "--options-file"},
+      Exec, CmdArgs, Inputs));
 }
 
 static bool shouldIncludePTX(const ArgList &Args, const char *gpu_arch) {
@@ -488,7 +492,11 @@ void NVPTX::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Args.MakeArgString(A));
 
   const char *Exec = Args.MakeArgString(TC.GetProgramPath("fatbinary"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this,
+      ResponseFileSupport{ResponseFileSupport::RF_Full, llvm::sys::WEM_UTF8,
+                          "--options-file"},
+      Exec, CmdArgs, Inputs));
 }
 
 void NVPTX::OpenMPLinker::ConstructJob(Compilation &C, const JobAction &JA,
@@ -565,7 +573,11 @@ void NVPTX::OpenMPLinker::ConstructJob(Compilation &C, const JobAction &JA,
 
   const char *Exec =
       Args.MakeArgString(getToolChain().GetProgramPath("nvlink"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this,
+      ResponseFileSupport{ResponseFileSupport::RF_Full, llvm::sys::WEM_UTF8,
+                          "--options-file"},
+      Exec, CmdArgs, Inputs));
 }
 
 /// CUDA toolchain.  Our assembler is ptxas, and our "linker" is fatbinary,
diff --git a/clang/lib/Driver/ToolChains/Cuda.h b/clang/lib/Driver/ToolChains/Cuda.h
index bf462d7025877..873eb7338a30a 100644
--- a/clang/lib/Driver/ToolChains/Cuda.h
+++ b/clang/lib/Driver/ToolChains/Cuda.h
@@ -89,9 +89,7 @@ namespace NVPTX {
 // Run ptxas, the NVPTX assembler.
 class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
  public:
-   Assembler(const ToolChain &TC)
-       : Tool("NVPTX::Assembler", "ptxas", TC, RF_Full, llvm::sys::WEM_UTF8,
-              "--options-file") {}
+   Assembler(const ToolChain &TC) : Tool("NVPTX::Assembler", "ptxas", TC) {}
 
    bool hasIntegratedCPP() const override { return false; }
 
@@ -105,9 +103,7 @@ class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 // assembly into a single output file.
 class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
  public:
-   Linker(const ToolChain &TC)
-       : Tool("NVPTX::Linker", "fatbinary", TC, RF_Full, llvm::sys::WEM_UTF8,
-              "--options-file") {}
+   Linker(const ToolChain &TC) : Tool("NVPTX::Linker", "fatbinary", TC) {}
 
    bool hasIntegratedCPP() const override { return false; }
 
@@ -120,8 +116,7 @@ class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 class LLVM_LIBRARY_VISIBILITY OpenMPLinker : public Tool {
  public:
    OpenMPLinker(const ToolChain &TC)
-       : Tool("NVPTX::OpenMPLinker", "nvlink", TC, RF_Full, llvm::sys::WEM_UTF8,
-              "--options-file") {}
+       : Tool("NVPTX::OpenMPLinker", "nvlink", TC) {}
 
    bool hasIntegratedCPP() const override { return false; }
 
diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp
index 3bf7f9cbc139b..2e1190c34ea7a 100644
--- a/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -148,7 +148,8 @@ void darwin::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
   // asm_final spec is empty.
 
   const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("as"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Exec, CmdArgs, Inputs));
 }
 
 void darwin::MachOTool::anchor() {}
@@ -202,16 +203,11 @@ static bool shouldLinkerNotDedup(bool IsLinkerOnlyAction, const ArgList &Args) {
 
 void darwin::Linker::AddLinkArgs(Compilation &C, const ArgList &Args,
                                  ArgStringList &CmdArgs,
-                                 const InputInfoList &Inputs) const {
+                                 const InputInfoList &Inputs,
+                                 unsigned Version[5]) const {
   const Driver &D = getToolChain().getDriver();
   const toolchains::MachO &MachOTC = getMachOToolChain();
 
-  unsigned Version[5] = {0, 0, 0, 0, 0};
-  if (Arg *A = Args.getLastArg(options::OPT_mlinker_version_EQ)) {
-    if (!Driver::GetReleaseVersion(A->getValue(), Version))
-      D.Diag(diag::err_drv_invalid_version_number) << A->getAsString(Args);
-  }
-
   // Newer linkers support -demangle. Pass it if supported and not disabled by
   // the user.
   if (Version[0] >= 100 && !Args.hasArg(options::OPT_Z_Xlinker__no_demangle))
@@ -525,13 +521,21 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     const char *Exec =
         Args.MakeArgString(getToolChain().GetProgramPath("touch"));
     CmdArgs.push_back(Output.getFilename());
-    C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, None));
+    C.addCommand(std::make_unique<Command>(
+        JA, *this, ResponseFileSupport::None(), Exec, CmdArgs, None));
     return;
   }
 
+  unsigned Version[5] = {0, 0, 0, 0, 0};
+  if (Arg *A = Args.getLastArg(options::OPT_mlinker_version_EQ)) {
+    if (!Driver::GetReleaseVersion(A->getValue(), Version))
+      getToolChain().getDriver().Diag(diag::err_drv_invalid_version_number)
+          << A->getAsString(Args);
+  }
+
   // I'm not sure why this particular decomposition exists in gcc, but
   // we follow suite for ease of comparison.
-  AddLinkArgs(C, Args, CmdArgs, Inputs);
+  AddLinkArgs(C, Args, CmdArgs, Inputs, Version);
 
   if (willEmitRemarks(Args) &&
       checkRemarksOptions(getToolChain().getDriver(), Args,
@@ -682,9 +686,16 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     }
   }
 
+  ResponseFileSupport ResponseSupport = ResponseFileSupport::AtFileUTF8();
+  if (Version[0] < 607) {
+    // For older versions of the linker, use the legacy filelist method instead.
+    ResponseSupport = {ResponseFileSupport::RF_FileList, llvm::sys::WEM_UTF8,
+                       "-filelist"};
+  }
+
   const char *Exec = Args.MakeArgString(getToolChain().GetLinkerPath());
-  std::unique_ptr<Command> Cmd =
-      std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs);
+  std::unique_ptr<Command> Cmd = std::make_unique<Command>(
+      JA, *this, ResponseSupport, Exec, CmdArgs, Inputs);
   Cmd->setInputFileList(std::move(InputFileList));
   C.addCommand(std::move(Cmd));
 }
@@ -708,7 +719,8 @@ void darwin::Lipo::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("lipo"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Exec, CmdArgs, Inputs));
 }
 
 void darwin::Dsymutil::ConstructJob(Compilation &C, const JobAction &JA,
@@ -728,7 +740,8 @@ void darwin::Dsymutil::ConstructJob(Compilation &C, const JobAction &JA,
 
   const char *Exec =
       Args.MakeArgString(getToolChain().GetProgramPath("dsymutil"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Exec, CmdArgs, Inputs));
 }
 
 void darwin::VerifyDebug::ConstructJob(Compilation &C, const JobAction &JA,
@@ -751,7 +764,8 @@ void darwin::VerifyDebug::ConstructJob(Compilation &C, const JobAction &JA,
 
   const char *Exec =
       Args.MakeArgString(getToolChain().GetProgramPath("dwarfdump"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Exec, CmdArgs, Inputs));
 }
 
 MachO::MachO(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
@@ -765,7 +779,7 @@ MachO::MachO(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
 /// Darwin - Darwin tool chain for i386 and x86_64.
 Darwin::Darwin(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
     : MachO(D, Triple, Args), TargetInitialized(false),
-      CudaInstallation(D, Triple, Args) {}
+      CudaInstallation(D, Triple, Args), RocmInstallation(D, Triple, Args) {}
 
 types::ID MachO::LookupTypeForExtension(StringRef Ext) const {
   types::ID Ty = ToolChain::LookupTypeForExtension(Ext);
@@ -817,6 +831,11 @@ void Darwin::AddCudaIncludeArgs(const ArgList &DriverArgs,
   CudaInstallation.AddCudaIncludeArgs(DriverArgs, CC1Args);
 }
 
+void Darwin::AddHIPIncludeArgs(const ArgList &DriverArgs,
+                               ArgStringList &CC1Args) const {
+  RocmInstallation.AddHIPIncludeArgs(DriverArgs, CC1Args);
+}
+
 // This is just a MachO name translation routine and there's no
 // way to join this into ARMTargetParser without breaking all
 // other assumptions. Maybe MachO should consider standardising
@@ -940,6 +959,10 @@ DarwinClang::DarwinClang(const Driver &D, const llvm::Triple &Triple,
     : Darwin(D, Triple, Args) {}
 
 void DarwinClang::addClangWarningOptions(ArgStringList &CC1Args) const {
+  // Always error about undefined 'TARGET_OS_*' macros.
+  CC1Args.push_back("-Wundef-prefix=TARGET_OS_");
+  CC1Args.push_back("-Werror=undef-prefix");
+
   // For modern targets, promote certain warnings to errors.
   if (isTargetWatchOSBased() || getTriple().isArch64Bit()) {
     // Always enable -Wdeprecated-objc-isa-usage and promote it
@@ -971,6 +994,8 @@ void DarwinClang::AddLinkARCArgs(const ArgList &Args,
   // Avoid linking compatibility stubs on i386 mac.
   if (isTargetMacOS() && getArch() == llvm::Triple::x86)
     return;
+  if (isTargetAppleSiliconMac())
+    return;
 
   ObjCRuntime runtime = getDefaultObjCRuntime(/*nonfragile*/ true);
 
@@ -1672,8 +1697,16 @@ inferDeploymentTargetFromArch(DerivedArgList &Args, const Darwin &Toolchain,
   llvm::Triple::OSType OSTy = llvm::Triple::UnknownOS;
 
   StringRef MachOArchName = Toolchain.getMachOArchName(Args);
-  if (MachOArchName == "armv7" || MachOArchName == "armv7s" ||
-      MachOArchName == "arm64")
+  if (MachOArchName == "arm64") {
+#if __arm64__
+    // A clang running on an Apple Silicon mac defaults
+    // to building for mac when building for arm64 rather than
+    // defaulting to iOS.
+    OSTy = llvm::Triple::MacOSX;
+#else
+    OSTy = llvm::Triple::IOS;
+#endif
+  } else if (MachOArchName == "armv7" || MachOArchName == "armv7s")
     OSTy = llvm::Triple::IOS;
   else if (MachOArchName == "armv7k" || MachOArchName == "arm64_32")
     OSTy = llvm::Triple::WatchOS;
@@ -1822,7 +1855,7 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
   if (Platform == MacOS) {
     if (!Driver::GetReleaseVersion(OSTarget->getOSVersion(), Major, Minor,
                                    Micro, HadExtra) ||
-        HadExtra || Major != 10 || Minor >= 100 || Micro >= 100)
+        HadExtra || Major < 10 || Major >= 100 || Minor >= 100 || Micro >= 100)
       getDriver().Diag(diag::err_drv_invalid_version_number)
           << OSTarget->getAsString(Args, Opts);
   } else if (Platform == IPhoneOS) {
@@ -2522,6 +2555,9 @@ void Darwin::addMinVersionArgs(const ArgList &Args,
     CmdArgs.push_back("-macosx_version_min");
   }
 
+  VersionTuple MinTgtVers = getEffectiveTriple().getMinimumSupportedOSVersion();
+  if (!MinTgtVers.empty() && MinTgtVers > TargetVersion)
+    TargetVersion = MinTgtVers;
   CmdArgs.push_back(Args.MakeArgString(TargetVersion.getAsString()));
 }
 
@@ -2554,6 +2590,9 @@ void Darwin::addPlatformVersionArgs(const llvm::opt::ArgList &Args,
     PlatformName += "-simulator";
   CmdArgs.push_back(Args.MakeArgString(PlatformName));
   VersionTuple TargetVersion = getTargetVersion().withoutBuild();
+  VersionTuple MinTgtVers = getEffectiveTriple().getMinimumSupportedOSVersion();
+  if (!MinTgtVers.empty() && MinTgtVers > TargetVersion)
+    TargetVersion = MinTgtVers;
   CmdArgs.push_back(Args.MakeArgString(TargetVersion.getAsString()));
   if (SDKInfo) {
     VersionTuple SDKVersion = SDKInfo->getVersion().withoutBuild();
@@ -2564,98 +2603,102 @@ void Darwin::addPlatformVersionArgs(const llvm::opt::ArgList &Args,
   }
 }
 
-void Darwin::addStartObjectFileArgs(const ArgList &Args,
-                                    ArgStringList &CmdArgs) const {
-  // Derived from startfile spec.
-  if (Args.hasArg(options::OPT_dynamiclib)) {
-    // Derived from darwin_dylib1 spec.
-    if (isTargetWatchOSBased()) {
-      ; // watchOS does not need dylib1.o.
-    } else if (isTargetIOSSimulator()) {
-      ; // iOS simulator does not need dylib1.o.
-    } else if (isTargetIPhoneOS()) {
-      if (isIPhoneOSVersionLT(3, 1))
-        CmdArgs.push_back("-ldylib1.o");
+// Add additional link args for the -dynamiclib option.
+static void addDynamicLibLinkArgs(const Darwin &D, const ArgList &Args,
+                                  ArgStringList &CmdArgs) {
+  // Derived from darwin_dylib1 spec.
+  if (D.isTargetIPhoneOS()) {
+    if (D.isIPhoneOSVersionLT(3, 1))
+      CmdArgs.push_back("-ldylib1.o");
+    return;
+  }
+
+  if (!D.isTargetMacOS())
+    return;
+  if (D.isMacosxVersionLT(10, 5))
+    CmdArgs.push_back("-ldylib1.o");
+  else if (D.isMacosxVersionLT(10, 6))
+    CmdArgs.push_back("-ldylib1.10.5.o");
+}
+
+// Add additional link args for the -bundle option.
+static void addBundleLinkArgs(const Darwin &D, const ArgList &Args,
+                              ArgStringList &CmdArgs) {
+  if (Args.hasArg(options::OPT_static))
+    return;
+  // Derived from darwin_bundle1 spec.
+  if ((D.isTargetIPhoneOS() && D.isIPhoneOSVersionLT(3, 1)) ||
+      (D.isTargetMacOS() && D.isMacosxVersionLT(10, 6)))
+    CmdArgs.push_back("-lbundle1.o");
+}
+
+// Add additional link args for the -pg option.
+static void addPgProfilingLinkArgs(const Darwin &D, const ArgList &Args,
+                                   ArgStringList &CmdArgs) {
+  if (D.isTargetMacOS() && D.isMacosxVersionLT(10, 9)) {
+    if (Args.hasArg(options::OPT_static) || Args.hasArg(options::OPT_object) ||
+        Args.hasArg(options::OPT_preload)) {
+      CmdArgs.push_back("-lgcrt0.o");
     } else {
-      if (isMacosxVersionLT(10, 5))
-        CmdArgs.push_back("-ldylib1.o");
-      else if (isMacosxVersionLT(10, 6))
-        CmdArgs.push_back("-ldylib1.10.5.o");
+      CmdArgs.push_back("-lgcrt1.o");
+
+      // darwin_crt2 spec is empty.
     }
+    // By default on OS X 10.8 and later, we don't link with a crt1.o
+    // file and the linker knows to use _main as the entry point.  But,
+    // when compiling with -pg, we need to link with the gcrt1.o file,
+    // so pass the -no_new_main option to tell the linker to use the
+    // "start" symbol as the entry point.
+    if (!D.isMacosxVersionLT(10, 8))
+      CmdArgs.push_back("-no_new_main");
   } else {
-    if (Args.hasArg(options::OPT_bundle)) {
-      if (!Args.hasArg(options::OPT_static)) {
-        // Derived from darwin_bundle1 spec.
-        if (isTargetWatchOSBased()) {
-          ; // watchOS does not need bundle1.o.
-        } else if (isTargetIOSSimulator()) {
-          ; // iOS simulator does not need bundle1.o.
-        } else if (isTargetIPhoneOS()) {
-          if (isIPhoneOSVersionLT(3, 1))
-            CmdArgs.push_back("-lbundle1.o");
-        } else {
-          if (isMacosxVersionLT(10, 6))
-            CmdArgs.push_back("-lbundle1.o");
-        }
-      }
-    } else {
-      if (Args.hasArg(options::OPT_pg) && SupportsProfiling()) {
-        if (isTargetMacOS() && isMacosxVersionLT(10, 9)) {
-          if (Args.hasArg(options::OPT_static) ||
-              Args.hasArg(options::OPT_object) ||
-              Args.hasArg(options::OPT_preload)) {
-            CmdArgs.push_back("-lgcrt0.o");
-          } else {
-            CmdArgs.push_back("-lgcrt1.o");
-
-            // darwin_crt2 spec is empty.
-          }
-          // By default on OS X 10.8 and later, we don't link with a crt1.o
-          // file and the linker knows to use _main as the entry point.  But,
-          // when compiling with -pg, we need to link with the gcrt1.o file,
-          // so pass the -no_new_main option to tell the linker to use the
-          // "start" symbol as the entry point.
-          if (isTargetMacOS() && !isMacosxVersionLT(10, 8))
-            CmdArgs.push_back("-no_new_main");
-        } else {
-          getDriver().Diag(diag::err_drv_clang_unsupported_opt_pg_darwin)
-              << isTargetMacOS();
-        }
-      } else {
-        if (Args.hasArg(options::OPT_static) ||
-            Args.hasArg(options::OPT_object) ||
-            Args.hasArg(options::OPT_preload)) {
-          CmdArgs.push_back("-lcrt0.o");
-        } else {
-          // Derived from darwin_crt1 spec.
-          if (isTargetWatchOSBased()) {
-            ; // watchOS does not need crt1.o.
-          } else if (isTargetIOSSimulator()) {
-            ; // iOS simulator does not need crt1.o.
-          } else if (isTargetIPhoneOS()) {
-            if (getArch() == llvm::Triple::aarch64)
-              ; // iOS does not need any crt1 files for arm64
-            else if (isIPhoneOSVersionLT(3, 1))
-              CmdArgs.push_back("-lcrt1.o");
-            else if (isIPhoneOSVersionLT(6, 0))
-              CmdArgs.push_back("-lcrt1.3.1.o");
-          } else {
-            if (isMacosxVersionLT(10, 5))
-              CmdArgs.push_back("-lcrt1.o");
-            else if (isMacosxVersionLT(10, 6))
-              CmdArgs.push_back("-lcrt1.10.5.o");
-            else if (isMacosxVersionLT(10, 8))
-              CmdArgs.push_back("-lcrt1.10.6.o");
-
-            // darwin_crt2 spec is empty.
-          }
-        }
-      }
-    }
+    D.getDriver().Diag(diag::err_drv_clang_unsupported_opt_pg_darwin)
+        << D.isTargetMacOS();
   }
+}
+
+static void addDefaultCRTLinkArgs(const Darwin &D, const ArgList &Args,
+                                  ArgStringList &CmdArgs) {
+  // Derived from darwin_crt1 spec.
+  if (D.isTargetIPhoneOS()) {
+    if (D.getArch() == llvm::Triple::aarch64)
+      ; // iOS does not need any crt1 files for arm64
+    else if (D.isIPhoneOSVersionLT(3, 1))
+      CmdArgs.push_back("-lcrt1.o");
+    else if (D.isIPhoneOSVersionLT(6, 0))
+      CmdArgs.push_back("-lcrt1.3.1.o");
+    return;
+  }
+
+  if (!D.isTargetMacOS())
+    return;
+  if (D.isMacosxVersionLT(10, 5))
+    CmdArgs.push_back("-lcrt1.o");
+  else if (D.isMacosxVersionLT(10, 6))
+    CmdArgs.push_back("-lcrt1.10.5.o");
+  else if (D.isMacosxVersionLT(10, 8))
+    CmdArgs.push_back("-lcrt1.10.6.o");
+  // darwin_crt2 spec is empty.
+}
+
+void Darwin::addStartObjectFileArgs(const ArgList &Args,
+                                    ArgStringList &CmdArgs) const {
+  // Derived from startfile spec.
+  if (Args.hasArg(options::OPT_dynamiclib))
+    addDynamicLibLinkArgs(*this, Args, CmdArgs);
+  else if (Args.hasArg(options::OPT_bundle))
+    addBundleLinkArgs(*this, Args, CmdArgs);
+  else if (Args.hasArg(options::OPT_pg) && SupportsProfiling())
+    addPgProfilingLinkArgs(*this, Args, CmdArgs);
+  else if (Args.hasArg(options::OPT_static) ||
+           Args.hasArg(options::OPT_object) ||
+           Args.hasArg(options::OPT_preload))
+    CmdArgs.push_back("-lcrt0.o");
+  else
+    addDefaultCRTLinkArgs(*this, Args, CmdArgs);
 
-  if (!isTargetIPhoneOS() && Args.hasArg(options::OPT_shared_libgcc) &&
-      !isTargetWatchOS() && isMacosxVersionLT(10, 5)) {
+  if (isTargetMacOS() && Args.hasArg(options::OPT_shared_libgcc) &&
+      isMacosxVersionLT(10, 5)) {
     const char *Str = Args.MakeArgString(GetFilePath("crt3.o"));
     CmdArgs.push_back(Str);
   }
@@ -2698,4 +2741,5 @@ SanitizerMask Darwin::getSupportedSanitizers() const {
 
 void Darwin::printVerboseInfo(raw_ostream &OS) const {
   CudaInstallation.print(OS);
+  RocmInstallation.print(OS);
 }
diff --git a/clang/lib/Driver/ToolChains/Darwin.h b/clang/lib/Driver/ToolChains/Darwin.h
index 1b193a4c4eb96..64c252efea7df 100644
--- a/clang/lib/Driver/ToolChains/Darwin.h
+++ b/clang/lib/Driver/ToolChains/Darwin.h
@@ -10,6 +10,7 @@
 #define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_DARWIN_H
 
 #include "Cuda.h"
+#include "ROCm.h"
 #include "clang/Driver/DarwinSDKInfo.h"
 #include "clang/Driver/Tool.h"
 #include "clang/Driver/ToolChain.h"
@@ -40,13 +41,8 @@ class LLVM_LIBRARY_VISIBILITY MachOTool : public Tool {
   }
 
 public:
-  MachOTool(
-      const char *Name, const char *ShortName, const ToolChain &TC,
-      ResponseFileSupport ResponseSupport = RF_None,
-      llvm::sys::WindowsEncodingMethod ResponseEncoding = llvm::sys::WEM_UTF8,
-      const char *ResponseFlag = "@")
-      : Tool(Name, ShortName, TC, ResponseSupport, ResponseEncoding,
-             ResponseFlag) {}
+  MachOTool(const char *Name, const char *ShortName, const ToolChain &TC)
+      : Tool(Name, ShortName, TC) {}
 };
 
 class LLVM_LIBRARY_VISIBILITY Assembler : public MachOTool {
@@ -66,12 +62,10 @@ class LLVM_LIBRARY_VISIBILITY Linker : public MachOTool {
   bool NeedsTempPath(const InputInfoList &Inputs) const;
   void AddLinkArgs(Compilation &C, const llvm::opt::ArgList &Args,
                    llvm::opt::ArgStringList &CmdArgs,
-                   const InputInfoList &Inputs) const;
+                   const InputInfoList &Inputs, unsigned Version[5]) const;
 
 public:
-  Linker(const ToolChain &TC)
-      : MachOTool("darwin::Linker", "linker", TC, RF_FileList,
-                  llvm::sys::WEM_UTF8, "-filelist") {}
+  Linker(const ToolChain &TC) : MachOTool("darwin::Linker", "linker", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
@@ -300,6 +294,7 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public MachO {
   mutable Optional<DarwinSDKInfo> SDKInfo;
 
   CudaInstallationDetector CudaInstallation;
+  RocmInstallationDetector RocmInstallation;
 
 private:
   void AddDeploymentTarget(llvm::opt::DerivedArgList &Args) const;
@@ -357,6 +352,7 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public MachO {
       const_cast<Darwin *>(this)->setTripleEnvironment(llvm::Triple::Simulator);
   }
 
+public:
   bool isTargetIPhoneOS() const {
     assert(TargetInitialized && "Target not initialized!");
     return (TargetPlatform == IPhoneOS || TargetPlatform == TvOS) &&
@@ -409,6 +405,17 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public MachO {
     return TargetPlatform == MacOS;
   }
 
+  bool isTargetMacOSBased() const {
+    assert(TargetInitialized && "Target not initialized!");
+    // FIXME (Alex L): Add remaining MacCatalyst suppport.
+    return TargetPlatform == MacOS;
+  }
+
+  bool isTargetAppleSiliconMac() const {
+    assert(TargetInitialized && "Target not initialized!");
+    return isTargetMacOSBased() && getArch() == llvm::Triple::aarch64;
+  }
+
   bool isTargetInitialized() const { return TargetInitialized; }
 
   VersionTuple getTargetVersion() const {
@@ -422,11 +429,20 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public MachO {
     return TargetVersion < VersionTuple(V0, V1, V2);
   }
 
+  /// Returns true if the minimum supported macOS version for the slice that's
+  /// being built is less than the specified version. If there's no minimum
+  /// supported macOS version, the deployment target version is compared to the
+  /// specifed version instead.
   bool isMacosxVersionLT(unsigned V0, unsigned V1 = 0, unsigned V2 = 0) const {
-    assert(isTargetMacOS() && "Unexpected call for non OS X target!");
-    return TargetVersion < VersionTuple(V0, V1, V2);
+    assert(isTargetMacOS() && getTriple().isMacOSX() &&
+           "Unexpected call for non OS X target!");
+    VersionTuple MinVers = getTriple().getMinimumSupportedOSVersion();
+    return (!MinVers.empty() && MinVers > TargetVersion
+                ? MinVers
+                : TargetVersion) < VersionTuple(V0, V1, V2);
   }
 
+protected:
   /// Return true if c++17 aligned allocation/deallocation functions are not
   /// implemented in the c++ standard library of the deployment target we are
   /// targeting.
@@ -461,6 +477,8 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public MachO {
 
   void AddCudaIncludeArgs(const llvm::opt::ArgList &DriverArgs,
                           llvm::opt::ArgStringList &CC1Args) const override;
+  void AddHIPIncludeArgs(const llvm::opt::ArgList &DriverArgs,
+                         llvm::opt::ArgStringList &CC1Args) const override;
 
   bool UseObjCMixedDispatch() const override {
     // This is only used with the non-fragile ABI and non-legacy dispatch.
diff --git a/clang/lib/Driver/ToolChains/DragonFly.cpp b/clang/lib/Driver/ToolChains/DragonFly.cpp
index 424331fbc6fe0..88dd0c899d8a8 100644
--- a/clang/lib/Driver/ToolChains/DragonFly.cpp
+++ b/clang/lib/Driver/ToolChains/DragonFly.cpp
@@ -45,7 +45,8 @@ void dragonfly::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(II.getFilename());
 
   const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("as"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 void dragonfly::Linker::ConstructJob(Compilation &C, const JobAction &JA,
@@ -169,7 +170,8 @@ void dragonfly::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   getToolChain().addProfileRTLibs(Args, CmdArgs);
 
   const char *Exec = Args.MakeArgString(getToolChain().GetLinkerPath());
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 /// DragonFly - DragonFly tool chain which can call as(1) and ld(1) directly.
diff --git a/clang/lib/Driver/ToolChains/DragonFly.h b/clang/lib/Driver/ToolChains/DragonFly.h
index 7e76904f10558..3ed5acefaefb5 100644
--- a/clang/lib/Driver/ToolChains/DragonFly.h
+++ b/clang/lib/Driver/ToolChains/DragonFly.h
@@ -18,10 +18,10 @@ namespace driver {
 namespace tools {
 /// dragonfly -- Directly call GNU Binutils assembler and linker
 namespace dragonfly {
-class LLVM_LIBRARY_VISIBILITY Assembler : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 public:
   Assembler(const ToolChain &TC)
-      : GnuTool("dragonfly::Assembler", "assembler", TC) {}
+      : Tool("dragonfly::Assembler", "assembler", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
 
@@ -31,9 +31,9 @@ class LLVM_LIBRARY_VISIBILITY Assembler : public GnuTool {
                     const char *LinkingOutput) const override;
 };
 
-class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC) : GnuTool("dragonfly::Linker", "linker", TC) {}
+  Linker(const ToolChain &TC) : Tool("dragonfly::Linker", "linker", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 9b9eb81fa111d..80f6db7ea6427 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -70,10 +70,10 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
 
   const auto& D = C.getDriver();
   const char* Exec = Args.MakeArgString(D.GetProgramPath("flang", TC));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileUTF8(), Exec, CmdArgs, Inputs));
 }
 
-Flang::Flang(const ToolChain &TC)
-    : Tool("flang", "flang frontend", TC, RF_Full) {}
+Flang::Flang(const ToolChain &TC) : Tool("flang", "flang frontend", TC) {}
 
 Flang::~Flang() {}
diff --git a/clang/lib/Driver/ToolChains/FreeBSD.cpp b/clang/lib/Driver/ToolChains/FreeBSD.cpp
index b2525e427cf95..909ac5e992129 100644
--- a/clang/lib/Driver/ToolChains/FreeBSD.cpp
+++ b/clang/lib/Driver/ToolChains/FreeBSD.cpp
@@ -128,7 +128,8 @@ void freebsd::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(II.getFilename());
 
   const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("as"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 void freebsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
@@ -358,7 +359,8 @@ void freebsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   ToolChain.addProfileRTLibs(Args, CmdArgs);
 
   const char *Exec = Args.MakeArgString(getToolChain().GetLinkerPath());
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 /// FreeBSD - FreeBSD tool chain which can call as(1) and ld(1) directly.
@@ -423,6 +425,11 @@ void FreeBSD::AddCudaIncludeArgs(const ArgList &DriverArgs,
   CudaInstallation.AddCudaIncludeArgs(DriverArgs, CC1Args);
 }
 
+void FreeBSD::AddHIPIncludeArgs(const ArgList &DriverArgs,
+                                ArgStringList &CC1Args) const {
+  RocmInstallation.AddHIPIncludeArgs(DriverArgs, CC1Args);
+}
+
 Tool *FreeBSD::buildAssembler() const {
   return new tools::freebsd::Assembler(*this);
 }
diff --git a/clang/lib/Driver/ToolChains/FreeBSD.h b/clang/lib/Driver/ToolChains/FreeBSD.h
index 7e13f48b7167a..abc0876cef260 100644
--- a/clang/lib/Driver/ToolChains/FreeBSD.h
+++ b/clang/lib/Driver/ToolChains/FreeBSD.h
@@ -19,10 +19,10 @@ namespace tools {
 
 /// freebsd -- Directly call GNU Binutils assembler and linker
 namespace freebsd {
-class LLVM_LIBRARY_VISIBILITY Assembler : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 public:
   Assembler(const ToolChain &TC)
-      : GnuTool("freebsd::Assembler", "assembler", TC) {}
+      : Tool("freebsd::Assembler", "assembler", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
 
@@ -32,9 +32,9 @@ class LLVM_LIBRARY_VISIBILITY Assembler : public GnuTool {
                     const char *LinkingOutput) const override;
 };
 
-class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC) : GnuTool("freebsd::Linker", "linker", TC) {}
+  Linker(const ToolChain &TC) : Tool("freebsd::Linker", "linker", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
@@ -68,6 +68,8 @@ class LLVM_LIBRARY_VISIBILITY FreeBSD : public Generic_ELF {
                            llvm::opt::ArgStringList &CmdArgs) const override;
   void AddCudaIncludeArgs(const llvm::opt::ArgList &DriverArgs,
                           llvm::opt::ArgStringList &CC1Args) const override;
+  void AddHIPIncludeArgs(const llvm::opt::ArgList &DriverArgs,
+                         llvm::opt::ArgStringList &CC1Args) const override;
 
   llvm::ExceptionHandling
   GetExceptionModel(const llvm::opt::ArgList &Args) const override;
diff --git a/clang/lib/Driver/ToolChains/Fuchsia.cpp b/clang/lib/Driver/ToolChains/Fuchsia.cpp
index 5ac10e37176c0..94e025e3055a3 100644
--- a/clang/lib/Driver/ToolChains/Fuchsia.cpp
+++ b/clang/lib/Driver/ToolChains/Fuchsia.cpp
@@ -164,7 +164,8 @@ void fuchsia::Linker::ConstructJob(Compilation &C, const JobAction &JA,
       CmdArgs.push_back("-lc");
   }
 
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Exec, CmdArgs, Inputs));
 }
 
 /// Fuchsia - Fuchsia tool chain which can call as(1) and ld(1) directly.
diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp
index 93a285d42a08e..c8a7fce07ef1c 100644
--- a/clang/lib/Driver/ToolChains/Gnu.cpp
+++ b/clang/lib/Driver/ToolChains/Gnu.cpp
@@ -37,8 +37,6 @@ using namespace llvm::opt;
 using tools::addMultilibFlag;
 using tools::addPathIfExists;
 
-void tools::GnuTool::anchor() {}
-
 static bool forwardToGCC(const Option &O) {
   // Don't forward inputs from the original command line.  They are added from
   // InputInfoList.
@@ -190,7 +188,8 @@ void tools::gcc::Common::ConstructJob(Compilation &C, const JobAction &JA,
     GCCName = "gcc";
 
   const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath(GCCName));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 void tools::gcc::Preprocessor::RenderExtraToolArgs(
@@ -305,6 +304,8 @@ static const char *getLDMOption(const llvm::Triple &T, const ArgList &Args) {
     if (T.getEnvironment() == llvm::Triple::GNUX32)
       return "elf32_x86_64";
     return "elf_x86_64";
+  case llvm::Triple::ve:
+    return "elf64ve";
   default:
     return nullptr;
   }
@@ -341,6 +342,44 @@ static bool getStatic(const ArgList &Args) {
       !Args.hasArg(options::OPT_static_pie);
 }
 
+void tools::gnutools::StaticLibTool::ConstructJob(
+    Compilation &C, const JobAction &JA, const InputInfo &Output,
+    const InputInfoList &Inputs, const ArgList &Args,
+    const char *LinkingOutput) const {
+  const Driver &D = getToolChain().getDriver();
+
+  // Silence warning for "clang -g foo.o -o foo"
+  Args.ClaimAllArgs(options::OPT_g_Group);
+  // and "clang -emit-llvm foo.o -o foo"
+  Args.ClaimAllArgs(options::OPT_emit_llvm);
+  // and for "clang -w foo.o -o foo". Other warning options are already
+  // handled somewhere else.
+  Args.ClaimAllArgs(options::OPT_w);
+  // Silence warnings when linking C code with a C++ '-stdlib' argument.
+  Args.ClaimAllArgs(options::OPT_stdlib_EQ);
+
+  // GNU ar tool command "ar <options> <output_file> <input_files>".
+  ArgStringList CmdArgs;
+  // Create and insert file members with a deterministic index.
+  CmdArgs.push_back("rcsD");
+  CmdArgs.push_back(Output.getFilename());
+  AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs, JA);
+
+  // Delete old output archive file if it already exists before generating a new
+  // archive file.
+  auto OutputFileName = Output.getFilename();
+  if (Output.isFilename() && llvm::sys::fs::exists(OutputFileName)) {
+    if (std::error_code EC = llvm::sys::fs::remove(OutputFileName)) {
+      D.Diag(diag::err_drv_unable_to_remove_file) << EC.message();
+      return;
+    }
+  }
+
+  const char *Exec = Args.MakeArgString(getToolChain().GetStaticLibToolPath());
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
+}
+
 void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
                                            const InputInfo &Output,
                                            const InputInfoList &Inputs,
@@ -359,6 +398,7 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   const llvm::Triple::ArchType Arch = ToolChain.getArch();
   const bool isAndroid = ToolChain.getTriple().isAndroid();
   const bool IsIAMCU = ToolChain.getTriple().isOSIAMCU();
+  const bool IsVE = ToolChain.getTriple().isVE();
   const bool IsPIE = getPIE(Args, ToolChain);
   const bool IsStaticPIE = getStaticPIE(Args, ToolChain);
   const bool IsStatic = getStatic(Args);
@@ -477,6 +517,11 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
       CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crti.o")));
     }
 
+    if (IsVE) {
+      CmdArgs.push_back("-z");
+      CmdArgs.push_back("max-page-size=0x4000000");
+    }
+
     if (IsIAMCU)
       CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crt0.o")));
     else if (HasCRTBeginEndFiles) {
@@ -625,14 +670,11 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     }
   }
 
-  // Add HIP offloading linker script args if required.
-  AddHIPLinkerScript(getToolChain(), C, Output, Inputs, Args, CmdArgs, JA,
-                     *this);
-
   Args.AddAllArgs(CmdArgs, options::OPT_T);
 
   const char *Exec = Args.MakeArgString(ToolChain.GetLinkerPath());
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 void tools::gnutools::Assembler::ConstructJob(Compilation &C,
@@ -650,6 +692,7 @@ void tools::gnutools::Assembler::ConstructJob(Compilation &C,
   llvm::Reloc::Model RelocationModel;
   unsigned PICLevel;
   bool IsPIE;
+  const char *DefaultAssembler = "as";
   std::tie(RelocationModel, PICLevel, IsPIE) =
       ParsePICArgs(getToolChain(), Args);
 
@@ -870,6 +913,8 @@ void tools::gnutools::Assembler::ConstructJob(Compilation &C,
     CmdArgs.push_back(Args.MakeArgString("-march=" + CPUName));
     break;
   }
+  case llvm::Triple::ve:
+    DefaultAssembler = "nas";
   }
 
   for (const Arg *A : Args.filtered(options::OPT_ffile_prefix_map_EQ,
@@ -894,8 +939,10 @@ void tools::gnutools::Assembler::ConstructJob(Compilation &C,
   for (const auto &II : Inputs)
     CmdArgs.push_back(II.getFilename());
 
-  const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("as"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  const char *Exec =
+      Args.MakeArgString(getToolChain().GetProgramPath(DefaultAssembler));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 
   // Handle the debug info splitting at object creation time if we're
   // creating an object.
@@ -2611,6 +2658,7 @@ void Generic_GCC::printVerboseInfo(raw_ostream &OS) const {
   // Print the information about how we detected the GCC installation.
   GCCInstallation.print(OS);
   CudaInstallation.print(OS);
+  RocmInstallation.print(OS);
 }
 
 bool Generic_GCC::IsUnwindTablesDefault(const ArgList &Args) const {
diff --git a/clang/lib/Driver/ToolChains/Gnu.h b/clang/lib/Driver/ToolChains/Gnu.h
index 8ef9b4fdb6cd9..52690ab4b83c2 100644
--- a/clang/lib/Driver/ToolChains/Gnu.h
+++ b/clang/lib/Driver/ToolChains/Gnu.h
@@ -36,23 +36,26 @@ bool findMIPSMultilibs(const Driver &D, const llvm::Triple &TargetTriple,
 
 namespace tools {
 
-/// Base class for all GNU tools that provide the same behavior when
-/// it comes to response files support
-class LLVM_LIBRARY_VISIBILITY GnuTool : public Tool {
-  virtual void anchor();
-
+/// Directly call GNU Binutils' assembler and linker.
+namespace gnutools {
+class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 public:
-  GnuTool(const char *Name, const char *ShortName, const ToolChain &TC)
-      : Tool(Name, ShortName, TC, RF_Full, llvm::sys::WEM_CurrentCodePage) {}
+  Assembler(const ToolChain &TC) : Tool("GNU::Assembler", "assembler", TC) {}
+
+  bool hasIntegratedCPP() const override { return false; }
+
+  void ConstructJob(Compilation &C, const JobAction &JA,
+                    const InputInfo &Output, const InputInfoList &Inputs,
+                    const llvm::opt::ArgList &TCArgs,
+                    const char *LinkingOutput) const override;
 };
 
-/// Directly call GNU Binutils' assembler and linker.
-namespace gnutools {
-class LLVM_LIBRARY_VISIBILITY Assembler : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Assembler(const ToolChain &TC) : GnuTool("GNU::Assembler", "assembler", TC) {}
+  Linker(const ToolChain &TC) : Tool("GNU::Linker", "linker", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
+  bool isLinkJob() const override { return true; }
 
   void ConstructJob(Compilation &C, const JobAction &JA,
                     const InputInfo &Output, const InputInfoList &Inputs,
@@ -60,9 +63,10 @@ class LLVM_LIBRARY_VISIBILITY Assembler : public GnuTool {
                     const char *LinkingOutput) const override;
 };
 
-class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY StaticLibTool : public Tool {
 public:
-  Linker(const ToolChain &TC) : GnuTool("GNU::Linker", "linker", TC) {}
+  StaticLibTool(const ToolChain &TC)
+      : Tool("GNU::StaticLibTool", "static-lib-linker", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
@@ -76,10 +80,10 @@ class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
 
 /// gcc - Generic GCC tool implementations.
 namespace gcc {
-class LLVM_LIBRARY_VISIBILITY Common : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Common : public Tool {
 public:
   Common(const char *Name, const char *ShortName, const ToolChain &TC)
-      : GnuTool(Name, ShortName, TC) {}
+      : Tool(Name, ShortName, TC) {}
 
   // A gcc tool has an "integrated" assembler that it will call to produce an
   // object. Let it use that assembler so that we don't have to deal with
diff --git a/clang/lib/Driver/ToolChains/HIP.cpp b/clang/lib/Driver/ToolChains/HIP.cpp
index b9143f98f1526..7d17f809690ea 100644
--- a/clang/lib/Driver/ToolChains/HIP.cpp
+++ b/clang/lib/Driver/ToolChains/HIP.cpp
@@ -57,18 +57,43 @@ void AMDGCN::Linker::constructLldCommand(Compilation &C, const JobAction &JA,
                                           const llvm::opt::ArgList &Args) const {
   // Construct lld command.
   // The output from ld.lld is an HSA code object file.
-  ArgStringList LldArgs{"-flavor",
-                        "gnu",
-                        "--no-undefined",
-                        "-shared",
-                        "-mllvm",
-                        "-amdgpu-internalize-symbols",
-                        "-o",
-                        Output.getFilename()};
+  ArgStringList LldArgs{"-flavor", "gnu", "--no-undefined", "-shared",
+                        "-plugin-opt=-amdgpu-internalize-symbols"};
+
+  auto &TC = getToolChain();
+  auto &D = TC.getDriver();
+  assert(!Inputs.empty() && "Must have at least one input.");
+  addLTOOptions(TC, Args, LldArgs, Output, Inputs[0],
+                D.getLTOMode() == LTOK_Thin);
+
+  // Extract all the -m options
+  std::vector<llvm::StringRef> Features;
+  amdgpu::getAMDGPUTargetFeatures(D, Args, Features);
+
+  // Add features to mattr such as cumode
+  std::string MAttrString = "-plugin-opt=-mattr=";
+  for (auto OneFeature : unifyTargetFeatures(Features)) {
+    MAttrString.append(Args.MakeArgString(OneFeature));
+    if (OneFeature != Features.back())
+      MAttrString.append(",");
+  }
+  if (!Features.empty())
+    LldArgs.push_back(Args.MakeArgString(MAttrString));
+
+  for (const Arg *A : Args.filtered(options::OPT_mllvm)) {
+    LldArgs.push_back(
+        Args.MakeArgString(Twine("-plugin-opt=") + A->getValue(0)));
+  }
+
+  if (C.getDriver().isSaveTempsEnabled())
+    LldArgs.push_back("-save-temps");
+
+  LldArgs.append({"-o", Output.getFilename()});
   for (auto Input : Inputs)
     LldArgs.push_back(Input.getFilename());
   const char *Lld = Args.MakeArgString(getToolChain().GetProgramPath("lld"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Lld, LldArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Lld, LldArgs, Inputs));
 }
 
 // Construct a clang-offload-bundler command to bundle code objects for
@@ -101,7 +126,78 @@ void AMDGCN::constructHIPFatbinCommand(Compilation &C, const JobAction &JA,
 
   const char *Bundler = Args.MakeArgString(
       T.getToolChain().GetProgramPath("clang-offload-bundler"));
-  C.addCommand(std::make_unique<Command>(JA, T, Bundler, BundlerArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, T, ResponseFileSupport::None(),
+                                         Bundler, BundlerArgs, Inputs));
+}
+
+/// Add Generated HIP Object File which has device images embedded into the
+/// host to the argument list for linking. Using MC directives, embed the
+/// device code and also define symbols required by the code generation so that
+/// the image can be retrieved at runtime.
+void AMDGCN::Linker::constructGenerateObjFileFromHIPFatBinary(
+    Compilation &C, const InputInfo &Output,
+    const InputInfoList &Inputs, const ArgList &Args,
+    const JobAction &JA) const {
+  const ToolChain &TC = getToolChain();
+  std::string Name =
+      std::string(llvm::sys::path::stem(Output.getFilename()));
+
+  // Create Temp Object File Generator,
+  // Offload Bundled file and Bundled Object file.
+  // Keep them if save-temps is enabled.
+  const char *McinFile;
+  const char *BundleFile;
+  if (C.getDriver().isSaveTempsEnabled()) {
+    McinFile = C.getArgs().MakeArgString(Name + ".mcin");
+    BundleFile = C.getArgs().MakeArgString(Name + ".hipfb");
+  } else {
+    auto TmpNameMcin = C.getDriver().GetTemporaryPath(Name, "mcin");
+    McinFile = C.addTempFile(C.getArgs().MakeArgString(TmpNameMcin));
+    auto TmpNameFb = C.getDriver().GetTemporaryPath(Name, "hipfb");
+    BundleFile = C.addTempFile(C.getArgs().MakeArgString(TmpNameFb));
+  }
+  constructHIPFatbinCommand(C, JA, BundleFile, Inputs, Args, *this);
+
+  // Create a buffer to write the contents of the temp obj generator.
+  std::string ObjBuffer;
+  llvm::raw_string_ostream ObjStream(ObjBuffer);
+
+  // Add MC directives to embed target binaries. We ensure that each
+  // section and image is 16-byte aligned. This is not mandatory, but
+  // increases the likelihood of data to be aligned with a cache block
+  // in several main host machines.
+  ObjStream << "#       HIP Object Generator\n";
+  ObjStream << "# *** Automatically generated by Clang ***\n";
+  ObjStream << "  .type __hip_fatbin,@object\n";
+  ObjStream << "  .section .hip_fatbin,\"aMS\",@progbits,1\n";
+  ObjStream << "  .data\n";
+  ObjStream << "  .globl __hip_fatbin\n";
+  ObjStream << "  .p2align 3\n";
+  ObjStream << "__hip_fatbin:\n";
+  ObjStream << "  .incbin \"" << BundleFile << "\"\n";
+  ObjStream.flush();
+
+  // Dump the contents of the temp object file gen if the user requested that.
+  // We support this option to enable testing of behavior with -###.
+  if (C.getArgs().hasArg(options::OPT_fhip_dump_offload_linker_script))
+    llvm::errs() << ObjBuffer;
+
+  // Open script file and write the contents.
+  std::error_code EC;
+  llvm::raw_fd_ostream Objf(McinFile, EC, llvm::sys::fs::OF_None);
+
+  if (EC) {
+    C.getDriver().Diag(clang::diag::err_unable_to_make_temp) << EC.message();
+    return;
+  }
+
+  Objf << ObjBuffer;
+
+  ArgStringList McArgs{"-o",      Output.getFilename(),
+                       McinFile,  "--filetype=obj"};
+  const char *Mc = Args.MakeArgString(TC.GetProgramPath("llvm-mc"));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Mc, McArgs, Inputs));
 }
 
 // For amdgcn the inputs of the linker job are device bitcode and output is
@@ -111,6 +207,10 @@ void AMDGCN::Linker::ConstructJob(Compilation &C, const JobAction &JA,
                                    const InputInfoList &Inputs,
                                    const ArgList &Args,
                                    const char *LinkingOutput) const {
+  if (Inputs.size() > 0 &&
+      Inputs[0].getType() == types::TY_Image &&
+      JA.getType() == types::TY_Object)
+    return constructGenerateObjFileFromHIPFatBinary(C, Output, Inputs, Args, JA);
 
   if (JA.getType() == types::TY_HIP_FATBIN)
     return constructHIPFatbinCommand(C, JA, Output.getFilename(), Inputs, Args, *this);
@@ -179,8 +279,7 @@ void HIPToolChain::addClangTargetOptions(
   ArgStringList LibraryPaths;
 
   // Find in --hip-device-lib-path and HIP_LIBRARY_PATH.
-  for (auto Path :
-       DriverArgs.getAllArgValues(options::OPT_hip_device_lib_path_EQ))
+  for (auto Path : RocmInstallation.getRocmDeviceLibPathArg())
     LibraryPaths.push_back(DriverArgs.MakeArgString(Path));
 
   addDirectoryList(DriverArgs, LibraryPaths, "", "HIP_DEVICE_LIB_PATH");
@@ -191,14 +290,14 @@ void HIPToolChain::addClangTargetOptions(
     for (auto Lib : BCLibs)
       addBCLib(getDriver(), DriverArgs, CC1Args, LibraryPaths, Lib);
   } else {
-    if (!RocmInstallation.isValid()) {
-      getDriver().Diag(diag::err_drv_no_rocm_installation);
+    if (!RocmInstallation.hasDeviceLibrary()) {
+      getDriver().Diag(diag::err_drv_no_rocm_device_lib) << 0;
       return;
     }
 
     std::string LibDeviceFile = RocmInstallation.getLibDeviceFile(CanonArch);
     if (LibDeviceFile.empty()) {
-      getDriver().Diag(diag::err_drv_no_rocm_device_lib) << GpuArch;
+      getDriver().Diag(diag::err_drv_no_rocm_device_lib) << 1 << GpuArch;
       return;
     }
 
diff --git a/clang/lib/Driver/ToolChains/HIP.h b/clang/lib/Driver/ToolChains/HIP.h
index fa6984ed9f52f..5e2be7138579a 100644
--- a/clang/lib/Driver/ToolChains/HIP.h
+++ b/clang/lib/Driver/ToolChains/HIP.h
@@ -42,6 +42,13 @@ class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
   void constructLldCommand(Compilation &C, const JobAction &JA,
                            const InputInfoList &Inputs, const InputInfo &Output,
                            const llvm::opt::ArgList &Args) const;
+
+  // Construct command for creating Object from HIP fatbin.
+  void constructGenerateObjFileFromHIPFatBinary(Compilation &C,
+                                                const InputInfo &Output,
+                                                const InputInfoList &Inputs,
+                                                const llvm::opt::ArgList &Args,
+                                                const JobAction &JA) const;
 };
 
 } // end namespace AMDGCN
diff --git a/clang/lib/Driver/ToolChains/Hexagon.cpp b/clang/lib/Driver/ToolChains/Hexagon.cpp
index cf0a0335fbdf7..775f6e1094fa6 100644
--- a/clang/lib/Driver/ToolChains/Hexagon.cpp
+++ b/clang/lib/Driver/ToolChains/Hexagon.cpp
@@ -189,7 +189,8 @@ void hexagon::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   auto *Exec = Args.MakeArgString(HTC.GetProgramPath(AsName));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 void hexagon::Linker::RenderExtraToolArgs(const JobAction &JA,
@@ -406,7 +407,8 @@ void hexagon::Linker::ConstructJob(Compilation &C, const JobAction &JA,
                            LinkingOutput);
 
   const char *Exec = Args.MakeArgString(HTC.GetLinkerPath());
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 // Hexagon tools end.
 
diff --git a/clang/lib/Driver/ToolChains/Hexagon.h b/clang/lib/Driver/ToolChains/Hexagon.h
index 7b2af9a799ccf..c32cb7f09591a 100644
--- a/clang/lib/Driver/ToolChains/Hexagon.h
+++ b/clang/lib/Driver/ToolChains/Hexagon.h
@@ -20,10 +20,10 @@ namespace hexagon {
 // For Hexagon, we do not need to instantiate tools for PreProcess, PreCompile
 // and Compile.
 // We simply use "clang -cc1" for those actions.
-class LLVM_LIBRARY_VISIBILITY Assembler : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 public:
   Assembler(const ToolChain &TC)
-      : GnuTool("hexagon::Assembler", "hexagon-as", TC) {}
+      : Tool("hexagon::Assembler", "hexagon-as", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
 
@@ -35,9 +35,9 @@ class LLVM_LIBRARY_VISIBILITY Assembler : public GnuTool {
                     const char *LinkingOutput) const override;
 };
 
-class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC) : GnuTool("hexagon::Linker", "hexagon-ld", TC) {}
+  Linker(const ToolChain &TC) : Tool("hexagon::Linker", "hexagon-ld", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
diff --git a/clang/lib/Driver/ToolChains/InterfaceStubs.cpp b/clang/lib/Driver/ToolChains/InterfaceStubs.cpp
index 8f947e79bd1f1..f7c11421e8094 100644
--- a/clang/lib/Driver/ToolChains/InterfaceStubs.cpp
+++ b/clang/lib/Driver/ToolChains/InterfaceStubs.cpp
@@ -54,8 +54,9 @@ void Merger::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Args.MakeArgString(InputFilename.c_str()));
   }
 
-  C.addCommand(std::make_unique<Command>(JA, *this, Args.MakeArgString(Merger),
-                                         CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Args.MakeArgString(Merger), CmdArgs,
+                                         Inputs));
 }
 } // namespace ifstool
 } // namespace tools
diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp
index 222c351016a79..180350476c38b 100644
--- a/clang/lib/Driver/ToolChains/Linux.cpp
+++ b/clang/lib/Driver/ToolChains/Linux.cpp
@@ -363,6 +363,10 @@ bool Linux::HasNativeLLVMSupport() const { return true; }
 
 Tool *Linux::buildLinker() const { return new tools::gnutools::Linker(*this); }
 
+Tool *Linux::buildStaticLibTool() const {
+  return new tools::gnutools::StaticLibTool(*this);
+}
+
 Tool *Linux::buildAssembler() const {
   return new tools::gnutools::Assembler(*this);
 }
@@ -537,6 +541,8 @@ std::string Linux::getDynamicLinker(const ArgList &Args) const {
     Loader = X32 ? "ld-linux-x32.so.2" : "ld-linux-x86-64.so.2";
     break;
   }
+  case llvm::Triple::ve:
+    return "/opt/nec/ve/lib/ld-linux-ve.so.1";
   }
 
   if (Distro == Distro::Exherbo &&
diff --git a/clang/lib/Driver/ToolChains/Linux.h b/clang/lib/Driver/ToolChains/Linux.h
index fba5c24441e48..6b16b0e64990d 100644
--- a/clang/lib/Driver/ToolChains/Linux.h
+++ b/clang/lib/Driver/ToolChains/Linux.h
@@ -57,6 +57,7 @@ class LLVM_LIBRARY_VISIBILITY Linux : public Generic_ELF {
 protected:
   Tool *buildAssembler() const override;
   Tool *buildLinker() const override;
+  Tool *buildStaticLibTool() const override;
 
   std::string getMultiarchTriple(const Driver &D,
                                  const llvm::Triple &TargetTriple,
diff --git a/clang/lib/Driver/ToolChains/MSP430.cpp b/clang/lib/Driver/ToolChains/MSP430.cpp
index 1c507655541f8..b0bc2e014b487 100644
--- a/clang/lib/Driver/ToolChains/MSP430.cpp
+++ b/clang/lib/Driver/ToolChains/MSP430.cpp
@@ -227,6 +227,7 @@ void msp430::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   }
   CmdArgs.push_back("-o");
   CmdArgs.push_back(Output.getFilename());
-  C.addCommand(std::make_unique<Command>(JA, *this, Args.MakeArgString(Linker),
-                                          CmdArgs, Inputs));
+  C.addCommand(
+      std::make_unique<Command>(JA, *this, ResponseFileSupport::AtFileCurCP(),
+                                Args.MakeArgString(Linker), CmdArgs, Inputs));
 }
diff --git a/clang/lib/Driver/ToolChains/MSP430.h b/clang/lib/Driver/ToolChains/MSP430.h
index ff5002b8b553d..58fd158cd12f9 100644
--- a/clang/lib/Driver/ToolChains/MSP430.h
+++ b/clang/lib/Driver/ToolChains/MSP430.h
@@ -52,10 +52,9 @@ class LLVM_LIBRARY_VISIBILITY MSP430ToolChain : public Generic_ELF {
 namespace tools {
 namespace msp430 {
 
-class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC)
-      : GnuTool("MSP430::Linker", "msp430-elf-ld", TC) {}
+  Linker(const ToolChain &TC) : Tool("MSP430::Linker", "msp430-elf-ld", TC) {}
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
   void ConstructJob(Compilation &C, const JobAction &JA,
diff --git a/clang/lib/Driver/ToolChains/MSVC.cpp b/clang/lib/Driver/ToolChains/MSVC.cpp
index 8271ca780f801..6b3c00e2ab6d5 100644
--- a/clang/lib/Driver/ToolChains/MSVC.cpp
+++ b/clang/lib/Driver/ToolChains/MSVC.cpp
@@ -592,8 +592,9 @@ void visualstudio::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     linkPath = TC.GetProgramPath(Linker.str().c_str());
   }
 
-  auto LinkCmd = std::make_unique<Command>(
-      JA, *this, Args.MakeArgString(linkPath), CmdArgs, Inputs);
+  auto LinkCmd =
+      std::make_unique<Command>(JA, *this, ResponseFileSupport::AtFileUTF16(),
+                                Args.MakeArgString(linkPath), CmdArgs, Inputs);
   if (!Environment.empty())
     LinkCmd->setEnvironment(Environment);
   C.addCommand(std::move(LinkCmd));
@@ -733,8 +734,9 @@ std::unique_ptr<Command> visualstudio::Compiler::GetCommand(
   CmdArgs.push_back(Fo);
 
   std::string Exec = FindVisualStudioExecutable(getToolChain(), "cl.exe");
-  return std::make_unique<Command>(JA, *this, Args.MakeArgString(Exec),
-                                    CmdArgs, Inputs);
+  return std::make_unique<Command>(JA, *this,
+                                   ResponseFileSupport::AtFileUTF16(),
+                                   Args.MakeArgString(Exec), CmdArgs, Inputs);
 }
 
 MSVCToolChain::MSVCToolChain(const Driver &D, const llvm::Triple &Triple,
@@ -805,6 +807,7 @@ void MSVCToolChain::AddHIPIncludeArgs(const ArgList &DriverArgs,
 
 void MSVCToolChain::printVerboseInfo(raw_ostream &OS) const {
   CudaInstallation.print(OS);
+  RocmInstallation.print(OS);
 }
 
 // Windows SDKs and VC Toolchains group their contents into subdirectories based
diff --git a/clang/lib/Driver/ToolChains/MSVC.h b/clang/lib/Driver/ToolChains/MSVC.h
index 85208eaa3cc31..dba99ed77246c 100644
--- a/clang/lib/Driver/ToolChains/MSVC.h
+++ b/clang/lib/Driver/ToolChains/MSVC.h
@@ -24,9 +24,7 @@ namespace tools {
 namespace visualstudio {
 class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC)
-      : Tool("visualstudio::Linker", "linker", TC, RF_Full,
-             llvm::sys::WEM_UTF16) {}
+  Linker(const ToolChain &TC) : Tool("visualstudio::Linker", "linker", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
@@ -40,8 +38,7 @@ class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 class LLVM_LIBRARY_VISIBILITY Compiler : public Tool {
 public:
   Compiler(const ToolChain &TC)
-      : Tool("visualstudio::Compiler", "compiler", TC, RF_Full,
-             llvm::sys::WEM_UTF16) {}
+      : Tool("visualstudio::Compiler", "compiler", TC) {}
 
   bool hasIntegratedAssembler() const override { return true; }
   bool hasIntegratedCPP() const override { return true; }
diff --git a/clang/lib/Driver/ToolChains/MinGW.cpp b/clang/lib/Driver/ToolChains/MinGW.cpp
index 3c2ba40323d92..a1a1b413fb6c6 100644
--- a/clang/lib/Driver/ToolChains/MinGW.cpp
+++ b/clang/lib/Driver/ToolChains/MinGW.cpp
@@ -50,7 +50,8 @@ void tools::MinGW::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(II.getFilename());
 
   const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("as"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Exec, CmdArgs, Inputs));
 
   if (Args.hasArg(options::OPT_gsplit_dwarf))
     SplitDebugInfo(getToolChain(), C, *this, JA, Args, Output,
@@ -321,7 +322,8 @@ void tools::MinGW::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     }
   }
   const char *Exec = Args.MakeArgString(TC.GetLinkerPath());
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileUTF8(), Exec, CmdArgs, Inputs));
 }
 
 // Simplified from Generic_GCC::GCCInstallationDetector::ScanLibDirForGCCTriple.
@@ -396,7 +398,8 @@ llvm::ErrorOr<std::string> toolchains::MinGW::findClangRelativeSysroot() {
 
 toolchains::MinGW::MinGW(const Driver &D, const llvm::Triple &Triple,
                          const ArgList &Args)
-    : ToolChain(D, Triple, Args), CudaInstallation(D, Triple, Args) {
+    : ToolChain(D, Triple, Args), CudaInstallation(D, Triple, Args),
+      RocmInstallation(D, Triple, Args) {
   getProgramPaths().push_back(getDriver().getInstalledDir());
 
   if (getDriver().SysRoot.size())
@@ -498,8 +501,14 @@ void toolchains::MinGW::AddCudaIncludeArgs(const ArgList &DriverArgs,
   CudaInstallation.AddCudaIncludeArgs(DriverArgs, CC1Args);
 }
 
+void toolchains::MinGW::AddHIPIncludeArgs(const ArgList &DriverArgs,
+                                          ArgStringList &CC1Args) const {
+  RocmInstallation.AddHIPIncludeArgs(DriverArgs, CC1Args);
+}
+
 void toolchains::MinGW::printVerboseInfo(raw_ostream &OS) const {
   CudaInstallation.print(OS);
+  RocmInstallation.print(OS);
 }
 
 // Include directories for various hosts:
diff --git a/clang/lib/Driver/ToolChains/MinGW.h b/clang/lib/Driver/ToolChains/MinGW.h
index 6752a405be879..2f1559fcf34cd 100644
--- a/clang/lib/Driver/ToolChains/MinGW.h
+++ b/clang/lib/Driver/ToolChains/MinGW.h
@@ -11,8 +11,10 @@
 
 #include "Cuda.h"
 #include "Gnu.h"
+#include "ROCm.h"
 #include "clang/Driver/Tool.h"
 #include "clang/Driver/ToolChain.h"
+#include "llvm/Support/ErrorOr.h"
 
 namespace clang {
 namespace driver {
@@ -34,8 +36,7 @@ class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 
 class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC)
-      : Tool("MinGW::Linker", "linker", TC, Tool::RF_Full) {}
+  Linker(const ToolChain &TC) : Tool("MinGW::Linker", "linker", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
@@ -81,6 +82,8 @@ class LLVM_LIBRARY_VISIBILITY MinGW : public ToolChain {
 
   void AddCudaIncludeArgs(const llvm::opt::ArgList &DriverArgs,
                           llvm::opt::ArgStringList &CC1Args) const override;
+  void AddHIPIncludeArgs(const llvm::opt::ArgList &DriverArgs,
+                         llvm::opt::ArgStringList &CC1Args) const override;
 
   void printVerboseInfo(raw_ostream &OS) const override;
 
@@ -91,6 +94,7 @@ class LLVM_LIBRARY_VISIBILITY MinGW : public ToolChain {
 
 private:
   CudaInstallationDetector CudaInstallation;
+  RocmInstallationDetector RocmInstallation;
 
   std::string Base;
   std::string GccLibDir;
diff --git a/clang/lib/Driver/ToolChains/Minix.cpp b/clang/lib/Driver/ToolChains/Minix.cpp
index 6947049ea52ee..d0314795620ce 100644
--- a/clang/lib/Driver/ToolChains/Minix.cpp
+++ b/clang/lib/Driver/ToolChains/Minix.cpp
@@ -36,7 +36,8 @@ void tools::minix::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(II.getFilename());
 
   const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("as"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 void tools::minix::Linker::ConstructJob(Compilation &C, const JobAction &JA,
@@ -88,7 +89,8 @@ void tools::minix::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   const char *Exec = Args.MakeArgString(getToolChain().GetLinkerPath());
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 /// Minix - Minix tool chain which can call as(1) and ld(1) directly.
diff --git a/clang/lib/Driver/ToolChains/Minix.h b/clang/lib/Driver/ToolChains/Minix.h
index 1ed6acebab9c0..af8d59c5085a9 100644
--- a/clang/lib/Driver/ToolChains/Minix.h
+++ b/clang/lib/Driver/ToolChains/Minix.h
@@ -18,10 +18,9 @@ namespace driver {
 namespace tools {
 /// minix -- Directly call GNU Binutils assembler and linker
 namespace minix {
-class LLVM_LIBRARY_VISIBILITY Assembler : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 public:
-  Assembler(const ToolChain &TC)
-      : GnuTool("minix::Assembler", "assembler", TC) {}
+  Assembler(const ToolChain &TC) : Tool("minix::Assembler", "assembler", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
 
@@ -31,9 +30,9 @@ class LLVM_LIBRARY_VISIBILITY Assembler : public GnuTool {
                     const char *LinkingOutput) const override;
 };
 
-class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC) : GnuTool("minix::Linker", "linker", TC) {}
+  Linker(const ToolChain &TC) : Tool("minix::Linker", "linker", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
diff --git a/clang/lib/Driver/ToolChains/Myriad.cpp b/clang/lib/Driver/ToolChains/Myriad.cpp
index 2ce0f13ce3d1a..84fe4748b6faf 100644
--- a/clang/lib/Driver/ToolChains/Myriad.cpp
+++ b/clang/lib/Driver/ToolChains/Myriad.cpp
@@ -77,8 +77,9 @@ void tools::SHAVE::Compiler::ConstructJob(Compilation &C, const JobAction &JA,
 
   std::string Exec =
       Args.MakeArgString(getToolChain().GetProgramPath("moviCompile"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Args.MakeArgString(Exec),
-                                          CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Args.MakeArgString(Exec), CmdArgs,
+                                         Inputs));
 }
 
 void tools::SHAVE::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
@@ -112,8 +113,9 @@ void tools::SHAVE::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
 
   std::string Exec =
       Args.MakeArgString(getToolChain().GetProgramPath("moviAsm"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Args.MakeArgString(Exec),
-                                          CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Args.MakeArgString(Exec), CmdArgs,
+                                         Inputs));
 }
 
 void tools::Myriad::Linker::ConstructJob(Compilation &C, const JobAction &JA,
@@ -198,8 +200,9 @@ void tools::Myriad::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   std::string Exec =
       Args.MakeArgString(TC.GetProgramPath("sparc-myriad-rtems-ld"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Args.MakeArgString(Exec),
-                                          CmdArgs, Inputs));
+  C.addCommand(
+      std::make_unique<Command>(JA, *this, ResponseFileSupport::AtFileCurCP(),
+                                Args.MakeArgString(Exec), CmdArgs, Inputs));
 }
 
 MyriadToolChain::MyriadToolChain(const Driver &D, const llvm::Triple &Triple,
diff --git a/clang/lib/Driver/ToolChains/Myriad.h b/clang/lib/Driver/ToolChains/Myriad.h
index 9f5225fbc62c9..cae574bdcfea3 100644
--- a/clang/lib/Driver/ToolChains/Myriad.h
+++ b/clang/lib/Driver/ToolChains/Myriad.h
@@ -49,9 +49,9 @@ class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 /// whereas the linker, which accepts code for a mixture of Sparc and SHAVE,
 /// is in the Myriad namespace.
 namespace Myriad {
-class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC) : GnuTool("shave::Linker", "ld", TC) {}
+  Linker(const ToolChain &TC) : Tool("shave::Linker", "ld", TC) {}
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
   void ConstructJob(Compilation &C, const JobAction &JA,
diff --git a/clang/lib/Driver/ToolChains/NaCl.cpp b/clang/lib/Driver/ToolChains/NaCl.cpp
index 97241c8840273..15a773675299a 100644
--- a/clang/lib/Driver/ToolChains/NaCl.cpp
+++ b/clang/lib/Driver/ToolChains/NaCl.cpp
@@ -193,7 +193,8 @@ void nacltools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   const char *Exec = Args.MakeArgString(ToolChain.GetLinkerPath());
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 /// NaCl Toolchain
diff --git a/clang/lib/Driver/ToolChains/NaCl.h b/clang/lib/Driver/ToolChains/NaCl.h
index ab243f8087bb1..5e5fdb583bb60 100644
--- a/clang/lib/Driver/ToolChains/NaCl.h
+++ b/clang/lib/Driver/ToolChains/NaCl.h
@@ -27,9 +27,9 @@ class LLVM_LIBRARY_VISIBILITY AssemblerARM : public gnutools::Assembler {
                     const char *LinkingOutput) const override;
 };
 
-class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC) : GnuTool("NaCl::Linker", "linker", TC) {}
+  Linker(const ToolChain &TC) : Tool("NaCl::Linker", "linker", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
diff --git a/clang/lib/Driver/ToolChains/NetBSD.cpp b/clang/lib/Driver/ToolChains/NetBSD.cpp
index 0100a387d6c35..253ee6ce0f721 100644
--- a/clang/lib/Driver/ToolChains/NetBSD.cpp
+++ b/clang/lib/Driver/ToolChains/NetBSD.cpp
@@ -103,7 +103,8 @@ void netbsd::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(II.getFilename());
 
   const char *Exec = Args.MakeArgString((getToolChain().GetProgramPath("as")));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
@@ -337,7 +338,8 @@ void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   ToolChain.addProfileRTLibs(Args, CmdArgs);
 
   const char *Exec = Args.MakeArgString(ToolChain.GetLinkerPath());
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 /// NetBSD - NetBSD tool chain which can call as(1) and ld(1) directly.
diff --git a/clang/lib/Driver/ToolChains/NetBSD.h b/clang/lib/Driver/ToolChains/NetBSD.h
index 6d404263f625c..8348554fd1492 100644
--- a/clang/lib/Driver/ToolChains/NetBSD.h
+++ b/clang/lib/Driver/ToolChains/NetBSD.h
@@ -19,10 +19,9 @@ namespace tools {
 
 /// netbsd -- Directly call GNU Binutils assembler and linker
 namespace netbsd {
-class LLVM_LIBRARY_VISIBILITY Assembler : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 public:
-  Assembler(const ToolChain &TC)
-      : GnuTool("netbsd::Assembler", "assembler", TC) {}
+  Assembler(const ToolChain &TC) : Tool("netbsd::Assembler", "assembler", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
 
@@ -32,9 +31,9 @@ class LLVM_LIBRARY_VISIBILITY Assembler : public GnuTool {
                     const char *LinkingOutput) const override;
 };
 
-class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC) : GnuTool("netbsd::Linker", "linker", TC) {}
+  Linker(const ToolChain &TC) : Tool("netbsd::Linker", "linker", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
diff --git a/clang/lib/Driver/ToolChains/OpenBSD.cpp b/clang/lib/Driver/ToolChains/OpenBSD.cpp
index 80343c0394cb5..9c1a9c5f82280 100644
--- a/clang/lib/Driver/ToolChains/OpenBSD.cpp
+++ b/clang/lib/Driver/ToolChains/OpenBSD.cpp
@@ -89,7 +89,8 @@ void openbsd::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(II.getFilename());
 
   const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("as"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 void openbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
@@ -227,7 +228,8 @@ void openbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   const char *Exec = Args.MakeArgString(ToolChain.GetLinkerPath());
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs));
 }
 
 SanitizerMask OpenBSD::getSupportedSanitizers() const {
diff --git a/clang/lib/Driver/ToolChains/OpenBSD.h b/clang/lib/Driver/ToolChains/OpenBSD.h
index 9f1ee0f66402b..897eee57ab684 100644
--- a/clang/lib/Driver/ToolChains/OpenBSD.h
+++ b/clang/lib/Driver/ToolChains/OpenBSD.h
@@ -19,10 +19,10 @@ namespace tools {
 
 /// openbsd -- Directly call GNU Binutils assembler and linker
 namespace openbsd {
-class LLVM_LIBRARY_VISIBILITY Assembler : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 public:
   Assembler(const ToolChain &TC)
-      : GnuTool("openbsd::Assembler", "assembler", TC) {}
+      : Tool("openbsd::Assembler", "assembler", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
 
@@ -32,9 +32,9 @@ class LLVM_LIBRARY_VISIBILITY Assembler : public GnuTool {
                     const char *LinkingOutput) const override;
 };
 
-class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC) : GnuTool("openbsd::Linker", "linker", TC) {}
+  Linker(const ToolChain &TC) : Tool("openbsd::Linker", "linker", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
diff --git a/clang/lib/Driver/ToolChains/PS4CPU.cpp b/clang/lib/Driver/ToolChains/PS4CPU.cpp
index 49f26c3701681..6dc81899cbaac 100644
--- a/clang/lib/Driver/ToolChains/PS4CPU.cpp
+++ b/clang/lib/Driver/ToolChains/PS4CPU.cpp
@@ -66,7 +66,8 @@ void tools::PS4cpu::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
 
   const char *Exec =
       Args.MakeArgString(getToolChain().GetProgramPath("orbis-as"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileUTF8(), Exec, CmdArgs, Inputs));
 }
 
 static void AddPS4SanitizerArgs(const ToolChain &TC, ArgStringList &CmdArgs) {
@@ -151,7 +152,8 @@ void tools::PS4cpu::Link::ConstructJob(Compilation &C, const JobAction &JA,
   const char *Exec =
       Args.MakeArgString(ToolChain.GetProgramPath("orbis-ld"));
 
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileUTF8(), Exec, CmdArgs, Inputs));
 }
 
 toolchains::PS4CPU::PS4CPU(const Driver &D, const llvm::Triple &Triple,
diff --git a/clang/lib/Driver/ToolChains/PS4CPU.h b/clang/lib/Driver/ToolChains/PS4CPU.h
index 6b1efd25f43a8..968be015d4113 100644
--- a/clang/lib/Driver/ToolChains/PS4CPU.h
+++ b/clang/lib/Driver/ToolChains/PS4CPU.h
@@ -26,8 +26,7 @@ void addSanitizerArgs(const ToolChain &TC, llvm::opt::ArgStringList &CmdArgs);
 
 class LLVM_LIBRARY_VISIBILITY Assemble : public Tool {
 public:
-  Assemble(const ToolChain &TC)
-      : Tool("PS4cpu::Assemble", "assembler", TC, RF_Full) {}
+  Assemble(const ToolChain &TC) : Tool("PS4cpu::Assemble", "assembler", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
 
@@ -40,7 +39,7 @@ class LLVM_LIBRARY_VISIBILITY Assemble : public Tool {
 
 class LLVM_LIBRARY_VISIBILITY Link : public Tool {
 public:
-  Link(const ToolChain &TC) : Tool("PS4cpu::Link", "linker", TC, RF_Full) {}
+  Link(const ToolChain &TC) : Tool("PS4cpu::Link", "linker", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
diff --git a/clang/lib/Driver/ToolChains/RISCVToolchain.cpp b/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
index 21106d0038590..cc912d94cb92f 100644
--- a/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
+++ b/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
@@ -191,7 +191,8 @@ void RISCV::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   CmdArgs.push_back("-o");
   CmdArgs.push_back(Output.getFilename());
-  C.addCommand(std::make_unique<Command>(JA, *this, Args.MakeArgString(Linker),
-                                          CmdArgs, Inputs));
+  C.addCommand(
+      std::make_unique<Command>(JA, *this, ResponseFileSupport::AtFileCurCP(),
+                                Args.MakeArgString(Linker), CmdArgs, Inputs));
 }
 // RISCV tools end.
diff --git a/clang/lib/Driver/ToolChains/RISCVToolchain.h b/clang/lib/Driver/ToolChains/RISCVToolchain.h
index b40ef366ff492..4734aee5f1ab7 100644
--- a/clang/lib/Driver/ToolChains/RISCVToolchain.h
+++ b/clang/lib/Driver/ToolChains/RISCVToolchain.h
@@ -46,9 +46,9 @@ class LLVM_LIBRARY_VISIBILITY RISCVToolChain : public Generic_ELF {
 
 namespace tools {
 namespace RISCV {
-class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC) : GnuTool("RISCV::Linker", "ld", TC) {}
+  Linker(const ToolChain &TC) : Tool("RISCV::Linker", "ld", TC) {}
   bool hasIntegratedCPP() const override { return false; }
   bool isLinkJob() const override { return true; }
   void ConstructJob(Compilation &C, const JobAction &JA,
diff --git a/clang/lib/Driver/ToolChains/ROCm.h b/clang/lib/Driver/ToolChains/ROCm.h
index 779e2e133becd..962c72fedfe07 100644
--- a/clang/lib/Driver/ToolChains/ROCm.h
+++ b/clang/lib/Driver/ToolChains/ROCm.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Option/ArgList.h"
+#include "llvm/Support/VersionTuple.h"
 
 namespace clang {
 namespace driver {
@@ -38,11 +39,43 @@ class RocmInstallationDetector {
     }
   };
 
+  // Installation path candidate.
+  struct Candidate {
+    llvm::SmallString<0> Path;
+    bool StrictChecking;
+
+    Candidate(std::string Path, bool StrictChecking = false)
+        : Path(Path), StrictChecking(StrictChecking) {}
+  };
+
   const Driver &D;
-  bool IsValid = false;
-  // RocmVersion Version = RocmVersion::UNKNOWN;
+  bool HasHIPRuntime = false;
+  bool HasDeviceLibrary = false;
+
+  // Default version if not detected or specified.
+  const unsigned DefaultVersionMajor = 3;
+  const unsigned DefaultVersionMinor = 5;
+  const char *DefaultVersionPatch = "0";
+
+  // The version string in Major.Minor.Patch format.
+  std::string DetectedVersion;
+  // Version containing major and minor.
+  llvm::VersionTuple VersionMajorMinor;
+  // Version containing patch.
+  std::string VersionPatch;
+
+  // ROCm path specified by --rocm-path.
+  StringRef RocmPathArg;
+  // ROCm device library paths specified by --rocm-device-lib-path.
+  std::vector<std::string> RocmDeviceLibPathArg;
+  // HIP version specified by --hip-version.
+  StringRef HIPVersionArg;
+  // Wheter -nogpulib is specified.
+  bool NoBuiltinLibs = false;
+
+  // Paths
   SmallString<0> InstallPath;
-  // SmallString<0> BinPath;
+  SmallString<0> BinPath;
   SmallString<0> LibPath;
   SmallString<0> LibDevicePath;
   SmallString<0> IncludePath;
@@ -74,11 +107,15 @@ class RocmInstallationDetector {
   // CheckRocmVersionSupportsArch.
   mutable llvm::SmallSet<CudaArch, 4> ArchsWithBadVersion;
 
-  void scanLibDevicePath();
+  void scanLibDevicePath(llvm::StringRef Path);
+  void ParseHIPVersionFile(llvm::StringRef V);
+  SmallVector<Candidate, 4> getInstallationPathCandidates();
 
 public:
   RocmInstallationDetector(const Driver &D, const llvm::Triple &HostTriple,
-                           const llvm::opt::ArgList &Args);
+                           const llvm::opt::ArgList &Args,
+                           bool DetectHIPRuntime = true,
+                           bool DetectDeviceLib = false);
 
   /// Add arguments needed to link default bitcode libraries.
   void addCommonBitcodeLibCC1Args(const llvm::opt::ArgList &DriverArgs,
@@ -93,8 +130,12 @@ class RocmInstallationDetector {
   /// most one error per Arch.
   void CheckRocmVersionSupportsArch(CudaArch Arch) const;
 
-  /// Check whether we detected a valid Rocm install.
-  bool isValid() const { return IsValid; }
+  /// Check whether we detected a valid HIP runtime.
+  bool hasHIPRuntime() const { return HasHIPRuntime; }
+
+  /// Check whether we detected a valid ROCm device library.
+  bool hasDeviceLibrary() const { return HasDeviceLibrary; }
+
   /// Print information about the detected ROCm installation.
   void print(raw_ostream &OS) const;
 
@@ -163,6 +204,22 @@ class RocmInstallationDetector {
 
   void AddHIPIncludeArgs(const llvm::opt::ArgList &DriverArgs,
                          llvm::opt::ArgStringList &CC1Args) const;
+
+  void detectDeviceLibrary();
+  void detectHIPRuntime();
+
+  /// Get the values for --rocm-device-lib-path arguments
+  std::vector<std::string> getRocmDeviceLibPathArg() const {
+    return RocmDeviceLibPathArg;
+  }
+
+  /// Get the value for --rocm-path argument
+  StringRef getRocmPathArg() const { return RocmPathArg; }
+
+  /// Get the value for --hip-version argument
+  StringRef getHIPVersionArg() const { return HIPVersionArg; }
+
+  std::string getHIPVersion() const { return DetectedVersion; }
 };
 
 } // end namespace driver
diff --git a/clang/lib/Driver/ToolChains/Solaris.cpp b/clang/lib/Driver/ToolChains/Solaris.cpp
index 0a1ac0b225868..b8fdc87478bc6 100644
--- a/clang/lib/Driver/ToolChains/Solaris.cpp
+++ b/clang/lib/Driver/ToolChains/Solaris.cpp
@@ -41,7 +41,8 @@ void solaris::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(II.getFilename());
 
   const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("as"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Exec, CmdArgs, Inputs));
 }
 
 void solaris::Linker::ConstructJob(Compilation &C, const JobAction &JA,
@@ -150,7 +151,8 @@ void solaris::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   getToolChain().addProfileRTLibs(Args, CmdArgs);
 
   const char *Exec = Args.MakeArgString(getToolChain().GetLinkerPath());
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Exec, CmdArgs, Inputs));
 }
 
 static StringRef getSolarisLibSuffix(const llvm::Triple &Triple) {
diff --git a/clang/lib/Driver/ToolChains/VEToolchain.cpp b/clang/lib/Driver/ToolChains/VEToolchain.cpp
new file mode 100644
index 0000000000000..6ea405c0269c7
--- /dev/null
+++ b/clang/lib/Driver/ToolChains/VEToolchain.cpp
@@ -0,0 +1,119 @@
+//===--- VE.cpp - VE ToolChain Implementations ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "VEToolchain.h"
+#include "CommonArgs.h"
+#include "clang/Driver/Compilation.h"
+#include "clang/Driver/Driver.h"
+#include "clang/Driver/Options.h"
+#include "llvm/Option/ArgList.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include <cstdlib> // ::getenv
+
+using namespace clang::driver;
+using namespace clang::driver::toolchains;
+using namespace clang;
+using namespace llvm::opt;
+
+/// VE tool chain
+VEToolChain::VEToolChain(const Driver &D, const llvm::Triple &Triple,
+                         const ArgList &Args)
+    : Linux(D, Triple, Args) {
+  getProgramPaths().push_back("/opt/nec/ve/bin");
+  // ProgramPaths are found via 'PATH' environment variable.
+
+  // default file paths are:
+  //   ${RESOURCEDIR}/lib/linux/ve (== getArchSpecificLibPath)
+  //   /lib/../lib64
+  //   /usr/lib/../lib64
+  //   ${BINPATH}/../lib
+  //   /lib
+  //   /usr/lib
+  //
+  // These are OK for host, but no go for VE.  So, defines them all
+  // from scratch here.
+  getFilePaths().clear();
+  getFilePaths().push_back(getArchSpecificLibPath());
+  getFilePaths().push_back(computeSysRoot() + "/opt/nec/ve/lib");
+}
+
+Tool *VEToolChain::buildAssembler() const {
+  return new tools::gnutools::Assembler(*this);
+}
+
+Tool *VEToolChain::buildLinker() const {
+  return new tools::gnutools::Linker(*this);
+}
+
+bool VEToolChain::isPICDefault() const { return false; }
+
+bool VEToolChain::isPIEDefault() const { return false; }
+
+bool VEToolChain::isPICDefaultForced() const { return false; }
+
+bool VEToolChain::SupportsProfiling() const { return false; }
+
+bool VEToolChain::hasBlocksRuntime() const { return false; }
+
+void VEToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
+                                            ArgStringList &CC1Args) const {
+  if (DriverArgs.hasArg(clang::driver::options::OPT_nostdinc))
+    return;
+
+  if (DriverArgs.hasArg(options::OPT_nobuiltininc) &&
+      DriverArgs.hasArg(options::OPT_nostdlibinc))
+    return;
+
+  if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) {
+    SmallString<128> P(getDriver().ResourceDir);
+    llvm::sys::path::append(P, "include");
+    addSystemInclude(DriverArgs, CC1Args, P);
+  }
+
+  if (!DriverArgs.hasArg(options::OPT_nostdlibinc)) {
+    if (const char *cl_include_dir = getenv("NCC_C_INCLUDE_PATH")) {
+      SmallVector<StringRef, 4> Dirs;
+      const char EnvPathSeparatorStr[] = {llvm::sys::EnvPathSeparator, '\0'};
+      StringRef(cl_include_dir).split(Dirs, StringRef(EnvPathSeparatorStr));
+      ArrayRef<StringRef> DirVec(Dirs);
+      addSystemIncludes(DriverArgs, CC1Args, DirVec);
+    } else {
+      addSystemInclude(DriverArgs, CC1Args,
+                       getDriver().SysRoot + "/opt/nec/ve/include");
+    }
+  }
+}
+
+void VEToolChain::addClangTargetOptions(const ArgList &DriverArgs,
+                                        ArgStringList &CC1Args,
+                                        Action::OffloadKind) const {
+  CC1Args.push_back("-nostdsysteminc");
+  bool UseInitArrayDefault = true;
+  if (!DriverArgs.hasFlag(options::OPT_fuse_init_array,
+                          options::OPT_fno_use_init_array, UseInitArrayDefault))
+    CC1Args.push_back("-fno-use-init-array");
+}
+
+void VEToolChain::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
+                                               ArgStringList &CC1Args) const {
+  // TODO upstream VE libc++ patches
+  llvm_unreachable("The VE target has no C++ stdlib for Clang yet");
+}
+
+void VEToolChain::AddCXXStdlibLibArgs(const ArgList &Args,
+                                      ArgStringList &CmdArgs) const {
+  // TODO upstream VE libc++ patches
+  llvm_unreachable("The VE target has no C++ stdlib for Clang yet");
+}
+
+llvm::ExceptionHandling
+VEToolChain::GetExceptionModel(const ArgList &Args) const {
+  // VE uses SjLj exceptions.
+  return llvm::ExceptionHandling::SjLj;
+}
diff --git a/clang/lib/Driver/ToolChains/VEToolchain.h b/clang/lib/Driver/ToolChains/VEToolchain.h
new file mode 100644
index 0000000000000..59069c0a75959
--- /dev/null
+++ b/clang/lib/Driver/ToolChains/VEToolchain.h
@@ -0,0 +1,66 @@
+//===--- VE.h - VE ToolChain Implementations --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_VE_H
+#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_VE_H
+
+#include "Linux.h"
+#include "clang/Driver/ToolChain.h"
+
+namespace clang {
+namespace driver {
+namespace toolchains {
+
+class LLVM_LIBRARY_VISIBILITY VEToolChain : public Linux {
+public:
+  VEToolChain(const Driver &D, const llvm::Triple &Triple,
+              const llvm::opt::ArgList &Args);
+
+protected:
+  Tool *buildAssembler() const override;
+  Tool *buildLinker() const override;
+
+public:
+  bool isPICDefault() const override;
+  bool isPIEDefault() const override;
+  bool isPICDefaultForced() const override;
+  bool SupportsProfiling() const override;
+  bool hasBlocksRuntime() const override;
+  void
+  AddClangSystemIncludeArgs(const llvm::opt::ArgList &DriverArgs,
+                            llvm::opt::ArgStringList &CC1Args) const override;
+  void
+  addClangTargetOptions(const llvm::opt::ArgList &DriverArgs,
+                        llvm::opt::ArgStringList &CC1Args,
+                        Action::OffloadKind DeviceOffloadKind) const override;
+  void AddClangCXXStdlibIncludeArgs(
+      const llvm::opt::ArgList &DriverArgs,
+      llvm::opt::ArgStringList &CC1Args) const override;
+  void AddCXXStdlibLibArgs(const llvm::opt::ArgList &Args,
+                           llvm::opt::ArgStringList &CmdArgs) const override;
+
+  llvm::ExceptionHandling
+  GetExceptionModel(const llvm::opt::ArgList &Args) const override;
+
+  CXXStdlibType
+  GetCXXStdlibType(const llvm::opt::ArgList &Args) const override {
+    return ToolChain::CST_Libcxx;
+  }
+
+  RuntimeLibType GetDefaultRuntimeLibType() const override {
+    return ToolChain::RLT_CompilerRT;
+  }
+
+  const char *getDefaultLinker() const override { return "nld"; }
+};
+
+} // end namespace toolchains
+} // end namespace driver
+} // end namespace clang
+
+#endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_VE_H
diff --git a/clang/lib/Driver/ToolChains/WebAssembly.cpp b/clang/lib/Driver/ToolChains/WebAssembly.cpp
index 48f9a9b603dbe..10168736400f8 100644
--- a/clang/lib/Driver/ToolChains/WebAssembly.cpp
+++ b/clang/lib/Driver/ToolChains/WebAssembly.cpp
@@ -62,6 +62,12 @@ void wasm::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   const char *Linker = Args.MakeArgString(getLinkerPath(Args));
   ArgStringList CmdArgs;
 
+  CmdArgs.push_back("-m");
+  if (getToolChain().getTriple().isArch64Bit())
+    CmdArgs.push_back("wasm64");
+  else
+    CmdArgs.push_back("wasm32");
+
   if (Args.hasArg(options::OPT_s))
     CmdArgs.push_back("--strip-all");
 
@@ -108,7 +114,8 @@ void wasm::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-o");
   CmdArgs.push_back(Output.getFilename());
 
-  C.addCommand(std::make_unique<Command>(JA, *this, Linker, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::AtFileCurCP(), Linker, CmdArgs, Inputs));
 
   // When optimizing, if wasm-opt is available, run it.
   if (Arg *A = Args.getLastArg(options::OPT_O_Group)) {
@@ -130,7 +137,9 @@ void wasm::Linker::ConstructJob(Compilation &C, const JobAction &JA,
         CmdArgs.push_back(Args.MakeArgString(llvm::Twine("-O") + OOpt));
         CmdArgs.push_back("-o");
         CmdArgs.push_back(Output.getFilename());
-        C.addCommand(std::make_unique<Command>(JA, *this, WasmOpt, CmdArgs, Inputs));
+        C.addCommand(std::make_unique<Command>(
+            JA, *this, ResponseFileSupport::AtFileCurCP(), WasmOpt, CmdArgs,
+            Inputs));
       }
     }
   }
diff --git a/clang/lib/Driver/ToolChains/WebAssembly.h b/clang/lib/Driver/ToolChains/WebAssembly.h
index 67d5fce845765..616bfb5d3d0c1 100644
--- a/clang/lib/Driver/ToolChains/WebAssembly.h
+++ b/clang/lib/Driver/ToolChains/WebAssembly.h
@@ -18,10 +18,9 @@ namespace driver {
 namespace tools {
 namespace wasm {
 
-class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool {
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  explicit Linker(const ToolChain &TC)
-      : GnuTool("wasm::Linker", "linker", TC) {}
+  explicit Linker(const ToolChain &TC) : Tool("wasm::Linker", "linker", TC) {}
   bool isLinkJob() const override { return true; }
   bool hasIntegratedCPP() const override { return false; }
   std::string getLinkerPath(const llvm::opt::ArgList &Args) const;
diff --git a/clang/lib/Driver/ToolChains/XCore.cpp b/clang/lib/Driver/ToolChains/XCore.cpp
index ba3a6d44addaf..5030c73c7d825 100644
--- a/clang/lib/Driver/ToolChains/XCore.cpp
+++ b/clang/lib/Driver/ToolChains/XCore.cpp
@@ -52,7 +52,8 @@ void tools::XCore::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(II.getFilename());
 
   const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("xcc"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Exec, CmdArgs, Inputs));
 }
 
 void tools::XCore::Linker::ConstructJob(Compilation &C, const JobAction &JA,
@@ -80,7 +81,8 @@ void tools::XCore::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs, JA);
 
   const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("xcc"));
-  C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Exec, CmdArgs, Inputs));
 }
 
 /// XCore tool chain
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index 2ac43b6cfa4bc..b1497651a8fef 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -1041,8 +1041,10 @@ unsigned ContinuationIndenter::getNewLineColumn(const LineState &State) {
       //    * not remove the 'lead' ContinuationIndentWidth
       //    * always un-indent by the operator when
       //    BreakBeforeTernaryOperators=true
-      unsigned Indent =
-          State.Stack.back().Indent - Style.ContinuationIndentWidth;
+      unsigned Indent = State.Stack.back().Indent;
+      if (Style.AlignOperands != FormatStyle::OAS_DontAlign) {
+        Indent -= Style.ContinuationIndentWidth;
+      }
       if (Style.BreakBeforeTernaryOperators &&
           State.Stack.back().UnindentOperator)
         Indent -= 2;
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index d83410b1f7e1a..0d277a6464af2 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -599,6 +599,8 @@ template <> struct MappingTraits<FormatStyle> {
     IO.mapOptional("TypenameMacros", Style.TypenameMacros);
     IO.mapOptional("UseCRLF", Style.UseCRLF);
     IO.mapOptional("UseTab", Style.UseTab);
+    IO.mapOptional("WhitespaceSensitiveMacros",
+                   Style.WhitespaceSensitiveMacros);
   }
 };
 
@@ -933,6 +935,9 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.SortUsingDeclarations = true;
   LLVMStyle.StatementMacros.push_back("Q_UNUSED");
   LLVMStyle.StatementMacros.push_back("QT_REQUIRE_VERSION");
+  LLVMStyle.WhitespaceSensitiveMacros.push_back("STRINGIZE");
+  LLVMStyle.WhitespaceSensitiveMacros.push_back("PP_STRINGIZE");
+  LLVMStyle.WhitespaceSensitiveMacros.push_back("BOOST_PP_STRINGIZE");
 
   // Defaults that differ when not C++.
   if (Language == FormatStyle::LK_TableGen) {
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index 2ad839a6a4f00..d4287f53fde37 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -102,6 +102,7 @@ namespace format {
   TYPE(TrailingUnaryOperator)                                                  \
   TYPE(TypenameMacro)                                                          \
   TYPE(UnaryOperator)                                                          \
+  TYPE(UntouchableMacroFunc)                                                   \
   TYPE(CSharpStringLiteral)                                                    \
   TYPE(CSharpNamedArgumentColon)                                               \
   TYPE(CSharpNullable)                                                         \
diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp
index f64ab389d02fb..1fd153d1112eb 100644
--- a/clang/lib/Format/FormatTokenLexer.cpp
+++ b/clang/lib/Format/FormatTokenLexer.cpp
@@ -22,13 +22,15 @@
 namespace clang {
 namespace format {
 
-FormatTokenLexer::FormatTokenLexer(const SourceManager &SourceMgr, FileID ID,
-                                   unsigned Column, const FormatStyle &Style,
-                                   encoding::Encoding Encoding)
+FormatTokenLexer::FormatTokenLexer(
+    const SourceManager &SourceMgr, FileID ID, unsigned Column,
+    const FormatStyle &Style, encoding::Encoding Encoding,
+    llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator,
+    IdentifierTable &IdentTable)
     : FormatTok(nullptr), IsFirstToken(true), StateStack({LexerState::NORMAL}),
       Column(Column), TrailingWhitespace(0), SourceMgr(SourceMgr), ID(ID),
-      Style(Style), IdentTable(getFormattingLangOpts(Style)),
-      Keywords(IdentTable), Encoding(Encoding), FirstInLineIndex(0),
+      Style(Style), IdentTable(IdentTable), Keywords(IdentTable),
+      Encoding(Encoding), Allocator(Allocator), FirstInLineIndex(0),
       FormattingDisabled(false), MacroBlockBeginRegex(Style.MacroBlockBegin),
       MacroBlockEndRegex(Style.MacroBlockEnd) {
   Lex.reset(new Lexer(ID, SourceMgr.getBuffer(ID), SourceMgr,
@@ -43,6 +45,11 @@ FormatTokenLexer::FormatTokenLexer(const SourceManager &SourceMgr, FileID ID,
     Macros.insert({&IdentTable.get(TypenameMacro), TT_TypenameMacro});
   for (const std::string &NamespaceMacro : Style.NamespaceMacros)
     Macros.insert({&IdentTable.get(NamespaceMacro), TT_NamespaceMacro});
+  for (const std::string &WhitespaceSensitiveMacro :
+       Style.WhitespaceSensitiveMacros) {
+    Macros.insert(
+        {&IdentTable.get(WhitespaceSensitiveMacro), TT_UntouchableMacroFunc});
+  }
 }
 
 ArrayRef<FormatToken *> FormatTokenLexer::lex() {
diff --git a/clang/lib/Format/FormatTokenLexer.h b/clang/lib/Format/FormatTokenLexer.h
index 219cd46f98edf..6b08677e33697 100644
--- a/clang/lib/Format/FormatTokenLexer.h
+++ b/clang/lib/Format/FormatTokenLexer.h
@@ -38,7 +38,9 @@ enum LexerState {
 class FormatTokenLexer {
 public:
   FormatTokenLexer(const SourceManager &SourceMgr, FileID ID, unsigned Column,
-                   const FormatStyle &Style, encoding::Encoding Encoding);
+                   const FormatStyle &Style, encoding::Encoding Encoding,
+                   llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator,
+                   IdentifierTable &IdentTable);
 
   ArrayRef<FormatToken *> lex();
 
@@ -103,10 +105,10 @@ class FormatTokenLexer {
   const SourceManager &SourceMgr;
   FileID ID;
   const FormatStyle &Style;
-  IdentifierTable IdentTable;
+  IdentifierTable &IdentTable;
   AdditionalKeywords Keywords;
   encoding::Encoding Encoding;
-  llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
+  llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator;
   // Index (in 'Tokens') of the last token that starts a new line.
   unsigned FirstInLineIndex;
   SmallVector<FormatToken *, 16> Tokens;
diff --git a/clang/lib/Format/TokenAnalyzer.cpp b/clang/lib/Format/TokenAnalyzer.cpp
index eb98a205d5260..f1459a808ff84 100644
--- a/clang/lib/Format/TokenAnalyzer.cpp
+++ b/clang/lib/Format/TokenAnalyzer.cpp
@@ -64,11 +64,16 @@ TokenAnalyzer::TokenAnalyzer(const Environment &Env, const FormatStyle &Style)
 
 std::pair<tooling::Replacements, unsigned> TokenAnalyzer::process() {
   tooling::Replacements Result;
-  FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(),
-                          Env.getFirstStartColumn(), Style, Encoding);
+  llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
+  IdentifierTable IdentTable(getFormattingLangOpts(Style));
+  FormatTokenLexer Lex(Env.getSourceManager(), Env.getFileID(),
+                       Env.getFirstStartColumn(), Style, Encoding, Allocator,
 
-  UnwrappedLineParser Parser(Style, Tokens.getKeywords(),
-                             Env.getFirstStartColumn(), Tokens.lex(), *this);
+                       IdentTable);
+  ArrayRef<FormatToken *> Toks(Lex.lex());
+  SmallVector<FormatToken *, 10> Tokens(Toks.begin(), Toks.end());
+  UnwrappedLineParser Parser(Style, Lex.getKeywords(),
+                             Env.getFirstStartColumn(), Tokens, *this);
   Parser.parse();
   assert(UnwrappedLines.rbegin()->empty());
   unsigned Penalty = 0;
@@ -76,14 +81,14 @@ std::pair<tooling::Replacements, unsigned> TokenAnalyzer::process() {
     LLVM_DEBUG(llvm::dbgs() << "Run " << Run << "...\n");
     SmallVector<AnnotatedLine *, 16> AnnotatedLines;
 
-    TokenAnnotator Annotator(Style, Tokens.getKeywords());
+    TokenAnnotator Annotator(Style, Lex.getKeywords());
     for (unsigned i = 0, e = UnwrappedLines[Run].size(); i != e; ++i) {
       AnnotatedLines.push_back(new AnnotatedLine(UnwrappedLines[Run][i]));
       Annotator.annotate(*AnnotatedLines.back());
     }
 
     std::pair<tooling::Replacements, unsigned> RunResult =
-        analyze(Annotator, AnnotatedLines, Tokens);
+        analyze(Annotator, AnnotatedLines, Lex);
 
     LLVM_DEBUG({
       llvm::dbgs() << "Replacements for run " << Run << ":\n";
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 6e8dc690eeb9a..7f8e351265127 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -160,6 +160,27 @@ class AnnotatingParser {
     return false;
   }
 
+  bool parseUntouchableParens() {
+    while (CurrentToken) {
+      CurrentToken->Finalized = true;
+      switch (CurrentToken->Tok.getKind()) {
+      case tok::l_paren:
+        next();
+        if (!parseUntouchableParens())
+          return false;
+        continue;
+      case tok::r_paren:
+        next();
+        return true;
+      default:
+        // no-op
+        break;
+      }
+      next();
+    }
+    return false;
+  }
+
   bool parseParens(bool LookForDecls = false) {
     if (!CurrentToken)
       return false;
@@ -171,6 +192,11 @@ class AnnotatingParser {
     Contexts.back().ColonIsForRangeExpr =
         Contexts.size() == 2 && Contexts[0].ColonIsForRangeExpr;
 
+    if (Left->Previous && Left->Previous->is(TT_UntouchableMacroFunc)) {
+      Left->Finalized = true;
+      return parseUntouchableParens();
+    }
+
     bool StartsObjCMethodExpr = false;
     if (FormatToken *MaybeSel = Left->Previous) {
       // @selector( starts a selector.
@@ -1311,7 +1337,7 @@ class AnnotatingParser {
             TT_TypenameMacro, TT_FunctionLBrace, TT_ImplicitStringLiteral,
             TT_InlineASMBrace, TT_JsFatArrow, TT_LambdaArrow, TT_NamespaceMacro,
             TT_OverloadedOperator, TT_RegexLiteral, TT_TemplateString,
-            TT_ObjCStringLiteral))
+            TT_ObjCStringLiteral, TT_UntouchableMacroFunc))
       CurrentToken->setType(TT_Unknown);
     CurrentToken->Role.reset();
     CurrentToken->MatchingParen = nullptr;
@@ -2818,6 +2844,11 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
             Left.Previous &&
             !Left.Previous->isOneOf(tok::l_paren, tok::coloncolon,
                                     tok::l_square));
+  // Ensure right pointer alignement with ellipsis e.g. int *...P
+  if (Left.is(tok::ellipsis) && Left.Previous &&
+      Left.Previous->isOneOf(tok::star, tok::amp, tok::ampamp))
+    return Style.PointerAlignment != FormatStyle::PAS_Right;
+
   if (Right.is(tok::star) && Left.is(tok::l_paren))
     return false;
   if (Left.is(tok::star) && Right.isOneOf(tok::star, tok::amp, tok::ampamp))
@@ -3970,7 +4001,7 @@ void TokenAnnotator::printDebugInfo(const AnnotatedLine &Line) {
                  << " C=" << Tok->CanBreakBefore
                  << " T=" << getTokenTypeName(Tok->getType())
                  << " S=" << Tok->SpacesRequiredBefore
-                 << " B=" << Tok->BlockParameterCount
+                 << " F=" << Tok->Finalized << " B=" << Tok->BlockParameterCount
                  << " BK=" << Tok->BlockKind << " P=" << Tok->SplitPenalty
                  << " Name=" << Tok->Tok.getName() << " L=" << Tok->TotalLength
                  << " PPK=" << Tok->PackingKind << " FakeLParens=";
diff --git a/clang/lib/Format/UnwrappedLineFormatter.cpp b/clang/lib/Format/UnwrappedLineFormatter.cpp
index 3392a055c0c67..22f27a668dccd 100644
--- a/clang/lib/Format/UnwrappedLineFormatter.cpp
+++ b/clang/lib/Format/UnwrappedLineFormatter.cpp
@@ -1230,7 +1230,8 @@ void UnwrappedLineFormatter::formatFirstToken(
 
   // If in Whitemsmiths mode, indent start and end of blocks
   if (Style.BreakBeforeBraces == FormatStyle::BS_Whitesmiths) {
-    if (RootToken.isOneOf(tok::l_brace, tok::r_brace, tok::kw_case))
+    if (RootToken.isOneOf(tok::l_brace, tok::r_brace, tok::kw_case,
+                          tok::kw_default))
       Indent += Style.IndentWidth;
   }
 
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 577b60bc51e22..a37386425aaed 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -2220,8 +2220,13 @@ void UnwrappedLineParser::parseLabel(bool LeftAlignLabel) {
     parseBlock(/*MustBeDeclaration=*/false);
     if (FormatTok->Tok.is(tok::kw_break)) {
       if (Style.BraceWrapping.AfterControlStatement ==
-          FormatStyle::BWACS_Always)
+          FormatStyle::BWACS_Always) {
         addUnwrappedLine();
+        if (!Style.IndentCaseBlocks &&
+            Style.BreakBeforeBraces == FormatStyle::BS_Whitesmiths) {
+          Line->Level++;
+        }
+      }
       parseStructuralElement();
     }
     addUnwrappedLine();
diff --git a/clang/lib/Frontend/ASTConsumers.cpp b/clang/lib/Frontend/ASTConsumers.cpp
index 043b2541b8f87..a73cc8876d5d8 100644
--- a/clang/lib/Frontend/ASTConsumers.cpp
+++ b/clang/lib/Frontend/ASTConsumers.cpp
@@ -36,10 +36,10 @@ namespace {
     enum Kind { DumpFull, Dump, Print, None };
     ASTPrinter(std::unique_ptr<raw_ostream> Out, Kind K,
                ASTDumpOutputFormat Format, StringRef FilterString,
-               bool DumpLookups = false)
+               bool DumpLookups = false, bool DumpDeclTypes = false)
         : Out(Out ? *Out : llvm::outs()), OwnedOut(std::move(Out)),
           OutputKind(K), OutputFormat(Format), FilterString(FilterString),
-          DumpLookups(DumpLookups) {}
+          DumpLookups(DumpLookups), DumpDeclTypes(DumpDeclTypes) {}
 
     void HandleTranslationUnit(ASTContext &Context) override {
       TranslationUnitDecl *D = Context.getTranslationUnitDecl();
@@ -91,8 +91,22 @@ namespace {
       } else if (OutputKind == Print) {
         PrintingPolicy Policy(D->getASTContext().getLangOpts());
         D->print(Out, Policy, /*Indentation=*/0, /*PrintInstantiation=*/true);
-      } else if (OutputKind != None)
+      } else if (OutputKind != None) {
         D->dump(Out, OutputKind == DumpFull, OutputFormat);
+      }
+
+      if (DumpDeclTypes) {
+        Decl *InnerD = D;
+        if (auto *TD = dyn_cast<TemplateDecl>(D))
+          InnerD = TD->getTemplatedDecl();
+
+        // FIXME: Support OutputFormat in type dumping.
+        // FIXME: Support combining -ast-dump-decl-types with -ast-dump-lookups.
+        if (auto *VD = dyn_cast<ValueDecl>(InnerD))
+          VD->getType().dump(Out, VD->getASTContext());
+        if (auto *TD = dyn_cast<TypeDecl>(InnerD))
+          TD->getTypeForDecl()->dump(Out, TD->getASTContext());
+      }
     }
 
     raw_ostream &Out;
@@ -111,6 +125,9 @@ namespace {
     /// results will be output with a format determined by OutputKind. This is
     /// incompatible with OutputKind == Print.
     bool DumpLookups;
+
+    /// Whether to dump the type for each declaration dumped.
+    bool DumpDeclTypes;
   };
 
   class ASTDeclNodeLister : public ASTConsumer,
@@ -146,13 +163,13 @@ clang::CreateASTPrinter(std::unique_ptr<raw_ostream> Out,
 std::unique_ptr<ASTConsumer>
 clang::CreateASTDumper(std::unique_ptr<raw_ostream> Out, StringRef FilterString,
                        bool DumpDecls, bool Deserialize, bool DumpLookups,
-                       ASTDumpOutputFormat Format) {
+                       bool DumpDeclTypes, ASTDumpOutputFormat Format) {
   assert((DumpDecls || Deserialize || DumpLookups) && "nothing to dump");
-  return std::make_unique<ASTPrinter>(std::move(Out),
-                                       Deserialize ? ASTPrinter::DumpFull :
-                                       DumpDecls ? ASTPrinter::Dump :
-                                       ASTPrinter::None, Format,
-                                       FilterString, DumpLookups);
+  return std::make_unique<ASTPrinter>(
+      std::move(Out),
+      Deserialize ? ASTPrinter::DumpFull
+                  : DumpDecls ? ASTPrinter::Dump : ASTPrinter::None,
+      Format, FilterString, DumpLookups, DumpDeclTypes);
 }
 
 std::unique_ptr<ASTConsumer> clang::CreateASTDeclNodeLister() {
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 0140a756b7dde..4613ed8d7f61f 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -815,17 +815,15 @@ std::unique_ptr<llvm::raw_pwrite_stream> CompilerInstance::createOutputFile(
 // Initialization Utilities
 
 bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input){
-  return InitializeSourceManager(
-      Input, getDiagnostics(), getFileManager(), getSourceManager(),
-      hasPreprocessor() ? &getPreprocessor().getHeaderSearchInfo() : nullptr,
-      getDependencyOutputOpts(), getFrontendOpts());
+  return InitializeSourceManager(Input, getDiagnostics(), getFileManager(),
+                                 getSourceManager());
 }
 
 // static
-bool CompilerInstance::InitializeSourceManager(
-    const FrontendInputFile &Input, DiagnosticsEngine &Diags,
-    FileManager &FileMgr, SourceManager &SourceMgr, HeaderSearch *HS,
-    DependencyOutputOptions &DepOpts, const FrontendOptions &Opts) {
+bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input,
+                                               DiagnosticsEngine &Diags,
+                                               FileManager &FileMgr,
+                                               SourceManager &SourceMgr) {
   SrcMgr::CharacteristicKind Kind =
       Input.getKind().getFormat() == InputKind::ModuleMap
           ? Input.isSystem() ? SrcMgr::C_System_ModuleMap
@@ -935,6 +933,19 @@ bool CompilerInstance::ExecuteAction(FrontendAction &Act) {
     setAuxTarget(TargetInfo::CreateTargetInfo(getDiagnostics(), TO));
   }
 
+  if (!getTarget().hasStrictFP() && !getLangOpts().ExpStrictFP) {
+    if (getLangOpts().getFPRoundingMode() !=
+        llvm::RoundingMode::NearestTiesToEven) {
+      getDiagnostics().Report(diag::warn_fe_backend_unsupported_fp_rounding);
+      getLangOpts().setFPRoundingMode(llvm::RoundingMode::NearestTiesToEven);
+    }
+    if (getLangOpts().getFPExceptionMode() != LangOptions::FPE_Ignore) {
+      getDiagnostics().Report(diag::warn_fe_backend_unsupported_fp_exceptions);
+      getLangOpts().setFPExceptionMode(LangOptions::FPE_Ignore);
+    }
+    // FIXME: can we disable FEnvAccess?
+  }
+
   // Inform the target of the language options.
   //
   // FIXME: We shouldn't need to do this, the target should be immutable once
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 1cd483081dc2b..863c6b3ca4f31 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -117,6 +117,62 @@ CompilerInvocationBase::CompilerInvocationBase(const CompilerInvocationBase &X)
 
 CompilerInvocationBase::~CompilerInvocationBase() = default;
 
+//===----------------------------------------------------------------------===//
+// Normalizers
+//===----------------------------------------------------------------------===//
+
+#define SIMPLE_ENUM_VALUE_TABLE
+#include "clang/Driver/Options.inc"
+#undef SIMPLE_ENUM_VALUE_TABLE
+
+static llvm::Optional<unsigned> normalizeSimpleEnum(OptSpecifier Opt,
+                                                    unsigned TableIndex,
+                                                    const ArgList &Args,
+                                                    DiagnosticsEngine &Diags) {
+  assert(TableIndex < SimpleEnumValueTablesSize);
+  const SimpleEnumValueTable &Table = SimpleEnumValueTables[TableIndex];
+
+  auto *Arg = Args.getLastArg(Opt);
+  if (!Arg)
+    return None;
+
+  StringRef ArgValue = Arg->getValue();
+  for (int I = 0, E = Table.Size; I != E; ++I)
+    if (ArgValue == Table.Table[I].Name)
+      return Table.Table[I].Value;
+
+  Diags.Report(diag::err_drv_invalid_value)
+      << Arg->getAsString(Args) << ArgValue;
+  return None;
+}
+
+static const char *denormalizeSimpleEnum(CompilerInvocation::StringAllocator SA,
+                                         unsigned TableIndex, unsigned Value) {
+  assert(TableIndex < SimpleEnumValueTablesSize);
+  const SimpleEnumValueTable &Table = SimpleEnumValueTables[TableIndex];
+  for (int I = 0, E = Table.Size; I != E; ++I)
+    if (Value == Table.Table[I].Value)
+      return Table.Table[I].Name;
+
+  llvm_unreachable("The simple enum value was not correctly defined in "
+                   "the tablegen option description");
+}
+
+static const char *denormalizeString(CompilerInvocation::StringAllocator SA,
+                                     unsigned TableIndex,
+                                     const std::string &Value) {
+  return SA(Value);
+}
+
+static Optional<std::string> normalizeTriple(OptSpecifier Opt, int TableIndex,
+                                             const ArgList &Args,
+                                             DiagnosticsEngine &Diags) {
+  auto *Arg = Args.getLastArg(Opt);
+  if (!Arg)
+    return None;
+  return llvm::Triple::normalize(Arg->getValue());
+}
+
 //===----------------------------------------------------------------------===//
 // Deserialization (from args)
 //===----------------------------------------------------------------------===//
@@ -528,25 +584,6 @@ static void ParseCommentArgs(CommentOptions &Opts, ArgList &Args) {
   Opts.ParseAllComments = Args.hasArg(OPT_fparse_all_comments);
 }
 
-static llvm::Reloc::Model getRelocModel(ArgList &Args,
-                                        DiagnosticsEngine &Diags) {
-  if (Arg *A = Args.getLastArg(OPT_mrelocation_model)) {
-    StringRef Value = A->getValue();
-    auto RM = llvm::StringSwitch<llvm::Optional<llvm::Reloc::Model>>(Value)
-                  .Case("static", llvm::Reloc::Static)
-                  .Case("pic", llvm::Reloc::PIC_)
-                  .Case("ropi", llvm::Reloc::ROPI)
-                  .Case("rwpi", llvm::Reloc::RWPI)
-                  .Case("ropi-rwpi", llvm::Reloc::ROPI_RWPI)
-                  .Case("dynamic-no-pic", llvm::Reloc::DynamicNoPIC)
-                  .Default(None);
-    if (RM.hasValue())
-      return *RM;
-    Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Value;
-  }
-  return llvm::Reloc::PIC_;
-}
-
 /// Create a new Regex instance out of the string value in \p RpassArg.
 /// It returns a pointer to the newly generated Regex instance.
 static std::shared_ptr<llvm::Regex>
@@ -750,7 +787,7 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
       Opts.setDebuggerTuning(static_cast<llvm::DebuggerKind>(Val));
   }
   Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 0, Diags);
-  Opts.DebugColumnInfo = Args.hasArg(OPT_dwarf_column_info);
+  Opts.DebugColumnInfo = !Args.hasArg(OPT_gno_column_info);
   Opts.EmitCodeView = Args.hasArg(OPT_gcodeview);
   Opts.CodeViewGHash = Args.hasArg(OPT_gcodeview_ghash);
   Opts.MacroDebugInfo = Args.hasArg(OPT_debug_info_macro);
@@ -823,7 +860,6 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
   Opts.RerollLoops = Args.hasArg(OPT_freroll_loops);
 
   Opts.DisableIntegratedAS = Args.hasArg(OPT_fno_integrated_as);
-  Opts.CallGraphProfile = !Opts.DisableIntegratedAS;
   Opts.Autolink = !Args.hasArg(OPT_fno_autolink);
   Opts.SampleProfileFile =
       std::string(Args.getLastArgValue(OPT_fprofile_sample_use_EQ));
@@ -905,7 +941,7 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
   Opts.StrictFloatCastOverflow =
       !Args.hasArg(OPT_fno_strict_float_cast_overflow);
 
-  Opts.NoZeroInitializedInBSS = Args.hasArg(OPT_mno_zero_initialized_in_bss);
+  Opts.NoZeroInitializedInBSS = Args.hasArg(OPT_fno_zero_initialized_in_bss);
   Opts.NumRegisterParameters = getLastArgIntValue(Args, OPT_mregparm, 0, Diags);
   Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack);
   Opts.SmallDataLimit =
@@ -927,7 +963,6 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
   Opts.StrictVTablePointers = Args.hasArg(OPT_fstrict_vtable_pointers);
   Opts.ForceEmitVTables = Args.hasArg(OPT_fforce_emit_vtables);
   Opts.UnwindTables = Args.hasArg(OPT_munwind_tables);
-  Opts.RelocationModel = getRelocModel(Args, Diags);
   Opts.ThreadModel =
       std::string(Args.getLastArgValue(OPT_mthread_model, "posix"));
   if (Opts.ThreadModel != "posix" && Opts.ThreadModel != "single")
@@ -1544,8 +1579,6 @@ bool clang::ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
   Opts.AbsolutePath = Args.hasArg(OPT_fdiagnostics_absolute_paths);
   Opts.ShowOptionNames = !Args.hasArg(OPT_fno_diagnostics_show_option);
 
-  llvm::sys::Process::UseANSIEscapeCodes(Args.hasArg(OPT_fansi_escape_codes));
-
   // Default behavior is to not to show note include stacks.
   Opts.ShowNoteIncludeStack = false;
   if (Arg *A = Args.getLastArg(OPT_fdiagnostics_show_note_include_stack,
@@ -1652,6 +1685,9 @@ bool clang::ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
   }
   Opts.MessageLength =
       getLastArgIntValue(Args, OPT_fmessage_length_EQ, 0, Diags);
+
+  Opts.UndefPrefixes = Args.getAllArgValues(OPT_Wundef_prefix_EQ);
+
   addDiagnosticArgs(Args, OPT_W_Group, OPT_W_value_Group, Opts.Warnings);
   addDiagnosticArgs(Args, OPT_R_Group, OPT_R_value_Group, Opts.Remarks);
 
@@ -1715,6 +1751,7 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
     case OPT_ast_dump:
     case OPT_ast_dump_all:
     case OPT_ast_dump_lookups:
+    case OPT_ast_dump_decl_types:
       Opts.ProgramAction = frontend::ASTDump; break;
     case OPT_ast_print:
       Opts.ProgramAction = frontend::ASTPrint; break;
@@ -1872,6 +1909,7 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
   Opts.ASTDumpAll = Args.hasArg(OPT_ast_dump_all, OPT_ast_dump_all_EQ);
   Opts.ASTDumpFilter = std::string(Args.getLastArgValue(OPT_ast_dump_filter));
   Opts.ASTDumpLookups = Args.hasArg(OPT_ast_dump_lookups);
+  Opts.ASTDumpDeclTypes = Args.hasArg(OPT_ast_dump_decl_types);
   Opts.UseGlobalModuleIndex = !Args.hasArg(OPT_fno_modules_global_index);
   Opts.GenerateGlobalModuleIndex = Opts.UseGlobalModuleIndex;
   Opts.ModuleMapFiles = Args.getAllArgValues(OPT_fmodule_map_file);
@@ -2105,7 +2143,6 @@ static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args,
     Opts.AddPrebuiltModulePath(A->getValue());
   Opts.DisableModuleHash = Args.hasArg(OPT_fdisable_module_hash);
   Opts.ModulesHashContent = Args.hasArg(OPT_fmodules_hash_content);
-  Opts.ModulesStrictContextHash = Args.hasArg(OPT_fmodules_strict_context_hash);
   Opts.ModulesValidateDiagnosticOptions =
       !Args.hasArg(OPT_fmodules_disable_diagnostic_validation);
   Opts.ImplicitModuleMaps = Args.hasArg(OPT_fimplicit_module_maps);
@@ -3062,8 +3099,8 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
     }
   }
 
-  // Check if -fopenmp is specified and set default version to 4.5.
-  Opts.OpenMP = Args.hasArg(options::OPT_fopenmp) ? 45 : 0;
+  // Check if -fopenmp is specified and set default version to 5.0.
+  Opts.OpenMP = Args.hasArg(options::OPT_fopenmp) ? 50 : 0;
   // Check if -fopenmp-simd is specified.
   bool IsSimdSpecified =
       Args.hasFlag(options::OPT_fopenmp_simd, options::OPT_fno_openmp_simd,
@@ -3081,10 +3118,8 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
   if (Opts.OpenMP || Opts.OpenMPSimd) {
     if (int Version = getLastArgIntValue(
             Args, OPT_fopenmp_version_EQ,
-            (IsSimdSpecified || IsTargetSpecified) ? 45 : Opts.OpenMP, Diags))
+            (IsSimdSpecified || IsTargetSpecified) ? 50 : Opts.OpenMP, Diags))
       Opts.OpenMP = Version;
-    else if (IsSimdSpecified || IsTargetSpecified)
-      Opts.OpenMP = 45;
     // Provide diagnostic when a given target is not expected to be an OpenMP
     // device or host.
     if (!Opts.OpenMPIsDevice) {
@@ -3160,6 +3195,12 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
   Opts.OpenMPCUDAMode = Opts.OpenMPIsDevice && (T.isNVPTX() || T.isAMDGCN()) &&
                         Args.hasArg(options::OPT_fopenmp_cuda_mode);
 
+  // Set CUDA support for parallel execution of target regions for OpenMP target
+  // NVPTX/AMDGCN if specified in options.
+  Opts.OpenMPCUDATargetParallel =
+      Opts.OpenMPIsDevice && (T.isNVPTX() || T.isAMDGCN()) &&
+      Args.hasArg(options::OPT_fopenmp_cuda_parallel_target_regions);
+
   // Set CUDA mode for OpenMP target NVPTX/AMDGCN if specified in options
   Opts.OpenMPCUDAForceFullRuntime =
       Opts.OpenMPIsDevice && (T.isNVPTX() || T.isAMDGCN()) &&
@@ -3239,6 +3280,9 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
       Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Val;
   }
 
+  if (Args.hasArg(OPT_fexperimental_strict_floating_point))
+    Opts.ExpStrictFP = true;
+
   auto FPRM = llvm::RoundingMode::NearestTiesToEven;
   if (Args.hasArg(OPT_frounding_math)) {
     FPRM = llvm::RoundingMode::Dynamic;
@@ -3607,11 +3651,6 @@ static void ParseTargetArgs(TargetOptions &Opts, ArgList &Args,
   Opts.FeaturesAsWritten = Args.getAllArgValues(OPT_target_feature);
   Opts.LinkerVersion =
       std::string(Args.getLastArgValue(OPT_target_linker_version));
-  Opts.Triple = std::string(Args.getLastArgValue(OPT_triple));
-  // Use the default target triple if unspecified.
-  if (Opts.Triple.empty())
-    Opts.Triple = llvm::sys::getDefaultTargetTriple();
-  Opts.Triple = llvm::Triple::normalize(Opts.Triple);
   Opts.OpenCLExtensionsAsWritten = Args.getAllArgValues(OPT_cl_ext_EQ);
   Opts.ForceEnableInt128 = Args.hasArg(OPT_fforce_enable_int128);
   Opts.NVPTXUseShortPointers = Args.hasFlag(
@@ -3626,6 +3665,31 @@ static void ParseTargetArgs(TargetOptions &Opts, ArgList &Args,
   }
 }
 
+bool CompilerInvocation::parseSimpleArgs(const ArgList &Args,
+                                         DiagnosticsEngine &Diags) {
+#define OPTION_WITH_MARSHALLING_FLAG(PREFIX_TYPE, NAME, ID, KIND, GROUP,       \
+                                     ALIAS, ALIASARGS, FLAGS, PARAM, HELPTEXT, \
+                                     METAVAR, VALUES, SPELLING, ALWAYS_EMIT,   \
+                                     KEYPATH, DEFAULT_VALUE, IS_POSITIVE)      \
+  this->KEYPATH = Args.hasArg(OPT_##ID) && IS_POSITIVE;
+
+#define OPTION_WITH_MARSHALLING_STRING(                                        \
+    PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
+    HELPTEXT, METAVAR, VALUES, SPELLING, ALWAYS_EMIT, KEYPATH, DEFAULT_VALUE,  \
+    TYPE, NORMALIZER, DENORMALIZER, TABLE_INDEX)                               \
+  {                                                                            \
+    if (auto MaybeValue = NORMALIZER(OPT_##ID, TABLE_INDEX, Args, Diags))      \
+      this->KEYPATH = static_cast<TYPE>(*MaybeValue);                          \
+    else                                                                       \
+      this->KEYPATH = DEFAULT_VALUE;                                           \
+  }
+
+#include "clang/Driver/Options.inc"
+#undef OPTION_WITH_MARSHALLING_STRING
+#undef OPTION_WITH_MARSHALLING_FLAG
+  return true;
+}
+
 bool CompilerInvocation::CreateFromArgs(CompilerInvocation &Res,
                                         ArrayRef<const char *> CommandLineArgs,
                                         DiagnosticsEngine &Diags,
@@ -3659,6 +3723,11 @@ bool CompilerInvocation::CreateFromArgs(CompilerInvocation &Res,
     Success = false;
   }
 
+  Success &= Res.parseSimpleArgs(Args, Diags);
+
+  llvm::sys::Process::UseANSIEscapeCodes(
+      Res.DiagnosticOpts->UseANSIEscapeCodes);
+
   Success &= ParseAnalyzerArgs(*Res.getAnalyzerOpts(), Args, Diags);
   Success &= ParseMigratorArgs(Res.getMigratorOpts(), Args);
   ParseDependencyOutputArgs(Res.getDependencyOutputOpts(), Args);
@@ -3787,6 +3856,10 @@ std::string CompilerInvocation::getModuleHash() const {
   for (StringRef Feature : LangOpts->ModuleFeatures)
     code = hash_combine(code, Feature);
 
+  code = hash_combine(code, LangOpts->ObjCRuntime);
+  const auto &BCN = LangOpts->CommentOpts.BlockCommandNames;
+  code = hash_combine(code, hash_combine_range(BCN.begin(), BCN.end()));
+
   // Extend the signature with the target options.
   code = hash_combine(code, TargetOpts->Triple, TargetOpts->CPU,
                       TargetOpts->ABI);
@@ -3866,6 +3939,33 @@ std::string CompilerInvocation::getModuleHash() const {
   return llvm::APInt(64, code).toString(36, /*Signed=*/false);
 }
 
+void CompilerInvocation::generateCC1CommandLine(
+    SmallVectorImpl<const char *> &Args, StringAllocator SA) const {
+#define OPTION_WITH_MARSHALLING_FLAG(PREFIX_TYPE, NAME, ID, KIND, GROUP,       \
+                                     ALIAS, ALIASARGS, FLAGS, PARAM, HELPTEXT, \
+                                     METAVAR, VALUES, SPELLING, ALWAYS_EMIT,   \
+                                     KEYPATH, DEFAULT_VALUE, IS_POSITIVE)      \
+  if ((FLAGS) & options::CC1Option &&                                            \
+      (ALWAYS_EMIT || this->KEYPATH != DEFAULT_VALUE))                         \
+    Args.push_back(SPELLING);
+
+#define OPTION_WITH_MARSHALLING_STRING(                                        \
+    PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
+    HELPTEXT, METAVAR, VALUES, SPELLING, ALWAYS_EMIT, KEYPATH, DEFAULT_VALUE,  \
+    NORMALIZER_RET_TY, NORMALIZER, DENORMALIZER, TABLE_INDEX)                  \
+  if (((FLAGS) & options::CC1Option) &&                                          \
+      (ALWAYS_EMIT || this->KEYPATH != DEFAULT_VALUE)) {                       \
+    if (Option::KIND##Class == Option::SeparateClass) {                        \
+      Args.push_back(SPELLING);                                                \
+      Args.push_back(DENORMALIZER(SA, TABLE_INDEX, this->KEYPATH));            \
+    }                                                                          \
+  }
+
+#include "clang/Driver/Options.inc"
+#undef OPTION_WITH_MARSHALLING_STRING
+#undef OPTION_WITH_MARSHALLING_FLAG
+}
+
 namespace clang {
 
 IntrusiveRefCntPtr<llvm::vfs::FileSystem>
diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp
index 3a86a8d05c49b..711e7336c8203 100644
--- a/clang/lib/Frontend/FrontendActions.cpp
+++ b/clang/lib/Frontend/FrontendActions.cpp
@@ -79,7 +79,8 @@ ASTDumpAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
   const FrontendOptions &Opts = CI.getFrontendOpts();
   return CreateASTDumper(nullptr /*Dump to stdout.*/, Opts.ASTDumpFilter,
                          Opts.ASTDumpDecls, Opts.ASTDumpAll,
-                         Opts.ASTDumpLookups, Opts.ASTDumpFormat);
+                         Opts.ASTDumpLookups, Opts.ASTDumpDeclTypes,
+                         Opts.ASTDumpFormat);
 }
 
 std::unique_ptr<ASTConsumer>
diff --git a/clang/lib/Frontend/InitHeaderSearch.cpp b/clang/lib/Frontend/InitHeaderSearch.cpp
index 159cd47f78316..16f1f1670e8de 100644
--- a/clang/lib/Frontend/InitHeaderSearch.cpp
+++ b/clang/lib/Frontend/InitHeaderSearch.cpp
@@ -381,6 +381,7 @@ void InitHeaderSearch::AddDefaultCPlusPlusIncludePaths(
   case llvm::Triple::Linux:
   case llvm::Triple::Hurd:
   case llvm::Triple::Solaris:
+  case llvm::Triple::AIX:
     llvm_unreachable("Include management is handled in the driver.");
     break;
   case llvm::Triple::Win32:
@@ -424,6 +425,7 @@ void InitHeaderSearch::AddDefaultIncludePaths(const LangOptions &Lang,
   case llvm::Triple::Hurd:
   case llvm::Triple::Solaris:
   case llvm::Triple::WASI:
+  case llvm::Triple::AIX:
     return;
 
   case llvm::Triple::Win32:
diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp
index ca3697606023c..6eef1e2376f6d 100644
--- a/clang/lib/Frontend/InitPreprocessor.cpp
+++ b/clang/lib/Frontend/InitPreprocessor.cpp
@@ -1076,12 +1076,12 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
     case 40:
       Builder.defineMacro("_OPENMP", "201307");
       break;
-    case 50:
-      Builder.defineMacro("_OPENMP", "201811");
+    case 45:
+      Builder.defineMacro("_OPENMP", "201511");
       break;
     default:
-      // Default version is OpenMP 4.5
-      Builder.defineMacro("_OPENMP", "201511");
+      // Default version is OpenMP 5.0
+      Builder.defineMacro("_OPENMP", "201811");
       break;
     }
   }
diff --git a/clang/lib/Frontend/Rewrite/CMakeLists.txt b/clang/lib/Frontend/Rewrite/CMakeLists.txt
index 61a22b5b13b4d..64aa651394dff 100644
--- a/clang/lib/Frontend/Rewrite/CMakeLists.txt
+++ b/clang/lib/Frontend/Rewrite/CMakeLists.txt
@@ -20,4 +20,7 @@ add_clang_library(clangRewriteFrontend
   clangLex
   clangRewrite
   clangSerialization
+
+  DEPENDS
+  omp_gen
   )
diff --git a/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp b/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp
index bb195ade1409b..e122b10e76d37 100644
--- a/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp
+++ b/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp
@@ -2590,7 +2590,7 @@ Stmt *RewriteModernObjC::RewriteObjCStringLiteral(ObjCStringLiteral *Exp) {
   Expr *Unop = UnaryOperator::Create(
       const_cast<ASTContext &>(*Context), DRE, UO_AddrOf,
       Context->getPointerType(DRE->getType()), VK_RValue, OK_Ordinary,
-      SourceLocation(), false, FPOptions(Context->getLangOpts()));
+      SourceLocation(), false, FPOptionsOverride());
   // cast to NSConstantString *
   CastExpr *cast = NoTypeInfoCStyleCastExpr(Context, Exp->getType(),
                                             CK_CPointerToObjCPointerCast, Unop);
@@ -3287,7 +3287,7 @@ Stmt *RewriteModernObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       SuperRep = UnaryOperator::Create(
           const_cast<ASTContext &>(*Context), SuperRep, UO_AddrOf,
           Context->getPointerType(SuperRep->getType()), VK_RValue, OK_Ordinary,
-          SourceLocation(), false, FPOptions(Context->getLangOpts()));
+          SourceLocation(), false, FPOptionsOverride());
       SuperRep = NoTypeInfoCStyleCastExpr(Context,
                                           Context->getPointerType(superType),
                                           CK_BitCast, SuperRep);
@@ -3305,7 +3305,7 @@ Stmt *RewriteModernObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       SuperRep = UnaryOperator::Create(
           const_cast<ASTContext &>(*Context), SuperRep, UO_AddrOf,
           Context->getPointerType(SuperRep->getType()), VK_RValue, OK_Ordinary,
-          SourceLocation(), false, FPOptions(Context->getLangOpts()));
+          SourceLocation(), false, FPOptionsOverride());
     }
     MsgExprs.push_back(SuperRep);
     break;
@@ -3382,7 +3382,7 @@ Stmt *RewriteModernObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       SuperRep = UnaryOperator::Create(
           const_cast<ASTContext &>(*Context), SuperRep, UO_AddrOf,
           Context->getPointerType(SuperRep->getType()), VK_RValue, OK_Ordinary,
-          SourceLocation(), false, FPOptions(Context->getLangOpts()));
+          SourceLocation(), false, FPOptionsOverride());
       SuperRep = NoTypeInfoCStyleCastExpr(Context,
                                Context->getPointerType(superType),
                                CK_BitCast, SuperRep);
@@ -4706,10 +4706,9 @@ Stmt *RewriteModernObjC::RewriteLocalVariableExternalStorage(DeclRefExpr *DRE) {
   if (VarDecl *Var = dyn_cast<VarDecl>(VD))
     if (!ImportedLocalExternalDecls.count(Var))
       return DRE;
-  Expr *Exp = UnaryOperator::Create(const_cast<ASTContext &>(*Context), DRE,
-                                    UO_Deref, DRE->getType(), VK_LValue,
-                                    OK_Ordinary, DRE->getLocation(), false,
-                                    FPOptions(Context->getLangOpts()));
+  Expr *Exp = UnaryOperator::Create(
+      const_cast<ASTContext &>(*Context), DRE, UO_Deref, DRE->getType(),
+      VK_LValue, OK_Ordinary, DRE->getLocation(), false, FPOptionsOverride());
   // Need parens to enforce precedence.
   ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(),
                                           Exp);
@@ -5300,7 +5299,7 @@ Stmt *RewriteModernObjC::SynthBlockInitExpr(BlockExpr *Exp,
       new (Context) DeclRefExpr(*Context, NewVD, false, Context->VoidPtrTy,
                                 VK_LValue, SourceLocation()),
       UO_AddrOf, Context->getPointerType(Context->VoidPtrTy), VK_RValue,
-      OK_Ordinary, SourceLocation(), false, FPOptions(Context->getLangOpts()));
+      OK_Ordinary, SourceLocation(), false, FPOptionsOverride());
   InitExprs.push_back(DescRefExpr);
 
   // Add initializers for any closure decl refs.
@@ -5317,10 +5316,9 @@ Stmt *RewriteModernObjC::SynthBlockInitExpr(BlockExpr *Exp,
         if (HasLocalVariableExternalStorage(*I)) {
           QualType QT = (*I)->getType();
           QT = Context->getPointerType(QT);
-          Exp = UnaryOperator::Create(const_cast<ASTContext &>(*Context), Exp,
-                                      UO_AddrOf, QT, VK_RValue, OK_Ordinary,
-                                      SourceLocation(), false,
-                                      FPOptions(Context->getLangOpts()));
+          Exp = UnaryOperator::Create(
+              const_cast<ASTContext &>(*Context), Exp, UO_AddrOf, QT, VK_RValue,
+              OK_Ordinary, SourceLocation(), false, FPOptionsOverride());
         }
       } else if (isTopLevelBlockPointerType((*I)->getType())) {
         FD = SynthBlockInitFunctionDecl((*I)->getName());
@@ -5335,10 +5333,9 @@ Stmt *RewriteModernObjC::SynthBlockInitExpr(BlockExpr *Exp,
         if (HasLocalVariableExternalStorage(*I)) {
           QualType QT = (*I)->getType();
           QT = Context->getPointerType(QT);
-          Exp = UnaryOperator::Create(const_cast<ASTContext &>(*Context), Exp,
-                                      UO_AddrOf, QT, VK_RValue, OK_Ordinary,
-                                      SourceLocation(), false,
-                                      FPOptions(Context->getLangOpts()));
+          Exp = UnaryOperator::Create(
+              const_cast<ASTContext &>(*Context), Exp, UO_AddrOf, QT, VK_RValue,
+              OK_Ordinary, SourceLocation(), false, FPOptionsOverride());
         }
 
       }
@@ -5379,7 +5376,7 @@ Stmt *RewriteModernObjC::SynthBlockInitExpr(BlockExpr *Exp,
         Exp = UnaryOperator::Create(
             const_cast<ASTContext &>(*Context), Exp, UO_AddrOf,
             Context->getPointerType(Exp->getType()), VK_RValue, OK_Ordinary,
-            SourceLocation(), false, FPOptions(Context->getLangOpts()));
+            SourceLocation(), false, FPOptionsOverride());
       Exp = NoTypeInfoCStyleCastExpr(Context, castT, CK_BitCast, Exp);
       InitExprs.push_back(Exp);
     }
@@ -5406,7 +5403,7 @@ Stmt *RewriteModernObjC::SynthBlockInitExpr(BlockExpr *Exp,
   NewRep = UnaryOperator::Create(
       const_cast<ASTContext &>(*Context), NewRep, UO_AddrOf,
       Context->getPointerType(NewRep->getType()), VK_RValue, OK_Ordinary,
-      SourceLocation(), false, FPOptions(Context->getLangOpts()));
+      SourceLocation(), false, FPOptionsOverride());
   NewRep = NoTypeInfoCStyleCastExpr(Context, FType, CK_BitCast,
                                     NewRep);
   // Put Paren around the call.
@@ -7494,7 +7491,7 @@ Stmt *RewriteModernObjC::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV) {
       BinaryOperator *addExpr = BinaryOperator::Create(
           *Context, castExpr, DRE, BO_Add,
           Context->getPointerType(Context->CharTy), VK_RValue, OK_Ordinary,
-          SourceLocation(), FPOptions(Context->getLangOpts()));
+          SourceLocation(), FPOptionsOverride());
       // Don't forget the parens to enforce the proper binding.
       ParenExpr *PE = new (Context) ParenExpr(SourceLocation(),
                                               SourceLocation(),
@@ -7545,10 +7542,9 @@ Stmt *RewriteModernObjC::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV) {
                                           CK_BitCast,
                                           PE);
 
-      Expr *Exp = UnaryOperator::Create(const_cast<ASTContext &>(*Context),
-                                        castExpr, UO_Deref, IvarT, VK_LValue,
-                                        OK_Ordinary, SourceLocation(), false,
-                                        FPOptions(Context->getLangOpts()));
+      Expr *Exp = UnaryOperator::Create(
+          const_cast<ASTContext &>(*Context), castExpr, UO_Deref, IvarT,
+          VK_LValue, OK_Ordinary, SourceLocation(), false, FPOptionsOverride());
       PE = new (Context) ParenExpr(OldRange.getBegin(),
                                    OldRange.getEnd(),
                                    Exp);
diff --git a/clang/lib/Frontend/Rewrite/RewriteObjC.cpp b/clang/lib/Frontend/Rewrite/RewriteObjC.cpp
index 8b618d90b47ec..3f320dc57aa69 100644
--- a/clang/lib/Frontend/Rewrite/RewriteObjC.cpp
+++ b/clang/lib/Frontend/Rewrite/RewriteObjC.cpp
@@ -2517,7 +2517,7 @@ Stmt *RewriteObjC::RewriteObjCStringLiteral(ObjCStringLiteral *Exp) {
   Expr *Unop = UnaryOperator::Create(
       const_cast<ASTContext &>(*Context), DRE, UO_AddrOf,
       Context->getPointerType(DRE->getType()), VK_RValue, OK_Ordinary,
-      SourceLocation(), false, FPOptions(Context->getLangOpts()));
+      SourceLocation(), false, FPOptionsOverride());
   // cast to NSConstantString *
   CastExpr *cast = NoTypeInfoCStyleCastExpr(Context, Exp->getType(),
                                             CK_CPointerToObjCPointerCast, Unop);
@@ -2718,7 +2718,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       SuperRep = UnaryOperator::Create(
           const_cast<ASTContext &>(*Context), SuperRep, UO_AddrOf,
           Context->getPointerType(SuperRep->getType()), VK_RValue, OK_Ordinary,
-          SourceLocation(), false, FPOptions(Context->getLangOpts()));
+          SourceLocation(), false, FPOptionsOverride());
       SuperRep = NoTypeInfoCStyleCastExpr(Context,
                                           Context->getPointerType(superType),
                                           CK_BitCast, SuperRep);
@@ -2736,7 +2736,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       SuperRep = UnaryOperator::Create(
           const_cast<ASTContext &>(*Context), SuperRep, UO_AddrOf,
           Context->getPointerType(SuperRep->getType()), VK_RValue, OK_Ordinary,
-          SourceLocation(), false, FPOptions(Context->getLangOpts()));
+          SourceLocation(), false, FPOptionsOverride());
     }
     MsgExprs.push_back(SuperRep);
     break;
@@ -2813,7 +2813,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       SuperRep = UnaryOperator::Create(
           const_cast<ASTContext &>(*Context), SuperRep, UO_AddrOf,
           Context->getPointerType(SuperRep->getType()), VK_RValue, OK_Ordinary,
-          SourceLocation(), false, FPOptions(Context->getLangOpts()));
+          SourceLocation(), false, FPOptionsOverride());
       SuperRep = NoTypeInfoCStyleCastExpr(Context,
                                Context->getPointerType(superType),
                                CK_BitCast, SuperRep);
@@ -2999,7 +2999,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
                                                    SourceLocation());
     BinaryOperator *lessThanExpr = BinaryOperator::Create(
         *Context, sizeofExpr, limit, BO_LE, Context->IntTy, VK_RValue,
-        OK_Ordinary, SourceLocation(), FPOptions(Context->getLangOpts()));
+        OK_Ordinary, SourceLocation(), FPOptionsOverride());
     // (sizeof(returnType) <= 8 ? objc_msgSend(...) : objc_msgSend_stret(...))
     ConditionalOperator *CondExpr =
       new (Context) ConditionalOperator(lessThanExpr,
@@ -3052,7 +3052,7 @@ Stmt *RewriteObjC::RewriteObjCProtocolExpr(ObjCProtocolExpr *Exp) {
   Expr *DerefExpr = UnaryOperator::Create(
       const_cast<ASTContext &>(*Context), DRE, UO_AddrOf,
       Context->getPointerType(DRE->getType()), VK_RValue, OK_Ordinary,
-      SourceLocation(), false, FPOptions(Context->getLangOpts()));
+      SourceLocation(), false, FPOptionsOverride());
   CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, DerefExpr->getType(),
                                                 CK_BitCast,
                                                 DerefExpr);
@@ -3877,10 +3877,9 @@ Stmt *RewriteObjC::RewriteLocalVariableExternalStorage(DeclRefExpr *DRE) {
   if (VarDecl *Var = dyn_cast<VarDecl>(VD))
     if (!ImportedLocalExternalDecls.count(Var))
       return DRE;
-  Expr *Exp = UnaryOperator::Create(const_cast<ASTContext &>(*Context), DRE,
-                                    UO_Deref, DRE->getType(), VK_LValue,
-                                    OK_Ordinary, DRE->getLocation(), false,
-                                    FPOptions(Context->getLangOpts()));
+  Expr *Exp = UnaryOperator::Create(
+      const_cast<ASTContext &>(*Context), DRE, UO_Deref, DRE->getType(),
+      VK_LValue, OK_Ordinary, DRE->getLocation(), false, FPOptionsOverride());
   // Need parens to enforce precedence.
   ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(),
                                           Exp);
@@ -4440,7 +4439,7 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
       new (Context) DeclRefExpr(*Context, NewVD, false, Context->VoidPtrTy,
                                 VK_LValue, SourceLocation()),
       UO_AddrOf, Context->getPointerType(Context->VoidPtrTy), VK_RValue,
-      OK_Ordinary, SourceLocation(), false, FPOptions(Context->getLangOpts()));
+      OK_Ordinary, SourceLocation(), false, FPOptionsOverride());
   InitExprs.push_back(DescRefExpr);
 
   // Add initializers for any closure decl refs.
@@ -4457,10 +4456,9 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
         if (HasLocalVariableExternalStorage(*I)) {
           QualType QT = (*I)->getType();
           QT = Context->getPointerType(QT);
-          Exp = UnaryOperator::Create(const_cast<ASTContext &>(*Context), Exp,
-                                      UO_AddrOf, QT, VK_RValue, OK_Ordinary,
-                                      SourceLocation(), false,
-                                      FPOptions(Context->getLangOpts()));
+          Exp = UnaryOperator::Create(
+              const_cast<ASTContext &>(*Context), Exp, UO_AddrOf, QT, VK_RValue,
+              OK_Ordinary, SourceLocation(), false, FPOptionsOverride());
         }
       } else if (isTopLevelBlockPointerType((*I)->getType())) {
         FD = SynthBlockInitFunctionDecl((*I)->getName());
@@ -4475,10 +4473,9 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
         if (HasLocalVariableExternalStorage(*I)) {
           QualType QT = (*I)->getType();
           QT = Context->getPointerType(QT);
-          Exp = UnaryOperator::Create(const_cast<ASTContext &>(*Context), Exp,
-                                      UO_AddrOf, QT, VK_RValue, OK_Ordinary,
-                                      SourceLocation(), false,
-                                      FPOptions(Context->getLangOpts()));
+          Exp = UnaryOperator::Create(
+              const_cast<ASTContext &>(*Context), Exp, UO_AddrOf, QT, VK_RValue,
+              OK_Ordinary, SourceLocation(), false, FPOptionsOverride());
         }
       }
       InitExprs.push_back(Exp);
@@ -4518,7 +4515,7 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
         Exp = UnaryOperator::Create(
             const_cast<ASTContext &>(*Context), Exp, UO_AddrOf,
             Context->getPointerType(Exp->getType()), VK_RValue, OK_Ordinary,
-            SourceLocation(), false, FPOptions(Context->getLangOpts()));
+            SourceLocation(), false, FPOptionsOverride());
       Exp = NoTypeInfoCStyleCastExpr(Context, castT, CK_BitCast, Exp);
       InitExprs.push_back(Exp);
     }
@@ -4537,7 +4534,7 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
   NewRep = UnaryOperator::Create(
       const_cast<ASTContext &>(*Context), NewRep, UO_AddrOf,
       Context->getPointerType(NewRep->getType()), VK_RValue, OK_Ordinary,
-      SourceLocation(), false, FPOptions(Context->getLangOpts()));
+      SourceLocation(), false, FPOptionsOverride());
   NewRep = NoTypeInfoCStyleCastExpr(Context, FType, CK_BitCast,
                                     NewRep);
   BlockDeclRefs.clear();
diff --git a/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp b/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp
index e3ca8fdec3934..462aeda6e027f 100644
--- a/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp
+++ b/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp
@@ -239,6 +239,9 @@ class SDiagsWriter : public DiagnosticConsumer {
   /// generated from child processes.
   bool MergeChildRecords;
 
+  /// Whether we've started finishing and tearing down this instance.
+  bool IsFinishing = false;
+
   /// State that is shared among the various clones of this diagnostic
   /// consumer.
   struct SharedState {
@@ -568,6 +571,17 @@ unsigned SDiagsWriter::getEmitDiagnosticFlag(StringRef FlagName) {
 
 void SDiagsWriter::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
                                     const Diagnostic &Info) {
+  assert(!IsFinishing &&
+         "Received a diagnostic after we've already started teardown.");
+  if (IsFinishing) {
+    SmallString<256> diagnostic;
+    Info.FormatDiagnostic(diagnostic);
+    getMetaDiags()->Report(
+        diag::warn_fe_serialized_diag_failure_during_finalisation)
+        << diagnostic;
+    return;
+  }
+
   // Enter the block for a non-note diagnostic immediately, rather than waiting
   // for beginDiagnostic, in case associated notes are emitted before we get
   // there.
@@ -761,6 +775,9 @@ void SDiagsWriter::RemoveOldDiagnostics() {
 }
 
 void SDiagsWriter::finish() {
+  assert(!IsFinishing);
+  IsFinishing = true;
+
   // The original instance is responsible for writing the file.
   if (!OriginalInstance)
     return;
@@ -786,12 +803,20 @@ void SDiagsWriter::finish() {
   if (EC) {
     getMetaDiags()->Report(diag::warn_fe_serialized_diag_failure)
         << State->OutputFile << EC.message();
+    OS->clear_error();
     return;
   }
 
   // Write the generated bitstream to "Out".
   OS->write((char *)&State->Buffer.front(), State->Buffer.size());
   OS->flush();
+
+  assert(!OS->has_error());
+  if (OS->has_error()) {
+    getMetaDiags()->Report(diag::warn_fe_serialized_diag_failure)
+        << State->OutputFile << OS->error().message();
+    OS->clear_error();
+  }
 }
 
 std::error_code SDiagsMerger::visitStartOfDiagnostic() {
diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index fd9e3a0d672f6..0692fe75a4417 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -2,6 +2,7 @@ set(files
   adxintrin.h
   altivec.h
   ammintrin.h
+  amxintrin.h
   arm_acle.h
   arm_cmse.h
   armintr.h
@@ -150,6 +151,8 @@ set(ppc_wrapper_files
 set(openmp_wrapper_files
   openmp_wrappers/math.h
   openmp_wrappers/cmath
+  openmp_wrappers/complex.h
+  openmp_wrappers/complex
   openmp_wrappers/__clang_openmp_device_functions.h
   openmp_wrappers/new
 )
diff --git a/clang/lib/Headers/__clang_cuda_cmath.h b/clang/lib/Headers/__clang_cuda_cmath.h
index f406112164e53..8ba182689a4f9 100644
--- a/clang/lib/Headers/__clang_cuda_cmath.h
+++ b/clang/lib/Headers/__clang_cuda_cmath.h
@@ -12,7 +12,7 @@
 #error "This file is for CUDA compilation only."
 #endif
 
-#ifndef _OPENMP
+#ifndef __OPENMP_NVPTX__
 #include <limits>
 #endif
 
@@ -32,7 +32,7 @@
 // implementation.  Declaring in the global namespace and pulling into namespace
 // std covers all of the known knowns.
 
-#ifdef _OPENMP
+#ifdef __OPENMP_NVPTX__
 #define __DEVICE__ static constexpr __attribute__((always_inline, nothrow))
 #else
 #define __DEVICE__ static __device__ __inline__ __attribute__((always_inline))
@@ -69,7 +69,7 @@ __DEVICE__ float frexp(float __arg, int *__exp) {
 // Windows. For OpenMP we omit these as some old system headers have
 // non-conforming `isinf(float)` and `isnan(float)` implementations that return
 // an `int`. The system versions of these functions should be fine anyway.
-#if !defined(_MSC_VER) && !defined(_OPENMP)
+#if !defined(_MSC_VER) && !defined(__OPENMP_NVPTX__)
 __DEVICE__ bool isinf(float __x) { return ::__isinff(__x); }
 __DEVICE__ bool isinf(double __x) { return ::__isinf(__x); }
 __DEVICE__ bool isfinite(float __x) { return ::__finitef(__x); }
@@ -146,7 +146,7 @@ __DEVICE__ float tanh(float __x) { return ::tanhf(__x); }
 // libdevice doesn't provide an implementation, and we don't want to be in the
 // business of implementing tricky libm functions in this header.
 
-#ifndef _OPENMP
+#ifndef __OPENMP_NVPTX__
 
 // Now we've defined everything we promised we'd define in
 // __clang_cuda_math_forward_declares.h.  We need to do two additional things to
@@ -463,7 +463,7 @@ _GLIBCXX_END_NAMESPACE_VERSION
 } // namespace std
 #endif
 
-#endif // _OPENMP
+#endif // __OPENMP_NVPTX__
 
 #undef __DEVICE__
 
diff --git a/clang/lib/Headers/__clang_cuda_complex_builtins.h b/clang/lib/Headers/__clang_cuda_complex_builtins.h
index 576a958b16bb2..8c10ff6b461fd 100644
--- a/clang/lib/Headers/__clang_cuda_complex_builtins.h
+++ b/clang/lib/Headers/__clang_cuda_complex_builtins.h
@@ -13,10 +13,57 @@
 // This header defines __muldc3, __mulsc3, __divdc3, and __divsc3.  These are
 // libgcc functions that clang assumes are available when compiling c99 complex
 // operations.  (These implementations come from libc++, and have been modified
-// to work with CUDA.)
+// to work with CUDA and OpenMP target offloading [in C and C++ mode].)
 
-extern "C" inline __device__ double _Complex __muldc3(double __a, double __b,
-                                                      double __c, double __d) {
+#pragma push_macro("__DEVICE__")
+#ifdef _OPENMP
+#pragma omp declare target
+#define __DEVICE__ __attribute__((noinline, nothrow, cold, weak))
+#else
+#define __DEVICE__ __device__ inline
+#endif
+
+// To make the algorithms available for C and C++ in CUDA and OpenMP we select
+// different but equivalent function versions. TODO: For OpenMP we currently
+// select the native builtins as the overload support for templates is lacking.
+#if !defined(_OPENMP)
+#define _ISNANd std::isnan
+#define _ISNANf std::isnan
+#define _ISINFd std::isinf
+#define _ISINFf std::isinf
+#define _ISFINITEd std::isfinite
+#define _ISFINITEf std::isfinite
+#define _COPYSIGNd std::copysign
+#define _COPYSIGNf std::copysign
+#define _SCALBNd std::scalbn
+#define _SCALBNf std::scalbn
+#define _ABSd std::abs
+#define _ABSf std::abs
+#define _LOGBd std::logb
+#define _LOGBf std::logb
+#else
+#define _ISNANd __nv_isnand
+#define _ISNANf __nv_isnanf
+#define _ISINFd __nv_isinfd
+#define _ISINFf __nv_isinff
+#define _ISFINITEd __nv_isfinited
+#define _ISFINITEf __nv_finitef
+#define _COPYSIGNd __nv_copysign
+#define _COPYSIGNf __nv_copysignf
+#define _SCALBNd __nv_scalbn
+#define _SCALBNf __nv_scalbnf
+#define _ABSd __nv_fabs
+#define _ABSf __nv_fabsf
+#define _LOGBd __nv_logb
+#define _LOGBf __nv_logbf
+#endif
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+__DEVICE__ double _Complex __muldc3(double __a, double __b, double __c,
+                                    double __d) {
   double __ac = __a * __c;
   double __bd = __b * __d;
   double __ad = __a * __d;
@@ -24,50 +71,49 @@ extern "C" inline __device__ double _Complex __muldc3(double __a, double __b,
   double _Complex z;
   __real__(z) = __ac - __bd;
   __imag__(z) = __ad + __bc;
-  if (std::isnan(__real__(z)) && std::isnan(__imag__(z))) {
+  if (_ISNANd(__real__(z)) && _ISNANd(__imag__(z))) {
     int __recalc = 0;
-    if (std::isinf(__a) || std::isinf(__b)) {
-      __a = std::copysign(std::isinf(__a) ? 1 : 0, __a);
-      __b = std::copysign(std::isinf(__b) ? 1 : 0, __b);
-      if (std::isnan(__c))
-        __c = std::copysign(0, __c);
-      if (std::isnan(__d))
-        __d = std::copysign(0, __d);
+    if (_ISINFd(__a) || _ISINFd(__b)) {
+      __a = _COPYSIGNd(_ISINFd(__a) ? 1 : 0, __a);
+      __b = _COPYSIGNd(_ISINFd(__b) ? 1 : 0, __b);
+      if (_ISNANd(__c))
+        __c = _COPYSIGNd(0, __c);
+      if (_ISNANd(__d))
+        __d = _COPYSIGNd(0, __d);
       __recalc = 1;
     }
-    if (std::isinf(__c) || std::isinf(__d)) {
-      __c = std::copysign(std::isinf(__c) ? 1 : 0, __c);
-      __d = std::copysign(std::isinf(__d) ? 1 : 0, __d);
-      if (std::isnan(__a))
-        __a = std::copysign(0, __a);
-      if (std::isnan(__b))
-        __b = std::copysign(0, __b);
+    if (_ISINFd(__c) || _ISINFd(__d)) {
+      __c = _COPYSIGNd(_ISINFd(__c) ? 1 : 0, __c);
+      __d = _COPYSIGNd(_ISINFd(__d) ? 1 : 0, __d);
+      if (_ISNANd(__a))
+        __a = _COPYSIGNd(0, __a);
+      if (_ISNANd(__b))
+        __b = _COPYSIGNd(0, __b);
       __recalc = 1;
     }
-    if (!__recalc && (std::isinf(__ac) || std::isinf(__bd) ||
-                      std::isinf(__ad) || std::isinf(__bc))) {
-      if (std::isnan(__a))
-        __a = std::copysign(0, __a);
-      if (std::isnan(__b))
-        __b = std::copysign(0, __b);
-      if (std::isnan(__c))
-        __c = std::copysign(0, __c);
-      if (std::isnan(__d))
-        __d = std::copysign(0, __d);
+    if (!__recalc &&
+        (_ISINFd(__ac) || _ISINFd(__bd) || _ISINFd(__ad) || _ISINFd(__bc))) {
+      if (_ISNANd(__a))
+        __a = _COPYSIGNd(0, __a);
+      if (_ISNANd(__b))
+        __b = _COPYSIGNd(0, __b);
+      if (_ISNANd(__c))
+        __c = _COPYSIGNd(0, __c);
+      if (_ISNANd(__d))
+        __d = _COPYSIGNd(0, __d);
       __recalc = 1;
     }
     if (__recalc) {
       // Can't use std::numeric_limits<double>::infinity() -- that doesn't have
       // a device overload (and isn't constexpr before C++11, naturally).
-      __real__(z) = __builtin_huge_valf() * (__a * __c - __b * __d);
-      __imag__(z) = __builtin_huge_valf() * (__a * __d + __b * __c);
+      __real__(z) = __builtin_huge_val() * (__a * __c - __b * __d);
+      __imag__(z) = __builtin_huge_val() * (__a * __d + __b * __c);
     }
   }
   return z;
 }
 
-extern "C" inline __device__ float _Complex __mulsc3(float __a, float __b,
-                                                     float __c, float __d) {
+__DEVICE__ float _Complex __mulsc3(float __a, float __b, float __c, float __d) {
   float __ac = __a * __c;
   float __bd = __b * __d;
   float __ad = __a * __d;
@@ -75,36 +121,36 @@ extern "C" inline __device__ float _Complex __mulsc3(float __a, float __b,
   float _Complex z;
   __real__(z) = __ac - __bd;
   __imag__(z) = __ad + __bc;
-  if (std::isnan(__real__(z)) && std::isnan(__imag__(z))) {
+  if (_ISNANf(__real__(z)) && _ISNANf(__imag__(z))) {
     int __recalc = 0;
-    if (std::isinf(__a) || std::isinf(__b)) {
-      __a = std::copysign(std::isinf(__a) ? 1 : 0, __a);
-      __b = std::copysign(std::isinf(__b) ? 1 : 0, __b);
-      if (std::isnan(__c))
-        __c = std::copysign(0, __c);
-      if (std::isnan(__d))
-        __d = std::copysign(0, __d);
+    if (_ISINFf(__a) || _ISINFf(__b)) {
+      __a = _COPYSIGNf(_ISINFf(__a) ? 1 : 0, __a);
+      __b = _COPYSIGNf(_ISINFf(__b) ? 1 : 0, __b);
+      if (_ISNANf(__c))
+        __c = _COPYSIGNf(0, __c);
+      if (_ISNANf(__d))
+        __d = _COPYSIGNf(0, __d);
       __recalc = 1;
     }
-    if (std::isinf(__c) || std::isinf(__d)) {
-      __c = std::copysign(std::isinf(__c) ? 1 : 0, __c);
-      __d = std::copysign(std::isinf(__d) ? 1 : 0, __d);
-      if (std::isnan(__a))
-        __a = std::copysign(0, __a);
-      if (std::isnan(__b))
-        __b = std::copysign(0, __b);
+    if (_ISINFf(__c) || _ISINFf(__d)) {
+      __c = _COPYSIGNf(_ISINFf(__c) ? 1 : 0, __c);
+      __d = _COPYSIGNf(_ISINFf(__d) ? 1 : 0, __d);
+      if (_ISNANf(__a))
+        __a = _COPYSIGNf(0, __a);
+      if (_ISNANf(__b))
+        __b = _COPYSIGNf(0, __b);
       __recalc = 1;
     }
-    if (!__recalc && (std::isinf(__ac) || std::isinf(__bd) ||
-                      std::isinf(__ad) || std::isinf(__bc))) {
-      if (std::isnan(__a))
-        __a = std::copysign(0, __a);
-      if (std::isnan(__b))
-        __b = std::copysign(0, __b);
-      if (std::isnan(__c))
-        __c = std::copysign(0, __c);
-      if (std::isnan(__d))
-        __d = std::copysign(0, __d);
+    if (!__recalc &&
+        (_ISINFf(__ac) || _ISINFf(__bd) || _ISINFf(__ad) || _ISINFf(__bc))) {
+      if (_ISNANf(__a))
+        __a = _COPYSIGNf(0, __a);
+      if (_ISNANf(__b))
+        __b = _COPYSIGNf(0, __b);
+      if (_ISNANf(__c))
+        __c = _COPYSIGNf(0, __c);
+      if (_ISNANf(__d))
+        __d = _COPYSIGNf(0, __d);
       __recalc = 1;
     }
     if (__recalc) {
@@ -115,36 +161,36 @@ extern "C" inline __device__ float _Complex __mulsc3(float __a, float __b,
   return z;
 }
 
-extern "C" inline __device__ double _Complex __divdc3(double __a, double __b,
-                                                      double __c, double __d) {
+__DEVICE__ double _Complex __divdc3(double __a, double __b, double __c,
+                                    double __d) {
   int __ilogbw = 0;
   // Can't use std::max, because that's defined in <algorithm>, and we don't
   // want to pull that in for every compile.  The CUDA headers define
   // ::max(float, float) and ::max(double, double), which is sufficient for us.
-  double __logbw = std::logb(max(std::abs(__c), std::abs(__d)));
-  if (std::isfinite(__logbw)) {
+  double __logbw = _LOGBd(max(_ABSd(__c), _ABSd(__d)));
+  if (_ISFINITEd(__logbw)) {
     __ilogbw = (int)__logbw;
-    __c = std::scalbn(__c, -__ilogbw);
-    __d = std::scalbn(__d, -__ilogbw);
+    __c = _SCALBNd(__c, -__ilogbw);
+    __d = _SCALBNd(__d, -__ilogbw);
   }
   double __denom = __c * __c + __d * __d;
   double _Complex z;
-  __real__(z) = std::scalbn((__a * __c + __b * __d) / __denom, -__ilogbw);
-  __imag__(z) = std::scalbn((__b * __c - __a * __d) / __denom, -__ilogbw);
-  if (std::isnan(__real__(z)) && std::isnan(__imag__(z))) {
-    if ((__denom == 0.0) && (!std::isnan(__a) || !std::isnan(__b))) {
-      __real__(z) = std::copysign(__builtin_huge_valf(), __c) * __a;
-      __imag__(z) = std::copysign(__builtin_huge_valf(), __c) * __b;
-    } else if ((std::isinf(__a) || std::isinf(__b)) && std::isfinite(__c) &&
-               std::isfinite(__d)) {
-      __a = std::copysign(std::isinf(__a) ? 1.0 : 0.0, __a);
-      __b = std::copysign(std::isinf(__b) ? 1.0 : 0.0, __b);
-      __real__(z) = __builtin_huge_valf() * (__a * __c + __b * __d);
-      __imag__(z) = __builtin_huge_valf() * (__b * __c - __a * __d);
-    } else if (std::isinf(__logbw) && __logbw > 0.0 && std::isfinite(__a) &&
-               std::isfinite(__b)) {
-      __c = std::copysign(std::isinf(__c) ? 1.0 : 0.0, __c);
-      __d = std::copysign(std::isinf(__d) ? 1.0 : 0.0, __d);
+  __real__(z) = _SCALBNd((__a * __c + __b * __d) / __denom, -__ilogbw);
+  __imag__(z) = _SCALBNd((__b * __c - __a * __d) / __denom, -__ilogbw);
+  if (_ISNANd(__real__(z)) && _ISNANd(__imag__(z))) {
+    if ((__denom == 0.0) && (!_ISNANd(__a) || !_ISNANd(__b))) {
+      __real__(z) = _COPYSIGNd(__builtin_huge_val(), __c) * __a;
+      __imag__(z) = _COPYSIGNd(__builtin_huge_val(), __c) * __b;
+    } else if ((_ISINFd(__a) || _ISINFd(__b)) && _ISFINITEd(__c) &&
+               _ISFINITEd(__d)) {
+      __a = _COPYSIGNd(_ISINFd(__a) ? 1.0 : 0.0, __a);
+      __b = _COPYSIGNd(_ISINFd(__b) ? 1.0 : 0.0, __b);
+      __real__(z) = __builtin_huge_val() * (__a * __c + __b * __d);
+      __imag__(z) = __builtin_huge_val() * (__b * __c - __a * __d);
+    } else if (_ISINFd(__logbw) && __logbw > 0.0 && _ISFINITEd(__a) &&
+               _ISFINITEd(__b)) {
+      __c = _COPYSIGNd(_ISINFd(__c) ? 1.0 : 0.0, __c);
+      __d = _COPYSIGNd(_ISINFd(__d) ? 1.0 : 0.0, __d);
       __real__(z) = 0.0 * (__a * __c + __b * __d);
       __imag__(z) = 0.0 * (__b * __c - __a * __d);
     }
@@ -152,33 +198,32 @@ extern "C" inline __device__ double _Complex __divdc3(double __a, double __b,
   return z;
 }
 
-extern "C" inline __device__ float _Complex __divsc3(float __a, float __b,
-                                                     float __c, float __d) {
+__DEVICE__ float _Complex __divsc3(float __a, float __b, float __c, float __d) {
   int __ilogbw = 0;
-  float __logbw = std::logb(max(std::abs(__c), std::abs(__d)));
-  if (std::isfinite(__logbw)) {
+  float __logbw = _LOGBf(max(_ABSf(__c), _ABSf(__d)));
+  if (_ISFINITEf(__logbw)) {
     __ilogbw = (int)__logbw;
-    __c = std::scalbn(__c, -__ilogbw);
-    __d = std::scalbn(__d, -__ilogbw);
+    __c = _SCALBNf(__c, -__ilogbw);
+    __d = _SCALBNf(__d, -__ilogbw);
   }
   float __denom = __c * __c + __d * __d;
   float _Complex z;
-  __real__(z) = std::scalbn((__a * __c + __b * __d) / __denom, -__ilogbw);
-  __imag__(z) = std::scalbn((__b * __c - __a * __d) / __denom, -__ilogbw);
-  if (std::isnan(__real__(z)) && std::isnan(__imag__(z))) {
-    if ((__denom == 0) && (!std::isnan(__a) || !std::isnan(__b))) {
-      __real__(z) = std::copysign(__builtin_huge_valf(), __c) * __a;
-      __imag__(z) = std::copysign(__builtin_huge_valf(), __c) * __b;
-    } else if ((std::isinf(__a) || std::isinf(__b)) && std::isfinite(__c) &&
-               std::isfinite(__d)) {
-      __a = std::copysign(std::isinf(__a) ? 1 : 0, __a);
-      __b = std::copysign(std::isinf(__b) ? 1 : 0, __b);
+  __real__(z) = _SCALBNf((__a * __c + __b * __d) / __denom, -__ilogbw);
+  __imag__(z) = _SCALBNf((__b * __c - __a * __d) / __denom, -__ilogbw);
+  if (_ISNANf(__real__(z)) && _ISNANf(__imag__(z))) {
+    if ((__denom == 0) && (!_ISNANf(__a) || !_ISNANf(__b))) {
+      __real__(z) = _COPYSIGNf(__builtin_huge_valf(), __c) * __a;
+      __imag__(z) = _COPYSIGNf(__builtin_huge_valf(), __c) * __b;
+    } else if ((_ISINFf(__a) || _ISINFf(__b)) && _ISFINITEf(__c) &&
+               _ISFINITEf(__d)) {
+      __a = _COPYSIGNf(_ISINFf(__a) ? 1 : 0, __a);
+      __b = _COPYSIGNf(_ISINFf(__b) ? 1 : 0, __b);
       __real__(z) = __builtin_huge_valf() * (__a * __c + __b * __d);
       __imag__(z) = __builtin_huge_valf() * (__b * __c - __a * __d);
-    } else if (std::isinf(__logbw) && __logbw > 0 && std::isfinite(__a) &&
-               std::isfinite(__b)) {
-      __c = std::copysign(std::isinf(__c) ? 1 : 0, __c);
-      __d = std::copysign(std::isinf(__d) ? 1 : 0, __d);
+    } else if (_ISINFf(__logbw) && __logbw > 0 && _ISFINITEf(__a) &&
+               _ISFINITEf(__b)) {
+      __c = _COPYSIGNf(_ISINFf(__c) ? 1 : 0, __c);
+      __d = _COPYSIGNf(_ISINFf(__d) ? 1 : 0, __d);
       __real__(z) = 0 * (__a * __c + __b * __d);
       __imag__(z) = 0 * (__b * __c - __a * __d);
     }
@@ -186,4 +231,29 @@ extern "C" inline __device__ float _Complex __divsc3(float __a, float __b,
   return z;
 }
 
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#undef _ISNANd
+#undef _ISNANf
+#undef _ISINFd
+#undef _ISINFf
+#undef _COPYSIGNd
+#undef _COPYSIGNf
+#undef _ISFINITEd
+#undef _ISFINITEf
+#undef _SCALBNd
+#undef _SCALBNf
+#undef _ABSd
+#undef _ABSf
+#undef _LOGBd
+#undef _LOGBf
+
+#ifdef _OPENMP
+#pragma omp end declare target
+#endif
+
+#pragma pop_macro("__DEVICE__")
+
 #endif // __CLANG_CUDA_COMPLEX_BUILTINS
diff --git a/clang/lib/Headers/__clang_cuda_device_functions.h b/clang/lib/Headers/__clang_cuda_device_functions.h
index 76c588997f187..f801e5426aa43 100644
--- a/clang/lib/Headers/__clang_cuda_device_functions.h
+++ b/clang/lib/Headers/__clang_cuda_device_functions.h
@@ -10,7 +10,7 @@
 #ifndef __CLANG_CUDA_DEVICE_FUNCTIONS_H__
 #define __CLANG_CUDA_DEVICE_FUNCTIONS_H__
 
-#ifndef _OPENMP
+#ifndef __OPENMP_NVPTX__
 #if CUDA_VERSION < 9000
 #error This file is intended to be used with CUDA-9+ only.
 #endif
@@ -20,7 +20,7 @@
 // we implement in this file. We need static in order to avoid emitting unused
 // functions and __forceinline__ helps inlining these wrappers at -O1.
 #pragma push_macro("__DEVICE__")
-#ifdef _OPENMP
+#ifdef __OPENMP_NVPTX__
 #define __DEVICE__ static __attribute__((always_inline, nothrow))
 #else
 #define __DEVICE__ static __device__ __forceinline__
@@ -1466,14 +1466,14 @@ __DEVICE__ unsigned int __vsubus4(unsigned int __a, unsigned int __b) {
 
 // For OpenMP we require the user to include <time.h> as we need to know what
 // clock_t is on the system.
-#ifndef _OPENMP
+#ifndef __OPENMP_NVPTX__
 __DEVICE__ /* clock_t= */ int clock() { return __nvvm_read_ptx_sreg_clock(); }
 #endif
 __DEVICE__ long long clock64() { return __nvvm_read_ptx_sreg_clock64(); }
 
 // These functions shouldn't be declared when including this header
 // for math function resolution purposes.
-#ifndef _OPENMP
+#ifndef __OPENMP_NVPTX__
 __DEVICE__ void *memcpy(void *__a, const void *__b, size_t __c) {
   return __builtin_memcpy(__a, __b, __c);
 }
diff --git a/clang/lib/Headers/__clang_cuda_libdevice_declares.h b/clang/lib/Headers/__clang_cuda_libdevice_declares.h
index 4d70353394c8f..6173b589e3eff 100644
--- a/clang/lib/Headers/__clang_cuda_libdevice_declares.h
+++ b/clang/lib/Headers/__clang_cuda_libdevice_declares.h
@@ -14,7 +14,7 @@
 extern "C" {
 #endif
 
-#if defined(_OPENMP)
+#if defined(__OPENMP_NVPTX__)
 #define __DEVICE__
 #elif defined(__CUDA__)
 #define __DEVICE__ __device__
diff --git a/clang/lib/Headers/__clang_cuda_math.h b/clang/lib/Headers/__clang_cuda_math.h
index 01db2f29af45b..332e616702acf 100644
--- a/clang/lib/Headers/__clang_cuda_math.h
+++ b/clang/lib/Headers/__clang_cuda_math.h
@@ -12,7 +12,7 @@
 #error "This file is for CUDA compilation only."
 #endif
 
-#ifndef _OPENMP
+#ifndef __OPENMP_NVPTX__
 #if CUDA_VERSION < 9000
 #error This file is intended to be used with CUDA-9+ only.
 #endif
@@ -22,7 +22,7 @@
 // we implement in this file. We need static in order to avoid emitting unused
 // functions and __forceinline__ helps inlining these wrappers at -O1.
 #pragma push_macro("__DEVICE__")
-#ifdef _OPENMP
+#ifdef __OPENMP_NVPTX__
 #if defined(__cplusplus)
 #define __DEVICE__ static constexpr __attribute__((always_inline, nothrow))
 #else
@@ -36,7 +36,7 @@
 // because the OpenMP overlay requires constexpr functions here but prior to
 // c++14 void return functions could not be constexpr.
 #pragma push_macro("__DEVICE_VOID__")
-#ifdef _OPENMP && defined(__cplusplus) && __cplusplus < 201402L
+#ifdef __OPENMP_NVPTX__ && defined(__cplusplus) && __cplusplus < 201402L
 #define __DEVICE_VOID__ static __attribute__((always_inline, nothrow))
 #else
 #define __DEVICE_VOID__ __DEVICE__
diff --git a/clang/lib/Headers/altivec.h b/clang/lib/Headers/altivec.h
index 385828aceced2..9a4009216930a 100644
--- a/clang/lib/Headers/altivec.h
+++ b/clang/lib/Headers/altivec.h
@@ -16777,6 +16777,56 @@ vec_pext(vector unsigned long long __a, vector unsigned long long __b) {
   return __builtin_altivec_vpextd(__a, __b);
 }
 
+/* vec_cfuge */
+
+static __inline__ vector unsigned long long __ATTRS_o_ai
+vec_cfuge(vector unsigned long long __a, vector unsigned long long __b) {
+  return __builtin_altivec_vcfuged(__a, __b);
+}
+
+/* vec_gnb */
+
+#define vec_gnb(__a, __b) __builtin_altivec_vgnb(__a, __b)
+
+/* vec_ternarylogic */
+#ifdef __VSX__
+#define vec_ternarylogic(__a, __b, __c, __imm)                                 \
+  _Generic((__a), vector unsigned char                                         \
+           : __builtin_vsx_xxeval((vector unsigned long long)(__a),            \
+                                  (vector unsigned long long)(__b),            \
+                                  (vector unsigned long long)(__c), (__imm)),  \
+             vector unsigned short                                             \
+           : __builtin_vsx_xxeval((vector unsigned long long)(__a),            \
+                                  (vector unsigned long long)(__b),            \
+                                  (vector unsigned long long)(__c), (__imm)),  \
+             vector unsigned int                                               \
+           : __builtin_vsx_xxeval((vector unsigned long long)(__a),            \
+                                  (vector unsigned long long)(__b),            \
+                                  (vector unsigned long long)(__c), (__imm)),  \
+             vector unsigned long long                                         \
+           : __builtin_vsx_xxeval((vector unsigned long long)(__a),            \
+                                  (vector unsigned long long)(__b),            \
+                                  (vector unsigned long long)(__c), (__imm)),  \
+             vector unsigned __int128                                          \
+           : __builtin_vsx_xxeval((vector unsigned long long)(__a),            \
+                                  (vector unsigned long long)(__b),            \
+                                  (vector unsigned long long)(__c), (__imm)))
+#endif /* __VSX__ */
+
+/* vec_genpcvm */
+
+#ifdef __VSX__
+#define vec_genpcvm(__a, __imm)                                                \
+  _Generic((__a), vector unsigned char                                         \
+           : __builtin_vsx_xxgenpcvbm((__a), (int)(__imm)),                    \
+             vector unsigned short                                             \
+           : __builtin_vsx_xxgenpcvhm((__a), (int)(__imm)),                    \
+             vector unsigned int                                               \
+           : __builtin_vsx_xxgenpcvwm((__a), (int)(__imm)),                    \
+             vector unsigned long long                                         \
+           : __builtin_vsx_xxgenpcvdm((__a), (int)(__imm)))
+#endif /* __VSX__ */
+
 /* vec_clrl */
 
 static __inline__ vector signed char __ATTRS_o_ai
@@ -16816,6 +16866,287 @@ vec_clrr(vector unsigned char __a, unsigned int __n) {
   return __builtin_altivec_vclrrb((vector signed char)__a, __n);
 #endif
 }
+
+/* vec_cntlzm */
+
+static __inline__ vector unsigned long long __ATTRS_o_ai
+vec_cntlzm(vector unsigned long long __a, vector unsigned long long __b) {
+  return __builtin_altivec_vclzdm(__a, __b);
+}
+
+/* vec_cnttzm */
+
+static __inline__ vector unsigned long long __ATTRS_o_ai
+vec_cnttzm(vector unsigned long long __a, vector unsigned long long __b) {
+  return __builtin_altivec_vctzdm(__a, __b);
+}
+
+/* vec_sldbi */
+
+#define vec_sldb(__a, __b, __c) __builtin_altivec_vsldbi(__a, __b, (__c & 0x7))
+
+/* vec_srdbi */
+
+#define vec_srdb(__a, __b, __c) __builtin_altivec_vsrdbi(__a, __b, (__c & 0x7))
+
+/* vec_insertl */
+
+static __inline__ vector unsigned char __ATTRS_o_ai
+vec_insertl(unsigned char __a, vector unsigned char __b, unsigned int __c) {
+#ifdef __LITTLE_ENDIAN__
+  return __builtin_altivec_vinsbrx(__b, __c, __a);
+#else
+  return __builtin_altivec_vinsblx(__b, __c, __a);
+#endif
+}
+
+static __inline__ vector unsigned short __ATTRS_o_ai
+vec_insertl(unsigned short __a, vector unsigned short __b, unsigned int __c) {
+#ifdef __LITTLE_ENDIAN__
+  return __builtin_altivec_vinshrx(__b, __c, __a);
+#else
+  return __builtin_altivec_vinshlx(__b, __c, __a);
+#endif
+}
+
+static __inline__ vector unsigned int __ATTRS_o_ai
+vec_insertl(unsigned int __a, vector unsigned int __b, unsigned int __c) {
+#ifdef __LITTLE_ENDIAN__
+  return __builtin_altivec_vinswrx(__b, __c, __a);
+#else
+  return __builtin_altivec_vinswlx(__b, __c, __a);
+#endif
+}
+
+static __inline__ vector unsigned long long __ATTRS_o_ai
+vec_insertl(unsigned long long __a, vector unsigned long long __b,
+            unsigned int __c) {
+#ifdef __LITTLE_ENDIAN__
+  return __builtin_altivec_vinsdrx(__b, __c, __a);
+#else
+  return __builtin_altivec_vinsdlx(__b, __c, __a);
+#endif
+}
+
+static __inline__ vector unsigned char __ATTRS_o_ai
+vec_insertl(vector unsigned char __a, vector unsigned char __b,
+            unsigned int __c) {
+#ifdef __LITTLE_ENDIAN__
+  return __builtin_altivec_vinsbvrx(__b, __c, __a);
+#else
+  return __builtin_altivec_vinsbvlx(__b, __c, __a);
+#endif
+}
+
+static __inline__ vector unsigned short __ATTRS_o_ai
+vec_insertl(vector unsigned short __a, vector unsigned short __b,
+            unsigned int __c) {
+#ifdef __LITTLE_ENDIAN__
+  return __builtin_altivec_vinshvrx(__b, __c, __a);
+#else
+  return __builtin_altivec_vinshvlx(__b, __c, __a);
+#endif
+}
+
+static __inline__ vector unsigned int __ATTRS_o_ai
+vec_insertl(vector unsigned int __a, vector unsigned int __b,
+            unsigned int __c) {
+#ifdef __LITTLE_ENDIAN__
+  return __builtin_altivec_vinswvrx(__b, __c, __a);
+#else
+  return __builtin_altivec_vinswvlx(__b, __c, __a);
+#endif
+}
+
+/* vec_inserth */
+
+static __inline__ vector unsigned char __ATTRS_o_ai
+vec_inserth(unsigned char __a, vector unsigned char __b, unsigned int __c) {
+#ifdef __LITTLE_ENDIAN__
+  return __builtin_altivec_vinsblx(__b, __c, __a);
+#else
+  return __builtin_altivec_vinsbrx(__b, __c, __a);
+#endif
+}
+
+static __inline__ vector unsigned short __ATTRS_o_ai
+vec_inserth(unsigned short __a, vector unsigned short __b, unsigned int __c) {
+#ifdef __LITTLE_ENDIAN__
+  return __builtin_altivec_vinshlx(__b, __c, __a);
+#else
+  return __builtin_altivec_vinshrx(__b, __c, __a);
+#endif
+}
+
+static __inline__ vector unsigned int __ATTRS_o_ai
+vec_inserth(unsigned int __a, vector unsigned int __b, unsigned int __c) {
+#ifdef __LITTLE_ENDIAN__
+  return __builtin_altivec_vinswlx(__b, __c, __a);
+#else
+  return __builtin_altivec_vinswrx(__b, __c, __a);
+#endif
+}
+
+static __inline__ vector unsigned long long __ATTRS_o_ai
+vec_inserth(unsigned long long __a, vector unsigned long long __b,
+            unsigned int __c) {
+#ifdef __LITTLE_ENDIAN__
+  return __builtin_altivec_vinsdlx(__b, __c, __a);
+#else
+  return __builtin_altivec_vinsdrx(__b, __c, __a);
+#endif
+}
+
+static __inline__ vector unsigned char __ATTRS_o_ai
+vec_inserth(vector unsigned char __a, vector unsigned char __b,
+            unsigned int __c) {
+#ifdef __LITTLE_ENDIAN__
+  return __builtin_altivec_vinsbvlx(__b, __c, __a);
+#else
+  return __builtin_altivec_vinsbvrx(__b, __c, __a);
+#endif
+}
+
+static __inline__ vector unsigned short __ATTRS_o_ai
+vec_inserth(vector unsigned short __a, vector unsigned short __b,
+            unsigned int __c) {
+#ifdef __LITTLE_ENDIAN__
+  return __builtin_altivec_vinshvlx(__b, __c, __a);
+#else
+  return __builtin_altivec_vinshvrx(__b, __c, __a);
+#endif
+}
+
+static __inline__ vector unsigned int __ATTRS_o_ai
+vec_inserth(vector unsigned int __a, vector unsigned int __b,
+            unsigned int __c) {
+#ifdef __LITTLE_ENDIAN__
+  return __builtin_altivec_vinswvlx(__b, __c, __a);
+#else
+  return __builtin_altivec_vinswvrx(__b, __c, __a);
+#endif
+}
+
+#ifdef __VSX__
+
+/* vec_permx */
+
+#define vec_permx(__a, __b, __c, __d)                                          \
+  __builtin_vsx_xxpermx((__a), (__b), (__c), (__d))
+
+/* vec_blendv */
+
+static __inline__ vector signed char __ATTRS_o_ai
+vec_blendv(vector signed char __a, vector signed char __b,
+           vector unsigned char __c) {
+  return __builtin_vsx_xxblendvb(__a, __b, __c);
+}
+
+static __inline__ vector unsigned char __ATTRS_o_ai
+vec_blendv(vector unsigned char __a, vector unsigned char __b,
+           vector unsigned char __c) {
+  return __builtin_vsx_xxblendvb(__a, __b, __c);
+}
+
+static __inline__ vector signed short __ATTRS_o_ai
+vec_blendv(vector signed short __a, vector signed short __b,
+           vector unsigned short __c) {
+  return __builtin_vsx_xxblendvh(__a, __b, __c);
+}
+
+static __inline__ vector unsigned short __ATTRS_o_ai
+vec_blendv(vector unsigned short __a, vector unsigned short __b,
+           vector unsigned short __c) {
+  return __builtin_vsx_xxblendvh(__a, __b, __c);
+}
+
+static __inline__ vector signed int __ATTRS_o_ai
+vec_blendv(vector signed int __a, vector signed int __b,
+           vector unsigned int __c) {
+  return __builtin_vsx_xxblendvw(__a, __b, __c);
+}
+
+static __inline__ vector unsigned int __ATTRS_o_ai
+vec_blendv(vector unsigned int __a, vector unsigned int __b,
+           vector unsigned int __c) {
+  return __builtin_vsx_xxblendvw(__a, __b, __c);
+}
+
+static __inline__ vector signed long long __ATTRS_o_ai
+vec_blendv(vector signed long long __a, vector signed long long __b,
+           vector unsigned long long __c) {
+  return __builtin_vsx_xxblendvd(__a, __b, __c);
+}
+
+static __inline__ vector unsigned long long __ATTRS_o_ai
+vec_blendv(vector unsigned long long __a, vector unsigned long long __b,
+           vector unsigned long long __c) {
+  return __builtin_vsx_xxblendvd(__a, __b, __c);
+}
+
+static __inline__ vector float __ATTRS_o_ai
+vec_blendv(vector float __a, vector float __b, vector unsigned int __c) {
+  return __builtin_vsx_xxblendvw(__a, __b, __c);
+}
+
+static __inline__ vector double __ATTRS_o_ai
+vec_blendv(vector double __a, vector double __b,
+           vector unsigned long long __c) {
+  return __builtin_vsx_xxblendvd(__a, __b, __c);
+}
+
+/* vec_splati */
+
+#define vec_splati(__a)                                                        \
+  _Generic((__a), signed int                                                   \
+           : ((vector signed int)__a), unsigned int                            \
+           : ((vector unsigned int)__a), float                                 \
+           : ((vector float)__a))
+
+/* vec_spatid */
+
+static __inline__ vector double __ATTRS_o_ai vec_splatid(const float __a) {
+  return ((vector double)((double)__a));
+}
+
+/* vec_splati_ins */
+
+static __inline__ vector signed int __ATTRS_o_ai vec_splati_ins(
+    vector signed int __a, const unsigned int __b, const signed int __c) {
+#ifdef __LITTLE_ENDIAN__
+  __a[1 - __b] = __c;
+  __a[3 - __b] = __c;
+#else
+  __a[__b] = __c;
+  __a[2 + __b] = __c;
+#endif
+  return __a;
+}
+
+static __inline__ vector unsigned int __ATTRS_o_ai vec_splati_ins(
+    vector unsigned int __a, const unsigned int __b, const unsigned int __c) {
+#ifdef __LITTLE_ENDIAN__
+  __a[1 - __b] = __c;
+  __a[3 - __b] = __c;
+#else
+  __a[__b] = __c;
+  __a[2 + __b] = __c;
+#endif
+  return __a;
+}
+
+static __inline__ vector float __ATTRS_o_ai
+vec_splati_ins(vector float __a, const unsigned int __b, const float __c) {
+#ifdef __LITTLE_ENDIAN__
+  __a[1 - __b] = __c;
+  __a[3 - __b] = __c;
+#else
+  __a[__b] = __c;
+  __a[2 + __b] = __c;
+#endif
+  return __a;
+}
+#endif /* __VSX__ */
 #endif /* __POWER10_VECTOR__ */
 
 #undef __ATTRS_o_ai
diff --git a/clang/lib/Headers/amxintrin.h b/clang/lib/Headers/amxintrin.h
new file mode 100644
index 0000000000000..58254e21c81a6
--- /dev/null
+++ b/clang/lib/Headers/amxintrin.h
@@ -0,0 +1,225 @@
+/*===--------------- amxintrin.h - AMX intrinsics -*- C/C++ -*---------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===------------------------------------------------------------------------===
+ */
+
+#ifndef __IMMINTRIN_H
+#error "Never use <amxintrin.h> directly; include <immintrin.h> instead."
+#endif /* __IMMINTRIN_H */
+
+#ifndef __AMXINTRIN_H
+#define __AMXINTRIN_H
+#ifdef __x86_64__
+
+#define __DEFAULT_FN_ATTRS \
+  __attribute__((__always_inline__, __nodebug__,  __target__("amx-tile")))
+
+/// Load tile configuration from a 64-byte memory location specified by
+/// "mem_addr". The tile configuration includes the tile type palette, the
+/// number of bytes per row, and the number of rows. If the specified
+/// palette_id is zero, that signifies the init state for both the tile
+/// config and the tile data, and the tiles are zeroed. Any invalid
+/// configurations will result in #GP fault.
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> LDTILECFG </c> instruction.
+///
+/// \param __config
+///    A pointer to 512-bits configuration
+static __inline__ void __DEFAULT_FN_ATTRS
+_tile_loadconfig(const void *__config)
+{
+  __builtin_ia32_tile_loadconfig(__config);
+}
+
+/// Stores the current tile configuration to a 64-byte memory location
+/// specified by "mem_addr". The tile configuration includes the tile type
+/// palette, the number of bytes per row, and the number of rows. If tiles
+/// are not configured, all zeroes will be stored to memory.
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> STTILECFG </c> instruction.
+///
+/// \param __config
+///    A pointer to 512-bits configuration
+static __inline__ void __DEFAULT_FN_ATTRS
+_tile_storeconfig(void *__config)
+{
+  __builtin_ia32_tile_storeconfig(__config);
+}
+
+/// Release the tile configuration to return to the init state, which
+/// releases all storage it currently holds.
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> TILERELEASE </c> instruction.
+static __inline__ void __DEFAULT_FN_ATTRS
+_tile_release(void)
+{
+  __builtin_ia32_tilerelease();
+}
+
+/// Load tile rows from memory specifieid by "base" address and "stride" into
+/// destination tile "dst" using the tile configuration previously configured
+/// via "_tile_loadconfig".
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> TILELOADD </c> instruction.
+///
+/// \param dst
+///    A destination tile. Max size is 1024 Bytes.
+/// \param base
+///    A pointer to base address.
+/// \param stride
+///    The stride between the rows' data to be loaded in memory.
+#define _tile_loadd(dst, base, stride) \
+  __builtin_ia32_tileloadd64((dst), ((const void *)(base)), (__SIZE_TYPE__)(stride))
+
+/// Load tile rows from memory specifieid by "base" address and "stride" into
+/// destination tile "dst" using the tile configuration previously configured
+/// via "_tile_loadconfig". This intrinsic provides a hint to the implementation
+/// that the data will likely not be reused in the near future and the data
+/// caching can be optimized accordingly.
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> TILELOADDT1 </c> instruction.
+///
+/// \param dst
+///    A destination tile. Max size is 1024 Bytes.
+/// \param base
+///    A pointer to base address.
+/// \param stride
+///    The stride between the rows' data to be loaded in memory.
+#define _tile_stream_loadd(dst, base, stride) \
+  __builtin_ia32_tileloaddt164((dst), ((const void *)(base)), (__SIZE_TYPE__)(stride))
+
+/// Store the tile specified by "src" to memory specifieid by "base" address and
+/// "stride" using the tile configuration previously configured via
+/// "_tile_loadconfig".
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> TILESTORED </c> instruction.
+///
+/// \param dst
+///    A destination tile. Max size is 1024 Bytes.
+/// \param base
+///    A pointer to base address.
+/// \param stride
+///    The stride between the rows' data to be stored in memory.
+#define _tile_stored(dst, base, stride) \
+  __builtin_ia32_tilestored64((dst), ((void *)(base)), (__SIZE_TYPE__)(stride))
+
+/// Zero the tile specified by "tdest".
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> TILEZERO </c> instruction.
+///
+/// \param tile
+///    The destination tile to be zero. Max size is 1024 Bytes.
+#define _tile_zero(tile) __builtin_ia32_tilezero((tile))
+
+/// Compute dot-product of bytes in tiles with a source/destination accumulator.
+/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in src0 with
+/// corresponding signed 8-bit integers in src1, producing 4 intermediate 32-bit
+/// results. Sum these 4 results with the corresponding 32-bit integer in "dst",
+/// and store the 32-bit result back to tile "dst".
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> TDPBSSD </c> instruction.
+///
+/// \param dst
+///    The destination tile. Max size is 1024 Bytes.
+/// \param src0
+///    The 1st source tile. Max size is 1024 Bytes.
+/// \param src1
+///    The 2nd source tile. Max size is 1024 Bytes.
+#define _tile_dpbssd(dst, src0, src1) __builtin_ia32_tdpbssd((dst), (src0), (src1))
+
+/// Compute dot-product of bytes in tiles with a source/destination accumulator.
+/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in src0 with
+/// corresponding unsigned 8-bit integers in src1, producing 4 intermediate
+/// 32-bit results. Sum these 4 results with the corresponding 32-bit integer
+/// in "dst", and store the 32-bit result back to tile "dst".
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> TDPBSUD </c> instruction.
+///
+/// \param dst
+///    The destination tile. Max size is 1024 Bytes.
+/// \param src0
+///    The 1st source tile. Max size is 1024 Bytes.
+/// \param src1
+///    The 2nd source tile. Max size is 1024 Bytes.
+#define _tile_dpbsud(dst, src0, src1) __builtin_ia32_tdpbsud((dst), (src0), (src1))
+
+/// Compute dot-product of bytes in tiles with a source/destination accumulator.
+/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in src0 with
+/// corresponding signed 8-bit integers in src1, producing 4 intermediate 32-bit
+/// results. Sum these 4 results with the corresponding 32-bit integer in "dst",
+/// and store the 32-bit result back to tile "dst".
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> TDPBUSD </c> instruction.
+///
+/// \param dst
+///    The destination tile. Max size is 1024 Bytes.
+/// \param src0
+///    The 1st source tile. Max size is 1024 Bytes.
+/// \param src1
+///    The 2nd source tile. Max size is 1024 Bytes.
+#define _tile_dpbusd(dst, src0, src1) __builtin_ia32_tdpbusd((dst), (src0), (src1))
+
+/// Compute dot-product of bytes in tiles with a source/destination accumulator.
+/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in src0 with
+/// corresponding unsigned 8-bit integers in src1, producing 4 intermediate
+/// 32-bit results. Sum these 4 results with the corresponding 32-bit integer in
+/// "dst", and store the 32-bit result back to tile "dst".
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> TDPBUUD </c> instruction.
+///
+/// \param dst
+///    The destination tile. Max size is 1024 Bytes.
+/// \param src0
+///    The 1st source tile. Max size is 1024 Bytes.
+/// \param src1
+///    The 2nd source tile. Max size is 1024 Bytes.
+#define _tile_dpbuud(dst, src0, src1) __builtin_ia32_tdpbuud((dst), (src0), (src1))
+
+/// Compute dot-product of BF16 (16-bit) floating-point pairs in tiles src0 and
+/// src1, accumulating the intermediate single-precision (32-bit) floating-point
+/// elements with elements in "dst", and store the 32-bit result back to tile
+/// "dst".
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> TDPBF16PS </c> instruction.
+///
+/// \param dst
+///    The destination tile. Max size is 1024 Bytes.
+/// \param src0
+///    The 1st source tile. Max size is 1024 Bytes.
+/// \param src1
+///    The 2nd source tile. Max size is 1024 Bytes.
+#define _tile_dpbf16ps(dst, src0, src1) \
+  __builtin_ia32_tdpbf16ps((dst), (src0), (src1))
+
+#undef __DEFAULT_FN_ATTRS
+
+#endif /* __x86_64__ */
+#endif /* __AMXINTRIN_H */
diff --git a/clang/lib/Headers/cpuid.h b/clang/lib/Headers/cpuid.h
index 6c38b578b30e5..2a88c042d0468 100644
--- a/clang/lib/Headers/cpuid.h
+++ b/clang/lib/Headers/cpuid.h
@@ -190,6 +190,9 @@
 #define bit_TSXLDTRK      0x00010000
 #define bit_PCONFIG       0x00040000
 #define bit_IBT           0x00100000
+#define bit_AMXBF16       0x00400000
+#define bit_AMXTILE       0x01000000
+#define bit_AMXINT8       0x02000000
 
 /* Features in %eax for leaf 7 sub-leaf 1 */
 #define bit_AVX512BF16    0x00000020
diff --git a/clang/lib/Headers/immintrin.h b/clang/lib/Headers/immintrin.h
index dd27ca2f6605b..e9dff2310fdf7 100644
--- a/clang/lib/Headers/immintrin.h
+++ b/clang/lib/Headers/immintrin.h
@@ -471,6 +471,11 @@ _storebe_i64(void * __P, long long __D) {
 #include <invpcidintrin.h>
 #endif
 
+#if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
+    defined(__AMXTILE__) || defined(__AMXINT8__) || defined(__AMXBF16__)
+#include <amxintrin.h>
+#endif
+
 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
     defined(__AVX512VP2INTERSECT__)
 #include <avx512vp2intersectintrin.h>
diff --git a/clang/lib/Headers/intrin.h b/clang/lib/Headers/intrin.h
index f85f7a2beb495..871b47ca82674 100644
--- a/clang/lib/Headers/intrin.h
+++ b/clang/lib/Headers/intrin.h
@@ -289,6 +289,9 @@ unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask);
 static __inline__
 unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask);
 
+#endif
+
+#if defined(__i386__) || defined(__x86_64__) || defined(__arm__) || defined(__aarch64__)
 static __inline__
 __int64 _InterlockedDecrement64(__int64 volatile *_Addend);
 static __inline__
diff --git a/clang/lib/Headers/openmp_wrappers/__clang_openmp_device_functions.h b/clang/lib/Headers/openmp_wrappers/__clang_openmp_device_functions.h
index 9ff0a186273ae..406c9748e286e 100644
--- a/clang/lib/Headers/openmp_wrappers/__clang_openmp_device_functions.h
+++ b/clang/lib/Headers/openmp_wrappers/__clang_openmp_device_functions.h
@@ -22,11 +22,15 @@ extern "C" {
 #endif
 
 #define __CUDA__
+#define __OPENMP_NVPTX__
+
 /// Include declarations for libdevice functions.
 #include <__clang_cuda_libdevice_declares.h>
 
 /// Provide definitions for these functions.
 #include <__clang_cuda_device_functions.h>
+
+#undef __OPENMP_NVPTX__
 #undef __CUDA__
 
 #ifdef __cplusplus
diff --git a/clang/lib/Headers/openmp_wrappers/cmath b/clang/lib/Headers/openmp_wrappers/cmath
index 05be252fa9fbf..bd6011eb6f6d5 100644
--- a/clang/lib/Headers/openmp_wrappers/cmath
+++ b/clang/lib/Headers/openmp_wrappers/cmath
@@ -28,7 +28,9 @@
     device = {arch(nvptx, nvptx64)}, implementation = {extension(match_any)})
 
 #define __CUDA__
+#define __OPENMP_NVPTX__
 #include <__clang_cuda_cmath.h>
+#undef __OPENMP_NVPTX__
 #undef __CUDA__
 
 // Overloads not provided by the CUDA wrappers but by the CUDA system headers.
diff --git a/clang/lib/Headers/openmp_wrappers/complex b/clang/lib/Headers/openmp_wrappers/complex
new file mode 100644
index 0000000000000..1ed0b14879efb
--- /dev/null
+++ b/clang/lib/Headers/openmp_wrappers/complex
@@ -0,0 +1,25 @@
+/*===-- complex --- OpenMP complex wrapper for target regions --------- c++ -===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __CLANG_OPENMP_COMPLEX__
+#define __CLANG_OPENMP_COMPLEX__
+
+#ifndef _OPENMP
+#error "This file is for OpenMP compilation only."
+#endif
+
+// We require std::math functions in the complex builtins below.
+#include <cmath>
+
+#define __CUDA__
+#include <__clang_cuda_complex_builtins.h>
+#endif
+
+// Grab the host header too.
+#include_next <complex>
diff --git a/clang/lib/Headers/openmp_wrappers/complex.h b/clang/lib/Headers/openmp_wrappers/complex.h
new file mode 100644
index 0000000000000..829c7a7857253
--- /dev/null
+++ b/clang/lib/Headers/openmp_wrappers/complex.h
@@ -0,0 +1,25 @@
+/*===-- complex --- OpenMP complex wrapper for target regions --------- c++ -===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __CLANG_OPENMP_COMPLEX_H__
+#define __CLANG_OPENMP_COMPLEX_H__
+
+#ifndef _OPENMP
+#error "This file is for OpenMP compilation only."
+#endif
+
+// We require math functions in the complex builtins below.
+#include <math.h>
+
+#define __CUDA__
+#include <__clang_cuda_complex_builtins.h>
+#endif
+
+// Grab the host header too.
+#include_next <complex.h>
diff --git a/clang/lib/Headers/openmp_wrappers/math.h b/clang/lib/Headers/openmp_wrappers/math.h
index e917a149b5c9a..c64af8b13ece4 100644
--- a/clang/lib/Headers/openmp_wrappers/math.h
+++ b/clang/lib/Headers/openmp_wrappers/math.h
@@ -41,7 +41,9 @@
     device = {arch(nvptx, nvptx64)}, implementation = {extension(match_any)})
 
 #define __CUDA__
+#define __OPENMP_NVPTX__
 #include <__clang_cuda_math.h>
+#undef __OPENMP_NVPTX__
 #undef __CUDA__
 
 #pragma omp end declare variant
diff --git a/clang/lib/Index/CMakeLists.txt b/clang/lib/Index/CMakeLists.txt
index e4ea7ea1660c9..68ebb5b7ede30 100644
--- a/clang/lib/Index/CMakeLists.txt
+++ b/clang/lib/Index/CMakeLists.txt
@@ -27,4 +27,7 @@ add_clang_library(clangIndex
   clangRewrite
   clangSerialization
   clangToolingCore
+
+  DEPENDS
+  omp_gen
   )
diff --git a/clang/lib/Lex/LiteralSupport.cpp b/clang/lib/Lex/LiteralSupport.cpp
index f44614b4bec46..eb16bc8c7da2d 100644
--- a/clang/lib/Lex/LiteralSupport.cpp
+++ b/clang/lib/Lex/LiteralSupport.cpp
@@ -525,8 +525,12 @@ static void EncodeUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
 ///
 NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
                                            SourceLocation TokLoc,
-                                           Preprocessor &PP)
-  : PP(PP), ThisTokBegin(TokSpelling.begin()), ThisTokEnd(TokSpelling.end()) {
+                                           const SourceManager &SM,
+                                           const LangOptions &LangOpts,
+                                           const TargetInfo &Target,
+                                           DiagnosticsEngine &Diags)
+    : SM(SM), LangOpts(LangOpts), Diags(Diags),
+      ThisTokBegin(TokSpelling.begin()), ThisTokEnd(TokSpelling.end()) {
 
   // This routine assumes that the range begin/end matches the regex for integer
   // and FP constants (specifically, the 'pp-number' regex), and assumes that
@@ -572,7 +576,7 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
   checkSeparator(TokLoc, s, CSK_AfterDigits);
 
   // Initial scan to lookahead for fixed point suffix.
-  if (PP.getLangOpts().FixedPoint) {
+  if (LangOpts.FixedPoint) {
     for (const char *c = s; c != ThisTokEnd; ++c) {
       if (*c == 'r' || *c == 'k' || *c == 'R' || *c == 'K') {
         saw_fixed_point_suffix = true;
@@ -592,14 +596,16 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
     switch (*s) {
     case 'R':
     case 'r':
-      if (!PP.getLangOpts().FixedPoint) break;
+      if (!LangOpts.FixedPoint)
+        break;
       if (isFract || isAccum) break;
       if (!(saw_period || saw_exponent)) break;
       isFract = true;
       continue;
     case 'K':
     case 'k':
-      if (!PP.getLangOpts().FixedPoint) break;
+      if (!LangOpts.FixedPoint)
+        break;
       if (isFract || isAccum) break;
       if (!(saw_period || saw_exponent)) break;
       isAccum = true;
@@ -607,7 +613,8 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
     case 'h':      // FP Suffix for "half".
     case 'H':
       // OpenCL Extension v1.2 s9.5 - h or H suffix for half type.
-      if (!(PP.getLangOpts().Half || PP.getLangOpts().FixedPoint)) break;
+      if (!(LangOpts.Half || LangOpts.FixedPoint))
+        break;
       if (isIntegerLiteral()) break;  // Error for integer constant.
       if (isHalf || isFloat || isLong) break; // HH, FH, LH invalid.
       isHalf = true;
@@ -621,8 +628,8 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
       // CUDA host and device may have different _Float16 support, therefore
       // allows f16 literals to avoid false alarm.
       // ToDo: more precise check for CUDA.
-      if ((PP.getTargetInfo().hasFloat16Type() || PP.getLangOpts().CUDA) &&
-          s + 2 < ThisTokEnd && s[1] == '1' && s[2] == '6') {
+      if ((Target.hasFloat16Type() || LangOpts.CUDA) && s + 2 < ThisTokEnd &&
+          s[1] == '1' && s[2] == '6') {
         s += 2; // success, eat up 2 characters.
         isFloat16 = true;
         continue;
@@ -657,10 +664,10 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
       } else {
         isLong = true;
       }
-      continue;  // Success.
+      continue; // Success.
     case 'i':
     case 'I':
-      if (PP.getLangOpts().MicrosoftExt) {
+      if (LangOpts.MicrosoftExt) {
         if (isLong || isLongLong || MicrosoftInteger)
           break;
 
@@ -713,7 +720,7 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
   if (s != ThisTokEnd || isImaginary) {
     // FIXME: Don't bother expanding UCNs if !tok.hasUCN().
     expandUCNs(UDSuffixBuf, StringRef(SuffixBegin, ThisTokEnd - SuffixBegin));
-    if (isValidUDSuffix(PP.getLangOpts(), UDSuffixBuf)) {
+    if (isValidUDSuffix(LangOpts, UDSuffixBuf)) {
       if (!isImaginary) {
         // Any suffix pieces we might have parsed are actually part of the
         // ud-suffix.
@@ -736,8 +743,9 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
 
     if (s != ThisTokEnd) {
       // Report an error if there are any.
-      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin - ThisTokBegin),
-              diag::err_invalid_suffix_constant)
+      Diags.Report(Lexer::AdvanceToTokenCharacter(
+                       TokLoc, SuffixBegin - ThisTokBegin, SM, LangOpts),
+                   diag::err_invalid_suffix_constant)
           << StringRef(SuffixBegin, ThisTokEnd - SuffixBegin)
           << (isFixedPointConstant ? 2 : isFPConstant);
       hadError = true;
@@ -758,9 +766,11 @@ void NumericLiteralParser::ParseDecimalOrOctalCommon(SourceLocation TokLoc){
   // If we have a hex digit other than 'e' (which denotes a FP exponent) then
   // the code is using an incorrect base.
   if (isHexDigit(*s) && *s != 'e' && *s != 'E' &&
-      !isValidUDSuffix(PP.getLangOpts(), StringRef(s, ThisTokEnd - s))) {
-    PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
-            diag::err_invalid_digit) << StringRef(s, 1) << (radix == 8 ? 1 : 0);
+      !isValidUDSuffix(LangOpts, StringRef(s, ThisTokEnd - s))) {
+    Diags.Report(
+        Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM, LangOpts),
+        diag::err_invalid_digit)
+        << StringRef(s, 1) << (radix == 8 ? 1 : 0);
     hadError = true;
     return;
   }
@@ -786,8 +796,9 @@ void NumericLiteralParser::ParseDecimalOrOctalCommon(SourceLocation TokLoc){
       s = first_non_digit;
     } else {
       if (!hadError) {
-        PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin),
-                diag::err_exponent_has_no_digits);
+        Diags.Report(Lexer::AdvanceToTokenCharacter(
+                         TokLoc, Exponent - ThisTokBegin, SM, LangOpts),
+                     diag::err_exponent_has_no_digits);
         hadError = true;
       }
       return;
@@ -833,9 +844,10 @@ void NumericLiteralParser::checkSeparator(SourceLocation TokLoc,
     return;
 
   if (isDigitSeparator(*Pos)) {
-    PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Pos - ThisTokBegin),
-            diag::err_digit_separator_not_between_digits)
-      << IsAfterDigits;
+    Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, Pos - ThisTokBegin, SM,
+                                                LangOpts),
+                 diag::err_digit_separator_not_between_digits)
+        << IsAfterDigits;
     hadError = true;
   }
 }
@@ -873,9 +885,10 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
     }
 
     if (!HasSignificandDigits) {
-      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin),
-              diag::err_hex_constant_requires)
-          << PP.getLangOpts().CPlusPlus << 1;
+      Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM,
+                                                  LangOpts),
+                   diag::err_hex_constant_requires)
+          << LangOpts.CPlusPlus << 1;
       hadError = true;
       return;
     }
@@ -891,8 +904,9 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
       const char *first_non_digit = SkipDigits(s);
       if (!containsDigits(s, first_non_digit)) {
         if (!hadError) {
-          PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin),
-                  diag::err_exponent_has_no_digits);
+          Diags.Report(Lexer::AdvanceToTokenCharacter(
+                           TokLoc, Exponent - ThisTokBegin, SM, LangOpts),
+                       diag::err_exponent_has_no_digits);
           hadError = true;
         }
         return;
@@ -900,16 +914,17 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
       checkSeparator(TokLoc, s, CSK_BeforeDigits);
       s = first_non_digit;
 
-      if (!PP.getLangOpts().HexFloats)
-        PP.Diag(TokLoc, PP.getLangOpts().CPlusPlus
-                            ? diag::ext_hex_literal_invalid
-                            : diag::ext_hex_constant_invalid);
-      else if (PP.getLangOpts().CPlusPlus17)
-        PP.Diag(TokLoc, diag::warn_cxx17_hex_literal);
+      if (!LangOpts.HexFloats)
+        Diags.Report(TokLoc, LangOpts.CPlusPlus
+                                 ? diag::ext_hex_literal_invalid
+                                 : diag::ext_hex_constant_invalid);
+      else if (LangOpts.CPlusPlus17)
+        Diags.Report(TokLoc, diag::warn_cxx17_hex_literal);
     } else if (saw_period) {
-      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin),
-              diag::err_hex_constant_requires)
-          << PP.getLangOpts().CPlusPlus << 0;
+      Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM,
+                                                  LangOpts),
+                   diag::err_hex_constant_requires)
+          << LangOpts.CPlusPlus << 0;
       hadError = true;
     }
     return;
@@ -918,12 +933,10 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
   // Handle simple binary numbers 0b01010
   if ((c1 == 'b' || c1 == 'B') && (s[1] == '0' || s[1] == '1')) {
     // 0b101010 is a C++1y / GCC extension.
-    PP.Diag(TokLoc,
-            PP.getLangOpts().CPlusPlus14
-              ? diag::warn_cxx11_compat_binary_literal
-              : PP.getLangOpts().CPlusPlus
-                ? diag::ext_binary_literal_cxx14
-                : diag::ext_binary_literal);
+    Diags.Report(TokLoc, LangOpts.CPlusPlus14
+                             ? diag::warn_cxx11_compat_binary_literal
+                         : LangOpts.CPlusPlus ? diag::ext_binary_literal_cxx14
+                                              : diag::ext_binary_literal);
     ++s;
     assert(s < ThisTokEnd && "didn't maximally munch?");
     radix = 2;
@@ -932,10 +945,11 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
     if (s == ThisTokEnd) {
       // Done.
     } else if (isHexDigit(*s) &&
-               !isValidUDSuffix(PP.getLangOpts(),
-                                StringRef(s, ThisTokEnd - s))) {
-      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
-              diag::err_invalid_digit) << StringRef(s, 1) << 2;
+               !isValidUDSuffix(LangOpts, StringRef(s, ThisTokEnd - s))) {
+      Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM,
+                                                  LangOpts),
+                   diag::err_invalid_digit)
+          << StringRef(s, 1) << 2;
       hadError = true;
     }
     // Other suffixes will be diagnosed by the caller.
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index 396ba529fc9a1..053ef1d2dd180 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -432,6 +432,7 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
     // Skip to the next '#endif' / '#else' / '#elif'.
     CurLexer->skipOver(*SkipLength);
   }
+  SourceLocation endLoc;
   while (true) {
     CurLexer->Lex(Tok);
 
@@ -538,7 +539,7 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
           // Restore the value of LexingRawMode so that trailing comments
           // are handled correctly, if we've reached the outermost block.
           CurPPLexer->LexingRawMode = false;
-          CheckEndOfDirective("endif");
+          endLoc = CheckEndOfDirective("endif");
           CurPPLexer->LexingRawMode = true;
           if (Callbacks)
             Callbacks->Endif(Tok.getLocation(), CondInfo.IfLoc);
@@ -565,7 +566,7 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
           // Restore the value of LexingRawMode so that trailing comments
           // are handled correctly.
           CurPPLexer->LexingRawMode = false;
-          CheckEndOfDirective("else");
+          endLoc = CheckEndOfDirective("else");
           CurPPLexer->LexingRawMode = true;
           if (Callbacks)
             Callbacks->Else(Tok.getLocation(), CondInfo.IfLoc);
@@ -621,7 +622,9 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
   // by the end of the preamble; we'll resume parsing after the preamble.
   if (Callbacks && (Tok.isNot(tok::eof) || !isRecordingPreamble()))
     Callbacks->SourceRangeSkipped(
-        SourceRange(HashTokenLoc, CurPPLexer->getSourceLocation()),
+        SourceRange(HashTokenLoc, endLoc.isValid()
+                                      ? endLoc
+                                      : CurPPLexer->getSourceLocation()),
         Tok.getLocation());
 }
 
diff --git a/clang/lib/Lex/PPExpressions.cpp b/clang/lib/Lex/PPExpressions.cpp
index e5ec2b99f5074..8c120c13d7d26 100644
--- a/clang/lib/Lex/PPExpressions.cpp
+++ b/clang/lib/Lex/PPExpressions.cpp
@@ -15,7 +15,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Lex/Preprocessor.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
@@ -26,9 +25,12 @@
 #include "clang/Lex/LiteralSupport.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/Token.h"
 #include "llvm/ADT/APSInt.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/SaveAndRestore.h"
@@ -251,8 +253,24 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
         // If this identifier isn't 'defined' or one of the special
         // preprocessor keywords and it wasn't macro expanded, it turns
         // into a simple 0
-        if (ValueLive)
+        if (ValueLive) {
           PP.Diag(PeekTok, diag::warn_pp_undef_identifier) << II;
+
+          const DiagnosticsEngine &DiagEngine = PP.getDiagnostics();
+          // If 'Wundef' is enabled, do not emit 'undef-prefix' diagnostics.
+          if (DiagEngine.isIgnored(diag::warn_pp_undef_identifier,
+                                   PeekTok.getLocation())) {
+            const std::vector<std::string> UndefPrefixes =
+                DiagEngine.getDiagnosticOptions().UndefPrefixes;
+            const StringRef IdentifierName = II->getName();
+            if (llvm::any_of(UndefPrefixes,
+                             [&IdentifierName](const std::string &Prefix) {
+                               return IdentifierName.startswith(Prefix);
+                             }))
+              PP.Diag(PeekTok, diag::warn_pp_undef_prefix)
+                  << AddFlagValue{llvm::join(UndefPrefixes, ",")} << II;
+          }
+        }
         Result.Val = 0;
         Result.Val.setIsUnsigned(false); // "0" is signed intmax_t 0.
         Result.setIdentifier(II);
@@ -277,7 +295,9 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
     if (NumberInvalid)
       return true; // a diagnostic was already reported
 
-    NumericLiteralParser Literal(Spelling, PeekTok.getLocation(), PP);
+    NumericLiteralParser Literal(Spelling, PeekTok.getLocation(),
+                                 PP.getSourceManager(), PP.getLangOpts(),
+                                 PP.getTargetInfo(), PP.getDiagnostics());
     if (Literal.hadError)
       return true; // a diagnostic was already reported.
 
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 105aa6683c8b5..160e2b6ed8846 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -1370,7 +1370,9 @@ bool Preprocessor::parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value) {
   StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid);
   if (NumberInvalid)
     return false;
-  NumericLiteralParser Literal(Spelling, Tok.getLocation(), *this);
+  NumericLiteralParser Literal(Spelling, Tok.getLocation(), getSourceManager(),
+                               getLangOpts(), getTargetInfo(),
+                               getDiagnostics());
   if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
     return false;
   llvm::APInt APVal(64, 0);
diff --git a/clang/lib/Parse/CMakeLists.txt b/clang/lib/Parse/CMakeLists.txt
index 3f7ab2a74af27..43cd8f6e12a24 100644
--- a/clang/lib/Parse/CMakeLists.txt
+++ b/clang/lib/Parse/CMakeLists.txt
@@ -27,4 +27,7 @@ add_clang_library(clangParse
   clangBasic
   clangLex
   clangSema
+
+  DEPENDS
+  omp_gen
   )
diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp
index 06e02ed0e11c6..d05332b5ac5a7 100644
--- a/clang/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp
@@ -133,9 +133,6 @@ NamedDecl *Parser::ParseCXXInlineMethodDef(
 
   LexedMethod* LM = new LexedMethod(this, FnD);
   getCurrentClass().LateParsedDeclarations.push_back(LM);
-  LM->TemplateScope = getCurScope()->isTemplateParamScope() ||
-      (FnD && isa<FunctionTemplateDecl>(FnD) &&
-       cast<FunctionTemplateDecl>(FnD)->isAbbreviated());
   CachedTokens &Toks = LM->Toks;
 
   tok::TokenKind kind = Tok.getKind();
@@ -225,6 +222,7 @@ Parser::LateParsedDeclaration::~LateParsedDeclaration() {}
 void Parser::LateParsedDeclaration::ParseLexedMethodDeclarations() {}
 void Parser::LateParsedDeclaration::ParseLexedMemberInitializers() {}
 void Parser::LateParsedDeclaration::ParseLexedMethodDefs() {}
+void Parser::LateParsedDeclaration::ParseLexedAttributes() {}
 void Parser::LateParsedDeclaration::ParseLexedPragmas() {}
 
 Parser::LateParsedClass::LateParsedClass(Parser *P, ParsingClass *C)
@@ -246,6 +244,10 @@ void Parser::LateParsedClass::ParseLexedMethodDefs() {
   Self->ParseLexedMethodDefs(*Class);
 }
 
+void Parser::LateParsedClass::ParseLexedAttributes() {
+  Self->ParseLexedAttributes(*Class);
+}
+
 void Parser::LateParsedClass::ParseLexedPragmas() {
   Self->ParseLexedPragmas(*Class);
 }
@@ -262,57 +264,79 @@ void Parser::LateParsedMemberInitializer::ParseLexedMemberInitializers() {
   Self->ParseLexedMemberInitializer(*this);
 }
 
+void Parser::LateParsedAttribute::ParseLexedAttributes() {
+  Self->ParseLexedAttribute(*this, true, false);
+}
+
 void Parser::LateParsedPragma::ParseLexedPragmas() {
   Self->ParseLexedPragma(*this);
 }
 
+/// Utility to re-enter a possibly-templated scope while parsing its
+/// late-parsed components.
+struct Parser::ReenterTemplateScopeRAII {
+  Parser &P;
+  MultiParseScope Scopes;
+  TemplateParameterDepthRAII CurTemplateDepthTracker;
+
+  ReenterTemplateScopeRAII(Parser &P, Decl *MaybeTemplated, bool Enter = true)
+      : P(P), Scopes(P), CurTemplateDepthTracker(P.TemplateParameterDepth) {
+    if (Enter) {
+      CurTemplateDepthTracker.addDepth(
+          P.ReenterTemplateScopes(Scopes, MaybeTemplated));
+    }
+  }
+};
+
+/// Utility to re-enter a class scope while parsing its late-parsed components.
+struct Parser::ReenterClassScopeRAII : ReenterTemplateScopeRAII {
+  ParsingClass &Class;
+
+  ReenterClassScopeRAII(Parser &P, ParsingClass &Class)
+      : ReenterTemplateScopeRAII(P, Class.TagOrTemplate,
+                                 /*Enter=*/!Class.TopLevelClass),
+        Class(Class) {
+    // If this is the top-level class, we're still within its scope.
+    if (Class.TopLevelClass)
+      return;
+
+    // Re-enter the class scope itself.
+    Scopes.Enter(Scope::ClassScope|Scope::DeclScope);
+    P.Actions.ActOnStartDelayedMemberDeclarations(P.getCurScope(),
+                                                  Class.TagOrTemplate);
+  }
+  ~ReenterClassScopeRAII() {
+    if (Class.TopLevelClass)
+      return;
+
+    P.Actions.ActOnFinishDelayedMemberDeclarations(P.getCurScope(),
+                                                   Class.TagOrTemplate);
+  }
+};
+
 /// ParseLexedMethodDeclarations - We finished parsing the member
 /// specification of a top (non-nested) C++ class. Now go over the
 /// stack of method declarations with some parts for which parsing was
 /// delayed (such as default arguments) and parse them.
 void Parser::ParseLexedMethodDeclarations(ParsingClass &Class) {
-  bool HasTemplateScope = !Class.TopLevelClass && Class.TemplateScope;
-  ParseScope ClassTemplateScope(this, Scope::TemplateParamScope,
-                                HasTemplateScope);
-  TemplateParameterDepthRAII CurTemplateDepthTracker(TemplateParameterDepth);
-  if (HasTemplateScope) {
-    Actions.ActOnReenterTemplateScope(getCurScope(), Class.TagOrTemplate);
-    ++CurTemplateDepthTracker;
-  }
-
-  // The current scope is still active if we're the top-level class.
-  // Otherwise we'll need to push and enter a new scope.
-  bool HasClassScope = !Class.TopLevelClass;
-  ParseScope ClassScope(this, Scope::ClassScope|Scope::DeclScope,
-                        HasClassScope);
-  if (HasClassScope)
-    Actions.ActOnStartDelayedMemberDeclarations(getCurScope(),
-                                                Class.TagOrTemplate);
-
-  for (size_t i = 0; i < Class.LateParsedDeclarations.size(); ++i) {
-    Class.LateParsedDeclarations[i]->ParseLexedMethodDeclarations();
-  }
+  ReenterClassScopeRAII InClassScope(*this, Class);
 
-  if (HasClassScope)
-    Actions.ActOnFinishDelayedMemberDeclarations(getCurScope(),
-                                                 Class.TagOrTemplate);
+  for (LateParsedDeclaration *LateD : Class.LateParsedDeclarations)
+    LateD->ParseLexedMethodDeclarations();
 }
 
 void Parser::ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM) {
   // If this is a member template, introduce the template parameter scope.
-  ParseScope TemplateScope(this, Scope::TemplateParamScope, LM.TemplateScope);
-  TemplateParameterDepthRAII CurTemplateDepthTracker(TemplateParameterDepth);
-  if (LM.TemplateScope) {
-    Actions.ActOnReenterTemplateScope(getCurScope(), LM.Method);
-    ++CurTemplateDepthTracker;
-  }
+  ReenterTemplateScopeRAII InFunctionTemplateScope(*this, LM.Method);
+
   // Start the delayed C++ method declaration
   Actions.ActOnStartDelayedCXXMethodDeclaration(getCurScope(), LM.Method);
 
   // Introduce the parameters into scope and parse their default
   // arguments.
-  ParseScope PrototypeScope(this, Scope::FunctionPrototypeScope |
-                            Scope::FunctionDeclarationScope | Scope::DeclScope);
+  InFunctionTemplateScope.Scopes.Enter(Scope::FunctionPrototypeScope |
+                                       Scope::FunctionDeclarationScope |
+                                       Scope::DeclScope);
   for (unsigned I = 0, N = LM.DefaultArgs.size(); I != N; ++I) {
     auto Param = cast<ParmVarDecl>(LM.DefaultArgs[I].Param);
     // Introduce the parameter into scope.
@@ -466,7 +490,7 @@ void Parser::ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM) {
     LM.ExceptionSpecTokens = nullptr;
   }
 
-  PrototypeScope.Exit();
+  InFunctionTemplateScope.Scopes.Exit();
 
   // Finish the delayed C++ method declaration.
   Actions.ActOnFinishDelayedCXXMethodDeclaration(getCurScope(), LM.Method);
@@ -476,30 +500,15 @@ void Parser::ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM) {
 /// (non-nested) C++ class. Now go over the stack of lexed methods that were
 /// collected during its parsing and parse them all.
 void Parser::ParseLexedMethodDefs(ParsingClass &Class) {
-  bool HasTemplateScope = !Class.TopLevelClass && Class.TemplateScope;
-  ParseScope ClassTemplateScope(this, Scope::TemplateParamScope, HasTemplateScope);
-  TemplateParameterDepthRAII CurTemplateDepthTracker(TemplateParameterDepth);
-  if (HasTemplateScope) {
-    Actions.ActOnReenterTemplateScope(getCurScope(), Class.TagOrTemplate);
-    ++CurTemplateDepthTracker;
-  }
-  bool HasClassScope = !Class.TopLevelClass;
-  ParseScope ClassScope(this, Scope::ClassScope|Scope::DeclScope,
-                        HasClassScope);
+  ReenterClassScopeRAII InClassScope(*this, Class);
 
-  for (size_t i = 0; i < Class.LateParsedDeclarations.size(); ++i) {
-    Class.LateParsedDeclarations[i]->ParseLexedMethodDefs();
-  }
+  for (LateParsedDeclaration *D : Class.LateParsedDeclarations)
+    D->ParseLexedMethodDefs();
 }
 
 void Parser::ParseLexedMethodDef(LexedMethod &LM) {
   // If this is a member template, introduce the template parameter scope.
-  ParseScope TemplateScope(this, Scope::TemplateParamScope, LM.TemplateScope);
-  TemplateParameterDepthRAII CurTemplateDepthTracker(TemplateParameterDepth);
-  if (LM.TemplateScope) {
-    Actions.ActOnReenterTemplateScope(getCurScope(), LM.D);
-    ++CurTemplateDepthTracker;
-  }
+  ReenterTemplateScopeRAII InFunctionTemplateScope(*this, LM.D);
 
   ParenBraceBracketBalancer BalancerRAIIObj(*this);
 
@@ -580,23 +589,7 @@ void Parser::ParseLexedMethodDef(LexedMethod &LM) {
 /// of a top (non-nested) C++ class. Now go over the stack of lexed data member
 /// initializers that were collected during its parsing and parse them all.
 void Parser::ParseLexedMemberInitializers(ParsingClass &Class) {
-  bool HasTemplateScope = !Class.TopLevelClass && Class.TemplateScope;
-  ParseScope ClassTemplateScope(this, Scope::TemplateParamScope,
-                                HasTemplateScope);
-  TemplateParameterDepthRAII CurTemplateDepthTracker(TemplateParameterDepth);
-  if (HasTemplateScope) {
-    Actions.ActOnReenterTemplateScope(getCurScope(), Class.TagOrTemplate);
-    ++CurTemplateDepthTracker;
-  }
-  // Set or update the scope flags.
-  bool AlreadyHasClassScope = Class.TopLevelClass;
-  unsigned ScopeFlags = Scope::ClassScope|Scope::DeclScope;
-  ParseScope ClassScope(this, ScopeFlags, !AlreadyHasClassScope);
-  ParseScopeFlags ClassScopeFlags(this, ScopeFlags, AlreadyHasClassScope);
-
-  if (!AlreadyHasClassScope)
-    Actions.ActOnStartDelayedMemberDeclarations(getCurScope(),
-                                                Class.TagOrTemplate);
+  ReenterClassScopeRAII InClassScope(*this, Class);
 
   if (!Class.LateParsedDeclarations.empty()) {
     // C++11 [expr.prim.general]p4:
@@ -604,18 +597,14 @@ void Parser::ParseLexedMemberInitializers(ParsingClass &Class) {
     //  (9.2) of a class X, the expression this is a prvalue of type "pointer
     //  to X" within the optional brace-or-equal-initializer. It shall not
     //  appear elsewhere in the member-declarator.
+    // FIXME: This should be done in ParseLexedMemberInitializer, not here.
     Sema::CXXThisScopeRAII ThisScope(Actions, Class.TagOrTemplate,
                                      Qualifiers());
 
-    for (size_t i = 0; i < Class.LateParsedDeclarations.size(); ++i) {
-      Class.LateParsedDeclarations[i]->ParseLexedMemberInitializers();
-    }
+    for (LateParsedDeclaration *D : Class.LateParsedDeclarations)
+      D->ParseLexedMemberInitializers();
   }
 
-  if (!AlreadyHasClassScope)
-    Actions.ActOnFinishDelayedMemberDeclarations(getCurScope(),
-                                                 Class.TagOrTemplate);
-
   Actions.ActOnFinishDelayedMemberInitializers(Class.TagOrTemplate);
 }
 
@@ -662,21 +651,115 @@ void Parser::ParseLexedMemberInitializer(LateParsedMemberInitializer &MI) {
     ConsumeAnyToken();
 }
 
-void Parser::ParseLexedPragmas(ParsingClass &Class) {
-  bool HasTemplateScope = !Class.TopLevelClass && Class.TemplateScope;
-  ParseScope ClassTemplateScope(this, Scope::TemplateParamScope,
-                                HasTemplateScope);
-  TemplateParameterDepthRAII CurTemplateDepthTracker(TemplateParameterDepth);
-  if (HasTemplateScope) {
-    Actions.ActOnReenterTemplateScope(getCurScope(), Class.TagOrTemplate);
-    ++CurTemplateDepthTracker;
+/// Wrapper class which calls ParseLexedAttribute, after setting up the
+/// scope appropriately.
+void Parser::ParseLexedAttributes(ParsingClass &Class) {
+  ReenterClassScopeRAII InClassScope(*this, Class);
+
+  for (LateParsedDeclaration *LateD : Class.LateParsedDeclarations)
+    LateD->ParseLexedAttributes();
+}
+
+/// Parse all attributes in LAs, and attach them to Decl D.
+void Parser::ParseLexedAttributeList(LateParsedAttrList &LAs, Decl *D,
+                                     bool EnterScope, bool OnDefinition) {
+  assert(LAs.parseSoon() &&
+         "Attribute list should be marked for immediate parsing.");
+  for (unsigned i = 0, ni = LAs.size(); i < ni; ++i) {
+    if (D)
+      LAs[i]->addDecl(D);
+    ParseLexedAttribute(*LAs[i], EnterScope, OnDefinition);
+    delete LAs[i];
   }
-  bool HasClassScope = !Class.TopLevelClass;
-  ParseScope ClassScope(this, Scope::ClassScope | Scope::DeclScope,
-                        HasClassScope);
+  LAs.clear();
+}
+
+/// Finish parsing an attribute for which parsing was delayed.
+/// This will be called at the end of parsing a class declaration
+/// for each LateParsedAttribute. We consume the saved tokens and
+/// create an attribute with the arguments filled in. We add this
+/// to the Attribute list for the decl.
+void Parser::ParseLexedAttribute(LateParsedAttribute &LA,
+                                 bool EnterScope, bool OnDefinition) {
+  // Create a fake EOF so that attribute parsing won't go off the end of the
+  // attribute.
+  Token AttrEnd;
+  AttrEnd.startToken();
+  AttrEnd.setKind(tok::eof);
+  AttrEnd.setLocation(Tok.getLocation());
+  AttrEnd.setEofData(LA.Toks.data());
+  LA.Toks.push_back(AttrEnd);
+
+  // Append the current token at the end of the new token stream so that it
+  // doesn't get lost.
+  LA.Toks.push_back(Tok);
+  PP.EnterTokenStream(LA.Toks, true, /*IsReinject=*/true);
+  // Consume the previously pushed token.
+  ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
+
+  ParsedAttributes Attrs(AttrFactory);
+  SourceLocation endLoc;
+
+  if (LA.Decls.size() > 0) {
+    Decl *D = LA.Decls[0];
+    NamedDecl *ND  = dyn_cast<NamedDecl>(D);
+    RecordDecl *RD = dyn_cast_or_null<RecordDecl>(D->getDeclContext());
+
+    // Allow 'this' within late-parsed attributes.
+    Sema::CXXThisScopeRAII ThisScope(Actions, RD, Qualifiers(),
+                                     ND && ND->isCXXInstanceMember());
+
+    if (LA.Decls.size() == 1) {
+      // If the Decl is templatized, add template parameters to scope.
+      ReenterTemplateScopeRAII InDeclScope(*this, D, EnterScope);
+
+      // If the Decl is on a function, add function parameters to the scope.
+      bool HasFunScope = EnterScope && D->isFunctionOrFunctionTemplate();
+      if (HasFunScope) {
+        InDeclScope.Scopes.Enter(Scope::FnScope | Scope::DeclScope |
+                                 Scope::CompoundStmtScope);
+        Actions.ActOnReenterFunctionContext(Actions.CurScope, D);
+      }
+
+      ParseGNUAttributeArgs(&LA.AttrName, LA.AttrNameLoc, Attrs, &endLoc,
+                            nullptr, SourceLocation(), ParsedAttr::AS_GNU,
+                            nullptr);
+
+      if (HasFunScope)
+        Actions.ActOnExitFunctionContext();
+    } else {
+      // If there are multiple decls, then the decl cannot be within the
+      // function scope.
+      ParseGNUAttributeArgs(&LA.AttrName, LA.AttrNameLoc, Attrs, &endLoc,
+                            nullptr, SourceLocation(), ParsedAttr::AS_GNU,
+                            nullptr);
+    }
+  } else {
+    Diag(Tok, diag::warn_attribute_no_decl) << LA.AttrName.getName();
+  }
+
+  if (OnDefinition && !Attrs.empty() && !Attrs.begin()->isCXX11Attribute() &&
+      Attrs.begin()->isKnownToGCC())
+    Diag(Tok, diag::warn_attribute_on_function_definition)
+      << &LA.AttrName;
+
+  for (unsigned i = 0, ni = LA.Decls.size(); i < ni; ++i)
+    Actions.ActOnFinishDelayedAttribute(getCurScope(), LA.Decls[i], Attrs);
+
+  // Due to a parsing error, we either went over the cached tokens or
+  // there are still cached tokens left, so we skip the leftover tokens.
+  while (Tok.isNot(tok::eof))
+    ConsumeAnyToken();
+
+  if (Tok.is(tok::eof) && Tok.getEofData() == AttrEnd.getEofData())
+    ConsumeAnyToken();
+}
+
+void Parser::ParseLexedPragmas(ParsingClass &Class) {
+  ReenterClassScopeRAII InClassScope(*this, Class);
 
-  for (LateParsedDeclaration *LPD : Class.LateParsedDeclarations)
-    LPD->ParseLexedPragmas();
+  for (LateParsedDeclaration *D : Class.LateParsedDeclarations)
+    D->ParseLexedPragmas();
 }
 
 void Parser::ParseLexedPragma(LateParsedPragma &LP) {
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 79a3b19bac576..c87d240a8206a 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -1409,154 +1409,6 @@ void Parser::ParseObjCBridgeRelatedAttribute(IdentifierInfo &ObjCBridgeRelated,
                Syntax);
 }
 
-// Late Parsed Attributes:
-// See other examples of late parsing in lib/Parse/ParseCXXInlineMethods
-
-void Parser::LateParsedDeclaration::ParseLexedAttributes() {}
-
-void Parser::LateParsedClass::ParseLexedAttributes() {
-  Self->ParseLexedAttributes(*Class);
-}
-
-void Parser::LateParsedAttribute::ParseLexedAttributes() {
-  Self->ParseLexedAttribute(*this, true, false);
-}
-
-/// Wrapper class which calls ParseLexedAttribute, after setting up the
-/// scope appropriately.
-void Parser::ParseLexedAttributes(ParsingClass &Class) {
-  // Deal with templates
-  // FIXME: Test cases to make sure this does the right thing for templates.
-  bool HasTemplateScope = !Class.TopLevelClass && Class.TemplateScope;
-  ParseScope ClassTemplateScope(this, Scope::TemplateParamScope,
-                                HasTemplateScope);
-  if (HasTemplateScope)
-    Actions.ActOnReenterTemplateScope(getCurScope(), Class.TagOrTemplate);
-
-  // Set or update the scope flags.
-  bool AlreadyHasClassScope = Class.TopLevelClass;
-  unsigned ScopeFlags = Scope::ClassScope|Scope::DeclScope;
-  ParseScope ClassScope(this, ScopeFlags, !AlreadyHasClassScope);
-  ParseScopeFlags ClassScopeFlags(this, ScopeFlags, AlreadyHasClassScope);
-
-  // Enter the scope of nested classes
-  if (!AlreadyHasClassScope)
-    Actions.ActOnStartDelayedMemberDeclarations(getCurScope(),
-                                                Class.TagOrTemplate);
-  if (!Class.LateParsedDeclarations.empty()) {
-    for (unsigned i = 0, ni = Class.LateParsedDeclarations.size(); i < ni; ++i){
-      Class.LateParsedDeclarations[i]->ParseLexedAttributes();
-    }
-  }
-
-  if (!AlreadyHasClassScope)
-    Actions.ActOnFinishDelayedMemberDeclarations(getCurScope(),
-                                                 Class.TagOrTemplate);
-}
-
-/// Parse all attributes in LAs, and attach them to Decl D.
-void Parser::ParseLexedAttributeList(LateParsedAttrList &LAs, Decl *D,
-                                     bool EnterScope, bool OnDefinition) {
-  assert(LAs.parseSoon() &&
-         "Attribute list should be marked for immediate parsing.");
-  for (unsigned i = 0, ni = LAs.size(); i < ni; ++i) {
-    if (D)
-      LAs[i]->addDecl(D);
-    ParseLexedAttribute(*LAs[i], EnterScope, OnDefinition);
-    delete LAs[i];
-  }
-  LAs.clear();
-}
-
-/// Finish parsing an attribute for which parsing was delayed.
-/// This will be called at the end of parsing a class declaration
-/// for each LateParsedAttribute. We consume the saved tokens and
-/// create an attribute with the arguments filled in. We add this
-/// to the Attribute list for the decl.
-void Parser::ParseLexedAttribute(LateParsedAttribute &LA,
-                                 bool EnterScope, bool OnDefinition) {
-  // Create a fake EOF so that attribute parsing won't go off the end of the
-  // attribute.
-  Token AttrEnd;
-  AttrEnd.startToken();
-  AttrEnd.setKind(tok::eof);
-  AttrEnd.setLocation(Tok.getLocation());
-  AttrEnd.setEofData(LA.Toks.data());
-  LA.Toks.push_back(AttrEnd);
-
-  // Append the current token at the end of the new token stream so that it
-  // doesn't get lost.
-  LA.Toks.push_back(Tok);
-  PP.EnterTokenStream(LA.Toks, true, /*IsReinject=*/true);
-  // Consume the previously pushed token.
-  ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
-
-  ParsedAttributes Attrs(AttrFactory);
-  SourceLocation endLoc;
-
-  if (LA.Decls.size() > 0) {
-    Decl *D = LA.Decls[0];
-    NamedDecl *ND  = dyn_cast<NamedDecl>(D);
-    RecordDecl *RD = dyn_cast_or_null<RecordDecl>(D->getDeclContext());
-
-    // Allow 'this' within late-parsed attributes.
-    Sema::CXXThisScopeRAII ThisScope(Actions, RD, Qualifiers(),
-                                     ND && ND->isCXXInstanceMember());
-
-    if (LA.Decls.size() == 1) {
-      // If the Decl is templatized, add template parameters to scope.
-      bool HasTemplateScope = EnterScope && D->isTemplateDecl();
-      ParseScope TempScope(this, Scope::TemplateParamScope, HasTemplateScope);
-      if (HasTemplateScope)
-        Actions.ActOnReenterTemplateScope(Actions.CurScope, D);
-
-      // If the Decl is on a function, add function parameters to the scope.
-      bool HasFunScope = EnterScope && D->isFunctionOrFunctionTemplate();
-      ParseScope FnScope(
-          this, Scope::FnScope | Scope::DeclScope | Scope::CompoundStmtScope,
-          HasFunScope);
-      if (HasFunScope)
-        Actions.ActOnReenterFunctionContext(Actions.CurScope, D);
-
-      ParseGNUAttributeArgs(&LA.AttrName, LA.AttrNameLoc, Attrs, &endLoc,
-                            nullptr, SourceLocation(), ParsedAttr::AS_GNU,
-                            nullptr);
-
-      if (HasFunScope) {
-        Actions.ActOnExitFunctionContext();
-        FnScope.Exit();  // Pop scope, and remove Decls from IdResolver
-      }
-      if (HasTemplateScope) {
-        TempScope.Exit();
-      }
-    } else {
-      // If there are multiple decls, then the decl cannot be within the
-      // function scope.
-      ParseGNUAttributeArgs(&LA.AttrName, LA.AttrNameLoc, Attrs, &endLoc,
-                            nullptr, SourceLocation(), ParsedAttr::AS_GNU,
-                            nullptr);
-    }
-  } else {
-    Diag(Tok, diag::warn_attribute_no_decl) << LA.AttrName.getName();
-  }
-
-  if (OnDefinition && !Attrs.empty() && !Attrs.begin()->isCXX11Attribute() &&
-      Attrs.begin()->isKnownToGCC())
-    Diag(Tok, diag::warn_attribute_on_function_definition)
-      << &LA.AttrName;
-
-  for (unsigned i = 0, ni = LA.Decls.size(); i < ni; ++i)
-    Actions.ActOnFinishDelayedAttribute(getCurScope(), LA.Decls[i], Attrs);
-
-  // Due to a parsing error, we either went over the cached tokens or
-  // there are still cached tokens left, so we skip the leftover tokens.
-  while (Tok.isNot(tok::eof))
-    ConsumeAnyToken();
-
-  if (Tok.is(tok::eof) && Tok.getEofData() == AttrEnd.getEofData())
-    ConsumeAnyToken();
-}
-
 void Parser::ParseTypeTagForDatatypeAttribute(IdentifierInfo &AttrName,
                                               SourceLocation AttrNameLoc,
                                               ParsedAttributes &Attrs,
@@ -2046,46 +1898,52 @@ Parser::DeclGroupPtrTy Parser::ParseDeclGroup(ParsingDeclSpec &DS,
   }
 
   // Check to see if we have a function *definition* which must have a body.
-  if (D.isFunctionDeclarator() &&
-      // Look at the next token to make sure that this isn't a function
-      // declaration.  We have to check this because __attribute__ might be the
-      // start of a function definition in GCC-extended K&R C.
-      !isDeclarationAfterDeclarator()) {
-
-    // Function definitions are only allowed at file scope and in C++ classes.
-    // The C++ inline method definition case is handled elsewhere, so we only
-    // need to handle the file scope definition case.
-    if (Context == DeclaratorContext::FileContext) {
-      if (isStartOfFunctionDefinition(D)) {
-        if (DS.getStorageClassSpec() == DeclSpec::SCS_typedef) {
-          Diag(Tok, diag::err_function_declared_typedef);
-
-          // Recover by treating the 'typedef' as spurious.
-          DS.ClearStorageClassSpecs();
-        }
+  if (D.isFunctionDeclarator()) {
+    if (Tok.is(tok::equal) && NextToken().is(tok::code_completion)) {
+      Actions.CodeCompleteAfterFunctionEquals(D);
+      cutOffParsing();
+      return nullptr;
+    }
+    // Look at the next token to make sure that this isn't a function
+    // declaration.  We have to check this because __attribute__ might be the
+    // start of a function definition in GCC-extended K&R C.
+    if (!isDeclarationAfterDeclarator()) {
+
+      // Function definitions are only allowed at file scope and in C++ classes.
+      // The C++ inline method definition case is handled elsewhere, so we only
+      // need to handle the file scope definition case.
+      if (Context == DeclaratorContext::FileContext) {
+        if (isStartOfFunctionDefinition(D)) {
+          if (DS.getStorageClassSpec() == DeclSpec::SCS_typedef) {
+            Diag(Tok, diag::err_function_declared_typedef);
+
+            // Recover by treating the 'typedef' as spurious.
+            DS.ClearStorageClassSpecs();
+          }
 
-        Decl *TheDecl =
-          ParseFunctionDefinition(D, ParsedTemplateInfo(), &LateParsedAttrs);
-        return Actions.ConvertDeclToDeclGroup(TheDecl);
-      }
+          Decl *TheDecl = ParseFunctionDefinition(D, ParsedTemplateInfo(),
+                                                  &LateParsedAttrs);
+          return Actions.ConvertDeclToDeclGroup(TheDecl);
+        }
 
-      if (isDeclarationSpecifier()) {
-        // If there is an invalid declaration specifier right after the
-        // function prototype, then we must be in a missing semicolon case
-        // where this isn't actually a body.  Just fall through into the code
-        // that handles it as a prototype, and let the top-level code handle
-        // the erroneous declspec where it would otherwise expect a comma or
-        // semicolon.
+        if (isDeclarationSpecifier()) {
+          // If there is an invalid declaration specifier right after the
+          // function prototype, then we must be in a missing semicolon case
+          // where this isn't actually a body.  Just fall through into the code
+          // that handles it as a prototype, and let the top-level code handle
+          // the erroneous declspec where it would otherwise expect a comma or
+          // semicolon.
+        } else {
+          Diag(Tok, diag::err_expected_fn_body);
+          SkipUntil(tok::semi);
+          return nullptr;
+        }
       } else {
-        Diag(Tok, diag::err_expected_fn_body);
-        SkipUntil(tok::semi);
-        return nullptr;
-      }
-    } else {
-      if (Tok.is(tok::l_brace)) {
-        Diag(Tok, diag::err_function_definition_not_allowed);
-        SkipMalformedDecl();
-        return nullptr;
+        if (Tok.is(tok::l_brace)) {
+          Diag(Tok, diag::err_function_definition_not_allowed);
+          SkipMalformedDecl();
+          return nullptr;
+        }
       }
     }
   }
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index a236a82213fb9..ddcbb5615feea 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -2185,7 +2185,6 @@ void Parser::HandleMemberFunctionDeclDelays(Declarator& DeclaratorInfo,
     // declarations.
     auto LateMethod = new LateParsedMethodDeclaration(this, ThisDecl);
     getCurrentClass().LateParsedDeclarations.push_back(LateMethod);
-    LateMethod->TemplateScope = getCurScope()->isTemplateParamScope();
 
     // Stash the exception-specification tokens in the late-pased method.
     LateMethod->ExceptionSpecTokens = FTI.ExceptionSpecTokens;
@@ -2714,6 +2713,11 @@ Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
           DefinitionKind = FDK_Defaulted;
         else if (KW.is(tok::kw_delete))
           DefinitionKind = FDK_Deleted;
+        else if (KW.is(tok::code_completion)) {
+          Actions.CodeCompleteAfterFunctionEquals(DeclaratorInfo);
+          cutOffParsing();
+          return nullptr;
+        }
       }
     }
     DeclaratorInfo.setFunctionDefinitionKind(DefinitionKind);
@@ -3372,8 +3376,10 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
     // to the levels specified on the command line.  Previous level
     // will be restored when the RAII object is destroyed.
     Sema::FPFeaturesStateRAII SaveFPFeaturesState(Actions);
-    FPOptions fpOptions(getLangOpts());
-    Actions.CurFPFeatures.getFromOpaqueInt(fpOptions.getAsOpaqueInt());
+    FPOptionsOverride NewOverrides;
+    Actions.CurFPFeatures = NewOverrides.applyOverrides(getLangOpts());
+    Actions.FpPragmaStack.Act(Tok.getLocation(), Sema::PSK_Reset, StringRef(),
+                              0 /*unused*/);
 
     SourceLocation SavedPrevTokLocation = PrevTokLocation;
     ParseLexedPragmas(getCurrentClass());
@@ -3931,8 +3937,8 @@ void Parser::PopParsingClass(Sema::ParsingClassState state) {
   // after the top-level class is completely defined. Therefore, add
   // it to the list of nested classes within its parent.
   assert(getCurScope()->isClassScope() && "Nested class outside of class scope?");
-  ClassStack.top()->LateParsedDeclarations.push_back(new LateParsedClass(this, Victim));
-  Victim->TemplateScope = getCurScope()->getParent()->isTemplateParamScope();
+  ClassStack.top()->LateParsedDeclarations.push_back(
+      new LateParsedClass(this, Victim));
 }
 
 /// Try to parse an 'identifier' which appears within an attribute-token.
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index bd003204a97ca..81e87582c6ade 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -920,6 +920,11 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
   auto SavedType = PreferredType;
   NotCastExpr = false;
 
+  // Are postfix-expression suffix operators permitted after this
+  // cast-expression? If not, and we find some, we'll parse them anyway and
+  // diagnose them.
+  bool AllowSuffix = true;
+
   // This handles all of cast-expression, unary-expression, postfix-expression,
   // and primary-expression.  We handle them together like this for efficiency
   // and to simplify handling of an expression starting with a '(' token: which
@@ -929,8 +934,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
   // If the parsed tokens consist of a primary-expression, the cases below
   // break out of the switch;  at the end we call ParsePostfixExpressionSuffix
   // to handle the postfix expression suffixes.  Cases that cannot be followed
-  // by postfix exprs should return without invoking
-  // ParsePostfixExpressionSuffix.
+  // by postfix exprs should set AllowSuffix to false.
   switch (SavedKind) {
   case tok::l_paren: {
     // If this expression is limited to being a unary-expression, the paren can
@@ -953,8 +957,11 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     Res = ParseParenExpression(ParenExprType, false/*stopIfCastExr*/,
                                isTypeCast == IsTypeCast, CastTy, RParenLoc);
 
+    // FIXME: What should we do if a vector literal is followed by a
+    // postfix-expression suffix? Usually postfix operators are permitted on
+    // literals.
     if (isVectorLiteral)
-        return Res;
+      return Res;
 
     switch (ParenExprType) {
     case SimpleExpr:   break;    // Nothing else to do.
@@ -992,11 +999,13 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
 
   case tok::kw___objc_yes:
   case tok::kw___objc_no:
-      return ParseObjCBoolLiteral();
+    Res = ParseObjCBoolLiteral();
+    break;
 
   case tok::kw_nullptr:
     Diag(Tok, diag::warn_cxx98_compat_nullptr);
-    return Actions.ActOnCXXNullPtrLiteral(ConsumeToken());
+    Res = Actions.ActOnCXXNullPtrLiteral(ConsumeToken());
+    break;
 
   case tok::annot_uneval_primary_expr:
   case tok::annot_primary_expr:
@@ -1295,7 +1304,8 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     Res = ParseGenericSelectionExpression();
     break;
   case tok::kw___builtin_available:
-    return ParseAvailabilityCheckExpr(Tok.getLocation());
+    Res = ParseAvailabilityCheckExpr(Tok.getLocation());
+    break;
   case tok::kw___builtin_va_arg:
   case tok::kw___builtin_offsetof:
   case tok::kw___builtin_choose_expr:
@@ -1307,9 +1317,11 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
   case tok::kw___builtin_LINE:
     if (NotPrimaryExpression)
       *NotPrimaryExpression = true;
+    // This parses the complete suffix; we can return early.
     return ParseBuiltinPrimaryExpression();
   case tok::kw___null:
-    return Actions.ActOnGNUNullExpr(ConsumeToken());
+    Res = Actions.ActOnGNUNullExpr(ConsumeToken());
+    break;
 
   case tok::plusplus:      // unary-expression: '++' unary-expression [C99]
   case tok::minusminus: {  // unary-expression: '--' unary-expression [C99]
@@ -1421,7 +1433,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
   case tok::kw___builtin_omp_required_simd_align:
     if (NotPrimaryExpression)
       *NotPrimaryExpression = true;
-    return ParseUnaryExprOrTypeTraitExpression();
+    AllowSuffix = false;
+    Res = ParseUnaryExprOrTypeTraitExpression();
+    break;
   case tok::ampamp: {      // unary-expression: '&&' identifier
     if (NotPrimaryExpression)
       *NotPrimaryExpression = true;
@@ -1437,7 +1451,8 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
                                                 Tok.getLocation());
     Res = Actions.ActOnAddrLabel(AmpAmpLoc, Tok.getLocation(), LD);
     ConsumeToken();
-    return Res;
+    AllowSuffix = false;
+    break;
   }
   case tok::kw_const_cast:
   case tok::kw_dynamic_cast:
@@ -1632,12 +1647,16 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     if (Tok.is(tok::kw_new)) {
       if (NotPrimaryExpression)
         *NotPrimaryExpression = true;
-      return ParseCXXNewExpression(true, CCLoc);
+      Res = ParseCXXNewExpression(true, CCLoc);
+      AllowSuffix = false;
+      break;
     }
     if (Tok.is(tok::kw_delete)) {
       if (NotPrimaryExpression)
         *NotPrimaryExpression = true;
-      return ParseCXXDeleteExpression(true, CCLoc);
+      Res = ParseCXXDeleteExpression(true, CCLoc);
+      AllowSuffix = false;
+      break;
     }
 
     // This is not a type name or scope specifier, it is an invalid expression.
@@ -1648,15 +1667,21 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
   case tok::kw_new: // [C++] new-expression
     if (NotPrimaryExpression)
       *NotPrimaryExpression = true;
-    return ParseCXXNewExpression(false, Tok.getLocation());
+    Res = ParseCXXNewExpression(false, Tok.getLocation());
+    AllowSuffix = false;
+    break;
 
   case tok::kw_delete: // [C++] delete-expression
     if (NotPrimaryExpression)
       *NotPrimaryExpression = true;
-    return ParseCXXDeleteExpression(false, Tok.getLocation());
+    Res = ParseCXXDeleteExpression(false, Tok.getLocation());
+    AllowSuffix = false;
+    break;
 
   case tok::kw_requires: // [C++2a] requires-expression
-    return ParseRequiresExpression();
+    Res = ParseRequiresExpression();
+    AllowSuffix = false;
+    break;
 
   case tok::kw_noexcept: { // [C++0x] 'noexcept' '(' expression ')'
     if (NotPrimaryExpression)
@@ -1672,32 +1697,36 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     //   which is an unevaluated operand, can throw an exception.
     EnterExpressionEvaluationContext Unevaluated(
         Actions, Sema::ExpressionEvaluationContext::Unevaluated);
-    ExprResult Result = ParseExpression();
+    Res = ParseExpression();
 
     T.consumeClose();
 
-    if (!Result.isInvalid())
-      Result = Actions.ActOnNoexceptExpr(KeyLoc, T.getOpenLocation(),
-                                         Result.get(), T.getCloseLocation());
-    return Result;
+    if (!Res.isInvalid())
+      Res = Actions.ActOnNoexceptExpr(KeyLoc, T.getOpenLocation(), Res.get(),
+                                      T.getCloseLocation());
+    AllowSuffix = false;
+    break;
   }
 
 #define TYPE_TRAIT(N,Spelling,K) \
   case tok::kw_##Spelling:
 #include "clang/Basic/TokenKinds.def"
-    return ParseTypeTrait();
+    Res = ParseTypeTrait();
+    break;
 
   case tok::kw___array_rank:
   case tok::kw___array_extent:
     if (NotPrimaryExpression)
       *NotPrimaryExpression = true;
-    return ParseArrayTypeTrait();
+    Res = ParseArrayTypeTrait();
+    break;
 
   case tok::kw___is_lvalue_expr:
   case tok::kw___is_rvalue_expr:
     if (NotPrimaryExpression)
       *NotPrimaryExpression = true;
-    return ParseExpressionTrait();
+    Res = ParseExpressionTrait();
+    break;
 
   case tok::at: {
     if (NotPrimaryExpression)
@@ -1754,6 +1783,41 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     // parsed.
     return Res;
 
+  if (!AllowSuffix) {
+    // FIXME: Don't parse a primary-expression suffix if we encountered a parse
+    // error already.
+    if (Res.isInvalid())
+      return Res;
+
+    switch (Tok.getKind()) {
+    case tok::l_square:
+    case tok::l_paren:
+    case tok::plusplus:
+    case tok::minusminus:
+      // "expected ';'" or similar is probably the right diagnostic here. Let
+      // the caller decide what to do.
+      if (Tok.isAtStartOfLine())
+        return Res;
+
+      LLVM_FALLTHROUGH;
+    case tok::period:
+    case tok::arrow:
+      break;
+
+    default:
+      return Res;
+    }
+
+    // This was a unary-expression for which a postfix-expression suffix is
+    // not permitted by the grammar (eg, a sizeof expression or
+    // new-expression or similar). Diagnose but parse the suffix anyway.
+    Diag(Tok.getLocation(), diag::err_postfix_after_unary_requires_parens)
+        << Tok.getKind() << Res.get()->getSourceRange()
+        << FixItHint::CreateInsertion(Res.get()->getBeginLoc(), "(")
+        << FixItHint::CreateInsertion(PP.getLocForEndOfToken(PrevTokLocation),
+                                      ")");
+  }
+
   // These can be followed by postfix-expr pieces.
   PreferredType = SavedType;
   Res = ParsePostfixExpressionSuffix(Res);
@@ -1845,8 +1909,8 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       BalancedDelimiterTracker T(*this, tok::l_square);
       T.consumeOpen();
       Loc = T.getOpenLocation();
-      ExprResult Idx, Length;
-      SourceLocation ColonLoc;
+      ExprResult Idx, Length, Stride;
+      SourceLocation ColonLocFirst, ColonLocSecond;
       PreferredType.enterSubscript(Actions, Tok.getLocation(), LHS.get());
       if (getLangOpts().CPlusPlus11 && Tok.is(tok::l_brace)) {
         Diag(Tok, diag::warn_cxx98_compat_generalized_initializer_lists);
@@ -1860,10 +1924,22 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
         }
         if (Tok.is(tok::colon)) {
           // Consume ':'
-          ColonLoc = ConsumeToken();
-          if (Tok.isNot(tok::r_square))
+          ColonLocFirst = ConsumeToken();
+          if (Tok.isNot(tok::r_square) &&
+              (getLangOpts().OpenMP < 50 ||
+               ((Tok.isNot(tok::colon) && getLangOpts().OpenMP >= 50))))
             Length = ParseExpression();
         }
+        if (getLangOpts().OpenMP >= 50 &&
+            (OMPClauseKind == llvm::omp::Clause::OMPC_to ||
+             OMPClauseKind == llvm::omp::Clause::OMPC_from) &&
+            Tok.is(tok::colon)) {
+          // Consume ':'
+          ColonLocSecond = ConsumeToken();
+          if (Tok.isNot(tok::r_square)) {
+            Stride = ParseExpression();
+          }
+        }
       } else
         Idx = ParseExpression();
 
@@ -1873,10 +1949,11 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       Idx = Actions.CorrectDelayedTyposInExpr(Idx);
       Length = Actions.CorrectDelayedTyposInExpr(Length);
       if (!LHS.isInvalid() && !Idx.isInvalid() && !Length.isInvalid() &&
-          Tok.is(tok::r_square)) {
-        if (ColonLoc.isValid()) {
-          LHS = Actions.ActOnOMPArraySectionExpr(LHS.get(), Loc, Idx.get(),
-                                                 ColonLoc, Length.get(), RLoc);
+          !Stride.isInvalid() && Tok.is(tok::r_square)) {
+        if (ColonLocFirst.isValid() || ColonLocSecond.isValid()) {
+          LHS = Actions.ActOnOMPArraySectionExpr(
+              LHS.get(), Loc, Idx.get(), ColonLocFirst, ColonLocSecond,
+              Length.get(), Stride.get(), RLoc);
         } else {
           LHS = Actions.ActOnArraySubscriptExpr(getCurScope(), LHS.get(), Loc,
                                                 Idx.get(), RLoc);
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index 48277dc854663..aa35200c33b66 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -1258,17 +1258,16 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
   };
 
   // FIXME: Consider allowing this as an extension for GCC compatibiblity.
-  const bool HasExplicitTemplateParams = Tok.is(tok::less);
-  ParseScope TemplateParamScope(this, Scope::TemplateParamScope,
-                                /*EnteredScope=*/HasExplicitTemplateParams);
-  if (HasExplicitTemplateParams) {
+  MultiParseScope TemplateParamScope(*this);
+  if (Tok.is(tok::less)) {
     Diag(Tok, getLangOpts().CPlusPlus20
                   ? diag::warn_cxx17_compat_lambda_template_parameter_list
                   : diag::ext_lambda_template_parameter_list);
 
     SmallVector<NamedDecl*, 4> TemplateParams;
     SourceLocation LAngleLoc, RAngleLoc;
-    if (ParseTemplateParameters(CurTemplateDepthTracker.getDepth(),
+    if (ParseTemplateParameters(TemplateParamScope,
+                                CurTemplateDepthTracker.getDepth(),
                                 TemplateParams, LAngleLoc, RAngleLoc)) {
       Actions.ActOnLambdaError(LambdaBeginLoc, getCurScope());
       return ExprError();
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 6cf50a5794f79..5223755c8fdf1 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -33,7 +33,7 @@ using namespace llvm::omp;
 
 namespace {
 enum OpenMPDirectiveKindEx {
-  OMPD_cancellation = unsigned(OMPD_unknown) + 1,
+  OMPD_cancellation = llvm::omp::Directive_enumSize + 1,
   OMPD_data,
   OMPD_declare,
   OMPD_end,
@@ -194,8 +194,9 @@ static OpenMPDirectiveKindExWrapper parseOpenMPDirectiveKind(Parser &P) {
       DKind = F[I][2];
     }
   }
-  return DKind < OMPD_unknown ? static_cast<OpenMPDirectiveKind>(DKind)
-                              : OMPD_unknown;
+  return unsigned(DKind) < llvm::omp::Directive_enumSize
+             ? static_cast<OpenMPDirectiveKind>(DKind)
+             : OMPD_unknown;
 }
 
 static DeclarationName parseOpenMPReductionId(Parser &P) {
@@ -647,16 +648,14 @@ namespace {
 class FNContextRAII final {
   Parser &P;
   Sema::CXXThisScopeRAII *ThisScope;
-  Parser::ParseScope *TempScope;
-  Parser::ParseScope *FnScope;
-  bool HasTemplateScope = false;
+  Parser::MultiParseScope Scopes;
   bool HasFunScope = false;
   FNContextRAII() = delete;
   FNContextRAII(const FNContextRAII &) = delete;
   FNContextRAII &operator=(const FNContextRAII &) = delete;
 
 public:
-  FNContextRAII(Parser &P, Parser::DeclGroupPtrTy Ptr) : P(P) {
+  FNContextRAII(Parser &P, Parser::DeclGroupPtrTy Ptr) : P(P), Scopes(P) {
     Decl *D = *Ptr.get().begin();
     NamedDecl *ND = dyn_cast<NamedDecl>(D);
     RecordDecl *RD = dyn_cast_or_null<RecordDecl>(D->getDeclContext());
@@ -667,29 +666,20 @@ class FNContextRAII final {
                                            ND && ND->isCXXInstanceMember());
 
     // If the Decl is templatized, add template parameters to scope.
-    HasTemplateScope = D->isTemplateDecl();
-    TempScope =
-        new Parser::ParseScope(&P, Scope::TemplateParamScope, HasTemplateScope);
-    if (HasTemplateScope)
-      Actions.ActOnReenterTemplateScope(Actions.getCurScope(), D);
+    // FIXME: Track CurTemplateDepth?
+    P.ReenterTemplateScopes(Scopes, D);
 
     // If the Decl is on a function, add function parameters to the scope.
-    HasFunScope = D->isFunctionOrFunctionTemplate();
-    FnScope = new Parser::ParseScope(
-        &P, Scope::FnScope | Scope::DeclScope | Scope::CompoundStmtScope,
-        HasFunScope);
-    if (HasFunScope)
+    if (D->isFunctionOrFunctionTemplate()) {
+      HasFunScope = true;
+      Scopes.Enter(Scope::FnScope | Scope::DeclScope |
+                   Scope::CompoundStmtScope);
       Actions.ActOnReenterFunctionContext(Actions.getCurScope(), D);
+    }
   }
   ~FNContextRAII() {
-    if (HasFunScope) {
+    if (HasFunScope)
       P.getActions().ActOnExitFunctionContext();
-      FnScope->Exit(); // Pop scope, and remove Decls from IdResolver
-    }
-    if (HasTemplateScope)
-      TempScope->Exit();
-    delete FnScope;
-    delete TempScope;
     delete ThisScope;
   }
 };
@@ -1451,7 +1441,7 @@ bool Parser::parseOMPDeclareVariantMatchClause(SourceLocation Loc,
 /// Parsing of simple OpenMP clauses like 'default' or 'proc_bind'.
 ///
 ///    default-clause:
-///         'default' '(' 'none' | 'shared' ')
+///         'default' '(' 'none' | 'shared'  | 'firstprivate' ')
 ///
 ///    proc_bind-clause:
 ///         'proc_bind' '(' 'master' | 'close' | 'spread' ')
@@ -1718,8 +1708,8 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
       SmallVector<OMPClause *, 1> Clauses;
       if (Tok.isNot(tok::annot_pragma_openmp_end)) {
         SmallVector<llvm::PointerIntPair<OMPClause *, 1, bool>,
-                    unsigned(OMPC_unknown) + 1>
-          FirstClauses(unsigned(OMPC_unknown) + 1);
+                    llvm::omp::Clause_enumSize + 1>
+            FirstClauses(llvm::omp::Clause_enumSize + 1);
         while (Tok.isNot(tok::annot_pragma_openmp_end)) {
           OpenMPClauseKind CKind =
               Tok.isAnnotation() ? OMPC_unknown
@@ -1754,8 +1744,8 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
     SourceLocation StartLoc = ConsumeToken();
     SmallVector<OMPClause *, 5> Clauses;
     SmallVector<llvm::PointerIntPair<OMPClause *, 1, bool>,
-                unsigned(OMPC_unknown) + 1>
-    FirstClauses(unsigned(OMPC_unknown) + 1);
+                llvm::omp::Clause_enumSize + 1>
+        FirstClauses(llvm::omp::Clause_enumSize + 1);
     if (Tok.is(tok::annot_pragma_openmp_end)) {
       Diag(Tok, diag::err_omp_expected_clause)
           << getOpenMPDirectiveName(OMPD_requires);
@@ -2019,6 +2009,8 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
     Diag(Tok, diag::err_omp_unexpected_directive)
         << 1 << getOpenMPDirectiveName(DKind);
     break;
+  default:
+    break;
   }
   while (Tok.isNot(tok::annot_pragma_openmp_end))
     ConsumeAnyToken();
@@ -2073,8 +2065,8 @@ Parser::ParseOpenMPDeclarativeOrExecutableDirective(ParsedStmtContext StmtCtx) {
   ParenBraceBracketBalancer BalancerRAIIObj(*this);
   SmallVector<OMPClause *, 5> Clauses;
   SmallVector<llvm::PointerIntPair<OMPClause *, 1, bool>,
-              unsigned(OMPC_unknown) + 1>
-  FirstClauses(unsigned(OMPC_unknown) + 1);
+              llvm::omp::Clause_enumSize + 1>
+      FirstClauses(llvm::omp::Clause_enumSize + 1);
   unsigned ScopeFlags = Scope::FnScope | Scope::DeclScope |
                         Scope::CompoundStmtScope | Scope::OpenMPDirectiveScope;
   SourceLocation Loc = ConsumeAnnotationToken(), EndLoc;
@@ -2119,8 +2111,8 @@ Parser::ParseOpenMPDeclarativeOrExecutableDirective(ParsedStmtContext StmtCtx) {
       SmallVector<OMPClause *, 1> Clauses;
       if (Tok.isNot(tok::annot_pragma_openmp_end)) {
         SmallVector<llvm::PointerIntPair<OMPClause *, 1, bool>,
-                    unsigned(OMPC_unknown) + 1>
-            FirstClauses(unsigned(OMPC_unknown) + 1);
+                    llvm::omp::Clause_enumSize + 1>
+            FirstClauses(llvm::omp::Clause_enumSize + 1);
         while (Tok.isNot(tok::annot_pragma_openmp_end)) {
           OpenMPClauseKind CKind =
               Tok.isAnnotation() ? OMPC_unknown
@@ -2368,6 +2360,7 @@ Parser::ParseOpenMPDeclarativeOrExecutableDirective(ParsedStmtContext StmtCtx) {
     SkipUntil(tok::annot_pragma_openmp_end);
     break;
   case OMPD_unknown:
+  default:
     Diag(Tok, diag::err_omp_unknown_directive);
     SkipUntil(tok::annot_pragma_openmp_end);
     break;
@@ -2505,6 +2498,7 @@ OMPClause *Parser::ParseOpenMPUsesAllocatorClause(OpenMPDirectiveKind DKind) {
 ///
 OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
                                      OpenMPClauseKind CKind, bool FirstClause) {
+  OMPClauseKind = CKind;
   OMPClause *Clause = nullptr;
   bool ErrorFound = false;
   bool WrongDirective = false;
@@ -2691,6 +2685,8 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
           << getOpenMPClauseName(CKind) << getOpenMPDirectiveName(DKind);
     SkipUntil(tok::comma, tok::annot_pragma_openmp_end, StopBeforeMatch);
     break;
+  default:
+    break;
   }
   return ErrorFound ? nullptr : Clause;
 }
@@ -2776,7 +2772,7 @@ OMPClause *Parser::ParseOpenMPSingleExprClause(OpenMPClauseKind Kind,
 /// Parsing of simple OpenMP clauses like 'default' or 'proc_bind'.
 ///
 ///    default-clause:
-///         'default' '(' 'none' | 'shared' ')'
+///         'default' '(' 'none' | 'shared' | 'firstprivate' ')'
 ///
 ///    proc_bind-clause:
 ///         'proc_bind' '(' 'master' | 'close' | 'spread' ')'
@@ -2789,6 +2785,14 @@ OMPClause *Parser::ParseOpenMPSimpleClause(OpenMPClauseKind Kind,
   llvm::Optional<SimpleClauseData> Val = parseOpenMPSimpleClause(*this, Kind);
   if (!Val || ParseOnly)
     return nullptr;
+  if (getLangOpts().OpenMP < 51 && Kind == OMPC_default &&
+      static_cast<DefaultKind>(Val.getValue().Type) ==
+          OMP_DEFAULT_firstprivate) {
+    Diag(Val.getValue().LOpen, diag::err_omp_invalid_dsa)
+        << getOpenMPClauseName(OMPC_firstprivate)
+        << getOpenMPClauseName(OMPC_default) << "5.1";
+    return nullptr;
+  }
   return Actions.ActOnOpenMPSimpleClause(
       Kind, Val.getValue().Type, Val.getValue().TypeLoc, Val.getValue().LOpen,
       Val.getValue().Loc, Val.getValue().RLoc);
diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp
index eacb54e81ae60..6402b31d00b29 100644
--- a/clang/lib/Parse/ParsePragma.cpp
+++ b/clang/lib/Parse/ParsePragma.cpp
@@ -654,8 +654,8 @@ void Parser::HandlePragmaFPContract() {
     break;
   }
 
-  Actions.ActOnPragmaFPContract(FPC);
-  ConsumeAnnotationToken();
+  SourceLocation PragmaLoc = ConsumeAnnotationToken();
+  Actions.ActOnPragmaFPContract(PragmaLoc, FPC);
 }
 
 void Parser::HandlePragmaFloatControl() {
@@ -2935,8 +2935,8 @@ void Parser::HandlePragmaFP() {
       reinterpret_cast<TokFPAnnotValue *>(Tok.getAnnotationValue());
 
   if (AnnotValue->FlagKind == TokFPAnnotValue::Reassociate)
-    Actions.ActOnPragmaFPReassociate(AnnotValue->FlagValue ==
-                                     TokFPAnnotValue::On);
+    Actions.ActOnPragmaFPReassociate(
+        Tok.getLocation(), AnnotValue->FlagValue == TokFPAnnotValue::On);
   else {
     LangOptions::FPModeKind FPC;
     switch (AnnotValue->FlagValue) {
@@ -2950,7 +2950,7 @@ void Parser::HandlePragmaFP() {
       FPC = LangOptions::FPM_Fast;
       break;
     }
-    Actions.ActOnPragmaFPContract(FPC);
+    Actions.ActOnPragmaFPContract(Tok.getLocation(), FPC);
   }
   ConsumeAnnotationToken();
 }
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index 773c31ff3aec1..89a6a2b829ae1 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -1156,10 +1156,14 @@ StmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
 /// should try to recover harder.  It returns false if the condition is
 /// successfully parsed.  Note that a successful parse can still have semantic
 /// errors in the condition.
+/// Additionally, if LParenLoc and RParenLoc are non-null, it will assign
+/// the location of the outer-most '(' and ')', respectively, to them.
 bool Parser::ParseParenExprOrCondition(StmtResult *InitStmt,
                                        Sema::ConditionResult &Cond,
                                        SourceLocation Loc,
-                                       Sema::ConditionKind CK) {
+                                       Sema::ConditionKind CK,
+                                       SourceLocation *LParenLoc,
+                                       SourceLocation *RParenLoc) {
   BalancedDelimiterTracker T(*this, tok::l_paren);
   T.consumeOpen();
 
@@ -1189,6 +1193,13 @@ bool Parser::ParseParenExprOrCondition(StmtResult *InitStmt,
   // Otherwise the condition is valid or the rparen is present.
   T.consumeClose();
 
+  if (LParenLoc != nullptr) {
+    *LParenLoc = T.getOpenLocation();
+  }
+  if (RParenLoc != nullptr) {
+    *RParenLoc = T.getCloseLocation();
+  }
+
   // Check for extraneous ')'s to catch things like "if (foo())) {".  We know
   // that all callers are looking for a statement after the condition, so ")"
   // isn't valid.
@@ -1348,6 +1359,8 @@ StmtResult Parser::ParseIfStatement(SourceLocation *TrailingElseLoc) {
   if (IsConstexpr)
     ConstexprCondition = Cond.getKnownValue();
 
+  bool IsBracedThen = Tok.is(tok::l_brace);
+
   // C99 6.8.4p3 - In C99, the body of the if statement is a scope, even if
   // there is no compound stmt.  C90 does not have this clause.  We only do this
   // if the body isn't a compound statement to avoid push/pop in common cases.
@@ -1366,7 +1379,7 @@ StmtResult Parser::ParseIfStatement(SourceLocation *TrailingElseLoc) {
   //    would have to notify ParseStatement not to create a new scope. It's
   //    simpler to let it create a new scope.
   //
-  ParseScope InnerScope(this, Scope::DeclScope, C99orCXX, Tok.is(tok::l_brace));
+  ParseScope InnerScope(this, Scope::DeclScope, C99orCXX, IsBracedThen);
 
   MisleadingIndentationChecker MIChecker(*this, MSK_if, IfLoc);
 
@@ -1427,7 +1440,7 @@ StmtResult Parser::ParseIfStatement(SourceLocation *TrailingElseLoc) {
     // Pop the 'else' scope if needed.
     InnerScope.Exit();
   } else if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteAfterIf(getCurScope());
+    Actions.CodeCompleteAfterIf(getCurScope(), IsBracedThen);
     cutOffParsing();
     return StmtError();
   } else if (InnerStatementTrailingElseLoc.isValid()) {
@@ -1580,8 +1593,10 @@ StmtResult Parser::ParseWhileStatement(SourceLocation *TrailingElseLoc) {
 
   // Parse the condition.
   Sema::ConditionResult Cond;
+  SourceLocation LParen;
+  SourceLocation RParen;
   if (ParseParenExprOrCondition(nullptr, Cond, WhileLoc,
-                                Sema::ConditionKind::Boolean))
+                                Sema::ConditionKind::Boolean, &LParen, &RParen))
     return StmtError();
 
   // C99 6.8.5p5 - In C99, the body of the while statement is a scope, even if
@@ -1611,7 +1626,7 @@ StmtResult Parser::ParseWhileStatement(SourceLocation *TrailingElseLoc) {
   if (Cond.isInvalid() || Body.isInvalid())
     return StmtError();
 
-  return Actions.ActOnWhileStmt(WhileLoc, Cond, Body.get());
+  return Actions.ActOnWhileStmt(WhileLoc, LParen, Cond, RParen, Body.get());
 }
 
 /// ParseDoStatement
diff --git a/clang/lib/Parse/ParseStmtAsm.cpp b/clang/lib/Parse/ParseStmtAsm.cpp
index 0f21037b61000..7d0818840a4fc 100644
--- a/clang/lib/Parse/ParseStmtAsm.cpp
+++ b/clang/lib/Parse/ParseStmtAsm.cpp
@@ -941,7 +941,7 @@ const char *Parser::GNUAsmQualifiers::getQualifierName(AQ Qualifier) {
     case AQ_goto: return "goto";
     case AQ_unspecified: return "unspecified";
   }
-  llvm_unreachable("Unkown GNUAsmQualifier");
+  llvm_unreachable("Unknown GNUAsmQualifier");
 }
 
 Parser::GNUAsmQualifiers::AQ
diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp
index 16aecffa494ad..3ef73f5791239 100644
--- a/clang/lib/Parse/ParseTemplate.cpp
+++ b/clang/lib/Parse/ParseTemplate.cpp
@@ -22,6 +22,16 @@
 #include "llvm/Support/TimeProfiler.h"
 using namespace clang;
 
+/// Re-enter a possible template scope, creating as many template parameter
+/// scopes as necessary.
+/// \return The number of template parameter scopes entered.
+unsigned Parser::ReenterTemplateScopes(MultiParseScope &S, Decl *D) {
+  return Actions.ActOnReenterTemplateScope(D, [&] {
+    S.Enter(Scope::TemplateParamScope);
+    return Actions.getCurScope();
+  });
+}
+
 /// Parse a template declaration, explicit instantiation, or
 /// explicit specialization.
 Decl *Parser::ParseDeclarationStartingWithTemplate(
@@ -67,8 +77,7 @@ Decl *Parser::ParseTemplateDeclarationOrSpecialization(
   assert(Tok.isOneOf(tok::kw_export, tok::kw_template) &&
          "Token does not start a template declaration.");
 
-  // Enter template-parameter scope.
-  ParseScope TemplateParmScope(this, Scope::TemplateParamScope);
+  MultiParseScope TemplateParamScopes(*this);
 
   // Tell the action that names should be checked in the context of
   // the declaration to come.
@@ -116,7 +125,8 @@ Decl *Parser::ParseTemplateDeclarationOrSpecialization(
     // Parse the '<' template-parameter-list '>'
     SourceLocation LAngleLoc, RAngleLoc;
     SmallVector<NamedDecl*, 4> TemplateParams;
-    if (ParseTemplateParameters(CurTemplateDepthTracker.getDepth(),
+    if (ParseTemplateParameters(TemplateParamScopes,
+                                CurTemplateDepthTracker.getDepth(),
                                 TemplateParams, LAngleLoc, RAngleLoc)) {
       // Skip until the semi-colon or a '}'.
       SkipUntil(tok::r_brace, StopAtSemi | StopBeforeMatch);
@@ -150,9 +160,6 @@ Decl *Parser::ParseTemplateDeclarationOrSpecialization(
         TemplateParams, RAngleLoc, OptionalRequiresClauseConstraintER.get()));
   } while (Tok.isOneOf(tok::kw_export, tok::kw_template));
 
-  unsigned NewFlags = getCurScope()->getFlags() & ~Scope::TemplateParamScope;
-  ParseScopeFlags TemplateScopeFlags(this, NewFlags, isSpecialization);
-
   // Parse the actual template declaration.
   if (Tok.is(tok::kw_concept))
     return ParseConceptDefinition(
@@ -430,8 +437,9 @@ Parser::ParseConceptDefinition(const ParsedTemplateInfo &TemplateInfo,
 ///
 /// \returns true if an error occurred, false otherwise.
 bool Parser::ParseTemplateParameters(
-    unsigned Depth, SmallVectorImpl<NamedDecl *> &TemplateParams,
-    SourceLocation &LAngleLoc, SourceLocation &RAngleLoc) {
+    MultiParseScope &TemplateScopes, unsigned Depth,
+    SmallVectorImpl<NamedDecl *> &TemplateParams, SourceLocation &LAngleLoc,
+    SourceLocation &RAngleLoc) {
   // Get the template parameter list.
   if (!TryConsumeToken(tok::less, LAngleLoc)) {
     Diag(Tok.getLocation(), diag::err_expected_less_after) << "template";
@@ -440,8 +448,11 @@ bool Parser::ParseTemplateParameters(
 
   // Try to parse the template parameter list.
   bool Failed = false;
-  if (!Tok.is(tok::greater) && !Tok.is(tok::greatergreater))
+  // FIXME: Missing greatergreatergreater support.
+  if (!Tok.is(tok::greater) && !Tok.is(tok::greatergreater)) {
+    TemplateScopes.Enter(Scope::TemplateParamScope);
     Failed = ParseTemplateParameterList(Depth, TemplateParams);
+  }
 
   if (Tok.is(tok::greatergreater)) {
     // No diagnostic required here: a template-parameter-list can only be
@@ -850,9 +861,9 @@ Parser::ParseTemplateTemplateParameter(unsigned Depth, unsigned Position) {
   SmallVector<NamedDecl*,8> TemplateParams;
   SourceLocation LAngleLoc, RAngleLoc;
   {
-    ParseScope TemplateParmScope(this, Scope::TemplateParamScope);
-    if (ParseTemplateParameters(Depth + 1, TemplateParams, LAngleLoc,
-                               RAngleLoc)) {
+    MultiParseScope TemplateParmScope(*this);
+    if (ParseTemplateParameters(TemplateParmScope, Depth + 1, TemplateParams,
+                                LAngleLoc, RAngleLoc)) {
       return nullptr;
     }
   }
@@ -1630,40 +1641,22 @@ void Parser::ParseLateTemplatedFuncDef(LateParsedTemplate &LPT) {
   Sema::ContextRAII GlobalSavedContext(
       Actions, Actions.Context.getTranslationUnitDecl());
 
-  SmallVector<ParseScope*, 4> TemplateParamScopeStack;
-
-  // Get the list of DeclContexts to reenter. For inline methods, we only want
-  // to push the DeclContext of the outermost class. This matches the way the
-  // parser normally parses bodies of inline methods when the outermost class is
-  // complete.
-  struct ContainingDC {
-    ContainingDC(DeclContext *DC, bool ShouldPush) : Pair(DC, ShouldPush) {}
-    llvm::PointerIntPair<DeclContext *, 1, bool> Pair;
-    DeclContext *getDC() { return Pair.getPointer(); }
-    bool shouldPushDC() { return Pair.getInt(); }
-  };
-  SmallVector<ContainingDC, 4> DeclContextsToReenter;
-  DeclContext *DD = FunD;
-  DeclContext *NextContaining = Actions.getContainingDC(DD);
-  while (DD && !DD->isTranslationUnit()) {
-    bool ShouldPush = DD == NextContaining;
-    DeclContextsToReenter.push_back({DD, ShouldPush});
-    if (ShouldPush)
-      NextContaining = Actions.getContainingDC(DD);
-    DD = DD->getLexicalParent();
-  }
-
-  // Reenter template scopes from outermost to innermost.
-  for (ContainingDC CDC : reverse(DeclContextsToReenter)) {
-    TemplateParamScopeStack.push_back(
-        new ParseScope(this, Scope::TemplateParamScope));
-    unsigned NumParamLists = Actions.ActOnReenterTemplateScope(
-        getCurScope(), cast<Decl>(CDC.getDC()));
-    CurTemplateDepthTracker.addDepth(NumParamLists);
-    if (CDC.shouldPushDC()) {
-      TemplateParamScopeStack.push_back(new ParseScope(this, Scope::DeclScope));
-      Actions.PushDeclContext(Actions.getCurScope(), CDC.getDC());
-    }
+  MultiParseScope Scopes(*this);
+
+  // Get the list of DeclContexts to reenter.
+  SmallVector<DeclContext*, 4> DeclContextsToReenter;
+  for (DeclContext *DC = FunD; DC && !DC->isTranslationUnit();
+       DC = DC->getLexicalParent())
+    DeclContextsToReenter.push_back(DC);
+
+  // Reenter scopes from outermost to innermost.
+  for (DeclContext *DC : reverse(DeclContextsToReenter)) {
+    CurTemplateDepthTracker.addDepth(
+        ReenterTemplateScopes(Scopes, cast<Decl>(DC)));
+    Scopes.Enter(Scope::DeclScope);
+    // We'll reenter the function context itself below.
+    if (DC != FunD)
+      Actions.PushDeclContext(Actions.getCurScope(), DC);
   }
 
   assert(!LPT.Toks.empty() && "Empty body!");
@@ -1684,8 +1677,7 @@ void Parser::ParseLateTemplatedFuncDef(LateParsedTemplate &LPT) {
                                Scope::CompoundStmtScope);
 
   // Recreate the containing function DeclContext.
-  Sema::ContextRAII FunctionSavedContext(Actions,
-                                         Actions.getContainingDC(FunD));
+  Sema::ContextRAII FunctionSavedContext(Actions, FunD->getLexicalParent());
 
   Actions.ActOnStartOfFunctionDef(getCurScope(), FunD);
 
@@ -1709,13 +1701,6 @@ void Parser::ParseLateTemplatedFuncDef(LateParsedTemplate &LPT) {
     } else
       Actions.ActOnFinishFunctionBody(LPT.D, nullptr);
   }
-
-  // Exit scopes.
-  FnScope.Exit();
-  SmallVectorImpl<ParseScope *>::reverse_iterator I =
-   TemplateParamScopeStack.rbegin();
-  for (; I != TemplateParamScopeStack.rend(); ++I)
-    delete *I;
 }
 
 /// Lex a delayed template function for late parsing.
diff --git a/clang/lib/Sema/CMakeLists.txt b/clang/lib/Sema/CMakeLists.txt
index b59fc30882f96..adadc06e209c3 100644
--- a/clang/lib/Sema/CMakeLists.txt
+++ b/clang/lib/Sema/CMakeLists.txt
@@ -72,6 +72,7 @@ add_clang_library(clangSema
 
   DEPENDS
   ClangOpenCLBuiltinsImpl
+  omp_gen
 
   LINK_LIBS
   clangAST
diff --git a/clang/lib/Sema/DeclSpec.cpp b/clang/lib/Sema/DeclSpec.cpp
index 6ad50e18cd528..f4c30c90ad271 100644
--- a/clang/lib/Sema/DeclSpec.cpp
+++ b/clang/lib/Sema/DeclSpec.cpp
@@ -1150,14 +1150,20 @@ void DeclSpec::Finish(Sema &S, const PrintingPolicy &Policy) {
         S.Diag(TSSLoc, diag::err_invalid_vector_bool_decl_spec)
           << getSpecifierName((TSS)TypeSpecSign);
       }
-
-      // Only char/int are valid with vector bool. (PIM 2.1)
+      // Only char/int are valid with vector bool prior to Power10.
+      // Power10 adds instructions that produce vector bool data
+      // for quadwords as well so allow vector bool __int128.
       if (((TypeSpecType != TST_unspecified) && (TypeSpecType != TST_char) &&
-           (TypeSpecType != TST_int)) || TypeAltiVecPixel) {
+           (TypeSpecType != TST_int) && (TypeSpecType != TST_int128)) ||
+          TypeAltiVecPixel) {
         S.Diag(TSTLoc, diag::err_invalid_vector_bool_decl_spec)
           << (TypeAltiVecPixel ? "__pixel" :
                                  getSpecifierName((TST)TypeSpecType, Policy));
       }
+      // vector bool __int128 requires Power10.
+      if ((TypeSpecType == TST_int128) &&
+          (!S.Context.getTargetInfo().hasFeature("power10-vector")))
+        S.Diag(TSTLoc, diag::err_invalid_vector_bool_int128_decl_spec);
 
       // Only 'short' and 'long long' are valid with vector bool. (PIM 2.1)
       if ((TypeSpecWidth != TSW_unspecified) && (TypeSpecWidth != TSW_short) &&
@@ -1174,7 +1180,7 @@ void DeclSpec::Finish(Sema &S, const PrintingPolicy &Policy) {
 
       // Elements of vector bool are interpreted as unsigned. (PIM 2.1)
       if ((TypeSpecType == TST_char) || (TypeSpecType == TST_int) ||
-          (TypeSpecWidth != TSW_unspecified))
+          (TypeSpecType == TST_int128) || (TypeSpecWidth != TSW_unspecified))
         TypeSpecSign = TSS_unsigned;
     } else if (TypeSpecType == TST_double) {
       // vector long double and vector long long double are never allowed.
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index fc3fb524c8f42..2f2b52106f3d4 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -159,9 +159,8 @@ Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
           LangOpts.getMSPointerToMemberRepresentationMethod()),
       VtorDispStack(LangOpts.getVtorDispMode()), PackStack(0),
       DataSegStack(nullptr), BSSSegStack(nullptr), ConstSegStack(nullptr),
-      CodeSegStack(nullptr), FpPragmaStack(CurFPFeatures.getAsOpaqueInt()),
-      CurInitSeg(nullptr), VisContext(nullptr),
-      PragmaAttributeCurrentTargetDecl(nullptr),
+      CodeSegStack(nullptr), FpPragmaStack(0xffffffff), CurInitSeg(nullptr),
+      VisContext(nullptr), PragmaAttributeCurrentTargetDecl(nullptr),
       IsBuildingRecoveryCallExpr(false), Cleanup{}, LateTemplateParser(nullptr),
       LateTemplateParserCleanup(nullptr), OpaqueParser(nullptr), IdResolver(pp),
       StdExperimentalNamespaceCache(nullptr), StdInitializerList(nullptr),
diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp
index 977e92475182d..b354e810974c4 100644
--- a/clang/lib/Sema/SemaAttr.cpp
+++ b/clang/lib/Sema/SemaAttr.cpp
@@ -417,8 +417,10 @@ void Sema::ActOnPragmaDetectMismatch(SourceLocation Loc, StringRef Name,
 void Sema::ActOnPragmaFloatControl(SourceLocation Loc,
                                    PragmaMsStackAction Action,
                                    PragmaFloatControlKind Value) {
-  auto NewValue = FpPragmaStack.CurrentValue;
-  FPOptions NewFPFeatures(NewValue);
+  unsigned NewValue = FpPragmaStack.hasValue()
+                          ? FpPragmaStack.CurrentValue
+                          : CurFPFeatureOverrides().getAsOpaqueInt();
+  FPOptionsOverride NewFPFeatures(NewValue);
   if ((Action == PSK_Push_Set || Action == PSK_Push || Action == PSK_Pop) &&
       !(CurContext->isTranslationUnit()) && !CurContext->isNamespace()) {
     // Push and pop can only occur at file or namespace scope.
@@ -429,38 +431,34 @@ void Sema::ActOnPragmaFloatControl(SourceLocation Loc,
   default:
     llvm_unreachable("invalid pragma float_control kind");
   case PFC_Precise:
-    CurFPFeatures.setFPPreciseEnabled(true);
-    NewValue = CurFPFeatures.getAsOpaqueInt();
+    NewFPFeatures.setFPPreciseEnabled(true);
+    NewValue = NewFPFeatures.getAsOpaqueInt();
     FpPragmaStack.Act(Loc, Action, StringRef(), NewValue);
     break;
   case PFC_NoPrecise:
-    if (CurFPFeatures.getExceptionMode() == LangOptions::FPE_Strict)
+    if (CurFPFeatures.getFPExceptionMode() == LangOptions::FPE_Strict)
       Diag(Loc, diag::err_pragma_fc_noprecise_requires_noexcept);
-    else if (CurFPFeatures.allowFEnvAccess())
+    else if (CurFPFeatures.getAllowFEnvAccess())
       Diag(Loc, diag::err_pragma_fc_noprecise_requires_nofenv);
     else
-      CurFPFeatures.setFPPreciseEnabled(false);
-    NewValue = CurFPFeatures.getAsOpaqueInt();
+      NewFPFeatures.setFPPreciseEnabled(false);
+    NewValue = NewFPFeatures.getAsOpaqueInt();
     FpPragmaStack.Act(Loc, Action, StringRef(), NewValue);
     break;
   case PFC_Except:
     if (!isPreciseFPEnabled())
       Diag(Loc, diag::err_pragma_fc_except_requires_precise);
     else
-      CurFPFeatures.setExceptionMode(LangOptions::FPE_Strict);
-    NewValue = CurFPFeatures.getAsOpaqueInt();
+      NewFPFeatures.setFPExceptionModeOverride(LangOptions::FPE_Strict);
+    NewValue = NewFPFeatures.getAsOpaqueInt();
     FpPragmaStack.Act(Loc, Action, StringRef(), NewValue);
     break;
   case PFC_NoExcept:
-    CurFPFeatures.setExceptionMode(LangOptions::FPE_Ignore);
-    NewValue = CurFPFeatures.getAsOpaqueInt();
+    NewFPFeatures.setFPExceptionModeOverride(LangOptions::FPE_Ignore);
+    NewValue = NewFPFeatures.getAsOpaqueInt();
     FpPragmaStack.Act(Loc, Action, StringRef(), NewValue);
     break;
   case PFC_Push:
-    if (FpPragmaStack.Stack.empty()) {
-      FpPragmaStack.Act(Loc, Sema::PSK_Set, StringRef(),
-                        CurFPFeatures.getAsOpaqueInt());
-    }
     FpPragmaStack.Act(Loc, Sema::PSK_Push_Set, StringRef(),
                       NewFPFeatures.getAsOpaqueInt());
     break;
@@ -472,9 +470,12 @@ void Sema::ActOnPragmaFloatControl(SourceLocation Loc,
     }
     FpPragmaStack.Act(Loc, Action, StringRef(), NewFPFeatures.getAsOpaqueInt());
     NewValue = FpPragmaStack.CurrentValue;
-    CurFPFeatures.getFromOpaqueInt(NewValue);
     break;
   }
+  FPOptionsOverride NewOverrides;
+  if (NewValue != FpPragmaStack.DefaultValue)
+    NewOverrides.getFromOpaqueInt(NewValue);
+  CurFPFeatures = NewOverrides.applyOverrides(getLangOpts());
 }
 
 void Sema::ActOnPragmaMSPointersToMembers(
@@ -493,44 +494,6 @@ void Sema::ActOnPragmaMSVtorDisp(PragmaMsStackAction Action,
   VtorDispStack.Act(PragmaLoc, Action, StringRef(), Mode);
 }
 
-template<typename ValueType>
-void Sema::PragmaStack<ValueType>::Act(SourceLocation PragmaLocation,
-                                       PragmaMsStackAction Action,
-                                       llvm::StringRef StackSlotLabel,
-                                       ValueType Value) {
-  if (Action == PSK_Reset) {
-    CurrentValue = DefaultValue;
-    CurrentPragmaLocation = PragmaLocation;
-    return;
-  }
-  if (Action & PSK_Push)
-    Stack.emplace_back(StackSlotLabel, CurrentValue, CurrentPragmaLocation,
-                       PragmaLocation);
-  else if (Action & PSK_Pop) {
-    if (!StackSlotLabel.empty()) {
-      // If we've got a label, try to find it and jump there.
-      auto I = llvm::find_if(llvm::reverse(Stack), [&](const Slot &x) {
-        return x.StackSlotLabel == StackSlotLabel;
-      });
-      // If we found the label so pop from there.
-      if (I != Stack.rend()) {
-        CurrentValue = I->Value;
-        CurrentPragmaLocation = I->PragmaLocation;
-        Stack.erase(std::prev(I.base()), Stack.end());
-      }
-    } else if (!Stack.empty()) {
-      // We do not have a label, just pop the last entry.
-      CurrentValue = Stack.back().Value;
-      CurrentPragmaLocation = Stack.back().PragmaLocation;
-      Stack.pop_back();
-    }
-  }
-  if (Action & PSK_Set) {
-    CurrentValue = Value;
-    CurrentPragmaLocation = PragmaLocation;
-  }
-}
-
 bool Sema::UnifySection(StringRef SectionName,
                         int SectionFlags,
                         DeclaratorDecl *Decl) {
@@ -1003,33 +966,70 @@ void Sema::ActOnPragmaVisibility(const IdentifierInfo* VisType,
   }
 }
 
-void Sema::ActOnPragmaFPContract(LangOptions::FPModeKind FPC) {
+void Sema::ActOnPragmaFPContract(SourceLocation Loc,
+                                 LangOptions::FPModeKind FPC) {
+  unsigned NewValue = FpPragmaStack.hasValue()
+                          ? FpPragmaStack.CurrentValue
+                          : CurFPFeatureOverrides().getAsOpaqueInt();
+  FPOptionsOverride NewFPFeatures(NewValue);
   switch (FPC) {
   case LangOptions::FPM_On:
-    CurFPFeatures.setAllowFPContractWithinStatement();
+    NewFPFeatures.setAllowFPContractWithinStatement();
     break;
   case LangOptions::FPM_Fast:
-    CurFPFeatures.setAllowFPContractAcrossStatement();
+    NewFPFeatures.setAllowFPContractAcrossStatement();
     break;
   case LangOptions::FPM_Off:
-    CurFPFeatures.setDisallowFPContract();
+    NewFPFeatures.setDisallowFPContract();
     break;
   }
+  CurFPFeatures = NewFPFeatures.applyOverrides(getLangOpts());
+  FpPragmaStack.Act(Loc, Sema::PSK_Set, StringRef(),
+                    NewFPFeatures.getAsOpaqueInt());
 }
 
-void Sema::ActOnPragmaFPReassociate(bool IsEnabled) {
-  CurFPFeatures.setAllowAssociativeMath(IsEnabled);
+void Sema::ActOnPragmaFPReassociate(SourceLocation Loc, bool IsEnabled) {
+  unsigned NewValue = FpPragmaStack.hasValue()
+                          ? FpPragmaStack.CurrentValue
+                          : CurFPFeatureOverrides().getAsOpaqueInt();
+  FPOptionsOverride NewFPFeatures(NewValue);
+  NewFPFeatures.setAllowFPReassociateOverride(IsEnabled);
+  NewValue = NewFPFeatures.getAsOpaqueInt();
+  FpPragmaStack.Act(Loc, PSK_Set, StringRef(), NewValue);
+  FPOptionsOverride NewOverrides(NewValue);
+  CurFPFeatures = NewOverrides.applyOverrides(getLangOpts());
 }
 
-void Sema::setRoundingMode(llvm::RoundingMode FPR) {
-  CurFPFeatures.setRoundingMode(FPR);
+void Sema::setRoundingMode(SourceLocation Loc, llvm::RoundingMode FPR) {
+  unsigned NewValue = FpPragmaStack.hasValue()
+                          ? FpPragmaStack.CurrentValue
+                          : CurFPFeatureOverrides().getAsOpaqueInt();
+  FPOptionsOverride NewFPFeatures(NewValue);
+  NewFPFeatures.setRoundingModeOverride(FPR);
+  NewValue = NewFPFeatures.getAsOpaqueInt();
+  FpPragmaStack.Act(Loc, PSK_Set, StringRef(), NewValue);
+  FPOptionsOverride NewOverrides(NewValue);
+  CurFPFeatures = NewOverrides.applyOverrides(getLangOpts());
 }
 
-void Sema::setExceptionMode(LangOptions::FPExceptionModeKind FPE) {
-  CurFPFeatures.setExceptionMode(FPE);
+void Sema::setExceptionMode(SourceLocation Loc,
+                            LangOptions::FPExceptionModeKind FPE) {
+  unsigned NewValue = FpPragmaStack.hasValue()
+                          ? FpPragmaStack.CurrentValue
+                          : CurFPFeatureOverrides().getAsOpaqueInt();
+  FPOptionsOverride NewFPFeatures(NewValue);
+  NewFPFeatures.setFPExceptionModeOverride(FPE);
+  NewValue = NewFPFeatures.getAsOpaqueInt();
+  FpPragmaStack.Act(Loc, PSK_Set, StringRef(), NewValue);
+  FPOptionsOverride NewOverrides(NewValue);
+  CurFPFeatures = NewOverrides.applyOverrides(getLangOpts());
 }
 
 void Sema::ActOnPragmaFEnvAccess(SourceLocation Loc, bool IsEnabled) {
+  unsigned NewValue = FpPragmaStack.hasValue()
+                          ? FpPragmaStack.CurrentValue
+                          : CurFPFeatureOverrides().getAsOpaqueInt();
+  FPOptionsOverride NewFPFeatures(NewValue);
   if (IsEnabled) {
     // Verify Microsoft restriction:
     // You can't enable fenv_access unless precise semantics are enabled.
@@ -1037,9 +1037,13 @@ void Sema::ActOnPragmaFEnvAccess(SourceLocation Loc, bool IsEnabled) {
     // pragma, or by using the /fp:precise or /fp:strict compiler options
     if (!isPreciseFPEnabled())
       Diag(Loc, diag::err_pragma_fenv_requires_precise);
-    CurFPFeatures.setAllowFEnvAccess();
+    NewFPFeatures.setAllowFEnvAccessOverride(true);
   } else
-    CurFPFeatures.setDisallowFEnvAccess();
+    NewFPFeatures.setAllowFEnvAccessOverride(false);
+  NewValue = NewFPFeatures.getAsOpaqueInt();
+  FpPragmaStack.Act(Loc, PSK_Set, StringRef(), NewValue);
+  FPOptionsOverride NewOverrides(NewValue);
+  CurFPFeatures = NewOverrides.applyOverrides(getLangOpts());
 }
 
 void Sema::PushNamespaceVisibilityAttr(const VisibilityAttr *Attr,
diff --git a/clang/lib/Sema/SemaCUDA.cpp b/clang/lib/Sema/SemaCUDA.cpp
index f54d97a2a3195..283a04683a32a 100644
--- a/clang/lib/Sema/SemaCUDA.cpp
+++ b/clang/lib/Sema/SemaCUDA.cpp
@@ -17,6 +17,7 @@
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/Lookup.h"
+#include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/SemaDiagnostic.h"
 #include "clang/Sema/SemaInternal.h"
@@ -746,20 +747,58 @@ bool Sema::CheckCUDACall(SourceLocation Loc, FunctionDecl *Callee) {
          DiagKind != DeviceDiagBuilder::K_ImmediateWithCallStack;
 }
 
+// Check the wrong-sided reference capture of lambda for CUDA/HIP.
+// A lambda function may capture a stack variable by reference when it is
+// defined and uses the capture by reference when the lambda is called. When
+// the capture and use happen on different sides, the capture is invalid and
+// should be diagnosed.
+void Sema::CUDACheckLambdaCapture(CXXMethodDecl *Callee,
+                                  const sema::Capture &Capture) {
+  // In host compilation we only need to check lambda functions emitted on host
+  // side. In such lambda functions, a reference capture is invalid only
+  // if the lambda structure is populated by a device function or kernel then
+  // is passed to and called by a host function. However that is impossible,
+  // since a device function or kernel can only call a device function, also a
+  // kernel cannot pass a lambda back to a host function since we cannot
+  // define a kernel argument type which can hold the lambda before the lambda
+  // itself is defined.
+  if (!LangOpts.CUDAIsDevice)
+    return;
+
+  // File-scope lambda can only do init captures for global variables, which
+  // results in passing by value for these global variables.
+  FunctionDecl *Caller = dyn_cast<FunctionDecl>(CurContext);
+  if (!Caller)
+    return;
+
+  // In device compilation, we only need to check lambda functions which are
+  // emitted on device side. For such lambdas, a reference capture is invalid
+  // only if the lambda structure is populated by a host function then passed
+  // to and called in a device function or kernel.
+  bool CalleeIsDevice = Callee->hasAttr<CUDADeviceAttr>();
+  bool CallerIsHost =
+      !Caller->hasAttr<CUDAGlobalAttr>() && !Caller->hasAttr<CUDADeviceAttr>();
+  bool ShouldCheck = CalleeIsDevice && CallerIsHost;
+  if (!ShouldCheck || !Capture.isReferenceCapture())
+    return;
+  auto DiagKind = DeviceDiagBuilder::K_Deferred;
+  if (Capture.isVariableCapture()) {
+    DeviceDiagBuilder(DiagKind, Capture.getLocation(),
+                      diag::err_capture_bad_target, Callee, *this)
+        << Capture.getVariable();
+  } else if (Capture.isThisCapture()) {
+    DeviceDiagBuilder(DiagKind, Capture.getLocation(),
+                      diag::err_capture_bad_target_this_ptr, Callee, *this);
+  }
+  return;
+}
+
 void Sema::CUDASetLambdaAttrs(CXXMethodDecl *Method) {
   assert(getLangOpts().CUDA && "Should only be called during CUDA compilation");
   if (Method->hasAttr<CUDAHostAttr>() || Method->hasAttr<CUDADeviceAttr>())
     return;
-  FunctionDecl *CurFn = dyn_cast<FunctionDecl>(CurContext);
-  if (!CurFn)
-    return;
-  CUDAFunctionTarget Target = IdentifyCUDATarget(CurFn);
-  if (Target == CFT_Global || Target == CFT_Device) {
-    Method->addAttr(CUDADeviceAttr::CreateImplicit(Context));
-  } else if (Target == CFT_HostDevice) {
-    Method->addAttr(CUDADeviceAttr::CreateImplicit(Context));
-    Method->addAttr(CUDAHostAttr::CreateImplicit(Context));
-  }
+  Method->addAttr(CUDADeviceAttr::CreateImplicit(Context));
+  Method->addAttr(CUDAHostAttr::CreateImplicit(Context));
 }
 
 void Sema::checkCUDATargetOverload(FunctionDecl *NewFD,
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 883ff17eb279b..efaf36a693061 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -88,6 +88,7 @@
 #include "llvm/Support/SaveAndRestore.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
+#include <bitset>
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
@@ -237,8 +238,8 @@ static bool SemaBuiltinAlignment(Sema &S, CallExpr *TheCall, unsigned ID) {
   }
   Expr::EvalResult AlignResult;
   unsigned MaxAlignmentBits = S.Context.getIntWidth(SrcTy) - 1;
-  // We can't check validity of alignment if it is type dependent.
-  if (!AlignOp->isInstantiationDependent() &&
+  // We can't check validity of alignment if it is value dependent.
+  if (!AlignOp->isValueDependent() &&
       AlignOp->EvaluateAsInt(AlignResult, S.Context,
                              Expr::SE_AllowSideEffects)) {
     llvm::APSInt AlignValue = AlignResult.Val.getInt();
@@ -1808,6 +1809,36 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
     TheCall->setType(Context.IntTy);
     break;
   }
+  case Builtin::BI__builtin_expect_with_probability: {
+    // We first want to ensure we are called with 3 arguments
+    if (checkArgCount(*this, TheCall, 3))
+      return ExprError();
+    // then check probability is constant float in range [0.0, 1.0]
+    const Expr *ProbArg = TheCall->getArg(2);
+    SmallVector<PartialDiagnosticAt, 8> Notes;
+    Expr::EvalResult Eval;
+    Eval.Diag = &Notes;
+    if ((!ProbArg->EvaluateAsConstantExpr(Eval, Expr::EvaluateForCodeGen,
+                                          Context)) ||
+        !Eval.Val.isFloat()) {
+      Diag(ProbArg->getBeginLoc(), diag::err_probability_not_constant_float)
+          << ProbArg->getSourceRange();
+      for (const PartialDiagnosticAt &PDiag : Notes)
+        Diag(PDiag.first, PDiag.second);
+      return ExprError();
+    }
+    llvm::APFloat Probability = Eval.Val.getFloat();
+    bool LoseInfo = false;
+    Probability.convert(llvm::APFloat::IEEEdouble(),
+                        llvm::RoundingMode::Dynamic, &LoseInfo);
+    if (!(Probability >= llvm::APFloat(0.0) &&
+          Probability <= llvm::APFloat(1.0))) {
+      Diag(ProbArg->getBeginLoc(), diag::err_probability_out_of_range)
+          << ProbArg->getSourceRange();
+      return ExprError();
+    }
+    break;
+  }
   case Builtin::BI__builtin_preserve_access_index:
     if (SemaBuiltinPreserveAI(*this, TheCall))
       return ExprError();
@@ -3094,6 +3125,16 @@ bool Sema::CheckPPCBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID,
            SemaBuiltinConstantArgRange(TheCall, 1, 0, 1);
   case PPC::BI__builtin_pack_vector_int128:
     return SemaVSXCheck(TheCall);
+  case PPC::BI__builtin_altivec_vgnb:
+     return SemaBuiltinConstantArgRange(TheCall, 1, 2, 7);
+  case PPC::BI__builtin_vsx_xxeval:
+     return SemaBuiltinConstantArgRange(TheCall, 3, 0, 255);
+  case PPC::BI__builtin_altivec_vsldbi:
+     return SemaBuiltinConstantArgRange(TheCall, 2, 0, 7);
+  case PPC::BI__builtin_altivec_vsrdbi:
+     return SemaBuiltinConstantArgRange(TheCall, 2, 0, 7);
+  case PPC::BI__builtin_vsx_xxpermx:
+     return SemaBuiltinConstantArgRange(TheCall, 3, 0, 7);
   }
   return SemaBuiltinConstantArgRange(TheCall, i, l, u);
 }
@@ -3567,6 +3608,64 @@ bool Sema::CheckX86BuiltinGatherScatterScale(unsigned BuiltinID,
          << Arg->getSourceRange();
 }
 
+enum { TileRegLow = 0, TileRegHigh = 7 };
+
+bool Sema::CheckX86BuiltinTileArgumentsRange(CallExpr *TheCall,
+                                    ArrayRef<int> ArgNums) {
+  for (int ArgNum : ArgNums) {
+    if (SemaBuiltinConstantArgRange(TheCall, ArgNum, TileRegLow, TileRegHigh))
+      return true;
+  }
+  return false;
+}
+
+bool Sema::CheckX86BuiltinTileArgumentsRange(CallExpr *TheCall, int ArgNum) {
+  return SemaBuiltinConstantArgRange(TheCall, ArgNum, TileRegLow, TileRegHigh);
+}
+
+bool Sema::CheckX86BuiltinTileDuplicate(CallExpr *TheCall,
+                                        ArrayRef<int> ArgNums) {
+  // Because the max number of tile register is TileRegHigh + 1, so here we use
+  // each bit to represent the usage of them in bitset.
+  std::bitset<TileRegHigh + 1> ArgValues;
+  for (int ArgNum : ArgNums) {
+    llvm::APSInt Arg;
+    SemaBuiltinConstantArg(TheCall, ArgNum, Arg);
+    int ArgExtValue = Arg.getExtValue();
+    assert((ArgExtValue >= TileRegLow || ArgExtValue <= TileRegHigh) &&
+           "Incorrect tile register num.");
+    if (ArgValues.test(ArgExtValue))
+      return Diag(TheCall->getBeginLoc(),
+                  diag::err_x86_builtin_tile_arg_duplicate)
+             << TheCall->getArg(ArgNum)->getSourceRange();
+    ArgValues.set(ArgExtValue);
+  }
+  return false;
+}
+
+bool Sema::CheckX86BuiltinTileRangeAndDuplicate(CallExpr *TheCall,
+                                                ArrayRef<int> ArgNums) {
+  return CheckX86BuiltinTileArgumentsRange(TheCall, ArgNums) ||
+         CheckX86BuiltinTileDuplicate(TheCall, ArgNums);
+}
+
+bool Sema::CheckX86BuiltinTileArguments(unsigned BuiltinID, CallExpr *TheCall) {
+  switch (BuiltinID) {
+  default:
+    return false;
+  case X86::BI__builtin_ia32_tileloadd64:
+  case X86::BI__builtin_ia32_tileloaddt164:
+  case X86::BI__builtin_ia32_tilestored64:
+  case X86::BI__builtin_ia32_tilezero:
+    return CheckX86BuiltinTileArgumentsRange(TheCall, 0);
+  case X86::BI__builtin_ia32_tdpbssd:
+  case X86::BI__builtin_ia32_tdpbsud:
+  case X86::BI__builtin_ia32_tdpbusd:
+  case X86::BI__builtin_ia32_tdpbuud:
+  case X86::BI__builtin_ia32_tdpbf16ps:
+    return CheckX86BuiltinTileRangeAndDuplicate(TheCall, {0, 1, 2});
+  }
+}
 static bool isX86_32Builtin(unsigned BuiltinID) {
   // These builtins only work on x86-32 targets.
   switch (BuiltinID) {
@@ -3600,6 +3699,10 @@ bool Sema::CheckX86BuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID,
   if (CheckX86BuiltinGatherScatterScale(BuiltinID, TheCall))
     return true;
 
+  // If the intrinsic has a tile arguments, make sure they are valid.
+  if (CheckX86BuiltinTileArguments(BuiltinID, TheCall))
+    return true;
+
   // For intrinsics which take an immediate value as part of the instruction,
   // range check them here.
   int i = 0, l = 0, u = 0;
@@ -11833,27 +11936,31 @@ static void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
       }
 }
 
-static void CheckConditionalOperator(Sema &S, ConditionalOperator *E,
+static void CheckConditionalOperator(Sema &S, AbstractConditionalOperator *E,
                                      SourceLocation CC, QualType T);
 
 static void CheckConditionalOperand(Sema &S, Expr *E, QualType T,
                                     SourceLocation CC, bool &ICContext) {
   E = E->IgnoreParenImpCasts();
 
-  if (isa<ConditionalOperator>(E))
-    return CheckConditionalOperator(S, cast<ConditionalOperator>(E), CC, T);
+  if (auto *CO = dyn_cast<AbstractConditionalOperator>(E))
+    return CheckConditionalOperator(S, CO, CC, T);
 
   AnalyzeImplicitConversions(S, E, CC);
   if (E->getType() != T)
     return CheckImplicitConversion(S, E, T, CC, &ICContext);
 }
 
-static void CheckConditionalOperator(Sema &S, ConditionalOperator *E,
+static void CheckConditionalOperator(Sema &S, AbstractConditionalOperator *E,
                                      SourceLocation CC, QualType T) {
   AnalyzeImplicitConversions(S, E->getCond(), E->getQuestionLoc());
 
+  Expr *TrueExpr = E->getTrueExpr();
+  if (auto *BCO = dyn_cast<BinaryConditionalOperator>(E))
+    TrueExpr = BCO->getCommon();
+
   bool Suspicious = false;
-  CheckConditionalOperand(S, E->getTrueExpr(), T, CC, Suspicious);
+  CheckConditionalOperand(S, TrueExpr, T, CC, Suspicious);
   CheckConditionalOperand(S, E->getFalseExpr(), T, CC, Suspicious);
 
   if (T->isBooleanType())
@@ -11872,7 +11979,7 @@ static void CheckConditionalOperator(Sema &S, ConditionalOperator *E,
   if (E->getType() == T) return;
 
   Suspicious = false;
-  CheckImplicitConversion(S, E->getTrueExpr()->IgnoreParenImpCasts(),
+  CheckImplicitConversion(S, TrueExpr->IgnoreParenImpCasts(),
                           E->getType(), CC, &Suspicious);
   if (!Suspicious)
     CheckImplicitConversion(S, E->getFalseExpr()->IgnoreParenImpCasts(),
@@ -11935,7 +12042,7 @@ static void AnalyzeImplicitConversions(
 
   // For conditional operators, we analyze the arguments as if they
   // were being fed directly into the output.
-  if (auto *CO = dyn_cast<ConditionalOperator>(SourceExpr)) {
+  if (auto *CO = dyn_cast<AbstractConditionalOperator>(SourceExpr)) {
     CheckConditionalOperator(S, CO, CC, T);
     return;
   }
@@ -13453,12 +13560,14 @@ static getBaseAlignmentAndOffsetFromLValue(const Expr *E, ASTContext &Ctx) {
   }
   case Stmt::MemberExprClass: {
     auto *ME = cast<MemberExpr>(E);
-    if (ME->isArrow())
-      break;
     auto *FD = dyn_cast<FieldDecl>(ME->getMemberDecl());
     if (!FD || FD->getType()->isReferenceType())
       break;
-    auto P = getBaseAlignmentAndOffsetFromLValue(ME->getBase(), Ctx);
+    Optional<std::pair<CharUnits, CharUnits>> P;
+    if (ME->isArrow())
+      P = getBaseAlignmentAndOffsetFromPtr(ME->getBase(), Ctx);
+    else
+      P = getBaseAlignmentAndOffsetFromLValue(ME->getBase(), Ctx);
     if (!P)
       break;
     const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(FD->getParent());
@@ -13522,6 +13631,11 @@ static getBaseAlignmentAndOffsetFromPtr(const Expr *E, ASTContext &Ctx) {
     }
     break;
   }
+  case Stmt::CXXThisExprClass: {
+    auto *RD = E->getType()->getPointeeType()->getAsCXXRecordDecl();
+    CharUnits Alignment = Ctx.getASTRecordLayout(RD).getNonVirtualAlignment();
+    return std::make_pair(Alignment, CharUnits::Zero());
+  }
   case Stmt::UnaryOperatorClass: {
     auto *UO = cast<UnaryOperator>(E);
     if (UO->getOpcode() == UO_AddrOf)
@@ -15265,7 +15379,8 @@ ExprResult Sema::SemaBuiltinMatrixColumnMajorLoad(CallExpr *TheCall,
   if (checkArgCount(*this, TheCall, 4))
     return ExprError();
 
-  Expr *PtrExpr = TheCall->getArg(0);
+  unsigned PtrArgIdx = 0;
+  Expr *PtrExpr = TheCall->getArg(PtrArgIdx);
   Expr *RowsExpr = TheCall->getArg(1);
   Expr *ColumnsExpr = TheCall->getArg(2);
   Expr *StrideExpr = TheCall->getArg(3);
@@ -15289,14 +15404,14 @@ ExprResult Sema::SemaBuiltinMatrixColumnMajorLoad(CallExpr *TheCall,
   QualType ElementTy;
   if (!PtrTy) {
     Diag(PtrExpr->getBeginLoc(), diag::err_builtin_matrix_pointer_arg)
-        << "first";
+        << PtrArgIdx + 1;
     ArgError = true;
   } else {
     ElementTy = PtrTy->getPointeeType().getUnqualifiedType();
 
     if (!ConstantMatrixType::isValidElementType(ElementTy)) {
       Diag(PtrExpr->getBeginLoc(), diag::err_builtin_matrix_pointer_arg)
-          << "first";
+          << PtrArgIdx + 1;
       ArgError = true;
     }
   }
@@ -15372,8 +15487,9 @@ ExprResult Sema::SemaBuiltinMatrixColumnMajorStore(CallExpr *TheCall,
   if (checkArgCount(*this, TheCall, 3))
     return ExprError();
 
+  unsigned PtrArgIdx = 1;
   Expr *MatrixExpr = TheCall->getArg(0);
-  Expr *PtrExpr = TheCall->getArg(1);
+  Expr *PtrExpr = TheCall->getArg(PtrArgIdx);
   Expr *StrideExpr = TheCall->getArg(2);
 
   bool ArgError = false;
@@ -15412,7 +15528,7 @@ ExprResult Sema::SemaBuiltinMatrixColumnMajorStore(CallExpr *TheCall,
   auto *PtrTy = PtrExpr->getType()->getAs<PointerType>();
   if (!PtrTy) {
     Diag(PtrExpr->getBeginLoc(), diag::err_builtin_matrix_pointer_arg)
-        << "second";
+        << PtrArgIdx + 1;
     ArgError = true;
   } else {
     QualType ElementTy = PtrTy->getPointeeType();
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 5539aef917d09..0a8a27068ebfd 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -24,11 +24,13 @@
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/CharInfo.h"
+#include "clang/Basic/OperatorKinds.h"
 #include "clang/Basic/Specifiers.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
+#include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/Designator.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Overload.h"
@@ -1687,11 +1689,9 @@ static void AddTypeSpecifierResults(const LangOptions &LangOpts,
     Results.AddResult(Result("class", CCP_Type));
     Results.AddResult(Result("wchar_t", CCP_Type));
 
-    // typename qualified-id
+    // typename name
     Builder.AddTypedTextChunk("typename");
     Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Builder.AddPlaceholderChunk("qualifier");
-    Builder.AddTextChunk("::");
     Builder.AddPlaceholderChunk("name");
     Results.AddResult(Result(Builder.TakeString()));
 
@@ -1818,6 +1818,18 @@ static void AddTypedefResult(ResultBuilder &Results) {
   Results.AddResult(CodeCompletionResult(Builder.TakeString()));
 }
 
+// using name = type
+static void AddUsingAliasResult(CodeCompletionBuilder &Builder,
+                                ResultBuilder &Results) {
+  Builder.AddTypedTextChunk("using");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("name");
+  Builder.AddChunk(CodeCompletionString::CK_Equal);
+  Builder.AddPlaceholderChunk("type");
+  Builder.AddChunk(CodeCompletionString::CK_SemiColon);
+  Results.AddResult(CodeCompletionResult(Builder.TakeString()));
+}
+
 static bool WantTypesInContext(Sema::ParserCompletionContext CCC,
                                const LangOptions &LangOpts) {
   switch (CCC) {
@@ -2061,6 +2073,9 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC, Scope *S,
       Builder.AddChunk(CodeCompletionString::CK_SemiColon);
       Results.AddResult(Result(Builder.TakeString()));
 
+      if (SemaRef.getLangOpts().CPlusPlus11)
+        AddUsingAliasResult(Builder, Results);
+
       // using typename qualifier::name (only in a dependent context)
       if (SemaRef.CurContext->isDependentContext()) {
         Builder.AddTypedTextChunk("using typename");
@@ -2141,6 +2156,9 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC, Scope *S,
 
   case Sema::PCC_RecoveryInFunction:
   case Sema::PCC_Statement: {
+    if (SemaRef.getLangOpts().CPlusPlus11)
+      AddUsingAliasResult(Builder, Results);
+
     AddTypedefResult(Results);
 
     if (SemaRef.getLangOpts().CPlusPlus && Results.includeCodePatterns() &&
@@ -5743,7 +5761,7 @@ void Sema::CodeCompleteInitializer(Scope *S, Decl *D) {
   CodeCompleteExpression(S, Data);
 }
 
-void Sema::CodeCompleteAfterIf(Scope *S) {
+void Sema::CodeCompleteAfterIf(Scope *S, bool IsBracedThen) {
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         mapCodeCompletionContext(*this, PCC_Statement));
@@ -5760,15 +5778,25 @@ void Sema::CodeCompleteAfterIf(Scope *S) {
   // "else" block
   CodeCompletionBuilder Builder(Results.getAllocator(),
                                 Results.getCodeCompletionTUInfo());
+
+  auto AddElseBodyPattern = [&] {
+    if (IsBracedThen) {
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
+      Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+      Builder.AddPlaceholderChunk("statements");
+      Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+      Builder.AddChunk(CodeCompletionString::CK_RightBrace);
+    } else {
+      Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddPlaceholderChunk("statement");
+      Builder.AddChunk(CodeCompletionString::CK_SemiColon);
+    }
+  };
   Builder.AddTypedTextChunk("else");
-  if (Results.includeCodePatterns()) {
-    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
-    Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
-    Builder.AddPlaceholderChunk("statements");
-    Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
-    Builder.AddChunk(CodeCompletionString::CK_RightBrace);
-  }
+  if (Results.includeCodePatterns())
+    AddElseBodyPattern();
   Results.AddResult(Builder.TakeString());
 
   // "else if" block
@@ -5781,12 +5809,7 @@ void Sema::CodeCompleteAfterIf(Scope *S) {
     Builder.AddPlaceholderChunk("expression");
   Builder.AddChunk(CodeCompletionString::CK_RightParen);
   if (Results.includeCodePatterns()) {
-    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
-    Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
-    Builder.AddPlaceholderChunk("statements");
-    Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
-    Builder.AddChunk(CodeCompletionString::CK_RightBrace);
+    AddElseBodyPattern();
   }
   Results.AddResult(Builder.TakeString());
 
@@ -6245,6 +6268,53 @@ void Sema::CodeCompleteLambdaIntroducer(Scope *S, LambdaIntroducer &Intro,
                             Results.data(), Results.size());
 }
 
+void Sema::CodeCompleteAfterFunctionEquals(Declarator &D) {
+  if (!LangOpts.CPlusPlus11)
+    return;
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompleter->getCodeCompletionTUInfo(),
+                        CodeCompletionContext::CCC_Other);
+  auto ShouldAddDefault = [&D, this]() {
+    if (!D.isFunctionDeclarator())
+      return false;
+    auto &Id = D.getName();
+    if (Id.getKind() == UnqualifiedIdKind::IK_DestructorName)
+      return true;
+    // FIXME(liuhui): Ideally, we should check the constructor parameter list to
+    // verify that it is the default, copy or move constructor?
+    if (Id.getKind() == UnqualifiedIdKind::IK_ConstructorName &&
+        D.getFunctionTypeInfo().NumParams <= 1)
+      return true;
+    if (Id.getKind() == UnqualifiedIdKind::IK_OperatorFunctionId) {
+      auto Op = Id.OperatorFunctionId.Operator;
+      // FIXME(liuhui): Ideally, we should check the function parameter list to
+      // verify that it is the copy or move assignment?
+      if (Op == OverloadedOperatorKind::OO_Equal)
+        return true;
+      if (LangOpts.CPlusPlus20 &&
+          (Op == OverloadedOperatorKind::OO_EqualEqual ||
+           Op == OverloadedOperatorKind::OO_ExclaimEqual ||
+           Op == OverloadedOperatorKind::OO_Less ||
+           Op == OverloadedOperatorKind::OO_LessEqual ||
+           Op == OverloadedOperatorKind::OO_Greater ||
+           Op == OverloadedOperatorKind::OO_GreaterEqual ||
+           Op == OverloadedOperatorKind::OO_Spaceship))
+        return true;
+    }
+    return false;
+  };
+
+  Results.EnterNewScope();
+  if (ShouldAddDefault())
+    Results.AddResult("default");
+  // FIXME(liuhui): Ideally, we should only provide `delete` completion for the
+  // first function declaration.
+  Results.AddResult("delete");
+  Results.ExitScope();
+  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
+                            Results.data(), Results.size());
+}
+
 /// Macro that optionally prepends an "@" to the string literal passed in via
 /// Keyword, depending on whether NeedAt is true or false.
 #define OBJC_AT_KEYWORD_NAME(NeedAt, Keyword) ((NeedAt) ? "@" Keyword : Keyword)
@@ -8133,8 +8203,8 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
         Builder.AddChunk(CodeCompletionString::CK_RightParen);
       }
 
-      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorId->getName()));
-      Builder.AddTypedTextChunk(":");
+      Builder.AddTypedTextChunk(
+          Allocator.CopyString(SelectorId->getName() + ":"));
       AddObjCPassingTypeChunk(Property->getType(), /*Quals=*/0, Context, Policy,
                               Builder);
       Builder.AddTextChunk(Key);
@@ -8722,39 +8792,43 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S, Optional<bool> IsInstanceMethod,
 
     Selector Sel = Method->getSelector();
 
-    // Add the first part of the selector to the pattern.
-    Builder.AddTypedTextChunk(
-        Builder.getAllocator().CopyString(Sel.getNameForSlot(0)));
-
-    // Add parameters to the pattern.
-    unsigned I = 0;
-    for (ObjCMethodDecl::param_iterator P = Method->param_begin(),
-                                        PEnd = Method->param_end();
-         P != PEnd; (void)++P, ++I) {
-      // Add the part of the selector name.
-      if (I == 0)
-        Builder.AddTypedTextChunk(":");
-      else if (I < Sel.getNumArgs()) {
-        Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
-        Builder.AddTypedTextChunk(
-            Builder.getAllocator().CopyString(Sel.getNameForSlot(I) + ":"));
-      } else
-        break;
-
-      // Add the parameter type.
-      QualType ParamType;
-      if ((*P)->getObjCDeclQualifier() & Decl::OBJC_TQ_CSNullability)
-        ParamType = (*P)->getType();
-      else
-        ParamType = (*P)->getOriginalType();
-      ParamType = ParamType.substObjCTypeArgs(
-          Context, {}, ObjCSubstitutionContext::Parameter);
-      AttributedType::stripOuterNullability(ParamType);
-      AddObjCPassingTypeChunk(ParamType, (*P)->getObjCDeclQualifier(), Context,
-                              Policy, Builder);
+    if (Sel.isUnarySelector()) {
+      // Unary selectors have no arguments.
+      Builder.AddTypedTextChunk(
+          Builder.getAllocator().CopyString(Sel.getNameForSlot(0)));
+    } else {
+      // Add all parameters to the pattern.
+      unsigned I = 0;
+      for (ObjCMethodDecl::param_iterator P = Method->param_begin(),
+                                          PEnd = Method->param_end();
+           P != PEnd; (void)++P, ++I) {
+        // Add the part of the selector name.
+        if (I == 0)
+          Builder.AddTypedTextChunk(
+              Builder.getAllocator().CopyString(Sel.getNameForSlot(I) + ":"));
+        else if (I < Sel.getNumArgs()) {
+          Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+          Builder.AddTypedTextChunk(
+              Builder.getAllocator().CopyString(Sel.getNameForSlot(I) + ":"));
+        } else
+          break;
 
-      if (IdentifierInfo *Id = (*P)->getIdentifier())
-        Builder.AddTextChunk(Builder.getAllocator().CopyString(Id->getName()));
+        // Add the parameter type.
+        QualType ParamType;
+        if ((*P)->getObjCDeclQualifier() & Decl::OBJC_TQ_CSNullability)
+          ParamType = (*P)->getType();
+        else
+          ParamType = (*P)->getOriginalType();
+        ParamType = ParamType.substObjCTypeArgs(
+            Context, {}, ObjCSubstitutionContext::Parameter);
+        AttributedType::stripOuterNullability(ParamType);
+        AddObjCPassingTypeChunk(ParamType, (*P)->getObjCDeclQualifier(),
+                                Context, Policy, Builder);
+
+        if (IdentifierInfo *Id = (*P)->getIdentifier())
+          Builder.AddTextChunk(
+              Builder.getAllocator().CopyString(Id->getName()));
+      }
     }
 
     if (Method->isVariadic()) {
diff --git a/clang/lib/Sema/SemaCoroutine.cpp b/clang/lib/Sema/SemaCoroutine.cpp
index 5ed0bbd6041d1..992cccac64050 100644
--- a/clang/lib/Sema/SemaCoroutine.cpp
+++ b/clang/lib/Sema/SemaCoroutine.cpp
@@ -24,6 +24,7 @@
 #include "clang/Sema/Overload.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaInternal.h"
+#include "llvm/ADT/SmallSet.h"
 
 using namespace clang;
 using namespace sema;
@@ -390,7 +391,13 @@ static Expr *maybeTailCall(Sema &S, QualType RetType, Expr *E,
     return nullptr;
 
   Expr *JustAddress = AddressExpr.get();
-  // FIXME: Check that the type of AddressExpr is void*
+
+  // Check that the type of AddressExpr is void*
+  if (!JustAddress->getType().getTypePtr()->isVoidPointerType())
+    S.Diag(cast<CallExpr>(JustAddress)->getCalleeDecl()->getLocation(),
+           diag::warn_coroutine_handle_address_invalid_return_type)
+        << JustAddress->getType();
+
   return buildBuiltinCall(S, Loc, Builtin::BI__builtin_coro_resume,
                           JustAddress);
 }
@@ -604,6 +611,80 @@ static FunctionScopeInfo *checkCoroutineContext(Sema &S, SourceLocation Loc,
   return ScopeInfo;
 }
 
+/// Recursively check \p E and all its children to see if any call target
+/// (including constructor call) is declared noexcept. Also any value returned
+/// from the call has a noexcept destructor.
+static void checkNoThrow(Sema &S, const Stmt *E,
+                         llvm::SmallPtrSetImpl<const Decl *> &ThrowingDecls) {
+  auto checkDeclNoexcept = [&](const Decl *D, bool IsDtor = false) {
+    // In the case of dtor, the call to dtor is implicit and hence we should
+    // pass nullptr to canCalleeThrow.
+    if (Sema::canCalleeThrow(S, IsDtor ? nullptr : cast<Expr>(E), D)) {
+      if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
+        // co_await promise.final_suspend() could end up calling
+        // __builtin_coro_resume for symmetric transfer if await_suspend()
+        // returns a handle. In that case, even __builtin_coro_resume is not
+        // declared as noexcept and may throw, it does not throw _into_ the
+        // coroutine that just suspended, but rather throws back out from
+        // whoever called coroutine_handle::resume(), hence we claim that
+        // logically it does not throw.
+        if (FD->getBuiltinID() == Builtin::BI__builtin_coro_resume)
+          return;
+      }
+      if (ThrowingDecls.empty()) {
+        // First time seeing an error, emit the error message.
+        S.Diag(cast<FunctionDecl>(S.CurContext)->getLocation(),
+               diag::err_coroutine_promise_final_suspend_requires_nothrow);
+      }
+      ThrowingDecls.insert(D);
+    }
+  };
+  auto SC = E->getStmtClass();
+  if (SC == Expr::CXXConstructExprClass) {
+    auto const *Ctor = cast<CXXConstructExpr>(E)->getConstructor();
+    checkDeclNoexcept(Ctor);
+    // Check the corresponding destructor of the constructor.
+    checkDeclNoexcept(Ctor->getParent()->getDestructor(), true);
+  } else if (SC == Expr::CallExprClass || SC == Expr::CXXMemberCallExprClass ||
+             SC == Expr::CXXOperatorCallExprClass) {
+    if (!cast<CallExpr>(E)->isTypeDependent()) {
+      checkDeclNoexcept(cast<CallExpr>(E)->getCalleeDecl());
+      auto ReturnType = cast<CallExpr>(E)->getCallReturnType(S.getASTContext());
+      // Check the destructor of the call return type, if any.
+      if (ReturnType.isDestructedType() ==
+          QualType::DestructionKind::DK_cxx_destructor) {
+        const auto *T =
+            cast<RecordType>(ReturnType.getCanonicalType().getTypePtr());
+        checkDeclNoexcept(
+            dyn_cast<CXXRecordDecl>(T->getDecl())->getDestructor(), true);
+      }
+    }
+  }
+  for (const auto *Child : E->children()) {
+    if (!Child)
+      continue;
+    checkNoThrow(S, Child, ThrowingDecls);
+  }
+}
+
+bool Sema::checkFinalSuspendNoThrow(const Stmt *FinalSuspend) {
+  llvm::SmallPtrSet<const Decl *, 4> ThrowingDecls;
+  // We first collect all declarations that should not throw but not declared
+  // with noexcept. We then sort them based on the location before printing.
+  // This is to avoid emitting the same note multiple times on the same
+  // declaration, and also provide a deterministic order for the messages.
+  checkNoThrow(*this, FinalSuspend, ThrowingDecls);
+  auto SortedDecls = llvm::SmallVector<const Decl *, 4>{ThrowingDecls.begin(),
+                                                        ThrowingDecls.end()};
+  sort(SortedDecls, [](const Decl *A, const Decl *B) {
+    return A->getEndLoc() < B->getEndLoc();
+  });
+  for (const auto *D : SortedDecls) {
+    Diag(D->getEndLoc(), diag::note_coroutine_function_declare_noexcept);
+  }
+  return ThrowingDecls.empty();
+}
+
 bool Sema::ActOnCoroutineBodyStart(Scope *SC, SourceLocation KWLoc,
                                    StringRef Keyword) {
   if (!checkCoroutineContext(*this, KWLoc, Keyword))
@@ -646,7 +727,7 @@ bool Sema::ActOnCoroutineBodyStart(Scope *SC, SourceLocation KWLoc,
     return true;
 
   StmtResult FinalSuspend = buildSuspends("final_suspend");
-  if (FinalSuspend.isInvalid())
+  if (FinalSuspend.isInvalid() || !checkFinalSuspendNoThrow(FinalSuspend.get()))
     return true;
 
   ScopeInfo->setCoroutineSuspends(InitSuspend.get(), FinalSuspend.get());
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 5eebfb7dc1f0c..f5e375134c293 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -750,7 +750,10 @@ void Sema::DiagnoseUnknownTypeName(IdentifierInfo *&II,
     Diag(IILoc, IsTemplateName ? diag::err_no_member_template
                                : diag::err_typename_nested_not_found)
         << II << DC << SS->getRange();
-  else if (isDependentScopeSpecifier(*SS)) {
+  else if (SS->isValid() && SS->getScopeRep()->containsErrors()) {
+    SuggestedType =
+        ActOnTypenameType(S, SourceLocation(), *SS, *II, IILoc).get();
+  } else if (isDependentScopeSpecifier(*SS)) {
     unsigned DiagID = diag::err_typename_missing;
     if (getLangOpts().MSVCCompat && isMicrosoftMissingTypename(SS, S))
       DiagID = diag::ext_typename_missing;
@@ -1257,47 +1260,8 @@ Sema::getTemplateNameKindForDiagnostics(TemplateName Name) {
   return TemplateNameKindForDiagnostics::DependentTemplate;
 }
 
-// Determines the context to return to after temporarily entering a
-// context.  This depends in an unnecessarily complicated way on the
-// exact ordering of callbacks from the parser.
-DeclContext *Sema::getContainingDC(DeclContext *DC) {
-
-  // Functions defined inline within classes aren't parsed until we've
-  // finished parsing the top-level class, so the top-level class is
-  // the context we'll need to return to.
-  // A Lambda call operator whose parent is a class must not be treated
-  // as an inline member function.  A Lambda can be used legally
-  // either as an in-class member initializer or a default argument.  These
-  // are parsed once the class has been marked complete and so the containing
-  // context would be the nested class (when the lambda is defined in one);
-  // If the class is not complete, then the lambda is being used in an
-  // ill-formed fashion (such as to specify the width of a bit-field, or
-  // in an array-bound) - in which case we still want to return the
-  // lexically containing DC (which could be a nested class).
-  if (isa<FunctionDecl>(DC) && !isLambdaCallOperator(DC)) {
-    DC = DC->getLexicalParent();
-
-    // A function not defined within a class will always return to its
-    // lexical context.
-    if (!isa<CXXRecordDecl>(DC))
-      return DC;
-
-    // A C++ inline method/friend is parsed *after* the topmost class
-    // it was declared in is fully parsed ("complete");  the topmost
-    // class is the context we need to return to.
-    while (CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(DC->getLexicalParent()))
-      DC = RD;
-
-    // Return the declaration context of the topmost class the inline method is
-    // declared in.
-    return DC;
-  }
-
-  return DC->getLexicalParent();
-}
-
 void Sema::PushDeclContext(Scope *S, DeclContext *DC) {
-  assert(getContainingDC(DC) == CurContext &&
+  assert(DC->getLexicalParent() == CurContext &&
       "The next DeclContext should be lexically contained in the current one.");
   CurContext = DC;
   S->setEntity(DC);
@@ -1306,7 +1270,7 @@ void Sema::PushDeclContext(Scope *S, DeclContext *DC) {
 void Sema::PopDeclContext() {
   assert(CurContext && "DeclContext imbalance!");
 
-  CurContext = getContainingDC(CurContext);
+  CurContext = CurContext->getLexicalParent();
   assert(CurContext && "Popped translation unit!");
 }
 
@@ -1358,6 +1322,12 @@ void Sema::EnterDeclaratorContext(Scope *S, DeclContext *DC) {
 
   CurContext = DC;
   S->setEntity(DC);
+
+  if (S->getParent()->isTemplateParamScope()) {
+    // Also set the corresponding entities for all immediately-enclosing
+    // template parameter scopes.
+    EnterTemplatedContext(S->getParent(), DC);
+  }
 }
 
 void Sema::ExitDeclaratorContext(Scope *S) {
@@ -1373,6 +1343,49 @@ void Sema::ExitDeclaratorContext(Scope *S) {
   // disappear.
 }
 
+void Sema::EnterTemplatedContext(Scope *S, DeclContext *DC) {
+  assert(S->isTemplateParamScope() &&
+         "expected to be initializing a template parameter scope");
+
+  // C++20 [temp.local]p7:
+  //   In the definition of a member of a class template that appears outside
+  //   of the class template definition, the name of a member of the class
+  //   template hides the name of a template-parameter of any enclosing class
+  //   templates (but not a template-parameter of the member if the member is a
+  //   class or function template).
+  // C++20 [temp.local]p9:
+  //   In the definition of a class template or in the definition of a member
+  //   of such a template that appears outside of the template definition, for
+  //   each non-dependent base class (13.8.2.1), if the name of the base class
+  //   or the name of a member of the base class is the same as the name of a
+  //   template-parameter, the base class name or member name hides the
+  //   template-parameter name (6.4.10).
+  //
+  // This means that a template parameter scope should be searched immediately
+  // after searching the DeclContext for which it is a template parameter
+  // scope. For example, for
+  //   template<typename T> template<typename U> template<typename V>
+  //     void N::A<T>::B<U>::f(...)
+  // we search V then B<U> (and base classes) then U then A<T> (and base
+  // classes) then T then N then ::.
+  unsigned ScopeDepth = getTemplateDepth(S);
+  for (; S && S->isTemplateParamScope(); S = S->getParent(), --ScopeDepth) {
+    DeclContext *SearchDCAfterScope = DC;
+    for (; DC; DC = DC->getLookupParent()) {
+      if (const TemplateParameterList *TPL =
+              cast<Decl>(DC)->getDescribedTemplateParams()) {
+        unsigned DCDepth = TPL->getDepth() + 1;
+        if (DCDepth > ScopeDepth)
+          continue;
+        if (ScopeDepth == DCDepth)
+          SearchDCAfterScope = DC = DC->getLookupParent();
+        break;
+      }
+    }
+    S->setLookupEntity(SearchDCAfterScope);
+  }
+}
+
 void Sema::ActOnReenterFunctionContext(Scope* S, Decl *D) {
   // We assume that the caller has already called
   // ActOnReenterTemplateScope so getTemplatedDecl() works.
@@ -16152,7 +16165,7 @@ Decl *Sema::ActOnObjCContainerStartDefinition(Decl *IDecl) {
   assert(isa<ObjCContainerDecl>(IDecl) &&
          "ActOnObjCContainerStartDefinition - Not ObjCContainerDecl");
   DeclContext *OCD = cast<DeclContext>(IDecl);
-  assert(getContainingDC(OCD) == CurContext &&
+  assert(OCD->getLexicalParent() == CurContext &&
       "The next DeclContext should be lexically contained in the current one.");
   CurContext = OCD;
   return IDecl;
@@ -17171,10 +17184,10 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
           I.setAccess((*I)->getAccess());
       }
 
-      if (!CXXRecord->isDependentType()) {
-        // Add any implicitly-declared members to this class.
-        AddImplicitlyDeclaredMembersToClass(CXXRecord);
+      // Add any implicitly-declared members to this class.
+      AddImplicitlyDeclaredMembersToClass(CXXRecord);
 
+      if (!CXXRecord->isDependentType()) {
         if (!CXXRecord->isInvalidDecl()) {
           // If we have virtual base classes, we may end up finding multiple
           // final overriders for a given virtual function. Check for this
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index a2b669c991258..515a2e9690ed1 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -2426,7 +2426,10 @@ Sema::CheckBaseSpecifier(CXXRecordDecl *Class,
                          TypeSourceInfo *TInfo,
                          SourceLocation EllipsisLoc) {
   QualType BaseType = TInfo->getType();
-
+  if (BaseType->containsErrors()) {
+    // Already emitted a diagnostic when parsing the error type.
+    return nullptr;
+  }
   // C++ [class.union]p1:
   //   A union shall not have base classes.
   if (Class->isUnion()) {
@@ -3042,7 +3045,7 @@ void Sema::CheckOverrideControl(NamedDecl *D) {
       << MD->getDeclName();
 }
 
-void Sema::DiagnoseAbsenceOfOverrideControl(NamedDecl *D) {
+void Sema::DiagnoseAbsenceOfOverrideControl(NamedDecl *D, bool Inconsistent) {
   if (D->isInvalidDecl() || D->hasAttr<OverrideAttr>())
     return;
   CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(D);
@@ -3058,12 +3061,22 @@ void Sema::DiagnoseAbsenceOfOverrideControl(NamedDecl *D) {
       return;
 
   if (MD->size_overridden_methods() > 0) {
-    unsigned DiagID = isa<CXXDestructorDecl>(MD)
-                          ? diag::warn_destructor_marked_not_override_overriding
-                          : diag::warn_function_marked_not_override_overriding;
-    Diag(MD->getLocation(), DiagID) << MD->getDeclName();
-    const CXXMethodDecl *OMD = *MD->begin_overridden_methods();
-    Diag(OMD->getLocation(), diag::note_overridden_virtual_function);
+    auto EmitDiag = [&](unsigned DiagInconsistent, unsigned DiagSuggest) {
+      unsigned DiagID =
+          Inconsistent && !Diags.isIgnored(DiagInconsistent, MD->getLocation())
+              ? DiagInconsistent
+              : DiagSuggest;
+      Diag(MD->getLocation(), DiagID) << MD->getDeclName();
+      const CXXMethodDecl *OMD = *MD->begin_overridden_methods();
+      Diag(OMD->getLocation(), diag::note_overridden_virtual_function);
+    };
+    if (isa<CXXDestructorDecl>(MD))
+      EmitDiag(
+          diag::warn_inconsistent_destructor_marked_not_override_overriding,
+          diag::warn_suggest_destructor_marked_not_override_overriding);
+    else
+      EmitDiag(diag::warn_inconsistent_function_marked_not_override_overriding,
+               diag::warn_suggest_function_marked_not_override_overriding);
   }
 }
 
@@ -6746,13 +6759,10 @@ void Sema::CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record) {
     }
   }
 
-  if (HasMethodWithOverrideControl &&
-      HasOverridingMethodWithoutOverrideControl) {
-    // At least one method has the 'override' control declared.
-    // Diagnose all other overridden methods which do not have 'override'
-    // specified on them.
+  if (HasOverridingMethodWithoutOverrideControl) {
+    bool HasInconsistentOverrideControl = HasMethodWithOverrideControl;
     for (auto *M : Record->methods())
-      DiagnoseAbsenceOfOverrideControl(M);
+      DiagnoseAbsenceOfOverrideControl(M, HasInconsistentOverrideControl);
   }
 
   // Check the defaulted secondary comparisons after any other member functions.
@@ -9716,6 +9726,10 @@ void Sema::checkIllFormedTrivialABIStruct(CXXRecordDecl &RD) {
 
   // Ill-formed if the copy and move constructors are deleted.
   auto HasNonDeletedCopyOrMoveConstructor = [&]() {
+    // If the type is dependent, then assume it might have
+    // implicit copy or move ctor because we won't know yet at this point.
+    if (RD.isDependentType())
+      return true;
     if (RD.needsImplicitCopyConstructor() &&
         !RD.defaultedCopyConstructorIsDeleted())
       return true;
@@ -9839,86 +9853,95 @@ static void findImplicitlyDeclaredEqualityComparisons(
 /// [special]p1).  This routine can only be executed just before the
 /// definition of the class is complete.
 void Sema::AddImplicitlyDeclaredMembersToClass(CXXRecordDecl *ClassDecl) {
-  if (ClassDecl->needsImplicitDefaultConstructor()) {
-    ++getASTContext().NumImplicitDefaultConstructors;
+  // Don't add implicit special members to templated classes.
+  // FIXME: This means unqualified lookups for 'operator=' within a class
+  // template don't work properly.
+  if (!ClassDecl->isDependentType()) {
+    if (ClassDecl->needsImplicitDefaultConstructor()) {
+      ++getASTContext().NumImplicitDefaultConstructors;
 
-    if (ClassDecl->hasInheritedConstructor())
-      DeclareImplicitDefaultConstructor(ClassDecl);
-  }
+      if (ClassDecl->hasInheritedConstructor())
+        DeclareImplicitDefaultConstructor(ClassDecl);
+    }
 
-  if (ClassDecl->needsImplicitCopyConstructor()) {
-    ++getASTContext().NumImplicitCopyConstructors;
+    if (ClassDecl->needsImplicitCopyConstructor()) {
+      ++getASTContext().NumImplicitCopyConstructors;
 
-    // If the properties or semantics of the copy constructor couldn't be
-    // determined while the class was being declared, force a declaration
-    // of it now.
-    if (ClassDecl->needsOverloadResolutionForCopyConstructor() ||
-        ClassDecl->hasInheritedConstructor())
-      DeclareImplicitCopyConstructor(ClassDecl);
-    // For the MS ABI we need to know whether the copy ctor is deleted. A
-    // prerequisite for deleting the implicit copy ctor is that the class has a
-    // move ctor or move assignment that is either user-declared or whose
-    // semantics are inherited from a subobject. FIXME: We should provide a more
-    // direct way for CodeGen to ask whether the constructor was deleted.
-    else if (Context.getTargetInfo().getCXXABI().isMicrosoft() &&
-             (ClassDecl->hasUserDeclaredMoveConstructor() ||
-              ClassDecl->needsOverloadResolutionForMoveConstructor() ||
-              ClassDecl->hasUserDeclaredMoveAssignment() ||
-              ClassDecl->needsOverloadResolutionForMoveAssignment()))
-      DeclareImplicitCopyConstructor(ClassDecl);
-  }
+      // If the properties or semantics of the copy constructor couldn't be
+      // determined while the class was being declared, force a declaration
+      // of it now.
+      if (ClassDecl->needsOverloadResolutionForCopyConstructor() ||
+          ClassDecl->hasInheritedConstructor())
+        DeclareImplicitCopyConstructor(ClassDecl);
+      // For the MS ABI we need to know whether the copy ctor is deleted. A
+      // prerequisite for deleting the implicit copy ctor is that the class has
+      // a move ctor or move assignment that is either user-declared or whose
+      // semantics are inherited from a subobject. FIXME: We should provide a
+      // more direct way for CodeGen to ask whether the constructor was deleted.
+      else if (Context.getTargetInfo().getCXXABI().isMicrosoft() &&
+               (ClassDecl->hasUserDeclaredMoveConstructor() ||
+                ClassDecl->needsOverloadResolutionForMoveConstructor() ||
+                ClassDecl->hasUserDeclaredMoveAssignment() ||
+                ClassDecl->needsOverloadResolutionForMoveAssignment()))
+        DeclareImplicitCopyConstructor(ClassDecl);
+    }
 
-  if (getLangOpts().CPlusPlus11 && ClassDecl->needsImplicitMoveConstructor()) {
-    ++getASTContext().NumImplicitMoveConstructors;
+    if (getLangOpts().CPlusPlus11 &&
+        ClassDecl->needsImplicitMoveConstructor()) {
+      ++getASTContext().NumImplicitMoveConstructors;
 
-    if (ClassDecl->needsOverloadResolutionForMoveConstructor() ||
-        ClassDecl->hasInheritedConstructor())
-      DeclareImplicitMoveConstructor(ClassDecl);
-  }
+      if (ClassDecl->needsOverloadResolutionForMoveConstructor() ||
+          ClassDecl->hasInheritedConstructor())
+        DeclareImplicitMoveConstructor(ClassDecl);
+    }
 
-  if (ClassDecl->needsImplicitCopyAssignment()) {
-    ++getASTContext().NumImplicitCopyAssignmentOperators;
+    if (ClassDecl->needsImplicitCopyAssignment()) {
+      ++getASTContext().NumImplicitCopyAssignmentOperators;
 
-    // If we have a dynamic class, then the copy assignment operator may be
-    // virtual, so we have to declare it immediately. This ensures that, e.g.,
-    // it shows up in the right place in the vtable and that we diagnose
-    // problems with the implicit exception specification.
-    if (ClassDecl->isDynamicClass() ||
-        ClassDecl->needsOverloadResolutionForCopyAssignment() ||
-        ClassDecl->hasInheritedAssignment())
-      DeclareImplicitCopyAssignment(ClassDecl);
-  }
+      // If we have a dynamic class, then the copy assignment operator may be
+      // virtual, so we have to declare it immediately. This ensures that, e.g.,
+      // it shows up in the right place in the vtable and that we diagnose
+      // problems with the implicit exception specification.
+      if (ClassDecl->isDynamicClass() ||
+          ClassDecl->needsOverloadResolutionForCopyAssignment() ||
+          ClassDecl->hasInheritedAssignment())
+        DeclareImplicitCopyAssignment(ClassDecl);
+    }
 
-  if (getLangOpts().CPlusPlus11 && ClassDecl->needsImplicitMoveAssignment()) {
-    ++getASTContext().NumImplicitMoveAssignmentOperators;
+    if (getLangOpts().CPlusPlus11 && ClassDecl->needsImplicitMoveAssignment()) {
+      ++getASTContext().NumImplicitMoveAssignmentOperators;
 
-    // Likewise for the move assignment operator.
-    if (ClassDecl->isDynamicClass() ||
-        ClassDecl->needsOverloadResolutionForMoveAssignment() ||
-        ClassDecl->hasInheritedAssignment())
-      DeclareImplicitMoveAssignment(ClassDecl);
-  }
+      // Likewise for the move assignment operator.
+      if (ClassDecl->isDynamicClass() ||
+          ClassDecl->needsOverloadResolutionForMoveAssignment() ||
+          ClassDecl->hasInheritedAssignment())
+        DeclareImplicitMoveAssignment(ClassDecl);
+    }
 
-  if (ClassDecl->needsImplicitDestructor()) {
-    ++getASTContext().NumImplicitDestructors;
+    if (ClassDecl->needsImplicitDestructor()) {
+      ++getASTContext().NumImplicitDestructors;
 
-    // If we have a dynamic class, then the destructor may be virtual, so we
-    // have to declare the destructor immediately. This ensures that, e.g., it
-    // shows up in the right place in the vtable and that we diagnose problems
-    // with the implicit exception specification.
-    if (ClassDecl->isDynamicClass() ||
-        ClassDecl->needsOverloadResolutionForDestructor())
-      DeclareImplicitDestructor(ClassDecl);
+      // If we have a dynamic class, then the destructor may be virtual, so we
+      // have to declare the destructor immediately. This ensures that, e.g., it
+      // shows up in the right place in the vtable and that we diagnose problems
+      // with the implicit exception specification.
+      if (ClassDecl->isDynamicClass() ||
+          ClassDecl->needsOverloadResolutionForDestructor())
+        DeclareImplicitDestructor(ClassDecl);
+    }
   }
 
   // C++2a [class.compare.default]p3:
   //   If the member-specification does not explicitly declare any member or
   //   friend named operator==, an == operator function is declared implicitly
-  //   for each defaulted three-way comparison operator function defined in the
-  //   member-specification
+  //   for each defaulted three-way comparison operator function defined in
+  //   the member-specification
   // FIXME: Consider doing this lazily.
-  if (getLangOpts().CPlusPlus20) {
-    llvm::SmallVector<FunctionDecl*, 4> DefaultedSpaceships;
+  // We do this during the initial parse for a class template, not during
+  // instantiation, so that we can handle unqualified lookups for 'operator=='
+  // when parsing the template.
+  if (getLangOpts().CPlusPlus20 && !inTemplateInstantiation()) {
+    llvm::SmallVector<FunctionDecl *, 4> DefaultedSpaceships;
     findImplicitlyDeclaredEqualityComparisons(Context, ClassDecl,
                                               DefaultedSpaceships);
     for (auto *FD : DefaultedSpaceships)
@@ -9926,19 +9949,17 @@ void Sema::AddImplicitlyDeclaredMembersToClass(CXXRecordDecl *ClassDecl) {
   }
 }
 
-unsigned Sema::ActOnReenterTemplateScope(Scope *S, Decl *D) {
+unsigned
+Sema::ActOnReenterTemplateScope(Decl *D,
+                                llvm::function_ref<Scope *()> EnterScope) {
   if (!D)
     return 0;
+  AdjustDeclIfTemplate(D);
 
-  // The order of template parameters is not important here. All names
-  // get added to the same scope.
+  // In order to get name lookup right, reenter template scopes in order from
+  // outermost to innermost.
   SmallVector<TemplateParameterList *, 4> ParameterLists;
-
-  if (TemplateDecl *TD = dyn_cast<TemplateDecl>(D))
-    D = TD->getTemplatedDecl();
-
-  if (auto *PSD = dyn_cast<ClassTemplatePartialSpecializationDecl>(D))
-    ParameterLists.push_back(PSD->getTemplateParameters());
+  DeclContext *LookupDC = dyn_cast<DeclContext>(D);
 
   if (DeclaratorDecl *DD = dyn_cast<DeclaratorDecl>(D)) {
     for (unsigned i = 0; i < DD->getNumTemplateParameterLists(); ++i)
@@ -9947,31 +9968,49 @@ unsigned Sema::ActOnReenterTemplateScope(Scope *S, Decl *D) {
     if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
       if (FunctionTemplateDecl *FTD = FD->getDescribedFunctionTemplate())
         ParameterLists.push_back(FTD->getTemplateParameters());
-    }
-  }
+    } else if (VarDecl *VD = dyn_cast<VarDecl>(D)) {
+      LookupDC = VD->getDeclContext();
 
-  if (TagDecl *TD = dyn_cast<TagDecl>(D)) {
+      if (VarTemplateDecl *VTD = VD->getDescribedVarTemplate())
+        ParameterLists.push_back(VTD->getTemplateParameters());
+      else if (auto *PSD = dyn_cast<VarTemplatePartialSpecializationDecl>(D))
+        ParameterLists.push_back(PSD->getTemplateParameters());
+    }
+  } else if (TagDecl *TD = dyn_cast<TagDecl>(D)) {
     for (unsigned i = 0; i < TD->getNumTemplateParameterLists(); ++i)
       ParameterLists.push_back(TD->getTemplateParameterList(i));
 
     if (CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(TD)) {
       if (ClassTemplateDecl *CTD = RD->getDescribedClassTemplate())
         ParameterLists.push_back(CTD->getTemplateParameters());
+      else if (auto *PSD = dyn_cast<ClassTemplatePartialSpecializationDecl>(D))
+        ParameterLists.push_back(PSD->getTemplateParameters());
     }
   }
+  // FIXME: Alias declarations and concepts.
 
   unsigned Count = 0;
+  Scope *InnermostTemplateScope = nullptr;
   for (TemplateParameterList *Params : ParameterLists) {
-    if (Params->size() > 0)
-      // Ignore explicit specializations; they don't contribute to the template
-      // depth.
-      ++Count;
+    // Ignore explicit specializations; they don't contribute to the template
+    // depth.
+    if (Params->size() == 0)
+      continue;
+
+    InnermostTemplateScope = EnterScope();
     for (NamedDecl *Param : *Params) {
       if (Param->getDeclName()) {
-        S->AddDecl(Param);
+        InnermostTemplateScope->AddDecl(Param);
         IdResolver.AddDecl(Param);
       }
     }
+    ++Count;
+  }
+
+  // Associate the new template scopes with the corresponding entities.
+  if (InnermostTemplateScope) {
+    assert(LookupDC && "no enclosing DeclContext for template lookup");
+    EnterTemplatedContext(InnermostTemplateScope, LookupDC);
   }
 
   return Count;
@@ -13498,11 +13537,11 @@ buildMemcpyForAssignmentOp(Sema &S, SourceLocation Loc, QualType T,
   Expr *From = FromB.build(S, Loc);
   From = UnaryOperator::Create(
       S.Context, From, UO_AddrOf, S.Context.getPointerType(From->getType()),
-      VK_RValue, OK_Ordinary, Loc, false, S.CurFPFeatures);
+      VK_RValue, OK_Ordinary, Loc, false, S.CurFPFeatureOverrides());
   Expr *To = ToB.build(S, Loc);
-  To = UnaryOperator::Create(S.Context, To, UO_AddrOf,
-                             S.Context.getPointerType(To->getType()), VK_RValue,
-                             OK_Ordinary, Loc, false, S.CurFPFeatures);
+  To = UnaryOperator::Create(
+      S.Context, To, UO_AddrOf, S.Context.getPointerType(To->getType()),
+      VK_RValue, OK_Ordinary, Loc, false, S.CurFPFeatureOverrides());
 
   const Type *E = T->getBaseElementTypeUnsafe();
   bool NeedsCollectableMemCpy =
@@ -13739,14 +13778,14 @@ buildSingleCopyAssignRecursively(Sema &S, SourceLocation Loc, QualType T,
   Expr *Comparison = BinaryOperator::Create(
       S.Context, IterationVarRefRVal.build(S, Loc),
       IntegerLiteral::Create(S.Context, Upper, SizeType, Loc), BO_NE,
-      S.Context.BoolTy, VK_RValue, OK_Ordinary, Loc, S.CurFPFeatures);
+      S.Context.BoolTy, VK_RValue, OK_Ordinary, Loc, S.CurFPFeatureOverrides());
 
   // Create the pre-increment of the iteration variable. We can determine
   // whether the increment will overflow based on the value of the array
   // bound.
   Expr *Increment = UnaryOperator::Create(
       S.Context, IterationVarRef.build(S, Loc), UO_PreInc, SizeType, VK_LValue,
-      OK_Ordinary, Loc, Upper.isMaxValue(), S.CurFPFeatures);
+      OK_Ordinary, Loc, Upper.isMaxValue(), S.CurFPFeatureOverrides());
 
   // Construct the loop that copies all elements of this array.
   return S.ActOnForStmt(
@@ -15184,12 +15223,6 @@ CheckOperatorNewDeleteTypes(Sema &SemaRef, const FunctionDecl *FnDecl,
   QualType ResultType =
       FnDecl->getType()->castAs<FunctionType>()->getReturnType();
 
-  // Check that the result type is not dependent.
-  if (ResultType->isDependentType())
-    return SemaRef.Diag(FnDecl->getLocation(),
-                        diag::err_operator_new_delete_dependent_result_type)
-    << FnDecl->getDeclName() << ExpectedResultType;
-
   // The operator is valid on any address space for OpenCL.
   if (SemaRef.getLangOpts().OpenCLCPlusPlus) {
     if (auto *PtrTy = ResultType->getAs<PointerType>()) {
@@ -15198,10 +15231,16 @@ CheckOperatorNewDeleteTypes(Sema &SemaRef, const FunctionDecl *FnDecl,
   }
 
   // Check that the result type is what we expect.
-  if (SemaRef.Context.getCanonicalType(ResultType) != ExpectedResultType)
-    return SemaRef.Diag(FnDecl->getLocation(),
-                        diag::err_operator_new_delete_invalid_result_type)
-    << FnDecl->getDeclName() << ExpectedResultType;
+  if (SemaRef.Context.getCanonicalType(ResultType) != ExpectedResultType) {
+    // Reject even if the type is dependent; an operator delete function is
+    // required to have a non-dependent result type.
+    return SemaRef.Diag(
+               FnDecl->getLocation(),
+               ResultType->isDependentType()
+                   ? diag::err_operator_new_delete_dependent_result_type
+                   : diag::err_operator_new_delete_invalid_result_type)
+           << FnDecl->getDeclName() << ExpectedResultType;
+  }
 
   // A function template must have at least 2 parameters.
   if (FnDecl->getDescribedFunctionTemplate() && FnDecl->getNumParams() < 2)
@@ -15215,13 +15254,7 @@ CheckOperatorNewDeleteTypes(Sema &SemaRef, const FunctionDecl *FnDecl,
                         diag::err_operator_new_delete_too_few_parameters)
       << FnDecl->getDeclName();
 
-  // Check the first parameter type is not dependent.
   QualType FirstParamType = FnDecl->getParamDecl(0)->getType();
-  if (FirstParamType->isDependentType())
-    return SemaRef.Diag(FnDecl->getLocation(), DependentParamTypeDiag)
-      << FnDecl->getDeclName() << ExpectedFirstParamType;
-
-  // Check that the first parameter type is what we expect.
   if (SemaRef.getLangOpts().OpenCLCPlusPlus) {
     // The operator is valid on any address space for OpenCL.
     if (auto *PtrTy =
@@ -15229,10 +15262,18 @@ CheckOperatorNewDeleteTypes(Sema &SemaRef, const FunctionDecl *FnDecl,
       FirstParamType = RemoveAddressSpaceFromPtr(SemaRef, PtrTy);
     }
   }
+
+  // Check that the first parameter type is what we expect.
   if (SemaRef.Context.getCanonicalType(FirstParamType).getUnqualifiedType() !=
-      ExpectedFirstParamType)
-    return SemaRef.Diag(FnDecl->getLocation(), InvalidParamTypeDiag)
-    << FnDecl->getDeclName() << ExpectedFirstParamType;
+      ExpectedFirstParamType) {
+    // The first parameter type is not allowed to be dependent. As a tentative
+    // DR resolution, we allow a dependent parameter type if it is the right
+    // type anyway, to allow destroying operator delete in class templates.
+    return SemaRef.Diag(FnDecl->getLocation(), FirstParamType->isDependentType()
+                                                   ? DependentParamTypeDiag
+                                                   : InvalidParamTypeDiag)
+           << FnDecl->getDeclName() << ExpectedFirstParamType;
+  }
 
   return false;
 }
diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 504a48b56639f..d7695f9d7d7a6 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -999,10 +999,8 @@ static CanThrowResult canSubStmtsThrow(Sema &Self, const Stmt *S) {
   return R;
 }
 
-/// Determine whether the callee of a particular function call can throw.
-/// E and D are both optional, but at least one of E and Loc must be specified.
-static CanThrowResult canCalleeThrow(Sema &S, const Expr *E, const Decl *D,
-                                     SourceLocation Loc = SourceLocation()) {
+CanThrowResult Sema::canCalleeThrow(Sema &S, const Expr *E, const Decl *D,
+                                    SourceLocation Loc) {
   // As an extension, we assume that __attribute__((nothrow)) functions don't
   // throw.
   if (D && isa<FunctionDecl>(D) && D->hasAttr<NoThrowAttr>())
@@ -1048,7 +1046,8 @@ static CanThrowResult canCalleeThrow(Sema &S, const Expr *E, const Decl *D,
   if (!FT)
     return CT_Can;
 
-  FT = S.ResolveExceptionSpec(Loc.isInvalid() ? E->getBeginLoc() : Loc, FT);
+  if (Loc.isValid() || (Loc.isInvalid() && E))
+    FT = S.ResolveExceptionSpec(Loc.isInvalid() ? E->getBeginLoc() : Loc, FT);
   if (!FT)
     return CT_Can;
 
@@ -1069,7 +1068,7 @@ static CanThrowResult canVarDeclThrow(Sema &Self, const VarDecl *VD) {
             VD->getType()->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) {
       if (auto *Dtor = RD->getDestructor()) {
         CT = mergeCanThrow(
-            CT, canCalleeThrow(Self, nullptr, Dtor, VD->getLocation()));
+            CT, Sema::canCalleeThrow(Self, nullptr, Dtor, VD->getLocation()));
       }
     }
   }
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index ec595ae7d723d..24b9c6777be17 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -964,6 +964,11 @@ ExprResult Sema::DefaultVariadicArgumentPromotion(Expr *E, VariadicCallType CT,
   ExprResult ExprRes = DefaultArgumentPromotion(E);
   if (ExprRes.isInvalid())
     return ExprError();
+
+  // Copy blocks to the heap.
+  if (ExprRes.get()->getType()->isBlockPointerType())
+    maybeExtendBlockObject(ExprRes);
+
   E = ExprRes.get();
 
   // Diagnostics regarding non-POD argument types are
@@ -3629,7 +3634,9 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok, Scope *UDLScope) {
   if (Invalid)
     return ExprError();
 
-  NumericLiteralParser Literal(TokSpelling, Tok.getLocation(), PP);
+  NumericLiteralParser Literal(TokSpelling, Tok.getLocation(),
+                               PP.getSourceManager(), PP.getLangOpts(),
+                               PP.getTargetInfo(), PP.getDiagnostics());
   if (Literal.hadError)
     return ExprError();
 
@@ -4555,7 +4562,8 @@ Sema::ActOnArraySubscriptExpr(Scope *S, Expr *base, SourceLocation lbLoc,
   if (base && !base->getType().isNull() &&
       base->getType()->isSpecificPlaceholderType(BuiltinType::OMPArraySection))
     return ActOnOMPArraySectionExpr(base, lbLoc, idx, SourceLocation(),
-                                    /*Length=*/nullptr, rbLoc);
+                                    SourceLocation(), /*Length*/ nullptr,
+                                    /*Stride=*/nullptr, rbLoc);
 
   // Since this might be a postfix expression, get rid of ParenListExprs.
   if (isa<ParenListExpr>(base)) {
@@ -4805,7 +4813,9 @@ void Sema::CheckSubscriptAccessOfNoDeref(const ArraySubscriptExpr *E) {
 
 ExprResult Sema::ActOnOMPArraySectionExpr(Expr *Base, SourceLocation LBLoc,
                                           Expr *LowerBound,
-                                          SourceLocation ColonLoc, Expr *Length,
+                                          SourceLocation ColonLocFirst,
+                                          SourceLocation ColonLocSecond,
+                                          Expr *Length, Expr *Stride,
                                           SourceLocation RBLoc) {
   if (Base->getType()->isPlaceholderType() &&
       !Base->getType()->isSpecificPlaceholderType(
@@ -4833,15 +4843,25 @@ ExprResult Sema::ActOnOMPArraySectionExpr(Expr *Base, SourceLocation LBLoc,
       return ExprError();
     Length = Result.get();
   }
+  if (Stride && Stride->getType()->isNonOverloadPlaceholderType()) {
+    ExprResult Result = CheckPlaceholderExpr(Stride);
+    if (Result.isInvalid())
+      return ExprError();
+    Result = DefaultLvalueConversion(Result.get());
+    if (Result.isInvalid())
+      return ExprError();
+    Stride = Result.get();
+  }
 
   // Build an unanalyzed expression if either operand is type-dependent.
   if (Base->isTypeDependent() ||
       (LowerBound &&
        (LowerBound->isTypeDependent() || LowerBound->isValueDependent())) ||
-      (Length && (Length->isTypeDependent() || Length->isValueDependent()))) {
-    return new (Context)
-        OMPArraySectionExpr(Base, LowerBound, Length, Context.DependentTy,
-                            VK_LValue, OK_Ordinary, ColonLoc, RBLoc);
+      (Length && (Length->isTypeDependent() || Length->isValueDependent())) ||
+      (Stride && (Stride->isTypeDependent() || Stride->isValueDependent()))) {
+    return new (Context) OMPArraySectionExpr(
+        Base, LowerBound, Length, Stride, Context.DependentTy, VK_LValue,
+        OK_Ordinary, ColonLocFirst, ColonLocSecond, RBLoc);
   }
 
   // Perform default conversions.
@@ -4885,6 +4905,20 @@ ExprResult Sema::ActOnOMPArraySectionExpr(Expr *Base, SourceLocation LBLoc,
       Diag(Length->getExprLoc(), diag::warn_omp_section_is_char)
           << 1 << Length->getSourceRange();
   }
+  if (Stride) {
+    ExprResult Res =
+        PerformOpenMPImplicitIntegerConversion(Stride->getExprLoc(), Stride);
+    if (Res.isInvalid())
+      return ExprError(Diag(Stride->getExprLoc(),
+                            diag::err_omp_typecheck_section_not_integer)
+                       << 1 << Stride->getSourceRange());
+    Stride = Res.get();
+
+    if (Stride->getType()->isSpecificBuiltinType(BuiltinType::Char_S) ||
+        Stride->getType()->isSpecificBuiltinType(BuiltinType::Char_U))
+      Diag(Stride->getExprLoc(), diag::warn_omp_section_is_char)
+          << 1 << Stride->getSourceRange();
+  }
 
   // C99 6.5.2.1p1: "shall have type "pointer to *object* type". Similarly,
   // C++ [expr.sub]p1: The type "T" shall be a completely-defined object
@@ -4903,7 +4937,7 @@ ExprResult Sema::ActOnOMPArraySectionExpr(Expr *Base, SourceLocation LBLoc,
   if (LowerBound && !OriginalTy->isAnyPointerType()) {
     Expr::EvalResult Result;
     if (LowerBound->EvaluateAsInt(Result, Context)) {
-      // OpenMP 4.5, [2.4 Array Sections]
+      // OpenMP 5.0, [2.1.5 Array Sections]
       // The array section must be a subset of the original array.
       llvm::APSInt LowerBoundValue = Result.Val.getInt();
       if (LowerBoundValue.isNegative()) {
@@ -4917,7 +4951,7 @@ ExprResult Sema::ActOnOMPArraySectionExpr(Expr *Base, SourceLocation LBLoc,
   if (Length) {
     Expr::EvalResult Result;
     if (Length->EvaluateAsInt(Result, Context)) {
-      // OpenMP 4.5, [2.4 Array Sections]
+      // OpenMP 5.0, [2.1.5 Array Sections]
       // The length must evaluate to non-negative integers.
       llvm::APSInt LengthValue = Result.Val.getInt();
       if (LengthValue.isNegative()) {
@@ -4927,17 +4961,32 @@ ExprResult Sema::ActOnOMPArraySectionExpr(Expr *Base, SourceLocation LBLoc,
         return ExprError();
       }
     }
-  } else if (ColonLoc.isValid() &&
+  } else if (ColonLocFirst.isValid() &&
              (OriginalTy.isNull() || (!OriginalTy->isConstantArrayType() &&
                                       !OriginalTy->isVariableArrayType()))) {
-    // OpenMP 4.5, [2.4 Array Sections]
+    // OpenMP 5.0, [2.1.5 Array Sections]
     // When the size of the array dimension is not known, the length must be
     // specified explicitly.
-    Diag(ColonLoc, diag::err_omp_section_length_undefined)
+    Diag(ColonLocFirst, diag::err_omp_section_length_undefined)
         << (!OriginalTy.isNull() && OriginalTy->isArrayType());
     return ExprError();
   }
 
+  if (Stride) {
+    Expr::EvalResult Result;
+    if (Stride->EvaluateAsInt(Result, Context)) {
+      // OpenMP 5.0, [2.1.5 Array Sections]
+      // The stride must evaluate to a positive integer.
+      llvm::APSInt StrideValue = Result.Val.getInt();
+      if (!StrideValue.isStrictlyPositive()) {
+        Diag(Stride->getExprLoc(), diag::err_omp_section_stride_non_positive)
+            << StrideValue.toString(/*Radix=*/10, /*Signed=*/true)
+            << Stride->getSourceRange();
+        return ExprError();
+      }
+    }
+  }
+
   if (!Base->getType()->isSpecificPlaceholderType(
           BuiltinType::OMPArraySection)) {
     ExprResult Result = DefaultFunctionArrayLvalueConversion(Base);
@@ -4945,9 +4994,9 @@ ExprResult Sema::ActOnOMPArraySectionExpr(Expr *Base, SourceLocation LBLoc,
       return ExprError();
     Base = Result.get();
   }
-  return new (Context)
-      OMPArraySectionExpr(Base, LowerBound, Length, Context.OMPArraySectionTy,
-                          VK_LValue, OK_Ordinary, ColonLoc, RBLoc);
+  return new (Context) OMPArraySectionExpr(
+      Base, LowerBound, Length, Stride, Context.OMPArraySectionTy, VK_LValue,
+      OK_Ordinary, ColonLocFirst, ColonLocSecond, RBLoc);
 }
 
 ExprResult Sema::ActOnOMPArrayShapingExpr(Expr *Base, SourceLocation LParenLoc,
@@ -5526,83 +5575,9 @@ bool Sema::CheckCXXDefaultArgExpr(SourceLocation CallLoc, FunctionDecl *FD,
     return true;
   }
 
-  if (Param->hasUninstantiatedDefaultArg()) {
-    Expr *UninstExpr = Param->getUninstantiatedDefaultArg();
-
-    EnterExpressionEvaluationContext EvalContext(
-        *this, ExpressionEvaluationContext::PotentiallyEvaluated, Param);
-
-    // Instantiate the expression.
-    //
-    // FIXME: Pass in a correct Pattern argument, otherwise
-    // getTemplateInstantiationArgs uses the lexical context of FD, e.g.
-    //
-    // template<typename T>
-    // struct A {
-    //   static int FooImpl();
-    //
-    //   template<typename Tp>
-    //   // bug: default argument A<T>::FooImpl() is evaluated with 2-level
-    //   // template argument list [[T], [Tp]], should be [[Tp]].
-    //   friend A<Tp> Foo(int a);
-    // };
-    //
-    // template<typename T>
-    // A<T> Foo(int a = A<T>::FooImpl());
-    MultiLevelTemplateArgumentList MutiLevelArgList
-      = getTemplateInstantiationArgs(FD, nullptr, /*RelativeToPrimary=*/true);
-
-    InstantiatingTemplate Inst(*this, CallLoc, Param,
-                               MutiLevelArgList.getInnermost());
-    if (Inst.isInvalid())
-      return true;
-    if (Inst.isAlreadyInstantiating()) {
-      Diag(Param->getBeginLoc(), diag::err_recursive_default_argument) << FD;
-      Param->setInvalidDecl();
-      return true;
-    }
-
-    ExprResult Result;
-    {
-      // C++ [dcl.fct.default]p5:
-      //   The names in the [default argument] expression are bound, and
-      //   the semantic constraints are checked, at the point where the
-      //   default argument expression appears.
-      ContextRAII SavedContext(*this, FD);
-      LocalInstantiationScope Local(*this);
-      runWithSufficientStackSpace(CallLoc, [&] {
-        Result = SubstInitializer(UninstExpr, MutiLevelArgList,
-                                  /*DirectInit*/false);
-      });
-    }
-    if (Result.isInvalid())
-      return true;
-
-    // Check the expression as an initializer for the parameter.
-    InitializedEntity Entity
-      = InitializedEntity::InitializeParameter(Context, Param);
-    InitializationKind Kind = InitializationKind::CreateCopy(
-        Param->getLocation(),
-        /*FIXME:EqualLoc*/ UninstExpr->getBeginLoc());
-    Expr *ResultE = Result.getAs<Expr>();
-
-    InitializationSequence InitSeq(*this, Entity, Kind, ResultE);
-    Result = InitSeq.Perform(*this, Entity, Kind, ResultE);
-    if (Result.isInvalid())
-      return true;
-
-    Result =
-        ActOnFinishFullExpr(Result.getAs<Expr>(), Param->getOuterLocStart(),
-                            /*DiscardedValue*/ false);
-    if (Result.isInvalid())
-      return true;
-
-    // Remember the instantiated default argument.
-    Param->setDefaultArg(Result.getAs<Expr>());
-    if (ASTMutationListener *L = getASTMutationListener()) {
-      L->DefaultArgumentInstantiated(Param);
-    }
-  }
+  if (Param->hasUninstantiatedDefaultArg() &&
+      InstantiateDefaultArgument(CallLoc, FD, Param))
+    return true;
 
   assert(Param->hasInit() && "default argument but no initializer?");
 
@@ -5941,9 +5916,6 @@ bool Sema::GatherArgumentsForCall(SourceLocation CallLoc, FunctionDecl *FDecl,
       for (Expr *A : Args.slice(ArgIx)) {
         ExprResult Arg = DefaultVariadicArgumentPromotion(A, CallType, FDecl);
         Invalid |= Arg.isInvalid();
-        // Copy blocks to the heap.
-        if (A->getType()->isBlockPointerType())
-          maybeExtendBlockObject(Arg);
         AllArgs.push_back(Arg.get());
       }
     }
@@ -13628,7 +13600,7 @@ static ExprResult convertHalfVecBinOp(Sema &S, ExprResult LHS, ExprResult RHS,
                                       BinaryOperatorKind Opc, QualType ResultTy,
                                       ExprValueKind VK, ExprObjectKind OK,
                                       bool IsCompAssign, SourceLocation OpLoc,
-                                      FPOptions FPFeatures) {
+                                      FPOptionsOverride FPFeatures) {
   auto &Context = S.getASTContext();
   assert((isVector(ResultTy, Context.HalfTy) ||
           isVector(ResultTy, Context.ShortTy)) &&
@@ -13951,9 +13923,9 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
   if (CompResultTy.isNull()) {
     if (ConvertHalfVec)
       return convertHalfVecBinOp(*this, LHS, RHS, Opc, ResultTy, VK, OK, false,
-                                 OpLoc, CurFPFeatures);
+                                 OpLoc, CurFPFeatureOverrides());
     return BinaryOperator::Create(Context, LHS.get(), RHS.get(), Opc, ResultTy,
-                                  VK, OK, OpLoc, CurFPFeatures);
+                                  VK, OK, OpLoc, CurFPFeatureOverrides());
   }
 
   // Handle compound assignments.
@@ -13971,11 +13943,11 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
 
   if (ConvertHalfVec)
     return convertHalfVecBinOp(*this, LHS, RHS, Opc, ResultTy, VK, OK, true,
-                               OpLoc, CurFPFeatures);
+                               OpLoc, CurFPFeatureOverrides());
 
-  return CompoundAssignOperator::Create(Context, LHS.get(), RHS.get(), Opc,
-                                        ResultTy, VK, OK, OpLoc, CurFPFeatures,
-                                        CompLHSTy, CompResultTy);
+  return CompoundAssignOperator::Create(
+      Context, LHS.get(), RHS.get(), Opc, ResultTy, VK, OK, OpLoc,
+      CurFPFeatureOverrides(), CompLHSTy, CompResultTy);
 }
 
 /// DiagnoseBitwisePrecedence - Emit a warning when bitwise and comparison
@@ -14563,8 +14535,9 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
   if (Opc != UO_AddrOf && Opc != UO_Deref)
     CheckArrayAccess(Input.get());
 
-  auto *UO = UnaryOperator::Create(Context, Input.get(), Opc, resultType, VK,
-                                   OK, OpLoc, CanOverflow, CurFPFeatures);
+  auto *UO =
+      UnaryOperator::Create(Context, Input.get(), Opc, resultType, VK, OK,
+                            OpLoc, CanOverflow, CurFPFeatureOverrides());
 
   if (Opc == UO_Deref && UO->getType()->hasAttr(attr::NoDeref) &&
       !isa<ArrayType>(UO->getType().getDesugaredType(Context)))
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 0a0bb3952cd89..d885920b6c14e 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -4203,8 +4203,8 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
     break;
 
   case ICK_Compatible_Conversion:
-      From = ImpCastExprToType(From, ToType, CK_NoOp,
-                               VK_RValue, /*BasePath=*/nullptr, CCK).get();
+    From = ImpCastExprToType(From, ToType, CK_NoOp, From->getValueKind(),
+                             /*BasePath=*/nullptr, CCK).get();
     break;
 
   case ICK_Writeback_Conversion:
@@ -4441,11 +4441,7 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
     break;
 
   case ICK_Qualification: {
-    // The qualification keeps the category of the inner expression, unless the
-    // target type isn't a reference.
-    ExprValueKind VK =
-        ToType->isReferenceType() ? From->getValueKind() : VK_RValue;
-
+    ExprValueKind VK = From->getValueKind();
     CastKind CK = CK_NoOp;
 
     if (ToType->isReferenceType() &&
diff --git a/clang/lib/Sema/SemaExprObjC.cpp b/clang/lib/Sema/SemaExprObjC.cpp
index 474aba6a0d446..228a1ec3ba1f9 100644
--- a/clang/lib/Sema/SemaExprObjC.cpp
+++ b/clang/lib/Sema/SemaExprObjC.cpp
@@ -1228,33 +1228,66 @@ static void DiagnoseMismatchedSelectors(Sema &S, SourceLocation AtLoc,
   }
 }
 
-static void HelperToDiagnoseDirectSelectorsExpr(Sema &S, SourceLocation AtLoc,
-                                                Selector Sel,
-                                                ObjCMethodList &MethList,
-                                                bool &onlyDirect) {
+static ObjCMethodDecl *LookupDirectMethodInMethodList(Sema &S, Selector Sel,
+                                                      ObjCMethodList &MethList,
+                                                      bool &onlyDirect,
+                                                      bool &anyDirect) {
+  (void)Sel;
   ObjCMethodList *M = &MethList;
-  for (M = M->getNext(); M; M = M->getNext()) {
+  ObjCMethodDecl *DirectMethod = nullptr;
+  for (; M; M = M->getNext()) {
     ObjCMethodDecl *Method = M->getMethod();
-    if (Method->getSelector() != Sel)
+    if (!Method)
       continue;
-    if (!Method->isDirectMethod())
+    assert(Method->getSelector() == Sel && "Method with wrong selector in method list");
+    if (Method->isDirectMethod()) {
+      anyDirect = true;
+      DirectMethod = Method;
+    } else
       onlyDirect = false;
   }
+
+  return DirectMethod;
 }
 
-static void DiagnoseDirectSelectorsExpr(Sema &S, SourceLocation AtLoc,
-                                        Selector Sel, bool &onlyDirect) {
-  for (Sema::GlobalMethodPool::iterator b = S.MethodPool.begin(),
-       e = S.MethodPool.end(); b != e; b++) {
-    // first, instance methods
-    ObjCMethodList &InstMethList = b->second.first;
-    HelperToDiagnoseDirectSelectorsExpr(S, AtLoc, Sel, InstMethList,
-                                        onlyDirect);
+// Search the global pool for (potentially) direct methods matching the given
+// selector. If a non-direct method is found, set \param onlyDirect to false. If
+// a direct method is found, set \param anyDirect to true. Returns a direct
+// method, if any.
+static ObjCMethodDecl *LookupDirectMethodInGlobalPool(Sema &S, Selector Sel,
+                                                      bool &onlyDirect,
+                                                      bool &anyDirect) {
+  auto Iter = S.MethodPool.find(Sel);
+  if (Iter == S.MethodPool.end())
+    return nullptr;
 
-    // second, class methods
-    ObjCMethodList &ClsMethList = b->second.second;
-    HelperToDiagnoseDirectSelectorsExpr(S, AtLoc, Sel, ClsMethList, onlyDirect);
-  }
+  ObjCMethodDecl *DirectInstance = LookupDirectMethodInMethodList(
+      S, Sel, Iter->second.first, onlyDirect, anyDirect);
+  ObjCMethodDecl *DirectClass = LookupDirectMethodInMethodList(
+      S, Sel, Iter->second.second, onlyDirect, anyDirect);
+
+  return DirectInstance ? DirectInstance : DirectClass;
+}
+
+static ObjCMethodDecl *findMethodInCurrentClass(Sema &S, Selector Sel) {
+  auto *CurMD = S.getCurMethodDecl();
+  if (!CurMD)
+    return nullptr;
+  ObjCInterfaceDecl *IFace = CurMD->getClassInterface();
+
+  // The language enforce that only one direct method is present in a given
+  // class, so we just need to find one method in the current class to know
+  // whether Sel is potentially direct in this context.
+  if (ObjCMethodDecl *MD = IFace->lookupMethod(Sel, /*isInstance=*/true))
+    return MD;
+  if (ObjCMethodDecl *MD = IFace->lookupPrivateMethod(Sel, /*isInstance=*/true))
+    return MD;
+  if (ObjCMethodDecl *MD = IFace->lookupMethod(Sel, /*isInstance=*/false))
+    return MD;
+  if (ObjCMethodDecl *MD = IFace->lookupPrivateMethod(Sel, /*isInstance=*/false))
+    return MD;
+
+  return nullptr;
 }
 
 ExprResult Sema::ParseObjCSelectorExpression(Selector Sel,
@@ -1280,15 +1313,38 @@ ExprResult Sema::ParseObjCSelectorExpression(Selector Sel,
     } else
         Diag(SelLoc, diag::warn_undeclared_selector) << Sel;
   } else {
-    bool onlyDirect = Method->isDirectMethod();
-    DiagnoseDirectSelectorsExpr(*this, AtLoc, Sel, onlyDirect);
     DiagnoseMismatchedSelectors(*this, AtLoc, Method, LParenLoc, RParenLoc,
                                 WarnMultipleSelectors);
+
+    bool onlyDirect = true;
+    bool anyDirect = false;
+    ObjCMethodDecl *GlobalDirectMethod =
+        LookupDirectMethodInGlobalPool(*this, Sel, onlyDirect, anyDirect);
+
     if (onlyDirect) {
       Diag(AtLoc, diag::err_direct_selector_expression)
           << Method->getSelector();
       Diag(Method->getLocation(), diag::note_direct_method_declared_at)
           << Method->getDeclName();
+    } else if (anyDirect) {
+      // If we saw any direct methods, see if we see a direct member of the
+      // current class. If so, the @selector will likely be used to refer to
+      // this direct method.
+      ObjCMethodDecl *LikelyTargetMethod = findMethodInCurrentClass(*this, Sel);
+      if (LikelyTargetMethod && LikelyTargetMethod->isDirectMethod()) {
+        Diag(AtLoc, diag::warn_potentially_direct_selector_expression) << Sel;
+        Diag(LikelyTargetMethod->getLocation(),
+             diag::note_direct_method_declared_at)
+            << LikelyTargetMethod->getDeclName();
+      } else if (!LikelyTargetMethod) {
+        // Otherwise, emit the "strict" variant of this diagnostic, unless
+        // LikelyTargetMethod is non-direct.
+        Diag(AtLoc, diag::warn_strict_potentially_direct_selector_expression)
+            << Sel;
+        Diag(GlobalDirectMethod->getLocation(),
+             diag::note_direct_method_declared_at)
+            << GlobalDirectMethod->getDeclName();
+      }
     }
   }
 
@@ -4487,7 +4543,8 @@ Expr *Sema::stripARCUnbridgedCast(Expr *e) {
     Expr *sub = stripARCUnbridgedCast(uo->getSubExpr());
     return UnaryOperator::Create(Context, sub, UO_Extension, sub->getType(),
                                  sub->getValueKind(), sub->getObjectKind(),
-                                 uo->getOperatorLoc(), false, CurFPFeatures);
+                                 uo->getOperatorLoc(), false,
+                                 CurFPFeatureOverrides());
   } else if (GenericSelectionExpr *gse = dyn_cast<GenericSelectionExpr>(e)) {
     assert(!gse->isResultDependent());
 
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index d46e7f86d6b33..eb07de65d2668 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -4693,6 +4693,9 @@ static bool isNonReferenceableGLValue(Expr *E) {
 }
 
 /// Reference initialization without resolving overloaded functions.
+///
+/// We also can get here in C if we call a builtin which is declared as
+/// a function with a parameter of reference type (such as __builtin_va_end()).
 static void TryReferenceInitializationCore(Sema &S,
                                            const InitializedEntity &Entity,
                                            const InitializationKind &Kind,
@@ -4769,15 +4772,20 @@ static void TryReferenceInitializationCore(Sema &S,
     // an rvalue. DR1287 removed the "implicitly" here.
     if (RefRelationship == Sema::Ref_Incompatible && T2->isRecordType() &&
         (isLValueRef || InitCategory.isRValue())) {
-      ConvOvlResult = TryRefInitWithConversionFunction(
-          S, Entity, Kind, Initializer, /*AllowRValues*/ isRValueRef,
-          /*IsLValueRef*/ isLValueRef, Sequence);
-      if (ConvOvlResult == OR_Success)
-        return;
-      if (ConvOvlResult != OR_No_Viable_Function)
-        Sequence.SetOverloadFailure(
-            InitializationSequence::FK_ReferenceInitOverloadFailed,
-            ConvOvlResult);
+      if (S.getLangOpts().CPlusPlus) {
+        // Try conversion functions only for C++.
+        ConvOvlResult = TryRefInitWithConversionFunction(
+            S, Entity, Kind, Initializer, /*AllowRValues*/ isRValueRef,
+            /*IsLValueRef*/ isLValueRef, Sequence);
+        if (ConvOvlResult == OR_Success)
+          return;
+        if (ConvOvlResult != OR_No_Viable_Function)
+          Sequence.SetOverloadFailure(
+              InitializationSequence::FK_ReferenceInitOverloadFailed,
+              ConvOvlResult);
+      } else {
+        ConvOvlResult = OR_No_Viable_Function;
+      }
     }
   }
 
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index e751a73957d0d..657ed13f207ad 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -990,8 +990,7 @@ void Sema::ActOnStartOfLambdaDefinition(LambdaIntroducer &Intro,
   // Attributes on the lambda apply to the method.
   ProcessDeclAttributes(CurScope, Method, ParamInfo);
 
-  // CUDA lambdas get implicit attributes based on the scope in which they're
-  // declared.
+  // CUDA lambdas get implicit host and device attributes.
   if (getLangOpts().CUDA)
     CUDASetLambdaAttrs(Method);
 
@@ -1780,6 +1779,9 @@ ExprResult Sema::BuildLambdaExpr(SourceLocation StartLoc, SourceLocation EndLoc,
       BuildCaptureField(Class, From);
       Captures.push_back(Capture);
       CaptureInits.push_back(Init.get());
+
+      if (LangOpts.CUDA)
+        CUDACheckLambdaCapture(CallOperator, From);
     }
 
     Class->setCaptures(Captures);
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index 06a4dfa0a636f..5757eaf3fac08 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -1153,73 +1153,14 @@ static bool isNamespaceOrTranslationUnitScope(Scope *S) {
   return false;
 }
 
-// Find the next outer declaration context from this scope. This
-// routine actually returns the semantic outer context, which may
-// differ from the lexical context (encoded directly in the Scope
-// stack) when we are parsing a member of a class template. In this
-// case, the second element of the pair will be true, to indicate that
-// name lookup should continue searching in this semantic context when
-// it leaves the current template parameter scope.
-static std::pair<DeclContext *, bool> findOuterContext(Scope *S) {
-  DeclContext *DC = S->getEntity();
-  DeclContext *Lexical = nullptr;
-  for (Scope *OuterS = S->getParent(); OuterS;
-       OuterS = OuterS->getParent()) {
-    if (OuterS->getEntity()) {
-      Lexical = OuterS->getEntity();
-      break;
-    }
-  }
-
-  // C++ [temp.local]p8:
-  //   In the definition of a member of a class template that appears
-  //   outside of the namespace containing the class template
-  //   definition, the name of a template-parameter hides the name of
-  //   a member of this namespace.
-  //
-  // Example:
-  //
-  //   namespace N {
-  //     class C { };
-  //
-  //     template<class T> class B {
-  //       void f(T);
-  //     };
-  //   }
-  //
-  //   template<class C> void N::B<C>::f(C) {
-  //     C b;  // C is the template parameter, not N::C
-  //   }
-  //
-  // In this example, the lexical context we return is the
-  // TranslationUnit, while the semantic context is the namespace N.
-  if (!Lexical || !DC || !S->getParent() ||
-      !S->getParent()->isTemplateParamScope())
-    return std::make_pair(Lexical, false);
-
-  // Find the outermost template parameter scope.
-  // For the example, this is the scope for the template parameters of
-  // template<class C>.
-  Scope *OutermostTemplateScope = S->getParent();
-  while (OutermostTemplateScope->getParent() &&
-         OutermostTemplateScope->getParent()->isTemplateParamScope())
-    OutermostTemplateScope = OutermostTemplateScope->getParent();
-
-  // Find the namespace context in which the original scope occurs. In
-  // the example, this is namespace N.
-  DeclContext *Semantic = DC;
-  while (!Semantic->isFileContext())
-    Semantic = Semantic->getParent();
-
-  // Find the declaration context just outside of the template
-  // parameter scope. This is the context in which the template is
-  // being lexically declaration (a namespace context). In the
-  // example, this is the global scope.
-  if (Lexical->isFileContext() && !Lexical->Equals(Semantic) &&
-      Lexical->Encloses(Semantic))
-    return std::make_pair(Semantic, true);
-
-  return std::make_pair(Lexical, false);
+/// Find the outer declaration context from this scope. This indicates the
+/// context that we should search up to (exclusive) before considering the
+/// parent of the specified scope.
+static DeclContext *findOuterContext(Scope *S) {
+  for (Scope *OuterS = S->getParent(); OuterS; OuterS = OuterS->getParent())
+    if (DeclContext *DC = OuterS->getLookupEntity())
+      return DC;
+  return nullptr;
 }
 
 namespace {
@@ -1286,13 +1227,11 @@ bool Sema::CppLookupName(LookupResult &R, Scope *S) {
   UnqualUsingDirectiveSet UDirs(*this);
   bool VisitedUsingDirectives = false;
   bool LeftStartingScope = false;
-  DeclContext *OutsideOfTemplateParamDC = nullptr;
 
   // When performing a scope lookup, we want to find local extern decls.
   FindLocalExternScope FindLocals(R);
 
   for (; S && !isNamespaceOrTranslationUnitScope(S); S = S->getParent()) {
-    DeclContext *Ctx = S->getEntity();
     bool SearchNamespaceScope = true;
     // Check whether the IdResolver has anything in this scope.
     for (; I != IEnd && S->isDeclScope(*I); ++I) {
@@ -1324,7 +1263,8 @@ bool Sema::CppLookupName(LookupResult &R, Scope *S) {
     if (!SearchNamespaceScope) {
       R.resolveKind();
       if (S->isClassScope())
-        if (CXXRecordDecl *Record = dyn_cast_or_null<CXXRecordDecl>(Ctx))
+        if (CXXRecordDecl *Record =
+                dyn_cast_or_null<CXXRecordDecl>(S->getEntity()))
           R.setNamingClass(Record);
       return true;
     }
@@ -1338,24 +1278,8 @@ bool Sema::CppLookupName(LookupResult &R, Scope *S) {
       return false;
     }
 
-    if (!Ctx && S->isTemplateParamScope() && OutsideOfTemplateParamDC &&
-        S->getParent() && !S->getParent()->isTemplateParamScope()) {
-      // We've just searched the last template parameter scope and
-      // found nothing, so look into the contexts between the
-      // lexical and semantic declaration contexts returned by
-      // findOuterContext(). This implements the name lookup behavior
-      // of C++ [temp.local]p8.
-      Ctx = OutsideOfTemplateParamDC;
-      OutsideOfTemplateParamDC = nullptr;
-    }
-
-    if (Ctx) {
-      DeclContext *OuterCtx;
-      bool SearchAfterTemplateScope;
-      std::tie(OuterCtx, SearchAfterTemplateScope) = findOuterContext(S);
-      if (SearchAfterTemplateScope)
-        OutsideOfTemplateParamDC = OuterCtx;
-
+    if (DeclContext *Ctx = S->getLookupEntity()) {
+      DeclContext *OuterCtx = findOuterContext(S);
       for (; Ctx && !Ctx->Equals(OuterCtx); Ctx = Ctx->getLookupParent()) {
         // We do not directly look into transparent contexts, since
         // those entities will be found in the nearest enclosing
@@ -1480,25 +1404,9 @@ bool Sema::CppLookupName(LookupResult &R, Scope *S) {
       return true;
     }
 
-    DeclContext *Ctx = S->getEntity();
-    if (!Ctx && S->isTemplateParamScope() && OutsideOfTemplateParamDC &&
-        S->getParent() && !S->getParent()->isTemplateParamScope()) {
-      // We've just searched the last template parameter scope and
-      // found nothing, so look into the contexts between the
-      // lexical and semantic declaration contexts returned by
-      // findOuterContext(). This implements the name lookup behavior
-      // of C++ [temp.local]p8.
-      Ctx = OutsideOfTemplateParamDC;
-      OutsideOfTemplateParamDC = nullptr;
-    }
-
+    DeclContext *Ctx = S->getLookupEntity();
     if (Ctx) {
-      DeclContext *OuterCtx;
-      bool SearchAfterTemplateScope;
-      std::tie(OuterCtx, SearchAfterTemplateScope) = findOuterContext(S);
-      if (SearchAfterTemplateScope)
-        OutsideOfTemplateParamDC = OuterCtx;
-
+      DeclContext *OuterCtx = findOuterContext(S);
       for (; Ctx && !Ctx->Equals(OuterCtx); Ctx = Ctx->getLookupParent()) {
         // We do not directly look into transparent contexts, since
         // those entities will be found in the nearest enclosing
@@ -3995,14 +3903,12 @@ class LookupVisibleHelper {
       }
     }
 
-    // FIXME: C++ [temp.local]p8
-    DeclContext *Entity = nullptr;
-    if (S->getEntity()) {
+    DeclContext *Entity = S->getLookupEntity();
+    if (Entity) {
       // Look into this scope's declaration context, along with any of its
       // parent lookup contexts (e.g., enclosing classes), up to the point
       // where we hit the context stored in the next outer scope.
-      Entity = S->getEntity();
-      DeclContext *OuterCtx = findOuterContext(S).first; // FIXME
+      DeclContext *OuterCtx = findOuterContext(S);
 
       for (DeclContext *Ctx = Entity; Ctx && !Ctx->Equals(OuterCtx);
            Ctx = Ctx->getLookupParent()) {
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index e005072da0aff..920463da40277 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -53,9 +53,10 @@ static const Expr *checkMapClauseExpressionBase(
 namespace {
 /// Default data sharing attributes, which can be applied to directive.
 enum DefaultDataSharingAttributes {
-  DSA_unspecified = 0, /// Data sharing attribute not specified.
-  DSA_none = 1 << 0,   /// Default data sharing attribute 'none'.
-  DSA_shared = 1 << 1, /// Default data sharing attribute 'shared'.
+  DSA_unspecified = 0,       /// Data sharing attribute not specified.
+  DSA_none = 1 << 0,         /// Default data sharing attribute 'none'.
+  DSA_shared = 1 << 1,       /// Default data sharing attribute 'shared'.
+  DSA_firstprivate = 1 << 2, /// Default data sharing attribute 'firstprivate'.
 };
 
 /// Stack for tracking declarations used in OpenMP directives and
@@ -684,6 +685,11 @@ class DSAStackTy {
     getTopOfStack().DefaultAttr = DSA_shared;
     getTopOfStack().DefaultAttrLoc = Loc;
   }
+  /// Set default data sharing attribute to firstprivate.
+  void setDefaultDSAFirstPrivate(SourceLocation Loc) {
+    getTopOfStack().DefaultAttr = DSA_firstprivate;
+    getTopOfStack().DefaultAttrLoc = Loc;
+  }
   /// Set default data mapping attribute to Modifier:Kind
   void setDefaultDMAAttr(OpenMPDefaultmapClauseModifier M,
                          OpenMPDefaultmapClauseKind Kind,
@@ -1183,6 +1189,15 @@ DSAStackTy::DSAVarData DSAStackTy::getDSA(const_iterator &Iter,
     return DVar;
   case DSA_none:
     return DVar;
+  case DSA_firstprivate:
+    if (VD->getStorageDuration() == SD_Static &&
+        VD->getDeclContext()->isFileContext()) {
+      DVar.CKind = OMPC_unknown;
+    } else {
+      DVar.CKind = OMPC_firstprivate;
+    }
+    DVar.ImplicitDSALoc = Iter->DefaultAttrLoc;
+    return DVar;
   case DSA_unspecified:
     // OpenMP [2.9.1.1, Data-sharing Attribute Rules for Variables Referenced
     // in a Construct, implicitly determined, p.2]
@@ -2058,7 +2073,13 @@ bool Sema::isOpenMPCapturedByRef(const ValueDecl *D, unsigned Level,
         // If the variable is artificial and must be captured by value - try to
         // capture by value.
         !(isa<OMPCapturedExprDecl>(D) && !D->hasAttr<OMPCaptureNoInitAttr>() &&
-          !cast<OMPCapturedExprDecl>(D)->getInit()->isGLValue());
+          !cast<OMPCapturedExprDecl>(D)->getInit()->isGLValue()) &&
+        // If the variable is implicitly firstprivate and scalar - capture by
+        // copy
+        !(DSAStack->getDefaultDSA() == DSA_firstprivate &&
+          !DSAStack->hasExplicitDSA(
+              D, [](OpenMPClauseKind K) { return K != OMPC_unknown; }, Level) &&
+          !DSAStack->isLoopControlVariable(D, Level).first);
   }
 
   // When passing data by copy, we need to make sure it fits the uintptr size
@@ -2185,10 +2206,13 @@ VarDecl *Sema::isOpenMPCapturedDecl(ValueDecl *D, bool CheckScopeInfo,
         DSAStack->isClauseParsingMode());
     // Global shared must not be captured.
     if (VD && !VD->hasLocalStorage() && DVarPrivate.CKind == OMPC_unknown &&
-        (DSAStack->getDefaultDSA() != DSA_none || DVarTop.CKind == OMPC_shared))
+        ((DSAStack->getDefaultDSA() != DSA_none &&
+          DSAStack->getDefaultDSA() != DSA_firstprivate) ||
+         DVarTop.CKind == OMPC_shared))
       return nullptr;
     if (DVarPrivate.CKind != OMPC_unknown ||
-        (VD && DSAStack->getDefaultDSA() == DSA_none))
+        (VD && (DSAStack->getDefaultDSA() == DSA_none ||
+                DSAStack->getDefaultDSA() == DSA_firstprivate)))
       return VD ? VD : cast<VarDecl>(DVarPrivate.PrivateCopy->getDecl());
   }
   return nullptr;
@@ -3333,10 +3357,19 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
       // in the construct, and does not have a predetermined data-sharing
       // attribute, must have its data-sharing attribute explicitly determined
       // by being listed in a data-sharing attribute clause.
-      if (DVar.CKind == OMPC_unknown && Stack->getDefaultDSA() == DSA_none &&
+      if (DVar.CKind == OMPC_unknown &&
+          (Stack->getDefaultDSA() == DSA_none ||
+           Stack->getDefaultDSA() == DSA_firstprivate) &&
           isImplicitOrExplicitTaskingRegion(DKind) &&
           VarsWithInheritedDSA.count(VD) == 0) {
-        VarsWithInheritedDSA[VD] = E;
+        bool InheritedDSA = Stack->getDefaultDSA() == DSA_none;
+        if (!InheritedDSA && Stack->getDefaultDSA() == DSA_firstprivate) {
+          DSAStackTy::DSAVarData DVar =
+              Stack->getImplicitDSA(VD, /*FromParent=*/false);
+          InheritedDSA = DVar.CKind == OMPC_unknown;
+        }
+        if (InheritedDSA)
+          VarsWithInheritedDSA[VD] = E;
         return;
       }
 
@@ -3438,7 +3471,9 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
 
       // Define implicit data-sharing attributes for task.
       DVar = Stack->getImplicitDSA(VD, /*FromParent=*/false);
-      if (isOpenMPTaskingDirective(DKind) && DVar.CKind != OMPC_shared &&
+      if (((isOpenMPTaskingDirective(DKind) && DVar.CKind != OMPC_shared) ||
+           (Stack->getDefaultDSA() == DSA_firstprivate &&
+            DVar.CKind == OMPC_firstprivate && !DVar.RefExpr)) &&
           !Stack->isLoopControlVariable(VD).first) {
         ImplicitFirstprivate.push_back(E);
         return;
@@ -4050,6 +4085,7 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
   case OMPD_end_declare_variant:
     llvm_unreachable("OpenMP Directive is not allowed");
   case OMPD_unknown:
+  default:
     llvm_unreachable("Unknown OpenMP directive");
   }
 }
@@ -4644,7 +4680,7 @@ static bool checkIfClauses(Sema &S, OpenMPDirectiveKind Kind,
   bool ErrorFound = false;
   unsigned NamedModifiersNumber = 0;
   llvm::IndexedMap<const OMPIfClause *, Kind2Unsigned> FoundNameModifiers;
-  FoundNameModifiers.resize(unsigned(OMPD_unknown) + 1);
+  FoundNameModifiers.resize(llvm::omp::Directive_enumSize + 1);
   SmallVector<SourceLocation, 4> NameModifierLoc;
   for (const OMPClause *C : Clauses) {
     if (const auto *IC = dyn_cast_or_null<OMPIfClause>(C)) {
@@ -5335,13 +5371,16 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
   case OMPD_end_declare_variant:
     llvm_unreachable("OpenMP Directive is not allowed");
   case OMPD_unknown:
+  default:
     llvm_unreachable("Unknown OpenMP directive");
   }
 
   ErrorFound = Res.isInvalid() || ErrorFound;
 
-  // Check variables in the clauses if default(none) was specified.
-  if (DSAStack->getDefaultDSA() == DSA_none) {
+  // Check variables in the clauses if default(none) or
+  // default(firstprivate) was specified.
+  if (DSAStack->getDefaultDSA() == DSA_none ||
+      DSAStack->getDefaultDSA() == DSA_firstprivate) {
     DSAAttrChecker DSAChecker(DSAStack, *this, nullptr);
     for (OMPClause *C : Clauses) {
       switch (C->getClauseKind()) {
@@ -5437,6 +5476,7 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
       case OMPC_atomic_default_mem_order:
       case OMPC_device_type:
       case OMPC_match:
+      default:
         llvm_unreachable("Unexpected clause");
       }
       for (Stmt *CC : C->children()) {
@@ -5451,7 +5491,8 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
     if (P.getFirst()->isImplicit() || isa<OMPCapturedExprDecl>(P.getFirst()))
       continue;
     ErrorFound = true;
-    if (DSAStack->getDefaultDSA() == DSA_none) {
+    if (DSAStack->getDefaultDSA() == DSA_none ||
+        DSAStack->getDefaultDSA() == DSA_firstprivate) {
       Diag(P.second->getExprLoc(), diag::err_omp_no_dsa_for_variable)
           << P.first << P.second->getSourceRange();
       Diag(DSAStack->getDefaultDSALocation(), diag::note_omp_default_dsa_none);
@@ -11683,6 +11724,7 @@ OMPClause *Sema::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind, Expr *Expr,
   case OMPC_exclusive:
   case OMPC_uses_allocators:
   case OMPC_affinity:
+  default:
     llvm_unreachable("Clause is not allowed.");
   }
   return Res;
@@ -11837,6 +11879,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_requires:
       llvm_unreachable("Unexpected OpenMP directive with if-clause");
     case OMPD_unknown:
+    default:
       llvm_unreachable("Unknown OpenMP directive");
     }
     break;
@@ -11915,6 +11958,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_requires:
       llvm_unreachable("Unexpected OpenMP directive with num_threads-clause");
     case OMPD_unknown:
+    default:
       llvm_unreachable("Unknown OpenMP directive");
     }
     break;
@@ -11991,6 +12035,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_requires:
       llvm_unreachable("Unexpected OpenMP directive with num_teams-clause");
     case OMPD_unknown:
+    default:
       llvm_unreachable("Unknown OpenMP directive");
     }
     break;
@@ -12067,6 +12112,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_requires:
       llvm_unreachable("Unexpected OpenMP directive with thread_limit-clause");
     case OMPD_unknown:
+    default:
       llvm_unreachable("Unknown OpenMP directive");
     }
     break;
@@ -12143,6 +12189,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_requires:
       llvm_unreachable("Unexpected OpenMP directive with schedule clause");
     case OMPD_unknown:
+    default:
       llvm_unreachable("Unknown OpenMP directive");
     }
     break;
@@ -12219,6 +12266,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_requires:
       llvm_unreachable("Unexpected OpenMP directive with schedule clause");
     case OMPD_unknown:
+    default:
       llvm_unreachable("Unknown OpenMP directive");
     }
     break;
@@ -12295,6 +12343,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_requires:
       llvm_unreachable("Unexpected OpenMP directive with num_teams-clause");
     case OMPD_unknown:
+    default:
       llvm_unreachable("Unknown OpenMP directive");
     }
     break;
@@ -12373,6 +12422,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_requires:
       llvm_unreachable("Unexpected OpenMP directive with grainsize-clause");
     case OMPD_unknown:
+    default:
       llvm_unreachable("Unknown OpenMP directive");
     }
     break;
@@ -12439,6 +12489,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
   case OMPC_exclusive:
   case OMPC_uses_allocators:
   case OMPC_affinity:
+  default:
     llvm_unreachable("Unexpected OpenMP clause.");
   }
   return CaptureRegion;
@@ -12880,6 +12931,7 @@ OMPClause *Sema::ActOnOpenMPSimpleClause(
   case OMPC_exclusive:
   case OMPC_uses_allocators:
   case OMPC_affinity:
+  default:
     llvm_unreachable("Clause is not allowed.");
   }
   return Res;
@@ -12918,10 +12970,20 @@ OMPClause *Sema::ActOnOpenMPDefaultClause(DefaultKind Kind,
         << getOpenMPClauseName(OMPC_default);
     return nullptr;
   }
-  if (Kind == OMP_DEFAULT_none)
+
+  switch (Kind) {
+  case OMP_DEFAULT_none:
     DSAStack->setDefaultDSANone(KindKwLoc);
-  else if (Kind == OMP_DEFAULT_shared)
+    break;
+  case OMP_DEFAULT_shared:
     DSAStack->setDefaultDSAShared(KindKwLoc);
+    break;
+  case OMP_DEFAULT_firstprivate:
+    DSAStack->setDefaultDSAFirstPrivate(KindKwLoc);
+    break;
+  default:
+    llvm_unreachable("DSA unexpected in OpenMP default clause");
+  }
 
   return new (Context)
       OMPDefaultClause(Kind, KindKwLoc, StartLoc, LParenLoc, EndLoc);
@@ -13108,6 +13170,7 @@ OMPClause *Sema::ActOnOpenMPSingleExprWithArgClause(
   case OMPC_exclusive:
   case OMPC_uses_allocators:
   case OMPC_affinity:
+  default:
     llvm_unreachable("Clause is not allowed.");
   }
   return Res;
@@ -13345,6 +13408,7 @@ OMPClause *Sema::ActOnOpenMPClause(OpenMPClauseKind Kind,
   case OMPC_exclusive:
   case OMPC_uses_allocators:
   case OMPC_affinity:
+  default:
     llvm_unreachable("Clause is not allowed.");
   }
   return Res;
@@ -13618,6 +13682,7 @@ OMPClause *Sema::ActOnOpenMPVarListClause(
   case OMPC_destroy:
   case OMPC_detach:
   case OMPC_uses_allocators:
+  default:
     llvm_unreachable("Clause is not allowed.");
   }
   return Res;
@@ -14675,7 +14740,7 @@ static bool checkOMPArraySectionConstantForReduction(
   if (Length == nullptr) {
     // For array sections of the form [1:] or [:], we would need to analyze
     // the lower bound...
-    if (OASE->getColonLoc().isValid())
+    if (OASE->getColonLocFirst().isValid())
       return false;
 
     // This is an array subscript which has implicit length 1!
@@ -14701,7 +14766,7 @@ static bool checkOMPArraySectionConstantForReduction(
     if (Length == nullptr) {
       // For array sections of the form [1:] or [:], we would need to analyze
       // the lower bound...
-      if (OASE->getColonLoc().isValid())
+      if (OASE->getColonLocFirst().isValid())
         return false;
 
       // This is an array subscript which has implicit length 1!
@@ -16458,7 +16523,8 @@ static bool checkArrayExpressionDoesNotReferToWholeSize(Sema &SemaRef,
   // If this is an array subscript, it refers to the whole size if the size of
   // the dimension is constant and equals 1. Also, an array section assumes the
   // format of an array subscript if no colon is used.
-  if (isa<ArraySubscriptExpr>(E) || (OASE && OASE->getColonLoc().isInvalid())) {
+  if (isa<ArraySubscriptExpr>(E) ||
+      (OASE && OASE->getColonLocFirst().isInvalid())) {
     if (const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.getTypePtr()))
       return ATy->getSize().getSExtValue() != 1;
     // Size can't be evaluated statically.
@@ -16514,7 +16580,8 @@ static bool checkArrayExpressionDoesNotReferToUnitySize(Sema &SemaRef,
 
   // An array subscript always refer to a single element. Also, an array section
   // assumes the format of an array subscript if no colon is used.
-  if (isa<ArraySubscriptExpr>(E) || (OASE && OASE->getColonLoc().isInvalid()))
+  if (isa<ArraySubscriptExpr>(E) ||
+      (OASE && OASE->getColonLocFirst().isInvalid()))
     return false;
 
   assert(OASE && "Expecting array section if not an array subscript.");
@@ -16558,7 +16625,7 @@ static bool checkArrayExpressionDoesNotReferToUnitySize(Sema &SemaRef,
 //
 // We want to retrieve the member expression 'this->S';
 
-// OpenMP 4.5 [2.15.5.1, map Clause, Restrictions, p.2]
+// OpenMP 5.0 [2.19.7.1, map Clause, Restrictions, p.2]
 //  If a list item is an array section, it must specify contiguous storage.
 //
 // For this restriction it is sufficient that we make sure only references
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index e57264dec9a09..8635397f48067 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -4709,7 +4709,7 @@ TryReferenceInit(Sema &S, Expr *Init, QualType DeclType,
                               Sema::ReferenceConversions::NestedQualification)
                              ? ICK_Qualification
                              : ICK_Identity;
-    ICS.Standard.FromTypePtr = T2.getAsOpaquePtr();
+    ICS.Standard.setFromType(T2);
     ICS.Standard.setToType(0, T2);
     ICS.Standard.setToType(1, T1);
     ICS.Standard.setToType(2, T1);
@@ -12818,7 +12818,7 @@ static QualType chooseRecoveryType(OverloadCandidateSet &CS,
   auto ConsiderCandidate = [&](const OverloadCandidate &Candidate) {
     if (!Candidate.Function)
       return;
-    QualType T = Candidate.Function->getCallResultType();
+    QualType T = Candidate.Function->getReturnType();
     if (T.isNull())
       return;
     if (!Result)
@@ -13041,7 +13041,7 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, UnaryOperatorKind Opc,
     if (Fns.empty())
       return UnaryOperator::Create(Context, Input, Opc, Context.DependentTy,
                                    VK_RValue, OK_Ordinary, OpLoc, false,
-                                   CurFPFeatures);
+                                   CurFPFeatureOverrides());
 
     CXXRecordDecl *NamingClass = nullptr; // lookup ignores member operators
     UnresolvedLookupExpr *Fn = UnresolvedLookupExpr::Create(
@@ -13049,7 +13049,7 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, UnaryOperatorKind Opc,
         /*ADL*/ true, IsOverloaded(Fns), Fns.begin(), Fns.end());
     return CXXOperatorCallExpr::Create(Context, Op, Fn, ArgsArray,
                                        Context.DependentTy, VK_RValue, OpLoc,
-                                       CurFPFeatures);
+                                       CurFPFeatureOverrides());
   }
 
   // Build an empty overload set.
@@ -13123,7 +13123,7 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, UnaryOperatorKind Opc,
       Args[0] = Input;
       CallExpr *TheCall = CXXOperatorCallExpr::Create(
           Context, Op, FnExpr.get(), ArgsArray, ResultTy, VK, OpLoc,
-          CurFPFeatures, Best->IsADLCandidate);
+          CurFPFeatureOverrides(), Best->IsADLCandidate);
 
       if (CheckCallReturnType(FnDecl->getReturnType(), OpLoc, TheCall, FnDecl))
         return ExprError();
@@ -13292,12 +13292,12 @@ ExprResult Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
       // If there are no functions to store, just build a dependent
       // BinaryOperator or CompoundAssignment.
       if (Opc <= BO_Assign || Opc > BO_OrAssign)
-        return BinaryOperator::Create(Context, Args[0], Args[1], Opc,
-                                      Context.DependentTy, VK_RValue,
-                                      OK_Ordinary, OpLoc, CurFPFeatures);
+        return BinaryOperator::Create(
+            Context, Args[0], Args[1], Opc, Context.DependentTy, VK_RValue,
+            OK_Ordinary, OpLoc, CurFPFeatureOverrides());
       return CompoundAssignOperator::Create(
           Context, Args[0], Args[1], Opc, Context.DependentTy, VK_LValue,
-          OK_Ordinary, OpLoc, CurFPFeatures, Context.DependentTy,
+          OK_Ordinary, OpLoc, CurFPFeatureOverrides(), Context.DependentTy,
           Context.DependentTy);
     }
 
@@ -13311,7 +13311,7 @@ ExprResult Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
         /*ADL*/ PerformADL, IsOverloaded(Fns), Fns.begin(), Fns.end());
     return CXXOperatorCallExpr::Create(Context, Op, Fn, Args,
                                        Context.DependentTy, VK_RValue, OpLoc,
-                                       CurFPFeatures);
+                                       CurFPFeatureOverrides());
   }
 
   // Always do placeholder-like conversions on the RHS.
@@ -13480,7 +13480,7 @@ ExprResult Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
 
         CXXOperatorCallExpr *TheCall = CXXOperatorCallExpr::Create(
             Context, ChosenOp, FnExpr.get(), Args, ResultTy, VK, OpLoc,
-            CurFPFeatures, Best->IsADLCandidate);
+            CurFPFeatureOverrides(), Best->IsADLCandidate);
 
         if (CheckCallReturnType(FnDecl->getReturnType(), OpLoc, TheCall,
                                 FnDecl))
@@ -13506,6 +13506,10 @@ ExprResult Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
         if (R.isInvalid())
           return ExprError();
 
+        R = CheckForImmediateInvocation(R, FnDecl);
+        if (R.isInvalid())
+          return ExprError();
+
         // For a rewritten candidate, we've already reversed the arguments
         // if needed. Perform the rest of the rewrite now.
         if ((Best->RewriteKind & CRK_DifferentOperator) ||
@@ -13541,7 +13545,7 @@ ExprResult Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
         if (Best->RewriteKind != CRK_None)
           R = new (Context) CXXRewrittenBinaryOperator(R.get(), IsReversed);
 
-        return CheckForImmediateInvocation(R, FnDecl);
+        return R;
       } else {
         // We matched a built-in operator. Convert the arguments, then
         // break out so that we will build the appropriate built-in
@@ -13748,7 +13752,7 @@ ExprResult Sema::BuildSynthesizedThreeWayComparison(
   Expr *SyntacticForm = BinaryOperator::Create(
       Context, OrigLHS, OrigRHS, BO_Cmp, Result.get()->getType(),
       Result.get()->getValueKind(), Result.get()->getObjectKind(), OpLoc,
-      CurFPFeatures);
+      CurFPFeatureOverrides());
   Expr *SemanticForm[] = {LHS, RHS, Result.get()};
   return PseudoObjectExpr::Create(Context, SyntacticForm, SemanticForm, 2);
 }
@@ -13779,7 +13783,7 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
 
     return CXXOperatorCallExpr::Create(Context, OO_Subscript, Fn, Args,
                                        Context.DependentTy, VK_RValue, RLoc,
-                                       CurFPFeatures);
+                                       CurFPFeatureOverrides());
   }
 
   // Handle placeholders on both operands.
@@ -13852,9 +13856,9 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
         ExprValueKind VK = Expr::getValueKindForType(ResultTy);
         ResultTy = ResultTy.getNonLValueExprType(Context);
 
-        CXXOperatorCallExpr *TheCall =
-            CXXOperatorCallExpr::Create(Context, OO_Subscript, FnExpr.get(),
-                                        Args, ResultTy, VK, RLoc, CurFPFeatures);
+        CXXOperatorCallExpr *TheCall = CXXOperatorCallExpr::Create(
+            Context, OO_Subscript, FnExpr.get(), Args, ResultTy, VK, RLoc,
+            CurFPFeatureOverrides());
         if (CheckCallReturnType(FnDecl->getReturnType(), LLoc, TheCall, FnDecl))
           return ExprError();
 
@@ -14476,9 +14480,9 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Obj,
   ExprValueKind VK = Expr::getValueKindForType(ResultTy);
   ResultTy = ResultTy.getNonLValueExprType(Context);
 
-  CXXOperatorCallExpr *TheCall =
-      CXXOperatorCallExpr::Create(Context, OO_Call, NewFn.get(), MethodArgs,
-                                  ResultTy, VK, RParenLoc, CurFPFeatures);
+  CXXOperatorCallExpr *TheCall = CXXOperatorCallExpr::Create(
+      Context, OO_Call, NewFn.get(), MethodArgs, ResultTy, VK, RParenLoc,
+      CurFPFeatureOverrides());
 
   if (CheckCallReturnType(Method->getReturnType(), LParenLoc, TheCall, Method))
     return true;
@@ -14594,8 +14598,9 @@ Sema::BuildOverloadedArrowExpr(Scope *S, Expr *Base, SourceLocation OpLoc,
   QualType ResultTy = Method->getReturnType();
   ExprValueKind VK = Expr::getValueKindForType(ResultTy);
   ResultTy = ResultTy.getNonLValueExprType(Context);
-  CXXOperatorCallExpr *TheCall = CXXOperatorCallExpr::Create(
-      Context, OO_Arrow, FnExpr.get(), Base, ResultTy, VK, OpLoc, CurFPFeatures);
+  CXXOperatorCallExpr *TheCall =
+      CXXOperatorCallExpr::Create(Context, OO_Arrow, FnExpr.get(), Base,
+                                  ResultTy, VK, OpLoc, CurFPFeatureOverrides());
 
   if (CheckCallReturnType(Method->getReturnType(), OpLoc, TheCall, Method))
     return ExprError();
@@ -14844,7 +14849,7 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
 
         return UnaryOperator::Create(
             Context, SubExpr, UO_AddrOf, MemPtrType, VK_RValue, OK_Ordinary,
-            UnOp->getOperatorLoc(), false, CurFPFeatures);
+            UnOp->getOperatorLoc(), false, CurFPFeatureOverrides());
       }
     }
     Expr *SubExpr = FixOverloadedFunctionReference(UnOp->getSubExpr(),
@@ -14852,9 +14857,10 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
     if (SubExpr == UnOp->getSubExpr())
       return UnOp;
 
-    return UnaryOperator::Create(
-        Context, SubExpr, UO_AddrOf, Context.getPointerType(SubExpr->getType()),
-        VK_RValue, OK_Ordinary, UnOp->getOperatorLoc(), false, CurFPFeatures);
+    return UnaryOperator::Create(Context, SubExpr, UO_AddrOf,
+                                 Context.getPointerType(SubExpr->getType()),
+                                 VK_RValue, OK_Ordinary, UnOp->getOperatorLoc(),
+                                 false, CurFPFeatureOverrides());
   }
 
   if (UnresolvedLookupExpr *ULE = dyn_cast<UnresolvedLookupExpr>(E)) {
diff --git a/clang/lib/Sema/SemaPseudoObject.cpp b/clang/lib/Sema/SemaPseudoObject.cpp
index da777201af882..d17599a6ed149 100644
--- a/clang/lib/Sema/SemaPseudoObject.cpp
+++ b/clang/lib/Sema/SemaPseudoObject.cpp
@@ -130,7 +130,7 @@ namespace {
         return UnaryOperator::Create(
             S.Context, e, uop->getOpcode(), uop->getType(), uop->getValueKind(),
             uop->getObjectKind(), uop->getOperatorLoc(), uop->canOverflow(),
-            S.CurFPFeatures);
+            S.CurFPFeatureOverrides());
       }
 
       if (GenericSelectionExpr *gse = dyn_cast<GenericSelectionExpr>(e)) {
@@ -446,9 +446,10 @@ PseudoOpBuilder::buildAssignmentOperation(Scope *Sc, SourceLocation opcLoc,
   ExprResult result;
   if (opcode == BO_Assign) {
     result = semanticRHS;
-    syntactic = BinaryOperator::Create(
-        S.Context, syntacticLHS, capturedRHS, opcode, capturedRHS->getType(),
-        capturedRHS->getValueKind(), OK_Ordinary, opcLoc, S.CurFPFeatures);
+    syntactic = BinaryOperator::Create(S.Context, syntacticLHS, capturedRHS,
+                                       opcode, capturedRHS->getType(),
+                                       capturedRHS->getValueKind(), OK_Ordinary,
+                                       opcLoc, S.CurFPFeatureOverrides());
 
   } else {
     ExprResult opLHS = buildGet();
@@ -462,8 +463,9 @@ PseudoOpBuilder::buildAssignmentOperation(Scope *Sc, SourceLocation opcLoc,
 
     syntactic = CompoundAssignOperator::Create(
         S.Context, syntacticLHS, capturedRHS, opcode, result.get()->getType(),
-        result.get()->getValueKind(), OK_Ordinary, opcLoc, S.CurFPFeatures,
-        opLHS.get()->getType(), result.get()->getType());
+        result.get()->getValueKind(), OK_Ordinary, opcLoc,
+        S.CurFPFeatureOverrides(), opLHS.get()->getType(),
+        result.get()->getType());
   }
 
   // The result of the assignment, if not void, is the value set into
@@ -531,7 +533,7 @@ PseudoOpBuilder::buildIncDecOperation(Scope *Sc, SourceLocation opcLoc,
                                 ? S.Context.getTypeSize(resultType) >=
                                       S.Context.getTypeSize(S.Context.IntTy)
                                 : false,
-                            S.CurFPFeatures);
+                            S.CurFPFeatureOverrides());
   return complete(syntactic);
 }
 
@@ -1553,7 +1555,7 @@ ExprResult Sema::checkPseudoObjectIncDec(Scope *Sc, SourceLocation opcLoc,
   if (op->isTypeDependent())
     return UnaryOperator::Create(Context, op, opcode, Context.DependentTy,
                                  VK_RValue, OK_Ordinary, opcLoc, false,
-                                 CurFPFeatures);
+                                 CurFPFeatureOverrides());
 
   assert(UnaryOperator::isIncrementDecrementOp(opcode));
   Expr *opaqueRef = op->IgnoreParens();
@@ -1584,7 +1586,7 @@ ExprResult Sema::checkPseudoObjectAssignment(Scope *S, SourceLocation opcLoc,
   if (LHS->isTypeDependent() || RHS->isTypeDependent())
     return BinaryOperator::Create(Context, LHS, RHS, opcode,
                                   Context.DependentTy, VK_RValue, OK_Ordinary,
-                                  opcLoc, CurFPFeatures);
+                                  opcLoc, CurFPFeatureOverrides());
 
   // Filter out non-overload placeholder types in the RHS.
   if (RHS->getType()->isNonOverloadPlaceholderType()) {
@@ -1640,7 +1642,7 @@ Expr *Sema::recreateSyntacticForm(PseudoObjectExpr *E) {
     return UnaryOperator::Create(Context, op, uop->getOpcode(), uop->getType(),
                                  uop->getValueKind(), uop->getObjectKind(),
                                  uop->getOperatorLoc(), uop->canOverflow(),
-                                 CurFPFeatures);
+                                 CurFPFeatureOverrides());
   } else if (CompoundAssignOperator *cop
                = dyn_cast<CompoundAssignOperator>(syntax)) {
     Expr *lhs = stripOpaqueValuesFromPseudoObjectRef(*this, cop->getLHS());
@@ -1648,7 +1650,7 @@ Expr *Sema::recreateSyntacticForm(PseudoObjectExpr *E) {
     return CompoundAssignOperator::Create(
         Context, lhs, rhs, cop->getOpcode(), cop->getType(),
         cop->getValueKind(), cop->getObjectKind(), cop->getOperatorLoc(),
-        CurFPFeatures, cop->getComputationLHSType(),
+        CurFPFeatureOverrides(), cop->getComputationLHSType(),
         cop->getComputationResultType());
 
   } else if (BinaryOperator *bop = dyn_cast<BinaryOperator>(syntax)) {
@@ -1657,7 +1659,7 @@ Expr *Sema::recreateSyntacticForm(PseudoObjectExpr *E) {
     return BinaryOperator::Create(Context, lhs, rhs, bop->getOpcode(),
                                   bop->getType(), bop->getValueKind(),
                                   bop->getObjectKind(), bop->getOperatorLoc(),
-                                  CurFPFeatures);
+                                  CurFPFeatureOverrides());
 
   } else if (isa<CallExpr>(syntax)) {
     return syntax;
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index a22a1116eb0bb..73f3183c163f3 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -1328,8 +1328,9 @@ Sema::DiagnoseAssignmentEnum(QualType DstType, QualType SrcType,
     }
 }
 
-StmtResult Sema::ActOnWhileStmt(SourceLocation WhileLoc, ConditionResult Cond,
-                                Stmt *Body) {
+StmtResult Sema::ActOnWhileStmt(SourceLocation WhileLoc,
+                                SourceLocation LParenLoc, ConditionResult Cond,
+                                SourceLocation RParenLoc, Stmt *Body) {
   if (Cond.isInvalid())
     return StmtError();
 
@@ -1344,7 +1345,7 @@ StmtResult Sema::ActOnWhileStmt(SourceLocation WhileLoc, ConditionResult Cond,
     getCurCompoundScope().setHasEmptyLoopBodies();
 
   return WhileStmt::Create(Context, CondVal.first, CondVal.second, Body,
-                           WhileLoc);
+                           WhileLoc, LParenLoc, RParenLoc);
 }
 
 StmtResult
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 073b4e818a249..c05ed0b14e3eb 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -51,8 +51,7 @@ unsigned Sema::getTemplateDepth(Scope *S) const {
 
   // Each template parameter scope represents one level of template parameter
   // depth.
-  for (Scope *TempParamScope = S->getTemplateParamParent();
-       TempParamScope && !Depth;
+  for (Scope *TempParamScope = S->getTemplateParamParent(); TempParamScope;
        TempParamScope = TempParamScope->getParent()->getTemplateParamParent()) {
     ++Depth;
   }
@@ -2038,6 +2037,7 @@ struct ConvertConstructorToDeductionGuideTransform {
       // a list of substituted template arguments as we go.
       for (NamedDecl *Param : *InnerParams) {
         MultiLevelTemplateArgumentList Args;
+        Args.setKind(TemplateSubstitutionKind::Rewrite);
         Args.addOuterTemplateArguments(SubstArgs);
         Args.addOuterRetainedLevel();
         NamedDecl *NewParam = transformTemplateParameter(Param, Args);
@@ -2057,6 +2057,7 @@ struct ConvertConstructorToDeductionGuideTransform {
     // substitute references to the old parameters into references to the
     // new ones.
     MultiLevelTemplateArgumentList Args;
+    Args.setKind(TemplateSubstitutionKind::Rewrite);
     if (FTD) {
       Args.addOuterTemplateArguments(SubstArgs);
       Args.addOuterRetainedLevel();
@@ -3558,9 +3559,8 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
     // Only substitute for the innermost template argument list.
     MultiLevelTemplateArgumentList TemplateArgLists;
     TemplateArgLists.addOuterTemplateArguments(&StackTemplateArgs);
-    unsigned Depth = AliasTemplate->getTemplateParameters()->getDepth();
-    for (unsigned I = 0; I < Depth; ++I)
-      TemplateArgLists.addOuterTemplateArguments(None);
+    TemplateArgLists.addOuterRetainedLevels(
+        AliasTemplate->getTemplateParameters()->getDepth());
 
     LocalInstantiationScope Scope(*this);
     InstantiatingTemplate Inst(*this, TemplateLoc, Template);
@@ -4844,10 +4844,7 @@ bool Sema::CheckTemplateTypeArgument(TemplateTypeParmDecl *Param,
     CXXScopeSpec SS;
     DeclarationNameInfo NameInfo;
 
-    if (DeclRefExpr *ArgExpr = dyn_cast<DeclRefExpr>(Arg.getAsExpr())) {
-      SS.Adopt(ArgExpr->getQualifierLoc());
-      NameInfo = ArgExpr->getNameInfo();
-    } else if (DependentScopeDeclRefExpr *ArgExpr =
+   if (DependentScopeDeclRefExpr *ArgExpr =
                dyn_cast<DependentScopeDeclRefExpr>(Arg.getAsExpr())) {
       SS.Adopt(ArgExpr->getQualifierLoc());
       NameInfo = ArgExpr->getNameInfo();
@@ -4866,6 +4863,7 @@ bool Sema::CheckTemplateTypeArgument(TemplateTypeParmDecl *Param,
       if (Result.getAsSingle<TypeDecl>() ||
           Result.getResultKind() ==
               LookupResult::NotFoundInCurrentInstantiation) {
+        assert(SS.getScopeRep() && "dependent scope expr must has a scope!");
         // Suggest that the user add 'typename' before the NNS.
         SourceLocation Loc = AL.getSourceRange().getBegin();
         Diag(Loc, getLangOpts().MSVCCompat
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 877020ed4dcf9..f3641afbbf8a0 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -738,8 +738,9 @@ class PackDeductionScope {
       // type, so we need to collect the pending deduced values for those packs.
       if (auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(
               TemplateParams->getParam(Index))) {
-        if (auto *Expansion = dyn_cast<PackExpansionType>(NTTP->getType()))
-          ExtraDeductions.push_back(Expansion->getPattern());
+        if (!NTTP->isExpandedParameterPack())
+          if (auto *Expansion = dyn_cast<PackExpansionType>(NTTP->getType()))
+            ExtraDeductions.push_back(Expansion->getPattern());
       }
       // FIXME: Also collect the unexpanded packs in any type and template
       // parameter packs that are pack expansions.
@@ -3040,8 +3041,13 @@ Sema::DeduceTemplateArguments(ClassTemplatePartialSpecializationDecl *Partial,
   if (Trap.hasErrorOccurred())
     return Sema::TDK_SubstitutionFailure;
 
-  return ::FinishTemplateArgumentDeduction(
-      *this, Partial, /*IsPartialOrdering=*/false, TemplateArgs, Deduced, Info);
+  TemplateDeductionResult Result;
+  runWithSufficientStackSpace(Info.getLocation(), [&] {
+    Result = ::FinishTemplateArgumentDeduction(*this, Partial,
+                                               /*IsPartialOrdering=*/false,
+                                               TemplateArgs, Deduced, Info);
+  });
+  return Result;
 }
 
 /// Perform template argument deduction to determine whether
@@ -3081,8 +3087,13 @@ Sema::DeduceTemplateArguments(VarTemplatePartialSpecializationDecl *Partial,
   if (Trap.hasErrorOccurred())
     return Sema::TDK_SubstitutionFailure;
 
-  return ::FinishTemplateArgumentDeduction(
-      *this, Partial, /*IsPartialOrdering=*/false, TemplateArgs, Deduced, Info);
+  TemplateDeductionResult Result;
+  runWithSufficientStackSpace(Info.getLocation(), [&] {
+    Result = ::FinishTemplateArgumentDeduction(*this, Partial,
+                                               /*IsPartialOrdering=*/false,
+                                               TemplateArgs, Deduced, Info);
+  });
+  return Result;
 }
 
 /// Determine whether the given type T is a simple-template-id type.
@@ -4031,13 +4042,12 @@ Sema::TemplateDeductionResult Sema::DeduceTemplateArguments(
   SmallVector<QualType, 8> ParamTypes;
   unsigned NumExplicitlySpecified = 0;
   if (ExplicitTemplateArgs) {
-    TemplateDeductionResult Result =
-      SubstituteExplicitTemplateArguments(FunctionTemplate,
-                                          *ExplicitTemplateArgs,
-                                          Deduced,
-                                          ParamTypes,
-                                          nullptr,
-                                          Info);
+    TemplateDeductionResult Result;
+    runWithSufficientStackSpace(Info.getLocation(), [&] {
+      Result = SubstituteExplicitTemplateArguments(
+          FunctionTemplate, *ExplicitTemplateArgs, Deduced, ParamTypes, nullptr,
+          Info);
+    });
     if (Result)
       return Result;
 
@@ -4139,12 +4149,16 @@ Sema::TemplateDeductionResult Sema::DeduceTemplateArguments(
   // that is needed when the accessibility of template arguments is checked.
   DeclContext *CallingCtx = CurContext;
 
-  return FinishTemplateArgumentDeduction(
-      FunctionTemplate, Deduced, NumExplicitlySpecified, Specialization, Info,
-      &OriginalCallArgs, PartialOverloading, [&, CallingCtx]() {
-        ContextRAII SavedContext(*this, CallingCtx);
-        return CheckNonDependent(ParamTypesForArgChecking);
-      });
+  TemplateDeductionResult Result;
+  runWithSufficientStackSpace(Info.getLocation(), [&] {
+    Result = FinishTemplateArgumentDeduction(
+        FunctionTemplate, Deduced, NumExplicitlySpecified, Specialization, Info,
+        &OriginalCallArgs, PartialOverloading, [&, CallingCtx]() {
+          ContextRAII SavedContext(*this, CallingCtx);
+          return CheckNonDependent(ParamTypesForArgChecking);
+        });
+  });
+  return Result;
 }
 
 QualType Sema::adjustCCAndNoReturn(QualType ArgFunctionType,
@@ -4230,11 +4244,13 @@ Sema::TemplateDeductionResult Sema::DeduceTemplateArguments(
   unsigned NumExplicitlySpecified = 0;
   SmallVector<QualType, 4> ParamTypes;
   if (ExplicitTemplateArgs) {
-    if (TemplateDeductionResult Result
-          = SubstituteExplicitTemplateArguments(FunctionTemplate,
-                                                *ExplicitTemplateArgs,
-                                                Deduced, ParamTypes,
-                                                &FunctionType, Info))
+    TemplateDeductionResult Result;
+    runWithSufficientStackSpace(Info.getLocation(), [&] {
+      Result = SubstituteExplicitTemplateArguments(
+          FunctionTemplate, *ExplicitTemplateArgs, Deduced, ParamTypes,
+          &FunctionType, Info);
+    });
+    if (Result)
       return Result;
 
     NumExplicitlySpecified = Deduced.size();
@@ -4276,10 +4292,13 @@ Sema::TemplateDeductionResult Sema::DeduceTemplateArguments(
       return Result;
   }
 
-  if (TemplateDeductionResult Result
-        = FinishTemplateArgumentDeduction(FunctionTemplate, Deduced,
-                                          NumExplicitlySpecified,
-                                          Specialization, Info))
+  TemplateDeductionResult Result;
+  runWithSufficientStackSpace(Info.getLocation(), [&] {
+    Result = FinishTemplateArgumentDeduction(FunctionTemplate, Deduced,
+                                             NumExplicitlySpecified,
+                                             Specialization, Info);
+  });
+  if (Result)
     return Result;
 
   // If the function has a deduced return type, deduce it now, so we can check
@@ -4436,9 +4455,11 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *ConversionTemplate,
   LocalInstantiationScope InstScope(*this);
   // Finish template argument deduction.
   FunctionDecl *ConversionSpecialized = nullptr;
-  TemplateDeductionResult Result
-      = FinishTemplateArgumentDeduction(ConversionTemplate, Deduced, 0,
-                                        ConversionSpecialized, Info);
+  TemplateDeductionResult Result;
+  runWithSufficientStackSpace(Info.getLocation(), [&] {
+    Result = FinishTemplateArgumentDeduction(ConversionTemplate, Deduced, 0,
+                                             ConversionSpecialized, Info);
+  });
   Specialization = cast_or_null<CXXConversionDecl>(ConversionSpecialized);
   return Result;
 }
@@ -5378,14 +5399,15 @@ static bool isAtLeastAsSpecializedAs(Sema &S, QualType T1, QualType T2,
   Sema::InstantiatingTemplate Inst(S, Info.getLocation(), P2, DeducedArgs,
                                    Info);
   auto *TST1 = T1->castAs<TemplateSpecializationType>();
-  if (FinishTemplateArgumentDeduction(
-          S, P2, /*IsPartialOrdering=*/true,
-          TemplateArgumentList(TemplateArgumentList::OnStack,
-                               TST1->template_arguments()),
-          Deduced, Info))
-    return false;
-
-  return true;
+  bool AtLeastAsSpecialized;
+  S.runWithSufficientStackSpace(Info.getLocation(), [&] {
+    AtLeastAsSpecialized = !FinishTemplateArgumentDeduction(
+        S, P2, /*IsPartialOrdering=*/true,
+        TemplateArgumentList(TemplateArgumentList::OnStack,
+                             TST1->template_arguments()),
+        Deduced, Info);
+  });
+  return AtLeastAsSpecialized;
 }
 
 /// Returns the more specialized class template partial specialization
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index bfda59d40c2a2..11e03c517d015 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -1362,6 +1362,19 @@ TemplateName TemplateInstantiator::TransformTemplateName(
 
       TemplateArgument Arg = TemplateArgs(TTP->getDepth(), TTP->getPosition());
 
+      if (TemplateArgs.isRewrite()) {
+        // We're rewriting the template parameter as a reference to another
+        // template parameter.
+        if (Arg.getKind() == TemplateArgument::Pack) {
+          assert(Arg.pack_size() == 1 && Arg.pack_begin()->isPackExpansion() &&
+                 "unexpected pack arguments in template rewrite");
+          Arg = Arg.pack_begin()->getPackExpansionPattern();
+        }
+        assert(Arg.getKind() == TemplateArgument::Template &&
+               "unexpected nontype template argument kind in template rewrite");
+        return Arg.getAsTemplate();
+      }
+
       if (TTP->isParameterPack()) {
         assert(Arg.getKind() == TemplateArgument::Pack &&
                "Missing argument pack");
@@ -1458,19 +1471,18 @@ TemplateInstantiator::TransformTemplateParmRefExpr(DeclRefExpr *E,
 
   TemplateArgument Arg = TemplateArgs(NTTP->getDepth(), NTTP->getPosition());
 
-  if (TemplateArgs.getNumLevels() != TemplateArgs.getNumSubstitutedLevels()) {
-    // We're performing a partial substitution, so the substituted argument
-    // could be dependent. As a result we can't create a SubstNonType*Expr
-    // node now, since that represents a fully-substituted argument.
-    // FIXME: We should have some AST representation for this.
+  if (TemplateArgs.isRewrite()) {
+    // We're rewriting the template parameter as a reference to another
+    // template parameter.
     if (Arg.getKind() == TemplateArgument::Pack) {
-      // FIXME: This won't work for alias templates.
       assert(Arg.pack_size() == 1 && Arg.pack_begin()->isPackExpansion() &&
-             "unexpected pack arguments in partial substitution");
+             "unexpected pack arguments in template rewrite");
       Arg = Arg.pack_begin()->getPackExpansionPattern();
     }
     assert(Arg.getKind() == TemplateArgument::Expression &&
-           "unexpected nontype template argument kind in partial substitution");
+           "unexpected nontype template argument kind in template rewrite");
+    // FIXME: This can lead to the same subexpression appearing multiple times
+    // in a complete expression.
     return Arg.getAsExpr();
   }
 
@@ -1782,6 +1794,24 @@ TemplateInstantiator::TransformTemplateTypeParmType(TypeLocBuilder &TLB,
 
     TemplateArgument Arg = TemplateArgs(T->getDepth(), T->getIndex());
 
+    if (TemplateArgs.isRewrite()) {
+      // We're rewriting the template parameter as a reference to another
+      // template parameter.
+      if (Arg.getKind() == TemplateArgument::Pack) {
+        assert(Arg.pack_size() == 1 && Arg.pack_begin()->isPackExpansion() &&
+               "unexpected pack arguments in template rewrite");
+        Arg = Arg.pack_begin()->getPackExpansionPattern();
+      }
+      assert(Arg.getKind() == TemplateArgument::Type &&
+             "unexpected nontype template argument kind in template rewrite");
+      QualType NewT = Arg.getAsType();
+      assert(isa<TemplateTypeParmType>(NewT) &&
+             "type parm not rewritten to type parm");
+      auto NewTL = TLB.push<TemplateTypeParmTypeLoc>(NewT);
+      NewTL.setNameLoc(TL.getNameLoc());
+      return NewT;
+    }
+
     if (T->isParameterPack()) {
       assert(Arg.getKind() == TemplateArgument::Pack &&
              "Missing argument pack");
@@ -2396,7 +2426,7 @@ ParmVarDecl *Sema::SubstParmVarDecl(ParmVarDecl *OldParm,
     UnparsedDefaultArgInstantiations[OldParm].push_back(NewParm);
   } else if (Expr *Arg = OldParm->getDefaultArg()) {
     FunctionDecl *OwningFunc = cast<FunctionDecl>(OldParm->getDeclContext());
-    if (OwningFunc->isInLocalScope()) {
+    if (OwningFunc->isInLocalScopeForInstantiation()) {
       // Instantiate default arguments for methods of local classes (DR1484)
       // and non-defining declarations.
       Sema::ContextRAII SavedContext(*this, OwningFunc);
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 1fff6b8b951e7..85adc4ef2dbde 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -3719,6 +3719,8 @@ FunctionDecl *Sema::SubstSpaceshipAsEqualEqual(CXXRecordDecl *RD,
   //   access and function-definition and in the same class scope as the
   //   three-way comparison operator function
   MultiLevelTemplateArgumentList NoTemplateArgs;
+  NoTemplateArgs.setKind(TemplateSubstitutionKind::Rewrite);
+  NoTemplateArgs.addOuterRetainedLevels(RD->getTemplateDepth());
   TemplateDeclInstantiator Instantiator(*this, RD, NoTemplateArgs);
   Decl *R;
   if (auto *MD = dyn_cast<CXXMethodDecl>(Spaceship)) {
@@ -4225,6 +4227,94 @@ static bool addInstantiatedParametersToScope(Sema &S, FunctionDecl *Function,
   return false;
 }
 
+bool Sema::InstantiateDefaultArgument(SourceLocation CallLoc, FunctionDecl *FD,
+                                      ParmVarDecl *Param) {
+  assert(Param->hasUninstantiatedDefaultArg());
+  Expr *UninstExpr = Param->getUninstantiatedDefaultArg();
+
+  EnterExpressionEvaluationContext EvalContext(
+      *this, ExpressionEvaluationContext::PotentiallyEvaluated, Param);
+
+  // Instantiate the expression.
+  //
+  // FIXME: Pass in a correct Pattern argument, otherwise
+  // getTemplateInstantiationArgs uses the lexical context of FD, e.g.
+  //
+  // template<typename T>
+  // struct A {
+  //   static int FooImpl();
+  //
+  //   template<typename Tp>
+  //   // bug: default argument A<T>::FooImpl() is evaluated with 2-level
+  //   // template argument list [[T], [Tp]], should be [[Tp]].
+  //   friend A<Tp> Foo(int a);
+  // };
+  //
+  // template<typename T>
+  // A<T> Foo(int a = A<T>::FooImpl());
+  MultiLevelTemplateArgumentList TemplateArgs
+    = getTemplateInstantiationArgs(FD, nullptr, /*RelativeToPrimary=*/true);
+
+  InstantiatingTemplate Inst(*this, CallLoc, Param,
+                             TemplateArgs.getInnermost());
+  if (Inst.isInvalid())
+    return true;
+  if (Inst.isAlreadyInstantiating()) {
+    Diag(Param->getBeginLoc(), diag::err_recursive_default_argument) << FD;
+    Param->setInvalidDecl();
+    return true;
+  }
+
+  ExprResult Result;
+  {
+    // C++ [dcl.fct.default]p5:
+    //   The names in the [default argument] expression are bound, and
+    //   the semantic constraints are checked, at the point where the
+    //   default argument expression appears.
+    ContextRAII SavedContext(*this, FD);
+    LocalInstantiationScope Local(*this);
+
+    FunctionDecl *Pattern = FD->getTemplateInstantiationPattern(
+        /*ForDefinition*/ false);
+    if (addInstantiatedParametersToScope(*this, FD, Pattern, Local,
+                                         TemplateArgs))
+      return true;
+
+    runWithSufficientStackSpace(CallLoc, [&] {
+      Result = SubstInitializer(UninstExpr, TemplateArgs,
+                                /*DirectInit*/false);
+    });
+  }
+  if (Result.isInvalid())
+    return true;
+
+  // Check the expression as an initializer for the parameter.
+  InitializedEntity Entity
+    = InitializedEntity::InitializeParameter(Context, Param);
+  InitializationKind Kind = InitializationKind::CreateCopy(
+      Param->getLocation(),
+      /*FIXME:EqualLoc*/ UninstExpr->getBeginLoc());
+  Expr *ResultE = Result.getAs<Expr>();
+
+  InitializationSequence InitSeq(*this, Entity, Kind, ResultE);
+  Result = InitSeq.Perform(*this, Entity, Kind, ResultE);
+  if (Result.isInvalid())
+    return true;
+
+  Result =
+      ActOnFinishFullExpr(Result.getAs<Expr>(), Param->getOuterLocStart(),
+                          /*DiscardedValue*/ false);
+  if (Result.isInvalid())
+    return true;
+
+  // Remember the instantiated default argument.
+  Param->setDefaultArg(Result.getAs<Expr>());
+  if (ASTMutationListener *L = getASTMutationListener())
+    L->DefaultArgumentInstantiated(Param);
+
+  return false;
+}
+
 void Sema::InstantiateExceptionSpec(SourceLocation PointOfInstantiation,
                                     FunctionDecl *Decl) {
   const FunctionProtoType *Proto = Decl->getType()->castAs<FunctionProtoType>();
@@ -4255,6 +4345,10 @@ void Sema::InstantiateExceptionSpec(SourceLocation PointOfInstantiation,
   MultiLevelTemplateArgumentList TemplateArgs =
     getTemplateInstantiationArgs(Decl, nullptr, /*RelativeToPrimary*/true);
 
+  // FIXME: We can't use getTemplateInstantiationPattern(false) in general
+  // here, because for a non-defining friend declaration in a class template,
+  // we don't store enough information to map back to the friend declaration in
+  // the template.
   FunctionDecl *Template = Proto->getExceptionSpecTemplate();
   if (addInstantiatedParametersToScope(*this, Decl, Template, Scope,
                                        TemplateArgs)) {
@@ -4364,7 +4458,7 @@ TemplateDeclInstantiator::InitFunctionInstantiation(FunctionDecl *New,
         EPI.ExceptionSpec.Type != EST_None &&
         EPI.ExceptionSpec.Type != EST_DynamicNone &&
         EPI.ExceptionSpec.Type != EST_BasicNoexcept &&
-        !Tmpl->isInLocalScope()) {
+        !Tmpl->isInLocalScopeForInstantiation()) {
       FunctionDecl *ExceptionSpecTemplate = Tmpl;
       if (EPI.ExceptionSpec.Type == EST_Uninstantiated)
         ExceptionSpecTemplate = EPI.ExceptionSpec.SourceTemplate;
@@ -5605,6 +5699,20 @@ DeclContext *Sema::FindInstantiatedContext(SourceLocation Loc, DeclContext* DC,
   } else return DC;
 }
 
+/// Determine whether the given context is dependent on template parameters at
+/// level \p Level or below.
+///
+/// Sometimes we only substitute an inner set of template arguments and leave
+/// the outer templates alone. In such cases, contexts dependent only on the
+/// outer levels are not effectively dependent.
+static bool isDependentContextAtLevel(DeclContext *DC, unsigned Level) {
+  if (!DC->isDependentContext())
+    return false;
+  if (!Level)
+    return true;
+  return cast<Decl>(DC)->getTemplateDepth() > Level;
+}
+
 /// Find the instantiation of the given declaration within the
 /// current instantiation.
 ///
@@ -5635,6 +5743,10 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
                           const MultiLevelTemplateArgumentList &TemplateArgs,
                           bool FindingInstantiatedContext) {
   DeclContext *ParentDC = D->getDeclContext();
+  // Determine whether our parent context depends on any of the tempalte
+  // arguments we're currently substituting.
+  bool ParentDependsOnArgs = isDependentContextAtLevel(
+      ParentDC, TemplateArgs.getNumRetainedOuterLevels());
   // FIXME: Parmeters of pointer to functions (y below) that are themselves
   // parameters (p below) can have their ParentDC set to the translation-unit
   // - thus we can not consistently check if the ParentDC of such a parameter
@@ -5651,15 +5763,14 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
   //  - as long as we have a ParmVarDecl whose parent is non-dependent and
   //    whose type is not instantiation dependent, do nothing to the decl
   //  - otherwise find its instantiated decl.
-  if (isa<ParmVarDecl>(D) && !ParentDC->isDependentContext() &&
+  if (isa<ParmVarDecl>(D) && !ParentDependsOnArgs &&
       !cast<ParmVarDecl>(D)->getType()->isInstantiationDependentType())
     return D;
   if (isa<ParmVarDecl>(D) || isa<NonTypeTemplateParmDecl>(D) ||
       isa<TemplateTypeParmDecl>(D) || isa<TemplateTemplateParmDecl>(D) ||
-      ((ParentDC->isFunctionOrMethod() ||
-        isa<OMPDeclareReductionDecl>(ParentDC) ||
-        isa<OMPDeclareMapperDecl>(ParentDC)) &&
-       ParentDC->isDependentContext()) ||
+      (ParentDependsOnArgs && (ParentDC->isFunctionOrMethod() ||
+                               isa<OMPDeclareReductionDecl>(ParentDC) ||
+                               isa<OMPDeclareMapperDecl>(ParentDC))) ||
       (isa<CXXRecordDecl>(D) && cast<CXXRecordDecl>(D)->isLambda())) {
     // D is a local of some kind. Look into the map of local
     // declarations to their instantiations.
@@ -5813,7 +5924,7 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
     // anonymous unions in class templates).
   }
 
-  if (!ParentDC->isDependentContext())
+  if (!ParentDependsOnArgs)
     return D;
 
   ParentDC = FindInstantiatedContext(Loc, ParentDC, TemplateArgs);
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 46fa8bc0608bd..b8f7f1a581590 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -1759,7 +1759,7 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) {
     //   The effect of a cv-qualifier-seq in a function declarator is not the
     //   same as adding cv-qualification on top of the function type. In the
     //   latter case, the cv-qualifiers are ignored.
-    if (TypeQuals && Result->isFunctionType()) {
+    if (Result->isFunctionType()) {
       diagnoseAndRemoveTypeQualifiers(
           S, DS, TypeQuals, Result, DeclSpec::TQ_const | DeclSpec::TQ_volatile,
           S.getLangOpts().CPlusPlus
@@ -2393,13 +2393,6 @@ QualType Sema::BuildArrayType(QualType T, ArrayType::ArraySizeModifier ASM,
                                          ? diag::err_typecheck_zero_array_size
                                          : diag::ext_typecheck_zero_array_size)
           << ArraySize->getSourceRange();
-
-      if (ASM == ArrayType::Static) {
-        Diag(ArraySize->getBeginLoc(),
-             diag::warn_typecheck_zero_static_array_size)
-            << ArraySize->getSourceRange();
-        ASM = ArrayType::Normal;
-      }
     } else if (!T->isDependentType() && !T->isVariablyModifiedType() &&
                !T->isIncompleteType() && !T->isUndeducedType()) {
       // Is the array too large?
@@ -3301,12 +3294,16 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state,
           D.isFunctionDeclarator())
         break;
       bool Cxx = SemaRef.getLangOpts().CPlusPlus;
-      switch (cast<TagDecl>(SemaRef.CurContext)->getTagKind()) {
-      case TTK_Enum: llvm_unreachable("unhandled tag kind");
-      case TTK_Struct: Error = Cxx ? 1 : 2; /* Struct member */ break;
-      case TTK_Union:  Error = Cxx ? 3 : 4; /* Union member */ break;
-      case TTK_Class:  Error = 5; /* Class member */ break;
-      case TTK_Interface: Error = 6; /* Interface member */ break;
+      if (isa<ObjCContainerDecl>(SemaRef.CurContext)) {
+        Error = 6; // Interface member.
+      } else {
+        switch (cast<TagDecl>(SemaRef.CurContext)->getTagKind()) {
+        case TTK_Enum: llvm_unreachable("unhandled tag kind");
+        case TTK_Struct: Error = Cxx ? 1 : 2; /* Struct member */ break;
+        case TTK_Union:  Error = Cxx ? 3 : 4; /* Union member */ break;
+        case TTK_Class:  Error = 5; /* Class member */ break;
+        case TTK_Interface: Error = 6; /* Interface member */ break;
+        }
       }
       if (D.getDeclSpec().isFriendSpecified())
         Error = 20; // Friend type
@@ -5135,8 +5132,8 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
             // FIXME: This really should be in BuildFunctionType.
             if (S.getLangOpts().OpenCL) {
               if (!S.getOpenCLOptions().isEnabled("cl_khr_fp16")) {
-                S.Diag(Param->getLocation(),
-                  diag::err_opencl_half_param) << ParamTy;
+                S.Diag(Param->getLocation(), diag::err_opencl_invalid_param)
+                    << ParamTy << 0;
                 D.setInvalidType();
                 Param->setInvalidDecl();
               }
@@ -5155,6 +5152,11 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
                 Param->setKNRPromoted(true);
               }
             }
+          } else if (S.getLangOpts().OpenCL && ParamTy->isBlockPointerType()) {
+            // OpenCL 2.0 s6.12.5: A block cannot be a parameter of a function.
+            S.Diag(Param->getLocation(), diag::err_opencl_invalid_param)
+                << ParamTy << 1 /*hint off*/;
+            D.setInvalidType();
           }
 
           if (LangOpts.ObjCAutoRefCount && Param->hasAttr<NSConsumedAttr>()) {
@@ -7031,15 +7033,15 @@ static bool checkNullabilityTypeSpecifier(TypeProcessingState &state,
   // attributes, require that the type be a single-level pointer.
   if (isContextSensitive) {
     // Make sure that the pointee isn't itself a pointer type.
-    const Type *pointeeType;
+    const Type *pointeeType = nullptr;
     if (desugared->isArrayType())
       pointeeType = desugared->getArrayElementTypeNoTypeQual();
-    else
+    else if (desugared->isAnyPointerType())
       pointeeType = desugared->getPointeeType().getTypePtr();
 
-    if (pointeeType->isAnyPointerType() ||
-        pointeeType->isObjCObjectPointerType() ||
-        pointeeType->isMemberPointerType()) {
+    if (pointeeType && (pointeeType->isAnyPointerType() ||
+                        pointeeType->isObjCObjectPointerType() ||
+                        pointeeType->isMemberPointerType())) {
       S.Diag(nullabilityLoc, diag::err_nullability_cs_multilevel)
         << DiagNullabilityKind(nullability, true)
         << type;
@@ -8106,6 +8108,15 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
     case ParsedAttr::AT_AcquireHandle: {
       if (!type->isFunctionType())
         return;
+
+      if (attr.getNumArgs() != 1) {
+        state.getSema().Diag(attr.getLoc(),
+                             diag::err_attribute_wrong_number_arguments)
+            << attr << 1;
+        attr.setInvalid();
+        return;
+      }
+
       StringRef HandleType;
       if (!state.getSema().checkStringLiteralArgumentAttr(attr, 0, HandleType))
         return;
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index ff9d4d610660a..ae0e9f1119b43 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1347,9 +1347,10 @@ class TreeTransform {
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  StmtResult RebuildWhileStmt(SourceLocation WhileLoc,
-                              Sema::ConditionResult Cond, Stmt *Body) {
-    return getSema().ActOnWhileStmt(WhileLoc, Cond, Body);
+  StmtResult RebuildWhileStmt(SourceLocation WhileLoc, SourceLocation LParenLoc,
+                              Sema::ConditionResult Cond,
+                              SourceLocation RParenLoc, Stmt *Body) {
+    return getSema().ActOnWhileStmt(WhileLoc, LParenLoc, Cond, RParenLoc, Body);
   }
 
   /// Build a new do-while statement.
@@ -2436,10 +2437,13 @@ class TreeTransform {
   /// Subclasses may override this routine to provide different behavior.
   ExprResult RebuildOMPArraySectionExpr(Expr *Base, SourceLocation LBracketLoc,
                                         Expr *LowerBound,
-                                        SourceLocation ColonLoc, Expr *Length,
+                                        SourceLocation ColonLocFirst,
+                                        SourceLocation ColonLocSecond,
+                                        Expr *Length, Expr *Stride,
                                         SourceLocation RBracketLoc) {
     return getSema().ActOnOMPArraySectionExpr(Base, LBracketLoc, LowerBound,
-                                              ColonLoc, Length, RBracketLoc);
+                                              ColonLocFirst, ColonLocSecond,
+                                              Length, Stride, RBracketLoc);
   }
 
   /// Build a new array shaping expression.
@@ -7332,7 +7336,8 @@ TreeTransform<Derived>::TransformWhileStmt(WhileStmt *S) {
       Body.get() == S->getBody())
     return Owned(S);
 
-  return getDerived().RebuildWhileStmt(S->getWhileLoc(), Cond, Body.get());
+  return getDerived().RebuildWhileStmt(S->getWhileLoc(), S->getLParenLoc(),
+                                       Cond, S->getRParenLoc(), Body.get());
 }
 
 template<typename Derived>
@@ -7630,7 +7635,8 @@ TreeTransform<Derived>::TransformCoroutineBodyStmt(CoroutineBodyStmt *S) {
     return StmtError();
   StmtResult FinalSuspend =
       getDerived().TransformStmt(S->getFinalSuspendStmt());
-  if (FinalSuspend.isInvalid())
+  if (FinalSuspend.isInvalid() ||
+      !SemaRef.checkFinalSuspendNoThrow(FinalSuspend.get()))
     return StmtError();
   ScopeInfo->setCoroutineSuspends(InitSuspend.get(), FinalSuspend.get());
   assert(isa<Expr>(InitSuspend.get()) && isa<Expr>(FinalSuspend.get()));
@@ -10336,13 +10342,21 @@ TreeTransform<Derived>::TransformOMPArraySectionExpr(OMPArraySectionExpr *E) {
       return ExprError();
   }
 
+  ExprResult Stride;
+  if (Expr *Str = E->getStride()) {
+    Stride = getDerived().TransformExpr(Str);
+    if (Stride.isInvalid())
+      return ExprError();
+  }
+
   if (!getDerived().AlwaysRebuild() && Base.get() == E->getBase() &&
       LowerBound.get() == E->getLowerBound() && Length.get() == E->getLength())
     return E;
 
   return getDerived().RebuildOMPArraySectionExpr(
-      Base.get(), E->getBase()->getEndLoc(), LowerBound.get(), E->getColonLoc(),
-      Length.get(), E->getRBracketLoc());
+      Base.get(), E->getBase()->getEndLoc(), LowerBound.get(),
+      E->getColonLocFirst(), E->getColonLocSecond(), Length.get(), Stride.get(),
+      E->getRBracketLoc());
 }
 
 template <typename Derived>
@@ -10563,8 +10577,10 @@ TreeTransform<Derived>::TransformBinaryOperator(BinaryOperator *E) {
     return getDerived().RebuildBinaryOperator(
         E->getOperatorLoc(), E->getOpcode(), LHS.get(), RHS.get());
   Sema::FPFeaturesStateRAII FPFeaturesState(getSema());
-  getSema().CurFPFeatures = E->getFPFeatures(getSema().getLangOpts());
-
+  FPOptionsOverride NewOverrides(E->getFPFeatures(getSema().getLangOpts()));
+  getSema().CurFPFeatures =
+      NewOverrides.applyOverrides(getSema().getLangOpts());
+  getSema().FpPragmaStack.CurrentValue = NewOverrides.getAsOpaqueInt();
   return getDerived().RebuildBinaryOperator(E->getOperatorLoc(), E->getOpcode(),
                                             LHS.get(), RHS.get());
 }
@@ -10618,7 +10634,10 @@ ExprResult
 TreeTransform<Derived>::TransformCompoundAssignOperator(
                                                       CompoundAssignOperator *E) {
   Sema::FPFeaturesStateRAII FPFeaturesState(getSema());
-  getSema().CurFPFeatures = E->getFPFeatures(getSema().getLangOpts());
+  FPOptionsOverride NewOverrides(E->getFPFeatures(getSema().getLangOpts()));
+  getSema().CurFPFeatures =
+      NewOverrides.applyOverrides(getSema().getLangOpts());
+  getSema().FpPragmaStack.CurrentValue = NewOverrides.getAsOpaqueInt();
   return getDerived().TransformBinaryOperator(E);
 }
 
@@ -11093,7 +11112,10 @@ TreeTransform<Derived>::TransformCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
     return SemaRef.MaybeBindToTemporary(E);
 
   Sema::FPFeaturesStateRAII FPFeaturesState(getSema());
-  getSema().CurFPFeatures = E->getFPFeatures();
+  FPOptionsOverride NewOverrides(E->getFPFeatures());
+  getSema().CurFPFeatures =
+      NewOverrides.applyOverrides(getSema().getLangOpts());
+  getSema().FpPragmaStack.CurrentValue = NewOverrides.getAsOpaqueInt();
 
   return getDerived().RebuildCXXOperatorCallExpr(E->getOperator(),
                                                  E->getOperatorLoc(),
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 4dd2054d4b079..4a1a995204e59 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -3234,7 +3234,8 @@ ASTReader::ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities) {
     case MODULAR_CODEGEN_DECLS:
       // FIXME: Skip reading this record if our ASTConsumer doesn't care about
       // them (ie: if we're not codegenerating this module).
-      if (F.Kind == MK_MainFile)
+      if (F.Kind == MK_MainFile ||
+          getContext().getLangOpts().BuildingPCHWithObjectFile)
         for (unsigned I = 0, N = Record.size(); I != N; ++I)
           EagerlyDeserializedDecls.push_back(getGlobalDeclID(F, Record[I]));
       break;
@@ -7843,7 +7844,9 @@ void ASTReader::InitializeSema(Sema &S) {
   // FIXME: What happens if these are changed by a module import?
   if (!FPPragmaOptions.empty()) {
     assert(FPPragmaOptions.size() == 1 && "Wrong number of FP_PRAGMA_OPTIONS");
-    SemaObj->CurFPFeatures = FPOptions(FPPragmaOptions[0]);
+    FPOptionsOverride NewOverrides(FPPragmaOptions[0]);
+    SemaObj->CurFPFeatures =
+        NewOverrides.applyOverrides(SemaObj->getLangOpts());
   }
 
   SemaObj->OpenCLFeatures.copy(OpenCLExtensions);
@@ -11992,6 +11995,8 @@ OMPClause *OMPClauseReader::readClause() {
   case llvm::omp::Enum:                                                        \
     break;
 #include "llvm/Frontend/OpenMP/OMPKinds.def"
+  default:
+    break;
   }
   assert(C && "Unknown OMPClause type");
 
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 359d5e567e672..eef4ab16ec15e 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -503,8 +503,12 @@ uint64_t ASTDeclReader::GetCurrentCursorOffset() {
 }
 
 void ASTDeclReader::ReadFunctionDefinition(FunctionDecl *FD) {
-  if (Record.readInt())
+  if (Record.readInt()) {
     Reader.DefinitionSource[FD] = Loc.F->Kind == ModuleKind::MK_MainFile;
+    if (Reader.getContext().getLangOpts().BuildingPCHWithObjectFile &&
+        Reader.DeclIsFromPCHWithObjectFile(FD))
+      Reader.DefinitionSource[FD] = true;
+  }
   if (auto *CD = dyn_cast<CXXConstructorDecl>(FD)) {
     CD->setNumCtorInitializers(Record.readInt());
     if (CD->getNumCtorInitializers())
@@ -1431,8 +1435,12 @@ ASTDeclReader::RedeclarableResult ASTDeclReader::VisitVarDeclImpl(VarDecl *VD) {
       Reader.getContext().setBlockVarCopyInit(VD, CopyExpr, Record.readInt());
   }
 
-  if (VD->getStorageDuration() == SD_Static && Record.readInt())
+  if (VD->getStorageDuration() == SD_Static && Record.readInt()) {
     Reader.DefinitionSource[VD] = Loc.F->Kind == ModuleKind::MK_MainFile;
+    if (Reader.getContext().getLangOpts().BuildingPCHWithObjectFile &&
+        Reader.DeclIsFromPCHWithObjectFile(VD))
+      Reader.DefinitionSource[VD] = true;
+  }
 
   enum VarKind {
     VarNotTemplate = 0, VarTemplate, StaticDataMemberSpecialization
@@ -1691,8 +1699,12 @@ void ASTDeclReader::ReadCXXDefinitionData(
   Data.ODRHash = Record.readInt();
   Data.HasODRHash = true;
 
-  if (Record.readInt())
+  if (Record.readInt()) {
     Reader.DefinitionSource[D] = Loc.F->Kind == ModuleKind::MK_MainFile;
+    if (Reader.getContext().getLangOpts().BuildingPCHWithObjectFile &&
+        Reader.DeclIsFromPCHWithObjectFile(D))
+      Reader.DefinitionSource[D] = true;
+  }
 
   Data.NumBases = Record.readInt();
   if (Data.NumBases)
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 0c379839ba839..a40c5499a6d77 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -271,6 +271,8 @@ void ASTStmtReader::VisitWhileStmt(WhileStmt *S) {
     S->setConditionVariable(Record.getContext(), readDeclAs<VarDecl>());
 
   S->setWhileLoc(readSourceLocation());
+  S->setLParenLoc(readSourceLocation());
+  S->setRParenLoc(readSourceLocation());
 }
 
 void ASTStmtReader::VisitDoStmt(DoStmt *S) {
@@ -706,7 +708,7 @@ void ASTStmtReader::VisitUnaryOperator(UnaryOperator *E) {
   E->setOperatorLoc(readSourceLocation());
   E->setCanOverflow(Record.readInt());
   if (hasFP_Features)
-    E->setStoredFPFeatures(FPOptions(Record.readInt()));
+    E->setStoredFPFeatures(FPOptionsOverride(Record.readInt()));
 }
 
 void ASTStmtReader::VisitOffsetOfExpr(OffsetOfExpr *E) {
@@ -937,7 +939,9 @@ void ASTStmtReader::VisitOMPArraySectionExpr(OMPArraySectionExpr *E) {
   E->setBase(Record.readSubExpr());
   E->setLowerBound(Record.readSubExpr());
   E->setLength(Record.readSubExpr());
-  E->setColonLoc(readSourceLocation());
+  E->setStride(Record.readSubExpr());
+  E->setColonLocFirst(readSourceLocation());
+  E->setColonLocSecond(readSourceLocation());
   E->setRBracketLoc(readSourceLocation());
 }
 
@@ -1089,7 +1093,7 @@ void ASTStmtReader::VisitBinaryOperator(BinaryOperator *E) {
   E->setRHS(Record.readSubExpr());
   E->setOperatorLoc(readSourceLocation());
   if (hasFP_Features)
-    E->setStoredFPFeatures(FPOptions(Record.readInt()));
+    E->setStoredFPFeatures(FPOptionsOverride(Record.readInt()));
 }
 
 void ASTStmtReader::VisitCompoundAssignOperator(CompoundAssignOperator *E) {
@@ -1657,8 +1661,8 @@ void ASTStmtReader::VisitMSDependentExistsStmt(MSDependentExistsStmt *S) {
 void ASTStmtReader::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
   VisitCallExpr(E);
   E->CXXOperatorCallExprBits.OperatorKind = Record.readInt();
-  E->CXXOperatorCallExprBits.FPFeatures = Record.readInt();
   E->Range = Record.readSourceRange();
+  E->setFPFeatures(FPOptionsOverride(Record.readInt()));
 }
 
 void ASTStmtReader::VisitCXXRewrittenBinaryOperator(
@@ -1715,12 +1719,12 @@ void ASTStmtReader::VisitLambdaExpr(LambdaExpr *E) {
 
   // Read capture initializers.
   for (LambdaExpr::capture_init_iterator C = E->capture_init_begin(),
-                                      CEnd = E->capture_init_end();
+                                         CEnd = E->capture_init_end();
        C != CEnd; ++C)
     *C = Record.readSubExpr();
 
-  // Ok, not one past the end.
-  E->getStoredStmts()[NumCaptures] = Record.readSubStmt();
+  // The body will be lazily deserialized when needed from the call operator
+  // declaration.
 }
 
 void
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 2a17360a67bc4..2345a12caeb2e 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -3960,7 +3960,7 @@ void ASTWriter::WriteDeclContextVisibleUpdate(const DeclContext *DC) {
 }
 
 /// Write an FP_PRAGMA_OPTIONS block for the given FPOptions.
-void ASTWriter::WriteFPPragmaOptions(const FPOptions &Opts) {
+void ASTWriter::WriteFPPragmaOptions(const FPOptionsOverride &Opts) {
   RecordData::value_type Record[] = {Opts.getAsOpaqueInt()};
   Stream.EmitRecord(FP_PRAGMA_OPTIONS, Record);
 }
@@ -4790,7 +4790,7 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
   WriteReferencedSelectorsPool(SemaRef);
   WriteLateParsedTemplates(SemaRef);
   WriteIdentifierTable(PP, SemaRef.IdResolver, isModule);
-  WriteFPPragmaOptions(SemaRef.getCurFPFeatures());
+  WriteFPPragmaOptions(SemaRef.CurFPFeatureOverrides());
   WriteOpenCLExtensions(SemaRef);
   WriteOpenCLExtensionTypes(SemaRef);
   WriteCUDAPragmas(SemaRef);
@@ -5688,8 +5688,8 @@ void ASTRecordWriter::AddCXXDefinitionData(const CXXRecordDecl *D) {
 
   // getODRHash will compute the ODRHash if it has not been previously computed.
   Record->push_back(D->getODRHash());
-  bool ModulesDebugInfo = Writer->Context->getLangOpts().ModulesDebugInfo &&
-                          Writer->WritingModule && !D->isDependentType();
+  bool ModulesDebugInfo =
+      Writer->Context->getLangOpts().ModulesDebugInfo && !D->isDependentType();
   Record->push_back(ModulesDebugInfo);
   if (ModulesDebugInfo)
     Writer->ModularCodegenDecls.push_back(Writer->GetDeclRef(D));
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index 19608c869613d..eecdf89c791ad 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -2458,9 +2458,10 @@ void ASTRecordWriter::AddFunctionDefinition(const FunctionDecl *FD) {
 
   assert(FD->doesThisDeclarationHaveABody());
   bool ModulesCodegen = false;
-  if (Writer->WritingModule && !FD->isDependentContext()) {
+  if (!FD->isDependentContext()) {
     Optional<GVALinkage> Linkage;
-    if (Writer->WritingModule->Kind == Module::ModuleInterfaceUnit) {
+    if (Writer->WritingModule &&
+        Writer->WritingModule->Kind == Module::ModuleInterfaceUnit) {
       // When building a C++ Modules TS module interface unit, a strong
       // definition in the module interface is provided by the compilation of
       // that module interface unit, not by its users. (Inline functions are
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index b13d0df9c3761..0767b3a24bf2f 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -194,6 +194,8 @@ void ASTStmtWriter::VisitWhileStmt(WhileStmt *S) {
     Record.AddDeclRef(S->getConditionVariable());
 
   Record.AddSourceLocation(S->getWhileLoc());
+  Record.AddSourceLocation(S->getLParenLoc());
+  Record.AddSourceLocation(S->getRParenLoc());
   Code = serialization::STMT_WHILE;
 }
 
@@ -797,7 +799,9 @@ void ASTStmtWriter::VisitOMPArraySectionExpr(OMPArraySectionExpr *E) {
   Record.AddStmt(E->getBase());
   Record.AddStmt(E->getLowerBound());
   Record.AddStmt(E->getLength());
-  Record.AddSourceLocation(E->getColonLoc());
+  Record.AddStmt(E->getStride());
+  Record.AddSourceLocation(E->getColonLocFirst());
+  Record.AddSourceLocation(E->getColonLocSecond());
   Record.AddSourceLocation(E->getRBracketLoc());
   Code = serialization::EXPR_OMP_ARRAY_SECTION;
 }
@@ -1545,8 +1549,8 @@ void ASTStmtWriter::VisitMSDependentExistsStmt(MSDependentExistsStmt *S) {
 void ASTStmtWriter::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
   VisitCallExpr(E);
   Record.push_back(E->getOperator());
-  Record.push_back(E->getFPFeatures().getAsOpaqueInt());
   Record.AddSourceRange(E->Range);
+  Record.push_back(E->getFPFeatures().getAsOpaqueInt());
   Code = serialization::EXPR_CXX_OPERATOR_CALL;
 }
 
@@ -1615,7 +1619,8 @@ void ASTStmtWriter::VisitLambdaExpr(LambdaExpr *E) {
     Record.AddStmt(*C);
   }
 
-  Record.AddStmt(E->getBody());
+  // Don't serialize the body. It belongs to the call operator declaration.
+  // LambdaExpr only stores a copy of the Stmt *.
 
   Code = serialization::EXPR_LAMBDA;
 }
diff --git a/clang/lib/Serialization/CMakeLists.txt b/clang/lib/Serialization/CMakeLists.txt
index 1f05117bb5fd4..a7c5b5e4ffa41 100644
--- a/clang/lib/Serialization/CMakeLists.txt
+++ b/clang/lib/Serialization/CMakeLists.txt
@@ -30,4 +30,7 @@ add_clang_library(clangSerialization
   clangBasic
   clangLex
   clangSema
+
+  DEPENDS
+  omp_gen
   )
diff --git a/clang/lib/Serialization/GlobalModuleIndex.cpp b/clang/lib/Serialization/GlobalModuleIndex.cpp
index 958cca7345722..9192b3b476bbc 100644
--- a/clang/lib/Serialization/GlobalModuleIndex.cpp
+++ b/clang/lib/Serialization/GlobalModuleIndex.cpp
@@ -321,7 +321,7 @@ bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
     = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
   IdentifierIndexTable::iterator Known = Table.find(Name);
   if (Known == Table.end()) {
-    return true;
+    return false;
   }
 
   SmallVector<unsigned, 2> ModuleIDs = *Known;
diff --git a/clang/lib/StaticAnalyzer/Checkers/AnalysisOrderChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/AnalysisOrderChecker.cpp
index 479e50b80867e..0e8cbc60689a7 100644
--- a/clang/lib/StaticAnalyzer/Checkers/AnalysisOrderChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/AnalysisOrderChecker.cpp
@@ -38,7 +38,7 @@ class AnalysisOrderChecker
           check::PostStmt<OffsetOfExpr>, check::PreCall, check::PostCall,
           check::EndFunction, check::EndAnalysis, check::NewAllocator,
           check::Bind, check::PointerEscape, check::RegionChanges,
-          check::LiveSymbols> {
+          check::LiveSymbols, eval::Call> {
 
   bool isCallbackEnabled(const AnalyzerOptions &Opts,
                          StringRef CallbackName) const {
@@ -122,6 +122,19 @@ class AnalysisOrderChecker
       llvm::errs() << "PostStmt<OffsetOfExpr>\n";
   }
 
+  bool evalCall(const CallEvent &Call, CheckerContext &C) const {
+    if (isCallbackEnabled(C, "EvalCall")) {
+      llvm::errs() << "EvalCall";
+      if (const NamedDecl *ND = dyn_cast_or_null<NamedDecl>(Call.getDecl()))
+        llvm::errs() << " (" << ND->getQualifiedNameAsString() << ')';
+      llvm::errs() << " {argno: " << Call.getNumArgs() << '}';
+      llvm::errs() << " [" << Call.getKindAsString() << ']';
+      llvm::errs() << '\n';
+      return true;
+    }
+    return false;
+  }
+
   void checkPreCall(const CallEvent &Call, CheckerContext &C) const {
     if (isCallbackEnabled(C, "PreCall")) {
       llvm::errs() << "PreCall";
diff --git a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
index 3eeb1e9a3502e..233ce57c3ac9f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
@@ -64,10 +64,12 @@ bool BuiltinFunctionChecker::evalCall(const CallEvent &Call,
 
   case Builtin::BI__builtin_unpredictable:
   case Builtin::BI__builtin_expect:
+  case Builtin::BI__builtin_expect_with_probability:
   case Builtin::BI__builtin_assume_aligned:
   case Builtin::BI__builtin_addressof: {
-    // For __builtin_unpredictable, __builtin_expect, and
-    // __builtin_assume_aligned, just return the value of the subexpression.
+    // For __builtin_unpredictable, __builtin_expect,
+    // __builtin_expect_with_probability and __builtin_assume_aligned,
+    // just return the value of the subexpression.
     // __builtin_addressof is going from a reference to a pointer, but those
     // are represented the same way in the analyzer.
     assert (Call.getNumArgs() > 0);
diff --git a/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt b/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
index 40f520d522fe0..9be1fdeb3ebfa 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
+++ b/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
@@ -98,6 +98,7 @@ add_clang_library(clangStaticAnalyzerCheckers
   ReturnValueChecker.cpp
   RunLoopAutoreleaseLeakChecker.cpp
   SimpleStreamChecker.cpp
+  SmartPtrChecker.cpp
   SmartPtrModeling.cpp
   StackAddrEscapeChecker.cpp
   StdLibraryFunctionsChecker.cpp
@@ -134,4 +135,7 @@ add_clang_library(clangStaticAnalyzerCheckers
   clangBasic
   clangLex
   clangStaticAnalyzerCore
+
+  DEPENDS
+  omp_gen
   )
diff --git a/clang/lib/StaticAnalyzer/Checkers/InvalidatedIteratorChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/InvalidatedIteratorChecker.cpp
index c6cd11b09a4ca..6955ba11a28ff 100644
--- a/clang/lib/StaticAnalyzer/Checkers/InvalidatedIteratorChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/InvalidatedIteratorChecker.cpp
@@ -26,7 +26,10 @@ using namespace iterator;
 namespace {
 
 class InvalidatedIteratorChecker
-  : public Checker<check::PreCall> {
+  : public Checker<check::PreCall, check::PreStmt<UnaryOperator>,
+                   check::PreStmt<BinaryOperator>,
+                   check::PreStmt<ArraySubscriptExpr>,
+                   check::PreStmt<MemberExpr>> {
 
   std::unique_ptr<BugType> InvalidatedBugType;
 
@@ -37,6 +40,10 @@ class InvalidatedIteratorChecker
   InvalidatedIteratorChecker();
 
   void checkPreCall(const CallEvent &Call, CheckerContext &C) const;
+  void checkPreStmt(const UnaryOperator *UO, CheckerContext &C) const;
+  void checkPreStmt(const BinaryOperator *BO, CheckerContext &C) const;
+  void checkPreStmt(const ArraySubscriptExpr *ASE, CheckerContext &C) const;
+  void checkPreStmt(const MemberExpr *ME, CheckerContext &C) const;
 
 };
 
@@ -65,6 +72,48 @@ void InvalidatedIteratorChecker::checkPreCall(const CallEvent &Call,
   }
 }
 
+void InvalidatedIteratorChecker::checkPreStmt(const UnaryOperator *UO,
+                                              CheckerContext &C) const {
+  if (isa<CXXThisExpr>(UO->getSubExpr()))
+    return;
+
+  ProgramStateRef State = C.getState();
+  UnaryOperatorKind OK = UO->getOpcode();
+  SVal SubVal = State->getSVal(UO->getSubExpr(), C.getLocationContext());
+
+  if (isAccessOperator(OK)) {
+    verifyAccess(C, SubVal);
+  }
+}
+
+void InvalidatedIteratorChecker::checkPreStmt(const BinaryOperator *BO,
+                                              CheckerContext &C) const {
+  ProgramStateRef State = C.getState();
+  BinaryOperatorKind OK = BO->getOpcode();
+  SVal LVal = State->getSVal(BO->getLHS(), C.getLocationContext());
+
+  if (isAccessOperator(OK)) {
+    verifyAccess(C, LVal);
+  }
+}
+
+void InvalidatedIteratorChecker::checkPreStmt(const ArraySubscriptExpr *ASE,
+                                              CheckerContext &C) const {
+  ProgramStateRef State = C.getState();
+  SVal LVal = State->getSVal(ASE->getLHS(), C.getLocationContext());
+  verifyAccess(C, LVal);
+}
+
+void InvalidatedIteratorChecker::checkPreStmt(const MemberExpr *ME,
+                                              CheckerContext &C) const {
+  if (!ME->isArrow() || ME->isImplicitAccess())
+    return;
+
+  ProgramStateRef State = C.getState();
+  SVal BaseVal = State->getSVal(ME->getBase(), C.getLocationContext());
+  verifyAccess(C, BaseVal);
+}
+
 void InvalidatedIteratorChecker::verifyAccess(CheckerContext &C, const SVal &Val) const {
   auto State = C.getState();
   const auto *Pos = getIteratorPosition(State, Val);
diff --git a/clang/lib/StaticAnalyzer/Checkers/Iterator.cpp b/clang/lib/StaticAnalyzer/Checkers/Iterator.cpp
index e80d8bc32deca..ac0f24603dd90 100644
--- a/clang/lib/StaticAnalyzer/Checkers/Iterator.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/Iterator.cpp
@@ -128,24 +128,54 @@ bool isAccessOperator(OverloadedOperatorKind OK) {
          isDecrementOperator(OK) || isRandomIncrOrDecrOperator(OK);
 }
 
+bool isAccessOperator(UnaryOperatorKind OK) {
+  return isDereferenceOperator(OK) || isIncrementOperator(OK) ||
+         isDecrementOperator(OK);
+}
+
+bool isAccessOperator(BinaryOperatorKind OK) {
+  return isDereferenceOperator(OK) || isRandomIncrOrDecrOperator(OK);
+}
+
 bool isDereferenceOperator(OverloadedOperatorKind OK) {
   return OK == OO_Star || OK == OO_Arrow || OK == OO_ArrowStar ||
          OK == OO_Subscript;
 }
 
+bool isDereferenceOperator(UnaryOperatorKind OK) {
+  return OK == UO_Deref;
+}
+
+bool isDereferenceOperator(BinaryOperatorKind OK) {
+  return OK == BO_PtrMemI;
+}
+
 bool isIncrementOperator(OverloadedOperatorKind OK) {
   return OK == OO_PlusPlus;
 }
 
+bool isIncrementOperator(UnaryOperatorKind OK) {
+  return OK == UO_PreInc || OK == UO_PostInc;
+}
+
 bool isDecrementOperator(OverloadedOperatorKind OK) {
   return OK == OO_MinusMinus;
 }
 
+bool isDecrementOperator(UnaryOperatorKind OK) {
+  return OK == UO_PreDec || OK == UO_PostDec;
+}
+
 bool isRandomIncrOrDecrOperator(OverloadedOperatorKind OK) {
   return OK == OO_Plus || OK == OO_PlusEqual || OK == OO_Minus ||
          OK == OO_MinusEqual;
 }
 
+bool isRandomIncrOrDecrOperator(BinaryOperatorKind OK) {
+  return OK == BO_Add || OK == BO_AddAssign ||
+         OK == BO_Sub || OK == BO_SubAssign;
+}
+
 const ContainerData *getContainerData(ProgramStateRef State,
                                       const MemRegion *Cont) {
   return State->get<ContainerMap>(Cont);
diff --git a/clang/lib/StaticAnalyzer/Checkers/Iterator.h b/clang/lib/StaticAnalyzer/Checkers/Iterator.h
index d5c6cb96ebc64..37157492fe3e9 100644
--- a/clang/lib/StaticAnalyzer/Checkers/Iterator.h
+++ b/clang/lib/StaticAnalyzer/Checkers/Iterator.h
@@ -115,9 +115,12 @@ class IteratorSymbolMap {};
 class IteratorRegionMap {};
 class ContainerMap {};
 
-using IteratorSymbolMapTy = CLANG_ENTO_PROGRAMSTATE_MAP(SymbolRef, IteratorPosition);
-using IteratorRegionMapTy = CLANG_ENTO_PROGRAMSTATE_MAP(const MemRegion *, IteratorPosition);
-using ContainerMapTy = CLANG_ENTO_PROGRAMSTATE_MAP(const MemRegion *, ContainerData);
+using IteratorSymbolMapTy =
+  CLANG_ENTO_PROGRAMSTATE_MAP(SymbolRef, IteratorPosition);
+using IteratorRegionMapTy =
+  CLANG_ENTO_PROGRAMSTATE_MAP(const MemRegion *, IteratorPosition);
+using ContainerMapTy =
+  CLANG_ENTO_PROGRAMSTATE_MAP(const MemRegion *, ContainerData);
 
 } // namespace iterator
 
@@ -149,10 +152,17 @@ bool isEraseCall(const FunctionDecl *Func);
 bool isEraseAfterCall(const FunctionDecl *Func);
 bool isEmplaceCall(const FunctionDecl *Func);
 bool isAccessOperator(OverloadedOperatorKind OK);
+bool isAccessOperator(UnaryOperatorKind OK);
+bool isAccessOperator(BinaryOperatorKind OK);
 bool isDereferenceOperator(OverloadedOperatorKind OK);
+bool isDereferenceOperator(UnaryOperatorKind OK);
+bool isDereferenceOperator(BinaryOperatorKind OK);
 bool isIncrementOperator(OverloadedOperatorKind OK);
+bool isIncrementOperator(UnaryOperatorKind OK);
 bool isDecrementOperator(OverloadedOperatorKind OK);
+bool isDecrementOperator(UnaryOperatorKind OK);
 bool isRandomIncrOrDecrOperator(OverloadedOperatorKind OK);
+bool isRandomIncrOrDecrOperator(BinaryOperatorKind OK);
 const ContainerData *getContainerData(ProgramStateRef State,
                                       const MemRegion *Cont);
 const IteratorPosition *getIteratorPosition(ProgramStateRef State,
diff --git a/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp
index e35918edbf89c..fd8cbd694b240 100644
--- a/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp
@@ -83,7 +83,9 @@ using namespace iterator;
 namespace {
 
 class IteratorModeling
-    : public Checker<check::PostCall, check::PostStmt<MaterializeTemporaryExpr>,
+    : public Checker<check::PostCall, check::PostStmt<UnaryOperator>,
+                     check::PostStmt<BinaryOperator>,
+                     check::PostStmt<MaterializeTemporaryExpr>,
                      check::Bind, check::LiveSymbols, check::DeadSymbols> {
 
   using AdvanceFn = void (IteratorModeling::*)(CheckerContext &, const Expr *,
@@ -108,6 +110,8 @@ class IteratorModeling
   void handleRandomIncrOrDecr(CheckerContext &C, const Expr *CE,
                               OverloadedOperatorKind Op, const SVal &RetVal,
                               const SVal &LHS, const SVal &RHS) const;
+  void handlePtrIncrOrDecr(CheckerContext &C, const Expr *Iterator,
+                           OverloadedOperatorKind OK, SVal Offset) const;
   void handleAdvance(CheckerContext &C, const Expr *CE, SVal RetVal, SVal Iter,
                      SVal Amount) const;
   void handlePrev(CheckerContext &C, const Expr *CE, SVal RetVal, SVal Iter,
@@ -144,6 +148,8 @@ class IteratorModeling
 
   void checkPostCall(const CallEvent &Call, CheckerContext &C) const;
   void checkBind(SVal Loc, SVal Val, const Stmt *S, CheckerContext &C) const;
+  void checkPostStmt(const UnaryOperator *UO, CheckerContext &C) const;
+  void checkPostStmt(const BinaryOperator *BO, CheckerContext &C) const;
   void checkPostStmt(const CXXConstructExpr *CCE, CheckerContext &C) const;
   void checkPostStmt(const DeclStmt *DS, CheckerContext &C) const;
   void checkPostStmt(const MaterializeTemporaryExpr *MTE,
@@ -153,6 +159,7 @@ class IteratorModeling
 };
 
 bool isSimpleComparisonOperator(OverloadedOperatorKind OK);
+bool isSimpleComparisonOperator(BinaryOperatorKind OK);
 ProgramStateRef removeIteratorPosition(ProgramStateRef State, const SVal &Val);
 ProgramStateRef relateSymbols(ProgramStateRef State, SymbolRef Sym1,
                               SymbolRef Sym2, bool Equal);
@@ -207,12 +214,15 @@ void IteratorModeling::checkPostCall(const CallEvent &Call,
   }
 
   // Assumption: if return value is an iterator which is not yet bound to a
-  //             container, then look for the first iterator argument, and
-  //             bind the return value to the same container. This approach
-  //             works for STL algorithms.
+  //             container, then look for the first iterator argument of the
+  //             same type as the return value and bind the return value to
+  //             the same container. This approach works for STL algorithms.
   // FIXME: Add a more conservative mode
   for (unsigned i = 0; i < Call.getNumArgs(); ++i) {
-    if (isIteratorType(Call.getArgExpr(i)->getType())) {
+    if (isIteratorType(Call.getArgExpr(i)->getType()) &&
+        Call.getArgExpr(i)->getType().getNonReferenceType().getDesugaredType(
+            C.getASTContext()).getTypePtr() ==
+        Call.getResultType().getDesugaredType(C.getASTContext()).getTypePtr()) {
       if (const auto *Pos = getIteratorPosition(State, Call.getArgSVal(i))) {
         assignToContainer(C, OrigExpr, Call.getReturnValue(),
                           Pos->getContainer());
@@ -238,6 +248,35 @@ void IteratorModeling::checkBind(SVal Loc, SVal Val, const Stmt *S,
   }
 }
 
+void IteratorModeling::checkPostStmt(const UnaryOperator *UO,
+                                     CheckerContext &C) const {
+  UnaryOperatorKind OK = UO->getOpcode();
+  if (!isIncrementOperator(OK) && !isDecrementOperator(OK))
+    return;
+
+  auto &SVB = C.getSValBuilder();
+  handlePtrIncrOrDecr(C, UO->getSubExpr(),
+                      isIncrementOperator(OK) ? OO_Plus : OO_Minus,
+                      SVB.makeArrayIndex(1));
+}
+
+void IteratorModeling::checkPostStmt(const BinaryOperator *BO,
+                                     CheckerContext &C) const {
+  ProgramStateRef State = C.getState();
+  BinaryOperatorKind OK = BO->getOpcode();
+  SVal RVal = State->getSVal(BO->getRHS(), C.getLocationContext());
+
+  if (isSimpleComparisonOperator(BO->getOpcode())) {
+    SVal LVal = State->getSVal(BO->getLHS(), C.getLocationContext());
+    SVal Result = State->getSVal(BO, C.getLocationContext());
+    handleComparison(C, BO, Result, LVal, RVal,
+                     BinaryOperator::getOverloadedOperator(OK));
+  } else if (isRandomIncrOrDecrOperator(OK)) {
+    handlePtrIncrOrDecr(C, BO->getLHS(),
+                        BinaryOperator::getOverloadedOperator(OK), RVal);
+  }
+}
+
 void IteratorModeling::checkPostStmt(const MaterializeTemporaryExpr *MTE,
                                      CheckerContext &C) const {
   /* Transfer iterator state to temporary objects */
@@ -458,7 +497,7 @@ void IteratorModeling::processComparison(CheckerContext &C,
     StateTrue = StateTrue->assume(*ConditionVal, true);
     C.addTransition(StateTrue);
   }
-  
+
   if (auto StateFalse = relateSymbols(State, Sym1, Sym2, Op != OO_EqualEqual)) {
     StateFalse = StateFalse->assume(*ConditionVal, false);
     C.addTransition(StateFalse);
@@ -540,20 +579,65 @@ void IteratorModeling::handleRandomIncrOrDecr(CheckerContext &C,
 
   auto &TgtVal = (Op == OO_PlusEqual || Op == OO_MinusEqual) ? LHS : RetVal;
 
-  auto NewState =
-    advancePosition(State, LHS, Op, *value);
-  if (NewState) {
-    const auto *NewPos = getIteratorPosition(NewState, LHS);
+  // `AdvancedState` is a state where the position of `LHS` is advanced. We
+  // only need this state to retrieve the new position, but we do not want
+  // to change the position of `LHS` (in every case).
+  auto AdvancedState = advancePosition(State, LHS, Op, *value);
+  if (AdvancedState) {
+    const auto *NewPos = getIteratorPosition(AdvancedState, LHS);
     assert(NewPos &&
            "Iterator should have position after successful advancement");
 
-    State = setIteratorPosition(NewState, TgtVal, *NewPos);
+    State = setIteratorPosition(State, TgtVal, *NewPos);
     C.addTransition(State);
   } else {
     assignToContainer(C, CE, TgtVal, Pos->getContainer());
   }
 }
 
+void IteratorModeling::handlePtrIncrOrDecr(CheckerContext &C,
+                                           const Expr *Iterator,
+                                           OverloadedOperatorKind OK,
+                                           SVal Offset) const {
+  QualType PtrType = Iterator->getType();
+  if (!PtrType->isPointerType())
+    return;
+  QualType ElementType = PtrType->getPointeeType();
+
+  ProgramStateRef State = C.getState();
+  SVal OldVal = State->getSVal(Iterator, C.getLocationContext());
+
+  const IteratorPosition *OldPos = getIteratorPosition(State, OldVal);
+  if (!OldPos)
+    return;
+
+  SVal NewVal;
+  if (OK == OO_Plus || OK == OO_PlusEqual)
+    NewVal = State->getLValue(ElementType, Offset, OldVal);
+  else {
+    const llvm::APSInt &OffsetInt =
+      Offset.castAs<nonloc::ConcreteInt>().getValue();
+    auto &BVF = C.getSymbolManager().getBasicVals();
+    SVal NegatedOffset = nonloc::ConcreteInt(BVF.getValue(-OffsetInt));
+    NewVal = State->getLValue(ElementType, NegatedOffset, OldVal);
+  }
+
+  // `AdvancedState` is a state where the position of `Old` is advanced. We
+  // only need this state to retrieve the new position, but we do not want
+  // ever to change the position of `OldVal`.
+  auto AdvancedState = advancePosition(State, OldVal, OK, Offset);
+  if (AdvancedState) {
+    const IteratorPosition *NewPos = getIteratorPosition(AdvancedState, OldVal);
+    assert(NewPos &&
+           "Iterator should have position after successful advancement");
+
+    ProgramStateRef NewState = setIteratorPosition(State, NewVal, *NewPos);
+    C.addTransition(NewState);
+  } else {
+    assignToContainer(C, Iterator, NewVal, OldPos->getContainer());
+  }
+}
+
 void IteratorModeling::handleAdvance(CheckerContext &C, const Expr *CE,
                                      SVal RetVal, SVal Iter,
                                      SVal Amount) const {
@@ -611,10 +695,15 @@ void IteratorModeling::printState(raw_ostream &Out, ProgramStateRef State,
                                   const char *NL, const char *Sep) const {
   auto SymbolMap = State->get<IteratorSymbolMap>();
   auto RegionMap = State->get<IteratorRegionMap>();
+  // Use a counter to add newlines before every line except the first one.
+  unsigned Count = 0;
 
   if (!SymbolMap.isEmpty() || !RegionMap.isEmpty()) {
     Out << Sep << "Iterator Positions :" << NL;
     for (const auto &Sym : SymbolMap) {
+      if (Count++)
+        Out << NL;
+
       Sym.first->dumpToStream(Out);
       Out << " : ";
       const auto Pos = Sym.second;
@@ -625,6 +714,9 @@ void IteratorModeling::printState(raw_ostream &Out, ProgramStateRef State,
     }
 
     for (const auto &Reg : RegionMap) {
+      if (Count++)
+        Out << NL;
+
       Reg.first->dumpToStream(Out);
       Out << " : ";
       const auto Pos = Reg.second;
@@ -642,6 +734,10 @@ bool isSimpleComparisonOperator(OverloadedOperatorKind OK) {
   return OK == OO_EqualEqual || OK == OO_ExclaimEqual;
 }
 
+bool isSimpleComparisonOperator(BinaryOperatorKind OK) {
+  return OK == BO_EQ || OK == BO_NE;
+}
+
 ProgramStateRef removeIteratorPosition(ProgramStateRef State, const SVal &Val) {
   if (auto Reg = Val.getAsRegion()) {
     Reg = Reg->getMostDerivedObjectRegion();
diff --git a/clang/lib/StaticAnalyzer/Checkers/IteratorRangeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/IteratorRangeChecker.cpp
index 6e86ef0c552a9..df8e379d1f20e 100644
--- a/clang/lib/StaticAnalyzer/Checkers/IteratorRangeChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/IteratorRangeChecker.cpp
@@ -27,7 +27,10 @@ using namespace iterator;
 namespace {
 
 class IteratorRangeChecker
-  : public Checker<check::PreCall> {
+  : public Checker<check::PreCall, check::PreStmt<UnaryOperator>,
+                   check::PreStmt<BinaryOperator>,
+                   check::PreStmt<ArraySubscriptExpr>,
+                   check::PreStmt<MemberExpr>> {
 
   std::unique_ptr<BugType> OutOfRangeBugType;
 
@@ -46,6 +49,10 @@ class IteratorRangeChecker
   IteratorRangeChecker();
 
   void checkPreCall(const CallEvent &Call, CheckerContext &C) const;
+  void checkPreStmt(const UnaryOperator *UO, CheckerContext &C) const;
+  void checkPreStmt(const BinaryOperator *BO, CheckerContext &C) const;
+  void checkPreStmt(const ArraySubscriptExpr *ASE, CheckerContext &C) const;
+  void checkPreStmt(const MemberExpr *ME, CheckerContext &C) const;
 
   using AdvanceFn = void (IteratorRangeChecker::*)(CheckerContext &, SVal,
                                                    SVal) const;
@@ -134,6 +141,56 @@ void IteratorRangeChecker::checkPreCall(const CallEvent &Call,
   }
 }
 
+void IteratorRangeChecker::checkPreStmt(const UnaryOperator *UO,
+                                        CheckerContext &C) const {
+  if (isa<CXXThisExpr>(UO->getSubExpr()))
+    return;
+
+  ProgramStateRef State = C.getState();
+  UnaryOperatorKind OK = UO->getOpcode();
+  SVal SubVal = State->getSVal(UO->getSubExpr(), C.getLocationContext());
+
+  if (isDereferenceOperator(OK)) {
+    verifyDereference(C, SubVal);
+  } else if (isIncrementOperator(OK)) {
+    verifyIncrement(C, SubVal);
+  } else if (isDecrementOperator(OK)) {
+    verifyDecrement(C, SubVal);
+  }
+}
+
+void IteratorRangeChecker::checkPreStmt(const BinaryOperator *BO,
+                                        CheckerContext &C) const {
+  ProgramStateRef State = C.getState();
+  BinaryOperatorKind OK = BO->getOpcode();
+  SVal LVal = State->getSVal(BO->getLHS(), C.getLocationContext());
+
+  if (isDereferenceOperator(OK)) {
+    verifyDereference(C, LVal);
+  } else if (isRandomIncrOrDecrOperator(OK)) {
+    SVal RVal = State->getSVal(BO->getRHS(), C.getLocationContext());
+    verifyRandomIncrOrDecr(C, BinaryOperator::getOverloadedOperator(OK), LVal,
+                           RVal);
+  }
+}
+
+void IteratorRangeChecker::checkPreStmt(const ArraySubscriptExpr *ASE,
+                                        CheckerContext &C) const {
+  ProgramStateRef State = C.getState();
+  SVal LVal = State->getSVal(ASE->getLHS(), C.getLocationContext());
+  verifyDereference(C, LVal);
+}
+
+void IteratorRangeChecker::checkPreStmt(const MemberExpr *ME,
+                                        CheckerContext &C) const {
+  if (!ME->isArrow() || ME->isImplicitAccess())
+    return;
+
+  ProgramStateRef State = C.getState();
+  SVal BaseVal = State->getSVal(ME->getBase(), C.getLocationContext());
+  verifyDereference(C, BaseVal);
+}
+
 void IteratorRangeChecker::verifyDereference(CheckerContext &C,
                                              SVal Val) const {
   auto State = C.getState();
diff --git a/clang/lib/StaticAnalyzer/Checkers/MismatchedIteratorChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MismatchedIteratorChecker.cpp
index d170b875856c2..1960873599f73 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MismatchedIteratorChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MismatchedIteratorChecker.cpp
@@ -28,7 +28,7 @@ using namespace iterator;
 namespace {
 
 class MismatchedIteratorChecker
-  : public Checker<check::PreCall> {
+  : public Checker<check::PreCall, check::PreStmt<BinaryOperator>> {
 
   std::unique_ptr<BugType> MismatchedBugType;
 
@@ -47,6 +47,7 @@ class MismatchedIteratorChecker
   MismatchedIteratorChecker();
 
   void checkPreCall(const CallEvent &Call, CheckerContext &C) const;
+  void checkPreStmt(const BinaryOperator *BO, CheckerContext &C) const;
 
 };
 
@@ -141,7 +142,7 @@ void MismatchedIteratorChecker::checkPreCall(const CallEvent &Call,
     // Example:
     // template<typename I1, typename I2>
     // void f(I1 first1, I1 last1, I2 first2, I2 last2);
-    // 
+    //
     // In this case the first two arguments to f() must be iterators must belong
     // to the same container and the last to also to the same container but
     // not necessarily to the same as the first two.
@@ -188,6 +189,17 @@ void MismatchedIteratorChecker::checkPreCall(const CallEvent &Call,
   }
 }
 
+void MismatchedIteratorChecker::checkPreStmt(const BinaryOperator *BO,
+                                             CheckerContext &C) const {
+  if (!BO->isComparisonOp())
+    return;
+
+  ProgramStateRef State = C.getState();
+  SVal LVal = State->getSVal(BO->getLHS(), C.getLocationContext());
+  SVal RVal = State->getSVal(BO->getRHS(), C.getLocationContext());
+  verifyMatch(C, LVal, RVal);
+}
+
 void MismatchedIteratorChecker::verifyMatch(CheckerContext &C, const SVal &Iter,
                                             const MemRegion *Cont) const {
   // Verify match between a container and the container of an iterator
diff --git a/clang/lib/StaticAnalyzer/Checkers/SmartPtr.h b/clang/lib/StaticAnalyzer/Checkers/SmartPtr.h
new file mode 100644
index 0000000000000..89b8965e4c9ad
--- /dev/null
+++ b/clang/lib/StaticAnalyzer/Checkers/SmartPtr.h
@@ -0,0 +1,40 @@
+//=== SmartPtr.h - Tracking smart pointer state. -------------------*- C++ -*-//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines inter-checker API for the smart pointer modeling. It allows
+// dependent checkers to figure out if an smart pointer is null or not.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_STATICANALYZER_CHECKERS_SMARTPTR_H
+#define LLVM_CLANG_LIB_STATICANALYZER_CHECKERS_SMARTPTR_H
+
+#include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
+
+namespace clang {
+namespace ento {
+namespace smartptr {
+
+/// Set of STL smart pointer class which we are trying to model.
+const llvm::StringSet<> StdSmartPtrs = {
+    "shared_ptr",
+    "unique_ptr",
+    "weak_ptr",
+};
+
+/// Returns true if the event call is on smart pointer.
+bool isStdSmartPtrCall(const CallEvent &Call);
+
+/// Returns whether the smart pointer is null or not.
+bool isNullSmartPtr(const ProgramStateRef State, const MemRegion *ThisRegion);
+
+} // namespace smartptr
+} // namespace ento
+} // namespace clang
+
+#endif // LLVM_CLANG_LIB_STATICANALYZER_CHECKERS_SMARTPTR_H
diff --git a/clang/lib/StaticAnalyzer/Checkers/SmartPtrChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/SmartPtrChecker.cpp
new file mode 100644
index 0000000000000..7bb25f397d010
--- /dev/null
+++ b/clang/lib/StaticAnalyzer/Checkers/SmartPtrChecker.cpp
@@ -0,0 +1,80 @@
+// SmartPtrChecker.cpp - Check for smart pointer dereference - C++ --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a checker that check for null dereference of C++ smart
+// pointer.
+//
+//===----------------------------------------------------------------------===//
+#include "SmartPtr.h"
+
+#include "clang/AST/DeclCXX.h"
+#include "clang/AST/ExprCXX.h"
+#include "clang/AST/Type.h"
+#include "clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/Checker.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SVals.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SymExpr.h"
+
+using namespace clang;
+using namespace ento;
+
+namespace {
+class SmartPtrChecker : public Checker<check::PreCall> {
+  BugType NullDereferenceBugType{this, "Null SmartPtr dereference",
+                                 "C++ Smart Pointer"};
+
+public:
+  void checkPreCall(const CallEvent &Call, CheckerContext &C) const;
+
+private:
+  void reportBug(CheckerContext &C, const CallEvent &Call) const;
+};
+} // end of anonymous namespace
+
+void SmartPtrChecker::checkPreCall(const CallEvent &Call,
+                                   CheckerContext &C) const {
+  if (!smartptr::isStdSmartPtrCall(Call))
+    return;
+  ProgramStateRef State = C.getState();
+  const auto *OC = dyn_cast<CXXMemberOperatorCall>(&Call);
+  if (!OC)
+    return;
+  const MemRegion *ThisRegion = OC->getCXXThisVal().getAsRegion();
+  if (!ThisRegion)
+    return;
+
+  OverloadedOperatorKind OOK = OC->getOverloadedOperator();
+  if (OOK == OO_Star || OOK == OO_Arrow) {
+    if (smartptr::isNullSmartPtr(State, ThisRegion))
+      reportBug(C, Call);
+  }
+}
+
+void SmartPtrChecker::reportBug(CheckerContext &C,
+                                const CallEvent &Call) const {
+  ExplodedNode *ErrNode = C.generateErrorNode();
+  if (!ErrNode)
+    return;
+
+  auto R = std::make_unique<PathSensitiveBugReport>(
+      NullDereferenceBugType, "Dereference of null smart pointer", ErrNode);
+  C.emitReport(std::move(R));
+}
+
+void ento::registerSmartPtrChecker(CheckerManager &Mgr) {
+  Mgr.registerChecker<SmartPtrChecker>();
+}
+
+bool ento::shouldRegisterSmartPtrChecker(const CheckerManager &mgr) {
+  const LangOptions &LO = mgr.getLangOpts();
+  return LO.CPlusPlus;
+}
diff --git a/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp
index 8250fd46b9266..91f2890788141 100644
--- a/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp
@@ -12,27 +12,80 @@
 //===----------------------------------------------------------------------===//
 
 #include "Move.h"
+#include "SmartPtr.h"
 
+#include "clang/AST/DeclCXX.h"
 #include "clang/AST/ExprCXX.h"
+#include "clang/AST/Type.h"
 #include "clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SVals.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SymExpr.h"
 
 using namespace clang;
 using namespace ento;
 
 namespace {
-class SmartPtrModeling : public Checker<eval::Call> {
+class SmartPtrModeling : public Checker<eval::Call, check::DeadSymbols> {
+
   bool isNullAfterMoveMethod(const CallEvent &Call) const;
 
 public:
+  // Whether the checker should model for null dereferences of smart pointers.
+  DefaultBool ModelSmartPtrDereference;
   bool evalCall(const CallEvent &Call, CheckerContext &C) const;
+  void checkPreCall(const CallEvent &Call, CheckerContext &C) const;
+  void checkDeadSymbols(SymbolReaper &SymReaper, CheckerContext &C) const;
+
+private:
+  ProgramStateRef updateTrackedRegion(const CallEvent &Call, CheckerContext &C,
+                                      const MemRegion *ThisValRegion) const;
+  void handleReset(const CallEvent &Call, CheckerContext &C) const;
+  void handleRelease(const CallEvent &Call, CheckerContext &C) const;
+  void handleSwap(const CallEvent &Call, CheckerContext &C) const;
+
+  using SmartPtrMethodHandlerFn =
+      void (SmartPtrModeling::*)(const CallEvent &Call, CheckerContext &) const;
+  CallDescriptionMap<SmartPtrMethodHandlerFn> SmartPtrMethodHandlers{
+      {{"reset"}, &SmartPtrModeling::handleReset},
+      {{"release"}, &SmartPtrModeling::handleRelease},
+      {{"swap", 1}, &SmartPtrModeling::handleSwap}};
 };
 } // end of anonymous namespace
 
+REGISTER_MAP_WITH_PROGRAMSTATE(TrackedRegionMap, const MemRegion *, SVal)
+
+// Define the inter-checker API.
+namespace clang {
+namespace ento {
+namespace smartptr {
+bool isStdSmartPtrCall(const CallEvent &Call) {
+  const auto *MethodDecl = dyn_cast_or_null<CXXMethodDecl>(Call.getDecl());
+  if (!MethodDecl || !MethodDecl->getParent())
+    return false;
+
+  const auto *RecordDecl = MethodDecl->getParent();
+  if (!RecordDecl || !RecordDecl->getDeclContext()->isStdNamespace())
+    return false;
+
+  if (RecordDecl->getDeclName().isIdentifier()) {
+    return smartptr::StdSmartPtrs.count(RecordDecl->getName().lower());
+  }
+  return false;
+}
+
+bool isNullSmartPtr(const ProgramStateRef State, const MemRegion *ThisRegion) {
+  const auto *InnerPointVal = State->get<TrackedRegionMap>(ThisRegion);
+  return InnerPointVal && InnerPointVal->isZeroConstant();
+}
+} // namespace smartptr
+} // namespace ento
+} // namespace clang
+
 bool SmartPtrModeling::isNullAfterMoveMethod(const CallEvent &Call) const {
   // TODO: Update CallDescription to support anonymous calls?
   // TODO: Handle other methods, such as .get() or .release().
@@ -44,27 +97,133 @@ bool SmartPtrModeling::isNullAfterMoveMethod(const CallEvent &Call) const {
 
 bool SmartPtrModeling::evalCall(const CallEvent &Call,
                                 CheckerContext &C) const {
-  if (!isNullAfterMoveMethod(Call))
+
+  if (!smartptr::isStdSmartPtrCall(Call))
     return false;
 
-  ProgramStateRef State = C.getState();
-  const MemRegion *ThisR =
-      cast<CXXInstanceCall>(&Call)->getCXXThisVal().getAsRegion();
+  if (isNullAfterMoveMethod(Call)) {
+    ProgramStateRef State = C.getState();
+    const MemRegion *ThisR =
+        cast<CXXInstanceCall>(&Call)->getCXXThisVal().getAsRegion();
+
+    if (!move::isMovedFrom(State, ThisR)) {
+      // TODO: Model this case as well. At least, avoid invalidation of globals.
+      return false;
+    }
+
+    // TODO: Add a note to bug reports describing this decision.
+    C.addTransition(
+        State->BindExpr(Call.getOriginExpr(), C.getLocationContext(),
+                        C.getSValBuilder().makeZeroVal(Call.getResultType())));
+    return true;
+  }
 
-  if (!move::isMovedFrom(State, ThisR)) {
-    // TODO: Model this case as well. At least, avoid invalidation of globals.
+  if (!ModelSmartPtrDereference)
     return false;
+
+  if (const auto *CC = dyn_cast<CXXConstructorCall>(&Call)) {
+    if (CC->getDecl()->isCopyOrMoveConstructor())
+      return false;
+
+    const MemRegion *ThisValRegion = CC->getCXXThisVal().getAsRegion();
+    if (!ThisValRegion)
+      return false;
+
+    auto State = updateTrackedRegion(Call, C, ThisValRegion);
+    C.addTransition(State);
+    return true;
+  }
+
+  const SmartPtrMethodHandlerFn *Handler = SmartPtrMethodHandlers.lookup(Call);
+  if (!Handler)
+    return false;
+  (this->**Handler)(Call, C);
+
+  return C.isDifferent();
+}
+
+void SmartPtrModeling::checkDeadSymbols(SymbolReaper &SymReaper,
+                                        CheckerContext &C) const {
+  ProgramStateRef State = C.getState();
+  // Clean up dead regions from the region map.
+  TrackedRegionMapTy TrackedRegions = State->get<TrackedRegionMap>();
+  for (auto E : TrackedRegions) {
+    const MemRegion *Region = E.first;
+    bool IsRegDead = !SymReaper.isLiveRegion(Region);
+
+    if (IsRegDead)
+      State = State->remove<TrackedRegionMap>(Region);
+  }
+  C.addTransition(State);
+}
+
+void SmartPtrModeling::handleReset(const CallEvent &Call,
+                                   CheckerContext &C) const {
+  const auto *IC = dyn_cast<CXXInstanceCall>(&Call);
+  if (!IC)
+    return;
+
+  const MemRegion *ThisValRegion = IC->getCXXThisVal().getAsRegion();
+  if (!ThisValRegion)
+    return;
+  auto State = updateTrackedRegion(Call, C, ThisValRegion);
+  C.addTransition(State);
+  // TODO: Make sure to ivalidate the the region in the Store if we don't have
+  // time to model all methods.
+}
+
+void SmartPtrModeling::handleRelease(const CallEvent &Call,
+                                     CheckerContext &C) const {
+  const auto *IC = dyn_cast<CXXInstanceCall>(&Call);
+  if (!IC)
+    return;
+
+  const MemRegion *ThisValRegion = IC->getCXXThisVal().getAsRegion();
+  if (!ThisValRegion)
+    return;
+
+  auto State = updateTrackedRegion(Call, C, ThisValRegion);
+
+  const auto *InnerPointVal = State->get<TrackedRegionMap>(ThisValRegion);
+  if (InnerPointVal) {
+    State = State->BindExpr(Call.getOriginExpr(), C.getLocationContext(),
+                            *InnerPointVal);
+  }
+  C.addTransition(State);
+  // TODO: Add support to enable MallocChecker to start tracking the raw
+  // pointer.
+}
+
+void SmartPtrModeling::handleSwap(const CallEvent &Call,
+                                  CheckerContext &C) const {
+  // TODO: Add support to handle swap method.
+}
+
+ProgramStateRef
+SmartPtrModeling::updateTrackedRegion(const CallEvent &Call, CheckerContext &C,
+                                      const MemRegion *ThisValRegion) const {
+  // TODO: Refactor and clean up handling too many things.
+  ProgramStateRef State = C.getState();
+  auto NumArgs = Call.getNumArgs();
+
+  if (NumArgs == 0) {
+    auto NullSVal = C.getSValBuilder().makeNull();
+    State = State->set<TrackedRegionMap>(ThisValRegion, NullSVal);
+  } else if (NumArgs == 1) {
+    auto ArgVal = Call.getArgSVal(0);
+    assert(Call.getArgExpr(0)->getType()->isPointerType() &&
+           "Adding a non pointer value to TrackedRegionMap");
+    State = State->set<TrackedRegionMap>(ThisValRegion, ArgVal);
   }
 
-  // TODO: Add a note to bug reports describing this decision.
-  C.addTransition(
-      State->BindExpr(Call.getOriginExpr(), C.getLocationContext(),
-                      C.getSValBuilder().makeZeroVal(Call.getResultType())));
-  return true;
+  return State;
 }
 
 void ento::registerSmartPtrModeling(CheckerManager &Mgr) {
-  Mgr.registerChecker<SmartPtrModeling>();
+  auto *Checker = Mgr.registerChecker<SmartPtrModeling>();
+  Checker->ModelSmartPtrDereference =
+      Mgr.getAnalyzerOptions().getCheckerBooleanOption(
+          Checker, "ModelSmartPtrDereference");
 }
 
 bool ento::shouldRegisterSmartPtrModeling(const CheckerManager &mgr) {
diff --git a/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
index 6feae56502f1f..8b575f4f4759c 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
@@ -461,6 +461,7 @@ class StdLibraryFunctionsChecker
   CheckerNameRef CheckNames[CK_NumCheckKinds];
 
   bool DisplayLoadedSummaries = false;
+  bool ModelPOSIX = false;
 
 private:
   Optional<Summary> findFunctionSummary(const FunctionDecl *FD,
@@ -725,7 +726,8 @@ StdLibraryFunctionsChecker::findFunctionSummary(const CallEvent &Call,
   return findFunctionSummary(FD, C);
 }
 
-llvm::Optional<QualType> lookupType(StringRef Name, const ASTContext &ACtx) {
+static llvm::Optional<QualType> lookupType(StringRef Name,
+                                           const ASTContext &ACtx) {
   IdentifierInfo &II = ACtx.Idents.get(Name);
   auto LookupRes = ACtx.getTranslationUnitDecl()->lookup(&II);
   if (LookupRes.size() == 0)
@@ -736,13 +738,18 @@ llvm::Optional<QualType> lookupType(StringRef Name, const ASTContext &ACtx) {
   //   typedef struct FILE FILE;
   // In this case, we have a RecordDecl 'struct FILE' with the name 'FILE' and
   // we have a TypedefDecl with the name 'FILE'.
-  for (Decl *D : LookupRes) {
+  for (Decl *D : LookupRes)
     if (auto *TD = dyn_cast<TypedefNameDecl>(D))
       return ACtx.getTypeDeclType(TD).getCanonicalType();
-  }
-  assert(LookupRes.size() == 1 && "Type identifier should be unique");
-  auto *D = cast<TypeDecl>(LookupRes.front());
-  return ACtx.getTypeDeclType(D).getCanonicalType();
+
+  // Find the first TypeDecl.
+  // There maybe cases when a function has the same name as a struct.
+  // E.g. in POSIX: `struct stat` and the function `stat()`:
+  //   int stat(const char *restrict path, struct stat *restrict buf);
+  for (Decl *D : LookupRes)
+    if (auto *TD = dyn_cast<TypeDecl>(D))
+      return ACtx.getTypeDeclType(TD).getCanonicalType();
+  return None;
 }
 
 void StdLibraryFunctionsChecker::initFunctionSummaries(
@@ -761,26 +768,46 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
   // of function summary for common cases (eg. ssize_t could be int or long
   // or long long, so three summary variants would be enough).
   // Of course, function variants are also useful for C++ overloads.
+  const QualType VoidTy = ACtx.VoidTy;
   const QualType IntTy = ACtx.IntTy;
+  const QualType UnsignedIntTy = ACtx.UnsignedIntTy;
   const QualType LongTy = ACtx.LongTy;
   const QualType LongLongTy = ACtx.LongLongTy;
   const QualType SizeTy = ACtx.getSizeType();
+
   const QualType VoidPtrTy = ACtx.VoidPtrTy; // void *
+  const QualType IntPtrTy = ACtx.getPointerType(IntTy); // int *
+  const QualType UnsignedIntPtrTy =
+      ACtx.getPointerType(UnsignedIntTy); // unsigned int *
   const QualType VoidPtrRestrictTy =
       ACtx.getLangOpts().C99 ? ACtx.getRestrictType(VoidPtrTy) // void *restrict
                              : VoidPtrTy;
   const QualType ConstVoidPtrTy =
       ACtx.getPointerType(ACtx.VoidTy.withConst()); // const void *
+  const QualType CharPtrTy = ACtx.getPointerType(ACtx.CharTy); // char *
+  const QualType CharPtrRestrictTy =
+      ACtx.getLangOpts().C99 ? ACtx.getRestrictType(CharPtrTy) // char *restrict
+                             : CharPtrTy;
   const QualType ConstCharPtrTy =
       ACtx.getPointerType(ACtx.CharTy.withConst()); // const char *
+  const QualType ConstCharPtrRestrictTy =
+      ACtx.getLangOpts().C99
+          ? ACtx.getRestrictType(ConstCharPtrTy) // const char *restrict
+          : ConstCharPtrTy;
+  const QualType Wchar_tPtrTy = ACtx.getPointerType(ACtx.WCharTy); // wchar_t *
+  const QualType ConstWchar_tPtrTy =
+      ACtx.getPointerType(ACtx.WCharTy.withConst()); // const wchar_t *
   const QualType ConstVoidPtrRestrictTy =
       ACtx.getLangOpts().C99
           ? ACtx.getRestrictType(ConstVoidPtrTy) // const void *restrict
           : ConstVoidPtrTy;
 
   const RangeInt IntMax = BVF.getMaxValue(IntTy).getLimitedValue();
+  const RangeInt UnsignedIntMax =
+      BVF.getMaxValue(UnsignedIntTy).getLimitedValue();
   const RangeInt LongMax = BVF.getMaxValue(LongTy).getLimitedValue();
   const RangeInt LongLongMax = BVF.getMaxValue(LongLongTy).getLimitedValue();
+  const RangeInt SizeMax = BVF.getMaxValue(SizeTy).getLimitedValue();
 
   // Set UCharRangeMax to min of int or uchar maximum value.
   // The C standard states that the arguments of functions like isalpha must
@@ -1110,6 +1137,573 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
                           {Getline(IntTy, IntMax), Getline(LongTy, LongMax),
                            Getline(LongLongTy, LongLongMax)});
 
+  if (ModelPOSIX) {
+
+    // long a64l(const char *str64);
+    addToFunctionSummaryMap(
+        "a64l", Summary(ArgTypes{ConstCharPtrTy}, RetType{LongTy}, NoEvalCall)
+                    .ArgConstraint(NotNull(ArgNo(0))));
+
+    // char *l64a(long value);
+    addToFunctionSummaryMap(
+        "l64a", Summary(ArgTypes{LongTy}, RetType{CharPtrTy}, NoEvalCall)
+                    .ArgConstraint(
+                        ArgumentCondition(0, WithinRange, Range(0, LongMax))));
+
+    // int access(const char *pathname, int amode);
+    addToFunctionSummaryMap("access", Summary(ArgTypes{ConstCharPtrTy, IntTy},
+                                              RetType{IntTy}, NoEvalCall)
+                                          .ArgConstraint(NotNull(ArgNo(0))));
+
+    // int faccessat(int dirfd, const char *pathname, int mode, int flags);
+    addToFunctionSummaryMap(
+        "faccessat", Summary(ArgTypes{IntTy, ConstCharPtrTy, IntTy, IntTy},
+                             RetType{IntTy}, NoEvalCall)
+                         .ArgConstraint(NotNull(ArgNo(1))));
+
+    // int dup(int fildes);
+    addToFunctionSummaryMap(
+        "dup", Summary(ArgTypes{IntTy}, RetType{IntTy}, NoEvalCall)
+                   .ArgConstraint(
+                       ArgumentCondition(0, WithinRange, Range(0, IntMax))));
+
+    // int dup2(int fildes1, int filedes2);
+    addToFunctionSummaryMap(
+        "dup2",
+        Summary(ArgTypes{IntTy, IntTy}, RetType{IntTy}, NoEvalCall)
+            .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
+            .ArgConstraint(
+                ArgumentCondition(1, WithinRange, Range(0, IntMax))));
+
+    // int fdatasync(int fildes);
+    addToFunctionSummaryMap(
+        "fdatasync", Summary(ArgTypes{IntTy}, RetType{IntTy}, NoEvalCall)
+                         .ArgConstraint(ArgumentCondition(0, WithinRange,
+                                                          Range(0, IntMax))));
+
+    // int fnmatch(const char *pattern, const char *string, int flags);
+    addToFunctionSummaryMap(
+        "fnmatch", Summary(ArgTypes{ConstCharPtrTy, ConstCharPtrTy, IntTy},
+                           RetType{IntTy}, EvalCallAsPure)
+                       .ArgConstraint(NotNull(ArgNo(0)))
+                       .ArgConstraint(NotNull(ArgNo(1))));
+
+    // int fsync(int fildes);
+    addToFunctionSummaryMap(
+        "fsync", Summary(ArgTypes{IntTy}, RetType{IntTy}, NoEvalCall)
+                     .ArgConstraint(
+                         ArgumentCondition(0, WithinRange, Range(0, IntMax))));
+
+    Optional<QualType> Off_tTy = lookupType("off_t", ACtx);
+
+    if (Off_tTy)
+      // int truncate(const char *path, off_t length);
+      addToFunctionSummaryMap("truncate",
+                              Summary(ArgTypes{ConstCharPtrTy, *Off_tTy},
+                                      RetType{IntTy}, NoEvalCall)
+                                  .ArgConstraint(NotNull(ArgNo(0))));
+
+    // int symlink(const char *oldpath, const char *newpath);
+    addToFunctionSummaryMap("symlink",
+                            Summary(ArgTypes{ConstCharPtrTy, ConstCharPtrTy},
+                                    RetType{IntTy}, NoEvalCall)
+                                .ArgConstraint(NotNull(ArgNo(0)))
+                                .ArgConstraint(NotNull(ArgNo(1))));
+
+    // int symlinkat(const char *oldpath, int newdirfd, const char *newpath);
+    addToFunctionSummaryMap(
+        "symlinkat",
+        Summary(ArgTypes{ConstCharPtrTy, IntTy, ConstCharPtrTy}, RetType{IntTy},
+                NoEvalCall)
+            .ArgConstraint(NotNull(ArgNo(0)))
+            .ArgConstraint(ArgumentCondition(1, WithinRange, Range(0, IntMax)))
+            .ArgConstraint(NotNull(ArgNo(2))));
+
+    if (Off_tTy)
+      // int lockf(int fd, int cmd, off_t len);
+      addToFunctionSummaryMap(
+          "lockf",
+          Summary(ArgTypes{IntTy, IntTy, *Off_tTy}, RetType{IntTy}, NoEvalCall)
+              .ArgConstraint(
+                  ArgumentCondition(0, WithinRange, Range(0, IntMax))));
+
+    Optional<QualType> Mode_tTy = lookupType("mode_t", ACtx);
+
+    if (Mode_tTy)
+      // int creat(const char *pathname, mode_t mode);
+      addToFunctionSummaryMap("creat",
+                              Summary(ArgTypes{ConstCharPtrTy, *Mode_tTy},
+                                      RetType{IntTy}, NoEvalCall)
+                                  .ArgConstraint(NotNull(ArgNo(0))));
+
+    // unsigned int sleep(unsigned int seconds);
+    addToFunctionSummaryMap(
+        "sleep",
+        Summary(ArgTypes{UnsignedIntTy}, RetType{UnsignedIntTy}, NoEvalCall)
+            .ArgConstraint(
+                ArgumentCondition(0, WithinRange, Range(0, UnsignedIntMax))));
+
+    Optional<QualType> DirTy = lookupType("DIR", ACtx);
+    Optional<QualType> DirPtrTy;
+    if (DirTy)
+      DirPtrTy = ACtx.getPointerType(*DirTy);
+
+    if (DirPtrTy)
+      // int dirfd(DIR *dirp);
+      addToFunctionSummaryMap(
+          "dirfd", Summary(ArgTypes{*DirPtrTy}, RetType{IntTy}, NoEvalCall)
+                       .ArgConstraint(NotNull(ArgNo(0))));
+
+    // unsigned int alarm(unsigned int seconds);
+    addToFunctionSummaryMap(
+        "alarm",
+        Summary(ArgTypes{UnsignedIntTy}, RetType{UnsignedIntTy}, NoEvalCall)
+            .ArgConstraint(
+                ArgumentCondition(0, WithinRange, Range(0, UnsignedIntMax))));
+
+    if (DirPtrTy)
+      // int closedir(DIR *dir);
+      addToFunctionSummaryMap(
+          "closedir", Summary(ArgTypes{*DirPtrTy}, RetType{IntTy}, NoEvalCall)
+                          .ArgConstraint(NotNull(ArgNo(0))));
+
+    // char *strdup(const char *s);
+    addToFunctionSummaryMap("strdup", Summary(ArgTypes{ConstCharPtrTy},
+                                              RetType{CharPtrTy}, NoEvalCall)
+                                          .ArgConstraint(NotNull(ArgNo(0))));
+
+    // char *strndup(const char *s, size_t n);
+    addToFunctionSummaryMap(
+        "strndup", Summary(ArgTypes{ConstCharPtrTy, SizeTy}, RetType{CharPtrTy},
+                           NoEvalCall)
+                       .ArgConstraint(NotNull(ArgNo(0)))
+                       .ArgConstraint(ArgumentCondition(1, WithinRange,
+                                                        Range(0, SizeMax))));
+
+    // wchar_t *wcsdup(const wchar_t *s);
+    addToFunctionSummaryMap("wcsdup", Summary(ArgTypes{ConstWchar_tPtrTy},
+                                              RetType{Wchar_tPtrTy}, NoEvalCall)
+                                          .ArgConstraint(NotNull(ArgNo(0))));
+
+    // int mkstemp(char *template);
+    addToFunctionSummaryMap(
+        "mkstemp", Summary(ArgTypes{CharPtrTy}, RetType{IntTy}, NoEvalCall)
+                       .ArgConstraint(NotNull(ArgNo(0))));
+
+    // char *mkdtemp(char *template);
+    addToFunctionSummaryMap(
+        "mkdtemp", Summary(ArgTypes{CharPtrTy}, RetType{CharPtrTy}, NoEvalCall)
+                       .ArgConstraint(NotNull(ArgNo(0))));
+
+    // char *getcwd(char *buf, size_t size);
+    addToFunctionSummaryMap(
+        "getcwd",
+        Summary(ArgTypes{CharPtrTy, SizeTy}, RetType{CharPtrTy}, NoEvalCall)
+            .ArgConstraint(
+                ArgumentCondition(1, WithinRange, Range(0, SizeMax))));
+
+    if (Mode_tTy) {
+      // int mkdir(const char *pathname, mode_t mode);
+      addToFunctionSummaryMap("mkdir",
+                              Summary(ArgTypes{ConstCharPtrTy, *Mode_tTy},
+                                      RetType{IntTy}, NoEvalCall)
+                                  .ArgConstraint(NotNull(ArgNo(0))));
+
+      // int mkdirat(int dirfd, const char *pathname, mode_t mode);
+      addToFunctionSummaryMap(
+          "mkdirat", Summary(ArgTypes{IntTy, ConstCharPtrTy, *Mode_tTy},
+                             RetType{IntTy}, NoEvalCall)
+                         .ArgConstraint(NotNull(ArgNo(1))));
+    }
+
+    Optional<QualType> Dev_tTy = lookupType("dev_t", ACtx);
+
+    if (Mode_tTy && Dev_tTy) {
+      // int mknod(const char *pathname, mode_t mode, dev_t dev);
+      addToFunctionSummaryMap(
+          "mknod", Summary(ArgTypes{ConstCharPtrTy, *Mode_tTy, *Dev_tTy},
+                           RetType{IntTy}, NoEvalCall)
+                       .ArgConstraint(NotNull(ArgNo(0))));
+
+      // int mknodat(int dirfd, const char *pathname, mode_t mode, dev_t dev);
+      addToFunctionSummaryMap("mknodat", Summary(ArgTypes{IntTy, ConstCharPtrTy,
+                                                          *Mode_tTy, *Dev_tTy},
+                                                 RetType{IntTy}, NoEvalCall)
+                                             .ArgConstraint(NotNull(ArgNo(1))));
+    }
+
+    if (Mode_tTy) {
+      // int chmod(const char *path, mode_t mode);
+      addToFunctionSummaryMap("chmod",
+                              Summary(ArgTypes{ConstCharPtrTy, *Mode_tTy},
+                                      RetType{IntTy}, NoEvalCall)
+                                  .ArgConstraint(NotNull(ArgNo(0))));
+
+      // int fchmodat(int dirfd, const char *pathname, mode_t mode, int flags);
+      addToFunctionSummaryMap(
+          "fchmodat", Summary(ArgTypes{IntTy, ConstCharPtrTy, *Mode_tTy, IntTy},
+                              RetType{IntTy}, NoEvalCall)
+                          .ArgConstraint(ArgumentCondition(0, WithinRange,
+                                                           Range(0, IntMax)))
+                          .ArgConstraint(NotNull(ArgNo(1))));
+
+      // int fchmod(int fildes, mode_t mode);
+      addToFunctionSummaryMap(
+          "fchmod",
+          Summary(ArgTypes{IntTy, *Mode_tTy}, RetType{IntTy}, NoEvalCall)
+              .ArgConstraint(
+                  ArgumentCondition(0, WithinRange, Range(0, IntMax))));
+    }
+
+    Optional<QualType> Uid_tTy = lookupType("uid_t", ACtx);
+    Optional<QualType> Gid_tTy = lookupType("gid_t", ACtx);
+
+    if (Uid_tTy && Gid_tTy) {
+      // int fchownat(int dirfd, const char *pathname, uid_t owner, gid_t group,
+      //              int flags);
+      addToFunctionSummaryMap(
+          "fchownat",
+          Summary(ArgTypes{IntTy, ConstCharPtrTy, *Uid_tTy, *Gid_tTy, IntTy},
+                  RetType{IntTy}, NoEvalCall)
+              .ArgConstraint(
+                  ArgumentCondition(0, WithinRange, Range(0, IntMax)))
+              .ArgConstraint(NotNull(ArgNo(1))));
+
+      // int chown(const char *path, uid_t owner, gid_t group);
+      addToFunctionSummaryMap(
+          "chown", Summary(ArgTypes{ConstCharPtrTy, *Uid_tTy, *Gid_tTy},
+                           RetType{IntTy}, NoEvalCall)
+                       .ArgConstraint(NotNull(ArgNo(0))));
+
+      // int lchown(const char *path, uid_t owner, gid_t group);
+      addToFunctionSummaryMap(
+          "lchown", Summary(ArgTypes{ConstCharPtrTy, *Uid_tTy, *Gid_tTy},
+                            RetType{IntTy}, NoEvalCall)
+                        .ArgConstraint(NotNull(ArgNo(0))));
+
+      // int fchown(int fildes, uid_t owner, gid_t group);
+      addToFunctionSummaryMap(
+          "fchown", Summary(ArgTypes{IntTy, *Uid_tTy, *Gid_tTy}, RetType{IntTy},
+                            NoEvalCall)
+                        .ArgConstraint(ArgumentCondition(0, WithinRange,
+                                                         Range(0, IntMax))));
+    }
+
+    // int rmdir(const char *pathname);
+    addToFunctionSummaryMap(
+        "rmdir", Summary(ArgTypes{ConstCharPtrTy}, RetType{IntTy}, NoEvalCall)
+                     .ArgConstraint(NotNull(ArgNo(0))));
+
+    // int chdir(const char *path);
+    addToFunctionSummaryMap(
+        "chdir", Summary(ArgTypes{ConstCharPtrTy}, RetType{IntTy}, NoEvalCall)
+                     .ArgConstraint(NotNull(ArgNo(0))));
+
+    // int link(const char *oldpath, const char *newpath);
+    addToFunctionSummaryMap("link",
+                            Summary(ArgTypes{ConstCharPtrTy, ConstCharPtrTy},
+                                    RetType{IntTy}, NoEvalCall)
+                                .ArgConstraint(NotNull(ArgNo(0)))
+                                .ArgConstraint(NotNull(ArgNo(1))));
+
+    // int linkat(int fd1, const char *path1, int fd2, const char *path2,
+    //            int flag);
+    addToFunctionSummaryMap(
+        "linkat",
+        Summary(ArgTypes{IntTy, ConstCharPtrTy, IntTy, ConstCharPtrTy, IntTy},
+                RetType{IntTy}, NoEvalCall)
+            .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
+            .ArgConstraint(NotNull(ArgNo(1)))
+            .ArgConstraint(ArgumentCondition(2, WithinRange, Range(0, IntMax)))
+            .ArgConstraint(NotNull(ArgNo(3))));
+
+    // int unlink(const char *pathname);
+    addToFunctionSummaryMap(
+        "unlink", Summary(ArgTypes{ConstCharPtrTy}, RetType{IntTy}, NoEvalCall)
+                      .ArgConstraint(NotNull(ArgNo(0))));
+
+    // int unlinkat(int fd, const char *path, int flag);
+    addToFunctionSummaryMap(
+        "unlinkat",
+        Summary(ArgTypes{IntTy, ConstCharPtrTy, IntTy}, RetType{IntTy},
+                NoEvalCall)
+            .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
+            .ArgConstraint(NotNull(ArgNo(1))));
+
+    Optional<QualType> StructStatTy = lookupType("stat", ACtx);
+    Optional<QualType> StructStatPtrTy, StructStatPtrRestrictTy;
+    if (StructStatTy) {
+      StructStatPtrTy = ACtx.getPointerType(*StructStatTy);
+      StructStatPtrRestrictTy = ACtx.getLangOpts().C99
+                                    ? ACtx.getRestrictType(*StructStatPtrTy)
+                                    : *StructStatPtrTy;
+    }
+
+    if (StructStatPtrTy)
+      // int fstat(int fd, struct stat *statbuf);
+      addToFunctionSummaryMap(
+          "fstat",
+          Summary(ArgTypes{IntTy, *StructStatPtrTy}, RetType{IntTy}, NoEvalCall)
+              .ArgConstraint(
+                  ArgumentCondition(0, WithinRange, Range(0, IntMax)))
+              .ArgConstraint(NotNull(ArgNo(1))));
+
+    if (StructStatPtrRestrictTy) {
+      // int stat(const char *restrict path, struct stat *restrict buf);
+      addToFunctionSummaryMap(
+          "stat",
+          Summary(ArgTypes{ConstCharPtrRestrictTy, *StructStatPtrRestrictTy},
+                  RetType{IntTy}, NoEvalCall)
+              .ArgConstraint(NotNull(ArgNo(0)))
+              .ArgConstraint(NotNull(ArgNo(1))));
+
+      // int lstat(const char *restrict path, struct stat *restrict buf);
+      addToFunctionSummaryMap(
+          "lstat",
+          Summary(ArgTypes{ConstCharPtrRestrictTy, *StructStatPtrRestrictTy},
+                  RetType{IntTy}, NoEvalCall)
+              .ArgConstraint(NotNull(ArgNo(0)))
+              .ArgConstraint(NotNull(ArgNo(1))));
+
+      // int fstatat(int fd, const char *restrict path,
+      //             struct stat *restrict buf, int flag);
+      addToFunctionSummaryMap(
+          "fstatat", Summary(ArgTypes{IntTy, ConstCharPtrRestrictTy,
+                                      *StructStatPtrRestrictTy, IntTy},
+                             RetType{IntTy}, NoEvalCall)
+                         .ArgConstraint(ArgumentCondition(0, WithinRange,
+                                                          Range(0, IntMax)))
+                         .ArgConstraint(NotNull(ArgNo(1)))
+                         .ArgConstraint(NotNull(ArgNo(2))));
+    }
+
+    if (DirPtrTy) {
+      // DIR *opendir(const char *name);
+      addToFunctionSummaryMap("opendir", Summary(ArgTypes{ConstCharPtrTy},
+                                                 RetType{*DirPtrTy}, NoEvalCall)
+                                             .ArgConstraint(NotNull(ArgNo(0))));
+
+      // DIR *fdopendir(int fd);
+      addToFunctionSummaryMap(
+          "fdopendir", Summary(ArgTypes{IntTy}, RetType{*DirPtrTy}, NoEvalCall)
+                           .ArgConstraint(ArgumentCondition(0, WithinRange,
+                                                            Range(0, IntMax))));
+    }
+
+    // int isatty(int fildes);
+    addToFunctionSummaryMap(
+        "isatty", Summary(ArgTypes{IntTy}, RetType{IntTy}, NoEvalCall)
+                      .ArgConstraint(
+                          ArgumentCondition(0, WithinRange, Range(0, IntMax))));
+
+    if (FilePtrTy) {
+      // FILE *popen(const char *command, const char *type);
+      addToFunctionSummaryMap("popen",
+                              Summary(ArgTypes{ConstCharPtrTy, ConstCharPtrTy},
+                                      RetType{*FilePtrTy}, NoEvalCall)
+                                  .ArgConstraint(NotNull(ArgNo(0)))
+                                  .ArgConstraint(NotNull(ArgNo(1))));
+
+      // int pclose(FILE *stream);
+      addToFunctionSummaryMap(
+          "pclose", Summary(ArgTypes{*FilePtrTy}, RetType{IntTy}, NoEvalCall)
+                        .ArgConstraint(NotNull(ArgNo(0))));
+    }
+
+    // int close(int fildes);
+    addToFunctionSummaryMap(
+        "close", Summary(ArgTypes{IntTy}, RetType{IntTy}, NoEvalCall)
+                     .ArgConstraint(
+                         ArgumentCondition(0, WithinRange, Range(0, IntMax))));
+
+    // long fpathconf(int fildes, int name);
+    addToFunctionSummaryMap(
+        "fpathconf",
+        Summary(ArgTypes{IntTy, IntTy}, RetType{LongTy}, NoEvalCall)
+            .ArgConstraint(
+                ArgumentCondition(0, WithinRange, Range(0, IntMax))));
+
+    // long pathconf(const char *path, int name);
+    addToFunctionSummaryMap("pathconf", Summary(ArgTypes{ConstCharPtrTy, IntTy},
+                                                RetType{LongTy}, NoEvalCall)
+                                            .ArgConstraint(NotNull(ArgNo(0))));
+
+    if (FilePtrTy)
+      // FILE *fdopen(int fd, const char *mode);
+      addToFunctionSummaryMap(
+          "fdopen", Summary(ArgTypes{IntTy, ConstCharPtrTy},
+                            RetType{*FilePtrTy}, NoEvalCall)
+                        .ArgConstraint(
+                            ArgumentCondition(0, WithinRange, Range(0, IntMax)))
+                        .ArgConstraint(NotNull(ArgNo(1))));
+
+    if (DirPtrTy) {
+      // void rewinddir(DIR *dir);
+      addToFunctionSummaryMap(
+          "rewinddir", Summary(ArgTypes{*DirPtrTy}, RetType{VoidTy}, NoEvalCall)
+                           .ArgConstraint(NotNull(ArgNo(0))));
+
+      // void seekdir(DIR *dirp, long loc);
+      addToFunctionSummaryMap("seekdir", Summary(ArgTypes{*DirPtrTy, LongTy},
+                                                 RetType{VoidTy}, NoEvalCall)
+                                             .ArgConstraint(NotNull(ArgNo(0))));
+    }
+
+    // int rand_r(unsigned int *seedp);
+    addToFunctionSummaryMap("rand_r", Summary(ArgTypes{UnsignedIntPtrTy},
+                                              RetType{IntTy}, NoEvalCall)
+                                          .ArgConstraint(NotNull(ArgNo(0))));
+
+    // int strcasecmp(const char *s1, const char *s2);
+    addToFunctionSummaryMap("strcasecmp",
+                            Summary(ArgTypes{ConstCharPtrTy, ConstCharPtrTy},
+                                    RetType{IntTy}, EvalCallAsPure)
+                                .ArgConstraint(NotNull(ArgNo(0)))
+                                .ArgConstraint(NotNull(ArgNo(1))));
+
+    // int strncasecmp(const char *s1, const char *s2, size_t n);
+    addToFunctionSummaryMap(
+        "strncasecmp", Summary(ArgTypes{ConstCharPtrTy, ConstCharPtrTy, SizeTy},
+                               RetType{IntTy}, EvalCallAsPure)
+                           .ArgConstraint(NotNull(ArgNo(0)))
+                           .ArgConstraint(NotNull(ArgNo(1)))
+                           .ArgConstraint(ArgumentCondition(
+                               2, WithinRange, Range(0, SizeMax))));
+
+    if (FilePtrTy && Off_tTy) {
+
+      // int fileno(FILE *stream);
+      addToFunctionSummaryMap(
+          "fileno", Summary(ArgTypes{*FilePtrTy}, RetType{IntTy}, NoEvalCall)
+                        .ArgConstraint(NotNull(ArgNo(0))));
+
+      // int fseeko(FILE *stream, off_t offset, int whence);
+      addToFunctionSummaryMap("fseeko",
+                              Summary(ArgTypes{*FilePtrTy, *Off_tTy, IntTy},
+                                      RetType{IntTy}, NoEvalCall)
+                                  .ArgConstraint(NotNull(ArgNo(0))));
+
+      // off_t ftello(FILE *stream);
+      addToFunctionSummaryMap(
+          "ftello", Summary(ArgTypes{*FilePtrTy}, RetType{*Off_tTy}, NoEvalCall)
+                        .ArgConstraint(NotNull(ArgNo(0))));
+    }
+
+    if (Off_tTy) {
+      Optional<RangeInt> Off_tMax = BVF.getMaxValue(*Off_tTy).getLimitedValue();
+
+      // void *mmap(void *addr, size_t length, int prot, int flags, int fd,
+      // off_t offset);
+      addToFunctionSummaryMap(
+          "mmap",
+          Summary(ArgTypes{VoidPtrTy, SizeTy, IntTy, IntTy, IntTy, *Off_tTy},
+                  RetType{VoidPtrTy}, NoEvalCall)
+              .ArgConstraint(
+                  ArgumentCondition(1, WithinRange, Range(1, SizeMax)))
+              .ArgConstraint(
+                  ArgumentCondition(4, WithinRange, Range(0, *Off_tMax))));
+    }
+
+    Optional<QualType> Off64_tTy = lookupType("off64_t", ACtx);
+    Optional<RangeInt> Off64_tMax;
+    if (Off64_tTy) {
+      Off64_tMax = BVF.getMaxValue(*Off_tTy).getLimitedValue();
+      // void *mmap64(void *addr, size_t length, int prot, int flags, int fd,
+      // off64_t offset);
+      addToFunctionSummaryMap(
+          "mmap64",
+          Summary(ArgTypes{VoidPtrTy, SizeTy, IntTy, IntTy, IntTy, *Off64_tTy},
+                  RetType{VoidPtrTy}, NoEvalCall)
+              .ArgConstraint(
+                  ArgumentCondition(1, WithinRange, Range(1, SizeMax)))
+              .ArgConstraint(
+                  ArgumentCondition(4, WithinRange, Range(0, *Off64_tMax))));
+    }
+
+    // int pipe(int fildes[2]);
+    addToFunctionSummaryMap(
+        "pipe", Summary(ArgTypes{IntPtrTy}, RetType{IntTy}, NoEvalCall)
+                    .ArgConstraint(NotNull(ArgNo(0))));
+
+    if (Off_tTy)
+      // off_t lseek(int fildes, off_t offset, int whence);
+      addToFunctionSummaryMap(
+          "lseek", Summary(ArgTypes{IntTy, *Off_tTy, IntTy}, RetType{*Off_tTy},
+                           NoEvalCall)
+                       .ArgConstraint(ArgumentCondition(0, WithinRange,
+                                                        Range(0, IntMax))));
+
+    Optional<QualType> Ssize_tTy = lookupType("ssize_t", ACtx);
+
+    if (Ssize_tTy) {
+      // ssize_t readlink(const char *restrict path, char *restrict buf,
+      //                  size_t bufsize);
+      addToFunctionSummaryMap(
+          "readlink",
+          Summary(ArgTypes{ConstCharPtrRestrictTy, CharPtrRestrictTy, SizeTy},
+                  RetType{*Ssize_tTy}, NoEvalCall)
+              .ArgConstraint(NotNull(ArgNo(0)))
+              .ArgConstraint(NotNull(ArgNo(1)))
+              .ArgConstraint(BufferSize(/*Buffer=*/ArgNo(1),
+                                        /*BufSize=*/ArgNo(2)))
+              .ArgConstraint(
+                  ArgumentCondition(2, WithinRange, Range(0, SizeMax))));
+
+      // ssize_t readlinkat(int fd, const char *restrict path,
+      //                    char *restrict buf, size_t bufsize);
+      addToFunctionSummaryMap(
+          "readlinkat", Summary(ArgTypes{IntTy, ConstCharPtrRestrictTy,
+                                         CharPtrRestrictTy, SizeTy},
+                                RetType{*Ssize_tTy}, NoEvalCall)
+                            .ArgConstraint(ArgumentCondition(0, WithinRange,
+                                                             Range(0, IntMax)))
+                            .ArgConstraint(NotNull(ArgNo(1)))
+                            .ArgConstraint(NotNull(ArgNo(2)))
+                            .ArgConstraint(BufferSize(/*Buffer=*/ArgNo(2),
+                                                      /*BufSize=*/ArgNo(3)))
+                            .ArgConstraint(ArgumentCondition(
+                                3, WithinRange, Range(0, SizeMax))));
+    }
+
+    // int renameat(int olddirfd, const char *oldpath, int newdirfd, const char
+    // *newpath);
+    addToFunctionSummaryMap("renameat", Summary(ArgTypes{IntTy, ConstCharPtrTy,
+                                                         IntTy, ConstCharPtrTy},
+                                                RetType{IntTy}, NoEvalCall)
+                                            .ArgConstraint(NotNull(ArgNo(1)))
+                                            .ArgConstraint(NotNull(ArgNo(3))));
+
+    // char *realpath(const char *restrict file_name,
+    //                char *restrict resolved_name);
+    addToFunctionSummaryMap(
+        "realpath", Summary(ArgTypes{ConstCharPtrRestrictTy, CharPtrRestrictTy},
+                            RetType{CharPtrTy}, NoEvalCall)
+                        .ArgConstraint(NotNull(ArgNo(0))));
+
+    QualType CharPtrConstPtr = ACtx.getPointerType(CharPtrTy.withConst());
+
+    // int execv(const char *path, char *const argv[]);
+    addToFunctionSummaryMap("execv",
+                            Summary(ArgTypes{ConstCharPtrTy, CharPtrConstPtr},
+                                    RetType{IntTy}, NoEvalCall)
+                                .ArgConstraint(NotNull(ArgNo(0))));
+
+    // int execvp(const char *file, char *const argv[]);
+    addToFunctionSummaryMap("execvp",
+                            Summary(ArgTypes{ConstCharPtrTy, CharPtrConstPtr},
+                                    RetType{IntTy}, NoEvalCall)
+                                .ArgConstraint(NotNull(ArgNo(0))));
+
+    // int getopt(int argc, char * const argv[], const char *optstring);
+    addToFunctionSummaryMap(
+        "getopt",
+        Summary(ArgTypes{IntTy, CharPtrConstPtr, ConstCharPtrTy},
+                RetType{IntTy}, NoEvalCall)
+            .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
+            .ArgConstraint(NotNull(ArgNo(1)))
+            .ArgConstraint(NotNull(ArgNo(2))));
+  }
+
   // Functions for testing.
   if (ChecksEnabled[CK_StdCLibraryFunctionsTesterChecker]) {
     addToFunctionSummaryMap(
@@ -1151,6 +1745,8 @@ void ento::registerStdCLibraryFunctionsChecker(CheckerManager &mgr) {
   Checker->DisplayLoadedSummaries =
       mgr.getAnalyzerOptions().getCheckerBooleanOption(
           Checker, "DisplayLoadedSummaries");
+  Checker->ModelPOSIX =
+      mgr.getAnalyzerOptions().getCheckerBooleanOption(Checker, "ModelPOSIX");
 }
 
 bool ento::shouldRegisterStdCLibraryFunctionsChecker(const CheckerManager &mgr) {
diff --git a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
index 7e10b2aa43568..f6abbe4f8f03b 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
@@ -195,29 +195,16 @@ ProgramStateRef bindInt(uint64_t Value, ProgramStateRef State,
 
 class StreamChecker : public Checker<check::PreCall, eval::Call,
                                      check::DeadSymbols, check::PointerEscape> {
-  BuiltinBug BT_FileNull{this, "NULL stream pointer",
-                         "Stream pointer might be NULL."};
-  BuiltinBug BT_UseAfterClose{
-      this, "Closed stream",
-      "Stream might be already closed. Causes undefined behaviour."};
-  BuiltinBug BT_UseAfterOpenFailed{this, "Invalid stream",
-                                   "Stream might be invalid after "
-                                   "(re-)opening it has failed. "
-                                   "Can cause undefined behaviour."};
-  BuiltinBug BT_IndeterminatePosition{
-      this, "Invalid stream state",
-      "File position of the stream might be 'indeterminate' "
-      "after a failed operation. "
-      "Can cause undefined behavior."};
-  BuiltinBug BT_IllegalWhence{this, "Illegal whence argument",
-                              "The whence argument to fseek() should be "
-                              "SEEK_SET, SEEK_END, or SEEK_CUR."};
-  BuiltinBug BT_StreamEof{this, "Stream already in EOF",
-                          "Read function called when stream is in EOF state. "
-                          "Function has no effect."};
-  BuiltinBug BT_ResourceLeak{
-      this, "Resource Leak",
-      "Opened File never closed. Potential Resource leak."};
+  BugType BT_FileNull{this, "NULL stream pointer", "Stream handling error"};
+  BugType BT_UseAfterClose{this, "Closed stream", "Stream handling error"};
+  BugType BT_UseAfterOpenFailed{this, "Invalid stream",
+                                "Stream handling error"};
+  BugType BT_IndeterminatePosition{this, "Invalid stream state",
+                                   "Stream handling error"};
+  BugType BT_IllegalWhence{this, "Illegal whence argument",
+                           "Stream handling error"};
+  BugType BT_StreamEof{this, "Stream already in EOF", "Stream handling error"};
+  BugType BT_ResourceLeak{this, "Resource leak", "Stream handling error"};
 
 public:
   void checkPreCall(const CallEvent &Call, CheckerContext &C) const;
@@ -365,6 +352,33 @@ class StreamChecker : public Checker<check::PreCall, eval::Call,
 
     return FnDescriptions.lookup(Call);
   }
+
+  /// Generate a message for BugReporterVisitor if the stored symbol is
+  /// marked as interesting by the actual bug report.
+  struct NoteFn {
+    const CheckerNameRef CheckerName;
+    SymbolRef StreamSym;
+    std::string Message;
+
+    std::string operator()(PathSensitiveBugReport &BR) const {
+      if (BR.isInteresting(StreamSym) &&
+          CheckerName == BR.getBugType().getCheckerName())
+        return Message;
+
+      return "";
+    }
+  };
+
+  const NoteTag *constructNoteTag(CheckerContext &C, SymbolRef StreamSym,
+                                  const std::string &Message) const {
+    return C.getNoteTag(NoteFn{getCheckerName(), StreamSym, Message});
+  }
+
+  /// Searches for the ExplodedNode where the file descriptor was acquired for
+  /// StreamSym.
+  static const ExplodedNode *getAcquisitionSite(const ExplodedNode *N,
+                                                SymbolRef StreamSym,
+                                                CheckerContext &C);
 };
 
 } // end anonymous namespace
@@ -376,6 +390,27 @@ inline void assertStreamStateOpened(const StreamState *SS) {
          "Previous create of error node for non-opened stream failed?");
 }
 
+const ExplodedNode *StreamChecker::getAcquisitionSite(const ExplodedNode *N,
+                                                      SymbolRef StreamSym,
+                                                      CheckerContext &C) {
+  ProgramStateRef State = N->getState();
+  // When bug type is resource leak, exploded node N may not have state info
+  // for leaked file descriptor, but predecessor should have it.
+  if (!State->get<StreamMap>(StreamSym))
+    N = N->getFirstPred();
+
+  const ExplodedNode *Pred = N;
+  while (N) {
+    State = N->getState();
+    if (!State->get<StreamMap>(StreamSym))
+      return Pred;
+    Pred = N;
+    N = N->getFirstPred();
+  }
+
+  return nullptr;
+}
+
 void StreamChecker::checkPreCall(const CallEvent &Call,
                                  CheckerContext &C) const {
   const FnDescription *Desc = lookupFn(Call);
@@ -421,7 +456,8 @@ void StreamChecker::evalFopen(const FnDescription *Desc, const CallEvent &Call,
   StateNull =
       StateNull->set<StreamMap>(RetSym, StreamState::getOpenFailed(Desc));
 
-  C.addTransition(StateNotNull);
+  C.addTransition(StateNotNull,
+                  constructNoteTag(C, RetSym, "Stream opened here"));
   C.addTransition(StateNull);
 }
 
@@ -476,7 +512,8 @@ void StreamChecker::evalFreopen(const FnDescription *Desc,
   StateRetNull =
       StateRetNull->set<StreamMap>(StreamSym, StreamState::getOpenFailed(Desc));
 
-  C.addTransition(StateRetNotNull);
+  C.addTransition(StateRetNotNull,
+                  constructNoteTag(C, StreamSym, "Stream reopened here"));
   C.addTransition(StateRetNull);
 }
 
@@ -784,7 +821,7 @@ StreamChecker::ensureStreamNonNull(SVal StreamVal, CheckerContext &C,
   if (!StateNotNull && StateNull) {
     if (ExplodedNode *N = C.generateErrorNode(StateNull)) {
       C.emitReport(std::make_unique<PathSensitiveBugReport>(
-          BT_FileNull, BT_FileNull.getDescription(), N));
+          BT_FileNull, "Stream pointer might be NULL.", N));
     }
     return nullptr;
   }
@@ -809,7 +846,8 @@ ProgramStateRef StreamChecker::ensureStreamOpened(SVal StreamVal,
     ExplodedNode *N = C.generateErrorNode();
     if (N) {
       C.emitReport(std::make_unique<PathSensitiveBugReport>(
-          BT_UseAfterClose, BT_UseAfterClose.getDescription(), N));
+          BT_UseAfterClose,
+          "Stream might be already closed. Causes undefined behaviour.", N));
       return nullptr;
     }
 
@@ -824,7 +862,11 @@ ProgramStateRef StreamChecker::ensureStreamOpened(SVal StreamVal,
     ExplodedNode *N = C.generateErrorNode();
     if (N) {
       C.emitReport(std::make_unique<PathSensitiveBugReport>(
-          BT_UseAfterOpenFailed, BT_UseAfterOpenFailed.getDescription(), N));
+          BT_UseAfterOpenFailed,
+          "Stream might be invalid after "
+          "(re-)opening it has failed. "
+          "Can cause undefined behaviour.",
+          N));
       return nullptr;
     }
     return State;
@@ -835,6 +877,11 @@ ProgramStateRef StreamChecker::ensureStreamOpened(SVal StreamVal,
 
 ProgramStateRef StreamChecker::ensureNoFilePositionIndeterminate(
     SVal StreamVal, CheckerContext &C, ProgramStateRef State) const {
+  static const char *BugMessage =
+      "File position of the stream might be 'indeterminate' "
+      "after a failed operation. "
+      "Can cause undefined behavior.";
+
   SymbolRef Sym = StreamVal.getAsSymbol();
   if (!Sym)
     return State;
@@ -855,8 +902,7 @@ ProgramStateRef StreamChecker::ensureNoFilePositionIndeterminate(
         return nullptr;
 
       C.emitReport(std::make_unique<PathSensitiveBugReport>(
-          BT_IndeterminatePosition, BT_IndeterminatePosition.getDescription(),
-          N));
+          BT_IndeterminatePosition, BugMessage, N));
       return State->set<StreamMap>(
           Sym, StreamState::getOpened(SS->LastOperation, ErrorFEof, false));
     }
@@ -866,8 +912,7 @@ ProgramStateRef StreamChecker::ensureNoFilePositionIndeterminate(
     ExplodedNode *N = C.generateErrorNode(State);
     if (N)
       C.emitReport(std::make_unique<PathSensitiveBugReport>(
-          BT_IndeterminatePosition, BT_IndeterminatePosition.getDescription(),
-          N));
+          BT_IndeterminatePosition, BugMessage, N));
 
     return nullptr;
   }
@@ -888,7 +933,10 @@ StreamChecker::ensureFseekWhenceCorrect(SVal WhenceVal, CheckerContext &C,
 
   if (ExplodedNode *N = C.generateNonFatalErrorNode(State)) {
     C.emitReport(std::make_unique<PathSensitiveBugReport>(
-        BT_IllegalWhence, BT_IllegalWhence.getDescription(), N));
+        BT_IllegalWhence,
+        "The whence argument to fseek() should be "
+        "SEEK_SET, SEEK_END, or SEEK_CUR.",
+        N));
     return nullptr;
   }
 
@@ -899,7 +947,10 @@ void StreamChecker::reportFEofWarning(CheckerContext &C,
                                       ProgramStateRef State) const {
   if (ExplodedNode *N = C.generateNonFatalErrorNode(State)) {
     C.emitReport(std::make_unique<PathSensitiveBugReport>(
-        BT_StreamEof, BT_StreamEof.getDescription(), N));
+        BT_StreamEof,
+        "Read function called when stream is in EOF state. "
+        "Function has no effect.",
+        N));
     return;
   }
   C.addTransition(State);
@@ -921,8 +972,39 @@ void StreamChecker::checkDeadSymbols(SymbolReaper &SymReaper,
     if (!N)
       continue;
 
-    C.emitReport(std::make_unique<PathSensitiveBugReport>(
-        BT_ResourceLeak, BT_ResourceLeak.getDescription(), N));
+    // Do not warn for non-closed stream at program exit.
+    ExplodedNode *Pred = C.getPredecessor();
+    if (Pred && Pred->getCFGBlock() &&
+        Pred->getCFGBlock()->hasNoReturnElement())
+      continue;
+
+    // Resource leaks can result in multiple warning that describe the same kind
+    // of programming error:
+    //  void f() {
+    //    FILE *F = fopen("a.txt");
+    //    if (rand()) // state split
+    //      return; // warning
+    //  } // warning
+    // While this isn't necessarily true (leaking the same stream could result
+    // from a different kinds of errors), the reduction in redundant reports
+    // makes this a worthwhile heuristic.
+    // FIXME: Add a checker option to turn this uniqueing feature off.
+
+    const ExplodedNode *StreamOpenNode = getAcquisitionSite(N, Sym, C);
+    assert(StreamOpenNode && "Could not find place of stream opening.");
+    PathDiagnosticLocation LocUsedForUniqueing =
+        PathDiagnosticLocation::createBegin(
+            StreamOpenNode->getStmtForDiagnostics(), C.getSourceManager(),
+            StreamOpenNode->getLocationContext());
+
+    std::unique_ptr<PathSensitiveBugReport> R =
+        std::make_unique<PathSensitiveBugReport>(
+            BT_ResourceLeak,
+            "Opened stream never closed. Potential resource leak.", N,
+            LocUsedForUniqueing,
+            StreamOpenNode->getLocationContext()->getDecl());
+    R->markInteresting(Sym);
+    C.emitReport(std::move(R));
   }
 }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/UninitializedObject/UninitializedObjectChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UninitializedObject/UninitializedObjectChecker.cpp
index e0be718decb29..4182b51c02b08 100644
--- a/clang/lib/StaticAnalyzer/Checkers/UninitializedObject/UninitializedObjectChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/UninitializedObject/UninitializedObjectChecker.cpp
@@ -541,14 +541,11 @@ static bool hasUnguardedAccess(const FieldDecl *FD, ProgramStateRef State) {
   auto FieldAccessM = memberExpr(hasDeclaration(equalsNode(FD))).bind("access");
 
   auto AssertLikeM = callExpr(callee(functionDecl(
-      anyOf(hasName("exit"), hasName("panic"), hasName("error"),
-            hasName("Assert"), hasName("assert"), hasName("ziperr"),
-            hasName("assfail"), hasName("db_error"), hasName("__assert"),
-            hasName("__assert2"), hasName("_wassert"), hasName("__assert_rtn"),
-            hasName("__assert_fail"), hasName("dtrace_assfail"),
-            hasName("yy_fatal_error"), hasName("_XCAssertionFailureHandler"),
-            hasName("_DTAssertionFailureHandler"),
-            hasName("_TSAssertionFailureHandler")))));
+      hasAnyName("exit", "panic", "error", "Assert", "assert", "ziperr",
+                 "assfail", "db_error", "__assert", "__assert2", "_wassert",
+                 "__assert_rtn", "__assert_fail", "dtrace_assfail",
+                 "yy_fatal_error", "_XCAssertionFailureHandler",
+                 "_DTAssertionFailureHandler", "_TSAssertionFailureHandler"))));
 
   auto NoReturnFuncM = callExpr(callee(functionDecl(isNoReturn())));
 
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
index efae511da83b4..72be4e81c83d6 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
@@ -34,10 +34,12 @@
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/CheckerRegistryData.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SVals.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -2106,6 +2108,53 @@ void BuiltinBug::anchor() {}
 // Methods for BugReport and subclasses.
 //===----------------------------------------------------------------------===//
 
+LLVM_ATTRIBUTE_USED static bool
+isDependency(const CheckerRegistryData &Registry, StringRef CheckerName) {
+  for (const std::pair<StringRef, StringRef> &Pair : Registry.Dependencies) {
+    if (Pair.second == CheckerName)
+      return true;
+  }
+  return false;
+}
+
+LLVM_ATTRIBUTE_USED static bool isHidden(const CheckerRegistryData &Registry,
+                                         StringRef CheckerName) {
+  for (const CheckerInfo &Checker : Registry.Checkers) {
+    if (Checker.FullName == CheckerName)
+      return Checker.IsHidden;
+  }
+  llvm_unreachable(
+      "Checker name not found in CheckerRegistry -- did you retrieve it "
+      "correctly from CheckerManager::getCurrentCheckerName?");
+}
+
+PathSensitiveBugReport::PathSensitiveBugReport(
+    const BugType &bt, StringRef shortDesc, StringRef desc,
+    const ExplodedNode *errorNode, PathDiagnosticLocation LocationToUnique,
+    const Decl *DeclToUnique)
+    : BugReport(Kind::PathSensitive, bt, shortDesc, desc), ErrorNode(errorNode),
+      ErrorNodeRange(getStmt() ? getStmt()->getSourceRange() : SourceRange()),
+      UniqueingLocation(LocationToUnique), UniqueingDecl(DeclToUnique) {
+  assert(!isDependency(ErrorNode->getState()
+                           ->getAnalysisManager()
+                           .getCheckerManager()
+                           ->getCheckerRegistryData(),
+                       bt.getCheckerName()) &&
+         "Some checkers depend on this one! We don't allow dependency "
+         "checkers to emit warnings, because checkers should depend on "
+         "*modeling*, not *diagnostics*.");
+
+  assert(
+      (bt.getCheckerName().startswith("debug") ||
+       !isHidden(ErrorNode->getState()
+                     ->getAnalysisManager()
+                     .getCheckerManager()
+                     ->getCheckerRegistryData(),
+                 bt.getCheckerName())) &&
+          "Hidden checkers musn't emit diagnostics as they are by definition "
+          "non-user facing!");
+}
+
 void PathSensitiveBugReport::addVisitor(
     std::unique_ptr<BugReporterVisitor> visitor) {
   if (!visitor)
@@ -2194,12 +2243,12 @@ static void insertToInterestingnessMap(
       return;
     case bugreporter::TrackingKind::Condition:
       return;
-  }
+    }
 
-  llvm_unreachable(
-      "BugReport::markInteresting currently can only handle 2 different "
-      "tracking kinds! Please define what tracking kind should this entitiy"
-      "have, if it was already marked as interesting with a different kind!");
+    llvm_unreachable(
+        "BugReport::markInteresting currently can only handle 2 different "
+        "tracking kinds! Please define what tracking kind should this entitiy"
+        "have, if it was already marked as interesting with a different kind!");
 }
 
 void PathSensitiveBugReport::markInteresting(SymbolRef sym,
@@ -2783,7 +2832,7 @@ Optional<PathDiagnosticBuilder> PathDiagnosticBuilder::findValidReport(
         R->clearVisitors();
         R->addVisitor(std::make_unique<FalsePositiveRefutationBRVisitor>());
 
-        // We don't overrite the notes inserted by other visitors because the
+        // We don't overwrite the notes inserted by other visitors because the
         // refutation manager does not add any new note to the path
         generateVisitorsDiagnostics(R, BugPath->ErrorNode, BRC);
       }
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
index ad79f7cb9359f..ef4d38ff498ff 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
@@ -2819,7 +2819,7 @@ void FalsePositiveRefutationBRVisitor::finalizeVisitor(
     BugReporterContext &BRC, const ExplodedNode *EndPathNode,
     PathSensitiveBugReport &BR) {
   // Collect new constraints
-  VisitNode(EndPathNode, BRC, BR);
+  addConstraints(EndPathNode, /*OverwriteConstraintsOnExistingSyms=*/true);
 
   // Create a refutation manager
   llvm::SMTSolverRef RefutationSolver = llvm::CreateZ3Solver();
@@ -2830,30 +2830,30 @@ void FalsePositiveRefutationBRVisitor::finalizeVisitor(
     const SymbolRef Sym = I.first;
     auto RangeIt = I.second.begin();
 
-    llvm::SMTExprRef Constraints = SMTConv::getRangeExpr(
+    llvm::SMTExprRef SMTConstraints = SMTConv::getRangeExpr(
         RefutationSolver, Ctx, Sym, RangeIt->From(), RangeIt->To(),
         /*InRange=*/true);
     while ((++RangeIt) != I.second.end()) {
-      Constraints = RefutationSolver->mkOr(
-          Constraints, SMTConv::getRangeExpr(RefutationSolver, Ctx, Sym,
-                                             RangeIt->From(), RangeIt->To(),
-                                             /*InRange=*/true));
+      SMTConstraints = RefutationSolver->mkOr(
+          SMTConstraints, SMTConv::getRangeExpr(RefutationSolver, Ctx, Sym,
+                                                RangeIt->From(), RangeIt->To(),
+                                                /*InRange=*/true));
     }
 
-    RefutationSolver->addConstraint(Constraints);
+    RefutationSolver->addConstraint(SMTConstraints);
   }
 
   // And check for satisfiability
-  Optional<bool> isSat = RefutationSolver->check();
-  if (!isSat.hasValue())
+  Optional<bool> IsSAT = RefutationSolver->check();
+  if (!IsSAT.hasValue())
     return;
 
-  if (!isSat.getValue())
+  if (!IsSAT.getValue())
     BR.markInvalid("Infeasible constraints", EndPathNode->getLocationContext());
 }
 
-PathDiagnosticPieceRef FalsePositiveRefutationBRVisitor::VisitNode(
-    const ExplodedNode *N, BugReporterContext &, PathSensitiveBugReport &) {
+void FalsePositiveRefutationBRVisitor::addConstraints(
+    const ExplodedNode *N, bool OverwriteConstraintsOnExistingSyms) {
   // Collect new constraints
   const ConstraintRangeTy &NewCs = N->getState()->get<ConstraintRange>();
   ConstraintRangeTy::Factory &CF =
@@ -2863,10 +2863,19 @@ PathDiagnosticPieceRef FalsePositiveRefutationBRVisitor::VisitNode(
   for (auto const &C : NewCs) {
     const SymbolRef &Sym = C.first;
     if (!Constraints.contains(Sym)) {
+      // This symbol is new, just add the constraint.
+      Constraints = CF.add(Constraints, Sym, C.second);
+    } else if (OverwriteConstraintsOnExistingSyms) {
+      // Overwrite the associated constraint of the Symbol.
+      Constraints = CF.remove(Constraints, Sym);
       Constraints = CF.add(Constraints, Sym, C.second);
     }
   }
+}
 
+PathDiagnosticPieceRef FalsePositiveRefutationBRVisitor::VisitNode(
+    const ExplodedNode *N, BugReporterContext &, PathSensitiveBugReport &) {
+  addConstraints(N, /*OverwriteConstraintsOnExistingSyms=*/false);
   return nullptr;
 }
 
diff --git a/clang/lib/StaticAnalyzer/Core/CMakeLists.txt b/clang/lib/StaticAnalyzer/Core/CMakeLists.txt
index 233ffaf799568..f2329ae6042ef 100644
--- a/clang/lib/StaticAnalyzer/Core/CMakeLists.txt
+++ b/clang/lib/StaticAnalyzer/Core/CMakeLists.txt
@@ -16,6 +16,7 @@ add_clang_library(clangStaticAnalyzerCore
   CheckerContext.cpp
   CheckerHelpers.cpp
   CheckerManager.cpp
+  CheckerRegistryData.cpp
   CommonBugCategories.cpp
   ConstraintManager.cpp
   CoreEngine.cpp
@@ -60,5 +61,8 @@ add_clang_library(clangStaticAnalyzerCore
   clangLex
   clangRewrite
   clangToolingCore
+
+  DEPENDS
+  omp_gen
   )
 
diff --git a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
index b8d8d467341d8..78d13ddfb773c 100644
--- a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
@@ -523,7 +523,7 @@ CallEvent::getReturnValueUnderConstruction() const {
   if (!CC)
     return None;
 
-  ExprEngine::EvalCallOptions CallOpts;
+  EvalCallOptions CallOpts;
   ExprEngine &Engine = getState()->getStateManager().getOwningEngine();
   SVal RetVal =
     Engine.computeObjectUnderConstruction(getOriginExpr(), getState(),
diff --git a/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp b/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
index 98504006030a7..86cecf6524f03 100644
--- a/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
@@ -653,7 +653,8 @@ CheckerManager::runCheckersForEvalAssume(ProgramStateRef state,
 void CheckerManager::runCheckersForEvalCall(ExplodedNodeSet &Dst,
                                             const ExplodedNodeSet &Src,
                                             const CallEvent &Call,
-                                            ExprEngine &Eng) {
+                                            ExprEngine &Eng,
+                                            const EvalCallOptions &CallOpts) {
   for (auto *const Pred : Src) {
     bool anyEvaluated = false;
 
@@ -665,10 +666,8 @@ void CheckerManager::runCheckersForEvalCall(ExplodedNodeSet &Dst,
       // TODO: Support the situation when the call doesn't correspond
       // to any Expr.
       ProgramPoint L = ProgramPoint::getProgramPoint(
-          cast<CallExpr>(Call.getOriginExpr()),
-          ProgramPoint::PostStmtKind,
-          Pred->getLocationContext(),
-          EvalCallChecker.Checker);
+          Call.getOriginExpr(), ProgramPoint::PostStmtKind,
+          Pred->getLocationContext(), EvalCallChecker.Checker);
       bool evaluated = false;
       { // CheckerContext generates transitions(populates checkDest) on
         // destruction, so introduce the scope to make sure it gets properly
@@ -690,7 +689,7 @@ void CheckerManager::runCheckersForEvalCall(ExplodedNodeSet &Dst,
     // If none of the checkers evaluated the call, ask ExprEngine to handle it.
     if (!anyEvaluated) {
       NodeBuilder B(Pred, Dst, Eng.getBuilderContext());
-      Eng.defaultEvalCall(B, Pred, Call);
+      Eng.defaultEvalCall(B, Pred, Call, CallOpts);
     }
   }
 }
diff --git a/clang/lib/StaticAnalyzer/Core/CheckerRegistryData.cpp b/clang/lib/StaticAnalyzer/Core/CheckerRegistryData.cpp
new file mode 100644
index 0000000000000..1b3e8b11549dd
--- /dev/null
+++ b/clang/lib/StaticAnalyzer/Core/CheckerRegistryData.cpp
@@ -0,0 +1,241 @@
+//===- CheckerRegistry.h - Maintains all available checkers -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/CheckerRegistryData.h"
+#include "clang/StaticAnalyzer/Core/AnalyzerOptions.h"
+#include "llvm/ADT/Twine.h"
+#include <map>
+
+using namespace clang;
+using namespace ento;
+
+//===----------------------------------------------------------------------===//
+// Methods of CmdLineOption, PackageInfo and CheckerInfo.
+//===----------------------------------------------------------------------===//
+
+LLVM_DUMP_METHOD void CmdLineOption::dump() const {
+  dumpToStream(llvm::errs());
+}
+
+LLVM_DUMP_METHOD void
+CmdLineOption::dumpToStream(llvm::raw_ostream &Out) const {
+  // The description can be just checked in Checkers.inc, the point here is to
+  // debug whether we succeeded in parsing it.
+  Out << OptionName << " (" << OptionType << ", "
+      << (IsHidden ? "hidden, " : "") << DevelopmentStatus << ") default: \""
+      << DefaultValStr;
+}
+
+static StringRef toString(StateFromCmdLine Kind) {
+  switch (Kind) {
+  case StateFromCmdLine::State_Disabled:
+    return "Disabled";
+  case StateFromCmdLine::State_Enabled:
+    return "Enabled";
+  case StateFromCmdLine::State_Unspecified:
+    return "Unspecified";
+  }
+  llvm_unreachable("Unhandled StateFromCmdLine enum");
+}
+
+LLVM_DUMP_METHOD void CheckerInfo::dump() const { dumpToStream(llvm::errs()); }
+
+LLVM_DUMP_METHOD void CheckerInfo::dumpToStream(llvm::raw_ostream &Out) const {
+  // The description can be just checked in Checkers.inc, the point here is to
+  // debug whether we succeeded in parsing it. Same with documentation uri.
+  Out << FullName << " (" << toString(State) << (IsHidden ? ", hidden" : "")
+      << ")\n";
+  Out << "  Options:\n";
+  for (const CmdLineOption &Option : CmdLineOptions) {
+    Out << "    ";
+    Option.dumpToStream(Out);
+    Out << '\n';
+  }
+  Out << "  Dependencies:\n";
+  for (const CheckerInfo *Dependency : Dependencies) {
+    Out << "  " << Dependency->FullName << '\n';
+  }
+  Out << "  Weak dependencies:\n";
+  for (const CheckerInfo *Dependency : WeakDependencies) {
+    Out << "    " << Dependency->FullName << '\n';
+  }
+}
+
+LLVM_DUMP_METHOD void PackageInfo::dump() const { dumpToStream(llvm::errs()); }
+
+LLVM_DUMP_METHOD void PackageInfo::dumpToStream(llvm::raw_ostream &Out) const {
+  Out << FullName << "\n";
+  Out << "  Options:\n";
+  for (const CmdLineOption &Option : CmdLineOptions) {
+    Out << "    ";
+    Option.dumpToStream(Out);
+    Out << '\n';
+  }
+}
+
+static constexpr char PackageSeparator = '.';
+
+static bool isInPackage(const CheckerInfo &Checker, StringRef PackageName) {
+  // Does the checker's full name have the package as a prefix?
+  if (!Checker.FullName.startswith(PackageName))
+    return false;
+
+  // Is the package actually just the name of a specific checker?
+  if (Checker.FullName.size() == PackageName.size())
+    return true;
+
+  // Is the checker in the package (or a subpackage)?
+  if (Checker.FullName[PackageName.size()] == PackageSeparator)
+    return true;
+
+  return false;
+}
+
+CheckerInfoListRange
+CheckerRegistryData::getMutableCheckersForCmdLineArg(StringRef CmdLineArg) {
+  auto It = checker_registry::binaryFind(Checkers, CmdLineArg);
+
+  if (!isInPackage(*It, CmdLineArg))
+    return {Checkers.end(), Checkers.end()};
+
+  // See how large the package is.
+  // If the package doesn't exist, assume the option refers to a single
+  // checker.
+  size_t Size = 1;
+  llvm::StringMap<size_t>::const_iterator PackageSize =
+      PackageSizes.find(CmdLineArg);
+
+  if (PackageSize != PackageSizes.end())
+    Size = PackageSize->getValue();
+
+  return {It, It + Size};
+}
+//===----------------------------------------------------------------------===//
+// Printing functions.
+//===----------------------------------------------------------------------===//
+
+void CheckerRegistryData::printCheckerWithDescList(
+    const AnalyzerOptions &AnOpts, raw_ostream &Out,
+    size_t MaxNameChars) const {
+  // FIXME: Print available packages.
+
+  Out << "CHECKERS:\n";
+
+  // Find the maximum option length.
+  size_t OptionFieldWidth = 0;
+  for (const auto &Checker : Checkers) {
+    // Limit the amount of padding we are willing to give up for alignment.
+    //   Package.Name     Description  [Hidden]
+    size_t NameLength = Checker.FullName.size();
+    if (NameLength <= MaxNameChars)
+      OptionFieldWidth = std::max(OptionFieldWidth, NameLength);
+  }
+
+  const size_t InitialPad = 2;
+
+  auto Print = [=](llvm::raw_ostream &Out, const CheckerInfo &Checker,
+                   StringRef Description) {
+    AnalyzerOptions::printFormattedEntry(Out, {Checker.FullName, Description},
+                                         InitialPad, OptionFieldWidth);
+    Out << '\n';
+  };
+
+  for (const auto &Checker : Checkers) {
+    // The order of this if branches is significant, we wouldn't like to display
+    // developer checkers even in the alpha output. For example,
+    // alpha.cplusplus.IteratorModeling is a modeling checker, hence it's hidden
+    // by default, and users (even when the user is a developer of an alpha
+    // checker) shouldn't normally tinker with whether they should be enabled.
+
+    if (Checker.IsHidden) {
+      if (AnOpts.ShowCheckerHelpDeveloper)
+        Print(Out, Checker, Checker.Desc);
+      continue;
+    }
+
+    if (Checker.FullName.startswith("alpha")) {
+      if (AnOpts.ShowCheckerHelpAlpha)
+        Print(Out, Checker,
+              ("(Enable only for development!) " + Checker.Desc).str());
+      continue;
+    }
+
+    if (AnOpts.ShowCheckerHelp)
+      Print(Out, Checker, Checker.Desc);
+  }
+}
+
+void CheckerRegistryData::printEnabledCheckerList(raw_ostream &Out) const {
+  for (const auto *i : EnabledCheckers)
+    Out << i->FullName << '\n';
+}
+
+void CheckerRegistryData::printCheckerOptionList(const AnalyzerOptions &AnOpts,
+                                                 raw_ostream &Out) const {
+  Out << "OVERVIEW: Clang Static Analyzer Checker and Package Option List\n\n";
+  Out << "USAGE: -analyzer-config <OPTION1=VALUE,OPTION2=VALUE,...>\n\n";
+  Out << "       -analyzer-config OPTION1=VALUE, -analyzer-config "
+         "OPTION2=VALUE, ...\n\n";
+  Out << "OPTIONS:\n\n";
+
+  // It's usually ill-advised to use multimap, but clang will terminate after
+  // this function.
+  std::multimap<StringRef, const CmdLineOption &> OptionMap;
+
+  for (const CheckerInfo &Checker : Checkers) {
+    for (const CmdLineOption &Option : Checker.CmdLineOptions) {
+      OptionMap.insert({Checker.FullName, Option});
+    }
+  }
+
+  for (const PackageInfo &Package : Packages) {
+    for (const CmdLineOption &Option : Package.CmdLineOptions) {
+      OptionMap.insert({Package.FullName, Option});
+    }
+  }
+
+  auto Print = [](llvm::raw_ostream &Out, StringRef FullOption,
+                  StringRef Desc) {
+    AnalyzerOptions::printFormattedEntry(Out, {FullOption, Desc},
+                                         /*InitialPad*/ 2,
+                                         /*EntryWidth*/ 50,
+                                         /*MinLineWidth*/ 90);
+    Out << "\n\n";
+  };
+  for (const std::pair<const StringRef, const CmdLineOption &> &Entry :
+       OptionMap) {
+    const CmdLineOption &Option = Entry.second;
+    std::string FullOption = (Entry.first + ":" + Option.OptionName).str();
+
+    std::string Desc =
+        ("(" + Option.OptionType + ") " + Option.Description + " (default: " +
+         (Option.DefaultValStr.empty() ? "\"\"" : Option.DefaultValStr) + ")")
+            .str();
+
+    // The list of these if branches is significant, we wouldn't like to
+    // display hidden alpha checker options for
+    // -analyzer-checker-option-help-alpha.
+
+    if (Option.IsHidden) {
+      if (AnOpts.ShowCheckerOptionDeveloperList)
+        Print(Out, FullOption, Desc);
+      continue;
+    }
+
+    if (Option.DevelopmentStatus == "alpha" ||
+        Entry.first.startswith("alpha")) {
+      if (AnOpts.ShowCheckerOptionAlphaList)
+        Print(Out, FullOption,
+              llvm::Twine("(Enable only for development!) " + Desc).str());
+      continue;
+    }
+
+    if (AnOpts.ShowCheckerOptionList)
+      Print(Out, FullOption, Desc);
+  }
+}
diff --git a/clang/lib/StaticAnalyzer/Core/Environment.cpp b/clang/lib/StaticAnalyzer/Core/Environment.cpp
index 1ccf4c6104a65..9e6d79bb7dcc9 100644
--- a/clang/lib/StaticAnalyzer/Core/Environment.cpp
+++ b/clang/lib/StaticAnalyzer/Core/Environment.cpp
@@ -183,12 +183,18 @@ EnvironmentManager::removeDeadBindings(Environment Env,
              F.getTreeFactory());
 
   // Iterate over the block-expr bindings.
-  for (Environment::iterator I = Env.begin(), E = Env.end();
-       I != E; ++I) {
+  for (Environment::iterator I = Env.begin(), E = Env.end(); I != E; ++I) {
     const EnvironmentEntry &BlkExpr = I.getKey();
     const SVal &X = I.getData();
 
-    if (SymReaper.isLive(BlkExpr.getStmt(), BlkExpr.getLocationContext())) {
+    const bool IsBlkExprLive =
+        SymReaper.isLive(BlkExpr.getStmt(), BlkExpr.getLocationContext());
+
+    assert((isa<Expr>(BlkExpr.getStmt()) || !IsBlkExprLive) &&
+           "Only Exprs can be live, LivenessAnalysis argues about the liveness "
+           "of *values*!");
+
+    if (IsBlkExprLive) {
       // Copy the binding to the new map.
       EBMapRef = EBMapRef.add(BlkExpr, X);
 
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
index ba2875aab7543..38a680eb04c00 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
@@ -615,7 +615,8 @@ void ExprEngine::handleConstructor(const Expr *E,
   } else {
     for (ExplodedNodeSet::iterator I = DstPreCall.begin(), E = DstPreCall.end();
          I != E; ++I)
-      defaultEvalCall(Bldr, *I, *Call, CallOpts);
+      getCheckerManager().runCheckersForEvalCall(DstEvaluated, *I, *Call, *this,
+                                                 CallOpts);
   }
 
   // If the CFG was constructed without elements for temporary destructors
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
index da7b5d9efa471..52ba17d59ae07 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
@@ -584,7 +584,7 @@ void ExprEngine::evalCall(ExplodedNodeSet &Dst, ExplodedNode *Pred,
   // defaultEvalCall if all of them fail.
   ExplodedNodeSet dstCallEvaluated;
   getCheckerManager().runCheckersForEvalCall(dstCallEvaluated, dstPreVisit,
-                                             Call, *this);
+                                             Call, *this, EvalCallOptions());
 
   // If there were other constructors called for object-type arguments
   // of this call, clean them up.
@@ -717,7 +717,7 @@ void ExprEngine::conservativeEvalCall(const CallEvent &Call, NodeBuilder &Bldr,
 ExprEngine::CallInlinePolicy
 ExprEngine::mayInlineCallKind(const CallEvent &Call, const ExplodedNode *Pred,
                               AnalyzerOptions &Opts,
-                              const ExprEngine::EvalCallOptions &CallOpts) {
+                              const EvalCallOptions &CallOpts) {
   const LocationContext *CurLC = Pred->getLocationContext();
   const StackFrameContext *CallerSFC = CurLC->getStackFrame();
   switch (Call.getKind()) {
diff --git a/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp b/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
index 08413c080d41c..cb6f61e86ae3b 100644
--- a/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
+++ b/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
@@ -672,7 +672,7 @@ class SymbolicRangeInferrer
     if (!BinaryOperator::isComparisonOp(CurrentOP) || (CurrentOP == BO_Cmp))
       return EmptyRangeSet;
 
-    static const OperatorRelationsTable CmpOpTable;
+    static const OperatorRelationsTable CmpOpTable{};
 
     const SymExpr *LHS = SSE->getLHS();
     const SymExpr *RHS = SSE->getRHS();
diff --git a/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp b/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
index 74c689730e58d..392049e21c6e5 100644
--- a/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
+++ b/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
@@ -154,7 +154,7 @@ class AnalysisConsumer : public AnalysisASTConsumer,
     break;
 #include "clang/StaticAnalyzer/Core/Analyses.def"
     default:
-      llvm_unreachable("Unkown analyzer output type!");
+      llvm_unreachable("Unknown analyzer output type!");
     }
 
     // Create the analyzer component creators.
diff --git a/clang/lib/StaticAnalyzer/Frontend/AnalyzerHelpFlags.cpp b/clang/lib/StaticAnalyzer/Frontend/AnalyzerHelpFlags.cpp
index d589e69bdf347..eb6014a0629df 100644
--- a/clang/lib/StaticAnalyzer/Frontend/AnalyzerHelpFlags.cpp
+++ b/clang/lib/StaticAnalyzer/Frontend/AnalyzerHelpFlags.cpp
@@ -33,7 +33,8 @@ void ento::printCheckerHelp(raw_ostream &out, CompilerInstance &CI) {
       *CI.getAnalyzerOpts(), CI.getLangOpts(), CI.getDiagnostics(),
       CI.getFrontendOpts().Plugins);
 
-  CheckerMgr->getCheckerRegistry().printCheckerWithDescList(out);
+  CheckerMgr->getCheckerRegistryData().printCheckerWithDescList(
+      *CI.getAnalyzerOpts(), out);
 }
 
 void ento::printEnabledCheckerList(raw_ostream &out, CompilerInstance &CI) {
@@ -43,7 +44,7 @@ void ento::printEnabledCheckerList(raw_ostream &out, CompilerInstance &CI) {
       *CI.getAnalyzerOpts(), CI.getLangOpts(), CI.getDiagnostics(),
       CI.getFrontendOpts().Plugins);
 
-  CheckerMgr->getCheckerRegistry().printEnabledCheckerList(out);
+  CheckerMgr->getCheckerRegistryData().printEnabledCheckerList(out);
 }
 
 void ento::printCheckerConfigList(raw_ostream &out, CompilerInstance &CI) {
@@ -52,7 +53,8 @@ void ento::printCheckerConfigList(raw_ostream &out, CompilerInstance &CI) {
       *CI.getAnalyzerOpts(), CI.getLangOpts(), CI.getDiagnostics(),
       CI.getFrontendOpts().Plugins);
 
-  CheckerMgr->getCheckerRegistry().printCheckerOptionList(out);
+  CheckerMgr->getCheckerRegistryData().printCheckerOptionList(
+      *CI.getAnalyzerOpts(), out);
 }
 
 void ento::printAnalyzerConfigList(raw_ostream &out) {
diff --git a/clang/lib/StaticAnalyzer/Frontend/CMakeLists.txt b/clang/lib/StaticAnalyzer/Frontend/CMakeLists.txt
index 0c100a121f777..5293f5e0a522d 100644
--- a/clang/lib/StaticAnalyzer/Frontend/CMakeLists.txt
+++ b/clang/lib/StaticAnalyzer/Frontend/CMakeLists.txt
@@ -23,4 +23,7 @@ add_clang_library(clangStaticAnalyzerFrontend
   clangLex
   clangStaticAnalyzerCheckers
   clangStaticAnalyzerCore
+
+  DEPENDS
+  omp_gen
   )
diff --git a/clang/lib/StaticAnalyzer/Frontend/CheckerRegistry.cpp b/clang/lib/StaticAnalyzer/Frontend/CheckerRegistry.cpp
index e690ed849e40d..528284ca89858 100644
--- a/clang/lib/StaticAnalyzer/Frontend/CheckerRegistry.cpp
+++ b/clang/lib/StaticAnalyzer/Frontend/CheckerRegistry.cpp
@@ -25,14 +25,13 @@
 
 using namespace clang;
 using namespace ento;
+using namespace checker_registry;
 using llvm::sys::DynamicLibrary;
 
 //===----------------------------------------------------------------------===//
 // Utilities.
 //===----------------------------------------------------------------------===//
 
-using RegisterCheckersFn = void (*)(CheckerRegistry &);
-
 static bool isCompatibleAPIVersion(const char *VersionString) {
   // If the version string is null, its not an analyzer plugin.
   if (!VersionString)
@@ -43,140 +42,17 @@ static bool isCompatibleAPIVersion(const char *VersionString) {
   return strcmp(VersionString, CLANG_ANALYZER_API_VERSION_STRING) == 0;
 }
 
-namespace {
-template <class T> struct FullNameLT {
-  bool operator()(const T &Lhs, const T &Rhs) {
-    return Lhs.FullName < Rhs.FullName;
-  }
-};
-
-using PackageNameLT = FullNameLT<CheckerRegistry::PackageInfo>;
-using CheckerNameLT = FullNameLT<CheckerRegistry::CheckerInfo>;
-} // end of anonymous namespace
-
-template <class CheckerOrPackageInfoList>
-static std::conditional_t<std::is_const<CheckerOrPackageInfoList>::value,
-                          typename CheckerOrPackageInfoList::const_iterator,
-                          typename CheckerOrPackageInfoList::iterator>
-binaryFind(CheckerOrPackageInfoList &Collection, StringRef FullName) {
-
-  using CheckerOrPackage = typename CheckerOrPackageInfoList::value_type;
-  using CheckerOrPackageFullNameLT = FullNameLT<CheckerOrPackage>;
-
-  assert(llvm::is_sorted(Collection, CheckerOrPackageFullNameLT{}) &&
-         "In order to efficiently gather checkers/packages, this function "
-         "expects them to be already sorted!");
-
-  return llvm::lower_bound(Collection, CheckerOrPackage(FullName),
-                           CheckerOrPackageFullNameLT{});
-}
-
 static constexpr char PackageSeparator = '.';
 
-static bool isInPackage(const CheckerRegistry::CheckerInfo &Checker,
-                        StringRef PackageName) {
-  // Does the checker's full name have the package as a prefix?
-  if (!Checker.FullName.startswith(PackageName))
-    return false;
-
-  // Is the package actually just the name of a specific checker?
-  if (Checker.FullName.size() == PackageName.size())
-    return true;
-
-  // Is the checker in the package (or a subpackage)?
-  if (Checker.FullName[PackageName.size()] == PackageSeparator)
-    return true;
-
-  return false;
-}
-
-//===----------------------------------------------------------------------===//
-// Methods of CmdLineOption, PackageInfo and CheckerInfo.
-//===----------------------------------------------------------------------===//
-
-LLVM_DUMP_METHOD void
-CheckerRegistry::CmdLineOption::dumpToStream(llvm::raw_ostream &Out) const {
-  // The description can be just checked in Checkers.inc, the point here is to
-  // debug whether we succeeded in parsing it.
-  Out << OptionName << " (" << OptionType << ", "
-      << (IsHidden ? "hidden, " : "") << DevelopmentStatus << ") default: \""
-      << DefaultValStr;
-}
-
-static StringRef toString(CheckerRegistry::StateFromCmdLine Kind) {
-  switch (Kind) {
-  case CheckerRegistry::StateFromCmdLine::State_Disabled:
-    return "Disabled";
-  case CheckerRegistry::StateFromCmdLine::State_Enabled:
-    return "Enabled";
-  case CheckerRegistry::StateFromCmdLine::State_Unspecified:
-    return "Unspecified";
-  }
-  llvm_unreachable("Unhandled CheckerRegistry::StateFromCmdLine enum");
-}
-
-LLVM_DUMP_METHOD void
-CheckerRegistry::CheckerInfo::dumpToStream(llvm::raw_ostream &Out) const {
-  // The description can be just checked in Checkers.inc, the point here is to
-  // debug whether we succeeded in parsing it. Same with documentation uri.
-  Out << FullName << " (" << toString(State) << (IsHidden ? ", hidden" : "")
-      << ")\n";
-  Out << "  Options:\n";
-  for (const CmdLineOption &Option : CmdLineOptions) {
-    Out << "    ";
-    Option.dumpToStream(Out);
-    Out << '\n';
-  }
-  Out << "  Dependencies:\n";
-  for (const CheckerInfo *Dependency : Dependencies) {
-    Out << "  " << Dependency->FullName << '\n';
-  }
-  Out << "  Weak dependencies:\n";
-  for (const CheckerInfo *Dependency : WeakDependencies) {
-    Out << "    " << Dependency->FullName << '\n';
-  }
-}
-
-LLVM_DUMP_METHOD void
-CheckerRegistry::PackageInfo::dumpToStream(llvm::raw_ostream &Out) const {
-  Out << FullName << "\n";
-  Out << "  Options:\n";
-  for (const CmdLineOption &Option : CmdLineOptions) {
-    Out << "    ";
-    Option.dumpToStream(Out);
-    Out << '\n';
-  }
-}
-
 //===----------------------------------------------------------------------===//
 // Methods of CheckerRegistry.
 //===----------------------------------------------------------------------===//
 
-CheckerRegistry::CheckerInfoListRange
-CheckerRegistry::getMutableCheckersForCmdLineArg(StringRef CmdLineArg) {
-  auto It = binaryFind(Checkers, CmdLineArg);
-
-  if (!isInPackage(*It, CmdLineArg))
-    return {Checkers.end(), Checkers.end()};
-
-  // See how large the package is.
-  // If the package doesn't exist, assume the option refers to a single
-  // checker.
-  size_t Size = 1;
-  llvm::StringMap<size_t>::const_iterator PackageSize =
-      PackageSizes.find(CmdLineArg);
-
-  if (PackageSize != PackageSizes.end())
-    Size = PackageSize->getValue();
-
-  return {It, It + Size};
-}
-
 CheckerRegistry::CheckerRegistry(
-    ArrayRef<std::string> Plugins, DiagnosticsEngine &Diags,
-    AnalyzerOptions &AnOpts,
+    CheckerRegistryData &Data, ArrayRef<std::string> Plugins,
+    DiagnosticsEngine &Diags, AnalyzerOptions &AnOpts,
     ArrayRef<std::function<void(CheckerRegistry &)>> CheckerRegistrationFns)
-    : Diags(Diags), AnOpts(AnOpts) {
+    : Data(Data), Diags(Diags), AnOpts(AnOpts) {
 
   // Register builtin checkers.
 #define GET_CHECKERS
@@ -216,9 +92,10 @@ CheckerRegistry::CheckerRegistry(
       continue;
     }
 
+    using RegisterPluginCheckerFn = void (*)(CheckerRegistry &);
     // Register its checkers.
-    RegisterCheckersFn RegisterPluginCheckers =
-        reinterpret_cast<RegisterCheckersFn>(
+    RegisterPluginCheckerFn RegisterPluginCheckers =
+        reinterpret_cast<RegisterPluginCheckerFn>(
             Lib.getAddressOfSymbol("clang_registerCheckers"));
     if (RegisterPluginCheckers)
       RegisterPluginCheckers(*this);
@@ -235,8 +112,8 @@ CheckerRegistry::CheckerRegistry(
   // FIXME: Alphabetical sort puts 'experimental' in the middle.
   // Would it be better to name it '~experimental' or something else
   // that's ASCIIbetically last?
-  llvm::sort(Packages, PackageNameLT{});
-  llvm::sort(Checkers, CheckerNameLT{});
+  llvm::sort(Data.Packages, checker_registry::PackageNameLT{});
+  llvm::sort(Data.Checkers, checker_registry::CheckerNameLT{});
 
 #define GET_CHECKER_DEPENDENCIES
 
@@ -274,8 +151,8 @@ CheckerRegistry::CheckerRegistry(
   resolveDependencies<false>();
 
 #ifndef NDEBUG
-  for (auto &DepPair : Dependencies) {
-    for (auto &WeakDepPair : WeakDependencies) {
+  for (auto &DepPair : Data.Dependencies) {
+    for (auto &WeakDepPair : Data.WeakDependencies) {
       // Some assertions to enforce that strong dependencies are relations in
       // between purely modeling checkers, and weak dependencies are about
       // diagnostics.
@@ -295,7 +172,7 @@ CheckerRegistry::CheckerRegistry(
   // command line.
   for (const std::pair<std::string, bool> &Opt : AnOpts.CheckersAndPackages) {
     CheckerInfoListRange CheckerForCmdLineArg =
-        getMutableCheckersForCmdLineArg(Opt.first);
+        Data.getMutableCheckersForCmdLineArg(Opt.first);
 
     if (CheckerForCmdLineArg.begin() == CheckerForCmdLineArg.end()) {
       Diags.Report(diag::err_unknown_analyzer_checker_or_package) << Opt.first;
@@ -315,19 +192,16 @@ CheckerRegistry::CheckerRegistry(
 //===----------------------------------------------------------------------===//
 
 template <typename IsEnabledFn>
-static bool
-collectStrongDependencies(const CheckerRegistry::ConstCheckerInfoList &Deps,
-                          const CheckerManager &Mgr,
-                          CheckerRegistry::CheckerInfoSet &Ret,
-                          IsEnabledFn IsEnabled);
+static bool collectStrongDependencies(const ConstCheckerInfoList &Deps,
+                                      const CheckerManager &Mgr,
+                                      CheckerInfoSet &Ret,
+                                      IsEnabledFn IsEnabled);
 
-/// Collects weak dependencies in \p enabledCheckers.
+/// Collects weak dependencies in \p enabledData.Checkers.
 template <typename IsEnabledFn>
-static void
-collectWeakDependencies(const CheckerRegistry::ConstCheckerInfoList &Deps,
-                        const CheckerManager &Mgr,
-                        CheckerRegistry::CheckerInfoSet &Ret,
-                        IsEnabledFn IsEnabled);
+static void collectWeakDependencies(const ConstCheckerInfoList &Deps,
+                                    const CheckerManager &Mgr,
+                                    CheckerInfoSet &Ret, IsEnabledFn IsEnabled);
 
 void CheckerRegistry::initializeRegistry(const CheckerManager &Mgr) {
   // First, we calculate the list of enabled checkers as specified by the
@@ -338,7 +212,7 @@ void CheckerRegistry::initializeRegistry(const CheckerManager &Mgr) {
   auto IsEnabledFromCmdLine = [&](const CheckerInfo *Checker) {
     return !Checker->isDisabled(Mgr);
   };
-  for (const CheckerInfo &Checker : Checkers) {
+  for (const CheckerInfo &Checker : Data.Checkers) {
     if (!Checker.isEnabled(Mgr))
       continue;
 
@@ -362,7 +236,7 @@ void CheckerRegistry::initializeRegistry(const CheckerManager &Mgr) {
   auto IsEnabled = [&](const CheckerInfo *Checker) {
     return llvm::is_contained(Tmp, Checker);
   };
-  for (const CheckerInfo &Checker : Checkers) {
+  for (const CheckerInfo &Checker : Data.Checkers) {
     if (!Checker.isEnabled(Mgr))
       continue;
 
@@ -378,19 +252,18 @@ void CheckerRegistry::initializeRegistry(const CheckerManager &Mgr) {
     }
 
     // Note that set_union also preserves the order of insertion.
-    EnabledCheckers.set_union(Deps);
-    EnabledCheckers.insert(&Checker);
+    Data.EnabledCheckers.set_union(Deps);
+    Data.EnabledCheckers.insert(&Checker);
   }
 }
 
 template <typename IsEnabledFn>
-static bool
-collectStrongDependencies(const CheckerRegistry::ConstCheckerInfoList &Deps,
-                          const CheckerManager &Mgr,
-                          CheckerRegistry::CheckerInfoSet &Ret,
-                          IsEnabledFn IsEnabled) {
+static bool collectStrongDependencies(const ConstCheckerInfoList &Deps,
+                                      const CheckerManager &Mgr,
+                                      CheckerInfoSet &Ret,
+                                      IsEnabledFn IsEnabled) {
 
-  for (const CheckerRegistry::CheckerInfo *Dependency : Deps) {
+  for (const CheckerInfo *Dependency : Deps) {
     if (!IsEnabled(Dependency))
       return false;
 
@@ -405,13 +278,12 @@ collectStrongDependencies(const CheckerRegistry::ConstCheckerInfoList &Deps,
 }
 
 template <typename IsEnabledFn>
-static void
-collectWeakDependencies(const CheckerRegistry::ConstCheckerInfoList &WeakDeps,
-                        const CheckerManager &Mgr,
-                        CheckerRegistry::CheckerInfoSet &Ret,
-                        IsEnabledFn IsEnabled) {
+static void collectWeakDependencies(const ConstCheckerInfoList &WeakDeps,
+                                    const CheckerManager &Mgr,
+                                    CheckerInfoSet &Ret,
+                                    IsEnabledFn IsEnabled) {
 
-  for (const CheckerRegistry::CheckerInfo *Dependency : WeakDeps) {
+  for (const CheckerInfo *Dependency : WeakDeps) {
     // Don't enable this checker if strong dependencies are unsatisfied, but
     // assume that weak dependencies are transitive.
     collectWeakDependencies(Dependency->WeakDependencies, Mgr, Ret, IsEnabled);
@@ -425,18 +297,26 @@ collectWeakDependencies(const CheckerRegistry::ConstCheckerInfoList &WeakDeps,
 
 template <bool IsWeak> void CheckerRegistry::resolveDependencies() {
   for (const std::pair<StringRef, StringRef> &Entry :
-       (IsWeak ? WeakDependencies : Dependencies)) {
+       (IsWeak ? Data.WeakDependencies : Data.Dependencies)) {
 
-    auto CheckerIt = binaryFind(Checkers, Entry.first);
-    assert(CheckerIt != Checkers.end() && CheckerIt->FullName == Entry.first &&
+    auto CheckerIt = binaryFind(Data.Checkers, Entry.first);
+    assert(CheckerIt != Data.Checkers.end() &&
+           CheckerIt->FullName == Entry.first &&
            "Failed to find the checker while attempting to set up its "
            "dependencies!");
 
-    auto DependencyIt = binaryFind(Checkers, Entry.second);
-    assert(DependencyIt != Checkers.end() &&
+    auto DependencyIt = binaryFind(Data.Checkers, Entry.second);
+    assert(DependencyIt != Data.Checkers.end() &&
            DependencyIt->FullName == Entry.second &&
            "Failed to find the dependency of a checker!");
 
+    // We do allow diagnostics from unit test/example dependency checkers.
+    assert((DependencyIt->FullName.startswith("test") ||
+            DependencyIt->FullName.startswith("example") || IsWeak ||
+            DependencyIt->IsHidden) &&
+           "Strong dependencies are modeling checkers, and as such "
+           "non-user facing! Mark them hidden in Checkers.td!");
+
     if (IsWeak)
       CheckerIt->WeakDependencies.emplace_back(&*DependencyIt);
     else
@@ -445,12 +325,12 @@ template <bool IsWeak> void CheckerRegistry::resolveDependencies() {
 }
 
 void CheckerRegistry::addDependency(StringRef FullName, StringRef Dependency) {
-  Dependencies.emplace_back(FullName, Dependency);
+  Data.Dependencies.emplace_back(FullName, Dependency);
 }
 
 void CheckerRegistry::addWeakDependency(StringRef FullName,
                                         StringRef Dependency) {
-  WeakDependencies.emplace_back(FullName, Dependency);
+  Data.WeakDependencies.emplace_back(FullName, Dependency);
 }
 
 //===----------------------------------------------------------------------===//
@@ -459,8 +339,7 @@ void CheckerRegistry::addWeakDependency(StringRef FullName,
 
 /// Insert the checker/package option to AnalyzerOptions' config table, and
 /// validate it, if the user supplied it on the command line.
-static void insertAndValidate(StringRef FullName,
-                              const CheckerRegistry::CmdLineOption &Option,
+static void insertAndValidate(StringRef FullName, const CmdLineOption &Option,
                               AnalyzerOptions &AnOpts,
                               DiagnosticsEngine &Diags) {
 
@@ -509,10 +388,10 @@ static void insertAndValidate(StringRef FullName,
 }
 
 template <class T>
-static void
-insertOptionToCollection(StringRef FullName, T &Collection,
-                         const CheckerRegistry::CmdLineOption &Option,
-                         AnalyzerOptions &AnOpts, DiagnosticsEngine &Diags) {
+static void insertOptionToCollection(StringRef FullName, T &Collection,
+                                     const CmdLineOption &Option,
+                                     AnalyzerOptions &AnOpts,
+                                     DiagnosticsEngine &Diags) {
   auto It = binaryFind(Collection, FullName);
   assert(It != Collection.end() &&
          "Failed to find the checker while attempting to add a command line "
@@ -525,20 +404,20 @@ insertOptionToCollection(StringRef FullName, T &Collection,
 
 void CheckerRegistry::resolveCheckerAndPackageOptions() {
   for (const std::pair<StringRef, CmdLineOption> &CheckerOptEntry :
-       CheckerOptions) {
-    insertOptionToCollection(CheckerOptEntry.first, Checkers,
+       Data.CheckerOptions) {
+    insertOptionToCollection(CheckerOptEntry.first, Data.Checkers,
                              CheckerOptEntry.second, AnOpts, Diags);
   }
 
   for (const std::pair<StringRef, CmdLineOption> &PackageOptEntry :
-       PackageOptions) {
-    insertOptionToCollection(PackageOptEntry.first, Packages,
+       Data.PackageOptions) {
+    insertOptionToCollection(PackageOptEntry.first, Data.Packages,
                              PackageOptEntry.second, AnOpts, Diags);
   }
 }
 
 void CheckerRegistry::addPackage(StringRef FullName) {
-  Packages.emplace_back(PackageInfo(FullName));
+  Data.Packages.emplace_back(PackageInfo(FullName));
 }
 
 void CheckerRegistry::addPackageOption(StringRef OptionType,
@@ -548,22 +427,22 @@ void CheckerRegistry::addPackageOption(StringRef OptionType,
                                        StringRef Description,
                                        StringRef DevelopmentStatus,
                                        bool IsHidden) {
-  PackageOptions.emplace_back(
+  Data.PackageOptions.emplace_back(
       PackageFullName, CmdLineOption{OptionType, OptionName, DefaultValStr,
                                      Description, DevelopmentStatus, IsHidden});
 }
 
-void CheckerRegistry::addChecker(InitializationFunction Rfn,
+void CheckerRegistry::addChecker(RegisterCheckerFn Rfn,
                                  ShouldRegisterFunction Sfn, StringRef Name,
                                  StringRef Desc, StringRef DocsUri,
                                  bool IsHidden) {
-  Checkers.emplace_back(Rfn, Sfn, Name, Desc, DocsUri, IsHidden);
+  Data.Checkers.emplace_back(Rfn, Sfn, Name, Desc, DocsUri, IsHidden);
 
   // Record the presence of the checker in its packages.
   StringRef PackageName, LeafName;
   std::tie(PackageName, LeafName) = Name.rsplit(PackageSeparator);
   while (!LeafName.empty()) {
-    PackageSizes[PackageName] += 1;
+    Data.PackageSizes[PackageName] += 1;
     std::tie(PackageName, LeafName) = PackageName.rsplit(PackageSeparator);
   }
 }
@@ -575,29 +454,28 @@ void CheckerRegistry::addCheckerOption(StringRef OptionType,
                                        StringRef Description,
                                        StringRef DevelopmentStatus,
                                        bool IsHidden) {
-  CheckerOptions.emplace_back(
+  Data.CheckerOptions.emplace_back(
       CheckerFullName, CmdLineOption{OptionType, OptionName, DefaultValStr,
                                      Description, DevelopmentStatus, IsHidden});
 }
 
 void CheckerRegistry::initializeManager(CheckerManager &CheckerMgr) const {
   // Initialize the CheckerManager with all enabled checkers.
-  for (const auto *Checker : EnabledCheckers) {
+  for (const auto *Checker : Data.EnabledCheckers) {
     CheckerMgr.setCurrentCheckerName(CheckerNameRef(Checker->FullName));
     Checker->Initialize(CheckerMgr);
   }
 }
 
-static void
-isOptionContainedIn(const CheckerRegistry::CmdLineOptionList &OptionList,
-                    StringRef SuppliedChecker, StringRef SuppliedOption,
-                    const AnalyzerOptions &AnOpts, DiagnosticsEngine &Diags) {
+static void isOptionContainedIn(const CmdLineOptionList &OptionList,
+                                StringRef SuppliedChecker,
+                                StringRef SuppliedOption,
+                                const AnalyzerOptions &AnOpts,
+                                DiagnosticsEngine &Diags) {
 
   if (!AnOpts.ShouldEmitErrorsOnInvalidConfigValue)
     return;
 
-  using CmdLineOption = CheckerRegistry::CmdLineOption;
-
   auto SameOptName = [SuppliedOption](const CmdLineOption &Opt) {
     return Opt.OptionName == SuppliedOption;
   };
@@ -631,16 +509,16 @@ void CheckerRegistry::validateCheckerOptions() const {
     // it would return with an iterator to the first checker in the core, so we
     // we really have to use find here, which uses operator==.
     auto CheckerIt =
-        llvm::find(Checkers, CheckerInfo(SuppliedCheckerOrPackage));
-    if (CheckerIt != Checkers.end()) {
+        llvm::find(Data.Checkers, CheckerInfo(SuppliedCheckerOrPackage));
+    if (CheckerIt != Data.Checkers.end()) {
       isOptionContainedIn(CheckerIt->CmdLineOptions, SuppliedCheckerOrPackage,
                           SuppliedOption, AnOpts, Diags);
       continue;
     }
 
     const auto *PackageIt =
-        llvm::find(Packages, PackageInfo(SuppliedCheckerOrPackage));
-    if (PackageIt != Packages.end()) {
+        llvm::find(Data.Packages, PackageInfo(SuppliedCheckerOrPackage));
+    if (PackageIt != Data.Packages.end()) {
       isOptionContainedIn(PackageIt->CmdLineOptions, SuppliedCheckerOrPackage,
                           SuppliedOption, AnOpts, Diags);
       continue;
@@ -651,122 +529,3 @@ void CheckerRegistry::validateCheckerOptions() const {
   }
 }
 
-//===----------------------------------------------------------------------===//
-// Printing functions.
-//===----------------------------------------------------------------------===//
-
-void CheckerRegistry::printCheckerWithDescList(raw_ostream &Out,
-                                               size_t MaxNameChars) const {
-  // FIXME: Print available packages.
-
-  Out << "CHECKERS:\n";
-
-  // Find the maximum option length.
-  size_t OptionFieldWidth = 0;
-  for (const auto &Checker : Checkers) {
-    // Limit the amount of padding we are willing to give up for alignment.
-    //   Package.Name     Description  [Hidden]
-    size_t NameLength = Checker.FullName.size();
-    if (NameLength <= MaxNameChars)
-      OptionFieldWidth = std::max(OptionFieldWidth, NameLength);
-  }
-
-  const size_t InitialPad = 2;
-
-  auto Print = [=](llvm::raw_ostream &Out, const CheckerInfo &Checker,
-                   StringRef Description) {
-    AnalyzerOptions::printFormattedEntry(Out, {Checker.FullName, Description},
-                                         InitialPad, OptionFieldWidth);
-    Out << '\n';
-  };
-
-  for (const auto &Checker : Checkers) {
-    // The order of this if branches is significant, we wouldn't like to display
-    // developer checkers even in the alpha output. For example,
-    // alpha.cplusplus.IteratorModeling is a modeling checker, hence it's hidden
-    // by default, and users (even when the user is a developer of an alpha
-    // checker) shouldn't normally tinker with whether they should be enabled.
-
-    if (Checker.IsHidden) {
-      if (AnOpts.ShowCheckerHelpDeveloper)
-        Print(Out, Checker, Checker.Desc);
-      continue;
-    }
-
-    if (Checker.FullName.startswith("alpha")) {
-      if (AnOpts.ShowCheckerHelpAlpha)
-        Print(Out, Checker,
-              ("(Enable only for development!) " + Checker.Desc).str());
-      continue;
-    }
-
-    if (AnOpts.ShowCheckerHelp)
-        Print(Out, Checker, Checker.Desc);
-  }
-}
-
-void CheckerRegistry::printEnabledCheckerList(raw_ostream &Out) const {
-  for (const auto *i : EnabledCheckers)
-    Out << i->FullName << '\n';
-}
-
-void CheckerRegistry::printCheckerOptionList(raw_ostream &Out) const {
-  Out << "OVERVIEW: Clang Static Analyzer Checker and Package Option List\n\n";
-  Out << "USAGE: -analyzer-config <OPTION1=VALUE,OPTION2=VALUE,...>\n\n";
-  Out << "       -analyzer-config OPTION1=VALUE, -analyzer-config "
-         "OPTION2=VALUE, ...\n\n";
-  Out << "OPTIONS:\n\n";
-
-  std::multimap<StringRef, const CmdLineOption &> OptionMap;
-
-  for (const CheckerInfo &Checker : Checkers) {
-    for (const CmdLineOption &Option : Checker.CmdLineOptions) {
-      OptionMap.insert({Checker.FullName, Option});
-    }
-  }
-
-  for (const PackageInfo &Package : Packages) {
-    for (const CmdLineOption &Option : Package.CmdLineOptions) {
-      OptionMap.insert({Package.FullName, Option});
-    }
-  }
-
-  auto Print = [] (llvm::raw_ostream &Out, StringRef FullOption, StringRef Desc) {
-    AnalyzerOptions::printFormattedEntry(Out, {FullOption, Desc},
-                                         /*InitialPad*/ 2,
-                                         /*EntryWidth*/ 50,
-                                         /*MinLineWidth*/ 90);
-    Out << "\n\n";
-  };
-  for (const std::pair<const StringRef, const CmdLineOption &> &Entry :
-       OptionMap) {
-    const CmdLineOption &Option = Entry.second;
-    std::string FullOption = (Entry.first + ":" + Option.OptionName).str();
-
-    std::string Desc =
-        ("(" + Option.OptionType + ") " + Option.Description + " (default: " +
-         (Option.DefaultValStr.empty() ? "\"\"" : Option.DefaultValStr) + ")")
-            .str();
-
-    // The list of these if branches is significant, we wouldn't like to
-    // display hidden alpha checker options for
-    // -analyzer-checker-option-help-alpha.
-
-    if (Option.IsHidden) {
-      if (AnOpts.ShowCheckerOptionDeveloperList)
-        Print(Out, FullOption, Desc);
-      continue;
-    }
-
-    if (Option.DevelopmentStatus == "alpha" ||
-        Entry.first.startswith("alpha")) {
-      if (AnOpts.ShowCheckerOptionAlphaList)
-        Print(Out, FullOption,
-              llvm::Twine("(Enable only for development!) " + Desc).str());
-      continue;
-    }
-
-    if (AnOpts.ShowCheckerOptionList)
-      Print(Out, FullOption, Desc);
-  }
-}
diff --git a/clang/lib/StaticAnalyzer/Frontend/CreateCheckerManager.cpp b/clang/lib/StaticAnalyzer/Frontend/CreateCheckerManager.cpp
index 140f807514439..21a60785eb525 100644
--- a/clang/lib/StaticAnalyzer/Frontend/CreateCheckerManager.cpp
+++ b/clang/lib/StaticAnalyzer/Frontend/CreateCheckerManager.cpp
@@ -23,11 +23,11 @@ CheckerManager::CheckerManager(
     ArrayRef<std::function<void(CheckerRegistry &)>> checkerRegistrationFns)
     : Context(&Context), LangOpts(Context.getLangOpts()), AOptions(AOptions),
       PP(&PP), Diags(Context.getDiagnostics()),
-      Registry(
-          std::make_unique<CheckerRegistry>(plugins, Context.getDiagnostics(),
-                                            AOptions, checkerRegistrationFns)) {
-  Registry->initializeRegistry(*this);
-  Registry->initializeManager(*this);
+      RegistryData(std::make_unique<CheckerRegistryData>()) {
+  CheckerRegistry Registry(*RegistryData, plugins, Context.getDiagnostics(),
+                           AOptions, checkerRegistrationFns);
+  Registry.initializeRegistry(*this);
+  Registry.initializeManager(*this);
   finishedCheckerRegistration();
 }
 
@@ -36,8 +36,9 @@ CheckerManager::CheckerManager(AnalyzerOptions &AOptions,
                                DiagnosticsEngine &Diags,
                                ArrayRef<std::string> plugins)
     : LangOpts(LangOpts), AOptions(AOptions), Diags(Diags),
-      Registry(std::make_unique<CheckerRegistry>(plugins, Diags, AOptions)) {
-  Registry->initializeRegistry(*this);
+      RegistryData(std::make_unique<CheckerRegistryData>()) {
+  CheckerRegistry Registry(*RegistryData, plugins, Diags, AOptions, {});
+  Registry.initializeRegistry(*this);
 }
 
 CheckerManager::~CheckerManager() {
diff --git a/clang/lib/Tooling/ASTDiff/CMakeLists.txt b/clang/lib/Tooling/ASTDiff/CMakeLists.txt
index 578d8ca0cbc11..38cf5370bc49d 100644
--- a/clang/lib/Tooling/ASTDiff/CMakeLists.txt
+++ b/clang/lib/Tooling/ASTDiff/CMakeLists.txt
@@ -8,4 +8,7 @@ add_clang_library(clangToolingASTDiff
   clangBasic
   clangAST
   clangLex
+
+  DEPENDS
+  omp_gen
   )
diff --git a/clang/lib/Tooling/CMakeLists.txt b/clang/lib/Tooling/CMakeLists.txt
index 71b6cc55e5040..7a58af59dad1e 100644
--- a/clang/lib/Tooling/CMakeLists.txt
+++ b/clang/lib/Tooling/CMakeLists.txt
@@ -31,6 +31,7 @@ add_clang_library(clangTooling
 
   DEPENDS
   ClangDriverOptions
+  omp_gen
 
   LINK_LIBS
   clangAST
diff --git a/clang/lib/Tooling/Refactoring/AtomicChange.cpp b/clang/lib/Tooling/Refactoring/AtomicChange.cpp
index 3be15b7d8509f..069e9c1eb36ef 100644
--- a/clang/lib/Tooling/Refactoring/AtomicChange.cpp
+++ b/clang/lib/Tooling/Refactoring/AtomicChange.cpp
@@ -204,6 +204,12 @@ AtomicChange::AtomicChange(const SourceManager &SM,
   Key = FilePath + ":" + std::to_string(FileIDAndOffset.second);
 }
 
+AtomicChange::AtomicChange(const SourceManager &SM, SourceLocation KeyPosition,
+                           llvm::Any M)
+    : AtomicChange(SM, KeyPosition) {
+  Metadata = std::move(M);
+}
+
 AtomicChange::AtomicChange(std::string Key, std::string FilePath,
                            std::string Error,
                            std::vector<std::string> InsertedHeaders,
diff --git a/clang/lib/Tooling/Refactoring/CMakeLists.txt b/clang/lib/Tooling/Refactoring/CMakeLists.txt
index db889d2a06b5b..0565de224bbca 100644
--- a/clang/lib/Tooling/Refactoring/CMakeLists.txt
+++ b/clang/lib/Tooling/Refactoring/CMakeLists.txt
@@ -22,4 +22,7 @@ add_clang_library(clangToolingRefactoring
   clangLex
   clangRewrite
   clangToolingCore
+
+  DEPENDS
+  omp_gen
   )
diff --git a/clang/lib/Tooling/Syntax/BuildTree.cpp b/clang/lib/Tooling/Syntax/BuildTree.cpp
index 3ee66aabfb6d8..1f192180ec451 100644
--- a/clang/lib/Tooling/Syntax/BuildTree.cpp
+++ b/clang/lib/Tooling/Syntax/BuildTree.cpp
@@ -12,6 +12,7 @@
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclarationName.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/Stmt.h"
 #include "clang/AST/TypeLoc.h"
@@ -22,6 +23,7 @@
 #include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TokenKinds.h"
 #include "clang/Lex/Lexer.h"
+#include "clang/Lex/LiteralSupport.h"
 #include "clang/Tooling/Syntax/Nodes.h"
 #include "clang/Tooling/Syntax/Tokens.h"
 #include "clang/Tooling/Syntax/Tree.h"
@@ -114,6 +116,86 @@ struct GetStartLoc : TypeLocVisitor<GetStartLoc, SourceLocation> {
 };
 } // namespace
 
+static syntax::NodeKind getOperatorNodeKind(const CXXOperatorCallExpr &E) {
+  switch (E.getOperator()) {
+  // Comparison
+  case OO_EqualEqual:
+  case OO_ExclaimEqual:
+  case OO_Greater:
+  case OO_GreaterEqual:
+  case OO_Less:
+  case OO_LessEqual:
+  case OO_Spaceship:
+  // Assignment
+  case OO_Equal:
+  case OO_SlashEqual:
+  case OO_PercentEqual:
+  case OO_CaretEqual:
+  case OO_PipeEqual:
+  case OO_LessLessEqual:
+  case OO_GreaterGreaterEqual:
+  case OO_PlusEqual:
+  case OO_MinusEqual:
+  case OO_StarEqual:
+  case OO_AmpEqual:
+  // Binary computation
+  case OO_Slash:
+  case OO_Percent:
+  case OO_Caret:
+  case OO_Pipe:
+  case OO_LessLess:
+  case OO_GreaterGreater:
+  case OO_AmpAmp:
+  case OO_PipePipe:
+  case OO_ArrowStar:
+  case OO_Comma:
+    return syntax::NodeKind::BinaryOperatorExpression;
+  case OO_Tilde:
+  case OO_Exclaim:
+    return syntax::NodeKind::PrefixUnaryOperatorExpression;
+  // Prefix/Postfix increment/decrement
+  case OO_PlusPlus:
+  case OO_MinusMinus:
+    switch (E.getNumArgs()) {
+    case 1:
+      return syntax::NodeKind::PrefixUnaryOperatorExpression;
+    case 2:
+      return syntax::NodeKind::PostfixUnaryOperatorExpression;
+    default:
+      llvm_unreachable("Invalid number of arguments for operator");
+    }
+  // Operators that can be unary or binary
+  case OO_Plus:
+  case OO_Minus:
+  case OO_Star:
+  case OO_Amp:
+    switch (E.getNumArgs()) {
+    case 1:
+      return syntax::NodeKind::PrefixUnaryOperatorExpression;
+    case 2:
+      return syntax::NodeKind::BinaryOperatorExpression;
+    default:
+      llvm_unreachable("Invalid number of arguments for operator");
+    }
+    return syntax::NodeKind::BinaryOperatorExpression;
+  // Not yet supported by SyntaxTree
+  case OO_New:
+  case OO_Delete:
+  case OO_Array_New:
+  case OO_Array_Delete:
+  case OO_Coawait:
+  case OO_Call:
+  case OO_Subscript:
+  case OO_Arrow:
+    return syntax::NodeKind::UnknownExpression;
+  case OO_Conditional: // not overloadable
+  case NUM_OVERLOADED_OPERATORS:
+  case OO_None:
+    llvm_unreachable("Not an overloadable operator");
+  }
+  llvm_unreachable("Unknown OverloadedOperatorKind enum");
+}
+
 /// Gets the range of declarator as defined by the C++ grammar. E.g.
 ///     `int a;` -> range of `a`,
 ///     `int *a;` -> range of `*a`,
@@ -135,7 +217,8 @@ static SourceRange getDeclaratorRange(const SourceManager &SM, TypeLoc T,
   }
   if (Initializer.isValid()) {
     auto InitializerEnd = Initializer.getEnd();
-    assert(SM.isBeforeInTranslationUnit(End, InitializerEnd) || End == InitializerEnd);
+    assert(SM.isBeforeInTranslationUnit(End, InitializerEnd) ||
+           End == InitializerEnd);
     End = InitializerEnd;
   }
   return SourceRange(Start, End);
@@ -470,8 +553,8 @@ class syntax::TreeBuilder {
 namespace {
 class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
 public:
-  explicit BuildTreeVisitor(ASTContext &Ctx, syntax::TreeBuilder &Builder)
-      : Builder(Builder), LangOpts(Ctx.getLangOpts()) {}
+  explicit BuildTreeVisitor(ASTContext &Context, syntax::TreeBuilder &Builder)
+      : Builder(Builder), Context(Context) {}
 
   bool shouldTraversePostOrder() const { return true; }
 
@@ -578,15 +661,19 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
     // RAV traverses it as a statement, we produce invalid node kinds in that
     // case.
     // FIXME: should do this in RAV instead?
-    if (S->getInit() && !TraverseStmt(S->getInit()))
-      return false;
-    if (S->getLoopVariable() && !TraverseDecl(S->getLoopVariable()))
-      return false;
-    if (S->getRangeInit() && !TraverseStmt(S->getRangeInit()))
-      return false;
-    if (S->getBody() && !TraverseStmt(S->getBody()))
-      return false;
-    return true;
+    bool Result = [&, this]() {
+      if (S->getInit() && !TraverseStmt(S->getInit()))
+        return false;
+      if (S->getLoopVariable() && !TraverseDecl(S->getLoopVariable()))
+        return false;
+      if (S->getRangeInit() && !TraverseStmt(S->getRangeInit()))
+        return false;
+      if (S->getBody() && !TraverseStmt(S->getBody()))
+        return false;
+      return true;
+    }();
+    WalkUpFromCXXForRangeStmt(S);
+    return Result;
   }
 
   bool TraverseStmt(Stmt *S) {
@@ -623,6 +710,55 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
     return NNS;
   }
 
+  bool TraverseUserDefinedLiteral(UserDefinedLiteral *S) {
+    // The semantic AST node `UserDefinedLiteral` (UDL) may have one child node
+    // referencing the location of the UDL suffix (`_w` in `1.2_w`). The
+    // UDL suffix location does not point to the beginning of a token, so we
+    // can't represent the UDL suffix as a separate syntax tree node.
+
+    return WalkUpFromUserDefinedLiteral(S);
+  }
+
+  syntax::UserDefinedLiteralExpression *
+  buildUserDefinedLiteral(UserDefinedLiteral *S) {
+    switch (S->getLiteralOperatorKind()) {
+    case clang::UserDefinedLiteral::LOK_Integer:
+      return new (allocator()) syntax::IntegerUserDefinedLiteralExpression;
+    case clang::UserDefinedLiteral::LOK_Floating:
+      return new (allocator()) syntax::FloatUserDefinedLiteralExpression;
+    case clang::UserDefinedLiteral::LOK_Character:
+      return new (allocator()) syntax::CharUserDefinedLiteralExpression;
+    case clang::UserDefinedLiteral::LOK_String:
+      return new (allocator()) syntax::StringUserDefinedLiteralExpression;
+    case clang::UserDefinedLiteral::LOK_Raw:
+    case clang::UserDefinedLiteral::LOK_Template:
+      // For raw literal operator and numeric literal operator template we
+      // cannot get the type of the operand in the semantic AST. We get this
+      // information from the token. As integer and floating point have the same
+      // token kind, we run `NumericLiteralParser` again to distinguish them.
+      auto TokLoc = S->getBeginLoc();
+      auto TokSpelling =
+          Builder.findToken(TokLoc)->text(Context.getSourceManager());
+      auto Literal =
+          NumericLiteralParser(TokSpelling, TokLoc, Context.getSourceManager(),
+                               Context.getLangOpts(), Context.getTargetInfo(),
+                               Context.getDiagnostics());
+      if (Literal.isIntegerLiteral())
+        return new (allocator()) syntax::IntegerUserDefinedLiteralExpression;
+      else {
+        assert(Literal.isFloatingLiteral());
+        return new (allocator()) syntax::FloatUserDefinedLiteralExpression;
+      }
+    }
+    llvm_unreachable("Unknown literal operator kind.");
+  }
+
+  bool WalkUpFromUserDefinedLiteral(UserDefinedLiteral *S) {
+    Builder.markChildToken(S->getBeginLoc(), syntax::NodeRole::LiteralToken);
+    Builder.foldNode(Builder.getExprRange(S), buildUserDefinedLiteral(S), S);
+    return true;
+  }
+
   bool WalkUpFromDeclRefExpr(DeclRefExpr *S) {
     if (auto *NNS = BuildNestedNameSpecifier(S->getQualifierLoc()))
       Builder.markChild(NNS, syntax::NodeRole::IdExpression_qualifier);
@@ -647,6 +783,16 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
     return true;
   }
 
+  bool WalkUpFromParenExpr(ParenExpr *S) {
+    Builder.markChildToken(S->getLParen(), syntax::NodeRole::OpenParen);
+    Builder.markExprChild(S->getSubExpr(),
+                          syntax::NodeRole::ParenExpression_subExpression);
+    Builder.markChildToken(S->getRParen(), syntax::NodeRole::CloseParen);
+    Builder.foldNode(Builder.getExprRange(S),
+                     new (allocator()) syntax::ParenExpression, S);
+    return true;
+  }
+
   bool WalkUpFromIntegerLiteral(IntegerLiteral *S) {
     Builder.markChildToken(S->getLocation(), syntax::NodeRole::LiteralToken);
     Builder.foldNode(Builder.getExprRange(S),
@@ -654,6 +800,34 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
     return true;
   }
 
+  bool WalkUpFromCharacterLiteral(CharacterLiteral *S) {
+    Builder.markChildToken(S->getLocation(), syntax::NodeRole::LiteralToken);
+    Builder.foldNode(Builder.getExprRange(S),
+                     new (allocator()) syntax::CharacterLiteralExpression, S);
+    return true;
+  }
+
+  bool WalkUpFromFloatingLiteral(FloatingLiteral *S) {
+    Builder.markChildToken(S->getLocation(), syntax::NodeRole::LiteralToken);
+    Builder.foldNode(Builder.getExprRange(S),
+                     new (allocator()) syntax::FloatingLiteralExpression, S);
+    return true;
+  }
+
+  bool WalkUpFromStringLiteral(StringLiteral *S) {
+    Builder.markChildToken(S->getBeginLoc(), syntax::NodeRole::LiteralToken);
+    Builder.foldNode(Builder.getExprRange(S),
+                     new (allocator()) syntax::StringLiteralExpression, S);
+    return true;
+  }
+
+  bool WalkUpFromCXXBoolLiteralExpr(CXXBoolLiteralExpr *S) {
+    Builder.markChildToken(S->getLocation(), syntax::NodeRole::LiteralToken);
+    Builder.foldNode(Builder.getExprRange(S),
+                     new (allocator()) syntax::BoolLiteralExpression, S);
+    return true;
+  }
+
   bool WalkUpFromCXXNullPtrLiteralExpr(CXXNullPtrLiteralExpr *S) {
     Builder.markChildToken(S->getLocation(), syntax::NodeRole::LiteralToken);
     Builder.foldNode(Builder.getExprRange(S),
@@ -691,8 +865,29 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
     return true;
   }
 
+  bool TraverseCXXOperatorCallExpr(CXXOperatorCallExpr *S) {
+    if (getOperatorNodeKind(*S) ==
+        syntax::NodeKind::PostfixUnaryOperatorExpression) {
+      // A postfix unary operator is declared as taking two operands. The
+      // second operand is used to distinguish from its prefix counterpart. In
+      // the semantic AST this "phantom" operand is represented as a
+      // `IntegerLiteral` with invalid `SourceLocation`. We skip visiting this
+      // operand because it does not correspond to anything written in source
+      // code
+      for (auto *child : S->children()) {
+        if (child->getSourceRange().isInvalid())
+          continue;
+        if (!TraverseStmt(child))
+          return false;
+      }
+      return WalkUpFromCXXOperatorCallExpr(S);
+    } else
+      return RecursiveASTVisitor::TraverseCXXOperatorCallExpr(S);
+  }
+
   bool WalkUpFromCXXOperatorCallExpr(CXXOperatorCallExpr *S) {
-    if (S->isInfixBinaryOp()) {
+    switch (getOperatorNodeKind(*S)) {
+    case syntax::NodeKind::BinaryOperatorExpression:
       Builder.markExprChild(
           S->getArg(0),
           syntax::NodeRole::BinaryOperatorExpression_leftHandSide);
@@ -705,8 +900,31 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
       Builder.foldNode(Builder.getExprRange(S),
                        new (allocator()) syntax::BinaryOperatorExpression, S);
       return true;
+    case syntax::NodeKind::PrefixUnaryOperatorExpression:
+      Builder.markChildToken(
+          S->getOperatorLoc(),
+          syntax::NodeRole::OperatorExpression_operatorToken);
+      Builder.markExprChild(S->getArg(0),
+                            syntax::NodeRole::UnaryOperatorExpression_operand);
+      Builder.foldNode(Builder.getExprRange(S),
+                       new (allocator()) syntax::PrefixUnaryOperatorExpression,
+                       S);
+      return true;
+    case syntax::NodeKind::PostfixUnaryOperatorExpression:
+      Builder.markChildToken(
+          S->getOperatorLoc(),
+          syntax::NodeRole::OperatorExpression_operatorToken);
+      Builder.markExprChild(S->getArg(0),
+                            syntax::NodeRole::UnaryOperatorExpression_operand);
+      Builder.foldNode(Builder.getExprRange(S),
+                       new (allocator()) syntax::PostfixUnaryOperatorExpression,
+                       S);
+      return true;
+    case syntax::NodeKind::UnknownExpression:
+      return RecursiveASTVisitor::WalkUpFromCXXOperatorCallExpr(S);
+    default:
+      llvm_unreachable("getOperatorNodeKind() does not return this value");
     }
-    return RecursiveASTVisitor::WalkUpFromCXXOperatorCallExpr(S);
   }
 
   bool WalkUpFromNamespaceDecl(NamespaceDecl *S) {
@@ -1058,7 +1276,7 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
   llvm::BumpPtrAllocator &allocator() { return Builder.allocator(); }
 
   syntax::TreeBuilder &Builder;
-  const LangOptions &LangOpts;
+  const ASTContext &Context;
 };
 } // namespace
 
diff --git a/clang/lib/Tooling/Syntax/CMakeLists.txt b/clang/lib/Tooling/Syntax/CMakeLists.txt
index eb266411036a0..e933faeb0f506 100644
--- a/clang/lib/Tooling/Syntax/CMakeLists.txt
+++ b/clang/lib/Tooling/Syntax/CMakeLists.txt
@@ -15,4 +15,7 @@ add_clang_library(clangToolingSyntax
   clangFrontend
   clangLex
   clangToolingCore
+
+  DEPENDS
+  omp_gen
   )
diff --git a/clang/lib/Tooling/Syntax/Nodes.cpp b/clang/lib/Tooling/Syntax/Nodes.cpp
index 623391a7d8446..2435ae0a91dd6 100644
--- a/clang/lib/Tooling/Syntax/Nodes.cpp
+++ b/clang/lib/Tooling/Syntax/Nodes.cpp
@@ -18,10 +18,28 @@ llvm::raw_ostream &syntax::operator<<(llvm::raw_ostream &OS, NodeKind K) {
     return OS << "TranslationUnit";
   case NodeKind::UnknownExpression:
     return OS << "UnknownExpression";
-  case NodeKind::CxxNullPtrExpression:
-    return OS << "CxxNullPtrExpression";
+  case NodeKind::ParenExpression:
+    return OS << "ParenExpression";
   case NodeKind::IntegerLiteralExpression:
     return OS << "IntegerLiteralExpression";
+  case NodeKind::CharacterLiteralExpression:
+    return OS << "CharacterLiteralExpression";
+  case NodeKind::FloatingLiteralExpression:
+    return OS << "FloatingLiteralExpression";
+  case NodeKind::StringLiteralExpression:
+    return OS << "StringLiteralExpression";
+  case NodeKind::BoolLiteralExpression:
+    return OS << "BoolLiteralExpression";
+  case NodeKind::CxxNullPtrExpression:
+    return OS << "CxxNullPtrExpression";
+  case NodeKind::IntegerUserDefinedLiteralExpression:
+    return OS << "IntegerUserDefinedLiteralExpression";
+  case NodeKind::FloatUserDefinedLiteralExpression:
+    return OS << "FloatUserDefinedLiteralExpression";
+  case NodeKind::CharUserDefinedLiteralExpression:
+    return OS << "CharUserDefinedLiteralExpression";
+  case NodeKind::StringUserDefinedLiteralExpression:
+    return OS << "StringUserDefinedLiteralExpression";
   case NodeKind::PrefixUnaryOperatorExpression:
     return OS << "PrefixUnaryOperatorExpression";
   case NodeKind::PostfixUnaryOperatorExpression:
@@ -172,6 +190,8 @@ llvm::raw_ostream &syntax::operator<<(llvm::raw_ostream &OS, NodeRole R) {
     return OS << "IdExpression_qualifier";
   case syntax::NodeRole::NestedNameSpecifier_specifier:
     return OS << "NestedNameSpecifier_specifier";
+  case syntax::NodeRole::ParenExpression_subExpression:
+    return OS << "ParenExpression_subExpression";
   }
   llvm_unreachable("invalid role");
 }
@@ -195,16 +215,56 @@ syntax::UnqualifiedId *syntax::IdExpression::unqualifiedId() {
       findChild(syntax::NodeRole::IdExpression_id));
 }
 
+syntax::Leaf *syntax::ParenExpression::openParen() {
+  return llvm::cast_or_null<syntax::Leaf>(
+      findChild(syntax::NodeRole::OpenParen));
+}
+
+syntax::Expression *syntax::ParenExpression::subExpression() {
+  return llvm::cast_or_null<syntax::Expression>(
+      findChild(syntax::NodeRole::ParenExpression_subExpression));
+}
+
+syntax::Leaf *syntax::ParenExpression::closeParen() {
+  return llvm::cast_or_null<syntax::Leaf>(
+      findChild(syntax::NodeRole::CloseParen));
+}
+
 syntax::Leaf *syntax::IntegerLiteralExpression::literalToken() {
   return llvm::cast_or_null<syntax::Leaf>(
       findChild(syntax::NodeRole::LiteralToken));
 }
 
+syntax::Leaf *syntax::CharacterLiteralExpression::literalToken() {
+  return llvm::cast_or_null<syntax::Leaf>(
+      findChild(syntax::NodeRole::LiteralToken));
+}
+
+syntax::Leaf *syntax::FloatingLiteralExpression::literalToken() {
+  return llvm::cast_or_null<syntax::Leaf>(
+      findChild(syntax::NodeRole::LiteralToken));
+}
+
+syntax::Leaf *syntax::StringLiteralExpression::literalToken() {
+  return llvm::cast_or_null<syntax::Leaf>(
+      findChild(syntax::NodeRole::LiteralToken));
+}
+
+syntax::Leaf *syntax::BoolLiteralExpression::literalToken() {
+  return llvm::cast_or_null<syntax::Leaf>(
+      findChild(syntax::NodeRole::LiteralToken));
+}
+
 syntax::Leaf *syntax::CxxNullPtrExpression::nullPtrKeyword() {
   return llvm::cast_or_null<syntax::Leaf>(
       findChild(syntax::NodeRole::LiteralToken));
 }
 
+syntax::Leaf *syntax::UserDefinedLiteralExpression::literalToken() {
+  return llvm::cast_or_null<syntax::Leaf>(
+      findChild(syntax::NodeRole::LiteralToken));
+}
+
 syntax::Expression *syntax::BinaryOperatorExpression::lhs() {
   return llvm::cast_or_null<syntax::Expression>(
       findChild(syntax::NodeRole::BinaryOperatorExpression_leftHandSide));
diff --git a/clang/lib/Tooling/Transformer/CMakeLists.txt b/clang/lib/Tooling/Transformer/CMakeLists.txt
index 150b71b1ffcdb..4ab4bd6f08989 100644
--- a/clang/lib/Tooling/Transformer/CMakeLists.txt
+++ b/clang/lib/Tooling/Transformer/CMakeLists.txt
@@ -19,4 +19,7 @@ add_clang_library(clangTransformer
   clangLex
   clangToolingCore
   clangToolingRefactoring
+
+  DEPENDS
+  omp_gen
   )
diff --git a/clang/lib/Tooling/Transformer/RangeSelector.cpp b/clang/lib/Tooling/Transformer/RangeSelector.cpp
index baf2e785bfd0b..29b1a5b0372ea 100644
--- a/clang/lib/Tooling/Transformer/RangeSelector.cpp
+++ b/clang/lib/Tooling/Transformer/RangeSelector.cpp
@@ -146,7 +146,7 @@ RangeSelector transformer::statement(std::string ID) {
   };
 }
 
-RangeSelector transformer::range(RangeSelector Begin, RangeSelector End) {
+RangeSelector transformer::enclose(RangeSelector Begin, RangeSelector End) {
   return [Begin, End](const MatchResult &Result) -> Expected<CharSourceRange> {
     Expected<CharSourceRange> BeginRange = Begin(Result);
     if (!BeginRange)
@@ -165,8 +165,9 @@ RangeSelector transformer::range(RangeSelector Begin, RangeSelector End) {
   };
 }
 
-RangeSelector transformer::range(std::string BeginID, std::string EndID) {
-  return transformer::range(node(std::move(BeginID)), node(std::move(EndID)));
+RangeSelector transformer::encloseNodes(std::string BeginID,
+                                        std::string EndID) {
+  return transformer::enclose(node(std::move(BeginID)), node(std::move(EndID)));
 }
 
 RangeSelector transformer::member(std::string ID) {
diff --git a/clang/lib/Tooling/Transformer/RewriteRule.cpp b/clang/lib/Tooling/Transformer/RewriteRule.cpp
index ddce6ce591851..995bec03cd669 100644
--- a/clang/lib/Tooling/Transformer/RewriteRule.cpp
+++ b/clang/lib/Tooling/Transformer/RewriteRule.cpp
@@ -47,6 +47,7 @@ translateEdits(const MatchResult &Result, ArrayRef<ASTEdit> ASTEdits) {
     transformer::Edit T;
     T.Range = *EditRange;
     T.Replacement = std::move(*Replacement);
+    T.Metadata = E.Metadata;
     Edits.push_back(std::move(T));
   }
   return Edits;
diff --git a/clang/lib/Tooling/Transformer/SourceCode.cpp b/clang/lib/Tooling/Transformer/SourceCode.cpp
index a29d2e796d948..26b204851f058 100644
--- a/clang/lib/Tooling/Transformer/SourceCode.cpp
+++ b/clang/lib/Tooling/Transformer/SourceCode.cpp
@@ -37,11 +37,17 @@ StringRef clang::tooling::getText(CharSourceRange Range,
 CharSourceRange clang::tooling::maybeExtendRange(CharSourceRange Range,
                                                  tok::TokenKind Next,
                                                  ASTContext &Context) {
-  Optional<Token> Tok = Lexer::findNextToken(
-      Range.getEnd(), Context.getSourceManager(), Context.getLangOpts());
-  if (!Tok || !Tok->is(Next))
+  CharSourceRange R = Lexer::getAsCharRange(Range, Context.getSourceManager(),
+                                            Context.getLangOpts());
+  if (R.isInvalid())
     return Range;
-  return CharSourceRange::getTokenRange(Range.getBegin(), Tok->getLocation());
+  Token Tok;
+  bool Err =
+      Lexer::getRawToken(R.getEnd(), Tok, Context.getSourceManager(),
+                         Context.getLangOpts(), /*IgnoreWhiteSpace=*/true);
+  if (Err || !Tok.is(Next))
+    return Range;
+  return CharSourceRange::getTokenRange(Range.getBegin(), Tok.getLocation());
 }
 
 llvm::Error clang::tooling::validateEditRange(const CharSourceRange &Range,
diff --git a/clang/lib/Tooling/Transformer/Stencil.cpp b/clang/lib/Tooling/Transformer/Stencil.cpp
index 31b6ad9332c81..2670bf7adabf3 100644
--- a/clang/lib/Tooling/Transformer/Stencil.cpp
+++ b/clang/lib/Tooling/Transformer/Stencil.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
 #include <atomic>
 #include <memory>
 #include <string>
@@ -232,8 +233,31 @@ Error evalData(const SelectorData &Data, const MatchFinder::MatchResult &Match,
     return RawRange.takeError();
   CharSourceRange Range = Lexer::makeFileCharRange(
       *RawRange, *Match.SourceManager, Match.Context->getLangOpts());
+  if (Range.isInvalid()) {
+    // Validate the original range to attempt to get a meaningful error message.
+    // If it's valid, then something else is the cause and we just return the
+    // generic failure message.
+    if (auto Err = tooling::validateEditRange(*RawRange, *Match.SourceManager))
+      return handleErrors(std::move(Err), [](std::unique_ptr<StringError> E) {
+        assert(E->convertToErrorCode() ==
+                   llvm::make_error_code(errc::invalid_argument) &&
+               "Validation errors must carry the invalid_argument code");
+        return llvm::createStringError(
+            errc::invalid_argument,
+            "selected range could not be resolved to a valid source range; " +
+                E->getMessage());
+      });
+    return llvm::createStringError(
+        errc::invalid_argument,
+        "selected range could not be resolved to a valid source range");
+  }
+  // Validate `Range`, because `makeFileCharRange` accepts some ranges that
+  // `validateEditRange` rejects.
   if (auto Err = tooling::validateEditRange(Range, *Match.SourceManager))
-    return Err;
+    return joinErrors(
+        llvm::createStringError(errc::invalid_argument,
+                                "selected range is not valid for editing"),
+        std::move(Err));
   *Result += tooling::getText(Range, *Match.Context);
   return Error::success();
 }
@@ -298,17 +322,11 @@ template <typename T> class StencilImpl : public StencilInterface {
 };
 } // namespace
 
-Stencil transformer::detail::makeStencil(StringRef Text) { return text(Text); }
-
-Stencil transformer::detail::makeStencil(RangeSelector Selector) {
-  return selection(std::move(Selector));
-}
-
-Stencil transformer::text(StringRef Text) {
+Stencil transformer::detail::makeStencil(StringRef Text) {
   return std::make_shared<StencilImpl<RawTextData>>(std::string(Text));
 }
 
-Stencil transformer::selection(RangeSelector Selector) {
+Stencil transformer::detail::makeStencil(RangeSelector Selector) {
   return std::make_shared<StencilImpl<SelectorData>>(std::move(Selector));
 }
 
diff --git a/clang/lib/Tooling/Transformer/Transformer.cpp b/clang/lib/Tooling/Transformer/Transformer.cpp
index 71340bf2f676d..e8fc00c4e953f 100644
--- a/clang/lib/Tooling/Transformer/Transformer.cpp
+++ b/clang/lib/Tooling/Transformer/Transformer.cpp
@@ -53,7 +53,7 @@ void Transformer::run(const MatchFinder::MatchResult &Result) {
     auto ID = Result.SourceManager->getFileID(T.Range.getBegin());
     auto Iter = ChangesByFileID
                     .emplace(ID, AtomicChange(*Result.SourceManager,
-                                              T.Range.getBegin()))
+                                              T.Range.getBegin(), T.Metadata))
                     .first;
     auto &AC = Iter->second;
     if (auto Err = AC.replace(*Result.SourceManager, T.Range, T.Replacement)) {
diff --git a/clang/test/AST/Inputs/std-coroutine.h b/clang/test/AST/Inputs/std-coroutine.h
index 7a424f1e99cf0..c1be9a19b8102 100644
--- a/clang/test/AST/Inputs/std-coroutine.h
+++ b/clang/test/AST/Inputs/std-coroutine.h
@@ -10,12 +10,12 @@ struct coroutine_traits { using promise_type = typename Ret::promise_type; };
 
 template <class Promise = void>
 struct coroutine_handle {
-  static coroutine_handle from_address(void *);
+  static coroutine_handle from_address(void *) noexcept;
 };
 template <>
 struct coroutine_handle<void> {
   template <class PromiseType>
-  coroutine_handle(coroutine_handle<PromiseType>);
+  coroutine_handle(coroutine_handle<PromiseType>) noexcept;
   static coroutine_handle from_address(void *);
 };
 
@@ -26,9 +26,9 @@ struct suspend_always {
 };
 
 struct suspend_never {
-  bool await_ready() { return true; }
-  void await_suspend(coroutine_handle<>) {}
-  void await_resume() {}
+  bool await_ready() noexcept { return true; }
+  void await_suspend(coroutine_handle<>) noexcept {}
+  void await_resume() noexcept {}
 };
 
 } // namespace experimental
diff --git a/clang/test/AST/alignas_maybe_odr_cleanup.cpp b/clang/test/AST/alignas_maybe_odr_cleanup.cpp
index cdff1bfe8b80e..3adef4cba1440 100644
--- a/clang/test/AST/alignas_maybe_odr_cleanup.cpp
+++ b/clang/test/AST/alignas_maybe_odr_cleanup.cpp
@@ -15,8 +15,9 @@ struct FOO {
   }
 };
 
-// CHECK: AlignedAttr {{.*}} alignas
-// CHECK: ConstantExpr {{.+}} 'int' Int: 32
-// CHECK: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
-// CHECK: DeclRefExpr {{.*}} 'const int' lvalue Var {{.*}} 'vec_align_bytes' 'const int' non_odr_use_constant
-// CHECK: NullStmt
+// CHECK:      |   `-AlignedAttr {{.*}} <col:14> alignas
+// CHECK-NEXT:      |     `-ConstantExpr {{.*}} <col:22> 'int'
+// CHECK-NEXT:      |       |-value: Int 32
+// CHECK-NEXT:      |       `-ImplicitCastExpr {{.*}} <col:22> 'int' <LValueToRValue>
+// CHECK-NEXT:      |         `-DeclRefExpr {{.*}} <col:22> 'const int' lvalue Var {{.*}} 'vec_align_bytes' 'const int' non_odr_use_constant
+// CHECK-NEXT:      `-NullStmt {{.*}} <line:14:5>
diff --git a/clang/test/AST/ast-dump-APValue-anon-union.cpp b/clang/test/AST/ast-dump-APValue-anon-union.cpp
new file mode 100644
index 0000000000000..1c9480c5a9430
--- /dev/null
+++ b/clang/test/AST/ast-dump-APValue-anon-union.cpp
@@ -0,0 +1,52 @@
+// Test without serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
+// RUN:            -ast-dump %s -ast-dump-filter Test \
+// RUN: | FileCheck --strict-whitespace --match-full-lines %s
+//
+// Test with serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 -emit-pch -o %t %s
+// RUN: %clang_cc1 -x c++ -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
+// RUN:           -include-pch %t -ast-dump-all -ast-dump-filter Test /dev/null \
+// RUN: | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" \
+// RUN: | FileCheck --strict-whitespace --match-full-lines %s
+
+struct S0 {
+  union {
+    int i = 42;
+  };
+};
+
+union U0 {
+  union {
+    float f = 3.1415f;
+  };
+};
+
+union U1 {
+  union {
+    float f;
+  };
+};
+
+void Test() {
+  constexpr S0 s0{};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} s0 'const S0' constexpr listinit
+  // CHECK-NEXT:  |   |-value: Struct
+  // CHECK-NEXT:  |   | `-field: Union .i Int 42
+
+  constexpr U0 u0a{};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} u0a 'const U0' constexpr listinit
+  // CHECK-NEXT:  |   |-value: Union None
+
+  constexpr U0 u0b{3.1415f};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} u0b 'const U0' constexpr listinit
+  // CHECK-NEXT:  |   |-value: Union . Union .f Float 3.141500e+00
+
+  constexpr U1 u1a{};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} u1a 'const U1' constexpr listinit
+  // CHECK-NEXT:  |   |-value: Union . Union .f Float 0.000000e+00
+
+  constexpr U1 u1b{3.1415f};
+  // CHECK:    `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} u1b 'const U1' constexpr listinit
+  // CHECK-NEXT:      |-value: Union . Union .f Float 3.141500e+00
+}
diff --git a/clang/test/AST/ast-dump-APValue-arithmetic.cpp b/clang/test/AST/ast-dump-APValue-arithmetic.cpp
new file mode 100644
index 0000000000000..835d8c8108346
--- /dev/null
+++ b/clang/test/AST/ast-dump-APValue-arithmetic.cpp
@@ -0,0 +1,49 @@
+// Test without serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
+// RUN:            -ast-dump %s -ast-dump-filter Test \
+// RUN: | FileCheck --strict-whitespace --match-full-lines %s
+//
+// Test with serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 -emit-pch -o %t %s
+// RUN: %clang_cc1 -x c++ -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
+// RUN:           -include-pch %t -ast-dump-all -ast-dump-filter Test /dev/null \
+// RUN: | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" \
+// RUN: | FileCheck --strict-whitespace --match-full-lines %s
+
+void Test() {
+  constexpr int Int = 42;
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} Int 'const int' constexpr cinit
+  // CHECK-NEXT:  |   |-value: Int 42
+
+  constexpr __int128 Int128 = (__int128)0xFFFFFFFFFFFFFFFF + (__int128)1;
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} Int128 'const __int128' constexpr cinit
+  // CHECK-NEXT:  |   |-value: Int 18446744073709551616
+
+  constexpr float Float = 3.1415f;
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} Float 'const float' constexpr cinit
+  // CHECK-NEXT:  |   |-value: Float 3.141500e+00
+
+  constexpr double Double = 3.1415f;
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} Double 'const double' constexpr cinit
+  // CHECK-NEXT:  |   |-value: Float 3.141500e+00
+
+  constexpr _Complex int ComplexInt = 42 + 24i;
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} referenced ComplexInt 'const _Complex int' constexpr cinit
+  // CHECK-NEXT:  |   |-value: ComplexInt 42 + 24i
+
+  constexpr _Complex float ComplexFloat = 3.1415f + 42i;
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} referenced ComplexFloat 'const _Complex float' constexpr cinit
+  // CHECK-NEXT:  |   |-value: ComplexFloat 3.141500e+00 + 4.200000e+01i
+
+  constexpr _Complex int ArrayOfComplexInt[10] = {ComplexInt, ComplexInt, ComplexInt, ComplexInt};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} ArrayOfComplexInt '_Complex int const[10]' constexpr cinit
+  // CHECK-NEXT:  |   |-value: Array size=10
+  // CHECK-NEXT:  |   | |-elements: ComplexInt 42 + 24i, ComplexInt 42 + 24i, ComplexInt 42 + 24i, ComplexInt 42 + 24i
+  // CHECK-NEXT:  |   | `-filler: 6 x ComplexInt 0 + 0i
+
+  constexpr _Complex float ArrayOfComplexFloat[10] = {ComplexFloat, ComplexFloat, ComplexInt, ComplexInt};
+  // CHECK:    `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} ArrayOfComplexFloat '_Complex float const[10]' constexpr cinit
+  // CHECK-NEXT:      |-value: Array size=10
+  // CHECK-NEXT:      | |-elements: ComplexFloat 3.141500e+00 + 4.200000e+01i, ComplexFloat 3.141500e+00 + 4.200000e+01i, ComplexFloat 4.200000e+01 + 2.400000e+01i, ComplexFloat 4.200000e+01 + 2.400000e+01i
+  // CHECK-NEXT:      | `-filler: 6 x ComplexFloat 0.000000e+00 + 0.000000e+00i
+}
diff --git a/clang/test/AST/ast-dump-APValue-array.cpp b/clang/test/AST/ast-dump-APValue-array.cpp
new file mode 100644
index 0000000000000..f9b38ec332a5b
--- /dev/null
+++ b/clang/test/AST/ast-dump-APValue-array.cpp
@@ -0,0 +1,82 @@
+// Test without serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
+// RUN:            -ast-dump %s -ast-dump-filter Test \
+// RUN: | FileCheck --strict-whitespace --match-full-lines %s
+//
+// Test with serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 -emit-pch -o %t %s
+// RUN: %clang_cc1 -x c++ -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
+// RUN:           -include-pch %t -ast-dump-all -ast-dump-filter Test /dev/null \
+// RUN: | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" \
+// RUN: | FileCheck --strict-whitespace --match-full-lines %s
+
+struct S0 {
+  int arr[2];
+};
+union U0 {
+  int i;
+  float f;
+};
+
+struct S1 {
+  S0 s0 = {1, 2};
+  U0 u0 = {.i = 42};
+};
+
+void Test() {
+  constexpr int __attribute__((vector_size(sizeof(int) * 5))) arr_v5i[5] = {
+      {1, 2, 3, 4, 5},
+      {1, 2, 3, 4},
+  };
+  // CHECK:  | `-VarDecl {{.*}} <line:{{.*}}, line:{{.*}}> line:{{.*}} arr_v5i '__attribute__((__vector_size__(5 * sizeof(int)))) int const[5]' constexpr cinit
+  // CHECK-NEXT:  |   |-value: Array size=5
+  // CHECK-NEXT:  |   | |-element: Vector length=5
+  // CHECK-NEXT:  |   | | |-elements: Int 1, Int 2, Int 3, Int 4
+  // CHECK-NEXT:  |   | | `-element: Int 5
+  // CHECK-NEXT:  |   | |-element: Vector length=5
+  // CHECK-NEXT:  |   | | |-elements: Int 1, Int 2, Int 3, Int 4
+  // CHECK-NEXT:  |   | | `-element: Int 0
+  // CHECK-NEXT:  |   | `-filler: 3 x Vector length=5
+  // CHECK-NEXT:  |   |   |-elements: Int 0, Int 0, Int 0, Int 0
+  // CHECK-NEXT:  |   |   `-element: Int 0
+
+  constexpr float arr_f[3][5] = {
+      {1, 2, 3, 4, 5},
+  };
+  // CHECK:  | `-VarDecl {{.*}} <line:{{.*}}, line:{{.*}}> line:{{.*}} arr_f 'float const[3][5]' constexpr cinit
+  // CHECK-NEXT:  |   |-value: Array size=3
+  // CHECK-NEXT:  |   | |-element: Array size=5
+  // CHECK-NEXT:  |   | | |-elements: Float 1.000000e+00, Float 2.000000e+00, Float 3.000000e+00, Float 4.000000e+00
+  // CHECK-NEXT:  |   | | `-element: Float 5.000000e+00
+  // CHECK-NEXT:  |   | `-filler: 2 x Array size=5
+  // CHECK-NEXT:  |   |   `-filler: 5 x Float 0.000000e+00
+
+  constexpr S0 arr_s0[2] = {{1, 2}, {3, 4}};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} arr_s0 'S0 const[2]' constexpr cinit
+  // CHECK-NEXT:  |   |-value: Array size=2
+  // CHECK-NEXT:  |   | |-element: Struct
+  // CHECK-NEXT:  |   | | `-field: Array size=2
+  // CHECK-NEXT:  |   | |   `-elements: Int 1, Int 2
+  // CHECK-NEXT:  |   | `-element: Struct
+  // CHECK-NEXT:  |   |   `-field: Array size=2
+  // CHECK-NEXT:  |   |     `-elements: Int 3, Int 4
+
+  constexpr U0 arr_u0[2] = {{.i = 42}, {.f = 3.1415f}};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} arr_u0 'U0 const[2]' constexpr cinit
+  // CHECK-NEXT:  |   |-value: Array size=2
+  // CHECK-NEXT:  |   | `-elements: Union .i Int 42, Union .f Float 3.141500e+00
+
+  constexpr S1 arr_s1[2] = {};
+  // CHECK:    `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} arr_s1 'S1 const[2]' constexpr cinit
+  // CHECK-NEXT:      |-value: Array size=2
+  // CHECK-NEXT:      | |-element: Struct
+  // CHECK-NEXT:      | | |-field: Struct
+  // CHECK-NEXT:      | | | `-field: Array size=2
+  // CHECK-NEXT:      | | |   `-elements: Int 1, Int 2
+  // CHECK-NEXT:      | | `-field: Union .i Int 42
+  // CHECK-NEXT:      | `-element: Struct
+  // CHECK-NEXT:      |   |-field: Struct
+  // CHECK-NEXT:      |   | `-field: Array size=2
+  // CHECK-NEXT:      |   |   `-elements: Int 1, Int 2
+  // CHECK-NEXT:      |   `-field: Union .i Int 42
+}
diff --git a/clang/test/AST/ast-dump-APValue-struct.cpp b/clang/test/AST/ast-dump-APValue-struct.cpp
new file mode 100644
index 0000000000000..4730404abc287
--- /dev/null
+++ b/clang/test/AST/ast-dump-APValue-struct.cpp
@@ -0,0 +1,113 @@
+// Test without serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
+// RUN:            -ast-dump %s -ast-dump-filter Test \
+// RUN: | FileCheck --strict-whitespace --match-full-lines %s
+//
+// Test with serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 -emit-pch -o %t %s
+// RUN: %clang_cc1 -x c++ -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
+// RUN:           -include-pch %t -ast-dump-all -ast-dump-filter Test /dev/null \
+// RUN: | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" \
+// RUN: | FileCheck --strict-whitespace --match-full-lines %s
+
+struct S0 {
+  int i = 0;
+  union {
+    int j = 0;
+  } u0;
+};
+
+struct S1 {
+  int i = 0;
+  union {
+    struct {
+      int j = 0;
+    } s;
+  } u1;
+};
+
+struct S2 {
+  int i = 0;
+  union {
+    union {
+      int j = 0;
+    } u;
+  } u2;
+};
+
+struct S3 {
+  int i = 0;
+  union {
+    union {
+      struct {
+        int j = 0;
+      } j;
+    } u;
+  } u3;
+};
+
+struct S4 : S0 {
+  int i = 1, j = 2, k = 3;
+  struct {
+  } s;
+  int a = 4, b = 5, c = 6;
+};
+
+struct S5 : S4 {
+  int arr0[8] = {1, 2, 3, 4};
+  int arr1[8] = {1, 2, 3, 4, 0, 0, 0, 0};
+};
+
+void Test() {
+  constexpr S0 s0{};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} s0 'const S0' constexpr listinit
+  // CHECK-NEXT:  |   |-value: Struct
+  // CHECK-NEXT:  |   | `-fields: Int 0, Union .j Int 0
+
+  constexpr S1 s1{};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} s1 'const S1' constexpr listinit
+  // CHECK-NEXT:  |   |-value: Struct
+  // CHECK-NEXT:  |   | |-field: Int 0
+  // CHECK-NEXT:  |   | `-field: Union .s
+  // CHECK-NEXT:  |   |   `-Struct
+  // CHECK-NEXT:  |   |     `-field: Int 0
+
+  constexpr S2 s2{};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} s2 'const S2' constexpr listinit
+  // CHECK-NEXT:  |   |-value: Struct
+  // CHECK-NEXT:  |   | `-fields: Int 0, Union .u Union .j Int 0
+
+  constexpr S3 s3{};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} s3 'const S3' constexpr listinit
+  // CHECK-NEXT:  |   |-value: Struct
+  // CHECK-NEXT:  |   | |-field: Int 0
+  // CHECK-NEXT:  |   | `-field: Union .u
+  // CHECK-NEXT:  |   |   `-Union .j
+  // CHECK-NEXT:  |   |     `-Struct
+  // CHECK-NEXT:  |   |       `-field: Int 0
+
+  constexpr S4 s4{};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} s4 'const S4' constexpr listinit
+  // CHECK-NEXT:  |   |-value: Struct
+  // CHECK-NEXT:  |   | |-base: Struct
+  // CHECK-NEXT:  |   | | `-fields: Int 0, Union .j Int 0
+  // CHECK-NEXT:  |   | |-fields: Int 1, Int 2, Int 3
+  // CHECK-NEXT:  |   | |-field: Struct
+  // CHECK-NEXT:  |   | `-fields: Int 4, Int 5, Int 6
+
+  constexpr S5 s5{};
+  // CHECK:    `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} s5 'const S5' constexpr listinit
+  // CHECK-NEXT:      |-value: Struct
+  // CHECK-NEXT:      | |-base: Struct
+  // CHECK-NEXT:      | | |-base: Struct
+  // CHECK-NEXT:      | | | `-fields: Int 0, Union .j Int 0
+  // CHECK-NEXT:      | | |-fields: Int 1, Int 2, Int 3
+  // CHECK-NEXT:      | | |-field: Struct
+  // CHECK-NEXT:      | | `-fields: Int 4, Int 5, Int 6
+  // CHECK-NEXT:      | |-field: Array size=8
+  // CHECK-NEXT:      | | |-elements: Int 1, Int 2, Int 3, Int 4
+  // CHECK-NEXT:      | | `-filler: 4 x Int 0
+  // CHECK-NEXT:      | `-field: Array size=8
+  // CHECK-NEXT:      |   |-elements: Int 1, Int 2, Int 3, Int 4
+  // CHECK-NEXT:      |   `-elements: Int 0, Int 0, Int 0, Int 0
+}
diff --git a/clang/test/AST/ast-dump-APValue-todo.cpp b/clang/test/AST/ast-dump-APValue-todo.cpp
new file mode 100644
index 0000000000000..78cc9cf36c73c
--- /dev/null
+++ b/clang/test/AST/ast-dump-APValue-todo.cpp
@@ -0,0 +1,26 @@
+// Test without serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
+// RUN:            -ast-dump %s -ast-dump-filter Test \
+// RUN: | FileCheck --strict-whitespace --match-full-lines %s
+//
+// Test with serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 -emit-pch -o %t %s
+// RUN: %clang_cc1 -x c++ -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
+// RUN:           -include-pch %t -ast-dump-all -ast-dump-filter Test /dev/null \
+// RUN: | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" \
+// RUN: | FileCheck --strict-whitespace --match-full-lines %s
+
+int i;
+struct S {
+  int i;
+};
+
+void Test() {
+  constexpr int *pi = &i;
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} pi 'int *const' constexpr cinit
+  // CHECK-NEXT:  |   |-value: LValue <todo>
+
+  constexpr int(S::*pmi) = &S::i;
+  // CHECK:    `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} pmi 'int (S::*const)' constexpr cinit
+  // CHECK-NEXT:      |-value: MemberPointer <todo>
+}
diff --git a/clang/test/AST/ast-dump-APValue-union.cpp b/clang/test/AST/ast-dump-APValue-union.cpp
new file mode 100644
index 0000000000000..c717b6ece7382
--- /dev/null
+++ b/clang/test/AST/ast-dump-APValue-union.cpp
@@ -0,0 +1,63 @@
+// Test without serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
+// RUN:            -ast-dump %s -ast-dump-filter Test \
+// RUN: | FileCheck --strict-whitespace --match-full-lines %s
+//
+// Test with serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 -emit-pch -o %t %s
+// RUN: %clang_cc1 -x c++ -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
+// RUN:           -include-pch %t -ast-dump-all -ast-dump-filter Test /dev/null \
+// RUN: | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" \
+// RUN: | FileCheck --strict-whitespace --match-full-lines %s
+
+union U0 {
+  int i = 42;
+  float f;
+};
+
+union U1 {
+  union Uinner {
+    int i;
+    float f = 3.1415f;
+  } uinner;
+};
+
+union U2 {
+  union Uinner {
+    double d;
+    int arr[2] = {1, 2};
+  } uinner;
+};
+
+union U3 {
+  union Uinner {
+    double d = 3.1415;
+    int arr[2];
+  } uinner;
+  float f;
+};
+
+void Test() {
+  constexpr U0 u0{};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} u0 'const U0' constexpr listinit
+  // CHECK-NEXT:  |   |-value: Union .i Int 42
+
+  constexpr U1 u1{};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} u1 'const U1' constexpr listinit
+  // CHECK-NEXT:  |   |-value: Union .uinner Union .f Float 3.141500e+00
+
+  constexpr U2 u2{};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} u2 'const U2' constexpr listinit
+  // CHECK-NEXT:  |   |-value: Union .uinner
+  // CHECK-NEXT:  |   | `-Union .arr
+  // CHECK-NEXT:  |   |   `-Array size=2
+  // CHECK-NEXT:  |   |     `-elements: Int 1, Int 2
+
+  constexpr U3 u3a = {.f = 3.1415};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} u3a 'const U3' constexpr cinit
+  // CHECK-NEXT:  |   |-value: Union .f Float 3.141500e+00
+
+  constexpr U3 u3b = {.uinner = {}};
+  // CHECK:    `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} u3b 'const U3' constexpr cinit
+  // CHECK-NEXT:      |-value: Union .uinner Union .d Float 3.141500e+00
+}
diff --git a/clang/test/AST/ast-dump-APValue-vector.cpp b/clang/test/AST/ast-dump-APValue-vector.cpp
new file mode 100644
index 0000000000000..a9ec1d361c412
--- /dev/null
+++ b/clang/test/AST/ast-dump-APValue-vector.cpp
@@ -0,0 +1,43 @@
+// Test without serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
+// RUN:            -ast-dump %s -ast-dump-filter Test \
+// RUN: | FileCheck --strict-whitespace --match-full-lines %s
+//
+// Test with serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 -emit-pch -o %t %s
+// RUN: %clang_cc1 -x c++ -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
+// RUN:           -include-pch %t -ast-dump-all -ast-dump-filter Test /dev/null \
+// RUN: | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" \
+// RUN: | FileCheck --strict-whitespace --match-full-lines %s
+
+void Test() {
+  constexpr int __attribute__((vector_size(sizeof(int) * 1))) v1i = {1};
+  // CHECK:  |-DeclStmt {{.*}} <line:{{.*}}, col:{{.*}}>
+  // CHECK-NEXT:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} v1i '__attribute__((__vector_size__(1 * sizeof(int)))) int const' constexpr cinit
+  // CHECK-NEXT:  |   |-value: Vector length=1
+  // CHECK-NEXT:  |   | `-element: Int 1
+
+  constexpr int __attribute__((vector_size(sizeof(int) * 4))) v4i = {1, 2, 3, 4};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} v4i '__attribute__((__vector_size__(4 * sizeof(int)))) int const' constexpr cinit
+  // CHECK-NEXT:  |   |-value: Vector length=4
+  // CHECK-NEXT:  |   | `-elements: Int 1, Int 2, Int 3, Int 4
+
+  constexpr int __attribute__((vector_size(sizeof(int) * 5))) v5i = {1, 2, 3, 4};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} v5i '__attribute__((__vector_size__(5 * sizeof(int)))) int const' constexpr cinit
+  // CHECK-NEXT:  |   |-value: Vector length=5
+  // CHECK-NEXT:  |   | |-elements: Int 1, Int 2, Int 3, Int 4
+  // CHECK-NEXT:  |   | `-element: Int 0
+
+  constexpr int __attribute__((vector_size(sizeof(int) * 8))) v8i = {1, 2, 3, 4};
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} v8i '__attribute__((__vector_size__(8 * sizeof(int)))) int const' constexpr cinit
+  // CHECK-NEXT:  |   |-value: Vector length=8
+  // CHECK-NEXT:  |   | |-elements: Int 1, Int 2, Int 3, Int 4
+  // CHECK-NEXT:  |   | `-elements: Int 0, Int 0, Int 0, Int 0
+
+  constexpr int __attribute__((vector_size(sizeof(int) * 9))) v9i = {1, 2, 3, 4};
+  // CHECK:    `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} v9i '__attribute__((__vector_size__(9 * sizeof(int)))) int const' constexpr cinit
+  // CHECK-NEXT:      |-value: Vector length=9
+  // CHECK-NEXT:      | |-elements: Int 1, Int 2, Int 3, Int 4
+  // CHECK-NEXT:      | |-elements: Int 0, Int 0, Int 0, Int 0
+  // CHECK-NEXT:      | `-element: Int 0
+}
diff --git a/clang/test/AST/ast-dump-attr.cpp b/clang/test/AST/ast-dump-attr.cpp
index 50eef3e0d97a0..95491a02f8b2d 100644
--- a/clang/test/AST/ast-dump-attr.cpp
+++ b/clang/test/AST/ast-dump-attr.cpp
@@ -45,6 +45,7 @@ int TestAlignedExpr __attribute__((aligned(4)));
 // CHECK:      VarDecl{{.*}}TestAlignedExpr
 // CHECK-NEXT:   AlignedAttr {{.*}} aligned
 // CHECK-NEXT:     ConstantExpr
+// CHECK-NEXT:       value: Int 4
 // CHECK-NEXT:       IntegerLiteral
 
 int TestEnum __attribute__((visibility("default")));
diff --git a/clang/test/AST/ast-dump-color.cpp b/clang/test/AST/ast-dump-color.cpp
index 6651ef2652e02..baaf9729c7ce7 100644
--- a/clang/test/AST/ast-dump-color.cpp
+++ b/clang/test/AST/ast-dump-color.cpp
@@ -49,13 +49,15 @@ struct Invalid {
 //CHECK: {{^}}[[Blue]]| |   |-[[RESET]][[MAGENTA]]IntegerLiteral[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:10:11[[RESET]]> [[Green]]'int'[[RESET]][[Cyan:.\[0;36m]][[RESET]][[Cyan]][[RESET]][[CYAN]] 1[[RESET]]{{$}}
 //CHECK: {{^}}[[Blue]]| |   `-[[RESET]][[MAGENTA]]CompoundStmt[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:14[[RESET]], [[Yellow]]line:15:3[[RESET]]>{{$}}
 //CHECK: {{^}}[[Blue]]| |     |-[[RESET]][[MAGENTA]]CaseStmt[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:11:3[[RESET]], [[Yellow]]line:12:27[[RESET]]>{{$}}
-//CHECK: {{^}}[[Blue]]| |     | |-[[RESET]][[MAGENTA]]ConstantExpr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:11:8[[RESET]]> [[Green]]'int'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]][[CYAN]] Int: 1[[RESET]]{{$}}
+//CHECK: {{^}}[[Blue]]| |     | |-[[RESET]][[MAGENTA]]ConstantExpr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:11:8[[RESET]]> [[Green]]'int'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]]{{$}}
+//CHECK: {{^}}[[Blue]]| |     | | |-value: [[RESET]][[Cyan]]Int [[CYAN]]1[[RESET]][[RESET]]{{$}}
 //CHECK: {{^}}[[Blue]]| |     | | `-[[RESET]][[MAGENTA]]IntegerLiteral[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:8[[RESET]]> [[Green]]'int'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]][[CYAN]] 1[[RESET]]{{$}}
 //CHECK: {{^}}[[Blue]]| |     | `-[[RESET]][[MAGENTA]]AttributedStmt[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:12:5[[RESET]], [[Yellow]]col:27[[RESET]]>{{$}}
 //CHECK: {{^}}[[Blue]]| |     |   |-[[RESET]][[BLUE]]FallThroughAttr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:7[[RESET]], [[Yellow]]col:14[[RESET]]>{{$}}
 //CHECK: {{^}}[[Blue]]| |     |   `-[[RESET]][[MAGENTA]]NullStmt[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:27[[RESET]]>{{$}}
 //CHECK: {{^}}[[Blue]]| |     `-[[RESET]][[MAGENTA]]CaseStmt[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:13:3[[RESET]], [[Yellow]]line:14:5[[RESET]]>{{$}}
-//CHECK: {{^}}[[Blue]]| |       |-[[RESET]][[MAGENTA]]ConstantExpr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:13:8[[RESET]]> [[Green]]'int'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]][[CYAN]] Int: 2[[RESET]]{{$}}
+//CHECK: {{^}}[[Blue]]| |       |-[[RESET]][[MAGENTA]]ConstantExpr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:13:8[[RESET]]> [[Green]]'int'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]]{{$}}
+//CHECK: {{^}}[[Blue]]| |       | |-value: [[RESET]][[Cyan]]Int [[CYAN]]2[[RESET]][[RESET]]{{$}}
 //CHECK: {{^}}[[Blue]]| |       | `-[[RESET]][[MAGENTA]]IntegerLiteral[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:8[[RESET]]> [[Green]]'int'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]][[CYAN]] 2[[RESET]]{{$}}
 //CHECK: {{^}}[[Blue]]| |       `-[[RESET]][[MAGENTA]]NullStmt[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:14:5[[RESET]]>{{$}}
 //CHECK: {{^}}[[Blue]]| `-[[RESET]][[Blue]]FullComment[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:8:4[[RESET]], [[Yellow]]col:11[[RESET]]>{{$}}
diff --git a/clang/test/AST/ast-dump-comment.cpp b/clang/test/AST/ast-dump-comment.cpp
index da73483f041a3..8df0416cd092b 100644
--- a/clang/test/AST/ast-dump-comment.cpp
+++ b/clang/test/AST/ast-dump-comment.cpp
@@ -1,4 +1,12 @@
-// RUN: %clang_cc1 -Wdocumentation -ast-dump -ast-dump-filter Test %s | FileCheck -strict-whitespace %s
+// Without serialization:
+// RUN: %clang_cc1 -Wdocumentation -ast-dump -ast-dump-filter Test %s \
+// RUN: | FileCheck -strict-whitespace %s
+//
+// With serialization:
+// RUN: %clang_cc1 -Wdocumentation -emit-pch -o %t %s
+// RUN: %clang_cc1 -x c++ -Wdocumentation -include-pch %t -ast-dump-all -ast-dump-filter Test /dev/null \
+// RUN: | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" \
+// RUN: | FileCheck --strict-whitespace %s
 
 /// Aaa
 int TestLocation;
diff --git a/clang/test/AST/ast-dump-constant-expr.cpp b/clang/test/AST/ast-dump-constant-expr.cpp
index c9ddb245ef647..43f68cac72067 100644
--- a/clang/test/AST/ast-dump-constant-expr.cpp
+++ b/clang/test/AST/ast-dump-constant-expr.cpp
@@ -54,27 +54,32 @@ void Test() {
 // CHECK-NEXT:FunctionDecl {{.*}} <{{.*}}ast-dump-constant-expr.cpp:42:1, line:52:1> line:42:6{{( imported)?}} Test 'void ()'
 // CHECK-NEXT:`-CompoundStmt {{.*}} <col:13, line:52:1>
 // CHECK-NEXT:  |-CStyleCastExpr {{.*}} <line:43:3, col:19> 'void' <ToVoid>
-// CHECK-NEXT:  | `-ConstantExpr {{.*}} <col:10, col:19> 'int' Int: 42
+// CHECK-NEXT:  | `-ConstantExpr {{.*}} <col:10, col:19> 'int'
+// CHECK-NEXT:  |   |-value: Int 42
 // CHECK-NEXT:  |   `-CallExpr {{.*}} <col:10, col:19> 'int'
 // CHECK-NEXT:  |     `-ImplicitCastExpr {{.*}} <col:10> 'int (*)()' <FunctionToPointerDecay>
 // CHECK-NEXT:  |       `-DeclRefExpr {{.*}} <col:10> 'int ()' lvalue Function {{.*}} 'test_Int' 'int ()'
 // CHECK-NEXT:  |-CStyleCastExpr {{.*}} <line:44:3, col:21> 'void' <ToVoid>
-// CHECK-NEXT:  | `-ConstantExpr {{.*}} <col:10, col:21> 'float' Float: 1.000000e+00
+// CHECK-NEXT:  | `-ConstantExpr {{.*}} <col:10, col:21> 'float'
+// CHECK-NEXT:  |   |-value: Float 1.000000e+00
 // CHECK-NEXT:  |   `-CallExpr {{.*}} <col:10, col:21> 'float'
 // CHECK-NEXT:  |     `-ImplicitCastExpr {{.*}} <col:10> 'float (*)()' <FunctionToPointerDecay>
 // CHECK-NEXT:  |       `-DeclRefExpr {{.*}} <col:10> 'float ()' lvalue Function {{.*}} 'test_Float' 'float ()'
 // CHECK-NEXT:  |-CStyleCastExpr {{.*}} <line:45:3, col:26> 'void' <ToVoid>
-// CHECK-NEXT:  | `-ConstantExpr {{.*}} <col:10, col:26> '_Complex int' ComplexInt: 1, 2
+// CHECK-NEXT:  | `-ConstantExpr {{.*}} <col:10, col:26> '_Complex int'
+// CHECK-NEXT:  |   |-value: ComplexInt 1 + 2i
 // CHECK-NEXT:  |   `-CallExpr {{.*}} <col:10, col:26> '_Complex int'
 // CHECK-NEXT:  |     `-ImplicitCastExpr {{.*}} <col:10> '_Complex int (*)()' <FunctionToPointerDecay>
 // CHECK-NEXT:  |       `-DeclRefExpr {{.*}} <col:10> '_Complex int ()' lvalue Function {{.*}} 'test_ComplexInt' '_Complex int ()'
 // CHECK-NEXT:  |-CStyleCastExpr {{.*}} <line:46:3, col:28> 'void' <ToVoid>
-// CHECK-NEXT:  | `-ConstantExpr {{.*}} <col:10, col:28> '_Complex float' ComplexFloat: 1.200000e+00, 3.400000e+00
+// CHECK-NEXT:  | `-ConstantExpr {{.*}} <col:10, col:28> '_Complex float'
+// CHECK-NEXT:  |   |-value: ComplexFloat 1.200000e+00 + 3.400000e+00i
 // CHECK-NEXT:  |   `-CallExpr {{.*}} <col:10, col:28> '_Complex float'
 // CHECK-NEXT:  |     `-ImplicitCastExpr {{.*}} <col:10> '_Complex float (*)()' <FunctionToPointerDecay>
 // CHECK-NEXT:  |       `-DeclRefExpr {{.*}} <col:10> '_Complex float ()' lvalue Function {{.*}} 'test_ComplexFloat' '_Complex float ()'
 // CHECK-NEXT:  `-CStyleCastExpr {{.*}} <line:47:3, col:22> 'void' <ToVoid>
-// CHECK-NEXT:    `-ConstantExpr {{.*}} <col:10, col:22> '__int128' Int: 18446744073709551616
+// CHECK-NEXT:    `-ConstantExpr {{.*}} <col:10, col:22> '__int128'
+// CHECK-NEXT:      |-value: Int 18446744073709551616
 // CHECK-NEXT:      `-CallExpr {{.*}} <col:10, col:22> '__int128'
 // CHECK-NEXT:        `-ImplicitCastExpr {{.*}} <col:10> '__int128 (*)()' <FunctionToPointerDecay>
 // CHECK-NEXT:          `-DeclRefExpr {{.*}} <col:10> '__int128 ()' lvalue Function {{.*}} 'test_Int128' '__int128 ()'
diff --git a/clang/test/AST/ast-dump-decl.cpp b/clang/test/AST/ast-dump-decl.cpp
index 3809683bd9dc7..97bb7964c47b8 100644
--- a/clang/test/AST/ast-dump-decl.cpp
+++ b/clang/test/AST/ast-dump-decl.cpp
@@ -224,19 +224,27 @@ namespace testFunctionTemplateDecl {
   // CHECK-NEXT:  | `-CompoundStmt 0x{{.+}} <col:53, col:55>
   // CHECK-NEXT:  |-FunctionDecl 0x{{.+}} <col:24, col:55> col:29 used TestFunctionTemplate 'void (testFunctionTemplateDecl::A)'
   // CHECK-NEXT:  | |-TemplateArgument type 'testFunctionTemplateDecl::A'
+  // CHECK-NEXT:  | | `-RecordType 0{{.+}} 'testFunctionTemplateDecl::A'
+  // CHECK-NEXT:  | |   `-CXXRecord 0x{{.+}} 'A'
   // CHECK-NEXT:  | |-ParmVarDecl 0x{{.+}} <col:50> col:51 'testFunctionTemplateDecl::A':'testFunctionTemplateDecl::A'
   // CHECK-NEXT:  | `-CompoundStmt 0x{{.+}} <col:53, col:55>
   // CHECK-NEXT:  |-Function 0x{{.+}} 'TestFunctionTemplate' 'void (testFunctionTemplateDecl::B)'
   // CHECK-NEXT:  |-FunctionDecl 0x{{.+}} <col:24, col:55> col:29 TestFunctionTemplate 'void (testFunctionTemplateDecl::C)'
   // CHECK-NEXT:  | |-TemplateArgument type 'testFunctionTemplateDecl::C'
+  // CHECK-NEXT:  | | `-RecordType 0{{.+}} 'testFunctionTemplateDecl::C'
+  // CHECK-NEXT:  | |   `-CXXRecord 0x{{.+}} 'C'
   // CHECK-NEXT:  | `-ParmVarDecl 0x{{.+}} <col:50> col:51 'testFunctionTemplateDecl::C':'testFunctionTemplateDecl::C'
   // CHECK-NEXT:  `-FunctionDecl 0x{{.+}} <col:24, col:55> col:29 TestFunctionTemplate 'void (testFunctionTemplateDecl::D)'
   // CHECK-NEXT:    |-TemplateArgument type 'testFunctionTemplateDecl::D'
+  // CHECK-NEXT:    | `-RecordType 0{{.+}} 'testFunctionTemplateDecl::D'
+  // CHECK-NEXT:    |   `-CXXRecord 0x{{.+}} 'D'
   // CHECK-NEXT:    |-ParmVarDecl 0x{{.+}} <col:50> col:51 'testFunctionTemplateDecl::D':'testFunctionTemplateDecl::D'
   // CHECK-NEXT:    `-CompoundStmt 0x{{.+}} <col:53, col:55>
 
-  // CHECK:       FunctionDecl 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-26]]:3, col:41> col:19 TestFunctionTemplate 'void (testFunctionTemplateDecl::B)'
+  // CHECK:       FunctionDecl 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-32]]:3, col:41> col:19 TestFunctionTemplate 'void (testFunctionTemplateDecl::B)'
   // CHECK-NEXT:  |-TemplateArgument type 'testFunctionTemplateDecl::B'
+  // CHECK-NEXT:  | `-RecordType 0{{.+}} 'testFunctionTemplateDecl::B'
+  // CHECK-NEXT:  |   `-CXXRecord 0x{{.+}} 'B'
   // CHECK-NEXT:  `-ParmVarDecl 0x{{.+}} <col:40> col:41 'testFunctionTemplateDecl::B'
 
 
@@ -311,19 +319,21 @@ namespace testClassTemplateDecl {
 // CHECK-NEXT:  | | |-MoveAssignment
 // CHECK-NEXT:  | | `-Destructor non_trivial user_declared
 // CHECK-NEXT:  | |-TemplateArgument type 'testClassTemplateDecl::A'
+// CHECK-NEXT:  | | `-RecordType 0{{.+}} 'testClassTemplateDecl::A'
+// CHECK-NEXT:  | |   `-CXXRecord 0x{{.+}} 'A'
 // CHECK-NEXT:  | |-CXXRecordDecl 0x{{.+}} prev 0x{{.+}} <col:24, col:30> col:30 implicit class TestClassTemplate
-// CHECK-NEXT:  | |-AccessSpecDecl 0x{{.+}} <line:[[@LINE-65]]:3, col:9> col:3 public
-// CHECK-NEXT:  | |-CXXConstructorDecl 0x{{.+}} <line:[[@LINE-65]]:5, col:23> col:5 used TestClassTemplate 'void ()'
-// CHECK-NEXT:  | |-CXXDestructorDecl 0x{{.+}} <line:[[@LINE-65]]:5, col:24> col:5 used ~TestClassTemplate 'void () noexcept'
-// CHECK-NEXT:  | |-CXXMethodDecl 0x{{.+}} <line:[[@LINE-65]]:5, col:11> col:9 j 'int ()'
-// CHECK-NEXT:  | |-FieldDecl 0x{{.+}} <line:[[@LINE-65]]:5, col:9> col:9 i 'int'
-// CHECK-NEXT:  | `-CXXConstructorDecl 0x{{.+}} <line:[[@LINE-71]]:30> col:30 implicit constexpr TestClassTemplate 'void (const testClassTemplateDecl::TestClassTemplate<testClassTemplateDecl::A> &)' inline default trivial noexcept-unevaluated 0x{{.+}}
+// CHECK-NEXT:  | |-AccessSpecDecl 0x{{.+}} <line:[[@LINE-67]]:3, col:9> col:3 public
+// CHECK-NEXT:  | |-CXXConstructorDecl 0x{{.+}} <line:[[@LINE-67]]:5, col:23> col:5 used TestClassTemplate 'void ()'
+// CHECK-NEXT:  | |-CXXDestructorDecl 0x{{.+}} <line:[[@LINE-67]]:5, col:24> col:5 used ~TestClassTemplate 'void () noexcept'
+// CHECK-NEXT:  | |-CXXMethodDecl 0x{{.+}} <line:[[@LINE-67]]:5, col:11> col:9 j 'int ()'
+// CHECK-NEXT:  | |-FieldDecl 0x{{.+}} <line:[[@LINE-67]]:5, col:9> col:9 i 'int'
+// CHECK-NEXT:  | `-CXXConstructorDecl 0x{{.+}} <line:[[@LINE-73]]:30> col:30 implicit constexpr TestClassTemplate 'void (const testClassTemplateDecl::TestClassTemplate<testClassTemplateDecl::A> &)' inline default trivial noexcept-unevaluated 0x{{.+}}
 // CHECK-NEXT:  |   `-ParmVarDecl 0x{{.+}} <col:30> col:30 'const testClassTemplateDecl::TestClassTemplate<testClassTemplateDecl::A> &'
 // CHECK-NEXT:  |-ClassTemplateSpecialization 0x{{.+}} 'TestClassTemplate'
 // CHECK-NEXT:  |-ClassTemplateSpecialization 0x{{.+}} 'TestClassTemplate'
 // CHECK-NEXT:  `-ClassTemplateSpecialization 0x{{.+}} 'TestClassTemplate'
 
-// CHECK:       ClassTemplateSpecializationDecl 0x{{.+}} <{{.+}}:[[@LINE-65]]:3, line:[[@LINE-63]]:3> line:[[@LINE-65]]:20 class TestClassTemplate definition
+// CHECK:       ClassTemplateSpecializationDecl 0x{{.+}} <{{.+}}:[[@LINE-67]]:3, line:[[@LINE-65]]:3> line:[[@LINE-67]]:20 class TestClassTemplate definition
 // CHECK-NEXT:  |-DefinitionData pass_in_registers standard_layout trivially_copyable trivial literal
 // CHECK-NEXT:  | |-DefaultConstructor exists trivial needs_implicit
 // CHECK-NEXT:  | |-CopyConstructor simple trivial has_const_param needs_implicit implicit_has_const_param
@@ -332,8 +342,10 @@ namespace testClassTemplateDecl {
 // CHECK-NEXT:  | |-MoveAssignment exists simple trivial needs_implicit
 // CHECK-NEXT:  | `-Destructor simple irrelevant trivial needs_implicit
 // CHECK-NEXT:  |-TemplateArgument type 'testClassTemplateDecl::B'
+// CHECK-NEXT:  | `-RecordType 0{{.+}} 'testClassTemplateDecl::B'
+// CHECK-NEXT:  |   `-CXXRecord 0x{{.+}} 'B'
 // CHECK-NEXT:  |-CXXRecordDecl 0x{{.+}} <col:14, col:20> col:20 implicit class TestClassTemplate
-// CHECK-NEXT:  `-FieldDecl 0x{{.+}} <line:[[@LINE-74]]:5, col:9> col:9 j 'int'
+// CHECK-NEXT:  `-FieldDecl 0x{{.+}} <line:[[@LINE-78]]:5, col:9> col:9 j 'int'
 
 // CHECK:       ClassTemplateSpecializationDecl 0x{{.+}} <{{.+}}:{{.*}}:3, col:44> col:25 class TestClassTemplate definition
 // CHECK-NEXT:  |-DefinitionData standard_layout has_user_declared_ctor can_const_default_init
@@ -344,14 +356,16 @@ namespace testClassTemplateDecl {
 // CHECK-NEXT:  | |-MoveAssignment
 // CHECK-NEXT:  | `-Destructor non_trivial user_declared
 // CHECK-NEXT:  |-TemplateArgument type 'testClassTemplateDecl::C'
-// CHECK-NEXT:  |-CXXRecordDecl 0x{{.+}} prev 0x{{.+}} <line:[[@LINE-98]]:24, col:30> col:30 implicit class TestClassTemplate
-// CHECK-NEXT:  |-AccessSpecDecl 0x{{.+}} <line:[[@LINE-98]]:3, col:9> col:3 public
-// CHECK-NEXT:  |-CXXConstructorDecl 0x{{.+}} <line:[[@LINE-98]]:5, col:23> col:5 TestClassTemplate 'void ()'
-// CHECK-NEXT:  |-CXXDestructorDecl 0x{{.+}} <line:[[@LINE-98]]:5, col:24> col:5 ~TestClassTemplate 'void ()' noexcept-unevaluated 0x{{.+}}
-// CHECK-NEXT:  |-CXXMethodDecl 0x{{.+}} <line:[[@LINE-98]]:5, col:11> col:9 j 'int ()'
-// CHECK-NEXT:  `-FieldDecl 0x{{.+}} <line:[[@LINE-98]]:5, col:9> col:9 i 'int'
-
-// CHECK:       ClassTemplateSpecializationDecl 0x{{.+}} <{{.+}}:[[@LINE-85]]:3, col:37> col:18 class TestClassTemplate definition
+// CHECK-NEXT:  | `-RecordType 0{{.+}} 'testClassTemplateDecl::C'
+// CHECK-NEXT:  |   `-CXXRecord 0x{{.+}} 'C'
+// CHECK-NEXT:  |-CXXRecordDecl 0x{{.+}} prev 0x{{.+}} <line:[[@LINE-104]]:24, col:30> col:30 implicit class TestClassTemplate
+// CHECK-NEXT:  |-AccessSpecDecl 0x{{.+}} <line:[[@LINE-104]]:3, col:9> col:3 public
+// CHECK-NEXT:  |-CXXConstructorDecl 0x{{.+}} <line:[[@LINE-104]]:5, col:23> col:5 TestClassTemplate 'void ()'
+// CHECK-NEXT:  |-CXXDestructorDecl 0x{{.+}} <line:[[@LINE-104]]:5, col:24> col:5 ~TestClassTemplate 'void ()' noexcept-unevaluated 0x{{.+}}
+// CHECK-NEXT:  |-CXXMethodDecl 0x{{.+}} <line:[[@LINE-104]]:5, col:11> col:9 j 'int ()'
+// CHECK-NEXT:  `-FieldDecl 0x{{.+}} <line:[[@LINE-104]]:5, col:9> col:9 i 'int'
+
+// CHECK:       ClassTemplateSpecializationDecl 0x{{.+}} <{{.+}}:[[@LINE-91]]:3, col:37> col:18 class TestClassTemplate definition
 // CHECK-NEXT:  |-DefinitionData standard_layout has_user_declared_ctor can_const_default_init
 // CHECK-NEXT:  | |-DefaultConstructor exists non_trivial user_provided
 // CHECK-NEXT:  | |-CopyConstructor simple trivial has_const_param needs_implicit implicit_has_const_param
@@ -360,17 +374,19 @@ namespace testClassTemplateDecl {
 // CHECK-NEXT:  | |-MoveAssignment
 // CHECK-NEXT:  | `-Destructor non_trivial user_declared
 // CHECK-NEXT:  |-TemplateArgument type 'testClassTemplateDecl::D'
-// CHECK-NEXT:  |-CXXRecordDecl 0x{{.+}} prev 0x{{.+}} <line:[[@LINE-114]]:24, col:30> col:30 implicit class TestClassTemplate
-// CHECK-NEXT:  |-AccessSpecDecl 0x{{.+}} <line:[[@LINE-114]]:3, col:9> col:3 public
-// CHECK-NEXT:  |-CXXConstructorDecl 0x{{.+}} <line:[[@LINE-114]]:5, col:23> col:5 TestClassTemplate 'void ()'
-// CHECK-NEXT:  |-CXXDestructorDecl 0x{{.+}} <line:[[@LINE-114]]:5, col:24> col:5 ~TestClassTemplate 'void ()' noexcept-unevaluated 0x{{.+}}
-// CHECK-NEXT:  |-CXXMethodDecl 0x{{.+}} <line:[[@LINE-114]]:5, col:11> col:9 j 'int ()'
-// CHECK-NEXT:  `-FieldDecl 0x{{.+}} <line:[[@LINE-114]]:5, col:9> col:9 i 'int'
-
-// CHECK:      ClassTemplateDecl 0x{{.+}} <{{.+}}:[[@LINE-98]]:3, line:[[@LINE-96]]:3> line:[[@LINE-98]]:44 TestClassTemplatePartial
+// CHECK-NEXT:  | `-RecordType 0{{.+}} 'testClassTemplateDecl::D'
+// CHECK-NEXT:  |   `-CXXRecord 0x{{.+}} 'D'
+// CHECK-NEXT:  |-CXXRecordDecl 0x{{.+}} prev 0x{{.+}} <line:[[@LINE-122]]:24, col:30> col:30 implicit class TestClassTemplate
+// CHECK-NEXT:  |-AccessSpecDecl 0x{{.+}} <line:[[@LINE-122]]:3, col:9> col:3 public
+// CHECK-NEXT:  |-CXXConstructorDecl 0x{{.+}} <line:[[@LINE-122]]:5, col:23> col:5 TestClassTemplate 'void ()'
+// CHECK-NEXT:  |-CXXDestructorDecl 0x{{.+}} <line:[[@LINE-122]]:5, col:24> col:5 ~TestClassTemplate 'void ()' noexcept-unevaluated 0x{{.+}}
+// CHECK-NEXT:  |-CXXMethodDecl 0x{{.+}} <line:[[@LINE-122]]:5, col:11> col:9 j 'int ()'
+// CHECK-NEXT:  `-FieldDecl 0x{{.+}} <line:[[@LINE-122]]:5, col:9> col:9 i 'int'
+
+// CHECK:      ClassTemplateDecl 0x{{.+}} <{{.+}}:[[@LINE-106]]:3, line:[[@LINE-104]]:3> line:[[@LINE-106]]:44 TestClassTemplatePartial
 // CHECK-NEXT:  |-TemplateTypeParmDecl 0x{{.+}} <col:12, col:21> col:21 typename depth 0 index 0 T1
 // CHECK-NEXT:  |-TemplateTypeParmDecl 0x{{.+}} <col:25, col:34> col:34 typename depth 0 index 1 T2
-// CHECK-NEXT:  `-CXXRecordDecl 0x{{.+}} <col:38, line:[[@LINE-99]]:3> line:[[@LINE-101]]:44 class TestClassTemplatePartial definition
+// CHECK-NEXT:  `-CXXRecordDecl 0x{{.+}} <col:38, line:[[@LINE-107]]:3> line:[[@LINE-109]]:44 class TestClassTemplatePartial definition
 // CHECK-NEXT:    |-DefinitionData standard_layout trivially_copyable trivial literal
 // CHECK-NEXT:    | |-DefaultConstructor exists trivial needs_implicit
 // CHECK-NEXT:    | |-CopyConstructor simple trivial has_const_param needs_implicit implicit_has_const_param
@@ -379,9 +395,9 @@ namespace testClassTemplateDecl {
 // CHECK-NEXT:    | |-MoveAssignment exists simple trivial needs_implicit
 // CHECK-NEXT:    | `-Destructor simple irrelevant trivial needs_implicit
 // CHECK-NEXT:    |-CXXRecordDecl 0x{{.+}} <col:38, col:44> col:44 implicit class TestClassTemplatePartial
-// CHECK-NEXT:    `-FieldDecl 0x{{.+}} <line:[[@LINE-109]]:5, col:9> col:9 i 'int'
+// CHECK-NEXT:    `-FieldDecl 0x{{.+}} <line:[[@LINE-117]]:5, col:9> col:9 i 'int'
 
-// CHECK:       ClassTemplatePartialSpecializationDecl 0x{{.+}} <{{.+}}:[[@LINE-109]]:3, line:[[@LINE-107]]:3> line:[[@LINE-109]]:31 class TestClassTemplatePartial definition
+// CHECK:       ClassTemplatePartialSpecializationDecl 0x{{.+}} <{{.+}}:[[@LINE-117]]:3, line:[[@LINE-115]]:3> line:[[@LINE-117]]:31 class TestClassTemplatePartial definition
 // CHECK-NEXT:  |-DefinitionData standard_layout trivially_copyable trivial literal
 // CHECK-NEXT:  | |-DefaultConstructor exists trivial needs_implicit
 // CHECK-NEXT:  | |-CopyConstructor simple trivial has_const_param needs_implicit implicit_has_const_param
@@ -390,20 +406,25 @@ namespace testClassTemplateDecl {
 // CHECK-NEXT:  | |-MoveAssignment exists simple trivial needs_implicit
 // CHECK-NEXT:  | `-Destructor simple irrelevant trivial needs_implicit
 // CHECK-NEXT:  |-TemplateArgument type 'type-parameter-0-0'
+// CHECK-NEXT:  | `-TemplateTypeParmType 0x{{.+}} 'type-parameter-0-0' dependent depth 0 index 0
 // CHECK-NEXT:  |-TemplateArgument type 'testClassTemplateDecl::A'
+// CHECK-NEXT:  | `-RecordType 0x{{.+}} 'testClassTemplateDecl::A'
+// CHECK-NEXT:  |   `-CXXRecord 0x{{.+}} 'A'
 // CHECK-NEXT:  |-TemplateTypeParmDecl 0x{{.+}} <col:12, col:21> col:21 referenced typename depth 0 index 0 T1
 // CHECK-NEXT:  |-CXXRecordDecl 0x{{.+}} <col:25, col:31> col:31 implicit class TestClassTemplatePartial
-// CHECK-NEXT:  `-FieldDecl 0x{{.+}} <line:[[@LINE-120]]:5, col:9> col:9 j 'int'
+// CHECK-NEXT:  `-FieldDecl 0x{{.+}} <line:[[@LINE-131]]:5, col:9> col:9 j 'int'
 
-// CHECK:       ClassTemplateDecl 0x{{.+}} <{{.+}}:[[@LINE-119]]:3, col:37> col:37 TestTemplateDefaultType
+// CHECK:       ClassTemplateDecl 0x{{.+}} <{{.+}}:[[@LINE-130]]:3, col:37> col:37 TestTemplateDefaultType
 // CHECK-NEXT:  |-TemplateTypeParmDecl 0x{{.+}} <col:12, col:25> col:21 typename depth 0 index 0 T
 // CHECK-NEXT:  | `-TemplateArgument type 'int'
+// CHECK-NEXT:  |   `-BuiltinType 0x{{.+}} 'int'
 // CHECK-NEXT:  `-CXXRecordDecl 0x{{.+}} <col:30, col:37> col:37 struct TestTemplateDefaultType
 
-// CHECK:       ClassTemplateDecl 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-123]]:3, col:57> col:31 TestTemplateDefaultType
+// CHECK:       ClassTemplateDecl 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-135]]:3, col:57> col:31 TestTemplateDefaultType
 // CHECK-NEXT:  |-TemplateTypeParmDecl 0x{{.+}} <col:12, col:21> col:21 typename depth 0 index 0 T
 // CHECK-NEXT:  | `-TemplateArgument type 'int'
-// CHECK-NEXT:  |   `-inherited from TemplateTypeParm 0x{{.+}} 'T'
+// CHECK-NEXT:  |   |-inherited from TemplateTypeParm 0x{{.+}} 'T'
+// CHECK-NEXT:  |   `-BuiltinType 0x{{.+}} 'int'
 // CHECK-NEXT:  `-CXXRecordDecl 0x{{.+}} prev 0x{{.+}} <col:24, col:57> col:31 struct TestTemplateDefaultType definition
 // CHECK-NEXT:    |-DefinitionData empty aggregate standard_layout trivially_copyable pod trivial literal has_constexpr_non_copy_move_ctor can_const_default_init
 // CHECK-NEXT:    | |-DefaultConstructor exists trivial constexpr needs_implicit defaulted_is_constexpr
@@ -414,10 +435,11 @@ namespace testClassTemplateDecl {
 // CHECK-NEXT:    | `-Destructor simple irrelevant trivial needs_implicit
 // CHECK-NEXT:    `-CXXRecordDecl 0x{{.+}} <col:24, col:31> col:31 implicit struct TestTemplateDefaultType
 
-// CHECK:       ClassTemplateDecl 0x{{.+}} <{{.+}}:[[@LINE-135]]:3, col:31> col:31 TestTemplateDefaultNonType
+// CHECK:       ClassTemplateDecl 0x{{.+}} <{{.+}}:[[@LINE-148]]:3, col:31> col:31 TestTemplateDefaultNonType
 // CHECK-NEXT:  |-NonTypeTemplateParmDecl 0x{{.+}} <col:12, col:20> col:16 'int' depth 0 index 0 I
 // CHECK-NEXT:  | `-TemplateArgument expr
 // CHECK-NEXT:  |   `-ConstantExpr 0x{{.+}} <col:20> 'int'
+// CHECK-NEXT:  |     |-value: Int 42
 // CHECK-NEXT:  |     `-IntegerLiteral 0x{{.+}} <col:20> 'int' 42
 // CHECK-NEXT:  `-CXXRecordDecl 0x{{.+}} <col:24, col:31> col:31 struct TestTemplateDefaultNonType
 
@@ -456,9 +478,11 @@ namespace testCanonicalTemplate {
   // CHECK-NEXT:   | `-ParmVarDecl 0x{{.*}} <col:50> col:51 'T'
   // CHECK-NEXT:   `-FunctionDecl 0x{{.*}} <line:[[@LINE-6]]:24, col:51> col:29 used TestFunctionTemplate 'void (testCanonicalTemplate::A)'
   // CHECK-NEXT:     |-TemplateArgument type 'testCanonicalTemplate::A'
+  // CHECK-NEXT:     | `-RecordType 0x{{.+}} 'testCanonicalTemplate::A'
+  // CHECK-NEXT:     |   `-CXXRecord 0x{{.+}} 'A'
   // CHECK-NEXT:     `-ParmVarDecl 0x{{.*}} <col:50> col:51 'testCanonicalTemplate::A':'testCanonicalTemplate::A'
 
-  // CHECK:      FunctionTemplateDecl 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-10]]:3, col:51> col:29 TestFunctionTemplate
+  // CHECK:      FunctionTemplateDecl 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-12]]:3, col:51> col:29 TestFunctionTemplate
   // CHECK-NEXT:   |-TemplateTypeParmDecl 0x{{.+}} <col:12, col:21> col:21 referenced typename depth 0 index 0 T
   // CHECK-NEXT:   |-FunctionDecl{{.*}} 0x{{.+}} prev 0x{{.+}} <col:24, col:51> col:29 TestFunctionTemplate 'void (T)'
   // CHECK-NEXT:   | `-ParmVarDecl 0x{{.+}} <col:50> col:51 'T'
@@ -493,13 +517,15 @@ namespace testCanonicalTemplate {
   // CHECK-NEXT:   | |-MoveAssignment exists simple trivial needs_implicit
   // CHECK-NEXT:   | `-Destructor simple irrelevant trivial needs_implicit
   // CHECK-NEXT:   |-TemplateArgument type 'testCanonicalTemplate::A'
+  // CHECK-NEXT:   | `-RecordType 0x{{.+}} 'testCanonicalTemplate::A'
+  // CHECK-NEXT:   |   `-CXXRecord 0x{{.+}} 'A'
   // CHECK-NEXT:   |-CXXRecordDecl 0x{{.+}} prev 0x{{.+}} <col:25, col:31> col:31 implicit class TestClassTemplate
-  // CHECK-NEXT:   |-FriendDecl 0x{{.+}} <line:[[@LINE-28]]:5, col:40> col:40
+  // CHECK-NEXT:   |-FriendDecl 0x{{.+}} <line:[[@LINE-30]]:5, col:40> col:40
   // CHECK-NEXT:   | `-ClassTemplateDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <col:5, col:40> col:40 TestClassTemplate
   // CHECK-NEXT:   |   |-TemplateTypeParmDecl 0x{{.+}} <col:14, col:23> col:23 typename depth 0 index 0 T2
   // CHECK-NEXT:   |   |-CXXRecordDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <col:34, col:40> col:40 class TestClassTemplate
   // CHECK-NEXT:   |   `-ClassTemplateSpecialization 0x{{.+}} 'TestClassTemplate'
-  // CHECK-NEXT:   |-CXXConstructorDecl 0x{{.+}} <line:[[@LINE-34]]:31> col:31 implicit used constexpr TestClassTemplate 'void () noexcept' inline default trivial
+  // CHECK-NEXT:   |-CXXConstructorDecl 0x{{.+}} <line:[[@LINE-36]]:31> col:31 implicit used constexpr TestClassTemplate 'void () noexcept' inline default trivial
   // CHECK-NEXT:   | `-CompoundStmt 0x{{.+}} <col:31>
   // CHECK-NEXT:   |-CXXConstructorDecl 0x{{.+}} <col:31> col:31 implicit constexpr TestClassTemplate 'void (const testCanonicalTemplate::TestClassTemplate<testCanonicalTemplate::A> &)' inline default trivial noexcept-unevaluated 0x{{.+}}
   // CHECK-NEXT:   | `-ParmVarDecl 0x{{.+}} <col:31> col:31 'const testCanonicalTemplate::TestClassTemplate<testCanonicalTemplate::A> &'
@@ -524,6 +550,8 @@ namespace testCanonicalTemplate {
   // CHECK-NEXT:   | |-MoveAssignment exists simple trivial needs_implicit
   // CHECK-NEXT:   | `-Destructor simple irrelevant trivial needs_implicit
   // CHECK-NEXT:   |-TemplateArgument type 'testCanonicalTemplate::A'
+  // CHECK-NEXT:   | `-RecordType 0x{{.+}} 'testCanonicalTemplate::A'
+  // CHECK-NEXT:   |   `-CXXRecord 0x{{.+}} 'A'
   // CHECK-NEXT:   |-CXXRecordDecl 0x{{.+}} prev 0x{{.+}} <col:25, col:31> col:31 implicit class TestClassTemplate2
   // CHECK-NEXT:   |-CXXConstructorDecl 0x{{.+}} <col:31> col:31 implicit used constexpr TestClassTemplate2 'void () noexcept' inline default trivial
   // CHECK-NEXT:   | `-CompoundStmt 0x{{.+}} <col:31>
@@ -532,14 +560,14 @@ namespace testCanonicalTemplate {
   // CHECK-NEXT:   `-CXXConstructorDecl 0x{{.+}} <col:31> col:31 implicit constexpr TestClassTemplate2 'void (testCanonicalTemplate::TestClassTemplate2<testCanonicalTemplate::A> &&)' inline default trivial noexcept-unevaluated 0x{{.+}}
   // CHECK-NEXT:     `-ParmVarDecl 0x{{.+}} <col:31> col:31 'testCanonicalTemplate::TestClassTemplate2<testCanonicalTemplate::A> &&'
 
-  // CHECK:      ClassTemplateDecl 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-24]]:3, col:31> col:31 TestClassTemplate2
+  // CHECK:      ClassTemplateDecl 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-26]]:3, col:31> col:31 TestClassTemplate2
   // CHECK-NEXT: |-TemplateTypeParmDecl 0x{{.+}} <col:12, col:21> col:21 typename depth 0 index 0 T1
   // CHECK-NEXT: |-CXXRecordDecl 0x{{.+}} prev 0x{{.+}} <col:25, col:31> col:31 class TestClassTemplate2
   // CHECK-NEXT: `-ClassTemplateSpecialization 0x{{.+}} 'TestClassTemplate2'
 
-  // CHECK:      ClassTemplateDecl 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-28]]:3, line:[[@LINE-27]]:3> line:[[@LINE-28]]:31 TestClassTemplate2
+  // CHECK:      ClassTemplateDecl 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-30]]:3, line:[[@LINE-29]]:3> line:[[@LINE-30]]:31 TestClassTemplate2
   // CHECK-NEXT: |-TemplateTypeParmDecl 0x{{.+}} <col:12, col:21> col:21 typename depth 0 index 0 T1
-  // CHECK-NEXT: |-CXXRecordDecl 0x{{.+}} prev 0x{{.+}} <col:25, line:[[@LINE-29]]:3> line:[[@LINE-30]]:31 class TestClassTemplate2 definition
+  // CHECK-NEXT: |-CXXRecordDecl 0x{{.+}} prev 0x{{.+}} <col:25, line:[[@LINE-31]]:3> line:[[@LINE-32]]:31 class TestClassTemplate2 definition
   // CHECK-NEXT: | |-DefinitionData empty aggregate standard_layout trivially_copyable pod trivial literal has_constexpr_non_copy_move_ctor can_const_default_init
   // CHECK-NEXT: | | |-DefaultConstructor exists trivial constexpr needs_implicit defaulted_is_constexpr
   // CHECK-NEXT: | | |-CopyConstructor simple trivial has_const_param needs_implicit implicit_has_const_param
@@ -567,22 +595,25 @@ namespace testCanonicalTemplate {
   // CHECK-NEXT: |-VarDecl 0x{{.+}} <col:28, col:43> col:43 TestVarTemplate 'const T' static
   // CHECK-NEXT: |-VarTemplateSpecializationDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <line:[[@LINE-11]]:3, col:34> col:14 referenced TestVarTemplate 'const int':'const int' cinit
   // CHECK-NEXT: | |-TemplateArgument type 'int'
+  // CHECK-NEXT: | | `-BuiltinType 0x{{.+}} 'int'
   // CHECK-NEXT: | `-InitListExpr 0x{{.+}} <col:32, col:34> 'int':'int'
-  // CHECK-NEXT: `-VarTemplateSpecializationDecl 0x{{.+}} <line:[[@LINE-17]]:28, col:43> col:43 referenced TestVarTemplate 'const int':'const int' static
+  // CHECK-NEXT: `-VarTemplateSpecializationDecl 0x{{.+}} <line:[[@LINE-18]]:28, col:43> col:43 referenced TestVarTemplate 'const int':'const int' static
   // CHECK-NEXT:   `-TemplateArgument type 'int'
 
-  // CHECK:     VarTemplateSpecializationDecl 0x{{.+}} <{{.+}}:[[@LINE-20]]:28, col:43> col:43 referenced TestVarTemplate 'const int':'const int' static
+  // CHECK:     VarTemplateSpecializationDecl 0x{{.+}} <{{.+}}:[[@LINE-21]]:28, col:43> col:43 referenced TestVarTemplate 'const int':'const int' static
   // CHECK-NEXT:`-TemplateArgument type 'int'
+  // CHECK-NEXT:  `-BuiltinType 0x{{.+}} 'int'
 
-  // CHECK:      VarTemplateDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-21]]:3, line:[[@LINE-20]]:34> col:14 TestVarTemplate
-  // CHECK-NEXT: |-TemplateTypeParmDecl 0x{{.+}} <line:[[@LINE-22]]:12, col:21> col:21 referenced typename depth 0 index 0 T
-  // CHECK-NEXT: |-VarDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <line:[[@LINE-22]]:3, col:34> col:14 TestVarTemplate 'const T' cinit
+  // CHECK:      VarTemplateDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-23]]:3, line:[[@LINE-22]]:34> col:14 TestVarTemplate
+  // CHECK-NEXT: |-TemplateTypeParmDecl 0x{{.+}} <line:[[@LINE-24]]:12, col:21> col:21 referenced typename depth 0 index 0 T
+  // CHECK-NEXT: |-VarDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <line:[[@LINE-24]]:3, col:34> col:14 TestVarTemplate 'const T' cinit
   // CHECK-NEXT: | `-InitListExpr 0x{{.+}} <col:32, col:34> 'void'
   // CHECK-NEXT: |-VarTemplateSpecialization 0x{{.+}} 'TestVarTemplate' 'const int':'const int'
   // CHECK-NEXT: `-VarTemplateSpecialization 0x{{.+}} 'TestVarTemplate' 'const int':'const int'
     
-  // CHECK:      VarTemplateSpecializationDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-27]]:3, col:34> col:14 referenced TestVarTemplate 'const int':'const int' cinit
+  // CHECK:      VarTemplateSpecializationDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-29]]:3, col:34> col:14 referenced TestVarTemplate 'const int':'const int' cinit
   // CHECK-NEXT: |-TemplateArgument type 'int'
+  // CHECK-NEXT: | `-BuiltinType 0x{{.+}} 'int'
   // CHECK-NEXT: `-InitListExpr 0x{{.+}} <col:32, col:34> 'int':'int'
 } 
 
@@ -614,6 +645,7 @@ namespace TestNonTypeTemplateParmDecl {
 // CHECK-NEXT:     NonTypeTemplateParmDecl{{.*}} 'int' depth 0 index 0 I
 // CHECK-NEXT:       TemplateArgument expr
 // CHECK-NEXT:         ConstantExpr{{.*}} 'int'
+// CHECK-NEXT:           value: Int 1
 // CHECK-NEXT:           IntegerLiteral{{.*}} 'int' 1
 // CHECK-NEXT:     NonTypeTemplateParmDecl{{.*}} 'int' depth 0 index 1 ... J
 
diff --git a/clang/test/AST/ast-dump-lambda-body-not-duplicated.cpp b/clang/test/AST/ast-dump-lambda-body-not-duplicated.cpp
new file mode 100644
index 0000000000000..558455e239c74
--- /dev/null
+++ b/clang/test/AST/ast-dump-lambda-body-not-duplicated.cpp
@@ -0,0 +1,40 @@
+// Test without serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -ast-dump %s \
+// RUN: | FileCheck %s
+//
+// Test with serialization:
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -emit-pch -o %t %s
+// RUN: %clang_cc1 -x c++ -triple x86_64-unknown-unknown -Wno-unused-value \
+// RUN: -include-pch %t -ast-dump-all /dev/null \
+// RUN: | FileCheck %s
+
+// Make sure that the Stmt * for the body of the LambdaExpr is
+// equal to the Stmt * for the body of the call operator.
+void Test0() {
+  []() {
+    return 42;
+  };
+}
+
+// CHECK: FunctionDecl {{.*}} Test0
+//
+// CHECK: CXXMethodDecl {{.*}} operator() 'int () const' inline
+// CHECK-NEXT: CompoundStmt 0x[[TMP0:.*]]
+// CHECK: IntegerLiteral {{.*}} 'int' 42
+//
+// CHECK: CompoundStmt 0x[[TMP0]]
+// Check: IntegerLiteral {{.*}} 'int' 42
+
+void Test1() {
+  [](auto x) { return x; };
+}
+
+// CHECK: FunctionDecl {{.*}} Test1
+//
+// CHECK: CXXMethodDecl {{.*}} operator() 'auto (auto) const' inline
+// CHECK-NEXT: ParmVarDecl {{.*}} referenced x 'auto'
+// CHECK-NEXT: CompoundStmt 0x[[TMP1:.*]]
+// CHECK: DeclRefExpr {{.*}} 'x' 'auto'
+//
+// CHECK: CompoundStmt 0x[[TMP1]]
+// CHECK: DeclRefExpr {{.*}} 'x' 'auto'
diff --git a/clang/test/AST/ast-dump-lambda.cpp b/clang/test/AST/ast-dump-lambda.cpp
index a3ca4a8fc534f..37fb62ef9930e 100644
--- a/clang/test/AST/ast-dump-lambda.cpp
+++ b/clang/test/AST/ast-dump-lambda.cpp
@@ -7,7 +7,7 @@
 // RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
 // RUN:            -emit-pch -o %t %s
 // RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
-// RUN:            -x c++ -include-pch %t -ast-dump-all -ast-dump-filter test \
+// RUN:            -x c++ -include-pch %t -ast-dump-all -ast-dump-filter test /dev/null \
 // RUN: | FileCheck -strict-whitespace --match-full-lines %s
 
 
diff --git a/clang/test/AST/ast-dump-openmp-begin-declare-variant_template_1.cpp b/clang/test/AST/ast-dump-openmp-begin-declare-variant_template_1.cpp
index 6a663d5d75d94..5916958b94625 100644
--- a/clang/test/AST/ast-dump-openmp-begin-declare-variant_template_1.cpp
+++ b/clang/test/AST/ast-dump-openmp-begin-declare-variant_template_1.cpp
@@ -118,7 +118,7 @@ int test() {
 // CHECK-NEXT: | |         `-IntegerLiteral [[ADDR_58:0x[a-z0-9]*]] <col:23> 'int' 0
 // CHECK-NEXT: | `-FunctionDecl [[ADDR_59:0x[a-z0-9]*]] <line:37:1, line:40:1> line:37:5 used test1 'int ({{.*}})'
 // CHECK-NEXT: |   |-TemplateArgument type 'double'
-// CHECK-NEXT: |   `-CompoundStmt [[ADDR_60:0x[a-z0-9]*]] <col:13, line:40:1>
+// CHECK:      |   `-CompoundStmt [[ADDR_60:0x[a-z0-9]*]] <col:13, line:40:1>
 // CHECK-NEXT: |     `-ReturnStmt [[ADDR_61:0x[a-z0-9]*]] <line:39:3, col:25>
 // CHECK-NEXT: |       `-PseudoObjectExpr [[ADDR_62:0x[a-z0-9]*]] <col:10, col:25> 'int'
 // CHECK-NEXT: |         |-CallExpr [[ADDR_63:0x[a-z0-9]*]] <col:10, col:25> 'int'
@@ -153,7 +153,8 @@ int test() {
 // CHECK-NEXT: |       `-PseudoObjectExpr [[ADDR_85:0x[a-z0-9]*]] <col:10, col:13> 'int'
 // CHECK-NEXT: |         |-CallExpr [[ADDR_86:0x[a-z0-9]*]] <col:10, col:13> 'int'
 // CHECK-NEXT: |         | `-SubstNonTypeTemplateParmExpr [[ADDR_87:0x[a-z0-9]*]] <col:10> 'int (*)({{.*}})'
-// CHECK-NEXT: |         |   `-UnaryOperator [[ADDR_88:0x[a-z0-9]*]] <col:10> 'int (*)({{.*}})' prefix '&' cannot overflow
+// CHECK-NEXT: |         |   |-NonTypeTemplateParmDecl {{.*}} referenced 'Ty':'int (*)()' depth 0 index 0 fn
+// CHECK-NEXT: |         |   `-UnaryOperator [[ADDR_88:0x[a-z0-9]*]] <line:47:10> 'int (*)({{.*}})' prefix '&' cannot overflow
 // CHECK-NEXT: |         |     `-DeclRefExpr [[ADDR_89:0x[a-z0-9]*]] <col:10> 'int ({{.*}})' {{.*}}Function [[ADDR_0]] 'also_before' 'int ({{.*}})'
 // CHECK-NEXT: |         `-CallExpr [[ADDR_90:0x[a-z0-9]*]] <line:21:1, line:47:13> 'int'
 // CHECK-NEXT: |           `-ImplicitCastExpr [[ADDR_91:0x[a-z0-9]*]] <line:21:1> 'int (*)({{.*}})' <FunctionToPointerDecay>
diff --git a/clang/test/AST/ast-dump-openmp-target-parallel-for-simd.c b/clang/test/AST/ast-dump-openmp-target-parallel-for-simd.c
index f45aac455ac4a..d65a729aa4022 100644
--- a/clang/test/AST/ast-dump-openmp-target-parallel-for-simd.c
+++ b/clang/test/AST/ast-dump-openmp-target-parallel-for-simd.c
@@ -71,7 +71,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:5:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       | | |   | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -102,7 +102,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:5:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -129,7 +129,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:5:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |     | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -202,9 +202,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:11:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       | | |   | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -250,9 +250,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:11:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -294,9 +294,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:11:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |     | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -387,9 +387,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:18:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       | | |   | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -435,9 +435,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:18:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -479,9 +479,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:18:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |     | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -572,9 +572,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:25:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:26:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       | | |   | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -620,9 +620,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:25:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:26:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -664,9 +664,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:25:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:26:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |     | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -774,11 +774,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | | | |-FieldDecl {{.*}} <line:32:23> col:23 implicit 'int'
-// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | |-FieldDecl {{.*}} <line:33:25> col:25 implicit 'int'
-// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         | | |   |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         | | |   | |-DeclStmt {{.*}} <line:32:8, col:17>
@@ -839,11 +839,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:32:23> col:23 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:33:25> col:25 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |   |-CapturedStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -900,11 +900,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         |   | |-FieldDecl {{.*}} <line:32:23> col:23 implicit 'int'
-// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | |-FieldDecl {{.*}} <line:33:25> col:25 implicit 'int'
-// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |     |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |     | |-DeclStmt {{.*}} <line:32:8, col:17>
diff --git a/clang/test/AST/ast-dump-openmp-target-parallel-for.c b/clang/test/AST/ast-dump-openmp-target-parallel-for.c
index cbc6445e30e07..abfa75ba10df4 100644
--- a/clang/test/AST/ast-dump-openmp-target-parallel-for.c
+++ b/clang/test/AST/ast-dump-openmp-target-parallel-for.c
@@ -71,7 +71,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:5:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       | | |   | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -102,7 +102,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:5:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -129,7 +129,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:5:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |     | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -202,9 +202,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:11:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       | | |   | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -250,9 +250,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:11:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -294,9 +294,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:11:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |     | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -387,9 +387,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:18:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       | | |   | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -435,9 +435,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:18:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -479,9 +479,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:18:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |     | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -572,9 +572,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:25:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:26:5> col:5 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       | | |   | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -620,9 +620,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:25:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:26:5> col:5 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -664,9 +664,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:25:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:26:5> col:5 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |     | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -774,11 +774,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | | | |-FieldDecl {{.*}} <line:32:3> col:3 implicit 'int'
-// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | |-FieldDecl {{.*}} <line:33:5> col:5 implicit 'int'
-// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         | | |   |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         | | |   | |-DeclStmt {{.*}} <line:32:8, col:17>
@@ -839,11 +839,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:32:3> col:3 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:33:5> col:5 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |   |-CapturedStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -900,11 +900,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         |   | |-FieldDecl {{.*}} <line:32:3> col:3 implicit 'int'
-// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | |-FieldDecl {{.*}} <line:33:5> col:5 implicit 'int'
-// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |     |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |     | |-DeclStmt {{.*}} <line:32:8, col:17>
diff --git a/clang/test/AST/ast-dump-openmp-target-simd.c b/clang/test/AST/ast-dump-openmp-target-simd.c
index d66cb2b42d741..404b1b197ab11 100644
--- a/clang/test/AST/ast-dump-openmp-target-simd.c
+++ b/clang/test/AST/ast-dump-openmp-target-simd.c
@@ -73,7 +73,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:5:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |   | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -146,9 +146,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:11:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |   | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -239,9 +239,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:18:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |   | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -332,9 +332,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:25:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:26:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |   | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -442,11 +442,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:32:23> col:23 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:33:25> col:25 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |   |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |   | |-DeclStmt {{.*}} <line:32:8, col:17>
diff --git a/clang/test/AST/ast-dump-openmp-target-teams-distribute-parallel-for-simd.c b/clang/test/AST/ast-dump-openmp-target-teams-distribute-parallel-for-simd.c
index 5edcaa5a1082c..acb3cf37f08cb 100644
--- a/clang/test/AST/ast-dump-openmp-target-teams-distribute-parallel-for-simd.c
+++ b/clang/test/AST/ast-dump-openmp-target-teams-distribute-parallel-for-simd.c
@@ -77,7 +77,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | | | `-FieldDecl {{.*}} <line:5:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | | | |   |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       | | | | |   | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -104,7 +104,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:5:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-CapturedStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       | | |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -135,7 +135,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |   | `-FieldDecl {{.*}} <line:5:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |     |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       | | |     | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -168,7 +168,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:5:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -201,7 +201,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | | | `-FieldDecl {{.*}} <line:5:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   | |   |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |   | |   | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -228,7 +228,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:5:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-CapturedStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |     | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -259,7 +259,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |     |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |     | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |     | `-FieldDecl {{.*}} <line:5:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |     `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |       |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |       | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -340,9 +340,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | | | |-FieldDecl {{.*}} <line:11:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | | | |   |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       | | | | |   | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -384,9 +384,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:11:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-CapturedStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       | | |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -432,9 +432,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |   | |-FieldDecl {{.*}} <line:11:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | |   | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |     |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       | | |     | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -482,9 +482,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:11:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -532,9 +532,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | | | |-FieldDecl {{.*}} <line:11:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   | |   |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |   | |   | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -576,9 +576,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:11:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-CapturedStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |     | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -624,9 +624,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |     |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |     | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |     | |-FieldDecl {{.*}} <line:11:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |     | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |     `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |       |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |       | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -725,9 +725,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | | | |-FieldDecl {{.*}} <line:18:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | | | |   |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       | | | | |   | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -769,9 +769,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:18:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-CapturedStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       | | |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -817,9 +817,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |   | |-FieldDecl {{.*}} <line:18:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | |   | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |     |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       | | |     | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -867,9 +867,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:18:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -917,9 +917,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | | | |-FieldDecl {{.*}} <line:18:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   | |   |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |   | |   | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -961,9 +961,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:18:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-CapturedStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |     | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -1009,9 +1009,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |     |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |     | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |     | |-FieldDecl {{.*}} <line:18:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |     | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |     `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |       |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |       | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -1110,9 +1110,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | | | |-FieldDecl {{.*}} <line:25:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | | | `-FieldDecl {{.*}} <line:26:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | | | |   |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       | | | | |   | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -1154,9 +1154,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:25:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:26:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-CapturedStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       | | |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -1202,9 +1202,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |   | |-FieldDecl {{.*}} <line:25:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | |   | `-FieldDecl {{.*}} <line:26:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |     |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       | | |     | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -1252,9 +1252,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:25:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:26:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -1302,9 +1302,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | | | |-FieldDecl {{.*}} <line:25:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | | | `-FieldDecl {{.*}} <line:26:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   | |   |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |   | |   | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -1346,9 +1346,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:25:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:26:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-CapturedStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |     | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -1394,9 +1394,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |     |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |     | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |     | |-FieldDecl {{.*}} <line:25:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |     | `-FieldDecl {{.*}} <line:26:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |     `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |       |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |       | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -1512,11 +1512,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | | | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | | | | | |-FieldDecl {{.*}} <line:32:23> col:23 implicit 'int'
-// CHECK-NEXT:         | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | | | |-FieldDecl {{.*}} <line:33:25> col:25 implicit 'int'
-// CHECK-NEXT:         | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         | | | | |   |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         | | | | |   | |-DeclStmt {{.*}} <line:32:8, col:17>
@@ -1573,11 +1573,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | | | |-FieldDecl {{.*}} <line:32:23> col:23 implicit 'int'
-// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | |-FieldDecl {{.*}} <line:33:25> col:25 implicit 'int'
-// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         | | |   |-CapturedStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         | | |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -1638,11 +1638,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | | |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | |   | |-FieldDecl {{.*}} <line:32:23> col:23 implicit 'int'
-// CHECK-NEXT:         | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |   | |-FieldDecl {{.*}} <line:33:25> col:25 implicit 'int'
-// CHECK-NEXT:         | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |   | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         | | |     |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         | | |     | |-DeclStmt {{.*}} <line:32:8, col:17>
@@ -1705,11 +1705,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:32:23> col:23 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:33:25> col:25 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |   |-CapturedStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -1772,11 +1772,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         |   | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         |   | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         |   | | | |-FieldDecl {{.*}} <line:32:23> col:23 implicit 'int'
-// CHECK-NEXT:         |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | | | |-FieldDecl {{.*}} <line:33:25> col:25 implicit 'int'
-// CHECK-NEXT:         |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |   | |   |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |   | |   | |-DeclStmt {{.*}} <line:32:8, col:17>
@@ -1833,11 +1833,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         |   | |-FieldDecl {{.*}} <line:32:23> col:23 implicit 'int'
-// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | |-FieldDecl {{.*}} <line:33:25> col:25 implicit 'int'
-// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |     |-CapturedStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |     | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -1898,11 +1898,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         |     |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         |     | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         |     | |-FieldDecl {{.*}} <line:32:23> col:23 implicit 'int'
-// CHECK-NEXT:         |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |     | |-FieldDecl {{.*}} <line:33:25> col:25 implicit 'int'
-// CHECK-NEXT:         |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |     | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |     `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |       |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |       | |-DeclStmt {{.*}} <line:32:8, col:17>
diff --git a/clang/test/AST/ast-dump-openmp-target-teams-distribute-parallel-for.c b/clang/test/AST/ast-dump-openmp-target-teams-distribute-parallel-for.c
index 0d816abf5f006..807d8f138c699 100644
--- a/clang/test/AST/ast-dump-openmp-target-teams-distribute-parallel-for.c
+++ b/clang/test/AST/ast-dump-openmp-target-teams-distribute-parallel-for.c
@@ -77,7 +77,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | | | `-FieldDecl {{.*}} <line:5:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | | | |   |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       | | | | |   | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -104,7 +104,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:5:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-CapturedStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       | | |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -135,7 +135,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |   | `-FieldDecl {{.*}} <line:5:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |     |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       | | |     | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -168,7 +168,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:5:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -201,7 +201,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | | | `-FieldDecl {{.*}} <line:5:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   | |   |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |   | |   | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -228,7 +228,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:5:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-CapturedStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |     | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -259,7 +259,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |     |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |     | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |     | `-FieldDecl {{.*}} <line:5:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |     `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |       |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |       | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -340,9 +340,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | | | |-FieldDecl {{.*}} <line:11:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | | | |   |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       | | | | |   | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -384,9 +384,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:11:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-CapturedStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       | | |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -432,9 +432,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |   | |-FieldDecl {{.*}} <line:11:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | |   | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |     |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       | | |     | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -482,9 +482,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:11:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -532,9 +532,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | | | |-FieldDecl {{.*}} <line:11:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   | |   |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |   | |   | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -576,9 +576,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:11:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-CapturedStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |     | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -624,9 +624,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |     |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |     | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |     | |-FieldDecl {{.*}} <line:11:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |     | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |     `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |       |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |       | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -725,9 +725,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | | | |-FieldDecl {{.*}} <line:18:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | | | |   |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       | | | | |   | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -769,9 +769,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:18:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-CapturedStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       | | |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -817,9 +817,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |   | |-FieldDecl {{.*}} <line:18:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | |   | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |     |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       | | |     | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -867,9 +867,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:18:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -917,9 +917,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | | | |-FieldDecl {{.*}} <line:18:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   | |   |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |   | |   | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -961,9 +961,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:18:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-CapturedStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |     | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -1009,9 +1009,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |     |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |     | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |     | |-FieldDecl {{.*}} <line:18:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |     | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |     `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |       |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |       | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -1110,9 +1110,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | | | |-FieldDecl {{.*}} <line:25:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | | | `-FieldDecl {{.*}} <line:26:5> col:5 implicit 'int'
-// CHECK-NEXT: |       | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | | | |   |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       | | | | |   | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -1154,9 +1154,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:25:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:26:5> col:5 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-CapturedStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       | | |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -1202,9 +1202,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |   | |-FieldDecl {{.*}} <line:25:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | |   | `-FieldDecl {{.*}} <line:26:5> col:5 implicit 'int'
-// CHECK-NEXT: |       | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |     |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       | | |     | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -1252,9 +1252,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:25:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:26:5> col:5 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -1302,9 +1302,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | | | |-FieldDecl {{.*}} <line:25:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | | | `-FieldDecl {{.*}} <line:26:5> col:5 implicit 'int'
-// CHECK-NEXT: |       |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   | |   |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |   | |   | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -1346,9 +1346,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:25:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:26:5> col:5 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-CapturedStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |     | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -1394,9 +1394,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |     |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |     | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |     | |-FieldDecl {{.*}} <line:25:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |     | `-FieldDecl {{.*}} <line:26:5> col:5 implicit 'int'
-// CHECK-NEXT: |       |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |     `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |       |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |       | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -1512,11 +1512,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | | | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | | | | | |-FieldDecl {{.*}} <line:32:3> col:3 implicit 'int'
-// CHECK-NEXT:         | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | | | |-FieldDecl {{.*}} <line:33:5> col:5 implicit 'int'
-// CHECK-NEXT:         | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         | | | | |   |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         | | | | |   | |-DeclStmt {{.*}} <line:32:8, col:17>
@@ -1573,11 +1573,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | | | |-FieldDecl {{.*}} <line:32:3> col:3 implicit 'int'
-// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | |-FieldDecl {{.*}} <line:33:5> col:5 implicit 'int'
-// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         | | |   |-CapturedStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         | | |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -1638,11 +1638,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | | |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | |   | |-FieldDecl {{.*}} <line:32:3> col:3 implicit 'int'
-// CHECK-NEXT:         | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |   | |-FieldDecl {{.*}} <line:33:5> col:5 implicit 'int'
-// CHECK-NEXT:         | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |   | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         | | |     |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         | | |     | |-DeclStmt {{.*}} <line:32:8, col:17>
@@ -1705,11 +1705,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:32:3> col:3 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:33:5> col:5 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |   |-CapturedStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -1772,11 +1772,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         |   | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         |   | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         |   | | | |-FieldDecl {{.*}} <line:32:3> col:3 implicit 'int'
-// CHECK-NEXT:         |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | | | |-FieldDecl {{.*}} <line:33:5> col:5 implicit 'int'
-// CHECK-NEXT:         |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |   | |   |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |   | |   | |-DeclStmt {{.*}} <line:32:8, col:17>
@@ -1833,11 +1833,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         |   | |-FieldDecl {{.*}} <line:32:3> col:3 implicit 'int'
-// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | |-FieldDecl {{.*}} <line:33:5> col:5 implicit 'int'
-// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |     |-CapturedStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |     | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -1898,11 +1898,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         |     |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         |     | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         |     | |-FieldDecl {{.*}} <line:32:3> col:3 implicit 'int'
-// CHECK-NEXT:         |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |     | |-FieldDecl {{.*}} <line:33:5> col:5 implicit 'int'
-// CHECK-NEXT:         |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |     | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |     | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |     |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |     `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |       |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |       | |-DeclStmt {{.*}} <line:32:8, col:17>
diff --git a/clang/test/AST/ast-dump-openmp-target-teams-distribute-simd.c b/clang/test/AST/ast-dump-openmp-target-teams-distribute-simd.c
index a113ce2ca4468..83e530c87e3f9 100644
--- a/clang/test/AST/ast-dump-openmp-target-teams-distribute-simd.c
+++ b/clang/test/AST/ast-dump-openmp-target-teams-distribute-simd.c
@@ -71,7 +71,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:5:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       | | |   | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -102,7 +102,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:5:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -129,7 +129,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:5:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |     | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -202,9 +202,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:11:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       | | |   | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -250,9 +250,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:11:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -294,9 +294,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:11:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |     | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -387,9 +387,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:18:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       | | |   | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -435,9 +435,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:18:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -479,9 +479,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:18:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |     | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -572,9 +572,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:25:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:26:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       | | |   | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -620,9 +620,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:25:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:26:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -664,9 +664,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:25:23> col:23 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:26:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |     | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -774,11 +774,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | | | |-FieldDecl {{.*}} <line:32:23> col:23 implicit 'int'
-// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | |-FieldDecl {{.*}} <line:33:25> col:25 implicit 'int'
-// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         | | |   |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         | | |   | |-DeclStmt {{.*}} <line:32:8, col:17>
@@ -839,11 +839,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:32:23> col:23 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:33:25> col:25 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |   |-CapturedStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -900,11 +900,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         |   | |-FieldDecl {{.*}} <line:32:23> col:23 implicit 'int'
-// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | |-FieldDecl {{.*}} <line:33:25> col:25 implicit 'int'
-// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |     |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |     | |-DeclStmt {{.*}} <line:32:8, col:17>
diff --git a/clang/test/AST/ast-dump-openmp-target-teams-distribute.c b/clang/test/AST/ast-dump-openmp-target-teams-distribute.c
index 9754294e0396b..a4ebf15356e24 100644
--- a/clang/test/AST/ast-dump-openmp-target-teams-distribute.c
+++ b/clang/test/AST/ast-dump-openmp-target-teams-distribute.c
@@ -71,7 +71,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:5:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       | | |   | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -102,7 +102,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:5:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -129,7 +129,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:5:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-ForStmt {{.*}} <col:3, line:6:5>
 // CHECK-NEXT: |       |     | |-DeclStmt {{.*}} <line:5:8, col:17>
@@ -202,9 +202,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:11:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       | | |   | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -250,9 +250,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:11:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -294,9 +294,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:11:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:12:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-ForStmt {{.*}} <line:11:3, line:13:7>
 // CHECK-NEXT: |       |     | |-DeclStmt {{.*}} <line:11:8, col:17>
@@ -387,9 +387,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:18:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       | | |   | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -435,9 +435,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:18:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -479,9 +479,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:18:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:19:25> col:25 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-ForStmt {{.*}} <line:18:3, line:20:7>
 // CHECK-NEXT: |       |     | |-DeclStmt {{.*}} <line:18:8, col:17>
@@ -572,9 +572,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | | | |-FieldDecl {{.*}} <line:25:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | | `-FieldDecl {{.*}} <line:26:5> col:5 implicit 'int'
-// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       | | |   |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       | | |   | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -620,9 +620,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:25:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:26:5> col:5 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-CapturedStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -664,9 +664,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       |   | |-FieldDecl {{.*}} <line:25:3> col:3 implicit 'int'
-// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   | `-FieldDecl {{.*}} <line:26:5> col:5 implicit 'int'
-// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |     |-ForStmt {{.*}} <line:25:3, line:27:7>
 // CHECK-NEXT: |       |     | |-DeclStmt {{.*}} <line:25:8, col:17>
@@ -774,11 +774,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | | | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | | | |-FieldDecl {{.*}} <line:32:3> col:3 implicit 'int'
-// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | |-FieldDecl {{.*}} <line:33:5> col:5 implicit 'int'
-// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         | | |   |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         | | |   | |-DeclStmt {{.*}} <line:32:8, col:17>
@@ -839,11 +839,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:32:3> col:3 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:33:5> col:5 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |   |-CapturedStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |   | |-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
@@ -900,11 +900,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         |   |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         |   | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         |   | |-FieldDecl {{.*}} <line:32:3> col:3 implicit 'int'
-// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | |-FieldDecl {{.*}} <line:33:5> col:5 implicit 'int'
-// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   | `-FieldDecl {{.*}} <line:34:27> col:27 implicit 'int'
-// CHECK-NEXT:         |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         |   |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         |   `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |     |-ForStmt {{.*}} <line:32:3, line:35:9>
 // CHECK-NEXT:         |     | |-DeclStmt {{.*}} <line:32:8, col:17>
diff --git a/clang/test/AST/ast-dump-openmp-teams-distribute-parallel-for-simd.c b/clang/test/AST/ast-dump-openmp-teams-distribute-parallel-for-simd.c
index 85a67e2f7e04e..204b4fe1365e6 100644
--- a/clang/test/AST/ast-dump-openmp-teams-distribute-parallel-for-simd.c
+++ b/clang/test/AST/ast-dump-openmp-teams-distribute-parallel-for-simd.c
@@ -189,7 +189,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:6:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-OMPTeamsDistributeParallelForSimdDirective {{.*}} <line:5:1, col:47>
 // CHECK-NEXT: |       |   | `-CapturedStmt {{.*}} <line:6:3, line:7:5>
@@ -532,9 +532,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:13:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:14:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-OMPTeamsDistributeParallelForSimdDirective {{.*}} <line:12:1, col:47>
 // CHECK-NEXT: |       |   | `-CapturedStmt {{.*}} <line:13:3, line:15:7>
@@ -943,9 +943,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:21:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:22:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-OMPTeamsDistributeParallelForSimdDirective {{.*}} <line:20:1, col:59>
 // CHECK-NEXT: |       |   | |-OMPCollapseClause {{.*}} <col:48, col:58>
@@ -1376,9 +1376,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:29:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:30:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-OMPTeamsDistributeParallelForSimdDirective {{.*}} <line:28:1, col:59>
 // CHECK-NEXT: |       |   | |-OMPCollapseClause {{.*}} <col:48, col:58>
@@ -1893,11 +1893,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:37:23> col:23 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:38:25> col:25 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | `-FieldDecl {{.*}} <line:39:27> col:27 implicit 'int'
-// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |   |-OMPTeamsDistributeParallelForSimdDirective {{.*}} <line:36:1, col:59>
 // CHECK-NEXT:         |   | |-OMPCollapseClause {{.*}} <col:48, col:58>
diff --git a/clang/test/AST/ast-dump-openmp-teams-distribute-parallel-for.c b/clang/test/AST/ast-dump-openmp-teams-distribute-parallel-for.c
index 68b99a09a5590..8c86d6c8e3765 100644
--- a/clang/test/AST/ast-dump-openmp-teams-distribute-parallel-for.c
+++ b/clang/test/AST/ast-dump-openmp-teams-distribute-parallel-for.c
@@ -189,7 +189,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:6:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-OMPTeamsDistributeParallelForDirective {{.*}} <line:5:1, col:42>
 // CHECK-NEXT: |       |   | `-CapturedStmt {{.*}} <line:6:3, line:7:5>
@@ -532,9 +532,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:13:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:14:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-OMPTeamsDistributeParallelForDirective {{.*}} <line:12:1, col:42>
 // CHECK-NEXT: |       |   | `-CapturedStmt {{.*}} <line:13:3, line:15:7>
@@ -943,9 +943,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:21:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:22:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-OMPTeamsDistributeParallelForDirective {{.*}} <line:20:1, col:54>
 // CHECK-NEXT: |       |   | |-OMPCollapseClause {{.*}} <col:43, col:53>
@@ -1376,9 +1376,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:29:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:30:5> col:5 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-OMPTeamsDistributeParallelForDirective {{.*}} <line:28:1, col:54>
 // CHECK-NEXT: |       |   | |-OMPCollapseClause {{.*}} <col:43, col:53>
@@ -1893,11 +1893,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:37:3> col:3 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:38:5> col:5 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | `-FieldDecl {{.*}} <line:39:27> col:27 implicit 'int'
-// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |   |-OMPTeamsDistributeParallelForDirective {{.*}} <line:36:1, col:54>
 // CHECK-NEXT:         |   | |-OMPCollapseClause {{.*}} <col:43, col:53>
diff --git a/clang/test/AST/ast-dump-openmp-teams-distribute-simd.c b/clang/test/AST/ast-dump-openmp-teams-distribute-simd.c
index bbdb722fd760d..a4ad08b8bef2c 100644
--- a/clang/test/AST/ast-dump-openmp-teams-distribute-simd.c
+++ b/clang/test/AST/ast-dump-openmp-teams-distribute-simd.c
@@ -125,7 +125,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:6:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-OMPTeamsDistributeSimdDirective {{.*}} <line:5:1, col:34>
 // CHECK-NEXT: |       |   | `-CapturedStmt {{.*}} <line:6:3, line:7:5>
@@ -308,9 +308,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:13:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:14:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-OMPTeamsDistributeSimdDirective {{.*}} <line:12:1, col:34>
 // CHECK-NEXT: |       |   | `-CapturedStmt {{.*}} <line:13:3, line:15:7>
@@ -527,9 +527,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:21:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:22:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-OMPTeamsDistributeSimdDirective {{.*}} <line:20:1, col:46>
 // CHECK-NEXT: |       |   | |-OMPCollapseClause {{.*}} <col:35, col:45>
@@ -768,9 +768,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:29:23> col:23 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:30:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-OMPTeamsDistributeSimdDirective {{.*}} <line:28:1, col:46>
 // CHECK-NEXT: |       |   | |-OMPCollapseClause {{.*}} <col:35, col:45>
@@ -1061,11 +1061,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:37:23> col:23 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:38:25> col:25 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | `-FieldDecl {{.*}} <line:39:27> col:27 implicit 'int'
-// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |   |-OMPTeamsDistributeSimdDirective {{.*}} <line:36:1, col:46>
 // CHECK-NEXT:         |   | |-OMPCollapseClause {{.*}} <col:35, col:45>
diff --git a/clang/test/AST/ast-dump-openmp-teams-distribute.c b/clang/test/AST/ast-dump-openmp-teams-distribute.c
index 17689709faabe..8c1e6b70cff69 100644
--- a/clang/test/AST/ast-dump-openmp-teams-distribute.c
+++ b/clang/test/AST/ast-dump-openmp-teams-distribute.c
@@ -125,7 +125,7 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:6:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-OMPTeamsDistributeDirective {{.*}} <line:5:1, col:29>
 // CHECK-NEXT: |       |   | `-CapturedStmt {{.*}} <line:6:3, line:7:5>
@@ -308,9 +308,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:13:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:14:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-OMPTeamsDistributeDirective {{.*}} <line:12:1, col:29>
 // CHECK-NEXT: |       |   | `-CapturedStmt {{.*}} <line:13:3, line:15:7>
@@ -527,9 +527,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:21:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:22:25> col:25 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-OMPTeamsDistributeDirective {{.*}} <line:20:1, col:41>
 // CHECK-NEXT: |       |   | |-OMPCollapseClause {{.*}} <col:30, col:40>
@@ -768,9 +768,9 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT: |       | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT: |       | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT: |       | | |-FieldDecl {{.*}} <line:29:3> col:3 implicit 'int'
-// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | | `-FieldDecl {{.*}} <line:30:5> col:5 implicit 'int'
-// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT: |       | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT: |       | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT: |       |   |-OMPTeamsDistributeDirective {{.*}} <line:28:1, col:41>
 // CHECK-NEXT: |       |   | |-OMPCollapseClause {{.*}} <col:30, col:40>
@@ -1061,11 +1061,11 @@ void test_five(int x, int y, int z) {
 // CHECK-NEXT:         | |-RecordDecl {{.*}} <col:1> col:1 implicit struct definition
 // CHECK-NEXT:         | | |-CapturedRecordAttr {{.*}} <<invalid sloc>> Implicit
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:37:3> col:3 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | |-FieldDecl {{.*}} <line:38:5> col:5 implicit 'int'
-// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | | | `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | | `-FieldDecl {{.*}} <line:39:27> col:27 implicit 'int'
-// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit 9
+// CHECK-NEXT:         | |   `-OMPCaptureKindAttr {{.*}} <<invalid sloc>> Implicit {{.*}}
 // CHECK-NEXT:         | `-CapturedDecl {{.*}} <<invalid sloc>> <invalid sloc> nothrow
 // CHECK-NEXT:         |   |-OMPTeamsDistributeDirective {{.*}} <line:36:1, col:41>
 // CHECK-NEXT:         |   | |-OMPCollapseClause {{.*}} <col:30, col:40>
diff --git a/clang/test/AST/ast-dump-records.cpp b/clang/test/AST/ast-dump-records.cpp
index f379b2dba529b..cb7ac83204312 100644
--- a/clang/test/AST/ast-dump-records.cpp
+++ b/clang/test/AST/ast-dump-records.cpp
@@ -15,7 +15,7 @@ struct B;
 // CHECK: CXXRecordDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:1, col:8> col:8 referenced struct B
 
 struct A {
-  // CHECK: CXXRecordDecl 0x{{[^ ]*}} prev 0x{{[^ ]*}} <line:[[@LINE-1]]:1, line:[[@LINE+29]]:1> line:[[@LINE-1]]:8 struct A definition
+  // CHECK: CXXRecordDecl 0x{{[^ ]*}} prev 0x{{[^ ]*}} <line:[[@LINE-1]]:1, line:[[@LINE+32]]:1> line:[[@LINE-1]]:8 struct A definition
   // CHECK-NEXT: DefinitionData pass_in_registers aggregate standard_layout trivially_copyable pod trivial literal
   // CHECK-NEXT: DefaultConstructor exists trivial needs_implicit
   // CHECK-NEXT: CopyConstructor simple trivial has_const_param needs_implicit implicit_has_const_param
@@ -33,14 +33,17 @@ struct A {
   int d : 12;
   // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:3, col:11> col:7 d 'int'
   // CHECK-NEXT: ConstantExpr 0x{{[^ ]*}} <col:11> 'int'
+  // CHECK-NEXT: value: Int 12
   // CHECK-NEXT: IntegerLiteral 0x{{[^ ]*}} <col:11> 'int' 12
   int : 0;
   // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:3, col:9> col:7 'int'
   // CHECK-NEXT: ConstantExpr 0x{{[^ ]*}} <col:9> 'int'
+  // CHECK-NEXT: value: Int 0
   // CHECK-NEXT: IntegerLiteral 0x{{[^ ]*}} <col:9> 'int' 0
   int e : 10;
   // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:3, col:11> col:7 e 'int'
   // CHECK-NEXT: ConstantExpr 0x{{[^ ]*}} <col:11> 'int'
+  // CHECK-NEXT: value: Int 10
   // CHECK-NEXT: IntegerLiteral 0x{{[^ ]*}} <col:11> 'int' 10
   B *f;
   // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:3, col:6> col:6 f 'B *'
@@ -141,7 +144,7 @@ union F;
 // CHECK: CXXRecordDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:1, col:7> col:7 union F
 
 union E {
-  // CHECK: CXXRecordDecl 0x{{[^ ]*}} prev 0x{{[^ ]*}} <line:[[@LINE-1]]:1, line:[[@LINE+29]]:1> line:[[@LINE-1]]:7 union E definition
+  // CHECK: CXXRecordDecl 0x{{[^ ]*}} prev 0x{{[^ ]*}} <line:[[@LINE-1]]:1, line:[[@LINE+32]]:1> line:[[@LINE-1]]:7 union E definition
   // CHECK-NEXT: DefinitionData pass_in_registers aggregate standard_layout trivially_copyable pod trivial literal
   // CHECK-NEXT: DefaultConstructor exists trivial needs_implicit
   // CHECK-NEXT: CopyConstructor simple trivial has_const_param needs_implicit implicit_has_const_param
@@ -159,14 +162,17 @@ union E {
   int d : 12;
   // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:3, col:11> col:7 d 'int'
   // CHECK-NEXT: ConstantExpr 0x{{[^ ]*}} <col:11> 'int'
+  // CHECK-NEXT: value: Int 12
   // CHECK-NEXT: IntegerLiteral 0x{{[^ ]*}} <col:11> 'int' 12
   int : 0;
   // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:3, col:9> col:7 'int'
   // CHECK-NEXT: ConstantExpr 0x{{[^ ]*}} <col:9> 'int'
+  // CHECK-NEXT: value: Int 0
   // CHECK-NEXT: IntegerLiteral 0x{{[^ ]*}} <col:9> 'int' 0
   int e : 10;
   // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:3, col:11> col:7 e 'int'
   // CHECK-NEXT: ConstantExpr 0x{{[^ ]*}} <col:11> 'int'
+  // CHECK-NEXT: value: Int 10
   // CHECK-NEXT: IntegerLiteral 0x{{[^ ]*}} <col:11> 'int' 10
   B *f;
   // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:3, col:6> col:6 f 'B *'
diff --git a/clang/test/AST/ast-dump-recovery.cpp b/clang/test/AST/ast-dump-recovery.cpp
index e2a7151306287..740864a26481b 100644
--- a/clang/test/AST/ast-dump-recovery.cpp
+++ b/clang/test/AST/ast-dump-recovery.cpp
@@ -4,7 +4,6 @@
 int some_func(int *);
 
 // CHECK:     VarDecl {{.*}} invalid_call
-// CHECK-NEXT: `-ImplicitCastExpr {{.*}} 'int' contains-errors
 // CHECK-NEXT:  `-RecoveryExpr {{.*}} 'int' contains-errors
 // CHECK-NEXT:    |-UnresolvedLookupExpr {{.*}} 'some_func'
 // CHECK-NEXT:    `-IntegerLiteral {{.*}} 123
@@ -34,7 +33,6 @@ int ambig_func(double);
 int ambig_func(float);
 
 // CHECK:     VarDecl {{.*}} ambig_call
-// CHECK-NEXT: `-ImplicitCastExpr {{.*}} 'int' contains-errors
 // CHECK-NEXT:  `-RecoveryExpr {{.*}} 'int' contains-errors
 // CHECK-NEXT:    |-UnresolvedLookupExpr {{.*}} 'ambig_func'
 // CHECK-NEXT:    `-IntegerLiteral {{.*}} 123
@@ -211,3 +209,16 @@ struct {
 } NoCrashOnInvalidInitList = {
   .abc = nullptr,
 };
+
+// Verify the value category of recovery expression.
+int prvalue(int);
+int &lvalue(int);
+int &&xvalue(int);
+void ValueCategory() {
+  // CHECK:  RecoveryExpr {{.*}} 'int' contains-errors
+  prvalue(); // call to a function (nonreference return type) yields a prvalue (not print by default)
+  // CHECK:  RecoveryExpr {{.*}} 'int' contains-errors lvalue
+  lvalue(); // call to a function (lvalue reference return type) yields an lvalue.
+  // CHECK:  RecoveryExpr {{.*}} 'int' contains-errors xvalue
+  xvalue(); // call to a function (rvalue reference return type) yields an xvalue.
+}
diff --git a/clang/test/AST/ast-dump-stmt.cpp b/clang/test/AST/ast-dump-stmt.cpp
index e4d8d8218b4e3..585a94bd1920a 100644
--- a/clang/test/AST/ast-dump-stmt.cpp
+++ b/clang/test/AST/ast-dump-stmt.cpp
@@ -130,6 +130,7 @@ void TestIf(bool b) {
     ;
   // CHECK: IfStmt 0x{{[^ ]*}} <line:[[@LINE-2]]:3, line:[[@LINE-1]]:5>
   // CHECK-NEXT: ConstantExpr 0x{{[^ ]*}} <line:[[@LINE-3]]:17, col:30> 'bool'
+  // CHECK-NEXT: value: Int 1
   // CHECK-NEXT: BinaryOperator
   // CHECK-NEXT: UnaryExprOrTypeTraitExpr
   // CHECK-NEXT: ParenExpr
@@ -144,6 +145,7 @@ void TestIf(bool b) {
     ;
   // CHECK: IfStmt 0x{{[^ ]*}} <line:[[@LINE-4]]:3, line:[[@LINE-1]]:5> has_else
   // CHECK-NEXT: ConstantExpr 0x{{[^ ]*}} <line:[[@LINE-5]]:17, col:30> 'bool'
+  // CHECK-NEXT: value: Int 1
   // CHECK-NEXT: BinaryOperator
   // CHECK-NEXT: UnaryExprOrTypeTraitExpr
   // CHECK-NEXT: ParenExpr
diff --git a/clang/test/AST/ast-dump-template-decls-json.cpp b/clang/test/AST/ast-dump-template-decls-json.cpp
index 658358c5789e6..f074d6f0137db 100644
--- a/clang/test/AST/ast-dump-template-decls-json.cpp
+++ b/clang/test/AST/ast-dump-template-decls-json.cpp
@@ -597,13 +597,31 @@ void V<Ty>::f() {}
 // CHECK-NEXT:      "kind": "TemplateArgument",
 // CHECK-NEXT:      "type": {
 // CHECK-NEXT:       "qualType": "float"
-// CHECK-NEXT:      }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "BuiltinType",
+// CHECK-NEXT:        "type": {
+// CHECK-NEXT:         "qualType": "float"
+// CHECK-NEXT:        }
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
 // CHECK-NEXT:     },
 // CHECK-NEXT:     {
 // CHECK-NEXT:      "kind": "TemplateArgument",
 // CHECK-NEXT:      "type": {
 // CHECK-NEXT:       "qualType": "int"
-// CHECK-NEXT:      }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "BuiltinType",
+// CHECK-NEXT:        "type": {
+// CHECK-NEXT:         "qualType": "int"
+// CHECK-NEXT:        }
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
 // CHECK-NEXT:     },
 // CHECK-NEXT:     {
 // CHECK-NEXT:      "id": "0x{{.*}}",
@@ -1120,7 +1138,16 @@ void V<Ty>::f() {}
 // CHECK-NEXT:        "kind": "TemplateArgument",
 // CHECK-NEXT:        "type": {
 // CHECK-NEXT:         "qualType": "int"
-// CHECK-NEXT:        }
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "inner": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:          "id": "0x{{.*}}",
+// CHECK-NEXT:          "kind": "BuiltinType",
+// CHECK-NEXT:          "type": {
+// CHECK-NEXT:           "qualType": "int"
+// CHECK-NEXT:          }
+// CHECK-NEXT:         }
+// CHECK-NEXT:        ]
 // CHECK-NEXT:       }
 // CHECK-NEXT:      ]
 // CHECK-NEXT:     },
@@ -1238,7 +1265,16 @@ void V<Ty>::f() {}
 // CHECK-NEXT:        "kind": "TemplateArgument",
 // CHECK-NEXT:        "type": {
 // CHECK-NEXT:         "qualType": "void"
-// CHECK-NEXT:        }
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "inner": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:          "id": "0x{{.*}}",
+// CHECK-NEXT:          "kind": "BuiltinType",
+// CHECK-NEXT:          "type": {
+// CHECK-NEXT:           "qualType": "void"
+// CHECK-NEXT:          }
+// CHECK-NEXT:         }
+// CHECK-NEXT:        ]
 // CHECK-NEXT:       }
 // CHECK-NEXT:      ]
 // CHECK-NEXT:     },
@@ -1511,7 +1547,16 @@ void V<Ty>::f() {}
 // CHECK-NEXT:      "kind": "TemplateArgument",
 // CHECK-NEXT:      "type": {
 // CHECK-NEXT:       "qualType": "int"
-// CHECK-NEXT:      }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "BuiltinType",
+// CHECK-NEXT:        "type": {
+// CHECK-NEXT:         "qualType": "int"
+// CHECK-NEXT:        }
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
 // CHECK-NEXT:     },
 // CHECK-NEXT:     {
 // CHECK-NEXT:      "id": "0x{{.*}}",
@@ -1799,13 +1844,38 @@ void V<Ty>::f() {}
 // CHECK-NEXT:      "kind": "TemplateArgument",
 // CHECK-NEXT:      "type": {
 // CHECK-NEXT:       "qualType": "type-parameter-0-0"
-// CHECK-NEXT:      }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "TemplateTypeParmType",
+// CHECK-NEXT:        "type": {
+// CHECK-NEXT:         "qualType": "type-parameter-0-0"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "isDependent": true,
+// CHECK-NEXT:        "isInstantiationDependent": true,
+// CHECK-NEXT:        "depth": 0,
+// CHECK-NEXT:        "index": 0,
+// CHECK-NEXT:        "decl": {
+// CHECK-NEXT:         "id": "0x0"
+// CHECK-NEXT:        }
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
 // CHECK-NEXT:     },
 // CHECK-NEXT:     {
 // CHECK-NEXT:      "kind": "TemplateArgument",
 // CHECK-NEXT:      "type": {
 // CHECK-NEXT:       "qualType": "int"
-// CHECK-NEXT:      }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "BuiltinType",
+// CHECK-NEXT:        "type": {
+// CHECK-NEXT:         "qualType": "int"
+// CHECK-NEXT:        }
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
 // CHECK-NEXT:     },
 // CHECK-NEXT:     {
 // CHECK-NEXT:      "id": "0x{{.*}}",
diff --git a/clang/test/AST/ast-dump-template-decls.cpp b/clang/test/AST/ast-dump-template-decls.cpp
index 29d2335ce0151..e58731ae6d51d 100644
--- a/clang/test/AST/ast-dump-template-decls.cpp
+++ b/clang/test/AST/ast-dump-template-decls.cpp
@@ -79,8 +79,8 @@ struct S {};
 template <typename Ty>
 // CHECK: ClassTemplatePartialSpecializationDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:1, line:[[@LINE+4]]:20> col:8 struct S definition
 // CHECK: TemplateArgument type 'type-parameter-0-0'
-// CHECK-NEXT: TemplateArgument type 'int'
-// CHECK-NEXT: TemplateTypeParmDecl 0x{{[^ ]*}} <line:[[@LINE-4]]:11, col:20> col:20 referenced typename depth 0 index 0 Ty
+// CHECK: TemplateArgument type 'int'
+// CHECK: TemplateTypeParmDecl 0x{{[^ ]*}} <line:[[@LINE-4]]:11, col:20> col:20 referenced typename depth 0 index 0 Ty
 struct S<Ty, int> {};
 
 template <auto>
diff --git a/clang/test/AST/coroutine-source-location-crash.cpp b/clang/test/AST/coroutine-source-location-crash.cpp
index 24fe3bcb224ef..6c0184d2076d4 100644
--- a/clang/test/AST/coroutine-source-location-crash.cpp
+++ b/clang/test/AST/coroutine-source-location-crash.cpp
@@ -24,7 +24,7 @@ struct coro_t {
   struct promise_type {
     coro_t get_return_object();
     suspend_never initial_suspend();
-    suspend_never final_suspend();
+    suspend_never final_suspend() noexcept;
     void return_void();
     static void unhandled_exception();
   };
diff --git a/clang/test/AST/float16.cpp b/clang/test/AST/float16.cpp
index c886aa9cf0f1b..807289ac0a7f1 100644
--- a/clang/test/AST/float16.cpp
+++ b/clang/test/AST/float16.cpp
@@ -178,7 +178,7 @@ template <class C> C func1t(C arg) {
 //CHECK-NEXT: | |       `-FloatingLiteral {{.*}} '_Float16' 2.000000e+00
 //CHECK-NEXT: | `-FunctionDecl {{.*}} used func1t '_Float16 (_Float16)'
 //CHECK-NEXT: |   |-TemplateArgument type '_Float16'
-//CHECK-NEXT: |   |-ParmVarDecl {{.*}} used arg '_Float16':'_Float16'
+//CHECK:      |   |-ParmVarDecl {{.*}} used arg '_Float16':'_Float16'
 //CHECK-NEXT: |   `-CompoundStmt
 //CHECK-NEXT: |     `-ReturnStmt
 //CHECK-NEXT: |       `-BinaryOperator {{.*}} '_Float16' '*'
diff --git a/clang/test/AST/pr43983.cpp b/clang/test/AST/pr43983.cpp
index dd7ce76a38bc8..c27d825dbb1b0 100644
--- a/clang/test/AST/pr43983.cpp
+++ b/clang/test/AST/pr43983.cpp
@@ -9,6 +9,7 @@
 
 struct B { _Alignas(64) struct { int b; };   };
 
-// CHECK: AlignedAttr {{.*}} _Alignas
-// CHECK: ConstantExpr {{.*}} 64
-// CHECK: IntegerLiteral {{.*}} 64
+// CHECK:  | `-AlignedAttr {{.*}} <col:12> _Alignas
+// CHECK-NEXT:  |   `-ConstantExpr {{.*}} <col:21> 'int'
+// CHECK-NEXT:  |     |-value: Int 64
+// CHECK-NEXT:  |     `-IntegerLiteral {{.*}} <col:21> 'int' 64
diff --git a/clang/test/AST/regression-new-expr-crash.cpp b/clang/test/AST/regression-new-expr-crash.cpp
new file mode 100644
index 0000000000000..81dd193b93e88
--- /dev/null
+++ b/clang/test/AST/regression-new-expr-crash.cpp
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fsyntax-only %s
+
+struct Bar {int a;};
+const Bar arr[2] = {{1}};
+
+struct Foo {};
+
+const int b = 2;
+
+void foo(int a) {
+  Foo *foo_array;
+  foo_array = new Foo[arr[0].a];
+}
diff --git a/clang/test/Analysis/Inputs/system-header-simulator-cxx.h b/clang/test/Analysis/Inputs/system-header-simulator-cxx.h
index 4729b0439ad0f..fe4b9d081e9c8 100644
--- a/clang/test/Analysis/Inputs/system-header-simulator-cxx.h
+++ b/clang/test/Analysis/Inputs/system-header-simulator-cxx.h
@@ -946,10 +946,15 @@ namespace std {
   template <typename T> // TODO: Implement the stub for deleter.
   class unique_ptr {
   public:
+    unique_ptr() {}
+    unique_ptr(T *) {}
     unique_ptr(const unique_ptr &) = delete;
     unique_ptr(unique_ptr &&);
 
     T *get() const;
+    T *release() const;
+    void reset(T *p = nullptr) const;
+    void swap(unique_ptr<T> &p) const;
 
     typename std::add_lvalue_reference<T>::type operator*() const;
     T *operator->() const;
diff --git a/clang/test/Analysis/analyzer-config.c b/clang/test/Analysis/analyzer-config.c
index 1d4a96389fb36..d286f1258c21a 100644
--- a/clang/test/Analysis/analyzer-config.c
+++ b/clang/test/Analysis/analyzer-config.c
@@ -13,6 +13,7 @@
 // CHECK-NEXT: alpha.security.MmapWriteExec:MmapProtRead = 0x01
 // CHECK-NEXT: alpha.security.taint.TaintPropagation:Config = ""
 // CHECK-NEXT: apiModeling.StdCLibraryFunctions:DisplayLoadedSummaries = false
+// CHECK-NEXT: apiModeling.StdCLibraryFunctions:ModelPOSIX = false
 // CHECK-NEXT: apply-fixits = false
 // CHECK-NEXT: avoid-suppressing-null-argument-paths = false
 // CHECK-NEXT: c++-allocator-inlining = true
@@ -39,9 +40,11 @@
 // CHECK-NEXT: core.CallAndMessage:ParameterCount = true
 // CHECK-NEXT: core.CallAndMessage:UndefReceiver = true
 // CHECK-NEXT: cplusplus.Move:WarnOn = KnownsAndLocals
+// CHECK-NEXT: cplusplus.SmartPtrModeling:ModelSmartPtrDereference = false
 // CHECK-NEXT: crosscheck-with-z3 = false
 // CHECK-NEXT: ctu-dir = ""
-// CHECK-NEXT: ctu-import-threshold = 100
+// CHECK-NEXT: ctu-import-cpp-threshold = 8
+// CHECK-NEXT: ctu-import-threshold = 24
 // CHECK-NEXT: ctu-index-name = externalDefMap.txt
 // CHECK-NEXT: ctu-invocation-list = invocations.yaml
 // CHECK-NEXT: deadcode.DeadStores:ShowFixIts = false
@@ -50,6 +53,7 @@
 // CHECK-NEXT: debug.AnalysisOrder:Bind = false
 // CHECK-NEXT: debug.AnalysisOrder:EndAnalysis = false
 // CHECK-NEXT: debug.AnalysisOrder:EndFunction = false
+// CHECK-NEXT: debug.AnalysisOrder:EvalCall = false
 // CHECK-NEXT: debug.AnalysisOrder:LiveSymbols = false
 // CHECK-NEXT: debug.AnalysisOrder:NewAllocator = false
 // CHECK-NEXT: debug.AnalysisOrder:PointerEscape = false
diff --git a/clang/test/Analysis/checker-plugins.c b/clang/test/Analysis/checker-plugins.c
index fbc9c9bd1c22b..69fab8fa6eed1 100644
--- a/clang/test/Analysis/checker-plugins.c
+++ b/clang/test/Analysis/checker-plugins.c
@@ -116,4 +116,5 @@ void caller() {
 // RUN:   2>&1 | FileCheck %s -check-prefix=CHECK-CHECKER-OPTION-HELP
 
 // CHECK-CHECKER-OPTION-HELP: example.MyChecker:ExampleOption  (bool) This is an
-// CHECK-CHECKER-OPTION-HELP-SAME: example checker opt. (default: false)
+// CHECK-CHECKER-OPTION-HELP-SAME: example checker opt. (default:
+// CHECK-CHECKER-OPTION-HELP-NEXT: false)
diff --git a/clang/test/Analysis/ctu-import-threshold.c b/clang/test/Analysis/ctu-import-threshold.c
index 82711b873d106..62bbeda20d5ba 100644
--- a/clang/test/Analysis/ctu-import-threshold.c
+++ b/clang/test/Analysis/ctu-import-threshold.c
@@ -1,5 +1,6 @@
 // Ensure analyzer option 'ctu-import-threshold' is a recognized option.
 //
 // RUN: %clang_cc1 -analyze -analyzer-config ctu-import-threshold=30 -verify %s
+// RUN: %clang_cc1 -analyze -analyzer-config ctu-import-cpp-threshold=30 -verify %s
 //
 // expected-no-diagnostics
diff --git a/clang/test/Analysis/cxxctr-evalcall-analysis-order.cpp b/clang/test/Analysis/cxxctr-evalcall-analysis-order.cpp
new file mode 100644
index 0000000000000..0e1ec2f9de566
--- /dev/null
+++ b/clang/test/Analysis/cxxctr-evalcall-analysis-order.cpp
@@ -0,0 +1,33 @@
+// RUN: %clang_analyze_cc1 %s \
+// RUN:  -analyzer-checker=debug.AnalysisOrder \
+// RUN:  -analyzer-config debug.AnalysisOrder:EvalCall=true \
+// RUN:  -analyzer-config debug.AnalysisOrder:PreCall=true \
+// RUN:  -analyzer-config debug.AnalysisOrder:PostCall=true \
+// RUN:  2>&1 | FileCheck %s
+
+// This test ensures that eval::Call event will be triggered for constructors.
+
+class C {
+public:
+  C(){};
+  C(int x){};
+  C(int x, int y){};
+};
+
+void foo() {
+  C C0;
+  C C1(42);
+  C *C2 = new C{2, 3};
+}
+
+// CHECK:  PreCall (C::C) [CXXConstructorCall]
+// CHECK-NEXT:  EvalCall (C::C) {argno: 0} [CXXConstructorCall]
+// CHECK-NEXT:  PostCall (C::C) [CXXConstructorCall]
+// CHECK-NEXT:  PreCall (C::C) [CXXConstructorCall]
+// CHECK-NEXT:  EvalCall (C::C) {argno: 1} [CXXConstructorCall]
+// CHECK-NEXT:  PostCall (C::C) [CXXConstructorCall]
+// CHECK-NEXT:  PreCall (operator new) [CXXAllocatorCall]
+// CHECK-NEXT:  PostCall (operator new) [CXXAllocatorCall]
+// CHECK-NEXT:  PreCall (C::C) [CXXConstructorCall]
+// CHECK-NEXT:  EvalCall (C::C) {argno: 2} [CXXConstructorCall]
+// CHECK-NEXT:  PostCall (C::C) [CXXConstructorCall]
diff --git a/clang/test/Analysis/exploded-graph-rewriter/l_name_starts_with_l.cpp b/clang/test/Analysis/exploded-graph-rewriter/l_name_starts_with_l.cpp
new file mode 100644
index 0000000000000..a5d2443ea0b45
--- /dev/null
+++ b/clang/test/Analysis/exploded-graph-rewriter/l_name_starts_with_l.cpp
@@ -0,0 +1,28 @@
+// CAUTION: The name of this file should start with `l` for proper tests.
+// FIXME: Figure out how to use %clang_analyze_cc1 with our lit.local.cfg.
+// RUN: %clang_cc1 -analyze -triple x86_64-unknown-linux-gnu \
+// RUN:                     -analyzer-checker=core \
+// RUN:                     -analyzer-dump-egraph=%t.dot %s
+// RUN: %exploded_graph_rewriter %t.dot | FileCheck %s
+// REQUIRES: asserts
+
+void test1() {
+  // Here __FILE__ macros produces a string with `\` delimiters on Windows
+  // and the name of the file starts with `l`.
+  char text[] = __FILE__;
+}
+
+void test2() {
+  // Here `\l` is in the middle of the literal.
+  char text[] = "string\\literal";
+}
+
+void test() {
+  test1();
+  test2();
+}
+
+// This test is passed if exploded_graph_rewriter handles dot file without errors.
+// CHECK: digraph "ExplodedGraph"
+// CHECK: shape=record,label=<<table border="0">
+// CHECK: char text[] = "string\\literal";
diff --git a/clang/test/Analysis/exploded-graph-rewriter/win_path_forbidden_chars.cpp b/clang/test/Analysis/exploded-graph-rewriter/win_path_forbidden_chars.cpp
new file mode 100644
index 0000000000000..4eac964a4f440
--- /dev/null
+++ b/clang/test/Analysis/exploded-graph-rewriter/win_path_forbidden_chars.cpp
@@ -0,0 +1,20 @@
+// FIXME: Figure out how to use %clang_analyze_cc1 with our lit.local.cfg.
+// RUN: %clang_cc1 -analyze -triple x86_64-unknown-linux-gnu \
+// RUN:                     -analyzer-checker=core \
+// RUN:                     -analyzer-dump-egraph=%t.dot %s
+// RUN: %exploded_graph_rewriter --verbose %t.dot 2>&1 | FileCheck %s
+// REQUIRES: asserts
+// UNSUPPORTED: !windows
+
+// Angle brackets shall not be presented in the field `file`,
+// because exploded_graph_rewriter handles it as a file path
+// and such symbols are forbidden on Windows platform.
+
+void test() {
+  // This produces angle brackets.
+  char text[] = __FILE__;
+}
+
+// This test is passed if exploded_graph_rewriter handles dot file without errors.
+// CHECK: DEBUG:root:Line: digraph "Exploded Graph"
+// CHECK: \"file\": \"scratch space\"
diff --git a/clang/test/Analysis/fuchsia_handle.cpp b/clang/test/Analysis/fuchsia_handle.cpp
index c20622be543f6..dade5261bd78e 100644
--- a/clang/test/Analysis/fuchsia_handle.cpp
+++ b/clang/test/Analysis/fuchsia_handle.cpp
@@ -261,7 +261,7 @@ struct HandleWrapperUnkonwDtor {
   T handle;
 };
 
-void doNotWarnOnUnkownDtor() {
+void doNotWarnOnUnknownDtor() {
   HandleWrapperUnkonwDtor<zx_handle_t> w1;
   zx_handle_t sb;
   if (zx_channel_create(0, w1.get_handle_address(), &sb))
diff --git a/clang/test/Analysis/invalidated-iterator.cpp b/clang/test/Analysis/invalidated-iterator.cpp
index a9ccc3b758348..778a8e01d9938 100644
--- a/clang/test/Analysis/invalidated-iterator.cpp
+++ b/clang/test/Analysis/invalidated-iterator.cpp
@@ -120,3 +120,80 @@ void assignment(std::vector<int> &V) {
   V.erase(i);
   auto j = V.cbegin(); // no-warning
 }
+
+template<typename T>
+struct cont_with_ptr_iterator {
+  T *begin() const;
+  T *end() const;
+  T &operator[](size_t);
+  void push_back(const T&);
+  T* erase(T*);
+};
+
+void invalidated_dereference_end_ptr_iterator(cont_with_ptr_iterator<int> &C) {
+  auto i = C.begin();
+  C.erase(i);
+  (void) *i; // expected-warning{{Invalidated iterator accessed}}
+}
+
+void invalidated_prefix_increment_end_ptr_iterator(
+    cont_with_ptr_iterator<int> &C) {
+  auto i = C.begin();
+  C.erase(i);
+  ++i; // expected-warning{{Invalidated iterator accessed}}
+}
+
+void invalidated_prefix_decrement_end_ptr_iterator(
+    cont_with_ptr_iterator<int> &C) {
+  auto i = C.begin() + 1;
+  C.erase(i);
+  --i; // expected-warning{{Invalidated iterator accessed}}
+}
+
+void invalidated_postfix_increment_end_ptr_iterator(
+    cont_with_ptr_iterator<int> &C) {
+  auto i = C.begin();
+  C.erase(i);
+  i++; // expected-warning{{Invalidated iterator accessed}}
+}
+
+void invalidated_postfix_decrement_end_ptr_iterator(
+    cont_with_ptr_iterator<int> &C) {
+  auto i = C.begin() + 1;
+  C.erase(i);
+  i--; // expected-warning{{Invalidated iterator accessed}}
+}
+
+void invalidated_increment_by_2_end_ptr_iterator(
+    cont_with_ptr_iterator<int> &C) {
+  auto i = C.begin();
+  C.erase(i);
+  i += 2; // expected-warning{{Invalidated iterator accessed}}
+}
+
+void invalidated_increment_by_2_copy_end_ptr_iterator(
+    cont_with_ptr_iterator<int> &C) {
+  auto i = C.begin();
+  C.erase(i);
+  auto j = i + 2; // expected-warning{{Invalidated iterator accessed}}
+}
+
+void invalidated_decrement_by_2_end_ptr_iterator(
+    cont_with_ptr_iterator<int> &C) {
+  auto i = C.begin();
+  C.erase(i);
+  i -= 2; // expected-warning{{Invalidated iterator accessed}}
+}
+
+void invalidated_decrement_by_2_copy_end_ptr_iterator(
+    cont_with_ptr_iterator<int> &C) {
+  auto i = C.begin();
+  C.erase(i);
+  auto j = i - 2; // expected-warning{{Invalidated iterator accessed}}
+}
+
+void invalidated_subscript_end_ptr_iterator(cont_with_ptr_iterator<int> &C) {
+  auto i = C.begin();
+  C.erase(i);
+  (void) i[1]; // expected-warning{{Invalidated iterator accessed}}
+}
diff --git a/clang/test/Analysis/iterator-modeling.cpp b/clang/test/Analysis/iterator-modeling.cpp
index bb37a7565ca16..f19848b8dc935 100644
--- a/clang/test/Analysis/iterator-modeling.cpp
+++ b/clang/test/Analysis/iterator-modeling.cpp
@@ -1,12 +1,12 @@
-// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,debug.DebugIteratorModeling,debug.ExprInspection -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=false %s -verify
+// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,debug.DebugIteratorModeling,debug.ExprInspection -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=false %s -verify -analyzer-config display-checker-name=false
 
-// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,debug.DebugIteratorModeling,debug.ExprInspection -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=true -DINLINE=1 %s -verify
+// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,debug.DebugIteratorModeling,debug.ExprInspection -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=true -DINLINE=1 %s -verify -analyzer-config display-checker-name=false
 
-// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,debug.DebugIteratorModeling,debug.ExprInspection -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=true -DINLINE=1 -DSTD_ADVANCE_INLINE_LEVEL=0 %s -verify
+// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,debug.DebugIteratorModeling,debug.ExprInspection -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=true -DINLINE=1 -DSTD_ADVANCE_INLINE_LEVEL=0 %s -verify -analyzer-config display-checker-name=false
 
-// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,debug.DebugIteratorModeling,debug.ExprInspection -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=true -DINLINE=1 -DSTD_ADVANCE_INLINE_LEVEL=1 %s -verify
+// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,debug.DebugIteratorModeling,debug.ExprInspection -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=true -DINLINE=1 -DSTD_ADVANCE_INLINE_LEVEL=1 %s -verify -analyzer-config display-checker-name=false
 
-// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,debug.DebugIteratorModeling,debug.ExprInspection -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=true -DINLINE=1 -DSTD_ADVANCE_INLINE_LEVEL=2 %s -verify
+// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,debug.DebugIteratorModeling,debug.ExprInspection -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=true -DINLINE=1 -DSTD_ADVANCE_INLINE_LEVEL=2 %s -verify -analyzer-config display-checker-name=false
 
 // RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,alpha.cplusplus.IteratorModeling,debug.ExprInspection -analyzer-config aggressive-binary-operation-simplification=true %s 2>&1 | FileCheck %s
 
@@ -18,6 +18,7 @@ template <typename Container>
 long clang_analyzer_container_end(const Container&);
 template <typename Iterator>
 long clang_analyzer_iterator_position(const Iterator&);
+long clang_analyzer_iterator_position(int*);
 template <typename Iterator>
 void* clang_analyzer_iterator_container(const Iterator&);
 template <typename Iterator>
@@ -33,7 +34,7 @@ void begin(const std::vector<int> &v) {
 
   clang_analyzer_eval(clang_analyzer_iterator_container(i) == &v); // expected-warning{{TRUE}}
   clang_analyzer_denote(clang_analyzer_container_begin(v), "$v.begin()");
-  clang_analyzer_express(clang_analyzer_iterator_position(i)); //expected-warning{{$v.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning-re {{$v.begin(){{$}}}}
 
   if (i != v.begin()) {
     clang_analyzer_warnIfReached();
@@ -45,7 +46,7 @@ void end(const std::vector<int> &v) {
 
   clang_analyzer_eval(clang_analyzer_iterator_container(i) == &v); // expected-warning{{TRUE}}
   clang_analyzer_denote(clang_analyzer_container_end(v), "$v.end()");
-  clang_analyzer_express(clang_analyzer_iterator_position(i)); //expected-warning{{$v.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning-re {{$v.end(){{$}}}}
 
   if (i != v.end()) {
     clang_analyzer_warnIfReached();
@@ -59,8 +60,8 @@ void prefix_increment(const std::vector<int> &v) {
 
   auto j = ++i;
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i)); //expected-warning{{$v.begin() + 1}}
-  clang_analyzer_express(clang_analyzer_iterator_position(j)); //expected-warning{{$v.begin() + 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning-re {{$v.begin() + 1{{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(j)); // expected-warning-re {{$v.begin() + 1{{$}}}}
 }
 
 void prefix_decrement(const std::vector<int> &v) {
@@ -70,8 +71,8 @@ void prefix_decrement(const std::vector<int> &v) {
 
   auto j = --i;
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i)); //expected-warning{{$v.end() - 1}}
-  clang_analyzer_express(clang_analyzer_iterator_position(j)); //expected-warning{{$v.end() - 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning-re {{$v.end() - 1{{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(j)); // expected-warning-re {{$v.end() - 1{{$}}}}
 }
 
 void postfix_increment(const std::vector<int> &v) {
@@ -81,8 +82,8 @@ void postfix_increment(const std::vector<int> &v) {
 
   auto j = i++;
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i)); //expected-warning{{$v.begin() + 1}}
-  clang_analyzer_express(clang_analyzer_iterator_position(j)); //expected-warning{{$v.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning-re {{$v.begin() + 1{{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(j)); // expected-warning-re {{$v.begin(){{$}}}}
 }
 
 void postfix_decrement(const std::vector<int> &v) {
@@ -92,8 +93,8 @@ void postfix_decrement(const std::vector<int> &v) {
 
   auto j = i--;
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i)); //expected-warning{{$v.end() - 1}}
-  clang_analyzer_express(clang_analyzer_iterator_position(j)); //expected-warning{{$v.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning-re {{$v.end() - 1{{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(j)); // expected-warning-re {{$v.end(){{$}}}}
 }
 
 void plus_equal(const std::vector<int> &v) {
@@ -103,7 +104,7 @@ void plus_equal(const std::vector<int> &v) {
 
   i += 2;
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i)); //expected-warning{{$v.begin() + 2}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning-re {{$v.begin() + 2{{$}}}}
 }
 
 void plus_equal_negative(const std::vector<int> &v) {
@@ -113,7 +114,7 @@ void plus_equal_negative(const std::vector<int> &v) {
 
   i += -2;
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i)); //expected-warning{{$v.end() - 2}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning-re {{$v.end() - 2{{$}}}}
 }
 
 void minus_equal(const std::vector<int> &v) {
@@ -123,7 +124,7 @@ void minus_equal(const std::vector<int> &v) {
 
   i -= 2;
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i)); //expected-warning{{$v.end() - 2}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning-re {{$v.end() - 2{{$}}}}
 }
 
 void minus_equal_negative(const std::vector<int> &v) {
@@ -133,7 +134,7 @@ void minus_equal_negative(const std::vector<int> &v) {
 
   i -= -2;
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i)); //expected-warning{{$v.begin() + 2}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning-re {{$v.begin() + 2{{$}}}}
 }
 
 void copy(const std::vector<int> &v) {
@@ -144,7 +145,8 @@ void copy(const std::vector<int> &v) {
   auto i2 = i1;
 
   clang_analyzer_eval(clang_analyzer_iterator_container(i2) == &v); // expected-warning{{TRUE}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); //expected-warning{{$v.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1));     // expected-warning-re {{$v.end(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2));     // expected-warning-re {{$v.end(){{$}}}}
 }
 
 void plus(const std::vector<int> &v) {
@@ -155,7 +157,8 @@ void plus(const std::vector<int> &v) {
   auto i2 = i1 + 2;
 
   clang_analyzer_eval(clang_analyzer_iterator_container(i2) == &v); // expected-warning{{TRUE}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); //expected-warning{{$v.begin() + 2}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1));     // expected-warning-re{{$v.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2));     // expected-warning-re{{$v.begin() + 2{{$}}}}
 }
 
 void plus_negative(const std::vector<int> &v) {
@@ -166,7 +169,8 @@ void plus_negative(const std::vector<int> &v) {
   auto i2 = i1 + (-2);
 
   clang_analyzer_eval(clang_analyzer_iterator_container(i2) == &v); // expected-warning{{TRUE}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); //expected-warning{{$v.end() - 2}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1));     // expected-warning-re {{$v.end(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2));     // expected-warning-re {{$v.end() - 2{{$}}}}
 }
 
 void minus(const std::vector<int> &v) {
@@ -177,7 +181,8 @@ void minus(const std::vector<int> &v) {
   auto i2 = i1 - 2;
 
   clang_analyzer_eval(clang_analyzer_iterator_container(i2) == &v); // expected-warning{{TRUE}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); //expected-warning{{$v.end() - 2}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1));     // expected-warning-re {{$v.end(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2));     // expected-warning-re {{$v.end() - 2{{$}}}}
 }
 
 void minus_negative(const std::vector<int> &v) {
@@ -188,7 +193,8 @@ void minus_negative(const std::vector<int> &v) {
   auto i2 = i1 - (-2);
 
   clang_analyzer_eval(clang_analyzer_iterator_container(i2) == &v); // expected-warning{{TRUE}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); //expected-warning{{$v.begin() + 2}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1));     // expected-warning-re {{$v.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2));     // expected-warning-re {{$v.begin() + 2{{$}}}}
 }
 
 void copy_and_increment1(const std::vector<int> &v) {
@@ -199,8 +205,8 @@ void copy_and_increment1(const std::vector<int> &v) {
   auto i2 = i1;
   ++i1;
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); //expected-warning{{$v.begin() + 1}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); //expected-warning{{$v.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$v.begin() + 1{{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$v.begin(){{$}}}}
 }
 
 void copy_and_increment2(const std::vector<int> &v) {
@@ -211,8 +217,8 @@ void copy_and_increment2(const std::vector<int> &v) {
   auto i2 = i1;
   ++i2;
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); //expected-warning{{$v.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); //expected-warning{{$v.begin() + 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$v.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$v.begin() + 1{{$}}}}
 }
 
 void copy_and_decrement1(const std::vector<int> &v) {
@@ -223,8 +229,8 @@ void copy_and_decrement1(const std::vector<int> &v) {
   auto i2 = i1;
   --i1;
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); //expected-warning{{$v.end() - 1}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); //expected-warning{{$v.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$v.end() - 1{{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$v.end(){{$}}}}
 }
 
 void copy_and_decrement2(const std::vector<int> &v) {
@@ -235,8 +241,8 @@ void copy_and_decrement2(const std::vector<int> &v) {
   auto i2 = i1;
   --i2;
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); //expected-warning{{$v.end()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); //expected-warning{{$v.end() - 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$v.end(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$v.end() - 1{{$}}}}
 }
 
 /// std::advance(), std::prev(), std::next()
@@ -248,7 +254,7 @@ void std_advance_minus(const std::vector<int> &v) {
 
   std::advance(i, -1);
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i)); //expected-warning{{$v.end() - 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning-re {{$v.end() - 1{{$}}}}
 }
 
 void std_advance_plus(const std::vector<int> &v) {
@@ -258,7 +264,7 @@ void std_advance_plus(const std::vector<int> &v) {
 
   std::advance(i, 1);
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i)); //expected-warning{{$v.begin() + 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning-re {{$v.begin() + 1{{$}}}}
 }
 
 void std_prev(const std::vector<int> &v) {
@@ -268,7 +274,7 @@ void std_prev(const std::vector<int> &v) {
 
   auto j = std::prev(i);
 
-  clang_analyzer_express(clang_analyzer_iterator_position(j)); //expected-warning{{$v.end() - 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(j)); // expected-warning-re {{$v.end() - 1{{$}}}}
 }
 
 void std_prev2(const std::vector<int> &v) {
@@ -278,7 +284,7 @@ void std_prev2(const std::vector<int> &v) {
 
   auto j = std::prev(i, 2);
 
-  clang_analyzer_express(clang_analyzer_iterator_position(j)); //expected-warning{{$v.end() - 2}}
+  clang_analyzer_express(clang_analyzer_iterator_position(j)); // expected-warning-re {{$v.end() - 2{{$}}}}
 }
 
 void std_next(const std::vector<int> &v) {
@@ -288,7 +294,7 @@ void std_next(const std::vector<int> &v) {
 
   auto j = std::next(i);
 
-  clang_analyzer_express(clang_analyzer_iterator_position(j)); //expected-warning{{$v.begin() + 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(j)); // expected-warning-re {{$v.begin() + 1{{$}}}}
 }
 
 void std_next2(const std::vector<int> &v) {
@@ -298,7 +304,7 @@ void std_next2(const std::vector<int> &v) {
 
   auto j = std::next(i, 2);
 
-  clang_analyzer_express(clang_analyzer_iterator_position(j)); //expected-warning{{$v.begin() + 2}}
+  clang_analyzer_express(clang_analyzer_iterator_position(j)); // expected-warning-re {{$v.begin() + 2{{$}}}}
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -356,7 +362,7 @@ void list_move_assignment(std::list<int> &L1, std::list<int> &L2) {
   clang_analyzer_eval(clang_analyzer_iterator_container(i1) == &L1); // expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_container(i2) == &L1); // expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$L2.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$L2.begin(){{$}}}}
 }
 
 void vector_move_assignment(std::vector<int> &V1, std::vector<int> &V2) {
@@ -374,7 +380,7 @@ void vector_move_assignment(std::vector<int> &V1, std::vector<int> &V2) {
   clang_analyzer_eval(clang_analyzer_iterator_container(i1) == &V1); // expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_container(i2) == &V1); // expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$V2.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$V2.begin(){{$}}}}
 }
 
 void deque_move_assignment(std::deque<int> &D1, std::deque<int> &D2) {
@@ -392,7 +398,7 @@ void deque_move_assignment(std::deque<int> &D1, std::deque<int> &D2) {
   clang_analyzer_eval(clang_analyzer_iterator_container(i1) == &D1); // expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_container(i2) == &D1); // expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$D2.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$D2.begin(){{$}}}}
 }
 
 void forward_list_move_assignment(std::forward_list<int> &FL1,
@@ -409,7 +415,7 @@ void forward_list_move_assignment(std::forward_list<int> &FL1,
 
   clang_analyzer_eval(clang_analyzer_iterator_container(i1) == &FL1); // expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$FL2.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$FL2.begin(){{$}}}}
 }
 
 
@@ -506,9 +512,9 @@ void list_push_back(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$L.end() - 1}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$L.end()}} FIXME: Should be $L.end() + 1
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$L.end() - 1{{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$L.end(){{$}}}} FIXME: Should be $L.end() + 1
 }
 
 /// std::vector-like containers: The past-the-end iterator is invalidated.
@@ -525,8 +531,8 @@ void vector_push_back(std::vector<int> &V, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{FALSE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$V.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$V.end() - 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$V.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$V.end() - 1{{$}}}}
 }
 
 /// std::deque-like containers: All iterators, including the past-the-end
@@ -566,9 +572,9 @@ void list_emplace_back(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$L.end()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$L.end()}}  FIXME: Should be $L.end() + 1
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$L.end() - 1{{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$L.end(){{$}}}}  FIXME: Should be $L.end() + 1
 }
 
 /// std::vector-like containers: The past-the-end iterator is invalidated.
@@ -585,8 +591,8 @@ void vector_emplace_back(std::vector<int> &V, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{FALSE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$V.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$V.end() - 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$V.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$V.end() - 1{{$}}}}
 }
 
 /// std::deque-like containers: All iterators, including the past-the-end
@@ -626,8 +632,8 @@ void list_pop_back(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{FALSE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$L.end()}}  FIXME: Should be $L.end() - 1
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$L.end(){{$}}}}  FIXME: Should be $L.end() - 1
 }
 
 /// std::vector-like containers: Iterators to the last element, as well as the
@@ -645,7 +651,7 @@ void vector_pop_back(std::vector<int> &V, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{FALSE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{FALSE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$V.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$V.begin(){{$}}}}
 }
 
 /// std::deque-like containers: Iterators to the last element are invalidated.
@@ -664,7 +670,7 @@ void deque_pop_back(std::deque<int> &D, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{FALSE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{FALSE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$D.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$D.begin(){{$}}}}
 }
 
 /// push_front()
@@ -687,8 +693,8 @@ void list_push_front(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i0)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$L.end(){{$}}}}
 }
 
 /// std::deque-like containers: All iterators, including the past-the-end
@@ -720,8 +726,8 @@ void forward_list_push_front(std::forward_list<int> &FL, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i0)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$FL.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$FL.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$FL.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$FL.end(){{$}}}}
 }
 
 /// emplace_front()
@@ -744,8 +750,8 @@ void list_emplace_front(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i0)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$L.end(){{$}}}}
 }
 
 /// std::deque-like containers: All iterators, including the past-the-end
@@ -777,8 +783,8 @@ void forward_list_emplace_front(std::forward_list<int> &FL, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i0)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$FL.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$FL.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$FL.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$FL.end(){{$}}}}
 }
 
 /// pop_front()
@@ -802,8 +808,8 @@ void list_pop_front(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$L.begin() + 1}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$L.begin() + 1{{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$L.end(){{$}}}}
 }
 
 /// std::deque-like containers: Iterators to the first element are invalidated.
@@ -821,8 +827,8 @@ void deque_pop_front(std::deque<int> &D, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$D.begin() + 1}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$D.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$D.begin() + 1{{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$D.end(){{$}}}}
 }
 
 /// std::forward_list-like containers: Iterators to the first element are
@@ -840,8 +846,8 @@ void forward_list_pop_front(std::list<int> &FL, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$FL.begin() + 1}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$FL.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$FL.begin() + 1{{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$FL.end(){{$}}}}
 }
 
 /// insert()
@@ -867,9 +873,9 @@ void list_insert_begin(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i0)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i2)); FIXME: expect warning $L.begin() - 1
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$L.end(){{$}}}}
 }
 
 void list_insert_behind_begin(std::list<int> &L, int n) {
@@ -884,10 +890,10 @@ void list_insert_behind_begin(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}} FIXME: Should be $L.begin() - 1
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$L.begin() + 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}} FIXME: Should be $L.begin() - 1
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$L.begin() + 1{{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $L.begin()
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$L.end(){{$}}}}
 }
 
 template <typename Iter> Iter return_any_iterator(const Iter &It);
@@ -905,10 +911,10 @@ void list_insert_unknown(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$i1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$i1{{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $i - 1
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$L.end(){{$}}}}
 }
 
 void list_insert_ahead_of_end(std::list<int> &L, int n) {
@@ -923,9 +929,9 @@ void list_insert_ahead_of_end(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$L.end() - 1}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$L.end() - 1{{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$L.end(){{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $L.end() - 2
 }
 
@@ -941,9 +947,9 @@ void list_insert_end(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$L.end() - 1}} FIXME: should be $L.end() - 2
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$L.end() - 1{{$}}}} FIXME: should be $L.end() - 2
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$L.end(){{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $L.end() - 1
 }
 
@@ -977,7 +983,7 @@ void vector_insert_behind_begin(std::vector<int> &V, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{FALSE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{FALSE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$V.begin()}} FIXME: Should be $V.begin() - 1
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$V.begin(){{$}}}} FIXME: Should be $V.begin() - 1
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); // FIXME: expect -warning $V.begin()
 }
 
@@ -994,7 +1000,7 @@ void vector_insert_unknown(std::vector<int> &V, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{FALSE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{FALSE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$V.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$V.begin(){{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expecte warning $i1 - 1
 }
 
@@ -1010,7 +1016,7 @@ void vector_insert_ahead_of_end(std::vector<int> &V, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{FALSE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{FALSE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$V.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$V.begin(){{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $V.end() - 2
 }
 
@@ -1026,8 +1032,8 @@ void vector_insert_end(std::vector<int> &V, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{FALSE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$V.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$V.end() - 1}} FIXME: Should be $V.end() - 2
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$V.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$V.end() - 1{{$}}}} FIXME: Should be $V.end() - 2
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $V.end() - 1
 }
 
@@ -1130,9 +1136,9 @@ void forward_list_insert_after_begin(std::forward_list<int> &FL, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i0)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$FL.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$FL.begin(){{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i2)); FIXME: expect warning $FL.begin() + 1
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$FL.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$FL.end(){{$}}}}
 }
 
 void forward_list_insert_after_behind_begin(std::forward_list<int> &FL, int n) {
@@ -1147,10 +1153,10 @@ void forward_list_insert_after_behind_begin(std::forward_list<int> &FL, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$FL.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$FL.begin() + 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$FL.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$FL.begin() + 1{{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $FL.begin() + 2
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$FL.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$FL.end(){{$}}}}
 }
 
 void forward_list_insert_after_unknown(std::forward_list<int> &FL, int n) {
@@ -1166,10 +1172,10 @@ void forward_list_insert_after_unknown(std::forward_list<int> &FL, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$FL.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$i1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$FL.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$i1{{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $i1 + 1
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$FL.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$FL.end(){{$}}}}
 }
 
 /// emplace()
@@ -1195,9 +1201,9 @@ void list_emplace_begin(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i0)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i2)); FIXME: expect warning $L.begin() - 1
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$L.end(){{$}}}}
 }
 
 void list_emplace_behind_begin(std::list<int> &L, int n) {
@@ -1212,10 +1218,10 @@ void list_emplace_behind_begin(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}} FIXME: Should be $L.begin() - 1
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$L.begin() + 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}} FIXME: Should be $L.begin() - 1
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$L.begin() + 1{{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $L.begin()
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$L.end(){{$}}}}
 }
 
 template <typename Iter> Iter return_any_iterator(const Iter &It);
@@ -1233,10 +1239,10 @@ void list_emplace_unknown(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$i1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$i1{{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $i - 1
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$L.end(){{$}}}}
 }
 
 void list_emplace_ahead_of_end(std::list<int> &L, int n) {
@@ -1251,9 +1257,9 @@ void list_emplace_ahead_of_end(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$L.end() - 1}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$L.end() - 1{{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$L.end(){{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $L.end() - 2
 }
 
@@ -1269,9 +1275,9 @@ void list_emplace_end(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$L.end() - 1}} FIXME: should be $L.end() - 2
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$L.end() - 1{{$}}}} FIXME: should be $L.end() - 2
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$L.end(){{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $L.end() - 1
 }
 
@@ -1304,7 +1310,7 @@ void vector_emplace_behind_begin(std::vector<int> &V, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{FALSE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{FALSE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$V.begin()}} FIXME: Should be $V.begin() - 1
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$V.begin(){{$}}}} FIXME: Should be $V.begin() - 1
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); // FIXME: expect -warning $V.begin()
 }
 
@@ -1321,7 +1327,7 @@ void vector_emplace_unknown(std::vector<int> &V, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{FALSE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{FALSE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$V.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$V.begin(){{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expecte warning $i1 - 1
 }
 
@@ -1337,7 +1343,7 @@ void vector_emplace_ahead_of_end(std::vector<int> &V, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{FALSE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{FALSE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$V.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$V.begin(){{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $V.end() - 2
 }
 
@@ -1353,8 +1359,8 @@ void vector_emplace_end(std::vector<int> &V, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{FALSE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$V.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$V.end() - 1}} FIXME: Should be $V.end() - 2
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$V.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$V.end() - 1{{$}}}} FIXME: Should be $V.end() - 2
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $V.end() - 1
 }
 
@@ -1455,9 +1461,9 @@ void forward_list_emplace_after_begin(std::forward_list<int> &FL, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i0)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$FL.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$FL.begin(){{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i2)); FIXME: expect warning $FL.begin() + 1
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$FL.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$FL.end(){{$}}}}
 }
 
 void forward_list_emplace_after_behind_begin(std::forward_list<int> &FL,
@@ -1473,10 +1479,10 @@ void forward_list_emplace_after_behind_begin(std::forward_list<int> &FL,
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$FL.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$FL.begin() + 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$FL.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$FL.begin() + 1{{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $FL.begin() + 2
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$FL.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$FL.end(){{$}}}}
 }
 
 void forward_list_emplace_after_unknown(std::forward_list<int> &FL, int n) {
@@ -1492,10 +1498,10 @@ void forward_list_emplace_after_unknown(std::forward_list<int> &FL, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$FL.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$i1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$FL.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$i1{{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $i1 + 1
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$FL.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$FL.end(){{$}}}}
 }
 
 /// erase()
@@ -1523,9 +1529,9 @@ void list_erase_begin(std::list<int> &L) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$L.begin() + 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$L.begin() + 1{{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $L.begin() + 1
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$L.end(){{$}}}}
 }
 
 void list_erase_behind_begin(std::list<int> &L, int n) {
@@ -1540,9 +1546,9 @@ void list_erase_behind_begin(std::list<int> &L, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{FALSE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}} FIXME: Should be $L.begin() + 1
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}} FIXME: Should be $L.begin() + 1
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $L.begin() + 2
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$L.end(){{$}}}}
 }
 
 void list_erase_unknown(std::list<int> &L) {
@@ -1558,9 +1564,9 @@ void list_erase_unknown(std::list<int> &L) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{FALSE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $i1 + 1
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$L.end(){{$}}}}
 }
 
 void list_erase_ahead_of_end(std::list<int> &L) {
@@ -1575,8 +1581,8 @@ void list_erase_ahead_of_end(std::list<int> &L) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{FALSE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$L.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$L.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$L.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$L.end(){{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $L.end()
 }
 
@@ -1610,7 +1616,7 @@ void vector_erase_behind_begin(std::vector<int> &V, int n) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{FALSE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{FALSE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$V.begin()}} FIXME: Should be $V.begin() + 1
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$V.begin(){{$}}}} FIXME: Should be $V.begin() + 1
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $V.begin() + 2
 }
 
@@ -1627,7 +1633,7 @@ void vector_erase_unknown(std::vector<int> &V) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{FALSE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{FALSE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$V.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$V.begin(){{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $i1 + 1
 }
 
@@ -1643,7 +1649,7 @@ void vector_erase_ahead_of_end(std::vector<int> &V) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i1)); //expected-warning{{FALSE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{FALSE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$V.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$V.begin(){{$}}}}
   // clang_analyzer_express(clang_analyzer_iterator_position(i3)); FIXME: expect warning $V.end()
 }
 
@@ -1743,10 +1749,10 @@ void forward_list_erase_after_begin(std::forward_list<int> &FL) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i2)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i3)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$FL.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$FL.begin() + 2}} FIXME: Should be $FL.begin() + 1
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$FL.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning-re {{$FL.begin() + 2{{$}}}} FIXME: Should be $FL.begin() + 1
   // clang_analyzer_express(clang_analyzer_iterator_position(i4)); FIXME: expect warning $FL.begin() + 1
-  clang_analyzer_express(clang_analyzer_iterator_position(i3)); // expected-warning{{$FL.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i3)); // expected-warning-re {{$FL.end(){{$}}}}
 }
 
 void forward_list_erase_after_unknown(std::forward_list<int> &FL) {
@@ -1768,11 +1774,11 @@ void forward_list_erase_after_unknown(std::forward_list<int> &FL) {
   clang_analyzer_eval(clang_analyzer_iterator_validity(i3)); //expected-warning{{TRUE}}
   clang_analyzer_eval(clang_analyzer_iterator_validity(i4)); //expected-warning{{TRUE}}
 
-  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning{{$FL.begin()}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$i1}}
-  clang_analyzer_express(clang_analyzer_iterator_position(i3)); // expected-warning{{$i1 + 2}} FIXME: Should be $i1 + 1
+  clang_analyzer_express(clang_analyzer_iterator_position(i0)); // expected-warning-re {{$FL.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning-re {{$i1{{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i3)); // expected-warning-re {{$i1 + 2{{$}}}} FIXME: Should be $i1 + 1
   // clang_analyzer_express(clang_analyzer_iterator_position(i5)); FIXME: expect warning $i1 + 1
-  clang_analyzer_express(clang_analyzer_iterator_position(i4)); // expected-warning{{$FL.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i4)); // expected-warning-re {{$FL.end(){{$}}}}
 }
 
 struct simple_iterator_base {
@@ -1806,7 +1812,7 @@ struct simple_container {
 
 void good_derived(simple_container c) {
   auto i0 = c.end();
-  
+
   if (i0 != c.end()) {
     clang_analyzer_warnIfReached();
   }
@@ -1859,24 +1865,134 @@ void non_std_find(std::vector<int> &V, int e) {
   }
 }
 
+template<typename T>
+struct cont_with_ptr_iterator {
+  typedef T* iterator;
+  T* begin() const;
+  T* end() const;
+};
+
+void begin_ptr_iterator(const cont_with_ptr_iterator<int> &c) {
+  auto i = c.begin();
+
+  clang_analyzer_eval(clang_analyzer_iterator_container(i) == &c); // expected-warning{{TRUE}}
+  clang_analyzer_denote(clang_analyzer_container_begin(c), "$c.begin()");
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning{{$c.begin()}}
+
+  if (i != c.begin()) {
+    clang_analyzer_warnIfReached();
+  }
+  }
+
+void prefix_increment_ptr_iterator(const cont_with_ptr_iterator<int> &c) {
+  auto i = c.begin();
+
+  clang_analyzer_denote(clang_analyzer_container_begin(c), "$c.begin()");
+
+  auto j = ++i;
+
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning{{$c.begin() + 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(j)); // expected-warning{{$c.begin() + 1}}
+}
+
+void prefix_decrement_ptr_iterator(const cont_with_ptr_iterator<int> &c) {
+  auto i = c.end();
+
+  clang_analyzer_denote(clang_analyzer_container_end(c), "$c.end()");
+
+  auto j = --i;
+
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning{{$c.end() - 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(j)); // expected-warning{{$c.end() - 1}}
+}
+
+void postfix_increment_ptr_iterator(const cont_with_ptr_iterator<int> &c) {
+  auto i = c.begin();
+
+  clang_analyzer_denote(clang_analyzer_container_begin(c), "$c.begin()");
+
+  auto j = i++;
+
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning{{$c.begin() + 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(j)); // expected-warning{{$c.begin()}}
+}
+
+void postfix_decrement_ptr_iterator(const cont_with_ptr_iterator<int> &c) {
+  auto i = c.end();
+
+  clang_analyzer_denote(clang_analyzer_container_end(c), "$c.end()");
+
+  auto j = i--;
+
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning{{$c.end() - 1}}
+  clang_analyzer_express(clang_analyzer_iterator_position(j)); // expected-warning{{$c.end()}}
+}
+
+void plus_equal_ptr_iterator(const cont_with_ptr_iterator<int> &c) {
+  auto i = c.begin();
+
+  clang_analyzer_denote(clang_analyzer_container_begin(c), "$c.begin()");
+
+  i += 2;
+
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning{{$c.begin() + 2}}
+}
+
+void minus_equal_ptr_iterator(const cont_with_ptr_iterator<int> &c) {
+  auto i = c.end();
+
+  clang_analyzer_denote(clang_analyzer_container_end(c), "$c.end()");
+
+  i -= 2;
+
+  clang_analyzer_express(clang_analyzer_iterator_position(i)); // expected-warning{{$c.end() - 2}}
+}
+
+void plus_ptr_iterator(const cont_with_ptr_iterator<int> &c) {
+  auto i1 = c.begin();
+
+  clang_analyzer_denote(clang_analyzer_container_begin(c), "$c.begin()");
+
+  auto i2 = i1 + 2;
+
+  clang_analyzer_eval(clang_analyzer_iterator_container(i2) == &c); // expected-warning{{TRUE}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$c.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$c.begin() + 2}}
+}
+
+void minus_ptr_iterator(const cont_with_ptr_iterator<int> &c) {
+  auto i1 = c.end();
+
+  clang_analyzer_denote(clang_analyzer_container_end(c), "$c.end()");
+
+  auto i2 = i1 - 2;
+
+  clang_analyzer_eval(clang_analyzer_iterator_container(i2) == &c); // expected-warning{{TRUE}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1)); // expected-warning{{$c.end()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$c.end() - 2}}
+}
+
 void clang_analyzer_printState();
 
 void print_state(std::vector<int> &V) {
   const auto i0 = V.cbegin();
   clang_analyzer_printState();
 
-// CHECK:      "checker_messages": [
-// CHECK:   { "checker": "alpha.cplusplus.IteratorModeling", "messages": [
-// CHECK-NEXT:     "Iterator Positions :",
-// CHECK-NEXT:     "i0 : Valid ; Container == SymRegion{reg_$[[#]]<std::vector<int> & V>} ; Offset == conj_$[[#]]{long, LC[[#]], S[[#]], #[[#]]}"
-// CHECK-NEXT:   ]}
+  // CHECK:      "checker_messages": [
+  // CHECK:   { "checker": "alpha.cplusplus.IteratorModeling", "messages": [
+  // CHECK-NEXT:     "Iterator Positions :",
+  // CHECK-NEXT:     "i0 : Valid ; Container == SymRegion{reg_$[[#]]<std::vector<int> & V>} ; Offset == conj_$[[#]]{long, LC[[#]], S[[#]], #[[#]]}"
+  // CHECK-NEXT:   ]}
 
+  *i0;
   const auto i1 = V.cend();
   clang_analyzer_printState();
-  
-// CHECK:      "checker_messages": [
-// CHECK:   { "checker": "alpha.cplusplus.IteratorModeling", "messages": [
-// CHECK-NEXT:     "Iterator Positions :",
-// CHECK-NEXT:     "i1 : Valid ; Container == SymRegion{reg_$[[#]]<std::vector<int> & V>} ; Offset == conj_$[[#]]{long, LC[[#]], S[[#]], #[[#]]}"
-// CHECK-NEXT:   ]}
+
+  // CHECK:      "checker_messages": [
+  // CHECK:   { "checker": "alpha.cplusplus.IteratorModeling", "messages": [
+  // CHECK-NEXT:     "Iterator Positions :",
+  // CHECK-NEXT:     "i1 : Valid ; Container == SymRegion{reg_$[[#]]<std::vector<int> & V>} ; Offset == conj_$[[#]]{long, LC[[#]], S[[#]], #[[#]]}"
+  // CHECK-NEXT:   ]}
+
+  *i1;
 }
diff --git a/clang/test/Analysis/iterator-range.cpp b/clang/test/Analysis/iterator-range.cpp
index ad8ce92ecfb88..657ae89998e81 100644
--- a/clang/test/Analysis/iterator-range.cpp
+++ b/clang/test/Analysis/iterator-range.cpp
@@ -854,3 +854,84 @@ void deref_end_after_pop_back(std::vector<int> &V) {
   *i; // expected-warning{{Past-the-end iterator dereferenced}}
       // expected-note@-1{{Past-the-end iterator dereferenced}}
 }
+
+template<typename T>
+struct cont_with_ptr_iterator {
+  T* begin() const;
+  T* end() const;
+};
+
+void deref_end_ptr_iterator(const cont_with_ptr_iterator<S> &c) {
+  auto i = c.end();
+  (void) *i; // expected-warning{{Past-the-end iterator dereferenced}}
+             // expected-note@-1{{Past-the-end iterator dereferenced}}
+}
+
+void array_deref_end_ptr_iterator(const cont_with_ptr_iterator<S> &c) {
+  auto i = c.end();
+  (void) i[0]; // expected-warning{{Past-the-end iterator dereferenced}}
+               // expected-note@-1{{Past-the-end iterator dereferenced}}
+}
+
+void arrow_deref_end_ptr_iterator(const cont_with_ptr_iterator<S> &c) {
+  auto i = c.end();
+  (void) i->n; // expected-warning{{Past-the-end iterator dereferenced}}
+               // expected-note@-1{{Past-the-end iterator dereferenced}}
+}
+
+void arrow_star_deref_end_ptr_iterator(const cont_with_ptr_iterator<S> &c,
+                                       int S::*p) {
+  auto i = c.end();
+  (void)(i->*p); // expected-warning{{Past-the-end iterator dereferenced}}
+                 // expected-note@-1{{Past-the-end iterator dereferenced}}
+}
+
+void prefix_incr_end_ptr_iterator(const cont_with_ptr_iterator<S> &c) {
+  auto i = c.end();
+  ++i; // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+       // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
+}
+
+void postfix_incr_end_ptr_iterator(const cont_with_ptr_iterator<S> &c) {
+  auto i = c.end();
+  i++; // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+       // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
+}
+
+void prefix_decr_begin_ptr_iterator(const cont_with_ptr_iterator<S> &c) {
+  auto i = c.begin();
+  --i; // expected-warning{{Iterator decremented ahead of its valid range}}
+       // expected-note@-1{{Iterator decremented ahead of its valid range}}
+}
+
+void postfix_decr_begin_ptr_iterator(const cont_with_ptr_iterator<S> &c) {
+  auto i = c.begin();
+  i--; // expected-warning{{Iterator decremented ahead of its valid range}}
+       // expected-note@-1{{Iterator decremented ahead of its valid range}}
+}
+
+void prefix_add_2_end_ptr_iterator(const cont_with_ptr_iterator<S> &c) {
+  auto i = c.end();
+  (void)(i + 2); // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+                 // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
+}
+
+void postfix_add_assign_2_end_ptr_iterator(const cont_with_ptr_iterator<S> &c) {
+  auto i = c.end();
+  i += 2; // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+          // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
+}
+
+void prefix_minus_2_begin_ptr_iterator(const cont_with_ptr_iterator<S> &c) {
+  auto i = c.begin();
+  (void)(i - 2); // expected-warning{{Iterator decremented ahead of its valid range}}
+                 // expected-note@-1{{Iterator decremented ahead of its valid range}}
+}
+
+void postfix_minus_assign_2_begin_ptr_iterator(
+    const cont_with_ptr_iterator<S> &c) {
+  auto i = c.begin();
+  i -= 2; // expected-warning{{Iterator decremented ahead of its valid range}}
+          // expected-note@-1{{Iterator decremented ahead of its valid range}}
+}
+
diff --git a/clang/test/Analysis/mismatched-iterator.cpp b/clang/test/Analysis/mismatched-iterator.cpp
index c58d14d72fda5..570e742751ead 100644
--- a/clang/test/Analysis/mismatched-iterator.cpp
+++ b/clang/test/Analysis/mismatched-iterator.cpp
@@ -118,3 +118,15 @@ void ignore_conjured2() {
 
   if (V1.cbegin() == V2.cbegin()) {} //no-warning
 }
+
+template<typename T>
+struct cont_with_ptr_iterator {
+  T *begin() const;
+  T *end() const;
+};
+
+void comparison_ptr_iterator(cont_with_ptr_iterator<int> &C1,
+                             cont_with_ptr_iterator<int> &C2) {
+  if (C1.begin() != C2.end()) {} // expected-warning{{Iterators of different containers used where the same container is expected}}
+}
+
diff --git a/clang/test/Analysis/more-dtors-cfg-output.cpp b/clang/test/Analysis/more-dtors-cfg-output.cpp
index c0df5953aa6e1..668210a0f61e0 100644
--- a/clang/test/Analysis/more-dtors-cfg-output.cpp
+++ b/clang/test/Analysis/more-dtors-cfg-output.cpp
@@ -278,16 +278,16 @@ void new_default_ctor_with_default_arg(long count) {
 namespace std::experimental {
   template <typename Promise>
   struct coroutine_handle {
-    static coroutine_handle from_address(void *);
+    static coroutine_handle from_address(void *) noexcept;
   };
 }
 
 struct TestPromise {
   TestPromise initial_suspend();
-  TestPromise final_suspend();
-  bool await_ready();
-  void await_suspend(const std::experimental::coroutine_handle<TestPromise> &);
-  void await_resume();
+  TestPromise final_suspend() noexcept;
+  bool await_ready() noexcept;
+  void await_suspend(const std::experimental::coroutine_handle<TestPromise> &) noexcept;
+  void await_resume() noexcept;
   Foo return_value(const Bar &);
   Bar get_return_object();
   void unhandled_exception();
diff --git a/clang/test/Analysis/new-ctor-conservative.cpp b/clang/test/Analysis/new-ctor-conservative.cpp
index 6cd403b50e5a4..83c4bed1530c5 100644
--- a/clang/test/Analysis/new-ctor-conservative.cpp
+++ b/clang/test/Analysis/new-ctor-conservative.cpp
@@ -27,6 +27,7 @@ void checkNewArray() {
   S *s = new S[10];
   // FIXME: Should be true once we inline array constructors.
   clang_analyzer_eval(s[0].x == 1); // expected-warning{{UNKNOWN}}
+  clang_analyzer_eval(s[1].x == 1); // expected-warning{{UNKNOWN}}
 }
 
 struct NullS {
diff --git a/clang/test/Analysis/smart-ptr.cpp b/clang/test/Analysis/smart-ptr.cpp
index ae6966b4e6f4a..4ab7e2bbd3bfc 100644
--- a/clang/test/Analysis/smart-ptr.cpp
+++ b/clang/test/Analysis/smart-ptr.cpp
@@ -1,16 +1,18 @@
 // RUN: %clang_analyze_cc1 -analyzer-checker=core,debug.ExprInspection\
-// RUN:   -analyzer-checker cplusplus.Move,cplusplus.SmartPtr\
+// RUN:   -analyzer-checker cplusplus.Move,alpha.cplusplus.SmartPtr\
+// RUN:   -analyzer-config cplusplus.SmartPtrModeling:ModelSmartPtrDereference=true\
 // RUN:   -std=c++11 -verify %s
 
 #include "Inputs/system-header-simulator-cxx.h"
 
 void clang_analyzer_warnIfReached();
+void clang_analyzer_numTimesReached();
 
 void derefAfterMove(std::unique_ptr<int> P) {
   std::unique_ptr<int> Q = std::move(P);
   if (Q)
     clang_analyzer_warnIfReached(); // expected-warning{{REACHABLE}}
-  *Q.get() = 1; // no-warning
+  *Q.get() = 1;                     // no-warning
   if (P)
     clang_analyzer_warnIfReached(); // no-warning
   // TODO: Report a null dereference (instead).
@@ -26,3 +28,76 @@ void bar(S *s, void (S::*func)(void)) {
   (s->*func)(); // no-crash
 }
 } // namespace testUnknownCallee
+
+class A {
+public:
+  A(){};
+  void foo();
+};
+
+A *return_null() {
+  return nullptr;
+}
+
+void derefAfterValidCtr() {
+  std::unique_ptr<A> P(new A());
+  P->foo(); // No warning.
+}
+
+void derefOfUnknown(std::unique_ptr<A> P) {
+  P->foo(); // No warning.
+}
+
+void derefAfterDefaultCtr() {
+  std::unique_ptr<A> P;
+  P->foo(); // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+}
+
+void derefAfterCtrWithNull() {
+  std::unique_ptr<A> P(nullptr);
+  *P; // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+}
+
+void derefAfterCtrWithNullReturnMethod() {
+  std::unique_ptr<A> P(return_null());
+  P->foo(); // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+}
+
+void derefAfterRelease() {
+  std::unique_ptr<A> P(new A());
+  P.release();
+  clang_analyzer_numTimesReached(); // expected-warning {{1}}
+  P->foo(); // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+}
+
+void derefAfterReset() {
+  std::unique_ptr<A> P(new A());
+  P.reset();
+  clang_analyzer_numTimesReached(); // expected-warning {{1}}
+  P->foo(); // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+}
+
+void derefAfterResetWithNull() {
+  std::unique_ptr<A> P(new A());
+  P.reset(nullptr);
+  P->foo(); // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+}
+
+void derefAfterResetWithNonNull() {
+  std::unique_ptr<A> P;
+  P.reset(new A());
+  P->foo(); // No warning.
+}
+
+void derefAfterReleaseAndResetWithNonNull() {
+  std::unique_ptr<A> P(new A());
+  P.release();
+  P.reset(new A());
+  P->foo(); // No warning.
+}
+
+void derefOnReleasedNullRawPtr() {
+  std::unique_ptr<A> P;
+  A *AP = P.release();
+  AP->foo(); // expected-warning {{Called C++ object pointer is null [core.CallAndMessage]}}
+}
diff --git a/clang/test/Analysis/std-c-library-functions-POSIX.c b/clang/test/Analysis/std-c-library-functions-POSIX.c
new file mode 100644
index 0000000000000..e6f5b19baa00e
--- /dev/null
+++ b/clang/test/Analysis/std-c-library-functions-POSIX.c
@@ -0,0 +1,178 @@
+// RUN: %clang_analyze_cc1 %s \
+// RUN:   -analyzer-checker=core \
+// RUN:   -analyzer-checker=apiModeling.StdCLibraryFunctions \
+// RUN:   -analyzer-config apiModeling.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:   -analyzer-config apiModeling.StdCLibraryFunctions:DisplayLoadedSummaries=true \
+// RUN:   -analyzer-checker=debug.ExprInspection \
+// RUN:   -analyzer-config eagerly-assume=false \
+// RUN:   -triple i686-unknown-linux 2>&1 | FileCheck %s
+
+// CHECK: Loaded summary for: long a64l(const char *str64)
+// CHECK: Loaded summary for: char *l64a(long value)
+// CHECK: Loaded summary for: int access(const char *pathname, int amode)
+// CHECK: Loaded summary for: int faccessat(int dirfd, const char *pathname, int mode, int flags)
+// CHECK: Loaded summary for: int dup(int fildes)
+// CHECK: Loaded summary for: int dup2(int fildes1, int filedes2)
+// CHECK: Loaded summary for: int fdatasync(int fildes)
+// CHECK: Loaded summary for: int fnmatch(const char *pattern, const char *string, int flags)
+// CHECK: Loaded summary for: int fsync(int fildes)
+// CHECK: Loaded summary for: int truncate(const char *path, off_t length)
+// CHECK: Loaded summary for: int symlink(const char *oldpath, const char *newpath)
+// CHECK: Loaded summary for: int symlinkat(const char *oldpath, int newdirfd, const char *newpath)
+// CHECK: Loaded summary for: int lockf(int fd, int cmd, off_t len)
+// CHECK: Loaded summary for: int creat(const char *pathname, mode_t mode)
+// CHECK: Loaded summary for: unsigned int sleep(unsigned int seconds)
+// CHECK: Loaded summary for: int dirfd(DIR *dirp)
+// CHECK: Loaded summary for: unsigned int alarm(unsigned int seconds)
+// CHECK: Loaded summary for: int closedir(DIR *dir)
+// CHECK: Loaded summary for: char *strdup(const char *s)
+// CHECK: Loaded summary for: char *strndup(const char *s, size_t n)
+// CHECK: Loaded summary for: int mkstemp(char *template)
+// CHECK: Loaded summary for: char *mkdtemp(char *template)
+// CHECK: Loaded summary for: char *getcwd(char *buf, size_t size)
+// CHECK: Loaded summary for: int mkdir(const char *pathname, mode_t mode)
+// CHECK: Loaded summary for: int mkdirat(int dirfd, const char *pathname, mode_t mode)
+// CHECK: Loaded summary for: int mknod(const char *pathname, mode_t mode, dev_t dev)
+// CHECK: Loaded summary for: int mknodat(int dirfd, const char *pathname, mode_t mode, dev_t dev)
+// CHECK: Loaded summary for: int chmod(const char *path, mode_t mode)
+// CHECK: Loaded summary for: int fchmodat(int dirfd, const char *pathname, mode_t mode, int flags)
+// CHECK: Loaded summary for: int fchmod(int fildes, mode_t mode)
+// CHECK: Loaded summary for: int fchownat(int dirfd, const char *pathname, uid_t owner, gid_t group, int flags)
+// CHECK: Loaded summary for: int chown(const char *path, uid_t owner, gid_t group)
+// CHECK: Loaded summary for: int lchown(const char *path, uid_t owner, gid_t group)
+// CHECK: Loaded summary for: int fchown(int fildes, uid_t owner, gid_t group)
+// CHECK: Loaded summary for: int rmdir(const char *pathname)
+// CHECK: Loaded summary for: int chdir(const char *path)
+// CHECK: Loaded summary for: int link(const char *oldpath, const char *newpath)
+// CHECK: Loaded summary for: int linkat(int fd1, const char *path1, int fd2, const char *path2, int flag)
+// CHECK: Loaded summary for: int unlink(const char *pathname)
+// CHECK: Loaded summary for: int unlinkat(int fd, const char *path, int flag)
+// CHECK: Loaded summary for: int fstat(int fd, struct stat *statbuf)
+// CHECK: Loaded summary for: int stat(const char *restrict path, struct stat *restrict buf)
+// CHECK: Loaded summary for: int lstat(const char *restrict path, struct stat *restrict buf)
+// CHECK: Loaded summary for: int fstatat(int fd, const char *restrict path, struct stat *restrict buf, int flag)
+// CHECK: Loaded summary for: DIR *opendir(const char *name)
+// CHECK: Loaded summary for: DIR *fdopendir(int fd)
+// CHECK: Loaded summary for: int isatty(int fildes)
+// CHECK: Loaded summary for: FILE *popen(const char *command, const char *type)
+// CHECK: Loaded summary for: int pclose(FILE *stream)
+// CHECK: Loaded summary for: int close(int fildes)
+// CHECK: Loaded summary for: long fpathconf(int fildes, int name)
+// CHECK: Loaded summary for: long pathconf(const char *path, int name)
+// CHECK: Loaded summary for: FILE *fdopen(int fd, const char *mode)
+// CHECK: Loaded summary for: void rewinddir(DIR *dir)
+// CHECK: Loaded summary for: void seekdir(DIR *dirp, long loc)
+// CHECK: Loaded summary for: int rand_r(unsigned int *seedp)
+// CHECK: Loaded summary for: int strcasecmp(const char *s1, const char *s2)
+// CHECK: Loaded summary for: int strncasecmp(const char *s1, const char *s2, size_t n)
+// CHECK: Loaded summary for: int fileno(FILE *stream)
+// CHECK: Loaded summary for: int fseeko(FILE *stream, off_t offset, int whence)
+// CHECK: Loaded summary for: off_t ftello(FILE *stream)
+// CHECK: Loaded summary for: void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset)
+// CHECK: Loaded summary for: void *mmap64(void *addr, size_t length, int prot, int flags, int fd, off64_t offset)
+// CHECK: Loaded summary for: int pipe(int fildes[2])
+// CHECK: Loaded summary for: off_t lseek(int fildes, off_t offset, int whence)
+// CHECK: Loaded summary for: ssize_t readlink(const char *restrict path, char *restrict buf, size_t bufsize)
+// CHECK: Loaded summary for: ssize_t readlinkat(int fd, const char *restrict path, char *restrict buf, size_t bufsize)
+// CHECK: Loaded summary for: int renameat(int olddirfd, const char *oldpath, int newdirfd, const char *newpath)
+// CHECK: Loaded summary for: char *realpath(const char *restrict file_name, char *restrict resolved_name)
+// CHECK: Loaded summary for: int execv(const char *path, char *const argv[])
+// CHECK: Loaded summary for: int execvp(const char *file, char *const argv[])
+// CHECK: Loaded summary for: int getopt(int argc, char *const argv[], const char *optstring)
+
+long a64l(const char *str64);
+char *l64a(long value);
+int access(const char *pathname, int amode);
+int faccessat(int dirfd, const char *pathname, int mode, int flags);
+int dup(int fildes);
+int dup2(int fildes1, int filedes2);
+int fdatasync(int fildes);
+int fnmatch(const char *pattern, const char *string, int flags);
+int fsync(int fildes);
+typedef unsigned long off_t;
+int truncate(const char *path, off_t length);
+int symlink(const char *oldpath, const char *newpath);
+int symlinkat(const char *oldpath, int newdirfd, const char *newpath);
+int lockf(int fd, int cmd, off_t len);
+typedef unsigned mode_t;
+int creat(const char *pathname, mode_t mode);
+unsigned int sleep(unsigned int seconds);
+typedef struct {
+  int a;
+} DIR;
+int dirfd(DIR *dirp);
+unsigned int alarm(unsigned int seconds);
+int closedir(DIR *dir);
+char *strdup(const char *s);
+typedef typeof(sizeof(int)) size_t;
+char *strndup(const char *s, size_t n);
+/*FIXME How to define wchar_t in the test?*/
+/*typedef __wchar_t wchar_t;*/
+/*wchar_t *wcsdup(const wchar_t *s);*/
+int mkstemp(char *template);
+char *mkdtemp(char *template);
+char *getcwd(char *buf, size_t size);
+int mkdir(const char *pathname, mode_t mode);
+int mkdirat(int dirfd, const char *pathname, mode_t mode);
+typedef int dev_t;
+int mknod(const char *pathname, mode_t mode, dev_t dev);
+int mknodat(int dirfd, const char *pathname, mode_t mode, dev_t dev);
+int chmod(const char *path, mode_t mode);
+int fchmodat(int dirfd, const char *pathname, mode_t mode, int flags);
+int fchmod(int fildes, mode_t mode);
+typedef int uid_t;
+typedef int gid_t;
+int fchownat(int dirfd, const char *pathname, uid_t owner, gid_t group, int flags);
+int chown(const char *path, uid_t owner, gid_t group);
+int lchown(const char *path, uid_t owner, gid_t group);
+int fchown(int fildes, uid_t owner, gid_t group);
+int rmdir(const char *pathname);
+int chdir(const char *path);
+int link(const char *oldpath, const char *newpath);
+int linkat(int fd1, const char *path1, int fd2, const char *path2, int flag);
+int unlink(const char *pathname);
+int unlinkat(int fd, const char *path, int flag);
+struct stat;
+int fstat(int fd, struct stat *statbuf);
+int stat(const char *restrict path, struct stat *restrict buf);
+int lstat(const char *restrict path, struct stat *restrict buf);
+int fstatat(int fd, const char *restrict path, struct stat *restrict buf, int flag);
+DIR *opendir(const char *name);
+DIR *fdopendir(int fd);
+int isatty(int fildes);
+typedef struct {
+  int x;
+} FILE;
+FILE *popen(const char *command, const char *type);
+int pclose(FILE *stream);
+int close(int fildes);
+long fpathconf(int fildes, int name);
+long pathconf(const char *path, int name);
+FILE *fdopen(int fd, const char *mode);
+void rewinddir(DIR *dir);
+void seekdir(DIR *dirp, long loc);
+int rand_r(unsigned int *seedp);
+int strcasecmp(const char *s1, const char *s2);
+int strncasecmp(const char *s1, const char *s2, size_t n);
+int fileno(FILE *stream);
+int fseeko(FILE *stream, off_t offset, int whence);
+off_t ftello(FILE *stream);
+void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset);
+typedef off_t off64_t;
+void *mmap64(void *addr, size_t length, int prot, int flags, int fd, off64_t offset);
+int pipe(int fildes[2]);
+off_t lseek(int fildes, off_t offset, int whence);
+typedef size_t ssize_t;
+ssize_t readlink(const char *restrict path, char *restrict buf, size_t bufsize);
+ssize_t readlinkat(int fd, const char *restrict path, char *restrict buf, size_t bufsize);
+int renameat(int olddirfd, const char *oldpath, int newdirfd, const char *newpath);
+char *realpath(const char *restrict file_name, char *restrict resolved_name);
+int execv(const char *path, char *const argv[]);
+int execvp(const char *file, char *const argv[]);
+int getopt(int argc, char *const argv[], const char *optstring);
+
+// Must have at least one call expression to initialize the summary map.
+int bar(void);
+void foo() {
+  bar();
+}
diff --git a/clang/test/Analysis/std-c-library-functions-arg-constraints.c b/clang/test/Analysis/std-c-library-functions-arg-constraints.c
index b99248d337b34..b23700a96f38f 100644
--- a/clang/test/Analysis/std-c-library-functions-arg-constraints.c
+++ b/clang/test/Analysis/std-c-library-functions-arg-constraints.c
@@ -2,7 +2,7 @@
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-checker=apiModeling.StdCLibraryFunctions \
-// RUN:   -analyzer-checker=alpha.apiModeling.StdCLibraryFunctionArgs \
+// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctionArgs \
 // RUN:   -analyzer-checker=debug.StdCLibraryFunctionsTester \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -triple x86_64-unknown-linux-gnu \
@@ -12,7 +12,7 @@
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-checker=apiModeling.StdCLibraryFunctions \
-// RUN:   -analyzer-checker=alpha.apiModeling.StdCLibraryFunctionArgs \
+// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctionArgs \
 // RUN:   -analyzer-checker=debug.StdCLibraryFunctionsTester \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -triple x86_64-unknown-linux-gnu \
diff --git a/clang/test/Analysis/std-c-library-functions-arg-constraints.cpp b/clang/test/Analysis/std-c-library-functions-arg-constraints.cpp
index ebc841a07fb73..ae26834402e69 100644
--- a/clang/test/Analysis/std-c-library-functions-arg-constraints.cpp
+++ b/clang/test/Analysis/std-c-library-functions-arg-constraints.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-checker=apiModeling.StdCLibraryFunctions \
-// RUN:   -analyzer-checker=alpha.apiModeling.StdCLibraryFunctionArgs \
+// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctionArgs \
 // RUN:   -analyzer-checker=debug.StdCLibraryFunctionsTester \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
diff --git a/clang/test/Analysis/stream-note.c b/clang/test/Analysis/stream-note.c
new file mode 100644
index 0000000000000..08927e8902be4
--- /dev/null
+++ b/clang/test/Analysis/stream-note.c
@@ -0,0 +1,48 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.unix.Stream -analyzer-output text -verify %s
+
+#include "Inputs/system-header-simulator.h"
+
+void check_note_at_correct_open() {
+  FILE *F1 = tmpfile(); // expected-note {{Stream opened here}}
+  if (!F1)
+    // expected-note@-1 {{'F1' is non-null}}
+    // expected-note@-2 {{Taking false branch}}
+    return;
+  FILE *F2 = tmpfile();
+  if (!F2) {
+    // expected-note@-1 {{'F2' is non-null}}
+    // expected-note@-2 {{Taking false branch}}
+    fclose(F1);
+    return;
+  }
+  rewind(F2);
+  fclose(F2);
+  rewind(F1);
+}
+// expected-warning@-1 {{Opened stream never closed. Potential resource leak}}
+// expected-note@-2 {{Opened stream never closed. Potential resource leak}}
+
+void check_note_fopen() {
+  FILE *F = fopen("file", "r"); // expected-note {{Stream opened here}}
+  if (!F)
+    // expected-note@-1 {{'F' is non-null}}
+    // expected-note@-2 {{Taking false branch}}
+    return;
+}
+// expected-warning@-1 {{Opened stream never closed. Potential resource leak}}
+// expected-note@-2 {{Opened stream never closed. Potential resource leak}}
+
+void check_note_freopen() {
+  FILE *F = fopen("file", "r"); // expected-note {{Stream opened here}}
+  if (!F)
+    // expected-note@-1 {{'F' is non-null}}
+    // expected-note@-2 {{Taking false branch}}
+    return;
+  F = freopen(0, "w", F); // expected-note {{Stream reopened here}}
+  if (!F)
+    // expected-note@-1 {{'F' is non-null}}
+    // expected-note@-2 {{Taking false branch}}
+    return;
+}
+// expected-warning@-1 {{Opened stream never closed. Potential resource leak}}
+// expected-note@-2 {{Opened stream never closed. Potential resource leak}}
diff --git a/clang/test/Analysis/stream.c b/clang/test/Analysis/stream.c
index dbbcc8715e0d2..cd5d28ae84554 100644
--- a/clang/test/Analysis/stream.c
+++ b/clang/test/Analysis/stream.c
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.unix.Stream -analyzer-store region -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.unix.Stream -verify %s
 
 #include "Inputs/system-header-simulator.h"
 
@@ -139,7 +139,7 @@ void f_leak(int c) {
   if (!p)
     return;
   if(c)
-    return; // expected-warning {{Opened File never closed. Potential Resource leak}}
+    return; // expected-warning {{Opened stream never closed. Potential resource leak}}
   fclose(p);
 }
 
@@ -240,3 +240,28 @@ void check_escape4() {
   fwrite("1", 1, 1, F); // expected-warning {{might be 'indeterminate'}}
   fclose(F);
 }
+
+int Test;
+_Noreturn void handle_error();
+
+void check_leak_noreturn_1() {
+  FILE *F1 = tmpfile();
+  if (!F1)
+    return;
+  if (Test == 1) {
+    handle_error(); // no warning
+  }
+  rewind(F1);
+} // expected-warning {{Opened stream never closed. Potential resource leak}}
+
+// Check that "location uniqueing" works.
+// This results in reporting only one occurence of resource leak for a stream.
+void check_leak_noreturn_2() {
+  FILE *F1 = tmpfile();
+  if (!F1)
+    return;
+  if (Test == 1) {
+    return; // expected-warning {{Opened stream never closed. Potential resource leak}}
+  }
+  rewind(F1);
+} // no warning
diff --git a/clang/test/Analysis/stream.cpp b/clang/test/Analysis/stream.cpp
index c76b9d7498d09..7eca505bcaf5d 100644
--- a/clang/test/Analysis/stream.cpp
+++ b/clang/test/Analysis/stream.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.unix.Stream -analyzer-store region -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.unix.Stream -verify %s
 
 typedef struct _IO_FILE FILE;
 extern FILE *fopen(const char *path, const char *mode);
@@ -19,4 +19,4 @@ void f1() {
 
 void f2() {
   FILE *f = fopen("file", "r");
-} // expected-warning {{Opened File never closed. Potential Resource leak}}
+} // expected-warning {{Opened stream never closed. Potential resource leak}}
diff --git a/clang/test/Analysis/use-after-move.cpp b/clang/test/Analysis/use-after-move.cpp
index c25f4393cdf92..d7b6c74fabd6c 100644
--- a/clang/test/Analysis/use-after-move.cpp
+++ b/clang/test/Analysis/use-after-move.cpp
@@ -1,36 +1,36 @@
 // RUN: %clang_analyze_cc1 -analyzer-checker=cplusplus.Move %s\
 // RUN:  -std=c++11 -analyzer-output=text -analyzer-config eagerly-assume=false\
 // RUN:  -analyzer-config exploration_strategy=unexplored_first_queue\
-// RUN:  -analyzer-checker core,cplusplus.SmartPtr,debug.ExprInspection\
+// RUN:  -analyzer-checker core,cplusplus.SmartPtrModeling,debug.ExprInspection\
 // RUN:  -verify=expected,peaceful,non-aggressive
 // RUN: %clang_analyze_cc1 -analyzer-checker=cplusplus.Move %s\
 // RUN:  -std=c++11 -analyzer-output=text -analyzer-config eagerly-assume=false\
 // RUN:  -analyzer-config exploration_strategy=dfs -DDFS\
-// RUN:  -analyzer-checker core,cplusplus.SmartPtr,debug.ExprInspection\
+// RUN:  -analyzer-checker core,cplusplus.SmartPtrModeling,debug.ExprInspection\
 // RUN:  -verify=expected,peaceful,non-aggressive
 // RUN: %clang_analyze_cc1 -analyzer-checker=cplusplus.Move %s\
 // RUN:  -std=c++11 -analyzer-output=text -analyzer-config eagerly-assume=false\
 // RUN:  -analyzer-config exploration_strategy=unexplored_first_queue\
 // RUN:  -analyzer-config cplusplus.Move:WarnOn=KnownsOnly\
-// RUN:  -analyzer-checker core,cplusplus.SmartPtr,debug.ExprInspection\
+// RUN:  -analyzer-checker core,cplusplus.SmartPtrModeling,debug.ExprInspection\
 // RUN:  -verify=expected,non-aggressive
 // RUN: %clang_analyze_cc1 -analyzer-checker=cplusplus.Move -verify %s\
 // RUN:  -std=c++11 -analyzer-output=text -analyzer-config eagerly-assume=false\
 // RUN:  -analyzer-config exploration_strategy=dfs -DDFS\
 // RUN:  -analyzer-config cplusplus.Move:WarnOn=KnownsOnly\
-// RUN:  -analyzer-checker core,cplusplus.SmartPtr,debug.ExprInspection\
+// RUN:  -analyzer-checker core,cplusplus.SmartPtrModeling,debug.ExprInspection\
 // RUN:  -verify=expected,non-aggressive
 // RUN: %clang_analyze_cc1 -analyzer-checker=cplusplus.Move %s\
 // RUN:  -std=c++11 -analyzer-output=text -analyzer-config eagerly-assume=false\
 // RUN:  -analyzer-config exploration_strategy=unexplored_first_queue\
 // RUN:  -analyzer-config cplusplus.Move:WarnOn=All\
-// RUN:  -analyzer-checker core,cplusplus.SmartPtr,debug.ExprInspection\
+// RUN:  -analyzer-checker core,cplusplus.SmartPtrModeling,debug.ExprInspection\
 // RUN:  -verify=expected,peaceful,aggressive
 // RUN: %clang_analyze_cc1 -analyzer-checker=cplusplus.Move %s\
 // RUN:  -std=c++11 -analyzer-output=text -analyzer-config eagerly-assume=false\
 // RUN:  -analyzer-config exploration_strategy=dfs -DDFS\
 // RUN:  -analyzer-config cplusplus.Move:WarnOn=All\
-// RUN:  -analyzer-checker core,cplusplus.SmartPtr,debug.ExprInspection\
+// RUN:  -analyzer-checker core,cplusplus.SmartPtrModeling,debug.ExprInspection\
 // RUN:  -verify=expected,peaceful,aggressive
 
 // RUN: not %clang_analyze_cc1 -verify %s \
@@ -49,7 +49,7 @@
 // RUN:  -std=c++11 -analyzer-output=text -analyzer-config eagerly-assume=false\
 // RUN:  -analyzer-config exploration_strategy=dfs -DDFS\
 // RUN:  -analyzer-config cplusplus.Move:WarnOn=All -DAGGRESSIVE_DFS\
-// RUN:  -analyzer-checker core,cplusplus.SmartPtr,debug.ExprInspection\
+// RUN:  -analyzer-checker core,cplusplus.SmartPtrModeling,debug.ExprInspection\
 // RUN:  -verify=expected,peaceful,aggressive %s 2>&1 | FileCheck %s
 
 #include "Inputs/system-header-simulator-cxx.h"
diff --git a/clang/test/Analysis/weak-dependencies.c b/clang/test/Analysis/weak-dependencies.c
index a2179ecaff57f..62cb10b5b5235 100644
--- a/clang/test/Analysis/weak-dependencies.c
+++ b/clang/test/Analysis/weak-dependencies.c
@@ -1,5 +1,5 @@
 // RUN: %clang_analyze_cc1 %s -verify \
-// RUN:   -analyzer-checker=alpha.apiModeling.StdCLibraryFunctionArgs \
+// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctionArgs \
 // RUN:   -analyzer-checker=core
 
 typedef __typeof(sizeof(int)) size_t;
diff --git a/clang/test/CXX/drs/dr4xx.cpp b/clang/test/CXX/drs/dr4xx.cpp
index 2c762237037d2..ff90ee82d4d40 100644
--- a/clang/test/CXX/drs/dr4xx.cpp
+++ b/clang/test/CXX/drs/dr4xx.cpp
@@ -690,7 +690,7 @@ namespace dr457 { // dr457: yes
   };
 }
 
-namespace dr458 { // dr458: no
+namespace dr458 { // dr458: 11
   struct A {
     int T;
     int f();
@@ -706,9 +706,9 @@ namespace dr458 { // dr458: no
   int A::f() {
     return T;
   }
-  template<typename T>
+  template<typename T> // expected-note {{declared here}}
   int A::g() {
-    return T; // FIXME: this is invalid, it finds the template parameter
+    return T; // expected-error {{'T' does not refer to a value}}
   }
 
   template<typename T>
@@ -719,9 +719,9 @@ namespace dr458 { // dr458: no
   int B<T>::g() {
     return T;
   }
-  template<typename U> template<typename T>
+  template<typename U> template<typename T> // expected-note {{declared here}}
   int B<U>::h() {
-    return T; // FIXME: this is invalid, it finds the template parameter
+    return T; // expected-error {{'T' does not refer to a value}}
   }
 }
 
diff --git a/clang/test/CXX/expr/expr.const/p2-0x.cpp b/clang/test/CXX/expr/expr.const/p2-0x.cpp
index c418767f8d12d..b9235eeeb1721 100644
--- a/clang/test/CXX/expr/expr.const/p2-0x.cpp
+++ b/clang/test/CXX/expr/expr.const/p2-0x.cpp
@@ -376,7 +376,7 @@ namespace References {
   int &d = c;
   constexpr int e = 42;
   int &f = const_cast<int&>(e);
-  extern int &g;
+  extern int &g; // expected-note {{here}}
   constexpr int &h(); // expected-note {{here}}
   int &i = h(); // expected-note {{here}}
   constexpr int &j() { return b; }
@@ -390,7 +390,7 @@ namespace References {
     int D2 : &d - &c + 1;
     int E : e / 2;
     int F : f - 11;
-    int G : g; // expected-error {{constant expression}}
+    int G : g; // expected-error {{constant expression}} expected-note {{initializer of 'g' is unknown}}
     int H : h(); // expected-error {{constant expression}} expected-note {{undefined function 'h'}}
     int I : i; // expected-error {{constant expression}} expected-note {{initializer of 'i' is not a constant expression}}
     int J : j();
diff --git a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.constexpr/p2.cpp b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.constexpr/p2.cpp
index 6c0df1aff231a..ecb82372bcb47 100644
--- a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.constexpr/p2.cpp
+++ b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.constexpr/p2.cpp
@@ -1,4 +1,6 @@
-// RUN: %clang_cc1 -std=c++98 -verify %s
+// RUN: %clang_cc1 -std=c++98 -verify=cxx98 %s
+// RUN: %clang_cc1 -std=c++11 -verify=cxx11 %s
+// cxx11-no-diagnostics
 
 template<int n> struct S;
 
@@ -13,9 +15,9 @@ template<int n> struct T {
     //  - a constant with literal type and is initialized with an expression
     //  that is value-dependent.
     const int k = n;
-    typename S<k>::T check3; // ok, u is value-dependent
+    typename S<k>::T check3; // ok, k is value-dependent
 
-    const int &i = k;
-    typename S<i>::T check4; // expected-error {{not an integral constant expression}}
+    const int &i = k; // cxx98-note {{declared here}}
+    typename S<i>::T check4; // cxx98-error {{not an integral constant expression}} cxx98-note {{read of variable 'i' of non-integral, non-enumeration type 'const int &'}}
   }
 };
diff --git a/clang/test/CXX/temp/temp.res/temp.local/p8.cpp b/clang/test/CXX/temp/temp.res/temp.local/p8.cpp
index fecfed06f109c..6b2071eb12ce0 100644
--- a/clang/test/CXX/temp/temp.res/temp.local/p8.cpp
+++ b/clang/test/CXX/temp/temp.res/temp.local/p8.cpp
@@ -1,5 +1,4 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s
-// expected-no-diagnostics
 
 namespace N { 
   enum { C };
@@ -52,3 +51,104 @@ void N::Y::f(D) {
   D d;
 }
 
+// Ensure we properly interleave the searches within classes and template parameter lists.
+namespace SearchClassBetweenTemplateParameterLists {
+  int AA, BB; // none of the below lookups should ever consider these
+
+  struct Base {
+    using AA = void;
+    using BB = void;
+  };
+  struct BaseT : Base {
+    using T = void;
+  };
+  struct BaseU : Base {
+    using U = void;
+  };
+
+  template<typename T> struct A {
+    using AA = void;
+    template<typename U> struct B {
+      using BB = void;
+      void f(U);
+      void g(U);
+      void h(T);
+      void i(T);
+      template<typename V> void j(V);
+      template<typename V> void k(U);
+
+      // OK: these find the template parameter not the member.
+      template<typename AA> void l(AA x) { AA aa; }
+      template<typename BB> void m(BB x) { BB bb; }
+
+      struct C : Base {
+        // All OK; these find the template parameters.
+        template<typename> void f(T x) { T t; }
+        template<typename> void g(U x) { U u; }
+        template<typename AA> void h(AA x) { AA aa; }
+        template<typename BB> void i(BB x) { BB bb; }
+      };
+
+      struct CT : BaseT {
+        template<typename> void f(T x) { // expected-error {{void}}
+          T t; // expected-error {{incomplete}}
+        }
+        template<typename> void g(U x) { U u; }
+        template<typename AA> void h(AA x) { AA aa; }
+        template<typename BB> void i(BB x) { BB bb; }
+      };
+
+      struct CU : BaseU {
+        template<typename> void f(T x) { T t; }
+        template<typename> void g(U x) { // expected-error {{void}}
+          U u; // expected-error {{incomplete}}
+        }
+        template<typename AA> void h(AA x) { AA aa; }
+        template<typename BB> void i(BB x) { BB bb; }
+      };
+    };
+  };
+
+  // Search order for the below is:
+  // 1) template parameter scope of the function itself (if any)
+  // 2) class of which function is a member
+  // 3) template parameter scope of inner class
+  // 4) class of which class is a member
+  // 5) template parameter scope of outer class
+
+  // OK, 'AA' found in (3)
+  template<typename T> template<typename AA>
+  void A<T>::B<AA>::f(AA) {
+    AA aa;
+  }
+
+  // error, 'BB' found in (2)
+  template<typename T> template<typename BB>
+  void A<T>::B<BB>::g(BB) { // expected-error {{does not match}}
+    BB bb; // expected-error {{incomplete type}}
+  }
+
+  // error, 'AA' found in (4)
+  template<typename AA> template<typename U>
+  void A<AA>::B<U>::h(AA) { // expected-error {{does not match}}
+    AA aa; // expected-error {{incomplete type}}
+  }
+
+  // error, 'BB' found in (2)
+  template<typename BB> template<typename U>
+  void A<BB>::B<U>::i(BB) { // expected-error {{does not match}}
+    BB bb; // expected-error {{incomplete type}}
+  }
+
+  // OK, 'BB' found in (1)
+  template<typename T> template<typename U> template<typename BB>
+  void A<T>::B<U>::j(BB) {
+    BB bb;
+  }
+
+  // error, 'BB' found in (2)
+  template<typename T> template<typename BB> template<typename V>
+  void A<T>::B<BB>::k(V) { // expected-error {{does not match}}
+    BB bb; // expected-error {{incomplete type}}
+  }
+}
diff --git a/clang/test/CodeCompletion/after-function-equals.cpp b/clang/test/CodeCompletion/after-function-equals.cpp
new file mode 100644
index 0000000000000..871c2a3f6d917
--- /dev/null
+++ b/clang/test/CodeCompletion/after-function-equals.cpp
@@ -0,0 +1,47 @@
+struct A {
+  A() = default;
+  // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:2:9 -std=gnu++11 %s -o - | FileCheck -check-prefix=CHECK-CC1 %s
+  // CHECK-CC1: COMPLETION: default
+  // CHECK-CC1-NEXT: COMPLETION: delete
+
+  A(const A &) = default;
+  // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:7:18 -std=gnu++11 %s -o - | FileCheck -check-prefix=CHECK-CC2 %s
+  // CHECK-CC2: COMPLETION: default
+  // CHECK-CC2-NEXT: COMPLETION: delete
+
+  A(const A &, int) = delete;
+  // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:12:23 -std=gnu++11 %s -o - | FileCheck -check-prefix=CHECK-CC3 %s
+  // CHECK-CC3-NOT: COMPLETION: default
+  // CHECK-CC3: COMPLETION: delete
+
+  A(A &&);
+
+  A &operator=(const A &) = default;
+  // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:19:29 -std=gnu++11 %s -o - | FileCheck -check-prefix=CHECK-CC4 %s
+  // CHECK-CC4: COMPLETION: default
+  // CHECK-CC4-NEXT: COMPLETION: delete
+
+  bool operator==(const A &) const = delete;
+  // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:24:38 -std=gnu++11 %s -o - | FileCheck -check-prefix=CHECK-CC5 %s
+  // CHECK-CC5-NOT: COMPLETION: default
+  // CHECK-CC5: COMPLETION: delete
+
+  // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:24:38 -std=gnu++20 %s -o - | FileCheck -check-prefix=CHECK-CC6 %s
+  // CHECK-CC6: COMPLETION: default
+  // CHECK-CC6-NEXT: COMPLETION: delete
+
+  void test() = delete;
+  // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:33:17 -std=gnu++11 %s -o - | FileCheck -check-prefix=CHECK-CC7 %s
+  // CHECK-CC7-NOT: COMPLETION: default
+  // CHECK-CC7: COMPLETION: delete
+};
+
+A::A(A &&) = default;
+// RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:39:14 -std=gnu++11 %s -o - | FileCheck -check-prefix=CHECK-CC8 %s
+// CHECK-CC8: COMPLETION: default
+// CHECK-CC8-NEXT: COMPLETION: delete
+
+void test() = delete;
+// RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:44:15 -std=gnu++11 %s -o - | FileCheck -check-prefix=CHECK-CC9 %s
+// CHECK-CC9-NOT: COMPLETION: default
+// CHECK-CC9: COMPLETION: delete
\ No newline at end of file
diff --git a/clang/test/CodeCompletion/ordinary-name-cxx11.cpp b/clang/test/CodeCompletion/ordinary-name-cxx11.cpp
index e568ee1ee3d8d..7593d00210e7c 100644
--- a/clang/test/CodeCompletion/ordinary-name-cxx11.cpp
+++ b/clang/test/CodeCompletion/ordinary-name-cxx11.cpp
@@ -57,11 +57,12 @@ void foo() {
   // CHECK-CC1: COMPLETION: TYPEDEF : TYPEDEF
   // CHECK-CC1-NEXT: COMPLETION: Pattern : typedef <#type#> <#name#>;
   // CHECK-CC1-NEXT: COMPLETION: Pattern : [#std::type_info#]typeid(<#expression-or-type#>)
-  // CHECK-CC1-NEXT: COMPLETION: Pattern : typename <#qualifier#>::<#name#>
+  // CHECK-CC1-NEXT: COMPLETION: Pattern : typename <#name#>
   // CHECK-CC1-NEXT: COMPLETION: Pattern : typeof <#expression#>
   // CHECK-CC1-NEXT: COMPLETION: Pattern : typeof(<#type#>)
   // CHECK-CC1-NEXT: COMPLETION: union
   // CHECK-CC1-NEXT: COMPLETION: unsigned
+  // CHECK-CC1-NEXT: COMPLETION: Pattern : using <#name#> = <#type#>;
   // CHECK-CC1-NEXT: COMPLETION: Pattern : using namespace <#identifier#>;
   // CHECK-CC1-NEXT: COMPLETION: void
   // CHECK-CC1-NEXT: COMPLETION: volatile
@@ -107,12 +108,13 @@ void foo() {
   // CHECK-CC2-NEXT: COMPLETION: thread_local
   // CHECK-CC2-NEXT: COMPLETION: TYPEDEF : TYPEDEF
   // CHECK-CC2-NEXT: COMPLETION: Pattern : typedef <#type#> <#name#>;
-  // CHECK-CC2-NEXT: COMPLETION: Pattern : typename <#qualifier#>::<#name#>
+  // CHECK-CC2-NEXT: COMPLETION: Pattern : typename <#name#>
   // CHECK-CC2-NEXT: COMPLETION: Pattern : typeof <#expression#>
   // CHECK-CC2-NEXT: COMPLETION: Pattern : typeof(<#type#>)
   // CHECK-CC2-NEXT: COMPLETION: union
   // CHECK-CC2-NEXT: COMPLETION: unsigned
   // CHECK-CC2-NEXT: COMPLETION: Pattern : using <#qualifier#>::<#name#>;
+  // CHECK-CC2-NEXT: COMPLETION: Pattern : using <#name#> = <#type#>;
   // CHECK-CC2-NEXT: COMPLETION: Pattern : using namespace <#identifier#>;
   // CHECK-CC2-NEXT: COMPLETION: void
   // CHECK-CC2-NEXT: COMPLETION: volatile
@@ -150,12 +152,13 @@ void foo() {
   // CHECK-CC3-NEXT: COMPLETION: Pattern : template<<#parameters#>>
   // CHECK-CC3-NEXT: COMPLETION: thread_local
   // CHECK-CC3-NEXT: COMPLETION: Pattern : typedef <#type#> <#name#>;
-  // CHECK-CC3-NEXT: COMPLETION: Pattern : typename <#qualifier#>::<#name#>
+  // CHECK-CC3-NEXT: COMPLETION: Pattern : typename <#name#>
   // CHECK-CC3-NEXT: COMPLETION: Pattern : typeof <#expression#>
   // CHECK-CC3-NEXT: COMPLETION: Pattern : typeof(<#type#>)
   // CHECK-CC3-NEXT: COMPLETION: union
   // CHECK-CC3-NEXT: COMPLETION: unsigned
   // CHECK-CC3-NEXT: COMPLETION: Pattern : using <#qualifier#>::<#name#>
+  // CHECK-CC3-NEXT: COMPLETION: Pattern : using <#name#> = <#type#>;
   // CHECK-CC3-NEXT: COMPLETION: virtual
   // CHECK-CC3-NEXT: COMPLETION: void
   // CHECK-CC3-NEXT: COMPLETION: volatile
@@ -198,7 +201,7 @@ void foo() {
   // CHECK-CC4-NEXT: COMPLETION: Pattern : [#bool#]true
   // CHECK-CC4-NEXT: COMPLETION: TYPEDEF : TYPEDEF
   // CHECK-CC4-NEXT: COMPLETION: Pattern : [#std::type_info#]typeid(<#expression-or-type#>)
-  // CHECK-CC4-NEXT: COMPLETION: Pattern : typename <#qualifier#>::<#name#>
+  // CHECK-CC4-NEXT: COMPLETION: Pattern : typename <#name#>
   // CHECK-CC4-NEXT: COMPLETION: Pattern : typeof <#expression#>
   // CHECK-CC4-NEXT: COMPLETION: Pattern : typeof(<#type#>)
   // CHECK-CC4-NEXT: COMPLETION: union
@@ -259,11 +262,12 @@ void foo() {
   // CHECK-NO-RTTI: COMPLETION: TYPEDEF : TYPEDEF
   // CHECK-NO-RTTI-NEXT: COMPLETION: Pattern : typedef <#type#> <#name#>;
   // CHECK-NO-RTTI-NOT: typeid
-  // CHECK-NO-RTTI: COMPLETION: Pattern : typename <#qualifier#>::<#name#>
+  // CHECK-NO-RTTI: COMPLETION: Pattern : typename <#name#>
   // CHECK-NO-RTTI-NEXT: COMPLETION: Pattern : typeof <#expression#>
   // CHECK-NO-RTTI-NEXT: COMPLETION: Pattern : typeof(<#type#>)
   // CHECK-NO-RTTI-NEXT: COMPLETION: union
   // CHECK-NO-RTTI-NEXT: COMPLETION: unsigned
+  // CHECK-NO-RTTI-NEXT: COMPLETION: Pattern : using <#name#> = <#type#>;
   // CHECK-NO-RTTI-NEXT: COMPLETION: Pattern : using namespace <#identifier#>;
   // CHECK-NO-RTTI-NEXT: COMPLETION: void
   // CHECK-NO-RTTI-NEXT: COMPLETION: volatile
diff --git a/clang/test/CodeCompletion/ordinary-name.cpp b/clang/test/CodeCompletion/ordinary-name.cpp
index 3f891307bb74b..3c3d0c5d68831 100644
--- a/clang/test/CodeCompletion/ordinary-name.cpp
+++ b/clang/test/CodeCompletion/ordinary-name.cpp
@@ -54,11 +54,12 @@ void foo() {
   // CHECK-CC1: COMPLETION: TYPEDEF : TYPEDEF
   // CHECK-CC1-NEXT: COMPLETION: Pattern : typedef <#type#> <#name#>;
   // CHECK-CC1-NEXT: COMPLETION: Pattern : [#std::type_info#]typeid(<#expression-or-type#>)
-  // CHECK-CC1-NEXT: COMPLETION: Pattern : typename <#qualifier#>::<#name#>
+  // CHECK-CC1-NEXT: COMPLETION: Pattern : typename <#name#>
   // CHECK-CC1-NEXT: COMPLETION: Pattern : typeof <#expression#>
   // CHECK-CC1-NEXT: COMPLETION: Pattern : typeof(<#type#>)
   // CHECK-CC1-NEXT: COMPLETION: union
   // CHECK-CC1-NEXT: COMPLETION: unsigned
+  // CHECK-CC1-NOT: COMPLETION: Pattern : using <#name#> = <#type#>;
   // CHECK-CC1-NEXT: COMPLETION: Pattern : using namespace <#identifier#>;
   // CHECK-CC1-NEXT: COMPLETION: void
   // CHECK-CC1-NEXT: COMPLETION: volatile
@@ -97,12 +98,13 @@ void foo() {
   // CHECK-CC2-NEXT: COMPLETION: Pattern : template<<#parameters#>>
   // CHECK-CC2-NEXT: COMPLETION: TYPEDEF : TYPEDEF
   // CHECK-CC2-NEXT: COMPLETION: Pattern : typedef <#type#> <#name#>;
-  // CHECK-CC2-NEXT: COMPLETION: Pattern : typename <#qualifier#>::<#name#>
+  // CHECK-CC2-NEXT: COMPLETION: Pattern : typename <#name#>
   // CHECK-CC2-NEXT: COMPLETION: Pattern : typeof <#expression#>
   // CHECK-CC2-NEXT: COMPLETION: Pattern : typeof(<#type#>)
   // CHECK-CC2-NEXT: COMPLETION: union
   // CHECK-CC2-NEXT: COMPLETION: unsigned
   // CHECK-CC2-NEXT: COMPLETION: Pattern : using <#qualifier#>::<#name#>;
+  // CHECK-CC2-NOT: COMPLETION: Pattern : using <#name#> = <#type#>;
   // CHECK-CC2-NEXT: COMPLETION: Pattern : using namespace <#identifier#>;
   // CHECK-CC2-NEXT: COMPLETION: void
   // CHECK-CC2-NEXT: COMPLETION: volatile
@@ -134,12 +136,13 @@ void foo() {
   // CHECK-CC3-NEXT: COMPLETION: struct
   // CHECK-CC3-NEXT: COMPLETION: Pattern : template<<#parameters#>>
   // CHECK-CC3-NEXT: COMPLETION: Pattern : typedef <#type#> <#name#>;
-  // CHECK-CC3-NEXT: COMPLETION: Pattern : typename <#qualifier#>::<#name#>
+  // CHECK-CC3-NEXT: COMPLETION: Pattern : typename <#name#>
   // CHECK-CC3-NEXT: COMPLETION: Pattern : typeof <#expression#>
   // CHECK-CC3-NEXT: COMPLETION: Pattern : typeof(<#type#>)
   // CHECK-CC3-NEXT: COMPLETION: union
   // CHECK-CC3-NEXT: COMPLETION: unsigned
   // CHECK-CC3-NEXT: COMPLETION: Pattern : using <#qualifier#>::<#name#>;
+  // CHECK-CC3-NOT: COMPLETION: Pattern : using <#name#> = <#type#>;
   // CHECK-CC3-NEXT: COMPLETION: virtual
   // CHECK-CC3-NEXT: COMPLETION: void
   // CHECK-CC3-NEXT: COMPLETION: volatile
@@ -176,7 +179,7 @@ void foo() {
   // CHECK-CC4-NEXT: COMPLETION: Pattern : [#bool#]true
   // CHECK-CC4-NEXT: COMPLETION: TYPEDEF : TYPEDEF
   // CHECK-CC4-NEXT: COMPLETION: Pattern : [#std::type_info#]typeid(<#expression-or-type#>)
-  // CHECK-CC4-NEXT: COMPLETION: Pattern : typename <#qualifier#>::<#name#>
+  // CHECK-CC4-NEXT: COMPLETION: Pattern : typename <#name#>
   // CHECK-CC4-NEXT: COMPLETION: Pattern : typeof <#expression#>
   // CHECK-CC4-NEXT: COMPLETION: Pattern : typeof(<#type#>)
   // CHECK-CC4-NEXT: COMPLETION: union
@@ -227,11 +230,12 @@ void foo() {
   // CHECK-NO-RTTI: COMPLETION: TYPEDEF : TYPEDEF
   // CHECK-NO-RTTI-NEXT: COMPLETION: Pattern : typedef <#type#> <#name#>;
   // CHECK-NO-RTTI-NOT: typeid
-  // CHECK-NO-RTTI: COMPLETION: Pattern : typename <#qualifier#>::<#name#>
+  // CHECK-NO-RTTI: COMPLETION: Pattern : typename <#name#>
   // CHECK-NO-RTTI-NEXT: COMPLETION: Pattern : typeof <#expression#>
   // CHECK-NO-RTTI-NEXT: COMPLETION: Pattern : typeof(<#type#>)
   // CHECK-NO-RTTI-NEXT: COMPLETION: union
   // CHECK-NO-RTTI-NEXT: COMPLETION: unsigned
+  // CHECK-NO-RTTI-NOT: COMPLETION: Pattern : using <#name#> = <#type#>;
   // CHECK-NO-RTTI-NEXT: COMPLETION: Pattern : using namespace <#identifier#>;
   // CHECK-NO-RTTI-NEXT: COMPLETION: void
   // CHECK-NO-RTTI-NEXT: COMPLETION: volatile
diff --git a/clang/test/CodeCompletion/patterns.cpp b/clang/test/CodeCompletion/patterns.cpp
index 5189e3e636d5a..9d18ced8aa6d1 100644
--- a/clang/test/CodeCompletion/patterns.cpp
+++ b/clang/test/CodeCompletion/patterns.cpp
@@ -74,3 +74,30 @@ int Cls::*memptr_return() {
 // RUN: %clang_cc1 -fsyntax-only -std=c++03 -code-completion-patterns -code-completion-at=%s:37:1 %s -o - | FileCheck -check-prefix=RETURN-PTR-STD03 %s
 // RUN: %clang_cc1 -fsyntax-only -std=c++03 -code-completion-patterns -code-completion-at=%s:41:1 %s -o - | FileCheck -check-prefix=RETURN-PTR-STD03 %s
 // RETURN-PTR-STD03-NOT: COMPLETION: Pattern : return nullptr;
+
+void something();
+
+void unbraced_if() {
+  if (true)
+    something();
+  // line 83
+}
+// RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:83:3 %s -o - | FileCheck -check-prefix=UNBRACED-IF %s
+// UNBRACED-IF: COMPLETION: Pattern : else
+// UNBRACED-IF-NEXT: <#statement#>;
+// UNBRACED-IF: COMPLETION: Pattern : else if (<#condition#>)
+// UNBRACED-IF-NEXT: <#statement#>;
+
+void braced_if() {
+  if (true) {
+    something();
+  }
+  // line 95
+}
+// RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:95:3 %s -o - | FileCheck -check-prefix=BRACED-IF %s
+// BRACED-IF: COMPLETION: Pattern : else {
+// BRACED-IF-NEXT: <#statements#>
+// BRACED-IF-NEXT: }
+// BRACED-IF: COMPLETION: Pattern : else if (<#condition#>) {
+// BRACED-IF-NEXT: <#statements#>
+// BRACED-IF-NEXT: }
diff --git a/clang/test/CodeGen/AMX/amx.c b/clang/test/CodeGen/AMX/amx.c
new file mode 100644
index 0000000000000..89b486f7a6010
--- /dev/null
+++ b/clang/test/CodeGen/AMX/amx.c
@@ -0,0 +1,32 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown  -target-feature +amx-int8  \
+// RUN: -target-feature +amx-bf16 -emit-llvm -o - -Werror -pedantic | FileCheck %s --check-prefixes=CHECK
+
+#include <immintrin.h>
+
+void test_amx(void *data) {
+  //CHECK-LABEL: @test_amx
+  //CHECK: call void @llvm.x86.ldtilecfg(i8* %{{.*}})
+  //CHECK: call void @llvm.x86.sttilecfg(i8* %{{.*}})
+  //CHECK: call void @llvm.x86.tilerelease()
+  //CHECK: call void @llvm.x86.tilezero(i8 3)
+  //CHECK: call void @llvm.x86.tileloadd64(i8 4, i8* %{{.*}}, i64 8)
+  //CHECK: call void @llvm.x86.tileloaddt164(i8 0, i8* %{{.*}}, i64 1)
+  //CHECK: call void @llvm.x86.tilestored64(i8 0, i8* %{{.*}}, i64 1)
+  //CHECK: call void @llvm.x86.tdpbssd(i8 1, i8 2, i8 3)
+  //CHECK: call void @llvm.x86.tdpbsud(i8 1, i8 2, i8 3)
+  //CHECK: call void @llvm.x86.tdpbusd(i8 1, i8 2, i8 3)
+  //CHECK: call void @llvm.x86.tdpbuud(i8 1, i8 2, i8 3)
+  //CHECK: call void @llvm.x86.tdpbf16ps(i8 1, i8 2, i8 3)
+  _tile_loadconfig(data);
+  _tile_storeconfig(data);
+  _tile_release();
+  _tile_zero(3);
+  _tile_loadd(4, data, 8);
+  _tile_stream_loadd(0, data, 1);
+  _tile_stored(0, data, 1);
+  _tile_dpbssd(1, 2, 3);
+  _tile_dpbsud(1, 2, 3);
+  _tile_dpbusd(1, 2, 3);
+  _tile_dpbuud(1, 2, 3);
+  _tile_dpbf16ps(1, 2, 3);
+}
diff --git a/clang/test/CodeGen/AMX/amx_errors.c b/clang/test/CodeGen/AMX/amx_errors.c
new file mode 100644
index 0000000000000..13a2b33b5a0a2
--- /dev/null
+++ b/clang/test/CodeGen/AMX/amx_errors.c
@@ -0,0 +1,17 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +amx-tile -target-feature +amx-int8 -target-feature +amx-bf16 -emit-llvm -fsyntax-only -verify
+
+#include <immintrin.h>
+
+void test_amx(void *data) {
+  _tile_zero(16); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
+  _tile_loadd(19, data, 16); // expected-error {{argument value 19 is outside the valid range [0, 7]}}
+  _tile_stream_loadd(23, data, 1); // expected-error {{argument value 23 is outside the valid range [0, 7]}}
+  _tile_stored(88, data, 1); // expected-error {{argument value 88 is outside the valid range [0, 7]}}
+  _tile_dpbssd(16, 2, 3); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
+  _tile_dpbssd(0, 16, 3); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
+  _tile_dpbuud(0, 2, 16); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
+  _tile_dpbsud(1, 1, 3); // expected-error {{tile arguments must refer to different tiles}}
+  _tile_dpbsud(7, 1, 7); // expected-error {{tile arguments must refer to different tiles}}
+  _tile_dpbsud(4, 3, 3); // expected-error {{tile arguments must refer to different tiles}}
+  _tile_dpbf16ps(4, 3, 3); // expected-error {{tile arguments must refer to different tiles}}
+}
diff --git a/clang/test/CodeGen/AMX/amx_inline_asm.c b/clang/test/CodeGen/AMX/amx_inline_asm.c
new file mode 100644
index 0000000000000..9d828f8ac94e3
--- /dev/null
+++ b/clang/test/CodeGen/AMX/amx_inline_asm.c
@@ -0,0 +1,11 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown  -target-feature +amx-int8 -target-feature +amx-bf16 -emit-llvm -o - -Wall -Werror -pedantic | FileCheck %s --check-prefixes=CHECK,X86_64
+
+void f_tilemul(short a)
+{
+  //CHECK:  call void asm sideeffect "tileloadd 0(%rsi,%r13,4), %tmm0   \0A\09tileloadd 0(%rdx,%r14,4), %tmm6   \0A\09tdpbf16ps %tmm6, %tmm0, %tmm7    \0A\09tilestored %tmm7, 0(%r12,%r15,4) \0A\09", "~{memory},~{tmm0},~{tmm6},~{tmm7},~{dirflag},~{fpsr},~{flags}"()
+  __asm__ volatile ("tileloadd 0(%%rsi,%%r13,4), %%tmm0   \n\t"
+                    "tileloadd 0(%%rdx,%%r14,4), %%tmm6   \n\t"
+                    "tdpbf16ps %%tmm6, %%tmm0, %%tmm7    \n\t"
+                    "tilestored %%tmm7, 0(%%r12,%%r15,4) \n\t"
+          ::: "memory", "tmm0", "tmm6", "tmm7");
+}
diff --git a/clang/test/CodeGen/Atomics.c b/clang/test/CodeGen/Atomics.c
index d960ac6df2f96..ebc9c139775a9 100644
--- a/clang/test/CodeGen/Atomics.c
+++ b/clang/test/CodeGen/Atomics.c
@@ -10,6 +10,8 @@ signed int si;
 unsigned int ui;
 signed long long sll;
 unsigned long long ull;
+__int128 s128;
+unsigned  __int128 u128;
 
 void test_op_ignore (void) // CHECK-LABEL: define void @test_op_ignore
 {
@@ -48,6 +50,8 @@ void test_op_ignore (void) // CHECK-LABEL: define void @test_op_ignore
   (void) __sync_fetch_and_xor (&ui, 1); // CHECK: atomicrmw xor i32
   (void) __sync_fetch_and_xor (&sll, 1); // CHECK: atomicrmw xor i64
   (void) __sync_fetch_and_xor (&ull, 1); // CHECK: atomicrmw xor i64
+  (void) __sync_fetch_and_xor (&u128, 1); // CHECK: atomicrmw xor i128
+  (void) __sync_fetch_and_xor (&s128, 1); // CHECK: atomicrmw xor i128
 
   (void) __sync_fetch_and_nand (&sc, 1); // CHECK: atomicrmw nand i8
   (void) __sync_fetch_and_nand (&uc, 1); // CHECK: atomicrmw nand i8
@@ -168,27 +172,43 @@ void test_op_and_fetch (void)
   sc = __sync_nand_and_fetch (&sc, uc); // CHECK: atomicrmw nand
                                         // CHECK: and
                                         // CHECK: xor
+                                        // CHECK: -1
   uc = __sync_nand_and_fetch (&uc, uc); // CHECK: atomicrmw nand
                                         // CHECK: and
                                         // CHECK: xor
+                                        // CHECK: -1
   ss = __sync_nand_and_fetch (&ss, uc); // CHECK: atomicrmw nand
                                         // CHECK: and
                                         // CHECK: xor
+                                        // CHECK: -1
   us = __sync_nand_and_fetch (&us, uc); // CHECK: atomicrmw nand
                                         // CHECK: and
                                         // CHECK: xor
+                                        // CHECK: -1
   si = __sync_nand_and_fetch (&si, uc); // CHECK: atomicrmw nand
                                         // CHECK: and
                                         // CHECK: xor
+                                        // CHECK: -1
   ui = __sync_nand_and_fetch (&ui, uc); // CHECK: atomicrmw nand
                                         // CHECK: and
                                         // CHECK: xor
+                                        // CHECK: -1
   sll = __sync_nand_and_fetch (&sll, uc); // CHECK: atomicrmw nand
                                           // CHECK: and
                                           // CHECK: xor
+                                          // CHECK: -1
   ull = __sync_nand_and_fetch (&ull, uc); // CHECK: atomicrmw nand
                                           // CHECK: and
                                           // CHECK: xor
+                                          // CHECK: -1
+  u128 = __sync_nand_and_fetch (&u128, uc); // CHECK: atomicrmw nand
+                                          // CHECK: and
+                                          // CHECK: xor
+                                          // CHECK: -1
+  s128 = __sync_nand_and_fetch (&s128, uc); // CHECK: atomicrmw nand
+                                          // CHECK: and
+                                          // CHECK: xor
+                                          // CHECK: -1
 
   sc = __sync_and_and_fetch (&sc, uc); // CHECK: atomicrmw and
   uc = __sync_and_and_fetch (&uc, uc); // CHECK: atomicrmw and
diff --git a/clang/test/CodeGen/aarch64-bf16-lane-intrinsics.c b/clang/test/CodeGen/aarch64-bf16-lane-intrinsics.c
new file mode 100644
index 0000000000000..8a8795cd8ad8b
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-bf16-lane-intrinsics.c
@@ -0,0 +1,145 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple aarch64-arm-none-eabi -target-feature +neon -target-feature +bf16 \
+// RUN:  -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg -instcombine | FileCheck --check-prefix=CHECK-LE %s
+// RUN: %clang_cc1 -triple aarch64_be-arm-none-eabi -target-feature +neon -target-feature +bf16 \
+// RUN:  -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg -instcombine | FileCheck --check-prefix=CHECK-BE %s
+
+#include <arm_neon.h>
+
+// CHECK-LE-LABEL: @test_vcopy_lane_bf16_v1(
+// CHECK-LE-NEXT:  entry:
+// CHECK-LE-NEXT:    [[VSET_LANE:%.*]] = shufflevector <4 x bfloat> [[A:%.*]], <4 x bfloat> [[B:%.*]], <4 x i32> <i32 0, i32 7, i32 2, i32 3>
+// CHECK-LE-NEXT:    ret <4 x bfloat> [[VSET_LANE]]
+//
+// CHECK-BE-LABEL: @test_vcopy_lane_bf16_v1(
+// CHECK-BE-NEXT:  entry:
+// CHECK-BE-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x bfloat> [[A:%.*]], <4 x bfloat> undef, <4 x i32> <i32 3, i32 undef, i32 1, i32 0>
+// CHECK-BE-NEXT:    [[VSET_LANE:%.*]] = shufflevector <4 x bfloat> [[SHUFFLE]], <4 x bfloat> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 3>
+// CHECK-BE-NEXT:    [[SHUFFLE5:%.*]] = shufflevector <4 x bfloat> [[VSET_LANE]], <4 x bfloat> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT:    ret <4 x bfloat> [[SHUFFLE5]]
+//
+bfloat16x4_t test_vcopy_lane_bf16_v1(bfloat16x4_t a, bfloat16x4_t b) {
+  return vcopy_lane_bf16(a, 1, b, 3);
+}
+
+// CHECK-LE-LABEL: @test_vcopy_lane_bf16_v2(
+// CHECK-LE-NEXT:  entry:
+// CHECK-LE-NEXT:    [[VSET_LANE:%.*]] = shufflevector <4 x bfloat> [[A:%.*]], <4 x bfloat> [[B:%.*]], <4 x i32> <i32 0, i32 1, i32 4, i32 3>
+// CHECK-LE-NEXT:    ret <4 x bfloat> [[VSET_LANE]]
+//
+// CHECK-BE-LABEL: @test_vcopy_lane_bf16_v2(
+// CHECK-BE-NEXT:  entry:
+// CHECK-BE-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x bfloat> [[A:%.*]], <4 x bfloat> undef, <4 x i32> <i32 3, i32 2, i32 undef, i32 0>
+// CHECK-BE-NEXT:    [[VSET_LANE:%.*]] = shufflevector <4 x bfloat> [[SHUFFLE]], <4 x bfloat> [[B:%.*]], <4 x i32> <i32 0, i32 1, i32 7, i32 3>
+// CHECK-BE-NEXT:    [[SHUFFLE5:%.*]] = shufflevector <4 x bfloat> [[VSET_LANE]], <4 x bfloat> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT:    ret <4 x bfloat> [[SHUFFLE5]]
+//
+bfloat16x4_t test_vcopy_lane_bf16_v2(bfloat16x4_t a, bfloat16x4_t b) {
+  return vcopy_lane_bf16(a, 2, b, 0);
+}
+
+// CHECK-LE-LABEL: @test_vcopyq_lane_bf16_v1(
+// CHECK-LE-NEXT:  entry:
+// CHECK-LE-NEXT:    [[TMP0:%.*]] = shufflevector <4 x bfloat> [[B:%.*]], <4 x bfloat> undef, <8 x i32> <i32 undef, i32 undef, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+// CHECK-LE-NEXT:    [[VSET_LANE:%.*]] = shufflevector <8 x bfloat> [[A:%.*]], <8 x bfloat> [[TMP0]], <8 x i32> <i32 10, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+// CHECK-LE-NEXT:    ret <8 x bfloat> [[VSET_LANE]]
+//
+// CHECK-BE-LABEL: @test_vcopyq_lane_bf16_v1(
+// CHECK-BE-NEXT:  entry:
+// CHECK-BE-NEXT:    [[TMP0:%.*]] = shufflevector <4 x bfloat> [[B:%.*]], <4 x bfloat> undef, <8 x i32> <i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+// CHECK-BE-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x bfloat> [[A:%.*]], <8 x bfloat> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT:    [[VSET_LANE:%.*]] = shufflevector <8 x bfloat> [[SHUFFLE]], <8 x bfloat> [[TMP0]], <8 x i32> <i32 9, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+// CHECK-BE-NEXT:    [[SHUFFLE5:%.*]] = shufflevector <8 x bfloat> [[VSET_LANE]], <8 x bfloat> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT:    ret <8 x bfloat> [[SHUFFLE5]]
+//
+bfloat16x8_t test_vcopyq_lane_bf16_v1(bfloat16x8_t a, bfloat16x4_t b) {
+  return vcopyq_lane_bf16(a, 0, b, 2);
+}
+
+// CHECK-LE-LABEL: @test_vcopyq_lane_bf16_v2(
+// CHECK-LE-NEXT:  entry:
+// CHECK-LE-NEXT:    [[TMP0:%.*]] = shufflevector <4 x bfloat> [[B:%.*]], <4 x bfloat> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+// CHECK-LE-NEXT:    [[VSET_LANE:%.*]] = shufflevector <8 x bfloat> [[A:%.*]], <8 x bfloat> [[TMP0]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 7>
+// CHECK-LE-NEXT:    ret <8 x bfloat> [[VSET_LANE]]
+//
+// CHECK-BE-LABEL: @test_vcopyq_lane_bf16_v2(
+// CHECK-BE-NEXT:  entry:
+// CHECK-BE-NEXT:    [[TMP0:%.*]] = shufflevector <4 x bfloat> [[B:%.*]], <4 x bfloat> undef, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+// CHECK-BE-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x bfloat> [[A:%.*]], <8 x bfloat> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 undef, i32 0>
+// CHECK-BE-NEXT:    [[VSET_LANE:%.*]] = shufflevector <8 x bfloat> [[SHUFFLE]], <8 x bfloat> [[TMP0]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 11, i32 7>
+// CHECK-BE-NEXT:    [[SHUFFLE5:%.*]] = shufflevector <8 x bfloat> [[VSET_LANE]], <8 x bfloat> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT:    ret <8 x bfloat> [[SHUFFLE5]]
+//
+bfloat16x8_t test_vcopyq_lane_bf16_v2(bfloat16x8_t a, bfloat16x4_t b) {
+  return vcopyq_lane_bf16(a, 6, b, 0);
+}
+
+// CHECK-LE-LABEL: @test_vcopy_laneq_bf16_v1(
+// CHECK-LE-NEXT:  entry:
+// CHECK-LE-NEXT:    [[VGETQ_LANE:%.*]] = extractelement <8 x bfloat> [[B:%.*]], i32 7
+// CHECK-LE-NEXT:    [[VSET_LANE:%.*]] = insertelement <4 x bfloat> [[A:%.*]], bfloat [[VGETQ_LANE]], i32 0
+// CHECK-LE-NEXT:    ret <4 x bfloat> [[VSET_LANE]]
+//
+// CHECK-BE-LABEL: @test_vcopy_laneq_bf16_v1(
+// CHECK-BE-NEXT:  entry:
+// CHECK-BE-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x bfloat> [[A:%.*]], <4 x bfloat> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT:    [[VGETQ_LANE:%.*]] = extractelement <8 x bfloat> [[B:%.*]], i32 0
+// CHECK-BE-NEXT:    [[VSET_LANE:%.*]] = insertelement <4 x bfloat> [[SHUFFLE]], bfloat [[VGETQ_LANE]], i32 0
+// CHECK-BE-NEXT:    [[SHUFFLE5:%.*]] = shufflevector <4 x bfloat> [[VSET_LANE]], <4 x bfloat> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT:    ret <4 x bfloat> [[SHUFFLE5]]
+//
+bfloat16x4_t test_vcopy_laneq_bf16_v1(bfloat16x4_t a, bfloat16x8_t b) {
+  return vcopy_laneq_bf16(a, 0, b, 7);
+}
+
+// CHECK-LE-LABEL: @test_vcopy_laneq_bf16_v2(
+// CHECK-LE-NEXT:  entry:
+// CHECK-LE-NEXT:    [[VGETQ_LANE:%.*]] = extractelement <8 x bfloat> [[B:%.*]], i32 4
+// CHECK-LE-NEXT:    [[VSET_LANE:%.*]] = insertelement <4 x bfloat> [[A:%.*]], bfloat [[VGETQ_LANE]], i32 3
+// CHECK-LE-NEXT:    ret <4 x bfloat> [[VSET_LANE]]
+//
+// CHECK-BE-LABEL: @test_vcopy_laneq_bf16_v2(
+// CHECK-BE-NEXT:  entry:
+// CHECK-BE-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x bfloat> [[A:%.*]], <4 x bfloat> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 undef>
+// CHECK-BE-NEXT:    [[VGETQ_LANE:%.*]] = extractelement <8 x bfloat> [[B:%.*]], i32 3
+// CHECK-BE-NEXT:    [[VSET_LANE:%.*]] = insertelement <4 x bfloat> [[SHUFFLE]], bfloat [[VGETQ_LANE]], i32 3
+// CHECK-BE-NEXT:    [[SHUFFLE5:%.*]] = shufflevector <4 x bfloat> [[VSET_LANE]], <4 x bfloat> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT:    ret <4 x bfloat> [[SHUFFLE5]]
+//
+bfloat16x4_t test_vcopy_laneq_bf16_v2(bfloat16x4_t a, bfloat16x8_t b) {
+  return vcopy_laneq_bf16(a, 3, b, 4);
+}
+
+// CHECK-LE-LABEL: @test_vcopyq_laneq_bf16_v1(
+// CHECK-LE-NEXT:  entry:
+// CHECK-LE-NEXT:    [[VSET_LANE:%.*]] = shufflevector <8 x bfloat> [[A:%.*]], <8 x bfloat> [[B:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 15, i32 4, i32 5, i32 6, i32 7>
+// CHECK-LE-NEXT:    ret <8 x bfloat> [[VSET_LANE]]
+//
+// CHECK-BE-LABEL: @test_vcopyq_laneq_bf16_v1(
+// CHECK-BE-NEXT:  entry:
+// CHECK-BE-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x bfloat> [[A:%.*]], <8 x bfloat> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 undef, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT:    [[VSET_LANE:%.*]] = shufflevector <8 x bfloat> [[SHUFFLE]], <8 x bfloat> [[B:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 8, i32 4, i32 5, i32 6, i32 7>
+// CHECK-BE-NEXT:    [[SHUFFLE5:%.*]] = shufflevector <8 x bfloat> [[VSET_LANE]], <8 x bfloat> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT:    ret <8 x bfloat> [[SHUFFLE5]]
+//
+bfloat16x8_t test_vcopyq_laneq_bf16_v1(bfloat16x8_t a, bfloat16x8_t b) {
+  return vcopyq_laneq_bf16(a, 3, b, 7);
+
+}
+
+// CHECK-LE-LABEL: @test_vcopyq_laneq_bf16_v2(
+// CHECK-LE-NEXT:  entry:
+// CHECK-LE-NEXT:    [[VSET_LANE:%.*]] = shufflevector <8 x bfloat> [[A:%.*]], <8 x bfloat> [[B:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 10, i32 7>
+// CHECK-LE-NEXT:    ret <8 x bfloat> [[VSET_LANE]]
+//
+// CHECK-BE-LABEL: @test_vcopyq_laneq_bf16_v2(
+// CHECK-BE-NEXT:  entry:
+// CHECK-BE-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x bfloat> [[A:%.*]], <8 x bfloat> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 undef, i32 0>
+// CHECK-BE-NEXT:    [[VSET_LANE:%.*]] = shufflevector <8 x bfloat> [[SHUFFLE]], <8 x bfloat> [[B:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 13, i32 7>
+// CHECK-BE-NEXT:    [[SHUFFLE5:%.*]] = shufflevector <8 x bfloat> [[VSET_LANE]], <8 x bfloat> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT:    ret <8 x bfloat> [[SHUFFLE5]]
+//
+bfloat16x8_t test_vcopyq_laneq_bf16_v2(bfloat16x8_t a, bfloat16x8_t b) {
+  return vcopyq_laneq_bf16(a, 6, b, 2);
+}
+
diff --git a/clang/test/CodeGen/aarch64-neon-misc-constrained.c b/clang/test/CodeGen/aarch64-neon-misc-constrained.c
index 0385358291c9b..a0a503df33754 100644
--- a/clang/test/CodeGen/aarch64-neon-misc-constrained.c
+++ b/clang/test/CodeGen/aarch64-neon-misc-constrained.c
@@ -3,6 +3,7 @@
 // RUN: | opt -S -mem2reg | FileCheck --check-prefix=COMMON --check-prefix=COMMONIR --check-prefix=UNCONSTRAINED %s
 // RUN: %clang_cc1 -triple arm64-none-linux-gnu -target-feature +neon \
 // RUN:  -ffp-exception-behavior=strict \
+// RUN:  -fexperimental-strict-floating-point \
 // RUN:  -disable-O0-optnone -fallow-half-arguments-and-returns -emit-llvm -o - %s \
 // RUN: | opt -S -mem2reg | FileCheck --check-prefix=COMMON --check-prefix=COMMONIR --check-prefix=CONSTRAINED %s
 // RUN: %clang_cc1 -triple arm64-none-linux-gnu -target-feature +neon \
@@ -10,6 +11,7 @@
 // RUN: | FileCheck --check-prefix=COMMON --check-prefix=CHECK-ASM %s
 // RUN: %clang_cc1 -triple arm64-none-linux-gnu -target-feature +neon \
 // RUN:  -ffp-exception-behavior=strict \
+// RUN:  -fexperimental-strict-floating-point \
 // RUN:  -disable-O0-optnone -fallow-half-arguments-and-returns -S -o - %s \
 // RUN: | FileCheck --check-prefix=COMMON --check-prefix=CHECK-ASM %s
 
diff --git a/clang/test/CodeGen/aarch64-neon-scalar-x-indexed-elem-constrained.c b/clang/test/CodeGen/aarch64-neon-scalar-x-indexed-elem-constrained.c
index cbe5627337fd9..311950362f332 100644
--- a/clang/test/CodeGen/aarch64-neon-scalar-x-indexed-elem-constrained.c
+++ b/clang/test/CodeGen/aarch64-neon-scalar-x-indexed-elem-constrained.c
@@ -3,6 +3,7 @@
 // RUN: | FileCheck --check-prefix=COMMON --check-prefix=COMMONIR --check-prefix=UNCONSTRAINED %s
 // RUN: %clang_cc1 -triple arm64-none-linux-gnu -target-feature +neon -target-cpu cyclone \
 // RUN: -ffp-exception-behavior=strict \
+// RUN: -fexperimental-strict-floating-point \
 // RUN: -disable-O0-optnone -emit-llvm -o - %s | opt -S -mem2reg \
 // RUN: | FileCheck --check-prefix=COMMON --check-prefix=COMMONIR --check-prefix=CONSTRAINED %s
 // RUN: %clang_cc1 -triple arm64-none-linux-gnu -target-feature +neon -target-cpu cyclone \
@@ -10,6 +11,7 @@
 // RUN: | FileCheck --check-prefix=COMMON --check-prefix=CHECK-ASM %s
 // RUN: %clang_cc1 -triple arm64-none-linux-gnu -target-feature +neon -target-cpu cyclone \
 // RUN: -ffp-exception-behavior=strict \
+// RUN: -fexperimental-strict-floating-point \
 // RUN: -disable-O0-optnone -emit-llvm -o - %s | opt -S -mem2reg | llc -o=- - \
 // RUN: | FileCheck --check-prefix=COMMON --check-prefix=CHECK-ASM %s
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_abd.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_abd.c
index dcaa5aa260295..96a814565981e 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_abd.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_abd.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_abs.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_abs.c
index 2313b3c01a4d6..c880f25f49434 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_abs.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_abs.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_acge.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_acge.c
index c5c3071225454..d1752815bcee5 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_acge.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_acge.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_acgt.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_acgt.c
index 00e408ca4d941..0377105a1ff05 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_acgt.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_acgt.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_acle.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_acle.c
index 6ba53d0bb8c3a..f411cd2cf6279 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_acle.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_acle.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_aclt.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_aclt.c
index 781e6633f69f2..cbbd6ed947538 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_aclt.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_aclt.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_add.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_add.c
index 0956efdebb13a..68c223f872f0e 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_add.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_add.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adda.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adda.c
index 104d112cc046b..853da8783faa7 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adda.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adda.c
@@ -1,8 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: rm -f -- %S/acle_sve_adda.s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_addv.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_addv.c
index fc00f38196f62..eab8ba56411f7 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_addv.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_addv.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adrb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adrb.c
index 4a5820aa52ef9..b70c3c0aca412 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adrb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adrb.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adrd.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adrd.c
index bd72564492210..5dc5aad81937e 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adrd.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adrd.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adrh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adrh.c
index 959a9f7ae7e89..7b11e478c1771 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adrh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adrh.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adrw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adrw.c
index 6050e0502e15e..3d771a5fdf781 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adrw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_adrw.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_and.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_and.c
index 2db91824ec2aa..61cc267702107 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_and.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_and.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_andv.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_andv.c
index 4967a488e6bad..02e2069727483 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_andv.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_andv.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_asr.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_asr.c
index ca0a97fb89491..6d973583f6ccc 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_asr.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_asr.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_asrd.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_asrd.c
index b91784b08053f..8a6febba9a21b 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_asrd.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_asrd.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfdot.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfdot.c
new file mode 100644
index 0000000000000..2d07f3419d739
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfdot.c
@@ -0,0 +1,40 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svfloat32_t test_bfdot_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) {
+  // CHECK-LABEL: @test_bfdot_f32(
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.bfdot(<vscale x 4 x float> %x, <vscale x 8 x bfloat> %y, <vscale x 8 x bfloat> %z)
+  // CHECK: ret <vscale x 4 x float> %[[RET]]
+  return SVE_ACLE_FUNC(svbfdot, _f32, , )(x, y, z);
+}
+
+svfloat32_t test_bfdot_lane_0_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) {
+  // CHECK-LABEL: @test_bfdot_lane_0_f32(
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.bfdot.lane(<vscale x 4 x float> %x, <vscale x 8 x bfloat> %y, <vscale x 8 x bfloat> %z, i64 0)
+  // CHECK: ret <vscale x 4 x float> %[[RET]]
+  return SVE_ACLE_FUNC(svbfdot_lane, _f32, , )(x, y, z, 0);
+}
+
+svfloat32_t test_bfdot_lane_3_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) {
+  // CHECK-LABEL: @test_bfdot_lane_3_f32(
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.bfdot.lane(<vscale x 4 x float> %x, <vscale x 8 x bfloat> %y, <vscale x 8 x bfloat> %z, i64 3)
+  // CHECK: ret <vscale x 4 x float> %[[RET]]
+  return SVE_ACLE_FUNC(svbfdot_lane, _f32, , )(x, y, z, 3);
+}
+
+svfloat32_t test_bfdot_n_f32(svfloat32_t x, svbfloat16_t y, bfloat16_t z) {
+  // CHECK-LABEL: @test_bfdot_n_f32(
+  // CHECK: %[[SPLAT:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.dup.x.nxv8bf16(bfloat %z)
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.bfdot(<vscale x 4 x float> %x, <vscale x 8 x bfloat> %y, <vscale x 8 x bfloat> %[[SPLAT]])
+  // CHECK: ret <vscale x 4 x float> %[[RET]]
+  return SVE_ACLE_FUNC(svbfdot, _n_f32, , )(x, y, z);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalb.c
new file mode 100644
index 0000000000000..dd88b486e57e1
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalb.c
@@ -0,0 +1,40 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16  -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svfloat32_t test_svbfmlalb_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) {
+  // CHECK-LABEL: @test_svbfmlalb_f32(
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalb(<vscale x 4 x float> %x, <vscale x 8 x bfloat> %y, <vscale x 8 x bfloat> %z)
+  // CHECK: ret <vscale x 4 x float> %[[RET]]
+  return SVE_ACLE_FUNC(svbfmlalb, _f32, , )(x, y, z);
+}
+
+svfloat32_t test_bfmlalb_lane_0_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) {
+  // CHECK-LABEL: @test_bfmlalb_lane_0_f32(
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalb.lane(<vscale x 4 x float> %x, <vscale x 8 x bfloat> %y, <vscale x 8 x bfloat> %z, i64 0)
+  // CHECK: ret <vscale x 4 x float> %[[RET]]
+  return SVE_ACLE_FUNC(svbfmlalb_lane, _f32, , )(x, y, z, 0);
+}
+
+svfloat32_t test_bfmlalb_lane_7_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) {
+  // CHECK-LABEL: @test_bfmlalb_lane_7_f32(
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalb.lane(<vscale x 4 x float> %x, <vscale x 8 x bfloat> %y, <vscale x 8 x bfloat> %z, i64 7)
+  // CHECK: ret <vscale x 4 x float> %[[RET]]
+  return SVE_ACLE_FUNC(svbfmlalb_lane, _f32, , )(x, y, z, 7);
+}
+
+svfloat32_t test_bfmlalb_n_f32(svfloat32_t x, svbfloat16_t y, bfloat16_t z) {
+  // CHECK-LABEL: @test_bfmlalb_n_f32(
+  // CHECK: %[[SPLAT:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.dup.x.nxv8bf16(bfloat %z)
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalb(<vscale x 4 x float> %x, <vscale x 8 x bfloat> %y, <vscale x 8 x bfloat> %[[SPLAT]])
+  // CHECK: ret <vscale x 4 x float> %[[RET]]
+  return SVE_ACLE_FUNC(svbfmlalb, _n_f32, , )(x, y, z);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalt.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalt.c
new file mode 100644
index 0000000000000..f68976458d56b
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmlalt.c
@@ -0,0 +1,40 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svfloat32_t test_svbfmlalt_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) {
+  // CHECK-LABEL: @test_svbfmlalt_f32(
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt(<vscale x 4 x float> %x, <vscale x 8 x bfloat> %y, <vscale x 8 x bfloat> %z)
+  // CHECK: ret <vscale x 4 x float> %[[RET]]
+  return SVE_ACLE_FUNC(svbfmlalt, _f32, , )(x, y, z);
+}
+
+svfloat32_t test_bfmlalt_lane_0_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) {
+  // CHECK-LABEL: @test_bfmlalt_lane_0_f32(
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt.lane(<vscale x 4 x float> %x, <vscale x 8 x bfloat> %y, <vscale x 8 x bfloat> %z, i64 0)
+  // CHECK: ret <vscale x 4 x float> %[[RET]]
+  return SVE_ACLE_FUNC(svbfmlalt_lane, _f32, , )(x, y, z, 0);
+}
+
+svfloat32_t test_bfmlalt_lane_7_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) {
+  // CHECK-LABEL: @test_bfmlalt_lane_7_f32(
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt.lane(<vscale x 4 x float> %x, <vscale x 8 x bfloat> %y, <vscale x 8 x bfloat> %z, i64 7)
+  // CHECK: ret <vscale x 4 x float> %[[RET]]
+  return SVE_ACLE_FUNC(svbfmlalt_lane, _f32, , )(x, y, z, 7);
+}
+
+svfloat32_t test_bfmlalt_n_f32(svfloat32_t x, svbfloat16_t y, bfloat16_t z) {
+  // CHECK-LABEL: @test_bfmlalt_n_f32(
+  // CHECK: %[[SPLAT:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.dup.x.nxv8bf16(bfloat %z)
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt(<vscale x 4 x float> %x, <vscale x 8 x bfloat> %y, <vscale x 8 x bfloat> %[[SPLAT]])
+  // CHECK: ret <vscale x 4 x float> %[[RET]]
+  return SVE_ACLE_FUNC(svbfmlalt, _n_f32, , )(x, y, z);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmmla.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmmla.c
new file mode 100644
index 0000000000000..e38baea73e709
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bfmmla.c
@@ -0,0 +1,18 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svfloat32_t test_bfmmla_f32(svfloat32_t x, svbfloat16_t y, svbfloat16_t z) {
+  // CHECK-LABEL: @test_bfmmla_f32(
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.bfmmla(<vscale x 4 x float> %x, <vscale x 8 x bfloat> %y, <vscale x 8 x bfloat> %z)
+  // CHECK: ret <vscale x 4 x float> %[[RET]]
+  return SVE_ACLE_FUNC(svbfmmla, _f32, , )(x, y, z);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bic.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bic.c
index d062d367fb601..7d5bb90ec41c4 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bic.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_bic.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brka.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brka.c
index b6bb5fe2e8ee8..71728997029a0 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brka.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brka.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brkb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brkb.c
index 07dd782f71fde..e950a6db629e8 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brkb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brkb.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brkn.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brkn.c
index 539c34aa6e421..c84360962b023 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brkn.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brkn.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brkpa.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brkpa.c
index bbcdead0a5614..b9b7d70f8f6d9 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brkpa.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brkpa.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brkpb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brkpb.c
index ca6b543135694..7723cf264c7bc 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brkpb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_brkpb.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cadd.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cadd.c
index 889f4af15e7a5..8f318f93a91a7 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cadd.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cadd.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clasta-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clasta-bfloat.c
new file mode 100644
index 0000000000000..3e66d7d98b076
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clasta-bfloat.c
@@ -0,0 +1,36 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svclasta_bf16(svbool_t pg, svbfloat16_t fallback, svbfloat16_t data) {
+  // CHECK-LABEL: test_svclasta_bf16
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.clasta.nxv8bf16(<vscale x 8 x i1> %[[PG]], <vscale x 8 x bfloat> %fallback, <vscale x 8 x bfloat> %data)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svclasta_bf16'}}
+  return SVE_ACLE_FUNC(svclasta, _bf16, , )(pg, fallback, data);
+}
+
+bfloat16_t test_svclasta_n_bf16(svbool_t pg, bfloat16_t fallback, svbfloat16_t data) {
+  // CHECK-LABEL: test_svclasta_n_bf16
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call bfloat @llvm.aarch64.sve.clasta.n.nxv8bf16(<vscale x 8 x i1> %[[PG]], bfloat %fallback, <vscale x 8 x bfloat> %data)
+  // CHECK: ret bfloat %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svclasta_n_bf16'}}
+  return SVE_ACLE_FUNC(svclasta, _n_bf16, , )(pg, fallback, data);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clasta.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clasta.c
index 0f49a4447fe78..5c869a02f3715 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clasta.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clasta.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clastb-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clastb-bfloat.c
new file mode 100644
index 0000000000000..0182f2e6775e7
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clastb-bfloat.c
@@ -0,0 +1,36 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svclastb_bf16(svbool_t pg, svbfloat16_t fallback, svbfloat16_t data) {
+  // CHECK-LABEL: test_svclastb_bf16
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.clastb.nxv8bf16(<vscale x 8 x i1> %[[PG]], <vscale x 8 x bfloat> %fallback, <vscale x 8 x bfloat> %data)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svclastb_bf16'}}
+  return SVE_ACLE_FUNC(svclastb, _bf16, , )(pg, fallback, data);
+}
+
+bfloat16_t test_svclastb_n_bf16(svbool_t pg, bfloat16_t fallback, svbfloat16_t data) {
+  // CHECK-LABEL: test_svclastb_n_bf16
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call bfloat @llvm.aarch64.sve.clastb.n.nxv8bf16(<vscale x 8 x i1> %[[PG]], bfloat %fallback, <vscale x 8 x bfloat> %data)
+  // CHECK: ret bfloat %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svclastb_n_bf16'}}
+  return SVE_ACLE_FUNC(svclastb, _n_bf16, , )(pg, fallback, data);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clastb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clastb.c
index 4ba3fbb1ff3a7..63bfc7c4c9ec9 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clastb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clastb.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cls.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cls.c
index 4538a9e7c57cd..92255b24cd515 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cls.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cls.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clz.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clz.c
index 864711ef34552..72e55ce8855d1 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clz.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_clz.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmla.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmla.c
index a8207acb9b3a5..2c8062daeb8f6 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmla.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmla.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpeq.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpeq.c
index b76bd8ee1bebf..269801c71fdc8 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpeq.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpeq.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpge.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpge.c
index a5fed360db3db..56c761c773180 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpge.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpge.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpgt.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpgt.c
index 76c0a63c2863a..f49520b3e360b 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpgt.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpgt.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmple.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmple.c
index a6cf374346579..cb93f1ece5fb2 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmple.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmple.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmplt.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmplt.c
index a5ed353609ee3..b19d515559569 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmplt.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmplt.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpne.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpne.c
index c98c40a120bc4..272af486f8950 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpne.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpne.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpuo.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpuo.c
index 1130d9812647b..bc9d506b5033a 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpuo.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cmpuo.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnot.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnot.c
index 3597cec12127d..1ef1ba8a7b28f 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnot.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnot.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnt-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnt-bfloat.c
new file mode 100644
index 0000000000000..e3e07a1ef18e4
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnt-bfloat.c
@@ -0,0 +1,44 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svuint16_t test_svcnt_bf16_z(svbool_t pg, svbfloat16_t op) {
+  // CHECK-LABEL: test_svcnt_bf16_z
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.cnt.nxv8bf16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> %[[PG]], <vscale x 8 x bfloat> %op)
+  // CHECK: ret <vscale x 8 x i16> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svcnt_bf16_z'}}
+  return SVE_ACLE_FUNC(svcnt, _bf16, _z, )(pg, op);
+}
+
+svuint16_t test_svcnt_bf16_m(svuint16_t inactive, svbool_t pg, svbfloat16_t op) {
+  // CHECK-LABEL: test_svcnt_bf16_m
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.cnt.nxv8bf16(<vscale x 8 x i16> %inactive, <vscale x 8 x i1> %[[PG]], <vscale x 8 x bfloat> %op)
+  // CHECK: ret <vscale x 8 x i16> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svcnt_bf16_m'}}
+  return SVE_ACLE_FUNC(svcnt, _bf16, _m, )(inactive, pg, op);
+}
+svuint16_t test_svcnt_bf16_x(svbool_t pg, svbfloat16_t op) {
+  // CHECK-LABEL: test_svcnt_bf16_x
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.cnt.nxv8bf16(<vscale x 8 x i16> undef, <vscale x 8 x i1> %[[PG]], <vscale x 8 x bfloat> %op)
+  // CHECK: ret <vscale x 8 x i16> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svcnt_bf16_x'}}
+  return SVE_ACLE_FUNC(svcnt, _bf16, _x, )(pg, op);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnt.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnt.c
index c3ab491a0dee8..43c03cc46ef6a 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnt.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnt.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntb.c
index 4e3febe41845d..f06ed67ecb2e3 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntb.c
@@ -1,6 +1,6 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntd.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntd.c
index aa6a024b9b1c8..1fe544e99c193 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntd.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntd.c
@@ -1,6 +1,6 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnth.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnth.c
index 8c2481f641192..0bd8b5b566c6d 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnth.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnth.c
@@ -1,6 +1,6 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntp.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntp.c
index 448ec9ba76852..cb9f2c7c6816a 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntp.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntp.c
@@ -1,6 +1,6 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntw.c
index 05bf30abb316c..40b00eb1bf25d 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntw.c
@@ -1,6 +1,6 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_compact.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_compact.c
index 2ebc928c0beb9..67ac08921a257 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_compact.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_compact.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create2-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create2-bfloat.c
new file mode 100644
index 0000000000000..52d92d04d6982
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create2-bfloat.c
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16x2_t test_svcreate2_bf16(svbfloat16_t x0, svbfloat16_t x1)
+{
+  // CHECK-LABEL: test_svcreate2_bf16
+  // CHECK: %[[CREATE:.*]] = call <vscale x 16 x bfloat> @llvm.aarch64.sve.tuple.create2.nxv16bf16.nxv8bf16(<vscale x 8 x bfloat> %x0, <vscale x 8 x bfloat> %x1)
+  // CHECK-NEXT: ret <vscale x 16 x bfloat> %[[CREATE]]
+  // expected-warning@+1 {{implicit declaration of function 'svcreate2_bf16'}}
+  return SVE_ACLE_FUNC(svcreate2,_bf16,,)(x0, x1);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create2.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create2.c
index 5f5de0a473e77..614f39c2bbfe9 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create2.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create2.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create3-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create3-bfloat.c
new file mode 100644
index 0000000000000..28e14972f0981
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create3-bfloat.c
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16x3_t test_svcreate3_bf16(svbfloat16_t x0, svbfloat16_t x1, svbfloat16_t x2)
+{
+  // CHECK-LABEL: test_svcreate3_bf16
+  // CHECK: %[[CREATE:.*]] = call <vscale x 24 x bfloat> @llvm.aarch64.sve.tuple.create3.nxv24bf16.nxv8bf16(<vscale x 8 x bfloat> %x0, <vscale x 8 x bfloat> %x1, <vscale x 8 x bfloat> %x2)
+  // CHECK-NEXT: ret <vscale x 24 x bfloat> %[[CREATE]]
+  // expected-warning@+1 {{implicit declaration of function 'svcreate3_bf16'}}
+  return SVE_ACLE_FUNC(svcreate3,_bf16,,)(x0, x1, x2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create3.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create3.c
index b08ce21f199c1..4c792b0b9e357 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create3.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create3.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create4-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create4-bfloat.c
new file mode 100644
index 0000000000000..36bdf7c05fca0
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create4-bfloat.c
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16x4_t test_svcreate4_bf16(svbfloat16_t x0, svbfloat16_t x1, svbfloat16_t x2, svbfloat16_t x4)
+{
+  // CHECK-LABEL: test_svcreate4_bf16
+  // CHECK: %[[CREATE:.*]] = call <vscale x 32 x bfloat> @llvm.aarch64.sve.tuple.create4.nxv32bf16.nxv8bf16(<vscale x 8 x bfloat> %x0, <vscale x 8 x bfloat> %x1, <vscale x 8 x bfloat> %x2, <vscale x 8 x bfloat> %x4)
+  // CHECK-NEXT: ret <vscale x 32 x bfloat> %[[CREATE]]
+  // expected-warning@+1 {{implicit declaration of function 'svcreate4_bf16'}}
+  return SVE_ACLE_FUNC(svcreate4,_bf16,,)(x0, x1, x2, x4);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create4.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create4.c
index ff05e4ac99467..6f93c4c44184d 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create4.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_create4.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt-bfloat.c
new file mode 100644
index 0000000000000..dea8e20dc267c
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt-bfloat.c
@@ -0,0 +1,35 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svcvt_bf16_f32_x(svbool_t pg, svfloat32_t op) {
+  // CHECK-LABEL: test_svcvt_bf16_f32_x
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> %[[PG]], <vscale x 4 x float> %op)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  return SVE_ACLE_FUNC(svcvt_bf16, _f32, _x, )(pg, op);
+}
+
+svbfloat16_t test_svcvt_bf16_f32_z(svbool_t pg, svfloat32_t op) {
+  // CHECK-LABEL: test_svcvt_bf16_f32_z
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x i1> %[[PG]], <vscale x 4 x float> %op)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  return SVE_ACLE_FUNC(svcvt_bf16, _f32, _z, )(pg, op);
+}
+
+svbfloat16_t test_svcvt_bf16_f32_m(svbfloat16_t inactive, svbool_t pg, svfloat32_t op) {
+  // CHECK-LABEL: test_svcvt_bf16_f32_m
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> %inactive, <vscale x 8 x i1> %[[PG]], <vscale x 4 x float> %op)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  return SVE_ACLE_FUNC(svcvt_bf16, _f32, _m, )(inactive, pg, op);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt.c
index f6c01cd6bfe66..58ced24326b0e 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvtnt.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvtnt.c
new file mode 100644
index 0000000000000..b5828227cac46
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvtnt.c
@@ -0,0 +1,27 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svcvtnt_bf16_f32_x(svbfloat16_t even, svbool_t pg, svfloat32_t op) {
+  // CHECK-LABEL: test_svcvtnt_bf16_f32_x
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32(<vscale x 8 x bfloat> %even, <vscale x 8 x i1> %[[PG]], <vscale x 4 x float> %op)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  return SVE_ACLE_FUNC(svcvtnt_bf16, _f32, _x, )(even, pg, op);
+}
+
+svbfloat16_t test_svcvtnt_bf16_f32_m(svbfloat16_t even, svbool_t pg, svfloat32_t op) {
+  // CHECK-LABEL: test_svcvtnt_bf16_f32_m
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32(<vscale x 8 x bfloat> %even, <vscale x 8 x i1> %[[PG]], <vscale x 4 x float> %op)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  return SVE_ACLE_FUNC(svcvtnt_bf16, _f32, _m, )(even, pg, op);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_div.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_div.c
index 0d8c1156a709b..e388e5c1271ca 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_div.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_div.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_divr.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_divr.c
index 2239441295ffe..462099ba6398f 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_divr.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_divr.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dot.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dot.c
index 7c5ee85afad30..2533662be82b2 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dot.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dot.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dup-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dup-bfloat.c
new file mode 100644
index 0000000000000..7ef02c3330e97
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dup-bfloat.c
@@ -0,0 +1,63 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svdup_n_bf16(bfloat16_t op) {
+  // CHECK-LABEL: test_svdup_n_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.dup.x.nxv8bf16(bfloat %op)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svdup_n_bf16'}}
+  return SVE_ACLE_FUNC(svdup, _n, _bf16, )(op);
+}
+
+svbfloat16_t test_svdup_n_bf16_z(svbool_t pg, bfloat16_t op) {
+  // CHECK-LABEL: test_svdup_n_bf16_z
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.dup.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x i1> %[[PG]], bfloat %op)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svdup_n_bf16_z'}}
+  return SVE_ACLE_FUNC(svdup, _n, _bf16_z, )(pg, op);
+}
+
+svbfloat16_t test_svdup_n_bf16_m(svbfloat16_t inactive, svbool_t pg, bfloat16_t op) {
+  // CHECK-LABEL: test_svdup_n_bf16_m
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.dup.nxv8bf16(<vscale x 8 x bfloat> %inactive, <vscale x 8 x i1> %[[PG]], bfloat %op)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svdup_n_bf16_m'}}
+  return SVE_ACLE_FUNC(svdup, _n, _bf16_m, )(inactive, pg, op);
+}
+
+svbfloat16_t test_svdup_n_bf16_x(svbool_t pg, bfloat16_t op) {
+  // CHECK-LABEL: test_svdup_n_bf16_x
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.dup.nxv8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> %[[PG]], bfloat %op)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svdup_n_bf16_x'}}
+  return SVE_ACLE_FUNC(svdup, _n, _bf16_x, )(pg, op);
+}
+
+svbfloat16_t test_svdup_lane_bf16(svbfloat16_t data, uint16_t index)
+{
+  // CHECK-LABEL: test_svdup_lane_bf16
+  // CHECK: %[[DUP:.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.dup.x.nxv8i16(i16 %index)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tbl.nxv8bf16(<vscale x 8 x bfloat> %data, <vscale x 8 x i16> %[[DUP]])
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svdup_lane_bf16'}}
+  return SVE_ACLE_FUNC(svdup_lane,_bf16,,)(data, index);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dup.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dup.c
index 9fb8cc6fabd19..5475d55564a63 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dup.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dup.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dupq-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dupq-bfloat.c
new file mode 100644
index 0000000000000..66694789162c7
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dupq-bfloat.c
@@ -0,0 +1,42 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svdupq_lane_bf16(svbfloat16_t data, uint64_t index) {
+  // CHECK-LABEL: test_svdupq_lane_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.dupq.lane.nxv8bf16(<vscale x 8 x bfloat> %data, i64 %index)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svdupq_lane_bf16'}}
+  return SVE_ACLE_FUNC(svdupq_lane, _bf16, , )(data, index);
+}
+svbfloat16_t test_svdupq_n_bf16(bfloat16_t x0, bfloat16_t x1, bfloat16_t x2, bfloat16_t x3,
+                                bfloat16_t x4, bfloat16_t x5, bfloat16_t x6, bfloat16_t x7) {
+  // CHECK-LABEL: test_svdupq_n_bf16
+  // CHECK: %[[ALLOCA:.*]] = alloca [8 x bfloat], align 16
+  // CHECK-DAG: %[[BASE:.*]] = getelementptr inbounds [8 x bfloat], [8 x bfloat]* %[[ALLOCA]], i64 0, i64 0
+  // CHECK-DAG: store bfloat %x0, bfloat* %[[BASE]], align 16
+  // <assume other stores>
+  // CHECK-DAG: %[[GEP:.*]] = getelementptr inbounds [8 x bfloat], [8 x bfloat]* %[[ALLOCA]], i64 0, i64 7
+  // CHECK: store bfloat %x7, bfloat* %[[GEP]], align 2
+  // CHECK-NOT: store
+  // CHECK: call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  // CHECK: %[[LOAD:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1rq.nxv8bf16(<vscale x 8 x i1> %{{.*}}, bfloat* nonnull %[[BASE]])
+  // CHECK: ret <vscale x 8 x bfloat> %[[LOAD]]
+  // expected-warning@+1 {{implicit declaration of function 'svdupq_n_bf16'}}
+  return SVE_ACLE_FUNC(svdupq, _n, _bf16, )(x0, x1, x2, x3, x4, x5, x6, x7);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dupq.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dupq.c
index 7393ef705e523..d604d13356e53 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dupq.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_dupq.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_eor.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_eor.c
index 88e1a1b38d092..bcc0333c67585 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_eor.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_eor.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_eorv.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_eorv.c
index 3398c6611f40c..69db72c2c2ece 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_eorv.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_eorv.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_expa.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_expa.c
index dfe56d5eadf10..96fbb9c705655 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_expa.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_expa.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext-bfloat.c
new file mode 100644
index 0000000000000..6ed1d36cf30ea
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext-bfloat.c
@@ -0,0 +1,26 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1  -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svext_bf16(svbfloat16_t op1, svbfloat16_t op2)
+{
+  // CHECK-LABEL: test_svext_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ext.nxv8bf16(<vscale x 8 x bfloat> %op1, <vscale x 8 x bfloat> %op2, i32 127)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svext_bf16'}}
+  return SVE_ACLE_FUNC(svext,_bf16,,)(op1, op2, 127);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext.c
index 7091e351fa817..7b970637eed08 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ext.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_extb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_extb.c
index 2e9ce0006d840..4e07f241c799a 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_extb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_extb.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_exth.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_exth.c
index 28f5084892acf..074feed23e83a 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_exth.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_exth.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_extw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_extw.c
index 043f9ed4f468b..ec07bb59cf1d9 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_extw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_extw.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get2-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get2-bfloat.c
new file mode 100644
index 0000000000000..cd585cc867e4d
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get2-bfloat.c
@@ -0,0 +1,30 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svget2_bf16_0(svbfloat16x2_t tuple)
+{
+  // CHECK-LABEL: test_svget2_bf16_0
+  // CHECK: %[[EXT:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> %tuple, i32 0)
+  // CHECK-NEXT: ret <vscale x 8 x bfloat> %[[EXT]]
+  // expected-warning@+1 {{implicit declaration of function 'svget2_bf16'}}
+  return SVE_ACLE_FUNC(svget2,_bf16,,)(tuple, 0);
+}
+
+svbfloat16_t test_svget2_bf16_1(svbfloat16x2_t tuple)
+{
+  // CHECK-LABEL: test_svget2_bf16_1
+  // CHECK: %[[EXT:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> %tuple, i32 1)
+  // CHECK-NEXT: ret <vscale x 8 x bfloat> %[[EXT]]
+  // expected-warning@+1 {{implicit declaration of function 'svget2_bf16'}}
+  return SVE_ACLE_FUNC(svget2,_bf16,,)(tuple, 1);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get2.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get2.c
index b05612bb7c607..7beb191cab30f 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get2.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get2.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get3-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get3-bfloat.c
new file mode 100644
index 0000000000000..ae9bc08eac113
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get3-bfloat.c
@@ -0,0 +1,39 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svget3_bf16_0(svbfloat16x3_t tuple)
+{
+  // CHECK-LABEL: test_svget3_bf16_0
+  // CHECK: %[[EXT:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv24bf16(<vscale x 24 x bfloat> %tuple, i32 0)
+  // CHECK-NEXT: ret <vscale x 8 x bfloat> %[[EXT]]
+  // expected-warning@+1 {{implicit declaration of function 'svget3_bf16'}}
+  return SVE_ACLE_FUNC(svget3,_bf16,,)(tuple, 0);
+}
+
+svbfloat16_t test_svget3_bf16_1(svbfloat16x3_t tuple)
+{
+  // CHECK-LABEL: test_svget3_bf16_1
+  // CHECK: %[[EXT:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv24bf16(<vscale x 24 x bfloat> %tuple, i32 1)
+  // CHECK-NEXT: ret <vscale x 8 x bfloat> %[[EXT]]
+  // expected-warning@+1 {{implicit declaration of function 'svget3_bf16'}}
+  return SVE_ACLE_FUNC(svget3,_bf16,,)(tuple, 1);
+}
+
+svbfloat16_t test_svget3_bf16_2(svbfloat16x3_t tuple)
+{
+  // CHECK-LABEL: test_svget3_bf16_2
+  // CHECK: %[[EXT:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv24bf16(<vscale x 24 x bfloat> %tuple, i32 2)
+  // CHECK-NEXT: ret <vscale x 8 x bfloat> %[[EXT]]
+  // expected-warning@+1 {{implicit declaration of function 'svget3_bf16'}}
+  return SVE_ACLE_FUNC(svget3,_bf16,,)(tuple, 2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get3.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get3.c
index 4b788919055f8..63e17c3e1e0fd 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get3.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get3.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get4-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get4-bfloat.c
new file mode 100644
index 0000000000000..4d5eb50e1b2ad
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get4-bfloat.c
@@ -0,0 +1,48 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svget4_bf16_0(svbfloat16x4_t tuple)
+{
+  // CHECK-LABEL: test_svget4_bf16_0
+  // CHECK: %[[EXT:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> %tuple, i32 0)
+  // CHECK-NEXT: ret <vscale x 8 x bfloat> %[[EXT]]
+  // expected-warning@+1 {{implicit declaration of function 'svget4_bf16'}}
+  return SVE_ACLE_FUNC(svget4,_bf16,,)(tuple, 0);
+}
+
+svbfloat16_t test_svget4_bf16_1(svbfloat16x4_t tuple)
+{
+  // CHECK-LABEL: test_svget4_bf16_1
+  // CHECK: %[[EXT:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> %tuple, i32 1)
+  // CHECK-NEXT: ret <vscale x 8 x bfloat> %[[EXT]]
+  // expected-warning@+1 {{implicit declaration of function 'svget4_bf16'}}
+  return SVE_ACLE_FUNC(svget4,_bf16,,)(tuple, 1);
+}
+
+svbfloat16_t test_svget4_bf16_2(svbfloat16x4_t tuple)
+{
+  // CHECK-LABEL: test_svget4_bf16_2
+  // CHECK: %[[EXT:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> %tuple, i32 2)
+  // CHECK-NEXT: ret <vscale x 8 x bfloat> %[[EXT]]
+  // expected-warning@+1 {{implicit declaration of function 'svget4_bf16'}}
+  return SVE_ACLE_FUNC(svget4,_bf16,,)(tuple, 2);
+}
+
+svbfloat16_t test_svget4_bf16_3(svbfloat16x4_t tuple)
+{
+  // CHECK-LABEL: test_svget4_bf16_3
+  // CHECK: %[[EXT:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> %tuple, i32 3)
+  // CHECK-NEXT: ret <vscale x 8 x bfloat> %[[EXT]]
+  // expected-warning@+1 {{implicit declaration of function 'svget4_bf16'}}
+  return SVE_ACLE_FUNC(svget4,_bf16,,)(tuple, 3);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get4.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get4.c
index c4ce575eac424..a34f41ff3b40a 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get4.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_get4.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_index.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_index.c
index d7241d1c014d2..f4d8478ec83ea 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_index.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_index.c
@@ -1,5 +1,10 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 svint8_t test_svindex_s8(int8_t base, int8_t step)
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_insr-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_insr-bfloat.c
new file mode 100644
index 0000000000000..9d1b4672d587d
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_insr-bfloat.c
@@ -0,0 +1,26 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svinsr_n_bf16(svbfloat16_t op1, bfloat16_t op2) {
+  // CHECK-LABEL: test_svinsr_n_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.insr.nxv8bf16(<vscale x 8 x bfloat> %op1, bfloat %op2)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svinsr_n_bf16'}}
+  return SVE_ACLE_FUNC(svinsr, _n_bf16, , )(op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_insr.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_insr.c
index ffa420b1ac5df..024b3a67d1c7a 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_insr.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_insr.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lasta-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lasta-bfloat.c
new file mode 100644
index 0000000000000..d5cc626b9733d
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lasta-bfloat.c
@@ -0,0 +1,27 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+bfloat16_t test_svlasta_bf16(svbool_t pg, svbfloat16_t op) {
+  // CHECK-LABEL: test_svlasta_bf16
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call bfloat @llvm.aarch64.sve.lasta.nxv8bf16(<vscale x 8 x i1> %[[PG]], <vscale x 8 x bfloat> %op)
+  // CHECK: ret bfloat %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svlasta_bf16'}}
+  return SVE_ACLE_FUNC(svlasta, _bf16, , )(pg, op);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lasta.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lasta.c
index 804b12a543b4c..fac57f7cde849 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lasta.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lasta.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lastb-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lastb-bfloat.c
new file mode 100644
index 0000000000000..57f0e79424314
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lastb-bfloat.c
@@ -0,0 +1,27 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+bfloat16_t test_svlastb_bf16(svbool_t pg, svbfloat16_t op) {
+  // CHECK-LABEL: test_svlastb_bf16
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call bfloat @llvm.aarch64.sve.lastb.nxv8bf16(<vscale x 8 x i1> %[[PG]], <vscale x 8 x bfloat> %op)
+  // CHECK: ret bfloat %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svlastb_bf16'}}
+  return SVE_ACLE_FUNC(svlastb, _bf16, , )(pg, op);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lastb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lastb.c
index 9cb60c49bacbd..cc16223f7de1b 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lastb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lastb.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1-bfloat.c
new file mode 100644
index 0000000000000..e85eced4b5a6b
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1-bfloat.c
@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svld1_bf16(svbool_t pg, const bfloat16_t *base)
+{
+  // CHECK-LABEL: test_svld1_bf16
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[LOAD:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1.nxv8bf16(<vscale x 8 x i1> %[[PG]], bfloat* %base)
+  // CHECK: ret <vscale x 8 x bfloat> %[[LOAD]]
+  // expected-warning@+1 {{implicit declaration of function 'svld1_bf16'}}
+  return SVE_ACLE_FUNC(svld1,_bf16,,)(pg, base);
+}
+
+svbfloat16_t test_svld1_vnum_bf16(svbool_t pg, const bfloat16_t *base, int64_t vnum)
+{
+  // CHECK-LABEL: test_svld1_vnum_bf16
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK-DAG: %[[BITCAST:.*]] = bitcast bfloat* %base to <vscale x 8 x bfloat>*
+  // CHECK-DAG: %[[GEP:.*]] = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %[[BITCAST]], i64 %vnum, i64 0
+  // CHECK: %[[LOAD:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1.nxv8bf16(<vscale x 8 x i1> %[[PG]], bfloat* %[[GEP]])
+  // CHECK: ret <vscale x 8 x bfloat> %[[LOAD]]
+  // expected-warning@+1 {{implicit declaration of function 'svld1_vnum_bf16'}}
+  return SVE_ACLE_FUNC(svld1_vnum,_bf16,,)(pg, base, vnum);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1.c
index 690ae89c6459e..7b6d2f38b81dc 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1ro-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1ro-bfloat.c
index 56137bd6c8a89..d9639a2c75b08 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1ro-bfloat.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1ro-bfloat.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -target-feature +f64mm -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -target-feature +f64mm -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1ro.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1ro.c
index 10608f2568320..bb4711943ad1e 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1ro.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1ro.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +f64mm -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +f64mm -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1rq-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1rq-bfloat.c
new file mode 100644
index 0000000000000..bd5fdc6da7a97
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1rq-bfloat.c
@@ -0,0 +1,22 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svld1rq_bf16(svbool_t pg, const bfloat16_t *base)
+{
+  // CHECK-LABEL: test_svld1rq_bf16
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1rq.nxv8bf16(<vscale x 8 x i1> %[[PG]], bfloat* %base)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svld1rq_bf16'}}
+  return SVE_ACLE_FUNC(svld1rq,_bf16,,)(pg, base);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1rq.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1rq.c
index 2e143a5cf3c8e..51b703a041ab3 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1rq.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1rq.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sb.c
index 2bc64c714f0ed..7a108331c9404 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sb.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sh.c
index 28d7ff448c90b..6475b19ab6533 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sh.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sw.c
index 110af5122c78b..3f4db7aec2440 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sw.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1ub.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1ub.c
index b0196a7066f0b..e3209fa8d9247 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1ub.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1ub.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1uh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1uh.c
index 9f371e795cf28..9219b687bd2cd 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1uh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1uh.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1uw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1uw.c
index 28a0c94fa4a0e..1c2b48becbb93 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1uw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1uw.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld2-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld2-bfloat.c
new file mode 100644
index 0000000000000..171e320883223
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld2-bfloat.c
@@ -0,0 +1,32 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16x2_t test_svld2_bf16(svbool_t pg, const bfloat16_t *base)
+{
+  // CHECK-LABEL: test_svld2_bf16
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[LOAD:.*]] = call <vscale x 16 x bfloat> @llvm.aarch64.sve.ld2.nxv16bf16.nxv8i1(<vscale x 8 x i1> %[[PG]], bfloat* %base)
+  // CHECK-NEXT: ret <vscale x 16 x bfloat> %[[LOAD]]
+  return SVE_ACLE_FUNC(svld2,_bf16,,)(pg, base);
+}
+
+
+svbfloat16x2_t test_svld2_vnum_bf16(svbool_t pg, const bfloat16_t *base, int64_t vnum)
+{
+  // CHECK-LABEL: test_svld2_vnum_bf16
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK-DAG: %[[BASE:.*]] = bitcast bfloat* %base to <vscale x 8 x bfloat>*
+  // CHECK-DAG: %[[GEP:.*]] = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %[[BASE]], i64 %vnum, i64 0
+  // CHECK: %[[LOAD:.*]] = call <vscale x 16 x bfloat> @llvm.aarch64.sve.ld2.nxv16bf16.nxv8i1(<vscale x 8 x i1> %[[PG]], bfloat* %[[GEP]])
+  // CHECK-NEXT: ret <vscale x 16 x bfloat> %[[LOAD]]
+  return SVE_ACLE_FUNC(svld2_vnum,_bf16,,)(pg, base, vnum);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld2.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld2.c
index 96c298077cb65..e232366cf744d 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld2.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld2.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld3-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld3-bfloat.c
new file mode 100644
index 0000000000000..94446e1041388
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld3-bfloat.c
@@ -0,0 +1,31 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16x3_t test_svld3_bf16(svbool_t pg, const bfloat16_t *base)
+{
+  // CHECK-LABEL: test_svld3_bf16
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[LOAD:.*]] = call <vscale x 24 x bfloat> @llvm.aarch64.sve.ld3.nxv24bf16.nxv8i1(<vscale x 8 x i1> %[[PG]], bfloat* %base)
+  // CHECK-NEXT: ret <vscale x 24 x bfloat> %[[LOAD]]
+  return SVE_ACLE_FUNC(svld3,_bf16,,)(pg, base);
+}
+
+svbfloat16x3_t test_svld3_vnum_bf16(svbool_t pg, const bfloat16_t *base, int64_t vnum)
+{
+  // CHECK-LABEL: test_svld3_vnum_bf16
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK-DAG: %[[BASE:.*]] = bitcast bfloat* %base to <vscale x 8 x bfloat>*
+  // CHECK-DAG: %[[GEP:.*]] = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %[[BASE]], i64 %vnum, i64 0
+  // CHECK: %[[LOAD:.*]] = call <vscale x 24 x bfloat> @llvm.aarch64.sve.ld3.nxv24bf16.nxv8i1(<vscale x 8 x i1> %[[PG]], bfloat* %[[GEP]])
+  // CHECK-NEXT: ret <vscale x 24 x bfloat> %[[LOAD]]
+  return SVE_ACLE_FUNC(svld3_vnum,_bf16,,)(pg, base, vnum);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld3.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld3.c
index b9d72033844c5..c3251c9205826 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld3.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld3.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld4-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld4-bfloat.c
new file mode 100644
index 0000000000000..953ebfb692262
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld4-bfloat.c
@@ -0,0 +1,31 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16x4_t test_svld4_bf16(svbool_t pg, const bfloat16_t *base)
+{
+  // CHECK-LABEL: test_svld4_bf16
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[LOAD:.*]] = call <vscale x 32 x bfloat> @llvm.aarch64.sve.ld4.nxv32bf16.nxv8i1(<vscale x 8 x i1> %[[PG]], bfloat* %base)
+  // CHECK-NEXT: ret <vscale x 32 x bfloat> %[[LOAD]]
+  return SVE_ACLE_FUNC(svld4,_bf16,,)(pg, base);
+}
+
+svbfloat16x4_t test_svld4_vnum_bf16(svbool_t pg, const bfloat16_t *base, int64_t vnum)
+{
+  // CHECK-LABEL: test_svld4_vnum_bf16
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK-DAG: %[[BASE:.*]] = bitcast bfloat* %base to <vscale x 8 x bfloat>*
+  // CHECK-DAG: %[[GEP:.*]] = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %[[BASE]], i64 %vnum, i64 0
+  // CHECK: %[[LOAD:.*]] = call <vscale x 32 x bfloat> @llvm.aarch64.sve.ld4.nxv32bf16.nxv8i1(<vscale x 8 x i1> %[[PG]], bfloat* %[[GEP]])
+  // CHECK-NEXT: ret <vscale x 32 x bfloat> %[[LOAD]]
+  return SVE_ACLE_FUNC(svld4_vnum,_bf16,,)(pg, base, vnum);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld4.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld4.c
index 8d904b49dfdb6..60deeb3c08962 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld4.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld4.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1-bfloat.c
new file mode 100644
index 0000000000000..7897a71ac2cc8
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1-bfloat.c
@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svldff1_bf16(svbool_t pg, const bfloat16_t *base)
+{
+  // CHECK-LABEL: test_svldff1_bf16
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[LOAD:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldff1.nxv8bf16(<vscale x 8 x i1> %[[PG]], bfloat* %base)
+  // CHECK: ret <vscale x 8 x bfloat> %[[LOAD]]
+  // expected-warning@+1 {{implicit declaration of function 'svldff1_bf16'}}
+  return SVE_ACLE_FUNC(svldff1,_bf16,,)(pg, base);
+}
+
+svbfloat16_t test_svldff1_vnum_bf16(svbool_t pg, const bfloat16_t *base, int64_t vnum)
+{
+  // CHECK-LABEL: test_svldff1_vnum_bf16
+  // CHECK-DAG: %[[BITCAST:.*]] = bitcast bfloat* %base to <vscale x 8 x bfloat>*
+  // CHECK-DAG: %[[GEP:.*]] = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %[[BITCAST]], i64 %vnum, i64 0
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[LOAD:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldff1.nxv8bf16(<vscale x 8 x i1> %[[PG]], bfloat* %[[GEP]])
+  // CHECK: ret <vscale x 8 x bfloat> %[[LOAD]]
+  // expected-warning@+1 {{implicit declaration of function 'svldff1_vnum_bf16'}}
+  return SVE_ACLE_FUNC(svldff1_vnum,_bf16,,)(pg, base, vnum);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1.c
index 5363445a4f59b..0797cb3c7deba 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sb.c
index 6d5a964e1cf7e..0915a8ae49599 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sb.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sh.c
index 3c60dcb8b2956..5c1fd27c3bbb9 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sh.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sw.c
index 6c65979c32d40..b7892b96d0bbd 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sw.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1ub.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1ub.c
index a6c117970877d..108980c190431 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1ub.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1ub.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1uh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1uh.c
index b335f3184d4c9..2a267e8301c83 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1uh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1uh.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1uw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1uw.c
index 0b47a0630d58a..6865c3ee6258b 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1uw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1uw.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1-bfloat.c
new file mode 100644
index 0000000000000..438e5d196cf73
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1-bfloat.c
@@ -0,0 +1,33 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svldnf1_bf16(svbool_t pg, const bfloat16_t *base)
+{
+  // CHECK-LABEL: test_svldnf1_bf16
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[LOAD:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnf1.nxv8bf16(<vscale x 8 x i1> %[[PG]], bfloat* %base)
+  // CHECK: ret <vscale x 8 x bfloat> %[[LOAD]]
+  // expected-warning@+1 {{implicit declaration of function 'svldnf1_bf16'}}
+  return SVE_ACLE_FUNC(svldnf1,_bf16,,)(pg, base);
+}
+
+svbfloat16_t test_svldnf1_vnum_bf16(svbool_t pg, const bfloat16_t *base, int64_t vnum)
+{
+  // CHECK-LABEL: test_svldnf1_vnum_bf16
+  // CHECK-DAG: %[[BITCAST:.*]] = bitcast bfloat* %base to <vscale x 8 x bfloat>*
+  // CHECK-DAG: %[[GEP:.*]] = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %[[BITCAST]], i64 %vnum, i64 0
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[LOAD:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnf1.nxv8bf16(<vscale x 8 x i1> %[[PG]], bfloat* %[[GEP]])
+  // CHECK: ret <vscale x 8 x bfloat> %[[LOAD]]
+  // expected-warning@+1 {{implicit declaration of function 'svldnf1_vnum_bf16'}}
+  return SVE_ACLE_FUNC(svldnf1_vnum,_bf16,,)(pg, base, vnum);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1.c
index bac5a26c249e4..55343bc803f13 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sb.c
index 274fe4249e355..841da37bc12f9 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sb.c
@@ -1,5 +1,10 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 svint16_t test_svldnf1sb_s16(svbool_t pg, const int8_t *base)
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sh.c
index d57fa1fd11104..e5a1666abd600 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sh.c
@@ -1,5 +1,10 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 svint32_t test_svldnf1sh_s32(svbool_t pg, const int16_t *base)
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sw.c
index 5c70e3068c62a..addb6825aa373 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sw.c
@@ -1,5 +1,10 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 svint64_t test_svldnf1sw_s64(svbool_t pg, const int32_t *base)
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1ub.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1ub.c
index 19b316d8774d0..63ea57c43c550 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1ub.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1ub.c
@@ -1,5 +1,10 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 svint16_t test_svldnf1ub_s16(svbool_t pg, const uint8_t *base)
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1uh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1uh.c
index 260bcb2e81a7e..3d70bba7f7d49 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1uh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1uh.c
@@ -1,5 +1,10 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 svint32_t test_svldnf1uh_s32(svbool_t pg, const uint16_t *base)
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1uw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1uw.c
index 011440da933f9..685a563c99ffe 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1uw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1uw.c
@@ -1,5 +1,10 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 svint64_t test_svldnf1uw_s64(svbool_t pg, const uint32_t *base)
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnt1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnt1-bfloat.c
new file mode 100644
index 0000000000000..db61ea68308ea
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnt1-bfloat.c
@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svldnt1_bf16(svbool_t pg, const bfloat16_t *base)
+{
+  // CHECK-LABEL: test_svldnt1_bf16
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[LOAD:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnt1.nxv8bf16(<vscale x 8 x i1> %[[PG]], bfloat* %base)
+  // CHECK: ret <vscale x 8 x bfloat> %[[LOAD]]
+  // expected-warning@+1 {{implicit declaration of function 'svldnt1_bf16'}}
+  return SVE_ACLE_FUNC(svldnt1,_bf16,,)(pg, base);
+}
+
+svbfloat16_t test_svldnt1_vnum_bf16(svbool_t pg, const bfloat16_t *base, int64_t vnum)
+{
+  // CHECK-LABEL: test_svldnt1_vnum_bf16
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK-DAG: %[[BITCAST:.*]] = bitcast bfloat* %base to <vscale x 8 x bfloat>*
+  // CHECK-DAG: %[[GEP:.*]] = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %[[BITCAST]], i64 %vnum, i64 0
+  // CHECK: %[[LOAD:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnt1.nxv8bf16(<vscale x 8 x i1> %[[PG]], bfloat* %[[GEP]])
+  // CHECK: ret <vscale x 8 x bfloat> %[[LOAD]]
+  // expected-warning@+1 {{implicit declaration of function 'svldnt1_vnum_bf16'}}
+  return SVE_ACLE_FUNC(svldnt1_vnum,_bf16,,)(pg, base, vnum);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnt1.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnt1.c
index 6806e2589bdfb..71685d899c034 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnt1.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnt1.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_len-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_len-bfloat.c
new file mode 100644
index 0000000000000..c084590039b3f
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_len-bfloat.c
@@ -0,0 +1,27 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+uint64_t test_svlen_bf16(svbfloat16_t op)
+{
+  // CHECK-LABEL: test_svlen_bf16
+  // CHECK: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
+  // CHECK: %[[SHL:.*]] = shl i64 %[[VSCALE]], 3
+  // CHECK: ret i64 %[[SHL]]
+  // expected-warning@+1 {{implicit declaration of function 'svlen_bf16'}}
+  return SVE_ACLE_FUNC(svlen,_bf16,,)(op);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_len.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_len.c
index 1d096dfd9cc7d..55fb6a4f5fea9 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_len.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_len.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lsl.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lsl.c
index d44b79864bebd..dc0bf2587ac00 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lsl.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lsl.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lsr.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lsr.c
index e64913a60670f..5271b38e4cf9b 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lsr.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_lsr.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mad.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mad.c
index d53382b98ffb8..0483884cb1343 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mad.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mad.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_matmul_fp32.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_matmul_fp32.c
index 422115c7aba5e..4ad3ed39a6d4a 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_matmul_fp32.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_matmul_fp32.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_FP32 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_FP32 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f32mm -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f32mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_matmul_fp64.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_matmul_fp64.c
index e918d271e53ba..e287734f1afe9 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_matmul_fp64.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_matmul_fp64.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_FP64 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_FP64 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_max.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_max.c
index 8759057aed2ec..9526a37357d48 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_max.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_max.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_maxnm.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_maxnm.c
index a6dd6eab6738e..0646356b40a8f 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_maxnm.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_maxnm.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_maxnmv.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_maxnmv.c
index 3d057a887c225..ea79022775e13 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_maxnmv.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_maxnmv.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_maxv.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_maxv.c
index a488c17e9df7f..8d33b78378d46 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_maxv.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_maxv.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_min.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_min.c
index 4fccd76389733..7966b57d9edf2 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_min.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_min.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_minnm.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_minnm.c
index 2aab3a138f92e..2957e8c5952de 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_minnm.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_minnm.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_minnmv.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_minnmv.c
index ca414449c0010..273e197b1e098 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_minnmv.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_minnmv.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_minv.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_minv.c
index 42b1462367746..466b90af12714 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_minv.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_minv.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mla.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mla.c
index 301a49c5fdbea..d187b79cbcc7b 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mla.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mla.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mls.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mls.c
index 2c2c99025ae8b..e50d4bceb4c02 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mls.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mls.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mmla.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mmla.c
index 8259bc0aa9a29..642cb1ca3ffa3 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mmla.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mmla.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_INT8 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_INT8 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +i8mm -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +i8mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mov.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mov.c
index 76ea092c38faf..88d4e9b5a2e77 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mov.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mov.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_msb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_msb.c
index f3aa553bc0860..c4fef3a4439da 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_msb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_msb.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mul.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mul.c
index e44a7b3b1fc02..804eac56608dd 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mul.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mul.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mulh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mulh.c
index 0384edeb27b0a..c899a08f9ff31 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mulh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mulh.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mulx.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mulx.c
index 0cbf5fa20e9cc..2e8a9c78b0d7a 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mulx.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_mulx.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nand.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nand.c
index e2db5919db394..450a493ee52cc 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nand.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nand.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_neg.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_neg.c
index 1e3a697733729..5cf52196c1d04 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_neg.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_neg.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nmad.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nmad.c
index c9432b5a562fc..f9a96f5bdf683 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nmad.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nmad.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nmla.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nmla.c
index 8613f7ba6a16b..7db8e950d8733 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nmla.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nmla.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nmls.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nmls.c
index 4af2c079f45a5..892da2b9254a6 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nmls.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nmls.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nmsb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nmsb.c
index 9b6149273099b..5792cb2e66e2d 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nmsb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nmsb.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nor.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nor.c
index 482b6ace99d50..ff2d8a67907e0 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nor.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_nor.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_not.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_not.c
index f6ba6b5fef299..43773ef21c9d9 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_not.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_not.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_orn.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_orn.c
index 40b5dbd2ad0d0..21156ef6cbe50 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_orn.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_orn.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_orr.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_orr.c
index 8a4c226e06dba..df64dce6aa57b 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_orr.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_orr.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_orv.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_orv.c
index e0810eff8f198..be15713f374c3 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_orv.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_orv.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_pfalse.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_pfalse.c
index 8702ea7c85b7f..1131978933278 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_pfalse.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_pfalse.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_pfirst.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_pfirst.c
index 2de1daed744c8..1486e948fa95b 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_pfirst.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_pfirst.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_pnext.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_pnext.c
index 2bc17afe12334..4d023fa34b8ff 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_pnext.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_pnext.c
@@ -1,5 +1,10 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 svbool_t test_svpnext_b8(svbool_t pg, svbool_t op)
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfb.c
index 4928ab87c070d..93aef4a5b4b06 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfb.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfd.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfd.c
index d2bcb9b7f413d..454b149199556 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfd.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfd.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfh.c
index ce0131f6ccb60..3b9f0b9050f11 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfh.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfw.c
index 775ebeb7ef9f6..53151a4c4b183 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfw.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ptest.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ptest.c
index 6290e47658253..b4ca9fa0000e9 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ptest.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ptest.c
@@ -1,6 +1,6 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ptrue.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ptrue.c
index f5a9ee8ec2825..98ea771ddf54f 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ptrue.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ptrue.c
@@ -1,5 +1,10 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 svbool_t test_svptrue_b8()
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qadd.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qadd.c
index 10e79ab7fa4cc..634be9daf3fb1 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qadd.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qadd.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdecb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdecb.c
index 26f09adac4fb3..507db3aab27f3 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdecb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdecb.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdecd.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdecd.c
index d69102e34ef28..7aad6fa51e5e2 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdecd.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdecd.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdech.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdech.c
index 1725cb4c8063c..6eb5e467f0f89 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdech.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdech.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdecp.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdecp.c
index 08298f72c029a..2cd4f75f578fd 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdecp.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdecp.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdecw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdecw.c
index 3f318020228d9..532a57a6bae1d 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdecw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qdecw.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qincb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qincb.c
index 2ae1c8aa1fd1e..7494af867da58 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qincb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qincb.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qincd.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qincd.c
index ac60f02c7d043..3780f5190d81b 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qincd.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qincd.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qinch.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qinch.c
index a0ea4fa9c6496..25984e4d0b63f 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qinch.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qinch.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qincp.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qincp.c
index 4052e309c00b6..2b1a0097eae94 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qincp.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qincp.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qincw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qincw.c
index 3f7cd70b64b0c..13787b5bf4ca0 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qincw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qincw.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qsub.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qsub.c
index b19bbf53d4330..48219a45e51a9 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qsub.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_qsub.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rbit.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rbit.c
index b8a2973ddb2ec..a3e4f67a93ac4 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rbit.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rbit.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rdffr.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rdffr.c
index 3e43b940402ff..dd4d6de5d6f7a 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rdffr.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rdffr.c
@@ -1,6 +1,6 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_recpe.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_recpe.c
index 7b4c5b8c7b864..644afb130195e 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_recpe.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_recpe.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_recps.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_recps.c
index b35f3b5546e45..2e170fa506fb9 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_recps.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_recps.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_recpx.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_recpx.c
index a375a2865d2a8..e9d19c8bf14fa 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_recpx.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_recpx.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_reinterpret-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_reinterpret-bfloat.c
new file mode 100644
index 0000000000000..90547a09b6e4e
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_reinterpret-bfloat.c
@@ -0,0 +1,177 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svint8_t test_svreinterpret_s8_bf16(svbfloat16_t op) {
+  // CHECK-LABEL: test_svreinterpret_s8_bf16
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 8 x bfloat> %op to <vscale x 16 x i8>
+  // CHECK: ret <vscale x 16 x i8> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_s8, _bf16, , )(op);
+}
+
+svint16_t test_svreinterpret_s16_bf16(svbfloat16_t op) {
+  // CHECK-LABEL: test_svreinterpret_s16_bf16
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 8 x bfloat> %op to <vscale x 8 x i16>
+  // CHECK: ret <vscale x 8 x i16> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_s16, _bf16, , )(op);
+}
+
+svint32_t test_svreinterpret_s32_bf16(svbfloat16_t op) {
+  // CHECK-LABEL: test_svreinterpret_s32_bf16
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 8 x bfloat> %op to <vscale x 4 x i32>
+  // CHECK: ret <vscale x 4 x i32> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_s32, _bf16, , )(op);
+}
+svint64_t test_svreinterpret_s64_bf16(svbfloat16_t op) {
+  // CHECK-LABEL: test_svreinterpret_s64_bf16
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 8 x bfloat> %op to <vscale x 2 x i64>
+  // CHECK: ret <vscale x 2 x i64> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_s64, _bf16, , )(op);
+}
+
+svuint8_t test_svreinterpret_u8_bf16(svbfloat16_t op) {
+  // CHECK-LABEL: test_svreinterpret_u8_bf16
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 8 x bfloat> %op to <vscale x 16 x i8>
+  // CHECK: ret <vscale x 16 x i8> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_u8, _bf16, , )(op);
+}
+
+svuint16_t test_svreinterpret_u16_bf16(svbfloat16_t op) {
+  // CHECK-LABEL: test_svreinterpret_u16_bf16
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 8 x bfloat> %op to <vscale x 8 x i16>
+  // CHECK: ret <vscale x 8 x i16> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_u16, _bf16, , )(op);
+}
+
+svuint32_t test_svreinterpret_u32_bf16(svbfloat16_t op) {
+  // CHECK-LABEL: test_svreinterpret_u32_bf16
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 8 x bfloat> %op to <vscale x 4 x i32>
+  // CHECK: ret <vscale x 4 x i32> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_u32, _bf16, , )(op);
+}
+
+svuint64_t test_svreinterpret_u64_bf16(svbfloat16_t op) {
+  // CHECK-LABEL: test_svreinterpret_u64_bf16
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 8 x bfloat> %op to <vscale x 2 x i64>
+  // CHECK: ret <vscale x 2 x i64> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_u64, _bf16, , )(op);
+}
+
+svbfloat16_t test_svreinterpret_bf16_s8(svint8_t op) {
+  // CHECK-LABEL: test_svreinterpret_bf16_s8
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 16 x i8> %op to <vscale x 8 x bfloat>
+  // CHECK: ret <vscale x 8 x bfloat> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_bf16, _s8, , )(op);
+}
+
+svbfloat16_t test_svreinterpret_bf16_s16(svint16_t op) {
+  // CHECK-LABEL: test_svreinterpret_bf16_s16
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 8 x i16> %op to <vscale x 8 x bfloat>
+  // CHECK: ret <vscale x 8 x bfloat> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_bf16, _s16, , )(op);
+}
+
+svbfloat16_t test_svreinterpret_bf16_s32(svint32_t op) {
+  // CHECK-LABEL: test_svreinterpret_bf16_s32
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 4 x i32> %op to <vscale x 8 x bfloat>
+  // CHECK: ret <vscale x 8 x bfloat> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_bf16, _s32, , )(op);
+}
+
+svbfloat16_t test_svreinterpret_bf16_s64(svint64_t op) {
+  // CHECK-LABEL: test_svreinterpret_bf16_s64
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 2 x i64> %op to <vscale x 8 x bfloat>
+  // CHECK: ret <vscale x 8 x bfloat> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_bf16, _s64, , )(op);
+}
+
+svbfloat16_t test_svreinterpret_bf16_u8(svuint8_t op) {
+  // CHECK-LABEL: test_svreinterpret_bf16_u8
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 16 x i8> %op to <vscale x 8 x bfloat>
+  // CHECK: ret <vscale x 8 x bfloat> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_bf16, _u8, , )(op);
+}
+
+svbfloat16_t test_svreinterpret_bf16_u16(svuint16_t op) {
+  // CHECK-LABEL: test_svreinterpret_bf16_u16
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 8 x i16> %op to <vscale x 8 x bfloat>
+  // CHECK: ret <vscale x 8 x bfloat> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_bf16, _u16, , )(op);
+}
+
+svbfloat16_t test_svreinterpret_bf16_u32(svuint32_t op) {
+  // CHECK-LABEL: test_svreinterpret_bf16_u32
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 4 x i32> %op to <vscale x 8 x bfloat>
+  // CHECK: ret <vscale x 8 x bfloat> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_bf16, _u32, , )(op);
+}
+
+svbfloat16_t test_svreinterpret_bf16_u64(svuint64_t op) {
+  // CHECK-LABEL: test_svreinterpret_bf16_u64
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 2 x i64> %op to <vscale x 8 x bfloat>
+  // CHECK: ret <vscale x 8 x bfloat> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_bf16, _u64, , )(op);
+}
+
+svbfloat16_t test_svreinterpret_bf16_bf16(svbfloat16_t op) {
+  // CHECK-LABEL: test_svreinterpret_bf16_bf16
+  // CHECK: ret <vscale x 8 x bfloat> %op
+  return SVE_ACLE_FUNC(svreinterpret_bf16, _bf16, , )(op);
+}
+
+svbfloat16_t test_svreinterpret_bf16_f16(svfloat16_t op) {
+  // CHECK-LABEL: test_svreinterpret_bf16_f16
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 8 x half> %op to <vscale x 8 x bfloat>
+  // CHECK: ret <vscale x 8 x bfloat> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_bf16, _f16, , )(op);
+}
+
+svbfloat16_t test_svreinterpret_bf16_f32(svfloat32_t op) {
+  // CHECK-LABEL: test_svreinterpret_bf16_f32
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 4 x float> %op to <vscale x 8 x bfloat>
+  // CHECK: ret <vscale x 8 x bfloat> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_bf16, _f32, , )(op);
+}
+
+svbfloat16_t test_svreinterpret_bf16_f64(svfloat64_t op) {
+  // CHECK-LABEL: test_svreinterpret_bf16_f64
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 2 x double> %op to <vscale x 8 x bfloat>
+  // CHECK: ret <vscale x 8 x bfloat> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_bf16, _f64, , )(op);
+}
+
+svfloat32_t test_svreinterpret_f32_bf16(svbfloat16_t op) {
+  // CHECK-LABEL: test_svreinterpret_f32_bf16
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 8 x bfloat> %op to <vscale x 4 x float>
+  // CHECK: ret <vscale x 4 x float> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_f32, _bf16, , )(op);
+}
+
+svfloat16_t test_svreinterpret_f16_bf16(svbfloat16_t op) {
+  // CHECK-LABEL: test_svreinterpret_f16_bf16
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 8 x bfloat> %op to <vscale x 8 x half>
+  // CHECK: ret <vscale x 8 x half> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_f16, _bf16, , )(op);
+}
+
+svfloat64_t test_svreinterpret_f64_bf16(svbfloat16_t op) {
+  // CHECK-LABEL: test_svreinterpret_f64_bf16
+  // CHECK: %[[CAST:.*]] = bitcast <vscale x 8 x bfloat> %op to <vscale x 2 x double>
+  // CHECK: ret <vscale x 2 x double> %[[CAST]]
+  return SVE_ACLE_FUNC(svreinterpret_f64, _bf16, , )(op);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_reinterpret.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_reinterpret.c
index 20abbabaeed22..0a50057fbc2a0 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_reinterpret.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_reinterpret.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rev-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rev-bfloat.c
new file mode 100644
index 0000000000000..1feab6ed6ec90
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rev-bfloat.c
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svrev_bf16(svbfloat16_t op)
+{
+  // CHECK-LABEL: test_svrev_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.rev.nxv8bf16(<vscale x 8 x bfloat> %op)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svrev_bf16'}}
+  return SVE_ACLE_FUNC(svrev,_bf16,,)(op);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rev.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rev.c
index fa275a8fab4ff..f48c6c71f4961 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rev.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rev.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_revb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_revb.c
index 2cf1c211cab1b..a61519f2930f0 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_revb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_revb.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_revh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_revh.c
index 1b72508cae859..aef91d64fcbdf 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_revh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_revh.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_revw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_revw.c
index db8b596426374..99ed94f8854ba 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_revw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_revw.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rinta.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rinta.c
index bcee6301ef065..4cc978ed9fddf 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rinta.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rinta.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rinti.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rinti.c
index febcb87e74d31..f595cdb13449b 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rinti.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rinti.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintm.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintm.c
index d781a130204fa..40db460ea6e76 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintm.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintm.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintn.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintn.c
index ba89f486190ca..73a7a2f68fab1 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintn.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintn.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintp.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintp.c
index eefdaea5d93aa..f014b3d3549a8 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintp.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintp.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintx.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintx.c
index a87be65edee2b..f54d704181041 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintx.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintx.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintz.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintz.c
index 3a6b120e90573..328b8375748b5 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintz.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rintz.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rsqrte.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rsqrte.c
index ae2bb3c357133..fb56c377a8b50 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rsqrte.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rsqrte.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rsqrts.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rsqrts.c
index 5dd5c909afb58..143461f235608 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rsqrts.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rsqrts.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_scale.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_scale.c
index 74d70482ee0f7..2ff293bdd88c5 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_scale.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_scale.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sel-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sel-bfloat.c
new file mode 100644
index 0000000000000..d803d2ffdf4d7
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sel-bfloat.c
@@ -0,0 +1,27 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svsel_bf16(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
+{
+  // CHECK-LABEL: test_svsel_bf16
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.sel.nxv8bf16(<vscale x 8 x i1> %[[PG]], <vscale x 8 x bfloat> %op1, <vscale x 8 x bfloat> %op2)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svsel_bf16'}}
+  return SVE_ACLE_FUNC(svsel,_bf16,,)(pg, op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sel.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sel.c
index aeedcdf970cfe..30b0d4dfa38a5 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sel.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sel.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set2-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set2-bfloat.c
new file mode 100644
index 0000000000000..b44d6264a711e
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set2-bfloat.c
@@ -0,0 +1,30 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16x2_t test_svset2_bf16_0(svbfloat16x2_t tuple, svbfloat16_t x)
+{
+  // CHECK-LABEL: test_svset2_bf16_0
+  // CHECK: %[[INSERT:.*]] = call <vscale x 16 x bfloat> @llvm.aarch64.sve.tuple.set.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> %tuple, i32 0, <vscale x 8 x bfloat> %x)
+  // CHECK-NEXT: ret <vscale x 16 x bfloat> %[[INSERT]]
+  // expected-warning@+1 {{implicit declaration of function 'svset2_bf16'}}
+  return SVE_ACLE_FUNC(svset2,_bf16,,)(tuple, 0, x);
+}
+
+svbfloat16x2_t test_svset2_bf16_1(svbfloat16x2_t tuple, svbfloat16_t x)
+{
+  // CHECK-LABEL: test_svset2_bf16_1
+  // CHECK: %[[INSERT:.*]] = call <vscale x 16 x bfloat> @llvm.aarch64.sve.tuple.set.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> %tuple, i32 1, <vscale x 8 x bfloat> %x)
+  // CHECK-NEXT: ret <vscale x 16 x bfloat> %[[INSERT]]
+  // expected-warning@+1 {{implicit declaration of function 'svset2_bf16'}}
+  return SVE_ACLE_FUNC(svset2,_bf16,,)(tuple, 1, x);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set2.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set2.c
index dc41543a4e835..9be45026d40c2 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set2.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set2.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set3-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set3-bfloat.c
new file mode 100644
index 0000000000000..1d5881d074e76
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set3-bfloat.c
@@ -0,0 +1,40 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+
+svbfloat16x3_t test_svset3_bf16_0(svbfloat16x3_t tuple, svbfloat16_t x)
+{
+  // CHECK-LABEL: test_svset3_bf16_0
+  // CHECK: %[[INSERT:.*]] = call <vscale x 24 x bfloat> @llvm.aarch64.sve.tuple.set.nxv24bf16.nxv8bf16(<vscale x 24 x bfloat> %tuple, i32 0, <vscale x 8 x bfloat> %x)
+  // CHECK-NEXT: ret <vscale x 24 x bfloat> %[[INSERT]]
+  // expected-warning@+1 {{implicit declaration of function 'svset3_bf16'}}
+  return SVE_ACLE_FUNC(svset3,_bf16,,)(tuple, 0, x);
+}
+
+svbfloat16x3_t test_svset3_bf16_1(svbfloat16x3_t tuple, svbfloat16_t x)
+{
+  // CHECK-LABEL: test_svset3_bf16_1
+  // CHECK: %[[INSERT:.*]] = call <vscale x 24 x bfloat> @llvm.aarch64.sve.tuple.set.nxv24bf16.nxv8bf16(<vscale x 24 x bfloat> %tuple, i32 1, <vscale x 8 x bfloat> %x)
+  // CHECK-NEXT: ret <vscale x 24 x bfloat> %[[INSERT]]
+  // expected-warning@+1 {{implicit declaration of function 'svset3_bf16'}}
+  return SVE_ACLE_FUNC(svset3,_bf16,,)(tuple, 1, x);
+}
+
+svbfloat16x3_t test_svset3_bf16_2(svbfloat16x3_t tuple, svbfloat16_t x)
+{
+  // CHECK-LABEL: test_svset3_bf16_2
+  // CHECK: %[[INSERT:.*]] = call <vscale x 24 x bfloat> @llvm.aarch64.sve.tuple.set.nxv24bf16.nxv8bf16(<vscale x 24 x bfloat> %tuple, i32 2, <vscale x 8 x bfloat> %x)
+  // CHECK-NEXT: ret <vscale x 24 x bfloat> %[[INSERT]]
+  // expected-warning@+1 {{implicit declaration of function 'svset3_bf16'}}
+  return SVE_ACLE_FUNC(svset3,_bf16,,)(tuple, 2, x);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set3.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set3.c
index 77fd69bbbbc2e..0f59a389e0d19 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set3.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set3.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set4-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set4-bfloat.c
new file mode 100644
index 0000000000000..b824cace2b78b
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set4-bfloat.c
@@ -0,0 +1,49 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+
+svbfloat16x4_t test_svset4_bf16_0(svbfloat16x4_t tuple, svbfloat16_t x)
+{
+  // CHECK-LABEL: test_svset4_bf16_0
+  // CHECK: %[[INSERT:.*]] = call <vscale x 32 x bfloat> @llvm.aarch64.sve.tuple.set.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> %tuple, i32 0, <vscale x 8 x bfloat> %x)
+  // CHECK-NEXT: ret <vscale x 32 x bfloat> %[[INSERT]]
+  // expected-warning@+1 {{implicit declaration of function 'svset4_bf16'}}
+  return SVE_ACLE_FUNC(svset4,_bf16,,)(tuple, 0, x);
+}
+
+svbfloat16x4_t test_svset4_bf16_1(svbfloat16x4_t tuple, svbfloat16_t x)
+{
+  // CHECK-LABEL: test_svset4_bf16_1
+  // CHECK: %[[INSERT:.*]] = call <vscale x 32 x bfloat> @llvm.aarch64.sve.tuple.set.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> %tuple, i32 1, <vscale x 8 x bfloat> %x)
+  // CHECK-NEXT: ret <vscale x 32 x bfloat> %[[INSERT]]
+  // expected-warning@+1 {{implicit declaration of function 'svset4_bf16'}}
+  return SVE_ACLE_FUNC(svset4,_bf16,,)(tuple, 1, x);
+}
+
+svbfloat16x4_t test_svset4_bf16_2(svbfloat16x4_t tuple, svbfloat16_t x)
+{
+  // CHECK-LABEL: test_svset4_bf16_2
+  // CHECK: %[[INSERT:.*]] = call <vscale x 32 x bfloat> @llvm.aarch64.sve.tuple.set.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> %tuple, i32 2, <vscale x 8 x bfloat> %x)
+  // CHECK-NEXT: ret <vscale x 32 x bfloat> %[[INSERT]]
+  // expected-warning@+1 {{implicit declaration of function 'svset4_bf16'}}
+  return SVE_ACLE_FUNC(svset4,_bf16,,)(tuple, 2, x);
+}
+
+svbfloat16x4_t test_svset4_bf16_3(svbfloat16x4_t tuple, svbfloat16_t x)
+{
+  // CHECK-LABEL: test_svset4_bf16_3
+  // CHECK: %[[INSERT:.*]] = call <vscale x 32 x bfloat> @llvm.aarch64.sve.tuple.set.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> %tuple, i32 3, <vscale x 8 x bfloat> %x)
+  // CHECK-NEXT: ret <vscale x 32 x bfloat> %[[INSERT]]
+  // expected-warning@+1 {{implicit declaration of function 'svset4_bf16'}}
+  return SVE_ACLE_FUNC(svset4,_bf16,,)(tuple, 3, x);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set4.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set4.c
index 6522948d8ba65..1e724d172d429 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set4.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_set4.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_setffr.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_setffr.c
index 4399e20e5351e..3a9dcf0c739f3 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_setffr.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_setffr.c
@@ -1,7 +1,11 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // CHECK-NOT: warning
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 void test_svsetffr()
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_splice-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_splice-bfloat.c
new file mode 100644
index 0000000000000..b9d0c7944ae40
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_splice-bfloat.c
@@ -0,0 +1,27 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svsplice_bf16(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
+{
+  // CHECK-LABEL: test_svsplice_bf16
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.splice.nxv8bf16(<vscale x 8 x i1> %[[PG]], <vscale x 8 x bfloat> %op1, <vscale x 8 x bfloat> %op2)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svsplice_bf16'}}
+  return SVE_ACLE_FUNC(svsplice,_bf16,,)(pg, op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_splice.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_splice.c
index c90b593396901..51212274903a0 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_splice.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_splice.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sqrt.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sqrt.c
index 1081fd3ce2136..43b6e63214bd6 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sqrt.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sqrt.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1-bfloat.c
new file mode 100644
index 0000000000000..feee5434e5431
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1-bfloat.c
@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+void test_svst1_bf16(svbool_t pg, bfloat16_t *base, svbfloat16_t data)
+{
+  // CHECK-LABEL: test_svst1_bf16
+  // CHECK: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: call void @llvm.aarch64.sve.st1.nxv8bf16(<vscale x 8 x bfloat> %data, <vscale x 8 x i1> %[[PG]], bfloat* %base)
+  // CHECK: ret void
+  // expected-warning@+1 {{implicit declaration of function 'svst1_bf16'}}
+  return SVE_ACLE_FUNC(svst1,_bf16,,)(pg, base, data);
+}
+
+void test_svst1_vnum_bf16(svbool_t pg, bfloat16_t *base, int64_t vnum, svbfloat16_t data)
+{
+  // CHECK-LABEL: test_svst1_vnum_bf16
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK-DAG: %[[BASE:.*]] = bitcast bfloat* %base to <vscale x 8 x bfloat>*
+  // CHECK-DAG: %[[GEP:.*]] = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %[[BASE]], i64 %vnum, i64 0
+  // CHECK: call void @llvm.aarch64.sve.st1.nxv8bf16(<vscale x 8 x bfloat> %data, <vscale x 8 x i1> %[[PG]], bfloat* %[[GEP]])
+  // CHECK: ret void
+  // expected-warning@+1 {{implicit declaration of function 'svst1_vnum_bf16'}}
+  return SVE_ACLE_FUNC(svst1_vnum,_bf16,,)(pg, base, vnum, data);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1.c
index 160319fd65108..fe91a5c52c514 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1b.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1b.c
index 25d7bf187ffc0..09731f7d6fe06 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1b.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1b.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - -emit-llvm %s 2>&1 | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - -emit-llvm %s 2>&1 | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - -emit-llvm %s 2>&1 | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - -emit-llvm %s 2>&1 | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1h.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1h.c
index 129f678218296..35b5c8416a8e6 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1h.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1h.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - -emit-llvm %s 2>&1 | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - -emit-llvm %s 2>&1 | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - -emit-llvm %s 2>&1 | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - -emit-llvm %s 2>&1 | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1w.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1w.c
index b4ddb5ed7e334..89adeb6f5da0b 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1w.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1w.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - -emit-llvm %s 2>&1 | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - -emit-llvm %s 2>&1 | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - -emit-llvm %s 2>&1 | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - -emit-llvm %s 2>&1 | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st2-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st2-bfloat.c
new file mode 100644
index 0000000000000..4b2235e63afe7
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st2-bfloat.c
@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+void test_svst2_bf16(svbool_t pg, bfloat16_t *base, svbfloat16x2_t data)
+{
+  // CHECK-LABEL: test_svst2_bf16
+  // CHECK-DAG: %[[V0:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> %data, i32 0)
+  // CHECK-DAG: %[[V1:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> %data, i32 1)
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: call void @llvm.aarch64.sve.st2.nxv8bf16(<vscale x 8 x bfloat> %[[V0]], <vscale x 8 x bfloat> %[[V1]], <vscale x 8 x i1> %[[PG]], bfloat* %base)
+  // CHECK-NEXT: ret
+  return SVE_ACLE_FUNC(svst2,_bf16,,)(pg, base, data);
+}
+
+void test_svst2_vnum_bf16(svbool_t pg, bfloat16_t *base, int64_t vnum, svbfloat16x2_t data)
+{
+  // CHECK-LABEL: test_svst2_vnum_bf16
+  // CHECK-DAG: %[[BITCAST:.*]] = bitcast bfloat* %base to <vscale x 8 x bfloat>*
+  // CHECK-DAG: %[[GEP:.*]] = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %[[BITCAST]], i64 %vnum, i64 0
+  // CHECK-DAG: %[[V0:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> %data, i32 0)
+  // CHECK-DAG: %[[V1:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> %data, i32 1)
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: call void @llvm.aarch64.sve.st2.nxv8bf16(<vscale x 8 x bfloat> %[[V0]], <vscale x 8 x bfloat> %[[V1]], <vscale x 8 x i1> %[[PG]], bfloat* %[[GEP]])
+  // CHECK-NEXT: ret
+  return SVE_ACLE_FUNC(svst2_vnum,_bf16,,)(pg, base, vnum, data);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st2.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st2.c
index 042612d7e0a9e..de21c59bb3b70 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st2.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st2.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st3-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st3-bfloat.c
new file mode 100644
index 0000000000000..37ec4af436468
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st3-bfloat.c
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+void test_svst3_bf16(svbool_t pg, bfloat16_t *base, svbfloat16x3_t data)
+{
+  // CHECK-LABEL: test_svst3_bf16
+  // CHECK-DAG: %[[V0:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv24bf16(<vscale x 24 x bfloat> %data, i32 0)
+  // CHECK-DAG: %[[V1:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv24bf16(<vscale x 24 x bfloat> %data, i32 1)
+  // CHECK-DAG: %[[V2:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv24bf16(<vscale x 24 x bfloat> %data, i32 2)
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: call void @llvm.aarch64.sve.st3.nxv8bf16(<vscale x 8 x bfloat> %[[V0]], <vscale x 8 x bfloat> %[[V1]], <vscale x 8 x bfloat> %[[V2]], <vscale x 8 x i1> %[[PG]], bfloat* %base)
+  // CHECK-NEXT: ret
+  return SVE_ACLE_FUNC(svst3,_bf16,,)(pg, base, data);
+}
+
+void test_svst3_vnum_bf16(svbool_t pg, bfloat16_t *base, int64_t vnum, svbfloat16x3_t data)
+{
+  // CHECK-LABEL: test_svst3_vnum_bf16
+  // CHECK-DAG: %[[BITCAST:.*]] = bitcast bfloat* %base to <vscale x 8 x bfloat>*
+  // CHECK-DAG: %[[GEP:.*]] = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %[[BITCAST]], i64 %vnum, i64 0
+  // CHECK-DAG: %[[V0:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv24bf16(<vscale x 24 x bfloat> %data, i32 0)
+  // CHECK-DAG: %[[V1:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv24bf16(<vscale x 24 x bfloat> %data, i32 1)
+  // CHECK-DAG: %[[V2:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv24bf16(<vscale x 24 x bfloat> %data, i32 2)
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: call void @llvm.aarch64.sve.st3.nxv8bf16(<vscale x 8 x bfloat> %[[V0]], <vscale x 8 x bfloat> %[[V1]], <vscale x 8 x bfloat> %[[V2]], <vscale x 8 x i1> %[[PG]], bfloat* %[[GEP]])
+  // CHECK-NEXT: ret
+  return SVE_ACLE_FUNC(svst3_vnum,_bf16,,)(pg, base, vnum, data);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st3.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st3.c
index 5fbff9c5ea408..e98b2e7cc8a2f 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st3.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st3.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st4-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st4-bfloat.c
new file mode 100644
index 0000000000000..538331b107dcb
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st4-bfloat.c
@@ -0,0 +1,39 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+void test_svst4_bf16(svbool_t pg, bfloat16_t *base, svbfloat16x4_t data)
+{
+  // CHECK-LABEL: test_svst4_bf16
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK-DAG: %[[V0:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> %data, i32 0)
+  // CHECK-DAG: %[[V1:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> %data, i32 1)
+  // CHECK-DAG: %[[V2:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> %data, i32 2)
+  // CHECK-DAG: %[[V3:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> %data, i32 3)
+  // CHECK: call void @llvm.aarch64.sve.st4.nxv8bf16(<vscale x 8 x bfloat> %[[V0]], <vscale x 8 x bfloat> %[[V1]], <vscale x 8 x bfloat> %[[V2]], <vscale x 8 x bfloat> %[[V3]], <vscale x 8 x i1> %[[PG]], bfloat* %base)
+  // CHECK-NEXT: ret
+  return SVE_ACLE_FUNC(svst4,_bf16,,)(pg, base, data);
+}
+
+void test_svst4_vnum_bf16(svbool_t pg, bfloat16_t *base, int64_t vnum, svbfloat16x4_t data)
+{
+  // CHECK-LABEL: test_svst4_vnum_bf16
+  // CHECK-DAG: %[[BITCAST:.*]] = bitcast bfloat* %base to <vscale x 8 x bfloat>*
+  // CHECK-DAG: %[[GEP:.*]] = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %[[BITCAST]], i64 %vnum, i64 0
+  // CHECK-DAG: %[[V0:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> %data, i32 0)
+  // CHECK-DAG: %[[V1:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> %data, i32 1)
+  // CHECK-DAG: %[[V2:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> %data, i32 2)
+  // CHECK-DAG: %[[V3:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> %data, i32 3)
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: call void @llvm.aarch64.sve.st4.nxv8bf16(<vscale x 8 x bfloat> %[[V0]], <vscale x 8 x bfloat> %[[V1]], <vscale x 8 x bfloat> %[[V2]], <vscale x 8 x bfloat> %[[V3]], <vscale x 8 x i1> %[[PG]], bfloat* %[[GEP]])
+  // CHECK-NEXT: ret
+  return SVE_ACLE_FUNC(svst4_vnum,_bf16,,)(pg, base, vnum, data);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st4.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st4.c
index e2f315e83c24b..2ea1ae655ee81 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st4.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st4.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_stnt1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_stnt1-bfloat.c
new file mode 100644
index 0000000000000..78ddddc6b1b7a
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_stnt1-bfloat.c
@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+void test_svstnt1_bf16(svbool_t pg, bfloat16_t *base, svbfloat16_t data)
+{
+  // CHECK-LABEL: test_svstnt1_bf16
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: call void @llvm.aarch64.sve.stnt1.nxv8bf16(<vscale x 8 x bfloat> %data, <vscale x 8 x i1> %[[PG]], bfloat* %base)
+  // CHECK-NEXT: ret
+  // expected-warning@+1 {{implicit declaration of function 'svstnt1_bf16'}}
+  return SVE_ACLE_FUNC(svstnt1,_bf16,,)(pg, base, data);
+}
+
+void test_svstnt1_vnum_bf16(svbool_t pg, bfloat16_t *base, int64_t vnum, svbfloat16_t data)
+{
+  // CHECK-LABEL: test_svstnt1_vnum_bf16
+  // CHECK-DAG: %[[BITCAST:.*]] = bitcast bfloat* %base to <vscale x 8 x bfloat>*
+  // CHECK-DAG: %[[GEP:.*]] = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %[[BITCAST]], i64 %vnum, i64 0
+  // CHECK-DAG: %[[PG:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
+  // CHECK: call void @llvm.aarch64.sve.stnt1.nxv8bf16(<vscale x 8 x bfloat> %data, <vscale x 8 x i1> %[[PG]], bfloat* %[[GEP]])
+  // CHECK-NEXT: ret
+  // expected-warning@+1 {{implicit declaration of function 'svstnt1_vnum_bf16'}}
+  return SVE_ACLE_FUNC(svstnt1_vnum,_bf16,,)(pg, base, vnum, data);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_stnt1.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_stnt1.c
index 32f222b5d47ec..0c5980808ba46 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_stnt1.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_stnt1.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sub.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sub.c
index d3f7da5862ff2..f4185179ba31e 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sub.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sub.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_subr.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_subr.c
index 81ef2839e347c..523171f8bf01d 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_subr.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_subr.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sudot.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sudot.c
index d678851bad586..70904c3fb8fd7 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sudot.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sudot.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_INT8 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_INT8 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +i8mm -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +i8mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tbl-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tbl-bfloat.c
new file mode 100644
index 0000000000000..1c68586d2fed5
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tbl-bfloat.c
@@ -0,0 +1,26 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svtbl_bf16(svbfloat16_t data, svuint16_t indices) {
+  // CHECK-LABEL: test_svtbl_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tbl.nxv8bf16(<vscale x 8 x bfloat> %data, <vscale x 8 x i16> %indices)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svtbl_bf16'}}
+  return SVE_ACLE_FUNC(svtbl, _bf16, , )(data, indices);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tbl.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tbl.c
index 14f36ca89c247..05e5fe98b79f8 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tbl.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tbl.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tmad.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tmad.c
index 70787cd634960..fa7139fa6e049 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tmad.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tmad.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-bfloat.c
new file mode 100644
index 0000000000000..fb60ce3e15b92
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-bfloat.c
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svtrn1_bf16(svbfloat16_t op1, svbfloat16_t op2)
+{
+  // CHECK-LABEL: test_svtrn1_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.trn1.nxv8bf16(<vscale x 8 x bfloat> %op1, <vscale x 8 x bfloat> %op2)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svtrn1_bf16'}}
+  return SVE_ACLE_FUNC(svtrn1,_bf16,,)(op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-fp64-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-fp64-bfloat.c
new file mode 100644
index 0000000000000..c6f8e4a92c742
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-fp64-bfloat.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -target-feature +f64mm -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svtrn1_bf16(svbfloat16_t op1, svbfloat16_t op2) {
+  // CHECK-LABEL: test_svtrn1_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.trn1q.nxv8bf16(<vscale x 8 x bfloat> %op1, <vscale x 8 x bfloat> %op2)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svtrn1q_bf16'}}
+  return SVE_ACLE_FUNC(svtrn1q, _bf16, , )(op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-fp64.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-fp64.c
index f968c1524a8b6..a8cbbc660b23a 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-fp64.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-fp64.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1.c
index 6e7cbed5350e9..7e2e8e133bda0 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-bfloat.c
new file mode 100644
index 0000000000000..62c9a7daf9ee3
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-bfloat.c
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svtrn2_bf16(svbfloat16_t op1, svbfloat16_t op2)
+{
+  // CHECK-LABEL: test_svtrn2_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.trn2.nxv8bf16(<vscale x 8 x bfloat> %op1, <vscale x 8 x bfloat> %op2)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svtrn2_bf16'}}
+  return SVE_ACLE_FUNC(svtrn2,_bf16,,)(op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-fp64-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-fp64-bfloat.c
new file mode 100644
index 0000000000000..16f4bce584611
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-fp64-bfloat.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -target-feature +f64mm -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svtrn2_bf16(svbfloat16_t op1, svbfloat16_t op2) {
+  // CHECK-LABEL: test_svtrn2_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.trn2q.nxv8bf16(<vscale x 8 x bfloat> %op1, <vscale x 8 x bfloat> %op2)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svtrn2q_bf16'}}
+  return SVE_ACLE_FUNC(svtrn2q, _bf16, , )(op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-fp64.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-fp64.c
index fc26ec2b5658f..21ea2cbc17a1f 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-fp64.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-fp64.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2.c
index 432370bb459f9..704b8d10f715d 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tsmul.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tsmul.c
index 11094fe61f4d1..698f5213fb626 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tsmul.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tsmul.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tssel.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tssel.c
index 69ab555908008..e130d836e5375 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tssel.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_tssel.c
@@ -1,7 +1,7 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef-bfloat.c
new file mode 100644
index 0000000000000..16b789ca84035
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef-bfloat.c
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+svbfloat16_t test_svundef_bf16()
+{
+  // CHECK-LABEL: test_svundef_bf16
+  // CHECK: ret <vscale x 8 x bfloat> undef
+  // expected-warning@+1 {{implicit declaration of function 'svundef_bf16'}}
+  return svundef_bf16();
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef.c
index 731f740846d03..581ea7a050d32 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef.c
@@ -1,5 +1,10 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 svint8_t test_svundef_s8()
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef2-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef2-bfloat.c
new file mode 100644
index 0000000000000..a65560d837b0f
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef2-bfloat.c
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+svbfloat16x2_t test_svundef2_bf16()
+{
+  // CHECK-LABEL: test_svundef2_bf16
+  // CHECK: ret <vscale x 16 x bfloat> undef
+  // expected-warning@+1 {{implicit declaration of function 'svundef2_bf16'}}
+  return svundef2_bf16();
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef2.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef2.c
index ab5fb1cc691c8..8259cfe307507 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef2.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef2.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O2 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O2 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef3-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef3-bfloat.c
new file mode 100644
index 0000000000000..6e0ae7ce1ce73
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef3-bfloat.c
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+svbfloat16x3_t test_svundef3_bf16()
+{
+  // CHECK-LABEL: test_svundef3_bf16
+  // CHECK: ret <vscale x 24 x bfloat> undef
+  // expected-warning@+1 {{implicit declaration of function 'svundef3_bf16'}}
+  return svundef3_bf16();
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef3.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef3.c
index 925e5aa43779e..929983a3ccde9 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef3.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef3.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O2 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O2 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef4-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef4-bfloat.c
new file mode 100644
index 0000000000000..6907451d2475f
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef4-bfloat.c
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+svbfloat16x4_t test_svundef4_bf16()
+{
+  // CHECK-LABEL: test_svundef4_bf16
+  // CHECK: ret <vscale x 32 x bfloat> undef
+  // expected-warning@+1 {{implicit declaration of function 'svundef4_bf16'}}
+  return svundef4_bf16();
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef4.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef4.c
index 7dc854f33159b..9c45a8936d1b6 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef4.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_undef4.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O2 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O2 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_unpkhi.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_unpkhi.c
index 5bab85658561f..cae68a3e0a858 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_unpkhi.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_unpkhi.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_unpklo.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_unpklo.c
index 343f61c5257f6..a73419779a281 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_unpklo.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_unpklo.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_usdot.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_usdot.c
index 1639cbb2b86f3..0c361bac167fd 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_usdot.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_usdot.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_INT8 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_INT8 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +i8mm -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +i8mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-bfloat.c
new file mode 100644
index 0000000000000..ffc98e9745b1d
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-bfloat.c
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svuzp1_bf16(svbfloat16_t op1, svbfloat16_t op2)
+{
+  // CHECK-LABEL: test_svuzp1_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.uzp1.nxv8bf16(<vscale x 8 x bfloat> %op1, <vscale x 8 x bfloat> %op2)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svuzp1_bf16'}}
+  return SVE_ACLE_FUNC(svuzp1,_bf16,,)(op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-fp64-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-fp64-bfloat.c
new file mode 100644
index 0000000000000..4aa8833073a43
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-fp64-bfloat.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -target-feature +f64mm -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svuzp1_bf16(svbfloat16_t op1, svbfloat16_t op2) {
+  // CHECK-LABEL: test_svuzp1_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.uzp1q.nxv8bf16(<vscale x 8 x bfloat> %op1, <vscale x 8 x bfloat> %op2)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svuzp1q_bf16'}}
+  return SVE_ACLE_FUNC(svuzp1q, _bf16, , )(op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-fp64.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-fp64.c
index cb96efa7f519a..ab63a5fd07144 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-fp64.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-fp64.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1.c
index d95cb5ec9bf89..6eea3bc17d98f 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-bfloat.c
new file mode 100644
index 0000000000000..bf3dd928c4fe3
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-bfloat.c
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svuzp2_bf16(svbfloat16_t op1, svbfloat16_t op2)
+{
+  // CHECK-LABEL: test_svuzp2_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.uzp2.nxv8bf16(<vscale x 8 x bfloat> %op1, <vscale x 8 x bfloat> %op2)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svuzp2_bf16'}}
+  return SVE_ACLE_FUNC(svuzp2,_bf16,,)(op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-fp64-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-fp64-bfloat.c
new file mode 100644
index 0000000000000..aaca01a5105dc
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-fp64-bfloat.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -target-feature +f64mm -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svuzp2_bf16(svbfloat16_t op1, svbfloat16_t op2) {
+  // CHECK-LABEL: test_svuzp2_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.uzp2q.nxv8bf16(<vscale x 8 x bfloat> %op1, <vscale x 8 x bfloat> %op2)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svuzp2q_bf16'}}
+  return SVE_ACLE_FUNC(svuzp2q, _bf16, , )(op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-fp64.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-fp64.c
index 122d5c536cc08..5b3b1b5443dea 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-fp64.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-fp64.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2.c
index b359aa68e8c06..645e96c4e55fe 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_whilele.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_whilele.c
index fc06fed94b015..6aa30f2ef59b7 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_whilele.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_whilele.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_whilelt.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_whilelt.c
index 1d2480c0a8092..6904dbc079b70 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_whilelt.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_whilelt.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_wrffr.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_wrffr.c
index 72ffe7ac2f954..c7d04db916e5d 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_wrffr.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_wrffr.c
@@ -1,6 +1,6 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t
 // RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
 
 // If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-bfloat.c
new file mode 100644
index 0000000000000..d9c6581e11f07
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-bfloat.c
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svzip1_bf16(svbfloat16_t op1, svbfloat16_t op2)
+{
+  // CHECK-LABEL: test_svzip1_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.zip1.nxv8bf16(<vscale x 8 x bfloat> %op1, <vscale x 8 x bfloat> %op2)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svzip1_bf16'}}
+  return SVE_ACLE_FUNC(svzip1,_bf16,,)(op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-fp64-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-fp64-bfloat.c
new file mode 100644
index 0000000000000..36ec40f5b91ea
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-fp64-bfloat.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -target-feature +f64mm -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svzip1_bf16(svbfloat16_t op1, svbfloat16_t op2) {
+  // CHECK-LABEL: test_svzip1_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.zip1q.nxv8bf16(<vscale x 8 x bfloat> %op1, <vscale x 8 x bfloat> %op2)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svzip1q_bf16'}}
+  return SVE_ACLE_FUNC(svzip1q, _bf16, , )(op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-fp64.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-fp64.c
index 1e9e81dc45bbe..47ced73fd8601 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-fp64.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-fp64.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1.c
index 3444ac9aa1abb..218f63764becc 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-bfloat.c
new file mode 100644
index 0000000000000..bde8800a57ca7
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-bfloat.c
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svzip2_bf16(svbfloat16_t op1, svbfloat16_t op2)
+{
+  // CHECK-LABEL: test_svzip2_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.zip2.nxv8bf16(<vscale x 8 x bfloat> %op1, <vscale x 8 x bfloat> %op2)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svzip2_bf16'}}
+  return SVE_ACLE_FUNC(svzip2,_bf16,,)(op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-fp64-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-fp64-bfloat.c
new file mode 100644
index 0000000000000..a3b04e6d67256
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-fp64-bfloat.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -target-feature +f64mm -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svzip2_bf16(svbfloat16_t op1, svbfloat16_t op2) {
+  // CHECK-LABEL: test_svzip2_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.zip2q.nxv8bf16(<vscale x 8 x bfloat> %op1, <vscale x 8 x bfloat> %op2)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // expected-warning@+1 {{implicit declaration of function 'svzip2q_bf16'}}
+  return SVE_ACLE_FUNC(svzip2q, _bf16, , )(op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-fp64.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-fp64.c
index ad33f565af89a..20a12479e8b7c 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-fp64.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-fp64.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -target-feature +f64mm -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2.c
index bba3aca2502a3..099d789586972 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2.c
@@ -1,6 +1,11 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o - %s >/dev/null 2>%t
+// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+
+// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it.
+// ASM-NOT: warning
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_asrd.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_asrd.c
index f57b94393096b..0df6a9b7b4507 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_asrd.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_asrd.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_cadd.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_cadd.c
index 39b1ac318ffd7..7d83addc05335 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_cadd.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_cadd.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_cmla.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_cmla.c
index ae507be463560..cafdd638b23e6 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_cmla.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_cmla.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_dot.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_dot.c
index 9ea5abfcb4108..29e5eba397750 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_dot.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_dot.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_ext.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_ext.c
index f8706d4e7590a..cce61cfd2b046 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_ext.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_ext.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_get2.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_get2.c
index 7bd084490c2b1..1c334cfe17cd4 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_get2.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_get2.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_get3.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_get3.c
index 91b91eb07a427..05f35f20a614f 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_get3.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_get3.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_get4.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_get4.c
index 10e5175fd6ab8..37c5d1838acb2 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_get4.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_get4.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_mla.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_mla.c
index 62249694f23e9..e107e02c7e8b2 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_mla.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_mla.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_mul.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_mul.c
index b8cb87dab9990..528999a5c1b1f 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_mul.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_mul.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfb.c
index 48fea8c540b7a..e8815a05c3a57 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfb.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfd.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfd.c
index 2efa9b6c9e138..cd15ac58d9614 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfd.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfd.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfh.c
index 16b19203cdc55..c18b66f06529b 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfh.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfw.c
index ea1d6e24419b5..6e3c27a488599 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfw.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qdecb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qdecb.c
index b8ac4ac5aa6a5..2878a6fa35b68 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qdecb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qdecb.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qdecd.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qdecd.c
index db380e7dc83ce..ac01ce0b770da 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qdecd.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qdecd.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qdech.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qdech.c
index daa87e1c2df79..adfc966363ede 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qdech.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qdech.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qdecw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qdecw.c
index 23705457131fb..316914fad06a5 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qdecw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qdecw.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qincb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qincb.c
index c4d9b473ebd5a..e2687e19240fa 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qincb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qincb.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qincd.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qincd.c
index 1905778e97acb..9ad619207647d 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qincd.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qincd.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qinch.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qinch.c
index b67c5ccdc88af..269ab07242f49 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qinch.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qinch.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qincw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qincw.c
index 7180f8bbd1649..f123b39a47f38 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qincw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_qincw.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_set2.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_set2.c
index 621e0c89926cc..5a3131b2da6b6 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_set2.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_set2.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_set3.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_set3.c
index 3b624588616bd..c4bb8cb880f27 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_set3.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_set3.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_set4.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_set4.c
index 8cc19246d620c..6036bcd159c07 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_set4.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_set4.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=note %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_tmad.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_tmad.c
index da85cb726257d..25f15402988ac 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_tmad.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_tmad.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/big_endian.c b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/big_endian.c
index fc44481c94c70..f7fdd76114af9 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/negative/big_endian.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/negative/big_endian.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64_be-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64_be-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 // expected-error@* {{Big endian is currently not supported for arm_sve.h}}
 #include <arm_sve.h>
diff --git a/clang/test/CodeGen/aarch64-sve.c b/clang/test/CodeGen/aarch64-sve.c
index 790b3a176182b..d21af74319f99 100644
--- a/clang/test/CodeGen/aarch64-sve.c
+++ b/clang/test/CodeGen/aarch64-sve.c
@@ -14,6 +14,7 @@
 // CHECK-DEBUG: cannot yet generate debug info for SVE type '__SVFloat16_t'
 // CHECK-DEBUG: cannot yet generate debug info for SVE type '__SVFloat32_t'
 // CHECK-DEBUG: cannot yet generate debug info for SVE type '__SVFloat64_t'
+// CHECK-DEBUG: cannot yet generate debug info for SVE type '__SVBFloat16_t'
 // CHECK-DEBUG: cannot yet generate debug info for SVE type '__SVBool_t'
 
 // CHECK: @ptr = global <vscale x 16 x i8>* null, align 8
@@ -28,6 +29,7 @@
 // CHECK: %f16 = alloca <vscale x 8 x half>, align 16
 // CHECK: %f32 = alloca <vscale x 4 x float>, align 16
 // CHECK: %f64 = alloca <vscale x 2 x double>, align 16
+// CHECK: %bf16 = alloca <vscale x 8 x bfloat>, align 16
 // CHECK: %b8 = alloca <vscale x 16 x i1>, align 2
 
 __SVInt8_t *ptr;
@@ -47,5 +49,7 @@ void test_locals(void) {
   __SVFloat32_t f32;
   __SVFloat64_t f64;
 
+  __SVBFloat16_t bf16;
+
   __SVBool_t b8;
 }
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aba.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aba.c
index 5ba165faf48fd..26aafead5cefe 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aba.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aba.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_abalb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_abalb.c
index 850a6722f645f..c50916c4e1316 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_abalb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_abalb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_abalt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_abalt.c
index 34a2cb915c5d7..b7480a5d89aa8 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_abalt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_abalt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_abdlb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_abdlb.c
index 2453fd99ca1ed..49d89ada0793d 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_abdlb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_abdlb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_abdlt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_abdlt.c
index dbd333ab764bb..b5bb45d1e2f9d 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_abdlt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_abdlt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_adalp.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_adalp.c
index 5d68f81099860..fa30a301b30a0 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_adalp.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_adalp.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_adclb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_adclb.c
index a02ecaaad5afb..215949fe15459 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_adclb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_adclb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_adclt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_adclt.c
index 2cee8a9dd5b3d..687574ed5d58d 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_adclt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_adclt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addhnb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addhnb.c
index 27efc36efedc6..24fdd559bf81f 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addhnb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addhnb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addhnt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addhnt.c
index 700c59d6b82a4..95137ad33a772 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addhnt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addhnt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addlb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addlb.c
index 39096523b223d..168a2c2bd0660 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addlb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addlb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addlbt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addlbt.c
index 3d47de8421871..c5d96fae7a325 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addlbt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addlbt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addlt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addlt.c
index 5d5806361e044..71cf4d7edac9c 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addlt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addlt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addp.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addp.c
index 0fd97cedb2013..a04ed5becc1cc 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addp.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addp.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addwb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addwb.c
index 85ec2f09aeb4c..00e3be0111abd 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addwb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addwb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addwt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addwt.c
index c4c40d295285b..57392da45022b 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addwt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_addwt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aesd.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aesd.c
index d037f452ce76d..a170585dd58a0 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aesd.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aesd.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_AES -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_AES -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aese.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aese.c
index 19235555e5e7f..d9b5f3eef2df8 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aese.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aese.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_AES -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_AES -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aesimc.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aesimc.c
index e3b1952e484dd..a79317c37f569 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aesimc.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aesimc.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_AES -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_AES -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aesmc.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aesmc.c
index fa53d5a51e760..5af2a29302ecf 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aesmc.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_aesmc.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_AES -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_AES -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bcax.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bcax.c
index 3cde14ad65df1..8547ff4d3fdd0 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bcax.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bcax.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bdep.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bdep.c
index eb5890b3f65f7..6543b30b417f8 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bdep.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bdep.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_BITPERM -triple aarch64-none-linux-gnu -target-feature +sve2-bitperm -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_BITPERM -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-bitperm -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2-bitperm -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-bitperm -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bext.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bext.c
index 8178506c37650..714364b6b4cbb 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bext.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bext.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_BITPERM -triple aarch64-none-linux-gnu -target-feature +sve2-bitperm -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_BITPERM -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-bitperm -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2-bitperm -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-bitperm -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bgrp.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bgrp.c
index aebfa11663d00..10d9fa85fa9f6 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bgrp.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bgrp.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_BITPERM -triple aarch64-none-linux-gnu -target-feature +sve2-bitperm -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_BITPERM -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-bitperm -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2-bitperm -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-bitperm -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bsl.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bsl.c
index 982d04b7d7cf1..fae32cbc9ba01 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bsl.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bsl.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bsl1n.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bsl1n.c
index 85c0951693e92..787a3f2ec9631 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bsl1n.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bsl1n.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bsl2n.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bsl2n.c
index a47dc2d190fed..d9771e31db874 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bsl2n.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_bsl2n.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cadd.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cadd.c
index 01c14f8636663..aada5acbd9e69 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cadd.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cadd.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cdot.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cdot.c
index c8094c8948924..bfbbee64eb3c8 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cdot.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cdot.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cmla.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cmla.c
index 4ce43feb61147..b9089aacb4d89 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cmla.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cmla.c
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cvtlt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cvtlt.c
index 99344a3ece2e2..e54128107c868 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cvtlt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cvtlt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cvtnt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cvtnt.c
index ee57e8de12b50..fd64108463b81 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cvtnt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cvtnt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cvtx.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cvtx.c
index a499a87ee2737..ba2cc769ebca7 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cvtx.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cvtx.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cvtxnt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cvtxnt.c
index a006c8e0c5502..d4ef80781a4da 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cvtxnt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_cvtxnt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_eor3.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_eor3.c
index 634cb65375952..79cccc7dc441e 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_eor3.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_eor3.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_eorbt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_eorbt.c
index 865ac2e444e57..586fea961439e 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_eorbt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_eorbt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_eortb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_eortb.c
index 6734d9fa0c6da..81abf27cf5e50 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_eortb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_eortb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_hadd.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_hadd.c
index 6a9f3f347f633..37d9dba6b6a57 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_hadd.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_hadd.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_histcnt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_histcnt.c
index 3f2138bc48c43..2572376fe5c48 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_histcnt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_histcnt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_histseg.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_histseg.c
index 230075fe7c6f4..b38c82a75911f 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_histseg.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_histseg.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_hsub.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_hsub.c
index 386bae82e07be..8bb1cdb46be83 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_hsub.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_hsub.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_hsubr.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_hsubr.c
index efc1544c6e3a1..f2fe38fc392af 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_hsubr.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_hsubr.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1.c
index 1f00744afe51b..fa055476f87d7 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1sb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1sb.c
index cd87d4491bc23..cb52acf5df9cc 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1sb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1sb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1sh.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1sh.c
index a595f70a9572d..0ffe8e337ffee 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1sh.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1sh.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1sw.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1sw.c
index 921861e4bcffa..e1e244ef56209 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1sw.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1sw.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1ub.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1ub.c
index 05fbfebb9a973..47b14d08657f9 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1ub.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1ub.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1uh.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1uh.c
index 5627060b878b5..cf3c8afd4caa9 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1uh.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1uh.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1uw.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1uw.c
index e64ec26be8dee..b90c0802406b6 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1uw.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_ldnt1uw.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_logb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_logb.c
index ab0a508b5a27f..92b4c4a18ba17 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_logb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_logb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_match.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_match.c
index 79591ad7b7365..456e1e05af7e6 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_match.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_match.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_maxnmp.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_maxnmp.c
index 66e4c9db60eae..5b7d487f70f80 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_maxnmp.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_maxnmp.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_maxp.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_maxp.c
index 3f09a62e28fc2..59bb50e695ffc 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_maxp.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_maxp.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_minnmp.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_minnmp.c
index 75d9319ce01a5..3480e7cf1a1ac 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_minnmp.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_minnmp.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_minp.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_minp.c
index 9f8c34bd21b5b..3a687870839c8 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_minp.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_minp.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mla.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mla.c
index b130780648c4e..e2585dbecc6b4 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mla.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mla.c
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mlalb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mlalb.c
index e4f80eb639dec..1e566a9bd9277 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mlalb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mlalb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mlalt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mlalt.c
index 49f67536d1767..2de60cd819f74 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mlalt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mlalt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mls.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mls.c
index 13ecb1fadfea5..b52cdd2efbaab 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mls.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mls.c
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mlslb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mlslb.c
index 429569e306d82..8f10ae47340ec 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mlslb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mlslb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mlslt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mlslt.c
index 4bca3ae5aefb0..e9cf85a739624 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mlslt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mlslt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_movlb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_movlb.c
index ba0d80deaf8af..f3ec86b843068 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_movlb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_movlb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_movlt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_movlt.c
index 71a3ea179a914..de9cb692ac80c 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_movlt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_movlt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mul.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mul.c
index fff05ca585fec..dca5752dee8a8 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mul.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mul.c
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mullb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mullb.c
index fca203bb60b91..a9bd191407450 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mullb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mullb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mullt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mullt.c
index 9590d4aa895c8..04055adfb431e 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mullt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_mullt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_nbsl.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_nbsl.c
index e20ca82990558..ce4072a27382e 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_nbsl.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_nbsl.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_nmatch.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_nmatch.c
index d55e7e6db1be0..2b3be07b96054 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_nmatch.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_nmatch.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmul.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmul.c
index 05dca4ecc4200..edca8a90d7ebe 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmul.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmul.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmullb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmullb.c
index a12d1fd09ffd0..647ad310aa511 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmullb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmullb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmullb_128.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmullb_128.c
index 2c8ff2cb000eb..5268cf80cf125 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmullb_128.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmullb_128.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_AES -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_AES -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmullt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmullt.c
index 5b4b05aa72fb5..3b7c443a8ae1e 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmullt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmullt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmullt_128.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmullt_128.c
index 69726c90d10cd..e9947f486ed40 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmullt_128.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_pmullt_128.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_AES -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_AES -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-aes -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qabs.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qabs.c
index 94442807d49cc..02dd525839a56 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qabs.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qabs.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qadd.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qadd.c
index 2c035b6a77d0b..8099471d8edec 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qadd.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qadd.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qcadd.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qcadd.c
index 028283b1e3c89..c6cd3428136f9 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qcadd.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qcadd.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlalb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlalb.c
index f1658279f15f8..2b237aa473504 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlalb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlalb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlalbt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlalbt.c
index 88f141838938e..3123182cad31e 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlalbt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlalbt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlalt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlalt.c
index f1554f37c8767..aeb9b083d14b6 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlalt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlalt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlslb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlslb.c
index 29e1fcbb2c793..bd072342d478c 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlslb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlslb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlslbt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlslbt.c
index 0bcc24e57022f..cd521d76a75f2 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlslbt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlslbt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlslt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlslt.c
index f3d78c06b3a47..e8398784b621b 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlslt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmlslt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmulh.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmulh.c
index 0de2aad076a60..f9a964940020c 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmulh.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmulh.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmullb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmullb.c
index e7101e43ad20a..e3b21aee8c976 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmullb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmullb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmullt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmullt.c
index 4943b503c6a8c..b8b3144c8cbdb 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmullt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qdmullt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qneg.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qneg.c
index a734f741d780b..195e8c4bbf426 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qneg.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qneg.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrdcmlah.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrdcmlah.c
index 33853acc600ff..9f125868143ab 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrdcmlah.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrdcmlah.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrdmlah.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrdmlah.c
index a142609407a6a..3f2c8369db3ac 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrdmlah.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrdmlah.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrdmlsh.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrdmlsh.c
index 921bca4c76699..a944719c46db4 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrdmlsh.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrdmlsh.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrdmulh.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrdmulh.c
index 07efb8ff0f8cd..57a26d4f9acf8 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrdmulh.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrdmulh.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshl.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshl.c
index e9233ab9804c8..dbfbc3f8d47aa 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshl.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshl.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshrnb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshrnb.c
index 991739d2ddc0d..de5e76b410e23 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshrnb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshrnb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshrnt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshrnt.c
index 33686277aa2fe..42ac313a86131 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshrnt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshrnt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshrunb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshrunb.c
index 7e3777819e0ee..dc84be5365d74 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshrunb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshrunb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshrunt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshrunt.c
index 5e96435b265b6..857079c7392ce 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshrunt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qrshrunt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshl.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshl.c
index d8c9e19afcd06..1527fd80e1a7e 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshl.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshl.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshlu.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshlu.c
index d61959573632e..8f7110a70890d 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshlu.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshlu.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshrnb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshrnb.c
index d4c1719bc0362..794301ee0600c 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshrnb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshrnb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshrnt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshrnt.c
index 5a68834978aa4..669a52f8ce4f8 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshrnt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshrnt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshrunb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshrunb.c
index e012484e86c57..f9a2843dcf7c5 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshrunb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshrunb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshrunt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshrunt.c
index d6db3fb8d8f60..07d68ebf3f13d 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshrunt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qshrunt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qsub.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qsub.c
index 63bbb8c0390eb..6aa030e01d3a6 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qsub.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qsub.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qsubr.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qsubr.c
index c0146876d648d..9f8f5cd24ff70 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qsubr.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qsubr.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qxtnb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qxtnb.c
index 1586d22fb86e5..c8a75a70f2970 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qxtnb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qxtnb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qxtnt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qxtnt.c
index 68c99bcda1d43..f7beb72880c55 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qxtnt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qxtnt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qxtunb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qxtunb.c
index 138fb51d85ba4..5c2ea3d163b81 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qxtunb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qxtunb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qxtunt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qxtunt.c
index 6f7a96e3b243d..49020b868c81a 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qxtunt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_qxtunt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_raddhnb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_raddhnb.c
index b8504b794d489..5a642a14eb959 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_raddhnb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_raddhnb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_raddhnt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_raddhnt.c
index fa313ef9cbabd..47c76d9209cb3 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_raddhnt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_raddhnt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rax1.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rax1.c
index 01ce8ba78b6d6..666ec9378bab6 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rax1.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rax1.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_SHA3 -triple aarch64-none-linux-gnu -target-feature +sve2-sha3 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_SHA3 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-sha3 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2-sha3 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-sha3 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_recpe.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_recpe.c
index 7f9c804746c11..a068e15301331 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_recpe.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_recpe.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rhadd.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rhadd.c
index e3731e5215689..b8f300d6ed1b2 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rhadd.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rhadd.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rshl.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rshl.c
index c17ab16632551..cfb2b058bc6db 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rshl.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rshl.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rshr.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rshr.c
index be46241417532..2c351cebf9558 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rshr.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rshr.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rshrnb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rshrnb.c
index 06cb1745d5418..8dcb355a138b5 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rshrnb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rshrnb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rshrnt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rshrnt.c
index 559d6c912c47e..ceb11ec9abf68 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rshrnt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rshrnt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rsqrte.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rsqrte.c
index 02fb5f61e9d27..19d1cbd14bc43 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rsqrte.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rsqrte.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rsra.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rsra.c
index 9e42b76032310..3196775cbfd3d 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rsra.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rsra.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rsubhnb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rsubhnb.c
index e424bef582188..f92817198b921 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rsubhnb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rsubhnb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rsubhnt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rsubhnt.c
index 8a3fc54a4a9b2..36bf0225f4098 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rsubhnt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_rsubhnt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sbclb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sbclb.c
index 37f4ae268c976..e40ab98212a0d 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sbclb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sbclb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sbclt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sbclt.c
index 0fcc4f4f94d17..2c05ae860eba8 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sbclt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sbclt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_shllb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_shllb.c
index e0b10d0959287..0e1d348bcba7a 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_shllb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_shllb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_shllt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_shllt.c
index 6faf3c819c7f9..313ad9aa8c381 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_shllt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_shllt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_shrnb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_shrnb.c
index 88403c5ede584..8c8ee83da4567 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_shrnb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_shrnb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_shrnt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_shrnt.c
index 35b0c23d26c8d..69eafcf7200be 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_shrnt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_shrnt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sli.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sli.c
index e25c9abf6e1d9..19361c6ab6099 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sli.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sli.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sm4e.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sm4e.c
index aab77947db3e5..b7f343bfd7f34 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sm4e.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sm4e.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_SM4 -triple aarch64-none-linux-gnu -target-feature +sve2-sm4 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_SM4 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-sm4 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2-sm4 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-sm4 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sm4ekey.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sm4ekey.c
index d2eb88bbda9f4..c2b72406de84f 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sm4ekey.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sm4ekey.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_SM4 -triple aarch64-none-linux-gnu -target-feature +sve2-sm4 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2_SM4 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-sm4 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2-sm4 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2-sm4 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sqadd.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sqadd.c
index 78f77adc33788..ae84e010c89af 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sqadd.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sqadd.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sra.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sra.c
index d6535a463430b..6a5f4ddc83c0c 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sra.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sra.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sri.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sri.c
index 7abb04d291c4a..276ceb2c8e285 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sri.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sri.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_stnt1.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_stnt1.c
index e088836648c12..665d1f2720bcc 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_stnt1.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_stnt1.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_stnt1b.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_stnt1b.c
index d94d6b4f34e37..a367406548ae8 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_stnt1b.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_stnt1b.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_stnt1h.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_stnt1h.c
index f457ada228f3c..9db44153732a6 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_stnt1h.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_stnt1h.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_stnt1w.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_stnt1w.c
index 98f1ecd544071..6b7ec77c6902c 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_stnt1w.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_stnt1w.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subhnb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subhnb.c
index 63bc103ce5135..388caf852183a 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subhnb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subhnb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subhnt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subhnt.c
index a2c8d1cd36a9a..42783c2d90fbc 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subhnt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subhnt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sublb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sublb.c
index e9b7b286fd912..0e0069051d43f 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sublb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sublb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sublbt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sublbt.c
index 36c414293b558..1b84cd1046e05 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sublbt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sublbt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sublt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sublt.c
index 02cc433ac2183..c21f03b76e1b2 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sublt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_sublt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subltb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subltb.c
index a3ad6db87a230..4b4a62e13f82d 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subltb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subltb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subwb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subwb.c
index e73e3481d6c9c..410e5f79bec50 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subwb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subwb.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subwt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subwt.c
index 08b9e97ed9a07..d927b1b6231aa 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subwt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_subwt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_tbl2-bfloat.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_tbl2-bfloat.c
new file mode 100644
index 0000000000000..e3169988466df
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_tbl2-bfloat.c
@@ -0,0 +1,26 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +bf16 -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svtbl2_bf16(svbfloat16x2_t data, svuint16_t indices) {
+  // CHECK-LABEL: test_svtbl2_bf16
+  // CHECK-DAG: %[[V0:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> %data, i32 0)
+  // CHECK-DAG: %[[V1:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> %data, i32 1)
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tbl2.nxv8bf16(<vscale x 8 x bfloat> %[[V0]], <vscale x 8 x bfloat> %[[V1]], <vscale x 8 x i16> %indices)
+  // CHECK-NEXT: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // overload-warning@+2 {{implicit declaration of function 'svtbl2'}}
+  // expected-warning@+1 {{implicit declaration of function 'svtbl2_bf16'}}
+  return SVE_ACLE_FUNC(svtbl2, _bf16, , )(data, indices);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_tbl2.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_tbl2.c
index d74b17f726630..4e4f70cb47a04 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_tbl2.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_tbl2.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_tbx-bfloat.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_tbx-bfloat.c
new file mode 100644
index 0000000000000..1b8cac3656f1c
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_tbx-bfloat.c
@@ -0,0 +1,24 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +bf16 -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svbfloat16_t test_svtbx_bf16(svbfloat16_t fallback, svbfloat16_t data, svuint16_t indices) {
+  // CHECK-LABEL: @test_svtbx_bf16(
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tbx.nxv8bf16(<vscale x 8 x bfloat> %fallback, <vscale x 8 x bfloat> %data, <vscale x 8 x i16> %indices)
+  // CHECK: ret <vscale x 8 x bfloat> %[[INTRINSIC]]
+  // overload-warning@+2 {{implicit declaration of function 'svtbx'}}
+  // expected-warning@+1 {{implicit declaration of function 'svtbx_bf16'}}
+  return SVE_ACLE_FUNC(svtbx, _bf16, , )(fallback, data, indices);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_tbx.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_tbx.c
index 78cc1b016417f..c3c6f5f0447b6 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_tbx.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_tbx.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_uqadd.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_uqadd.c
index 19245bd0425e5..090df7318edd2 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_uqadd.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_uqadd.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilege.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilege.c
index 72f331609b8fa..bbfcdce54a85e 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilege.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilege.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilegt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilegt.c
index 464f7c111e343..ea263e4ad1036 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilegt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilegt.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilerw-bfloat.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilerw-bfloat.c
new file mode 100644
index 0000000000000..9ea4ebc9ddceb
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilerw-bfloat.c
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve2 -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbool_t test_svwhilerw_bf16(const bfloat16_t *op1, const bfloat16_t *op2)
+{
+  // CHECK-LABEL: test_svwhilerw_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.whilerw.h.nxv8i1.p0bf16(bfloat* %op1, bfloat* %op2)
+  // CHECK: %[[INTRINSIC_REINT:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> %[[INTRINSIC]])
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC_REINT]]
+  // overload-warning@+2 {{implicit declaration of function 'svwhilerw'}}
+  // expected-warning@+1 {{implicit declaration of function 'svwhilerw_bf16'}}
+  return SVE_ACLE_FUNC(svwhilerw,_bf16,,)(op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilerw.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilerw.c
index cfb2600aaf05d..0fcd6be3043d1 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilerw.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilerw.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilewr-bfloat.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilewr-bfloat.c
new file mode 100644
index 0000000000000..a8d09379b7163
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilewr-bfloat.c
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve2 -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+svbool_t test_svwhilewr_bf16(const bfloat16_t *op1, const bfloat16_t *op2)
+{
+  // CHECK-LABEL: test_svwhilewr_bf16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.whilewr.h.nxv8i1.p0bf16(bfloat* %op1, bfloat* %op2)
+  // CHECK: %[[INTRINSIC_REINT:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> %[[INTRINSIC]])
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC_REINT]]
+  // overload-warning@+2 {{implicit declaration of function 'svwhilewr'}}
+  // expected-warning@+1 {{implicit declaration of function 'svwhilewr_bf16'}}
+  return SVE_ACLE_FUNC(svwhilewr,_bf16,,)(op1, op2);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilewr.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilewr.c
index 8495a30cd801c..4541c31971485 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilewr.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_whilewr.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_xar.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_xar.c
index 5cd7cf0c4fd47..95542c07a44c6 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_xar.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_xar.c
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify=overload -verify-ignore-unexpected=error %s
 
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_cadd.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_cadd.c
index 70e8f3192a82f..12d695d1a4a7b 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_cadd.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_cadd.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_cdot.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_cdot.c
index cb177b943cd5b..88b662d089f9d 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_cdot.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_cdot.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=warning %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=warning %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=warning %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=warning %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_cmla.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_cmla.c
index 338e1f10f8dc3..4c0592addb5d8 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_cmla.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_cmla.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mla.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mla.c
index 969317db8a0b0..ee679750b9f6b 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mla.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mla.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mlalb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mlalb.c
index 17b8282c60010..da67b6de26bbf 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mlalb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mlalb.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mlalt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mlalt.c
index f93de7cdf8d23..8f2b4e4897c07 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mlalt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mlalt.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mls.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mls.c
index 1467931a4b312..e727f5745bd66 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mls.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mls.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mlslb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mlslb.c
index d6c44447dd02d..cda90d870b687 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mlslb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mlslb.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mlslt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mlslt.c
index 5ac82f7c48d7b..c51ccff08b792 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mlslt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mlslt.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mul.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mul.c
index 73d7d6f79d345..fe407224e4dfb 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mul.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mul.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mullb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mullb.c
index 010e969d5a580..9902a4a89618d 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mullb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mullb.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mullt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mullt.c
index 3dca8dd37e34b..f29b01d278738 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mullt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_mullt.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qcadd.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qcadd.c
index adf04c9928468..8dea17f829a8f 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qcadd.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qcadd.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmlalb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmlalb.c
index fc40b7010ceb6..24adade56741d 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmlalb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmlalb.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmlalt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmlalt.c
index 32d28bfaf87a7..b3dd7bd940dc1 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmlalt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmlalt.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmlslb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmlslb.c
index 6e5203d3c45d5..250e6755794e6 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmlslb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmlslb.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmlslt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmlslt.c
index f47dc027c00d1..9d38a7849cfdd 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmlslt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmlslt.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmulh.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmulh.c
index 60207813d8a39..58401229adaae 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmulh.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmulh.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmullb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmullb.c
index f40c221f286ab..5b22db2f8ed09 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmullb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmullb.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmullt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmullt.c
index 85b96b4ef7772..597ac7e08a679 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmullt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qdmullt.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrdcmlah.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrdcmlah.c
index 547da4c17ed99..ce1bafe877559 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrdcmlah.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrdcmlah.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrdmlah.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrdmlah.c
index 94a5f351d6e80..266d658af31bc 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrdmlah.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrdmlah.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrdmlsh.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrdmlsh.c
index 5ca6e6e1917a9..8074a605041b4 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrdmlsh.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrdmlsh.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrdmulh.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrdmulh.c
index ead75193692d1..14834aa42596b 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrdmulh.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrdmulh.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrshrnb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrshrnb.c
index bbbbfc418cc96..f92fe6011e7d5 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrshrnb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrshrnb.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrshrnt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrshrnt.c
index 4a323b56c1b37..0c791e005bfdc 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrshrnt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrshrnt.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrshrunb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrshrunb.c
index ff45e95c5d520..541ae62837d1b 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrshrunb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrshrunb.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrshrunt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrshrunt.c
index 849500d3a4cb3..4b6dc9e7f164f 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrshrunt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qrshrunt.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshlu.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshlu.c
index 6b7e5a5f69376..ca26aec41bdb4 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshlu.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshlu.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshrnb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshrnb.c
index dc7189583b8ca..19d4d8a97a60b 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshrnb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshrnb.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshrnt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshrnt.c
index 0e05ce5093237..18d9ccc04a425 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshrnt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshrnt.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshrunb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshrunb.c
index e4da43cb607b4..bee61489d4d61 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshrunb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshrunb.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshrunt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshrunt.c
index 4caf7bc3c6d2c..8b7c202fba80e 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshrunt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_qshrunt.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_rshr.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_rshr.c
index 2f7513db3a347..9da56a0163ce0 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_rshr.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_rshr.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_rshrnb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_rshrnb.c
index 4c9fd97eca03f..dfba6aeb1d760 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_rshrnb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_rshrnb.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_rshrnt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_rshrnt.c
index 99c751fafbe6c..c859022738528 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_rshrnt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_rshrnt.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_rsra.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_rsra.c
index e279a384e01d9..acf5065a9348f 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_rsra.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_rsra.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_shllb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_shllb.c
index 79a7b59299371..5e4af80a5bb9d 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_shllb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_shllb.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_shllt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_shllt.c
index e989544048f57..6b67e2dd45c34 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_shllt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_shllt.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_shrnb.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_shrnb.c
index 60cf3a1154ed0..f783779ac2232 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_shrnb.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_shrnb.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_shrnt.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_shrnt.c
index d6d0a23ae5049..8bca15b643170 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_shrnt.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_shrnt.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_sli.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_sli.c
index f8846952dd4a6..ce159bcf679ab 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_sli.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_sli.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_sra.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_sra.c
index efd4a70c37ec5..b70d4351ad887 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_sra.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_sra.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_sri.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_sri.c
index 48aea5c124276..19acfebc661a7 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_sri.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_sri.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_xar.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_xar.c
index 2d96ab558781f..d0d7f5d0dc133 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_xar.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/negative/acle_sve2_xar.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
-// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE2 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2 -fallow-half-arguments-and-returns -fsyntax-only -verify %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
diff --git a/clang/test/CodeGen/aarch64-v8.2a-fp16-intrinsics-constrained.c b/clang/test/CodeGen/aarch64-v8.2a-fp16-intrinsics-constrained.c
index 473d3ba53e968..d42fb1e02cade 100644
--- a/clang/test/CodeGen/aarch64-v8.2a-fp16-intrinsics-constrained.c
+++ b/clang/test/CodeGen/aarch64-v8.2a-fp16-intrinsics-constrained.c
@@ -12,7 +12,7 @@
 // RUN: %clang_cc1 -triple arm64-none-linux-gnu -target-feature +fullfp16 \
 // RUN: -ffp-exception-behavior=strict \
 // RUN: -fallow-half-arguments-and-returns -S -disable-O0-optnone -o - %s \
-// RUN: | FileCheck -vv --check-prefix=COMMON --check-prefix=CHECK-ASM %s
+// RUN: | FileCheck --check-prefix=COMMON --check-prefix=CHECK-ASM %s
 
 // REQUIRES: aarch64-registered-target
 
diff --git a/clang/test/CodeGen/aarch64-v8.2a-neon-intrinsics-constrained.c b/clang/test/CodeGen/aarch64-v8.2a-neon-intrinsics-constrained.c
index 6058e6f92832f..478c4a27c3e7a 100644
--- a/clang/test/CodeGen/aarch64-v8.2a-neon-intrinsics-constrained.c
+++ b/clang/test/CodeGen/aarch64-v8.2a-neon-intrinsics-constrained.c
@@ -4,6 +4,7 @@
 // RUN: | FileCheck --check-prefix=COMMON --check-prefix=COMMONIR --check-prefix=UNCONSTRAINED %s
 // RUN: %clang_cc1 -triple arm64-none-linux-gnu -target-feature +neon -target-feature +fullfp16 -target-feature +v8.2a\
 // RUN: -ffp-exception-behavior=strict \
+// RUN: -fexperimental-strict-floating-point \
 // RUN: -fallow-half-arguments-and-returns -flax-vector-conversions=none -S -disable-O0-optnone -emit-llvm -o - %s \
 // RUN: | opt -S -mem2reg \
 // RUN: | FileCheck --check-prefix=COMMON --check-prefix=COMMONIR --check-prefix=CONSTRAINED %s
@@ -13,6 +14,7 @@
 // RUN: | FileCheck --check-prefix=COMMON --check-prefix=CHECK-ASM %s
 // RUN: %clang_cc1 -triple arm64-none-linux-gnu -target-feature +neon -target-feature +fullfp16 -target-feature +v8.2a\
 // RUN: -ffp-exception-behavior=strict \
+// RUN: -fexperimental-strict-floating-point \
 // RUN: -fallow-half-arguments-and-returns -flax-vector-conversions=none -S -disable-O0-optnone -emit-llvm -o - %s \
 // RUN: | opt -S -mem2reg | llc -o=- - \
 // RUN: | FileCheck --check-prefix=COMMON --check-prefix=CHECK-ASM %s
diff --git a/clang/test/CodeGen/align_value.cpp b/clang/test/CodeGen/align_value.cpp
index 3badcd74f46d6..acbfbaf2ba5c7 100644
--- a/clang/test/CodeGen/align_value.cpp
+++ b/clang/test/CodeGen/align_value.cpp
@@ -1,103 +1,192 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // RUN: %clang_cc1 -triple x86_64-unknown-unknown -emit-llvm -o - %s | FileCheck %s
 
 typedef double * __attribute__((align_value(64))) aligned_double;
 
+// CHECK-LABEL: define {{[^@]+}}@_Z3fooPdS_Rd
+// CHECK-SAME: (double* align 64 [[X:%.*]], double* align 32 [[Y:%.*]], double* nonnull align 128 dereferenceable(8) [[Z:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca double*, align 8
+// CHECK-NEXT:    [[Y_ADDR:%.*]] = alloca double*, align 8
+// CHECK-NEXT:    [[Z_ADDR:%.*]] = alloca double*, align 8
+// CHECK-NEXT:    store double* [[X]], double** [[X_ADDR]], align 8
+// CHECK-NEXT:    store double* [[Y]], double** [[Y_ADDR]], align 8
+// CHECK-NEXT:    store double* [[Z]], double** [[Z_ADDR]], align 8
+// CHECK-NEXT:    ret void
+//
 void foo(aligned_double x, double * y __attribute__((align_value(32))),
          double & z __attribute__((align_value(128)))) { };
-// CHECK: define void @_Z3fooPdS_Rd(double* align 64 %x, double* align 32 %y, double* nonnull align 128 dereferenceable(8) %z)
 
 struct ad_struct {
   aligned_double a;
 };
 
+// CHECK-LABEL: define {{[^@]+}}@_Z3fooR9ad_struct
+// CHECK-SAME: (%struct.ad_struct* nonnull align 8 dereferenceable(8) [[X:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca %struct.ad_struct*, align 8
+// CHECK-NEXT:    store %struct.ad_struct* [[X]], %struct.ad_struct** [[X_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load %struct.ad_struct*, %struct.ad_struct** [[X_ADDR]], align 8
+// CHECK-NEXT:    [[A:%.*]] = getelementptr inbounds [[STRUCT_AD_STRUCT:%.*]], %struct.ad_struct* [[TMP0]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP1:%.*]] = load double*, double** [[A]], align 8
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint double* [[TMP1]] to i64
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 63
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    ret double* [[TMP1]]
+//
 double *foo(ad_struct& x) {
-// CHECK-LABEL: @_Z3fooR9ad_struct
 
-// CHECK: [[PTRINT1:%.+]] = ptrtoint
-// CHECK: [[MASKEDPTR1:%.+]] = and i64 [[PTRINT1]], 63
-// CHECK: [[MASKCOND1:%.+]] = icmp eq i64 [[MASKEDPTR1]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND1]])
   return x.a;
 }
 
+// CHECK-LABEL: define {{[^@]+}}@_Z3gooP9ad_struct
+// CHECK-SAME: (%struct.ad_struct* [[X:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca %struct.ad_struct*, align 8
+// CHECK-NEXT:    store %struct.ad_struct* [[X]], %struct.ad_struct** [[X_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load %struct.ad_struct*, %struct.ad_struct** [[X_ADDR]], align 8
+// CHECK-NEXT:    [[A:%.*]] = getelementptr inbounds [[STRUCT_AD_STRUCT:%.*]], %struct.ad_struct* [[TMP0]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP1:%.*]] = load double*, double** [[A]], align 8
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint double* [[TMP1]] to i64
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 63
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    ret double* [[TMP1]]
+//
 double *goo(ad_struct *x) {
-// CHECK-LABEL: @_Z3gooP9ad_struct
 
-// CHECK: [[PTRINT2:%.+]] = ptrtoint
-// CHECK: [[MASKEDPTR2:%.+]] = and i64 [[PTRINT2]], 63
-// CHECK: [[MASKCOND2:%.+]] = icmp eq i64 [[MASKEDPTR2]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND2]])
   return x->a;
 }
 
+// CHECK-LABEL: define {{[^@]+}}@_Z3barPPd
+// CHECK-SAME: (double** [[X:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca double**, align 8
+// CHECK-NEXT:    store double** [[X]], double*** [[X_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load double**, double*** [[X_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load double*, double** [[TMP0]], align 8
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint double* [[TMP1]] to i64
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 63
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    ret double* [[TMP1]]
+//
 double *bar(aligned_double *x) {
-// CHECK-LABEL: @_Z3barPPd
 
-// CHECK: [[PTRINT3:%.+]] = ptrtoint
-// CHECK: [[MASKEDPTR3:%.+]] = and i64 [[PTRINT3]], 63
-// CHECK: [[MASKCOND3:%.+]] = icmp eq i64 [[MASKEDPTR3]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND3]])
   return *x;
 }
 
+// CHECK-LABEL: define {{[^@]+}}@_Z3carRPd
+// CHECK-SAME: (double** nonnull align 8 dereferenceable(8) [[X:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca double**, align 8
+// CHECK-NEXT:    store double** [[X]], double*** [[X_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load double**, double*** [[X_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load double*, double** [[TMP0]], align 8
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint double* [[TMP1]] to i64
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 63
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    ret double* [[TMP1]]
+//
 double *car(aligned_double &x) {
-// CHECK-LABEL: @_Z3carRPd
 
-// CHECK: [[PTRINT4:%.+]] = ptrtoint
-// CHECK: [[MASKEDPTR4:%.+]] = and i64 [[PTRINT4]], 63
-// CHECK: [[MASKCOND4:%.+]] = icmp eq i64 [[MASKEDPTR4]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND4]])
   return x;
 }
 
+// CHECK-LABEL: define {{[^@]+}}@_Z3darPPd
+// CHECK-SAME: (double** [[X:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca double**, align 8
+// CHECK-NEXT:    store double** [[X]], double*** [[X_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load double**, double*** [[X_ADDR]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double*, double** [[TMP0]], i64 5
+// CHECK-NEXT:    [[TMP1:%.*]] = load double*, double** [[ARRAYIDX]], align 8
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint double* [[TMP1]] to i64
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 63
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    ret double* [[TMP1]]
+//
 double *dar(aligned_double *x) {
-// CHECK-LABEL: @_Z3darPPd
 
-// CHECK: [[PTRINT5:%.+]] = ptrtoint
-// CHECK: [[MASKEDPTR5:%.+]] = and i64 [[PTRINT5]], 63
-// CHECK: [[MASKCOND5:%.+]] = icmp eq i64 [[MASKEDPTR5]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND5]])
   return x[5];
 }
 
 aligned_double eep();
+// CHECK-LABEL: define {{[^@]+}}@_Z3retv() #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CALL:%.*]] = call double* @_Z3eepv()
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint double* [[CALL]] to i64
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 63
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    ret double* [[CALL]]
+//
 double *ret() {
-// CHECK-LABEL: @_Z3retv
 
-// CHECK: [[PTRINT6:%.+]] = ptrtoint
-// CHECK: [[MASKEDPTR6:%.+]] = and i64 [[PTRINT6]], 63
-// CHECK: [[MASKCOND6:%.+]] = icmp eq i64 [[MASKEDPTR6]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND6]])
   return eep();
 }
 
+// CHECK-LABEL: define {{[^@]+}}@_Z3no1PPd
+// CHECK-SAME: (double** [[X:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca double**, align 8
+// CHECK-NEXT:    store double** [[X]], double*** [[X_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load double**, double*** [[X_ADDR]], align 8
+// CHECK-NEXT:    ret double** [[TMP0]]
+//
 double **no1(aligned_double *x) {
-// CHECK-LABEL: @_Z3no1PPd
   return x;
-// CHECK-NOT: call void @llvm.assume
 }
 
+// CHECK-LABEL: define {{[^@]+}}@_Z3no2RPd
+// CHECK-SAME: (double** nonnull align 8 dereferenceable(8) [[X:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca double**, align 8
+// CHECK-NEXT:    store double** [[X]], double*** [[X_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load double**, double*** [[X_ADDR]], align 8
+// CHECK-NEXT:    ret double** [[TMP0]]
+//
 double *&no2(aligned_double &x) {
-// CHECK-LABEL: @_Z3no2RPd
   return x;
-// CHECK-NOT: call void @llvm.assume
 }
 
+// CHECK-LABEL: define {{[^@]+}}@_Z3no3RPd
+// CHECK-SAME: (double** nonnull align 8 dereferenceable(8) [[X:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca double**, align 8
+// CHECK-NEXT:    store double** [[X]], double*** [[X_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load double**, double*** [[X_ADDR]], align 8
+// CHECK-NEXT:    ret double** [[TMP0]]
+//
 double **no3(aligned_double &x) {
-// CHECK-LABEL: @_Z3no3RPd
   return &x;
-// CHECK-NOT: call void @llvm.assume
 }
 
+// CHECK-LABEL: define {{[^@]+}}@_Z3no3Pd
+// CHECK-SAME: (double* align 64 [[X:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca double*, align 8
+// CHECK-NEXT:    store double* [[X]], double** [[X_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load double*, double** [[X_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load double, double* [[TMP0]], align 8
+// CHECK-NEXT:    ret double [[TMP1]]
+//
 double no3(aligned_double x) {
-// CHECK-LABEL: @_Z3no3Pd
   return *x;
-// CHECK-NOT: call void @llvm.assume
 }
 
+// CHECK-LABEL: define {{[^@]+}}@_Z3no4Pd
+// CHECK-SAME: (double* align 64 [[X:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca double*, align 8
+// CHECK-NEXT:    store double* [[X]], double** [[X_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load double*, double** [[X_ADDR]], align 8
+// CHECK-NEXT:    ret double* [[TMP0]]
+//
 double *no4(aligned_double x) {
-// CHECK-LABEL: @_Z3no4Pd
   return x;
-// CHECK-NOT: call void @llvm.assume
 }
 
diff --git a/clang/test/CodeGen/alloc-align-attr.c b/clang/test/CodeGen/alloc-align-attr.c
index 6294450d04445..9517c50dbb1db 100644
--- a/clang/test/CodeGen/alloc-align-attr.c
+++ b/clang/test/CodeGen/alloc-align-attr.c
@@ -1,57 +1,90 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // RUN: %clang_cc1 -triple x86_64-pc-linux -emit-llvm -o - %s | FileCheck %s
 
 __INT32_TYPE__*m1(__INT32_TYPE__ i) __attribute__((alloc_align(1)));
 
 // Condition where parameter to m1 is not size_t.
+// CHECK-LABEL: define {{[^@]+}}@test1
+// CHECK-SAME: (i32 [[A:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 [[A]], i32* [[A_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A_ADDR]], align 4
+// CHECK-NEXT:    [[CALL:%.*]] = call i32* @m1(i32 [[TMP0]])
+// CHECK-NEXT:    [[ALIGNMENTCAST:%.*]] = zext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[MASK:%.*]] = sub i64 [[ALIGNMENTCAST]], 1
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[CALL]] to i64
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], [[MASK]]
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[CALL]], align 4
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
 __INT32_TYPE__ test1(__INT32_TYPE__ a) {
-// CHECK: define i32 @test1
   return *m1(a);
-// CHECK: call i32* @m1(i32 [[PARAM1:%[^\)]+]])
-// CHECK: [[ALIGNCAST1:%.+]] = zext i32 [[PARAM1]] to i64
-// CHECK: [[MASK1:%.+]] = sub i64 [[ALIGNCAST1]], 1
-// CHECK: [[PTRINT1:%.+]] = ptrtoint
-// CHECK: [[MASKEDPTR1:%.+]] = and i64 [[PTRINT1]], [[MASK1]]
-// CHECK: [[MASKCOND1:%.+]] = icmp eq i64 [[MASKEDPTR1]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND1]])
 }
 // Condition where test2 param needs casting.
+// CHECK-LABEL: define {{[^@]+}}@test2
+// CHECK-SAME: (i64 [[A:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i64, align 8
+// CHECK-NEXT:    store i64 [[A]], i64* [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A_ADDR]], align 8
+// CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP0]] to i32
+// CHECK-NEXT:    [[CALL:%.*]] = call i32* @m1(i32 [[CONV]])
+// CHECK-NEXT:    [[ALIGNMENTCAST:%.*]] = zext i32 [[CONV]] to i64
+// CHECK-NEXT:    [[MASK:%.*]] = sub i64 [[ALIGNMENTCAST]], 1
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[CALL]] to i64
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], [[MASK]]
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[CALL]], align 4
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
 __INT32_TYPE__ test2(__SIZE_TYPE__ a) {
-// CHECK: define i32 @test2
   return *m1(a);
-// CHECK: [[CONV2:%.+]] = trunc i64 %{{.+}} to i32
-// CHECK: call i32* @m1(i32 [[CONV2]])
-// CHECK: [[ALIGNCAST2:%.+]] = zext i32 [[CONV2]] to i64
-// CHECK: [[MASK2:%.+]] = sub i64 [[ALIGNCAST2]], 1
-// CHECK: [[PTRINT2:%.+]] = ptrtoint
-// CHECK: [[MASKEDPTR2:%.+]] = and i64 [[PTRINT2]], [[MASK2]]
-// CHECK: [[MASKCOND2:%.+]] = icmp eq i64 [[MASKEDPTR2]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND2]])
 }
 __INT32_TYPE__ *m2(__SIZE_TYPE__ i) __attribute__((alloc_align(1)));
 
 // test3 param needs casting, but 'm2' is correct.
+// CHECK-LABEL: define {{[^@]+}}@test3
+// CHECK-SAME: (i32 [[A:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 [[A]], i32* [[A_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A_ADDR]], align 4
+// CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[CALL:%.*]] = call i32* @m2(i64 [[CONV]])
+// CHECK-NEXT:    [[MASK:%.*]] = sub i64 [[CONV]], 1
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[CALL]] to i64
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], [[MASK]]
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[CALL]], align 4
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
 __INT32_TYPE__ test3(__INT32_TYPE__ a) {
-// CHECK: define i32 @test3
   return *m2(a);
-// CHECK: [[CONV3:%.+]] = sext i32 %{{.+}} to i64
-// CHECK: call i32* @m2(i64 [[CONV3]])
-// CHECK: [[MASK3:%.+]] = sub i64 [[CONV3]], 1
-// CHECK: [[PTRINT3:%.+]] = ptrtoint
-// CHECK: [[MASKEDPTR3:%.+]] = and i64 [[PTRINT3]], [[MASK3]]
-// CHECK: [[MASKCOND3:%.+]] = icmp eq i64 [[MASKEDPTR3]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND3]])
 }
 
 // Every type matches, canonical example.
+// CHECK-LABEL: define {{[^@]+}}@test4
+// CHECK-SAME: (i64 [[A:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i64, align 8
+// CHECK-NEXT:    store i64 [[A]], i64* [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A_ADDR]], align 8
+// CHECK-NEXT:    [[CALL:%.*]] = call i32* @m2(i64 [[TMP0]])
+// CHECK-NEXT:    [[MASK:%.*]] = sub i64 [[TMP0]], 1
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[CALL]] to i64
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], [[MASK]]
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[CALL]], align 4
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
 __INT32_TYPE__ test4(__SIZE_TYPE__ a) {
-// CHECK: define i32 @test4
   return *m2(a);
-// CHECK: call i32* @m2(i64 [[PARAM4:%[^\)]+]])
-// CHECK: [[MASK4:%.+]] = sub i64 [[PARAM4]], 1
-// CHECK: [[PTRINT4:%.+]] = ptrtoint
-// CHECK: [[MASKEDPTR4:%.+]] = and i64 [[PTRINT4]], [[MASK4]]
-// CHECK: [[MASKCOND4:%.+]] = icmp eq i64 [[MASKEDPTR4]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND4]])
 }
 
 
@@ -60,30 +93,81 @@ struct MultiArgs { __INT64_TYPE__ a, b;};
 // Struct parameter doesn't take up an IR parameter, 'i' takes up 2.
 // Truncation to i64 is permissible, since alignments of greater than 2^64 are insane.
 __INT32_TYPE__ *m3(struct Empty s, __int128_t i) __attribute__((alloc_align(2)));
+// CHECK-LABEL: define {{[^@]+}}@test5
+// CHECK-SAME: (i64 [[A_COERCE0:%.*]], i64 [[A_COERCE1:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = alloca i128, align 16
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i128, align 16
+// CHECK-NEXT:    [[E:%.*]] = alloca [[STRUCT_EMPTY:%.*]], align 1
+// CHECK-NEXT:    [[COERCE:%.*]] = alloca i128, align 16
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast i128* [[A]] to { i64, i64 }*
+// CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds { i64, i64 }, { i64, i64 }* [[TMP0]], i32 0, i32 0
+// CHECK-NEXT:    store i64 [[A_COERCE0]], i64* [[TMP1]], align 16
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds { i64, i64 }, { i64, i64 }* [[TMP0]], i32 0, i32 1
+// CHECK-NEXT:    store i64 [[A_COERCE1]], i64* [[TMP2]], align 8
+// CHECK-NEXT:    [[A1:%.*]] = load i128, i128* [[A]], align 16
+// CHECK-NEXT:    store i128 [[A1]], i128* [[A_ADDR]], align 16
+// CHECK-NEXT:    [[TMP3:%.*]] = load i128, i128* [[A_ADDR]], align 16
+// CHECK-NEXT:    store i128 [[TMP3]], i128* [[COERCE]], align 16
+// CHECK-NEXT:    [[TMP4:%.*]] = bitcast i128* [[COERCE]] to { i64, i64 }*
+// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds { i64, i64 }, { i64, i64 }* [[TMP4]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP6:%.*]] = load i64, i64* [[TMP5]], align 16
+// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds { i64, i64 }, { i64, i64 }* [[TMP4]], i32 0, i32 1
+// CHECK-NEXT:    [[TMP8:%.*]] = load i64, i64* [[TMP7]], align 8
+// CHECK-NEXT:    [[CALL:%.*]] = call i32* @m3(i64 [[TMP6]], i64 [[TMP8]])
+// CHECK-NEXT:    [[ALIGNMENTCAST:%.*]] = trunc i128 [[TMP3]] to i64
+// CHECK-NEXT:    [[MASK:%.*]] = sub i64 [[ALIGNMENTCAST]], 1
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[CALL]] to i64
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], [[MASK]]
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* [[CALL]], align 4
+// CHECK-NEXT:    ret i32 [[TMP9]]
+//
 __INT32_TYPE__ test5(__int128_t a) {
-// CHECK: define i32 @test5
   struct Empty e;
   return *m3(e, a);
-// CHECK: call i32* @m3(i64 %{{.*}}, i64 %{{.*}})
-// CHECK: [[ALIGNCAST5:%.+]] = trunc i128 %{{.*}} to i64
-// CHECK: [[MASK5:%.+]] = sub i64 [[ALIGNCAST5]], 1
-// CHECK: [[PTRINT5:%.+]] = ptrtoint
-// CHECK: [[MASKEDPTR5:%.+]] = and i64 [[PTRINT5]], [[MASK5]]
-// CHECK: [[MASKCOND5:%.+]] = icmp eq i64 [[MASKEDPTR5]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND5]])
 }
 // Struct parameter takes up 2 parameters, 'i' takes up 2.
 __INT32_TYPE__ *m4(struct MultiArgs s, __int128_t i) __attribute__((alloc_align(2)));
+// CHECK-LABEL: define {{[^@]+}}@test6
+// CHECK-SAME: (i64 [[A_COERCE0:%.*]], i64 [[A_COERCE1:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = alloca i128, align 16
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i128, align 16
+// CHECK-NEXT:    [[E:%.*]] = alloca [[STRUCT_MULTIARGS:%.*]], align 8
+// CHECK-NEXT:    [[COERCE:%.*]] = alloca i128, align 16
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast i128* [[A]] to { i64, i64 }*
+// CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds { i64, i64 }, { i64, i64 }* [[TMP0]], i32 0, i32 0
+// CHECK-NEXT:    store i64 [[A_COERCE0]], i64* [[TMP1]], align 16
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds { i64, i64 }, { i64, i64 }* [[TMP0]], i32 0, i32 1
+// CHECK-NEXT:    store i64 [[A_COERCE1]], i64* [[TMP2]], align 8
+// CHECK-NEXT:    [[A1:%.*]] = load i128, i128* [[A]], align 16
+// CHECK-NEXT:    store i128 [[A1]], i128* [[A_ADDR]], align 16
+// CHECK-NEXT:    [[TMP3:%.*]] = load i128, i128* [[A_ADDR]], align 16
+// CHECK-NEXT:    [[TMP4:%.*]] = bitcast %struct.MultiArgs* [[E]] to { i64, i64 }*
+// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds { i64, i64 }, { i64, i64 }* [[TMP4]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP6:%.*]] = load i64, i64* [[TMP5]], align 8
+// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds { i64, i64 }, { i64, i64 }* [[TMP4]], i32 0, i32 1
+// CHECK-NEXT:    [[TMP8:%.*]] = load i64, i64* [[TMP7]], align 8
+// CHECK-NEXT:    store i128 [[TMP3]], i128* [[COERCE]], align 16
+// CHECK-NEXT:    [[TMP9:%.*]] = bitcast i128* [[COERCE]] to { i64, i64 }*
+// CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds { i64, i64 }, { i64, i64 }* [[TMP9]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP10]], align 16
+// CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds { i64, i64 }, { i64, i64 }* [[TMP9]], i32 0, i32 1
+// CHECK-NEXT:    [[TMP13:%.*]] = load i64, i64* [[TMP12]], align 8
+// CHECK-NEXT:    [[CALL:%.*]] = call i32* @m4(i64 [[TMP6]], i64 [[TMP8]], i64 [[TMP11]], i64 [[TMP13]])
+// CHECK-NEXT:    [[ALIGNMENTCAST:%.*]] = trunc i128 [[TMP3]] to i64
+// CHECK-NEXT:    [[MASK:%.*]] = sub i64 [[ALIGNMENTCAST]], 1
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[CALL]] to i64
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], [[MASK]]
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    [[TMP14:%.*]] = load i32, i32* [[CALL]], align 4
+// CHECK-NEXT:    ret i32 [[TMP14]]
+//
 __INT32_TYPE__ test6(__int128_t a) {
-// CHECK: define i32 @test6
   struct MultiArgs e;
   return *m4(e, a);
-// CHECK: call i32* @m4(i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}})
-// CHECK: [[ALIGNCAST6:%.+]] = trunc i128 %{{.*}} to i64
-// CHECK: [[MASK6:%.+]] = sub i64 [[ALIGNCAST6]], 1
-// CHECK: [[PTRINT6:%.+]] = ptrtoint
-// CHECK: [[MASKEDPTR6:%.+]] = and i64 [[PTRINT6]], [[MASK6]]
-// CHECK: [[MASKCOND6:%.+]] = icmp eq i64 [[MASKEDPTR6]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND6]])
 }
 
diff --git a/clang/test/CodeGen/arm-bf16-convert-intrinsics.c b/clang/test/CodeGen/arm-bf16-convert-intrinsics.c
new file mode 100644
index 0000000000000..d36d8def70dea
--- /dev/null
+++ b/clang/test/CodeGen/arm-bf16-convert-intrinsics.c
@@ -0,0 +1,88 @@
+// RUN: %clang_cc1 \
+// RUN:   -triple aarch64-arm-none-eabi -target-feature +neon -target-feature +bf16 \
+// RUN:   -disable-O0-optnone -emit-llvm -o - %s \
+// RUN:   | opt -S -mem2reg -instcombine \
+// RUN:   | FileCheck --check-prefixes=CHECK,CHECK-A64 %s
+// RUN: %clang_cc1 \
+// RUN:   -triple armv8.6a-arm-none-eabi -target-feature +neon \
+// RUN:   -target-feature +bf16 -mfloat-abi hard \
+// RUN:   -disable-O0-optnone -emit-llvm -o - %s \
+// RUN:   | opt -S -mem2reg -instcombine \
+// RUN:   | FileCheck --check-prefixes=CHECK,CHECK-A32-HARDFP %s
+// RUN: %clang_cc1 \
+// RUN:   -triple armv8.6a-arm-none-eabi -target-feature +neon \
+// RUN:   -target-feature +bf16 -mfloat-abi softfp \
+// RUN:   -disable-O0-optnone -emit-llvm -o - %s \
+// RUN:   | opt -S -mem2reg -instcombine \
+// RUN:   | FileCheck --check-prefixes=CHECK,CHECK-A32-SOFTFP %s
+
+#include <arm_neon.h>
+
+// CHECK-LABEL: test_vcvt_f32_bf16
+// CHECK: %[[EXT:.*]] = zext <4 x i16> %{{.*}} to <4 x i32>
+// CHECK: shl nuw <4 x i32> %[[EXT]], <i32 16, i32 16, i32 16, i32 16>
+float32x4_t test_vcvt_f32_bf16(bfloat16x4_t a) {
+  return vcvt_f32_bf16(a);
+}
+
+// CHECK-LABEL: test_vcvtq_low_f32_bf16
+// CHECK: shufflevector <8 x bfloat> %{{.*}}, <8 x bfloat> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK: %[[EXT:.*]] = zext <4 x i16> %{{.*}} to <4 x i32>
+// CHECK: shl nuw <4 x i32> %[[EXT]], <i32 16, i32 16, i32 16, i32 16>
+float32x4_t test_vcvtq_low_f32_bf16(bfloat16x8_t a) {
+  return vcvtq_low_f32_bf16(a);
+}
+
+// CHECK-LABEL: test_vcvtq_high_f32_bf16
+// CHECK: shufflevector <8 x bfloat> %{{.*}}, <8 x bfloat> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+// CHECK: %[[EXT:.*]] = zext <4 x i16> %{{.*}} to <4 x i32>
+// CHECK: shl nuw <4 x i32> %[[EXT]], <i32 16, i32 16, i32 16, i32 16>
+float32x4_t test_vcvtq_high_f32_bf16(bfloat16x8_t a) {
+  return vcvtq_high_f32_bf16(a);
+}
+
+// CHECK-LABEL: test_vcvt_bf16_f32
+// CHECK-A64: %[[CVT:.*]] = call <8 x bfloat> @llvm.aarch64.neon.bfcvtn(<4 x float> %a)
+// CHECK-A64: shufflevector <8 x bfloat> %[[CVT]], <8 x bfloat> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-A32-HARDFP: call <4 x bfloat> @llvm.arm.neon.vcvtfp2bf.v4bf16(<4 x float> %a)
+// CHECK-A32-SOFTFP: call <4 x i16> @llvm.arm.neon.vcvtfp2bf.v4i16(<4 x float> %a)
+bfloat16x4_t test_vcvt_bf16_f32(float32x4_t a) {
+  return vcvt_bf16_f32(a);
+}
+
+// CHECK-LABEL: test_vcvtq_low_bf16_f32
+// CHECK-A64: call <8 x bfloat> @llvm.aarch64.neon.bfcvtn(<4 x float> %a)
+// CHECK-A32-HARDFP: %[[CVT:.*]] = call <4 x bfloat> @llvm.arm.neon.vcvtfp2bf.v4bf16
+// CHECK-A32-HARDFP: shufflevector <4 x bfloat> zeroinitializer, <4 x bfloat> %[[CVT]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+// CHECK-A32-SOFTFP: call <4 x i16> @llvm.arm.neon.vcvtfp2bf.v4i16
+// CHECK-A32-SOFTFP: shufflevector <4 x bfloat> zeroinitializer, <4 x bfloat> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+bfloat16x8_t test_vcvtq_low_bf16_f32(float32x4_t a) {
+  return vcvtq_low_bf16_f32(a);
+}
+
+// CHECK-LABEL: test_vcvtq_high_bf16_f32
+// CHECK-A64: call <8 x bfloat> @llvm.aarch64.neon.bfcvtn2(<8 x bfloat> %inactive, <4 x float> %a)
+// CHECK-A32-HARDFP: %[[CVT:.*]] = call <4 x bfloat> @llvm.arm.neon.vcvtfp2bf.v4bf16(<4 x float> %a)
+// CHECK-A32-HARDFP: %[[INACT:.*]] = shufflevector <8 x bfloat> %inactive, <8 x bfloat> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-A32-HARDFP: shufflevector <4 x bfloat> %[[CVT]], <4 x bfloat> %[[INACT]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+// CHECK-A32-SOFTFP: call <4 x i16> @llvm.arm.neon.vcvtfp2bf.v4i16(<4 x float> %a)
+// CHECK-A32-SOFTFP: shufflevector <8 x bfloat> %{{.*}}, <8 x bfloat> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-A32-SOFTFP: shufflevector <4 x bfloat> %{{.*}}, <4 x bfloat> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+bfloat16x8_t test_vcvtq_high_bf16_f32(bfloat16x8_t inactive, float32x4_t a) {
+  return vcvtq_high_bf16_f32(inactive, a);
+}
+
+// CHECK-LABEL: test_vcvth_bf16_f32
+// CHECK-A64: call bfloat @llvm.aarch64.neon.bfcvt(float %a)
+// CHECK-A32-HARDFP: call bfloat @llvm.arm.neon.vcvtbfp2bf(float %a)
+// CHECK-A32-SOFTFP: call bfloat @llvm.arm.neon.vcvtbfp2bf(float %a)
+bfloat16_t test_vcvth_bf16_f32(float32_t a) {
+  return vcvth_bf16_f32(a);
+}
+
+// CHECK-LABEL: test_vcvtah_f32_bf16
+// CHECK: shl i32 %{{.*}}, 16
+float32_t test_vcvtah_f32_bf16(bfloat16_t a) {
+  return vcvtah_f32_bf16(a);
+}
+
diff --git a/clang/test/CodeGen/arm-bf16-dotprod-intrinsics.c b/clang/test/CodeGen/arm-bf16-dotprod-intrinsics.c
new file mode 100644
index 0000000000000..0eb130a377bdd
--- /dev/null
+++ b/clang/test/CodeGen/arm-bf16-dotprod-intrinsics.c
@@ -0,0 +1,166 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple armv8-arm-none-eabi \
+// RUN:   -target-feature +neon -target-feature +bf16 -mfloat-abi soft \
+// RUN:   -disable-O0-optnone -S -emit-llvm -o - %s \
+// RUN: | opt -S -mem2reg -instcombine | FileCheck %s
+// RUN: %clang_cc1 -triple armv8-arm-none-eabi \
+// RUN:   -target-feature +neon -target-feature +bf16 -mfloat-abi hard \
+// RUN:   -disable-O0-optnone -S -emit-llvm -o - %s \
+// RUN: | opt -S -mem2reg -instcombine | FileCheck %s
+
+#include <arm_neon.h>
+
+// CHECK-LABEL: @test_vbfdot_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x bfloat> [[A:%.*]] to <8 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x bfloat> [[B:%.*]] to <8 x i8>
+// CHECK-NEXT:    [[VBFDOT1_I:%.*]] = call <2 x float> @llvm.arm.neon.bfdot.v2f32.v8i8(<2 x float> [[R:%.*]], <8 x i8> [[TMP0]], <8 x i8> [[TMP1]]) #3
+// CHECK-NEXT:    ret <2 x float> [[VBFDOT1_I]]
+//
+float32x2_t test_vbfdot_f32(float32x2_t r, bfloat16x4_t a, bfloat16x4_t b) {
+  return vbfdot_f32(r, a, b);
+}
+
+// CHECK-LABEL: @test_vbfdotq_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[B:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[VBFDOT1_I:%.*]] = call <4 x float> @llvm.arm.neon.bfdot.v4f32.v16i8(<4 x float> [[R:%.*]], <16 x i8> [[TMP0]], <16 x i8> [[TMP1]]) #3
+// CHECK-NEXT:    ret <4 x float> [[VBFDOT1_I]]
+//
+float32x4_t test_vbfdotq_f32(float32x4_t r, bfloat16x8_t a, bfloat16x8_t b){
+  return vbfdotq_f32(r, a, b);
+}
+
+// CHECK-LABEL: @test_vbfdot_lane_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DOTCAST:%.*]] = bitcast <4 x bfloat> [[B:%.*]] to <2 x float>
+// CHECK-NEXT:    [[LANE:%.*]] = shufflevector <2 x float> [[DOTCAST]], <2 x float> undef, <2 x i32> zeroinitializer
+// CHECK-NEXT:    [[DOTCAST1:%.*]] = bitcast <2 x float> [[LANE]] to <8 x i8>
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x bfloat> [[A:%.*]] to <8 x i8>
+// CHECK-NEXT:    [[VBFDOT1_I:%.*]] = call <2 x float> @llvm.arm.neon.bfdot.v2f32.v8i8(<2 x float> [[R:%.*]], <8 x i8> [[TMP0]], <8 x i8> [[DOTCAST1]]) #3
+// CHECK-NEXT:    ret <2 x float> [[VBFDOT1_I]]
+//
+float32x2_t test_vbfdot_lane_f32(float32x2_t r, bfloat16x4_t a, bfloat16x4_t b){
+  return vbfdot_lane_f32(r, a, b, 0);
+}
+
+// CHECK-LABEL: @test_vbfdotq_laneq_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DOTCAST:%.*]] = bitcast <8 x bfloat> [[B:%.*]] to <4 x float>
+// CHECK-NEXT:    [[LANE:%.*]] = shufflevector <4 x float> [[DOTCAST]], <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+// CHECK-NEXT:    [[DOTCAST1:%.*]] = bitcast <4 x float> [[LANE]] to <16 x i8>
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[VBFDOT1_I:%.*]] = call <4 x float> @llvm.arm.neon.bfdot.v4f32.v16i8(<4 x float> [[R:%.*]], <16 x i8> [[TMP0]], <16 x i8> [[DOTCAST1]]) #3
+// CHECK-NEXT:    ret <4 x float> [[VBFDOT1_I]]
+//
+float32x4_t test_vbfdotq_laneq_f32(float32x4_t r, bfloat16x8_t a, bfloat16x8_t b) {
+  return vbfdotq_laneq_f32(r, a, b, 3);
+}
+
+// CHECK-LABEL: @test_vbfdot_laneq_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DOTCAST:%.*]] = bitcast <8 x bfloat> [[B:%.*]] to <4 x float>
+// CHECK-NEXT:    [[LANE:%.*]] = shufflevector <4 x float> [[DOTCAST]], <4 x float> undef, <2 x i32> <i32 3, i32 3>
+// CHECK-NEXT:    [[DOTCAST1:%.*]] = bitcast <2 x float> [[LANE]] to <8 x i8>
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x bfloat> [[A:%.*]] to <8 x i8>
+// CHECK-NEXT:    [[VBFDOT1_I:%.*]] = call <2 x float> @llvm.arm.neon.bfdot.v2f32.v8i8(<2 x float> [[R:%.*]], <8 x i8> [[TMP0]], <8 x i8> [[DOTCAST1]]) #3
+// CHECK-NEXT:    ret <2 x float> [[VBFDOT1_I]]
+//
+float32x2_t test_vbfdot_laneq_f32(float32x2_t r, bfloat16x4_t a, bfloat16x8_t b) {
+  return vbfdot_laneq_f32(r, a, b, 3);
+}
+
+// CHECK-LABEL: @test_vbfdotq_lane_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DOTCAST:%.*]] = bitcast <4 x bfloat> [[B:%.*]] to <2 x float>
+// CHECK-NEXT:    [[LANE:%.*]] = shufflevector <2 x float> [[DOTCAST]], <2 x float> undef, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[DOTCAST1:%.*]] = bitcast <4 x float> [[LANE]] to <16 x i8>
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[VBFDOT1_I:%.*]] = call <4 x float> @llvm.arm.neon.bfdot.v4f32.v16i8(<4 x float> [[R:%.*]], <16 x i8> [[TMP0]], <16 x i8> [[DOTCAST1]]) #3
+// CHECK-NEXT:    ret <4 x float> [[VBFDOT1_I]]
+//
+float32x4_t test_vbfdotq_lane_f32(float32x4_t r, bfloat16x8_t a, bfloat16x4_t b) {
+  return vbfdotq_lane_f32(r, a, b, 0);
+}
+
+// CHECK-LABEL: @test_vbfmmlaq_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[B:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[VBFMMLA1_I:%.*]] = call <4 x float> @llvm.arm.neon.bfmmla.v4f32.v16i8(<4 x float> [[R:%.*]], <16 x i8> [[TMP0]], <16 x i8> [[TMP1]]) #3
+// CHECK-NEXT:    ret <4 x float> [[VBFMMLA1_I]]
+//
+float32x4_t test_vbfmmlaq_f32(float32x4_t r, bfloat16x8_t a, bfloat16x8_t b) {
+  return vbfmmlaq_f32(r, a, b);
+}
+
+// CHECK-LABEL: @test_vbfmlalbq_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[B:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[VBFMLALB1_I:%.*]] = call <4 x float> @llvm.arm.neon.bfmlalb.v4f32.v16i8(<4 x float> [[R:%.*]], <16 x i8> [[TMP0]], <16 x i8> [[TMP1]]) #3
+// CHECK-NEXT:    ret <4 x float> [[VBFMLALB1_I]]
+//
+float32x4_t test_vbfmlalbq_f32(float32x4_t r, bfloat16x8_t a, bfloat16x8_t b) {
+  return vbfmlalbq_f32(r, a, b);
+}
+
+// CHECK-LABEL: @test_vbfmlaltq_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[B:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[VBFMLALT1_I:%.*]] = call <4 x float> @llvm.arm.neon.bfmlalt.v4f32.v16i8(<4 x float> [[R:%.*]], <16 x i8> [[TMP0]], <16 x i8> [[TMP1]]) #3
+// CHECK-NEXT:    ret <4 x float> [[VBFMLALT1_I]]
+//
+float32x4_t test_vbfmlaltq_f32(float32x4_t r, bfloat16x8_t a, bfloat16x8_t b) {
+  return vbfmlaltq_f32(r, a, b);
+}
+
+// CHECK-LABEL: @test_vbfmlalbq_lane_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[VECINIT35:%.*]] = shufflevector <4 x bfloat> [[B:%.*]], <4 x bfloat> undef, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[VECINIT35]] to <16 x i8>
+// CHECK-NEXT:    [[VBFMLALB1_I:%.*]] = call <4 x float> @llvm.arm.neon.bfmlalb.v4f32.v16i8(<4 x float> [[R:%.*]], <16 x i8> [[TMP0]], <16 x i8> [[TMP1]]) #3
+// CHECK-NEXT:    ret <4 x float> [[VBFMLALB1_I]]
+//
+float32x4_t test_vbfmlalbq_lane_f32(float32x4_t r, bfloat16x8_t a, bfloat16x4_t b) {
+  return vbfmlalbq_lane_f32(r, a, b, 0);
+}
+
+// CHECK-LABEL: @test_vbfmlalbq_laneq_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[VECINIT35:%.*]] = shufflevector <8 x bfloat> [[B:%.*]], <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[VECINIT35]] to <16 x i8>
+// CHECK-NEXT:    [[VBFMLALB1_I:%.*]] = call <4 x float> @llvm.arm.neon.bfmlalb.v4f32.v16i8(<4 x float> [[R:%.*]], <16 x i8> [[TMP0]], <16 x i8> [[TMP1]]) #3
+// CHECK-NEXT:    ret <4 x float> [[VBFMLALB1_I]]
+//
+float32x4_t test_vbfmlalbq_laneq_f32(float32x4_t r, bfloat16x8_t a, bfloat16x8_t b) {
+  return vbfmlalbq_laneq_f32(r, a, b, 3);
+}
+
+// CHECK-LABEL: @test_vbfmlaltq_lane_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[VECINIT35:%.*]] = shufflevector <4 x bfloat> [[B:%.*]], <4 x bfloat> undef, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[VECINIT35]] to <16 x i8>
+// CHECK-NEXT:    [[VBFMLALT1_I:%.*]] = call <4 x float> @llvm.arm.neon.bfmlalt.v4f32.v16i8(<4 x float> [[R:%.*]], <16 x i8> [[TMP0]], <16 x i8> [[TMP1]]) #3
+// CHECK-NEXT:    ret <4 x float> [[VBFMLALT1_I]]
+//
+float32x4_t test_vbfmlaltq_lane_f32(float32x4_t r, bfloat16x8_t a, bfloat16x4_t b) {
+  return vbfmlaltq_lane_f32(r, a, b, 0);
+}
+
+// CHECK-LABEL: @test_vbfmlaltq_laneq_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[VECINIT35:%.*]] = shufflevector <8 x bfloat> [[B:%.*]], <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[VECINIT35]] to <16 x i8>
+// CHECK-NEXT:    [[VBFMLALT1_I:%.*]] = call <4 x float> @llvm.arm.neon.bfmlalt.v4f32.v16i8(<4 x float> [[R:%.*]], <16 x i8> [[TMP0]], <16 x i8> [[TMP1]]) #3
+// CHECK-NEXT:    ret <4 x float> [[VBFMLALT1_I]]
+//
+float32x4_t test_vbfmlaltq_laneq_f32(float32x4_t r, bfloat16x8_t a, bfloat16x8_t b) {
+  return vbfmlaltq_laneq_f32(r, a, b, 3);
+}
diff --git a/clang/test/CodeGen/arm-bf16-getset-intrinsics.c b/clang/test/CodeGen/arm-bf16-getset-intrinsics.c
index ed35f810d18fc..a1a5b7b54f3ed 100644
--- a/clang/test/CodeGen/arm-bf16-getset-intrinsics.c
+++ b/clang/test/CodeGen/arm-bf16-getset-intrinsics.c
@@ -1,6 +1,8 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // RUN: %clang_cc1 -triple armv8.6a-arm-none-eabi -target-feature +neon -target-feature +bf16 -mfloat-abi hard \
 // RUN:  -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg -instcombine | FileCheck %s
+// RUN: %clang_cc1 -triple armv8.6a-arm-none-eabi -target-feature +neon -target-feature +bf16 -mfloat-abi soft \
+// RUN:  -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg -instcombine | FileCheck %s
 
 #include <arm_neon.h>
 
@@ -98,8 +100,8 @@ bfloat16x4_t test_vget_low_bf16(bfloat16x8_t a) {
 
 // CHECK-LABEL: @test_vget_lane_bf16(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[DOTCAST1:%.*]] = extractelement <4 x bfloat> [[V:%.*]], i32 1
-// CHECK-NEXT:    ret bfloat [[DOTCAST1]]
+// CHECK-NEXT:    [[VGET_LANE:%.*]] = extractelement <4 x bfloat> [[V:%.*]], i32 1
+// CHECK-NEXT:    ret bfloat [[VGET_LANE]]
 //
 bfloat16_t test_vget_lane_bf16(bfloat16x4_t v) {
   return vget_lane_bf16(v, 1);
@@ -107,8 +109,8 @@ bfloat16_t test_vget_lane_bf16(bfloat16x4_t v) {
 
 // CHECK-LABEL: @test_vgetq_lane_bf16(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[DOTCAST1:%.*]] = extractelement <8 x bfloat> [[V:%.*]], i32 7
-// CHECK-NEXT:    ret bfloat [[DOTCAST1]]
+// CHECK-NEXT:    [[VGET_LANE:%.*]] = extractelement <8 x bfloat> [[V:%.*]], i32 7
+// CHECK-NEXT:    ret bfloat [[VGET_LANE]]
 //
 bfloat16_t test_vgetq_lane_bf16(bfloat16x8_t v) {
   return vgetq_lane_bf16(v, 7);
@@ -116,8 +118,8 @@ bfloat16_t test_vgetq_lane_bf16(bfloat16x8_t v) {
 
 // CHECK-LABEL: @test_vset_lane_bf16(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x bfloat> [[V:%.*]], bfloat [[A:%.*]], i32 1
-// CHECK-NEXT:    ret <4 x bfloat> [[TMP0]]
+// CHECK-NEXT:    [[VSET_LANE:%.*]] = insertelement <4 x bfloat> [[V:%.*]], bfloat [[A:%.*]], i32 1
+// CHECK-NEXT:    ret <4 x bfloat> [[VSET_LANE]]
 //
 bfloat16x4_t test_vset_lane_bf16(bfloat16_t a, bfloat16x4_t v) {
   return vset_lane_bf16(a, v, 1);
@@ -125,8 +127,8 @@ bfloat16x4_t test_vset_lane_bf16(bfloat16_t a, bfloat16x4_t v) {
 
 // CHECK-LABEL: @test_vsetq_lane_bf16(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x bfloat> [[V:%.*]], bfloat [[A:%.*]], i32 7
-// CHECK-NEXT:    ret <8 x bfloat> [[TMP0]]
+// CHECK-NEXT:    [[VSET_LANE:%.*]] = insertelement <8 x bfloat> [[V:%.*]], bfloat [[A:%.*]], i32 7
+// CHECK-NEXT:    ret <8 x bfloat> [[VSET_LANE]]
 //
 bfloat16x8_t test_vsetq_lane_bf16(bfloat16_t a, bfloat16x8_t v) {
   return vsetq_lane_bf16(a, v, 7);
@@ -143,8 +145,8 @@ bfloat16_t test_vduph_lane_bf16(bfloat16x4_t v) {
 
 // CHECK-LABEL: @test_vduph_laneq_bf16(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[VGETQ_LANE:%.*]] = extractelement <8 x bfloat> [[V:%.*]], i32 7
-// CHECK-NEXT:    ret bfloat [[VGETQ_LANE]]
+// CHECK-NEXT:    [[VGET_LANE:%.*]] = extractelement <8 x bfloat> [[V:%.*]], i32 7
+// CHECK-NEXT:    ret bfloat [[VGET_LANE]]
 //
 bfloat16_t test_vduph_laneq_bf16(bfloat16x8_t v) {
   return vduph_laneq_bf16(v, 7);
diff --git a/clang/test/CodeGen/arm-mve-intrinsics/dup.c b/clang/test/CodeGen/arm-mve-intrinsics/dup.c
index 283c082570056..b443917cb2582 100644
--- a/clang/test/CodeGen/arm-mve-intrinsics/dup.c
+++ b/clang/test/CodeGen/arm-mve-intrinsics/dup.c
@@ -242,7 +242,8 @@ uint32x4_t test_vdupq_m_n_u32(uint32x4_t inactive, uint32_t a, mve_pred16_t p)
 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <8 x half> undef, half [[A:%.*]], i32 0
 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <8 x half> [[DOTSPLATINSERT]], <8 x half> undef, <8 x i32> zeroinitializer
-// CHECK-NEXT:    ret <8 x half> [[DOTSPLAT]]
+// CHECK-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x half> [[DOTSPLAT]], <8 x half> undef
+// CHECK-NEXT:    ret <8 x half> [[TMP2]]
 //
 float16x8_t test_vdupq_x_n_f16(float16_t a, mve_pred16_t p)
 {
@@ -255,7 +256,8 @@ float16x8_t test_vdupq_x_n_f16(float16_t a, mve_pred16_t p)
 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> undef, <4 x i32> zeroinitializer
-// CHECK-NEXT:    ret <4 x float> [[DOTSPLAT]]
+// CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[DOTSPLAT]], <4 x float> undef
+// CHECK-NEXT:    ret <4 x float> [[TMP2]]
 //
 float32x4_t test_vdupq_x_n_f32(float32_t a, mve_pred16_t p)
 {
@@ -268,7 +270,8 @@ float32x4_t test_vdupq_x_n_f32(float32_t a, mve_pred16_t p)
 // CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <16 x i8> undef, i8 [[A:%.*]], i32 0
 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <16 x i8> [[DOTSPLATINSERT]], <16 x i8> undef, <16 x i32> zeroinitializer
-// CHECK-NEXT:    ret <16 x i8> [[DOTSPLAT]]
+// CHECK-NEXT:    [[TMP2:%.*]] = select <16 x i1> [[TMP1]], <16 x i8> [[DOTSPLAT]], <16 x i8> undef
+// CHECK-NEXT:    ret <16 x i8> [[TMP2]]
 //
 int8x16_t test_vdupq_x_n_s8(int8_t a, mve_pred16_t p)
 {
@@ -281,7 +284,8 @@ int8x16_t test_vdupq_x_n_s8(int8_t a, mve_pred16_t p)
 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <8 x i16> undef, i16 [[A:%.*]], i32 0
 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <8 x i16> [[DOTSPLATINSERT]], <8 x i16> undef, <8 x i32> zeroinitializer
-// CHECK-NEXT:    ret <8 x i16> [[DOTSPLAT]]
+// CHECK-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x i16> [[DOTSPLAT]], <8 x i16> undef
+// CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 //
 int16x8_t test_vdupq_x_n_s16(int16_t a, mve_pred16_t p)
 {
@@ -294,7 +298,8 @@ int16x8_t test_vdupq_x_n_s16(int16_t a, mve_pred16_t p)
 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> undef, i32 [[A:%.*]], i32 0
 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> undef, <4 x i32> zeroinitializer
-// CHECK-NEXT:    ret <4 x i32> [[DOTSPLAT]]
+// CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[DOTSPLAT]], <4 x i32> undef
+// CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 //
 int32x4_t test_vdupq_x_n_s32(int32_t a, mve_pred16_t p)
 {
@@ -307,7 +312,8 @@ int32x4_t test_vdupq_x_n_s32(int32_t a, mve_pred16_t p)
 // CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <16 x i8> undef, i8 [[A:%.*]], i32 0
 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <16 x i8> [[DOTSPLATINSERT]], <16 x i8> undef, <16 x i32> zeroinitializer
-// CHECK-NEXT:    ret <16 x i8> [[DOTSPLAT]]
+// CHECK-NEXT:    [[TMP2:%.*]] = select <16 x i1> [[TMP1]], <16 x i8> [[DOTSPLAT]], <16 x i8> undef
+// CHECK-NEXT:    ret <16 x i8> [[TMP2]]
 //
 uint8x16_t test_vdupq_x_n_u8(uint8_t a, mve_pred16_t p)
 {
@@ -320,7 +326,8 @@ uint8x16_t test_vdupq_x_n_u8(uint8_t a, mve_pred16_t p)
 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <8 x i16> undef, i16 [[A:%.*]], i32 0
 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <8 x i16> [[DOTSPLATINSERT]], <8 x i16> undef, <8 x i32> zeroinitializer
-// CHECK-NEXT:    ret <8 x i16> [[DOTSPLAT]]
+// CHECK-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x i16> [[DOTSPLAT]], <8 x i16> undef
+// CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 //
 uint16x8_t test_vdupq_x_n_u16(uint16_t a, mve_pred16_t p)
 {
@@ -333,7 +340,8 @@ uint16x8_t test_vdupq_x_n_u16(uint16_t a, mve_pred16_t p)
 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> undef, i32 [[A:%.*]], i32 0
 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> undef, <4 x i32> zeroinitializer
-// CHECK-NEXT:    ret <4 x i32> [[DOTSPLAT]]
+// CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[DOTSPLAT]], <4 x i32> undef
+// CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 //
 uint32x4_t test_vdupq_x_n_u32(uint32_t a, mve_pred16_t p)
 {
diff --git a/clang/test/CodeGen/arm-neon-directed-rounding-constrained.c b/clang/test/CodeGen/arm-neon-directed-rounding-constrained.c
index 5246993173f83..7dedab375c81e 100644
--- a/clang/test/CodeGen/arm-neon-directed-rounding-constrained.c
+++ b/clang/test/CodeGen/arm-neon-directed-rounding-constrained.c
@@ -7,10 +7,12 @@
 
 // RUN: %clang_cc1 -triple thumbv8-linux-gnueabihf -target-cpu cortex-a57 \
 // RUN:     -ffp-exception-behavior=strict \
+// RUN:     -fexperimental-strict-floating-point \
 // RUN:     -ffreestanding -disable-O0-optnone -emit-llvm %s -o - | \
 // RUN:     opt -S -mem2reg | FileCheck -check-prefixes=COMMON,COMMONIR,CONSTRAINED %s
 // RUN: %clang_cc1 -triple arm64-linux-gnueabihf -target-feature +neon \
 // RUN:     -ffp-exception-behavior=strict \
+// RUN:     -fexperimental-strict-floating-point \
 // RUN:     -ffreestanding -disable-O0-optnone -emit-llvm %s -o - | \
 // RUN:     opt -S -mem2reg | FileCheck -check-prefixes=COMMON,COMMONIR,CONSTRAINED %s
 
@@ -23,10 +25,12 @@
 
 // RUN: %clang_cc1 -triple thumbv8-linux-gnueabihf -target-cpu cortex-a57 \
 // RUN:     -ffp-exception-behavior=strict \
+// RUN:     -fexperimental-strict-floating-point \
 // RUN:     -ffreestanding -disable-O0-optnone -emit-llvm %s -o - | \
 // RUN:     opt -S -mem2reg | llc -o=- - | FileCheck -check-prefixes=COMMON,CHECK-ASM32 %s
 // RUN: %clang_cc1 -triple arm64-linux-gnueabihf -target-feature +neon \
 // RUN:     -ffp-exception-behavior=strict \
+// RUN:     -fexperimental-strict-floating-point \
 // RUN:     -ffreestanding -disable-O0-optnone -emit-llvm %s -o - | \
 // RUN:     opt -S -mem2reg | llc -o=- - | FileCheck -check-prefixes=COMMON,CHECK-ASM64 %s
 
diff --git a/clang/test/CodeGen/arm64-vrnd-constrained.c b/clang/test/CodeGen/arm64-vrnd-constrained.c
index bbded8f2c7f66..ee2edbcbd0964 100644
--- a/clang/test/CodeGen/arm64-vrnd-constrained.c
+++ b/clang/test/CodeGen/arm64-vrnd-constrained.c
@@ -1,10 +1,10 @@
 // RUN: %clang_cc1 -triple arm64-apple-ios7 -target-feature +neon -ffreestanding -flax-vector-conversions=none -emit-llvm -o - %s \
 // RUN: | FileCheck --check-prefix=COMMON --check-prefix=COMMONIR --check-prefix=UNCONSTRAINED %s
-// RUN: %clang_cc1 -triple arm64-apple-ios7 -target-feature +neon -ffreestanding -flax-vector-conversions=none -ffp-exception-behavior=strict -emit-llvm -o - %s \
+// RUN: %clang_cc1 -triple arm64-apple-ios7 -target-feature +neon -ffreestanding -flax-vector-conversions=none -fexperimental-strict-floating-point -ffp-exception-behavior=strict -emit-llvm -o - %s \
 // RUN: | FileCheck --check-prefix=COMMON --check-prefix=COMMONIR --check-prefix=CONSTRAINED %s
 // RUN: %clang_cc1 -triple arm64-apple-ios7 -target-feature +neon -ffreestanding -flax-vector-conversions=none -emit-llvm -o - %s | llc -o=- - \
 // RUN: | FileCheck --check-prefix=COMMON --check-prefix=CHECK-ASM %s
-// RUN: %clang_cc1 -triple arm64-apple-ios7 -target-feature +neon -ffreestanding -flax-vector-conversions=none -ffp-exception-behavior=strict -emit-llvm -o - %s | llc -o=- - \
+// RUN: %clang_cc1 -triple arm64-apple-ios7 -target-feature +neon -ffreestanding -flax-vector-conversions=none -fexperimental-strict-floating-point -ffp-exception-behavior=strict -emit-llvm -o - %s | llc -o=- - \
 // RUN: | FileCheck --check-prefix=COMMON --check-prefix=CHECK-ASM %s
 
 // REQUIRES: aarch64-registered-target
diff --git a/clang/test/CodeGen/attr-target-x86.c b/clang/test/CodeGen/attr-target-x86.c
index 73486e5ee1fe7..feed175cce0bc 100644
--- a/clang/test/CodeGen/attr-target-x86.c
+++ b/clang/test/CodeGen/attr-target-x86.c
@@ -50,9 +50,9 @@ int __attribute__((target("arch=lakemont,mmx"))) use_before_def(void) {
 // CHECK: use_before_def{{.*}} #7
 // CHECK: #0 = {{.*}}"target-cpu"="i686" "target-features"="+cx8,+x87"
 // CHECK: #1 = {{.*}}"target-cpu"="ivybridge" "target-features"="+avx,+cx16,+cx8,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt"
-// CHECK: #2 = {{.*}}"target-cpu"="i686" "target-features"="+cx8,+x87,-aes,-avx,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-f16c,-fma,-fma4,-gfni,-pclmul,-sha,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-xop,-xsave,-xsaveopt"
+// CHECK: #2 = {{.*}}"target-cpu"="i686" "target-features"="+cx8,+x87,-aes,-avx,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-f16c,-fma,-fma4,-gfni,-pclmul,-sha,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-xop"
 // CHECK: #3 = {{.*}}"target-cpu"="i686" "target-features"="+cx8,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87"
-// CHECK: #4 = {{.*}}"target-cpu"="i686" "target-features"="+cx8,+x87,-avx,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-f16c,-fma,-fma4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop,-xsave,-xsaveopt"
+// CHECK: #4 = {{.*}}"target-cpu"="i686" "target-features"="+cx8,+x87,-avx,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-f16c,-fma,-fma4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop"
 // CHECK: #5 = {{.*}}"target-cpu"="ivybridge" "target-features"="+avx,+cx16,+cx8,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt,-aes,-vaes"
 // CHECK: #6 = {{.*}}"target-cpu"="i686" "target-features"="+cx8,+x87,-3dnow,-3dnowa,-mmx"
 // CHECK: #7 = {{.*}}"target-cpu"="lakemont" "target-features"="+cx8,+mmx"
diff --git a/clang/test/CodeGen/builtin-align.c b/clang/test/CodeGen/builtin-align.c
index 8fb51822164d0..7e66e2b5c0b9b 100644
--- a/clang/test/CodeGen/builtin-align.c
+++ b/clang/test/CodeGen/builtin-align.c
@@ -1,21 +1,22 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 /// Check the code generation for the alignment builtins
 /// To make the test case easier to read, run SROA after generating IR to remove the alloca instructions.
 // RUN: %clang_cc1 -triple=x86_64-unknown-unknown -DTEST_VOID_PTR \
 // RUN:   -o - -emit-llvm %s -disable-O0-optnone | opt -S -sroa | \
-// RUN:   FileCheck %s -check-prefixes CHECK,POINTER,ALIGNMENT_EXT \
+// RUN:   FileCheck %s -check-prefixes CHECK,CHECK-VOID_PTR \
 // RUN:   -enable-var-scope '-D$PTRTYPE=i8'
 // RUN: %clang_cc1 -triple=x86_64-unknown-unknown -DTEST_FLOAT_PTR \
 // RUN:   -o - -emit-llvm %s -disable-O0-optnone | opt -S -sroa | \
-// RUN:   FileCheck %s -check-prefixes CHECK,POINTER,NON_I8_POINTER,ALIGNMENT_EXT \
+// RUN:   FileCheck %s -check-prefixes CHECK,CHECK-FLOAT_PTR \
 // RUN:   -enable-var-scope '-D$PTRTYPE=f32'
 // RUN: %clang_cc1 -triple=x86_64-unknown-unknown -DTEST_LONG \
 // RUN:   -o - -emit-llvm %s -disable-O0-optnone | opt -S -sroa | \
-// RUN:   FileCheck %s -check-prefixes CHECK,INTEGER,ALIGNMENT_EXT -enable-var-scope
+// RUN:   FileCheck %s -check-prefixes CHECK,CHECK-LONG -enable-var-scope
 /// Check that we can handle the case where the alignment parameter is wider
 /// than the source type (generate a trunc on alignment instead of zext)
 // RUN: %clang_cc1 -triple=x86_64-unknown-unknown -DTEST_USHORT \
 // RUN:   -o - -emit-llvm %s -disable-O0-optnone | opt -S -sroa | \
-// RUN:   FileCheck %s -check-prefixes CHECK,INTEGER,ALIGNMENT_TRUNC -enable-var-scope
+// RUN:   FileCheck %s -check-prefixes CHECK,CHECK-USHORT -enable-var-scope
 
 
 #ifdef TEST_VOID_PTR
@@ -24,8 +25,6 @@
 #define TYPE float *
 #elif defined(TEST_LONG)
 #define TYPE long
-#elif defined(TEST_CAP)
-#define TYPE void *__capability
 #elif defined(TEST_USHORT)
 #define TYPE unsigned short
 #else
@@ -49,78 +48,185 @@ int up_2 = __builtin_align_up(256, 32);
 // CHECK: @up_2 = global i32 256, align 4
 
 /// Capture the IR type here to use in the remaining FileCheck captures:
-// CHECK: define {{[^@]+}}@get_type() #0
-// CHECK-NEXT:  entry:
-// POINTER-NEXT:    ret [[$TYPE:.+]] null
-// INTEGER-NEXT:    ret [[$TYPE:.+]] 0
+// CHECK-VOID_PTR-LABEL: define {{[^@]+}}@get_type() #0
+// CHECK-VOID_PTR-NEXT:  entry:
+// CHECK-VOID_PTR-NEXT:    ret i8* null
+//
+// CHECK-FLOAT_PTR-LABEL: define {{[^@]+}}@get_type() #0
+// CHECK-FLOAT_PTR-NEXT:  entry:
+// CHECK-FLOAT_PTR-NEXT:    ret float* null
+//
+// CHECK-LONG-LABEL: define {{[^@]+}}@get_type() #0
+// CHECK-LONG-NEXT:  entry:
+// CHECK-LONG-NEXT:    ret i64 0
+//
+// CHECK-USHORT-LABEL: define {{[^@]+}}@get_type() #0
+// CHECK-USHORT-NEXT:  entry:
+// CHECK-USHORT-NEXT:    ret i16 0
 //
 TYPE get_type(void) {
   return (TYPE)0;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@is_aligned
-// CHECK-SAME: ([[$TYPE]] {{[^%]*}}[[PTR:%.*]], i32 [[ALIGN:%.*]]) #0
-// CHECK-NEXT:  entry:
-// ALIGNMENT_EXT-NEXT:   [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to [[ALIGN_TYPE:i64]]
-// ALIGNMENT_TRUNC-NEXT: [[ALIGNMENT:%.*]] = trunc i32 [[ALIGN]] to [[ALIGN_TYPE:i16]]
-// CHECK-NEXT:           [[MASK:%.*]] = sub [[ALIGN_TYPE]] [[ALIGNMENT]], 1
-// POINTER-NEXT:         [[PTR:%.*]] = ptrtoint [[$TYPE]] %ptr to i64
-// CHECK-NEXT:           [[SET_BITS:%.*]] = and [[ALIGN_TYPE]] [[PTR]], [[MASK]]
-// CHECK-NEXT:           [[IS_ALIGNED:%.*]] = icmp eq [[ALIGN_TYPE]] [[SET_BITS]], 0
-// CHECK-NEXT:           ret i1 [[IS_ALIGNED]]
+// CHECK-VOID_PTR-LABEL: define {{[^@]+}}@is_aligned
+// CHECK-VOID_PTR-SAME: (i8* [[PTR:%.*]], i32 [[ALIGN:%.*]]) #0
+// CHECK-VOID_PTR-NEXT:  entry:
+// CHECK-VOID_PTR-NEXT:    [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
+// CHECK-VOID_PTR-NEXT:    [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
+// CHECK-VOID_PTR-NEXT:    [[SRC_ADDR:%.*]] = ptrtoint i8* [[PTR]] to i64
+// CHECK-VOID_PTR-NEXT:    [[SET_BITS:%.*]] = and i64 [[SRC_ADDR]], [[MASK]]
+// CHECK-VOID_PTR-NEXT:    [[IS_ALIGNED:%.*]] = icmp eq i64 [[SET_BITS]], 0
+// CHECK-VOID_PTR-NEXT:    ret i1 [[IS_ALIGNED]]
+//
+// CHECK-FLOAT_PTR-LABEL: define {{[^@]+}}@is_aligned
+// CHECK-FLOAT_PTR-SAME: (float* [[PTR:%.*]], i32 [[ALIGN:%.*]]) #0
+// CHECK-FLOAT_PTR-NEXT:  entry:
+// CHECK-FLOAT_PTR-NEXT:    [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
+// CHECK-FLOAT_PTR-NEXT:    [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
+// CHECK-FLOAT_PTR-NEXT:    [[SRC_ADDR:%.*]] = ptrtoint float* [[PTR]] to i64
+// CHECK-FLOAT_PTR-NEXT:    [[SET_BITS:%.*]] = and i64 [[SRC_ADDR]], [[MASK]]
+// CHECK-FLOAT_PTR-NEXT:    [[IS_ALIGNED:%.*]] = icmp eq i64 [[SET_BITS]], 0
+// CHECK-FLOAT_PTR-NEXT:    ret i1 [[IS_ALIGNED]]
+//
+// CHECK-LONG-LABEL: define {{[^@]+}}@is_aligned
+// CHECK-LONG-SAME: (i64 [[PTR:%.*]], i32 [[ALIGN:%.*]]) #0
+// CHECK-LONG-NEXT:  entry:
+// CHECK-LONG-NEXT:    [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
+// CHECK-LONG-NEXT:    [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
+// CHECK-LONG-NEXT:    [[SET_BITS:%.*]] = and i64 [[PTR]], [[MASK]]
+// CHECK-LONG-NEXT:    [[IS_ALIGNED:%.*]] = icmp eq i64 [[SET_BITS]], 0
+// CHECK-LONG-NEXT:    ret i1 [[IS_ALIGNED]]
+//
+// CHECK-USHORT-LABEL: define {{[^@]+}}@is_aligned
+// CHECK-USHORT-SAME: (i16 zeroext [[PTR:%.*]], i32 [[ALIGN:%.*]]) #0
+// CHECK-USHORT-NEXT:  entry:
+// CHECK-USHORT-NEXT:    [[ALIGNMENT:%.*]] = trunc i32 [[ALIGN]] to i16
+// CHECK-USHORT-NEXT:    [[MASK:%.*]] = sub i16 [[ALIGNMENT]], 1
+// CHECK-USHORT-NEXT:    [[SET_BITS:%.*]] = and i16 [[PTR]], [[MASK]]
+// CHECK-USHORT-NEXT:    [[IS_ALIGNED:%.*]] = icmp eq i16 [[SET_BITS]], 0
+// CHECK-USHORT-NEXT:    ret i1 [[IS_ALIGNED]]
 //
 _Bool is_aligned(TYPE ptr, unsigned align) {
   return __builtin_is_aligned(ptr, align);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@align_up
-// CHECK-SAME: ([[$TYPE]] {{[^%]*}}[[PTR:%.*]], i32 [[ALIGN:%.*]]) #0
-// CHECK-NEXT:  entry:
-// ALIGNMENT_EXT-NEXT:   [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to [[ALIGN_TYPE:i64]]
-// ALIGNMENT_TRUNC-NEXT: [[ALIGNMENT:%.*]] = trunc i32 [[ALIGN]] to [[ALIGN_TYPE:i16]]
-// CHECK-NEXT:           [[MASK:%.*]] = sub [[ALIGN_TYPE]] [[ALIGNMENT]], 1
-// INTEGER-NEXT:         [[OVER_BOUNDARY:%.*]] = add [[$TYPE]] [[PTR]], [[MASK]]
 // NOTYET-POINTER-NEXT:  [[ALIGNED_RESULT:%.*]] = call [[$TYPE]] @llvm.ptrmask.p0[[$PTRTYPE]].p0i8.i64(i8* [[OVER_BOUNDARY]], [[ALIGN_TYPE]] [[INVERTED_MASK]])
-// POINTER-NEXT:         [[INTPTR:%.*]] = ptrtoint [[$TYPE]] [[PTR]] to [[ALIGN_TYPE]]
-// POINTER-NEXT:         [[OVER_BOUNDARY:%.*]] = add [[ALIGN_TYPE]] [[INTPTR]], [[MASK]]
-// CHECK-NEXT:           [[INVERTED_MASK:%.*]] = xor [[ALIGN_TYPE]] [[MASK]], -1
-// CHECK-NEXT:           [[ALIGNED_RESULT:%.*]] = and [[ALIGN_TYPE]] [[OVER_BOUNDARY]], [[INVERTED_MASK]]
-// POINTER-NEXT:         [[DIFF:%.*]] = sub i64 [[ALIGNED_RESULT]], [[INTPTR]]
-// NON_I8_POINTER-NEXT:  [[PTR:%.*]] = bitcast [[$TYPE]] {{%.*}} to i8*
-// POINTER-NEXT:         [[ALIGNED_RESULT:%.*]] = getelementptr inbounds i8, i8* [[PTR]], i64 [[DIFF]]
-// NON_I8_POINTER-NEXT:  [[ALIGNED_RESULT:%.*]] = bitcast i8* {{%.*}} to [[$TYPE]]
-// POINTER-NEXT:         [[ASSUME_MASK:%.*]] = sub i64 %alignment, 1
-// POINTER-NEXT:         [[ASSUME_INTPTR:%.*]]= ptrtoint [[$TYPE]] [[ALIGNED_RESULT]] to i64
-// POINTER-NEXT:         [[MASKEDPTR:%.*]] = and i64 %ptrint, [[ASSUME_MASK]]
-// POINTER-NEXT:         [[MASKEDCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
-// POINTER-NEXT:         call void @llvm.assume(i1 [[MASKEDCOND]])
-// CHECK-NEXT:           ret [[$TYPE]] [[ALIGNED_RESULT]]
+// CHECK-VOID_PTR-LABEL: define {{[^@]+}}@align_up
+// CHECK-VOID_PTR-SAME: (i8* [[PTR:%.*]], i32 [[ALIGN:%.*]]) #0
+// CHECK-VOID_PTR-NEXT:  entry:
+// CHECK-VOID_PTR-NEXT:    [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
+// CHECK-VOID_PTR-NEXT:    [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
+// CHECK-VOID_PTR-NEXT:    [[INTPTR:%.*]] = ptrtoint i8* [[PTR]] to i64
+// CHECK-VOID_PTR-NEXT:    [[OVER_BOUNDARY:%.*]] = add i64 [[INTPTR]], [[MASK]]
+// CHECK-VOID_PTR-NEXT:    [[INVERTED_MASK:%.*]] = xor i64 [[MASK]], -1
+// CHECK-VOID_PTR-NEXT:    [[ALIGNED_INTPTR:%.*]] = and i64 [[OVER_BOUNDARY]], [[INVERTED_MASK]]
+// CHECK-VOID_PTR-NEXT:    [[DIFF:%.*]] = sub i64 [[ALIGNED_INTPTR]], [[INTPTR]]
+// CHECK-VOID_PTR-NEXT:    [[ALIGNED_RESULT:%.*]] = getelementptr inbounds i8, i8* [[PTR]], i64 [[DIFF]]
+// CHECK-VOID_PTR-NEXT:    [[MASK1:%.*]] = sub i64 [[ALIGNMENT]], 1
+// CHECK-VOID_PTR-NEXT:    [[PTRINT:%.*]] = ptrtoint i8* [[ALIGNED_RESULT]] to i64
+// CHECK-VOID_PTR-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], [[MASK1]]
+// CHECK-VOID_PTR-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-VOID_PTR-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-VOID_PTR-NEXT:    ret i8* [[ALIGNED_RESULT]]
+//
+// CHECK-FLOAT_PTR-LABEL: define {{[^@]+}}@align_up
+// CHECK-FLOAT_PTR-SAME: (float* [[PTR:%.*]], i32 [[ALIGN:%.*]]) #0
+// CHECK-FLOAT_PTR-NEXT:  entry:
+// CHECK-FLOAT_PTR-NEXT:    [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
+// CHECK-FLOAT_PTR-NEXT:    [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
+// CHECK-FLOAT_PTR-NEXT:    [[INTPTR:%.*]] = ptrtoint float* [[PTR]] to i64
+// CHECK-FLOAT_PTR-NEXT:    [[OVER_BOUNDARY:%.*]] = add i64 [[INTPTR]], [[MASK]]
+// CHECK-FLOAT_PTR-NEXT:    [[INVERTED_MASK:%.*]] = xor i64 [[MASK]], -1
+// CHECK-FLOAT_PTR-NEXT:    [[ALIGNED_INTPTR:%.*]] = and i64 [[OVER_BOUNDARY]], [[INVERTED_MASK]]
+// CHECK-FLOAT_PTR-NEXT:    [[DIFF:%.*]] = sub i64 [[ALIGNED_INTPTR]], [[INTPTR]]
+// CHECK-FLOAT_PTR-NEXT:    [[TMP0:%.*]] = bitcast float* [[PTR]] to i8*
+// CHECK-FLOAT_PTR-NEXT:    [[ALIGNED_RESULT:%.*]] = getelementptr inbounds i8, i8* [[TMP0]], i64 [[DIFF]]
+// CHECK-FLOAT_PTR-NEXT:    [[TMP1:%.*]] = bitcast i8* [[ALIGNED_RESULT]] to float*
+// CHECK-FLOAT_PTR-NEXT:    [[MASK1:%.*]] = sub i64 [[ALIGNMENT]], 1
+// CHECK-FLOAT_PTR-NEXT:    [[PTRINT:%.*]] = ptrtoint float* [[TMP1]] to i64
+// CHECK-FLOAT_PTR-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], [[MASK1]]
+// CHECK-FLOAT_PTR-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-FLOAT_PTR-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-FLOAT_PTR-NEXT:    ret float* [[TMP1]]
+//
+// CHECK-LONG-LABEL: define {{[^@]+}}@align_up
+// CHECK-LONG-SAME: (i64 [[PTR:%.*]], i32 [[ALIGN:%.*]]) #0
+// CHECK-LONG-NEXT:  entry:
+// CHECK-LONG-NEXT:    [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
+// CHECK-LONG-NEXT:    [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
+// CHECK-LONG-NEXT:    [[OVER_BOUNDARY:%.*]] = add i64 [[PTR]], [[MASK]]
+// CHECK-LONG-NEXT:    [[INVERTED_MASK:%.*]] = xor i64 [[MASK]], -1
+// CHECK-LONG-NEXT:    [[ALIGNED_RESULT:%.*]] = and i64 [[OVER_BOUNDARY]], [[INVERTED_MASK]]
+// CHECK-LONG-NEXT:    ret i64 [[ALIGNED_RESULT]]
+//
+// CHECK-USHORT-LABEL: define {{[^@]+}}@align_up
+// CHECK-USHORT-SAME: (i16 zeroext [[PTR:%.*]], i32 [[ALIGN:%.*]]) #0
+// CHECK-USHORT-NEXT:  entry:
+// CHECK-USHORT-NEXT:    [[ALIGNMENT:%.*]] = trunc i32 [[ALIGN]] to i16
+// CHECK-USHORT-NEXT:    [[MASK:%.*]] = sub i16 [[ALIGNMENT]], 1
+// CHECK-USHORT-NEXT:    [[OVER_BOUNDARY:%.*]] = add i16 [[PTR]], [[MASK]]
+// CHECK-USHORT-NEXT:    [[INVERTED_MASK:%.*]] = xor i16 [[MASK]], -1
+// CHECK-USHORT-NEXT:    [[ALIGNED_RESULT:%.*]] = and i16 [[OVER_BOUNDARY]], [[INVERTED_MASK]]
+// CHECK-USHORT-NEXT:    ret i16 [[ALIGNED_RESULT]]
 //
 TYPE align_up(TYPE ptr, unsigned align) {
   return __builtin_align_up(ptr, align);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@align_down
-// CHECK-SAME: ([[$TYPE]] {{[^%]*}}[[PTR:%.*]], i32 [[ALIGN:%.*]]) #0
-// CHECK-NEXT:  entry:
-// ALIGNMENT_EXT-NEXT:   [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to [[ALIGN_TYPE:i64]]
-// ALIGNMENT_TRUNC-NEXT: [[ALIGNMENT:%.*]] = trunc i32 [[ALIGN]] to [[ALIGN_TYPE:i16]]
-// CHECK-NEXT:           [[MASK:%.*]] = sub [[ALIGN_TYPE]] [[ALIGNMENT]], 1
 // NOTYET-POINTER-NEXT:  [[ALIGNED_RESULT:%.*]] = call [[$TYPE]] @llvm.ptrmask.p0[[$PTRTYPE]].p0[[$PTRTYPE]].i64([[$TYPE]] [[PTR]], [[ALIGN_TYPE]] [[INVERTED_MASK]])
-// POINTER-NEXT:         [[INTPTR:%.*]] = ptrtoint [[$TYPE]] [[PTR]] to [[ALIGN_TYPE]]
-// CHECK-NEXT:           [[INVERTED_MASK:%.*]] = xor [[ALIGN_TYPE]] [[MASK]], -1
-// POINTER-NEXT:         [[ALIGNED_INTPTR:%.*]] = and [[ALIGN_TYPE]] [[INTPTR]], [[INVERTED_MASK]]
-// POINTER-NEXT:         [[DIFF:%.*]] = sub i64 [[ALIGNED_INTPTR]], [[INTPTR]]
-// NON_I8_POINTER-NEXT:  [[PTR:%.*]] = bitcast [[$TYPE]] {{%.*}} to i8*
-// POINTER-NEXT:         [[ALIGNED_RESULT:%.*]] = getelementptr inbounds i8, i8* [[PTR]], i64 [[DIFF]]
-// NON_I8_POINTER-NEXT:  [[ALIGNED_RESULT:%.*]] = bitcast i8* {{%.*}} to [[$TYPE]]
-// INTEGER-NEXT:         [[ALIGNED_RESULT:%.*]] = and [[ALIGN_TYPE]] [[PTR]], [[INVERTED_MASK]]
-// POINTER-NEXT:         [[ASSUME_MASK:%.*]] = sub i64 %alignment, 1
-// POINTER-NEXT:         [[ASSUME_INTPTR:%.*]]= ptrtoint [[$TYPE]] [[ALIGNED_RESULT]] to i64
-// POINTER-NEXT:         [[MASKEDPTR:%.*]] = and i64 %ptrint, [[ASSUME_MASK]]
-// POINTER-NEXT:         [[MASKEDCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
-// POINTER-NEXT:         call void @llvm.assume(i1 [[MASKEDCOND]])
-// CHECK-NEXT:           ret [[$TYPE]] [[ALIGNED_RESULT]]
+// CHECK-VOID_PTR-LABEL: define {{[^@]+}}@align_down
+// CHECK-VOID_PTR-SAME: (i8* [[PTR:%.*]], i32 [[ALIGN:%.*]]) #0
+// CHECK-VOID_PTR-NEXT:  entry:
+// CHECK-VOID_PTR-NEXT:    [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
+// CHECK-VOID_PTR-NEXT:    [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
+// CHECK-VOID_PTR-NEXT:    [[INTPTR:%.*]] = ptrtoint i8* [[PTR]] to i64
+// CHECK-VOID_PTR-NEXT:    [[INVERTED_MASK:%.*]] = xor i64 [[MASK]], -1
+// CHECK-VOID_PTR-NEXT:    [[ALIGNED_INTPTR:%.*]] = and i64 [[INTPTR]], [[INVERTED_MASK]]
+// CHECK-VOID_PTR-NEXT:    [[DIFF:%.*]] = sub i64 [[ALIGNED_INTPTR]], [[INTPTR]]
+// CHECK-VOID_PTR-NEXT:    [[ALIGNED_RESULT:%.*]] = getelementptr inbounds i8, i8* [[PTR]], i64 [[DIFF]]
+// CHECK-VOID_PTR-NEXT:    [[MASK1:%.*]] = sub i64 [[ALIGNMENT]], 1
+// CHECK-VOID_PTR-NEXT:    [[PTRINT:%.*]] = ptrtoint i8* [[ALIGNED_RESULT]] to i64
+// CHECK-VOID_PTR-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], [[MASK1]]
+// CHECK-VOID_PTR-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-VOID_PTR-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-VOID_PTR-NEXT:    ret i8* [[ALIGNED_RESULT]]
+//
+// CHECK-FLOAT_PTR-LABEL: define {{[^@]+}}@align_down
+// CHECK-FLOAT_PTR-SAME: (float* [[PTR:%.*]], i32 [[ALIGN:%.*]]) #0
+// CHECK-FLOAT_PTR-NEXT:  entry:
+// CHECK-FLOAT_PTR-NEXT:    [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
+// CHECK-FLOAT_PTR-NEXT:    [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
+// CHECK-FLOAT_PTR-NEXT:    [[INTPTR:%.*]] = ptrtoint float* [[PTR]] to i64
+// CHECK-FLOAT_PTR-NEXT:    [[INVERTED_MASK:%.*]] = xor i64 [[MASK]], -1
+// CHECK-FLOAT_PTR-NEXT:    [[ALIGNED_INTPTR:%.*]] = and i64 [[INTPTR]], [[INVERTED_MASK]]
+// CHECK-FLOAT_PTR-NEXT:    [[DIFF:%.*]] = sub i64 [[ALIGNED_INTPTR]], [[INTPTR]]
+// CHECK-FLOAT_PTR-NEXT:    [[TMP0:%.*]] = bitcast float* [[PTR]] to i8*
+// CHECK-FLOAT_PTR-NEXT:    [[ALIGNED_RESULT:%.*]] = getelementptr inbounds i8, i8* [[TMP0]], i64 [[DIFF]]
+// CHECK-FLOAT_PTR-NEXT:    [[TMP1:%.*]] = bitcast i8* [[ALIGNED_RESULT]] to float*
+// CHECK-FLOAT_PTR-NEXT:    [[MASK1:%.*]] = sub i64 [[ALIGNMENT]], 1
+// CHECK-FLOAT_PTR-NEXT:    [[PTRINT:%.*]] = ptrtoint float* [[TMP1]] to i64
+// CHECK-FLOAT_PTR-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], [[MASK1]]
+// CHECK-FLOAT_PTR-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-FLOAT_PTR-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-FLOAT_PTR-NEXT:    ret float* [[TMP1]]
+//
+// CHECK-LONG-LABEL: define {{[^@]+}}@align_down
+// CHECK-LONG-SAME: (i64 [[PTR:%.*]], i32 [[ALIGN:%.*]]) #0
+// CHECK-LONG-NEXT:  entry:
+// CHECK-LONG-NEXT:    [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
+// CHECK-LONG-NEXT:    [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
+// CHECK-LONG-NEXT:    [[INVERTED_MASK:%.*]] = xor i64 [[MASK]], -1
+// CHECK-LONG-NEXT:    [[ALIGNED_RESULT:%.*]] = and i64 [[PTR]], [[INVERTED_MASK]]
+// CHECK-LONG-NEXT:    ret i64 [[ALIGNED_RESULT]]
+//
+// CHECK-USHORT-LABEL: define {{[^@]+}}@align_down
+// CHECK-USHORT-SAME: (i16 zeroext [[PTR:%.*]], i32 [[ALIGN:%.*]]) #0
+// CHECK-USHORT-NEXT:  entry:
+// CHECK-USHORT-NEXT:    [[ALIGNMENT:%.*]] = trunc i32 [[ALIGN]] to i16
+// CHECK-USHORT-NEXT:    [[MASK:%.*]] = sub i16 [[ALIGNMENT]], 1
+// CHECK-USHORT-NEXT:    [[INVERTED_MASK:%.*]] = xor i16 [[MASK]], -1
+// CHECK-USHORT-NEXT:    [[ALIGNED_RESULT:%.*]] = and i16 [[PTR]], [[INVERTED_MASK]]
+// CHECK-USHORT-NEXT:    ret i16 [[ALIGNED_RESULT]]
 //
 TYPE align_down(TYPE ptr, unsigned align) {
   return __builtin_align_down(ptr, align);
diff --git a/clang/test/CodeGen/builtin-assume-aligned.c b/clang/test/CodeGen/builtin-assume-aligned.c
index 40532d914f7a1..90693cc215200 100644
--- a/clang/test/CodeGen/builtin-assume-aligned.c
+++ b/clang/test/CodeGen/builtin-assume-aligned.c
@@ -1,50 +1,106 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // RUN: %clang_cc1 -triple x86_64-unknown-unknown -emit-llvm -o - %s | FileCheck %s
 
-// CHECK-LABEL: @test1
+// CHECK-LABEL: define {{[^@]+}}@test1
+// CHECK-SAME: (i32* [[A:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32*, align 8
+// CHECK-NEXT:    store i32* [[A]], i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32*, i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[TMP0]] to i8*
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i8* [[TMP1]] to i64
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[TMP1]] to i32*
+// CHECK-NEXT:    store i32* [[TMP2]], i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32*, i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TMP3]], i64 0
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+// CHECK-NEXT:    ret i32 [[TMP4]]
+//
 int test1(int *a) {
-// CHECK: [[PTRINT1:%.+]] = ptrtoint
-// CHECK: [[MASKEDPTR1:%.+]] = and i64 [[PTRINT1]], 31
-// CHECK: [[MASKCOND1:%.+]] = icmp eq i64 [[MASKEDPTR1]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND1]])
   a = __builtin_assume_aligned(a, 32, 0ull);
   return a[0];
 }
 
-// CHECK-LABEL: @test2
+// CHECK-LABEL: define {{[^@]+}}@test2
+// CHECK-SAME: (i32* [[A:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32*, align 8
+// CHECK-NEXT:    store i32* [[A]], i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32*, i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[TMP0]] to i8*
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i8* [[TMP1]] to i64
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[TMP1]] to i32*
+// CHECK-NEXT:    store i32* [[TMP2]], i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32*, i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TMP3]], i64 0
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+// CHECK-NEXT:    ret i32 [[TMP4]]
+//
 int test2(int *a) {
-// CHECK: [[PTRINT2:%.+]] = ptrtoint
-// CHECK: [[MASKEDPTR2:%.+]] = and i64 [[PTRINT2]], 31
-// CHECK: [[MASKCOND2:%.+]] = icmp eq i64 [[MASKEDPTR2]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND2]])
   a = __builtin_assume_aligned(a, 32, 0);
   return a[0];
 }
 
-// CHECK-LABEL: @test3
+// CHECK-LABEL: define {{[^@]+}}@test3
+// CHECK-SAME: (i32* [[A:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32*, align 8
+// CHECK-NEXT:    store i32* [[A]], i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32*, i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[TMP0]] to i8*
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i8* [[TMP1]] to i64
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[TMP1]] to i32*
+// CHECK-NEXT:    store i32* [[TMP2]], i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32*, i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TMP3]], i64 0
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+// CHECK-NEXT:    ret i32 [[TMP4]]
+//
 int test3(int *a) {
-// CHECK: [[PTRINT3:%.+]] = ptrtoint
-// CHECK: [[MASKEDPTR3:%.+]] = and i64 [[PTRINT3]], 31
-// CHECK: [[MASKCOND3:%.+]] = icmp eq i64 [[MASKEDPTR3]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND3]])
   a = __builtin_assume_aligned(a, 32);
   return a[0];
 }
 
-// CHECK-LABEL: @test4
+// CHECK-LABEL: define {{[^@]+}}@test4
+// CHECK-SAME: (i32* [[A:%.*]], i32 [[B:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32*, align 8
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32* [[A]], i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    store i32 [[B]], i32* [[B_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32*, i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[TMP0]] to i8*
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[B_ADDR]], align 4
+// CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[TMP2]] to i64
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i8* [[TMP1]] to i64
+// CHECK-NEXT:    [[OFFSETPTR:%.*]] = sub i64 [[PTRINT]], [[CONV]]
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[OFFSETPTR]], 31
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP1]] to i32*
+// CHECK-NEXT:    store i32* [[TMP3]], i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32*, i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i64 0
+// CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+// CHECK-NEXT:    ret i32 [[TMP5]]
+//
 int test4(int *a, int b) {
-// CHECK-DAG: [[PTRINT4:%.+]] = ptrtoint
-// CHECK-DAG: [[CONV4:%.+]] = sext i32
-// CHECK: [[OFFSETPTR4:%.+]] = sub i64 [[PTRINT4]], [[CONV4]]
-// CHECK: [[MASKEDPTR4:%.+]] = and i64 [[OFFSETPTR4]], 31
-// CHECK: [[MASKCOND4:%.+]] = icmp eq i64 [[MASKEDPTR4]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND4]])
   a = __builtin_assume_aligned(a, 32, b);
   return a[0];
 }
 
 int *m1() __attribute__((assume_aligned(64)));
 
-// CHECK-LABEL: @test5(
+// CHECK-LABEL: define {{[^@]+}}@test5() #0
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[CALL:%.*]] = call align 64 i32* (...) @m1()
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[CALL]], align 4
@@ -56,22 +112,40 @@ int test5() {
 
 int *m2() __attribute__((assume_aligned(64, 12)));
 
-// CHECK-LABEL: @test6
+// CHECK-LABEL: define {{[^@]+}}@test6() #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CALL:%.*]] = call i32* (...) @m2()
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[CALL]] to i64
+// CHECK-NEXT:    [[OFFSETPTR:%.*]] = sub i64 [[PTRINT]], 12
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[OFFSETPTR]], 63
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[CALL]], align 4
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
 int test6() {
   return *m2();
-// CHECK: [[PTRINT6:%.+]] = ptrtoint
-// CHECK: [[OFFSETPTR6:%.+]] = sub i64 [[PTRINT6]], 12
-// CHECK: [[MASKEDPTR6:%.+]] = and i64 [[OFFSETPTR6]], 63
-// CHECK: [[MASKCOND6:%.+]] = icmp eq i64 [[MASKEDPTR6]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND6]])
 }
 
-// CHECK-LABEL: @pr43638
+// CHECK-LABEL: define {{[^@]+}}@pr43638
+// CHECK-SAME: (i32* [[A:%.*]]) #0
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32*, align 8
+// CHECK-NEXT:    store i32* [[A]], i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32*, i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[TMP0]] to i8*
+// CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i8* [[TMP1]] to i64
+// CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 536870911
+// CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+// CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+// CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[TMP1]] to i32*
+// CHECK-NEXT:    store i32* [[TMP2]], i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32*, i32** [[A_ADDR]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TMP3]], i64 0
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+// CHECK-NEXT:    ret i32 [[TMP4]]
+//
 int pr43638(int *a) {
   a = __builtin_assume_aligned(a, 4294967296);
 return a[0];
-// CHECK: [[PTRINT7:%.+]] = ptrtoint
-// CHECK: [[MASKEDPTR7:%.+]] = and i64 [[PTRINT7]], 536870911
-// CHECK: [[MASKCOND7:%.+]] = icmp eq i64 [[MASKEDPTR7]], 0
-// CHECK: call void @llvm.assume(i1 [[MASKCOND7]])
 }
diff --git a/clang/test/CodeGen/builtin-expect-with-probability.cpp b/clang/test/CodeGen/builtin-expect-with-probability.cpp
new file mode 100644
index 0000000000000..ba1d71321a3ff
--- /dev/null
+++ b/clang/test/CodeGen/builtin-expect-with-probability.cpp
@@ -0,0 +1,101 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -emit-llvm -o - %s -O1 -disable-llvm-passes | FileCheck %s --check-prefix=ALL --check-prefix=O1
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -emit-llvm -o - %s -O0 | FileCheck %s --check-prefix=ALL --check-prefix=O0
+extern int global;
+
+int expect_taken(int x) {
+// ALL-LABEL: expect_taken
+// O1:        call i64 @llvm.expect.with.probability.i64(i64 {{%.*}}, i64 1, double 9.000000e-01)
+// O0-NOT:    @llvm.expect.with.probability
+
+  if (__builtin_expect_with_probability(x == 100, 1, 0.9)) {
+    return 0;
+  }
+  return x;
+}
+
+int expect_not_taken(int x) {
+// ALL-LABEL: expect_not_taken
+// O1:        call i64 @llvm.expect.with.probability.i64(i64 {{%.*}}, i64 0, double 9.000000e-01)
+// O0-NOT:    @llvm.expect.with.probability
+
+  if (__builtin_expect_with_probability(x == 100, 0, 0.9)) {
+    return 0;
+  }
+  return x;
+}
+
+struct S {
+  static constexpr int prob = 1;
+};
+
+template<typename T>
+int expect_taken_template(int x) {
+// ALL-LABEL: expect_taken_template
+// O1:        call i64 @llvm.expect.with.probability.i64(i64 {{%.*}}, i64 1, double 1.000000e+00)
+// O0-NOT:    @llvm.expect.with.probability
+
+	if (__builtin_expect_with_probability (x == 100, 1, T::prob)) {
+		return 0;
+	}
+	return x;
+}
+
+int f() {
+  return expect_taken_template<S>(global);
+}
+
+int x;
+extern "C" {
+  int y(void);
+}
+void foo();
+
+void expect_value_side_effects() {
+// ALL-LABEL: expect_value_side_effects
+// ALL: [[CALL:%.*]] = call i32 @y
+// O1:  [[SEXT:%.*]] = sext i32 [[CALL]] to i64
+// O1:  call i64 @llvm.expect.with.probability.i64(i64 {{%.*}}, i64 [[SEXT]], double 6.000000e-01)
+// O0-NOT: @llvm.expect.with.probability
+
+  if (__builtin_expect_with_probability(x, y(), 0.6))
+    foo();
+}
+
+int switch_cond(int x) {
+// ALL-LABEL: switch_cond
+// O1:        call i64 @llvm.expect.with.probability.i64(i64 {{%.*}}, i64 1, double 8.000000e-01)
+// O0-NOT:    @llvm.expect.with.probability
+
+  switch (__builtin_expect_with_probability(x, 1, 0.8)) {
+  case 0:
+    x = x + 0;
+  case 1:
+    x = x + 1;
+  case 2:
+    x = x + 2;
+  case 5:
+    x = x + 5;
+  default:
+    x = x + 6;
+  }
+  return x;
+}
+
+constexpr double prob = 0.8;
+
+int variable_expected(int stuff) {
+// ALL-LABEL: variable_expected
+// O1: call i64 @llvm.expect.with.probability.i64(i64 {{%.*}}, i64 {{%.*}}, double 8.000000e-01)
+// O0-NOT: @llvm.expect.with.probability
+
+  int res = 0;
+
+  switch(__builtin_expect_with_probability(stuff, stuff, prob)) {
+    case 0:
+      res = 1;
+      break;
+    default:
+      break;
+  }
+  return res;
+}
diff --git a/clang/test/CodeGen/builtins-ppc-fpconstrained.c b/clang/test/CodeGen/builtins-ppc-fpconstrained.c
new file mode 100644
index 0000000000000..c8b08c3fb5d4a
--- /dev/null
+++ b/clang/test/CodeGen/builtins-ppc-fpconstrained.c
@@ -0,0 +1,161 @@
+// REQUIRES: powerpc-registered-target
+// RUN: %clang_cc1 -triple powerpc64le-gnu-linux -target-feature +vsx \
+// RUN: -emit-llvm %s -o - | FileCheck --check-prefix=CHECK-UNCONSTRAINED %s
+// RUN: %clang_cc1 -triple powerpc64le-gnu-linux -target-feature +vsx \
+// RUN: -fexperimental-strict-floating-point \
+// RUN:  -ffp-exception-behavior=strict -emit-llvm %s -o - | FileCheck \
+// RUN: --check-prefix=CHECK-CONSTRAINED -vv %s
+// RUN: %clang_cc1 -triple powerpc64le-gnu-linux -target-feature +vsx \
+// RUN: -fallow-half-arguments-and-returns -S -o - %s | \
+// RUN: FileCheck --check-prefix=CHECK-ASM --check-prefix=NOT-FIXME-CHECK  %s
+// RUN: %clang_cc1 -triple powerpc64le-gnu-linux -target-feature +vsx \
+// RUN: -fexperimental-strict-floating-point \
+// RUN: -fallow-half-arguments-and-returns -S -ffp-exception-behavior=strict \
+// RUN: -o - %s | FileCheck --check-prefix=CHECK-ASM \
+// RUN: --check-prefix=FIXME-CHECK  %s
+
+typedef __attribute__((vector_size(4 * sizeof(float)))) float vec_float;
+typedef __attribute__((vector_size(2 * sizeof(double)))) double vec_double;
+
+volatile vec_double vd;
+volatile vec_float vf;
+
+void test_float(void) {
+  vf = __builtin_vsx_xvsqrtsp(vf);
+  // CHECK-LABEL: try-xvsqrtsp
+  // CHECK-UNCONSTRAINED: @llvm.sqrt.v4f32(<4 x float> %{{.*}})
+  // CHECK-CONSTRAINED: @llvm.experimental.constrained.sqrt.v4f32(<4 x float> %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // CHECK-ASM: xvsqrtsp
+
+  vd = __builtin_vsx_xvsqrtdp(vd);
+  // CHECK-LABEL: try-xvsqrtdp
+  // CHECK-UNCONSTRAINED: @llvm.sqrt.v2f64(<2 x double> %{{.*}})
+  // CHECK-CONSTRAINED: @llvm.experimental.constrained.sqrt.v2f64(<2 x double> %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // CHECK-ASM: xvsqrtdp
+
+  vf = __builtin_vsx_xvrspim(vf);
+  // CHECK-LABEL: try-xvrspim
+  // CHECK-UNCONSTRAINED: @llvm.floor.v4f32(<4 x float> %{{.*}})
+  // CHECK-CONSTRAINED: @llvm.experimental.constrained.floor.v4f32(<4 x float> %{{.*}}, metadata !"fpexcept.strict")
+  // CHECK-ASM: xvrspim
+
+  vd = __builtin_vsx_xvrdpim(vd);
+  // CHECK-LABEL: try-xvrdpim
+  // CHECK-UNCONSTRAINED: @llvm.floor.v2f64(<2 x double> %{{.*}})
+  // CHECK-CONSTRAINED: @llvm.experimental.constrained.floor.v2f64(<2 x double> %{{.*}}, metadata !"fpexcept.strict")
+  // CHECK-ASM: xvrdpim
+
+  vf = __builtin_vsx_xvrspi(vf);
+  // CHECK-LABEL: try-xvrspi
+  // CHECK-UNCONSTRAINED: @llvm.round.v4f32(<4 x float> %{{.*}})
+  // CHECK-CONSTRAINED: @llvm.experimental.constrained.round.v4f32(<4 x float> %{{.*}}, metadata !"fpexcept.strict")
+  // CHECK-ASM: xvrspi
+
+  vd = __builtin_vsx_xvrdpi(vd);
+  // CHECK-LABEL: try-xvrdpi
+  // CHECK-UNCONSTRAINED: @llvm.round.v2f64(<2 x double> %{{.*}})
+  // CHECK-CONSTRAINED: @llvm.experimental.constrained.round.v2f64(<2 x double> %{{.*}}, metadata !"fpexcept.strict")
+  // CHECK-ASM: xvrdpi
+
+  vf = __builtin_vsx_xvrspic(vf);
+  // CHECK-LABEL: try-xvrspic
+  // CHECK-UNCONSTRAINED: @llvm.nearbyint.v4f32(<4 x float> %{{.*}})
+  // CHECK-CONSTRAINED: @llvm.experimental.constrained.nearbyint.v4f32(<4 x float> %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // CHECK-ASM: xvrspic
+
+  vd = __builtin_vsx_xvrdpic(vd);
+  // CHECK-LABEL: try-xvrdpic
+  // CHECK-UNCONSTRAINED: @llvm.nearbyint.v2f64(<2 x double> %{{.*}})
+  // CHECK-CONSTRAINED: @llvm.experimental.constrained.nearbyint.v2f64(<2 x double> %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // CHECK-ASM: xvrdpic
+
+  vf = __builtin_vsx_xvrspip(vf);
+  // CHECK-LABEL: try-xvrspip
+  // CHECK-UNCONSTRAINED: @llvm.ceil.v4f32(<4 x float> %{{.*}})
+  // CHECK-CONSTRAINED: @llvm.experimental.constrained.ceil.v4f32(<4 x float> %{{.*}}, metadata !"fpexcept.strict")
+  // CHECK-ASM: xvrspip
+
+  vd = __builtin_vsx_xvrdpip(vd);
+  // CHECK-LABEL: try-xvrdpip
+  // CHECK-UNCONSTRAINED: @llvm.ceil.v2f64(<2 x double> %{{.*}})
+  // CHECK-CONSTRAINED: @llvm.experimental.constrained.ceil.v2f64(<2 x double> %{{.*}}, metadata !"fpexcept.strict")
+  // CHECK-ASM: xvrdpip
+
+  vf = __builtin_vsx_xvrspiz(vf);
+  // CHECK-LABEL: try-xvrspiz
+  // CHECK-UNCONSTRAINED: @llvm.trunc.v4f32(<4 x float> %{{.*}})
+  // CHECK-CONSTRAINED: @llvm.experimental.constrained.trunc.v4f32(<4 x float> %{{.*}}, metadata !"fpexcept.strict")
+  // CHECK-ASM: xvrspiz
+
+  vd = __builtin_vsx_xvrdpiz(vd);
+  // CHECK-LABEL: try-xvrdpiz
+  // CHECK-UNCONSTRAINED: @llvm.trunc.v2f64(<2 x double> %{{.*}})
+  // CHECK-CONSTRAINED: @llvm.experimental.constrained.trunc.v2f64(<2 x double> %{{.*}}, metadata !"fpexcept.strict")
+  // CHECK-ASM: xvrdpiz
+
+  vf = __builtin_vsx_xvmaddasp(vf, vf, vf);
+  // CHECK-LABEL: try-xvmaddasp
+  // CHECK-UNCONSTRAINED: @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK-CONSTRAINED: @llvm.experimental.constrained.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // CHECK-ASM: xvmaddasp
+
+  vd = __builtin_vsx_xvmaddadp(vd, vd, vd);
+  // CHECK-LABEL: try-xvmaddadp
+  // CHECK-UNCONSTRAINED: @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK-CONSTRAINED: @llvm.experimental.constrained.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // CHECK-ASM: xvmaddadp
+
+  vf = __builtin_vsx_xvnmaddasp(vf, vf, vf);
+  // CHECK-LABEL: try-xvnmaddasp
+  // CHECK-UNCONSTRAINED: [[RESULT:%[^ ]+]] = call <4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK-UNCONSTRAINED: fneg <4 x float> [[RESULT]]
+  // CHECK-CONSTRAINED: [[RESULT:%[^ ]+]] = call <4 x float> @llvm.experimental.constrained.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // CHECK-CONSTRAINED: fneg <4 x float> [[RESULT]]
+  // NOT-FIXME-CHECK: xvnmaddasp
+  // FIXME-CHECK: xvmaddasp
+  // FIXME-CHECK: xvnegsp
+
+  vd = __builtin_vsx_xvnmaddadp(vd, vd, vd);
+  // CHECK-LABEL: try-xvnmaddadp
+  // CHECK-UNCONSTRAINED: [[RESULT:%[^ ]+]] = call <2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK-UNCONSTRAINED: fneg <2 x double> [[RESULT]]
+  // CHECK-CONSTRAINED: [[RESULT:%[^ ]+]] = call <2 x double> @llvm.experimental.constrained.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // CHECK-CONSTRAINED: fneg <2 x double> [[RESULT]]
+  // CHECK-ASM: xvnmaddadp
+
+  vf = __builtin_vsx_xvmsubasp(vf, vf, vf);
+  // CHECK-LABEL: try-xvmsubasp
+  // CHECK-UNCONSTRAINED: [[RESULT:%[^ ]+]] = fneg <4 x float> %{{.*}}
+  // CHECK-UNCONSTRAINED: @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> [[RESULT]])
+  // CHECK-CONSTRAINED: [[RESULT:%[^ ]+]] = fneg <4 x float> %{{.*}}
+  // CHECK-CONSTRAINED: @llvm.experimental.constrained.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> [[RESULT]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // CHECK-ASM: xvmsubasp
+
+  vd = __builtin_vsx_xvmsubadp(vd, vd, vd);
+  // CHECK-LABEL: try-xvmsubadp
+  // CHECK-UNCONSTRAINED: [[RESULT:%[^ ]+]] = fneg <2 x double> %{{.*}}
+  // CHECK-UNCONSTRAINED: @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> [[RESULT]])
+  // CHECK-CONSTRAINED: [[RESULT:%[^ ]+]] = fneg <2 x double> %{{.*}}
+  // CHECK-CONSTRAINED: @llvm.experimental.constrained.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> [[RESULT]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // CHECK-ASM:  xvmsubadp
+
+  vf = __builtin_vsx_xvnmsubasp(vf, vf, vf);
+  // CHECK-LABEL: try-xvnmsubasp
+  // CHECK-UNCONSTRAINED: [[RESULT0:%[^ ]+]] = fneg <4 x float> %{{.*}}
+  // CHECK-UNCONSTRAINED: [[RESULT1:%[^ ]+]] = call <4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> [[RESULT0]])
+  // CHECK-UNCONSTRAINED: fneg <4 x float> [[RESULT1]]
+  // CHECK-CONSTRAINED: [[RESULT0:%[^ ]+]] = fneg <4 x float> %{{.*}}
+  // CHECK-CONSTRAINED: [[RESULT1:%[^ ]+]] = call <4 x float> @llvm.experimental.constrained.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> [[RESULT0]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // CHECK-CONSTRAINED: fneg <4 x float> [[RESULT1]]
+  // CHECK-ASM: xvnmsubasp
+
+  vd = __builtin_vsx_xvnmsubadp(vd, vd, vd);
+  // CHECK-LABEL: try-xvnmsubadp
+  // CHECK-UNCONSTRAINED: [[RESULT0:%[^ ]+]] = fneg <2 x double> %{{.*}}
+  // CHECK-UNCONSTRAINED: [[RESULT1:%[^ ]+]] = call <2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> [[RESULT0]])
+  // CHECK-UNCONSTRAINED: fneg <2 x double> [[RESULT1]]
+  // CHECK-CONSTRAINED: [[RESULT0:%[^ ]+]] = fneg <2 x double> %{{.*}}
+  // CHECK-CONSTRAINED: [[RESULT1:%[^ ]+]] = call <2 x double> @llvm.experimental.constrained.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> [[RESULT0]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // CHECK-CONSTRAINED: fneg <2 x double> [[RESULT1]]
+  // CHECK-ASM: xvnmsubadp
+}
diff --git a/clang/test/CodeGen/builtins-ppc-p10.c b/clang/test/CodeGen/builtins-ppc-p10.c
index c21e8026d0c9a..38c8988a3e37c 100644
--- a/clang/test/CodeGen/builtins-ppc-p10.c
+++ b/clang/test/CodeGen/builtins-ppc-p10.c
@@ -13,3 +13,18 @@ unsigned long long test_pextd(void) {
   // CHECK: @llvm.ppc.pextd
   return __builtin_pextd(ulla, ullb);
 }
+
+unsigned long long test_cfuged(void) {
+  // CHECK: @llvm.ppc.cfuged
+  return __builtin_cfuged(ulla, ullb);
+}
+
+unsigned long long test_cntlzdm(void) {
+  // CHECK: @llvm.ppc.cntlzdm
+  return __builtin_cntlzdm(ulla, ullb);
+}
+
+unsigned long long test_cnttzdm(void) {
+  // CHECK: @llvm.ppc.cnttzdm
+  return __builtin_cnttzdm(ulla, ullb);
+}
diff --git a/clang/test/CodeGen/builtins-ppc-p10vector.c b/clang/test/CodeGen/builtins-ppc-p10vector.c
index 829ef97435eb4..22b4e7a6f3ecf 100644
--- a/clang/test/CodeGen/builtins-ppc-p10vector.c
+++ b/clang/test/CodeGen/builtins-ppc-p10vector.c
@@ -3,12 +3,31 @@
 // RUN:   -target-cpu pwr10 -triple powerpc64le-unknown-unknown -emit-llvm %s \
 // RUN:   -o - | FileCheck %s
 
+// RUN: %clang_cc1 -target-feature +vsx -target-feature +altivec \
+// RUN:   -target-cpu pwr10 -triple powerpc64-unknown-unknown -emit-llvm %s \
+// RUN:   -o - | FileCheck %s -check-prefix=CHECK-BE
+
+// RUN: %clang_cc1 -target-feature +vsx -target-feature +altivec \
+// RUN:   -target-cpu pwr10 -triple powerpc64le-unknown-unknown -emit-llvm %s \
+// RUN:   -o - | FileCheck %s -check-prefix=CHECK-LE
+
 #include <altivec.h>
 
-vector signed char vsca;
-vector unsigned char vuca;
-vector unsigned long long vulla, vullb;
-unsigned int uia;
+vector signed char vsca, vscb;
+vector unsigned char vuca, vucb, vucc;
+vector signed short vssa, vssb;
+vector unsigned short vusa, vusb, vusc;
+vector signed int vsia, vsib;
+vector unsigned int vuia, vuib, vuic;
+vector signed long long vslla, vsllb;
+vector unsigned long long vulla, vullb, vullc;
+vector unsigned __int128 vui128a, vui128b, vui128c;
+vector float vfa, vfb;
+vector double vda, vdb;
+unsigned int uia, uib;
+unsigned char uca;
+unsigned short usa;
+unsigned long long ulla;
 
 vector unsigned long long test_vpdepd(void) {
   // CHECK: @llvm.ppc.altivec.vpdepd(<2 x i64>
@@ -22,6 +41,84 @@ vector unsigned long long test_vpextd(void) {
   return vec_pext(vulla, vullb);
 }
 
+vector unsigned long long test_vcfuged(void) {
+  // CHECK: @llvm.ppc.altivec.vcfuged(<2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_cfuge(vulla, vullb);
+}
+
+unsigned long long test_vgnb_1(void) {
+  // CHECK: @llvm.ppc.altivec.vgnb(<1 x i128> %{{.+}}, i32 2)
+  // CHECK-NEXT: ret i64
+  return vec_gnb(vui128a, 2);
+}
+
+unsigned long long test_vgnb_2(void) {
+  // CHECK: @llvm.ppc.altivec.vgnb(<1 x i128> %{{.+}}, i32 7)
+  // CHECK-NEXT: ret i64
+  return vec_gnb(vui128a, 7);
+}
+
+unsigned long long test_vgnb_3(void) {
+  // CHECK: @llvm.ppc.altivec.vgnb(<1 x i128> %{{.+}}, i32 5)
+  // CHECK-NEXT: ret i64
+  return vec_gnb(vui128a, 5);
+}
+
+vector unsigned char test_xxeval_uc(void) {
+  // CHECK: @llvm.ppc.vsx.xxeval(<2 x i64> %{{.+}}, <2 x i64> %{{.+}}, <2 x i64> %{{.+}}, i32 0)
+  // CHECK: ret <16 x i8>
+  return vec_ternarylogic(vuca, vucb, vucc, 0);
+}
+
+vector unsigned short test_xxeval_us(void) {
+  // CHECK: @llvm.ppc.vsx.xxeval(<2 x i64> %{{.+}}, <2 x i64> %{{.+}}, <2 x i64> %{{.+}}, i32 255)
+  // CHECK: ret <8 x i16>
+  return vec_ternarylogic(vusa, vusb, vusc, 255);
+}
+
+vector unsigned int test_xxeval_ui(void) {
+  // CHECK: @llvm.ppc.vsx.xxeval(<2 x i64> %{{.+}}, <2 x i64> %{{.+}}, <2 x i64> %{{.+}}, i32 150)
+  // CHECK: ret <4 x i32>
+  return vec_ternarylogic(vuia, vuib, vuic, 150);
+}
+
+vector unsigned long long test_xxeval_ull(void) {
+  // CHECK: @llvm.ppc.vsx.xxeval(<2 x i64> %{{.+}}, <2 x i64> %{{.+}}, <2 x i64> %{{.+}}, i32 1)
+  // CHECK: ret <2 x i64>
+  return vec_ternarylogic(vulla, vullb, vullc, 1);
+}
+
+vector unsigned __int128 test_xxeval_ui128(void) {
+  // CHECK: @llvm.ppc.vsx.xxeval(<2 x i64> %{{.+}}, <2 x i64> %{{.+}}, <2 x i64> %{{.+}}, i32 246)
+  // CHECK: ret <1 x i128>
+  return vec_ternarylogic(vui128a, vui128b, vui128c, 246);
+}
+
+vector unsigned char test_xxgenpcvbm(void) {
+  // CHECK: @llvm.ppc.vsx.xxgenpcvbm(<16 x i8> %{{.+}}, i32
+  // CHECK-NEXT: ret <16 x i8>
+  return vec_genpcvm(vuca, 0);
+}
+
+vector unsigned short test_xxgenpcvhm(void) {
+  // CHECK: @llvm.ppc.vsx.xxgenpcvhm(<8 x i16> %{{.+}}, i32
+  // CHECK-NEXT: ret <8 x i16>
+  return vec_genpcvm(vusa, 0);
+}
+
+vector unsigned int test_xxgenpcvwm(void) {
+  // CHECK: @llvm.ppc.vsx.xxgenpcvwm(<4 x i32> %{{.+}}, i32
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_genpcvm(vuia, 0);
+}
+
+vector unsigned long long test_xxgenpcvdm(void) {
+  // CHECK: @llvm.ppc.vsx.xxgenpcvdm(<2 x i64> %{{.+}}, i32
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_genpcvm(vulla, 0);
+}
+
 vector signed char test_vec_vclrl_sc(void) {
   // CHECK-BE: @llvm.ppc.altivec.vclrlb(<16 x i8>
   // CHECK-BE-NEXT: ret <16 x i8>
@@ -53,3 +150,434 @@ vector unsigned char test_vec_clrr_uc(void) {
   // CHECK-LE-NEXT: ret <16 x i8>
   return vec_clrr(vuca, uia);
 }
+
+vector unsigned long long test_vclzdm(void) {
+  // CHECK: @llvm.ppc.altivec.vclzdm(<2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_cntlzm(vulla, vullb);
+}
+
+vector unsigned long long test_vctzdm(void) {
+  // CHECK: @llvm.ppc.altivec.vctzdm(<2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_cnttzm(vulla, vullb);
+}
+
+vector signed char test_vec_sldb_sc(void) {
+  // CHECK: @llvm.ppc.altivec.vsldbi(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32 0
+  // CHECK-NEXT: ret <16 x i8>
+  return vec_sldb(vsca, vscb, 0);
+  }
+
+vector unsigned char test_vec_sldb_uc(void) {
+  // CHECK: @llvm.ppc.altivec.vsldbi(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32 1
+  // CHECK-NEXT: ret <16 x i8>
+  return vec_sldb(vuca, vucb, 1);
+}
+
+vector signed short test_vec_sldb_ss(void) {
+  // CHECK: @llvm.ppc.altivec.vsldbi(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32 2
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <8 x i16>
+  // CHECK-NEXT: ret <8 x i16>
+  return vec_sldb(vssa, vssb, 2);
+}
+
+vector unsigned short test_vec_sldb_us(void) {
+  // CHECK: @llvm.ppc.altivec.vsldbi(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32 3
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <8 x i16>
+  // CHECK-NEXT: ret <8 x i16>
+  return vec_sldb(vusa, vusb, 3);
+}
+
+vector signed int test_vec_sldb_si(void) {
+  // CHECK: @llvm.ppc.altivec.vsldbi(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32 4
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <4 x i32>
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_sldb(vsia, vsib, 4);
+}
+
+vector unsigned int test_vec_sldb_ui(void) {
+  // CHECK: @llvm.ppc.altivec.vsldbi(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32 5
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <4 x i32>
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_sldb(vuia, vuib, 5);
+}
+
+vector signed long long test_vec_sldb_sll(void) {
+  // CHECK: @llvm.ppc.altivec.vsldbi(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32 6
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_sldb(vslla, vsllb, 6);
+}
+
+vector unsigned long long test_vec_sldb_ull(void) {
+  // CHECK: @llvm.ppc.altivec.vsldbi(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32 7
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_sldb(vulla, vullb, 7);
+}
+
+vector signed char test_vec_srdb_sc(void) {
+  // CHECK: @llvm.ppc.altivec.vsrdbi(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32 0
+  // CHECK-NEXT: ret <16 x i8>
+  return vec_srdb(vsca, vscb, 8);
+}
+
+vector unsigned char test_vec_srdb_uc(void) {
+  // CHECK: @llvm.ppc.altivec.vsrdbi(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32 1
+  // CHECK-NEXT: ret <16 x i8>
+  return vec_srdb(vuca, vucb, 9);
+}
+
+vector signed short test_vec_srdb_ss(void) {
+  // CHECK: @llvm.ppc.altivec.vsrdbi(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32 2
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <8 x i16>
+  // CHECK-NEXT: ret <8 x i16>
+  return vec_srdb(vssa, vssb, 10);
+}
+
+vector unsigned short test_vec_srdb_us(void) {
+  // CHECK: @llvm.ppc.altivec.vsrdbi(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32 3
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <8 x i16>
+  // CHECK-NEXT: ret <8 x i16>
+  return vec_srdb(vusa, vusb, 3);
+}
+
+vector signed int test_vec_srdb_si(void) {
+  // CHECK: @llvm.ppc.altivec.vsrdbi(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32 4
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <4 x i32>
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_srdb(vsia, vsib, 4);
+}
+
+vector unsigned int test_vec_srdb_ui(void) {
+  // CHECK: @llvm.ppc.altivec.vsrdbi(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32 5
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <4 x i32>
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_srdb(vuia, vuib, 5);
+}
+
+vector signed long long test_vec_srdb_sll(void) {
+  // CHECK: @llvm.ppc.altivec.vsrdbi(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32 6
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_srdb(vslla, vsllb, 6);
+}
+
+vector unsigned long long test_vec_srdb_ull(void) {
+  // CHECK: @llvm.ppc.altivec.vsrdbi(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32 7
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_srdb(vulla, vullb, 7);
+}
+
+vector signed char test_vec_permx_sc(void) {
+  // CHECK: @llvm.ppc.vsx.xxpermx(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32
+  // CHECK-NEXT: ret <16 x i8>
+  return vec_permx(vsca, vscb, vucc, 0);
+}
+
+vector unsigned char test_vec_permx_uc(void) {
+  // CHECK: @llvm.ppc.vsx.xxpermx(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32
+  // CHECK-NEXT: ret <16 x i8>
+  return vec_permx(vuca, vucb, vucc, 1);
+}
+
+vector signed short test_vec_permx_ss(void) {
+  // CHECK: @llvm.ppc.vsx.xxpermx(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <8 x i16>
+  // CHECK-NEXT: ret <8 x i16>
+  return vec_permx(vssa, vssb, vucc, 2);
+}
+
+vector unsigned short test_vec_permx_us(void) {
+  // CHECK: @llvm.ppc.vsx.xxpermx(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <8 x i16>
+  // CHECK-NEXT: ret <8 x i16>
+  return vec_permx(vusa, vusb, vucc, 3);
+}
+
+vector signed int test_vec_permx_si(void) {
+  // CHECK: @llvm.ppc.vsx.xxpermx(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <4 x i32>
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_permx(vsia, vsib, vucc, 4);
+}
+
+vector unsigned int test_vec_permx_ui(void) {
+  // CHECK: @llvm.ppc.vsx.xxpermx(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <4 x i32>
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_permx(vuia, vuib, vucc, 5);
+}
+
+vector signed long long test_vec_permx_sll(void) {
+  // CHECK: @llvm.ppc.vsx.xxpermx(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_permx(vslla, vsllb, vucc, 6);
+}
+
+vector unsigned long long test_vec_permx_ull(void) {
+  // CHECK: @llvm.ppc.vsx.xxpermx(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_permx(vulla, vullb, vucc, 7);
+}
+
+vector float test_vec_permx_f(void) {
+  // CHECK: @llvm.ppc.vsx.xxpermx(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <4 x float>
+  // CHECK-NEXT: ret <4 x float>
+  return vec_permx(vfa, vfb, vucc, 0);
+}
+
+vector double test_vec_permx_d(void) {
+  // CHECK: @llvm.ppc.vsx.xxpermx(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, <16 x i8> %{{.+}}, i32
+  // CHECK-NEXT: bitcast <16 x i8> %{{.*}} to <2 x double>
+  // CHECK-NEXT: ret <2 x double>
+  return vec_permx(vda, vdb, vucc, 1);
+}
+
+vector signed char test_vec_blend_sc(void) {
+  // CHECK: @llvm.ppc.vsx.xxblendvb(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, <16 x i8>
+  // CHECK-NEXT: ret <16 x i8>
+  return vec_blendv(vsca, vscb, vucc);
+}
+
+vector unsigned char test_vec_blend_uc(void) {
+  // CHECK: @llvm.ppc.vsx.xxblendvb(<16 x i8> %{{.+}}, <16 x i8> %{{.+}}, <16 x i8>
+  // CHECK-NEXT: ret <16 x i8>
+  return vec_blendv(vuca, vucb, vucc);
+}
+
+vector signed short test_vec_blend_ss(void) {
+  // CHECK: @llvm.ppc.vsx.xxblendvh(<8 x i16> %{{.+}}, <8 x i16> %{{.+}}, <8 x i16>
+  // CHECK-NEXT: ret <8 x i16>
+  return vec_blendv(vssa, vssb, vusc);
+}
+
+vector unsigned short test_vec_blend_us(void) {
+  // CHECK: @llvm.ppc.vsx.xxblendvh(<8 x i16> %{{.+}}, <8 x i16> %{{.+}}, <8 x i16>
+  // CHECK-NEXT: ret <8 x i16>
+  return vec_blendv(vusa, vusb, vusc);
+}
+
+vector signed int test_vec_blend_si(void) {
+  // CHECK: @llvm.ppc.vsx.xxblendvw(<4 x i32> %{{.+}}, <4 x i32> %{{.+}}, <4 x i32>
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_blendv(vsia, vsib, vuic);
+}
+
+vector unsigned int test_vec_blend_ui(void) {
+  // CHECK: @llvm.ppc.vsx.xxblendvw(<4 x i32> %{{.+}}, <4 x i32> %{{.+}}, <4 x i32>
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_blendv(vuia, vuib, vuic);
+}
+
+vector signed long long test_vec_blend_sll(void) {
+  // CHECK: @llvm.ppc.vsx.xxblendvd(<2 x i64> %{{.+}}, <2 x i64> %{{.+}}, <2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_blendv(vslla, vsllb, vullc);
+}
+
+vector unsigned long long test_vec_blend_ull(void) {
+  // CHECK: @llvm.ppc.vsx.xxblendvd(<2 x i64> %{{.+}}, <2 x i64> %{{.+}}, <2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_blendv(vulla, vullb, vullc);
+}
+
+vector float test_vec_blend_f(void) {
+  // CHECK: @llvm.ppc.vsx.xxblendvw(<4 x i32> %{{.+}}, <4 x i32> %{{.+}}, <4 x i32>
+  // CHECK-NEXT: bitcast <4 x i32> %{{.*}} to <4 x float>
+  // CHECK-NEXT: ret <4 x float>
+  return vec_blendv(vfa, vfb, vuic);
+}
+
+vector double test_vec_blend_d(void) {
+  // CHECK: @llvm.ppc.vsx.xxblendvd(<2 x i64> %{{.+}}, <2 x i64> %{{.+}}, <2 x i64>
+  // CHECK-NEXT: bitcast <2 x i64> %{{.*}} to <2 x double>
+  // CHECK-NEXT: ret <2 x double>
+  return vec_blendv(vda, vdb, vullc);
+}
+
+vector unsigned char test_vec_insertl_uc(void) {
+  // CHECK-BE: @llvm.ppc.altivec.vinsblx(<16 x i8> %{{.+}}, i64 %{{.+}}, i64
+  // CHECK-BE-NEXT: ret <16 x i8>
+  // CHECK-LE: @llvm.ppc.altivec.vinsbrx(<16 x i8> %{{.+}}, i64 %{{.+}}, i64
+  // CHECK-LE-NEXT: ret <16 x i8>
+  return vec_insertl(uca, vuca, uia);
+}
+
+vector unsigned short test_vec_insertl_us(void) {
+  // CHECK-BE: @llvm.ppc.altivec.vinshlx(<8 x i16> %{{.+}}, i64 %{{.+}}, i64
+  // CHECK-BE-NEXT: ret <8 x i16>
+  // CHECK-LE: @llvm.ppc.altivec.vinshrx(<8 x i16> %{{.+}}, i64 %{{.+}}, i64
+  // CHECK-LE-NEXT: ret <8 x i16>
+  return vec_insertl(usa, vusa, uia);
+}
+
+vector unsigned int test_vec_insertl_ui(void) {
+  // CHECK-BE: @llvm.ppc.altivec.vinswlx(<4 x i32> %{{.+}}, i64 %{{.+}}, i64
+  // CHECK-BE-NEXT: ret <4 x i32>
+  // CHECK-LE: @llvm.ppc.altivec.vinswrx(<4 x i32> %{{.+}}, i64 %{{.+}}, i64
+  // CHECK-LE-NEXT: ret <4 x i32>
+  return vec_insertl(uib, vuia, uia);
+}
+
+vector unsigned long long test_vec_insertl_ul(void) {
+  // CHECK-BE: @llvm.ppc.altivec.vinsdlx(<2 x i64> %{{.+}}, i64 %{{.+}}, i64
+  // CHECK-BE-NEXT: ret <2 x i64>
+  // CHECK-LE: @llvm.ppc.altivec.vinsdrx(<2 x i64> %{{.+}}, i64 %{{.+}}, i64
+  // CHECK-LE-NEXT: ret <2 x i64>
+  return vec_insertl(ulla, vulla, uia);
+}
+
+vector unsigned char test_vec_insertl_ucv(void) {
+  // CHECK-BE: @llvm.ppc.altivec.vinsbvlx(<16 x i8> %{{.+}}, i64 %{{.+}}, <16 x i8>
+  // CHECK-BE-NEXT: ret <16 x i8>
+  // CHECK-LE: @llvm.ppc.altivec.vinsbvrx(<16 x i8> %{{.+}}, i64 %{{.+}}, <16 x i8>
+  // CHECK-LE-NEXT: ret <16 x i8>
+  return vec_insertl(vuca, vucb, uia);
+}
+
+vector unsigned short test_vec_insertl_usv(void) {
+  // CHECK-BE: @llvm.ppc.altivec.vinshvlx(<8 x i16> %{{.+}}, i64 %{{.+}}, <8 x i16>
+  // CHECK-BE-NEXT: ret <8 x i16>
+  // CHECK-LE: @llvm.ppc.altivec.vinshvrx(<8 x i16> %{{.+}}, i64 %{{.+}}, <8 x i16>
+  // CHECK-LE-NEXT: ret <8 x i16>
+  return vec_insertl(vusa, vusb, uia);
+}
+
+vector unsigned int test_vec_insertl_uiv(void) {
+  // CHECK-BE: @llvm.ppc.altivec.vinswvlx(<4 x i32> %{{.+}}, i64 %{{.+}}, <4 x i32>
+  // CHECK-BE-NEXT: ret <4 x i32>
+  // CHECK-LE: @llvm.ppc.altivec.vinswvrx(<4 x i32> %{{.+}}, i64 %{{.+}}, <4 x i32>
+  // CHECK-LE-NEXT: ret <4 x i32>
+  return vec_insertl(vuia, vuib, uia);
+}
+
+vector unsigned char test_vec_inserth_uc(void) {
+  // CHECK-BE: @llvm.ppc.altivec.vinsbrx(<16 x i8> %{{.+}}, i64 %{{.+}}, i64
+  // CHECK-BE-NEXT: ret <16 x i8>
+  // CHECK-LE: @llvm.ppc.altivec.vinsblx(<16 x i8> %{{.+}}, i64 %{{.+}}, i64
+  // CHECK-LE-NEXT: ret <16 x i8>
+  return vec_inserth(uca, vuca, uia);
+}
+
+vector unsigned short test_vec_inserth_us(void) {
+  // CHECK-BE: @llvm.ppc.altivec.vinshrx(<8 x i16> %{{.+}}, i64 %{{.+}}, i64
+  // CHECK-BE-NEXT: ret <8 x i16>
+  // CHECK-LE: @llvm.ppc.altivec.vinshlx(<8 x i16> %{{.+}}, i64 %{{.+}}, i64
+  // CHECK-LE-NEXT: ret <8 x i16>
+  return vec_inserth(usa, vusa, uia);
+}
+
+vector unsigned int test_vec_inserth_ui(void) {
+  // CHECK-BE: @llvm.ppc.altivec.vinswrx(<4 x i32> %{{.+}}, i64 %{{.+}}, i64
+  // CHECK-BE-NEXT: ret <4 x i32>
+  // CHECK-LE: @llvm.ppc.altivec.vinswlx(<4 x i32> %{{.+}}, i64 %{{.+}}, i64
+  // CHECK-LE-NEXT: ret <4 x i32>
+  return vec_inserth(uib, vuia, uia);
+}
+
+vector unsigned long long test_vec_inserth_ul(void) {
+  // CHECK-BE: @llvm.ppc.altivec.vinsdrx(<2 x i64> %{{.+}}, i64 %{{.+}}, i64
+  // CHECK-BE-NEXT: ret <2 x i64>
+  // CHECK-LE: @llvm.ppc.altivec.vinsdlx(<2 x i64> %{{.+}}, i64 %{{.+}}, i64
+  // CHECK-LE-NEXT: ret <2 x i64>
+  return vec_inserth(ulla, vulla, uia);
+}
+
+vector unsigned char test_vec_inserth_ucv(void) {
+  // CHECK-BE: @llvm.ppc.altivec.vinsbvrx(<16 x i8> %{{.+}}, i64 %{{.+}}, <16 x i8>
+  // CHECK-BE-NEXT: ret <16 x i8>
+  // CHECK-LE: @llvm.ppc.altivec.vinsbvlx(<16 x i8> %{{.+}}, i64 %{{.+}}, <16 x i8>
+  // CHECK-LE-NEXT: ret <16 x i8>
+  return vec_inserth(vuca, vucb, uia);
+}
+
+vector unsigned short test_vec_inserth_usv(void) {
+  // CHECK-BE: @llvm.ppc.altivec.vinshvrx(<8 x i16> %{{.+}}, i64 %{{.+}}, <8 x i16>
+  // CHECK-BE-NEXT: ret <8 x i16>
+  // CHECK-LE: @llvm.ppc.altivec.vinshvlx(<8 x i16> %{{.+}}, i64 %{{.+}}, <8 x i16>
+  // CHECK-LE-NEXT: ret <8 x i16>
+  return vec_inserth(vusa, vusb, uia);
+}
+
+vector unsigned int test_vec_inserth_uiv(void) {
+  // CHECK-BE: @llvm.ppc.altivec.vinswvrx(<4 x i32> %{{.+}}, i64 %{{.+}}, <4 x i32>
+  // CHECK-BE-NEXT: ret <4 x i32>
+  // CHECK-LE: @llvm.ppc.altivec.vinswvlx(<4 x i32> %{{.+}}, i64 %{{.+}}, <4 x i32>
+  // CHECK-LE-NEXT: ret <4 x i32>
+  return vec_inserth(vuia, vuib, uia);
+}
+
+vector signed int test_vec_vec_splati_si(void) {
+  // CHECK-BE: ret <4 x i32> <i32 -17, i32 -17, i32 -17, i32 -17>
+  // CHECK: ret <4 x i32> <i32 -17, i32 -17, i32 -17, i32 -17>
+  return vec_splati(-17);
+}
+
+vector unsigned int test_vec_vec_splati_ui(void) {
+  // CHECK-BE: ret <4 x i32> <i32 16, i32 16, i32 16, i32 16>
+  // CHECK: ret <4 x i32> <i32 16, i32 16, i32 16, i32 16>
+  return vec_splati(16U);
+}
+
+vector float test_vec_vec_splati_f(void) {
+  // CHECK-BE: ret <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  // CHECK: ret <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  return vec_splati(1.0f);
+}
+
+vector double test_vec_vec_splatid(void) {
+  // CHECK-BE: [[T1:%.+]] = fpext float %{{.+}} to double
+  // CHECK-BE-NEXT: [[T2:%.+]] = insertelement <2 x double> undef, double [[T1:%.+]], i32 0
+  // CHECK-BE-NEXT: [[T3:%.+]] = shufflevector <2 x double> [[T2:%.+]], <2 x double> undef, <2 x i32> zeroinitialize
+  // CHECK-BE-NEXT: ret <2 x double> [[T3:%.+]]
+  // CHECK: [[T1:%.+]] = fpext float %{{.+}} to double
+  // CHECK-NEXT: [[T2:%.+]] = insertelement <2 x double> undef, double [[T1:%.+]], i32 0
+  // CHECK-NEXT: [[T3:%.+]] = shufflevector <2 x double> [[T2:%.+]], <2 x double> undef, <2 x i32> zeroinitialize
+  // CHECK-NEXT: ret <2 x double> [[T3:%.+]]
+  return vec_splatid(1.0);
+}
+
+vector signed int test_vec_vec_splati_ins_si(void) {
+  // CHECK-BE: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 %{{.+}}
+  // CHECK-BE:  [[T1:%.+]] = add i32 2, %{{.+}}
+  // CHECK-BE: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 [[T1]]
+  // CHECK-BE: ret <4 x i32>
+  // CHECK:  [[T1:%.+]] = sub i32 1, %{{.+}}
+  // CHECK: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 [[T1]]
+  // CHECK:  [[T2:%.+]] = sub i32 3, %{{.+}}
+  // CHECK: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 [[T2]]
+  // CHECK: ret <4 x i32>
+  return vec_splati_ins(vsia, 0, -17);
+}
+
+vector unsigned int test_vec_vec_splati_ins_ui(void) {
+  // CHECK-BE: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 %{{.+}}
+  // CHECK-BE:  [[T1:%.+]] = add i32 2, %{{.+}}
+  // CHECK-BE: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 [[T1]]
+  // CHECK-BE: ret <4 x i32>
+  // CHECK:  [[T1:%.+]] = sub i32 1, %{{.+}}
+  // CHECK: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 [[T1]]
+  // CHECK:  [[T2:%.+]] = sub i32 3, %{{.+}}
+  // CHECK: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 [[T2]]
+  // CHECK: ret <4 x i32>
+  return vec_splati_ins(vuia, 1, 16U);
+}
+
+vector float test_vec_vec_splati_ins_f(void) {
+  // CHECK-BE: insertelement <4 x float> %{{.+}}, float %{{.+}}, i32 %{{.+}}
+  // CHECK-BE:  [[T1:%.+]] = add i32 2, %{{.+}}
+  // CHECK-BE: insertelement <4 x float> %{{.+}}, float %{{.+}}, i32 [[T1]]
+  // CHECK-BE: ret <4 x float>
+  // CHECK:  [[T1:%.+]] = sub i32 1, %{{.+}}
+  // CHECK: insertelement <4 x float> %{{.+}}, float %{{.+}}, i32 [[T1]]
+  // CHECK:  [[T2:%.+]] = sub i32 3, %{{.+}}
+  // CHECK: insertelement <4 x float> %{{.+}}, float %{{.+}}, i32 [[T2]]
+  // CHECK: ret <4 x float>
+  return vec_splati_ins(vfa, 0, 1.0f);
+}
diff --git a/clang/test/CodeGen/builtins-wasm.c b/clang/test/CodeGen/builtins-wasm.c
index 36d259f7405d7..f7e3dc1ea5e7a 100644
--- a/clang/test/CodeGen/builtins-wasm.c
+++ b/clang/test/CodeGen/builtins-wasm.c
@@ -26,18 +26,6 @@ __SIZE_TYPE__ memory_grow(__SIZE_TYPE__ delta) {
   // WEBASSEMBLY64: call i64 @llvm.wasm.memory.grow.i64(i32 0, i64 %{{.*}})
 }
 
-void memory_init(void *dest, int offset, int size) {
-  __builtin_wasm_memory_init(3, 0, dest, offset, size);
-  // WEBASSEMBLY32: call void @llvm.wasm.memory.init(i32 3, i32 0, i8* %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
-  // WEBASSEMBLY64: call void @llvm.wasm.memory.init(i32 3, i32 0, i8* %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
-}
-
-void data_drop() {
-  __builtin_wasm_data_drop(3);
-  // WEBASSEMBLY32: call void @llvm.wasm.data.drop(i32 3)
-  // WEBASSEMBLY64: call void @llvm.wasm.data.drop(i32 3)
-}
-
 __SIZE_TYPE__ tls_size() {
   return __builtin_wasm_tls_size();
   // WEBASSEMBLY32: call i32 @llvm.wasm.tls.size.i32()
diff --git a/clang/test/CodeGen/code-coverage.c b/clang/test/CodeGen/code-coverage.c
index 251e8bc77b312..34ba3554f5b54 100644
--- a/clang/test/CodeGen/code-coverage.c
+++ b/clang/test/CodeGen/code-coverage.c
@@ -9,7 +9,7 @@
 // RUN: %clang_cc1 -emit-llvm -disable-red-zone -femit-coverage-data %s -o - | \
 // RUN:   FileCheck --check-prefixes=CHECK,408 %s
 
-// RUN: %clang_cc1 -emit-llvm -disable-red-zone -femit-coverage-data -coverage-notes-file=aaa.gcno -coverage-data-file=bbb.gcda -dwarf-column-info -debug-info-kind=limited -dwarf-version=4 %s -o - | FileCheck %s --check-prefix GCOV_FILE_INFO
+// RUN: %clang_cc1 -emit-llvm -disable-red-zone -femit-coverage-data -coverage-notes-file=aaa.gcno -coverage-data-file=bbb.gcda -debug-info-kind=limited -dwarf-version=4 %s -o - | FileCheck %s --check-prefix GCOV_FILE_INFO
 
 // RUN: %clang_cc1 -emit-llvm-bc -o /dev/null -fexperimental-new-pass-manager -fdebug-pass-manager -femit-coverage-data %s 2>&1 | FileCheck --check-prefix=NEWPM %s
 // RUN: %clang_cc1 -emit-llvm-bc -o /dev/null -fexperimental-new-pass-manager -fdebug-pass-manager -femit-coverage-data -O3 %s 2>&1 | FileCheck --check-prefix=NEWPM-O3 %s
diff --git a/clang/test/CodeGen/fp-floatcontrol-pragma.cpp b/clang/test/CodeGen/fp-floatcontrol-pragma.cpp
index 7d8843220f44d..548809521f7f2 100644
--- a/clang/test/CodeGen/fp-floatcontrol-pragma.cpp
+++ b/clang/test/CodeGen/fp-floatcontrol-pragma.cpp
@@ -119,6 +119,24 @@ float fma_test1(float a, float b, float c) {
   return x;
 }
 
+#pragma float_control(push)
+#pragma float_control(precise, on)
+struct Distance {};
+Distance operator+(Distance, Distance);
+
+template <class T>
+T add(T lhs, T rhs) {
+#pragma float_control(except, on)
+  return lhs + rhs;
+}
+#pragma float_control(pop)
+
+float test_OperatorCall() {
+  return add(1.0f, 2.0f);
+  //CHECK: llvm.experimental.constrained.fadd{{.*}}fpexcept.strict
+}
+// CHECK-LABEL define float  {{.*}}test_OperatorCall{{.*}}
+
 #if FENV_ON
 // expected-warning@+1{{pragma STDC FENV_ACCESS ON is not supported, ignoring pragma}}
 #pragma STDC FENV_ACCESS ON
diff --git a/clang/test/CodeGen/fp-strictfp-exp.cpp b/clang/test/CodeGen/fp-strictfp-exp.cpp
new file mode 100644
index 0000000000000..7b9718a50ff2b
--- /dev/null
+++ b/clang/test/CodeGen/fp-strictfp-exp.cpp
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 -triple mips64-linux-gnu -fexperimental-strict-floating-point -frounding-math -ffp-exception-behavior=strict -O2 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple mips64-linux-gnu -fexperimental-strict-floating-point -ffp-exception-behavior=strict -O2 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple mips64-linux-gnu -fexperimental-strict-floating-point -frounding-math -O2 -emit-llvm -o - %s | FileCheck %s
+//
+// Verify that constrained intrinsics are used due to the experimental flag.
+// As more targets gain support for constrained intrinsics the triple
+// in this test will need to change.
+
+float fp_precise_1(float a, float b, float c) {
+// CHECK-LABEL: define float @_Z12fp_precise_1fff
+// CHECK: %[[M:.+]] = tail call float @llvm.experimental.constrained.fmul.f32(float {{.*}}, float {{.*}}, metadata {{.*}})
+// CHECK: tail call float @llvm.experimental.constrained.fadd.f32(float %[[M]], float %c, metadata {{.*}})
+  return a * b + c;
+}
diff --git a/clang/test/CodeGen/fp-strictfp.cpp b/clang/test/CodeGen/fp-strictfp.cpp
new file mode 100644
index 0000000000000..bfa13b4dd3545
--- /dev/null
+++ b/clang/test/CodeGen/fp-strictfp.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -triple mips64-linux-gnu -frounding-math -ffp-exception-behavior=strict -O2 -verify=rounding,exception -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple mips64-linux-gnu -ffp-exception-behavior=strict -O2 -verify=exception -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple mips64-linux-gnu -frounding-math -O2 -verify=rounding -emit-llvm -o - %s | FileCheck %s
+//
+// Verify that constrained intrinsics are not used.
+// As more targets gain support for constrained intrinsics the triple
+// in this test will need to change.
+
+// rounding-warning@* {{overriding currently unsupported rounding mode on this target}}
+// exception-warning@* {{overriding currently unsupported use of floating point exceptions on this target}}
+float fp_precise_1(float a, float b, float c) {
+// CHECK: define float @_Z12fp_precise_1fff
+// CHECK: %[[M:.+]] = fmul float{{.*}}
+// CHECK: fadd float %[[M]], %c
+  return a * b + c;
+}
diff --git a/clang/test/CodeGen/fpconstrained-cmp-double.c b/clang/test/CodeGen/fpconstrained-cmp-double.c
index 2819970a3fcff..335b7e49b01df 100644
--- a/clang/test/CodeGen/fpconstrained-cmp-double.c
+++ b/clang/test/CodeGen/fpconstrained-cmp-double.c
@@ -1,9 +1,9 @@
 // RUN: %clang_cc1 -ffp-exception-behavior=ignore -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=FCMP
-// RUN: %clang_cc1 -ffp-exception-behavior=strict -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=EXCEPT
-// RUN: %clang_cc1 -ffp-exception-behavior=maytrap -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=MAYTRAP
-// RUN: %clang_cc1 -frounding-math -ffp-exception-behavior=ignore -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=IGNORE
-// RUN: %clang_cc1 -frounding-math -ffp-exception-behavior=strict -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=EXCEPT
-// RUN: %clang_cc1 -frounding-math -ffp-exception-behavior=maytrap -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=MAYTRAP
+// RUN: %clang_cc1 -ffp-exception-behavior=strict -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=EXCEPT
+// RUN: %clang_cc1 -ffp-exception-behavior=maytrap -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=MAYTRAP
+// RUN: %clang_cc1 -frounding-math -ffp-exception-behavior=ignore -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=IGNORE
+// RUN: %clang_cc1 -frounding-math -ffp-exception-behavior=strict -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=EXCEPT
+// RUN: %clang_cc1 -frounding-math -ffp-exception-behavior=maytrap -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=MAYTRAP
 
 _Bool QuietEqual(double f1, double f2) {
   // CHECK-LABEL: define {{.*}}i1 @QuietEqual(double %f1, double %f2)
diff --git a/clang/test/CodeGen/fpconstrained-cmp-float.c b/clang/test/CodeGen/fpconstrained-cmp-float.c
index 0265fc54c02b5..e9667904122ba 100644
--- a/clang/test/CodeGen/fpconstrained-cmp-float.c
+++ b/clang/test/CodeGen/fpconstrained-cmp-float.c
@@ -1,9 +1,9 @@
-// RUN: %clang_cc1 -ffp-exception-behavior=ignore -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=FCMP
-// RUN: %clang_cc1 -ffp-exception-behavior=strict -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=EXCEPT
-// RUN: %clang_cc1 -ffp-exception-behavior=maytrap -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=MAYTRAP
-// RUN: %clang_cc1 -frounding-math -ffp-exception-behavior=ignore -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=IGNORE
-// RUN: %clang_cc1 -frounding-math -ffp-exception-behavior=strict -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=EXCEPT
-// RUN: %clang_cc1 -frounding-math -ffp-exception-behavior=maytrap -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=MAYTRAP
+// RUN: %clang_cc1 -ffp-exception-behavior=ignore -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=FCMP
+// RUN: %clang_cc1 -ffp-exception-behavior=strict -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=EXCEPT
+// RUN: %clang_cc1 -ffp-exception-behavior=maytrap -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=MAYTRAP
+// RUN: %clang_cc1 -frounding-math -ffp-exception-behavior=ignore -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=IGNORE
+// RUN: %clang_cc1 -frounding-math -ffp-exception-behavior=strict -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=EXCEPT
+// RUN: %clang_cc1 -frounding-math -ffp-exception-behavior=maytrap -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=MAYTRAP
 
 _Bool QuietEqual(float f1, float f2) {
   // CHECK-LABEL: define {{.*}}i1 @QuietEqual(float %f1, float %f2)
diff --git a/clang/test/CodeGen/fpconstrained.c b/clang/test/CodeGen/fpconstrained.c
index 902d6b5baf490..0307ebbd357f5 100644
--- a/clang/test/CodeGen/fpconstrained.c
+++ b/clang/test/CodeGen/fpconstrained.c
@@ -1,10 +1,11 @@
-// RUN: %clang_cc1 -ftrapping-math -frounding-math -ffp-exception-behavior=strict -emit-llvm -o - %s | FileCheck %s -check-prefix=FPMODELSTRICT
+// RUN: %clang_cc1 -ftrapping-math -frounding-math -ffp-exception-behavior=strict -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=FPMODELSTRICT
 // RUN: %clang_cc1 -ffp-contract=fast -emit-llvm -o - %s | FileCheck %s -check-prefix=PRECISE
 // RUN: %clang_cc1 -ffast-math -ffp-contract=fast -emit-llvm -o - %s | FileCheck %s -check-prefix=FAST
 // RUN: %clang_cc1 -ffast-math -emit-llvm -o - %s | FileCheck %s -check-prefix=FASTNOCONTRACT
 // RUN: %clang_cc1 -ffast-math -ffp-contract=fast -ffp-exception-behavior=ignore -emit-llvm -o - %s | FileCheck %s -check-prefix=FAST
-// RUN: %clang_cc1 -ffast-math -ffp-contract=fast -ffp-exception-behavior=strict -emit-llvm -o - %s | FileCheck %s -check-prefix=EXCEPT
-// RUN: %clang_cc1 -ffast-math -ffp-contract=fast -ffp-exception-behavior=maytrap -emit-llvm -o - %s | FileCheck %s -check-prefix=MAYTRAP
+// RUN: %clang_cc1 -ffast-math -ffp-contract=fast -ffp-exception-behavior=strict -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=EXCEPT
+// RUN: %clang_cc1 -ffast-math -ffp-contract=fast -ffp-exception-behavior=maytrap -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=MAYTRAP
+
 float f0, f1, f2;
 
 void foo() {
diff --git a/clang/test/CodeGen/fpconstrained.cpp b/clang/test/CodeGen/fpconstrained.cpp
index e914abcc69268..305c3684486d9 100644
--- a/clang/test/CodeGen/fpconstrained.cpp
+++ b/clang/test/CodeGen/fpconstrained.cpp
@@ -1,10 +1,11 @@
-// RUN: %clang_cc1 -x c++ -ftrapping-math -fexceptions -fcxx-exceptions -frounding-math -ffp-exception-behavior=strict -emit-llvm -o - %s | FileCheck %s -check-prefix=FPMODELSTRICT
+// RUN: %clang_cc1 -x c++ -ftrapping-math -fexceptions -fcxx-exceptions -frounding-math -ffp-exception-behavior=strict -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=FPMODELSTRICT
 // RUN: %clang_cc1 -x c++ -ffp-contract=fast -fexceptions -fcxx-exceptions -emit-llvm -o - %s | FileCheck %s -check-prefix=PRECISE
 // RUN: %clang_cc1 -x c++ -ffast-math -fexceptions -fcxx-exceptions -ffp-contract=fast -emit-llvm -o - %s | FileCheck %s -check-prefix=FAST
 // RUN: %clang_cc1 -x c++ -ffast-math -fexceptions -fcxx-exceptions -emit-llvm -o - %s | FileCheck %s -check-prefix=FASTNOCONTRACT
 // RUN: %clang_cc1 -x c++ -ffast-math -fexceptions -fcxx-exceptions -ffp-contract=fast -ffp-exception-behavior=ignore -emit-llvm -o - %s | FileCheck %s -check-prefix=FAST
-// RUN: %clang_cc1 -x c++ -ffast-math -fexceptions -fcxx-exceptions -ffp-contract=fast -ffp-exception-behavior=strict -emit-llvm -o - %s | FileCheck %s -check-prefix=EXCEPT
-// RUN: %clang_cc1 -x c++ -ffast-math -fexceptions -fcxx-exceptions -ffp-contract=fast -ffp-exception-behavior=maytrap -emit-llvm -o - %s | FileCheck %s -check-prefix=MAYTRAP
+// RUN: %clang_cc1 -x c++ -ffast-math -fexceptions -fcxx-exceptions -ffp-contract=fast -ffp-exception-behavior=strict -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=EXCEPT
+// RUN: %clang_cc1 -x c++ -ffast-math -fexceptions -fcxx-exceptions -ffp-contract=fast -ffp-exception-behavior=maytrap -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=MAYTRAP
+
 float f0, f1, f2;
 
   template <class>
diff --git a/clang/test/CodeGen/linetable-endscope.c b/clang/test/CodeGen/linetable-endscope.c
index 6eefbea2fc667..03291a2ea4aa9 100644
--- a/clang/test/CodeGen/linetable-endscope.c
+++ b/clang/test/CodeGen/linetable-endscope.c
@@ -11,7 +11,7 @@
 void foo(char c)
 {
   int i;
-  // CHECK: ![[CONV]] = !DILocation(line: [[@LINE+1]], scope: !{{.*}})
+  // CHECK: ![[CONV]] = !DILocation(line: [[@LINE+1]], column: 5, scope: !{{.*}})
   i = c;
-  // CHECK: ![[RET]] = !DILocation(line: [[@LINE+1]], scope: !{{.*}})
+  // CHECK: ![[RET]] = !DILocation(line: [[@LINE+1]], column: 1, scope: !{{.*}})
 }
diff --git a/clang/test/CodeGen/ms-intrinsics.c b/clang/test/CodeGen/ms-intrinsics.c
index fed789e60d373..5182ae4039849 100644
--- a/clang/test/CodeGen/ms-intrinsics.c
+++ b/clang/test/CodeGen/ms-intrinsics.c
@@ -523,72 +523,72 @@ void test_iso_volatile_store64(__int64 volatile *p, __int64 v) { __iso_volatile_
 // CHECK: store volatile i64 %v, i64* %p
 
 
-#if defined(__x86_64__) || defined(__arm__) || defined(__aarch64__)
+#if defined(__i386__) || defined(__x86_64__) || defined(__arm__) || defined(__aarch64__)
 __int64 test_InterlockedExchange64(__int64 volatile *value, __int64 mask) {
   return _InterlockedExchange64(value, mask);
 }
-// CHECK-ARM-X64: define{{.*}}i64 @test_InterlockedExchange64(i64*{{[a-z_ ]*}}%value, i64{{[a-z_ ]*}}%mask){{.*}}{
-// CHECK-ARM-X64:   [[RESULT:%[0-9]+]] = atomicrmw xchg i64* %value, i64 %mask seq_cst
-// CHECK-ARM-X64:   ret i64 [[RESULT:%[0-9]+]]
-// CHECK-ARM-X64: }
+// CHECK: define{{.*}}i64 @test_InterlockedExchange64(i64*{{[a-z_ ]*}}%value, i64{{[a-z_ ]*}}%mask){{.*}}{
+// CHECK:   [[RESULT:%[0-9]+]] = atomicrmw xchg i64* %value, i64 %mask seq_cst
+// CHECK:   ret i64 [[RESULT:%[0-9]+]]
+// CHECK: }
 
 __int64 test_InterlockedExchangeAdd64(__int64 volatile *value, __int64 mask) {
   return _InterlockedExchangeAdd64(value, mask);
 }
-// CHECK-ARM-X64: define{{.*}}i64 @test_InterlockedExchangeAdd64(i64*{{[a-z_ ]*}}%value, i64{{[a-z_ ]*}}%mask){{.*}}{
-// CHECK-ARM-X64:   [[RESULT:%[0-9]+]] = atomicrmw add i64* %value, i64 %mask seq_cst
-// CHECK-ARM-X64:   ret i64 [[RESULT:%[0-9]+]]
-// CHECK-ARM-X64: }
+// CHECK: define{{.*}}i64 @test_InterlockedExchangeAdd64(i64*{{[a-z_ ]*}}%value, i64{{[a-z_ ]*}}%mask){{.*}}{
+// CHECK:   [[RESULT:%[0-9]+]] = atomicrmw add i64* %value, i64 %mask seq_cst
+// CHECK:   ret i64 [[RESULT:%[0-9]+]]
+// CHECK: }
 
 __int64 test_InterlockedExchangeSub64(__int64 volatile *value, __int64 mask) {
   return _InterlockedExchangeSub64(value, mask);
 }
-// CHECK-ARM-X64: define{{.*}}i64 @test_InterlockedExchangeSub64(i64*{{[a-z_ ]*}}%value, i64{{[a-z_ ]*}}%mask){{.*}}{
-// CHECK-ARM-X64:   [[RESULT:%[0-9]+]] = atomicrmw sub i64* %value, i64 %mask seq_cst
-// CHECK-ARM-X64:   ret i64 [[RESULT:%[0-9]+]]
-// CHECK-ARM-X64: }
+// CHECK: define{{.*}}i64 @test_InterlockedExchangeSub64(i64*{{[a-z_ ]*}}%value, i64{{[a-z_ ]*}}%mask){{.*}}{
+// CHECK:   [[RESULT:%[0-9]+]] = atomicrmw sub i64* %value, i64 %mask seq_cst
+// CHECK:   ret i64 [[RESULT:%[0-9]+]]
+// CHECK: }
 
 __int64 test_InterlockedOr64(__int64 volatile *value, __int64 mask) {
   return _InterlockedOr64(value, mask);
 }
-// CHECK-ARM-X64: define{{.*}}i64 @test_InterlockedOr64(i64*{{[a-z_ ]*}}%value, i64{{[a-z_ ]*}}%mask){{.*}}{
-// CHECK-ARM-X64:   [[RESULT:%[0-9]+]] = atomicrmw or i64* %value, i64 %mask seq_cst
-// CHECK-ARM-X64:   ret i64 [[RESULT:%[0-9]+]]
-// CHECK-ARM-X64: }
+// CHECK: define{{.*}}i64 @test_InterlockedOr64(i64*{{[a-z_ ]*}}%value, i64{{[a-z_ ]*}}%mask){{.*}}{
+// CHECK:   [[RESULT:%[0-9]+]] = atomicrmw or i64* %value, i64 %mask seq_cst
+// CHECK:   ret i64 [[RESULT:%[0-9]+]]
+// CHECK: }
 
 __int64 test_InterlockedXor64(__int64 volatile *value, __int64 mask) {
   return _InterlockedXor64(value, mask);
 }
-// CHECK-ARM-X64: define{{.*}}i64 @test_InterlockedXor64(i64*{{[a-z_ ]*}}%value, i64{{[a-z_ ]*}}%mask){{.*}}{
-// CHECK-ARM-X64:   [[RESULT:%[0-9]+]] = atomicrmw xor i64* %value, i64 %mask seq_cst
-// CHECK-ARM-X64:   ret i64 [[RESULT:%[0-9]+]]
-// CHECK-ARM-X64: }
+// CHECK: define{{.*}}i64 @test_InterlockedXor64(i64*{{[a-z_ ]*}}%value, i64{{[a-z_ ]*}}%mask){{.*}}{
+// CHECK:   [[RESULT:%[0-9]+]] = atomicrmw xor i64* %value, i64 %mask seq_cst
+// CHECK:   ret i64 [[RESULT:%[0-9]+]]
+// CHECK: }
 
 __int64 test_InterlockedAnd64(__int64 volatile *value, __int64 mask) {
   return _InterlockedAnd64(value, mask);
 }
-// CHECK-ARM-X64: define{{.*}}i64 @test_InterlockedAnd64(i64*{{[a-z_ ]*}}%value, i64{{[a-z_ ]*}}%mask){{.*}}{
-// CHECK-ARM-X64:   [[RESULT:%[0-9]+]] = atomicrmw and i64* %value, i64 %mask seq_cst
-// CHECK-ARM-X64:   ret i64 [[RESULT:%[0-9]+]]
-// CHECK-ARM-X64: }
+// CHECK: define{{.*}}i64 @test_InterlockedAnd64(i64*{{[a-z_ ]*}}%value, i64{{[a-z_ ]*}}%mask){{.*}}{
+// CHECK:   [[RESULT:%[0-9]+]] = atomicrmw and i64* %value, i64 %mask seq_cst
+// CHECK:   ret i64 [[RESULT:%[0-9]+]]
+// CHECK: }
 
 __int64 test_InterlockedIncrement64(__int64 volatile *Addend) {
   return _InterlockedIncrement64(Addend);
 }
-// CHECK-ARM-X64: define{{.*}}i64 @test_InterlockedIncrement64(i64*{{[a-z_ ]*}}%Addend){{.*}}{
-// CHECK-ARM-X64: [[TMP:%[0-9]+]] = atomicrmw add i64* %Addend, i64 1 seq_cst
-// CHECK-ARM-X64: [[RESULT:%[0-9]+]] = add i64 [[TMP]], 1
-// CHECK-ARM-X64: ret i64 [[RESULT]]
-// CHECK-ARM-X64: }
+// CHECK: define{{.*}}i64 @test_InterlockedIncrement64(i64*{{[a-z_ ]*}}%Addend){{.*}}{
+// CHECK: [[TMP:%[0-9]+]] = atomicrmw add i64* %Addend, i64 1 seq_cst
+// CHECK: [[RESULT:%[0-9]+]] = add i64 [[TMP]], 1
+// CHECK: ret i64 [[RESULT]]
+// CHECK: }
 
 __int64 test_InterlockedDecrement64(__int64 volatile *Addend) {
   return _InterlockedDecrement64(Addend);
 }
-// CHECK-ARM-X64: define{{.*}}i64 @test_InterlockedDecrement64(i64*{{[a-z_ ]*}}%Addend){{.*}}{
-// CHECK-ARM-X64: [[TMP:%[0-9]+]] = atomicrmw sub i64* %Addend, i64 1 seq_cst
-// CHECK-ARM-X64: [[RESULT:%[0-9]+]] = add i64 [[TMP]], -1
-// CHECK-ARM-X64: ret i64 [[RESULT]]
-// CHECK-ARM-X64: }
+// CHECK: define{{.*}}i64 @test_InterlockedDecrement64(i64*{{[a-z_ ]*}}%Addend){{.*}}{
+// CHECK: [[TMP:%[0-9]+]] = atomicrmw sub i64* %Addend, i64 1 seq_cst
+// CHECK: [[RESULT:%[0-9]+]] = add i64 [[TMP]], -1
+// CHECK: ret i64 [[RESULT]]
+// CHECK: }
 
 #endif
 
diff --git a/clang/test/CodeGen/msp430-abi-complex.c b/clang/test/CodeGen/msp430-abi-complex.c
new file mode 100644
index 0000000000000..faafcd2cde3f8
--- /dev/null
+++ b/clang/test/CodeGen/msp430-abi-complex.c
@@ -0,0 +1,226 @@
+// REQUIRES: msp430-registered-target
+// RUN: %clang -target msp430 -Os -S -o- %s | FileCheck %s
+
+volatile int N;
+volatile int i16_1, i16_2;
+volatile long i32_1, i32_2;
+volatile long long i64_1, i64_2;
+volatile float f1, f2;
+volatile double d1, d2;
+
+_Static_assert(sizeof(int) == 2, "Assumption failed");
+_Static_assert(sizeof(long) == 4, "Assumption failed");
+_Static_assert(sizeof(long long) == 8, "Assumption failed");
+
+void complex_i16_arg_first(int _Complex x, int n) {
+// CHECK-LABEL: @complex_i16_arg_first
+  i16_1 = __real__ x;
+// CHECK-DAG: mov r12, &i16_1
+  i16_2 = __imag__ x;
+// CHECK-DAG: mov r13, &i16_2
+  N = n;
+// CHECK-DAG: mov r14, &N
+// CHECK:     ret
+}
+
+void complex_i16_arg_second(int n, int _Complex x) {
+// CHECK-LABEL: @complex_i16_arg_second
+  N = n;
+// CHECK-DAG: mov r12, &N
+  i16_1 = __real__ x;
+// CHECK-DAG: mov r13, &i16_1
+  i16_2 = __imag__ x;
+// CHECK-DAG: mov r14, &i16_2
+// CHECK:     ret
+}
+
+void complex_i32_arg_first(long _Complex x, int n) {
+// CHECK-LABEL: @complex_i32_arg_first
+  i32_1 = __real__ x;
+// CHECK-DAG: mov r12, &i32_1
+// CHECK-DAG: mov r13, &i32_1+2
+  i32_2 = __imag__ x;
+// CHECK-DAG: mov r14, &i32_2
+// CHECK-DAG: mov r15, &i32_2+2
+  N = n;
+// CHECK-DAG: mov 2(r1), &N
+// CHECK:     ret
+}
+
+void complex_i32_arg_second(int n, long _Complex x) {
+// CHECK-LABEL: @complex_i32_arg_second
+  N = n;
+// CHECK-DAG: mov r12, &N
+  i32_1 = __real__ x;
+// CHECK-DAG: mov 2(r1), &i32_1
+// CHECK-DAG: mov 4(r1), &i32_1+2
+  i32_2 = __imag__ x;
+// CHECK-DAG: mov 6(r1), &i32_2
+// CHECK-DAG: mov 8(r1), &i32_2+2
+// CHECK:     ret
+}
+
+void complex_i64_arg_first(long long _Complex x, int n) {
+// CHECK-LABEL: @complex_i64_arg_first
+  i64_1 = __real__ x;
+// CHECK-DAG: mov 2(r1), &i64_1
+// CHECK-DAG: mov 4(r1), &i64_1+2
+// CHECK-DAG: mov 6(r1), &i64_1+4
+// CHECK-DAG: mov 8(r1), &i64_1+6
+  i64_2 = __imag__ x;
+// CHECK-DAG: mov 10(r1), &i64_2
+// CHECK-DAG: mov 12(r1), &i64_2+2
+// CHECK-DAG: mov 14(r1), &i64_2+4
+// CHECK-DAG: mov 16(r1), &i64_2+6
+  N = n;
+// CHECK-DAG: mov r12, &N
+// CHECK:     ret
+}
+
+void complex_i64_arg_second(int n, long long _Complex x) {
+// CHECK-LABEL: @complex_i64_arg_second
+  N = n;
+// CHECK-DAG: mov r12, &N
+  i64_1 = __real__ x;
+// CHECK-DAG: mov 2(r1), &i64_1
+// CHECK-DAG: mov 4(r1), &i64_1+2
+// CHECK-DAG: mov 6(r1), &i64_1+4
+// CHECK-DAG: mov 8(r1), &i64_1+6
+  i64_2 = __imag__ x;
+// CHECK-DAG: mov 10(r1), &i64_2
+// CHECK-DAG: mov 12(r1), &i64_2+2
+// CHECK-DAG: mov 14(r1), &i64_2+4
+// CHECK-DAG: mov 16(r1), &i64_2+6
+// CHECK:     ret
+}
+
+void complex_float_arg_first(float _Complex x, int n) {
+// CHECK-LABEL: @complex_float_arg_first
+  f1 = __real__ x;
+// CHECK-DAG: mov r12, &f1
+// CHECK-DAG: mov r13, &f1+2
+  f2 = __imag__ x;
+// CHECK-DAG: mov r14, &f2
+// CHECK-DAG: mov r15, &f2+2
+  N = n;
+// CHECK-DAG: mov 2(r1), &N
+// CHECK:     ret
+}
+
+void complex_float_arg_second(int n, float _Complex x) {
+// CHECK-LABEL: @complex_float_arg_second
+  N = n;
+// CHECK-DAG: mov r12, &N
+  f1 = __real__ x;
+// CHECK-DAG: mov 2(r1), &f1
+// CHECK-DAG: mov 4(r1), &f1+2
+  f2 = __imag__ x;
+// CHECK-DAG: mov 6(r1), &f2
+// CHECK-DAG: mov 8(r1), &f2+2
+// CHECK:     ret
+}
+
+void complex_double_arg_first(double _Complex x, int n) {
+// CHECK-LABEL: @complex_double_arg_first
+  d1 = __real__ x;
+// CHECK-DAG: mov 2(r1), &d1
+// CHECK-DAG: mov 4(r1), &d1+2
+// CHECK-DAG: mov 6(r1), &d1+4
+// CHECK-DAG: mov 8(r1), &d1+6
+  d2 = __imag__ x;
+// CHECK-DAG: mov 10(r1), &d2
+// CHECK-DAG: mov 12(r1), &d2+2
+// CHECK-DAG: mov 14(r1), &d2+4
+// CHECK-DAG: mov 16(r1), &d2+6
+  N = n;
+// CHECK-DAG: mov r12, &N
+// CHECK:     ret
+}
+
+void complex_double_arg_second(int n, double _Complex x) {
+// CHECK-LABEL: @complex_double_arg_second
+  d1 = __real__ x;
+// CHECK-DAG: mov 2(r1), &d1
+// CHECK-DAG: mov 4(r1), &d1+2
+// CHECK-DAG: mov 6(r1), &d1+4
+// CHECK-DAG: mov 8(r1), &d1+6
+  d2 = __imag__ x;
+// CHECK-DAG: mov 10(r1), &d2
+// CHECK-DAG: mov 12(r1), &d2+2
+// CHECK-DAG: mov 14(r1), &d2+4
+// CHECK-DAG: mov 16(r1), &d2+6
+  N = n;
+// CHECK-DAG: mov r12, &N
+// CHECK:     ret
+}
+
+int _Complex complex_i16_res(void) {
+// CHECK-LABEL: @complex_i16_res
+  int _Complex res;
+  __real__ res = 0x1122;
+// CHECK-DAG: mov #4386, r12
+  __imag__ res = 0x3344;
+// CHECK-DAG: mov #13124, r13
+  return res;
+// CHECK:     ret
+}
+
+long _Complex complex_i32_res(void) {
+// CHECK-LABEL: @complex_i32_res
+  long _Complex res;
+  __real__ res = 0x11223344;
+// CHECK-DAG: mov #13124, r12
+// CHECK-DAG: mov #4386,  r13
+  __imag__ res = 0x55667788;
+// CHECK-DAG: mov #30600, r14
+// CHECK-DAG: mov #21862, r15
+  return res;
+// CHECK:     ret
+}
+
+long long _Complex complex_i64_res(void) {
+// CHECK-LABEL: @complex_i64_res
+  long long _Complex res;
+  __real__ res = 0x1122334455667788;
+// CHECK-DAG: mov #30600,  0(r12)
+// CHECK-DAG: mov #21862,  2(r12)
+// CHECK-DAG: mov #13124,  4(r12)
+// CHECK-DAG: mov #4386,   6(r12)
+  __imag__ res = 0x99aabbccddeeff00;
+// CHECK-DAG: mov #-256,   8(r12)
+// CHECK-DAG: mov #-8722,  10(r12)
+// CHECK-DAG: mov #-17460, 12(r12)
+// CHECK-DAG: mov #-26198, 14(r12)
+  return res;
+// CHECK:     ret
+}
+
+float _Complex complex_float_res(void) {
+// CHECK-LABEL: @complex_float_res
+  float _Complex res;
+  __real__ res = 1;
+// CHECK-DAG: clr r12
+// CHECK-DAG: mov #16256, r13
+  __imag__ res = -1;
+// CHECK-DAG: clr r14
+// CHECK-DAG: mov #-16512, r15
+  return res;
+// CHECK:     ret
+}
+
+double _Complex complex_double_res(void) {
+// CHECK-LABEL: @complex_double_res
+  double _Complex res;
+  __real__ res = 1;
+// CHECK-DAG: clr 0(r12)
+// CHECK-DAG: clr 2(r12)
+// CHECK-DAG: clr 4(r12)
+// CHECK-DAG: mov #16368, 6(r12)
+  __imag__ res = -1;
+// CHECK-DAG: clr 8(r12)
+// CHECK-DAG: clr 10(r12)
+// CHECK-DAG: clr 12(r12)
+// CHECK-DAG: mov #-16400, 14(r12)
+  return res;
+// CHECK:     ret
+}
diff --git a/clang/test/CodeGen/msp430-register-names.c b/clang/test/CodeGen/msp430-register-names.c
new file mode 100644
index 0000000000000..106b6c933a49b
--- /dev/null
+++ b/clang/test/CodeGen/msp430-register-names.c
@@ -0,0 +1,105 @@
+// Registers R0 - R3 have different names inside the LLVM MSP430 target code.
+// Test that they are handled properly when used inside clobber lists.
+// At the time of writing, llc silently ignores unknown register names.
+
+// REQUIRES: msp430-registered-target
+// RUN: %clang -target msp430 -c %s -mllvm -stop-after=finalize-isel -o- | FileCheck %s
+
+void test_function(void) {
+  asm volatile(""
+               :
+               :
+               : "r0");
+  asm volatile(""
+               :
+               :
+               : "r1");
+  asm volatile(""
+               :
+               :
+               : "r2");
+  asm volatile(""
+               :
+               :
+               : "r3");
+  asm volatile(""
+               :
+               :
+               : "r4");
+  asm volatile(""
+               :
+               :
+               : "r5");
+  asm volatile(""
+               :
+               :
+               : "r6");
+  asm volatile(""
+               :
+               :
+               : "r7");
+  asm volatile(""
+               :
+               :
+               : "r8");
+  asm volatile(""
+               :
+               :
+               : "r9");
+  asm volatile(""
+               :
+               :
+               : "r10");
+  asm volatile(""
+               :
+               :
+               : "r11");
+  asm volatile(""
+               :
+               :
+               : "r12");
+  asm volatile(""
+               :
+               :
+               : "r13");
+  asm volatile(""
+               :
+               :
+               : "r14");
+  asm volatile(""
+               :
+               :
+               : "r15");
+  // CHECK: call void asm sideeffect "", "~{pc}"()
+  // CHECK: call void asm sideeffect "", "~{sp}"()
+  // CHECK: call void asm sideeffect "", "~{sr}"()
+  // CHECK: call void asm sideeffect "", "~{cg}"()
+  // CHECK: call void asm sideeffect "", "~{r4}"()
+  // CHECK: call void asm sideeffect "", "~{r5}"()
+  // CHECK: call void asm sideeffect "", "~{r6}"()
+  // CHECK: call void asm sideeffect "", "~{r7}"()
+  // CHECK: call void asm sideeffect "", "~{r8}"()
+  // CHECK: call void asm sideeffect "", "~{r9}"()
+  // CHECK: call void asm sideeffect "", "~{r10}"()
+  // CHECK: call void asm sideeffect "", "~{r11}"()
+  // CHECK: call void asm sideeffect "", "~{r12}"()
+  // CHECK: call void asm sideeffect "", "~{r13}"()
+  // CHECK: call void asm sideeffect "", "~{r14}"()
+  // CHECK: call void asm sideeffect "", "~{r15}"()
+  // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $pc
+  // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $sp
+  // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $sr
+  // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $cg
+  // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r4
+  // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r5
+  // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r6
+  // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r7
+  // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r8
+  // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r9
+  // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r10
+  // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r11
+  // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r12
+  // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r13
+  // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r14
+  // CHECK: INLINEASM &"", {{.*}} implicit-def early-clobber $r15
+}
diff --git a/clang/test/CodeGen/opt-record-MIR.c b/clang/test/CodeGen/opt-record-MIR.c
index 731c6591c2372..40832d6e776a7 100644
--- a/clang/test/CodeGen/opt-record-MIR.c
+++ b/clang/test/CodeGen/opt-record-MIR.c
@@ -1,11 +1,11 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -triple arm64-apple-ios -S -o /dev/null %s -O2 -dwarf-column-info -Rpass-missed=regalloc 2>&1 | FileCheck -check-prefix=REMARK %s
-// RUN: %clang_cc1 -triple arm64-apple-ios -S -o /dev/null %s -O2 -dwarf-column-info 2>&1 | FileCheck -allow-empty -check-prefix=NO_REMARK %s
-// RUN: %clang_cc1 -triple arm64-apple-ios -S -o /dev/null %s -O2 -dwarf-column-info -opt-record-file %t.yaml
+// RUN: %clang_cc1 -triple arm64-apple-ios -S -o /dev/null %s -O2 -Rpass-missed=regalloc 2>&1 | FileCheck -check-prefix=REMARK %s
+// RUN: %clang_cc1 -triple arm64-apple-ios -S -o /dev/null %s -O2 2>&1 | FileCheck -allow-empty -check-prefix=NO_REMARK %s
+// RUN: %clang_cc1 -triple arm64-apple-ios -S -o /dev/null %s -O2 -opt-record-file %t.yaml
 // RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
-// RUN: %clang_cc1 -triple arm64-apple-ios -S -o /dev/null %s -O2 -dwarf-column-info -opt-record-file %t.yaml -opt-record-passes asm-printer
+// RUN: %clang_cc1 -triple arm64-apple-ios -S -o /dev/null %s -O2 -opt-record-file %t.yaml -opt-record-passes asm-printer
 // RUN: cat %t.yaml | FileCheck -check-prefix=PASSES %s
-// RUN: %clang_cc1 -triple arm64-apple-ios -S -o /dev/null %s -O2 -dwarf-column-info -opt-record-file %t.yaml -opt-record-format yaml
+// RUN: %clang_cc1 -triple arm64-apple-ios -S -o /dev/null %s -O2 -opt-record-file %t.yaml -opt-record-format yaml
 // RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
 
 void bar(float);
diff --git a/clang/test/CodeGen/opt-record.c b/clang/test/CodeGen/opt-record.c
index 481b45d9e9589..243d05d26e318 100644
--- a/clang/test/CodeGen/opt-record.c
+++ b/clang/test/CodeGen/opt-record.c
@@ -1,14 +1,14 @@
-// RUN: %clang_cc1 -O3 -triple x86_64-unknown-linux-gnu -target-cpu x86-64 %s -o %t -dwarf-column-info -opt-record-file %t.yaml -emit-obj
+// RUN: %clang_cc1 -O3 -triple x86_64-unknown-linux-gnu -target-cpu x86-64 %s -o %t -opt-record-file %t.yaml -emit-obj
 // RUN: cat %t.yaml | FileCheck %s
 // RUN: llvm-profdata merge %S/Inputs/opt-record.proftext -o %t.profdata
-// RUN: %clang_cc1 -O3 -triple x86_64-unknown-linux-gnu -target-cpu x86-64 -fprofile-instrument-use-path=%t.profdata %s -o %t -dwarf-column-info -opt-record-file %t.yaml -emit-obj
+// RUN: %clang_cc1 -O3 -triple x86_64-unknown-linux-gnu -target-cpu x86-64 -fprofile-instrument-use-path=%t.profdata %s -o %t -opt-record-file %t.yaml -emit-obj
 // RUN: cat %t.yaml | FileCheck -check-prefix=CHECK -check-prefix=CHECK-PGO %s
-// RUN: %clang_cc1 -O3 -triple x86_64-unknown-linux-gnu -target-cpu x86-64 %s -o %t -dwarf-column-info -opt-record-file %t.yaml -opt-record-passes inline -emit-obj
+// RUN: %clang_cc1 -O3 -triple x86_64-unknown-linux-gnu -target-cpu x86-64 %s -o %t -opt-record-file %t.yaml -opt-record-passes inline -emit-obj
 // RUN: cat %t.yaml | FileCheck -check-prefix=CHECK-PASSES %s
-// RUN: not %clang_cc1 -O3 -triple x86_64-unknown-linux-gnu -target-cpu x86-64 %s -o %t -dwarf-column-info -opt-record-file %t.yaml -opt-record-passes "(foo" -emit-obj 2>&1 | FileCheck -check-prefix=CHECK-PATTERN-ERROR %s
-// RUN: %clang_cc1 -O3 -triple x86_64-unknown-linux-gnu -target-cpu x86-64 %s -o %t -dwarf-column-info -opt-record-file %t.yaml -opt-record-format yaml -emit-obj
+// RUN: not %clang_cc1 -O3 -triple x86_64-unknown-linux-gnu -target-cpu x86-64 %s -o %t -opt-record-file %t.yaml -opt-record-passes "(foo" -emit-obj 2>&1 | FileCheck -check-prefix=CHECK-PATTERN-ERROR %s
+// RUN: %clang_cc1 -O3 -triple x86_64-unknown-linux-gnu -target-cpu x86-64 %s -o %t -opt-record-file %t.yaml -opt-record-format yaml -emit-obj
 // RUN: cat %t.yaml | FileCheck %s
-// RUN: not %clang_cc1 -O3 -triple x86_64-unknown-linux-gnu -target-cpu x86-64 %s -o %t -dwarf-column-info -opt-record-file %t.yaml -opt-record-format "unknown-format" -emit-obj 2>&1 | FileCheck -check-prefix=CHECK-FORMAT-ERROR %s
+// RUN: not %clang_cc1 -O3 -triple x86_64-unknown-linux-gnu -target-cpu x86-64 %s -o %t -opt-record-file %t.yaml -opt-record-format "unknown-format" -emit-obj 2>&1 | FileCheck -check-prefix=CHECK-FORMAT-ERROR %s
 // REQUIRES: x86-registered-target
 
 void bar();
diff --git a/clang/test/CodeGen/stack-clash-protection.c b/clang/test/CodeGen/stack-clash-protection.c
index eb48da8ff9e9a..54699f044ae48 100644
--- a/clang/test/CodeGen/stack-clash-protection.c
+++ b/clang/test/CodeGen/stack-clash-protection.c
@@ -1,6 +1,8 @@
 // Check the correct function attributes are generated
 // RUN: %clang_cc1 -triple x86_64-linux -O0 -S -emit-llvm -o- %s -fstack-clash-protection | FileCheck %s
 // RUN: %clang_cc1 -triple s390x-linux-gnu -O0 -S -emit-llvm -o- %s -fstack-clash-protection | FileCheck %s
+// RUN: %clang_cc1 -triple powerpc64le-linux-gnu -O0 -S -emit-llvm -o- %s -fstack-clash-protection | FileCheck %s
+// RUN: %clang_cc1 -triple powerpc64-linux-gnu -O0 -S -emit-llvm -o- %s -fstack-clash-protection | FileCheck %s
 
 // CHECK: define void @large_stack() #[[A:.*]] {
 void large_stack() {
diff --git a/clang/test/CodeGen/systemz-abi.c b/clang/test/CodeGen/systemz-abi.c
index 35adbbe301c47..9f9cb2275bfa4 100644
--- a/clang/test/CodeGen/systemz-abi.c
+++ b/clang/test/CodeGen/systemz-abi.c
@@ -155,6 +155,17 @@ struct agg_nofloat3 pass_agg_nofloat3(struct agg_nofloat3 arg) { return arg; }
 // CHECK-LABEL: define void @pass_agg_nofloat3(%struct.agg_nofloat3* noalias sret align 4 %{{.*}}, i32 %{{.*}})
 
 
+// Union types likewise are *not* float-like aggregate types
+
+union union_float { float a; };
+union union_float pass_union_float(union union_float arg) { return arg; }
+// CHECK-LABEL: define void @pass_union_float(%union.union_float* noalias sret align 4 %{{.*}}, i32 %{{.*}})
+
+union union_double { double a; };
+union union_double pass_union_double(union union_double arg) { return arg; }
+// CHECK-LABEL: define void @pass_union_double(%union.union_double* noalias sret align 8 %{{.*}}, i64 %{{.*}})
+
+
 // Accessing variable argument lists
 
 int va_int(__builtin_va_list l) { return __builtin_va_arg(l, int); }
diff --git a/clang/test/CodeGen/systemz-abi.cpp b/clang/test/CodeGen/systemz-abi.cpp
index cb381e88dd8f1..a91cb72ae33e2 100644
--- a/clang/test/CodeGen/systemz-abi.cpp
+++ b/clang/test/CodeGen/systemz-abi.cpp
@@ -2,10 +2,58 @@
 // RUN: %clang_cc1 -triple s390x-linux-gnu -emit-llvm -x c++ -o - %s -mfloat-abi soft \
 // RUN:   | FileCheck %s --check-prefix=SOFT-FLOAT
 
+// Verify that class types are also recognized as float-like aggregate types
+
+class agg_float_class { float a; };
+class agg_float_class pass_agg_float_class(class agg_float_class arg) { return arg; }
+// CHECK-LABEL: define void @_Z20pass_agg_float_class15agg_float_class(%class.agg_float_class* noalias sret align 4 %{{.*}}, float %{{.*}})
+// SOFT-FLOAT-LABEL: define void @_Z20pass_agg_float_class15agg_float_class(%class.agg_float_class* noalias sret align 4 %{{.*}}, i32 %{{.*}})
+
+class agg_double_class { double a; };
+class agg_double_class pass_agg_double_class(class agg_double_class arg) { return arg; }
+// CHECK-LABEL: define void @_Z21pass_agg_double_class16agg_double_class(%class.agg_double_class* noalias sret align 8 %{{.*}}, double %{{.*}})
+// SOFT-FLOAT-LABEL: define void @_Z21pass_agg_double_class16agg_double_class(%class.agg_double_class* noalias sret align 8 %{{.*}}, i64 %{{.*}})
+
+
 // For compatibility with GCC, this structure is passed in an FPR in C++,
 // but passed in a GPR in C (checked in systemz-abi.c).
 
 struct agg_float_cpp { float a; int : 0; };
 struct agg_float_cpp pass_agg_float_cpp(struct agg_float_cpp arg) { return arg; }
 // CHECK-LABEL: define void @_Z18pass_agg_float_cpp13agg_float_cpp(%struct.agg_float_cpp* noalias sret align 4 %{{.*}}, float %{{.*}})
-// SOFT-FLOAT:  define void @_Z18pass_agg_float_cpp13agg_float_cpp(%struct.agg_float_cpp* noalias sret align 4 %{{.*}}, i32 %{{.*}})
+// SOFT-FLOAT-LABEL:  define void @_Z18pass_agg_float_cpp13agg_float_cpp(%struct.agg_float_cpp* noalias sret align 4 %{{.*}}, i32 %{{.*}})
+
+
+// A field member of empty class type in C++ makes the record nonhomogeneous,
+// unless it is marked as [[no_unique_address]].  This does not apply to arrays.
+struct empty { };
+struct agg_nofloat_empty { float a; empty dummy; };
+struct agg_nofloat_empty pass_agg_nofloat_empty(struct agg_nofloat_empty arg) { return arg; }
+// CHECK-LABEL: define void @_Z22pass_agg_nofloat_empty17agg_nofloat_empty(%struct.agg_nofloat_empty* noalias sret align 4 %{{.*}}, i64 %{{.*}})
+// SOFT-FLOAT-LABEL:  define void @_Z22pass_agg_nofloat_empty17agg_nofloat_empty(%struct.agg_nofloat_empty* noalias sret align 4 %{{.*}}, i64 %{{.*}})
+struct agg_float_empty { float a; [[no_unique_address]] empty dummy; };
+struct agg_float_empty pass_agg_float_empty(struct agg_float_empty arg) { return arg; }
+// CHECK-LABEL: define void @_Z20pass_agg_float_empty15agg_float_empty(%struct.agg_float_empty* noalias sret align 4 %{{.*}}, float %{{.*}})
+// SOFT-FLOAT-LABEL:  define void @_Z20pass_agg_float_empty15agg_float_empty(%struct.agg_float_empty* noalias sret align 4 %{{.*}}, i32 %{{.*}})
+struct agg_nofloat_emptyarray { float a; [[no_unique_address]] empty dummy[3]; };
+struct agg_nofloat_emptyarray pass_agg_nofloat_emptyarray(struct agg_nofloat_emptyarray arg) { return arg; }
+// CHECK-LABEL: define void @_Z27pass_agg_nofloat_emptyarray22agg_nofloat_emptyarray(%struct.agg_nofloat_emptyarray* noalias sret align 4 %{{.*}}, i64 %{{.*}})
+// SOFT-FLOAT-LABEL:  define void @_Z27pass_agg_nofloat_emptyarray22agg_nofloat_emptyarray(%struct.agg_nofloat_emptyarray* noalias sret align 4 %{{.*}}, i64 %{{.*}})
+
+// And likewise for members of base classes.
+struct noemptybase { empty dummy; };
+struct agg_nofloat_emptybase : noemptybase { float a; };
+struct agg_nofloat_emptybase pass_agg_nofloat_emptybase(struct agg_nofloat_emptybase arg) { return arg; }
+// CHECK-LABEL: define void @_Z26pass_agg_nofloat_emptybase21agg_nofloat_emptybase(%struct.agg_nofloat_emptybase* noalias sret align 4 %{{.*}}, i64 %{{.*}})
+// SOFT-FLOAT-LABEL:  define void @_Z26pass_agg_nofloat_emptybase21agg_nofloat_emptybase(%struct.agg_nofloat_emptybase* noalias sret align 4 %{{.*}}, i64 %{{.*}})
+struct emptybase { [[no_unique_address]] empty dummy; };
+struct agg_float_emptybase : emptybase { float a; };
+struct agg_float_emptybase pass_agg_float_emptybase(struct agg_float_emptybase arg) { return arg; }
+// CHECK-LABEL: define void @_Z24pass_agg_float_emptybase19agg_float_emptybase(%struct.agg_float_emptybase* noalias sret align 4 %{{.*}}, float %{{.*}})
+// SOFT-FLOAT-LABEL:  define void @_Z24pass_agg_float_emptybase19agg_float_emptybase(%struct.agg_float_emptybase* noalias sret align 4 %{{.*}}, i32 %{{.*}})
+struct noemptybasearray { [[no_unique_address]] empty dummy[3]; };
+struct agg_nofloat_emptybasearray : noemptybasearray { float a; };
+struct agg_nofloat_emptybasearray pass_agg_nofloat_emptybasearray(struct agg_nofloat_emptybasearray arg) { return arg; }
+// CHECK-LABEL: define void @_Z31pass_agg_nofloat_emptybasearray26agg_nofloat_emptybasearray(%struct.agg_nofloat_emptybasearray* noalias sret align 4 %{{.*}}, i64 %{{.*}})
+// SOFT-FLOAT-LABEL:  define void @_Z31pass_agg_nofloat_emptybasearray26agg_nofloat_emptybasearray(%struct.agg_nofloat_emptybasearray* noalias sret align 4 %{{.*}}, i64 %{{.*}})
+
diff --git a/clang/test/CodeGen/target-avx-abi-diag.c b/clang/test/CodeGen/target-avx-abi-diag.c
new file mode 100644
index 0000000000000..f3d4462a552d2
--- /dev/null
+++ b/clang/test/CodeGen/target-avx-abi-diag.c
@@ -0,0 +1,51 @@
+// RUN: %clang_cc1 %s -triple=x86_64-linux-gnu -verify=no256,no512 -o - -S
+// RUN: %clang_cc1 %s -triple=x86_64-linux-gnu -target-feature +avx -verify=no512 -o - -S
+// RUN: %clang_cc1 %s -triple=x86_64-linux-gnu -target-feature +avx512f -verify=both -o - -S
+// REQUIRES: x86-registered-target
+
+// both-no-diagnostics
+
+typedef short avx512fType __attribute__((vector_size(64)));
+typedef short avx256Type __attribute__((vector_size(32)));
+
+__attribute__((target("avx"))) void takesAvx256(avx256Type t);
+__attribute__((target("avx512f"))) void takesAvx512(avx512fType t);
+void takesAvx256_no_target(avx256Type t);
+void takesAvx512_no_target(avx512fType t);
+
+void variadic(int i, ...);
+__attribute__((target("avx512f"))) void variadic_err(int i, ...);
+
+// If neither side has an attribute, warn.
+void call_warn(void) {
+  avx256Type t1;
+  takesAvx256_no_target(t1); // no256-warning {{AVX vector argument of type 'avx256Type' (vector of 16 'short' values) without 'avx' enabled changes the ABI}}
+
+  avx512fType t2;
+  takesAvx512_no_target(t2); // no512-warning {{AVX vector argument of type 'avx512fType' (vector of 32 'short' values) without 'avx512f' enabled changes the ABI}}
+
+  variadic(1, t1); // no256-warning {{AVX vector argument of type 'avx256Type' (vector of 16 'short' values) without 'avx' enabled changes the ABI}}
+  variadic(3, t2); // no512-warning {{AVX vector argument of type 'avx512fType' (vector of 32 'short' values) without 'avx512f' enabled changes the ABI}}
+}
+
+// If only 1 side has an attribute, error.
+void call_errors(void) {
+  avx256Type t1;
+  takesAvx256(t1); // no256-error {{AVX vector argument of type 'avx256Type' (vector of 16 'short' values) without 'avx' enabled changes the ABI}}
+  avx512fType t2;
+  takesAvx512(t2); // no512-error {{AVX vector argument of type 'avx512fType' (vector of 32 'short' values) without 'avx512f' enabled changes the ABI}}
+
+  variadic_err(1, t1); // no256-error {{AVX vector argument of type 'avx256Type' (vector of 16 'short' values) without 'avx' enabled changes the ABI}}
+  variadic_err(3, t2); // no512-error {{AVX vector argument of type 'avx512fType' (vector of 32 'short' values) without 'avx512f' enabled changes the ABI}}
+}
+
+// These two don't diagnose anything, since these are valid calls.
+__attribute__((target("avx"))) void call_avx256_ok(void) {
+  avx256Type t;
+  takesAvx256(t);
+}
+
+__attribute__((target("avx512f"))) void call_avx512_ok(void) {
+  avx512fType t;
+  takesAvx512(t);
+}
diff --git a/clang/test/CodeGen/target-builtin-error-3.c b/clang/test/CodeGen/target-builtin-error-3.c
index 5beb474befe07..3de76e253d9b1 100644
--- a/clang/test/CodeGen/target-builtin-error-3.c
+++ b/clang/test/CodeGen/target-builtin-error-3.c
@@ -18,11 +18,12 @@ static inline half8 __attribute__((__overloadable__)) convert_half( float8 a ) {
   return __extension__ ({ __m256 __a = (a); (__m128i)__builtin_ia32_vcvtps2ph256((__v8sf)__a, (0x00)); }); // expected-error {{'__builtin_ia32_vcvtps2ph256' needs target feature f16c}}
 }
 static inline half16 __attribute__((__overloadable__)) convert_half( float16 a ) {
-  half16 r; 
-  r.lo = convert_half( a.lo); 
+  half16 r;
+  r.lo = convert_half(a.lo);
   return r;
 }
 void avx_test( uint16_t *destData, float16 argbF)
 {
-   ((half16U*)destData)[0] = convert_half(argbF);
+  // expected-warning@+1{{AVX vector argument of type 'float16' (vector of 16 'float' values) without 'avx512f' enabled changes the ABI}}
+  ((half16U *)destData)[0] = convert_half(argbF);
 }
diff --git a/clang/test/CodeGen/target-builtin-noerror.c b/clang/test/CodeGen/target-builtin-noerror.c
index b2d18fa0b2c00..339e5b15c88d7 100644
--- a/clang/test/CodeGen/target-builtin-noerror.c
+++ b/clang/test/CodeGen/target-builtin-noerror.c
@@ -6,15 +6,15 @@
 
 // No warnings.
 extern __m256i a;
-int __attribute__((target("avx"))) bar(__m256i a) {
+int __attribute__((target("avx"))) bar() {
   return _mm256_extract_epi32(a, 3);
 }
 
 int baz() {
-  return bar(a);
+  return bar();
 }
 
-int __attribute__((target("avx"))) qq_avx(__m256i a) {
+int __attribute__((target("avx"))) qq_avx() {
   return _mm256_extract_epi32(a, 3);
 }
 
@@ -25,7 +25,7 @@ int qq_noavx() {
 extern __m256i a;
 int qq() {
   if (__builtin_cpu_supports("avx"))
-    return qq_avx(a);
+    return qq_avx();
   else
     return qq_noavx();
 }
diff --git a/clang/test/CodeGen/target-data.c b/clang/test/CodeGen/target-data.c
index e619843f4bdbd..8c740119cd1ba 100644
--- a/clang/test/CodeGen/target-data.c
+++ b/clang/test/CodeGen/target-data.c
@@ -250,3 +250,7 @@
 // RUN: %clang_cc1 -triple bpfeb -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=BPFEB
 // BPFEB: target datalayout = "E-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
+
+// RUN: %clang_cc1 -triple ve -o - -emit-llvm %s | \
+// RUN: FileCheck %s -check-prefix=VE
+// VE: target datalayout = "e-m:e-i64:64-n32:64-S128"
diff --git a/clang/test/CodeGen/thinlto-distributed-cfi-devirt.ll b/clang/test/CodeGen/thinlto-distributed-cfi-devirt.ll
index b46845afba26b..5c753ba6f93c3 100644
--- a/clang/test/CodeGen/thinlto-distributed-cfi-devirt.ll
+++ b/clang/test/CodeGen/thinlto-distributed-cfi-devirt.ll
@@ -93,7 +93,7 @@ cont2:
   ; CHECK-IR: br i1 {{.*}}, label %trap
 
   ; We still have to call it as virtual.
-  ; CHECK-IR: %call3 = tail call i32 %8
+  ; CHECK-IR: %call3 = tail call i32 %7
   %call3 = tail call i32 %8(%struct.A* nonnull %obj, i32 %call)
   ret i32 %call3
 }
diff --git a/clang/test/CodeGen/thinlto-inline-asm.c b/clang/test/CodeGen/thinlto-inline-asm.c
new file mode 100644
index 0000000000000..a41cba6fadc1c
--- /dev/null
+++ b/clang/test/CodeGen/thinlto-inline-asm.c
@@ -0,0 +1,21 @@
+// REQUIRES: x86-registered-target
+
+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc19.11.0 -emit-llvm-bc \
+// RUN:   -flto=thin -mllvm -x86-asm-syntax=intel -v \
+// RUN:   -o %t.obj %s 2>&1 | FileCheck --check-prefix=CLANG %s
+//
+// RUN: llvm-lto2 dump-symtab %t.obj | FileCheck --check-prefix=SYMTAB %s
+
+// Module-level inline asm is parsed with At&t syntax. Test that the
+// -x86-asm-syntax flag does not affect this.
+
+// CLANG-NOT: unknown token in expression
+// SYMTAB: D------X foo
+// SYMTAB: D------X bar
+
+void foo() {}
+
+asm(".globl bar      \n"
+    "bar:            \n"
+    "  xor %eax, %eax\n"
+    "  ret           \n");
diff --git a/clang/test/CodeGen/ve-abi.c b/clang/test/CodeGen/ve-abi.c
new file mode 100644
index 0000000000000..aa35095d5dea0
--- /dev/null
+++ b/clang/test/CodeGen/ve-abi.c
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 -triple ve-linux-gnu -emit-llvm %s -o - | FileCheck %s
+
+// CHECK-LABEL: define { float, float } @p(float %a.coerce0, float %a.coerce1, float %b.coerce0, float %b.coerce1) #0 {
+float __complex__ p(float __complex__ a, float __complex__ b) {
+}
+
+// CHECK-LABEL: define { double, double } @q(double %a.coerce0, double %a.coerce1, double %b.coerce0, double %b.coerce1) #0 {
+double __complex__ q(double __complex__ a, double __complex__ b) {
+}
+
+void func() {
+  // CHECK-LABEL: %call = call i32 (i32, i32, i32, i32, i32, i32, i32, ...) bitcast (i32 (...)* @hoge to i32 (i32, i32, i32, i32, i32, i32, i32, ...)*)(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7)
+  hoge(1, 2, 3, 4, 5, 6, 7);
+}
diff --git a/clang/test/CodeGen/vla.c b/clang/test/CodeGen/vla.c
index 37243cd172905..16b82f4acc7d3 100644
--- a/clang/test/CodeGen/vla.c
+++ b/clang/test/CodeGen/vla.c
@@ -206,3 +206,7 @@ void test9(int n, int a[static n]) { }
 // NULL-INVALID: define void @test9(i32 %n, i32* nonnull %a)
 // NULL-VALID: define void @test9(i32 %n, i32* %a)
 
+// Make sure a zero-sized static array extent is still required to be nonnull.
+void test10(int a[static 0]) {}
+// NULL-INVALID: define void @test10(i32* nonnull %a)
+// NULL-VALID: define void @test10(i32* %a)
diff --git a/clang/test/CodeGen/windows-seh-filter-inFinally.c b/clang/test/CodeGen/windows-seh-filter-inFinally.c
new file mode 100644
index 0000000000000..f9dfca14f0209
--- /dev/null
+++ b/clang/test/CodeGen/windows-seh-filter-inFinally.c
@@ -0,0 +1,36 @@
+// RUN: %clang_cc1 -triple x86_64-windows -fms-extensions -Wno-implicit-function-declaration -S -emit-llvm %s -o - | FileCheck %s
+
+// CHECK: %[[dst:[0-9-]+]] = call i8* @llvm.eh.recoverfp(i8* bitcast (void (i8, i8*)* @"?fin$0@0@main@@" to i8*), i8* %frame_pointer)
+// CHECK-NEXT: %[[dst1:[0-9-]+]] = call i8* @llvm.localrecover(i8* bitcast (void (i8, i8*)* @"?fin$0@0@main@@" to i8*), i8* %[[dst]], i32 0)
+// CHECK-NEXT: %[[dst2:[0-9-]+]] = bitcast i8* %[[dst1]] to i8**
+// CHECK-NEXT: = load i8*, i8** %[[dst2]], align 8
+
+int
+main(int argc, char *argv[])
+{
+    int Counter = 0;
+    //
+    // Try/except within the finally clause of a try/finally.
+    //
+    __try {
+      Counter -= 1;
+    }
+    __finally {
+      __try {
+        Counter += 2;
+        // RtlRaiseStatus(STATUS_INTEGER_OVERFLOW);
+      } __except(Counter) {
+        __try {
+          Counter += 3;
+        }
+        __finally {
+          if (abnormal_termination() == 1) {
+            Counter += 5;
+          }
+        }
+      }
+    }
+    // expect Counter == 9
+    return 1;
+}
+
diff --git a/clang/test/CodeGen/xcore-unused-inline.c b/clang/test/CodeGen/xcore-unused-inline.c
new file mode 100644
index 0000000000000..8b5bdbf2ed6de
--- /dev/null
+++ b/clang/test/CodeGen/xcore-unused-inline.c
@@ -0,0 +1,9 @@
+// REQUIRES: xcore-registered-target
+// RUN: %clang_cc1 -triple xcore-unknown-unknown -emit-llvm -o - %s
+
+// D77068 fixes a segmentation fault and assertion failure "Unexpected null
+// Value" in the case of an unused inline function, when targeting xcore. This
+// test verifies that clang does not crash and does not produce code for such a
+// function.
+
+inline void dead_function(void) {}
diff --git a/clang/test/CodeGenCUDA/amdgpu-kernel-arg-pointer-type.cu b/clang/test/CodeGenCUDA/amdgpu-kernel-arg-pointer-type.cu
index 73ab9edf318e5..2660a5f14f90d 100644
--- a/clang/test/CodeGenCUDA/amdgpu-kernel-arg-pointer-type.cu
+++ b/clang/test/CodeGenCUDA/amdgpu-kernel-arg-pointer-type.cu
@@ -1,37 +1,55 @@
-// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -fcuda-is-device -emit-llvm -x hip %s -o - | FileCheck %s
+// REQUIRES: x86-registered-target
+// REQUIRES: amdgpu-registered-target
+
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -fcuda-is-device -emit-llvm -x hip %s -o - | FileCheck --check-prefixes=COMMON,CHECK %s
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -fcuda-is-device -emit-llvm -x hip %s -disable-O0-optnone -o - | opt -S -O2 | FileCheck %s --check-prefixes=COMMON,OPT
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -x hip %s -o - | FileCheck -check-prefix=HOST %s
 
 #include "Inputs/cuda.h"
 
 // Coerced struct from `struct S` without all generic pointers lowered into
 // global ones.
-// CHECK: %struct.S.coerce = type { i32 addrspace(1)*, float addrspace(1)* }
-// CHECK: %struct.T.coerce = type { [2 x float addrspace(1)*] }
+// COMMON: %struct.S.coerce = type { i32 addrspace(1)*, float addrspace(1)* }
+// COMMON: %struct.T.coerce = type { [2 x float addrspace(1)*] }
 
 // On the host-side compilation, generic pointer won't be coerced.
 // HOST-NOT: %struct.S.coerce
 // HOST-NOT: %struct.T.coerce
 
-// CHECK: define amdgpu_kernel void  @_Z7kernel1Pi(i32 addrspace(1)* %x.coerce)
 // HOST: define void @_Z22__device_stub__kernel1Pi(i32* %x)
+// COMMON-LABEL: define amdgpu_kernel void @_Z7kernel1Pi(i32 addrspace(1)*{{.*}} %x.coerce)
+// CHECK:     = addrspacecast [[TYPE:.*]] addrspace(1)* %{{.*}} to [[TYPE]]*
+// CHECK-NOT: = addrspacecast [[TYPE:.*]] addrspace(1)* %{{.*}} to [[TYPE]]*
+// OPT: [[VAL:%.*]] = load i32, i32 addrspace(1)* %x.coerce, align 4
+// OPT: [[INC:%.*]] = add nsw i32 [[VAL]], 1
+// OPT: store i32 [[INC]], i32 addrspace(1)* %x.coerce, align 4
+// OPT: ret void
 __global__ void kernel1(int *x) {
   x[0]++;
 }
 
-// CHECK: define amdgpu_kernel void  @_Z7kernel2Ri(i32 addrspace(1)* nonnull align 4 dereferenceable(4) %x.coerce)
 // HOST: define void @_Z22__device_stub__kernel2Ri(i32* nonnull align 4 dereferenceable(4) %x)
+// COMMON-LABEL: define amdgpu_kernel void @_Z7kernel2Ri(i32 addrspace(1)*{{.*}} nonnull align 4 dereferenceable(4) %x.coerce)
+// CHECK:     = addrspacecast [[TYPE:.*]] addrspace(1)* %{{.*}} to [[TYPE]]*
+// CHECK-NOT: = addrspacecast [[TYPE:.*]] addrspace(1)* %{{.*}} to [[TYPE]]*
+// OPT: [[VAL:%.*]] = load i32, i32 addrspace(1)* %x.coerce, align 4
+// OPT: [[INC:%.*]] = add nsw i32 [[VAL]], 1
+// OPT: store i32 [[INC]], i32 addrspace(1)* %x.coerce, align 4
+// OPT: ret void
 __global__ void kernel2(int &x) {
   x++;
 }
 
-// CHECK: define amdgpu_kernel void  @_Z7kernel3PU3AS2iPU3AS1i(i32 addrspace(2)* %x, i32 addrspace(1)* %y)
 // HOST: define void @_Z22__device_stub__kernel3PU3AS2iPU3AS1i(i32 addrspace(2)* %x, i32 addrspace(1)* %y)
+// CHECK-LABEL: define amdgpu_kernel void  @_Z7kernel3PU3AS2iPU3AS1i(i32 addrspace(2)*{{.*}} %x, i32 addrspace(1)*{{.*}} %y)
+// CHECK-NOT: = addrspacecast [[TYPE:.*]] addrspace(1)* %{{.*}} to [[TYPE]]*
 __global__ void kernel3(__attribute__((address_space(2))) int *x,
                         __attribute__((address_space(1))) int *y) {
   y[0] = x[0];
 }
 
-// CHECK: define void @_Z4funcPi(i32* %x)
+// COMMON-LABEL: define void @_Z4funcPi(i32*{{.*}} %x)
+// CHECK-NOT: = addrspacecast [[TYPE:.*]] addrspace(1)* %{{.*}} to [[TYPE]]*
 __device__ void func(int *x) {
   x[0]++;
 }
@@ -42,16 +60,25 @@ struct S {
 };
 // `by-val` struct will be coerced into a similar struct with all generic
 // pointers lowerd into global ones.
-// CHECK: define amdgpu_kernel void @_Z7kernel41S(%struct.S.coerce %s.coerce)
 // HOST: define void @_Z22__device_stub__kernel41S(i32* %s.coerce0, float* %s.coerce1)
+// COMMON-LABEL: define amdgpu_kernel void @_Z7kernel41S(%struct.S.coerce %s.coerce)
+// OPT: [[P0:%.*]] = extractvalue %struct.S.coerce %s.coerce, 0
+// OPT: [[P1:%.*]] = extractvalue %struct.S.coerce %s.coerce, 1
+// OPT: [[V0:%.*]] = load i32, i32 addrspace(1)* [[P0]], align 4
+// OPT: [[INC:%.*]] = add nsw i32 [[V0]], 1
+// OPT: store i32 [[INC]], i32 addrspace(1)* [[P0]], align 4
+// OPT: [[V1:%.*]] = load float, float addrspace(1)* [[P1]], align 4
+// OPT: [[ADD:%.*]] = fadd contract float [[V1]], 1.000000e+00
+// OPT: store float [[ADD]], float addrspace(1)* [[P1]], align 4
+// OPT: ret void
 __global__ void kernel4(struct S s) {
   s.x[0]++;
   s.y[0] += 1.f;
 }
 
 // If a pointer to struct is passed, only the pointer itself is coerced into the global one.
-// CHECK: define amdgpu_kernel void @_Z7kernel5P1S(%struct.S addrspace(1)* %s.coerce)
 // HOST: define void @_Z22__device_stub__kernel5P1S(%struct.S* %s)
+// COMMON-LABEL: define amdgpu_kernel void @_Z7kernel5P1S(%struct.S addrspace(1)*{{.*}} %s.coerce)
 __global__ void kernel5(struct S *s) {
   s->x[0]++;
   s->y[0] += 1.f;
@@ -61,16 +88,26 @@ struct T {
   float *x[2];
 };
 // `by-val` array is also coerced.
-// CHECK: define amdgpu_kernel void @_Z7kernel61T(%struct.T.coerce %t.coerce)
 // HOST: define void @_Z22__device_stub__kernel61T(float* %t.coerce0, float* %t.coerce1)
+// COMMON-LABEL: define amdgpu_kernel void @_Z7kernel61T(%struct.T.coerce %t.coerce)
+// OPT: [[ARR:%.*]] = extractvalue %struct.T.coerce %t.coerce, 0
+// OPT: [[P0:%.*]] = extractvalue [2 x float addrspace(1)*] [[ARR]], 0
+// OPT: [[P1:%.*]] = extractvalue [2 x float addrspace(1)*] [[ARR]], 1
+// OPT: [[V0:%.*]] = load float, float addrspace(1)* [[P0]], align 4
+// OPT: [[ADD0:%.*]] = fadd contract float [[V0]], 1.000000e+00
+// OPT: store float [[ADD0]], float addrspace(1)* [[P0]], align 4
+// OPT: [[V1:%.*]] = load float, float addrspace(1)* [[P1]], align 4
+// OPT: [[ADD1:%.*]] = fadd contract float [[V1]], 2.000000e+00
+// OPT: store float [[ADD1]], float addrspace(1)* [[P1]], align 4
+// OPT: ret void
 __global__ void kernel6(struct T t) {
   t.x[0][0] += 1.f;
   t.x[1][0] += 2.f;
 }
 
 // Check that coerced pointers retain the noalias attribute when qualified with __restrict.
-// CHECK: define amdgpu_kernel void @_Z7kernel7Pi(i32 addrspace(1)* noalias %x.coerce)
 // HOST: define void @_Z22__device_stub__kernel7Pi(i32* noalias %x)
+// COMMON-LABEL: define amdgpu_kernel void @_Z7kernel7Pi(i32 addrspace(1)* noalias{{.*}} %x.coerce)
 __global__ void kernel7(int *__restrict x) {
   x[0]++;
 }
diff --git a/clang/test/CodeGenCUDA/lambda.cu b/clang/test/CodeGenCUDA/lambda.cu
new file mode 100644
index 0000000000000..8a639c90f2668
--- /dev/null
+++ b/clang/test/CodeGenCUDA/lambda.cu
@@ -0,0 +1,85 @@
+// RUN: %clang_cc1 -x hip -emit-llvm -std=c++11 %s -o - \
+// RUN:   -triple x86_64-linux-gnu \
+// RUN:   | FileCheck -check-prefix=HOST %s
+// RUN: %clang_cc1 -x hip -emit-llvm -std=c++11 %s -o - \
+// RUN:   -triple amdgcn-amd-amdhsa -fcuda-is-device \
+// RUN:   | FileCheck -check-prefix=DEV %s
+
+#include "Inputs/cuda.h"
+
+// Device side kernel name.
+// HOST: @[[KERN_CAPTURE:[0-9]+]] = {{.*}} c"_Z1gIZ12test_capturevEUlvE_EvT_\00"
+// HOST: @[[KERN_RESOLVE:[0-9]+]] = {{.*}} c"_Z1gIZ12test_resolvevEUlvE_EvT_\00"
+
+// Check functions emitted for test_capture in host compilation.
+// Check lambda is not emitted in host compilation.
+// HOST-LABEL: define void @_Z12test_capturev
+// HOST:  call void @_Z19test_capture_helperIZ12test_capturevEUlvE_EvT_
+// HOST-LABEL: define internal void @_Z19test_capture_helperIZ12test_capturevEUlvE_EvT_
+// HOST:  call void @_Z16__device_stub__gIZ12test_capturevEUlvE_EvT_
+// HOST-NOT: define{{.*}}@_ZZ4mainENKUlvE_clEv
+
+// Check functions emitted for test_resolve in host compilation.
+// Check host version of template function 'overloaded' is emitted and called
+// by the lambda function.
+// HOST-LABEL: define void @_Z12test_resolvev
+// HOST:  call void @_Z19test_resolve_helperIZ12test_resolvevEUlvE_EvT_()
+// HOST-LABEL: define internal void @_Z19test_resolve_helperIZ12test_resolvevEUlvE_EvT_
+// HOST:  call void @_Z16__device_stub__gIZ12test_resolvevEUlvE_EvT_
+// HOST:  call void @_ZZ12test_resolvevENKUlvE_clEv
+// HOST-LABEL: define internal void @_ZZ12test_resolvevENKUlvE_clEv
+// HOST:  call i32 @_Z10overloadedIiET_v
+// HOST-LABEL: define linkonce_odr i32 @_Z10overloadedIiET_v
+// HOST:  ret i32 2
+
+// Check kernel is registered with correct device side kernel name.
+// HOST: @__hipRegisterFunction({{.*}}@[[KERN_CAPTURE]]
+// HOST: @__hipRegisterFunction({{.*}}@[[KERN_RESOLVE]]
+
+// DEV: @a = addrspace(1) externally_initialized global i32 0
+
+// Check functions emitted for test_capture in device compilation.
+// Check lambda is emitted in device compilation and accessing device variable.
+// DEV-LABEL: define amdgpu_kernel void @_Z1gIZ12test_capturevEUlvE_EvT_
+// DEV:  call void @_ZZ12test_capturevENKUlvE_clEv
+// DEV-LABEL: define internal void @_ZZ12test_capturevENKUlvE_clEv
+// DEV:  store i32 1, i32* addrspacecast (i32 addrspace(1)* @a to i32*)
+
+// Check functions emitted for test_resolve in device compilation.
+// Check device version of template function 'overloaded' is emitted and called
+// by the lambda function.
+// DEV-LABEL: define amdgpu_kernel void @_Z1gIZ12test_resolvevEUlvE_EvT_
+// DEV:  call void @_ZZ12test_resolvevENKUlvE_clEv
+// DEV-LABEL: define internal void @_ZZ12test_resolvevENKUlvE_clEv
+// DEV:  call i32 @_Z10overloadedIiET_v
+// DEV-LABEL: define linkonce_odr i32 @_Z10overloadedIiET_v
+// DEV:  ret i32 1
+
+__device__ int a;
+
+template<class T>
+__device__ T overloaded() { return 1; }
+
+template<class T>
+__host__ T overloaded() { return 2; }
+
+template<class F>
+__global__ void g(F f) { f(); }
+
+template<class F>
+void test_capture_helper(F f) { g<<<1,1>>>(f); }
+
+template<class F>
+void test_resolve_helper(F f) { g<<<1,1>>>(f); f(); }
+
+// Test capture of device variable in lambda function.
+void test_capture(void) {
+  test_capture_helper([](){ a = 1;});
+}
+
+// Test resolving host/device function in lambda function.
+// Callee should resolve to correct host/device function based on where
+// the lambda function is called, not where it is defined.
+void test_resolve(void) {
+  test_resolve_helper([](){ overloaded<int>();});
+}
diff --git a/clang/test/CodeGenCXX/PR20038.cpp b/clang/test/CodeGenCXX/PR20038.cpp
index 195e4e521240b..b6d12f6603127 100644
--- a/clang/test/CodeGenCXX/PR20038.cpp
+++ b/clang/test/CodeGenCXX/PR20038.cpp
@@ -7,8 +7,8 @@ extern bool b;
 // CHECK: call {{.*}}, !dbg [[DTOR_CALL1_LOC:![0-9]*]]
 // CHECK: call {{.*}}, !dbg [[DTOR_CALL2_LOC:![0-9]*]]
 // CHECK: [[FUN1:.*]] = distinct !DISubprogram(name: "fun1",{{.*}} DISPFlagDefinition
-// CHECK: [[DTOR_CALL1_LOC]] = !DILocation(line: [[@LINE+1]], scope: [[FUN1]])
+// CHECK: [[DTOR_CALL1_LOC]] = !DILocation(line: [[@LINE+1]], column: 15, scope: [[FUN1]])
 void fun1() { b && (C(), 1); }
 // CHECK: [[FUN2:.*]] = distinct !DISubprogram(name: "fun2",{{.*}} DISPFlagDefinition
-// CHECK: [[DTOR_CALL2_LOC]] = !DILocation(line: [[@LINE+1]], scope: [[FUN2]])
+// CHECK: [[DTOR_CALL2_LOC]] = !DILocation(line: [[@LINE+1]], column: 15, scope: [[FUN2]])
 bool fun2() { return (C(), b) && 0; }
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/dynamic-cast.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/dynamic-cast.cpp
index 56b56a1b93984..62d674669661d 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/dynamic-cast.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/dynamic-cast.cpp
@@ -35,13 +35,13 @@
 // CHECK-NEXT:   [[isnull:%[0-9]+]] = icmp eq %class.B* %b, null
 // CHECK-NEXT:   br i1 [[isnull]], label %[[dynamic_cast_end:[a-z0-9._]+]], label %[[dynamic_cast_notnull:[a-z0-9._]+]]
 // CHECK:      [[dynamic_cast_notnull]]:
-// CHECK-NEXT:   [[b2:%[0-9]+]] = bitcast %class.B* %b to i32**
-// CHECK-NEXT:   [[vtable:%[a-z0-9]+]] = load i32*, i32** [[b2]], align 8
-// CHECK-NEXT:   [[offset_ptr:%.+]] = getelementptr inbounds i32, i32* [[vtable]], i64 -2
-// CHECK-NEXT:   [[offset_to_top:%.+]] = load i32, i32* [[offset_ptr]], align 4
-// CHECK-NEXT:   [[b:%[0-9]+]] = bitcast %class.B* %b to i8*
-// CHECK-NEXT:   [[offset_to_top2:%.+]] = sext i32 [[offset_to_top]] to i64
-// CHECK-NEXT:   [[casted:%.+]] = getelementptr inbounds i8, i8* [[b]], i64 [[offset_to_top2]]
+// CHECK-DAG:    [[b2:%[0-9]+]] = bitcast %class.B* %b to i32**
+// CHECK-DAG:    [[vtable:%[a-z0-9]+]] = load i32*, i32** [[b2]], align 8
+// CHECK-DAG:    [[offset_ptr:%.+]] = getelementptr inbounds i32, i32* [[vtable]], i64 -2
+// CHECK-DAG:    [[offset_to_top:%.+]] = load i32, i32* [[offset_ptr]], align 4
+// CHECK-DAG:    [[b:%[0-9]+]] = bitcast %class.B* %b to i8*
+// CHECK-DAG:    [[offset_to_top2:%.+]] = sext i32 [[offset_to_top]] to i64
+// CHECK-DAG:    [[casted:%.+]] = getelementptr inbounds i8, i8* [[b]], i64 [[offset_to_top2]]
 // CHECK-NEXT:   br label %[[dynamic_cast_end]]
 // CHECK:      [[dynamic_cast_end]]:
 // CHECK-NEXT:   [[res:%[0-9]+]] = phi i8* [ [[casted]], %[[dynamic_cast_notnull]] ], [ null, %entry ]
diff --git a/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp b/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp
index 66e3c1306dd25..d75efbd25c484 100644
--- a/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp
+++ b/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu %s -emit-llvm -o - \
 // RUN:   | FileCheck %s
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu %s -emit-llvm -o - \
-// RUN:   -target-feature +sve | FileCheck %s
+// RUN:   -target-feature +sve,+bf16 | FileCheck %s
 
 template<typename T> struct S {};
 
@@ -27,8 +27,10 @@ void f9(S<__SVFloat16_t>) {}
 void f10(S<__SVFloat32_t>) {}
 // CHECK: _Z3f111SIu13__SVFloat64_tE
 void f11(S<__SVFloat64_t>) {}
-// CHECK: _Z3f121SIu10__SVBool_tE
-void f12(S<__SVBool_t>) {}
+// CHECK: _Z3f121SIu14__SVBFloat16_tE
+void f12(S<__SVBFloat16_t>) {}
+// CHECK: _Z3f131SIu10__SVBool_tE
+void f13(S<__SVBool_t>) {}
 
 // The tuple types don't use the internal name for mangling.
 
@@ -98,3 +100,9 @@ void f43(S<__clang_svfloat64x2_t>) {}
 void f44(S<__clang_svfloat64x3_t>) {}
 // CHECK: _Z3f451SI13svfloat64x4_tE
 void f45(S<__clang_svfloat64x4_t>) {}
+// CHECK: _Z3f461SI14svbfloat16x2_tE
+void f46(S<__clang_svbfloat16x2_t>) {}
+// CHECK: _Z3f471SI14svbfloat16x3_tE
+void f47(S<__clang_svbfloat16x3_t>) {}
+// CHECK: _Z3f481SI14svbfloat16x4_tE
+void f48(S<__clang_svbfloat16x4_t>) {}
diff --git a/clang/test/CodeGenCXX/aarch64-sve-typeinfo.cpp b/clang/test/CodeGenCXX/aarch64-sve-typeinfo.cpp
index 5627efaf6a6a4..6081ab69a2344 100644
--- a/clang/test/CodeGenCXX/aarch64-sve-typeinfo.cpp
+++ b/clang/test/CodeGenCXX/aarch64-sve-typeinfo.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu %s -emit-llvm -o - \
 // RUN:   | FileCheck %s
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu %s -emit-llvm -o - \
-// RUN:   -target-feature +sve | FileCheck %s
+// RUN:   -target-feature +sve,+bf16 | FileCheck %s
 
 namespace std { class type_info; };
 
@@ -19,6 +19,8 @@ auto &f16 = typeid(__SVFloat16_t);
 auto &f32 = typeid(__SVFloat32_t);
 auto &f64 = typeid(__SVFloat64_t);
 
+auto &bf16 = typeid(__SVBFloat16_t);
+
 auto &b8 = typeid(__SVBool_t);
 
 // CHECK-DAG: @_ZTSu10__SVInt8_t = {{.*}} c"u10__SVInt8_t\00"
@@ -53,6 +55,9 @@ auto &b8 = typeid(__SVBool_t);
 
 // CHECK-DAG: @_ZTSu13__SVFloat64_t = {{.*}} c"u13__SVFloat64_t\00"
 // CHECK-DAG: @_ZTIu13__SVFloat64_t = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTSu13__SVFloat64_t
+//
+// CHECK-DAG: @_ZTSu14__SVBFloat16_t = {{.*}} c"u14__SVBFloat16_t\00"
+// CHECK-DAG: @_ZTIu14__SVBFloat16_t = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTSu14__SVBFloat16_t
 
 // CHECK-DAG: @_ZTSu10__SVBool_t = {{.*}} c"u10__SVBool_t\00"
 // CHECK-DAG: @_ZTIu10__SVBool_t = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTSu10__SVBool_t
diff --git a/clang/test/CodeGenCXX/alignment.cpp b/clang/test/CodeGenCXX/alignment.cpp
index 37509fcb4dd55..c9378bf20a47c 100644
--- a/clang/test/CodeGenCXX/alignment.cpp
+++ b/clang/test/CodeGenCXX/alignment.cpp
@@ -308,4 +308,20 @@ namespace test1 {
     D d;
     AlignedArray result = d.bArray;
   }
+
+  // CHECK-LABEL: @_ZN5test11hEPA_NS_1BE
+  void h(B (*b)[]) {
+    // CHECK: [[RESULT:%.*]] = alloca [[ARRAY]], align 64
+    // CHECK: [[B_P:%.*]] = load [0 x [[B]]]*, [0 x [[B]]]**
+    // CHECK: [[ELEMENT_P:%.*]] = getelementptr inbounds [0 x [[B]]], [0 x [[B]]]* [[B_P]], i64 0
+    // CHECK: [[ARRAY_P:%.*]] = getelementptr inbounds [[B]], [[B]]* [[ELEMENT_P]], i32 0, i32 2
+    // CHECK: [[T0:%.*]] = bitcast [[ARRAY]]* [[RESULT]] to i8*
+    // CHECK: [[T1:%.*]] = bitcast [[ARRAY]]* [[ARRAY_P]] to i8*
+    // CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 64 [[T0]], i8* align 16 [[T1]], i64 16, i1 false)
+    AlignedArray result = (*b)->bArray;
+  }
 }
+
+// CHECK-LABEL: @_Z22incomplete_array_derefPA_i
+// CHECK: load i32, i32* {{%.*}}, align 4
+int incomplete_array_deref(int (*p)[]) { return (*p)[2]; }
diff --git a/clang/test/CodeGenCXX/builtin-amdgcn-atomic-inc-dec.cpp b/clang/test/CodeGenCXX/builtin-amdgcn-atomic-inc-dec.cpp
index 535c3d754954b..77ea3d485c8aa 100644
--- a/clang/test/CodeGenCXX/builtin-amdgcn-atomic-inc-dec.cpp
+++ b/clang/test/CodeGenCXX/builtin-amdgcn-atomic-inc-dec.cpp
@@ -2,9 +2,9 @@
 // RUN: %clang_cc1 %s -x hip -fcuda-is-device -emit-llvm -O0 -o - \
 // RUN:   -triple=amdgcn-amd-amdhsa  | opt -S | FileCheck %s
 
-__attribute__((device)) void test_non_volatile_parameter32(int *ptr) {
+__attribute__((device)) void test_non_volatile_parameter32(__UINT32_TYPE__ *ptr) {
   // CHECK-LABEL: test_non_volatile_parameter32
-  int res;
+  __UINT32_TYPE__ res;
   // CHECK: %ptr.addr = alloca i32*, align 8, addrspace(5)
   // CHECK-NEXT: %ptr.addr.ascast = addrspacecast i32* addrspace(5)* %ptr.addr to i32**
   // CHECK-NEXT: %res = alloca i32, align 4, addrspace(5)
@@ -25,9 +25,9 @@ __attribute__((device)) void test_non_volatile_parameter32(int *ptr) {
   res = __builtin_amdgcn_atomic_dec32(ptr, *ptr, __ATOMIC_SEQ_CST, "workgroup");
 }
 
-__attribute__((device)) void test_non_volatile_parameter64(__INT64_TYPE__ *ptr) {
+__attribute__((device)) void test_non_volatile_parameter64(__UINT64_TYPE__ *ptr) {
   // CHECK-LABEL: test_non_volatile_parameter64
-  __INT64_TYPE__ res;
+  __UINT64_TYPE__ res;
   // CHECK: %ptr.addr = alloca i64*, align 8, addrspace(5)
   // CHECK-NEXT: %ptr.addr.ascast = addrspacecast i64* addrspace(5)* %ptr.addr to i64**
   // CHECK-NEXT: %res = alloca i64, align 8, addrspace(5)
@@ -48,9 +48,9 @@ __attribute__((device)) void test_non_volatile_parameter64(__INT64_TYPE__ *ptr)
   res = __builtin_amdgcn_atomic_dec64(ptr, *ptr, __ATOMIC_SEQ_CST, "workgroup");
 }
 
-__attribute__((device)) void test_volatile_parameter32(volatile int *ptr) {
+__attribute__((device)) void test_volatile_parameter32(volatile __UINT32_TYPE__ *ptr) {
   // CHECK-LABEL: test_volatile_parameter32
-  int res;
+  __UINT32_TYPE__ res;
   // CHECK: %ptr.addr = alloca i32*, align 8, addrspace(5)
   // CHECK-NEXT: %ptr.addr.ascast = addrspacecast i32* addrspace(5)* %ptr.addr to i32**
   // CHECK-NEXT: %res = alloca i32, align 4, addrspace(5)
@@ -71,9 +71,9 @@ __attribute__((device)) void test_volatile_parameter32(volatile int *ptr) {
   res = __builtin_amdgcn_atomic_dec32(ptr, *ptr, __ATOMIC_SEQ_CST, "workgroup");
 }
 
-__attribute__((device)) void test_volatile_parameter64(volatile __INT64_TYPE__ *ptr) {
+__attribute__((device)) void test_volatile_parameter64(volatile __UINT64_TYPE__ *ptr) {
   // CHECK-LABEL: test_volatile_parameter64
-  __INT64_TYPE__ res;
+  __UINT64_TYPE__ res;
   // CHECK: %ptr.addr = alloca i64*, align 8, addrspace(5)
   // CHECK-NEXT: %ptr.addr.ascast = addrspacecast i64* addrspace(5)* %ptr.addr to i64**
   // CHECK-NEXT: %res = alloca i64, align 8, addrspace(5)
@@ -96,7 +96,7 @@ __attribute__((device)) void test_volatile_parameter64(volatile __INT64_TYPE__ *
 
 __attribute__((device)) void test_shared32() {
   // CHECK-LABEL: test_shared32
-  __attribute__((shared)) int val;
+  __attribute__((shared)) __UINT32_TYPE__ val;
 
   // CHECK: %0 = load i32, i32* addrspacecast (i32 addrspace(3)* @_ZZ13test_shared32vE3val to i32*), align 4
   // CHECK-NEXT: %1 = call i32 @llvm.amdgcn.atomic.inc.i32.p0i32(i32* addrspacecast (i32 addrspace(3)* @_ZZ13test_shared32vE3val to i32*), i32 %0, i32 7, i32 2, i1 false)
@@ -111,7 +111,7 @@ __attribute__((device)) void test_shared32() {
 
 __attribute__((device)) void test_shared64() {
   // CHECK-LABEL: test_shared64
-  __attribute__((shared)) __INT64_TYPE__ val;
+  __attribute__((shared)) __UINT64_TYPE__ val;
 
   // CHECK: %0 = load i64, i64* addrspacecast (i64 addrspace(3)* @_ZZ13test_shared64vE3val to i64*), align 8
   // CHECK-NEXT: %1 = call i64 @llvm.amdgcn.atomic.inc.i64.p0i64(i64* addrspacecast (i64 addrspace(3)* @_ZZ13test_shared64vE3val to i64*), i64 %0, i32 7, i32 2, i1 false)
@@ -124,7 +124,7 @@ __attribute__((device)) void test_shared64() {
   val = __builtin_amdgcn_atomic_dec64(&val, val, __ATOMIC_SEQ_CST, "workgroup");
 }
 
-int global_val32;
+__UINT32_TYPE__ global_val32;
 __attribute__((device)) void test_global32() {
   // CHECK-LABEL: test_global32
   // CHECK: %0 = load i32, i32* addrspacecast (i32 addrspace(1)* @global_val32 to i32*), align 4
@@ -138,7 +138,7 @@ __attribute__((device)) void test_global32() {
   global_val32 = __builtin_amdgcn_atomic_dec32(&global_val32, global_val32, __ATOMIC_SEQ_CST, "workgroup");
 }
 
-__INT64_TYPE__ global_val64;
+__UINT64_TYPE__ global_val64;
 __attribute__((device)) void test_global64() {
   // CHECK-LABEL: test_global64
   // CHECK: %0 = load i64, i64* addrspacecast (i64 addrspace(1)* @global_val64 to i64*), align 8
@@ -152,10 +152,10 @@ __attribute__((device)) void test_global64() {
   global_val64 = __builtin_amdgcn_atomic_dec64(&global_val64, global_val64, __ATOMIC_SEQ_CST, "workgroup");
 }
 
-__attribute__((constant)) int cval32;
+__attribute__((constant)) __UINT32_TYPE__ cval32;
 __attribute__((device)) void test_constant32() {
   // CHECK-LABEL: test_constant32
-  int local_val;
+  __UINT32_TYPE__ local_val;
 
   // CHECK: %0 = load i32, i32* addrspacecast (i32 addrspace(4)* @cval32 to i32*), align 4
   // CHECK-NEXT: %1 = call i32 @llvm.amdgcn.atomic.inc.i32.p0i32(i32* addrspacecast (i32 addrspace(4)* @cval32 to i32*), i32 %0, i32 7, i32 2, i1 false)
@@ -168,10 +168,10 @@ __attribute__((device)) void test_constant32() {
   local_val = __builtin_amdgcn_atomic_dec32(&cval32, cval32, __ATOMIC_SEQ_CST, "workgroup");
 }
 
-__attribute__((constant)) __INT64_TYPE__ cval64;
+__attribute__((constant)) __UINT64_TYPE__ cval64;
 __attribute__((device)) void test_constant64() {
   // CHECK-LABEL: test_constant64
-  __INT64_TYPE__ local_val;
+  __UINT64_TYPE__ local_val;
 
   // CHECK: %0 = load i64, i64* addrspacecast (i64 addrspace(4)* @cval64 to i64*), align 8
   // CHECK-NEXT: %1 = call i64 @llvm.amdgcn.atomic.inc.i64.p0i64(i64* addrspacecast (i64 addrspace(4)* @cval64 to i64*), i64 %0, i32 7, i32 2, i1 false)
@@ -186,7 +186,7 @@ __attribute__((device)) void test_constant64() {
 
 __attribute__((device)) void test_order32() {
   // CHECK-LABEL: test_order32
-  __attribute__((shared)) int val;
+  __attribute__((shared)) __UINT32_TYPE__ val;
 
   // CHECK: %1 = call i32 @llvm.amdgcn.atomic.inc.i32.p0i32(i32* addrspacecast (i32 addrspace(3)* @_ZZ12test_order32vE3val to i32*), i32 %0, i32 4, i32 2, i1 false)
   val = __builtin_amdgcn_atomic_inc32(&val, val, __ATOMIC_ACQUIRE, "workgroup");
@@ -203,7 +203,7 @@ __attribute__((device)) void test_order32() {
 
 __attribute__((device)) void test_order64() {
   // CHECK-LABEL: test_order64
-  __attribute__((shared)) __INT64_TYPE__ val;
+  __attribute__((shared)) __UINT64_TYPE__ val;
 
   // CHECK: %1 = call i64 @llvm.amdgcn.atomic.inc.i64.p0i64(i64* addrspacecast (i64 addrspace(3)* @_ZZ12test_order64vE3val to i64*), i64 %0, i32 4, i32 2, i1 false)
   val = __builtin_amdgcn_atomic_inc64(&val, val, __ATOMIC_ACQUIRE, "workgroup");
@@ -220,7 +220,7 @@ __attribute__((device)) void test_order64() {
 
 __attribute__((device)) void test_scope32() {
   // CHECK-LABEL: test_scope32
-  __attribute__((shared)) int val;
+  __attribute__((shared)) __UINT32_TYPE__ val;
 
   // CHECK: %1 = call i32 @llvm.amdgcn.atomic.inc.i32.p0i32(i32* addrspacecast (i32 addrspace(3)* @_ZZ12test_scope32vE3val to i32*), i32 %0, i32 7, i32 1, i1 false)
   val = __builtin_amdgcn_atomic_inc32(&val, val, __ATOMIC_SEQ_CST, "");
@@ -237,7 +237,7 @@ __attribute__((device)) void test_scope32() {
 
 __attribute__((device)) void test_scope64() {
   // CHECK-LABEL: test_scope64
-  __attribute__((shared)) __INT64_TYPE__ val;
+  __attribute__((shared)) __UINT64_TYPE__ val;
 
   // CHECK: %1 = call i64 @llvm.amdgcn.atomic.inc.i64.p0i64(i64* addrspacecast (i64 addrspace(3)* @_ZZ12test_scope64vE3val to i64*), i64 %0, i32 7, i32 1, i1 false)
   val = __builtin_amdgcn_atomic_inc64(&val, val, __ATOMIC_SEQ_CST, "");
diff --git a/clang/test/CodeGenCXX/debug-info-inheriting-constructor.cpp b/clang/test/CodeGenCXX/debug-info-inheriting-constructor.cpp
index 3a2605078220c..d523649ed5ab9 100644
--- a/clang/test/CodeGenCXX/debug-info-inheriting-constructor.cpp
+++ b/clang/test/CodeGenCXX/debug-info-inheriting-constructor.cpp
@@ -19,7 +19,7 @@ A::A(int i, ...) {}
 // CHECK-DAG: ![[A:.*]] = distinct !DISubprogram(name: "A", linkageName: "_ZN1BCI11AEiz"
 void foo() {
 // CHECK-DAG: ![[LOC]] = !DILocation(line: 0, scope: ![[A]], inlinedAt: ![[INL:[0-9]+]])
-// CHECK-DAG: ![[INL]] = !DILocation(line: [[@LINE+1]], scope: ![[FOO]])
+// CHECK-DAG: ![[INL]] = !DILocation(line: [[@LINE+1]], column: 5, scope: ![[FOO]])
   B b(0);
-// CHECK: ![[NOINL]] = !DILocation(line: [[@LINE+1]], scope: !{{[0-9]+}})
+// CHECK: ![[NOINL]] = !DILocation(line: [[@LINE+1]], column: 1, scope: !{{[0-9]+}})
 }
diff --git a/clang/test/CodeGenCXX/debug-info-inlined.cpp b/clang/test/CodeGenCXX/debug-info-inlined.cpp
index d36715bcfcb6f..22b60504000ff 100644
--- a/clang/test/CodeGenCXX/debug-info-inlined.cpp
+++ b/clang/test/CodeGenCXX/debug-info-inlined.cpp
@@ -25,5 +25,5 @@ B::B() : Forward(WithDtor()) {}
 // CHECK-SAME: %class.Forward** %
 // CHECK-SAME: !dbg ![[INL:[0-9]+]]
 
-// CHECK: ![[INL]] = !DILocation(line: 10, scope: ![[SP:[0-9]+]], inlinedAt:
+// CHECK: ![[INL]] = !DILocation(line: 10, column: 15, scope: ![[SP:[0-9]+]], inlinedAt:
 // CHECK: ![[SP]] = distinct !DISubprogram(name: "Base", {{.*}} DISPFlagDefinition
diff --git a/clang/test/CodeGenCXX/debug-info-lambda.cpp b/clang/test/CodeGenCXX/debug-info-lambda.cpp
index a10765de37778..7f79648aa6730 100644
--- a/clang/test/CodeGenCXX/debug-info-lambda.cpp
+++ b/clang/test/CodeGenCXX/debug-info-lambda.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm \
-// RUN:   -debug-info-kind=line-tables-only -dwarf-column-info -std=c++11 %s -o - | FileCheck %s
+// RUN:   -debug-info-kind=line-tables-only -std=c++11 %s -o - | FileCheck %s
 
 // CHECK-LABEL: define{{.*}}lambda_in_func
 void lambda_in_func(int &ref) {
diff --git a/clang/test/CodeGenCXX/debug-info-line-if.cpp b/clang/test/CodeGenCXX/debug-info-line-if.cpp
index 7e6cdb2eb9d81..3b54b5c7c65d0 100644
--- a/clang/test/CodeGenCXX/debug-info-line-if.cpp
+++ b/clang/test/CodeGenCXX/debug-info-line-if.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -debug-info-kind=limited -std=c++11 -S -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap %s
+// RUN: %clang_cc1 -debug-info-kind=limited -gno-column-info -std=c++11 -S -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap %s
 // PR19864
 extern int v[2];
 int a = 0, b = 0;
diff --git a/clang/test/CodeGenCXX/debug-info-member-call.cpp b/clang/test/CodeGenCXX/debug-info-member-call.cpp
index 3b5adb8e4b826..2b60de8aee491 100644
--- a/clang/test/CodeGenCXX/debug-info-member-call.cpp
+++ b/clang/test/CodeGenCXX/debug-info-member-call.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple x86_64-unknown_unknown -emit-llvm -debug-info-kind=standalone -dwarf-column-info %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown_unknown -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck %s
 void ext();
 
 struct Bar {
diff --git a/clang/test/CodeGenCXX/debug-info-nested-exprs.cpp b/clang/test/CodeGenCXX/debug-info-nested-exprs.cpp
index bd70373e22858..fa934bf2aa008 100644
--- a/clang/test/CodeGenCXX/debug-info-nested-exprs.cpp
+++ b/clang/test/CodeGenCXX/debug-info-nested-exprs.cpp
@@ -1,13 +1,13 @@
 // RUN: %clang_cc1 -triple=x86_64-pc-windows-msvc -debug-info-kind=limited \
-// RUN:    -std=c++11 -gcodeview -emit-llvm -o - %s \
+// RUN:    -std=c++11 -gcodeview -gno-column-info -emit-llvm -o - %s \
 // RUN:    | FileCheck -check-prefix=NONEST %s
 // RUN: %clang_cc1 -triple=x86_64-pc-windows-msvc -debug-info-kind=limited \
-// RUN:    -std=c++11 -gcodeview -dwarf-column-info -emit-llvm -o - %s \
+// RUN:    -std=c++11 -gcodeview -emit-llvm -o - %s \
 // RUN:    | FileCheck -check-prefix=COLUMNS %s
 // RUN: %clang_cc1 -triple=x86_64-unknown-linux-gnu -debug-info-kind=limited \
-// RUN:    -std=c++11 -emit-llvm -o - %s | FileCheck -check-prefix=NESTED %s
+// RUN:    -std=c++11 -gno-column-info -emit-llvm -o - %s | FileCheck -check-prefix=NESTED %s
 // RUN: %clang_cc1 -triple=x86_64-unknown-linux-gnu -debug-info-kind=limited \
-// RUN:    -std=c++11 -dwarf-column-info -emit-llvm -o - %s \
+// RUN:    -std=c++11 -emit-llvm -o - %s \
 // RUN:    | FileCheck -check-prefix=COLUMNS %s
 
 class Foo {
diff --git a/clang/test/CodeGenCXX/debug-info-scope.cpp b/clang/test/CodeGenCXX/debug-info-scope.cpp
index 1124282697492..b55ae9e858787 100644
--- a/clang/test/CodeGenCXX/debug-info-scope.cpp
+++ b/clang/test/CodeGenCXX/debug-info-scope.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -debug-info-kind=limited -std=c++11 -emit-llvm %s -o -| FileCheck %s
+// RUN: %clang_cc1 -debug-info-kind=limited -gno-column-info -std=c++11 -emit-llvm %s -o -| FileCheck %s
 //
 // Two variables with the same name in subsequent if staments need to be in separate scopes.
 //
diff --git a/clang/test/CodeGenCXX/linetable-cleanup.cpp b/clang/test/CodeGenCXX/linetable-cleanup.cpp
index 748f056918b9b..c8daf40757045 100644
--- a/clang/test/CodeGenCXX/linetable-cleanup.cpp
+++ b/clang/test/CodeGenCXX/linetable-cleanup.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -emit-llvm -debug-info-kind=limited -triple x86_64-apple-darwin10 %s -o - | FileCheck %s
-// RUN: %clang_cc1 -emit-llvm -debug-info-kind=limited -triple x86_64-apple-darwin10 -std=c++98 %s -o - | FileCheck %s
-// RUN: %clang_cc1 -emit-llvm -debug-info-kind=limited -triple x86_64-apple-darwin10 -std=c++11 %s -o - | FileCheck %s
+// RUN: %clang_cc1 -emit-llvm -debug-info-kind=limited -gno-column-info -triple x86_64-apple-darwin10 %s -o - | FileCheck %s
+// RUN: %clang_cc1 -emit-llvm -debug-info-kind=limited -gno-column-info -triple x86_64-apple-darwin10 -std=c++98 %s -o - | FileCheck %s
+// RUN: %clang_cc1 -emit-llvm -debug-info-kind=limited -gno-column-info -triple x86_64-apple-darwin10 -std=c++11 %s -o - | FileCheck %s
 
 // Check the line numbers for cleanup code with EH in combination with
 // simple return expressions.
diff --git a/clang/test/CodeGenCXX/linetable-eh.cpp b/clang/test/CodeGenCXX/linetable-eh.cpp
index b31df54f7c456..6821655a0bb14 100644
--- a/clang/test/CodeGenCXX/linetable-eh.cpp
+++ b/clang/test/CodeGenCXX/linetable-eh.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -emit-llvm -debug-info-kind=limited -triple x86_64-apple-macosx10.9.0 -munwind-tables -std=c++11 -fcxx-exceptions -fexceptions %s -o - | FileCheck -allow-deprecated-dag-overlap %s
+// RUN: %clang_cc1 -emit-llvm -debug-info-kind=limited -gno-column-info -triple x86_64-apple-macosx10.9.0 -munwind-tables -std=c++11 -fcxx-exceptions -fexceptions %s -o - | FileCheck -allow-deprecated-dag-overlap %s
 
 // Test that emitting a landing pad does not affect the line table
 // entries for the code that triggered it.
diff --git a/clang/test/CodeGenCXX/linetable-fnbegin.cpp b/clang/test/CodeGenCXX/linetable-fnbegin.cpp
index 3b93d70bd7054..d64b5a64b174a 100644
--- a/clang/test/CodeGenCXX/linetable-fnbegin.cpp
+++ b/clang/test/CodeGenCXX/linetable-fnbegin.cpp
@@ -9,7 +9,7 @@
 // CHECK-SAME:                               file: [[HPP]], line: 22
 // CHECK-SAME:                               DISPFlagDefinition
 // We shouldn't need a lexical block for this function.
-// CHECK: [[DBG]] = !DILocation(line: 23, scope: [[SP]])
+// CHECK: [[DBG]] = !DILocation(line: 23, column: 3, scope: [[SP]])
 
 
 # 1 "./template.h" 1
diff --git a/clang/test/CodeGenCXX/lpad-linetable.cpp b/clang/test/CodeGenCXX/lpad-linetable.cpp
index 9b429bc2a223d..b20106f5f98fe 100644
--- a/clang/test/CodeGenCXX/lpad-linetable.cpp
+++ b/clang/test/CodeGenCXX/lpad-linetable.cpp
@@ -4,7 +4,7 @@
 // CHECK: ret i32
 // CHECK: landingpad {{.*}}
 // CHECK-NEXT: !dbg ![[LPAD:[0-9]+]]
-// CHECK: ![[LPAD]] = !DILocation(line: 24, scope: !{{.*}})
+// CHECK: ![[LPAD]] = !DILocation(line: 24, column: 1, scope: !{{.*}})
 
 # 1 "/usr/include/c++/4.2.1/vector" 1 3
 typedef long unsigned int __darwin_size_t;
diff --git a/clang/test/CodeGenCXX/trivial_abi.cpp b/clang/test/CodeGenCXX/trivial_abi.cpp
index e7e7f840944d9..cb9b9dfeb4bd9 100644
--- a/clang/test/CodeGenCXX/trivial_abi.cpp
+++ b/clang/test/CodeGenCXX/trivial_abi.cpp
@@ -43,6 +43,31 @@ struct HasNonTrivial {
   NonTrivial m;
 };
 
+struct B0 {
+  virtual Small m0();
+};
+
+struct B1 {
+  virtual Small m0();
+};
+
+struct D0 : B0, B1 {
+  Small m0() override;
+};
+
+// CHECK-LABEL: define i64 @_ZThn8_N2D02m0Ev(
+// CHECK: %[[RETVAL:.*]] = alloca %[[STRUCT_SMALL]], align 8
+// CHECK: %[[CALL:.*]] = tail call i64 @_ZN2D02m0Ev(
+// CHECK: %[[COERCE_DIVE:.*]] = getelementptr inbounds %[[STRUCT_SMALL]], %[[STRUCT_SMALL]]* %[[RETVAL]], i32 0, i32 0
+// CHECK: %[[COERCE_VAL_IP:.*]] = inttoptr i64 %[[CALL]] to i32*
+// CHECK: store i32* %[[COERCE_VAL_IP]], i32** %[[COERCE_DIVE]], align 8
+// CHECK: %[[COERCE_DIVE2:.*]] = getelementptr inbounds %[[STRUCT_SMALL]], %[[STRUCT_SMALL]]* %[[RETVAL]], i32 0, i32 0
+// CHECK: %[[V3:.*]] = load i32*, i32** %[[COERCE_DIVE2]], align 8
+// CHECK: %[[COERCE_VAL_PI:.*]] = ptrtoint i32* %[[V3]] to i64
+// CHECK: ret i64 %[[COERCE_VAL_PI]]
+
+Small D0::m0() { return {}; }
+
 // CHECK: define void @_Z14testParamSmall5Small(i64 %[[A_COERCE:.*]])
 // CHECK: %[[A:.*]] = alloca %[[STRUCT_SMALL]], align 8
 // CHECK: %[[COERCE_DIVE:.*]] = getelementptr inbounds %[[STRUCT_SMALL]], %[[STRUCT_SMALL]]* %[[A]], i32 0, i32 0
diff --git a/clang/test/CodeGenCXX/ubsan-coroutines.cpp b/clang/test/CodeGenCXX/ubsan-coroutines.cpp
index 8728c1511122b..dacc229ce519e 100644
--- a/clang/test/CodeGenCXX/ubsan-coroutines.cpp
+++ b/clang/test/CodeGenCXX/ubsan-coroutines.cpp
@@ -32,7 +32,7 @@ struct task {
   struct promise_type {
     task get_return_object() { return task(); }
     suspend_always initial_suspend() { return {}; }
-    suspend_always final_suspend() { return {}; }
+    suspend_always final_suspend() noexcept { return {}; }
     void return_void() {}
     void unhandled_exception() {}
   };
diff --git a/clang/test/CodeGenCXX/vtable-consteval.cpp b/clang/test/CodeGenCXX/vtable-consteval.cpp
new file mode 100644
index 0000000000000..ed4a25290d359
--- /dev/null
+++ b/clang/test/CodeGenCXX/vtable-consteval.cpp
@@ -0,0 +1,64 @@
+// RUN: %clang_cc1 -std=c++20 %s -emit-llvm -o - -triple x86_64-linux | FileCheck %s --check-prefix=ITANIUM --implicit-check-not=DoNotEmit
+// RUN: %clang_cc1 -std=c++20 %s -emit-llvm -o - -triple x86_64-windows | FileCheck %s --check-prefix=MSABI --implicit-check-not=DoNotEmit
+
+// FIXME: The MSVC ABI rule in use here was discussed with MS folks prior to
+// them implementing virtual consteval functions, but we do not know for sure
+// if this is the ABI rule they will use.
+
+// ITANIUM-DAG: @_ZTV1A = {{.*}} constant { [2 x i8*] } {{.*}} null, {{.*}} @_ZTI1A
+// MSABI-DAG: @[[A_VFTABLE:.*]] = {{.*}} constant { [1 x i8*] } {{.*}} @"??_R4A@@6B@"
+struct A {
+  virtual consteval void DoNotEmit_f() {}
+};
+// ITANIUM-DAG: @a = {{.*}}global { i8** } { {{.*}} @_ZTV1A,
+// MSABI-DAG: @"?a@@3UA@@A" = {{.*}}global { i8** } { i8** @"??_7A@@6B@" }
+A a;
+
+// ITANIUM-DAG: @_ZTV1B = {{.*}} constant { [4 x i8*] } {{.*}} null, {{.*}} @_ZTI1B {{.*}} @_ZN1B1fEv {{.*}} @_ZN1B1hEv
+// MSABI-DAG: @[[B_VFTABLE:.*]] = {{.*}} constant { [3 x i8*] } {{.*}} @"??_R4B@@6B@" {{.*}} @"?f@B@@UEAAXXZ" {{.*}} @"?h@B@@UEAAXXZ"
+struct B {
+  virtual void f() {}
+  virtual consteval void DoNotEmit_g() {}
+  virtual void h() {}
+};
+// ITANIUM-DAG: @b = {{.*}}global { i8** } { {{.*}} @_ZTV1B,
+// MSABI-DAG: @"?b@@3UB@@A" = {{.*}}global { i8** } { i8** @"??_7B@@6B@" }
+B b;
+
+// ITANIUM-DAG: @_ZTV1C = {{.*}} constant { [4 x i8*] } {{.*}} null, {{.*}} @_ZTI1C {{.*}} @_ZN1CD1Ev {{.*}} @_ZN1CD0Ev
+// MSABI-DAG: @[[C_VFTABLE:.*]] = {{.*}} constant { [2 x i8*] } {{.*}} @"??_R4C@@6B@" {{.*}} @"??_GC@@UEAAPEAXI@Z"
+struct C {
+  virtual ~C() = default;
+  virtual consteval C &operator=(const C&) = default;
+};
+// ITANIUM-DAG: @c = {{.*}}global { i8** } { {{.*}} @_ZTV1C,
+// MSABI-DAG: @"?c@@3UC@@A" = {{.*}}global { i8** } { i8** @"??_7C@@6B@" }
+C c;
+
+// ITANIUM-DAG: @_ZTV1D = {{.*}} constant { [4 x i8*] } {{.*}} null, {{.*}} @_ZTI1D {{.*}} @_ZN1DD1Ev {{.*}} @_ZN1DD0Ev
+// MSABI-DAG: @[[D_VFTABLE:.*]] = {{.*}} constant { [2 x i8*] } {{.*}} @"??_R4D@@6B@" {{.*}} @"??_GD@@UEAAPEAXI@Z"
+struct D : C {};
+// ITANIUM-DAG: @d = {{.*}}global { i8** } { {{.*}} @_ZTV1D,
+// MSABI-DAG: @"?d@@3UD@@A" = {{.*}}global { i8** } { i8** @"??_7D@@6B@" }
+D d;
+
+// ITANIUM-DAG: @_ZTV1E = {{.*}} constant { [3 x i8*] } {{.*}} null, {{.*}} @_ZTI1E {{.*}} @_ZN1E1fEv
+// MSABI-DAG: @[[E_VFTABLE:.*]] = {{.*}} constant { [2 x i8*] } {{.*}} @"??_R4E@@6B@" {{.*}} @"?f@E@@UEAAXXZ"
+struct E { virtual void f() {} };
+// ITANIUM-DAG: @e = {{.*}}global { i8** } { {{.*}} @_ZTV1E,
+// MSABI-DAG: @"?e@@3UE@@A" = {{.*}}global { i8** } { i8** @"??_7E@@6B@" }
+E e;
+
+// ITANIUM-DAG: @_ZTV1F = {{.*}} constant { [3 x i8*] } {{.*}} null, {{.*}} @_ZTI1F {{.*}} @_ZN1E1fEv
+// MSABI-DAG: @[[F_VFTABLE:.*]] = {{.*}} constant { [2 x i8*] } {{.*}} @"??_R4F@@6B@" {{.*}} @"?f@E@@UEAAXXZ"
+struct F : E { virtual consteval void DoNotEmit_g(); };
+// ITANIUM-DAG: @f = {{.*}}global { i8** } { {{.*}} @_ZTV1F,
+// MSABI-DAG: @"?f@@3UF@@A" = {{.*}}global { i8** } { i8** @"??_7F@@6B@" }
+F f;
+
+// MSABI-DAG: @"??_7A@@6B@" = {{.*}} alias {{.*}} @[[A_VFTABLE]],
+// MSABI-DAG: @"??_7B@@6B@" = {{.*}} alias {{.*}} @[[B_VFTABLE]],
+// MSABI-DAG: @"??_7C@@6B@" = {{.*}} alias {{.*}} @[[C_VFTABLE]],
+// MSABI-DAG: @"??_7D@@6B@" = {{.*}} alias {{.*}} @[[D_VFTABLE]],
+// MSABI-DAG: @"??_7E@@6B@" = {{.*}} alias {{.*}} @[[E_VFTABLE]],
+// MSABI-DAG: @"??_7F@@6B@" = {{.*}} alias {{.*}} @[[F_VFTABLE]],
diff --git a/clang/test/CodeGenCXX/vtable-constexpr.cpp b/clang/test/CodeGenCXX/vtable-constexpr.cpp
new file mode 100644
index 0000000000000..e398a9e5835d1
--- /dev/null
+++ b/clang/test/CodeGenCXX/vtable-constexpr.cpp
@@ -0,0 +1,27 @@
+// RUN: %clang_cc1 -std=c++20 %s -emit-llvm -o - -triple %itanium_abi_triple | FileCheck %s --implicit-check-not=DoNotEmit
+
+// constexpr virtual functions can be called at runtime and go in the vtable as
+// normal. But they are implicitly inline so are never the key function.
+
+struct DoNotEmit {
+  virtual constexpr void f();
+};
+constexpr void DoNotEmit::f() {}
+
+// CHECK-DAG: @_ZTV1B = {{.*}} constant { [3 x i8*] } { {{.*}} null, {{.*}} @_ZTI1B {{.*}} @_ZN1B1fEv
+struct B {
+  // CHECK-DAG: define {{.*}} @_ZN1B1fEv
+  virtual constexpr void f() {}
+};
+B b;
+
+struct CBase {
+  virtual constexpr void f(); // not key function
+};
+
+// CHECK-DAG: @_ZTV1C = {{.*}} constant {{.*}} null, {{.*}} @_ZTI1C {{.*}} @_ZN1C1fEv
+struct C : CBase {
+  void f(); // key function
+};
+// CHECK-DAG: define {{.*}} @_ZN1C1fEv
+void C::f() {}
diff --git a/clang/test/CodeGenCoroutines/Inputs/coroutine.h b/clang/test/CodeGenCoroutines/Inputs/coroutine.h
index d58212b1d5280..5cc78a4904aad 100644
--- a/clang/test/CodeGenCoroutines/Inputs/coroutine.h
+++ b/clang/test/CodeGenCoroutines/Inputs/coroutine.h
@@ -72,9 +72,9 @@ struct suspend_always {
   void await_resume() {}
 };
 struct suspend_never {
-  bool await_ready() { return true; }
-  void await_suspend(coroutine_handle<>) {}
-  void await_resume() {}
+  bool await_ready() noexcept { return true; }
+  void await_suspend(coroutine_handle<>) noexcept {}
+  void await_resume() noexcept {}
 };
 
 }}}
diff --git a/clang/test/CodeGenCoroutines/coro-alloc.cpp b/clang/test/CodeGenCoroutines/coro-alloc.cpp
index a73b7d85a4be6..bf8edf012a33f 100644
--- a/clang/test/CodeGenCoroutines/coro-alloc.cpp
+++ b/clang/test/CodeGenCoroutines/coro-alloc.cpp
@@ -10,7 +10,7 @@ struct coroutine_traits; // expected-note {{declared here}}
 template <class Promise = void>
 struct coroutine_handle {
   coroutine_handle() = default;
-  static coroutine_handle from_address(void *) { return {}; }
+  static coroutine_handle from_address(void *) noexcept { return {}; }
 };
 
 template <>
@@ -18,7 +18,7 @@ struct coroutine_handle<void> {
   static coroutine_handle from_address(void *) { return {}; }
   coroutine_handle() = default;
   template <class PromiseType>
-  coroutine_handle(coroutine_handle<PromiseType>) {}
+  coroutine_handle(coroutine_handle<PromiseType>) noexcept {}
 };
 
 } // end namespace experimental
@@ -36,9 +36,9 @@ void  operator delete(void* __p, const std::nothrow_t&) noexcept;
 
 
 struct suspend_always {
-  bool await_ready() { return false; }
-  void await_suspend(std::experimental::coroutine_handle<>) {}
-  void await_resume() {}
+  bool await_ready() noexcept { return false; }
+  void await_suspend(std::experimental::coroutine_handle<>) noexcept {}
+  void await_resume() noexcept {}
 };
 
 struct global_new_delete_tag {};
@@ -48,7 +48,7 @@ struct std::experimental::coroutine_traits<void, global_new_delete_tag> {
   struct promise_type {
     void get_return_object() {}
     suspend_always initial_suspend() { return {}; }
-    suspend_always final_suspend() { return {}; }
+    suspend_always final_suspend() noexcept { return {}; }
     void return_void() {}
   };
 };
@@ -89,7 +89,7 @@ struct std::experimental::coroutine_traits<void, promise_new_tag> {
     void *operator new(unsigned long);
     void get_return_object() {}
     suspend_always initial_suspend() { return {}; }
-    suspend_always final_suspend() { return {}; }
+    suspend_always final_suspend() noexcept { return {}; }
     void return_void() {}
   };
 };
@@ -115,7 +115,7 @@ struct std::experimental::coroutine_traits<void, promise_matching_placement_new_
                        int, float, double);
     void get_return_object() {}
     suspend_always initial_suspend() { return {}; }
-    suspend_always final_suspend() { return {}; }
+    suspend_always final_suspend() noexcept { return {}; }
     void return_void() {}
   };
 };
@@ -145,7 +145,7 @@ struct std::experimental::coroutine_traits<void, promise_matching_global_placeme
   struct promise_type {
     void get_return_object() {}
     suspend_always initial_suspend() { return {}; }
-    suspend_always final_suspend() { return {}; }
+    suspend_always final_suspend() noexcept { return {}; }
     void return_void() {}
   };
 };
@@ -168,7 +168,7 @@ struct std::experimental::coroutine_traits<void, promise_delete_tag> {
     void operator delete(void*);
     void get_return_object() {}
     suspend_always initial_suspend() { return {}; }
-    suspend_always final_suspend() { return {}; }
+    suspend_always final_suspend() noexcept { return {}; }
     void return_void() {}
   };
 };
@@ -193,7 +193,7 @@ struct std::experimental::coroutine_traits<void, promise_sized_delete_tag> {
     void operator delete(void*, unsigned long);
     void get_return_object() {}
     suspend_always initial_suspend() { return {}; }
-    suspend_always final_suspend() { return {}; }
+    suspend_always final_suspend() noexcept { return {}; }
     void return_void() {}
   };
 };
@@ -218,7 +218,7 @@ struct std::experimental::coroutine_traits<int, promise_on_alloc_failure_tag> {
   struct promise_type {
     int get_return_object() { return 0; }
     suspend_always initial_suspend() { return {}; }
-    suspend_always final_suspend() { return {}; }
+    suspend_always final_suspend() noexcept { return {}; }
     void return_void() {}
     static int get_return_object_on_allocation_failure() { return -1; }
   };
diff --git a/clang/test/CodeGenCoroutines/coro-always-inline.cpp b/clang/test/CodeGenCoroutines/coro-always-inline.cpp
index a2e4bba45c0c9..e4aa14a6ac397 100644
--- a/clang/test/CodeGenCoroutines/coro-always-inline.cpp
+++ b/clang/test/CodeGenCoroutines/coro-always-inline.cpp
@@ -14,22 +14,22 @@ namespace experimental {
 struct handle {};
 
 struct awaitable {
-  bool await_ready() { return true; }
+  bool await_ready() noexcept { return true; }
   // CHECK-NOT: await_suspend
-  inline void __attribute__((__always_inline__)) await_suspend(handle) {}
-  bool await_resume() { return true; }
+  inline void __attribute__((__always_inline__)) await_suspend(handle) noexcept {}
+  bool await_resume() noexcept { return true; }
 };
 
 template <typename T>
 struct coroutine_handle {
-  static handle from_address(void *address) { return {}; }
+  static handle from_address(void *address) noexcept { return {}; }
 };
 
 template <typename T = void>
 struct coroutine_traits {
   struct promise_type {
     awaitable initial_suspend() { return {}; }
-    awaitable final_suspend() { return {}; }
+    awaitable final_suspend() noexcept { return {}; }
     void return_void() {}
     T get_return_object() { return T(); }
     void unhandled_exception() {}
diff --git a/clang/test/CodeGenCoroutines/coro-await-domination.cpp b/clang/test/CodeGenCoroutines/coro-await-domination.cpp
index 5df22374a6e7f..3ce8cd7a6227f 100644
--- a/clang/test/CodeGenCoroutines/coro-await-domination.cpp
+++ b/clang/test/CodeGenCoroutines/coro-await-domination.cpp
@@ -7,7 +7,7 @@ struct coro {
   struct promise_type {
     coro get_return_object();
     suspend_never initial_suspend();
-    suspend_never final_suspend();
+    suspend_never final_suspend() noexcept;
     void return_void();
     static void unhandled_exception();
   };
@@ -35,4 +35,3 @@ extern "C" coro f(int) {
   x = co_await A{};
   consume(x);
 }
-
diff --git a/clang/test/CodeGenCoroutines/coro-await-resume-eh.cpp b/clang/test/CodeGenCoroutines/coro-await-resume-eh.cpp
index f0f8855fbd4d3..b5dcc3f9da78d 100644
--- a/clang/test/CodeGenCoroutines/coro-await-resume-eh.cpp
+++ b/clang/test/CodeGenCoroutines/coro-await-resume-eh.cpp
@@ -22,7 +22,7 @@ struct throwing_task {
   struct promise_type {
     auto get_return_object() { return throwing_task{}; }
     auto initial_suspend() { return throwing_awaitable{}; }
-    auto final_suspend() { return coro::suspend_never{}; }
+    auto final_suspend() noexcept { return coro::suspend_never{}; }
     void return_void() {}
     void unhandled_exception() {}
   };
@@ -76,7 +76,7 @@ throwing_task f() {
   // CHECK-NEXT: br label %[[COROFINAL]]
 
   // CHECK: [[COROFINAL]]:
-  // CHECK-NEXT: invoke void @_ZN13throwing_task12promise_type13final_suspendEv
+  // CHECK-NEXT: call void @_ZN13throwing_task12promise_type13final_suspendEv
   co_return;
 }
 
@@ -90,7 +90,7 @@ struct noexcept_task {
   struct promise_type {
     auto get_return_object() { return noexcept_task{}; }
     auto initial_suspend() { return noexcept_awaitable{}; }
-    auto final_suspend() { return coro::suspend_never{}; }
+    auto final_suspend() noexcept { return coro::suspend_never{}; }
     void return_void() {}
     void unhandled_exception() {}
   };
diff --git a/clang/test/CodeGenCoroutines/coro-await.cpp b/clang/test/CodeGenCoroutines/coro-await.cpp
index acd3c4aa97162..90da9be5976d4 100644
--- a/clang/test/CodeGenCoroutines/coro-await.cpp
+++ b/clang/test/CodeGenCoroutines/coro-await.cpp
@@ -17,7 +17,7 @@ struct coroutine_handle<void> {
 
 template <typename Promise>
 struct coroutine_handle : coroutine_handle<> {
-  static coroutine_handle from_address(void *);
+  static coroutine_handle from_address(void *) noexcept;
 };
 
 }
@@ -29,9 +29,9 @@ struct init_susp {
   void await_resume();
 };
 struct final_susp {
-  bool await_ready();
-  void await_suspend(std::experimental::coroutine_handle<>);
-  void await_resume();
+  bool await_ready() noexcept;
+  void await_suspend(std::experimental::coroutine_handle<>) noexcept;
+  void await_resume() noexcept;
 };
 
 struct suspend_always {
@@ -46,7 +46,7 @@ struct std::experimental::coroutine_traits<void> {
   struct promise_type {
     void get_return_object();
     init_susp initial_suspend();
-    final_susp final_suspend();
+    final_susp final_suspend() noexcept;
     void return_void();
   };
 };
@@ -119,7 +119,7 @@ struct std::experimental::coroutine_traits<void,int> {
   struct promise_type {
     void get_return_object();
     init_susp initial_suspend();
-    final_susp final_suspend();
+    final_susp final_suspend() noexcept;
     void return_void();
     suspend_maybe yield_value(int);
   };
@@ -295,7 +295,7 @@ struct std::experimental::coroutine_traits<void,double> {
   struct promise_type {
     void get_return_object();
     init_susp initial_suspend();
-    final_susp final_suspend();
+    final_susp final_suspend() noexcept;
     void return_void();
     AwaitResumeReturnsLValue yield_value(int);
   };
diff --git a/clang/test/CodeGenCoroutines/coro-dest-slot.cpp b/clang/test/CodeGenCoroutines/coro-dest-slot.cpp
index 4c7395ba608c3..0c8ef6b045821 100644
--- a/clang/test/CodeGenCoroutines/coro-dest-slot.cpp
+++ b/clang/test/CodeGenCoroutines/coro-dest-slot.cpp
@@ -8,7 +8,7 @@ struct coro {
   struct promise_type {
     coro get_return_object();
     suspend_always initial_suspend();
-    suspend_never final_suspend();
+    suspend_never final_suspend() noexcept;
     void return_void();
     static void unhandled_exception();
   };
diff --git a/clang/test/CodeGenCoroutines/coro-gro-nrvo.cpp b/clang/test/CodeGenCoroutines/coro-gro-nrvo.cpp
index 42856f4479ec3..787cb01a53e6e 100644
--- a/clang/test/CodeGenCoroutines/coro-gro-nrvo.cpp
+++ b/clang/test/CodeGenCoroutines/coro-gro-nrvo.cpp
@@ -21,7 +21,7 @@ template <class RetObject>
 struct promise_type {
     RetObject get_return_object();
     suspend_always initial_suspend();
-    suspend_never final_suspend();
+    suspend_never final_suspend() noexcept;
     void return_void();
     static void unhandled_exception();
 };
@@ -52,7 +52,7 @@ struct promise_type_with_on_alloc_failure {
     static RetObject get_return_object_on_allocation_failure();
     RetObject get_return_object();
     suspend_always initial_suspend();
-    suspend_never final_suspend();
+    suspend_never final_suspend() noexcept;
     void return_void();
     static void unhandled_exception();
 };
diff --git a/clang/test/CodeGenCoroutines/coro-newpm-pipeline.cpp b/clang/test/CodeGenCoroutines/coro-newpm-pipeline.cpp
index cea71a1acc6bc..2a1b0d34658fb 100644
--- a/clang/test/CodeGenCoroutines/coro-newpm-pipeline.cpp
+++ b/clang/test/CodeGenCoroutines/coro-newpm-pipeline.cpp
@@ -33,19 +33,19 @@ namespace experimental {
 struct handle {};
 
 struct awaitable {
-  bool await_ready() { return true; }
-  void await_suspend(handle) {}
-  bool await_resume() { return true; }
+  bool await_ready() noexcept { return true; }
+  void await_suspend(handle) noexcept {}
+  bool await_resume() noexcept { return true; }
 };
 
 template <typename T> struct coroutine_handle {
-  static handle from_address(void *address) { return {}; }
+  static handle from_address(void *address) noexcept { return {}; }
 };
 
 template <typename T = void> struct coroutine_traits {
   struct promise_type {
     awaitable initial_suspend() { return {}; }
-    awaitable final_suspend() { return {}; }
+    awaitable final_suspend() noexcept { return {}; }
     void return_void() {}
     T get_return_object() { return T(); }
     void unhandled_exception() {}
diff --git a/clang/test/CodeGenCoroutines/coro-params.cpp b/clang/test/CodeGenCoroutines/coro-params.cpp
index 1302d734d4442..5f8b28b40b2b1 100644
--- a/clang/test/CodeGenCoroutines/coro-params.cpp
+++ b/clang/test/CodeGenCoroutines/coro-params.cpp
@@ -142,7 +142,7 @@ struct std::experimental::coroutine_traits<void, promise_matching_constructor, i
     promise_type() = delete;
     void get_return_object() {}
     suspend_always initial_suspend() { return {}; }
-    suspend_always final_suspend() { return {}; }
+    suspend_always final_suspend() noexcept { return {}; }
     void return_void() {}
     void unhandled_exception() {}
   };
@@ -166,7 +166,7 @@ template <typename... Args> struct std::experimental::coroutine_traits<method, A
     promise_type(some_class&, float);
     method get_return_object();
     suspend_always initial_suspend();
-    suspend_always final_suspend();
+    suspend_always final_suspend() noexcept;
     void return_void();
     void unhandled_exception();
   };
diff --git a/clang/test/CodeGenCoroutines/coro-promise-dtor.cpp b/clang/test/CodeGenCoroutines/coro-promise-dtor.cpp
index bc190c9d1cb52..4fb0f0fef7940 100644
--- a/clang/test/CodeGenCoroutines/coro-promise-dtor.cpp
+++ b/clang/test/CodeGenCoroutines/coro-promise-dtor.cpp
@@ -11,7 +11,7 @@ struct coro_t {
   struct promise_type {
     coro_t get_return_object();
     coro::suspend_never initial_suspend();
-    coro::suspend_never final_suspend();
+    coro::suspend_never final_suspend() noexcept;
     void return_void();
     promise_type();
     ~promise_type();
diff --git a/clang/test/CodeGenCoroutines/coro-ret-void.cpp b/clang/test/CodeGenCoroutines/coro-ret-void.cpp
index 6ebb44dfaefab..0007c36fa942d 100644
--- a/clang/test/CodeGenCoroutines/coro-ret-void.cpp
+++ b/clang/test/CodeGenCoroutines/coro-ret-void.cpp
@@ -8,7 +8,7 @@ struct coro1 {
   struct promise_type {
     coro1 get_return_object();
     coro::suspend_never initial_suspend();
-    coro::suspend_never final_suspend();
+    coro::suspend_never final_suspend() noexcept;
     void return_void();
   };
 };
@@ -39,7 +39,7 @@ struct coro2 {
   struct promise_type {
     coro2 get_return_object();
     coro::suspend_never initial_suspend();
-    coro::suspend_never final_suspend();
+    coro::suspend_never final_suspend() noexcept;
     void return_value(int);
   };
 };
diff --git a/clang/test/CodeGenCoroutines/coro-return-voidtype-initlist.cpp b/clang/test/CodeGenCoroutines/coro-return-voidtype-initlist.cpp
index 8526b6f8a8dad..8bd6c79b497de 100644
--- a/clang/test/CodeGenCoroutines/coro-return-voidtype-initlist.cpp
+++ b/clang/test/CodeGenCoroutines/coro-return-voidtype-initlist.cpp
@@ -14,7 +14,7 @@ template <>
 struct coroutine_handle<> {};
 template <typename>
 struct coroutine_handle : coroutine_handle<> {
-  static coroutine_handle from_address(void *);
+  static coroutine_handle from_address(void *) noexcept;
 };
 struct e {
   int await_ready();
@@ -29,13 +29,13 @@ template <typename>
 struct f;
 struct g {
   struct h {
-    int await_ready();
+    int await_ready() noexcept;
     template <typename al>
-    void await_suspend(std::experimental::coroutine_handle<al>);
-    void await_resume();
+    void await_suspend(std::experimental::coroutine_handle<al>) noexcept;
+    void await_resume() noexcept;
   };
   std::experimental::e initial_suspend();
-  h final_suspend();
+  h final_suspend() noexcept;
   template <typename ag>
   auto await_transform(ag) { return ah(ag()); }
 };
diff --git a/clang/test/CodeGenCoroutines/coro-return.cpp b/clang/test/CodeGenCoroutines/coro-return.cpp
index 17356f944ce1f..65c0655b86797 100644
--- a/clang/test/CodeGenCoroutines/coro-return.cpp
+++ b/clang/test/CodeGenCoroutines/coro-return.cpp
@@ -5,27 +5,27 @@ template <typename... T> struct coroutine_traits;
 
 template <class Promise = void> struct coroutine_handle {
   coroutine_handle() = default;
-  static coroutine_handle from_address(void *) { return {}; }
+  static coroutine_handle from_address(void *) noexcept { return {}; }
 };
 template <> struct coroutine_handle<void> {
   static coroutine_handle from_address(void *) { return {}; }
   coroutine_handle() = default;
   template <class PromiseType>
-  coroutine_handle(coroutine_handle<PromiseType>) {}
+  coroutine_handle(coroutine_handle<PromiseType>) noexcept {}
 };
 }
 
 struct suspend_always {
-  bool await_ready();
-  void await_suspend(std::experimental::coroutine_handle<>);
-  void await_resume();
+  bool await_ready() noexcept;
+  void await_suspend(std::experimental::coroutine_handle<>) noexcept;
+  void await_resume() noexcept;
 };
 
 template <> struct std::experimental::coroutine_traits<void> {
   struct promise_type {
     void get_return_object();
     suspend_always initial_suspend();
-    suspend_always final_suspend();
+    suspend_always final_suspend() noexcept;
     void return_void();
   };
 };
@@ -44,7 +44,7 @@ struct std::experimental::coroutine_traits<int> {
   struct promise_type {
     int get_return_object();
     suspend_always initial_suspend();
-    suspend_always final_suspend();
+    suspend_always final_suspend() noexcept;
     void return_value(int);
   };
 };
diff --git a/clang/test/CodeGenCoroutines/coro-unhandled-exception.cpp b/clang/test/CodeGenCoroutines/coro-unhandled-exception.cpp
index 039f02352cfca..870e67477ed14 100644
--- a/clang/test/CodeGenCoroutines/coro-unhandled-exception.cpp
+++ b/clang/test/CodeGenCoroutines/coro-unhandled-exception.cpp
@@ -17,7 +17,7 @@ struct coro_t {
       return {};
     }
     coro::suspend_never initial_suspend() { return {}; }
-    coro::suspend_never final_suspend() { return {}; }
+    coro::suspend_never final_suspend() noexcept { return {}; }
     void return_void(){}
     void unhandled_exception() noexcept;
   };
@@ -48,11 +48,9 @@ coro_t f() {
 // CHECK: [[CATCHRETDEST]]:
 // CHECK-NEXT: br label %[[TRYCONT:.+]]
 // CHECK: [[TRYCONT]]:
-// CHECK-NEXT: br label %[[RESUMECONT:.+]]
-// CHECK: [[RESUMECONT]]:
 // CHECK-NEXT: br label %[[COROFIN:.+]]
 // CHECK: [[COROFIN]]:
-// CHECK-NEXT: invoke void @"?final_suspend@promise_type@coro_t@@QEAA?AUsuspend_never@coroutines_v1@experimental@std@@XZ"(
+// CHECK-NEXT: call void @"?final_suspend@promise_type@coro_t@@QEAA?AUsuspend_never@coroutines_v1@experimental@std@@XZ"(
 
 // CHECK-LPAD: @_Z1fv(
 // CHECK-LPAD:   invoke void @_Z9may_throwv()
@@ -69,8 +67,6 @@ coro_t f() {
 // CHECK-LPAD: [[CATCHRETDEST]]:
 // CHECK-LPAD-NEXT: br label %[[TRYCONT:.+]]
 // CHECK-LPAD: [[TRYCONT]]:
-// CHECK-LPAD: br label %[[RESUMECONT:.+]]
-// CHECK-LPAD: [[RESUMECONT]]:
-// CHECK-LPAD-NEXT: br label %[[COROFIN:.+]]
+// CHECK-LPAD: br label %[[COROFIN:.+]]
 // CHECK-LPAD: [[COROFIN]]:
-// CHECK-LPAD-NEXT: invoke void @_ZN6coro_t12promise_type13final_suspendEv(
+// CHECK-LPAD-NEXT: call void @_ZN6coro_t12promise_type13final_suspendEv(
diff --git a/clang/test/CodeGenHIP/lit.local.cfg b/clang/test/CodeGenHIP/lit.local.cfg
new file mode 100644
index 0000000000000..ded4330455aeb
--- /dev/null
+++ b/clang/test/CodeGenHIP/lit.local.cfg
@@ -0,0 +1 @@
+config.suffixes = ['.cpp', '.hip']
diff --git a/clang/test/CodeGenObjC/aarch64-sve-types.m b/clang/test/CodeGenObjC/aarch64-sve-types.m
index 625c752965923..cc0a95a044068 100644
--- a/clang/test/CodeGenObjC/aarch64-sve-types.m
+++ b/clang/test/CodeGenObjC/aarch64-sve-types.m
@@ -1,7 +1,7 @@
 // RUN: not %clang_cc1 -triple aarch64-none-linux-gnu %s -emit-llvm -o - \
 // RUN:   2>&1 | FileCheck %s
 // RUN: not %clang_cc1 -triple aarch64-none-linux-gnu %s -emit-llvm -o - \
-// RUN:   -target-feature +sve 2>&1 | FileCheck %s
+// RUN:   -target-feature +sve,+bf16 2>&1 | FileCheck %s
 
 // CHECK: error: cannot yet @encode type __SVInt8_t
 const char s8[] = @encode(__SVInt8_t);
@@ -28,5 +28,8 @@
 // CHECK: error: cannot yet @encode type __SVFloat64_t
 const char f64[] = @encode(__SVFloat64_t);
 
+// CHECK: error: cannot yet @encode type __SVBFloat16_t
+const char bf16[] = @encode(__SVBFloat16_t);
+
 // CHECK: error: cannot yet @encode type __SVBool_t
 const char b8[] = @encode(__SVBool_t);
diff --git a/clang/test/CodeGenObjC/arc-blocks.m b/clang/test/CodeGenObjC/arc-blocks.m
index 3851773c68b76..0694155b5f4aa 100644
--- a/clang/test/CodeGenObjC/arc-blocks.m
+++ b/clang/test/CodeGenObjC/arc-blocks.m
@@ -747,5 +747,18 @@ id test22(int c, id x) {
   return c ? test22_0() : ({ id (^b)(void) = ^{ return x; }; test22_1(); b(); });
 }
 
+@interface Test23
+-(void)m:(int)i, ...;
+@end
+
+// CHECK-COMMON-LABEL: define void @test23(
+// CHECK-COMMON: %[[V9:.*]] = call i8* @llvm.objc.retainBlock(
+// CHECK-COMMON: %[[V10:.*]] = bitcast i8* %[[V9]] to void ()*
+// CHECK-COMMON: call void (i8*, i8*, i32, ...) bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to void (i8*, i8*, i32, ...)*)(i8* %{{.*}}, i8* %{{.*}}, i32 123, void ()* %[[V10]])
+
+void test23(id x, Test23 *t) {
+  [t m:123, ^{ (void)x; }];
+}
+
 // CHECK: attributes [[NUW]] = { nounwind }
 // CHECK-UNOPT: attributes [[NUW]] = { nounwind }
diff --git a/clang/test/CodeGenObjC/arc-linetable-autorelease.m b/clang/test/CodeGenObjC/arc-linetable-autorelease.m
index 0804b0700a085..93eeacab902f6 100644
--- a/clang/test/CodeGenObjC/arc-linetable-autorelease.m
+++ b/clang/test/CodeGenObjC/arc-linetable-autorelease.m
@@ -32,8 +32,8 @@ - (NSBezierPath *)_createBezierPathWithWidth:(CGFloat)width height:(CGFloat)heig
   // CHECK: call void @llvm.objc.storeStrong{{.*}} !dbg ![[ARC:[0-9]+]]
   // CHECK: call {{.*}} @llvm.objc.autoreleaseReturnValue{{.*}} !dbg ![[ARC]]
   // CHECK: ret {{.*}} !dbg ![[ARC]]
-  // CHECK: ![[RET]] = !DILocation(line: [[@LINE+1]], scope: !{{.*}})
+  // CHECK: ![[RET]] = !DILocation(line: [[@LINE+1]], column: 10, scope: !{{.*}})
   return path;
-  // CHECK: ![[ARC]] = !DILocation(line: [[@LINE+1]], scope: !{{.*}})
+  // CHECK: ![[ARC]] = !DILocation(line: [[@LINE+1]], column: 1, scope: !{{.*}})
 }
 @end
diff --git a/clang/test/CodeGenObjC/arc-linetable.m b/clang/test/CodeGenObjC/arc-linetable.m
index 3883e0a3a93f2..646136a1cd035 100644
--- a/clang/test/CodeGenObjC/arc-linetable.m
+++ b/clang/test/CodeGenObjC/arc-linetable.m
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -emit-llvm -fblocks -fobjc-arc -debug-info-kind=standalone -dwarf-version=4 -disable-llvm-passes -triple x86_64-apple-darwin10 %s -o - | FileCheck %s
+// RUN: %clang_cc1 -emit-llvm -fblocks -fobjc-arc -debug-info-kind=standalone -dwarf-version=4 -gno-column-info -disable-llvm-passes -triple x86_64-apple-darwin10 %s -o - | FileCheck %s
 
 // Legend: EXP = Return expression, RET = ret instruction
 
diff --git a/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm b/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm
index 7a7781f68891b..d19534f6922dc 100644
--- a/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm
+++ b/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm
@@ -178,3 +178,32 @@ void testParamContainsNonTrivial(ContainsNonTrivial a) {
 void testCallContainsNonTrivial(ContainsNonTrivial *a) {
   testParamContainsNonTrivial(*a);
 }
+
+namespace testThunk {
+
+// CHECK-LABEL: define i64 @_ZThn8_N9testThunk2D02m0Ev(
+// CHECK: %[[RETVAL:.*]] = alloca %[[STRUCT_STRONG]], align 8
+// CHECK: %[[CALL:.*]] = tail call i64 @_ZN9testThunk2D02m0Ev(
+// CHECK: %[[COERCE_DIVE:.*]] = getelementptr inbounds %[[STRUCT_STRONG]], %[[STRUCT_STRONG]]* %[[RETVAL]], i32 0, i32 0
+// CHECK: %[[COERCE_VAL_IP:.*]] = inttoptr i64 %[[CALL]] to i8*
+// CHECK: store i8* %[[COERCE_VAL_IP]], i8** %[[COERCE_DIVE]], align 8
+// CHECK: %[[COERCE_DIVE2:.*]] = getelementptr inbounds %[[STRUCT_STRONG]], %[[STRUCT_STRONG]]* %[[RETVAL]], i32 0, i32 0
+// CHECK: %[[V3:.*]] = load i8*, i8** %[[COERCE_DIVE2]], align 8
+// CHECK: %[[COERCE_VAL_PI:.*]] = ptrtoint i8* %[[V3]] to i64
+// CHECK: ret i64 %[[COERCE_VAL_PI]]
+
+struct B0 {
+  virtual Strong m0();
+};
+
+struct B1 {
+  virtual Strong m0();
+};
+
+struct D0 : B0, B1 {
+  Strong m0() override;
+};
+
+Strong D0::m0() { return {}; }
+
+}
diff --git a/clang/test/CodeGenOpenCL/blocks.cl b/clang/test/CodeGenOpenCL/blocks.cl
index c3e26855dfe01..f8315515f1d9d 100644
--- a/clang/test/CodeGenOpenCL/blocks.cl
+++ b/clang/test/CodeGenOpenCL/blocks.cl
@@ -70,26 +70,6 @@ void foo(){
 
 // COMMON-NOT: define{{.*}}@__foo_block_invoke_kernel
 
-// Test that we support block arguments.
-// COMMON-LABEL: define {{.*}} @blockArgFunc
-int blockArgFunc(int (^ bl)(void)) {
-  return bl();
-}
-
-// COMMON-LABEL: define {{.*}} @get21
-// COMMON: define {{.*}} @__get21_block_invoke
-// COMMON: ret i32 21
-int get21() {
-  return blockArgFunc(^{return 21;});
-}
-
-// COMMON-LABEL: define {{.*}} @get42
-// COMMON: define {{.*}} @__get42_block_invoke
-// COMMON: ret i32 42
-int get42() {
-  return blockArgFunc(^{return 42;});
-}
-
 // COMMON-LABEL: define {{.*}}@call_block
 // call {{.*}}@__call_block_block_invoke
 int call_block() {
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-vi.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-vi.cl
index e3e6b81271d1f..5884f84ab081b 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-vi.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-vi.cl
@@ -22,6 +22,13 @@ void test_rcp_f16(global half* out, half a)
   *out = __builtin_amdgcn_rcph(a);
 }
 
+// CHECK-LABEL: @test_sqrt_f16
+// CHECK: call half @llvm.amdgcn.sqrt.f16
+void test_sqrt_f16(global half* out, half a)
+{
+  *out = __builtin_amdgcn_sqrth(a);
+}
+
 // CHECK-LABEL: @test_rsq_f16
 // CHECK: call half @llvm.amdgcn.rsq.f16
 void test_rsq_f16(global half* out, half a)
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
index 3563ad464c66a..56c83df6b6b41 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
@@ -116,6 +116,20 @@ void test_rcp_f64(global double* out, double a)
   *out = __builtin_amdgcn_rcp(a);
 }
 
+// CHECK-LABEL: @test_sqrt_f32
+// CHECK: call float @llvm.amdgcn.sqrt.f32
+void test_sqrt_f32(global float* out, float a)
+{
+  *out = __builtin_amdgcn_sqrtf(a);
+}
+
+// CHECK-LABEL: @test_sqrt_f64
+// CHECK: call double @llvm.amdgcn.sqrt.f64
+void test_sqrt_f64(global double* out, double a)
+{
+  *out = __builtin_amdgcn_sqrt(a);
+}
+
 // CHECK-LABEL: @test_rsq_f32
 // CHECK: call float @llvm.amdgcn.rsq.f32
 void test_rsq_f32(global float* out, float a)
diff --git a/clang/test/CodeGenOpenCL/convergent.cl b/clang/test/CodeGenOpenCL/convergent.cl
index 193d391ced207..49d182579e4c8 100644
--- a/clang/test/CodeGenOpenCL/convergent.cl
+++ b/clang/test/CodeGenOpenCL/convergent.cl
@@ -70,9 +70,9 @@ void test_merge_if(int a) {
 // CHECK-NOT: call spir_func void @g()
 // CHECK: br label %[[if_end]]
 // CHECK: [[if_end]]:
-// CHECK:  %[[tobool_pr:.+]] = phi i1 [ true, %[[if_then]] ], [ false, %{{.+}} ]
+// CHECK:  %[[tobool_not_pr:.+]] = phi i1 [ true, %{{.+}} ], [ false, %[[if_then]] ]
 // CHECK:  tail call spir_func void @convfun() #[[attr4:.+]]
-// CHECK:  br i1 %[[tobool_pr]], label %[[if_then2:.+]], label %[[if_end3:.+]]
+// CHECK:  br i1 %[[tobool_not_pr]], label %[[if_end3:.+]], label %[[if_then2:.+]]
 // CHECK: [[if_then2]]:
 // CHECK: tail call spir_func void @g()
 // CHECK:  br label %[[if_end3:.+]]
diff --git a/clang/test/CodeGenOpenCL/enqueue-kernel-non-entry-block.cl b/clang/test/CodeGenOpenCL/enqueue-kernel-non-entry-block.cl
index 8b5c5df2e4bae..1e8917b5324bc 100644
--- a/clang/test/CodeGenOpenCL/enqueue-kernel-non-entry-block.cl
+++ b/clang/test/CodeGenOpenCL/enqueue-kernel-non-entry-block.cl
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -cl-std=CL2.0 -O0 -emit-llvm -o - -triple amdgcn < %s | FileCheck %s --check-prefixes=COMMON,AMDGPU
 // RUN: %clang_cc1 -cl-std=CL2.0 -O0 -emit-llvm -o - -triple "spir-unknown-unknown" < %s | FileCheck %s --check-prefixes=COMMON,SPIR32
 // RUN: %clang_cc1 -cl-std=CL2.0 -O0 -emit-llvm -o - -triple "spir64-unknown-unknown" < %s | FileCheck %s --check-prefixes=COMMON,SPIR64
-// RUN: %clang_cc1 -cl-std=CL2.0 -O0 -debug-info-kind=limited -emit-llvm -o - -triple amdgcn < %s | FileCheck %s --check-prefixes=CHECK-DEBUG
+// RUN: %clang_cc1 -cl-std=CL2.0 -O0 -debug-info-kind=limited -gno-column-info -emit-llvm -o - -triple amdgcn < %s | FileCheck %s --check-prefixes=CHECK-DEBUG
 
 // Check that the enqueue_kernel array temporary is in the entry block to avoid
 // a dynamic alloca
diff --git a/clang/test/CodeGenOpenCL/func-call-dbg-loc.cl b/clang/test/CodeGenOpenCL/func-call-dbg-loc.cl
index 4ed082fa9f1ce..117e69ae92dd2 100644
--- a/clang/test/CodeGenOpenCL/func-call-dbg-loc.cl
+++ b/clang/test/CodeGenOpenCL/func-call-dbg-loc.cl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple amdgcn---amdgizcl -debug-info-kind=limited -O0 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn---amdgizcl -debug-info-kind=limited -gno-column-info -O0 -emit-llvm -o - %s | FileCheck %s
 
 typedef struct
 {
diff --git a/clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl b/clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl
index 37090772f6646..259c12384f2c8 100644
--- a/clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl
+++ b/clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl
@@ -33,3 +33,17 @@ kernel void test_agpr() {
          : "={a1}"(reg_a)
          : "{a1}"(reg_b));
 }
+
+kernel void test_constraint_DA() {
+  const long x = 0x200000001;
+  int res;
+  // CHECK: call i32 asm sideeffect "v_mov_b32 $0, $1 & 0xFFFFFFFF", "=v,^DA"(i64 8589934593)
+  __asm volatile("v_mov_b32 %0, %1 & 0xFFFFFFFF" : "=v"(res) : "DA"(x));
+}
+
+kernel void test_constraint_DB() {
+  const long x = 0x200000001;
+  int res;
+  // CHECK: call i32 asm sideeffect "v_mov_b32 $0, $1 & 0xFFFFFFFF", "=v,^DB"(i64 8589934593)
+  __asm volatile("v_mov_b32 %0, %1 & 0xFFFFFFFF" : "=v"(res) : "DB"(x));
+}
diff --git a/clang/test/CodeGenOpenCL/relaxed-fpmath.cl b/clang/test/CodeGenOpenCL/relaxed-fpmath.cl
index ced15f6a8f3dc..1f12392d861ba 100644
--- a/clang/test/CodeGenOpenCL/relaxed-fpmath.cl
+++ b/clang/test/CodeGenOpenCL/relaxed-fpmath.cl
@@ -5,6 +5,17 @@
 // RUN: %clang_cc1 %s -emit-llvm -cl-mad-enable -o - | FileCheck %s -check-prefix=MAD
 // RUN: %clang_cc1 %s -emit-llvm -cl-no-signed-zeros -o - | FileCheck %s -check-prefix=NOSIGNED
 
+// Check the fp options are correct with PCH.
+// RUN: %clang_cc1 %s -DGEN_PCH=1 -finclude-default-header -triple spir-unknown-unknown -emit-pch -o %t.pch
+// RUN: %clang_cc1 %s -include-pch %t.pch -fno-validate-pch -emit-llvm -o - | FileCheck %s -check-prefix=NORMAL
+// RUN: %clang_cc1 %s -include-pch %t.pch -fno-validate-pch -emit-llvm -cl-fast-relaxed-math -o - | FileCheck %s -check-prefix=FAST
+// RUN: %clang_cc1 %s -include-pch %t.pch -fno-validate-pch -emit-llvm -cl-finite-math-only -o - | FileCheck %s -check-prefix=FINITE
+// RUN: %clang_cc1 %s -include-pch %t.pch -fno-validate-pch -emit-llvm -cl-unsafe-math-optimizations -o - | FileCheck %s -check-prefix=UNSAFE
+// RUN: %clang_cc1 %s -include-pch %t.pch -fno-validate-pch -emit-llvm -cl-mad-enable -o - | FileCheck %s -check-prefix=MAD
+// RUN: %clang_cc1 %s -include-pch %t.pch -fno-validate-pch -emit-llvm -cl-no-signed-zeros -o - | FileCheck %s -check-prefix=NOSIGNED
+
+#if !GEN_PCH
+
 float spscalardiv(float a, float b) {
   // CHECK: @spscalardiv(
 
@@ -53,3 +64,8 @@ float spscalardiv(float a, float b) {
 // NOSIGNED: "no-nans-fp-math"="false"
 // NOSIGNED: "no-signed-zeros-fp-math"="true"
 // NOSIGNED: "unsafe-fp-math"="false"
+
+#else
+// Undefine this to avoid putting it in the PCH.
+#undef GEN_PCH
+#endif
diff --git a/clang/test/CoverageMapping/coroutine.cpp b/clang/test/CoverageMapping/coroutine.cpp
new file mode 100644
index 0000000000000..dc9473348fc90
--- /dev/null
+++ b/clang/test/CoverageMapping/coroutine.cpp
@@ -0,0 +1,47 @@
+// fixme: the following line is added to cleanup bots, will be removed in weeks.
+// RUN: rm -f %S/coroutine.ll
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fcoroutines-ts -std=c++14 -emit-llvm -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping %s -o - | FileCheck %s
+
+namespace std::experimental {
+template <typename... T>
+struct coroutine_traits;
+
+template <class Promise = void>
+struct coroutine_handle {
+  coroutine_handle() = default;
+  static coroutine_handle from_address(void *) noexcept { return {}; }
+};
+template <>
+struct coroutine_handle<void> {
+  static coroutine_handle from_address(void *) { return {}; }
+  coroutine_handle() = default;
+  template <class PromiseType>
+  coroutine_handle(coroutine_handle<PromiseType>) noexcept {}
+};
+} // namespace std::experimental
+
+struct suspend_always {
+  bool await_ready() noexcept;
+  void await_suspend(std::experimental::coroutine_handle<>) noexcept;
+  void await_resume() noexcept;
+};
+
+template <>
+struct std::experimental::coroutine_traits<int, int> {
+  struct promise_type {
+    int get_return_object();
+    suspend_always initial_suspend();
+    suspend_always final_suspend() noexcept;
+    void return_value(int);
+  };
+};
+
+// CHECK-LABEL: _Z2f1i:
+int f1(int x) {       // CHECK-NEXT: File 0, [[@LINE]]:15 -> [[@LINE+7]]:2 = #0
+  if (x > 42) {       // CHECK-NEXT: File 0, [[@LINE]]:7 -> [[@LINE]]:13 = #0
+    ++x;              // CHECK-NEXT: Gap,File 0, [[@LINE-1]]:14 -> [[@LINE-1]]:15 = #1
+  } else {            // CHECK-NEXT: File 0, [[@LINE-2]]:15 -> [[@LINE]]:4 = #1
+    co_return x + 42; // CHECK-NEXT: Gap,File 0, [[@LINE-1]]:4 -> [[@LINE-1]]:10 = (#0 - #1)
+  }                   // CHECK-NEXT: File 0, [[@LINE-2]]:10 -> [[@LINE]]:4 = (#0 - #1)
+  co_return x;        // CHECK-NEXT: Gap,File 0, [[@LINE-1]]:4 -> [[@LINE]]:3 = #1
+} // CHECK-NEXT: File 0, [[@LINE-1]]:3 -> [[@LINE]]:2 = #1
diff --git a/clang/test/CoverageMapping/preprocessor.c b/clang/test/CoverageMapping/preprocessor.c
index b3ebc7bd4ec06..9225c9f162a20 100644
--- a/clang/test/CoverageMapping/preprocessor.c
+++ b/clang/test/CoverageMapping/preprocessor.c
@@ -3,7 +3,7 @@
                  // CHECK: func
 void func() {    // CHECK: File 0, [[@LINE]]:13 -> [[@LINE+5]]:2 = #0
   int i = 0;
-#ifdef MACRO     // CHECK-NEXT: Skipped,File 0, [[@LINE]]:1 -> [[@LINE+3]]:1 = 0
+#ifdef MACRO     // CHECK-NEXT: Skipped,File 0, [[@LINE]]:1 -> [[@LINE+2]]:7 = 0
   int x = i;
 #endif
 }
@@ -11,7 +11,7 @@ void func() {    // CHECK: File 0, [[@LINE]]:13 -> [[@LINE+5]]:2 = #0
                  // CHECK: main
 int main() {     // CHECK-NEXT: File 0, [[@LINE]]:12 -> {{[0-9]+}}:2 = #0
   int i = 0;
-#  if 0            // CHECK-NEXT: Skipped,File 0, [[@LINE]]:1 -> [[@LINE+5]]:1 = 0
+#  if 0            // CHECK-NEXT: Skipped,File 0, [[@LINE]]:1 -> [[@LINE+4]]:29 = 0
   if(i == 0) {
     i = 1;
   }
@@ -22,44 +22,44 @@ int main() {     // CHECK-NEXT: File 0, [[@LINE]]:12 -> {{[0-9]+}}:2 = #0
   if(i == 0) {   // CHECK: File 0, [[@LINE]]:14 -> [[@LINE+2]]:4 = #1
     i = 1;
   }
-#else            // CHECK-NEXT: Skipped,File 0, [[@LINE]]:1 -> [[@LINE+6]]:1 = 0
+#else            // CHECK-NEXT: Skipped,File 0, [[@LINE]]:1 -> [[@LINE+5]]:7 = 0
   if(i == 1) {
     i = 0;
   }
 }
 #endif
 
-  // CHECK-NEXT: Skipped,File 0, [[@LINE+1]]:1 -> [[@LINE+5]]:1
+  // CHECK-NEXT: Skipped,File 0, [[@LINE+1]]:1 -> [[@LINE+4]]:24
 #\
   if 0
 #\
   endif // also skipped
 
 #if 1
-  // CHECK-NEXT: Skipped,File 0, [[@LINE+1]]:1 -> [[@LINE+4]]:1
+  // CHECK-NEXT: Skipped,File 0, [[@LINE+1]]:1 -> [[@LINE+3]]:7
 #\
   elif 0
 #endif
 
 #if 1
-  // CHECK-NEXT: Skipped,File 0, [[@LINE+1]]:1 -> [[@LINE+4]]:1
+  // CHECK-NEXT: Skipped,File 0, [[@LINE+1]]:1 -> [[@LINE+3]]:7
 #\
   else
 #endif
 
-  // CHECK-NEXT: Skipped,File 0, [[@LINE+1]]:1 -> [[@LINE+5]]:1
+  // CHECK-NEXT: Skipped,File 0, [[@LINE+1]]:1 -> [[@LINE+4]]:8
 #\
   ifdef NOT_DEFINED
 #\
   endif
 
-  // CHECK-NEXT: Skipped,File 0, [[@LINE+1]]:1 -> [[@LINE+5]]:1
+  // CHECK-NEXT: Skipped,File 0, [[@LINE+1]]:1 -> [[@LINE+4]]:8
 #\
   ifndef __FILE__
 #\
   endif
 
-  // CHECK-NEXT: Skipped,File 0, [[@LINE+1]]:1 -> [[@LINE+7]]:1
+  // CHECK-NEXT: Skipped,File 0, [[@LINE+1]]:1 -> [[@LINE+6]]:26
 #\
   ifdef NOT_DEFINED
 #\
diff --git a/clang/test/Driver/Inputs/rocm/bin/.hipVersion b/clang/test/Driver/Inputs/rocm/bin/.hipVersion
new file mode 100644
index 0000000000000..48ee6f10c3e4e
--- /dev/null
+++ b/clang/test/Driver/Inputs/rocm/bin/.hipVersion
@@ -0,0 +1,4 @@
+# Auto-generated by cmake
+HIP_VERSION_MAJOR=3
+HIP_VERSION_MINOR=6
+HIP_VERSION_PATCH=20214-a2917cd
diff --git a/clang/test/Driver/aarch64-cpus.c b/clang/test/Driver/aarch64-cpus.c
index f774e5c4d8db2..f39241bee8a6d 100644
--- a/clang/test/Driver/aarch64-cpus.c
+++ b/clang/test/Driver/aarch64-cpus.c
@@ -171,6 +171,13 @@
 // ARM64-CORTEX-A76: "-cc1"{{.*}} "-triple" "arm64{{.*}}" "-target-cpu" "cortex-a76"
 // ARM64-CORTEX-A76-TUNE: "-cc1"{{.*}} "-triple" "arm64{{.*}}" "-target-cpu" "generic"
 
+// RUN: %clang -target aarch64 -mcpu=cortex-a77  -### -c %s 2>&1 | FileCheck -check-prefix=CORTEX-A77 %s
+// CORTEX-A77: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "cortex-a77"
+// RUN: %clang -target aarch64 -mcpu=cortex-x1  -### -c %s 2>&1 | FileCheck -check-prefix=CORTEXX1 %s
+// CORTEXX1: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "cortex-x1"
+// RUN: %clang -target aarch64 -mcpu=cortex-a78  -### -c %s 2>&1 | FileCheck -check-prefix=CORTEXA78 %s
+// CORTEXA78: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "cortex-a78"
+
 // RUN: %clang -target aarch64_be -mcpu=exynos-m3 -### -c %s 2>&1 | FileCheck -check-prefix=M3 %s
 // RUN: %clang -target aarch64 -mbig-endian -mcpu=exynos-m3 -### -c %s 2>&1 | FileCheck -check-prefix=M3 %s
 // RUN: %clang -target aarch64_be -mbig-endian -mcpu=exynos-m3 -### -c %s 2>&1 | FileCheck -check-prefix=M3 %s
diff --git a/clang/test/Driver/aix-object-mode.c b/clang/test/Driver/aix-object-mode.c
new file mode 100644
index 0000000000000..a4bc79a4b41d5
--- /dev/null
+++ b/clang/test/Driver/aix-object-mode.c
@@ -0,0 +1,22 @@
+// Check that setting an OBJECT_MODE converts the AIX triple to the right variant.
+// RUN: env OBJECT_MODE=64 \
+// RUN: %clang -target powerpc-ibm-aix -print-target-triple | FileCheck -check-prefix=CHECK64 %s
+
+// RUN: env OBJECT_MODE=32 \
+// RUN: %clang -target powerpc64-ibm-aix -print-target-triple | FileCheck -check-prefix=CHECK32 %s
+
+// Command-line options win.
+// RUN: env OBJECT_MODE=64 \
+// RUN: %clang -target powerpc64-ibm-aix -print-target-triple -m32 | FileCheck -check-prefix=CHECK32 %s
+
+// RUN: env OBJECT_MODE=32 \
+// RUN: %clang -target powerpc-ibm-aix -print-target-triple -m64 | FileCheck -check-prefix=CHECK64 %s
+
+// CHECK32: powerpc-ibm-aix
+// CHECK64: powerpc64-ibm-aix
+
+// Emit a diagnostic if there is an invalid mode.
+// RUN: env OBJECT_MODE=31 \
+// RUN: not %clang -target powerpc-ibm-aix 2>&1 | FileCheck -check-prefix=DIAG %s
+
+// DIAG: error: OBJECT_MODE setting 31 is not recognized and is not a valid setting.
diff --git a/clang/test/Driver/aix-toolchain-include.cpp b/clang/test/Driver/aix-toolchain-include.cpp
new file mode 100644
index 0000000000000..dc8b272936a48
--- /dev/null
+++ b/clang/test/Driver/aix-toolchain-include.cpp
@@ -0,0 +1,136 @@
+// Tests that the AIX toolchain adds system includes to its search path.
+
+// Check powerpc-ibm-aix, 32-bit/64-bit.
+// RUN: %clangxx -### -no-canonical-prefixes %s 2>&1 \
+// RUN:		-target powerpc-ibm-aix \
+// RUN:		-resource-dir=%S/Inputs/resource_dir \
+// RUN:		--sysroot=%S/Inputs/basic_aix_tree \
+// RUN:   | FileCheck -check-prefix=CHECK-INTERNAL-INCLUDE %s
+
+// RUN: %clangxx -### -no-canonical-prefixes %s 2>&1 \
+// RUN:		-target powerpc64-ibm-aix \
+// RUN:		-resource-dir=%S/Inputs/resource_dir \
+// RUN:		--sysroot=%S/Inputs/basic_aix_tree \
+// RUN:   | FileCheck -check-prefix=CHECK-INTERNAL-INCLUDE %s
+
+// RUN: %clang -### -xc -no-canonical-prefixes %s 2>&1 \
+// RUN:		-target powerpc-ibm-aix \
+// RUN:		-resource-dir=%S/Inputs/resource_dir \
+// RUN:		--sysroot=%S/Inputs/basic_aix_tree \
+// RUN:   | FileCheck -check-prefix=CHECK-INTERNAL-INCLUDE %s
+
+// RUN: %clang -### -xc -no-canonical-prefixes %s 2>&1 \
+// RUN:		-target powerpc64-ibm-aix \
+// RUN:		-resource-dir=%S/Inputs/resource_dir \
+// RUN:		--sysroot=%S/Inputs/basic_aix_tree \
+// RUN:   | FileCheck -check-prefix=CHECK-INTERNAL-INCLUDE %s
+
+// CHECK-INTERNAL-INCLUDE:	{{.*}}clang{{.*}}" "-cc1"
+// CHECK-INTERNAL-INCLUDE:	"-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
+// CHECK-INTERNAL-INCLUDE:	"-isysroot" "[[SYSROOT:[^"]+]]"
+// CHECK-INTERNAL-INCLUDE:	"-internal-isystem" "[[RESOURCE_DIR]]{{(/|\\\\)}}include"
+// CHECK-INTERNAL-INCLUDE:	"-internal-isystem" "[[SYSROOT]]/usr/include"
+
+// Check powerpc-ibm-aix, 32-bit/64-bit. -nostdinc option.
+// RUN: %clangxx -### -no-canonical-prefixes %s 2>&1 \
+// RUN:		-target powerpc-ibm-aix \
+// RUN:		-resource-dir=%S/Inputs/resource_dir \
+// RUN:		--sysroot=%S/Inputs/basic_aix_tree \
+// RUN:		-nostdinc \
+// RUN:   | FileCheck -check-prefix=CHECK-NOSTDINC-INCLUDE %s
+
+// RUN: %clangxx -### -no-canonical-prefixes %s 2>&1 \
+// RUN:		-target powerpc64-ibm-aix \
+// RUN:		-resource-dir=%S/Inputs/resource_dir \
+// RUN:		--sysroot=%S/Inputs/basic_aix_tree \
+// RUN:		-nostdinc \
+// RUN:   | FileCheck -check-prefix=CHECK-NOSTDINC-INCLUDE %s
+
+// RUN: %clang -### -xc -no-canonical-prefixes %s 2>&1 \
+// RUN:		-target powerpc-ibm-aix \
+// RUN:		-resource-dir=%S/Inputs/resource_dir \
+// RUN:		--sysroot=%S/Inputs/basic_aix_tree \
+// RUN:		-nostdinc \
+// RUN:   | FileCheck -check-prefix=CHECK-NOSTDINC-INCLUDE %s
+
+// RUN: %clang -### -xc -no-canonical-prefixes %s 2>&1 \
+// RUN:		-target powerpc64-ibm-aix \
+// RUN:		-resource-dir=%S/Inputs/resource_dir \
+// RUN:		--sysroot=%S/Inputs/basic_aix_tree \
+// RUN:		-nostdinc \
+// RUN:   | FileCheck -check-prefix=CHECK-NOSTDINC-INCLUDE %s
+
+// CHECK-NOSTDINC-INCLUDE:	{{.*}}clang{{.*}}" "-cc1"
+// CHECK-NOSTDINC-INCLUDE:	"-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
+// CHECK-NOSTDINC-INCLUDE:	"-isysroot" "[[SYSROOT:[^"]+]]"
+// CHECK-NOSTDINC-INCLUDE-NOT:	"-internal-isystem"
+
+// Check powerpc-ibm-aix, 32-bit/64-bit. -nostdlibinc option.
+// RUN: %clangxx -### -no-canonical-prefixes %s 2>&1 \
+// RUN:		-target powerpc-ibm-aix \
+// RUN:		-resource-dir=%S/Inputs/resource_dir \
+// RUN:		--sysroot=%S/Inputs/basic_aix_tree \
+// RUN:		-nostdlibinc \
+// RUN:   | FileCheck -check-prefix=CHECK-NOSTDLIBINC-INCLUDE %s
+
+// RUN: %clangxx -### -no-canonical-prefixes %s 2>&1 \
+// RUN:		-target powerpc64-ibm-aix \
+// RUN:		-resource-dir=%S/Inputs/resource_dir \
+// RUN:		--sysroot=%S/Inputs/basic_aix_tree \
+// RUN:		-nostdlibinc \
+// RUN:   | FileCheck -check-prefix=CHECK-NOSTDLIBINC-INCLUDE %s
+
+// RUN: %clang -### -xc -no-canonical-prefixes %s 2>&1 \
+// RUN:		-target powerpc-ibm-aix \
+// RUN:		-resource-dir=%S/Inputs/resource_dir \
+// RUN:		--sysroot=%S/Inputs/basic_aix_tree \
+// RUN:		-nostdlibinc \
+// RUN:   | FileCheck -check-prefix=CHECK-NOSTDLIBINC-INCLUDE %s
+
+// RUN: %clang -### -xc -no-canonical-prefixes %s 2>&1 \
+// RUN:		-target powerpc64-ibm-aix \
+// RUN:		-resource-dir=%S/Inputs/resource_dir \
+// RUN:		--sysroot=%S/Inputs/basic_aix_tree \
+// RUN:		-nostdlibinc \
+// RUN:   | FileCheck -check-prefix=CHECK-NOSTDLIBINC-INCLUDE %s
+
+// CHECK-NOSTDLIBINC-INCLUDE:	{{.*}}clang{{.*}}" "-cc1"
+// CHECK-NOSTDLIBINC-INCLUDE:	"-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
+// CHECK-NOSTDLIBINC-INCLUDE:	"-isysroot" "[[SYSROOT:[^"]+]]"
+// CHECK-NOSTDLIBINC-INCLUDE:	"-internal-isystem" "[[RESOURCE_DIR]]{{(/|\\\\)}}include"
+// CHECK-NOSTDLIBINC-INCLUDE-NOT:	"-internal-isystem" "[[SYSROOT]]/usr/include"
+
+// Check powerpc-ibm-aix, 32-bit/64-bit. -nobuiltininc option.
+// RUN: %clangxx -### -no-canonical-prefixes %s 2>&1 \
+// RUN:		-target powerpc-ibm-aix \
+// RUN:		-resource-dir=%S/Inputs/resource_dir \
+// RUN:		--sysroot=%S/Inputs/basic_aix_tree \
+// RUN:		-nobuiltininc \
+// RUN:   | FileCheck -check-prefix=CHECK-NOBUILTININC-INCLUDE %s
+
+// RUN: %clangxx -### -no-canonical-prefixes %s 2>&1 \
+// RUN:		-target powerpc64-ibm-aix \
+// RUN:		-resource-dir=%S/Inputs/resource_dir \
+// RUN:		--sysroot=%S/Inputs/basic_aix_tree \
+// RUN:		-nobuiltininc \
+// RUN:   | FileCheck -check-prefix=CHECK-NOBUILTININC-INCLUDE %s
+
+// RUN: %clang -### -xc -no-canonical-prefixes %s 2>&1 \
+// RUN:		-target powerpc-ibm-aix \
+// RUN:		-resource-dir=%S/Inputs/resource_dir \
+// RUN:		--sysroot=%S/Inputs/basic_aix_tree \
+// RUN:		-nobuiltininc \
+// RUN:   | FileCheck -check-prefix=CHECK-NOBUILTININC-INCLUDE %s
+
+// RUN: %clang -### -xc -no-canonical-prefixes %s 2>&1 \
+// RUN:		-target powerpc64-ibm-aix \
+// RUN:		-resource-dir=%S/Inputs/resource_dir \
+// RUN:		--sysroot=%S/Inputs/basic_aix_tree \
+// RUN:		-nobuiltininc \
+// RUN:   | FileCheck -check-prefix=CHECK-NOBUILTININC-INCLUDE %s
+
+// CHECK-NOBUILTININC-INCLUDE:	{{.*}}clang{{.*}}" "-cc1"
+// CHECK-NOBUILTININC-INCLUDE:	"-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
+// CHECK-NOBUILTININC-INCLUDE:	"-isysroot" "[[SYSROOT:[^"]+]]"
+// CHECK-NOBUILTININC-INCLUDE-NOT:	"-internal-isystem" "[[RESOURCE_DIR]]{{(/|\\\\)}}include"
+// CHECK-NOBUILTININC-INCLUDE:	"-internal-isystem" "[[SYSROOT]]/usr/include"
diff --git a/clang/test/Driver/apple-arm64-arch.c b/clang/test/Driver/apple-arm64-arch.c
new file mode 100644
index 0000000000000..a37346b1a9bb0
--- /dev/null
+++ b/clang/test/Driver/apple-arm64-arch.c
@@ -0,0 +1,7 @@
+// RUN: env SDKROOT="/" %clang -arch arm64 -c -### %s 2>&1 | \
+// RUN:   FileCheck %s
+//
+// REQUIRES: system-darwin
+// XFAIL: apple-silicon-mac
+//
+// CHECK: "-triple" "arm64-apple-ios{{[0-9.]+}}"
diff --git a/clang/test/Driver/apple-silicon-arch.c b/clang/test/Driver/apple-silicon-arch.c
new file mode 100644
index 0000000000000..b1201fa2d7ddb
--- /dev/null
+++ b/clang/test/Driver/apple-silicon-arch.c
@@ -0,0 +1,6 @@
+// RUN: env SDKROOT="/" %clang -arch arm64 -c -### %s 2>&1 | \
+// RUN:   FileCheck %s
+//
+// REQUIRES: apple-silicon-mac
+//
+// CHECK: "-triple" "arm64-apple-macosx{{[0-9.]+}}"
diff --git a/clang/test/Driver/arclite-link.c b/clang/test/Driver/arclite-link.c
index 2cc0271fc0222..a53b12daa47bd 100644
--- a/clang/test/Driver/arclite-link.c
+++ b/clang/test/Driver/arclite-link.c
@@ -15,3 +15,6 @@
 // RUN: %clang -### -target x86_64-apple-darwin10 -fobjc-link-runtime -fobjc-arc -mmacosx-version-min=10.10 %s 2>&1 | FileCheck -check-prefix=CHECK-UNUSED %s
 
 // CHECK-UNUSED-NOT: warning: argument unused during compilation: '-fobjc-link-runtime'
+
+// RUN: %clang -### -target arm64-apple-macos10.8 -fobjc-link-runtime %t.o 2>&1 | FileCheck -check-prefix=CHECK-ARCLITE-ARM-MAC %s
+// CHECK-ARCLITE-ARM-MAC-NOT: libarclite
diff --git a/clang/test/Driver/arm-cortex-cpus.c b/clang/test/Driver/arm-cortex-cpus.c
index c1ce9c08ca874..6de1040e94204 100644
--- a/clang/test/Driver/arm-cortex-cpus.c
+++ b/clang/test/Driver/arm-cortex-cpus.c
@@ -686,10 +686,12 @@
 // RUN: %clang -target arm -mcpu=cortex-a75 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A %s
 // RUN: %clang -target arm -mcpu=cortex-a76 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A %s
 // RUN: %clang -target arm -mcpu=cortex-a76ae -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A %s
+// RUN: %clang -target arm -mcpu=cortex-a77 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A %s
 // RUN: %clang -target arm -mcpu=cortex-a55 -mlittle-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A %s
 // RUN: %clang -target arm -mcpu=cortex-a75 -mlittle-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A %s
 // RUN: %clang -target arm -mcpu=cortex-a76 -mlittle-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A %s
 // RUN: %clang -target arm -mcpu=cortex-a76ae -mlittle-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A %s
+// RUN: %clang -target arm -mcpu=cortex-a77 -mlittle-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A %s
 //
 // RUN: %clang -target arm -mcpu=exynos-m4 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A %s
 // RUN: %clang -target arm -mcpu=exynos-m4 -mlittle-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A %s
@@ -718,10 +720,12 @@
 // RUN: %clang -target armeb -mcpu=cortex-a75 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A %s
 // RUN: %clang -target armeb -mcpu=cortex-a76 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A %s
 // RUN: %clang -target armeb -mcpu=cortex-a76ae -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A %s
+// RUN: %clang -target armeb -mcpu=cortex-a77 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A %s
 // RUN: %clang -target arm -mcpu=cortex-a55 -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A %s
 // RUN: %clang -target arm -mcpu=cortex-a75 -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A %s
 // RUN: %clang -target arm -mcpu=cortex-a76 -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A %s
 // RUN: %clang -target arm -mcpu=cortex-a76ae -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A %s
+// RUN: %clang -target arm -mcpu=cortex-a77 -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A %s
 //
 // RUN: %clang -target armeb -mcpu=exynos-m4 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A %s
 // RUN: %clang -target arm -mcpu=exynos-m4 -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A %s
@@ -753,10 +757,12 @@
 // RUN: %clang -target arm -mcpu=cortex-a75 -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A-THUMB %s
 // RUN: %clang -target arm -mcpu=cortex-a76 -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A-THUMB %s
 // RUN: %clang -target arm -mcpu=cortex-a76ae -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A-THUMB %s
+// RUN: %clang -target arm -mcpu=cortex-a77 -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A-THUMB %s
 // RUN: %clang -target arm -mcpu=cortex-a55 -mlittle-endian -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A-THUMB %s
 // RUN: %clang -target arm -mcpu=cortex-a75 -mlittle-endian -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A-THUMB %s
 // RUN: %clang -target arm -mcpu=cortex-a76 -mlittle-endian -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A-THUMB %s
 // RUN: %clang -target arm -mcpu=cortex-a76ae -mlittle-endian -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A-THUMB %s
+// RUN: %clang -target arm -mcpu=cortex-a77 -mlittle-endian -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A-THUMB %s
 //
 // RUN: %clang -target arm -mcpu=exynos-m4 -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A-THUMB %s
 // RUN: %clang -target arm -mcpu=exynos-m4 -mlittle-endian -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV82A-THUMB %s
@@ -785,10 +791,12 @@
 // RUN: %clang -target armeb -mcpu=cortex-a75 -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A-THUMB %s
 // RUN: %clang -target armeb -mcpu=cortex-a76 -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A-THUMB %s
 // RUN: %clang -target armeb -mcpu=cortex-a76ae -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A-THUMB %s
+// RUN: %clang -target armeb -mcpu=cortex-a77 -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A-THUMB %s
 // RUN: %clang -target arm -mcpu=cortex-a55 -mbig-endian -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A-THUMB %s
 // RUN: %clang -target arm -mcpu=cortex-a75 -mbig-endian -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A-THUMB %s
 // RUN: %clang -target arm -mcpu=cortex-a76 -mbig-endian -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A-THUMB %s
 // RUN: %clang -target arm -mcpu=cortex-a76ae -mbig-endian -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A-THUMB %s
+// RUN: %clang -target arm -mcpu=cortex-a77 -mbig-endian -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A-THUMB %s
 //
 // RUN: %clang -target armeb -mcpu=exynos-m4 -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A-THUMB %s
 // RUN: %clang -target arm -mcpu=exynos-m4 -mbig-endian -mthumb -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-CPUV82A-THUMB %s
@@ -832,6 +840,18 @@
 // CHECK-CORTEX-A76AE-SOFT: "-target-feature" "+soft-float"
 // CHECK-CORTEX-A76AE-SOFT: "-target-feature" "+soft-float-abi"
 
+// RUN: %clang -target armv8a-arm-none-eabi -mcpu=cortex-x1 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-X1 %s
+// RUN: %clang -target armv8a-arm-none-eabi -mcpu=cortex-x1 -mfpu=crypto-neon-fp-armv8 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-X1-MFPU %s
+// CHECK-CORTEX-X1: "-cc1"{{.*}} "-triple" "armv8.2a-{{.*}} "-target-cpu" "cortex-x1"
+// CHECK-CORTEX-X1-MFPU: "-cc1"{{.*}} "-target-feature" "+fp-armv8"
+// CHECK-CORTEX-X1-MFPU: "-target-feature" "+crypto"
+
+// RUN: %clang -target armv8a-arm-none-eabi -mcpu=cortex-a78 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-A78 %s
+// RUN: %clang -target armv8a-arm-none-eabi -mcpu=cortex-a78 -mfpu=crypto-neon-fp-armv8 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-A78-MFPU %s
+// CHECK-CORTEX-A78: "-cc1"{{.*}} "-triple" "armv8.2a-{{.*}} "-target-cpu" "cortex-a78"
+// CHECK-CORTEX-A78-MFPU: "-cc1"{{.*}} "-target-feature" "+fp-armv8"
+// CHECK-CORTEX-A78-MFPU: "-target-feature" "+crypto"
+
 // RUN: %clang -target arm -mcpu=cortex-m23 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CPUV8MBASE %s
 // CHECK-CPUV8MBASE:  "-cc1"{{.*}} "-triple" "thumbv8m.base-
 
diff --git a/clang/test/Driver/bindings.c b/clang/test/Driver/bindings.c
index 880667b739348..d7f323419d39e 100644
--- a/clang/test/Driver/bindings.c
+++ b/clang/test/Driver/bindings.c
@@ -23,3 +23,7 @@
 // CHECK14: "clang", inputs: ["{{.*}}bindings.c"], output: "{{.*}}.s"
 // CHECK14: "darwin::Assembler", inputs: ["{{.*}}.s"], output: "{{.*}}.o"
 // CHECK14: "darwin::Linker", inputs: ["{{.*}}.o"], output: "a.out"
+
+// GNU StaticLibTool binding
+// RUN: %clang -target x86_64-linux-gnu -ccc-print-bindings --emit-static-lib %s 2>&1 | FileCheck %s --check-prefix=CHECK15
+// CHECK15: "x86_64-unknown-linux-gnu" - "GNU::StaticLibTool", inputs: ["{{.*}}.o"], output: "a.out"
diff --git a/clang/test/Driver/cl-options.c b/clang/test/Driver/cl-options.c
index 848a532b24f93..0dcaf61088069 100644
--- a/clang/test/Driver/cl-options.c
+++ b/clang/test/Driver/cl-options.c
@@ -56,15 +56,6 @@
 // fpstrict-NOT: -menable-unsafe-fp-math
 // fpstrict-NOT: -ffast-math
 
-// RUN: %clang_cl /Z7 -gcolumn-info -### -- %s 2>&1 | FileCheck -check-prefix=gcolumn %s
-// gcolumn: -dwarf-column-info
-
-// RUN: %clang_cl /Z7 -gno-column-info -### -- %s 2>&1 | FileCheck -check-prefix=gnocolumn %s
-// gnocolumn-NOT: -dwarf-column-info
-
-// RUN: %clang_cl /Z7 -### -- %s 2>&1 | FileCheck -check-prefix=gdefcolumn %s
-// gdefcolumn-NOT: -dwarf-column-info
-
 // RUN: %clang_cl -### /FA -fprofile-instr-generate -- %s 2>&1 | FileCheck -check-prefix=CHECK-PROFILE-INSTR-GENERATE %s
 // RUN: %clang_cl -### /FA -fprofile-instr-generate=/tmp/somefile.profraw -- %s 2>&1 | FileCheck -check-prefix=CHECK-PROFILE-INSTR-GENERATE-FILE %s
 // CHECK-PROFILE-INSTR-GENERATE: "-fprofile-instrument=clang" "--dependent-lib=clang_rt.profile-{{[^"]*}}.lib"
@@ -533,11 +524,11 @@
 
 // RUN: %clang_cl /Zi /c -### -- %s 2>&1 | FileCheck -check-prefix=Zi %s
 // Zi: "-gcodeview"
-// Zi: "-debug-info-kind=limited"
+// Zi: "-debug-info-kind=constructor"
 
 // RUN: %clang_cl /Z7 /c -### -- %s 2>&1 | FileCheck -check-prefix=Z7 %s
 // Z7: "-gcodeview"
-// Z7: "-debug-info-kind=limited"
+// Z7: "-debug-info-kind=constructor"
 
 // RUN: %clang_cl /Zd /c -### -- %s 2>&1 | FileCheck -check-prefix=Z7GMLT %s
 // Z7GMLT: "-gcodeview"
@@ -566,7 +557,7 @@
 // which made it "win". This test could not detect that bug.
 // RUN: %clang_cl /Z7 -gdwarf /c -### -- %s 2>&1 | FileCheck -check-prefix=Z7_gdwarf %s
 // Z7_gdwarf: "-gcodeview"
-// Z7_gdwarf: "-debug-info-kind=limited"
+// Z7_gdwarf: "-debug-info-kind=constructor"
 // Z7_gdwarf: "-dwarf-version=4"
 
 // RUN: %clang_cl -fmsc-version=1800 -TP -### -- %s 2>&1 | FileCheck -check-prefix=CXX11 %s
diff --git a/clang/test/Driver/clang-g-opts.c b/clang/test/Driver/clang-g-opts.c
index bc714b6c93791..60c97790b7dae 100644
--- a/clang/test/Driver/clang-g-opts.c
+++ b/clang/test/Driver/clang-g-opts.c
@@ -31,7 +31,7 @@
 // RUN:             | FileCheck --check-prefix=CHECK-WITH-G-DWARF2 %s
 
 // CHECK-WITHOUT-G-NOT: -debug-info-kind
-// CHECK-WITH-G: "-debug-info-kind=limited"
+// CHECK-WITH-G: "-debug-info-kind=constructor"
 // CHECK-WITH-G: "-dwarf-version=4"
 // CHECK-WITH-G-DWARF2: "-dwarf-version=2"
 
diff --git a/clang/test/Driver/codeview-column-info.c b/clang/test/Driver/codeview-column-info.c
index 6b524accd7b70..4cabefac06e64 100644
--- a/clang/test/Driver/codeview-column-info.c
+++ b/clang/test/Driver/codeview-column-info.c
@@ -1,4 +1,4 @@
-// Check that -dwarf-column-info does not get added to the cc1 line:
+// Check that -gno-column-info gets added to the cc1 line:
 // 1) When -gcodeview is present via the clang or clang++ driver
 // 2) When /Z7 is present via the cl driver.
 
@@ -11,5 +11,8 @@
 // RUN: %clang_cl -### --target=x86_64-windows-msvc /c /Z7 -- %s 2> %t2
 // RUN: FileCheck < %t2 %s
 
-// CHECK: "-cc1"
-// CHECK-NOT: "-dwarf-column-info"
+// CHECK: "-gno-column-info"
+
+// RUN: %clang_cl -### /Z7 -gcolumn-info -- %s 2>&1 | FileCheck --check-prefix=COLUMN %s
+
+// COLUMN-NOT: "-gno-column-info"
diff --git a/clang/test/Driver/crash-report-crashfile.m b/clang/test/Driver/crash-report-crashfile.m
index 980b1acd9fed7..fd26246073faa 100644
--- a/clang/test/Driver/crash-report-crashfile.m
+++ b/clang/test/Driver/crash-report-crashfile.m
@@ -18,6 +18,7 @@
 const int x = MODULE_MACRO;
 
 // CRASH_ENV: failing because environment variable 'FORCE_CLANG_DIAGNOSTICS_CRASH' is set
+// CRASH_ENV: PLEASE submit a bug report to {{.*}} and include the crash backtrace, preprocessed source, and associated run script.
 // CRASH_ENV: Preprocessed source(s) and associated run script(s) are located at:
 // CRASH_ENV-NEXT: note: diagnostic msg: {{.*}}.m
 // CRASH_ENV-NEXT: note: diagnostic msg: {{.*}}.cache
diff --git a/clang/test/Driver/crash-report-modules.m b/clang/test/Driver/crash-report-modules.m
index ded31b4ed4fed..e6d0335337910 100644
--- a/clang/test/Driver/crash-report-modules.m
+++ b/clang/test/Driver/crash-report-modules.m
@@ -19,6 +19,7 @@
 @import simple;
 const int x = MODULE_MACRO;
 
+// CHECK: PLEASE submit a bug report to {{.*}} and include the crash backtrace, preprocessed source, and associated run script.
 // CHECK: Preprocessed source(s) and associated run script(s) are located at:
 // CHECK-NEXT: note: diagnostic msg: {{.*}}.m
 // CHECK-NEXT: note: diagnostic msg: {{.*}}.cache
diff --git a/clang/test/Driver/crash-report-null.test b/clang/test/Driver/crash-report-null.test
index 7281f5309eb33..05309bf63f275 100644
--- a/clang/test/Driver/crash-report-null.test
+++ b/clang/test/Driver/crash-report-null.test
@@ -3,5 +3,6 @@
 // FIXME: Investigating. "fatal error: file 'nul' modified since it was first processed"
 // XFAIL: windows-gnu
 
+// CHECK: PLEASE submit a bug report to {{.*}} and include the crash backtrace, preprocessed source, and associated run script.
 // CHECK: Preprocessed source(s) and associated run script(s) are located at:
 // CHECK-NEXT: note: diagnostic msg: {{.*}}null-{{.*}}.c
diff --git a/clang/test/Driver/cuda-dwarf-2.cu b/clang/test/Driver/cuda-dwarf-2.cu
index bcfb2444bc516..92b8919729fc4 100644
--- a/clang/test/Driver/cuda-dwarf-2.cu
+++ b/clang/test/Driver/cuda-dwarf-2.cu
@@ -49,7 +49,7 @@
 
 // HAS_DEBUG-NOT: warning: debug
 // HAS_DEBUG: "-fcuda-is-device"
-// HAS_DEBUG-SAME: "-debug-info-kind={{limited|line-tables-only}}"
+// HAS_DEBUG-SAME: "-debug-info-kind={{constructor|line-tables-only}}"
 // HAS_DEBUG-SAME: "-dwarf-version=2"
 // HAS_DEBUG: ptxas
 // HAS_DEBUG-SAME: "-g"
diff --git a/clang/test/Driver/darwin-ld-platform-version-macos.c b/clang/test/Driver/darwin-ld-platform-version-macos.c
index c1b940a3f35f3..d3f49093ef868 100644
--- a/clang/test/Driver/darwin-ld-platform-version-macos.c
+++ b/clang/test/Driver/darwin-ld-platform-version-macos.c
@@ -7,9 +7,22 @@
 // RUN: env SDKROOT=%S/Inputs/MacOSX10.14.sdk %clang -target x86_64-apple-macos10.13.0.1 -mlinker-version=520 -### %t.o 2>&1 \
 // RUN:   | FileCheck --check-prefix=LINKER-NEW %s
 
+// RUN: %clang -target arm64-apple-macos10.13 -isysroot %S/Inputs/MacOSX10.14.sdk -mlinker-version=520 -### %t.o 2>&1 \
+// RUN:   | FileCheck --check-prefix=ARM64_NEW %s
+// RUN: %clang -target arm64-apple-darwin19 -isysroot %S/Inputs/MacOSX10.14.sdk -mlinker-version=520 -### %t.o 2>&1 \
+// RUN:   | FileCheck --check-prefix=ARM64_NEW %s
+// RUN: %clang -target arm64-apple-macos11.1 -isysroot %S/Inputs/MacOSX10.14.sdk -mlinker-version=520 -### %t.o 2>&1 \
+// RUN:   | FileCheck --check-prefix=ARM64_NEW_1 %s
+// RUN: %clang -target arm64-apple-macos10.13 -isysroot %S/Inputs/MacOSX10.14.sdk -mlinker-version=400 -### %t.o 2>&1 \
+// RUN:   | FileCheck --check-prefix=ARM64_OLD %s
+
 // LINKER-OLD: "-macosx_version_min" "10.13.0"
 // LINKER-NEW: "-platform_version" "macos" "10.13.0" "10.14"
 
+// ARM64_NEW: "-platform_version" "macos" "11.0.0" "10.14"
+// ARM64_NEW_1: "-platform_version" "macos" "11.1.0" "10.14"
+// ARM64_OLD: "-macosx_version_min" "11.0.0"
+
 // RUN: %clang -target x86_64-apple-macos10.13  -mlinker-version=520 -### %t.o 2>&1 \
 // RUN:   | FileCheck --check-prefix=NOSDK %s
 // NOSDK: "-platform_version" "macos" "10.13.0" "0.0.0"
diff --git a/clang/test/Driver/darwin-sdk-vs-os-version.c b/clang/test/Driver/darwin-sdk-vs-os-version.c
index 391f4d5a73052..94e52a9036ede 100644
--- a/clang/test/Driver/darwin-sdk-vs-os-version.c
+++ b/clang/test/Driver/darwin-sdk-vs-os-version.c
@@ -2,9 +2,9 @@
 
 // Ensure that we never pick a version that's based on the SDK that's newer than
 // the system version:
-// RUN: rm -rf %t/SDKs/MacOSX10.99.99.sdk
-// RUN: mkdir -p %t/SDKs/MacOSX10.99.99.sdk
-// RUN: %clang -target x86_64-apple-darwin -isysroot %t/SDKs/MacOSX10.99.99.sdk %s -### 2>&1 \
+// RUN: rm -rf %t/SDKs/MacOSX99.99.99.sdk
+// RUN: mkdir -p %t/SDKs/MacOSX99.99.99.sdk
+// RUN: %clang -target x86_64-apple-darwin -isysroot %t/SDKs/MacOSX99.99.99.sdk %s -### 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-MACOSX-SYSTEM-VERSION %s
 
-// CHECK-MACOSX-SYSTEM-VERSION-NOT: 10.99.99"
+// CHECK-MACOSX-SYSTEM-VERSION-NOT: 99.99.99"
diff --git a/clang/test/Driver/darwin-version.c b/clang/test/Driver/darwin-version.c
index 7885b59646266..3471552c937f2 100644
--- a/clang/test/Driver/darwin-version.c
+++ b/clang/test/Driver/darwin-version.c
@@ -305,3 +305,13 @@
 // RUN: %clang -target armv7k-apple-ios10.1-simulator -c %s -### 2>&1 | \
 // RUN:   FileCheck --check-prefix=CHECK-VERSION-TENV-SIM2 %s
 // CHECK-VERSION-TENV-SIM2: "thumbv7k-apple-ios10.1.0-simulator"
+
+
+// RUN: %clang -target x86_64-apple-macos11 -c %s -### 2>&1 | \
+// RUN:   FileCheck --check-prefix=CHECK-MACOS11 %s
+// RUN: %clang -target x86_64-apple-darwin20 -c %s -### 2>&1 | \
+// RUN:   FileCheck --check-prefix=CHECK-MACOS11 %s
+// RUN: %clang -target x86_64-apple-darwin -mmacos-version-min=11 -c %s -### 2>&1 | \
+// RUN:   FileCheck --check-prefix=CHECK-MACOS11 %s
+
+// CHECK-MACOS11: "x86_64-apple-macosx11.0.0"
diff --git a/clang/test/Driver/darwin-warning-options.c b/clang/test/Driver/darwin-warning-options.c
new file mode 100644
index 0000000000000..b0a591eac8204
--- /dev/null
+++ b/clang/test/Driver/darwin-warning-options.c
@@ -0,0 +1,7 @@
+// REQUIRES: system-darwin
+
+// Always error about undefined 'TARGET_OS_*' macros on Darwin.
+// RUN: %clang -### %s 2>&1 | FileCheck %s
+
+// CHECK-DAG: "-Wundef-prefix=TARGET_OS_"
+// CHECK-DAG: "-Werror=undef-prefix"
diff --git a/clang/test/Driver/debug-options-as.c b/clang/test/Driver/debug-options-as.c
index 51475680e9b18..4808219702e76 100644
--- a/clang/test/Driver/debug-options-as.c
+++ b/clang/test/Driver/debug-options-as.c
@@ -23,7 +23,7 @@
 // RUN:   | FileCheck %s
 //
 // CHECK: "-cc1as"
-// CHECK: "-debug-info-kind=limited"
+// CHECK: "-debug-info-kind=constructor"
 
 // Check to make sure clang with -g on a .s file gets passed -dwarf-debug-producer.
 // rdar://12955296
diff --git a/clang/test/Driver/debug-options.c b/clang/test/Driver/debug-options.c
index 1dd95db4b92ab..2d1a0b2d5cd8f 100644
--- a/clang/test/Driver/debug-options.c
+++ b/clang/test/Driver/debug-options.c
@@ -274,18 +274,18 @@
 // GLIO_ONLY_DWARF2: "-dwarf-version=2"
 //
 // G_ONLY: "-cc1"
-// G_ONLY: "-debug-info-kind=limited"
+// G_ONLY: "-debug-info-kind=constructor"
 //
 // These tests assert that "-gline-tables-only" "-g" uses the latter,
 // but otherwise not caring about the DebugInfoKind.
 // G_ONLY_DWARF2: "-cc1"
-// G_ONLY_DWARF2: "-debug-info-kind={{standalone|limited}}"
+// G_ONLY_DWARF2: "-debug-info-kind={{standalone|constructor}}"
 // G_ONLY_DWARF2: "-dwarf-version=2"
 //
 // G_STANDALONE: "-cc1"
 // G_STANDALONE: "-debug-info-kind=standalone"
 // G_LIMITED: "-cc1"
-// G_LIMITED: "-debug-info-kind=limited"
+// G_LIMITED: "-debug-info-kind=constructor"
 // G_DWARF2: "-dwarf-version=2"
 // G_DWARF4: "-dwarf-version=4"
 //
@@ -334,12 +334,12 @@
 // NOFDTS-NOT: "-mllvm" "-generate-type-units"
 // NOFDTSE-NOT: error: unsupported option '-fdebug-types-section' for target 'x86_64-apple-darwin'
 //
-// CI: "-dwarf-column-info"
+// CI-NOT: "-gno-column-info"
 //
-// NOCI-NOT: "-dwarf-column-info"
+// NOCI: "-gno-column-info"
 //
 // GEXTREFS: "-dwarf-ext-refs" "-fmodule-format=obj"
-// GEXTREFS: "-debug-info-kind={{standalone|limited}}"
+// GEXTREFS: "-debug-info-kind={{standalone|constructor}}"
 
 // RUN: not %clang -cc1 -debug-info-kind=watkind 2>&1 | FileCheck -check-prefix=BADSTRING1 %s
 // BADSTRING1: error: invalid value 'watkind' in '-debug-info-kind=watkind'
diff --git a/clang/test/Driver/fsanitize.c b/clang/test/Driver/fsanitize.c
index 9ff2bdf58d6cd..7340bfb35e40c 100644
--- a/clang/test/Driver/fsanitize.c
+++ b/clang/test/Driver/fsanitize.c
@@ -868,9 +868,7 @@
 // RUN: %clang -target x86_64-linux-gnu -fsanitize=hwaddress -fsanitize-hwaddress-abi=platform %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-HWASAN-PLATFORM-ABI
 // RUN: %clang -target x86_64-linux-gnu -fsanitize=hwaddress -fsanitize-hwaddress-abi=foo %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-HWASAN-FOO-ABI
 // CHECK-HWASAN-INTERCEPTOR-ABI: "-default-function-attr" "hwasan-abi=interceptor"
-// CHECK-HWASAN-INTERCEPTOR-ABI: "-target-feature" "+tagged-globals"
 // CHECK-HWASAN-PLATFORM-ABI: "-default-function-attr" "hwasan-abi=platform"
-// CHECK-HWASAN-PLATFORM-ABI: "-target-feature" "+tagged-globals"
 // CHECK-HWASAN-FOO-ABI: error: invalid value 'foo' in '-fsanitize-hwaddress-abi=foo'
 
 // RUN: %clang -target x86_64-linux-gnu -fsanitize=address,pointer-compare,pointer-subtract %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-POINTER-ALL
diff --git a/clang/test/Driver/fzero-initialized-in-bss.c b/clang/test/Driver/fzero-initialized-in-bss.c
new file mode 100644
index 0000000000000..34a1dc9788cda
--- /dev/null
+++ b/clang/test/Driver/fzero-initialized-in-bss.c
@@ -0,0 +1,8 @@
+// RUN: %clang -### %s -c -fzero-initialized-in-bss 2>&1 | FileCheck %s --check-prefix=NO
+// RUN: %clang -### %s -c 2>&1 | FileCheck %s --check-prefix=NO
+
+// NO-NOT: -fno-zero-initialized-in-bss
+
+// RUN: %clang -### %s -c -fzero-initialized-in-bss -fno-zero-initialized-in-bss 2>&1 | FileCheck %s
+
+// CHECK: -fno-zero-initialized-in-bss
diff --git a/clang/test/Driver/hip-binding.hip b/clang/test/Driver/hip-binding.hip
index 53fe9af0e653c..3d83930223462 100644
--- a/clang/test/Driver/hip-binding.hip
+++ b/clang/test/Driver/hip-binding.hip
@@ -25,12 +25,15 @@
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 -fgpu-rdc %t.o\
 // RUN: 2>&1 | FileCheck %s
 
-// CHECK: # "amdgcn-amd-amdhsa" - "offload bundler", inputs: ["[[IN:.*o]]"], outputs: ["[[OBJ1:.*o]]", "[[OBJ2:.*o]]", "[[OBJ3:.*o]]"] 
-// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[OBJ2]]"], output: "[[IMG2:.*out]]"
+// CHECK: # "x86_64-unknown-linux-gnu" - "offload bundler", inputs: ["[[IN:.*o]]"], outputs: ["[[HOSTOBJ:.*o]]", "{{.*o}}", "{{.*o}}"]
+// CHECK: # "amdgcn-amd-amdhsa" - "offload bundler", inputs: ["[[IN]]"], outputs: ["{{.*o}}", "[[DOBJ1:.*o]]", "[[DOBJ2:.*o]]"]
+// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[DOBJ1]]"], output: "[[IMG1:.*out]]"
 // CHECK-NOT: offload bundler
-// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[OBJ3]]"], output: "[[IMG3:.*out]]"
+// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[DOBJ2]]"], output: "[[IMG2:.*out]]"
 // CHECK-NOT: offload bundler
-// CHECK: # "x86_64-unknown-linux-gnu" - "GNU::Linker", inputs: ["[[OBJ1]]", "[[IMG2]]", "[[IMG3]]"], output: "a.out"
+// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[IMG1]]", "[[IMG2]]"], output: "[[FATBINOBJ:.*o]]"
+// CHECK-NOT: offload bundler
+// CHECK: # "x86_64-unknown-linux-gnu" - "GNU::Linker", inputs: ["[[HOSTOBJ]]", "[[FATBINOBJ]]"], output: "a.out"
 
 // RUN: %clang --hip-link -ccc-print-bindings -target x86_64-linux-gnu \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t.o\
diff --git a/clang/test/Driver/hip-include-path.hip b/clang/test/Driver/hip-include-path.hip
index 996c53bc3f286..7af06fabe5aec 100644
--- a/clang/test/Driver/hip-include-path.hip
+++ b/clang/test/Driver/hip-include-path.hip
@@ -19,9 +19,10 @@
 // COMMON-LABEL: "{{[^"]*}}clang{{[^"]*}}" "-cc1"
 // CLANG-SAME: "-internal-isystem" "{{.*}}clang/{{.*}}/include/cuda_wrappers"
 // NOCLANG-NOT: "{{.*}}clang/{{.*}}/include/cuda_wrappers"
-// CLANG-SAME: "-include" "__clang_hip_runtime_wrapper.h"
 // HIP-SAME: "-internal-isystem" "{{.*}}Inputs/rocm/include"
 // NOHIP-NOT: "{{.*}}Inputs/rocm/include"
+// HIP-SAME: "-include" "__clang_hip_runtime_wrapper.h"
+// NOHIP-NOT: "-include" "__clang_hip_runtime_wrapper.h"
 // skip check of standard C++ include path
 // CLANG-SAME: "-internal-isystem" "{{.*}}clang/{{.*}}/include"
 // NOCLANG-NOT: "{{.*}}clang/{{.*}}/include"
@@ -29,9 +30,22 @@
 // COMMON-LABEL: "{{[^"]*}}clang{{[^"]*}}" "-cc1"
 // CLANG-SAME: "-internal-isystem" "{{.*}}clang/{{.*}}/include/cuda_wrappers"
 // NOCLANG-NOT: "{{.*}}clang/{{.*}}/include/cuda_wrappers"
-// CLANG-SAME: "-include" "__clang_hip_runtime_wrapper.h"
 // HIP-SAME: "-internal-isystem" "{{.*}}Inputs/rocm/include"
 // NOHIP-NOT: "{{.*}}Inputs/rocm/include"
+// HIP-SAME: "-include" "__clang_hip_runtime_wrapper.h"
+// NOHIP-NOT: "-include" "__clang_hip_runtime_wrapper.h"
 // skip check of standard C++ include path
 // CLANG-SAME: "-internal-isystem" "{{.*}}clang/{{.*}}/include"
 // NOCLANG-NOT: "{{.*}}clang/{{.*}}/include"
+
+// RUN: %clang -c -### -target x86_64-unknown-linux-gnu --cuda-gpu-arch=gfx900 \
+// RUN:   -std=c++11 --rocm-path=%S/Inputs/rocm -nogpulib %s 2>&1 \
+// RUN:   --hip-version=3.5 | FileCheck -check-prefixes=ROCM35 %s
+
+// ROCM35-LABEL: "{{[^"]*}}clang{{[^"]*}}" "-cc1"
+// ROCM35-NOT: "{{.*}}clang/{{.*}}/include/cuda_wrappers"
+// ROCM35-SAME: "-internal-isystem" "{{[^"]*}}clang/{{[^"]*}}"
+// ROCM35-SAME: "-internal-isystem" "{{[^"]*}}Inputs/rocm/include"
+// ROCM35-NOT: "-include" "__clang_hip_runtime_wrapper.h"
+// skip check of standard C++ include path
+// ROCM35-SAME: "-internal-isystem" "{{[^"]*}}clang/{{[^"]*}}/include"
diff --git a/clang/test/Driver/hip-launch-api.hip b/clang/test/Driver/hip-launch-api.hip
new file mode 100644
index 0000000000000..a7d7d15c9b5ae
--- /dev/null
+++ b/clang/test/Driver/hip-launch-api.hip
@@ -0,0 +1,17 @@
+// REQUIRES: clang-driver
+// REQUIRES: x86-registered-target
+// REQUIRES: amdgpu-registered-target
+
+// By default FE assumes -fhip-new-launch-api.
+
+// RUN: %clang -### -target x86_64-unknown-linux-gnu -offload-arch=gfx906 %s \
+// RUN:   2>&1 | FileCheck -check-prefixes=NEW %s
+// NEW: "-fhip-new-launch-api"
+
+// RUN: %clang -### -target x86_64-unknown-linux-gnu -offload-arch=gfx906 %s \
+// RUN:   -fhip-new-launch-api 2>&1 | FileCheck -check-prefixes=NEW %s
+// NEW: "-fhip-new-launch-api"
+
+// RUN: %clang -### -target x86_64-unknown-linux-gnu -offload-arch=gfx906 %s \
+// RUN:   -fno-hip-new-launch-api 2>&1 | FileCheck -check-prefixes=OLD %s
+// OLD-NOT: "-fhip-new-launch-api"
diff --git a/clang/test/Driver/hip-link-save-temps.hip b/clang/test/Driver/hip-link-save-temps.hip
index 304be921ebc62..f2d3630dd119f 100644
--- a/clang/test/Driver/hip-link-save-temps.hip
+++ b/clang/test/Driver/hip-link-save-temps.hip
@@ -18,16 +18,35 @@
 // RUN:   --offload-arch=gfx906  %T/obj1.o %T/obj2.o 2>&1 | \
 // RUN:   FileCheck -check-prefixes=CHECK,NOUT %s
 
+// -fgpu-rdc link with output and --emit-static-lib
+// RUN: touch %T/obj1.o
+// RUN: touch %T/obj2.o
+// RUN: %clang -### -target x86_64-linux-gnu -nogpulib -save-temps \
+// RUN:   --hip-link -o libTest.a -fgpu-rdc --cuda-gpu-arch=gfx900 \
+// RUN:   --emit-static-lib \
+// RUN:   --offload-arch=gfx906  %T/obj1.o %T/obj2.o 2>&1 | \
+// RUN:   FileCheck -check-prefixes=CHECK,SLO %s
+
+// -fgpu-rdc link without output and --emit-static-lib
+// RUN: touch %T/obj1.o
+// RUN: touch %T/obj2.o
+// RUN: %clang -### -target x86_64-linux-gnu -nogpulib -save-temps \
+// RUN:   --hip-link -fgpu-rdc --cuda-gpu-arch=gfx900 \
+// RUN:   --emit-static-lib \
+// RUN:   --offload-arch=gfx906  %T/obj1.o %T/obj2.o 2>&1 | \
+// RUN:   FileCheck -check-prefixes=CHECK,SLNO %s
+
 // CHECK: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=obj1-host-x86_64-unknown-linux-gnu.o,obj1-hip-amdgcn-amd-amdhsa-gfx900.o,obj1-hip-amdgcn-amd-amdhsa-gfx906.o" "-unbundle"
 // CHECK: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=obj2-host-x86_64-unknown-linux-gnu.o,obj2-hip-amdgcn-amd-amdhsa-gfx900.o,obj2-hip-amdgcn-amd-amdhsa-gfx906.o" "-unbundle"
-// CHECK-NOT: llvm-link
-// CHECK-NOT: opt
-// CHECK-NOT: llc
-// CHECK: "{{.*lld.*}}" {{.*}} "-mllvm" "-amdgpu-internalize-symbols"
+// CHECK-NOT: {{".*/llvm-link"}}
+// CHECK-NOT: {{".*/opt"}}
+// CHECK-NOT: {{".*/llc"}}
+// CHECK: "{{.*lld.*}}" {{.*}} "-plugin-opt=-amdgpu-internalize-symbols"
 // CHECK-SAME: "-o" "a.out-hip-amdgcn-amd-amdhsa-gfx900" "obj1-hip-amdgcn-amd-amdhsa-gfx900.o" "obj2-hip-amdgcn-amd-amdhsa-gfx900.o"
-// CHECK: "{{.*lld.*}}" {{.*}} "-mllvm" "-amdgpu-internalize-symbols"
+// CHECK: "{{.*lld.*}}" {{.*}} "-plugin-opt=-amdgpu-internalize-symbols"
 // CHECK-SAME: "-o" "a.out-hip-amdgcn-amd-amdhsa-gfx906" "obj1-hip-amdgcn-amd-amdhsa-gfx906.o" "obj2-hip-amdgcn-amd-amdhsa-gfx906.o"
-// OUT: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=executable.hipfb"
-// OUT: "{{.*ld.*}}" {{.*}} "-o" "executable" {{.*}} "-T" "executable.lk"
-// NOUT: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=a.out.hipfb"
-// NOUT: "{{.*ld.*}}" {{.*}} "-o" "a.out" {{.*}} "-T" "a.out.lk"
+// CHECK: {{".*llvm-mc.*"}} "-o" "[[OBJBUNDLE:.*.o]]" "{{.*}}.mcin" "--filetype=obj"
+// OUT: "{{.*ld.*}}" {{.*}} "-o" "executable" {{.*}} "[[OBJBUNDLE]]"
+// NOUT: "{{.*ld.*}}" {{.*}} "-o" "a.out" {{.*}} "[[OBJBUNDLE]]"
+// SLO: "{{.*llvm-ar.*}}" "rcsD" "libTest.a" {{.*}} "[[OBJBUNDLE]]"
+// SLNO: "{{.*llvm-ar.*}}" "rcsD" "a.out" {{.*}} "[[OBJBUNDLE]]"
diff --git a/clang/test/Driver/hip-link-shared-library.hip b/clang/test/Driver/hip-link-shared-library.hip
index cb409d1a874d8..895bd53225e12 100644
--- a/clang/test/Driver/hip-link-shared-library.hip
+++ b/clang/test/Driver/hip-link-shared-library.hip
@@ -3,10 +3,13 @@
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t.o %S/Inputs/in.so \
 // RUN:   -fgpu-rdc 2>&1 | FileCheck %s
 
-// CHECK: # "amdgcn-amd-amdhsa" - "offload bundler", inputs: ["[[IN:.*o]]"], outputs: ["[[OBJ1:.*o]]", "[[OBJ2:.*o]]", "[[OBJ3:.*o]]"]
-// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[OBJ2]]"], output: "[[IMG2:.*out]]"
+// CHECK: # "x86_64-unknown-linux-gnu" - "offload bundler", inputs: ["[[IN:.*o]]"], outputs: ["[[HOSTOBJ:.*o]]", "{{.*o}}", "{{.*o}}"]
+// CHECK: # "amdgcn-amd-amdhsa" - "offload bundler", inputs: ["[[IN]]"], outputs: ["{{.*o}}", "[[DOBJ1:.*o]]", "[[DOBJ2:.*o]]"]
+// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[DOBJ1]]"], output: "[[IMG1:.*out]]"
 // CHECK-NOT: offload bundler
-// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[OBJ3]]"], output: "[[IMG3:.*out]]"
+// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[DOBJ2]]"], output: "[[IMG2:.*out]]"
 // CHECK-NOT: offload bundler
-// CHECK: # "x86_64-unknown-linux-gnu" - "GNU::Linker", inputs: ["[[OBJ1]]", "{{.*}}/Inputs/in.so", "[[IMG2]]", "[[IMG3]]"], output: "a.out"
+// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[IMG1]]", "[[IMG2]]"], output: "[[FATBINOBJ:.*o]]"
+// CHECK-NOT: offload bundler
+// CHECK: # "x86_64-unknown-linux-gnu" - "GNU::Linker", inputs: ["[[HOSTOBJ]]", "{{.*}}/Inputs/in.so", "[[FATBINOBJ]]"], output: "a.out"
 
diff --git a/clang/test/Driver/hip-link-static-library.hip b/clang/test/Driver/hip-link-static-library.hip
new file mode 100644
index 0000000000000..55c5a5acc5ca6
--- /dev/null
+++ b/clang/test/Driver/hip-link-static-library.hip
@@ -0,0 +1,27 @@
+// REQUIRES: clang-driver
+// REQUIRES: x86-registered-target
+// REQUIRES: amdgpu-registered-target
+
+// RUN: touch %t.o
+// RUN: %clang --hip-link -ccc-print-bindings -target x86_64-linux-gnu \
+// RUN:   --emit-static-lib \
+// RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 -fgpu-rdc %t.o\
+// RUN: 2>&1 | FileCheck %s
+
+// CHECK: # "x86_64-unknown-linux-gnu" - "offload bundler", inputs: ["[[IN:.*o]]"], outputs: ["[[HOSTOBJ:.*o]]", "{{.*o}}", "{{.*o}}"]
+// CHECK: # "amdgcn-amd-amdhsa" - "offload bundler", inputs: ["[[IN]]"], outputs: ["{{.*o}}", "[[DOBJ1:.*o]]", "[[DOBJ2:.*o]]"]
+// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[DOBJ1]]"], output: "[[IMG1:.*out]]"
+// CHECK-NOT: offload bundler
+// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[DOBJ2]]"], output: "[[IMG2:.*out]]"
+// CHECK-NOT: offload bundler
+// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[IMG1]]", "[[IMG2]]"], output: "[[FATBINOBJ:.*o]]"
+// CHECK-NOT: offload bundler
+// CHECK: # "x86_64-unknown-linux-gnu" - "GNU::StaticLibTool", inputs: ["[[HOSTOBJ]]", "[[FATBINOBJ]]"], output: "a.out"
+
+// RUN: %clang --hip-link -ccc-print-bindings -target x86_64-linux-gnu \
+// RUN:   --emit-static-lib \
+// RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t.o\
+// RUN: 2>&1 | FileCheck -check-prefix=NORDC %s
+
+// NORDC-NOT: offload bundler
+// NORDC: # "x86_64-unknown-linux-gnu" - "GNU::StaticLibTool", inputs: ["{{.*o}}"], output: "a.out"
diff --git a/clang/test/Driver/hip-phases.hip b/clang/test/Driver/hip-phases.hip
index b27ae44167a21..7c2dc1384ccd4 100644
--- a/clang/test/Driver/hip-phases.hip
+++ b/clang/test/Driver/hip-phases.hip
@@ -21,25 +21,27 @@
 // BIN-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
 // BIN-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
 // BIN-DAG: [[P2:[0-9]+]]: compiler, {[[P1]]}, ir, (host-[[T]])
+// RDC-DAG: [[P12:[0-9]+]]: backend, {[[P2]]}, assembler, (host-[[T]])
+// RDC-DAG: [[P13:[0-9]+]]: assembler, {[[P12]]}, object, (host-[[T]])
 
 // BIN-DAG: [[P3:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T]], (device-[[T]], [[ARCH:gfx803]])
 // BIN-DAG: [[P4:[0-9]+]]: preprocessor, {[[P3]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH]])
 // BIN-DAG: [[P5:[0-9]+]]: compiler, {[[P4]]}, ir, (device-[[T]], [[ARCH]])
 // NRD-DAG: [[P6:[0-9]+]]: backend, {[[P5]]}, assembler, (device-[[T]], [[ARCH]])
 // NRD-DAG: [[P7:[0-9]+]]: assembler, {[[P6]]}, object, (device-[[T]], [[ARCH]])
-// NRD-DAG: [[P8:[0-9]+]]: linker, {[[P7]]}, image, (device-[[T]], [[ARCH]])
-// NRD-DAG: [[P9:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH]])" {[[P8]]}, image
+// RDC-DAG: [[P7:[0-9]+]]: backend, {[[P5]]}, ir, (device-[[T]], [[ARCH]])
+// BIN-DAG: [[P8:[0-9]+]]: linker, {[[P7]]}, image, (device-[[T]], [[ARCH]])
+// BIN-DAG: [[P9:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH]])" {[[P8]]}, image
 // NRD-DAG: [[P10:[0-9]+]]: linker, {[[P9]]}, hip-fatbin, (device-[[T]])
-// RDC-DAG: [[P6:[0-9]+]]: backend, {[[P5]]}, ir, (device-[[T]], [[ARCH]])
-// RDC-DAG: [[P10:[0-9]+]]: linker, {[[P6]]}, image, (device-[[T]], [[ARCH]])
+// RDC-DAG: [[P10:[0-9]+]]: linker, {[[P9]]}, object, (device-[[T]])
 
-// NRD-DAG: [[P12:[0-9]+]]: offload, "host-[[T]] (x86_64-unknown-linux-gnu)" {[[P2]]}, "device-[[T]] (amdgcn-amd-amdhsa)" {[[P10]]}, ir
+// NRD-DAG: [[P11:[0-9]+]]: offload, "host-[[T]] (x86_64-unknown-linux-gnu)" {[[P2]]}, "device-[[T]] (amdgcn-amd-amdhsa)" {[[P10]]}, ir
+// RDC-DAG: [[P11:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa)" {[[P10]]}, object
+// NRD-DAG: [[P12:[0-9]+]]: backend, {[[P11]]}, assembler, (host-[[T]])
+// NRD-DAG: [[P13:[0-9]+]]: assembler, {[[P12]]}, object, (host-[[T]])
+// NRD-DAG: [[P14:[0-9]+]]: linker, {[[P13]]}, image, (host-[[T]])
+// RDC-DAG: [[P14:[0-9]+]]: linker, {[[P13]], [[P11]]}, image, (host-[[T]])
 
-// NRD-DAG: [[P13:[0-9]+]]: backend, {[[P12]]}, assembler, (host-[[T]])
-// RDC-DAG: [[P13:[0-9]+]]: backend, {[[P2]]}, assembler, (host-[[T]])
-// BIN-DAG: [[P14:[0-9]+]]: assembler, {[[P13]]}, object, (host-[[T]])
-// BIN-DAG: [[P15:[0-9]+]]: linker, {[[P14]]}, image, (host-[[T]])
-// RDC-DAG: [[P16:[0-9]+]]: offload, "host-[[T]] (x86_64-unknown-linux-gnu)" {[[P15]]}, "device-[[T]] (amdgcn-amd-amdhsa:gfx803)" {[[P10]]}, image
 //
 // Test single gpu architecture up to the assemble phase.
 //
@@ -56,59 +58,84 @@
 // ASM-DAG: [[P8:[0-9]+]]: backend, {[[P7]]}, assembler, (host-[[T]])
 
 //
-// Test two gpu architectures with complete compilation.
+// Test two gpu architectures with complete compilation with -fno-gpu-rdc.
 //
 // RUN: %clang -x hip -target x86_64-unknown-linux-gnu -ccc-print-phases \
 // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s 2>&1 \
-// RUN: | FileCheck -check-prefixes=BIN2,NRD2,CL2 %s
+// RUN: | FileCheck -check-prefixes=NRD2,NCL2 %s
 
 // RUN: %clang -x hip -target x86_64-unknown-linux-gnu -ccc-print-phases \
 // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -c 2>&1 \
-// RUN: | FileCheck -check-prefixes=BIN2,NRD2 %s
+// RUN: | FileCheck -check-prefixes=NRD2 %s
 
+// NRD2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
+// NRD2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
+// NRD2-DAG: [[P2:[0-9]+]]: compiler, {[[P1]]}, ir, (host-[[T]])
+
+// NRD2-DAG: [[P3:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T]], (device-[[T]], [[ARCH1:gfx803]])
+// NRD2-DAG: [[P4:[0-9]+]]: preprocessor, {[[P3]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH1]])
+// NRD2-DAG: [[P5:[0-9]+]]: compiler, {[[P4]]}, ir, (device-[[T]], [[ARCH1]])
+// NRD2-DAG: [[P6:[0-9]+]]: backend, {[[P5]]}, assembler, (device-[[T]], [[ARCH1]])
+// NRD2-DAG: [[P7:[0-9]+]]: assembler, {[[P6]]}, object, (device-[[T]], [[ARCH1]])
+// NRD2-DAG: [[P8:[0-9]+]]: linker, {[[P7]]}, image, (device-[[T]], [[ARCH1]])
+// NRD2-DAG: [[P9:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH1]])" {[[P8]]}, image
+
+// NRD2-DAG: [[P10:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T]], (device-[[T]], [[ARCH2:gfx900]])
+// NRD2-DAG: [[P11:[0-9]+]]: preprocessor, {[[P10]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH2]])
+// NRD2-DAG: [[P12:[0-9]+]]: compiler, {[[P11]]}, ir, (device-[[T]], [[ARCH2]])
+// NRD2-DAG: [[P13:[0-9]+]]: backend, {[[P12]]}, assembler, (device-[[T]], [[ARCH2]])
+// NRD2-DAG: [[P14:[0-9]+]]: assembler, {[[P13]]}, object, (device-[[T]], [[ARCH2]])
+// NRD2-DAG: [[P15:[0-9]+]]: linker, {[[P14]]}, image, (device-[[T]], [[ARCH2]])
+// NRD2-DAG: [[P16:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH2]])" {[[P15]]}, image
+// NRD2-DAG: [[P17:[0-9]+]]: linker, {[[P9]], [[P16]]}, hip-fatbin, (device-[[T]])
+// NRD2-DAG: [[P18:[0-9]+]]: offload, "host-[[T]] (x86_64-unknown-linux-gnu)" {[[P2]]}, "device-[[T]] (amdgcn-amd-amdhsa)" {[[P17]]}, ir
+// NRD2-DAG: [[P19:[0-9]+]]: backend, {[[P18]]}, assembler, (host-[[T]])
+// NRD2-DAG: [[P20:[0-9]+]]: assembler, {[[P19]]}, object, (host-[[T]])
+// NCL2-DAG: [[P21:[0-9]+]]: linker, {[[P20]]}, image, (host-[[T]])
+
+//
+// Test two gpu architectures with complete compilation with -fgpu-rdc.
+//
 // RUN: %clang -x hip -target x86_64-unknown-linux-gnu -ccc-print-phases \
 // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -fgpu-rdc 2>&1 \
-// RUN: | FileCheck -check-prefixes=BIN2,RDC2,CL2,RCL2 %s
+// RUN: | FileCheck -check-prefixes=RDC2,CL2,RCL2 %s
 
 // RUN: %clang -x hip -target x86_64-unknown-linux-gnu -ccc-print-phases \
 // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -fgpu-rdc -c 2>&1 \
-// RUN: | FileCheck -check-prefixes=BIN2,RDC2,RC2 %s
+// RUN: | FileCheck -check-prefixes=RDC2,RC2 %s
 
-// BIN2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
-// BIN2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
-// BIN2-DAG: [[P2:[0-9]+]]: compiler, {[[P1]]}, ir, (host-[[T]])
+// RCL2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
+// RCL2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
+// RCL2-DAG: [[P2:[0-9]+]]: compiler, {[[P1]]}, ir, (host-[[T]])
+// RCL2-DAG: [[P19:[0-9]+]]: backend, {[[P2]]}, assembler, (host-[[T]])
+// RCL2-DAG: [[P20:[0-9]+]]: assembler, {[[P19]]}, object, (host-[[T]])
 
-// BIN2-DAG: [[P3:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T]], (device-[[T]], [[ARCH1:gfx803]])
-// BIN2-DAG: [[P4:[0-9]+]]: preprocessor, {[[P3]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH1]])
-// BIN2-DAG: [[P5:[0-9]+]]: compiler, {[[P4]]}, ir, (device-[[T]], [[ARCH1]])
-// NRD2-DAG: [[P6:[0-9]+]]: backend, {[[P5]]}, assembler, (device-[[T]], [[ARCH1]])
+// RDC2-DAG: [[P3:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (device-[[T]], [[ARCH1:gfx803]])
+// RDC2-DAG: [[P4:[0-9]+]]: preprocessor, {[[P3]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH1]])
+// RDC2-DAG: [[P5:[0-9]+]]: compiler, {[[P4]]}, ir, (device-[[T]], [[ARCH1]])
 // RDC2-DAG: [[P6:[0-9]+]]: backend, {[[P5]]}, ir, (device-[[T]], [[ARCH1]])
-// NRD2-DAG: [[P7:[0-9]+]]: assembler, {[[P6]]}, object, (device-[[T]], [[ARCH1]])
-// NRD2-DAG: [[P8:[0-9]+]]: linker, {[[P7]]}, image, (device-[[T]], [[ARCH1]])
 // RCL2-DAG: [[P8:[0-9]+]]: linker, {[[P6]]}, image, (device-[[T]], [[ARCH1]])
-// NRD2-DAG: [[P9:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH1]])" {[[P8]]}, image
+// RCL2-DAG: [[P9:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH1]])" {[[P8]]}, image
 // RC2-DAG: [[P9:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH1]])" {[[P6]]}, ir
 
-// BIN2-DAG: [[P10:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T]], (device-[[T]], [[ARCH2:gfx900]])
-// BIN2-DAG: [[P11:[0-9]+]]: preprocessor, {[[P10]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH2]])
-// BIN2-DAG: [[P12:[0-9]+]]: compiler, {[[P11]]}, ir, (device-[[T]], [[ARCH2]])
-// NRD2-DAG: [[P13:[0-9]+]]: backend, {[[P12]]}, assembler, (device-[[T]], [[ARCH2]])
+// RDC2-DAG: [[P10:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T]], (device-[[T]], [[ARCH2:gfx900]])
+// RDC2-DAG: [[P11:[0-9]+]]: preprocessor, {[[P10]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH2]])
+// RDC2-DAG: [[P12:[0-9]+]]: compiler, {[[P11]]}, ir, (device-[[T]], [[ARCH2]])
 // RDC2-DAG: [[P13:[0-9]+]]: backend, {[[P12]]}, ir, (device-[[T]], [[ARCH2]])
-// NRD2-DAG: [[P14:[0-9]+]]: assembler, {[[P13]]}, object, (device-[[T]], [[ARCH2]])
-// NRD2-DAG: [[P15:[0-9]+]]: linker, {[[P14]]}, image, (device-[[T]], [[ARCH2]])
 // RCL2-DAG: [[P15:[0-9]+]]: linker, {[[P13]]}, image, (device-[[T]], [[ARCH2]])
-// NRD2-DAG: [[P16:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH2]])" {[[P15]]}, image
+// RCL2-DAG: [[P16:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH2]])" {[[P15]]}, image
 // RC2-DAG: [[P16:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH2]])" {[[P13]]}, ir
 
-// NRD2-DAG: [[P17:[0-9]+]]: linker, {[[P9]], [[P16]]}, hip-fatbin, (device-[[T]])
+// RC2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
+// RC2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
+// RC2-DAG: [[P2:[0-9]+]]: compiler, {[[P1]]}, ir, (host-[[T]])
+// RC2-DAG: [[P19:[0-9]+]]: backend, {[[P2]]}, assembler, (host-[[T]])
+// RC2-DAG: [[P20:[0-9]+]]: assembler, {[[P19]]}, object, (host-[[T]])
 
-// NRD2-DAG: [[P18:[0-9]+]]: offload, "host-[[T]] (x86_64-unknown-linux-gnu)" {[[P2]]}, "device-[[T]] (amdgcn-amd-amdhsa)" {[[P17]]}, ir
-// NRD2-DAG: [[P19:[0-9]+]]: backend, {[[P18]]}, assembler, (host-[[T]])
-// RDC2-DAG: [[P19:[0-9]+]]: backend, {[[P2]]}, assembler, (host-[[T]])
-// BIN2-DAG: [[P20:[0-9]+]]: assembler, {[[P19]]}, object, (host-[[T]])
-// CL2-DAG: [[P21:[0-9]+]]: linker, {[[P20]]}, image, (host-[[T]])
-// RCL2-DAG: [[P22:[0-9]+]]: offload, "host-[[T]] (x86_64-unknown-linux-gnu)" {[[P21]]}, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH1]])" {[[P8]]}, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH2]])" {[[P15]]}, image
-// RC2-DAG: [[P22:[0-9]+]]: clang-offload-bundler, {[[P9]], [[P16]], [[P20]]}, object, (host-[[T]])
+// RCL2-DAG: [[P17:[0-9]+]]: linker, {[[P9]], [[P16]]}, object, (device-[[T]])
+// RCL2-DAG: [[P22:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa)" {[[P17]]}, object
+// RCL2-DAG: [[P23:[0-9]+]]: linker, {[[P20]], [[P22]]}, image, (host-[[T]])
+// RC2-DAG: [[P23:[0-9]+]]: clang-offload-bundler, {[[P9]], [[P16]], [[P20]]}, object, (host-[[T]])
 
 //
 // Test two gpu architecturess up to the assemble phase.
@@ -253,8 +280,13 @@
 // RL2-DAG: [[P1:[0-9]+]]: clang-offload-unbundler, {[[P0]]}, object, (host-[[T]])
 // L2-DAG: [[P2:[0-9]+]]: input, "{{.*}}obj2.o", object, (host-[[T]])
 // RL2-DAG: [[P3:[0-9]+]]: clang-offload-unbundler, {[[P2]]}, object, (host-[[T]])
-// NL2-DAG: [[P4:[0-9]+]]: linker, {[[P0]], [[P2]]}, image, (host-[[T]])
-// RL2-DAG: [[P4:[0-9]+]]: linker, {[[P1]], [[P3]]}, image, (host-[[T]])
-// RL2-DAG: [[P5:[0-9]+]]: linker, {[[P1]], [[P3]]}, image, (device-[[T]], [[ARCH1:gfx803]])
+
+// RL2-DAG: [[P4:[0-9]+]]: linker, {[[P1]], [[P3]]}, image, (device-[[T]], [[ARCH1:gfx803]])
+// RL2-DAG: [[P5:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH1]])" {[[P4]]}, image
 // RL2-DAG: [[P6:[0-9]+]]: linker, {[[P1]], [[P3]]}, image, (device-[[T]], [[ARCH2:gfx900]])
-// RL2-DAG: [[P7:[0-9]+]]: offload, "host-[[T]] (x86_64-unknown-linux-gnu)" {[[P4]]}, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH1]])" {[[P5]]}, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH2]])" {[[P6]]}, image
+// RL2-DAG: [[P7:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH2]])" {[[P6]]}, image
+// RL2-DAG: [[P8:[0-9]+]]: linker, {[[P5]], [[P7]]}, object, (device-[[T]])
+// RL2-DAG: [[P9:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa)" {[[P8]]}, object
+
+// NL2-DAG: [[P4:[0-9]+]]: linker, {[[P0]], [[P2]]}, image, (host-[[T]])
+// RL2-DAG: [[P4:[0-9]+]]: linker, {[[P1]], [[P3]], [[P9]]}, image, (host-[[T]])
diff --git a/clang/test/Driver/hip-save-temps.hip b/clang/test/Driver/hip-save-temps.hip
index 1ac506aa6fba1..b5d54a8092a92 100644
--- a/clang/test/Driver/hip-save-temps.hip
+++ b/clang/test/Driver/hip-save-temps.hip
@@ -25,13 +25,20 @@
 // -fgpu-rdc without -o
 // RUN: %clang -### -target x86_64-linux-gnu -nogpulib -save-temps \
 // RUN:   -fgpu-rdc --cuda-gpu-arch=gfx900 %s 2>&1 | \
-// RUN:   FileCheck -check-prefixes=CHECK,RDC,RDCL,RDC-NOUT,NOUT %s
+// RUN:   FileCheck -check-prefixes=CHECK,RDC,RDCL,NOUT %s
 
 // -fgpu-rdc with -o
-// RUN: %clang -### -target x86_64-linux-gnu -nogpulib -save-temps \
-// RUN:   -o executable -fgpu-rdc --cuda-gpu-arch=gfx900 %s 2>&1 | \
-// RUN:   FileCheck -check-prefixes=CHECK,RDC,RDCL,RDC-WOUT,WOUT %s
+// UN: %clang -### -target x86_64-linux-gnu -nogpulib -save-temps \
+// UN:   -o executable -fgpu-rdc --cuda-gpu-arch=gfx900 %s 2>&1 | \
+// UN:   FileCheck -check-prefixes=CHECK,RDC,RDCL,WOUT %s
+
+// -fgpu-rdc host object path
+// RDCL: "{{.*clang.*}}" "-cc1" {{.*}} "-E" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.cui"
+// RDCL: "{{.*clang.*}}" "-cc1" {{.*}} "-emit-llvm-bc" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.bc"
+// RDCL: "{{.*clang.*}}" "-cc1" {{.*}} "-S" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.s"
+// RDCL: "{{.*clang.*}}" "-cc1as" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.o"
 
+// device object paths
 // CHECK: {{".*clang.*"}} "-cc1" {{.*}} "-E" {{.*}} [[CPU:"-target-cpu" "gfx900"]] {{.*}}  "-o" "hip-save-temps-hip-amdgcn-amd-amdhsa-gfx900.cui"
 // NORDC: {{".*clang.*"}} "-cc1" {{.*}} "-emit-llvm-bc" {{.*}} [[CPU]] {{.*}} "-disable-llvm-passes" {{.*}} "-o" "hip-save-temps-hip-amdgcn-amd-amdhsa-gfx900.bc"
 // RDC: {{".*clang.*"}} "-cc1" {{.*}} "-emit-llvm-bc" {{.*}} [[CPU]] {{.*}} "-disable-llvm-passes" {{.*}} "-o" "hip-save-temps-hip-amdgcn-amd-amdhsa-gfx900.tmp.bc"
@@ -43,22 +50,26 @@
 
 // RDC: {{".*clang.*"}} "-cc1" {{.*}} "-emit-llvm-bc" {{.*}} [[CPU]] {{.*}} "-o" "hip-save-temps-hip-amdgcn-amd-amdhsa-gfx900.bc"
 // NORDC: {{".*clang.*"}} "-cc1as" {{.*}} "-filetype" "obj" {{.*}} [[CPU]] {{.*}} "-o" "hip-save-temps-hip-amdgcn-amd-amdhsa-gfx900.o"
-// CHECK-NOT: llvm-link
-// CHECK-NOT: opt
-// CHECK-NOT: llc
+// CHECK-NOT: "{{.*}}llvm-link"
+// CHECK-NOT: "{{.*}}opt"
+// CHECK-NOT: "{{.*}}llc"
 // NORDC: {{.*lld.*}}"-o" "hip-save-temps-hip-amdgcn-amd-amdhsa-gfx900.out"
-
-// RDCL: "{{.*lld.*}}" {{.*}} "-mllvm" "-amdgpu-internalize-symbols"
+// RDCL: "{{.*lld.*}}" {{.*}} "-plugin-opt=-amdgpu-internalize-symbols" {{.*}}"-save-temps"
 // RDCL-SAME: "-o" "a.out-hip-amdgcn-amd-amdhsa-gfx900"
+// RDCC: "{{.*clang.*}}" "-cc1" {{.*}} "-E" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.cui"
+// RDCC: "{{.*clang.*}}" "-cc1" {{.*}} "-emit-llvm-bc" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.bc"
+// RDCC: "{{.*clang.*}}" "-cc1" {{.*}} "-S" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.s"
+// RDCC: "{{.*clang.*}}" "-cc1as" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.o"
+// RDCC: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=hip-save-temps.o"
+// RDCL: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=hip-save-temps-hip-amdgcn-amd-amdhsa.hipfb"
+// RDCL: {{.*}}llvm-mc{{.*}}"-o" "hip-save-temps-hip-amdgcn-amd-amdhsa.o" "hip-save-temps-hip-amdgcn-amd-amdhsa.mcin" "--filetype=obj"
 
-// NORDC: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=hip-save-temps.hip-hip-amdgcn-amd-amdhsa.hipfb"
-// CHECK: "{{.*clang.*}}" "-cc1" {{.*}} "-E" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.cui"
+// -fno-gpu-rdc host object path
+// NORDC: "{{.*clang.*}}" "-cc1" {{.*}} "-E" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.cui"
 // NORDC: "{{.*clang.*}}" "-cc1" {{.*}} "-emit-llvm-bc" {{.*}}  "-fcuda-include-gpubinary" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.bc"
-// RDC: "{{.*clang.*}}" "-cc1" {{.*}} "-emit-llvm-bc" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.bc"
-// CHECK: "{{.*clang.*}}" "-cc1" {{.*}} "-S" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.s"
-// CHECK: "{{.*clang.*}}" "-cc1as" {{.*}} "-o" "hip-save-temps{{.*}}.o"
-// RDCC: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=hip-save-temps.o"
-// RDC-NOUT: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=a.out.hipfb"
-// RDC-WOUT: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=executable.hipfb"
-// NOUT: "{{.*ld.*}}" {{.*}} "-o" "a.out"
-// WOUT: "{{.*ld.*}}" {{.*}} "-o" "executable"
\ No newline at end of file
+// NORDC: "{{.*clang.*}}" "-cc1" {{.*}} "-S" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.s"
+// NORDC: "{{.*clang.*}}" "-cc1as" {{.*}} "-o" "hip-save-temps{{.*}}.o"
+
+// output to default a.out or -o specified file name
+// NOUT: {{.*}}ld{{.*}}"-o" "a.out"
+// WOUT: {{.*}}ld{{.*}}"-o" "executable"
diff --git a/clang/test/Driver/hip-std.hip b/clang/test/Driver/hip-std.hip
new file mode 100644
index 0000000000000..4bdf05ab495d5
--- /dev/null
+++ b/clang/test/Driver/hip-std.hip
@@ -0,0 +1,23 @@
+// REQUIRES: clang-driver
+// REQUIRES: x86-registered-target
+// REQUIRES: amdgpu-registered-target
+
+// RUN: %clang -### -target x86_64-unknown-linux-gnu -offload-arch=gfx906 %s \
+// RUN:   2>&1 | FileCheck -check-prefixes=DEFAULT %s
+// DEFAULT: "{{.*}}clang{{.*}}" "-cc1"{{.*}}"-fcuda-is-device"{{.*}}"-std=c++11"
+// DEFAULT: "{{.*}}clang{{.*}}" "-cc1"{{.*}}"-std=c++11"
+
+// RUN: %clang -### -target x86_64-unknown-linux-gnu -offload-arch=gfx906 %s \
+// RUN:   -std=c++17 %s 2>&1 | FileCheck -check-prefixes=SPECIFIED %s
+// SPECIFIED: "{{.*}}clang{{.*}}" "-cc1"{{.*}}"-fcuda-is-device"{{.*}}"-std=c++17"
+// SPECIFIED: "{{.*}}clang{{.*}}" "-cc1"{{.*}}"-std=c++17"
+
+// RUN: %clang -### -target x86_64-pc-windows-msvc -offload-arch=gfx906 %s \
+// RUN:   2>&1 | FileCheck -check-prefixes=MSVC-DEF %s
+// MSVC-DEF: "{{.*}}clang{{.*}}" "-cc1"{{.*}}"-fcuda-is-device"{{.*}}"-std=c++14"
+// MSVC-DEF: "{{.*}}clang{{.*}}" "-cc1"{{.*}}"-std=c++14"
+
+// RUN: %clang -### -target x86_64-pc-windows-msvc -offload-arch=gfx906 %s \
+// RUN:   -std=c++17 %s 2>&1 | FileCheck -check-prefixes=MSVC-SPEC %s
+// MSVC-SPEC: "{{.*}}clang{{.*}}" "-cc1"{{.*}}"-fcuda-is-device"{{.*}}"-std=c++17"
+// MSVC-SPEC: "{{.*}}clang{{.*}}" "-cc1"{{.*}}"-std=c++17"
diff --git a/clang/test/Driver/hip-toolchain-features.hip b/clang/test/Driver/hip-toolchain-features.hip
index 3b90d4101ee3d..c236d2c7102fb 100644
--- a/clang/test/Driver/hip-toolchain-features.hip
+++ b/clang/test/Driver/hip-toolchain-features.hip
@@ -2,36 +2,53 @@
 // REQUIRES: x86-registered-target
 // REQUIRES: amdgpu-registered-target
 
-// RUN: %clang -### -c -target x86_64-linux-gnu -fgpu-rdc \
+// RUN: %clang -### -target x86_64-linux-gnu -fgpu-rdc -nogpulib \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
 // RUN:   -mxnack 2>&1 | FileCheck %s -check-prefix=XNACK
-// RUN: %clang -### -c -target x86_64-linux-gnu -fgpu-rdc \
+// RUN: %clang -### -target x86_64-linux-gnu -fgpu-rdc -nogpulib \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
 // RUN:   -mno-xnack 2>&1 | FileCheck %s -check-prefix=NOXNACK
 
 // XNACK: {{.*}}clang{{.*}}"-target-feature" "+xnack"
+// XNACK: {{.*}}lld{{.*}}"-plugin-opt=-mattr=+xnack"
 // NOXNACK: {{.*}}clang{{.*}}"-target-feature" "-xnack"
+// NOXNACK: {{.*}}lld{{.*}}"-plugin-opt=-mattr=-xnack"
 
 
-// RUN: %clang -### -c -target x86_64-linux-gnu -fgpu-rdc \
+// RUN: %clang -### -target x86_64-linux-gnu -fgpu-rdc -nogpulib \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
 // RUN:   -msram-ecc 2>&1 | FileCheck %s -check-prefix=SRAM
-// RUN: %clang -### -c -target x86_64-linux-gnu -fgpu-rdc \
+// RUN: %clang -### -target x86_64-linux-gnu -fgpu-rdc -nogpulib \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
 // RUN:   -mno-sram-ecc 2>&1 | FileCheck %s -check-prefix=NOSRAM
 
 // SRAM: {{.*}}clang{{.*}}"-target-feature" "+sram-ecc"
+// SRAM: {{.*}}lld{{.*}}"-plugin-opt=-mattr=+sram-ecc"
 // NOSRAM: {{.*}}clang{{.*}}"-target-feature" "-sram-ecc"
+// NOSRAM: {{.*}}lld{{.*}}"-plugin-opt=-mattr=-sram-ecc"
 
 
-// RUN: %clang -### -c -target x86_64-linux-gnu -fgpu-rdc \
+// RUN: %clang -### -target x86_64-linux-gnu -fgpu-rdc -nogpulib \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
 // RUN:   -mxnack -msram-ecc \
 // RUN:   2>&1 | FileCheck %s -check-prefix=ALL3
-// RUN: %clang -### -c -target x86_64-linux-gnu -fgpu-rdc \
+// RUN: %clang -### -target x86_64-linux-gnu -fgpu-rdc -nogpulib \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
 // RUN:   -mno-xnack -mno-sram-ecc \
 // RUN:   2>&1 | FileCheck %s -check-prefix=NOALL3
 
 // ALL3: {{.*}}clang{{.*}}"-target-feature" "+xnack" "-target-feature" "+sram-ecc"
+// ALL3: {{.*}}lld{{.*}}"-plugin-opt=-mattr=+xnack,+sram-ecc"
 // NOALL3: {{.*}}clang{{.*}}"-target-feature" "-xnack" "-target-feature" "-sram-ecc"
+// NOALL3: {{.*}}lld{{.*}}"-plugin-opt=-mattr=-xnack,-sram-ecc"
+
+// RUN: %clang -### -target x86_64-linux-gnu -fgpu-rdc -nogpulib \
+// RUN:   --cuda-gpu-arch=gfx1010 %s \
+// RUN:   -mcumode -mcumode -mno-cumode -mwavefrontsize64 -mcumode \
+// RUN:   -mwavefrontsize64 -mno-wavefrontsize64 2>&1 \
+// RUN:   | FileCheck %s -check-prefix=DUP
+// DUP: {{.*}}clang{{.*}} "-target-feature" "-wavefrontsize16"
+// DUP-SAME: "-target-feature" "+wavefrontsize32"
+// DUP-SAME: "-target-feature" "-wavefrontsize64"
+// DUP-SAME: "-target-feature" "+cumode"
+// DUP: {{.*}}lld{{.*}} "-plugin-opt=-mattr=-wavefrontsize16,+wavefrontsize32,-wavefrontsize64,+cumode"
diff --git a/clang/test/Driver/hip-toolchain-mllvm.hip b/clang/test/Driver/hip-toolchain-mllvm.hip
index 66556d0b726b8..5b47e1e0b5d86 100644
--- a/clang/test/Driver/hip-toolchain-mllvm.hip
+++ b/clang/test/Driver/hip-toolchain-mllvm.hip
@@ -7,20 +7,25 @@
 // RUN:   -mllvm -amdgpu-function-calls=0 \
 // RUN:   %s 2>&1 | FileCheck %s
 
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
+// RUN:   -fgpu-rdc -mllvm -amdgpu-function-calls=0 \
+// RUN:   %s 2>&1 | FileCheck -check-prefixes=CHECK,RDC %s
+
 // CHECK: [[CLANG:".*clang.*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
-// CHECK-SAME: "-emit-obj"
 // CHECK-SAME: {{.*}} "-target-cpu" "gfx803"
 // CHECK-SAME: {{.*}} "-mllvm" "-amdgpu-function-calls=0" {{.*}}
 
 // CHECK-NOT: {{".*opt"}}
 // CHECK-NOT: {{".*llc"}}
+// RDC: [[LLD:".*lld.*"]] {{.*}} "-plugin-opt=-amdgpu-function-calls=0"
 
 // CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
-// CHECK-SAME: "-emit-obj"
 // CHECK-SAME: {{.*}} "-target-cpu" "gfx900"
 // CHECK-SAME: {{.*}} "-mllvm" "-amdgpu-function-calls=0" {{.*}}
 
 // CHECK-NOT: {{".*opt"}}
 // CHECK-NOT: {{".*llc"}}
+// RDC: [[LLD:".*lld.*"]] {{.*}} "-plugin-opt=-amdgpu-function-calls=0"
diff --git a/clang/test/Driver/hip-toolchain-opt.hip b/clang/test/Driver/hip-toolchain-opt.hip
index 4a470dba38557..d58471aec5f51 100644
--- a/clang/test/Driver/hip-toolchain-opt.hip
+++ b/clang/test/Driver/hip-toolchain-opt.hip
@@ -72,6 +72,16 @@
 
 // ALL-NOT: "{{.*}}llc"
 
+// ALL: "{{.*}}lld{{.*}}" {{.*}} "-plugin-opt=mcpu=gfx900"
+// DEFAULT-NOT: "-plugin-opt=O{{.*}}"
+// O0-SAME: "-plugin-opt=O0"
+// O1-SAME: "-plugin-opt=O1"
+// O2-SAME: "-plugin-opt=O2"
+// O3-SAME: "-plugin-opt=O3"
+// Os-SAME: "-plugin-opt=O2"
+// Oz-SAME: "-plugin-opt=O2"
+// Og-SAME: "-plugin-opt=O1"
+
 // ALL: "{{.*}}clang{{.*}}" "-cc1" "-triple" "x86_64-unknown-linux-gnu"
 // DEFAULT-NOT: "-O{{.}}"
 // O0-SAME: "-O0"
diff --git a/clang/test/Driver/hip-toolchain-rdc-separate.hip b/clang/test/Driver/hip-toolchain-rdc-separate.hip
index 25661f399a82f..ab89e19256be9 100644
--- a/clang/test/Driver/hip-toolchain-rdc-separate.hip
+++ b/clang/test/Driver/hip-toolchain-rdc-separate.hip
@@ -86,29 +86,43 @@
 
 // LINK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
 // LINK-SAME: "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
-// LINK-SAME: "-inputs=[[A_O:.*a.o]]" "-outputs=[[A_OBJ_HOST:.*o]],[[A_BC1:.*o]],[[A_BC2:.*o]]"
+// LINK-SAME: "-inputs=[[A_O:.*a.o]]" "-outputs=[[A_OBJ_HOST:.*o]],{{.*o}},{{.*o}}"
 // LINK: "-unbundle"
 
 // LINK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
 // LINK-SAME: "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
-// LINK-SAME: "-inputs=[[B_O:.*b.o]]" "-outputs=[[B_OBJ_HOST:.*o]],[[B_BC1:.*o]],[[B_BC2:.*o]]"
+// LINK-SAME: "-inputs=[[B_O:.*b.o]]" "-outputs=[[B_OBJ_HOST:.*o]],{{.*o}},{{.*o}}"
+// LINK: "-unbundle"
+
+// LINK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
+// LINK-SAME: "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// LINK-SAME: "-inputs=[[A_O]]" "-outputs={{.*o}},[[A_BC1:.*o]],[[A_BC2:.*o]]"
+// LINK: "-unbundle"
+
+// LINK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
+// LINK-SAME: "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// LINK-SAME: "-inputs=[[B_O]]" "-outputs={{.*o}},[[B_BC1:.*o]],[[B_BC2:.*o]]"
 // LINK: "-unbundle"
 
 // LINK-NOT: "*.llvm-link"
 // LINK-NOT: ".*opt"
 // LINK-NOT: ".*llc"
-// LINK: {{".*lld.*"}} {{.*}} "-mllvm" "-amdgpu-internalize-symbols"
+// LINK: {{".*lld.*"}} {{.*}} "-plugin-opt=-amdgpu-internalize-symbols"
+// LINK: "-plugin-opt=mcpu=gfx803"
 // LINK-SAME: "-o" "[[IMG_DEV1:.*.out]]" "[[A_BC1]]" "[[B_BC1]]"
 
 // LINK-NOT: "*.llvm-link"
 // LINK-NOT: ".*opt"
 // LINK-NOT: ".*llc"
-// LINK: {{".*lld.*"}} {{.*}} "-mllvm" "-amdgpu-internalize-symbols"
+// LINK: {{".*lld.*"}} {{.*}} "-plugin-opt=-amdgpu-internalize-symbols"
+// LINK: "-plugin-opt=mcpu=gfx900"
 // LINK-SAME: "-o" "[[IMG_DEV2:.*.out]]" "[[A_BC2]]" "[[B_BC2]]"
 
 // LINK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
 // LINK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
 // LINK-SAME: "-inputs={{.*}},[[IMG_DEV1]],[[IMG_DEV2]]" "-outputs=[[BUNDLE:.*hipfb]]"
 
-// LINK: [[LD:".*ld.*"]] {{.*}} "[[A_OBJ_HOST]]" "[[B_OBJ_HOST]]"
-// LINK-SAME: {{.*}} "-T" "{{.*}}.lk"
+// LINK: {{".*llvm-mc.*"}} "-o" "[[OBJBUNDLE:.*o]]" "{{.*}}.mcin" "--filetype=obj"
+
+// LINK: [[LD:".*ld.*"]] {{.*}} "-o" "a.out" {{.*}} "[[A_OBJ_HOST]]"
+// LINK-SAME: "[[B_OBJ_HOST]]" "[[OBJBUNDLE]]"
diff --git a/clang/test/Driver/hip-toolchain-rdc-static-lib.hip b/clang/test/Driver/hip-toolchain-rdc-static-lib.hip
new file mode 100644
index 0000000000000..dc29b0f87e366
--- /dev/null
+++ b/clang/test/Driver/hip-toolchain-rdc-static-lib.hip
@@ -0,0 +1,83 @@
+// REQUIRES: clang-driver
+// REQUIRES: x86-registered-target
+// REQUIRES: amdgpu-registered-target
+
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
+// RUN:   --emit-static-lib -nogpulib \
+// RUN:   -fuse-ld=lld -fgpu-rdc -nogpuinc \
+// RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
+// RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
+// RUN: 2>&1 | FileCheck %s
+
+// emit objects for host side path
+// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-triple" "x86_64-unknown-linux-gnu"
+// CHECK-SAME: "-aux-triple" "amdgcn-amd-amdhsa"
+// CHECK-SAME: "-emit-obj"
+// CHECK-SAME: {{.*}} "-main-file-name" "a.cu"
+// CHECK-SAME: {{.*}} "-o" [[A_OBJ_HOST:".*o"]] "-x" "hip"
+// CHECK-SAME: {{.*}} [[A_SRC:".*a.cu"]]
+
+// CHECK: [[CLANG]] "-cc1" "-triple" "x86_64-unknown-linux-gnu"
+// CHECK-SAME: "-aux-triple" "amdgcn-amd-amdhsa"
+// CHECK-SAME: "-emit-obj"
+// CHECK-SAME: {{.*}} "-main-file-name" "b.hip"
+// CHECK-SAME: {{.*}} "-o" [[B_OBJ_HOST:".*o"]] "-x" "hip"
+// CHECK-SAME: {{.*}} [[B_SRC:".*b.hip"]]
+
+// generate image for device side path on gfx803
+// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
+// CHECK-SAME: "-emit-llvm-bc"
+// CHECK-SAME: {{.*}} "-main-file-name" "a.cu"
+// CHECK-SAME: "-fcuda-is-device" "-fgpu-rdc"
+// CHECK-SAME: "-target-cpu" "gfx803"
+// CHECK-SAME: {{.*}} "-o" [[A_BC1:".*bc"]] "-x" "hip"
+// CHECK-SAME: {{.*}} [[A_SRC]]
+
+// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
+// CHECK-SAME: "-emit-llvm-bc"
+// CHECK-SAME: {{.*}} "-main-file-name" "b.hip"
+// CHECK-SAME: "-fcuda-is-device" "-fgpu-rdc"
+// CHECK-SAME: "-target-cpu" "gfx803"
+// CHECK-SAME: {{.*}} "-o" [[B_BC1:".*bc"]] "-x" "hip"
+// CHECK-SAME: {{.*}} [[B_SRC]]
+
+// CHECK-NOT: "*.llvm-link"
+// CHECK-NOT: ".*opt"
+// CHECK-NOT: ".*llc"
+// CHECK: [[LLD: ".*lld"]] {{.*}} "-o" "[[IMG_DEV1:.*out]]" [[A_BC1]] [[B_BC1]]
+
+// generate image for device side path on gfx900
+// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
+// CHECK-SAME: "-emit-llvm-bc"
+// CHECK-SAME: {{.*}} "-main-file-name" "a.cu"
+// CHECK-SAME: "-fcuda-is-device" "-fgpu-rdc"
+// CHECK-SAME: "-target-cpu" "gfx900"
+// CHECK-SAME: {{.*}} "-o" [[A_BC2:".*bc"]] "-x" "hip"
+// CHECK-SAME: {{.*}} [[A_SRC]]
+
+// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
+// CHECK-SAME: "-emit-llvm-bc"
+// CHECK-SAME: {{.*}} "-main-file-name" "b.hip"
+// CHECK-SAME: "-fcuda-is-device" "-fgpu-rdc"
+// CHECK-SAME: "-target-cpu" "gfx900"
+// CHECK-SAME: {{.*}} "-o" [[B_BC2:".*bc"]] "-x" "hip"
+// CHECK-SAME: {{.*}} [[B_SRC]]
+
+// CHECK-NOT: "*.llvm-link"
+// CHECK-NOT: ".*opt"
+// CHECK-NOT: ".*llc"
+// CHECK: [[LLD]] {{.*}} "-o" "[[IMG_DEV2:.*out]]" [[A_BC2]] [[B_BC2]]
+
+// combine images generated into hip fat binary object
+// CHECK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
+// CHECK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// CHECK-SAME: "-inputs={{.*}},[[IMG_DEV1]],[[IMG_DEV2]]" "-outputs=[[BUNDLE:.*hipfb]]"
+
+// CHECK: [[MC:".*llvm-mc"]] "-o" [[OBJBUNDLE:".*o"]] "{{.*}}.mcin" "--filetype=obj"
+
+// CHECK: [[AR:".*llvm-ar.*"]] "rcsD" "{{.*}}.out" [[A_OBJ_HOST]] [[B_OBJ_HOST]] [[OBJBUNDLE]]
diff --git a/clang/test/Driver/hip-toolchain-rdc.hip b/clang/test/Driver/hip-toolchain-rdc.hip
index f520236abcb9f..97d5e59c0c4b1 100644
--- a/clang/test/Driver/hip-toolchain-rdc.hip
+++ b/clang/test/Driver/hip-toolchain-rdc.hip
@@ -12,7 +12,23 @@
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
 // RUN: 2>&1 | FileCheck %s
 
-// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// emit objects for host side path
+// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-triple" "x86_64-unknown-linux-gnu"
+// CHECK-SAME: "-aux-triple" "amdgcn-amd-amdhsa"
+// CHECK-SAME: "-emit-obj"
+// CHECK-SAME: {{.*}} "-main-file-name" "a.cu"
+// CHECK-SAME: {{.*}} "-o" [[A_OBJ_HOST:".*o"]] "-x" "hip"
+// CHECK-SAME: {{.*}} [[A_SRC:".*a.cu"]]
+
+// CHECK: [[CLANG]] "-cc1" "-triple" "x86_64-unknown-linux-gnu"
+// CHECK-SAME: "-aux-triple" "amdgcn-amd-amdhsa"
+// CHECK-SAME: "-emit-obj"
+// CHECK-SAME: {{.*}} "-main-file-name" "b.hip"
+// CHECK-SAME: {{.*}} "-o" [[B_OBJ_HOST:".*o"]] "-x" "hip"
+// CHECK-SAME: {{.*}} [[B_SRC:".*b.hip"]]
+
+// generate image for device side path on gfx803
+// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-llvm-bc"
 // CHECK-SAME: {{.*}} "-main-file-name" "a.cu"
@@ -21,7 +37,7 @@
 // CHECK-SAME: "{{.*}}lib1.bc" "{{.*}}lib2.bc"
 // CHECK-SAME: "-target-cpu" "gfx803"
 // CHECK-SAME: {{.*}} "-o" [[A_BC1:".*bc"]] "-x" "hip"
-// CHECK-SAME: {{.*}} [[A_SRC:".*a.cu"]]
+// CHECK-SAME: {{.*}} [[A_SRC]]
 
 // CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
@@ -32,14 +48,16 @@
 // CHECK-SAME: "{{.*}}lib1.bc" "{{.*}}lib2.bc"
 // CHECK-SAME: "-target-cpu" "gfx803"
 // CHECK-SAME: {{.*}} "-o" [[B_BC1:".*bc"]] "-x" "hip"
-// CHECK-SAME: {{.*}} [[B_SRC:".*b.hip"]]
+// CHECK-SAME: {{.*}} [[B_SRC]]
 
 // CHECK-NOT: "*.llvm-link"
 // CHECK-NOT: ".*opt"
 // CHECK-NOT: ".*llc"
-// CHECK: {{".*lld.*"}} {{.*}} "-mllvm" "-amdgpu-internalize-symbols"
+// CHECK: {{".*lld.*"}} {{.*}} "-plugin-opt=-amdgpu-internalize-symbols"
+// CHECK-SAME: "-plugin-opt=mcpu=gfx803"
 // CHECK-SAME: "-o" "[[IMG_DEV1:.*.out]]" [[A_BC1]] [[B_BC1]]
 
+// generate image for device side path on gfx900
 // CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-llvm-bc"
@@ -63,26 +81,16 @@
 // CHECK-NOT: "*.llvm-link"
 // CHECK-NOT: ".*opt"
 // CHECK-NOT: ".*llc"
-// CHECK: {{".*lld.*"}} {{.*}} "-mllvm" "-amdgpu-internalize-symbols"
+// CHECK: {{".*lld.*"}} {{.*}} "-plugin-opt=-amdgpu-internalize-symbols"
+// CHECK-SAME: "-plugin-opt=mcpu=gfx900"
 // CHECK-SAME: "-o" "[[IMG_DEV2:.*.out]]" [[A_BC2]] [[B_BC2]]
 
-// CHECK: [[CLANG]] "-cc1" "-triple" "x86_64-unknown-linux-gnu"
-// CHECK-SAME: "-aux-triple" "amdgcn-amd-amdhsa"
-// CHECK-SAME: "-emit-obj"
-// CHECK-SAME: {{.*}} "-main-file-name" "a.cu"
-// CHECK-SAME: {{.*}} "-o" [[A_OBJ_HOST:".*o"]] "-x" "hip"
-// CHECK-SAME: {{.*}} [[A_SRC]]
-
-// CHECK: [[CLANG]] "-cc1" "-triple" "x86_64-unknown-linux-gnu"
-// CHECK-SAME: "-aux-triple" "amdgcn-amd-amdhsa"
-// CHECK-SAME: "-emit-obj"
-// CHECK-SAME: {{.*}} "-main-file-name" "b.hip"
-// CHECK-SAME: {{.*}} "-o" [[B_OBJ_HOST:".*o"]] "-x" "hip"
-// CHECK-SAME: {{.*}} [[B_SRC]]
-
+// combine images generated into hip fat binary object
 // CHECK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
 // CHECK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
 // CHECK-SAME: "-inputs={{.*}},[[IMG_DEV1]],[[IMG_DEV2]]" "-outputs=[[BUNDLE:.*hipfb]]"
 
-// CHECK: [[LD:".*ld.*"]] {{.*}} [[A_OBJ_HOST]] [[B_OBJ_HOST]]
-// CHECK-SAME: {{.*}} "-T" "{{.*}}.lk"
+// CHECK: [[MC:".*llvm-mc"]] "-o" [[OBJBUNDLE:".*o"]] "{{.*}}.mcin" "--filetype=obj"
+
+// output the executable
+// CHECK: [[LD:".*ld.*"]] {{.*}}"-o" "a.out" {{.*}} [[A_OBJ_HOST]] [[B_OBJ_HOST]] [[OBJBUNDLE]]
diff --git a/clang/test/Driver/hip-version.hip b/clang/test/Driver/hip-version.hip
new file mode 100644
index 0000000000000..eb1295210cfc1
--- /dev/null
+++ b/clang/test/Driver/hip-version.hip
@@ -0,0 +1,42 @@
+// REQUIRES: clang-driver
+// REQUIRES: x86-registered-target
+// REQUIRES: amdgpu-registered-target
+
+// RUN: %clang -v --rocm-path=%S/Inputs/rocm 2>&1 \
+// RUN:   | FileCheck -check-prefixes=FOUND %s
+
+// RUN: %clang -v --rocm-path=%S/Inputs/rocm 2>&1 \
+// RUN:   -target amdgcn-amd-amdhsa \
+// RUN:   | FileCheck -check-prefixes=FOUND %s
+
+// FOUND: Found HIP installation: {{.*Inputs.*rocm}}, version 3.6.20214-a2917cd
+
+// When --rocm-path is set and .hipVersion is not found, use default version
+
+// RUN: %clang -v --rocm-path=%S 2>&1 \
+// RUN:   | FileCheck -check-prefixes=DEFAULT %s
+
+// RUN: %clang -v --rocm-path=%S 2>&1 \
+// RUN:   -target amdgcn-amd-amdhsa \
+// RUN:   | FileCheck -check-prefixes=DEFAULT %s
+
+// DEFAULT: Found HIP installation: {{.*Driver}}, version 3.5.
+
+// RUN: %clang -v --rocm-path=%S --hip-version=3.7.0 2>&1 \
+// RUN:   | FileCheck -check-prefixes=SPECIFIED %s
+
+// RUN: %clang -v --rocm-path=%S --hip-version=3.7.0 2>&1 \
+// RUN:   -target amdgcn-amd-amdhsa \
+// RUN:   | FileCheck -check-prefixes=SPECIFIED %s
+
+// SPECIFIED: Found HIP installation: {{.*Driver}}, version 3.7.0
+
+// RUN: %clang -v --rocm-path=%S --hip-version=3.7 2>&1 \
+// RUN:   | FileCheck -check-prefixes=SPECIFIED2 %s
+
+// SPECIFIED2: Found HIP installation: {{.*Driver}}, version 3.7.0
+
+// RUN: not %clang -v --rocm-path=%S --hip-version=x.y 2>&1 \
+// RUN:   | FileCheck -check-prefixes=INVALID %s
+
+// INVALID: error: invalid value 'x.y' in '--hip-version=x.y'
diff --git a/clang/test/Driver/integrated-as.s b/clang/test/Driver/integrated-as.s
index 0194a3d5a4382..05999cfe002b5 100644
--- a/clang/test/Driver/integrated-as.s
+++ b/clang/test/Driver/integrated-as.s
@@ -27,19 +27,19 @@
 // XA_INCLUDE2: "-Ifoo_dir"
 
 // RUN: %clang -### -target x86_64--- -c -integrated-as %s -gdwarf-4 -gdwarf-2 2>&1 | FileCheck --check-prefix=DWARF2 %s
-// DWARF2: "-debug-info-kind=limited" "-dwarf-version=2"
+// DWARF2: "-debug-info-kind=constructor" "-dwarf-version=2"
 
 // RUN: %clang -### -target x86_64--- -c -integrated-as %s -gdwarf-3 2>&1 | FileCheck --check-prefix=DWARF3 %s
-// DWARF3: "-debug-info-kind=limited" "-dwarf-version=3"
+// DWARF3: "-debug-info-kind=constructor" "-dwarf-version=3"
 
 // RUN: %clang -### -target x86_64--- -c -integrated-as %s -gdwarf-4 2>&1 | FileCheck --check-prefix=DWARF4 %s
-// DWARF4: "-debug-info-kind=limited" "-dwarf-version=4"
+// DWARF4: "-debug-info-kind=constructor" "-dwarf-version=4"
 
 // RUN: %clang -### -target x86_64--- -c -integrated-as %s -Xassembler -gdwarf-2 2>&1 | FileCheck --check-prefix=DWARF2XASSEMBLER %s
-// DWARF2XASSEMBLER: "-debug-info-kind=limited" "-dwarf-version=2"
+// DWARF2XASSEMBLER: "-debug-info-kind=constructor" "-dwarf-version=2"
 
 // RUN: %clang -### -target x86_64--- -c -integrated-as %s -Wa,-gdwarf-2 2>&1 | FileCheck --check-prefix=DWARF2WA %s
-// DWARF2WA: "-debug-info-kind=limited" "-dwarf-version=2"
+// DWARF2WA: "-debug-info-kind=constructor" "-dwarf-version=2"
 
 // A dwarf version number that driver can't parse is just stuffed in.
 // RUN: %clang -### -target x86_64--- -c -integrated-as %s -Wa,-gdwarf-huh 2>&1 | FileCheck --check-prefix=BOGODWARF %s
diff --git a/clang/test/Driver/macos-apple-silicon-slice-link-libs.cpp b/clang/test/Driver/macos-apple-silicon-slice-link-libs.cpp
new file mode 100644
index 0000000000000..522fda34987e9
--- /dev/null
+++ b/clang/test/Driver/macos-apple-silicon-slice-link-libs.cpp
@@ -0,0 +1,42 @@
+// RUN: %clang -### -target arm64-apple-macos10.7 %s 2>&1 | FileCheck -check-prefix=ARM64-10_7 %s
+// RUN: %clang -### -target x86_64-apple-macos10.7 %s 2>&1 | FileCheck -check-prefix=x86_64-10_7 %s
+
+// RUN: %clang -### -target arm64-apple-macos10.5 %s 2>&1 | FileCheck -check-prefix=ARM64-10_5 %s
+// RUN: %clang -### -target x86_64-apple-macos10.5 %s 2>&1 | FileCheck -check-prefix=x86_64-10_5 %s
+
+// RUN: %clang -### -target arm64-apple-macos10.4 %s 2>&1 | FileCheck -check-prefix=ARM64-10_4 %s
+// RUN: %clang -### -target x86_64-apple-macos10.4 %s 2>&1 | FileCheck -check-prefix=x86_64-10_4 %s
+
+// RUN: %clang -### -target arm64-apple-macos10.5 -bundle %s 2>&1 | FileCheck -check-prefix=ARM64-BUNDLE %s
+// RUN: %clang -### -target x86_64-apple-macos10.5 -bundle %s 2>&1 | FileCheck -check-prefix=x86_64-BUNDLE %s
+
+// RUN: %clang -### -target arm64-apple-macos10.5 -dynamiclib %s 2>&1 | FileCheck -check-prefix=ARM64-10_5-DYNAMICLIB %s
+// RUN: %clang -### -target x86_64-apple-macos10.5 -dynamiclib %s 2>&1 | FileCheck -check-prefix=x86_64-10_5-DYNAMICLIB %s
+
+// RUN: %clang -### -target arm64-apple-macos10.4 -dynamiclib %s 2>&1 | FileCheck -check-prefix=ARM64-10_4-DYNAMICLIB %s
+// RUN: %clang -### -target arm64-apple-darwin8 -dynamiclib %s 2>&1 | FileCheck -check-prefix=ARM64-10_4-DYNAMICLIB %s
+// RUN: %clang -### -target x86_64-apple-macos10.4 -dynamiclib %s 2>&1 | FileCheck -check-prefix=x86_64-10_4-DYNAMICLIB %s
+// RUN: %clang -### -target x86_64-apple-darwin8 -dynamiclib %s 2>&1 | FileCheck -check-prefix=x86_64-10_4-DYNAMICLIB %s
+
+// RUN: %clang -### -target arm64-apple-macos10.7 -static %s 2>&1 | FileCheck -check-prefix=STATIC %s
+// RUN: %clang -### -target x86_64-apple-macos10.7 -static %s 2>&1 | FileCheck -check-prefix=STATIC %s
+
+// ARM64-10_7-NOT: -lcrt1.10.6.o
+// x86_64-10_7:    -lcrt1.10.6.o
+
+// ARM64-10_5-NOT: -lcrt1.10.5.o
+// x86_64-10_5:    -lcrt1.10.5.o
+
+// ARM64-10_4-NOT: -lcrt1.o
+// x86_64-10_4:    -lcrt1.o
+
+// ARM64-BUNDLE-NOT: -lbundle1.o
+// x86_64-BUNDLE:    -lbundle1.o
+
+// ARM64-10_5-DYNAMICLIB-NOT: -ldylib1.10.5.o
+// x86_64-10_5-DYNAMICLIB:    -ldylib1.10.5.o
+
+// ARM64-10_4-DYNAMICLIB-NOT: -ldylib1.o
+// x86_64-10_4-DYNAMICLIB:    -ldylib1.o
+
+// STATIC: -lcrt0.o
diff --git a/clang/test/Driver/modules-cache-path.m b/clang/test/Driver/modules-cache-path.m
index 419d6d479f6b2..51df6739a5055 100644
--- a/clang/test/Driver/modules-cache-path.m
+++ b/clang/test/Driver/modules-cache-path.m
@@ -1,5 +1,2 @@
-// RUN: env USERNAME=asdf LOGNAME=asdf %clang -fmodules -### %s 2>&1 | FileCheck %s -check-prefix=CHECK-SET
-// CHECK-SET: -fmodules-cache-path={{.*}}org.llvm.clang.asdf{{[/\\]+}}ModuleCache
-
 // RUN: %clang -fmodules -### %s 2>&1 | FileCheck %s -check-prefix=CHECK-DEFAULT
-// CHECK-DEFAULT: -fmodules-cache-path={{.*}}org.llvm.clang.{{[A-Za-z0-9_]*[/\\]+}}ModuleCache
+// CHECK-DEFAULT: -fmodules-cache-path={{.*}}clang{{[/\\]+}}ModuleCache
diff --git a/clang/test/Driver/myriad-toolchain.c b/clang/test/Driver/myriad-toolchain.c
index 215a02fd0dec1..a4bd260a14986 100644
--- a/clang/test/Driver/myriad-toolchain.c
+++ b/clang/test/Driver/myriad-toolchain.c
@@ -83,7 +83,7 @@
 // NOSTDLIB-NOT: "-lc"
 
 // RUN: %clang -### -c -g %s -target sparc-myriad 2>&1 | FileCheck -check-prefix=G_SPARC %s
-// G_SPARC: "-debug-info-kind=limited" "-dwarf-version=2"
+// G_SPARC: "-debug-info-kind=constructor" "-dwarf-version=2"
 
 // RUN: %clang -### -c %s -target sparc-myriad-rtems -fuse-init-array 2>&1 \
 // RUN: | FileCheck -check-prefix=USE-INIT-ARRAY %s
diff --git a/clang/test/Driver/openmp-offload-gpu.c b/clang/test/Driver/openmp-offload-gpu.c
index 6415f1d61b720..3ddd6446d1176 100644
--- a/clang/test/Driver/openmp-offload-gpu.c
+++ b/clang/test/Driver/openmp-offload-gpu.c
@@ -241,7 +241,7 @@
 
 // HAS_DEBUG-NOT: warning: debug
 // HAS_DEBUG: "-triple" "nvptx64-nvidia-cuda"
-// HAS_DEBUG-SAME: "-debug-info-kind={{limited|line-tables-only}}"
+// HAS_DEBUG-SAME: "-debug-info-kind={{constructor|line-tables-only}}"
 // HAS_DEBUG-SAME: "-dwarf-version=2"
 // HAS_DEBUG-SAME: "-fopenmp-is-device"
 // HAS_DEBUG: ptxas
diff --git a/clang/test/Driver/program-path-priority.c b/clang/test/Driver/program-path-priority.c
new file mode 100644
index 0000000000000..ba893e7e2e2cd
--- /dev/null
+++ b/clang/test/Driver/program-path-priority.c
@@ -0,0 +1,109 @@
+/// Don't create symlinks on Windows
+// UNSUPPORTED: system-windows
+
+/// Check the priority used when searching for tools
+/// Names and locations are usually in this order:
+/// <triple>-tool, tool, <default triple>-tool
+/// program path, PATH
+/// (from highest to lowest priority)
+/// A higher priority name found in a lower priority
+/// location will win over a lower priority name in a
+/// higher priority location.
+/// Prefix dirs (added with -B) override the location,
+/// so only name priority is accounted for, unless we fail to find
+/// anything at all in the prefix.
+
+/// Symlink clang to a new dir which will be its
+/// "program path" for these tests
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: ln -s %clang %t/clang
+
+/// No gccs at all, nothing is found
+// RUN: env "PATH=" %t/clang -### -target notreal-none-elf %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=NO_NOTREAL_GCC %s
+// NO_NOTREAL_GCC-NOT: notreal-none-elf-gcc
+// NO_NOTREAL_GCC-NOT: /gcc
+
+/// <triple>-gcc in program path is found
+// RUN: touch %t/notreal-none-elf-gcc && chmod +x %t/notreal-none-elf-gcc
+// RUN: env "PATH=" %t/clang -### -target notreal-none-elf %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=PROG_PATH_NOTREAL_GCC %s
+// PROG_PATH_NOTREAL_GCC: notreal-none-elf-gcc
+
+/// <triple>-gcc on the PATH is found
+// RUN: mkdir -p %t/env
+// RUN: rm %t/notreal-none-elf-gcc
+// RUN: touch %t/env/notreal-none-elf-gcc && chmod +x %t/env/notreal-none-elf-gcc
+// RUN: env "PATH=%t/env/" %t/clang -### -target notreal-none-elf %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=ENV_PATH_NOTREAL_GCC %s
+// ENV_PATH_NOTREAL_GCC: env/notreal-none-elf-gcc
+
+/// <triple>-gcc in program path is preferred to one on the PATH
+// RUN: touch %t/notreal-none-elf-gcc && chmod +x %t/notreal-none-elf-gcc
+// RUN: env "PATH=%t/env/" %t/clang -### -target notreal-none-elf %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=BOTH_NOTREAL_GCC %s
+// BOTH_NOTREAL_GCC: notreal-none-elf-gcc
+// BOTH_NOTREAL_GCC-NOT: env/notreal-none-elf-gcc
+
+/// On program path, <triple>-gcc is preferred to plain gcc
+// RUN: touch %t/gcc && chmod +x %t/gcc
+// RUN: env "PATH=" %t/clang -### -target notreal-none-elf %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=NOTREAL_GCC_PREFERRED %s
+// NOTREAL_GCC_PREFERRED: notreal-none-elf-gcc
+// NOTREAL_GCC_PREFERRED-NOT: /gcc
+
+/// <triple>-gcc on the PATH is preferred to gcc in program path
+// RUN: rm %t/notreal-none-elf-gcc
+// RUN: env "PATH=%t/env/" %t/clang -### -target notreal-none-elf %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=NOTREAL_PATH_OVER_GCC_PROG %s
+// NOTREAL_PATH_OVER_GCC_PROG: env/notreal-none-elf-gcc
+// NOTREAL_PATH_OVER_GCC_PROG-NOT: /gcc
+
+/// <triple>-gcc on the PATH is preferred to gcc on the PATH
+// RUN: rm %t/gcc
+// RUN: touch %t/env/gcc && chmod +x %t/env/gcc
+// RUN: env "PATH=%t/env/" %t/clang -### -target notreal-none-elf %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=NOTREAL_PATH_OVER_GCC_PATH %s
+// NOTREAL_PATH_OVER_GCC_PATH: env/notreal-none-elf-gcc
+// NOTREAL_PATH_OVER_GCC_PATH-NOT: /gcc
+
+/// <default-triple>-gcc has lowest priority so <triple>-gcc
+/// on PATH beats default triple in program path
+/// Darwin triples have a version appended to them, even if set via
+/// LLVM_DEFAULT_TARGET_TRIPLE. So the only way to know for sure is to ask clang.
+// RUN: DEFAULT_TRIPLE=`%t/clang --version | grep "Target:" | cut -d ' ' -f2`
+// RUN: touch %t/$DEFAULT_TRIPLE-gcc && chmod +x %t/$DEFAULT_TRIPLE-gcc
+// RUN: env "PATH=%t/env/" %t/clang -### -target notreal-none-elf %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=DEFAULT_TRIPLE_GCC %s
+// DEFAULT_TRIPLE_GCC: env/notreal-none-elf-gcc
+
+/// plain gcc on PATH beats default triple in program path
+// RUN: rm %t/env/notreal-none-elf-gcc
+// RUN: env "PATH=%t/env/" %t/clang -### -target notreal-none-elf %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=DEFAULT_TRIPLE_NO_NOTREAL %s
+// DEFAULT_TRIPLE_NO_NOTREAL: env/gcc
+// DEFAULT_TRIPLE_NO_NOTREAL-NOT: -gcc
+
+/// default triple only chosen when no others are present
+// RUN: rm %t/env/gcc
+// RUN: env "PATH=%t/env/" %t/clang -### -target notreal-none-elf %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=DEFAULT_TRIPLE_NO_OTHERS %s
+// DEFAULT_TRIPLE_NO_OTHERS: -gcc
+// DEFAULT_TRIPLE_NO_OTHERS-NOT: notreal-none-elf-gcc
+// DEFAULT_TRIPLE_NO_OTHERS-NOT: /gcc
+
+/// -B paths are searched separately so default triple will win
+/// if put in one of those even if other paths have higher priority names
+// RUN: mkdir -p %t/prefix
+// RUN: mv %t/$DEFAULT_TRIPLE-gcc %t/prefix
+// RUN: touch %t/notreal-none-elf-gcc && chmod +x %t/notreal-none-elf-gcc
+// RUN: env "PATH=" %t/clang -### -target notreal-none-elf %s -B %t/prefix 2>&1 | \
+// RUN:   FileCheck --check-prefix=DEFAULT_TRIPLE_IN_PREFIX %s
+// DEFAULT_TRIPLE_IN_PREFIX: prefix/{{.*}}-gcc
+// DEFAULT_TRIPLE_IN_PREFIX-NOT: notreal-none-elf-gcc
+
+/// Only if there is nothing in the prefix will we search other paths
+// RUN: rm %t/prefix/$DEFAULT_TRIPLE-gcc
+// RUN: env "PATH=" %t/clang -### -target notreal-none-elf %s -B %t/prefix 2>&1 | \
+// RUN:   FileCheck --check-prefix=EMPTY_PREFIX_DIR %s
+// EMPTY_PREFIX_DIR: notreal-none-elf-gcc
diff --git a/clang/test/Driver/riscv-arch.c b/clang/test/Driver/riscv-arch.c
index ba93be9caa36e..e3062feb7dee0 100644
--- a/clang/test/Driver/riscv-arch.c
+++ b/clang/test/Driver/riscv-arch.c
@@ -360,3 +360,22 @@
 // RUN: -fsyntax-only 2>&1 | FileCheck -check-prefix=RV32-EXPERIMENTAL-ZBB-ZBP %s
 // RV32-EXPERIMENTAL-ZBB-ZBP: "-target-feature" "+experimental-zbb"
 // RV32-EXPERIMENTAL-ZBB-ZBP: "-target-feature" "+experimental-zbp"
+
+// RUN: %clang -target riscv32-unknown-elf -march=rv32iv -### %s -c 2>&1 | \
+// RUN:   FileCheck -check-prefix=RV32-EXPERIMENTAL-V-NOFLAG %s
+// RV32-EXPERIMENTAL-V-NOFLAG: error: invalid arch name 'rv32iv'
+// RV32-EXPERIMENTAL-V-NOFLAG: requires '-menable-experimental-extensions'
+
+// RUN: %clang -target riscv32-unknown-elf -march=rv32iv -menable-experimental-extensions -### %s -c 2>&1 | \
+// RUN:   FileCheck -check-prefix=RV32-EXPERIMENTAL-V-NOVERS %s
+// RV32-EXPERIMENTAL-V-NOVERS: error: invalid arch name 'rv32iv'
+// RV32-EXPERIMENTAL-V-NOVERS: experimental extension requires explicit version number
+
+// RUN: %clang -target riscv32-unknown-elf -march=rv32iv0p1 -menable-experimental-extensions -### %s -c 2>&1 | \
+// RUN:   FileCheck -check-prefix=RV32-EXPERIMENTAL-V-BADVERS %s
+// RV32-EXPERIMENTAL-V-BADVERS: error: invalid arch name 'rv32iv0p1'
+// RV32-EXPERIMENTAL-V-BADVERS: unsupported version number 0.1 for experimental extension
+
+// RUN: %clang -target riscv32-unknown-elf -march=rv32iv0p8 -menable-experimental-extensions -### %s -c 2>&1 | \
+// RUN:   FileCheck -check-prefix=RV32-EXPERIMENTAL-V-GOODVERS %s
+// RV32-EXPERIMENTAL-V-GOODVERS: "-target-feature" "+experimental-v"
diff --git a/clang/test/Driver/rocm-detect.cl b/clang/test/Driver/rocm-detect.cl
index 75378bf003bee..2fac3aa7cc8d2 100644
--- a/clang/test/Driver/rocm-detect.cl
+++ b/clang/test/Driver/rocm-detect.cl
@@ -16,6 +16,6 @@
 // RUN:   | FileCheck -check-prefixes=COMMON,GFX902,NODEFAULTLIBS %s
 
 
-// GFX902-DEFAULTLIBS: error: cannot find device library for gfx902. Provide path to different ROCm installation via --rocm-path, or pass -nogpulib to build without linking default libraries.
+// GFX902-DEFAULTLIBS: error: cannot find ROCm device library for gfx902. Provide its path via --rocm-path or --rocm-device-lib-path, or pass -nogpulib to build without ROCm device library
 
 // NODEFAULTLIBS-NOT: error: cannot find
diff --git a/clang/test/Driver/rocm-detect.hip b/clang/test/Driver/rocm-detect.hip
index 9490ec9ba3762..3a3a028facb48 100644
--- a/clang/test/Driver/rocm-detect.hip
+++ b/clang/test/Driver/rocm-detect.hip
@@ -22,6 +22,6 @@
 // RUN:   | FileCheck -check-prefixes=COMMON,GFX902,NODEFAULTLIBS %s
 
 
-// GFX902-DEFAULTLIBS: error: cannot find device library for gfx902. Provide path to different ROCm installation via --rocm-path, or pass -nogpulib to build without linking default libraries.
+// GFX902-DEFAULTLIBS: error: cannot find ROCm device library for gfx902. Provide its path via --rocm-path or --rocm-device-lib-path, or pass -nogpulib to build without ROCm device library
 
 // NODEFAULTLIBS-NOT: error: cannot find
diff --git a/clang/test/Driver/rocm-not-found.cl b/clang/test/Driver/rocm-not-found.cl
index ee931971d9e6a..122f826714b51 100644
--- a/clang/test/Driver/rocm-not-found.cl
+++ b/clang/test/Driver/rocm-not-found.cl
@@ -5,7 +5,7 @@
 
 // RUN: %clang -### --sysroot=%s/no-rocm-there -target amdgcn--amdhsa %s 2>&1 | FileCheck %s --check-prefix ERR
 // RUN: %clang -### --rocm-path=%s/no-rocm-there -target amdgcn--amdhsa %s 2>&1 | FileCheck %s --check-prefix ERR
-// ERR: cannot find ROCm installation. Provide its path via --rocm-path, or pass -nogpulib and -nogpuinc to build without ROCm device library and HIP includes.
+// ERR: cannot find ROCm device library. Provide its path via --rocm-path or --rocm-device-lib-path, or pass -nogpulib to build without ROCm device library
 
 // Accept nogpulib or nostdlib for OpenCL.
 // RUN: %clang -### -nogpulib --rocm-path=%s/no-rocm-there %s 2>&1 | FileCheck %s --check-prefix OK
diff --git a/clang/test/Driver/split-debug.c b/clang/test/Driver/split-debug.c
index d40207d5ae3b6..70f8d91d48e01 100644
--- a/clang/test/Driver/split-debug.c
+++ b/clang/test/Driver/split-debug.c
@@ -68,18 +68,18 @@
 // RUN: FileCheck -check-prefix=CHECK-NOINLINE-WITHOUT-SPLIT < %t %s
 //
 // CHECK-NOINLINE-WITHOUT-SPLIT: "-fno-split-dwarf-inlining"
-// CHECK-NOINLINE-WITHOUT-SPLIT: "-debug-info-kind=limited"
+// CHECK-NOINLINE-WITHOUT-SPLIT: "-debug-info-kind=constructor"
 
 // RUN: %clang -target x86_64-unknown-linux-gnu -gmlt -gsplit-dwarf -fno-split-dwarf-inlining -S -### %s 2> %t
 // RUN: FileCheck -check-prefix=CHECK-SPLIT-WITH-GMLT < %t %s
 //
-// CHECK-SPLIT-WITH-GMLT: "-debug-info-kind=limited"
+// CHECK-SPLIT-WITH-GMLT: "-debug-info-kind=constructor"
 // CHECK-SPLIT-WITH-GMLT: "-split-dwarf-output"
 
 // RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf -fno-split-dwarf-inlining -S -### %s 2> %t
 // RUN: FileCheck -check-prefix=CHECK-SPLIT-WITH-NOINL < %t %s
 //
-// CHECK-SPLIT-WITH-NOINL: "-debug-info-kind=limited"
+// CHECK-SPLIT-WITH-NOINL: "-debug-info-kind=constructor"
 // CHECK-SPLIT-WITH-NOINL: "-split-dwarf-output"
 
 // RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf -gmlt -fsplit-dwarf-inlining -S -### %s 2> %t
@@ -92,7 +92,7 @@
 // RUN: %clang -target x86_64-unknown-linux-gnu -gmlt -gsplit-dwarf -S -### %s 2> %t
 // RUN: FileCheck -check-prefix=CHECK-SPLIT-OVER-GMLT < %t %s
 //
-// CHECK-SPLIT-OVER-GMLT: "-debug-info-kind=limited"
+// CHECK-SPLIT-OVER-GMLT: "-debug-info-kind=constructor"
 // CHECK-SPLIT-OVER-GMLT: "-split-dwarf-file"
 // CHECK-SPLIT-OVER-GMLT: "-split-dwarf-output"
 
@@ -117,6 +117,6 @@
 // RUN: %clang -target x86_64-unknown-linux-gnu -g0 -gsplit-dwarf=split -S -### %s 2> %t
 // RUN: FileCheck -check-prefix=CHECK-SPLIT-OVER-G0 < %t %s
 //
-// CHECK-SPLIT-OVER-G0: "-debug-info-kind=limited"
+// CHECK-SPLIT-OVER-G0: "-debug-info-kind=constructor"
 // CHECK-SPLIT-OVER-G0: "-split-dwarf-file"
 // CHECK-SPLIT-OVER-G0: "-split-dwarf-output"
diff --git a/clang/test/Driver/ve-toolchain.c b/clang/test/Driver/ve-toolchain.c
new file mode 100644
index 0000000000000..7b93797ee5970
--- /dev/null
+++ b/clang/test/Driver/ve-toolchain.c
@@ -0,0 +1,82 @@
+/// Check the behavior of toolchain for NEC Aurora VE
+/// REQUIRES: ve-registered-target
+
+///-----------------------------------------------------------------------------
+/// Checking dwarf-version
+
+// RUN: %clang -### -g -target ve %s 2>&1 | FileCheck -check-prefix=DWARF_VER %s
+// DWARF_VER: "-dwarf-version=4"
+
+///-----------------------------------------------------------------------------
+/// Checking dynamic-linker
+
+// RUN: %clang -### -target ve %s 2>&1 | FileCheck -check-prefix=DYNLINKER %s
+// DYNLINKER: nld{{.*}} "-dynamic-linker" "/opt/nec/ve/lib/ld-linux-ve.so.1"
+
+///-----------------------------------------------------------------------------
+/// Checking VE specific option
+
+// RUN: %clang -### -target ve %s 2>&1 | FileCheck -check-prefix=VENLDOPT %s
+// VENLDOPT: nld{{.*}} "-z" "max-page-size=0x4000000"
+
+///-----------------------------------------------------------------------------
+/// Checking include-path
+
+// RUN: %clang -### -target ve %s 2>&1 | FileCheck -check-prefix=DEFINC %s
+// DEFINC: clang{{.*}} "-cc1"
+// DEFINC: "-nostdsysteminc"
+// DEFINC: "-internal-isystem" "{{.*}}/lib/clang/{{[0-9.]*}}/include"
+// DEFINC: "-internal-isystem" "/opt/nec/ve/include"
+
+// RUN: %clang -### -target ve %s -nostdlibinc 2>&1 | \
+// RUN:    FileCheck -check-prefix=NOSTDLIBINC %s
+// NOSTDLIBINC: clang{{.*}} "-cc1"
+// NOSTDLIBINC: "-internal-isystem" "{{.*}}/lib/clang/{{[0-9.]*}}/include"
+// NOSTDLIBINC-NOT: "-internal-isystem" "/opt/nec/ve/include"
+
+// RUN: %clang -### -target ve %s -nobuiltininc 2>&1 | \
+// RUN:    FileCheck -check-prefix=NOBUILTININC %s
+// NOBUILTININC: clang{{.*}} "-cc1"
+// NOBUILTININC: "-nobuiltininc"
+// NOBUILTININC-NOT: "-internal-isystem" "{{.*}}/lib/clang/{{[0-9.]*}}/include"
+// NOBUILTININC: "-internal-isystem" "/opt/nec/ve/include"
+
+// RUN: %clang -### -target ve %s -nostdinc 2>&1 | \
+// RUN:    FileCheck -check-prefix=NOSTDINC %s
+// NOSTDINC: clang{{.*}} "-cc1"
+// NOSTDINC: "-nobuiltininc"
+// NOSTDINC-NOT: "-internal-isystem" "{{.*}}/lib/clang/{{[0-9.]*}}/include"
+// NOSTDINC-NOT: "-internal-isystem" "/opt/nec/ve/include"
+
+///-----------------------------------------------------------------------------
+/// Checking -fuse-init-array
+
+// RUN: %clang -### -target ve %s 2>&1 | FileCheck -check-prefix=DEFINITARRAY %s
+// DEFINITARRAY: clang{{.*}} "-cc1"
+// DEFINITARRAY-NOT: "-fuse-init-array"
+
+// RUN: %clang -### -target ve %s -fno-use-init-array 2>&1 | \
+// RUN:     FileCheck -check-prefix=NOTINITARRAY %s
+// NOTINITARRAY: clang{{.*}} "-cc1"
+// NOTINITARRAY: "-fno-use-init-array"
+
+///-----------------------------------------------------------------------------
+/// Checking exceptions
+
+// RUN: %clang -### -target ve %s 2>&1 | FileCheck -check-prefix=DEFEXCEPTION %s
+// DEFEXCEPTION: clang{{.*}} "-cc1"
+// DEFEXCEPTION: "-fsjlj-exceptions"
+
+///-----------------------------------------------------------------------------
+/// Passing -fintegrated-as
+
+// RUN: %clang -### -target ve -x assembler %s 2>&1 | \
+// RUN:    FileCheck -check-prefix=NAS_LINK %s
+// RUN: %clang -### -target ve -fintegrated-as -x assembler %s 2>&1 | \
+// RUN:    FileCheck -check-prefix=AS_LINK %s
+
+// NAS_LINK: nas{{.*}}
+// NAS_LINK: nld{{.*}}
+
+// AS_LINK: clang{{.*}} "-cc1as"
+// AS_LINK: nld{{.*}}
diff --git a/clang/test/Driver/windows-thumbv7em.cpp b/clang/test/Driver/windows-thumbv7em.cpp
new file mode 100644
index 0000000000000..5d7c00b31fd16
--- /dev/null
+++ b/clang/test/Driver/windows-thumbv7em.cpp
@@ -0,0 +1,8 @@
+// RUN: %clang -target thumb-none-windows-eabi-coff -mcpu=cortex-m7 -### -c %s 2>&1 \
+// RUN: | FileCheck %s --check-prefix CHECK-V7
+// CHECK-V7-NOT: error: the target architecture 'thumbv7em' is not supported by the target 'thumbv7em-none-windows-eabi'
+
+// RUN: %clang -target thumb-none-windows-eabi-coff -mcpu=cortex-m1 -### -c %s 2>&1 \
+// RUN: | FileCheck %s --check-prefix CHECK-V6
+// CHECK-V6: error: the target architecture 'thumbv6m' is not supported by the target 'thumbv6m-none-windows-eabi'
+
diff --git a/clang/test/Driver/x86-target-features.c b/clang/test/Driver/x86-target-features.c
index b96eed287bd9f..85a9374ab9057 100644
--- a/clang/test/Driver/x86-target-features.c
+++ b/clang/test/Driver/x86-target-features.c
@@ -178,6 +178,27 @@
 // RUN: %clang -target i386-linux-gnu -mlvi-hardening -mretpoline-external-thunk %s -### -o %t.o 2>&1 | FileCheck -check-prefix=LVIHARDENING-RETPOLINE-EXTERNAL-THUNK %s
 // LVIHARDENING-RETPOLINE-EXTERNAL-THUNK: error: invalid argument 'mretpoline-external-thunk' not allowed with 'mlvi-hardening'
 
+// RUN: %clang -target i386-linux-gnu -mseses %s -### -o %t.o 2>&1 | FileCheck -check-prefix=SESES %s
+// RUN: %clang -target i386-linux-gnu -mno-seses %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-SESES %s
+// SESES: "-target-feature" "+seses"
+// SESES: "-target-feature" "+lvi-cfi"
+// NO-SESES-NOT: seses
+// NO-SESES-NOT: lvi-cfi
+
+// RUN: %clang -target i386-linux-gnu -mseses -mno-lvi-cfi %s -### -o %t.o 2>&1 | FileCheck -check-prefix=SESES-NOLVICFI %s
+// SESES-NOLVICFI: "-target-feature" "+seses"
+// SESES-NOLVICFI-NOT: lvi-cfi
+
+// RUN: %clang -target i386-linux-gnu -mseses -mspeculative-load-hardening %s -### -o %t.o 2>&1 | FileCheck -check-prefix=SESES-SLH %s
+// SESES-SLH: error: invalid argument 'mspeculative-load-hardening' not allowed with 'mseses'
+// RUN: %clang -target i386-linux-gnu -mseses -mretpoline %s -### -o %t.o 2>&1 | FileCheck -check-prefix=SESES-RETPOLINE %s
+// SESES-RETPOLINE: error: invalid argument 'mretpoline' not allowed with 'mseses'
+// RUN: %clang -target i386-linux-gnu -mseses -mretpoline-external-thunk %s -### -o %t.o 2>&1 | FileCheck -check-prefix=SESES-RETPOLINE-EXTERNAL-THUNK %s
+// SESES-RETPOLINE-EXTERNAL-THUNK: error: invalid argument 'mretpoline-external-thunk' not allowed with 'mseses'
+
+// RUN: %clang -target i386-linux-gnu -mseses -mlvi-hardening %s -### -o %t.o 2>&1 | FileCheck -check-prefix=SESES-LVIHARDENING %s
+// SESES-LVIHARDENING: error: invalid argument 'mlvi-hardening' not allowed with 'mseses'
+
 // RUN: %clang -target i386-linux-gnu -mwaitpkg %s -### -o %t.o 2>&1 | FileCheck -check-prefix=WAITPKG %s
 // RUN: %clang -target i386-linux-gnu -mno-waitpkg %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-WAITPKG %s
 // WAITPKG: "-target-feature" "+waitpkg"
@@ -232,3 +253,18 @@
 // RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-tsxldtrk %s -### -o %t.o 2>&1 | FileCheck --check-prefix=NO-TSXLDTRK %s
 // TSXLDTRK: "-target-feature" "+tsxldtrk"
 // NO-TSXLDTRK: "-target-feature" "-tsxldtrk"
+
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mamx-tile %s -### -o %t.o 2>&1 | FileCheck --check-prefix=AMX-TILE %s
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-amx-tile %s -### -o %t.o 2>&1 | FileCheck --check-prefix=NO-AMX-TILE %s
+// AMX-TILE: "-target-feature" "+amx-tile"
+// NO-AMX-TILE: "-target-feature" "-amx-tile"
+
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mamx-bf16 %s -### -o %t.o 2>&1 | FileCheck --check-prefix=AMX-BF16 %s
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-amx-bf16 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-AMX-BF16 %s
+// AMX-BF16: "-target-feature" "+amx-bf16"
+// NO-AMX-BF16: "-target-feature" "-amx-bf16"
+
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mamx-int8 %s -### -o %t.o 2>&1 | FileCheck --check-prefix=AMX-INT8 %s
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-amx-int8 %s -### -o %t.o 2>&1 | FileCheck --check-prefix=NO-AMX-INT8 %s
+// AMX-INT8: "-target-feature" "+amx-int8"
+// NO-AMX-INT8: "-target-feature" "-amx-int8"
diff --git a/clang/test/Frontend/fixed_point_add.c b/clang/test/Frontend/fixed_point_add.c
index be3d5a8f5e9e0..078976503ecd0 100644
--- a/clang/test/Frontend/fixed_point_add.c
+++ b/clang/test/Frontend/fixed_point_add.c
@@ -413,7 +413,7 @@ void SaturatedAddition() {
   // SIGNED-NEXT: [[USA_SAT_RESIZE:%[a-z0-9]+]] = zext i16 [[USA_SAT]] to i40
   // SIGNED-NEXT: [[I_RESIZE:%[a-z0-9]+]] = sext i32 [[I]] to i40
   // SIGNED-NEXT: [[I_UPSCALE:%[a-z0-9]+]] = shl i40 [[I_RESIZE]], 8
-  // SIGNED-NEXT: [[SUM:%[0-9]+]] = call i40 @llvm.uadd.sat.i40(i40 [[USA_SAT_RESIZE]], i40 [[I_UPSCALE]])
+  // SIGNED-NEXT: [[SUM:%[0-9]+]] = call i40 @llvm.sadd.sat.i40(i40 [[USA_SAT_RESIZE]], i40 [[I_UPSCALE]])
   // SIGNED-NEXT: [[USE_MAX:%[0-9]+]] = icmp sgt i40 [[SUM]], 65535
   // SIGNED-NEXT: [[RESULT:%[a-z0-9]+]] = select i1 [[USE_MAX]], i40 65535, i40 [[SUM]]
   // SIGNED-NEXT: [[USE_MIN:%[0-9]+]] = icmp slt i40 [[RESULT]], 0
@@ -422,7 +422,7 @@ void SaturatedAddition() {
   // UNSIGNED-NEXT: [[USA_SAT_RESIZE:%[a-z0-9]+]] = zext i16 [[USA_SAT]] to i39
   // UNSIGNED-NEXT: [[I_RESIZE:%[a-z0-9]+]] = sext i32 [[I]] to i39
   // UNSIGNED-NEXT: [[I_UPSCALE:%[a-z0-9]+]] = shl i39 [[I_RESIZE]], 7
-  // UNSIGNED-NEXT: [[SUM:%[0-9]+]] = call i39 @llvm.uadd.sat.i39(i39 [[USA_SAT_RESIZE]], i39 [[I_UPSCALE]])
+  // UNSIGNED-NEXT: [[SUM:%[0-9]+]] = call i39 @llvm.sadd.sat.i39(i39 [[USA_SAT_RESIZE]], i39 [[I_UPSCALE]])
   // UNSIGNED-NEXT: [[USE_MAX:%[0-9]+]] = icmp sgt i39 [[SUM]], 32767
   // UNSIGNED-NEXT: [[RESULT:%[a-z0-9]+]] = select i1 [[USE_MAX]], i39 32767, i39 [[SUM]]
   // UNSIGNED-NEXT: [[USE_MIN:%[0-9]+]] = icmp slt i39 [[RESULT]], 0
diff --git a/clang/test/Frontend/fixed_point_crash.c b/clang/test/Frontend/fixed_point_crash.c
new file mode 100644
index 0000000000000..12dc1944f018e
--- /dev/null
+++ b/clang/test/Frontend/fixed_point_crash.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -verify -ffixed-point %s
+
+union a {
+  _Accum x;
+  int i;
+};
+
+int fn1() {
+  union a m;
+  m.x = 5.6k;
+  return m.i;
+}
+
+int fn2() {
+  union a m;
+  m.x = 7, 5.6k; // expected-warning {{expression result unused}}
+  return m.x, m.i; // expected-warning {{expression result unused}}
+}
+
+_Accum acc = (0.5r, 6.9k); // expected-warning {{expression result unused}}
diff --git a/clang/test/Frontend/fixed_point_div.c b/clang/test/Frontend/fixed_point_div.c
index ec17e082558cb..b18ba59ae735c 100644
--- a/clang/test/Frontend/fixed_point_div.c
+++ b/clang/test/Frontend/fixed_point_div.c
@@ -1,6 +1,72 @@
 // RUN: %clang_cc1 -ffixed-point -triple x86_64-unknown-linux-gnu -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,SIGNED
 // RUN: %clang_cc1 -ffixed-point -triple x86_64-unknown-linux-gnu -fpadding-on-unsigned-fixed-point -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,UNSIGNED
 
+// Division between different fixed point types
+short _Accum sa_const = 1.0hk / 2.0hk;  // CHECK-DAG: @sa_const  = {{.*}}global i16 64, align 2
+_Accum a_const = 1.0hk / 2.0k;          // CHECK-DAG: @a_const   = {{.*}}global i32 16384, align 4
+long _Accum la_const = 1.0hk / 2.0lk;   // CHECK-DAG: @la_const  = {{.*}}global i64 1073741824, align 8
+short _Accum sa_const2 = 0.5hr / 2.0hk; // CHECK-DAG: @sa_const2  = {{.*}}global i16 32, align 2
+short _Accum sa_const3 = 0.5r / 2.0hk;  // CHECK-DAG: @sa_const3  = {{.*}}global i16 32, align 2
+short _Accum sa_const4 = 0.5lr / 2.0hk; // CHECK-DAG: @sa_const4  = {{.*}}global i16 32, align 2
+short _Accum sa_const5 = 2.0hk / 0.5lr; // CHECK-DAG: @sa_const5  = {{.*}}global i16 512, align 2
+
+// Unsigned division
+unsigned short _Accum usa_const = 3.0uhk / 2.0uhk;
+// CHECK-SIGNED-DAG:   @usa_const = {{.*}}global i16 192, align 2
+// CHECK-UNSIGNED-DAG: @usa_const = {{.*}}global i16 384, align 2
+
+// Unsigned / signed
+short _Accum sa_const6 = 1.0uhk / 2.0hk;
+// CHECK-DAG: @sa_const6 = {{.*}}global i16 64, align 2
+
+// Division with negative number
+short _Accum sa_const7 = 0.5hr / (-2.0hk);
+// CHECK-DAG: @sa_const7 = {{.*}}global i16 -32, align 2
+
+// Int division
+unsigned short _Accum usa_const2 = 2 / 0.5uhk;
+// CHECK-SIGNED-DAG:   @usa_const2 = {{.*}}global i16 512, align 2
+// CHECK-UNSIGNED-DAG: @usa_const2 = {{.*}}global i16 1024, align 2
+short _Accum sa_const8 = 2 / (-0.5hk);   // CHECK-DAG: @sa_const8 = {{.*}}global i16 -512, align 2
+short _Accum sa_const9 = 256 / 2.0hk;    // CHECK-DAG: @sa_const9 = {{.*}}global i16 16384, align 2
+long _Fract lf_const = 0.5lr / -1;       // CHECK-DAG: @lf_const  = {{.*}}global i32 -1073741824, align 4
+
+// Saturated division
+_Sat short _Accum sat_sa_const = (_Sat short _Accum)128.0hk / (-0.25hk);
+// CHECK-DAG: @sat_sa_const = {{.*}}global i16 -32768, align 2
+_Sat unsigned short _Accum sat_usa_const = (_Sat unsigned short _Accum)128.0uhk / (0.25uhk);
+// CHECK-SIGNED-DAG:   @sat_usa_const = {{.*}}global i16 65535, align 2
+// CHECK-UNSIGNED-DAG: @sat_usa_const = {{.*}}global i16 32767, align 2
+_Sat short _Accum sat_sa_const2 = (_Sat short _Accum)-128.0hk / (-0.0125hr);
+// CHECK-DAG: @sat_sa_const2 = {{.*}}global i16 32767, align 2
+_Sat unsigned short _Accum sat_usa_const2 = (_Sat unsigned short _Accum)128.0uhk / (-128);
+// CHECK-SIGNED-DAG:   @sat_usa_const2 = {{.*}}global i16 65535, align 2
+// CHECK-UNSIGNED-DAG: @sat_usa_const2 = {{.*}}global i16 32767, align 2
+_Sat unsigned short _Accum sat_usa_const3 = (_Sat unsigned short _Accum)0.5uhk / -1;
+// CHECK-DAG:   @sat_usa_const3 = {{.*}}global i16 0, align 2
+_Sat short _Accum sat_sa_const3 = (_Sat short _Accum)-128.0hk / 128;
+// CHECK-DAG: @sat_sa_const3 = {{.*}}global i16 -128, align 2
+_Sat short _Accum sat_sa_const4 = (_Sat short _Accum)-25.7hk / 0.1lk;
+// CHECK-DAG: @sat_sa_const4 = {{.*}}global i16 -32768, align 2
+
+// Some more cases
+short _Accum sa_const10 = 255.9921875hk / 255.9921875hk;
+// CHECK-DAG: @sa_const10 = {{.*}}global i16 128, align 2
+short _Accum sat_sa_const5 = (_Sat short _Accum)(-255.0hk - 1.0hk) / 0.0078125hk;
+// CHECK-DAG: @sat_sa_const5 = {{.*}}global i16 -32768, align 2
+_Sat short _Accum sat_sa_const6 = (_Sat short _Accum)(-255.0hk - 1.0hk) / -0.0078125hk;
+// CHECK-DAG: @sat_sa_const6 = {{.*}}global i16 32767, align 2
+short _Accum sa_const12 = 255.9921875hk / -1.0hk;
+// CHECK-DAG: @sa_const12 = {{.*}}global i16 -32767, align 2
+_Sat short _Accum sat_sa_const7 = (_Sat short _Accum)(-255.0hk - 1.0hk) / -1.0hk;
+// CHECK-DAG: @sat_sa_const7 = {{.*}}global i16 32767, align 2
+short _Accum sa_const13 = 0.0234375hk / 2.0hk;
+// CHECK-DAG: @sa_const13 = {{.*}}global i16 1, align 2
+short _Accum sa_const14 = -0.0234375hk / 2.0hk;
+// CHECK-DAG: @sa_const14 = {{.*}}global i16 -2, align 2
+short _Accum sa_const15 = -0.0078125hk / 255.28125hk;
+// CHECK-DAG: @sa_const15 = {{.*}}global i16 -1, align 2
+
 void SignedDivision() {
   // CHECK-LABEL: SignedDivision
   short _Accum sa;
@@ -233,12 +299,12 @@ void IntDivision() {
   // SIGNED-NEXT:   [[RESIZE7:%.*]] = zext i16 [[TMP6]] to i40
   // SIGNED-NEXT:   [[RESIZE8:%.*]] = sext i32 [[TMP7]] to i40
   // SIGNED-NEXT:   [[UPSCALE9:%.*]] = shl i40 [[RESIZE8]], 8
-  // SIGNED-NEXT:   [[TMP8:%.*]] = call i40 @llvm.udiv.fix.i40(i40 [[RESIZE7]], i40 [[UPSCALE9]], i32 8)
+  // SIGNED-NEXT:   [[TMP8:%.*]] = call i40 @llvm.sdiv.fix.i40(i40 [[RESIZE7]], i40 [[UPSCALE9]], i32 8)
   // SIGNED-NEXT:   [[RESIZE10:%.*]] = trunc i40 [[TMP8]] to i16
   // UNSIGNED-NEXT: [[RESIZE7:%.*]] = zext i16 [[TMP6]] to i39
   // UNSIGNED-NEXT: [[RESIZE8:%.*]] = sext i32 [[TMP7]] to i39
   // UNSIGNED-NEXT: [[UPSCALE9:%.*]] = shl i39 [[RESIZE8]], 7
-  // UNSIGNED-NEXT: [[TMP8:%.*]] = call i39 @llvm.udiv.fix.i39(i39 [[RESIZE7]], i39 [[UPSCALE9]], i32 7)
+  // UNSIGNED-NEXT: [[TMP8:%.*]] = call i39 @llvm.sdiv.fix.i39(i39 [[RESIZE7]], i39 [[UPSCALE9]], i32 7)
   // UNSIGNED-NEXT: [[RESIZE10:%.*]] = trunc i39 [[TMP8]] to i16
   // CHECK-NEXT:    store i16 [[RESIZE10]], i16* %usa, align 2
   usa = usa / i;
@@ -411,7 +477,7 @@ void SaturatedDivision() {
   // SIGNED-NEXT: [[USA_SAT_RESIZE:%[a-z0-9]+]] = zext i16 [[USA_SAT]] to i40
   // SIGNED-NEXT: [[I_RESIZE:%[a-z0-9]+]] = sext i32 [[I]] to i40
   // SIGNED-NEXT: [[I_UPSCALE:%[a-z0-9]+]] = shl i40 [[I_RESIZE]], 8
-  // SIGNED-NEXT: [[SUM:%[0-9]+]] = call i40 @llvm.udiv.fix.sat.i40(i40 [[USA_SAT_RESIZE]], i40 [[I_UPSCALE]], i32 8)
+  // SIGNED-NEXT: [[SUM:%[0-9]+]] = call i40 @llvm.sdiv.fix.sat.i40(i40 [[USA_SAT_RESIZE]], i40 [[I_UPSCALE]], i32 8)
   // SIGNED-NEXT: [[USE_MAX:%[0-9]+]] = icmp sgt i40 [[SUM]], 65535
   // SIGNED-NEXT: [[RESULT:%[a-z0-9]+]] = select i1 [[USE_MAX]], i40 65535, i40 [[SUM]]
   // SIGNED-NEXT: [[USE_MIN:%[0-9]+]] = icmp slt i40 [[RESULT]], 0
@@ -420,7 +486,7 @@ void SaturatedDivision() {
   // UNSIGNED-NEXT: [[USA_SAT_RESIZE:%[a-z0-9]+]] = zext i16 [[USA_SAT]] to i39
   // UNSIGNED-NEXT: [[I_RESIZE:%[a-z0-9]+]] = sext i32 [[I]] to i39
   // UNSIGNED-NEXT: [[I_UPSCALE:%[a-z0-9]+]] = shl i39 [[I_RESIZE]], 7
-  // UNSIGNED-NEXT: [[SUM:%[0-9]+]] = call i39 @llvm.udiv.fix.sat.i39(i39 [[USA_SAT_RESIZE]], i39 [[I_UPSCALE]], i32 7)
+  // UNSIGNED-NEXT: [[SUM:%[0-9]+]] = call i39 @llvm.sdiv.fix.sat.i39(i39 [[USA_SAT_RESIZE]], i39 [[I_UPSCALE]], i32 7)
   // UNSIGNED-NEXT: [[USE_MAX:%[0-9]+]] = icmp sgt i39 [[SUM]], 32767
   // UNSIGNED-NEXT: [[RESULT:%[a-z0-9]+]] = select i1 [[USE_MAX]], i39 32767, i39 [[SUM]]
   // UNSIGNED-NEXT: [[USE_MIN:%[0-9]+]] = icmp slt i39 [[RESULT]], 0
diff --git a/clang/test/Frontend/fixed_point_errors.c b/clang/test/Frontend/fixed_point_errors.c
index 9b600fbc2642b..2fd8cfd1fdf70 100644
--- a/clang/test/Frontend/fixed_point_errors.c
+++ b/clang/test/Frontend/fixed_point_errors.c
@@ -250,3 +250,20 @@ unsigned u_const = -2.5hk;                // expected-warning{{implicit conversi
 char c_const = 256.0uk;                   // expected-warning{{implicit conversion from 256.0 cannot fit within the range of values for 'char'}}
 short _Accum sa_const5 = 256;             // expected-warning{{implicit conversion from 256 cannot fit within the range of values for 'short _Accum'}}
 unsigned short _Accum usa_const2 = -2;    // expected-warning{{implicit conversion from -2 cannot fit within the range of values for 'unsigned short _Accum'}}
+
+short _Accum add_ovf1 = 255.0hk + 20.0hk;                     // expected-warning {{overflow in expression; result is -237.0 with type 'short _Accum'}}
+short _Accum add_ovf2 = 10 + 0.5hr;                           // expected-warning {{overflow in expression; result is 0.5 with type 'short _Fract'}}
+short _Accum sub_ovf1 = 16.0uhk - 32.0uhk;                    // expected-warning {{overflow in expression; result is 240.0 with type 'unsigned short _Accum'}}
+short _Accum sub_ovf2 = -255.0hk - 20;                        // expected-warning {{overflow in expression; result is 237.0 with type 'short _Accum'}}
+short _Accum mul_ovf1 = 200.0uhk * 10.0uhk;                   // expected-warning {{overflow in expression; result is 208.0 with type 'unsigned short _Accum'}}
+short _Accum mul_ovf2 = (-0.5hr - 0.5hr) * (-0.5hr - 0.5hr);  // expected-warning {{overflow in expression; result is -1.0 with type 'short _Fract'}}
+short _Accum div_ovf1 = 255.0hk / 0.5hk;                      // expected-warning {{overflow in expression; result is -2.0 with type 'short _Accum'}}
+
+// No warnings for saturation
+short _Fract add_sat  = (_Sat short _Fract)0.5hr + 0.5hr;
+short _Accum sub_sat  = (_Sat short _Accum)-200.0hk - 80.0hk;
+short _Accum mul_sat  = (_Sat short _Accum)80.0hk * 10.0hk;
+short _Fract div_sat  = (_Sat short _Fract)0.9hr / 0.1hr;
+
+// Division by zero
+short _Accum div_zero = 4.5k / 0.0lr;  // expected-error {{initializer element is not a compile-time constant}}
diff --git a/clang/test/Frontend/fixed_point_mul.c b/clang/test/Frontend/fixed_point_mul.c
index 274528fe9d9a6..5e4741de1fe75 100644
--- a/clang/test/Frontend/fixed_point_mul.c
+++ b/clang/test/Frontend/fixed_point_mul.c
@@ -1,6 +1,49 @@
 // RUN: %clang_cc1 -ffixed-point -triple x86_64-unknown-linux-gnu -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,SIGNED
 // RUN: %clang_cc1 -ffixed-point -triple x86_64-unknown-linux-gnu -fpadding-on-unsigned-fixed-point -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,UNSIGNED
 
+// Multiplication between different fixed point types
+short _Accum sa_const = 2.0hk * 2.0hk;  // CHECK-DAG: @sa_const  = {{.*}}global i16 512, align 2
+_Accum a_const = 3.0hk * 2.0k;          // CHECK-DAG: @a_const   = {{.*}}global i32 196608, align 4
+long _Accum la_const = 4.0hk * 2.0lk;   // CHECK-DAG: @la_const  = {{.*}}global i64 17179869184, align 8
+short _Accum sa_const2 = 0.5hr * 2.0hk; // CHECK-DAG: @sa_const2  = {{.*}}global i16 128, align 2
+short _Accum sa_const3 = 0.5r * 3.0hk;  // CHECK-DAG: @sa_const3  = {{.*}}global i16 192, align 2
+short _Accum sa_const4 = 0.5lr * 4.0hk; // CHECK-DAG: @sa_const4  = {{.*}}global i16 256, align 2
+
+// Unsigned multiplication
+unsigned short _Accum usa_const = 1.0uhk * 2.0uhk;
+// CHECK-SIGNED-DAG:   @usa_const = {{.*}}global i16 768, align 2
+// CHECK-UNSIGNED-DAG: @usa_const = {{.*}}global i16 384, align 2
+
+// Unsigned * signed
+short _Accum sa_const5 = 20.0uhk * 3.0hk;
+// CHECK-DAG: @sa_const5 = {{.*}}global i16 7680, align 2
+
+// Multiplication with negative number
+short _Accum sa_const6 = 0.5hr * (-2.0hk);
+// CHECK-DAG: @sa_const6 = {{.*}}global i16 -128, align 2
+
+// Int multiplication
+unsigned short _Accum usa_const2 = 5 * 10.5uhk;
+// CHECK-SIGNED-DAG:   @usa_const2 = {{.*}}global i16 640, align 2
+// CHECK-UNSIGNED-DAG: @usa_const2 = {{.*}}global i16 320, align 2
+short _Accum sa_const7 = 3 * (-0.5hk);   // CHECK-DAG: @sa_const7 = {{.*}}global i16 -192, align 2
+short _Accum sa_const8 = 100 * (-2.0hk); // CHECK-DAG: @sa_const8 = {{.*}}global i16 -25600, align 2
+long _Fract lf_const = -0.25lr * 3;      // CHECK-DAG: @lf_const  = {{.*}}global i32 -1610612736, align 4
+
+// Saturated multiplication
+_Sat short _Accum sat_sa_const = (_Sat short _Accum)128.0hk * 3.0hk;
+// CHECK-DAG: @sat_sa_const = {{.*}}global i16 32767, align 2
+_Sat unsigned short _Accum sat_usa_const = (_Sat unsigned short _Accum)128.0uhk * 128.0uhk;
+// CHECK-SIGNED-DAG:   @sat_usa_const = {{.*}}global i16 65535, align 2
+// CHECK-UNSIGNED-DAG: @sat_usa_const = {{.*}}global i16 32767, align 2
+_Sat short _Accum sat_sa_const2 = (_Sat short _Accum)128.0hk * -128;
+// CHECK-DAG: @sat_sa_const2 = {{.*}}global i16 -32768, align 2
+_Sat unsigned short _Accum sat_usa_const2 = (_Sat unsigned short _Accum)128.0uhk * 30;
+// CHECK-SIGNED-DAG:   @sat_usa_const2 = {{.*}}global i16 65535, align 2
+// CHECK-UNSIGNED-DAG: @sat_usa_const2 = {{.*}}global i16 32767, align 2
+_Sat unsigned short _Accum sat_usa_const3 = (_Sat unsigned short _Accum)0.5uhk * (-2);
+// CHECK-DAG:   @sat_usa_const3 = {{.*}}global i16 0, align 2
+
 void SignedMultiplication() {
   // CHECK-LABEL: SignedMultiplication
   short _Accum sa;
@@ -233,12 +276,12 @@ void IntMultiplication() {
   // SIGNED-NEXT:   [[RESIZE7:%.*]] = zext i16 [[TMP6]] to i40
   // SIGNED-NEXT:   [[RESIZE8:%.*]] = sext i32 [[TMP7]] to i40
   // SIGNED-NEXT:   [[UPSCALE9:%.*]] = shl i40 [[RESIZE8]], 8
-  // SIGNED-NEXT:   [[TMP8:%.*]] = call i40 @llvm.umul.fix.i40(i40 [[RESIZE7]], i40 [[UPSCALE9]], i32 8)
+  // SIGNED-NEXT:   [[TMP8:%.*]] = call i40 @llvm.smul.fix.i40(i40 [[RESIZE7]], i40 [[UPSCALE9]], i32 8)
   // SIGNED-NEXT:   [[RESIZE10:%.*]] = trunc i40 [[TMP8]] to i16
   // UNSIGNED-NEXT: [[RESIZE7:%.*]] = zext i16 [[TMP6]] to i39
   // UNSIGNED-NEXT: [[RESIZE8:%.*]] = sext i32 [[TMP7]] to i39
   // UNSIGNED-NEXT: [[UPSCALE9:%.*]] = shl i39 [[RESIZE8]], 7
-  // UNSIGNED-NEXT: [[TMP8:%.*]] = call i39 @llvm.umul.fix.i39(i39 [[RESIZE7]], i39 [[UPSCALE9]], i32 7)
+  // UNSIGNED-NEXT: [[TMP8:%.*]] = call i39 @llvm.smul.fix.i39(i39 [[RESIZE7]], i39 [[UPSCALE9]], i32 7)
   // UNSIGNED-NEXT: [[RESIZE10:%.*]] = trunc i39 [[TMP8]] to i16
   // CHECK-NEXT:    store i16 [[RESIZE10]], i16* %usa, align 2
   usa = usa * i;
@@ -411,7 +454,7 @@ void SaturatedMultiplication() {
   // SIGNED-NEXT: [[USA_SAT_RESIZE:%[a-z0-9]+]] = zext i16 [[USA_SAT]] to i40
   // SIGNED-NEXT: [[I_RESIZE:%[a-z0-9]+]] = sext i32 [[I]] to i40
   // SIGNED-NEXT: [[I_UPSCALE:%[a-z0-9]+]] = shl i40 [[I_RESIZE]], 8
-  // SIGNED-NEXT: [[SUM:%[0-9]+]] = call i40 @llvm.umul.fix.sat.i40(i40 [[USA_SAT_RESIZE]], i40 [[I_UPSCALE]], i32 8)
+  // SIGNED-NEXT: [[SUM:%[0-9]+]] = call i40 @llvm.smul.fix.sat.i40(i40 [[USA_SAT_RESIZE]], i40 [[I_UPSCALE]], i32 8)
   // SIGNED-NEXT: [[USE_MAX:%[0-9]+]] = icmp sgt i40 [[SUM]], 65535
   // SIGNED-NEXT: [[RESULT:%[a-z0-9]+]] = select i1 [[USE_MAX]], i40 65535, i40 [[SUM]]
   // SIGNED-NEXT: [[USE_MIN:%[0-9]+]] = icmp slt i40 [[RESULT]], 0
@@ -420,7 +463,7 @@ void SaturatedMultiplication() {
   // UNSIGNED-NEXT: [[USA_SAT_RESIZE:%[a-z0-9]+]] = zext i16 [[USA_SAT]] to i39
   // UNSIGNED-NEXT: [[I_RESIZE:%[a-z0-9]+]] = sext i32 [[I]] to i39
   // UNSIGNED-NEXT: [[I_UPSCALE:%[a-z0-9]+]] = shl i39 [[I_RESIZE]], 7
-  // UNSIGNED-NEXT: [[SUM:%[0-9]+]] = call i39 @llvm.umul.fix.sat.i39(i39 [[USA_SAT_RESIZE]], i39 [[I_UPSCALE]], i32 7)
+  // UNSIGNED-NEXT: [[SUM:%[0-9]+]] = call i39 @llvm.smul.fix.sat.i39(i39 [[USA_SAT_RESIZE]], i39 [[I_UPSCALE]], i32 7)
   // UNSIGNED-NEXT: [[USE_MAX:%[0-9]+]] = icmp sgt i39 [[SUM]], 32767
   // UNSIGNED-NEXT: [[RESULT:%[a-z0-9]+]] = select i1 [[USE_MAX]], i39 32767, i39 [[SUM]]
   // UNSIGNED-NEXT: [[USE_MIN:%[0-9]+]] = icmp slt i39 [[RESULT]], 0
diff --git a/clang/test/Frontend/fixed_point_sub.c b/clang/test/Frontend/fixed_point_sub.c
index 59b2e0a43aed6..b30825e555be0 100644
--- a/clang/test/Frontend/fixed_point_sub.c
+++ b/clang/test/Frontend/fixed_point_sub.c
@@ -1,6 +1,55 @@
 // RUN: %clang_cc1 -ffixed-point -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,SIGNED
 // RUN: %clang_cc1 -ffixed-point -fpadding-on-unsigned-fixed-point -S -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,UNSIGNED
 
+// Subtraction between different fixed point types
+short _Accum sa_const = 1.0hk - 2.0hk;  // CHECK-DAG: @sa_const  = {{.*}}global i16 -128, align 2
+_Accum a_const = 1.0hk - 2.0k;          // CHECK-DAG: @a_const   = {{.*}}global i32 -32768, align 4
+long _Accum la_const = 1.0hk - 2.0lk;   // CHECK-DAG: @la_const  = {{.*}}global i64 -2147483648, align 8
+short _Accum sa_const2 = 0.5hr - 2.0hk; // CHECK-DAG: @sa_const2  = {{.*}}global i16 -192, align 2
+short _Accum sa_const3 = 0.5r - 2.0hk;  // CHECK-DAG: @sa_const3  = {{.*}}global i16 -192, align 2
+short _Accum sa_const4 = 0.5lr - 2.0hk; // CHECK-DAG: @sa_const4  = {{.*}}global i16 -192, align 2
+short _Accum sa_const5 = 2.0hk - 0.5lr; // CHECK-DAG: @sa_const5  = {{.*}}global i16 192, align 2
+
+// Unsigned subtraction
+unsigned short _Accum usa_const = 3.0uhk - 2.0uhk;
+// CHECK-SIGNED-DAG:   @usa_const = {{.*}}global i16 768, align 2
+// CHECK-UNSIGNED-DAG: @usa_const = {{.*}}global i16 384, align 2
+
+// Unsigned - signed
+short _Accum sa_const6 = 1.0uhk - 2.0hk;
+// CHECK-DAG: @sa_const6 = {{.*}}global i16 -128, align 2
+
+// Subtraction with negative number
+short _Accum sa_const7 = 0.5hr - (-2.0hk);
+// CHECK-DAG: @sa_const7 = {{.*}}global i16 320, align 2
+
+// Int subtraction
+unsigned short _Accum usa_const2 = 2 - 0.5uhk;
+// CHECK-SIGNED-DAG:   @usa_const2 = {{.*}}global i16 640, align 2
+// CHECK-UNSIGNED-DAG: @usa_const2 = {{.*}}global i16 320, align 2
+short _Accum sa_const8 = 2 - (-0.5hk);   // CHECK-DAG: @sa_const8 = {{.*}}global i16 320, align 2
+short _Accum sa_const9 = 257 - 2.0hk;    // CHECK-DAG: @sa_const9 = {{.*}}global i16 32640, align 2
+long _Fract lf_const = 0.5lr - 1;        // CHECK-DAG: @lf_const  = {{.*}}global i32 -1073741824, align 4
+
+// Saturated subtraction
+_Sat short _Accum sat_sa_const = (_Sat short _Accum)128.0hk - (-128.0hk);
+// CHECK-DAG: @sat_sa_const = {{.*}}global i16 32767, align 2
+_Sat unsigned short _Accum sat_usa_const = (_Sat unsigned short _Accum)128.0uhk - (-128.0uhk);
+// CHECK-SIGNED-DAG:   @sat_usa_const = {{.*}}global i16 65535, align 2
+// CHECK-UNSIGNED-DAG: @sat_usa_const = {{.*}}global i16 32767, align 2
+_Sat short _Accum sat_sa_const2 = (_Sat short _Accum)128.0hk - (-128);
+// CHECK-DAG: @sat_sa_const2 = {{.*}}global i16 32767, align 2
+_Sat unsigned short _Accum sat_usa_const2 = (_Sat unsigned short _Accum)128.0uhk - (-128);
+// CHECK-SIGNED-DAG:   @sat_usa_const2 = {{.*}}global i16 65535, align 2
+// CHECK-UNSIGNED-DAG: @sat_usa_const2 = {{.*}}global i16 32767, align 2
+_Sat unsigned short _Accum sat_usa_const3 = (_Sat unsigned short _Accum)0.5uhk - 2;
+// CHECK-DAG:   @sat_usa_const3 = {{.*}}global i16 0, align 2
+_Sat short _Accum sat_sa_const3 = (_Sat short _Accum)-128.0hk - 128;
+// CHECK-DAG: @sat_sa_const3 = {{.*}}global i16 -32768, align 2
+_Sat short _Accum sat_sa_const4 = (_Sat short _Accum)-150.0hk - 130.0lk;
+// CHECK-DAG: @sat_sa_const4 = {{.*}}global i16 -32768, align 2
+
+
 void SignedSubtraction() {
   // CHECK-LABEL: SignedSubtraction
   short _Accum sa;
@@ -370,7 +419,7 @@ void SaturatedSubtraction() {
   // SIGNED-NEXT: [[USA_SAT_RESIZE:%[a-z0-9]+]] = zext i16 [[USA_SAT]] to i40
   // SIGNED-NEXT: [[I_RESIZE:%[a-z0-9]+]] = sext i32 [[I]] to i40
   // SIGNED-NEXT: [[I_UPSCALE:%[a-z0-9]+]] = shl i40 [[I_RESIZE]], 8
-  // SIGNED-NEXT: [[SUM:%[0-9]+]] = call i40 @llvm.usub.sat.i40(i40 [[USA_SAT_RESIZE]], i40 [[I_UPSCALE]])
+  // SIGNED-NEXT: [[SUM:%[0-9]+]] = call i40 @llvm.ssub.sat.i40(i40 [[USA_SAT_RESIZE]], i40 [[I_UPSCALE]])
   // SIGNED-NEXT: [[USE_MAX:%[0-9]+]] = icmp sgt i40 [[SUM]], 65535
   // SIGNED-NEXT: [[RESULT:%[a-z0-9]+]] = select i1 [[USE_MAX]], i40 65535, i40 [[SUM]]
   // SIGNED-NEXT: [[USE_MIN:%[0-9]+]] = icmp slt i40 [[RESULT]], 0
@@ -379,7 +428,7 @@ void SaturatedSubtraction() {
   // UNSIGNED-NEXT: [[USA_SAT_RESIZE:%[a-z0-9]+]] = zext i16 [[USA_SAT]] to i39
   // UNSIGNED-NEXT: [[I_RESIZE:%[a-z0-9]+]] = sext i32 [[I]] to i39
   // UNSIGNED-NEXT: [[I_UPSCALE:%[a-z0-9]+]] = shl i39 [[I_RESIZE]], 7
-  // UNSIGNED-NEXT: [[SUM:%[0-9]+]] = call i39 @llvm.usub.sat.i39(i39 [[USA_SAT_RESIZE]], i39 [[I_UPSCALE]])
+  // UNSIGNED-NEXT: [[SUM:%[0-9]+]] = call i39 @llvm.ssub.sat.i39(i39 [[USA_SAT_RESIZE]], i39 [[I_UPSCALE]])
   // UNSIGNED-NEXT: [[USE_MAX:%[0-9]+]] = icmp sgt i39 [[SUM]], 32767
   // UNSIGNED-NEXT: [[RESULT:%[a-z0-9]+]] = select i1 [[USE_MAX]], i39 32767, i39 [[SUM]]
   // UNSIGNED-NEXT: [[USE_MIN:%[0-9]+]] = icmp slt i39 [[RESULT]], 0
diff --git a/clang/test/Frontend/opencl.cl b/clang/test/Frontend/opencl.cl
index ea7d3d4c87317..e25a4b8fac6a6 100644
--- a/clang/test/Frontend/opencl.cl
+++ b/clang/test/Frontend/opencl.cl
@@ -21,7 +21,7 @@ int member;
 #endif
 #endif
 
-void f(void (^g)(void)) {
+typedef int (^bl_t)(void);
 #if defined(__OPENCL_C_VERSION__) || defined(__OPENCL_CPP_VERSION__)
 #if !defined(BLOCKS) && (defined(__OPENCL_CPP_VERSION__)  || __OPENCL_C_VERSION__ != CL_VERSION_2_0)
   // expected-error@-3{{blocks support disabled - compile with -fblocks or for OpenCL 2.0}}
@@ -29,7 +29,6 @@ void f(void (^g)(void)) {
 #else
   // expected-error@-6{{blocks support disabled - compile with -fblocks or pick a deployment target that supports them}}
 #endif
-}
 
 // CHECK-INVALID-OPENCL-VERSION11: warning: OpenCL version 1.1 does not support the option '-cl-strict-aliasing'
 // CHECK-INVALID-OPENCL-VERSION12: warning: OpenCL version 1.2 does not support the option '-cl-strict-aliasing'
diff --git a/clang/test/Frontend/optimization-remark-line-directive.c b/clang/test/Frontend/optimization-remark-line-directive.c
index 516555ff45503..095791e505f8b 100644
--- a/clang/test/Frontend/optimization-remark-line-directive.c
+++ b/clang/test/Frontend/optimization-remark-line-directive.c
@@ -2,11 +2,11 @@
 // directives. We cannot map #line directives back to
 // a SourceLocation.
 
-// RUN: %clang_cc1 %s -Rpass=inline -debug-info-kind=line-tables-only -dwarf-column-info -emit-llvm-only -verify -fno-experimental-new-pass-manager
+// RUN: %clang_cc1 %s -Rpass=inline -debug-info-kind=line-tables-only -emit-llvm-only -verify -fno-experimental-new-pass-manager
 
 // The new PM inliner is not added to the default pipeline at O0, so we add
 // some optimizations to trigger it.
-// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -debug-info-kind=line-tables-only -dwarf-column-info -emit-llvm-only -verify
+// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -debug-info-kind=line-tables-only -emit-llvm-only -verify
 
 int foo(int x, int y) __attribute__((always_inline));
 int foo(int x, int y) { return x + y; }
diff --git a/clang/test/Headers/Inputs/include/cmath b/clang/test/Headers/Inputs/include/cmath
index 0cadc131d2111..5e4e8b67514f0 100644
--- a/clang/test/Headers/Inputs/include/cmath
+++ b/clang/test/Headers/Inputs/include/cmath
@@ -49,8 +49,12 @@ double fma(double, double, double);
 float fma(float, float, float);
 double fmax(double, double);
 float fmax(float, float);
+float max(float, float);
+double max(double, double);
 double fmin(double, double);
 float fmin(float, float);
+float min(float, float);
+double min(double, double);
 double fmod(double, double);
 float fmod(float, float);
 int fpclassify(double);
diff --git a/clang/test/Headers/Inputs/include/complex b/clang/test/Headers/Inputs/include/complex
new file mode 100644
index 0000000000000..f3aefab7954be
--- /dev/null
+++ b/clang/test/Headers/Inputs/include/complex
@@ -0,0 +1,301 @@
+#pragma once
+
+#include <cmath>
+
+#define INFINITY (__builtin_inff())
+
+namespace std {
+
+// Taken from libc++
+template <class _Tp>
+class complex {
+public:
+  typedef _Tp value_type;
+
+private:
+  value_type __re_;
+  value_type __im_;
+
+public:
+  complex(const value_type &__re = value_type(), const value_type &__im = value_type())
+      : __re_(__re), __im_(__im) {}
+  template <class _Xp>
+  complex(const complex<_Xp> &__c)
+      : __re_(__c.real()), __im_(__c.imag()) {}
+
+  value_type real() const { return __re_; }
+  value_type imag() const { return __im_; }
+
+  void real(value_type __re) { __re_ = __re; }
+  void imag(value_type __im) { __im_ = __im; }
+
+  complex &operator=(const value_type &__re) {
+    __re_ = __re;
+    __im_ = value_type();
+    return *this;
+  }
+  complex &operator+=(const value_type &__re) {
+    __re_ += __re;
+    return *this;
+  }
+  complex &operator-=(const value_type &__re) {
+    __re_ -= __re;
+    return *this;
+  }
+  complex &operator*=(const value_type &__re) {
+    __re_ *= __re;
+    __im_ *= __re;
+    return *this;
+  }
+  complex &operator/=(const value_type &__re) {
+    __re_ /= __re;
+    __im_ /= __re;
+    return *this;
+  }
+
+  template <class _Xp>
+  complex &operator=(const complex<_Xp> &__c) {
+    __re_ = __c.real();
+    __im_ = __c.imag();
+    return *this;
+  }
+  template <class _Xp>
+  complex &operator+=(const complex<_Xp> &__c) {
+    __re_ += __c.real();
+    __im_ += __c.imag();
+    return *this;
+  }
+  template <class _Xp>
+  complex &operator-=(const complex<_Xp> &__c) {
+    __re_ -= __c.real();
+    __im_ -= __c.imag();
+    return *this;
+  }
+  template <class _Xp>
+  complex &operator*=(const complex<_Xp> &__c) {
+    *this = *this * complex(__c.real(), __c.imag());
+    return *this;
+  }
+  template <class _Xp>
+  complex &operator/=(const complex<_Xp> &__c) {
+    *this = *this / complex(__c.real(), __c.imag());
+    return *this;
+  }
+};
+
+template <class _Tp>
+inline complex<_Tp>
+operator+(const complex<_Tp> &__x, const complex<_Tp> &__y) {
+  complex<_Tp> __t(__x);
+  __t += __y;
+  return __t;
+}
+
+template <class _Tp>
+inline complex<_Tp>
+operator+(const complex<_Tp> &__x, const _Tp &__y) {
+  complex<_Tp> __t(__x);
+  __t += __y;
+  return __t;
+}
+
+template <class _Tp>
+inline complex<_Tp>
+operator+(const _Tp &__x, const complex<_Tp> &__y) {
+  complex<_Tp> __t(__y);
+  __t += __x;
+  return __t;
+}
+
+template <class _Tp>
+inline complex<_Tp>
+operator-(const complex<_Tp> &__x, const complex<_Tp> &__y) {
+  complex<_Tp> __t(__x);
+  __t -= __y;
+  return __t;
+}
+
+template <class _Tp>
+inline complex<_Tp>
+operator-(const complex<_Tp> &__x, const _Tp &__y) {
+  complex<_Tp> __t(__x);
+  __t -= __y;
+  return __t;
+}
+
+template <class _Tp>
+inline complex<_Tp>
+operator-(const _Tp &__x, const complex<_Tp> &__y) {
+  complex<_Tp> __t(-__y);
+  __t += __x;
+  return __t;
+}
+
+template <class _Tp>
+complex<_Tp>
+operator*(const complex<_Tp> &__z, const complex<_Tp> &__w) {
+  _Tp __a = __z.real();
+  _Tp __b = __z.imag();
+  _Tp __c = __w.real();
+  _Tp __d = __w.imag();
+  _Tp __ac = __a * __c;
+  _Tp __bd = __b * __d;
+  _Tp __ad = __a * __d;
+  _Tp __bc = __b * __c;
+  _Tp __x = __ac - __bd;
+  _Tp __y = __ad + __bc;
+  if (std::isnan(__x) && std::isnan(__y)) {
+    bool __recalc = false;
+    if (std::isinf(__a) || std::isinf(__b)) {
+      __a = copysign(std::isinf(__a) ? _Tp(1) : _Tp(0), __a);
+      __b = copysign(std::isinf(__b) ? _Tp(1) : _Tp(0), __b);
+      if (std::isnan(__c))
+        __c = copysign(_Tp(0), __c);
+      if (std::isnan(__d))
+        __d = copysign(_Tp(0), __d);
+      __recalc = true;
+    }
+    if (std::isinf(__c) || std::isinf(__d)) {
+      __c = copysign(std::isinf(__c) ? _Tp(1) : _Tp(0), __c);
+      __d = copysign(std::isinf(__d) ? _Tp(1) : _Tp(0), __d);
+      if (std::isnan(__a))
+        __a = copysign(_Tp(0), __a);
+      if (std::isnan(__b))
+        __b = copysign(_Tp(0), __b);
+      __recalc = true;
+    }
+    if (!__recalc && (std::isinf(__ac) || std::isinf(__bd) ||
+                      std::isinf(__ad) || std::isinf(__bc))) {
+      if (std::isnan(__a))
+        __a = copysign(_Tp(0), __a);
+      if (std::isnan(__b))
+        __b = copysign(_Tp(0), __b);
+      if (std::isnan(__c))
+        __c = copysign(_Tp(0), __c);
+      if (std::isnan(__d))
+        __d = copysign(_Tp(0), __d);
+      __recalc = true;
+    }
+    if (__recalc) {
+      __x = _Tp(INFINITY) * (__a * __c - __b * __d);
+      __y = _Tp(INFINITY) * (__a * __d + __b * __c);
+    }
+  }
+  return complex<_Tp>(__x, __y);
+}
+
+template <class _Tp>
+inline complex<_Tp>
+operator*(const complex<_Tp> &__x, const _Tp &__y) {
+  complex<_Tp> __t(__x);
+  __t *= __y;
+  return __t;
+}
+
+template <class _Tp>
+inline complex<_Tp>
+operator*(const _Tp &__x, const complex<_Tp> &__y) {
+  complex<_Tp> __t(__y);
+  __t *= __x;
+  return __t;
+}
+
+template <class _Tp>
+complex<_Tp>
+operator/(const complex<_Tp> &__z, const complex<_Tp> &__w) {
+  int __ilogbw = 0;
+  _Tp __a = __z.real();
+  _Tp __b = __z.imag();
+  _Tp __c = __w.real();
+  _Tp __d = __w.imag();
+  _Tp __logbw = logb(fmax(fabs(__c), fabs(__d)));
+  if (std::isfinite(__logbw)) {
+    __ilogbw = static_cast<int>(__logbw);
+    __c = scalbn(__c, -__ilogbw);
+    __d = scalbn(__d, -__ilogbw);
+  }
+  _Tp __denom = __c * __c + __d * __d;
+  _Tp __x = scalbn((__a * __c + __b * __d) / __denom, -__ilogbw);
+  _Tp __y = scalbn((__b * __c - __a * __d) / __denom, -__ilogbw);
+  if (std::isnan(__x) && std::isnan(__y)) {
+    if ((__denom == _Tp(0)) && (!std::isnan(__a) || !std::isnan(__b))) {
+      __x = copysign(_Tp(INFINITY), __c) * __a;
+      __y = copysign(_Tp(INFINITY), __c) * __b;
+    } else if ((std::isinf(__a) || std::isinf(__b)) && std::isfinite(__c) && std::isfinite(__d)) {
+      __a = copysign(std::isinf(__a) ? _Tp(1) : _Tp(0), __a);
+      __b = copysign(std::isinf(__b) ? _Tp(1) : _Tp(0), __b);
+      __x = _Tp(INFINITY) * (__a * __c + __b * __d);
+      __y = _Tp(INFINITY) * (__b * __c - __a * __d);
+    } else if (std::isinf(__logbw) && __logbw > _Tp(0) && std::isfinite(__a) && std::isfinite(__b)) {
+      __c = copysign(std::isinf(__c) ? _Tp(1) : _Tp(0), __c);
+      __d = copysign(std::isinf(__d) ? _Tp(1) : _Tp(0), __d);
+      __x = _Tp(0) * (__a * __c + __b * __d);
+      __y = _Tp(0) * (__b * __c - __a * __d);
+    }
+  }
+  return complex<_Tp>(__x, __y);
+}
+
+template <class _Tp>
+inline complex<_Tp>
+operator/(const complex<_Tp> &__x, const _Tp &__y) {
+  return complex<_Tp>(__x.real() / __y, __x.imag() / __y);
+}
+
+template <class _Tp>
+inline complex<_Tp>
+operator/(const _Tp &__x, const complex<_Tp> &__y) {
+  complex<_Tp> __t(__x);
+  __t /= __y;
+  return __t;
+}
+
+template <class _Tp>
+inline complex<_Tp>
+operator+(const complex<_Tp> &__x) {
+  return __x;
+}
+
+template <class _Tp>
+inline complex<_Tp>
+operator-(const complex<_Tp> &__x) {
+  return complex<_Tp>(-__x.real(), -__x.imag());
+}
+
+template <class _Tp>
+inline bool
+operator==(const complex<_Tp> &__x, const complex<_Tp> &__y) {
+  return __x.real() == __y.real() && __x.imag() == __y.imag();
+}
+
+template <class _Tp>
+inline bool
+operator==(const complex<_Tp> &__x, const _Tp &__y) {
+  return __x.real() == __y && __x.imag() == 0;
+}
+
+template <class _Tp>
+inline bool
+operator==(const _Tp &__x, const complex<_Tp> &__y) {
+  return __x == __y.real() && 0 == __y.imag();
+}
+
+template <class _Tp>
+inline bool
+operator!=(const complex<_Tp> &__x, const complex<_Tp> &__y) {
+  return !(__x == __y);
+}
+
+template <class _Tp>
+inline bool
+operator!=(const complex<_Tp> &__x, const _Tp &__y) {
+  return !(__x == __y);
+}
+
+template <class _Tp>
+inline bool
+operator!=(const _Tp &__x, const complex<_Tp> &__y) {
+  return !(__x == __y);
+}
+
+} // namespace std
diff --git a/clang/test/Headers/Inputs/include/cstdlib b/clang/test/Headers/Inputs/include/cstdlib
index 00e81e8c4a150..1d1864a98976b 100644
--- a/clang/test/Headers/Inputs/include/cstdlib
+++ b/clang/test/Headers/Inputs/include/cstdlib
@@ -24,4 +24,8 @@ inline long long
 abs(long long __x) { return __builtin_llabs (__x); }
 
 float fabs(float __x) { return __builtin_fabs(__x); }
+
+float abs(float __x) { return fabs(__x); }
+double abs(double __x) { return fabs(__x); }
+
 }
diff --git a/clang/test/Headers/nvptx_device_math_complex.c b/clang/test/Headers/nvptx_device_math_complex.c
index 43f4ec6a6b590..6e3e8bffbd24e 100644
--- a/clang/test/Headers/nvptx_device_math_complex.c
+++ b/clang/test/Headers/nvptx_device_math_complex.c
@@ -1,10 +1,44 @@
 // REQUIRES: nvptx-registered-target
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -internal-isystem %S/Inputs/include -fopenmp -x c -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -internal-isystem %S/../../lib/Headers/openmp_wrappers -include __clang_openmp_device_functions.h -internal-isystem %S/Inputs/include -fopenmp -x c -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -aux-triple powerpc64le-unknown-unknown -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -internal-isystem %S/Inputs/include -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -internal-isystem %S/../../lib/Headers/openmp_wrappers -include __clang_openmp_device_functions.h -internal-isystem %S/Inputs/include -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -aux-triple powerpc64le-unknown-unknown -o - | FileCheck %s
 // expected-no-diagnostics
 
-// CHECK-DAG: call { float, float } @__divsc3(
-// CHECK-DAG: call { float, float } @__mulsc3(
+#ifdef __cplusplus
+#include <complex>
+#else
+#include <complex.h>
+#endif
+
+// CHECK: define weak {{.*}} @__muldc3
+// CHECK-DAG: call i32 @__nv_isnand(
+// CHECK-DAG: call i32 @__nv_isinfd(
+// CHECK-DAG: call double @__nv_copysign(
+
+// CHECK: define weak {{.*}} @__mulsc3
+// CHECK-DAG: call i32 @__nv_isnanf(
+// CHECK-DAG: call i32 @__nv_isinff(
+// CHECK-DAG: call float @__nv_copysignf(
+
+// CHECK: define weak {{.*}} @__divdc3
+// CHECK-DAG: call i32 @__nv_isnand(
+// CHECK-DAG: call i32 @__nv_isinfd(
+// CHECK-DAG: call i32 @__nv_isfinited(
+// CHECK-DAG: call double @__nv_copysign(
+// CHECK-DAG: call double @__nv_scalbn(
+// CHECK-DAG: call double @__nv_fabs(
+// CHECK-DAG: call double @__nv_logb(
+
+// CHECK: define weak {{.*}} @__divsc3
+// CHECK-DAG: call i32 @__nv_isnanf(
+// CHECK-DAG: call i32 @__nv_isinff(
+// CHECK-DAG: call i32 @__nv_finitef(
+// CHECK-DAG: call float @__nv_copysignf(
+// CHECK-DAG: call float @__nv_scalbnf(
+// CHECK-DAG: call float @__nv_fabsf(
+// CHECK-DAG: call float @__nv_logbf(
+
 void test_scmplx(float _Complex a) {
 #pragma omp target
   {
@@ -12,9 +46,6 @@ void test_scmplx(float _Complex a) {
   }
 }
 
-
-// CHECK-DAG: call { double, double } @__divdc3(
-// CHECK-DAG: call { double, double } @__muldc3(
 void test_dcmplx(double _Complex a) {
 #pragma omp target
   {
diff --git a/clang/test/Headers/nvptx_device_math_complex.cpp b/clang/test/Headers/nvptx_device_math_complex.cpp
new file mode 100644
index 0000000000000..e4b78deb05d7b
--- /dev/null
+++ b/clang/test/Headers/nvptx_device_math_complex.cpp
@@ -0,0 +1,48 @@
+// REQUIRES: nvptx-registered-target
+// RUN: %clang_cc1 -verify -internal-isystem %S/Inputs/include -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -internal-isystem %S/../../lib/Headers/openmp_wrappers -include __clang_openmp_device_functions.h -internal-isystem %S/Inputs/include -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -aux-triple powerpc64le-unknown-unknown -o - | FileCheck %s
+// expected-no-diagnostics
+
+#include <complex>
+
+// CHECK: define weak {{.*}} @__muldc3
+// CHECK-DAG: call i32 @__nv_isnand(
+// CHECK-DAG: call i32 @__nv_isinfd(
+// CHECK-DAG: call double @__nv_copysign(
+
+// CHECK: define weak {{.*}} @__mulsc3
+// CHECK-DAG: call i32 @__nv_isnanf(
+// CHECK-DAG: call i32 @__nv_isinff(
+// CHECK-DAG: call float @__nv_copysignf(
+
+// CHECK: define weak {{.*}} @__divdc3
+// CHECK-DAG: call i32 @__nv_isnand(
+// CHECK-DAG: call i32 @__nv_isinfd(
+// CHECK-DAG: call i32 @__nv_isfinited(
+// CHECK-DAG: call double @__nv_copysign(
+// CHECK-DAG: call double @__nv_scalbn(
+// CHECK-DAG: call double @__nv_fabs(
+// CHECK-DAG: call double @__nv_logb(
+
+// CHECK: define weak {{.*}} @__divsc3
+// CHECK-DAG: call i32 @__nv_isnanf(
+// CHECK-DAG: call i32 @__nv_isinff(
+// CHECK-DAG: call i32 @__nv_finitef(
+// CHECK-DAG: call float @__nv_copysignf(
+// CHECK-DAG: call float @__nv_scalbnf(
+// CHECK-DAG: call float @__nv_fabsf(
+// CHECK-DAG: call float @__nv_logbf(
+
+void test_scmplx(std::complex<float> a) {
+#pragma omp target
+  {
+    (void)(a * (a / a));
+  }
+}
+
+void test_dcmplx(std::complex<double> a) {
+#pragma omp target
+  {
+    (void)(a * (a / a));
+  }
+}
diff --git a/clang/test/Import/switch-stmt/test.cpp b/clang/test/Import/switch-stmt/test.cpp
index e274e895c444e..fcd8263727eaa 100644
--- a/clang/test/Import/switch-stmt/test.cpp
+++ b/clang/test/Import/switch-stmt/test.cpp
@@ -5,20 +5,26 @@
 // CHECK-NEXT: CompoundStmt
 // CHECK-NEXT: CaseStmt
 // CHECK-NEXT: ConstantExpr
+// CHECK-NEXT: value: Int 1
 // CHECK-NEXT: IntegerLiteral
 // CHECK-NEXT: CaseStmt
 // CHECK-NEXT: ConstantExpr
+// CHECK-NEXT: value: Int 2
 // CHECK-NEXT: IntegerLiteral
 // CHECK-NEXT: BreakStmt
 // CHECK-NEXT: CaseStmt
 // CHECK-NEXT: ConstantExpr
+// CHECK-NEXT: value: Int 3
 // CHECK-NEXT: IntegerLiteral
 // CHECK-NEXT: ConstantExpr
+// CHECK-NEXT: value: Int 4
 // CHECK-NEXT: IntegerLiteral
 // CHECK-NEXT: CaseStmt
 // CHECK-NEXT: ConstantExpr
+// CHECK-NEXT: value: Int 5
 // CHECK-NEXT: IntegerLiteral
 // CHECK-NEXT: ConstantExpr
+// CHECK-NEXT: value: Int 5
 // CHECK-NEXT: IntegerLiteral
 // CHECK-NEXT: BreakStmt
 
@@ -30,16 +36,20 @@
 // CHECK-NEXT: CompoundStmt
 // CHECK-NEXT: CaseStmt
 // CHECK-NEXT: ConstantExpr
+// CHECK-NEXT: value: Int 1
 // CHECK-NEXT: IntegerLiteral
 // CHECK-NEXT: BreakStmt
 // CHECK-NEXT: CaseStmt
 // CHECK-NEXT: ConstantExpr
+// CHECK-NEXT: value: Int 2
 // CHECK-NEXT: IntegerLiteral
 // CHECK-NEXT: BreakStmt
 // CHECK-NEXT: CaseStmt
 // CHECK-NEXT: ConstantExpr
+// CHECK-NEXT: value: Int 3
 // CHECK-NEXT: IntegerLiteral
 // CHECK-NEXT: ConstantExpr
+// CHECK-NEXT: value: Int 5
 // CHECK-NEXT: IntegerLiteral
 // CHECK-NEXT: BreakStmt
 
diff --git a/clang/test/Index/complete-method-decls.m b/clang/test/Index/complete-method-decls.m
index f561f81496a90..66c1bc56e8c95 100644
--- a/clang/test/Index/complete-method-decls.m
+++ b/clang/test/Index/complete-method-decls.m
@@ -111,40 +111,40 @@ -(void)foo {}
 // CHECK-CC1: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText abc} (40)
 // CHECK-CC1: ObjCInstanceMethodDecl:{LeftParen (}{Text int}{RightParen )}{TypedText getInt} (40)
 // CHECK-CC1: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText getSelf} (40)
-// CHECK-CC1: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText initWithInt}{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text x} (40)
-// CHECK-CC1: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText initWithTwoInts}{TypedText :}{LeftParen (}{Text inout }{Text int}{RightParen )}{Text x}{HorizontalSpace  }{TypedText second:}{LeftParen (}{Text int}{RightParen )}{Text y} (40)
+// CHECK-CC1: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText initWithInt:}{LeftParen (}{Text int}{RightParen )}{Text x} (40)
+// CHECK-CC1: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText initWithTwoInts:}{LeftParen (}{Text inout }{Text int}{RightParen )}{Text x}{HorizontalSpace  }{TypedText second:}{LeftParen (}{Text int}{RightParen )}{Text y} (40)
 // RUN: c-index-test -code-completion-at=%s:17:7 %s | FileCheck -check-prefix=CHECK-CC2 %s
 // CHECK-CC2: ObjCInstanceMethodDecl:{TypedText abc}
 // CHECK-CC2-NEXT: ObjCInstanceMethodDecl:{TypedText getSelf}
-// CHECK-CC2: ObjCInstanceMethodDecl:{TypedText initWithInt}{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text x}
-// CHECK-CC2: ObjCInstanceMethodDecl:{TypedText initWithTwoInts}{TypedText :}{LeftParen (}{Text inout }{Text int}{RightParen )}{Text x}{HorizontalSpace  }{TypedText second:}{LeftParen (}{Text int}{RightParen )}{Text y}
+// CHECK-CC2: ObjCInstanceMethodDecl:{TypedText initWithInt:}{LeftParen (}{Text int}{RightParen )}{Text x}
+// CHECK-CC2: ObjCInstanceMethodDecl:{TypedText initWithTwoInts:}{LeftParen (}{Text inout }{Text int}{RightParen )}{Text x}{HorizontalSpace  }{TypedText second:}{LeftParen (}{Text int}{RightParen )}{Text y}
 // RUN: c-index-test -code-completion-at=%s:24:7 %s | FileCheck -check-prefix=CHECK-CC3 %s
 // CHECK-CC3: ObjCInstanceMethodDecl:{TypedText abc}
 // CHECK-CC3-NEXT: ObjCInstanceMethodDecl:{TypedText getSelf}
 // CHECK-CC3: ObjCInstanceMethodDecl:{TypedText init}
-// CHECK-CC3: ObjCInstanceMethodDecl:{TypedText initWithInt}{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text x}
-// CHECK-CC3: ObjCInstanceMethodDecl:{TypedText initWithTwoInts}{TypedText :}{LeftParen (}{Text inout }{Text int}{RightParen )}{Text x}{HorizontalSpace  }{TypedText second:}{LeftParen (}{Text int}{RightParen )}{Text y}
+// CHECK-CC3: ObjCInstanceMethodDecl:{TypedText initWithInt:}{LeftParen (}{Text int}{RightParen )}{Text x}
+// CHECK-CC3: ObjCInstanceMethodDecl:{TypedText initWithTwoInts:}{LeftParen (}{Text inout }{Text int}{RightParen )}{Text x}{HorizontalSpace  }{TypedText second:}{LeftParen (}{Text int}{RightParen )}{Text y}
 // RUN: env CINDEXTEST_CODE_COMPLETE_PATTERNS=1 c-index-test -code-completion-at=%s:33:3 %s | FileCheck -check-prefix=CHECK-CC4 %s
 // CHECK-CC4: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText abc}{HorizontalSpace  }{LeftBrace {}{VerticalSpace  }{Text return}{HorizontalSpace  }{Placeholder expression}{SemiColon ;}{VerticalSpace  }{RightBrace }} (42)
 // CHECK-CC4: ObjCInstanceMethodDecl:{LeftParen (}{Text int}{RightParen )}{TypedText getInt}{HorizontalSpace  }{LeftBrace {}{VerticalSpace
 // CHECK-CC4: ObjCInstanceMethodDecl:{LeftParen (}{Text int}{RightParen )}{TypedText getSecondValue}{HorizontalSpace  }{LeftBrace {}{VerticalSpace
 // CHECK-CC4: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText getSelf}{HorizontalSpace  }{LeftBrace {}{VerticalSpace  }{Text return}{HorizontalSpace  }{Placeholder expression}{SemiColon ;}{VerticalSpace  }{RightBrace }} (40)
-// CHECK-CC4: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText initWithInt}{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text x}{HorizontalSpace  }{LeftBrace {}{VerticalSpace
-// CHECK-CC4: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText initWithTwoInts}{TypedText :}{LeftParen (}{Text inout }{Text int}{RightParen )}{Text x}{HorizontalSpace  }{TypedText second:}{LeftParen (}{Text int}{RightParen )}{Text y}{HorizontalSpace  }{LeftBrace {}{VerticalSpace
-// CHECK-CC4: ObjCInstanceMethodDecl:{LeftParen (}{Text int}{RightParen )}{TypedText setValue}{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text x}{HorizontalSpace  }{LeftBrace {}{VerticalSpace
+// CHECK-CC4: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText initWithInt:}{LeftParen (}{Text int}{RightParen )}{Text x}{HorizontalSpace  }{LeftBrace {}{VerticalSpace
+// CHECK-CC4: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText initWithTwoInts:}{LeftParen (}{Text inout }{Text int}{RightParen )}{Text x}{HorizontalSpace  }{TypedText second:}{LeftParen (}{Text int}{RightParen )}{Text y}{HorizontalSpace  }{LeftBrace {}{VerticalSpace
+// CHECK-CC4: ObjCInstanceMethodDecl:{LeftParen (}{Text int}{RightParen )}{TypedText setValue:}{LeftParen (}{Text int}{RightParen )}{Text x}{HorizontalSpace  }{LeftBrace {}{VerticalSpace
 // RUN: env CINDEXTEST_CODE_COMPLETE_PATTERNS=1 c-index-test -code-completion-at=%s:33:8 %s | FileCheck -check-prefix=CHECK-CC5 %s
 // CHECK-CC5: ObjCInstanceMethodDecl:{TypedText getInt}{HorizontalSpace  }{LeftBrace {}{VerticalSpace  }{Text return}{HorizontalSpace  }{Placeholder expression}{SemiColon ;}{VerticalSpace  }{RightBrace }} (42)
 // CHECK-CC5: ObjCInstanceMethodDecl:{TypedText getSecondValue}{HorizontalSpace  }{LeftBrace {}{VerticalSpace  }{Text return}{HorizontalSpace  }{Placeholder expression}{SemiColon ;}{VerticalSpace  }{RightBrace }} (40)
 // CHECK-CC5-NOT: {TypedText getSelf}{HorizontalSpace  }{LeftBrace {}{VerticalSpace
-// CHECK-CC5: ObjCInstanceMethodDecl:{TypedText setValue}{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text x}{HorizontalSpace  }{LeftBrace {}{VerticalSpace
+// CHECK-CC5: ObjCInstanceMethodDecl:{TypedText setValue:}{LeftParen (}{Text int}{RightParen )}{Text x}{HorizontalSpace  }{LeftBrace {}{VerticalSpace
 // RUN: env CINDEXTEST_CODE_COMPLETE_PATTERNS=1 c-index-test -code-completion-at=%s:37:7 %s | FileCheck -check-prefix=CHECK-CC6 %s
-// CHECK-CC6: ObjCInstanceMethodDecl:{TypedText abc}{HorizontalSpace  }{LeftBrace {}{VerticalSpace 
+// CHECK-CC6: ObjCInstanceMethodDecl:{TypedText abc}{HorizontalSpace  }{LeftBrace {}{VerticalSpace
 // CHECK-CC6: ObjCInstanceMethodDecl:{TypedText getSelf}{HorizontalSpace  }{LeftBrace {}{VerticalSpace  }{Text return}{HorizontalSpace  }{Placeholder expression}{SemiColon ;}{VerticalSpace  }{RightBrace }} (40)
-// CHECK-CC6: ObjCInstanceMethodDecl:{TypedText initWithInt}{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text x}{HorizontalSpace  }{LeftBrace {}{VerticalSpace 
-// CHECK-CC6: ObjCInstanceMethodDecl:{TypedText initWithTwoInts}{TypedText :}{LeftParen (}{Text inout }{Text int}{RightParen )}{Text x}{HorizontalSpace  }{TypedText second:}{LeftParen (}{Text int}{RightParen )}{Text y}{HorizontalSpace  }{LeftBrace {}{VerticalSpace 
+// CHECK-CC6: ObjCInstanceMethodDecl:{TypedText initWithInt:}{LeftParen (}{Text int}{RightParen )}{Text x}{HorizontalSpace  }{LeftBrace {}{VerticalSpace
+// CHECK-CC6: ObjCInstanceMethodDecl:{TypedText initWithTwoInts:}{LeftParen (}{Text inout }{Text int}{RightParen )}{Text x}{HorizontalSpace  }{TypedText second:}{LeftParen (}{Text int}{RightParen )}{Text y}{HorizontalSpace  }{LeftBrace {}{VerticalSpace
 // RUN: env CINDEXTEST_CODE_COMPLETE_PATTERNS=1 c-index-test -code-completion-at=%s:42:3 %s | FileCheck -check-prefix=CHECK-CC7 %s
 // CHECK-CC7: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText abc}{HorizontalSpace  }{LeftBrace {}{VerticalSpace  }{Text return}{HorizontalSpace  }{Placeholder expression}{SemiColon ;}{VerticalSpace  }{RightBrace }} (42)
-// CHECK-CC7: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText categoryFunction}{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text x}{HorizontalSpace  }{LeftBrace {}{VerticalSpace 
+// CHECK-CC7: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText categoryFunction:}{LeftParen (}{Text int}{RightParen )}{Text x}{HorizontalSpace  }{LeftBrace {}{VerticalSpace
 // CHECK-CC7: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText getSelf}{HorizontalSpace  }{LeftBrace {}{VerticalSpace  }{Text return}{HorizontalSpace  }{Placeholder expression}{SemiColon ;}{VerticalSpace  }{RightBrace }} (42)
 // RUN: env CINDEXTEST_CODE_COMPLETE_PATTERNS=1 c-index-test -code-completion-at=%s:52:21 %s | FileCheck -check-prefix=CHECK-CC8 %s
 // CHECK-CC8: ObjCInstanceMethodDecl:{ResultType id}{Informative first:}{TypedText second2:}{Text (float)y2}{HorizontalSpace  }{TypedText third:}{Text (double)z} (35)
@@ -157,9 +157,9 @@ -(void)foo {}
 // RUN: env CINDEXTEST_CODE_COMPLETE_PATTERNS=1 c-index-test -code-completion-at=%s:52:36 %s | FileCheck -check-prefix=CHECK-CCA %s
 // CHECK-CCA: NotImplemented:{TypedText y2} (40)
 // RUN: c-index-test -code-completion-at=%s:56:3 %s | FileCheck -check-prefix=CHECK-CCB %s
-// CHECK-CCB: ObjCInstanceMethodDecl:{LeftParen (}{Text int}{RightParen )}{TypedText first}{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text x}{HorizontalSpace  }{TypedText second2:}{LeftParen (}{Text float}{RightParen )}{Text y}{HorizontalSpace  }{TypedText third:}{LeftParen (}{Text double}{RightParen )}{Text z} (40)
+// CHECK-CCB: ObjCInstanceMethodDecl:{LeftParen (}{Text int}{RightParen )}{TypedText first:}{LeftParen (}{Text int}{RightParen )}{Text x}{HorizontalSpace  }{TypedText second2:}{LeftParen (}{Text float}{RightParen )}{Text y}{HorizontalSpace  }{TypedText third:}{LeftParen (}{Text double}{RightParen )}{Text z} (40)
 // RUN: c-index-test -code-completion-at=%s:56:8 %s | FileCheck -check-prefix=CHECK-CCC %s
-// CHECK-CCC: ObjCInstanceMethodDecl:{TypedText first}{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text x}{HorizontalSpace  }{TypedText second2:}{LeftParen (}{Text float}{RightParen )}{Text y}{HorizontalSpace  }{TypedText third:}{LeftParen (}{Text double}{RightParen )}{Text z} (40)
+// CHECK-CCC: ObjCInstanceMethodDecl:{TypedText first:}{LeftParen (}{Text int}{RightParen )}{Text x}{HorizontalSpace  }{TypedText second2:}{LeftParen (}{Text float}{RightParen )}{Text y}{HorizontalSpace  }{TypedText third:}{LeftParen (}{Text double}{RightParen )}{Text z} (40)
 // RUN: c-index-test -code-completion-at=%s:56:21 %s | FileCheck -check-prefix=CHECK-CCD %s
 // CHECK-CCD: ObjCInstanceMethodDecl:{ResultType id}{Informative first:}{TypedText second2:}{Text (float)y2}{HorizontalSpace  }{TypedText third:}{Text (double)z} (35)
 // CHECK-CCD: ObjCInstanceMethodDecl:{ResultType int}{Informative first:}{TypedText second2:}{Text (float)y}{HorizontalSpace  }{TypedText third:}{Text (double)z} (8)
@@ -214,11 +214,11 @@ -(void)foo {}
 
 // <rdar://problem/8939352>
 // RUN: c-index-test -code-completion-at=%s:68:9 %s | FileCheck -check-prefix=CHECK-8939352 %s
-// CHECK-8939352: ObjCInstanceMethodDecl:{TypedText method}{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text x}{HorizontalSpace  }{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text y} (40)
+// CHECK-8939352: ObjCInstanceMethodDecl:{TypedText method:}{LeftParen (}{Text int}{RightParen )}{Text x}{HorizontalSpace  }{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text y} (40)
 
 
 // RUN: c-index-test -code-completion-at=%s:72:2 %s | FileCheck -check-prefix=CHECK-ONEWAY %s
-// CHECK-ONEWAY: ObjCInstanceMethodDecl:{LeftParen (}{Text oneway }{Text void}{RightParen )}{TypedText method}{TypedText :}{LeftParen (}{Text in }{Text id}{RightParen )}{Text x} (40)
+// CHECK-ONEWAY: ObjCInstanceMethodDecl:{LeftParen (}{Text oneway }{Text void}{RightParen )}{TypedText method:}{LeftParen (}{Text in }{Text id}{RightParen )}{Text x} (40)
 
 // RUN: c-index-test -code-completion-at=%s:85:2 %s | FileCheck -check-prefix=CHECK-CLASSTY %s
 // CHECK-CLASSTY: ObjCInstanceMethodDecl:{LeftParen (}{Text Class<P1>}{RightParen )}{TypedText meth}
@@ -226,16 +226,16 @@ -(void)foo {}
 // CHECK-CLASSTY: ObjCInstanceMethodDecl:{LeftParen (}{Text MyObjectRef}{RightParen )}{TypedText meth3}
 
 // RUN: c-index-test -code-completion-at=%s:96:2 %s -target x86_64-apple-macosx10.7 | FileCheck -check-prefix=CHECK-NULLABILITY %s
-// CHECK-NULLABILITY: ObjCInstanceMethodDecl:{LeftParen (}{Text I2 *}{RightParen )}{TypedText produceI2}{TypedText :}{LeftParen (}{Text I2 *}{RightParen )}{Text i2} (40)
+// CHECK-NULLABILITY: ObjCInstanceMethodDecl:{LeftParen (}{Text I2 *}{RightParen )}{TypedText produceI2:}{LeftParen (}{Text I2 *}{RightParen )}{Text i2} (40)
 // CHECK-NULLABILITY: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText prop}
 // CHECK-NULLABILITY: ObjCInstanceMethodDecl:{LeftParen (}{Text void (^)(int * _Nullable)}{RightParen )}{TypedText propWB}
-// CHECK-NULLABILITY: ObjCInstanceMethodDecl:{LeftParen (}{Text void}{RightParen )}{TypedText setProp}{TypedText :}{LeftParen (}{Text id}{RightParen )}{Text prop}
-// CHECK-NULLABILITY: ObjCInstanceMethodDecl:{LeftParen (}{Text void}{RightParen )}{TypedText setPropWB}{TypedText :}{LeftParen (}{Text void (^)(int * _Nullable)}{RightParen )}{Text propWB}
-// CHECK-NULLABILITY: ObjCInstanceMethodDecl:{LeftParen (}{Text int * _Nullable *}{RightParen )}{TypedText something}{TypedText :}{LeftParen (}{Text void (^)(int * _Nullable)}{RightParen )}{Text b}
+// CHECK-NULLABILITY: ObjCInstanceMethodDecl:{LeftParen (}{Text void}{RightParen )}{TypedText setProp:}{LeftParen (}{Text id}{RightParen )}{Text prop}
+// CHECK-NULLABILITY: ObjCInstanceMethodDecl:{LeftParen (}{Text void}{RightParen )}{TypedText setPropWB:}{LeftParen (}{Text void (^)(int * _Nullable)}{RightParen )}{Text propWB}
+// CHECK-NULLABILITY: ObjCInstanceMethodDecl:{LeftParen (}{Text int * _Nullable *}{RightParen )}{TypedText something:}{LeftParen (}{Text void (^)(int * _Nullable)}{RightParen )}{Text b}
 
 // RUN: c-index-test -code-completion-at=%s:107:2 %s -target x86_64-apple-macosx10.7 | FileCheck -check-prefix=CHECK-NULLABILITY2 %s
 // CHECK-NULLABILITY2: ObjCInstanceMethodDecl:{LeftParen (}{Text instancetype}{RightParen )}{TypedText getI3} (40)
-// CHECK-NULLABILITY2: ObjCInstanceMethodDecl:{LeftParen (}{Text I3 *}{RightParen )}{TypedText produceI3}{TypedText :}{LeftParen (}{Text I3 *}{RightParen )}{Text i3} (40)
+// CHECK-NULLABILITY2: ObjCInstanceMethodDecl:{LeftParen (}{Text I3 *}{RightParen )}{TypedText produceI3:}{LeftParen (}{Text I3 *}{RightParen )}{Text i3} (40)
 
 @interface CompleteWithoutLeadingPrefix
 
@@ -253,8 +253,21 @@ @implementation CompleteWithoutLeadingPrefix
 
 // RUN: c-index-test -code-completion-at=%s:250:1 %s | FileCheck -check-prefix=CHECK-COMP-NO-PREFIX %s
 // CHECK-COMP-NO-PREFIX: NotImplemented:{TypedText @end} (40)
-// CHECK-COMP-NO-PREFIX: ObjCClassMethodDecl:{Text +}{HorizontalSpace  }{LeftParen (}{Text int}{RightParen )}{TypedText aClassMethod}{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text x} (40)
+// CHECK-COMP-NO-PREFIX: ObjCClassMethodDecl:{Text +}{HorizontalSpace  }{LeftParen (}{Text int}{RightParen )}{TypedText aClassMethod:}{LeftParen (}{Text int}{RightParen )}{Text x} (40)
 // CHECK-COMP-NO-PREFIX: ObjCInstanceMethodDecl:{Text -}{HorizontalSpace  }{LeftParen (}{Text void}{RightParen )}{TypedText aMethod} (40)
 // CHECK-COMP-NO-PREFIX: ObjCInterfaceDecl:{TypedText I1}
 // CHECK-COMP-NO-PREFIX: ObjCInstanceMethodDecl:{Text -}{HorizontalSpace  }{LeftParen (}{Text int}{RightParen )}{TypedText p} (40)
-// CHECK-COMP-NO-PREFIX: ObjCInstanceMethodDecl:{Text -}{HorizontalSpace  }{LeftParen (}{Text void}{RightParen )}{TypedText setP}{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text p} (40)
+// CHECK-COMP-NO-PREFIX: ObjCInstanceMethodDecl:{Text -}{HorizontalSpace  }{LeftParen (}{Text void}{RightParen )}{TypedText setP:}{LeftParen (}{Text int}{RightParen )}{Text p} (40)
+
+@interface NoNameSelectors
+- (void):(int)a;
+- (void):(int)a :(int)b;
+@end
+
+@implementation NoNameSelectors
+
+@end
+
+// RUN: c-index-test -code-completion-at=%s:268:1 %s | FileCheck -check-prefix=CHECK-NNS %s
+// CHECK-NNS: ObjCInstanceMethodDecl:{Text -}{HorizontalSpace }{LeftParen (}{Text void}{RightParen )}{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text a}
+// CHECK-NNS: ObjCInstanceMethodDecl:{Text -}{HorizontalSpace }{LeftParen (}{Text void}{RightParen )}{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text a}{HorizontalSpace }{TypedText :}{LeftParen (}{Text int}{RightParen )}{Text b}
diff --git a/clang/test/Index/complete-parameterized-classes.m b/clang/test/Index/complete-parameterized-classes.m
index ffef991acbeec..701cf5d410b35 100644
--- a/clang/test/Index/complete-parameterized-classes.m
+++ b/clang/test/Index/complete-parameterized-classes.m
@@ -65,9 +65,9 @@ void test3() {
 // CHECK-CC5: ObjCIvarDecl:{ResultType __kindof NSObject *}{TypedText myVar} (35)
 
 // RUN: c-index-test -code-completion-at=%s:37:2 %s | FileCheck -check-prefix=CHECK-CC6 %s
-// CHECK-CC6: ObjCInstanceMethodDecl:{LeftParen (}{Text void}{RightParen )}{TypedText apply2}{TypedText :}{LeftParen (}{Text void (^)(id, NSObject *)}{RightParen )}{Text block} (40)
-// CHECK-CC6: ObjCInstanceMethodDecl:{LeftParen (}{Text void}{RightParen )}{TypedText apply}{TypedText :}{LeftParen (}{Text void (^)(id, NSObject *)}{RightParen )}{Text block} (40)
-// CHECK-CC6: ObjCInstanceMethodDecl:{LeftParen (}{Text NSObject *}{RightParen )}{TypedText getit}{TypedText :}{LeftParen (}{Text id}{RightParen )}{Text val} (40)
+// CHECK-CC6: ObjCInstanceMethodDecl:{LeftParen (}{Text void}{RightParen )}{TypedText apply2:}{LeftParen (}{Text void (^)(id, NSObject *)}{RightParen )}{Text block} (40)
+// CHECK-CC6: ObjCInstanceMethodDecl:{LeftParen (}{Text void}{RightParen )}{TypedText apply:}{LeftParen (}{Text void (^)(id, NSObject *)}{RightParen )}{Text block} (40)
+// CHECK-CC6: ObjCInstanceMethodDecl:{LeftParen (}{Text NSObject *}{RightParen )}{TypedText getit:}{LeftParen (}{Text id}{RightParen )}{Text val} (40)
 // CHECK-CC6: ObjCInstanceMethodDecl:{LeftParen (}{Text id}{RightParen )}{TypedText prop} (40)
 
 // RUN: c-index-test -code-completion-at=%s:41:8 %s | FileCheck -check-prefix=CHECK-CC7 %s
diff --git a/clang/test/Index/coroutines.cpp b/clang/test/Index/coroutines.cpp
index 5853437926c68..000327ffaec49 100644
--- a/clang/test/Index/coroutines.cpp
+++ b/clang/test/Index/coroutines.cpp
@@ -7,7 +7,7 @@ using std::experimental::suspend_never;
 struct promise_void {
   void get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void return_void();
   void unhandled_exception();
 };
diff --git a/clang/test/Index/evaluate-cursor.cpp b/clang/test/Index/evaluate-cursor.cpp
index af396318aab7e..901e988bc99e5 100644
--- a/clang/test/Index/evaluate-cursor.cpp
+++ b/clang/test/Index/evaluate-cursor.cpp
@@ -26,6 +26,15 @@ template <typename d> class e {
   static const auto g = alignof(f);
 };
 
+constexpr static int calc_val() { return 1 + 2; }
+const auto the_value = calc_val() + sizeof(char);
+
+void vlaTest() {
+  int msize = 4;
+  float arr[msize];
+  [&arr] {};
+}
+
 // RUN: c-index-test -evaluate-cursor-at=%s:4:7 \
 // RUN:    -evaluate-cursor-at=%s:8:7 \
 // RUN:    -evaluate-cursor-at=%s:8:11 -std=c++11 %s | FileCheck %s
@@ -53,3 +62,16 @@ template <typename d> class e {
 // RUN:    -evaluate-cursor-at=%s:26:21 \
 // RUN:    -std=c++11 %s | FileCheck -check-prefix=CHECK-DOES-NOT-CRASH %s
 // CHECK-DOES-NOT-CRASH: Not Evaluatable
+
+// RUN: c-index-test -evaluate-cursor-at=%s:30:1 \
+// RUN:    -evaluate-cursor-at=%s:30:32 \
+// RUN:    -evaluate-cursor-at=%s:30:35 \
+// RUN:    -evaluate-cursor-at=%s:30:37 -std=c++11 %s | FileCheck %s -check-prefix=CHECK-EXPR
+// CHECK-EXPR: unsigned, Value: 4
+// CHECK-EXPR: Value: 3
+// CHECK-EXPR: unsigned, Value: 4
+// CHECK-EXPR: unsigned, Value: 1
+
+// RUN: c-index-test -evaluate-cursor-at=%s:35:5 \
+// RUN:    -std=c++11 %s | FileCheck -check-prefix=VLA %s
+// VLA: Not Evaluatable
diff --git a/clang/test/Misc/warning-flags.c b/clang/test/Misc/warning-flags.c
index 841c748b8025d..e4f9069b88c86 100644
--- a/clang/test/Misc/warning-flags.c
+++ b/clang/test/Misc/warning-flags.c
@@ -18,7 +18,7 @@ This test serves two purposes:
 
 The list of warnings below should NEVER grow.  It should gradually shrink to 0.
 
-CHECK: Warnings without flags (69):
+CHECK: Warnings without flags (68):
 
 CHECK-NEXT:   ext_expected_semi_decl_list
 CHECK-NEXT:   ext_explicit_specialization_storage_class
@@ -58,7 +58,6 @@ CHECK-NEXT:   warn_fe_macro_contains_embedded_newline
 CHECK-NEXT:   warn_ignoring_ftabstop_value
 CHECK-NEXT:   warn_implements_nscopying
 CHECK-NEXT:   warn_incompatible_qualified_id
-CHECK-NEXT:   warn_inline_namespace_reopened_noninline
 CHECK-NEXT:   warn_invalid_asm_cast_lvalue
 CHECK-NEXT:   warn_maynot_respond
 CHECK-NEXT:   warn_method_param_redefinition
diff --git a/clang/test/Modules/Inputs/codegen-flags/foo.h b/clang/test/Modules/Inputs/codegen-flags/foo.h
index 7b9c1cd8e086e..74cfab1771f62 100644
--- a/clang/test/Modules/Inputs/codegen-flags/foo.h
+++ b/clang/test/Modules/Inputs/codegen-flags/foo.h
@@ -1,4 +1,7 @@
+#ifndef FOO_H
+#define FOO_H
 struct foo {
 };
 inline void f1() {
 }
+#endif
diff --git a/clang/test/Modules/context-hash.c b/clang/test/Modules/context-hash.c
index 33dfb2f15a2ce..8bb7422f6a54f 100644
--- a/clang/test/Modules/context-hash.c
+++ b/clang/test/Modules/context-hash.c
@@ -1,3 +1,6 @@
+// This test verifies that only strict hashing includes search paths and
+// diagnostics in the module context hash.
+
 // RUN: rm -rf %t
 // RUN: %clang_cc1 -fsyntax-only -internal-isystem \
 // RUN:   %S/Inputs/System/usr/include -fmodules -fimplicit-module-maps \
@@ -20,8 +23,25 @@
 // RUN: echo %t > %t.path
 // RUN: cat %t.path %t1 %t2 %t3 %t4 | FileCheck %s
 
-// This test verifies that only strict hashing includes search paths and
-// diagnostics in the module context hash.
+// This tests things verified by ASTReader::checkLanguageOptions that are not
+// part of LangOpts.def.
+
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -fsyntax-only -internal-isystem \
+// RUN:   %S/Inputs/System/usr/include -fmodules -fimplicit-module-maps \
+// RUN:   -fmodules-cache-path=%t -x objective-c %s -Rmodule-build 2> %t1
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -fsyntax-only -internal-isystem \
+// RUN:   %S/Inputs/System/usr/include -fmodules -fimplicit-module-maps \
+// RUN:   -fobjc-runtime=macosx-1.0.0.0 \
+// RUN:   -fmodules-cache-path=%t -x objective-c %s -Rmodule-build 2> %t2
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -fsyntax-only -internal-isystem \
+// RUN:   %S/Inputs/System/usr/include -fmodules -fimplicit-module-maps \
+// RUN:   -fcomment-block-commands=lp,bj \
+// RUN:   -fmodules-cache-path=%t -x objective-c %s -Rmodule-build 2> %t3
+// RUN: echo %t > %t.path
+// RUN: cat %t.path %t1 %t2 %t3 | FileCheck --check-prefix=LANGOPTS %s
 
 #include <stdio.h>
 
@@ -32,3 +52,10 @@
 // CHECK: cstd-[[AST_HASH]].pcm'
 // CHECK-NOT: building module 'cstd' as '{{.*[/\\]}}[[CONTEXT_HASH]]{{[/\\]}}
 // CHECK: cstd-[[AST_HASH]].pcm'
+
+// LANGOPTS: [[PREFIX:(.*[/\\])+[a-zA-Z0-9.-]+]]
+// LANGOPTS: building module 'cstd' as '[[PREFIX]]{{[/\\]}}[[CONTEXT_HASH:[A-Z0-9]+]]{{[/\\]}}cstd-[[AST_HASH:[A-Z0-9]+]].pcm'
+// LANGOPTS-NOT: building module 'cstd' as '{{.*[/\\]}}[[CONTEXT_HASH]]{{[/\\]}}
+// LANGOPTS: cstd-[[AST_HASH]].pcm'
+// LANGOPTS-NOT: building module 'cstd' as '{{.*[/\\]}}[[CONTEXT_HASH]]{{[/\\]}}
+// LANGOPTS: cstd-[[AST_HASH]].pcm'
diff --git a/clang/test/OpenMP/declare_reduction_codegen.c b/clang/test/OpenMP/declare_reduction_codegen.c
index efc2466657f0a..f5695ffaac342 100644
--- a/clang/test/OpenMP/declare_reduction_codegen.c
+++ b/clang/test/OpenMP/declare_reduction_codegen.c
@@ -2,6 +2,9 @@
 // RUN: %clang_cc1 -fopenmp -x c -triple %itanium_abi_triple -emit-pch -o %t %s -femit-all-decls -disable-llvm-passes
 // RUN: %clang_cc1 -fopenmp -x c -triple %itanium_abi_triple -include-pch %t -verify %s -emit-llvm -o - -femit-all-decls -disable-llvm-passes | FileCheck --check-prefix=CHECK-LOAD %s
 
+// RUN: %clang_cc1 -fopenmp -x c -triple %itanium_abi_triple -emit-pch -o %t %s -femit-all-decls -disable-llvm-passes -fopenmp-version=45
+// RUN: %clang_cc1 -fopenmp -x c -triple %itanium_abi_triple -include-pch %t -verify %s -emit-llvm -o - -femit-all-decls -disable-llvm-passes -fopenmp-version=45 | FileCheck --check-prefixes=CHECK-LOAD,OMP45-LOAD %s
+
 // RUN: %clang_cc1 -verify -fopenmp-simd -x c -emit-llvm %s -triple %itanium_abi_triple -o - -femit-all-decls -disable-llvm-passes | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c -triple %itanium_abi_triple -emit-pch -o %t %s -femit-all-decls -disable-llvm-passes
 // RUN: %clang_cc1 -fopenmp-simd -x c -triple %itanium_abi_triple -include-pch %t -verify %s -emit-llvm -o - -femit-all-decls -disable-llvm-passes | FileCheck --check-prefix SIMD-ONLY0 %s
@@ -146,18 +149,18 @@ int main() {
   return 0;
 }
 
-// CHECK-LOAD: define internal {{.*}}void @{{[^(]+}}(i32* noalias %0, i32* noalias %1)
-// CHECK-LOAD: [[MUL:%.+]] = mul nsw i32
-// CHECK-LOAD-NEXT: store i32 [[MUL]], i32*
-// CHECK-LOAD-NEXT: ret void
-// CHECK-LOAD-NEXT: }
+// OMP45-LOAD: define internal {{.*}}void @{{[^(]+}}(i32* noalias %0, i32* noalias %1)
+// OMP45-LOAD: [[MUL:%.+]] = mul nsw i32
+// OMP45-LOAD-NEXT: store i32 [[MUL]], i32*
+// OMP45-LOAD-NEXT: ret void
+// OMP45-LOAD-NEXT: }
 
-// CHECK-LOAD: define internal {{.*}}void @{{[^(]+}}(i8* noalias %0, i8* noalias %1)
-// CHECK-LOAD: sext i8
-// CHECK-LOAD: sext i8
-// CHECK-LOAD: [[MUL:%.+]] = mul nsw i32
-// CHECK-LOAD-NEXT: [[TRUNC:%.+]] = trunc i32 [[MUL]] to i8
-// CHECK-LOAD-NEXT: store i8 [[TRUNC]], i8*
-// CHECK-LOAD-NEXT: ret void
-// CHECK-LOAD-NEXT: }
+// OMP45-LOAD: define internal {{.*}}void @{{[^(]+}}(i8* noalias %0, i8* noalias %1)
+// OMP45-LOAD: sext i8
+// OMP45-LOAD: sext i8
+// OMP45-LOAD: [[MUL:%.+]] = mul nsw i32
+// OMP45-LOAD-NEXT: [[TRUNC:%.+]] = trunc i32 [[MUL]] to i8
+// OMP45-LOAD-NEXT: store i8 [[TRUNC]], i8*
+// OMP45-LOAD-NEXT: ret void
+// OMP45-LOAD-NEXT: }
 #endif
diff --git a/clang/test/OpenMP/declare_reduction_codegen.cpp b/clang/test/OpenMP/declare_reduction_codegen.cpp
index 8d49d81cd0e36..6209e00ba0a3e 100644
--- a/clang/test/OpenMP/declare_reduction_codegen.cpp
+++ b/clang/test/OpenMP/declare_reduction_codegen.cpp
@@ -1,6 +1,10 @@
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -emit-llvm %s -triple x86_64-linux -fexceptions -fcxx-exceptions -o - -femit-all-decls -disable-llvm-passes | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-linux -fexceptions -fcxx-exceptions -emit-pch -o %t %s -femit-all-decls -disable-llvm-passes
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-linux -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -femit-all-decls -disable-llvm-passes | FileCheck --check-prefix=CHECK-LOAD %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-linux -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -femit-all-decls -disable-llvm-passes | FileCheck --check-prefixes=CHECK-LOAD,OMP50-LOAD %s
+
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -emit-llvm %s -triple x86_64-linux -fexceptions -fcxx-exceptions -o - -femit-all-decls -disable-llvm-passes | FileCheck %s --check-prefixes=CHECK,OMP45
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple x86_64-linux -fexceptions -fcxx-exceptions -emit-pch -o %t %s -femit-all-decls -disable-llvm-passes
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple x86_64-linux -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -femit-all-decls -disable-llvm-passes | FileCheck --check-prefixes=CHECK-LOAD,OMP45-LOAD %s
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -emit-llvm %s -triple x86_64-linux -fexceptions -fcxx-exceptions -o - -femit-all-decls -disable-llvm-passes | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-linux -fexceptions -fcxx-exceptions -emit-pch -o %t %s -femit-all-decls -disable-llvm-passes
@@ -14,25 +18,25 @@
 // CHECK: [[SSS_INT:.+]] = type { i32 }
 // CHECK-LOAD: [[SSS_INT:.+]] = type { i32 }
 
-// CHECK: add
+// OMP45: add
 void add(short &out, short &in) {}
 
 #pragma omp declare reduction(my_add : short : add(omp_out, omp_in))
 
-// CHECK: define internal void @.
-// CHECK: call void @{{.+}}add{{.+}}(
-// CHECK: ret void
+// OMP45: define internal void @.
+// OMP45: call void @{{.+}}add{{.+}}(
+// OMP45: ret void
 
-// CHECK: foo_reduction_array
+// OMP45: foo_reduction_array
 void foo_reduction_array() {
   short y[1];
-  // CHECK: call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(
+  // OMP45: call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(
 #pragma omp parallel for reduction(my_add : y)
   for (int i = 0; i < 1; i++) {
   }
 }
 
-// CHECK: define internal void @
+// OMP45: define internal void @
 
 #pragma omp declare reduction(+ : int, char : omp_out *= omp_in)
 // CHECK: define internal {{.*}}void @{{[^(]+}}(i32* noalias %0, i32* noalias %1)
@@ -181,17 +185,17 @@ int main() {
 // CHECK-LABEL: i32 @{{.+}}foo{{[^(].+}}(i32
 // CHECK-LOAD-LABEL: i32 @{{.+}}foo{{[^(].+}}(i32
 
-// CHECK-LOAD: define internal {{.*}}void @{{[^(]+}}(i32* noalias %0, i32* noalias %1)
-// CHECK-LOAD: [[XOR:%.+]] = xor i32
-// CHECK-LOAD-NEXT: store i32 [[XOR]], i32*
-// CHECK-LOAD-NEXT: ret void
-// CHECK-LOAD-NEXT: }
-
-// CHECK-LOAD: define internal {{.*}}void @{{[^(]+}}(i32* noalias %0, i32* noalias %1)
-// CHECK-LOAD: [[ADD:%.+]] = add nsw i32 24,
-// CHECK-LOAD-NEXT: store i32 [[ADD]], i32*
-// CHECK-LOAD-NEXT: ret void
-// CHECK-LOAD-NEXT: }
+// OMP45-LOAD: define internal {{.*}}void @{{[^(]+}}(i32* noalias %0, i32* noalias %1)
+// OMP45-LOAD: [[XOR:%.+]] = xor i32
+// OMP45-LOAD-NEXT: store i32 [[XOR]], i32*
+// OMP45-LOAD-NEXT: ret void
+// OMP45-LOAD-NEXT: }
+
+// OMP45-LOAD: define internal {{.*}}void @{{[^(]+}}(i32* noalias %0, i32* noalias %1)
+// OMP45-LOAD: [[ADD:%.+]] = add nsw i32 24,
+// OMP45-LOAD-NEXT: store i32 [[ADD]], i32*
+// OMP45-LOAD-NEXT: ret void
+// OMP45-LOAD-NEXT: }
 
 // CHECK: define internal {{.*}}void @{{[^(]+}}(i32* noalias %0, i32* noalias %1)
 // CHECK: [[ADD:%.+]] = add nsw i32
diff --git a/clang/test/OpenMP/declare_target_ast_print.cpp b/clang/test/OpenMP/declare_target_ast_print.cpp
index 510162f808d2e..14115e6a3ad6a 100644
--- a/clang/test/OpenMP/declare_target_ast_print.cpp
+++ b/clang/test/OpenMP/declare_target_ast_print.cpp
@@ -33,9 +33,7 @@ void bazz();
 #endif // _OPENMP
 
 int out_decl_target = 0;
-#if _OPENMP == 201811
 #pragma omp declare target (out_decl_target)
-#endif // _OPENMP
 
 // CHECK: #pragma omp declare target{{$}}
 // CHECK: int out_decl_target = 0;
diff --git a/clang/test/OpenMP/declare_target_codegen.cpp b/clang/test/OpenMP/declare_target_codegen.cpp
index e55397ca74e75..0cd725ac5665f 100644
--- a/clang/test/OpenMP/declare_target_codegen.cpp
+++ b/clang/test/OpenMP/declare_target_codegen.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc -DLOAD
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -DLOAD | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -emit-pch -o %t
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -o - -DLOAD | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc -DLOAD
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -DLOAD | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -emit-pch -o %t
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -o - -DLOAD | FileCheck %s
 
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - -fopenmp-version=50 -DOMP5 | FileCheck %s --check-prefix HOST5
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc -fopenmp-version=50 -DOMP5
@@ -13,10 +13,10 @@
 // RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc -fopenmp-version=50 -DOMP5
 // RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-version=50 -DOMP5 | FileCheck %s --check-prefix SIMD-ONLY
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o -| FileCheck %s --check-prefix SIMD-ONLY
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -emit-pch -o %t
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -o - | FileCheck %s --check-prefix SIMD-ONLY
+// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc -fopenmp-version=45
+// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-version=45 | FileCheck %s --check-prefix SIMD-ONLY
+// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -emit-pch -o %t -fopenmp-version=45
+// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -o - -fopenmp-version=45 | FileCheck %s --check-prefix SIMD-ONLY
 
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/declare_target_messages.cpp b/clang/test/OpenMP/declare_target_messages.cpp
index 1a371d699789f..3a78e492af587 100644
--- a/clang/test/OpenMP/declare_target_messages.cpp
+++ b/clang/test/OpenMP/declare_target_messages.cpp
@@ -1,10 +1,10 @@
-// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp45 -fopenmp -fnoopenmp-use-tls -ferror-limit 100 -o - %s
+// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp45 -fopenmp -fopenmp-version=45 -fnoopenmp-use-tls -ferror-limit 100 -o - %s
 // RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp5,host5 -fopenmp -fopenmp-version=50 -fnoopenmp-use-tls -ferror-limit 100 -o - %s
 // RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp5,dev5 -fopenmp -fopenmp-is-device -fopenmp-targets=x86_64-apple-macos10.7.0 -aux-triple x86_64-apple-macos10.7.0 -fopenmp-version=50 -fnoopenmp-use-tls -ferror-limit 100 -o - %s
 
 // RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp5,host5 -fopenmp-simd -fopenmp-version=50 -fnoopenmp-use-tls -ferror-limit 100 -o - %s
 // RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp5,host5 -fopenmp-simd -fopenmp-is-device -fopenmp-version=50 -fnoopenmp-use-tls -ferror-limit 100 -o - %s
-// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp45 -fopenmp-simd -fnoopenmp-use-tls -ferror-limit 100 -o - %s
+// RUN: %clang_cc1 -triple x86_64-apple-macos10.7.0 -verify=expected,omp45 -fopenmp-version=45 -fopenmp-simd -fnoopenmp-use-tls -ferror-limit 100 -o - %s
 
 #pragma omp end declare target // expected-error {{unexpected OpenMP directive '#pragma omp end declare target'}}
 
diff --git a/clang/test/OpenMP/declare_variant_implementation_vendor_codegen.cpp b/clang/test/OpenMP/declare_variant_implementation_vendor_codegen.cpp
index bc34d174c1491..70e24c027eb82 100644
--- a/clang/test/OpenMP/declare_variant_implementation_vendor_codegen.cpp
+++ b/clang/test/OpenMP/declare_variant_implementation_vendor_codegen.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple %itanium_abi_triple -emit-llvm %s -fexceptions -fcxx-exceptions -o - -fsanitize-address-use-after-scope | FileCheck %s --implicit-check-not='ret i32 {{6|7|8|9|10|12|13|14|15|19|21|22|23|24}}'
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple %itanium_abi_triple -emit-llvm %s -fexceptions -fcxx-exceptions -o - -fsanitize-address-use-after-scope -fopenmp-version=45 | FileCheck %s --implicit-check-not='ret i32 {{6|7|8|9|10|12|13|14|15|19|21|22|23|24}}'
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple %itanium_abi_triple -fexceptions -fcxx-exceptions -emit-pch -o %t -fopenmp-version=50 %s
 // RUN: %clang_cc1 -fopenmp -x c++ -triple %itanium_abi_triple -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -fopenmp-version=50 | FileCheck %s --implicit-check-not='ret i32 {{6|7|8|9|10|12|13|14|15|19|21|22|23|24}}'
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/distribute_parallel_for_default_messages.cpp b/clang/test/OpenMP/distribute_parallel_for_default_messages.cpp
index 0629ba096d0c2..67e4615ae8c01 100644
--- a/clang/test/OpenMP/distribute_parallel_for_default_messages.cpp
+++ b/clang/test/OpenMP/distribute_parallel_for_default_messages.cpp
@@ -2,8 +2,17 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp -ferror-limit 100 -o - %s -Wuninitialized
+
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp-simd -ferror-limit 100 -o - %s -Wuninitialized
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 template <class T, int N>
 T tmain(T argc) {
   int i;
@@ -14,12 +23,12 @@ T tmain(T argc) {
     foo();
 #pragma omp target
 #pragma omp teams
-#pragma omp distribute parallel for default( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp distribute parallel for default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (i = 0; i < argc; ++i)
     foo();
 #pragma omp target
 #pragma omp teams
-#pragma omp distribute parallel for default() // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp distribute parallel for default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
 #pragma omp target
@@ -34,7 +43,7 @@ T tmain(T argc) {
     foo();
 #pragma omp target
 #pragma omp teams
-#pragma omp distribute parallel for default(x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp distribute parallel for default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
 #pragma omp target
@@ -62,12 +71,12 @@ int main(int argc, char **argv) {
     foo();
 #pragma omp target
 #pragma omp teams
-#pragma omp distribute parallel for default( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp distribute parallel for default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (i = 0; i < argc; ++i)
     foo();
 #pragma omp target
 #pragma omp teams
-#pragma omp distribute parallel for default() // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp distribute parallel for default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
 #pragma omp target
@@ -82,7 +91,7 @@ int main(int argc, char **argv) {
     foo();
 #pragma omp target
 #pragma omp teams
-#pragma omp distribute parallel for default(x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp distribute parallel for default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
 #pragma omp target
@@ -98,5 +107,15 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
     foo();
 
+#ifdef OMP51
+#pragma omp target
+#pragma omp teams
+#pragma omp distribute parallel for default(firstprivate) // expected-note 2 {{explicit data sharing attribute requested here}}
+  for (i = 0; i < argc; ++i) {
+    ++x; // expected-error {{variable 'x' must have explicitly specified data sharing attributes}}
+    ++y; // expected-error {{variable 'y' must have explicitly specified data sharing attributes}}
+  }
+#endif
+
   return (tmain<int, 5>(argc) + tmain<char, 1>(argv[0][0])); // expected-note {{in instantiation of function template specialization 'tmain<int, 5>' requested here}} expected-note {{in instantiation of function template specialization 'tmain<char, 1>' requested here}}
 }
diff --git a/clang/test/OpenMP/distribute_parallel_for_simd_default_messages.cpp b/clang/test/OpenMP/distribute_parallel_for_simd_default_messages.cpp
index b9c5546ec5d95..9aab00f16c48f 100644
--- a/clang/test/OpenMP/distribute_parallel_for_simd_default_messages.cpp
+++ b/clang/test/OpenMP/distribute_parallel_for_simd_default_messages.cpp
@@ -2,8 +2,17 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 -o - %s -Wuninitialized -DOMP51 -fopenmp-version=51
+
+// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - %s -Wuninitialized -DOMP51 -fopenmp-version=51
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 template <class T, int N>
 T tmain(T argc) {
   int i;
@@ -14,12 +23,12 @@ T tmain(T argc) {
     foo();
 #pragma omp target
 #pragma omp teams
-#pragma omp distribute parallel for simd default( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp distribute parallel for simd default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (i = 0; i < argc; ++i)
     foo();
 #pragma omp target
 #pragma omp teams
-#pragma omp distribute parallel for simd default() // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp distribute parallel for simd default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
 #pragma omp target
@@ -34,7 +43,7 @@ T tmain(T argc) {
     foo();
 #pragma omp target
 #pragma omp teams
-#pragma omp distribute parallel for simd default(x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp distribute parallel for simd default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
 #pragma omp target
@@ -62,12 +71,12 @@ int main(int argc, char **argv) {
     foo();
 #pragma omp target
 #pragma omp teams
-#pragma omp distribute parallel for simd default( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp distribute parallel for simd default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (i = 0; i < argc; ++i)
     foo();
 #pragma omp target
 #pragma omp teams
-#pragma omp distribute parallel for simd default() // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp distribute parallel for simd default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
 #pragma omp target
@@ -82,7 +91,7 @@ int main(int argc, char **argv) {
     foo();
 #pragma omp target
 #pragma omp teams
-#pragma omp distribute parallel for simd default(x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp distribute parallel for simd default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
 #pragma omp target
@@ -90,6 +99,15 @@ int main(int argc, char **argv) {
 #pragma omp distribute parallel for simd default(none) // expected-note {{explicit data sharing attribute requested here}}
   for (i = 0; i < argc; ++i)  // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
     foo();
+#ifdef OpenMP51
+#pragma omp target
+#pragma omp teams
+#pragma omp distribute parallel for simd default(firstprivate) // expected-note 2 {{explicit data sharing attribute requested here}}
+  for (i = 0; i < argc; ++i) {
+    ++x; // expected-error {{variable 'x' must have explicitly specified data sharing attributes}}
+    ++y; // expected-error {{variable 'y' must have explicitly specified data sharing attributes}}
+  }
+#endif
 
 #pragma omp parallel default(none) // expected-note 2 {{explicit data sharing attribute requested here}}
 #pragma omp target
diff --git a/clang/test/OpenMP/distribute_parallel_for_simd_loop_messages.cpp b/clang/test/OpenMP/distribute_parallel_for_simd_loop_messages.cpp
index c58c56b6ca7af..8f7f3c44f7a77 100644
--- a/clang/test/OpenMP/distribute_parallel_for_simd_loop_messages.cpp
+++ b/clang/test/OpenMP/distribute_parallel_for_simd_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wno-openmp-mapping -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/distribute_simd_loop_messages.cpp b/clang/test/OpenMP/distribute_simd_loop_messages.cpp
index ff50109a1c405..5838ac7592a2a 100644
--- a/clang/test/OpenMP/distribute_simd_loop_messages.cpp
+++ b/clang/test/OpenMP/distribute_simd_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
 class S5 {
diff --git a/clang/test/OpenMP/driver.c b/clang/test/OpenMP/driver.c
index dad4479833e12..047478256f9f5 100644
--- a/clang/test/OpenMP/driver.c
+++ b/clang/test/OpenMP/driver.c
@@ -15,9 +15,9 @@
 // RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-version=1 | FileCheck --check-prefix=CHECK-DEFAULT-VERSION %s
 // RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-version=0 | FileCheck --check-prefix=CHECK-DEFAULT-VERSION %s
 // RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-version=100 | FileCheck --check-prefix=CHECK-DEFAULT-VERSION %s
-// RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-version=45 | FileCheck --check-prefix=CHECK-DEFAULT-VERSION %s
+// RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-version=50 | FileCheck --check-prefix=CHECK-DEFAULT-VERSION %s
 
-// CHECK-DEFAULT-VERSION: #define _OPENMP 201511
+// CHECK-DEFAULT-VERSION: #define _OPENMP 201811
 
 // RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-version=31 | FileCheck --check-prefix=CHECK-31-VERSION %s
 // CHECK-31-VERSION: #define _OPENMP 201107
@@ -26,8 +26,8 @@
 // CHECK-40-VERSION: #define _OPENMP 201307
 
 // RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-version=45 | FileCheck --check-prefix=CHECK-45-VERSION %s
-// RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-simd | FileCheck --check-prefix=CHECK-45-VERSION %s
-// RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-targets=x86_64-unknown-unknown -o - | FileCheck --check-prefix=CHECK-45-VERSION --check-prefix=CHECK-45-VERSION2 %s
+// RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-version=45 -fopenmp-simd | FileCheck --check-prefix=CHECK-45-VERSION %s
+// RUN: %clang %s -c -E -dM -fopenmp=libomp -fopenmp-version=45 -fopenmp-targets=x86_64-unknown-unknown -o - | FileCheck --check-prefix=CHECK-45-VERSION --check-prefix=CHECK-45-VERSION2 %s
 // CHECK-45-VERSION: #define _OPENMP 201511
 // CHECK-45-VERSION2: #define _OPENMP 201511
 
@@ -47,6 +47,7 @@
 // RUN: %clang %s -c -E -dM -fopenmp-simd -fopenmp-version=31 | FileCheck --check-prefix=CHECK-VERSION %s
 // RUN: %clang %s -c -E -dM -fopenmp-simd -fopenmp-version=40 | FileCheck --check-prefix=CHECK-VERSION %s
 // RUN: %clang %s -c -E -dM -fopenmp-simd -fopenmp-version=45 | FileCheck --check-prefix=CHECK-VERSION %s
+// RUN: %clang %s -c -E -dM -fopenmp-simd -fopenmp-version=51 | FileCheck --check-prefix=CHECK-VERSION %s
 
 // CHECK-VERSION-NOT: #define _OPENMP
 
diff --git a/clang/test/OpenMP/for_codegen.cpp b/clang/test/OpenMP/for_codegen.cpp
index f91301c8a897c..26b09c574f3c7 100644
--- a/clang/test/OpenMP/for_codegen.cpp
+++ b/clang/test/OpenMP/for_codegen.cpp
@@ -1,9 +1,10 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - -fsanitize-address-use-after-scope | FileCheck %s --check-prefix=CHECK --check-prefix=LIFETIME --check-prefix=OMP45
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - -fsanitize-address-use-after-scope | FileCheck %s --check-prefix=CHECK --check-prefix=LIFETIME --check-prefix=OMP45
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - -fsanitize-address-use-after-scope | FileCheck %s --check-prefix=CHECK --check-prefix=LIFETIME --check-prefix=OMP5
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t -fopenmp-version=50 %s
 // RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -fopenmp-version=50 | FileCheck %s --check-prefix=CHECK --check-prefix=OMP5
-// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK --check-prefix=OMP45
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp -fexceptions -fcxx-exceptions -debug-info-kind=line-tables-only -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=TERM_DEBUG
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -fopenmp-version=45 -emit-llvm -o - | FileCheck %s --check-prefix=CHECK --check-prefix=OMP45
+// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp -fexceptions -fcxx-exceptions -debug-info-kind=line-tables-only -gno-column-info -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=TERM_DEBUG
 // RUN: %clang_cc1 -main-file-name for_codegen.cpp %s -o - -emit-llvm -fprofile-instrument=clang -fprofile-instrument-path=for_codegen-test.profraw | FileCheck %s --check-prefix=PROF-INSTR-PATH
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck --check-prefix SIMD-ONLY0 %s
diff --git a/clang/test/OpenMP/for_collapse_messages.cpp b/clang/test/OpenMP/for_collapse_messages.cpp
index 2316ca6ba3f0d..6fe6ef79c4c39 100644
--- a/clang/test/OpenMP/for_collapse_messages.cpp
+++ b/clang/test/OpenMP/for_collapse_messages.cpp
@@ -1,13 +1,13 @@
-// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp %s -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp -std=c++98 %s -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp -fopenmp-version=45 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp -fopenmp-version=45 -std=c++98 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp -fopenmp-version=45 -std=c++11 %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,omp50 -fopenmp -fopenmp-version=50 %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,omp50 -fopenmp -fopenmp-version=50 -std=c++98 %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,omp50 -fopenmp -fopenmp-version=50 -std=c++11 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp-simd %s -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp-simd -std=c++98 %s -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp-simd -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp-simd -fopenmp-version=45 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp-simd -fopenmp-version=45 -std=c++98 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp-simd -fopenmp-version=45 -std=c++11 %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,omp50 -fopenmp-simd -fopenmp-version=50 %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,omp50 -fopenmp-simd -fopenmp-version=50 -std=c++98 %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,omp50 -fopenmp-simd -fopenmp-version=50 -std=c++11 %s -Wuninitialized
diff --git a/clang/test/OpenMP/for_loop_messages.cpp b/clang/test/OpenMP/for_loop_messages.cpp
index 087db755273a2..b423bfa0f31fa 100644
--- a/clang/test/OpenMP/for_loop_messages.cpp
+++ b/clang/test/OpenMP/for_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/for_simd_loop_messages.cpp b/clang/test/OpenMP/for_simd_loop_messages.cpp
index 99f30a64eb5e3..6c5820ab7c451 100644
--- a/clang/test/OpenMP/for_simd_loop_messages.cpp
+++ b/clang/test/OpenMP/for_simd_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/master_taskloop_loop_messages.cpp b/clang/test/OpenMP/master_taskloop_loop_messages.cpp
index 9b50439705c42..10a60f306a904 100644
--- a/clang/test/OpenMP/master_taskloop_loop_messages.cpp
+++ b/clang/test/OpenMP/master_taskloop_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/master_taskloop_simd_loop_messages.cpp b/clang/test/OpenMP/master_taskloop_simd_loop_messages.cpp
index e39dbceda0fce..a4eb21bf24b69 100644
--- a/clang/test/OpenMP/master_taskloop_simd_loop_messages.cpp
+++ b/clang/test/OpenMP/master_taskloop_simd_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/nesting_of_regions.cpp b/clang/test/OpenMP/nesting_of_regions.cpp
index 9824c1df5310e..f391d21df616d 100644
--- a/clang/test/OpenMP/nesting_of_regions.cpp
+++ b/clang/test/OpenMP/nesting_of_regions.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -verify=expected,omp45,omp45warn %s
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -verify=expected,omp45,omp45warn %s
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -verify=expected,omp50 %s
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -verify=expected,omp45 -Wno-openmp %s
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -verify=expected,omp45 -Wno-source-uses-openmp %s
diff --git a/clang/test/OpenMP/nvptx_data_sharing.cpp b/clang/test/OpenMP/nvptx_data_sharing.cpp
index 1905999977358..1372246c7fc8c 100644
--- a/clang/test/OpenMP/nvptx_data_sharing.cpp
+++ b/clang/test/OpenMP/nvptx_data_sharing.cpp
@@ -2,7 +2,8 @@
 ///==========================================================================///
 
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix CK1
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix CK1 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -disable-llvm-optzns -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CK1 --check-prefix PAR
 
 // expected-no-diagnostics
 
@@ -26,11 +27,11 @@ void test_ds(){
     }
   }
 }
-// CK1: [[MEM_TY:%.+]] = type { [128 x i8] }
-// CK1-DAG: [[SHARED_GLOBAL_RD:@.+]] = common addrspace(3) global [[MEM_TY]] zeroinitializer
-// CK1-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
-// CK1-DAG: [[KERNEL_SIZE:@.+]] = internal unnamed_addr constant i64 8
-// CK1-DAG: [[KERNEL_SHARED:@.+]] = internal unnamed_addr constant i16 1
+// SEQ: [[MEM_TY:%.+]] = type { [128 x i8] }
+// SEQ-DAG: [[SHARED_GLOBAL_RD:@.+]] = common addrspace(3) global [[MEM_TY]] zeroinitializer
+// SEQ-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
+// SEQ-DAG: [[KERNEL_SIZE:@.+]] = internal unnamed_addr constant i64 8
+// SEQ-DAG: [[KERNEL_SHARED:@.+]] = internal unnamed_addr constant i16 1
 
 /// ========= In the worker function ========= ///
 // CK1: {{.*}}define internal void @__omp_offloading{{.*}}test_ds{{.*}}_worker()
@@ -44,16 +45,17 @@ void test_ds(){
 // CK1: [[SHAREDARGS2:%.+]] = alloca i8**
 // CK1: call void @__kmpc_kernel_init
 // CK1: call void @__kmpc_data_sharing_init_stack
-// CK1: [[SHARED_MEM_FLAG:%.+]] = load i16, i16* [[KERNEL_SHARED]],
-// CK1: [[SIZE:%.+]] = load i64, i64* [[KERNEL_SIZE]],
-// CK1: call void @__kmpc_get_team_static_memory(i16 0, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i64 [[SIZE]], i16 [[SHARED_MEM_FLAG]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
-// CK1: [[KERNEL_RD:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
-// CK1: [[GLOBALSTACK:%.+]] = getelementptr inbounds i8, i8* [[KERNEL_RD]], i64 0
+// SEQ: [[SHARED_MEM_FLAG:%.+]] = load i16, i16* [[KERNEL_SHARED]],
+// SEQ: [[SIZE:%.+]] = load i64, i64* [[KERNEL_SIZE]],
+// SEQ: call void @__kmpc_get_team_static_memory(i16 0, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i64 [[SIZE]], i16 [[SHARED_MEM_FLAG]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
+// SEQ: [[KERNEL_RD:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
+// SEQ: [[GLOBALSTACK:%.+]] = getelementptr inbounds i8, i8* [[KERNEL_RD]], i64 0
+// PAR: [[GLOBALSTACK:%.+]] = call i8* @__kmpc_data_sharing_push_stack(i{{32|64}} 8, i16 1)
 // CK1: [[GLOBALSTACK2:%.+]] = bitcast i8* [[GLOBALSTACK]] to %struct._globalized_locals_ty*
 // CK1: [[A:%.+]] = getelementptr inbounds %struct._globalized_locals_ty, %struct._globalized_locals_ty* [[GLOBALSTACK2]], i32 0, i32 0
 // CK1: [[B:%.+]] = getelementptr inbounds %struct._globalized_locals_ty, %struct._globalized_locals_ty* [[GLOBALSTACK2]], i32 0, i32 1
 // CK1: store i32 10, i32* [[A]]
-// CK1: call void @__kmpc_kernel_prepare_parallel({{.*}}, i16 1)
+// CK1: call void @__kmpc_kernel_prepare_parallel({{.*}})
 // CK1: call void @__kmpc_begin_sharing_variables(i8*** [[SHAREDARGS1]], i64 1)
 // CK1: [[SHARGSTMP1:%.+]] = load i8**, i8*** [[SHAREDARGS1]]
 // CK1: [[SHARGSTMP2:%.+]] = getelementptr inbounds i8*, i8** [[SHARGSTMP1]], i64 0
@@ -63,7 +65,7 @@ void test_ds(){
 // CK1: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
 // CK1: call void @__kmpc_end_sharing_variables()
 // CK1: store i32 100, i32* [[B]]
-// CK1: call void @__kmpc_kernel_prepare_parallel({{.*}}, i16 1)
+// CK1: call void @__kmpc_kernel_prepare_parallel({{.*}})
 // CK1: call void @__kmpc_begin_sharing_variables(i8*** [[SHAREDARGS2]], i64 2)
 // CK1: [[SHARGSTMP3:%.+]] = load i8**, i8*** [[SHAREDARGS2]]
 // CK1: [[SHARGSTMP4:%.+]] = getelementptr inbounds i8*, i8** [[SHARGSTMP3]], i64 0
@@ -75,8 +77,9 @@ void test_ds(){
 // CK1: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
 // CK1: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
 // CK1: call void @__kmpc_end_sharing_variables()
-// CK1: [[SHARED_MEM_FLAG:%.+]] = load i16, i16* [[KERNEL_SHARED]],
-// CK1: call void @__kmpc_restore_team_static_memory(i16 0, i16 [[SHARED_MEM_FLAG]])
+// SEQ: [[SHARED_MEM_FLAG:%.+]] = load i16, i16* [[KERNEL_SHARED]],
+// SEQ: call void @__kmpc_restore_team_static_memory(i16 0, i16 [[SHARED_MEM_FLAG]])
+// PAR: call void @__kmpc_data_sharing_pop_stack(i8* [[GLOBALSTACK]])
 // CK1: call void @__kmpc_kernel_deinit(i16 1)
 
 /// ========= In the data sharing wrapper function ========= ///
diff --git a/clang/test/OpenMP/nvptx_distribute_parallel_generic_mode_codegen.cpp b/clang/test/OpenMP/nvptx_distribute_parallel_generic_mode_codegen.cpp
index 87d2c622015de..62cbeaa1326a7 100644
--- a/clang/test/OpenMP/nvptx_distribute_parallel_generic_mode_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_distribute_parallel_generic_mode_codegen.cpp
@@ -1,9 +1,12 @@
 // Test target codegen - host bc file has to be created first.
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -disable-llvm-optzns -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix PAR
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -disable-llvm-optzns -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix PAR
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -disable-llvm-optzns -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix PAR
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
@@ -21,19 +24,20 @@ int main(int argc, char **argv) {
   return 0;
 }
 
-// CHECK: [[MEM_TY:%.+]] = type { [128 x i8] }
-// CHECK-DAG: [[SHARED_GLOBAL_RD:@.+]] = common addrspace(3) global [[MEM_TY]] zeroinitializer
-// CHECK-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
-// CHECK-DAG: [[KERNEL_SIZE:@.+]] = internal unnamed_addr constant i{{64|32}} 40
-// CHECK-DAG: [[KERNEL_SHARED:@.+]] = internal unnamed_addr constant i16 1
-// CHECK-DAG: @__omp_offloading_{{.*}}_main_l17_exec_mode = weak constant i8 0
-
-// CHECK: define weak void @__omp_offloading_{{.*}}_main_l17([10 x i32]* nonnull align 4 dereferenceable(40) %{{.+}}, [10 x i32]* nonnull align 4 dereferenceable(40) %{{.+}}, i32* nonnull align 4 dereferenceable(4) %{{.+}}, i{{64|32}} %{{.+}}, [10 x i32]* nonnull align 4 dereferenceable(40) %{{.+}})
-// CHECK: [[SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
-// CHECK: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE]],
-// CHECK: call void @__kmpc_get_team_static_memory(i16 1, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[SHARED]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
-// CHECK: [[PTR:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
-// CHECK: [[GEP:%.+]] = getelementptr inbounds i8, i8* [[PTR]], i{{64|32}} 0
+// SEQ: [[MEM_TY:%.+]] = type { [128 x i8] }
+// SEQ-DAG: [[SHARED_GLOBAL_RD:@.+]] = common addrspace(3) global [[MEM_TY]] zeroinitializer
+// SEQ-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
+// SEQ-DAG: [[KERNEL_SIZE:@.+]] = internal unnamed_addr constant i{{64|32}} 40
+// SEQ-DAG: [[KERNEL_SHARED:@.+]] = internal unnamed_addr constant i16 1
+// CHECK-DAG: @__omp_offloading_{{.*}}_main_l20_exec_mode = weak constant i8 0
+
+// CHECK: define weak void @__omp_offloading_{{.*}}_main_l20([10 x i32]* nonnull align 4 dereferenceable(40) %{{.+}}, [10 x i32]* nonnull align 4 dereferenceable(40) %{{.+}}, i32* nonnull align 4 dereferenceable(4) %{{.+}}, i{{64|32}} %{{.+}}, [10 x i32]* nonnull align 4 dereferenceable(40) %{{.+}})
+// SEQ: [[SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
+// SEQ: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE]],
+// SEQ: call void @__kmpc_get_team_static_memory(i16 1, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[SHARED]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
+// SEQ: [[PTR:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
+// SEQ: [[GEP:%.+]] = getelementptr inbounds i8, i8* [[PTR]], i{{64|32}} 0
+// PAR: [[GEP:%.+]] = call i8* @__kmpc_data_sharing_push_stack(i{{32|64}} 40, i16 1)
 // CHECK: [[STACK:%.+]] = bitcast i8* [[GEP]] to %struct._globalized_locals_ty*
 // CHECK: getelementptr inbounds %struct._globalized_locals_ty, %struct._globalized_locals_ty* [[STACK]], i{{32|64}} 0, i{{32|64}} 0
 // CHECK-NOT: getelementptr inbounds %struct._globalized_locals_ty, %struct._globalized_locals_ty* [[STACK]],
@@ -43,8 +47,9 @@ int main(int argc, char **argv) {
 
 // CHECK: call void @__kmpc_for_static_fini(%struct.ident_t* @
 
-// CHECK: [[SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
-// CHECK: call void @__kmpc_restore_team_static_memory(i16 1, i16 [[SHARED]])
+// SEQ: [[SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
+// SEQ: call void @__kmpc_restore_team_static_memory(i16 1, i16 [[SHARED]])
+// PAR: call void @__kmpc_data_sharing_pop_stack(i8* [[GEP]])
 
 // CHECK: define internal void [[PARALLEL]](
 // CHECK-NOT: call i8* @__kmpc_data_sharing_push_stack(
diff --git a/clang/test/OpenMP/nvptx_parallel_codegen.cpp b/clang/test/OpenMP/nvptx_parallel_codegen.cpp
index 9acc5aef72d1f..ad25e0d775d12 100644
--- a/clang/test/OpenMP/nvptx_parallel_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_parallel_codegen.cpp
@@ -1,9 +1,12 @@
 // Test target codegen - host bc file has to be created first.
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -disable-llvm-optzns -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix PAR
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
-// RUN: %clang_cc1 -verify -fopenmp -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -disable-llvm-optzns -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix PAR
+// RUN: %clang_cc1 -verify -fopenmp -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -disable-llvm-optzns -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix PAR
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
@@ -72,15 +75,15 @@ int bar(int n){
   return a;
 }
 
-// CHECK: [[MEM_TY:%.+]] = type { [128 x i8] }
-// CHECK-DAG: [[SHARED_GLOBAL_RD:@.+]] = common addrspace(3) global [[MEM_TY]] zeroinitializer
-// CHECK-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
-// CHECK-DAG: [[KERNEL_SIZE:@.+]] = internal unnamed_addr constant i{{64|32}} 4
-// CHECK-DAG: [[KERNEL_SHARED:@.+]] = internal unnamed_addr constant i16 1
+// SEQ: [[MEM_TY:%.+]] = type { [128 x i8] }
+// SEQ-DAG: [[SHARED_GLOBAL_RD:@.+]] = common addrspace(3) global [[MEM_TY]] zeroinitializer
+// SEQ-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
+// SEQ-DAG: [[KERNEL_SIZE:@.+]] = internal unnamed_addr constant i{{64|32}} 4
+// SEQ-DAG: [[KERNEL_SHARED:@.+]] = internal unnamed_addr constant i16 1
 
-// CHECK-NOT: define {{.*}}void {{@__omp_offloading_.+template.+l17}}_worker()
+// CHECK-NOT: define {{.*}}void {{@__omp_offloading_.+template.+l20}}_worker()
 
-// CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l26}}_worker()
+// CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l29}}_worker()
 // CHECK-DAG: [[OMP_EXEC_STATUS:%.+]] = alloca i8,
 // CHECK-DAG: [[OMP_WORK_FN:%.+]] = alloca i8*,
 // CHECK: store i8* null, i8** [[OMP_WORK_FN]],
@@ -89,7 +92,7 @@ int bar(int n){
 //
 // CHECK: [[AWAIT_WORK]]
 // CHECK: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0) #[[#CONVERGENT:]]
-// CHECK: [[KPR:%.+]] = call i1 @__kmpc_kernel_parallel(i8** [[OMP_WORK_FN]]
+// CHECK: [[KPR:%.+]] = call i1 @__kmpc_kernel_parallel(i8** [[OMP_WORK_FN]])
 // CHECK: [[KPRB:%.+]] = zext i1 [[KPR]] to i8
 // store i8 [[KPRB]], i8* [[OMP_EXEC_STATUS]], align 1
 // CHECK: [[WORK:%.+]] = load i8*, i8** [[OMP_WORK_FN]],
@@ -133,7 +136,7 @@ int bar(int n){
 // CHECK: [[EXIT]]
 // CHECK: ret void
 
-// CHECK: define {{.*}}void [[T6:@__omp_offloading_.+template.+l26]](i[[SZ:32|64]]
+// CHECK: define {{.*}}void [[T6:@__omp_offloading_.+template.+l29]](i[[SZ:32|64]]
 // Create local storage for each capture.
 // CHECK:  [[LOCAL_A:%.+]] = alloca i[[SZ]],
 // CHECK-DAG:  store i[[SZ]] [[ARG_A:%.+]], i[[SZ]]* [[LOCAL_A]]
@@ -163,13 +166,13 @@ int bar(int n){
 // CHECK-DAG: [[MWS:%.+]] = call i32 @llvm.nvvm.read.ptx.sreg.warpsize()
 // CHECK: [[MTMP1:%.+]] = sub nuw i32 [[MNTH]], [[MWS]]
 // CHECK: call void @__kmpc_kernel_init(i32 [[MTMP1]]
-// CHECK: call void @__kmpc_kernel_prepare_parallel(i8* bitcast (void (i16, i32)* [[PARALLEL_FN1]]_wrapper to i8*),
+// CHECK: call void @__kmpc_kernel_prepare_parallel(i8* bitcast (void (i16, i32)* [[PARALLEL_FN1]]_wrapper to i8*))
 // CHECK: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
 // CHECK: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
 // CHECK: call void @__kmpc_serialized_parallel(
 // CHECK: {{call|invoke}} void [[PARALLEL_FN3:@.+]](
 // CHECK: call void @__kmpc_end_serialized_parallel(
-// CHECK: call void @__kmpc_kernel_prepare_parallel(i8* bitcast (void (i16, i32)* [[PARALLEL_FN2]]_wrapper to i8*),
+// CHECK: call void @__kmpc_kernel_prepare_parallel(i8* bitcast (void (i16, i32)* [[PARALLEL_FN2]]_wrapper to i8*))
 // CHECK: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
 // CHECK: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
 // CHECK-64-DAG: load i32, i32* [[REF_A]]
@@ -199,7 +202,7 @@ int bar(int n){
 // CHECK: store i[[SZ]] 44, i[[SZ]]* %a,
 // CHECK: ret void
 
-// CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l43}}_worker()
+// CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l46}}_worker()
 // CHECK-DAG: [[OMP_EXEC_STATUS:%.+]] = alloca i8,
 // CHECK-DAG: [[OMP_WORK_FN:%.+]] = alloca i8*,
 // CHECK: store i8* null, i8** [[OMP_WORK_FN]],
@@ -208,7 +211,7 @@ int bar(int n){
 //
 // CHECK: [[AWAIT_WORK]]
 // CHECK: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
-// CHECK: [[KPR:%.+]] = call i1 @__kmpc_kernel_parallel(i8** [[OMP_WORK_FN]],
+// CHECK: [[KPR:%.+]] = call i1 @__kmpc_kernel_parallel(i8** [[OMP_WORK_FN]])
 // CHECK: [[KPRB:%.+]] = zext i1 [[KPR]] to i8
 // store i8 [[KPRB]], i8* [[OMP_EXEC_STATUS]], align 1
 // CHECK: [[WORK:%.+]] = load i8*, i8** [[OMP_WORK_FN]],
@@ -243,7 +246,7 @@ int bar(int n){
 // CHECK: [[EXIT]]
 // CHECK: ret void
 
-// CHECK: define {{.*}}void [[T6:@__omp_offloading_.+template.+l43]](i[[SZ:32|64]]
+// CHECK: define {{.*}}void [[T6:@__omp_offloading_.+template.+l46]](i[[SZ:32|64]]
 // Create local storage for each capture.
 // CHECK:  [[LOCAL_N:%.+]] = alloca i[[SZ]],
 // CHECK:  [[LOCAL_A:%.+]] = alloca i[[SZ]],
@@ -288,7 +291,7 @@ int bar(int n){
 // CHECK: br i1 [[CMP]], label {{%?}}[[IF_THEN:.+]], label {{%?}}[[IF_ELSE:.+]]
 //
 // CHECK: [[IF_THEN]]
-// CHECK: call void @__kmpc_kernel_prepare_parallel(i8* bitcast (void (i16, i32)* [[PARALLEL_FN4]]_wrapper to i8*),
+// CHECK: call void @__kmpc_kernel_prepare_parallel(i8* bitcast (void (i16, i32)* [[PARALLEL_FN4]]_wrapper to i8*))
 // CHECK: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
 // CHECK: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
 // CHECK: br label {{%?}}[[IF_END:.+]]
@@ -323,23 +326,25 @@ int bar(int n){
 
 // CHECK: declare void @__kmpc_barrier(%struct.ident_t*, i32) #[[#CONVERGENT]]
 
-// CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l55}}_worker()
-// CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l55}}(
+// CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l58}}_worker()
+// CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l58}}(
 // CHECK-32: [[A_ADDR:%.+]] = alloca i32,
 // CHECK-64: [[A_ADDR:%.+]] = alloca i64,
 // CHECK-64: [[CONV:%.+]] = bitcast i64* [[A_ADDR]] to i32*
-// CHECK: [[IS_SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
-// CHECK: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE]],
-// CHECK: call void @__kmpc_get_team_static_memory(i16 0, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[IS_SHARED]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
-// CHECK: [[KERNEL_RD:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
-// CHECK: [[STACK:%.+]] = getelementptr inbounds i8, i8* [[KERNEL_RD]], i{{64|32}} 0
+// SEQ: [[IS_SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
+// SEQ: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE]],
+// SEQ: call void @__kmpc_get_team_static_memory(i16 0, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[IS_SHARED]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
+// SEQ: [[KERNEL_RD:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
+// SEQ: [[STACK:%.+]] = getelementptr inbounds i8, i8* [[KERNEL_RD]], i{{64|32}} 0
+// PAR: [[STACK:%.+]] = call i8* @__kmpc_data_sharing_push_stack(i{{32|64}} 4, i16 1)
 // CHECK: [[BC:%.+]] = bitcast i8* [[STACK]] to %struct._globalized_locals_ty*
 // CHECK-32: [[A:%.+]] = load i32, i32* [[A_ADDR]],
 // CHECK-64: [[A:%.+]] = load i32, i32* [[CONV]],
 // CHECK: [[GLOBAL_A_ADDR:%.+]] = getelementptr inbounds %struct._globalized_locals_ty, %struct._globalized_locals_ty* [[BC]], i{{[0-9]+}} 0, i{{[0-9]+}} 0
 // CHECK: store i32 [[A]], i32* [[GLOBAL_A_ADDR]],
-// CHECK: [[IS_SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
-// CHECK: call void @__kmpc_restore_team_static_memory(i16 0, i16 [[IS_SHARED]])
+// SEQ: [[IS_SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
+// SEQ: call void @__kmpc_restore_team_static_memory(i16 0, i16 [[IS_SHARED]])
+// PAR: call void @__kmpc_data_sharing_pop_stack(i8* [[STACK]])
 
 // CHECK-LABEL: define internal void @{{.+}}(i32* noalias %{{.+}}, i32* noalias %{{.+}}, i32* nonnull align {{[0-9]+}} dereferenceable{{.*}})
 // CHECK:  [[CC:%.+]] = alloca i32,
diff --git a/clang/test/OpenMP/nvptx_parallel_for_codegen.cpp b/clang/test/OpenMP/nvptx_parallel_for_codegen.cpp
index 94b4959853d8e..4ef167de3b8a8 100644
--- a/clang/test/OpenMP/nvptx_parallel_for_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_parallel_for_codegen.cpp
@@ -1,6 +1,7 @@
 // Test target codegen - host bc file has to be created first.
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -disable-llvm-optzns -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix PAR
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
@@ -30,33 +31,35 @@ int bar(int n){
   return a;
 }
 
-// CHECK: [[MEM_TY:%.+]] = type { [128 x i8] }
-// CHECK-DAG: [[SHARED_GLOBAL_RD:@.+]] = common addrspace(3) global [[MEM_TY]] zeroinitializer
-// CHECK-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
-// CHECK-DAG: [[KERNEL_SIZE:@.+]] = internal unnamed_addr constant i{{64|32}} 4
-// CHECK-DAG: [[KERNEL_SHARED:@.+]] = internal unnamed_addr constant i16 1
+// SEQ: [[MEM_TY:%.+]] = type { [128 x i8] }
+// SEQ-DAG: [[SHARED_GLOBAL_RD:@.+]] = common addrspace(3) global [[MEM_TY]] zeroinitializer
+// SEQ-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
+// SEQ-DAG: [[KERNEL_SIZE:@.+]] = internal unnamed_addr constant i{{64|32}} 4
+// SEQ-DAG: [[KERNEL_SHARED:@.+]] = internal unnamed_addr constant i16 1
 
-// CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l12}}_worker()
+// CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l13}}_worker()
 // CHECK: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
 // CHECK: call i1 @__kmpc_kernel_parallel(
 // CHECK: call void @__omp_outlined___wrapper(
 
-// CHECK: define weak void @__omp_offloading_{{.*}}l12(
-// CHECK: call void @__omp_offloading_{{.*}}l12_worker()
+// CHECK: define weak void @__omp_offloading_{{.*}}l13(
+// CHECK: call void @__omp_offloading_{{.*}}l13_worker()
 // CHECK: call void @__kmpc_kernel_init(
 // CHECK: call void @__kmpc_data_sharing_init_stack()
-// CHECK: [[IS_SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
-// CHECK: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE]],
-// CHECK: call void @__kmpc_get_team_static_memory(i16 0, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i64 %7, i16 %6, i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
-// CHECK: [[KERNEL_RD:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
-// CHECK: [[STACK:%.+]] = getelementptr inbounds i8, i8* [[KERNEL_RD]], i{{64|32}} 0
+// SEQ: [[IS_SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
+// SEQ: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE]],
+// SEQ: call void @__kmpc_get_team_static_memory(i16 0, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i64 %7, i16 %6, i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
+// SEQ: [[KERNEL_RD:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
+// SEQ: [[STACK:%.+]] = getelementptr inbounds i8, i8* [[KERNEL_RD]], i{{64|32}} 0
+// PAR: [[STACK:%.+]] = call i8* @__kmpc_data_sharing_push_stack(i{{32|64}} 4, i16 1)
 // CHECK: call void @__kmpc_kernel_prepare_parallel(
 // CHECK: call void @__kmpc_begin_sharing_variables({{.*}}, i64 2)
 // CHECK: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
 // CHECK: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
 // CHECK: call void @__kmpc_end_sharing_variables()
-// CHECK: [[IS_SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
-// CHECK: call void @__kmpc_restore_team_static_memory(i16 0, i16 [[IS_SHARED]])
+// SEQ: [[IS_SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
+// SEQ: call void @__kmpc_restore_team_static_memory(i16 0, i16 [[IS_SHARED]])
+// PAR: call void @__kmpc_data_sharing_pop_stack(i8* [[STACK]])
 // CHECK: call void @__kmpc_kernel_deinit(i16 1)
 
 // CHECK: define internal void @__omp_outlined__(
diff --git a/clang/test/OpenMP/nvptx_target_codegen.cpp b/clang/test/OpenMP/nvptx_target_codegen.cpp
index 91f31185d8c1a..56f04cb01f0aa 100644
--- a/clang/test/OpenMP/nvptx_target_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_target_codegen.cpp
@@ -612,7 +612,7 @@ int baz(int f, double &a) {
 // CHECK: call void @__kmpc_end_serialized_parallel(%struct.ident_t* [[UNKNOWN]], i32 [[GTID]])
 // CHECK: br label
 
-// CHECK: call void @__kmpc_kernel_prepare_parallel(i8* bitcast (void (i16, i32)* @{{.+}} to i8*), i16 1)
+// CHECK: call void @__kmpc_kernel_prepare_parallel(i8* bitcast (void (i16, i32)* @{{.+}} to i8*))
 // CHECK: call void @__kmpc_begin_sharing_variables(i8*** [[SHARED_PTR:%.+]], i{{64|32}} 2)
 // CHECK: [[SHARED:%.+]] = load i8**, i8*** [[SHARED_PTR]],
 // CHECK: [[REF:%.+]] = getelementptr inbounds i8*, i8** [[SHARED]], i{{64|32}} 0
diff --git a/clang/test/OpenMP/nvptx_target_simd_codegen.cpp b/clang/test/OpenMP/nvptx_target_simd_codegen.cpp
index 073d6fa2f14e3..7a1f01c1f1ad4 100644
--- a/clang/test/OpenMP/nvptx_target_simd_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_target_simd_codegen.cpp
@@ -78,7 +78,6 @@ int bar(int n){
 // CHECK: call void @__kmpc_spmd_kernel_init(i32 %{{.+}}, i16 0, i16 0)
 // CHECK-NOT: call void @__kmpc_for_static_init
 // CHECK-NOT: call void @__kmpc_for_static_fini
-// CHECK-NOT: call i32 @__kmpc_nvptx_simd_reduce_nowait(
 // CHECK-NOT: call void @__kmpc_nvptx_end_reduce_nowait(
 // CHECK: call void @__kmpc_spmd_kernel_deinit_v2(i16 0)
 // CHECK: ret void
diff --git a/clang/test/OpenMP/nvptx_target_teams_codegen.cpp b/clang/test/OpenMP/nvptx_target_teams_codegen.cpp
index 3ab955fa85080..8ff393f074e4a 100644
--- a/clang/test/OpenMP/nvptx_target_teams_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_target_teams_codegen.cpp
@@ -68,7 +68,7 @@ int bar(int n){
   //
   // CHECK: [[AWAIT_WORK]]
   // CHECK: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
-  // CHECK: [[KPR:%.+]] = call i1 @__kmpc_kernel_parallel(i8** [[OMP_WORK_FN]], i16 1)
+  // CHECK: [[KPR:%.+]] = call i1 @__kmpc_kernel_parallel(i8** [[OMP_WORK_FN]])
   // CHECK: [[KPRB:%.+]] = zext i1 [[KPR]] to i8
   // store i8 [[KPRB]], i8* [[OMP_EXEC_STATUS]], align 1
   // CHECK: [[WORK:%.+]] = load i8*, i8** [[OMP_WORK_FN]],
@@ -154,7 +154,7 @@ int bar(int n){
   //
   // CHECK: [[AWAIT_WORK]]
   // CHECK: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
-  // CHECK: [[KPR:%.+]] = call i1 @__kmpc_kernel_parallel(i8** [[OMP_WORK_FN]], i16 1)
+  // CHECK: [[KPR:%.+]] = call i1 @__kmpc_kernel_parallel(i8** [[OMP_WORK_FN]])
   // CHECK: [[KPRB:%.+]] = zext i1 [[KPR]] to i8
   // store i8 [[KPRB]], i8* [[OMP_EXEC_STATUS]], align 1
   // CHECK: [[WORK:%.+]] = load i8*, i8** [[OMP_WORK_FN]],
diff --git a/clang/test/OpenMP/nvptx_target_teams_distribute_codegen.cpp b/clang/test/OpenMP/nvptx_target_teams_distribute_codegen.cpp
index abd9f4ae715f2..4f23f18730cc2 100644
--- a/clang/test/OpenMP/nvptx_target_teams_distribute_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_target_teams_distribute_codegen.cpp
@@ -1,17 +1,20 @@
 // Test target codegen - host bc file has to be created first.
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix PAR
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
-// RUN: %clang_cc1 -verify -fopenmp -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix PAR
+// RUN: %clang_cc1 -verify -fopenmp -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix PAR
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
 
-// CHECK: [[MEM_TY:%.+]] = type { [128 x i8] }
-// CHECK-DAG: {{@__omp_offloading_.+}}_l20_exec_mode = weak constant i8 1
-// CHECK-DAG: [[KERNEL_SIZE:@.+]] = internal unnamed_addr constant i{{64|32}} 4
-// CHECK-DAG: [[KERNEL_SHARED:@.+]] = internal unnamed_addr constant i16 1
+// SEQ: [[MEM_TY:%.+]] = type { [128 x i8] }
+// SEQ-DAG: {{@__omp_offloading_.+}}_l23_exec_mode = weak constant i8 1
+// SEQ-DAG: [[KERNEL_SIZE:@.+]] = internal unnamed_addr constant i{{64|32}} 4
+// SEQ-DAG: [[KERNEL_SHARED:@.+]] = internal unnamed_addr constant i16 1
 
 template<typename tx>
 tx ftemplate(int n) {
@@ -35,10 +38,10 @@ int bar(int n){
   return a;
 }
 
-  // CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l20}}_worker()
+  // CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l23}}_worker()
   // CHECK: ret void
 
-  // CHECK: define {{.*}}void {{@__omp_offloading_.+template.+l20}}()
+  // CHECK: define {{.*}}void {{@__omp_offloading_.+template.+l23}}()
 
   // CHECK-DAG: [[TID:%.+]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x()
   // CHECK-DAG: [[NTH:%.+]] = call i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
@@ -48,7 +51,7 @@ int bar(int n){
   // CHECK: br i1 [[IS_WORKER]], label {{%?}}[[WORKER:.+]], label {{%?}}[[CHECK_MASTER:.+]]
   //
   // CHECK: [[WORKER]]
-  // CHECK: {{call|invoke}} void {{@__omp_offloading_.+template.+l20}}_worker()
+  // CHECK: {{call|invoke}} void {{@__omp_offloading_.+template.+l23}}_worker()
   // CHECK: br label {{%?}}[[EXIT:.+]]
   //
   // CHECK: [[CHECK_MASTER]]
@@ -75,16 +78,17 @@ int bar(int n){
   // CHECK: ret void
 
   // CHECK: define internal void [[PARALLEL]](i32* noalias %{{.+}}, i32* noalias %{{.+}})
-  // CHECK: [[SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
-  // CHECK: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE]],
-  // CHECK: call void @__kmpc_get_team_static_memory(i16 0, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* @{{.+}}, i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[SHARED]], i8** addrspacecast (i8* addrspace(3)* [[BUF:@.+]] to i8**))
-  // CHECK: [[PTR:%.+]] = load i8*, i8* addrspace(3)* [[BUF]],
-  // CHECK: [[ADDR:%.+]] = getelementptr inbounds i8, i8* [[PTR]], i{{64|32}} 0
+  // SEQ: [[SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
+  // SEQ: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE]],
+  // SEQ: call void @__kmpc_get_team_static_memory(i16 0, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* @{{.+}}, i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[SHARED]], i8** addrspacecast (i8* addrspace(3)* [[BUF:@.+]] to i8**))
+  // SEQ: [[PTR:%.+]] = load i8*, i8* addrspace(3)* [[BUF]],
+  // SEQ: [[ADDR:%.+]] = getelementptr inbounds i8, i8* [[PTR]], i{{64|32}} 0
+  // PAR: [[ADDR:%.+]] = call i8* @__kmpc_data_sharing_push_stack(i{{32|64}} 4, i16 1)
   // CHECK: [[RD:%.+]] = bitcast i8* [[ADDR]] to [[GLOB_TY:%.+]]*
   // CHECK: [[I_ADDR:%.+]] = getelementptr inbounds [[GLOB_TY]], [[GLOB_TY]]* [[RD]], i32 0, i32 0
   //
   // CHECK: call void @__kmpc_for_static_init_4(
-  // CHECK: call void @__kmpc_kernel_prepare_parallel(i8* bitcast (void (i16, i32)* @{{.+}} to i8*), i16 1)
+  // CHECK: call void @__kmpc_kernel_prepare_parallel(i8* bitcast (void (i16, i32)* @{{.+}} to i8*))
   // CHECK: call void @__kmpc_begin_sharing_variables(i8*** [[SHARED_VARS_PTR:%.+]], i{{64|32}} 1)
   // CHECK: [[SHARED_VARS_BUF:%.+]] = load i8**, i8*** [[SHARED_VARS_PTR]],
   // CHECK: [[VARS_BUF:%.+]] = getelementptr inbounds i8*, i8** [[SHARED_VARS_BUF]], i{{64|32}} 0
diff --git a/clang/test/OpenMP/nvptx_target_teams_distribute_parallel_for_codegen.cpp b/clang/test/OpenMP/nvptx_target_teams_distribute_parallel_for_codegen.cpp
index a7e81ba1b7405..c90a3d2248e84 100644
--- a/clang/test/OpenMP/nvptx_target_teams_distribute_parallel_for_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_target_teams_distribute_parallel_for_codegen.cpp
@@ -1,21 +1,25 @@
 // Test target codegen - host bc file has to be created first.
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix CHECK-DIV64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -fopenmp-optimistic-collapse -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-DIV32
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix CHECK-DIV64 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix CHECK-DIV64 --check-prefix PAR
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -fopenmp-optimistic-collapse -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-DIV32 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -fopenmp-optimistic-collapse -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-DIV32 --check-prefix PAR
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple i386-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix PAR
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix PAR
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
 
 // Check that the execution mode of all 5 target regions on the gpu is set to SPMD Mode.
-// CHECK-DAG: {{@__omp_offloading_.+l34}}_exec_mode = weak constant i8 0
-// CHECK-DAG: {{@__omp_offloading_.+l40}}_exec_mode = weak constant i8 0
-// CHECK-DAG: {{@__omp_offloading_.+l45}}_exec_mode = weak constant i8 0
-// CHECK-DAG: {{@__omp_offloading_.+l50}}_exec_mode = weak constant i8 0
-// CHECK-DAG: {{@__omp_offloading_.+l58}}_exec_mode = weak constant i8 0
-// CHECK-DAG: {{@__omp_offloading_.+l65}}_exec_mode = weak constant i8 0
+// CHECK-DAG: {{@__omp_offloading_.+l38}}_exec_mode = weak constant i8 0
+// CHECK-DAG: {{@__omp_offloading_.+l44}}_exec_mode = weak constant i8 0
+// CHECK-DAG: {{@__omp_offloading_.+l49}}_exec_mode = weak constant i8 0
+// CHECK-DAG: {{@__omp_offloading_.+l54}}_exec_mode = weak constant i8 0
+// CHECK-DAG: {{@__omp_offloading_.+l62}}_exec_mode = weak constant i8 0
+// CHECK-DAG: {{@__omp_offloading_.+l69}}_exec_mode = weak constant i8 0
 
 #define N 1000
 #define M 10
@@ -76,31 +80,33 @@ int bar(int n){
   return a;
 }
 
-// CHECK-DAG: [[MEM_TY:%.+]] = type { [128 x i8] }
-// CHECK-DAG: [[SHARED_GLOBAL_RD:@.+]] = common addrspace(3) global [[MEM_TY]] zeroinitializer
-// CHECK-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
-// CHECK-DAG: [[KERNEL_SIZE:@.+]] = internal unnamed_addr constant i{{64|32}} 4
-// CHECK-DAG: [[KERNEL_SHARED:@.+]] = internal unnamed_addr constant i16 1
+// SEQ-DAG: [[MEM_TY:%.+]] = type { [128 x i8] }
+// SEQ-DAG: [[SHARED_GLOBAL_RD:@.+]] = common addrspace(3) global [[MEM_TY]] zeroinitializer
+// SEQ-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
+// SEQ-DAG: [[KERNEL_SIZE:@.+]] = internal unnamed_addr constant i{{64|32}} 4
+// SEQ-DAG: [[KERNEL_SHARED:@.+]] = internal unnamed_addr constant i16 1
 
-// CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+}}_l34(
+// CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+}}_l38(
 // CHECK-DAG: [[THREAD_LIMIT:%.+]] = call i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
 // CHECK: call void @__kmpc_spmd_kernel_init(i32 [[THREAD_LIMIT]], i16 0, i16 0)
 // CHECK: call void [[PARALLEL:@.+]](
 // CHECK: call void @__kmpc_spmd_kernel_deinit_v2(i16 0)
 
 // CHECK: define internal void [[PARALLEL]](
-// CHECK: [[SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
-// CHECK: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE]],
-// CHECK: call void @__kmpc_get_team_static_memory(i16 1, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[SHARED]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
-// CHECK: [[TEAM_ALLOC:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
-// CHECK: [[ADDR:%.+]] = getelementptr inbounds i8, i8* [[TEAM_ALLOC]], i{{64|32}} 0
+// SEQ: [[SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
+// SEQ: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE]],
+// SEQ: call void @__kmpc_get_team_static_memory(i16 1, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[SHARED]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
+// SEQ: [[TEAM_ALLOC:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
+// SEQ: [[ADDR:%.+]] = getelementptr inbounds i8, i8* [[TEAM_ALLOC]], i{{64|32}} 0
+// PAR: [[ADDR:%.+]] = call i8* @__kmpc_data_sharing_push_stack(i{{32|64}} 4, i16 1)
 // CHECK: [[BC:%.+]] = bitcast i8* [[ADDR]] to [[REC:%.+]]*
 // CHECK: getelementptr inbounds [[REC]], [[REC]]* [[BC]], i{{[0-9]+}} 0, i{{[0-9]+}} 0
 // CHECK: call void @__kmpc_for_static_init_4({{.+}}, {{.+}}, {{.+}} 91,
 // CHECK: {{call|invoke}} void [[OUTL1:@.+]](
 // CHECK: call void @__kmpc_for_static_fini(
-// CHECK: [[SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
-// CHECK: call void @__kmpc_restore_team_static_memory(i16 1, i16 [[SHARED]])
+// SEQ: [[SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
+// SEQ: call void @__kmpc_restore_team_static_memory(i16 1, i16 [[SHARED]])
+// PAR: call void @__kmpc_data_sharing_pop_stack(i8* [[ADDR]])
 // CHECK: ret void
 
 // CHECK: define internal void [[OUTL1]](
@@ -233,13 +239,13 @@ int bar(int n){
 // CHECK: call void @__kmpc_for_static_fini(
 // CHECK: ret void
 
-// CHECK: define weak void @__omp_offloading_{{.*}}_l58(i[[SZ:64|32]] %{{[^,]+}}, [10 x [10 x i32]]* nonnull align {{[0-9]+}} dereferenceable{{.*}})
+// CHECK: define weak void @__omp_offloading_{{.*}}_l62(i[[SZ:64|32]] %{{[^,]+}}, [10 x [10 x i32]]* nonnull align {{[0-9]+}} dereferenceable{{.*}})
 // CHECK: call void [[OUTLINED:@__omp_outlined.*]](i32* %{{.+}}, i32* %{{.+}}, i[[SZ]] %{{.*}}, i[[SZ]] %{{.*}}, i[[SZ]] %{{.*}}, [10 x [10 x i32]]* %{{.*}})
 // CHECK: define internal void [[OUTLINED]](i32* noalias %{{.*}}, i32* noalias %{{.*}} i[[SZ]] %{{.+}}, i[[SZ]] %{{.+}}, i[[SZ]] %{{.+}}, [10 x [10 x i32]]* nonnull align {{[0-9]+}} dereferenceable{{.*}})
 // CHECK-DIV64: div i64
 // CHECK-DIV32-NO: div i64
 
-// CHECK: define weak void @__omp_offloading_{{.*}}_l65(i[[SZ:64|32]] %{{[^,]+}}, [1000 x i32]* nonnull align {{[0-9]+}} dereferenceable{{.*}}, i32* %{{[^)]+}})
+// CHECK: define weak void @__omp_offloading_{{.*}}_l69(i[[SZ:64|32]] %{{[^,]+}}, [1000 x i32]* nonnull align {{[0-9]+}} dereferenceable{{.*}}, i32* %{{[^)]+}})
 // CHECK: call void [[OUTLINED:@__omp_outlined.*]](i32* %{{.+}}, i32* %{{.+}}, i[[SZ]] %{{.*}}, i[[SZ]] %{{.*}}, i[[SZ]] %{{.*}}, [1000 x i32]* %{{.*}}, i32* %{{.*}})
 // CHECK: define internal void [[OUTLINED]](i32* noalias %{{.*}}, i32* noalias %{{.*}} i[[SZ]] %{{.+}}, i[[SZ]] %{{.+}}, i[[SZ]] %{{.+}}, [1000 x i32]* nonnull align {{[0-9]+}} dereferenceable{{.*}}, i32* %{{.*}})
 
diff --git a/clang/test/OpenMP/nvptx_target_teams_distribute_parallel_for_simd_codegen.cpp b/clang/test/OpenMP/nvptx_target_teams_distribute_parallel_for_simd_codegen.cpp
index 2d43ffed70163..50ce394461066 100644
--- a/clang/test/OpenMP/nvptx_target_teams_distribute_parallel_for_simd_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_target_teams_distribute_parallel_for_simd_codegen.cpp
@@ -1,18 +1,21 @@
 // Test target codegen - host bc file has to be created first.
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix PAR
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple i386-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix PAR
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fopenmp-cuda-mode -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix PAR
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
 
 // Check that the execution mode of all 4 target regions on the gpu is set to SPMD Mode.
-// CHECK-DAG: {{@__omp_offloading_.+l30}}_exec_mode = weak constant i8 0
-// CHECK-DAG: {{@__omp_offloading_.+l36}}_exec_mode = weak constant i8 0
-// CHECK-DAG: {{@__omp_offloading_.+l41}}_exec_mode = weak constant i8 0
-// CHECK-DAG: {{@__omp_offloading_.+l46}}_exec_mode = weak constant i8 0
+// CHECK-DAG: {{@__omp_offloading_.+l33}}_exec_mode = weak constant i8 0
+// CHECK-DAG: {{@__omp_offloading_.+l39}}_exec_mode = weak constant i8 0
+// CHECK-DAG: {{@__omp_offloading_.+l44}}_exec_mode = weak constant i8 0
+// CHECK-DAG: {{@__omp_offloading_.+l49}}_exec_mode = weak constant i8 0
 
 #define N 1000
 #define M 10
@@ -62,29 +65,31 @@ int bar(int n){
   return a;
 }
 
-// CHECK-DAG: [[MEM_TY:%.+]] = type { [128 x i8] }
-// CHECK-DAG: [[SHARED_GLOBAL_RD:@.+]] = common addrspace(3) global [[MEM_TY]] zeroinitializer
-// CHECK-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
-// CHECK-DAG: [[KERNEL_SIZE:@.+]] = internal unnamed_addr constant i{{64|32}} 4
-// CHECK-DAG: [[KERNEL_SHARED:@.+]] = internal unnamed_addr constant i16 1
+// SEQ-DAG: [[MEM_TY:%.+]] = type { [128 x i8] }
+// SEQ-DAG: [[SHARED_GLOBAL_RD:@.+]] = common addrspace(3) global [[MEM_TY]] zeroinitializer
+// SEQ-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
+// SEQ-DAG: [[KERNEL_SIZE:@.+]] = internal unnamed_addr constant i{{64|32}} 4
+// SEQ-DAG: [[KERNEL_SHARED:@.+]] = internal unnamed_addr constant i16 1
 
-// CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+}}_l30(
+// CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+}}_l33(
 // CHECK-DAG: [[THREAD_LIMIT:%.+]] = call i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
 // CHECK: call void @__kmpc_spmd_kernel_init(i32 [[THREAD_LIMIT]], i16 0, i16 0)
 // CHECK: call void @__kmpc_spmd_kernel_deinit_v2(i16 0)
 
-// CHECK: [[SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
-// CHECK: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE]],
-// CHECK: call void @__kmpc_get_team_static_memory(i16 1, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[SHARED]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
-// CHECK: [[TEAM_ALLOC:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
-// CHECK: [[PTR:%.+]] = getelementptr inbounds i8, i8* [[TEAM_ALLOC]], i{{64|32}} 0
+// SEQ: [[SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
+// SEQ: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE]],
+// SEQ: call void @__kmpc_get_team_static_memory(i16 1, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[SHARED]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
+// SEQ: [[TEAM_ALLOC:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
+// SEQ: [[PTR:%.+]] = getelementptr inbounds i8, i8* [[TEAM_ALLOC]], i{{64|32}} 0
+// PAR: [[PTR:%.+]] = call i8* @__kmpc_data_sharing_push_stack(i{{32|64}} 4, i16 1)
 // CHECK: [[BC:%.+]] = bitcast i8* [[PTR]] to [[REC:%.+]]*
 // CHECK: getelementptr inbounds [[REC]], [[REC]]* [[BC]], i{{[0-9]+}} 0, i{{[0-9]+}} 0
 // CHECK: call void @__kmpc_for_static_init_4({{.+}}, {{.+}}, {{.+}} 91,
 // CHECK: {{call|invoke}} void [[OUTL1:@.+]](
 // CHECK: call void @__kmpc_for_static_fini(
-// CHECK: [[SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
-// CHECK: call void @__kmpc_restore_team_static_memory(i16 1, i16 [[SHARED]])
+// SEQ: [[SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED]],
+// SEQ: call void @__kmpc_restore_team_static_memory(i16 1, i16 [[SHARED]])
+// PAR: call void @__kmpc_data_sharing_pop_stack(i8* [[PTR]])
 // CHECK: ret void
 
 // CHECK: define internal void [[OUTL1]](
diff --git a/clang/test/OpenMP/nvptx_teams_codegen.cpp b/clang/test/OpenMP/nvptx_teams_codegen.cpp
index a796ccc3e9677..36bb972175a54 100644
--- a/clang/test/OpenMP/nvptx_teams_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_teams_codegen.cpp
@@ -1,8 +1,10 @@
 // Test target codegen - host bc file has to be created first.
 // RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64 --check-prefix SEQ
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CK1 --check-prefix CK1-64 --check-prefix PAR
 // RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32 --check-prefix SEQ
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CK1 --check-prefix CK1-32 --check-prefix PAR
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
@@ -27,13 +29,13 @@ int main (int argc, char **argv) {
   return tmain(argv);
 }
 
-// CK1: [[MEM_TY:%.+]] = type { [128 x i8] }
-// CK1-DAG: [[SHARED_GLOBAL_RD:@.+]] = common addrspace(3) global [[MEM_TY]] zeroinitializer
-// CK1-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
-// CK1-DAG: [[KERNEL_SIZE1:@.+]] = internal unnamed_addr constant i{{64|32}} 4
-// CK1-DAG: [[KERNEL_SIZE2:@.+]] = internal unnamed_addr constant i{{64|32}} {{8|4}}
-// CK1-DAG: [[KERNEL_SHARED1:@.+]] = internal unnamed_addr constant i16 1
-// CK1-DAG: [[KERNEL_SHARED2:@.+]] = internal unnamed_addr constant i16 1
+// SEQ: [[MEM_TY:%.+]] = type { [128 x i8] }
+// SEQ-DAG: [[SHARED_GLOBAL_RD:@.+]] = common addrspace(3) global [[MEM_TY]] zeroinitializer
+// SEQ-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
+// SEQ-DAG: [[KERNEL_SIZE1:@.+]] = internal unnamed_addr constant i{{64|32}} 4
+// SEQ-DAG: [[KERNEL_SIZE2:@.+]] = internal unnamed_addr constant i{{64|32}} {{8|4}}
+// SEQ-DAG: [[KERNEL_SHARED1:@.+]] = internal unnamed_addr constant i16 1
+// SEQ-DAG: [[KERNEL_SHARED2:@.+]] = internal unnamed_addr constant i16 1
 
 // only nvptx side: do not outline teams region and do not call fork_teams
 // CK1:  define {{.*}}void @{{[^,]+}}(i{{[0-9]+}} [[ARGC:%.+]])
@@ -41,11 +43,12 @@ int main (int argc, char **argv) {
 // CK1:  store {{.+}} 0, {{.+}},
 // CK1:  store i{{[0-9]+}} [[ARGC]], i{{[0-9]+}}* [[ARGCADDR]],
 // CK1-64:  [[CONV:%.+]] = bitcast i{{[0-9]+}}* [[ARGCADDR]] to i{{[0-9]+}}*
-// CK1: [[IS_SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED1]],
-// CK1: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE1]],
-// CK1: call void @__kmpc_get_team_static_memory(i16 0, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[IS_SHARED]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
-// CK1: [[KERNEL_RD:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
-// CK1: [[GLOBALSTACK:%.+]] = getelementptr inbounds i8, i8* [[KERNEL_RD]], i{{64|32}} 0
+// SEQ: [[IS_SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED1]],
+// SEQ: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE1]],
+// SEQ: call void @__kmpc_get_team_static_memory(i16 0, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[IS_SHARED]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
+// SEQ: [[KERNEL_RD:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
+// SEQ: [[GLOBALSTACK:%.+]] = getelementptr inbounds i8, i8* [[KERNEL_RD]], i{{64|32}} 0
+// PAR: [[GLOBALSTACK:%.+]] = call i8* @__kmpc_data_sharing_push_stack(i{{32|64}} 4, i16 1)
 // CK1-64:  [[ARG:%.+]] = load i{{[0-9]+}}, i{{[0-9]+}}* [[CONV]]
 // CK1-32:  [[ARG:%.+]] = load i{{[0-9]+}}, i{{[0-9]+}}* [[ARGCADDR]]
 // CK1:  [[ARGCADDR:%.+]] = getelementptr inbounds %struct.{{.*}}, %struct.{{.*}}* %{{.*}}, i{{[0-9]+}} 0, i{{[0-9]+}} 0
@@ -62,11 +65,12 @@ int main (int argc, char **argv) {
 // CK1: define {{.*}}void @{{[^,]+}}(i{{.+}}** [[ARGC:%.+]])
 // CK1: [[ARGCADDR:%.+]] = alloca i{{.+}}**,
 // CK1: store i{{.+}}** [[ARGC]], i{{.+}}*** [[ARGCADDR]]
-// CK1: [[IS_SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED2]],
-// CK1: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE2]],
-// CK1: call void @__kmpc_get_team_static_memory(i16 0, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[IS_SHARED]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
-// CK1: [[KERNEL_RD:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
-// CK1: [[GLOBALSTACK:%.+]] = getelementptr inbounds i8, i8* [[KERNEL_RD]], i{{64|32}} 0
+// SEQ: [[IS_SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED2]],
+// SEQ: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE2]],
+// SEQ: call void @__kmpc_get_team_static_memory(i16 0, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[IS_SHARED]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
+// SEQ: [[KERNEL_RD:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
+// SEQ: [[GLOBALSTACK:%.+]] = getelementptr inbounds i8, i8* [[KERNEL_RD]], i{{64|32}} 0
+// PAR: [[GLOBALSTACK:%.+]] = call i8* @__kmpc_data_sharing_push_stack(i{{32|64}} {{4|8}}, i16 1)
 // CK1: [[ARG:%.+]] = load i{{[0-9]+}}**, i{{[0-9]+}}*** [[ARGCADDR]]
 // CK1: [[ARGCADDR:%.+]] = getelementptr inbounds %struct.{{.*}}, %struct.{{.*}}* %{{.*}}, i{{[0-9]+}} 0, i{{[0-9]+}} 0
 // CK1: store i{{[0-9]+}}** [[ARG]], i{{[0-9]+}}*** [[ARGCADDR]],
@@ -83,9 +87,11 @@ int main (int argc, char **argv) {
 
 // Test target codegen - host bc file has to be created first.
 // RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64 --check-prefix SEQ2
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CK2 --check-prefix CK2-64 --check-prefix PAR2
 // RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32 --check-prefix SEQ2
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CK2 --check-prefix CK2-32 --check-prefix PAR2
 // expected-no-diagnostics
 #ifdef CK2
 
@@ -112,13 +118,13 @@ int main (int argc, char **argv) {
   return tmain(argv);
 }
 
-// CK2: [[MEM_TY:%.+]] = type { [128 x i8] }
-// CK2-DAG: [[SHARED_GLOBAL_RD:@.+]] = common addrspace(3) global [[MEM_TY]] zeroinitializer
-// CK2-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
-// CK2-DAG: [[KERNEL_SIZE1:@.+]] = internal unnamed_addr constant i{{64|32}} 4
-// CK2-DAG: [[KERNEL_SIZE2:@.+]] = internal unnamed_addr constant i{{64|32}} {{8|4}}
-// CK2-DAG: [[KERNEL_SHARED1:@.+]] = internal unnamed_addr constant i16 1
-// CK2-DAG: [[KERNEL_SHARED2:@.+]] = internal unnamed_addr constant i16 1
+// SEQ2: [[MEM_TY:%.+]] = type { [128 x i8] }
+// SEQ2-DAG: [[SHARED_GLOBAL_RD:@.+]] = common addrspace(3) global [[MEM_TY]] zeroinitializer
+// SEQ2-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
+// SEQ2-DAG: [[KERNEL_SIZE1:@.+]] = internal unnamed_addr constant i{{64|32}} 4
+// SEQ2-DAG: [[KERNEL_SIZE2:@.+]] = internal unnamed_addr constant i{{64|32}} {{8|4}}
+// SEQ2-DAG: [[KERNEL_SHARED1:@.+]] = internal unnamed_addr constant i16 1
+// SEQ2-DAG: [[KERNEL_SHARED2:@.+]] = internal unnamed_addr constant i16 1
 
 // CK2: define {{.*}}void @{{[^,]+}}(i{{[0-9]+}} [[A_IN:%.+]], i{{[0-9]+}} [[B_IN:%.+]], i{{[0-9]+}} [[ARGC_IN:.+]])
 // CK2: [[AADDR:%.+]] = alloca i{{[0-9]+}},
@@ -130,11 +136,12 @@ int main (int argc, char **argv) {
 // CK2-64: [[ACONV:%.+]] = bitcast i64* [[AADDR]] to i32*
 // CK2-64: [[BCONV:%.+]] = bitcast i64* [[BADDR]] to i32*
 // CK2-64: [[CONV:%.+]] = bitcast i64* [[ARGCADDR]] to i32*
-// CK2: [[IS_SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED1]],
-// CK2: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE1]],
-// CK2: call void @__kmpc_get_team_static_memory(i16 0, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[IS_SHARED]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
-// CK2: [[KERNEL_RD:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
-// CK2: [[GLOBALSTACK:%.+]] = getelementptr inbounds i8, i8* [[KERNEL_RD]], i{{64|32}} 0
+// SEQ2: [[IS_SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED1]],
+// SEQ2: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE1]],
+// SEQ2: call void @__kmpc_get_team_static_memory(i16 0, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[IS_SHARED]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
+// SEQ2: [[KERNEL_RD:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
+// SEQ2: [[GLOBALSTACK:%.+]] = getelementptr inbounds i8, i8* [[KERNEL_RD]], i{{64|32}} 0
+// PAR2: [[GLOBALSTACK:%.+]] = call i8* @__kmpc_data_sharing_push_stack(i{{32|64}} 4, i16 1)
 // CK2-64:  [[ARG:%.+]] = load i{{[0-9]+}}, i{{[0-9]+}}* [[CONV]]
 // CK2-32:  [[ARG:%.+]] = load i{{[0-9]+}}, i{{[0-9]+}}* [[ARGCADDR]]
 // CK2:  [[ARGCADDR:%.+]] = getelementptr inbounds %struct.{{.*}}, %struct.{{.*}}* %{{.*}}, i{{[0-9]+}} 0, i{{[0-9]+}} 0
@@ -155,11 +162,12 @@ int main (int argc, char **argv) {
 // CK2: store i{{[0-9]+}} [[A_IN]], i{{[0-9]+}}* [[AADDR]],
 // CK2: store i{{[0-9]+}} [[B_IN]], i{{[0-9]+}}* [[BADDR]],
 // CK2: store i{{[0-9]+}}** [[ARGC]], i{{[0-9]+}}*** [[ARGCADDR]],
-// CK2: [[IS_SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED2]],
-// CK2: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE2]],
-// CK2: call void @__kmpc_get_team_static_memory(i16 0, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[IS_SHARED]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
-// CK2: [[KERNEL_RD:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
-// CK2: [[GLOBALSTACK:%.+]] = getelementptr inbounds i8, i8* [[KERNEL_RD]], i{{64|32}} 0
+// SEQ2: [[IS_SHARED:%.+]] = load i16, i16* [[KERNEL_SHARED2]],
+// SEQ2: [[SIZE:%.+]] = load i{{64|32}}, i{{64|32}}* [[KERNEL_SIZE2]],
+// SEQ2: call void @__kmpc_get_team_static_memory(i16 0, i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([[MEM_TY]], [[MEM_TY]] addrspace(3)* [[SHARED_GLOBAL_RD]], i32 0, i32 0, i32 0) to i8*), i{{64|32}} [[SIZE]], i16 [[IS_SHARED]], i8** addrspacecast (i8* addrspace(3)* [[KERNEL_PTR]] to i8**))
+// SEQ2: [[KERNEL_RD:%.+]] = load i8*, i8* addrspace(3)* [[KERNEL_PTR]],
+// SEQ2: [[GLOBALSTACK:%.+]] = getelementptr inbounds i8, i8* [[KERNEL_RD]], i{{64|32}} 0
+// PAR2: [[GLOBALSTACK:%.+]] = call i8* @__kmpc_data_sharing_push_stack(i{{32|64}} {{4|8}}, i16 1)
 // CK2: [[ARG:%.+]] = load i{{[0-9]+}}**, i{{[0-9]+}}*** [[ARGCADDR]]
 // CK2: [[ARGCADDR:%.+]] = getelementptr inbounds %struct.{{.*}}, %struct.{{.*}}* %{{.*}}, i{{[0-9]+}} 0, i{{[0-9]+}} 0
 // CK2: store i{{[0-9]+}}** [[ARG]], i{{[0-9]+}}*** [[ARGCADDR]],
diff --git a/clang/test/OpenMP/nvptx_teams_reduction_codegen.cpp b/clang/test/OpenMP/nvptx_teams_reduction_codegen.cpp
index 465b0e8de6d5e..5aa3e1978b106 100644
--- a/clang/test/OpenMP/nvptx_teams_reduction_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_teams_reduction_codegen.cpp
@@ -1,9 +1,12 @@
 // Test target codegen - host bc file has to be created first.
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix  PAR
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
-// RUN: %clang_cc1 -verify -fopenmp -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -fopenmp-cuda-teams-reduction-recs-num=2048 -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -fopenmp-cuda-teams-reduction-recs-num=2048 -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix SEQ
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix PAR
+// RUN: %clang_cc1 -verify -fopenmp -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -fopenmp-cuda-teams-reduction-recs-num=2048 -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -fopenmp-cuda-parallel-target-regions | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix PAR
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
@@ -12,21 +15,21 @@
 // CHECK-DAG: [[TEAM2_REDUCE_TY:%.+]] = type { [{{1024|2048}} x i8], [{{1024|2048}} x float] }
 // CHECK-DAG: [[TEAM3_REDUCE_TY:%.+]] = type { [{{1024|2048}} x i32], [{{1024|2048}} x i16] }
 // CHECK-DAG: [[TEAMS_REDUCE_UNION_TY:%.+]] = type { [[TEAM1_REDUCE_TY]] }
-// CHECK-DAG: [[MAP_TY:%.+]] = type { [128 x i8] }
+// SEQ-DAG: [[MAP_TY:%.+]] = type { [128 x i8] }
 
-// CHECK-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
-// CHECK-DAG: [[KERNEL_SHARED1:@.+]] = internal unnamed_addr constant i16 1
-// CHECK-DAG: [[KERNEL_SHARED2:@.+]] = internal unnamed_addr constant i16 1
-// CHECK-DAG: [[KERNEL_SIZE1:@.+]] = internal unnamed_addr constant i{{64|32}} {{16|8}}
-// CHECK-DAG: [[KERNEL_SIZE2:@.+]] = internal unnamed_addr constant i{{64|32}} 16
+// SEQ-DAG: [[KERNEL_PTR:@.+]] = internal addrspace(3) global i8* null
+// SEQ-DAG: [[KERNEL_SHARED1:@.+]] = internal unnamed_addr constant i16 1
+// SEQ-DAG: [[KERNEL_SHARED2:@.+]] = internal unnamed_addr constant i16 1
+// SEQ-DAG: [[KERNEL_SIZE1:@.+]] = internal unnamed_addr constant i{{64|32}} {{16|8}}
+// SEQ-DAG: [[KERNEL_SIZE2:@.+]] = internal unnamed_addr constant i{{64|32}} 16
 
 // Check for the data transfer medium in shared memory to transfer the reduction list to the first warp.
 // CHECK-DAG: [[TRANSFER_STORAGE:@.+]] = common addrspace([[SHARED_ADDRSPACE:[0-9]+]]) global [32 x i32]
 
 // Check that the execution mode of 2 target regions is set to Non-SPMD and the 3rd is in SPMD.
-// CHECK-DAG: {{@__omp_offloading_.+l41}}_exec_mode = weak constant i8 1
-// CHECK-DAG: {{@__omp_offloading_.+l47}}_exec_mode = weak constant i8 1
-// CHECK-DAG: {{@__omp_offloading_.+l54}}_exec_mode = weak constant i8 0
+// CHECK-DAG: {{@__omp_offloading_.+l44}}_exec_mode = weak constant i8 1
+// CHECK-DAG: {{@__omp_offloading_.+l50}}_exec_mode = weak constant i8 1
+// CHECK-DAG: {{@__omp_offloading_.+l57}}_exec_mode = weak constant i8 0
 
 // CHECK-DAG: [[TEAMS_RED_BUFFER:@.+]] = internal global [[TEAMS_REDUCE_UNION_TY]] zeroinitializer
 
@@ -70,9 +73,9 @@ int bar(int n){
   return a;
 }
 
-  // CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l41}}_worker()
+  // CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l44}}_worker()
 
-  // CHECK: define {{.*}}void [[T1:@__omp_offloading_.+template.+l41]](
+  // CHECK: define {{.*}}void [[T1:@__omp_offloading_.+template.+l44]](
   //
   // CHECK: {{call|invoke}} void [[T1]]_worker()
   //
@@ -337,9 +340,9 @@ int bar(int n){
   // CHECK: call void [[REDUCTION_FUNC]](i8* [[RL_BC]], i8* [[LOCAL_RL_BC]])
   // CHECK: ret void
 
-  // CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l47}}_worker()
+  // CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l50}}_worker()
 
-  // CHECK: define {{.*}}void [[T2:@__omp_offloading_.+template.+l47]](
+  // CHECK: define {{.*}}void [[T2:@__omp_offloading_.+template.+l50]](
   //
   // CHECK: {{call|invoke}} void [[T2]]_worker()
 
@@ -704,13 +707,13 @@ int bar(int n){
   // CHECK: call void [[REDUCTION_FUNC]](i8* [[RL_BC]], i8* [[LOCAL_RL_BC]])
   // CHECK: ret void
 
-  // CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l54}}(
+  // CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l57}}(
   //
   // CHECK: call void @__kmpc_spmd_kernel_init(
   // CHECK: call void @__kmpc_data_sharing_init_stack_spmd()
   // CHECK: call void @__kmpc_spmd_kernel_deinit_v2(i16 1)
 
-  // CHECK-NOT: call void @__kmpc_get_team_static_memory
+  // CHECK-NOT: call void @{{__kmpc_get_team_static_memory|__kmpc_data_sharing_push_stack}}
   // CHECK: store i32 0,
   // CHECK: store i32 0,
   // CHECK: store i32 0, i32* [[A_ADDR:%.+]], align
diff --git a/clang/test/OpenMP/ordered_codegen.cpp b/clang/test/OpenMP/ordered_codegen.cpp
index 7797dad0d4846..07ecee45974c5 100644
--- a/clang/test/OpenMP/ordered_codegen.cpp
+++ b/clang/test/OpenMP/ordered_codegen.cpp
@@ -1,7 +1,11 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefixes=CHECK,OMP50
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
 
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -fopenmp-version=45 -o - | FileCheck %s --check-prefixes=CHECK,OMP45
+// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -fopenmp-version=45 -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+
 // RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
@@ -65,7 +69,8 @@ void static_not_chunked(float *a, float *b, float *c, float *d) {
 void dynamic1(float *a, float *b, float *c, float *d) {
 // CHECK: [[GTID:%.+]] = call i32 @__kmpc_global_thread_num([[IDENT_T_TY]]* [[DEFAULT_LOC:[@%].+]])
   #pragma omp for schedule(dynamic) ordered
-// CHECK: call void @__kmpc_dispatch_init_8u([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID]], i32 67, i64 0, i64 16908287, i64 1, i64 1)
+// OMP45: call void @__kmpc_dispatch_init_8u([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID]], i32 67, i64 0, i64 16908287, i64 1, i64 1)
+// OMP50: call void @__kmpc_dispatch_init_8u([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID]], i32 1073741891, i64 0, i64 16908287, i64 1, i64 1)
 //
 // CHECK: [[HASWORK:%.+]] = call i32 @__kmpc_dispatch_next_8u([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID]], i32* [[OMP_ISLAST:%[^,]+]], i64* [[OMP_LB:%[^,]+]], i64* [[OMP_UB:%[^,]+]], i64* [[OMP_ST:%[^,]+]])
 // CHECK-NEXT: [[O_CMP:%.+]] = icmp ne i32 [[HASWORK]], 0
@@ -119,7 +124,8 @@ void test_auto(float *a, float *b, float *c, float *d) {
   unsigned int y = 0;
 // CHECK: [[GTID:%.+]] = call i32 @__kmpc_global_thread_num([[IDENT_T_TY]]* [[DEFAULT_LOC:[@%].+]])
   #pragma omp for schedule(auto) collapse(2) ordered
-// CHECK: call void @__kmpc_dispatch_init_8([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID]], i32 70, i64 0, i64 [[LAST_ITER:%[^,]+]], i64 1, i64 1)
+// OMP45: call void @__kmpc_dispatch_init_8([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID]], i32 70, i64 0, i64 [[LAST_ITER:%[^,]+]], i64 1, i64 1)
+// OMP50: call void @__kmpc_dispatch_init_8([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID]], i32 1073741894, i64 0, i64 [[LAST_ITER:%[^,]+]], i64 1, i64 1)
 //
 // CHECK: [[HASWORK:%.+]] = call i32 @__kmpc_dispatch_next_8([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID]], i32* [[OMP_ISLAST:%[^,]+]], i64* [[OMP_LB:%[^,]+]], i64* [[OMP_UB:%[^,]+]], i64* [[OMP_ST:%[^,]+]])
 // CHECK-NEXT: [[O_CMP:%.+]] = icmp ne i32 [[HASWORK]], 0
@@ -172,7 +178,8 @@ void runtime(float *a, float *b, float *c, float *d) {
   int x = 0;
 // CHECK: [[GTID:%.+]] = call i32 @__kmpc_global_thread_num([[IDENT_T_TY]]* [[DEFAULT_LOC:[@%].+]])
   #pragma omp for collapse(2) schedule(runtime) ordered
-// CHECK: call void @__kmpc_dispatch_init_4([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID]], i32 69, i32 0, i32 199, i32 1, i32 1)
+// OMP45: call void @__kmpc_dispatch_init_4([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID]], i32 69, i32 0, i32 199, i32 1, i32 1)
+// OMP50: call void @__kmpc_dispatch_init_4([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID]], i32 1073741893, i32 0, i32 199, i32 1, i32 1)
 //
 // CHECK: [[HASWORK:%.+]] = call i32 @__kmpc_dispatch_next_4([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID]], i32* [[OMP_ISLAST:%[^,]+]], i32* [[OMP_LB:%[^,]+]], i32* [[OMP_UB:%[^,]+]], i32* [[OMP_ST:%[^,]+]])
 // CHECK-NEXT: [[O_CMP:%.+]] = icmp ne i32 [[HASWORK]], 0
diff --git a/clang/test/OpenMP/parallel_codegen.cpp b/clang/test/OpenMP/parallel_codegen.cpp
index d903fdd1542a3..6fd394c0bbc9b 100644
--- a/clang/test/OpenMP/parallel_codegen.cpp
+++ b/clang/test/OpenMP/parallel_codegen.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefixes=ALL-DEBUG,CHECK-DEBUG %s
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-enable-irbuilder -DIRBUILDER -x c++ -emit-llvm %s -triple %itanium_abi_triple -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefixes=ALL,IRBUILDER
 // RUN: %clang_cc1 -fopenmp -fopenmp-enable-irbuilder -DIRBUILDER -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-enable-irbuilder -DIRBUILDER -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefixes=ALL-DEBUG,IRBUILDER-DEBUG %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-enable-irbuilder -DIRBUILDER -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -gno-column-info -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefixes=ALL-DEBUG,IRBUILDER-DEBUG %s
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -emit-llvm %s -triple %itanium_abi_triple -fexceptions -fcxx-exceptions -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
diff --git a/clang/test/OpenMP/parallel_default_messages.cpp b/clang/test/OpenMP/parallel_default_messages.cpp
index 6b8ad67051850..b098c43852a85 100644
--- a/clang/test/OpenMP/parallel_default_messages.cpp
+++ b/clang/test/OpenMP/parallel_default_messages.cpp
@@ -4,18 +4,25 @@
 // RUN: %clang_cc1 -verify=expected,ge40 -fopenmp-version=40 -fopenmp -ferror-limit 100 -o - %s -Wuninitialized
 // RUN: %clang_cc1 -verify -fopenmp-version=31 -fopenmp -ferror-limit 100 -o - %s -Wuninitialized
 // RUN: %clang_cc1 -verify -fopenmp-version=30 -fopenmp -ferror-limit 100 -o - %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge40 -fopenmp-version=51 -fopenmp -DOMP51 -ferror-limit 100 -o - %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge40 -fopenmp-version=51 -fopenmp-simd -DOMP51 -ferror-limit 100 -o - %s -Wuninitialized
 
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
   const int c = 0;
 
   #pragma omp parallel default // expected-error {{expected '(' after 'default'}}
-  #pragma omp parallel default ( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
-  #pragma omp parallel default () // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
-  #pragma omp parallel default (none // expected-error {{expected ')'}} expected-note {{to match this '('}}
-  #pragma omp parallel default (shared), default(shared) // expected-error {{directive '#pragma omp parallel' cannot contain more than one 'default' clause}}
-  #pragma omp parallel default (x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp parallel default(  // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp parallel default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
+#pragma omp parallel default(none                     // expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp parallel default(shared), default(shared) // expected-error {{directive '#pragma omp parallel' cannot contain more than one 'default' clause}}
+#pragma omp parallel default(x)                       // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   foo();
 
   #pragma omp parallel default(none) // expected-note {{explicit data sharing attribute requested here}}
@@ -27,5 +34,14 @@ int main(int argc, char **argv) {
 
   #pragma omp parallel default(none) // ge40-note {{explicit data sharing attribute requested here}}
   (void)c; // ge40-error {{variable 'c' must have explicitly specified data sharing attributes}}
+
+#ifdef OMP51
+#pragma omp parallel default(firstprivate) // expected-note 2 {{explicit data sharing attribute requested here}}
+  {
+    ++x; // expected-error {{variable 'x' must have explicitly specified data sharing attributes}}
+    ++y; // expected-error {{variable 'y' must have explicitly specified data sharing attributes}}
+  }
+#endif
+
   return 0;
 }
diff --git a/clang/test/OpenMP/parallel_for_codegen.cpp b/clang/test/OpenMP/parallel_for_codegen.cpp
index c45f3e911fc66..de445634470bb 100644
--- a/clang/test/OpenMP/parallel_for_codegen.cpp
+++ b/clang/test/OpenMP/parallel_for_codegen.cpp
@@ -1,7 +1,13 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefixes=OMP50,CHECK
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefixes=OMP45,CHECK
+
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp -fexceptions -fcxx-exceptions -debug-info-kind=line-tables-only -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=TERM_DEBUG
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefixes=OMP50,CHECK
+
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefixes=OMP45,CHECK
+
+// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp -fexceptions -fcxx-exceptions -debug-info-kind=line-tables-only -gno-column-info -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=TERM_DEBUG
 // RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -O1 -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=CLEANUP
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck --check-prefix SIMD-ONLY0 %s
@@ -194,7 +200,8 @@ void dynamic1(float *a, float *b, float *c, float *d) {
 // CHECK: call void ([[IDENT_T_TY]]*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call([[IDENT_T_TY]]* [[DEFAULT_LOC:[@%].+]], i32 4, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, float**, float**, float**, float**)* [[OMP_PARALLEL_FUNC:@.+]] to void (i32*, i32*, ...)*),
 // CHECK: define internal void [[OMP_PARALLEL_FUNC]](i32* noalias [[GTID_PARAM_ADDR:%.+]], i32* noalias %{{.+}}, float** nonnull align 8 dereferenceable(8) %{{.+}}, float** nonnull align 8 dereferenceable(8) %{{.+}}, float** nonnull align 8 dereferenceable(8) %{{.+}}, float** nonnull align 8 dereferenceable(8) %{{.+}})
 // CHECK: store i32* [[GTID_PARAM_ADDR]], i32** [[GTID_REF_ADDR:%.+]],
-// CHECK: call void @__kmpc_dispatch_init_8u([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID:%.+]], i32 35, i64 0, i64 16908287, i64 1, i64 1)
+// OMP45: call void @__kmpc_dispatch_init_8u([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID:%.+]], i32 35, i64 0, i64 16908287, i64 1, i64 1)
+// OMP50: call void @__kmpc_dispatch_init_8u([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID:%.+]], i32 1073741859, i64 0, i64 16908287, i64 1, i64 1)
 //
 // CHECK: [[HASWORK:%.+]] = call i32 @__kmpc_dispatch_next_8u([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID]], i32* [[OMP_ISLAST:%[^,]+]], i64* [[OMP_LB:%[^,]+]], i64* [[OMP_UB:%[^,]+]], i64* [[OMP_ST:%[^,]+]])
 // CHECK-NEXT: [[O_CMP:%.+]] = icmp ne i32 [[HASWORK]], 0
@@ -237,7 +244,8 @@ void guided7(float *a, float *b, float *c, float *d) {
 // CHECK: call void ([[IDENT_T_TY]]*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call([[IDENT_T_TY]]* [[DEFAULT_LOC:[@%].+]], i32 4, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, float**, float**, float**, float**)* [[OMP_PARALLEL_FUNC:@.+]] to void (i32*, i32*, ...)*),
 // CHECK: define internal void [[OMP_PARALLEL_FUNC]](i32* noalias [[GTID_PARAM_ADDR:%.+]], i32* noalias %{{.+}}, float** nonnull align 8 dereferenceable(8) %{{.+}}, float** nonnull align 8 dereferenceable(8) %{{.+}}, float** nonnull align 8 dereferenceable(8) %{{.+}}, float** nonnull align 8 dereferenceable(8) %{{.+}})
 // CHECK: store i32* [[GTID_PARAM_ADDR]], i32** [[GTID_REF_ADDR:%.+]],
-// CHECK: call void @__kmpc_dispatch_init_8u([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID:%.+]], i32 36, i64 0, i64 16908287, i64 1, i64 7)
+// OMP45: call void @__kmpc_dispatch_init_8u([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID:%.+]], i32 36, i64 0, i64 16908287, i64 1, i64 7)
+// OMP50: call void @__kmpc_dispatch_init_8u([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID:%.+]], i32 1073741860, i64 0, i64 16908287, i64 1, i64 7)
 //
 // CHECK: [[HASWORK:%.+]] = call i32 @__kmpc_dispatch_next_8u([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID]], i32* [[OMP_ISLAST:%[^,]+]], i64* [[OMP_LB:%[^,]+]], i64* [[OMP_UB:%[^,]+]], i64* [[OMP_ST:%[^,]+]])
 // CHECK-NEXT: [[O_CMP:%.+]] = icmp ne i32 [[HASWORK]], 0
@@ -282,7 +290,8 @@ void test_auto(float *a, float *b, float *c, float *d) {
 // CHECK: call void ([[IDENT_T_TY]]*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call([[IDENT_T_TY]]* [[DEFAULT_LOC:[@%].+]], i32 5, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i32*, float**, float**, float**, float**)* [[OMP_PARALLEL_FUNC:@.+]] to void (i32*, i32*, ...)*),
 // CHECK: define internal void [[OMP_PARALLEL_FUNC]](i32* noalias [[GTID_PARAM_ADDR:%.+]], i32* noalias %{{.+}}, i32* nonnull align 4 dereferenceable(4) %{{.+}}, float** nonnull align 8 dereferenceable(8) %{{.+}}, float** nonnull align 8 dereferenceable(8) %{{.+}}, float** nonnull align 8 dereferenceable(8) %{{.+}}, float** nonnull align 8 dereferenceable(8) %{{.+}})
 // CHECK: store i32* [[GTID_PARAM_ADDR]], i32** [[GTID_REF_ADDR:%.+]],
-// CHECK: call void @__kmpc_dispatch_init_8([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID:%.+]], i32 38, i64 0, i64 [[LAST_ITER:%[^,]+]], i64 1, i64 1)
+// OMP45: call void @__kmpc_dispatch_init_8([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID:%.+]], i32 38, i64 0, i64 [[LAST_ITER:%[^,]+]], i64 1, i64 1)
+// OMP50: call void @__kmpc_dispatch_init_8([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID:%.+]], i32 1073741862, i64 0, i64 [[LAST_ITER:%[^,]+]], i64 1, i64 1)
 //
 // CHECK: [[HASWORK:%.+]] = call i32 @__kmpc_dispatch_next_8([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID:%.+]], i32* [[OMP_ISLAST:%[^,]+]], i64* [[OMP_LB:%[^,]+]], i64* [[OMP_UB:%[^,]+]], i64* [[OMP_ST:%[^,]+]])
 // CHECK-NEXT: [[O_CMP:%.+]] = icmp ne i32 [[HASWORK]], 0
@@ -326,7 +335,8 @@ void runtime(float *a, float *b, float *c, float *d) {
 // CHECK: call void ([[IDENT_T_TY]]*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call([[IDENT_T_TY]]* [[DEFAULT_LOC:[@%].+]], i32 4, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, float**, float**, float**, float**)* [[OMP_PARALLEL_FUNC:@.+]] to void (i32*, i32*, ...)*),
 // CHECK: define internal void [[OMP_PARALLEL_FUNC]](i32* noalias [[GTID_PARAM_ADDR:%.+]], i32* noalias %{{.+}}, float** nonnull align 8 dereferenceable(8) %{{.+}}, float** nonnull align 8 dereferenceable(8) %{{.+}}, float** nonnull align 8 dereferenceable(8) %{{.+}}, float** nonnull align 8 dereferenceable(8) %{{.+}})
 // CHECK: store i32* [[GTID_PARAM_ADDR]], i32** [[GTID_REF_ADDR:%.+]],
-// CHECK: call void @__kmpc_dispatch_init_4([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID:%.+]], i32 37, i32 0, i32 199, i32 1, i32 1)
+// OMP45: call void @__kmpc_dispatch_init_4([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID:%.+]], i32 37, i32 0, i32 199, i32 1, i32 1)
+// OMP50: call void @__kmpc_dispatch_init_4([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID:%.+]], i32 1073741861, i32 0, i32 199, i32 1, i32 1)
 //
 // CHECK: [[HASWORK:%.+]] = call i32 @__kmpc_dispatch_next_4([[IDENT_T_TY]]* [[DEFAULT_LOC]], i32 [[GTID]], i32* [[OMP_ISLAST:%[^,]+]], i32* [[OMP_LB:%[^,]+]], i32* [[OMP_UB:%[^,]+]], i32* [[OMP_ST:%[^,]+]])
 // CHECK-NEXT: [[O_CMP:%.+]] = icmp ne i32 [[HASWORK]], 0
diff --git a/clang/test/OpenMP/parallel_for_default_messages.cpp b/clang/test/OpenMP/parallel_for_default_messages.cpp
index b02fa8803a3b3..c64b76948c018 100644
--- a/clang/test/OpenMP/parallel_for_default_messages.cpp
+++ b/clang/test/OpenMP/parallel_for_default_messages.cpp
@@ -2,17 +2,26 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=51 -DOMP51 -ferror-limit 100 -o - %s -Wuninitialized
+
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=51 -DOMP51 -ferror-limit 100 -o - %s -Wuninitialized
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
   int i;
 #pragma omp parallel for default // expected-error {{expected '(' after 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
-#pragma omp parallel for default( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp parallel for default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (i = 0; i < argc; ++i)
     foo();
-#pragma omp parallel for default() // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp parallel for default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
 #pragma omp parallel for default(none // expected-error {{expected ')'}} expected-note {{to match this '('}} expected-note {{explicit data sharing attribute requested here}}
@@ -21,7 +30,7 @@ int main(int argc, char **argv) {
 #pragma omp parallel for default(shared), default(shared) // expected-error {{directive '#pragma omp parallel for' cannot contain more than one 'default' clause}}
   for (i = 0; i < argc; ++i)
     foo();
-#pragma omp parallel for default(x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp parallel for default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
 
@@ -34,5 +43,13 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
     foo();
 
+#ifdef OMP51
+#pragma omp parallel for default(firstprivate) // expected-note 2 {{explicit data sharing attribute requested here}}
+  for (i = 0; i < argc; ++i) {
+    ++y; // expected-error {{variable 'y' must have explicitly specified data sharing attributes}}
+    ++x; // expected-error {{variable 'x' must have explicitly specified data sharing attributes}}
+  }
+#endif
+
   return 0;
 }
diff --git a/clang/test/OpenMP/parallel_for_loop_messages.cpp b/clang/test/OpenMP/parallel_for_loop_messages.cpp
index beaf56e7bcb6d..cbcc18419dcc1 100644
--- a/clang/test/OpenMP/parallel_for_loop_messages.cpp
+++ b/clang/test/OpenMP/parallel_for_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/parallel_for_simd_codegen.cpp b/clang/test/OpenMP/parallel_for_simd_codegen.cpp
index 01f2b4c42a243..e9cc2f302eafc 100644
--- a/clang/test/OpenMP/parallel_for_simd_codegen.cpp
+++ b/clang/test/OpenMP/parallel_for_simd_codegen.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=OMP45 --check-prefix=CHECK
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=OMP45 --check-prefix=CHECK
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
 // RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp -fexceptions -fcxx-exceptions -debug-info-kind=line-tables-only -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=TERM_DEBUG
diff --git a/clang/test/OpenMP/parallel_for_simd_default_messages.cpp b/clang/test/OpenMP/parallel_for_simd_default_messages.cpp
index 570ee14bbc84b..6368d280de5db 100644
--- a/clang/test/OpenMP/parallel_for_simd_default_messages.cpp
+++ b/clang/test/OpenMP/parallel_for_simd_default_messages.cpp
@@ -2,17 +2,26 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -o - %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp -o - %s -Wuninitialized
+
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp-simd -o - %s -Wuninitialized
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
   int i;
 #pragma omp parallel for simd default // expected-error {{expected '(' after 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
-#pragma omp parallel for simd default( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp parallel for simd default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (i = 0; i < argc; ++i)
     foo();
-#pragma omp parallel for simd default() // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp parallel for simd default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
 #pragma omp parallel for simd default(none // expected-error {{expected ')'}} expected-note {{to match this '('}} expected-note {{explicit data sharing attribute requested here}}
@@ -21,7 +30,7 @@ int main(int argc, char **argv) {
 #pragma omp parallel for simd default(shared), default(shared) // expected-error {{directive '#pragma omp parallel for simd' cannot contain more than one 'default' clause}}
   for (i = 0; i < argc; ++i)
     foo();
-#pragma omp parallel for simd default(x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp parallel for simd default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
 
@@ -34,5 +43,13 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}} expected-error {{variable 'i' must have explicitly specified data sharing attributes}}
     foo();
 
+#ifdef OMP51
+#pragma omp parallel for default(firstprivate) // expected-note 2 {{explicit data sharing attribute requested here}}
+  for (i = 0; i < argc; ++i) {
+    x++; // expected-error {{variable 'x' must have explicitly specified data sharing attributes}}
+    y++; // expected-error {{variable 'y' must have explicitly specified data sharing attributes}}
+  }
+#endif
+
   return 0;
 }
diff --git a/clang/test/OpenMP/parallel_for_simd_loop_messages.cpp b/clang/test/OpenMP/parallel_for_simd_loop_messages.cpp
index 5a0202c4e5766..cbc14ed6663a9 100644
--- a/clang/test/OpenMP/parallel_for_simd_loop_messages.cpp
+++ b/clang/test/OpenMP/parallel_for_simd_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/parallel_for_simd_scan_codegen.cpp b/clang/test/OpenMP/parallel_for_simd_scan_codegen.cpp
new file mode 100644
index 0000000000000..42fb9de348649
--- /dev/null
+++ b/clang/test/OpenMP/parallel_for_simd_scan_codegen.cpp
@@ -0,0 +1,315 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -triple x86_64-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -x c++ -triple x86_64-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -triple x86_64-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -triple x86_64-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
+// expected-no-diagnostics
+#ifndef HEADER
+#define HEADER
+
+void foo();
+void bar();
+
+// CHECK: define void @{{.*}}baz{{.*}}(i32 %n)
+void baz(int n) {
+  static float a[10];
+  static double b;
+
+  // CHECK: call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(
+  // CHECK: call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(
+
+  // CHECK: call i8* @llvm.stacksave()
+  // CHECK: [[A_BUF_SIZE:%.+]] = mul nuw i64 10, [[NUM_ELEMS:%[^,]+]]
+
+  // float a_buffer[10][n];
+  // CHECK: [[A_BUF:%.+]] = alloca float, i64 [[A_BUF_SIZE]],
+
+  // double b_buffer[10];
+  // CHECK: [[B_BUF:%.+]] = alloca double, i64 10,
+#pragma omp parallel for simd reduction(inscan, +:a[:n], b)
+  for (int i = 0; i < 10; ++i) {
+    // CHECK: call void @__kmpc_for_static_init_4(
+    // CHECK: call i8* @llvm.stacksave()
+    // CHECK: store float 0.000000e+00, float* %
+    // CHECK: store double 0.000000e+00, double* [[B_PRIV_ADDR:%.+]],
+    // CHECK: br label %[[DISPATCH:[^,]+]]
+    // CHECK: [[INPUT_PHASE:.+]]:
+    // CHECK: call void @{{.+}}foo{{.+}}()
+
+    // a_buffer[i][0..n] = a_priv[[0..n];
+    // CHECK: [[BASE_IDX_I:%.+]] = load i32, i32* [[IV_ADDR:%.+]],
+    // CHECK: [[BASE_IDX:%.+]] = zext i32 [[BASE_IDX_I]] to i64
+    // CHECK: [[IDX:%.+]] = mul nsw i64 [[BASE_IDX]], [[NUM_ELEMS]]
+    // CHECK: [[A_BUF_IDX:%.+]] = getelementptr inbounds float, float* [[A_BUF]], i64 [[IDX]]
+    // CHECK: [[A_PRIV:%.+]] = getelementptr inbounds [10 x float], [10 x float]* [[A_PRIV_ADDR:%.+]], i64 0, i64 0
+    // CHECK: [[BYTES:%.+]] = mul nuw i64 [[NUM_ELEMS:%.+]], 4
+    // CHECK: [[DEST:%.+]] = bitcast float* [[A_BUF_IDX]] to i8*
+    // CHECK: [[SRC:%.+]] = bitcast float* [[A_PRIV]] to i8*
+    // CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* {{.*}}[[DEST]], i8* {{.*}}[[SRC]], i64 [[BYTES]], i1 false)
+
+    // b_buffer[i] = b_priv;
+    // CHECK: [[B_BUF_IDX:%.+]] = getelementptr inbounds double, double* [[B_BUF]], i64 [[BASE_IDX]]
+    // CHECK: [[B_PRIV:%.+]] = load double, double* [[B_PRIV_ADDR]],
+    // CHECK: store double [[B_PRIV]], double* [[B_BUF_IDX]],
+    // CHECK: br label %[[LOOP_CONTINUE:.+]]
+
+    // CHECK: [[DISPATCH]]:
+    // CHECK: br label %[[INPUT_PHASE]]
+    // CHECK: [[LOOP_CONTINUE]]:
+    // CHECK: call void @llvm.stackrestore(i8* %
+    // CHECK: call void @__kmpc_for_static_fini(
+    // CHECK: call void @__kmpc_barrier(
+    foo();
+#pragma omp scan inclusive(a[:n], b)
+    // CHECK: [[LOG2_10:%.+]] = call double @llvm.log2.f64(double 1.000000e+01)
+    // CHECK: [[CEIL_LOG2_10:%.+]] = call double @llvm.ceil.f64(double [[LOG2_10]])
+    // CHECK: [[CEIL_LOG2_10_INT:%.+]] = fptoui double [[CEIL_LOG2_10]] to i32
+    // CHECK: br label %[[OUTER_BODY:[^,]+]]
+    // CHECK: [[OUTER_BODY]]:
+    // CHECK: [[K:%.+]] = phi i32 [ 0, %{{.+}} ], [ [[K_NEXT:%.+]], %{{.+}} ]
+    // CHECK: [[K2POW:%.+]] = phi i64 [ 1, %{{.+}} ], [ [[K2POW_NEXT:%.+]], %{{.+}} ]
+    // CHECK: [[CMP:%.+]] = icmp uge i64 9, [[K2POW]]
+    // CHECK: br i1 [[CMP]], label %[[INNER_BODY:[^,]+]], label %[[INNER_EXIT:[^,]+]]
+    // CHECK: [[INNER_BODY]]:
+    // CHECK: [[I:%.+]] = phi i64 [ 9, %[[OUTER_BODY]] ], [ [[I_PREV:%.+]], %{{.+}} ]
+
+    // a_buffer[i] += a_buffer[i-pow(2, k)];
+    // CHECK: [[IDX:%.+]] = mul nsw i64 [[I]], [[NUM_ELEMS]]
+    // CHECK: [[A_BUF_IDX:%.+]] = getelementptr inbounds float, float* [[A_BUF]], i64 [[IDX]]
+    // CHECK: [[IDX_SUB_K2POW:%.+]] = sub nuw i64 [[I]], [[K2POW]]
+    // CHECK: [[IDX:%.+]] = mul nsw i64 [[IDX_SUB_K2POW]], [[NUM_ELEMS]]
+    // CHECK: [[A_BUF_IDX_SUB_K2POW:%.+]] = getelementptr inbounds float, float* [[A_BUF]], i64 [[IDX]]
+    // CHECK: [[B_BUF_IDX:%.+]] = getelementptr inbounds double, double* [[B_BUF]], i64 [[I]]
+    // CHECK: [[IDX_SUB_K2POW:%.+]] = sub nuw i64 [[I]], [[K2POW]]
+    // CHECK: [[B_BUF_IDX_SUB_K2POW:%.+]] = getelementptr inbounds double, double* [[B_BUF]], i64 [[IDX_SUB_K2POW]]
+    // CHECK: [[A_BUF_END:%.+]] = getelementptr float, float* [[A_BUF_IDX]], i64 [[NUM_ELEMS]]
+    // CHECK: [[ISEMPTY:%.+]] = icmp eq float* [[A_BUF_IDX]], [[A_BUF_END]]
+    // CHECK: br i1 [[ISEMPTY]], label %[[RED_DONE:[^,]+]], label %[[RED_BODY:[^,]+]]
+    // CHECK: [[RED_BODY]]:
+    // CHECK: [[A_BUF_IDX_SUB_K2POW_ELEM:%.+]] = phi float* [ [[A_BUF_IDX_SUB_K2POW]], %[[INNER_BODY]] ], [ [[A_BUF_IDX_SUB_K2POW_NEXT:%.+]], %[[RED_BODY]] ]
+    // CHECK: [[A_BUF_IDX_ELEM:%.+]] = phi float* [ [[A_BUF_IDX]], %[[INNER_BODY]] ], [ [[A_BUF_IDX_NEXT:%.+]], %[[RED_BODY]] ]
+    // CHECK: [[A_BUF_IDX_VAL:%.+]] = load float, float* [[A_BUF_IDX_ELEM]],
+    // CHECK: [[A_BUF_IDX_SUB_K2POW_VAL:%.+]] = load float, float* [[A_BUF_IDX_SUB_K2POW_ELEM]],
+    // CHECK: [[RED:%.+]] = fadd float [[A_BUF_IDX_VAL]], [[A_BUF_IDX_SUB_K2POW_VAL]]
+    // CHECK: store float [[RED]], float* [[A_BUF_IDX_ELEM]],
+    // CHECK: [[A_BUF_IDX_NEXT]] = getelementptr float, float* [[A_BUF_IDX_ELEM]], i32 1
+    // CHECK: [[A_BUF_IDX_SUB_K2POW_NEXT]] = getelementptr float, float* [[A_BUF_IDX_SUB_K2POW_ELEM]], i32 1
+    // CHECK: [[DONE:%.+]] = icmp eq float* [[A_BUF_IDX_NEXT]], [[A_BUF_END]]
+    // CHECK: br i1 [[DONE]], label %[[RED_DONE]], label %[[RED_BODY]]
+    // CHECK: [[RED_DONE]]:
+
+    // b_buffer[i] += b_buffer[i-pow(2, k)];
+    // CHECK: [[B_BUF_IDX_VAL:%.+]] = load double, double* [[B_BUF_IDX]],
+    // CHECK: [[B_BUF_IDX_SUB_K2POW_VAL:%.+]] = load double, double* [[B_BUF_IDX_SUB_K2POW]],
+    // CHECK: [[RED:%.+]] = fadd double [[B_BUF_IDX_VAL]], [[B_BUF_IDX_SUB_K2POW_VAL]]
+    // CHECK: store double [[RED]], double* [[B_BUF_IDX]],
+
+    // --i;
+    // CHECK: [[I_PREV:%.+]] = sub nuw i64 [[I]], 1
+    // CHECK: [[CMP:%.+]] = icmp uge i64 [[I_PREV]], [[K2POW]]
+    // CHECK: br i1 [[CMP]], label %[[INNER_BODY]], label %[[INNER_EXIT]]
+    // CHECK: [[INNER_EXIT]]:
+
+    // ++k;
+    // CHECK: [[K_NEXT]] = add nuw i32 [[K]], 1
+    // k2pow <<= 1;
+    // CHECK: [[K2POW_NEXT]] = shl nuw i64 [[K2POW]], 1
+    // CHECK: [[CMP:%.+]] = icmp ne i32 [[K_NEXT]], [[CEIL_LOG2_10_INT]]
+    // CHECK: br i1 [[CMP]], label %[[OUTER_BODY]], label %[[OUTER_EXIT:[^,]+]]
+    // CHECK: [[OUTER_EXIT]]:
+    bar();
+    // CHECK: call void @__kmpc_for_static_init_4(
+    // CHECK: call i8* @llvm.stacksave()
+    // CHECK: store float 0.000000e+00, float* %
+    // CHECK: store double 0.000000e+00, double* [[B_PRIV_ADDR:%.+]],
+    // CHECK: br label %[[DISPATCH:[^,]+]]
+
+    // Skip the before scan body.
+    // CHECK: call void @{{.+}}foo{{.+}}()
+
+    // CHECK: [[EXIT_INSCAN:[^,]+]]:
+    // CHECK: br label %[[LOOP_CONTINUE:[^,]+]]
+
+    // CHECK: [[DISPATCH]]:
+    // a_priv[[0..n] = a_buffer[i][0..n];
+    // CHECK: [[BASE_IDX_I:%.+]] = load i32, i32* [[IV_ADDR:%.+]],
+    // CHECK: [[BASE_IDX:%.+]] = zext i32 [[BASE_IDX_I]] to i64
+    // CHECK: [[IDX:%.+]] = mul nsw i64 [[BASE_IDX]], [[NUM_ELEMS]]
+    // CHECK: [[A_BUF_IDX:%.+]] = getelementptr inbounds float, float* [[A_BUF]], i64 [[IDX]]
+    // CHECK: [[A_PRIV:%.+]] = getelementptr inbounds [10 x float], [10 x float]* [[A_PRIV_ADDR:%.+]], i64 0, i64 0
+    // CHECK: [[BYTES:%.+]] = mul nuw i64 [[NUM_ELEMS:%.+]], 4
+    // CHECK: [[DEST:%.+]] = bitcast float* [[A_PRIV]] to i8*
+    // CHECK: [[SRC:%.+]] = bitcast float* [[A_BUF_IDX]] to i8*
+    // CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* {{.*}}[[DEST]], i8* {{.*}}[[SRC]], i64 [[BYTES]], i1 false)
+
+    // b_priv = b_buffer[i];
+    // CHECK: [[B_BUF_IDX:%.+]] = getelementptr inbounds double, double* [[B_BUF]], i64 [[BASE_IDX]]
+    // CHECK: [[B_BUF_IDX_VAL:%.+]] = load double, double* [[B_BUF_IDX]],
+    // CHECK: store double [[B_BUF_IDX_VAL]], double* [[B_PRIV_ADDR]],
+    // CHECK: br label %[[SCAN_PHASE:[^,]+]]
+
+    // CHECK: [[SCAN_PHASE]]:
+    // CHECK: call void @{{.+}}bar{{.+}}()
+    // CHECK: br label %[[EXIT_INSCAN]]
+
+    // CHECK: [[LOOP_CONTINUE]]:
+    // CHECK: call void @llvm.stackrestore(i8* %
+    // CHECK: call void @__kmpc_for_static_fini(
+    // CHECK: call void @llvm.stackrestore(i8*
+  }
+
+  // CHECK: call i8* @llvm.stacksave()
+  // CHECK: [[A_BUF_SIZE:%.+]] = mul nuw i64 10, [[NUM_ELEMS:%[^,]+]]
+
+  // float a_buffer[10][n];
+  // CHECK: [[A_BUF:%.+]] = alloca float, i64 [[A_BUF_SIZE]],
+
+  // double b_buffer[10];
+  // CHECK: [[B_BUF:%.+]] = alloca double, i64 10,
+#pragma omp parallel for simd reduction(inscan, +:a[:n], b)
+  for (int i = 0; i < 10; ++i) {
+    // CHECK: call void @__kmpc_for_static_init_4(
+    // CHECK: call i8* @llvm.stacksave()
+    // CHECK: store float 0.000000e+00, float* %
+    // CHECK: store double 0.000000e+00, double* [[B_PRIV_ADDR:%.+]],
+    // CHECK: br label %[[DISPATCH:[^,]+]]
+
+    // Skip the before scan body.
+    // CHECK: call void @{{.+}}foo{{.+}}()
+
+    // CHECK: [[EXIT_INSCAN:[^,]+]]:
+
+    // a_buffer[i][0..n] = a_priv[[0..n];
+    // CHECK: [[BASE_IDX_I:%.+]] = load i32, i32* [[IV_ADDR:%.+]],
+    // CHECK: [[BASE_IDX:%.+]] = zext i32 [[BASE_IDX_I]] to i64
+    // CHECK: [[IDX:%.+]] = mul nsw i64 [[BASE_IDX]], [[NUM_ELEMS]]
+    // CHECK: [[A_BUF_IDX:%.+]] = getelementptr inbounds float, float* [[A_BUF]], i64 [[IDX]]
+    // CHECK: [[A_PRIV:%.+]] = getelementptr inbounds [10 x float], [10 x float]* [[A_PRIV_ADDR:%.+]], i64 0, i64 0
+    // CHECK: [[BYTES:%.+]] = mul nuw i64 [[NUM_ELEMS:%.+]], 4
+    // CHECK: [[DEST:%.+]] = bitcast float* [[A_BUF_IDX]] to i8*
+    // CHECK: [[SRC:%.+]] = bitcast float* [[A_PRIV]] to i8*
+    // CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* {{.*}}[[DEST]], i8* {{.*}}[[SRC]], i64 [[BYTES]], i1 false)
+
+    // b_buffer[i] = b_priv;
+    // CHECK: [[B_BUF_IDX:%.+]] = getelementptr inbounds double, double* [[B_BUF]], i64 [[BASE_IDX]]
+    // CHECK: [[B_PRIV:%.+]] = load double, double* [[B_PRIV_ADDR]],
+    // CHECK: store double [[B_PRIV]], double* [[B_BUF_IDX]],
+    // CHECK: br label %[[LOOP_CONTINUE:[^,]+]]
+
+    // CHECK: [[DISPATCH]]:
+    // CHECK: br label %[[INPUT_PHASE:[^,]+]]
+
+    // CHECK: [[INPUT_PHASE]]:
+    // CHECK: call void @{{.+}}bar{{.+}}()
+    // CHECK: br label %[[EXIT_INSCAN]]
+
+    // CHECK: [[LOOP_CONTINUE]]:
+    // CHECK: call void @llvm.stackrestore(i8* %
+    // CHECK: call void @__kmpc_for_static_fini(
+    // CHECK: call void @__kmpc_barrier(
+    foo();
+#pragma omp scan exclusive(a[:n], b)
+    // CHECK: [[LOG2_10:%.+]] = call double @llvm.log2.f64(double 1.000000e+01)
+    // CHECK: [[CEIL_LOG2_10:%.+]] = call double @llvm.ceil.f64(double [[LOG2_10]])
+    // CHECK: [[CEIL_LOG2_10_INT:%.+]] = fptoui double [[CEIL_LOG2_10]] to i32
+    // CHECK: br label %[[OUTER_BODY:[^,]+]]
+    // CHECK: [[OUTER_BODY]]:
+    // CHECK: [[K:%.+]] = phi i32 [ 0, %{{.+}} ], [ [[K_NEXT:%.+]], %{{.+}} ]
+    // CHECK: [[K2POW:%.+]] = phi i64 [ 1, %{{.+}} ], [ [[K2POW_NEXT:%.+]], %{{.+}} ]
+    // CHECK: [[CMP:%.+]] = icmp uge i64 9, [[K2POW]]
+    // CHECK: br i1 [[CMP]], label %[[INNER_BODY:[^,]+]], label %[[INNER_EXIT:[^,]+]]
+    // CHECK: [[INNER_BODY]]:
+    // CHECK: [[I:%.+]] = phi i64 [ 9, %[[OUTER_BODY]] ], [ [[I_PREV:%.+]], %{{.+}} ]
+
+    // a_buffer[i] += a_buffer[i-pow(2, k)];
+    // CHECK: [[IDX:%.+]] = mul nsw i64 [[I]], [[NUM_ELEMS]]
+    // CHECK: [[A_BUF_IDX:%.+]] = getelementptr inbounds float, float* [[A_BUF]], i64 [[IDX]]
+    // CHECK: [[IDX_SUB_K2POW:%.+]] = sub nuw i64 [[I]], [[K2POW]]
+    // CHECK: [[IDX:%.+]] = mul nsw i64 [[IDX_SUB_K2POW]], [[NUM_ELEMS]]
+    // CHECK: [[A_BUF_IDX_SUB_K2POW:%.+]] = getelementptr inbounds float, float* [[A_BUF]], i64 [[IDX]]
+    // CHECK: [[B_BUF_IDX:%.+]] = getelementptr inbounds double, double* [[B_BUF]], i64 [[I]]
+    // CHECK: [[IDX_SUB_K2POW:%.+]] = sub nuw i64 [[I]], [[K2POW]]
+    // CHECK: [[B_BUF_IDX_SUB_K2POW:%.+]] = getelementptr inbounds double, double* [[B_BUF]], i64 [[IDX_SUB_K2POW]]
+    // CHECK: [[A_BUF_END:%.+]] = getelementptr float, float* [[A_BUF_IDX]], i64 [[NUM_ELEMS]]
+    // CHECK: [[ISEMPTY:%.+]] = icmp eq float* [[A_BUF_IDX]], [[A_BUF_END]]
+    // CHECK: br i1 [[ISEMPTY]], label %[[RED_DONE:[^,]+]], label %[[RED_BODY:[^,]+]]
+    // CHECK: [[RED_BODY]]:
+    // CHECK: [[A_BUF_IDX_SUB_K2POW_ELEM:%.+]] = phi float* [ [[A_BUF_IDX_SUB_K2POW]], %[[INNER_BODY]] ], [ [[A_BUF_IDX_SUB_K2POW_NEXT:%.+]], %[[RED_BODY]] ]
+    // CHECK: [[A_BUF_IDX_ELEM:%.+]] = phi float* [ [[A_BUF_IDX]], %[[INNER_BODY]] ], [ [[A_BUF_IDX_NEXT:%.+]], %[[RED_BODY]] ]
+    // CHECK: [[A_BUF_IDX_VAL:%.+]] = load float, float* [[A_BUF_IDX_ELEM]],
+    // CHECK: [[A_BUF_IDX_SUB_K2POW_VAL:%.+]] = load float, float* [[A_BUF_IDX_SUB_K2POW_ELEM]],
+    // CHECK: [[RED:%.+]] = fadd float [[A_BUF_IDX_VAL]], [[A_BUF_IDX_SUB_K2POW_VAL]]
+    // CHECK: store float [[RED]], float* [[A_BUF_IDX_ELEM]],
+    // CHECK: [[A_BUF_IDX_NEXT]] = getelementptr float, float* [[A_BUF_IDX_ELEM]], i32 1
+    // CHECK: [[A_BUF_IDX_SUB_K2POW_NEXT]] = getelementptr float, float* [[A_BUF_IDX_SUB_K2POW_ELEM]], i32 1
+    // CHECK: [[DONE:%.+]] = icmp eq float* [[A_BUF_IDX_NEXT]], [[A_BUF_END]]
+    // CHECK: br i1 [[DONE]], label %[[RED_DONE]], label %[[RED_BODY]]
+    // CHECK: [[RED_DONE]]:
+
+    // b_buffer[i] += b_buffer[i-pow(2, k)];
+    // CHECK: [[B_BUF_IDX_VAL:%.+]] = load double, double* [[B_BUF_IDX]],
+    // CHECK: [[B_BUF_IDX_SUB_K2POW_VAL:%.+]] = load double, double* [[B_BUF_IDX_SUB_K2POW]],
+    // CHECK: [[RED:%.+]] = fadd double [[B_BUF_IDX_VAL]], [[B_BUF_IDX_SUB_K2POW_VAL]]
+    // CHECK: store double [[RED]], double* [[B_BUF_IDX]],
+
+    // --i;
+    // CHECK: [[I_PREV:%.+]] = sub nuw i64 [[I]], 1
+    // CHECK: [[CMP:%.+]] = icmp uge i64 [[I_PREV]], [[K2POW]]
+    // CHECK: br i1 [[CMP]], label %[[INNER_BODY]], label %[[INNER_EXIT]]
+    // CHECK: [[INNER_EXIT]]:
+
+    // ++k;
+    // CHECK: [[K_NEXT]] = add nuw i32 [[K]], 1
+    // k2pow <<= 1;
+    // CHECK: [[K2POW_NEXT]] = shl nuw i64 [[K2POW]], 1
+    // CHECK: [[CMP:%.+]] = icmp ne i32 [[K_NEXT]], [[CEIL_LOG2_10_INT]]
+    // CHECK: br i1 [[CMP]], label %[[OUTER_BODY]], label %[[OUTER_EXIT:[^,]+]]
+    // CHECK: [[OUTER_EXIT]]:
+    bar();
+    // CHECK: call void @__kmpc_for_static_init_4(
+    // CHECK: call i8* @llvm.stacksave()
+    // CHECK: store float 0.000000e+00, float* %
+    // CHECK: store double 0.000000e+00, double* [[B_PRIV_ADDR:%.+]],
+    // CHECK: br label %[[DISPATCH:[^,]+]]
+
+    // CHECK: [[SCAN_PHASE:.+]]:
+    // CHECK: call void @{{.+}}foo{{.+}}()
+    // CHECK: br label %[[LOOP_CONTINUE:.+]]
+
+    // CHECK: [[DISPATCH]]:
+    // if (i >0)
+    //   a_priv[[0..n] = a_buffer[i-1][0..n];
+    // CHECK: [[BASE_IDX_I:%.+]] = load i32, i32* [[IV_ADDR:%.+]],
+    // CHECK: [[BASE_IDX:%.+]] = zext i32 [[BASE_IDX_I]] to i64
+    // CHECK: [[CMP:%.+]] = icmp eq i64 [[BASE_IDX]], 0
+    // CHECK: br i1 [[CMP]], label %[[IF_DONE:[^,]+]], label %[[IF_THEN:[^,]+]]
+    // CHECK: [[IF_THEN]]:
+    // CHECK: [[BASE_IDX_SUB_1:%.+]] = sub nuw i64 [[BASE_IDX]], 1
+    // CHECK: [[IDX:%.+]] = mul nsw i64 [[BASE_IDX_SUB_1]], [[NUM_ELEMS]]
+    // CHECK: [[A_BUF_IDX:%.+]] = getelementptr inbounds float, float* [[A_BUF]], i64 [[IDX]]
+    // CHECK: [[A_PRIV:%.+]] = getelementptr inbounds [10 x float], [10 x float]* [[A_PRIV_ADDR:%.+]], i64 0, i64 0
+    // CHECK: [[BYTES:%.+]] = mul nuw i64 [[NUM_ELEMS:%.+]], 4
+    // CHECK: [[DEST:%.+]] = bitcast float* [[A_PRIV]] to i8*
+    // CHECK: [[SRC:%.+]] = bitcast float* [[A_BUF_IDX]] to i8*
+    // CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* {{.*}}[[DEST]], i8* {{.*}}[[SRC]], i64 [[BYTES]], i1 false)
+
+    // b_priv = b_buffer[i];
+    // CHECK: [[B_BUF_IDX:%.+]] = getelementptr inbounds double, double* [[B_BUF]], i64 [[BASE_IDX_SUB_1]]
+    // CHECK: [[B_BUF_IDX_VAL:%.+]] = load double, double* [[B_BUF_IDX]],
+    // CHECK: store double [[B_BUF_IDX_VAL]], double* [[B_PRIV_ADDR]],
+    // CHECK: br label %[[SCAN_PHASE]]
+
+    // CHECK: [[LOOP_CONTINUE]]:
+    // CHECK: call void @llvm.stackrestore(i8* %
+    // CHECK: call void @__kmpc_for_static_fini(
+    // CHECK: call void @llvm.stackrestore(i8*
+  }
+}
+
+// CHECK: !{!"llvm.loop.vectorize.enable", i1 true}
+
+#endif
+
diff --git a/clang/test/OpenMP/parallel_master_codegen.cpp b/clang/test/OpenMP/parallel_master_codegen.cpp
index 9ffa941314b98..82e18c80f103e 100644
--- a/clang/test/OpenMP/parallel_master_codegen.cpp
+++ b/clang/test/OpenMP/parallel_master_codegen.cpp
@@ -118,6 +118,162 @@ void parallel_master_private() {
 
 #endif
 
+#ifdef CK31
+///==========================================================================///
+// RUN: %clang_cc1 -DCK31 -fopenmp-version=51 -verify -fopenmp -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix CK31
+// RUN: %clang_cc1 -DCK31 -fopenmp-version=51 -fopenmp -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK31 -fopenmp-version=51 -fopenmp -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK31
+
+// RUN: %clang_cc1 -DCK31 -fopenmp-version=51 -verify -fopenmp-simd -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK31 -fopenmp-version=51 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK31 -fopenmp-version=51 -fopenmp-simd -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
+
+// CK31-DAG:   %struct.ident_t = type { i32, i32, i32, i32, i8* }
+// CK31-DAG:   [[STR:@.+]] = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00"
+
+void parallel_master_default_firstprivate() {
+  int a;
+#pragma omp parallel master default(firstprivate)
+  a++;
+}
+
+// CK31-LABEL: define void @{{.+}}parallel_master{{.+}}
+// CK31:       [[A_VAL:%.+]] = alloca i32{{.+}}
+// CK31:       [[A_CASTED:%.+]] = alloca i64
+// CK31:       [[ZERO_VAL:%.+]] = load i32, i32* [[A_VAL]]
+// CK31:       [[CONV:%.+]] = bitcast i64* [[A_CASTED]] to i32*
+// CK31:       store i32 [[ZERO_VAL]], i32* [[CONV]]
+// CK31:       [[ONE_VAL:%.+]] = load i64, i64* [[A_CASTED]]
+// CK31:       call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* @0, i32 1, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i64)* @.omp_outlined. to void (i32*, i32*, ...)*), i64 [[ONE_VAL]])
+// CK31:       ret void
+
+// CK31:       [[GLOBAL_TID_ADDR:%.+]] = alloca i32*
+// CK31:       [[BOUND_TID_ADDR:%.+]] = alloca i32*
+// CK31:       [[A_ADDR:%.+]] = alloca i64{{.+}}
+// CK31:       store i32* [[GLOBAL_TID:%.+]], i32** [[GLOBAL_TID_ADDR]]{{.+}}
+// CK31:       store i32* [[BOUND_TID:%.+]], i32** [[BOUND_TID_ADDR]]
+// CK31:       store i64 [[A_VAL]], i64* [[A_ADDR]]
+// CK31:       [[CONV]] = bitcast i64* [[A_ADDR]]
+// CK31:       [[ZERO_VAL]] = load i32*, i32** [[GLOBAL_TID_ADDR]]
+// CK31:       [[ONE_VAL]] = load i32, i32* [[ZERO_VAL]]
+// CK31:       [[TWO_VAL:%.+]] = call i32 @__kmpc_master(%struct.ident_t* @0, i32 [[ONE_VAL]])
+// CK31:       [[THREE:%.+]] = icmp ne i32 [[TWO_VAL]], 0
+// CK31:       br i1 %3, label [[OMP_IF_THEN:%.+]], label [[OMP_IF_END:%.+]]
+
+// CK31:       [[FOUR:%.+]] = load i32, i32* [[CONV:%.+]]
+// CK31:       [[INC:%.+]] = add nsw i32 [[FOUR]]
+// CK31:       store i32 [[INC]], i32* [[CONV]]
+// CK31:       call void @__kmpc_end_master(%struct.ident_t* @0, i32 [[ONE_VAL]])
+// CK31:       br label [[OMP_IF_END]]
+
+// CK31:       ret void
+
+#endif
+
+#ifdef CK32
+///==========================================================================///
+// RUN: %clang_cc1 -DCK32 -fopenmp-version=51 -verify -fopenmp -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix CK32
+// RUN: %clang_cc1 -DCK32 -fopenmp-version=51 -fopenmp -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK32 -fopenmp-version=51 -fopenmp -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK32
+
+// RUN: %clang_cc1 -DCK32 -fopenmp-version=51 -verify -fopenmp-simd -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK32 -fopenmp-version=51 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK32 -fopenmp-version=51 -fopenmp-simd -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
+
+// CK32-DAG:   %struct.ident_t = type { i32, i32, i32, i32, i8* }
+// CK32-DAG:   [[STR:@.+]] = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00"
+
+struct St {
+  int a, b;
+  static int y;
+  St() : a(0), b(0) {}
+  ~St() {}
+};
+int St::y = 0;
+
+void parallel_master_default_firstprivate() {
+  St a = St();
+  static int y = 0;
+#pragma omp parallel master default(firstprivate)
+  {
+    a.a += 1;
+    a.b += 1;
+    y++;
+    a.y++;
+  }
+}
+
+// CK32-LABEL: define {{.+}} @{{.+}}parallel_master_default_firstprivate{{.+}}
+// CK32: [[A_VAL:%.+]] = alloca %struct.St{{.+}}
+// CK32: [[Y_CASTED:%.+]] = alloca i64
+// CK32: call void @[[CTOR:.+]](%struct.St* [[A_VAL]])
+// CK32: [[ZERO:%.+]] = load i32, i32* @{{.+}}parallel_master_default_firstprivate{{.+}}
+// CK32: [[CONV:%.+]] = bitcast i64* [[Y_CASTED]] to i32*
+// CK32: store i32 [[ZERO]], i32* [[CONV]]
+// CK32: [[ONE:%.+]] = load i64, i64* [[Y_CASTED]]
+// CK32: call void {{.+}}@{{.+}} %struct.St* [[A_VAL]], i64 [[ONE]])
+// CK32: call void [[DTOR:@.+]](%struct.St* [[A_VAL]])
+
+// CK32: [[THIS_ADDR:%.+]] = alloca %struct.St*
+// CK32: store %struct.St* [[THIS:%.+]], %struct.St** [[THIS_ADDR]]
+// CK32: [[THIS_ONE:%.+]] = load %struct.St*, %struct.St** [[THIS_ADDR]]
+// CK32: call void [[CTOR_2:.+]](%struct.St* [[THIS_ONE]])
+// CK32: ret void
+
+// CK32: [[GLOBAL_TID_ADDR:%.+]] = alloca i32*
+// CK32: [[BOUND_TID_ADDR:%.+]] = alloca i32*
+// CK32: [[A_ADDR:%.+]] = alloca %struct.St
+// CK32: [[Y_ADDR:%.+]] = alloca i64
+// CK32: store i32* [[GLOBAL_TID:%.+]], i32** [[GLOBAL_TID_ADDR]]
+// CK32: store i32* %.bound_tid., i32** [[BOUND_TID_ADDR]]
+// CK32: store %struct.St* [[A_VAL]], %struct.St** [[A_ADDR]]{{.+}}
+// CK32: store i64 [[Y:%.+]], i64* [[Y_ADDR]]
+// CK32: [[ONE:%.+]] = load i32*, i32** [[GLOBAL_TID_ADDR]]
+// CK32: [[TWO:%.+]] = load i32, i32* [[ONE]]
+// CK32: [[THREE:%.+]] = call i32 @{{.+}} i32 [[TWO]])
+// CK32: [[FOUR:%.+]] = icmp ne i32 [[THREE]], 0
+// CK32: br i1 [[FOUR]], label [[IF_THEN:%.+]], label [[IF_END:%.+]]
+
+// CK32: [[A_1:%.+]] = getelementptr inbounds %struct.St, %struct.St* [[ZERO]], i32 0, i32 0
+// CK32: [[FIVE:%.+]] = load i32, i32* [[A_1]]
+// CK32: [[ADD:%.+]] = add nsw i32 [[FIVE]], 1
+// CK32: store i32 [[ADD]], i32* [[A_1]]
+// CK32: [[B:%.+]] = getelementptr inbounds %struct.St, %struct.St* [[ZERO]], i32 0, i32 1
+// CK32: [[SIX:%.+]] = load i32, i32* [[B]]
+// CK32: [[ADD_2:%.+]] = add nsw i32 [[SIX]], 1
+// CK32: store i32 [[ADD_2]], i32* [[B]]
+// CK32: [[SEVEN:%.+]] = load i32, i32* [[CONV]]
+// CK32: [[INC:%.+]] = add nsw i32 [[SEVEN]], 1
+// CK32: store i32 [[INC]], i32* [[CONV]]
+// CK32: [[EIGHT:%.+]] = load i32, i32* [[FUNC:@.+]]
+// CK32: [[INC_3:%.+]] = add nsw i32 [[EIGHT]], 1
+// CK32: store i32 [[INC_3]], i32* @{{.+}}
+// CK32: call void @{{.+}} i32 [[TWO]])
+// CK32: br label [[IF_END]]
+
+// CK32: [[DTOR]](%struct.St* [[THIS]])
+// CK32: [[THIS_ADDR]] = alloca %struct.St*
+// CK32: store %struct.St* [[THIS]], %struct.St** [[THIS_ADDR]]
+// CK32: [[THIS_ONE]] = load %struct.St*, %struct.St** [[THIS_ADDR]]
+// CK32: call void @_ZN2StD2Ev(%struct.St* [[THIS_ONE]])
+
+// CK32: [[THIS_ADDR]] = alloca %struct.St*
+// CK32: store %struct.St* [[THIS]], %struct.St** [[THIS_ADDR]]
+// CK32: [[THIS_ONE]] = load %struct.St*, %struct.St** [[THIS_ADDR]]
+// CK32: [[A_VAL]] = getelementptr inbounds %struct.St, %struct.St* [[THIS_ONE]], i32 0, i32 0
+// CK32: store i32 0, i32* [[A_VAL]]
+// CK32: [[B_VAL:%.+]] = getelementptr inbounds %struct.St, %struct.St* [[THIS_ONE]], i32 0, i32 1
+// CK32: store i32 0, i32* [[B_VAL]]
+// CK32: ret void
+
+// CK32: [[THIS_ADDR:%.+]] = alloca %struct.St*
+// CK32: store %struct.St* %this, %struct.St** [[THIS_ADDR]]
+// CK32: [[THIS_ONE]] = load %struct.St*, %struct.St** [[THIS_ADDR]]
+
+#endif
+
 #ifdef CK4
 ///==========================================================================///
 // RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix CK4
diff --git a/clang/test/OpenMP/parallel_master_default_messages.cpp b/clang/test/OpenMP/parallel_master_default_messages.cpp
index 557cba5aa322a..39f78ea53ae16 100644
--- a/clang/test/OpenMP/parallel_master_default_messages.cpp
+++ b/clang/test/OpenMP/parallel_master_default_messages.cpp
@@ -2,20 +2,29 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp -ferror-limit 100 -o - %s -Wuninitialized
+
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp-simd -ferror-limit 100 -o - %s -Wuninitialized
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
 #pragma omp parallel master default // expected-error {{expected '(' after 'default'}}
   {
-#pragma omp parallel master default( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp parallel master default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
     {
-#pragma omp parallel master default() // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp parallel master default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
       {
 #pragma omp parallel master default(none // expected-error {{expected ')'}} expected-note {{to match this '('}}
         {
 #pragma omp parallel master default(shared), default(shared) // expected-error {{directive '#pragma omp parallel master' cannot contain more than one 'default' clause}}
           {
-#pragma omp parallel master default(x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp parallel master default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
             {
               foo();
             }
@@ -37,5 +46,14 @@ int main(int argc, char **argv) {
       ++argc;  // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
     }
   }
+
+#ifdef OMP51
+#pragma omp parallel master default(firstprivate) // expected-note 2 {{explicit data sharing attribute requested here}}
+  {
+    ++x; // expected-error {{variable 'x' must have explicitly specified data sharing attributes}}
+    ++y; // expected-error {{variable 'y' must have explicitly specified data sharing attributes}}
+  }
+#endif
+
   return 0;
 }
diff --git a/clang/test/OpenMP/parallel_master_taskloop_loop_messages.cpp b/clang/test/OpenMP/parallel_master_taskloop_loop_messages.cpp
index 76411969924bd..2f0edc4539127 100644
--- a/clang/test/OpenMP/parallel_master_taskloop_loop_messages.cpp
+++ b/clang/test/OpenMP/parallel_master_taskloop_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/parallel_master_taskloop_simd_loop_messages.cpp b/clang/test/OpenMP/parallel_master_taskloop_simd_loop_messages.cpp
index 9f6c55e1c4c54..354844f94ec04 100644
--- a/clang/test/OpenMP/parallel_master_taskloop_simd_loop_messages.cpp
+++ b/clang/test/OpenMP/parallel_master_taskloop_simd_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/parallel_sections_default_messages.cpp b/clang/test/OpenMP/parallel_sections_default_messages.cpp
index d6a10fe56b344..cfa95445fb536 100644
--- a/clang/test/OpenMP/parallel_sections_default_messages.cpp
+++ b/clang/test/OpenMP/parallel_sections_default_messages.cpp
@@ -7,15 +7,15 @@ void foo();
 int main(int argc, char **argv) {
 #pragma omp parallel sections default // expected-error {{expected '(' after 'default'}}
   {
-#pragma omp parallel sections default( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp parallel sections default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
     {
-#pragma omp parallel sections default() // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp parallel sections default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
       {
 #pragma omp parallel sections default(none // expected-error {{expected ')'}} expected-note {{to match this '('}}
         {
 #pragma omp parallel sections default(shared), default(shared) // expected-error {{directive '#pragma omp parallel sections' cannot contain more than one 'default' clause}}
           {
-#pragma omp parallel sections default(x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp parallel sections default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
             {
               foo();
             }
diff --git a/clang/test/OpenMP/simd_loop_messages.cpp b/clang/test/OpenMP/simd_loop_messages.cpp
index d905991682829..f046ce3ec7c89 100644
--- a/clang/test/OpenMP/simd_loop_messages.cpp
+++ b/clang/test/OpenMP/simd_loop_messages.cpp
@@ -1,9 +1,9 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4,expectedw %s -Wuninitialized
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4,expectedw %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4,expectedw %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4,expectedw %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5,expectedw %s -fopenmp-version=50 -DOMP50 -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5,expectedw %s -fopenmp-version=50 -DOMP50 -Wuninitialized
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized -Wno-openmp-loop-form
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized -Wno-openmp
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized -Wno-openmp-loop-form
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized -Wno-openmp
 
 static int sii;
 // expected-note@+1 {{defined as threadprivate or thread local}}
diff --git a/clang/test/OpenMP/single_codegen.cpp b/clang/test/OpenMP/single_codegen.cpp
index ef1e32d27fed2..a56cdb0ae81a4 100644
--- a/clang/test/OpenMP/single_codegen.cpp
+++ b/clang/test/OpenMP/single_codegen.cpp
@@ -1,6 +1,12 @@
-// RUN: %clang_cc1 -verify -fopenmp -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s
-// RUN: %clang_cc1 -fopenmp -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fnoopenmp-use-tls -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefixes=OMP50,CHECK
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -fopenmp-version=45 -o - | FileCheck %s --check-prefixes=OMP45,CHECK
+
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fnoopenmp-use-tls -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefixes=OMP50,CHECK
+
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fnoopenmp-use-tls -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefixes=OMP45,CHECK
+
 // RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -std=c++11 -fopenmp -fnoopenmp-use-tls -fexceptions -fcxx-exceptions -debug-info-kind=line-tables-only -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=TERM_DEBUG
 // RUN: %clang_cc1 -verify -fopenmp -fnoopenmp-use-tls -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
@@ -149,6 +155,7 @@ int main() {
   return a;
 }
 
+// OMP50-LABEL: declare i8* @__kmpc_threadprivate_cached(
 // CHECK: void [[COPY_FUNC]](i8* %0, i8* %1)
 // CHECK: store i8* %0, i8** [[DST_ADDR_REF:%.+]],
 // CHECK: store i8* %1, i8** [[SRC_ADDR_REF:%.+]],
@@ -192,7 +199,58 @@ int main() {
 // CHECK: br i1
 // CHECK: ret void
 
-// CHECK-LABEL:      parallel_single
+
+// OMP50-LABEL: void @_ZN3SSTIdEC2Ev(
+// OMP50: getelementptr inbounds [[SST_TY]], [[SST_TY]]* %{{.+}}, i32 0, i32 0
+// OMP50-NEXT: store double 0.000000e+00, double* %
+// OMP50-NEXT: getelementptr inbounds [[SST_TY]], [[SST_TY]]* %{{.+}}, i32 0, i32 0
+// OMP50-NEXT: store double* %{{.+}}, double** %
+// OMP50-NEXT: load double*, double** %
+// OMP50-NEXT: load double, double* %
+// OMP50-NEXT: bitcast i64* %{{.+}} to double*
+// OMP50-NEXT: store double %{{.+}}, double* %
+// OMP50-NEXT: load i64, i64* %
+// OMP50-NEXT: call void ([[IDENT_T_TY]]*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call([[IDENT_T_TY]]* @{{.+}}, i32 2, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, [[SST_TY]]*, i64)* [[SST_MICROTASK:@.+]] to void
+// OMP50-NEXT: ret void
+
+// OMP50: define internal void [[SST_MICROTASK]](i32* {{[^,]+}}, i32* {{[^,]+}}, [[SST_TY]]* {{.+}}, i64 {{.+}})
+// OMP50: [[RES:%.+]] = call i32 @__kmpc_single([[IDENT_T_TY]]* @{{.+}}, i32 %{{.+}})
+// OMP50-NEXT: icmp ne i32 [[RES]], 0
+// OMP50-NEXT: br i1
+
+// OMP50: getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 1
+// OMP50-NEXT: load double*, double** %
+// OMP50-NEXT: store double* %
+// OMP50-LABEL: invoke void @_ZZN3SSTIdEC1EvENKUlvE_clEv(
+
+// OMP50: call void @__kmpc_end_single([[IDENT_T_TY]]* @{{.+}}, i32 %{{.+}})
+// OMP50-NEXT: store i32 1, i32* [[DID_IT]],
+// OMP50-NEXT: br label
+
+// OMP50: call void @__kmpc_end_single([[IDENT_T_TY]]* @{{.+}}, i32 %{{.+}})
+// OMP50-NEXT: br label
+
+// OMP50: getelementptr inbounds [1 x i8*], [1 x i8*]* [[LIST:%.+]], i64 0, i64 0
+// OMP50: load double*, double** %
+// OMP50-NEXT: bitcast double* %
+// OMP50-NEXT: store i8* %
+// OMP50-NEXT: bitcast [1 x i8*]* [[LIST]] to i8*
+// OMP50-NEXT: load i32, i32* [[DID_IT]],
+// OMP50-NEXT: call void @__kmpc_copyprivate([[IDENT_T_TY]]* @{{.+}}, i32 %{{.+}}, i64 8, i8* %{{.+}}, void (i8*, i8*)* [[COPY_FUNC:@[^,]+]], i32 %{{.+}})
+// OMP50-NEXT:  ret void
+
+// OMP50-LABEL: @_ZZN3SSTIdEC1EvENKUlvE_clEv(
+// OMP50: getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 1
+// OMP50-NEXT: getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 1
+// OMP50-NEXT: load double*, double** %
+// OMP50-NEXT: store double* %
+// OMP50-LABEL: call void @_ZZZN3SSTIdEC1EvENKUlvE_clEvENKUlvE_clEv(
+// OMP50-NEXT: ret void
+
+// OMP50: define internal void [[COPY_FUNC]](i8* %0, i8* %1)
+// OMP50: ret void
+
+// OMP45-LABEL:      parallel_single
 // TERM_DEBUG-LABEL: parallel_single
 void parallel_single() {
 #pragma omp parallel
@@ -385,54 +443,54 @@ void array_func(int n, int a[n], St s[2]) {
 // CHECK: define internal void [[COPY_FUNC]](i8* %0, i8* %1)
 // CHECK: ret void
 
-// CHECK-LABEL: @_ZN3SSTIdEC2Ev
-// CHECK: getelementptr inbounds [[SST_TY]], [[SST_TY]]* %{{.+}}, i32 0, i32 0
-// CHECK-NEXT: store double 0.000000e+00, double* %
-// CHECK-NEXT: getelementptr inbounds [[SST_TY]], [[SST_TY]]* %{{.+}}, i32 0, i32 0
-// CHECK-NEXT: store double* %{{.+}}, double** %
-// CHECK-NEXT: load double*, double** %
-// CHECK-NEXT: load double, double* %
-// CHECK-NEXT: bitcast i64* %{{.+}} to double*
-// CHECK-NEXT: store double %{{.+}}, double* %
-// CHECK-NEXT: load i64, i64* %
-// CHECK-NEXT: call void ([[IDENT_T_TY]]*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call([[IDENT_T_TY]]* @{{.+}}, i32 2, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, [[SST_TY]]*, i64)* [[SST_MICROTASK:@.+]] to void
-// CHECK-NEXT: ret void
-
-// CHECK: define internal void [[SST_MICROTASK]](i32* {{[^,]+}}, i32* {{[^,]+}}, [[SST_TY]]* {{.+}}, i64 {{.+}})
-// CHECK: [[RES:%.+]] = call i32 @__kmpc_single([[IDENT_T_TY]]* @{{.+}}, i32 %{{.+}})
-// CHECK-NEXT: icmp ne i32 [[RES]], 0
-// CHECK-NEXT: br i1
-
-// CHECK: getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 1
-// CHECK-NEXT: load double*, double** %
-// CHECK-NEXT: store double* %
-// CHECK-LABEL: invoke void @_ZZN3SSTIdEC1EvENKUlvE_clEv(
-
-// CHECK: call void @__kmpc_end_single([[IDENT_T_TY]]* @{{.+}}, i32 %{{.+}})
-// CHECK-NEXT: store i32 1, i32* [[DID_IT]],
-// CHECK-NEXT: br label
-
-// CHECK: call void @__kmpc_end_single([[IDENT_T_TY]]* @{{.+}}, i32 %{{.+}})
-// CHECK-NEXT: br label
-
-// CHECK: getelementptr inbounds [1 x i8*], [1 x i8*]* [[LIST:%.+]], i64 0, i64 0
-// CHECK: load double*, double** %
-// CHECK-NEXT: bitcast double* %
-// CHECK-NEXT: store i8* %
-// CHECK-NEXT: bitcast [1 x i8*]* [[LIST]] to i8*
-// CHECK-NEXT: load i32, i32* [[DID_IT]],
-// CHECK-NEXT: call void @__kmpc_copyprivate([[IDENT_T_TY]]* @{{.+}}, i32 %{{.+}}, i64 8, i8* %{{.+}}, void (i8*, i8*)* [[COPY_FUNC:@[^,]+]], i32 %{{.+}})
-// CHECK-NEXT:  ret void
-
-// CHECK-LABEL: @_ZZN3SSTIdEC1EvENKUlvE_clEv(
-// CHECK: getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 1
-// CHECK-NEXT: getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 1
-// CHECK-NEXT: load double*, double** %
-// CHECK-NEXT: store double* %
-// CHECK-LABEL: call void @_ZZZN3SSTIdEC1EvENKUlvE_clEvENKUlvE_clEv(
-// CHECK-NEXT: ret void
-
-// CHECK: define internal void [[COPY_FUNC]](i8* %0, i8* %1)
-// CHECK: ret void
-
-// CHECK-LABEL: @_ZZZN3SSTIdEC1EvENKUlvE_clEvENKUlvE_clEv(
+// OMP45-LABEL: void @_ZN3SSTIdEC2Ev(
+// OMP45: getelementptr inbounds [[SST_TY]], [[SST_TY]]* %{{.+}}, i32 0, i32 0
+// OMP45-NEXT: store double 0.000000e+00, double* %
+// OMP45-NEXT: getelementptr inbounds [[SST_TY]], [[SST_TY]]* %{{.+}}, i32 0, i32 0
+// OMP45-NEXT: store double* %{{.+}}, double** %
+// OMP45-NEXT: load double*, double** %
+// OMP45-NEXT: load double, double* %
+// OMP45-NEXT: bitcast i64* %{{.+}} to double*
+// OMP45-NEXT: store double %{{.+}}, double* %
+// OMP45-NEXT: load i64, i64* %
+// OMP45-NEXT: call void ([[IDENT_T_TY]]*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call([[IDENT_T_TY]]* @{{.+}}, i32 2, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, [[SST_TY]]*, i64)* [[SST_MICROTASK:@.+]] to void
+// OMP45-NEXT: ret void
+
+// OMP45: define internal void [[SST_MICROTASK]](i32* {{[^,]+}}, i32* {{[^,]+}}, [[SST_TY]]* {{.+}}, i64 {{.+}})
+// OMP45: [[RES:%.+]] = call i32 @__kmpc_single([[IDENT_T_TY]]* @{{.+}}, i32 %{{.+}})
+// OMP45-NEXT: icmp ne i32 [[RES]], 0
+// OMP45-NEXT: br i1
+
+// OMP45: getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 1
+// OMP45-NEXT: load double*, double** %
+// OMP45-NEXT: store double* %
+// OMP45-LABEL: invoke void @_ZZN3SSTIdEC1EvENKUlvE_clEv(
+
+// OMP45: call void @__kmpc_end_single([[IDENT_T_TY]]* @{{.+}}, i32 %{{.+}})
+// OMP45-NEXT: store i32 1, i32* [[DID_IT]],
+// OMP45-NEXT: br label
+
+// OMP45: call void @__kmpc_end_single([[IDENT_T_TY]]* @{{.+}}, i32 %{{.+}})
+// OMP45-NEXT: br label
+
+// OMP45: getelementptr inbounds [1 x i8*], [1 x i8*]* [[LIST:%.+]], i64 0, i64 0
+// OMP45: load double*, double** %
+// OMP45-NEXT: bitcast double* %
+// OMP45-NEXT: store i8* %
+// OMP45-NEXT: bitcast [1 x i8*]* [[LIST]] to i8*
+// OMP45-NEXT: load i32, i32* [[DID_IT]],
+// OMP45-NEXT: call void @__kmpc_copyprivate([[IDENT_T_TY]]* @{{.+}}, i32 %{{.+}}, i64 8, i8* %{{.+}}, void (i8*, i8*)* [[COPY_FUNC:@[^,]+]], i32 %{{.+}})
+// OMP45-NEXT:  ret void
+
+// OMP45-LABEL: @_ZZN3SSTIdEC1EvENKUlvE_clEv(
+// OMP45: getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 1
+// OMP45-NEXT: getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 1
+// OMP45-NEXT: load double*, double** %
+// OMP45-NEXT: store double* %
+// OMP45-LABEL: call void @_ZZZN3SSTIdEC1EvENKUlvE_clEvENKUlvE_clEv(
+// OMP45-NEXT: ret void
+
+// OMP45: define internal void [[COPY_FUNC]](i8* %0, i8* %1)
+// OMP45: ret void
+
+// OMP45-LABEL: @_ZZZN3SSTIdEC1EvENKUlvE_clEvENKUlvE_clEv(
diff --git a/clang/test/OpenMP/target_codegen.cpp b/clang/test/OpenMP/target_codegen.cpp
index 940edc34b1758..b8fd7cf34b37a 100644
--- a/clang/test/OpenMP/target_codegen.cpp
+++ b/clang/test/OpenMP/target_codegen.cpp
@@ -1,10 +1,17 @@
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP45
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP45
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP45
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP45
+
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP50
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP50
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP50
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP50
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
@@ -15,6 +22,15 @@
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // Test target codegen - host bc file has to be created first.
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
@@ -690,31 +706,31 @@ int bar(int n){
 
 // CHECK:       [[IFEND]]
 
-// CHECK: define {{.*}}@{{.*}}zee{{.*}}
-
-// CHECK:       [[LOCAL_THIS:%.+]] = alloca [[S2]]*
-// CHECK:       [[BP:%.+]] = alloca [1 x i8*]
-// CHECK:       [[P:%.+]] = alloca [1 x i8*]
-// CHECK:       [[LOCAL_THIS1:%.+]] = load [[S2]]*, [[S2]]** [[LOCAL_THIS]]
-// CHECK:       [[ARR_IDX:%.+]] = getelementptr inbounds [[S2]], [[S2]]* [[LOCAL_THIS1]], i[[SZ]] 0
-// CHECK:       [[ARR_IDX2:%.+]] = getelementptr inbounds [[S2]], [[S2]]* [[LOCAL_THIS1]], i[[SZ]] 0
-
-// CHECK-DAG:   [[BPADDR0:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* [[BP]], i32 0, i32 0
-// CHECK-DAG:   [[PADDR0:%.+]] =  getelementptr inbounds [1 x i8*], [1 x i8*]* [[P]], i32 0, i32 0
-// CHECK-DAG:   [[CBPADDR0:%.+]] = bitcast i8** [[BPADDR0]] to [[S2]]**
-// CHECK-DAG:   [[CPADDR0:%.+]] = bitcast i8** [[PADDR0]] to [[S2]]**
-// CHECK-DAG:   store [[S2]]* [[ARR_IDX]], [[S2]]** [[CBPADDR0]]
-// CHECK-DAG:   store [[S2]]* [[ARR_IDX2]], [[S2]]** [[CPADDR0]]
-
-// CHECK:       [[BPR:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* [[BP]], i32 0, i32 0
-// CHECK:       [[PR:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* [[P]], i32 0, i32 0
-// CHECK:       [[RET:%.+]] = call i32 @__tgt_target(i64 -1, i8* @{{[^,]+}}, i32 1, i8** [[BPR:%[^,]+]], i8** [[PR:%[^,]+]], i64* getelementptr inbounds ([1 x i64], [1 x i64]* [[SIZET9]], i32 0, i32 0), i64* getelementptr inbounds ([1 x i64], [1 x i64]* [[MAPT10]], i32 0, i32 0))
-// CHECK-NEXT:  [[ERROR:%.+]] = icmp ne i32 [[RET]], 0
-// CHECK-NEXT:  br i1 [[ERROR]], label %[[FAIL:[^,]+]], label %[[END:[^,]+]]
-// CHECK:       [[FAIL]]
-// CHECK:       call void [[HVT0:@.+]]([[S2]]* [[LOCAL_THIS1]])
-// CHECK-NEXT:  br label %[[END]]
-// CHECK:       [[END]]
+// OMP45: define {{.*}}@{{.*}}zee{{.*}}
+
+// OMP45:       [[LOCAL_THIS:%.+]] = alloca [[S2]]*
+// OMP45:       [[BP:%.+]] = alloca [1 x i8*]
+// OMP45:       [[P:%.+]] = alloca [1 x i8*]
+// OMP45:       [[LOCAL_THIS1:%.+]] = load [[S2]]*, [[S2]]** [[LOCAL_THIS]]
+// OMP45:       [[ARR_IDX:%.+]] = getelementptr inbounds [[S2]], [[S2]]* [[LOCAL_THIS1]], i[[SZ]] 0
+// OMP45:       [[ARR_IDX2:%.+]] = getelementptr inbounds [[S2]], [[S2]]* [[LOCAL_THIS1]], i[[SZ]] 0
+
+// OMP45-DAG:   [[BPADDR0:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* [[BP]], i32 0, i32 0
+// OMP45-DAG:   [[PADDR0:%.+]] =  getelementptr inbounds [1 x i8*], [1 x i8*]* [[P]], i32 0, i32 0
+// OMP45-DAG:   [[CBPADDR0:%.+]] = bitcast i8** [[BPADDR0]] to [[S2]]**
+// OMP45-DAG:   [[CPADDR0:%.+]] = bitcast i8** [[PADDR0]] to [[S2]]**
+// OMP45-DAG:   store [[S2]]* [[ARR_IDX]], [[S2]]** [[CBPADDR0]]
+// OMP45-DAG:   store [[S2]]* [[ARR_IDX2]], [[S2]]** [[CPADDR0]]
+
+// OMP45:       [[BPR:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* [[BP]], i32 0, i32 0
+// OMP45:       [[PR:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* [[P]], i32 0, i32 0
+// OMP45:       [[RET:%.+]] = call i32 @__tgt_target(i64 -1, i8* @{{[^,]+}}, i32 1, i8** [[BPR:%[^,]+]], i8** [[PR:%[^,]+]], i64* getelementptr inbounds ([1 x i64], [1 x i64]* [[SIZET9]], i32 0, i32 0), i64* getelementptr inbounds ([1 x i64], [1 x i64]* [[MAPT10]], i32 0, i32 0))
+// OMP45-NEXT:  [[ERROR:%.+]] = icmp ne i32 [[RET]], 0
+// OMP45-NEXT:  br i1 [[ERROR]], label %[[FAIL:[^,]+]], label %[[END:[^,]+]]
+// OMP45:       [[FAIL]]
+// OMP45:       call void [[HVT0:@.+]]([[S2]]* [[LOCAL_THIS1]])
+// OMP45-NEXT:  br label %[[END]]
+// OMP45:       [[END]]
 
 // Check that the offloading functions are emitted and that the arguments are
 // correct and loaded correctly for the target regions of the callees of bar().
@@ -784,6 +800,33 @@ int bar(int n){
 // CHECK-DAG:   load i16, i16* [[REF_AA]]
 // CHECK-DAG:   getelementptr inbounds [10 x i32], [10 x i32]* [[REF_B]], i[[SZ]] 0, i[[SZ]] 2
 
+
+// OMP50: define {{.*}}@{{.*}}zee{{.*}}
+
+// OMP50:       [[LOCAL_THIS:%.+]] = alloca [[S2]]*
+// OMP50:       [[BP:%.+]] = alloca [1 x i8*]
+// OMP50:       [[P:%.+]] = alloca [1 x i8*]
+// OMP50:       [[LOCAL_THIS1:%.+]] = load [[S2]]*, [[S2]]** [[LOCAL_THIS]]
+// OMP50:       [[ARR_IDX:%.+]] = getelementptr inbounds [[S2]], [[S2]]* [[LOCAL_THIS1]], i[[SZ]] 0
+// OMP50:       [[ARR_IDX2:%.+]] = getelementptr inbounds [[S2]], [[S2]]* [[LOCAL_THIS1]], i[[SZ]] 0
+
+// OMP50-DAG:   [[BPADDR0:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* [[BP]], i32 0, i32 0
+// OMP50-DAG:   [[PADDR0:%.+]] =  getelementptr inbounds [1 x i8*], [1 x i8*]* [[P]], i32 0, i32 0
+// OMP50-DAG:   [[CBPADDR0:%.+]] = bitcast i8** [[BPADDR0]] to [[S2]]**
+// OMP50-DAG:   [[CPADDR0:%.+]] = bitcast i8** [[PADDR0]] to [[S2]]**
+// OMP50-DAG:   store [[S2]]* [[ARR_IDX]], [[S2]]** [[CBPADDR0]]
+// OMP50-DAG:   store [[S2]]* [[ARR_IDX2]], [[S2]]** [[CPADDR0]]
+
+// OMP50:       [[BPR:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* [[BP]], i32 0, i32 0
+// OMP50:       [[PR:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* [[P]], i32 0, i32 0
+// OMP50:       [[RET:%.+]] = call i32 @__tgt_target(i64 -1, i8* @{{[^,]+}}, i32 1, i8** [[BPR:%[^,]+]], i8** [[PR:%[^,]+]], i64* getelementptr inbounds ([1 x i64], [1 x i64]* [[SIZET9]], i32 0, i32 0), i64* getelementptr inbounds ([1 x i64], [1 x i64]* [[MAPT10]], i32 0, i32 0))
+// OMP50-NEXT:  [[ERROR:%.+]] = icmp ne i32 [[RET]], 0
+// OMP50-NEXT:  br i1 [[ERROR]], label %[[FAIL:[^,]+]], label %[[END:[^,]+]]
+// OMP50:       [[FAIL]]
+// OMP50:       call void [[HVT0:@.+]]([[S2]]* [[LOCAL_THIS1]])
+// OMP50-NEXT:  br label %[[END]]
+// OMP50:       [[END]]
+ 
 void bar () {
 #define pragma_target _Pragma("omp target")
 pragma_target
diff --git a/clang/test/OpenMP/target_data_messages.c b/clang/test/OpenMP/target_data_messages.c
index 7a7fc0012af2b..bc164c3f1796d 100644
--- a/clang/test/OpenMP/target_data_messages.c
+++ b/clang/test/OpenMP/target_data_messages.c
@@ -45,5 +45,12 @@ int main(int argc, char **argv) {
   {
     foo();
   }
+
+  const int b = 5;
+  int marr[10][10], iarr[5];
+#pragma omp target data map(to: marr[10][0:2:2]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+  {}
+#pragma omp target data map(alloc: iarr[:2:b]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+  {}
   return 0;
 }
diff --git a/clang/test/OpenMP/target_defaultmap_codegen.cpp b/clang/test/OpenMP/target_defaultmap_codegen.cpp
index 140574c18c746..3deff63273d53 100644
--- a/clang/test/OpenMP/target_defaultmap_codegen.cpp
+++ b/clang/test/OpenMP/target_defaultmap_codegen.cpp
@@ -20,7 +20,7 @@
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
 // SIMD-ONLY10-NOT: {{__kmpc|__tgt}}
 
-// CK1-LABEL: @.__omp_offloading_{{.*}}implicit_maps_double_complex{{.*}}_l44.region_id = weak constant i8 0
+// CK1-LABEL: @.__omp_offloading_{{.*}}implicit_maps_double_complex{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK1-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 16]
 // Map types: OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 544
@@ -70,7 +70,7 @@ void implicit_maps_double_complex (int a){
 // SIMD-ONLY10-NOT: {{__kmpc|__tgt}}
 #ifdef CK2
 
-// CK2-LABEL: @.__omp_offloading_{{.*}}implicit_maps_double_complex{{.*}}_l94.region_id = weak constant i8 0
+// CK2-LABEL: @.__omp_offloading_{{.*}}implicit_maps_double_complex{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK2-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 16]
 // Map types: OMP_MAP_TO  | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 545
@@ -120,7 +120,7 @@ void implicit_maps_double_complex (int a){
 // SIMD-ONLY10-NOT: {{__kmpc|__tgt}}
 #ifdef CK3
 
-// CK3-LABEL: @.__omp_offloading_{{.*}}implicit_maps_double_complex{{.*}}_l144.region_id = weak constant i8 0
+// CK3-LABEL: @.__omp_offloading_{{.*}}implicit_maps_double_complex{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK3-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 16]
 // Map types: OMP_MAP_FROM | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 546
@@ -173,7 +173,7 @@ void implicit_maps_double_complex (int a){
 // For a 32-bit targets, the value doesn't fit the size of the pointer,
 // therefore it is passed by reference with a map 'to' specification.
 
-// CK4-LABEL: @.__omp_offloading_{{.*}}implicit_maps_double{{.*}}_l209.region_id = weak constant i8 0
+// CK4-LABEL: @.__omp_offloading_{{.*}}implicit_maps_double{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK4-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 8]
 // Map types: OMP_MAP_PRIVATE_VAL | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 800
@@ -242,7 +242,7 @@ void implicit_maps_double (int a){
 // SIMD-ONLY8-NOT: {{__kmpc|__tgt}}
 #ifdef CK5
 
-// CK5-LABEL: @.__omp_offloading_{{.*}}implicit_maps_array{{.*}}_l266.region_id = weak constant i8 0
+// CK5-LABEL: @.__omp_offloading_{{.*}}implicit_maps_array{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK5-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 16]
 // Map types: OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 544
@@ -293,7 +293,7 @@ void implicit_maps_array (int a){
 // SIMD-ONLY8-NOT: {{__kmpc|__tgt}}
 #ifdef CK6
 
-// CK6-LABEL: @.__omp_offloading_{{.*}}implicit_maps_array{{.*}}_l317.region_id = weak constant i8 0
+// CK6-LABEL: @.__omp_offloading_{{.*}}implicit_maps_array{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK6-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 16]
 // Map types: OMP_MAP_TO | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 545
@@ -344,7 +344,7 @@ void implicit_maps_array (int a){
 // SIMD-ONLY8-NOT: {{__kmpc|__tgt}}
 #ifdef CK7
 
-// CK7-LABEL: @.__omp_offloading_{{.*}}implicit_maps_array{{.*}}_l368.region_id = weak constant i8 0
+// CK7-LABEL: @.__omp_offloading_{{.*}}implicit_maps_array{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK7-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 16]
 // Map types: OMP_MAP_FROM | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 546
@@ -395,7 +395,7 @@ void implicit_maps_array (int a){
 // SIMD-ONLY8-NOT: {{__kmpc|__tgt}}
 #ifdef CK8
 
-// CK8-LABEL: @.__omp_offloading_{{.*}}implicit_maps_array{{.*}}_l419.region_id = weak constant i8 0
+// CK8-LABEL: @.__omp_offloading_{{.*}}implicit_maps_array{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK8-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 16]
 // Map types: OMP_MAP_TO | OMP_MAP_FROM | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 547
@@ -498,7 +498,7 @@ void zero_size_section_and_private_maps (int ii){
 #ifdef CK10
 
 
-// CK10-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l523.region_id = weak constant i8 0
+// CK10-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK10: [[SIZE:@.+]] = private {{.*}}constant [1 x i64] [i64 {{8|4}}]
 // Map types: OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 544
@@ -546,7 +546,7 @@ void explicit_maps_single (){
 #ifdef CK11
 
 
-// CK11-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l571.region_id = weak constant i8 0
+// CK11-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK11: [[SIZE09:@.+]] = private {{.*}}constant [1 x i64] [i64 {{8|4}}]
 // Map types: OMP_MAP_TO | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 545
@@ -594,7 +594,7 @@ void explicit_maps_single (){
 #ifdef CK12
 
 
-// CK12-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l619.region_id = weak constant i8 0
+// CK12-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK12: [[SIZE09:@.+]] = private {{.*}}constant [1 x i64] [i64 {{8|4}}]
 // Map types: OMP_MAP_FROM | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 546
@@ -642,7 +642,7 @@ void explicit_maps_single (){
 #ifdef CK13
 
 
-// CK13-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l667.region_id = weak constant i8 0
+// CK13-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK13: [[SIZE09:@.+]] = private {{.*}}constant [1 x i64] [i64 {{8|4}}]
 // Map types: OMP_MAP_TO | OMP_MAP_FROM | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 547
@@ -690,7 +690,7 @@ void explicit_maps_single (){
 #ifdef CK14
 
 
-// CK14-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l715.region_id = weak constant i8 0
+// CK14-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK14: [[SIZE09:@.+]] = private {{.*}}constant [1 x i64] zeroinitializer
 // Map types: OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 544
@@ -737,7 +737,7 @@ void explicit_maps_single (){
 // SIMD-ONLY12-NOT: {{__kmpc|__tgt}}
 #ifdef CK15
 
-// CK15-LABEL: @.__omp_offloading_{{.*}}implicit_maps_variable_length_array{{.*}}_l787.region_id = weak constant i8 0
+// CK15-LABEL: @.__omp_offloading_{{.*}}implicit_maps_variable_length_array{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // We don't have a constant map size for VLAs.
 // Map types:
@@ -820,7 +820,7 @@ void implicit_maps_variable_length_array (int a){
 #ifdef CK16
 
 // CK16-DAG: [[ST:%.+]] = type { i32, double }
-// CK16-LABEL: @.__omp_offloading_{{.*}}implicit_maps_struct{{.*}}_l849.region_id = weak constant i8 0
+// CK16-LABEL: @.__omp_offloading_{{.*}}implicit_maps_struct{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK16-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 {{16|12}}]
 // Map types: OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 544
 // CK16-DAG: [[TYPES:@.+]] = {{.+}}constant [1 x i64] [i64 544]
@@ -877,7 +877,7 @@ void implicit_maps_struct (int a){
 #ifdef CK17
 
 // CK17-DAG: [[ST:%.+]] = type { i32, double }
-// CK17-LABEL: @.__omp_offloading_{{.*}}implicit_maps_struct{{.*}}_l906.region_id = weak constant i8 0
+// CK17-LABEL: @.__omp_offloading_{{.*}}implicit_maps_struct{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK17-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 {{16|12}}]
 // Map types: OMP_MAP_TO | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 545
 // CK17-DAG: [[TYPES:@.+]] = {{.+}}constant [1 x i64] [i64 545]
@@ -934,7 +934,7 @@ void implicit_maps_struct (int a){
 #ifdef CK18
 
 // CK18-DAG: [[ST:%.+]] = type { i32, double }
-// CK18-LABEL: @.__omp_offloading_{{.*}}implicit_maps_struct{{.*}}_l963.region_id = weak constant i8 0
+// CK18-LABEL: @.__omp_offloading_{{.*}}implicit_maps_struct{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK18-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 {{16|12}}]
 // Map types: OMP_MAP_FROM | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 546
 // CK18-DAG: [[TYPES:@.+]] = {{.+}}constant [1 x i64] [i64 546]
@@ -991,7 +991,7 @@ void implicit_maps_struct (int a){
 #ifdef CK19
 
 // CK19-DAG: [[ST:%.+]] = type { i32, double }
-// CK19-LABEL: @.__omp_offloading_{{.*}}implicit_maps_struct{{.*}}_l1020.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}implicit_maps_struct{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 {{16|12}}]
 // Map types: OMP_MAP_TO | OMP_MAP_FROM | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 547
 // CK19-DAG: [[TYPES:@.+]] = {{.+}}constant [1 x i64] [i64 547]
@@ -1050,7 +1050,7 @@ void implicit_maps_struct (int a){
 // For a 32-bit targets, the value doesn't fit the size of the pointer,
 // therefore it is passed by reference with a map 'to' specification.
 
-// CK20-LABEL: @.__omp_offloading_{{.*}}implicit_maps_double{{.*}}_l1086.region_id = weak constant i8 0
+// CK20-LABEL: @.__omp_offloading_{{.*}}implicit_maps_double{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK20-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 8]
 // Map types: OMP_MAP_LITERAL | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 800
@@ -1120,7 +1120,7 @@ void implicit_maps_double (int a){
 #ifdef CK21
 
 // CK21-DAG: [[ST:%.+]] = type { i32, double }
-// CK21-LABEL: @.__omp_offloading_{{.*}}implicit_maps_struct{{.*}}_l1149.region_id = weak constant i8 0
+// CK21-LABEL: @.__omp_offloading_{{.*}}implicit_maps_struct{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK21-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 {{16|12}}]
 // Map types: OMP_MAP_TO + OMP_MAP_FROM + OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 547
 // CK21-DAG: [[TYPES:@.+]] = {{.+}}constant [1 x i64] [i64 547]
@@ -1176,7 +1176,7 @@ void implicit_maps_struct (int a){
 // SIMD-ONLY9-NOT: {{__kmpc|__tgt}}
 #ifdef CK22
 
-// CK22-LABEL: @.__omp_offloading_{{.*}}implicit_maps_pointer{{.*}}_l1200.region_id = weak constant i8 0
+// CK22-LABEL: @.__omp_offloading_{{.*}}implicit_maps_pointer{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK22-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] zeroinitializer
 // Map types: OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 544
@@ -1413,11 +1413,11 @@ void bar(float *&a, int *&b) {
 // SIMD-ONLY18-NOT: {{__kmpc|__tgt}}
 #ifdef CK24
 
-// CK24-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1443.region_id = weak constant i8 0
+// CK24-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK24: [[SIZE00:@.+]] = private {{.*}}constant [1 x i[[Z:64|32]]] [i[[Z:64|32]] 4]
 // CK24: [[MTYPE00:@.+]] = private {{.*}}constant [1 x i64] [i64 1059]
 
-// CK24-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1462.region_id = weak constant i8 0
+// CK24-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK24: [[SIZE01:@.+]] = private {{.*}}constant [1 x i[[Z:64|32]]] [i[[Z:64|32]] 4]
 // CK24: [[MTYPE01:@.+]] = private {{.*}}constant [1 x i64] [i64 1063]
 
@@ -1488,7 +1488,7 @@ void explicit_maps_single (int ii){
 extern int x;
 #pragma omp declare target to(x)
 
-// CK25-LABEL: @.__omp_offloading_{{.*}}declare_target_to{{.*}}_l1499.region_id = weak constant i8 0
+// CK25-LABEL: @.__omp_offloading_{{.*}}declare_target_to{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 void declare_target_to()
 {
diff --git a/clang/test/OpenMP/target_depend_messages.cpp b/clang/test/OpenMP/target_depend_messages.cpp
index 3ff6e6860ff83..9e5f28c04f9e4 100644
--- a/clang/test/OpenMP/target_depend_messages.cpp
+++ b/clang/test/OpenMP/target_depend_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
 void xxx(int argc) {
   int x; // expected-note {{initialize the variable 'x' to silence this warning}}
@@ -30,27 +32,27 @@ int main(int argc, char **argv, char *env[]) {
 
   #pragma omp target depend // expected-error {{expected '(' after 'depend'}}
   foo();
-  #pragma omp target depend ( // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target depend( // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target depend () // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target depend() // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target depend (argc // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target depend(argc // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target depend (source : argc) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target depend(source : argc) // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target depend (source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
   #pragma omp target depend (in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target' are ignored}}
   foo();
   #pragma omp target depend (out: ) // expected-error {{expected expression}}
   foo();
-  #pragma omp target depend (inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
+#pragma omp target depend(inout : foobool(argc)), depend(in, argc) // omp4-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target depend (out :S1) // expected-error {{'S1' does not refer to a value}}
   foo();
   #pragma omp target depend(in : argv[1][1] = '2')
   foo();
-  #pragma omp target depend (in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
+#pragma omp target depend(in : vec[1]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target depend (in : argv[0])
   foo();
@@ -58,7 +60,7 @@ int main(int argc, char **argv, char *env[]) {
   foo();
   #pragma omp target depend (in : main)
   foo();
-  #pragma omp target depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
+#pragma omp target depend(in : a[0]) // omp4-error{{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target depend (in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
   foo();
@@ -78,7 +80,7 @@ int main(int argc, char **argv, char *env[]) {
   foo();
   #pragma omp target depend (in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   foo();
-  #pragma omp target depend (in : argv[3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+  #pragma omp target depend(in : argv [3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
   foo();
   #pragma omp target depend(in:a[0:1]) // expected-error {{subscripted value is not an array or pointer}}
   foo();
diff --git a/clang/test/OpenMP/target_enter_data_depend_messages.cpp b/clang/test/OpenMP/target_enter_data_depend_messages.cpp
index 8a4d96f60a23f..4b28067b8eb33 100644
--- a/clang/test/OpenMP/target_enter_data_depend_messages.cpp
+++ b/clang/test/OpenMP/target_enter_data_depend_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
 void foo() {
 }
@@ -26,27 +28,27 @@ int tmain(T argc, S **argv, R *env[]) {
   int i;
   #pragma omp target enter data map(to: i) depend // expected-error {{expected '(' after 'depend'}}
   foo();
-  #pragma omp target enter data map(to: i) depend ( // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target enter data map(to : i) depend( // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target enter data map(to: i) depend () // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target enter data map(to : i) depend() // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target enter data map(to: i) depend (argc // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target enter data map(to : i) depend(argc // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target enter data map(to: i) depend (source : argc) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target enter data map(to : i) depend(source : argc) // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target enter data map(to: i) depend (source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target enter data map(to : i) depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
   #pragma omp target enter data map(to: i) depend (in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target enter data' are ignored}}
   foo();
-  #pragma omp target enter data map(to: i) depend (out: ) // expected-error {{expected expression}}
+#pragma omp target enter data map(to : i) depend(out:) // expected-error {{expected expression}}
   foo();
-  #pragma omp target enter data map(to: i) depend (inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
+#pragma omp target enter data map(to : i) depend(inout : foobool(argc)), depend(in, argc) // omp4-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target enter data map(to: i) depend (out :S1) // expected-error {{'S1' does not refer to a value}}
   foo();
-  #pragma omp target enter data map(to: i) depend(in : argv[1][1] = '2') // expected-error {{expected addressable lvalue expression, array element or array section}}
+#pragma omp target enter data map(to : i) depend(in : argv[1][1] = '2') // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
-  #pragma omp target enter data map(to: i) depend (in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
+#pragma omp target enter data map(to : i) depend(in : vec[1]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target enter data map(to: i) depend (in : argv[0])
   foo();
@@ -54,7 +56,7 @@ int tmain(T argc, S **argv, R *env[]) {
   foo();
   #pragma omp target enter data map(to: i) depend (in : tmain)
   foo();
-  #pragma omp target enter data map(to: i) depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
+#pragma omp target enter data map(to : i) depend(in : a[0]) // omp4-error{{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target enter data map(to: i) depend (in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
   foo();
@@ -70,11 +72,11 @@ int tmain(T argc, S **argv, R *env[]) {
   foo();
   #pragma omp target enter data map(to: i) depend (in : argv[0:-1]) // expected-error {{section length is evaluated to a negative value -1}}
   foo();
-  #pragma omp target enter data map(to: i) depend (in : argv[-1:0]) // expected-error {{zero-length array section is not allowed in 'depend' clause}}
+#pragma omp target enter data map(to : i) depend(in : argv [-1:0]) // expected-error {{zero-length array section is not allowed in 'depend' clause}}
   foo();
   #pragma omp target enter data map(to: i) depend (in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   foo();
-  #pragma omp target enter data map(to: i) depend (in : argv[3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+#pragma omp target enter data map(to : i) depend(in : argv [3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
   foo();
   #pragma omp target enter data map(to: i) depend(in:a[0:1]) // expected-error {{subscripted value is not an array or pointer}}
   foo();
@@ -86,7 +88,7 @@ int tmain(T argc, S **argv, R *env[]) {
   foo();
   #pragma omp target enter data map(to: i) depend(in : argv[ : argc][1 : argc - 1])
   foo();
-  #pragma omp target enter data map(to: i) depend(in : arr[0])
+#pragma omp target enter data map(to : i) depend(in : arr[0])
   foo();
 
   return 0;
@@ -101,27 +103,27 @@ int main(int argc, char **argv, char *env[]) {
   int i;
   #pragma omp target enter data map(to: i) depend // expected-error {{expected '(' after 'depend'}}
   foo();
-  #pragma omp target enter data map(to: i) depend ( // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target enter data map(to : i) depend( // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target enter data map(to: i) depend () // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target enter data map(to : i) depend() // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target enter data map(to: i) depend (argc // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target enter data map(to : i) depend(argc // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target enter data map(to: i) depend (source : argc) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target enter data map(to : i) depend(source : argc) // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target enter data map(to: i) depend (source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target enter data map(to : i) depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
   #pragma omp target enter data map(to: i) depend (in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target enter data' are ignored}}
   foo();
   #pragma omp target enter data map(to: i) depend (out: ) // expected-error {{expected expression}}
   foo();
-  #pragma omp target enter data map(to: i) depend (inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
+#pragma omp target enter data map(to : i) depend(inout : foobool(argc)), depend(in, argc) // omp4-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target enter data map(to: i) depend (out :S1) // expected-error {{'S1' does not refer to a value}}
   foo();
   #pragma omp target enter data map(to: i) depend(in : argv[1][1] = '2')
   foo();
-  #pragma omp target enter data map(to: i) depend (in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
+#pragma omp target enter data map(to : i) depend(in : vec[1]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target enter data map(to: i) depend (in : argv[0])
   foo();
@@ -129,7 +131,7 @@ int main(int argc, char **argv, char *env[]) {
   foo();
   #pragma omp target enter data map(to: i) depend (in : main)
   foo();
-  #pragma omp target enter data map(to: i) depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
+#pragma omp target enter data map(to : i) depend(in : a[0]) // omp4-error{{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target enter data map(to: i) depend (in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
   foo();
@@ -147,9 +149,9 @@ int main(int argc, char **argv, char *env[]) {
   foo();
   #pragma omp target enter data map(to: i) depend (in : argv[-1:0]) // expected-error {{zero-length array section is not allowed in 'depend' clause}}
   foo();
-  #pragma omp target enter data map(to: i) depend (in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
+  #pragma omp target enter data map(to: i) depend(in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   foo();
-  #pragma omp target enter data map(to: i) depend (in : argv[3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+  #pragma omp target enter data map(to : i) depend(in : argv [3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
   foo();
   #pragma omp target enter data map(to: i) depend(in:a[0:1]) // expected-error {{subscripted value is not an array or pointer}}
   foo();
diff --git a/clang/test/OpenMP/target_exit_data_depend_messages.cpp b/clang/test/OpenMP/target_exit_data_depend_messages.cpp
index ae26139d239fc..a1bcdb0f92065 100644
--- a/clang/test/OpenMP/target_exit_data_depend_messages.cpp
+++ b/clang/test/OpenMP/target_exit_data_depend_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
 void foo() {
 }
@@ -26,27 +28,27 @@ int tmain(T argc, S **argv, R *env[]) {
   int i;
   #pragma omp target exit data map(from: i) depend // expected-error {{expected '(' after 'depend'}}
   foo();
-  #pragma omp target exit data map(from: i) depend ( // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target exit data map(from : i) depend( // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target exit data map(from: i) depend () // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target exit data map(from : i) depend() // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target exit data map(from: i) depend (argc // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target exit data map(from : i) depend(argc // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target exit data map(from: i) depend (source : argc) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target exit data map(from : i) depend(source : argc) // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target exit data map(from: i) depend (source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target exit data map(from : i) depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
   #pragma omp target exit data map(from: i) depend (in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target exit data' are ignored}}
   foo();
-  #pragma omp target exit data map(from: i) depend (out: ) // expected-error {{expected expression}}
+#pragma omp target exit data map(from : i) depend(out:) // expected-error {{expected expression}}
   foo();
-  #pragma omp target exit data map(from: i) depend (inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
+#pragma omp target exit data map(from : foobool(argc)), depend(in, argc) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}} omp5-error {{expected addressable lvalue in 'map' clause}}
   foo();
   #pragma omp target exit data map(from: i) depend (out :S1) // expected-error {{'S1' does not refer to a value}}
   foo();
-  #pragma omp target exit data map(from: i) depend(in : argv[1][1] = '2') // expected-error {{expected addressable lvalue expression, array element or array section}}
+#pragma omp target exit data map(from : i) depend(in : argv[1][1] = '2') // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
-  #pragma omp target exit data map(from: i) depend (in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
+#pragma omp target exit data map(from : i) depend(in : vec[1]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target exit data map(from: i) depend (in : argv[0])
   foo();
@@ -54,7 +56,7 @@ int tmain(T argc, S **argv, R *env[]) {
   foo();
   #pragma omp target exit data map(from: i) depend (in : tmain)
   foo();
-  #pragma omp target exit data map(from: i) depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
+#pragma omp target exit data map(from : i) depend(in : a[0]) // omp4-error{{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target exit data map(from: i) depend (in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
   foo();
@@ -70,11 +72,11 @@ int tmain(T argc, S **argv, R *env[]) {
   foo();
   #pragma omp target exit data map(from: i) depend (in : argv[0:-1]) // expected-error {{section length is evaluated to a negative value -1}}
   foo();
-  #pragma omp target exit data map(from: i) depend (in : argv[-1:0]) // expected-error {{zero-length array section is not allowed in 'depend' clause}}
+#pragma omp target exit data map(from : i) depend(in : argv [-1:0]) // expected-error {{zero-length array section is not allowed in 'depend' clause}}
   foo();
   #pragma omp target exit data map(from: i) depend (in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   foo();
-  #pragma omp target exit data map(from: i) depend (in : argv[3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+  #pragma omp target exit data map(from : i) depend(in : argv [3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
   foo();
   #pragma omp target exit data map(from: i) depend(in:a[0:1]) // expected-error {{subscripted value is not an array or pointer}}
   foo();
@@ -86,7 +88,7 @@ int tmain(T argc, S **argv, R *env[]) {
   foo();
   #pragma omp target exit data map(from: i) depend(in : argv[ : argc][1 : argc - 1])
   foo();
-  #pragma omp target exit data map(from: i) depend(in : arr[0])
+#pragma omp target exit data map(from : i) depend(in : arr[0])
   foo();
 
   return 0;
@@ -101,27 +103,27 @@ int main(int argc, char **argv, char *env[]) {
   int i;
   #pragma omp target exit data map(from: i) depend // expected-error {{expected '(' after 'depend'}}
   foo();
-  #pragma omp target exit data map(from: i) depend ( // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target exit data map(from : i) depend( // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target exit data map(from: i) depend () // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target exit data map(from : i) depend() // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target exit data map(from: i) depend (argc // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target exit data map(from : i) depend(argc // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target exit data map(from: i) depend (source : argc) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target exit data map(from : i) depend(source : argc) // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target exit data map(from: i) depend (source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target exit data map(from : i) depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
   #pragma omp target exit data map(from: i) depend (in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target exit data' are ignored}}
   foo();
   #pragma omp target exit data map(from: i) depend (out: ) // expected-error {{expected expression}}
   foo();
-  #pragma omp target exit data map(from: i) depend (inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
+#pragma omp target exit data map(from  : i) depend(inout : foobool(argc)), depend(in, argc) // omp4-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target exit data map(from: i) depend (out :S1) // expected-error {{'S1' does not refer to a value}}
   foo();
   #pragma omp target exit data map(from: i) depend(in : argv[1][1] = '2')
   foo();
-  #pragma omp target exit data map(from: i) depend (in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
+#pragma omp target exit data map(from : i) depend(in : vec[1]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target exit data map(from: i) depend (in : argv[0])
   foo();
@@ -129,7 +131,7 @@ int main(int argc, char **argv, char *env[]) {
   foo();
   #pragma omp target exit data map(from: i) depend (in : main)
   foo();
-  #pragma omp target exit data map(from: i) depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
+#pragma omp target exit data map(from : i) depend(in : a[0]) // omp4-error{{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target exit data map(from: i) depend (in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
   foo();
@@ -149,7 +151,7 @@ int main(int argc, char **argv, char *env[]) {
   foo();
   #pragma omp target exit data map(from: i) depend (in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   foo();
-  #pragma omp target exit data map(from: i) depend (in : argv[3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+  #pragma omp target exit data map(from : i) depend(in : argv [3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
   foo();
   #pragma omp target exit data map(from: i) depend(in:a[0:1]) // expected-error {{subscripted value is not an array or pointer}}
   foo();
diff --git a/clang/test/OpenMP/target_map_codegen.cpp b/clang/test/OpenMP/target_map_codegen.cpp
index ecfe50c01ea6d..92e0224a2de3b 100644
--- a/clang/test/OpenMP/target_map_codegen.cpp
+++ b/clang/test/OpenMP/target_map_codegen.cpp
@@ -39,7 +39,7 @@ class B {
 };
 double B::VAR = 1.0;
 
-// CK1-LABEL: @.__omp_offloading_{{.*}}implicit_maps_integer{{.*}}_l68.region_id = weak constant i8 0
+// CK1-LABEL: @.__omp_offloading_{{.*}}implicit_maps_integer{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK1-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 4]
 // Map types: OMP_MAP_PRIVATE_VAL | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 800
@@ -98,12 +98,12 @@ void implicit_maps_integer (int a){
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 #ifdef CK2
 
-// CK2-LABEL: @.__omp_offloading_{{.*}}implicit_maps_reference{{.*}}_l128.region_id = weak constant i8 0
+// CK2-LABEL: @.__omp_offloading_{{.*}}implicit_maps_reference{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK2: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 4]
 // Map types: OMP_MAP_PRIVATE_VAL | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 800
 // CK2: [[TYPES:@.+]] = {{.+}}constant [1 x i64] [i64 800]
-// CK2-LABEL: @.__omp_offloading_{{.*}}implicit_maps_reference{{.*}}_l147.region_id = weak constant i8 0
+// CK2-LABEL: @.__omp_offloading_{{.*}}implicit_maps_reference{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK2: [[SIZES2:@.+]] = {{.+}}constant [1 x i64] zeroinitializer
 // Map types: OMP_MAP_IS_PTR | OMP_MAP_IMPLICIT = 544
 // CK2: [[TYPES2:@.+]] = {{.+}}constant [1 x i64] [i64 544]
@@ -188,7 +188,7 @@ void implicit_maps_reference (int a, int *b){
 // SIMD-ONLY2-NOT: {{__kmpc|__tgt}}
 #ifdef CK3
 
-// CK3-LABEL: @.__omp_offloading_{{.*}}implicit_maps_parameter{{.*}}_l214.region_id = weak constant i8 0
+// CK3-LABEL: @.__omp_offloading_{{.*}}implicit_maps_parameter{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK3-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 4]
 // Map types: OMP_MAP_PRIVATE_VAL | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 800
@@ -242,7 +242,7 @@ void implicit_maps_parameter (int a){
 // SIMD-ONLY3-NOT: {{__kmpc|__tgt}}
 #ifdef CK4
 
-// CK4-LABEL: @.__omp_offloading_{{.*}}implicit_maps_nested_integer{{.*}}_l276.region_id = weak constant i8 0
+// CK4-LABEL: @.__omp_offloading_{{.*}}implicit_maps_nested_integer{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK4-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 4]
 // Map types: OMP_MAP_PRIVATE_VAL | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 800
@@ -308,7 +308,7 @@ void implicit_maps_nested_integer (int a){
 // SIMD-ONLY4-NOT: {{__kmpc|__tgt}}
 #ifdef CK5
 
-// CK5-LABEL: @.__omp_offloading_{{.*}}implicit_maps_nested_integer_and_enum{{.*}}_l340.region_id = weak constant i8 0
+// CK5-LABEL: @.__omp_offloading_{{.*}}implicit_maps_nested_integer_and_enum{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK5-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 4]
 // Map types: OMP_MAP_PRIVATE_VAL | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 800
@@ -353,12 +353,12 @@ void implicit_maps_nested_integer_and_enum (int a){
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK6 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK6 --check-prefix CK6-64
-// RUN: %clang_cc1 -DCK6 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6 --check-prefix CK6-64
-// RUN: %clang_cc1 -DCK6 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6 --check-prefix CK6-32
-// RUN: %clang_cc1 -DCK6 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6 --check-prefix CK6-32
+// RUN: %clang_cc1 -DCK6 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK6 --check-prefix CK6-64
+// RUN: %clang_cc1 -DCK6 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6 --check-prefix CK6-64
+// RUN: %clang_cc1 -DCK6 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6 --check-prefix CK6-32
+// RUN: %clang_cc1 -DCK6 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6 --check-prefix CK6-32
 
 // RUN: %clang_cc1 -DCK6 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY5 %s
 // RUN: %clang_cc1 -DCK6 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
@@ -368,7 +368,7 @@ void implicit_maps_nested_integer_and_enum (int a){
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY5 %s
 // SIMD-ONLY5-NOT: {{__kmpc|__tgt}}
 #ifdef CK6
-// CK6-LABEL: @.__omp_offloading_{{.*}}implicit_maps_host_global{{.*}}_l397.region_id = weak constant i8 0
+// CK6-LABEL: @.__omp_offloading_{{.*}}implicit_maps_host_global{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK6-DAG: [[GBL:@Gi]] = global i32 0
 // CK6-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 4]
@@ -428,7 +428,7 @@ void implicit_maps_host_global (int a){
 // For a 32-bit targets, the value doesn't fit the size of the pointer,
 // therefore it is passed by reference with a map 'to' specification.
 
-// CK7-LABEL: @.__omp_offloading_{{.*}}implicit_maps_double{{.*}}_l464.region_id = weak constant i8 0
+// CK7-LABEL: @.__omp_offloading_{{.*}}implicit_maps_double{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK7-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 8]
 // Map types: OMP_MAP_PRIVATE_VAL | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 800
@@ -497,7 +497,7 @@ void implicit_maps_double (int a){
 // SIMD-ONLY7-NOT: {{__kmpc|__tgt}}
 #ifdef CK8
 
-// CK8-LABEL: @.__omp_offloading_{{.*}}implicit_maps_float{{.*}}_l524.region_id = weak constant i8 0
+// CK8-LABEL: @.__omp_offloading_{{.*}}implicit_maps_float{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK8-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 4]
 // Map types: OMP_MAP_PRIVATE_VAL | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 800
@@ -551,7 +551,7 @@ void implicit_maps_float (int a){
 // SIMD-ONLY8-NOT: {{__kmpc|__tgt}}
 #ifdef CK9
 
-// CK9-LABEL: @.__omp_offloading_{{.*}}implicit_maps_array{{.*}}_l575.region_id = weak constant i8 0
+// CK9-LABEL: @.__omp_offloading_{{.*}}implicit_maps_array{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK9-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 16]
 // Map types: OMP_MAP_TO + OMP_MAP_FROM + OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 547
@@ -602,7 +602,7 @@ void implicit_maps_array (int a){
 // SIMD-ONLY9-NOT: {{__kmpc|__tgt}}
 #ifdef CK10
 
-// CK10-LABEL: @.__omp_offloading_{{.*}}implicit_maps_pointer{{.*}}_l626.region_id = weak constant i8 0
+// CK10-LABEL: @.__omp_offloading_{{.*}}implicit_maps_pointer{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK10-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] zeroinitializer
 // Map types: OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 544
@@ -638,12 +638,12 @@ void implicit_maps_pointer (){
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK11 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK11
-// RUN: %clang_cc1 -DCK11 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK11
-// RUN: %clang_cc1 -DCK11 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK11
-// RUN: %clang_cc1 -DCK11 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK11
+// RUN: %clang_cc1 -DCK11 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK11
+// RUN: %clang_cc1 -DCK11 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK11
+// RUN: %clang_cc1 -DCK11 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK11
+// RUN: %clang_cc1 -DCK11 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK11
 
 // RUN: %clang_cc1 -DCK11 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
 // RUN: %clang_cc1 -DCK11 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
@@ -654,7 +654,7 @@ void implicit_maps_pointer (){
 // SIMD-ONLY10-NOT: {{__kmpc|__tgt}}
 #ifdef CK11
 
-// CK11-LABEL: @.__omp_offloading_{{.*}}implicit_maps_double_complex{{.*}}_l678.region_id = weak constant i8 0
+// CK11-LABEL: @.__omp_offloading_{{.*}}implicit_maps_double_complex{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK11-DAG: [[SIZES:@.+]] = {{.+}}constant [2 x i64] [i64 16, i64 {{8|4}}]
 // Map types: OMP_MAP_TO  | OMP_MAP_FROM | OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 547
@@ -707,7 +707,7 @@ void implicit_maps_double_complex (int a, int *b){
 // For a 32-bit targets, the value doesn't fit the size of the pointer,
 // therefore it is passed by reference with a map 'to' specification.
 
-// CK12-LABEL: @.__omp_offloading_{{.*}}implicit_maps_float_complex{{.*}}_l743.region_id = weak constant i8 0
+// CK12-LABEL: @.__omp_offloading_{{.*}}implicit_maps_float_complex{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK12-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 8]
 // Map types: OMP_MAP_PRIVATE_VAL + OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 800
@@ -775,7 +775,7 @@ void implicit_maps_float_complex (int a){
 // SIMD-ONLY12-NOT: {{__kmpc|__tgt}}
 #ifdef CK13
 
-// CK13-LABEL: @.__omp_offloading_{{.*}}implicit_maps_variable_length_array{{.*}}_l825.region_id = weak constant i8 0
+// CK13-LABEL: @.__omp_offloading_{{.*}}implicit_maps_variable_length_array{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // We don't have a constant map size for VLAs.
 // Map types:
@@ -859,7 +859,7 @@ void implicit_maps_variable_length_array (int a){
 
 // CK14-DAG: [[ST:%.+]] = type { i32, double }
 
-// CK14-LABEL: @.__omp_offloading_{{.*}}foo{{.*}}_l877.region_id = weak constant i8 0
+// CK14-LABEL: @.__omp_offloading_{{.*}}foo{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 
 // Map types:
@@ -1207,7 +1207,7 @@ void implicit_maps_templated_function (int a){
 #ifdef CK17
 
 // CK17-DAG: [[ST:%.+]] = type { i32, double }
-// CK17-LABEL: @.__omp_offloading_{{.*}}implicit_maps_struct{{.*}}_l1236.region_id = weak constant i8 0
+// CK17-LABEL: @.__omp_offloading_{{.*}}implicit_maps_struct{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK17-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 {{16|12}}]
 // Map types: OMP_MAP_TO + OMP_MAP_FROM + OMP_MAP_TARGET_PARAM | OMP_MAP_IMPLICIT = 547
 // CK17-DAG: [[TYPES:@.+]] = {{.+}}constant [1 x i64] [i64 547]
@@ -1323,173 +1323,173 @@ void implicit_maps_template_type_capture (int a){
 // SIMD-ONLY18-NOT: {{__kmpc|__tgt}}
 #ifdef CK19
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1514.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE00:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK19: [[MTYPE00:@.+]] = private {{.*}}constant [1 x i64] [i64 32]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1535.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE00n:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK19: [[MTYPE00n:@.+]] = private {{.*}}constant [1 x i64] [i64 32]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1557.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE01:@.+]] = private {{.*}}constant [1 x i64] [i64 400]
 // CK19: [[MTYPE01:@.+]] = private {{.*}}constant [1 x i64] [i64 33]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1576.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE02:@.+]] = private {{.*}}constant [1 x i64] [i64 240]
 // CK19: [[MTYPE02:@.+]] = private {{.*}}constant [1 x i64] [i64 34]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1595.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE03:@.+]] = private {{.*}}constant [1 x i64] [i64 240]
 // CK19: [[MTYPE03:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1614.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE04:@.+]] = private {{.*}}constant [1 x i64] [i64 400]
 // CK19: [[MTYPE04:@.+]] = private {{.*}}constant [1 x i64] [i64 32]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1633.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE05:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK19: [[MTYPE05:@.+]] = private {{.*}}constant [1 x i64] [i64 33]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1656.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[MTYPE06:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1679.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[MTYPE07:@.+]] = private {{.*}}constant [1 x i64] [i64 32]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1698.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE08:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK19: [[MTYPE08:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1719.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE09:@.+]] = private {{.*}}constant [1 x i64] [i64 {{8|4}}]
 // CK19: [[MTYPE09:@.+]] = private {{.*}}constant [1 x i64] [i64 34]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1740.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE10:@.+]] = private {{.*}}constant [1 x i64] [i64 240]
 // CK19: [[MTYPE10:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1761.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE11:@.+]] = private {{.*}}constant [1 x i64] [i64 240]
 // CK19: [[MTYPE11:@.+]] = private {{.*}}constant [1 x i64] [i64 32]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1782.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE12:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK19: [[MTYPE12:@.+]] = private {{.*}}constant [1 x i64] [i64 33]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1807.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[MTYPE13:@.+]] = private {{.*}}constant [1 x i64] [i64 32]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1832.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[MTYPE14:@.+]] = private {{.*}}constant [1 x i64] [i64 33]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1853.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE15:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK19: [[MTYPE15:@.+]] = private {{.*}}constant [1 x i64] [i64 34]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1887.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[MTYPE16:@.+]] = private {{.*}}constant [2 x i64] [i64 800, i64 33]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1913.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE17:@.+]] = private {{.*}}constant [2 x i64] [i64 {{8|4}}, i64 240]
 // CK19: [[MTYPE17:@.+]] = private {{.*}}constant [2 x i64] [i64 800, i64 34]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1939.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE18:@.+]] = private {{.*}}constant [2 x i64] [i64 {{8|4}}, i64 240]
 // CK19: [[MTYPE18:@.+]] = private {{.*}}constant [2 x i64] [i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1971.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[MTYPE19:@.+]] = private {{.*}}constant [2 x i64] [i64 800, i64 32]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1997.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE20:@.+]] = private {{.*}}constant [2 x i64] [i64 {{8|4}}, i64 4]
 // CK19: [[MTYPE20:@.+]] = private {{.*}}constant [2 x i64] [i64 800, i64 33]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2029.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[MTYPE21:@.+]] = private {{.*}}constant [2 x i64] [i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2055.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE22:@.+]] = private {{.*}}constant [2 x i64] [i64 {{8|4}}, i64 4]
 // CK19: [[MTYPE22:@.+]] = private {{.*}}constant [2 x i64] [i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2074.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE23:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK19: [[MTYPE23:@.+]] = private {{.*}}constant [1 x i64] [i64 39]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2096.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE24:@.+]] = private {{.*}}constant [1 x i64] [i64 480]
 // CK19: [[MTYPE24:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2117.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE25:@.+]] = private {{.*}}constant [1 x i64] [i64 16]
 // CK19: [[MTYPE25:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2138.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE26:@.+]] = private {{.*}}constant [1 x i64] [i64 24]
 // CK19: [[MTYPE26:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2159.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE27:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK19: [[MTYPE27:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2204.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE28:@.+]] = private {{.*}}constant [3 x i64] [i64 {{8|4}}, i64 {{8|4}}, i64 16]
 // CK19: [[MTYPE28:@.+]] = private {{.*}}constant [3 x i64] [i64 35, i64 16, i64 19]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2249.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE29:@.+]] = private {{.*}}constant [3 x i64] [i64 {{8|4}}, i64 {{8|4}}, i64 4]
 // CK19: [[MTYPE29:@.+]] = private {{.*}}constant [3 x i64] [i64 35, i64 16, i64 19]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2305.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[MTYPE30:@.+]] = private {{.*}}constant [4 x i64] [i64 800, i64 800, i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2349.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE31:@.+]] = private {{.*}}constant [4 x i64] [i64 {{8|4}}, i64 {{8|4}}, i64 {{8|4}}, i64 40]
 // CK19: [[MTYPE31:@.+]] = private {{.*}}constant [4 x i64] [i64 800, i64 800, i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2372.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE32:@.+]] = private {{.*}}constant [1 x i64] [i64 13728]
 // CK19: [[MTYPE32:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2391.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE33:@.+]] = private {{.*}}constant [1 x i64] [i64 13728]
 // CK19: [[MTYPE33:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2410.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE34:@.+]] = private {{.*}}constant [1 x i64] [i64 13728]
 // CK19: [[MTYPE34:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2435.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[MTYPE35:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2456.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE36:@.+]] = private {{.*}}constant [1 x i64] [i64 208]
 // CK19: [[MTYPE36:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2496.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[MTYPE37:@.+]] = private {{.*}}constant [3 x i64] [i64 800, i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2538.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[MTYPE38:@.+]] = private {{.*}}constant [3 x i64] [i64 800, i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2580.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[MTYPE39:@.+]] = private {{.*}}constant [3 x i64] [i64 800, i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2622.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[MTYPE40:@.+]] = private {{.*}}constant [3 x i64] [i64 800, i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2657.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE41:@.+]] = private {{.*}}constant [3 x i64] [i64 {{8|4}}, i64 {{8|4}}, i64 208]
 // CK19: [[MTYPE41:@.+]] = private {{.*}}constant [3 x i64] [i64 800, i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2702.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE42:@.+]] = private {{.*}}constant [3 x i64] [i64 {{8|4}}, i64 {{8|4}}, i64 104]
 // CK19: [[MTYPE42:@.+]] = private {{.*}}constant [3 x i64] [i64 35, i64 16, i64 19]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2727.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[MTYPE43:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2746.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK19: [[SIZE44:@.+]] = private {{.*}}constant [1 x i64] [i64 320]
 // CK19: [[MTYPE44:@.+]] = private {{.*}}constant [1 x i64] [i64 34]
 
@@ -2814,19 +2814,19 @@ void explicit_maps_single (int ii){
 // SIMD-ONLY19-NOT: {{__kmpc|__tgt}}
 #ifdef CK20
 
-// CK20-LABEL: @.__omp_offloading_{{.*}}explicit_maps_references_and_function_args{{.*}}_l2856.region_id = weak constant i8 0
+// CK20-LABEL: @.__omp_offloading_{{.*}}explicit_maps_references_and_function_args{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK20: [[SIZE00:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK20: [[MTYPE00:@.+]] = private {{.*}}constant [1 x i64] [i64 33]
 
-// CK20-LABEL: @.__omp_offloading_{{.*}}explicit_maps_references_and_function_args{{.*}}_l2877.region_id = weak constant i8 0
+// CK20-LABEL: @.__omp_offloading_{{.*}}explicit_maps_references_and_function_args{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK20: [[SIZE01:@.+]] = private {{.*}}constant [1 x i64] [i64 20]
 // CK20: [[MTYPE01:@.+]] = private {{.*}}constant [1 x i64] [i64 33]
 
-// CK20-LABEL: @.__omp_offloading_{{.*}}explicit_maps_references_and_function_args{{.*}}_l2895.region_id = weak constant i8 0
+// CK20-LABEL: @.__omp_offloading_{{.*}}explicit_maps_references_and_function_args{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK20: [[SIZE02:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK20: [[MTYPE02:@.+]] = private {{.*}}constant [1 x i64] [i64 34]
 
-// CK20-LABEL: @.__omp_offloading_{{.*}}explicit_maps_references_and_function_args{{.*}}_l2916.region_id = weak constant i8 0
+// CK20-LABEL: @.__omp_offloading_{{.*}}explicit_maps_references_and_function_args{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK20: [[SIZE03:@.+]] = private {{.*}}constant [1 x i64] [i64 12]
 // CK20: [[MTYPE03:@.+]] = private {{.*}}constant [1 x i64] [i64 34]
 
@@ -5299,11 +5299,11 @@ void map_with_deep_copy() {
 // SIMD-ONLY18-NOT: {{__kmpc|__tgt}}
 #ifdef CK31
 
-// CK31-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l5329.region_id = weak constant i8 0
+// CK31-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK31: [[SIZE00:@.+]] = private {{.*}}constant [1 x i[[Z:64|32]]] [i[[Z:64|32]] 4]
 // CK31: [[MTYPE00:@.+]] = private {{.*}}constant [1 x i64] [i64 1059]
 
-// CK31-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l5348.region_id = weak constant i8 0
+// CK31-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK31: [[SIZE01:@.+]] = private {{.*}}constant [1 x i[[Z:64|32]]] [i[[Z:64|32]] 4]
 // CK31: [[MTYPE01:@.+]] = private {{.*}}constant [1 x i64] [i64 1063]
 
diff --git a/clang/test/OpenMP/target_map_messages.cpp b/clang/test/OpenMP/target_map_messages.cpp
index 92edd12e9449e..833f509cd7f0e 100644
--- a/clang/test/OpenMP/target_map_messages.cpp
+++ b/clang/test/OpenMP/target_map_messages.cpp
@@ -1,10 +1,10 @@
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,le50 -fopenmp -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp -fopenmp-version=40 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp -fopenmp-version=45 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le50 -fopenmp -fopenmp-version=50 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp -ferror-limit 200 -x c %s -Wno-openmp -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp-simd -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,le50 -fopenmp-simd -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp-simd -fopenmp-version=40 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp-simd -fopenmp-version=45 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le50 -fopenmp-simd -fopenmp-version=50 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
@@ -550,6 +550,12 @@ T tmain(T argc) {
 #pragma omp target data map(tofrom, close: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
 #pragma omp target data map(close, tofrom: close, tofrom, x)
   foo();
+
+  T marr[10][10], iarr[5];
+#pragma omp target data map(marr[10][0:2:2]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+  {}
+#pragma omp target data map(iarr[:2:d]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+  {}
   return 0;
 }
 
@@ -737,6 +743,13 @@ int main(int argc, char **argv) {
 #pragma omp target map(release: a) // expected-error {{map type 'release' is not allowed for '#pragma omp target'}}
   {}
 
+  int marr[10][10], iarr[5];
+
+#pragma omp target map(marr[10][0:2:2]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+  {}
+#pragma omp target map(iarr[:2:d]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+  {}
+
   return tmain<int, 3>(argc)+tmain<from, 4>(argc); // expected-note {{in instantiation of function template specialization 'tmain<int, 3>' requested here}} expected-note {{in instantiation of function template specialization 'tmain<int, 4>' requested here}}
 }
 #endif
diff --git a/clang/test/OpenMP/target_messages.cpp b/clang/test/OpenMP/target_messages.cpp
index 5bba976b9f9f6..a40a7d03ac80d 100644
--- a/clang/test/OpenMP/target_messages.cpp
+++ b/clang/test/OpenMP/target_messages.cpp
@@ -1,7 +1,10 @@
-// RUN: %clang_cc1 -verify -fopenmp -std=c++11 -o - %s
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -std=c++11 -o - %s
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -o - -std=c++11 %s -Wuninitialized
+
 // RUN: not %clang_cc1 -fopenmp -std=c++11 -fopenmp-targets=aaa-bbb-ccc-ddd -o - %s 2>&1 | FileCheck %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -std=c++11 -o - %s
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -std=c++11 -o - %s
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -std=c++11 -o - %s
 // CHECK: error: OpenMP target is invalid: 'aaa-bbb-ccc-ddd'
 // RUN: not %clang_cc1 -fopenmp -std=c++11 -triple nvptx64-nvidia-cuda -o - %s 2>&1 | FileCheck --check-prefix CHECK-UNSUPPORTED-HOST-TARGET %s
 // RUN: not %clang_cc1 -fopenmp -std=c++11 -triple nvptx-nvidia-cuda -o - %s 2>&1 | FileCheck --check-prefix CHECK-UNSUPPORTED-HOST-TARGET %s
@@ -13,7 +16,8 @@
 // NO-HOST-BC: The provided host compiler IR file '1111.bc' is required to generate code for OpenMP target regions but cannot be found.
 
 // RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc -DREGION_HOST
-// RUN: not %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -DREGION_DEVICE 2>&1
+// RUN: not %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -DREGION_DEVICE 2>&1
+// RUN: not %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -DREGION_DEVICE 2>&1
 
 #if defined(REGION_HOST) || defined(REGION_DEVICE)
 void foo() {
@@ -50,11 +54,11 @@ class S {
       int b;
     #pragma omp target map(this[1]) // expected-note {{expected 'this' subscript expression on map clause to be 'this[0]'}} // expected-error {{invalid 'this' expression on 'map' clause}}
       int c;
-    #pragma omp target map(foo) // expected-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target map(foo)         // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
       int d;
-    #pragma omp target map(zee) // expected-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target map(zee)         // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
       int e;
-    #pragma omp target map(this->zee) // expected-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target map(this->zee)   // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
       int f;
   }
 };
@@ -116,12 +120,12 @@ int main(int argc, char **argv) {
   #pragma omp target
   for (int n = 0; n < 100; ++n) {}
 
-  #pragma omp target map(foo) // expected-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target map(foo) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
   {}
 
   S s;
 
-  #pragma omp target map(s.zee) // expected-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target map(s.zee) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
   {}
 
   return 0;
diff --git a/clang/test/OpenMP/target_parallel_default_messages.cpp b/clang/test/OpenMP/target_parallel_default_messages.cpp
index 0691cdf37e4eb..c8f68659438fe 100644
--- a/clang/test/OpenMP/target_parallel_default_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_default_messages.cpp
@@ -2,20 +2,29 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp -ferror-limit 100 -o - %s -Wuninitialized
+
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp-simd -ferror-limit 100 -o - %s -Wuninitialized
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
   #pragma omp target parallel default // expected-error {{expected '(' after 'default'}}
   foo();
-  #pragma omp target parallel default ( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target parallel default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   foo();
-  #pragma omp target parallel default () // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp target parallel default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   foo();
   #pragma omp target parallel default (none // expected-error {{expected ')'}} expected-note {{to match this '('}}
   foo();
   #pragma omp target parallel default (shared), default(shared) // expected-error {{directive '#pragma omp target parallel' cannot contain more than one 'default' clause}}
   foo();
-  #pragma omp target parallel default (x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp target parallel default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   foo();
 
   #pragma omp target parallel default(none) // expected-note {{explicit data sharing attribute requested here}}
@@ -28,5 +37,14 @@ int main(int argc, char **argv) {
   #pragma omp target parallel default(none) // expected-note {{explicit data sharing attribute requested here}}
   #pragma omp parallel default(shared)
   ++argc; // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
+
+#ifndef OMP51
+#pragma omp target parallel default(firstprivate) // expected-error {{data-sharing attribute 'firstprivate' in 'default' clause requires OpenMP version 5.1 or above}}
+  {
+    ++x;
+    ++y;
+  }
+#endif
+
   return 0;
 }
diff --git a/clang/test/OpenMP/target_parallel_depend_messages.cpp b/clang/test/OpenMP/target_parallel_depend_messages.cpp
index 85111a8b14b5a..997669922d368 100644
--- a/clang/test/OpenMP/target_parallel_depend_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_depend_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
 void foo() {
 }
@@ -24,35 +26,35 @@ int main(int argc, char **argv, char *env[]) {
 
   #pragma omp target parallel depend // expected-error {{expected '(' after 'depend'}}
   foo();
-  #pragma omp target parallel depend ( // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target parallel depend( // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target parallel depend () // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target parallel depend() // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target parallel depend (argc // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target parallel depend(argc // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target parallel depend (source : argc) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target parallel depend(source : argc) // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-  #pragma omp target parallel depend (source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target parallel depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
   #pragma omp target parallel depend (in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target parallel' are ignored}}
   foo();
   #pragma omp target parallel depend (out: ) // expected-error {{expected expression}}
   foo();
-  #pragma omp target parallel depend (inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
+#pragma omp target parallel depend(inout : foobool(argc)), depend(in, argc) // omp4-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target parallel depend (out :S1) // expected-error {{'S1' does not refer to a value}}
   foo();
-  #pragma omp target parallel depend(in : argv[1][1] = '2')
+#pragma omp target parallel depend(in : argv[1][1] = '2')
   foo();
-  #pragma omp target parallel depend (in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
+#pragma omp target parallel depend(in : vec[1]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target parallel depend (in : argv[0])
   foo();
-  #pragma omp target parallel depend (in : ) // expected-error {{expected expression}}
+#pragma omp target parallel depend(in:) // expected-error {{expected expression}}
   foo();
   #pragma omp target parallel depend (in : main)
   foo();
-  #pragma omp target parallel depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
+#pragma omp target parallel depend(in : a[0]) // omp4-error{{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
   #pragma omp target parallel depend (in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
   foo();
@@ -72,7 +74,7 @@ int main(int argc, char **argv, char *env[]) {
   foo();
   #pragma omp target parallel depend (in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   foo();
-  #pragma omp target parallel depend (in : argv[3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+  #pragma omp target parallel depend(in : argv [3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
   foo();
   #pragma omp target parallel depend(in:a[0:1]) // expected-error {{subscripted value is not an array or pointer}}
   foo();
diff --git a/clang/test/OpenMP/target_parallel_for_default_messages.cpp b/clang/test/OpenMP/target_parallel_for_default_messages.cpp
index fc6ba43138d76..4a3aae68e0865 100644
--- a/clang/test/OpenMP/target_parallel_for_default_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_for_default_messages.cpp
@@ -2,17 +2,26 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=51 -DOMP51 -ferror-limit 100 -o - %s -Wuninitialized
+
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=51 -DOMP51 -ferror-limit 100 -o - %s -Wuninitialized
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
   int i;
 #pragma omp target parallel for default // expected-error {{expected '(' after 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
-#pragma omp target parallel for default( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target parallel for default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (i = 0; i < argc; ++i)
     foo();
-#pragma omp target parallel for default() // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp target parallel for default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
 #pragma omp target parallel for default(none // expected-error {{expected ')'}} expected-note {{to match this '('}} expected-note {{explicit data sharing attribute requested here}}
@@ -21,7 +30,7 @@ int main(int argc, char **argv) {
 #pragma omp target parallel for default(shared), default(shared) // expected-error {{directive '#pragma omp target parallel for' cannot contain more than one 'default' clause}}
   for (i = 0; i < argc; ++i)
     foo();
-#pragma omp target parallel for default(x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp target parallel for default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
 
@@ -34,5 +43,13 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
     foo();
 
+#ifndef OMP51
+#pragma omp target parallel for default(firstprivate) // expected-error {{data-sharing attribute 'firstprivate' in 'default' clause requires OpenMP version 5.1 or above}}
+  for (i = 0; i < argc; ++i) {
+    ++x;
+    ++y;
+  }
+#endif
+
   return 0;
 }
diff --git a/clang/test/OpenMP/target_parallel_for_depend_messages.cpp b/clang/test/OpenMP/target_parallel_for_depend_messages.cpp
index 34d11ea860d6c..bb8a282fb9b05 100644
--- a/clang/test/OpenMP/target_parallel_for_depend_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_for_depend_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
 void foo() {
 }
@@ -25,35 +27,42 @@ int main(int argc, char **argv, char *env[]) {
 
   #pragma omp target parallel for depend // expected-error {{expected '(' after 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for depend ( // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target parallel for depend( // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for depend () // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target parallel for depend() // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for depend (argc // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target parallel for depend(argc // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for depend (source : argc) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for depend (source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target parallel for depend(source : argc) // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target parallel for depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
   #pragma omp target parallel for depend (in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target parallel for' are ignored}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for depend (out: ) // expected-error {{expected expression}}
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for depend (inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for depend (out :S1) // expected-error {{'S1' does not refer to a value}}
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for depend(in : argv[1][1] = '2')
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for depend (in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for depend (in : argv[0])
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for depend (in : ) // expected-error {{expected expression}}
+#pragma omp target parallel for depend(out:) // expected-error {{expected expression}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target parallel for depend(inout : foobool(argc)), depend(in, argc) // omp4-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target parallel for depend(out : S1) // expected-error {{'S1' does not refer to a value}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target parallel for depend(in : argv[1][1] = '2')
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target parallel for depend(in : vec[1]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target parallel for depend(in : argv[0])
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target parallel for depend(in:) // expected-error {{expected expression}}
   for (i = 0; i < argc; ++i) foo();
   #pragma omp target parallel for depend (in : main)
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
+#pragma omp target parallel for depend(in : a[0]) // omp4-error{{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   for (i = 0; i < argc; ++i) foo();
   #pragma omp target parallel for depend (in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
   for (i = 0; i < argc; ++i) foo();
@@ -73,7 +82,7 @@ int main(int argc, char **argv, char *env[]) {
   for (i = 0; i < argc; ++i) foo();
   #pragma omp target parallel for depend (in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for depend (in : argv[3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+  #pragma omp target parallel for depend(in : argv [3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
   for (i = 0; i < argc; ++i) foo();
   #pragma omp target parallel for depend(in:a[0:1]) // expected-error {{subscripted value is not an array or pointer}}
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_parallel_for_loop_messages.cpp b/clang/test/OpenMP/target_parallel_for_loop_messages.cpp
index 671f116adffd8..2cbf858bf66f4 100644
--- a/clang/test/OpenMP/target_parallel_for_loop_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_for_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wno-openmp-mapping -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/target_parallel_for_map_messages.cpp b/clang/test/OpenMP/target_parallel_for_map_messages.cpp
index 0d75d702f78cf..b44c330e2260d 100644
--- a/clang/test/OpenMP/target_parallel_for_map_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_for_map_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
@@ -103,7 +105,7 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for map(T) // expected-error {{'T' does not refer to a value}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target parallel for map(I) // expected-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target parallel for map(I) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for map(S2::S2s)
   for (i = 0; i < argc; ++i) foo();
@@ -121,7 +123,8 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target parallel for map(tofrom: argc > 0 ? x : y) // expected-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} 
+#pragma omp target parallel for map(tofrom \
+                                    : argc > 0 ? x : y) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -228,8 +231,10 @@ int main(int argc, char **argv) {
 #pragma omp target parallel for map(to, x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
-  for (i = 0; i < argc; ++i) foo();
-#pragma omp target parallel for map(tofrom: argc > 0 ? argv[1] : argv[2]) // expected-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target parallel for map(tofrom \
+                                    : argc > 0 ? argv[1] : argv[2]) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for map(argc)
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_parallel_for_simd_default_messages.cpp b/clang/test/OpenMP/target_parallel_for_simd_default_messages.cpp
index daa93b9c9050b..48489309ef037 100644
--- a/clang/test/OpenMP/target_parallel_for_simd_default_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_for_simd_default_messages.cpp
@@ -2,17 +2,26 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp -ferror-limit 100 -o - %s -Wuninitialized
+
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp-simd -ferror-limit 100 -o - %s -Wuninitialized
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
   int i;
 #pragma omp target parallel for simd default // expected-error {{expected '(' after 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
-#pragma omp target parallel for simd default( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target parallel for simd default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (i = 0; i < argc; ++i)
     foo();
-#pragma omp target parallel for simd default() // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp target parallel for simd default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
 #pragma omp target parallel for simd default(none // expected-error {{expected ')'}} expected-note {{to match this '('}} expected-note {{explicit data sharing attribute requested here}}
@@ -21,7 +30,7 @@ int main(int argc, char **argv) {
 #pragma omp target parallel for simd default(shared), default(shared) // expected-error {{directive '#pragma omp target parallel for simd' cannot contain more than one 'default' clause}}
   for (i = 0; i < argc; ++i)
     foo();
-#pragma omp target parallel for simd default(x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp target parallel for simd default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (i = 0; i < argc; ++i)
     foo();
 
@@ -34,5 +43,13 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) // expected-error {{variable 'i' must have explicitly specified data sharing attributes}} expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
     foo();
 
+#ifndef OMP51
+#pragma omp target parallel for simd default(firstprivate) // expected-error {{data-sharing attribute 'firstprivate' in 'default' clause requires OpenMP version 5.1 or above}}
+  for (int i = 0; i < argc; i++) {
+    ++x;
+    ++y;
+  }
+#endif
+
   return 0;
 }
diff --git a/clang/test/OpenMP/target_parallel_for_simd_depend_messages.cpp b/clang/test/OpenMP/target_parallel_for_simd_depend_messages.cpp
index 14e202ba53b57..64e4c05ab5f0e 100644
--- a/clang/test/OpenMP/target_parallel_for_simd_depend_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_for_simd_depend_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -std=c++11 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -std=c++11 %s -Wuninitialized
 
 void foo() {
 }
@@ -25,35 +27,42 @@ int main(int argc, char **argv, char *env[]) {
 
   #pragma omp target parallel for simd depend // expected-error {{expected '(' after 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for simd depend ( // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target parallel for simd depend( // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for simd depend () // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target parallel for simd depend() // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for simd depend (argc // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target parallel for simd depend(argc // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for simd depend (source : argc) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for simd depend (source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target parallel for simd depend(source : argc) // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target parallel for simd depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
   #pragma omp target parallel for simd depend (in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target parallel for simd' are ignored}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for simd depend (out: ) // expected-error {{expected expression}}
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for simd depend (inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for simd depend (out :S1) // expected-error {{'S1' does not refer to a value}}
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for simd depend(in : argv[1][1] = '2')
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for simd depend (in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for simd depend (in : argv[0])
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for simd depend (in : ) // expected-error {{expected expression}}
+#pragma omp target parallel for simd depend(out:) // expected-error {{expected expression}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target parallel for simd depend(inout : foobool(argc)), depend(in, argc) // omp4-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target parallel for simd depend(out : S1) // expected-error {{'S1' does not refer to a value}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target parallel for simd depend(in : argv[1][1] = '2')
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target parallel for simd depend(in : vec[1]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target parallel for simd depend(in : argv[0])
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target parallel for simd depend(in:) // expected-error {{expected expression}}
   for (i = 0; i < argc; ++i) foo();
   #pragma omp target parallel for simd depend (in : main)
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for simd depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
+#pragma omp target parallel for simd depend(in : a[0]) // omp4-error{{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   for (i = 0; i < argc; ++i) foo();
   #pragma omp target parallel for simd depend (in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
   for (i = 0; i < argc; ++i) foo();
@@ -73,7 +82,7 @@ int main(int argc, char **argv, char *env[]) {
   for (i = 0; i < argc; ++i) foo();
   #pragma omp target parallel for simd depend (in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target parallel for simd depend (in : argv[3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+  #pragma omp target parallel for simd depend(in : argv [3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
   for (i = 0; i < argc; ++i) foo();
   #pragma omp target parallel for simd depend(in:a[0:1]) // expected-error {{subscripted value is not an array or pointer}}
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_parallel_for_simd_is_device_ptr_messages.cpp b/clang/test/OpenMP/target_parallel_for_simd_is_device_ptr_messages.cpp
index 47e498a44c6db..77fc1338518c2 100644
--- a/clang/test/OpenMP/target_parallel_for_simd_is_device_ptr_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_for_simd_is_device_ptr_messages.cpp
@@ -1,6 +1,9 @@
-// RUN: %clang_cc1 -std=c++11 -verify -fopenmp %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify=expected,omp4 -fopenmp -fopenmp-version=45 %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify -fopenmp -fopenmp-version=50 %s -Wuninitialized
+
+// RUN: %clang_cc1 -std=c++11 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify -fopenmp-simd -fopenmp-version=50 %s -Wuninitialized
 
-// RUN: %clang_cc1 -std=c++11 -verify -fopenmp-simd %s -Wuninitialized
 struct ST {
   int *a;
 };
@@ -228,13 +231,13 @@ T tmain(T argc) {
 #pragma omp target parallel for simd is_device_ptr(ps) map(ps->a) // expected-error{{pointer cannot be mapped along with a section derived from itself}} expected-note{{used here}}
   for (int i=0; i<100; i++)
     ;
-#pragma omp target parallel for simd is_device_ptr(ps) firstprivate(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target parallel for simd' directive}}
+#pragma omp target parallel for simd is_device_ptr(ps) firstprivate(ps) // omp4-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target parallel for simd' directive}}
   for (int i=0; i<100; i++)
     ;
 #pragma omp target parallel for simd firstprivate(ps) is_device_ptr(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target parallel for simd' directive}} expected-note{{defined as firstprivate}}
   for (int i=0; i<100; i++)
     ;
-#pragma omp target parallel for simd is_device_ptr(ps) private(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target parallel for simd' directive}}
+#pragma omp target parallel for simd is_device_ptr(ps) private(ps) // omp4-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target parallel for simd' directive}}
   for (int i=0; i<100; i++)
     ;
 #pragma omp target parallel for simd private(ps) is_device_ptr(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target parallel for simd' directive}} expected-note{{defined as private}}
@@ -323,13 +326,13 @@ int main(int argc, char **argv) {
 #pragma omp target parallel for simd is_device_ptr(ps) map(ps->a) // expected-error{{pointer cannot be mapped along with a section derived from itself}} expected-note{{used here}}
   for (int i=0; i<100; i++)
     ;
-#pragma omp target parallel for simd is_device_ptr(ps) firstprivate(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target parallel for simd' directive}}
+#pragma omp target parallel for simd is_device_ptr(ps) firstprivate(ps) // omp4-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target parallel for simd' directive}}
   for (int i=0; i<100; i++)
     ;
 #pragma omp target parallel for simd firstprivate(ps) is_device_ptr(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target parallel for simd' directive}} expected-note{{defined as firstprivate}}
   for (int i=0; i<100; i++)
     ;
-#pragma omp target parallel for simd is_device_ptr(ps) private(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target parallel for simd' directive}}
+#pragma omp target parallel for simd is_device_ptr(ps) private(ps) // omp4-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target parallel for simd' directive}}
   for (int i=0; i<100; i++)
     ;
 #pragma omp target parallel for simd private(ps) is_device_ptr(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target parallel for simd' directive}} expected-note{{defined as private}}
diff --git a/clang/test/OpenMP/target_parallel_for_simd_map_messages.cpp b/clang/test/OpenMP/target_parallel_for_simd_map_messages.cpp
index 8156ad8f308c0..d4c0b30f57664 100644
--- a/clang/test/OpenMP/target_parallel_for_simd_map_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_for_simd_map_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
@@ -103,7 +105,7 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for simd map(T) // expected-error {{'T' does not refer to a value}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target parallel for simd map(I) // expected-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target parallel for simd map(I) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for simd map(S2::S2s)
   for (i = 0; i < argc; ++i) foo();
@@ -121,7 +123,8 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for simd map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target parallel for simd map(tofrom: argc > 0 ? x : y) // expected-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} 
+#pragma omp target parallel for simd map(tofrom \
+                                         : argc > 0 ? x : y) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for simd map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -228,8 +231,10 @@ int main(int argc, char **argv) {
 #pragma omp target parallel for simd map(to, x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for simd map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
-  for (i = 0; i < argc; ++i) foo();
-#pragma omp target parallel for simd map(tofrom: argc > 0 ? argv[1] : argv[2]) // expected-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target parallel for simd map(tofrom \
+                                         : argc > 0 ? argv[1] : argv[2]) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for simd map(argc)
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_parallel_is_device_ptr_messages.cpp b/clang/test/OpenMP/target_parallel_is_device_ptr_messages.cpp
index 9913133734a74..c8aef46d703ed 100644
--- a/clang/test/OpenMP/target_parallel_is_device_ptr_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_is_device_ptr_messages.cpp
@@ -1,6 +1,8 @@
 // RUN: %clang_cc1 -std=c++11 -verify -fopenmp -ferror-limit 200 %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 200 %s -Wuninitialized
 
 // RUN: %clang_cc1 -std=c++11 -verify -fopenmp-simd -ferror-limit 200 %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 200 %s -Wuninitialized
 struct ST {
   int *a;
 };
@@ -189,11 +191,11 @@ T tmain(T argc) {
   {}
 #pragma omp target parallel is_device_ptr(ps) map(ps->a) // expected-error{{pointer cannot be mapped along with a section derived from itself}} expected-note{{used here}}
   {}
-#pragma omp target parallel is_device_ptr(ps) firstprivate(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target parallel' directive}}
+#pragma omp target parallel is_device_ptr(ps) firstprivate(ps) // omp4-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target parallel' directive}}
   {}
 #pragma omp target parallel firstprivate(ps) is_device_ptr(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target parallel' directive}} expected-note{{defined as firstprivate}}
   {}
-#pragma omp target parallel is_device_ptr(ps) private(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target parallel' directive}}
+#pragma omp target parallel is_device_ptr(ps) private(ps) // omp4-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target parallel' directive}}
   {}
 #pragma omp target parallel private(ps) is_device_ptr(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target parallel' directive}} expected-note{{defined as private}}
   {}
@@ -258,11 +260,11 @@ int main(int argc, char **argv) {
   {}
 #pragma omp target parallel is_device_ptr(ps) map(ps->a) // expected-error{{pointer cannot be mapped along with a section derived from itself}} expected-note{{used here}}
   {}
-#pragma omp target parallel is_device_ptr(ps) firstprivate(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target parallel' directive}}
+#pragma omp target parallel is_device_ptr(ps) firstprivate(ps) // omp4-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target parallel' directive}}
   {}
 #pragma omp target parallel firstprivate(ps) is_device_ptr(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target parallel' directive}} expected-note{{defined as firstprivate}}
   {}
-#pragma omp target parallel is_device_ptr(ps) private(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target parallel' directive}}
+#pragma omp target parallel is_device_ptr(ps) private(ps) // omp4-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target parallel' directive}}
   {}
 #pragma omp target parallel private(ps) is_device_ptr(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target parallel' directive}} expected-note{{defined as private}}
   {}
diff --git a/clang/test/OpenMP/target_parallel_map_messages.cpp b/clang/test/OpenMP/target_parallel_map_messages.cpp
index c35bb99d8d59a..6e080fda9226b 100644
--- a/clang/test/OpenMP/target_parallel_map_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_map_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
@@ -103,7 +105,7 @@ T tmain(T argc) {
   foo();
 #pragma omp target parallel map(T) // expected-error {{'T' does not refer to a value}}
   foo();
-#pragma omp target parallel map(I) // expected-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target parallel map(I) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
   foo();
 #pragma omp target parallel map(S2::S2s)
   foo();
@@ -121,7 +123,8 @@ T tmain(T argc) {
   foo();
 #pragma omp target parallel map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   foo();
-#pragma omp target parallel map(tofrom: argc > 0 ? x : y) // expected-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} 
+#pragma omp target parallel map(tofrom \
+                                : argc > 0 ? x : y) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
   foo();
 #pragma omp target parallel map(argc)
   foo();
@@ -228,7 +231,8 @@ int main(int argc, char **argv) {
   foo();
 #pragma omp target parallel map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   foo();
-#pragma omp target parallel map(tofrom: argc > 0 ? argv[1] : argv[2]) // expected-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target parallel map(tofrom \
+                                : argc > 0 ? argv[1] : argv[2]) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
   foo();
 #pragma omp target parallel map(argc)
   foo();
diff --git a/clang/test/OpenMP/target_simd_depend_codegen.cpp b/clang/test/OpenMP/target_simd_depend_codegen.cpp
index efbb71772b31f..72cd550207b67 100644
--- a/clang/test/OpenMP/target_simd_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_simd_depend_codegen.cpp
@@ -1,10 +1,10 @@
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
-// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
-// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
diff --git a/clang/test/OpenMP/target_simd_depend_messages.cpp b/clang/test/OpenMP/target_simd_depend_messages.cpp
index f4fefb013325c..a4f6e535194a3 100644
--- a/clang/test/OpenMP/target_simd_depend_messages.cpp
+++ b/clang/test/OpenMP/target_simd_depend_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -std=c++11 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -std=c++11 %s -Wuninitialized
 
 void foo() {
 }
@@ -25,35 +27,42 @@ int main(int argc, char **argv, char *env[]) {
 
   #pragma omp target simd depend // expected-error {{expected '(' after 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target simd depend ( // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target simd depend( // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target simd depend () // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target simd depend() // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target simd depend (argc // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target simd depend(argc // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target simd depend (source : argc) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target simd depend (source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target simd depend(source : argc) // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target simd depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
   #pragma omp target simd depend (in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target simd' are ignored}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target simd depend (out: ) // expected-error {{expected expression}}
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target simd depend (inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target simd depend (out :S1) // expected-error {{'S1' does not refer to a value}}
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target simd depend(in : argv[1][1] = '2')
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target simd depend (in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target simd depend (in : argv[0])
-  for (i = 0; i < argc; ++i) foo();
-  #pragma omp target simd depend (in : ) // expected-error {{expected expression}}
+#pragma omp target simd depend(out:) // expected-error {{expected expression}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target simd depend(inout : foobool(argc)), depend(in, argc) // omp4-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target simd depend(out : S1) // expected-error {{'S1' does not refer to a value}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target simd depend(in : argv[1][1] = '2')
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target simd depend(in : vec[1]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target simd depend(in : argv[0])
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target simd depend(in:) // expected-error {{expected expression}}
   for (i = 0; i < argc; ++i) foo();
   #pragma omp target simd depend (in : main)
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target simd depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
+#pragma omp target simd depend(in : a[0]) // omp4-error{{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   for (i = 0; i < argc; ++i) foo();
   #pragma omp target simd depend (in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
   for (i = 0; i < argc; ++i) foo();
@@ -73,7 +82,7 @@ int main(int argc, char **argv, char *env[]) {
   for (i = 0; i < argc; ++i) foo();
   #pragma omp target simd depend (in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   for (i = 0; i < argc; ++i) foo();
-  #pragma omp target simd depend (in : argv[3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+  #pragma omp target simd depend(in : argv [3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
   for (i = 0; i < argc; ++i) foo();
   #pragma omp target simd depend(in:a[0:1]) // expected-error {{subscripted value is not an array or pointer}}
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_simd_map_messages.cpp b/clang/test/OpenMP/target_simd_map_messages.cpp
index b8ee39aa03d3d..57929e276260e 100644
--- a/clang/test/OpenMP/target_simd_map_messages.cpp
+++ b/clang/test/OpenMP/target_simd_map_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
@@ -97,7 +99,7 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target simd map(T) // expected-error {{'T' does not refer to a value}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target simd map(I) // expected-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target simd map(I) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target simd map(S2::S2s)
   for (i = 0; i < argc; ++i) foo();
@@ -115,7 +117,8 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target simd map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target simd map(tofrom: argc > 0 ? x : y) // expected-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} 
+#pragma omp target simd map(tofrom \
+                            : argc > 0 ? x : y) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target simd map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -216,8 +219,10 @@ int main(int argc, char **argv) {
 #pragma omp target simd map(to, x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target simd map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
-  for (i = 0; i < argc; ++i) foo();
-#pragma omp target simd map(tofrom: argc > 0 ? argv[1] : argv[2]) // expected-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target simd map(tofrom \
+                            : argc > 0 ? argv[1] : argv[2]) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target simd map(argc)
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_teams_default_messages.cpp b/clang/test/OpenMP/target_teams_default_messages.cpp
index 21fa8270ef6a7..85c417f8f9853 100644
--- a/clang/test/OpenMP/target_teams_default_messages.cpp
+++ b/clang/test/OpenMP/target_teams_default_messages.cpp
@@ -2,20 +2,29 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -o - %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp -o - %s -Wuninitialized
+
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp-simd -o - %s -Wuninitialized
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
 #pragma omp target teams default // expected-error {{expected '(' after 'default'}}
   foo();
-#pragma omp target teams default ( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target teams default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   foo();
-#pragma omp target teams default () // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp target teams default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   foo();
 #pragma omp target teams default (none // expected-error {{expected ')'}} expected-note {{to match this '('}}
   foo();
 #pragma omp target teams default (shared), default(shared) // expected-error {{directive '#pragma omp target teams' cannot contain more than one 'default' clause}}
   foo();
-#pragma omp target teams default (x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp target teams default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   foo();
 
 #pragma omp target teams default(none) // expected-note {{explicit data sharing attribute requested here}}
@@ -24,5 +33,14 @@ int main(int argc, char **argv) {
 #pragma omp target teams default(none) // expected-note {{explicit data sharing attribute requested here}}
 #pragma omp parallel default(shared)
   ++argc; // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
+
+#ifndef OMP51
+#pragma omp target teams default(firstprivate) // expected-error {{data-sharing attribute 'firstprivate' in 'default' clause requires OpenMP version 5.1 or above}}
+  {
+    ++x;
+    ++y;
+  }
+#endif
+
   return 0;
 }
diff --git a/clang/test/OpenMP/target_teams_depend_messages.cpp b/clang/test/OpenMP/target_teams_depend_messages.cpp
index 1575e163db827..5baf73a9ba5ea 100644
--- a/clang/test/OpenMP/target_teams_depend_messages.cpp
+++ b/clang/test/OpenMP/target_teams_depend_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
 void foo() {
 }
@@ -24,27 +26,27 @@ int main(int argc, char **argv, char *env[]) {
 
 #pragma omp target teams depend // expected-error {{expected '(' after 'depend'}}
   foo();
-#pragma omp target teams depend ( // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target teams depend( // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-#pragma omp target teams depend () // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target teams depend() // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-#pragma omp target teams depend (argc // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target teams depend(argc // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-#pragma omp target teams depend (source : argc) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target teams depend(source : argc) // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
-#pragma omp target teams depend (source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target teams depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   foo();
 #pragma omp target teams depend (in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target teams' are ignored}}
   foo();
 #pragma omp target teams depend (out: ) // expected-error {{expected expression}}
   foo();
-#pragma omp target teams depend (inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
+#pragma omp target teams depend(inout : foobool(argc)), depend(in, argc) // omp4-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
 #pragma omp target teams depend (out :S1) // expected-error {{'S1' does not refer to a value}}
   foo();
 #pragma omp target teams depend(in : argv[1][1] = '2')
   foo();
-#pragma omp target teams depend (in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
+#pragma omp target teams depend(in : vec[1]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
 #pragma omp target teams depend (in : argv[0])
   foo();
@@ -52,7 +54,7 @@ int main(int argc, char **argv, char *env[]) {
   foo();
 #pragma omp target teams depend (in : main)
   foo();
-#pragma omp target teams depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
+#pragma omp target teams depend(in : a[0]) // omp4-error{{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   foo();
 #pragma omp target teams depend (in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
   foo();
@@ -72,7 +74,7 @@ int main(int argc, char **argv, char *env[]) {
   foo();
 #pragma omp target teams depend (in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   foo();
-#pragma omp target teams depend (in : argv[3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+#pragma omp target teams depend(in : argv [3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
   foo();
 #pragma omp target teams depend(in:a[0:1]) // expected-error {{subscripted value is not an array or pointer}}
   foo();
diff --git a/clang/test/OpenMP/target_teams_distribute_default_messages.cpp b/clang/test/OpenMP/target_teams_distribute_default_messages.cpp
index fd834e7cba32c..a490ad61385ff 100644
--- a/clang/test/OpenMP/target_teams_distribute_default_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_default_messages.cpp
@@ -2,24 +2,41 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=51 -DOMP51 %s -Wuninitialized
+
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=51 -DOMP51 %s -Wuninitialized
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
   #pragma omp target teams distribute default // expected-error {{expected '(' after 'default'}}
   for (int i=0; i<200; i++) foo();
-  #pragma omp target teams distribute default ( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target teams distribute default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (int i=0; i<200; i++) foo();
-  #pragma omp target teams distribute default () // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp target teams distribute default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target teams distribute default (none // expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target teams distribute default (shared), default(shared) // expected-error {{directive '#pragma omp target teams distribute' cannot contain more than one 'default' clause}}
   for (int i=0; i<200; i++) foo();
-  #pragma omp target teams distribute default (x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp target teams distribute default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (int i=0; i<200; i++) foo();
 
   #pragma omp target teams distribute default(none) // expected-note {{explicit data sharing attribute requested here}}
   for (int i=0; i<200; i++) ++argc; // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
 
+#ifndef OMP51
+#pragma omp target teams distribute default(firstprivate) // expected-error {{data-sharing attribute 'firstprivate' in 'default' clause requires OpenMP version 5.1 or above}}
+  for (int i = 0; i < 200; i++) {
+    ++x;
+    ++y;
+  }
+#endif
+
   return 0;
 }
diff --git a/clang/test/OpenMP/target_teams_distribute_depend_messages.cpp b/clang/test/OpenMP/target_teams_distribute_depend_messages.cpp
index 740eff22e723d..fab61b34126ec 100644
--- a/clang/test/OpenMP/target_teams_distribute_depend_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_depend_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
 void foo() {
 }
@@ -25,35 +27,38 @@ int main(int argc, char **argv, char *env[]) {
 
 #pragma omp target teams distribute depend // expected-error {{expected '(' after 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute depend ( // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target teams distribute depend( // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute depend () // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target teams distribute depend() // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute depend (argc // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target teams distribute depend(argc // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute depend (source : argc) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
-  for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute depend (source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target teams distribute depend(source : argc) // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target teams distribute depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute depend (in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target teams distribute' are ignored}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute depend (out: ) // expected-error {{expected expression}}
-  for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute depend (inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target teams distribute depend(inout : foobool(argc)), depend(in, argc) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute depend (out :S1) // expected-error {{'S1' does not refer to a value}}
-  for (i = 0; i < argc; ++i) foo();
+  for (i = 0; i < argc; ++i)
+    foo();
 #pragma omp target teams distribute depend(in : argv[1][1] = '2')
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute depend (in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
+#pragma omp target teams distribute depend(in : vec[1]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute depend (in : argv[0])
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute depend (in : ) // expected-error {{expected expression}}
+#pragma omp target teams distribute depend(in:) // expected-error {{expected expression}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute depend (in : main)
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
+#pragma omp target teams distribute depend(in : a[0]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute depend (in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
   for (i = 0; i < argc; ++i) foo();
@@ -73,7 +78,7 @@ int main(int argc, char **argv, char *env[]) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute depend (in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute depend (in : argv[3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+#pragma omp target teams distribute depend(in : argv [3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute depend(in:a[0:1]) // expected-error {{subscripted value is not an array or pointer}}
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_teams_distribute_loop_messages.cpp b/clang/test/OpenMP/target_teams_distribute_loop_messages.cpp
index e20b04bd94362..dcaf5824dfaf2 100644
--- a/clang/test/OpenMP/target_teams_distribute_loop_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wno-openmp-mapping -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/target_teams_distribute_map_messages.cpp b/clang/test/OpenMP/target_teams_distribute_map_messages.cpp
index d5f87f74672bb..38855680c1fe5 100644
--- a/clang/test/OpenMP/target_teams_distribute_map_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_map_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
@@ -103,7 +105,7 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute map(T) // expected-error {{'T' does not refer to a value}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute map(I) // expected-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams distribute map(I) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute map(S2::S2s)
   for (i = 0; i < argc; ++i) foo();
@@ -121,7 +123,8 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute map(tofrom: argc > 0 ? x : y) // expected-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} 
+#pragma omp target teams distribute map(tofrom \
+                                        : argc > 0 ? x : y) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -228,8 +231,10 @@ int main(int argc, char **argv) {
 #pragma omp target teams distribute map(to, x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
-  for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute map(tofrom: argc > 0 ? argv[1] : argv[2]) // expected-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target teams distribute map(tofrom \
+                                        : argc > 0 ? argv[1] : argv[2]) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute map(argc)
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_default_messages.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_default_messages.cpp
index 00e0704a6ccac..2fe7931369618 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_default_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_default_messages.cpp
@@ -2,24 +2,41 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp %s -Wuninitialized
+
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp-simd %s -Wuninitialized
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
 #pragma omp target teams distribute parallel for default // expected-error {{expected '(' after 'default'}}
   for (int i=0; i<200; i++) foo();
- #pragma omp target teams distribute parallel for default ( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target teams distribute parallel for default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (int i=0; i<200; i++) foo();
-#pragma omp target teams distribute parallel for default () // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp target teams distribute parallel for default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (int i=0; i<200; i++) foo();
 #pragma omp target teams distribute parallel for default (none // expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (int i=0; i<200; i++) foo();
 #pragma omp target teams distribute parallel for default (shared), default(shared) // expected-error {{directive '#pragma omp target teams distribute parallel for' cannot contain more than one 'default' clause}}
   for (int i=0; i<200; i++) foo();
-#pragma omp target teams distribute parallel for default (x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp target teams distribute parallel for default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (int i=0; i<200; i++) foo();
 
 #pragma omp target teams distribute parallel for default(none) // expected-note {{explicit data sharing attribute requested here}}
   for (int i=0; i<200; i++) ++argc; // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
 
+#ifndef OMP51
+#pragma omp target teams distribute parallel for default(firstprivate) // expected-error {{data-sharing attribute 'firstprivate' in 'default' clause requires OpenMP version 5.1 or above}}
+  for (int i = 0; i < 200; i++) {
+    ++x;
+    ++y;
+  }
+#endif
+
   return 0;
 }
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_depend_messages.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_depend_messages.cpp
index 34d655c6a2576..c8adbe53b7423 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_depend_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_depend_messages.cpp
@@ -1,7 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
-
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 void foo() {
 }
 
@@ -25,35 +26,40 @@ int main(int argc, char **argv, char *env[]) {
 
 #pragma omp target teams distribute parallel for depend // expected-error {{expected '(' after 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for depend ( // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
-  for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for depend () // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target teams distribute parallel for depend( // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for depend (argc // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target teams distribute parallel for depend() // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for depend (source : argc) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target teams distribute parallel for depend(argc // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for depend (source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target teams distribute parallel for depend(source : argc) // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target teams distribute parallel for depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for depend (in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target teams distribute parallel for' are ignored}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for depend (out: ) // expected-error {{expected expression}}
-  for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for depend (inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
-  for (i = 0; i < argc; ++i) foo();
+#pragma omp target teams distribute parallel for depend(out:) // expected-error {{expected expression}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target teams distribute parallel for depend(inout : foobool(argc)), depend(in, argc) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
+  for (i = 0; i < argc; ++i)
+    foo();
 #pragma omp target teams distribute parallel for depend (out :S1) // expected-error {{'S1' does not refer to a value}}
-  for (i = 0; i < argc; ++i) foo();
+  for (i = 0; i < argc; ++i)
+    foo();
 #pragma omp target teams distribute parallel for depend(in : argv[1][1] = '2')
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for depend (in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
-  for (i = 0; i < argc; ++i) foo();
+#pragma omp target teams distribute parallel for depend(in : vec[1]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
+  for (i = 0; i < argc; ++i)
+    foo();
 #pragma omp target teams distribute parallel for depend (in : argv[0])
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for depend (in : ) // expected-error {{expected expression}}
+#pragma omp target teams distribute parallel for depend(in:) // expected-error {{expected expression}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for depend (in : main)
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
+#pragma omp target teams distribute parallel for depend(in : a[0]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for depend (in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
   for (i = 0; i < argc; ++i) foo();
@@ -73,7 +79,7 @@ int main(int argc, char **argv, char *env[]) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for depend (in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for depend (in : argv[3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+#pragma omp target teams distribute parallel for depend(in : argv [3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for depend(in:a[0:1]) // expected-error {{subscripted value is not an array or pointer}}
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_is_device_ptr_messages.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_is_device_ptr_messages.cpp
index fdf98d40ea857..070ca7beaaafe 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_is_device_ptr_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_is_device_ptr_messages.cpp
@@ -1,6 +1,8 @@
 // RUN: %clang_cc1 -std=c++11 -verify -fopenmp %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify=expected,omp4 -fopenmp -fopenmp-version=45 %s -Wuninitialized
 
 // RUN: %clang_cc1 -std=c++11 -verify -fopenmp-simd %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 %s -Wuninitialized
 struct ST {
   int *a;
 };
@@ -228,13 +230,13 @@ T tmain(T argc) {
 #pragma omp target teams distribute parallel for is_device_ptr(ps) map(ps->a) // expected-error{{pointer cannot be mapped along with a section derived from itself}} expected-note{{used here}}
   for (int i=0; i<100; i++)
     ;
-#pragma omp target teams distribute parallel for is_device_ptr(ps) firstprivate(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for' directive}}
+#pragma omp target teams distribute parallel for is_device_ptr(ps) firstprivate(ps) // omp4-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for' directive}}
   for (int i=0; i<100; i++)
     ;
 #pragma omp target teams distribute parallel for firstprivate(ps) is_device_ptr(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for' directive}} expected-note{{defined as firstprivate}}
   for (int i=0; i<100; i++)
     ;
-#pragma omp target teams distribute parallel for is_device_ptr(ps) private(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for' directive}}
+#pragma omp target teams distribute parallel for is_device_ptr(ps) private(ps) // omp4-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for' directive}}
   for (int i=0; i<100; i++)
     ;
 #pragma omp target teams distribute parallel for private(ps) is_device_ptr(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for' directive}} expected-note{{defined as private}}
@@ -323,13 +325,13 @@ int main(int argc, char **argv) {
 #pragma omp target teams distribute parallel for is_device_ptr(ps) map(ps->a) // expected-error{{pointer cannot be mapped along with a section derived from itself}} expected-note{{used here}}
   for (int i=0; i<100; i++)
     ;
-#pragma omp target teams distribute parallel for is_device_ptr(ps) firstprivate(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for' directive}}
+#pragma omp target teams distribute parallel for is_device_ptr(ps) firstprivate(ps) // omp4-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for' directive}}
   for (int i=0; i<100; i++)
     ;
 #pragma omp target teams distribute parallel for firstprivate(ps) is_device_ptr(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for' directive}} expected-note{{defined as firstprivate}}
   for (int i=0; i<100; i++)
     ;
-#pragma omp target teams distribute parallel for is_device_ptr(ps) private(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for' directive}}
+#pragma omp target teams distribute parallel for is_device_ptr(ps) private(ps) // omp4-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for' directive}}
   for (int i=0; i<100; i++)
     ;
 #pragma omp target teams distribute parallel for private(ps) is_device_ptr(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for' directive}} expected-note{{defined as private}}
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_lastprivate_messages.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_lastprivate_messages.cpp
index 34cd01f3eecc7..ff279dcf8fb91 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_lastprivate_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_lastprivate_messages.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp50 -fopenmp %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp-version=40 -fopenmp %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp-version=45 -fopenmp %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,omp50 -fopenmp-version=50 -fopenmp %s -Wuninitialized
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_loop_messages.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_loop_messages.cpp
index a57f213615bd0..a440cbdcb6234 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_loop_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wno-openmp-mapping -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_map_messages.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_map_messages.cpp
index 0be31256a42a9..a1af4600b59a4 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_map_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_map_messages.cpp
@@ -1,6 +1,7 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
@@ -103,7 +104,7 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for map(T) // expected-error {{'T' does not refer to a value}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for map(I) // expected-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams distribute parallel for map(I) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for map(S2::S2s)
   for (i = 0; i < argc; ++i) foo();
@@ -121,7 +122,8 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for map(tofrom: argc > 0 ? x : y) // expected-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} 
+#pragma omp target teams distribute parallel for map(tofrom \
+                                                     : argc > 0 ? x : y) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -229,7 +231,8 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for map(tofrom: argc > 0 ? argv[1] : argv[2]) // expected-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams distribute parallel for map(tofrom \
+                                                     : argc > 0 ? argv[1] : argv[2]) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for map(argc)
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_schedule_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_schedule_codegen.cpp
index 8eb6c443ca2d8..3ec1782f2c09d 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_schedule_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_schedule_codegen.cpp
@@ -3,12 +3,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-OMP45 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-OMP45 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-OMP45 --check-prefix CK1-32
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-OMP45 --check-prefix CK1-32
+
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-OMP50 --check-prefix CK1-64
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-OMP50 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-OMP50 --check-prefix CK1-32
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-OMP50 --check-prefix CK1-32
 
 // RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix SIMD-ONLY 
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
@@ -116,7 +123,8 @@ struct SS{
   // CK1: ret void
 
   // CK1: define internal void @[[PAR_OUTL4]]({{.+}})
-  // CK1: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+  // CK1-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+  // CK1-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
   // CK1: call {{.+}} @__kmpc_dispatch_next_4(
   // CK1: ret void
 
@@ -131,7 +139,8 @@ struct SS{
   // CK1: ret void
 
   // CK1: define internal void @[[PAR_OUTL5]]({{.+}})
-  // CK1: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+  // CK1-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+  // CK1-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
   // CK1: call {{.+}} @__kmpc_dispatch_next_4(
   // CK1: ret void
 
@@ -147,12 +156,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64 --check-prefix OMP45
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64 --check-prefix OMP45
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32 --check-prefix OMP45
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32 --check-prefix OMP45
+
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64 --check-prefix OMP50
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64 --check-prefix OMP50
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32 --check-prefix OMP50
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32 --check-prefix OMP50
 
 // RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix SIMD-ONLY 
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
@@ -287,7 +303,8 @@ int main (int argc, char **argv) {
 // CK2: ret void
 
 // CK2: define internal void @[[PAR_OUTL4]]({{.+}})
-// CK2: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
 // CK2: call {{.+}} @__kmpc_dispatch_next_4(
 // CK2: ret void
 
@@ -303,7 +320,8 @@ int main (int argc, char **argv) {
 // CK2: ret void
 
 // CK2: define internal void @[[PAR_OUTL5]]({{.+}})
-// CK2: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
 // CK2: call {{.+}} @__kmpc_dispatch_next_4(
 // CK2: ret void
 
@@ -377,7 +395,8 @@ int main (int argc, char **argv) {
 // CK2: ret void
 
 // CK2: define internal void @[[PAR_OUTLT4]]({{.+}})
-// CK2: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
 // CK2: call {{.+}} @__kmpc_dispatch_next_4(
 // CK2: ret void
 
@@ -392,7 +411,8 @@ int main (int argc, char **argv) {
 // CK2: ret void
 
 // CK2: define internal void @[[PAR_OUTLT5]]({{.+}})
-// CK2: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
 // CK2: call {{.+}} @__kmpc_dispatch_next_4(
 // CK2: ret void
 
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_default_messages.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_default_messages.cpp
index 7c46c964d2ec3..e5ff856222501 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_default_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_default_messages.cpp
@@ -2,16 +2,25 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp %s -Wuninitialized
+
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp-simd %s -Wuninitialized
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
 #pragma omp target teams distribute parallel for simd default // expected-error {{expected '(' after 'default'}}
   for (int i=0; i<200; i++) foo();
 
-#pragma omp target teams distribute parallel for simd default ( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target teams distribute parallel for simd default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (int i=0; i<200; i++) foo();
 
-#pragma omp target teams distribute parallel for simd default () // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp target teams distribute parallel for simd default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (int i=0; i<200; i++) foo();
 
 #pragma omp target teams distribute parallel for simd default (none // expected-error {{expected ')'}} expected-note {{to match this '('}}
@@ -20,11 +29,19 @@ int main(int argc, char **argv) {
 #pragma omp target teams distribute parallel for simd default (shared), default(shared) // expected-error {{directive '#pragma omp target teams distribute parallel for simd' cannot contain more than one 'default' clause}}
   for (int i=0; i<200; i++) foo();
 
-#pragma omp target teams distribute parallel for simd default (x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp target teams distribute parallel for simd default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (int i=0; i<200; i++) foo();
 
 #pragma omp target teams distribute parallel for simd default(none) // expected-note {{explicit data sharing attribute requested here}}
   for (int i=0; i<200; i++) ++argc; // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
 
+#ifndef OMP51
+#pragma omp target teams distribute parallel for simd default(firstprivate) // expected-error {{data-sharing attribute 'firstprivate' in 'default' clause requires OpenMP version 5.1 or above}}
+  for (int i = 0; i < argc; ++i) {
+    ++x;
+    ++y;
+  }
+#endif
+
   return 0;
 }
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_depend_messages.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_depend_messages.cpp
index b0378423aaa06..dc584508e7b59 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_depend_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_depend_messages.cpp
@@ -1,7 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
-
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 void foo() {
 }
 
@@ -25,35 +26,40 @@ int main(int argc, char **argv, char *env[]) {
 
 #pragma omp target teams distribute parallel for simd depend // expected-error {{expected '(' after 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd depend ( // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
-  for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd depend () // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target teams distribute parallel for simd depend( // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd depend (argc // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target teams distribute parallel for simd depend() // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd depend (source : argc) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target teams distribute parallel for simd depend(argc // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd depend (source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target teams distribute parallel for simd depend(source : argc) // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target teams distribute parallel for simd depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd depend (in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target teams distribute parallel for simd' are ignored}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd depend (out: ) // expected-error {{expected expression}}
-  for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd depend (inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
-  for (i = 0; i < argc; ++i) foo();
+#pragma omp target teams distribute parallel for simd depend(out:) // expected-error {{expected expression}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target teams distribute parallel for simd depend(inout : foobool(argc)), depend(in, argc) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
+  for (i = 0; i < argc; ++i)
+    foo();
 #pragma omp target teams distribute parallel for simd depend (out :S1) // expected-error {{'S1' does not refer to a value}}
-  for (i = 0; i < argc; ++i) foo();
+  for (i = 0; i < argc; ++i)
+    foo();
 #pragma omp target teams distribute parallel for simd depend(in : argv[1][1] = '2')
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd depend (in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
-  for (i = 0; i < argc; ++i) foo();
+#pragma omp target teams distribute parallel for simd depend(in : vec[1]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
+  for (i = 0; i < argc; ++i)
+    foo();
 #pragma omp target teams distribute parallel for simd depend (in : argv[0])
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd depend (in : ) // expected-error {{expected expression}}
+#pragma omp target teams distribute parallel for simd depend(in:) // expected-error {{expected expression}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd depend (in : main)
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
+#pragma omp target teams distribute parallel for simd depend(in : a[0]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd depend (in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
   for (i = 0; i < argc; ++i) foo();
@@ -73,7 +79,7 @@ int main(int argc, char **argv, char *env[]) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd depend (in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd depend (in : argv[3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+#pragma omp target teams distribute parallel for simd depend(in : argv [3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd depend(in:a[0:1]) // expected-error {{subscripted value is not an array or pointer}}
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_firstprivate_messages.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_firstprivate_messages.cpp
index 73125b62bb3dd..fb64d4e9ee167 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_firstprivate_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_firstprivate_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 %s -Wuninitialized
 
 #pragma omp requires dynamic_allocators
 typedef void **omp_allocator_handle_t;
@@ -133,8 +135,9 @@ int main(int argc, char **argv) {
 #pragma omp target teams distribute parallel for simd firstprivate(h) // expected-error {{threadprivate or thread local variable cannot be firstprivate}}
   for (i = 0; i < argc; ++i) foo();
 
-#pragma omp target teams distribute parallel for simd private(i), firstprivate(i) // expected-error {{private variable cannot be firstprivate}} expected-note 2 {{defined as private}}
-  for (i = 0; i < argc; ++i) foo(); // expected-error {{loop iteration variable in the associated loop of 'omp target teams distribute parallel for simd' directive may not be private, predetermined as linear}}
+#pragma omp target teams distribute parallel for simd private(i), firstprivate(i) // expected-error {{private variable cannot be firstprivate}}  expected-note {{defined as private}} omp4-note {{defined as private}}
+  for (i = 0; i < argc; ++i) // omp4-error {{loop iteration variable in the associated loop of 'omp target teams distribute parallel for simd' directive may not be private, predetermined as linear}}
+    foo();
 
 #pragma omp target teams distribute parallel for simd firstprivate(i)
   for (j = 0; j < argc; ++j) foo();
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_is_device_ptr_messages.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_is_device_ptr_messages.cpp
index b368f2a3f5486..1f8f71afcd4ab 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_is_device_ptr_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_is_device_ptr_messages.cpp
@@ -1,6 +1,9 @@
 // RUN: %clang_cc1 -std=c++11 -verify -fopenmp %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify=expected,omp4 -fopenmp -fopenmp-version=45 %s -Wuninitialized
 
 // RUN: %clang_cc1 -std=c++11 -verify -fopenmp-simd %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 %s -Wuninitialized
+
 struct ST {
   int *a;
 };
@@ -228,13 +231,13 @@ T tmain(T argc) {
 #pragma omp target teams distribute parallel for simd is_device_ptr(ps) map(ps->a) // expected-error{{pointer cannot be mapped along with a section derived from itself}} expected-note{{used here}}
   for (int i=0; i<100; i++)
     ;
-#pragma omp target teams distribute parallel for simd is_device_ptr(ps) firstprivate(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for simd' directive}}
+#pragma omp target teams distribute parallel for simd is_device_ptr(ps) firstprivate(ps) // omp4-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for simd' directive}}
   for (int i=0; i<100; i++)
     ;
 #pragma omp target teams distribute parallel for simd firstprivate(ps) is_device_ptr(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for simd' directive}} expected-note{{defined as firstprivate}}
   for (int i=0; i<100; i++)
     ;
-#pragma omp target teams distribute parallel for simd is_device_ptr(ps) private(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for simd' directive}}
+#pragma omp target teams distribute parallel for simd is_device_ptr(ps) private(ps) // omp4-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for simd' directive}}
   for (int i=0; i<100; i++)
     ;
 #pragma omp target teams distribute parallel for simd private(ps) is_device_ptr(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for simd' directive}} expected-note{{defined as private}}
@@ -323,13 +326,13 @@ int main(int argc, char **argv) {
 #pragma omp target teams distribute parallel for simd is_device_ptr(ps) map(ps->a) // expected-error{{pointer cannot be mapped along with a section derived from itself}} expected-note{{used here}}
   for (int i=0; i<100; i++)
     ;
-#pragma omp target teams distribute parallel for simd is_device_ptr(ps) firstprivate(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for simd' directive}}
+#pragma omp target teams distribute parallel for simd is_device_ptr(ps) firstprivate(ps) // omp4-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for simd' directive}}
   for (int i=0; i<100; i++)
     ;
 #pragma omp target teams distribute parallel for simd firstprivate(ps) is_device_ptr(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for simd' directive}} expected-note{{defined as firstprivate}}
   for (int i=0; i<100; i++)
     ;
-#pragma omp target teams distribute parallel for simd is_device_ptr(ps) private(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for simd' directive}}
+#pragma omp target teams distribute parallel for simd is_device_ptr(ps) private(ps) // omp4-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for simd' directive}}
   for (int i=0; i<100; i++)
     ;
 #pragma omp target teams distribute parallel for simd private(ps) is_device_ptr(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute parallel for simd' directive}} expected-note{{defined as private}}
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_lastprivate_messages.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_lastprivate_messages.cpp
index 49aa77c953c11..3be5fc7bcdf92 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_lastprivate_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_lastprivate_messages.cpp
@@ -1,9 +1,9 @@
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,le50 -fopenmp %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp-version=40 -fopenmp %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp-version=45 -fopenmp %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le50 -fopenmp-version=50 -fopenmp %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp-simd %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,le50 -fopenmp-simd %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le50 -fopenmp-version=50 -fopenmp-simd %s -Wuninitialized
 
 typedef void **omp_allocator_handle_t;
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_map_messages.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_map_messages.cpp
index 25c5d938d9151..b53a506bf234b 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_map_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_map_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
@@ -103,7 +105,7 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd map(T) // expected-error {{'T' does not refer to a value}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd map(I) // expected-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams distribute parallel for simd map(I) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd map(S2::S2s)
   for (i = 0; i < argc; ++i) foo();
@@ -121,7 +123,8 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd map(tofrom: argc > 0 ? x : y) // expected-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} 
+#pragma omp target teams distribute parallel for simd map(tofrom \
+                                                          : argc > 0 ? x : y) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -229,7 +232,8 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd map(tofrom: argc > 0 ? argv[1] : argv[2]) // expected-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams distribute parallel for simd map(tofrom \
+                                                          : argc > 0 ? argv[1] : argv[2]) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd map(argc)
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_schedule_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_schedule_codegen.cpp
index 799769739d976..9852a75187481 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_schedule_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_schedule_codegen.cpp
@@ -3,12 +3,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64 --check-prefix CK1-OMP45
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64 --check-prefix CK1-OMP45
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32 --check-prefix CK1-OMP45
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32 --check-prefix CK1-OMP45
+
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64 --check-prefix CK1-OMP50
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64  --check-prefix CK1-OMP50
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32 --check-prefix CK1-OMP50
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32 --check-prefix CK1-OMP50
 
 // RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix SIMD-ONLY
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
@@ -116,7 +123,8 @@ struct SS{
   // CK1: ret void
 
   // CK1: define internal void @[[PAR_OUTL4]]({{.+}})
-  // CK1: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+  // CK1-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+  // CK1-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
   // CK1: call {{.+}} @__kmpc_dispatch_next_4(
   // CK1: ret void
 
@@ -131,7 +139,8 @@ struct SS{
   // CK1: ret void
 
   // CK1: define internal void @[[PAR_OUTL5]]({{.+}})
-  // CK1: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+  // CK1-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+  // CK1-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
   // CK1: call {{.+}} @__kmpc_dispatch_next_4(
   // CK1: ret void
 
@@ -147,12 +156,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64 --check-prefix CK2-OMP45
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64 --check-prefix CK2-OMP45
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32 --check-prefix CK2-OMP45
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32 --check-prefix CK2-OMP45
+
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64 --check-prefix CK2-OMP50
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64 --check-prefix CK2-OMP50
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32 --check-prefix CK2-OMP50
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32 --check-prefix CK2-OMP50
 
 // RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix SIMD-ONLY
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
@@ -287,7 +303,8 @@ int main (int argc, char **argv) {
 // CK2: ret void
 
 // CK2: define internal void @[[PAR_OUTL4]]({{.+}})
-// CK2: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
 // CK2: call {{.+}} @__kmpc_dispatch_next_4(
 // CK2: ret void
 
@@ -303,7 +320,8 @@ int main (int argc, char **argv) {
 // CK2: ret void
 
 // CK2: define internal void @[[PAR_OUTL5]]({{.+}})
-// CK2: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
 // CK2: call {{.+}} @__kmpc_dispatch_next_4(
 // CK2: ret void
 
@@ -377,7 +395,8 @@ int main (int argc, char **argv) {
 // CK2: ret void
 
 // CK2: define internal void @[[PAR_OUTLT4]]({{.+}})
-// CK2: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
 // CK2: call {{.+}} @__kmpc_dispatch_next_4(
 // CK2: ret void
 
@@ -392,7 +411,8 @@ int main (int argc, char **argv) {
 // CK2: ret void
 
 // CK2: define internal void @[[PAR_OUTLT5]]({{.+}})
-// CK2: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
 // CK2: call {{.+}} @__kmpc_dispatch_next_4(
 // CK2: ret void
 
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_depend_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_simd_depend_codegen.cpp
index 8e1240aa596e1..4e8ff2f70234a 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_depend_codegen.cpp
@@ -1,10 +1,19 @@
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP50 --check-prefix OMP50-64
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP45
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP50 --check-prefix OMP50-64
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP50 --check-prefix OMP50-32
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP50 --check-prefix OMP50-32
+
+
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP45
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP45
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP45
+
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
@@ -43,9 +52,14 @@
 
 // TCHECK: [[ENTTY:%.+]] = type { i8*, i8*, i{{32|64}}, i32, i32 }
 
-// CHECK-DAG: [[SIZET:@.+]] = private unnamed_addr constant [2 x i64] [i64 0, i64 4]
-// CHECK-DAG: [[MAPT:@.+]] = private unnamed_addr constant [2 x i64] [i64 544, i64 800]
-// CHECK-DAG: @{{.*}} = weak constant i8 0
+// OMP45-DAG: [[SIZET:@.+]] = private unnamed_addr constant [2 x i64] [i64 0, i64 4]
+// OMP45-DAG: [[MAPT:@.+]] = private unnamed_addr constant [2 x i64] [i64 544, i64 800]
+// OMP45-DAG: @{{.*}} = weak constant i8 0
+
+// OMP50-DAG: [[SIZET:@.+]] = private unnamed_addr constant [3 x i64] [i64 0, i64 4, i64 1]
+// OMP50-DAG: [[MAPT:@.+]] = private unnamed_addr constant [3 x i64] [i64 544, i64 800, i64 800]
+// OMP50-DAG: @{{.*}} = weak constant i8 0
+
 
 // TCHECK: @{{.+}} = weak constant [[ENTTY]]
 // TCHECK: @{{.+}} = {{.*}}constant [[ENTTY]]
@@ -98,31 +112,52 @@ int foo(int n) {
   // CHECK:       [[ADD:%.+]] = add nsw i32
   // CHECK:       store i32 [[ADD]], i32* [[DEVICE_CAP:%.+]],
 
-  // CHECK:       [[BOOL:%.+]] = icmp ne i32 %{{.+}}, 0
-  // CHECK:       br i1 [[BOOL]], label %[[THEN:.+]], label %[[ELSE:.+]]
+  // OMP45:       [[BOOL:%.+]] = icmp ne i32 %{{.+}}, 0
+  // OMP45:       br i1 [[BOOL]], label %[[THEN:.+]], label %[[ELSE:.+]]
+  // OMP50:       br i1 {{.+}}, label %[[THEN:.+]], label %[[ELSE:.+]]
   // CHECK:       [[THEN]]:
-  // CHECK-DAG:   [[BPADDR0:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[BP:%.+]], i32 0, i32 0
-  // CHECK-DAG:   [[PADDR0:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[P:%.+]], i32 0, i32 0
+  // OMP45-DAG:   [[BPADDR0:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[BP:%.+]], i32 0, i32 0
+  // OMP45-DAG:   [[PADDR0:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[P:%.+]], i32 0, i32 0
+  // OMP50-DAG:   [[BPADDR0:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* [[BP:%.+]], i32 0, i32 0
+  // OMP50-DAG:   [[PADDR0:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* [[P:%.+]], i32 0, i32 0
   // CHECK-DAG:   [[CBPADDR0:%.+]] = bitcast i8** [[BPADDR0]] to i[[SZ]]**
   // CHECK-DAG:   [[CPADDR0:%.+]] = bitcast i8** [[PADDR0]] to i[[SZ]]**
   // CHECK-DAG:   store i[[SZ]]* [[BP0:%[^,]+]], i[[SZ]]** [[CBPADDR0]]
   // CHECK-DAG:   store i[[SZ]]* [[BP0]], i[[SZ]]** [[CPADDR0]]
 
-  // CHECK-DAG:   [[BPADDR1:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[BP]], i32 0, i32 1
-  // CHECK-DAG:   [[PADDR1:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[P]], i32 0, i32 1
+  // OMP45-DAG:   [[BPADDR1:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[BP]], i32 0, i32 1
+  // OMP45-DAG:   [[PADDR1:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[P]], i32 0, i32 1
+
+  // OMP50-DAG:   [[BPADDR1:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* [[BP]], i32 0, i32 1
+  // OMP50-DAG:   [[PADDR1:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* [[P]], i32 0, i32 1
   // CHECK-DAG:   [[CBPADDR1:%.+]] = bitcast i8** [[BPADDR1]] to i[[SZ]]*
   // CHECK-DAG:   [[CPADDR1:%.+]] = bitcast i8** [[PADDR1]] to i[[SZ]]*
   // CHECK-DAG:   store i[[SZ]] [[BP1:%[^,]+]], i[[SZ]]* [[CBPADDR1]]
   // CHECK-DAG:   store i[[SZ]] [[BP1]], i[[SZ]]* [[CPADDR1]]
-  // CHECK-DAG:   getelementptr inbounds [2 x i8*], [2 x i8*]* [[BP]], i32 0, i32 0
-  // CHECK-DAG:   getelementptr inbounds [2 x i8*], [2 x i8*]* [[P]], i32 0, i32 0
-  // CHECK:       [[GEP:%.+]] = getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 2
+
+  // OMP50-DAG:   [[BPADDR2:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* [[BP]], i32 0, i32 2
+  // OMP50-DAG:   [[PADDR2:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* [[P]], i32 0, i32 2
+  // OMP50-DAG:   [[CBPADDR2:%.+]] = bitcast i8** [[BPADDR2]] to i[[SZ]]*
+  // OMP50-DAG:   [[CPADDR2:%.+]] = bitcast i8** [[PADDR2]] to i[[SZ]]*
+  // OMP50-DAG:   store i[[SZ]] [[BP2:%[^,]+]], i[[SZ]]* [[CBPADDR2]]
+  // OMP50-DAG:   store i[[SZ]] [[BP2]], i[[SZ]]* [[CPADDR2]]
+
+
+  // OMP45-DAG:   getelementptr inbounds [2 x i8*], [2 x i8*]* [[BP]], i32 0, i32 0
+  // OMP45-DAG:   getelementptr inbounds [2 x i8*], [2 x i8*]* [[P]], i32 0, i32 0
+  // OMP50-DAG:   getelementptr inbounds [3 x i8*], [3 x i8*]* [[BP]], i32 0, i32 0
+  // OMP50-DAG:   getelementptr inbounds [3 x i8*], [3 x i8*]* [[P]], i32 0, i32 0
+
+  // OMP45:       [[GEP:%.+]] = getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 2
+  // OMP50-64:       [[GEP:%.+]] = getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 4
+  // OMP50-32:    [[GEP:%.+]] = getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 3
   // CHECK:       [[DEV:%.+]] = load i32, i32* [[DEVICE_CAP]],
   // CHECK:       store i32 [[DEV]], i32* [[GEP]],
   // CHECK:       [[DEV1:%.+]] = load i32, i32* [[DEVICE_CAP]],
   // CHECK:       [[DEV2:%.+]] = sext i32 [[DEV1]] to i64
 
-  // CHECK:       [[TASK:%.+]] = call i8* @__kmpc_omp_target_task_alloc(%struct.ident_t* [[ID]], i32 [[GTID]], i32 1, i[[SZ]] {{104|60}}, i[[SZ]] {{16|12}}, i32 (i32, i8*)* bitcast (i32 (i32, %{{.+}}*)* [[TASK_ENTRY1_:@.+]] to i32 (i32, i8*)*), i64 [[DEV2]])
+  // OMP45:       [[TASK:%.+]] = call i8* @__kmpc_omp_target_task_alloc(%struct.ident_t* [[ID]], i32 [[GTID]], i32 1, i[[SZ]] {{104|60}}, i[[SZ]] {{16|12}}, i32 (i32, i8*)* bitcast (i32 (i32, %{{.+}}*)* [[TASK_ENTRY1_:@.+]] to i32 (i32, i8*)*), i64 [[DEV2]])
+  // OMP50:       [[TASK:%.+]] = call i8* @__kmpc_omp_target_task_alloc(%struct.ident_t* [[ID]], i32 [[GTID]], i32 1, i[[SZ]] {{28|128|76}}, i[[SZ]] {{16|12|24}}, i32 (i32, i8*)* bitcast (i32 (i32, %{{.+}}*)* [[TASK_ENTRY1_:@.+]] to i32 (i32, i8*)*), i64 [[DEV2]])
   // CHECK:       [[BC_TASK:%.+]] = bitcast i8* [[TASK]] to [[TASK_TY1_:%.+]]*
   // CHECK:       getelementptr %struct.kmp_depend_info, %struct.kmp_depend_info* %{{.+}}, i[[SZ]] 0
   // CHECK:       getelementptr %struct.kmp_depend_info, %struct.kmp_depend_info* %{{.+}}, i[[SZ]] 1
@@ -132,14 +167,18 @@ int foo(int n) {
   // CHECK:       br label %[[EXIT:.+]]
 
   // CHECK:       [[ELSE]]:
-  // CHECK-NOT:   getelementptr inbounds [2 x i8*], [2 x i8*]*
-  // CHECK:       [[GEP:%.+]] = getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 2
+  // OMP45-NOT:   getelementptr inbounds [2 x i8*], [2 x i8*]*
+  // OMP50-NOT:   getelementptr inbounds [3 x i8*], [3 x i8*]*
+  // OMP45:       [[GEP:%.+]] = getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 2
+  // OMP50-64:       [[GEP:%.+]] = getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 4
+  // OMP50-32:       [[GEP:%.+]] = getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 3
   // CHECK:       [[DEV:%.+]] = load i32, i32* [[DEVICE_CAP]],
   // CHECK:       store i32 [[DEV]], i32* [[GEP]],
   // CHECK:       [[DEV1:%.+]] = load i32, i32* [[DEVICE_CAP]],
   // CHECK:       [[DEV2:%.+]] = sext i32 [[DEV1]] to i64
 
-  // CHECK:       [[TASK:%.+]] = call i8* @__kmpc_omp_target_task_alloc(%struct.ident_t* [[ID]], i32 [[GTID]], i32 1, i[[SZ]] {{56|28}}, i[[SZ]] {{16|12}}, i32 (i32, i8*)* bitcast (i32 (i32, %{{.+}}*)* [[TASK_ENTRY1__:@.+]] to i32 (i32, i8*)*), i64 [[DEV2]])
+  // OMP45:       [[TASK:%.+]] = call i8* @__kmpc_omp_target_task_alloc(%struct.ident_t* [[ID]], i32 [[GTID]], i32 1, i[[SZ]] {{56|28}}, i[[SZ]] {{16|12}}, i32 (i32, i8*)* bitcast (i32 (i32, %{{.+}}*)* [[TASK_ENTRY1__:@.+]] to i32 (i32, i8*)*), i64 [[DEV2]])
+  // OMP50:       [[TASK:%.+]] = call i8* @__kmpc_omp_target_task_alloc(%struct.ident_t* [[ID]], i32 [[GTID]], i32 1, i[[SZ]] {{56|28}}, i[[SZ]] {{16|12|24}}, i32 (i32, i8*)* bitcast (i32 (i32, %{{.+}}*)* [[TASK_ENTRY1__:@.+]] to i32 (i32, i8*)*), i64 [[DEV2]])
   // CHECK:       [[BC_TASK:%.+]] = bitcast i8* [[TASK]] to [[TASK_TY1__:%.+]]*
   // CHECK:       getelementptr %struct.kmp_depend_info, %struct.kmp_depend_info* %{{.+}}, i[[SZ]] 0
   // CHECK:       getelementptr %struct.kmp_depend_info, %struct.kmp_depend_info* %{{.+}}, i[[SZ]] 1
@@ -195,11 +234,15 @@ int foo(int n) {
 
 // CHECK:       define internal{{.*}} i32 [[TASK_ENTRY1_]](i32{{.*}}, [[TASK_TY1_]]* noalias %1)
 // CHECK:       call void (i8*, ...) %
-// CHECK:       [[SZT:%.+]] = getelementptr inbounds [2 x i64], [2 x i64]* %{{.+}}, i[[SZ]] 0, i[[SZ]] 0
-// CHECK:       [[DEVICE_CAP:%.+]] = getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 2
+// OMP45:       [[SZT:%.+]] = getelementptr inbounds [2 x i64], [2 x i64]* %{{.+}}, i[[SZ]] 0, i[[SZ]] 0
+// OMP50:       [[SZT:%.+]] = getelementptr inbounds [3 x i64], [3 x i64]* %{{.+}}, i[[SZ]] 0, i[[SZ]] 0
+// OMP45:       [[DEVICE_CAP:%.+]] = getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 2
+// OMP50-64:       [[DEVICE_CAP:%.+]] = getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 4
+// OMP50-32:       [[DEVICE_CAP:%.+]] = getelementptr inbounds %{{.+}}, %{{.+}}* %{{.+}}, i32 0, i32 3
 // CHECK:       [[DEV:%.+]] = load i32, i32* [[DEVICE_CAP]],
 // CHECK:       [[DEVICE:%.+]] = sext i32 [[DEV]] to i64
-// CHECK:       [[RET:%.+]] = call i32 @__tgt_target_teams_nowait(i64 [[DEVICE]], i8* @{{[^,]+}}, i32 2, i8** [[BPR:%[^,]+]], i8** [[PR:%[^,]+]], i64* [[SZT]], i64* getelementptr inbounds ([2 x i64], [2 x i64]* [[MAPT]], i32 0, i32 0), i32 0, i32 1)
+// OMP45:       [[RET:%.+]] = call i32 @__tgt_target_teams_nowait(i64 [[DEVICE]], i8* @{{[^,]+}}, i32 2, i8** [[BPR:%[^,]+]], i8** [[PR:%[^,]+]], i64* [[SZT]], i64* getelementptr inbounds ([2 x i64], [2 x i64]* [[MAPT]], i32 0, i32 0), i32 0, i32 1)
+// OMP50:       [[RET:%.+]] = call i32 @__tgt_target_teams_nowait(i64 [[DEVICE]], i8* @{{[^,]+}}, i32 3, i8** [[BPR:%[^,]+]], i8** [[PR:%[^,]+]], i64* [[SZT]], i64* getelementptr inbounds ([3 x i64], [3 x i64]* [[MAPT]], i32 0, i32 0), i32 0, i32 1)
 
 // CHECK:       [[ERROR:%.+]] = icmp ne i32 [[RET]], 0
 // CHECK-NEXT:  br i1 [[ERROR]], label %[[FAIL:[^,]+]], label %[[END:[^,]+]]
@@ -210,7 +253,10 @@ int foo(int n) {
 // CHECK-64:    store i32 [[BP1_I32]], i32* [[BP1_CAST]],
 // CHECK-32:    store i32 [[BP1_I32]], i32* [[BP1_PTR:%.+]],
 // CHECK:       [[BP1:%.+]] = load i[[SZ]], i[[SZ]]* [[BP1_PTR]],
-// CHECK:       call void [[HVT1]](i[[SZ]]* [[BP0]], i[[SZ]] [[BP1]])
+// OMP45:       call void [[HVT1]](i[[SZ]]* [[BP0]], i[[SZ]] [[BP1]])
+
+// OMP50:       [[BP2:%.+]] = load i[[SZ]], i[[SZ]]*
+// OMP50:       call void [[HVT1]](i[[SZ]]* [[BP0]], i[[SZ]] [[BP1]], i[[SZ]] [[BP2]])
 // CHECK-NEXT:  br label %[[END]]
 // CHECK:       [[END]]
 // CHECK:       ret i32 0
@@ -224,7 +270,10 @@ int foo(int n) {
 // CHECK-64:    store i32 [[BP1_I32]], i32* [[BP1_CAST]],
 // CHECK-32:    store i32 [[BP1_I32]], i32* [[BP1_PTR:%.+]],
 // CHECK:       [[BP1:%.+]] = load i[[SZ]], i[[SZ]]* [[BP1_PTR]],
-// CHECK:       call void [[HVT1]](i[[SZ]]* [[BP0]], i[[SZ]] [[BP1]])
+// OMP50:       [[BP2:%.+]] = load i[[SZ]], i[[SZ]]*
+// OMP45:       call void [[HVT1]](i[[SZ]]* [[BP0]], i[[SZ]] [[BP1]])
+// OMP50:       call void [[HVT1]](i[[SZ]]* [[BP0]], i[[SZ]] [[BP1]], i[[SZ]] [[BP2]])
+
 // CHECK:       ret i32 0
 
 // CHECK:       define internal void [[HVT2:@.+]](i[[SZ]] %{{.+}})
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_depend_messages.cpp b/clang/test/OpenMP/target_teams_distribute_simd_depend_messages.cpp
index 8c9332a3e1237..a864e21a6968b 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_depend_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_depend_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
 void foo() {
 }
@@ -25,35 +27,38 @@ int main(int argc, char **argv, char *env[]) {
 
 #pragma omp target teams distribute simd depend // expected-error {{expected '(' after 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute simd depend ( // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target teams distribute simd depend( // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute simd depend () // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target teams distribute simd depend() // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute simd depend (argc // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target teams distribute simd depend(argc // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute simd depend (source : argc) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
-  for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute simd depend (source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp target teams distribute simd depend(source : argc) // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target teams distribute simd depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd depend (in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target teams distribute simd' are ignored}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd depend (out: ) // expected-error {{expected expression}}
-  for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute simd depend (inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target teams distribute simd depend(inout : foobool(argc)), depend(in, argc) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd depend (out :S1) // expected-error {{'S1' does not refer to a value}}
-  for (i = 0; i < argc; ++i) foo();
+  for (i = 0; i < argc; ++i)
+    foo();
 #pragma omp target teams distribute simd depend(in : argv[1][1] = '2')
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute simd depend (in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
+#pragma omp target teams distribute simd depend(in : vec[1]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd depend (in : argv[0])
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute simd depend (in : ) // expected-error {{expected expression}}
+#pragma omp target teams distribute simd depend(in:) // expected-error {{expected expression}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd depend (in : main)
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute simd depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
+#pragma omp target teams distribute simd depend(in : a[0]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd depend (in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
   for (i = 0; i < argc; ++i) foo();
@@ -73,7 +78,7 @@ int main(int argc, char **argv, char *env[]) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd depend (in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute simd depend (in : argv[3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+#pragma omp target teams distribute simd depend(in : argv [3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd depend(in:a[0:1]) // expected-error {{subscripted value is not an array or pointer}}
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_firstprivate_messages.cpp b/clang/test/OpenMP/target_teams_distribute_simd_firstprivate_messages.cpp
index 04f993f051f4e..28fb06e4add9e 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_firstprivate_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_firstprivate_messages.cpp
@@ -1,9 +1,9 @@
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,le50 -fopenmp %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp-version=40 -fopenmp %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp-version=45 -fopenmp %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected -fopenmp-version=50 -fopenmp %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp-simd %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,le50 -fopenmp-simd %s -Wuninitialized
 
 #pragma omp requires dynamic_allocators
 typedef void **omp_allocator_handle_t;
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_is_device_ptr_messages.cpp b/clang/test/OpenMP/target_teams_distribute_simd_is_device_ptr_messages.cpp
index 3de17bf0ceccd..e311cf1a698e0 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_is_device_ptr_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_is_device_ptr_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -std=c++11 -verify -fopenmp %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify=expected,omp4 -fopenmp -fopenmp-version=45 %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify -fopenmp -fopenmp-version=50 %s -Wuninitialized
 
-// RUN: %clang_cc1 -std=c++11 -verify -fopenmp-simd %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify -fopenmp-simd -fopenmp-version=50 %s -Wuninitialized
 struct ST {
   int *a;
 };
@@ -228,13 +230,13 @@ T tmain(T argc) {
 #pragma omp target teams distribute simd is_device_ptr(ps) map(ps->a) // expected-error{{pointer cannot be mapped along with a section derived from itself}} expected-note{{used here}}
   for (int i=0; i<100; i++)
     ;
-#pragma omp target teams distribute simd is_device_ptr(ps) firstprivate(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute simd' directive}}
+#pragma omp target teams distribute simd is_device_ptr(ps) firstprivate(ps) // omp4-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute simd' directive}}
   for (int i=0; i<100; i++)
     ;
 #pragma omp target teams distribute simd firstprivate(ps) is_device_ptr(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute simd' directive}} expected-note{{defined as firstprivate}}
   for (int i=0; i<100; i++)
     ;
-#pragma omp target teams distribute simd is_device_ptr(ps) private(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute simd' directive}}
+#pragma omp target teams distribute simd is_device_ptr(ps) private(ps) // omp4-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute simd' directive}}
   for (int i=0; i<100; i++)
     ;
 #pragma omp target teams distribute simd private(ps) is_device_ptr(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute simd' directive}} expected-note{{defined as private}}
@@ -323,13 +325,13 @@ int main(int argc, char **argv) {
 #pragma omp target teams distribute simd is_device_ptr(ps) map(ps->a) // expected-error{{pointer cannot be mapped along with a section derived from itself}} expected-note{{used here}}
   for (int i=0; i<100; i++)
     ;
-#pragma omp target teams distribute simd is_device_ptr(ps) firstprivate(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute simd' directive}}
+#pragma omp target teams distribute simd is_device_ptr(ps) firstprivate(ps) // omp4-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute simd' directive}}
   for (int i=0; i<100; i++)
     ;
 #pragma omp target teams distribute simd firstprivate(ps) is_device_ptr(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute simd' directive}} expected-note{{defined as firstprivate}}
   for (int i=0; i<100; i++)
     ;
-#pragma omp target teams distribute simd is_device_ptr(ps) private(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute simd' directive}}
+#pragma omp target teams distribute simd is_device_ptr(ps) private(ps) // omp4-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute simd' directive}}
   for (int i=0; i<100; i++)
     ;
 #pragma omp target teams distribute simd private(ps) is_device_ptr(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams distribute simd' directive}} expected-note{{defined as private}}
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_lastprivate_messages.cpp b/clang/test/OpenMP/target_teams_distribute_simd_lastprivate_messages.cpp
index ac132f6b8f9fb..44295325e0d29 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_lastprivate_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_lastprivate_messages.cpp
@@ -1,9 +1,9 @@
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,le50 -fopenmp %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp-version=40 -fopenmp %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp-version=45 -fopenmp %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le50 -fopenmp-version=50 -fopenmp %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp-simd %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,le50 -fopenmp-simd %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le50 -fopenmp-version=50 -fopenmp-simd %s -Wuninitialized
 
 typedef void **omp_allocator_handle_t;
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_loop_messages.cpp b/clang/test/OpenMP/target_teams_distribute_simd_loop_messages.cpp
index 46dd35acfab3c..df0a1393a3735 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_loop_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_loop_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wno-openmp-mapping -Wuninitialized
 
 class S {
   int a;
@@ -93,18 +95,12 @@ int test_iteration_spaces() {
     c[ii] = a[ii];
 
 #pragma omp target teams distribute simd
-// omp4-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', or '>=') of loop variable 'i'}}
-  for (int i = 0; i; i++)
-    c[i] = a[i];
-
-#pragma omp target teams distribute simd
-// omp4-error@+2 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', or '>=') of loop variable 'i'}}
-// expected-error@+1 {{increment clause of OpenMP for loop must perform simple addition or subtraction on loop variable 'i'}}
+// omp4-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', or '>=') of loop variable 'i'}} expected-error@+1 {{increment clause of OpenMP for loop must perform simple addition or subtraction on loop variable 'i'}} omp5-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', '>=', or '!=') of loop variable 'i'}}
   for (int i = 0; jj < kk; ii++)
     c[i] = a[i];
 
 #pragma omp target teams distribute simd
-// omp4-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', or '>=') of loop variable 'i'}}
+// omp4-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', or '>=') of loop variable 'i'}} omp5-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', '>=', or '!=') of loop variable 'i'}}
   for (int i = 0; !!i; i++)
     c[i] = a[i];
 
@@ -114,7 +110,7 @@ int test_iteration_spaces() {
     c[i] = a[i];
 
 #pragma omp target teams distribute simd
-// omp4-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', or '>=') of loop variable 'i'}}
+// omp4-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', or '>=') of loop variable 'i'}} omp5-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', '>=', or '!=') of loop variable 'i'}}
   for (int i = 0;; i++)
     c[i] = a[i];
 
@@ -244,13 +240,13 @@ int test_iteration_spaces() {
   for (ii = 0; ii < 10; ii++)
     c[ii] = a[ii];
 
-#pragma omp target teams distribute simd private(ii) // expected-note {{defined as private}}
-// expected-error@+1 {{loop iteration variable in the associated loop of 'omp target teams distribute simd' directive may not be private, predetermined as linear}}
+#pragma omp target teams distribute simd private(ii) // omp4-note {{defined as private}}
+// omp4-error@+1 {{loop iteration variable in the associated loop of 'omp target teams distribute simd' directive may not be private, predetermined as linear}}
   for (ii = 0; ii < 10; ii++)
     c[ii] = a[ii];
 
-#pragma omp target teams distribute simd lastprivate(ii) // expected-note {{defined as lastprivate}}
-// expected-error@+1 {{loop iteration variable in the associated loop of 'omp target teams distribute simd' directive may not be lastprivate, predetermined as linear}}
+#pragma omp target teams distribute simd lastprivate(ii) // omp4-note {{defined as lastprivate}}
+// omp4-error@+1 {{loop iteration variable in the associated loop of 'omp target teams distribute simd' directive may not be lastprivate, predetermined as linear}}
   for (ii = 0; ii < 10; ii++)
     c[ii] = a[ii];
 
@@ -267,7 +263,7 @@ int test_iteration_spaces() {
   }
 
 #pragma omp target teams distribute simd
-// expected-error@+1 {{statement after '#pragma omp target teams distribute simd' must be a for loop}}
+// omp4-error@+1 {{statement after '#pragma omp target teams distribute simd' must be a for loop}}
   for (auto &item : a) {
     item = item + 1;
   }
@@ -403,15 +399,15 @@ int test_with_random_access_iterator() {
   for (begin = end; begin < end; ++begin)
     ++begin;
 #pragma omp target teams distribute simd
-// omp4-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', or '>=') of loop variable 'I'}}
+// omp4-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', or '>=') of loop variable 'I'}} omp5-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', '>=', or '!=') of loop variable 'I'}}
   for (GoodIter I = begin; I - I; ++I)
     ++I;
 #pragma omp target teams distribute simd
-// omp4-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', or '>=') of loop variable 'I'}}
+// omp4-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', or '>=') of loop variable 'I'}} omp5-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', '>=', or '!=') of loop variable 'I'}}
   for (GoodIter I = begin; begin < end; ++I)
     ++I;
 #pragma omp target teams distribute simd
-// omp4-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', or '>=') of loop variable 'I'}}
+// omp4-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', or '>=') of loop variable 'I'}} omp5-error@+1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', '>=', or '!=') of loop variable 'I'}}
   for (GoodIter I = begin; !I; ++I)
     ++I;
 #pragma omp target teams distribute simd
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_map_messages.cpp b/clang/test/OpenMP/target_teams_distribute_simd_map_messages.cpp
index 71d2ee68b1d94..3bf612b7637f0 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_map_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_map_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
@@ -103,7 +105,7 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd map(T) // expected-error {{'T' does not refer to a value}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute simd map(I) // expected-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams distribute simd map(I) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd map(S2::S2s)
   for (i = 0; i < argc; ++i) foo();
@@ -121,7 +123,8 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute simd map(tofrom: argc > 0 ? x : y) // expected-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} 
+#pragma omp target teams distribute simd map(tofrom \
+                                             : argc > 0 ? x : y) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -228,8 +231,10 @@ int main(int argc, char **argv) {
 #pragma omp target teams distribute simd map(to, x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
-  for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute simd map(tofrom: argc > 0 ? argv[1] : argv[2]) // expected-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+  for (i = 0; i < argc; ++i)
+    foo();
+#pragma omp target teams distribute simd map(tofrom \
+                                             : argc > 0 ? argv[1] : argv[2]) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd map(argc)
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_private_messages.cpp b/clang/test/OpenMP/target_teams_distribute_simd_private_messages.cpp
index fa57ef93b4a74..47d5a8fa6773a 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_private_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_private_messages.cpp
@@ -1,9 +1,9 @@
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,le50 -fopenmp %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp-version=40 -fopenmp %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp-version=45 -fopenmp %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected -fopenmp-version=50 -fopenmp %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp-simd %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,le50 -fopenmp-simd %s -Wuninitialized
 // RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 %s -Wuninitialized
 
 typedef void **omp_allocator_handle_t;
diff --git a/clang/test/OpenMP/target_teams_is_device_ptr_messages.cpp b/clang/test/OpenMP/target_teams_is_device_ptr_messages.cpp
index 1885d1e6994bd..d662390cdf1f0 100644
--- a/clang/test/OpenMP/target_teams_is_device_ptr_messages.cpp
+++ b/clang/test/OpenMP/target_teams_is_device_ptr_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -std=c++11 -verify -fopenmp -ferror-limit 200 %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 200 %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify -fopenmp -fopenmp-version=50 -ferror-limit 200 %s -Wuninitialized
 
-// RUN: %clang_cc1 -std=c++11 -verify -fopenmp-simd -ferror-limit 200 %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 200 %s -Wuninitialized
+// RUN: %clang_cc1 -std=c++11 -verify -fopenmp-simd -fopenmp-version=50 -ferror-limit 200 %s -Wuninitialized
 struct ST {
   int *a;
 };
@@ -189,11 +191,11 @@ T tmain(T argc) {
   {}
 #pragma omp target teams is_device_ptr(ps) map(ps->a) // expected-error{{pointer cannot be mapped along with a section derived from itself}} expected-note{{used here}}
   {}
-#pragma omp target teams is_device_ptr(ps) firstprivate(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams' directive}}
+#pragma omp target teams is_device_ptr(ps) firstprivate(ps) // omp4-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams' directive}}
   {}
 #pragma omp target teams firstprivate(ps) is_device_ptr(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams' directive}} expected-note{{defined as firstprivate}}
   {}
-#pragma omp target teams is_device_ptr(ps) private(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams' directive}}
+#pragma omp target teams is_device_ptr(ps) private(ps) // omp4-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams' directive}}
   {}
 #pragma omp target teams private(ps) is_device_ptr(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams' directive}} expected-note{{defined as private}}
   {}
@@ -258,11 +260,11 @@ int main(int argc, char **argv) {
   {}
 #pragma omp target teams is_device_ptr(ps) map(ps->a) // expected-error{{pointer cannot be mapped along with a section derived from itself}} expected-note{{used here}}
   {}
-#pragma omp target teams is_device_ptr(ps) firstprivate(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams' directive}}
+#pragma omp target teams is_device_ptr(ps) firstprivate(ps) // omp4-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams' directive}}
   {}
 #pragma omp target teams firstprivate(ps) is_device_ptr(ps) // expected-error{{firstprivate variable cannot be in a is_device_ptr clause in '#pragma omp target teams' directive}} expected-note{{defined as firstprivate}}
   {}
-#pragma omp target teams is_device_ptr(ps) private(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams' directive}}
+#pragma omp target teams is_device_ptr(ps) private(ps) // omp4-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams' directive}}
   {}
 #pragma omp target teams private(ps) is_device_ptr(ps) // expected-error{{private variable cannot be in a is_device_ptr clause in '#pragma omp target teams' directive}} expected-note{{defined as private}}
   {}
diff --git a/clang/test/OpenMP/target_teams_map_messages.cpp b/clang/test/OpenMP/target_teams_map_messages.cpp
index 11115d501912a..7828099d3e41e 100644
--- a/clang/test/OpenMP/target_teams_map_messages.cpp
+++ b/clang/test/OpenMP/target_teams_map_messages.cpp
@@ -1,9 +1,9 @@
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,le50 -fopenmp -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp-version=40 -fopenmp -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp-version=45 -fopenmp -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le50 -fopenmp-version=50 -fopenmp -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp-simd -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,le50 -fopenmp-simd -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -DCCODE -verify=expected,le45 -fopenmp -ferror-limit 200 -x c %s -Wno-openmp-mapping -Wuninitialized
 #ifdef CCODE
 void foo(int arg) {
diff --git a/clang/test/OpenMP/target_update_ast_print.cpp b/clang/test/OpenMP/target_update_ast_print.cpp
index fb6440b87cea3..0111432fde8f6 100644
--- a/clang/test/OpenMP/target_update_ast_print.cpp
+++ b/clang/test/OpenMP/target_update_ast_print.cpp
@@ -20,6 +20,11 @@ T foo(T targ, U uarg) {
 #pragma omp target update to(([a][targ])p, a) if(l>5) device(l) nowait depend(inout:l)
 
 #pragma omp target update from(b, ([a][targ])p) if(l<5) device(l-1) nowait depend(inout:l)
+
+  int arr[100][100];
+#pragma omp target update to(arr[2][0:1:2])
+
+#pragma omp target update from(arr[2][0:1:2])
   return a + targ + (T)b;
 }
 // CHECK:      static T a, *p;
@@ -37,6 +42,9 @@ T foo(T targ, U uarg) {
 // CHECK-NEXT: int l;
 // CHECK-NEXT: #pragma omp target update to(([a][targ])p,a) if(l > 5) device(l) nowait depend(inout : l)
 // CHECK-NEXT: #pragma omp target update from(b,([a][targ])p) if(l < 5) device(l - 1) nowait depend(inout : l)
+// CHECK:      int arr[100][100];
+// CHECK-NEXT: #pragma omp target update to(arr[2][0:1:2])
+// CHECK-NEXT: #pragma omp target update from(arr[2][0:1:2])
 
 int main(int argc, char **argv) {
   static int a;
@@ -50,6 +58,11 @@ int main(int argc, char **argv) {
 // CHECK-NEXT: #pragma omp target update to(a) if(f > 0.) device(n) nowait depend(in : n)
 #pragma omp target update from(f) if(f<0.0) device(n+1) nowait depend(in:n)
 // CHECK-NEXT: #pragma omp target update from(f) if(f < 0.) device(n + 1) nowait depend(in : n)
+#pragma omp target update to(argv[2][0:1:2])
+// CHECK-NEXT: #pragma omp target update to(argv[2][0:1:2])
+#pragma omp target update from(argv[2][0:1:2])
+// CHECK-NEXT: #pragma omp target update from(argv[2][0:1:2])
+
   return foo(argc, f) + foo(argv[0][0], f) + a;
 }
 
diff --git a/clang/test/OpenMP/target_update_depend_messages.cpp b/clang/test/OpenMP/target_update_depend_messages.cpp
index b570f0c8672bc..b218d4d21889e 100644
--- a/clang/test/OpenMP/target_update_depend_messages.cpp
+++ b/clang/test/OpenMP/target_update_depend_messages.cpp
@@ -1,7 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
-
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 void foo() {
 }
 
@@ -29,37 +30,37 @@ int tmain(T argc, S **argv, R *env[]) {
   {}
 
   #pragma omp target update to(z) depend // expected-error {{expected '(' after 'depend'}}
-  #pragma omp target update to(z) depend( // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
-  #pragma omp target update to(z) depend() // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
-  #pragma omp target update to(z) depend(argc // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
-  #pragma omp target update to(z) depend(source : argc) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
-  #pragma omp target update to(z) depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
-  #pragma omp target update to(z) depend(in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target update' are ignored}}
-  #pragma omp target update to(z) depend(out: ) // expected-error {{expected expression}}
-  #pragma omp target update to(z) depend(inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
-  #pragma omp target update to(z) depend(out :S1) // expected-error {{'S1' does not refer to a value}}
-  #pragma omp target update to(z) depend(in : argv[1][1] = '2') // expected-error {{expected addressable lvalue expression, array element or array section}}
-  #pragma omp target update to(z) depend(in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
-  #pragma omp target update to(z) depend(in : argv[0])
-  #pragma omp target update to(z) depend(in : ) // expected-error {{expected expression}}
-  #pragma omp target update to(z) depend(in : tmain)
-  #pragma omp target update to(z) depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
-  #pragma omp target update to(z) depend(in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
-  #pragma omp target update to(z) depend(in : argv[ // expected-error {{expected expression}} expected-error {{expected ']'}} expected-error {{expected ')'}} expected-note {{to match this '['}} expected-note {{to match this '('}}
-  #pragma omp target update to(z) depend(in : argv[: // expected-error {{expected expression}} expected-error {{expected ']'}} expected-error {{expected ')'}} expected-note {{to match this '['}} expected-note {{to match this '('}}
-  #pragma omp target update to(z) depend(in : argv[:] // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}} expected-error {{expected ')'}} expected-note {{to match this '('}}
-  #pragma omp target update to(z) depend(in : argv[argc: // expected-error {{expected expression}} expected-error {{expected ']'}} expected-error {{expected ')'}} expected-note {{to match this '['}} expected-note {{to match this '('}}
-  #pragma omp target update to(z) depend(in : argv[argc:argc] // expected-error {{expected ')'}} expected-note {{to match this '('}}
-  #pragma omp target update to(z) depend(in : argv[0:-1]) // expected-error {{section length is evaluated to a negative value -1}}
-  #pragma omp target update to(z) depend(in : argv[-1:0]) // expected-error {{zero-length array section is not allowed in 'depend' clause}}
-  #pragma omp target update to(z) depend(in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
-  #pragma omp target update to(z) depend(in : argv[3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
-  #pragma omp target update to(z) depend(in:a[0:1]) // expected-error {{subscripted value is not an array or pointer}}
-  #pragma omp target update to(z) depend(in:argv[argv[:2]:1]) // expected-error {{OpenMP array section is not allowed here}}
-  #pragma omp target update to(z) depend(in:argv[0:][:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
-  #pragma omp target update to(z) depend(in:env[0:][:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is an array of unknown bound}}
-  #pragma omp target update to(z) depend(in : argv[ : argc][1 : argc - 1])
-  #pragma omp target update to(z) depend(in : arr[0])
+#pragma omp target update to(z) depend(  // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target update to(z) depend() // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target update to(z) depend(argc // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target update to(z) depend(source : argc) // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+#pragma omp target update to(z) depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+#pragma omp target update to(z) depend(in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target update' are ignored}}
+#pragma omp target update to(z) depend(out:)    // expected-error {{expected expression}}
+#pragma omp target update to(z) depend(inout : foobool(argc)), depend(in, argc) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
+#pragma omp target update to(z) depend(out : S1) // expected-error {{'S1' does not refer to a value}}
+#pragma omp target update to(z) depend(in : argv[1][1] = '2') // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
+#pragma omp target update to(z) depend(in : vec[1]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
+#pragma omp target update to(z) depend(in : argv[0])
+#pragma omp target update to(z) depend(in:) // expected-error {{expected expression}}
+#pragma omp target update to(z) depend(in : tmain)
+#pragma omp target update to(z) depend(in : a[0]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
+#pragma omp target update to(z) depend(in : vec [1:2]) // expected-error {{ value is not an array or pointer}}
+#pragma omp target update to(z) depend(in : argv[ // expected-error {{expected expression}} expected-error {{expected ']'}} expected-error {{expected ')'}} expected-note {{to match this '['}} expected-note {{to match this '('}}
+#pragma omp target update to(z) depend(in : argv[: // expected-error {{expected expression}} expected-error {{expected ']'}} expected-error {{expected ')'}} expected-note {{to match this '['}} expected-note {{to match this '('}}
+#pragma omp target update to(z) depend(in : argv[:] // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target update to(z) depend(in : argv [argc: // expected-error {{expected expression}} expected-error {{expected ']'}} expected-error {{expected ')'}} expected-note {{to match this '['}} expected-note {{to match this '('}}
+#pragma omp target update to(z) depend(in : argv [argc:argc] // expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target update to(z) depend(in : argv [0:-1]) // expected-error {{section length is evaluated to a negative value -1}}
+#pragma omp target update to(z) depend(in : argv [-1:0]) // expected-error {{zero-length array section is not allowed in 'depend' clause}}
+#pragma omp target update to(z) depend(in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
+#pragma omp target update to(z) depend(in : argv [3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+#pragma omp target update to(z) depend(in : a [0:1]) // expected-error {{subscripted value is not an array or pointer}}
+#pragma omp target update to(z) depend(in : argv [argv[:2]:1]) // expected-error {{OpenMP array section is not allowed here}}
+#pragma omp target update to(z) depend(in : argv [0:][:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
+#pragma omp target update to(z) depend(in : env [0:][:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is an array of unknown bound}}
+#pragma omp target update to(z) depend(in : argv[:argc] [1:argc - 1])
+#pragma omp target update to(z) depend(in : arr[0])
 
   return 0;
 }
@@ -76,36 +77,36 @@ int main(int argc, char **argv, char *env[]) {
   {}
 
   #pragma omp target update to(z) depend // expected-error {{expected '(' after 'depend'}}
-  #pragma omp target update to(z) depend( // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
-  #pragma omp target update to(z) depend() // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
-  #pragma omp target update to(z) depend(argc // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
-  #pragma omp target update to(z) depend(source : argc) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
-  #pragma omp target update to(z) depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
-  #pragma omp target update to(z) depend(in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target update' are ignored}}
-  #pragma omp target update to(z) depend(out: ) // expected-error {{expected expression}}
-  #pragma omp target update to(z) depend(inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
-  #pragma omp target update to(z) depend(out :S1) // expected-error {{'S1' does not refer to a value}}
-  #pragma omp target update to(z) depend(in : argv[1][1] = '2')
-  #pragma omp target update to(z) depend(in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
-  #pragma omp target update to(z) depend(in : argv[0])
-  #pragma omp target update to(z) depend(in : ) // expected-error {{expected expression}}
-  #pragma omp target update to(z) depend(in : main)
-  #pragma omp target update to(z) depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
-  #pragma omp target update to(z) depend(in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
-  #pragma omp target update to(z) depend(in : argv[ // expected-error {{expected expression}} expected-error {{expected ']'}} expected-error {{expected ')'}} expected-note {{to match this '['}} expected-note {{to match this '('}}
-  #pragma omp target update to(z) depend(in : argv[: // expected-error {{expected expression}} expected-error {{expected ']'}} expected-error {{expected ')'}} expected-note {{to match this '['}} expected-note {{to match this '('}}
-  #pragma omp target update to(z) depend(in : argv[:] // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}} expected-error {{expected ')'}} expected-note {{to match this '('}}
-  #pragma omp target update to(z) depend(in : argv[argc: // expected-error {{expected expression}} expected-error {{expected ']'}} expected-error {{expected ')'}} expected-note {{to match this '['}} expected-note {{to match this '('}}
-  #pragma omp target update to(z) depend(in : argv[argc:argc] // expected-error {{expected ')'}} expected-note {{to match this '('}}
-  #pragma omp target update to(z) depend(in : argv[0:-1]) // expected-error {{section length is evaluated to a negative value -1}}
-  #pragma omp target update to(z) depend(in : argv[-1:0]) // expected-error {{zero-length array section is not allowed in 'depend' clause}}
-  #pragma omp target update to(z) depend(in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
-  #pragma omp target update to(z) depend(in : argv[3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
-  #pragma omp target update to(z) depend(in:a[0:1]) // expected-error {{subscripted value is not an array or pointer}}
-  #pragma omp target update to(z) depend(in:argv[argv[:2]:1]) // expected-error {{OpenMP array section is not allowed here}}
-  #pragma omp target update to(z) depend(in:argv[0:][:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
-  #pragma omp target update to(z) depend(in:env[0:][:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is an array of unknown bound}}
-  #pragma omp target update to(z) depend(in : argv[ : argc][1 : argc - 1])
-  #pragma omp target update to(z) depend(in : arr[0])
+#pragma omp target update to(z) depend(  // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target update to(z) depend() // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}}
+#pragma omp target update to(z) depend(argc // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target update to(z) depend(source : argc) // omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+#pragma omp target update to(z) depend(source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp4-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp5-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+#pragma omp target update to(z) depend(in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp target update' are ignored}}
+#pragma omp target update to(z) depend(out:)    // expected-error {{expected expression}}
+#pragma omp target update to(z) depend(inout : foobool(argc)), depend(in, argc) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
+#pragma omp target update to(z) depend(out : S1) // expected-error {{'S1' does not refer to a value}}
+#pragma omp target update to(z) depend(in : argv[1][1] = '2')
+#pragma omp target update to(z) depend(in : vec[1]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
+#pragma omp target update to(z) depend(in : argv[0])
+#pragma omp target update to(z) depend(in:) // expected-error {{expected expression}}
+#pragma omp target update to(z) depend(in : main)
+#pragma omp target update to(z) depend(in : a[0]) // omp4-error {{expected addressable lvalue expression, array element or array section}} omp5-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
+#pragma omp target update to(z) depend(in : vec [1:2]) // expected-error {{ value is not an array or pointer}}
+#pragma omp target update to(z) depend(in : argv[ // expected-error {{expected expression}} expected-error {{expected ']'}} expected-error {{expected ')'}} expected-note {{to match this '['}} expected-note {{to match this '('}}
+#pragma omp target update to(z) depend(in : argv[: // expected-error {{expected expression}} expected-error {{expected ']'}} expected-error {{expected ')'}} expected-note {{to match this '['}} expected-note {{to match this '('}}
+#pragma omp target update to(z) depend(in : argv[:] // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target update to(z) depend(in : argv [argc: // expected-error {{expected expression}} expected-error {{expected ']'}} expected-error {{expected ')'}} expected-note {{to match this '['}} expected-note {{to match this '('}}
+#pragma omp target update to(z) depend(in : argv [argc:argc] // expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp target update to(z) depend(in : argv [0:-1]) // expected-error {{section length is evaluated to a negative value -1}}
+#pragma omp target update to(z) depend(in : argv [-1:0]) // expected-error {{zero-length array section is not allowed in 'depend' clause}}
+#pragma omp target update to(z) depend(in : argv[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
+#pragma omp target update to(z) depend(in : argv [3:4:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+#pragma omp target update to(z) depend(in : a [0:1]) // expected-error {{subscripted value is not an array or pointer}}
+#pragma omp target update to(z) depend(in : argv [argv[:2]:1]) // expected-error {{OpenMP array section is not allowed here}}
+#pragma omp target update to(z) depend(in : argv [0:][:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
+#pragma omp target update to(z) depend(in : env [0:][:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is an array of unknown bound}}
+#pragma omp target update to(z) depend(in : argv[:argc] [1:argc - 1])
+#pragma omp target update to(z) depend(in : arr[0])
   return tmain(argc, argv, env); // expected-note {{in instantiation of function template specialization 'tmain<int, char, char>' requested here}}
 }
diff --git a/clang/test/OpenMP/target_update_from_messages.cpp b/clang/test/OpenMP/target_update_from_messages.cpp
index 141cfc38ffeb1..3dc377c4ca4ce 100644
--- a/clang/test/OpenMP/target_update_from_messages.cpp
+++ b/clang/test/OpenMP/target_update_from_messages.cpp
@@ -1,9 +1,9 @@
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,le50 -fopenmp -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp -fopenmp-version=40 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le50 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp-simd -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,le50 -fopenmp-simd -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
diff --git a/clang/test/OpenMP/target_update_messages.cpp b/clang/test/OpenMP/target_update_messages.cpp
index 581fdd8e0a9bd..4092b623bdc97 100644
--- a/clang/test/OpenMP/target_update_messages.cpp
+++ b/clang/test/OpenMP/target_update_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp50 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp50 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
 void xxx(int argc) {
   int x; // expected-note {{initialize the variable 'x' to silence this warning}}
@@ -36,5 +38,11 @@ int main(int argc, char **argv) {
   {
     foo();
   }
+
+  int iarr[5][5];
+#pragma omp target update to(iarr[0:][1:2:-1]) // omp50-error {{section stride is evaluated to a non-positive value -1}} omp45-error {{expected ']'}} omp45-note {{to match this '['}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  {}
+#pragma omp target update from(iarr[0:][1:2:-1]) // omp50-error {{section stride is evaluated to a non-positive value -1}} omp45-error {{expected ']'}} omp45-note {{to match this '['}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+
   return tmain(argc, argv);
 }
diff --git a/clang/test/OpenMP/target_update_to_messages.cpp b/clang/test/OpenMP/target_update_to_messages.cpp
index 832adc0dd4d09..5b840c056115e 100644
--- a/clang/test/OpenMP/target_update_to_messages.cpp
+++ b/clang/test/OpenMP/target_update_to_messages.cpp
@@ -1,9 +1,9 @@
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,le50 -fopenmp -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp -fopenmp-version=40 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le50 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp-simd -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,le50 -fopenmp-simd -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp-simd -fopenmp-version=40 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le45 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,le50 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
diff --git a/clang/test/OpenMP/task_default_messages.cpp b/clang/test/OpenMP/task_default_messages.cpp
index 4826c253aa043..8b6809ee05d56 100644
--- a/clang/test/OpenMP/task_default_messages.cpp
+++ b/clang/test/OpenMP/task_default_messages.cpp
@@ -2,15 +2,24 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 -o - %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp -ferror-limit 100 -o - %s -Wuninitialized
+
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp-simd -ferror-limit 100 -o - %s -Wuninitialized
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
 #pragma omp task default                          // expected-error {{expected '(' after 'default'}}
-#pragma omp task default(                         // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
-#pragma omp task default()                        // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp task default(                         // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp task default()                        // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
 #pragma omp task default(none                     // expected-error {{expected ')'}} expected-note {{to match this '('}}
 #pragma omp task default(shared), default(shared) // expected-error {{directive '#pragma omp task' cannot contain more than one 'default' clause}}
-#pragma omp task default(x)                       // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp task default(x)                       // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   foo();
 
 #pragma omp task default(none) // expected-note {{explicit data sharing attribute requested here}}
@@ -19,5 +28,13 @@ int main(int argc, char **argv) {
 #pragma omp task default(none) // expected-note {{explicit data sharing attribute requested here}}
 #pragma omp task default(shared)
   ++argc; // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
+
+#ifdef OMP51
+#pragma omp task default(firstprivate) // expected-note 2 {{explicit data sharing attribute requested here}}
+  {
+    ++x; // expected-error {{variable 'x' must have explicitly specified data sharing attributes}}
+    ++y; // expected-error {{variable 'y' must have explicitly specified data sharing attributes}}
+  }
+#endif
   return 0;
 }
diff --git a/clang/test/OpenMP/task_messages.cpp b/clang/test/OpenMP/task_messages.cpp
index 8b3183e0bd93e..13cbfb6c45693 100644
--- a/clang/test/OpenMP/task_messages.cpp
+++ b/clang/test/OpenMP/task_messages.cpp
@@ -4,6 +4,9 @@
 // RUN: %clang_cc1 -verify=expected,omp45 -fopenmp-version=45 -fopenmp-simd -ferror-limit 200 -std=c++11 -o - %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,omp50 -fopenmp-version=50 -fopenmp-simd -ferror-limit 200 -std=c++11 -o - %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify=expected,omp50 -fopenmp-version=51 -DOMP51 -fopenmp -ferror-limit 100 -std=c++11 -o - %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp50 -fopenmp-version=51 -DOMP51 -fopenmp-simd -ferror-limit 100 -std=c++11 -o - %s -Wuninitialized
+
 void xxx(int argc) {
   int x; // expected-note {{initialize the variable 'x' to silence this warning}}
 #pragma omp task
@@ -16,6 +19,10 @@ void foo() {
 }
 
 typedef unsigned long omp_event_handle_t;
+namespace {
+static int y = 0;
+}
+static int x = 0;
 
 #pragma omp task // expected-error {{unexpected OpenMP directive '#pragma omp task'}}
 
@@ -52,6 +59,15 @@ int foo() {
 #pragma omp task default(none) // expected-note 2 {{explicit data sharing attribute requested here}}
 #pragma omp task default(shared)
   ++a; // expected-error 2 {{variable 'a' must have explicitly specified data sharing attributes}}
+#ifdef OMP51
+#pragma omp task default(firstprivate) // expected-note 4 {{explicit data sharing attribute requested here}}
+#pragma omp task
+  {
+    ++x; // expected-error 2 {{variable 'x' must have explicitly specified data sharing attributes}}
+    ++y; // expected-error 2 {{variable 'y' must have explicitly specified data sharing attributes}}
+  }
+#endif
+
 #pragma omp task default(none) // expected-note 2 {{explicit data sharing attribute requested here}}
 #pragma omp task
   // expected-error@+1 {{calling a private constructor of class 'S'}}
diff --git a/clang/test/OpenMP/taskloop_loop_messages.cpp b/clang/test/OpenMP/taskloop_loop_messages.cpp
index 1783a5265d9f5..b3b24e96abc9d 100644
--- a/clang/test/OpenMP/taskloop_loop_messages.cpp
+++ b/clang/test/OpenMP/taskloop_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/taskloop_simd_loop_messages.cpp b/clang/test/OpenMP/taskloop_simd_loop_messages.cpp
index 9a345d6a6300c..b782525a3bbf0 100644
--- a/clang/test/OpenMP/taskloop_simd_loop_messages.cpp
+++ b/clang/test/OpenMP/taskloop_simd_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/teams_default_messages.cpp b/clang/test/OpenMP/teams_default_messages.cpp
index a025050406000..b117ef4948a0f 100644
--- a/clang/test/OpenMP/teams_default_messages.cpp
+++ b/clang/test/OpenMP/teams_default_messages.cpp
@@ -2,17 +2,26 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -o - %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp -o - %s -Wuninitialized
+
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp-simd -o - %s -Wuninitialized
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
   #pragma omp target
   #pragma omp teams default // expected-error {{expected '(' after 'default'}}
   foo();
   #pragma omp target
-  #pragma omp teams default ( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp teams default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   foo();
   #pragma omp target
-  #pragma omp teams default () // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp teams default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   foo();
   #pragma omp target
   #pragma omp teams default (none // expected-error {{expected ')'}} expected-note {{to match this '('}}
@@ -21,7 +30,7 @@ int main(int argc, char **argv) {
   #pragma omp teams default (shared), default(shared) // expected-error {{directive '#pragma omp teams' cannot contain more than one 'default' clause}}
   foo();
   #pragma omp target
-  #pragma omp teams default (x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp teams default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   foo();
 
   #pragma omp target
@@ -32,5 +41,14 @@ int main(int argc, char **argv) {
   #pragma omp teams default(none) // expected-note {{explicit data sharing attribute requested here}}
   #pragma omp parallel default(shared)
   ++argc; // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
+
+#ifdef OMP51
+#pragma omp target
+#pragma omp teams default(firstprivate) // expected-note 2 {{explicit data sharing attribute requested here}}
+  {
+    ++x; // expected-error {{variable 'x' must have explicitly specified data sharing attributes}}
+    ++y; // expected-error {{variable 'y' must have explicitly specified data sharing attributes}}
+  }
+#endif
   return 0;
 }
diff --git a/clang/test/OpenMP/teams_distribute_default_messages.cpp b/clang/test/OpenMP/teams_distribute_default_messages.cpp
index 7f000208303b7..1d5fd40c53a6b 100644
--- a/clang/test/OpenMP/teams_distribute_default_messages.cpp
+++ b/clang/test/OpenMP/teams_distribute_default_messages.cpp
@@ -2,17 +2,26 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp %s -Wuninitialized
+
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp-simd %s -Wuninitialized
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
   #pragma omp target
   #pragma omp teams distribute default // expected-error {{expected '(' after 'default'}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target
-  #pragma omp teams distribute default ( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp teams distribute default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target
-  #pragma omp teams distribute default () // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp teams distribute default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target
   #pragma omp teams distribute default (none // expected-error {{expected ')'}} expected-note {{to match this '('}}
@@ -21,12 +30,21 @@ int main(int argc, char **argv) {
   #pragma omp teams distribute default (shared), default(shared) // expected-error {{directive '#pragma omp teams distribute' cannot contain more than one 'default' clause}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target
-  #pragma omp teams distribute default (x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp teams distribute default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (int i=0; i<200; i++) foo();
 
   #pragma omp target
   #pragma omp teams distribute default(none) // expected-note {{explicit data sharing attribute requested here}}
   for (int i=0; i<200; i++) ++argc; // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
 
+#ifdef OMP51
+#pragma omp target
+#pragma omp teams distribute default(firstprivate) // expected-note 2 {{explicit data sharing attribute requested here}}
+  for (int i = 0; i < 200; i++) {
+    ++x; // expected-error {{variable 'x' must have explicitly specified data sharing attributes}}
+    ++y; // expected-error {{variable 'y' must have explicitly specified data sharing attributes}}
+  }
+#endif
+
   return 0;
 }
diff --git a/clang/test/OpenMP/teams_distribute_loop_messages.cpp b/clang/test/OpenMP/teams_distribute_loop_messages.cpp
index 1edc926c6ce14..36f3830829c5e 100644
--- a/clang/test/OpenMP/teams_distribute_loop_messages.cpp
+++ b/clang/test/OpenMP/teams_distribute_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_default_messages.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_default_messages.cpp
index 2c46623985070..3a414543be806 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_default_messages.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_default_messages.cpp
@@ -2,17 +2,26 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp %s -Wuninitialized
+
+// RUN: %clang_cc1 -verify -fopenmp-version=51 -DOMP51 -fopenmp-simd %s -Wuninitialized
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
   #pragma omp target
   #pragma omp teams distribute parallel for default // expected-error {{expected '(' after 'default'}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target
-  #pragma omp teams distribute parallel for default ( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp teams distribute parallel for default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target
-  #pragma omp teams distribute parallel for default () // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp teams distribute parallel for default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target
   #pragma omp teams distribute parallel for default (none // expected-error {{expected ')'}} expected-note {{to match this '('}}
@@ -21,12 +30,21 @@ int main(int argc, char **argv) {
   #pragma omp teams distribute parallel for default (shared), default(shared) // expected-error {{directive '#pragma omp teams distribute parallel for' cannot contain more than one 'default' clause}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target
-  #pragma omp teams distribute parallel for default (x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp teams distribute parallel for default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (int i=0; i<200; i++) foo();
 
   #pragma omp target
   #pragma omp teams distribute parallel for default(none) // expected-note {{explicit data sharing attribute requested here}}
   for (int i=0; i<200; i++) ++argc; // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
 
+#ifdef OMP51
+#pragma omp target
+#pragma omp teams distribute parallel for default(firstprivate) // expected-note 2 {{explicit data sharing attribute requested here}}
+  for (int i = 0; i < 200; i++) {
+    ++x; // expected-error {{variable 'x' must have explicitly specified data sharing attributes}}
+    ++y; // expected-error {{variable 'y' must have explicitly specified data sharing attributes}}
+  }
+#endif
+
   return 0;
 }
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_loop_messages.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_loop_messages.cpp
index 5916bedea529b..237addabb50fc 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_loop_messages.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_schedule_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_schedule_codegen.cpp
index f7b11a7191940..f095296dab5c5 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_schedule_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_schedule_codegen.cpp
@@ -3,12 +3,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64 --check-prefix CK1-OMP50
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64 --check-prefix CK1-OMP50
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32 --check-prefix CK1-OMP50
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32 --check-prefix CK1-OMP50
+
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64 --check-prefix CK1-OMP45
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64 --check-prefix CK1-OMP45
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32 --check-prefix CK1-OMP45
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32 --check-prefix CK1-OMP45
 
 // RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
@@ -120,7 +127,8 @@ struct SS{
   // CK1: ret void
 
   // CK1: define internal void @[[PAR_OUTL4]]({{.+}})
-  // CK1: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+  // CK1-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+  // CK1-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
   // CK1: call {{.+}} @__kmpc_dispatch_next_4(
   // CK1: ret void
 
@@ -135,7 +143,8 @@ struct SS{
   // CK1: ret void
 
   // CK1: define internal void @[[PAR_OUTL5]]({{.+}})
-  // CK1: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+  // CK1-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+  // CK1-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
   // CK1: call {{.+}} @__kmpc_dispatch_next_4(
   // CK1: ret void
 
@@ -151,12 +160,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64 --check-prefix CK2-OMP50
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64 --check-prefix CK2-OMP50
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32  --check-prefix CK2-OMP50
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32 --check-prefix CK2-OMP50
+
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64 --check-prefix CK2-OMP45
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64 --check-prefix CK2-OMP45
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32  --check-prefix CK2-OMP45
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32 --check-prefix CK2-OMP45
 
 // RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
@@ -301,7 +317,8 @@ int main (int argc, char **argv) {
 // CK2: ret void
 
 // CK2: define internal void @[[PAR_OUTL4]]({{.+}})
-// CK2: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
 // CK2: call {{.+}} @__kmpc_dispatch_next_4(
 // CK2: ret void
 
@@ -317,7 +334,8 @@ int main (int argc, char **argv) {
 // CK2: ret void
 
 // CK2: define internal void @[[PAR_OUTL5]]({{.+}})
-// CK2: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
 // CK2: call {{.+}} @__kmpc_dispatch_next_4(
 // CK2: ret void
 
@@ -391,7 +409,8 @@ int main (int argc, char **argv) {
 // CK2: ret void
 
 // CK2: define internal void @[[PAR_OUTLT4]]({{.+}})
-// CK2: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
 // CK2: call {{.+}} @__kmpc_dispatch_next_4(
 // CK2: ret void
 
@@ -406,7 +425,8 @@ int main (int argc, char **argv) {
 // CK2: ret void
 
 // CK2: define internal void @[[PAR_OUTLT5]]({{.+}})
-// CK2: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
 // CK2: call {{.+}} @__kmpc_dispatch_next_4(
 // CK2: ret void
 
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_simd_default_messages.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_simd_default_messages.cpp
index 93017a8233ffe..ce7f35b479592 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_simd_default_messages.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_simd_default_messages.cpp
@@ -2,17 +2,26 @@
 
 // RUN: %clang_cc1 -verify -fopenmp-simd %s -Wuninitialized
 
+// RUN: %clang_cc1 -verify -fopenmp %s -Wuninitialized -fopenmp-version=51 -DOMP51
+
+// RUN: %clang_cc1 -verify -fopenmp-simd %s -Wuninitialized -fopenmp-version=51 -DOMP51
+
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
   #pragma omp target
   #pragma omp teams distribute parallel for simd default // expected-error {{expected '(' after 'default'}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target
-  #pragma omp teams distribute parallel for simd default ( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp teams distribute parallel for simd default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target
-  #pragma omp teams distribute parallel for simd default () // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp teams distribute parallel for simd default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target
   #pragma omp teams distribute parallel for simd default (none // expected-error {{expected ')'}} expected-note {{to match this '('}}
@@ -21,12 +30,20 @@ int main(int argc, char **argv) {
   #pragma omp teams distribute parallel for simd default (shared), default(shared) // expected-error {{directive '#pragma omp teams distribute parallel for simd' cannot contain more than one 'default' clause}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target
-  #pragma omp teams distribute parallel for simd default (x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp teams distribute parallel for simd default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (int i=0; i<200; i++) foo();
 
   #pragma omp target
   #pragma omp teams distribute parallel for simd default(none) // expected-note {{explicit data sharing attribute requested here}}
   for (int i=0; i<200; i++) ++argc; // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
 
+#ifdef OpenMP51
+#pragma omp teams distribute parallel for default(firstprivate) // expected-note 2 {{explicit data sharing attribute requested here}}
+  for (int i = 0; i < 200; i++) {
+    ++x; // expected-error {{variable 'x' must have explicitly specified data sharing attributes}}
+    ++y; // expected-error {{variable 'y' must have explicitly specified data sharing attributes}}
+  }
+#endif
+
   return 0;
 }
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_simd_firstprivate_messages.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_simd_firstprivate_messages.cpp
index 898c1e2615074..9d8137b1962ba 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_simd_firstprivate_messages.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_simd_firstprivate_messages.cpp
@@ -1,7 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp %s -Wno-openmp-mapping -Wuninitialized
-
-// RUN: %clang_cc1 -verify -fopenmp-simd %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 %s -Wno-openmp-mapping -Wuninitialized
 
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 %s -Wno-openmp-mapping -Wuninitialized
 extern int omp_default_mem_alloc;
 void foo() {
 }
@@ -140,8 +141,8 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) foo();
 
 #pragma omp target
-#pragma omp teams distribute parallel for simd private(i), firstprivate(i) // expected-error {{private variable cannot be firstprivate}} expected-note 2 {{defined as private}}
-  for (i = 0; i < argc; ++i) foo(); // expected-error {{loop iteration variable in the associated loop of 'omp teams distribute parallel for simd' directive may not be private, predetermined as linear}}
+#pragma omp teams distribute parallel for simd private(i), firstprivate(i) // expected-error {{private variable cannot be firstprivate}} omp4-note 2 {{defined as private}} omp5-note {{defined as private}} 
+  for (i = 0; i < argc; ++i) foo(); // omp4-error {{loop iteration variable in the associated loop of 'omp teams distribute parallel for simd' directive may not be private, predetermined as linear}}
 
 #pragma omp target
 #pragma omp teams distribute parallel for simd firstprivate(i)
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_simd_loop_messages.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_simd_loop_messages.cpp
index 0b2d46211cc85..6e741ce00b678 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_simd_loop_messages.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_simd_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_simd_schedule_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_simd_schedule_codegen.cpp
index ea99cc7b569d2..4662c46fcce34 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_simd_schedule_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_simd_schedule_codegen.cpp
@@ -3,10 +3,18 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64 --check-prefix CK1-OMP45
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64 --check-prefix CK1-OMP50
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
+
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32
 // RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32
+
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-32
 
@@ -120,7 +128,8 @@ struct SS{
   // CK1: ret void
 
   // CK1: define internal void @[[PAR_OUTL4]]({{.+}})
-  // CK1: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+  // CK1-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+  // CK1-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
   // CK1: call {{.+}} @__kmpc_dispatch_next_4(
   // CK1: ret void
 
@@ -135,7 +144,8 @@ struct SS{
   // CK1: ret void
 
   // CK1: define internal void @[[PAR_OUTL5]]({{.+}})
-  // CK1: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+  // CK1-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+  // CK1-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
   // CK1: call {{.+}} @__kmpc_dispatch_next_4(
   // CK1: ret void
 
@@ -153,9 +163,20 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64 --check-prefix CK2-OMP45
+
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64 --check-prefix CK2-OMP50
+
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
+
+
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32
+
 // RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-32
@@ -303,7 +324,8 @@ int main (int argc, char **argv) {
 // CK2: ret void
 
 // CK2: define internal void @[[PAR_OUTL4]]({{.+}})
-// CK2: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
 // CK2: call {{.+}} @__kmpc_dispatch_next_4(
 // CK2: ret void
 
@@ -319,7 +341,9 @@ int main (int argc, char **argv) {
 // CK2: ret void
 
 // CK2: define internal void @[[PAR_OUTL5]]({{.+}})
-// CK2: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
+
 // CK2: call {{.+}} @__kmpc_dispatch_next_4(
 // CK2: ret void
 
@@ -393,7 +417,9 @@ int main (int argc, char **argv) {
 // CK2: ret void
 
 // CK2: define internal void @[[PAR_OUTLT4]]({{.+}})
-// CK2: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
+
 // CK2: call {{.+}} @__kmpc_dispatch_next_4(
 // CK2: ret void
 
@@ -408,7 +434,9 @@ int main (int argc, char **argv) {
 // CK2: ret void
 
 // CK2: define internal void @[[PAR_OUTLT5]]({{.+}})
-// CK2: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP45: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 35,
+// CK2-OMP50: call void @__kmpc_dispatch_init_4({{.+}}, {{.+}}, i32 1073741859,
+
 // CK2: call {{.+}} @__kmpc_dispatch_next_4(
 // CK2: ret void
 
diff --git a/clang/test/OpenMP/teams_distribute_simd_default_messages.cpp b/clang/test/OpenMP/teams_distribute_simd_default_messages.cpp
index 2775210ae048f..11f5d1cd1fc8f 100644
--- a/clang/test/OpenMP/teams_distribute_simd_default_messages.cpp
+++ b/clang/test/OpenMP/teams_distribute_simd_default_messages.cpp
@@ -1,18 +1,23 @@
-// RUN: %clang_cc1 -verify -fopenmp %s -Wuninitialized
+// RUN: %clang_cc1 -verify -fopenmp %s -Wuninitialized -fopenmp-version=51
 
-// RUN: %clang_cc1 -verify -fopenmp-simd %s -Wuninitialized
+// RUN: %clang_cc1 -verify -fopenmp-simd %s -Wuninitialized -fopenmp-version=51
 
 void foo();
 
+namespace {
+static int y = 0;
+}
+static int x = 0;
+
 int main(int argc, char **argv) {
   #pragma omp target
   #pragma omp teams distribute simd default // expected-error {{expected '(' after 'default'}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target
-  #pragma omp teams distribute simd default ( // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp teams distribute simd default( // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}} expected-error {{expected ')'}} expected-note {{to match this '('}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target
-  #pragma omp teams distribute simd default () // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp teams distribute simd default() // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target
   #pragma omp teams distribute simd default (none // expected-error {{expected ')'}} expected-note {{to match this '('}}
@@ -21,12 +26,22 @@ int main(int argc, char **argv) {
   #pragma omp teams distribute simd default (shared), default(shared) // expected-error {{directive '#pragma omp teams distribute simd' cannot contain more than one 'default' clause}}
   for (int i=0; i<200; i++) foo();
   #pragma omp target
-  #pragma omp teams distribute simd default (x) // expected-error {{expected 'none' or 'shared' in OpenMP clause 'default'}}
+#pragma omp teams distribute simd default(x) // expected-error {{expected 'none', 'shared' or 'firstprivate' in OpenMP clause 'default'}}
   for (int i=0; i<200; i++) foo();
 
   #pragma omp target
   #pragma omp teams distribute simd default(none) // expected-note {{explicit data sharing attribute requested here}}
   for (int i=0; i<200; i++) ++argc; // expected-error {{variable 'argc' must have explicitly specified data sharing attributes}}
 
+#pragma omp target
+#pragma omp teams distribute simd default(firstprivate) // expected-note {{explicit data sharing attribute requested here}}
+  for (int i = 0; i < 200; i++)
+    ++x; // expected-error {{variable 'x' must have explicitly specified data sharing attributes}}
+
+#pragma omp target
+#pragma omp teams distribute simd default(firstprivate) // expected-note {{explicit data sharing attribute requested here}}
+  for (int i = 0; i < 200; i++)
+    ++y; // expected-error {{variable 'y' must have explicitly specified data sharing attributes}}
+
   return 0;
 }
diff --git a/clang/test/OpenMP/teams_distribute_simd_firstprivate_messages.cpp b/clang/test/OpenMP/teams_distribute_simd_firstprivate_messages.cpp
index 23eeb5efc17ac..c8a95c2f1553e 100644
--- a/clang/test/OpenMP/teams_distribute_simd_firstprivate_messages.cpp
+++ b/clang/test/OpenMP/teams_distribute_simd_firstprivate_messages.cpp
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 %s -Wno-openmp-mapping -Wuninitialized
 
 extern int omp_default_mem_alloc;
 void foo() {
@@ -140,8 +142,8 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) foo();
 
 #pragma omp target
-#pragma omp teams distribute simd private(i), firstprivate(i) // expected-error {{private variable cannot be firstprivate}} expected-note 2 {{defined as private}}
-  for (i = 0; i < argc; ++i) foo(); // expected-error {{loop iteration variable in the associated loop of 'omp teams distribute simd' directive may not be private, predetermined as linear}}
+#pragma omp teams distribute simd private(i), firstprivate(i) // expected-error {{private variable cannot be firstprivate}} omp4-note 2 {{defined as private}} omp5-note {{defined as private}} 
+  for (i = 0; i < argc; ++i) foo(); // omp4-error {{loop iteration variable in the associated loop of 'omp teams distribute simd' directive may not be private, predetermined as linear}}
 
 #pragma omp target
 #pragma omp teams distribute simd firstprivate(i)
diff --git a/clang/test/OpenMP/teams_distribute_simd_loop_messages.cpp b/clang/test/OpenMP/teams_distribute_simd_loop_messages.cpp
index 7ab32c94da586..a79afb2260732 100644
--- a/clang/test/OpenMP/teams_distribute_simd_loop_messages.cpp
+++ b/clang/test/OpenMP/teams_distribute_simd_loop_messages.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -fopenmp -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
-// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
+// RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp4 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -fexceptions -fcxx-exceptions -verify=expected,omp5 %s -Wuninitialized
 
 class S {
diff --git a/clang/test/OpenMP/teams_messages.cpp b/clang/test/OpenMP/teams_messages.cpp
index 1154220a18c5c..9a37ac7b78fd0 100644
--- a/clang/test/OpenMP/teams_messages.cpp
+++ b/clang/test/OpenMP/teams_messages.cpp
@@ -1,7 +1,8 @@
-// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp -std=c++11 -o - %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp-version=45 -fopenmp -std=c++11 -o - %s -Wuninitialized
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 -std=c++11 -o - %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp-simd -std=c++11 -o - %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp-version=45 -fopenmp-simd -std=c++11 -o - %s -Wuninitialized
+// RUN: %clang_cc1 -verify -fopenmp-version=50 -fopenmp-simd -std=c++11 -o - %s -Wuninitialized
 
 void xxx(int argc) {
   int x; // expected-note {{initialize the variable 'x' to silence this warning}}
diff --git a/clang/test/OpenMP/threadprivate_codegen.cpp b/clang/test/OpenMP/threadprivate_codegen.cpp
index 52ed8bb4b1aa6..c24d1ea787454 100644
--- a/clang/test/OpenMP/threadprivate_codegen.cpp
+++ b/clang/test/OpenMP/threadprivate_codegen.cpp
@@ -1,19 +1,27 @@
-// RUN: %clang_cc1 -verify -fopenmp -fnoopenmp-use-tls -DBODY -triple x86_64-unknown-unknown -x c++ -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s
-// RUN: %clang_cc1 -fopenmp -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fnoopenmp-use-tls -DBODY -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix=CHECK-DEBUG %s
+// RUN: %clang_cc1 -verify -fopenmp -fnoopenmp-use-tls -DBODY -triple x86_64-unknown-unknown -x c++ -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefixes=CHECK,OMP50
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fnoopenmp-use-tls -DBODY -triple x86_64-unknown-unknown -x c++ -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefixes=CHECK,OMP45
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -fnoopenmp-use-tls -DBODY -triple x86_64-unknown-unknown -x c++ -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fnoopenmp-use-tls -DBODY -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
-// RUN: %clang_cc1 -verify -fopenmp -DBODY -triple x86_64-unknown-unknown -x c++ -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=CHECK-TLS
+// RUN: %clang_cc1 -verify -fopenmp -DBODY -triple x86_64-unknown-unknown -x c++ -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefixes=CHECK-TLS,OMP50-TLS
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -DBODY -triple x86_64-unknown-unknown -x c++ -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefixes=CHECK-TLS,OMP45-TLS
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -DBODY -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix=CHECK-TLS %s
+// RUN: %clang_cc1 -fopenmp -DBODY -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefixes=CHECK-TLS,OMP50-TLS %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -DBODY -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefixes=CHECK-TLS,OMP45-TLS %s
 
 // RUN: %clang_cc1 -verify -fopenmp-simd -DBODY -triple x86_64-unknown-unknown -x c++ -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -DBODY -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+
+// RUN: %clang_cc1 -fopenmp -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fnoopenmp-use-tls -DBODY -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefixes=CHECK-DEBUG,OMP50-DEBUG %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fnoopenmp-use-tls -DBODY -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefixes=CHECK-DEBUG,OMP45-DEBUG %s
+
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
@@ -154,26 +162,28 @@ struct S5 {
 // CHECK-DEBUG-DAG: [[ST_INT_ST:@.+]] = linkonce_odr global i32 23
 // CHECK-DEBUG-DAG: [[ST_FLOAT_ST:@.+]] = linkonce_odr global float 2.300000e+01
 // CHECK-DEBUG-DAG: [[ST_S4_ST:@.+]] = linkonce_odr global %struct.S4 zeroinitializer
-// CHECK-DEBUG-DAG: [[LOC1:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;;201;1;;\00"
-// CHECK-DEBUG-DAG: [[LOC2:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;;256;1;;\00"
-// CHECK-DEBUG-DAG: [[LOC3:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;;343;19;;\00"
-// CHECK-DEBUG-DAG: [[LOC4:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;380;1;;\00"
-// CHECK-DEBUG-DAG: [[LOC5:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;397;9;;\00"
-// CHECK-DEBUG-DAG: [[LOC6:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;419;10;;\00"
-// CHECK-DEBUG-DAG: [[LOC7:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;442;10;;\00"
-// CHECK-DEBUG-DAG: [[LOC8:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;478;10;;\00"
-// CHECK-DEBUG-DAG: [[LOC9:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;507;10;;\00"
-// CHECK-DEBUG-DAG: [[LOC10:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;527;10;;\00"
-// CHECK-DEBUG-DAG: [[LOC11:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;550;27;;\00"
-// CHECK-DEBUG-DAG: [[LOC12:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;573;10;;\00"
-// CHECK-DEBUG-DAG: [[LOC13:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;foobar;693;9;;\00"
-// CHECK-DEBUG-DAG: [[LOC14:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;foobar;716;10;;\00"
-// CHECK-DEBUG-DAG: [[LOC15:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;foobar;752;10;;\00"
-// CHECK-DEBUG-DAG: [[LOC16:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;foobar;781;10;;\00"
-// CHECK-DEBUG-DAG: [[LOC17:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;foobar;801;10;;\00"
-// CHECK-DEBUG-DAG: [[LOC18:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;foobar;824;27;;\00"
-// CHECK-DEBUG-DAG: [[LOC19:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;foobar;847;10;;\00"
-// CHECK-DEBUG-DAG: [[LOC20:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;;315;1;;\00"
+
+// CHECK-DEBUG-DAG: [[LOC1:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;;249;1;;\00"
+// CHECK-DEBUG-DAG: [[LOC2:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;;304;1;;\00"
+// CHECK-DEBUG-DAG: [[LOC3:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;;422;19;;\00"
+// CHECK-DEBUG-DAG: [[LOC4:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;459;1;;\00"
+// CHECK-DEBUG-DAG: [[LOC5:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;476;9;;\00"
+// CHECK-DEBUG-DAG: [[LOC6:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;498;10;;\00"
+// CHECK-DEBUG-DAG: [[LOC7:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;521;10;;\00"
+// CHECK-DEBUG-DAG: [[LOC8:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;557;10;;\00"
+// CHECK-DEBUG-DAG: [[LOC9:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;586;10;;\00"
+// CHECK-DEBUG-DAG: [[LOC10:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;606;10;;\00"
+// CHECK-DEBUG-DAG: [[LOC11:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;629;27;;\00"
+// CHECK-DEBUG-DAG: [[LOC12:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;main;652;10;;\00"
+// CHECK-DEBUG-DAG: [[LOC13:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;foobar;774;9;;\00"
+// CHECK-DEBUG-DAG: [[LOC14:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;foobar;797;10;;\00"
+// CHECK-DEBUG-DAG: [[LOC15:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;foobar;833;10;;\00"
+// CHECK-DEBUG-DAG: [[LOC16:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;foobar;862;10;;\00"
+// CHECK-DEBUG-DAG: [[LOC17:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;foobar;882;10;;\00"
+// CHECK-DEBUG-DAG: [[LOC18:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;foobar;905;27;;\00"
+// CHECK-DEBUG-DAG: [[LOC19:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;foobar;928;10;;\00"
+// CHECK-DEBUG-DAG: [[LOC20:@.*]] = private unnamed_addr constant [{{[0-9]+}} x i8] c";{{.*}}threadprivate_codegen.cpp;;363;1;;\00"
+
 // CHECK-TLS-DAG:  [[GS1:@.+]] = internal thread_local global [[S1]] zeroinitializer
 // CHECK-TLS-DAG:  [[GS2:@.+]] = internal global [[S2]] zeroinitializer
 // CHECK-TLS-DAG:  [[ARR_X:@.+]] = thread_local global [2 x [3 x [[S1]]]] zeroinitializer
@@ -192,6 +202,44 @@ struct S5 {
 
 // CHECK-TLS-DAG:  [[ST_S4_ST_TLS_INIT:@_ZTHN2STI2S4E2stE]] = linkonce_odr alias void (), void ()* [[ST_S4_ST_CXX_INIT:@[^, ]*]]
 
+
+// OMP50-TLS: define internal void [[GS1_CXX_INIT:@.*]]()
+// OMP50-TLS: call void [[GS1_CTOR1:@.*]]([[S1]]* [[GS1]], i32 5)
+// OMP50-TLS: call i32 @__cxa_thread_atexit(void (i8*)* bitcast (void ([[S1]]*)* [[GS1_DTOR1:.*]] to void (i8*)*), i8* bitcast ([[S1]]* [[GS1]] to i8*)
+// OMP50-TLS: }
+// OMP50-TLS: define {{.*}}void [[GS1_CTOR1]]([[S1]]* {{.*}}, i32 {{.*}})
+// OMP50-TLS: call void [[GS1_CTOR2:@.*]]([[S1]]* {{.*}}, i32 {{.*}})
+// OMP50-TLS: }
+// OMP50-TLS: define {{.*}}void [[GS1_DTOR1]]([[S1]]* {{.*}})
+// OMP50-TLS: call void [[GS1_DTOR2:@.*]]([[S1]]* {{.*}})
+// OMP50-TLS: }
+// OMP50-TLS: define {{.*}}void [[GS1_CTOR2]]([[S1]]* {{.*}}, i32 {{.*}})
+// OMP50-TLS: define {{.*}}void [[GS1_DTOR2]]([[S1]]* {{.*}})
+
+// OMP50-TLS: define internal void [[GS2_CXX_INIT:@.*]]()
+// OMP50-TLS: call void [[GS2_CTOR1:@.*]]([[S2]]* [[GS2]], i32 27)
+// OMP50-TLS: call i32 @__cxa_atexit(void (i8*)* bitcast (void ([[S2]]*)* [[GS2_DTOR1:.*]] to void (i8*)*), i8* bitcast ([[S2]]* [[GS2]] to i8*)
+// OMP50-TLS: }
+// OMP50-TLS: define {{.*}}void [[GS2_CTOR1]]([[S2]]* {{.*}}, i32 {{.*}})
+// OMP50-TLS: call void [[GS2_CTOR2:@.*]]([[S2]]* {{.*}}, i32 {{.*}})
+// OMP50-TLS: }
+// OMP50-TLS: define {{.*}}void [[GS2_DTOR1]]([[S2]]* {{.*}})
+// OMP50-TLS: call void [[GS2_DTOR2:@.*]]([[S2]]* {{.*}})
+// OMP50-TLS: }
+// OMP50-TLS: define {{.*}}void [[GS2_CTOR2]]([[S2]]* {{.*}}, i32 {{.*}})
+// OMP50-TLS: define {{.*}}void [[GS2_DTOR2]]([[S2]]* {{.*}})
+
+// OMP50-TLS: define internal void [[ARR_X_CXX_INIT:@.*]]()
+// OMP50-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 0, i{{.*}} 0), i{{.*}} 1)
+// OMP50-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 0, i{{.*}} 1), i{{.*}} 2)
+// OMP50-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 0, i{{.*}} 2), i{{.*}} 3)
+// OMP50-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 1, i{{.*}} 0), i{{.*}} 4)
+// OMP50-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 1, i{{.*}} 1), i{{.*}} 5)
+// OMP50-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 1, i{{.*}} 2), i{{.*}} 6)
+// OMP50-TLS: call i32 @__cxa_thread_atexit(void (i8*)* [[ARR_X_CXX_DTOR:@[^,]+]]
+// OMP50-TLS: define internal void [[ARR_X_CXX_DTOR]](i8* %0)
+// OMP50-TLS: void [[GS1_DTOR1]]([[S1]]* {{.*}})
+
 struct Static {
   static S3 s;
 #pragma omp threadprivate(s)
@@ -315,6 +363,37 @@ struct ST {
 #pragma omp threadprivate(st)
 };
 
+
+// OMP50-DEBUG:      [[KMPC_LOC_ADDR:%.*]] = alloca [[IDENT]]
+// OMP50-DEBUG:      [[KMPC_LOC_ADDR_PSOURCE:%.*]] = getelementptr inbounds [[IDENT]], [[IDENT]]* [[KMPC_LOC_ADDR]], i{{.*}} 0, i{{.*}} 4
+// OMP50-DEBUG-NEXT: store i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[LOC20]], i{{.*}} 0, i{{.*}} 0), i8** [[KMPC_LOC_ADDR_PSOURCE]]
+// OMP50-DEBUG:      @__kmpc_global_thread_num
+// OMP50-DEBUG:      call {{.*}}void @__kmpc_threadprivate_register([[IDENT]]* [[KMPC_LOC_ADDR]], i8* bitcast ([[S4]]* [[ST_S4_ST]] to i8*), i8* (i8*)* [[ST_S4_ST_CTOR:@\.__kmpc_global_ctor_\..+]], i8* (i8*, i8*)* null, void (i8*)* [[ST_S4_ST_DTOR:@\.__kmpc_global_dtor_\..+]])
+// OMP50-DEBUG:      define internal {{.*}}i8* [[ST_S4_ST_CTOR]](i8* %0)
+// OMP50-DEBUG:      }
+// OMP50-DEBUG:      define {{.*}} [[S4_CTOR:@.*]]([[S4]]* {{.*}},
+// OMP50-DEBUG:      define internal {{.*}}void [[ST_S4_ST_DTOR]](i8* %0)
+// OMP50-DEBUG:      }
+// OMP50-DEBUG:      define {{.*}} [[S4_DTOR:@.*]]([[S4]]* {{.*}})
+
+// OMP50:      call {{.*}}void @__kmpc_threadprivate_register([[IDENT]]* [[DEFAULT_LOC]], i8* bitcast ([[S4]]* [[ST_S4_ST]] to i8*), i8* (i8*)* [[ST_S4_ST_CTOR:@\.__kmpc_global_ctor_\..+]], i8* (i8*, i8*)* null, void (i8*)* [[ST_S4_ST_DTOR:@\.__kmpc_global_dtor_\..+]])
+// OMP50:      define internal {{.*}}i8* [[ST_S4_ST_CTOR]](i8* %0)
+// OMP50:      store i8* %0, i8** [[ARG_ADDR:%.*]],
+// OMP50:      [[ARG:%.+]] = load i8*, i8** [[ARG_ADDR]]
+// OMP50:      [[RES:%.*]] = bitcast i8* [[ARG]] to [[S4]]*
+// OMP50-NEXT: call {{.*}} [[S4_CTOR:@.+]]([[S4]]* [[RES]], {{.*}} 23)
+// OMP50:      [[ARG:%.+]] = load i8*, i8** [[ARG_ADDR]]
+// OMP50-NEXT: ret i8* [[ARG]]
+// OMP50-NEXT: }
+// OMP50:      define {{.*}} [[S4_CTOR]]([[S4]]* {{.*}},
+// OMP50:      define internal {{.*}}void [[ST_S4_ST_DTOR]](i8* %0)
+// OMP50:      store i8* %0, i8** [[ARG_ADDR:%.*]],
+// OMP50:      [[ARG:%.+]] = load i8*, i8** [[ARG_ADDR]]
+// OMP50:      [[RES:%.*]] = bitcast i8* [[ARG]] to [[S4]]*
+// OMP50-NEXT: call {{.*}} [[S4_DTOR:@.+]]([[S4]]* [[RES]])
+// OMP50-NEXT: ret void
+// OMP50-NEXT: }
+// OMP50:      define {{.*}} [[S4_DTOR]]([[S4]]* {{.*}})
 template <class T>
 T ST<T>::st(23);
 
@@ -662,11 +741,13 @@ int main() {
 // CHECK-TLS: }
 
 #endif
+// OMP50-TLS: define {{.*}}void [[SM_CTOR2]]([[SMAIN]]* {{.*}}, i32 {{.*}})
+// OMP50-TLS: define {{.*}}void [[SM_DTOR2]]([[SMAIN]]* {{.*}})
 
 #ifdef BODY
 // CHECK-LABEL:  @{{.*}}foobar{{.*}}()
 // CHECK-DEBUG-LABEL: @{{.*}}foobar{{.*}}()
-// CHECK-TLS: @{{.*}}foobar{{.*}}()
+// CHECK-TLS-LABEL: @{{.*}}foobar{{.*}}()
 int foobar() {
   // CHECK-DEBUG:      [[KMPC_LOC_ADDR:%.*]] = alloca [[IDENT]]
   int Res;
@@ -793,11 +874,11 @@ int foobar() {
   // CHECK-DEBUG-NEXT: [[RES:%.*]] = load [[INT]], [[INT]]* [[RES_ADDR]]
   // CHECK-DEBUG-NEXT: [[ADD:%.*]] = add {{.*}} [[INT]] [[RES]], [[ST_INT_ST_VAL]]
   // CHECK-DEBUG-NEXT: store [[INT]] [[ADD]], [[INT]]* [[RES:.+]]
-  // CHECK-TLS:      [[ST_INT_ST_ADDR:%.*]] = call i32* [[ST_INT_ST_TLS_INITD]]
-  // CHECK-TLS-NEXT: [[ST_INT_ST_VAL:%.*]] = load [[INT]], [[INT]]* [[ST_INT_ST_ADDR]]
-  // CHECK-TLS-NEXT: [[RES:%.*]] = load [[INT]], [[INT]]* [[RES_ADDR]]
-  // CHECK-TLS-NEXT: [[ADD:%.*]] = add {{.*}} [[INT]] [[RES]], [[ST_INT_ST_VAL]]
-  // CHECK-TLS-NEXT: store [[INT]] [[ADD]], [[INT]]* [[RES:.+]]
+  // OMP45-TLS:      [[ST_INT_ST_ADDR:%.*]] = call i32* [[ST_INT_ST_TLS_INITD]]
+  // OMP45-TLS-NEXT: [[ST_INT_ST_VAL:%.*]] = load [[INT]], [[INT]]* [[ST_INT_ST_ADDR]]
+  // OMP45-TLS-NEXT: [[RES:%.*]] = load [[INT]], [[INT]]* [[RES_ADDR]]
+  // OMP45-TLS-NEXT: [[ADD:%.*]] = add {{.*}} [[INT]] [[RES]], [[ST_INT_ST_VAL]]
+  // OMP45-TLS-NEXT: store [[INT]] [[ADD]], [[INT]]* [[RES:.+]]
   Res += ST<int>::st;
   // CHECK:      [[ST_FLOAT_ST_TEMP_ADDR:%.*]] = call {{.*}}i8* @__kmpc_threadprivate_cached([[IDENT]]* [[DEFAULT_LOC]], i32 [[THREAD_NUM]], i8* bitcast (float* [[ST_FLOAT_ST]] to i8*), i{{.*}} {{[0-9]+}}, i8*** [[ST_FLOAT_ST]].cache.)
   // CHECK-NEXT: [[ST_FLOAT_ST_ADDR:%.*]] = bitcast i8* [[ST_FLOAT_ST_TEMP_ADDR]] to float*
@@ -815,12 +896,12 @@ int foobar() {
   // CHECK-DEBUG-NEXT: [[RES:%.*]] = load [[INT]], [[INT]]* [[RES_ADDR]]
   // CHECK-DEBUG-NEXT: [[ADD:%.*]] = add {{.*}} [[INT]] [[RES]], [[FLOAT_TO_INT_CONV]]
   // CHECK-DEBUG-NEXT: store [[INT]] [[ADD]], [[INT]]* [[RES:.+]]
-  // CHECK-TLS:      [[ST_FLOAT_ST_ADDR:%.*]] = call float* [[ST_FLOAT_ST_TLS_INITD]]
-  // CHECK-TLS-NEXT: [[ST_FLOAT_ST_VAL:%.*]] = load float, float* [[ST_FLOAT_ST_ADDR]]
-  // CHECK-TLS-NEXT: [[FLOAT_TO_INT_CONV:%.*]] = fptosi float [[ST_FLOAT_ST_VAL]] to [[INT]]
-  // CHECK-TLS-NEXT: [[RES:%.*]] = load [[INT]], [[INT]]* [[RES_ADDR]]
-  // CHECK-TLS-NEXT: [[ADD:%.*]] = add {{.*}} [[INT]] [[RES]], [[FLOAT_TO_INT_CONV]]
-  // CHECK-TLS-NEXT: store [[INT]] [[ADD]], [[INT]]* [[RES:.+]]
+  // OMP45-TLS:      [[ST_FLOAT_ST_ADDR:%.*]] = call float* [[ST_FLOAT_ST_TLS_INITD]]
+  // OMP45-TLS-NEXT: [[ST_FLOAT_ST_VAL:%.*]] = load float, float* [[ST_FLOAT_ST_ADDR]]
+  // OMP45-TLS-NEXT: [[FLOAT_TO_INT_CONV:%.*]] = fptosi float [[ST_FLOAT_ST_VAL]] to [[INT]]
+  // OMP45-TLS-NEXT: [[RES:%.*]] = load [[INT]], [[INT]]* [[RES_ADDR]]
+  // OMP45-TLS-NEXT: [[ADD:%.*]] = add {{.*}} [[INT]] [[RES]], [[FLOAT_TO_INT_CONV]]
+  // OMP45-TLS-NEXT: store [[INT]] [[ADD]], [[INT]]* [[RES:.+]]
   Res += static_cast<int>(ST<float>::st);
   // CHECK:      [[ST_S4_ST_TEMP_ADDR:%.*]] = call {{.*}}i8* @__kmpc_threadprivate_cached([[IDENT]]* [[DEFAULT_LOC]], i32 [[THREAD_NUM]], i8* bitcast ([[S4]]* [[ST_S4_ST]] to i8*), i{{.*}} {{[0-9]+}}, i8*** [[ST_S4_ST]].cache.)
   // CHECK-NEXT: [[ST_S4_ST_ADDR:%.*]] = bitcast i8* [[ST_S4_ST_TEMP_ADDR]] to [[S4]]*
@@ -855,35 +936,36 @@ int foobar() {
 }
 #endif
 
-// CHECK:      call {{.*}}void @__kmpc_threadprivate_register([[IDENT]]* [[DEFAULT_LOC]], i8* bitcast ([[S4]]* [[ST_S4_ST]] to i8*), i8* (i8*)* [[ST_S4_ST_CTOR:@\.__kmpc_global_ctor_\..+]], i8* (i8*, i8*)* null, void (i8*)* [[ST_S4_ST_DTOR:@\.__kmpc_global_dtor_\..+]])
-// CHECK:      define internal {{.*}}i8* [[ST_S4_ST_CTOR]](i8* %0)
-// CHECK:      store i8* %0, i8** [[ARG_ADDR:%.*]],
-// CHECK:      [[ARG:%.+]] = load i8*, i8** [[ARG_ADDR]]
-// CHECK:      [[RES:%.*]] = bitcast i8* [[ARG]] to [[S4]]*
-// CHECK-NEXT: call {{.*}} [[S4_CTOR:@.+]]([[S4]]* [[RES]], {{.*}} 23)
-// CHECK:      [[ARG:%.+]] = load i8*, i8** [[ARG_ADDR]]
-// CHECK-NEXT: ret i8* [[ARG]]
-// CHECK-NEXT: }
-// CHECK:      define {{.*}} [[S4_CTOR]]([[S4]]* {{.*}},
-// CHECK:      define internal {{.*}}void [[ST_S4_ST_DTOR]](i8* %0)
-// CHECK:      store i8* %0, i8** [[ARG_ADDR:%.*]],
-// CHECK:      [[ARG:%.+]] = load i8*, i8** [[ARG_ADDR]]
-// CHECK:      [[RES:%.*]] = bitcast i8* [[ARG]] to [[S4]]*
-// CHECK-NEXT: call {{.*}} [[S4_DTOR:@.+]]([[S4]]* [[RES]])
-// CHECK-NEXT: ret void
-// CHECK-NEXT: }
-// CHECK:      define {{.*}} [[S4_DTOR]]([[S4]]* {{.*}})
-// CHECK-DEBUG:      [[KMPC_LOC_ADDR:%.*]] = alloca [[IDENT]]
-// CHECK-DEBUG:      [[KMPC_LOC_ADDR_PSOURCE:%.*]] = getelementptr inbounds [[IDENT]], [[IDENT]]* [[KMPC_LOC_ADDR]], i{{.*}} 0, i{{.*}} 4
-// CHECK-DEBUG-NEXT: store i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[LOC20]], i{{.*}} 0, i{{.*}} 0), i8** [[KMPC_LOC_ADDR_PSOURCE]]
-// CHECK-DEBUG:      @__kmpc_global_thread_num
-// CHECK-DEBUG:      call {{.*}}void @__kmpc_threadprivate_register([[IDENT]]* [[KMPC_LOC_ADDR]], i8* bitcast ([[S4]]* [[ST_S4_ST]] to i8*), i8* (i8*)* [[ST_S4_ST_CTOR:@\.__kmpc_global_ctor_\..+]], i8* (i8*, i8*)* null, void (i8*)* [[ST_S4_ST_DTOR:@\.__kmpc_global_dtor_\..+]])
-// CHECK-DEBUG:      define internal {{.*}}i8* [[ST_S4_ST_CTOR]](i8* %0)
-// CHECK-DEBUG:      }
-// CHECK-DEBUG:      define {{.*}} [[S4_CTOR:@.*]]([[S4]]* {{.*}},
-// CHECK-DEBUG:      define internal {{.*}}void [[ST_S4_ST_DTOR]](i8* %0)
-// CHECK-DEBUG:      }
-// CHECK-DEBUG:      define {{.*}} [[S4_DTOR:@.*]]([[S4]]* {{.*}})
+// OMP45:      call {{.*}}void @__kmpc_threadprivate_register([[IDENT]]* [[DEFAULT_LOC]], i8* bitcast ([[S4]]* [[ST_S4_ST]] to i8*), i8* (i8*)* [[ST_S4_ST_CTOR:@\.__kmpc_global_ctor_\..+]], i8* (i8*, i8*)* null, void (i8*)* [[ST_S4_ST_DTOR:@\.__kmpc_global_dtor_\..+]])
+// OMP45:      define internal {{.*}}i8* [[ST_S4_ST_CTOR]](i8* %0)
+// OMP45:      store i8* %0, i8** [[ARG_ADDR:%.*]],
+// OMP45:      [[ARG:%.+]] = load i8*, i8** [[ARG_ADDR]]
+// OMP45:      [[RES:%.*]] = bitcast i8* [[ARG]] to [[S4]]*
+// OMP45-NEXT: call {{.*}} [[S4_CTOR:@.+]]([[S4]]* [[RES]], {{.*}} 23)
+// OMP45:      [[ARG:%.+]] = load i8*, i8** [[ARG_ADDR]]
+// OMP45-NEXT: ret i8* [[ARG]]
+// OMP45-NEXT: }
+// OMP45:      define {{.*}} [[S4_CTOR]]([[S4]]* {{.*}},
+// OMP45:      define internal {{.*}}void [[ST_S4_ST_DTOR]](i8* %0)
+// OMP45:      store i8* %0, i8** [[ARG_ADDR:%.*]],
+// OMP45:      [[ARG:%.+]] = load i8*, i8** [[ARG_ADDR]]
+// OMP45:      [[RES:%.*]] = bitcast i8* [[ARG]] to [[S4]]*
+// OMP45-NEXT: call {{.*}} [[S4_DTOR:@.+]]([[S4]]* [[RES]])
+// OMP45-NEXT: ret void
+// OMP45-NEXT: }
+// OMP45:      define {{.*}} [[S4_DTOR]]([[S4]]* {{.*}})
+
+// OMP45-DEBUG:      [[KMPC_LOC_ADDR:%.*]] = alloca [[IDENT]]
+// OMP45-DEBUG:      [[KMPC_LOC_ADDR_PSOURCE:%.*]] = getelementptr inbounds [[IDENT]], [[IDENT]]* [[KMPC_LOC_ADDR]], i{{.*}} 0, i{{.*}} 4
+// OMP45-DEBUG-NEXT: store i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[LOC20]], i{{.*}} 0, i{{.*}} 0), i8** [[KMPC_LOC_ADDR_PSOURCE]]
+// OMP45-DEBUG:      @__kmpc_global_thread_num
+// OMP45-DEBUG:      call {{.*}}void @__kmpc_threadprivate_register([[IDENT]]* [[KMPC_LOC_ADDR]], i8* bitcast ([[S4]]* [[ST_S4_ST]] to i8*), i8* (i8*)* [[ST_S4_ST_CTOR:@\.__kmpc_global_ctor_\..+]], i8* (i8*, i8*)* null, void (i8*)* [[ST_S4_ST_DTOR:@\.__kmpc_global_dtor_\..+]])
+// OMP45-DEBUG:      define internal {{.*}}i8* [[ST_S4_ST_CTOR]](i8* %0)
+// OMP45-DEBUG:      }
+// OMP45-DEBUG:      define {{.*}} [[S4_CTOR:@.*]]([[S4]]* {{.*}},
+// OMP45-DEBUG:      define internal {{.*}}void [[ST_S4_ST_DTOR]](i8* %0)
+// OMP45-DEBUG:      }
+// OMP45-DEBUG:      define {{.*}} [[S4_DTOR:@.*]]([[S4]]* {{.*}})
 
 // CHECK:      define internal {{.*}}void {{@.*}}()
 // CHECK-DAG:  call {{.*}}void [[GS1_INIT]]()
@@ -892,58 +974,72 @@ int foobar() {
 // CHECK-DEBUG:      define internal {{.*}}void {{@.*}}()
 // CHECK-DEBUG:      ret void
 
-// CHECK-TLS: define internal void [[GS1_CXX_INIT:@.*]]()
-// CHECK-TLS: call void [[GS1_CTOR1:@.*]]([[S1]]* [[GS1]], i32 5)
-// CHECK-TLS: call i32 @__cxa_thread_atexit(void (i8*)* bitcast (void ([[S1]]*)* [[GS1_DTOR1:.*]] to void (i8*)*), i8* bitcast ([[S1]]* [[GS1]] to i8*)
-// CHECK-TLS: }
-// CHECK-TLS: define {{.*}}void [[GS1_CTOR1]]([[S1]]* {{.*}}, i32 {{.*}})
-// CHECK-TLS: call void [[GS1_CTOR2:@.*]]([[S1]]* {{.*}}, i32 {{.*}})
-// CHECK-TLS: }
-// CHECK-TLS: define {{.*}}void [[GS1_DTOR1]]([[S1]]* {{.*}})
-// CHECK-TLS: call void [[GS1_DTOR2:@.*]]([[S1]]* {{.*}})
-// CHECK-TLS: }
-// CHECK-TLS: define {{.*}}void [[GS1_CTOR2]]([[S1]]* {{.*}}, i32 {{.*}})
-// CHECK-TLS: define {{.*}}void [[GS1_DTOR2]]([[S1]]* {{.*}})
+// OMP45-TLS: define internal void [[GS1_CXX_INIT:@.*]]()
+// OMP45-TLS: call void [[GS1_CTOR1:@.*]]([[S1]]* [[GS1]], i32 5)
+// OMP45-TLS: call i32 @__cxa_thread_atexit(void (i8*)* bitcast (void ([[S1]]*)* [[GS1_DTOR1:.*]] to void (i8*)*), i8* bitcast ([[S1]]* [[GS1]] to i8*)
+// OMP45-TLS: }
+// OMP45-TLS: define {{.*}}void [[GS1_CTOR1]]([[S1]]* {{.*}}, i32 {{.*}})
+// OMP45-TLS: call void [[GS1_CTOR2:@.*]]([[S1]]* {{.*}}, i32 {{.*}})
+// OMP45-TLS: }
+// OMP45-TLS: define {{.*}}void [[GS1_DTOR1]]([[S1]]* {{.*}})
+// OMP45-TLS: call void [[GS1_DTOR2:@.*]]([[S1]]* {{.*}})
+// OMP45-TLS: }
+// OMP45-TLS: define {{.*}}void [[GS1_CTOR2]]([[S1]]* {{.*}}, i32 {{.*}})
+// OMP45-TLS: define {{.*}}void [[GS1_DTOR2]]([[S1]]* {{.*}})
 
-// CHECK-TLS: define internal void [[GS2_CXX_INIT:@.*]]()
-// CHECK-TLS: call void [[GS2_CTOR1:@.*]]([[S2]]* [[GS2]], i32 27)
-// CHECK-TLS: call i32 @__cxa_atexit(void (i8*)* bitcast (void ([[S2]]*)* [[GS2_DTOR1:.*]] to void (i8*)*), i8* bitcast ([[S2]]* [[GS2]] to i8*)
-// CHECK-TLS: }
-// CHECK-TLS: define {{.*}}void [[GS2_CTOR1]]([[S2]]* {{.*}}, i32 {{.*}})
-// CHECK-TLS: call void [[GS2_CTOR2:@.*]]([[S2]]* {{.*}}, i32 {{.*}})
-// CHECK-TLS: }
-// CHECK-TLS: define {{.*}}void [[GS2_DTOR1]]([[S2]]* {{.*}})
-// CHECK-TLS: call void [[GS2_DTOR2:@.*]]([[S2]]* {{.*}})
-// CHECK-TLS: }
-// CHECK-TLS: define {{.*}}void [[GS2_CTOR2]]([[S2]]* {{.*}}, i32 {{.*}})
-// CHECK-TLS: define {{.*}}void [[GS2_DTOR2]]([[S2]]* {{.*}})
-
-// CHECK-TLS: define internal void [[ARR_X_CXX_INIT:@.*]]()
-// CHECK-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 0, i{{.*}} 0), i{{.*}} 1)
-// CHECK-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 0, i{{.*}} 1), i{{.*}} 2)
-// CHECK-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 0, i{{.*}} 2), i{{.*}} 3)
-// CHECK-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 1, i{{.*}} 0), i{{.*}} 4)
-// CHECK-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 1, i{{.*}} 1), i{{.*}} 5)
-// CHECK-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 1, i{{.*}} 2), i{{.*}} 6)
-// CHECK-TLS: call i32 @__cxa_thread_atexit(void (i8*)* [[ARR_X_CXX_DTOR:@[^,]+]]
-// CHECK-TLS: define internal void [[ARR_X_CXX_DTOR]](i8* %0)
-// CHECK-TLS: void [[GS1_DTOR1]]([[S1]]* {{.*}})
-
-// CHECK-TLS: define {{.*}}void [[SM_CTOR2]]([[SMAIN]]* {{.*}}, i32 {{.*}})
-// CHECK-TLS: define {{.*}}void [[SM_DTOR2]]([[SMAIN]]* {{.*}})
-
-// CHECK-TLS: define internal void [[ST_S4_ST_CXX_INIT]]()
-// CHECK-TLS: call void [[ST_S4_ST_CTOR1:@.*]]([[S4]]* [[ST_S4_ST]], i32 23)
-// CHECK-TLS: call i32 @__cxa_thread_atexit(void (i8*)* bitcast (void ([[S4]]*)* [[ST_S4_ST_DTOR1:.*]] to void (i8*)*), i8* bitcast ([[S4]]* [[ST_S4_ST]] to i8*)
-// CHECK-TLS: }
-// CHECK-TLS: define {{.*}}void [[ST_S4_ST_CTOR1]]([[S4]]* {{.*}}, i32 {{.*}})
-// CHECK-TLS: call void [[ST_S4_ST_CTOR2:@.*]]([[S4]]* {{.*}}, i32 {{.*}})
-// CHECK-TLS: }
-// CHECK-TLS: define {{.*}}void [[ST_S4_ST_DTOR1]]([[S4]]* {{.*}})
-// CHECK-TLS: call void [[ST_S4_ST_DTOR2:@.*]]([[S4]]* {{.*}})
-// CHECK-TLS: }
-// CHECK-TLS: define {{.*}}void [[ST_S4_ST_CTOR2]]([[S4]]* {{.*}}, i32 {{.*}})
-// CHECK-TLS: define {{.*}}void [[ST_S4_ST_DTOR2]]([[S4]]* {{.*}})
+// OMP45-TLS: define internal void [[GS2_CXX_INIT:@.*]]()
+// OMP45-TLS: call void [[GS2_CTOR1:@.*]]([[S2]]* [[GS2]], i32 27)
+// OMP45-TLS: call i32 @__cxa_atexit(void (i8*)* bitcast (void ([[S2]]*)* [[GS2_DTOR1:.*]] to void (i8*)*), i8* bitcast ([[S2]]* [[GS2]] to i8*)
+// OMP45-TLS: }
+// OMP45-TLS: define {{.*}}void [[GS2_CTOR1]]([[S2]]* {{.*}}, i32 {{.*}})
+// OMP45-TLS: call void [[GS2_CTOR2:@.*]]([[S2]]* {{.*}}, i32 {{.*}})
+// OMP45-TLS: }
+// OMP45-TLS: define {{.*}}void [[GS2_DTOR1]]([[S2]]* {{.*}})
+// OMP45-TLS: call void [[GS2_DTOR2:@.*]]([[S2]]* {{.*}})
+// OMP45-TLS: }
+// OMP45-TLS: define {{.*}}void [[GS2_CTOR2]]([[S2]]* {{.*}}, i32 {{.*}})
+// OMP45-TLS: define {{.*}}void [[GS2_DTOR2]]([[S2]]* {{.*}})
+
+// OMP45-TLS: define internal void [[ARR_X_CXX_INIT:@.*]]()
+// OMP45-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 0, i{{.*}} 0), i{{.*}} 1)
+// OMP45-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 0, i{{.*}} 1), i{{.*}} 2)
+// OMP45-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 0, i{{.*}} 2), i{{.*}} 3)
+// OMP45-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 1, i{{.*}} 0), i{{.*}} 4)
+// OMP45-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 1, i{{.*}} 1), i{{.*}} 5)
+// OMP45-TLS: invoke void [[GS1_CTOR1]]([[S1]]* getelementptr inbounds ([2 x [3 x [[S1]]]], [2 x [3 x [[S1]]]]* [[ARR_X]], i{{.*}} 0, i{{.*}} 1, i{{.*}} 2), i{{.*}} 6)
+// OMP45-TLS: call i32 @__cxa_thread_atexit(void (i8*)* [[ARR_X_CXX_DTOR:@[^,]+]]
+// OMP45-TLS: define internal void [[ARR_X_CXX_DTOR]](i8* %0)
+// OMP45-TLS: void [[GS1_DTOR1]]([[S1]]* {{.*}})
+
+// OMP45-TLS: define {{.*}}void [[SM_CTOR2]]([[SMAIN]]* {{.*}}, i32 {{.*}})
+// OMP45-TLS: define {{.*}}void [[SM_DTOR2]]([[SMAIN]]* {{.*}})
+
+// OMP45-TLS: define internal void [[ST_S4_ST_CXX_INIT]]()
+// OMP45-TLS: call void [[ST_S4_ST_CTOR1:@.*]]([[S4]]* [[ST_S4_ST]], i32 23)
+// OMP45-TLS: call i32 @__cxa_thread_atexit(void (i8*)* bitcast (void ([[S4]]*)* [[ST_S4_ST_DTOR1:.*]] to void (i8*)*), i8* bitcast ([[S4]]* [[ST_S4_ST]] to i8*)
+// OMP45-TLS: }
+// OMP45-TLS: define {{.*}}void [[ST_S4_ST_CTOR1]]([[S4]]* {{.*}}, i32 {{.*}})
+// OMP45-TLS: call void [[ST_S4_ST_CTOR2:@.*]]([[S4]]* {{.*}}, i32 {{.*}})
+// OMP45-TLS: }
+// OMP45-TLS: define {{.*}}void [[ST_S4_ST_DTOR1]]([[S4]]* {{.*}})
+// OMP45-TLS: call void [[ST_S4_ST_DTOR2:@.*]]([[S4]]* {{.*}})
+// OMP45-TLS: }
+// OMP45-TLS: define {{.*}}void [[ST_S4_ST_CTOR2]]([[S4]]* {{.*}}, i32 {{.*}})
+// OMP45-TLS: define {{.*}}void [[ST_S4_ST_DTOR2]]([[S4]]* {{.*}})
+
+// OMP50-TLS: define internal void [[ST_S4_ST_CXX_INIT]]()
+// OMP50-TLS: call void [[ST_S4_ST_CTOR1:@.*]]([[S4]]* [[ST_S4_ST]], i32 23)
+
+// OMP50-TLS: call i32 @__cxa_thread_atexit(void (i8*)* bitcast (void ([[S4]]*)* [[ST_S4_ST_DTOR1:.*]] to void (i8*)*), i8* bitcast ([[S4]]* [[ST_S4_ST]] to i8*)
+// OMP50-TLS: }
+// OMP50-TLS: define {{.*}}void [[ST_S4_ST_CTOR1]]([[S4]]* {{.*}}, i32 {{.*}})
+// OMP50-TLS: call void [[ST_S4_ST_CTOR2:@.*]]([[S4]]* {{.*}}, i32 {{.*}})
+// OMP50-TLS: }
+// OMP50-TLS: define {{.*}}void [[ST_S4_ST_DTOR1]]([[S4]]* {{.*}})
+// OMP50-TLS: call void [[ST_S4_ST_DTOR2:@.*]]([[S4]]* {{.*}})
+// OMP50-TLS: }
+// OMP50-TLS: define {{.*}}void [[ST_S4_ST_CTOR2]]([[S4]]* {{.*}}, i32 {{.*}})
+// OMP50-TLS: define {{.*}}void [[ST_S4_ST_DTOR2]]([[S4]]* {{.*}})
 
 // CHECK-TLS:      define internal void @__tls_init()
 // CHECK-TLS:      [[GRD:%.*]] = load i8, i8* @__tls_guard
diff --git a/clang/test/PCH/aarch64-sve-types.c b/clang/test/PCH/aarch64-sve-types.c
index 27f16306927d4..d9be6daa127e7 100644
--- a/clang/test/PCH/aarch64-sve-types.c
+++ b/clang/test/PCH/aarch64-sve-types.c
@@ -18,4 +18,6 @@ __SVFloat16_t *f16;
 __SVFloat32_t *f32;
 __SVFloat64_t *f64;
 
+__SVBFloat16_t *bf16;
+
 __SVBool_t *b8;
diff --git a/clang/test/PCH/codegen.cpp b/clang/test/PCH/codegen.cpp
new file mode 100644
index 0000000000000..49ed30aeaf054
--- /dev/null
+++ b/clang/test/PCH/codegen.cpp
@@ -0,0 +1,42 @@
+// This test is the PCH version of Modules/codegen-flags.test . It uses its inputs.
+// The purpose of this test is just verify that the codegen options work with PCH as well.
+// All the codegen functionality should be tested in Modules/.
+
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// REQUIRES: x86-registered-target
+
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -fmodules-codegen -x c++-header -building-pch-with-obj -emit-pch %S/../Modules/Inputs/codegen-flags/foo.h -o %t/foo-cg.pch
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -fmodules-debuginfo -x c++-header -building-pch-with-obj -emit-pch %S/../Modules/Inputs/codegen-flags/foo.h -o %t/foo-di.pch
+
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - %s -include-pch %t/foo-cg.pch -building-pch-with-obj -fmodules-codegen | FileCheck --check-prefix=CG %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - %s -include-pch %t/foo-di.pch -building-pch-with-obj -fmodules-debuginfo | FileCheck --check-prefix=DI %s
+
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - -include-pch %t/foo-cg.pch %S/../Modules/Inputs/codegen-flags/use.cpp | FileCheck --check-prefix=CG-USE %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - -include-pch %t/foo-di.pch %S/../Modules/Inputs/codegen-flags/use.cpp | FileCheck --check-prefix=DI-USE %s
+
+// Test with template instantiation in the pch.
+
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -fmodules-codegen -x c++-header -building-pch-with-obj -emit-pch -fpch-instantiate-templates %S/../Modules/Inputs/codegen-flags/foo.h -o %t/foo-cg.pch
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -fmodules-debuginfo -x c++-header -building-pch-with-obj -emit-pch -fpch-instantiate-templates %S/../Modules/Inputs/codegen-flags/foo.h -o %t/foo-di.pch
+
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - %s -include-pch %t/foo-cg.pch -building-pch-with-obj -fmodules-codegen | FileCheck --check-prefix=CG %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - %s -include-pch %t/foo-di.pch -building-pch-with-obj -fmodules-debuginfo | FileCheck --check-prefix=DI %s
+
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - -include-pch %t/foo-cg.pch %S/../Modules/Inputs/codegen-flags/use.cpp | FileCheck --check-prefix=CG-USE %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - -include-pch %t/foo-di.pch %S/../Modules/Inputs/codegen-flags/use.cpp | FileCheck --check-prefix=DI-USE %s
+
+
+// CG: define weak_odr void @_Z2f1v
+// CG: DICompileUnit
+// CG-NOT: DICompositeType
+
+// CG-USE: declare void @_Z2f1v
+// CG-USE: DICompileUnit
+// CG-USE: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo"
+
+// DI-NOT: define
+// DI: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo"
+
+// DI-USE: define linkonce_odr void @_Z2f1v
+// DI-USE: = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", {{.*}}, flags: DIFlagFwdDecl
diff --git a/clang/test/PCH/cxx2a-defaulted-comparison.cpp b/clang/test/PCH/cxx2a-defaulted-comparison.cpp
index 8fea0fb028478..8aeb1683961af 100644
--- a/clang/test/PCH/cxx2a-defaulted-comparison.cpp
+++ b/clang/test/PCH/cxx2a-defaulted-comparison.cpp
@@ -1,4 +1,4 @@
-// RxN: %clang_cc1 -std=c++2a -verify -Wno-defaulted-function-deleted -include %s %s
+// RUN: %clang_cc1 -std=c++2a -verify -Wno-defaulted-function-deleted -include %s %s
 //
 // RUN: %clang_cc1 -std=c++2a -emit-pch %s -o %t.pch
 // RUN: %clang_cc1 -std=c++2a -include-pch %t.pch %s -verify
diff --git a/clang/test/PCH/pragma-floatcontrol.c b/clang/test/PCH/pragma-floatcontrol.c
index a0918dd53597b..8be46dbd16550 100644
--- a/clang/test/PCH/pragma-floatcontrol.c
+++ b/clang/test/PCH/pragma-floatcontrol.c
@@ -6,6 +6,18 @@
 // Test with pch.
 // RUN: %clang_cc1 %s -DSET -emit-pch -o %t
 // RUN: %clang_cc1 %s -DSET -include-pch %t -emit-llvm -o - | FileCheck --check-prefix=CHECK-EBSTRICT %s
+// RUN: %clang_cc1 %s -ffp-contract=on -DSET -emit-pch -o %t
+// RUN: %clang_cc1 %s -DSET -include-pch %t -emit-llvm -o - | FileCheck --check-prefix=CHECK-EBSTRICT %s
+// RUN: %clang_cc1 %s -menable-no-nans -DSET -emit-pch -o %t
+// RUN: %clang_cc1 %s -DSET -include-pch %t -emit-llvm -o - | FileCheck --check-prefix=CHECK-EBSTRICT %s
+// RUN: %clang_cc1 %s -frounding-math -DSET -emit-pch -o %t
+// RUN: %clang_cc1 %s -DSET -include-pch %t -emit-llvm -o - | FileCheck --check-prefix=CHECK-EBSTRICT %s
+// RUN: %clang_cc1 %s -ffp-exception-behavior=maytrap -DSET -emit-pch -o %t
+// RUN: %clang_cc1 %s -DSET -include-pch %t -emit-llvm -o - | FileCheck --check-prefix=CHECK-EBSTRICT %s
+// RUN: %clang_cc1 %s -ffp-contract=fast -DSET -emit-pch -o %t
+// RUN: %clang_cc1 %s -DSET -include-pch %t -emit-llvm -o - | FileCheck --check-prefix=CHECK-EBSTRICT %s
+// RUN: %clang_cc1 %s -DSET -emit-pch -o %t
+// RUN: %clang_cc1 %s -ffp-contract=on -DSET -include-pch %t -emit-llvm -o - | FileCheck --check-prefix=CHECK-CONTRACT %s
 // RUN: %clang_cc1 %s -DPUSH -emit-pch -o %t
 // RUN: %clang_cc1 %s -DPUSH -verify -include-pch %t
 // RUN: %clang_cc1 %s -DPUSH_POP -emit-pch -o %t
@@ -36,6 +48,7 @@ float fun(float a, float b) {
   // CHECK-LABEL: define float @fun{{.*}}
   //CHECK-EBSTRICT: llvm.experimental.constrained.fmul{{.*}}tonearest{{.*}}strict
   //CHECK-EBSTRICT: llvm.experimental.constrained.fadd{{.*}}tonearest{{.*}}strict
+  //CHECK-CONTRACT: llvm.experimental.constrained.fmuladd{{.*}}tonearest{{.*}}strict
   return a * b + 2;
 }
 #pragma float_control(pop) // expected-warning {{#pragma float_control(pop, ...) failed: stack empty}}
diff --git a/clang/test/Parser/altivec-bool-128.c b/clang/test/Parser/altivec-bool-128.c
new file mode 100644
index 0000000000000..049ca3cc839d2
--- /dev/null
+++ b/clang/test/Parser/altivec-bool-128.c
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -triple=powerpc64-unknown-linux-gnu \
+// RUN:            -target-feature +altivec -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple=powerpc64le-unknown-linux-gnu \
+// RUN:            -target-feature +altivec -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple=powerpc64le-unknown-linux-gnu -target-cpu pwr10 \
+// RUN:            -target-feature +vsx -target-feature -power10-vector \
+// RUN:            -fsyntax-only -verify %s
+
+// Test 'vector bool __int128' type.
+
+// These should have errors.
+__vector bool __int128 v1_bi128;          // expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+__vector __bool __int128 v2_bi128;        // expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+vector bool __int128 v3_bi128;            // expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+vector __bool __int128 v4_bi128;          // expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+__vector bool unsigned __int128 v5_bi128; // expected-error {{cannot use 'unsigned' with '__vector bool'}} expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+__vector bool signed __int128 v6_bi128;   // expected-error {{cannot use 'signed' with '__vector bool'}} expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+vector bool unsigned __int128 v7_bi128;   // expected-error {{cannot use 'unsigned' with '__vector bool'}} expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+vector bool signed __int128 v8_bi128;     // expected-error {{cannot use 'signed' with '__vector bool'}} expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+__vector __bool signed __int128 v9_bi128; // expected-error {{cannot use 'signed' with '__vector bool'}} expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+vector __bool signed __int128 v10_bi128;  // expected-error {{cannot use 'signed' with '__vector bool'}} expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
diff --git a/clang/test/Parser/cxx-altivec-bool-128.cpp b/clang/test/Parser/cxx-altivec-bool-128.cpp
new file mode 100644
index 0000000000000..36ffefc9a18bf
--- /dev/null
+++ b/clang/test/Parser/cxx-altivec-bool-128.cpp
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -triple=powerpc64-unknown-linux-gnu \
+// RUN:            -target-feature +altivec -fsyntax-only -verify -std=c++11 %s
+// RUN: %clang_cc1 -triple=powerpc64le-unknown-linux-gnu \
+// RUN:            -target-feature +altivec -fsyntax-only -verify -std=c++11 %s
+// RUN: %clang_cc1 -triple=powerpc64le-unknown-linux-gnu -target-cpu pwr10 \
+// RUN:            -target-feature +altivec -target-feature +vsx \
+// RUN:            -target-feature -power10-vector -fsyntax-only -verify %s
+
+#include <altivec.h>
+
+// Test 'vector bool __int128' type.
+
+// These should have errors.
+__vector bool __int128 v1_bi128;          // expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+__vector __bool __int128 v2_bi128;        // expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+vector bool __int128 v3_bi128;            // expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+vector __bool __int128 v4_bi128;          // expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+__vector bool unsigned __int128 v5_bi128; // expected-error {{cannot use 'unsigned' with '__vector bool'}} expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+__vector bool signed __int128 v6_bi128;   // expected-error {{cannot use 'signed' with '__vector bool'}} expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+vector bool unsigned __int128 v7_bi128;   // expected-error {{cannot use 'unsigned' with '__vector bool'}} expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+vector bool signed __int128 v8_bi128;     // expected-error {{cannot use 'signed' with '__vector bool'}} expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+__vector __bool signed __int128 v9_bi128; // expected-error {{cannot use 'signed' with '__vector bool'}} expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
+vector __bool signed __int128 v10_bi128;  // expected-error {{cannot use 'signed' with '__vector bool'}} expected-error {{use of '__int128' with '__vector bool' requires VSX support enabled (on POWER10 or later)}}
diff --git a/clang/test/Parser/cxx-template-decl.cpp b/clang/test/Parser/cxx-template-decl.cpp
index 64e7ca921f575..24cc13cde91fa 100644
--- a/clang/test/Parser/cxx-template-decl.cpp
+++ b/clang/test/Parser/cxx-template-decl.cpp
@@ -286,3 +286,17 @@ namespace PR45239 {
   template<int> int b;
   template<int> auto f() -> b<0>; // expected-error +{{}}
 }
+
+namespace NoCrashOnNullNNSTypoCorrection {
+
+int AddObservation(); // expected-note {{declared here}}
+
+template <typename T, typename... Args> // expected-note {{template parameter is declared here}}
+class UsingImpl {};
+class AddObservation {
+  using Using =
+    UsingImpl<AddObservationFn, const int>; // expected-error {{use of undeclared identifier 'AddObservationFn'; did you mean}} \
+                                               expected-error {{template argument for template type parameter must be a type}}
+};
+
+}
diff --git a/clang/test/Parser/expressions.cpp b/clang/test/Parser/expressions.cpp
new file mode 100644
index 0000000000000..6d3123bd1de7c
--- /dev/null
+++ b/clang/test/Parser/expressions.cpp
@@ -0,0 +1,38 @@
+// RUN: %clang_cc1 -std=c++2a -verify %s
+
+void *operator new(__SIZE_TYPE__, void*);
+
+// Check that we give a good diagnostic for an attempt to use a postfix
+// operator after a unary-expression.
+namespace postfix_after_unary {
+  struct A { int n; };
+  int &a = new A->n; // expected-error {{expression cannot be followed by a postfix '->' operator; add parentheses}}
+
+  struct B { B(int); int operator()(int); };
+  int n = new (0) (B) (int()) (int()); // expected-error {{cannot be followed by a postfix '(}} expected-error {{not a function or function pointer}}
+
+  char x = sizeof(int)["hello"]; // expected-error {{cannot be followed by a postfix '[}}
+  char y = alignof(int)["hello"]; // expected-error {{cannot be followed by a postfix '[}}
+  char z = noexcept(0)["hello"]; // expected-error {{cannot be followed by a postfix '[}}
+  char w = requires { x == x; }["ny"]; // expected-error {{cannot be followed by a postfix '[}}
+
+  int f() {
+  label:
+    return &&label->n; // expected-error {{cannot be followed by a postfix}} expected-error {{not a structure or union}}
+  }
+
+  char k = sizeof(int) // expected-error {{expected ';'}}
+  [[noreturn]] void g();
+}
+
+// Check that we do parse postfix-expression suffixes after some more unusual
+// kinds of postfix-expressions (null literals and builtin calls).
+namespace unusual_primary_exprs {
+  int a = nullptr["foo"]; // expected-error {{array subscript is not an integer}}
+  int b = __builtin_COLUMN()["sufficiently long string constant"];
+  int c = __builtin_available(*)["ny"];
+  static_assert(__null["foo"] == 'f'); // FIXME: Warn about converting __null to integer in array subscripting.
+  static_assert(__is_standard_layout(int)["ny"] == 'y');
+  static_assert(__array_rank(int[1][2])["0123"] == '2');
+  static_assert(__is_lvalue_expr(a)["ny"] == 'y');
+}
diff --git a/clang/test/Parser/p10-vector-bool-128.c b/clang/test/Parser/p10-vector-bool-128.c
new file mode 100644
index 0000000000000..df14b04e7e37c
--- /dev/null
+++ b/clang/test/Parser/p10-vector-bool-128.c
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -triple=powerpc64-unknown-linux-gnu -target-cpu pwr10 \
+// RUN:            -target-feature +vsx -target-feature +power10-vector \
+// RUN:            -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple=powerpc64le-unknown-linux-gnu -target-cpu pwr10 \
+// RUN:            -target-feature +power10-vector -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Test legitimate uses of 'vector bool __int128' with VSX.
+__vector bool __int128 v1_bi128;
+__vector __bool __int128 v2_bi128;
+vector bool __int128 v3_bi128;
+vector __bool __int128 v4_bi128;
diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c
index 784c67cd643d1..bef145930697f 100644
--- a/clang/test/Preprocessor/aarch64-target-features.c
+++ b/clang/test/Preprocessor/aarch64-target-features.c
@@ -93,7 +93,39 @@
 // RUN: %clang -target aarch64 -mtune=cyclone -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MTUNE-CYCLONE %s
 
 // RUN: %clang -target aarch64-none-linux-gnu -march=armv8-a+sve -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE %s
-// CHECK-SVE-NOT: __ARM_FEATURE_SVE 1
+// CHECK-SVE: __ARM_FEATURE_SVE 1
+
+// RUN: %clang -target aarch64-none-linux-gnu -march=armv8-a+sve+bf16 -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE-BF16 %s
+// CHECK-SVE-BF16: __ARM_FEATURE_BF16_SCALAR_ARITHMETIC 1
+// CHECK-SVE-BF16: __ARM_FEATURE_SVE 1
+// CHECK-SVE-BF16: __ARM_FEATURE_SVE_BF16 1
+
+// RUN: %clang -target aarch64-none-linux-gnu -march=armv8-a+sve+i8mm -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE-I8MM %s
+// CHECK-SVE-I8MM: __ARM_FEATURE_SVE 1
+// CHECK-SVE-I8MM: __ARM_FEATURE_SVE_MATMUL_INT8 1
+
+// RUN: %clang -target aarch64-none-linux-gnu -march=armv8-a+sve+f32mm -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE-F32MM %s
+// CHECK-SVE-F32MM: __ARM_FEATURE_SVE 1
+// CHECK-SVE-F32MM: __ARM_FEATURE_SVE_MATMUL_FP32 1
+
+// RUN: %clang -target aarch64-none-linux-gnu -march=armv8-a+sve+f64mm -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE-F64MM %s
+// CHECK-SVE-F64MM: __ARM_FEATURE_SVE 1
+// CHECK-SVE-F64MM: __ARM_FEATURE_SVE_MATMUL_FP64 1
+
+// The following tests may need to be revised in the future since
+// SVE2 is currently still part of Future Architecture Technologies
+// (https://developer.arm.com/docs/ddi0602/latest)
+//
+// RUN: %clang -target aarch64-none-linux-gnu -march=armv8-a+sve2 -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE2 %s
+// CHECK-SVE2: __ARM_FEATURE_SVE2 1
+// RUN: %clang -target aarch64-none-linux-gnu -march=armv8-a+sve2-aes -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE2AES %s
+// CHECK-SVE2AES: __ARM_FEATURE_SVE2_AES 1
+// RUN: %clang -target aarch64-none-linux-gnu -march=armv8-a+sve2-sha3 -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE2SHA3 %s
+// CHECK-SVE2SHA3: __ARM_FEATURE_SVE2_SHA3 1
+// RUN: %clang -target aarch64-none-linux-gnu -march=armv8-a+sve2-sm4 -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE2SM4 %s
+// CHECK-SVE2SM4: __ARM_FEATURE_SVE2_SM4 1
+// RUN: %clang -target aarch64-none-linux-gnu -march=armv8-a+sve2-bitperm -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE2BITPERM %s
+// CHECK-SVE2BITPERM: __ARM_FEATURE_SVE2_BITPERM 1
 
 // RUN: %clang -target aarch64-none-linux-gnu -march=armv8.2a+dotprod -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-DOTPROD %s
 // CHECK-DOTPROD: __ARM_FEATURE_DOTPROD 1
diff --git a/clang/test/Preprocessor/init-ve.c b/clang/test/Preprocessor/init-ve.c
new file mode 100644
index 0000000000000..4686315f4ea06
--- /dev/null
+++ b/clang/test/Preprocessor/init-ve.c
@@ -0,0 +1,274 @@
+/// Check predefinitions for NEC Aurora VE
+/// REQUIRES: ve-registered-target
+
+// RUN: %clang_cc1 -E -dM -triple=ve < /dev/null | \
+// RUN:     FileCheck -match-full-lines -check-prefix VE %s
+// RUN: %clang_cc1 -x c++ -E -dM -triple=ve < /dev/null | \
+// RUN:     FileCheck -match-full-lines -check-prefix VE -check-prefix VE-CXX %s
+//
+// VE:#define _LP64 1
+// VE:#define __BIGGEST_ALIGNMENT__ 8
+// VE:#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
+// VE:#define __CHAR16_TYPE__ unsigned short
+// VE:#define __CHAR32_TYPE__ unsigned int
+// VE:#define __CHAR_BIT__ 8
+// VE:#define __DBL_DECIMAL_DIG__ 17
+// VE:#define __DBL_DENORM_MIN__ 4.9406564584124654e-324
+// VE:#define __DBL_DIG__ 15
+// VE:#define __DBL_EPSILON__ 2.2204460492503131e-16
+// VE:#define __DBL_HAS_DENORM__ 1
+// VE:#define __DBL_HAS_INFINITY__ 1
+// VE:#define __DBL_HAS_QUIET_NAN__ 1
+// VE:#define __DBL_MANT_DIG__ 53
+// VE:#define __DBL_MAX_10_EXP__ 308
+// VE:#define __DBL_MAX_EXP__ 1024
+// VE:#define __DBL_MAX__ 1.7976931348623157e+308
+// VE:#define __DBL_MIN_10_EXP__ (-307)
+// VE:#define __DBL_MIN_EXP__ (-1021)
+// VE:#define __DBL_MIN__ 2.2250738585072014e-308
+// VE:#define __DECIMAL_DIG__ __LDBL_DECIMAL_DIG__
+// VE-NOT:#define __FAST_MATH__ 1
+// VE:#define __FLT_DECIMAL_DIG__ 9
+// VE:#define __FLT_DENORM_MIN__ 1.40129846e-45F
+// VE:#define __FLT_DIG__ 6
+// VE:#define __FLT_EPSILON__ 1.19209290e-7F
+// VE:#define __FLT_EVAL_METHOD__ 0
+// VE:#define __FLT_HAS_DENORM__ 1
+// VE:#define __FLT_HAS_INFINITY__ 1
+// VE:#define __FLT_HAS_QUIET_NAN__ 1
+// VE:#define __FLT_MANT_DIG__ 24
+// VE:#define __FLT_MAX_10_EXP__ 38
+// VE:#define __FLT_MAX_EXP__ 128
+// VE:#define __FLT_MAX__ 3.40282347e+38F
+// VE:#define __FLT_MIN_10_EXP__ (-37)
+// VE:#define __FLT_MIN_EXP__ (-125)
+// VE:#define __FLT_MIN__ 1.17549435e-38F
+// VE:#define __FLT_RADIX__ 2
+// VE:#define __INT16_C_SUFFIX__
+// VE:#define __INT16_FMTd__ "hd"
+// VE:#define __INT16_FMTi__ "hi"
+// VE:#define __INT16_MAX__ 32767
+// VE:#define __INT16_TYPE__ short
+// VE:#define __INT32_C_SUFFIX__
+// VE:#define __INT32_FMTd__ "d"
+// VE:#define __INT32_FMTi__ "i"
+// VE:#define __INT32_MAX__ 2147483647
+// VE:#define __INT32_TYPE__ int
+// VE:#define __INT64_C_SUFFIX__ L
+// VE:#define __INT64_FMTd__ "ld"
+// VE:#define __INT64_FMTi__ "li"
+// VE:#define __INT64_MAX__ 9223372036854775807L
+// VE:#define __INT64_TYPE__ long int
+// VE:#define __INT8_C_SUFFIX__
+// VE:#define __INT8_FMTd__ "hhd"
+// VE:#define __INT8_FMTi__ "hhi"
+// VE:#define __INT8_MAX__ 127
+// VE:#define __INT8_TYPE__ signed char
+// VE:#define __INTMAX_C_SUFFIX__ L
+// VE:#define __INTMAX_FMTd__ "ld"
+// VE:#define __INTMAX_FMTi__ "li"
+// VE:#define __INTMAX_MAX__ 9223372036854775807L
+// VE:#define __INTMAX_TYPE__ long int
+// VE:#define __INTMAX_WIDTH__ 64
+// VE:#define __INTPTR_FMTd__ "ld"
+// VE:#define __INTPTR_FMTi__ "li"
+// VE:#define __INTPTR_MAX__ 9223372036854775807L
+// VE:#define __INTPTR_TYPE__ long int
+// VE:#define __INTPTR_WIDTH__ 64
+// VE:#define __INT_FAST16_FMTd__ "hd"
+// VE:#define __INT_FAST16_FMTi__ "hi"
+// VE:#define __INT_FAST16_MAX__ 32767
+// VE:#define __INT_FAST16_TYPE__ short
+// VE:#define __INT_FAST32_FMTd__ "d"
+// VE:#define __INT_FAST32_FMTi__ "i"
+// VE:#define __INT_FAST32_MAX__ 2147483647
+// VE:#define __INT_FAST32_TYPE__ int
+// VE:#define __INT_FAST64_FMTd__ "ld"
+// VE:#define __INT_FAST64_FMTi__ "li"
+// VE:#define __INT_FAST64_MAX__ 9223372036854775807L
+// VE:#define __INT_FAST64_TYPE__ long int
+// VE:#define __INT_FAST8_FMTd__ "hhd"
+// VE:#define __INT_FAST8_FMTi__ "hhi"
+// VE:#define __INT_FAST8_MAX__ 127
+// VE:#define __INT_FAST8_TYPE__ signed char
+// VE:#define __INT_LEAST16_FMTd__ "hd"
+// VE:#define __INT_LEAST16_FMTi__ "hi"
+// VE:#define __INT_LEAST16_MAX__ 32767
+// VE:#define __INT_LEAST16_TYPE__ short
+// VE:#define __INT_LEAST32_FMTd__ "d"
+// VE:#define __INT_LEAST32_FMTi__ "i"
+// VE:#define __INT_LEAST32_MAX__ 2147483647
+// VE:#define __INT_LEAST32_TYPE__ int
+// VE:#define __INT_LEAST64_FMTd__ "ld"
+// VE:#define __INT_LEAST64_FMTi__ "li"
+// VE:#define __INT_LEAST64_MAX__ 9223372036854775807L
+// VE:#define __INT_LEAST64_TYPE__ long int
+// VE:#define __INT_LEAST8_FMTd__ "hhd"
+// VE:#define __INT_LEAST8_FMTi__ "hhi"
+// VE:#define __INT_LEAST8_MAX__ 127
+// VE:#define __INT_LEAST8_TYPE__ signed char
+// VE:#define __INT_MAX__ 2147483647
+// VE:#define __LDBL_DECIMAL_DIG__ 36
+// VE:#define __LDBL_DENORM_MIN__ 6.47517511943802511092443895822764655e-4966L
+// VE:#define __LDBL_DIG__ 33
+// VE:#define __LDBL_EPSILON__ 1.92592994438723585305597794258492732e-34L
+// VE:#define __LDBL_HAS_DENORM__ 1
+// VE:#define __LDBL_HAS_INFINITY__ 1
+// VE:#define __LDBL_HAS_QUIET_NAN__ 1
+// VE:#define __LDBL_MANT_DIG__ 113
+// VE:#define __LDBL_MAX_10_EXP__ 4932
+// VE:#define __LDBL_MAX_EXP__ 16384
+// VE:#define __LDBL_MAX__ 1.18973149535723176508575932662800702e+4932L
+// VE:#define __LDBL_MIN_10_EXP__ (-4931)
+// VE:#define __LDBL_MIN_EXP__ (-16381)
+// VE:#define __LDBL_MIN__ 3.36210314311209350626267781732175260e-4932L
+// VE:#define __LITTLE_ENDIAN__ 1
+// VE-NOT:#define __LONGDOUBLE128 1
+// VE-NOT:#define __LONG_DOUBLE_128__ 1
+// VE:#define __LONG_LONG_MAX__ 9223372036854775807LL
+// VE:#define __LONG_MAX__ 9223372036854775807L
+// VE:#define __LP64__ 1
+// VE:#define __NEC__ 1
+// VE-NOT:#define __OPTIMIZE__
+// VE:#define __POINTER_WIDTH__ 64
+// VE:#define __PTRDIFF_FMTd__ "ld"
+// VE:#define __PTRDIFF_FMTi__ "li"
+// VE:#define __PTRDIFF_MAX__ 9223372036854775807L
+// VE:#define __PTRDIFF_TYPE__ long int
+// VE:#define __PTRDIFF_WIDTH__ 64
+// VE:#define __SCHAR_MAX__ 127
+// VE:#define __SHRT_MAX__ 32767
+// VE:#define __SIG_ATOMIC_MAX__ 2147483647
+// VE:#define __SIG_ATOMIC_WIDTH__ 32
+// VE:#define __SIZEOF_DOUBLE__ 8
+// VE:#define __SIZEOF_FLOAT__ 4
+// VE:#define __SIZEOF_INT128__ 16
+// VE:#define __SIZEOF_INT__ 4
+// VE:#define __SIZEOF_LONG_DOUBLE__ 16
+// VE:#define __SIZEOF_LONG_LONG__ 8
+// VE:#define __SIZEOF_LONG__ 8
+// VE:#define __SIZEOF_POINTER__ 8
+// VE:#define __SIZEOF_PTRDIFF_T__ 8
+// VE:#define __SIZEOF_SHORT__ 2
+// VE:#define __SIZEOF_SIZE_T__ 8
+// VE:#define __SIZEOF_WCHAR_T__ 4
+// VE:#define __SIZEOF_WINT_T__ 4
+// VE:#define __SIZE_FMTX__ "lX"
+// VE:#define __SIZE_FMTo__ "lo"
+// VE:#define __SIZE_FMTu__ "lu"
+// VE:#define __SIZE_FMTx__ "lx"
+// VE:#define __SIZE_MAX__ 18446744073709551615UL
+// VE:#define __SIZE_TYPE__ long unsigned int
+// VE:#define __SIZE_WIDTH__ 64
+// VE-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 16UL
+// VE:#define __STDC_HOSTED__ 1
+// VE:#define __UINT16_C_SUFFIX__
+// VE:#define __UINT16_FMTX__ "hX"
+// VE:#define __UINT16_FMTo__ "ho"
+// VE:#define __UINT16_FMTu__ "hu"
+// VE:#define __UINT16_FMTx__ "hx"
+// VE:#define __UINT16_MAX__ 65535
+// VE:#define __UINT16_TYPE__ unsigned short
+// VE:#define __UINT32_C_SUFFIX__ U
+// VE:#define __UINT32_FMTX__ "X"
+// VE:#define __UINT32_FMTo__ "o"
+// VE:#define __UINT32_FMTu__ "u"
+// VE:#define __UINT32_FMTx__ "x"
+// VE:#define __UINT32_MAX__ 4294967295U
+// VE:#define __UINT32_TYPE__ unsigned int
+// VE:#define __UINT64_C_SUFFIX__ UL
+// VE:#define __UINT64_FMTX__ "lX"
+// VE:#define __UINT64_FMTo__ "lo"
+// VE:#define __UINT64_FMTu__ "lu"
+// VE:#define __UINT64_FMTx__ "lx"
+// VE:#define __UINT64_MAX__ 18446744073709551615UL
+// VE:#define __UINT64_TYPE__ long unsigned int
+// VE:#define __UINT8_C_SUFFIX__
+// VE:#define __UINT8_FMTX__ "hhX"
+// VE:#define __UINT8_FMTo__ "hho"
+// VE:#define __UINT8_FMTu__ "hhu"
+// VE:#define __UINT8_FMTx__ "hhx"
+// VE:#define __UINT8_MAX__ 255
+// VE:#define __UINT8_TYPE__ unsigned char
+// VE:#define __UINTMAX_C_SUFFIX__ UL
+// VE:#define __UINTMAX_FMTX__ "lX"
+// VE:#define __UINTMAX_FMTo__ "lo"
+// VE:#define __UINTMAX_FMTu__ "lu"
+// VE:#define __UINTMAX_FMTx__ "lx"
+// VE:#define __UINTMAX_MAX__ 18446744073709551615UL
+// VE:#define __UINTMAX_TYPE__ long unsigned int
+// VE:#define __UINTMAX_WIDTH__ 64
+// VE:#define __UINTPTR_FMTX__ "lX"
+// VE:#define __UINTPTR_FMTo__ "lo"
+// VE:#define __UINTPTR_FMTu__ "lu"
+// VE:#define __UINTPTR_FMTx__ "lx"
+// VE:#define __UINTPTR_MAX__ 18446744073709551615UL
+// VE:#define __UINTPTR_TYPE__ long unsigned int
+// VE:#define __UINTPTR_WIDTH__ 64
+// VE:#define __UINT_FAST16_FMTX__ "hX"
+// VE:#define __UINT_FAST16_FMTo__ "ho"
+// VE:#define __UINT_FAST16_FMTu__ "hu"
+// VE:#define __UINT_FAST16_FMTx__ "hx"
+// VE:#define __UINT_FAST16_MAX__ 65535
+// VE:#define __UINT_FAST16_TYPE__ unsigned short
+// VE:#define __UINT_FAST32_FMTX__ "X"
+// VE:#define __UINT_FAST32_FMTo__ "o"
+// VE:#define __UINT_FAST32_FMTu__ "u"
+// VE:#define __UINT_FAST32_FMTx__ "x"
+// VE:#define __UINT_FAST32_MAX__ 4294967295U
+// VE:#define __UINT_FAST32_TYPE__ unsigned int
+// VE:#define __UINT_FAST64_FMTX__ "lX"
+// VE:#define __UINT_FAST64_FMTo__ "lo"
+// VE:#define __UINT_FAST64_FMTu__ "lu"
+// VE:#define __UINT_FAST64_FMTx__ "lx"
+// VE:#define __UINT_FAST64_MAX__ 18446744073709551615UL
+// VE:#define __UINT_FAST64_TYPE__ long unsigned int
+// VE:#define __UINT_FAST8_FMTX__ "hhX"
+// VE:#define __UINT_FAST8_FMTo__ "hho"
+// VE:#define __UINT_FAST8_FMTu__ "hhu"
+// VE:#define __UINT_FAST8_FMTx__ "hhx"
+// VE:#define __UINT_FAST8_MAX__ 255
+// VE:#define __UINT_FAST8_TYPE__ unsigned char
+// VE:#define __UINT_LEAST16_FMTX__ "hX"
+// VE:#define __UINT_LEAST16_FMTo__ "ho"
+// VE:#define __UINT_LEAST16_FMTu__ "hu"
+// VE:#define __UINT_LEAST16_FMTx__ "hx"
+// VE:#define __UINT_LEAST16_MAX__ 65535
+// VE:#define __UINT_LEAST16_TYPE__ unsigned short
+// VE:#define __UINT_LEAST32_FMTX__ "X"
+// VE:#define __UINT_LEAST32_FMTo__ "o"
+// VE:#define __UINT_LEAST32_FMTu__ "u"
+// VE:#define __UINT_LEAST32_FMTx__ "x"
+// VE:#define __UINT_LEAST32_MAX__ 4294967295U
+// VE:#define __UINT_LEAST32_TYPE__ unsigned int
+// VE:#define __UINT_LEAST64_FMTX__ "lX"
+// VE:#define __UINT_LEAST64_FMTo__ "lo"
+// VE:#define __UINT_LEAST64_FMTu__ "lu"
+// VE:#define __UINT_LEAST64_FMTx__ "lx"
+// VE:#define __UINT_LEAST64_MAX__ 18446744073709551615UL
+// VE:#define __UINT_LEAST64_TYPE__ long unsigned int
+// VE:#define __UINT_LEAST8_FMTX__ "hhX"
+// VE:#define __UINT_LEAST8_FMTo__ "hho"
+// VE:#define __UINT_LEAST8_FMTu__ "hhu"
+// VE:#define __UINT_LEAST8_FMTx__ "hhx"
+// VE:#define __UINT_LEAST8_MAX__ 255
+// VE:#define __UINT_LEAST8_TYPE__ unsigned char
+// VE:#define __USER_LABEL_PREFIX__
+// VE-NOT:#define __VECTOR__
+// VE:#define __WCHAR_MAX__ 4294967295U
+// VE:#define __WCHAR_TYPE__ unsigned int
+// VE:#define __WCHAR_UNSIGNED__ 1
+// VE:#define __WCHAR_WIDTH__ 32
+// VE:#define __WINT_MAX__ 4294967295U
+// VE:#define __WINT_TYPE__ unsigned int
+// VE:#define __WINT_UNSIGNED__ 1
+// VE:#define __WINT_WIDTH__ 32
+// VE:#define __linux 1
+// VE:#define __linux__ 1
+// VE:#define __llvm__ 1
+// VE:#define __unix 1
+// VE:#define __unix__ 1
+// VE:#define __ve 1
+// VE:#define __ve__ 1
+// VE:#define linux 1
+// VE:#define unix 1
diff --git a/clang/test/Preprocessor/predefined-arch-macros.c b/clang/test/Preprocessor/predefined-arch-macros.c
index f7de81028327e..e457a0479b330 100644
--- a/clang/test/Preprocessor/predefined-arch-macros.c
+++ b/clang/test/Preprocessor/predefined-arch-macros.c
@@ -2379,6 +2379,7 @@
 // CHECK_AMDFAM10_M32: #define __LZCNT__ 1
 // CHECK_AMDFAM10_M32: #define __MMX__ 1
 // CHECK_AMDFAM10_M32: #define __POPCNT__ 1
+// CHECK_AMDFAM10_M32: #define __PRFCHW__ 1
 // CHECK_AMDFAM10_M32: #define __SSE2_MATH__ 1
 // CHECK_AMDFAM10_M32: #define __SSE2__ 1
 // CHECK_AMDFAM10_M32: #define __SSE3__ 1
@@ -2396,9 +2397,11 @@
 // RUN:   | FileCheck -match-full-lines %s -check-prefix=CHECK_AMDFAM10_M64
 // CHECK_AMDFAM10_M64: #define __3dNOW_A__ 1
 // CHECK_AMDFAM10_M64: #define __3dNOW__ 1
+// CHECK_AMDFAM10_M64: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 1
 // CHECK_AMDFAM10_M64: #define __LZCNT__ 1
 // CHECK_AMDFAM10_M64: #define __MMX__ 1
 // CHECK_AMDFAM10_M64: #define __POPCNT__ 1
+// CHECK_AMDFAM10_M64: #define __PRFCHW__ 1
 // CHECK_AMDFAM10_M64: #define __SSE2_MATH__ 1
 // CHECK_AMDFAM10_M64: #define __SSE2__ 1
 // CHECK_AMDFAM10_M64: #define __SSE3__ 1
@@ -2750,9 +2753,11 @@
 // CHECK_BDVER4_M32: #define __LWP__ 1
 // CHECK_BDVER4_M32: #define __LZCNT__ 1
 // CHECK_BDVER4_M32: #define __MMX__ 1
+// CHECK_BDVER4_M32: #define __MOVBE__ 1
 // CHECK_BDVER4_M32: #define __PCLMUL__ 1
 // CHECK_BDVER4_M32: #define __POPCNT__ 1
 // CHECK_BDVER4_M32: #define __PRFCHW__ 1
+// CHECK_BDVER4_M32: #define __RDRND__ 1
 // CHECK_BDVER4_M32: #define __SSE2_MATH__ 1
 // CHECK_BDVER4_M32: #define __SSE2__ 1
 // CHECK_BDVER4_M32: #define __SSE3__ 1
@@ -2788,9 +2793,11 @@
 // CHECK_BDVER4_M64: #define __LWP__ 1
 // CHECK_BDVER4_M64: #define __LZCNT__ 1
 // CHECK_BDVER4_M64: #define __MMX__ 1
+// CHECK_BDVER4_M64: #define __MOVBE__ 1
 // CHECK_BDVER4_M64: #define __PCLMUL__ 1
 // CHECK_BDVER4_M64: #define __POPCNT__ 1
 // CHECK_BDVER4_M64: #define __PRFCHW__ 1
+// CHECK_BDVER4_M64: #define __RDRND__ 1
 // CHECK_BDVER4_M64: #define __SSE2_MATH__ 1
 // CHECK_BDVER4_M64: #define __SSE2__ 1
 // CHECK_BDVER4_M64: #define __SSE3__ 1
@@ -2825,6 +2832,7 @@
 // CHECK_ZNVER1_M32: #define __CLFLUSHOPT__ 1
 // CHECK_ZNVER1_M32: #define __CLZERO__ 1
 // CHECK_ZNVER1_M32: #define __F16C__ 1
+// CHECK_ZNVER1_M32-NOT: #define __FMA4__ 1
 // CHECK_ZNVER1_M32: #define __FMA__ 1
 // CHECK_ZNVER1_M32: #define __FSGSBASE__ 1
 // CHECK_ZNVER1_M32: #define __LZCNT__ 1
@@ -2845,6 +2853,8 @@
 // CHECK_ZNVER1_M32: #define __SSE_MATH__ 1
 // CHECK_ZNVER1_M32: #define __SSE__ 1
 // CHECK_ZNVER1_M32: #define __SSSE3__ 1
+// CHECK_ZNVER1_M32-NOT: #define __TBM__ 1
+// CHECK_ZNVER1_M32-NOT: #define __XOP__ 1
 // CHECK_ZNVER1_M32: #define __XSAVEC__ 1
 // CHECK_ZNVER1_M32: #define __XSAVEOPT__ 1
 // CHECK_ZNVER1_M32: #define __XSAVES__ 1
@@ -2869,6 +2879,7 @@
 // CHECK_ZNVER1_M64: #define __CLFLUSHOPT__ 1
 // CHECK_ZNVER1_M64: #define __CLZERO__ 1
 // CHECK_ZNVER1_M64: #define __F16C__ 1
+// CHECK_ZNVER1_M64-NOT: #define __FMA4__ 1
 // CHECK_ZNVER1_M64: #define __FMA__ 1
 // CHECK_ZNVER1_M64: #define __FSGSBASE__ 1
 // CHECK_ZNVER1_M64: #define __LZCNT__ 1
@@ -2889,6 +2900,8 @@
 // CHECK_ZNVER1_M64: #define __SSE_MATH__ 1
 // CHECK_ZNVER1_M64: #define __SSE__ 1
 // CHECK_ZNVER1_M64: #define __SSSE3__ 1
+// CHECK_ZNVER1_M64-NOT: #define __TBM__ 1
+// CHECK_ZNVER1_M64-NOT: #define __XOP__ 1
 // CHECK_ZNVER1_M64: #define __XSAVEC__ 1
 // CHECK_ZNVER1_M64: #define __XSAVEOPT__ 1
 // CHECK_ZNVER1_M64: #define __XSAVES__ 1
@@ -2916,6 +2929,7 @@
 // CHECK_ZNVER2_M32: #define __CLWB__ 1
 // CHECK_ZNVER2_M32: #define __CLZERO__ 1
 // CHECK_ZNVER2_M32: #define __F16C__ 1
+// CHECK_ZNVER2_M32-NOT: #define __FMA4__ 1
 // CHECK_ZNVER2_M32: #define __FMA__ 1
 // CHECK_ZNVER2_M32: #define __FSGSBASE__ 1
 // CHECK_ZNVER2_M32: #define __LZCNT__ 1
@@ -2936,7 +2950,9 @@
 // CHECK_ZNVER2_M32: #define __SSE_MATH__ 1
 // CHECK_ZNVER2_M32: #define __SSE__ 1
 // CHECK_ZNVER2_M32: #define __SSSE3__ 1
+// CHECK_ZNVER2_M32-NOT: #define __TBM__ 1
 // CHECK_ZNVER2_M32: #define __WBNOINVD__ 1
+// CHECK_ZNVER2_M32-NOT: #define __XOP__ 1
 // CHECK_ZNVER2_M32: #define __XSAVEC__ 1
 // CHECK_ZNVER2_M32: #define __XSAVEOPT__ 1
 // CHECK_ZNVER2_M32: #define __XSAVES__ 1
@@ -2962,6 +2978,7 @@
 // CHECK_ZNVER2_M64: #define __CLWB__ 1
 // CHECK_ZNVER2_M64: #define __CLZERO__ 1
 // CHECK_ZNVER2_M64: #define __F16C__ 1
+// CHECK_ZNVER2_M64-NOT: #define __FMA4__ 1
 // CHECK_ZNVER2_M64: #define __FMA__ 1
 // CHECK_ZNVER2_M64: #define __FSGSBASE__ 1
 // CHECK_ZNVER2_M64: #define __LZCNT__ 1
@@ -2982,7 +2999,9 @@
 // CHECK_ZNVER2_M64: #define __SSE_MATH__ 1
 // CHECK_ZNVER2_M64: #define __SSE__ 1
 // CHECK_ZNVER2_M64: #define __SSSE3__ 1
+// CHECK_ZNVER2_M64-NOT: #define __TBM__ 1
 // CHECK_ZNVER2_M64: #define __WBNOINVD__ 1
+// CHECK_ZNVER2_M64-NOT: #define __XOP__ 1
 // CHECK_ZNVER2_M64: #define __XSAVEC__ 1
 // CHECK_ZNVER2_M64: #define __XSAVEOPT__ 1
 // CHECK_ZNVER2_M64: #define __XSAVES__ 1
diff --git a/clang/test/Preprocessor/warn-macro-undef.c b/clang/test/Preprocessor/warn-macro-undef.c
new file mode 100644
index 0000000000000..e7d16c836076a
--- /dev/null
+++ b/clang/test/Preprocessor/warn-macro-undef.c
@@ -0,0 +1,52 @@
+// RUN: %clang_cc1 %s -Eonly -Wundef -verify=undef
+// RUN: %clang_cc1 %s -Eonly -Wundef-prefix=A,BC -verify=undef-prefix
+// RUN: %clang_cc1 %s -Eonly -Wundef -Wundef-prefix=A,BC -verify=both
+// RUN: %clang_cc1 %s -Eonly -Werror=undef -verify=undef-error
+// RUN: %clang_cc1 %s -Eonly -Werror=undef-prefix -Wundef-prefix=A,BC -verify=undef-prefix-error
+// RUN: %clang_cc1 %s -Eonly -Werror=undef -Wundef-prefix=A,BC -verify=both-error
+
+extern int x;
+
+#if AB // #1
+#endif
+// undef-warning@#1 {{'AB' is not defined, evaluates to 0}}
+// undef-prefix-warning@#1 {{'AB' is not defined, evaluates to 0}}
+// both-warning@#1 {{'AB' is not defined, evaluates to 0}}
+// undef-error-error@#1 {{'AB' is not defined, evaluates to 0}}
+// undef-prefix-error-error@#1 {{'AB' is not defined, evaluates to 0}}
+// both-error-error@#1 {{'AB' is not defined, evaluates to 0}}
+
+#if B // #2
+#endif
+// undef-warning@#2 {{'B' is not defined, evaluates to 0}}
+// no warning for undef-prefix
+// both-warning@#2 {{'B' is not defined, evaluates to 0}}
+// undef-error-error@#2 {{'B' is not defined, evaluates to 0}}
+// no error for undef-prefix
+// both-error-error@#2 {{'B' is not defined, evaluates to 0}}
+
+#define BC 0
+#if BC // no warning/error
+#endif
+
+#undef BC
+#if BC // #3
+#endif
+// undef-warning@#3 {{'BC' is not defined, evaluates to 0}}
+// undef-prefix-warning@#3 {{'BC' is not defined, evaluates to 0}}
+// both-warning@#3 {{'BC' is not defined, evaluates to 0}}
+// undef-error-error@#3 {{'BC' is not defined, evaluates to 0}}
+// undef-prefix-error-error@#3 {{'BC' is not defined, evaluates to 0}}
+// both-error-error@#3 {{'BC' is not defined, evaluates to 0}}
+
+// Test that #pragma-enabled 'Wundef' can override 'Wundef-prefix'
+#pragma clang diagnostic error "-Wundef"
+
+#if C // #4
+#endif
+// undef-error@#4 {{'C' is not defined, evaluates to 0}}
+// undef-prefix-error@#4 {{'C' is not defined, evaluates to 0}}
+// both-error@#4 {{'C' is not defined, evaluates to 0}}
+// undef-error-error@#4 {{'C' is not defined, evaluates to 0}}
+// undef-prefix-error-error@#4 {{'C' is not defined, evaluates to 0}}
+// both-error-error@#4 {{'C' is not defined, evaluates to 0}}
diff --git a/clang/test/Preprocessor/x86_amx_target_features.c b/clang/test/Preprocessor/x86_amx_target_features.c
new file mode 100644
index 0000000000000..68a3d7f950b10
--- /dev/null
+++ b/clang/test/Preprocessor/x86_amx_target_features.c
@@ -0,0 +1,35 @@
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mamx-tile -x c -E -dM -o - %s | FileCheck  -check-prefix=AMX-TILE %s
+
+// AMX-TILE: #define __AMXTILE__ 1
+
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mamx-bf16 -x c -E -dM -o - %s | FileCheck -check-prefix=AMX-BF16 %s
+
+// AMX-BF16: #define __AMXBF16__ 1
+// AMX-BF16: #define __AMXTILE__ 1
+
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mamx-int8 -x c -E -dM -o - %s | FileCheck -check-prefix=AMX-INT8 %s
+
+// AMX-INT8: #define __AMXINT8__ 1
+// AMX-INT8: #define __AMXTILE__ 1
+
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-amx-tile -x c -E -dM -o - %s | FileCheck  -check-prefix=NOAMX-TILE %s
+
+// NOAMX-TILE-NOT: #define __AMXTILE__ 1
+
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-amx-bf16 -x c -E -dM -o - %s | FileCheck  -check-prefix=NOAMX-BF16 %s
+
+// NOAMX-BF16-NOT: #define __AMXBF16__ 1
+
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -amx-bf16 -mno-amx-tile -x c -E -dM -o - %s | FileCheck  -check-prefix=NOAMX-BF16 %s
+
+// NOAMX-BF16-NOT: #define __AMXTILE__ 1
+// NOAMX-BF16-NOT: #define __AMXBF16__ 1
+
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-amx-int8 -x c -E -dM -o - %s | FileCheck  -check-prefix=NOAMX-INT8 %s
+
+// NOAMX-INT8-NOT: #define __AMXINT8__ 1
+
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -amx-int8 -mno-amx-tile -x c -E -dM -o - %s | FileCheck  -check-prefix=NOAMX-INT8 %s
+
+// NOAMX-INT8-NOT: #define __AMXTILE__ 1
+// NOAMX-INT8-NOT: #define __AMXINT8__ 1
diff --git a/clang/test/Preprocessor/x86_target_features.c b/clang/test/Preprocessor/x86_target_features.c
index 8e24c15187208..4a46a131afa74 100644
--- a/clang/test/Preprocessor/x86_target_features.c
+++ b/clang/test/Preprocessor/x86_target_features.c
@@ -269,6 +269,12 @@
 
 // NOSSE42POPCNT: #define __POPCNT__ 1
 
+// RUN: %clang -target i386-unknown-unknown -march=nehalem -mno-sse4.2 -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=CPUPOPCNT %s
+// RUN: %clang -target i386-unknown-unknown -march=silvermont -mno-sse4.2 -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=CPUPOPCNT %s
+// RUN: %clang -target i386-unknown-unknown -march=knl -mno-sse4.2 -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=CPUPOPCNT %s
+
+// CPUPOPCNT: #define __POPCNT__ 1
+
 // RUN: %clang -target i386-unknown-unknown -march=pentium -msse -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=SSEMMX %s
 
 // SSEMMX: #define __MMX__ 1
@@ -374,10 +380,12 @@
 
 // RUN: %clang -target i386-unknown-unknown -march=atom -m3dnow -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=3DNOWPRFCHW %s
 
-// 3DNOWPRFCHW: #define __PRFCHW__ 1
+// 3DNOWPRFCHW: #define __3dNOW__ 1
+// 3DNOWPRFCHW-NOT: #define __PRFCHW__ 1
 
 // RUN: %clang -target i386-unknown-unknown -march=atom -mno-prfchw -m3dnow -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=3DNOWNOPRFCHW %s
 
+// 3DNOWNOPRFCHW: #define __3dNOW__ 1
 // 3DNOWNOPRFCHW-NOT: #define __PRFCHW__ 1
 
 // RUN: %clang -target i386-unknown-unknown -march=atom -mprfchw -mno-3dnow -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=NO3DNOWPRFCHW %s
@@ -415,9 +423,11 @@
 // XSAVES: #define __XSAVES__ 1
 // XSAVES: #define __XSAVE__ 1
 
-// RUN: %clang -target i386-unknown-unknown -march=atom -mxsaveopt -mno-xsave -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=NOXSAVE %s
+// RUN: %clang -target i386-unknown-unknown -march=atom -mxsaveopt -mxsavec -mxsaves -mno-xsave -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=NOXSAVE %s
 
+// NOXSAVE-NOT: #define __XSAVEC__ 1
 // NOXSAVE-NOT: #define __XSAVEOPT__ 1
+// NOXSAVE-NOT: #define __XSAVES__ 1
 // NOXSAVE-NOT: #define __XSAVE__ 1
 
 // RUN: %clang -target i386-unknown-unknown -march=atom -mclflushopt -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=CLFLUSHOPT %s
diff --git a/clang/test/Sema/aarch64-neon-bf16-ranges.c b/clang/test/Sema/aarch64-neon-bf16-ranges.c
new file mode 100644
index 0000000000000..bbed036846e57
--- /dev/null
+++ b/clang/test/Sema/aarch64-neon-bf16-ranges.c
@@ -0,0 +1,47 @@
+// RUN: %clang_cc1 -fsyntax-only -verify \
+// RUN: -triple aarch64-arm-none-eabi -target-feature +neon \
+// RUN: -target-feature +bf16 %s
+
+#include <arm_neon.h>
+
+int x;
+
+void test_vcopy_lane_bf16(bfloat16x4_t a, bfloat16x8_t b) {
+  // 0 <= lane1 <= 3; 0 <= lane2 <= 3
+  (void)vcopy_lane_bf16(a, 3, a, 3);
+  (void)vcopy_lane_bf16(a, 0, a, 4);    // expected-error {{argument value 4 is outside the valid range [0, 3]}}
+  (void)vcopy_lane_bf16(a, 1, a, -1);   // expected-error {{argument value -1 is outside the valid range [0, 3]}}
+  (void)vcopy_lane_bf16(a, 4, a, 0);    // expected-error {{argument value 4 is outside the valid range [0, 3]}}
+  (void)vcopy_lane_bf16(a, -1, a, 1);   // expected-error {{argument value -1 is outside the valid range [0, 3]}}
+  (void)vcopy_lane_bf16(a, 0, a, x);    // expected-error-re {{argument {{.*}} must be a constant integer}}
+  (void)vcopy_lane_bf16(a, x, a, 0);    // expected-error-re {{argument {{.*}} must be a constant integer}}
+
+  // 0 <= lane1 <= 7; 0 <= lane2 <= 3
+  (void)vcopyq_lane_bf16(b, 7, a, 3);
+  (void)vcopyq_lane_bf16(b, 0, a, 4);   // expected-error {{argument value 4 is outside the valid range [0, 3]}}
+  (void)vcopyq_lane_bf16(b, 1, a, -1);  // expected-error {{argument value -1 is outside the valid range [0, 3]}}
+  (void)vcopyq_lane_bf16(b, 8, a, 0);   // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+  (void)vcopyq_lane_bf16(b, -1, a, 1);  // expected-error {{argument value -1 is outside the valid range [0, 7]}}
+  (void)vcopyq_lane_bf16(b, 0, a, x);   // expected-error-re {{argument {{.*}} must be a constant integer}}
+  (void)vcopyq_lane_bf16(b, x, a, 0);   // expected-error-re {{argument {{.*}} must be a constant integer}}
+
+  // 0 <= lane1 <= 3; 0 <= lane2 <= 7
+  (void)vcopy_laneq_bf16(a, 3, b, 7);
+  (void)vcopy_laneq_bf16(a, 0, b, 8);   // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+  (void)vcopy_laneq_bf16(a, 1, b, -1);  // expected-error {{argument value -1 is outside the valid range [0, 7]}}
+  (void)vcopy_laneq_bf16(a, 4, b, 0);   // expected-error {{argument value 4 is outside the valid range [0, 3]}}
+  (void)vcopy_laneq_bf16(a, -1, b, 1);  // expected-error {{argument value -1 is outside the valid range [0, 3]}}
+  (void)vcopy_laneq_bf16(a, 0, b, x);   // expected-error-re {{argument {{.*}} must be a constant integer}}
+  (void)vcopy_laneq_bf16(a, x, b, 0);   // expected-error-re {{argument {{.*}} must be a constant integer}}
+
+
+  // 0 <= lane1 <= 7; 0 <= lane2 <= 7
+  (void)vcopyq_laneq_bf16(b, 7, b, 7);
+  (void)vcopyq_laneq_bf16(b, 0, b, 8);  // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+  (void)vcopyq_laneq_bf16(b, 1, b, -1); // expected-error {{argument value -1 is outside the valid range [0, 7]}}
+  (void)vcopyq_laneq_bf16(b, 8, b, 0);  // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+  (void)vcopyq_laneq_bf16(b, -1, b, 1); // expected-error {{argument value -1 is outside the valid range [0, 7]}}
+  (void)vcopyq_laneq_bf16(b, 0, b, x);  // expected-error-re {{argument {{.*}} must be a constant integer}}
+  (void)vcopyq_laneq_bf16(b, x, b, 0);  // expected-error-re {{argument {{.*}} must be a constant integer}}
+}
+
diff --git a/clang/test/Sema/aarch64-sve-types.c b/clang/test/Sema/aarch64-sve-types.c
index 989bc71d33c12..54233005f9344 100644
--- a/clang/test/Sema/aarch64-sve-types.c
+++ b/clang/test/Sema/aarch64-sve-types.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple aarch64-none-linux-gnu -target-feature +sve -fsyntax-only -verify
+// RUN: %clang_cc1 %s -triple aarch64-none-linux-gnu -target-feature +sve,+bf16 -fsyntax-only -verify
 
 void f() {
   int size_s8[sizeof(__SVInt8_t) == 0 ? 1 : -1];        // expected-error {{invalid application of 'sizeof' to sizeless type '__SVInt8_t'}}
@@ -34,6 +34,9 @@ void f() {
   int size_f64[sizeof(__SVFloat64_t) == 0 ? 1 : -1];        // expected-error {{invalid application of 'sizeof' to sizeless type '__SVFloat64_t'}}
   int align_f64[__alignof__(__SVFloat64_t) == 16 ? 1 : -1]; // expected-error {{invalid application of '__alignof' to sizeless type '__SVFloat64_t'}}
 
+  int size_bf16[sizeof(__SVBFloat16_t) == 0 ? 1 : -1];        // expected-error {{invalid application of 'sizeof' to sizeless type '__SVBFloat16_t'}}
+  int align_bf16[__alignof__(__SVBFloat16_t) == 16 ? 1 : -1]; // expected-error {{invalid application of '__alignof' to sizeless type '__SVBFloat16_t'}}
+
   int size_b8[sizeof(__SVBool_t) == 0 ? 1 : -1];       // expected-error {{invalid application of 'sizeof' to sizeless type '__SVBool_t'}}
   int align_b8[__alignof__(__SVBool_t) == 2 ? 1 : -1]; // expected-error {{invalid application of '__alignof' to sizeless type '__SVBool_t'}}
 }
diff --git a/clang/test/Sema/attr-handles.cpp b/clang/test/Sema/attr-handles.cpp
index 5abb1e8d00bc6..135467b6c0a8c 100644
--- a/clang/test/Sema/attr-handles.cpp
+++ b/clang/test/Sema/attr-handles.cpp
@@ -17,6 +17,7 @@ int (* __attribute__((acquire_handle("Fuchsia"))) fpt)(char *); // expected-warn
 auto lambdat = [](int handle __attribute__((use_handle("Fuchsia"))))
     __attribute__((acquire_handle("Fuchsia"))) -> int { return 0; };
 int __attribute((acquire_handle("Fuchsia"))) ta; // expected-warning {{'acquire_handle' attribute only applies to functions, typedefs, and parameters}}
+int open(const char *path, int flags, ...) [[clang::acquire_handle]]; // expected-error {{'acquire_handle' attribute takes one argument}}
 
 // Typedefs.
 typedef int callback(char *) __attribute__((acquire_handle("Fuchsia")));
diff --git a/clang/test/Sema/builtin-amdgcn-atomic-inc-dec-failure.cpp b/clang/test/Sema/builtin-amdgcn-atomic-inc-dec-failure.cpp
index c08b00bac0cf2..9351b4ecb032d 100644
--- a/clang/test/Sema/builtin-amdgcn-atomic-inc-dec-failure.cpp
+++ b/clang/test/Sema/builtin-amdgcn-atomic-inc-dec-failure.cpp
@@ -2,17 +2,18 @@
 // RUN: not %clang_cc1 %s -x hip -fcuda-is-device -o - -emit-llvm -triple=amdgcn-amd-amdhsa 2>&1 | FileCheck %s
 
 void test_host() {
-  int val;
+  __UINT32_TYPE__ val32;
+  __UINT64_TYPE__ val64;
 
   // CHECK: error: reference to __device__ function '__builtin_amdgcn_atomic_inc32' in __host__ function
-  val = __builtin_amdgcn_atomic_inc32(&val, val, __ATOMIC_SEQ_CST, "");
+  val32 = __builtin_amdgcn_atomic_inc32(&val32, val32, __ATOMIC_SEQ_CST, "");
 
   // CHECK: error: reference to __device__ function '__builtin_amdgcn_atomic_inc64' in __host__ function
-  val = __builtin_amdgcn_atomic_inc64(&val, val, __ATOMIC_SEQ_CST, "");
+  val64 = __builtin_amdgcn_atomic_inc64(&val64, val64, __ATOMIC_SEQ_CST, "");
 
   // CHECK: error: reference to __device__ function '__builtin_amdgcn_atomic_dec32' in __host__ function
-  val = __builtin_amdgcn_atomic_dec32(&val, val, __ATOMIC_SEQ_CST, "");
+  val32 = __builtin_amdgcn_atomic_dec32(&val32, val32, __ATOMIC_SEQ_CST, "");
 
   // CHECK: error: reference to __device__ function '__builtin_amdgcn_atomic_dec64' in __host__ function
-  val = __builtin_amdgcn_atomic_dec64(&val, val, __ATOMIC_SEQ_CST, "");
+  val64 = __builtin_amdgcn_atomic_dec64(&val64, val64, __ATOMIC_SEQ_CST, "");
 }
diff --git a/clang/test/Sema/builtin-expect-with-probability-avr.cpp b/clang/test/Sema/builtin-expect-with-probability-avr.cpp
new file mode 100644
index 0000000000000..1dbb1d52396ad
--- /dev/null
+++ b/clang/test/Sema/builtin-expect-with-probability-avr.cpp
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -triple avr -fsyntax-only -verify %s
+
+void test(int x, double p) { // expected-note {{declared here}}
+  bool dummy = false;
+  dummy = __builtin_expect_with_probability(x > 0, 1, 0.9);
+  dummy = __builtin_expect_with_probability(x > 0, 1, 1.1); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}}
+  dummy = __builtin_expect_with_probability(x > 0, 1, -1); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}}
+  dummy = __builtin_expect_with_probability(x > 0, 1, p); // expected-error {{probability argument to __builtin_expect_with_probability must be constant floating-point expression}} expected-note {{read of non-constexpr variable 'p' is not allowed in a constant expression}}
+  dummy = __builtin_expect_with_probability(x > 0, 1, "aa"); // expected-error {{cannot initialize a parameter of type 'double' with an lvalue of type 'const char [3]'}}
+  dummy = __builtin_expect_with_probability(x > 0, 1, __builtin_nan("")); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}}
+  dummy = __builtin_expect_with_probability(x > 0, 1, __builtin_inf()); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}}
+  dummy = __builtin_expect_with_probability(x > 0, 1, -0.0);
+  dummy = __builtin_expect_with_probability(x > 0, 1, 1.0 + __DBL_EPSILON__); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}}
+  dummy = __builtin_expect_with_probability(x > 0, 1, -__DBL_DENORM_MIN__); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}}
+}
diff --git a/clang/test/Sema/builtin-expect-with-probability.cpp b/clang/test/Sema/builtin-expect-with-probability.cpp
new file mode 100644
index 0000000000000..3aa20cb0e6dd7
--- /dev/null
+++ b/clang/test/Sema/builtin-expect-with-probability.cpp
@@ -0,0 +1,58 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+
+__attribute__((noreturn)) extern void bar();
+
+int test_no_warn(int x) {
+  if (x) {
+    if (__builtin_expect_with_probability(1, 1, 1))
+      bar();
+  } else {
+    return 0;
+  }
+} // should not emit warn "control may reach end of non-void function" here since expr is constantly true, so the "if(__bui..)" should be constantly true condition and be ignored
+
+template <int b> void tempf() {
+  static_assert(b == 1, "should be evaluated as 1"); // should not have error here
+}
+
+constexpr int constf() {
+  return __builtin_expect_with_probability(1, 1, 1);
+}
+
+void foo() {
+  tempf<__builtin_expect_with_probability(1, 1, 1)>();
+  constexpr int f = constf();
+  static_assert(f == 1, "should be evaluated as 1"); // should not have error here
+}
+
+extern int global;
+
+struct S {
+  static constexpr float prob = 0.7;
+};
+
+template<typename T>
+void expect_taken(int x) {
+  if (__builtin_expect_with_probability(x > 0, 1, T::prob)) {
+    global++;
+  }
+}
+
+void test(int x, double p) { // expected-note {{declared here}}
+  bool dummy;
+  dummy = __builtin_expect_with_probability(x > 0, 1, 0.9);
+  dummy = __builtin_expect_with_probability(x > 0, 1, 1.1); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}}
+  dummy = __builtin_expect_with_probability(x > 0, 1, -1); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}}
+  dummy = __builtin_expect_with_probability(x > 0, 1, p); // expected-error {{probability argument to __builtin_expect_with_probability must be constant floating-point expression}} expected-note {{read of non-constexpr variable 'p' is not allowed in a constant expression}}
+  dummy = __builtin_expect_with_probability(x > 0, 1, "aa"); // expected-error {{cannot initialize a parameter of type 'double' with an lvalue of type 'const char [3]'}}
+  dummy = __builtin_expect_with_probability(x > 0, 1, __builtin_nan("")); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}}
+  dummy = __builtin_expect_with_probability(x > 0, 1, __builtin_inf()); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}}
+  dummy = __builtin_expect_with_probability(x > 0, 1, -0.0);
+  dummy = __builtin_expect_with_probability(x > 0, 1, 1.0 + __DBL_EPSILON__); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}}
+  dummy = __builtin_expect_with_probability(x > 0, 1, -__DBL_DENORM_MIN__); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}}
+  constexpr double pd = 0.7;
+  dummy = __builtin_expect_with_probability(x > 0, 1, pd);
+  constexpr int pi = 1;
+  dummy = __builtin_expect_with_probability(x > 0, 1, pi);
+  expect_taken<S>(x);
+}
diff --git a/clang/test/Sema/format-strings.c b/clang/test/Sema/format-strings.c
index 11acfcd1b9a62..e8cd478d25126 100644
--- a/clang/test/Sema/format-strings.c
+++ b/clang/test/Sema/format-strings.c
@@ -290,8 +290,11 @@ void test11(void *p, char *s) {
   printf("%0p", p); // expected-warning{{flag '0' results in undefined behavior with 'p' conversion specifier}}
   printf("%s", s); // no-warning
   printf("%+s", p); // expected-warning{{flag '+' results in undefined behavior with 's' conversion specifier}}
+                    // expected-warning@-1 {{format specifies type 'char *' but the argument has type 'void *'}}
   printf("% s", p); // expected-warning{{flag ' ' results in undefined behavior with 's' conversion specifier}}
+                    // expected-warning@-1 {{format specifies type 'char *' but the argument has type 'void *'}}
   printf("%0s", p); // expected-warning{{flag '0' results in undefined behavior with 's' conversion specifier}}
+                    // expected-warning@-1 {{format specifies type 'char *' but the argument has type 'void *'}}
 }
 
 void test12(char *b) {
@@ -707,3 +710,7 @@ void PR30481() {
   // This caused crashes due to invalid casts.
   printf(1 > 0); // expected-warning{{format string is not a string literal}} expected-warning{{incompatible integer to pointer conversion}} expected-note@format-strings.c:*{{passing argument to parameter here}} expected-note{{to avoid this}}
 }
+
+void test_printf_opaque_ptr(void *op) {
+  printf("%s", op); // expected-warning{{format specifies type 'char *' but the argument has type 'void *'}}
+}
diff --git a/clang/test/Sema/init-ref-c.c b/clang/test/Sema/init-ref-c.c
new file mode 100644
index 0000000000000..38d8c44e2fdf5
--- /dev/null
+++ b/clang/test/Sema/init-ref-c.c
@@ -0,0 +1,7 @@
+// RUN: %clang_cc1 -triple arm-unknown-gnu -fsyntax-only -verify %s
+
+void f() {
+  struct EmptyStruct {};
+  struct EmptyStruct S;
+  __builtin_va_end(S); // no-crash, expected-error {{non-const lvalue reference to type '__builtin_va_list' cannot bind to a value of unrelated type 'struct EmptyStruct'}}
+}
\ No newline at end of file
diff --git a/clang/test/Sema/inline-asm-validate-amdgpu.cl b/clang/test/Sema/inline-asm-validate-amdgpu.cl
index 3d6488227ef29..418952c0e7272 100644
--- a/clang/test/Sema/inline-asm-validate-amdgpu.cl
+++ b/clang/test/Sema/inline-asm-validate-amdgpu.cl
@@ -18,9 +18,35 @@ kernel void test () {
   // vgpr constraints
   __asm__ ("v_mov_b32 %0, %1" : "=v" (vgpr) : "v" (imm) : );
 
-  // 'A' constraint
+  // 'I' constraint (an immediate integer in the range -16 to 64)
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "I" (imm) : );
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "I" (-16) : );
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "I" (64) : );
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "I" (-17) : ); // expected-error {{value '-17' out of range for constraint 'I'}}
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "I" (65) : ); // expected-error {{value '65' out of range for constraint 'I'}}
+
+  // 'J' constraint (an immediate 16-bit signed integer)
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "J" (imm) : );
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "J" (-32768) : );
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "J" (32767) : );
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "J" (-32769) : ); // expected-error {{value '-32769' out of range for constraint 'J'}}
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "J" (32768) : ); // expected-error {{value '32768' out of range for constraint 'J'}}
+
+  // 'A' constraint (an immediate constant that can be inlined)
   __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "A" (imm) : );
 
+  // 'B' constraint (an immediate 32-bit signed integer)
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "B" (imm) : );
+
+  // 'C' constraint (an immediate 32-bit unsigned integer or 'A' constraint)
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "C" (imm) : );
+
+  // 'DA' constraint (an immediate 64-bit constant that can be split into two 'A' constants)
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "DA" (imm) : );
+
+  // 'DB' constraint (an immediate 64-bit constant that can be split into two 'B' constants)
+  __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "DB" (imm) : );
+
 }
 
 __kernel void
diff --git a/clang/test/Sema/invalid-bitwidth-expr.mm b/clang/test/Sema/invalid-bitwidth-expr.mm
index 63aced1a3bf4d..41ca9496de4f9 100644
--- a/clang/test/Sema/invalid-bitwidth-expr.mm
+++ b/clang/test/Sema/invalid-bitwidth-expr.mm
@@ -18,3 +18,22 @@ auto func() {
   int X : func();
 };
 constexpr int sss = sizeof(Y);
+
+bool Foo(int *); // expected-note {{candidate function not viable}}
+template <typename T>
+struct Base {};
+template <typename T>
+auto func() {
+  // error-bit should be propagated from TemplateArgument to NestNameSpecifier.
+  class Base<decltype(Foo(T()))>::type C; // expected-error {{no matching function for call to 'Foo'}}
+  return C;
+}
+struct Z {
+  int X : func<int>(); // expected-note {{in instantiation of function template}}
+};
+constexpr int ssss = sizeof(Z);
+
+struct Z2 {
+  int X : sizeof(_ExtInt(invalid())); // expected-error {{use of undeclared identifier}}
+};
+constexpr int sssss = sizeof(Z2);
diff --git a/clang/test/Sema/matrix-type-builtins.c b/clang/test/Sema/matrix-type-builtins.c
index d8da11aa05aa6..2f7e4549e4950 100644
--- a/clang/test/Sema/matrix-type-builtins.c
+++ b/clang/test/Sema/matrix-type-builtins.c
@@ -11,11 +11,11 @@ void transpose(sx5x10_t a, ix3x2_t b, dx3x3 c, int *d, int e) {
   b = __builtin_matrix_transpose(b);
   // expected-error@-1 {{assigning to 'ix3x2_t' (aka 'int __attribute__((matrix_type(3, 2)))') from incompatible type 'int __attribute__((matrix_type(2, 3)))'}}
   __builtin_matrix_transpose(d);
-  // expected-error@-1 {{first argument must be a matrix}}
+  // expected-error@-1 {{1st argument must be a matrix}}
   __builtin_matrix_transpose(e);
-  // expected-error@-1 {{first argument must be a matrix}}
+  // expected-error@-1 {{1st argument must be a matrix}}
   __builtin_matrix_transpose("test");
-  // expected-error@-1 {{first argument must be a matrix}}
+  // expected-error@-1 {{1st argument must be a matrix}}
 
   ix3x3 m = __builtin_matrix_transpose(c);
   // expected-error@-1 {{initializing 'ix3x3' (aka 'unsigned int __attribute__((matrix_type(3, 3)))') with an expression of incompatible type 'double __attribute__((matrix_type(3, 3)))'}}
@@ -43,10 +43,10 @@ void column_major_load(float *p1, int *p2, _Bool *p3, struct Foo *p4) {
   // expected-error@-1 {{stride must be greater or equal to the number of rows}}
 
   sx5x10_t a8 = __builtin_matrix_column_major_load(p3, 5, 10, 6);
-  // expected-error@-1 {{first argument must be a pointer to a valid matrix element type}}
+  // expected-error@-1 {{1st argument must be a pointer to a valid matrix element type}}
 
   sx5x10_t a9 = __builtin_matrix_column_major_load(p4, 5, 10, 6);
-  // expected-error@-1 {{first argument must be a pointer to a valid matrix element type}}
+  // expected-error@-1 {{1st argument must be a pointer to a valid matrix element type}}
 
   sx5x10_t a10 = __builtin_matrix_column_major_load(p1, 1ull << 21, 10, 6);
   // expected-error@-1 {{row dimension is outside the allowed range [1, 1048575}}
@@ -54,13 +54,13 @@ void column_major_load(float *p1, int *p2, _Bool *p3, struct Foo *p4) {
   // expected-error@-1 {{column dimension is outside the allowed range [1, 1048575}}
 
   sx5x10_t a12 = __builtin_matrix_column_major_load(
-      10,         // expected-error {{first argument must be a pointer to a valid matrix element type}}
+      10,         // expected-error {{1st argument must be a pointer to a valid matrix element type}}
       1ull << 21, // expected-error {{row dimension is outside the allowed range [1, 1048575]}}
       1ull << 21, // expected-error {{column dimension is outside the allowed range [1, 1048575]}}
       "");        // expected-warning {{incompatible pointer to integer conversion casting 'char [1]' to type 'unsigned long'}}
 
   sx5x10_t a13 = __builtin_matrix_column_major_load(
-      10,  // expected-error {{first argument must be a pointer to a valid matrix element type}}
+      10,  // expected-error {{1st argument must be a pointer to a valid matrix element type}}
       *p4, // expected-error {{casting 'struct Foo' to incompatible type 'unsigned long'}}
       "",  // expected-error {{column argument must be a constant unsigned integer expression}}
            // expected-warning@-1 {{incompatible pointer to integer conversion casting 'char [1]' to type 'unsigned long'}}
@@ -73,18 +73,18 @@ void column_major_store(sx5x10_t *m1, ix3x2_t *m2, float *p1, int *p2, struct Fo
   __builtin_matrix_column_major_store(*m1, p1, 0);
   // expected-error@-1 {{stride must be greater or equal to the number of rows}}
   __builtin_matrix_column_major_store(*m1, p2, 10);
-  // expected-error@-1 {{the pointee of the second argument must match the element type of the first argument ('int' != 'float')}}
+  // expected-error@-1 {{the pointee of the 2nd argument must match the element type of the 1st argument ('int' != 'float')}}
   __builtin_matrix_column_major_store(p1, p2, 10);
-  // expected-error@-1 {{first argument must be a matrix}}
+  // expected-error@-1 {{1st argument must be a matrix}}
 
   __builtin_matrix_column_major_store(
-      "",   // expected-error {{first argument must be a matrix}}
-      10,   // expected-error {{second argument must be a pointer to a valid matrix element type}}
+      "",   // expected-error {{1st argument must be a matrix}}
+      10,   // expected-error {{2nd argument must be a pointer to a valid matrix element type}}
       *p3); // expected-error {{casting 'struct Foo' to incompatible type 'unsigned long'}}
 
   __builtin_matrix_column_major_store(
       *m1,
-      10, // expected-error {{second argument must be a pointer to a valid matrix element type}}
+      10, // expected-error {{2nd argument must be a pointer to a valid matrix element type}}
       10);
 
   *m1 = __builtin_matrix_column_major_store(*m1, p1, 10);
diff --git a/clang/test/Sema/static-array.c b/clang/test/Sema/static-array.c
index cc1043fe9c471..ef070718dc63f 100644
--- a/clang/test/Sema/static-array.c
+++ b/clang/test/Sema/static-array.c
@@ -1,6 +1,7 @@
-// RUN: %clang_cc1 -triple x86_64-apple-macosx10.14.0 -fsyntax-only -fblocks -verify %s
+// RUN: %clang_cc1 -triple x86_64-apple-macosx10.14.0 -fsyntax-only -fblocks -pedantic -verify %s
 
-void cat0(int a[static 0]) {} // expected-warning {{'static' has no effect on zero-length arrays}}
+void cat0(int a[static 0]) {} // expected-warning {{zero size arrays are an extension}} \
+                              // expected-note {{callee declares array parameter as static here}}
 
 void cat(int a[static 3]) {} // expected-note 4 {{callee declares array parameter as static here}} expected-note 2 {{passing argument to parameter 'a' here}}
 
@@ -9,7 +10,7 @@ void vat(int i, int a[static i]) {} // expected-note {{callee declares array par
 void f(int *p) {
   int a[2], b[3], c[4];
 
-  cat0(0);
+  cat0(0); // expected-warning {{null passed to a callee that requires a non-null argument}}
 
   cat(0); // expected-warning {{null passed to a callee that requires a non-null argument}}
   cat(a); // expected-warning {{array argument is too small; contains 2 elements, callee requires at least 3}}
diff --git a/clang/test/SemaCUDA/Inputs/cuda.h b/clang/test/SemaCUDA/Inputs/cuda.h
index 2600bfa9c470b..901f8e0c17cff 100644
--- a/clang/test/SemaCUDA/Inputs/cuda.h
+++ b/clang/test/SemaCUDA/Inputs/cuda.h
@@ -17,6 +17,19 @@ struct dim3 {
   __host__ __device__ dim3(unsigned x, unsigned y = 1, unsigned z = 1) : x(x), y(y), z(z) {}
 };
 
+#ifdef __HIP__
+typedef struct hipStream *hipStream_t;
+typedef enum hipError {} hipError_t;
+int hipConfigureCall(dim3 gridSize, dim3 blockSize, size_t sharedSize = 0,
+                     hipStream_t stream = 0);
+extern "C" hipError_t __hipPushCallConfiguration(dim3 gridSize, dim3 blockSize,
+                                                 size_t sharedSize = 0,
+                                                 hipStream_t stream = 0);
+extern "C" hipError_t hipLaunchKernel(const void *func, dim3 gridDim,
+                                      dim3 blockDim, void **args,
+                                      size_t sharedMem,
+                                      hipStream_t stream);
+#else
 typedef struct cudaStream *cudaStream_t;
 typedef enum cudaError {} cudaError_t;
 
@@ -29,6 +42,7 @@ extern "C" int __cudaPushCallConfiguration(dim3 gridSize, dim3 blockSize,
 extern "C" cudaError_t cudaLaunchKernel(const void *func, dim3 gridDim,
                                         dim3 blockDim, void **args,
                                         size_t sharedMem, cudaStream_t stream);
+#endif
 
 // Host- and device-side placement new overloads.
 void *operator new(__SIZE_TYPE__, void *p) { return p; }
diff --git a/clang/test/SemaCUDA/lambda.cu b/clang/test/SemaCUDA/lambda.cu
new file mode 100644
index 0000000000000..6f305a683c009
--- /dev/null
+++ b/clang/test/SemaCUDA/lambda.cu
@@ -0,0 +1,73 @@
+// RUN: %clang_cc1 -std=c++17 -fsyntax-only -verify=com %s
+// RUN: %clang_cc1 -std=c++17 -fsyntax-only -fcuda-is-device -verify=com,dev %s
+
+#include "Inputs/cuda.h"
+
+auto global_lambda = [] () { return 123; };
+
+template<class F>
+__global__ void kernel(F f) { f(); }
+// dev-note@-1 7{{called by 'kernel<(lambda}}
+
+__host__ __device__ void hd(int x);
+
+class A {
+  int b;
+public:
+  void test() {
+    [=](){ hd(b); }();
+
+    [&](){ hd(b); }();
+
+    kernel<<<1,1>>>([](){ hd(0); });
+
+    kernel<<<1,1>>>([=](){ hd(b); });
+    // dev-error@-1 {{capture host side class data member by this pointer in device or host device lambda function}}
+
+    kernel<<<1,1>>>([&](){ hd(b); });
+    // dev-error@-1 {{capture host side class data member by this pointer in device or host device lambda function}}
+
+    kernel<<<1,1>>>([&] __device__ (){ hd(b); });
+    // dev-error@-1 {{capture host side class data member by this pointer in device or host device lambda function}}
+
+    kernel<<<1,1>>>([&](){
+      auto f = [&]{ hd(b); };
+      // dev-error@-1 {{capture host side class data member by this pointer in device or host device lambda function}}
+      f();
+    });
+  }
+};
+
+int main(void) {
+  auto lambda_kernel = [&]__global__(){};
+  // com-error@-1 {{kernel function 'operator()' must be a free function or static member function}}
+
+  int b;
+  [&](){ hd(b); }();
+
+  [=, &b](){ hd(b); }();
+
+  kernel<<<1,1>>>(global_lambda);
+
+  kernel<<<1,1>>>([](){ hd(0); });
+
+  kernel<<<1,1>>>([=](){ hd(b); });
+
+  kernel<<<1,1>>>([b](){ hd(b); });
+
+  kernel<<<1,1>>>([&](){ hd(b); });
+  // dev-error@-1 {{capture host variable 'b' by reference in device or host device lambda function}}
+
+  kernel<<<1,1>>>([=, &b](){ hd(b); });
+  // dev-error@-1 {{capture host variable 'b' by reference in device or host device lambda function}}
+
+  kernel<<<1,1>>>([&, b](){ hd(b); });
+
+  kernel<<<1,1>>>([&](){
+      auto f = [&]{ hd(b); };
+      // dev-error@-1 {{capture host variable 'b' by reference in device or host device lambda function}}
+      f();
+  });
+
+  return 0;
+}
diff --git a/clang/test/SemaCXX/Inputs/std-coroutine.h b/clang/test/SemaCXX/Inputs/std-coroutine.h
index 7a424f1e99cf0..e9af21aa51945 100644
--- a/clang/test/SemaCXX/Inputs/std-coroutine.h
+++ b/clang/test/SemaCXX/Inputs/std-coroutine.h
@@ -10,25 +10,25 @@ struct coroutine_traits { using promise_type = typename Ret::promise_type; };
 
 template <class Promise = void>
 struct coroutine_handle {
-  static coroutine_handle from_address(void *);
+  static coroutine_handle from_address(void *) noexcept;
 };
 template <>
 struct coroutine_handle<void> {
   template <class PromiseType>
-  coroutine_handle(coroutine_handle<PromiseType>);
+  coroutine_handle(coroutine_handle<PromiseType>) noexcept;
   static coroutine_handle from_address(void *);
 };
 
 struct suspend_always {
-  bool await_ready() { return false; }
-  void await_suspend(coroutine_handle<>) {}
-  void await_resume() {}
+  bool await_ready() noexcept { return false; }
+  void await_suspend(coroutine_handle<>) noexcept {}
+  void await_resume() noexcept {}
 };
 
 struct suspend_never {
-  bool await_ready() { return true; }
-  void await_suspend(coroutine_handle<>) {}
-  void await_resume() {}
+  bool await_ready() noexcept { return true; }
+  void await_suspend(coroutine_handle<>) noexcept {}
+  void await_resume() noexcept {}
 };
 
 } // namespace experimental
diff --git a/clang/test/SemaCXX/attr-trivial-abi.cpp b/clang/test/SemaCXX/attr-trivial-abi.cpp
new file mode 100644
index 0000000000000..334b471d2ab87
--- /dev/null
+++ b/clang/test/SemaCXX/attr-trivial-abi.cpp
@@ -0,0 +1,112 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s -std=c++11
+
+void __attribute__((trivial_abi)) foo(); // expected-warning {{'trivial_abi' attribute only applies to classes}}
+
+// Should not crash.
+template <class>
+class __attribute__((trivial_abi)) a { a(a &&); };
+
+struct [[clang::trivial_abi]] S0 {
+  int a;
+};
+
+struct __attribute__((trivial_abi)) S1 {
+  int a;
+};
+
+struct __attribute__((trivial_abi)) S3 { // expected-warning {{'trivial_abi' cannot be applied to 'S3'}} expected-note {{is polymorphic}}
+  virtual void m();
+};
+
+struct S3_2 {
+  virtual void m();
+} __attribute__((trivial_abi)); // expected-warning {{'trivial_abi' cannot be applied to 'S3_2'}} expected-note {{is polymorphic}}
+
+struct __attribute__((trivial_abi)) S3_3 { // expected-warning {{'trivial_abi' cannot be applied to 'S3_3'}} expected-note {{has a field of a non-trivial class type}}
+  S3_3(S3_3 &&);
+  S3_2 s32;
+};
+
+// Diagnose invalid trivial_abi even when the type is templated because it has a non-trivial field.
+template <class T>
+struct __attribute__((trivial_abi)) S3_4 { // expected-warning {{'trivial_abi' cannot be applied to 'S3_4'}} expected-note {{has a field of a non-trivial class type}}
+  S3_4(S3_4 &&);
+  S3_2 s32;
+};
+
+struct S4 {
+  int a;
+};
+
+struct __attribute__((trivial_abi)) S5 : public virtual S4 { // expected-warning {{'trivial_abi' cannot be applied to 'S5'}} expected-note {{has a virtual base}}
+};
+
+struct __attribute__((trivial_abi)) S9 : public S4 {
+};
+
+struct __attribute__((trivial_abi(1))) S8 { // expected-error {{'trivial_abi' attribute takes no arguments}}
+  int a;
+};
+
+// Do not warn about deleted ctors  when 'trivial_abi' is used to annotate a template class.
+template <class T>
+struct __attribute__((trivial_abi)) S10 {
+  T p;
+};
+
+S10<int *> p1;
+
+template <class T>
+struct S14 {
+  T a;
+};
+
+template <class T>
+struct __attribute__((trivial_abi)) S15 : S14<T> {
+};
+
+S15<int> s15;
+
+template <class T>
+struct __attribute__((trivial_abi)) S16 {
+  S14<T> a;
+};
+
+S16<int> s16;
+
+template <class T>
+struct __attribute__((trivial_abi)) S17 {
+};
+
+S17<int> s17;
+
+namespace deletedCopyMoveConstructor {
+struct __attribute__((trivial_abi)) CopyMoveDeleted { // expected-warning {{'trivial_abi' cannot be applied to 'CopyMoveDeleted'}} expected-note {{copy constructors and move constructors are all deleted}}
+  CopyMoveDeleted(const CopyMoveDeleted &) = delete;
+  CopyMoveDeleted(CopyMoveDeleted &&) = delete;
+};
+
+struct __attribute__((trivial_abi)) S18 { // expected-warning {{'trivial_abi' cannot be applied to 'S18'}} expected-note {{copy constructors and move constructors are all deleted}}
+  CopyMoveDeleted a;
+};
+
+struct __attribute__((trivial_abi)) CopyDeleted {
+  CopyDeleted(const CopyDeleted &) = delete;
+  CopyDeleted(CopyDeleted &&) = default;
+};
+
+struct __attribute__((trivial_abi)) MoveDeleted {
+  MoveDeleted(const MoveDeleted &) = default;
+  MoveDeleted(MoveDeleted &&) = delete;
+};
+
+struct __attribute__((trivial_abi)) S19 { // expected-warning {{'trivial_abi' cannot be applied to 'S19'}} expected-note {{copy constructors and move constructors are all deleted}}
+  CopyDeleted a;
+  MoveDeleted b;
+};
+
+// This is fine since the move constructor isn't deleted.
+struct __attribute__((trivial_abi)) S20 {
+  int &&a; // a member of rvalue reference type deletes the copy constructor.
+};
+} // namespace deletedCopyMoveConstructor
diff --git a/clang/test/SemaCXX/builtin-align-cxx.cpp b/clang/test/SemaCXX/builtin-align-cxx.cpp
index 190f352c26d4a..848ed86525bbe 100644
--- a/clang/test/SemaCXX/builtin-align-cxx.cpp
+++ b/clang/test/SemaCXX/builtin-align-cxx.cpp
@@ -37,7 +37,7 @@ void test() {
   // expected-note@-1{{in instantiation of function template specialization 'test_templated_arguments<int, 7, 16>'}}
 }
 
-template <typename T>
+template <typename T, long ArraySize>
 void test_incorrect_alignment_without_instatiation(T value) {
   int array[32];
   static_assert(__is_same(decltype(__builtin_align_up(array, 31)), int *), // expected-error{{requested alignment is not a power of 2}}
@@ -52,6 +52,10 @@ void test_incorrect_alignment_without_instatiation(T value) {
   __builtin_align_up(array, 31);   // expected-error{{requested alignment is not a power of 2}}
   __builtin_align_up(value, 31);   // This shouldn't want since the type is dependent
   __builtin_align_up(value);       // Same here
+
+  __builtin_align_up(array, sizeof(sizeof(value)) - 1); // expected-error{{requested alignment is not a power of 2}}
+  __builtin_align_up(array, value); // no diagnostic as the alignment is value dependent.
+  (void)__builtin_align_up(array, ArraySize); // The same above here
 }
 
 // The original fix for the issue above broke some legitimate code.
diff --git a/clang/test/SemaCXX/co_await-range-for.cpp b/clang/test/SemaCXX/co_await-range-for.cpp
index 4d999ea7db5e5..b6c6e6c40f977 100644
--- a/clang/test/SemaCXX/co_await-range-for.cpp
+++ b/clang/test/SemaCXX/co_await-range-for.cpp
@@ -44,7 +44,7 @@ struct MyForLoopArrayAwaiter {
     void return_void();
     void unhandled_exception();
     suspend_never initial_suspend();
-    suspend_never final_suspend();
+    suspend_never final_suspend() noexcept;
     template <class T>
     Awaiter<T *> await_transform(T *) = delete; // expected-note {{explicitly deleted}}
   };
@@ -62,7 +62,7 @@ struct ForLoopAwaiterBadBeginTransform {
     void return_void();
     void unhandled_exception();
     suspend_never initial_suspend();
-    suspend_never final_suspend();
+    suspend_never final_suspend() noexcept;
 
     template <class T>
     Awaiter<T> await_transform(BeginTag<T>) = delete; // expected-note 1+ {{explicitly deleted}}
@@ -96,7 +96,7 @@ struct ForLoopAwaiterBadIncTransform {
     void return_void();
     void unhandled_exception();
     suspend_never initial_suspend();
-    suspend_never final_suspend();
+    suspend_never final_suspend() noexcept;
 
     template <class T>
     Awaiter<T> await_transform(BeginTag<T> e);
@@ -137,7 +137,7 @@ struct ForLoopAwaiterCoawaitLookup {
     void return_void();
     void unhandled_exception();
     suspend_never initial_suspend();
-    suspend_never final_suspend();
+    suspend_never final_suspend() noexcept;
     template <class T>
     CoawaitTag<T, false> await_transform(BeginTag<T> e);
     template <class T>
diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp
index dd6d7ccba19f8..78e9fef96c8da 100644
--- a/clang/test/SemaCXX/constant-expression-cxx11.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp
@@ -682,12 +682,12 @@ template<bool B, typename T> struct S : T {
   }
 };
 
-extern const int n;
+extern const int n; // expected-note {{declared here}}
 template<typename T> void f() {
   // This is ill-formed, because a hypothetical instantiation at the point of
   // template definition would be ill-formed due to a construct that does not
   // depend on a template parameter.
-  constexpr int k = n; // expected-error {{must be initialized by a constant expression}}
+  constexpr int k = n; // expected-error {{must be initialized by a constant expression}} expected-note {{initializer of 'n' is unknown}}
 }
 // It doesn't matter that the instantiation could later become valid:
 constexpr int n = 4;
@@ -1258,7 +1258,7 @@ constexpr int m1b = const_cast<const int&>(n1); // expected-error {{constant exp
 constexpr int m2b = const_cast<const int&>(n2); // expected-error {{constant expression}} expected-note {{read of volatile object 'n2'}}
 
 struct T { int n; };
-const T t = { 42 };
+const T t = { 42 }; // expected-note {{declared here}}
 
 constexpr int f(volatile int &&r) {
   return r; // expected-note {{read of volatile-qualified type 'volatile int'}}
@@ -1372,7 +1372,7 @@ namespace InstantiateCaseStmt {
 
 namespace ConvertedConstantExpr {
   extern int &m;
-  extern int &n;
+  extern int &n; // expected-note 2{{declared here}}
 
   constexpr int k = 4;
   int &m = const_cast<int&>(k);
@@ -1381,9 +1381,9 @@ namespace ConvertedConstantExpr {
   // useless note and instead just point to the non-constant subexpression.
   enum class E {
     em = m,
-    en = n, // expected-error {{not a constant expression}}
-    eo = (m +
-          n // expected-error {{not a constant expression}}
+    en = n, // expected-error {{not a constant expression}} expected-note {{initializer of 'n' is unknown}}
+    eo = (m + // expected-error {{not a constant expression}}
+          n // expected-note {{initializer of 'n' is unknown}}
           ),
     eq = reinterpret_cast<long>((int*)0) // expected-error {{not a constant expression}} expected-note {{reinterpret_cast}}
   };
@@ -2302,3 +2302,23 @@ namespace PR41854 {
   f &d = reinterpret_cast<f&>(a);
   unsigned b = d.c;
 }
+
+namespace array_size {
+  template<int N> struct array {
+    static constexpr int size() { return N; }
+  };
+  template<typename T> void f1(T t) {
+    constexpr int k = t.size();
+  }
+  template<typename T> void f2(const T &t) {
+    constexpr int k = t.size(); // expected-error {{constant}} expected-note {{function parameter 't' with unknown value cannot be used in a constant expression}}
+  }
+  template<typename T> void f3(const T &t) {
+    constexpr int k = T::size();
+  }
+  void g(array<3> a) {
+    f1(a);
+    f2(a); // expected-note {{instantiation of}}
+    f3(a);
+  }
+}
diff --git a/clang/test/SemaCXX/constant-expression-cxx1y.cpp b/clang/test/SemaCXX/constant-expression-cxx1y.cpp
index 5a414799f7552..8bc4f88a63a96 100644
--- a/clang/test/SemaCXX/constant-expression-cxx1y.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx1y.cpp
@@ -1009,7 +1009,7 @@ constexpr int sum(const char (&Arr)[N]) {
 // As an extension, we support evaluating some things that are `const` as though
 // they were `constexpr` when folding, but it should not be allowed in normal
 // constexpr evaluation.
-const char Cs[] = {'a', 'b'};
+const char Cs[] = {'a', 'b'}; // expected-note 2{{declared here}}
 void foo() __attribute__((enable_if(sum(Cs) == 'a' + 'b', "")));
 void run() { foo(); }
 
diff --git a/clang/test/SemaCXX/constexpr-default-init-value-crash.cpp b/clang/test/SemaCXX/constexpr-default-init-value-crash.cpp
new file mode 100644
index 0000000000000..d20d28d199d41
--- /dev/null
+++ b/clang/test/SemaCXX/constexpr-default-init-value-crash.cpp
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 %s -std=c++20 -fsyntax-only -verify
+// RUN: %clang_cc1 %s -std=c++20 -fsyntax-only -verify -fno-recovery-ast
+
+namespace NoCrash {
+struct ForwardDecl; // expected-note {{forward declaration of}}
+struct Foo {        // expected-note 2{{candidate constructor}}
+  ForwardDecl f;    // expected-error {{field has incomplete type}}
+};
+
+constexpr Foo getFoo() {
+  Foo e = 123; // expected-error {{no viable conversion from 'int' to 'NoCrash::Foo'}}
+  return e;
+}
+}
diff --git a/clang/test/SemaCXX/constexpr-vectors.cpp b/clang/test/SemaCXX/constexpr-vectors.cpp
new file mode 100644
index 0000000000000..1ba0e33253321
--- /dev/null
+++ b/clang/test/SemaCXX/constexpr-vectors.cpp
@@ -0,0 +1,616 @@
+// RUN: %clang_cc1 -std=c++14 -Wno-unused-value %s -disable-llvm-passes -triple x86_64-linux-gnu -emit-llvm -o - | FileCheck %s
+
+// FIXME: Unfortunately there is no good way to validate that our values are
+// correct since Vector types don't have operator [] implemented for constexpr.
+// Instead, we need to use filecheck to ensure the emitted IR is correct. Once
+// someone implements array subscript operator for these types as constexpr,
+// this test should modified to jsut use static asserts.
+
+using FourCharsVecSize __attribute__((vector_size(4))) = char;
+using FourIntsVecSize __attribute__((vector_size(16))) = int;
+using FourLongLongsVecSize __attribute__((vector_size(32))) = long long;
+using FourFloatsVecSize __attribute__((vector_size(16))) = float;
+using FourDoublesVecSize __attribute__((vector_size(32))) = double;
+
+using FourCharsExtVec __attribute__((ext_vector_type(4))) = char;
+using FourIntsExtVec __attribute__((ext_vector_type(4))) = int;
+using FourLongLongsExtVec __attribute__((ext_vector_type(4))) = long long;
+using FourFloatsExtVec __attribute__((ext_vector_type(4))) = float;
+using FourDoublesExtVec __attribute__((ext_vector_type(4))) = double;
+
+// Next a series of tests to make sure these operations are usable in
+// constexpr functions. Template instantiations don't emit Winvalid-constexpr,
+// so we have to do these as macros.
+#define MathShiftOps(Type)                            \
+  constexpr auto MathShiftOps##Type(Type a, Type b) { \
+    a = a + b;                                        \
+    a = a - b;                                        \
+    a = a * b;                                        \
+    a = a / b;                                        \
+    b = a + 1;                                        \
+    b = a - 1;                                        \
+    b = a * 1;                                        \
+    b = a / 1;                                        \
+    a += a;                                           \
+    a -= a;                                           \
+    a *= a;                                           \
+    a /= a;                                           \
+    b += a;                                           \
+    b -= a;                                           \
+    b *= a;                                           \
+    b /= a;                                           \
+    a < b;                                            \
+    a > b;                                            \
+    a <= b;                                           \
+    a >= b;                                           \
+    a == b;                                           \
+    a != b;                                           \
+    a &&b;                                            \
+    a || b;                                           \
+    auto c = (a, b);                                  \
+    return c;                                         \
+  }
+
+// Ops specific to Integers.
+#define MathShiftOpsInts(Type)                            \
+  constexpr auto MathShiftopsInts##Type(Type a, Type b) { \
+    a = a << b;                                           \
+    a = a >> b;                                           \
+    a = a << 3;                                           \
+    a = a >> 3;                                           \
+    a = 3 << b;                                           \
+    a = 3 >> b;                                           \
+    a <<= b;                                              \
+    a >>= b;                                              \
+    a <<= 3;                                              \
+    a >>= 3;                                              \
+    a = a % b;                                            \
+    a &b;                                                 \
+    a | b;                                                \
+    a ^ b;                                                \
+    return a;                                             \
+  }
+
+MathShiftOps(FourCharsVecSize);
+MathShiftOps(FourIntsVecSize);
+MathShiftOps(FourLongLongsVecSize);
+MathShiftOps(FourFloatsVecSize);
+MathShiftOps(FourDoublesVecSize);
+MathShiftOps(FourCharsExtVec);
+MathShiftOps(FourIntsExtVec);
+MathShiftOps(FourLongLongsExtVec);
+MathShiftOps(FourFloatsExtVec);
+MathShiftOps(FourDoublesExtVec);
+
+MathShiftOpsInts(FourCharsVecSize);
+MathShiftOpsInts(FourIntsVecSize);
+MathShiftOpsInts(FourLongLongsVecSize);
+MathShiftOpsInts(FourCharsExtVec);
+MathShiftOpsInts(FourIntsExtVec);
+MathShiftOpsInts(FourLongLongsExtVec);
+
+template <typename T, typename U>
+constexpr auto CmpMul(T t, U u) {
+  t *= u;
+  return t;
+}
+template <typename T, typename U>
+constexpr auto CmpDiv(T t, U u) {
+  t /= u;
+  return t;
+}
+template <typename T, typename U>
+constexpr auto CmpRem(T t, U u) {
+  t %= u;
+  return t;
+}
+
+template <typename T, typename U>
+constexpr auto CmpAdd(T t, U u) {
+  t += u;
+  return t;
+}
+
+template <typename T, typename U>
+constexpr auto CmpSub(T t, U u) {
+  t -= u;
+  return t;
+}
+
+template <typename T, typename U>
+constexpr auto CmpLSH(T t, U u) {
+  t <<= u;
+  return t;
+}
+
+template <typename T, typename U>
+constexpr auto CmpRSH(T t, U u) {
+  t >>= u;
+  return t;
+}
+
+template <typename T, typename U>
+constexpr auto CmpBinAnd(T t, U u) {
+  t &= u;
+  return t;
+}
+
+template <typename T, typename U>
+constexpr auto CmpBinXOr(T t, U u) {
+  t ^= u;
+  return t;
+}
+
+template <typename T, typename U>
+constexpr auto CmpBinOr(T t, U u) {
+  t |= u;
+  return t;
+}
+
+// Only int vs float makes a difference here, so we only need to test 1 of each.
+// Test Char to make sure the mixed-nature of shifts around char is evident.
+void CharUsage() {
+  constexpr auto a = FourCharsVecSize{6, 3, 2, 1} +
+                     FourCharsVecSize{12, 15, 5, 7};
+  // CHECK: store <4 x i8> <i8 18, i8 18, i8 7, i8 8>
+  constexpr auto b = FourCharsVecSize{19, 15, 13, 12} -
+                     FourCharsVecSize{13, 14, 5, 3};
+  // CHECK: store <4 x i8> <i8 6, i8 1, i8 8, i8 9>
+  constexpr auto c = FourCharsVecSize{8, 4, 2, 1} *
+                     FourCharsVecSize{3, 4, 5, 6};
+  // CHECK: store <4 x i8> <i8 24, i8 16, i8 10, i8 6>
+  constexpr auto d = FourCharsVecSize{12, 12, 10, 10} /
+                     FourCharsVecSize{6, 4, 5, 2};
+  // CHECK: store <4 x i8> <i8 2, i8 3, i8 2, i8 5>
+  constexpr auto e = FourCharsVecSize{12, 12, 10, 10} %
+                     FourCharsVecSize{6, 4, 4, 3};
+  // CHECK: store <4 x i8> <i8 0, i8 0, i8 2, i8 1>
+
+  constexpr auto f = FourCharsVecSize{6, 3, 2, 1} + 3;
+  // CHECK: store <4 x i8> <i8 9, i8 6, i8 5, i8 4>
+  constexpr auto g = FourCharsVecSize{19, 15, 12, 10} - 3;
+  // CHECK: store <4 x i8> <i8 16, i8 12, i8 9, i8 7>
+  constexpr auto h = FourCharsVecSize{8, 4, 2, 1} * 3;
+  // CHECK: store <4 x i8> <i8 24, i8 12, i8 6, i8 3>
+  constexpr auto j = FourCharsVecSize{12, 15, 18, 21} / 3;
+  // CHECK: store <4 x i8> <i8 4, i8 5, i8 6, i8 7>
+  constexpr auto k = FourCharsVecSize{12, 17, 19, 22} % 3;
+  // CHECK: store <4 x i8> <i8 0, i8 2, i8 1, i8 1>
+
+  constexpr auto l = 3 + FourCharsVecSize{6, 3, 2, 1};
+  // CHECK: store <4 x i8> <i8 9, i8 6, i8 5, i8 4>
+  constexpr auto m = 20 - FourCharsVecSize{19, 15, 12, 10};
+  // CHECK: store <4 x i8> <i8 1, i8 5, i8 8, i8 10>
+  constexpr auto n = 3 * FourCharsVecSize{8, 4, 2, 1};
+  // CHECK: store <4 x i8> <i8 24, i8 12, i8 6, i8 3>
+  constexpr auto o = 100 / FourCharsVecSize{12, 15, 18, 21};
+  // CHECK: store <4 x i8> <i8 8, i8 6, i8 5, i8 4>
+  constexpr auto p = 100 % FourCharsVecSize{12, 15, 18, 21};
+  // CHECK: store <4 x i8> <i8 4, i8 10, i8 10, i8 16>
+
+  constexpr auto q = FourCharsVecSize{6, 3, 2, 1} << FourCharsVecSize{1, 1, 2, 2};
+  // CHECK: store <4 x i8> <i8 12, i8 6, i8 8, i8 4>
+  constexpr auto r = FourCharsVecSize{19, 15, 12, 10} >>
+                     FourCharsVecSize{1, 1, 2, 2};
+  // CHECK: store <4 x i8> <i8 9, i8 7, i8 3, i8 2>
+  constexpr auto s = FourCharsVecSize{6, 3, 5, 10} << 1;
+  // CHECK: store <4 x i8> <i8 12, i8 6, i8 10, i8 20>
+  constexpr auto t = FourCharsVecSize{19, 15, 10, 20} >> 1;
+  // CHECK: store <4 x i8> <i8 9, i8 7, i8 5, i8 10>
+  constexpr auto u = 12 << FourCharsVecSize{1, 2, 3, 3};
+  // CHECK: store <4 x i8> <i8 24, i8 48, i8 96, i8 96>
+  constexpr auto v = 12 >> FourCharsVecSize{1, 2, 2, 1};
+  // CHECK: store <4 x i8> <i8 6, i8 3, i8 3, i8 6>
+
+  constexpr auto w = FourCharsVecSize{1, 2, 3, 4} <
+                     FourCharsVecSize{4, 3, 2, 1};
+  // CHECK: store <4 x i8> <i8 1, i8 1, i8 0, i8 0>
+  constexpr auto x = FourCharsVecSize{1, 2, 3, 4} >
+                     FourCharsVecSize{4, 3, 2, 1};
+  // CHECK: store <4 x i8> <i8 0, i8 0, i8 1, i8 1>
+  constexpr auto y = FourCharsVecSize{1, 2, 3, 4} <=
+                     FourCharsVecSize{4, 3, 3, 1};
+  // CHECK: store <4 x i8> <i8 1, i8 1, i8 1, i8 0>
+  constexpr auto z = FourCharsVecSize{1, 2, 3, 4} >=
+                     FourCharsVecSize{4, 3, 3, 1};
+  // CHECK: store <4 x i8> <i8 0, i8 0, i8 1, i8 1>
+  constexpr auto A = FourCharsVecSize{1, 2, 3, 4} ==
+                     FourCharsVecSize{4, 3, 3, 1};
+  // CHECK: store <4 x i8> <i8 0, i8 0, i8 1, i8 0>
+  constexpr auto B = FourCharsVecSize{1, 2, 3, 4} !=
+                     FourCharsVecSize{4, 3, 3, 1};
+  // CHECK: store <4 x i8> <i8 1, i8 1, i8 0, i8 1>
+
+  constexpr auto C = FourCharsVecSize{1, 2, 3, 4} < 3;
+  // CHECK: store <4 x i8> <i8 1, i8 1, i8 0, i8 0>
+  constexpr auto D = FourCharsVecSize{1, 2, 3, 4} > 3;
+  // CHECK: store <4 x i8> <i8 0, i8 0, i8 0, i8 1>
+  constexpr auto E = FourCharsVecSize{1, 2, 3, 4} <= 3;
+  // CHECK: store <4 x i8> <i8 1, i8 1, i8 1, i8 0>
+  constexpr auto F = FourCharsVecSize{1, 2, 3, 4} >= 3;
+  // CHECK: store <4 x i8> <i8 0, i8 0, i8 1, i8 1>
+  constexpr auto G = FourCharsVecSize{1, 2, 3, 4} == 3;
+  // CHECK: store <4 x i8> <i8 0, i8 0, i8 1, i8 0>
+  constexpr auto H = FourCharsVecSize{1, 2, 3, 4} != 3;
+  // CHECK: store <4 x i8> <i8 1, i8 1, i8 0, i8 1>
+
+  constexpr auto I = FourCharsVecSize{1, 2, 3, 4} &
+                     FourCharsVecSize{4, 3, 2, 1};
+  // CHECK: store <4 x i8> <i8 0, i8 2, i8 2, i8 0>
+  constexpr auto J = FourCharsVecSize{1, 2, 3, 4} ^
+                     FourCharsVecSize { 4, 3, 2, 1 };
+  // CHECK: store <4 x i8> <i8 5, i8 1, i8 1, i8 5>
+  constexpr auto K = FourCharsVecSize{1, 2, 3, 4} |
+                     FourCharsVecSize{4, 3, 2, 1};
+  // CHECK: store <4 x i8> <i8 5, i8 3, i8 3, i8 5>
+  constexpr auto L = FourCharsVecSize{1, 2, 3, 4} & 3;
+  // CHECK: store <4 x i8> <i8 1, i8 2, i8 3, i8 0>
+  constexpr auto M = FourCharsVecSize{1, 2, 3, 4} ^ 3;
+  // CHECK: store <4 x i8> <i8 2, i8 1, i8 0, i8 7>
+  constexpr auto N = FourCharsVecSize{1, 2, 3, 4} | 3;
+  // CHECK: store <4 x i8> <i8 3, i8 3, i8 3, i8 7>
+
+  constexpr auto O = FourCharsVecSize{5, 0, 6, 0} &&
+                     FourCharsVecSize{5, 5, 0, 0};
+  // CHECK: store <4 x i8> <i8 1, i8 0, i8 0, i8 0>
+  constexpr auto P = FourCharsVecSize{5, 0, 6, 0} ||
+                     FourCharsVecSize{5, 5, 0, 0};
+  // CHECK: store <4 x i8> <i8 1, i8 1, i8 1, i8 0>
+
+  constexpr auto Q = FourCharsVecSize{5, 0, 6, 0} && 3;
+  // CHECK: store <4 x i8> <i8 1, i8 0, i8 1, i8 0>
+  constexpr auto R = FourCharsVecSize{5, 0, 6, 0} || 3;
+  // CHECK: store <4 x i8> <i8 1, i8 1, i8 1, i8 1>
+
+  constexpr auto T = CmpMul(a, b);
+  // CHECK: store <4 x i8> <i8 108, i8 18, i8 56, i8 72>
+
+  constexpr auto U = CmpDiv(a, b);
+  // CHECK: store <4 x i8> <i8 3, i8 18, i8 0, i8 0>
+
+  constexpr auto V = CmpRem(a, b);
+  // CHECK: store <4 x i8> <i8 0, i8 0, i8 7, i8 8>
+
+  constexpr auto X = CmpAdd(a, b);
+  // CHECK: store <4 x i8> <i8 24, i8 19, i8 15, i8 17>
+
+  constexpr auto Y = CmpSub(a, b);
+  // CHECK: store <4 x i8> <i8 12, i8 17, i8 -1, i8 -1>
+
+  constexpr auto Z = CmpLSH(a, H);
+  // CHECK: store <4 x i8> <i8 36, i8 36, i8 7, i8 16>
+
+  constexpr auto aa = CmpRSH(a, H);
+  // CHECK: store <4 x i8> <i8 9, i8 9, i8 7, i8 4>
+
+  constexpr auto ab = CmpBinAnd(a, b);
+  // CHECK: store <4 x i8> <i8 2, i8 0, i8 0, i8 8>
+
+  constexpr auto ac = CmpBinXOr(a, b);
+  // CHECK: store <4 x i8> <i8 20, i8 19, i8 15, i8 1>
+
+  constexpr auto ad = CmpBinOr(a, b);
+  // CHECK: store <4 x i8> <i8 22, i8 19, i8 15, i8 9>
+}
+
+void CharExtVecUsage() {
+  constexpr auto a = FourCharsExtVec{6, 3, 2, 1} +
+                     FourCharsExtVec{12, 15, 5, 7};
+  // CHECK: store <4 x i8> <i8 18, i8 18, i8 7, i8 8>
+  constexpr auto b = FourCharsExtVec{19, 15, 13, 12} -
+                     FourCharsExtVec{13, 14, 5, 3};
+  // CHECK: store <4 x i8> <i8 6, i8 1, i8 8, i8 9>
+  constexpr auto c = FourCharsExtVec{8, 4, 2, 1} *
+                     FourCharsExtVec{3, 4, 5, 6};
+  // CHECK: store <4 x i8> <i8 24, i8 16, i8 10, i8 6>
+  constexpr auto d = FourCharsExtVec{12, 12, 10, 10} /
+                     FourCharsExtVec{6, 4, 5, 2};
+  // CHECK: store <4 x i8> <i8 2, i8 3, i8 2, i8 5>
+  constexpr auto e = FourCharsExtVec{12, 12, 10, 10} %
+                     FourCharsExtVec{6, 4, 4, 3};
+  // CHECK: store <4 x i8> <i8 0, i8 0, i8 2, i8 1>
+
+  constexpr auto f = FourCharsExtVec{6, 3, 2, 1} + 3;
+  // CHECK: store <4 x i8> <i8 9, i8 6, i8 5, i8 4>
+  constexpr auto g = FourCharsExtVec{19, 15, 12, 10} - 3;
+  // CHECK: store <4 x i8> <i8 16, i8 12, i8 9, i8 7>
+  constexpr auto h = FourCharsExtVec{8, 4, 2, 1} * 3;
+  // CHECK: store <4 x i8> <i8 24, i8 12, i8 6, i8 3>
+  constexpr auto j = FourCharsExtVec{12, 15, 18, 21} / 3;
+  // CHECK: store <4 x i8> <i8 4, i8 5, i8 6, i8 7>
+  constexpr auto k = FourCharsExtVec{12, 17, 19, 22} % 3;
+  // CHECK: store <4 x i8> <i8 0, i8 2, i8 1, i8 1>
+
+  constexpr auto l = 3 + FourCharsExtVec{6, 3, 2, 1};
+  // CHECK: store <4 x i8> <i8 9, i8 6, i8 5, i8 4>
+  constexpr auto m = 20 - FourCharsExtVec{19, 15, 12, 10};
+  // CHECK: store <4 x i8> <i8 1, i8 5, i8 8, i8 10>
+  constexpr auto n = 3 * FourCharsExtVec{8, 4, 2, 1};
+  // CHECK: store <4 x i8> <i8 24, i8 12, i8 6, i8 3>
+  constexpr auto o = 100 / FourCharsExtVec{12, 15, 18, 21};
+  // CHECK: store <4 x i8> <i8 8, i8 6, i8 5, i8 4>
+  constexpr auto p = 100 % FourCharsExtVec{12, 15, 18, 21};
+  // CHECK: store <4 x i8> <i8 4, i8 10, i8 10, i8 16>
+
+  constexpr auto q = FourCharsExtVec{6, 3, 2, 1} << FourCharsVecSize{1, 1, 2, 2};
+  // CHECK: store <4 x i8> <i8 12, i8 6, i8 8, i8 4>
+  constexpr auto r = FourCharsExtVec{19, 15, 12, 10} >>
+                     FourCharsExtVec{1, 1, 2, 2};
+  // CHECK: store <4 x i8> <i8 9, i8 7, i8 3, i8 2>
+  constexpr auto s = FourCharsExtVec{6, 3, 5, 10} << 1;
+  // CHECK: store <4 x i8> <i8 12, i8 6, i8 10, i8 20>
+  constexpr auto t = FourCharsExtVec{19, 15, 10, 20} >> 1;
+  // CHECK: store <4 x i8> <i8 9, i8 7, i8 5, i8 10>
+  constexpr auto u = 12 << FourCharsExtVec{1, 2, 3, 3};
+  // CHECK: store <4 x i8> <i8 24, i8 48, i8 96, i8 96>
+  constexpr auto v = 12 >> FourCharsExtVec{1, 2, 2, 1};
+  // CHECK: store <4 x i8> <i8 6, i8 3, i8 3, i8 6>
+
+  constexpr auto w = FourCharsExtVec{1, 2, 3, 4} <
+                     FourCharsExtVec{4, 3, 2, 1};
+  // CHECK: store <4 x i8> <i8 1, i8 1, i8 0, i8 0>
+  constexpr auto x = FourCharsExtVec{1, 2, 3, 4} >
+                     FourCharsExtVec{4, 3, 2, 1};
+  // CHECK: store <4 x i8> <i8 0, i8 0, i8 1, i8 1>
+  constexpr auto y = FourCharsExtVec{1, 2, 3, 4} <=
+                     FourCharsExtVec{4, 3, 3, 1};
+  // CHECK: store <4 x i8> <i8 1, i8 1, i8 1, i8 0>
+  constexpr auto z = FourCharsExtVec{1, 2, 3, 4} >=
+                     FourCharsExtVec{4, 3, 3, 1};
+  // CHECK: store <4 x i8> <i8 0, i8 0, i8 1, i8 1>
+  constexpr auto A = FourCharsExtVec{1, 2, 3, 4} ==
+                     FourCharsExtVec{4, 3, 3, 1};
+  // CHECK: store <4 x i8> <i8 0, i8 0, i8 1, i8 0>
+  constexpr auto B = FourCharsExtVec{1, 2, 3, 4} !=
+                     FourCharsExtVec{4, 3, 3, 1};
+  // CHECK: store <4 x i8> <i8 1, i8 1, i8 0, i8 1>
+
+  constexpr auto C = FourCharsExtVec{1, 2, 3, 4} < 3;
+  // CHECK: store <4 x i8> <i8 1, i8 1, i8 0, i8 0>
+  constexpr auto D = FourCharsExtVec{1, 2, 3, 4} > 3;
+  // CHECK: store <4 x i8> <i8 0, i8 0, i8 0, i8 1>
+  constexpr auto E = FourCharsExtVec{1, 2, 3, 4} <= 3;
+  // CHECK: store <4 x i8> <i8 1, i8 1, i8 1, i8 0>
+  constexpr auto F = FourCharsExtVec{1, 2, 3, 4} >= 3;
+  // CHECK: store <4 x i8> <i8 0, i8 0, i8 1, i8 1>
+  constexpr auto G = FourCharsExtVec{1, 2, 3, 4} == 3;
+  // CHECK: store <4 x i8> <i8 0, i8 0, i8 1, i8 0>
+  constexpr auto H = FourCharsExtVec{1, 2, 3, 4} != 3;
+  // CHECK: store <4 x i8> <i8 1, i8 1, i8 0, i8 1>
+
+  constexpr auto I = FourCharsExtVec{1, 2, 3, 4} &
+                     FourCharsExtVec{4, 3, 2, 1};
+  // CHECK: store <4 x i8> <i8 0, i8 2, i8 2, i8 0>
+  constexpr auto J = FourCharsExtVec{1, 2, 3, 4} ^
+                     FourCharsExtVec { 4, 3, 2, 1 };
+  // CHECK: store <4 x i8> <i8 5, i8 1, i8 1, i8 5>
+  constexpr auto K = FourCharsExtVec{1, 2, 3, 4} |
+                     FourCharsExtVec{4, 3, 2, 1};
+  // CHECK: store <4 x i8> <i8 5, i8 3, i8 3, i8 5>
+  constexpr auto L = FourCharsExtVec{1, 2, 3, 4} & 3;
+  // CHECK: store <4 x i8> <i8 1, i8 2, i8 3, i8 0>
+  constexpr auto M = FourCharsExtVec{1, 2, 3, 4} ^ 3;
+  // CHECK: store <4 x i8> <i8 2, i8 1, i8 0, i8 7>
+  constexpr auto N = FourCharsExtVec{1, 2, 3, 4} | 3;
+  // CHECK: store <4 x i8> <i8 3, i8 3, i8 3, i8 7>
+
+  constexpr auto O = FourCharsExtVec{5, 0, 6, 0} &&
+                     FourCharsExtVec{5, 5, 0, 0};
+  // CHECK: store <4 x i8> <i8 1, i8 0, i8 0, i8 0>
+  constexpr auto P = FourCharsExtVec{5, 0, 6, 0} ||
+                     FourCharsExtVec{5, 5, 0, 0};
+  // CHECK: store <4 x i8> <i8 1, i8 1, i8 1, i8 0>
+
+  constexpr auto Q = FourCharsExtVec{5, 0, 6, 0} && 3;
+  // CHECK: store <4 x i8> <i8 1, i8 0, i8 1, i8 0>
+  constexpr auto R = FourCharsExtVec{5, 0, 6, 0} || 3;
+  // CHECK: store <4 x i8> <i8 1, i8 1, i8 1, i8 1>
+
+  constexpr auto T = CmpMul(a, b);
+  // CHECK: store <4 x i8> <i8 108, i8 18, i8 56, i8 72>
+
+  constexpr auto U = CmpDiv(a, b);
+  // CHECK: store <4 x i8> <i8 3, i8 18, i8 0, i8 0>
+
+  constexpr auto V = CmpRem(a, b);
+  // CHECK: store <4 x i8> <i8 0, i8 0, i8 7, i8 8>
+
+  constexpr auto X = CmpAdd(a, b);
+  // CHECK: store <4 x i8> <i8 24, i8 19, i8 15, i8 17>
+
+  constexpr auto Y = CmpSub(a, b);
+  // CHECK: store <4 x i8> <i8 12, i8 17, i8 -1, i8 -1>
+
+  constexpr auto Z = CmpLSH(a, H);
+  // CHECK: store <4 x i8> <i8 36, i8 36, i8 7, i8 16>
+
+  constexpr auto aa = CmpRSH(a, H);
+  // CHECK: store <4 x i8> <i8 9, i8 9, i8 7, i8 4>
+
+  constexpr auto ab = CmpBinAnd(a, b);
+  // CHECK: store <4 x i8> <i8 2, i8 0, i8 0, i8 8>
+
+  constexpr auto ac = CmpBinXOr(a, b);
+  // CHECK: store <4 x i8> <i8 20, i8 19, i8 15, i8 1>
+
+  constexpr auto ad = CmpBinOr(a, b);
+  // CHECK: store <4 x i8> <i8 22, i8 19, i8 15, i8 9>
+}
+
+void FloatUsage() {
+  constexpr auto a = FourFloatsVecSize{6, 3, 2, 1} +
+                     FourFloatsVecSize{12, 15, 5, 7};
+  // CHECK: <4 x float> <float 1.800000e+01, float 1.800000e+01, float 7.000000e+00, float 8.000000e+00>
+  constexpr auto b = FourFloatsVecSize{19, 15, 13, 12} -
+                     FourFloatsVecSize{13, 14, 5, 3};
+  // CHECK: store <4 x float> <float 6.000000e+00, float 1.000000e+00, float 8.000000e+00, float 9.000000e+00>
+  constexpr auto c = FourFloatsVecSize{8, 4, 2, 1} *
+                     FourFloatsVecSize{3, 4, 5, 6};
+  // CHECK: store <4 x float> <float 2.400000e+01, float 1.600000e+01, float 1.000000e+01, float 6.000000e+00>
+  constexpr auto d = FourFloatsVecSize{12, 12, 10, 10} /
+                     FourFloatsVecSize{6, 4, 5, 2};
+  // CHECK: store <4 x float> <float 2.000000e+00, float 3.000000e+00, float 2.000000e+00, float 5.000000e+00>
+
+  constexpr auto f = FourFloatsVecSize{6, 3, 2, 1} + 3;
+  // CHECK: store <4 x float> <float 9.000000e+00, float 6.000000e+00, float 5.000000e+00, float 4.000000e+00>
+  constexpr auto g = FourFloatsVecSize{19, 15, 12, 10} - 3;
+  // CHECK: store <4 x float> <float 1.600000e+01, float 1.200000e+01, float 9.000000e+00, float 7.000000e+00>
+  constexpr auto h = FourFloatsVecSize{8, 4, 2, 1} * 3;
+  // CHECK: store <4 x float> <float 2.400000e+01, float 1.200000e+01, float 6.000000e+00, float 3.000000e+00>
+  constexpr auto j = FourFloatsVecSize{12, 15, 18, 21} / 3;
+  // CHECK: store <4 x float> <float 4.000000e+00, float 5.000000e+00, float 6.000000e+00, float 7.000000e+00>
+
+  constexpr auto l = 3 + FourFloatsVecSize{6, 3, 2, 1};
+  // CHECK: store <4 x float> <float 9.000000e+00, float 6.000000e+00, float 5.000000e+00, float 4.000000e+00>
+  constexpr auto m = 20 - FourFloatsVecSize{19, 15, 12, 10};
+  // CHECK: store <4 x float> <float 1.000000e+00, float 5.000000e+00, float 8.000000e+00, float 1.000000e+01>
+  constexpr auto n = 3 * FourFloatsVecSize{8, 4, 2, 1};
+  // CHECK: store <4 x float> <float 2.400000e+01, float 1.200000e+01, float 6.000000e+00, float 3.000000e+00>
+  constexpr auto o = 100 / FourFloatsVecSize{12, 15, 18, 21};
+  // CHECK: store <4 x float> <float 0x4020AAAAA0000000, float 0x401AAAAAA0000000, float 0x401638E380000000, float 0x40130C30C0000000>
+
+  constexpr auto w = FourFloatsVecSize{1, 2, 3, 4} <
+                     FourFloatsVecSize{4, 3, 2, 1};
+  // CHECK: store <4 x i32> <i32 1, i32 1, i32 0, i32 0>
+  constexpr auto x = FourFloatsVecSize{1, 2, 3, 4} >
+                     FourFloatsVecSize{4, 3, 2, 1};
+  // CHECK: store <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+  constexpr auto y = FourFloatsVecSize{1, 2, 3, 4} <=
+                     FourFloatsVecSize{4, 3, 3, 1};
+  // CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 0>
+  constexpr auto z = FourFloatsVecSize{1, 2, 3, 4} >=
+                     FourFloatsVecSize{4, 3, 3, 1};
+  // CHECK: store <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+  constexpr auto A = FourFloatsVecSize{1, 2, 3, 4} ==
+                     FourFloatsVecSize{4, 3, 3, 1};
+  // CHECK: store <4 x i32> <i32 0, i32 0, i32 1, i32 0>
+  constexpr auto B = FourFloatsVecSize{1, 2, 3, 4} !=
+                     FourFloatsVecSize{4, 3, 3, 1};
+  // CHECK: store <4 x i32> <i32 1, i32 1, i32 0, i32 1>
+
+  constexpr auto C = FourFloatsVecSize{1, 2, 3, 4} < 3;
+  // CHECK: store <4 x i32> <i32 1, i32 1, i32 0, i32 0>
+  constexpr auto D = FourFloatsVecSize{1, 2, 3, 4} > 3;
+  // CHECK: store <4 x i32> <i32 0, i32 0, i32 0, i32 1>
+  constexpr auto E = FourFloatsVecSize{1, 2, 3, 4} <= 3;
+  // CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 0>
+  constexpr auto F = FourFloatsVecSize{1, 2, 3, 4} >= 3;
+  // CHECK: store <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+  constexpr auto G = FourFloatsVecSize{1, 2, 3, 4} == 3;
+  // CHECK: store <4 x i32> <i32 0, i32 0, i32 1, i32 0>
+  constexpr auto H = FourFloatsVecSize{1, 2, 3, 4} != 3;
+  // CHECK: store <4 x i32> <i32 1, i32 1, i32 0, i32 1>
+
+  constexpr auto O = FourFloatsVecSize{5, 0, 6, 0} &&
+                     FourFloatsVecSize{5, 5, 0, 0};
+  // CHECK: store <4 x i32> <i32 1, i32 0, i32 0, i32 0>
+  constexpr auto P = FourFloatsVecSize{5, 0, 6, 0} ||
+                     FourFloatsVecSize{5, 5, 0, 0};
+  // CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 0>
+
+  constexpr auto Q = FourFloatsVecSize{5, 0, 6, 0} && 3;
+  // CHECK: store <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+  constexpr auto R = FourFloatsVecSize{5, 0, 6, 0} || 3;
+  // CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+
+  constexpr auto T = CmpMul(a, b);
+  // CHECK: store <4 x float> <float 1.080000e+02, float 1.800000e+01, float 5.600000e+01, float 7.200000e+01>
+
+  constexpr auto U = CmpDiv(a, b);
+  // CHECK: store <4 x float> <float 3.000000e+00, float 1.800000e+01, float 8.750000e-01, float 0x3FEC71C720000000>
+
+  constexpr auto X = CmpAdd(a, b);
+  // CHECK: store <4 x float> <float 2.400000e+01, float 1.900000e+01, float 1.500000e+01, float 1.700000e+01>
+
+  constexpr auto Y = CmpSub(a, b);
+  // CHECK: store <4 x float> <float 1.200000e+01, float 1.700000e+01, float -1.000000e+00, float -1.000000e+00>
+}
+
+void FloatVecUsage() {
+  constexpr auto a = FourFloatsVecSize{6, 3, 2, 1} +
+                     FourFloatsVecSize{12, 15, 5, 7};
+  // CHECK: <4 x float> <float 1.800000e+01, float 1.800000e+01, float 7.000000e+00, float 8.000000e+00>
+  constexpr auto b = FourFloatsVecSize{19, 15, 13, 12} -
+                     FourFloatsVecSize{13, 14, 5, 3};
+  // CHECK: store <4 x float> <float 6.000000e+00, float 1.000000e+00, float 8.000000e+00, float 9.000000e+00>
+  constexpr auto c = FourFloatsVecSize{8, 4, 2, 1} *
+                     FourFloatsVecSize{3, 4, 5, 6};
+  // CHECK: store <4 x float> <float 2.400000e+01, float 1.600000e+01, float 1.000000e+01, float 6.000000e+00>
+  constexpr auto d = FourFloatsVecSize{12, 12, 10, 10} /
+                     FourFloatsVecSize{6, 4, 5, 2};
+  // CHECK: store <4 x float> <float 2.000000e+00, float 3.000000e+00, float 2.000000e+00, float 5.000000e+00>
+
+  constexpr auto f = FourFloatsVecSize{6, 3, 2, 1} + 3;
+  // CHECK: store <4 x float> <float 9.000000e+00, float 6.000000e+00, float 5.000000e+00, float 4.000000e+00>
+  constexpr auto g = FourFloatsVecSize{19, 15, 12, 10} - 3;
+  // CHECK: store <4 x float> <float 1.600000e+01, float 1.200000e+01, float 9.000000e+00, float 7.000000e+00>
+  constexpr auto h = FourFloatsVecSize{8, 4, 2, 1} * 3;
+  // CHECK: store <4 x float> <float 2.400000e+01, float 1.200000e+01, float 6.000000e+00, float 3.000000e+00>
+  constexpr auto j = FourFloatsVecSize{12, 15, 18, 21} / 3;
+  // CHECK: store <4 x float> <float 4.000000e+00, float 5.000000e+00, float 6.000000e+00, float 7.000000e+00>
+
+  constexpr auto l = 3 + FourFloatsVecSize{6, 3, 2, 1};
+  // CHECK: store <4 x float> <float 9.000000e+00, float 6.000000e+00, float 5.000000e+00, float 4.000000e+00>
+  constexpr auto m = 20 - FourFloatsVecSize{19, 15, 12, 10};
+  // CHECK: store <4 x float> <float 1.000000e+00, float 5.000000e+00, float 8.000000e+00, float 1.000000e+01>
+  constexpr auto n = 3 * FourFloatsVecSize{8, 4, 2, 1};
+  // CHECK: store <4 x float> <float 2.400000e+01, float 1.200000e+01, float 6.000000e+00, float 3.000000e+00>
+  constexpr auto o = 100 / FourFloatsVecSize{12, 15, 18, 21};
+  // CHECK: store <4 x float> <float 0x4020AAAAA0000000, float 0x401AAAAAA0000000, float 0x401638E380000000, float 0x40130C30C0000000>
+
+  constexpr auto w = FourFloatsVecSize{1, 2, 3, 4} <
+                     FourFloatsVecSize{4, 3, 2, 1};
+  // CHECK: store <4 x i32> <i32 1, i32 1, i32 0, i32 0>
+  constexpr auto x = FourFloatsVecSize{1, 2, 3, 4} >
+                     FourFloatsVecSize{4, 3, 2, 1};
+  // CHECK: store <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+  constexpr auto y = FourFloatsVecSize{1, 2, 3, 4} <=
+                     FourFloatsVecSize{4, 3, 3, 1};
+  // CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 0>
+  constexpr auto z = FourFloatsVecSize{1, 2, 3, 4} >=
+                     FourFloatsVecSize{4, 3, 3, 1};
+  // CHECK: store <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+  constexpr auto A = FourFloatsVecSize{1, 2, 3, 4} ==
+                     FourFloatsVecSize{4, 3, 3, 1};
+  // CHECK: store <4 x i32> <i32 0, i32 0, i32 1, i32 0>
+  constexpr auto B = FourFloatsVecSize{1, 2, 3, 4} !=
+                     FourFloatsVecSize{4, 3, 3, 1};
+  // CHECK: store <4 x i32> <i32 1, i32 1, i32 0, i32 1>
+
+  constexpr auto C = FourFloatsVecSize{1, 2, 3, 4} < 3;
+  // CHECK: store <4 x i32> <i32 1, i32 1, i32 0, i32 0>
+  constexpr auto D = FourFloatsVecSize{1, 2, 3, 4} > 3;
+  // CHECK: store <4 x i32> <i32 0, i32 0, i32 0, i32 1>
+  constexpr auto E = FourFloatsVecSize{1, 2, 3, 4} <= 3;
+  // CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 0>
+  constexpr auto F = FourFloatsVecSize{1, 2, 3, 4} >= 3;
+  // CHECK: store <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+  constexpr auto G = FourFloatsVecSize{1, 2, 3, 4} == 3;
+  // CHECK: store <4 x i32> <i32 0, i32 0, i32 1, i32 0>
+  constexpr auto H = FourFloatsVecSize{1, 2, 3, 4} != 3;
+  // CHECK: store <4 x i32> <i32 1, i32 1, i32 0, i32 1>
+
+  constexpr auto O = FourFloatsVecSize{5, 0, 6, 0} &&
+                     FourFloatsVecSize{5, 5, 0, 0};
+  // CHECK: store <4 x i32> <i32 1, i32 0, i32 0, i32 0>
+  constexpr auto P = FourFloatsVecSize{5, 0, 6, 0} ||
+                     FourFloatsVecSize{5, 5, 0, 0};
+  // CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 0>
+
+  constexpr auto Q = FourFloatsVecSize{5, 0, 6, 0} && 3;
+  // CHECK: store <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+  constexpr auto R = FourFloatsVecSize{5, 0, 6, 0} || 3;
+  // CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+
+  constexpr auto T = CmpMul(a, b);
+  // CHECK: store <4 x float> <float 1.080000e+02, float 1.800000e+01, float 5.600000e+01, float 7.200000e+01>
+
+  constexpr auto U = CmpDiv(a, b);
+  // CHECK: store <4 x float> <float 3.000000e+00, float 1.800000e+01, float 8.750000e-01, float 0x3FEC71C720000000>
+
+  constexpr auto X = CmpAdd(a, b);
+  // CHECK: store <4 x float> <float 2.400000e+01, float 1.900000e+01, float 1.500000e+01, float 1.700000e+01>
+
+  constexpr auto Y = CmpSub(a, b);
+  // CHECK: store <4 x float> <float 1.200000e+01, float 1.700000e+01, float -1.000000e+00, float -1.000000e+00>
+}
diff --git a/clang/test/SemaCXX/coreturn-eh.cpp b/clang/test/SemaCXX/coreturn-eh.cpp
index 79065736c0a41..591ab8ec5c5e4 100644
--- a/clang/test/SemaCXX/coreturn-eh.cpp
+++ b/clang/test/SemaCXX/coreturn-eh.cpp
@@ -17,7 +17,7 @@ struct object { ~object() {} };
 struct promise_void_return_value {
   void get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void unhandled_exception();
   void return_value(object);
 };
@@ -26,7 +26,7 @@ struct VoidTagReturnValue {
   struct promise_type {
     VoidTagReturnValue get_return_object();
     suspend_always initial_suspend();
-    suspend_always final_suspend();
+    suspend_always final_suspend() noexcept;
     void unhandled_exception();
     void return_value(object);
   };
diff --git a/clang/test/SemaCXX/coreturn.cpp b/clang/test/SemaCXX/coreturn.cpp
index 4d8bfa11d832c..eaa462016de55 100644
--- a/clang/test/SemaCXX/coreturn.cpp
+++ b/clang/test/SemaCXX/coreturn.cpp
@@ -13,7 +13,7 @@ struct awaitable {
 struct promise_void {
   void get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void return_void();
   void unhandled_exception();
 };
@@ -21,7 +21,7 @@ struct promise_void {
 struct promise_void_return_value {
   void get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void unhandled_exception();
   void return_value(int);
 };
@@ -30,7 +30,7 @@ struct VoidTagNoReturn {
   struct promise_type {
     VoidTagNoReturn get_return_object();
     suspend_always initial_suspend();
-    suspend_always final_suspend();
+    suspend_always final_suspend() noexcept;
     void unhandled_exception();
   };
 };
@@ -39,7 +39,7 @@ struct VoidTagReturnValue {
   struct promise_type {
     VoidTagReturnValue get_return_object();
     suspend_always initial_suspend();
-    suspend_always final_suspend();
+    suspend_always final_suspend() noexcept;
     void unhandled_exception();
     void return_value(int);
   };
@@ -49,7 +49,7 @@ struct VoidTagReturnVoid {
   struct promise_type {
     VoidTagReturnVoid get_return_object();
     suspend_always initial_suspend();
-    suspend_always final_suspend();
+    suspend_always final_suspend() noexcept;
     void unhandled_exception();
     void return_void();
   };
@@ -58,7 +58,7 @@ struct VoidTagReturnVoid {
 struct promise_float {
   float get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void return_void();
   void unhandled_exception();
 };
@@ -66,7 +66,7 @@ struct promise_float {
 struct promise_int {
   int get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void return_value(int);
   void unhandled_exception();
 };
diff --git a/clang/test/SemaCXX/coroutine-final-suspend-noexcept.cpp b/clang/test/SemaCXX/coroutine-final-suspend-noexcept.cpp
new file mode 100644
index 0000000000000..48c65f8afb951
--- /dev/null
+++ b/clang/test/SemaCXX/coroutine-final-suspend-noexcept.cpp
@@ -0,0 +1,72 @@
+// This file contains references to sections of the Coroutines TS, which can be
+// found at http://wg21.link/coroutines.
+
+// RUN: %clang_cc1 -std=c++14 -fcoroutines-ts -verify %s -fcxx-exceptions -fexceptions -Wunused-result
+
+namespace std {
+namespace experimental {
+
+template <class Ret, typename... T>
+struct coroutine_traits { using promise_type = typename Ret::promise_type; };
+
+template <class Promise = void>
+struct coroutine_handle {
+  static coroutine_handle from_address(void *); // expected-note 2 {{must be declared with 'noexcept'}}
+};
+template <>
+struct coroutine_handle<void> {
+  template <class PromiseType>
+  coroutine_handle(coroutine_handle<PromiseType>); // expected-note 2 {{must be declared with 'noexcept'}}
+};
+
+struct suspend_never {
+  bool await_ready() { return true; }       // expected-note 2 {{must be declared with 'noexcept'}}
+  void await_suspend(coroutine_handle<>) {} // expected-note 2 {{must be declared with 'noexcept'}}
+  void await_resume() {}                    // expected-note 2 {{must be declared with 'noexcept'}}
+  ~suspend_never() noexcept(false);         // expected-note 2 {{must be declared with 'noexcept'}}
+};
+
+struct suspend_always {
+  bool await_ready() { return false; }
+  void await_suspend(coroutine_handle<>) {}
+  void await_resume() {}
+  suspend_never operator co_await(); // expected-note 2 {{must be declared with 'noexcept'}}
+  ~suspend_always() noexcept(false); // expected-note 2 {{must be declared with 'noexcept'}}
+};
+
+} // namespace experimental
+} // namespace std
+
+using namespace std::experimental;
+
+struct A {
+  bool await_ready();
+  void await_resume();
+  template <typename F>
+  void await_suspend(F);
+};
+
+struct coro_t {
+  struct promise_type {
+    coro_t get_return_object();
+    suspend_never initial_suspend();
+    suspend_always final_suspend(); // expected-note 2 {{must be declared with 'noexcept'}}
+    void return_void();
+    static void unhandled_exception();
+  };
+};
+
+coro_t f(int n) { // expected-error {{the expression 'co_await __promise.final_suspend()' is required to be non-throwing}}
+  A a{};
+  co_await a;
+}
+
+template <typename T>
+coro_t f_dep(T n) { // expected-error {{the expression 'co_await __promise.final_suspend()' is required to be non-throwing}}
+  A a{};
+  co_await a;
+}
+
+void foo() {
+  f_dep<int>(5); // expected-note {{in instantiation of function template specialization 'f_dep<int>' requested here}}
+}
diff --git a/clang/test/SemaCXX/coroutine-rvo.cpp b/clang/test/SemaCXX/coroutine-rvo.cpp
index 8521b8506fd04..1cd025052dca8 100644
--- a/clang/test/SemaCXX/coroutine-rvo.cpp
+++ b/clang/test/SemaCXX/coroutine-rvo.cpp
@@ -49,7 +49,7 @@ template <typename T>
 struct task {
   struct promise_type {
     auto initial_suspend() { return suspend_never{}; }
-    auto final_suspend() { return suspend_never{}; }
+    auto final_suspend() noexcept { return suspend_never{}; }
     auto get_return_object() { return task{}; }
     static void unhandled_exception() {}
     void return_value(T&& value) {}
diff --git a/clang/test/SemaCXX/coroutine-unhandled_exception-warning.cpp b/clang/test/SemaCXX/coroutine-unhandled_exception-warning.cpp
index d819580462c8e..88fae2e8acb26 100644
--- a/clang/test/SemaCXX/coroutine-unhandled_exception-warning.cpp
+++ b/clang/test/SemaCXX/coroutine-unhandled_exception-warning.cpp
@@ -23,7 +23,7 @@ struct promise_void {
 #endif
   void get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void return_void();
 };
 
diff --git a/clang/test/SemaCXX/coroutine-uninitialized-warning-crash.cpp b/clang/test/SemaCXX/coroutine-uninitialized-warning-crash.cpp
index 5bdb232d5307c..391f64d37e5bf 100644
--- a/clang/test/SemaCXX/coroutine-uninitialized-warning-crash.cpp
+++ b/clang/test/SemaCXX/coroutine-uninitialized-warning-crash.cpp
@@ -16,7 +16,7 @@ struct coro_t {
   struct promise_type {
     coro_t get_return_object() { return {}; }
     suspend_never initial_suspend() { return {}; }
-    suspend_never final_suspend() { return {}; }
+    suspend_never final_suspend() noexcept { return {}; }
     A yield_value(int) { return {}; }
     void return_void() {}
     static void unhandled_exception() {}
diff --git a/clang/test/SemaCXX/coroutine_handle-addres-return-type.cpp b/clang/test/SemaCXX/coroutine_handle-addres-return-type.cpp
new file mode 100644
index 0000000000000..a951383652344
--- /dev/null
+++ b/clang/test/SemaCXX/coroutine_handle-addres-return-type.cpp
@@ -0,0 +1,75 @@
+// RUN: %clang_cc1 -verify %s -stdlib=libc++ -std=c++1z -fcoroutines-ts -fsyntax-only
+
+namespace std::experimental {
+template <class Promise = void>
+struct coroutine_handle;
+
+template <>
+struct coroutine_handle<void> {
+  coroutine_handle() = default;
+  static coroutine_handle from_address(void *) noexcept;
+  void *address() const;
+};
+
+template <class Promise>
+struct coroutine_handle : public coroutine_handle<> {
+};
+
+template <class... Args>
+struct void_t_imp {
+  using type = void;
+};
+template <class... Args>
+using void_t = typename void_t_imp<Args...>::type;
+
+template <class T, class = void>
+struct traits_sfinae_base {};
+
+template <class T>
+struct traits_sfinae_base<T, void_t<typename T::promise_type>> {
+  using promise_type = typename T::promise_type;
+};
+
+template <class Ret, class... Args>
+struct coroutine_traits : public traits_sfinae_base<Ret> {};
+} // namespace std::experimental
+
+struct suspend_never {
+  bool await_ready() noexcept;
+  void await_suspend(std::experimental::coroutine_handle<>) noexcept;
+  void await_resume() noexcept;
+};
+
+struct task {
+  struct promise_type {
+    auto initial_suspend() { return suspend_never{}; }
+    auto final_suspend() noexcept { return suspend_never{}; }
+    auto get_return_object() { return task{}; }
+    static void unhandled_exception() {}
+    void return_void() {}
+  };
+};
+
+namespace std::experimental {
+template <>
+struct coroutine_handle<task::promise_type> : public coroutine_handle<> {
+  coroutine_handle<task::promise_type> *address() const; // expected-warning {{return type of 'coroutine_handle<>::address should be 'void*'}}
+};
+} // namespace std::experimental
+
+struct awaitable {
+  bool await_ready();
+
+  std::experimental::coroutine_handle<task::promise_type>
+  await_suspend(std::experimental::coroutine_handle<> handle);
+  void await_resume();
+} a;
+
+task f() {
+  co_await a;
+}
+
+int main() {
+  f();
+  return 0;
+}
diff --git a/clang/test/SemaCXX/coroutines.cpp b/clang/test/SemaCXX/coroutines.cpp
index 9e94fe8c9c109..f354b1f2a4bb0 100644
--- a/clang/test/SemaCXX/coroutines.cpp
+++ b/clang/test/SemaCXX/coroutines.cpp
@@ -52,21 +52,24 @@ struct std::experimental::coroutine_traits<coro<Promise>, Ps...> {
 };
 
 struct awaitable {
-  bool await_ready();
-  template <typename F> void await_suspend(F);
-  void await_resume();
+  bool await_ready() noexcept;
+  template <typename F>
+  void await_suspend(F) noexcept;
+  void await_resume() noexcept;
 } a;
 
 struct suspend_always {
-  bool await_ready() { return false; }
-  template <typename F> void await_suspend(F);
-  void await_resume() {}
+  bool await_ready() noexcept { return false; }
+  template <typename F>
+  void await_suspend(F) noexcept;
+  void await_resume() noexcept {}
 };
 
 struct suspend_never {
-  bool await_ready() { return true; }
-  template <typename F> void await_suspend(F);
-  void await_resume() {}
+  bool await_ready() noexcept { return true; }
+  template <typename F>
+  void await_suspend(F) noexcept;
+  void await_resume() noexcept {}
 };
 
 struct auto_await_suspend {
@@ -127,7 +130,7 @@ struct not_awaitable {};
 struct promise {
   void get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   awaitable yield_value(int); // expected-note 2{{candidate}}
   awaitable yield_value(yielded_thing); // expected-note 2{{candidate}}
   not_awaitable yield_value(void()); // expected-note 2{{candidate}}
@@ -138,7 +141,7 @@ struct promise {
 struct promise_void {
   void get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void return_void();
   void unhandled_exception();
 };
@@ -152,13 +155,13 @@ namespace std {
 namespace experimental {
 template <class PromiseType = void>
 struct coroutine_handle {
-  static coroutine_handle from_address(void *);
+  static coroutine_handle from_address(void *) noexcept;
 };
 template <>
 struct coroutine_handle<void> {
   template <class PromiseType>
-  coroutine_handle(coroutine_handle<PromiseType>);
-  static coroutine_handle from_address(void *);
+  coroutine_handle(coroutine_handle<PromiseType>) noexcept;
+  static coroutine_handle from_address(void *) noexcept;
 };
 }} // namespace std::experimental
 
@@ -402,7 +405,7 @@ struct await_arg_2 {};
 
 namespace adl_ns {
 struct coawait_arg_type {};
-awaitable operator co_await(coawait_arg_type);
+awaitable operator co_await(coawait_arg_type) noexcept;
 }
 
 namespace dependent_operator_co_await_lookup {
@@ -434,7 +437,7 @@ namespace dependent_operator_co_await_lookup {
     typedef transform_awaitable await_arg;
     coro<transform_promise> get_return_object();
     transformed initial_suspend();
-    ::adl_ns::coawait_arg_type final_suspend();
+    ::adl_ns::coawait_arg_type final_suspend() noexcept;
     transformed await_transform(transform_awaitable);
     void unhandled_exception();
     void return_void();
@@ -444,7 +447,7 @@ namespace dependent_operator_co_await_lookup {
     typedef AwaitArg await_arg;
     coro<basic_promise> get_return_object();
     awaitable initial_suspend();
-    awaitable final_suspend();
+    awaitable final_suspend() noexcept;
     void unhandled_exception();
     void return_void();
   };
@@ -529,7 +532,7 @@ struct std::experimental::coroutine_traits<void, yield_fn_tag> {
     void return_value(int());
 
     suspend_never initial_suspend();
-    suspend_never final_suspend();
+    suspend_never final_suspend() noexcept;
     void get_return_object();
     void unhandled_exception();
   };
@@ -563,7 +566,7 @@ namespace placeholder {
 
 struct bad_promise_1 {
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void unhandled_exception();
   void return_void();
 };
@@ -573,7 +576,7 @@ coro<bad_promise_1> missing_get_return_object() { // expected-error {{no member
 
 struct bad_promise_2 {
   coro<bad_promise_2> get_return_object();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void unhandled_exception();
   void return_void();
 };
@@ -588,14 +591,14 @@ struct bad_promise_3 {
   void unhandled_exception();
   void return_void();
 };
-coro<bad_promise_3> missing_final_suspend() { // expected-error {{no member named 'final_suspend' in 'bad_promise_3'}}
+coro<bad_promise_3> missing_final_suspend() noexcept { // expected-error {{no member named 'final_suspend' in 'bad_promise_3'}}
   co_await a;
 }
 
 struct bad_promise_4 {
   coro<bad_promise_4> get_return_object();
   not_awaitable initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void return_void();
 };
 // FIXME: This diagnostic is terrible.
@@ -607,7 +610,7 @@ coro<bad_promise_4> bad_initial_suspend() { // expected-error {{no member named
 struct bad_promise_5 {
   coro<bad_promise_5> get_return_object();
   suspend_always initial_suspend();
-  not_awaitable final_suspend();
+  not_awaitable final_suspend() noexcept;
   void return_void();
 };
 // FIXME: This diagnostic is terrible.
@@ -619,7 +622,7 @@ coro<bad_promise_5> bad_final_suspend() { // expected-error {{no member named 'a
 struct bad_promise_6 {
   coro<bad_promise_6> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void unhandled_exception();
   void return_void();           // expected-note 2 {{member 'return_void' first declared here}}
   void return_value(int) const; // expected-note 2 {{member 'return_value' first declared here}}
@@ -638,7 +641,7 @@ template coro<bad_promise_6> bad_implicit_return_dependent(bad_promise_6); // ex
 struct bad_promise_7 { // expected-note 2 {{defined here}}
   coro<bad_promise_7> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void return_void();
 };
 coro<bad_promise_7> no_unhandled_exception() { // expected-error {{'bad_promise_7' is required to declare the member 'unhandled_exception()'}}
@@ -658,7 +661,7 @@ struct bad_promise_base {
 struct bad_promise_8 : bad_promise_base {
   coro<bad_promise_8> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void unhandled_exception() __attribute__((unavailable)); // expected-note 2 {{marked unavailable here}}
   void unhandled_exception() const;
   void unhandled_exception(void *) const;
@@ -680,7 +683,7 @@ template coro<bad_promise_8> calls_unhandled_exception_dependent(bad_promise_8);
 struct bad_promise_9 {
   coro<bad_promise_9> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void await_transform(void *);
   awaitable await_transform(int) __attribute__((unavailable)); // expected-note {{explicitly marked unavailable}}
   void return_void();
@@ -693,7 +696,7 @@ coro<bad_promise_9> calls_await_transform() {
 struct bad_promise_10 {
   coro<bad_promise_10> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   int await_transform;
   void return_void();
   void unhandled_exception();
@@ -712,7 +715,7 @@ void ret_void();
 struct good_promise_1 {
   coro<good_promise_1> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void unhandled_exception();
   static const call_operator await_transform;
   using Fn = void (*)();
@@ -750,7 +753,7 @@ int main(int, const char**) {
 struct good_promise_2 {
   float get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void return_void();
   void unhandled_exception();
 };
@@ -783,7 +786,7 @@ struct std::experimental::coroutine_traits<int, promise_on_alloc_failure_tag> {
   struct promise_type {
     int get_return_object() {}
     suspend_always initial_suspend() { return {}; }
-    suspend_always final_suspend() { return {}; }
+    suspend_always final_suspend() noexcept { return {}; }
     void return_void() {}
     int get_return_object_on_allocation_failure(); // expected-error{{'promise_type': 'get_return_object_on_allocation_failure()' must be a static member function}}
     void unhandled_exception();
@@ -797,7 +800,7 @@ extern "C" int f(promise_on_alloc_failure_tag) {
 struct bad_promise_11 {
   coro<bad_promise_11> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void unhandled_exception();
   void return_void();
 
@@ -820,7 +823,7 @@ template coro<bad_promise_11> dependent_private_alloc_failure_handler(bad_promis
 struct bad_promise_12 {
   coro<bad_promise_12> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void unhandled_exception();
   void return_void();
   static coro<bad_promise_12> get_return_object_on_allocation_failure();
@@ -842,7 +845,7 @@ template coro<bad_promise_12> dependent_throwing_in_class_new(bad_promise_12); /
 struct good_promise_13 {
   coro<good_promise_13> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void unhandled_exception();
   void return_void();
   static coro<good_promise_13> get_return_object_on_allocation_failure();
@@ -860,7 +863,7 @@ template coro<good_promise_13> dependent_uses_nothrow_new(good_promise_13);
 struct good_promise_custom_new_operator {
   coro<good_promise_custom_new_operator> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void return_void();
   void unhandled_exception();
   void *operator new(SizeT, double, float, int);
@@ -876,7 +879,7 @@ struct coroutine_nonstatic_member_struct;
 struct good_promise_nonstatic_member_custom_new_operator {
   coro<good_promise_nonstatic_member_custom_new_operator> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void return_void();
   void unhandled_exception();
   void *operator new(SizeT, coroutine_nonstatic_member_struct &, double);
@@ -886,7 +889,7 @@ struct good_promise_noexcept_custom_new_operator {
   static coro<good_promise_noexcept_custom_new_operator> get_return_object_on_allocation_failure();
   coro<good_promise_noexcept_custom_new_operator> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void return_void();
   void unhandled_exception();
   void *operator new(SizeT, double, float, int) noexcept;
@@ -903,7 +906,7 @@ struct std::experimental::coroutine_traits<int, mismatch_gro_type_tag1> {
   struct promise_type {
     void get_return_object() {} //expected-note {{member 'get_return_object' declared here}}
     suspend_always initial_suspend() { return {}; }
-    suspend_always final_suspend() { return {}; }
+    suspend_always final_suspend() noexcept { return {}; }
     void return_void() {}
     void unhandled_exception();
   };
@@ -920,7 +923,7 @@ struct std::experimental::coroutine_traits<int, mismatch_gro_type_tag2> {
   struct promise_type {
     void *get_return_object() {} //expected-note {{member 'get_return_object' declared here}}
     suspend_always initial_suspend() { return {}; }
-    suspend_always final_suspend() { return {}; }
+    suspend_always final_suspend() noexcept { return {}; }
     void return_void() {}
     void unhandled_exception();
   };
@@ -938,7 +941,7 @@ struct std::experimental::coroutine_traits<int, mismatch_gro_type_tag3> {
     int get_return_object() {}
     static void get_return_object_on_allocation_failure() {} //expected-note {{member 'get_return_object_on_allocation_failure' declared here}}
     suspend_always initial_suspend() { return {}; }
-    suspend_always final_suspend() { return {}; }
+    suspend_always final_suspend() noexcept { return {}; }
     void return_void() {}
     void unhandled_exception();
   };
@@ -957,7 +960,7 @@ struct std::experimental::coroutine_traits<int, mismatch_gro_type_tag4> {
     int get_return_object() {}
     static char *get_return_object_on_allocation_failure() {} //expected-note {{member 'get_return_object_on_allocation_failure' declared}}
     suspend_always initial_suspend() { return {}; }
-    suspend_always final_suspend() { return {}; }
+    suspend_always final_suspend() noexcept { return {}; }
     void return_void() {}
     void unhandled_exception();
   };
@@ -971,7 +974,7 @@ extern "C" int f(mismatch_gro_type_tag4) {
 struct bad_promise_no_return_func { // expected-note {{'bad_promise_no_return_func' defined here}}
   coro<bad_promise_no_return_func> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void unhandled_exception();
 };
 // FIXME: The PDTS currently specifies this as UB, technically forbidding a
@@ -1083,7 +1086,7 @@ struct CoroMemberPromise {
 
   CoroMemberTag get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
 
   AwaitTestT yield_value(int);
 
@@ -1292,7 +1295,7 @@ struct bad_promise_deleted_constructor {
   bad_promise_deleted_constructor() = delete;
   coro<bad_promise_deleted_constructor> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void return_void();
   void unhandled_exception();
 };
@@ -1314,7 +1317,7 @@ struct good_promise_default_constructor {
   good_promise_default_constructor() = default;
   coro<good_promise_default_constructor> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void return_void();
   void unhandled_exception();
 };
@@ -1332,7 +1335,7 @@ struct good_promise_custom_constructor {
   good_promise_custom_constructor() = delete;
   coro<good_promise_custom_constructor> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void return_void();
   void unhandled_exception();
 };
@@ -1359,7 +1362,7 @@ struct bad_promise_no_matching_constructor {
   bad_promise_no_matching_constructor() = delete;
   coro<bad_promise_no_matching_constructor> get_return_object();
   suspend_always initial_suspend();
-  suspend_always final_suspend();
+  suspend_always final_suspend() noexcept;
   void return_void();
   void unhandled_exception();
 };
@@ -1383,26 +1386,25 @@ bad_coroutine_calls_with_no_matching_constructor(int, int, int) {
 class awaitable_no_unused_warn {
 public:
   using handle_type = std::experimental::coroutine_handle<>;
-  constexpr bool await_ready()  { return false; }
+  constexpr bool await_ready() noexcept { return false; }
   void await_suspend(handle_type) noexcept {}
-  int await_resume() { return 1; }
+  int await_resume() noexcept { return 1; }
 };
 
 
 class awaitable_unused_warn {
 public:
   using handle_type = std::experimental::coroutine_handle<>;
-  constexpr bool await_ready()  { return false; }
+  constexpr bool await_ready() noexcept { return false; }
   void await_suspend(handle_type) noexcept {}
-  [[nodiscard]]
-  int await_resume() { return 1; }
+  [[nodiscard]] int await_resume() noexcept { return 1; }
 };
 
 template <class Await>
 struct check_warning_promise {
   coro<check_warning_promise> get_return_object();
   Await initial_suspend();
-  Await final_suspend();
+  Await final_suspend() noexcept;
   Await yield_value(int);
   void return_void();
   void unhandled_exception();
diff --git a/clang/test/SemaCXX/cxx1y-variable-templates_in_class.cpp b/clang/test/SemaCXX/cxx1y-variable-templates_in_class.cpp
index a16a5b54d8e0d..fc49ec88d5537 100644
--- a/clang/test/SemaCXX/cxx1y-variable-templates_in_class.cpp
+++ b/clang/test/SemaCXX/cxx1y-variable-templates_in_class.cpp
@@ -5,7 +5,7 @@
 #define CONST const
 
 #ifdef PRECXX11
-#define static_assert(expr, msg) typedef int static_assert[(expr) ? 1 : -1];
+#define static_assert _Static_assert
 #endif
 
 class A {
@@ -237,7 +237,7 @@ namespace in_class_template {
   namespace definition_after_outer_instantiation {
     template<typename A> struct S {
       template<typename B> static const int V1;
-      template<typename B> static const int V2;
+      template<typename B> static const int V2; // expected-note 3{{here}}
     };
     template struct S<int>;
     template<typename A> template<typename B> const int S<A>::V1 = 123;
@@ -250,11 +250,11 @@ namespace in_class_template {
     // is instantiated. This is kind of implied by [temp.class.spec.mfunc]/2,
     // and matches our behavior for member class templates, but it's not clear
     // that this is intentional. See PR17294 and core-24030.
-    static_assert(S<int>::V2<int*> == 456, ""); // FIXME expected-error {{}}
-    static_assert(S<int>::V2<int&> == 789, ""); // expected-error {{}}
+    static_assert(S<int>::V2<int*> == 456, ""); // FIXME expected-error {{}} expected-note {{initializer of 'V2<int *>' is unknown}}
+    static_assert(S<int>::V2<int&> == 789, ""); // expected-error {{}} expected-note {{initializer of 'V2<int &>' is unknown}}
 
     template<typename A> template<typename B> const int S<A>::V2<B&> = 789;
-    static_assert(S<int>::V2<int&> == 789, ""); // FIXME expected-error {{}}
+    static_assert(S<int>::V2<int&> == 789, ""); // FIXME expected-error {{}} expected-note {{initializer of 'V2<int &>' is unknown}}
 
     // All is OK if the partial specialization is declared before the implicit
     // instantiation of the class template specialization.
diff --git a/clang/test/SemaCXX/cxx2a-consteval.cpp b/clang/test/SemaCXX/cxx2a-consteval.cpp
index 0a6d137e96709..3231107add09c 100644
--- a/clang/test/SemaCXX/cxx2a-consteval.cpp
+++ b/clang/test/SemaCXX/cxx2a-consteval.cpp
@@ -476,6 +476,25 @@ namespace override {
   }
 }
 
+namespace operator_rewrite {
+  struct A {
+    friend consteval int operator<=>(const A&, const A&) { return 0; }
+  };
+  const bool k = A() < A();
+  static_assert(!k);
+
+  A a;
+  bool k2 = A() < a; // OK, does not access 'a'.
+
+  struct B {
+    friend consteval int operator<=>(const B &l, const B &r) { return r.n - l.n; } // expected-note {{read of }}
+    int n;
+  };
+  static_assert(B() >= B());
+  B b; // expected-note {{here}}
+  bool k3 = B() < b; // expected-error-re {{call to consteval function '{{.*}}::operator<=>' is not a constant expression}} expected-note {{in call}}
+}
+
 struct A {
   int(*ptr)();
   consteval A(int(*p)() = nullptr) : ptr(p) {}
diff --git a/clang/test/SemaCXX/cxx2a-destroying-delete.cpp b/clang/test/SemaCXX/cxx2a-destroying-delete.cpp
index 553b7a7080fe5..015d11e655262 100644
--- a/clang/test/SemaCXX/cxx2a-destroying-delete.cpp
+++ b/clang/test/SemaCXX/cxx2a-destroying-delete.cpp
@@ -123,3 +123,21 @@ namespace first_param_conversion {
     delete e; // expected-error {{ambiguous conversion from derived class 'first_param_conversion::E' to base class 'first_param_conversion::B':}}
   }
 }
+
+namespace templated {
+  template<typename T> using id_alias = T;
+  template<typename T> struct id_struct { using type = T; };
+
+  template<typename T> struct A {
+    void operator delete(A *, std::destroying_delete_t);
+  };
+  template<typename T> struct B {
+    void operator delete(B<T> *, std::destroying_delete_t);
+  };
+  template<typename T> struct C {
+    void operator delete(id_alias<C> *, std::destroying_delete_t);
+  };
+  template<typename T> struct D {
+    void operator delete(typename id_struct<D>::type *, std::destroying_delete_t); // expected-error {{use 'D<T> *'}}
+  };
+}
diff --git a/clang/test/SemaCXX/invalid-template-base-specifier.cpp b/clang/test/SemaCXX/invalid-template-base-specifier.cpp
new file mode 100644
index 0000000000000..7a1a7f801c45e
--- /dev/null
+++ b/clang/test/SemaCXX/invalid-template-base-specifier.cpp
@@ -0,0 +1,28 @@
+// RUN: %clang_cc1 -frecovery-ast -verify %s
+
+bool Foo(int *); // expected-note 3{{candidate function not viable}}
+
+template <typename T>
+struct Crash : decltype(Foo(T())) { // expected-error {{no matching function for call to 'Foo'}}
+  Crash(){};
+};
+
+void test() { Crash<int>(); } // expected-note {{in instantiation of template class}}
+
+template <typename T>
+using Alias = decltype(Foo(T())); // expected-error {{no matching function for call to 'Foo'}}
+template <typename T>
+struct Crash2 : decltype(Alias<T>()) { // expected-note {{in instantiation of template type alias 'Alias' requested here}}
+  Crash2(){};
+};
+
+void test2() { Crash2<int>(); } // expected-note {{in instantiation of template class 'Crash2<int>' requested here}}
+
+template <typename T>
+class Base {};
+template <typename T>
+struct Crash3 : Base<decltype(Foo(T()))> { // expected-error {{no matching function for call to 'Foo'}}
+  Crash3(){};
+};
+
+void test3() { Crash3<int>(); } // expected-note {{in instantiation of template class}}
diff --git a/clang/test/SemaCXX/lambda-expressions.cpp b/clang/test/SemaCXX/lambda-expressions.cpp
index 0f4edc4d1f343..3240d5351fc5b 100644
--- a/clang/test/SemaCXX/lambda-expressions.cpp
+++ b/clang/test/SemaCXX/lambda-expressions.cpp
@@ -601,16 +601,24 @@ namespace ConversionOperatorDoesNotHaveDeducedReturnType {
 #if __cplusplus > 201402L
     friend constexpr auto T::operator()(int) const;
     friend constexpr T::operator ExpectedTypeT() const noexcept;
+
+    template<typename T>
+      friend constexpr void U::operator()(T&) const;
+    // FIXME: This should not match; the return type is specified as behaving
+    // "as if it were a decltype-specifier denoting the return type of
+    // [operator()]", which is not equivalent to this alias template.
+    template<typename T>
+      friend constexpr U::operator ExpectedTypeU<T>() const noexcept;
 #else
     friend auto T::operator()(int) const;
     friend T::operator ExpectedTypeT() const;
-#endif
 
-    // FIXME: The first of these should match. The second should not.
     template<typename T>
-      friend void U::operator()(T&) const; // expected-error {{does not match}}
+      friend void U::operator()(T&) const;
+    // FIXME: This should not match, as above.
     template<typename T>
-      friend U::operator ExpectedTypeU<T>() const; // expected-error {{does not match}}
+      friend U::operator ExpectedTypeU<T>() const;
+#endif
 
   private:
     int n;
diff --git a/clang/test/SemaCXX/matrix-type-builtins.cpp b/clang/test/SemaCXX/matrix-type-builtins.cpp
index 3850e4ff901b9..fce4d6fb2162b 100644
--- a/clang/test/SemaCXX/matrix-type-builtins.cpp
+++ b/clang/test/SemaCXX/matrix-type-builtins.cpp
@@ -15,9 +15,9 @@ typename MyMatrix<EltTy1, R1, C1>::matrix_t transpose(MyMatrix<EltTy0, R0, C0> &
   // expected-error@-3 {{cannot initialize a variable of type 'char *' with an rvalue of type 'unsigned int __attribute__((matrix_type(3, 3)))'}}
 
   __builtin_matrix_transpose(A);
-  // expected-error@-1 {{first argument must be a matrix}}
-  // expected-error@-2 {{first argument must be a matrix}}
-  // expected-error@-3 {{first argument must be a matrix}}
+  // expected-error@-1 {{1st argument must be a matrix}}
+  // expected-error@-2 {{1st argument must be a matrix}}
+  // expected-error@-3 {{1st argument must be a matrix}}
 
   return __builtin_matrix_transpose(A.value);
   // expected-error@-1 {{cannot initialize return object of type 'typename MyMatrix<unsigned int, 2U, 3U>::matrix_t' (aka 'unsigned int __attribute__((matrix_type(2, 3)))') with an rvalue of type 'unsigned int __attribute__((matrix_type(3, 2)))'}}
@@ -99,13 +99,13 @@ void call_column_major_load_temp(unsigned *Ptr, unsigned X) {
   // expected-error@-1 {{row argument must be a constant unsigned integer expression}}
   // expected-error@-2 {{column argument must be a constant unsigned integer expression}}
   (void)__builtin_matrix_column_major_load(X, 2, 2, 2);
-  // expected-error@-1 {{first argument must be a pointer to a valid matrix element type}}
+  // expected-error@-1 {{1st argument must be a pointer to a valid matrix element type}}
 }
 
 template <typename EltTy0, unsigned R0, unsigned C0, typename PtrTy>
 void column_major_store(MyMatrix<EltTy0, R0, C0> &A, PtrTy Ptr, unsigned Stride) {
   __builtin_matrix_column_major_store(A.value, Ptr, Stride);
-  // expected-error@-1 {{the pointee of the second argument must match the element type of the first argument ('float' != 'unsigned int')}}
+  // expected-error@-1 {{the pointee of the 2nd argument must match the element type of the 1st argument ('float' != 'unsigned int')}}
 }
 
 template <typename MTy, typename PtrTy, unsigned Stride>
@@ -126,14 +126,14 @@ template <typename EltTy0, unsigned R0, unsigned C0, typename EltTy1>
 void column_major_store(MyMatrix<EltTy0, R0, C0> &A, EltTy1 *Ptr) {
   __builtin_matrix_column_major_store(A.value, Ptr, 1);
   // expected-error@-1 3 {{stride must be greater or equal to the number of rows}}
-  // expected-error@-2 {{the pointee of the second argument must match the element type of the first argument ('float' != 'unsigned int')}}
-  // expected-error@-3 {{the pointee of the second argument must match the element type of the first argument ('unsigned int' != 'float')}}
+  // expected-error@-2 {{the pointee of the 2nd argument must match the element type of the 1st argument ('float' != 'unsigned int')}}
+  // expected-error@-3 {{the pointee of the 2nd argument must match the element type of the 1st argument ('unsigned int' != 'float')}}
 
   char *s;
   return __builtin_matrix_column_major_store(A.value, s, 20);
-  // expected-error@-1 {{the pointee of the second argument must match the element type of the first argument ('char' != 'unsigned int')}}
-  // expected-error@-2 {{the pointee of the second argument must match the element type of the first argument ('char' != 'unsigned int')}}
-  // expected-error@-3 {{he pointee of the second argument must match the element type of the first argument ('char' != 'float')}}
+  // expected-error@-1 {{the pointee of the 2nd argument must match the element type of the 1st argument ('char' != 'unsigned int')}}
+  // expected-error@-2 {{the pointee of the 2nd argument must match the element type of the 1st argument ('char' != 'unsigned int')}}
+  // expected-error@-3 {{he pointee of the 2nd argument must match the element type of the 1st argument ('char' != 'float')}}
 }
 
 void test_column_major_store_template(unsigned *Ptr1, float *Ptr2) {
@@ -152,9 +152,9 @@ void test_column_major_store_constexpr(unsigned *Ptr, MyMatrix<unsigned, 3, 3> &
   __builtin_matrix_column_major_store(M.value, Ptr, constexpr1());
   // expected-error@-1 {{stride must be greater or equal to the number of rows}}
   __builtin_matrix_column_major_store(constexpr1(), Ptr, 1);
-  // expected-error@-1 {{first argument must be a matrix}}
+  // expected-error@-1 {{1st argument must be a matrix}}
   __builtin_matrix_column_major_store(M.value, constexpr1(), 1);
-  // expected-error@-1 {{second argument must be a pointer to a valid matrix element type}}
+  // expected-error@-1 {{2nd argument must be a pointer to a valid matrix element type}}
   // expected-error@-2 {{stride must be greater or equal to the number of rows}}
 }
 
@@ -162,5 +162,5 @@ void test_column_major_store_wrapper(unsigned *Ptr, MyMatrix<unsigned, 3, 3> &M,
   __builtin_matrix_column_major_store(M.value, Ptr, W);
 
   __builtin_matrix_column_major_store(W, Ptr, W);
-  // expected-error@-1 {{first argument must be a matrix}}
+  // expected-error@-1 {{1st argument must be a matrix}}
 }
diff --git a/clang/test/SemaCXX/recovery-expr-type.cpp b/clang/test/SemaCXX/recovery-expr-type.cpp
index df40e5cd60218..6cd79326e8c34 100644
--- a/clang/test/SemaCXX/recovery-expr-type.cpp
+++ b/clang/test/SemaCXX/recovery-expr-type.cpp
@@ -62,3 +62,9 @@ constexpr auto x2 = AA<int>::foo2(); // expected-error {{be initialized by a con
                                      // expected-note {{in instantiation of member function}} \
                                      // expected-note {{in call to}}
 }
+
+// verify no assertion failure on violating value category.
+namespace test4 {
+int &&f(int);  // expected-note {{candidate function not viable}}
+int &&k = f(); // expected-error {{no matching function for call}}
+}
diff --git a/clang/test/SemaCXX/references.cpp b/clang/test/SemaCXX/references.cpp
index f30e16d990ebe..eaab1ae833e4e 100644
--- a/clang/test/SemaCXX/references.cpp
+++ b/clang/test/SemaCXX/references.cpp
@@ -201,3 +201,9 @@ namespace RefCollapseTypePrinting {
   template void add_rref<const int&>(); // expected-note {{instantiation of}}
   template void add_rref<const int&&>(); // expected-note {{instantiation of}}
 }
+
+namespace PR45521 {
+  struct a { template<class b> a(const b * const&); };
+  int *d;
+  const a &r = d;
+}
diff --git a/clang/test/SemaCXX/warn-cast-align.cpp b/clang/test/SemaCXX/warn-cast-align.cpp
index 53cd75fb14052..1e84ba9cd67af 100644
--- a/clang/test/SemaCXX/warn-cast-align.cpp
+++ b/clang/test/SemaCXX/warn-cast-align.cpp
@@ -44,9 +44,16 @@ void test1(void *P) {
   c = IntPtr(P);
 }
 
+struct __attribute__((aligned(16))) AlignedS {
+  char m[16];
+};
+
 struct __attribute__((aligned(16))) A {
   char m0[16];
   char m1[16];
+  AlignedS *getAlignedS() {
+    return (AlignedS *)m1;
+  }
 };
 
 struct B0 {
@@ -92,6 +99,9 @@ struct __attribute__((aligned(16))) D4 : virtual D2 {
 
 struct D5 : virtual D0 {
   char m0[16];
+  AlignedS *get() {
+    return (AlignedS *)m0; // expected-warning {{cast from 'char *' to 'AlignedS *'}}
+  }
 };
 
 struct D6 : virtual D5 {
diff --git a/clang/test/SemaCXX/warn-inline-namespace-reopened-noninline-disable.cpp b/clang/test/SemaCXX/warn-inline-namespace-reopened-noninline-disable.cpp
new file mode 100644
index 0000000000000..3cc683006c412
--- /dev/null
+++ b/clang/test/SemaCXX/warn-inline-namespace-reopened-noninline-disable.cpp
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -fsyntax-only -Wall -verify -std=c++11 %s
+// RUN: %clang_cc1 -fsyntax-only -Wall -Wno-inline-namespace-reopened-noninline -DSILENCE -verify -std=c++11 %s
+
+namespace X {
+  #ifndef SILENCE
+    inline namespace {} // expected-note {{previous definition}}
+    namespace {} // expected-warning {{inline namespace reopened as a non-inline namespace}}
+  #else
+    // expected-no-diagnostics
+    inline namespace {}
+    namespace {}
+  #endif
+}
diff --git a/clang/test/SemaCXX/warn-suggest-destructor-override b/clang/test/SemaCXX/warn-suggest-destructor-override
new file mode 100644
index 0000000000000..1cfff748678f2
--- /dev/null
+++ b/clang/test/SemaCXX/warn-suggest-destructor-override
@@ -0,0 +1,27 @@
+// RUN: %clang_cc1 -fsyntax-only -std=c++11 %s -verify -Wsuggest-destructor-override
+
+struct A {
+  ~A();
+  virtual void run();
+};
+
+struct B : public A {
+  ~B();
+};
+
+struct C {
+  virtual void run();
+  virtual ~C();  // expected-note 2{{overridden virtual function is here}}
+};
+
+struct D : public C {
+  void run();
+  ~D();
+  // expected-warning@-1 {{'~D' overrides a destructor but is not marked 'override'}}
+};
+
+struct E : public C {
+  void run();
+  virtual ~E();
+  // expected-warning@-1 {{'~E' overrides a destructor but is not marked 'override'}}
+};
diff --git a/clang/test/SemaCXX/warn-suggest-override b/clang/test/SemaCXX/warn-suggest-override
new file mode 100644
index 0000000000000..e06c939ff001f
--- /dev/null
+++ b/clang/test/SemaCXX/warn-suggest-override
@@ -0,0 +1,38 @@
+// RUN: %clang_cc1 -fsyntax-only -std=c++11 %s -verify -Wsuggest-override
+
+struct A {
+  ~A();
+  void run();
+};
+
+struct B : public A {
+  ~B();
+  void run();
+};
+
+struct C {
+  virtual void run(); // expected-note 2{{overridden virtual function is here}}
+  virtual ~C();
+};
+
+struct D : public C {
+  void run();
+  // expected-warning@-1 {{'run()' overrides a member function but is not marked 'override'}}
+  ~D();
+};
+
+struct E : public C {
+  virtual void run();
+  // expected-warning@-1 {{'run()' overrides a member function but is not marked 'override'}}
+  virtual ~E();
+};
+
+struct F : public C {
+  void run() override;
+  ~F() override;
+};
+
+struct G : public C {
+  void run() final;
+  ~G() final;
+};
diff --git a/clang/test/SemaCXX/warn-uninitialized-const-reference.cpp b/clang/test/SemaCXX/warn-uninitialized-const-reference.cpp
index 292793ba483a6..d24b561441d8f 100644
--- a/clang/test/SemaCXX/warn-uninitialized-const-reference.cpp
+++ b/clang/test/SemaCXX/warn-uninitialized-const-reference.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -Wuninitialized-const-reference -verify %s
+// RUN: %clang_cc1 -fsyntax-only -fcxx-exceptions -fexceptions -Wuninitialized-const-reference -verify %s
 
 class A {
 public:
@@ -9,6 +9,16 @@ class A {
   bool operator!=(const A &);
 };
 
+template <class T>
+void ignore_template(const T &) {}
+void ignore(const int &i) {}
+void dont_ignore_non_empty(const int &i) { ; } // Calling this won't silence the warning for you
+void dont_ignore_block(const int &i) {
+  {}
+} // Calling this won't silence the warning for you
+void ignore_function_try_block_maybe_who_knows(const int &) try {
+} catch (...) {
+}
 A const_ref_use_A(const A &a);
 int const_ref_use(const int &i);
 A const_use_A(const A a);
@@ -33,4 +43,13 @@ void f(int a) {
   if (a < 42)
     m = 1;
   const_ref_use(m);
+
+  int l;
+  ignore_template(l); // This is a pattern to avoid "unused variable" warnings (e.g. boost::ignore_unused).
+  ignore(l);
+  dont_ignore_non_empty(l); // expected-warning {{variable 'l' is uninitialized when passed as a const reference argument here}}
+  int l1;
+  dont_ignore_block(l1); // expected-warning {{variable 'l1' is uninitialized when passed as a const reference argument here}}
+  int l2;
+  ignore_function_try_block_maybe_who_knows(l2); // expected-warning {{variable 'l2' is uninitialized when passed as a const reference argument here}}
 }
diff --git a/clang/test/SemaObjC/aarch64-sve-types.m b/clang/test/SemaObjC/aarch64-sve-types.m
index 9e1af311de6fb..376ffe648e2c2 100644
--- a/clang/test/SemaObjC/aarch64-sve-types.m
+++ b/clang/test/SemaObjC/aarch64-sve-types.m
@@ -18,5 +18,7 @@ @interface foo
 @property(nullable) __SVFloat32_t f32; // expected-error {{cannot be applied to non-pointer type}}
 @property(nullable) __SVFloat64_t f64; // expected-error {{cannot be applied to non-pointer type}}
 
+@property(nullable) __SVBFloat16_t bf16; // expected-error {{cannot be applied to non-pointer type}}
+
 @property(nullable) __SVBool_t b8; // expected-error {{cannot be applied to non-pointer type}}
 @end
diff --git a/clang/test/SemaObjC/matrix-type-builtins.m b/clang/test/SemaObjC/matrix-type-builtins.m
index dc18905963e40..21b8bf864271d 100644
--- a/clang/test/SemaObjC/matrix-type-builtins.m
+++ b/clang/test/SemaObjC/matrix-type-builtins.m
@@ -22,10 +22,10 @@ void test_element_type_mismatch(u4x4 m, MatrixValue *mv) {
 
 double test_store(MatrixValue *mv, float *Ptr) {
   __builtin_matrix_column_major_store(mv.value, Ptr, 1);
-  // expected-error@-1 {{the pointee of the second argument must match the element type of the first argument ('float' != 'double')}}
+  // expected-error@-1 {{the pointee of the 2nd argument must match the element type of the 1st argument ('float' != 'double')}}
   // expected-error@-2 {{stride must be greater or equal to the number of rows}}
 
   __builtin_matrix_column_major_store(mv.value, mv.value, mv.value);
-  // expected-error@-1 {{second argument must be a pointer to a valid matrix element type}}
+  // expected-error@-1 {{2nd argument must be a pointer to a valid matrix element type}}
   // expected-error@-2 {{casting 'double4x4' (aka 'double __attribute__((matrix_type(4, 4)))') to incompatible type 'unsigned long}}
 }
diff --git a/clang/test/SemaObjC/potentially-direct-selector.m b/clang/test/SemaObjC/potentially-direct-selector.m
new file mode 100644
index 0000000000000..04f9d2111c9b1
--- /dev/null
+++ b/clang/test/SemaObjC/potentially-direct-selector.m
@@ -0,0 +1,157 @@
+// RUN: %clang_cc1 %s -Wpotentially-direct-selector -verify
+// RUN: %clang_cc1 %s -Wstrict-potentially-direct-selector -verify=expected,strict
+
+#define NS_DIRECT __attribute__((objc_direct))
+
+__attribute__((objc_root_class))
+@interface Dummies
+-(void)inBase;
+-(void)inBaseImpl;
+-(void)inBaseCat;
+-(void)inBaseCatImpl;
+-(void)inDerived;
+-(void)inDerivedImpl;
+-(void)inDerivedCat;
+-(void)inDerivedCatImpl;
++(void)inBaseClass;
++(void)inDerivedClass;
++(void)inDerivedCatClass;
+@end
+
+__attribute__((objc_root_class))
+@interface Base
+-(void)inBase NS_DIRECT; // expected-note + {{direct method}}
++(void)inBaseClass NS_DIRECT;  // expected-note + {{direct method}}
+@end
+
+@implementation Base
+-(void)inBaseImpl NS_DIRECT { // expected-note + {{direct method}}
+}
+-(void)inBase {}
++(void)inBaseClass {}
+@end
+
+@interface Base (Cat)
+-(void)inBaseCat NS_DIRECT; // expected-note + {{direct method}}
+@end
+
+@implementation Base (Cat)
+-(void)inBaseCatImpl NS_DIRECT { // expected-note + {{direct method}}
+}
+-(void)inBaseCat {}
+@end
+
+@interface Derived : Base
+-(void)inDerived NS_DIRECT; // expected-note + {{direct method}}
++(void)inDerivedClass NS_DIRECT;  // expected-note + {{direct method}}
+@end
+
+@implementation Derived
+-(void)inDerivedImpl NS_DIRECT { // expected-note + {{direct method}}
+}
+-(void)inDerived {}
++(void)inDerivedClass {}
+@end
+
+@interface Derived (Cat)
+-(void)inDerivedCat NS_DIRECT; // expected-note + {{direct method}}
++(void)inDerivedCatClass NS_DIRECT; // expected-note + {{direct method}}
+@end
+
+@implementation Derived (Cat)
+-(void)inDerivedCatImpl NS_DIRECT { // expected-note + {{direct method}}
+}
+-(void)inDerivedCat {}
++(void)inDerivedCatClass {}
+
+-(void)test1 {
+  (void)@selector(inBase); // expected-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inBaseImpl); // expected-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inBaseCat); // expected-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inBaseCatImpl); // expected-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerived); // expected-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedImpl); // expected-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedCat); // expected-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedCatImpl); // expected-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedClass); // expected-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inBaseClass); // expected-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedCatClass); // expected-warning{{@selector expression formed with potentially direct selector}}
+}
+@end
+
+void test2() {
+  (void)@selector(inBase); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inBaseImpl); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inBaseCat); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inBaseCatImpl); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerived); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedImpl); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedCat); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedCatImpl); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedClass); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inBaseClass); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedCatClass); // strict-warning{{@selector expression formed with potentially direct selector}}
+}
+
+@interface OnlyBase : Base @end
+@implementation OnlyBase
+-(void)test3 {
+  (void)@selector(inBase); // expected-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inBaseImpl); // expected-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inBaseCat); // expected-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inBaseCatImpl); // expected-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerived); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedImpl); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedCat); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedCatImpl); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedClass); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inBaseClass); // expected-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedCatClass); // strict-warning{{@selector expression formed with potentially direct selector}}
+}
+@end
+
+__attribute__((objc_root_class))
+@interface Unrelated @end
+@implementation Unrelated
+-(void)test4 {
+  (void)@selector(inBase); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inBaseImpl); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inBaseCat); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inBaseCatImpl); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerived); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedImpl); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedCat); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedCatImpl); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedClass); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inBaseClass); // strict-warning{{@selector expression formed with potentially direct selector}}
+  (void)@selector(inDerivedCatClass); // strict-warning{{@selector expression formed with potentially direct selector}}
+}
+@end
+
+@implementation Dummies
+-(void)inBase {}
+-(void)inBaseImpl {}
+-(void)inBaseCat {}
+-(void)inBaseCatImpl {}
+-(void)inDerived {}
+-(void)inDerivedImpl {}
+-(void)inDerivedCat {}
+-(void)inDerivedCatImpl {}
++(void)inBaseClass {}
++(void)inDerivedClass {}
++(void)inDerivedCatClass {}
+
+-(void)test5 {
+  (void)@selector(inBase);
+  (void)@selector(inBaseImpl);
+  (void)@selector(inBaseCat);
+  (void)@selector(inBaseCatImpl);
+  (void)@selector(inDerived);
+  (void)@selector(inDerivedImpl);
+  (void)@selector(inDerivedCat);
+  (void)@selector(inDerivedCatImpl);
+  (void)@selector(inDerivedClass);
+  (void)@selector(inBaseClass);
+  (void)@selector(inDerivedCatClass);
+}
+@end
diff --git a/clang/test/SemaObjC/signed-char-bool-conversion.m b/clang/test/SemaObjC/signed-char-bool-conversion.m
index 183f60fafcd5a..9442d8fa86a90 100644
--- a/clang/test/SemaObjC/signed-char-bool-conversion.m
+++ b/clang/test/SemaObjC/signed-char-bool-conversion.m
@@ -108,3 +108,15 @@ int main() {
   f<short>(); // expected-note {{in instantiation of function template specialization 'f<short>' requested here}}
 }
 #endif
+
+void t5(BOOL b) {
+  int i;
+  b = b ?: YES; // no warning
+  b = b ?: i; // expected-warning {{implicit conversion from integral type 'int' to 'BOOL'}}
+  b = (b = i) // expected-warning {{implicit conversion from integral type 'int' to 'BOOL'}}
+               ?: YES;
+  b = (1 ? YES : i) ?: YES; // expected-warning {{implicit conversion from integral type 'int' to 'BOOL'}}
+  b = b ?: (1 ? i : i); // expected-warning 2 {{implicit conversion from integral type 'int' to 'BOOL'}}
+
+  b = b ? YES : (i ?: 0); // expected-warning {{implicit conversion from integral type 'int' to 'BOOL'}}
+}
diff --git a/clang/test/SemaObjCXX/attr-trivial-abi.mm b/clang/test/SemaObjCXX/attr-trivial-abi.mm
index 537c1390a54a2..87b79c14d07a6 100644
--- a/clang/test/SemaObjCXX/attr-trivial-abi.mm
+++ b/clang/test/SemaObjCXX/attr-trivial-abi.mm
@@ -57,7 +57,7 @@ struct __attribute__((trivial_abi(1))) S8 { // expected-error {{'trivial_abi' at
 };
 
 // Do not warn when 'trivial_abi' is used to annotate a template class.
-template<class T>
+template <class T>
 struct __attribute__((trivial_abi)) S10 {
   T p;
 };
@@ -76,21 +76,27 @@ struct __attribute__((trivial_abi)) S10<id> { // expected-warning {{'trivial_abi
   __weak id b;
 };
 
-template<class T>
+template <class T>
 struct __attribute__((trivial_abi)) S15 : S14<T> {
 };
 
 S15<int> s15;
 
-template<class T>
+template <class T>
 struct __attribute__((trivial_abi)) S16 {
   S14<T> a;
 };
 
 S16<int> s16;
 
-template<class T>
+template <class T>
 struct __attribute__((trivial_abi)) S17 { // expected-warning {{'trivial_abi' cannot be applied to 'S17'}} expected-note {{has a __weak field}}
+  S17();
+  S17(S17 &&);
+  __weak id a;
+};
+
+struct __attribute__((trivial_abi)) S18 { // expected-warning {{'trivial_abi' cannot be applied to 'S18'}} expected-note {{has a __weak field}}
   __weak id a;
 };
 
diff --git a/clang/test/SemaObjCXX/property-invalid-type.mm b/clang/test/SemaObjCXX/property-invalid-type.mm
index 9122426f4c300..35a5f3a5a525f 100644
--- a/clang/test/SemaObjCXX/property-invalid-type.mm
+++ b/clang/test/SemaObjCXX/property-invalid-type.mm
@@ -21,3 +21,13 @@ void foo(I *i)
 {
   i.helper; // expected-warning{{property access result unused - getters should not be used for side effects}}
 }
+
+@interface J
+@property (nonnull) auto a; // expected-error {{'auto' not allowed in interface member}}
+@property auto b; // expected-error {{'auto' not allowed in interface member}}
+@property (nullable) auto c; // expected-error {{'auto' not allowed in interface member}}
+@end
+
+@interface J (Cat)
+@property (nonnull) auto catprop; // expected-error {{'auto' not allowed in interface member}}
+@end
diff --git a/clang/test/SemaOpenCL/block-array-capturing.cl b/clang/test/SemaOpenCL/block-array-capturing.cl
index e8073fce11e83..6ad4d1e493af0 100644
--- a/clang/test/SemaOpenCL/block-array-capturing.cl
+++ b/clang/test/SemaOpenCL/block-array-capturing.cl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -O0 -cl-std=CL2.0 -triple spir64-unkown-unkown -emit-llvm %s -o -| FileCheck %s
+// RUN: %clang_cc1 -O0 -cl-std=CL2.0 -triple spir64-unknown-unknown -emit-llvm %s -o -| FileCheck %s
 // expected-no-diagnostics
 
 typedef int (^block_t)();
diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-error-f16.cl b/clang/test/SemaOpenCL/builtins-amdgcn-error-f16.cl
index 3487b1a5a8038..fdb2f3f3c9816 100644
--- a/clang/test/SemaOpenCL/builtins-amdgcn-error-f16.cl
+++ b/clang/test/SemaOpenCL/builtins-amdgcn-error-f16.cl
@@ -8,6 +8,7 @@ void test_f16_tahiti(global half *out, half a, half b, half c)
 {
   *out = __builtin_amdgcn_div_fixuph(a, b, c); // expected-error {{'__builtin_amdgcn_div_fixuph' needs target feature 16-bit-insts}}
   *out = __builtin_amdgcn_rcph(a); // expected-error {{'__builtin_amdgcn_rcph' needs target feature 16-bit-insts}}
+  *out = __builtin_amdgcn_sqrth(a); // expected-error {{'__builtin_amdgcn_sqrth' needs target feature 16-bit-insts}}
   *out = __builtin_amdgcn_rsqh(a); // expected-error {{'__builtin_amdgcn_rsqh' needs target feature 16-bit-insts}}
   *out = __builtin_amdgcn_sinh(a); // expected-error {{'__builtin_amdgcn_sinh' needs target feature 16-bit-insts}}
   *out = __builtin_amdgcn_cosh(a); // expected-error {{'__builtin_amdgcn_cosh' needs target feature 16-bit-insts}}
diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-error.cl b/clang/test/SemaOpenCL/builtins-amdgcn-error.cl
index e2d9082f8f723..8001cf3b59e00 100644
--- a/clang/test/SemaOpenCL/builtins-amdgcn-error.cl
+++ b/clang/test/SemaOpenCL/builtins-amdgcn-error.cl
@@ -146,7 +146,7 @@ void test_s_setreg(int x, int y) {
 }
 
 void test_atomic_inc32() {
-  int val = 17;
+  uint val = 17;
   val = __builtin_amdgcn_atomic_inc32(&val, val, __ATOMIC_SEQ_CST + 1, "workgroup"); // expected-warning {{memory order argument to atomic operation is invalid}}
   val = __builtin_amdgcn_atomic_inc32(&val, val, __ATOMIC_ACQUIRE - 1, "workgroup"); // expected-warning {{memory order argument to atomic operation is invalid}}
   val = __builtin_amdgcn_atomic_inc32(4);                                            // expected-error {{too few arguments to function call, expected 4}}
@@ -155,10 +155,12 @@ void test_atomic_inc32() {
   val = __builtin_amdgcn_atomic_inc32(&val, val, __ATOMIC_ACQUIRE, 5);               // expected-warning {{incompatible integer to pointer conversion passing 'int' to parameter of type 'const char *'}}
   const char ptr[] = "workgroup";
   val = __builtin_amdgcn_atomic_inc32(&val, val, __ATOMIC_ACQUIRE, ptr); // expected-error {{expression is not a string literal}}
+  int signedVal = 15;
+  signedVal = __builtin_amdgcn_atomic_inc32(&signedVal, signedVal, __ATOMIC_ACQUIRE, ""); // expected-warning {{passing '__private int *' to parameter of type 'volatile __private unsigned int *' converts between pointers to integer types with different sign}}
 }
 
 void test_atomic_inc64() {
-  __INT64_TYPE__ val = 17;
+  __UINT64_TYPE__ val = 17;
   val = __builtin_amdgcn_atomic_inc64(&val, val, __ATOMIC_SEQ_CST + 1, "workgroup"); // expected-warning {{memory order argument to atomic operation is invalid}}
   val = __builtin_amdgcn_atomic_inc64(&val, val, __ATOMIC_ACQUIRE - 1, "workgroup"); // expected-warning {{memory order argument to atomic operation is invalid}}
   val = __builtin_amdgcn_atomic_inc64(4);                                            // expected-error {{too few arguments to function call, expected 4}}
@@ -167,10 +169,12 @@ void test_atomic_inc64() {
   val = __builtin_amdgcn_atomic_inc64(&val, val, __ATOMIC_ACQUIRE, 5);               // expected-warning {{incompatible integer to pointer conversion passing 'int' to parameter of type 'const char *'}}
   const char ptr[] = "workgroup";
   val = __builtin_amdgcn_atomic_inc64(&val, val, __ATOMIC_ACQUIRE, ptr); // expected-error {{expression is not a string literal}}
+  __INT64_TYPE__ signedVal = 15;
+  signedVal = __builtin_amdgcn_atomic_inc64(&signedVal, signedVal, __ATOMIC_ACQUIRE, ""); // expected-warning {{passing '__private long *' to parameter of type 'volatile __private unsigned long *' converts between pointers to integer types with different sign}}
 }
 
 void test_atomic_dec32() {
-  int val = 17;
+  uint val = 17;
   val = __builtin_amdgcn_atomic_dec32(&val, val, __ATOMIC_SEQ_CST + 1, "workgroup"); // expected-warning {{memory order argument to atomic operation is invalid}}
   val = __builtin_amdgcn_atomic_dec32(&val, val, __ATOMIC_ACQUIRE - 1, "workgroup"); // expected-warning {{memory order argument to atomic operation is invalid}}
   val = __builtin_amdgcn_atomic_dec32(4);                                            // expected-error {{too few arguments to function call, expected 4}}
@@ -179,10 +183,12 @@ void test_atomic_dec32() {
   val = __builtin_amdgcn_atomic_dec32(&val, val, __ATOMIC_ACQUIRE, 5);               // expected-warning {{incompatible integer to pointer conversion passing 'int' to parameter of type 'const char *'}}
   const char ptr[] = "workgroup";
   val = __builtin_amdgcn_atomic_dec32(&val, val, __ATOMIC_ACQUIRE, ptr); // expected-error {{expression is not a string literal}}
+  int signedVal = 15;
+  signedVal = __builtin_amdgcn_atomic_dec32(&signedVal, signedVal, __ATOMIC_ACQUIRE, ""); // expected-warning {{passing '__private int *' to parameter of type 'volatile __private unsigned int *' converts between pointers to integer types with different sign}}
 }
 
 void test_atomic_dec64() {
-  __INT64_TYPE__ val = 17;
+  __UINT64_TYPE__ val = 17;
   val = __builtin_amdgcn_atomic_dec64(&val, val, __ATOMIC_SEQ_CST + 1, "workgroup"); // expected-warning {{memory order argument to atomic operation is invalid}}
   val = __builtin_amdgcn_atomic_dec64(&val, val, __ATOMIC_ACQUIRE - 1, "workgroup"); // expected-warning {{memory order argument to atomic operation is invalid}}
   val = __builtin_amdgcn_atomic_dec64(4);                                            // expected-error {{too few arguments to function call, expected 4}}
@@ -191,4 +197,6 @@ void test_atomic_dec64() {
   val = __builtin_amdgcn_atomic_dec64(&val, val, __ATOMIC_ACQUIRE, 5);               // expected-warning {{incompatible integer to pointer conversion passing 'int' to parameter of type 'const char *'}}
   const char ptr[] = "workgroup";
   val = __builtin_amdgcn_atomic_dec64(&val, val, __ATOMIC_ACQUIRE, ptr); // expected-error {{expression is not a string literal}}
+  __INT64_TYPE__ signedVal = 15;
+  signedVal = __builtin_amdgcn_atomic_dec64(&signedVal, signedVal, __ATOMIC_ACQUIRE, ""); // expected-warning {{passing '__private long *' to parameter of type 'volatile __private unsigned long *' converts between pointers to integer types with different sign}}
 }
diff --git a/clang/test/SemaOpenCL/fp-options.cl b/clang/test/SemaOpenCL/fp-options.cl
new file mode 100644
index 0000000000000..413afd61819d3
--- /dev/null
+++ b/clang/test/SemaOpenCL/fp-options.cl
@@ -0,0 +1,4 @@
+// RUN: %clang_cc1 %s -finclude-default-header -triple spir-unknown-unknown -emit-pch -o %t.pch
+// RUN: %clang_cc1 %s -finclude-default-header -cl-no-signed-zeros -triple spir-unknown-unknown -include-pch %t.pch -fsyntax-only -verify
+// expected-no-diagnostics
+
diff --git a/clang/test/SemaOpenCL/invalid-block.cl b/clang/test/SemaOpenCL/invalid-block.cl
index ec74d16cc9b8c..0fb2e26920736 100644
--- a/clang/test/SemaOpenCL/invalid-block.cl
+++ b/clang/test/SemaOpenCL/invalid-block.cl
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -verify -fblocks -cl-std=CL2.0 %s
 
 // OpenCL v2.0 s6.12.5
-void f0(int (^const bl)());
+void f0(int (^const bl)()); // expected-error{{declaring function parameter of type 'int (__generic ^const __private)(void)' is not allowed}}
 // All blocks declarations must be const qualified and initialized.
 void f1() {
   int (^bl1)(void) = ^() {
@@ -26,9 +26,18 @@ void f2() {
   };
 }
 
-// A block cannot be the return value of a function.
+// A block cannot be the return value or parameter of a function.
 typedef int (^bl_t)(void);
-bl_t f3(bl_t bl); // expected-error{{declaring function return value of type 'bl_t' (aka 'int (__generic ^const)(void)') is not allowed}}
+bl_t f3a(int);     // expected-error{{declaring function return value of type 'bl_t' (aka 'int (__generic ^const)(void)') is not allowed}}
+bl_t f3b(bl_t bl);
+// expected-error@-1{{declaring function return value of type 'bl_t' (aka 'int (__generic ^const)(void)') is not allowed}}
+// expected-error@-2{{declaring function parameter of type '__private bl_t' (aka 'int (__generic ^const __private)(void)') is not allowed}}
+void f3c() {
+  // Block with a block argument.
+  int (^const bl2)(bl_t block_arg) = ^() { // expected-error{{declaring function parameter of type '__private bl_t' (aka 'int (__generic ^const __private)(void)') is not allowed}}
+    return block_arg(); // expected-error{{implicit declaration of function 'block_arg' is invalid in OpenCL}}
+  };
+}
 
 struct bl_s {
   int (^bl)(void); // expected-error {{the 'int (__generic ^const)(void)' type cannot be used to declare a structure or union field}}
diff --git a/clang/test/SemaTemplate/alias-templates.cpp b/clang/test/SemaTemplate/alias-templates.cpp
index 80678bf229852..6dffd94892943 100644
--- a/clang/test/SemaTemplate/alias-templates.cpp
+++ b/clang/test/SemaTemplate/alias-templates.cpp
@@ -265,3 +265,28 @@ namespace an_alias_template_is_not_a_class_template {
     int z = Bar(); // expected-error {{use of template template parameter 'Bar' requires template arguments}}
   }
 }
+
+namespace resolved_nttp {
+  template <typename T> struct A {
+    template <int N> using Arr = T[N];
+    Arr<3> a;
+  };
+  using TA = decltype(A<int>::a);
+  using TA = int[3];
+
+  template <typename T> struct B {
+    template <int... N> using Fn = T(int(*...A)[N]);
+    Fn<1, 2, 3> *p;
+  };
+  using TB = decltype(B<int>::p);
+  using TB = int (*)(int (*)[1], int (*)[2], int (*)[3]);
+
+  template <typename T, int ...M> struct C {
+    template <T... N> using Fn = T(int(*...A)[N]);
+    Fn<1, M..., 4> *p; // expected-error-re 3{{evaluates to {{[234]}}, which cannot be narrowed to type 'bool'}}
+  };
+  using TC = decltype(C<int, 2, 3>::p);
+  using TC = int (*)(int (*)[1], int (*)[2], int (*)[3], int (*)[4]);
+
+  using TC2 = decltype(C<bool, 2, 3>::p); // expected-note {{instantiation of}}
+}
diff --git a/clang/test/SemaTemplate/deduction-guide.cpp b/clang/test/SemaTemplate/deduction-guide.cpp
new file mode 100644
index 0000000000000..8a964903ebfcc
--- /dev/null
+++ b/clang/test/SemaTemplate/deduction-guide.cpp
@@ -0,0 +1,208 @@
+// RUN: %clang_cc1 -std=c++2a -verify -ast-dump -ast-dump-decl-types -ast-dump-filter "deduction guide" %s | FileCheck %s --strict-whitespace
+
+template<auto ...> struct X {};
+template<template<typename X, X> typename> struct Y {};
+template<typename ...> struct Z {};
+
+template<typename T, typename ...Ts> struct A {
+  template<Ts ...Ns, T *...Ps> A(X<Ps...>, Ts (*...qs)[Ns]);
+};
+int arr1[3], arr2[3];
+short arr3[4];
+A a(X<&arr1, &arr2>{}, &arr1, &arr2, &arr3);
+using AT = decltype(a);
+using AT = A<int[3], int, int, short>;
+
+// CHECK-LABEL: Dumping <deduction guide for A>:
+// CHECK: FunctionTemplateDecl
+// CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 0 T
+// CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 1 ... Ts
+// CHECK: |-NonTypeTemplateParmDecl {{.*}} 'Ts...' depth 0 index 2 ... Ns
+// CHECK: |-NonTypeTemplateParmDecl {{.*}} 'T *' depth 0 index 3 ... Ps
+// CHECK: |-CXXDeductionGuideDecl
+// CHECK: | |-ParmVarDecl {{.*}} 'X<Ps...>'
+// CHECK: | `-ParmVarDecl {{.*}} 'Ts (*)[Ns]...' pack
+// CHECK: `-CXXDeductionGuideDecl
+// CHECK:   |-TemplateArgument type 'int [3]'
+// CHECK:   |-TemplateArgument pack
+// CHECK:   | |-TemplateArgument type 'int'
+// CHECK:   | |-TemplateArgument type 'int'
+// CHECK:   | `-TemplateArgument type 'short'
+// CHECK:   |-TemplateArgument pack
+// CHECK:   | |-TemplateArgument integral 3
+// CHECK:   | |-TemplateArgument integral 3
+// CHECK:   | `-TemplateArgument integral 4
+// CHECK:   |-TemplateArgument pack
+// CHECK:   | |-TemplateArgument decl
+// CHECK:   | | `-Var {{.*}} 'arr1' 'int [3]'
+// CHECK:   | `-TemplateArgument decl
+// CHECK:   |   `-Var {{.*}} 'arr2' 'int [3]'
+// CHECK:   |-ParmVarDecl {{.*}} 'X<&arr1, &arr2>':'X<&arr1, &arr2>'
+// CHECK:   |-ParmVarDecl {{.*}} 'int (*)[3]'
+// CHECK:   |-ParmVarDecl {{.*}} 'int (*)[3]'
+// CHECK:   `-ParmVarDecl {{.*}} 'short (*)[4]'
+// CHECK: FunctionProtoType {{.*}} 'auto (X<Ps...>, Ts (*)[Ns]...) -> A<T, Ts...>' dependent trailing_return
+// CHECK: |-InjectedClassNameType {{.*}} 'A<T, Ts...>' dependent
+// CHECK: |-TemplateSpecializationType {{.*}} 'X<Ps...>' dependent X
+// CHECK: | `-TemplateArgument expr
+// CHECK: |   `-PackExpansionExpr {{.*}} 'T *'
+// CHECK: |     `-DeclRefExpr {{.*}} 'T *' NonTypeTemplateParm {{.*}} 'Ps' 'T *'
+// CHECK: `-PackExpansionType {{.*}} 'Ts (*)[Ns]...' dependent
+// CHECK:   `-PointerType {{.*}} 'Ts (*)[Ns]' dependent contains_unexpanded_pack
+// CHECK:     `-ParenType {{.*}} 'Ts [Ns]' sugar dependent contains_unexpanded_pack
+// CHECK:       `-DependentSizedArrayType {{.*}} 'Ts [Ns]' dependent contains_unexpanded_pack
+// CHECK:         |-TemplateTypeParmType {{.*}} 'Ts' dependent contains_unexpanded_pack depth 0 index 1 pack
+// CHECK:         | `-TemplateTypeParm {{.*}} 'Ts'
+// CHECK:         `-DeclRefExpr {{.*}} 'Ts' NonTypeTemplateParm {{.*}} 'Ns' 'Ts...'
+
+template<typename T, T V> struct B {
+  template<typename U, U W> B(X<W, V>);
+};
+B b(X<nullptr, 'x'>{});
+using BT = decltype(b);
+using BT = B<char, 'x'>;
+
+// CHECK-LABEL: Dumping <deduction guide for B>:
+// CHECK: FunctionTemplateDecl
+// CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 0 T
+// CHECK: |-NonTypeTemplateParmDecl {{.*}} 'T' depth 0 index 1 V
+// CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 2 U
+// CHECK: |-NonTypeTemplateParmDecl {{.*}} 'type-parameter-0-2' depth 0 index 3 W
+// CHECK: |-CXXDeductionGuideDecl {{.*}} 'auto (X<W, V>) -> B<T, V>'
+// CHECK: | `-ParmVarDecl {{.*}} 'X<W, V>'
+// CHECK: `-CXXDeductionGuideDecl {{.*}} 'auto (X<nullptr, 'x'>) -> B<char, 'x'>'
+// CHECK:   |-TemplateArgument type 'char'
+// CHECK:   |-TemplateArgument integral 120
+// CHECK:   |-TemplateArgument type 'nullptr_t'
+// CHECK:   |-TemplateArgument nullptr
+// CHECK:   `-ParmVarDecl {{.*}} 'X<nullptr, 'x'>':'X<nullptr, 'x'>'
+// CHECK: FunctionProtoType {{.*}} 'auto (X<W, V>) -> B<T, V>' dependent trailing_return
+// CHECK: |-InjectedClassNameType {{.*}} 'B<T, V>' dependent
+// CHECK: `-TemplateSpecializationType {{.*}} 'X<W, V>' dependent X
+// CHECK:   |-TemplateArgument expr
+// CHECK:   | `-DeclRefExpr {{.*}} 'type-parameter-0-2' NonTypeTemplateParm {{.*}} 'W' 'type-parameter-0-2'
+// CHECK:   `-TemplateArgument expr
+// CHECK:     `-DeclRefExpr {{.*}} 'T' NonTypeTemplateParm {{.*}} 'V' 'T'
+
+template<typename A> struct C {
+  template<template<typename X, X> typename T, typename U, U V = 0> C(A, Y<T>, U);
+};
+C c(1, Y<B>{}, 2);
+using CT = decltype(c);
+using CT = C<int>;
+
+// CHECK-LABEL: Dumping <deduction guide for C>:
+// CHECK: FunctionTemplateDecl
+// CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 0 A
+// CHECK: |-TemplateTemplateParmDecl {{.*}} depth 0 index 1 T
+// CHECK: | |-TemplateTypeParmDecl {{.*}} typename depth 1 index 0 X
+// CHECK: | `-NonTypeTemplateParmDecl {{.*}} 'X' depth 1 index 1
+// CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 2 U
+// CHECK: |-NonTypeTemplateParmDecl {{.*}} 'type-parameter-0-2' depth 0 index 3 V
+// CHECK: | `-TemplateArgument expr
+// CHECK: |   `-IntegerLiteral {{.*}} 'int' 0
+// CHECK: |-CXXDeductionGuideDecl {{.*}} 'auto (A, Y<>, type-parameter-0-2) -> C<A>'
+// CHECK: | |-ParmVarDecl {{.*}} 'A'
+// CHECK: | |-ParmVarDecl {{.*}} 'Y<>'
+// CHECK: | `-ParmVarDecl {{.*}} 'type-parameter-0-2'
+// CHECK: `-CXXDeductionGuideDecl {{.*}} 'auto (int, Y<B>, int) -> C<int>'
+// CHECK:  |-TemplateArgument type 'int'
+// CHECK:  |-TemplateArgument template B
+// CHECK:  |-TemplateArgument type 'int'
+// CHECK:  |-TemplateArgument integral 0
+// CHECK:  |-ParmVarDecl {{.*}} 'int':'int'
+// CHECK:  |-ParmVarDecl {{.*}} 'Y<B>':'Y<B>'
+// CHECK:  `-ParmVarDecl {{.*}} 'int':'int'
+// CHECK: FunctionProtoType {{.*}} 'auto (A, Y<>, type-parameter-0-2) -> C<A>' dependent trailing_return cdecl
+// CHECK: |-InjectedClassNameType {{.*}} 'C<A>' dependent
+// CHECK: |-TemplateTypeParmType {{.*}} 'A' dependent depth 0 index 0
+// CHECK: | `-TemplateTypeParm {{.*}} 'A'
+// CHECK: |-TemplateSpecializationType {{.*}} 'Y<>' dependent Y
+// CHECK: | `-TemplateArgument template 
+// CHECK: `-TemplateTypeParmType {{.*}} 'type-parameter-0-2' dependent depth 0 index 2
+
+template<typename ...T> struct D { // expected-note {{candidate}}
+  template<typename... U> using B = int(int (*...p)(T, U));
+  template<typename U1, typename U2> D(B<U1, U2>*); // expected-note {{candidate}}
+};
+int f(int(int, int), int(int, int));
+// FIXME: We can't deduce this because we can't deduce through a
+// SubstTemplateTypeParmPackType.
+D d = f; // expected-error {{no viable}}
+using DT = decltype(d);
+using DT = D<int, int>;
+
+// CHECK-LABEL: Dumping <deduction guide for D>:
+// CHECK: FunctionTemplateDecl
+// CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 0 ... T
+// CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 1 U1
+// CHECK: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 2 U2
+// CHECK: `-CXXDeductionGuideDecl {{.*}} 'auto (B<type-parameter-0-1, type-parameter-0-2> *) -> D<T...>'  
+// CHECK:   `-ParmVarDecl {{.*}} 'B<type-parameter-0-1, type-parameter-0-2> *'
+// CHECK: FunctionProtoType {{.*}} 'auto (B<type-parameter-0-1, type-parameter-0-2> *) -> D<T...>' dependent trailing_return
+// CHECK: |-InjectedClassNameType {{.*}} 'D<T...>' dependent
+// CHECK: `-PointerType {{.*}} 'B<type-parameter-0-1, type-parameter-0-2> *' dependent
+// CHECK:   `-TemplateSpecializationType {{.*}} 'B<type-parameter-0-1, type-parameter-0-2>' sugar dependent alias B
+// CHECK:     |-TemplateArgument type 'type-parameter-0-1'
+// CHECK:     |-TemplateArgument type 'type-parameter-0-2'
+// CHECK:     `-FunctionProtoType {{.*}} 'int (int (*)(T, U)...)' dependent cdecl
+// CHECK:       |-BuiltinType {{.*}} 'int'
+// CHECK:       `-PackExpansionType {{.*}} 'int (*)(T, U)...' dependent expansions 2
+// CHECK:         `-PointerType {{.*}} 'int (*)(T, U)' dependent contains_unexpanded_pack
+// CHECK:           `-ParenType {{.*}} 'int (T, U)' sugar dependent contains_unexpanded_pack
+// CHECK:             `-FunctionProtoType {{.*}} 'int (T, U)' dependent contains_unexpanded_pack cdecl
+// CHECK:               |-BuiltinType {{.*}} 'int'
+// CHECK:               |-TemplateTypeParmType {{.*}} 'T' dependent contains_unexpanded_pack depth 0 index 0 pack
+// CHECK:               | `-TemplateTypeParm {{.*}} 'T'
+// CHECK:               `-SubstTemplateTypeParmPackType {{.*}} 'U' dependent contains_unexpanded_pack
+// CHECK:                 |-TemplateTypeParmType {{.*}} 'U' dependent contains_unexpanded_pack depth 1 index 0 pack
+// CHECK:                 | `-TemplateTypeParm {{.*}} 'U'
+// CHECK:                 `-TemplateArgument pack
+// CHECK:                   |-TemplateArgument type 'type-parameter-0-1'
+// CHECK-NOT: Subst
+// CHECK:                   | `-TemplateTypeParmType
+// CHECK:                   `-TemplateArgument type 'type-parameter-0-2'
+// CHECK-NOT: Subst
+// CHECK:                     `-TemplateTypeParmType
+
+template<int ...N> struct E { // expected-note {{candidate}}
+  template<int ...M> using B = Z<X<N, M>...>;
+  template<int M1, int M2> E(B<M1, M2>); // expected-note {{candidate}}
+};
+// FIXME: We can't deduce this because we can't deduce through a
+// SubstNonTypeTemplateParmPackExpr.
+E e = Z<X<1, 2>, X<3, 4>>(); // expected-error {{no viable}}
+using ET = decltype(e);
+using ET = E<1, 3>;
+
+// CHECK-LABEL: Dumping <deduction guide for E>:
+// CHECK: FunctionTemplateDecl
+// CHECK: |-NonTypeTemplateParmDecl [[N:0x[0-9a-f]*]] {{.*}} 'int' depth 0 index 0 ... N
+// CHECK: |-NonTypeTemplateParmDecl [[M1:0x[0-9a-f]*]] {{.*}} 'int' depth 0 index 1 M1
+// CHECK: |-NonTypeTemplateParmDecl [[M2:0x[0-9a-f]*]] {{.*}} 'int' depth 0 index 2 M2
+// CHECK: `-CXXDeductionGuideDecl {{.*}} 'auto (B<M1, M2>) -> E<N...>'
+// CHECK:   `-ParmVarDecl {{.*}} 'B<M1, M2>':'Z<X<N, M>...>'
+// CHECK: FunctionProtoType {{.*}} 'auto (B<M1, M2>) -> E<N...>' dependent trailing_return
+// CHECK: |-InjectedClassNameType {{.*}} 'E<N...>' dependent
+// CHECK: `-TemplateSpecializationType {{.*}} 'B<M1, M2>' sugar dependent alias B
+// CHECK:   |-TemplateArgument expr
+// CHECK:   | `-DeclRefExpr {{.*}} 'int' NonTypeTemplateParm {{.*}} 'M1' 'int'
+// CHECK:   |-TemplateArgument expr
+// CHECK:   | `-DeclRefExpr {{.*}} 'int' NonTypeTemplateParm {{.*}} 'M2' 'int'
+// CHECK:   `-TemplateSpecializationType {{.*}} 'Z<X<N, M>...>' dependent Z
+// CHECK:     `-TemplateArgument type 'X<N, M>...'
+// CHECK:       `-PackExpansionType {{.*}} 'X<N, M>...' dependent expansions 2
+// CHECK:         `-TemplateSpecializationType {{.*}} 'X<N, M>' dependent contains_unexpanded_pack X
+// CHECK:           |-TemplateArgument expr
+// CHECK-NOT: Subst
+// CHECK:           | `-DeclRefExpr {{.*}} 'int' NonTypeTemplateParm [[N]] 'N' 'int'
+// CHECK:           `-TemplateArgument expr
+// CHECK:             `-SubstNonTypeTemplateParmPackExpr {{.*}} 'int'
+// CHECK:               |-NonTypeTemplateParmDecl {{.*}} referenced 'int' depth 1 index 0 ... M
+// CHECK:               `-TemplateArgument pack
+// CHECK:                 |-TemplateArgument expr
+// CHECK-NOT: Subst
+// CHECK:                 | `-DeclRefExpr {{.*}} 'int' NonTypeTemplateParm [[M1]] 'M1' 'int'
+// CHECK:                 `-TemplateArgument expr
+// CHECK-NOT: Subst
+// CHECK:                   `-DeclRefExpr {{.*}} 'int' NonTypeTemplateParm [[M2]] 'M2' 'int'
diff --git a/clang/test/SemaTemplate/default-arguments-cxx0x.cpp b/clang/test/SemaTemplate/default-arguments-cxx0x.cpp
index 2114cc94e6c60..1aa456553599c 100644
--- a/clang/test/SemaTemplate/default-arguments-cxx0x.cpp
+++ b/clang/test/SemaTemplate/default-arguments-cxx0x.cpp
@@ -1,5 +1,6 @@
 // RUN: %clang_cc1 -fsyntax-only -std=c++11 -verify %s
 // RUN: %clang_cc1 -fsyntax-only -std=c++14 -verify %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++20 -verify %s
 // expected-no-diagnostics
 
 // Test default template arguments for function templates.
@@ -116,6 +117,11 @@ namespace rdar34167492 {
   };
 }
 
+namespace use_of_earlier_param {
+  template<typename T> void f(T a, int = decltype(a)());
+  void g() { f(0); }
+}
+
 #if __cplusplus >= 201402L
 namespace lambda {
   // Verify that a default argument in a lambda can refer to the type of a
@@ -127,5 +133,32 @@ namespace lambda {
   void foo() {
     bar<int>();
   }
+
+#if __cplusplus >= 202002L
+  // PR46648: ensure we don't reject this by triggering default argument
+  // instantiation spuriously.
+  auto x = []<typename T>(T x = 123) {};
+  void y() { x(nullptr); }
+
+  template<int A> struct X {
+    template<int B> constexpr int f() {
+      auto l = []<int C>(int n = A + B + C) { return n; };
+      return l.template operator()<3>();
+    }
+  };
+  static_assert(X<100>().f<20>() == 123);
+
+  template<> template<int B> constexpr int X<200>::f() {
+    auto l = []<int C>(int n = 300 + B + C) { return n; };
+    return l.template operator()<1>();
+  }
+  static_assert(X<200>().f<20>() == 321);
+
+  template<> template<> constexpr int X<300>::f<20>() {
+    auto l = []<int C>(int n = 450 + C) { return n; };
+    return l.template operator()<6>();
+  }
+  static_assert(X<300>().f<20>() == 456);
+#endif
 } // namespace lambda
 #endif
diff --git a/clang/test/SemaTemplate/defaulted.cpp b/clang/test/SemaTemplate/defaulted.cpp
new file mode 100644
index 0000000000000..15111f71202df
--- /dev/null
+++ b/clang/test/SemaTemplate/defaulted.cpp
@@ -0,0 +1,10 @@
+// RUN: %clang_cc1 -std=c++20 -verify %s
+// expected-no-diagnostics
+
+namespace SpaceshipImpliesEq {
+  template<typename T> struct A {
+    int operator<=>(const A&) const = default;
+    constexpr bool f() { return operator==(*this); }
+  };
+  static_assert(A<int>().f());
+}
diff --git a/clang/test/SemaTemplate/dependent-expr.cpp b/clang/test/SemaTemplate/dependent-expr.cpp
index 0ce4cb39d3493..abdb8e9c4a9fd 100644
--- a/clang/test/SemaTemplate/dependent-expr.cpp
+++ b/clang/test/SemaTemplate/dependent-expr.cpp
@@ -144,7 +144,7 @@ namespace PR45083 {
   void h(auto a, void*) {} // expected-error {{redefinition}}
 
   void i(auto a) {
-    [](auto a, int = ({decltype(a) i; i * 2;})){}(a); // expected-error {{no matching function}} expected-note {{substitution failure}}
+    [](auto a, int = ({decltype(a) i; i * 2;})){}(a); // expected-error {{invalid operands to binary expression ('decltype(a)' (aka 'void *') and 'int')}} expected-note {{in instantiation of}}
   }
   void use_i() {
     i(0);
diff --git a/clang/test/SemaTemplate/nss-recovery.cpp b/clang/test/SemaTemplate/nss-recovery.cpp
new file mode 100644
index 0000000000000..08974cbbff113
--- /dev/null
+++ b/clang/test/SemaTemplate/nss-recovery.cpp
@@ -0,0 +1,6 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+
+// There should be no extra errors about missing 'typename' keywords.
+void f() {
+  decltype(undef())::Type T; // expected-error {{use of undeclared identifier}}
+}
diff --git a/clang/test/SemaTemplate/stack-exhaustion.cpp b/clang/test/SemaTemplate/stack-exhaustion.cpp
index 7f76212d2a492..c7bfea4132d5e 100644
--- a/clang/test/SemaTemplate/stack-exhaustion.cpp
+++ b/clang/test/SemaTemplate/stack-exhaustion.cpp
@@ -1,4 +1,6 @@
-// RUN: %clang_cc1 -verify %s
+// RUN: %clang_cc1 -verify %s -DTEST=1
+// RUN: %clang_cc1 -verify %s -DTEST=2
+// RUN: %clang_cc1 -verify %s -DTEST=3
 // REQUIRES: thread_support
 
 // FIXME: Detection of, or recovery from, stack exhaustion does not work on
@@ -6,9 +8,14 @@
 // implementation limits, just disable the test.
 // UNSUPPORTED: system-netbsd
 
+// asan has own stack-overflow check.
+// UNSUPPORTED: asan
+
 // expected-warning@* 0-1{{stack nearly exhausted}}
 // expected-note@* 0+{{}}
 
+#if TEST == 1
+
 template<int N> struct X : X<N-1> {};
 template<> struct X<0> {};
 X<1000> x;
@@ -21,3 +28,38 @@ int f(X<0>);
 template<int N> auto f(X<N>) -> f(X<N-1>());
 
 int k = f(X<1000>());
+
+#elif TEST == 2
+
+namespace template_argument_recursion {
+  struct ostream;
+  template<typename T> T &&declval();
+
+  namespace mlir {
+    template<typename T, typename = decltype(declval<ostream&>() << declval<T&>())>
+    ostream &operator<<(ostream& os, const T& obj); // expected-error {{exceeded maximum depth}}
+    struct Value;
+  }
+
+  void printFunctionalType(ostream &os, mlir::Value &v) { os << v; }
+}
+
+#elif TEST == 3
+
+namespace template_parameter_type_recursion {
+  struct ostream;
+  template<typename T> T &&declval();
+  template<bool B, typename T> struct enable_if { using type = T; };
+
+  namespace mlir {
+    template<typename T, typename enable_if<declval<ostream&>() << declval<T&>(), void*>::type = nullptr>
+    ostream &operator<<(ostream& os, const T& obj); // expected-error {{exceeded maximum depth}}
+    struct Value;
+  }
+
+  void printFunctionalType(ostream &os, mlir::Value &v) { os << v; }
+}
+
+#else
+#error unknown test
+#endif
diff --git a/clang/test/Tooling/clang-check-ast-dump.cpp b/clang/test/Tooling/clang-check-ast-dump.cpp
index 496d489fd6090..bd9ae5fcc2a59 100644
--- a/clang/test/Tooling/clang-check-ast-dump.cpp
+++ b/clang/test/Tooling/clang-check-ast-dump.cpp
@@ -33,6 +33,7 @@
 // CHECK-ATTR-NEXT: FieldDecl{{.*}}n
 // CHECK-ATTR-NEXT:   AlignedAttr
 // CHECK-ATTR-NEXT:     ConstantExpr
+// CHECK-ATTR-NEXT:       value: Int 2
 // CHECK-ATTR-NEXT:       BinaryOperator
 //
 // RUN: clang-check -ast-dump -ast-dump-filter test_namespace::AfterNullNode "%s" -- 2>&1 | FileCheck -check-prefix CHECK-AFTER-NULL %s
diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py
index 413f811754204..ade32988b9a82 100644
--- a/clang/test/lit.cfg.py
+++ b/clang/test/lit.cfg.py
@@ -155,6 +155,10 @@ def is_filesystem_case_insensitive():
 if not re.match(r'.*-(cygwin)$', config.target_triple):
     config.available_features.add('clang-driver')
 
+# Tests that are specific to the Apple Silicon macOS.
+if re.match(r'^arm64(e)?-apple-(macos|darwin)', config.target_triple):
+    config.available_features.add('apple-silicon-mac')
+
 # [PR18856] Depends to remove opened file. On win32, a file could be removed
 # only if all handles were closed.
 if platform.system() not in ['Windows']:
diff --git a/clang/test/utils/update_cc_test_checks/Inputs/mangled_names.c.funcsig.expected b/clang/test/utils/update_cc_test_checks/Inputs/mangled_names.c.funcsig.expected
index 005b2f2427473..e76cf074bdb78 100644
--- a/clang/test/utils/update_cc_test_checks/Inputs/mangled_names.c.funcsig.expected
+++ b/clang/test/utils/update_cc_test_checks/Inputs/mangled_names.c.funcsig.expected
@@ -1,4 +1,4 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature
 // Example input for update_cc_test_checks
 // RUN: %clang_cc1 -triple=x86_64-unknown-linux-gnu -emit-llvm -o - %s | FileCheck %s
 
diff --git a/clang/test/utils/update_cc_test_checks/Inputs/on_the_fly_arg_change.c b/clang/test/utils/update_cc_test_checks/Inputs/on_the_fly_arg_change.c
new file mode 100644
index 0000000000000..8956e6b52a210
--- /dev/null
+++ b/clang/test/utils/update_cc_test_checks/Inputs/on_the_fly_arg_change.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -triple=x86_64-unknown-linux-gnu -emit-llvm -o - %s | FileCheck %s
+
+int checks_please() {
+  return 1;
+}
+
+// UTC_ARGS: --disable
+
+int no_checks_please() {
+  // Manual CHECK line should be retained:
+  // CHECK: manual check line
+  return -1;
+}
+
+// UTC_ARGS: --enable
+
+
+int checks_again() {
+  return 2;
+}
diff --git a/clang/test/utils/update_cc_test_checks/Inputs/on_the_fly_arg_change.c.expected b/clang/test/utils/update_cc_test_checks/Inputs/on_the_fly_arg_change.c.expected
new file mode 100644
index 0000000000000..cb7846c7b3d5a
--- /dev/null
+++ b/clang/test/utils/update_cc_test_checks/Inputs/on_the_fly_arg_change.c.expected
@@ -0,0 +1,29 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple=x86_64-unknown-linux-gnu -emit-llvm -o - %s | FileCheck %s
+
+// CHECK-LABEL: @checks_please(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+int checks_please() {
+  return 1;
+}
+
+// UTC_ARGS: --disable
+
+int no_checks_please() {
+  // Manual CHECK line should be retained:
+  // CHECK: manual check line
+  return -1;
+}
+
+// UTC_ARGS: --enable
+
+
+// CHECK-LABEL: @checks_again(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 2
+//
+int checks_again() {
+  return 2;
+}
diff --git a/clang/test/utils/update_cc_test_checks/mangled_names.test b/clang/test/utils/update_cc_test_checks/mangled_names.test
index 082ed74304f06..bc88c9b5a382f 100644
--- a/clang/test/utils/update_cc_test_checks/mangled_names.test
+++ b/clang/test/utils/update_cc_test_checks/mangled_names.test
@@ -8,6 +8,11 @@
 ## Also try the --function-signature flag
 # RUN: %update_cc_test_checks %t.c --function-signature
 # RUN: diff -u %t.c %S/Inputs/mangled_names.c.funcsig.expected
-## Verify that running without the --function-signature flag removes the -SAME: lines:
+## Running it again should implicitly add the function-signature flag due to UTC_ARGS:
 # RUN: %update_cc_test_checks %t.c
-# RUN: diff -u %t.c %S/Inputs/mangled_names.c.expected
+# RUN: diff -u %t.c %S/Inputs/mangled_names.c.funcsig.expected
+## Verify that running without the --function-signature flag removes the -SAME: lines:
+## We have to remove the UTC_ARGS comment first:
+# RUN: grep -v UTC_ARGS %t.c > %t-no-args.c
+# RUN: %update_cc_test_checks %t-no-args.c
+# RUN: diff -u %t-no-args.c %S/Inputs/mangled_names.c.expected
diff --git a/clang/test/utils/update_cc_test_checks/on_the_fly_arg_change.test b/clang/test/utils/update_cc_test_checks/on_the_fly_arg_change.test
new file mode 100644
index 0000000000000..629b01f9d0660
--- /dev/null
+++ b/clang/test/utils/update_cc_test_checks/on_the_fly_arg_change.test
@@ -0,0 +1,6 @@
+# RUN: cp -f %S/Inputs/on_the_fly_arg_change.c %t.c
+# RUN: %update_cc_test_checks %t.c
+# RUN: diff -u %t.c %S/Inputs/on_the_fly_arg_change.c.expected
+## Check that running the script again does not change the result:
+# RUN: %update_cc_test_checks %t.c
+# RUN: diff -u %t.c %S/Inputs/on_the_fly_arg_change.c.expected
diff --git a/clang/tools/clang-check/ClangCheck.cpp b/clang/tools/clang-check/ClangCheck.cpp
index 2f59e2b82d343..6565aa2290307 100644
--- a/clang/tools/clang-check/ClangCheck.cpp
+++ b/clang/tools/clang-check/ClangCheck.cpp
@@ -141,6 +141,7 @@ class ClangCheckActionFactory {
                                     /*DumpDecls=*/true,
                                     /*Deserialize=*/false,
                                     /*DumpLookups=*/false,
+                                    /*DumpDeclTypes=*/false,
                                     clang::ADOF_Default);
     if (ASTPrint)
       return clang::CreateASTPrinter(nullptr, ASTDumpFilter);
diff --git a/clang/tools/clang-import-test/CMakeLists.txt b/clang/tools/clang-import-test/CMakeLists.txt
index 39a5c41c1512d..4ccc2d752aac4 100644
--- a/clang/tools/clang-import-test/CMakeLists.txt
+++ b/clang/tools/clang-import-test/CMakeLists.txt
@@ -7,7 +7,7 @@ if(NOT CLANG_BUILT_STANDALONE)
   set(tablegen_deps intrinsics_gen)
 endif()
 
-add_clang_tool(clang-import-test
+add_clang_executable(clang-import-test
   clang-import-test.cpp
   DEPENDS
   ${tablegen_deps}
diff --git a/clang/tools/clang-import-test/clang-import-test.cpp b/clang/tools/clang-import-test/clang-import-test.cpp
index 2245c6aff8836..0c9bd237b3ba5 100644
--- a/clang/tools/clang-import-test/clang-import-test.cpp
+++ b/clang/tools/clang-import-test/clang-import-test.cpp
@@ -320,9 +320,9 @@ llvm::Expected<CIAndOrigins> Parse(const std::string &Path,
   auto &CG = *static_cast<CodeGenerator *>(ASTConsumers.back().get());
 
   if (ShouldDumpAST)
-    ASTConsumers.push_back(
-        CreateASTDumper(nullptr /*Dump to stdout.*/, "", true, false, false,
-                        clang::ADOF_Default));
+    ASTConsumers.push_back(CreateASTDumper(nullptr /*Dump to stdout.*/, "",
+                                           true, false, false, false,
+                                           clang::ADOF_Default));
 
   CI.getDiagnosticClient().BeginSourceFile(
       CI.getCompilerInstance().getLangOpts(),
diff --git a/clang/tools/clang-shlib/CMakeLists.txt b/clang/tools/clang-shlib/CMakeLists.txt
index 07ee0f0a9a929..5949223fc8e34 100644
--- a/clang/tools/clang-shlib/CMakeLists.txt
+++ b/clang/tools/clang-shlib/CMakeLists.txt
@@ -13,7 +13,12 @@ foreach (lib ${clang_libs})
   else()
     list(APPEND _OBJECTS $<TARGET_OBJECTS:obj.${lib}>)
   endif()
-  list(APPEND _DEPS $<TARGET_PROPERTY:${lib},INTERFACE_LINK_LIBRARIES>)
+  if (BUILD_SHARED_LIBS)
+    # If we are building static libraries, then we don't need to add the static
+    # libraries as a depedency, because we are already linking against the
+    # individual object files.
+    list(APPEND _DEPS $<TARGET_PROPERTY:${lib},INTERFACE_LINK_LIBRARIES>)
+  endif()
 
   # clang libraries are redundant since we are linking all the individual
   # object files into libclang-cpp.so, so filter them out from _DEPS.
diff --git a/clang/tools/driver/driver.cpp b/clang/tools/driver/driver.cpp
index 5e92fc32e6f6c..f24fd61e61a54 100644
--- a/clang/tools/driver/driver.cpp
+++ b/clang/tools/driver/driver.cpp
@@ -511,6 +511,11 @@ int main(int argc_, const char **argv_) {
       for (const auto &J : C->getJobs())
         if (const Command *C = dyn_cast<Command>(&J))
           FailingCommands.push_back(std::make_pair(-1, C));
+
+      // Print the bug report message that would be printed if we did actually
+      // crash, but only if we're crashing due to FORCE_CLANG_DIAGNOSTICS_CRASH.
+      if (::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH"))
+        llvm::dbgs() << llvm::getBugReportMsg();
     }
 
     for (const auto &P : FailingCommands) {
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 4a65624268d84..93f9797a965ec 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -3272,7 +3272,7 @@ bool CursorVisitor::RunVisitorWorkList(VisitorWorkList &WL) {
       }
       // Visit init captures
       for (auto InitExpr : E->capture_inits()) {
-        if (Visit(InitExpr))
+        if (InitExpr && Visit(InitExpr))
           return true;
       }
 
@@ -4056,10 +4056,14 @@ static const Expr *evaluateCompoundStmtExpr(const CompoundStmt *CS) {
 }
 
 CXEvalResult clang_Cursor_Evaluate(CXCursor C) {
-  if (const Expr *E =
-          clang_getCursorKind(C) == CXCursor_CompoundStmt
-              ? evaluateCompoundStmtExpr(cast<CompoundStmt>(getCursorStmt(C)))
-              : evaluateDeclExpr(getCursorDecl(C)))
+  const Expr *E = nullptr;
+  if (clang_getCursorKind(C) == CXCursor_CompoundStmt)
+    E = evaluateCompoundStmtExpr(cast<CompoundStmt>(getCursorStmt(C)));
+  else if (clang_isDeclaration(C.kind))
+    E = evaluateDeclExpr(getCursorDecl(C));
+  else if (clang_isExpression(C.kind))
+    E = getCursorExpr(C);
+  if (E)
     return const_cast<CXEvalResult>(
         reinterpret_cast<const void *>(evaluateExpr(const_cast<Expr *>(E), C)));
   return nullptr;
diff --git a/clang/tools/libclang/CIndexInclusionStack.cpp b/clang/tools/libclang/CIndexInclusionStack.cpp
index f1c5b53c5ef61..7572512ae5764 100644
--- a/clang/tools/libclang/CIndexInclusionStack.cpp
+++ b/clang/tools/libclang/CIndexInclusionStack.cpp
@@ -18,10 +18,9 @@
 #include "clang/Frontend/ASTUnit.h"
 using namespace clang;
 
-static void getInclusions(const SrcMgr::SLocEntry &(SourceManager::*Getter)(unsigned, bool*) const, unsigned n,
-                          CXTranslationUnit TU, CXInclusionVisitor CB,
-                          CXClientData clientData)
-{
+namespace {
+void getInclusions(bool IsLocal, unsigned n, CXTranslationUnit TU,
+                   CXInclusionVisitor CB, CXClientData clientData) {
   ASTUnit *CXXUnit = cxtu::getASTUnit(TU);
   SourceManager &SM = CXXUnit->getSourceManager();
   ASTContext &Ctx = CXXUnit->getASTContext();
@@ -30,8 +29,8 @@ static void getInclusions(const SrcMgr::SLocEntry &(SourceManager::*Getter)(unsi
 
   for (unsigned i = 0 ; i < n ; ++i) {
     bool Invalid = false;
-    const SrcMgr::SLocEntry &SL = (SM.*Getter)(i, &Invalid);
-
+    const SrcMgr::SLocEntry &SL =
+        IsLocal ? SM.getLocalSLocEntry(i) : SM.getLoadedSLocEntry(i, &Invalid);
     if (!SL.isFile() || Invalid)
       continue;
 
@@ -61,11 +60,11 @@ static void getInclusions(const SrcMgr::SLocEntry &(SourceManager::*Getter)(unsi
     // Callback to the client.
     // FIXME: We should have a function to construct CXFiles.
     CB(static_cast<CXFile>(
-         const_cast<FileEntry *>(FI.getContentCache()->OrigEntry)),
+           const_cast<FileEntry *>(FI.getContentCache()->OrigEntry)),
        InclusionStack.data(), InclusionStack.size(), clientData);
   }
 }
-
+} // namespace
 
 void clang_getInclusions(CXTranslationUnit TU, CXInclusionVisitor CB,
                          CXClientData clientData) {
@@ -83,14 +82,13 @@ void clang_getInclusions(CXTranslationUnit TU, CXInclusionVisitor CB,
   // a AST/PCH file, but this file has a pre-compiled preamble, we also need
   // to look in that file.
   if (n == 1 || SM.getPreambleFileID().isValid()) {
-    getInclusions(&SourceManager::getLoadedSLocEntry,
-                  SM.loaded_sloc_entry_size(), TU, CB, clientData);
+    getInclusions(/*IsLocal=*/false, SM.loaded_sloc_entry_size(), TU, CB,
+                  clientData);
   }
 
   // Not a PCH/AST file. Note, if there is a preamble, it could still be that
   // there are #includes in this file (e.g. for any include after the first
   // declaration).
   if (n != 1)
-    getInclusions(&SourceManager::getLocalSLocEntry, n, TU, CB, clientData);
-
+    getInclusions(/*IsLocal=*/true, n, TU, CB, clientData);
 }
diff --git a/clang/unittests/AST/ASTImporterTest.cpp b/clang/unittests/AST/ASTImporterTest.cpp
index 172fbb461a7e3..f92ea8de16516 100644
--- a/clang/unittests/AST/ASTImporterTest.cpp
+++ b/clang/unittests/AST/ASTImporterTest.cpp
@@ -3416,6 +3416,26 @@ TEST_P(ASTImporterOptionSpecificTestBase,
   EXPECT_TRUE(ToCtor->hasBody());
 }
 
+TEST_P(ASTImporterOptionSpecificTestBase, ClassTemplateFriendDecl) {
+  const auto *Code =
+      R"(
+      template <class T> class X {  friend T; };
+      struct Y {};
+      template class X<Y>;
+    )";
+  Decl *ToTU = getToTuDecl(Code, Lang_CXX11);
+  Decl *FromTU = getTuDecl(Code, Lang_CXX11);
+  auto *FromSpec = FirstDeclMatcher<ClassTemplateSpecializationDecl>().match(
+      FromTU, classTemplateSpecializationDecl());
+  auto *ToSpec = FirstDeclMatcher<ClassTemplateSpecializationDecl>().match(
+      ToTU, classTemplateSpecializationDecl());
+
+  auto *ImportedSpec = Import(FromSpec, Lang_CXX11);
+  EXPECT_EQ(ImportedSpec, ToSpec);
+  EXPECT_EQ(1u, DeclCounter<ClassTemplateSpecializationDecl>().match(
+                    ToTU, classTemplateSpecializationDecl()));
+}
+
 TEST_P(ASTImporterOptionSpecificTestBase,
        ClassTemplatePartialSpecializationsShouldNotBeDuplicated) {
   auto Code =
@@ -3954,6 +3974,56 @@ TEST_P(ImportFriendClasses, ImportOfClassDefinitionAndFwdFriendShouldBeLinked) {
   EXPECT_EQ(ImportedFwd, ImportedDef->getPreviousDecl());
 }
 
+TEST_P(ImportFriendClasses, ImportOfRepeatedFriendType) {
+  const char *Code =
+      R"(
+      class Container {
+        friend class X;
+        friend class X;
+      };
+      )";
+  Decl *ToTu = getToTuDecl(Code, Lang_CXX03);
+  Decl *FromTu = getTuDecl(Code, Lang_CXX03, "from.cc");
+
+  auto *ToFriend1 = FirstDeclMatcher<FriendDecl>().match(ToTu, friendDecl());
+  auto *ToFriend2 = LastDeclMatcher<FriendDecl>().match(ToTu, friendDecl());
+  auto *FromFriend1 =
+      FirstDeclMatcher<FriendDecl>().match(FromTu, friendDecl());
+  auto *FromFriend2 = LastDeclMatcher<FriendDecl>().match(FromTu, friendDecl());
+
+  FriendDecl *ToImportedFriend1 = Import(FromFriend1, Lang_CXX03);
+  FriendDecl *ToImportedFriend2 = Import(FromFriend2, Lang_CXX03);
+
+  EXPECT_NE(ToImportedFriend1, ToImportedFriend2);
+  EXPECT_EQ(ToFriend1, ToImportedFriend1);
+  EXPECT_EQ(ToFriend2, ToImportedFriend2);
+}
+
+TEST_P(ImportFriendClasses, ImportOfRepeatedFriendDecl) {
+  const char *Code =
+      R"(
+      class Container {
+        friend void f();
+        friend void f();
+      };
+      )";
+  Decl *ToTu = getToTuDecl(Code, Lang_CXX03);
+  Decl *FromTu = getTuDecl(Code, Lang_CXX03, "from.cc");
+
+  auto *ToFriend1 = FirstDeclMatcher<FriendDecl>().match(ToTu, friendDecl());
+  auto *ToFriend2 = LastDeclMatcher<FriendDecl>().match(ToTu, friendDecl());
+  auto *FromFriend1 =
+      FirstDeclMatcher<FriendDecl>().match(FromTu, friendDecl());
+  auto *FromFriend2 = LastDeclMatcher<FriendDecl>().match(FromTu, friendDecl());
+
+  FriendDecl *ToImportedFriend1 = Import(FromFriend1, Lang_CXX03);
+  FriendDecl *ToImportedFriend2 = Import(FromFriend2, Lang_CXX03);
+
+  EXPECT_NE(ToImportedFriend1, ToImportedFriend2);
+  EXPECT_EQ(ToFriend1, ToImportedFriend1);
+  EXPECT_EQ(ToFriend2, ToImportedFriend2);
+}
+
 TEST_P(ASTImporterOptionSpecificTestBase, FriendFunInClassTemplate) {
   auto *Code = R"(
   template <class T>
diff --git a/clang/unittests/AST/ASTTraverserTest.cpp b/clang/unittests/AST/ASTTraverserTest.cpp
index 5585238939fea..2f131c3fd04f0 100644
--- a/clang/unittests/AST/ASTTraverserTest.cpp
+++ b/clang/unittests/AST/ASTTraverserTest.cpp
@@ -244,6 +244,7 @@ FullComment
   verifyWithDynNode(TA,
                     R"cpp(
 TemplateArgument
+`-BuiltinType
 )cpp");
 
   Func = getFunctionNode(AST.get(), "parmvardecl_attr");
@@ -430,6 +431,7 @@ VarDecl 'c1'
 StaticAssertDecl
 |-ImplicitCastExpr
 | `-SubstNonTypeTemplateParmExpr
+|   |-NonTypeTemplateParmDecl 'alignment'
 |   `-IntegerLiteral
 `-StringLiteral
 )cpp");
diff --git a/clang/unittests/AST/CommentParser.cpp b/clang/unittests/AST/CommentParser.cpp
index 327cabd619bdb..ba8b34ebcd38b 100644
--- a/clang/unittests/AST/CommentParser.cpp
+++ b/clang/unittests/AST/CommentParser.cpp
@@ -64,7 +64,7 @@ FullComment *CommentParserTest::parseString(const char *Source) {
 
   if (MY_DEBUG) {
     llvm::errs() << "=== Source:\n" << Source << "\n=== AST:\n";
-    FC->dump(llvm::errs(), &Traits, &SourceMgr);
+    FC->dump();
   }
 
   Token Tok;
diff --git a/clang/unittests/AST/DeclPrinterTest.cpp b/clang/unittests/AST/DeclPrinterTest.cpp
index 018e99237ae94..939c8b52c12c1 100644
--- a/clang/unittests/AST/DeclPrinterTest.cpp
+++ b/clang/unittests/AST/DeclPrinterTest.cpp
@@ -76,14 +76,16 @@ ::testing::AssertionResult
 PrintedDeclMatches(StringRef Code, const std::vector<std::string> &Args,
                    const DeclarationMatcher &NodeMatch,
                    StringRef ExpectedPrinted, StringRef FileName,
-                   PrintingPolicyModifier PolicyModifier = nullptr) {
+                   PrintingPolicyModifier PolicyModifier = nullptr,
+                   bool AllowError = false) {
   PrintMatch Printer(PolicyModifier);
   MatchFinder Finder;
   Finder.addMatcher(NodeMatch, &Printer);
   std::unique_ptr<FrontendActionFactory> Factory(
       newFrontendActionFactory(&Finder));
 
-  if (!runToolOnCodeWithArgs(Factory->create(), Code, Args, FileName))
+  if (!runToolOnCodeWithArgs(Factory->create(), Code, Args, FileName) &&
+      !AllowError)
     return testing::AssertionFailure()
       << "Parsing error in \"" << Code.str() << "\"";
 
@@ -170,16 +172,12 @@ PrintedDeclCXX1ZMatches(StringRef Code, const DeclarationMatcher &NodeMatch,
                             "input.cc");
 }
 
-::testing::AssertionResult PrintedDeclObjCMatches(
-                                  StringRef Code,
-                                  const DeclarationMatcher &NodeMatch,
-                                  StringRef ExpectedPrinted) {
+::testing::AssertionResult
+PrintedDeclObjCMatches(StringRef Code, const DeclarationMatcher &NodeMatch,
+                       StringRef ExpectedPrinted, bool AllowError = false) {
   std::vector<std::string> Args(1, "");
-  return PrintedDeclMatches(Code,
-                            Args,
-                            NodeMatch,
-                            ExpectedPrinted,
-                            "input.m");
+  return PrintedDeclMatches(Code, Args, NodeMatch, ExpectedPrinted, "input.m",
+                            /*PolicyModifier=*/nullptr, AllowError);
 }
 
 } // unnamed namespace
@@ -1321,3 +1319,17 @@ TEST(DeclPrinter, TestObjCProtocol2) {
     namedDecl(hasName("P1")).bind("id"),
     "@protocol P1<P2>\n@end"));
 }
+
+TEST(DeclPrinter, TestObjCCategoryInvalidInterface) {
+  ASSERT_TRUE(PrintedDeclObjCMatches(
+      "@interface I (Extension) @end",
+      namedDecl(hasName("Extension")).bind("id"),
+      "@interface <<error-type>>(Extension)\n@end", /*AllowError=*/true));
+}
+
+TEST(DeclPrinter, TestObjCCategoryImplInvalidInterface) {
+  ASSERT_TRUE(PrintedDeclObjCMatches(
+      "@implementation I (Extension) @end",
+      namedDecl(hasName("Extension")).bind("id"),
+      "@implementation <<error-type>>(Extension)\n@end", /*AllowError=*/true));
+}
diff --git a/clang/unittests/AST/MatchVerifier.h b/clang/unittests/AST/MatchVerifier.h
index 3fb22ef00f98f..217c1abcff693 100644
--- a/clang/unittests/AST/MatchVerifier.h
+++ b/clang/unittests/AST/MatchVerifier.h
@@ -271,7 +271,7 @@ class DumpVerifier : public MatchVerifier<DynTypedNode> {
               const DynTypedNode &Node) override {
     std::string DumpStr;
     llvm::raw_string_ostream Dump(DumpStr);
-    Node.dump(Dump, *Result.SourceManager);
+    Node.dump(Dump, *Result.Context);
 
     if (Dump.str().find(ExpectSubstring) == std::string::npos) {
       std::string MsgStr;
diff --git a/clang/unittests/AST/SourceLocationTest.cpp b/clang/unittests/AST/SourceLocationTest.cpp
index cb96afed64d31..32dc382aa05c3 100644
--- a/clang/unittests/AST/SourceLocationTest.cpp
+++ b/clang/unittests/AST/SourceLocationTest.cpp
@@ -60,6 +60,59 @@ TEST(RangeVerifier, WrongRange) {
   EXPECT_FALSE(Verifier.match("int i;", varDecl()));
 }
 
+class WhileParenLocationVerifier : public MatchVerifier<WhileStmt> {
+  unsigned ExpectLParenLine = 0, ExpectLParenColumn = 0;
+  unsigned ExpectRParenLine = 0, ExpectRParenColumn = 0;
+
+public:
+  void expectLocations(unsigned LParenLine, unsigned LParenColumn,
+                       unsigned RParenLine, unsigned RParenColumn) {
+    ExpectLParenLine = LParenLine;
+    ExpectLParenColumn = LParenColumn;
+    ExpectRParenLine = RParenLine;
+    ExpectRParenColumn = RParenColumn;
+  }
+
+protected:
+  void verify(const MatchFinder::MatchResult &Result,
+              const WhileStmt &Node) override {
+    SourceLocation LParenLoc = Node.getLParenLoc();
+    SourceLocation RParenLoc = Node.getRParenLoc();
+    unsigned LParenLine =
+        Result.SourceManager->getSpellingLineNumber(LParenLoc);
+    unsigned LParenColumn =
+        Result.SourceManager->getSpellingColumnNumber(LParenLoc);
+    unsigned RParenLine =
+        Result.SourceManager->getSpellingLineNumber(RParenLoc);
+    unsigned RParenColumn =
+        Result.SourceManager->getSpellingColumnNumber(RParenLoc);
+
+    if (LParenLine != ExpectLParenLine || LParenColumn != ExpectLParenColumn ||
+        RParenLine != ExpectRParenLine || RParenColumn != ExpectRParenColumn) {
+      std::string MsgStr;
+      llvm::raw_string_ostream Msg(MsgStr);
+      Msg << "Expected LParen Location <" << ExpectLParenLine << ":"
+          << ExpectLParenColumn << ">, found <";
+      LParenLoc.print(Msg, *Result.SourceManager);
+      Msg << ">\n";
+
+      Msg << "Expected RParen Location <" << ExpectRParenLine << ":"
+          << ExpectRParenColumn << ">, found <";
+      RParenLoc.print(Msg, *Result.SourceManager);
+      Msg << ">";
+
+      this->setFailure(Msg.str());
+    }
+  }
+};
+
+TEST(LocationVerifier, WhileParenLoc) {
+  WhileParenLocationVerifier Verifier;
+  Verifier.expectLocations(1, 17, 1, 38);
+  EXPECT_TRUE(Verifier.match("void f() { while(true/*some comment*/) {} }",
+                             whileStmt()));
+}
+
 class LabelDeclRangeVerifier : public RangeVerifier<LabelStmt> {
 protected:
   SourceRange getRange(const LabelStmt &Node) override {
diff --git a/clang/unittests/AST/StructuralEquivalenceTest.cpp b/clang/unittests/AST/StructuralEquivalenceTest.cpp
index 08512d6062793..2b5ce0fed51d6 100644
--- a/clang/unittests/AST/StructuralEquivalenceTest.cpp
+++ b/clang/unittests/AST/StructuralEquivalenceTest.cpp
@@ -759,6 +759,27 @@ TEST_F(StructuralEquivalenceRecordTest, RecordsWithDifferentBody) {
   EXPECT_FALSE(testStructuralMatch(t));
 }
 
+TEST_F(StructuralEquivalenceRecordTest, SameFriendMultipleTimes) {
+  auto t = makeNamedDecls("struct foo { friend class X; };",
+                          "struct foo { friend class X; friend class X; };",
+                          Lang_CXX11);
+  EXPECT_FALSE(testStructuralMatch(t));
+}
+
+TEST_F(StructuralEquivalenceRecordTest, SameFriendsDifferentOrder) {
+  auto t = makeNamedDecls("struct foo { friend class X; friend class Y; };",
+                          "struct foo { friend class Y; friend class X; };",
+                          Lang_CXX11);
+  EXPECT_FALSE(testStructuralMatch(t));
+}
+
+TEST_F(StructuralEquivalenceRecordTest, SameFriendsSameOrder) {
+  auto t = makeNamedDecls("struct foo { friend class X; friend class Y; };",
+                          "struct foo { friend class X; friend class Y; };",
+                          Lang_CXX11);
+  EXPECT_TRUE(testStructuralMatch(t));
+}
+
 struct StructuralEquivalenceLambdaTest : StructuralEquivalenceTest {};
 
 TEST_F(StructuralEquivalenceLambdaTest, LambdaClassesWithDifferentMethods) {
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
index 0070c22164a6b..687908043a8d3 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -103,9 +103,9 @@ TEST(IsExpandedFromMacro, ShouldMatchFromCommandLine) {
   StringRef input = R"cc(
     void Test() { FOUR_PLUS_FOUR; }
   )cc";
-  EXPECT_TRUE(matchesConditionally(input,
-                                   binaryOperator(isExpandedFromMacro("FOUR_PLUS_FOUR")),
-                                   true, {"-std=c++11", "-DFOUR_PLUS_FOUR=4+4"}));
+  EXPECT_TRUE(matchesConditionally(
+      input, binaryOperator(isExpandedFromMacro("FOUR_PLUS_FOUR")), true,
+      {"-std=c++11", "-DFOUR_PLUS_FOUR=4+4"}));
 }
 
 TEST(IsExpandedFromMacro, ShouldNotMatchBeginOnly) {
@@ -143,31 +143,31 @@ TEST(IsExpandedFromMacro, ShouldNotMatchDifferentInstances) {
 }
 
 TEST(AllOf, AllOverloadsWork) {
-  const char Program[] =
-      "struct T { };"
-      "int f(int, T*, int, int);"
-      "void g(int x) { T t; f(x, &t, 3, 4); }";
-  EXPECT_TRUE(matches(Program,
-      callExpr(allOf(callee(functionDecl(hasName("f"))),
-                     hasArgument(0, declRefExpr(to(varDecl())))))));
-  EXPECT_TRUE(matches(Program,
-      callExpr(allOf(callee(functionDecl(hasName("f"))),
-                     hasArgument(0, declRefExpr(to(varDecl()))),
-                     hasArgument(1, hasType(pointsTo(
-                                        recordDecl(hasName("T")))))))));
-  EXPECT_TRUE(matches(Program,
-      callExpr(allOf(callee(functionDecl(hasName("f"))),
-                     hasArgument(0, declRefExpr(to(varDecl()))),
-                     hasArgument(1, hasType(pointsTo(
-                                        recordDecl(hasName("T"))))),
-                     hasArgument(2, integerLiteral(equals(3)))))));
-  EXPECT_TRUE(matches(Program,
-      callExpr(allOf(callee(functionDecl(hasName("f"))),
-                     hasArgument(0, declRefExpr(to(varDecl()))),
-                     hasArgument(1, hasType(pointsTo(
-                                        recordDecl(hasName("T"))))),
-                     hasArgument(2, integerLiteral(equals(3))),
-                     hasArgument(3, integerLiteral(equals(4)))))));
+  const char Program[] = "struct T { };"
+                         "int f(int, T*, int, int);"
+                         "void g(int x) { T t; f(x, &t, 3, 4); }";
+  EXPECT_TRUE(matches(
+      Program, callExpr(allOf(callee(functionDecl(hasName("f"))),
+                              hasArgument(0, declRefExpr(to(varDecl())))))));
+  EXPECT_TRUE(matches(
+      Program,
+      callExpr(
+          allOf(callee(functionDecl(hasName("f"))),
+                hasArgument(0, declRefExpr(to(varDecl()))),
+                hasArgument(1, hasType(pointsTo(recordDecl(hasName("T")))))))));
+  EXPECT_TRUE(matches(
+      Program, callExpr(allOf(
+                   callee(functionDecl(hasName("f"))),
+                   hasArgument(0, declRefExpr(to(varDecl()))),
+                   hasArgument(1, hasType(pointsTo(recordDecl(hasName("T"))))),
+                   hasArgument(2, integerLiteral(equals(3)))))));
+  EXPECT_TRUE(matches(
+      Program, callExpr(allOf(
+                   callee(functionDecl(hasName("f"))),
+                   hasArgument(0, declRefExpr(to(varDecl()))),
+                   hasArgument(1, hasType(pointsTo(recordDecl(hasName("T"))))),
+                   hasArgument(2, integerLiteral(equals(3))),
+                   hasArgument(3, integerLiteral(equals(4)))))));
 }
 
 TEST(DeclarationMatcher, MatchHas) {
@@ -176,127 +176,103 @@ TEST(DeclarationMatcher, MatchHas) {
   EXPECT_TRUE(matches("class X {};", HasClassX));
 
   DeclarationMatcher YHasClassX =
-    recordDecl(hasName("Y"), has(recordDecl(hasName("X"))));
+      recordDecl(hasName("Y"), has(recordDecl(hasName("X"))));
   EXPECT_TRUE(matches("class Y { class X {}; };", YHasClassX));
   EXPECT_TRUE(notMatches("class X {};", YHasClassX));
-  EXPECT_TRUE(
-    notMatches("class Y { class Z { class X {}; }; };", YHasClassX));
+  EXPECT_TRUE(notMatches("class Y { class Z { class X {}; }; };", YHasClassX));
 }
 
 TEST(DeclarationMatcher, MatchHasRecursiveAllOf) {
   DeclarationMatcher Recursive =
-    recordDecl(
-      has(recordDecl(
-        has(recordDecl(hasName("X"))),
-        has(recordDecl(hasName("Y"))),
-        hasName("Z"))),
-      has(recordDecl(
-        has(recordDecl(hasName("A"))),
-        has(recordDecl(hasName("B"))),
-        hasName("C"))),
-      hasName("F"));
-
-  EXPECT_TRUE(matches(
-    "class F {"
-      "  class Z {"
-      "    class X {};"
-      "    class Y {};"
-      "  };"
-      "  class C {"
-      "    class A {};"
-      "    class B {};"
-      "  };"
-      "};", Recursive));
-
-  EXPECT_TRUE(matches(
-    "class F {"
-      "  class Z {"
-      "    class A {};"
-      "    class X {};"
-      "    class Y {};"
-      "  };"
-      "  class C {"
-      "    class X {};"
-      "    class A {};"
-      "    class B {};"
-      "  };"
-      "};", Recursive));
-
-  EXPECT_TRUE(matches(
-    "class O1 {"
-      "  class O2 {"
-      "    class F {"
-      "      class Z {"
-      "        class A {};"
-      "        class X {};"
-      "        class Y {};"
-      "      };"
-      "      class C {"
-      "        class X {};"
-      "        class A {};"
-      "        class B {};"
-      "      };"
-      "    };"
-      "  };"
-      "};", Recursive));
+      recordDecl(has(recordDecl(has(recordDecl(hasName("X"))),
+                                has(recordDecl(hasName("Y"))), hasName("Z"))),
+                 has(recordDecl(has(recordDecl(hasName("A"))),
+                                has(recordDecl(hasName("B"))), hasName("C"))),
+                 hasName("F"));
+
+  EXPECT_TRUE(matches("class F {"
+                      "  class Z {"
+                      "    class X {};"
+                      "    class Y {};"
+                      "  };"
+                      "  class C {"
+                      "    class A {};"
+                      "    class B {};"
+                      "  };"
+                      "};",
+                      Recursive));
+
+  EXPECT_TRUE(matches("class F {"
+                      "  class Z {"
+                      "    class A {};"
+                      "    class X {};"
+                      "    class Y {};"
+                      "  };"
+                      "  class C {"
+                      "    class X {};"
+                      "    class A {};"
+                      "    class B {};"
+                      "  };"
+                      "};",
+                      Recursive));
+
+  EXPECT_TRUE(matches("class O1 {"
+                      "  class O2 {"
+                      "    class F {"
+                      "      class Z {"
+                      "        class A {};"
+                      "        class X {};"
+                      "        class Y {};"
+                      "      };"
+                      "      class C {"
+                      "        class X {};"
+                      "        class A {};"
+                      "        class B {};"
+                      "      };"
+                      "    };"
+                      "  };"
+                      "};",
+                      Recursive));
 }
 
 TEST(DeclarationMatcher, MatchHasRecursiveAnyOf) {
-  DeclarationMatcher Recursive =
-    recordDecl(
-      anyOf(
-        has(recordDecl(
-          anyOf(
-            has(recordDecl(
-              hasName("X"))),
-            has(recordDecl(
-              hasName("Y"))),
-            hasName("Z")))),
-        has(recordDecl(
-          anyOf(
-            hasName("C"),
-            has(recordDecl(
-              hasName("A"))),
-            has(recordDecl(
-              hasName("B")))))),
-        hasName("F")));
+  DeclarationMatcher Recursive = recordDecl(
+      anyOf(has(recordDecl(anyOf(has(recordDecl(hasName("X"))),
+                                 has(recordDecl(hasName("Y"))), hasName("Z")))),
+            has(recordDecl(anyOf(hasName("C"), has(recordDecl(hasName("A"))),
+                                 has(recordDecl(hasName("B")))))),
+            hasName("F")));
 
   EXPECT_TRUE(matches("class F {};", Recursive));
   EXPECT_TRUE(matches("class Z {};", Recursive));
   EXPECT_TRUE(matches("class C {};", Recursive));
   EXPECT_TRUE(matches("class M { class N { class X {}; }; };", Recursive));
   EXPECT_TRUE(matches("class M { class N { class B {}; }; };", Recursive));
-  EXPECT_TRUE(
-    matches("class O1 { class O2 {"
-              "  class M { class N { class B {}; }; }; "
-              "}; };", Recursive));
+  EXPECT_TRUE(matches("class O1 { class O2 {"
+                      "  class M { class N { class B {}; }; }; "
+                      "}; };",
+                      Recursive));
 }
 
 TEST(DeclarationMatcher, MatchNot) {
   DeclarationMatcher NotClassX =
-    cxxRecordDecl(
-      isDerivedFrom("Y"),
-      unless(hasName("X")));
+      cxxRecordDecl(isDerivedFrom("Y"), unless(hasName("X")));
   EXPECT_TRUE(notMatches("", NotClassX));
   EXPECT_TRUE(notMatches("class Y {};", NotClassX));
   EXPECT_TRUE(matches("class Y {}; class Z : public Y {};", NotClassX));
   EXPECT_TRUE(notMatches("class Y {}; class X : public Y {};", NotClassX));
   EXPECT_TRUE(
-    notMatches("class Y {}; class Z {}; class X : public Y {};",
-               NotClassX));
+      notMatches("class Y {}; class Z {}; class X : public Y {};", NotClassX));
 
   DeclarationMatcher ClassXHasNotClassY =
-    recordDecl(
-      hasName("X"),
-      has(recordDecl(hasName("Z"))),
-      unless(
-        has(recordDecl(hasName("Y")))));
+      recordDecl(hasName("X"), has(recordDecl(hasName("Z"))),
+                 unless(has(recordDecl(hasName("Y")))));
   EXPECT_TRUE(matches("class X { class Z {}; };", ClassXHasNotClassY));
-  EXPECT_TRUE(notMatches("class X { class Y {}; class Z {}; };",
-                         ClassXHasNotClassY));
+  EXPECT_TRUE(
+      notMatches("class X { class Y {}; class Z {}; };", ClassXHasNotClassY));
 
   DeclarationMatcher NamedNotRecord =
-    namedDecl(hasName("Foo"), unless(recordDecl()));
+      namedDecl(hasName("Foo"), unless(recordDecl()));
   EXPECT_TRUE(matches("void Foo(){}", NamedNotRecord));
   EXPECT_TRUE(notMatches("struct Foo {};", NamedNotRecord));
 }
@@ -318,67 +294,61 @@ TEST(CastExpression, HasCastKind) {
 
 TEST(DeclarationMatcher, HasDescendant) {
   DeclarationMatcher ZDescendantClassX =
-    recordDecl(
-      hasDescendant(recordDecl(hasName("X"))),
-      hasName("Z"));
+      recordDecl(hasDescendant(recordDecl(hasName("X"))), hasName("Z"));
   EXPECT_TRUE(matches("class Z { class X {}; };", ZDescendantClassX));
   EXPECT_TRUE(
-    matches("class Z { class Y { class X {}; }; };", ZDescendantClassX));
+      matches("class Z { class Y { class X {}; }; };", ZDescendantClassX));
+  EXPECT_TRUE(matches("class Z { class A { class Y { class X {}; }; }; };",
+                      ZDescendantClassX));
   EXPECT_TRUE(
-    matches("class Z { class A { class Y { class X {}; }; }; };",
-            ZDescendantClassX));
-  EXPECT_TRUE(
-    matches("class Z { class A { class B { class Y { class X {}; }; }; }; };",
-            ZDescendantClassX));
+      matches("class Z { class A { class B { class Y { class X {}; }; }; }; };",
+              ZDescendantClassX));
   EXPECT_TRUE(notMatches("class Z {};", ZDescendantClassX));
 
-  DeclarationMatcher ZDescendantClassXHasClassY =
-    recordDecl(
-      hasDescendant(recordDecl(has(recordDecl(hasName("Y"))),
-                               hasName("X"))),
+  DeclarationMatcher ZDescendantClassXHasClassY = recordDecl(
+      hasDescendant(recordDecl(has(recordDecl(hasName("Y"))), hasName("X"))),
       hasName("Z"));
   EXPECT_TRUE(matches("class Z { class X { class Y {}; }; };",
                       ZDescendantClassXHasClassY));
   EXPECT_TRUE(
-    matches("class Z { class A { class B { class X { class Y {}; }; }; }; };",
-            ZDescendantClassXHasClassY));
-  EXPECT_TRUE(notMatches(
-    "class Z {"
-      "  class A {"
-      "    class B {"
-      "      class X {"
-      "        class C {"
-      "          class Y {};"
-      "        };"
-      "      };"
-      "    }; "
-      "  };"
-      "};", ZDescendantClassXHasClassY));
+      matches("class Z { class A { class B { class X { class Y {}; }; }; }; };",
+              ZDescendantClassXHasClassY));
+  EXPECT_TRUE(notMatches("class Z {"
+                         "  class A {"
+                         "    class B {"
+                         "      class X {"
+                         "        class C {"
+                         "          class Y {};"
+                         "        };"
+                         "      };"
+                         "    }; "
+                         "  };"
+                         "};",
+                         ZDescendantClassXHasClassY));
 
   DeclarationMatcher ZDescendantClassXDescendantClassY =
-    recordDecl(
-      hasDescendant(recordDecl(hasDescendant(recordDecl(hasName("Y"))),
-                               hasName("X"))),
-      hasName("Z"));
-  EXPECT_TRUE(
-    matches("class Z { class A { class X { class B { class Y {}; }; }; }; };",
-            ZDescendantClassXDescendantClassY));
-  EXPECT_TRUE(matches(
-    "class Z {"
-      "  class A {"
-      "    class X {"
-      "      class B {"
-      "        class Y {};"
-      "      };"
-      "      class Y {};"
-      "    };"
-      "  };"
-      "};", ZDescendantClassXDescendantClassY));
+      recordDecl(hasDescendant(recordDecl(
+                     hasDescendant(recordDecl(hasName("Y"))), hasName("X"))),
+                 hasName("Z"));
+  EXPECT_TRUE(
+      matches("class Z { class A { class X { class B { class Y {}; }; }; }; };",
+              ZDescendantClassXDescendantClassY));
+  EXPECT_TRUE(matches("class Z {"
+                      "  class A {"
+                      "    class X {"
+                      "      class B {"
+                      "        class Y {};"
+                      "      };"
+                      "      class Y {};"
+                      "    };"
+                      "  };"
+                      "};",
+                      ZDescendantClassXDescendantClassY));
 }
 
 TEST(DeclarationMatcher, HasDescendantMemoization) {
   DeclarationMatcher CannotMemoize =
-    decl(hasDescendant(typeLoc().bind("x")), has(decl()));
+      decl(hasDescendant(typeLoc().bind("x")), has(decl()));
   EXPECT_TRUE(matches("void f() { int i; }", CannotMemoize));
 }
 
@@ -401,39 +371,36 @@ TEST(DeclarationMatcher, HasAncestorMemoization) {
   // That node can't be memoized so we have to check for it before trying to put
   // it on the cache.
   DeclarationMatcher CannotMemoize = classTemplateSpecializationDecl(
-    hasAnyTemplateArgument(templateArgument().bind("targ")),
-    forEach(fieldDecl(hasAncestor(forStmt()))));
+      hasAnyTemplateArgument(templateArgument().bind("targ")),
+      forEach(fieldDecl(hasAncestor(forStmt()))));
 
   EXPECT_TRUE(notMatches("template <typename T> struct S;"
-                           "template <> struct S<int>{ int i; int j; };",
+                         "template <> struct S<int>{ int i; int j; };",
                          CannotMemoize));
 }
 
 TEST(DeclarationMatcher, HasAttr) {
   EXPECT_TRUE(matches("struct __attribute__((warn_unused)) X {};",
                       decl(hasAttr(clang::attr::WarnUnused))));
-  EXPECT_FALSE(matches("struct X {};",
-                       decl(hasAttr(clang::attr::WarnUnused))));
+  EXPECT_FALSE(matches("struct X {};", decl(hasAttr(clang::attr::WarnUnused))));
 }
 
-
 TEST(DeclarationMatcher, MatchAnyOf) {
   DeclarationMatcher YOrZDerivedFromX = cxxRecordDecl(
-    anyOf(hasName("Y"), allOf(isDerivedFrom("X"), hasName("Z"))));
+      anyOf(hasName("Y"), allOf(isDerivedFrom("X"), hasName("Z"))));
   EXPECT_TRUE(matches("class X {}; class Z : public X {};", YOrZDerivedFromX));
   EXPECT_TRUE(matches("class Y {};", YOrZDerivedFromX));
   EXPECT_TRUE(
-    notMatches("class X {}; class W : public X {};", YOrZDerivedFromX));
+      notMatches("class X {}; class W : public X {};", YOrZDerivedFromX));
   EXPECT_TRUE(notMatches("class Z {};", YOrZDerivedFromX));
 
   DeclarationMatcher XOrYOrZOrU =
-    recordDecl(anyOf(hasName("X"), hasName("Y"), hasName("Z"), hasName("U")));
+      recordDecl(anyOf(hasName("X"), hasName("Y"), hasName("Z"), hasName("U")));
   EXPECT_TRUE(matches("class X {};", XOrYOrZOrU));
   EXPECT_TRUE(notMatches("class V {};", XOrYOrZOrU));
 
-  DeclarationMatcher XOrYOrZOrUOrV =
-    recordDecl(anyOf(hasName("X"), hasName("Y"), hasName("Z"), hasName("U"),
-                     hasName("V")));
+  DeclarationMatcher XOrYOrZOrUOrV = recordDecl(anyOf(
+      hasName("X"), hasName("Y"), hasName("Z"), hasName("U"), hasName("V")));
   EXPECT_TRUE(matches("class X {};", XOrYOrZOrUOrV));
   EXPECT_TRUE(matches("class Y {};", XOrYOrZOrUOrV));
   EXPECT_TRUE(matches("class Z {};", XOrYOrZOrUOrV));
@@ -447,8 +414,8 @@ TEST(DeclarationMatcher, MatchAnyOf) {
   EXPECT_TRUE(notMatches("int F() { return 1; }", MixedTypes));
 
   EXPECT_TRUE(
-    matches("void f() try { } catch (int) { } catch (...) { }",
-            cxxCatchStmt(anyOf(hasDescendant(varDecl()), isCatchAll()))));
+      matches("void f() try { } catch (int) { } catch (...) { }",
+              cxxCatchStmt(anyOf(hasDescendant(varDecl()), isCatchAll()))));
 }
 
 TEST(DeclarationMatcher, ClassIsDerived) {
@@ -460,19 +427,17 @@ TEST(DeclarationMatcher, ClassIsDerived) {
   EXPECT_TRUE(notMatches("class Y;", IsDerivedFromX));
   EXPECT_TRUE(notMatches("", IsDerivedFromX));
   EXPECT_TRUE(matches("class X {}; template<int N> class Y : Y<N-1>, X {};",
-    IsDerivedFromX));
+                      IsDerivedFromX));
   EXPECT_TRUE(matches("class X {}; template<int N> class Y : X, Y<N-1> {};",
-    IsDerivedFromX));
+                      IsDerivedFromX));
 
-  DeclarationMatcher IsZDerivedFromX = cxxRecordDecl(hasName("Z"),
-    isDerivedFrom("X"));
-  EXPECT_TRUE(
-    matches(
-      "class X {};"
-      "template<int N> class Y : Y<N-1> {};"
-      "template<> class Y<0> : X {};"
-      "class Z : Y<1> {};",
-      IsZDerivedFromX));
+  DeclarationMatcher IsZDerivedFromX =
+      cxxRecordDecl(hasName("Z"), isDerivedFrom("X"));
+  EXPECT_TRUE(matches("class X {};"
+                      "template<int N> class Y : Y<N-1> {};"
+                      "template<> class Y<0> : X {};"
+                      "class Z : Y<1> {};",
+                      IsZDerivedFromX));
 
   DeclarationMatcher IsDirectlyDerivedFromX =
       cxxRecordDecl(isDirectlyDerivedFrom("X"));
@@ -493,145 +458,138 @@ TEST(DeclarationMatcher, ClassIsDerived) {
   EXPECT_TRUE(notMatches("", IsAX));
 
   DeclarationMatcher ZIsDerivedFromX =
-    cxxRecordDecl(hasName("Z"), isDerivedFrom("X"));
+      cxxRecordDecl(hasName("Z"), isDerivedFrom("X"));
   DeclarationMatcher ZIsDirectlyDerivedFromX =
       cxxRecordDecl(hasName("Z"), isDirectlyDerivedFrom("X"));
   EXPECT_TRUE(
-    matches("class X {}; class Y : public X {}; class Z : public Y {};",
-            ZIsDerivedFromX));
+      matches("class X {}; class Y : public X {}; class Z : public Y {};",
+              ZIsDerivedFromX));
   EXPECT_TRUE(
       notMatches("class X {}; class Y : public X {}; class Z : public Y {};",
                  ZIsDirectlyDerivedFromX));
-  EXPECT_TRUE(
-    matches("class X {};"
-              "template<class T> class Y : public X {};"
-              "class Z : public Y<int> {};", ZIsDerivedFromX));
+  EXPECT_TRUE(matches("class X {};"
+                      "template<class T> class Y : public X {};"
+                      "class Z : public Y<int> {};",
+                      ZIsDerivedFromX));
   EXPECT_TRUE(notMatches("class X {};"
                          "template<class T> class Y : public X {};"
                          "class Z : public Y<int> {};",
                          ZIsDirectlyDerivedFromX));
   EXPECT_TRUE(matches("class X {}; template<class T> class Z : public X {};",
                       ZIsDerivedFromX));
+  EXPECT_TRUE(matches("template<class T> class X {}; "
+                      "template<class T> class Z : public X<T> {};",
+                      ZIsDerivedFromX));
+  EXPECT_TRUE(matches("template<class T, class U=T> class X {}; "
+                      "template<class T> class Z : public X<T> {};",
+                      ZIsDerivedFromX));
   EXPECT_TRUE(
-    matches("template<class T> class X {}; "
-              "template<class T> class Z : public X<T> {};",
-            ZIsDerivedFromX));
-  EXPECT_TRUE(
-    matches("template<class T, class U=T> class X {}; "
-              "template<class T> class Z : public X<T> {};",
-            ZIsDerivedFromX));
-  EXPECT_TRUE(
-    notMatches("template<class X> class A { class Z : public X {}; };",
-               ZIsDerivedFromX));
+      notMatches("template<class X> class A { class Z : public X {}; };",
+                 ZIsDerivedFromX));
   EXPECT_TRUE(
-    matches("template<class X> class A { public: class Z : public X {}; }; "
-              "class X{}; void y() { A<X>::Z z; }", ZIsDerivedFromX));
+      matches("template<class X> class A { public: class Z : public X {}; }; "
+              "class X{}; void y() { A<X>::Z z; }",
+              ZIsDerivedFromX));
   EXPECT_TRUE(
-    matches("template <class T> class X {}; "
+      matches("template <class T> class X {}; "
               "template<class Y> class A { class Z : public X<Y> {}; };",
-            ZIsDerivedFromX));
-  EXPECT_TRUE(
-    notMatches("template<template<class T> class X> class A { "
-                 "  class Z : public X<int> {}; };", ZIsDerivedFromX));
-  EXPECT_TRUE(
-    matches("template<template<class T> class X> class A { "
-              "  public: class Z : public X<int> {}; }; "
-              "template<class T> class X {}; void y() { A<X>::Z z; }",
-            ZIsDerivedFromX));
-  EXPECT_TRUE(
-    notMatches("template<class X> class A { class Z : public X::D {}; };",
-               ZIsDerivedFromX));
-  EXPECT_TRUE(
-    matches("template<class X> class A { public: "
-              "  class Z : public X::D {}; }; "
-              "class Y { public: class X {}; typedef X D; }; "
-              "void y() { A<Y>::Z z; }", ZIsDerivedFromX));
-  EXPECT_TRUE(
-    matches("class X {}; typedef X Y; class Z : public Y {};",
-            ZIsDerivedFromX));
+              ZIsDerivedFromX));
+  EXPECT_TRUE(notMatches("template<template<class T> class X> class A { "
+                         "  class Z : public X<int> {}; };",
+                         ZIsDerivedFromX));
+  EXPECT_TRUE(matches("template<template<class T> class X> class A { "
+                      "  public: class Z : public X<int> {}; }; "
+                      "template<class T> class X {}; void y() { A<X>::Z z; }",
+                      ZIsDerivedFromX));
   EXPECT_TRUE(
-    matches("template<class T> class Y { typedef typename T::U X; "
-              "  class Z : public X {}; };", ZIsDerivedFromX));
-  EXPECT_TRUE(matches("class X {}; class Z : public ::X {};",
+      notMatches("template<class X> class A { class Z : public X::D {}; };",
+                 ZIsDerivedFromX));
+  EXPECT_TRUE(matches("template<class X> class A { public: "
+                      "  class Z : public X::D {}; }; "
+                      "class Y { public: class X {}; typedef X D; }; "
+                      "void y() { A<Y>::Z z; }",
+                      ZIsDerivedFromX));
+  EXPECT_TRUE(matches("class X {}; typedef X Y; class Z : public Y {};",
                       ZIsDerivedFromX));
+  EXPECT_TRUE(matches("template<class T> class Y { typedef typename T::U X; "
+                      "  class Z : public X {}; };",
+                      ZIsDerivedFromX));
+  EXPECT_TRUE(matches("class X {}; class Z : public ::X {};", ZIsDerivedFromX));
   EXPECT_TRUE(
-    notMatches("template<class T> class X {}; "
+      notMatches("template<class T> class X {}; "
                  "template<class T> class A { class Z : public X<T>::D {}; };",
-               ZIsDerivedFromX));
+                 ZIsDerivedFromX));
   EXPECT_TRUE(
-    matches("template<class T> class X { public: typedef X<T> D; }; "
+      matches("template<class T> class X { public: typedef X<T> D; }; "
               "template<class T> class A { public: "
               "  class Z : public X<T>::D {}; }; void y() { A<int>::Z z; }",
-            ZIsDerivedFromX));
-  EXPECT_TRUE(
-    notMatches("template<class X> class A { class Z : public X::D::E {}; };",
-               ZIsDerivedFromX));
+              ZIsDerivedFromX));
   EXPECT_TRUE(
-    matches("class X {}; typedef X V; typedef V W; class Z : public W {};",
-            ZIsDerivedFromX));
+      notMatches("template<class X> class A { class Z : public X::D::E {}; };",
+                 ZIsDerivedFromX));
   EXPECT_TRUE(
-    matches("class X {}; class Y : public X {}; "
-              "typedef Y V; typedef V W; class Z : public W {};",
-            ZIsDerivedFromX));
+      matches("class X {}; typedef X V; typedef V W; class Z : public W {};",
+              ZIsDerivedFromX));
+  EXPECT_TRUE(matches("class X {}; class Y : public X {}; "
+                      "typedef Y V; typedef V W; class Z : public W {};",
+                      ZIsDerivedFromX));
   EXPECT_TRUE(notMatches("class X {}; class Y : public X {}; "
                          "typedef Y V; typedef V W; class Z : public W {};",
                          ZIsDirectlyDerivedFromX));
   EXPECT_TRUE(
-    matches("template<class T, class U> class X {}; "
+      matches("template<class T, class U> class X {}; "
               "template<class T> class A { class Z : public X<T, int> {}; };",
-            ZIsDerivedFromX));
+              ZIsDerivedFromX));
   EXPECT_TRUE(
-    notMatches("template<class X> class D { typedef X A; typedef A B; "
+      notMatches("template<class X> class D { typedef X A; typedef A B; "
                  "  typedef B C; class Z : public C {}; };",
-               ZIsDerivedFromX));
-  EXPECT_TRUE(
-    matches("class X {}; typedef X A; typedef A B; "
-              "class Z : public B {};", ZIsDerivedFromX));
-  EXPECT_TRUE(
-    matches("class X {}; typedef X A; typedef A B; typedef B C; "
-              "class Z : public C {};", ZIsDerivedFromX));
-  EXPECT_TRUE(
-    matches("class U {}; typedef U X; typedef X V; "
-              "class Z : public V {};", ZIsDerivedFromX));
-  EXPECT_TRUE(
-    matches("class Base {}; typedef Base X; "
-              "class Z : public Base {};", ZIsDerivedFromX));
-  EXPECT_TRUE(
-    matches("class Base {}; typedef Base Base2; typedef Base2 X; "
-              "class Z : public Base {};", ZIsDerivedFromX));
-  EXPECT_TRUE(
-    notMatches("class Base {}; class Base2 {}; typedef Base2 X; "
-                 "class Z : public Base {};", ZIsDerivedFromX));
-  EXPECT_TRUE(
-    matches("class A {}; typedef A X; typedef A Y; "
-              "class Z : public Y {};", ZIsDerivedFromX));
-  EXPECT_TRUE(
-    notMatches("template <typename T> class Z;"
-                 "template <> class Z<void> {};"
-                 "template <typename T> class Z : public Z<void> {};",
-               IsDerivedFromX));
-  EXPECT_TRUE(
-    matches("template <typename T> class X;"
-              "template <> class X<void> {};"
-              "template <typename T> class X : public X<void> {};",
-            IsDerivedFromX));
-  EXPECT_TRUE(matches(
-    "class X {};"
-      "template <typename T> class Z;"
-      "template <> class Z<void> {};"
-      "template <typename T> class Z : public Z<void>, public X {};",
-    ZIsDerivedFromX));
-  EXPECT_TRUE(
-    notMatches("template<int> struct X;"
+                 ZIsDerivedFromX));
+  EXPECT_TRUE(matches("class X {}; typedef X A; typedef A B; "
+                      "class Z : public B {};",
+                      ZIsDerivedFromX));
+  EXPECT_TRUE(matches("class X {}; typedef X A; typedef A B; typedef B C; "
+                      "class Z : public C {};",
+                      ZIsDerivedFromX));
+  EXPECT_TRUE(matches("class U {}; typedef U X; typedef X V; "
+                      "class Z : public V {};",
+                      ZIsDerivedFromX));
+  EXPECT_TRUE(matches("class Base {}; typedef Base X; "
+                      "class Z : public Base {};",
+                      ZIsDerivedFromX));
+  EXPECT_TRUE(matches("class Base {}; typedef Base Base2; typedef Base2 X; "
+                      "class Z : public Base {};",
+                      ZIsDerivedFromX));
+  EXPECT_TRUE(notMatches("class Base {}; class Base2 {}; typedef Base2 X; "
+                         "class Z : public Base {};",
+                         ZIsDerivedFromX));
+  EXPECT_TRUE(matches("class A {}; typedef A X; typedef A Y; "
+                      "class Z : public Y {};",
+                      ZIsDerivedFromX));
+  EXPECT_TRUE(notMatches("template <typename T> class Z;"
+                         "template <> class Z<void> {};"
+                         "template <typename T> class Z : public Z<void> {};",
+                         IsDerivedFromX));
+  EXPECT_TRUE(matches("template <typename T> class X;"
+                      "template <> class X<void> {};"
+                      "template <typename T> class X : public X<void> {};",
+                      IsDerivedFromX));
+  EXPECT_TRUE(
+      matches("class X {};"
+              "template <typename T> class Z;"
+              "template <> class Z<void> {};"
+              "template <typename T> class Z : public Z<void>, public X {};",
+              ZIsDerivedFromX));
+  EXPECT_TRUE(
+      notMatches("template<int> struct X;"
                  "template<int i> struct X : public X<i-1> {};",
-               cxxRecordDecl(isDerivedFrom(recordDecl(hasName("Some"))))));
+                 cxxRecordDecl(isDerivedFrom(recordDecl(hasName("Some"))))));
   EXPECT_TRUE(matches(
-    "struct A {};"
+      "struct A {};"
       "template<int> struct X;"
       "template<int i> struct X : public X<i-1> {};"
       "template<> struct X<0> : public A {};"
       "struct B : public X<42> {};",
-    cxxRecordDecl(hasName("B"), isDerivedFrom(recordDecl(hasName("A"))))));
+      cxxRecordDecl(hasName("B"), isDerivedFrom(recordDecl(hasName("A"))))));
   EXPECT_TRUE(notMatches(
       "struct A {};"
       "template<int> struct X;"
@@ -645,7 +603,7 @@ TEST(DeclarationMatcher, ClassIsDerived) {
   // get rid of the Variable(...) matching and match the right template
   // declarations directly.
   const char *RecursiveTemplateOneParameter =
-    "class Base1 {}; class Base2 {};"
+      "class Base1 {}; class Base2 {};"
       "template <typename T> class Z;"
       "template <> class Z<void> : public Base1 {};"
       "template <> class Z<int> : public Base2 {};"
@@ -654,21 +612,21 @@ TEST(DeclarationMatcher, ClassIsDerived) {
       "template <typename T> class Z : public Z<float>, public Z<double> {};"
       "void f() { Z<float> z_float; Z<double> z_double; Z<char> z_char; }";
   EXPECT_TRUE(matches(
-    RecursiveTemplateOneParameter,
-    varDecl(hasName("z_float"),
-            hasInitializer(hasType(cxxRecordDecl(isDerivedFrom("Base1")))))));
+      RecursiveTemplateOneParameter,
+      varDecl(hasName("z_float"),
+              hasInitializer(hasType(cxxRecordDecl(isDerivedFrom("Base1")))))));
   EXPECT_TRUE(notMatches(
-    RecursiveTemplateOneParameter,
-    varDecl(hasName("z_float"),
-            hasInitializer(hasType(cxxRecordDecl(isDerivedFrom("Base2")))))));
-  EXPECT_TRUE(matches(
-    RecursiveTemplateOneParameter,
-    varDecl(hasName("z_char"),
-            hasInitializer(hasType(cxxRecordDecl(isDerivedFrom("Base1"),
-                                                 isDerivedFrom("Base2")))))));
+      RecursiveTemplateOneParameter,
+      varDecl(hasName("z_float"),
+              hasInitializer(hasType(cxxRecordDecl(isDerivedFrom("Base2")))))));
+  EXPECT_TRUE(
+      matches(RecursiveTemplateOneParameter,
+              varDecl(hasName("z_char"),
+                      hasInitializer(hasType(cxxRecordDecl(
+                          isDerivedFrom("Base1"), isDerivedFrom("Base2")))))));
 
   const char *RecursiveTemplateTwoParameters =
-    "class Base1 {}; class Base2 {};"
+      "class Base1 {}; class Base2 {};"
       "template <typename T1, typename T2> class Z;"
       "template <typename T> class Z<void, T> : public Base1 {};"
       "template <typename T> class Z<int, T> : public Base2 {};"
@@ -679,34 +637,31 @@ TEST(DeclarationMatcher, ClassIsDerived) {
       "void f() { Z<float, void> z_float; Z<double, void> z_double; "
       "           Z<char, void> z_char; }";
   EXPECT_TRUE(matches(
-    RecursiveTemplateTwoParameters,
-    varDecl(hasName("z_float"),
-            hasInitializer(hasType(cxxRecordDecl(isDerivedFrom("Base1")))))));
+      RecursiveTemplateTwoParameters,
+      varDecl(hasName("z_float"),
+              hasInitializer(hasType(cxxRecordDecl(isDerivedFrom("Base1")))))));
   EXPECT_TRUE(notMatches(
-    RecursiveTemplateTwoParameters,
-    varDecl(hasName("z_float"),
-            hasInitializer(hasType(cxxRecordDecl(isDerivedFrom("Base2")))))));
-  EXPECT_TRUE(matches(
-    RecursiveTemplateTwoParameters,
-    varDecl(hasName("z_char"),
-            hasInitializer(hasType(cxxRecordDecl(isDerivedFrom("Base1"),
-                                                 isDerivedFrom("Base2")))))));
-  EXPECT_TRUE(matches(
-    "namespace ns { class X {}; class Y : public X {}; }",
-    cxxRecordDecl(isDerivedFrom("::ns::X"))));
-  EXPECT_TRUE(notMatches(
-    "class X {}; class Y : public X {};",
-    cxxRecordDecl(isDerivedFrom("::ns::X"))));
+      RecursiveTemplateTwoParameters,
+      varDecl(hasName("z_float"),
+              hasInitializer(hasType(cxxRecordDecl(isDerivedFrom("Base2")))))));
+  EXPECT_TRUE(
+      matches(RecursiveTemplateTwoParameters,
+              varDecl(hasName("z_char"),
+                      hasInitializer(hasType(cxxRecordDecl(
+                          isDerivedFrom("Base1"), isDerivedFrom("Base2")))))));
+  EXPECT_TRUE(matches("namespace ns { class X {}; class Y : public X {}; }",
+                      cxxRecordDecl(isDerivedFrom("::ns::X"))));
+  EXPECT_TRUE(notMatches("class X {}; class Y : public X {};",
+                         cxxRecordDecl(isDerivedFrom("::ns::X"))));
 
   EXPECT_TRUE(matches(
-    "class X {}; class Y : public X {};",
-    cxxRecordDecl(isDerivedFrom(recordDecl(hasName("X")).bind("test")))));
+      "class X {}; class Y : public X {};",
+      cxxRecordDecl(isDerivedFrom(recordDecl(hasName("X")).bind("test")))));
 
-  EXPECT_TRUE(matches(
-    "template<typename T> class X {};"
-      "template<typename T> using Z = X<T>;"
-      "template <typename T> class Y : Z<T> {};",
-    cxxRecordDecl(isDerivedFrom(namedDecl(hasName("X"))))));
+  EXPECT_TRUE(matches("template<typename T> class X {};"
+                      "template<typename T> using Z = X<T>;"
+                      "template <typename T> class Y : Z<T> {};",
+                      cxxRecordDecl(isDerivedFrom(namedDecl(hasName("X"))))));
 }
 
 TEST(DeclarationMatcher, IsDerivedFromEmptyName) {
@@ -737,24 +692,24 @@ TEST(DeclarationMatcher, ObjCClassIsDerived) {
 
   DeclarationMatcher IsDirectlyDerivedFromX =
       objcInterfaceDecl(isDirectlyDerivedFrom("X"));
-  EXPECT_TRUE(
-      matchesObjC("@interface X @end @interface Y : X @end", IsDirectlyDerivedFromX));
+  EXPECT_TRUE(matchesObjC("@interface X @end @interface Y : X @end",
+                          IsDirectlyDerivedFromX));
   EXPECT_TRUE(matchesObjC(
       "@interface X @end @interface Y<__covariant ObjectType> : X @end",
       IsDirectlyDerivedFromX));
   EXPECT_TRUE(matchesObjC(
       "@interface X @end @compatibility_alias Y X; @interface Z : Y @end",
       IsDirectlyDerivedFromX));
-  EXPECT_TRUE(matchesObjC(
-      "@interface X @end typedef X Y; @interface Z : Y @end",
-      IsDirectlyDerivedFromX));
+  EXPECT_TRUE(
+      matchesObjC("@interface X @end typedef X Y; @interface Z : Y @end",
+                  IsDirectlyDerivedFromX));
   EXPECT_TRUE(notMatchesObjC("@interface X @end", IsDirectlyDerivedFromX));
   EXPECT_TRUE(notMatchesObjC("@class X;", IsDirectlyDerivedFromX));
   EXPECT_TRUE(notMatchesObjC("@class Y;", IsDirectlyDerivedFromX));
   EXPECT_TRUE(notMatchesObjC("@interface X @end @compatibility_alias Y X;",
                              IsDirectlyDerivedFromX));
-  EXPECT_TRUE(notMatchesObjC("@interface X @end typedef X Y;",
-                             IsDirectlyDerivedFromX));
+  EXPECT_TRUE(
+      notMatchesObjC("@interface X @end typedef X Y;", IsDirectlyDerivedFromX));
 
   DeclarationMatcher IsAX = objcInterfaceDecl(isSameOrDerivedFrom("X"));
   EXPECT_TRUE(matchesObjC("@interface X @end @interface Y : X @end", IsAX));
@@ -775,9 +730,9 @@ TEST(DeclarationMatcher, ObjCClassIsDerived) {
                           ZIsDerivedFromX));
   EXPECT_TRUE(matchesObjC(
       "@interface X @end typedef X Y; @interface Z : Y @end", ZIsDerivedFromX));
-  EXPECT_TRUE(matchesObjC(
-      "@interface X @end typedef X Y; @interface Z : Y @end",
-      ZIsDirectlyDerivedFromX));
+  EXPECT_TRUE(
+      matchesObjC("@interface X @end typedef X Y; @interface Z : Y @end",
+                  ZIsDirectlyDerivedFromX));
   EXPECT_TRUE(matchesObjC(
       "@interface A @end typedef A X; typedef A Y; @interface Z : Y @end",
       ZIsDerivedFromX));
@@ -798,27 +753,33 @@ TEST(DeclarationMatcher, ObjCClassIsDerived) {
       ZIsDirectlyDerivedFromX));
   EXPECT_TRUE(matchesObjC(
       "@interface A @end @compatibility_alias X A; @compatibility_alias Y A;"
-      "@interface Z : Y @end", ZIsDerivedFromX));
+      "@interface Z : Y @end",
+      ZIsDerivedFromX));
   EXPECT_TRUE(matchesObjC(
       "@interface A @end @compatibility_alias X A; @compatibility_alias Y A;"
-      "@interface Z : Y @end", ZIsDirectlyDerivedFromX));
-  EXPECT_TRUE(matchesObjC(
-      "@interface Y @end typedef Y X; @interface Z : X @end", ZIsDerivedFromX));
-  EXPECT_TRUE(matchesObjC(
-      "@interface Y @end typedef Y X; @interface Z : X @end",
+      "@interface Z : Y @end",
       ZIsDirectlyDerivedFromX));
   EXPECT_TRUE(matchesObjC(
-      "@interface A @end @compatibility_alias Y A; typedef Y X;"
-      "@interface Z : A @end", ZIsDerivedFromX));
-  EXPECT_TRUE(matchesObjC(
-      "@interface A @end @compatibility_alias Y A; typedef Y X;"
-      "@interface Z : A @end", ZIsDirectlyDerivedFromX));
-  EXPECT_TRUE(matchesObjC(
-      "@interface A @end typedef A Y; @compatibility_alias X Y;"
-      "@interface Z : A @end", ZIsDerivedFromX));
-  EXPECT_TRUE(matchesObjC(
-      "@interface A @end typedef A Y; @compatibility_alias X Y;"
-      "@interface Z : A @end", ZIsDirectlyDerivedFromX));
+      "@interface Y @end typedef Y X; @interface Z : X @end", ZIsDerivedFromX));
+  EXPECT_TRUE(
+      matchesObjC("@interface Y @end typedef Y X; @interface Z : X @end",
+                  ZIsDirectlyDerivedFromX));
+  EXPECT_TRUE(
+      matchesObjC("@interface A @end @compatibility_alias Y A; typedef Y X;"
+                  "@interface Z : A @end",
+                  ZIsDerivedFromX));
+  EXPECT_TRUE(
+      matchesObjC("@interface A @end @compatibility_alias Y A; typedef Y X;"
+                  "@interface Z : A @end",
+                  ZIsDirectlyDerivedFromX));
+  EXPECT_TRUE(
+      matchesObjC("@interface A @end typedef A Y; @compatibility_alias X Y;"
+                  "@interface Z : A @end",
+                  ZIsDerivedFromX));
+  EXPECT_TRUE(
+      matchesObjC("@interface A @end typedef A Y; @compatibility_alias X Y;"
+                  "@interface Z : A @end",
+                  ZIsDirectlyDerivedFromX));
 }
 
 TEST(DeclarationMatcher, IsLambda) {
@@ -830,42 +791,41 @@ TEST(DeclarationMatcher, IsLambda) {
 TEST(Matcher, BindMatchedNodes) {
   DeclarationMatcher ClassX = has(recordDecl(hasName("::X")).bind("x"));
 
-  EXPECT_TRUE(matchAndVerifyResultTrue("class X {};",
-                                       ClassX, std::make_unique<VerifyIdIsBoundTo<CXXRecordDecl>>("x")));
+  EXPECT_TRUE(matchAndVerifyResultTrue(
+      "class X {};", ClassX,
+      std::make_unique<VerifyIdIsBoundTo<CXXRecordDecl>>("x")));
 
-  EXPECT_TRUE(matchAndVerifyResultFalse("class X {};",
-                                        ClassX, std::make_unique<VerifyIdIsBoundTo<CXXRecordDecl>>("other-id")));
+  EXPECT_TRUE(matchAndVerifyResultFalse(
+      "class X {};", ClassX,
+      std::make_unique<VerifyIdIsBoundTo<CXXRecordDecl>>("other-id")));
 
   TypeMatcher TypeAHasClassB = hasDeclaration(
-    recordDecl(hasName("A"), has(recordDecl(hasName("B")).bind("b"))));
+      recordDecl(hasName("A"), has(recordDecl(hasName("B")).bind("b"))));
 
-  EXPECT_TRUE(matchAndVerifyResultTrue("class A { public: A *a; class B {}; };",
-                                       TypeAHasClassB,
-                                       std::make_unique<VerifyIdIsBoundTo<Decl>>("b")));
+  EXPECT_TRUE(matchAndVerifyResultTrue(
+      "class A { public: A *a; class B {}; };", TypeAHasClassB,
+      std::make_unique<VerifyIdIsBoundTo<Decl>>("b")));
 
   StatementMatcher MethodX =
-    callExpr(callee(cxxMethodDecl(hasName("x")))).bind("x");
+      callExpr(callee(cxxMethodDecl(hasName("x")))).bind("x");
 
-  EXPECT_TRUE(matchAndVerifyResultTrue("class A { void x() { x(); } };",
-                                       MethodX,
-                                       std::make_unique<VerifyIdIsBoundTo<CXXMemberCallExpr>>("x")));
+  EXPECT_TRUE(matchAndVerifyResultTrue(
+      "class A { void x() { x(); } };", MethodX,
+      std::make_unique<VerifyIdIsBoundTo<CXXMemberCallExpr>>("x")));
 }
 
 TEST(Matcher, BindTheSameNameInAlternatives) {
   StatementMatcher matcher = anyOf(
-    binaryOperator(hasOperatorName("+"),
-                   hasLHS(expr().bind("x")),
-                   hasRHS(integerLiteral(equals(0)))),
-    binaryOperator(hasOperatorName("+"),
-                   hasLHS(integerLiteral(equals(0))),
-                   hasRHS(expr().bind("x"))));
+      binaryOperator(hasOperatorName("+"), hasLHS(expr().bind("x")),
+                     hasRHS(integerLiteral(equals(0)))),
+      binaryOperator(hasOperatorName("+"), hasLHS(integerLiteral(equals(0))),
+                     hasRHS(expr().bind("x"))));
 
   EXPECT_TRUE(matchAndVerifyResultTrue(
-    // The first branch of the matcher binds x to 0 but then fails.
-    // The second branch binds x to f() and succeeds.
-    "int f() { return 0 + f(); }",
-    matcher,
-    std::make_unique<VerifyIdIsBoundTo<CallExpr>>("x")));
+      // The first branch of the matcher binds x to 0 but then fails.
+      // The second branch binds x to f() and succeeds.
+      "int f() { return 0 + f(); }", matcher,
+      std::make_unique<VerifyIdIsBoundTo<CallExpr>>("x")));
 }
 
 TEST(Matcher, BindsIDForMemoizedResults) {
@@ -873,48 +833,48 @@ TEST(Matcher, BindsIDForMemoizedResults) {
   // kick in.
   DeclarationMatcher ClassX = recordDecl(hasName("X")).bind("x");
   EXPECT_TRUE(matchAndVerifyResultTrue(
-    "class A { class B { class X {}; }; };",
-    DeclarationMatcher(anyOf(
-      recordDecl(hasName("A"), hasDescendant(ClassX)),
-      recordDecl(hasName("B"), hasDescendant(ClassX)))),
-    std::make_unique<VerifyIdIsBoundTo<Decl>>("x", 2)));
+      "class A { class B { class X {}; }; };",
+      DeclarationMatcher(
+          anyOf(recordDecl(hasName("A"), hasDescendant(ClassX)),
+                recordDecl(hasName("B"), hasDescendant(ClassX)))),
+      std::make_unique<VerifyIdIsBoundTo<Decl>>("x", 2)));
 }
 
 TEST(HasType, MatchesAsString) {
   EXPECT_TRUE(
-    matches("class Y { public: void x(); }; void z() {Y* y; y->x(); }",
-            cxxMemberCallExpr(on(hasType(asString("class Y *"))))));
+      matches("class Y { public: void x(); }; void z() {Y* y; y->x(); }",
+              cxxMemberCallExpr(on(hasType(asString("class Y *"))))));
   EXPECT_TRUE(
-    matches("class X { void x(int x) {} };",
-            cxxMethodDecl(hasParameter(0, hasType(asString("int"))))));
+      matches("class X { void x(int x) {} };",
+              cxxMethodDecl(hasParameter(0, hasType(asString("int"))))));
   EXPECT_TRUE(matches("namespace ns { struct A {}; }  struct B { ns::A a; };",
                       fieldDecl(hasType(asString("ns::A")))));
-  EXPECT_TRUE(matches("namespace { struct A {}; }  struct B { A a; };",
-                      fieldDecl(hasType(asString("struct (anonymous namespace)::A")))));
+  EXPECT_TRUE(
+      matches("namespace { struct A {}; }  struct B { A a; };",
+              fieldDecl(hasType(asString("struct (anonymous namespace)::A")))));
 }
 
 TEST(Matcher, HasOperatorNameForOverloadedOperatorCall) {
   StatementMatcher OpCallAndAnd =
-    cxxOperatorCallExpr(hasOverloadedOperatorName("&&"));
+      cxxOperatorCallExpr(hasOverloadedOperatorName("&&"));
   EXPECT_TRUE(matches("class Y { }; "
-                        "bool operator&&(Y x, Y y) { return true; }; "
-                        "Y a; Y b; bool c = a && b;", OpCallAndAnd));
+                      "bool operator&&(Y x, Y y) { return true; }; "
+                      "Y a; Y b; bool c = a && b;",
+                      OpCallAndAnd));
   StatementMatcher OpCallLessLess =
-    cxxOperatorCallExpr(hasOverloadedOperatorName("<<"));
+      cxxOperatorCallExpr(hasOverloadedOperatorName("<<"));
   EXPECT_TRUE(notMatches("class Y { }; "
-                           "bool operator&&(Y x, Y y) { return true; }; "
-                           "Y a; Y b; bool c = a && b;",
+                         "bool operator&&(Y x, Y y) { return true; }; "
+                         "Y a; Y b; bool c = a && b;",
                          OpCallLessLess));
   StatementMatcher OpStarCall =
-    cxxOperatorCallExpr(hasOverloadedOperatorName("*"));
-  EXPECT_TRUE(matches("class Y; int operator*(Y &); void f(Y &y) { *y; }",
-                      OpStarCall));
+      cxxOperatorCallExpr(hasOverloadedOperatorName("*"));
+  EXPECT_TRUE(
+      matches("class Y; int operator*(Y &); void f(Y &y) { *y; }", OpStarCall));
   DeclarationMatcher ClassWithOpStar =
-    cxxRecordDecl(hasMethod(hasOverloadedOperatorName("*")));
-  EXPECT_TRUE(matches("class Y { int operator*(); };",
-                      ClassWithOpStar));
-  EXPECT_TRUE(notMatches("class Y { void myOperator(); };",
-                         ClassWithOpStar)) ;
+      cxxRecordDecl(hasMethod(hasOverloadedOperatorName("*")));
+  EXPECT_TRUE(matches("class Y { int operator*(); };", ClassWithOpStar));
+  EXPECT_TRUE(notMatches("class Y { void myOperator(); };", ClassWithOpStar));
   DeclarationMatcher AnyOpStar = functionDecl(hasOverloadedOperatorName("*"));
   EXPECT_TRUE(matches("class Y; int operator*(Y &);", AnyOpStar));
   EXPECT_TRUE(matches("class Y { int operator*(); };", AnyOpStar));
@@ -926,23 +886,22 @@ TEST(Matcher, HasOperatorNameForOverloadedOperatorCall) {
   EXPECT_TRUE(matches("class Y { Y operator&&(Y &); };", AnyAndOp));
 }
 
-
 TEST(Matcher, NestedOverloadedOperatorCalls) {
   EXPECT_TRUE(matchAndVerifyResultTrue(
-    "class Y { }; "
+      "class Y { }; "
       "Y& operator&&(Y& x, Y& y) { return x; }; "
       "Y a; Y b; Y c; Y d = a && b && c;",
-    cxxOperatorCallExpr(hasOverloadedOperatorName("&&")).bind("x"),
-    std::make_unique<VerifyIdIsBoundTo<CXXOperatorCallExpr>>("x", 2)));
+      cxxOperatorCallExpr(hasOverloadedOperatorName("&&")).bind("x"),
+      std::make_unique<VerifyIdIsBoundTo<CXXOperatorCallExpr>>("x", 2)));
   EXPECT_TRUE(matches("class Y { }; "
-                        "Y& operator&&(Y& x, Y& y) { return x; }; "
-                        "Y a; Y b; Y c; Y d = a && b && c;",
+                      "Y& operator&&(Y& x, Y& y) { return x; }; "
+                      "Y a; Y b; Y c; Y d = a && b && c;",
                       cxxOperatorCallExpr(hasParent(cxxOperatorCallExpr()))));
   EXPECT_TRUE(
-    matches("class Y { }; "
+      matches("class Y { }; "
               "Y& operator&&(Y& x, Y& y) { return x; }; "
               "Y a; Y b; Y c; Y d = a && b && c;",
-            cxxOperatorCallExpr(hasDescendant(cxxOperatorCallExpr()))));
+              cxxOperatorCallExpr(hasDescendant(cxxOperatorCallExpr()))));
 }
 
 TEST(Matcher, VarDecl_Storage) {
@@ -971,9 +930,9 @@ TEST(Matcher, VarDecl_StorageDuration) {
 
   EXPECT_TRUE(matches(T, varDecl(hasName("x"), hasAutomaticStorageDuration())));
   EXPECT_TRUE(
-    notMatches(T, varDecl(hasName("y"), hasAutomaticStorageDuration())));
+      notMatches(T, varDecl(hasName("y"), hasAutomaticStorageDuration())));
   EXPECT_TRUE(
-    notMatches(T, varDecl(hasName("a"), hasAutomaticStorageDuration())));
+      notMatches(T, varDecl(hasName("a"), hasAutomaticStorageDuration())));
 
   EXPECT_TRUE(matches(T, varDecl(hasName("y"), hasStaticStorageDuration())));
   EXPECT_TRUE(matches(T, varDecl(hasName("a"), hasStaticStorageDuration())));
@@ -991,48 +950,48 @@ TEST(Matcher, VarDecl_StorageDuration) {
 }
 
 TEST(Matcher, FindsVarDeclInFunctionParameter) {
-  EXPECT_TRUE(matches(
-    "void f(int i) {}",
-    varDecl(hasName("i"))));
+  EXPECT_TRUE(matches("void f(int i) {}", varDecl(hasName("i"))));
 }
 
 TEST(UnaryExpressionOrTypeTraitExpression, MatchesCorrectType) {
-  EXPECT_TRUE(matches("void x() { int a = sizeof(a); }", sizeOfExpr(
-    hasArgumentOfType(asString("int")))));
-  EXPECT_TRUE(notMatches("void x() { int a = sizeof(a); }", sizeOfExpr(
-    hasArgumentOfType(asString("float")))));
+  EXPECT_TRUE(matches("void x() { int a = sizeof(a); }",
+                      sizeOfExpr(hasArgumentOfType(asString("int")))));
+  EXPECT_TRUE(notMatches("void x() { int a = sizeof(a); }",
+                         sizeOfExpr(hasArgumentOfType(asString("float")))));
   EXPECT_TRUE(matches(
-    "struct A {}; void x() { A a; int b = sizeof(a); }",
-    sizeOfExpr(hasArgumentOfType(hasDeclaration(recordDecl(hasName("A")))))));
-  EXPECT_TRUE(notMatches("void x() { int a = sizeof(a); }", sizeOfExpr(
-    hasArgumentOfType(hasDeclaration(recordDecl(hasName("string")))))));
+      "struct A {}; void x() { A a; int b = sizeof(a); }",
+      sizeOfExpr(hasArgumentOfType(hasDeclaration(recordDecl(hasName("A")))))));
+  EXPECT_TRUE(notMatches("void x() { int a = sizeof(a); }",
+                         sizeOfExpr(hasArgumentOfType(
+                             hasDeclaration(recordDecl(hasName("string")))))));
 }
 
 TEST(IsInteger, MatchesIntegers) {
   EXPECT_TRUE(matches("int i = 0;", varDecl(hasType(isInteger()))));
-  EXPECT_TRUE(matches(
-    "long long i = 0; void f(long long) { }; void g() {f(i);}",
-    callExpr(hasArgument(0, declRefExpr(
-      to(varDecl(hasType(isInteger()))))))));
+  EXPECT_TRUE(
+      matches("long long i = 0; void f(long long) { }; void g() {f(i);}",
+              callExpr(hasArgument(
+                  0, declRefExpr(to(varDecl(hasType(isInteger()))))))));
 }
 
 TEST(IsInteger, ReportsNoFalsePositives) {
   EXPECT_TRUE(notMatches("int *i;", varDecl(hasType(isInteger()))));
-  EXPECT_TRUE(notMatches("struct T {}; T t; void f(T *) { }; void g() {f(&t);}",
-                         callExpr(hasArgument(0, declRefExpr(
-                           to(varDecl(hasType(isInteger()))))))));
+  EXPECT_TRUE(
+      notMatches("struct T {}; T t; void f(T *) { }; void g() {f(&t);}",
+                 callExpr(hasArgument(
+                     0, declRefExpr(to(varDecl(hasType(isInteger()))))))));
 }
 
 TEST(IsSignedInteger, MatchesSignedIntegers) {
   EXPECT_TRUE(matches("int i = 0;", varDecl(hasType(isSignedInteger()))));
-  EXPECT_TRUE(notMatches("unsigned i = 0;",
-                         varDecl(hasType(isSignedInteger()))));
+  EXPECT_TRUE(
+      notMatches("unsigned i = 0;", varDecl(hasType(isSignedInteger()))));
 }
 
 TEST(IsUnsignedInteger, MatchesUnsignedIntegers) {
   EXPECT_TRUE(notMatches("int i = 0;", varDecl(hasType(isUnsignedInteger()))));
-  EXPECT_TRUE(matches("unsigned i = 0;",
-                      varDecl(hasType(isUnsignedInteger()))));
+  EXPECT_TRUE(
+      matches("unsigned i = 0;", varDecl(hasType(isUnsignedInteger()))));
 }
 
 TEST(IsAnyPointer, MatchesPointers) {
@@ -1059,8 +1018,8 @@ TEST(IsAnyCharacter, ReportsNoFalsePositives) {
 TEST(IsArrow, MatchesMemberVariablesViaArrow) {
   EXPECT_TRUE(matches("class Y { void x() { this->y; } int y; };",
                       memberExpr(isArrow())));
-  EXPECT_TRUE(matches("class Y { void x() { y; } int y; };",
-                      memberExpr(isArrow())));
+  EXPECT_TRUE(
+      matches("class Y { void x() { y; } int y; };", memberExpr(isArrow())));
   EXPECT_TRUE(notMatches("class Y { void x() { (*this).y; } int y; };",
                          memberExpr(isArrow())));
   EXPECT_TRUE(matches("template <class T> class Y { void x() { this->m; } };",
@@ -1080,10 +1039,9 @@ TEST(IsArrow, MatchesStaticMemberVariablesViaArrow) {
 }
 
 TEST(IsArrow, MatchesMemberCallsViaArrow) {
-  EXPECT_TRUE(matches("class Y { void x() { this->x(); } };",
-                      memberExpr(isArrow())));
-  EXPECT_TRUE(matches("class Y { void x() { x(); } };",
-                      memberExpr(isArrow())));
+  EXPECT_TRUE(
+      matches("class Y { void x() { this->x(); } };", memberExpr(isArrow())));
+  EXPECT_TRUE(matches("class Y { void x() { x(); } };", memberExpr(isArrow())));
   EXPECT_TRUE(notMatches("class Y { void x() { Y y; y.x(); } };",
                          memberExpr(isArrow())));
   EXPECT_TRUE(
@@ -1128,20 +1086,18 @@ TEST(Matcher, ParameterCount) {
 }
 
 TEST(Matcher, References) {
-  DeclarationMatcher ReferenceClassX = varDecl(
-    hasType(references(recordDecl(hasName("X")))));
-  EXPECT_TRUE(matches("class X {}; void y(X y) { X &x = y; }",
-                      ReferenceClassX));
+  DeclarationMatcher ReferenceClassX =
+      varDecl(hasType(references(recordDecl(hasName("X")))));
   EXPECT_TRUE(
-    matches("class X {}; void y(X y) { const X &x = y; }", ReferenceClassX));
+      matches("class X {}; void y(X y) { X &x = y; }", ReferenceClassX));
+  EXPECT_TRUE(
+      matches("class X {}; void y(X y) { const X &x = y; }", ReferenceClassX));
   // The match here is on the implicit copy constructor code for
   // class X, not on code 'X x = y'.
+  EXPECT_TRUE(matches("class X {}; void y(X y) { X x = y; }", ReferenceClassX));
+  EXPECT_TRUE(notMatches("class X {}; extern X x;", ReferenceClassX));
   EXPECT_TRUE(
-    matches("class X {}; void y(X y) { X x = y; }", ReferenceClassX));
-  EXPECT_TRUE(
-    notMatches("class X {}; extern X x;", ReferenceClassX));
-  EXPECT_TRUE(
-    notMatches("class X {}; void y(X *y) { X *&x = y; }", ReferenceClassX));
+      notMatches("class X {}; void y(X *y) { X *&x = y; }", ReferenceClassX));
 }
 
 TEST(QualType, hasLocalQualifiers) {
@@ -1149,16 +1105,15 @@ TEST(QualType, hasLocalQualifiers) {
                          varDecl(hasType(hasLocalQualifiers()))));
   EXPECT_TRUE(matches("int *const j = nullptr;",
                       varDecl(hasType(hasLocalQualifiers()))));
-  EXPECT_TRUE(matches("int *volatile k;",
-                      varDecl(hasType(hasLocalQualifiers()))));
-  EXPECT_TRUE(notMatches("int m;",
-                         varDecl(hasType(hasLocalQualifiers()))));
+  EXPECT_TRUE(
+      matches("int *volatile k;", varDecl(hasType(hasLocalQualifiers()))));
+  EXPECT_TRUE(notMatches("int m;", varDecl(hasType(hasLocalQualifiers()))));
 }
 
 TEST(IsExternC, MatchesExternCFunctionDeclarations) {
   EXPECT_TRUE(matches("extern \"C\" void f() {}", functionDecl(isExternC())));
-  EXPECT_TRUE(matches("extern \"C\" { void f() {} }",
-                      functionDecl(isExternC())));
+  EXPECT_TRUE(
+      matches("extern \"C\" { void f() {} }", functionDecl(isExternC())));
   EXPECT_TRUE(notMatches("void f() {}", functionDecl(isExternC())));
 }
 
@@ -1186,7 +1141,7 @@ TEST(IsDefaulted, MatchesDefaultedFunctionDeclarations) {
 
 TEST(IsDeleted, MatchesDeletedFunctionDeclarations) {
   EXPECT_TRUE(
-    notMatches("void Func();", functionDecl(hasName("Func"), isDeleted())));
+      notMatches("void Func();", functionDecl(hasName("Func"), isDeleted())));
   EXPECT_TRUE(matches("void Func() = delete;",
                       functionDecl(hasName("Func"), isDeleted())));
 }
@@ -1195,14 +1150,15 @@ TEST(IsNoThrow, MatchesNoThrowFunctionDeclarations) {
   EXPECT_TRUE(notMatches("void f();", functionDecl(isNoThrow())));
   EXPECT_TRUE(notMatches("void f() throw(int);", functionDecl(isNoThrow())));
   EXPECT_TRUE(
-    notMatches("void f() noexcept(false);", functionDecl(isNoThrow())));
+      notMatches("void f() noexcept(false);", functionDecl(isNoThrow())));
   EXPECT_TRUE(matches("void f() throw();", functionDecl(isNoThrow())));
   EXPECT_TRUE(matches("void f() noexcept;", functionDecl(isNoThrow())));
 
   EXPECT_TRUE(notMatches("void f();", functionProtoType(isNoThrow())));
-  EXPECT_TRUE(notMatches("void f() throw(int);", functionProtoType(isNoThrow())));
   EXPECT_TRUE(
-    notMatches("void f() noexcept(false);", functionProtoType(isNoThrow())));
+      notMatches("void f() throw(int);", functionProtoType(isNoThrow())));
+  EXPECT_TRUE(
+      notMatches("void f() noexcept(false);", functionProtoType(isNoThrow())));
   EXPECT_TRUE(matches("void f() throw();", functionProtoType(isNoThrow())));
   EXPECT_TRUE(matches("void f() noexcept;", functionProtoType(isNoThrow())));
 }
@@ -1249,41 +1205,41 @@ TEST(hasInitStatement, MatchesRangeForInitializers) {
 
 TEST(TemplateArgumentCountIs, Matches) {
   EXPECT_TRUE(
-    matches("template<typename T> struct C {}; C<int> c;",
-            classTemplateSpecializationDecl(templateArgumentCountIs(1))));
+      matches("template<typename T> struct C {}; C<int> c;",
+              classTemplateSpecializationDecl(templateArgumentCountIs(1))));
   EXPECT_TRUE(
-    notMatches("template<typename T> struct C {}; C<int> c;",
-               classTemplateSpecializationDecl(templateArgumentCountIs(2))));
+      notMatches("template<typename T> struct C {}; C<int> c;",
+                 classTemplateSpecializationDecl(templateArgumentCountIs(2))));
 
   EXPECT_TRUE(matches("template<typename T> struct C {}; C<int> c;",
                       templateSpecializationType(templateArgumentCountIs(1))));
   EXPECT_TRUE(
-    notMatches("template<typename T> struct C {}; C<int> c;",
-               templateSpecializationType(templateArgumentCountIs(2))));
+      notMatches("template<typename T> struct C {}; C<int> c;",
+                 templateSpecializationType(templateArgumentCountIs(2))));
 }
 
 TEST(IsIntegral, Matches) {
-  EXPECT_TRUE(matches("template<int T> struct C {}; C<42> c;",
-                      classTemplateSpecializationDecl(
-                        hasAnyTemplateArgument(isIntegral()))));
+  EXPECT_TRUE(matches(
+      "template<int T> struct C {}; C<42> c;",
+      classTemplateSpecializationDecl(hasAnyTemplateArgument(isIntegral()))));
   EXPECT_TRUE(notMatches("template<typename T> struct C {}; C<int> c;",
                          classTemplateSpecializationDecl(hasAnyTemplateArgument(
-                           templateArgument(isIntegral())))));
+                             templateArgument(isIntegral())))));
 }
 
 TEST(EqualsIntegralValue, Matches) {
   EXPECT_TRUE(matches("template<int T> struct C {}; C<42> c;",
                       classTemplateSpecializationDecl(
-                        hasAnyTemplateArgument(equalsIntegralValue("42")))));
+                          hasAnyTemplateArgument(equalsIntegralValue("42")))));
   EXPECT_TRUE(matches("template<int T> struct C {}; C<-42> c;",
                       classTemplateSpecializationDecl(
-                        hasAnyTemplateArgument(equalsIntegralValue("-42")))));
+                          hasAnyTemplateArgument(equalsIntegralValue("-42")))));
   EXPECT_TRUE(matches("template<int T> struct C {}; C<-0042> c;",
                       classTemplateSpecializationDecl(
-                        hasAnyTemplateArgument(equalsIntegralValue("-34")))));
+                          hasAnyTemplateArgument(equalsIntegralValue("-34")))));
   EXPECT_TRUE(notMatches("template<int T> struct C {}; C<42> c;",
                          classTemplateSpecializationDecl(hasAnyTemplateArgument(
-                           equalsIntegralValue("0042")))));
+                             equalsIntegralValue("0042")))));
 }
 
 TEST(Matcher, MatchesAccessSpecDecls) {
@@ -1304,7 +1260,7 @@ TEST(Matcher, MatchesFinal) {
                       cxxMethodDecl(isFinal())));
   EXPECT_TRUE(notMatches("class X {};", cxxRecordDecl(isFinal())));
   EXPECT_TRUE(
-    notMatches("class X { virtual void f(); };", cxxMethodDecl(isFinal())));
+      notMatches("class X { virtual void f(); };", cxxMethodDecl(isFinal())));
 }
 
 TEST(Matcher, MatchesVirtualMethod) {
@@ -1315,12 +1271,12 @@ TEST(Matcher, MatchesVirtualMethod) {
 
 TEST(Matcher, MatchesVirtualAsWrittenMethod) {
   EXPECT_TRUE(matches("class A { virtual int f(); };"
-                        "class B : public A { int f(); };",
+                      "class B : public A { int f(); };",
                       cxxMethodDecl(isVirtualAsWritten(), hasName("::A::f"))));
   EXPECT_TRUE(
-    notMatches("class A { virtual int f(); };"
+      notMatches("class A { virtual int f(); };"
                  "class B : public A { int f(); };",
-               cxxMethodDecl(isVirtualAsWritten(), hasName("::B::f"))));
+                 cxxMethodDecl(isVirtualAsWritten(), hasName("::B::f"))));
 }
 
 TEST(Matcher, MatchesPureMethod) {
@@ -1358,26 +1314,26 @@ TEST(Matcher, MatchesMoveAssignmentOperator) {
 
 TEST(Matcher, MatchesConstMethod) {
   EXPECT_TRUE(
-    matches("struct A { void foo() const; };", cxxMethodDecl(isConst())));
+      matches("struct A { void foo() const; };", cxxMethodDecl(isConst())));
   EXPECT_TRUE(
-    notMatches("struct A { void foo(); };", cxxMethodDecl(isConst())));
+      notMatches("struct A { void foo(); };", cxxMethodDecl(isConst())));
 }
 
 TEST(Matcher, MatchesOverridingMethod) {
   EXPECT_TRUE(matches("class X { virtual int f(); }; "
-                        "class Y : public X { int f(); };",
+                      "class Y : public X { int f(); };",
                       cxxMethodDecl(isOverride(), hasName("::Y::f"))));
   EXPECT_TRUE(notMatches("class X { virtual int f(); }; "
-                           "class Y : public X { int f(); };",
+                         "class Y : public X { int f(); };",
                          cxxMethodDecl(isOverride(), hasName("::X::f"))));
   EXPECT_TRUE(notMatches("class X { int f(); }; "
-                           "class Y : public X { int f(); };",
+                         "class Y : public X { int f(); };",
                          cxxMethodDecl(isOverride())));
   EXPECT_TRUE(notMatches("class X { int f(); int f(int); }; ",
                          cxxMethodDecl(isOverride())));
   EXPECT_TRUE(
-    matches("template <typename Base> struct Y : Base { void f() override;};",
-            cxxMethodDecl(isOverride(), hasName("::Y::f"))));
+      matches("template <typename Base> struct Y : Base { void f() override;};",
+              cxxMethodDecl(isOverride(), hasName("::Y::f"))));
 }
 
 TEST(Matcher, ConstructorArgument) {
@@ -1385,44 +1341,38 @@ TEST(Matcher, ConstructorArgument) {
       ast_type_traits::TK_AsIs,
       cxxConstructExpr(hasArgument(0, declRefExpr(to(varDecl(hasName("y")))))));
 
+  EXPECT_TRUE(matches(
+      "class X { public: X(int); }; void x() { int y; X x(y); }", Constructor));
   EXPECT_TRUE(
-    matches("class X { public: X(int); }; void x() { int y; X x(y); }",
-            Constructor));
-  EXPECT_TRUE(
-    matches("class X { public: X(int); }; void x() { int y; X x = X(y); }",
-            Constructor));
-  EXPECT_TRUE(
-    matches("class X { public: X(int); }; void x() { int y; X x = y; }",
-            Constructor));
+      matches("class X { public: X(int); }; void x() { int y; X x = X(y); }",
+              Constructor));
   EXPECT_TRUE(
-    notMatches("class X { public: X(int); }; void x() { int z; X x(z); }",
-               Constructor));
+      matches("class X { public: X(int); }; void x() { int y; X x = y; }",
+              Constructor));
+  EXPECT_TRUE(notMatches(
+      "class X { public: X(int); }; void x() { int z; X x(z); }", Constructor));
 
   StatementMatcher WrongIndex =
       traverse(ast_type_traits::TK_AsIs,
                cxxConstructExpr(
                    hasArgument(42, declRefExpr(to(varDecl(hasName("y")))))));
-  EXPECT_TRUE(
-    notMatches("class X { public: X(int); }; void x() { int y; X x(y); }",
-               WrongIndex));
+  EXPECT_TRUE(notMatches(
+      "class X { public: X(int); }; void x() { int y; X x(y); }", WrongIndex));
 }
 
 TEST(Matcher, ConstructorArgumentCount) {
   auto Constructor1Arg =
       traverse(ast_type_traits::TK_AsIs, cxxConstructExpr(argumentCountIs(1)));
 
+  EXPECT_TRUE(matches("class X { public: X(int); }; void x() { X x(0); }",
+                      Constructor1Arg));
+  EXPECT_TRUE(matches("class X { public: X(int); }; void x() { X x = X(0); }",
+                      Constructor1Arg));
+  EXPECT_TRUE(matches("class X { public: X(int); }; void x() { X x = 0; }",
+                      Constructor1Arg));
   EXPECT_TRUE(
-    matches("class X { public: X(int); }; void x() { X x(0); }",
-            Constructor1Arg));
-  EXPECT_TRUE(
-    matches("class X { public: X(int); }; void x() { X x = X(0); }",
-            Constructor1Arg));
-  EXPECT_TRUE(
-    matches("class X { public: X(int); }; void x() { X x = 0; }",
-            Constructor1Arg));
-  EXPECT_TRUE(
-    notMatches("class X { public: X(int, int); }; void x() { X x(0, 0); }",
-               Constructor1Arg));
+      notMatches("class X { public: X(int, int); }; void x() { X x(0, 0); }",
+                 Constructor1Arg));
 }
 
 TEST(Matcher, ConstructorListInitialization) {
@@ -1430,19 +1380,16 @@ TEST(Matcher, ConstructorListInitialization) {
       traverse(ast_type_traits::TK_AsIs,
                varDecl(has(cxxConstructExpr(isListInitialization()))));
 
-  EXPECT_TRUE(
-    matches("class X { public: X(int); }; void x() { X x{0}; }",
-            ConstructorListInit));
-  EXPECT_FALSE(
-    matches("class X { public: X(int); }; void x() { X x(0); }",
-            ConstructorListInit));
+  EXPECT_TRUE(matches("class X { public: X(int); }; void x() { X x{0}; }",
+                      ConstructorListInit));
+  EXPECT_FALSE(matches("class X { public: X(int); }; void x() { X x(0); }",
+                       ConstructorListInit));
 }
 
 TEST(ConstructorDeclaration, IsImplicit) {
   // This one doesn't match because the constructor is not added by the
   // compiler (it is not needed).
-  EXPECT_TRUE(notMatches("class Foo { };",
-                         cxxConstructorDecl(isImplicit())));
+  EXPECT_TRUE(notMatches("class Foo { };", cxxConstructorDecl(isImplicit())));
   // The compiler added the implicit default constructor.
   EXPECT_TRUE(matches("class Foo { }; Foo* f = new Foo();",
                       cxxConstructorDecl(isImplicit())));
@@ -1456,8 +1403,8 @@ TEST(ConstructorDeclaration, IsImplicit) {
 TEST(ConstructorDeclaration, IsExplicit) {
   EXPECT_TRUE(matches("struct S { explicit S(int); };",
                       cxxConstructorDecl(isExplicit())));
-  EXPECT_TRUE(notMatches("struct S { S(int); };",
-                         cxxConstructorDecl(isExplicit())));
+  EXPECT_TRUE(
+      notMatches("struct S { S(int); };", cxxConstructorDecl(isExplicit())));
   EXPECT_TRUE(notMatches("template<bool b> struct S { explicit(b) S(int);};",
                          cxxConstructorDecl(isExplicit()), langCxx20OrLater()));
   EXPECT_TRUE(matches("struct S { explicit(true) S(int);};",
@@ -1488,9 +1435,9 @@ TEST(DeductionGuideDeclaration, IsExplicit) {
 }
 
 TEST(ConstructorDeclaration, Kinds) {
-  EXPECT_TRUE(matches(
-      "struct S { S(); };",
-      cxxConstructorDecl(isDefaultConstructor(), unless(isImplicit()))));
+  EXPECT_TRUE(
+      matches("struct S { S(); };", cxxConstructorDecl(isDefaultConstructor(),
+                                                       unless(isImplicit()))));
   EXPECT_TRUE(notMatches(
       "struct S { S(); };",
       cxxConstructorDecl(isCopyConstructor(), unless(isImplicit()))));
@@ -1501,9 +1448,9 @@ TEST(ConstructorDeclaration, Kinds) {
   EXPECT_TRUE(notMatches(
       "struct S { S(const S&); };",
       cxxConstructorDecl(isDefaultConstructor(), unless(isImplicit()))));
-  EXPECT_TRUE(matches(
-      "struct S { S(const S&); };",
-      cxxConstructorDecl(isCopyConstructor(), unless(isImplicit()))));
+  EXPECT_TRUE(
+      matches("struct S { S(const S&); };",
+              cxxConstructorDecl(isCopyConstructor(), unless(isImplicit()))));
   EXPECT_TRUE(notMatches(
       "struct S { S(const S&); };",
       cxxConstructorDecl(isMoveConstructor(), unless(isImplicit()))));
@@ -1514,9 +1461,9 @@ TEST(ConstructorDeclaration, Kinds) {
   EXPECT_TRUE(notMatches(
       "struct S { S(S&&); };",
       cxxConstructorDecl(isCopyConstructor(), unless(isImplicit()))));
-  EXPECT_TRUE(matches(
-      "struct S { S(S&&); };",
-      cxxConstructorDecl(isMoveConstructor(), unless(isImplicit()))));
+  EXPECT_TRUE(
+      matches("struct S { S(S&&); };",
+              cxxConstructorDecl(isMoveConstructor(), unless(isImplicit()))));
 }
 
 TEST(ConstructorDeclaration, IsUserProvided) {
@@ -1527,7 +1474,7 @@ TEST(ConstructorDeclaration, IsUserProvided) {
   EXPECT_TRUE(notMatches("struct S { S() = delete; };",
                          cxxConstructorDecl(isUserProvided())));
   EXPECT_TRUE(
-    matches("struct S { S(); };", cxxConstructorDecl(isUserProvided())));
+      matches("struct S { S(); };", cxxConstructorDecl(isUserProvided())));
   EXPECT_TRUE(matches("struct S { S(); }; S::S(){}",
                       cxxConstructorDecl(isUserProvided())));
 }
@@ -1538,11 +1485,11 @@ TEST(ConstructorDeclaration, IsDelegatingConstructor) {
   EXPECT_TRUE(notMatches("struct S { S(){} S(int X) : X(X) {} int X; };",
                          cxxConstructorDecl(isDelegatingConstructor())));
   EXPECT_TRUE(matches(
-    "struct S { S() : S(0) {} S(int X) : X(X) {} int X; };",
-    cxxConstructorDecl(isDelegatingConstructor(), parameterCountIs(0))));
+      "struct S { S() : S(0) {} S(int X) : X(X) {} int X; };",
+      cxxConstructorDecl(isDelegatingConstructor(), parameterCountIs(0))));
   EXPECT_TRUE(matches(
-    "struct S { S(); S(int X); int X; }; S::S(int X) : S() {}",
-    cxxConstructorDecl(isDelegatingConstructor(), parameterCountIs(1))));
+      "struct S { S(); S(int X); int X; }; S::S(int X) : S() {}",
+      cxxConstructorDecl(isDelegatingConstructor(), parameterCountIs(1))));
 }
 
 TEST(StringLiteral, HasSize) {
@@ -1584,38 +1531,28 @@ TEST(Matcher, HasNameSupportsNamespaces) {
 }
 
 TEST(Matcher, HasNameSupportsOuterClasses) {
-  EXPECT_TRUE(
-    matches("class A { class B { class C; }; };",
-            recordDecl(hasName("A::B::C"))));
-  EXPECT_TRUE(
-    matches("class A { class B { class C; }; };",
-            recordDecl(hasName("::A::B::C"))));
-  EXPECT_TRUE(
-    matches("class A { class B { class C; }; };",
-            recordDecl(hasName("B::C"))));
-  EXPECT_TRUE(
-    matches("class A { class B { class C; }; };",
-            recordDecl(hasName("C"))));
-  EXPECT_TRUE(
-    notMatches("class A { class B { class C; }; };",
-               recordDecl(hasName("c::B::C"))));
-  EXPECT_TRUE(
-    notMatches("class A { class B { class C; }; };",
-               recordDecl(hasName("A::c::C"))));
-  EXPECT_TRUE(
-    notMatches("class A { class B { class C; }; };",
-               recordDecl(hasName("A::B::A"))));
-  EXPECT_TRUE(
-    notMatches("class A { class B { class C; }; };",
-               recordDecl(hasName("::C"))));
-  EXPECT_TRUE(
-    notMatches("class A { class B { class C; }; };",
-               recordDecl(hasName("::B::C"))));
+  EXPECT_TRUE(matches("class A { class B { class C; }; };",
+                      recordDecl(hasName("A::B::C"))));
+  EXPECT_TRUE(matches("class A { class B { class C; }; };",
+                      recordDecl(hasName("::A::B::C"))));
+  EXPECT_TRUE(matches("class A { class B { class C; }; };",
+                      recordDecl(hasName("B::C"))));
+  EXPECT_TRUE(
+      matches("class A { class B { class C; }; };", recordDecl(hasName("C"))));
+  EXPECT_TRUE(notMatches("class A { class B { class C; }; };",
+                         recordDecl(hasName("c::B::C"))));
+  EXPECT_TRUE(notMatches("class A { class B { class C; }; };",
+                         recordDecl(hasName("A::c::C"))));
+  EXPECT_TRUE(notMatches("class A { class B { class C; }; };",
+                         recordDecl(hasName("A::B::A"))));
+  EXPECT_TRUE(notMatches("class A { class B { class C; }; };",
+                         recordDecl(hasName("::C"))));
+  EXPECT_TRUE(notMatches("class A { class B { class C; }; };",
+                         recordDecl(hasName("::B::C"))));
   EXPECT_TRUE(notMatches("class A { class B { class C; }; };",
                          recordDecl(hasName("z::A::B::C"))));
-  EXPECT_TRUE(
-    notMatches("class A { class B { class C; }; };",
-               recordDecl(hasName("A+B::C"))));
+  EXPECT_TRUE(notMatches("class A { class B { class C; }; };",
+                         recordDecl(hasName("A+B::C"))));
 }
 
 TEST(Matcher, HasNameSupportsInlinedNamespaces) {
@@ -1629,10 +1566,10 @@ TEST(Matcher, HasNameSupportsInlinedNamespaces) {
 TEST(Matcher, HasNameSupportsAnonymousNamespaces) {
   StringRef code = "namespace a { namespace { class C; } }";
   EXPECT_TRUE(
-    matches(code, recordDecl(hasName("a::(anonymous namespace)::C"))));
+      matches(code, recordDecl(hasName("a::(anonymous namespace)::C"))));
   EXPECT_TRUE(matches(code, recordDecl(hasName("a::C"))));
   EXPECT_TRUE(
-    matches(code, recordDecl(hasName("::a::(anonymous namespace)::C"))));
+      matches(code, recordDecl(hasName("::a::(anonymous namespace)::C"))));
   EXPECT_TRUE(matches(code, recordDecl(hasName("::a::C"))));
 }
 
@@ -1689,7 +1626,7 @@ TEST(Matcher, HasAnyName) {
 
   EXPECT_TRUE(notMatches(Code, recordDecl(hasAnyName("::C", "::b::C"))));
   EXPECT_TRUE(
-    matches(Code, recordDecl(hasAnyName("::C", "::b::C", "::a::b::C"))));
+      matches(Code, recordDecl(hasAnyName("::C", "::b::C", "::a::b::C"))));
 
   std::vector<StringRef> Names = {"::C", "::b::C", "::a::b::C"};
   EXPECT_TRUE(matches(Code, recordDecl(hasAnyName(Names))));
@@ -1697,27 +1634,27 @@ TEST(Matcher, HasAnyName) {
 
 TEST(Matcher, IsDefinition) {
   DeclarationMatcher DefinitionOfClassA =
-    recordDecl(hasName("A"), isDefinition());
+      recordDecl(hasName("A"), isDefinition());
   EXPECT_TRUE(matches("class A {};", DefinitionOfClassA));
   EXPECT_TRUE(notMatches("class A;", DefinitionOfClassA));
 
   DeclarationMatcher DefinitionOfVariableA =
-    varDecl(hasName("a"), isDefinition());
+      varDecl(hasName("a"), isDefinition());
   EXPECT_TRUE(matches("int a;", DefinitionOfVariableA));
   EXPECT_TRUE(notMatches("extern int a;", DefinitionOfVariableA));
 
   DeclarationMatcher DefinitionOfMethodA =
-    cxxMethodDecl(hasName("a"), isDefinition());
+      cxxMethodDecl(hasName("a"), isDefinition());
   EXPECT_TRUE(matches("class A { void a() {} };", DefinitionOfMethodA));
   EXPECT_TRUE(notMatches("class A { void a(); };", DefinitionOfMethodA));
 
   DeclarationMatcher DefinitionOfObjCMethodA =
-    objcMethodDecl(hasName("a"), isDefinition());
+      objcMethodDecl(hasName("a"), isDefinition());
   EXPECT_TRUE(matchesObjC("@interface A @end "
                           "@implementation A; -(void)a {} @end",
                           DefinitionOfObjCMethodA));
-  EXPECT_TRUE(notMatchesObjC("@interface A; - (void)a; @end",
-                             DefinitionOfObjCMethodA));
+  EXPECT_TRUE(
+      notMatchesObjC("@interface A; - (void)a; @end", DefinitionOfObjCMethodA));
 }
 
 TEST(Matcher, HandlesNullQualTypes) {
@@ -1728,7 +1665,7 @@ TEST(Matcher, HandlesNullQualTypes) {
   // We don't really care whether this matcher succeeds; we're testing that
   // it completes without crashing.
   EXPECT_TRUE(matches(
-    "struct A { };"
+      "struct A { };"
       "template <typename T>"
       "void f(T t) {"
       "  T local_t(t /* this becomes a null QualType in the AST */);"
@@ -1736,13 +1673,10 @@ TEST(Matcher, HandlesNullQualTypes) {
       "void g() {"
       "  f(0);"
       "}",
-    expr(hasType(TypeMatcher(
-      anyOf(
-        TypeMatcher(hasDeclaration(anything())),
-        pointsTo(AnyType),
-        references(AnyType)
-        // Other QualType matchers should go here.
-      ))))));
+      expr(hasType(TypeMatcher(anyOf(TypeMatcher(hasDeclaration(anything())),
+                                     pointsTo(AnyType), references(AnyType)
+                                     // Other QualType matchers should go here.
+                                     ))))));
 }
 
 TEST(ObjCIvarRefExprMatcher, IvarExpr) {
@@ -1750,10 +1684,10 @@ TEST(ObjCIvarRefExprMatcher, IvarExpr) {
       "@interface A @end "
       "@implementation A { A *x; } - (void) func { x = 0; } @end";
   EXPECT_TRUE(matchesObjC(ObjCString, objcIvarRefExpr()));
-  EXPECT_TRUE(matchesObjC(ObjCString, objcIvarRefExpr(
-        hasDeclaration(namedDecl(hasName("x"))))));
-  EXPECT_FALSE(matchesObjC(ObjCString, objcIvarRefExpr(
-        hasDeclaration(namedDecl(hasName("y"))))));
+  EXPECT_TRUE(matchesObjC(
+      ObjCString, objcIvarRefExpr(hasDeclaration(namedDecl(hasName("x"))))));
+  EXPECT_FALSE(matchesObjC(
+      ObjCString, objcIvarRefExpr(hasDeclaration(namedDecl(hasName("y"))))));
 }
 
 TEST(BlockExprMatcher, BlockExpr) {
@@ -1761,24 +1695,19 @@ TEST(BlockExprMatcher, BlockExpr) {
 }
 
 TEST(StatementCountIs, FindsNoStatementsInAnEmptyCompoundStatement) {
-  EXPECT_TRUE(matches("void f() { }",
-                      compoundStmt(statementCountIs(0))));
-  EXPECT_TRUE(notMatches("void f() {}",
-                         compoundStmt(statementCountIs(1))));
+  EXPECT_TRUE(matches("void f() { }", compoundStmt(statementCountIs(0))));
+  EXPECT_TRUE(notMatches("void f() {}", compoundStmt(statementCountIs(1))));
 }
 
 TEST(StatementCountIs, AppearsToMatchOnlyOneCount) {
-  EXPECT_TRUE(matches("void f() { 1; }",
-                      compoundStmt(statementCountIs(1))));
-  EXPECT_TRUE(notMatches("void f() { 1; }",
-                         compoundStmt(statementCountIs(0))));
-  EXPECT_TRUE(notMatches("void f() { 1; }",
-                         compoundStmt(statementCountIs(2))));
+  EXPECT_TRUE(matches("void f() { 1; }", compoundStmt(statementCountIs(1))));
+  EXPECT_TRUE(notMatches("void f() { 1; }", compoundStmt(statementCountIs(0))));
+  EXPECT_TRUE(notMatches("void f() { 1; }", compoundStmt(statementCountIs(2))));
 }
 
 TEST(StatementCountIs, WorksWithMultipleStatements) {
-  EXPECT_TRUE(matches("void f() { 1; 2; 3; }",
-                      compoundStmt(statementCountIs(3))));
+  EXPECT_TRUE(
+      matches("void f() { 1; 2; 3; }", compoundStmt(statementCountIs(3))));
 }
 
 TEST(StatementCountIs, WorksWithNestedCompoundStatements) {
@@ -1806,19 +1735,19 @@ TEST(Member, DoesNotMatchTheBaseExpression) {
 
 TEST(Member, MatchesInMemberFunctionCall) {
   EXPECT_TRUE(matches("void f() {"
-                        "  struct { void first() {}; } s;"
-                        "  s.first();"
-                        "};",
+                      "  struct { void first() {}; } s;"
+                      "  s.first();"
+                      "};",
                       memberExpr(member(hasName("first")))));
 }
 
 TEST(Member, MatchesMember) {
-  EXPECT_TRUE(matches(
-    "struct A { int i; }; void f() { A a; a.i = 2; }",
-    memberExpr(hasDeclaration(fieldDecl(hasType(isInteger()))))));
-  EXPECT_TRUE(notMatches(
-    "struct A { float f; }; void f() { A a; a.f = 2.0f; }",
-    memberExpr(hasDeclaration(fieldDecl(hasType(isInteger()))))));
+  EXPECT_TRUE(
+      matches("struct A { int i; }; void f() { A a; a.i = 2; }",
+              memberExpr(hasDeclaration(fieldDecl(hasType(isInteger()))))));
+  EXPECT_TRUE(
+      notMatches("struct A { float f; }; void f() { A a; a.f = 2.0f; }",
+                 memberExpr(hasDeclaration(fieldDecl(hasType(isInteger()))))));
 }
 
 TEST(Member, BitFields) {
@@ -1841,26 +1770,26 @@ TEST(Member, InClassInitializer) {
 }
 
 TEST(Member, UnderstandsAccess) {
-  EXPECT_TRUE(matches(
-    "struct A { int i; };", fieldDecl(isPublic(), hasName("i"))));
-  EXPECT_TRUE(notMatches(
-    "struct A { int i; };", fieldDecl(isProtected(), hasName("i"))));
-  EXPECT_TRUE(notMatches(
-    "struct A { int i; };", fieldDecl(isPrivate(), hasName("i"))));
+  EXPECT_TRUE(
+      matches("struct A { int i; };", fieldDecl(isPublic(), hasName("i"))));
+  EXPECT_TRUE(notMatches("struct A { int i; };",
+                         fieldDecl(isProtected(), hasName("i"))));
+  EXPECT_TRUE(
+      notMatches("struct A { int i; };", fieldDecl(isPrivate(), hasName("i"))));
 
-  EXPECT_TRUE(notMatches(
-    "class A { int i; };", fieldDecl(isPublic(), hasName("i"))));
-  EXPECT_TRUE(notMatches(
-    "class A { int i; };", fieldDecl(isProtected(), hasName("i"))));
-  EXPECT_TRUE(matches(
-    "class A { int i; };", fieldDecl(isPrivate(), hasName("i"))));
+  EXPECT_TRUE(
+      notMatches("class A { int i; };", fieldDecl(isPublic(), hasName("i"))));
+  EXPECT_TRUE(notMatches("class A { int i; };",
+                         fieldDecl(isProtected(), hasName("i"))));
+  EXPECT_TRUE(
+      matches("class A { int i; };", fieldDecl(isPrivate(), hasName("i"))));
 
-  EXPECT_TRUE(notMatches(
-    "class A { protected: int i; };", fieldDecl(isPublic(), hasName("i"))));
+  EXPECT_TRUE(notMatches("class A { protected: int i; };",
+                         fieldDecl(isPublic(), hasName("i"))));
   EXPECT_TRUE(matches("class A { protected: int i; };",
                       fieldDecl(isProtected(), hasName("i"))));
-  EXPECT_TRUE(notMatches(
-    "class A { protected: int i; };", fieldDecl(isPrivate(), hasName("i"))));
+  EXPECT_TRUE(notMatches("class A { protected: int i; };",
+                         fieldDecl(isPrivate(), hasName("i"))));
 
   // Non-member decls have the AccessSpecifier AS_none and thus aren't matched.
   EXPECT_TRUE(notMatches("int i;", varDecl(isPublic(), hasName("i"))));
@@ -1883,35 +1812,35 @@ TEST(hasDynamicExceptionSpec, MatchesDynamicExceptionSpecifications) {
   EXPECT_TRUE(
       matches("void l() throw(...);", functionDecl(hasDynamicExceptionSpec())));
 
-  EXPECT_TRUE(notMatches("void f();", functionProtoType(hasDynamicExceptionSpec())));
+  EXPECT_TRUE(
+      notMatches("void f();", functionProtoType(hasDynamicExceptionSpec())));
   EXPECT_TRUE(notMatches("void g() noexcept;",
                          functionProtoType(hasDynamicExceptionSpec())));
   EXPECT_TRUE(notMatches("void h() noexcept(true);",
                          functionProtoType(hasDynamicExceptionSpec())));
   EXPECT_TRUE(notMatches("void i() noexcept(false);",
                          functionProtoType(hasDynamicExceptionSpec())));
-  EXPECT_TRUE(
-      matches("void j() throw();", functionProtoType(hasDynamicExceptionSpec())));
-  EXPECT_TRUE(
-      matches("void k() throw(int);", functionProtoType(hasDynamicExceptionSpec())));
-  EXPECT_TRUE(
-      matches("void l() throw(...);", functionProtoType(hasDynamicExceptionSpec())));
+  EXPECT_TRUE(matches("void j() throw();",
+                      functionProtoType(hasDynamicExceptionSpec())));
+  EXPECT_TRUE(matches("void k() throw(int);",
+                      functionProtoType(hasDynamicExceptionSpec())));
+  EXPECT_TRUE(matches("void l() throw(...);",
+                      functionProtoType(hasDynamicExceptionSpec())));
 }
 
 TEST(HasObjectExpression, DoesNotMatchMember) {
   EXPECT_TRUE(notMatches(
-    "class X {}; struct Z { X m; }; void f(Z z) { z.m; }",
-    memberExpr(hasObjectExpression(hasType(recordDecl(hasName("X")))))));
+      "class X {}; struct Z { X m; }; void f(Z z) { z.m; }",
+      memberExpr(hasObjectExpression(hasType(recordDecl(hasName("X")))))));
 }
 
 TEST(HasObjectExpression, MatchesBaseOfVariable) {
   EXPECT_TRUE(matches(
-    "struct X { int m; }; void f(X x) { x.m; }",
-    memberExpr(hasObjectExpression(hasType(recordDecl(hasName("X")))))));
-  EXPECT_TRUE(matches(
-    "struct X { int m; }; void f(X* x) { x->m; }",
-    memberExpr(hasObjectExpression(
-      hasType(pointsTo(recordDecl(hasName("X"))))))));
+      "struct X { int m; }; void f(X x) { x.m; }",
+      memberExpr(hasObjectExpression(hasType(recordDecl(hasName("X")))))));
+  EXPECT_TRUE(matches("struct X { int m; }; void f(X* x) { x->m; }",
+                      memberExpr(hasObjectExpression(
+                          hasType(pointsTo(recordDecl(hasName("X"))))))));
   EXPECT_TRUE(matches("template <class T> struct X { void f() { T t; t.m; } };",
                       cxxDependentScopeMemberExpr(hasObjectExpression(
                           declRefExpr(to(namedDecl(hasName("t"))))))));
@@ -1936,14 +1865,12 @@ TEST(HasObjectExpression, MatchesBaseOfMemberFunc) {
 
 TEST(HasObjectExpression,
      MatchesObjectExpressionOfImplicitlyFormedMemberExpression) {
-  EXPECT_TRUE(matches(
-    "class X {}; struct S { X m; void f() { this->m; } };",
-    memberExpr(hasObjectExpression(
-      hasType(pointsTo(recordDecl(hasName("S"))))))));
-  EXPECT_TRUE(matches(
-    "class X {}; struct S { X m; void f() { m; } };",
-    memberExpr(hasObjectExpression(
-      hasType(pointsTo(recordDecl(hasName("S"))))))));
+  EXPECT_TRUE(matches("class X {}; struct S { X m; void f() { this->m; } };",
+                      memberExpr(hasObjectExpression(
+                          hasType(pointsTo(recordDecl(hasName("S"))))))));
+  EXPECT_TRUE(matches("class X {}; struct S { X m; void f() { m; } };",
+                      memberExpr(hasObjectExpression(
+                          hasType(pointsTo(recordDecl(hasName("S"))))))));
 }
 
 TEST(Field, DoesNotMatchNonFieldMembers) {
@@ -1958,17 +1885,17 @@ TEST(Field, MatchesField) {
 }
 
 TEST(IsVolatileQualified, QualifiersMatch) {
-  EXPECT_TRUE(matches("volatile int i = 42;",
-                      varDecl(hasType(isVolatileQualified()))));
-  EXPECT_TRUE(notMatches("volatile int *i;",
-                         varDecl(hasType(isVolatileQualified()))));
+  EXPECT_TRUE(
+      matches("volatile int i = 42;", varDecl(hasType(isVolatileQualified()))));
+  EXPECT_TRUE(
+      notMatches("volatile int *i;", varDecl(hasType(isVolatileQualified()))));
   EXPECT_TRUE(matches("typedef volatile int v_int; v_int i = 42;",
                       varDecl(hasType(isVolatileQualified()))));
 }
 
 TEST(IsConstQualified, MatchesConstInt) {
-  EXPECT_TRUE(matches("const int i = 42;",
-                      varDecl(hasType(isConstQualified()))));
+  EXPECT_TRUE(
+      matches("const int i = 42;", varDecl(hasType(isConstQualified()))));
 }
 
 TEST(IsConstQualified, MatchesConstPointer) {
@@ -1986,43 +1913,41 @@ TEST(IsConstQualified, MatchesThroughTypedef) {
 TEST(IsConstQualified, DoesNotMatchInappropriately) {
   EXPECT_TRUE(notMatches("typedef int nonconst_int; nonconst_int i = 42;",
                          varDecl(hasType(isConstQualified()))));
-  EXPECT_TRUE(notMatches("int const* p;",
-                         varDecl(hasType(isConstQualified()))));
+  EXPECT_TRUE(
+      notMatches("int const* p;", varDecl(hasType(isConstQualified()))));
 }
 
 TEST(DeclCount, DeclCountIsCorrect) {
-  EXPECT_TRUE(matches("void f() {int i,j;}",
-                      declStmt(declCountIs(2))));
-  EXPECT_TRUE(notMatches("void f() {int i,j; int k;}",
-                         declStmt(declCountIs(3))));
-  EXPECT_TRUE(notMatches("void f() {int i,j, k, l;}",
-                         declStmt(declCountIs(3))));
+  EXPECT_TRUE(matches("void f() {int i,j;}", declStmt(declCountIs(2))));
+  EXPECT_TRUE(
+      notMatches("void f() {int i,j; int k;}", declStmt(declCountIs(3))));
+  EXPECT_TRUE(
+      notMatches("void f() {int i,j, k, l;}", declStmt(declCountIs(3))));
 }
 
-
 TEST(EachOf, TriggersForEachMatch) {
   EXPECT_TRUE(matchAndVerifyResultTrue(
-    "class A { int a; int b; };",
-    recordDecl(eachOf(has(fieldDecl(hasName("a")).bind("v")),
-                      has(fieldDecl(hasName("b")).bind("v")))),
-    std::make_unique<VerifyIdIsBoundTo<FieldDecl>>("v", 2)));
+      "class A { int a; int b; };",
+      recordDecl(eachOf(has(fieldDecl(hasName("a")).bind("v")),
+                        has(fieldDecl(hasName("b")).bind("v")))),
+      std::make_unique<VerifyIdIsBoundTo<FieldDecl>>("v", 2)));
 }
 
 TEST(EachOf, BehavesLikeAnyOfUnlessBothMatch) {
   EXPECT_TRUE(matchAndVerifyResultTrue(
-    "class A { int a; int c; };",
-    recordDecl(eachOf(has(fieldDecl(hasName("a")).bind("v")),
-                      has(fieldDecl(hasName("b")).bind("v")))),
-    std::make_unique<VerifyIdIsBoundTo<FieldDecl>>("v", 1)));
+      "class A { int a; int c; };",
+      recordDecl(eachOf(has(fieldDecl(hasName("a")).bind("v")),
+                        has(fieldDecl(hasName("b")).bind("v")))),
+      std::make_unique<VerifyIdIsBoundTo<FieldDecl>>("v", 1)));
   EXPECT_TRUE(matchAndVerifyResultTrue(
-    "class A { int c; int b; };",
-    recordDecl(eachOf(has(fieldDecl(hasName("a")).bind("v")),
-                      has(fieldDecl(hasName("b")).bind("v")))),
-    std::make_unique<VerifyIdIsBoundTo<FieldDecl>>("v", 1)));
-  EXPECT_TRUE(notMatches(
-    "class A { int c; int d; };",
-    recordDecl(eachOf(has(fieldDecl(hasName("a")).bind("v")),
-                      has(fieldDecl(hasName("b")).bind("v"))))));
+      "class A { int c; int b; };",
+      recordDecl(eachOf(has(fieldDecl(hasName("a")).bind("v")),
+                        has(fieldDecl(hasName("b")).bind("v")))),
+      std::make_unique<VerifyIdIsBoundTo<FieldDecl>>("v", 1)));
+  EXPECT_TRUE(
+      notMatches("class A { int c; int d; };",
+                 recordDecl(eachOf(has(fieldDecl(hasName("a")).bind("v")),
+                                   has(fieldDecl(hasName("b")).bind("v"))))));
 }
 
 TEST(Optionally, SubmatchersDoNotMatch) {
@@ -2056,29 +1981,30 @@ TEST(IsTemplateInstantiation, MatchesImplicitClassTemplateInstantiation) {
   // Make sure that we can both match the class by name (::X) and by the type
   // the template was instantiated with (via a field).
 
-  EXPECT_TRUE(matches(
-    "template <typename T> class X {}; class A {}; X<A> x;",
-    cxxRecordDecl(hasName("::X"), isTemplateInstantiation())));
+  EXPECT_TRUE(
+      matches("template <typename T> class X {}; class A {}; X<A> x;",
+              cxxRecordDecl(hasName("::X"), isTemplateInstantiation())));
 
   EXPECT_TRUE(matches(
-    "template <typename T> class X { T t; }; class A {}; X<A> x;",
-    cxxRecordDecl(isTemplateInstantiation(), hasDescendant(
-      fieldDecl(hasType(recordDecl(hasName("A"))))))));
+      "template <typename T> class X { T t; }; class A {}; X<A> x;",
+      cxxRecordDecl(
+          isTemplateInstantiation(),
+          hasDescendant(fieldDecl(hasType(recordDecl(hasName("A"))))))));
 }
 
 TEST(IsTemplateInstantiation, MatchesImplicitFunctionTemplateInstantiation) {
   EXPECT_TRUE(matches(
-    "template <typename T> void f(T t) {} class A {}; void g() { f(A()); }",
-    functionDecl(hasParameter(0, hasType(recordDecl(hasName("A")))),
-                 isTemplateInstantiation())));
+      "template <typename T> void f(T t) {} class A {}; void g() { f(A()); }",
+      functionDecl(hasParameter(0, hasType(recordDecl(hasName("A")))),
+                   isTemplateInstantiation())));
 }
 
 TEST(IsTemplateInstantiation, MatchesExplicitClassTemplateInstantiation) {
-  EXPECT_TRUE(matches(
-    "template <typename T> class X { T t; }; class A {};"
-      "template class X<A>;",
-    cxxRecordDecl(isTemplateInstantiation(), hasDescendant(
-      fieldDecl(hasType(recordDecl(hasName("A"))))))));
+  EXPECT_TRUE(matches("template <typename T> class X { T t; }; class A {};"
+                      "template class X<A>;",
+                      cxxRecordDecl(isTemplateInstantiation(),
+                                    hasDescendant(fieldDecl(
+                                        hasType(recordDecl(hasName("A"))))))));
 
   // Make sure that we match the instantiation instead of the template
   // definition by checking whether the member function is present.
@@ -2091,21 +2017,21 @@ TEST(IsTemplateInstantiation, MatchesExplicitClassTemplateInstantiation) {
 
 TEST(IsTemplateInstantiation,
      MatchesInstantiationOfPartiallySpecializedClassTemplate) {
-  EXPECT_TRUE(matches(
-    "template <typename T> class X {};"
-      "template <typename T> class X<T*> {}; class A {}; X<A*> x;",
-    cxxRecordDecl(hasName("::X"), isTemplateInstantiation())));
+  EXPECT_TRUE(
+      matches("template <typename T> class X {};"
+              "template <typename T> class X<T*> {}; class A {}; X<A*> x;",
+              cxxRecordDecl(hasName("::X"), isTemplateInstantiation())));
 }
 
 TEST(IsTemplateInstantiation,
      MatchesInstantiationOfClassTemplateNestedInNonTemplate) {
-  EXPECT_TRUE(matches(
-    "class A {};"
-      "class X {"
-      "  template <typename U> class Y { U u; };"
-      "  Y<A> y;"
-      "};",
-    cxxRecordDecl(hasName("::X::Y"), isTemplateInstantiation())));
+  EXPECT_TRUE(
+      matches("class A {};"
+              "class X {"
+              "  template <typename U> class Y { U u; };"
+              "  Y<A> y;"
+              "};",
+              cxxRecordDecl(hasName("::X::Y"), isTemplateInstantiation())));
 }
 
 TEST(IsTemplateInstantiation, DoesNotMatchInstantiationsInsideOfInstantiation) {
@@ -2113,31 +2039,30 @@ TEST(IsTemplateInstantiation, DoesNotMatchInstantiationsInsideOfInstantiation) {
   // normal use case as long as the uppermost instantiation always is marked
   // as template instantiation, but it might be confusing as a predicate.
   EXPECT_TRUE(matches(
-    "class A {};"
+      "class A {};"
       "template <typename T> class X {"
       "  template <typename U> class Y { U u; };"
       "  Y<T> y;"
       "}; X<A> x;",
-    cxxRecordDecl(hasName("::X<A>::Y"), unless(isTemplateInstantiation()))));
+      cxxRecordDecl(hasName("::X<A>::Y"), unless(isTemplateInstantiation()))));
 }
 
 TEST(IsTemplateInstantiation, DoesNotMatchExplicitClassTemplateSpecialization) {
-  EXPECT_TRUE(notMatches(
-    "template <typename T> class X {}; class A {};"
-      "template <> class X<A> {}; X<A> x;",
-    cxxRecordDecl(hasName("::X"), isTemplateInstantiation())));
+  EXPECT_TRUE(
+      notMatches("template <typename T> class X {}; class A {};"
+                 "template <> class X<A> {}; X<A> x;",
+                 cxxRecordDecl(hasName("::X"), isTemplateInstantiation())));
 }
 
 TEST(IsTemplateInstantiation, DoesNotMatchNonTemplate) {
-  EXPECT_TRUE(notMatches(
-    "class A {}; class Y { A a; };",
-    cxxRecordDecl(isTemplateInstantiation())));
+  EXPECT_TRUE(notMatches("class A {}; class Y { A a; };",
+                         cxxRecordDecl(isTemplateInstantiation())));
 }
 
 TEST(IsInstantiated, MatchesInstantiation) {
   EXPECT_TRUE(
-    matches("template<typename T> class A { T i; }; class Y { A<int> a; };",
-            cxxRecordDecl(isInstantiated())));
+      matches("template<typename T> class A { T i; }; class Y { A<int> a; };",
+              cxxRecordDecl(isInstantiated())));
 }
 
 TEST(IsInstantiated, NotMatchesDefinition) {
@@ -2147,7 +2072,7 @@ TEST(IsInstantiated, NotMatchesDefinition) {
 
 TEST(IsInTemplateInstantiation, MatchesInstantiationStmt) {
   EXPECT_TRUE(matches("template<typename T> struct A { A() { T i; } };"
-                        "class Y { A<int> a; }; Y y;",
+                      "class Y { A<int> a; }; Y y;",
                       declStmt(isInTemplateInstantiation())));
 }
 
@@ -2158,8 +2083,8 @@ TEST(IsInTemplateInstantiation, NotMatchesDefinitionStmt) {
 
 TEST(IsInstantiated, MatchesFunctionInstantiation) {
   EXPECT_TRUE(
-    matches("template<typename T> void A(T t) { T i; } void x() { A(0); }",
-            functionDecl(isInstantiated())));
+      matches("template<typename T> void A(T t) { T i; } void x() { A(0); }",
+              functionDecl(isInstantiated())));
 }
 
 TEST(IsInstantiated, NotMatchesFunctionDefinition) {
@@ -2169,8 +2094,8 @@ TEST(IsInstantiated, NotMatchesFunctionDefinition) {
 
 TEST(IsInTemplateInstantiation, MatchesFunctionInstantiationStmt) {
   EXPECT_TRUE(
-    matches("template<typename T> void A(T t) { T i; } void x() { A(0); }",
-            declStmt(isInTemplateInstantiation())));
+      matches("template<typename T> void A(T t) { T i; } void x() { A(0); }",
+              declStmt(isInTemplateInstantiation())));
 }
 
 TEST(IsInTemplateInstantiation, NotMatchesFunctionDefinitionStmt) {
@@ -2183,11 +2108,11 @@ TEST(IsInTemplateInstantiation, Sharing) {
   // FIXME: Node sharing is an implementation detail, exposing it is ugly
   // and makes the matcher behave in non-obvious ways.
   EXPECT_TRUE(notMatches(
-    "int j; template<typename T> void A(T t) { j += 42; } void x() { A(0); }",
-    Matcher));
+      "int j; template<typename T> void A(T t) { j += 42; } void x() { A(0); }",
+      Matcher));
   EXPECT_TRUE(matches(
-    "int j; template<typename T> void A(T t) { j += t; } void x() { A(0); }",
-    Matcher));
+      "int j; template<typename T> void A(T t) { j += t; } void x() { A(0); }",
+      Matcher));
 }
 
 TEST(IsInstantiationDependent, MatchesNonValueTypeDependent) {
@@ -2232,48 +2157,41 @@ TEST(IsValueDependent, MatchesInstantiationDependent) {
       expr(isValueDependent())));
 }
 
-TEST(IsExplicitTemplateSpecialization,
-     DoesNotMatchPrimaryTemplate) {
-  EXPECT_TRUE(notMatches(
-    "template <typename T> class X {};",
-    cxxRecordDecl(isExplicitTemplateSpecialization())));
-  EXPECT_TRUE(notMatches(
-    "template <typename T> void f(T t);",
-    functionDecl(isExplicitTemplateSpecialization())));
+TEST(IsExplicitTemplateSpecialization, DoesNotMatchPrimaryTemplate) {
+  EXPECT_TRUE(notMatches("template <typename T> class X {};",
+                         cxxRecordDecl(isExplicitTemplateSpecialization())));
+  EXPECT_TRUE(notMatches("template <typename T> void f(T t);",
+                         functionDecl(isExplicitTemplateSpecialization())));
 }
 
 TEST(IsExplicitTemplateSpecialization,
      DoesNotMatchExplicitTemplateInstantiations) {
-  EXPECT_TRUE(notMatches(
-    "template <typename T> class X {};"
-      "template class X<int>; extern template class X<long>;",
-    cxxRecordDecl(isExplicitTemplateSpecialization())));
-  EXPECT_TRUE(notMatches(
-    "template <typename T> void f(T t) {}"
-      "template void f(int t); extern template void f(long t);",
-    functionDecl(isExplicitTemplateSpecialization())));
+  EXPECT_TRUE(
+      notMatches("template <typename T> class X {};"
+                 "template class X<int>; extern template class X<long>;",
+                 cxxRecordDecl(isExplicitTemplateSpecialization())));
+  EXPECT_TRUE(
+      notMatches("template <typename T> void f(T t) {}"
+                 "template void f(int t); extern template void f(long t);",
+                 functionDecl(isExplicitTemplateSpecialization())));
 }
 
 TEST(IsExplicitTemplateSpecialization,
      DoesNotMatchImplicitTemplateInstantiations) {
-  EXPECT_TRUE(notMatches(
-    "template <typename T> class X {}; X<int> x;",
-    cxxRecordDecl(isExplicitTemplateSpecialization())));
-  EXPECT_TRUE(notMatches(
-    "template <typename T> void f(T t); void g() { f(10); }",
-    functionDecl(isExplicitTemplateSpecialization())));
+  EXPECT_TRUE(notMatches("template <typename T> class X {}; X<int> x;",
+                         cxxRecordDecl(isExplicitTemplateSpecialization())));
+  EXPECT_TRUE(
+      notMatches("template <typename T> void f(T t); void g() { f(10); }",
+                 functionDecl(isExplicitTemplateSpecialization())));
 }
 
-TEST(IsExplicitTemplateSpecialization,
-     MatchesExplicitTemplateSpecializations) {
-  EXPECT_TRUE(matches(
-    "template <typename T> class X {};"
-      "template<> class X<int> {};",
-    cxxRecordDecl(isExplicitTemplateSpecialization())));
-  EXPECT_TRUE(matches(
-    "template <typename T> void f(T t) {}"
-      "template<> void f(int t) {}",
-    functionDecl(isExplicitTemplateSpecialization())));
+TEST(IsExplicitTemplateSpecialization, MatchesExplicitTemplateSpecializations) {
+  EXPECT_TRUE(matches("template <typename T> class X {};"
+                      "template<> class X<int> {};",
+                      cxxRecordDecl(isExplicitTemplateSpecialization())));
+  EXPECT_TRUE(matches("template <typename T> void f(T t) {}"
+                      "template<> void f(int t) {}",
+                      functionDecl(isExplicitTemplateSpecialization())));
 }
 
 TEST(TypeMatching, MatchesNoReturn) {
@@ -2314,8 +2232,8 @@ TEST(TypeMatching, MatchesNoReturn) {
 
   EXPECT_TRUE(
       matches("struct S { [[noreturn]] S(); };", functionDecl(isNoReturn())));
-  EXPECT_TRUE(matches("struct S { [[noreturn]] S() {} };",
-                      functionDecl(isNoReturn())));
+  EXPECT_TRUE(
+      matches("struct S { [[noreturn]] S() {} };", functionDecl(isNoReturn())));
 
   // ---
 
@@ -2344,14 +2262,12 @@ TEST(TypeMatching, MatchesNoReturn) {
   // ---
 
   EXPECT_TRUE(matchesC("__attribute__((noreturn)) void func();",
-                      functionDecl(isNoReturn())));
+                       functionDecl(isNoReturn())));
   EXPECT_TRUE(matchesC("__attribute__((noreturn)) void func() {}",
-                      functionDecl(isNoReturn())));
+                       functionDecl(isNoReturn())));
 
-  EXPECT_TRUE(matchesC("_Noreturn void func();",
-                      functionDecl(isNoReturn())));
-  EXPECT_TRUE(matchesC("_Noreturn void func() {}",
-                      functionDecl(isNoReturn())));
+  EXPECT_TRUE(matchesC("_Noreturn void func();", functionDecl(isNoReturn())));
+  EXPECT_TRUE(matchesC("_Noreturn void func() {}", functionDecl(isNoReturn())));
 }
 
 TEST(TypeMatching, MatchesBool) {
@@ -2383,45 +2299,42 @@ TEST(TypeMatching, MatchesArrayTypes) {
   EXPECT_TRUE(notMatches("struct A {}; A a[7];",
                          arrayType(hasElementType(builtinType()))));
 
+  EXPECT_TRUE(matches("int const a[] = { 2, 3 };",
+                      qualType(arrayType(hasElementType(builtinType())))));
   EXPECT_TRUE(matches(
-    "int const a[] = { 2, 3 };",
-    qualType(arrayType(hasElementType(builtinType())))));
-  EXPECT_TRUE(matches(
-    "int const a[] = { 2, 3 };",
-    qualType(isConstQualified(), arrayType(hasElementType(builtinType())))));
-  EXPECT_TRUE(matches(
-    "typedef const int T; T x[] = { 1, 2 };",
-    qualType(isConstQualified(), arrayType())));
+      "int const a[] = { 2, 3 };",
+      qualType(isConstQualified(), arrayType(hasElementType(builtinType())))));
+  EXPECT_TRUE(matches("typedef const int T; T x[] = { 1, 2 };",
+                      qualType(isConstQualified(), arrayType())));
 
   EXPECT_TRUE(notMatches(
-    "int a[] = { 2, 3 };",
-    qualType(isConstQualified(), arrayType(hasElementType(builtinType())))));
-  EXPECT_TRUE(notMatches(
-    "int a[] = { 2, 3 };",
-    qualType(arrayType(hasElementType(isConstQualified(), builtinType())))));
+      "int a[] = { 2, 3 };",
+      qualType(isConstQualified(), arrayType(hasElementType(builtinType())))));
   EXPECT_TRUE(notMatches(
-    "int const a[] = { 2, 3 };",
-    qualType(arrayType(hasElementType(builtinType())),
-             unless(isConstQualified()))));
+      "int a[] = { 2, 3 };",
+      qualType(arrayType(hasElementType(isConstQualified(), builtinType())))));
+  EXPECT_TRUE(notMatches("int const a[] = { 2, 3 };",
+                         qualType(arrayType(hasElementType(builtinType())),
+                                  unless(isConstQualified()))));
 
-  EXPECT_TRUE(matches("int a[2];",
-                      constantArrayType(hasElementType(builtinType()))));
+  EXPECT_TRUE(
+      matches("int a[2];", constantArrayType(hasElementType(builtinType()))));
   EXPECT_TRUE(matches("const int a = 0;", qualType(isInteger())));
 }
 
 TEST(TypeMatching, DecayedType) {
-  EXPECT_TRUE(matches("void f(int i[]);", valueDecl(hasType(decayedType(hasDecayedType(pointerType()))))));
+  EXPECT_TRUE(
+      matches("void f(int i[]);",
+              valueDecl(hasType(decayedType(hasDecayedType(pointerType()))))));
   EXPECT_TRUE(notMatches("int i[7];", decayedType()));
 }
 
 TEST(TypeMatching, MatchesComplexTypes) {
   EXPECT_TRUE(matches("_Complex float f;", complexType()));
-  EXPECT_TRUE(matches(
-    "_Complex float f;",
-    complexType(hasElementType(builtinType()))));
-  EXPECT_TRUE(notMatches(
-    "_Complex float f;",
-    complexType(hasElementType(isInteger()))));
+  EXPECT_TRUE(
+      matches("_Complex float f;", complexType(hasElementType(builtinType()))));
+  EXPECT_TRUE(notMatches("_Complex float f;",
+                         complexType(hasElementType(isInteger()))));
 }
 
 TEST(NS, Anonymous) {
@@ -2482,38 +2395,38 @@ TEST(DeclarationMatcher, InStdNamespace) {
 
 TEST(EqualsBoundNodeMatcher, QualType) {
   EXPECT_TRUE(matches(
-    "int i = 1;", varDecl(hasType(qualType().bind("type")),
-                          hasInitializer(ignoringParenImpCasts(
-                            hasType(qualType(equalsBoundNode("type"))))))));
+      "int i = 1;", varDecl(hasType(qualType().bind("type")),
+                            hasInitializer(ignoringParenImpCasts(
+                                hasType(qualType(equalsBoundNode("type"))))))));
   EXPECT_TRUE(notMatches("int i = 1.f;",
                          varDecl(hasType(qualType().bind("type")),
                                  hasInitializer(ignoringParenImpCasts(hasType(
-                                   qualType(equalsBoundNode("type"))))))));
+                                     qualType(equalsBoundNode("type"))))))));
 }
 
 TEST(EqualsBoundNodeMatcher, NonMatchingTypes) {
   EXPECT_TRUE(notMatches(
-    "int i = 1;", varDecl(namedDecl(hasName("i")).bind("name"),
-                          hasInitializer(ignoringParenImpCasts(
-                            hasType(qualType(equalsBoundNode("type"))))))));
+      "int i = 1;", varDecl(namedDecl(hasName("i")).bind("name"),
+                            hasInitializer(ignoringParenImpCasts(
+                                hasType(qualType(equalsBoundNode("type"))))))));
 }
 
 TEST(EqualsBoundNodeMatcher, Stmt) {
   EXPECT_TRUE(
-    matches("void f() { if(true) {} }",
-            stmt(allOf(ifStmt().bind("if"),
-                       hasParent(stmt(has(stmt(equalsBoundNode("if")))))))));
+      matches("void f() { if(true) {} }",
+              stmt(allOf(ifStmt().bind("if"),
+                         hasParent(stmt(has(stmt(equalsBoundNode("if")))))))));
 
   EXPECT_TRUE(notMatches(
-    "void f() { if(true) { if (true) {} } }",
-    stmt(allOf(ifStmt().bind("if"), has(stmt(equalsBoundNode("if")))))));
+      "void f() { if(true) { if (true) {} } }",
+      stmt(allOf(ifStmt().bind("if"), has(stmt(equalsBoundNode("if")))))));
 }
 
 TEST(EqualsBoundNodeMatcher, Decl) {
   EXPECT_TRUE(matches(
-    "class X { class Y {}; };",
-    decl(allOf(recordDecl(hasName("::X::Y")).bind("record"),
-               hasParent(decl(has(decl(equalsBoundNode("record")))))))));
+      "class X { class Y {}; };",
+      decl(allOf(recordDecl(hasName("::X::Y")).bind("record"),
+                 hasParent(decl(has(decl(equalsBoundNode("record")))))))));
 
   EXPECT_TRUE(notMatches("class X { class Y {}; };",
                          decl(allOf(recordDecl(hasName("::X")).bind("record"),
@@ -2522,21 +2435,21 @@ TEST(EqualsBoundNodeMatcher, Decl) {
 
 TEST(EqualsBoundNodeMatcher, Type) {
   EXPECT_TRUE(matches(
-    "class X { int a; int b; };",
-    recordDecl(
-      has(fieldDecl(hasName("a"), hasType(type().bind("t")))),
-      has(fieldDecl(hasName("b"), hasType(type(equalsBoundNode("t"))))))));
+      "class X { int a; int b; };",
+      recordDecl(
+          has(fieldDecl(hasName("a"), hasType(type().bind("t")))),
+          has(fieldDecl(hasName("b"), hasType(type(equalsBoundNode("t"))))))));
 
   EXPECT_TRUE(notMatches(
-    "class X { int a; double b; };",
-    recordDecl(
-      has(fieldDecl(hasName("a"), hasType(type().bind("t")))),
-      has(fieldDecl(hasName("b"), hasType(type(equalsBoundNode("t"))))))));
+      "class X { int a; double b; };",
+      recordDecl(
+          has(fieldDecl(hasName("a"), hasType(type().bind("t")))),
+          has(fieldDecl(hasName("b"), hasType(type(equalsBoundNode("t"))))))));
 }
 
 TEST(EqualsBoundNodeMatcher, UsingForEachDescendant) {
   EXPECT_TRUE(matchAndVerifyResultTrue(
-    "int f() {"
+      "int f() {"
       "  if (1) {"
       "    int i = 9;"
       "  }"
@@ -2546,68 +2459,78 @@ TEST(EqualsBoundNodeMatcher, UsingForEachDescendant) {
       "  }"
       "  return 0;"
       "}",
-    // Look for variable declarations within functions whose type is the same
-    // as the function return type.
-    functionDecl(returns(qualType().bind("type")),
-                 forEachDescendant(varDecl(hasType(
-                   qualType(equalsBoundNode("type")))).bind("decl"))),
-    // Only i and j should match, not k.
-    std::make_unique<VerifyIdIsBoundTo<VarDecl>>("decl", 2)));
+      // Look for variable declarations within functions whose type is the same
+      // as the function return type.
+      functionDecl(
+          returns(qualType().bind("type")),
+          forEachDescendant(varDecl(hasType(qualType(equalsBoundNode("type"))))
+                                .bind("decl"))),
+      // Only i and j should match, not k.
+      std::make_unique<VerifyIdIsBoundTo<VarDecl>>("decl", 2)));
 }
 
 TEST(EqualsBoundNodeMatcher, FiltersMatchedCombinations) {
   EXPECT_TRUE(matchAndVerifyResultTrue(
-    "void f() {"
+      "void f() {"
       "  int x;"
       "  double d;"
       "  x = d + x - d + x;"
       "}",
-    functionDecl(
-      hasName("f"), forEachDescendant(varDecl().bind("d")),
-      forEachDescendant(declRefExpr(to(decl(equalsBoundNode("d")))))),
-    std::make_unique<VerifyIdIsBoundTo<VarDecl>>("d", 5)));
+      functionDecl(
+          hasName("f"), forEachDescendant(varDecl().bind("d")),
+          forEachDescendant(declRefExpr(to(decl(equalsBoundNode("d")))))),
+      std::make_unique<VerifyIdIsBoundTo<VarDecl>>("d", 5)));
 }
 
 TEST(EqualsBoundNodeMatcher, UnlessDescendantsOfAncestorsMatch) {
   EXPECT_TRUE(matchAndVerifyResultTrue(
-    "struct StringRef { int size() const; const char* data() const; };"
+      "struct StringRef { int size() const; const char* data() const; };"
       "void f(StringRef v) {"
       "  v.data();"
       "}",
-    cxxMemberCallExpr(
-      callee(cxxMethodDecl(hasName("data"))),
-      on(declRefExpr(to(
-        varDecl(hasType(recordDecl(hasName("StringRef")))).bind("var")))),
-      unless(hasAncestor(stmt(hasDescendant(cxxMemberCallExpr(
-        callee(cxxMethodDecl(anyOf(hasName("size"), hasName("length")))),
-        on(declRefExpr(to(varDecl(equalsBoundNode("var")))))))))))
-      .bind("data"),
-    std::make_unique<VerifyIdIsBoundTo<Expr>>("data", 1)));
+      cxxMemberCallExpr(
+          callee(cxxMethodDecl(hasName("data"))),
+          on(declRefExpr(to(
+              varDecl(hasType(recordDecl(hasName("StringRef")))).bind("var")))),
+          unless(hasAncestor(stmt(hasDescendant(cxxMemberCallExpr(
+              callee(cxxMethodDecl(anyOf(hasName("size"), hasName("length")))),
+              on(declRefExpr(to(varDecl(equalsBoundNode("var")))))))))))
+          .bind("data"),
+      std::make_unique<VerifyIdIsBoundTo<Expr>>("data", 1)));
 
   EXPECT_FALSE(matches(
-    "struct StringRef { int size() const; const char* data() const; };"
+      "struct StringRef { int size() const; const char* data() const; };"
       "void f(StringRef v) {"
       "  v.data();"
       "  v.size();"
       "}",
-    cxxMemberCallExpr(
-      callee(cxxMethodDecl(hasName("data"))),
-      on(declRefExpr(to(
-        varDecl(hasType(recordDecl(hasName("StringRef")))).bind("var")))),
-      unless(hasAncestor(stmt(hasDescendant(cxxMemberCallExpr(
-        callee(cxxMethodDecl(anyOf(hasName("size"), hasName("length")))),
-        on(declRefExpr(to(varDecl(equalsBoundNode("var")))))))))))
-      .bind("data")));
+      cxxMemberCallExpr(
+          callee(cxxMethodDecl(hasName("data"))),
+          on(declRefExpr(to(
+              varDecl(hasType(recordDecl(hasName("StringRef")))).bind("var")))),
+          unless(hasAncestor(stmt(hasDescendant(cxxMemberCallExpr(
+              callee(cxxMethodDecl(anyOf(hasName("size"), hasName("length")))),
+              on(declRefExpr(to(varDecl(equalsBoundNode("var")))))))))))
+          .bind("data")));
 }
 
 TEST(NullPointerConstants, Basic) {
   EXPECT_TRUE(matches("#define NULL ((void *)0)\n"
-                        "void *v1 = NULL;", expr(nullPointerConstant())));
+                      "void *v1 = NULL;",
+                      expr(nullPointerConstant())));
   EXPECT_TRUE(matches("void *v2 = nullptr;", expr(nullPointerConstant())));
   EXPECT_TRUE(matches("void *v3 = __null;", expr(nullPointerConstant())));
   EXPECT_TRUE(matches("char *cp = (char *)0;", expr(nullPointerConstant())));
   EXPECT_TRUE(matches("int *ip = 0;", expr(nullPointerConstant())));
   EXPECT_TRUE(matches("int i = 0;", expr(nullPointerConstant())));
+  const char kTest[] = R"(
+    template <typename T>
+    struct MyTemplate {
+      MyTemplate() : field_(__null) {}
+      T* field_;
+    };
+  )";
+  EXPECT_TRUE(matches(kTest, expr(nullPointerConstant())));
 }
 
 TEST(HasExternalFormalLinkage, Basic) {
@@ -2627,10 +2550,10 @@ TEST(HasExternalFormalLinkage, Basic) {
 }
 
 TEST(HasDefaultArgument, Basic) {
-  EXPECT_TRUE(matches("void x(int val = 0) {}",
-                      parmVarDecl(hasDefaultArgument())));
-  EXPECT_TRUE(notMatches("void x(int val) {}",
-                      parmVarDecl(hasDefaultArgument())));
+  EXPECT_TRUE(
+      matches("void x(int val = 0) {}", parmVarDecl(hasDefaultArgument())));
+  EXPECT_TRUE(
+      notMatches("void x(int val) {}", parmVarDecl(hasDefaultArgument())));
 }
 
 TEST(IsAtPosition, Basic) {
@@ -2683,24 +2606,18 @@ TEST(HasArraySize, Basic) {
 }
 
 TEST(HasDefinition, MatchesStructDefinition) {
-  EXPECT_TRUE(matches("struct x {};",
-                      cxxRecordDecl(hasDefinition())));
-  EXPECT_TRUE(notMatches("struct x;",
-                      cxxRecordDecl(hasDefinition())));
+  EXPECT_TRUE(matches("struct x {};", cxxRecordDecl(hasDefinition())));
+  EXPECT_TRUE(notMatches("struct x;", cxxRecordDecl(hasDefinition())));
 }
 
 TEST(HasDefinition, MatchesClassDefinition) {
-  EXPECT_TRUE(matches("class x {};",
-                      cxxRecordDecl(hasDefinition())));
-  EXPECT_TRUE(notMatches("class x;",
-                      cxxRecordDecl(hasDefinition())));
+  EXPECT_TRUE(matches("class x {};", cxxRecordDecl(hasDefinition())));
+  EXPECT_TRUE(notMatches("class x;", cxxRecordDecl(hasDefinition())));
 }
 
 TEST(HasDefinition, MatchesUnionDefinition) {
-  EXPECT_TRUE(matches("union x {};",
-                      cxxRecordDecl(hasDefinition())));
-  EXPECT_TRUE(notMatches("union x;",
-                      cxxRecordDecl(hasDefinition())));
+  EXPECT_TRUE(matches("union x {};", cxxRecordDecl(hasDefinition())));
+  EXPECT_TRUE(notMatches("union x;", cxxRecordDecl(hasDefinition())));
 }
 
 TEST(IsScopedEnum, MatchesScopedEnum) {
@@ -2719,19 +2636,19 @@ TEST(HasTrailingReturn, MatchesTrailingReturn) {
   EXPECT_TRUE(matches("auto Y() -> int { return 0; }",
                       functionDecl(hasTrailingReturn())));
   EXPECT_TRUE(matches("auto X() -> int;", functionDecl(hasTrailingReturn())));
-  EXPECT_TRUE(notMatches("int X() { return 0; }",
-                      functionDecl(hasTrailingReturn())));
+  EXPECT_TRUE(
+      notMatches("int X() { return 0; }", functionDecl(hasTrailingReturn())));
   EXPECT_TRUE(notMatches("int X();", functionDecl(hasTrailingReturn())));
   EXPECT_TRUE(notMatchesC("void X();", functionDecl(hasTrailingReturn())));
 }
 
 TEST(HasTrailingReturn, MatchesLambdaTrailingReturn) {
   EXPECT_TRUE(matches(
-          "auto lambda2 = [](double x, double y) -> double {return x + y;};",
-          functionDecl(hasTrailingReturn())));
-  EXPECT_TRUE(notMatches(
-          "auto lambda2 = [](double x, double y) {return x + y;};",
-          functionDecl(hasTrailingReturn())));
+      "auto lambda2 = [](double x, double y) -> double {return x + y;};",
+      functionDecl(hasTrailingReturn())));
+  EXPECT_TRUE(
+      notMatches("auto lambda2 = [](double x, double y) {return x + y;};",
+                 functionDecl(hasTrailingReturn())));
 }
 
 TEST(IsAssignmentOperator, Basic) {
@@ -2764,23 +2681,15 @@ TEST(IsComparisonOperator, Basic) {
 }
 
 TEST(HasInit, Basic) {
-  EXPECT_TRUE(
-    matches("int x{0};",
-            initListExpr(hasInit(0, expr()))));
-  EXPECT_FALSE(
-    matches("int x{0};",
-            initListExpr(hasInit(1, expr()))));
-  EXPECT_FALSE(
-    matches("int x;",
-            initListExpr(hasInit(0, expr()))));
+  EXPECT_TRUE(matches("int x{0};", initListExpr(hasInit(0, expr()))));
+  EXPECT_FALSE(matches("int x{0};", initListExpr(hasInit(1, expr()))));
+  EXPECT_FALSE(matches("int x;", initListExpr(hasInit(0, expr()))));
 }
 
 TEST(Matcher, isMain) {
-  EXPECT_TRUE(
-    matches("int main() {}", functionDecl(isMain())));
+  EXPECT_TRUE(matches("int main() {}", functionDecl(isMain())));
 
-  EXPECT_TRUE(
-    notMatches("int main2() {}", functionDecl(isMain())));
+  EXPECT_TRUE(notMatches("int main2() {}", functionDecl(isMain())));
 }
 
 TEST(OMPExecutableDirective, isStandaloneDirective) {
@@ -2859,11 +2768,18 @@ void x() {
   EXPECT_TRUE(matchesWithOpenMP(Source3, Matcher));
 
   StringRef Source4 = R"(
+void x() {
+#pragma omp parallel default(firstprivate)
+;
+})";
+  EXPECT_TRUE(matchesWithOpenMP51(Source4, Matcher));
+
+  StringRef Source5 = R"(
 void x(int x) {
 #pragma omp parallel num_threads(x)
 ;
 })";
-  EXPECT_TRUE(matchesWithOpenMP(Source4, Matcher));
+  EXPECT_TRUE(matchesWithOpenMP(Source5, Matcher));
 }
 
 TEST(OMPDefaultClause, isNoneKind) {
@@ -2899,10 +2815,17 @@ void x() {
 
   StringRef Source4 = R"(
 void x(int x) {
+#pragma omp parallel default(firstprivate)
+;
+})";
+  EXPECT_TRUE(notMatchesWithOpenMP51(Source4, Matcher));
+
+  const std::string Source5 = R"(
+void x(int x) {
 #pragma omp parallel num_threads(x)
 ;
 })";
-  EXPECT_TRUE(notMatchesWithOpenMP(Source4, Matcher));
+  EXPECT_TRUE(notMatchesWithOpenMP(Source5, Matcher));
 }
 
 TEST(OMPDefaultClause, isSharedKind) {
@@ -2938,10 +2861,63 @@ void x() {
 
   StringRef Source4 = R"(
 void x(int x) {
+#pragma omp parallel default(firstprivate)
+;
+})";
+  EXPECT_TRUE(notMatchesWithOpenMP51(Source4, Matcher));
+
+  const std::string Source5 = R"(
+void x(int x) {
+#pragma omp parallel num_threads(x)
+;
+})";
+  EXPECT_TRUE(notMatchesWithOpenMP(Source5, Matcher));
+}
+
+TEST(OMPDefaultClause, isFirstPrivateKind) {
+  auto Matcher = ompExecutableDirective(
+      hasAnyClause(ompDefaultClause(isFirstPrivateKind())));
+
+  const std::string Source0 = R"(
+void x() {
+;
+})";
+  EXPECT_TRUE(notMatchesWithOpenMP(Source0, Matcher));
+
+  const std::string Source1 = R"(
+void x() {
+#pragma omp parallel
+;
+})";
+  EXPECT_TRUE(notMatchesWithOpenMP(Source1, Matcher));
+
+  const std::string Source2 = R"(
+void x() {
+#pragma omp parallel default(shared)
+;
+})";
+  EXPECT_TRUE(notMatchesWithOpenMP(Source2, Matcher));
+
+  const std::string Source3 = R"(
+void x() {
+#pragma omp parallel default(none)
+;
+})";
+  EXPECT_TRUE(notMatchesWithOpenMP(Source3, Matcher));
+
+  const std::string Source4 = R"(
+void x(int x) {
+#pragma omp parallel default(firstprivate)
+;
+})";
+  EXPECT_TRUE(matchesWithOpenMP51(Source4, Matcher));
+
+  const std::string Source5 = R"(
+void x(int x) {
 #pragma omp parallel num_threads(x)
 ;
 })";
-  EXPECT_TRUE(notMatchesWithOpenMP(Source4, Matcher));
+  EXPECT_TRUE(notMatchesWithOpenMP(Source5, Matcher));
 }
 
 TEST(OMPExecutableDirective, isAllowedToContainClauseKind) {
@@ -2976,24 +2952,31 @@ void x() {
   EXPECT_TRUE(matchesWithOpenMP(Source3, Matcher));
 
   StringRef Source4 = R"(
+void x() {
+#pragma omp parallel default(firstprivate)
+;
+})";
+  EXPECT_TRUE(matchesWithOpenMP51(Source4, Matcher));
+
+  StringRef Source5 = R"(
 void x(int x) {
 #pragma omp parallel num_threads(x)
 ;
 })";
-  EXPECT_TRUE(matchesWithOpenMP(Source4, Matcher));
+  EXPECT_TRUE(matchesWithOpenMP(Source5, Matcher));
 
-  StringRef Source5 = R"(
+  StringRef Source6 = R"(
 void x() {
 #pragma omp taskyield
 })";
-  EXPECT_TRUE(notMatchesWithOpenMP(Source5, Matcher));
+  EXPECT_TRUE(notMatchesWithOpenMP(Source6, Matcher));
 
-  StringRef Source6 = R"(
+  StringRef Source7 = R"(
 void x() {
 #pragma omp task
 ;
 })";
-  EXPECT_TRUE(matchesWithOpenMP(Source6, Matcher));
+  EXPECT_TRUE(matchesWithOpenMP(Source7, Matcher));
 }
 
 TEST(HasAnyBase, DirectBase) {
@@ -3117,5 +3100,44 @@ TEST(IsVirtual, NoVirtualBase) {
                          cxxRecordDecl(hasAnyBase(isVirtual()))));
 }
 
+TEST(BaseSpecifier, hasDirectBase) {
+  EXPECT_TRUE(matches(
+      R"cc(
+    class Base {};
+    class Derived : Base{};
+    )cc",
+      cxxRecordDecl(hasName("Derived"),
+                    hasDirectBase(hasType(cxxRecordDecl(hasName("Base")))))));
+
+  StringRef MultiDerived = R"cc(
+    class Base {};
+    class Base2 {};
+    class Derived : Base, Base2{};
+    )cc";
+
+  EXPECT_TRUE(matches(
+      MultiDerived,
+      cxxRecordDecl(hasName("Derived"),
+                    hasDirectBase(hasType(cxxRecordDecl(hasName("Base")))))));
+  EXPECT_TRUE(matches(
+      MultiDerived,
+      cxxRecordDecl(hasName("Derived"),
+                    hasDirectBase(hasType(cxxRecordDecl(hasName("Base2")))))));
+
+  StringRef Indirect = R"cc(
+    class Base {};
+    class Intermediate : Base {};
+    class Derived : Intermediate{};
+    )cc";
+
+  EXPECT_TRUE(
+      matches(Indirect, cxxRecordDecl(hasName("Derived"),
+                                      hasDirectBase(hasType(cxxRecordDecl(
+                                          hasName("Intermediate")))))));
+  EXPECT_TRUE(notMatches(
+      Indirect,
+      cxxRecordDecl(hasName("Derived"),
+                    hasDirectBase(hasType(cxxRecordDecl(hasName("Base")))))));
+}
 } // namespace ast_matchers
 } // namespace clang
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
index 1d262fc1a6558..895c8ae48adc1 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
@@ -18,52 +18,47 @@
 namespace clang {
 namespace ast_matchers {
 
-TEST(Finder, DynamicOnlyAcceptsSomeMatchers) {
-  MatchFinder Finder;
-  EXPECT_TRUE(Finder.addDynamicMatcher(decl(), nullptr));
-  EXPECT_TRUE(Finder.addDynamicMatcher(callExpr(), nullptr));
-  EXPECT_TRUE(Finder.addDynamicMatcher(constantArrayType(hasSize(42)),
-                                       nullptr));
-
-  // Do not accept non-toplevel matchers.
-  EXPECT_FALSE(Finder.addDynamicMatcher(isMain(), nullptr));
-  EXPECT_FALSE(Finder.addDynamicMatcher(hasName("x"), nullptr));
-}
-
-TEST(Decl, MatchesDeclarations) {
+TEST_P(ASTMatchersTest, Decl_CXX) {
+  if (!GetParam().isCXX()) {
+    // FIXME: Add a test for `decl()` that does not depend on C++.
+    return;
+  }
   EXPECT_TRUE(notMatches("", decl(usingDecl())));
-  EXPECT_TRUE(matches("namespace x { class X {}; } using x::X;",
-                      decl(usingDecl())));
+  EXPECT_TRUE(
+      matches("namespace x { class X {}; } using x::X;", decl(usingDecl())));
 }
 
-TEST(NameableDeclaration, MatchesVariousDecls) {
+TEST_P(ASTMatchersTest, NameableDeclaration_MatchesVariousDecls) {
   DeclarationMatcher NamedX = namedDecl(hasName("X"));
   EXPECT_TRUE(matches("typedef int X;", NamedX));
   EXPECT_TRUE(matches("int X;", NamedX));
-  EXPECT_TRUE(matches("class foo { virtual void X(); };", NamedX));
-  EXPECT_TRUE(matches("void foo() try { } catch(int X) { }", NamedX));
   EXPECT_TRUE(matches("void foo() { int X; }", NamedX));
-  EXPECT_TRUE(matches("namespace X { }", NamedX));
   EXPECT_TRUE(matches("enum X { A, B, C };", NamedX));
 
   EXPECT_TRUE(notMatches("#define X 1", NamedX));
 }
 
-TEST(NameableDeclaration, REMatchesVariousDecls) {
+TEST_P(ASTMatchersTest, NamedDecl_CXX) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  DeclarationMatcher NamedX = namedDecl(hasName("X"));
+  EXPECT_TRUE(matches("class foo { virtual void X(); };", NamedX));
+  EXPECT_TRUE(matches("void foo() try { } catch(int X) { }", NamedX));
+  EXPECT_TRUE(matches("namespace X { }", NamedX));
+}
+
+TEST_P(ASTMatchersTest, MatchesNameRE) {
   DeclarationMatcher NamedX = namedDecl(matchesName("::X"));
   EXPECT_TRUE(matches("typedef int Xa;", NamedX));
   EXPECT_TRUE(matches("int Xb;", NamedX));
-  EXPECT_TRUE(matches("class foo { virtual void Xc(); };", NamedX));
-  EXPECT_TRUE(matches("void foo() try { } catch(int Xdef) { }", NamedX));
   EXPECT_TRUE(matches("void foo() { int Xgh; }", NamedX));
-  EXPECT_TRUE(matches("namespace Xij { }", NamedX));
   EXPECT_TRUE(matches("enum X { A, B, C };", NamedX));
 
   EXPECT_TRUE(notMatches("#define Xkl 1", NamedX));
 
   DeclarationMatcher StartsWithNo = namedDecl(matchesName("::no"));
   EXPECT_TRUE(matches("int no_foo;", StartsWithNo));
-  EXPECT_TRUE(matches("class foo { virtual void nobody(); };", StartsWithNo));
 
   DeclarationMatcher Abc = namedDecl(matchesName("a.*b.*c"));
   EXPECT_TRUE(matches("int abc;", Abc));
@@ -74,18 +69,36 @@ TEST(NameableDeclaration, REMatchesVariousDecls) {
   DeclarationMatcher StartsWithK = namedDecl(matchesName(":k[^:]*$"));
   EXPECT_TRUE(matches("int k;", StartsWithK));
   EXPECT_TRUE(matches("int kAbc;", StartsWithK));
+}
+
+TEST_P(ASTMatchersTest, MatchesNameRE_CXX) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  DeclarationMatcher NamedX = namedDecl(matchesName("::X"));
+  EXPECT_TRUE(matches("class foo { virtual void Xc(); };", NamedX));
+  EXPECT_TRUE(matches("void foo() try { } catch(int Xdef) { }", NamedX));
+  EXPECT_TRUE(matches("namespace Xij { }", NamedX));
+
+  DeclarationMatcher StartsWithNo = namedDecl(matchesName("::no"));
+  EXPECT_TRUE(matches("class foo { virtual void nobody(); };", StartsWithNo));
+
+  DeclarationMatcher StartsWithK = namedDecl(matchesName(":k[^:]*$"));
   EXPECT_TRUE(matches("namespace x { int kTest; }", StartsWithK));
   EXPECT_TRUE(matches("class C { int k; };", StartsWithK));
   EXPECT_TRUE(notMatches("class C { int ckc; };", StartsWithK));
-}
+  EXPECT_TRUE(notMatches("int K;", StartsWithK));
 
-TEST(DeclarationMatcher, MatchClass) {
-  DeclarationMatcher ClassMatcher(recordDecl());
+  DeclarationMatcher StartsWithKIgnoreCase =
+      namedDecl(matchesName(":k[^:]*$", llvm::Regex::IgnoreCase));
+  EXPECT_TRUE(matches("int k;", StartsWithKIgnoreCase));
+  EXPECT_TRUE(matches("int K;", StartsWithKIgnoreCase));
+}
 
-  // This passes on Windows only because we explicitly pass -target
-  // i386-unknown-unknown.  If we were to compile with the default target
-  // triple, we'd want to EXPECT_TRUE if it's Win32 or MSVC.
-  EXPECT_FALSE(matches("", ClassMatcher));
+TEST_P(ASTMatchersTest, DeclarationMatcher_MatchClass) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
 
   DeclarationMatcher ClassX = recordDecl(recordDecl(hasName("X")));
   EXPECT_TRUE(matches("class X;", ClassX));
@@ -94,72 +107,103 @@ TEST(DeclarationMatcher, MatchClass) {
   EXPECT_TRUE(notMatches("", ClassX));
 }
 
-TEST(DeclarationMatcher, translationUnitDecl) {
+TEST_P(ASTMatchersTest, TranslationUnitDecl) {
+  if (!GetParam().isCXX()) {
+    // FIXME: Add a test for `translationUnitDecl()` that does not depend on
+    // C++.
+    return;
+  }
   StringRef Code = "int MyVar1;\n"
                    "namespace NameSpace {\n"
                    "int MyVar2;\n"
                    "}  // namespace NameSpace\n";
   EXPECT_TRUE(matches(
-    Code, varDecl(hasName("MyVar1"), hasDeclContext(translationUnitDecl()))));
+      Code, varDecl(hasName("MyVar1"), hasDeclContext(translationUnitDecl()))));
   EXPECT_FALSE(matches(
-    Code, varDecl(hasName("MyVar2"), hasDeclContext(translationUnitDecl()))));
+      Code, varDecl(hasName("MyVar2"), hasDeclContext(translationUnitDecl()))));
   EXPECT_TRUE(matches(
-    Code,
-    varDecl(hasName("MyVar2"),
-            hasDeclContext(decl(hasDeclContext(translationUnitDecl()))))));
+      Code,
+      varDecl(hasName("MyVar2"),
+              hasDeclContext(decl(hasDeclContext(translationUnitDecl()))))));
 }
 
-TEST(DeclarationMatcher, LinkageSpecification) {
+TEST_P(ASTMatchersTest, LinkageSpecDecl) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("extern \"C\" { void foo() {}; }", linkageSpecDecl()));
   EXPECT_TRUE(notMatches("void foo() {};", linkageSpecDecl()));
 }
 
-TEST(ClassTemplate, DoesNotMatchClass) {
+TEST_P(ASTMatchersTest, ClassTemplateDecl_DoesNotMatchClass) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   DeclarationMatcher ClassX = classTemplateDecl(hasName("X"));
   EXPECT_TRUE(notMatches("class X;", ClassX));
   EXPECT_TRUE(notMatches("class X {};", ClassX));
 }
 
-TEST(ClassTemplate, MatchesClassTemplate) {
+TEST_P(ASTMatchersTest, ClassTemplateDecl_MatchesClassTemplate) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   DeclarationMatcher ClassX = classTemplateDecl(hasName("X"));
   EXPECT_TRUE(matches("template<typename T> class X {};", ClassX));
   EXPECT_TRUE(matches("class Z { template<class T> class X {}; };", ClassX));
 }
 
-TEST(ClassTemplate, DoesNotMatchClassTemplateExplicitSpecialization) {
-  EXPECT_TRUE(notMatches("template<typename T> class X { };"
-                           "template<> class X<int> { int a; };",
-                         classTemplateDecl(hasName("X"),
-                                           hasDescendant(fieldDecl(hasName("a"))))));
+TEST_P(ASTMatchersTest,
+       ClassTemplateDecl_DoesNotMatchClassTemplateExplicitSpecialization) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(notMatches(
+      "template<typename T> class X { };"
+      "template<> class X<int> { int a; };",
+      classTemplateDecl(hasName("X"), hasDescendant(fieldDecl(hasName("a"))))));
 }
 
-TEST(ClassTemplate, DoesNotMatchClassTemplatePartialSpecialization) {
-  EXPECT_TRUE(notMatches("template<typename T, typename U> class X { };"
-                           "template<typename T> class X<T, int> { int a; };",
-                         classTemplateDecl(hasName("X"),
-                                           hasDescendant(fieldDecl(hasName("a"))))));
+TEST_P(ASTMatchersTest,
+       ClassTemplateDecl_DoesNotMatchClassTemplatePartialSpecialization) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(notMatches(
+      "template<typename T, typename U> class X { };"
+      "template<typename T> class X<T, int> { int a; };",
+      classTemplateDecl(hasName("X"), hasDescendant(fieldDecl(hasName("a"))))));
 }
 
-TEST(DeclarationMatcher, MatchCudaDecl) {
+TEST(ASTMatchersTestCUDA, CUDAKernelCallExpr) {
   EXPECT_TRUE(matchesWithCuda("__global__ void f() { }"
-                                "void g() { f<<<1, 2>>>(); }",
+                              "void g() { f<<<1, 2>>>(); }",
                               cudaKernelCallExpr()));
+  EXPECT_TRUE(notMatchesWithCuda("void f() {}", cudaKernelCallExpr()));
+}
+
+TEST(ASTMatchersTestCUDA, HasAttrCUDA) {
   EXPECT_TRUE(matchesWithCuda("__attribute__((device)) void f() {}",
                               hasAttr(clang::attr::CUDADevice)));
-  EXPECT_TRUE(notMatchesWithCuda("void f() {}",
-                                 cudaKernelCallExpr()));
   EXPECT_FALSE(notMatchesWithCuda("__attribute__((global)) void f() {}",
                                   hasAttr(clang::attr::CUDAGlobal)));
 }
 
-TEST(ValueDecl, Matches) {
+TEST_P(ASTMatchersTest, ValueDecl) {
+  if (!GetParam().isCXX()) {
+    // FIXME: Fix this test in non-C++ language modes.
+    return;
+  }
   EXPECT_TRUE(matches("enum EnumType { EnumValue };",
                       valueDecl(hasType(asString("enum EnumType")))));
   EXPECT_TRUE(matches("void FunctionDecl();",
                       valueDecl(hasType(asString("void (void)")))));
 }
 
-TEST(FriendDecl, Matches) {
+TEST_P(ASTMatchersTest, FriendDecl) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("class Y { friend class X; };",
                       friendDecl(hasType(asString("class X")))));
   EXPECT_TRUE(matches("class Y { friend class X; };",
@@ -169,43 +213,69 @@ TEST(FriendDecl, Matches) {
                       functionDecl(hasName("f"), hasParent(friendDecl()))));
 }
 
-TEST(Enum, DoesNotMatchClasses) {
+TEST_P(ASTMatchersTest, EnumDecl_DoesNotMatchClasses) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(notMatches("class X {};", enumDecl(hasName("X"))));
 }
 
-TEST(Enum, MatchesEnums) {
+TEST_P(ASTMatchersTest, EnumDecl_MatchesEnums) {
+  if (!GetParam().isCXX()) {
+    // FIXME: Fix this test in non-C++ language modes.
+    return;
+  }
   EXPECT_TRUE(matches("enum X {};", enumDecl(hasName("X"))));
 }
 
-TEST(EnumConstant, Matches) {
+TEST_P(ASTMatchersTest, EnumConstantDecl) {
+  if (!GetParam().isCXX()) {
+    // FIXME: Fix this test in non-C++ language modes.
+    return;
+  }
   DeclarationMatcher Matcher = enumConstantDecl(hasName("A"));
   EXPECT_TRUE(matches("enum X{ A };", Matcher));
   EXPECT_TRUE(notMatches("enum X{ B };", Matcher));
   EXPECT_TRUE(notMatches("enum X {};", Matcher));
 }
 
-TEST(TagDecl, MatchesTagDecls) {
+TEST_P(ASTMatchersTest, TagDecl) {
+  if (!GetParam().isCXX()) {
+    // FIXME: Fix this test in non-C++ language modes.
+    return;
+  }
   EXPECT_TRUE(matches("struct X {};", tagDecl(hasName("X"))));
-  EXPECT_TRUE(matches("class C {};", tagDecl(hasName("C"))));
   EXPECT_TRUE(matches("union U {};", tagDecl(hasName("U"))));
   EXPECT_TRUE(matches("enum E {};", tagDecl(hasName("E"))));
 }
 
-TEST(Matcher, UnresolvedLookupExpr) {
-  // FIXME: The test is known to be broken on Windows with delayed template
-  // parsing.
-  EXPECT_TRUE(matchesConditionally("template<typename T>"
-                                   "T foo() { T a; return a; }"
-                                   "template<typename T>"
-                                   "void bar() {"
-                                   "  foo<T>();"
-                                   "}",
-                                   unresolvedLookupExpr(),
-                                   /*ExpectMatch=*/true,
-                                   {"-fno-delayed-template-parsing"}));
+TEST_P(ASTMatchersTest, TagDecl_CXX) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(matches("class C {};", tagDecl(hasName("C"))));
 }
 
-TEST(Matcher, ADLCall) {
+TEST_P(ASTMatchersTest, UnresolvedLookupExpr) {
+  if (!GetParam().isCXX() || GetParam().hasDelayedTemplateParsing()) {
+    // FIXME: Fix this test to work with delayed template parsing.
+    return;
+  }
+
+  EXPECT_TRUE(matches("template<typename T>"
+                      "T foo() { T a; return a; }"
+                      "template<typename T>"
+                      "void bar() {"
+                      "  foo<T>();"
+                      "}",
+                      unresolvedLookupExpr()));
+}
+
+TEST_P(ASTMatchersTest, UsesADL) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+
   StatementMatcher ADLMatch = callExpr(usesADL());
   StatementMatcher ADLMatchOper = cxxOperatorCallExpr(usesADL());
   StringRef NS_Str = R"cpp(
@@ -237,121 +307,140 @@ TEST(Matcher, ADLCall) {
   EXPECT_TRUE(notMatches(MkStr("NS::X x; NS::operator+(x, x);"), ADLMatch));
 }
 
-TEST(Matcher, Call) {
+TEST_P(ASTMatchersTest, CallExpr_CXX) {
+  if (!GetParam().isCXX()) {
+    // FIXME: Add a test for `callExpr()` that does not depend on C++.
+    return;
+  }
   // FIXME: Do we want to overload Call() to directly take
   // Matcher<Decl>, too?
   StatementMatcher MethodX =
-    callExpr(hasDeclaration(cxxMethodDecl(hasName("x"))));
+      callExpr(hasDeclaration(cxxMethodDecl(hasName("x"))));
 
   EXPECT_TRUE(matches("class Y { void x() { x(); } };", MethodX));
   EXPECT_TRUE(notMatches("class Y { void x() {} };", MethodX));
 
   StatementMatcher MethodOnY =
-    cxxMemberCallExpr(on(hasType(recordDecl(hasName("Y")))));
+      cxxMemberCallExpr(on(hasType(recordDecl(hasName("Y")))));
 
-  EXPECT_TRUE(
-    matches("class Y { public: void x(); }; void z() { Y y; y.x(); }",
-            MethodOnY));
-  EXPECT_TRUE(
-    matches("class Y { public: void x(); }; void z(Y &y) { y.x(); }",
-            MethodOnY));
-  EXPECT_TRUE(
-    notMatches("class Y { public: void x(); }; void z(Y *&y) { y->x(); }",
-               MethodOnY));
-  EXPECT_TRUE(
-    notMatches("class Y { public: void x(); }; void z(Y y[]) { y->x(); }",
-               MethodOnY));
-  EXPECT_TRUE(
-    notMatches("class Y { public: void x(); }; void z() { Y *y; y->x(); }",
-               MethodOnY));
+  EXPECT_TRUE(matches("class Y { public: void x(); }; void z() { Y y; y.x(); }",
+                      MethodOnY));
+  EXPECT_TRUE(matches("class Y { public: void x(); }; void z(Y &y) { y.x(); }",
+                      MethodOnY));
+  EXPECT_TRUE(notMatches(
+      "class Y { public: void x(); }; void z(Y *&y) { y->x(); }", MethodOnY));
+  EXPECT_TRUE(notMatches(
+      "class Y { public: void x(); }; void z(Y y[]) { y->x(); }", MethodOnY));
+  EXPECT_TRUE(notMatches(
+      "class Y { public: void x(); }; void z() { Y *y; y->x(); }", MethodOnY));
 
   StatementMatcher MethodOnYPointer =
-    cxxMemberCallExpr(on(hasType(pointsTo(recordDecl(hasName("Y"))))));
+      cxxMemberCallExpr(on(hasType(pointsTo(recordDecl(hasName("Y"))))));
 
   EXPECT_TRUE(
-    matches("class Y { public: void x(); }; void z() { Y *y; y->x(); }",
-            MethodOnYPointer));
+      matches("class Y { public: void x(); }; void z() { Y *y; y->x(); }",
+              MethodOnYPointer));
   EXPECT_TRUE(
-    matches("class Y { public: void x(); }; void z(Y *&y) { y->x(); }",
-            MethodOnYPointer));
+      matches("class Y { public: void x(); }; void z(Y *&y) { y->x(); }",
+              MethodOnYPointer));
   EXPECT_TRUE(
-    matches("class Y { public: void x(); }; void z(Y y[]) { y->x(); }",
-            MethodOnYPointer));
+      matches("class Y { public: void x(); }; void z(Y y[]) { y->x(); }",
+              MethodOnYPointer));
   EXPECT_TRUE(
-    notMatches("class Y { public: void x(); }; void z() { Y y; y.x(); }",
-               MethodOnYPointer));
+      notMatches("class Y { public: void x(); }; void z() { Y y; y.x(); }",
+                 MethodOnYPointer));
   EXPECT_TRUE(
-    notMatches("class Y { public: void x(); }; void z(Y &y) { y.x(); }",
-               MethodOnYPointer));
+      notMatches("class Y { public: void x(); }; void z(Y &y) { y.x(); }",
+                 MethodOnYPointer));
 }
-TEST(Matcher, Lambda) {
-  EXPECT_TRUE(matches("auto f = [] (int i) { return i; };",
-                      lambdaExpr()));
+
+TEST_P(ASTMatchersTest, LambdaExpr) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
+  EXPECT_TRUE(matches("auto f = [] (int i) { return i; };", lambdaExpr()));
+}
+
+TEST_P(ASTMatchersTest, CXXForRangeStmt) {
+  EXPECT_TRUE(
+      notMatches("void f() { for (int i; i<5; ++i); }", cxxForRangeStmt()));
 }
 
-TEST(Matcher, ForRange) {
+TEST_P(ASTMatchersTest, CXXForRangeStmt_CXX11) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
   EXPECT_TRUE(matches("int as[] = { 1, 2, 3 };"
-                        "void f() { for (auto &a : as); }",
+                      "void f() { for (auto &a : as); }",
                       cxxForRangeStmt()));
-  EXPECT_TRUE(notMatches("void f() { for (int i; i<5; ++i); }",
-                         cxxForRangeStmt()));
 }
 
-TEST(Matcher, SubstNonTypeTemplateParm) {
+TEST_P(ASTMatchersTest, SubstNonTypeTemplateParmExpr) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_FALSE(matches("template<int N>\n"
-                         "struct A {  static const int n = 0; };\n"
-                         "struct B : public A<42> {};",
-                         traverse(TK_AsIs,
-                       substNonTypeTemplateParmExpr())));
+                       "struct A {  static const int n = 0; };\n"
+                       "struct B : public A<42> {};",
+                       traverse(TK_AsIs, substNonTypeTemplateParmExpr())));
   EXPECT_TRUE(matches("template<int N>\n"
-                        "struct A {  static const int n = N; };\n"
-                        "struct B : public A<42> {};",
-                         traverse(TK_AsIs,
-                      substNonTypeTemplateParmExpr())));
+                      "struct A {  static const int n = N; };\n"
+                      "struct B : public A<42> {};",
+                      traverse(TK_AsIs, substNonTypeTemplateParmExpr())));
 }
 
-TEST(Matcher, NonTypeTemplateParmDecl) {
+TEST_P(ASTMatchersTest, NonTypeTemplateParmDecl) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("template <int N> void f();",
                       nonTypeTemplateParmDecl(hasName("N"))));
   EXPECT_TRUE(
-    notMatches("template <typename T> void f();", nonTypeTemplateParmDecl()));
+      notMatches("template <typename T> void f();", nonTypeTemplateParmDecl()));
 }
 
-TEST(Matcher, templateTypeParmDecl) {
+TEST_P(ASTMatchersTest, TemplateTypeParmDecl) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("template <typename T> void f();",
                       templateTypeParmDecl(hasName("T"))));
-  EXPECT_TRUE(
-    notMatches("template <int N> void f();", templateTypeParmDecl()));
+  EXPECT_TRUE(notMatches("template <int N> void f();", templateTypeParmDecl()));
 }
 
-TEST(Matcher, UserDefinedLiteral) {
+TEST_P(ASTMatchersTest, UserDefinedLiteral) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
   EXPECT_TRUE(matches("constexpr char operator \"\" _inc (const char i) {"
-                        "  return i + 1;"
-                        "}"
-                        "char c = 'a'_inc;",
+                      "  return i + 1;"
+                      "}"
+                      "char c = 'a'_inc;",
                       userDefinedLiteral()));
 }
 
-TEST(Matcher, FlowControl) {
-  EXPECT_TRUE(matches("void f() { while(true) { break; } }", breakStmt()));
-  EXPECT_TRUE(matches("void f() { while(true) { continue; } }",
-                      continueStmt()));
+TEST_P(ASTMatchersTest, FlowControl) {
+  EXPECT_TRUE(matches("void f() { while(1) { break; } }", breakStmt()));
+  EXPECT_TRUE(matches("void f() { while(1) { continue; } }", continueStmt()));
   EXPECT_TRUE(matches("void f() { goto FOO; FOO: ;}", gotoStmt()));
   EXPECT_TRUE(matches("void f() { goto FOO; FOO: ;}",
-                      labelStmt(
-                        hasDeclaration(
-                          labelDecl(hasName("FOO"))))));
+                      labelStmt(hasDeclaration(labelDecl(hasName("FOO"))))));
   EXPECT_TRUE(matches("void f() { FOO: ; void *ptr = &&FOO; goto *ptr; }",
                       addrLabelExpr()));
   EXPECT_TRUE(matches("void f() { return; }", returnStmt()));
 }
 
-TEST(Matcher, OverloadedOperatorCall) {
+TEST_P(ASTMatchersTest, CXXOperatorCallExpr) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+
   StatementMatcher OpCall = cxxOperatorCallExpr();
   // Unary operator
   EXPECT_TRUE(matches("class Y { }; "
-                        "bool operator!(Y x) { return false; }; "
-                        "Y y; bool c = !y;", OpCall));
+                      "bool operator!(Y x) { return false; }; "
+                      "Y y; bool c = !y;",
+                      OpCall));
   // No match -- special operators like "new", "delete"
   // FIXME: operator new takes size_t, for which we need stddef.h, for which
   // we need to figure out include paths in the test.
@@ -360,100 +449,108 @@ TEST(Matcher, OverloadedOperatorCall) {
   //             "void *operator new(size_t size) { return 0; } "
   //             "Y *y = new Y;", OpCall));
   EXPECT_TRUE(notMatches("class Y { }; "
-                           "void operator delete(void *p) { } "
-                           "void a() {Y *y = new Y; delete y;}", OpCall));
+                         "void operator delete(void *p) { } "
+                         "void a() {Y *y = new Y; delete y;}",
+                         OpCall));
   // Binary operator
   EXPECT_TRUE(matches("class Y { }; "
-                        "bool operator&&(Y x, Y y) { return true; }; "
-                        "Y a; Y b; bool c = a && b;",
+                      "bool operator&&(Y x, Y y) { return true; }; "
+                      "Y a; Y b; bool c = a && b;",
                       OpCall));
   // No match -- normal operator, not an overloaded one.
   EXPECT_TRUE(notMatches("bool x = true, y = true; bool t = x && y;", OpCall));
   EXPECT_TRUE(notMatches("int t = 5 << 2;", OpCall));
 }
 
-TEST(Matcher, ThisPointerType) {
+TEST_P(ASTMatchersTest, ThisPointerType) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+
   StatementMatcher MethodOnY =
       traverse(ast_type_traits::TK_AsIs,
                cxxMemberCallExpr(thisPointerType(recordDecl(hasName("Y")))));
 
-  EXPECT_TRUE(
-    matches("class Y { public: void x(); }; void z() { Y y; y.x(); }",
-            MethodOnY));
-  EXPECT_TRUE(
-    matches("class Y { public: void x(); }; void z(Y &y) { y.x(); }",
-            MethodOnY));
-  EXPECT_TRUE(
-    matches("class Y { public: void x(); }; void z(Y *&y) { y->x(); }",
-            MethodOnY));
-  EXPECT_TRUE(
-    matches("class Y { public: void x(); }; void z(Y y[]) { y->x(); }",
-            MethodOnY));
-  EXPECT_TRUE(
-    matches("class Y { public: void x(); }; void z() { Y *y; y->x(); }",
-            MethodOnY));
-
-  EXPECT_TRUE(matches(
-    "class Y {"
-      "  public: virtual void x();"
-      "};"
-      "class X : public Y {"
-      "  public: virtual void x();"
-      "};"
-      "void z() { X *x; x->Y::x(); }", MethodOnY));
-}
-
-TEST(Matcher, VariableUsage) {
-  StatementMatcher Reference =
-    declRefExpr(to(
-      varDecl(hasInitializer(
-        cxxMemberCallExpr(thisPointerType(recordDecl(hasName("Y"))))))));
-
-  EXPECT_TRUE(matches(
-    "class Y {"
-      " public:"
-      "  bool x() const;"
-      "};"
-      "void z(const Y &y) {"
-      "  bool b = y.x();"
-      "  if (b) {}"
-      "}", Reference));
-
-  EXPECT_TRUE(notMatches(
-    "class Y {"
-      " public:"
-      "  bool x() const;"
-      "};"
-      "void z(const Y &y) {"
-      "  bool b = y.x();"
-      "}", Reference));
-}
-
-TEST(Matcher, CalledVariable) {
-  StatementMatcher CallOnVariableY =
-    cxxMemberCallExpr(on(declRefExpr(to(varDecl(hasName("y"))))));
-
-  EXPECT_TRUE(matches(
-    "class Y { public: void x() { Y y; y.x(); } };", CallOnVariableY));
+  EXPECT_TRUE(matches("class Y { public: void x(); }; void z() { Y y; y.x(); }",
+                      MethodOnY));
+  EXPECT_TRUE(matches("class Y { public: void x(); }; void z(Y &y) { y.x(); }",
+                      MethodOnY));
   EXPECT_TRUE(matches(
-    "class Y { public: void x() const { Y y; y.x(); } };", CallOnVariableY));
+      "class Y { public: void x(); }; void z(Y *&y) { y->x(); }", MethodOnY));
   EXPECT_TRUE(matches(
-    "class Y { public: void x(); };"
-      "class X : public Y { void z() { X y; y.x(); } };", CallOnVariableY));
+      "class Y { public: void x(); }; void z(Y y[]) { y->x(); }", MethodOnY));
   EXPECT_TRUE(matches(
-    "class Y { public: void x(); };"
-      "class X : public Y { void z() { X *y; y->x(); } };", CallOnVariableY));
+      "class Y { public: void x(); }; void z() { Y *y; y->x(); }", MethodOnY));
+
+  EXPECT_TRUE(matches("class Y {"
+                      "  public: virtual void x();"
+                      "};"
+                      "class X : public Y {"
+                      "  public: virtual void x();"
+                      "};"
+                      "void z() { X *x; x->Y::x(); }",
+                      MethodOnY));
+}
+
+TEST_P(ASTMatchersTest, DeclRefExpr) {
+  if (!GetParam().isCXX()) {
+    // FIXME: Add a test for `declRefExpr()` that does not depend on C++.
+    return;
+  }
+  StatementMatcher Reference = declRefExpr(to(varDecl(hasInitializer(
+      cxxMemberCallExpr(thisPointerType(recordDecl(hasName("Y"))))))));
+
+  EXPECT_TRUE(matches("class Y {"
+                      " public:"
+                      "  bool x() const;"
+                      "};"
+                      "void z(const Y &y) {"
+                      "  bool b = y.x();"
+                      "  if (b) {}"
+                      "}",
+                      Reference));
+
+  EXPECT_TRUE(notMatches("class Y {"
+                         " public:"
+                         "  bool x() const;"
+                         "};"
+                         "void z(const Y &y) {"
+                         "  bool b = y.x();"
+                         "}",
+                         Reference));
+}
+
+TEST_P(ASTMatchersTest, CXXMemberCallExpr) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  StatementMatcher CallOnVariableY =
+      cxxMemberCallExpr(on(declRefExpr(to(varDecl(hasName("y"))))));
+
+  EXPECT_TRUE(matches("class Y { public: void x() { Y y; y.x(); } };",
+                      CallOnVariableY));
+  EXPECT_TRUE(matches("class Y { public: void x() const { Y y; y.x(); } };",
+                      CallOnVariableY));
+  EXPECT_TRUE(matches("class Y { public: void x(); };"
+                      "class X : public Y { void z() { X y; y.x(); } };",
+                      CallOnVariableY));
+  EXPECT_TRUE(matches("class Y { public: void x(); };"
+                      "class X : public Y { void z() { X *y; y->x(); } };",
+                      CallOnVariableY));
   EXPECT_TRUE(notMatches(
-    "class Y { public: void x(); };"
+      "class Y { public: void x(); };"
       "class X : public Y { void z() { unsigned long y; ((X*)y)->x(); } };",
-    CallOnVariableY));
+      CallOnVariableY));
+}
+
+TEST_P(ASTMatchersTest, UnaryExprOrTypeTraitExpr) {
+  EXPECT_TRUE(
+      matches("void x() { int a = sizeof(a); }", unaryExprOrTypeTraitExpr()));
 }
 
-TEST(UnaryExprOrTypeTraitExpr, MatchesSizeOfAndAlignOf) {
-  EXPECT_TRUE(matches("void x() { int a = sizeof(a); }",
-                      unaryExprOrTypeTraitExpr()));
-  EXPECT_TRUE(notMatches("void x() { int a = sizeof(a); }",
-                         alignOfExpr(anything())));
+TEST_P(ASTMatchersTest, AlignOfExpr) {
+  EXPECT_TRUE(
+      notMatches("void x() { int a = sizeof(a); }", alignOfExpr(anything())));
   // FIXME: Uncomment once alignof is enabled.
   // EXPECT_TRUE(matches("void x() { int a = alignof(a); }",
   //                     unaryExprOrTypeTraitExpr()));
@@ -461,14 +558,21 @@ TEST(UnaryExprOrTypeTraitExpr, MatchesSizeOfAndAlignOf) {
   //                        sizeOfExpr()));
 }
 
-TEST(MemberExpression, DoesNotMatchClasses) {
+TEST_P(ASTMatchersTest, MemberExpr_DoesNotMatchClasses) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(notMatches("class Y { void x() {} };", memberExpr()));
   EXPECT_TRUE(notMatches("class Y { void x() {} };", unresolvedMemberExpr()));
   EXPECT_TRUE(
       notMatches("class Y { void x() {} };", cxxDependentScopeMemberExpr()));
 }
 
-TEST(MemberExpression, MatchesMemberFunctionCall) {
+TEST_P(ASTMatchersTest, MemberExpr_MatchesMemberFunctionCall) {
+  if (!GetParam().isCXX() || GetParam().hasDelayedTemplateParsing()) {
+    // FIXME: Fix this test to work with delayed template parsing.
+    return;
+  }
   EXPECT_TRUE(matches("class Y { void x() { x(); } };", memberExpr()));
   EXPECT_TRUE(matches("class Y { template <class T> void x() { x<T>(); } };",
                       unresolvedMemberExpr()));
@@ -476,13 +580,16 @@ TEST(MemberExpression, MatchesMemberFunctionCall) {
                       cxxDependentScopeMemberExpr()));
 }
 
-TEST(MemberExpression, MatchesVariable) {
-  EXPECT_TRUE(
-    matches("class Y { void x() { this->y; } int y; };", memberExpr()));
+TEST_P(ASTMatchersTest, MemberExpr_MatchesVariable) {
+  if (!GetParam().isCXX() || GetParam().hasDelayedTemplateParsing()) {
+    // FIXME: Fix this test to work with delayed template parsing.
+    return;
+  }
   EXPECT_TRUE(
-    matches("class Y { void x() { y; } int y; };", memberExpr()));
+      matches("class Y { void x() { this->y; } int y; };", memberExpr()));
+  EXPECT_TRUE(matches("class Y { void x() { y; } int y; };", memberExpr()));
   EXPECT_TRUE(
-    matches("class Y { void x() { Y y; y.y; } int y; };", memberExpr()));
+      matches("class Y { void x() { Y y; y.y; } int y; };", memberExpr()));
   EXPECT_TRUE(matches("template <class T>"
                       "class X : T { void f() { this->T::v; } };",
                       cxxDependentScopeMemberExpr()));
@@ -492,76 +599,110 @@ TEST(MemberExpression, MatchesVariable) {
                       cxxDependentScopeMemberExpr()));
 }
 
-TEST(MemberExpression, MatchesStaticVariable) {
+TEST_P(ASTMatchersTest, MemberExpr_MatchesStaticVariable) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("class Y { void x() { this->y; } static int y; };",
                       memberExpr()));
-  EXPECT_TRUE(notMatches("class Y { void x() { y; } static int y; };",
-                         memberExpr()));
+  EXPECT_TRUE(
+      notMatches("class Y { void x() { y; } static int y; };", memberExpr()));
   EXPECT_TRUE(notMatches("class Y { void x() { Y::y; } static int y; };",
                          memberExpr()));
 }
 
-TEST(Function, MatchesFunctionDeclarations) {
+TEST_P(ASTMatchersTest, FunctionDecl) {
   StatementMatcher CallFunctionF = callExpr(callee(functionDecl(hasName("f"))));
 
   EXPECT_TRUE(matches("void f() { f(); }", CallFunctionF));
   EXPECT_TRUE(notMatches("void f() { }", CallFunctionF));
 
-  if (llvm::Triple(llvm::sys::getDefaultTargetTriple()).getOS() !=
-    llvm::Triple::Win32) {
-    // FIXME: Make this work for MSVC.
+  EXPECT_TRUE(notMatches("void f(int);", functionDecl(isVariadic())));
+  EXPECT_TRUE(notMatches("void f();", functionDecl(isVariadic())));
+  EXPECT_TRUE(matches("void f(int, ...);", functionDecl(parameterCountIs(1))));
+}
+
+TEST_P(ASTMatchersTest, FunctionDecl_C) {
+  if (!GetParam().isC()) {
+    return;
+  }
+  EXPECT_TRUE(notMatches("void f();", functionDecl(isVariadic())));
+  EXPECT_TRUE(matches("void f();", functionDecl(parameterCountIs(0))));
+}
+
+TEST_P(ASTMatchersTest, FunctionDecl_CXX) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+
+  StatementMatcher CallFunctionF = callExpr(callee(functionDecl(hasName("f"))));
+
+  if (!GetParam().hasDelayedTemplateParsing()) {
+    // FIXME: Fix this test to work with delayed template parsing.
     // Dependent contexts, but a non-dependent call.
-    EXPECT_TRUE(matches("void f(); template <int N> void g() { f(); }",
-                        CallFunctionF));
     EXPECT_TRUE(
-      matches("void f(); template <int N> struct S { void g() { f(); } };",
-              CallFunctionF));
+        matches("void f(); template <int N> void g() { f(); }", CallFunctionF));
+    EXPECT_TRUE(
+        matches("void f(); template <int N> struct S { void g() { f(); } };",
+                CallFunctionF));
   }
 
   // Depedent calls don't match.
   EXPECT_TRUE(
-    notMatches("void f(int); template <typename T> void g(T t) { f(t); }",
-               CallFunctionF));
+      notMatches("void f(int); template <typename T> void g(T t) { f(t); }",
+                 CallFunctionF));
   EXPECT_TRUE(
-    notMatches("void f(int);"
+      notMatches("void f(int);"
                  "template <typename T> struct S { void g(T t) { f(t); } };",
-               CallFunctionF));
+                 CallFunctionF));
 
   EXPECT_TRUE(matches("void f(...);", functionDecl(isVariadic())));
-  EXPECT_TRUE(notMatches("void f(int);", functionDecl(isVariadic())));
+  EXPECT_TRUE(matches("void f(...);", functionDecl(parameterCountIs(0))));
+}
+
+TEST_P(ASTMatchersTest, FunctionDecl_CXX11) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
+
   EXPECT_TRUE(notMatches("template <typename... Ts> void f(Ts...);",
                          functionDecl(isVariadic())));
-  EXPECT_TRUE(notMatches("void f();", functionDecl(isVariadic())));
-  EXPECT_TRUE(notMatchesC("void f();", functionDecl(isVariadic())));
-  EXPECT_TRUE(matches("void f(...);", functionDecl(parameterCountIs(0))));
-  EXPECT_TRUE(matchesC("void f();", functionDecl(parameterCountIs(0))));
-  EXPECT_TRUE(matches("void f(int, ...);", functionDecl(parameterCountIs(1))));
 }
 
-TEST(FunctionTemplate, MatchesFunctionTemplateDeclarations) {
-  EXPECT_TRUE(
-    matches("template <typename T> void f(T t) {}",
-            functionTemplateDecl(hasName("f"))));
+TEST_P(ASTMatchersTest,
+       FunctionTemplateDecl_MatchesFunctionTemplateDeclarations) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(matches("template <typename T> void f(T t) {}",
+                      functionTemplateDecl(hasName("f"))));
 }
 
-TEST(FunctionTemplate, DoesNotMatchFunctionDeclarations) {
+TEST_P(ASTMatchersTest, FunctionTemplate_DoesNotMatchFunctionDeclarations) {
   EXPECT_TRUE(
-    notMatches("void f(double d); void f(int t) {}",
-               functionTemplateDecl(hasName("f"))));
+      notMatches("void f(double d);", functionTemplateDecl(hasName("f"))));
+  EXPECT_TRUE(
+      notMatches("void f(int t) {}", functionTemplateDecl(hasName("f"))));
 }
 
-TEST(FunctionTemplate, DoesNotMatchFunctionTemplateSpecializations) {
-  EXPECT_TRUE(
-    notMatches("void g(); template <typename T> void f(T t) {}"
-                 "template <> void f(int t) { g(); }",
-               functionTemplateDecl(hasName("f"),
-                                    hasDescendant(declRefExpr(to(
-                                      functionDecl(hasName("g"))))))));
+TEST_P(ASTMatchersTest,
+       FunctionTemplateDecl_DoesNotMatchFunctionTemplateSpecializations) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(notMatches(
+      "void g(); template <typename T> void f(T t) {}"
+      "template <> void f(int t) { g(); }",
+      functionTemplateDecl(hasName("f"), hasDescendant(declRefExpr(to(
+                                             functionDecl(hasName("g"))))))));
 }
 
-TEST(Matcher, MatchesClassTemplateSpecialization) {
+TEST_P(ASTMatchersTest, ClassTemplateSpecializationDecl) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("template<typename T> struct A {};"
-                        "template<> struct A<int> {};",
+                      "template<> struct A<int> {};",
                       classTemplateSpecializationDecl()));
   EXPECT_TRUE(matches("template<typename T> struct A {}; A<int> a;",
                       classTemplateSpecializationDecl()));
@@ -569,85 +710,114 @@ TEST(Matcher, MatchesClassTemplateSpecialization) {
                          classTemplateSpecializationDecl()));
 }
 
-TEST(DeclaratorDecl, MatchesDeclaratorDecls) {
+TEST_P(ASTMatchersTest, DeclaratorDecl) {
   EXPECT_TRUE(matches("int x;", declaratorDecl()));
+  EXPECT_TRUE(notMatches("struct A {};", declaratorDecl()));
+}
+
+TEST_P(ASTMatchersTest, DeclaratorDecl_CXX) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(notMatches("class A {};", declaratorDecl()));
 }
 
-TEST(ParmVarDecl, MatchesParmVars) {
+TEST_P(ASTMatchersTest, ParmVarDecl) {
   EXPECT_TRUE(matches("void f(int x);", parmVarDecl()));
   EXPECT_TRUE(notMatches("void f();", parmVarDecl()));
 }
 
-TEST(Matcher, ConstructorCall) {
+TEST_P(ASTMatchersTest, Matcher_ConstructorCall) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+
   StatementMatcher Constructor =
       traverse(ast_type_traits::TK_AsIs, cxxConstructExpr());
 
   EXPECT_TRUE(
-    matches("class X { public: X(); }; void x() { X x; }", Constructor));
-  EXPECT_TRUE(
-    matches("class X { public: X(); }; void x() { X x = X(); }",
-            Constructor));
-  EXPECT_TRUE(
-    matches("class X { public: X(int); }; void x() { X x = 0; }",
-            Constructor));
+      matches("class X { public: X(); }; void x() { X x; }", Constructor));
+  EXPECT_TRUE(matches("class X { public: X(); }; void x() { X x = X(); }",
+                      Constructor));
+  EXPECT_TRUE(matches("class X { public: X(int); }; void x() { X x = 0; }",
+                      Constructor));
   EXPECT_TRUE(matches("class X {}; void x(int) { X x; }", Constructor));
 }
 
-TEST(Match, ConstructorInitializers) {
+TEST_P(ASTMatchersTest, Match_ConstructorInitializers) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("class C { int i; public: C(int ii) : i(ii) {} };",
                       cxxCtorInitializer(forField(hasName("i")))));
 }
 
-TEST(Matcher, ThisExpr) {
+TEST_P(ASTMatchersTest, Matcher_ThisExpr) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(
-    matches("struct X { int a; int f () { return a; } };", cxxThisExpr()));
+      matches("struct X { int a; int f () { return a; } };", cxxThisExpr()));
   EXPECT_TRUE(
-    notMatches("struct X { int f () { int a; return a; } };", cxxThisExpr()));
+      notMatches("struct X { int f () { int a; return a; } };", cxxThisExpr()));
 }
 
-TEST(Matcher, BindTemporaryExpression) {
+TEST_P(ASTMatchersTest, Matcher_BindTemporaryExpression) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+
   StatementMatcher TempExpression =
       traverse(ast_type_traits::TK_AsIs, cxxBindTemporaryExpr());
 
   StringRef ClassString = "class string { public: string(); ~string(); }; ";
 
-  EXPECT_TRUE(
-    matches(ClassString +
-              "string GetStringByValue();"
-                "void FunctionTakesString(string s);"
-                "void run() { FunctionTakesString(GetStringByValue()); }",
-            TempExpression));
+  EXPECT_TRUE(matches(
+      ClassString + "string GetStringByValue();"
+                    "void FunctionTakesString(string s);"
+                    "void run() { FunctionTakesString(GetStringByValue()); }",
+      TempExpression));
 
-  EXPECT_TRUE(
-    notMatches(ClassString +
-                 "string* GetStringPointer(); "
-                   "void FunctionTakesStringPtr(string* s);"
-                   "void run() {"
-                   "  string* s = GetStringPointer();"
-                   "  FunctionTakesStringPtr(GetStringPointer());"
-                   "  FunctionTakesStringPtr(s);"
-                   "}",
-               TempExpression));
+  EXPECT_TRUE(notMatches(ClassString +
+                             "string* GetStringPointer(); "
+                             "void FunctionTakesStringPtr(string* s);"
+                             "void run() {"
+                             "  string* s = GetStringPointer();"
+                             "  FunctionTakesStringPtr(GetStringPointer());"
+                             "  FunctionTakesStringPtr(s);"
+                             "}",
+                         TempExpression));
 
-  EXPECT_TRUE(
-    notMatches("class no_dtor {};"
-                 "no_dtor GetObjByValue();"
-                 "void ConsumeObj(no_dtor param);"
-                 "void run() { ConsumeObj(GetObjByValue()); }",
-               TempExpression));
+  EXPECT_TRUE(notMatches("class no_dtor {};"
+                         "no_dtor GetObjByValue();"
+                         "void ConsumeObj(no_dtor param);"
+                         "void run() { ConsumeObj(GetObjByValue()); }",
+                         TempExpression));
 }
 
-TEST(MaterializeTemporaryExpr, MatchesTemporary) {
-  StringRef ClassString = "class string { public: string(); int length(); }; ";
+TEST_P(ASTMatchersTest, MaterializeTemporaryExpr_MatchesTemporaryCXX11CXX14) {
+  if (GetParam().Language != Lang_CXX11 && GetParam().Language != Lang_CXX14) {
+    return;
+  }
+
   StatementMatcher TempExpression =
       traverse(ast_type_traits::TK_AsIs, materializeTemporaryExpr());
 
-  EXPECT_TRUE(matches(
-      ClassString + "string GetStringByValue();"
-                    "void FunctionTakesString(string s);"
-                    "void run() { FunctionTakesString(GetStringByValue()); }",
-      TempExpression));
+  EXPECT_TRUE(matches("class string { public: string(); }; "
+                      "string GetStringByValue();"
+                      "void FunctionTakesString(string s);"
+                      "void run() { FunctionTakesString(GetStringByValue()); }",
+                      TempExpression));
+}
+
+TEST_P(ASTMatchersTest, MaterializeTemporaryExpr_MatchesTemporary) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+
+  StringRef ClassString = "class string { public: string(); int length(); }; ";
+  StatementMatcher TempExpression =
+      traverse(ast_type_traits::TK_AsIs, materializeTemporaryExpr());
 
   EXPECT_TRUE(notMatches(ClassString +
                              "string* GetStringPointer(); "
@@ -669,23 +839,32 @@ TEST(MaterializeTemporaryExpr, MatchesTemporary) {
                          TempExpression));
 }
 
-TEST(Matcher, NewExpression) {
+TEST_P(ASTMatchersTest, Matcher_NewExpression) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+
   StatementMatcher New = cxxNewExpr();
 
   EXPECT_TRUE(matches("class X { public: X(); }; void x() { new X; }", New));
+  EXPECT_TRUE(matches("class X { public: X(); }; void x() { new X(); }", New));
   EXPECT_TRUE(
-    matches("class X { public: X(); }; void x() { new X(); }", New));
-  EXPECT_TRUE(
-    matches("class X { public: X(int); }; void x() { new X(0); }", New));
+      matches("class X { public: X(int); }; void x() { new X(0); }", New));
   EXPECT_TRUE(matches("class X {}; void x(int) { new X; }", New));
 }
 
-TEST(Matcher, DeleteExpression) {
-  EXPECT_TRUE(matches("struct A {}; void f(A* a) { delete a; }",
-                      cxxDeleteExpr()));
+TEST_P(ASTMatchersTest, Matcher_DeleteExpression) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(
+      matches("struct A {}; void f(A* a) { delete a; }", cxxDeleteExpr()));
 }
 
-TEST(Matcher, NoexceptExpression) {
+TEST_P(ASTMatchersTest, Matcher_NoexceptExpression) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
   StatementMatcher NoExcept = cxxNoexceptExpr();
   EXPECT_TRUE(matches("void foo(); bool bar = noexcept(foo());", NoExcept));
   EXPECT_TRUE(
@@ -695,37 +874,49 @@ TEST(Matcher, NoexceptExpression) {
   EXPECT_TRUE(matches("void foo() noexcept(noexcept(1+1));", NoExcept));
 }
 
-TEST(Matcher, DefaultArgument) {
+TEST_P(ASTMatchersTest, Matcher_DefaultArgument) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   StatementMatcher Arg = cxxDefaultArgExpr();
-
   EXPECT_TRUE(matches("void x(int, int = 0) { int y; x(y); }", Arg));
   EXPECT_TRUE(
-    matches("class X { void x(int, int = 0) { int y; x(y); } };", Arg));
+      matches("class X { void x(int, int = 0) { int y; x(y); } };", Arg));
   EXPECT_TRUE(notMatches("void x(int, int = 0) { int y; x(y, 0); }", Arg));
 }
 
-TEST(Matcher, StringLiterals) {
+TEST_P(ASTMatchersTest, StringLiteral) {
   StatementMatcher Literal = stringLiteral();
   EXPECT_TRUE(matches("const char *s = \"string\";", Literal));
-  // wide string
-  EXPECT_TRUE(matches("const wchar_t *s = L\"string\";", Literal));
   // with escaped characters
   EXPECT_TRUE(matches("const char *s = \"\x05five\";", Literal));
   // no matching -- though the data type is the same, there is no string literal
   EXPECT_TRUE(notMatches("const char s[1] = {'a'};", Literal));
 }
 
-TEST(Matcher, CharacterLiterals) {
-  StatementMatcher CharLiteral = characterLiteral();
-  EXPECT_TRUE(matches("const char c = 'c';", CharLiteral));
+TEST_P(ASTMatchersTest, StringLiteral_CXX) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(matches("const wchar_t *s = L\"string\";", stringLiteral()));
+}
+
+TEST_P(ASTMatchersTest, CharacterLiteral) {
+  EXPECT_TRUE(matches("const char c = 'c';", characterLiteral()));
+  EXPECT_TRUE(notMatches("const char c = 0x1;", characterLiteral()));
+}
+
+TEST_P(ASTMatchersTest, CharacterLiteral_CXX) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   // wide character
-  EXPECT_TRUE(matches("const char c = L'c';", CharLiteral));
+  EXPECT_TRUE(matches("const char c = L'c';", characterLiteral()));
   // wide character, Hex encoded, NOT MATCHED!
-  EXPECT_TRUE(notMatches("const wchar_t c = 0x2126;", CharLiteral));
-  EXPECT_TRUE(notMatches("const char c = 0x1;", CharLiteral));
+  EXPECT_TRUE(notMatches("const wchar_t c = 0x2126;", characterLiteral()));
 }
 
-TEST(Matcher, IntegerLiterals) {
+TEST_P(ASTMatchersTest, IntegerLiteral) {
   StatementMatcher HasIntLiteral = integerLiteral();
   EXPECT_TRUE(matches("int i = 10;", HasIntLiteral));
   EXPECT_TRUE(matches("int i = 0x1AB;", HasIntLiteral));
@@ -734,7 +925,7 @@ TEST(Matcher, IntegerLiterals) {
 
   // Non-matching cases (character literals, float and double)
   EXPECT_TRUE(notMatches("int i = L'a';",
-                         HasIntLiteral));  // this is actually a character
+                         HasIntLiteral)); // this is actually a character
   // literal cast to int
   EXPECT_TRUE(notMatches("int i = 'a';", HasIntLiteral));
   EXPECT_TRUE(notMatches("int i = 1e10;", HasIntLiteral));
@@ -747,7 +938,7 @@ TEST(Matcher, IntegerLiterals) {
                             hasUnaryOperand(integerLiteral(equals(10))))));
 }
 
-TEST(Matcher, FloatLiterals) {
+TEST_P(ASTMatchersTest, FloatLiteral) {
   StatementMatcher HasFloatLiteral = floatLiteral();
   EXPECT_TRUE(matches("float i = 10.0;", HasFloatLiteral));
   EXPECT_TRUE(matches("float i = 10.0f;", HasFloatLiteral));
@@ -757,51 +948,47 @@ TEST(Matcher, FloatLiterals) {
   EXPECT_TRUE(matches("double i = 5.0;", floatLiteral(equals(5.0))));
   EXPECT_TRUE(matches("double i = 5.0;", floatLiteral(equals(5.0f))));
   EXPECT_TRUE(
-    matches("double i = 5.0;", floatLiteral(equals(llvm::APFloat(5.0)))));
+      matches("double i = 5.0;", floatLiteral(equals(llvm::APFloat(5.0)))));
 
   EXPECT_TRUE(notMatches("float i = 10;", HasFloatLiteral));
   EXPECT_TRUE(notMatches("double i = 5.0;", floatLiteral(equals(6.0))));
   EXPECT_TRUE(notMatches("double i = 5.0;", floatLiteral(equals(6.0f))));
   EXPECT_TRUE(
-    notMatches("double i = 5.0;", floatLiteral(equals(llvm::APFloat(6.0)))));
+      notMatches("double i = 5.0;", floatLiteral(equals(llvm::APFloat(6.0)))));
 }
 
-TEST(Matcher, NullPtrLiteral) {
+TEST_P(ASTMatchersTest, CXXNullPtrLiteralExpr) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
   EXPECT_TRUE(matches("int* i = nullptr;", cxxNullPtrLiteralExpr()));
 }
 
-TEST(Matcher, ChooseExpr) {
-  EXPECT_TRUE(matchesC("void f() { (void)__builtin_choose_expr(1, 2, 3); }",
-                       chooseExpr()));
+TEST_P(ASTMatchersTest, ChooseExpr) {
+  EXPECT_TRUE(matches("void f() { (void)__builtin_choose_expr(1, 2, 3); }",
+                      chooseExpr()));
 }
 
-TEST(Matcher, GNUNullExpr) {
+TEST_P(ASTMatchersTest, GNUNullExpr) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("int* i = __null;", gnuNullExpr()));
 }
 
-TEST(Matcher, AtomicExpr) {
+TEST_P(ASTMatchersTest, AtomicExpr) {
   EXPECT_TRUE(matches("void foo() { int *ptr; __atomic_load_n(ptr, 1); }",
                       atomicExpr()));
 }
 
-TEST(Matcher, Initializers) {
-  const char *ToMatch = "void foo() { struct point { double x; double y; };"
-    "  struct point ptarray[10] = "
-    "      { [2].y = 1.0, [2].x = 2.0, [0].x = 1.0 }; }";
-  EXPECT_TRUE(matchesConditionally(
-      ToMatch,
-      initListExpr(
-          has(cxxConstructExpr(requiresZeroInitialization())),
-          has(initListExpr(
-              hasType(asString("struct point")), has(floatLiteral(equals(1.0))),
-              has(implicitValueInitExpr(hasType(asString("double")))))),
-          has(initListExpr(hasType(asString("struct point")),
-                           has(floatLiteral(equals(2.0))),
-                           has(floatLiteral(equals(1.0)))))),
-      true, {"-std=gnu++03"}));
-
-  EXPECT_TRUE(matchesC99(
-      ToMatch,
+TEST_P(ASTMatchersTest, Initializers_C99) {
+  if (!GetParam().isC99OrLater()) {
+    return;
+  }
+  EXPECT_TRUE(matches(
+      "void foo() { struct point { double x; double y; };"
+      "  struct point ptarray[10] = "
+      "      { [2].y = 1.0, [2].x = 2.0, [0].x = 1.0 }; }",
       initListExpr(hasSyntacticForm(initListExpr(
           has(designatedInitExpr(designatorCountIs(2),
                                  hasDescendant(floatLiteral(equals(1.0))),
@@ -814,33 +1001,59 @@ TEST(Matcher, Initializers) {
               hasDescendant(integerLiteral(equals(0))))))))));
 }
 
-TEST(Matcher, ParenListExpr) {
+TEST_P(ASTMatchersTest, Initializers_CXX) {
+  if (GetParam().Language != Lang_CXX03) {
+    // FIXME: Make this test pass with other C++ standard versions.
+    return;
+  }
+  EXPECT_TRUE(matches(
+      "void foo() { struct point { double x; double y; };"
+      "  struct point ptarray[10] = "
+      "      { [2].y = 1.0, [2].x = 2.0, [0].x = 1.0 }; }",
+      initListExpr(
+          has(cxxConstructExpr(requiresZeroInitialization())),
+          has(initListExpr(
+              hasType(asString("struct point")), has(floatLiteral(equals(1.0))),
+              has(implicitValueInitExpr(hasType(asString("double")))))),
+          has(initListExpr(hasType(asString("struct point")),
+                           has(floatLiteral(equals(2.0))),
+                           has(floatLiteral(equals(1.0))))))));
+}
+
+TEST_P(ASTMatchersTest, ParenListExpr) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(
-    matches("template<typename T> class foo { void bar() { foo X(*this); } };"
+      matches("template<typename T> class foo { void bar() { foo X(*this); } };"
               "template class foo<int>;",
-            varDecl(hasInitializer(parenListExpr(has(unaryOperator()))))));
+              varDecl(hasInitializer(parenListExpr(has(unaryOperator()))))));
 }
 
-TEST(Matcher, StmtExpr) {
+TEST_P(ASTMatchersTest, StmtExpr) {
   EXPECT_TRUE(matches("void declToImport() { int C = ({int X=4; X;}); }",
                       varDecl(hasInitializer(stmtExpr()))));
 }
 
-TEST(Matcher, ImportPredefinedExpr) {
+TEST_P(ASTMatchersTest, PredefinedExpr) {
   // __func__ expands as StringLiteral("foo")
   EXPECT_TRUE(matches("void foo() { __func__; }",
-                      predefinedExpr(
-                        hasType(asString("const char [4]")),
-                        has(stringLiteral()))));
+                      predefinedExpr(hasType(asString("const char [4]")),
+                                     has(stringLiteral()))));
 }
 
-TEST(Matcher, AsmStatement) {
+TEST_P(ASTMatchersTest, AsmStatement) {
   EXPECT_TRUE(matches("void foo() { __asm(\"mov al, 2\"); }", asmStmt()));
 }
 
-TEST(Matcher, Conditions) {
+TEST_P(ASTMatchersTest, HasCondition) {
+  if (!GetParam().isCXX()) {
+    // FIXME: Add a test for `hasCondition()` that does not depend on C++.
+    return;
+  }
+
   StatementMatcher Condition =
-    ifStmt(hasCondition(cxxBoolLiteral(equals(true))));
+      ifStmt(hasCondition(cxxBoolLiteral(equals(true))));
 
   EXPECT_TRUE(matches("void x() { if (true) {} }", Condition));
   EXPECT_TRUE(notMatches("void x() { if (false) {} }", Condition));
@@ -849,28 +1062,39 @@ TEST(Matcher, Conditions) {
   EXPECT_TRUE(notMatches("void x() { if (1) {} }", Condition));
 }
 
-TEST(Matcher, ConditionalOperator) {
-  StatementMatcher Conditional = conditionalOperator(
-    hasCondition(cxxBoolLiteral(equals(true))),
-    hasTrueExpression(cxxBoolLiteral(equals(false))));
+TEST_P(ASTMatchersTest, ConditionalOperator) {
+  if (!GetParam().isCXX()) {
+    // FIXME: Add a test for `conditionalOperator()` that does not depend on
+    // C++.
+    return;
+  }
+
+  StatementMatcher Conditional =
+      conditionalOperator(hasCondition(cxxBoolLiteral(equals(true))),
+                          hasTrueExpression(cxxBoolLiteral(equals(false))));
 
   EXPECT_TRUE(matches("void x() { true ? false : true; }", Conditional));
   EXPECT_TRUE(notMatches("void x() { false ? false : true; }", Conditional));
   EXPECT_TRUE(notMatches("void x() { true ? true : false; }", Conditional));
 
-  StatementMatcher ConditionalFalse = conditionalOperator(
-    hasFalseExpression(cxxBoolLiteral(equals(false))));
+  StatementMatcher ConditionalFalse =
+      conditionalOperator(hasFalseExpression(cxxBoolLiteral(equals(false))));
 
   EXPECT_TRUE(matches("void x() { true ? true : false; }", ConditionalFalse));
   EXPECT_TRUE(
-    notMatches("void x() { true ? false : true; }", ConditionalFalse));
+      notMatches("void x() { true ? false : true; }", ConditionalFalse));
 
   EXPECT_TRUE(matches("void x() { true ? true : false; }", ConditionalFalse));
   EXPECT_TRUE(
-    notMatches("void x() { true ? false : true; }", ConditionalFalse));
+      notMatches("void x() { true ? false : true; }", ConditionalFalse));
 }
 
-TEST(Matcher, BinaryConditionalOperator) {
+TEST_P(ASTMatchersTest, BinaryConditionalOperator) {
+  if (!GetParam().isCXX()) {
+    // FIXME: This test should work in non-C++ language modes.
+    return;
+  }
+
   StatementMatcher AlwaysOne =
       traverse(ast_type_traits::TK_AsIs,
                binaryConditionalOperator(
@@ -881,144 +1105,192 @@ TEST(Matcher, BinaryConditionalOperator) {
   EXPECT_TRUE(matches("void x() { 1 ?: 0; }", AlwaysOne));
 
   StatementMatcher FourNotFive = binaryConditionalOperator(
-    hasTrueExpression(opaqueValueExpr(
-      hasSourceExpression((integerLiteral(equals(4)))))),
-    hasFalseExpression(integerLiteral(equals(5))));
+      hasTrueExpression(
+          opaqueValueExpr(hasSourceExpression((integerLiteral(equals(4)))))),
+      hasFalseExpression(integerLiteral(equals(5))));
 
   EXPECT_TRUE(matches("void x() { 4 ?: 5; }", FourNotFive));
 }
 
-TEST(ArraySubscriptMatchers, ArraySubscripts) {
-  EXPECT_TRUE(matches("int i[2]; void f() { i[1] = 1; }",
-                      arraySubscriptExpr()));
-  EXPECT_TRUE(notMatches("int i; void f() { i = 1; }",
-                         arraySubscriptExpr()));
+TEST_P(ASTMatchersTest, ArraySubscriptExpr) {
+  EXPECT_TRUE(
+      matches("int i[2]; void f() { i[1] = 1; }", arraySubscriptExpr()));
+  EXPECT_TRUE(notMatches("int i; void f() { i = 1; }", arraySubscriptExpr()));
 }
 
-TEST(For, FindsForLoops) {
+TEST_P(ASTMatchersTest, ForStmt) {
   EXPECT_TRUE(matches("void f() { for(;;); }", forStmt()));
-  EXPECT_TRUE(matches("void f() { if(true) for(;;); }", forStmt()));
+  EXPECT_TRUE(matches("void f() { if(1) for(;;); }", forStmt()));
+}
+
+TEST_P(ASTMatchersTest, ForStmt_CXX11) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
   EXPECT_TRUE(notMatches("int as[] = { 1, 2, 3 };"
-                           "void f() { for (auto &a : as); }",
+                         "void f() { for (auto &a : as); }",
                          forStmt()));
 }
 
-TEST(For, ReportsNoFalsePositives) {
+TEST_P(ASTMatchersTest, ForStmt_NoFalsePositives) {
   EXPECT_TRUE(notMatches("void f() { ; }", forStmt()));
-  EXPECT_TRUE(notMatches("void f() { if(true); }", forStmt()));
+  EXPECT_TRUE(notMatches("void f() { if(1); }", forStmt()));
 }
 
-TEST(CompoundStatement, HandlesSimpleCases) {
+TEST_P(ASTMatchersTest, CompoundStatement) {
   EXPECT_TRUE(notMatches("void f();", compoundStmt()));
   EXPECT_TRUE(matches("void f() {}", compoundStmt()));
   EXPECT_TRUE(matches("void f() {{}}", compoundStmt()));
 }
 
-TEST(CompoundStatement, DoesNotMatchEmptyStruct) {
+TEST_P(ASTMatchersTest, CompoundStatement_DoesNotMatchEmptyStruct) {
+  if (!GetParam().isCXX()) {
+    // FIXME: Add a similar test that does not depend on C++.
+    return;
+  }
   // It's not a compound statement just because there's "{}" in the source
   // text. This is an AST search, not grep.
-  EXPECT_TRUE(notMatches("namespace n { struct S {}; }",
-                         compoundStmt()));
-  EXPECT_TRUE(matches("namespace n { struct S { void f() {{}} }; }",
-                      compoundStmt()));
+  EXPECT_TRUE(notMatches("namespace n { struct S {}; }", compoundStmt()));
+  EXPECT_TRUE(
+      matches("namespace n { struct S { void f() {{}} }; }", compoundStmt()));
 }
 
-TEST(CastExpression, MatchesExplicitCasts) {
-  EXPECT_TRUE(matches("char *p = reinterpret_cast<char *>(&p);",castExpr()));
+TEST_P(ASTMatchersTest, CastExpr_MatchesExplicitCasts) {
   EXPECT_TRUE(matches("void *p = (void *)(&p);", castExpr()));
+}
+
+TEST_P(ASTMatchersTest, CastExpr_MatchesExplicitCasts_CXX) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(matches("char *p = reinterpret_cast<char *>(&p);", castExpr()));
   EXPECT_TRUE(matches("char q, *p = const_cast<char *>(&q);", castExpr()));
   EXPECT_TRUE(matches("char c = char(0);", castExpr()));
 }
-TEST(CastExpression, MatchesImplicitCasts) {
+
+TEST_P(ASTMatchersTest, CastExpression_MatchesImplicitCasts) {
   // This test creates an implicit cast from int to char.
   EXPECT_TRUE(
       matches("char c = 0;", traverse(ast_type_traits::TK_AsIs, castExpr())));
   // This test creates an implicit cast from lvalue to rvalue.
-  EXPECT_TRUE(matches("char c = 0, d = c;",
+  EXPECT_TRUE(matches("void f() { char c = 0, d = c; }",
                       traverse(ast_type_traits::TK_AsIs, castExpr())));
 }
 
-TEST(CastExpression, DoesNotMatchNonCasts) {
-  EXPECT_TRUE(notMatches("char c = '0';", castExpr()));
-  EXPECT_TRUE(notMatches("char c, &q = c;", castExpr()));
+TEST_P(ASTMatchersTest, CastExpr_DoesNotMatchNonCasts) {
+  if (GetParam().Language == Lang_C89 || GetParam().Language == Lang_C99) {
+    // This does have a cast in C
+    EXPECT_TRUE(matches("char c = '0';", implicitCastExpr()));
+  } else {
+    EXPECT_TRUE(notMatches("char c = '0';", castExpr()));
+  }
   EXPECT_TRUE(notMatches("int i = (0);", castExpr()));
   EXPECT_TRUE(notMatches("int i = 0;", castExpr()));
 }
 
-TEST(ReinterpretCast, MatchesSimpleCase) {
+TEST_P(ASTMatchersTest, CastExpr_DoesNotMatchNonCasts_CXX) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(notMatches("char c, &q = c;", castExpr()));
+}
+
+TEST_P(ASTMatchersTest, CXXReinterpretCastExpr) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("char* p = reinterpret_cast<char*>(&p);",
                       cxxReinterpretCastExpr()));
 }
 
-TEST(ReinterpretCast, DoesNotMatchOtherCasts) {
+TEST_P(ASTMatchersTest, CXXReinterpretCastExpr_DoesNotMatchOtherCasts) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(notMatches("char* p = (char*)(&p);", cxxReinterpretCastExpr()));
   EXPECT_TRUE(notMatches("char q, *p = const_cast<char*>(&q);",
                          cxxReinterpretCastExpr()));
   EXPECT_TRUE(notMatches("void* p = static_cast<void*>(&p);",
                          cxxReinterpretCastExpr()));
   EXPECT_TRUE(notMatches("struct B { virtual ~B() {} }; struct D : B {};"
-                           "B b;"
-                           "D* p = dynamic_cast<D*>(&b);",
+                         "B b;"
+                         "D* p = dynamic_cast<D*>(&b);",
                          cxxReinterpretCastExpr()));
 }
 
-TEST(FunctionalCast, MatchesSimpleCase) {
+TEST_P(ASTMatchersTest, CXXFunctionalCastExpr_MatchesSimpleCase) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   StringRef foo_class = "class Foo { public: Foo(const char*); };";
   EXPECT_TRUE(matches(foo_class + "void r() { Foo f = Foo(\"hello world\"); }",
                       cxxFunctionalCastExpr()));
 }
 
-TEST(FunctionalCast, DoesNotMatchOtherCasts) {
+TEST_P(ASTMatchersTest, CXXFunctionalCastExpr_DoesNotMatchOtherCasts) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   StringRef FooClass = "class Foo { public: Foo(const char*); };";
   EXPECT_TRUE(
-    notMatches(FooClass + "void r() { Foo f = (Foo) \"hello world\"; }",
-               cxxFunctionalCastExpr()));
-  EXPECT_TRUE(
-    notMatches(FooClass + "void r() { Foo f = \"hello world\"; }",
-               cxxFunctionalCastExpr()));
+      notMatches(FooClass + "void r() { Foo f = (Foo) \"hello world\"; }",
+                 cxxFunctionalCastExpr()));
+  EXPECT_TRUE(notMatches(FooClass + "void r() { Foo f = \"hello world\"; }",
+                         cxxFunctionalCastExpr()));
 }
 
-TEST(DynamicCast, MatchesSimpleCase) {
+TEST_P(ASTMatchersTest, CXXDynamicCastExpr) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("struct B { virtual ~B() {} }; struct D : B {};"
-                        "B b;"
-                        "D* p = dynamic_cast<D*>(&b);",
+                      "B b;"
+                      "D* p = dynamic_cast<D*>(&b);",
                       cxxDynamicCastExpr()));
 }
 
-TEST(StaticCast, MatchesSimpleCase) {
-  EXPECT_TRUE(matches("void* p(static_cast<void*>(&p));",
-                      cxxStaticCastExpr()));
+TEST_P(ASTMatchersTest, CXXStaticCastExpr_MatchesSimpleCase) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(matches("void* p(static_cast<void*>(&p));", cxxStaticCastExpr()));
 }
 
-TEST(StaticCast, DoesNotMatchOtherCasts) {
+TEST_P(ASTMatchersTest, CXXStaticCastExpr_DoesNotMatchOtherCasts) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(notMatches("char* p = (char*)(&p);", cxxStaticCastExpr()));
-  EXPECT_TRUE(notMatches("char q, *p = const_cast<char*>(&q);",
-                         cxxStaticCastExpr()));
+  EXPECT_TRUE(
+      notMatches("char q, *p = const_cast<char*>(&q);", cxxStaticCastExpr()));
   EXPECT_TRUE(notMatches("void* p = reinterpret_cast<char*>(&p);",
                          cxxStaticCastExpr()));
   EXPECT_TRUE(notMatches("struct B { virtual ~B() {} }; struct D : B {};"
-                           "B b;"
-                           "D* p = dynamic_cast<D*>(&b);",
+                         "B b;"
+                         "D* p = dynamic_cast<D*>(&b);",
                          cxxStaticCastExpr()));
 }
 
-TEST(CStyleCast, MatchesSimpleCase) {
+TEST_P(ASTMatchersTest, CStyleCastExpr_MatchesSimpleCase) {
   EXPECT_TRUE(matches("int i = (int) 2.2f;", cStyleCastExpr()));
 }
 
-TEST(CStyleCast, DoesNotMatchOtherCasts) {
+TEST_P(ASTMatchersTest, CStyleCastExpr_DoesNotMatchOtherCasts) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(notMatches("char* p = static_cast<char*>(0);"
-                           "char q, *r = const_cast<char*>(&q);"
-                           "void* s = reinterpret_cast<char*>(&s);"
-                           "struct B { virtual ~B() {} }; struct D : B {};"
-                           "B b;"
-                           "D* t = dynamic_cast<D*>(&b);",
+                         "char q, *r = const_cast<char*>(&q);"
+                         "void* s = reinterpret_cast<char*>(&s);"
+                         "struct B { virtual ~B() {} }; struct D : B {};"
+                         "B b;"
+                         "D* t = dynamic_cast<D*>(&b);",
                          cStyleCastExpr()));
 }
 
-TEST(ImplicitCast, MatchesSimpleCase) {
+TEST_P(ASTMatchersTest, ImplicitCastExpr_MatchesSimpleCase) {
   // This test creates an implicit const cast.
-  EXPECT_TRUE(matches("int x = 0; const int y = x;",
+  EXPECT_TRUE(matches("void f() { int x = 0; const int y = x; }",
                       traverse(ast_type_traits::TK_AsIs,
                                varDecl(hasInitializer(implicitCastExpr())))));
   // This test creates an implicit cast from int to char.
@@ -1031,42 +1303,49 @@ TEST(ImplicitCast, MatchesSimpleCase) {
                                varDecl(hasInitializer(implicitCastExpr())))));
 }
 
-TEST(ImplicitCast, DoesNotMatchIncorrectly) {
-  // This test verifies that implicitCastExpr() matches exactly when implicit casts
-  // are present, and that it ignores explicit and paren casts.
+TEST_P(ASTMatchersTest, ImplicitCastExpr_DoesNotMatchIncorrectly) {
+  // This test verifies that implicitCastExpr() matches exactly when implicit
+  // casts are present, and that it ignores explicit and paren casts.
 
   // These two test cases have no casts.
-  EXPECT_TRUE(notMatches("int x = 0;",
-                         varDecl(hasInitializer(implicitCastExpr()))));
-  EXPECT_TRUE(notMatches("int x = 0, &y = x;",
+  EXPECT_TRUE(
+      notMatches("int x = 0;", varDecl(hasInitializer(implicitCastExpr()))));
+  EXPECT_TRUE(
+      notMatches("int x = (0);", varDecl(hasInitializer(implicitCastExpr()))));
+  EXPECT_TRUE(notMatches("void f() { int x = 0; double d = (double) x; }",
                          varDecl(hasInitializer(implicitCastExpr()))));
+}
 
-  EXPECT_TRUE(notMatches("int x = 0; double d = (double) x;",
+TEST_P(ASTMatchersTest, ImplicitCastExpr_DoesNotMatchIncorrectly_CXX) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(notMatches("int x = 0, &y = x;",
                          varDecl(hasInitializer(implicitCastExpr()))));
   EXPECT_TRUE(notMatches("const int *p; int *q = const_cast<int *>(p);",
                          varDecl(hasInitializer(implicitCastExpr()))));
-
-  EXPECT_TRUE(notMatches("int x = (0);",
-                         varDecl(hasInitializer(implicitCastExpr()))));
 }
 
-TEST(Statement, DoesNotMatchDeclarations) {
-  EXPECT_TRUE(notMatches("class X {};", stmt()));
+TEST_P(ASTMatchersTest, Stmt_DoesNotMatchDeclarations) {
+  EXPECT_TRUE(notMatches("struct X {};", stmt()));
 }
 
-TEST(Statement, MatchesCompoundStatments) {
+TEST_P(ASTMatchersTest, Stmt_MatchesCompoundStatments) {
   EXPECT_TRUE(matches("void x() {}", stmt()));
 }
 
-TEST(DeclarationStatement, DoesNotMatchCompoundStatements) {
+TEST_P(ASTMatchersTest, DeclStmt_DoesNotMatchCompoundStatements) {
   EXPECT_TRUE(notMatches("void x() {}", declStmt()));
 }
 
-TEST(DeclarationStatement, MatchesVariableDeclarationStatements) {
+TEST_P(ASTMatchersTest, DeclStmt_MatchesVariableDeclarationStatements) {
   EXPECT_TRUE(matches("void x() { int a; }", declStmt()));
 }
 
-TEST(ExprWithCleanups, MatchesExprWithCleanups) {
+TEST_P(ASTMatchersTest, ExprWithCleanups_MatchesExprWithCleanups) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("struct Foo { ~Foo(); };"
                       "const Foo f = Foo();",
                       traverse(ast_type_traits::TK_AsIs,
@@ -1077,20 +1356,30 @@ TEST(ExprWithCleanups, MatchesExprWithCleanups) {
                                 varDecl(hasInitializer(exprWithCleanups())))));
 }
 
-TEST(InitListExpression, MatchesInitListExpression) {
+TEST_P(ASTMatchersTest, InitListExpr) {
   EXPECT_TRUE(matches("int a[] = { 1, 2 };",
                       initListExpr(hasType(asString("int [2]")))));
-  EXPECT_TRUE(matches("struct B { int x, y; }; B b = { 5, 6 };",
+  EXPECT_TRUE(matches("struct B { int x, y; }; struct B b = { 5, 6 };",
                       initListExpr(hasType(recordDecl(hasName("B"))))));
+  EXPECT_TRUE(
+      matches("int i[1] = {42, [0] = 43};", integerLiteral(equals(42))));
+}
+
+TEST_P(ASTMatchersTest, InitListExpr_CXX) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("struct S { S(void (*a)()); };"
-                        "void f();"
-                        "S s[1] = { &f };",
+                      "void f();"
+                      "S s[1] = { &f };",
                       declRefExpr(to(functionDecl(hasName("f"))))));
-  EXPECT_TRUE(
-    matches("int i[1] = {42, [0] = 43};", integerLiteral(equals(42))));
 }
 
-TEST(CXXStdInitializerListExpression, MatchesCXXStdInitializerListExpression) {
+TEST_P(ASTMatchersTest,
+       CXXStdInitializerListExpression_MatchesCXXStdInitializerListExpression) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
   StringRef code = "namespace std {"
                    "template <typename> class initializer_list {"
                    "  public: initializer_list() noexcept {}"
@@ -1117,62 +1406,72 @@ TEST(CXXStdInitializerListExpression, MatchesCXXStdInitializerListExpression) {
                          cxxStdInitializerListExpr()));
 }
 
-TEST(UsingDeclaration, MatchesUsingDeclarations) {
-  EXPECT_TRUE(matches("namespace X { int x; } using X::x;",
-                      usingDecl()));
+TEST_P(ASTMatchersTest, UsingDecl_MatchesUsingDeclarations) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(matches("namespace X { int x; } using X::x;", usingDecl()));
 }
 
-TEST(UsingDeclaration, MatchesShadowUsingDelcarations) {
+TEST_P(ASTMatchersTest, UsingDecl_MatchesShadowUsingDelcarations) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("namespace f { int a; } using f::a;",
                       usingDecl(hasAnyUsingShadowDecl(hasName("a")))));
 }
 
-TEST(UsingDirectiveDeclaration, MatchesUsingNamespace) {
+TEST_P(ASTMatchersTest, UsingDirectiveDecl_MatchesUsingNamespace) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("namespace X { int x; } using namespace X;",
                       usingDirectiveDecl()));
   EXPECT_FALSE(
-    matches("namespace X { int x; } using X::x;", usingDirectiveDecl()));
+      matches("namespace X { int x; } using X::x;", usingDirectiveDecl()));
 }
 
-
-TEST(While, MatchesWhileLoops) {
+TEST_P(ASTMatchersTest, WhileStmt) {
   EXPECT_TRUE(notMatches("void x() {}", whileStmt()));
-  EXPECT_TRUE(matches("void x() { while(true); }", whileStmt()));
-  EXPECT_TRUE(notMatches("void x() { do {} while(true); }", whileStmt()));
+  EXPECT_TRUE(matches("void x() { while(1); }", whileStmt()));
+  EXPECT_TRUE(notMatches("void x() { do {} while(1); }", whileStmt()));
 }
 
-TEST(Do, MatchesDoLoops) {
-  EXPECT_TRUE(matches("void x() { do {} while(true); }", doStmt()));
-  EXPECT_TRUE(matches("void x() { do ; while(false); }", doStmt()));
+TEST_P(ASTMatchersTest, DoStmt_MatchesDoLoops) {
+  EXPECT_TRUE(matches("void x() { do {} while(1); }", doStmt()));
+  EXPECT_TRUE(matches("void x() { do ; while(0); }", doStmt()));
 }
 
-TEST(Do, DoesNotMatchWhileLoops) {
-  EXPECT_TRUE(notMatches("void x() { while(true) {} }", doStmt()));
+TEST_P(ASTMatchersTest, DoStmt_DoesNotMatchWhileLoops) {
+  EXPECT_TRUE(notMatches("void x() { while(1) {} }", doStmt()));
 }
 
-TEST(SwitchCase, MatchesCase) {
+TEST_P(ASTMatchersTest, SwitchCase_MatchesCase) {
   EXPECT_TRUE(matches("void x() { switch(42) { case 42:; } }", switchCase()));
   EXPECT_TRUE(matches("void x() { switch(42) { default:; } }", switchCase()));
   EXPECT_TRUE(matches("void x() { switch(42) default:; }", switchCase()));
   EXPECT_TRUE(notMatches("void x() { switch(42) {} }", switchCase()));
 }
 
-TEST(SwitchCase, MatchesSwitch) {
+TEST_P(ASTMatchersTest, SwitchCase_MatchesSwitch) {
   EXPECT_TRUE(matches("void x() { switch(42) { case 42:; } }", switchStmt()));
   EXPECT_TRUE(matches("void x() { switch(42) { default:; } }", switchStmt()));
   EXPECT_TRUE(matches("void x() { switch(42) default:; }", switchStmt()));
   EXPECT_TRUE(notMatches("void x() {}", switchStmt()));
 }
 
-TEST(ExceptionHandling, SimpleCases) {
+TEST_P(ASTMatchersTest, CxxExceptionHandling_SimpleCases) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("void foo() try { } catch(int X) { }", cxxCatchStmt()));
   EXPECT_TRUE(matches("void foo() try { } catch(int X) { }", cxxTryStmt()));
   EXPECT_TRUE(
-    notMatches("void foo() try { } catch(int X) { }", cxxThrowExpr()));
-  EXPECT_TRUE(matches("void foo() try { throw; } catch(int X) { }",
-                      cxxThrowExpr()));
-  EXPECT_TRUE(matches("void foo() try { throw 5;} catch(int X) { }",
-                      cxxThrowExpr()));
+      notMatches("void foo() try { } catch(int X) { }", cxxThrowExpr()));
+  EXPECT_TRUE(
+      matches("void foo() try { throw; } catch(int X) { }", cxxThrowExpr()));
+  EXPECT_TRUE(
+      matches("void foo() try { throw 5;} catch(int X) { }", cxxThrowExpr()));
   EXPECT_TRUE(matches("void foo() try { throw; } catch(...) { }",
                       cxxCatchStmt(isCatchAll())));
   EXPECT_TRUE(notMatches("void foo() try { throw; } catch(int) { }",
@@ -1183,18 +1482,18 @@ TEST(ExceptionHandling, SimpleCases) {
                          varDecl(isExceptionVariable())));
 }
 
-TEST(ParenExpression, SimpleCases) {
+TEST_P(ASTMatchersTest, ParenExpr_SimpleCases) {
   EXPECT_TRUE(
       matches("int i = (3);", traverse(ast_type_traits::TK_AsIs, parenExpr())));
   EXPECT_TRUE(matches("int i = (3 + 7);",
                       traverse(ast_type_traits::TK_AsIs, parenExpr())));
   EXPECT_TRUE(notMatches("int i = 3;",
                          traverse(ast_type_traits::TK_AsIs, parenExpr())));
-  EXPECT_TRUE(notMatches("int foo() { return 1; }; int a = foo();",
+  EXPECT_TRUE(notMatches("int f() { return 1; }; void g() { int a = f(); }",
                          traverse(ast_type_traits::TK_AsIs, parenExpr())));
 }
 
-TEST(ParenExpression, IgnoringParens) {
+TEST_P(ASTMatchersTest, IgnoringParens) {
   EXPECT_FALSE(matches(
       "const char* str = (\"my-string\");",
       traverse(ast_type_traits::TK_AsIs,
@@ -1205,31 +1504,33 @@ TEST(ParenExpression, IgnoringParens) {
                                    ignoringParens(stringLiteral()))))));
 }
 
-TEST(TypeMatching, MatchesTypes) {
+TEST_P(ASTMatchersTest, QualType) {
   EXPECT_TRUE(matches("struct S {};", qualType().bind("loc")));
 }
 
-TEST(TypeMatching, MatchesConstantArrayTypes) {
+TEST_P(ASTMatchersTest, ConstantArrayType) {
   EXPECT_TRUE(matches("int a[2];", constantArrayType()));
-  EXPECT_TRUE(notMatches(
-    "void f() { int a[] = { 2, 3 }; int b[a[0]]; }",
-    constantArrayType(hasElementType(builtinType()))));
+  EXPECT_TRUE(notMatches("void f() { int a[] = { 2, 3 }; int b[a[0]]; }",
+                         constantArrayType(hasElementType(builtinType()))));
 
   EXPECT_TRUE(matches("int a[42];", constantArrayType(hasSize(42))));
   EXPECT_TRUE(matches("int b[2*21];", constantArrayType(hasSize(42))));
   EXPECT_TRUE(notMatches("int c[41], d[43];", constantArrayType(hasSize(42))));
 }
 
-TEST(TypeMatching, MatchesDependentSizedArrayTypes) {
-  EXPECT_TRUE(matches(
-    "template <typename T, int Size> class array { T data[Size]; };",
-    dependentSizedArrayType()));
-  EXPECT_TRUE(notMatches(
-    "int a[42]; int b[] = { 2, 3 }; void f() { int c[b[0]]; }",
-    dependentSizedArrayType()));
+TEST_P(ASTMatchersTest, DependentSizedArrayType) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(
+      matches("template <typename T, int Size> class array { T data[Size]; };",
+              dependentSizedArrayType()));
+  EXPECT_TRUE(
+      notMatches("int a[42]; int b[] = { 2, 3 }; void f() { int c[b[0]]; }",
+                 dependentSizedArrayType()));
 }
 
-TEST(TypeMatching, MatchesIncompleteArrayType) {
+TEST_P(ASTMatchersTest, IncompleteArrayType) {
   EXPECT_TRUE(matches("int a[] = { 2, 3 };", incompleteArrayType()));
   EXPECT_TRUE(matches("void f(int a[]) {}", incompleteArrayType()));
 
@@ -1237,31 +1538,32 @@ TEST(TypeMatching, MatchesIncompleteArrayType) {
                          incompleteArrayType()));
 }
 
-TEST(TypeMatching, MatchesVariableArrayType) {
+TEST_P(ASTMatchersTest, VariableArrayType) {
   EXPECT_TRUE(matches("void f(int b) { int a[b]; }", variableArrayType()));
   EXPECT_TRUE(notMatches("int a[] = {2, 3}; int b[42];", variableArrayType()));
 
-  EXPECT_TRUE(matches(
-    "void f(int b) { int a[b]; }",
-    variableArrayType(hasSizeExpr(ignoringImpCasts(declRefExpr(to(
-      varDecl(hasName("b")))))))));
+  EXPECT_TRUE(matches("void f(int b) { int a[b]; }",
+                      variableArrayType(hasSizeExpr(ignoringImpCasts(
+                          declRefExpr(to(varDecl(hasName("b")))))))));
 }
 
-
-TEST(TypeMatching, MatchesAtomicTypes) {
+TEST_P(ASTMatchersTest, AtomicType) {
   if (llvm::Triple(llvm::sys::getDefaultTargetTriple()).getOS() !=
-    llvm::Triple::Win32) {
+      llvm::Triple::Win32) {
     // FIXME: Make this work for MSVC.
     EXPECT_TRUE(matches("_Atomic(int) i;", atomicType()));
 
-    EXPECT_TRUE(matches("_Atomic(int) i;",
-                        atomicType(hasValueType(isInteger()))));
-    EXPECT_TRUE(notMatches("_Atomic(float) f;",
-                           atomicType(hasValueType(isInteger()))));
+    EXPECT_TRUE(
+        matches("_Atomic(int) i;", atomicType(hasValueType(isInteger()))));
+    EXPECT_TRUE(
+        notMatches("_Atomic(float) f;", atomicType(hasValueType(isInteger()))));
   }
 }
 
-TEST(TypeMatching, MatchesAutoTypes) {
+TEST_P(ASTMatchersTest, AutoType) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
   EXPECT_TRUE(matches("auto i = 2;", autoType()));
   EXPECT_TRUE(matches("int v[] = { 2, 3 }; void f() { for (int i : v) {} }",
                       autoType()));
@@ -1272,285 +1574,351 @@ TEST(TypeMatching, MatchesAutoTypes) {
 
   // FIXME: Matching against the type-as-written can't work here, because the
   //        type as written was not deduced.
-  //EXPECT_TRUE(matches("auto a = 1;",
+  // EXPECT_TRUE(matches("auto a = 1;",
   //                    autoType(hasDeducedType(isInteger()))));
-  //EXPECT_TRUE(notMatches("auto b = 2.0;",
+  // EXPECT_TRUE(notMatches("auto b = 2.0;",
   //                       autoType(hasDeducedType(isInteger()))));
 }
 
-TEST(TypeMatching, MatchesDeclTypes) {
+TEST_P(ASTMatchersTest, DecltypeType) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
   EXPECT_TRUE(matches("decltype(1 + 1) sum = 1 + 1;", decltypeType()));
   EXPECT_TRUE(matches("decltype(1 + 1) sum = 1 + 1;",
                       decltypeType(hasUnderlyingType(isInteger()))));
 }
 
-TEST(TypeMatching, MatchesFunctionTypes) {
+TEST_P(ASTMatchersTest, FunctionType) {
   EXPECT_TRUE(matches("int (*f)(int);", functionType()));
   EXPECT_TRUE(matches("void f(int i) {}", functionType()));
 }
 
-TEST(TypeMatching, IgnoringParens) {
+TEST_P(ASTMatchersTest, IgnoringParens_Type) {
   EXPECT_TRUE(
       notMatches("void (*fp)(void);", pointerType(pointee(functionType()))));
   EXPECT_TRUE(matches("void (*fp)(void);",
                       pointerType(pointee(ignoringParens(functionType())))));
 }
 
-TEST(TypeMatching, MatchesFunctionProtoTypes) {
+TEST_P(ASTMatchersTest, FunctionProtoType) {
   EXPECT_TRUE(matches("int (*f)(int);", functionProtoType()));
   EXPECT_TRUE(matches("void f(int i);", functionProtoType()));
+  EXPECT_TRUE(matches("void f(void);", functionProtoType(parameterCountIs(0))));
+}
+
+TEST_P(ASTMatchersTest, FunctionProtoType_C) {
+  if (!GetParam().isC()) {
+    return;
+  }
+  EXPECT_TRUE(notMatches("void f();", functionProtoType()));
+}
+
+TEST_P(ASTMatchersTest, FunctionProtoType_CXX) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("void f();", functionProtoType(parameterCountIs(0))));
-  EXPECT_TRUE(notMatchesC("void f();", functionProtoType()));
-  EXPECT_TRUE(
-    matchesC("void f(void);", functionProtoType(parameterCountIs(0))));
 }
 
-TEST(TypeMatching, MatchesParenType) {
+TEST_P(ASTMatchersTest, ParenType) {
   EXPECT_TRUE(
-    matches("int (*array)[4];", varDecl(hasType(pointsTo(parenType())))));
+      matches("int (*array)[4];", varDecl(hasType(pointsTo(parenType())))));
   EXPECT_TRUE(notMatches("int *array[4];", varDecl(hasType(parenType()))));
 
   EXPECT_TRUE(matches(
-    "int (*ptr_to_func)(int);",
-    varDecl(hasType(pointsTo(parenType(innerType(functionType())))))));
+      "int (*ptr_to_func)(int);",
+      varDecl(hasType(pointsTo(parenType(innerType(functionType())))))));
   EXPECT_TRUE(notMatches(
-    "int (*ptr_to_array)[4];",
-    varDecl(hasType(pointsTo(parenType(innerType(functionType())))))));
+      "int (*ptr_to_array)[4];",
+      varDecl(hasType(pointsTo(parenType(innerType(functionType())))))));
 }
 
-TEST(TypeMatching, PointerTypes) {
+TEST_P(ASTMatchersTest, PointerType) {
   // FIXME: Reactive when these tests can be more specific (not matching
   // implicit code on certain platforms), likely when we have hasDescendant for
   // Types/TypeLocs.
-  //EXPECT_TRUE(matchAndVerifyResultTrue(
+  // EXPECT_TRUE(matchAndVerifyResultTrue(
   //    "int* a;",
   //    pointerTypeLoc(pointeeLoc(typeLoc().bind("loc"))),
   //    std::make_unique<VerifyIdIsBoundTo<TypeLoc>>("loc", 1)));
-  //EXPECT_TRUE(matchAndVerifyResultTrue(
+  // EXPECT_TRUE(matchAndVerifyResultTrue(
   //    "int* a;",
   //    pointerTypeLoc().bind("loc"),
   //    std::make_unique<VerifyIdIsBoundTo<TypeLoc>>("loc", 1)));
-  EXPECT_TRUE(matches(
-    "int** a;",
-    loc(pointerType(pointee(qualType())))));
-  EXPECT_TRUE(matches(
-    "int** a;",
-    loc(pointerType(pointee(pointerType())))));
-  EXPECT_TRUE(matches(
-    "int* b; int* * const a = &b;",
-    loc(qualType(isConstQualified(), pointerType()))));
+  EXPECT_TRUE(matches("int** a;", loc(pointerType(pointee(qualType())))));
+  EXPECT_TRUE(matches("int** a;", loc(pointerType(pointee(pointerType())))));
+  EXPECT_TRUE(matches("int* b; int* * const a = &b;",
+                      loc(qualType(isConstQualified(), pointerType()))));
+
+  StringRef Fragment = "int *ptr;";
+  EXPECT_TRUE(notMatches(Fragment,
+                         varDecl(hasName("ptr"), hasType(blockPointerType()))));
+  EXPECT_TRUE(notMatches(
+      Fragment, varDecl(hasName("ptr"), hasType(memberPointerType()))));
+  EXPECT_TRUE(
+      matches(Fragment, varDecl(hasName("ptr"), hasType(pointerType()))));
+  EXPECT_TRUE(
+      notMatches(Fragment, varDecl(hasName("ptr"), hasType(referenceType()))));
+}
 
+TEST_P(ASTMatchersTest, PointerType_CXX) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   StringRef Fragment = "struct A { int i; }; int A::* ptr = &A::i;";
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("ptr"),
-                                           hasType(blockPointerType()))));
-  EXPECT_TRUE(matches(Fragment, varDecl(hasName("ptr"),
-                                        hasType(memberPointerType()))));
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("ptr"),
-                                           hasType(pointerType()))));
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("ptr"),
-                                           hasType(referenceType()))));
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("ptr"),
-                                           hasType(lValueReferenceType()))));
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("ptr"),
-                                           hasType(rValueReferenceType()))));
-
-  Fragment = "int *ptr;";
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("ptr"),
-                                           hasType(blockPointerType()))));
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("ptr"),
-                                           hasType(memberPointerType()))));
-  EXPECT_TRUE(matches(Fragment, varDecl(hasName("ptr"),
-                                        hasType(pointerType()))));
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("ptr"),
-                                           hasType(referenceType()))));
+  EXPECT_TRUE(notMatches(Fragment,
+                         varDecl(hasName("ptr"), hasType(blockPointerType()))));
+  EXPECT_TRUE(
+      matches(Fragment, varDecl(hasName("ptr"), hasType(memberPointerType()))));
+  EXPECT_TRUE(
+      notMatches(Fragment, varDecl(hasName("ptr"), hasType(pointerType()))));
+  EXPECT_TRUE(
+      notMatches(Fragment, varDecl(hasName("ptr"), hasType(referenceType()))));
+  EXPECT_TRUE(notMatches(
+      Fragment, varDecl(hasName("ptr"), hasType(lValueReferenceType()))));
+  EXPECT_TRUE(notMatches(
+      Fragment, varDecl(hasName("ptr"), hasType(rValueReferenceType()))));
 
   Fragment = "int a; int &ref = a;";
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("ref"),
-                                           hasType(blockPointerType()))));
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("ref"),
-                                           hasType(memberPointerType()))));
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("ref"),
-                                           hasType(pointerType()))));
-  EXPECT_TRUE(matches(Fragment, varDecl(hasName("ref"),
-                                        hasType(referenceType()))));
-  EXPECT_TRUE(matches(Fragment, varDecl(hasName("ref"),
-                                        hasType(lValueReferenceType()))));
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("ref"),
-                                           hasType(rValueReferenceType()))));
-
-  Fragment = "int &&ref = 2;";
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("ref"),
-                                           hasType(blockPointerType()))));
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("ref"),
-                                           hasType(memberPointerType()))));
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("ref"),
-                                           hasType(pointerType()))));
-  EXPECT_TRUE(matches(Fragment, varDecl(hasName("ref"),
-                                        hasType(referenceType()))));
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("ref"),
-                                           hasType(lValueReferenceType()))));
-  EXPECT_TRUE(matches(Fragment, varDecl(hasName("ref"),
-                                        hasType(rValueReferenceType()))));
-}
-
-TEST(TypeMatching, AutoRefTypes) {
+  EXPECT_TRUE(notMatches(Fragment,
+                         varDecl(hasName("ref"), hasType(blockPointerType()))));
+  EXPECT_TRUE(notMatches(
+      Fragment, varDecl(hasName("ref"), hasType(memberPointerType()))));
+  EXPECT_TRUE(
+      notMatches(Fragment, varDecl(hasName("ref"), hasType(pointerType()))));
+  EXPECT_TRUE(
+      matches(Fragment, varDecl(hasName("ref"), hasType(referenceType()))));
+  EXPECT_TRUE(matches(Fragment,
+                      varDecl(hasName("ref"), hasType(lValueReferenceType()))));
+  EXPECT_TRUE(notMatches(
+      Fragment, varDecl(hasName("ref"), hasType(rValueReferenceType()))));
+}
+
+TEST_P(ASTMatchersTest, PointerType_CXX11) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
+  StringRef Fragment = "int &&ref = 2;";
+  EXPECT_TRUE(notMatches(Fragment,
+                         varDecl(hasName("ref"), hasType(blockPointerType()))));
+  EXPECT_TRUE(notMatches(
+      Fragment, varDecl(hasName("ref"), hasType(memberPointerType()))));
+  EXPECT_TRUE(
+      notMatches(Fragment, varDecl(hasName("ref"), hasType(pointerType()))));
+  EXPECT_TRUE(
+      matches(Fragment, varDecl(hasName("ref"), hasType(referenceType()))));
+  EXPECT_TRUE(notMatches(
+      Fragment, varDecl(hasName("ref"), hasType(lValueReferenceType()))));
+  EXPECT_TRUE(matches(Fragment,
+                      varDecl(hasName("ref"), hasType(rValueReferenceType()))));
+}
+
+TEST_P(ASTMatchersTest, AutoRefTypes) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
+
   StringRef Fragment = "auto a = 1;"
                        "auto b = a;"
                        "auto &c = a;"
                        "auto &&d = c;"
                        "auto &&e = 2;";
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("a"),
-                                           hasType(referenceType()))));
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("b"),
-                                           hasType(referenceType()))));
-  EXPECT_TRUE(matches(Fragment, varDecl(hasName("c"),
-                                        hasType(referenceType()))));
-  EXPECT_TRUE(matches(Fragment, varDecl(hasName("c"),
-                                        hasType(lValueReferenceType()))));
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("c"),
-                                           hasType(rValueReferenceType()))));
-  EXPECT_TRUE(matches(Fragment, varDecl(hasName("d"),
-                                        hasType(referenceType()))));
-  EXPECT_TRUE(matches(Fragment, varDecl(hasName("d"),
-                                        hasType(lValueReferenceType()))));
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("d"),
-                                           hasType(rValueReferenceType()))));
-  EXPECT_TRUE(matches(Fragment, varDecl(hasName("e"),
-                                        hasType(referenceType()))));
-  EXPECT_TRUE(notMatches(Fragment, varDecl(hasName("e"),
-                                           hasType(lValueReferenceType()))));
-  EXPECT_TRUE(matches(Fragment, varDecl(hasName("e"),
-                                        hasType(rValueReferenceType()))));
-}
-
-TEST(TypeMatching, MatchesEnumTypes) {
-  EXPECT_TRUE(matches("enum Color { Green }; Color color;",
-                      loc(enumType())));
-  EXPECT_TRUE(matches("enum class Color { Green }; Color color;",
-                      loc(enumType())));
-}
-
-TEST(TypeMatching, MatchesPointersToConstTypes) {
-  EXPECT_TRUE(matches("int b; int * const a = &b;",
-                      loc(pointerType())));
-  EXPECT_TRUE(matches("int b; int * const a = &b;",
-                      loc(pointerType())));
-  EXPECT_TRUE(matches(
-    "int b; const int * a = &b;",
-    loc(pointerType(pointee(builtinType())))));
-  EXPECT_TRUE(matches(
-    "int b; const int * a = &b;",
-    pointerType(pointee(builtinType()))));
+  EXPECT_TRUE(
+      notMatches(Fragment, varDecl(hasName("a"), hasType(referenceType()))));
+  EXPECT_TRUE(
+      notMatches(Fragment, varDecl(hasName("b"), hasType(referenceType()))));
+  EXPECT_TRUE(
+      matches(Fragment, varDecl(hasName("c"), hasType(referenceType()))));
+  EXPECT_TRUE(
+      matches(Fragment, varDecl(hasName("c"), hasType(lValueReferenceType()))));
+  EXPECT_TRUE(notMatches(
+      Fragment, varDecl(hasName("c"), hasType(rValueReferenceType()))));
+  EXPECT_TRUE(
+      matches(Fragment, varDecl(hasName("d"), hasType(referenceType()))));
+  EXPECT_TRUE(
+      matches(Fragment, varDecl(hasName("d"), hasType(lValueReferenceType()))));
+  EXPECT_TRUE(notMatches(
+      Fragment, varDecl(hasName("d"), hasType(rValueReferenceType()))));
+  EXPECT_TRUE(
+      matches(Fragment, varDecl(hasName("e"), hasType(referenceType()))));
+  EXPECT_TRUE(notMatches(
+      Fragment, varDecl(hasName("e"), hasType(lValueReferenceType()))));
+  EXPECT_TRUE(
+      matches(Fragment, varDecl(hasName("e"), hasType(rValueReferenceType()))));
+}
+
+TEST_P(ASTMatchersTest, EnumType) {
+  EXPECT_TRUE(
+      matches("enum Color { Green }; enum Color color;", loc(enumType())));
+}
+
+TEST_P(ASTMatchersTest, EnumType_CXX) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(matches("enum Color { Green }; Color color;", loc(enumType())));
 }
 
-TEST(TypeMatching, MatchesTypedefTypes) {
-  EXPECT_TRUE(matches("typedef int X; X a;", varDecl(hasName("a"),
-                                                     hasType(typedefType()))));
+TEST_P(ASTMatchersTest, EnumType_CXX11) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
+  EXPECT_TRUE(
+      matches("enum class Color { Green }; Color color;", loc(enumType())));
+}
+
+TEST_P(ASTMatchersTest, PointerType_MatchesPointersToConstTypes) {
+  EXPECT_TRUE(matches("int b; int * const a = &b;", loc(pointerType())));
+  EXPECT_TRUE(matches("int b; int * const a = &b;", loc(pointerType())));
+  EXPECT_TRUE(matches("int b; const int * a = &b;",
+                      loc(pointerType(pointee(builtinType())))));
+  EXPECT_TRUE(matches("int b; const int * a = &b;",
+                      pointerType(pointee(builtinType()))));
 }
 
-TEST(TypeMatching, MatchesTemplateSpecializationType) {
+TEST_P(ASTMatchersTest, TypedefType) {
+  EXPECT_TRUE(matches("typedef int X; X a;",
+                      varDecl(hasName("a"), hasType(typedefType()))));
+}
+
+TEST_P(ASTMatchersTest, TemplateSpecializationType) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("template <typename T> class A{}; A<int> a;",
                       templateSpecializationType()));
 }
 
-TEST(TypeMatching, MatchesDeucedTemplateSpecializationType) {
+TEST_P(ASTMatchersTest, DeducedTemplateSpecializationType) {
+  if (!GetParam().isCXX17OrLater()) {
+    return;
+  }
   EXPECT_TRUE(
       matches("template <typename T> class A{ public: A(T) {} }; A a(1);",
-              deducedTemplateSpecializationType(), langCxx17OrLater()));
+              deducedTemplateSpecializationType()));
 }
 
-TEST(TypeMatching, MatchesRecordType) {
-  EXPECT_TRUE(matches("class C{}; C c;", recordType()));
-  EXPECT_TRUE(matches("struct S{}; S s;",
+TEST_P(ASTMatchersTest, RecordType) {
+  EXPECT_TRUE(matches("struct S {}; struct S s;",
                       recordType(hasDeclaration(recordDecl(hasName("S"))))));
   EXPECT_TRUE(notMatches("int i;",
                          recordType(hasDeclaration(recordDecl(hasName("S"))))));
 }
 
-TEST(TypeMatching, MatchesElaboratedType) {
-  EXPECT_TRUE(matches(
-    "namespace N {"
-      "  namespace M {"
-      "    class D {};"
-      "  }"
-      "}"
-      "N::M::D d;", elaboratedType()));
+TEST_P(ASTMatchersTest, RecordType_CXX) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(matches("class C {}; C c;", recordType()));
+  EXPECT_TRUE(matches("struct S {}; S s;",
+                      recordType(hasDeclaration(recordDecl(hasName("S"))))));
+}
+
+TEST_P(ASTMatchersTest, ElaboratedType) {
+  if (!GetParam().isCXX()) {
+    // FIXME: Add a test for `elaboratedType()` that does not depend on C++.
+    return;
+  }
+  EXPECT_TRUE(matches("namespace N {"
+                      "  namespace M {"
+                      "    class D {};"
+                      "  }"
+                      "}"
+                      "N::M::D d;",
+                      elaboratedType()));
   EXPECT_TRUE(matches("class C {} c;", elaboratedType()));
   EXPECT_TRUE(notMatches("class C {}; C c;", elaboratedType()));
 }
 
-TEST(TypeMatching, MatchesSubstTemplateTypeParmType) {
+TEST_P(ASTMatchersTest, SubstTemplateTypeParmType) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   StringRef code = "template <typename T>"
                    "int F() {"
                    "  return 1 + T();"
                    "}"
                    "int i = F<int>();";
   EXPECT_FALSE(matches(code, binaryOperator(hasLHS(
-    expr(hasType(substTemplateTypeParmType()))))));
+                                 expr(hasType(substTemplateTypeParmType()))))));
   EXPECT_TRUE(matches(code, binaryOperator(hasRHS(
-    expr(hasType(substTemplateTypeParmType()))))));
+                                expr(hasType(substTemplateTypeParmType()))))));
 }
 
-TEST(NNS, MatchesNestedNameSpecifiers) {
-  EXPECT_TRUE(matches("namespace ns { struct A {}; } ns::A a;",
-                      nestedNameSpecifier()));
+TEST_P(ASTMatchersTest, NestedNameSpecifier) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(
+      matches("namespace ns { struct A {}; } ns::A a;", nestedNameSpecifier()));
   EXPECT_TRUE(matches("template <typename T> class A { typename T::B b; };",
                       nestedNameSpecifier()));
-  EXPECT_TRUE(matches("struct A { void f(); }; void A::f() {}",
-                      nestedNameSpecifier()));
+  EXPECT_TRUE(
+      matches("struct A { void f(); }; void A::f() {}", nestedNameSpecifier()));
   EXPECT_TRUE(matches("namespace a { namespace b {} } namespace ab = a::b;",
                       nestedNameSpecifier()));
 
-  EXPECT_TRUE(matches(
-    "struct A { static void f() {} }; void g() { A::f(); }",
-    nestedNameSpecifier()));
-  EXPECT_TRUE(notMatches(
-    "struct A { static void f() {} }; void g(A* a) { a->f(); }",
-    nestedNameSpecifier()));
+  EXPECT_TRUE(matches("struct A { static void f() {} }; void g() { A::f(); }",
+                      nestedNameSpecifier()));
+  EXPECT_TRUE(
+      notMatches("struct A { static void f() {} }; void g(A* a) { a->f(); }",
+                 nestedNameSpecifier()));
 }
 
-TEST(NullStatement, SimpleCases) {
+TEST_P(ASTMatchersTest, NullStmt) {
   EXPECT_TRUE(matches("void f() {int i;;}", nullStmt()));
   EXPECT_TRUE(notMatches("void f() {int i;}", nullStmt()));
 }
 
-TEST(NS, Alias) {
+TEST_P(ASTMatchersTest, NamespaceAliasDecl) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches("namespace test {} namespace alias = ::test;",
                       namespaceAliasDecl(hasName("alias"))));
 }
 
-TEST(NNS, MatchesTypes) {
+TEST_P(ASTMatchersTest, NestedNameSpecifier_MatchesTypes) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   NestedNameSpecifierMatcher Matcher = nestedNameSpecifier(
-    specifiesType(hasDeclaration(recordDecl(hasName("A")))));
+      specifiesType(hasDeclaration(recordDecl(hasName("A")))));
   EXPECT_TRUE(matches("struct A { struct B {}; }; A::B b;", Matcher));
-  EXPECT_TRUE(matches("struct A { struct B { struct C {}; }; }; A::B::C c;",
-                      Matcher));
+  EXPECT_TRUE(
+      matches("struct A { struct B { struct C {}; }; }; A::B::C c;", Matcher));
   EXPECT_TRUE(notMatches("namespace A { struct B {}; } A::B b;", Matcher));
 }
 
-TEST(NNS, MatchesNamespaceDecls) {
-  NestedNameSpecifierMatcher Matcher = nestedNameSpecifier(
-    specifiesNamespace(hasName("ns")));
+TEST_P(ASTMatchersTest, NestedNameSpecifier_MatchesNamespaceDecls) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  NestedNameSpecifierMatcher Matcher =
+      nestedNameSpecifier(specifiesNamespace(hasName("ns")));
   EXPECT_TRUE(matches("namespace ns { struct A {}; } ns::A a;", Matcher));
   EXPECT_TRUE(notMatches("namespace xx { struct A {}; } xx::A a;", Matcher));
   EXPECT_TRUE(notMatches("struct ns { struct A {}; }; ns::A a;", Matcher));
 }
 
-TEST(NNS, MatchesNestedNameSpecifierPrefixes) {
-  EXPECT_TRUE(matches(
-    "struct A { struct B { struct C {}; }; }; A::B::C c;",
-    nestedNameSpecifier(hasPrefix(specifiesType(asString("struct A"))))));
+TEST_P(ASTMatchersTest,
+       NestedNameSpecifier_MatchesNestedNameSpecifierPrefixes) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matches(
-    "struct A { struct B { struct C {}; }; }; A::B::C c;",
-    nestedNameSpecifierLoc(hasPrefix(
-      specifiesTypeLoc(loc(qualType(asString("struct A"))))))));
+      "struct A { struct B { struct C {}; }; }; A::B::C c;",
+      nestedNameSpecifier(hasPrefix(specifiesType(asString("struct A"))))));
+  EXPECT_TRUE(matches("struct A { struct B { struct C {}; }; }; A::B::C c;",
+                      nestedNameSpecifierLoc(hasPrefix(specifiesTypeLoc(
+                          loc(qualType(asString("struct A"))))))));
   EXPECT_TRUE(matches(
-    "namespace N { struct A { struct B { struct C {}; }; }; } N::A::B::C c;",
-    nestedNameSpecifierLoc(hasPrefix(
-      specifiesTypeLoc(loc(qualType(asString("struct N::A"))))))));
+      "namespace N { struct A { struct B { struct C {}; }; }; } N::A::B::C c;",
+      nestedNameSpecifierLoc(hasPrefix(
+          specifiesTypeLoc(loc(qualType(asString("struct N::A"))))))));
 }
 
-
 template <typename T>
 class VerifyAncestorHasChildIsEqual : public BoundNodesCallback {
 public:
@@ -1566,18 +1934,18 @@ class VerifyAncestorHasChildIsEqual : public BoundNodesCallback {
     // to equalsNode.
     const T *TypedNode = cast<T>(Node);
     return selectFirst<T>(
-      "", match(stmt(hasParent(
-        stmt(has(stmt(equalsNode(TypedNode)))).bind(""))),
-                *Node, Context)) != nullptr;
+               "", match(stmt(hasParent(
+                             stmt(has(stmt(equalsNode(TypedNode)))).bind(""))),
+                         *Node, Context)) != nullptr;
   }
   bool verify(const BoundNodes &Nodes, ASTContext &Context, const Decl *Node) {
     // Use the original typed pointer to verify we can pass pointers to subtypes
     // to equalsNode.
     const T *TypedNode = cast<T>(Node);
     return selectFirst<T>(
-      "", match(decl(hasParent(
-        decl(has(decl(equalsNode(TypedNode)))).bind(""))),
-                *Node, Context)) != nullptr;
+               "", match(decl(hasParent(
+                             decl(has(decl(equalsNode(TypedNode)))).bind(""))),
+                         *Node, Context)) != nullptr;
   }
   bool verify(const BoundNodes &Nodes, ASTContext &Context, const Type *Node) {
     // Use the original typed pointer to verify we can pass pointers to subtypes
@@ -1585,47 +1953,74 @@ class VerifyAncestorHasChildIsEqual : public BoundNodesCallback {
     const T *TypedNode = cast<T>(Node);
     const auto *Dec = Nodes.getNodeAs<FieldDecl>("decl");
     return selectFirst<T>(
-      "", match(fieldDecl(hasParent(decl(has(fieldDecl(
-        hasType(type(equalsNode(TypedNode)).bind(""))))))),
-                *Dec, Context)) != nullptr;
+               "", match(fieldDecl(hasParent(decl(has(fieldDecl(
+                             hasType(type(equalsNode(TypedNode)).bind(""))))))),
+                         *Dec, Context)) != nullptr;
   }
 };
 
-TEST(IsEqualTo, MatchesNodesByIdentity) {
+TEST_P(ASTMatchersTest, IsEqualTo_MatchesNodesByIdentity) {
   EXPECT_TRUE(matchAndVerifyResultTrue(
-    "class X { class Y {}; };", recordDecl(hasName("::X::Y")).bind(""),
-    std::make_unique<VerifyAncestorHasChildIsEqual<CXXRecordDecl>>()));
+      "void f() { if (1) if(1) {} }", ifStmt().bind(""),
+      std::make_unique<VerifyAncestorHasChildIsEqual<IfStmt>>()));
+}
+
+TEST_P(ASTMatchersTest, IsEqualTo_MatchesNodesByIdentity_Cxx) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
   EXPECT_TRUE(matchAndVerifyResultTrue(
-    "void f() { if (true) if(true) {} }", ifStmt().bind(""),
-    std::make_unique<VerifyAncestorHasChildIsEqual<IfStmt>>()));
+      "class X { class Y {}; };", recordDecl(hasName("::X::Y")).bind(""),
+      std::make_unique<VerifyAncestorHasChildIsEqual<CXXRecordDecl>>()));
   EXPECT_TRUE(matchAndVerifyResultTrue(
-    "class X { class Y {} y; };",
-    fieldDecl(hasName("y"), hasType(type().bind(""))).bind("decl"),
-    std::make_unique<VerifyAncestorHasChildIsEqual<Type>>()));
+      "class X { class Y {} y; };",
+      fieldDecl(hasName("y"), hasType(type().bind(""))).bind("decl"),
+      std::make_unique<VerifyAncestorHasChildIsEqual<Type>>()));
 }
 
-TEST(TypedefDeclMatcher, Match) {
+TEST_P(ASTMatchersTest, TypedefDecl) {
   EXPECT_TRUE(matches("typedef int typedefDeclTest;",
                       typedefDecl(hasName("typedefDeclTest"))));
-  EXPECT_TRUE(notMatches("using typedefDeclTest2 = int;",
-                         typedefDecl(hasName("typedefDeclTest2"))));
 }
 
-TEST(TypeAliasDeclMatcher, Match) {
-  EXPECT_TRUE(matches("using typeAliasTest2 = int;",
-                      typeAliasDecl(hasName("typeAliasTest2"))));
+TEST_P(ASTMatchersTest, TypedefDecl_Cxx) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
+  EXPECT_TRUE(notMatches("using typedefDeclTest = int;",
+                         typedefDecl(hasName("typedefDeclTest"))));
+}
+
+TEST_P(ASTMatchersTest, TypeAliasDecl) {
   EXPECT_TRUE(notMatches("typedef int typeAliasTest;",
                          typeAliasDecl(hasName("typeAliasTest"))));
 }
 
-TEST(TypedefNameDeclMatcher, Match) {
+TEST_P(ASTMatchersTest, TypeAliasDecl_CXX) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
+  EXPECT_TRUE(matches("using typeAliasTest = int;",
+                      typeAliasDecl(hasName("typeAliasTest"))));
+}
+
+TEST_P(ASTMatchersTest, TypedefNameDecl) {
   EXPECT_TRUE(matches("typedef int typedefNameDeclTest1;",
                       typedefNameDecl(hasName("typedefNameDeclTest1"))));
-  EXPECT_TRUE(matches("using typedefNameDeclTest2 = int;",
-                      typedefNameDecl(hasName("typedefNameDeclTest2"))));
 }
 
-TEST(TypeAliasTemplateDeclMatcher, Match) {
+TEST_P(ASTMatchersTest, TypedefNameDecl_CXX) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
+  EXPECT_TRUE(matches("using typedefNameDeclTest = int;",
+                      typedefNameDecl(hasName("typedefNameDeclTest"))));
+}
+
+TEST_P(ASTMatchersTest, TypeAliasTemplateDecl) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
   StringRef Code = R"(
     template <typename T>
     class X { T t; };
@@ -1641,8 +2036,8 @@ TEST(TypeAliasTemplateDeclMatcher, Match) {
       notMatches(Code, typeAliasTemplateDecl(hasName("typeAliasDecl"))));
 }
 
-TEST(ObjCMessageExprMatcher, SimpleExprs) {
-  // don't find ObjCMessageExpr where none are present
+TEST(ASTMatchersTestObjC, ObjCMessageExpr) {
+  // Don't find ObjCMessageExpr where none are present.
   EXPECT_TRUE(notMatchesObjC("", objcMessageExpr(anything())));
 
   StringRef Objc1String = "@interface Str "
@@ -1659,46 +2054,34 @@ TEST(ObjCMessageExprMatcher, SimpleExprs) {
                           "  Str *up = [text uppercaseString];"
                           "} "
                           "@end ";
-  EXPECT_TRUE(matchesObjC(
-    Objc1String,
-    objcMessageExpr(anything())));
+  EXPECT_TRUE(matchesObjC(Objc1String, objcMessageExpr(anything())));
   EXPECT_TRUE(matchesObjC(Objc1String,
-                          objcMessageExpr(hasAnySelector({
-                                          "contents", "meth:"}))
+                          objcMessageExpr(hasAnySelector({"contents", "meth:"}))
 
-                         ));
-  EXPECT_TRUE(matchesObjC(
-    Objc1String,
-    objcMessageExpr(hasSelector("contents"))));
-  EXPECT_TRUE(matchesObjC(
-    Objc1String,
-    objcMessageExpr(hasAnySelector("contents", "contentsA"))));
-  EXPECT_FALSE(matchesObjC(
-    Objc1String,
-    objcMessageExpr(hasAnySelector("contentsB", "contentsC"))));
+                              ));
+  EXPECT_TRUE(
+      matchesObjC(Objc1String, objcMessageExpr(hasSelector("contents"))));
   EXPECT_TRUE(matchesObjC(
-    Objc1String,
-    objcMessageExpr(matchesSelector("cont*"))));
+      Objc1String, objcMessageExpr(hasAnySelector("contents", "contentsA"))));
   EXPECT_FALSE(matchesObjC(
-    Objc1String,
-    objcMessageExpr(matchesSelector("?cont*"))));
-  EXPECT_TRUE(notMatchesObjC(
-    Objc1String,
-    objcMessageExpr(hasSelector("contents"), hasNullSelector())));
-  EXPECT_TRUE(matchesObjC(
-    Objc1String,
-    objcMessageExpr(hasSelector("contents"), hasUnarySelector())));
-  EXPECT_TRUE(matchesObjC(
-    Objc1String,
-    objcMessageExpr(hasSelector("contents"), numSelectorArgs(0))));
-  EXPECT_TRUE(matchesObjC(
-    Objc1String,
-    objcMessageExpr(matchesSelector("uppercase*"),
-                    argumentCountIs(0)
-    )));
+      Objc1String, objcMessageExpr(hasAnySelector("contentsB", "contentsC"))));
+  EXPECT_TRUE(
+      matchesObjC(Objc1String, objcMessageExpr(matchesSelector("cont*"))));
+  EXPECT_FALSE(
+      matchesObjC(Objc1String, objcMessageExpr(matchesSelector("?cont*"))));
+  EXPECT_TRUE(
+      notMatchesObjC(Objc1String, objcMessageExpr(hasSelector("contents"),
+                                                  hasNullSelector())));
+  EXPECT_TRUE(matchesObjC(Objc1String, objcMessageExpr(hasSelector("contents"),
+                                                       hasUnarySelector())));
+  EXPECT_TRUE(matchesObjC(Objc1String, objcMessageExpr(hasSelector("contents"),
+                                                       numSelectorArgs(0))));
+  EXPECT_TRUE(
+      matchesObjC(Objc1String, objcMessageExpr(matchesSelector("uppercase*"),
+                                               argumentCountIs(0))));
 }
 
-TEST(ObjCDeclMatcher, CoreDecls) {
+TEST(ASTMatchersTestObjC, ObjCDecls) {
   StringRef ObjCString = "@protocol Proto "
                          "- (void)protoDidThing; "
                          "@end "
@@ -1716,36 +2099,20 @@ TEST(ObjCDeclMatcher, CoreDecls) {
                          "- (void)abc_doThing {} "
                          "@end ";
 
-  EXPECT_TRUE(matchesObjC(
-    ObjCString,
-    objcProtocolDecl(hasName("Proto"))));
-  EXPECT_TRUE(matchesObjC(
-    ObjCString,
-    objcImplementationDecl(hasName("Thing"))));
-  EXPECT_TRUE(matchesObjC(
-    ObjCString,
-    objcCategoryDecl(hasName("ABC"))));
-  EXPECT_TRUE(matchesObjC(
-    ObjCString,
-    objcCategoryImplDecl(hasName("ABC"))));
-  EXPECT_TRUE(matchesObjC(
-    ObjCString,
-    objcMethodDecl(hasName("protoDidThing"))));
-  EXPECT_TRUE(matchesObjC(
-    ObjCString,
-    objcMethodDecl(hasName("abc_doThing"))));
-  EXPECT_TRUE(matchesObjC(
-    ObjCString,
-    objcMethodDecl(hasName("anything"))));
-  EXPECT_TRUE(matchesObjC(
-    ObjCString,
-    objcIvarDecl(hasName("_ivar"))));
-  EXPECT_TRUE(matchesObjC(
-    ObjCString,
-    objcPropertyDecl(hasName("enabled"))));
+  EXPECT_TRUE(matchesObjC(ObjCString, objcProtocolDecl(hasName("Proto"))));
+  EXPECT_TRUE(
+      matchesObjC(ObjCString, objcImplementationDecl(hasName("Thing"))));
+  EXPECT_TRUE(matchesObjC(ObjCString, objcCategoryDecl(hasName("ABC"))));
+  EXPECT_TRUE(matchesObjC(ObjCString, objcCategoryImplDecl(hasName("ABC"))));
+  EXPECT_TRUE(
+      matchesObjC(ObjCString, objcMethodDecl(hasName("protoDidThing"))));
+  EXPECT_TRUE(matchesObjC(ObjCString, objcMethodDecl(hasName("abc_doThing"))));
+  EXPECT_TRUE(matchesObjC(ObjCString, objcMethodDecl(hasName("anything"))));
+  EXPECT_TRUE(matchesObjC(ObjCString, objcIvarDecl(hasName("_ivar"))));
+  EXPECT_TRUE(matchesObjC(ObjCString, objcPropertyDecl(hasName("enabled"))));
 }
 
-TEST(ObjCStmtMatcher, ExceptionStmts) {
+TEST(ASTMatchersTestObjC, ObjCExceptionStmts) {
   StringRef ObjCString = "void f(id obj) {"
                          "  @try {"
                          "    @throw obj;"
@@ -1753,21 +2120,13 @@ TEST(ObjCStmtMatcher, ExceptionStmts) {
                          "  } @finally {}"
                          "}";
 
-  EXPECT_TRUE(matchesObjC(
-    ObjCString,
-    objcTryStmt()));
-  EXPECT_TRUE(matchesObjC(
-    ObjCString,
-    objcThrowStmt()));
-  EXPECT_TRUE(matchesObjC(
-    ObjCString,
-    objcCatchStmt()));
-  EXPECT_TRUE(matchesObjC(
-    ObjCString,
-    objcFinallyStmt()));
+  EXPECT_TRUE(matchesObjC(ObjCString, objcTryStmt()));
+  EXPECT_TRUE(matchesObjC(ObjCString, objcThrowStmt()));
+  EXPECT_TRUE(matchesObjC(ObjCString, objcCatchStmt()));
+  EXPECT_TRUE(matchesObjC(ObjCString, objcFinallyStmt()));
 }
 
-TEST(ObjCAutoreleaseMatcher, AutoreleasePool) {
+TEST(ASTMatchersTestObjC, ObjCAutoreleasePoolStmt) {
   StringRef ObjCString = "void f() {"
                          "@autoreleasepool {"
                          "  int x = 1;"
@@ -1778,7 +2137,7 @@ TEST(ObjCAutoreleaseMatcher, AutoreleasePool) {
   EXPECT_FALSE(matchesObjC(ObjCStringNoPool, autoreleasePoolStmt()));
 }
 
-TEST(OMPExecutableDirective, Matches) {
+TEST(ASTMatchersTestOpenMP, OMPExecutableDirective) {
   auto Matcher = stmt(ompExecutableDirective());
 
   StringRef Source0 = R"(
@@ -1802,7 +2161,7 @@ void x() {
   EXPECT_TRUE(notMatchesWithOpenMP(Source2, Matcher));
 }
 
-TEST(OMPDefaultClause, Matches) {
+TEST(ASTMatchersTestOpenMP, OMPDefaultClause) {
   auto Matcher = ompExecutableDirective(hasAnyClause(ompDefaultClause()));
 
   StringRef Source0 = R"(
@@ -1833,15 +2192,33 @@ void x() {
   EXPECT_TRUE(matchesWithOpenMP(Source3, Matcher));
 
   StringRef Source4 = R"(
+void x() {
+#pragma omp parallel default(firstprivate)
+;
+})";
+  EXPECT_TRUE(matchesWithOpenMP51(Source4, Matcher));
+
+  StringRef Source5 = R"(
 void x(int x) {
 #pragma omp parallel num_threads(x)
 ;
 })";
-  EXPECT_TRUE(notMatchesWithOpenMP(Source4, Matcher));
+  EXPECT_TRUE(notMatchesWithOpenMP(Source5, Matcher));
 }
 
-TEST(MatchFinderAPI, matchesDynamic) {
+TEST(ASTMatchersTest, Finder_DynamicOnlyAcceptsSomeMatchers) {
+  MatchFinder Finder;
+  EXPECT_TRUE(Finder.addDynamicMatcher(decl(), nullptr));
+  EXPECT_TRUE(Finder.addDynamicMatcher(callExpr(), nullptr));
+  EXPECT_TRUE(
+      Finder.addDynamicMatcher(constantArrayType(hasSize(42)), nullptr));
+
+  // Do not accept non-toplevel matchers.
+  EXPECT_FALSE(Finder.addDynamicMatcher(isMain(), nullptr));
+  EXPECT_FALSE(Finder.addDynamicMatcher(hasName("x"), nullptr));
+}
 
+TEST(MatchFinderAPI, MatchesDynamic) {
   StringRef SourceCode = "struct A { void f() {} };";
   auto Matcher = functionDecl(isDefinition()).bind("method");
 
@@ -1864,5 +2241,35 @@ TEST(MatchFinderAPI, matchesDynamic) {
   EXPECT_EQ(MethodNode, GlobalMethodNode);
 }
 
+static std::vector<TestClangConfig> allTestClangConfigs() {
+  std::vector<TestClangConfig> all_configs;
+  for (TestLanguage lang : {Lang_C89, Lang_C99, Lang_CXX03, Lang_CXX11,
+                            Lang_CXX14, Lang_CXX17, Lang_CXX20}) {
+    TestClangConfig config;
+    config.Language = lang;
+
+    // Use an unknown-unknown triple so we don't instantiate the full system
+    // toolchain.  On Linux, instantiating the toolchain involves stat'ing
+    // large portions of /usr/lib, and this slows down not only this test, but
+    // all other tests, via contention in the kernel.
+    //
+    // FIXME: This is a hack to work around the fact that there's no way to do
+    // the equivalent of runToolOnCodeWithArgs without instantiating a full
+    // Driver.  We should consider having a function, at least for tests, that
+    // invokes cc1.
+    config.Target = "i386-unknown-unknown";
+    all_configs.push_back(config);
+
+    // Windows target is interesting to test because it enables
+    // `-fdelayed-template-parsing`.
+    config.Target = "x86_64-pc-win32-msvc";
+    all_configs.push_back(config);
+  }
+  return all_configs;
+}
+
+INSTANTIATE_TEST_CASE_P(ASTMatchersTests, ASTMatchersTest,
+                        testing::ValuesIn(allTestClangConfigs()), );
+
 } // namespace ast_matchers
 } // namespace clang
diff --git a/clang/unittests/ASTMatchers/ASTMatchersTest.h b/clang/unittests/ASTMatchers/ASTMatchersTest.h
index fec033c4d72e6..bde6297f82ddc 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersTest.h
+++ b/clang/unittests/ASTMatchers/ASTMatchersTest.h
@@ -12,6 +12,7 @@
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/Frontend/ASTUnit.h"
 #include "clang/Testing/CommandLineArgs.h"
+#include "clang/Testing/TestClangConfig.h"
 #include "clang/Tooling/Tooling.h"
 #include "gtest/gtest.h"
 
@@ -19,10 +20,10 @@ namespace clang {
 namespace ast_matchers {
 
 using clang::tooling::buildASTFromCodeWithArgs;
+using clang::tooling::FileContentMappings;
+using clang::tooling::FrontendActionFactory;
 using clang::tooling::newFrontendActionFactory;
 using clang::tooling::runToolOnCodeWithArgs;
-using clang::tooling::FrontendActionFactory;
-using clang::tooling::FileContentMappings;
 
 class BoundNodesCallback {
 public:
@@ -37,7 +38,8 @@ class BoundNodesCallback {
 // If 'FindResultVerifier' is NULL, sets *Verified to true when Run is called.
 class VerifyMatch : public MatchFinder::MatchCallback {
 public:
-  VerifyMatch(std::unique_ptr<BoundNodesCallback> FindResultVerifier, bool *Verified)
+  VerifyMatch(std::unique_ptr<BoundNodesCallback> FindResultVerifier,
+              bool *Verified)
       : Verified(Verified), FindResultReviewer(std::move(FindResultVerifier)) {}
 
   void run(const MatchFinder::MatchResult &Result) override {
@@ -94,34 +96,45 @@ testing::AssertionResult matchesConditionally(
     return testing::AssertionFailure() << "Could not add dynamic matcher";
   std::unique_ptr<FrontendActionFactory> Factory(
       newFrontendActionFactory(&Finder));
-  // Some tests need rtti/exceptions on.  Use an unknown-unknown triple so we
-  // don't instantiate the full system toolchain.  On Linux, instantiating the
-  // toolchain involves stat'ing large portions of /usr/lib, and this slows down
-  // not only this test, but all other tests, via contention in the kernel.
-  //
-  // FIXME: This is a hack to work around the fact that there's no way to do the
-  // equivalent of runToolOnCodeWithArgs without instantiating a full Driver.
-  // We should consider having a function, at least for tests, that invokes cc1.
-  std::vector<std::string> Args(CompileArgs.begin(), CompileArgs.end());
-  Args.insert(Args.end(), {"-frtti", "-fexceptions",
-                           "-target", "i386-unknown-unknown"});
+  std::vector<std::string> Args = {
+      // Some tests need rtti/exceptions on.
+      "-frtti", "-fexceptions",
+      // Ensure that tests specify the C++ standard version that they need.
+      "-Werror=c++14-extensions", "-Werror=c++17-extensions",
+      "-Werror=c++20-extensions"};
+  // Append additional arguments at the end to allow overriding the default
+  // choices that we made above.
+  llvm::copy(CompileArgs, std::back_inserter(Args));
+  if (llvm::find(Args, "-target") == Args.end()) {
+    // Use an unknown-unknown triple so we don't instantiate the full system
+    // toolchain.  On Linux, instantiating the toolchain involves stat'ing
+    // large portions of /usr/lib, and this slows down not only this test, but
+    // all other tests, via contention in the kernel.
+    //
+    // FIXME: This is a hack to work around the fact that there's no way to do
+    // the equivalent of runToolOnCodeWithArgs without instantiating a full
+    // Driver.  We should consider having a function, at least for tests, that
+    // invokes cc1.
+    Args.push_back("-target");
+    Args.push_back("i386-unknown-unknown");
+  }
+
   if (!runToolOnCodeWithArgs(
           Factory->create(), Code, Args, Filename, "clang-tool",
           std::make_shared<PCHContainerOperations>(), VirtualMappedFiles)) {
     return testing::AssertionFailure() << "Parsing error in \"" << Code << "\"";
   }
   if (Found != DynamicFound) {
-    return testing::AssertionFailure() << "Dynamic match result ("
-                                       << DynamicFound
-                                       << ") does not match static result ("
-                                       << Found << ")";
+    return testing::AssertionFailure()
+           << "Dynamic match result (" << DynamicFound
+           << ") does not match static result (" << Found << ")";
   }
   if (!Found && ExpectMatch) {
     return testing::AssertionFailure()
-      << "Could not find match in \"" << Code << "\"";
+           << "Could not find match in \"" << Code << "\"";
   } else if (Found && !ExpectMatch) {
     return testing::AssertionFailure()
-      << "Found unexpected match in \"" << Code << "\"";
+           << "Found unexpected match in \"" << Code << "\"";
   }
   return testing::AssertionSuccess();
 }
@@ -131,13 +144,9 @@ testing::AssertionResult
 matchesConditionally(const Twine &Code, const T &AMatcher, bool ExpectMatch,
                      ArrayRef<TestLanguage> TestLanguages) {
   for (auto Lang : TestLanguages) {
-    std::vector<std::string> Args = getCommandLineArgsForTesting(Lang);
-    Args.insert(Args.end(),
-                {"-Werror=c++14-extensions", "-Werror=c++17-extensions",
-                 "-Werror=c++20-extensions"});
-    auto Result = matchesConditionally(Code, AMatcher, ExpectMatch, Args,
-                                       FileContentMappings(),
-                                       getFilenameForTesting(Lang));
+    auto Result = matchesConditionally(
+        Code, AMatcher, ExpectMatch, getCommandLineArgsForTesting(Lang),
+        FileContentMappings(), getFilenameForTesting(Lang));
     if (!Result)
       return Result;
   }
@@ -174,11 +183,6 @@ testing::AssertionResult matchesC(const Twine &Code, const T &AMatcher) {
                               "input.c");
 }
 
-template <typename T>
-testing::AssertionResult matchesC99(const Twine &Code, const T &AMatcher) {
-  return matchesConditionally(Code, AMatcher, true, {Lang_C99});
-}
-
 template <typename T>
 testing::AssertionResult notMatchesC(const Twine &Code, const T &AMatcher) {
   return matchesConditionally(Code, AMatcher, false, {Lang_C89});
@@ -212,7 +216,8 @@ matchesConditionallyWithCuda(const Twine &Code, const T &AMatcher,
       "                      size_t sharedSize = 0,"
       "                      cudaStream_t stream = 0);"
       "extern \"C\" unsigned __cudaPushCallConfiguration("
-      "    dim3 gridDim, dim3 blockDim, size_t sharedMem = 0, void *stream = 0);";
+      "    dim3 gridDim, dim3 blockDim, size_t sharedMem = 0, void *stream = "
+      "0);";
 
   bool Found = false, DynamicFound = false;
   MatchFinder Finder;
@@ -229,22 +234,20 @@ matchesConditionallyWithCuda(const Twine &Code, const T &AMatcher,
   std::vector<std::string> Args = {
       "-xcuda",  "-fno-ms-extensions",     "--cuda-host-only",     "-nocudainc",
       "-target", "x86_64-unknown-unknown", std::string(CompileArg)};
-  if (!runToolOnCodeWithArgs(Factory->create(),
-                             CudaHeader + Code, Args)) {
+  if (!runToolOnCodeWithArgs(Factory->create(), CudaHeader + Code, Args)) {
     return testing::AssertionFailure() << "Parsing error in \"" << Code << "\"";
   }
   if (Found != DynamicFound) {
-    return testing::AssertionFailure() << "Dynamic match result ("
-                                       << DynamicFound
-                                       << ") does not match static result ("
-                                       << Found << ")";
+    return testing::AssertionFailure()
+           << "Dynamic match result (" << DynamicFound
+           << ") does not match static result (" << Found << ")";
   }
   if (!Found && ExpectMatch) {
     return testing::AssertionFailure()
-      << "Could not find match in \"" << Code << "\"";
+           << "Could not find match in \"" << Code << "\"";
   } else if (Found && !ExpectMatch) {
     return testing::AssertionFailure()
-      << "Found unexpected match in \"" << Code << "\"";
+           << "Found unexpected match in \"" << Code << "\"";
   }
   return testing::AssertionSuccess();
 }
@@ -272,13 +275,28 @@ testing::AssertionResult notMatchesWithOpenMP(const Twine &Code,
   return matchesConditionally(Code, AMatcher, false, {"-fopenmp=libomp"});
 }
 
+template <typename T>
+testing::AssertionResult matchesWithOpenMP51(const Twine &Code,
+                                             const T &AMatcher) {
+  return matchesConditionally(Code, AMatcher, true,
+                              {"-fopenmp=libomp", "-fopenmp-version=51"});
+}
+
+template <typename T>
+testing::AssertionResult notMatchesWithOpenMP51(const Twine &Code,
+                                                const T &AMatcher) {
+  return matchesConditionally(Code, AMatcher, false,
+                              {"-fopenmp=libomp", "-fopenmp-version=51"});
+}
+
 template <typename T>
 testing::AssertionResult matchAndVerifyResultConditionally(
     const Twine &Code, const T &AMatcher,
     std::unique_ptr<BoundNodesCallback> FindResultVerifier, bool ExpectResult) {
   bool VerifiedResult = false;
   MatchFinder Finder;
-  VerifyMatch VerifyVerifiedResult(std::move(FindResultVerifier), &VerifiedResult);
+  VerifyMatch VerifyVerifiedResult(std::move(FindResultVerifier),
+                                   &VerifiedResult);
   Finder.addMatcher(AMatcher, &VerifyVerifiedResult);
   std::unique_ptr<FrontendActionFactory> Factory(
       newFrontendActionFactory(&Finder));
@@ -292,10 +310,10 @@ testing::AssertionResult matchAndVerifyResultConditionally(
   }
   if (!VerifiedResult && ExpectResult) {
     return testing::AssertionFailure()
-      << "Could not verify result in \"" << Code << "\"";
+           << "Could not verify result in \"" << Code << "\"";
   } else if (VerifiedResult && !ExpectResult) {
     return testing::AssertionFailure()
-      << "Verified unexpected result in \"" << Code << "\"";
+           << "Verified unexpected result in \"" << Code << "\"";
   }
 
   VerifiedResult = false;
@@ -303,15 +321,15 @@ testing::AssertionResult matchAndVerifyResultConditionally(
   std::unique_ptr<ASTUnit> AST(
       buildASTFromCodeWithArgs(Code.toStringRef(Buffer), Args));
   if (!AST.get())
-    return testing::AssertionFailure() << "Parsing error in \"" << Code
-                                       << "\" while building AST";
+    return testing::AssertionFailure()
+           << "Parsing error in \"" << Code << "\" while building AST";
   Finder.matchAST(AST->getASTContext());
   if (!VerifiedResult && ExpectResult) {
     return testing::AssertionFailure()
-      << "Could not verify result in \"" << Code << "\" with AST";
+           << "Could not verify result in \"" << Code << "\" with AST";
   } else if (VerifiedResult && !ExpectResult) {
     return testing::AssertionFailure()
-      << "Verified unexpected result in \"" << Code << "\" with AST";
+           << "Verified unexpected result in \"" << Code << "\" with AST";
   }
 
   return testing::AssertionSuccess();
@@ -323,8 +341,8 @@ template <typename T>
 testing::AssertionResult matchAndVerifyResultTrue(
     const Twine &Code, const T &AMatcher,
     std::unique_ptr<BoundNodesCallback> FindResultVerifier) {
-  return matchAndVerifyResultConditionally(
-      Code, AMatcher, std::move(FindResultVerifier), true);
+  return matchAndVerifyResultConditionally(Code, AMatcher,
+                                           std::move(FindResultVerifier), true);
 }
 
 template <typename T>
@@ -338,8 +356,7 @@ testing::AssertionResult matchAndVerifyResultFalse(
 // Implements a run method that returns whether BoundNodes contains a
 // Decl bound to Id that can be dynamically cast to T.
 // Optionally checks that the check succeeded a specific number of times.
-template <typename T>
-class VerifyIdIsBoundTo : public BoundNodesCallback {
+template <typename T> class VerifyIdIsBoundTo : public BoundNodesCallback {
 public:
   // Create an object that checks that a node of type \c T was bound to \c Id.
   // Does not check for a certain number of matches.
@@ -382,7 +399,7 @@ class VerifyIdIsBoundTo : public BoundNodesCallback {
       if (const NamedDecl *Named = Nodes->getNodeAs<NamedDecl>(Id)) {
         Name = Named->getNameAsString();
       } else if (const NestedNameSpecifier *NNS =
-        Nodes->getNodeAs<NestedNameSpecifier>(Id)) {
+                     Nodes->getNodeAs<NestedNameSpecifier>(Id)) {
         llvm::raw_string_ostream OS(Name);
         NNS->print(OS, PrintingPolicy(LangOptions()));
       }
@@ -394,7 +411,7 @@ class VerifyIdIsBoundTo : public BoundNodesCallback {
       return true;
     }
     EXPECT_TRUE(M.count(Id) == 0 ||
-      M.find(Id)->second.template get<T>() == nullptr);
+                M.find(Id)->second.template get<T>() == nullptr);
     return false;
   }
 
@@ -410,7 +427,27 @@ class VerifyIdIsBoundTo : public BoundNodesCallback {
   std::string Name;
 };
 
+class ASTMatchersTest : public ::testing::Test,
+                        public ::testing::WithParamInterface<TestClangConfig> {
+protected:
+  template <typename T>
+  testing::AssertionResult matches(const Twine &Code, const T &AMatcher) {
+    const TestClangConfig &TestConfig = GetParam();
+    return clang::ast_matchers::matchesConditionally(
+        Code, AMatcher, /*ExpectMatch=*/true, TestConfig.getCommandLineArgs(),
+        FileContentMappings(), getFilenameForTesting(TestConfig.Language));
+  }
+
+  template <typename T>
+  testing::AssertionResult notMatches(const Twine &Code, const T &AMatcher) {
+    const TestClangConfig &TestConfig = GetParam();
+    return clang::ast_matchers::matchesConditionally(
+        Code, AMatcher, /*ExpectMatch=*/false, TestConfig.getCommandLineArgs(),
+        FileContentMappings(), getFilenameForTesting(TestConfig.Language));
+  }
+};
+
 } // namespace ast_matchers
 } // namespace clang
 
-#endif  // LLVM_CLANG_UNITTESTS_AST_MATCHERS_AST_MATCHERS_TEST_H
+#endif // LLVM_CLANG_UNITTESTS_AST_MATCHERS_AST_MATCHERS_TEST_H
diff --git a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
index 50d101167ff20..380538697f461 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
@@ -2864,6 +2864,34 @@ TEST(HasParent, MatchesOnlyParent) {
     compoundStmt(hasParent(ifStmt()))));
 }
 
+TEST(MatcherMemoize, HasParentDiffersFromHas) {
+  // Test introduced after detecting a bug in memoization
+  constexpr auto code = "void f() { throw 1; }";
+  EXPECT_TRUE(notMatches(
+    code,
+    cxxThrowExpr(hasParent(expr()))));
+  EXPECT_TRUE(matches(
+    code,
+    cxxThrowExpr(has(expr()))));
+  EXPECT_TRUE(matches(
+    code,
+    cxxThrowExpr(anyOf(hasParent(expr()), has(expr())))));
+}
+
+TEST(MatcherMemoize, HasDiffersFromHasDescendant) {
+  // Test introduced after detecting a bug in memoization
+  constexpr auto code = "void f() { throw 1+1; }";
+  EXPECT_TRUE(notMatches(
+    code,
+    cxxThrowExpr(has(integerLiteral()))));
+  EXPECT_TRUE(matches(
+    code,
+    cxxThrowExpr(hasDescendant(integerLiteral()))));
+  EXPECT_TRUE(notMatches(code, 
+    cxxThrowExpr(allOf(
+      hasDescendant(integerLiteral()),
+      has(integerLiteral())))));
+}
 TEST(HasAncestor, MatchesAllAncestors) {
   EXPECT_TRUE(matches(
     "template <typename T> struct C { static void f() { 42; } };"
diff --git a/clang/unittests/ASTMatchers/Dynamic/ParserTest.cpp b/clang/unittests/ASTMatchers/Dynamic/ParserTest.cpp
index f5e324a3b158d..3af5574216f19 100644
--- a/clang/unittests/ASTMatchers/Dynamic/ParserTest.cpp
+++ b/clang/unittests/ASTMatchers/Dynamic/ParserTest.cpp
@@ -259,6 +259,15 @@ TEST(ParserTest, FullParserTest) {
   EXPECT_TRUE(matches("unsigned X = sizeof(int);", MStmt));
   EXPECT_FALSE(matches("unsigned X = alignof(int);", MStmt));
 
+  Code =
+      R"query(namedDecl(matchesName("^::[ABC]*$", "IgnoreCase | BasicRegex")))query";
+  llvm::Optional<DynTypedMatcher> MatchesName(
+      Parser::parseMatcherExpression(Code, nullptr, nullptr, &Error));
+  EXPECT_EQ("", Error.toStringFull());
+  M = MatchesName->unconditionalConvertTo<Decl>();
+  EXPECT_TRUE(matches("unsigned AAACCBB;", M));
+  EXPECT_TRUE(matches("unsigned aaaccbb;", M));
+
   Code = "hasInitializer(\n    binaryOperator(hasLHS(\"A\")))";
   EXPECT_TRUE(!Parser::parseMatcherExpression(Code, &Error).hasValue());
   EXPECT_EQ("1:1: Error parsing argument 1 for matcher hasInitializer.\n"
@@ -348,6 +357,26 @@ TEST(ParserTest, Errors) {
             "1:14: Incorrect type for arg 1. (Expected = string) != (Actual = "
             "String)",
             ParseMatcherWithError(R"query(decl(hasAttr("unrelated")))query"));
+  EXPECT_EQ(
+      "1:1: Error parsing argument 1 for matcher namedDecl.\n"
+      "1:11: Error building matcher matchesName.\n"
+      "1:33: Unknown value 'Ignorecase' for arg 2; did you mean 'IgnoreCase'",
+      ParseMatcherWithError(
+          R"query(namedDecl(matchesName("[ABC]*", "Ignorecase")))query"));
+  EXPECT_EQ(
+      "1:1: Error parsing argument 1 for matcher namedDecl.\n"
+      "1:11: Error building matcher matchesName.\n"
+      "1:33: Incorrect type for arg 2. (Expected = string) != (Actual = "
+      "String)",
+      ParseMatcherWithError(
+          R"query(namedDecl(matchesName("[ABC]*", "IgnoreCase & BasicRegex")))query"));
+  EXPECT_EQ(
+      "1:1: Error parsing argument 1 for matcher namedDecl.\n"
+      "1:11: Error building matcher matchesName.\n"
+      "1:33: Unknown value 'IgnoreCase | Basicregex' for arg 2; did you mean "
+      "'IgnoreCase | BasicRegex'",
+      ParseMatcherWithError(
+          R"query(namedDecl(matchesName("[ABC]*", "IgnoreCase | Basicregex")))query"));
 }
 
 TEST(ParserTest, OverloadErrors) {
diff --git a/clang/unittests/CrossTU/CrossTranslationUnitTest.cpp b/clang/unittests/CrossTU/CrossTranslationUnitTest.cpp
index 36a697d54c8a0..5495f27f5b32a 100644
--- a/clang/unittests/CrossTU/CrossTranslationUnitTest.cpp
+++ b/clang/unittests/CrossTU/CrossTranslationUnitTest.cpp
@@ -8,6 +8,7 @@
 
 #include "clang/CrossTU/CrossTranslationUnit.h"
 #include "clang/AST/ASTConsumer.h"
+#include "clang/AST/ParentMapContext.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendAction.h"
 #include "clang/Tooling/Tooling.h"
@@ -44,6 +45,10 @@ class CTUASTConsumer : public clang::ASTConsumer {
     assert(FD && FD->getName() == "f");
     bool OrigFDHasBody = FD->hasBody();
 
+    const DynTypedNodeList ParentsBeforeImport =
+        Ctx.getParentMapContext().getParents<Decl>(*FD);
+    ASSERT_FALSE(ParentsBeforeImport.empty());
+
     // Prepare the index file and the AST file.
     int ASTFD;
     llvm::SmallString<256> ASTFileName;
@@ -105,10 +110,29 @@ class CTUASTConsumer : public clang::ASTConsumer {
             EXPECT_EQ(OrigSLoc, FDWithDefinition->getLocation());
           }
         }
+
+        // Check parent map.
+        const DynTypedNodeList ParentsAfterImport =
+            Ctx.getParentMapContext().getParents<Decl>(*FD);
+        const DynTypedNodeList ParentsOfImported =
+            Ctx.getParentMapContext().getParents<Decl>(*NewFD);
+        EXPECT_TRUE(
+            checkParentListsEq(ParentsBeforeImport, ParentsAfterImport));
+        EXPECT_FALSE(ParentsOfImported.empty());
       }
     }
   }
 
+  static bool checkParentListsEq(const DynTypedNodeList &L1,
+                                 const DynTypedNodeList &L2) {
+    if (L1.size() != L2.size())
+      return false;
+    for (unsigned int I = 0; I < L1.size(); ++I)
+      if (L1[I] != L2[I])
+        return false;
+    return true;
+  }
+
 private:
   CrossTranslationUnitContext CTU;
   bool *Success;
@@ -123,6 +147,7 @@ class CTUAction : public clang::ASTFrontendAction {
   std::unique_ptr<clang::ASTConsumer>
   CreateASTConsumer(clang::CompilerInstance &CI, StringRef) override {
     CI.getAnalyzerOpts()->CTUImportThreshold = OverrideLimit;
+    CI.getAnalyzerOpts()->CTUImportCppThreshold = OverrideLimit;
     return std::make_unique<CTUASTConsumer>(CI, Success);
   }
 
diff --git a/clang/unittests/Driver/ModuleCacheTest.cpp b/clang/unittests/Driver/ModuleCacheTest.cpp
index db3395f4abb2b..6a0f68f26a676 100644
--- a/clang/unittests/Driver/ModuleCacheTest.cpp
+++ b/clang/unittests/Driver/ModuleCacheTest.cpp
@@ -21,7 +21,7 @@ TEST(ModuleCacheTest, GetTargetAndMode) {
   SmallString<128> Buf;
   Driver::getDefaultModuleCachePath(Buf);
   StringRef Path = Buf;
-  EXPECT_TRUE(Path.find("org.llvm.clang") != Path.npos);
+  EXPECT_TRUE(Path.find("clang") != Path.npos);
   EXPECT_TRUE(Path.endswith("ModuleCache"));  
 }
 } // end anonymous namespace.
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index dce71b0f06923..6ac3ffbffd1c8 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -5325,7 +5325,7 @@ TEST_F(FormatTest, DeductionGuides) {
   verifyFormat("template <class... Ts> S(Ts...) -> S<Ts...>;");
   verifyFormat(
       "template <class... T>\n"
-      "array(T &&... t) -> array<std::common_type_t<T...>, sizeof...(T)>;");
+      "array(T &&...t) -> array<std::common_type_t<T...>, sizeof...(T)>;");
   verifyFormat("template <class T> A() -> A<decltype(p->foo<3>())>;");
   verifyFormat("template <class T> A() -> A<decltype(foo<traits<1>>)>;");
   verifyFormat("template <class T> A() -> A<sizeof(p->foo<1>)>;");
@@ -6281,6 +6281,17 @@ TEST_F(FormatTest, BreaksConditionalExpressions) {
                "       : bbbbbbbbbbbbbbbbbbbbbbb ? 2222222222222222\n"
                "                                 : 3333333333333333;",
                Style);
+
+  Style.AlignOperands = FormatStyle::OAS_DontAlign;
+  Style.BreakBeforeTernaryOperators = false;
+  // FIXME: Aligning the question marks is weird given DontAlign.
+  // Consider disabling this alignment in this case. Also check whether this
+  // will render the adjustment from https://reviews.llvm.org/D82199
+  // unnecessary.
+  verifyFormat("int x = aaaaaaaaaaaaaaa ? aaaaaaaaaaaaaaaaaa :\n"
+               "    bbbb                ? cccccccccccccccccc :\n"
+               "                          ddddd;\n",
+               Style);
 }
 
 TEST_F(FormatTest, BreaksConditionalExpressionsAfterOperator) {
@@ -8168,13 +8179,20 @@ TEST_F(FormatTest, AttributePenaltyBreaking) {
 }
 
 TEST_F(FormatTest, UnderstandsEllipsis) {
+  FormatStyle Style = getLLVMStyle();
   verifyFormat("int printf(const char *fmt, ...);");
   verifyFormat("template <class... Ts> void Foo(Ts... ts) { Foo(ts...); }");
-  verifyFormat("template <class... Ts> void Foo(Ts *... ts) {}");
+  verifyFormat("template <class... Ts> void Foo(Ts *...ts) {}");
+
+  verifyFormat("template <int *...PP> a;", Style);
 
-  FormatStyle PointersLeft = getLLVMStyle();
-  PointersLeft.PointerAlignment = FormatStyle::PAS_Left;
-  verifyFormat("template <class... Ts> void Foo(Ts*... ts) {}", PointersLeft);
+  Style.PointerAlignment = FormatStyle::PAS_Left;
+  verifyFormat("template <class... Ts> void Foo(Ts*... ts) {}", Style);
+
+  verifyFormat("template <int*... PP> a;", Style);
+
+  Style.PointerAlignment = FormatStyle::PAS_Middle;
+  verifyFormat("template <int *... PP> a;", Style);
 }
 
 TEST_F(FormatTest, AdaptivelyFormatsPointersAndReferences) {
@@ -12968,9 +12986,7 @@ TEST_F(FormatTest, WhitesmithsBraceBreaking) {
                "  }\n",
                WhitesmithsBraceStyle);
 
-  // FIXME: the block and the break under case 2 in this test don't get indented
-  // correctly
-  /*
+  WhitesmithsBraceStyle.IndentCaseBlocks = true;
   verifyFormat("void switchTest1(int a)\n"
                "  {\n"
                "  switch (a)\n"
@@ -12978,35 +12994,101 @@ TEST_F(FormatTest, WhitesmithsBraceBreaking) {
                "    case 2:\n"
                "      {\n"
                "      }\n"
-               "      break;\n"
+               "    break;\n"
                "    }\n"
                "  }\n",
                WhitesmithsBraceStyle);
-  */
 
-  // FIXME: the block and the break under case 2 in this test don't get indented
-  // correctly
-  /*
   verifyFormat("void switchTest2(int a)\n"
                "  {\n"
                "  switch (a)\n"
                "    {\n"
-               "  case 0:\n"
+               "    case 0:\n"
                "    break;\n"
-               "  case 1:\n"
+               "    case 1:\n"
+               "      {\n"
+               "      break;\n"
+               "      }\n"
+               "    case 2:\n"
+               "      {\n"
+               "      }\n"
+               "    break;\n"
+               "    default:\n"
+               "    break;\n"
+               "    }\n"
+               "  }\n",
+               WhitesmithsBraceStyle);
+
+  verifyFormat("void switchTest3(int a)\n"
+               "  {\n"
+               "  switch (a)\n"
                "    {\n"
+               "    case 0:\n"
+               "      {\n"
+               "      foo(x);\n"
+               "      }\n"
+               "    break;\n"
+               "    default:\n"
+               "      {\n"
+               "      foo(1);\n"
+               "      }\n"
                "    break;\n"
                "    }\n"
-               "  case 2:\n"
+               "  }\n",
+               WhitesmithsBraceStyle);
+
+  WhitesmithsBraceStyle.IndentCaseBlocks = false;
+
+  verifyFormat("void switchTest4(int a)\n"
+               "  {\n"
+               "  switch (a)\n"
+               "    {\n"
+               "    case 2:\n"
                "    {\n"
                "    }\n"
                "    break;\n"
-               "  default:\n"
+               "    }\n"
+               "  }\n",
+               WhitesmithsBraceStyle);
+
+  verifyFormat("void switchTest5(int a)\n"
+               "  {\n"
+               "  switch (a)\n"
+               "    {\n"
+               "    case 0:\n"
+               "    break;\n"
+               "    case 1:\n"
+               "    {\n"
+               "    foo();\n"
+               "    break;\n"
+               "    }\n"
+               "    case 2:\n"
+               "    {\n"
+               "    }\n"
+               "    break;\n"
+               "    default:\n"
+               "    break;\n"
+               "    }\n"
+               "  }\n",
+               WhitesmithsBraceStyle);
+
+  verifyFormat("void switchTest6(int a)\n"
+               "  {\n"
+               "  switch (a)\n"
+               "    {\n"
+               "    case 0:\n"
+               "    {\n"
+               "    foo(x);\n"
+               "    }\n"
+               "    break;\n"
+               "    default:\n"
+               "    {\n"
+               "    foo(1);\n"
+               "    }\n"
                "    break;\n"
                "    }\n"
                "  }\n",
                WhitesmithsBraceStyle);
-  */
 
   verifyFormat("enum X\n"
                "  {\n"
@@ -13946,6 +14028,19 @@ TEST_F(FormatTest, ParsesConfiguration) {
   CHECK_PARSE("NamespaceMacros: [TESTSUITE, SUITE]", NamespaceMacros,
               std::vector<std::string>({"TESTSUITE", "SUITE"}));
 
+  Style.WhitespaceSensitiveMacros.clear();
+  CHECK_PARSE("WhitespaceSensitiveMacros: [STRINGIZE]",
+              WhitespaceSensitiveMacros, std::vector<std::string>{"STRINGIZE"});
+  CHECK_PARSE("WhitespaceSensitiveMacros: [STRINGIZE, ASSERT]",
+              WhitespaceSensitiveMacros,
+              std::vector<std::string>({"STRINGIZE", "ASSERT"}));
+  Style.WhitespaceSensitiveMacros.clear();
+  CHECK_PARSE("WhitespaceSensitiveMacros: ['STRINGIZE']",
+              WhitespaceSensitiveMacros, std::vector<std::string>{"STRINGIZE"});
+  CHECK_PARSE("WhitespaceSensitiveMacros: ['STRINGIZE', 'ASSERT']",
+              WhitespaceSensitiveMacros,
+              std::vector<std::string>({"STRINGIZE", "ASSERT"}));
+
   Style.IncludeStyle.IncludeCategories.clear();
   std::vector<tooling::IncludeStyle::IncludeCategory> ExpectedCategories = {
       {"abc/.*", 2, 0}, {".*", 1, 0}};
@@ -16455,6 +16550,36 @@ TEST_F(FormatTest, OperatorPassedAsAFunctionPtr) {
   verifyFormat("foo(operator, , -42);", Style);
 }
 
+TEST_F(FormatTest, WhitespaceSensitiveMacros) {
+  FormatStyle Style = getLLVMStyle();
+  Style.WhitespaceSensitiveMacros.push_back("FOO");
+
+  // Don't use the helpers here, since 'mess up' will change the whitespace
+  // and these are all whitespace sensitive by definition
+  EXPECT_EQ("FOO(String-ized&Messy+But(: :Still)=Intentional);",
+            format("FOO(String-ized&Messy+But(: :Still)=Intentional);", Style));
+  EXPECT_EQ(
+      "FOO(String-ized&Messy+But\\(: :Still)=Intentional);",
+      format("FOO(String-ized&Messy+But\\(: :Still)=Intentional);", Style));
+  EXPECT_EQ("FOO(String-ized&Messy+But,: :Still=Intentional);",
+            format("FOO(String-ized&Messy+But,: :Still=Intentional);", Style));
+  EXPECT_EQ("FOO(String-ized&Messy+But,: :\n"
+            "       Still=Intentional);",
+            format("FOO(String-ized&Messy+But,: :\n"
+                   "       Still=Intentional);",
+                   Style));
+  Style.AlignConsecutiveAssignments = true;
+  EXPECT_EQ("FOO(String-ized=&Messy+But,: :\n"
+            "       Still=Intentional);",
+            format("FOO(String-ized=&Messy+But,: :\n"
+                   "       Still=Intentional);",
+                   Style));
+
+  Style.ColumnLimit = 21;
+  EXPECT_EQ("FOO(String-ized&Messy+But: :Still=Intentional);",
+            format("FOO(String-ized&Messy+But: :Still=Intentional);", Style));
+}
+
 TEST_F(FormatTest, VeryLongNamespaceCommentSplit) {
   // These tests are not in NamespaceFixer because that doesn't
   // test its interaction with line wrapping
diff --git a/clang/unittests/Frontend/CMakeLists.txt b/clang/unittests/Frontend/CMakeLists.txt
index cde19e9106142..d247089e92955 100644
--- a/clang/unittests/Frontend/CMakeLists.txt
+++ b/clang/unittests/Frontend/CMakeLists.txt
@@ -4,6 +4,7 @@ set(LLVM_LINK_COMPONENTS
 
 add_clang_unittest(FrontendTests
   ASTUnitTest.cpp
+  CompilerInvocationTest.cpp
   CompilerInstanceTest.cpp
   FixedPointString.cpp
   FrontendActionTest.cpp
diff --git a/clang/unittests/Frontend/CompilerInvocationTest.cpp b/clang/unittests/Frontend/CompilerInvocationTest.cpp
new file mode 100644
index 0000000000000..ed82d678462fc
--- /dev/null
+++ b/clang/unittests/Frontend/CompilerInvocationTest.cpp
@@ -0,0 +1,118 @@
+//===- unittests/Frontend/CompilerInvocationTest.cpp - CI tests //---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "llvm/Support/Host.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace clang;
+
+using ::testing::Contains;
+using ::testing::Each;
+using ::testing::StrEq;
+using ::testing::StrNe;
+
+namespace {
+
+class CC1CommandLineGenerationTest : public ::testing::Test {
+public:
+  IntrusiveRefCntPtr<DiagnosticsEngine> Diags;
+  SmallVector<const char *, 32> GeneratedArgs;
+  SmallVector<std::string, 32> GeneratedArgsStorage;
+
+  const char *operator()(const Twine &Arg) {
+    return GeneratedArgsStorage.emplace_back(Arg.str()).c_str();
+  }
+
+  CC1CommandLineGenerationTest()
+      : Diags(CompilerInstance::createDiagnostics(new DiagnosticOptions())) {}
+};
+
+TEST_F(CC1CommandLineGenerationTest, CanGenerateCC1CommandLineFlag) {
+  const char *Args[] = {"clang", "-xc++", "-fmodules-strict-context-hash", "-"};
+
+  CompilerInvocation CInvok;
+  CompilerInvocation::CreateFromArgs(CInvok, Args, *Diags);
+
+  CInvok.generateCC1CommandLine(GeneratedArgs, *this);
+
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq("-fmodules-strict-context-hash")));
+}
+
+TEST_F(CC1CommandLineGenerationTest, CanGenerateCC1CommandLineSeparate) {
+  const char *TripleCStr = "i686-apple-darwin9";
+  const char *Args[] = {"clang", "-xc++", "-triple", TripleCStr, "-"};
+
+  CompilerInvocation CInvok;
+  CompilerInvocation::CreateFromArgs(CInvok, Args, *Diags);
+
+  CInvok.generateCC1CommandLine(GeneratedArgs, *this);
+
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq(TripleCStr)));
+}
+
+TEST_F(CC1CommandLineGenerationTest,
+       CanGenerateCC1CommandLineSeparateRequiredPresent) {
+  const std::string DefaultTriple =
+      llvm::Triple::normalize(llvm::sys::getDefaultTargetTriple());
+  const char *Args[] = {"clang", "-xc++", "-triple", DefaultTriple.c_str(),
+                        "-"};
+
+  CompilerInvocation CInvok;
+  CompilerInvocation::CreateFromArgs(CInvok, Args, *Diags);
+
+  CInvok.generateCC1CommandLine(GeneratedArgs, *this);
+
+  // Triple should always be emitted even if it is the default
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq(DefaultTriple.c_str())));
+}
+
+TEST_F(CC1CommandLineGenerationTest,
+       CanGenerateCC1CommandLineSeparateRequiredAbsent) {
+  const std::string DefaultTriple =
+      llvm::Triple::normalize(llvm::sys::getDefaultTargetTriple());
+  const char *Args[] = {"clang", "-xc++", "-"};
+
+  CompilerInvocation CInvok;
+  CompilerInvocation::CreateFromArgs(CInvok, Args, *Diags);
+
+  CInvok.generateCC1CommandLine(GeneratedArgs, *this);
+
+  // Triple should always be emitted even if it is the default
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq(DefaultTriple.c_str())));
+}
+
+TEST_F(CC1CommandLineGenerationTest, CanGenerateCC1CommandLineSeparateEnum) {
+  const char *RelocationModelCStr = "static";
+  const char *Args[] = {"clang", "-xc++", "-mrelocation-model",
+                        RelocationModelCStr, "-"};
+
+  CompilerInvocation CInvok;
+  CompilerInvocation::CreateFromArgs(CInvok, Args, *Diags);
+
+  CInvok.generateCC1CommandLine(GeneratedArgs, *this);
+
+  // Non default relocation model
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq(RelocationModelCStr)));
+  GeneratedArgs.clear();
+
+  RelocationModelCStr = "pic";
+  Args[3] = RelocationModelCStr;
+
+  CompilerInvocation CInvok1;
+  CompilerInvocation::CreateFromArgs(CInvok1, Args, *Diags);
+
+  CInvok1.generateCC1CommandLine(GeneratedArgs, *this);
+  ASSERT_THAT(GeneratedArgs, Each(StrNe(RelocationModelCStr)));
+}
+
+} // anonymous namespace
diff --git a/clang/unittests/StaticAnalyzer/CMakeLists.txt b/clang/unittests/StaticAnalyzer/CMakeLists.txt
index 564eba879f245..0e6d8763d96d2 100644
--- a/clang/unittests/StaticAnalyzer/CMakeLists.txt
+++ b/clang/unittests/StaticAnalyzer/CMakeLists.txt
@@ -7,6 +7,7 @@ add_clang_unittest(StaticAnalysisTests
   AnalyzerOptionsTest.cpp
   CallDescriptionTest.cpp
   CallEventTest.cpp
+  FalsePositiveRefutationBRVisitorTest.cpp
   ParamRegionTest.cpp
   RangeSetTest.cpp
   RegisterCustomCheckersTest.cpp
diff --git a/clang/unittests/StaticAnalyzer/CheckerRegistration.h b/clang/unittests/StaticAnalyzer/CheckerRegistration.h
index 0bbed9b7784ff..e02b856e0e9c4 100644
--- a/clang/unittests/StaticAnalyzer/CheckerRegistration.h
+++ b/clang/unittests/StaticAnalyzer/CheckerRegistration.h
@@ -14,6 +14,7 @@
 #include "clang/StaticAnalyzer/Frontend/AnalysisConsumer.h"
 #include "clang/StaticAnalyzer/Frontend/CheckerRegistry.h"
 #include "clang/Tooling/Tooling.h"
+#include "gtest/gtest.h"
 
 namespace clang {
 namespace ento {
@@ -65,10 +66,21 @@ class TestAction : public ASTFrontendAction {
   }
 };
 
+inline SmallString<80> getCurrentTestNameAsFileName() {
+  const ::testing::TestInfo *Info =
+      ::testing::UnitTest::GetInstance()->current_test_info();
+
+  SmallString<80> FileName;
+  (Twine{Info->name()} + ".cc").toVector(FileName);
+  return FileName;
+}
+
 template <AddCheckerFn... Fns>
 bool runCheckerOnCode(const std::string &Code, std::string &Diags) {
+  const SmallVectorImpl<char> &FileName = getCurrentTestNameAsFileName();
   llvm::raw_string_ostream OS(Diags);
-  return tooling::runToolOnCode(std::make_unique<TestAction<Fns...>>(OS), Code);
+  return tooling::runToolOnCode(std::make_unique<TestAction<Fns...>>(OS), Code,
+                                FileName);
 }
 
 template <AddCheckerFn... Fns>
@@ -77,5 +89,22 @@ bool runCheckerOnCode(const std::string &Code) {
   return runCheckerOnCode<Fns...>(Code, Diags);
 }
 
+template <AddCheckerFn... Fns>
+bool runCheckerOnCodeWithArgs(const std::string &Code,
+                              const std::vector<std::string> &Args,
+                              std::string &Diags) {
+  const SmallVectorImpl<char> &FileName = getCurrentTestNameAsFileName();
+  llvm::raw_string_ostream OS(Diags);
+  return tooling::runToolOnCodeWithArgs(
+      std::make_unique<TestAction<Fns...>>(OS), Code, Args, FileName);
+}
+
+template <AddCheckerFn... Fns>
+bool runCheckerOnCodeWithArgs(const std::string &Code,
+                              const std::vector<std::string> &Args) {
+  std::string Diags;
+  return runCheckerOnCodeWithArgs<Fns...>(Code, Args, Diags);
+}
+
 } // namespace ento
 } // namespace clang
diff --git a/clang/unittests/StaticAnalyzer/FalsePositiveRefutationBRVisitorTest.cpp b/clang/unittests/StaticAnalyzer/FalsePositiveRefutationBRVisitorTest.cpp
new file mode 100644
index 0000000000000..7c151c1821133
--- /dev/null
+++ b/clang/unittests/StaticAnalyzer/FalsePositiveRefutationBRVisitorTest.cpp
@@ -0,0 +1,223 @@
+//===- unittests/StaticAnalyzer/FalsePositiveRefutationBRVisitorTest.cpp --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "CheckerRegistration.h"
+#include "Reusables.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/Checker.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
+#include "clang/StaticAnalyzer/Frontend/AnalysisConsumer.h"
+#include "clang/StaticAnalyzer/Frontend/CheckerRegistry.h"
+#include "llvm/Config/config.h"
+#include "gtest/gtest.h"
+
+// FIXME: Use GTEST_SKIP() instead if GTest is updated to version 1.10.0
+#ifdef LLVM_WITH_Z3
+#define SKIP_WITHOUT_Z3
+#else
+#define SKIP_WITHOUT_Z3 return
+#endif
+
+namespace clang {
+namespace ento {
+namespace {
+
+class FalsePositiveGenerator : public Checker<eval::Call> {
+  using Self = FalsePositiveGenerator;
+  const BuiltinBug FalsePositiveGeneratorBug{this, "FalsePositiveGenerator"};
+  using HandlerFn = bool (Self::*)(const CallEvent &Call,
+                                   CheckerContext &) const;
+  CallDescriptionMap<HandlerFn> Callbacks = {
+      {{"reachedWithContradiction", 0}, &Self::reachedWithContradiction},
+      {{"reachedWithNoContradiction", 0}, &Self::reachedWithNoContradiction},
+      {{"reportIfCanBeTrue", 1}, &Self::reportIfCanBeTrue},
+  };
+
+  bool report(CheckerContext &C, ProgramStateRef State,
+              StringRef Description) const {
+    ExplodedNode *Node = C.generateNonFatalErrorNode(State);
+    if (!Node)
+      return false;
+
+    auto Report = std::make_unique<PathSensitiveBugReport>(
+        FalsePositiveGeneratorBug, Description, Node);
+    C.emitReport(std::move(Report));
+    return true;
+  }
+
+  bool reachedWithNoContradiction(const CallEvent &, CheckerContext &C) const {
+    return report(C, C.getState(), "REACHED_WITH_NO_CONTRADICTION");
+  }
+
+  bool reachedWithContradiction(const CallEvent &, CheckerContext &C) const {
+    return report(C, C.getState(), "REACHED_WITH_CONTRADICTION");
+  }
+
+  // Similar to ExprInspectionChecker::analyzerEval except it emits warning only
+  // if the argument can be true. The report emits the report in the state where
+  // the assertion true.
+  bool reportIfCanBeTrue(const CallEvent &Call, CheckerContext &C) const {
+    // A specific instantiation of an inlined function may have more constrained
+    // values than can generally be assumed. Skip the check.
+    if (C.getPredecessor()->getLocationContext()->getStackFrame()->getParent())
+      return false;
+
+    SVal AssertionVal = Call.getArgSVal(0);
+    if (AssertionVal.isUndef())
+      return false;
+
+    ProgramStateRef State = C.getPredecessor()->getState();
+    ProgramStateRef StTrue;
+    std::tie(StTrue, std::ignore) =
+        State->assume(AssertionVal.castAs<DefinedOrUnknownSVal>());
+    if (StTrue)
+      return report(C, StTrue, "CAN_BE_TRUE");
+    return false;
+  }
+
+public:
+  bool evalCall(const CallEvent &Call, CheckerContext &C) const {
+    if (const HandlerFn *Callback = Callbacks.lookup(Call))
+      return (this->*(*Callback))(Call, C);
+    return false;
+  }
+};
+
+void addFalsePositiveGenerator(AnalysisASTConsumer &AnalysisConsumer,
+                               AnalyzerOptions &AnOpts) {
+  AnOpts.CheckersAndPackages = {{"test.FalsePositiveGenerator", true},
+                                {"debug.ViewExplodedGraph", false}};
+  AnalysisConsumer.AddCheckerRegistrationFn([](CheckerRegistry &Registry) {
+    Registry.addChecker<FalsePositiveGenerator>(
+        "test.FalsePositiveGenerator", "EmptyDescription", "EmptyDocsUri");
+  });
+}
+
+// C++20 use constexpr below.
+const std::vector<std::string> LazyAssumeArgs{
+    "-Xclang", "-analyzer-config", "-Xclang", "eagerly-assume=false"};
+const std::vector<std::string> LazyAssumeAndCrossCheckArgs{
+    "-Xclang", "-analyzer-config", "-Xclang", "eagerly-assume=false",
+    "-Xclang", "-analyzer-config", "-Xclang", "crosscheck-with-z3=true"};
+
+TEST(FalsePositiveRefutationBRVisitor, UnSatInTheMiddleNoReport) {
+  SKIP_WITHOUT_Z3;
+  constexpr auto Code = R"(
+     void reachedWithContradiction();
+     void reachedWithNoContradiction();
+     void test(int x, int y) {
+       if (x * y == 0)
+         return;
+       reachedWithNoContradiction();
+       if (x == 0) {
+         reachedWithContradiction();
+         // x * y != 0  =>  x != 0 && y != 0  => contradict with x == 0
+       }
+     })";
+
+  std::string Diags;
+  EXPECT_TRUE(runCheckerOnCodeWithArgs<addFalsePositiveGenerator>(
+      Code, LazyAssumeAndCrossCheckArgs, Diags));
+  EXPECT_EQ(Diags,
+            "test.FalsePositiveGenerator:REACHED_WITH_NO_CONTRADICTION\n");
+  // Single warning. The second report was invalidated by the visitor.
+
+  // Without enabling the crosscheck-with-z3 both reports are displayed.
+  std::string Diags2;
+  EXPECT_TRUE(runCheckerOnCodeWithArgs<addFalsePositiveGenerator>(
+      Code, LazyAssumeArgs, Diags2));
+  EXPECT_EQ(Diags2,
+            "test.FalsePositiveGenerator:REACHED_WITH_NO_CONTRADICTION\n"
+            "test.FalsePositiveGenerator:REACHED_WITH_CONTRADICTION\n");
+}
+
+TEST(FalsePositiveRefutationBRVisitor, UnSatAtErrorNodeWithNewSymbolNoReport) {
+  SKIP_WITHOUT_Z3;
+  constexpr auto Code = R"(
+    void reportIfCanBeTrue(bool);
+    void reachedWithNoContradiction();
+    void test(int x, int y) {
+      if (x * y == 0)
+       return;
+      // We know that 'x * y': {[MIN,-1], [1,MAX]}
+      reachedWithNoContradiction();
+      reportIfCanBeTrue(x == 0); // contradiction
+      // The function introduces the 'x == 0' constraint in the ErrorNode which
+      // leads to contradiction with the constraint of 'x * y'.
+      // Note that the new constraint was bound to a new symbol 'x'.
+    })";
+  std::string Diags;
+  EXPECT_TRUE(runCheckerOnCodeWithArgs<addFalsePositiveGenerator>(
+      Code, LazyAssumeAndCrossCheckArgs, Diags));
+  EXPECT_EQ(Diags,
+            "test.FalsePositiveGenerator:REACHED_WITH_NO_CONTRADICTION\n");
+  // Single warning. The second report was invalidated by the visitor.
+
+  // Without enabling the crosscheck-with-z3 both reports are displayed.
+  std::string Diags2;
+  EXPECT_TRUE(runCheckerOnCodeWithArgs<addFalsePositiveGenerator>(
+      Code, LazyAssumeArgs, Diags2));
+  EXPECT_EQ(Diags2,
+            "test.FalsePositiveGenerator:REACHED_WITH_NO_CONTRADICTION\n"
+            "test.FalsePositiveGenerator:CAN_BE_TRUE\n");
+}
+
+TEST(FalsePositiveRefutationBRVisitor,
+     UnSatAtErrorNodeDueToRefinedConstraintNoReport) {
+  SKIP_WITHOUT_Z3;
+  constexpr auto Code = R"(
+    void reportIfCanBeTrue(bool);
+    void reachedWithNoContradiction();
+    void test(unsigned x, unsigned n) {
+      if (n >= 1 && n <= 2) {
+        if (x >= 3)
+          return;
+        // x: [0,2] and n: [1,2]
+        int y = x + n; // y: '(x+n)' Which is in approximately between 1 and 4.
+
+        // Registers the symbol 'y' with the constraint [1, MAX] in the true
+        // branch.
+        if (y > 0) {
+          // Since the x: [0,2] and n: [1,2], the 'y' is indeed greater than
+          // zero. If we emit a warning here, the constraints on the BugPath is
+          // SAT. Therefore that report is NOT invalidated.
+          reachedWithNoContradiction(); // 'y' can be greater than zero. OK
+
+          // If we ask the analyzer whether the 'y' can be 5. It won't know,
+          // therefore, the state will be created where the 'y' expression is 5.
+          // Although, this assumption is false!
+          // 'y' can not be 5 if the maximal value of both x and n is 2.
+          // The BugPath which become UnSAT in the ErrorNode with a refined
+          // constraint, should be invalidated.
+          reportIfCanBeTrue(y == 5);
+        }
+      }
+    })";
+
+  std::string Diags;
+  EXPECT_TRUE(runCheckerOnCodeWithArgs<addFalsePositiveGenerator>(
+      Code, LazyAssumeAndCrossCheckArgs, Diags));
+  EXPECT_EQ(Diags,
+            "test.FalsePositiveGenerator:REACHED_WITH_NO_CONTRADICTION\n");
+  // Single warning. The second report was invalidated by the visitor.
+
+  // Without enabling the crosscheck-with-z3 both reports are displayed.
+  std::string Diags2;
+  EXPECT_TRUE(runCheckerOnCodeWithArgs<addFalsePositiveGenerator>(
+      Code, LazyAssumeArgs, Diags2));
+  EXPECT_EQ(Diags2,
+            "test.FalsePositiveGenerator:REACHED_WITH_NO_CONTRADICTION\n"
+            "test.FalsePositiveGenerator:CAN_BE_TRUE\n");
+}
+
+} // namespace
+} // namespace ento
+} // namespace clang
diff --git a/clang/unittests/StaticAnalyzer/RegisterCustomCheckersTest.cpp b/clang/unittests/StaticAnalyzer/RegisterCustomCheckersTest.cpp
index 30a4efc06a4a0..60b8aafa0d84e 100644
--- a/clang/unittests/StaticAnalyzer/RegisterCustomCheckersTest.cpp
+++ b/clang/unittests/StaticAnalyzer/RegisterCustomCheckersTest.cpp
@@ -11,6 +11,7 @@
 #include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "clang/StaticAnalyzer/Core/Checker.h"
+#include "clang/StaticAnalyzer/Core/CheckerRegistryData.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Frontend/AnalysisConsumer.h"
@@ -41,17 +42,16 @@ class CustomChecker : public Checker<check::ASTCodeBody> {
 
 void addCustomChecker(AnalysisASTConsumer &AnalysisConsumer,
                       AnalyzerOptions &AnOpts) {
-  AnOpts.CheckersAndPackages = {{"custom.CustomChecker", true}};
+  AnOpts.CheckersAndPackages = {{"test.CustomChecker", true}};
   AnalysisConsumer.AddCheckerRegistrationFn([](CheckerRegistry &Registry) {
-    Registry.addChecker<CustomChecker>("custom.CustomChecker", "Description",
-                                       "");
+    Registry.addChecker<CustomChecker>("test.CustomChecker", "Description", "");
   });
 }
 
 TEST(RegisterCustomCheckers, RegisterChecker) {
   std::string Diags;
   EXPECT_TRUE(runCheckerOnCode<addCustomChecker>("void f() {;}", Diags));
-  EXPECT_EQ(Diags, "custom.CustomChecker:Custom diagnostic description\n");
+  EXPECT_EQ(Diags, "test.CustomChecker:Custom diagnostic description\n");
 }
 
 //===----------------------------------------------------------------------===//
@@ -100,7 +100,7 @@ class CheckerRegistrationOrderPrinter
     llvm::raw_svector_ostream OS(Buf);
     C.getAnalysisManager()
         .getCheckerManager()
-        ->getCheckerRegistry()
+        ->getCheckerRegistryData()
         .printEnabledCheckerList(OS);
     // Strip a newline off.
     auto R =
@@ -120,7 +120,7 @@ bool shouldRegisterCheckerRegistrationOrderPrinter(const CheckerManager &mgr) {
 void addCheckerRegistrationOrderPrinter(CheckerRegistry &Registry) {
   Registry.addChecker(registerCheckerRegistrationOrderPrinter,
                       shouldRegisterCheckerRegistrationOrderPrinter,
-                      "custom.RegistrationOrder", "Description", "", false);
+                      "test.RegistrationOrder", "Description", "", false);
 }
 
 #define UNITTEST_CHECKER(CHECKER_NAME, DIAG_MSG)                               \
@@ -141,7 +141,7 @@ void addCheckerRegistrationOrderPrinter(CheckerRegistry &Registry) {
   }                                                                            \
   void add##CHECKER_NAME(CheckerRegistry &Registry) {                          \
     Registry.addChecker(register##CHECKER_NAME, shouldRegister##CHECKER_NAME,  \
-                        "custom." #CHECKER_NAME, "Description", "", false);    \
+                        "test." #CHECKER_NAME, "Description", "", false);      \
   }
 
 UNITTEST_CHECKER(StrongDep, "Strong")
@@ -154,22 +154,22 @@ bool shouldRegisterStrongFALSE(const CheckerManager &mgr) {
 
 void addDep(AnalysisASTConsumer &AnalysisConsumer,
                   AnalyzerOptions &AnOpts) {
-  AnOpts.CheckersAndPackages = {{"custom.Dep", true},
-                                {"custom.RegistrationOrder", true}};
+  AnOpts.CheckersAndPackages = {{"test.Dep", true},
+                                {"test.RegistrationOrder", true}};
   AnalysisConsumer.AddCheckerRegistrationFn([](CheckerRegistry &Registry) {
     Registry.addChecker(registerStrongDep, shouldRegisterStrongFALSE,
-                        "custom.Strong", "Description", "", false);
+                        "test.Strong", "Description", "", false);
     addStrongDep(Registry);
     addDep(Registry);
     addCheckerRegistrationOrderPrinter(Registry);
-    Registry.addDependency("custom.Dep", "custom.Strong");
+    Registry.addDependency("test.Dep", "test.Strong");
   });
 }
 
 TEST(RegisterDeps, UnsatisfiedDependency) {
   std::string Diags;
   EXPECT_TRUE(runCheckerOnCode<addDep>("void f() {int i;}", Diags));
-  EXPECT_EQ(Diags, "custom.RegistrationOrder:custom.RegistrationOrder\n");
+  EXPECT_EQ(Diags, "test.RegistrationOrder:test.RegistrationOrder\n");
 }
 
 //===----------------------------------------------------------------------===//
@@ -180,52 +180,52 @@ UNITTEST_CHECKER(WeakDep, "Weak")
 
 void addWeakDepCheckerBothEnabled(AnalysisASTConsumer &AnalysisConsumer,
                                   AnalyzerOptions &AnOpts) {
-  AnOpts.CheckersAndPackages = {{"custom.Dep", true},
-                                {"custom.WeakDep", true},
-                                {"custom.RegistrationOrder", true}};
+  AnOpts.CheckersAndPackages = {{"test.Dep", true},
+                                {"test.WeakDep", true},
+                                {"test.RegistrationOrder", true}};
   AnalysisConsumer.AddCheckerRegistrationFn([=](CheckerRegistry &Registry) {
     addWeakDep(Registry);
     addDep(Registry);
     addCheckerRegistrationOrderPrinter(Registry);
-    Registry.addWeakDependency("custom.Dep", "custom.WeakDep");
+    Registry.addWeakDependency("test.Dep", "test.WeakDep");
   });
 }
 
 void addWeakDepCheckerBothEnabledSwitched(AnalysisASTConsumer &AnalysisConsumer,
                                           AnalyzerOptions &AnOpts) {
-  AnOpts.CheckersAndPackages = {{"custom.Dep", true},
-                                {"custom.WeakDep", true},
-                                {"custom.RegistrationOrder", true}};
+  AnOpts.CheckersAndPackages = {{"test.Dep", true},
+                                {"test.WeakDep", true},
+                                {"test.RegistrationOrder", true}};
   AnalysisConsumer.AddCheckerRegistrationFn([=](CheckerRegistry &Registry) {
     addWeakDep(Registry);
     addDep(Registry);
     addCheckerRegistrationOrderPrinter(Registry);
-    Registry.addWeakDependency("custom.WeakDep", "custom.Dep");
+    Registry.addWeakDependency("test.WeakDep", "test.Dep");
   });
 }
 
 void addWeakDepCheckerDepDisabled(AnalysisASTConsumer &AnalysisConsumer,
                                   AnalyzerOptions &AnOpts) {
-  AnOpts.CheckersAndPackages = {{"custom.Dep", true},
-                                {"custom.WeakDep", false},
-                                {"custom.RegistrationOrder", true}};
+  AnOpts.CheckersAndPackages = {{"test.Dep", true},
+                                {"test.WeakDep", false},
+                                {"test.RegistrationOrder", true}};
   AnalysisConsumer.AddCheckerRegistrationFn([=](CheckerRegistry &Registry) {
     addWeakDep(Registry);
     addDep(Registry);
     addCheckerRegistrationOrderPrinter(Registry);
-    Registry.addWeakDependency("custom.Dep", "custom.WeakDep");
+    Registry.addWeakDependency("test.Dep", "test.WeakDep");
   });
 }
 
 void addWeakDepCheckerDepUnspecified(AnalysisASTConsumer &AnalysisConsumer,
                                      AnalyzerOptions &AnOpts) {
-  AnOpts.CheckersAndPackages = {{"custom.Dep", true},
-                                {"custom.RegistrationOrder", true}};
+  AnOpts.CheckersAndPackages = {{"test.Dep", true},
+                                {"test.RegistrationOrder", true}};
   AnalysisConsumer.AddCheckerRegistrationFn([=](CheckerRegistry &Registry) {
     addWeakDep(Registry);
     addDep(Registry);
     addCheckerRegistrationOrderPrinter(Registry);
-    Registry.addWeakDependency("custom.Dep", "custom.WeakDep");
+    Registry.addWeakDependency("test.Dep", "test.WeakDep");
   });
 }
 
@@ -234,10 +234,10 @@ UNITTEST_CHECKER(Dep2, "Dep2")
 
 void addWeakDepHasWeakDep(AnalysisASTConsumer &AnalysisConsumer,
                           AnalyzerOptions &AnOpts) {
-  AnOpts.CheckersAndPackages = {{"custom.Dep", true},
-                                {"custom.WeakDep", true},
-                                {"custom.WeakDep2", true},
-                                {"custom.RegistrationOrder", true}};
+  AnOpts.CheckersAndPackages = {{"test.Dep", true},
+                                {"test.WeakDep", true},
+                                {"test.WeakDep2", true},
+                                {"test.RegistrationOrder", true}};
   AnalysisConsumer.AddCheckerRegistrationFn([=](CheckerRegistry &Registry) {
     addStrongDep(Registry);
     addWeakDep(Registry);
@@ -245,17 +245,17 @@ void addWeakDepHasWeakDep(AnalysisASTConsumer &AnalysisConsumer,
     addDep(Registry);
     addDep2(Registry);
     addCheckerRegistrationOrderPrinter(Registry);
-    Registry.addWeakDependency("custom.Dep", "custom.WeakDep");
-    Registry.addWeakDependency("custom.WeakDep", "custom.WeakDep2");
+    Registry.addWeakDependency("test.Dep", "test.WeakDep");
+    Registry.addWeakDependency("test.WeakDep", "test.WeakDep2");
   });
 }
 
 void addWeakDepTransitivity(AnalysisASTConsumer &AnalysisConsumer,
                             AnalyzerOptions &AnOpts) {
-  AnOpts.CheckersAndPackages = {{"custom.Dep", true},
-                                {"custom.WeakDep", false},
-                                {"custom.WeakDep2", true},
-                                {"custom.RegistrationOrder", true}};
+  AnOpts.CheckersAndPackages = {{"test.Dep", true},
+                                {"test.WeakDep", false},
+                                {"test.WeakDep2", true},
+                                {"test.RegistrationOrder", true}};
   AnalysisConsumer.AddCheckerRegistrationFn([=](CheckerRegistry &Registry) {
     addStrongDep(Registry);
     addWeakDep(Registry);
@@ -263,8 +263,8 @@ void addWeakDepTransitivity(AnalysisASTConsumer &AnalysisConsumer,
     addDep(Registry);
     addDep2(Registry);
     addCheckerRegistrationOrderPrinter(Registry);
-    Registry.addWeakDependency("custom.Dep", "custom.WeakDep");
-    Registry.addWeakDependency("custom.WeakDep", "custom.WeakDep2");
+    Registry.addWeakDependency("test.Dep", "test.WeakDep");
+    Registry.addWeakDependency("test.WeakDep", "test.WeakDep2");
   });
 }
 
@@ -272,42 +272,40 @@ TEST(RegisterDeps, SimpleWeakDependency) {
   std::string Diags;
   EXPECT_TRUE(runCheckerOnCode<addWeakDepCheckerBothEnabled>(
       "void f() {int i;}", Diags));
-  EXPECT_EQ(Diags, "custom.RegistrationOrder:custom.WeakDep\ncustom."
-                   "Dep\ncustom.RegistrationOrder\n");
+  EXPECT_EQ(Diags, "test.RegistrationOrder:test.WeakDep\ntest."
+                   "Dep\ntest.RegistrationOrder\n");
   Diags.clear();
 
   // Mind that AnalyzerOption listed the enabled checker list in the same order,
   // but the dependencies are switched.
   EXPECT_TRUE(runCheckerOnCode<addWeakDepCheckerBothEnabledSwitched>(
       "void f() {int i;}", Diags));
-  EXPECT_EQ(Diags, "custom.RegistrationOrder:custom.Dep\ncustom."
-                   "RegistrationOrder\ncustom.WeakDep\n");
+  EXPECT_EQ(Diags, "test.RegistrationOrder:test.Dep\ntest."
+                   "RegistrationOrder\ntest.WeakDep\n");
   Diags.clear();
 
   // Weak dependencies dont prevent dependent checkers from being enabled.
   EXPECT_TRUE(runCheckerOnCode<addWeakDepCheckerDepDisabled>(
       "void f() {int i;}", Diags));
-  EXPECT_EQ(Diags,
-            "custom.RegistrationOrder:custom.Dep\ncustom.RegistrationOrder\n");
+  EXPECT_EQ(Diags, "test.RegistrationOrder:test.Dep\ntest.RegistrationOrder\n");
   Diags.clear();
 
   // Nor will they be enabled just because a dependent checker is.
   EXPECT_TRUE(runCheckerOnCode<addWeakDepCheckerDepUnspecified>(
       "void f() {int i;}", Diags));
-  EXPECT_EQ(Diags,
-            "custom.RegistrationOrder:custom.Dep\ncustom.RegistrationOrder\n");
+  EXPECT_EQ(Diags, "test.RegistrationOrder:test.Dep\ntest.RegistrationOrder\n");
   Diags.clear();
 
   EXPECT_TRUE(
       runCheckerOnCode<addWeakDepTransitivity>("void f() {int i;}", Diags));
-  EXPECT_EQ(Diags, "custom.RegistrationOrder:custom.WeakDep2\ncustom."
-                   "Dep\ncustom.RegistrationOrder\n");
+  EXPECT_EQ(Diags, "test.RegistrationOrder:test.WeakDep2\ntest."
+                   "Dep\ntest.RegistrationOrder\n");
   Diags.clear();
 
   EXPECT_TRUE(
       runCheckerOnCode<addWeakDepHasWeakDep>("void f() {int i;}", Diags));
-  EXPECT_EQ(Diags, "custom.RegistrationOrder:custom.WeakDep2\ncustom."
-                   "WeakDep\ncustom.Dep\ncustom.RegistrationOrder\n");
+  EXPECT_EQ(Diags, "test.RegistrationOrder:test.WeakDep2\ntest."
+                   "WeakDep\ntest.Dep\ntest.RegistrationOrder\n");
   Diags.clear();
 }
 
@@ -317,98 +315,98 @@ TEST(RegisterDeps, SimpleWeakDependency) {
 
 void addWeakDepHasStrongDep(AnalysisASTConsumer &AnalysisConsumer,
                             AnalyzerOptions &AnOpts) {
-  AnOpts.CheckersAndPackages = {{"custom.Dep", true},
-                                {"custom.StrongDep", true},
-                                {"custom.WeakDep", true},
-                                {"custom.RegistrationOrder", true}};
+  AnOpts.CheckersAndPackages = {{"test.Dep", true},
+                                {"test.StrongDep", true},
+                                {"test.WeakDep", true},
+                                {"test.RegistrationOrder", true}};
   AnalysisConsumer.AddCheckerRegistrationFn([=](CheckerRegistry &Registry) {
     addStrongDep(Registry);
     addWeakDep(Registry);
     addDep(Registry);
     addCheckerRegistrationOrderPrinter(Registry);
-    Registry.addDependency("custom.WeakDep", "custom.StrongDep");
-    Registry.addWeakDependency("custom.Dep", "custom.WeakDep");
+    Registry.addDependency("test.WeakDep", "test.StrongDep");
+    Registry.addWeakDependency("test.Dep", "test.WeakDep");
   });
 }
 
 void addWeakDepAndStrongDep(AnalysisASTConsumer &AnalysisConsumer,
                             AnalyzerOptions &AnOpts) {
-  AnOpts.CheckersAndPackages = {{"custom.Dep", true},
-                                {"custom.StrongDep", true},
-                                {"custom.WeakDep", true},
-                                {"custom.RegistrationOrder", true}};
+  AnOpts.CheckersAndPackages = {{"test.Dep", true},
+                                {"test.StrongDep", true},
+                                {"test.WeakDep", true},
+                                {"test.RegistrationOrder", true}};
   AnalysisConsumer.AddCheckerRegistrationFn([=](CheckerRegistry &Registry) {
     addStrongDep(Registry);
     addWeakDep(Registry);
     addDep(Registry);
     addCheckerRegistrationOrderPrinter(Registry);
-    Registry.addDependency("custom.Dep", "custom.StrongDep");
-    Registry.addWeakDependency("custom.Dep", "custom.WeakDep");
+    Registry.addDependency("test.Dep", "test.StrongDep");
+    Registry.addWeakDependency("test.Dep", "test.WeakDep");
   });
 }
 
 void addDisabledWeakDepHasStrongDep(AnalysisASTConsumer &AnalysisConsumer,
                                     AnalyzerOptions &AnOpts) {
-  AnOpts.CheckersAndPackages = {{"custom.Dep", true},
-                                {"custom.StrongDep", true},
-                                {"custom.WeakDep", false},
-                                {"custom.RegistrationOrder", true}};
+  AnOpts.CheckersAndPackages = {{"test.Dep", true},
+                                {"test.StrongDep", true},
+                                {"test.WeakDep", false},
+                                {"test.RegistrationOrder", true}};
   AnalysisConsumer.AddCheckerRegistrationFn([=](CheckerRegistry &Registry) {
     addStrongDep(Registry);
     addWeakDep(Registry);
     addDep(Registry);
     addCheckerRegistrationOrderPrinter(Registry);
-    Registry.addDependency("custom.WeakDep", "custom.StrongDep");
-    Registry.addWeakDependency("custom.Dep", "custom.WeakDep");
+    Registry.addDependency("test.WeakDep", "test.StrongDep");
+    Registry.addWeakDependency("test.Dep", "test.WeakDep");
   });
 }
 
 void addDisabledWeakDepHasUnspecifiedStrongDep(
     AnalysisASTConsumer &AnalysisConsumer, AnalyzerOptions &AnOpts) {
-  AnOpts.CheckersAndPackages = {{"custom.Dep", true},
-                                {"custom.WeakDep", false},
-                                {"custom.RegistrationOrder", true}};
+  AnOpts.CheckersAndPackages = {{"test.Dep", true},
+                                {"test.WeakDep", false},
+                                {"test.RegistrationOrder", true}};
   AnalysisConsumer.AddCheckerRegistrationFn([=](CheckerRegistry &Registry) {
     addStrongDep(Registry);
     addWeakDep(Registry);
     addDep(Registry);
     addCheckerRegistrationOrderPrinter(Registry);
-    Registry.addDependency("custom.WeakDep", "custom.StrongDep");
-    Registry.addWeakDependency("custom.Dep", "custom.WeakDep");
+    Registry.addDependency("test.WeakDep", "test.StrongDep");
+    Registry.addWeakDependency("test.Dep", "test.WeakDep");
   });
 }
 
 void addWeakDepHasDisabledStrongDep(AnalysisASTConsumer &AnalysisConsumer,
                                     AnalyzerOptions &AnOpts) {
-  AnOpts.CheckersAndPackages = {{"custom.Dep", true},
-                                {"custom.StrongDep", false},
-                                {"custom.WeakDep", true},
-                                {"custom.RegistrationOrder", true}};
+  AnOpts.CheckersAndPackages = {{"test.Dep", true},
+                                {"test.StrongDep", false},
+                                {"test.WeakDep", true},
+                                {"test.RegistrationOrder", true}};
   AnalysisConsumer.AddCheckerRegistrationFn([=](CheckerRegistry &Registry) {
     addStrongDep(Registry);
     addWeakDep(Registry);
     addDep(Registry);
     addCheckerRegistrationOrderPrinter(Registry);
-    Registry.addDependency("custom.WeakDep", "custom.StrongDep");
-    Registry.addWeakDependency("custom.Dep", "custom.WeakDep");
+    Registry.addDependency("test.WeakDep", "test.StrongDep");
+    Registry.addWeakDependency("test.Dep", "test.WeakDep");
   });
 }
 
 void addWeakDepHasUnspecifiedButLaterEnabledStrongDep(
     AnalysisASTConsumer &AnalysisConsumer, AnalyzerOptions &AnOpts) {
-  AnOpts.CheckersAndPackages = {{"custom.Dep", true},
-                                {"custom.Dep2", true},
-                                {"custom.WeakDep", true},
-                                {"custom.RegistrationOrder", true}};
+  AnOpts.CheckersAndPackages = {{"test.Dep", true},
+                                {"test.Dep2", true},
+                                {"test.WeakDep", true},
+                                {"test.RegistrationOrder", true}};
   AnalysisConsumer.AddCheckerRegistrationFn([=](CheckerRegistry &Registry) {
     addStrongDep(Registry);
     addWeakDep(Registry);
     addDep(Registry);
     addDep2(Registry);
     addCheckerRegistrationOrderPrinter(Registry);
-    Registry.addDependency("custom.WeakDep", "custom.StrongDep");
-    Registry.addDependency("custom.Dep2", "custom.StrongDep");
-    Registry.addWeakDependency("custom.Dep", "custom.WeakDep");
+    Registry.addDependency("test.WeakDep", "test.StrongDep");
+    Registry.addDependency("test.Dep2", "test.StrongDep");
+    Registry.addWeakDependency("test.Dep", "test.WeakDep");
   });
 }
 
@@ -416,8 +414,8 @@ TEST(RegisterDeps, DependencyInteraction) {
   std::string Diags;
   EXPECT_TRUE(
       runCheckerOnCode<addWeakDepHasStrongDep>("void f() {int i;}", Diags));
-  EXPECT_EQ(Diags, "custom.RegistrationOrder:custom.StrongDep\ncustom."
-                   "WeakDep\ncustom.Dep\ncustom.RegistrationOrder\n");
+  EXPECT_EQ(Diags, "test.RegistrationOrder:test.StrongDep\ntest."
+                   "WeakDep\ntest.Dep\ntest.RegistrationOrder\n");
   Diags.clear();
 
   // Weak dependencies are registered before strong dependencies. This is most
@@ -426,39 +424,36 @@ TEST(RegisterDeps, DependencyInteraction) {
   // established in between the modeling portion and the weak dependency.
   EXPECT_TRUE(
       runCheckerOnCode<addWeakDepAndStrongDep>("void f() {int i;}", Diags));
-  EXPECT_EQ(Diags, "custom.RegistrationOrder:custom.WeakDep\ncustom."
-                   "StrongDep\ncustom.Dep\ncustom.RegistrationOrder\n");
+  EXPECT_EQ(Diags, "test.RegistrationOrder:test.WeakDep\ntest."
+                   "StrongDep\ntest.Dep\ntest.RegistrationOrder\n");
   Diags.clear();
 
   // If a weak dependency is disabled, the checker itself can still be enabled.
   EXPECT_TRUE(runCheckerOnCode<addDisabledWeakDepHasStrongDep>(
       "void f() {int i;}", Diags));
-  EXPECT_EQ(Diags, "custom.RegistrationOrder:custom.Dep\ncustom."
-                   "RegistrationOrder\ncustom.StrongDep\n");
+  EXPECT_EQ(Diags, "test.RegistrationOrder:test.Dep\ntest."
+                   "RegistrationOrder\ntest.StrongDep\n");
   Diags.clear();
 
   // If a weak dependency is disabled, the checker itself can still be enabled,
   // but it shouldn't enable a strong unspecified dependency.
   EXPECT_TRUE(runCheckerOnCode<addDisabledWeakDepHasUnspecifiedStrongDep>(
       "void f() {int i;}", Diags));
-  EXPECT_EQ(Diags,
-            "custom.RegistrationOrder:custom.Dep\ncustom.RegistrationOrder\n");
+  EXPECT_EQ(Diags, "test.RegistrationOrder:test.Dep\ntest.RegistrationOrder\n");
   Diags.clear();
 
   // A strong dependency of a weak dependency is disabled, so neither of them
   // should be enabled.
   EXPECT_TRUE(runCheckerOnCode<addWeakDepHasDisabledStrongDep>(
       "void f() {int i;}", Diags));
-  EXPECT_EQ(Diags,
-            "custom.RegistrationOrder:custom.Dep\ncustom.RegistrationOrder\n");
+  EXPECT_EQ(Diags, "test.RegistrationOrder:test.Dep\ntest.RegistrationOrder\n");
   Diags.clear();
 
   EXPECT_TRUE(
       runCheckerOnCode<addWeakDepHasUnspecifiedButLaterEnabledStrongDep>(
           "void f() {int i;}", Diags));
-  EXPECT_EQ(Diags,
-            "custom.RegistrationOrder:custom.StrongDep\ncustom.WeakDep\ncustom."
-            "Dep\ncustom.Dep2\ncustom.RegistrationOrder\n");
+  EXPECT_EQ(Diags, "test.RegistrationOrder:test.StrongDep\ntest.WeakDep\ntest."
+                   "Dep\ntest.Dep2\ntest.RegistrationOrder\n");
   Diags.clear();
 }
 } // namespace
diff --git a/clang/unittests/Tooling/CMakeLists.txt b/clang/unittests/Tooling/CMakeLists.txt
index 67ab6bea20160..c439f5a786375 100644
--- a/clang/unittests/Tooling/CMakeLists.txt
+++ b/clang/unittests/Tooling/CMakeLists.txt
@@ -9,6 +9,7 @@ set(LLVM_LINK_COMPONENTS
 if (MSVC)
   set_source_files_properties(RecursiveASTVisitorTest.cpp PROPERTIES COMPILE_FLAGS /bigobj)
   set_source_files_properties(RecursiveASTVisitorTestExprVisitor.cpp PROPERTIES COMPILE_FLAGS /bigobj)
+  set_source_files_properties(RecursiveASTVisitorTests/Callbacks.cpp PROPERTIES COMPILE_FLAGS /bigobj)
   set_source_files_properties(SourceCodeTest.cpp PROPERTIES COMPILE_FLAGS /bigobj)
 endif()
 
@@ -27,6 +28,7 @@ add_clang_unittest(ToolingTests
   QualTypeNamesTest.cpp
   RangeSelectorTest.cpp
   RecursiveASTVisitorTests/Attr.cpp
+  RecursiveASTVisitorTests/Callbacks.cpp
   RecursiveASTVisitorTests/Class.cpp
   RecursiveASTVisitorTests/ConstructExpr.cpp
   RecursiveASTVisitorTests/CXXBoolLiteralExpr.cpp
diff --git a/clang/unittests/Tooling/RangeSelectorTest.cpp b/clang/unittests/Tooling/RangeSelectorTest.cpp
index da5b3c524e4b0..e2d7723eab116 100644
--- a/clang/unittests/Tooling/RangeSelectorTest.cpp
+++ b/clang/unittests/Tooling/RangeSelectorTest.cpp
@@ -201,7 +201,7 @@ TEST(RangeSelectorTest, RangeOpNodes) {
   )cc";
   auto Matcher = callExpr(hasArgument(0, expr().bind("a0")),
                           hasArgument(1, expr().bind("a1")));
-  RangeSelector R = range("a0", "a1");
+  RangeSelector R = encloseNodes("a0", "a1");
   TestMatch Match = matchCode(Code, Matcher);
   EXPECT_THAT_EXPECTED(select(R, Match), HasValue("3, 7"));
 }
@@ -213,7 +213,7 @@ TEST(RangeSelectorTest, RangeOpGeneral) {
   )cc";
   auto Matcher = callExpr(hasArgument(0, expr().bind("a0")),
                           hasArgument(1, expr().bind("a1")));
-  RangeSelector R = range(node("a0"), node("a1"));
+  RangeSelector R = enclose(node("a0"), node("a1"));
   TestMatch Match = matchCode(Code, Matcher);
   EXPECT_THAT_EXPECTED(select(R, Match), HasValue("3, 7"));
 }
diff --git a/clang/unittests/Tooling/RecursiveASTVisitorTests/Callbacks.cpp b/clang/unittests/Tooling/RecursiveASTVisitorTests/Callbacks.cpp
new file mode 100644
index 0000000000000..71f2bf6be599f
--- /dev/null
+++ b/clang/unittests/Tooling/RecursiveASTVisitorTests/Callbacks.cpp
@@ -0,0 +1,1209 @@
+//===--- clang/unittests/Tooling/RecursiveASTVisitorTests/Callbacks.cpp ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestVisitor.h"
+
+using namespace clang;
+
+namespace {
+
+enum class ShouldTraversePostOrder : bool {
+  No = false,
+  Yes = true,
+};
+
+/// Base class for tests for RecursiveASTVisitor tests that validate the
+/// sequence of calls to user-defined callbacks like Traverse*(), WalkUp*(),
+/// Visit*().
+template <typename Derived>
+class RecordingVisitorBase : public TestVisitor<Derived> {
+  ShouldTraversePostOrder ShouldTraversePostOrderValue;
+
+public:
+  RecordingVisitorBase(ShouldTraversePostOrder ShouldTraversePostOrderValue)
+      : ShouldTraversePostOrderValue(ShouldTraversePostOrderValue) {}
+
+  bool shouldTraversePostOrder() const {
+    return static_cast<bool>(ShouldTraversePostOrderValue);
+  }
+
+  // Callbacks received during traversal.
+  std::string CallbackLog;
+  unsigned CallbackLogIndent = 0;
+
+  std::string stmtToString(Stmt *S) {
+    StringRef ClassName = S->getStmtClassName();
+    if (IntegerLiteral *IL = dyn_cast<IntegerLiteral>(S)) {
+      return (ClassName + "(" + IL->getValue().toString(10, false) + ")").str();
+    }
+    if (UnaryOperator *UO = dyn_cast<UnaryOperator>(S)) {
+      return (ClassName + "(" + UnaryOperator::getOpcodeStr(UO->getOpcode()) +
+              ")")
+          .str();
+    }
+    if (BinaryOperator *BO = dyn_cast<BinaryOperator>(S)) {
+      return (ClassName + "(" + BinaryOperator::getOpcodeStr(BO->getOpcode()) +
+              ")")
+          .str();
+    }
+    if (CallExpr *CE = dyn_cast<CallExpr>(S)) {
+      if (FunctionDecl *Callee = CE->getDirectCallee()) {
+        if (Callee->getIdentifier()) {
+          return (ClassName + "(" + Callee->getName() + ")").str();
+        }
+      }
+    }
+    if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(S)) {
+      if (NamedDecl *ND = DRE->getFoundDecl()) {
+        if (ND->getIdentifier()) {
+          return (ClassName + "(" + ND->getName() + ")").str();
+        }
+      }
+    }
+    return ClassName.str();
+  }
+
+  /// Record the fact that the user-defined callback member function
+  /// \p CallbackName was called with the argument \p S. Then, record the
+  /// effects of calling the default implementation \p CallDefaultFn.
+  template <typename CallDefault>
+  void recordCallback(StringRef CallbackName, Stmt *S,
+                      CallDefault CallDefaultFn) {
+    for (unsigned i = 0; i != CallbackLogIndent; ++i) {
+      CallbackLog += "  ";
+    }
+    CallbackLog += (CallbackName + " " + stmtToString(S) + "\n").str();
+    ++CallbackLogIndent;
+    CallDefaultFn();
+    --CallbackLogIndent;
+  }
+};
+
+template <typename VisitorTy>
+::testing::AssertionResult visitorCallbackLogEqual(VisitorTy Visitor,
+                                                   StringRef Code,
+                                                   StringRef ExpectedLog) {
+  Visitor.runOver(Code);
+  // EXPECT_EQ shows the diff between the two strings if they are different.
+  EXPECT_EQ(ExpectedLog.trim().str(),
+            StringRef(Visitor.CallbackLog).trim().str());
+  if (ExpectedLog.trim() != StringRef(Visitor.CallbackLog).trim()) {
+    return ::testing::AssertionFailure();
+  }
+  return ::testing::AssertionSuccess();
+}
+
+} // namespace
+
+TEST(RecursiveASTVisitor, StmtCallbacks_TraverseLeaf) {
+  class RecordingVisitor : public RecordingVisitorBase<RecordingVisitor> {
+  public:
+    RecordingVisitor(ShouldTraversePostOrder ShouldTraversePostOrderValue)
+        : RecordingVisitorBase(ShouldTraversePostOrderValue) {}
+
+    bool TraverseIntegerLiteral(IntegerLiteral *IL) {
+      recordCallback(__func__, IL, [&]() {
+        RecordingVisitorBase::TraverseIntegerLiteral(IL);
+      });
+      return true;
+    }
+
+    bool WalkUpFromStmt(Stmt *S) {
+      recordCallback(__func__, S,
+                     [&]() { RecordingVisitorBase::WalkUpFromStmt(S); });
+      return true;
+    }
+  };
+
+  StringRef Code = R"cpp(
+void add(int, int);
+void test() {
+  1;
+  2 + 3;
+  add(4, 5);
+}
+)cpp";
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::No), Code,
+      R"txt(
+WalkUpFromStmt CompoundStmt
+TraverseIntegerLiteral IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+WalkUpFromStmt BinaryOperator(+)
+TraverseIntegerLiteral IntegerLiteral(2)
+  WalkUpFromStmt IntegerLiteral(2)
+TraverseIntegerLiteral IntegerLiteral(3)
+  WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromStmt CallExpr(add)
+WalkUpFromStmt ImplicitCastExpr
+WalkUpFromStmt DeclRefExpr(add)
+TraverseIntegerLiteral IntegerLiteral(4)
+  WalkUpFromStmt IntegerLiteral(4)
+TraverseIntegerLiteral IntegerLiteral(5)
+  WalkUpFromStmt IntegerLiteral(5)
+)txt"));
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::Yes), Code,
+      R"txt(
+TraverseIntegerLiteral IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+TraverseIntegerLiteral IntegerLiteral(2)
+  WalkUpFromStmt IntegerLiteral(2)
+TraverseIntegerLiteral IntegerLiteral(3)
+  WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromStmt BinaryOperator(+)
+WalkUpFromStmt DeclRefExpr(add)
+WalkUpFromStmt ImplicitCastExpr
+TraverseIntegerLiteral IntegerLiteral(4)
+  WalkUpFromStmt IntegerLiteral(4)
+TraverseIntegerLiteral IntegerLiteral(5)
+  WalkUpFromStmt IntegerLiteral(5)
+WalkUpFromStmt CallExpr(add)
+WalkUpFromStmt CompoundStmt
+)txt"));
+}
+
+TEST(RecursiveASTVisitor, StmtCallbacks_TraverseLeaf_WalkUpFromLeaf) {
+  class RecordingVisitor : public RecordingVisitorBase<RecordingVisitor> {
+  public:
+    RecordingVisitor(ShouldTraversePostOrder ShouldTraversePostOrderValue)
+        : RecordingVisitorBase(ShouldTraversePostOrderValue) {}
+
+    bool TraverseIntegerLiteral(IntegerLiteral *IL) {
+      recordCallback(__func__, IL, [&]() {
+        RecordingVisitorBase::TraverseIntegerLiteral(IL);
+      });
+      return true;
+    }
+
+    bool WalkUpFromStmt(Stmt *S) {
+      recordCallback(__func__, S,
+                     [&]() { RecordingVisitorBase::WalkUpFromStmt(S); });
+      return true;
+    }
+
+    bool WalkUpFromExpr(Expr *E) {
+      recordCallback(__func__, E,
+                     [&]() { RecordingVisitorBase::WalkUpFromExpr(E); });
+      return true;
+    }
+
+    bool WalkUpFromIntegerLiteral(IntegerLiteral *IL) {
+      recordCallback(__func__, IL, [&]() {
+        RecordingVisitorBase::WalkUpFromIntegerLiteral(IL);
+      });
+      return true;
+    }
+  };
+
+  StringRef Code = R"cpp(
+void add(int, int);
+void test() {
+  1;
+  2 + 3;
+  add(4, 5);
+}
+)cpp";
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::No), Code,
+      R"txt(
+WalkUpFromStmt CompoundStmt
+TraverseIntegerLiteral IntegerLiteral(1)
+  WalkUpFromIntegerLiteral IntegerLiteral(1)
+    WalkUpFromExpr IntegerLiteral(1)
+      WalkUpFromStmt IntegerLiteral(1)
+WalkUpFromExpr BinaryOperator(+)
+  WalkUpFromStmt BinaryOperator(+)
+TraverseIntegerLiteral IntegerLiteral(2)
+  WalkUpFromIntegerLiteral IntegerLiteral(2)
+    WalkUpFromExpr IntegerLiteral(2)
+      WalkUpFromStmt IntegerLiteral(2)
+TraverseIntegerLiteral IntegerLiteral(3)
+  WalkUpFromIntegerLiteral IntegerLiteral(3)
+    WalkUpFromExpr IntegerLiteral(3)
+      WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromExpr CallExpr(add)
+  WalkUpFromStmt CallExpr(add)
+WalkUpFromExpr ImplicitCastExpr
+  WalkUpFromStmt ImplicitCastExpr
+WalkUpFromExpr DeclRefExpr(add)
+  WalkUpFromStmt DeclRefExpr(add)
+TraverseIntegerLiteral IntegerLiteral(4)
+  WalkUpFromIntegerLiteral IntegerLiteral(4)
+    WalkUpFromExpr IntegerLiteral(4)
+      WalkUpFromStmt IntegerLiteral(4)
+TraverseIntegerLiteral IntegerLiteral(5)
+  WalkUpFromIntegerLiteral IntegerLiteral(5)
+    WalkUpFromExpr IntegerLiteral(5)
+      WalkUpFromStmt IntegerLiteral(5)
+)txt"));
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::Yes), Code,
+      R"txt(
+TraverseIntegerLiteral IntegerLiteral(1)
+  WalkUpFromIntegerLiteral IntegerLiteral(1)
+    WalkUpFromExpr IntegerLiteral(1)
+      WalkUpFromStmt IntegerLiteral(1)
+TraverseIntegerLiteral IntegerLiteral(2)
+  WalkUpFromIntegerLiteral IntegerLiteral(2)
+    WalkUpFromExpr IntegerLiteral(2)
+      WalkUpFromStmt IntegerLiteral(2)
+TraverseIntegerLiteral IntegerLiteral(3)
+  WalkUpFromIntegerLiteral IntegerLiteral(3)
+    WalkUpFromExpr IntegerLiteral(3)
+      WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromExpr BinaryOperator(+)
+  WalkUpFromStmt BinaryOperator(+)
+WalkUpFromExpr DeclRefExpr(add)
+  WalkUpFromStmt DeclRefExpr(add)
+WalkUpFromExpr ImplicitCastExpr
+  WalkUpFromStmt ImplicitCastExpr
+TraverseIntegerLiteral IntegerLiteral(4)
+  WalkUpFromIntegerLiteral IntegerLiteral(4)
+    WalkUpFromExpr IntegerLiteral(4)
+      WalkUpFromStmt IntegerLiteral(4)
+TraverseIntegerLiteral IntegerLiteral(5)
+  WalkUpFromIntegerLiteral IntegerLiteral(5)
+    WalkUpFromExpr IntegerLiteral(5)
+      WalkUpFromStmt IntegerLiteral(5)
+WalkUpFromExpr CallExpr(add)
+  WalkUpFromStmt CallExpr(add)
+WalkUpFromStmt CompoundStmt
+)txt"));
+}
+
+TEST(RecursiveASTVisitor, StmtCallbacks_WalkUpFromLeaf) {
+  class RecordingVisitor : public RecordingVisitorBase<RecordingVisitor> {
+  public:
+    RecordingVisitor(ShouldTraversePostOrder ShouldTraversePostOrderValue)
+        : RecordingVisitorBase(ShouldTraversePostOrderValue) {}
+
+    bool WalkUpFromStmt(Stmt *S) {
+      recordCallback(__func__, S,
+                     [&]() { RecordingVisitorBase::WalkUpFromStmt(S); });
+      return true;
+    }
+
+    bool WalkUpFromExpr(Expr *E) {
+      recordCallback(__func__, E,
+                     [&]() { RecordingVisitorBase::WalkUpFromExpr(E); });
+      return true;
+    }
+
+    bool WalkUpFromIntegerLiteral(IntegerLiteral *IL) {
+      recordCallback(__func__, IL, [&]() {
+        RecordingVisitorBase::WalkUpFromIntegerLiteral(IL);
+      });
+      return true;
+    }
+  };
+
+  StringRef Code = R"cpp(
+void add(int, int);
+void test() {
+  1;
+  2 + 3;
+  add(4, 5);
+}
+)cpp";
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::No), Code,
+      R"txt(
+WalkUpFromStmt CompoundStmt
+WalkUpFromIntegerLiteral IntegerLiteral(1)
+  WalkUpFromExpr IntegerLiteral(1)
+    WalkUpFromStmt IntegerLiteral(1)
+WalkUpFromExpr BinaryOperator(+)
+  WalkUpFromStmt BinaryOperator(+)
+WalkUpFromIntegerLiteral IntegerLiteral(2)
+  WalkUpFromExpr IntegerLiteral(2)
+    WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromIntegerLiteral IntegerLiteral(3)
+  WalkUpFromExpr IntegerLiteral(3)
+    WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromExpr CallExpr(add)
+  WalkUpFromStmt CallExpr(add)
+WalkUpFromExpr ImplicitCastExpr
+  WalkUpFromStmt ImplicitCastExpr
+WalkUpFromExpr DeclRefExpr(add)
+  WalkUpFromStmt DeclRefExpr(add)
+WalkUpFromIntegerLiteral IntegerLiteral(4)
+  WalkUpFromExpr IntegerLiteral(4)
+    WalkUpFromStmt IntegerLiteral(4)
+WalkUpFromIntegerLiteral IntegerLiteral(5)
+  WalkUpFromExpr IntegerLiteral(5)
+    WalkUpFromStmt IntegerLiteral(5)
+)txt"));
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::Yes), Code,
+      R"txt(
+WalkUpFromIntegerLiteral IntegerLiteral(1)
+  WalkUpFromExpr IntegerLiteral(1)
+    WalkUpFromStmt IntegerLiteral(1)
+WalkUpFromIntegerLiteral IntegerLiteral(2)
+  WalkUpFromExpr IntegerLiteral(2)
+    WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromIntegerLiteral IntegerLiteral(3)
+  WalkUpFromExpr IntegerLiteral(3)
+    WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromExpr BinaryOperator(+)
+  WalkUpFromStmt BinaryOperator(+)
+WalkUpFromExpr DeclRefExpr(add)
+  WalkUpFromStmt DeclRefExpr(add)
+WalkUpFromExpr ImplicitCastExpr
+  WalkUpFromStmt ImplicitCastExpr
+WalkUpFromIntegerLiteral IntegerLiteral(4)
+  WalkUpFromExpr IntegerLiteral(4)
+    WalkUpFromStmt IntegerLiteral(4)
+WalkUpFromIntegerLiteral IntegerLiteral(5)
+  WalkUpFromExpr IntegerLiteral(5)
+    WalkUpFromStmt IntegerLiteral(5)
+WalkUpFromExpr CallExpr(add)
+  WalkUpFromStmt CallExpr(add)
+WalkUpFromStmt CompoundStmt
+)txt"));
+}
+
+TEST(RecursiveASTVisitor, StmtCallbacks_TraverseUnaryOperator) {
+  class RecordingVisitor : public RecordingVisitorBase<RecordingVisitor> {
+  public:
+    RecordingVisitor(ShouldTraversePostOrder ShouldTraversePostOrderValue)
+        : RecordingVisitorBase(ShouldTraversePostOrderValue) {}
+
+    bool TraverseUnaryOperator(UnaryOperator *UO) {
+      recordCallback(__func__, UO, [&]() {
+        RecordingVisitorBase::TraverseUnaryOperator(UO);
+      });
+      return true;
+    }
+
+    bool WalkUpFromStmt(Stmt *S) {
+      recordCallback(__func__, S,
+                     [&]() { RecordingVisitorBase::WalkUpFromStmt(S); });
+      return true;
+    }
+  };
+
+  StringRef Code = R"cpp(
+void test() {
+  1;
+  -2;
+  3;
+}
+)cpp";
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::No), Code,
+      R"txt(
+WalkUpFromStmt CompoundStmt
+WalkUpFromStmt IntegerLiteral(1)
+TraverseUnaryOperator UnaryOperator(-)
+  WalkUpFromStmt UnaryOperator(-)
+  WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromStmt IntegerLiteral(3)
+)txt"));
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::Yes), Code,
+      R"txt(
+WalkUpFromStmt IntegerLiteral(1)
+TraverseUnaryOperator UnaryOperator(-)
+  WalkUpFromStmt IntegerLiteral(2)
+  WalkUpFromStmt UnaryOperator(-)
+WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromStmt CompoundStmt
+)txt"));
+}
+
+TEST(RecursiveASTVisitor,
+     StmtCallbacks_TraverseUnaryOperator_WalkUpFromUnaryOperator) {
+  class RecordingVisitor : public RecordingVisitorBase<RecordingVisitor> {
+  public:
+    RecordingVisitor(ShouldTraversePostOrder ShouldTraversePostOrderValue)
+        : RecordingVisitorBase(ShouldTraversePostOrderValue) {}
+
+    bool TraverseUnaryOperator(UnaryOperator *UO) {
+      recordCallback(__func__, UO, [&]() {
+        RecordingVisitorBase::TraverseUnaryOperator(UO);
+      });
+      return true;
+    }
+
+    bool WalkUpFromStmt(Stmt *S) {
+      recordCallback(__func__, S,
+                     [&]() { RecordingVisitorBase::WalkUpFromStmt(S); });
+      return true;
+    }
+
+    bool WalkUpFromExpr(Expr *E) {
+      recordCallback(__func__, E,
+                     [&]() { RecordingVisitorBase::WalkUpFromExpr(E); });
+      return true;
+    }
+
+    bool WalkUpFromUnaryOperator(UnaryOperator *UO) {
+      recordCallback(__func__, UO, [&]() {
+        RecordingVisitorBase::WalkUpFromUnaryOperator(UO);
+      });
+      return true;
+    }
+  };
+
+  StringRef Code = R"cpp(
+void test() {
+  1;
+  -2;
+  3;
+}
+)cpp";
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::No), Code,
+      R"txt(
+WalkUpFromStmt CompoundStmt
+WalkUpFromExpr IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+TraverseUnaryOperator UnaryOperator(-)
+  WalkUpFromUnaryOperator UnaryOperator(-)
+    WalkUpFromExpr UnaryOperator(-)
+      WalkUpFromStmt UnaryOperator(-)
+  WalkUpFromExpr IntegerLiteral(2)
+    WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromExpr IntegerLiteral(3)
+  WalkUpFromStmt IntegerLiteral(3)
+)txt"));
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::Yes), Code,
+      R"txt(
+WalkUpFromExpr IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+TraverseUnaryOperator UnaryOperator(-)
+  WalkUpFromExpr IntegerLiteral(2)
+    WalkUpFromStmt IntegerLiteral(2)
+  WalkUpFromUnaryOperator UnaryOperator(-)
+    WalkUpFromExpr UnaryOperator(-)
+      WalkUpFromStmt UnaryOperator(-)
+WalkUpFromExpr IntegerLiteral(3)
+  WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromStmt CompoundStmt
+)txt"));
+}
+
+TEST(RecursiveASTVisitor, StmtCallbacks_WalkUpFromUnaryOperator) {
+  class RecordingVisitor : public RecordingVisitorBase<RecordingVisitor> {
+  public:
+    RecordingVisitor(ShouldTraversePostOrder ShouldTraversePostOrderValue)
+        : RecordingVisitorBase(ShouldTraversePostOrderValue) {}
+
+    bool WalkUpFromStmt(Stmt *S) {
+      recordCallback(__func__, S,
+                     [&]() { RecordingVisitorBase::WalkUpFromStmt(S); });
+      return true;
+    }
+
+    bool WalkUpFromExpr(Expr *E) {
+      recordCallback(__func__, E,
+                     [&]() { RecordingVisitorBase::WalkUpFromExpr(E); });
+      return true;
+    }
+
+    bool WalkUpFromUnaryOperator(UnaryOperator *UO) {
+      recordCallback(__func__, UO, [&]() {
+        RecordingVisitorBase::WalkUpFromUnaryOperator(UO);
+      });
+      return true;
+    }
+  };
+
+  StringRef Code = R"cpp(
+void test() {
+  1;
+  -2;
+  3;
+}
+)cpp";
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::No), Code,
+      R"txt(
+WalkUpFromStmt CompoundStmt
+WalkUpFromExpr IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+WalkUpFromUnaryOperator UnaryOperator(-)
+  WalkUpFromExpr UnaryOperator(-)
+    WalkUpFromStmt UnaryOperator(-)
+WalkUpFromExpr IntegerLiteral(2)
+  WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromExpr IntegerLiteral(3)
+  WalkUpFromStmt IntegerLiteral(3)
+)txt"));
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::Yes), Code,
+      R"txt(
+WalkUpFromExpr IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+WalkUpFromExpr IntegerLiteral(2)
+  WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromUnaryOperator UnaryOperator(-)
+  WalkUpFromExpr UnaryOperator(-)
+    WalkUpFromStmt UnaryOperator(-)
+WalkUpFromExpr IntegerLiteral(3)
+  WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromStmt CompoundStmt
+)txt"));
+}
+
+TEST(RecursiveASTVisitor, StmtCallbacks_TraverseBinaryOperator) {
+  class RecordingVisitor : public RecordingVisitorBase<RecordingVisitor> {
+  public:
+    RecordingVisitor(ShouldTraversePostOrder ShouldTraversePostOrderValue)
+        : RecordingVisitorBase(ShouldTraversePostOrderValue) {}
+
+    bool TraverseBinaryOperator(BinaryOperator *BO) {
+      recordCallback(__func__, BO, [&]() {
+        RecordingVisitorBase::TraverseBinaryOperator(BO);
+      });
+      return true;
+    }
+
+    bool WalkUpFromStmt(Stmt *S) {
+      recordCallback(__func__, S,
+                     [&]() { RecordingVisitorBase::WalkUpFromStmt(S); });
+      return true;
+    }
+  };
+
+  StringRef Code = R"cpp(
+void test() {
+  1;
+  2 + 3;
+  4;
+}
+)cpp";
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::No), Code,
+      R"txt(
+WalkUpFromStmt CompoundStmt
+WalkUpFromStmt IntegerLiteral(1)
+TraverseBinaryOperator BinaryOperator(+)
+  WalkUpFromStmt BinaryOperator(+)
+  WalkUpFromStmt IntegerLiteral(2)
+  WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromStmt IntegerLiteral(4)
+)txt"));
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::Yes), Code,
+      R"txt(
+WalkUpFromStmt IntegerLiteral(1)
+TraverseBinaryOperator BinaryOperator(+)
+  WalkUpFromStmt IntegerLiteral(2)
+  WalkUpFromStmt IntegerLiteral(3)
+  WalkUpFromStmt BinaryOperator(+)
+WalkUpFromStmt IntegerLiteral(4)
+WalkUpFromStmt CompoundStmt
+)txt"));
+}
+
+TEST(RecursiveASTVisitor,
+     StmtCallbacks_TraverseBinaryOperator_WalkUpFromBinaryOperator) {
+  class RecordingVisitor : public RecordingVisitorBase<RecordingVisitor> {
+  public:
+    RecordingVisitor(ShouldTraversePostOrder ShouldTraversePostOrderValue)
+        : RecordingVisitorBase(ShouldTraversePostOrderValue) {}
+
+    bool TraverseBinaryOperator(BinaryOperator *BO) {
+      recordCallback(__func__, BO, [&]() {
+        RecordingVisitorBase::TraverseBinaryOperator(BO);
+      });
+      return true;
+    }
+
+    bool WalkUpFromStmt(Stmt *S) {
+      recordCallback(__func__, S,
+                     [&]() { RecordingVisitorBase::WalkUpFromStmt(S); });
+      return true;
+    }
+
+    bool WalkUpFromExpr(Expr *E) {
+      recordCallback(__func__, E,
+                     [&]() { RecordingVisitorBase::WalkUpFromExpr(E); });
+      return true;
+    }
+
+    bool WalkUpFromBinaryOperator(BinaryOperator *BO) {
+      recordCallback(__func__, BO, [&]() {
+        RecordingVisitorBase::WalkUpFromBinaryOperator(BO);
+      });
+      return true;
+    }
+  };
+
+  StringRef Code = R"cpp(
+void test() {
+  1;
+  2 + 3;
+  4;
+}
+)cpp";
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::No), Code,
+      R"txt(
+WalkUpFromStmt CompoundStmt
+WalkUpFromExpr IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+TraverseBinaryOperator BinaryOperator(+)
+  WalkUpFromBinaryOperator BinaryOperator(+)
+    WalkUpFromExpr BinaryOperator(+)
+      WalkUpFromStmt BinaryOperator(+)
+  WalkUpFromExpr IntegerLiteral(2)
+    WalkUpFromStmt IntegerLiteral(2)
+  WalkUpFromExpr IntegerLiteral(3)
+    WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromExpr IntegerLiteral(4)
+  WalkUpFromStmt IntegerLiteral(4)
+)txt"));
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::Yes), Code,
+      R"txt(
+WalkUpFromExpr IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+TraverseBinaryOperator BinaryOperator(+)
+  WalkUpFromExpr IntegerLiteral(2)
+    WalkUpFromStmt IntegerLiteral(2)
+  WalkUpFromExpr IntegerLiteral(3)
+    WalkUpFromStmt IntegerLiteral(3)
+  WalkUpFromBinaryOperator BinaryOperator(+)
+    WalkUpFromExpr BinaryOperator(+)
+      WalkUpFromStmt BinaryOperator(+)
+WalkUpFromExpr IntegerLiteral(4)
+  WalkUpFromStmt IntegerLiteral(4)
+WalkUpFromStmt CompoundStmt
+)txt"));
+}
+
+TEST(RecursiveASTVisitor, StmtCallbacks_WalkUpFromBinaryOperator) {
+  class RecordingVisitor : public RecordingVisitorBase<RecordingVisitor> {
+  public:
+    RecordingVisitor(ShouldTraversePostOrder ShouldTraversePostOrderValue)
+        : RecordingVisitorBase(ShouldTraversePostOrderValue) {}
+
+    bool WalkUpFromStmt(Stmt *S) {
+      recordCallback(__func__, S,
+                     [&]() { RecordingVisitorBase::WalkUpFromStmt(S); });
+      return true;
+    }
+
+    bool WalkUpFromExpr(Expr *E) {
+      recordCallback(__func__, E,
+                     [&]() { RecordingVisitorBase::WalkUpFromExpr(E); });
+      return true;
+    }
+
+    bool WalkUpFromBinaryOperator(BinaryOperator *BO) {
+      recordCallback(__func__, BO, [&]() {
+        RecordingVisitorBase::WalkUpFromBinaryOperator(BO);
+      });
+      return true;
+    }
+  };
+
+  StringRef Code = R"cpp(
+void test() {
+  1;
+  2 + 3;
+  4;
+}
+)cpp";
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::No), Code,
+      R"txt(
+WalkUpFromStmt CompoundStmt
+WalkUpFromExpr IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+WalkUpFromBinaryOperator BinaryOperator(+)
+  WalkUpFromExpr BinaryOperator(+)
+    WalkUpFromStmt BinaryOperator(+)
+WalkUpFromExpr IntegerLiteral(2)
+  WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromExpr IntegerLiteral(3)
+  WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromExpr IntegerLiteral(4)
+  WalkUpFromStmt IntegerLiteral(4)
+)txt"));
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::Yes), Code,
+      R"txt(
+WalkUpFromExpr IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+WalkUpFromExpr IntegerLiteral(2)
+  WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromExpr IntegerLiteral(3)
+  WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromBinaryOperator BinaryOperator(+)
+  WalkUpFromExpr BinaryOperator(+)
+    WalkUpFromStmt BinaryOperator(+)
+WalkUpFromExpr IntegerLiteral(4)
+  WalkUpFromStmt IntegerLiteral(4)
+WalkUpFromStmt CompoundStmt
+)txt"));
+}
+
+TEST(RecursiveASTVisitor, StmtCallbacks_TraverseCompoundAssignOperator) {
+  class RecordingVisitor : public RecordingVisitorBase<RecordingVisitor> {
+  public:
+    RecordingVisitor(ShouldTraversePostOrder ShouldTraversePostOrderValue)
+        : RecordingVisitorBase(ShouldTraversePostOrderValue) {}
+
+    bool TraverseCompoundAssignOperator(CompoundAssignOperator *CAO) {
+      recordCallback(__func__, CAO, [&]() {
+        RecordingVisitorBase::TraverseCompoundAssignOperator(CAO);
+      });
+      return true;
+    }
+
+    bool WalkUpFromStmt(Stmt *S) {
+      recordCallback(__func__, S,
+                     [&]() { RecordingVisitorBase::WalkUpFromStmt(S); });
+      return true;
+    }
+  };
+
+  StringRef Code = R"cpp(
+void test(int a) {
+  1;
+  a += 2;
+  3;
+}
+)cpp";
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::No), Code,
+      R"txt(
+WalkUpFromStmt CompoundStmt
+WalkUpFromStmt IntegerLiteral(1)
+TraverseCompoundAssignOperator CompoundAssignOperator(+=)
+  WalkUpFromStmt CompoundAssignOperator(+=)
+  WalkUpFromStmt DeclRefExpr(a)
+  WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromStmt IntegerLiteral(3)
+)txt"));
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::Yes), Code,
+      R"txt(
+WalkUpFromStmt IntegerLiteral(1)
+TraverseCompoundAssignOperator CompoundAssignOperator(+=)
+  WalkUpFromStmt DeclRefExpr(a)
+  WalkUpFromStmt IntegerLiteral(2)
+  WalkUpFromStmt CompoundAssignOperator(+=)
+WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromStmt CompoundStmt
+)txt"));
+}
+
+TEST(
+    RecursiveASTVisitor,
+    StmtCallbacks_TraverseCompoundAssignOperator_WalkUpFromCompoundAssignOperator) {
+  class RecordingVisitor : public RecordingVisitorBase<RecordingVisitor> {
+  public:
+    RecordingVisitor(ShouldTraversePostOrder ShouldTraversePostOrderValue)
+        : RecordingVisitorBase(ShouldTraversePostOrderValue) {}
+
+    bool TraverseCompoundAssignOperator(CompoundAssignOperator *CAO) {
+      recordCallback(__func__, CAO, [&]() {
+        RecordingVisitorBase::TraverseCompoundAssignOperator(CAO);
+      });
+      return true;
+    }
+
+    bool WalkUpFromStmt(Stmt *S) {
+      recordCallback(__func__, S,
+                     [&]() { RecordingVisitorBase::WalkUpFromStmt(S); });
+      return true;
+    }
+
+    bool WalkUpFromExpr(Expr *E) {
+      recordCallback(__func__, E,
+                     [&]() { RecordingVisitorBase::WalkUpFromExpr(E); });
+      return true;
+    }
+
+    bool WalkUpFromCompoundAssignOperator(CompoundAssignOperator *CAO) {
+      recordCallback(__func__, CAO, [&]() {
+        RecordingVisitorBase::WalkUpFromCompoundAssignOperator(CAO);
+      });
+      return true;
+    }
+  };
+
+  StringRef Code = R"cpp(
+void test(int a) {
+  1;
+  a += 2;
+  3;
+}
+)cpp";
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::No), Code,
+      R"txt(
+WalkUpFromStmt CompoundStmt
+WalkUpFromExpr IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+TraverseCompoundAssignOperator CompoundAssignOperator(+=)
+  WalkUpFromCompoundAssignOperator CompoundAssignOperator(+=)
+    WalkUpFromExpr CompoundAssignOperator(+=)
+      WalkUpFromStmt CompoundAssignOperator(+=)
+  WalkUpFromExpr DeclRefExpr(a)
+    WalkUpFromStmt DeclRefExpr(a)
+  WalkUpFromExpr IntegerLiteral(2)
+    WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromExpr IntegerLiteral(3)
+  WalkUpFromStmt IntegerLiteral(3)
+)txt"));
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::Yes), Code,
+      R"txt(
+WalkUpFromExpr IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+TraverseCompoundAssignOperator CompoundAssignOperator(+=)
+  WalkUpFromExpr DeclRefExpr(a)
+    WalkUpFromStmt DeclRefExpr(a)
+  WalkUpFromExpr IntegerLiteral(2)
+    WalkUpFromStmt IntegerLiteral(2)
+  WalkUpFromCompoundAssignOperator CompoundAssignOperator(+=)
+    WalkUpFromExpr CompoundAssignOperator(+=)
+      WalkUpFromStmt CompoundAssignOperator(+=)
+WalkUpFromExpr IntegerLiteral(3)
+  WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromStmt CompoundStmt
+)txt"));
+}
+
+TEST(RecursiveASTVisitor, StmtCallbacks_WalkUpFromCompoundAssignOperator) {
+  class RecordingVisitor : public RecordingVisitorBase<RecordingVisitor> {
+  public:
+    RecordingVisitor(ShouldTraversePostOrder ShouldTraversePostOrderValue)
+        : RecordingVisitorBase(ShouldTraversePostOrderValue) {}
+
+    bool WalkUpFromStmt(Stmt *S) {
+      recordCallback(__func__, S,
+                     [&]() { RecordingVisitorBase::WalkUpFromStmt(S); });
+      return true;
+    }
+
+    bool WalkUpFromExpr(Expr *E) {
+      recordCallback(__func__, E,
+                     [&]() { RecordingVisitorBase::WalkUpFromExpr(E); });
+      return true;
+    }
+
+    bool WalkUpFromCompoundAssignOperator(CompoundAssignOperator *CAO) {
+      recordCallback(__func__, CAO, [&]() {
+        RecordingVisitorBase::WalkUpFromCompoundAssignOperator(CAO);
+      });
+      return true;
+    }
+  };
+
+  StringRef Code = R"cpp(
+void test(int a) {
+  1;
+  a += 2;
+  3;
+}
+)cpp";
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::No), Code,
+      R"txt(
+WalkUpFromStmt CompoundStmt
+WalkUpFromExpr IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+WalkUpFromCompoundAssignOperator CompoundAssignOperator(+=)
+  WalkUpFromExpr CompoundAssignOperator(+=)
+    WalkUpFromStmt CompoundAssignOperator(+=)
+WalkUpFromExpr DeclRefExpr(a)
+  WalkUpFromStmt DeclRefExpr(a)
+WalkUpFromExpr IntegerLiteral(2)
+  WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromExpr IntegerLiteral(3)
+  WalkUpFromStmt IntegerLiteral(3)
+)txt"));
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::Yes), Code,
+      R"txt(
+WalkUpFromExpr IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+WalkUpFromExpr DeclRefExpr(a)
+  WalkUpFromStmt DeclRefExpr(a)
+WalkUpFromExpr IntegerLiteral(2)
+  WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromCompoundAssignOperator CompoundAssignOperator(+=)
+  WalkUpFromExpr CompoundAssignOperator(+=)
+    WalkUpFromStmt CompoundAssignOperator(+=)
+WalkUpFromExpr IntegerLiteral(3)
+  WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromStmt CompoundStmt
+)txt"));
+}
+
+TEST(RecursiveASTVisitor, StmtCallbacks_TraverseCallExpr) {
+  class RecordingVisitor : public RecordingVisitorBase<RecordingVisitor> {
+  public:
+    RecordingVisitor(ShouldTraversePostOrder ShouldTraversePostOrderValue)
+        : RecordingVisitorBase(ShouldTraversePostOrderValue) {}
+
+    bool TraverseCallExpr(CallExpr *CE) {
+      recordCallback(__func__, CE,
+                     [&]() { RecordingVisitorBase::TraverseCallExpr(CE); });
+      return true;
+    }
+
+    bool WalkUpFromStmt(Stmt *S) {
+      recordCallback(__func__, S,
+                     [&]() { RecordingVisitorBase::WalkUpFromStmt(S); });
+      return true;
+    }
+  };
+
+  StringRef Code = R"cpp(
+void add(int, int);
+void test() {
+  1;
+  2 + 3;
+  add(4, 5);
+}
+)cpp";
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::No), Code,
+      R"txt(
+WalkUpFromStmt CompoundStmt
+WalkUpFromStmt IntegerLiteral(1)
+WalkUpFromStmt BinaryOperator(+)
+WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromStmt IntegerLiteral(3)
+TraverseCallExpr CallExpr(add)
+  WalkUpFromStmt CallExpr(add)
+  WalkUpFromStmt ImplicitCastExpr
+  WalkUpFromStmt DeclRefExpr(add)
+  WalkUpFromStmt IntegerLiteral(4)
+  WalkUpFromStmt IntegerLiteral(5)
+)txt"));
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::Yes), Code,
+      R"txt(
+WalkUpFromStmt IntegerLiteral(1)
+WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromStmt BinaryOperator(+)
+TraverseCallExpr CallExpr(add)
+  WalkUpFromStmt DeclRefExpr(add)
+  WalkUpFromStmt ImplicitCastExpr
+  WalkUpFromStmt IntegerLiteral(4)
+  WalkUpFromStmt IntegerLiteral(5)
+  WalkUpFromStmt CallExpr(add)
+WalkUpFromStmt CompoundStmt
+)txt"));
+}
+
+TEST(RecursiveASTVisitor, StmtCallbacks_TraverseCallExpr_WalkUpFromCallExpr) {
+  class RecordingVisitor : public RecordingVisitorBase<RecordingVisitor> {
+  public:
+    RecordingVisitor(ShouldTraversePostOrder ShouldTraversePostOrderValue)
+        : RecordingVisitorBase(ShouldTraversePostOrderValue) {}
+
+    bool TraverseCallExpr(CallExpr *CE) {
+      recordCallback(__func__, CE,
+                     [&]() { RecordingVisitorBase::TraverseCallExpr(CE); });
+      return true;
+    }
+
+    bool WalkUpFromStmt(Stmt *S) {
+      recordCallback(__func__, S,
+                     [&]() { RecordingVisitorBase::WalkUpFromStmt(S); });
+      return true;
+    }
+
+    bool WalkUpFromExpr(Expr *E) {
+      recordCallback(__func__, E,
+                     [&]() { RecordingVisitorBase::WalkUpFromExpr(E); });
+      return true;
+    }
+
+    bool WalkUpFromCallExpr(CallExpr *CE) {
+      recordCallback(__func__, CE,
+                     [&]() { RecordingVisitorBase::WalkUpFromCallExpr(CE); });
+      return true;
+    }
+  };
+
+  StringRef Code = R"cpp(
+void add(int, int);
+void test() {
+  1;
+  2 + 3;
+  add(4, 5);
+}
+)cpp";
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::No), Code,
+      R"txt(
+WalkUpFromStmt CompoundStmt
+WalkUpFromExpr IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+WalkUpFromExpr BinaryOperator(+)
+  WalkUpFromStmt BinaryOperator(+)
+WalkUpFromExpr IntegerLiteral(2)
+  WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromExpr IntegerLiteral(3)
+  WalkUpFromStmt IntegerLiteral(3)
+TraverseCallExpr CallExpr(add)
+  WalkUpFromCallExpr CallExpr(add)
+    WalkUpFromExpr CallExpr(add)
+      WalkUpFromStmt CallExpr(add)
+  WalkUpFromExpr ImplicitCastExpr
+    WalkUpFromStmt ImplicitCastExpr
+  WalkUpFromExpr DeclRefExpr(add)
+    WalkUpFromStmt DeclRefExpr(add)
+  WalkUpFromExpr IntegerLiteral(4)
+    WalkUpFromStmt IntegerLiteral(4)
+  WalkUpFromExpr IntegerLiteral(5)
+    WalkUpFromStmt IntegerLiteral(5)
+)txt"));
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::Yes), Code,
+      R"txt(
+WalkUpFromExpr IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+WalkUpFromExpr IntegerLiteral(2)
+  WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromExpr IntegerLiteral(3)
+  WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromExpr BinaryOperator(+)
+  WalkUpFromStmt BinaryOperator(+)
+TraverseCallExpr CallExpr(add)
+  WalkUpFromExpr DeclRefExpr(add)
+    WalkUpFromStmt DeclRefExpr(add)
+  WalkUpFromExpr ImplicitCastExpr
+    WalkUpFromStmt ImplicitCastExpr
+  WalkUpFromExpr IntegerLiteral(4)
+    WalkUpFromStmt IntegerLiteral(4)
+  WalkUpFromExpr IntegerLiteral(5)
+    WalkUpFromStmt IntegerLiteral(5)
+  WalkUpFromCallExpr CallExpr(add)
+    WalkUpFromExpr CallExpr(add)
+      WalkUpFromStmt CallExpr(add)
+WalkUpFromStmt CompoundStmt
+)txt"));
+}
+
+TEST(RecursiveASTVisitor, StmtCallbacks_WalkUpFromCallExpr) {
+  class RecordingVisitor : public RecordingVisitorBase<RecordingVisitor> {
+  public:
+    RecordingVisitor(ShouldTraversePostOrder ShouldTraversePostOrderValue)
+        : RecordingVisitorBase(ShouldTraversePostOrderValue) {}
+
+    bool WalkUpFromStmt(Stmt *S) {
+      recordCallback(__func__, S,
+                     [&]() { RecordingVisitorBase::WalkUpFromStmt(S); });
+      return true;
+    }
+
+    bool WalkUpFromExpr(Expr *E) {
+      recordCallback(__func__, E,
+                     [&]() { RecordingVisitorBase::WalkUpFromExpr(E); });
+      return true;
+    }
+
+    bool WalkUpFromCallExpr(CallExpr *CE) {
+      recordCallback(__func__, CE,
+                     [&]() { RecordingVisitorBase::WalkUpFromCallExpr(CE); });
+      return true;
+    }
+  };
+
+  StringRef Code = R"cpp(
+void add(int, int);
+void test() {
+  1;
+  2 + 3;
+  add(4, 5);
+}
+)cpp";
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::No), Code,
+      R"txt(
+WalkUpFromStmt CompoundStmt
+WalkUpFromExpr IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+WalkUpFromExpr BinaryOperator(+)
+  WalkUpFromStmt BinaryOperator(+)
+WalkUpFromExpr IntegerLiteral(2)
+  WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromExpr IntegerLiteral(3)
+  WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromCallExpr CallExpr(add)
+  WalkUpFromExpr CallExpr(add)
+    WalkUpFromStmt CallExpr(add)
+WalkUpFromExpr ImplicitCastExpr
+  WalkUpFromStmt ImplicitCastExpr
+WalkUpFromExpr DeclRefExpr(add)
+  WalkUpFromStmt DeclRefExpr(add)
+WalkUpFromExpr IntegerLiteral(4)
+  WalkUpFromStmt IntegerLiteral(4)
+WalkUpFromExpr IntegerLiteral(5)
+  WalkUpFromStmt IntegerLiteral(5)
+)txt"));
+
+  EXPECT_TRUE(visitorCallbackLogEqual(
+      RecordingVisitor(ShouldTraversePostOrder::Yes), Code,
+      R"txt(
+WalkUpFromExpr IntegerLiteral(1)
+  WalkUpFromStmt IntegerLiteral(1)
+WalkUpFromExpr IntegerLiteral(2)
+  WalkUpFromStmt IntegerLiteral(2)
+WalkUpFromExpr IntegerLiteral(3)
+  WalkUpFromStmt IntegerLiteral(3)
+WalkUpFromExpr BinaryOperator(+)
+  WalkUpFromStmt BinaryOperator(+)
+WalkUpFromExpr DeclRefExpr(add)
+  WalkUpFromStmt DeclRefExpr(add)
+WalkUpFromExpr ImplicitCastExpr
+  WalkUpFromStmt ImplicitCastExpr
+WalkUpFromExpr IntegerLiteral(4)
+  WalkUpFromStmt IntegerLiteral(4)
+WalkUpFromExpr IntegerLiteral(5)
+  WalkUpFromStmt IntegerLiteral(5)
+WalkUpFromCallExpr CallExpr(add)
+  WalkUpFromExpr CallExpr(add)
+    WalkUpFromStmt CallExpr(add)
+WalkUpFromStmt CompoundStmt
+)txt"));
+}
diff --git a/clang/unittests/Tooling/RefactoringTest.cpp b/clang/unittests/Tooling/RefactoringTest.cpp
index d65c6dba77b24..97a26a71deec7 100644
--- a/clang/unittests/Tooling/RefactoringTest.cpp
+++ b/clang/unittests/Tooling/RefactoringTest.cpp
@@ -1296,6 +1296,18 @@ TEST_F(AtomicChangeTest, InsertAfterWithInvalidLocation) {
       Replacement(Context.Sources, SourceLocation(), 0, "b")));
 }
 
+TEST_F(AtomicChangeTest, Metadata) {
+  AtomicChange Change(Context.Sources, DefaultLoc, 17);
+  const llvm::Any &Metadata = Change.getMetadata();
+  ASSERT_TRUE(llvm::any_isa<int>(Metadata));
+  EXPECT_EQ(llvm::any_cast<int>(Metadata), 17);
+}
+
+TEST_F(AtomicChangeTest, NoMetadata) {
+  AtomicChange Change(Context.Sources, DefaultLoc);
+  EXPECT_FALSE(Change.getMetadata().hasValue());
+}
+
 class ApplyAtomicChangesTest : public ::testing::Test {
 protected:
   ApplyAtomicChangesTest() : FilePath("file.cc") {
diff --git a/clang/unittests/Tooling/ReplacementsYamlTest.cpp b/clang/unittests/Tooling/ReplacementsYamlTest.cpp
index c8fe9c4db412b..3328d9bad55c7 100644
--- a/clang/unittests/Tooling/ReplacementsYamlTest.cpp
+++ b/clang/unittests/Tooling/ReplacementsYamlTest.cpp
@@ -65,7 +65,7 @@ TEST(ReplacementsYamlTest, serializesNewLines) {
                "  - FilePath:        '/path/to/file1.h'\n"
                "    Offset:          0\n"
                "    Length:          0\n"
-               "    ReplacementText: '#include <utility>\n\n'\n"
+               "    ReplacementText: \"#include <utility>\\n\"\n"
                "...\n",
                YamlContentStream.str().c_str());
 }
diff --git a/clang/unittests/Tooling/SourceCodeTest.cpp b/clang/unittests/Tooling/SourceCodeTest.cpp
index aeb0ca36582bc..eb652cf030644 100644
--- a/clang/unittests/Tooling/SourceCodeTest.cpp
+++ b/clang/unittests/Tooling/SourceCodeTest.cpp
@@ -9,6 +9,8 @@
 #include "clang/Tooling/Transformer/SourceCode.h"
 #include "TestVisitor.h"
 #include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Lex/Lexer.h"
 #include "llvm/Testing/Support/Annotations.h"
 #include "llvm/Testing/Support/Error.h"
 #include "llvm/Testing/Support/SupportHelpers.h"
@@ -21,9 +23,11 @@ using llvm::Failed;
 using llvm::Succeeded;
 using llvm::ValueIs;
 using tooling::getAssociatedRange;
+using tooling::getExtendedRange;
 using tooling::getExtendedText;
 using tooling::getRangeForEdit;
 using tooling::getText;
+using tooling::maybeExtendRange;
 using tooling::validateEditRange;
 
 namespace {
@@ -52,7 +56,7 @@ MATCHER_P(EqualsRange, R, "") {
          arg.getBegin() == R.getBegin() && arg.getEnd() == R.getEnd();
 }
 
-MATCHER_P2(EqualsAnnotatedRange, SM, R, "") {
+MATCHER_P2(EqualsAnnotatedRange, Context, R, "") {
   if (arg.getBegin().isMacroID()) {
     *result_listener << "which starts in a macro";
     return false;
@@ -62,15 +66,13 @@ MATCHER_P2(EqualsAnnotatedRange, SM, R, "") {
     return false;
   }
 
-  unsigned Begin = SM->getFileOffset(arg.getBegin());
-  unsigned End = SM->getFileOffset(arg.getEnd());
+  CharSourceRange Range = Lexer::getAsCharRange(
+      arg, Context->getSourceManager(), Context->getLangOpts());
+  unsigned Begin = Context->getSourceManager().getFileOffset(Range.getBegin());
+  unsigned End = Context->getSourceManager().getFileOffset(Range.getEnd());
 
-  *result_listener << "which is [" << Begin << ",";
-  if (arg.isTokenRange()) {
-    *result_listener << End << "]";
-    return Begin == R.Begin && End + 1 == R.End;
-  }
-  *result_listener << End << ")";
+  *result_listener << "which is a " << (arg.isTokenRange() ? "Token" : "Char")
+                   << " range [" << Begin << "," << End << ")";
   return Begin == R.Begin && End == R.End;
 }
 
@@ -84,11 +86,13 @@ static ::testing::Matcher<CharSourceRange> AsRange(const SourceManager &SM,
 // Base class for visitors that expect a single match corresponding to a
 // specific annotated range.
 template <typename T> class AnnotatedCodeVisitor : public TestVisitor<T> {
-  llvm::Annotations Code;
+protected:
   int MatchCount = 0;
+  llvm::Annotations Code;
 
 public:
   AnnotatedCodeVisitor() : Code("$r[[]]") {}
+  // Helper for tests of `getAssociatedRange`.
   bool VisitDeclHelper(Decl *Decl) {
     // Only consider explicit declarations.
     if (Decl->isImplicit())
@@ -96,8 +100,7 @@ template <typename T> class AnnotatedCodeVisitor : public TestVisitor<T> {
 
     ++MatchCount;
     EXPECT_THAT(getAssociatedRange(*Decl, *this->Context),
-                EqualsAnnotatedRange(&this->Context->getSourceManager(),
-                                     Code.range("r")))
+                EqualsAnnotatedRange(this->Context, Code.range("r")))
         << Code.code();
     return true;
   }
@@ -183,6 +186,45 @@ TEST(SourceCodeTest, getExtendedText) {
   Visitor.runOver("int foo() { return foo() + 3; }");
 }
 
+TEST(SourceCodeTest, maybeExtendRange_TokenRange) {
+  struct ExtendTokenRangeVisitor
+      : AnnotatedCodeVisitor<ExtendTokenRangeVisitor> {
+    bool VisitCallExpr(CallExpr *CE) {
+      ++MatchCount;
+      EXPECT_THAT(getExtendedRange(*CE, tok::TokenKind::semi, *Context),
+                  EqualsAnnotatedRange(Context, Code.range("r")));
+      return true;
+    }
+  };
+
+  ExtendTokenRangeVisitor Visitor;
+  // Extends to include semicolon.
+  Visitor.runOverAnnotated("void f(int x, int y) { $r[[f(x, y);]] }");
+  // Does not extend to include semicolon.
+  Visitor.runOverAnnotated(
+      "int f(int x, int y) { if (0) return $r[[f(x, y)]] + 3; }");
+}
+
+TEST(SourceCodeTest, maybeExtendRange_CharRange) {
+  struct ExtendCharRangeVisitor : AnnotatedCodeVisitor<ExtendCharRangeVisitor> {
+    bool VisitCallExpr(CallExpr *CE) {
+      ++MatchCount;
+      CharSourceRange Call = Lexer::getAsCharRange(CE->getSourceRange(),
+                                                   Context->getSourceManager(),
+                                                   Context->getLangOpts());
+      EXPECT_THAT(maybeExtendRange(Call, tok::TokenKind::semi, *Context),
+                  EqualsAnnotatedRange(Context, Code.range("r")));
+      return true;
+    }
+  };
+  ExtendCharRangeVisitor Visitor;
+  // Extends to include semicolon.
+  Visitor.runOverAnnotated("void f(int x, int y) { $r[[f(x, y);]] }");
+  // Does not extend to include semicolon.
+  Visitor.runOverAnnotated(
+      "int f(int x, int y) { if (0) return $r[[f(x, y)]] + 3; }");
+}
+
 TEST(SourceCodeTest, getAssociatedRange) {
   struct VarDeclsVisitor : AnnotatedCodeVisitor<VarDeclsVisitor> {
     bool VisitVarDecl(VarDecl *Decl) { return VisitDeclHelper(Decl); }
diff --git a/clang/unittests/Tooling/StencilTest.cpp b/clang/unittests/Tooling/StencilTest.cpp
index 9408a1e62dfa3..c843e33dd0daa 100644
--- a/clang/unittests/Tooling/StencilTest.cpp
+++ b/clang/unittests/Tooling/StencilTest.cpp
@@ -180,12 +180,12 @@ TEST_F(StencilTest, SelectionOp) {
 
 TEST_F(StencilTest, IfBoundOpBound) {
   StringRef Id = "id";
-  testExpr(Id, "3;", ifBound(Id, text("5"), text("7")), "5");
+  testExpr(Id, "3;", ifBound(Id, cat("5"), cat("7")), "5");
 }
 
 TEST_F(StencilTest, IfBoundOpUnbound) {
   StringRef Id = "id";
-  testExpr(Id, "3;", ifBound("other", text("5"), text("7")), "7");
+  testExpr(Id, "3;", ifBound("other", cat("5"), cat("7")), "7");
 }
 
 TEST_F(StencilTest, ExpressionOpNoParens) {
@@ -293,7 +293,7 @@ TEST_F(StencilTest, AccessOpValueExplicitText) {
     x;
   )cc";
   StringRef Id = "id";
-  testExpr(Id, Snippet, access(Id, text("field")), "x.field");
+  testExpr(Id, Snippet, access(Id, cat("field")), "x.field");
 }
 
 TEST_F(StencilTest, AccessOpValueAddress) {
@@ -428,7 +428,12 @@ TEST_F(StencilTest, CatOfInvalidRangeFails) {
                                   hasArgument(0, expr().bind("arg"))));
   ASSERT_TRUE(StmtMatch);
   Stencil S = cat(node("arg"));
-  EXPECT_THAT_EXPECTED(S->eval(StmtMatch->Result), Failed<StringError>());
+  Expected<std::string> Result = S->eval(StmtMatch->Result);
+  ASSERT_THAT_EXPECTED(Result, Failed<StringError>());
+  llvm::handleAllErrors(Result.takeError(), [](const llvm::StringError &E) {
+    EXPECT_THAT(E.getMessage(), AllOf(HasSubstr("selected range"),
+                                      HasSubstr("macro expansion")));
+  });
 }
 
 TEST(StencilToStringTest, RawTextOp) {
@@ -479,7 +484,7 @@ TEST(StencilToStringTest, AccessOpText) {
 }
 
 TEST(StencilToStringTest, AccessOpSelector) {
-  auto S = access("Id", selection(name("otherId")));
+  auto S = access("Id", cat(name("otherId")));
   StringRef Expected = R"repr(access("Id", selection(...)))repr";
   EXPECT_EQ(S->toString(), Expected);
 }
@@ -491,7 +496,7 @@ TEST(StencilToStringTest, AccessOpStencil) {
 }
 
 TEST(StencilToStringTest, IfBoundOp) {
-  auto S = ifBound("Id", text("trueText"), access("exprId", "memberData"));
+  auto S = ifBound("Id", cat("trueText"), access("exprId", "memberData"));
   StringRef Expected =
       R"repr(ifBound("Id", "trueText", access("exprId", "memberData")))repr";
   EXPECT_EQ(S->toString(), Expected);
@@ -505,7 +510,7 @@ TEST(StencilToStringTest, RunOp) {
 
 TEST(StencilToStringTest, Sequence) {
   auto S = cat("foo", access("x", "m()"), "bar",
-               ifBound("x", text("t"), access("e", "f")));
+               ifBound("x", cat("t"), access("e", "f")));
   StringRef Expected = R"repr(seq("foo", access("x", "m()"), "bar", )repr"
                        R"repr(ifBound("x", "t", access("e", "f"))))repr";
   EXPECT_EQ(S->toString(), Expected);
@@ -524,8 +529,8 @@ TEST(StencilToStringTest, SequenceSingle) {
 }
 
 TEST(StencilToStringTest, SequenceFromVector) {
-  auto S = catVector({text("foo"), access("x", "m()"), text("bar"),
-                      ifBound("x", text("t"), access("e", "f"))});
+  auto S = catVector({cat("foo"), access("x", "m()"), cat("bar"),
+                      ifBound("x", cat("t"), access("e", "f"))});
   StringRef Expected = R"repr(seq("foo", access("x", "m()"), "bar", )repr"
                        R"repr(ifBound("x", "t", access("e", "f"))))repr";
   EXPECT_EQ(S->toString(), Expected);
diff --git a/clang/unittests/Tooling/Syntax/TreeTest.cpp b/clang/unittests/Tooling/Syntax/TreeTest.cpp
index 5f48ef1299885..bd639aa581661 100644
--- a/clang/unittests/Tooling/Syntax/TreeTest.cpp
+++ b/clang/unittests/Tooling/Syntax/TreeTest.cpp
@@ -18,6 +18,7 @@
 #include "clang/Frontend/TextDiagnosticPrinter.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Testing/CommandLineArgs.h"
+#include "clang/Testing/TestClangConfig.h"
 #include "clang/Tooling/Core/Replacement.h"
 #include "clang/Tooling/Syntax/BuildTree.h"
 #include "clang/Tooling/Syntax/Mutations.h"
@@ -47,77 +48,6 @@ static llvm::ArrayRef<syntax::Token> tokens(syntax::Node *N) {
                             T->lastLeaf()->token() + 1);
 }
 
-struct TestClangConfig {
-  TestLanguage Language;
-  std::string Target;
-
-  bool isC99OrLater() const { return Language == Lang_C99; }
-
-  bool isCXX() const {
-    return Language == Lang_CXX03 || Language == Lang_CXX11 ||
-           Language == Lang_CXX14 || Language == Lang_CXX17 ||
-           Language == Lang_CXX20;
-  }
-
-  bool isCXX11OrLater() const {
-    return Language == Lang_CXX11 || Language == Lang_CXX14 ||
-           Language == Lang_CXX17 || Language == Lang_CXX20;
-  }
-
-  bool isCXX14OrLater() const {
-    return Language == Lang_CXX14 || Language == Lang_CXX17 ||
-           Language == Lang_CXX20;
-  }
-
-  bool supportsCXXDynamicExceptionSpecification() const {
-    return Language == Lang_CXX03 || Language == Lang_CXX11 ||
-           Language == Lang_CXX14;
-  }
-
-  bool hasDelayedTemplateParsing() const {
-    return Target == "x86_64-pc-win32-msvc";
-  }
-
-  std::vector<std::string> getCommandLineArgs() const {
-    std::vector<std::string> Result = getCommandLineArgsForTesting(Language);
-    Result.push_back("-target");
-    Result.push_back(Target);
-    return Result;
-  }
-
-  std::string toString() const {
-    std::string Result;
-    llvm::raw_string_ostream OS(Result);
-    OS << "{ Language=" << Language << ", Target=" << Target << " }";
-    return OS.str();
-  }
-
-  friend std::ostream &operator<<(std::ostream &OS,
-                                  const TestClangConfig &ClangConfig) {
-    return OS << ClangConfig.toString();
-  }
-
-  static std::vector<TestClangConfig> &allConfigs() {
-    static std::vector<TestClangConfig> all_configs = []() {
-      std::vector<TestClangConfig> all_configs;
-      for (TestLanguage lang : {Lang_C89, Lang_C99, Lang_CXX03, Lang_CXX11,
-                                Lang_CXX14, Lang_CXX17, Lang_CXX20}) {
-        TestClangConfig config;
-        config.Language = lang;
-        config.Target = "x86_64-pc-linux-gnu";
-        all_configs.push_back(config);
-
-        // Windows target is interesting to test because it enables
-        // `-fdelayed-template-parsing`.
-        config.Target = "x86_64-pc-win32-msvc";
-        all_configs.push_back(config);
-      }
-      return all_configs;
-    }();
-    return all_configs;
-  }
-};
-
 class SyntaxTreeTest : public ::testing::Test,
                        public ::testing::WithParamInterface<TestClangConfig> {
 protected:
@@ -1199,14 +1129,512 @@ void test(S s) {
 )txt"));
 }
 
-TEST_P(SyntaxTreeTest, CxxNullPtrLiteral) {
-  if (!GetParam().isCXX11OrLater()) {
+TEST_P(SyntaxTreeTest, ParenExpr) {
+  EXPECT_TRUE(treeDumpEqual(
+      R"cpp(
+void test() {
+  (1);
+  ((1));
+  (1 + (2));
+}
+)cpp",
+      R"txt(
+*: TranslationUnit
+`-SimpleDeclaration
+  |-void
+  |-SimpleDeclarator
+  | |-test
+  | `-ParametersAndQualifiers
+  |   |-(
+  |   `-)
+  `-CompoundStatement
+    |-{
+    |-ExpressionStatement
+    | |-ParenExpression
+    | | |-(
+    | | |-IntegerLiteralExpression
+    | | | `-1
+    | | `-)
+    | `-;
+    |-ExpressionStatement
+    | |-ParenExpression
+    | | |-(
+    | | |-ParenExpression
+    | | | |-(
+    | | | |-IntegerLiteralExpression
+    | | | | `-1
+    | | | `-)
+    | | `-)
+    | `-;
+    |-ExpressionStatement
+    | |-ParenExpression
+    | | |-(
+    | | |-BinaryOperatorExpression
+    | | | |-IntegerLiteralExpression
+    | | | | `-1
+    | | | |-+
+    | | | `-ParenExpression
+    | | |   |-(
+    | | |   |-IntegerLiteralExpression
+    | | |   | `-2
+    | | |   `-)
+    | | `-)
+    | `-;
+    `-}
+)txt"));
+}
+
+TEST_P(SyntaxTreeTest, UserDefinedLiteral) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
+  EXPECT_TRUE(treeDumpEqual(
+      R"cpp(
+typedef decltype(sizeof(void *)) size_t;
+
+unsigned operator "" _i(unsigned long long);
+unsigned operator "" _f(long double);
+unsigned operator "" _c(char);
+unsigned operator "" _s(const char*, size_t);
+unsigned operator "" _r(const char*);
+template <char...>
+unsigned operator "" _t();
+
+void test() {
+  12_i;   // call: operator "" _i(12uLL)      | kind: integer
+  1.2_f;  // call: operator "" _f(1.2L)       | kind: float
+  '2'_c;  // call: operator "" _c('2')        | kind: char
+  "12"_s; // call: operator "" _s("12")       | kind: string
+
+  12_r;   // call: operator "" _r("12")       | kind: integer
+  1.2_r;  // call: operator "" _i("1.2")      | kind: float
+  12_t;   // call: operator<'1', '2'> "" _x() | kind: integer
+  1.2_t;  // call: operator<'1', '2'> "" _x() | kind: float
+}
+    )cpp",
+      R"txt(
+*: TranslationUnit
+|-SimpleDeclaration
+| |-typedef
+| |-decltype
+| |-(
+| |-UnknownExpression
+| | |-sizeof
+| | |-(
+| | |-void
+| | |-*
+| | `-)
+| |-)
+| |-SimpleDeclarator
+| | `-size_t
+| `-;
+|-SimpleDeclaration
+| |-unsigned
+| |-SimpleDeclarator
+| | |-operator
+| | |-""
+| | |-_i
+| | `-ParametersAndQualifiers
+| |   |-(
+| |   |-SimpleDeclaration
+| |   | |-unsigned
+| |   | |-long
+| |   | `-long
+| |   `-)
+| `-;
+|-SimpleDeclaration
+| |-unsigned
+| |-SimpleDeclarator
+| | |-operator
+| | |-""
+| | |-_f
+| | `-ParametersAndQualifiers
+| |   |-(
+| |   |-SimpleDeclaration
+| |   | |-long
+| |   | `-double
+| |   `-)
+| `-;
+|-SimpleDeclaration
+| |-unsigned
+| |-SimpleDeclarator
+| | |-operator
+| | |-""
+| | |-_c
+| | `-ParametersAndQualifiers
+| |   |-(
+| |   |-SimpleDeclaration
+| |   | `-char
+| |   `-)
+| `-;
+|-SimpleDeclaration
+| |-unsigned
+| |-SimpleDeclarator
+| | |-operator
+| | |-""
+| | |-_s
+| | `-ParametersAndQualifiers
+| |   |-(
+| |   |-SimpleDeclaration
+| |   | |-const
+| |   | |-char
+| |   | `-SimpleDeclarator
+| |   |   `-*
+| |   |-,
+| |   |-SimpleDeclaration
+| |   | `-size_t
+| |   `-)
+| `-;
+|-SimpleDeclaration
+| |-unsigned
+| |-SimpleDeclarator
+| | |-operator
+| | |-""
+| | |-_r
+| | `-ParametersAndQualifiers
+| |   |-(
+| |   |-SimpleDeclaration
+| |   | |-const
+| |   | |-char
+| |   | `-SimpleDeclarator
+| |   |   `-*
+| |   `-)
+| `-;
+|-TemplateDeclaration
+| |-template
+| |-<
+| |-SimpleDeclaration
+| | `-char
+| |-...
+| |->
+| `-SimpleDeclaration
+|   |-unsigned
+|   |-SimpleDeclarator
+|   | |-operator
+|   | |-""
+|   | |-_t
+|   | `-ParametersAndQualifiers
+|   |   |-(
+|   |   `-)
+|   `-;
+`-SimpleDeclaration
+  |-void
+  |-SimpleDeclarator
+  | |-test
+  | `-ParametersAndQualifiers
+  |   |-(
+  |   `-)
+  `-CompoundStatement
+    |-{
+    |-ExpressionStatement
+    | |-IntegerUserDefinedLiteralExpression
+    | | `-12_i
+    | `-;
+    |-ExpressionStatement
+    | |-FloatUserDefinedLiteralExpression
+    | | `-1.2_f
+    | `-;
+    |-ExpressionStatement
+    | |-CharUserDefinedLiteralExpression
+    | | `-'2'_c
+    | `-;
+    |-ExpressionStatement
+    | |-StringUserDefinedLiteralExpression
+    | | `-"12"_s
+    | `-;
+    |-ExpressionStatement
+    | |-IntegerUserDefinedLiteralExpression
+    | | `-12_r
+    | `-;
+    |-ExpressionStatement
+    | |-FloatUserDefinedLiteralExpression
+    | | `-1.2_r
+    | `-;
+    |-ExpressionStatement
+    | |-IntegerUserDefinedLiteralExpression
+    | | `-12_t
+    | `-;
+    |-ExpressionStatement
+    | |-FloatUserDefinedLiteralExpression
+    | | `-1.2_t
+    | `-;
+    `-}
+)txt"));
+}
+
+TEST_P(SyntaxTreeTest, IntegerLiteralLongLong) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
+  EXPECT_TRUE(treeDumpEqual(
+      R"cpp(
+void test() {
+  12ll;
+  12ull;
+}
+)cpp",
+      R"txt(
+*: TranslationUnit
+`-SimpleDeclaration
+  |-void
+  |-SimpleDeclarator
+  | |-test
+  | `-ParametersAndQualifiers
+  |   |-(
+  |   `-)
+  `-CompoundStatement
+    |-{
+    |-ExpressionStatement
+    | |-IntegerLiteralExpression
+    | | `-12ll
+    | `-;
+    |-ExpressionStatement
+    | |-IntegerLiteralExpression
+    | | `-12ull
+    | `-;
+    `-}
+)txt"));
+}
+
+TEST_P(SyntaxTreeTest, IntegerLiteralBinary) {
+  if (!GetParam().isCXX14OrLater()) {
+    return;
+  }
+  EXPECT_TRUE(treeDumpEqual(
+      R"cpp(
+void test() {
+  0b1100;
+}
+)cpp",
+      R"txt(
+*: TranslationUnit
+`-SimpleDeclaration
+  |-void
+  |-SimpleDeclarator
+  | |-test
+  | `-ParametersAndQualifiers
+  |   |-(
+  |   `-)
+  `-CompoundStatement
+    |-{
+    |-ExpressionStatement
+    | |-IntegerLiteralExpression
+    | | `-0b1100
+    | `-;
+    `-}
+)txt"));
+}
+
+TEST_P(SyntaxTreeTest, IntegerLiteralWithDigitSeparators) {
+  if (!GetParam().isCXX14OrLater()) {
+    return;
+  }
+  EXPECT_TRUE(treeDumpEqual(
+      R"cpp(
+void test() {
+  1'2'0ull;
+}
+)cpp",
+      R"txt(
+*: TranslationUnit
+`-SimpleDeclaration
+  |-void
+  |-SimpleDeclarator
+  | |-test
+  | `-ParametersAndQualifiers
+  |   |-(
+  |   `-)
+  `-CompoundStatement
+    |-{
+    |-ExpressionStatement
+    | |-IntegerLiteralExpression
+    | | `-1'2'0ull
+    | `-;
+    `-}
+)txt"));
+}
+
+TEST_P(SyntaxTreeTest, CharacterLiteral) {
+  EXPECT_TRUE(treeDumpEqual(
+      R"cpp(
+void test() {
+  'a';
+  '\n';
+  '\x20';
+  '\0';
+  L'a';
+  L'α';
+}
+)cpp",
+      R"txt(
+*: TranslationUnit
+`-SimpleDeclaration
+  |-void
+  |-SimpleDeclarator
+  | |-test
+  | `-ParametersAndQualifiers
+  |   |-(
+  |   `-)
+  `-CompoundStatement
+    |-{
+    |-ExpressionStatement
+    | |-CharacterLiteralExpression
+    | | `-'a'
+    | `-;
+    |-ExpressionStatement
+    | |-CharacterLiteralExpression
+    | | `-'\n'
+    | `-;
+    |-ExpressionStatement
+    | |-CharacterLiteralExpression
+    | | `-'\x20'
+    | `-;
+    |-ExpressionStatement
+    | |-CharacterLiteralExpression
+    | | `-'\0'
+    | `-;
+    |-ExpressionStatement
+    | |-CharacterLiteralExpression
+    | | `-L'a'
+    | `-;
+    |-ExpressionStatement
+    | |-CharacterLiteralExpression
+    | | `-L'α'
+    | `-;
+    `-}
+)txt"));
+}
+
+TEST_P(SyntaxTreeTest, CharacterLiteralUtf) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
+  EXPECT_TRUE(treeDumpEqual(
+      R"cpp(
+void test() {
+  u'a';
+  u'構';
+  U'a';
+  U'🌲';
+}
+)cpp",
+      R"txt(
+*: TranslationUnit
+`-SimpleDeclaration
+  |-void
+  |-SimpleDeclarator
+  | |-test
+  | `-ParametersAndQualifiers
+  |   |-(
+  |   `-)
+  `-CompoundStatement
+    |-{
+    |-ExpressionStatement
+    | |-CharacterLiteralExpression
+    | | `-u'a'
+    | `-;
+    |-ExpressionStatement
+    | |-CharacterLiteralExpression
+    | | `-u'構'
+    | `-;
+    |-ExpressionStatement
+    | |-CharacterLiteralExpression
+    | | `-U'a'
+    | `-;
+    |-ExpressionStatement
+    | |-CharacterLiteralExpression
+    | | `-U'🌲'
+    | `-;
+    `-}
+)txt"));
+}
+
+TEST_P(SyntaxTreeTest, CharacterLiteralUtf8) {
+  if (!GetParam().isCXX17OrLater()) {
+    return;
+  }
+  EXPECT_TRUE(treeDumpEqual(
+      R"cpp(
+void test() {
+  u8'a';
+  u8'\x7f';
+}
+)cpp",
+      R"txt(
+*: TranslationUnit
+`-SimpleDeclaration
+  |-void
+  |-SimpleDeclarator
+  | |-test
+  | `-ParametersAndQualifiers
+  |   |-(
+  |   `-)
+  `-CompoundStatement
+    |-{
+    |-ExpressionStatement
+    | |-CharacterLiteralExpression
+    | | `-u8'a'
+    | `-;
+    |-ExpressionStatement
+    | |-CharacterLiteralExpression
+    | | `-u8'\x7f'
+    | `-;
+    `-}
+)txt"));
+}
+
+TEST_P(SyntaxTreeTest, FloatingLiteral) {
+  EXPECT_TRUE(treeDumpEqual(
+      R"cpp(
+void test() {
+  1e-2;
+  2.;
+  .2;
+  2.f;
+}
+)cpp",
+      R"txt(
+*: TranslationUnit
+`-SimpleDeclaration
+  |-void
+  |-SimpleDeclarator
+  | |-test
+  | `-ParametersAndQualifiers
+  |   |-(
+  |   `-)
+  `-CompoundStatement
+    |-{
+    |-ExpressionStatement
+    | |-FloatingLiteralExpression
+    | | `-1e-2
+    | `-;
+    |-ExpressionStatement
+    | |-FloatingLiteralExpression
+    | | `-2.
+    | `-;
+    |-ExpressionStatement
+    | |-FloatingLiteralExpression
+    | | `-.2
+    | `-;
+    |-ExpressionStatement
+    | |-FloatingLiteralExpression
+    | | `-2.f
+    | `-;
+    `-}
+)txt"));
+}
+
+TEST_P(SyntaxTreeTest, FloatingLiteralHexadecimal) {
+  if (!GetParam().isCXX17OrLater()) {
     return;
   }
   EXPECT_TRUE(treeDumpEqual(
       R"cpp(
 void test() {
-  nullptr;
+  0xfp1;
+  0xf.p1;
+  0x.fp1;
+  0xf.fp1f;
 }
 )cpp",
       R"txt(
@@ -1221,23 +1649,31 @@ void test() {
   `-CompoundStatement
     |-{
     |-ExpressionStatement
-    | |-CxxNullPtrExpression
-    | | `-nullptr
+    | |-FloatingLiteralExpression
+    | | `-0xfp1
+    | `-;
+    |-ExpressionStatement
+    | |-FloatingLiteralExpression
+    | | `-0xf.p1
+    | `-;
+    |-ExpressionStatement
+    | |-FloatingLiteralExpression
+    | | `-0x.fp1
+    | `-;
+    |-ExpressionStatement
+    | |-FloatingLiteralExpression
+    | | `-0xf.fp1f
     | `-;
     `-}
 )txt"));
 }
 
-TEST_P(SyntaxTreeTest, IntegerLiteral) {
+TEST_P(SyntaxTreeTest, StringLiteral) {
   EXPECT_TRUE(treeDumpEqual(
       R"cpp(
 void test() {
-  12;
-  12u;
-  12l;
-  12ul;
-  014;
-  0XC;
+  "a\n\0\x20";
+  L"αβ";
 }
 )cpp",
       R"txt(
@@ -1252,42 +1688,27 @@ void test() {
   `-CompoundStatement
     |-{
     |-ExpressionStatement
-    | |-IntegerLiteralExpression
-    | | `-12
-    | `-;
-    |-ExpressionStatement
-    | |-IntegerLiteralExpression
-    | | `-12u
-    | `-;
-    |-ExpressionStatement
-    | |-IntegerLiteralExpression
-    | | `-12l
-    | `-;
-    |-ExpressionStatement
-    | |-IntegerLiteralExpression
-    | | `-12ul
-    | `-;
-    |-ExpressionStatement
-    | |-IntegerLiteralExpression
-    | | `-014
+    | |-StringLiteralExpression
+    | | `-"a\n\0\x20"
     | `-;
     |-ExpressionStatement
-    | |-IntegerLiteralExpression
-    | | `-0XC
+    | |-StringLiteralExpression
+    | | `-L"αβ"
     | `-;
     `-}
 )txt"));
 }
 
-TEST_P(SyntaxTreeTest, IntegerLiteralLongLong) {
+TEST_P(SyntaxTreeTest, StringLiteralUtf) {
   if (!GetParam().isCXX11OrLater()) {
     return;
   }
   EXPECT_TRUE(treeDumpEqual(
       R"cpp(
 void test() {
-  12ll;
-  12ull;
+  u8"a\x1f\x05";
+  u"C++抽象構文木";
+  U"📖🌲\n";
 }
 )cpp",
       R"txt(
@@ -1302,25 +1723,63 @@ void test() {
   `-CompoundStatement
     |-{
     |-ExpressionStatement
-    | |-IntegerLiteralExpression
-    | | `-12ll
+    | |-StringLiteralExpression
+    | | `-u8"a\x1f\x05"
     | `-;
     |-ExpressionStatement
-    | |-IntegerLiteralExpression
-    | | `-12ull
+    | |-StringLiteralExpression
+    | | `-u"C++抽象構文木"
+    | `-;
+    |-ExpressionStatement
+    | |-StringLiteralExpression
+    | | `-U"📖🌲\n"
     | `-;
     `-}
 )txt"));
 }
 
-TEST_P(SyntaxTreeTest, IntegerLiteralBinary) {
-  if (!GetParam().isCXX14OrLater()) {
+TEST_P(SyntaxTreeTest, StringLiteralRaw) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
+  // This test uses regular string literals instead of raw string literals to
+  // hold source code and expected output because of a bug in MSVC up to MSVC
+  // 2019 16.2:
+  // https://developercommunity.visualstudio.com/content/problem/67300/stringifying-raw-string-literal.html
+  EXPECT_TRUE(treeDumpEqual( //
+      "void test() {\n"
+      "  R\"SyntaxTree(\n"
+      "  Hello \"Syntax\" \\\"\n"
+      "  )SyntaxTree\";\n"
+      "}\n",
+      "*: TranslationUnit\n"
+      "`-SimpleDeclaration\n"
+      "  |-void\n"
+      "  |-SimpleDeclarator\n"
+      "  | |-test\n"
+      "  | `-ParametersAndQualifiers\n"
+      "  |   |-(\n"
+      "  |   `-)\n"
+      "  `-CompoundStatement\n"
+      "    |-{\n"
+      "    |-ExpressionStatement\n"
+      "    | |-StringLiteralExpression\n"
+      "    | | `-R\"SyntaxTree(\n"
+      "  Hello \"Syntax\" \\\"\n"
+      "  )SyntaxTree\"\n"
+      "    | `-;\n"
+      "    `-}\n"));
+}
+
+TEST_P(SyntaxTreeTest, BoolLiteral) {
+  if (GetParam().isC()) {
     return;
   }
   EXPECT_TRUE(treeDumpEqual(
       R"cpp(
 void test() {
-  0b1100;
+  true;
+  false;
 }
 )cpp",
       R"txt(
@@ -1335,21 +1794,25 @@ void test() {
   `-CompoundStatement
     |-{
     |-ExpressionStatement
-    | |-IntegerLiteralExpression
-    | | `-0b1100
+    | |-BoolLiteralExpression
+    | | `-true
+    | `-;
+    |-ExpressionStatement
+    | |-BoolLiteralExpression
+    | | `-false
     | `-;
     `-}
 )txt"));
 }
 
-TEST_P(SyntaxTreeTest, IntegerLiteralWithDigitSeparators) {
-  if (!GetParam().isCXX14OrLater()) {
+TEST_P(SyntaxTreeTest, CxxNullPtrLiteral) {
+  if (!GetParam().isCXX11OrLater()) {
     return;
   }
   EXPECT_TRUE(treeDumpEqual(
       R"cpp(
 void test() {
-  1'2'0ull;
+  nullptr;
 }
 )cpp",
       R"txt(
@@ -1364,8 +1827,8 @@ void test() {
   `-CompoundStatement
     |-{
     |-ExpressionStatement
-    | |-IntegerLiteralExpression
-    | | `-1'2'0ull
+    | |-CxxNullPtrExpression
+    | | `-nullptr
     | `-;
     `-}
 )txt"));
@@ -1691,18 +2154,18 @@ void test(int a) {
     |-{
     |-ExpressionStatement
     | |-BinaryOperatorExpression
-    | | |-UnknownExpression
+    | | |-BoolLiteralExpression
     | | | `-true
     | | |-||
-    | | `-UnknownExpression
+    | | `-BoolLiteralExpression
     | |   `-false
     | `-;
     |-ExpressionStatement
     | |-BinaryOperatorExpression
-    | | |-UnknownExpression
+    | | |-BoolLiteralExpression
     | | | `-true
     | | |-or
-    | | `-UnknownExpression
+    | | `-BoolLiteralExpression
     | |   `-false
     | `-;
     |-ExpressionStatement
@@ -1759,7 +2222,7 @@ void test(int a, int b) {
     |-{
     |-ExpressionStatement
     | |-BinaryOperatorExpression
-    | | |-UnknownExpression
+    | | |-ParenExpression
     | | | |-(
     | | | |-BinaryOperatorExpression
     | | | | |-IntegerLiteralExpression
@@ -1769,7 +2232,7 @@ void test(int a, int b) {
     | | | |   `-2
     | | | `-)
     | | |-*
-    | | `-UnknownExpression
+    | | `-ParenExpression
     | |   |-(
     | |   |-BinaryOperatorExpression
     | |   | |-IntegerLiteralExpression
@@ -1857,11 +2320,18 @@ struct X {
   X& operator=(const X&);
   friend X operator+(X, const X&);
   friend bool operator<(const X&, const X&);
+  friend X operator<<(X&, const X&);
+  X operator,(X&);
+  // TODO: Fix crash on member function pointer and add a test for `->*`
+  // TODO: Unbox operators in syntax tree. 
+  // Represent operators by `+` instead of `IdExpression-UnqualifiedId-+`
 };
 void test(X x, X y) {
   x = y;
   x + y;
   x < y;
+  x << y;
+  x, y;
 }
 )cpp",
       R"txt(
@@ -1926,6 +2396,40 @@ void test(X x, X y) {
 | |   |   |   `-&
 | |   |   `-)
 | |   `-;
+| |-UnknownDeclaration
+| | `-SimpleDeclaration
+| |   |-friend
+| |   |-X
+| |   |-SimpleDeclarator
+| |   | |-operator
+| |   | |-<<
+| |   | `-ParametersAndQualifiers
+| |   |   |-(
+| |   |   |-SimpleDeclaration
+| |   |   | |-X
+| |   |   | `-SimpleDeclarator
+| |   |   |   `-&
+| |   |   |-,
+| |   |   |-SimpleDeclaration
+| |   |   | |-const
+| |   |   | |-X
+| |   |   | `-SimpleDeclarator
+| |   |   |   `-&
+| |   |   `-)
+| |   `-;
+| |-SimpleDeclaration
+| | |-X
+| | |-SimpleDeclarator
+| | | |-operator
+| | | |-,
+| | | `-ParametersAndQualifiers
+| | |   |-(
+| | |   |-SimpleDeclaration
+| | |   | |-X
+| | |   | `-SimpleDeclarator
+| | |   |   `-&
+| | |   `-)
+| | `-;
 | |-}
 | `-;
 `-SimpleDeclaration
@@ -1983,6 +2487,185 @@ void test(X x, X y) {
     | |   `-UnqualifiedId
     | |     `-y
     | `-;
+    |-ExpressionStatement
+    | |-BinaryOperatorExpression
+    | | |-IdExpression
+    | | | `-UnqualifiedId
+    | | |   `-x
+    | | |-IdExpression
+    | | | `-UnqualifiedId
+    | | |   `-<<
+    | | `-IdExpression
+    | |   `-UnqualifiedId
+    | |     `-y
+    | `-;
+    |-ExpressionStatement
+    | |-BinaryOperatorExpression
+    | | |-IdExpression
+    | | | `-UnqualifiedId
+    | | |   `-x
+    | | |-IdExpression
+    | | | `-UnqualifiedId
+    | | |   `-,
+    | | `-IdExpression
+    | |   `-UnqualifiedId
+    | |     `-y
+    | `-;
+    `-}
+)txt"));
+}
+
+TEST_P(SyntaxTreeTest, UserDefinedUnaryPrefixOperator) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(treeDumpEqual(
+      R"cpp(
+struct X {
+  X operator++();
+  bool operator!();
+  X* operator&();
+};
+void test(X x) {
+  ++x;
+  !x;
+  &x;
+}
+)cpp",
+      R"txt(
+*: TranslationUnit
+|-SimpleDeclaration
+| |-struct
+| |-X
+| |-{
+| |-SimpleDeclaration
+| | |-X
+| | |-SimpleDeclarator
+| | | |-operator
+| | | |-++
+| | | `-ParametersAndQualifiers
+| | |   |-(
+| | |   `-)
+| | `-;
+| |-SimpleDeclaration
+| | |-bool
+| | |-SimpleDeclarator
+| | | |-operator
+| | | |-!
+| | | `-ParametersAndQualifiers
+| | |   |-(
+| | |   `-)
+| | `-;
+| |-SimpleDeclaration
+| | |-X
+| | |-SimpleDeclarator
+| | | |-*
+| | | |-operator
+| | | |-&
+| | | `-ParametersAndQualifiers
+| | |   |-(
+| | |   `-)
+| | `-;
+| |-}
+| `-;
+`-SimpleDeclaration
+  |-void
+  |-SimpleDeclarator
+  | |-test
+  | `-ParametersAndQualifiers
+  |   |-(
+  |   |-SimpleDeclaration
+  |   | |-X
+  |   | `-SimpleDeclarator
+  |   |   `-x
+  |   `-)
+  `-CompoundStatement
+    |-{
+    |-ExpressionStatement
+    | |-PrefixUnaryOperatorExpression
+    | | |-IdExpression
+    | | | `-UnqualifiedId
+    | | |   `-++
+    | | `-IdExpression
+    | |   `-UnqualifiedId
+    | |     `-x
+    | `-;
+    |-ExpressionStatement
+    | |-PrefixUnaryOperatorExpression
+    | | |-IdExpression
+    | | | `-UnqualifiedId
+    | | |   `-!
+    | | `-IdExpression
+    | |   `-UnqualifiedId
+    | |     `-x
+    | `-;
+    |-ExpressionStatement
+    | |-PrefixUnaryOperatorExpression
+    | | |-IdExpression
+    | | | `-UnqualifiedId
+    | | |   `-&
+    | | `-IdExpression
+    | |   `-UnqualifiedId
+    | |     `-x
+    | `-;
+    `-}
+)txt"));
+}
+
+TEST_P(SyntaxTreeTest, UserDefinedUnaryPostfixOperator) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(treeDumpEqual(
+      R"cpp(
+struct X {
+  X operator++(int);
+};
+void test(X x) {
+  x++;
+}
+)cpp",
+      R"txt(
+*: TranslationUnit
+|-SimpleDeclaration
+| |-struct
+| |-X
+| |-{
+| |-SimpleDeclaration
+| | |-X
+| | |-SimpleDeclarator
+| | | |-operator
+| | | |-++
+| | | `-ParametersAndQualifiers
+| | |   |-(
+| | |   |-SimpleDeclaration
+| | |   | `-int
+| | |   `-)
+| | `-;
+| |-}
+| `-;
+`-SimpleDeclaration
+  |-void
+  |-SimpleDeclarator
+  | |-test
+  | `-ParametersAndQualifiers
+  |   |-(
+  |   |-SimpleDeclaration
+  |   | |-X
+  |   | `-SimpleDeclarator
+  |   |   `-x
+  |   `-)
+  `-CompoundStatement
+    |-{
+    |-ExpressionStatement
+    | |-PostfixUnaryOperatorExpression
+    | | |-IdExpression
+    | | | `-UnqualifiedId
+    | | |   `-x
+    | | `-IdExpression
+    | |   `-UnqualifiedId
+    | |     `-++
+    | `-;
     `-}
 )txt"));
 }
@@ -2556,17 +3239,17 @@ static_assert(true);
 |-StaticAssertDeclaration
 | |-static_assert
 | |-(
-| |-UnknownExpression
+| |-BoolLiteralExpression
 | | `-true
 | |-,
-| |-UnknownExpression
+| |-StringLiteralExpression
 | | `-"message"
 | |-)
 | `-;
 `-StaticAssertDeclaration
   |-static_assert
   |-(
-  |-UnknownExpression
+  |-BoolLiteralExpression
   | `-true
   |-)
   `-;
@@ -3186,7 +3869,7 @@ int b() noexcept(true);
   |   |-)
   |   |-noexcept
   |   |-(
-  |   |-UnknownExpression
+  |   |-BoolLiteralExpression
   |   | `-true
   |   `-)
   `-;
@@ -3536,7 +4219,24 @@ TEST_P(SyntaxTreeTest, SynthesizedNodes) {
   EXPECT_TRUE(S->isDetached());
 }
 
+static std::vector<TestClangConfig> allTestClangConfigs() {
+  std::vector<TestClangConfig> all_configs;
+  for (TestLanguage lang : {Lang_C89, Lang_C99, Lang_CXX03, Lang_CXX11,
+                            Lang_CXX14, Lang_CXX17, Lang_CXX20}) {
+    TestClangConfig config;
+    config.Language = lang;
+    config.Target = "x86_64-pc-linux-gnu";
+    all_configs.push_back(config);
+
+    // Windows target is interesting to test because it enables
+    // `-fdelayed-template-parsing`.
+    config.Target = "x86_64-pc-win32-msvc";
+    all_configs.push_back(config);
+  }
+  return all_configs;
+}
+
 INSTANTIATE_TEST_CASE_P(SyntaxTreeTests, SyntaxTreeTest,
-                        testing::ValuesIn(TestClangConfig::allConfigs()), );
+                        testing::ValuesIn(allTestClangConfigs()), );
 
 } // namespace
diff --git a/clang/unittests/Tooling/TransformerTest.cpp b/clang/unittests/Tooling/TransformerTest.cpp
index d19f747a69b57..7d6b632937487 100644
--- a/clang/unittests/Tooling/TransformerTest.cpp
+++ b/clang/unittests/Tooling/TransformerTest.cpp
@@ -439,6 +439,29 @@ TEST_F(TransformerTest, RemoveEdit) {
       Input, Expected);
 }
 
+TEST_F(TransformerTest, WithMetadata) {
+  std::string Input = R"cc(
+    int f() {
+      int x = 5;
+      return 7;
+    }
+  )cc";
+
+  Transformer T(
+      makeRule(declStmt().bind("decl"),
+               withMetadata(remove(statement(std::string("decl"))), 17)),
+      consumer());
+  T.registerMatchers(&MatchFinder);
+  auto Factory = newFrontendActionFactory(&MatchFinder);
+  EXPECT_TRUE(runToolOnCodeWithArgs(
+      Factory->create(), Input, std::vector<std::string>(), "input.cc",
+      "clang-tool", std::make_shared<PCHContainerOperations>(), {}));
+  ASSERT_EQ(Changes.size(), 1u);
+  const llvm::Any &Metadata = Changes[0].getMetadata();
+  ASSERT_TRUE(llvm::any_isa<int>(Metadata));
+  EXPECT_THAT(llvm::any_cast<int>(Metadata), 17);
+}
+
 TEST_F(TransformerTest, MultiChange) {
   std::string Input = R"cc(
     void foo() {
diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp
index 7494f05c85fdc..d5bf59ef04adf 100644
--- a/clang/utils/TableGen/NeonEmitter.cpp
+++ b/clang/utils/TableGen/NeonEmitter.cpp
@@ -1062,7 +1062,8 @@ std::string Intrinsic::mangleName(std::string Name, ClassKind LocalCK) const {
   std::string S = Name;
 
   if (Name == "vcvt_f16_f32" || Name == "vcvt_f32_f16" ||
-      Name == "vcvt_f32_f64" || Name == "vcvt_f64_f32")
+      Name == "vcvt_f32_f64" || Name == "vcvt_f64_f32" ||
+      Name == "vcvt_f32_bf16")
     return Name;
 
   if (!typeCode.empty()) {
@@ -2311,9 +2312,14 @@ void NeonEmitter::run(raw_ostream &OS) {
   OS << "#ifndef __ARM_NEON_H\n";
   OS << "#define __ARM_NEON_H\n\n";
 
+  OS << "#ifndef __ARM_FP\n";
+  OS << "#error \"NEON intrinsics not available with the soft-float ABI. "
+        "Please use -mfloat-abi=softfp or -mfloat-abi=hard\"\n";
+  OS << "#else\n\n";
+
   OS << "#if !defined(__ARM_NEON)\n";
   OS << "#error \"NEON support not enabled\"\n";
-  OS << "#endif\n\n";
+  OS << "#else\n\n";
 
   OS << "#include <stdint.h>\n\n";
 
@@ -2403,6 +2409,8 @@ void NeonEmitter::run(raw_ostream &OS) {
 
   OS << "\n";
   OS << "#undef __ai\n\n";
+  OS << "#endif /* if !defined(__ARM_NEON) */\n";
+  OS << "#endif /* ifndef __ARM_FP */\n";
   OS << "#endif /* __ARM_NEON_H */\n";
 }
 
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index c8e16a9283ffe..1d42edd8a94a8 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -92,8 +92,8 @@ class SVEType {
   bool isChar() const { return ElementBitwidth == 8; }
   bool isVoid() const { return Void & !Pointer; }
   bool isDefault() const { return DefaultType; }
-  bool isFloat() const { return Float; }
-  bool isBFloat() const { return BFloat; }
+  bool isFloat() const { return Float && !BFloat; }
+  bool isBFloat() const { return BFloat && !Float; }
   bool isFloatingPoint() const { return Float || BFloat; }
   bool isInteger() const { return !isFloatingPoint() && !Predicate; }
   bool isScalarPredicate() const {
@@ -248,13 +248,13 @@ class SVEEmitter {
     const char *Type;
     const char *BuiltinType;
   };
-  SmallVector<ReinterpretTypeInfo, 11> Reinterprets = {
+  SmallVector<ReinterpretTypeInfo, 12> Reinterprets = {
       {"s8", "svint8_t", "q16Sc"},   {"s16", "svint16_t", "q8Ss"},
       {"s32", "svint32_t", "q4Si"},  {"s64", "svint64_t", "q2SWi"},
       {"u8", "svuint8_t", "q16Uc"},  {"u16", "svuint16_t", "q8Us"},
       {"u32", "svuint32_t", "q4Ui"}, {"u64", "svuint64_t", "q2UWi"},
-      {"f16", "svfloat16_t", "q8h"}, {"f32", "svfloat32_t", "q4f"},
-      {"f64", "svfloat64_t", "q2d"}};
+      {"f16", "svfloat16_t", "q8h"}, {"bf16", "svbfloat16_t", "q8y"},
+      {"f32", "svfloat32_t", "q4f"}, {"f64", "svfloat64_t", "q2d"}};
 
   RecordKeeper &Records;
   llvm::StringMap<uint64_t> EltTypes;
@@ -491,6 +491,7 @@ void SVEType::applyTypespec() {
       break;
     case 'b':
       BFloat = true;
+      Float = false;
       ElementBitwidth = 16;
       break;
     default:
@@ -538,6 +539,7 @@ void SVEType::applyModifier(char Mod) {
   case 'b':
     Signed = false;
     Float = false;
+    BFloat = false;
     ElementBitwidth /= 4;
     break;
   case 'o':
@@ -567,18 +569,21 @@ void SVEType::applyModifier(char Mod) {
   case '@':
     Signed = false;
     Float = false;
+    BFloat = false;
     ElementBitwidth /= 4;
     NumVectors = 0;
     break;
   case 'K':
     Signed = true;
     Float = false;
+    BFloat = false;
     Bitwidth = ElementBitwidth;
     NumVectors = 0;
     break;
   case 'L':
     Signed = false;
     Float = false;
+    BFloat = false;
     Bitwidth = ElementBitwidth;
     NumVectors = 0;
     break;
@@ -586,15 +591,18 @@ void SVEType::applyModifier(char Mod) {
     Predicate = false;
     Signed = false;
     Float = false;
+    BFloat = false;
     break;
   case 'x':
     Predicate = false;
     Signed = true;
     Float = false;
+    BFloat = false;
     break;
   case 'i':
     Predicate = false;
     Float = false;
+    BFloat = false;
     ElementBitwidth = Bitwidth = 64;
     NumVectors = 0;
     Signed = false;
@@ -603,6 +611,7 @@ void SVEType::applyModifier(char Mod) {
   case 'I':
     Predicate = false;
     Float = false;
+    BFloat = false;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
     Signed = true;
@@ -612,6 +621,7 @@ void SVEType::applyModifier(char Mod) {
   case 'J':
     Predicate = false;
     Float = false;
+    BFloat = false;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
     Signed = true;
@@ -622,6 +632,7 @@ void SVEType::applyModifier(char Mod) {
     Predicate = false;
     Signed = true;
     Float = false;
+    BFloat = false;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
     break;
@@ -629,6 +640,7 @@ void SVEType::applyModifier(char Mod) {
     Predicate = false;
     Signed = true;
     Float = false;
+    BFloat = false;
     ElementBitwidth = Bitwidth = 64;
     NumVectors = 0;
     break;
@@ -636,6 +648,7 @@ void SVEType::applyModifier(char Mod) {
     Predicate = false;
     Signed = false;
     Float = false;
+    BFloat = false;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
     break;
@@ -643,6 +656,7 @@ void SVEType::applyModifier(char Mod) {
     Predicate = false;
     Signed = false;
     Float = false;
+    BFloat = false;
     ElementBitwidth = Bitwidth = 64;
     NumVectors = 0;
     break;
@@ -661,16 +675,19 @@ void SVEType::applyModifier(char Mod) {
   case 'g':
     Signed = false;
     Float = false;
+    BFloat = false;
     ElementBitwidth = 64;
     break;
   case 't':
     Signed = true;
     Float = false;
+    BFloat = false;
     ElementBitwidth = 32;
     break;
   case 'z':
     Signed = false;
     Float = false;
+    BFloat = false;
     ElementBitwidth = 32;
     break;
   case 'O':
@@ -681,6 +698,7 @@ void SVEType::applyModifier(char Mod) {
   case 'M':
     Predicate = false;
     Float = true;
+    BFloat = false;
     ElementBitwidth = 32;
     break;
   case 'N':
@@ -1085,10 +1103,20 @@ void SVEEmitter::createHeader(raw_ostream &OS) {
   OS << "typedef __SVUint16_t svuint16_t;\n";
   OS << "typedef __SVUint32_t svuint32_t;\n";
   OS << "typedef __SVUint64_t svuint64_t;\n";
-  OS << "typedef __SVFloat16_t svfloat16_t;\n";
-  OS << "typedef __SVBFloat16_t svbfloat16_t;\n\n";
+  OS << "typedef __SVFloat16_t svfloat16_t;\n\n";
+
+  OS << "#if defined(__ARM_FEATURE_SVE_BF16) && "
+        "!defined(__ARM_FEATURE_BF16_SCALAR_ARITHMETIC)\n";
+  OS << "#error \"__ARM_FEATURE_BF16_SCALAR_ARITHMETIC must be defined when "
+        "__ARM_FEATURE_SVE_BF16 is defined\"\n";
+  OS << "#endif\n\n";
+
+  OS << "#if defined(__ARM_FEATURE_SVE_BF16)\n";
+  OS << "typedef __SVBFloat16_t svbfloat16_t;\n";
+  OS << "#endif\n\n";
 
-  OS << "#ifdef __ARM_FEATURE_BF16_SCALAR_ARITHMETIC\n";
+  OS << "#if defined(__ARM_FEATURE_BF16_SCALAR_ARITHMETIC)\n";
+  OS << "#include <arm_bf16.h>\n";
   OS << "typedef __bf16 bfloat16_t;\n";
   OS << "#endif\n\n";
 
@@ -1129,6 +1157,12 @@ void SVEEmitter::createHeader(raw_ostream &OS) {
   OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n";
   OS << "typedef __SVBool_t  svbool_t;\n\n";
 
+  OS << "#ifdef __ARM_FEATURE_SVE_BF16\n";
+  OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";
+  OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";
+  OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";
+  OS << "#endif\n";
+
   OS << "typedef enum\n";
   OS << "{\n";
   OS << "  SV_POW2 = 0,\n";
@@ -1174,6 +1208,10 @@ void SVEEmitter::createHeader(raw_ostream &OS) {
   for (auto ShortForm : { false, true } )
     for (const ReinterpretTypeInfo &From : Reinterprets)
       for (const ReinterpretTypeInfo &To : Reinterprets) {
+        const bool IsBFloat = StringRef(From.Suffix).equals("bf16") ||
+                              StringRef(To.Suffix).equals("bf16");
+        if (IsBFloat)
+          OS << "#if defined(__ARM_FEATURE_SVE_BF16)\n";
         if (ShortForm) {
           OS << "__aio " << From.Type << " svreinterpret_" << From.Suffix;
           OS << "(" << To.Type << " op) {\n";
@@ -1184,6 +1222,8 @@ void SVEEmitter::createHeader(raw_ostream &OS) {
           OS << "#define svreinterpret_" << From.Suffix << "_" << To.Suffix
              << "(...) __builtin_sve_reinterpret_" << From.Suffix << "_"
              << To.Suffix << "(__VA_ARGS__)\n";
+        if (IsBFloat)
+          OS << "#endif /* #if defined(__ARM_FEATURE_SVE_BF16) */\n";
       }
 
   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
@@ -1222,6 +1262,11 @@ void SVEEmitter::createHeader(raw_ostream &OS) {
   if (!InGuard.empty())
     OS << "#endif  //" << InGuard << "\n";
 
+  OS << "#if defined(__ARM_FEATURE_SVE_BF16)\n";
+  OS << "#define svcvtnt_bf16_x      svcvtnt_bf16_m\n";
+  OS << "#define svcvtnt_bf16_f32_x  svcvtnt_bf16_f32_m\n";
+  OS << "#endif /*__ARM_FEATURE_SVE_BF16 */\n\n";
+
   OS << "#if defined(__ARM_FEATURE_SVE2)\n";
   OS << "#define svcvtnt_f16_x      svcvtnt_f16_m\n";
   OS << "#define svcvtnt_f16_f32_x  svcvtnt_f16_f32_m\n";
diff --git a/clang/utils/analyzer/.dockerignore b/clang/utils/analyzer/.dockerignore
new file mode 100644
index 0000000000000..2322248c5da91
--- /dev/null
+++ b/clang/utils/analyzer/.dockerignore
@@ -0,0 +1 @@
+./projects
diff --git a/clang/utils/analyzer/Dockerfile b/clang/utils/analyzer/Dockerfile
new file mode 100644
index 0000000000000..21906011c7dc2
--- /dev/null
+++ b/clang/utils/analyzer/Dockerfile
@@ -0,0 +1,60 @@
+FROM ubuntu:bionic
+
+RUN apt-get update && apt-get install -y \
+    apt-transport-https \
+    ca-certificates \
+    gnupg \
+    software-properties-common \
+    wget
+
+# newer CMake is required by LLVM
+RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null
+RUN apt-add-repository -y 'deb https://apt.kitware.com/ubuntu/ bionic main'
+
+# test system dependencies
+RUN apt-get update && apt-get install -y \
+    git=1:2.17.1-1ubuntu0.7 \
+    gettext=0.19.8.1-6ubuntu0.3 \
+    python3=3.6.7-1~18.04 \
+    python3-pip=9.0.1-2.3~ubuntu1.18.04.1 \
+    cmake=3.17.3-0kitware1 \
+    ninja-build=1.8.2-1
+
+# box2d dependencies
+RUN apt-get install -y \
+    libx11-dev=2:1.6.4-3ubuntu0.2 \
+    libxrandr-dev=2:1.5.1-1 \
+    libxinerama-dev=2:1.1.3-1 \
+    libxcursor-dev=1:1.1.15-1 \
+    libxi-dev=2:1.7.9-1
+
+# symengine dependencies
+RUN apt-get install -y \
+    libgmp10=2:6.1.2+dfsg-2 \
+    libgmp-dev=2:6.1.2+dfsg-2
+
+# simbody dependencies
+RUN apt-get install -y \
+    liblapack-dev=3.7.1-4ubuntu1
+
+# drogon dependencies
+RUN apt-get install -y \
+    libjsonrpccpp-dev=0.7.0-1build2 \
+    uuid-dev=2.31.1-0.4ubuntu3.6
+
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+
+VOLUME /analyzer
+VOLUME /projects
+VOLUME /llvm-project
+VOLUME /build
+VOLUME /scripts
+
+ENV PATH="/analyzer/bin:${PATH}"
+
+ADD entrypoint.py /entrypoint.py
+
+ADD requirements.txt /requirements.txt
+RUN pip3 install -r /requirements.txt
+
+ENTRYPOINT ["python", "/entrypoint.py"]
diff --git a/clang/utils/analyzer/SATest.py b/clang/utils/analyzer/SATest.py
index f45f593d08ec6..16f1dce0c584e 100755
--- a/clang/utils/analyzer/SATest.py
+++ b/clang/utils/analyzer/SATest.py
@@ -1,17 +1,23 @@
 #!/usr/bin/env python
 
-import SATestAdd
-import SATestBuild
-import SATestUpdateDiffs
-import CmpRuns
-
-from ProjectMap import ProjectInfo, ProjectMap
-
 import argparse
 import sys
+import os
+
+from subprocess import call
+
+SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__))
+PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects")
+DEFAULT_LLVM_DIR = os.path.realpath(os.path.join(SCRIPTS_DIR,
+                                                 os.path.pardir,
+                                                 os.path.pardir,
+                                                 os.path.pardir))
 
 
 def add(parser, args):
+    import SATestAdd
+    from ProjectMap import ProjectInfo
+
     if args.source == "git" and (args.origin == "" or args.commit == ""):
         parser.error(
             "Please provide both --origin and --commit if source is 'git'")
@@ -27,6 +33,9 @@ def add(parser, args):
 
 
 def build(parser, args):
+    import SATestBuild
+    from ProjectMap import ProjectMap
+
     SATestBuild.VERBOSE = args.verbose
 
     project_map = ProjectMap()
@@ -62,6 +71,16 @@ def build(parser, args):
 
 
 def compare(parser, args):
+    import CmpRuns
+
+    choices = [CmpRuns.HistogramType.RELATIVE.value,
+               CmpRuns.HistogramType.LOG_RELATIVE.value,
+               CmpRuns.HistogramType.ABSOLUTE.value]
+
+    if args.histogram is not None and args.histogram not in choices:
+        parser.error("Incorrect histogram type, available choices are {}"
+                     .format(choices))
+
     dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old)
     dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new)
 
@@ -73,11 +92,77 @@ def compare(parser, args):
 
 
 def update(parser, args):
+    import SATestUpdateDiffs
+    from ProjectMap import ProjectMap
+
     project_map = ProjectMap()
     for project in project_map.projects:
         SATestUpdateDiffs.update_reference_results(project)
 
 
+def docker(parser, args):
+    if len(args.rest) > 0:
+        if args.rest[0] != "--":
+            parser.error("REST arguments should start with '--'")
+        args.rest = args.rest[1:]
+
+    if args.build_image:
+        docker_build_image()
+    elif args.shell:
+        docker_shell(args)
+    else:
+        sys.exit(docker_run(args, ' '.join(args.rest)))
+
+
+def docker_build_image():
+    sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR),
+                  shell=True))
+
+
+def docker_shell(args):
+    try:
+        # First we need to start the docker container in a waiting mode,
+        # so it doesn't do anything, but most importantly keeps working
+        # while the shell session is in progress.
+        docker_run(args, "--wait", "--detach")
+        # Since the docker container is running, we can actually connect to it
+        call("docker exec -it satest bash", shell=True)
+
+    except KeyboardInterrupt:
+        pass
+
+    finally:
+        docker_cleanup()
+
+
+def docker_run(args, command, docker_args=""):
+    try:
+        return call("docker run --rm --name satest "
+                    "-v {llvm}:/llvm-project "
+                    "-v {build}:/build "
+                    "-v {clang}:/analyzer "
+                    "-v {scripts}:/scripts "
+                    "-v {projects}:/projects "
+                    "{docker_args} "
+                    "satest-image:latest {command}"
+                    .format(llvm=args.llvm_project_dir,
+                            build=args.build_dir,
+                            clang=args.clang_dir,
+                            scripts=SCRIPTS_DIR,
+                            projects=PROJECTS_DIR,
+                            docker_args=docker_args,
+                            command=command),
+                    shell=True)
+
+    except KeyboardInterrupt:
+        docker_cleanup()
+
+
+def docker_cleanup():
+    print("Please wait for docker to clean up")
+    call("docker stop satest", shell=True)
+
+
 def main():
     parser = argparse.ArgumentParser()
     subparsers = parser.add_subparsers()
@@ -101,8 +186,7 @@ def main():
                             "(please provide --origin and --commit), "
                             "'zip' for unpacking source from a zip file, "
                             "'script' for downloading source by running "
-                            "a custom script {}"
-                            .format(SATestBuild.DOWNLOAD_SCRIPT))
+                            "a custom script")
     add_parser.add_argument("--origin", action="store", default="",
                             help="Origin link for a git repository")
     add_parser.add_argument("--commit", action="store", default="",
@@ -163,9 +247,6 @@ def main():
                             dest="show_stats", default=False,
                             help="Show change in statistics")
     cmp_parser.add_argument("--histogram", action="store", default=None,
-                            choices=[CmpRuns.HistogramType.RELATIVE.value,
-                                     CmpRuns.HistogramType.LOG_RELATIVE.value,
-                                     CmpRuns.HistogramType.ABSOLUTE.value],
                             help="Show histogram of paths differences. "
                             "Requires matplotlib")
     cmp_parser.add_argument("old", nargs=1, help="Directory with old results")
@@ -180,6 +261,29 @@ def main():
     # TODO: add option to decide whether we should use git
     upd_parser.set_defaults(func=update)
 
+    # docker subcommand
+    dock_parser = subparsers.add_parser(
+        "docker",
+        help="Run regression system in the docker.")
+
+    dock_parser.add_argument("--build-image", action="store_true",
+                             help="Build docker image for running tests.")
+    dock_parser.add_argument("--shell", action="store_true",
+                             help="Start a shell on docker.")
+    dock_parser.add_argument("--llvm-project-dir", action="store",
+                             default=DEFAULT_LLVM_DIR,
+                             help="Path to LLVM source code. Defaults "
+                             "to the repo where this script is located. ")
+    dock_parser.add_argument("--build-dir", action="store", default="",
+                             help="Path to a directory where docker should "
+                             "build LLVM code.")
+    dock_parser.add_argument("--clang-dir", action="store", default="",
+                             help="Path to find/install LLVM installation.")
+    dock_parser.add_argument("rest", nargs=argparse.REMAINDER, default=[],
+                             help="Additionall args that will be forwarded "
+                             "to the docker's entrypoint.")
+    dock_parser.set_defaults(func=docker)
+
     args = parser.parse_args()
     args.func(parser, args)
 
diff --git a/clang/utils/analyzer/SATestBuild.py b/clang/utils/analyzer/SATestBuild.py
index d83ff1e7d0096..eefab869f6ef9 100644
--- a/clang/utils/analyzer/SATestBuild.py
+++ b/clang/utils/analyzer/SATestBuild.py
@@ -43,7 +43,7 @@
 variable. It should contain a comma separated list.
 """
 import CmpRuns
-import SATestUtils
+import SATestUtils as utils
 from ProjectMap import DownloadType, ProjectInfo
 
 import glob
@@ -63,16 +63,34 @@
 # and this is we can shush that false positive
 from plistlib import InvalidFileException  # type:ignore
 from subprocess import CalledProcessError, check_call
-from typing import Dict, IO, List, NamedTuple, Optional, TYPE_CHECKING
+from typing import Dict, IO, List, NamedTuple, Optional, TYPE_CHECKING, Tuple
 
 
 ###############################################################################
 # Helper functions.
 ###############################################################################
 
+class StreamToLogger:
+    def __init__(self, logger: logging.Logger,
+                 log_level: int = logging.INFO):
+        self.logger = logger
+        self.log_level = log_level
+
+    def write(self, message: str):
+        # Rstrip in order not to write an extra newline.
+        self.logger.log(self.log_level, message.rstrip())
+
+    def flush(self):
+        pass
+
+    def fileno(self) -> int:
+        return 0
+
+
+Logger = logging.getLogger("main")
 LOCAL = threading.local()
-LOCAL.stdout = sys.stdout
-LOCAL.stderr = sys.stderr
+LOCAL.stdout = StreamToLogger(Logger, logging.INFO)
+LOCAL.stderr = StreamToLogger(Logger, logging.ERROR)
 
 
 def stderr(message: str):
@@ -97,7 +115,7 @@ def stdout(message: str):
 if 'CC' in os.environ:
     cc_candidate: Optional[str] = os.environ['CC']
 else:
-    cc_candidate = SATestUtils.which("clang", os.environ['PATH'])
+    cc_candidate = utils.which("clang", os.environ['PATH'])
 if not cc_candidate:
     stderr("Error: cannot find 'clang' in PATH")
     sys.exit(1)
@@ -164,23 +182,6 @@ def stdout(message: str):
 VERBOSE = 0
 
 
-class StreamToLogger:
-    def __init__(self, logger: logging.Logger,
-                 log_level: int = logging.INFO):
-        self.logger = logger
-        self.log_level = log_level
-
-    def write(self, message: str):
-        # Rstrip in order not to write an extra newline.
-        self.logger.log(self.log_level, message.rstrip())
-
-    def flush(self):
-        pass
-
-    def fileno(self) -> int:
-        return 0
-
-
 ###############################################################################
 # Test harness logic.
 ###############################################################################
@@ -193,9 +194,9 @@ def run_cleanup_script(directory: str, build_log_file: IO):
     cwd = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)
     script_path = os.path.join(directory, CLEANUP_SCRIPT)
 
-    SATestUtils.run_script(script_path, build_log_file, cwd,
-                           out=LOCAL.stdout, err=LOCAL.stderr,
-                           verbose=VERBOSE)
+    utils.run_script(script_path, build_log_file, cwd,
+                     out=LOCAL.stdout, err=LOCAL.stderr,
+                     verbose=VERBOSE)
 
 
 class TestInfo(NamedTuple):
@@ -350,8 +351,6 @@ def get_output_dir(self) -> str:
             return OUTPUT_DIR_NAME
 
     def build(self, directory: str, output_dir: str):
-        time_start = time.time()
-
         build_log_path = get_build_log_path(output_dir)
 
         stdout(f"Log file: {build_log_path}\n")
@@ -374,19 +373,23 @@ def build(self, directory: str, output_dir: str):
             if self.project.mode == 1:
                 self._download_and_patch(directory, build_log_file)
                 run_cleanup_script(directory, build_log_file)
-                self.scan_build(directory, output_dir, build_log_file)
+                build_time, memory = self.scan_build(directory, output_dir,
+                                               build_log_file)
             else:
-                self.analyze_preprocessed(directory, output_dir)
+                build_time, memory = self.analyze_preprocessed(directory,
+                                                               output_dir)
 
             if self.is_reference_build:
                 run_cleanup_script(directory, build_log_file)
                 normalize_reference_results(directory, output_dir,
                                             self.project.mode)
 
-        stdout(f"Build complete (time: {time.time() - time_start:.2f}). "
+        stdout(f"Build complete (time: {utils.time_to_str(build_time)}, "
+               f"peak memory: {utils.memory_to_str(memory)}). "
                f"See the log for more details: {build_log_path}\n")
 
-    def scan_build(self, directory: str, output_dir: str, build_log_file: IO):
+    def scan_build(self, directory: str, output_dir: str,
+                   build_log_file: IO) -> Tuple[float, int]:
         """
         Build the project with scan-build by reading in the commands and
         prefixing them with the scan-build options.
@@ -415,6 +418,10 @@ def scan_build(self, directory: str, output_dir: str, build_log_file: IO):
             options += "--override-compiler "
 
         extra_env: Dict[str, str] = {}
+
+        execution_time = 0.0
+        peak_memory = 0
+
         try:
             command_file = open(build_script_path, "r")
             command_prefix = "scan-build " + options + " "
@@ -450,11 +457,15 @@ def scan_build(self, directory: str, output_dir: str, build_log_file: IO):
                 if VERBOSE >= 1:
                     stdout(f"  Executing: {command_to_run}\n")
 
-                check_call(command_to_run, cwd=cwd,
-                           stderr=build_log_file,
-                           stdout=build_log_file,
-                           env=dict(os.environ, **extra_env),
-                           shell=True)
+                time, mem = utils.check_and_measure_call(
+                    command_to_run, cwd=cwd,
+                    stderr=build_log_file,
+                    stdout=build_log_file,
+                    env=dict(os.environ, **extra_env),
+                    shell=True)
+
+                execution_time += time
+                peak_memory = max(peak_memory, mem)
 
         except CalledProcessError:
             stderr("Error: scan-build failed. Its output was: \n")
@@ -462,7 +473,10 @@ def scan_build(self, directory: str, output_dir: str, build_log_file: IO):
             shutil.copyfileobj(build_log_file, LOCAL.stderr)
             sys.exit(1)
 
-    def analyze_preprocessed(self, directory: str, output_dir: str):
+        return execution_time, peak_memory
+
+    def analyze_preprocessed(self, directory: str,
+                             output_dir: str) -> Tuple[float, int]:
         """
         Run analysis on a set of preprocessed files.
         """
@@ -486,14 +500,17 @@ def analyze_preprocessed(self, directory: str, output_dir: str):
         fail_path = os.path.join(plist_path, "failures")
         os.makedirs(fail_path)
 
+        execution_time = 0.0
+        peak_memory = 0
+
         for full_file_name in glob.glob(directory + "/*"):
             file_name = os.path.basename(full_file_name)
             failed = False
 
             # Only run the analyzes on supported files.
-            if SATestUtils.has_no_extension(file_name):
+            if utils.has_no_extension(file_name):
                 continue
-            if not SATestUtils.is_valid_single_input_file(file_name):
+            if not utils.is_valid_single_input_file(file_name):
                 stderr(f"Error: Invalid single input file {full_file_name}.\n")
                 raise Exception()
 
@@ -508,8 +525,12 @@ def analyze_preprocessed(self, directory: str, output_dir: str):
                     if VERBOSE >= 1:
                         stdout(f"  Executing: {command}\n")
 
-                    check_call(command, cwd=directory, stderr=log_file,
-                               stdout=log_file, shell=True)
+                    time, mem = utils.check_and_measure_call(
+                        command, cwd=directory, stderr=log_file,
+                        stdout=log_file, shell=True)
+
+                    execution_time += time
+                    peak_memory = max(peak_memory, mem)
 
                 except CalledProcessError as e:
                     stderr(f"Error: Analyzes of {full_file_name} failed. "
@@ -521,6 +542,8 @@ def analyze_preprocessed(self, directory: str, output_dir: str):
                 if not failed:
                     os.remove(log_file.name)
 
+        return execution_time, peak_memory
+
     def generate_config(self) -> str:
         out = "serialize-stats=true,stable-report-filename=true"
 
@@ -570,7 +593,7 @@ def _download(self, directory: str, build_log_file: IO):
 
     def _download_from_git(self, directory: str, build_log_file: IO):
         cached_source = os.path.join(directory, CACHED_SOURCE_DIR_NAME)
-        check_call(f"git clone {self.project.origin} {cached_source}",
+        check_call(f"git clone --recursive {self.project.origin} {cached_source}",
                    cwd=directory, stderr=build_log_file,
                    stdout=build_log_file, shell=True)
         check_call(f"git checkout --quiet {self.project.commit}",
@@ -578,7 +601,7 @@ def _download_from_git(self, directory: str, build_log_file: IO):
                    stdout=build_log_file, shell=True)
 
     def _unpack_zip(self, directory: str, build_log_file: IO):
-        zip_files = list(glob.glob(os.path.join(directory, "/*.zip")))
+        zip_files = list(glob.glob(directory + "/*.zip"))
 
         if len(zip_files) == 0:
             raise ValueError(
@@ -597,9 +620,9 @@ def _unpack_zip(self, directory: str, build_log_file: IO):
     @staticmethod
     def _run_download_script(directory: str, build_log_file: IO):
         script_path = os.path.join(directory, DOWNLOAD_SCRIPT)
-        SATestUtils.run_script(script_path, build_log_file, directory,
-                               out=LOCAL.stdout, err=LOCAL.stderr,
-                               verbose=VERBOSE)
+        utils.run_script(script_path, build_log_file, directory,
+                         out=LOCAL.stdout, err=LOCAL.stderr,
+                         verbose=VERBOSE)
 
     @staticmethod
     def _apply_patch(directory: str, build_log_file: IO):
diff --git a/clang/utils/analyzer/SATestUtils.py b/clang/utils/analyzer/SATestUtils.py
index 4e126e66b869f..3947e183d82f3 100644
--- a/clang/utils/analyzer/SATestUtils.py
+++ b/clang/utils/analyzer/SATestUtils.py
@@ -1,8 +1,9 @@
 import os
 import sys
+import time
 
 from subprocess import CalledProcessError, check_call
-from typing import List, IO, Optional
+from typing import List, IO, Optional, Tuple
 
 
 def which(command: str, paths: Optional[str] = None) -> Optional[str]:
@@ -47,6 +48,87 @@ def is_valid_single_input_file(file_name: str) -> bool:
     return ext in (".i", ".ii", ".c", ".cpp", ".m", "")
 
 
+def time_to_str(time: float) -> str:
+    """
+    Convert given time in seconds into a human-readable string.
+    """
+    return f"{time:.2f}s"
+
+
+def memory_to_str(memory: int) -> str:
+    """
+    Convert given number of bytes into a human-readable string.
+    """
+    if memory:
+        try:
+            import humanize
+            return humanize.naturalsize(memory, gnu=True)
+        except ImportError:
+            # no formatter installed, let's keep it in bytes
+            return f"{memory}B"
+
+    # If memory is 0, we didn't succeed measuring it.
+    return "N/A"
+
+
+def check_and_measure_call(*popenargs, **kwargs) -> Tuple[float, int]:
+    """
+    Run command with arguments.  Wait for command to complete and measure
+    execution time and peak memory consumption.
+    If the exit code was zero then return, otherwise raise
+    CalledProcessError.  The CalledProcessError object will have the
+    return code in the returncode attribute.
+
+    The arguments are the same as for the call and check_call functions.
+
+    Return a tuple of execution time and peak memory.
+    """
+    peak_mem = 0
+    start_time = time.time()
+
+    try:
+        import psutil as ps
+
+        def get_memory(process: ps.Process) -> int:
+            mem = 0
+
+            # we want to gather memory usage from all of the child processes
+            descendants = list(process.children(recursive=True))
+            descendants.append(process)
+
+            for subprocess in descendants:
+                try:
+                    mem += subprocess.memory_info().rss
+                except (ps.NoSuchProcess, ps.AccessDenied):
+                    continue
+
+            return mem
+
+        with ps.Popen(*popenargs, **kwargs) as process:
+            # while the process is running calculate resource utilization.
+            while (process.is_running() and
+                   process.status() != ps.STATUS_ZOMBIE):
+                # track the peak utilization of the process
+                peak_mem = max(peak_mem, get_memory(process))
+                time.sleep(.5)
+
+            if process.is_running():
+                process.kill()
+
+        if process.returncode != 0:
+            cmd = kwargs.get("args")
+            if cmd is None:
+                cmd = popenargs[0]
+            raise CalledProcessError(process.returncode, cmd)
+
+    except ImportError:
+        # back off to subprocess if we don't have psutil installed
+        peak_mem = 0
+        check_call(*popenargs, **kwargs)
+
+    return time.time() - start_time, peak_mem
+
+
 def run_script(script_path: str, build_log_file: IO, cwd: str,
                out=sys.stdout, err=sys.stderr, verbose: int = 0):
     """
diff --git a/clang/utils/analyzer/entrypoint.py b/clang/utils/analyzer/entrypoint.py
new file mode 100644
index 0000000000000..b440e776b57cb
--- /dev/null
+++ b/clang/utils/analyzer/entrypoint.py
@@ -0,0 +1,72 @@
+import argparse
+import os
+import sys
+
+from subprocess import call, check_call, CalledProcessError
+from time import sleep
+from typing import List, Tuple
+
+
+def main():
+    settings, rest = parse_arguments()
+    if settings.wait:
+        wait()
+    if settings.build_llvm or settings.build_llvm_only:
+        build_llvm()
+    if settings.build_llvm_only:
+        return
+    sys.exit(test(rest))
+
+
+def wait():
+    # It is an easy on CPU way of keeping the docker container running
+    # while the user has a terminal session in that container.
+    while True:
+        sleep(3600)
+
+
+def parse_arguments() -> Tuple[argparse.Namespace, List[str]]:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--wait', action='store_true')
+    parser.add_argument('--build-llvm', action='store_true')
+    parser.add_argument('--build-llvm-only', action='store_true')
+    return parser.parse_known_args()
+
+
+def build_llvm():
+    os.chdir('/build')
+    try:
+        if is_cmake_needed():
+            cmake()
+        ninja()
+    except CalledProcessError:
+        print("Build failed!")
+        sys.exit(1)
+
+
+def is_cmake_needed():
+    return "build.ninja" not in os.listdir()
+
+
+CMAKE_COMMAND = "cmake -G Ninja -DCMAKE_BUILD_TYPE=Release " \
+    "-DCMAKE_INSTALL_PREFIX=/analyzer -DLLVM_TARGETS_TO_BUILD=X86 " \
+    "-DLLVM_ENABLE_PROJECTS=clang -DLLVM_BUILD_RUNTIME=OFF " \
+    "-DLLVM_ENABLE_TERMINFO=OFF -DCLANG_ENABLE_ARCMT=OFF " \
+    "-DCLANG_ENABLE_STATIC_ANALYZER=ON"
+
+
+def cmake():
+    check_call(CMAKE_COMMAND + ' /llvm-project/llvm', shell=True)
+
+
+def ninja():
+    check_call("ninja install", shell=True)
+
+
+def test(args: List[str]) -> int:
+    os.chdir("/projects")
+    return call("/scripts/SATest.py " + " ".join(args), shell=True)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/clang/utils/analyzer/exploded-graph-rewriter.py b/clang/utils/analyzer/exploded-graph-rewriter.py
index bae863cb6d046..54d87addd3da3 100755
--- a/clang/utils/analyzer/exploded-graph-rewriter.py
+++ b/clang/utils/analyzer/exploded-graph-rewriter.py
@@ -368,8 +368,7 @@ def add_raw_line(self, raw_line):
                 self.root_id = node_id
             # Note: when writing tests you don't need to escape everything,
             # even though in a valid dot file everything is escaped.
-            node_label = result.group(2).replace('\\l', '') \
-                                        .replace('&nbsp;', '') \
+            node_label = result.group(2).replace('&nbsp;', '') \
                                         .replace('\\"', '"') \
                                         .replace('\\{', '{') \
                                         .replace('\\}', '}') \
@@ -378,6 +377,13 @@ def add_raw_line(self, raw_line):
                                         .replace('\\<', '\\\\<') \
                                         .replace('\\>', '\\\\>') \
                                         .rstrip(',')
+            # Handle `\l` separately because a string literal can be in code
+            # like "string\\literal" with the `\l` inside.
+            # Also on Windows macros __FILE__ produces specific delimiters `\`
+            # and a directory or file may starts with the letter `l`.
+            # Find all `\l` (like `,\l`, `}\l`, `[\l`) except `\\l`,
+            # because the literal as a rule containes multiple `\` before `\l`.
+            node_label = re.sub(r'(?<!\\)\\l', '', node_label)
             logging.debug(node_label)
             json_node = json.loads(node_label)
             self.nodes[node_id].construct(node_id, json_node)
@@ -422,8 +428,8 @@ def _dump(self, s):
              .replace('}', '\\}') \
              .replace('\\<', '&lt;') \
              .replace('\\>', '&gt;') \
-             .replace('\\l', '<br />') \
              .replace('|', '\\|')
+        s = re.sub(r'(?<!\\)\\l', '<br />', s)
         if self._gray_mode:
             s = re.sub(r'<font color="[a-z0-9]*">', '', s)
             s = re.sub(r'</font>', '', s)
diff --git a/clang/utils/analyzer/projects/box2d/cleanup_run_static_analyzer.sh b/clang/utils/analyzer/projects/box2d/cleanup_run_static_analyzer.sh
new file mode 100755
index 0000000000000..e14c423280ec5
--- /dev/null
+++ b/clang/utils/analyzer/projects/box2d/cleanup_run_static_analyzer.sh
@@ -0,0 +1 @@
+rm -rf ./build
diff --git a/clang/utils/analyzer/projects/box2d/run_static_analyzer.cmd b/clang/utils/analyzer/projects/box2d/run_static_analyzer.cmd
new file mode 100644
index 0000000000000..6678fe635db32
--- /dev/null
+++ b/clang/utils/analyzer/projects/box2d/run_static_analyzer.cmd
@@ -0,0 +1,2 @@
+cmake . -DCMAKE_BUILD_TYPE=Debug -Bbuild -GNinja
+cmake --build build
diff --git a/clang/utils/analyzer/projects/cxxopts/cleanup_run_static_analyzer.sh b/clang/utils/analyzer/projects/cxxopts/cleanup_run_static_analyzer.sh
new file mode 100755
index 0000000000000..e14c423280ec5
--- /dev/null
+++ b/clang/utils/analyzer/projects/cxxopts/cleanup_run_static_analyzer.sh
@@ -0,0 +1 @@
+rm -rf ./build
diff --git a/clang/utils/analyzer/projects/cxxopts/run_static_analyzer.cmd b/clang/utils/analyzer/projects/cxxopts/run_static_analyzer.cmd
new file mode 100644
index 0000000000000..6678fe635db32
--- /dev/null
+++ b/clang/utils/analyzer/projects/cxxopts/run_static_analyzer.cmd
@@ -0,0 +1,2 @@
+cmake . -DCMAKE_BUILD_TYPE=Debug -Bbuild -GNinja
+cmake --build build
diff --git a/clang/utils/analyzer/projects/drogon/cleanup_run_static_analyzer.sh b/clang/utils/analyzer/projects/drogon/cleanup_run_static_analyzer.sh
new file mode 100755
index 0000000000000..e14c423280ec5
--- /dev/null
+++ b/clang/utils/analyzer/projects/drogon/cleanup_run_static_analyzer.sh
@@ -0,0 +1 @@
+rm -rf ./build
diff --git a/clang/utils/analyzer/projects/drogon/run_static_analyzer.cmd b/clang/utils/analyzer/projects/drogon/run_static_analyzer.cmd
new file mode 100644
index 0000000000000..6678fe635db32
--- /dev/null
+++ b/clang/utils/analyzer/projects/drogon/run_static_analyzer.cmd
@@ -0,0 +1,2 @@
+cmake . -DCMAKE_BUILD_TYPE=Debug -Bbuild -GNinja
+cmake --build build
diff --git a/clang/utils/analyzer/projects/duckdb/cleanup_run_static_analyzer.sh b/clang/utils/analyzer/projects/duckdb/cleanup_run_static_analyzer.sh
new file mode 100755
index 0000000000000..e14c423280ec5
--- /dev/null
+++ b/clang/utils/analyzer/projects/duckdb/cleanup_run_static_analyzer.sh
@@ -0,0 +1 @@
+rm -rf ./build
diff --git a/clang/utils/analyzer/projects/duckdb/run_static_analyzer.cmd b/clang/utils/analyzer/projects/duckdb/run_static_analyzer.cmd
new file mode 100644
index 0000000000000..6678fe635db32
--- /dev/null
+++ b/clang/utils/analyzer/projects/duckdb/run_static_analyzer.cmd
@@ -0,0 +1,2 @@
+cmake . -DCMAKE_BUILD_TYPE=Debug -Bbuild -GNinja
+cmake --build build
diff --git a/clang/utils/analyzer/projects/fmt/cleanup_run_static_analyzer.sh b/clang/utils/analyzer/projects/fmt/cleanup_run_static_analyzer.sh
new file mode 100755
index 0000000000000..e14c423280ec5
--- /dev/null
+++ b/clang/utils/analyzer/projects/fmt/cleanup_run_static_analyzer.sh
@@ -0,0 +1 @@
+rm -rf ./build
diff --git a/clang/utils/analyzer/projects/fmt/run_static_analyzer.cmd b/clang/utils/analyzer/projects/fmt/run_static_analyzer.cmd
new file mode 100644
index 0000000000000..6678fe635db32
--- /dev/null
+++ b/clang/utils/analyzer/projects/fmt/run_static_analyzer.cmd
@@ -0,0 +1,2 @@
+cmake . -DCMAKE_BUILD_TYPE=Debug -Bbuild -GNinja
+cmake --build build
diff --git a/clang/utils/analyzer/projects/libsoundio/cleanup_run_static_analyzer.sh b/clang/utils/analyzer/projects/libsoundio/cleanup_run_static_analyzer.sh
new file mode 100755
index 0000000000000..e14c423280ec5
--- /dev/null
+++ b/clang/utils/analyzer/projects/libsoundio/cleanup_run_static_analyzer.sh
@@ -0,0 +1 @@
+rm -rf ./build
diff --git a/clang/utils/analyzer/projects/libsoundio/run_static_analyzer.cmd b/clang/utils/analyzer/projects/libsoundio/run_static_analyzer.cmd
new file mode 100644
index 0000000000000..6678fe635db32
--- /dev/null
+++ b/clang/utils/analyzer/projects/libsoundio/run_static_analyzer.cmd
@@ -0,0 +1,2 @@
+cmake . -DCMAKE_BUILD_TYPE=Debug -Bbuild -GNinja
+cmake --build build
diff --git a/clang/utils/analyzer/projects/oatpp/cleanup_run_static_analyzer.sh b/clang/utils/analyzer/projects/oatpp/cleanup_run_static_analyzer.sh
new file mode 100755
index 0000000000000..e14c423280ec5
--- /dev/null
+++ b/clang/utils/analyzer/projects/oatpp/cleanup_run_static_analyzer.sh
@@ -0,0 +1 @@
+rm -rf ./build
diff --git a/clang/utils/analyzer/projects/oatpp/run_static_analyzer.cmd b/clang/utils/analyzer/projects/oatpp/run_static_analyzer.cmd
new file mode 100644
index 0000000000000..6678fe635db32
--- /dev/null
+++ b/clang/utils/analyzer/projects/oatpp/run_static_analyzer.cmd
@@ -0,0 +1,2 @@
+cmake . -DCMAKE_BUILD_TYPE=Debug -Bbuild -GNinja
+cmake --build build
diff --git a/clang/utils/analyzer/projects/projects.json b/clang/utils/analyzer/projects/projects.json
new file mode 100644
index 0000000000000..e3d853ac6f6a5
--- /dev/null
+++ b/clang/utils/analyzer/projects/projects.json
@@ -0,0 +1,107 @@
+[
+  {
+    "name": "cxxopts",
+    "mode": 1,
+    "source": "git",
+    "origin": "https://github.com/jarro2783/cxxopts.git",
+    "commit": "794c975"
+  },
+  {
+    "name": "box2d",
+    "mode": 1,
+    "source": "git",
+    "origin": "https://github.com/erincatto/box2d.git",
+    "commit": "1025f9a"
+  },
+  {
+    "name": "tinyexpr",
+    "mode": 1,
+    "source": "git",
+    "origin": "https://github.com/codeplea/tinyexpr.git",
+    "commit": "ffb0d41"
+  },
+  {
+    "name": "symengine",
+    "mode": 1,
+    "source": "git",
+    "origin": "https://github.com/symengine/symengine.git",
+    "commit": "4f669d59"
+  },
+  {
+    "name": "termbox",
+    "mode": 1,
+    "source": "git",
+    "origin": "https://github.com/nsf/termbox.git",
+    "commit": "0df1355"
+  },
+  {
+    "name": "tinyvm",
+    "mode": 1,
+    "source": "git",
+    "origin": "https://github.com/jakogut/tinyvm.git",
+    "commit": "10c25d8"
+  },
+  {
+    "name": "tinyspline",
+    "mode": 1,
+    "source": "git",
+    "origin": "https://github.com/msteinbeck/tinyspline.git",
+    "commit": "f8b1ab7"
+  },
+  {
+    "name": "oatpp",
+    "mode": 1,
+    "source": "git",
+    "origin": "https://github.com/oatpp/oatpp.git",
+    "commit": "d3e60fb"
+  },
+  {
+    "name": "libsoundio",
+    "mode": 1,
+    "source": "git",
+    "origin": "https://github.com/andrewrk/libsoundio.git",
+    "commit": "b810bf2"
+  },
+  {
+    "name": "zstd",
+    "mode": 1,
+    "source": "git",
+    "origin": "https://github.com/facebook/zstd.git",
+    "commit": "2af4e073"
+  },
+  {
+    "name": "simbody",
+    "mode": 1,
+    "source": "git",
+    "origin": "https://github.com/simbody/simbody.git",
+    "commit": "5cf513d"
+  },
+  {
+    "name": "duckdb",
+    "mode": 1,
+    "source": "git",
+    "origin": "https://github.com/cwida/duckdb.git",
+    "commit": "d098c9f"
+  },
+  {
+    "name": "drogon",
+    "mode": 1,
+    "source": "git",
+    "origin": "https://github.com/an-tao/drogon.git",
+    "commit": "fd2a612"
+  },
+  {
+    "name": "fmt",
+    "mode": 1,
+    "source": "git",
+    "origin": "https://github.com/fmtlib/fmt.git",
+    "commit": "5e7c70e"
+  },
+  {
+    "name": "re2",
+    "mode": 1,
+    "source": "git",
+    "origin": "https://github.com/google/re2.git",
+    "commit": "2b25567"
+  }
+]
diff --git a/clang/utils/analyzer/projects/re2/cleanup_run_static_analyzer.sh b/clang/utils/analyzer/projects/re2/cleanup_run_static_analyzer.sh
new file mode 100755
index 0000000000000..e14c423280ec5
--- /dev/null
+++ b/clang/utils/analyzer/projects/re2/cleanup_run_static_analyzer.sh
@@ -0,0 +1 @@
+rm -rf ./build
diff --git a/clang/utils/analyzer/projects/re2/run_static_analyzer.cmd b/clang/utils/analyzer/projects/re2/run_static_analyzer.cmd
new file mode 100644
index 0000000000000..6678fe635db32
--- /dev/null
+++ b/clang/utils/analyzer/projects/re2/run_static_analyzer.cmd
@@ -0,0 +1,2 @@
+cmake . -DCMAKE_BUILD_TYPE=Debug -Bbuild -GNinja
+cmake --build build
diff --git a/clang/utils/analyzer/projects/simbody/cleanup_run_static_analyzer.sh b/clang/utils/analyzer/projects/simbody/cleanup_run_static_analyzer.sh
new file mode 100755
index 0000000000000..e14c423280ec5
--- /dev/null
+++ b/clang/utils/analyzer/projects/simbody/cleanup_run_static_analyzer.sh
@@ -0,0 +1 @@
+rm -rf ./build
diff --git a/clang/utils/analyzer/projects/simbody/run_static_analyzer.cmd b/clang/utils/analyzer/projects/simbody/run_static_analyzer.cmd
new file mode 100644
index 0000000000000..6678fe635db32
--- /dev/null
+++ b/clang/utils/analyzer/projects/simbody/run_static_analyzer.cmd
@@ -0,0 +1,2 @@
+cmake . -DCMAKE_BUILD_TYPE=Debug -Bbuild -GNinja
+cmake --build build
diff --git a/clang/utils/analyzer/projects/symengine/cleanup_run_static_analyzer.sh b/clang/utils/analyzer/projects/symengine/cleanup_run_static_analyzer.sh
new file mode 100755
index 0000000000000..e14c423280ec5
--- /dev/null
+++ b/clang/utils/analyzer/projects/symengine/cleanup_run_static_analyzer.sh
@@ -0,0 +1 @@
+rm -rf ./build
diff --git a/clang/utils/analyzer/projects/symengine/run_static_analyzer.cmd b/clang/utils/analyzer/projects/symengine/run_static_analyzer.cmd
new file mode 100644
index 0000000000000..aaf6bca468d47
--- /dev/null
+++ b/clang/utils/analyzer/projects/symengine/run_static_analyzer.cmd
@@ -0,0 +1,2 @@
+cmake . -DCMAKE_BUILD_TYPE=Debug -DWITH_COTIRE=OFF -Bbuild -GNinja
+cmake --build build
diff --git a/clang/utils/analyzer/projects/termbox/cleanup_run_static_analyzer.sh b/clang/utils/analyzer/projects/termbox/cleanup_run_static_analyzer.sh
new file mode 100755
index 0000000000000..3d0f2153c1c59
--- /dev/null
+++ b/clang/utils/analyzer/projects/termbox/cleanup_run_static_analyzer.sh
@@ -0,0 +1,2 @@
+./waf clean
+exit 0
diff --git a/clang/utils/analyzer/projects/termbox/run_static_analyzer.cmd b/clang/utils/analyzer/projects/termbox/run_static_analyzer.cmd
new file mode 100644
index 0000000000000..1b8c416343e5e
--- /dev/null
+++ b/clang/utils/analyzer/projects/termbox/run_static_analyzer.cmd
@@ -0,0 +1,2 @@
+./waf configure
+./waf
diff --git a/clang/utils/analyzer/projects/tinyexpr/cleanup_run_static_analyzer.sh b/clang/utils/analyzer/projects/tinyexpr/cleanup_run_static_analyzer.sh
new file mode 100755
index 0000000000000..efcd16e590a18
--- /dev/null
+++ b/clang/utils/analyzer/projects/tinyexpr/cleanup_run_static_analyzer.sh
@@ -0,0 +1 @@
+make clean
diff --git a/clang/utils/analyzer/projects/tinyexpr/run_static_analyzer.cmd b/clang/utils/analyzer/projects/tinyexpr/run_static_analyzer.cmd
new file mode 100644
index 0000000000000..8f58e6df93d2f
--- /dev/null
+++ b/clang/utils/analyzer/projects/tinyexpr/run_static_analyzer.cmd
@@ -0,0 +1 @@
+make
diff --git a/clang/utils/analyzer/projects/tinyspline/cleanup_run_static_analyzer.sh b/clang/utils/analyzer/projects/tinyspline/cleanup_run_static_analyzer.sh
new file mode 100755
index 0000000000000..e14c423280ec5
--- /dev/null
+++ b/clang/utils/analyzer/projects/tinyspline/cleanup_run_static_analyzer.sh
@@ -0,0 +1 @@
+rm -rf ./build
diff --git a/clang/utils/analyzer/projects/tinyspline/run_static_analyzer.cmd b/clang/utils/analyzer/projects/tinyspline/run_static_analyzer.cmd
new file mode 100644
index 0000000000000..6678fe635db32
--- /dev/null
+++ b/clang/utils/analyzer/projects/tinyspline/run_static_analyzer.cmd
@@ -0,0 +1,2 @@
+cmake . -DCMAKE_BUILD_TYPE=Debug -Bbuild -GNinja
+cmake --build build
diff --git a/clang/utils/analyzer/projects/tinyvm/cleanup_run_static_analyzer.sh b/clang/utils/analyzer/projects/tinyvm/cleanup_run_static_analyzer.sh
new file mode 100755
index 0000000000000..efcd16e590a18
--- /dev/null
+++ b/clang/utils/analyzer/projects/tinyvm/cleanup_run_static_analyzer.sh
@@ -0,0 +1 @@
+make clean
diff --git a/clang/utils/analyzer/projects/tinyvm/run_static_analyzer.cmd b/clang/utils/analyzer/projects/tinyvm/run_static_analyzer.cmd
new file mode 100644
index 0000000000000..8f58e6df93d2f
--- /dev/null
+++ b/clang/utils/analyzer/projects/tinyvm/run_static_analyzer.cmd
@@ -0,0 +1 @@
+make
diff --git a/clang/utils/analyzer/projects/zstd/cleanup_run_static_analyzer.sh b/clang/utils/analyzer/projects/zstd/cleanup_run_static_analyzer.sh
new file mode 100755
index 0000000000000..2f4ec2f3631bf
--- /dev/null
+++ b/clang/utils/analyzer/projects/zstd/cleanup_run_static_analyzer.sh
@@ -0,0 +1 @@
+rm -rf ./build_analyzer
diff --git a/clang/utils/analyzer/projects/zstd/run_static_analyzer.cmd b/clang/utils/analyzer/projects/zstd/run_static_analyzer.cmd
new file mode 100644
index 0000000000000..5253ac025f051
--- /dev/null
+++ b/clang/utils/analyzer/projects/zstd/run_static_analyzer.cmd
@@ -0,0 +1,2 @@
+cmake ./build/cmake -DCMAKE_BUILD_TYPE=Debug -Bbuild_analyzer -GNinja
+cmake --build build_analyzer
diff --git a/clang/utils/analyzer/requirements.txt b/clang/utils/analyzer/requirements.txt
new file mode 100644
index 0000000000000..ec4f669299523
--- /dev/null
+++ b/clang/utils/analyzer/requirements.txt
@@ -0,0 +1,4 @@
+graphviz
+humanize
+matplotlib
+psutil
diff --git a/clang/utils/make-ast-dump-check.sh b/clang/utils/make-ast-dump-check.sh
index f0c268c883e92..365f227cc1c66 100755
--- a/clang/utils/make-ast-dump-check.sh
+++ b/clang/utils/make-ast-dump-check.sh
@@ -70,6 +70,10 @@ BEGIN {
   if ($generate_serialization_test == 1) {
     gsub(" imported", "{{( imported)?}}", s)
     gsub(" <undeserialized declarations>", "{{( <undeserialized declarations>)?}}", s)
+    gsub("line:[0-9]+:[0-9]+", "line:{{.*}}", s)
+    gsub("line:[0-9]+", "line:{{.*}}", s)
+    gsub("col:[0-9]+", "col:{{.*}}", s)
+    gsub(":[0-9]+:[0-9]+", "{{.*}}", s)
   }
 }
 
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 97879c2e64c79..65f196ad59d0f 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -2789,7 +2789,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://wg21.link/cwg458">458</a></td>
     <td>C++11</td>
     <td>Hiding of member template parameters by other members</td>
-    <td class="none" align="center">No</td>
+    <td class="unreleased" align="center">Clang 11</td>
   </tr>
   <tr class="open" id="459">
     <td><a href="https://wg21.link/cwg459">459</a></td>
diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt
index 8619b6e6280ca..fa62814b635d3 100644
--- a/compiler-rt/CMakeLists.txt
+++ b/compiler-rt/CMakeLists.txt
@@ -535,8 +535,6 @@ else()
   set(COMPILER_RT_LLD_PATH ${LLVM_MAIN_SRC_DIR}/../lld)
   if(EXISTS ${COMPILER_RT_LLD_PATH}/ AND LLVM_TOOL_LLD_BUILD)
     set(COMPILER_RT_HAS_LLD TRUE)
-  else()
-    set(COMPILER_RT_HAS_LLD ${COMPILER_RT_HAS_FUSE_LD_LLD_FLAG})
   endif()
 endif()
 pythonize_bool(COMPILER_RT_HAS_LLD)
diff --git a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
index 657d2c97eac8e..425de8bffdf72 100644
--- a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
+++ b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
@@ -169,25 +169,46 @@ function(darwin_test_archs os valid_archs)
     CACHE STRING "List of valid architectures for platform ${os}." FORCE)
 endfunction()
 
-# This function checks the host cpusubtype to see if it is post-haswell. Haswell
-# and later machines can run x86_64h binaries. Haswell is cpusubtype 8.
+# This function checks the host cputype/cpusubtype to filter supported
+# architecture for the host OS. This is used to determine which tests are
+# available for the host.
 function(darwin_filter_host_archs input output)
   list_intersect(tmp_var DARWIN_osx_ARCHS ${input})
   execute_process(
-    COMMAND sysctl hw.cpusubtype
-    OUTPUT_VARIABLE SUBTYPE)
-
-  string(REGEX MATCH "hw.cpusubtype: ([0-9]*)"
-         SUBTYPE_MATCHED "${SUBTYPE}")
-  set(HASWELL_SUPPORTED Off)
-  if(SUBTYPE_MATCHED)
-    if(${CMAKE_MATCH_1} GREATER 7)
-      set(HASWELL_SUPPORTED On)
+    COMMAND sysctl hw.cputype
+    OUTPUT_VARIABLE CPUTYPE)
+  string(REGEX MATCH "hw.cputype: ([0-9]*)"
+         CPUTYPE_MATCHED "${CPUTYPE}")
+  set(ARM_HOST Off)
+  if(CPUTYPE_MATCHED)
+    # ARM cputype is (0x01000000 | 12) and X86(_64) is always 7.
+    if(${CMAKE_MATCH_1} GREATER 11)
+      set(ARM_HOST On)
     endif()
   endif()
-  if(NOT HASWELL_SUPPORTED)
-    list(REMOVE_ITEM tmp_var x86_64h)
+
+  if(ARM_HOST)
+    list(REMOVE_ITEM tmp_var i386)
+  else()
+    list(REMOVE_ITEM tmp_var arm64)
+    list(REMOVE_ITEM tmp_var arm64e)
+    execute_process(
+      COMMAND sysctl hw.cpusubtype
+      OUTPUT_VARIABLE SUBTYPE)
+    string(REGEX MATCH "hw.cpusubtype: ([0-9]*)"
+           SUBTYPE_MATCHED "${SUBTYPE}")
+
+    set(HASWELL_SUPPORTED Off)
+    if(SUBTYPE_MATCHED)
+      if(${CMAKE_MATCH_1} GREATER 7)
+        set(HASWELL_SUPPORTED On)
+      endif()
+    endif()
+    if(NOT HASWELL_SUPPORTED)
+      list(REMOVE_ITEM tmp_var x86_64h)
+    endif()
   endif()
+
   set(${output} ${tmp_var} PARENT_SCOPE)
 endfunction()
 
diff --git a/compiler-rt/cmake/builtin-config-ix.cmake b/compiler-rt/cmake/builtin-config-ix.cmake
index 5f4275ae54d4c..8de901513beb5 100644
--- a/compiler-rt/cmake/builtin-config-ix.cmake
+++ b/compiler-rt/cmake/builtin-config-ix.cmake
@@ -64,11 +64,34 @@ if(APPLE)
   find_darwin_sdk_dir(DARWIN_tvossim_SYSROOT appletvsimulator)
   find_darwin_sdk_dir(DARWIN_tvos_SYSROOT appletvos)
 
+  # Get supported architecture from SDKSettings.
+  function(sdk_has_arch_support sdk_path os arch has_support)
+    execute_process(COMMAND
+        /usr/libexec/PlistBuddy -c "Print :SupportedTargets:${os}:Archs" ${sdk_path}/SDKSettings.plist
+      OUTPUT_VARIABLE SDK_SUPPORTED_ARCHS
+      RESULT_VARIABLE PLIST_ERROR)
+    if (PLIST_ERROR EQUAL 0 AND
+        SDK_SUPPORTED_ARCHS MATCHES " ${arch}\n")
+      message(STATUS "Found ${arch} support in ${sdk_path}/SDKSettings.plist")
+      set("${has_support}" On PARENT_SCOPE)
+    else()
+      message(STATUS "No ${arch} support in ${sdk_path}/SDKSettings.plist")
+      set("${has_support}" Off PARENT_SCOPE)
+    endif()
+  endfunction()
+
   set(DARWIN_EMBEDDED_PLATFORMS)
   set(DARWIN_osx_BUILTIN_MIN_VER 10.5)
   set(DARWIN_osx_BUILTIN_MIN_VER_FLAG
       -mmacosx-version-min=${DARWIN_osx_BUILTIN_MIN_VER})
   set(DARWIN_osx_BUILTIN_ALL_POSSIBLE_ARCHS ${X86} ${X86_64})
+  # Add support for arm64 macOS if available in SDK.
+  foreach(arch ${ARM64})
+    sdk_has_arch_support(${DARWIN_osx_SYSROOT} macosx ${arch} MACOS_ARM_SUPPORT)
+    if (MACOS_ARM_SUPPORT)
+     list(APPEND DARWIN_osx_BUILTIN_ALL_POSSIBLE_ARCHS ${arch})
+    endif()
+  endforeach(arch)
 
   if(COMPILER_RT_ENABLE_IOS)
     list(APPEND DARWIN_EMBEDDED_PLATFORMS ios)
diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake
index 1f3697ff6f659..2edc1dabd90d3 100644
--- a/compiler-rt/cmake/config-ix.cmake
+++ b/compiler-rt/cmake/config-ix.cmake
@@ -650,7 +650,7 @@ endif()
 
 # TODO: Add builtins support.
 
-if (CRT_SUPPORTED_ARCH AND OS_NAME MATCHES "Linux")
+if (CRT_SUPPORTED_ARCH AND OS_NAME MATCHES "Linux" AND NOT LLVM_USE_SANITIZER)
   set(COMPILER_RT_HAS_CRT TRUE)
 else()
   set(COMPILER_RT_HAS_CRT FALSE)
diff --git a/compiler-rt/lib/asan/tests/asan_test.cpp b/compiler-rt/lib/asan/tests/asan_test.cpp
index 83b0b0e8d33e9..edc98ed185202 100644
--- a/compiler-rt/lib/asan/tests/asan_test.cpp
+++ b/compiler-rt/lib/asan/tests/asan_test.cpp
@@ -588,9 +588,6 @@ NOINLINE void TouchStackFunc() {
     A[i] = i*i;
 }
 
-// Disabled due to rdar://problem/62141412
-#if !(defined(__APPLE__) && defined(__i386__))
-
 // Test that we handle longjmp and do not report false positives on stack.
 TEST(AddressSanitizer, LongJmpTest) {
   static jmp_buf buf;
@@ -600,7 +597,6 @@ TEST(AddressSanitizer, LongJmpTest) {
     TouchStackFunc();
   }
 }
-#endif
 
 #if !defined(_WIN32)  // Only basic longjmp is available on Windows.
 NOINLINE void UnderscopeLongJmpFunc1(jmp_buf buf) {
@@ -662,8 +658,6 @@ TEST(AddressSanitizer, UnderscopeLongJmpTest) {
   }
 }
 
-// Disabled due to rdar://problem/62141412
-#if !(defined(__APPLE__) && defined(__i386__))
 TEST(AddressSanitizer, SigLongJmpTest) {
   static sigjmp_buf buf;
   if (!sigsetjmp(buf, 1)) {
@@ -674,8 +668,6 @@ TEST(AddressSanitizer, SigLongJmpTest) {
 }
 #endif
 
-#endif
-
 // FIXME: Why does clang-cl define __EXCEPTIONS?
 #if defined(__EXCEPTIONS) && !defined(_WIN32)
 NOINLINE void ThrowFunc() {
diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index f8431bdcf059f..5e3c901322ec6 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -46,7 +46,6 @@ set(GENERIC_SOURCES
   absvti2.c
   adddf3.c
   addsf3.c
-  addtf3.c
   addvdi3.c
   addvsi3.c
   addvti3.c
@@ -75,9 +74,7 @@ set(GENERIC_SOURCES
   divsc3.c
   divsf3.c
   divsi3.c
-  divtc3.c
   divti3.c
-  divtf3.c
   extendsfdf2.c
   extendhfsf2.c
   ffsdi2.c
@@ -123,7 +120,6 @@ set(GENERIC_SOURCES
   mulsc3.c
   mulsf3.c
   multi3.c
-  multf3.c
   mulvdi3.c
   mulvsi3.c
   mulvti3.c
@@ -143,13 +139,11 @@ set(GENERIC_SOURCES
   popcountti2.c
   powidf2.c
   powisf2.c
-  powitf2.c
   subdf3.c
   subsf3.c
   subvdi3.c
   subvsi3.c
   subvti3.c
-  subtf3.c
   trampoline_setup.c
   truncdfhf2.c
   truncdfsf2.c
@@ -168,7 +162,10 @@ set(GENERIC_SOURCES
 )
 
 set(GENERIC_TF_SOURCES
+  addtf3.c
   comparetf2.c
+  divtc3.c
+  divtf3.c
   extenddftf2.c
   extendsftf2.c
   fixtfdi.c
@@ -184,6 +181,9 @@ set(GENERIC_TF_SOURCES
   floatunsitf.c
   floatuntitf.c
   multc3.c
+  multf3.c
+  powitf2.c
+  subtf3.c
   trunctfdf2.c
   trunctfsf2.c
 )
@@ -505,13 +505,9 @@ set(hexagon_SOURCES
   hexagon/dfsqrt.S
   hexagon/divdi3.S
   hexagon/divsi3.S
-  hexagon/fabs_opt.S
   hexagon/fastmath2_dlib_asm.S
   hexagon/fastmath2_ldlib_asm.S
   hexagon/fastmath_dlib_asm.S
-  hexagon/fma_opt.S
-  hexagon/fmax_opt.S
-  hexagon/fmin_opt.S
   hexagon/memcpy_forward_vp4cp4n2.S
   hexagon/memcpy_likely_aligned.S
   hexagon/moddi3.S
diff --git a/compiler-rt/lib/builtins/README.txt b/compiler-rt/lib/builtins/README.txt
index e603dfa053566..f9e1bc805092e 100644
--- a/compiler-rt/lib/builtins/README.txt
+++ b/compiler-rt/lib/builtins/README.txt
@@ -20,13 +20,18 @@ Here is the specification for this library:
 
 http://gcc.gnu.org/onlinedocs/gccint/Libgcc.html#Libgcc
 
+Please note that the libgcc specification explicitly mentions actual types of
+arguments and returned values being expressed with machine modes.
+In some cases particular types such as "int", "unsigned", "long long", etc.
+may be specified just as examples there.
+
 Here is a synopsis of the contents of this library:
 
-typedef      int si_int;
-typedef unsigned su_int;
+typedef  int32_t si_int;
+typedef uint32_t su_int;
 
-typedef          long long di_int;
-typedef unsigned long long du_int;
+typedef  int64_t di_int;
+typedef uint64_t du_int;
 
 // Integral bit manipulation
 
@@ -38,24 +43,24 @@ ti_int __ashrti3(ti_int a, si_int b);      // a >> b  arithmetic (sign fill)
 di_int __lshrdi3(di_int a, si_int b);      // a >> b  logical    (zero fill)
 ti_int __lshrti3(ti_int a, si_int b);      // a >> b  logical    (zero fill)
 
-si_int __clzsi2(si_int a);  // count leading zeros
-si_int __clzdi2(di_int a);  // count leading zeros
-si_int __clzti2(ti_int a);  // count leading zeros
-si_int __ctzsi2(si_int a);  // count trailing zeros
-si_int __ctzdi2(di_int a);  // count trailing zeros
-si_int __ctzti2(ti_int a);  // count trailing zeros
+int __clzsi2(si_int a);  // count leading zeros
+int __clzdi2(di_int a);  // count leading zeros
+int __clzti2(ti_int a);  // count leading zeros
+int __ctzsi2(si_int a);  // count trailing zeros
+int __ctzdi2(di_int a);  // count trailing zeros
+int __ctzti2(ti_int a);  // count trailing zeros
 
-si_int __ffssi2(si_int a);  // find least significant 1 bit
-si_int __ffsdi2(di_int a);  // find least significant 1 bit
-si_int __ffsti2(ti_int a);  // find least significant 1 bit
+int __ffssi2(si_int a);  // find least significant 1 bit
+int __ffsdi2(di_int a);  // find least significant 1 bit
+int __ffsti2(ti_int a);  // find least significant 1 bit
 
-si_int __paritysi2(si_int a);  // bit parity
-si_int __paritydi2(di_int a);  // bit parity
-si_int __parityti2(ti_int a);  // bit parity
+int __paritysi2(si_int a);  // bit parity
+int __paritydi2(di_int a);  // bit parity
+int __parityti2(ti_int a);  // bit parity
 
-si_int __popcountsi2(si_int a);  // bit population
-si_int __popcountdi2(di_int a);  // bit population
-si_int __popcountti2(ti_int a);  // bit population
+int __popcountsi2(si_int a);  // bit population
+int __popcountdi2(di_int a);  // bit population
+int __popcountti2(ti_int a);  // bit population
 
 uint32_t __bswapsi2(uint32_t a);   // a byteswapped
 uint64_t __bswapdi2(uint64_t a);   // a byteswapped
@@ -169,10 +174,10 @@ long double __floatuntixf(tu_int a);
 
 //  Floating point raised to integer power
 
-float       __powisf2(      float a, si_int b);  // a ^ b
-double      __powidf2(     double a, si_int b);  // a ^ b
-long double __powixf2(long double a, si_int b);  // a ^ b
-long double __powitf2(long double a, si_int b);  // ppc only, a ^ b
+float       __powisf2(      float a, int b);  // a ^ b
+double      __powidf2(     double a, int b);  // a ^ b
+long double __powixf2(long double a, int b);  // a ^ b
+long double __powitf2(long double a, int b);  // ppc only, a ^ b
 
 //  Complex arithmetic
 
diff --git a/compiler-rt/lib/builtins/clzdi2.c b/compiler-rt/lib/builtins/clzdi2.c
index d64e9eda85130..12c17982a5cb1 100644
--- a/compiler-rt/lib/builtins/clzdi2.c
+++ b/compiler-rt/lib/builtins/clzdi2.c
@@ -21,12 +21,12 @@
 // ctz instruction, gcc resolves __builtin_clz to __clzdi2 rather than
 // __clzsi2, leading to infinite recursion.
 #define __builtin_clz(a) __clzsi2(a)
-extern si_int __clzsi2(si_int);
+extern int __clzsi2(si_int);
 #endif
 
 // Precondition: a != 0
 
-COMPILER_RT_ABI si_int __clzdi2(di_int a) {
+COMPILER_RT_ABI int __clzdi2(di_int a) {
   dwords x;
   x.all = a;
   const si_int f = -(x.s.high == 0);
diff --git a/compiler-rt/lib/builtins/clzsi2.c b/compiler-rt/lib/builtins/clzsi2.c
index 3f9f27f413315..d75f56d937b07 100644
--- a/compiler-rt/lib/builtins/clzsi2.c
+++ b/compiler-rt/lib/builtins/clzsi2.c
@@ -16,7 +16,7 @@
 
 // Precondition: a != 0
 
-COMPILER_RT_ABI si_int __clzsi2(si_int a) {
+COMPILER_RT_ABI int __clzsi2(si_int a) {
   su_int x = (su_int)a;
   si_int t = ((x & 0xFFFF0000) == 0) << 4; // if (x is small) t = 16 else 0
   x >>= 16 - t;                            // x = [0 - 0xFFFF]
diff --git a/compiler-rt/lib/builtins/clzti2.c b/compiler-rt/lib/builtins/clzti2.c
index 0c787104caa2a..25d30119f271c 100644
--- a/compiler-rt/lib/builtins/clzti2.c
+++ b/compiler-rt/lib/builtins/clzti2.c
@@ -18,7 +18,7 @@
 
 // Precondition: a != 0
 
-COMPILER_RT_ABI si_int __clzti2(ti_int a) {
+COMPILER_RT_ABI int __clzti2(ti_int a) {
   twords x;
   x.all = a;
   const di_int f = -(x.s.high == 0);
diff --git a/compiler-rt/lib/builtins/cpu_model.c b/compiler-rt/lib/builtins/cpu_model.c
index 59c6974207c79..8346bb62dcfb4 100644
--- a/compiler-rt/lib/builtins/cpu_model.c
+++ b/compiler-rt/lib/builtins/cpu_model.c
@@ -125,7 +125,8 @@ enum ProcessorFeatures {
   FEATURE_AVX512VNNI,
   FEATURE_AVX512BITALG,
   FEATURE_AVX512BF16,
-  FEATURE_AVX512VP2INTERSECT
+  FEATURE_AVX512VP2INTERSECT,
+  CPU_FEATURE_MAX
 };
 
 // The check below for i386 was copied from clang's cpuid.h (__get_cpuid_max).
@@ -271,12 +272,17 @@ static void detectX86FamilyModel(unsigned EAX, unsigned *Family,
   }
 }
 
-static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
-                                            const unsigned *Features,
-                                            unsigned *Type, unsigned *Subtype) {
+static const char *
+getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
+                                const unsigned *Features,
+                                unsigned *Type, unsigned *Subtype) {
 #define testFeature(F)                                                         \
   (Features[F / 32] & (F % 32)) != 0
 
+  // We select CPU strings to match the code in Host.cpp, but we don't use them
+  // in compiler-rt.
+  const char *CPU = 0;
+
   switch (Family) {
   case 6:
     switch (Model) {
@@ -287,13 +293,17 @@ static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
                // 0Fh. All processors are manufactured using the 65 nm process.
     case 0x16: // Intel Celeron processor model 16h. All processors are
                // manufactured using the 65 nm process
+      CPU = "core2";
+      *Type = INTEL_CORE2;
+      break;
     case 0x17: // Intel Core 2 Extreme processor, Intel Xeon processor, model
                // 17h. All processors are manufactured using the 45 nm process.
                //
                // 45nm: Penryn , Wolfdale, Yorkfield (XE)
     case 0x1d: // Intel Xeon processor MP. All processors are manufactured using
                // the 45 nm process.
-      *Type = INTEL_CORE2; // "penryn"
+      CPU = "penryn";
+      *Type = INTEL_CORE2;
       break;
     case 0x1a: // Intel Core i7 processor and Intel Xeon processor. All
                // processors are manufactured using the 45 nm process.
@@ -301,25 +311,29 @@ static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
                // As found in a Summer 2010 model iMac.
     case 0x1f:
     case 0x2e:              // Nehalem EX
-      *Type = INTEL_COREI7; // "nehalem"
+      CPU = "nehalem";
+      *Type = INTEL_COREI7;
       *Subtype = INTEL_COREI7_NEHALEM;
       break;
     case 0x25: // Intel Core i7, laptop version.
     case 0x2c: // Intel Core i7 processor and Intel Xeon processor. All
                // processors are manufactured using the 32 nm process.
     case 0x2f: // Westmere EX
-      *Type = INTEL_COREI7; // "westmere"
+      CPU = "westmere";
+      *Type = INTEL_COREI7;
       *Subtype = INTEL_COREI7_WESTMERE;
       break;
     case 0x2a: // Intel Core i7 processor. All processors are manufactured
                // using the 32 nm process.
     case 0x2d:
-      *Type = INTEL_COREI7; //"sandybridge"
+      CPU = "sandybridge";
+      *Type = INTEL_COREI7;
       *Subtype = INTEL_COREI7_SANDYBRIDGE;
       break;
     case 0x3a:
     case 0x3e:              // Ivy Bridge EP
-      *Type = INTEL_COREI7; // "ivybridge"
+      CPU = "ivybridge";
+      *Type = INTEL_COREI7;
       *Subtype = INTEL_COREI7_IVYBRIDGE;
       break;
 
@@ -328,7 +342,8 @@ static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
     case 0x3f:
     case 0x45:
     case 0x46:
-      *Type = INTEL_COREI7; // "haswell"
+      CPU = "haswell";
+      *Type = INTEL_COREI7;
       *Subtype = INTEL_COREI7_HASWELL;
       break;
 
@@ -337,7 +352,8 @@ static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
     case 0x47:
     case 0x4f:
     case 0x56:
-      *Type = INTEL_COREI7; // "broadwell"
+      CPU = "broadwell";
+      *Type = INTEL_COREI7;
       *Subtype = INTEL_COREI7_BROADWELL;
       break;
 
@@ -348,39 +364,47 @@ static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
     case 0x9e:              // Kaby Lake desktop
     case 0xa5:              // Comet Lake-H/S
     case 0xa6:              // Comet Lake-U
-      *Type = INTEL_COREI7; // "skylake"
+      CPU = "skylake";
+      *Type = INTEL_COREI7;
       *Subtype = INTEL_COREI7_SKYLAKE;
       break;
 
     // Skylake Xeon:
     case 0x55:
       *Type = INTEL_COREI7;
-      if (testFeature(FEATURE_AVX512BF16))
-        *Subtype = INTEL_COREI7_COOPERLAKE; // "cooperlake"
-      else if (testFeature(FEATURE_AVX512VNNI))
-        *Subtype = INTEL_COREI7_CASCADELAKE; // "cascadelake"
-      else
-        *Subtype = INTEL_COREI7_SKYLAKE_AVX512; // "skylake-avx512"
+      if (testFeature(FEATURE_AVX512BF16)) {
+        CPU = "cooperlake";
+        *Subtype = INTEL_COREI7_COOPERLAKE;
+      } else if (testFeature(FEATURE_AVX512VNNI)) {
+        CPU = "cascadelake";
+        *Subtype = INTEL_COREI7_CASCADELAKE;
+      } else {
+        CPU = "skylake-avx512";
+        *Subtype = INTEL_COREI7_SKYLAKE_AVX512;
+      }
       break;
 
     // Cannonlake:
     case 0x66:
+      CPU = "cannonlake";
       *Type = INTEL_COREI7;
-      *Subtype = INTEL_COREI7_CANNONLAKE; // "cannonlake"
+      *Subtype = INTEL_COREI7_CANNONLAKE;
       break;
 
     // Icelake:
     case 0x7d:
     case 0x7e:
+      CPU = "icelake-client";
       *Type = INTEL_COREI7;
-      *Subtype = INTEL_COREI7_ICELAKE_CLIENT; // "icelake-client"
+      *Subtype = INTEL_COREI7_ICELAKE_CLIENT;
       break;
 
     // Icelake Xeon:
     case 0x6a:
     case 0x6c:
+      CPU = "icelake-server";
       *Type = INTEL_COREI7;
-      *Subtype = INTEL_COREI7_ICELAKE_SERVER; // "icelake-server"
+      *Subtype = INTEL_COREI7_ICELAKE_SERVER;
       break;
 
     case 0x1c: // Most 45 nm Intel Atom processors
@@ -388,8 +412,9 @@ static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
     case 0x27: // 32 nm Atom Medfield
     case 0x35: // 32 nm Atom Midview
     case 0x36: // 32 nm Atom Midview
+      CPU = "bonnell";
       *Type = INTEL_BONNELL;
-      break; // "bonnell"
+      break;
 
     // Atom Silvermont codes from the Intel software optimization guide.
     case 0x37:
@@ -398,26 +423,32 @@ static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
     case 0x5a:
     case 0x5d:
     case 0x4c: // really airmont
+      CPU = "silvermont";
       *Type = INTEL_SILVERMONT;
-      break; // "silvermont"
+      break;
     // Goldmont:
     case 0x5c: // Apollo Lake
     case 0x5f: // Denverton
+      CPU = "goldmont";
       *Type = INTEL_GOLDMONT;
       break; // "goldmont"
     case 0x7a:
+      CPU = "goldmont-plus";
       *Type = INTEL_GOLDMONT_PLUS;
       break;
     case 0x86:
+      CPU = "tremont";
       *Type = INTEL_TREMONT;
       break;
 
     case 0x57:
-      *Type = INTEL_KNL; // knl
+      CPU = "knl";
+      *Type = INTEL_KNL;
       break;
 
     case 0x85:
-      *Type = INTEL_KNM; // knm
+      CPU = "knm";
+      *Type = INTEL_KNM;
       break;
 
     default: // Unknown family 6 CPU.
@@ -427,17 +458,22 @@ static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
   default:
     break; // Unknown.
   }
+
+  return CPU;
 }
 
-static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
-                                          const unsigned *Features,
-                                          unsigned *Type, unsigned *Subtype) {
-  // FIXME: this poorly matches the generated SubtargetFeatureKV table.  There
-  // appears to be no way to generate the wide variety of AMD-specific targets
-  // from the information returned from CPUID.
+static const char *
+getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
+                              const unsigned *Features,
+                              unsigned *Type, unsigned *Subtype) {
+  // We select CPU strings to match the code in Host.cpp, but we don't use them
+  // in compiler-rt.
+  const char *CPU = 0;
+
   switch (Family) {
   case 16:
-    *Type = AMDFAM10H; // "amdfam10"
+    CPU = "amdfam10";
+    *Type = AMDFAM10H;
     switch (Model) {
     case 2:
       *Subtype = AMDFAM10H_BARCELONA;
@@ -451,50 +487,58 @@ static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
     }
     break;
   case 20:
+    CPU = "btver1";
     *Type = AMD_BTVER1;
-    break; // "btver1";
+    break;
   case 21:
+    CPU = "bdver1";
     *Type = AMDFAM15H;
     if (Model >= 0x60 && Model <= 0x7f) {
+      CPU = "bdver4";
       *Subtype = AMDFAM15H_BDVER4;
-      break; // "bdver4"; 60h-7Fh: Excavator
+      break; // 60h-7Fh: Excavator
     }
     if (Model >= 0x30 && Model <= 0x3f) {
+      CPU = "bdver3";
       *Subtype = AMDFAM15H_BDVER3;
-      break; // "bdver3"; 30h-3Fh: Steamroller
+      break; // 30h-3Fh: Steamroller
     }
     if ((Model >= 0x10 && Model <= 0x1f) || Model == 0x02) {
+      CPU = "bdver2";
       *Subtype = AMDFAM15H_BDVER2;
-      break; // "bdver2"; 02h, 10h-1Fh: Piledriver
+      break; // 02h, 10h-1Fh: Piledriver
     }
     if (Model <= 0x0f) {
       *Subtype = AMDFAM15H_BDVER1;
-      break; // "bdver1"; 00h-0Fh: Bulldozer
+      break; // 00h-0Fh: Bulldozer
     }
     break;
   case 22:
+    CPU = "btver2";
     *Type = AMD_BTVER2;
-    break; // "btver2"
+    break;
   case 23:
+    CPU = "znver1";
     *Type = AMDFAM17H;
     if ((Model >= 0x30 && Model <= 0x3f) || Model == 0x71) {
+      CPU = "znver2";
       *Subtype = AMDFAM17H_ZNVER2;
-      break; // "znver2"; 30h-3fh, 71h: Zen2
+      break; // 30h-3fh, 71h: Zen2
     }
     if (Model <= 0x0f) {
       *Subtype = AMDFAM17H_ZNVER1;
-      break; // "znver1"; 00h-0Fh: Zen1
+      break; // 00h-0Fh: Zen1
     }
     break;
   default:
-    break; // "generic"
+    break; // Unknown AMD CPU.
   }
+
+  return CPU;
 }
 
 static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
                                  unsigned *Features) {
-  Features[0] = 0;
-  Features[1] = 0;
   unsigned EAX, EBX;
 
 #define setFeature(F)                                                          \
@@ -653,7 +697,7 @@ int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) {
   unsigned MaxLeaf = 5;
   unsigned Vendor;
   unsigned Model, Family;
-  unsigned Features[2];
+  unsigned Features[(CPU_FEATURE_MAX + 31) / 32] = {0};
 
   // This function needs to run just once.
   if (__cpu_model.__cpu_vendor)
@@ -670,6 +714,8 @@ int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) {
 
   // Find available features.
   getAvailableFeatures(ECX, EDX, MaxLeaf, &Features[0]);
+
+  assert((sizeof(Features)/sizeof(Features[0])) == 2);
   __cpu_model.__cpu_features[0] = Features[0];
   __cpu_features2 = Features[1];
 
diff --git a/compiler-rt/lib/builtins/ctzdi2.c b/compiler-rt/lib/builtins/ctzdi2.c
index 8335f5cbb314e..26c908d876ac4 100644
--- a/compiler-rt/lib/builtins/ctzdi2.c
+++ b/compiler-rt/lib/builtins/ctzdi2.c
@@ -21,7 +21,7 @@
 // ctz instruction, gcc resolves __builtin_ctz to __ctzdi2 rather than
 // __ctzsi2, leading to infinite recursion.
 #define __builtin_ctz(a) __ctzsi2(a)
-extern si_int __ctzsi2(si_int);
+extern int __ctzsi2(si_int);
 #endif
 
 // Precondition: a != 0
diff --git a/compiler-rt/lib/builtins/ctzsi2.c b/compiler-rt/lib/builtins/ctzsi2.c
index 09c6863b74e3a..ed95c60579339 100644
--- a/compiler-rt/lib/builtins/ctzsi2.c
+++ b/compiler-rt/lib/builtins/ctzsi2.c
@@ -16,7 +16,7 @@
 
 // Precondition: a != 0
 
-COMPILER_RT_ABI si_int __ctzsi2(si_int a) {
+COMPILER_RT_ABI int __ctzsi2(si_int a) {
   su_int x = (su_int)a;
   si_int t = ((x & 0x0000FFFF) == 0)
              << 4; // if (x has no small bits) t = 16 else 0
diff --git a/compiler-rt/lib/builtins/ctzti2.c b/compiler-rt/lib/builtins/ctzti2.c
index 2a1312c8437de..fb136d0de1c0d 100644
--- a/compiler-rt/lib/builtins/ctzti2.c
+++ b/compiler-rt/lib/builtins/ctzti2.c
@@ -18,7 +18,7 @@
 
 // Precondition: a != 0
 
-COMPILER_RT_ABI si_int __ctzti2(ti_int a) {
+COMPILER_RT_ABI int __ctzti2(ti_int a) {
   twords x;
   x.all = a;
   const di_int f = -(x.s.low == 0);
diff --git a/compiler-rt/lib/builtins/ffsti2.c b/compiler-rt/lib/builtins/ffsti2.c
index a2d7ce08ada13..a2177d148a099 100644
--- a/compiler-rt/lib/builtins/ffsti2.c
+++ b/compiler-rt/lib/builtins/ffsti2.c
@@ -17,7 +17,7 @@
 // Returns: the index of the least significant 1-bit in a, or
 // the value zero if a is zero. The least significant bit is index one.
 
-COMPILER_RT_ABI si_int __ffsti2(ti_int a) {
+COMPILER_RT_ABI int __ffsti2(ti_int a) {
   twords x;
   x.all = a;
   if (x.s.low == 0) {
diff --git a/compiler-rt/lib/builtins/floatdidf.c b/compiler-rt/lib/builtins/floatdidf.c
index 8f887314b9e1b..b2d8f2b44b6d3 100644
--- a/compiler-rt/lib/builtins/floatdidf.c
+++ b/compiler-rt/lib/builtins/floatdidf.c
@@ -87,7 +87,7 @@ COMPILER_RT_ABI double __floatdidf(di_int a) {
   }
   double_bits fb;
   fb.u.s.high = ((su_int)s & 0x80000000) |        // sign
-                ((e + 1023) << 20) |              // exponent
+                ((su_int)(e + 1023) << 20) |      // exponent
                 ((su_int)(a >> 32) & 0x000FFFFF); // mantissa-high
   fb.u.s.low = (su_int)a;                         // mantissa-low
   return fb.f;
diff --git a/compiler-rt/lib/builtins/floatundidf.c b/compiler-rt/lib/builtins/floatundidf.c
index e7c6aae5ce382..4c445b118080b 100644
--- a/compiler-rt/lib/builtins/floatundidf.c
+++ b/compiler-rt/lib/builtins/floatundidf.c
@@ -90,7 +90,7 @@ COMPILER_RT_ABI double __floatundidf(du_int a) {
     // a is now rounded to DBL_MANT_DIG bits
   }
   double_bits fb;
-  fb.u.s.high = ((e + 1023) << 20) |              // exponent
+  fb.u.s.high = ((su_int)(e + 1023) << 20) |      // exponent
                 ((su_int)(a >> 32) & 0x000FFFFF); // mantissa-high
   fb.u.s.low = (su_int)a;                         // mantissa-low
   return fb.f;
diff --git a/compiler-rt/lib/builtins/fp_lib.h b/compiler-rt/lib/builtins/fp_lib.h
index e6ba077d23130..bd1f180f499e8 100644
--- a/compiler-rt/lib/builtins/fp_lib.h
+++ b/compiler-rt/lib/builtins/fp_lib.h
@@ -46,7 +46,7 @@ typedef float fp_t;
 #define REP_C UINT32_C
 #define significandBits 23
 
-static __inline int rep_clz(rep_t a) { return __builtin_clz(a); }
+static __inline int rep_clz(rep_t a) { return clzsi(a); }
 
 // 32x32 --> 64 bit multiply
 static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) {
diff --git a/compiler-rt/lib/builtins/hexagon/dffma.S b/compiler-rt/lib/builtins/hexagon/dffma.S
index c201d3d8be5e1..843e88b3cab84 100644
--- a/compiler-rt/lib/builtins/hexagon/dffma.S
+++ b/compiler-rt/lib/builtins/hexagon/dffma.S
@@ -104,13 +104,11 @@
         .type __hexagon_fmadf4,@function
 	.global __hexagon_fmadf5
         .type __hexagon_fmadf5,@function
-	.global fma
-	.type fma,@function
 	Q6_ALIAS(fmadf5)
 	.p2align 5
 __hexagon_fmadf4:
 __hexagon_fmadf5:
-fma:
+.Lfma_begin:
 	{
 		P_TMP = dfclass(A,#2)
 		P_TMP = dfclass(B,#2)
@@ -561,7 +559,7 @@ fma:
 		B = insert(BTMP,#63,#0)
 		AH -= asl(TMP,#HI_MANTBITS)
 	}
-	jump fma
+	jump .Lfma_begin
 
 .Lfma_ab_tiny:
 	ATMP = combine(##0x00100000,#0)
@@ -569,7 +567,7 @@ fma:
 		A = insert(ATMP,#63,#0)
 		B = insert(ATMP,#63,#0)
 	}
-	jump fma
+	jump .Lfma_begin
 
 .Lab_inf:
 	{
diff --git a/compiler-rt/lib/builtins/hexagon/fabs_opt.S b/compiler-rt/lib/builtins/hexagon/fabs_opt.S
deleted file mode 100644
index 6bf9b84b3d209..0000000000000
--- a/compiler-rt/lib/builtins/hexagon/fabs_opt.S
+++ /dev/null
@@ -1,36 +0,0 @@
-//===----------------------Hexagon builtin routine ------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-.macro FUNCTION_BEGIN name
-.text
-.p2align 5
-.globl \name
-.type  \name, @function
-\name:
-.endm
-
-.macro FUNCTION_END name
-.size  \name, . - \name
-.endm
-
-FUNCTION_BEGIN fabs
-  {
-    r1 = clrbit(r1, #31)
-    jumpr r31
-  }
-FUNCTION_END fabs
-
-FUNCTION_BEGIN fabsf
-  {
-    r0 = clrbit(r0, #31)
-    jumpr r31
-  }
-FUNCTION_END fabsf
-
-  .globl fabsl
-  .set fabsl, fabs
diff --git a/compiler-rt/lib/builtins/hexagon/fma_opt.S b/compiler-rt/lib/builtins/hexagon/fma_opt.S
deleted file mode 100644
index 7f566adffd6a4..0000000000000
--- a/compiler-rt/lib/builtins/hexagon/fma_opt.S
+++ /dev/null
@@ -1,30 +0,0 @@
-//===----------------------Hexagon builtin routine ------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-.macro FUNCTION_BEGIN name
-.text
-.p2align 5
-.globl \name
-.type  \name, @function
-\name:
-.endm
-
-.macro FUNCTION_END name
-.size  \name, . - \name
-.endm
-
-FUNCTION_BEGIN fmaf
-  r2 += sfmpy(r0, r1)
-  {
-    r0 = r2
-    jumpr r31
-  }
-FUNCTION_END fmaf
-
-  .globl fmal
-  .set fmal, fma
diff --git a/compiler-rt/lib/builtins/hexagon/fmax_opt.S b/compiler-rt/lib/builtins/hexagon/fmax_opt.S
deleted file mode 100644
index 81d711dff8d20..0000000000000
--- a/compiler-rt/lib/builtins/hexagon/fmax_opt.S
+++ /dev/null
@@ -1,29 +0,0 @@
-//===----------------------Hexagon builtin routine ------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-.macro FUNCTION_BEGIN name
-.text
-.p2align 5
-.globl \name
-.type  \name, @function
-\name:
-.endm
-
-.macro FUNCTION_END name
-.size  \name, . - \name
-.endm
-
-FUNCTION_BEGIN fmaxf
-  {
-    r0 = sfmax(r0, r1)
-    jumpr r31
-  }
-FUNCTION_END fmaxf
-
-  .globl fmaxl
-  .set fmaxl, fmax
diff --git a/compiler-rt/lib/builtins/hexagon/fmin_opt.S b/compiler-rt/lib/builtins/hexagon/fmin_opt.S
deleted file mode 100644
index d043f1d7a6987..0000000000000
--- a/compiler-rt/lib/builtins/hexagon/fmin_opt.S
+++ /dev/null
@@ -1,29 +0,0 @@
-//===----------------------Hexagon builtin routine ------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-.macro FUNCTION_BEGIN name
-.text
-.p2align 5
-.globl \name
-.type  \name, @function
-\name:
-.endm
-
-.macro FUNCTION_END name
-.size  \name, . - \name
-.endm
-
-FUNCTION_BEGIN fminf
-  {
-    r0 = sfmin(r0, r1)
-    jumpr r31
-  }
-FUNCTION_END fminf
-
-  .globl fminl
-  .set fminl, fmin
diff --git a/compiler-rt/lib/builtins/int_lib.h b/compiler-rt/lib/builtins/int_lib.h
index 7e1dabe29d01a..991c4a99ea6e2 100644
--- a/compiler-rt/lib/builtins/int_lib.h
+++ b/compiler-rt/lib/builtins/int_lib.h
@@ -92,8 +92,8 @@
 // Include internal utility function declarations.
 #include "int_util.h"
 
-COMPILER_RT_ABI si_int __paritysi2(si_int a);
-COMPILER_RT_ABI si_int __paritydi2(di_int a);
+COMPILER_RT_ABI int __paritysi2(si_int a);
+COMPILER_RT_ABI int __paritydi2(di_int a);
 
 COMPILER_RT_ABI di_int __divdi3(di_int a, di_int b);
 COMPILER_RT_ABI si_int __divsi3(si_int a, si_int b);
@@ -102,7 +102,7 @@ COMPILER_RT_ABI su_int __udivsi3(su_int n, su_int d);
 COMPILER_RT_ABI su_int __udivmodsi4(su_int a, su_int b, su_int *rem);
 COMPILER_RT_ABI du_int __udivmoddi4(du_int a, du_int b, du_int *rem);
 #ifdef CRT_HAS_128BIT
-COMPILER_RT_ABI si_int __clzti2(ti_int a);
+COMPILER_RT_ABI int __clzti2(ti_int a);
 COMPILER_RT_ABI tu_int __udivmodti4(tu_int a, tu_int b, tu_int *rem);
 #endif
 
@@ -110,14 +110,14 @@ COMPILER_RT_ABI tu_int __udivmodti4(tu_int a, tu_int b, tu_int *rem);
 #if defined(_MSC_VER) && !defined(__clang__)
 #include <intrin.h>
 
-uint32_t __inline __builtin_ctz(uint32_t value) {
+int __inline __builtin_ctz(uint32_t value) {
   unsigned long trailing_zero = 0;
   if (_BitScanForward(&trailing_zero, value))
     return trailing_zero;
   return 32;
 }
 
-uint32_t __inline __builtin_clz(uint32_t value) {
+int __inline __builtin_clz(uint32_t value) {
   unsigned long leading_zero = 0;
   if (_BitScanReverse(&leading_zero, value))
     return 31 - leading_zero;
@@ -125,14 +125,14 @@ uint32_t __inline __builtin_clz(uint32_t value) {
 }
 
 #if defined(_M_ARM) || defined(_M_X64)
-uint32_t __inline __builtin_clzll(uint64_t value) {
+int __inline __builtin_clzll(uint64_t value) {
   unsigned long leading_zero = 0;
   if (_BitScanReverse64(&leading_zero, value))
     return 63 - leading_zero;
   return 64;
 }
 #else
-uint32_t __inline __builtin_clzll(uint64_t value) {
+int __inline __builtin_clzll(uint64_t value) {
   if (value == 0)
     return 64;
   uint32_t msh = (uint32_t)(value >> 32);
diff --git a/compiler-rt/lib/builtins/paritydi2.c b/compiler-rt/lib/builtins/paritydi2.c
index dd9d45e63ea47..58e85f89e0437 100644
--- a/compiler-rt/lib/builtins/paritydi2.c
+++ b/compiler-rt/lib/builtins/paritydi2.c
@@ -14,7 +14,7 @@
 
 // Returns: 1 if number of bits is odd else returns 0
 
-COMPILER_RT_ABI si_int __paritydi2(di_int a) {
+COMPILER_RT_ABI int __paritydi2(di_int a) {
   dwords x;
   x.all = a;
   return __paritysi2(x.s.high ^ x.s.low);
diff --git a/compiler-rt/lib/builtins/paritysi2.c b/compiler-rt/lib/builtins/paritysi2.c
index 3efa961f2f85f..a4b84e0806328 100644
--- a/compiler-rt/lib/builtins/paritysi2.c
+++ b/compiler-rt/lib/builtins/paritysi2.c
@@ -14,7 +14,7 @@
 
 // Returns: 1 if number of bits is odd else returns 0
 
-COMPILER_RT_ABI si_int __paritysi2(si_int a) {
+COMPILER_RT_ABI int __paritysi2(si_int a) {
   su_int x = (su_int)a;
   x ^= x >> 16;
   x ^= x >> 8;
diff --git a/compiler-rt/lib/builtins/parityti2.c b/compiler-rt/lib/builtins/parityti2.c
index f3942ba8378c7..79e920d8a02df 100644
--- a/compiler-rt/lib/builtins/parityti2.c
+++ b/compiler-rt/lib/builtins/parityti2.c
@@ -16,7 +16,7 @@
 
 // Returns: 1 if number of bits is odd else returns 0
 
-COMPILER_RT_ABI si_int __parityti2(ti_int a) {
+COMPILER_RT_ABI int __parityti2(ti_int a) {
   twords x;
   x.all = a;
   return __paritydi2(x.s.high ^ x.s.low);
diff --git a/compiler-rt/lib/builtins/popcountsi2.c b/compiler-rt/lib/builtins/popcountsi2.c
index 75e592a778d98..4d346c45d9cee 100644
--- a/compiler-rt/lib/builtins/popcountsi2.c
+++ b/compiler-rt/lib/builtins/popcountsi2.c
@@ -14,7 +14,7 @@
 
 // Returns: count of 1 bits
 
-COMPILER_RT_ABI si_int __popcountsi2(si_int a) {
+COMPILER_RT_ABI int __popcountsi2(si_int a) {
   su_int x = (su_int)a;
   x = x - ((x >> 1) & 0x55555555);
   // Every 2 bits holds the sum of every pair of bits
diff --git a/compiler-rt/lib/builtins/popcountti2.c b/compiler-rt/lib/builtins/popcountti2.c
index 853fd722309ee..79cbb2fb34c00 100644
--- a/compiler-rt/lib/builtins/popcountti2.c
+++ b/compiler-rt/lib/builtins/popcountti2.c
@@ -17,7 +17,7 @@
 
 // Returns: count of 1 bits
 
-COMPILER_RT_ABI si_int __popcountti2(ti_int a) {
+COMPILER_RT_ABI int __popcountti2(ti_int a) {
   tu_int x3 = (tu_int)a;
   x3 = x3 - ((x3 >> 1) &
              (((tu_int)0x5555555555555555uLL << 64) | 0x5555555555555555uLL));
diff --git a/compiler-rt/lib/builtins/powidf2.c b/compiler-rt/lib/builtins/powidf2.c
index 9697588484e74..81058af508297 100644
--- a/compiler-rt/lib/builtins/powidf2.c
+++ b/compiler-rt/lib/builtins/powidf2.c
@@ -14,7 +14,7 @@
 
 // Returns: a ^ b
 
-COMPILER_RT_ABI double __powidf2(double a, si_int b) {
+COMPILER_RT_ABI double __powidf2(double a, int b) {
   const int recip = b < 0;
   double r = 1;
   while (1) {
diff --git a/compiler-rt/lib/builtins/powisf2.c b/compiler-rt/lib/builtins/powisf2.c
index 469402348825c..d0ab26167bbd2 100644
--- a/compiler-rt/lib/builtins/powisf2.c
+++ b/compiler-rt/lib/builtins/powisf2.c
@@ -14,7 +14,7 @@
 
 // Returns: a ^ b
 
-COMPILER_RT_ABI float __powisf2(float a, si_int b) {
+COMPILER_RT_ABI float __powisf2(float a, int b) {
   const int recip = b < 0;
   float r = 1;
   while (1) {
diff --git a/compiler-rt/lib/builtins/powitf2.c b/compiler-rt/lib/builtins/powitf2.c
index 141a3a0ea727f..8e639a03a3c4b 100644
--- a/compiler-rt/lib/builtins/powitf2.c
+++ b/compiler-rt/lib/builtins/powitf2.c
@@ -17,7 +17,7 @@
 
 // Returns: a ^ b
 
-COMPILER_RT_ABI long double __powitf2(long double a, si_int b) {
+COMPILER_RT_ABI long double __powitf2(long double a, int b) {
   const int recip = b < 0;
   long double r = 1;
   while (1) {
diff --git a/compiler-rt/lib/builtins/powixf2.c b/compiler-rt/lib/builtins/powixf2.c
index b7b52095afa17..3edfe9fd7af59 100644
--- a/compiler-rt/lib/builtins/powixf2.c
+++ b/compiler-rt/lib/builtins/powixf2.c
@@ -16,7 +16,7 @@
 
 // Returns: a ^ b
 
-COMPILER_RT_ABI long double __powixf2(long double a, si_int b) {
+COMPILER_RT_ABI long double __powixf2(long double a, int b) {
   const int recip = b < 0;
   long double r = 1;
   while (1) {
diff --git a/compiler-rt/lib/builtins/udivmodti4.c b/compiler-rt/lib/builtins/udivmodti4.c
index dd14a8b579ca5..55def37c9e1fe 100644
--- a/compiler-rt/lib/builtins/udivmodti4.c
+++ b/compiler-rt/lib/builtins/udivmodti4.c
@@ -14,182 +14,145 @@
 
 #ifdef CRT_HAS_128BIT
 
+// Returns the 128 bit division result by 64 bit. Result must fit in 64 bits.
+// Remainder stored in r.
+// Taken and adjusted from libdivide libdivide_128_div_64_to_64 division
+// fallback. For a correctness proof see the reference for this algorithm
+// in Knuth, Volume 2, section 4.3.1, Algorithm D.
+UNUSED
+static inline du_int udiv128by64to64default(du_int u1, du_int u0, du_int v,
+                                            du_int *r) {
+  const unsigned n_udword_bits = sizeof(du_int) * CHAR_BIT;
+  const du_int b = (1ULL << (n_udword_bits / 2)); // Number base (32 bits)
+  du_int un1, un0;                                // Norm. dividend LSD's
+  du_int vn1, vn0;                                // Norm. divisor digits
+  du_int q1, q0;                                  // Quotient digits
+  du_int un64, un21, un10;                        // Dividend digit pairs
+  du_int rhat;                                    // A remainder
+  si_int s;                                       // Shift amount for normalization
+
+  s = __builtin_clzll(v);
+  if (s > 0) {
+    // Normalize the divisor.
+    v = v << s;
+    un64 = (u1 << s) | (u0 >> (n_udword_bits - s));
+    un10 = u0 << s; // Shift dividend left
+  } else {
+    // Avoid undefined behavior of (u0 >> 64).
+    un64 = u1;
+    un10 = u0;
+  }
+
+  // Break divisor up into two 32-bit digits.
+  vn1 = v >> (n_udword_bits / 2);
+  vn0 = v & 0xFFFFFFFF;
+
+  // Break right half of dividend into two digits.
+  un1 = un10 >> (n_udword_bits / 2);
+  un0 = un10 & 0xFFFFFFFF;
+
+  // Compute the first quotient digit, q1.
+  q1 = un64 / vn1;
+  rhat = un64 - q1 * vn1;
+
+  // q1 has at most error 2. No more than 2 iterations.
+  while (q1 >= b || q1 * vn0 > b * rhat + un1) {
+    q1 = q1 - 1;
+    rhat = rhat + vn1;
+    if (rhat >= b)
+      break;
+  }
+
+  un21 = un64 * b + un1 - q1 * v;
+
+  // Compute the second quotient digit.
+  q0 = un21 / vn1;
+  rhat = un21 - q0 * vn1;
+
+  // q0 has at most error 2. No more than 2 iterations.
+  while (q0 >= b || q0 * vn0 > b * rhat + un0) {
+    q0 = q0 - 1;
+    rhat = rhat + vn1;
+    if (rhat >= b)
+      break;
+  }
+
+  *r = (un21 * b + un0 - q0 * v) >> s;
+  return q1 * b + q0;
+}
+
+static inline du_int udiv128by64to64(du_int u1, du_int u0, du_int v,
+                                     du_int *r) {
+#if defined(__x86_64__)
+  du_int result;
+  __asm__("divq %[v]"
+          : "=a"(result), "=d"(*r)
+          : [ v ] "r"(v), "a"(u0), "d"(u1));
+  return result;
+#else
+  return udiv128by64to64default(u1, u0, v, r);
+#endif
+}
+
 // Effects: if rem != 0, *rem = a % b
 // Returns: a / b
 
-// Translated from Figure 3-40 of The PowerPC Compiler Writer's Guide
-
 COMPILER_RT_ABI tu_int __udivmodti4(tu_int a, tu_int b, tu_int *rem) {
-  const unsigned n_udword_bits = sizeof(du_int) * CHAR_BIT;
   const unsigned n_utword_bits = sizeof(tu_int) * CHAR_BIT;
-  utwords n;
-  n.all = a;
-  utwords d;
-  d.all = b;
-  utwords q;
-  utwords r;
-  unsigned sr;
-  // special cases, X is unknown, K != 0
-  if (n.s.high == 0) {
-    if (d.s.high == 0) {
-      // 0 X
-      // ---
-      // 0 X
-      if (rem)
-        *rem = n.s.low % d.s.low;
-      return n.s.low / d.s.low;
-    }
-    // 0 X
-    // ---
-    // K X
+  utwords dividend;
+  dividend.all = a;
+  utwords divisor;
+  divisor.all = b;
+  utwords quotient;
+  utwords remainder;
+  if (divisor.all > dividend.all) {
     if (rem)
-      *rem = n.s.low;
+      *rem = dividend.all;
     return 0;
   }
-  // n.s.high != 0
-  if (d.s.low == 0) {
-    if (d.s.high == 0) {
-      // K X
-      // ---
-      // 0 0
-      if (rem)
-        *rem = n.s.high % d.s.low;
-      return n.s.high / d.s.low;
-    }
-    // d.s.high != 0
-    if (n.s.low == 0) {
-      // K 0
-      // ---
-      // K 0
-      if (rem) {
-        r.s.high = n.s.high % d.s.high;
-        r.s.low = 0;
-        *rem = r.all;
-      }
-      return n.s.high / d.s.high;
-    }
-    // K K
-    // ---
-    // K 0
-    if ((d.s.high & (d.s.high - 1)) == 0) /* if d is a power of 2 */ {
-      if (rem) {
-        r.s.low = n.s.low;
-        r.s.high = n.s.high & (d.s.high - 1);
-        *rem = r.all;
-      }
-      return n.s.high >> __builtin_ctzll(d.s.high);
-    }
-    // K K
-    // ---
-    // K 0
-    sr = __builtin_clzll(d.s.high) - __builtin_clzll(n.s.high);
-    // 0 <= sr <= n_udword_bits - 2 or sr large
-    if (sr > n_udword_bits - 2) {
-      if (rem)
-        *rem = n.all;
-      return 0;
-    }
-    ++sr;
-    // 1 <= sr <= n_udword_bits - 1
-    // q.all = n.all << (n_utword_bits - sr);
-    q.s.low = 0;
-    q.s.high = n.s.low << (n_udword_bits - sr);
-    // r.all = n.all >> sr;
-    r.s.high = n.s.high >> sr;
-    r.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr);
-  } else /* d.s.low != 0 */ {
-    if (d.s.high == 0) {
-      // K X
-      // ---
-      // 0 K
-      if ((d.s.low & (d.s.low - 1)) == 0) /* if d is a power of 2 */ {
-        if (rem)
-          *rem = n.s.low & (d.s.low - 1);
-        if (d.s.low == 1)
-          return n.all;
-        sr = __builtin_ctzll(d.s.low);
-        q.s.high = n.s.high >> sr;
-        q.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr);
-        return q.all;
-      }
-      // K X
-      // ---
-      // 0 K
-      sr = 1 + n_udword_bits + __builtin_clzll(d.s.low) -
-           __builtin_clzll(n.s.high);
-      // 2 <= sr <= n_utword_bits - 1
-      // q.all = n.all << (n_utword_bits - sr);
-      // r.all = n.all >> sr;
-      if (sr == n_udword_bits) {
-        q.s.low = 0;
-        q.s.high = n.s.low;
-        r.s.high = 0;
-        r.s.low = n.s.high;
-      } else if (sr < n_udword_bits) /* 2 <= sr <= n_udword_bits - 1 */ {
-        q.s.low = 0;
-        q.s.high = n.s.low << (n_udword_bits - sr);
-        r.s.high = n.s.high >> sr;
-        r.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr);
-      } else /* n_udword_bits + 1 <= sr <= n_utword_bits - 1 */ {
-        q.s.low = n.s.low << (n_utword_bits - sr);
-        q.s.high = (n.s.high << (n_utword_bits - sr)) |
-                   (n.s.low >> (sr - n_udword_bits));
-        r.s.high = 0;
-        r.s.low = n.s.high >> (sr - n_udword_bits);
-      }
+  // When the divisor fits in 64 bits, we can use an optimized path.
+  if (divisor.s.high == 0) {
+    remainder.s.high = 0;
+    if (dividend.s.high < divisor.s.low) {
+      // The result fits in 64 bits.
+      quotient.s.low = udiv128by64to64(dividend.s.high, dividend.s.low,
+                                       divisor.s.low, &remainder.s.low);
+      quotient.s.high = 0;
     } else {
-      // K X
-      // ---
-      // K K
-      sr = __builtin_clzll(d.s.high) - __builtin_clzll(n.s.high);
-      // 0 <= sr <= n_udword_bits - 1 or sr large
-      if (sr > n_udword_bits - 1) {
-        if (rem)
-          *rem = n.all;
-        return 0;
-      }
-      ++sr;
-      // 1 <= sr <= n_udword_bits
-      // q.all = n.all << (n_utword_bits - sr);
-      // r.all = n.all >> sr;
-      q.s.low = 0;
-      if (sr == n_udword_bits) {
-        q.s.high = n.s.low;
-        r.s.high = 0;
-        r.s.low = n.s.high;
-      } else {
-        r.s.high = n.s.high >> sr;
-        r.s.low = (n.s.high << (n_udword_bits - sr)) | (n.s.low >> sr);
-        q.s.high = n.s.low << (n_udword_bits - sr);
-      }
+      // First, divide with the high part to get the remainder in dividend.s.high.
+      // After that dividend.s.high < divisor.s.low.
+      quotient.s.high = dividend.s.high / divisor.s.low;
+      dividend.s.high = dividend.s.high % divisor.s.low;
+      quotient.s.low = udiv128by64to64(dividend.s.high, dividend.s.low,
+                                       divisor.s.low, &remainder.s.low);
     }
+    if (rem)
+      *rem = remainder.all;
+    return quotient.all;
   }
-  // Not a special case
-  // q and r are initialized with:
-  // q.all = n.all << (n_utword_bits - sr);
-  // r.all = n.all >> sr;
-  // 1 <= sr <= n_utword_bits - 1
-  su_int carry = 0;
-  for (; sr > 0; --sr) {
-    // r:q = ((r:q)  << 1) | carry
-    r.s.high = (r.s.high << 1) | (r.s.low >> (n_udword_bits - 1));
-    r.s.low = (r.s.low << 1) | (q.s.high >> (n_udword_bits - 1));
-    q.s.high = (q.s.high << 1) | (q.s.low >> (n_udword_bits - 1));
-    q.s.low = (q.s.low << 1) | carry;
-    // carry = 0;
-    // if (r.all >= d.all)
+  // 0 <= shift <= 63.
+  si_int shift =
+      __builtin_clzll(divisor.s.high) - __builtin_clzll(dividend.s.high);
+  divisor.all <<= shift;
+  quotient.s.high = 0;
+  quotient.s.low = 0;
+  for (; shift >= 0; --shift) {
+    quotient.s.low <<= 1;
+    // Branch free version of.
+    // if (dividend.all >= divisor.all)
     // {
-    //     r.all -= d.all;
-    //      carry = 1;
+    //    dividend.all -= divisor.all;
+    //    carry = 1;
     // }
-    const ti_int s = (ti_int)(d.all - r.all - 1) >> (n_utword_bits - 1);
-    carry = s & 1;
-    r.all -= d.all & s;
+    const ti_int s =
+        (ti_int)(divisor.all - dividend.all - 1) >> (n_utword_bits - 1);
+    quotient.s.low |= s & 1;
+    dividend.all -= divisor.all & s;
+    divisor.all >>= 1;
   }
-  q.all = (q.all << 1) | carry;
   if (rem)
-    *rem = r.all;
-  return q.all;
+    *rem = dividend.all;
+  return quotient.all;
 }
 
 #endif // CRT_HAS_128BIT
diff --git a/compiler-rt/lib/profile/GCDAProfiling.c b/compiler-rt/lib/profile/GCDAProfiling.c
index d15540d7c18a8..57d8dec423cc0 100644
--- a/compiler-rt/lib/profile/GCDAProfiling.c
+++ b/compiler-rt/lib/profile/GCDAProfiling.c
@@ -628,8 +628,14 @@ void llvm_writeout_files(void) {
   }
 }
 
-COMPILER_RT_VISIBILITY
-void llvm_delete_writeout_function_list(void) {
+#ifndef _WIN32
+// __attribute__((destructor)) and destructors whose priorities are greater than
+// 100 run before this function and can thus be tracked. The priority is
+// compatible with GCC 7 onwards.
+__attribute__((destructor(100)))
+#endif
+static void llvm_writeout_and_clear(void) {
+  llvm_writeout_files();
   fn_list_remove(&writeout_fn_list);
 }
 
@@ -710,8 +716,9 @@ void llvm_gcov_init(fn_ptr wfn, fn_ptr ffn, fn_ptr rfn) {
     /* Make sure we write out the data and delete the data structures. */
     atexit(llvm_delete_reset_function_list);
     atexit(llvm_delete_flush_function_list);
-    atexit(llvm_delete_writeout_function_list);
-    atexit(llvm_writeout_files);
+#ifdef _WIN32
+    atexit(llvm_writeout_and_clear);
+#endif
   }
 }
 
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index 57f8b2d29442c..ea9c71ba88032 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -3092,6 +3092,34 @@ INTERCEPTOR(int, sendmmsg, int fd, struct __sanitizer_mmsghdr *msgvec,
 #define INIT_SENDMMSG
 #endif
 
+#if SANITIZER_INTERCEPT_SYSMSG
+INTERCEPTOR(int, msgsnd, int msqid, const void *msgp, SIZE_T msgsz,
+            int msgflg) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, msgsnd, msqid, msgp, msgsz, msgflg);
+  if (msgp)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, msgp, sizeof(long) + msgsz);
+  int res = REAL(msgsnd)(msqid, msgp, msgsz, msgflg);
+  return res;
+}
+
+INTERCEPTOR(SSIZE_T, msgrcv, int msqid, void *msgp, SIZE_T msgsz,
+            long msgtyp, int msgflg) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, msgrcv, msqid, msgp, msgsz, msgtyp, msgflg);
+  SSIZE_T len = REAL(msgrcv)(msqid, msgp, msgsz, msgtyp, msgflg);
+  if (len != -1)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, msgp, sizeof(long) + len);
+  return len;
+}
+
+#define INIT_SYSMSG                  \
+  COMMON_INTERCEPT_FUNCTION(msgsnd); \
+  COMMON_INTERCEPT_FUNCTION(msgrcv);
+#else
+#define INIT_SYSMSG
+#endif
+
 #if SANITIZER_INTERCEPT_GETPEERNAME
 INTERCEPTOR(int, getpeername, int sockfd, void *addr, unsigned *addrlen) {
   void *ctx;
@@ -7271,23 +7299,26 @@ INTERCEPTOR(int, setttyentpath, char *path) {
 #endif
 
 #if SANITIZER_INTERCEPT_PROTOENT
-INTERCEPTOR(struct __sanitizer_protoent *, getprotoent) {
-  void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, getprotoent);
-  struct __sanitizer_protoent *p = REAL(getprotoent)();
-  if (p) {
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
+static void write_protoent(void *ctx, struct __sanitizer_protoent *p) {
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
 
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_name, REAL(strlen)(p->p_name) + 1);
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_name, REAL(strlen)(p->p_name) + 1);
 
-    SIZE_T pp_size = 1; // One handles the trailing \0
+  SIZE_T pp_size = 1; // One handles the trailing \0
 
-    for (char **pp = p->p_aliases; *pp; ++pp, ++pp_size)
-       COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *pp, REAL(strlen)(*pp) + 1);
+  for (char **pp = p->p_aliases; *pp; ++pp, ++pp_size)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *pp, REAL(strlen)(*pp) + 1);
 
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases,
-                                   pp_size * sizeof(char **));
-  }
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases,
+                                  pp_size * sizeof(char **));
+}
+
+INTERCEPTOR(struct __sanitizer_protoent *, getprotoent) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getprotoent);
+  struct __sanitizer_protoent *p = REAL(getprotoent)();
+  if (p)
+    write_protoent(ctx, p);
   return p;
 }
 
@@ -7297,19 +7328,8 @@ INTERCEPTOR(struct __sanitizer_protoent *, getprotobyname, const char *name) {
   if (name)
     COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
   struct __sanitizer_protoent *p = REAL(getprotobyname)(name);
-  if (p) {
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
-
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_name, REAL(strlen)(p->p_name) + 1);
-
-    SIZE_T pp_size = 1; // One handles the trailing \0
-
-    for (char **pp = p->p_aliases; *pp; ++pp, ++pp_size)
-       COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *pp, REAL(strlen)(*pp) + 1);
-
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases,
-                                   pp_size * sizeof(char **));
-  }
+  if (p)
+    write_protoent(ctx, p);
   return p;
 }
 
@@ -7317,19 +7337,8 @@ INTERCEPTOR(struct __sanitizer_protoent *, getprotobynumber, int proto) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, getprotobynumber, proto);
   struct __sanitizer_protoent *p = REAL(getprotobynumber)(proto);
-  if (p) {
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
-
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_name, REAL(strlen)(p->p_name) + 1);
-
-    SIZE_T pp_size = 1; // One handles the trailing \0
-
-    for (char **pp = p->p_aliases; *pp; ++pp, ++pp_size)
-       COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *pp, REAL(strlen)(*pp) + 1);
-
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases,
-                                   pp_size * sizeof(char **));
-  }
+  if (p)
+    write_protoent(ctx, p);
   return p;
 }
 #define INIT_PROTOENT \
@@ -7340,6 +7349,58 @@ INTERCEPTOR(struct __sanitizer_protoent *, getprotobynumber, int proto) {
 #define INIT_PROTOENT
 #endif
 
+#if SANITIZER_INTERCEPT_PROTOENT_R
+INTERCEPTOR(int, getprotoent_r, struct __sanitizer_protoent *result_buf,
+            char *buf, SIZE_T buflen, struct __sanitizer_protoent **result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getprotoent_r, result_buf, buf, buflen,
+                           result);
+  int res = REAL(getprotoent_r)(result_buf, buf, buflen, result);
+
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof *result);
+  if (!res && *result)
+    write_protoent(ctx, *result);
+  return res;
+}
+
+INTERCEPTOR(int, getprotobyname_r, const char *name,
+            struct __sanitizer_protoent *result_buf, char *buf, SIZE_T buflen,
+            struct __sanitizer_protoent **result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getprotobyname_r, name, result_buf, buf,
+                           buflen, result);
+  if (name)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  int res = REAL(getprotobyname_r)(name, result_buf, buf, buflen, result);
+
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof *result);
+  if (!res && *result)
+    write_protoent(ctx, *result);
+  return res;
+}
+
+INTERCEPTOR(int, getprotobynumber_r, int num,
+            struct __sanitizer_protoent *result_buf, char *buf,
+            SIZE_T buflen, struct __sanitizer_protoent **result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getprotobynumber_r, num, result_buf, buf,
+                           buflen, result);
+  int res = REAL(getprotobynumber_r)(num, result_buf, buf, buflen, result);
+
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof *result);
+  if (!res && *result)
+    write_protoent(ctx, *result);
+  return res;
+}
+
+#define INIT_PROTOENT_R \
+  COMMON_INTERCEPT_FUNCTION(getprotoent_r); \
+  COMMON_INTERCEPT_FUNCTION(getprotobyname_r); \
+  COMMON_INTERCEPT_FUNCTION(getprotobynumber_r);
+#else
+#define INIT_PROTOENT_R
+#endif
+
 #if SANITIZER_INTERCEPT_NETENT
 INTERCEPTOR(struct __sanitizer_netent *, getnetent) {
   void *ctx;
@@ -9879,6 +9940,7 @@ static void InitializeCommonInterceptors() {
   INIT_SENDMSG;
   INIT_RECVMMSG;
   INIT_SENDMMSG;
+  INIT_SYSMSG;
   INIT_GETPEERNAME;
   INIT_IOCTL;
   INIT_INET_ATON;
@@ -10042,6 +10104,7 @@ static void InitializeCommonInterceptors() {
   INIT_STRMODE;
   INIT_TTYENT;
   INIT_PROTOENT;
+  INIT_PROTOENT_R;
   INIT_NETENT;
   INIT_GETMNTINFO;
   INIT_MI_VECTOR_HASH;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
index dffe2c9c47376..91caa6a35693b 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
@@ -2210,7 +2210,7 @@ void CheckNoDeepBind(const char *filename, int flag) {
   if (flag & RTLD_DEEPBIND) {
     Report(
         "You are trying to dlopen a %s shared library with RTLD_DEEPBIND flag"
-        " which is incompatibe with sanitizer runtime "
+        " which is incompatible with sanitizer runtime "
         "(see https://github.com/google/sanitizers/issues/611 for details"
         "). If you want to run %s library under sanitizers please remove "
         "RTLD_DEEPBIND from dlopen flags.\n",
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index c22e7517fc6f8..7a3dfbcc27607 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -606,12 +606,22 @@ HandleSignalMode GetHandleSignalMode(int signum) {
   return result;
 }
 
+// This corresponds to Triple::getMacOSXVersion() in the Clang driver.
 static MacosVersion GetMacosAlignedVersionInternal() {
   u16 kernel_major = GetDarwinKernelVersion().major;
-  const u16 version_offset = 4;
-  CHECK_GE(kernel_major, version_offset);
-  u16 macos_major = kernel_major - version_offset;
-  return MacosVersion(10, macos_major);
+  // Darwin 0-3  -> unsupported
+  // Darwin 4-19 -> macOS 10.x
+  // Darwin 20+  -> macOS 11+
+  CHECK_GE(kernel_major, 4);
+  u16 major, minor;
+  if (kernel_major < 20) {
+    major = 10;
+    minor = kernel_major - 4;
+  } else {
+    major = 11 + kernel_major - 20;
+    minor = 0;
+  }
+  return MacosVersion(major, minor);
 }
 
 static_assert(sizeof(MacosVersion) == sizeof(atomic_uint32_t::Type),
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.h b/compiler-rt/lib/sanitizer_common/sanitizer_mac.h
index 922307a9b9779..90ecff4815c2e 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.h
@@ -47,9 +47,7 @@ struct VersionBase {
 };
 
 struct MacosVersion : VersionBase<MacosVersion> {
-  MacosVersion(u16 ten, u16 major) : VersionBase(ten, major) {
-    CHECK_EQ(ten, 10);
-  }
+  MacosVersion(u16 major, u16 minor) : VersionBase(major, minor) {}
 };
 
 struct DarwinKernelVersion : VersionBase<DarwinKernelVersion> {
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform.h
index c68bfa2587558..f0b1e04d1dd68 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform.h
@@ -132,6 +132,12 @@
 # define SANITIZER_X32 0
 #endif
 
+#if defined(__i386__) || defined(_M_IX86)
+# define SANITIZER_I386 1
+#else
+# define SANITIZER_I386 0
+#endif
+
 #if defined(__mips__)
 # define SANITIZER_MIPS 1
 # if defined(__mips64)
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
index 9dd6d285f5940..2d48e9d0ae1ad 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
@@ -270,6 +270,7 @@
 #define SANITIZER_INTERCEPT_SENDMSG SI_POSIX
 #define SANITIZER_INTERCEPT_RECVMMSG SI_LINUX
 #define SANITIZER_INTERCEPT_SENDMMSG SI_LINUX
+#define SANITIZER_INTERCEPT_SYSMSG SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_GETPEERNAME SI_POSIX
 #define SANITIZER_INTERCEPT_IOCTL SI_POSIX
 #define SANITIZER_INTERCEPT_INET_ATON SI_POSIX
@@ -341,7 +342,7 @@
 #define SANITIZER_INTERCEPT_STATFS \
   (SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_STATFS64 \
-  ((SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID)
+  (((SI_MAC && !TARGET_CPU_ARM64) && !SI_IOS) || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_STATVFS \
   (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_STATVFS64 SI_LINUX_NOT_ANDROID
@@ -544,7 +545,8 @@
 #define SANITIZER_INTERCEPT_FGETLN (SI_NETBSD || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_STRMODE (SI_NETBSD || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_TTYENT SI_NETBSD
-#define SANITIZER_INTERCEPT_PROTOENT SI_NETBSD
+#define SANITIZER_INTERCEPT_PROTOENT (SI_NETBSD || SI_LINUX)
+#define SANITIZER_INTERCEPT_PROTOENT_R (SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_NETENT SI_NETBSD
 #define SANITIZER_INTERCEPT_SETVBUF (SI_NETBSD || SI_FREEBSD || \
   SI_LINUX || SI_MAC)
@@ -596,7 +598,10 @@
 #define SANITIZER_INTERCEPT_QSORT \
   (SI_POSIX && !SI_IOSSIM && !SI_WATCHOS && !SI_TVOS && !SI_ANDROID)
 #define SANITIZER_INTERCEPT_QSORT_R (SI_LINUX && !SI_ANDROID)
-#define SANITIZER_INTERCEPT_SIGALTSTACK SI_POSIX
+// sigaltstack on i386 macOS cannot be intercepted due to setjmp()
+// calling it and assuming that it does not clobber registers.
+#define SANITIZER_INTERCEPT_SIGALTSTACK \
+  (SI_POSIX && !(SANITIZER_MAC && SANITIZER_I386))
 #define SANITIZER_INTERCEPT_UNAME (SI_POSIX && !SI_FREEBSD)
 #define SANITIZER_INTERCEPT___XUNAME SI_FREEBSD
 
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h
index d80280d9bf8c8..ae54a8cf105ee 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h
@@ -129,12 +129,6 @@ struct __sanitizer_shmid_ds {
   void *_shm_internal;
 };
 
-struct __sanitizer_protoent {
-  char *p_name;
-  char **p_aliases;
-  int p_proto;
-};
-
 struct __sanitizer_netent {
   char *n_name;
   char **n_aliases;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp
index e71515f12e94e..c052aa2bc9509 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp
@@ -170,9 +170,9 @@ typedef struct user_fpregs elf_fpregset_t;
 namespace __sanitizer {
   unsigned struct_utsname_sz = sizeof(struct utsname);
   unsigned struct_stat_sz = sizeof(struct stat);
-#if !SANITIZER_IOS
+#if !SANITIZER_IOS && !(SANITIZER_MAC && TARGET_CPU_ARM64)
   unsigned struct_stat64_sz = sizeof(struct stat64);
-#endif // !SANITIZER_IOS
+#endif // !SANITIZER_IOS && !(SANITIZER_MAC && TARGET_CPU_ARM64)
   unsigned struct_rusage_sz = sizeof(struct rusage);
   unsigned struct_tm_sz = sizeof(struct tm);
   unsigned struct_passwd_sz = sizeof(struct passwd);
@@ -197,9 +197,9 @@ namespace __sanitizer {
   unsigned struct_regex_sz = sizeof(regex_t);
   unsigned struct_regmatch_sz = sizeof(regmatch_t);
 
-#if SANITIZER_MAC && !SANITIZER_IOS
+#if (SANITIZER_MAC && !TARGET_CPU_ARM64) && !SANITIZER_IOS
   unsigned struct_statfs64_sz = sizeof(struct statfs64);
-#endif // SANITIZER_MAC && !SANITIZER_IOS
+#endif // (SANITIZER_MAC && !TARGET_CPU_ARM64) && !SANITIZER_IOS
 
 #if !SANITIZER_ANDROID
   unsigned struct_fstab_sz = sizeof(struct fstab);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
index f6c8a1450a932..658b0abaece82 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
@@ -704,6 +704,12 @@ struct __sanitizer_dl_phdr_info {
 extern unsigned struct_ElfW_Phdr_sz;
 #endif
 
+struct __sanitizer_protoent {
+  char *p_name;
+  char **p_aliases;
+  int p_proto;
+};
+
 struct __sanitizer_addrinfo {
   int ai_flags;
   int ai_family;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp
index d7b931bc23795..3c379a8480250 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp
@@ -78,13 +78,6 @@ static void InitializeSwiftDemangler() {
 // Attempts to demangle a Swift name. The demangler will return nullptr if a
 // non-Swift name is passed in.
 const char *DemangleSwift(const char *name) {
-  if (!name) return nullptr;
-
-  // Check if we are dealing with a Swift mangled name first.
-  if (name[0] != '_' || name[1] != 'T') {
-    return nullptr;
-  }
-
   if (swift_demangle_f)
     return swift_demangle_f(name, internal_strlen(name), 0, 0, 0);
 
@@ -321,9 +314,10 @@ class Addr2LinePool : public SymbolizerTool {
 
 #if SANITIZER_SUPPORTS_WEAK_HOOKS
 extern "C" {
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-bool __sanitizer_symbolize_code(const char *ModuleName, u64 ModuleOffset,
-                                char *Buffer, int MaxLength);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
+__sanitizer_symbolize_code(const char *ModuleName, u64 ModuleOffset,
+                           char *Buffer, int MaxLength,
+                           bool SymbolizeInlineFrames);
 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
 bool __sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset,
                                 char *Buffer, int MaxLength);
@@ -346,7 +340,8 @@ class InternalSymbolizer : public SymbolizerTool {
 
   bool SymbolizePC(uptr addr, SymbolizedStack *stack) override {
     bool result = __sanitizer_symbolize_code(
-        stack->info.module, stack->info.module_offset, buffer_, kBufferSize);
+        stack->info.module, stack->info.module_offset, buffer_, kBufferSize,
+        common_flags()->symbolize_inline_frames);
     if (result) ParseSymbolizePCOutput(buffer_, stack);
     return result;
   }
diff --git a/compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp b/compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp
index ba285bc1e8844..4902be0bf51ec 100644
--- a/compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp
+++ b/compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <stdio.h>
+
 #include <string>
 
 #include "llvm/DebugInfo/Symbolize/DIPrinter.h"
@@ -32,17 +33,25 @@ extern "C" {
 typedef uint64_t u64;
 
 bool __sanitizer_symbolize_code(const char *ModuleName, uint64_t ModuleOffset,
-                                char *Buffer, int MaxLength) {
+                                char *Buffer, int MaxLength,
+                                bool SymbolizeInlineFrames) {
   std::string Result;
   {
     llvm::raw_string_ostream OS(Result);
     llvm::symbolize::DIPrinter Printer(OS);
     // TODO: it is neccessary to set proper SectionIndex here.
     // object::SectionedAddress::UndefSection works for only absolute addresses.
-    auto ResOrErr = getDefaultSymbolizer()->symbolizeInlinedCode(
-        ModuleName,
-        {ModuleOffset, llvm::object::SectionedAddress::UndefSection});
-    Printer << (ResOrErr ? ResOrErr.get() : llvm::DIInliningInfo());
+    if (SymbolizeInlineFrames) {
+      auto ResOrErr = getDefaultSymbolizer()->symbolizeInlinedCode(
+          ModuleName,
+          {ModuleOffset, llvm::object::SectionedAddress::UndefSection});
+      Printer << (ResOrErr ? ResOrErr.get() : llvm::DIInliningInfo());
+    } else {
+      auto ResOrErr = getDefaultSymbolizer()->symbolizeCode(
+          ModuleName,
+          {ModuleOffset, llvm::object::SectionedAddress::UndefSection});
+      Printer << (ResOrErr ? ResOrErr.get() : llvm::DILineInfo());
+    }
   }
   return __sanitizer::internal_snprintf(Buffer, MaxLength, "%s",
                                         Result.c_str()) < MaxLength;
diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_mac_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_mac_test.cpp
index b327ba96e2239..c8658ea55d034 100644
--- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_mac_test.cpp
+++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_mac_test.cpp
@@ -24,8 +24,12 @@ namespace __sanitizer {
 
 TEST(SanitizerMac, GetMacosAlignedVersion) {
   MacosVersion vers = GetMacosAlignedVersion();
-  EXPECT_EQ(vers.major, 10);
-  EXPECT_EQ(vers.minor, GetDarwinKernelVersion().major - 4);
+  u16 kernel_major = GetDarwinKernelVersion().major;
+  bool macos_11 = (kernel_major >= 20);
+  u16 expected_major = macos_11 ? (kernel_major - 9) : 10;
+  u16 expected_minor = macos_11 ? 0 : (kernel_major - 4);
+  EXPECT_EQ(vers.major, expected_major);
+  EXPECT_EQ(vers.minor, expected_minor);
 }
 
 void ParseVersion(const char *vers, u16 *major, u16 *minor);
diff --git a/compiler-rt/test/builtins/Unit/clzdi2_test.c b/compiler-rt/test/builtins/Unit/clzdi2_test.c
index 3cbfb4b68b0dc..962f1dff45113 100644
--- a/compiler-rt/test/builtins/Unit/clzdi2_test.c
+++ b/compiler-rt/test/builtins/Unit/clzdi2_test.c
@@ -8,11 +8,11 @@
 
 // Precondition: a != 0
 
-COMPILER_RT_ABI si_int __clzdi2(di_int a);
+COMPILER_RT_ABI int __clzdi2(di_int a);
 
-int test__clzdi2(di_int a, si_int expected)
+int test__clzdi2(di_int a, int expected)
 {
-    si_int x = __clzdi2(a);
+    int x = __clzdi2(a);
     if (x != expected)
         printf("error in __clzdi2(0x%llX) = %d, expected %d\n", a, x, expected);
     return x != expected;
diff --git a/compiler-rt/test/builtins/Unit/clzsi2_test.c b/compiler-rt/test/builtins/Unit/clzsi2_test.c
index d12c59cf56a54..57562c497c5d0 100644
--- a/compiler-rt/test/builtins/Unit/clzsi2_test.c
+++ b/compiler-rt/test/builtins/Unit/clzsi2_test.c
@@ -8,11 +8,11 @@
 
 // Precondition: a != 0
 
-COMPILER_RT_ABI si_int __clzsi2(si_int a);
+COMPILER_RT_ABI int __clzsi2(si_int a);
 
-int test__clzsi2(si_int a, si_int expected)
+int test__clzsi2(si_int a, int expected)
 {
-    si_int x = __clzsi2(a);
+    int x = __clzsi2(a);
     if (x != expected)
         printf("error in __clzsi2(0x%X) = %d, expected %d\n", a, x, expected);
     return x != expected;
diff --git a/compiler-rt/test/builtins/Unit/clzti2_test.c b/compiler-rt/test/builtins/Unit/clzti2_test.c
index 3dbf907540210..13e0e698422f2 100644
--- a/compiler-rt/test/builtins/Unit/clzti2_test.c
+++ b/compiler-rt/test/builtins/Unit/clzti2_test.c
@@ -11,11 +11,11 @@
 
 // Precondition: a != 0
 
-COMPILER_RT_ABI si_int __clzti2(ti_int a);
+COMPILER_RT_ABI int __clzti2(ti_int a);
 
-int test__clzti2(ti_int a, si_int expected)
+int test__clzti2(ti_int a, int expected)
 {
-    si_int x = __clzti2(a);
+    int x = __clzti2(a);
     if (x != expected)
     {
         twords at;
diff --git a/compiler-rt/test/builtins/Unit/ctzsi2_test.c b/compiler-rt/test/builtins/Unit/ctzsi2_test.c
index 96db33983cc48..db6d4ce6a1621 100644
--- a/compiler-rt/test/builtins/Unit/ctzsi2_test.c
+++ b/compiler-rt/test/builtins/Unit/ctzsi2_test.c
@@ -8,11 +8,11 @@
 
 // Precondition: a != 0
 
-COMPILER_RT_ABI si_int __ctzsi2(si_int a);
+COMPILER_RT_ABI int __ctzsi2(si_int a);
 
-int test__ctzsi2(si_int a, si_int expected)
+int test__ctzsi2(si_int a, int expected)
 {
-    si_int x = __ctzsi2(a);
+    int x = __ctzsi2(a);
     if (x != expected)
         printf("error in __ctzsi2(0x%X) = %d, expected %d\n", a, x, expected);
     return x != expected;
diff --git a/compiler-rt/test/builtins/Unit/ctzti2_test.c b/compiler-rt/test/builtins/Unit/ctzti2_test.c
index a19de13c9a074..054776ef71cff 100644
--- a/compiler-rt/test/builtins/Unit/ctzti2_test.c
+++ b/compiler-rt/test/builtins/Unit/ctzti2_test.c
@@ -11,11 +11,11 @@
 
 // Precondition: a != 0
 
-COMPILER_RT_ABI si_int __ctzti2(ti_int a);
+COMPILER_RT_ABI int __ctzti2(ti_int a);
 
-int test__ctzti2(ti_int a, si_int expected)
+int test__ctzti2(ti_int a, int expected)
 {
-    si_int x = __ctzti2(a);
+    int x = __ctzti2(a);
     if (x != expected)
     {
         twords at;
diff --git a/compiler-rt/test/builtins/Unit/ffsti2_test.c b/compiler-rt/test/builtins/Unit/ffsti2_test.c
index 3ec7f02bca453..446b16fe3871c 100644
--- a/compiler-rt/test/builtins/Unit/ffsti2_test.c
+++ b/compiler-rt/test/builtins/Unit/ffsti2_test.c
@@ -10,11 +10,11 @@
 // Returns: the index of the least significant 1-bit in a, or
 // the value zero if a is zero. The least significant bit is index one.
 
-COMPILER_RT_ABI si_int __ffsti2(ti_int a);
+COMPILER_RT_ABI int __ffsti2(ti_int a);
 
-int test__ffsti2(ti_int a, si_int expected)
+int test__ffsti2(ti_int a, int expected)
 {
-    si_int x = __ffsti2(a);
+    int x = __ffsti2(a);
     if (x != expected)
     {
         twords at;
diff --git a/compiler-rt/test/builtins/Unit/lit.cfg.py b/compiler-rt/test/builtins/Unit/lit.cfg.py
index 8fdb1a216ee26..c8888078be507 100644
--- a/compiler-rt/test/builtins/Unit/lit.cfg.py
+++ b/compiler-rt/test/builtins/Unit/lit.cfg.py
@@ -5,6 +5,17 @@
 
 import lit.formats
 
+# Choose between lit's internal shell pipeline runner and a real shell.  If
+# LIT_USE_INTERNAL_SHELL is in the environment, we use that as an override.
+use_lit_shell = os.environ.get("LIT_USE_INTERNAL_SHELL")
+if use_lit_shell:
+    # 0 is external, "" is default, and everything else is internal.
+    execute_external = (use_lit_shell == "0")
+else:
+    # Otherwise we default to internal on Windows and external elsewhere, as
+    # bash on Windows is usually very slow.
+    execute_external = (not sys.platform in ['win32'])
+
 def get_required_attr(config, attr_name):
   attr_value = getattr(config, attr_name, None)
   if attr_value == None:
@@ -35,10 +46,16 @@ def get_required_attr(config, attr_name):
 else:
   base_lib = os.path.join(config.compiler_rt_libdir, "libclang_rt.builtins%s.a"
                           % config.target_suffix)
+  if sys.platform in ['win32'] and execute_external:
+    # Don't pass dosish path separator to msys bash.exe.
+    base_lib = base_lib.replace('\\', '/')
   config.substitutions.append( ("%librt ", base_lib + ' -lc -lm ') )
 
 builtins_source_dir = os.path.join(
   get_required_attr(config, "compiler_rt_src_root"), "lib", "builtins")
+if sys.platform in ['win32'] and execute_external:
+  # Don't pass dosish path separator to msys bash.exe.
+  builtins_source_dir = builtins_source_dir.replace('\\', '/')
 builtins_lit_source_dir = get_required_attr(config, "builtins_lit_source_dir")
 
 extra_link_flags = ["-nodefaultlibs"]
diff --git a/compiler-rt/test/builtins/Unit/paritydi2_test.c b/compiler-rt/test/builtins/Unit/paritydi2_test.c
index 13218a9beeb62..2ea1ddc0977d8 100644
--- a/compiler-rt/test/builtins/Unit/paritydi2_test.c
+++ b/compiler-rt/test/builtins/Unit/paritydi2_test.c
@@ -7,7 +7,7 @@
 
 // Returns: 1 if number of bits is odd else returns 0
 
-COMPILER_RT_ABI si_int __paritydi2(di_int a);
+COMPILER_RT_ABI int __paritydi2(di_int a);
 
 int naive_parity(di_int a)
 {
diff --git a/compiler-rt/test/builtins/Unit/paritysi2_test.c b/compiler-rt/test/builtins/Unit/paritysi2_test.c
index 7aae131b7b07b..2bed010d9d4ad 100644
--- a/compiler-rt/test/builtins/Unit/paritysi2_test.c
+++ b/compiler-rt/test/builtins/Unit/paritysi2_test.c
@@ -7,7 +7,7 @@
 
 // Returns: 1 if number of bits is odd else returns 0
 
-COMPILER_RT_ABI si_int __paritysi2(si_int a);
+COMPILER_RT_ABI int __paritysi2(si_int a);
 
 int naive_parity(si_int a)
 {
diff --git a/compiler-rt/test/builtins/Unit/parityti2_test.c b/compiler-rt/test/builtins/Unit/parityti2_test.c
index 6644ee0ef0fa0..ec4cf45bf30d4 100644
--- a/compiler-rt/test/builtins/Unit/parityti2_test.c
+++ b/compiler-rt/test/builtins/Unit/parityti2_test.c
@@ -10,7 +10,7 @@
 
 // Returns: 1 if number of bits is odd else returns 0
 
-COMPILER_RT_ABI si_int __parityti2(ti_int a);
+COMPILER_RT_ABI int __parityti2(ti_int a);
 
 int naive_parity(ti_int a)
 {
diff --git a/compiler-rt/test/builtins/Unit/popcountsi2_test.c b/compiler-rt/test/builtins/Unit/popcountsi2_test.c
index 731e6212c06fc..7689267514744 100644
--- a/compiler-rt/test/builtins/Unit/popcountsi2_test.c
+++ b/compiler-rt/test/builtins/Unit/popcountsi2_test.c
@@ -7,7 +7,7 @@
 
 // Returns: count of 1 bits
 
-COMPILER_RT_ABI si_int __popcountsi2(si_int a);
+COMPILER_RT_ABI int __popcountsi2(si_int a);
 
 int naive_popcount(si_int a)
 {
diff --git a/compiler-rt/test/builtins/Unit/popcountti2_test.c b/compiler-rt/test/builtins/Unit/popcountti2_test.c
index 53edd8c43b933..c3919dccdcb8c 100644
--- a/compiler-rt/test/builtins/Unit/popcountti2_test.c
+++ b/compiler-rt/test/builtins/Unit/popcountti2_test.c
@@ -10,7 +10,7 @@
 
 // Returns: count of 1 bits
 
-COMPILER_RT_ABI si_int __popcountti2(ti_int a);
+COMPILER_RT_ABI int __popcountti2(ti_int a);
 
 int naive_popcount(ti_int a)
 {
diff --git a/compiler-rt/test/builtins/Unit/powidf2_test.c b/compiler-rt/test/builtins/Unit/powidf2_test.c
index 0c4e2138734bd..aa16b8542dae3 100644
--- a/compiler-rt/test/builtins/Unit/powidf2_test.c
+++ b/compiler-rt/test/builtins/Unit/powidf2_test.c
@@ -7,9 +7,9 @@
 
 // Returns: a ^ b
 
-COMPILER_RT_ABI double __powidf2(double a, si_int b);
+COMPILER_RT_ABI double __powidf2(double a, int b);
 
-int test__powidf2(double a, si_int b, double expected)
+int test__powidf2(double a, int b, double expected)
 {
     double x = __powidf2(a, b);
     int correct = (x == expected) && (signbit(x) == signbit(expected));
@@ -51,9 +51,9 @@ int main()
         return 1;
     if (test__powidf2(0, 4, 0))
         return 1;
-    if (test__powidf2(0, 0x7FFFFFFE, 0))
+    if (test__powidf2(0, INT_MAX - 1, 0))
         return 1;
-    if (test__powidf2(0, 0x7FFFFFFF, 0))
+    if (test__powidf2(0, INT_MAX, 0))
         return 1;
 
     if (test__powidf2(-0., 1, -0.))
@@ -64,9 +64,9 @@ int main()
         return 1;
     if (test__powidf2(-0., 4, 0))
         return 1;
-    if (test__powidf2(-0., 0x7FFFFFFE, 0))
+    if (test__powidf2(-0., INT_MAX - 1, 0))
         return 1;
-    if (test__powidf2(-0., 0x7FFFFFFF, -0.))
+    if (test__powidf2(-0., INT_MAX, -0.))
         return 1;
 
     if (test__powidf2(1, 1, 1))
@@ -77,9 +77,9 @@ int main()
         return 1;
     if (test__powidf2(1, 4, 1))
         return 1;
-    if (test__powidf2(1, 0x7FFFFFFE, 1))
+    if (test__powidf2(1, INT_MAX - 1, 1))
         return 1;
-    if (test__powidf2(1, 0x7FFFFFFF, 1))
+    if (test__powidf2(1, INT_MAX, 1))
         return 1;
 
     if (test__powidf2(INFINITY, 1, INFINITY))
@@ -90,9 +90,9 @@ int main()
         return 1;
     if (test__powidf2(INFINITY, 4, INFINITY))
         return 1;
-    if (test__powidf2(INFINITY, 0x7FFFFFFE, INFINITY))
+    if (test__powidf2(INFINITY, INT_MAX - 1, INFINITY))
         return 1;
-    if (test__powidf2(INFINITY, 0x7FFFFFFF, INFINITY))
+    if (test__powidf2(INFINITY, INT_MAX, INFINITY))
         return 1;
 
     if (test__powidf2(-INFINITY, 1, -INFINITY))
@@ -103,9 +103,9 @@ int main()
         return 1;
     if (test__powidf2(-INFINITY, 4, INFINITY))
         return 1;
-    if (test__powidf2(-INFINITY, 0x7FFFFFFE, INFINITY))
+    if (test__powidf2(-INFINITY, INT_MAX - 1, INFINITY))
         return 1;
-    if (test__powidf2(-INFINITY, 0x7FFFFFFF, -INFINITY))
+    if (test__powidf2(-INFINITY, INT_MAX, -INFINITY))
         return 1;
 
     if (test__powidf2(0, -1, INFINITY))
@@ -116,11 +116,11 @@ int main()
         return 1;
     if (test__powidf2(0, -4, INFINITY))
         return 1;
-    if (test__powidf2(0, 0x80000002, INFINITY))
+    if (test__powidf2(0, INT_MIN + 2, INFINITY))
         return 1;
-    if (test__powidf2(0, 0x80000001, INFINITY))
+    if (test__powidf2(0, INT_MIN + 1, INFINITY))
         return 1;
-    if (test__powidf2(0, 0x80000000, INFINITY))
+    if (test__powidf2(0, INT_MIN, INFINITY))
         return 1;
 
     if (test__powidf2(-0., -1, -INFINITY))
@@ -131,11 +131,11 @@ int main()
         return 1;
     if (test__powidf2(-0., -4, INFINITY))
         return 1;
-    if (test__powidf2(-0., 0x80000002, INFINITY))
+    if (test__powidf2(-0., INT_MIN + 2, INFINITY))
         return 1;
-    if (test__powidf2(-0., 0x80000001, -INFINITY))
+    if (test__powidf2(-0., INT_MIN + 1, -INFINITY))
         return 1;
-    if (test__powidf2(-0., 0x80000000, INFINITY))
+    if (test__powidf2(-0., INT_MIN, INFINITY))
         return 1;
 
     if (test__powidf2(1, -1, 1))
@@ -146,11 +146,11 @@ int main()
         return 1;
     if (test__powidf2(1, -4, 1))
         return 1;
-    if (test__powidf2(1, 0x80000002, 1))
+    if (test__powidf2(1, INT_MIN + 2, 1))
         return 1;
-    if (test__powidf2(1, 0x80000001, 1))
+    if (test__powidf2(1, INT_MIN + 1, 1))
         return 1;
-    if (test__powidf2(1, 0x80000000, 1))
+    if (test__powidf2(1, INT_MIN, 1))
         return 1;
 
     if (test__powidf2(INFINITY, -1, 0))
@@ -161,11 +161,11 @@ int main()
         return 1;
     if (test__powidf2(INFINITY, -4, 0))
         return 1;
-    if (test__powidf2(INFINITY, 0x80000002, 0))
+    if (test__powidf2(INFINITY, INT_MIN + 2, 0))
         return 1;
-    if (test__powidf2(INFINITY, 0x80000001, 0))
+    if (test__powidf2(INFINITY, INT_MIN + 1, 0))
         return 1;
-    if (test__powidf2(INFINITY, 0x80000000, 0))
+    if (test__powidf2(INFINITY, INT_MIN, 0))
         return 1;
 
     if (test__powidf2(-INFINITY, -1, -0.))
@@ -176,11 +176,11 @@ int main()
         return 1;
     if (test__powidf2(-INFINITY, -4, 0))
         return 1;
-    if (test__powidf2(-INFINITY, 0x80000002, 0))
+    if (test__powidf2(-INFINITY, INT_MIN + 2, 0))
         return 1;
-    if (test__powidf2(-INFINITY, 0x80000001, -0.))
+    if (test__powidf2(-INFINITY, INT_MIN + 1, -0.))
         return 1;
-    if (test__powidf2(-INFINITY, 0x80000000, 0))
+    if (test__powidf2(-INFINITY, INT_MIN, 0))
         return 1;
 
     if (test__powidf2(2, 10, 1024.))
diff --git a/compiler-rt/test/builtins/Unit/powisf2_test.c b/compiler-rt/test/builtins/Unit/powisf2_test.c
index e86901d574ad3..6da42d908d048 100644
--- a/compiler-rt/test/builtins/Unit/powisf2_test.c
+++ b/compiler-rt/test/builtins/Unit/powisf2_test.c
@@ -7,9 +7,9 @@
 
 // Returns: a ^ b
 
-COMPILER_RT_ABI float __powisf2(float a, si_int b);
+COMPILER_RT_ABI float __powisf2(float a, int b);
 
-int test__powisf2(float a, si_int b, float expected)
+int test__powisf2(float a, int b, float expected)
 {
     float x = __powisf2(a, b);
     int correct = (x == expected) && (signbit(x) == signbit(expected));
@@ -51,9 +51,9 @@ int main()
         return 1;
     if (test__powisf2(0, 4, 0))
         return 1;
-    if (test__powisf2(0, 0x7FFFFFFE, 0))
+    if (test__powisf2(0, INT_MAX -1, 0))
         return 1;
-    if (test__powisf2(0, 0x7FFFFFFF, 0))
+    if (test__powisf2(0, INT_MAX, 0))
         return 1;
 
     if (test__powisf2(-0., 1, -0.))
@@ -64,9 +64,9 @@ int main()
         return 1;
     if (test__powisf2(-0., 4, 0))
         return 1;
-    if (test__powisf2(-0., 0x7FFFFFFE, 0))
+    if (test__powisf2(-0., INT_MAX - 1, 0))
         return 1;
-    if (test__powisf2(-0., 0x7FFFFFFF, -0.))
+    if (test__powisf2(-0., INT_MAX, -0.))
         return 1;
 
     if (test__powisf2(1, 1, 1))
@@ -77,9 +77,9 @@ int main()
         return 1;
     if (test__powisf2(1, 4, 1))
         return 1;
-    if (test__powisf2(1, 0x7FFFFFFE, 1))
+    if (test__powisf2(1, INT_MAX - 1, 1))
         return 1;
-    if (test__powisf2(1, 0x7FFFFFFF, 1))
+    if (test__powisf2(1, INT_MAX, 1))
         return 1;
 
     if (test__powisf2(INFINITY, 1, INFINITY))
@@ -90,9 +90,9 @@ int main()
         return 1;
     if (test__powisf2(INFINITY, 4, INFINITY))
         return 1;
-    if (test__powisf2(INFINITY, 0x7FFFFFFE, INFINITY))
+    if (test__powisf2(INFINITY, INT_MAX - 1, INFINITY))
         return 1;
-    if (test__powisf2(INFINITY, 0x7FFFFFFF, INFINITY))
+    if (test__powisf2(INFINITY, INT_MAX, INFINITY))
         return 1;
 
     if (test__powisf2(-INFINITY, 1, -INFINITY))
@@ -103,9 +103,9 @@ int main()
         return 1;
     if (test__powisf2(-INFINITY, 4, INFINITY))
         return 1;
-    if (test__powisf2(-INFINITY, 0x7FFFFFFE, INFINITY))
+    if (test__powisf2(-INFINITY, INT_MAX - 1, INFINITY))
         return 1;
-    if (test__powisf2(-INFINITY, 0x7FFFFFFF, -INFINITY))
+    if (test__powisf2(-INFINITY, INT_MAX, -INFINITY))
         return 1;
 
     if (test__powisf2(0, -1, INFINITY))
@@ -116,11 +116,11 @@ int main()
         return 1;
     if (test__powisf2(0, -4, INFINITY))
         return 1;
-    if (test__powisf2(0, 0x80000002, INFINITY))
+    if (test__powisf2(0, INT_MIN + 2, INFINITY))
         return 1;
-    if (test__powisf2(0, 0x80000001, INFINITY))
+    if (test__powisf2(0, INT_MIN + 1, INFINITY))
         return 1;
-    if (test__powisf2(0, 0x80000000, INFINITY))
+    if (test__powisf2(0, INT_MIN, INFINITY))
         return 1;
 
     if (test__powisf2(-0., -1, -INFINITY))
@@ -131,11 +131,11 @@ int main()
         return 1;
     if (test__powisf2(-0., -4, INFINITY))
         return 1;
-    if (test__powisf2(-0., 0x80000002, INFINITY))
+    if (test__powisf2(-0., INT_MIN + 2, INFINITY))
         return 1;
-    if (test__powisf2(-0., 0x80000001, -INFINITY))
+    if (test__powisf2(-0., INT_MIN + 1, -INFINITY))
         return 1;
-    if (test__powisf2(-0., 0x80000000, INFINITY))
+    if (test__powisf2(-0., INT_MIN, INFINITY))
         return 1;
 
     if (test__powisf2(1, -1, 1))
@@ -146,11 +146,11 @@ int main()
         return 1;
     if (test__powisf2(1, -4, 1))
         return 1;
-    if (test__powisf2(1, 0x80000002, 1))
+    if (test__powisf2(1, INT_MIN + 2, 1))
         return 1;
-    if (test__powisf2(1, 0x80000001, 1))
+    if (test__powisf2(1, INT_MIN + 1, 1))
         return 1;
-    if (test__powisf2(1, 0x80000000, 1))
+    if (test__powisf2(1, INT_MIN, 1))
         return 1;
 
     if (test__powisf2(INFINITY, -1, 0))
@@ -161,11 +161,11 @@ int main()
         return 1;
     if (test__powisf2(INFINITY, -4, 0))
         return 1;
-    if (test__powisf2(INFINITY, 0x80000002, 0))
+    if (test__powisf2(INFINITY, INT_MIN + 2, 0))
         return 1;
-    if (test__powisf2(INFINITY, 0x80000001, 0))
+    if (test__powisf2(INFINITY, INT_MIN + 1, 0))
         return 1;
-    if (test__powisf2(INFINITY, 0x80000000, 0))
+    if (test__powisf2(INFINITY, INT_MIN, 0))
         return 1;
 
     if (test__powisf2(-INFINITY, -1, -0.))
@@ -176,11 +176,11 @@ int main()
         return 1;
     if (test__powisf2(-INFINITY, -4, 0))
         return 1;
-    if (test__powisf2(-INFINITY, 0x80000002, 0))
+    if (test__powisf2(-INFINITY, INT_MIN + 2, 0))
         return 1;
-    if (test__powisf2(-INFINITY, 0x80000001, -0.))
+    if (test__powisf2(-INFINITY, INT_MIN + 1, -0.))
         return 1;
-    if (test__powisf2(-INFINITY, 0x80000000, 0))
+    if (test__powisf2(-INFINITY, INT_MIN, 0))
         return 1;
 
     if (test__powisf2(2, 10, 1024.))
diff --git a/compiler-rt/test/builtins/Unit/powitf2_test.c b/compiler-rt/test/builtins/Unit/powitf2_test.c
index be16c5f45c32c..81331098730ac 100644
--- a/compiler-rt/test/builtins/Unit/powitf2_test.c
+++ b/compiler-rt/test/builtins/Unit/powitf2_test.c
@@ -10,9 +10,9 @@
 
 // Returns: a ^ b
 
-COMPILER_RT_ABI long double __powitf2(long double a, si_int b);
+COMPILER_RT_ABI long double __powitf2(long double a, int b);
 
-int test__powitf2(long double a, si_int b, long double expected)
+int test__powitf2(long double a, int b, long double expected)
 {
     long double x = __powitf2(a, b);
     int correct = (x == expected) && (signbit(x) == signbit(expected));
@@ -57,9 +57,9 @@ int main()
         return 1;
     if (test__powitf2(0, 4, 0))
         return 1;
-    if (test__powitf2(0, 0x7FFFFFFE, 0))
+    if (test__powitf2(0, INT_MAX - 1, 0))
         return 1;
-    if (test__powitf2(0, 0x7FFFFFFF, 0))
+    if (test__powitf2(0, INT_MAX, 0))
         return 1;
 
     if (test__powitf2(-0., 1, -0.))
@@ -70,9 +70,9 @@ int main()
         return 1;
     if (test__powitf2(-0., 4, 0))
         return 1;
-    if (test__powitf2(-0., 0x7FFFFFFE, 0))
+    if (test__powitf2(-0., INT_MAX - 1, 0))
         return 1;
-    if (test__powitf2(-0., 0x7FFFFFFF, -0.))
+    if (test__powitf2(-0., INT_MAX, -0.))
         return 1;
 
     if (test__powitf2(1, 1, 1))
@@ -83,9 +83,9 @@ int main()
         return 1;
     if (test__powitf2(1, 4, 1))
         return 1;
-    if (test__powitf2(1, 0x7FFFFFFE, 1))
+    if (test__powitf2(1, INT_MAX - 1, 1))
         return 1;
-    if (test__powitf2(1, 0x7FFFFFFF, 1))
+    if (test__powitf2(1, INT_MAX, 1))
         return 1;
 
     if (test__powitf2(INFINITY, 1, INFINITY))
@@ -96,9 +96,9 @@ int main()
         return 1;
     if (test__powitf2(INFINITY, 4, INFINITY))
         return 1;
-    if (test__powitf2(INFINITY, 0x7FFFFFFE, INFINITY))
+    if (test__powitf2(INFINITY, INT_MAX - 1, INFINITY))
         return 1;
-    if (test__powitf2(INFINITY, 0x7FFFFFFF, INFINITY))
+    if (test__powitf2(INFINITY, INT_MAX, INFINITY))
         return 1;
 
     if (test__powitf2(-INFINITY, 1, -INFINITY))
@@ -109,9 +109,9 @@ int main()
         return 1;
     if (test__powitf2(-INFINITY, 4, INFINITY))
         return 1;
-    if (test__powitf2(-INFINITY, 0x7FFFFFFE, INFINITY))
+    if (test__powitf2(-INFINITY, INT_MAX - 1, INFINITY))
         return 1;
-    if (test__powitf2(-INFINITY, 0x7FFFFFFF, -INFINITY))
+    if (test__powitf2(-INFINITY, INT_MAX, -INFINITY))
         return 1;
 
     if (test__powitf2(0, -1, INFINITY))
@@ -122,11 +122,11 @@ int main()
         return 1;
     if (test__powitf2(0, -4, INFINITY))
         return 1;
-    if (test__powitf2(0, 0x80000002, INFINITY))
+    if (test__powitf2(0, INT_MIN + 2, INFINITY))
         return 1;
-    if (test__powitf2(0, 0x80000001, INFINITY))
+    if (test__powitf2(0, INT_MIN + 1, INFINITY))
         return 1;
-    if (test__powitf2(0, 0x80000000, INFINITY))
+    if (test__powitf2(0, INT_MIN, INFINITY))
         return 1;
 
     if (test__powitf2(-0., -1, -INFINITY))
@@ -137,11 +137,11 @@ int main()
         return 1;
     if (test__powitf2(-0., -4, INFINITY))
         return 1;
-    if (test__powitf2(-0., 0x80000002, INFINITY))
+    if (test__powitf2(-0., INT_MIN + 2, INFINITY))
         return 1;
-    if (test__powitf2(-0., 0x80000001, -INFINITY))
+    if (test__powitf2(-0., INT_MIN + 1, -INFINITY))
         return 1;
-    if (test__powitf2(-0., 0x80000000, INFINITY))
+    if (test__powitf2(-0., INT_MIN, INFINITY))
         return 1;
 
     if (test__powitf2(1, -1, 1))
@@ -152,11 +152,11 @@ int main()
         return 1;
     if (test__powitf2(1, -4, 1))
         return 1;
-    if (test__powitf2(1, 0x80000002, 1))
+    if (test__powitf2(1, INT_MIN + 2, 1))
         return 1;
-    if (test__powitf2(1, 0x80000001, 1))
+    if (test__powitf2(1, INT_MIN + 1, 1))
         return 1;
-    if (test__powitf2(1, 0x80000000, 1))
+    if (test__powitf2(1, INT_MIN, 1))
         return 1;
 
     if (test__powitf2(INFINITY, -1, 0))
@@ -167,11 +167,11 @@ int main()
         return 1;
     if (test__powitf2(INFINITY, -4, 0))
         return 1;
-    if (test__powitf2(INFINITY, 0x80000002, 0))
+    if (test__powitf2(INFINITY, INT_MIN + 2, 0))
         return 1;
-    if (test__powitf2(INFINITY, 0x80000001, 0))
+    if (test__powitf2(INFINITY, INT_MIN + 1, 0))
         return 1;
-    if (test__powitf2(INFINITY, 0x80000000, 0))
+    if (test__powitf2(INFINITY, INT_MIN, 0))
         return 1;
 
     if (test__powitf2(-INFINITY, -1, -0.))
@@ -182,11 +182,11 @@ int main()
         return 1;
     if (test__powitf2(-INFINITY, -4, 0))
         return 1;
-    if (test__powitf2(-INFINITY, 0x80000002, 0))
+    if (test__powitf2(-INFINITY, INT_MIN + 2, 0))
         return 1;
-    if (test__powitf2(-INFINITY, 0x80000001, -0.))
+    if (test__powitf2(-INFINITY, INT_MIN + 1, -0.))
         return 1;
-    if (test__powitf2(-INFINITY, 0x80000000, 0))
+    if (test__powitf2(-INFINITY, INT_MIN, 0))
         return 1;
 
     if (test__powitf2(2, 10, 1024.))
diff --git a/compiler-rt/test/builtins/Unit/powixf2_test.c b/compiler-rt/test/builtins/Unit/powixf2_test.c
index 5ce3e522af458..856012ab32dbb 100644
--- a/compiler-rt/test/builtins/Unit/powixf2_test.c
+++ b/compiler-rt/test/builtins/Unit/powixf2_test.c
@@ -11,9 +11,9 @@
 
 // Returns: a ^ b
 
-COMPILER_RT_ABI long double __powixf2(long double a, si_int b);
+COMPILER_RT_ABI long double __powixf2(long double a, int b);
 
-int test__powixf2(long double a, si_int b, long double expected)
+int test__powixf2(long double a, int b, long double expected)
 {
     long double x = __powixf2(a, b);
     int correct = (x == expected) && (signbit(x) == signbit(expected));
@@ -58,9 +58,9 @@ int main()
         return 1;
     if (test__powixf2(0, 4, 0))
         return 1;
-    if (test__powixf2(0, 0x7FFFFFFE, 0))
+    if (test__powixf2(0, INT_MAX - 1, 0))
         return 1;
-    if (test__powixf2(0, 0x7FFFFFFF, 0))
+    if (test__powixf2(0, INT_MAX, 0))
         return 1;
 
     if (test__powixf2(-0., 1, -0.))
@@ -71,9 +71,9 @@ int main()
         return 1;
     if (test__powixf2(-0., 4, 0))
         return 1;
-    if (test__powixf2(-0., 0x7FFFFFFE, 0))
+    if (test__powixf2(-0., INT_MAX - 1, 0))
         return 1;
-    if (test__powixf2(-0., 0x7FFFFFFF, -0.))
+    if (test__powixf2(-0., INT_MAX, -0.))
         return 1;
 
     if (test__powixf2(1, 1, 1))
@@ -84,9 +84,9 @@ int main()
         return 1;
     if (test__powixf2(1, 4, 1))
         return 1;
-    if (test__powixf2(1, 0x7FFFFFFE, 1))
+    if (test__powixf2(1, INT_MAX - 1, 1))
         return 1;
-    if (test__powixf2(1, 0x7FFFFFFF, 1))
+    if (test__powixf2(1, INT_MAX, 1))
         return 1;
 
     if (test__powixf2(INFINITY, 1, INFINITY))
@@ -97,9 +97,9 @@ int main()
         return 1;
     if (test__powixf2(INFINITY, 4, INFINITY))
         return 1;
-    if (test__powixf2(INFINITY, 0x7FFFFFFE, INFINITY))
+    if (test__powixf2(INFINITY, INT_MAX - 1, INFINITY))
         return 1;
-    if (test__powixf2(INFINITY, 0x7FFFFFFF, INFINITY))
+    if (test__powixf2(INFINITY, INT_MAX, INFINITY))
         return 1;
 
     if (test__powixf2(-INFINITY, 1, -INFINITY))
@@ -110,9 +110,9 @@ int main()
         return 1;
     if (test__powixf2(-INFINITY, 4, INFINITY))
         return 1;
-    if (test__powixf2(-INFINITY, 0x7FFFFFFE, INFINITY))
+    if (test__powixf2(-INFINITY, INT_MAX - 1, INFINITY))
         return 1;
-    if (test__powixf2(-INFINITY, 0x7FFFFFFF, -INFINITY))
+    if (test__powixf2(-INFINITY, INT_MAX, -INFINITY))
         return 1;
 
     if (test__powixf2(0, -1, INFINITY))
@@ -123,11 +123,11 @@ int main()
         return 1;
     if (test__powixf2(0, -4, INFINITY))
         return 1;
-    if (test__powixf2(0, 0x80000002, INFINITY))
+    if (test__powixf2(0, INT_MIN + 2, INFINITY))
         return 1;
-    if (test__powixf2(0, 0x80000001, INFINITY))
+    if (test__powixf2(0, INT_MIN + 1, INFINITY))
         return 1;
-    if (test__powixf2(0, 0x80000000, INFINITY))
+    if (test__powixf2(0, INT_MIN, INFINITY))
         return 1;
 
     if (test__powixf2(-0., -1, -INFINITY))
@@ -138,11 +138,11 @@ int main()
         return 1;
     if (test__powixf2(-0., -4, INFINITY))
         return 1;
-    if (test__powixf2(-0., 0x80000002, INFINITY))
+    if (test__powixf2(-0., INT_MIN + 2, INFINITY))
         return 1;
-    if (test__powixf2(-0., 0x80000001, -INFINITY))
+    if (test__powixf2(-0., INT_MIN + 1, -INFINITY))
         return 1;
-    if (test__powixf2(-0., 0x80000000, INFINITY))
+    if (test__powixf2(-0., INT_MIN, INFINITY))
         return 1;
 
     if (test__powixf2(1, -1, 1))
@@ -153,11 +153,11 @@ int main()
         return 1;
     if (test__powixf2(1, -4, 1))
         return 1;
-    if (test__powixf2(1, 0x80000002, 1))
+    if (test__powixf2(1, INT_MIN + 2, 1))
         return 1;
-    if (test__powixf2(1, 0x80000001, 1))
+    if (test__powixf2(1, INT_MIN + 1, 1))
         return 1;
-    if (test__powixf2(1, 0x80000000, 1))
+    if (test__powixf2(1, INT_MIN, 1))
         return 1;
 
     if (test__powixf2(INFINITY, -1, 0))
@@ -168,11 +168,11 @@ int main()
         return 1;
     if (test__powixf2(INFINITY, -4, 0))
         return 1;
-    if (test__powixf2(INFINITY, 0x80000002, 0))
+    if (test__powixf2(INFINITY, INT_MIN + 2, 0))
         return 1;
-    if (test__powixf2(INFINITY, 0x80000001, 0))
+    if (test__powixf2(INFINITY, INT_MIN + 1, 0))
         return 1;
-    if (test__powixf2(INFINITY, 0x80000000, 0))
+    if (test__powixf2(INFINITY, INT_MIN, 0))
         return 1;
 
     if (test__powixf2(-INFINITY, -1, -0.))
@@ -183,11 +183,11 @@ int main()
         return 1;
     if (test__powixf2(-INFINITY, -4, 0))
         return 1;
-    if (test__powixf2(-INFINITY, 0x80000002, 0))
+    if (test__powixf2(-INFINITY, INT_MIN + 2, 0))
         return 1;
-    if (test__powixf2(-INFINITY, 0x80000001, -0.))
+    if (test__powixf2(-INFINITY, INT_MIN + 1, -0.))
         return 1;
-    if (test__powixf2(-INFINITY, 0x80000000, 0))
+    if (test__powixf2(-INFINITY, INT_MIN, 0))
         return 1;
 
     if (test__powixf2(2, 10, 1024.))
diff --git a/compiler-rt/test/crt/lit.cfg.py b/compiler-rt/test/crt/lit.cfg.py
index dc15e456fe199..68e7eda7d59ba 100644
--- a/compiler-rt/test/crt/lit.cfg.py
+++ b/compiler-rt/test/crt/lit.cfg.py
@@ -26,15 +26,15 @@ def get_library_path(file):
                             config.target_cflags.strip(),
                             '-print-file-name=%s' % file],
                            stdout=subprocess.PIPE,
-                           env=config.environment)
+                           env=config.environment,
+                           universal_newlines=True)
     if not cmd.stdout:
       lit_config.fatal("Couldn't find the library path for '%s'" % file)
     dir = cmd.stdout.read().strip()
     if sys.platform in ['win32'] and execute_external:
         # Don't pass dosish path separator to msys bash.exe.
         dir = dir.replace('\\', '/')
-    # Ensure the result is an ascii string, across Python2.5+ - Python3.
-    return str(dir.decode('ascii'))
+    return dir
 
 
 def get_libgcc_file_name():
@@ -42,15 +42,15 @@ def get_libgcc_file_name():
                             config.target_cflags.strip(),
                             '-print-libgcc-file-name'],
                            stdout=subprocess.PIPE,
-                           env=config.environment)
+                           env=config.environment,
+                           universal_newlines=True)
     if not cmd.stdout:
       lit_config.fatal("Couldn't find the library path for '%s'" % file)
     dir = cmd.stdout.read().strip()
     if sys.platform in ['win32'] and execute_external:
         # Don't pass dosish path separator to msys bash.exe.
         dir = dir.replace('\\', '/')
-    # Ensure the result is an ascii string, across Python2.5+ - Python3.
-    return str(dir.decode('ascii'))
+    return dir
 
 
 def build_invocation(compile_flags):
@@ -66,6 +66,11 @@ def build_invocation(compile_flags):
 
 base_lib = os.path.join(
     config.compiler_rt_libdir, "clang_rt.%%s%s.o" % config.target_suffix)
+
+if sys.platform in ['win32'] and execute_external:
+    # Don't pass dosish path separator to msys bash.exe.
+    base_lib = base_lib.replace('\\', '/')
+
 config.substitutions.append(('%crtbegin', base_lib % "crtbegin"))
 config.substitutions.append(('%crtend', base_lib % "crtend"))
 
diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py
index 98a2f3c03e609..9d0c214bd9a76 100644
--- a/compiler-rt/test/lit.common.cfg.py
+++ b/compiler-rt/test/lit.common.cfg.py
@@ -23,6 +23,9 @@
     # bash on Windows is usually very slow.
     execute_external = (not sys.platform in ['win32'])
 
+# Allow expanding substitutions that are based on other substitutions
+config.recursiveExpansionLimit = 10
+
 # Setup test format.
 config.test_format = lit.formats.ShTest(execute_external)
 if execute_external:
diff --git a/compiler-rt/test/lit.common.configured.in b/compiler-rt/test/lit.common.configured.in
index a0203c0855651..4a3e268c8a6f9 100644
--- a/compiler-rt/test/lit.common.configured.in
+++ b/compiler-rt/test/lit.common.configured.in
@@ -48,6 +48,11 @@ config.available_features.add('target-is-%s' % config.target_arch)
 
 if config.enable_per_target_runtime_dir:
   set_default("target_suffix", "")
+elif config.android:
+  if config.target_arch == "i386":
+    set_default("target_suffix", "-i686-android")
+  else:
+    set_default("target_suffix", "-%s-android" % config.target_arch)
 else:
   set_default("target_suffix", "-%s" % config.target_arch)
 
diff --git a/compiler-rt/test/msan/sigwait.cpp b/compiler-rt/test/msan/sigwait.cpp
index 222fc34a16982..80f3f21ffff3b 100644
--- a/compiler-rt/test/msan/sigwait.cpp
+++ b/compiler-rt/test/msan/sigwait.cpp
@@ -4,6 +4,7 @@
 #include <assert.h>
 #include <signal.h>
 #include <sys/time.h>
+#include <sys/wait.h>
 #include <unistd.h>
 
 #include <sanitizer/msan_interface.h>
@@ -19,7 +20,9 @@ void test_sigwait() {
 
   if (pid_t pid = fork()) {
     kill(pid, SIGUSR1);
-    _exit(0);
+    int child_stat;
+    wait(&child_stat);
+    _exit(!WIFEXITED(child_stat));
   } else {
     int sig;
     int res = sigwait(&s, &sig);
diff --git a/compiler-rt/test/profile/ContinuousSyncMode/online-merging.c b/compiler-rt/test/profile/ContinuousSyncMode/online-merging.c
index a9f32b9c51e15..b3c33f339713c 100644
--- a/compiler-rt/test/profile/ContinuousSyncMode/online-merging.c
+++ b/compiler-rt/test/profile/ContinuousSyncMode/online-merging.c
@@ -8,9 +8,9 @@
 // Create two DSOs and a driver program that uses them.
 // RUN: echo "void dso1(void) {}" > dso1.c
 // RUN: echo "void dso2(void) {}" > dso2.c
-// RUN: %clang_pgogen -dynamiclib -o %t.dir/dso1.dylib dso1.c
-// RUN: %clang_pgogen -dynamiclib -o %t.dir/dso2.dylib dso2.c
-// RUN: %clang_pgogen -o main.exe %s %t.dir/dso1.dylib %t.dir/dso2.dylib
+// RUN: %clang_pgogen -dynamiclib -o %t.dir/dso1.dylib dso1.c -mllvm -instrprof-atomic-counter-update-all=1
+// RUN: %clang_pgogen -dynamiclib -o %t.dir/dso2.dylib dso2.c -mllvm -instrprof-atomic-counter-update-all=1
+// RUN: %clang_pgogen -o main.exe %s %t.dir/dso1.dylib %t.dir/dso2.dylib -mllvm -instrprof-atomic-counter-update-all=1
 //
 // === Round 1 ===
 // Test merging+continuous mode without any file contention.
diff --git a/compiler-rt/test/profile/Inputs/instrprof-gcov-__gcov_flush-multiple.c b/compiler-rt/test/profile/Inputs/instrprof-gcov-__gcov_flush-multiple.c
deleted file mode 100644
index 56e0538b1e830..0000000000000
--- a/compiler-rt/test/profile/Inputs/instrprof-gcov-__gcov_flush-multiple.c
+++ /dev/null
@@ -1,18 +0,0 @@
-extern void __gcov_flush();
-extern int remove(const char *);
-int main(void) {
-  __gcov_flush();
-
-  if (remove("instrprof-gcov-__gcov_flush-multiple.gcda") != 0) {
-    return 1;
-  }
-
-  __gcov_flush();
-  __gcov_flush();
-
-  if (remove("instrprof-gcov-__gcov_flush-multiple.gcda") != 0) {
-    return 1;
-  }
-
-  return 0;
-}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-gcov-__gcov_flush-multiple.c.gcov b/compiler-rt/test/profile/Inputs/instrprof-gcov-__gcov_flush-multiple.c.gcov
deleted file mode 100644
index b0921d22461e2..0000000000000
--- a/compiler-rt/test/profile/Inputs/instrprof-gcov-__gcov_flush-multiple.c.gcov
+++ /dev/null
@@ -1,23 +0,0 @@
-// CHECK:        -:    0:Source:{{.*}}Inputs{{[/\\]}}instrprof-gcov-__gcov_flush-multiple.c
-// CHECK-NEXT:        -:    0:Graph:instrprof-gcov-__gcov_flush-multiple.gcno
-// CHECK-NEXT:        -:    0:Data:instrprof-gcov-__gcov_flush-multiple.gcda
-// CHECK-NEXT:        -:    0:Runs:1
-// CHECK-NEXT:        -:    0:Programs:1
-// CHECK-NEXT:        -:    1:extern void __gcov_flush();
-// CHECK-NEXT:        -:    2:extern int remove(const char *);
-// CHECK-NEXT:    #####:    3:int main(void) {
-// CHECK-NEXT:    #####:    4:  __gcov_flush();
-// CHECK-NEXT:        -:    5:
-// CHECK-NEXT:    #####:    6:  if (remove("instrprof-gcov-__gcov_flush-multiple.gcda") != 0) {
-// CHECK-NEXT:    #####:    7:    return 1;
-// CHECK-NEXT:        -:    8:  }
-// CHECK-NEXT:        -:    9:
-// CHECK-NEXT:    #####:   10:  __gcov_flush();
-// CHECK-NEXT:    #####:   11:  __gcov_flush();
-// CHECK-NEXT:        -:   12:
-// CHECK-NEXT:    #####:   13:  if (remove("instrprof-gcov-__gcov_flush-multiple.gcda") != 0) {
-// CHECK-NEXT:    #####:   14:    return 1;
-// CHECK-NEXT:        -:   15:  }
-// CHECK-NEXT:        -:   16:
-// CHECK-NEXT:        1:   17:  return 0;
-// CHECK-NEXT:        1:   18:}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-gcov-execlp.c b/compiler-rt/test/profile/Inputs/instrprof-gcov-execlp.c
deleted file mode 100644
index 5213a5b80360e..0000000000000
--- a/compiler-rt/test/profile/Inputs/instrprof-gcov-execlp.c
+++ /dev/null
@@ -1,15 +0,0 @@
-#include <unistd.h>
-
-void func1() {}
-void func2() {}
-
-int main(void)
-{
-  func1();
-
-  execlp("ls", "-l", "-h", (char*)0);
-
-  func2();
-
-  return 0;
-}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-gcov-execlp.c.gcov b/compiler-rt/test/profile/Inputs/instrprof-gcov-execlp.c.gcov
deleted file mode 100644
index 810046aac396f..0000000000000
--- a/compiler-rt/test/profile/Inputs/instrprof-gcov-execlp.c.gcov
+++ /dev/null
@@ -1,23 +0,0 @@
-//CHECK:        -:    0:Source:{{.*}}Inputs{{[/\\]}}instrprof-gcov-execlp.c
-//CHECK-NEXT:        -:    0:Graph:instrprof-gcov-execlp.gcno
-//CHECK-NEXT:        -:    0:Data:instrprof-gcov-execlp.gcda
-//CHECK-NEXT:        -:    0:Runs:1
-//CHECK-NEXT:        -:    0:Programs:1
-//CHECK-NEXT:        -:    1:#include <unistd.h>
-//CHECK-NEXT:        -:    2:
-//CHECK-NEXT:function func1 called 1 returned 100% blocks executed 100%
-//CHECK-NEXT:        1:    3:void func1() {}
-//CHECK-NEXT:function func2 called 0 returned 0% blocks executed 0%
-//CHECK-NEXT:    #####:    4:void func2() {}
-//CHECK-NEXT:        -:    5:
-//CHECK-NEXT:function main called 1 returned 0% blocks executed 33%
-//CHECK-NEXT:        1:    6:int main(void)
-//CHECK-NEXT:        -:    7:{
-//CHECK-NEXT:        1:    8:  func1();
-//CHECK-NEXT:        -:    9:
-//CHECK-NEXT:        1:   10:  execlp("ls", "-l", "-h", (char*)0);
-//CHECK-NEXT:        -:   11:
-//CHECK-NEXT:    #####:   12:  func2();
-//CHECK-NEXT:        -:   13:
-//CHECK-NEXT:    #####:   14:  return 0;
-//CHECK-NEXT:        -:   15:}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-gcov-execvp.c b/compiler-rt/test/profile/Inputs/instrprof-gcov-execvp.c
deleted file mode 100644
index f2e411c2a42e8..0000000000000
--- a/compiler-rt/test/profile/Inputs/instrprof-gcov-execvp.c
+++ /dev/null
@@ -1,17 +0,0 @@
-#include <unistd.h>
-
-void func1() {}
-void func2() {}
-
-int main(void)
-{
-  char *const args[] = {"-l", "-h", (char*)0};
-
-  func1();
-
-  execvp("ls", args);
-
-  func2();
-
-  return 0;
-}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-gcov-execvp.c.gcov b/compiler-rt/test/profile/Inputs/instrprof-gcov-execvp.c.gcov
deleted file mode 100644
index e896cb8917e70..0000000000000
--- a/compiler-rt/test/profile/Inputs/instrprof-gcov-execvp.c.gcov
+++ /dev/null
@@ -1,25 +0,0 @@
-//CHECK:        -:    0:Source:{{.*}}Inputs{{[/\\]}}instrprof-gcov-execvp.c
-//CHECK-NEXT:        -:    0:Graph:instrprof-gcov-execvp.gcno
-//CHECK-NEXT:        -:    0:Data:instrprof-gcov-execvp.gcda
-//CHECK-NEXT:        -:    0:Runs:1
-//CHECK-NEXT:        -:    0:Programs:1
-//CHECK-NEXT:        -:    1:#include <unistd.h>
-//CHECK-NEXT:        -:    2:
-//CHECK-NEXT:function func1 called 1 returned 100% blocks executed 100%
-//CHECK-NEXT:        1:    3:void func1() {}
-//CHECK-NEXT:function func2 called 0 returned 0% blocks executed 0%
-//CHECK-NEXT:    #####:    4:void func2() {}
-//CHECK-NEXT:        -:    5:
-//CHECK-NEXT:function main called 1 returned 0% blocks executed 33%
-//CHECK-NEXT:        1:    6:int main(void)
-//CHECK-NEXT:        -:    7:{
-//CHECK-NEXT:        1:    8:  char *const args[] = {"-l", "-h", (char*)0};
-//CHECK-NEXT:        -:    9:
-//CHECK-NEXT:        1:   10:  func1();
-//CHECK-NEXT:        -:   11:
-//CHECK-NEXT:        1:   12:  execvp("ls", args);
-//CHECK-NEXT:        -:   13:
-//CHECK-NEXT:    #####:   14:  func2();
-//CHECK-NEXT:        -:   15:
-//CHECK-NEXT:    #####:   16:  return 0;
-//CHECK-NEXT:        -:   17:}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-gcov-fork.c b/compiler-rt/test/profile/Inputs/instrprof-gcov-fork.c
deleted file mode 100644
index 818c459c50ddc..0000000000000
--- a/compiler-rt/test/profile/Inputs/instrprof-gcov-fork.c
+++ /dev/null
@@ -1,15 +0,0 @@
-#include <unistd.h>
-
-void func1() {}
-void func2() {}
-
-int main(void)
-{
-  func1();
-
-  fork();
-
-  func2();
-
-  return 0;
-}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-gcov-fork.c.gcov b/compiler-rt/test/profile/Inputs/instrprof-gcov-fork.c.gcov
deleted file mode 100644
index 2825bd5802a79..0000000000000
--- a/compiler-rt/test/profile/Inputs/instrprof-gcov-fork.c.gcov
+++ /dev/null
@@ -1,23 +0,0 @@
-// CHECK:        -:    0:Source:{{.*}}Inputs{{[/\\]}}instrprof-gcov-fork.c
-// CHECK-NEXT:        -:    0:Graph:instrprof-gcov-fork.gcno
-// CHECK-NEXT:        -:    0:Data:instrprof-gcov-fork.gcda
-// CHECK-NEXT:        -:    0:Runs:1
-// CHECK-NEXT:        -:    0:Programs:1
-// CHECK-NEXT:        -:    1:#include <unistd.h>
-// CHECK-NEXT:        -:    2:
-// CHECK-NEXT:function func1 called 1 returned 100% blocks executed 100%
-// CHECK-NEXT:        1:    3:void func1() {}
-// CHECK-NEXT:function func2 called 2 returned 100% blocks executed 100%
-// CHECK-NEXT:        2:    4:void func2() {}
-// CHECK-NEXT:        -:    5:
-// CHECK-NEXT:function main called 1 returned 200% blocks executed 100%
-// CHECK-NEXT:        1:    6:int main(void)
-// CHECK-NEXT:        -:    7:{
-// CHECK-NEXT:        1:    8:  func1();
-// CHECK-NEXT:        -:    9:
-// CHECK-NEXT:        1:   10:  fork();
-// CHECK-NEXT:        -:   11:
-// CHECK-NEXT:        2:   12:  func2();
-// CHECK-NEXT:        -:   13:
-// CHECK-NEXT:        2:   14:  return 0;
-// CHECK-NEXT:        -:   15:}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-shared-lib.c.gcov b/compiler-rt/test/profile/Inputs/instrprof-shared-lib.c.gcov
deleted file mode 100644
index 7e6d741d5b104..0000000000000
--- a/compiler-rt/test/profile/Inputs/instrprof-shared-lib.c.gcov
+++ /dev/null
@@ -1,14 +0,0 @@
-// CHECK:        -:    0:Source:{{.*}}Inputs{{[/\\]}}instrprof-shared-lib.c
-// CHECK-NEXT:        -:    0:Graph:instrprof-shared-lib.gcno
-// CHECK-NEXT:        -:    0:Data:instrprof-shared-lib.gcda
-// CHECK-NEXT:        -:    0:Runs:1
-// CHECK-NEXT:        -:    0:Programs:1
-// CHECK-NEXT:        -:    1:int g1 = 0;
-// CHECK-NEXT:        -:    2:int g2 = 1;
-// CHECK-NEXT:        -:    3:
-// CHECK-NEXT:        1:    4:void foo(int n) {
-// CHECK-NEXT:        1:    5:  if (n % 5 == 0)
-// CHECK-NEXT:    #####:    6:    g1++;
-// CHECK-NEXT:        -:    7:  else
-// CHECK-NEXT:        1:    8:    g2++;
-// CHECK-NEXT:        1:    9:}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-shared-lib_called-twice.c.gcov b/compiler-rt/test/profile/Inputs/instrprof-shared-lib_called-twice.c.gcov
deleted file mode 100644
index 993c6cc508b28..0000000000000
--- a/compiler-rt/test/profile/Inputs/instrprof-shared-lib_called-twice.c.gcov
+++ /dev/null
@@ -1,14 +0,0 @@
-// CHECK:        -:    0:Source:{{.*}}Inputs{{[/\\]}}instrprof-shared-lib.c
-// CHECK-NEXT:        -:    0:Graph:instrprof-shared-lib.gcno
-// CHECK-NEXT:        -:    0:Data:instrprof-shared-lib.gcda
-// CHECK-NEXT:        -:    0:Runs:1
-// CHECK-NEXT:        -:    0:Programs:1
-// CHECK-NEXT:        -:    1:int g1 = 0;
-// CHECK-NEXT:        -:    2:int g2 = 1;
-// CHECK-NEXT:        -:    3:
-// CHECK-NEXT:        2:    4:void foo(int n) {
-// CHECK-NEXT:        2:    5:  if (n % 5 == 0)
-// CHECK-NEXT:    #####:    6:    g1++;
-// CHECK-NEXT:        -:    7:  else
-// CHECK-NEXT:        2:    8:    g2++;
-// CHECK-NEXT:        2:    9:}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-shared-main-gcov-flush.c b/compiler-rt/test/profile/Inputs/instrprof-shared-main-gcov-flush.c
deleted file mode 100644
index 9f41b7e6362aa..0000000000000
--- a/compiler-rt/test/profile/Inputs/instrprof-shared-main-gcov-flush.c
+++ /dev/null
@@ -1,36 +0,0 @@
-extern void foo(int n);
-extern void __gcov_flush(void);
-
-int bar1 = 0;
-int bar2 = 1;
-
-void bar(int n) {
-  if (n % 5 == 0)
-    bar1++;
-  else
-    bar2++;
-}
-
-int main(int argc, char *argv[]) {
-#ifdef SHARED_CALL_BEFORE_GCOV_FLUSH
-  foo(1);
-#endif
-
-  bar(5);
-
-  __gcov_flush();
-
-  bar(5);
-
-#ifdef SHARED_CALL_AFTER_GCOV_FLUSH
-  foo(1);
-#endif
-
-#ifdef EXIT_ABRUPTLY
-  _exit(0);
-#endif
-
-  bar(5);
-
-  return 0;
-}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-shared-main-gcov-flush_no-writeout.c.gcov b/compiler-rt/test/profile/Inputs/instrprof-shared-main-gcov-flush_no-writeout.c.gcov
deleted file mode 100644
index 82a489158a33a..0000000000000
--- a/compiler-rt/test/profile/Inputs/instrprof-shared-main-gcov-flush_no-writeout.c.gcov
+++ /dev/null
@@ -1,41 +0,0 @@
-// CHECK:        -:    0:Source:{{.*}}Inputs{{[/\\]}}instrprof-shared-main-gcov-flush.c
-// CHECK-NEXT:        -:    0:Graph:instrprof-shared-main-gcov-flush.gcno
-// CHECK-NEXT:        -:    0:Data:instrprof-shared-main-gcov-flush.gcda
-// CHECK-NEXT:        -:    0:Runs:1
-// CHECK-NEXT:        -:    0:Programs:1
-// CHECK-NEXT:        -:    1:extern void foo(int n);
-// CHECK-NEXT:        -:    2:extern void __gcov_flush(void);
-// CHECK-NEXT:        -:    3:
-// CHECK-NEXT:        -:    4:int bar1 = 0;
-// CHECK-NEXT:        -:    5:int bar2 = 1;
-// CHECK-NEXT:        -:    6:
-// CHECK-NEXT:        1:    7:void bar(int n) {
-// CHECK-NEXT:        1:    8:  if (n % 5 == 0)
-// CHECK-NEXT:        1:    9:    bar1++;
-// CHECK-NEXT:        -:   10:  else
-// CHECK-NEXT:    #####:   11:    bar2++;
-// CHECK-NEXT:        1:   12:}
-// CHECK-NEXT:        -:   13:
-// CHECK-NEXT:        1:   14:int main(int argc, char *argv[]) {
-// CHECK-NEXT:        -:   15:#ifdef SHARED_CALL_BEFORE_GCOV_FLUSH
-// CHECK-NEXT:        1:   16:  foo(1);
-// CHECK-NEXT:        -:   17:#endif
-// CHECK-NEXT:        -:   18:
-// CHECK-NEXT:        1:   19:  bar(5);
-// CHECK-NEXT:        -:   20:
-// CHECK-NEXT:        1:   21:  __gcov_flush();
-// CHECK-NEXT:        -:   22:
-// CHECK-NEXT:        1:   23:  bar(5);
-// CHECK-NEXT:        -:   24:
-// CHECK-NEXT:        -:   25:#ifdef SHARED_CALL_AFTER_GCOV_FLUSH
-// CHECK-NEXT:        1:   26:  foo(1);
-// CHECK-NEXT:        -:   27:#endif
-// CHECK-NEXT:        -:   28:
-// CHECK-NEXT:        -:   29:#ifdef EXIT_ABRUPTLY
-// CHECK-NEXT:        1:   30:  _exit(0);
-// CHECK-NEXT:        -:   31:#endif
-// CHECK-NEXT:        -:   32:
-// CHECK-NEXT:        -:   33:  bar(5);
-// CHECK-NEXT:        -:   34:
-// CHECK-NEXT:        -:   35:  return 0;
-// CHECK-NEXT:        -:   36:}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-shared-main-gcov-flush_shared-call-after.c.gcov b/compiler-rt/test/profile/Inputs/instrprof-shared-main-gcov-flush_shared-call-after.c.gcov
deleted file mode 100644
index 5cc26580b2226..0000000000000
--- a/compiler-rt/test/profile/Inputs/instrprof-shared-main-gcov-flush_shared-call-after.c.gcov
+++ /dev/null
@@ -1,41 +0,0 @@
-// CHECK:        -:    0:Source:{{.*}}Inputs{{[/\\]}}instrprof-shared-main-gcov-flush.c
-// CHECK-NEXT:        -:    0:Graph:instrprof-shared-main-gcov-flush.gcno
-// CHECK-NEXT:        -:    0:Data:instrprof-shared-main-gcov-flush.gcda
-// CHECK-NEXT:        -:    0:Runs:1
-// CHECK-NEXT:        -:    0:Programs:1
-// CHECK-NEXT:        -:    1:extern void foo(int n);
-// CHECK-NEXT:        -:    2:extern void __gcov_flush(void);
-// CHECK-NEXT:        -:    3:
-// CHECK-NEXT:        -:    4:int bar1 = 0;
-// CHECK-NEXT:        -:    5:int bar2 = 1;
-// CHECK-NEXT:        -:    6:
-// CHECK-NEXT:        3:    7:void bar(int n) {
-// CHECK-NEXT:        3:    8:  if (n % 5 == 0)
-// CHECK-NEXT:        3:    9:    bar1++;
-// CHECK-NEXT:        -:   10:  else
-// CHECK-NEXT:    #####:   11:    bar2++;
-// CHECK-NEXT:        3:   12:}
-// CHECK-NEXT:        -:   13:
-// CHECK-NEXT:        1:   14:int main(int argc, char *argv[]) {
-// CHECK-NEXT:        -:   15:#ifdef SHARED_CALL_BEFORE_GCOV_FLUSH
-// CHECK-NEXT:        -:   16:  foo(1);
-// CHECK-NEXT:        -:   17:#endif
-// CHECK-NEXT:        -:   18:
-// CHECK-NEXT:        1:   19:  bar(5);
-// CHECK-NEXT:        -:   20:
-// CHECK-NEXT:        1:   21:  __gcov_flush();
-// CHECK-NEXT:        -:   22:
-// CHECK-NEXT:        1:   23:  bar(5);
-// CHECK-NEXT:        -:   24:
-// CHECK-NEXT:        -:   25:#ifdef SHARED_CALL_AFTER_GCOV_FLUSH
-// CHECK-NEXT:        1:   26:  foo(1);
-// CHECK-NEXT:        -:   27:#endif
-// CHECK-NEXT:        -:   28:
-// CHECK-NEXT:        -:   29:#ifdef EXIT_ABRUPTLY
-// CHECK-NEXT:        -:   30:  _exit(0);
-// CHECK-NEXT:        -:   31:#endif
-// CHECK-NEXT:        -:   32:
-// CHECK-NEXT:        1:   33:  bar(5);
-// CHECK-NEXT:        -:   34:
-// CHECK-NEXT:        1:   35:  return 0;
-// CHECK-NEXT:        -:   36:}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-shared-main-gcov-flush_shared-call-before-after.c.gcov b/compiler-rt/test/profile/Inputs/instrprof-shared-main-gcov-flush_shared-call-before-after.c.gcov
deleted file mode 100644
index 7a6800c47f84f..0000000000000
--- a/compiler-rt/test/profile/Inputs/instrprof-shared-main-gcov-flush_shared-call-before-after.c.gcov
+++ /dev/null
@@ -1,41 +0,0 @@
-// CHECK:        -:    0:Source:{{.*}}Inputs{{[/\\]}}instrprof-shared-main-gcov-flush.c
-// CHECK-NEXT:        -:    0:Graph:instrprof-shared-main-gcov-flush.gcno
-// CHECK-NEXT:        -:    0:Data:instrprof-shared-main-gcov-flush.gcda
-// CHECK-NEXT:        -:    0:Runs:1
-// CHECK-NEXT:        -:    0:Programs:1
-// CHECK-NEXT:        -:    1:extern void foo(int n);
-// CHECK-NEXT:        -:    2:extern void __gcov_flush(void);
-// CHECK-NEXT:        -:    3:
-// CHECK-NEXT:        -:    4:int bar1 = 0;
-// CHECK-NEXT:        -:    5:int bar2 = 1;
-// CHECK-NEXT:        -:    6:
-// CHECK-NEXT:        3:    7:void bar(int n) {
-// CHECK-NEXT:        3:    8:  if (n % 5 == 0)
-// CHECK-NEXT:        3:    9:    bar1++;
-// CHECK-NEXT:        -:   10:  else
-// CHECK-NEXT:    #####:   11:    bar2++;
-// CHECK-NEXT:        3:   12:}
-// CHECK-NEXT:        -:   13:
-// CHECK-NEXT:        1:   14:int main(int argc, char *argv[]) {
-// CHECK-NEXT:        -:   15:#ifdef SHARED_CALL_BEFORE_GCOV_FLUSH
-// CHECK-NEXT:        1:   16:  foo(1);
-// CHECK-NEXT:        -:   17:#endif
-// CHECK-NEXT:        -:   18:
-// CHECK-NEXT:        1:   19:  bar(5);
-// CHECK-NEXT:        -:   20:
-// CHECK-NEXT:        1:   21:  __gcov_flush();
-// CHECK-NEXT:        -:   22:
-// CHECK-NEXT:        1:   23:  bar(5);
-// CHECK-NEXT:        -:   24:
-// CHECK-NEXT:        -:   25:#ifdef SHARED_CALL_AFTER_GCOV_FLUSH
-// CHECK-NEXT:        1:   26:  foo(1);
-// CHECK-NEXT:        -:   27:#endif
-// CHECK-NEXT:        -:   28:
-// CHECK-NEXT:        -:   29:#ifdef EXIT_ABRUPTLY
-// CHECK-NEXT:        -:   30:  _exit(0);
-// CHECK-NEXT:        -:   31:#endif
-// CHECK-NEXT:        -:   32:
-// CHECK-NEXT:        1:   33:  bar(5);
-// CHECK-NEXT:        -:   34:
-// CHECK-NEXT:        1:   35:  return 0;
-// CHECK-NEXT:        -:   36:}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-shared-main-gcov-flush_shared-call-before.c.gcov b/compiler-rt/test/profile/Inputs/instrprof-shared-main-gcov-flush_shared-call-before.c.gcov
deleted file mode 100644
index 49995fdc86350..0000000000000
--- a/compiler-rt/test/profile/Inputs/instrprof-shared-main-gcov-flush_shared-call-before.c.gcov
+++ /dev/null
@@ -1,41 +0,0 @@
-// CHECK:        -:    0:Source:{{.*}}Inputs{{[/\\]}}instrprof-shared-main-gcov-flush.c
-// CHECK-NEXT:        -:    0:Graph:instrprof-shared-main-gcov-flush.gcno
-// CHECK-NEXT:        -:    0:Data:instrprof-shared-main-gcov-flush.gcda
-// CHECK-NEXT:        -:    0:Runs:1
-// CHECK-NEXT:        -:    0:Programs:1
-// CHECK-NEXT:        -:    1:extern void foo(int n);
-// CHECK-NEXT:        -:    2:extern void __gcov_flush(void);
-// CHECK-NEXT:        -:    3:
-// CHECK-NEXT:        -:    4:int bar1 = 0;
-// CHECK-NEXT:        -:    5:int bar2 = 1;
-// CHECK-NEXT:        -:    6:
-// CHECK-NEXT:        3:    7:void bar(int n) {
-// CHECK-NEXT:        3:    8:  if (n % 5 == 0)
-// CHECK-NEXT:        3:    9:    bar1++;
-// CHECK-NEXT:        -:   10:  else
-// CHECK-NEXT:    #####:   11:    bar2++;
-// CHECK-NEXT:        3:   12:}
-// CHECK-NEXT:        -:   13:
-// CHECK-NEXT:        1:   14:int main(int argc, char *argv[]) {
-// CHECK-NEXT:        -:   15:#ifdef SHARED_CALL_BEFORE_GCOV_FLUSH
-// CHECK-NEXT:        1:   16:  foo(1);
-// CHECK-NEXT:        -:   17:#endif
-// CHECK-NEXT:        -:   18:
-// CHECK-NEXT:        1:   19:  bar(5);
-// CHECK-NEXT:        -:   20:
-// CHECK-NEXT:        1:   21:  __gcov_flush();
-// CHECK-NEXT:        -:   22:
-// CHECK-NEXT:        1:   23:  bar(5);
-// CHECK-NEXT:        -:   24:
-// CHECK-NEXT:        -:   25:#ifdef SHARED_CALL_AFTER_GCOV_FLUSH
-// CHECK-NEXT:        -:   26:  foo(1);
-// CHECK-NEXT:        -:   27:#endif
-// CHECK-NEXT:        -:   28:
-// CHECK-NEXT:        -:   29:#ifdef EXIT_ABRUPTLY
-// CHECK-NEXT:        -:   30:  _exit(0);
-// CHECK-NEXT:        -:   31:#endif
-// CHECK-NEXT:        -:   32:
-// CHECK-NEXT:        1:   33:  bar(5);
-// CHECK-NEXT:        -:   34:
-// CHECK-NEXT:        1:   35:  return 0;
-// CHECK-NEXT:        -:   36:}
diff --git a/compiler-rt/test/profile/Posix/gcov-destructor.c b/compiler-rt/test/profile/Posix/gcov-destructor.c
new file mode 100644
index 0000000000000..c6fa144bcb39f
--- /dev/null
+++ b/compiler-rt/test/profile/Posix/gcov-destructor.c
@@ -0,0 +1,34 @@
+/// Test that destructors and destructors whose priorities are greater than 100 are tracked.
+// RUN: mkdir -p %t.dir && cd %t.dir
+// RUN: %clang --coverage %s -o %t
+// RUN: rm -f gcov-destructor.gcda && %run %t
+// RUN: llvm-cov gcov -t gcov-destructor.gcda | FileCheck %s
+// UNSUPPORTED: darwin
+
+#include <unistd.h>
+
+void before_exec() {}                   // CHECK:      1: [[#@LINE]]:void before_exec
+void after_exec() {}                    // CHECK-NEXT: 1: [[#@LINE]]:void after_exec
+
+__attribute__((constructor))            // CHECK:      -: [[#@LINE]]:__attribute__
+void constructor() {}                   // CHECK-NEXT: 1: [[#@LINE]]:
+
+/// Runs before __llvm_gcov_writeout.
+__attribute__((destructor))             // CHECK:      -: [[#@LINE]]:__attribute__
+void destructor() {}                    // CHECK-NEXT: 1: [[#@LINE]]:
+
+__attribute__((destructor(101)))        // CHECK:      -: [[#@LINE]]:__attribute__
+void destructor_101() {}                // CHECK-NEXT: 1: [[#@LINE]]:
+
+/// Runs after __llvm_gcov_writeout.
+__attribute__((destructor(99)))         // CHECK:      -: [[#@LINE]]:__attribute__
+void destructor_99() {}                 // CHECK-NEXT: #####: [[#@LINE]]:
+
+int main() {
+  before_exec();
+  // Implicit writeout.
+  execl("/not_exist", "not_exist", (char *)0);
+  // Still tracked.
+  after_exec();
+  return 0;
+}
diff --git a/compiler-rt/test/profile/Posix/gcov-execlp.c b/compiler-rt/test/profile/Posix/gcov-execlp.c
new file mode 100644
index 0000000000000..0b70e9c697904
--- /dev/null
+++ b/compiler-rt/test/profile/Posix/gcov-execlp.c
@@ -0,0 +1,28 @@
+/// A basic block with fork/exec* is split. .gcda is flushed immediately before
+/// fork/exec* so the lines before exec* are counted once while succeeding
+/// lines are not counted.
+// RUN: mkdir -p %t.d && cd %t.d
+// RUN: %clang --coverage %s -o %t
+// RUN: test -f gcov-execlp.gcno
+// RUN: rm -f gcov-execlp.gcda && %run %t
+// RUN: llvm-cov gcov -t gcov-execlp.gcda | FileCheck %s --check-prefixes=CHECK,EXECLP
+
+// RUN: %clang --coverage -DEXECVP %s -o %t
+// RUN: rm -f gcov-execlp.gcda && %run %t
+// RUN: llvm-cov gcov -t gcov-execlp.gcda | FileCheck %s --check-prefixes=CHECK,EXECVP
+
+#include <unistd.h>
+
+void func1(void) {}                // CHECK:          1: [[#@LINE]]:void func1(void)
+void func2(void) {}                // CHECK-NEXT: #####: [[#@LINE]]:
+int main(void) {                   // CHECK-NEXT:     1: [[#@LINE]]:
+  func1();                         // CHECK-NEXT:     1: [[#@LINE]]:
+#ifdef EXECVP
+  char *argv[] = {"ls", "-l", "-h", (char *)0};
+  execvp("ls", argv);              // EXECVP:         1: [[#@LINE]]:  execvp
+#else
+  execlp("ls", "-l", "-h", (char *)0); // EXECLP:     1: [[#@LINE]]:  execlp
+#endif
+  func2();                         // CHECK:      #####: [[#@LINE]]:  func2
+  return 0;                        // CHECK-NEXT: #####: [[#@LINE]]:
+}
diff --git a/compiler-rt/test/profile/Posix/gcov-fork.c b/compiler-rt/test/profile/Posix/gcov-fork.c
new file mode 100644
index 0000000000000..4942d5ac92888
--- /dev/null
+++ b/compiler-rt/test/profile/Posix/gcov-fork.c
@@ -0,0 +1,24 @@
+/// A basic block with fork/exec* is split. .gcda is flushed immediately before
+/// fork/exec* so the lines before fork are counted once while succeeding
+/// lines are counted twice.
+// UNSUPPORTED: darwin
+/// FIXME: http://lab.llvm.org:8011/builders/clang-ppc64be-linux/builds/50913
+// UNSUPPORTED: host-byteorder-big-endian
+
+// RUN: mkdir -p %t.d && cd %t.d
+// RUN: %clang --coverage %s -o %t
+// RUN: test -f gcov-fork.gcno
+
+// RUN: rm -f gcov-fork.gcda && %run %t
+// RUN: llvm-cov gcov -t gcov-fork.gcda | FileCheck %s
+
+#include <unistd.h>
+
+void func1() {}                    // CHECK:      1: [[#@LINE]]:void func1()
+void func2() {}                    // CHECK-NEXT: 2: [[#@LINE]]:
+int main(void) {                   // CHECK-NEXT: 1: [[#@LINE]]:
+  func1();                         // CHECK-NEXT: 1: [[#@LINE]]:
+  if (fork() == -1) return 1;      // CHECK-NEXT: 1: [[#@LINE]]:
+  func2();                         // CHECK-NEXT: 2: [[#@LINE]]:
+  return 0;                        // CHECK-NEXT: 2: [[#@LINE]]:
+}
diff --git a/compiler-rt/test/profile/Posix/gcov-shared-flush.c b/compiler-rt/test/profile/Posix/gcov-shared-flush.c
new file mode 100644
index 0000000000000..97d44ad5204e1
--- /dev/null
+++ b/compiler-rt/test/profile/Posix/gcov-shared-flush.c
@@ -0,0 +1,113 @@
+/// This test fails on Mac (https://bugs.llvm.org/show_bug.cgi?id=38134)
+// XFAIL: darwin
+
+// RUN: mkdir -p %t.d && cd %t.d
+
+// RUN: %clang -E -DSHARED %s -o shared.c
+// RUN: %clang --coverage -fPIC -shared shared.c -o libfunc.so
+// RUN: test -f shared.gcno
+
+/// Test the case where we exit abruptly after calling __gcov_flush, which means we don't write out the counters at exit.
+// RUN: %clang -DEXIT_ABRUPTLY -DSHARED_CALL_BEFORE_FLUSH -DSHARED_CALL_AFTER_FLUSH --coverage %s -L%t.d -rpath %t.d -lfunc -o %t
+// RUN: test -f gcov-shared-flush.gcno
+
+// RUN: rm -f gcov-shared-flush.gcda shared.gcda
+// RUN: %run %t
+// RUN: llvm-cov gcov -t gcov-shared-flush.gcda | FileCheck %s --check-prefix=NO_WRITEOUT
+// RUN: llvm-cov gcov -t shared.gcda | FileCheck %s --check-prefix=SHARED
+
+// NO_WRITEOUT:      -: [[#%u,L:]]:#ifdef EXIT_ABRUPTLY
+// NO_WRITEOUT-NEXT: 1: [[#%u,L+1]]: _exit(0);
+
+// SHARED: 1: {{[[0-9]+}}:void foo(int n)
+
+/// Test the case where we exit normally and we have a call to the shared library function before __gcov_flush.
+// RUN: %clang -DSHARED_CALL_BEFORE_FLUSH --coverage %s -L%t.d -rpath %t.d -lfunc -o %t
+// RUN: test -f gcov-shared-flush.gcno
+
+// RUN: rm -f gcov-shared-flush.gcda shared.gcda
+// RUN: %run %t
+// RUN: llvm-cov gcov -t gcov-shared-flush.gcda | FileCheck %s --check-prefix=BEFORE
+// RUN: llvm-cov gcov -t shared.gcda | FileCheck %s --check-prefix=SHARED_ONCE
+
+// BEFORE:      -: {{[0-9]+}}:#ifdef SHARED_CALL_BEFORE_FLUSH
+// BEFORE-NEXT: 1: {{[0-9]+}}:  foo(1);
+// BEFORE:      1: {{[0-9]+}}:  __gcov_flush();
+// BEFORE:      -: {{[0-9]+}}:#ifdef SHARED_CALL_AFTER_FLUSH
+// BEFORE-NEXT: -: {{[0-9]+}}:  foo(1);
+// BEFORE:      1: {{[0-9]+}}:  bar(5);
+
+// SHARED_ONCE: 1: {{[0-9]+}}:void foo(int n)
+
+// # Test the case where we exit normally and we have a call to the shared library function after __gcov_flush.
+// RUN: %clang -DSHARED_CALL_AFTER_FLUSH --coverage %s -L%t.d -rpath %t.d -lfunc -o %t
+// RUN: test -f gcov-shared-flush.gcno
+
+// RUN: rm -f gcov-shared-flush.gcda shared.gcda
+// RUN: %run %t
+// RUN: llvm-cov gcov -t gcov-shared-flush.gcda | FileCheck %s --check-prefix=AFTER
+// RUN: llvm-cov gcov -t shared.gcda > 2s.txt
+
+// AFTER:      -: {{[0-9]+}}:#ifdef SHARED_CALL_BEFORE_FLUSH
+// AFTER-NEXT: -: {{[0-9]+}}:  foo(1);
+// AFTER:      1: {{[0-9]+}}:  __gcov_flush();
+// AFTER:      -: {{[0-9]+}}:#ifdef SHARED_CALL_AFTER_FLUSH
+// AFTER-NEXT: 1: {{[0-9]+}}:  foo(1);
+// AFTER:      1: {{[0-9]+}}:  bar(5);
+
+// # Test the case where we exit normally and we have calls to the shared library function before and after __gcov_flush.
+// RUN: %clang -DSHARED_CALL_BEFORE_FLUSH -DSHARED_CALL_AFTER_FLUSH --coverage %s -L%t.d -rpath %t.d -lfunc -o %t
+// RUN: test -f gcov-shared-flush.gcno
+
+// RUN: rm -f gcov-shared-flush.gcda shared.gcda
+// RUN: %run %t
+// RUN: llvm-cov gcov -t gcov-shared-flush.gcda | FileCheck %s --check-prefix=BEFORE_AFTER
+// RUN: llvm-cov gcov -t shared.gcda | FileCheck %s --check-prefix=SHARED_TWICE
+
+// BEFORE_AFTER:      -: {{[0-9]+}}:#ifdef SHARED_CALL_BEFORE_FLUSH
+// BEFORE_AFTER-NEXT: 1: {{[0-9]+}}:  foo(1);
+// BEFORE_AFTER:      1: {{[0-9]+}}:  __gcov_flush();
+// BEFORE_AFTER:      -: {{[0-9]+}}:#ifdef SHARED_CALL_AFTER_FLUSH
+// BEFORE_AFTER-NEXT: 1: {{[0-9]+}}:  foo(1);
+// BEFORE_AFTER:      1: {{[0-9]+}}:  bar(5);
+
+// SHARED_TWICE: 2: {{[0-9]+}}:void foo(int n)
+
+#ifdef SHARED
+void foo(int n) {
+}
+#else
+extern void foo(int n);
+extern void __gcov_flush(void);
+
+int bar1 = 0;
+int bar2 = 1;
+
+void bar(int n) {
+  if (n % 5 == 0)
+    bar1++;
+  else
+    bar2++;
+}
+
+int main(int argc, char *argv[]) {
+#ifdef SHARED_CALL_BEFORE_FLUSH
+  foo(1);
+#endif
+
+  bar(5);
+  __gcov_flush();
+  bar(5);
+
+#ifdef SHARED_CALL_AFTER_FLUSH
+  foo(1);
+#endif
+
+#ifdef EXIT_ABRUPTLY
+  _exit(0);
+#endif
+
+  bar(5);
+  return 0;
+}
+#endif
diff --git a/compiler-rt/test/profile/Posix/instrprof-gcov-execlp.test b/compiler-rt/test/profile/Posix/instrprof-gcov-execlp.test
deleted file mode 100644
index 1d136ce97907e..0000000000000
--- a/compiler-rt/test/profile/Posix/instrprof-gcov-execlp.test
+++ /dev/null
@@ -1,10 +0,0 @@
-RUN: mkdir -p %t.d
-RUN: cd %t.d
-
-RUN: %clang --coverage -o %t %S/../Inputs/instrprof-gcov-execlp.c
-RUN: test -f instrprof-gcov-execlp.gcno
-
-RUN: rm -f instrprof-gcov-execlp.gcda
-RUN: %run %t
-RUN: llvm-cov gcov -b -c instrprof-gcov-execlp.gcda
-RUN: FileCheck --match-full-lines --strict-whitespace --input-file instrprof-gcov-execlp.c.gcov %S/../Inputs/instrprof-gcov-execlp.c.gcov
diff --git a/compiler-rt/test/profile/Posix/instrprof-gcov-execvp.test b/compiler-rt/test/profile/Posix/instrprof-gcov-execvp.test
deleted file mode 100644
index 8e5cbc6095092..0000000000000
--- a/compiler-rt/test/profile/Posix/instrprof-gcov-execvp.test
+++ /dev/null
@@ -1,10 +0,0 @@
-RUN: mkdir -p %t.d
-RUN: cd %t.d
-
-RUN: %clang --coverage -o %t %S/../Inputs/instrprof-gcov-execvp.c
-RUN: test -f instrprof-gcov-execvp.gcno
-
-RUN: rm -f instrprof-gcov-execvp.gcda
-RUN: %run %t
-RUN: llvm-cov gcov -b -c instrprof-gcov-execvp.gcda
-RUN: FileCheck --match-full-lines --strict-whitespace --input-file instrprof-gcov-execvp.c.gcov %S/../Inputs/instrprof-gcov-execvp.c.gcov
diff --git a/compiler-rt/test/profile/Posix/instrprof-gcov-fork.test b/compiler-rt/test/profile/Posix/instrprof-gcov-fork.test
deleted file mode 100644
index 31b6a23be49cd..0000000000000
--- a/compiler-rt/test/profile/Posix/instrprof-gcov-fork.test
+++ /dev/null
@@ -1,13 +0,0 @@
-UNSUPPORTED: linux
-UNSUPPORTED: darwin
-
-RUN: mkdir -p %t.d
-RUN: cd %t.d
-
-RUN: %clang --coverage -o %t %S/../Inputs/instrprof-gcov-fork.c
-RUN: test -f instrprof-gcov-fork.gcno
-
-RUN: rm -f instrprof-gcov-fork.gcda
-RUN: %run %t
-RUN: llvm-cov gcov -b -c instrprof-gcov-fork.gcda
-RUN: FileCheck --match-full-lines --strict-whitespace --input-file instrprof-gcov-fork.c.gcov %S/../Inputs/instrprof-gcov-fork.c.gcov
diff --git a/compiler-rt/test/profile/Posix/instrprof-shared-gcov-flush.test b/compiler-rt/test/profile/Posix/instrprof-shared-gcov-flush.test
deleted file mode 100644
index 8d530fd452a7b..0000000000000
--- a/compiler-rt/test/profile/Posix/instrprof-shared-gcov-flush.test
+++ /dev/null
@@ -1,52 +0,0 @@
-# This test fails on Mac (https://bugs.llvm.org/show_bug.cgi?id=38134)
-XFAIL: darwin
-
-RUN: mkdir -p %t.d
-RUN: cd %t.d
-
-RUN: %clang --coverage -o libfunc.so -fPIC -shared %S/../Inputs/instrprof-shared-lib.c
-RUN: test -f instrprof-shared-lib.gcno
-
-# Test the case where we exit abruptly after calling __gcov_flush, which means we don't write out the counters at exit.
-RUN: %clang -DEXIT_ABRUPTLY -DSHARED_CALL_BEFORE_GCOV_FLUSH -DSHARED_CALL_AFTER_GCOV_FLUSH --coverage -o %t -L%t.d -rpath %t.d -lfunc %S/../Inputs/instrprof-shared-main-gcov-flush.c
-RUN: test -f instrprof-shared-main-gcov-flush.gcno
-
-RUN: rm -f instrprof-shared-main-gcov-flush.gcda instrprof-shared-lib.gcda
-RUN: %run %t
-RUN: llvm-cov gcov instrprof-shared-main-gcov-flush.gcda
-RUN: FileCheck --match-full-lines --strict-whitespace --input-file instrprof-shared-main-gcov-flush.c.gcov %S/../Inputs/instrprof-shared-main-gcov-flush_no-writeout.c.gcov
-RUN: llvm-cov gcov instrprof-shared-lib.gcda
-RUN: FileCheck --match-full-lines --strict-whitespace --input-file instrprof-shared-lib.c.gcov %S/../Inputs/instrprof-shared-lib.c.gcov
-
-# Test the case where we exit normally and we have a call to the shared library function before __gcov_flush.
-RUN: %clang -DSHARED_CALL_BEFORE_GCOV_FLUSH --coverage -o %t -L%t.d -rpath %t.d -lfunc %S/../Inputs/instrprof-shared-main-gcov-flush.c
-RUN: test -f instrprof-shared-main-gcov-flush.gcno
-
-RUN: rm -f instrprof-shared-main-gcov-flush.gcda instrprof-shared-lib.gcda
-RUN: %run %t
-RUN: llvm-cov gcov instrprof-shared-main-gcov-flush.gcda
-RUN: FileCheck --match-full-lines --strict-whitespace --input-file instrprof-shared-main-gcov-flush.c.gcov %S/../Inputs/instrprof-shared-main-gcov-flush_shared-call-before.c.gcov
-RUN: llvm-cov gcov instrprof-shared-lib.gcda
-RUN: FileCheck --match-full-lines --strict-whitespace --input-file instrprof-shared-lib.c.gcov %S/../Inputs/instrprof-shared-lib.c.gcov
-
-# Test the case where we exit normally and we have a call to the shared library function after __gcov_flush.
-RUN: %clang -DSHARED_CALL_AFTER_GCOV_FLUSH --coverage -o %t -L%t.d -rpath %t.d -lfunc %S/../Inputs/instrprof-shared-main-gcov-flush.c
-RUN: test -f instrprof-shared-main-gcov-flush.gcno
-
-RUN: rm -f instrprof-shared-main-gcov-flush.gcda instrprof-shared-lib.gcda
-RUN: %run %t
-RUN: llvm-cov gcov instrprof-shared-main-gcov-flush.gcda
-RUN: FileCheck --match-full-lines --strict-whitespace --input-file instrprof-shared-main-gcov-flush.c.gcov %S/../Inputs/instrprof-shared-main-gcov-flush_shared-call-after.c.gcov
-RUN: llvm-cov gcov instrprof-shared-lib.gcda
-RUN: FileCheck --match-full-lines --strict-whitespace --input-file instrprof-shared-lib.c.gcov %S/../Inputs/instrprof-shared-lib.c.gcov
-
-# Test the case where we exit normally and we have calls to the shared library function before and after __gcov_flush.
-RUN: %clang -DSHARED_CALL_BEFORE_GCOV_FLUSH -DSHARED_CALL_AFTER_GCOV_FLUSH --coverage -o %t -L%t.d -rpath %t.d -lfunc %S/../Inputs/instrprof-shared-main-gcov-flush.c
-RUN: test -f instrprof-shared-main-gcov-flush.gcno
-
-RUN: rm -f instrprof-shared-main-gcov-flush.gcda instrprof-shared-lib.gcda
-RUN: %run %t
-RUN: llvm-cov gcov instrprof-shared-main-gcov-flush.gcda
-RUN: FileCheck --match-full-lines --strict-whitespace --input-file instrprof-shared-main-gcov-flush.c.gcov %S/../Inputs/instrprof-shared-main-gcov-flush_shared-call-before-after.c.gcov
-RUN: llvm-cov gcov instrprof-shared-lib.gcda
-RUN: FileCheck --match-full-lines --strict-whitespace --input-file instrprof-shared-lib.c.gcov %S/../Inputs/instrprof-shared-lib_called-twice.c.gcov
diff --git a/compiler-rt/test/profile/gcov-__gcov_flush-terminate.c b/compiler-rt/test/profile/gcov-__gcov_flush-terminate.c
index c0c443c0e9ddc..5303e045063e8 100644
--- a/compiler-rt/test/profile/gcov-__gcov_flush-terminate.c
+++ b/compiler-rt/test/profile/gcov-__gcov_flush-terminate.c
@@ -9,23 +9,14 @@
 
 // CHECK:             -:    0:Runs:1
 // CHECK-NEXT:        -:    0:Programs:1
-// CHECK:             -:    1:void __gcov_flush(void);
-// CHECK-NEXT:        -:    2:
-// CHECK-NEXT:        1:    3:int main(void) {
-// CHECK-NEXT:        1:    4:  int i = 22;
-// CHECK-NEXT:        1:    5:  __gcov_flush();
-// CHECK-NEXT:        1:    6:  i = 42;
-// CHECK-NEXT:        1:    7:  __builtin_trap();
-// CHECK-NEXT:        1:    8:  i = 84;
-// CHECK-NEXT:        1:    9:  return 0;
 
 void __gcov_flush(void);
 
-int main(void) {
-  int i = 22;
-  __gcov_flush();
-  i = 42;
-  __builtin_trap();
-  i = 84;
-  return 0;
+int main(void) {                   // CHECK:      1: [[#@LINE]]:int main(void)
+  int i = 22;                      // CHECK-NEXT: 1: [[#@LINE]]:
+  __gcov_flush();                  // CHECK-NEXT: 1: [[#@LINE]]:
+  i = 42;                          // CHECK-NEXT: 1: [[#@LINE]]:
+  __builtin_trap();                // CHECK-NEXT: 1: [[#@LINE]]:
+  i = 84;                          // CHECK-NEXT: 1: [[#@LINE]]:
+  return 0;                        // CHECK-NEXT: 1: [[#@LINE]]:
 }
diff --git a/compiler-rt/test/profile/gcov-dump-and-remove.c b/compiler-rt/test/profile/gcov-dump-and-remove.c
new file mode 100644
index 0000000000000..1dcf7b5bd5ca8
--- /dev/null
+++ b/compiler-rt/test/profile/gcov-dump-and-remove.c
@@ -0,0 +1,24 @@
+/// Test we close file handle on flush, so the .gcda file can be deleted on
+/// Windows while the process is still running. In addition, test we create
+/// a new .gcda on flush, so there is a file when the process exists.
+// RUN: mkdir -p %t.d && cd %t.d
+// RUN: %clang --coverage -o %t %s
+// RUN: test -f gcov-dump-and-remove.gcno
+
+// RUN: rm -f gcov-dump-and-remove.gcda && %run %t
+// RUN: llvm-cov gcov -t gcov-dump-and-remove.gcda | FileCheck %s
+
+extern void __gcov_flush(void);
+extern int remove(const char *);   // CHECK:          -: [[#@LINE]]:extern int remove
+int main(void) {                   // CHECK-NEXT: #####: [[#@LINE]]:
+  __gcov_flush();                  // CHECK-NEXT: #####: [[#@LINE]]:
+  if (remove("gcov-dump-and-remove.gcda") != 0) // CHECK-NEXT: #####: [[#@LINE]]:
+    return 1;                      // CHECK-NEXT: #####: [[#@LINE]]: return 1;
+                                   // CHECK-NEXT:     -: [[#@LINE]]:
+  __gcov_flush();                  // CHECK-NEXT: #####: [[#@LINE]]:
+  __gcov_flush();                  // CHECK-NEXT: #####: [[#@LINE]]:
+  if (remove("gcov-dump-and-remove.gcda") != 0) // CHECK-NEXT: #####: [[#@LINE]]:
+    return 1;                      // CHECK-NEXT: #####: [[#@LINE]]: return 1;
+
+  return 0;
+}
diff --git a/compiler-rt/test/profile/instrprof-gcov-__gcov_flush-multiple.test b/compiler-rt/test/profile/instrprof-gcov-__gcov_flush-multiple.test
deleted file mode 100644
index fdc93c93e0b06..0000000000000
--- a/compiler-rt/test/profile/instrprof-gcov-__gcov_flush-multiple.test
+++ /dev/null
@@ -1,10 +0,0 @@
-RUN: mkdir -p %t.d
-RUN: cd %t.d
-
-RUN: %clang --coverage -o %t %S/Inputs/instrprof-gcov-__gcov_flush-multiple.c
-RUN: test -f instrprof-gcov-__gcov_flush-multiple.gcno
-
-RUN: rm -f instrprof-gcov-__gcov_flush-multiple.gcda
-RUN: %run %t
-RUN: llvm-cov gcov instrprof-gcov-__gcov_flush-multiple.gcda
-RUN: FileCheck --match-full-lines --strict-whitespace --input-file instrprof-gcov-__gcov_flush-multiple.c.gcov %S/Inputs/instrprof-gcov-__gcov_flush-multiple.c.gcov
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/protoent.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/protoent.cpp
new file mode 100644
index 0000000000000..a1a93badf6b81
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/protoent.cpp
@@ -0,0 +1,69 @@
+// RUN: %clangxx -std=c++11 -O0 -g %s -o %t && %run %t 2>&1 | FileCheck %s
+// REQUIRES: !android
+
+#include <assert.h>
+#include <errno.h>
+#include <netdb.h>
+#include <stdio.h>
+
+void print_protoent(protoent *curr_entry) {
+  fprintf(stderr, "%s (%d)\n", curr_entry->p_name, curr_entry->p_proto);
+
+  char **aliases = curr_entry->p_aliases;
+  while (char *alias = *aliases++) {
+    fprintf(stderr, "  alias %s\n", alias);
+  }
+}
+
+void print_all_protoent() {
+  protoent entry;
+  char buf[1024];
+  protoent *curr_entry;
+
+  while (getprotoent_r(&entry, buf, sizeof(buf), &curr_entry) != ENOENT && curr_entry) {
+    print_protoent(curr_entry);
+  }
+}
+
+void print_protoent_by_name(const char *name) {
+  protoent entry;
+  char buf[1024];
+  protoent *curr_entry;
+
+  int res = getprotobyname_r(name, &entry, buf, sizeof(buf), &curr_entry);
+  assert(!res && curr_entry);
+  print_protoent(curr_entry);
+}
+
+void print_protoent_by_num(int num) {
+  protoent entry;
+  char buf[1024];
+  protoent *curr_entry;
+
+  int res = getprotobynumber_r(num, &entry, buf, sizeof(buf), &curr_entry);
+  assert(!res && curr_entry);
+  print_protoent(curr_entry);
+}
+
+int main() {
+  // CHECK: All protoent
+  // CHECK: ip (0)
+  // CHECK-NEXT: alias IP
+  // CHECK: ipv6 (41)
+  // CHECK-NEXT: alias IPv6
+  fprintf(stderr, "All protoent\n");
+  print_all_protoent();
+
+  // CHECK: Protoent by name
+  // CHECK-NEXT: ipv6 (41)
+  // CHECK-NEXT: alias IPv6
+  fprintf(stderr, "Protoent by name\n");
+  print_protoent_by_name("ipv6");
+
+  // CHECK: Protoent by num
+  // CHECK-NEXT: udp (17)
+  // CHECK-NEXT: alias UDP
+  fprintf(stderr, "Protoent by num\n");
+  print_protoent_by_num(17);
+  return 0;
+}
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/sysmsg.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/sysmsg.c
new file mode 100644
index 0000000000000..bc7776f2b3d39
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/sysmsg.c
@@ -0,0 +1,36 @@
+// RUN: %clang -O1 %s -o %t && %run %t
+// UNSUPPORTED: android
+#include <assert.h>
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/msg.h>
+
+#define CHECK_STRING "hello, world!"
+#define MSG_BUFLEN 0x100
+
+int main() {
+  int msgq = msgget(IPC_PRIVATE, 0666);
+  assert(msgq != -1);
+
+  struct msg_s {
+    long mtype;
+    char string[MSG_BUFLEN];
+  };
+
+  struct msg_s msg = {
+      .mtype = 1};
+  strcpy(msg.string, CHECK_STRING);
+  int res = msgsnd(msgq, &msg, MSG_BUFLEN, IPC_NOWAIT);
+  if (res) {
+    fprintf(stderr, "Error sending message! %s\n", strerror(errno));
+    return -1;
+  }
+
+  struct msg_s rcv_msg;
+  ssize_t len = msgrcv(msgq, &rcv_msg, MSG_BUFLEN, msg.mtype, IPC_NOWAIT);
+  assert(len == MSG_BUFLEN);
+  assert(msg.mtype == rcv_msg.mtype);
+  assert(!memcmp(msg.string, rcv_msg.string, MSG_BUFLEN));
+  return 0;
+}
diff --git a/debuginfo-tests/CMakeLists.txt b/debuginfo-tests/CMakeLists.txt
index addf13d50f49b..d3ac0a4aad90a 100644
--- a/debuginfo-tests/CMakeLists.txt
+++ b/debuginfo-tests/CMakeLists.txt
@@ -16,11 +16,27 @@ set(DEBUGINFO_TEST_DEPS
   clang
   FileCheck
   count
+  llvm-config
   llvm-objdump
   check-gdb-llvm-support
   not
   )
 
+if("compiler-rt" IN_LIST LLVM_ENABLE_PROJECTS)
+  # llgdb-tests/asan.c and other asan* files.
+  if(TARGET asan)
+    list(APPEND DEBUGINFO_TEST_DEPS asan)
+  endif()
+  # llgdb-tests/safestack.c
+  if(TARGET safestack)
+    list(APPEND DEBUGINFO_TEST_DEPS safestack)
+  endif()
+endif()
+# Many dexter tests depend on lldb.
+if("lldb" IN_LIST LLVM_ENABLE_PROJECTS)
+  list(APPEND DEBUGINFO_TEST_DEPS lldb lldb-server)
+endif()
+
 # The Windows builder scripts pass -fuse-ld=lld.
 if (WIN32)
   set(DEBUGINFO_TEST_DEPS ${DEBUGINFO_TEST_DEPS} lld)
@@ -42,4 +58,9 @@ add_lit_testsuite(check-debuginfo "Running debug info integration tests"
   DEPENDS ${DEBUGINFO_TEST_DEPS}
   )
 
+# Add check-debuginfo-* targets.
+add_lit_testsuites(DEBUGINFO ${CMAKE_CURRENT_SOURCE_DIR}
+  DEPENDS ${DEBUGINFO_TEST_DEPS}
+  )
+
 set_target_properties(check-debuginfo PROPERTIES FOLDER "Debug info tests")
diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/lit.local.cfg b/debuginfo-tests/dexter/feature_tests/commands/perfect/lit.local.cfg
new file mode 100644
index 0000000000000..7d1dd8b7b9026
--- /dev/null
+++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'lldb' in config.available_features:
+    config.unsupported = True
diff --git a/debuginfo-tests/dexter/feature_tests/subtools/view.cpp b/debuginfo-tests/dexter/feature_tests/subtools/view.cpp
index 03e0b9420fb1d..93a51062b4421 100644
--- a/debuginfo-tests/dexter/feature_tests/subtools/view.cpp
+++ b/debuginfo-tests/dexter/feature_tests/subtools/view.cpp
@@ -1,3 +1,4 @@
+// REQUIRES: lldb
 // Purpose:
 //      Check the `view` subtool works with typical inputs.
 //
diff --git a/flang/.clang-tidy b/flang/.clang-tidy
new file mode 100644
index 0000000000000..be9a2b704edd0
--- /dev/null
+++ b/flang/.clang-tidy
@@ -0,0 +1 @@
+Checks: '-*,llvm-*,-llvm-include-order,misc-*,-misc-no-recursion,-misc-unused-parameters,-misc-non-private-member-variables-in-classes'
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 5c4aad9bd65d7..feb9544209bf0 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -50,6 +50,7 @@ option(FLANG_ENABLE_WERROR "Fail and stop building flang if a warning is trigger
 if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
   message("Building Flang as a standalone project.")
   project(Flang)
+  set(FLANG_STANDALONE_BUILD ON)
 
   set(FLANG_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
   if (NOT MSVC_IDE)
@@ -179,6 +180,7 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
   endif()
 
 else()
+  set(FLANG_STANDALONE_BUILD OFF)
   option(FLANG_INCLUDE_TESTS
          "Generate build targets for the Flang unit tests."
          ${LLVM_INCLUDE_TESTS})
diff --git a/flang/documentation/Intrinsics.md b/flang/documentation/Intrinsics.md
index 381f3a3c34b85..8fd06766710c6 100644
--- a/flang/documentation/Intrinsics.md
+++ b/flang/documentation/Intrinsics.md
@@ -701,7 +701,7 @@ This phase currently supports all the intrinsic procedures listed above but the
 
 | Intrinsic Category | Intrinsic Procedures Lacking Support |
 | --- | --- |
-| Coarray intrinsic functions | LCOBOUND, UCOBOUND, FAILED_IMAGES, GET_TEAM, IMAGE_INDEX, NUM_IMAGES, STOPPED_IMAGES, TEAM_NUMBER, THIS_IMAGE, COSHAPE |
+| Coarray intrinsic functions | LCOBOUND, UCOBOUND, FAILED_IMAGES, GET_TEAM, IMAGE_INDEX, STOPPED_IMAGES, TEAM_NUMBER, THIS_IMAGE, COSHAPE |
 | Object characteristic inquiry functions | ALLOCATED, ASSOCIATED, EXTENDS_TYPE_OF, IS_CONTIGUOUS, PRESENT, RANK, SAME_TYPE, STORAGE_SIZE |
 | Type inquiry intrinsic functions | BIT_SIZE, DIGITS, EPSILON, HUGE, KIND, MAXEXPONENT, MINEXPONENT, NEW_LINE, PRECISION, RADIX, RANGE, TINY|
 | Non-standard intrinsic functions | AND, OR, XOR, LSHIFT, RSHIFT, SHIFT, ZEXT, IZEXT, COSD, SIND, TAND, ACOSD, ASIND, ATAND, ATAN2D, COMPL, DCMPLX, EQV, NEQV, INT8, JINT, JNINT, KNINT, LOC, QCMPLX, DREAL, DFLOAT, QEXT, QFLOAT, QREAL, DNUM, NUM, JNUM, KNUM, QNUM, RNUM, RAN, RANF, ILEN, SIZEOF, MCLOCK, SECNDS, COTAN, IBCHNG, ISHA, ISHC, ISHL, IXOR, IARG, IARGC, NARGS, NUMARG, BADDRESS, IADDR, CACHESIZE, EOF, FP_CLASS, INT_PTR_KIND, ISNAN, MALLOC |
diff --git a/flang/include/flang/Common/enum-set.h b/flang/include/flang/Common/enum-set.h
index 5346cceaadfab..a7bdc757a1c97 100644
--- a/flang/include/flang/Common/enum-set.h
+++ b/flang/include/flang/Common/enum-set.h
@@ -16,7 +16,6 @@
 
 #include "constexpr-bitset.h"
 #include "idioms.h"
-#include "llvm/Support/raw_ostream.h"
 #include <bitset>
 #include <cstddef>
 #include <initializer_list>
@@ -190,7 +189,7 @@ template <typename ENUM, std::size_t BITS> class EnumSet {
       // std::bitset: just iterate
       for (std::size_t j{0}; j < BITS; ++j) {
         auto enumerator{static_cast<enumerationType>(j)};
-        if (bitset_.test(enumerator)) {
+        if (bitset_.test(j)) {
           return {enumerator};
         }
       }
@@ -206,8 +205,8 @@ template <typename ENUM, std::size_t BITS> class EnumSet {
     }
   }
 
-  llvm::raw_ostream &Dump(
-      llvm::raw_ostream &o, std::string EnumToString(enumerationType)) const {
+  template <typename STREAM>
+  STREAM &Dump(STREAM &o, std::string EnumToString(enumerationType)) const {
     char sep{'{'};
     IterateOverMembers([&](auto e) {
       o << sep << EnumToString(e);
diff --git a/flang/include/flang/Common/idioms.h b/flang/include/flang/Common/idioms.h
index 3f4f8f2eec6ba..6e499eb04f36a 100644
--- a/flang/include/flang/Common/idioms.h
+++ b/flang/include/flang/Common/idioms.h
@@ -23,6 +23,7 @@
 #error g++ >= 7.2 is required
 #endif
 
+#include "llvm/Support/Compiler.h"
 #include <functional>
 #include <list>
 #include <optional>
@@ -108,10 +109,6 @@ template <typename... LAMBDAS> visitors(LAMBDAS... x) -> visitors<LAMBDAS...>;
   } \
   template <typename A> constexpr bool T{class_trait_ns_##T::trait_value<A>()};
 
-#if !defined ATTRIBUTE_UNUSED && (__clang__ || __GNUC__)
-#define ATTRIBUTE_UNUSED __attribute__((unused))
-#endif
-
 // Define enum class NAME with the given enumerators, a static
 // function EnumToString() that maps enumerators to std::string,
 // and a constant NAME_enumSize that captures the number of items
@@ -126,11 +123,11 @@ template <typename A> struct ListItemCount {
 
 #define ENUM_CLASS(NAME, ...) \
   enum class NAME { __VA_ARGS__ }; \
-  ATTRIBUTE_UNUSED static constexpr std::size_t NAME##_enumSize{[] { \
+  LLVM_ATTRIBUTE_UNUSED static constexpr std::size_t NAME##_enumSize{[] { \
     enum { __VA_ARGS__ }; \
     return Fortran::common::ListItemCount{__VA_ARGS__}.value; \
   }()}; \
-  ATTRIBUTE_UNUSED static inline std::string EnumToString(NAME e) { \
+  LLVM_ATTRIBUTE_UNUSED static inline std::string EnumToString(NAME e) { \
     return Fortran::common::EnumIndexToString( \
         static_cast<int>(e), #__VA_ARGS__); \
   }
diff --git a/flang/include/flang/Common/indirection.h b/flang/include/flang/Common/indirection.h
index f34d561ae0492..fec3bc7ae8dad 100644
--- a/flang/include/flang/Common/indirection.h
+++ b/flang/include/flang/Common/indirection.h
@@ -122,20 +122,49 @@ template <typename A> class Indirection<A, true> {
 
 template <typename A> using CopyableIndirection = Indirection<A, true>;
 
-// For use with std::unique_ptr<> when declaring owning pointers to
-// forward-referenced types, here's a minimal custom deleter that avoids
-// some of the drama with std::default_delete<>.  Invoke DEFINE_DELETER()
-// later in exactly one C++ source file where a complete definition of the
-// type is visible.  Be advised, std::unique_ptr<> does not have copy
-// semantics; if you need ownership, copy semantics, and nullability,
-// std::optional<CopyableIndirection<>> works.
-template <typename A> class Deleter {
+// A variation of std::unique_ptr<> with a reified deletion routine.
+// Used to avoid dependence cycles between shared libraries.
+template <typename A> class ForwardOwningPointer {
 public:
-  void operator()(A *) const;
+  ForwardOwningPointer() {}
+  ForwardOwningPointer(A *p, void (*del)(A *)) : p_{p}, deleter_{del} {}
+  ForwardOwningPointer(ForwardOwningPointer &&that)
+      : p_{that.p_}, deleter_{that.deleter_} {
+    that.p_ = nullptr;
+  }
+  ForwardOwningPointer &operator=(ForwardOwningPointer &&that) {
+    p_ = that.p_;
+    that.p_ = nullptr;
+    deleter_ = that.deleter_;
+    return *this;
+  }
+  ~ForwardOwningPointer() {
+    if (p_) {
+      deleter_(p_);
+    }
+  }
+
+  A &operator*() const { return *p_; }
+  A *operator->() const { return p_; }
+  operator bool() const { return p_ != nullptr; }
+  A *get() { return p_; }
+  A *release() {
+    A *result{p_};
+    p_ = nullptr;
+    return result;
+  }
+
+  void Reset(A *p, void (*del)(A *)) {
+    if (p_) {
+      deleter_(p_);
+    }
+    p_ = p;
+    deleter_ = del;
+  }
+
+private:
+  A *p_{nullptr};
+  void (*deleter_)(A *){nullptr};
 };
 } // namespace Fortran::common
-#define DEFINE_DELETER(A) \
-  template <> void Fortran::common::Deleter<A>::operator()(A *p) const { \
-    delete p; \
-  }
 #endif // FORTRAN_COMMON_INDIRECTION_H_
diff --git a/flang/include/flang/Evaluate/call.h b/flang/include/flang/Evaluate/call.h
index 15fe5b879dc4c..71e0610549282 100644
--- a/flang/include/flang/Evaluate/call.h
+++ b/flang/include/flang/Evaluate/call.h
@@ -190,9 +190,12 @@ struct ProcedureDesignator {
 class ProcedureRef {
 public:
   CLASS_BOILERPLATE(ProcedureRef)
-  ProcedureRef(ProcedureDesignator &&p, ActualArguments &&a)
-      : proc_{std::move(p)}, arguments_(std::move(a)) {}
+  ProcedureRef(ProcedureDesignator &&p, ActualArguments &&a,
+      bool hasAlternateReturns = false)
+      : proc_{std::move(p)}, arguments_{std::move(a)},
+        hasAlternateReturns_{hasAlternateReturns} {}
   ~ProcedureRef();
+  static void Deleter(ProcedureRef *);
 
   ProcedureDesignator &proc() { return proc_; }
   const ProcedureDesignator &proc() const { return proc_; }
@@ -202,12 +205,14 @@ class ProcedureRef {
   std::optional<Expr<SubscriptInteger>> LEN() const;
   int Rank() const;
   bool IsElemental() const { return proc_.IsElemental(); }
+  bool hasAlternateReturns() const { return hasAlternateReturns_; }
   bool operator==(const ProcedureRef &) const;
   llvm::raw_ostream &AsFortran(llvm::raw_ostream &) const;
 
 protected:
   ProcedureDesignator proc_;
   ActualArguments arguments_;
+  bool hasAlternateReturns_;
 };
 
 template <typename A> class FunctionRef : public ProcedureRef {
diff --git a/flang/include/flang/Evaluate/expression.h b/flang/include/flang/Evaluate/expression.h
index 095783231b660..3569b68fd9d2e 100644
--- a/flang/include/flang/Evaluate/expression.h
+++ b/flang/include/flang/Evaluate/expression.h
@@ -841,6 +841,7 @@ struct GenericExprWrapper {
   explicit GenericExprWrapper(std::optional<Expr<SomeType>> &&x)
       : v{std::move(x)} {}
   ~GenericExprWrapper();
+  static void Deleter(GenericExprWrapper *);
   std::optional<Expr<SomeType>> v; // vacant if error
 };
 
@@ -849,6 +850,7 @@ struct GenericAssignmentWrapper {
   GenericAssignmentWrapper() {}
   explicit GenericAssignmentWrapper(Assignment &&x) : v{std::move(x)} {}
   ~GenericAssignmentWrapper();
+  static void Deleter(GenericAssignmentWrapper *);
   std::optional<Assignment> v; // vacant if error
 };
 
diff --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index 84de29bd7fec8..e64aaece15bf8 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -908,6 +908,7 @@ bool IsProcedure(const Symbol &);
 bool IsProcedurePointer(const Symbol &);
 bool IsSaved(const Symbol &); // saved implicitly or explicitly
 bool IsDummy(const Symbol &);
+bool IsFunctionResult(const Symbol &);
 
 // Follow use, host, and construct assocations to a variable, if any.
 const Symbol *GetAssociationRoot(const Symbol &);
diff --git a/flang/include/flang/Evaluate/type.h b/flang/include/flang/Evaluate/type.h
index 197fb3291dd42..c62d633563430 100644
--- a/flang/include/flang/Evaluate/type.h
+++ b/flang/include/flang/Evaluate/type.h
@@ -146,7 +146,7 @@ class DynamicType {
   DynamicType ResultTypeForMultiply(const DynamicType &) const;
 
   bool IsAssumedLengthCharacter() const;
-  bool IsUnknownLengthCharacter() const;
+  bool IsNonConstantLengthCharacter() const;
   bool IsTypelessIntrinsicArgument() const;
   constexpr bool IsAssumedType() const { // TYPE(*)
     return kind_ == AssumedTypeKind;
diff --git a/flang/include/flang/Lower/.clang-tidy b/flang/include/flang/Lower/.clang-tidy
new file mode 100644
index 0000000000000..87ec2ff53af6e
--- /dev/null
+++ b/flang/include/flang/Lower/.clang-tidy
@@ -0,0 +1,19 @@
+# Almost identical to the top-level .clang-tidy, except that {Member,Parameter,Variable}Case use camelBack.
+Checks: '-*,clang-diagnostic-*,llvm-*,misc-*,-misc-unused-parameters,-misc-non-private-member-variables-in-classes,readability-identifier-naming'
+CheckOptions:
+  - key:             readability-identifier-naming.ClassCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.EnumCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.FunctionCase
+    value:           camelBack
+  - key:             readability-identifier-naming.MemberCase
+    value:           camelBack
+  - key:             readability-identifier-naming.ParameterCase
+    value:           camelBack
+  - key:             readability-identifier-naming.UnionCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.VariableCase
+    value:           camelBack
+  - key:             readability-identifier-naming.IgnoreMainLikeFunctions
+    value:           1
diff --git a/flang/include/flang/Lower/AbstractConverter.h b/flang/include/flang/Lower/AbstractConverter.h
index 0716b841fa1e7..190e69230c8e1 100644
--- a/flang/include/flang/Lower/AbstractConverter.h
+++ b/flang/include/flang/Lower/AbstractConverter.h
@@ -9,11 +9,127 @@
 #ifndef FORTRAN_LOWER_ABSTRACTCONVERTER_H
 #define FORTRAN_LOWER_ABSTRACTCONVERTER_H
 
-namespace Fortran::lower {
+#include "flang/Common/Fortran.h"
+#include "mlir/IR/Module.h"
 
-// temporary stub
-class AbstractConverter;
+namespace Fortran {
+namespace common {
+template <typename>
+class Reference;
+}
+namespace evaluate {
+struct DataRef;
+template <typename>
+class Expr;
+class FoldingContext;
+struct SomeType;
+} // namespace evaluate
 
-} // namespace Fortran::lower
+namespace parser {
+class CharBlock;
+}
+namespace semantics {
+class Symbol;
+}
+
+namespace lower {
+namespace pft {
+struct Variable;
+}
+
+using SomeExpr = Fortran::evaluate::Expr<Fortran::evaluate::SomeType>;
+using SymbolRef = Fortran::common::Reference<const Fortran::semantics::Symbol>;
+class FirOpBuilder;
+
+//===----------------------------------------------------------------------===//
+// AbstractConverter interface
+//===----------------------------------------------------------------------===//
+
+/// The abstract interface for converter implementations to lower Fortran
+/// front-end fragments such as expressions, types, etc. to the FIR dialect of
+/// MLIR.
+class AbstractConverter {
+public:
+  //===--------------------------------------------------------------------===//
+  // Symbols
+  //===--------------------------------------------------------------------===//
+
+  /// Get the mlir instance of a symbol.
+  virtual mlir::Value getSymbolAddress(SymbolRef sym) = 0;
+
+  //===--------------------------------------------------------------------===//
+  // Expressions
+  //===--------------------------------------------------------------------===//
+
+  /// Generate the address of the location holding the expression, someExpr
+  virtual mlir::Value genExprAddr(const SomeExpr &,
+                                  mlir::Location *loc = nullptr) = 0;
+  /// Generate the address of the location holding the expression, someExpr
+  mlir::Value genExprAddr(const SomeExpr *someExpr, mlir::Location loc) {
+    return genExprAddr(*someExpr, &loc);
+  }
+
+  /// Generate the computations of the expression to produce a value
+  virtual mlir::Value genExprValue(const SomeExpr &,
+                                   mlir::Location *loc = nullptr) = 0;
+  /// Generate the computations of the expression, someExpr, to produce a value
+  mlir::Value genExprValue(const SomeExpr *someExpr, mlir::Location loc) {
+    return genExprValue(*someExpr, &loc);
+  }
+
+  /// Get FoldingContext that is required for some expression
+  /// analysis.
+  virtual Fortran::evaluate::FoldingContext &getFoldingContext() = 0;
+
+  //===--------------------------------------------------------------------===//
+  // Types
+  //===--------------------------------------------------------------------===//
+
+  /// Generate the type of a DataRef
+  virtual mlir::Type genType(const Fortran::evaluate::DataRef &) = 0;
+  /// Generate the type of an Expr
+  virtual mlir::Type genType(const SomeExpr &) = 0;
+  /// Generate the type of a Symbol
+  virtual mlir::Type genType(SymbolRef) = 0;
+  /// Generate the type from a category
+  virtual mlir::Type genType(Fortran::common::TypeCategory tc) = 0;
+  /// Generate the type from a category and kind
+  virtual mlir::Type genType(Fortran::common::TypeCategory tc, int kind) = 0;
+  /// Generate the type from a Variable
+  virtual mlir::Type genType(const pft::Variable &) = 0;
+
+  //===--------------------------------------------------------------------===//
+  // Locations
+  //===--------------------------------------------------------------------===//
+
+  /// Get the converter's current location
+  virtual mlir::Location getCurrentLocation() = 0;
+  /// Generate a dummy location
+  virtual mlir::Location genLocation() = 0;
+  /// Generate the location as converted from a CharBlock
+  virtual mlir::Location genLocation(const Fortran::parser::CharBlock &) = 0;
+
+  //===--------------------------------------------------------------------===//
+  // FIR/MLIR
+  //===--------------------------------------------------------------------===//
+
+  /// Get the OpBuilder
+  virtual Fortran::lower::FirOpBuilder &getFirOpBuilder() = 0;
+  /// Get the ModuleOp
+  virtual mlir::ModuleOp &getModuleOp() = 0;
+  /// Get the MLIRContext
+  virtual mlir::MLIRContext &getMLIRContext() = 0;
+  /// Unique a symbol
+  virtual std::string mangleName(const Fortran::semantics::Symbol &) = 0;
+  /// Unique a compiler generated identifier. A short prefix should be provided
+  /// to hint at the origin of the identifier.
+  virtual std::string uniqueCGIdent(llvm::StringRef prefix,
+                                    llvm::StringRef name) = 0;
+
+  virtual ~AbstractConverter() = default;
+};
+
+} // namespace lower
+} // namespace Fortran
 
 #endif // FORTRAN_LOWER_ABSTRACTCONVERTER_H
diff --git a/flang/include/flang/Lower/Bridge.h b/flang/include/flang/Lower/Bridge.h
new file mode 100644
index 0000000000000..aee7a0ef5bd8d
--- /dev/null
+++ b/flang/include/flang/Lower/Bridge.h
@@ -0,0 +1,117 @@
+//===-- Lower/Bridge.h -- main interface to lowering ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Implements lowering. Convert Fortran source to
+/// [MLIR](https://github.com/tensorflow/mlir).
+///
+/// [Coding style](https://llvm.org/docs/CodingStandards.html)
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_LOWER_BRIDGE_H
+#define FORTRAN_LOWER_BRIDGE_H
+
+#include "flang/Common/Fortran.h"
+#include "flang/Lower/AbstractConverter.h"
+#include "flang/Optimizer/Support/KindMapping.h"
+#include "mlir/IR/Module.h"
+
+namespace fir {
+struct NameUniquer;
+}
+
+namespace Fortran {
+namespace common {
+class IntrinsicTypeDefaultKinds;
+} // namespace common
+namespace evaluate {
+class IntrinsicProcTable;
+} // namespace evaluate
+namespace parser {
+class CookedSource;
+struct Program;
+} // namespace parser
+namespace semantics {
+class SemanticsContext;
+} // namespace semantics
+
+namespace lower {
+
+//===----------------------------------------------------------------------===//
+// Lowering bridge
+//===----------------------------------------------------------------------===//
+
+/// The lowering bridge converts the front-end parse trees and semantics
+/// checking residual to MLIR (FIR dialect) code.
+class LoweringBridge {
+public:
+  /// Create a lowering bridge instance.
+  static LoweringBridge
+  create(const Fortran::common::IntrinsicTypeDefaultKinds &defaultKinds,
+         const Fortran::evaluate::IntrinsicProcTable &intrinsics,
+         const Fortran::parser::CookedSource &cooked) {
+    return LoweringBridge{defaultKinds, intrinsics, cooked};
+  }
+
+  //===--------------------------------------------------------------------===//
+  // Getters
+  //===--------------------------------------------------------------------===//
+
+  mlir::MLIRContext &getMLIRContext() { return *context.get(); }
+  mlir::ModuleOp &getModule() { return *module.get(); }
+  const Fortran::common::IntrinsicTypeDefaultKinds &getDefaultKinds() const {
+    return defaultKinds;
+  }
+  const Fortran::evaluate::IntrinsicProcTable &getIntrinsicTable() const {
+    return intrinsics;
+  }
+  const Fortran::parser::CookedSource *getCookedSource() const {
+    return cooked;
+  }
+
+  /// Get the kind map.
+  const fir::KindMapping &getKindMap() const { return kindMap; }
+
+  /// Create a folding context. Careful: this is very expensive.
+  Fortran::evaluate::FoldingContext createFoldingContext() const;
+
+  bool validModule() { return getModule(); }
+
+  //===--------------------------------------------------------------------===//
+  // Perform the creation of an mlir::ModuleOp
+  //===--------------------------------------------------------------------===//
+
+  /// Read in an MLIR input file rather than lowering Fortran sources.
+  /// This is intended to be used for testing.
+  void parseSourceFile(llvm::SourceMgr &);
+
+  /// Cross the bridge from the Fortran parse-tree, etc. to MLIR dialects
+  void lower(const Fortran::parser::Program &program, fir::NameUniquer &uniquer,
+             const Fortran::semantics::SemanticsContext &semanticsContext);
+
+private:
+  explicit LoweringBridge(
+      const Fortran::common::IntrinsicTypeDefaultKinds &defaultKinds,
+      const Fortran::evaluate::IntrinsicProcTable &intrinsics,
+      const Fortran::parser::CookedSource &cooked);
+  LoweringBridge() = delete;
+  LoweringBridge(const LoweringBridge &) = delete;
+
+  const Fortran::common::IntrinsicTypeDefaultKinds &defaultKinds;
+  const Fortran::evaluate::IntrinsicProcTable &intrinsics;
+  const Fortran::parser::CookedSource *cooked;
+  std::unique_ptr<mlir::MLIRContext> context;
+  std::unique_ptr<mlir::ModuleOp> module;
+  fir::KindMapping kindMap;
+};
+
+} // namespace lower
+} // namespace Fortran
+
+#endif // FORTRAN_LOWER_BRIDGE_H
diff --git a/flang/include/flang/Lower/CharacterExpr.h b/flang/include/flang/Lower/CharacterExpr.h
new file mode 100644
index 0000000000000..6f75448a5dbbf
--- /dev/null
+++ b/flang/include/flang/Lower/CharacterExpr.h
@@ -0,0 +1,153 @@
+//===-- Lower/CharacterExpr.h -- lowering of characters ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_LOWER_CHARACTEREXPR_H
+#define FORTRAN_LOWER_CHARACTEREXPR_H
+
+#include "flang/Lower/FIRBuilder.h"
+#include "flang/Lower/Support/BoxValue.h"
+
+namespace Fortran::lower {
+
+/// Helper to facilitate lowering of CHARACTER in FIR.
+class CharacterExprHelper {
+public:
+  /// Constructor.
+  explicit CharacterExprHelper(FirOpBuilder &builder, mlir::Location loc)
+      : builder{builder}, loc{loc} {}
+  CharacterExprHelper(const CharacterExprHelper &) = delete;
+
+  /// Unless otherwise stated, all mlir::Value inputs of these pseudo-fir ops
+  /// must be of type:
+  /// - fir.boxchar<kind> (dynamic length character),
+  /// - fir.ref<fir.array<len x fir.char<kind>>> (character with compile time
+  ///      constant length),
+  /// - fir.array<len x fir.char<kind>> (compile time constant character)
+
+  /// Copy the \p count first characters of \p src into \p dest.
+  /// \p count can have any integer type.
+  void createCopy(mlir::Value dest, mlir::Value src, mlir::Value count);
+
+  /// Set characters of \p str at position [\p lower, \p upper) to blanks.
+  /// \p lower and \upper bounds are zero based.
+  /// If \p upper <= \p lower, no padding is done.
+  /// \p upper and \p lower can have any integer type.
+  void createPadding(mlir::Value str, mlir::Value lower, mlir::Value upper);
+
+  /// Create str(lb:ub), lower bounds must always be specified, upper
+  /// bound is optional.
+  mlir::Value createSubstring(mlir::Value str,
+                              llvm::ArrayRef<mlir::Value> bounds);
+
+  /// Return blank character of given \p type !fir.char<kind>
+  mlir::Value createBlankConstant(fir::CharacterType type);
+
+  /// Lower \p lhs = \p rhs where \p lhs and \p rhs are scalar characters.
+  /// It handles cases where \p lhs and \p rhs may overlap.
+  void createAssign(mlir::Value lhs, mlir::Value rhs);
+
+  /// Lower an assignment where the buffer and LEN parameter are known and do
+  /// not need to be unboxed.
+  void createAssign(mlir::Value lptr, mlir::Value llen, mlir::Value rptr,
+                    mlir::Value rlen);
+
+  /// Create lhs // rhs in temp obtained with fir.alloca
+  mlir::Value createConcatenate(mlir::Value lhs, mlir::Value rhs);
+
+  /// LEN_TRIM intrinsic.
+  mlir::Value createLenTrim(mlir::Value str);
+
+  /// Embox \p addr and \p len and return fir.boxchar.
+  /// Take care of type conversions before emboxing.
+  /// \p len is converted to the integer type for character lengths if needed.
+  mlir::Value createEmboxChar(mlir::Value addr, mlir::Value len);
+
+  /// Unbox \p boxchar into (fir.ref<fir.char<kind>>, getLengthType()).
+  std::pair<mlir::Value, mlir::Value> createUnboxChar(mlir::Value boxChar);
+
+  /// Allocate a temp of fir::CharacterType type and length len.
+  /// Returns related fir.ref<fir.char<kind>>.
+  mlir::Value createCharacterTemp(mlir::Type type, mlir::Value len);
+
+  /// Allocate a temp of compile time constant length.
+  /// Returns related fir.ref<fir.array<len x fir.char<kind>>>.
+  mlir::Value createCharacterTemp(mlir::Type type, int len) {
+    return createTemp(type, len);
+  }
+
+  /// Return buffer/length pair of character str, if str is a constant,
+  /// it is allocated into a temp, otherwise, its memory reference is
+  /// returned as the buffer.
+  /// The buffer type of str is of type:
+  ///   - fir.ref<fir.array<len x fir.char<kind>>> if str has compile time
+  ///      constant length.
+  ///   - fir.ref<fir.char<kind>> if str has dynamic length.
+  std::pair<mlir::Value, mlir::Value> materializeCharacter(mlir::Value str);
+
+  /// Return true if \p type is a character literal type (is
+  /// fir.array<len x fir.char<kind>>).;
+  static bool isCharacterLiteral(mlir::Type type);
+
+  /// Return true if \p type is one of the following type
+  /// - fir.boxchar<kind>
+  /// - fir.ref<fir.array<len x fir.char<kind>>>
+  /// - fir.array<len x fir.char<kind>>
+  static bool isCharacter(mlir::Type type);
+
+  /// Extract the kind of a character type
+  static int getCharacterKind(mlir::Type type);
+
+  /// Return the integer type that must be used to manipulate
+  /// Character lengths. TODO: move this to FirOpBuilder?
+  mlir::Type getLengthType() { return builder.getIndexType(); }
+
+  /// Create an extended value from:
+  /// - fir.boxchar<kind>
+  /// - fir.ref<fir.array<len x fir.char<kind>>>
+  /// - fir.array<len x fir.char<kind>>
+  /// - fir.char<kind>
+  /// - fir.ref<char<kind>>
+  /// If the no length is passed, it is attempted to be extracted from \p
+  /// character (or its type). This will crash if this is not possible.
+  /// The returned value is a CharBoxValue if \p character is a scalar,
+  /// otherwise it is a CharArrayBoxValue.
+  fir::ExtendedValue toExtendedValue(mlir::Value character,
+                                     mlir::Value len = {});
+
+private:
+  fir::CharBoxValue materializeValue(const fir::CharBoxValue &str);
+  fir::CharBoxValue toDataLengthPair(mlir::Value character);
+  mlir::Type getReferenceType(const fir::CharBoxValue &c) const;
+  mlir::Value createEmbox(const fir::CharBoxValue &str);
+  mlir::Value createLoadCharAt(const fir::CharBoxValue &str, mlir::Value index);
+  void createStoreCharAt(const fir::CharBoxValue &str, mlir::Value index,
+                         mlir::Value c);
+  void createCopy(const fir::CharBoxValue &dest, const fir::CharBoxValue &src,
+                  mlir::Value count);
+  void createPadding(const fir::CharBoxValue &str, mlir::Value lower,
+                     mlir::Value upper);
+  fir::CharBoxValue createTemp(mlir::Type type, mlir::Value len);
+  void createLengthOneAssign(const fir::CharBoxValue &lhs,
+                             const fir::CharBoxValue &rhs);
+  void createAssign(const fir::CharBoxValue &lhs, const fir::CharBoxValue &rhs);
+  fir::CharBoxValue createConcatenate(const fir::CharBoxValue &lhs,
+                                      const fir::CharBoxValue &rhs);
+  fir::CharBoxValue createSubstring(const fir::CharBoxValue &str,
+                                    llvm::ArrayRef<mlir::Value> bounds);
+  mlir::Value createLenTrim(const fir::CharBoxValue &str);
+  mlir::Value createTemp(mlir::Type type, int len);
+  mlir::Value createBlankConstantCode(fir::CharacterType type);
+
+private:
+  FirOpBuilder &builder;
+  mlir::Location loc;
+};
+
+} // namespace Fortran::lower
+
+#endif // FORTRAN_LOWER_CHARACTEREXPR_H
diff --git a/flang/include/flang/Lower/CharacterRuntime.h b/flang/include/flang/Lower/CharacterRuntime.h
new file mode 100644
index 0000000000000..d2992f76406ae
--- /dev/null
+++ b/flang/include/flang/Lower/CharacterRuntime.h
@@ -0,0 +1,36 @@
+//===-- Lower/CharacterRuntime.h -- lower CHARACTER operations --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_LOWER_CHARACTERRUNTIME_H
+#define FORTRAN_LOWER_CHARACTERRUNTIME_H
+
+#include "mlir/Dialect/StandardOps/IR/Ops.h"
+
+namespace Fortran {
+namespace lower {
+class AbstractConverter;
+
+/// Generate call to a character comparison for two ssa-values of type
+/// `boxchar`.
+mlir::Value genBoxCharCompare(AbstractConverter &converter, mlir::Location loc,
+                              mlir::CmpIPredicate cmp, mlir::Value lhs,
+                              mlir::Value rhs);
+
+/// Generate call to a character comparison op for two unboxed variables. There
+/// are 4 arguments, 2 for the lhs and 2 for the rhs. Each CHARACTER must pass a
+/// reference to its buffer (`ref<char<K>>`) and its LEN type parameter (some
+/// integral type).
+mlir::Value genRawCharCompare(AbstractConverter &converter, mlir::Location loc,
+                              mlir::CmpIPredicate cmp, mlir::Value lhsBuff,
+                              mlir::Value lhsLen, mlir::Value rhsBuff,
+                              mlir::Value rhsLen);
+
+} // namespace lower
+} // namespace Fortran
+
+#endif // FORTRAN_LOWER_CHARACTERRUNTIME_H
diff --git a/flang/include/flang/Lower/Coarray.h b/flang/include/flang/Lower/Coarray.h
new file mode 100644
index 0000000000000..e32c82d36e55e
--- /dev/null
+++ b/flang/include/flang/Lower/Coarray.h
@@ -0,0 +1,78 @@
+//===-- Lower/Coarray.h -- image related lowering ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_LOWER_COARRAY_H
+#define FORTRAN_LOWER_COARRAY_H
+
+#include "flang/Lower/AbstractConverter.h"
+#include "flang/Lower/Support/BoxValue.h"
+
+namespace Fortran {
+
+namespace parser {
+struct ChangeTeamConstruct;
+struct ChangeTeamStmt;
+struct EndChangeTeamStmt;
+struct FormTeamStmt;
+} // namespace parser
+
+namespace evaluate {
+class CoarrayRef;
+} // namespace evaluate
+
+namespace lower {
+
+class SymMap;
+
+namespace pft {
+struct Evaluation;
+} // namespace pft
+
+//===----------------------------------------------------------------------===//
+// TEAM constructs
+//===----------------------------------------------------------------------===//
+
+void genChangeTeamConstruct(AbstractConverter &, pft::Evaluation &eval,
+                            const parser::ChangeTeamConstruct &);
+void genChangeTeamStmt(AbstractConverter &, pft::Evaluation &eval,
+                       const parser::ChangeTeamStmt &);
+void genEndChangeTeamStmt(AbstractConverter &, pft::Evaluation &eval,
+                          const parser::EndChangeTeamStmt &);
+void genFormTeamStatement(AbstractConverter &, pft::Evaluation &eval,
+                          const parser::FormTeamStmt &);
+
+//===----------------------------------------------------------------------===//
+// COARRAY expressions
+//===----------------------------------------------------------------------===//
+
+/// Coarray expression lowering helper. A coarray expression is expected to be
+/// lowered into runtime support calls. For example, expressions may use a
+/// message-passing runtime to access another image's data.
+class CoarrayExprHelper {
+public:
+  explicit CoarrayExprHelper(AbstractConverter &converter, mlir::Location loc,
+                             SymMap &syms)
+      : converter{converter}, symMap{syms}, loc{loc} {}
+  CoarrayExprHelper(const CoarrayExprHelper &) = delete;
+
+  /// Generate the address of a co-array expression.
+  fir::ExtendedValue genAddr(const evaluate::CoarrayRef &expr);
+
+  /// Generate the value of a co-array expression.
+  fir::ExtendedValue genValue(const evaluate::CoarrayRef &expr);
+
+private:
+  AbstractConverter &converter;
+  SymMap &symMap;
+  mlir::Location loc;
+};
+
+} // namespace lower
+} // namespace Fortran
+
+#endif // FORTRAN_LOWER_COARRAY_H
diff --git a/flang/include/flang/Lower/FIRBuilder.h b/flang/include/flang/Lower/FIRBuilder.h
index f1fb15048c955..fa9efe267b54a 100644
--- a/flang/include/flang/Lower/FIRBuilder.h
+++ b/flang/include/flang/Lower/FIRBuilder.h
@@ -70,6 +70,10 @@ class FirOpBuilder : public mlir::OpBuilder {
   /// Safely create a reference type to the type `eleTy`.
   mlir::Type getRefType(mlir::Type eleTy);
 
+  /// Create a null constant of type RefType and value 0. Need to pass in the
+  /// Location information.
+  mlir::Value createNullConstant(mlir::Location loc);
+
   /// Create an integer constant of type \p type and value \p i.
   mlir::Value createIntegerConstant(mlir::Location loc, mlir::Type integerType,
                                     std::int64_t i);
diff --git a/flang/include/flang/Lower/IO.h b/flang/include/flang/Lower/IO.h
new file mode 100644
index 0000000000000..9d5147f8e42a9
--- /dev/null
+++ b/flang/include/flang/Lower/IO.h
@@ -0,0 +1,98 @@
+//===-- Lower/IO.h -- lower I/O statements ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_LOWER_IO_H
+#define FORTRAN_LOWER_IO_H
+
+#include "flang/Common/reference.h"
+#include "flang/Semantics/symbol.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallSet.h"
+
+namespace mlir {
+class Value;
+} // namespace mlir
+
+namespace Fortran {
+namespace parser {
+using Label = std::uint64_t;
+struct BackspaceStmt;
+struct CloseStmt;
+struct EndfileStmt;
+struct FlushStmt;
+struct InquireStmt;
+struct OpenStmt;
+struct PrintStmt;
+struct ReadStmt;
+struct RewindStmt;
+struct WaitStmt;
+struct WriteStmt;
+} // namespace parser
+
+namespace lower {
+
+class AbstractConverter;
+class BridgeImpl;
+
+namespace pft {
+struct Evaluation;
+using LabelEvalMap = llvm::DenseMap<Fortran::parser::Label, Evaluation *>;
+using SymbolRef = Fortran::common::Reference<const Fortran::semantics::Symbol>;
+using LabelSet = llvm::SmallSet<Fortran::parser::Label, 5>;
+using SymbolLabelMap = llvm::DenseMap<SymbolRef, LabelSet>;
+} // namespace pft
+
+/// Generate IO call(s) for BACKSPACE; return the IOSTAT code
+mlir::Value genBackspaceStatement(AbstractConverter &,
+                                  const parser::BackspaceStmt &);
+
+/// Generate IO call(s) for CLOSE; return the IOSTAT code
+mlir::Value genCloseStatement(AbstractConverter &, const parser::CloseStmt &);
+
+/// Generate IO call(s) for ENDFILE; return the IOSTAT code
+mlir::Value genEndfileStatement(AbstractConverter &,
+                                const parser::EndfileStmt &);
+
+/// Generate IO call(s) for FLUSH; return the IOSTAT code
+mlir::Value genFlushStatement(AbstractConverter &, const parser::FlushStmt &);
+
+/// Generate IO call(s) for INQUIRE; return the IOSTAT code
+mlir::Value genInquireStatement(AbstractConverter &,
+                                const parser::InquireStmt &);
+
+/// Generate IO call(s) for OPEN; return the IOSTAT code
+mlir::Value genOpenStatement(AbstractConverter &, const parser::OpenStmt &);
+
+/// Generate IO call(s) for PRINT
+void genPrintStatement(AbstractConverter &converter,
+                       const parser::PrintStmt &stmt,
+                       pft::LabelEvalMap &labelMap,
+                       pft::SymbolLabelMap &assignMap);
+
+/// Generate IO call(s) for READ; return the IOSTAT code
+mlir::Value genReadStatement(AbstractConverter &converter,
+                             const parser::ReadStmt &stmt,
+                             pft::LabelEvalMap &labelMap,
+                             pft::SymbolLabelMap &assignMap);
+
+/// Generate IO call(s) for REWIND; return the IOSTAT code
+mlir::Value genRewindStatement(AbstractConverter &, const parser::RewindStmt &);
+
+/// Generate IO call(s) for WAIT; return the IOSTAT code
+mlir::Value genWaitStatement(AbstractConverter &, const parser::WaitStmt &);
+
+/// Generate IO call(s) for WRITE; return the IOSTAT code
+mlir::Value genWriteStatement(AbstractConverter &converter,
+                              const parser::WriteStmt &stmt,
+                              pft::LabelEvalMap &labelMap,
+                              pft::SymbolLabelMap &assignMap);
+
+} // namespace lower
+} // namespace Fortran
+
+#endif // FORTRAN_LOWER_IO_H
diff --git a/flang/include/flang/Lower/IntrinsicCall.h b/flang/include/flang/Lower/IntrinsicCall.h
new file mode 100644
index 0000000000000..2db1bda335b53
--- /dev/null
+++ b/flang/include/flang/Lower/IntrinsicCall.h
@@ -0,0 +1,67 @@
+//===-- Lower/IntrinsicCall.h -- lowering of intrinsics ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_LOWER_INTRINSICCALL_H
+#define FORTRAN_LOWER_INTRINSICCALL_H
+
+#include "flang/Lower/FIRBuilder.h"
+
+namespace fir {
+class ExtendedValue;
+}
+
+namespace Fortran::lower {
+
+// TODO: Expose interface to get specific intrinsic function address.
+// TODO: Handle intrinsic subroutine.
+// TODO: Intrinsics that do not require their arguments to be defined
+//   (e.g shape inquiries) might not fit in the current interface that
+//   requires mlir::Value to be provided.
+// TODO: Error handling interface ?
+// TODO: Implementation is incomplete. Many intrinsics to tbd.
+
+/// Helper for building calls to intrinsic functions in the runtime support
+/// libraries.
+
+/// Generate the FIR+MLIR operations for the generic intrinsic \p name
+/// with arguments \p args and expected result type \p resultType.
+/// Returned mlir::Value is the returned Fortran intrinsic value.
+fir::ExtendedValue genIntrinsicCall(FirOpBuilder &, mlir::Location,
+                                    llvm::StringRef name, mlir::Type resultType,
+                                    llvm::ArrayRef<fir::ExtendedValue> args);
+
+/// Get SymbolRefAttr of runtime (or wrapper function containing inlined
+// implementation) of an unrestricted intrinsic (defined by its signature
+// and generic name)
+mlir::SymbolRefAttr
+getUnrestrictedIntrinsicSymbolRefAttr(FirOpBuilder &, mlir::Location,
+                                      llvm::StringRef name,
+                                      mlir::FunctionType signature);
+
+//===--------------------------------------------------------------------===//
+// Direct access to intrinsics that may be used by lowering outside
+// of intrinsic call lowering.
+//===--------------------------------------------------------------------===//
+
+/// Generate maximum. There must be at least one argument and all arguments
+/// must have the same type.
+mlir::Value genMax(FirOpBuilder &, mlir::Location,
+                   llvm::ArrayRef<mlir::Value> args);
+
+/// Generate minimum. Same constraints as genMax.
+mlir::Value genMin(FirOpBuilder &, mlir::Location,
+                   llvm::ArrayRef<mlir::Value> args);
+
+/// Generate power function x**y with given the expected
+/// result type.
+mlir::Value genPow(FirOpBuilder &, mlir::Location, mlir::Type resultType,
+                   mlir::Value x, mlir::Value y);
+
+} // namespace Fortran::lower
+
+#endif // FORTRAN_LOWER_INTRINSICCALL_H
diff --git a/flang/include/flang/Lower/Mangler.h b/flang/include/flang/Lower/Mangler.h
index b13b51355c812..7e18c069fc343 100644
--- a/flang/include/flang/Lower/Mangler.h
+++ b/flang/include/flang/Lower/Mangler.h
@@ -5,19 +5,32 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
 
-#ifndef FORTRAN_LOWER_MANGLER_H_
-#define FORTRAN_LOWER_MANGLER_H_
+#ifndef FORTRAN_LOWER_MANGLER_H
+#define FORTRAN_LOWER_MANGLER_H
 
+#include "mlir/IR/StandardTypes.h"
+#include "llvm/ADT/StringRef.h"
 #include <string>
 
 namespace fir {
 struct NameUniquer;
-}
 
-namespace llvm {
-class StringRef;
-}
+/// Returns a name suitable to define mlir functions for Fortran intrinsic
+/// Procedure. These names are guaranteed to not conflict with user defined
+/// procedures. This is needed to implement Fortran generic intrinsics as
+/// several mlir functions specialized for the argument types.
+/// The result is guaranteed to be distinct for different mlir::FunctionType
+/// arguments. The mangling pattern is:
+///    fir.<generic name>.<result type>.<arg type>...
+/// e.g ACOS(COMPLEX(4)) is mangled as fir.acos.z4.z4
+std::string mangleIntrinsicProcedure(llvm::StringRef genericName,
+                                     mlir::FunctionType);
+} // namespace fir
 
 namespace Fortran {
 namespace common {
@@ -41,4 +54,4 @@ std::string demangleName(llvm::StringRef name);
 } // namespace lower
 } // namespace Fortran
 
-#endif // FORTRAN_LOWER_MANGLER_H_
+#endif // FORTRAN_LOWER_MANGLER_H
diff --git a/flang/include/flang/Lower/PFTBuilder.h b/flang/include/flang/Lower/PFTBuilder.h
index 852700b8c0b15..1201470de704c 100644
--- a/flang/include/flang/Lower/PFTBuilder.h
+++ b/flang/include/flang/Lower/PFTBuilder.h
@@ -234,6 +234,12 @@ struct Evaluation : EvaluationVariant {
     return visit(common::visitors{
         [](auto &r) { return pft::isDirective<std::decay_t<decltype(r)>>; }});
   }
+  constexpr bool isNopConstructStmt() const {
+    return visit(common::visitors{[](auto &r) {
+      return pft::isNopConstructStmt<std::decay_t<decltype(r)>>;
+    }});
+  }
+
   /// Return the predicate:  "This is a non-initial, non-terminal construct
   /// statement."  For an IfConstruct, this is ElseIfStmt and ElseStmt.
   constexpr bool isIntermediateConstructStmt() const {
@@ -241,14 +247,40 @@ struct Evaluation : EvaluationVariant {
       return pft::isIntermediateConstructStmt<std::decay_t<decltype(r)>>;
     }});
   }
-  constexpr bool isNopConstructStmt() const {
-    return visit(common::visitors{[](auto &r) {
-      return pft::isNopConstructStmt<std::decay_t<decltype(r)>>;
-    }});
+
+  /// Return the first non-nop successor of an evaluation, possibly exiting
+  /// from one or more enclosing constructs.
+  Evaluation &nonNopSuccessor() const {
+    Evaluation *successor = lexicalSuccessor;
+    if (successor && successor->isNopConstructStmt()) {
+      successor = successor->parentConstruct->constructExit;
+    }
+    assert(successor && "missing successor");
+    return *successor;
   }
 
-  /// Return FunctionLikeUnit to which this evaluation
-  /// belongs. Nullptr if it does not belong to such unit.
+  /// Return true if this Evaluation has at least one nested evaluation.
+  bool hasNestedEvaluations() const {
+    return evaluationList && !evaluationList->empty();
+  }
+
+  /// Return nested evaluation list.
+  EvaluationList &getNestedEvaluations() {
+    assert(evaluationList && "no nested evaluations");
+    return *evaluationList;
+  }
+
+  Evaluation &getFirstNestedEvaluation() {
+    assert(hasNestedEvaluations() && "no nested evaluations");
+    return evaluationList->front();
+  }
+
+  Evaluation &getLastNestedEvaluation() {
+    assert(hasNestedEvaluations() && "no nested evaluations");
+    return evaluationList->back();
+  }
+
+  /// Return the FunctionLikeUnit containing this evaluation (or nullptr).
   FunctionLikeUnit *getOwningProcedure() const;
 
   bool lowerAsStructured() const;
@@ -297,9 +329,9 @@ struct Evaluation : EvaluationVariant {
   Evaluation *controlSuccessor{nullptr}; // set for some statements
   Evaluation *constructExit{nullptr};    // set for constructs
   bool isNewBlock{false};                // evaluation begins a new basic block
-  bool isUnstructured{false};        // evaluation has unstructured control flow
-  bool skip{false};                  // evaluation has been processed in advance
-  class mlir::Block *block{nullptr}; // isNewBlock block
+  bool isUnstructured{false};  // evaluation has unstructured control flow
+  bool skip{false};            // evaluation has been processed in advance
+  mlir::Block *block{nullptr}; // isNewBlock block
   llvm::SmallVector<mlir::Block *, 1> localBlocks{}; // construct local blocks
   int printIndex{0}; // (ActionStmt, ConstructStmt) evaluation index for dumps
 };
@@ -333,13 +365,13 @@ struct Variable {
       : sym{&sym}, depth{depth}, global{global} {}
 
   const Fortran::semantics::Symbol &getSymbol() const { return *sym; }
-  
+
   bool isGlobal() const { return global; }
   bool isHeapAlloc() const { return heapAlloc; }
   bool isPointer() const { return pointer; }
   bool isTarget() const { return target; }
   int getDepth() const { return depth; }
-  
+
   void setHeapAlloc(bool to = true) { heapAlloc = to; }
   void setPointer(bool to = true) { pointer = to; }
   void setTarget(bool to = true) { target = to; }
diff --git a/flang/include/flang/Lower/Runtime.h b/flang/include/flang/Lower/Runtime.h
new file mode 100644
index 0000000000000..dcfce8ff63c31
--- /dev/null
+++ b/flang/include/flang/Lower/Runtime.h
@@ -0,0 +1,57 @@
+//===-- Lower/Runtime.h -- Fortran runtime codegen interface ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Builder routines for constructing the FIR dialect of MLIR. As FIR is a
+// dialect of MLIR, it makes extensive use of MLIR interfaces and MLIR's coding
+// style (https://mlir.llvm.org/getting_started/DeveloperGuide/) is used in this
+// module.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_LOWER_RUNTIME_H
+#define FORTRAN_LOWER_RUNTIME_H
+
+namespace Fortran {
+
+namespace parser {
+struct EventPostStmt;
+struct EventWaitStmt;
+struct LockStmt;
+struct PauseStmt;
+struct StopStmt;
+struct SyncAllStmt;
+struct SyncImagesStmt;
+struct SyncMemoryStmt;
+struct SyncTeamStmt;
+struct UnlockStmt;
+} // namespace parser
+
+namespace lower {
+
+class AbstractConverter;
+
+// Lowering of Fortran statement related runtime (other than IO and maths)
+
+void genEventPostStatement(AbstractConverter &, const parser::EventPostStmt &);
+void genEventWaitStatement(AbstractConverter &, const parser::EventWaitStmt &);
+void genLockStatement(AbstractConverter &, const parser::LockStmt &);
+void genFailImageStatement(AbstractConverter &);
+void genStopStatement(AbstractConverter &, const parser::StopStmt &);
+void genSyncAllStatement(AbstractConverter &, const parser::SyncAllStmt &);
+void genSyncImagesStatement(AbstractConverter &,
+                            const parser::SyncImagesStmt &);
+void genSyncMemoryStatement(AbstractConverter &,
+                            const parser::SyncMemoryStmt &);
+void genSyncTeamStatement(AbstractConverter &, const parser::SyncTeamStmt &);
+void genUnlockStatement(AbstractConverter &, const parser::UnlockStmt &);
+void genPauseStatement(AbstractConverter &, const parser::PauseStmt &);
+
+} // namespace lower
+} // namespace Fortran
+
+#endif // FORTRAN_LOWER_RUNTIME_H
diff --git a/flang/include/flang/Lower/Support/BoxValue.h b/flang/include/flang/Lower/Support/BoxValue.h
index 3516cd352969c..0d5dec97ef097 100644
--- a/flang/include/flang/Lower/Support/BoxValue.h
+++ b/flang/include/flang/Lower/Support/BoxValue.h
@@ -11,6 +11,7 @@
 
 #include "mlir/IR/Value.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include <utility>
 #include <variant>
@@ -67,7 +68,7 @@ class CharBoxValue : public AbstractBox {
 
   friend llvm::raw_ostream &operator<<(llvm::raw_ostream &,
                                        const CharBoxValue &);
-  void dump() const { llvm::errs() << *this; }
+  LLVM_DUMP_METHOD void dump() const { llvm::errs() << *this; }
 
 protected:
   mlir::Value len;
@@ -117,7 +118,7 @@ class ArrayBoxValue : public AbstractBox, public AbstractArrayBox {
 
   friend llvm::raw_ostream &operator<<(llvm::raw_ostream &,
                                        const ArrayBoxValue &);
-  void dump() const { operator<<(llvm::errs(), *this); }
+  LLVM_DUMP_METHOD void dump() const { operator<<(llvm::errs(), *this); }
 };
 
 /// Expressions of type CHARACTER and with rank > 0.
@@ -134,7 +135,7 @@ class CharArrayBoxValue : public CharBoxValue, public AbstractArrayBox {
 
   friend llvm::raw_ostream &operator<<(llvm::raw_ostream &,
                                        const CharArrayBoxValue &);
-  void dump() const { operator<<(llvm::errs(), *this); }
+  LLVM_DUMP_METHOD void dump() const { operator<<(llvm::errs(), *this); }
 };
 
 /// Expressions that are procedure POINTERs may need a set of references to
@@ -152,7 +153,7 @@ class ProcBoxValue : public AbstractBox {
 
   friend llvm::raw_ostream &operator<<(llvm::raw_ostream &,
                                        const ProcBoxValue &);
-  void dump() const { operator<<(llvm::errs(), *this); }
+  LLVM_DUMP_METHOD void dump() const { operator<<(llvm::errs(), *this); }
 
 protected:
   mlir::Value hostContext;
@@ -185,7 +186,7 @@ class BoxValue : public AbstractBox, public AbstractArrayBox {
   }
 
   friend llvm::raw_ostream &operator<<(llvm::raw_ostream &, const BoxValue &);
-  void dump() const { operator<<(llvm::errs(), *this); }
+  LLVM_DUMP_METHOD void dump() const { operator<<(llvm::errs(), *this); }
 
 protected:
   mlir::Value len;
@@ -220,7 +221,7 @@ class ExtendedValue {
   }
 
   /// LLVM style debugging of extended values
-  void dump() const { llvm::errs() << *this << '\n'; }
+  LLVM_DUMP_METHOD void dump() const { llvm::errs() << *this << '\n'; }
 
   friend llvm::raw_ostream &operator<<(llvm::raw_ostream &,
                                        const ExtendedValue &);
diff --git a/flang/include/flang/Optimizer/.clang-tidy b/flang/include/flang/Optimizer/.clang-tidy
new file mode 100644
index 0000000000000..87ec2ff53af6e
--- /dev/null
+++ b/flang/include/flang/Optimizer/.clang-tidy
@@ -0,0 +1,19 @@
+# Almost identical to the top-level .clang-tidy, except that {Member,Parameter,Variable}Case use camelBack.
+Checks: '-*,clang-diagnostic-*,llvm-*,misc-*,-misc-unused-parameters,-misc-non-private-member-variables-in-classes,readability-identifier-naming'
+CheckOptions:
+  - key:             readability-identifier-naming.ClassCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.EnumCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.FunctionCase
+    value:           camelBack
+  - key:             readability-identifier-naming.MemberCase
+    value:           camelBack
+  - key:             readability-identifier-naming.ParameterCase
+    value:           camelBack
+  - key:             readability-identifier-naming.UnionCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.VariableCase
+    value:           camelBack
+  - key:             readability-identifier-naming.IgnoreMainLikeFunctions
+    value:           1
diff --git a/flang/include/flang/Optimizer/CMakeLists.txt b/flang/include/flang/Optimizer/CMakeLists.txt
index 0ca0f41c5af49..0fa521d5b9670 100644
--- a/flang/include/flang/Optimizer/CMakeLists.txt
+++ b/flang/include/flang/Optimizer/CMakeLists.txt
@@ -1 +1,3 @@
+add_subdirectory(CodeGen)
 add_subdirectory(Dialect)
+add_subdirectory(Transforms)
diff --git a/flang/include/flang/Optimizer/CodeGen/CGPasses.td b/flang/include/flang/Optimizer/CodeGen/CGPasses.td
new file mode 100644
index 0000000000000..187147b8375fd
--- /dev/null
+++ b/flang/include/flang/Optimizer/CodeGen/CGPasses.td
@@ -0,0 +1,24 @@
+//===-- CGPasses.td - code gen pass definition file --------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains definitions for passes within the Optimizer/CodeGen/
+//  directory.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FLANG_OPTIMIZER_CODEGEN_PASSES
+#define FLANG_OPTIMIZER_CODEGEN_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+def CodeGenRewrite : FunctionPass<"cg-rewrite"> {
+  let summary = "Rewrite some FIR ops into their code-gen forms.";
+  let constructor = "fir::createFirCodeGenRewritePass()";
+}
+
+#endif // FLANG_OPTIMIZER_CODEGEN_PASSES
diff --git a/flang/include/flang/Optimizer/CodeGen/CMakeLists.txt b/flang/include/flang/Optimizer/CodeGen/CMakeLists.txt
new file mode 100644
index 0000000000000..ab6526ee18330
--- /dev/null
+++ b/flang/include/flang/Optimizer/CodeGen/CMakeLists.txt
@@ -0,0 +1,6 @@
+
+set(LLVM_TARGET_DEFINITIONS CGPasses.td)
+mlir_tablegen(CGPasses.h.inc -gen-pass-decls)
+add_public_tablegen_target(FIROptCodeGenPassIncGen)
+
+add_mlir_doc(Passes -gen-pass-doc OptimizerCodeGenPasses ./)
diff --git a/flang/include/flang/Optimizer/CodeGen/CodeGen.h b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
index 9f6936e34e17b..9b968172f3486 100644
--- a/flang/include/flang/Optimizer/CodeGen/CodeGen.h
+++ b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
@@ -9,19 +9,18 @@
 #ifndef OPTIMIZER_CODEGEN_CODEGEN_H
 #define OPTIMIZER_CODEGEN_CODEGEN_H
 
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassRegistry.h"
 #include <memory>
 
-namespace llvm {
-class raw_ostream;
-}
-namespace mlir {
-class Pass;
-}
-
 namespace fir {
 
 struct NameUniquer;
 
+/// Prerequiste pass for code gen. Perform intermediate rewrites to perform
+/// the code gen (to LLVM-IR dialect) conversion.
+std::unique_ptr<mlir::Pass> createFirCodeGenRewritePass();
+
 /// Convert FIR to the LLVM IR dialect
 std::unique_ptr<mlir::Pass> createFIRToLLVMPass(NameUniquer &uniquer);
 
@@ -29,6 +28,13 @@ std::unique_ptr<mlir::Pass> createFIRToLLVMPass(NameUniquer &uniquer);
 std::unique_ptr<mlir::Pass>
 createLLVMDialectToLLVMPass(llvm::raw_ostream &output);
 
+inline void registerOptCodeGenPasses() {
+  using mlir::Pass;
+// declarative passes
+#define GEN_PASS_REGISTRATION
+#include "flang/Optimizer/CodeGen/CGPasses.h.inc"
+}
+
 } // namespace fir
 
 #endif // OPTIMIZER_CODEGEN_CODEGEN_H
diff --git a/flang/include/flang/Optimizer/Dialect/FIRAttr.h b/flang/include/flang/Optimizer/Dialect/FIRAttr.h
index 57ce5f13bf652..0b22bc21224cb 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRAttr.h
+++ b/flang/include/flang/Optimizer/Dialect/FIRAttr.h
@@ -75,7 +75,8 @@ class SubclassAttr
 /// A case selector of `CASE (n:m)` corresponds to any value from `n` to `m` and
 /// is encoded as `#fir.interval, %n, %m`.
 class ClosedIntervalAttr
-    : public mlir::Attribute::AttrBase<ClosedIntervalAttr> {
+    : public mlir::Attribute::AttrBase<ClosedIntervalAttr, mlir::Attribute,
+                                       mlir::AttributeStorage> {
 public:
   using Base::Base;
 
@@ -91,7 +92,9 @@ class ClosedIntervalAttr
 /// an ssa-value.
 /// A case selector of `CASE (:m)` corresponds to any value up to and including
 /// `m` and is encoded as `#fir.upper, %m`.
-class UpperBoundAttr : public mlir::Attribute::AttrBase<UpperBoundAttr> {
+class UpperBoundAttr
+    : public mlir::Attribute::AttrBase<UpperBoundAttr, mlir::Attribute,
+                                       mlir::AttributeStorage> {
 public:
   using Base::Base;
 
@@ -107,7 +110,9 @@ class UpperBoundAttr : public mlir::Attribute::AttrBase<UpperBoundAttr> {
 /// an ssa-value.
 /// A case selector of `CASE (n:)` corresponds to any value down to and
 /// including `n` and is encoded as `#fir.lower, %n`.
-class LowerBoundAttr : public mlir::Attribute::AttrBase<LowerBoundAttr> {
+class LowerBoundAttr
+    : public mlir::Attribute::AttrBase<LowerBoundAttr, mlir::Attribute,
+                                       mlir::AttributeStorage> {
 public:
   using Base::Base;
 
@@ -123,7 +128,9 @@ class LowerBoundAttr : public mlir::Attribute::AttrBase<LowerBoundAttr> {
 /// interval contains exactly one value.
 /// A case selector of `CASE (p)` corresponds to exactly the value `p` and is
 /// encoded as `#fir.point, %p`.
-class PointIntervalAttr : public mlir::Attribute::AttrBase<PointIntervalAttr> {
+class PointIntervalAttr
+    : public mlir::Attribute::AttrBase<PointIntervalAttr, mlir::Attribute,
+                                       mlir::AttributeStorage> {
 public:
   using Base::Base;
 
diff --git a/flang/include/flang/Optimizer/Dialect/FIRType.h b/flang/include/flang/Optimizer/Dialect/FIRType.h
index 3749c87b9e947..b1f1cc85b744a 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRType.h
+++ b/flang/include/flang/Optimizer/Dialect/FIRType.h
@@ -242,7 +242,7 @@ class DimsType : public mlir::Type::TypeBase<DimsType, mlir::Type,
   static bool kindof(unsigned kind) { return kind == TypeKind::FIR_DIMS; }
 
   /// returns -1 if the rank is unknown
-  int getRank() const;
+  unsigned getRank() const;
 };
 
 /// The type of a field name. Implementations may defer the layout of a Fortran
@@ -437,6 +437,12 @@ inline bool isa_real(mlir::Type t) {
   return t.isa<fir::RealType>() || t.isa<mlir::FloatType>();
 }
 
+/// Is `t` an integral type?
+inline bool isa_integer(mlir::Type t) {
+  return t.isa<mlir::IndexType>() || t.isa<mlir::IntegerType>() ||
+         t.isa<fir::IntType>();
+}
+
 /// Is `t` a FIR or MLIR Complex type?
 inline bool isa_complex(mlir::Type t) {
   return t.isa<fir::CplxType>() || t.isa<mlir::ComplexType>();
diff --git a/flang/include/flang/Optimizer/Support/KindMapping.h b/flang/include/flang/Optimizer/Support/KindMapping.h
index 8c8ed1f24afdd..b65f8285c3a81 100644
--- a/flang/include/flang/Optimizer/Support/KindMapping.h
+++ b/flang/include/flang/Optimizer/Support/KindMapping.h
@@ -1,4 +1,4 @@
-//===-- Optimizer/Support/KindMapping.h -------------------------*- C++ -*-===//
+//===-- Optimizer/Support/KindMapping.h -- support kind mapping -*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -14,15 +14,9 @@
 #include "llvm/IR/Type.h"
 
 namespace llvm {
-template <typename>
-class Optional;
 struct fltSemantics;
 } // namespace llvm
 
-namespace mlir {
-class MLIRContext;
-} // namespace mlir
-
 namespace fir {
 
 /// The kind mapping is an encoded string that informs FIR how the Fortran KIND
@@ -57,24 +51,27 @@ class KindMapping {
   explicit KindMapping(mlir::MLIRContext *context, llvm::StringRef map);
 
   /// Get the size in bits of !fir.char<kind>
-  Bitsize getCharacterBitsize(KindTy kind);
+  Bitsize getCharacterBitsize(KindTy kind) const;
 
   /// Get the size in bits of !fir.int<kind>
-  Bitsize getIntegerBitsize(KindTy kind);
+  Bitsize getIntegerBitsize(KindTy kind) const;
 
   /// Get the size in bits of !fir.logical<kind>
-  Bitsize getLogicalBitsize(KindTy kind);
+  Bitsize getLogicalBitsize(KindTy kind) const;
+
+  /// Get the size in bits of !fir.real<kind>
+  Bitsize getRealBitsize(KindTy kind) const;
 
   /// Get the LLVM Type::TypeID of !fir.real<kind>
-  LLVMTypeID getRealTypeID(KindTy kind);
+  LLVMTypeID getRealTypeID(KindTy kind) const;
 
   /// Get the LLVM Type::TypeID of !fir.complex<kind>
-  LLVMTypeID getComplexTypeID(KindTy kind);
+  LLVMTypeID getComplexTypeID(KindTy kind) const;
 
   mlir::MLIRContext *getContext() const { return context; }
 
   /// Get the float semantics of !fir.real<kind>
-  const llvm::fltSemantics &getFloatSemantics(KindTy kind);
+  const llvm::fltSemantics &getFloatSemantics(KindTy kind) const;
 
 private:
   MatchResult badMapString(const llvm::Twine &ptr);
diff --git a/flang/include/flang/Optimizer/Transforms/CMakeLists.txt b/flang/include/flang/Optimizer/Transforms/CMakeLists.txt
new file mode 100644
index 0000000000000..fde17eb88622e
--- /dev/null
+++ b/flang/include/flang/Optimizer/Transforms/CMakeLists.txt
@@ -0,0 +1,6 @@
+
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls)
+add_public_tablegen_target(FIROptTransformsPassIncGen)
+
+add_mlir_doc(Passes -gen-pass-doc OptimizerTransformPasses ./)
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h
new file mode 100644
index 0000000000000..9377c2dc61cc3
--- /dev/null
+++ b/flang/include/flang/Optimizer/Transforms/Passes.h
@@ -0,0 +1,58 @@
+//===-- Optimizer/Transforms/Passes.h ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPTIMIZER_TRANSFORMS_PASSES_H
+#define OPTIMIZER_TRANSFORMS_PASSES_H
+
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassRegistry.h"
+#include <memory>
+
+namespace mlir {
+class BlockAndValueMapping;
+class Operation;
+class Pass;
+class Region;
+} // namespace mlir
+
+namespace fir {
+
+/// Convert fir.select_type to the standard dialect
+std::unique_ptr<mlir::Pass> createControlFlowLoweringPass();
+
+/// Effects aware CSE pass
+std::unique_ptr<mlir::Pass> createCSEPass();
+
+/// Convert FIR loop constructs to the Affine dialect
+std::unique_ptr<mlir::Pass> createPromoteToAffinePass();
+
+/// Convert `fir.do_loop` and `fir.if` to a CFG.  This
+/// conversion enables the `createLowerToCFGPass` to transform these to CFG
+/// form.
+std::unique_ptr<mlir::Pass> createFirToCfgPass();
+
+/// A pass to convert the FIR dialect from "Mem-SSA" form to "Reg-SSA"
+/// form. This pass is a port of LLVM's mem2reg pass, but modified for the FIR
+/// dialect as well as the restructuring of MLIR's representation to present PHI
+/// nodes as block arguments.
+std::unique_ptr<mlir::Pass> createMemToRegPass();
+
+/// Support for inlining on FIR.
+bool canLegallyInline(mlir::Operation *op, mlir::Region *reg,
+                      mlir::BlockAndValueMapping &map);
+
+inline void registerOptTransformPasses() {
+using mlir::Pass;
+// declarative passes
+#define GEN_PASS_REGISTRATION
+#include "flang/Optimizer/Transforms/Passes.h.inc"
+}
+
+} // namespace fir
+
+#endif // OPTIMIZER_TRANSFORMS_PASSES_H
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td
new file mode 100644
index 0000000000000..c558d99af8dc3
--- /dev/null
+++ b/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -0,0 +1,51 @@
+//===-- Passes.td - Transforms pass definition file --------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains definitions for passes within the Optimizer/Transforms/
+//  directory.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FLANG_OPTIMIZER_TRANSFORMS_PASSES
+#define FLANG_OPTIMIZER_TRANSFORMS_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+def AffineDialectPromotion : FunctionPass<"promote-to-affine"> {
+  let summary = "Promotes fir.loop and fir.where to affine.for and affine.if where possible";
+  let description = [{
+    TODO
+  }];
+  let constructor = "fir::createPromoteToAffinePass()";
+}
+
+def BasicCSE : FunctionPass<"basic-cse"> {
+  let summary = "Basic common sub-expression elimination";
+  let description = [{
+      TODO
+  }];
+  let constructor = "fir::createCSEPass()";
+}
+
+def ControlFlowLowering : FunctionPass<"lower-control-flow"> {
+  let summary = "Convert affine dialect, fir.select_type to standard dialect";
+  let description = [{
+      TODO
+  }];
+  let constructor = "fir::createControlFlowLoweringPass()";
+}
+
+def CFGConversion : FunctionPass<"cfg-conversion"> {
+  let summary = "Convert FIR structured control flow ops to CFG ops.";
+  let description = [{
+      TODO
+  }];
+  let constructor = "fir::createFirToCfgPass()";
+}
+
+#endif // FLANG_OPTIMIZER_TRANSFORMS_PASSES
diff --git a/flang/include/flang/Parser/characters.h b/flang/include/flang/Parser/characters.h
index c18140780956a..eefb524e5d879 100644
--- a/flang/include/flang/Parser/characters.h
+++ b/flang/include/flang/Parser/characters.h
@@ -96,9 +96,9 @@ inline constexpr bool IsSameApartFromCase(char x, char y) {
 inline constexpr char DecimalDigitValue(char ch) { return ch - '0'; }
 
 inline constexpr char HexadecimalDigitValue(char ch) {
-  return IsUpperCaseLetter(ch)
-      ? ch - 'A' + 10
-      : IsLowerCaseLetter(ch) ? ch - 'a' + 10 : DecimalDigitValue(ch);
+  return IsUpperCaseLetter(ch) ? ch - 'A' + 10
+      : IsLowerCaseLetter(ch)  ? ch - 'a' + 10
+                               : DecimalDigitValue(ch);
 }
 
 inline constexpr std::optional<char> BackslashEscapeValue(char ch) {
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index ad93fcd257954..59333c7405ffa 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -409,7 +409,11 @@ class ParseTreeDumper {
   NODE(parser, OmpBeginLoopDirective)
   NODE(parser, OmpBeginSectionsDirective)
   NODE(parser, OmpBlockDirective)
-  NODE_ENUM(OmpBlockDirective, Directive)
+  static std::string GetNodeName(const llvm::omp::Directive &x) {
+    return llvm::Twine(
+        "llvm::omp::Directive = ", llvm::omp::getOpenMPDirectiveName(x))
+        .str();
+  }
   NODE(parser, OmpCancelType)
   NODE_ENUM(OmpCancelType, Type)
   NODE(parser, OmpClause)
@@ -477,7 +481,6 @@ class ParseTreeDumper {
   NODE(parser, OmpLinearModifier)
   NODE_ENUM(OmpLinearModifier, Type)
   NODE(parser, OmpLoopDirective)
-  NODE_ENUM(OmpLoopDirective, Directive)
   NODE(parser, OmpMapClause)
   NODE(parser, OmpMapType)
   NODE(OmpMapType, Always)
@@ -505,9 +508,7 @@ class ParseTreeDumper {
   NODE_ENUM(OmpScheduleModifierType, ModType)
   NODE(parser, OmpSectionBlocks)
   NODE(parser, OmpSectionsDirective)
-  NODE_ENUM(OmpSectionsDirective, Directive)
   NODE(parser, OmpSimpleStandaloneDirective)
-  NODE_ENUM(OmpSimpleStandaloneDirective, Directive)
   NODE(parser, Only)
   NODE(parser, OpenMPAtomicConstruct)
   NODE(parser, OpenMPBlockConstruct)
@@ -520,6 +521,8 @@ class ParseTreeDumper {
   NODE(parser, OpenMPDeclareReductionConstruct)
   NODE(parser, OpenMPDeclareSimdConstruct)
   NODE(parser, OpenMPDeclareTargetConstruct)
+  NODE(parser, OmpFlushMemoryClause)
+  NODE_ENUM(OmpFlushMemoryClause, FlushMemoryOrder)
   NODE(parser, OpenMPFlushConstruct)
   NODE(parser, OpenMPLoopConstruct)
   NODE(parser, OpenMPSimpleStandaloneConstruct)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 933638d039d35..67fd5741b0975 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -25,6 +25,7 @@
 #include "flang/Common/Fortran.h"
 #include "flang/Common/idioms.h"
 #include "flang/Common/indirection.h"
+#include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include <cinttypes>
 #include <list>
 #include <memory>
@@ -1394,8 +1395,7 @@ WRAPPER_CLASS(ContiguousStmt, std::list<ObjectName>);
 using ConstantSubobject = Constant<common::Indirection<Designator>>;
 
 // Represents an analyzed expression
-using TypedExpr = std::unique_ptr<evaluate::GenericExprWrapper,
-    common::Deleter<evaluate::GenericExprWrapper>>;
+using TypedExpr = common::ForwardOwningPointer<evaluate::GenericExprWrapper>;
 
 // R845 data-stmt-constant ->
 //        scalar-constant | scalar-constant-subobject |
@@ -1918,8 +1918,8 @@ struct DeallocateStmt {
 // R1032 assignment-stmt -> variable = expr
 struct AssignmentStmt {
   TUPLE_CLASS_BOILERPLATE(AssignmentStmt);
-  using TypedAssignment = std::unique_ptr<evaluate::GenericAssignmentWrapper,
-      common::Deleter<evaluate::GenericAssignmentWrapper>>;
+  using TypedAssignment =
+      common::ForwardOwningPointer<evaluate::GenericAssignmentWrapper>;
   mutable TypedAssignment typedAssignment;
   std::tuple<Variable, Expr> t;
 };
@@ -3139,8 +3139,7 @@ struct FunctionReference {
 // R1521 call-stmt -> CALL procedure-designator [( [actual-arg-spec-list] )]
 struct CallStmt {
   WRAPPER_CLASS_BOILERPLATE(CallStmt, Call);
-  mutable std::unique_ptr<evaluate::ProcedureRef,
-      common::Deleter<evaluate::ProcedureRef>>
+  mutable common::ForwardOwningPointer<evaluate::ProcedureRef>
       typedCall; // filled by semantics
 };
 
@@ -3495,8 +3494,7 @@ struct OmpClauseList {
 // 2.7.2 SECTIONS
 // 2.11.2 PARALLEL SECTIONS
 struct OmpSectionsDirective {
-  ENUM_CLASS(Directive, Sections, ParallelSections);
-  WRAPPER_CLASS_BOILERPLATE(OmpSectionsDirective, Directive);
+  WRAPPER_CLASS_BOILERPLATE(OmpSectionsDirective, llvm::omp::Directive);
   CharBlock source;
 };
 
@@ -3527,10 +3525,7 @@ struct OpenMPSectionsConstruct {
 
 // OpenMP directive beginning or ending a block
 struct OmpBlockDirective {
-  ENUM_CLASS(Directive, Master, Ordered, Parallel, ParallelWorkshare, Single,
-      Target, TargetData, TargetParallel, TargetTeams, Task, Taskgroup, Teams,
-      Workshare);
-  WRAPPER_CLASS_BOILERPLATE(OmpBlockDirective, Directive);
+  WRAPPER_CLASS_BOILERPLATE(OmpBlockDirective, llvm::omp::Directive);
   CharBlock source;
 };
 
@@ -3684,15 +3679,7 @@ struct OpenMPAtomicConstruct {
 
 // OpenMP directives that associate with loop(s)
 struct OmpLoopDirective {
-  ENUM_CLASS(Directive, Distribute, DistributeParallelDo,
-      DistributeParallelDoSimd, DistributeSimd, ParallelDo, ParallelDoSimd, Do,
-      DoSimd, Simd, TargetParallelDo, TargetParallelDoSimd,
-      TargetTeamsDistribute, TargetTeamsDistributeParallelDo,
-      TargetTeamsDistributeParallelDoSimd, TargetTeamsDistributeSimd,
-      TargetSimd, Taskloop, TaskloopSimd, TeamsDistribute,
-      TeamsDistributeParallelDo, TeamsDistributeParallelDoSimd,
-      TeamsDistributeSimd)
-  WRAPPER_CLASS_BOILERPLATE(OmpLoopDirective, Directive);
+  WRAPPER_CLASS_BOILERPLATE(OmpLoopDirective, llvm::omp::Directive);
   CharBlock source;
 };
 
@@ -3718,17 +3705,27 @@ struct OpenMPCancelConstruct {
   std::tuple<Verbatim, OmpCancelType, std::optional<If>> t;
 };
 
-// 2.13.7 flush -> FLUSH [(variable-name-list)]
+// 2.17.8 Flush Construct [OpenMP 5.0]
+// memory-order-clause -> acq_rel
+//                        release
+//                        acquire
+struct OmpFlushMemoryClause {
+  ENUM_CLASS(FlushMemoryOrder, AcqRel, Release, Acquire)
+  WRAPPER_CLASS_BOILERPLATE(OmpFlushMemoryClause, FlushMemoryOrder);
+  CharBlock source;
+};
+
+// 2.17.8 flush -> FLUSH [memory-order-clause] [(variable-name-list)]
 struct OpenMPFlushConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPFlushConstruct);
   CharBlock source;
-  std::tuple<Verbatim, std::optional<OmpObjectList>> t;
+  std::tuple<Verbatim, std::optional<OmpFlushMemoryClause>,
+      std::optional<OmpObjectList>>
+      t;
 };
 
 struct OmpSimpleStandaloneDirective {
-  ENUM_CLASS(Directive, Barrier, Taskwait, Taskyield, TargetEnterData,
-      TargetExitData, TargetUpdate, Ordered)
-  WRAPPER_CLASS_BOILERPLATE(OmpSimpleStandaloneDirective, Directive);
+  WRAPPER_CLASS_BOILERPLATE(OmpSimpleStandaloneDirective, llvm::omp::Directive);
   CharBlock source;
 };
 
diff --git a/flang/include/flang/Parser/tools.h b/flang/include/flang/Parser/tools.h
index c918425a29787..fa6ecd08a3186 100644
--- a/flang/include/flang/Parser/tools.h
+++ b/flang/include/flang/Parser/tools.h
@@ -99,6 +99,8 @@ template <typename A, typename B> A *Unwrap(B &x) {
 // Get the CoindexedNamedObject if the entity is a coindexed object.
 const CoindexedNamedObject *GetCoindexedNamedObject(const AllocateObject &);
 const CoindexedNamedObject *GetCoindexedNamedObject(const DataRef &);
+const CoindexedNamedObject *GetCoindexedNamedObject(const Designator &);
+const CoindexedNamedObject *GetCoindexedNamedObject(const Variable &);
 
 // Detects parse tree nodes with "source" members.
 template <typename A, typename = int> struct HasSource : std::false_type {};
diff --git a/flang/include/flang/Semantics/expression.h b/flang/include/flang/Semantics/expression.h
index 1b94ce62a996e..4148a0d605d86 100644
--- a/flang/include/flang/Semantics/expression.h
+++ b/flang/include/flang/Semantics/expression.h
@@ -70,7 +70,8 @@ class IntrinsicProcTable;
 struct SetExprHelper {
   explicit SetExprHelper(GenericExprWrapper &&expr) : expr_{std::move(expr)} {}
   void Set(parser::TypedExpr &x) {
-    x.reset(new GenericExprWrapper{std::move(expr_)});
+    x.Reset(new GenericExprWrapper{std::move(expr_)},
+        evaluate::GenericExprWrapper::Deleter);
   }
   void Set(const parser::Expr &x) { Set(x.typedExpr); }
   void Set(const parser::Variable &x) { Set(x.typedExpr); }
diff --git a/flang/include/flang/Semantics/scope.h b/flang/include/flang/Semantics/scope.h
index 3913889ac27b9..a67a008889276 100644
--- a/flang/include/flang/Semantics/scope.h
+++ b/flang/include/flang/Semantics/scope.h
@@ -38,9 +38,10 @@ class SemanticsContext;
 // the indices for an array element, and the lower bound for a substring.
 struct EquivalenceObject {
   EquivalenceObject(Symbol &symbol, std::vector<ConstantSubscript> subscripts,
-      std::optional<ConstantSubscript> substringStart)
-      : symbol{symbol}, subscripts{subscripts}, substringStart{substringStart} {
-  }
+      std::optional<ConstantSubscript> substringStart, parser::CharBlock source)
+      : symbol{symbol}, subscripts{subscripts},
+        substringStart{substringStart}, source{source} {}
+
   bool operator==(const EquivalenceObject &) const;
   bool operator<(const EquivalenceObject &) const;
   std::string AsFortran() const;
@@ -48,6 +49,7 @@ struct EquivalenceObject {
   Symbol &symbol;
   std::vector<ConstantSubscript> subscripts; // for array elem
   std::optional<ConstantSubscript> substringStart;
+  parser::CharBlock source;
 };
 using EquivalenceSet = std::vector<EquivalenceObject>;
 
@@ -263,8 +265,9 @@ class Scope {
 // Inline so that it can be called from Evaluate without a link-time dependency.
 
 inline const Symbol *Scope::GetSymbol() const {
-  return symbol_ ? symbol_
-                 : derivedTypeSpec_ ? &derivedTypeSpec_->typeSymbol() : nullptr;
+  return symbol_         ? symbol_
+      : derivedTypeSpec_ ? &derivedTypeSpec_->typeSymbol()
+                         : nullptr;
 }
 
 } // namespace Fortran::semantics
diff --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h
index 0de6e462133d1..227f951aef5f0 100644
--- a/flang/include/flang/Semantics/symbol.h
+++ b/flang/include/flang/Semantics/symbol.h
@@ -423,7 +423,6 @@ struct GenericKind {
 class GenericDetails {
 public:
   GenericDetails() {}
-  GenericDetails(const SymbolVector &specificProcs);
 
   GenericKind kind() const { return kind_; }
   void set_kind(GenericKind kind) { kind_ = kind; }
diff --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h
index 86a766bf963c4..f63e4ccbc687c 100644
--- a/flang/include/flang/Semantics/tools.h
+++ b/flang/include/flang/Semantics/tools.h
@@ -85,7 +85,6 @@ bool IsPointerDummy(const Symbol &);
 bool IsBindCProcedure(const Symbol &);
 bool IsBindCProcedure(const Scope &);
 bool IsProcName(const Symbol &); // proc-name
-bool IsFunctionResult(const Symbol &);
 bool IsFunctionResultWithSameNameAsFunction(const Symbol &);
 bool IsExtensibleType(const DerivedTypeSpec *);
 bool IsBuiltinDerivedType(const DerivedTypeSpec *derived, const char *name);
@@ -100,6 +99,7 @@ bool IsInitialized(const Symbol &, bool ignoreDATAstatements = false);
 bool HasIntrinsicTypeName(const Symbol &);
 bool IsSeparateModuleProcedureInterface(const Symbol *);
 bool IsAutomatic(const Symbol &);
+bool HasAlternateReturns(const Symbol &);
 
 // Return an ultimate component of type that matches predicate, or nullptr.
 const Symbol *FindUltimateComponent(const DerivedTypeSpec &type,
diff --git a/flang/lib/Decimal/big-radix-floating-point.h b/flang/lib/Decimal/big-radix-floating-point.h
index 18626e2b55df4..53bd9d7249145 100644
--- a/flang/lib/Decimal/big-radix-floating-point.h
+++ b/flang/lib/Decimal/big-radix-floating-point.h
@@ -27,7 +27,6 @@
 #include "flang/Common/unsigned-const-division.h"
 #include "flang/Decimal/binary-floating-point.h"
 #include "flang/Decimal/decimal.h"
-#include "llvm/Support/raw_ostream.h"
 #include <cinttypes>
 #include <limits>
 #include <type_traits>
@@ -112,7 +111,7 @@ template <int PREC, int LOG10RADIX = 16> class BigRadixFloatingPointNumber {
   void Minimize(
       BigRadixFloatingPointNumber &&less, BigRadixFloatingPointNumber &&more);
 
-  llvm::raw_ostream &Dump(llvm::raw_ostream &) const;
+  template <typename STREAM> STREAM &Dump(STREAM &) const;
 
 private:
   BigRadixFloatingPointNumber(const BigRadixFloatingPointNumber &that)
diff --git a/flang/lib/Decimal/binary-to-decimal.cpp b/flang/lib/Decimal/binary-to-decimal.cpp
index 02e39c241ee1a..ad30b4d854033 100644
--- a/flang/lib/Decimal/binary-to-decimal.cpp
+++ b/flang/lib/Decimal/binary-to-decimal.cpp
@@ -389,8 +389,8 @@ ConversionToDecimalResult ConvertLongDoubleToDecimal(char *buffer,
 }
 
 template <int PREC, int LOG10RADIX>
-llvm::raw_ostream &BigRadixFloatingPointNumber<PREC, LOG10RADIX>::Dump(
-    llvm::raw_ostream &o) const {
+template <typename STREAM>
+STREAM &BigRadixFloatingPointNumber<PREC, LOG10RADIX>::Dump(STREAM &o) const {
   if (isNegative_) {
     o << '-';
   }
diff --git a/flang/lib/Evaluate/CMakeLists.txt b/flang/lib/Evaluate/CMakeLists.txt
index ddcdc80186586..9ab83d7c1c258 100644
--- a/flang/lib/Evaluate/CMakeLists.txt
+++ b/flang/lib/Evaluate/CMakeLists.txt
@@ -1,3 +1,13 @@
+if (LIBPGMATH_DIR)
+  # If pgmath library is found, it can be used for constant folding.
+  find_library(LIBPGMATH pgmath PATHS ${LIBPGMATH_DIR})
+  if(LIBPGMATH)
+    add_compile_definitions(LINK_WITH_LIBPGMATH)
+    message(STATUS "Found libpgmath: ${LIBPGMATH}")
+  else()
+    message(STATUS "Libpgmath not found in: ${LIBPGMATH_DIR}")
+  endif()
+endif()
 
 add_flang_library(FortranEvaluate
   call.cpp
@@ -31,18 +41,10 @@ add_flang_library(FortranEvaluate
   LINK_LIBS
   FortranCommon
   FortranDecimal
-  FortranSemantics
   FortranParser
+  ${LIBPGMATH}
+
+  DEPENDS
+  omp_gen
 )
 
-if (LIBPGMATH_DIR)
-  # If pgmath library is found, it can be used for constant folding.
-  find_library(LIBPGMATH pgmath PATHS ${LIBPGMATH_DIR})
-  if(LIBPGMATH)
-    add_compile_definitions(LINK_WITH_LIBPGMATH)
-    target_link_libraries(FortranEvaluate ${LIBPGMATH})
-    message(STATUS "Found libpgmath: ${LIBPGMATH}")
-  else()
-    message(STATUS "Libpgmath not found in: ${LIBPGMATH_DIR}")
-  endif()
-endif()
diff --git a/flang/lib/Evaluate/call.cpp b/flang/lib/Evaluate/call.cpp
index c9d6fe0ddb807..b4cf0dc3af3ab 100644
--- a/flang/lib/Evaluate/call.cpp
+++ b/flang/lib/Evaluate/call.cpp
@@ -212,6 +212,7 @@ int ProcedureRef::Rank() const {
 
 ProcedureRef::~ProcedureRef() {}
 
+void ProcedureRef::Deleter(ProcedureRef *p) { delete p; }
+
 FOR_EACH_SPECIFIC_TYPE(template class FunctionRef, )
 } // namespace Fortran::evaluate
-DEFINE_DELETER(Fortran::evaluate::ProcedureRef)
diff --git a/flang/lib/Evaluate/expression.cpp b/flang/lib/Evaluate/expression.cpp
index 604e4ff41ce23..5a456648b8254 100644
--- a/flang/lib/Evaluate/expression.cpp
+++ b/flang/lib/Evaluate/expression.cpp
@@ -223,8 +223,14 @@ StructureConstructor &StructureConstructor::Add(
 
 GenericExprWrapper::~GenericExprWrapper() {}
 
+void GenericExprWrapper::Deleter(GenericExprWrapper *p) { delete p; }
+
 GenericAssignmentWrapper::~GenericAssignmentWrapper() {}
 
+void GenericAssignmentWrapper::Deleter(GenericAssignmentWrapper *p) {
+  delete p;
+}
+
 template <TypeCategory CAT> int Expr<SomeKind<CAT>>::GetKind() const {
   return std::visit(
       [](const auto &kx) { return std::decay_t<decltype(kx)>::Result::kind; },
@@ -243,5 +249,3 @@ std::optional<Expr<SubscriptInteger>> Expr<SomeCharacter>::LEN() const {
 
 INSTANTIATE_EXPRESSION_TEMPLATES
 } // namespace Fortran::evaluate
-DEFINE_DELETER(Fortran::evaluate::GenericExprWrapper)
-DEFINE_DELETER(Fortran::evaluate::GenericAssignmentWrapper)
diff --git a/flang/lib/Evaluate/fold-integer.cpp b/flang/lib/Evaluate/fold-integer.cpp
index 3472f9a444c15..9b96a479df597 100644
--- a/flang/lib/Evaluate/fold-integer.cpp
+++ b/flang/lib/Evaluate/fold-integer.cpp
@@ -157,9 +157,8 @@ Expr<Type<TypeCategory::Integer, KIND>> FoldIntrinsicFunction(
   } else if (name == "ceiling" || name == "floor" || name == "nint") {
     if (const auto *cx{UnwrapExpr<Expr<SomeReal>>(args[0])}) {
       // NINT rounds ties away from zero, not to even
-      common::RoundingMode mode{name == "ceiling"
-              ? common::RoundingMode::Up
-              : name == "floor" ? common::RoundingMode::Down
+      common::RoundingMode mode{name == "ceiling" ? common::RoundingMode::Up
+              : name == "floor"                   ? common::RoundingMode::Down
                                 : common::RoundingMode::TiesAwayFromZero};
       return std::visit(
           [&](const auto &kx) {
@@ -322,8 +321,8 @@ Expr<Type<TypeCategory::Integer, KIND>> FoldIntrinsicFunction(
                         return name == "index"
                             ? CharacterUtils<TC::kind>::INDEX(str, other)
                             : name == "scan"
-                                ? CharacterUtils<TC::kind>::SCAN(str, other)
-                                : CharacterUtils<TC::kind>::VERIFY(str, other);
+                            ? CharacterUtils<TC::kind>::SCAN(str, other)
+                            : CharacterUtils<TC::kind>::VERIFY(str, other);
                       }});
             }
           },
@@ -613,7 +612,9 @@ Expr<Type<TypeCategory::Integer, KIND>> FoldOperation(
         if (iter != scope->end()) {
           const Symbol &symbol{*iter->second};
           const auto *details{symbol.detailsIf<semantics::TypeParamDetails>()};
-          if (details && details->init()) {
+          if (details && details->init() &&
+              (details->attr() == common::TypeParamAttr::Kind ||
+                  IsConstantExpr(*details->init()))) {
             Expr<SomeInteger> expr{*details->init()};
             return Fold(context,
                 Expr<IntKIND>{
diff --git a/flang/lib/Evaluate/fold-real.cpp b/flang/lib/Evaluate/fold-real.cpp
index 657bc0c2033f3..e3db35caf7e2e 100644
--- a/flang/lib/Evaluate/fold-real.cpp
+++ b/flang/lib/Evaluate/fold-real.cpp
@@ -42,7 +42,7 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
     return RewriteSpecificMINorMAX(context, std::move(funcRef));
   } else if (name == "atan" || name == "atan2" || name == "hypot" ||
       name == "mod") {
-    std::string localName{name == "atan2" ? "atan" : name};
+    std::string localName{name == "atan" ? "atan2" : name};
     CHECK(args.size() == 2);
     if (auto callable{
             context.hostIntrinsicsLibrary()
diff --git a/flang/lib/Evaluate/host.cpp b/flang/lib/Evaluate/host.cpp
index ea831274b9425..c9aa5c4bbce3b 100644
--- a/flang/lib/Evaluate/host.cpp
+++ b/flang/lib/Evaluate/host.cpp
@@ -11,6 +11,9 @@
 #include "flang/Common/idioms.h"
 #include "llvm/Support/Errno.h"
 #include <cfenv>
+#if __x86_64__
+#include <xmmintrin.h>
+#endif
 
 namespace Fortran::evaluate::host {
 using namespace Fortran::parser::literals;
@@ -18,39 +21,42 @@ using namespace Fortran::parser::literals;
 void HostFloatingPointEnvironment::SetUpHostFloatingPointEnvironment(
     FoldingContext &context) {
   errno = 0;
+  std::fenv_t currentFenv;
   if (feholdexcept(&originalFenv_) != 0) {
     common::die("Folding with host runtime: feholdexcept() failed: %s",
         llvm::sys::StrError(errno).c_str());
     return;
   }
-  if (fegetenv(&currentFenv_) != 0) {
+  if (fegetenv(&currentFenv) != 0) {
     common::die("Folding with host runtime: fegetenv() failed: %s",
         llvm::sys::StrError(errno).c_str());
     return;
   }
 #if __x86_64__
   hasSubnormalFlushingHardwareControl_ = true;
+  originalMxcsr = _mm_getcsr();
+  unsigned int currentMxcsr{originalMxcsr};
   if (context.flushSubnormalsToZero()) {
-    currentFenv_.__mxcsr |= 0x8000; // result
-    currentFenv_.__mxcsr |= 0x0040; // operands
+    currentMxcsr |= 0x8000;
+    currentMxcsr |= 0x0040;
   } else {
-    currentFenv_.__mxcsr &= ~0x8000; // result
-    currentFenv_.__mxcsr &= ~0x0040; // operands
+    currentMxcsr &= ~0x8000;
+    currentMxcsr &= ~0x0040;
   }
 #elif defined(__aarch64__)
 #if defined(__GNU_LIBRARY__)
   hasSubnormalFlushingHardwareControl_ = true;
   if (context.flushSubnormalsToZero()) {
-    currentFenv_.__fpcr |= (1U << 24); // control register
+    currentFenv.__fpcr |= (1U << 24); // control register
   } else {
-    currentFenv_.__fpcr &= ~(1U << 24); // control register
+    currentFenv.__fpcr &= ~(1U << 24); // control register
   }
 #elif defined(__BIONIC__)
   hasSubnormalFlushingHardwareControl_ = true;
   if (context.flushSubnormalsToZero()) {
-    currentFenv_.__control |= (1U << 24); // control register
+    currentFenv.__control |= (1U << 24); // control register
   } else {
-    currentFenv_.__control &= ~(1U << 24); // control register
+    currentFenv.__control &= ~(1U << 24); // control register
   }
 #else
   // If F18 is built with other C libraries on AArch64, software flushing will
@@ -70,11 +76,15 @@ void HostFloatingPointEnvironment::SetUpHostFloatingPointEnvironment(
   hardwareFlagsAreReliable_ = false;
 #endif
   errno = 0;
-  if (fesetenv(&currentFenv_) != 0) {
+  if (fesetenv(&currentFenv) != 0) {
     common::die("Folding with host runtime: fesetenv() failed: %s",
         llvm::sys::StrError(errno).c_str());
     return;
   }
+#if __x86_64__
+  _mm_setcsr(currentMxcsr);
+#endif
+
   switch (context.rounding().mode) {
   case common::RoundingMode::TiesToEven:
     fesetround(FE_TONEAREST);
@@ -141,6 +151,10 @@ void HostFloatingPointEnvironment::CheckAndRestoreFloatingPointEnvironment(
         "Folding with host runtime: fesetenv() failed while restoring fenv: %s",
         llvm::sys::StrError(errno).c_str());
   }
+#if __x86_64__
+  _mm_setcsr(originalMxcsr);
+#endif
+
   errno = 0;
 }
 } // namespace Fortran::evaluate::host
diff --git a/flang/lib/Evaluate/host.h b/flang/lib/Evaluate/host.h
index 20355ae3147a9..2fac0424f17cc 100644
--- a/flang/lib/Evaluate/host.h
+++ b/flang/lib/Evaluate/host.h
@@ -41,7 +41,9 @@ class HostFloatingPointEnvironment {
 
 private:
   std::fenv_t originalFenv_;
-  std::fenv_t currentFenv_;
+#if __x86_64__
+  unsigned int originalMxcsr;
+#endif
   RealFlags flags_;
   bool hasSubnormalFlushingHardwareControl_{false};
   bool hardwareFlagsAreReliable_{true};
diff --git a/flang/lib/Evaluate/intrinsics-library.cpp b/flang/lib/Evaluate/intrinsics-library.cpp
index 84c71e145bb60..c81ad4f34563a 100644
--- a/flang/lib/Evaluate/intrinsics-library.cpp
+++ b/flang/lib/Evaluate/intrinsics-library.cpp
@@ -57,7 +57,7 @@ static void AddLibmRealHostProcedures(
       {"asin", F{std::asin}, true},
       {"asinh", F{std::asinh}, true},
       {"atan", F{std::atan}, true},
-      {"atan", F2{std::atan2}, true},
+      {"atan2", F2{std::atan2}, true},
       {"atanh", F{std::atanh}, true},
       {"cos", F{std::cos}, true},
       {"cosh", F{std::cosh}, true},
@@ -135,7 +135,8 @@ static void AddLibmComplexHostProcedures(
   }
 }
 
-void InitHostIntrinsicLibraryWithLibm(HostIntrinsicProceduresLibrary &lib) {
+[[maybe_unused]] static void InitHostIntrinsicLibraryWithLibm(
+    HostIntrinsicProceduresLibrary &lib) {
   if constexpr (host::FortranTypeExists<float>()) {
     AddLibmRealHostProcedures<float>(lib);
   }
@@ -158,397 +159,213 @@ void InitHostIntrinsicLibraryWithLibm(HostIntrinsicProceduresLibrary &lib) {
 }
 
 #if LINK_WITH_LIBPGMATH
-namespace pgmath {
-// Define mapping between numerical intrinsics and libpgmath symbols
-// namespace is used to have shorter names on repeated patterns.
-// A class would be better to hold all these defs, but GCC does not
-// support specialization of template variables inside class even
-// if it is C++14 standard compliant here because there are only full
-// specializations.
-
-// List of intrinsics that have libpgmath implementations that can be used for
-// constant folding The tag names must match the name used inside libpgmath name
-// so that the macro below work.
-enum class I {
-  acos,
-  acosh,
-  asin,
-  asinh,
-  atan,
-  atan2,
-  atanh,
-  bessel_j0,
-  bessel_j1,
-  bessel_jn,
-  bessel_y0,
-  bessel_y1,
-  bessel_yn,
-  cos,
-  cosh,
-  erf,
-  erfc,
-  erfc_scaled,
-  exp,
-  gamma,
-  hypot,
-  log,
-  log10,
-  log_gamma,
-  mod,
-  pow,
-  sin,
-  sinh,
-  sqrt,
-  tan,
-  tanh
-};
+// Only use libpgmath for folding if it is available.
+// First declare all libpgmaths functions
+#define PGMATH_DECLARE
+#include "../runtime/pgmath.h.inc"
 
 // Library versions: P for Precise, R for Relaxed, F for Fast
 enum class L { F, R, P };
 
-struct NoSuchRuntimeSymbol {};
-template <L, I, typename> constexpr auto Sym{NoSuchRuntimeSymbol{}};
-
-// Macros to declare fast/relaxed/precise libpgmath variants.
-#define DECLARE_PGMATH_FAST_REAL(func) \
-  extern "C" float __fs_##func##_1(float); \
-  extern "C" double __fd_##func##_1(double); \
-  template <> constexpr auto Sym<L::F, I::func, float>{__fs_##func##_1}; \
-  template <> constexpr auto Sym<L::F, I::func, double>{__fd_##func##_1};
-
-#define DECLARE_PGMATH_FAST_COMPLEX(func) \
-  extern "C" float _Complex __fc_##func##_1(float _Complex); \
-  extern "C" double _Complex __fz_##func##_1(double _Complex); \
-  template <> \
-  constexpr auto Sym<L::F, I::func, std::complex<float>>{__fc_##func##_1}; \
-  template <> \
-  constexpr auto Sym<L::F, I::func, std::complex<double>>{__fz_##func##_1};
-
-#define DECLARE_PGMATH_FAST_ALL_FP(func) \
-  DECLARE_PGMATH_FAST_REAL(func) \
-  DECLARE_PGMATH_FAST_COMPLEX(func)
-
-#define DECLARE_PGMATH_PRECISE_REAL(func) \
-  extern "C" float __ps_##func##_1(float); \
-  extern "C" double __pd_##func##_1(double); \
-  template <> constexpr auto Sym<L::P, I::func, float>{__ps_##func##_1}; \
-  template <> constexpr auto Sym<L::P, I::func, double>{__pd_##func##_1};
-
-#define DECLARE_PGMATH_PRECISE_COMPLEX(func) \
-  extern "C" float _Complex __pc_##func##_1(float _Complex); \
-  extern "C" double _Complex __pz_##func##_1(double _Complex); \
-  template <> \
-  constexpr auto Sym<L::P, I::func, std::complex<float>>{__pc_##func##_1}; \
-  template <> \
-  constexpr auto Sym<L::P, I::func, std::complex<double>>{__pz_##func##_1};
-
-#define DECLARE_PGMATH_PRECISE_ALL_FP(func) \
-  DECLARE_PGMATH_PRECISE_REAL(func) \
-  DECLARE_PGMATH_PRECISE_COMPLEX(func)
-
-#define DECLARE_PGMATH_RELAXED_REAL(func) \
-  extern "C" float __rs_##func##_1(float); \
-  extern "C" double __rd_##func##_1(double); \
-  template <> constexpr auto Sym<L::R, I::func, float>{__rs_##func##_1}; \
-  template <> constexpr auto Sym<L::R, I::func, double>{__rd_##func##_1};
-
-#define DECLARE_PGMATH_RELAXED_COMPLEX(func) \
-  extern "C" float _Complex __rc_##func##_1(float _Complex); \
-  extern "C" double _Complex __rz_##func##_1(double _Complex); \
-  template <> \
-  constexpr auto Sym<L::R, I::func, std::complex<float>>{__rc_##func##_1}; \
-  template <> \
-  constexpr auto Sym<L::R, I::func, std::complex<double>>{__rz_##func##_1};
-
-#define DECLARE_PGMATH_RELAXED_ALL_FP(func) \
-  DECLARE_PGMATH_RELAXED_REAL(func) \
-  DECLARE_PGMATH_RELAXED_COMPLEX(func)
-
-#define DECLARE_PGMATH_REAL(func) \
-  DECLARE_PGMATH_FAST_REAL(func) \
-  DECLARE_PGMATH_PRECISE_REAL(func) \
-  DECLARE_PGMATH_RELAXED_REAL(func)
-
-#define DECLARE_PGMATH_COMPLEX(func) \
-  DECLARE_PGMATH_FAST_COMPLEX(func) \
-  DECLARE_PGMATH_PRECISE_COMPLEX(func) \
-  DECLARE_PGMATH_RELAXED_COMPLEX(func)
-
-#define DECLARE_PGMATH_ALL(func) \
-  DECLARE_PGMATH_REAL(func) \
-  DECLARE_PGMATH_COMPLEX(func)
-
-// Macros to declare fast/relaxed/precise libpgmath variants with two arguments.
-#define DECLARE_PGMATH_FAST_REAL2(func) \
-  extern "C" float __fs_##func##_1(float, float); \
-  extern "C" double __fd_##func##_1(double, double); \
-  template <> constexpr auto Sym<L::F, I::func, float>{__fs_##func##_1}; \
-  template <> constexpr auto Sym<L::F, I::func, double>{__fd_##func##_1};
-
-#define DECLARE_PGMATH_FAST_COMPLEX2(func) \
-  extern "C" float _Complex __fc_##func##_1(float _Complex, float _Complex); \
-  extern "C" double _Complex __fz_##func##_1( \
-      double _Complex, double _Complex); \
-  template <> \
-  constexpr auto Sym<L::F, I::func, std::complex<float>>{__fc_##func##_1}; \
-  template <> \
-  constexpr auto Sym<L::F, I::func, std::complex<double>>{__fz_##func##_1};
-
-#define DECLARE_PGMATH_FAST_ALL_FP2(func) \
-  DECLARE_PGMATH_FAST_REAL2(func) \
-  DECLARE_PGMATH_FAST_COMPLEX2(func)
-
-#define DECLARE_PGMATH_PRECISE_REAL2(func) \
-  extern "C" float __ps_##func##_1(float, float); \
-  extern "C" double __pd_##func##_1(double, double); \
-  template <> constexpr auto Sym<L::P, I::func, float>{__ps_##func##_1}; \
-  template <> constexpr auto Sym<L::P, I::func, double>{__pd_##func##_1};
-
-#define DECLARE_PGMATH_PRECISE_COMPLEX2(func) \
-  extern "C" float _Complex __pc_##func##_1(float _Complex, float _Complex); \
-  extern "C" double _Complex __pz_##func##_1( \
-      double _Complex, double _Complex); \
-  template <> \
-  constexpr auto Sym<L::P, I::func, std::complex<float>>{__pc_##func##_1}; \
-  template <> \
-  constexpr auto Sym<L::P, I::func, std::complex<double>>{__pz_##func##_1};
-
-#define DECLARE_PGMATH_PRECISE_ALL_FP2(func) \
-  DECLARE_PGMATH_PRECISE_REAL2(func) \
-  DECLARE_PGMATH_PRECISE_COMPLEX2(func)
-
-#define DECLARE_PGMATH_RELAXED_REAL2(func) \
-  extern "C" float __rs_##func##_1(float, float); \
-  extern "C" double __rd_##func##_1(double, double); \
-  template <> constexpr auto Sym<L::R, I::func, float>{__rs_##func##_1}; \
-  template <> constexpr auto Sym<L::R, I::func, double>{__rd_##func##_1};
-
-#define DECLARE_PGMATH_RELAXED_COMPLEX2(func) \
-  extern "C" float _Complex __rc_##func##_1(float _Complex, float _Complex); \
-  extern "C" double _Complex __rz_##func##_1( \
-      double _Complex, double _Complex); \
-  template <> \
-  constexpr auto Sym<L::R, I::func, std::complex<float>>{__rc_##func##_1}; \
-  template <> \
-  constexpr auto Sym<L::R, I::func, std::complex<double>>{__rz_##func##_1};
-
-#define DECLARE_PGMATH_RELAXED_ALL_FP2(func) \
-  DECLARE_PGMATH_RELAXED_REAL2(func) \
-  DECLARE_PGMATH_RELAXED_COMPLEX2(func)
-
-#define DECLARE_PGMATH_REAL2(func) \
-  DECLARE_PGMATH_FAST_REAL2(func) \
-  DECLARE_PGMATH_PRECISE_REAL2(func) \
-  DECLARE_PGMATH_RELAXED_REAL2(func)
-
-#define DECLARE_PGMATH_COMPLEX2(func) \
-  DECLARE_PGMATH_FAST_COMPLEX2(func) \
-  DECLARE_PGMATH_PRECISE_COMPLEX2(func) \
-  DECLARE_PGMATH_RELAXED_COMPLEX2(func)
-
-#define DECLARE_PGMATH_ALL2(func) \
-  DECLARE_PGMATH_REAL2(func) \
-  DECLARE_PGMATH_COMPLEX2(func)
-
-// Marcos to declare __mth_i libpgmath variants
-#define DECLARE_PGMATH_MTH_VERSION_REAL(func) \
-  extern "C" float __mth_i_##func(float); \
-  extern "C" double __mth_i_d##func(double); \
-  template <> constexpr auto Sym<L::F, I::func, float>{__mth_i_##func}; \
-  template <> constexpr auto Sym<L::F, I::func, double>{__mth_i_d##func}; \
-  template <> constexpr auto Sym<L::P, I::func, float>{__mth_i_##func}; \
-  template <> constexpr auto Sym<L::P, I::func, double>{__mth_i_d##func}; \
-  template <> constexpr auto Sym<L::R, I::func, float>{__mth_i_##func}; \
-  template <> constexpr auto Sym<L::R, I::func, double>{__mth_i_d##func};
-
-// Actual libpgmath declarations
-DECLARE_PGMATH_ALL(acos)
-DECLARE_PGMATH_MTH_VERSION_REAL(acosh)
-DECLARE_PGMATH_ALL(asin)
-DECLARE_PGMATH_MTH_VERSION_REAL(asinh)
-DECLARE_PGMATH_ALL(atan)
-DECLARE_PGMATH_REAL2(atan2)
-DECLARE_PGMATH_MTH_VERSION_REAL(atanh)
-DECLARE_PGMATH_MTH_VERSION_REAL(bessel_j0)
-DECLARE_PGMATH_MTH_VERSION_REAL(bessel_j1)
-DECLARE_PGMATH_MTH_VERSION_REAL(bessel_y0)
-DECLARE_PGMATH_MTH_VERSION_REAL(bessel_y1)
-// bessel_jn and bessel_yn takes an int as first arg
-extern "C" float __mth_i_bessel_jn(int, float);
-extern "C" double __mth_i_dbessel_jn(int, double);
-template <> constexpr auto Sym<L::F, I::bessel_jn, float>{__mth_i_bessel_jn};
-template <> constexpr auto Sym<L::F, I::bessel_jn, double>{__mth_i_dbessel_jn};
-template <> constexpr auto Sym<L::P, I::bessel_jn, float>{__mth_i_bessel_jn};
-template <> constexpr auto Sym<L::P, I::bessel_jn, double>{__mth_i_dbessel_jn};
-template <> constexpr auto Sym<L::R, I::bessel_jn, float>{__mth_i_bessel_jn};
-template <> constexpr auto Sym<L::R, I::bessel_jn, double>{__mth_i_dbessel_jn};
-extern "C" float __mth_i_bessel_yn(int, float);
-extern "C" double __mth_i_dbessel_yn(int, double);
-template <> constexpr auto Sym<L::F, I::bessel_yn, float>{__mth_i_bessel_yn};
-template <> constexpr auto Sym<L::F, I::bessel_yn, double>{__mth_i_dbessel_yn};
-template <> constexpr auto Sym<L::P, I::bessel_yn, float>{__mth_i_bessel_yn};
-template <> constexpr auto Sym<L::P, I::bessel_yn, double>{__mth_i_dbessel_yn};
-template <> constexpr auto Sym<L::R, I::bessel_yn, float>{__mth_i_bessel_yn};
-template <> constexpr auto Sym<L::R, I::bessel_yn, double>{__mth_i_dbessel_yn};
-DECLARE_PGMATH_ALL(cos)
-DECLARE_PGMATH_ALL(cosh)
-DECLARE_PGMATH_MTH_VERSION_REAL(erf)
-DECLARE_PGMATH_MTH_VERSION_REAL(erfc)
-DECLARE_PGMATH_MTH_VERSION_REAL(erfc_scaled)
-DECLARE_PGMATH_ALL(exp)
-DECLARE_PGMATH_MTH_VERSION_REAL(gamma)
-extern "C" float __mth_i_hypot(float, float);
-extern "C" double __mth_i_dhypot(double, double);
-template <> constexpr auto Sym<L::F, I::hypot, float>{__mth_i_hypot};
-template <> constexpr auto Sym<L::F, I::hypot, double>{__mth_i_dhypot};
-template <> constexpr auto Sym<L::P, I::hypot, float>{__mth_i_hypot};
-template <> constexpr auto Sym<L::P, I::hypot, double>{__mth_i_dhypot};
-template <> constexpr auto Sym<L::R, I::hypot, float>{__mth_i_hypot};
-template <> constexpr auto Sym<L::R, I::hypot, double>{__mth_i_dhypot};
-DECLARE_PGMATH_ALL(log)
-DECLARE_PGMATH_REAL(log10)
-DECLARE_PGMATH_MTH_VERSION_REAL(log_gamma)
-// no function for modulo in libpgmath
-extern "C" float __fs_mod_1(float, float);
-extern "C" double __fd_mod_1(double, double);
-template <> constexpr auto Sym<L::F, I::mod, float>{__fs_mod_1};
-template <> constexpr auto Sym<L::F, I::mod, double>{__fd_mod_1};
-template <> constexpr auto Sym<L::P, I::mod, float>{__fs_mod_1};
-template <> constexpr auto Sym<L::P, I::mod, double>{__fd_mod_1};
-template <> constexpr auto Sym<L::R, I::mod, float>{__fs_mod_1};
-template <> constexpr auto Sym<L::R, I::mod, double>{__fd_mod_1};
-DECLARE_PGMATH_ALL2(pow)
-DECLARE_PGMATH_ALL(sin)
-DECLARE_PGMATH_ALL(sinh)
-DECLARE_PGMATH_MTH_VERSION_REAL(sqrt)
-DECLARE_PGMATH_COMPLEX(sqrt) // real versions are __mth_i...
-DECLARE_PGMATH_ALL(tan)
-DECLARE_PGMATH_ALL(tanh)
-
 // Fill the function map used for folding with libpgmath symbols
-template <L Lib, typename HostT>
-static void AddLibpgmathRealHostProcedures(
+template <L Lib>
+static void AddLibpgmathFloatHostProcedures(
     HostIntrinsicProceduresLibrary &hostIntrinsicLibrary) {
-  static_assert(std::is_same_v<HostT, float> || std::is_same_v<HostT, double>);
-  HostRuntimeIntrinsicProcedure pgmathSymbols[]{
-      {"acos", Sym<Lib, I::acos, HostT>, true},
-      {"acosh", Sym<Lib, I::acosh, HostT>, true},
-      {"asin", Sym<Lib, I::asin, HostT>, true},
-      {"asinh", Sym<Lib, I::asinh, HostT>, true},
-      {"atan", Sym<Lib, I::atan, HostT>, true},
-      {"atan", Sym<Lib, I::atan2, HostT>,
-          true}, // atan is also the generic name for atan2
-      {"atanh", Sym<Lib, I::atanh, HostT>, true},
-      {"bessel_j0", Sym<Lib, I::bessel_j0, HostT>, true},
-      {"bessel_j1", Sym<Lib, I::bessel_j1, HostT>, true},
-      {"bessel_jn", Sym<Lib, I::bessel_jn, HostT>, true},
-      {"bessel_y0", Sym<Lib, I::bessel_y0, HostT>, true},
-      {"bessel_y1", Sym<Lib, I::bessel_y1, HostT>, true},
-      {"bessel_yn", Sym<Lib, I::bessel_yn, HostT>, true},
-      {"cos", Sym<Lib, I::cos, HostT>, true},
-      {"cosh", Sym<Lib, I::cosh, HostT>, true},
-      {"erf", Sym<Lib, I::erf, HostT>, true},
-      {"erfc", Sym<Lib, I::erfc, HostT>, true},
-      {"erfc_scaled", Sym<Lib, I::erfc_scaled, HostT>, true},
-      {"exp", Sym<Lib, I::exp, HostT>, true},
-      {"gamma", Sym<Lib, I::gamma, HostT>, true},
-      {"hypot", Sym<Lib, I::hypot, HostT>, true},
-      {"log", Sym<Lib, I::log, HostT>, true},
-      {"log10", Sym<Lib, I::log10, HostT>, true},
-      {"log_gamma", Sym<Lib, I::log_gamma, HostT>, true},
-      {"mod", Sym<Lib, I::mod, HostT>, true},
-      {"pow", Sym<Lib, I::pow, HostT>, true},
-      {"sin", Sym<Lib, I::sin, HostT>, true},
-      {"sinh", Sym<Lib, I::sinh, HostT>, true},
-      {"sqrt", Sym<Lib, I::sqrt, HostT>, true},
-      {"tan", Sym<Lib, I::tan, HostT>, true},
-      {"tanh", Sym<Lib, I::tanh, HostT>, true},
-  };
+  if constexpr (Lib == L::F) {
+    HostRuntimeIntrinsicProcedure pgmathSymbols[]{
+#define PGMATH_FAST
+#define PGMATH_USE_S(name, function) {#name, function, true},
+#include "../runtime/pgmath.h.inc"
+    };
+    for (auto sym : pgmathSymbols) {
+      hostIntrinsicLibrary.AddProcedure(std::move(sym));
+    }
+  } else if constexpr (Lib == L::R) {
+    HostRuntimeIntrinsicProcedure pgmathSymbols[]{
+#define PGMATH_RELAXED
+#define PGMATH_USE_S(name, function) {#name, function, true},
+#include "../runtime/pgmath.h.inc"
+    };
+    for (auto sym : pgmathSymbols) {
+      hostIntrinsicLibrary.AddProcedure(std::move(sym));
+    }
+  } else {
+    static_assert(Lib == L::P && "unexpected libpgmath version");
+    HostRuntimeIntrinsicProcedure pgmathSymbols[]{
+#define PGMATH_PRECISE
+#define PGMATH_USE_S(name, function) {#name, function, true},
+#include "../runtime/pgmath.h.inc"
+    };
+    for (auto sym : pgmathSymbols) {
+      hostIntrinsicLibrary.AddProcedure(std::move(sym));
+    }
+  }
+}
 
-  for (auto sym : pgmathSymbols) {
-    hostIntrinsicLibrary.AddProcedure(std::move(sym));
+template <L Lib>
+static void AddLibpgmathDoubleHostProcedures(
+    HostIntrinsicProceduresLibrary &hostIntrinsicLibrary) {
+  if constexpr (Lib == L::F) {
+    HostRuntimeIntrinsicProcedure pgmathSymbols[]{
+#define PGMATH_FAST
+#define PGMATH_USE_D(name, function) {#name, function, true},
+#include "../runtime/pgmath.h.inc"
+    };
+    for (auto sym : pgmathSymbols) {
+      hostIntrinsicLibrary.AddProcedure(std::move(sym));
+    }
+  } else if constexpr (Lib == L::R) {
+    HostRuntimeIntrinsicProcedure pgmathSymbols[]{
+#define PGMATH_RELAXED
+#define PGMATH_USE_D(name, function) {#name, function, true},
+#include "../runtime/pgmath.h.inc"
+    };
+    for (auto sym : pgmathSymbols) {
+      hostIntrinsicLibrary.AddProcedure(std::move(sym));
+    }
+  } else {
+    static_assert(Lib == L::P && "unexpected libpgmath version");
+    HostRuntimeIntrinsicProcedure pgmathSymbols[]{
+#define PGMATH_PRECISE
+#define PGMATH_USE_D(name, function) {#name, function, true},
+#include "../runtime/pgmath.h.inc"
+    };
+    for (auto sym : pgmathSymbols) {
+      hostIntrinsicLibrary.AddProcedure(std::move(sym));
+    }
   }
 }
 
-// Note: std::complex and _complex are layout compatible but are not guaranteed
+// Note: Lipgmath uses _Complex but the front-end use std::complex for folding.
+// std::complex and _Complex are layout compatible but are not guaranteed
 // to be linkage compatible. For instance, on i386, float _Complex is returned
 // by a pair of register but std::complex<float> is returned by structure
 // address. To fix the issue, wrapper around C _Complex functions are defined
 // below.
-template <FuncPointer<float _Complex, float _Complex> func>
-static std::complex<float> ComplexCFuncWrapper(std::complex<float> &arg) {
-  float _Complex res{func(*reinterpret_cast<float _Complex *>(&arg))};
-  return *reinterpret_cast<std::complex<float> *>(&res);
-}
 
-template <FuncPointer<double _Complex, double _Complex> func>
-static std::complex<double> ComplexCFuncWrapper(std::complex<double> &arg) {
-  double _Complex res{func(*reinterpret_cast<double _Complex *>(&arg))};
-  return *reinterpret_cast<std::complex<double> *>(&res);
-}
-
-template <FuncPointer<float _Complex, float _Complex, float _Complex> func>
-static std::complex<float> ComplexCFuncWrapper(
-    std::complex<float> &arg1, std::complex<float> &arg2) {
-  float _Complex res{func(*reinterpret_cast<float _Complex *>(&arg1),
-      *reinterpret_cast<float _Complex *>(&arg2))};
-  return *reinterpret_cast<std::complex<float> *>(&res);
-}
+template <typename T> struct ToStdComplex {
+  using Type = T;
+  using AType = Type;
+};
 
-template <FuncPointer<double _Complex, double _Complex, double _Complex> func>
-static std::complex<double> ComplexCFuncWrapper(
-    std::complex<double> &arg1, std::complex<double> &arg2) {
-  double _Complex res{func(*reinterpret_cast<double _Complex *>(&arg1),
-      *reinterpret_cast<double _Complex *>(&arg2))};
-  return *reinterpret_cast<std::complex<double> *>(&res);
-}
+template <typename F, F func> struct CComplexFunc {};
+template <typename R, typename... A, FuncPointer<R, A...> func>
+struct CComplexFunc<FuncPointer<R, A...>, func> {
+  static typename ToStdComplex<R>::Type wrapper(
+      typename ToStdComplex<A>::AType... args) {
+    R res{func(*reinterpret_cast<A *>(&args)...)};
+    return *reinterpret_cast<typename ToStdComplex<R>::Type *>(&res);
+  }
+};
 
-template <L Lib, typename HostT>
+template <L Lib>
 static void AddLibpgmathComplexHostProcedures(
     HostIntrinsicProceduresLibrary &hostIntrinsicLibrary) {
-  static_assert(std::is_same_v<HostT, float> || std::is_same_v<HostT, double>);
-  using CHostT = std::complex<HostT>;
+  if constexpr (Lib == L::F) {
+    HostRuntimeIntrinsicProcedure pgmathSymbols[]{
+#define PGMATH_FAST
+#define PGMATH_USE_C(name, function) \
+  {#name, CComplexFunc<decltype(&function), &function>::wrapper, true},
+#include "../runtime/pgmath.h.inc"
+    };
+    for (auto sym : pgmathSymbols) {
+      hostIntrinsicLibrary.AddProcedure(std::move(sym));
+    }
+  } else if constexpr (Lib == L::R) {
+    HostRuntimeIntrinsicProcedure pgmathSymbols[]{
+#define PGMATH_RELAXED
+#define PGMATH_USE_C(name, function) \
+  {#name, CComplexFunc<decltype(&function), &function>::wrapper, true},
+#include "../runtime/pgmath.h.inc"
+    };
+    for (auto sym : pgmathSymbols) {
+      hostIntrinsicLibrary.AddProcedure(std::move(sym));
+    }
+  } else {
+    static_assert(Lib == L::P && "unexpected libpgmath version");
+    HostRuntimeIntrinsicProcedure pgmathSymbols[]{
+#define PGMATH_PRECISE
+#define PGMATH_USE_C(name, function) \
+  {#name, CComplexFunc<decltype(&function), &function>::wrapper, true},
+#include "../runtime/pgmath.h.inc"
+    };
+    for (auto sym : pgmathSymbols) {
+      hostIntrinsicLibrary.AddProcedure(std::move(sym));
+    }
+  }
+
   // cmath is used to complement pgmath when symbols are not available
+  using HostT = float;
+  using CHostT = std::complex<HostT>;
   using CmathF = FuncPointer<CHostT, const CHostT &>;
-  HostRuntimeIntrinsicProcedure pgmathSymbols[]{
-      {"abs", FuncPointer<HostT, const CHostT &>{std::abs}, true},
-      {"acos", ComplexCFuncWrapper<Sym<Lib, I::acos, CHostT>>, true},
-      {"acosh", CmathF{std::acosh}, true},
-      {"asin", ComplexCFuncWrapper<Sym<Lib, I::asin, CHostT>>, true},
-      {"asinh", CmathF{std::asinh}, true},
-      {"atan", ComplexCFuncWrapper<Sym<Lib, I::atan, CHostT>>, true},
-      {"atanh", CmathF{std::atanh}, true},
-      {"cos", ComplexCFuncWrapper<Sym<Lib, I::cos, CHostT>>, true},
-      {"cosh", ComplexCFuncWrapper<Sym<Lib, I::cosh, CHostT>>, true},
-      {"exp", ComplexCFuncWrapper<Sym<Lib, I::exp, CHostT>>, true},
-      {"log", ComplexCFuncWrapper<Sym<Lib, I::log, CHostT>>, true},
-      {"pow", ComplexCFuncWrapper<Sym<Lib, I::pow, CHostT>>, true},
-      {"sin", ComplexCFuncWrapper<Sym<Lib, I::sin, CHostT>>, true},
-      {"sinh", ComplexCFuncWrapper<Sym<Lib, I::sinh, CHostT>>, true},
-      {"sqrt", ComplexCFuncWrapper<Sym<Lib, I::sqrt, CHostT>>, true},
-      {"tan", ComplexCFuncWrapper<Sym<Lib, I::tan, CHostT>>, true},
-      {"tanh", ComplexCFuncWrapper<Sym<Lib, I::tanh, CHostT>>, true},
-  };
+  hostIntrinsicLibrary.AddProcedure(
+      {"abs", FuncPointer<HostT, const CHostT &>{std::abs}, true});
+  hostIntrinsicLibrary.AddProcedure({"acosh", CmathF{std::acosh}, true});
+  hostIntrinsicLibrary.AddProcedure({"asinh", CmathF{std::asinh}, true});
+  hostIntrinsicLibrary.AddProcedure({"atanh", CmathF{std::atanh}, true});
+}
 
-  for (auto sym : pgmathSymbols) {
-    hostIntrinsicLibrary.AddProcedure(std::move(sym));
+template <L Lib>
+static void AddLibpgmathDoubleComplexHostProcedures(
+    HostIntrinsicProceduresLibrary &hostIntrinsicLibrary) {
+  if constexpr (Lib == L::F) {
+    HostRuntimeIntrinsicProcedure pgmathSymbols[]{
+#define PGMATH_FAST
+#define PGMATH_USE_Z(name, function) \
+  {#name, CComplexFunc<decltype(&function), &function>::wrapper, true},
+#include "../runtime/pgmath.h.inc"
+    };
+    for (auto sym : pgmathSymbols) {
+      hostIntrinsicLibrary.AddProcedure(std::move(sym));
+    }
+  } else if constexpr (Lib == L::R) {
+    HostRuntimeIntrinsicProcedure pgmathSymbols[]{
+#define PGMATH_RELAXED
+#define PGMATH_USE_Z(name, function) \
+  {#name, CComplexFunc<decltype(&function), &function>::wrapper, true},
+#include "../runtime/pgmath.h.inc"
+    };
+    for (auto sym : pgmathSymbols) {
+      hostIntrinsicLibrary.AddProcedure(std::move(sym));
+    }
+  } else {
+    static_assert(Lib == L::P && "unexpected libpgmath version");
+    HostRuntimeIntrinsicProcedure pgmathSymbols[]{
+#define PGMATH_PRECISE
+#define PGMATH_USE_Z(name, function) \
+  {#name, CComplexFunc<decltype(&function), &function>::wrapper, true},
+#include "../runtime/pgmath.h.inc"
+    };
+    for (auto sym : pgmathSymbols) {
+      hostIntrinsicLibrary.AddProcedure(std::move(sym));
+    }
   }
+
+  // cmath is used to complement pgmath when symbols are not available
+  using HostT = double;
+  using CHostT = std::complex<HostT>;
+  using CmathF = FuncPointer<CHostT, const CHostT &>;
+  hostIntrinsicLibrary.AddProcedure(
+      {"abs", FuncPointer<HostT, const CHostT &>{std::abs}, true});
+  hostIntrinsicLibrary.AddProcedure({"acosh", CmathF{std::acosh}, true});
+  hostIntrinsicLibrary.AddProcedure({"asinh", CmathF{std::asinh}, true});
+  hostIntrinsicLibrary.AddProcedure({"atanh", CmathF{std::atanh}, true});
 }
 
 template <L Lib>
 static void InitHostIntrinsicLibraryWithLibpgmath(
     HostIntrinsicProceduresLibrary &lib) {
   if constexpr (host::FortranTypeExists<float>()) {
-    AddLibpgmathRealHostProcedures<Lib, float>(lib);
+    AddLibpgmathFloatHostProcedures<Lib>(lib);
   }
   if constexpr (host::FortranTypeExists<double>()) {
-    AddLibpgmathRealHostProcedures<Lib, double>(lib);
+    AddLibpgmathDoubleHostProcedures<Lib>(lib);
   }
   if constexpr (host::FortranTypeExists<std::complex<float>>()) {
-    AddLibpgmathComplexHostProcedures<Lib, float>(lib);
+    AddLibpgmathComplexHostProcedures<Lib>(lib);
   }
   if constexpr (host::FortranTypeExists<std::complex<double>>()) {
-    AddLibpgmathComplexHostProcedures<Lib, double>(lib);
+    AddLibpgmathDoubleComplexHostProcedures<Lib>(lib);
   }
   // No long double functions in libpgmath
   if constexpr (host::FortranTypeExists<long double>()) {
@@ -558,7 +375,6 @@ static void InitHostIntrinsicLibraryWithLibpgmath(
     AddLibmComplexHostProcedures<long double>(lib);
   }
 }
-} // namespace pgmath
 #endif // LINK_WITH_LIBPGMATH
 
 // Define which host runtime functions will be used for folding
@@ -571,11 +387,11 @@ HostIntrinsicProceduresLibrary::HostIntrinsicProceduresLibrary() {
   // to silence clang warnings on unused symbols if all declared pgmath
   // symbols are not used somewhere.
   if (true) {
-    pgmath::InitHostIntrinsicLibraryWithLibpgmath<pgmath::L::P>(*this);
+    InitHostIntrinsicLibraryWithLibpgmath<L::P>(*this);
   } else if (false) {
-    pgmath::InitHostIntrinsicLibraryWithLibpgmath<pgmath::L::F>(*this);
+    InitHostIntrinsicLibraryWithLibpgmath<L::F>(*this);
   } else {
-    pgmath::InitHostIntrinsicLibraryWithLibpgmath<pgmath::L::R>(*this);
+    InitHostIntrinsicLibraryWithLibpgmath<L::R>(*this);
   }
 #else
   InitHostIntrinsicLibraryWithLibm(*this);
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 0f92dd8e7f505..6237499d12ef7 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -496,28 +496,28 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"logical", {{"l", AnyLogical}, DefaultingKIND}, KINDLogical},
     {"log_gamma", {{"x", SameReal}}, SameReal},
     {"matmul",
-        {{"array_a", AnyLogical, Rank::vector},
-            {"array_b", AnyLogical, Rank::matrix}},
+        {{"matrix_a", AnyLogical, Rank::vector},
+            {"matrix_b", AnyLogical, Rank::matrix}},
         ResultLogical, Rank::vector, IntrinsicClass::transformationalFunction},
     {"matmul",
-        {{"array_a", AnyLogical, Rank::matrix},
-            {"array_b", AnyLogical, Rank::vector}},
+        {{"matrix_a", AnyLogical, Rank::matrix},
+            {"matrix_b", AnyLogical, Rank::vector}},
         ResultLogical, Rank::vector, IntrinsicClass::transformationalFunction},
     {"matmul",
-        {{"array_a", AnyLogical, Rank::matrix},
-            {"array_b", AnyLogical, Rank::matrix}},
+        {{"matrix_a", AnyLogical, Rank::matrix},
+            {"matrix_b", AnyLogical, Rank::matrix}},
         ResultLogical, Rank::matrix, IntrinsicClass::transformationalFunction},
     {"matmul",
-        {{"array_a", AnyNumeric, Rank::vector},
-            {"array_b", AnyNumeric, Rank::matrix}},
+        {{"matrix_a", AnyNumeric, Rank::vector},
+            {"matrix_b", AnyNumeric, Rank::matrix}},
         ResultNumeric, Rank::vector, IntrinsicClass::transformationalFunction},
     {"matmul",
-        {{"array_a", AnyNumeric, Rank::matrix},
-            {"array_b", AnyNumeric, Rank::vector}},
+        {{"matrix_a", AnyNumeric, Rank::matrix},
+            {"matrix_b", AnyNumeric, Rank::vector}},
         ResultNumeric, Rank::vector, IntrinsicClass::transformationalFunction},
     {"matmul",
-        {{"array_a", AnyNumeric, Rank::matrix},
-            {"array_b", AnyNumeric, Rank::matrix}},
+        {{"matrix_a", AnyNumeric, Rank::matrix},
+            {"matrix_b", AnyNumeric, Rank::matrix}},
         ResultNumeric, Rank::matrix, IntrinsicClass::transformationalFunction},
     {"maskl", {{"i", AnyInt}, DefaultingKIND}, KINDInt},
     {"maskr", {{"i", AnyInt}, DefaultingKIND}, KINDInt},
@@ -581,6 +581,10 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
         Rank::dimReduced, IntrinsicClass::transformationalFunction},
     {"not", {{"i", SameInt}}, SameInt},
     // NULL() is a special case handled in Probe() below
+    {"num_images", {}, DefaultInt, Rank::scalar,
+        IntrinsicClass::transformationalFunction},
+    {"num_images", {{"team_number", AnyInt, Rank::scalar}}, DefaultInt,
+        Rank::scalar, IntrinsicClass::transformationalFunction},
     {"out_of_range",
         {{"x", AnyIntOrReal}, {"mold", AnyIntOrReal, Rank::scalar}},
         DefaultLogical},
@@ -724,7 +728,7 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
 
 // TODO: Coarray intrinsic functions
 //   LCOBOUND, UCOBOUND, FAILED_IMAGES, GET_TEAM, IMAGE_INDEX,
-//   NUM_IMAGES, STOPPED_IMAGES, TEAM_NUMBER, THIS_IMAGE,
+//   STOPPED_IMAGES, TEAM_NUMBER, THIS_IMAGE,
 //   COSHAPE
 // TODO: Non-standard intrinsic functions
 //  AND, OR, XOR, LSHIFT, RSHIFT, SHIFT, ZEXT, IZEXT,
@@ -1904,7 +1908,6 @@ std::optional<SpecificCall> IntrinsicProcTable::Implementation::Probe(
   }
 
   if (call.isSubroutineCall) {
-    parser::Messages buffer;
     auto subrRange{subroutines_.equal_range(call.name)};
     for (auto iter{subrRange.first}; iter != subrRange.second; ++iter) {
       if (auto specificCall{
diff --git a/flang/lib/Evaluate/shape.cpp b/flang/lib/Evaluate/shape.cpp
index 507de428b54a9..90d864466e2dc 100644
--- a/flang/lib/Evaluate/shape.cpp
+++ b/flang/lib/Evaluate/shape.cpp
@@ -205,7 +205,7 @@ auto GetLowerBoundHelper::operator()(const Symbol &symbol0) -> Result {
       if (j++ == dimension_) {
         if (const auto &bound{shapeSpec.lbound().GetExplicit()}) {
           return Fold(context_, common::Clone(*bound));
-        } else if (semantics::IsDescriptor(symbol)) {
+        } else if (IsDescriptor(symbol)) {
           return ExtentExpr{DescriptorInquiry{NamedEntity{symbol0},
               DescriptorInquiry::Field::LowerBound, dimension_}};
         } else {
@@ -230,7 +230,7 @@ auto GetLowerBoundHelper::operator()(const Component &component) -> Result {
         if (j++ == dimension_) {
           if (const auto &bound{shapeSpec.lbound().GetExplicit()}) {
             return Fold(context_, common::Clone(*bound));
-          } else if (semantics::IsDescriptor(symbol)) {
+          } else if (IsDescriptor(symbol)) {
             return ExtentExpr{
                 DescriptorInquiry{NamedEntity{common::Clone(component)},
                     DescriptorInquiry::Field::LowerBound, dimension_}};
@@ -545,6 +545,23 @@ auto GetShapeHelper::operator()(const ProcedureRef &call) const -> Result {
       if (!call.arguments().empty()) {
         return (*this)(call.arguments()[0]);
       }
+    } else if (intrinsic->name == "matmul") {
+      if (call.arguments().size() == 2) {
+        if (auto ashape{(*this)(call.arguments()[0])}) {
+          if (auto bshape{(*this)(call.arguments()[1])}) {
+            if (ashape->size() == 1 && bshape->size() == 2) {
+              bshape->erase(bshape->begin());
+              return std::move(*bshape); // matmul(vector, matrix)
+            } else if (ashape->size() == 2 && bshape->size() == 1) {
+              ashape->pop_back();
+              return std::move(*ashape); // matmul(matrix, vector)
+            } else if (ashape->size() == 2 && bshape->size() == 2) {
+              (*ashape)[1] = std::move((*bshape)[1]);
+              return std::move(*ashape); // matmul(matrix, matrix)
+            }
+          }
+        }
+      }
     } else if (intrinsic->name == "reshape") {
       if (call.arguments().size() >= 2 && call.arguments().at(1)) {
         // SHAPE(RESHAPE(array,shape)) -> shape
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 3538cd587f970..677b263408b39 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -1003,7 +1003,7 @@ bool IsSaved(const Symbol &original) {
     } else if (const Symbol * block{FindCommonBlockContaining(symbol)};
                block && block->attrs().test(Attr::SAVE)) {
       return true;
-    } else if (IsDummy(symbol)) {
+    } else if (IsDummy(symbol) || IsFunctionResult(symbol)) {
       return false;
     } else {
       for (; !scope->IsGlobal(); scope = &scope->parent()) {
@@ -1026,6 +1026,13 @@ bool IsDummy(const Symbol &symbol) {
       symbol.details());
 }
 
+bool IsFunctionResult(const Symbol &symbol) {
+  return (symbol.has<ObjectEntityDetails>() &&
+             symbol.get<ObjectEntityDetails>().isFuncResult()) ||
+      (symbol.has<ProcEntityDetails>() &&
+          symbol.get<ProcEntityDetails>().isFuncResult());
+}
+
 int CountLenParameters(const DerivedTypeSpec &type) {
   return std::count_if(type.parameters().begin(), type.parameters().end(),
       [](const auto &pair) { return pair.second.isLen(); });
diff --git a/flang/lib/Evaluate/type.cpp b/flang/lib/Evaluate/type.cpp
index 0a823cd20398f..293cd18648148 100644
--- a/flang/lib/Evaluate/type.cpp
+++ b/flang/lib/Evaluate/type.cpp
@@ -22,17 +22,21 @@
 
 // IsDescriptor() predicate: true when a symbol is implemented
 // at runtime with a descriptor.
-// TODO there's probably a better place for this predicate than here
 namespace Fortran::semantics {
 
-static bool IsDescriptor(const ObjectEntityDetails &details) {
-  if (const auto *type{details.type()}) {
+static bool IsDescriptor(const DeclTypeSpec *type) {
+  if (type) {
     if (auto dynamicType{evaluate::DynamicType::From(*type)}) {
-      if (dynamicType->RequiresDescriptor()) {
-        return true;
-      }
+      return dynamicType->RequiresDescriptor();
     }
   }
+  return false;
+}
+
+static bool IsDescriptor(const ObjectEntityDetails &details) {
+  if (IsDescriptor(details.type())) {
+    return true;
+  }
   // TODO: Automatic (adjustable) arrays - are they descriptors?
   for (const ShapeSpec &shapeSpec : details.shape()) {
     const auto &lb{shapeSpec.lbound().GetExplicit()};
@@ -62,6 +66,7 @@ bool IsDescriptor(const Symbol &symbol) {
                        symbol.attrs().test(Attr::EXTERNAL)) &&
                 IsDescriptor(d);
           },
+          [&](const EntityDetails &d) { return IsDescriptor(d.type()); },
           [](const AssocEntityDetails &d) {
             if (const auto &expr{d.expr()}) {
               if (expr->Rank() > 0) {
@@ -149,7 +154,7 @@ bool DynamicType::IsAssumedLengthCharacter() const {
       charLength_->isAssumed();
 }
 
-bool DynamicType::IsUnknownLengthCharacter() const {
+bool DynamicType::IsNonConstantLengthCharacter() const {
   if (category_ != TypeCategory::Character) {
     return false;
   } else if (!charLength_) {
@@ -471,7 +476,7 @@ DynamicType DynamicType::ResultTypeForMultiply(const DynamicType &that) const {
 }
 
 bool DynamicType::RequiresDescriptor() const {
-  return IsPolymorphic() || IsUnknownLengthCharacter() ||
+  return IsPolymorphic() || IsNonConstantLengthCharacter() ||
       (derived_ && CountNonConstantLenParameters(*derived_) > 0);
 }
 
diff --git a/flang/lib/Evaluate/variable.cpp b/flang/lib/Evaluate/variable.cpp
index 3a6cf560384c8..8c9bd18285af4 100644
--- a/flang/lib/Evaluate/variable.cpp
+++ b/flang/lib/Evaluate/variable.cpp
@@ -8,6 +8,7 @@
 
 #include "flang/Evaluate/variable.h"
 #include "flang/Common/idioms.h"
+#include "flang/Evaluate/check-expression.h"
 #include "flang/Evaluate/fold.h"
 #include "flang/Evaluate/tools.h"
 #include "flang/Parser/char-block.h"
@@ -259,16 +260,13 @@ static std::optional<Expr<SubscriptInteger>> SymbolLEN(const Symbol &sym) {
     if (const semantics::ParamValue * len{dyType->charLength()}) {
       if (len->isExplicit()) {
         if (auto intExpr{len->GetExplicit()}) {
-          return ConvertToType<SubscriptInteger>(*std::move(intExpr));
-        } else {
-          // There was an error constructing this symbol's type.  It should
-          // have a length expression, but we couldn't retrieve it
-          return std::nullopt;
+          if (IsConstantExpr(*intExpr)) {
+            return ConvertToType<SubscriptInteger>(*std::move(intExpr));
+          }
         }
-      } else {
-        return Expr<SubscriptInteger>{
-            DescriptorInquiry{NamedEntity{sym}, DescriptorInquiry::Field::Len}};
       }
+      return Expr<SubscriptInteger>{
+          DescriptorInquiry{NamedEntity{sym}, DescriptorInquiry::Field::Len}};
     }
   }
   return std::nullopt;
diff --git a/flang/lib/Lower/.clang-tidy b/flang/lib/Lower/.clang-tidy
new file mode 100644
index 0000000000000..87ec2ff53af6e
--- /dev/null
+++ b/flang/lib/Lower/.clang-tidy
@@ -0,0 +1,19 @@
+# Almost identical to the top-level .clang-tidy, except that {Member,Parameter,Variable}Case use camelBack.
+Checks: '-*,clang-diagnostic-*,llvm-*,misc-*,-misc-unused-parameters,-misc-non-private-member-variables-in-classes,readability-identifier-naming'
+CheckOptions:
+  - key:             readability-identifier-naming.ClassCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.EnumCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.FunctionCase
+    value:           camelBack
+  - key:             readability-identifier-naming.MemberCase
+    value:           camelBack
+  - key:             readability-identifier-naming.ParameterCase
+    value:           camelBack
+  - key:             readability-identifier-naming.UnionCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.VariableCase
+    value:           camelBack
+  - key:             readability-identifier-naming.IgnoreMainLikeFunctions
+    value:           1
diff --git a/flang/lib/Lower/CMakeLists.txt b/flang/lib/Lower/CMakeLists.txt
index 8602c23cc578e..3cd71c007a00a 100644
--- a/flang/lib/Lower/CMakeLists.txt
+++ b/flang/lib/Lower/CMakeLists.txt
@@ -2,9 +2,15 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error -Wno-unused-parameter")
 get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 
 add_flang_library(FortranLower
+  CharacterExpr.cpp
+  CharacterRuntime.cpp
+  Coarray.cpp
   ComplexExpr.cpp
   ConvertType.cpp
   DoLoopHelper.cpp
+  FIRBuilder.cpp
+  IntrinsicCall.cpp
+  IO.cpp
   Mangler.cpp
   OpenMP.cpp
   PFTBuilder.cpp
@@ -16,6 +22,10 @@ add_flang_library(FortranLower
   LINK_LIBS
   FIROptimizer
   ${dialect_libs}
+  FortranCommon
+  FortranParser
+  FortranEvaluate
+  FortranSemantics
   MLIRAffineToStandard
   MLIRLLVMIR
   MLIRSCFToStandard
diff --git a/flang/lib/Lower/CharacterExpr.cpp b/flang/lib/Lower/CharacterExpr.cpp
new file mode 100644
index 0000000000000..eadf934019394
--- /dev/null
+++ b/flang/lib/Lower/CharacterExpr.cpp
@@ -0,0 +1,486 @@
+//===-- CharacterExpr.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Lower/CharacterExpr.h"
+#include "flang/Lower/ConvertType.h"
+#include "flang/Lower/DoLoopHelper.h"
+#include "flang/Lower/IntrinsicCall.h"
+
+//===----------------------------------------------------------------------===//
+// CharacterExprHelper implementation
+//===----------------------------------------------------------------------===//
+
+/// Get fir.char<kind> type with the same kind as inside str.
+static fir::CharacterType getCharacterType(mlir::Type type) {
+  if (auto boxType = type.dyn_cast<fir::BoxCharType>())
+    return boxType.getEleTy();
+  if (auto refType = type.dyn_cast<fir::ReferenceType>())
+    type = refType.getEleTy();
+  if (auto seqType = type.dyn_cast<fir::SequenceType>()) {
+    assert(seqType.getShape().size() == 1 && "rank must be 1");
+    type = seqType.getEleTy();
+  }
+  if (auto charType = type.dyn_cast<fir::CharacterType>())
+    return charType;
+  llvm_unreachable("Invalid character value type");
+}
+
+static fir::CharacterType getCharacterType(const fir::CharBoxValue &box) {
+  return getCharacterType(box.getBuffer().getType());
+}
+
+static bool needToMaterialize(const fir::CharBoxValue &box) {
+  return box.getBuffer().getType().isa<fir::SequenceType>() ||
+         box.getBuffer().getType().isa<fir::CharacterType>();
+}
+
+static std::optional<fir::SequenceType::Extent>
+getCompileTimeLength(const fir::CharBoxValue &box) {
+  // FIXME: should this just return box.getLen() ??
+  auto type = box.getBuffer().getType();
+  if (type.isa<fir::CharacterType>())
+    return 1;
+  if (auto refType = type.dyn_cast<fir::ReferenceType>())
+    type = refType.getEleTy();
+  if (auto seqType = type.dyn_cast<fir::SequenceType>()) {
+    auto shape = seqType.getShape();
+    assert(shape.size() == 1 && "only scalar character supported");
+    if (shape[0] != fir::SequenceType::getUnknownExtent())
+      return shape[0];
+  }
+  return {};
+}
+
+fir::CharBoxValue Fortran::lower::CharacterExprHelper::materializeValue(
+    const fir::CharBoxValue &str) {
+  if (!needToMaterialize(str))
+    return str;
+  auto variable = builder.create<fir::AllocaOp>(loc, str.getBuffer().getType());
+  builder.create<fir::StoreOp>(loc, str.getBuffer(), variable);
+  return {variable, str.getLen()};
+}
+
+fir::CharBoxValue
+Fortran::lower::CharacterExprHelper::toDataLengthPair(mlir::Value character) {
+  // TODO: get rid of toDataLengthPair when adding support for arrays
+  auto charBox = toExtendedValue(character).getCharBox();
+  assert(charBox && "Array unsupported in character lowering helper");
+  return *charBox;
+}
+
+fir::ExtendedValue
+Fortran::lower::CharacterExprHelper::toExtendedValue(mlir::Value character,
+                                                     mlir::Value len) {
+  auto lenType = getLengthType();
+  auto type = character.getType();
+  auto base = character;
+  mlir::Value resultLen = len;
+  llvm::SmallVector<mlir::Value, 2> extents;
+
+  if (auto refType = type.dyn_cast<fir::ReferenceType>())
+    type = refType.getEleTy();
+
+  if (auto arrayType = type.dyn_cast<fir::SequenceType>()) {
+    type = arrayType.getEleTy();
+    auto shape = arrayType.getShape();
+    auto cstLen = shape[0];
+    if (!resultLen && cstLen != fir::SequenceType::getUnknownExtent())
+      resultLen = builder.createIntegerConstant(loc, lenType, cstLen);
+    // FIXME: only allow `?` in last dimension ?
+    auto typeExtents =
+        llvm::ArrayRef<fir::SequenceType::Extent>{shape}.drop_front();
+    auto indexType = builder.getIndexType();
+    for (auto extent : typeExtents) {
+      if (extent == fir::SequenceType::getUnknownExtent())
+        break;
+      extents.emplace_back(
+          builder.createIntegerConstant(loc, indexType, extent));
+    }
+    // Last extent might be missing in case of assumed-size. If more extents
+    // could not be deduced from type, that's an error (a fir.box should
+    // have been used in the interface).
+    if (extents.size() + 1 < typeExtents.size())
+      mlir::emitError(loc, "cannot retrieve array extents from type");
+  } else if (type.isa<fir::CharacterType>()) {
+    if (!resultLen)
+      resultLen = builder.createIntegerConstant(loc, lenType, 1);
+  } else if (auto boxCharType = type.dyn_cast<fir::BoxCharType>()) {
+    auto refType = builder.getRefType(boxCharType.getEleTy());
+    auto unboxed =
+        builder.create<fir::UnboxCharOp>(loc, refType, lenType, character);
+    base = unboxed.getResult(0);
+    if (!resultLen)
+      resultLen = unboxed.getResult(1);
+  } else if (type.isa<fir::BoxType>()) {
+    mlir::emitError(loc, "descriptor or derived type not yet handled");
+  } else {
+    llvm_unreachable("Cannot translate mlir::Value to character ExtendedValue");
+  }
+
+  if (!resultLen)
+    mlir::emitError(loc, "no dynamic length found for character");
+  if (!extents.empty())
+    return fir::CharArrayBoxValue{base, resultLen, extents};
+  return fir::CharBoxValue{base, resultLen};
+}
+
+/// Get fir.ref<fir.char<kind>> type.
+mlir::Type Fortran::lower::CharacterExprHelper::getReferenceType(
+    const fir::CharBoxValue &box) const {
+  return builder.getRefType(getCharacterType(box));
+}
+
+mlir::Value
+Fortran::lower::CharacterExprHelper::createEmbox(const fir::CharBoxValue &box) {
+  // BoxChar require a reference.
+  auto str = box;
+  if (needToMaterialize(box))
+    str = materializeValue(box);
+  auto kind = getCharacterType(str).getFKind();
+  auto boxCharType = fir::BoxCharType::get(builder.getContext(), kind);
+  auto refType = getReferenceType(str);
+  // So far, fir.emboxChar fails lowering to llvm when it is given
+  // fir.ref<fir.array<len x fir.char<kind>>> types, so convert to
+  // fir.ref<fir.char<kind>> if needed.
+  auto buff = str.getBuffer();
+  buff = builder.createConvert(loc, refType, buff);
+  // Convert in case the provided length is not of the integer type that must
+  // be used in boxchar.
+  auto lenType = getLengthType();
+  auto len = str.getLen();
+  len = builder.createConvert(loc, lenType, len);
+  return builder.create<fir::EmboxCharOp>(loc, boxCharType, buff, len);
+}
+
+mlir::Value Fortran::lower::CharacterExprHelper::createLoadCharAt(
+    const fir::CharBoxValue &str, mlir::Value index) {
+  // In case this is addressing a length one character scalar simply return
+  // the single character.
+  if (str.getBuffer().getType().isa<fir::CharacterType>())
+    return str.getBuffer();
+  auto addr = builder.create<fir::CoordinateOp>(loc, getReferenceType(str),
+                                                str.getBuffer(), index);
+  return builder.create<fir::LoadOp>(loc, addr);
+}
+
+void Fortran::lower::CharacterExprHelper::createStoreCharAt(
+    const fir::CharBoxValue &str, mlir::Value index, mlir::Value c) {
+  assert(!needToMaterialize(str) && "not in memory");
+  auto addr = builder.create<fir::CoordinateOp>(loc, getReferenceType(str),
+                                                str.getBuffer(), index);
+  builder.create<fir::StoreOp>(loc, c, addr);
+}
+
+void Fortran::lower::CharacterExprHelper::createCopy(
+    const fir::CharBoxValue &dest, const fir::CharBoxValue &src,
+    mlir::Value count) {
+  Fortran::lower::DoLoopHelper{builder, loc}.createLoop(
+      count, [&](Fortran::lower::FirOpBuilder &, mlir::Value index) {
+        auto charVal = createLoadCharAt(src, index);
+        createStoreCharAt(dest, index, charVal);
+      });
+}
+
+void Fortran::lower::CharacterExprHelper::createPadding(
+    const fir::CharBoxValue &str, mlir::Value lower, mlir::Value upper) {
+  auto blank = createBlankConstant(getCharacterType(str));
+  // Always create the loop, if upper < lower, no iteration will be
+  // executed.
+  Fortran::lower::DoLoopHelper{builder, loc}.createLoop(
+      lower, upper, [&](Fortran::lower::FirOpBuilder &, mlir::Value index) {
+        createStoreCharAt(str, index, blank);
+      });
+}
+
+fir::CharBoxValue
+Fortran::lower::CharacterExprHelper::createTemp(mlir::Type type,
+                                                mlir::Value len) {
+  assert(type.isa<fir::CharacterType>() && "expected fir character type");
+  llvm::SmallVector<mlir::Value, 3> sizes{len};
+  auto ref = builder.allocateLocal(loc, type, llvm::StringRef{}, sizes);
+  return {ref, len};
+}
+
+// Simple length one character assignment without loops.
+void Fortran::lower::CharacterExprHelper::createLengthOneAssign(
+    const fir::CharBoxValue &lhs, const fir::CharBoxValue &rhs) {
+  auto addr = lhs.getBuffer();
+  auto val = rhs.getBuffer();
+  // If rhs value resides in memory, load it.
+  if (!needToMaterialize(rhs))
+    val = builder.create<fir::LoadOp>(loc, val);
+  auto valTy = val.getType();
+  // Precondition is rhs is size 1, but it may be wrapped in a fir.array.
+  if (auto seqTy = valTy.dyn_cast<fir::SequenceType>()) {
+    auto zero = builder.createIntegerConstant(loc, builder.getIndexType(), 0);
+    valTy = seqTy.getEleTy();
+    val = builder.create<fir::ExtractValueOp>(loc, valTy, val, zero);
+  }
+  auto addrTy = fir::ReferenceType::get(valTy);
+  addr = builder.createConvert(loc, addrTy, addr);
+  assert(fir::dyn_cast_ptrEleTy(addr.getType()) == val.getType());
+  builder.create<fir::StoreOp>(loc, val, addr);
+}
+
+void Fortran::lower::CharacterExprHelper::createAssign(
+    const fir::CharBoxValue &lhs, const fir::CharBoxValue &rhs) {
+  auto rhsCstLen = getCompileTimeLength(rhs);
+  auto lhsCstLen = getCompileTimeLength(lhs);
+  bool compileTimeSameLength =
+      lhsCstLen && rhsCstLen && *lhsCstLen == *rhsCstLen;
+
+  if (compileTimeSameLength && *lhsCstLen == 1) {
+    createLengthOneAssign(lhs, rhs);
+    return;
+  }
+
+  // Copy the minimum of the lhs and rhs lengths and pad the lhs remainder
+  // if needed.
+  mlir::Value copyCount = lhs.getLen();
+  if (!compileTimeSameLength)
+    copyCount =
+        Fortran::lower::genMin(builder, loc, {lhs.getLen(), rhs.getLen()});
+
+  fir::CharBoxValue safeRhs = rhs;
+  if (needToMaterialize(rhs)) {
+    // TODO: revisit now that character constant handling changed.
+    // Need to materialize the constant to get its elements.
+    // (No equivalent of fir.coordinate_of for array value).
+    safeRhs = materializeValue(rhs);
+  } else {
+    // If rhs is in memory, always assumes rhs might overlap with lhs
+    // in a way that require a temp for the copy. That can be optimize later.
+    // Only create a temp of copyCount size because we do not need more from
+    // rhs.
+    auto temp = createTemp(getCharacterType(rhs), copyCount);
+    createCopy(temp, rhs, copyCount);
+    safeRhs = temp;
+  }
+
+  // Actual copy
+  createCopy(lhs, safeRhs, copyCount);
+
+  // Pad if needed.
+  if (!compileTimeSameLength) {
+    auto one = builder.createIntegerConstant(loc, lhs.getLen().getType(), 1);
+    auto maxPadding = builder.create<mlir::SubIOp>(loc, lhs.getLen(), one);
+    createPadding(lhs, copyCount, maxPadding);
+  }
+}
+
+fir::CharBoxValue Fortran::lower::CharacterExprHelper::createConcatenate(
+    const fir::CharBoxValue &lhs, const fir::CharBoxValue &rhs) {
+  mlir::Value len =
+      builder.create<mlir::AddIOp>(loc, lhs.getLen(), rhs.getLen());
+  auto temp = createTemp(getCharacterType(rhs), len);
+  createCopy(temp, lhs, lhs.getLen());
+  auto one = builder.createIntegerConstant(loc, len.getType(), 1);
+  auto upperBound = builder.create<mlir::SubIOp>(loc, len, one);
+  auto lhsLen =
+      builder.createConvert(loc, builder.getIndexType(), lhs.getLen());
+  Fortran::lower::DoLoopHelper{builder, loc}.createLoop(
+      lhs.getLen(), upperBound, one,
+      [&](Fortran::lower::FirOpBuilder &bldr, mlir::Value index) {
+        auto rhsIndex = bldr.create<mlir::SubIOp>(loc, index, lhsLen);
+        auto charVal = createLoadCharAt(rhs, rhsIndex);
+        createStoreCharAt(temp, index, charVal);
+      });
+  return temp;
+}
+
+fir::CharBoxValue Fortran::lower::CharacterExprHelper::createSubstring(
+    const fir::CharBoxValue &box, llvm::ArrayRef<mlir::Value> bounds) {
+  // Constant need to be materialize in memory to use fir.coordinate_of.
+  auto str = box;
+  if (needToMaterialize(box))
+    str = materializeValue(box);
+
+  auto nbounds{bounds.size()};
+  if (nbounds < 1 || nbounds > 2) {
+    mlir::emitError(loc, "Incorrect number of bounds in substring");
+    return {mlir::Value{}, mlir::Value{}};
+  }
+  mlir::SmallVector<mlir::Value, 2> castBounds;
+  // Convert bounds to length type to do safe arithmetic on it.
+  for (auto bound : bounds)
+    castBounds.push_back(builder.createConvert(loc, getLengthType(), bound));
+  auto lowerBound = castBounds[0];
+  // FIR CoordinateOp is zero based but Fortran substring are one based.
+  auto one = builder.createIntegerConstant(loc, lowerBound.getType(), 1);
+  auto offset = builder.create<mlir::SubIOp>(loc, lowerBound, one).getResult();
+  auto idxType = builder.getIndexType();
+  if (offset.getType() != idxType)
+    offset = builder.createConvert(loc, idxType, offset);
+  auto substringRef = builder.create<fir::CoordinateOp>(
+      loc, getReferenceType(str), str.getBuffer(), offset);
+
+  // Compute the length.
+  mlir::Value substringLen{};
+  if (nbounds < 2) {
+    substringLen =
+        builder.create<mlir::SubIOp>(loc, str.getLen(), castBounds[0]);
+  } else {
+    substringLen =
+        builder.create<mlir::SubIOp>(loc, castBounds[1], castBounds[0]);
+  }
+  substringLen = builder.create<mlir::AddIOp>(loc, substringLen, one);
+
+  // Set length to zero if bounds were reversed (Fortran 2018 9.4.1)
+  auto zero = builder.createIntegerConstant(loc, substringLen.getType(), 0);
+  auto cdt = builder.create<mlir::CmpIOp>(loc, mlir::CmpIPredicate::slt,
+                                          substringLen, zero);
+  substringLen = builder.create<mlir::SelectOp>(loc, cdt, zero, substringLen);
+
+  return {substringRef, substringLen};
+}
+
+mlir::Value Fortran::lower::CharacterExprHelper::createLenTrim(
+    const fir::CharBoxValue &str) {
+  // Note: Runtime for LEN_TRIM should also be available at some
+  // point. For now use an inlined implementation.
+  auto indexType = builder.getIndexType();
+  auto len = builder.createConvert(loc, indexType, str.getLen());
+  auto one = builder.createIntegerConstant(loc, indexType, 1);
+  auto minusOne = builder.createIntegerConstant(loc, indexType, -1);
+  auto zero = builder.createIntegerConstant(loc, indexType, 0);
+  auto trueVal = builder.createIntegerConstant(loc, builder.getI1Type(), 1);
+  auto blank = createBlankConstantCode(getCharacterType(str));
+  mlir::Value lastChar = builder.create<mlir::SubIOp>(loc, len, one);
+
+  auto iterWhile = builder.create<fir::IterWhileOp>(
+      loc, lastChar, zero, minusOne, trueVal, lastChar);
+  auto insPt = builder.saveInsertionPoint();
+  builder.setInsertionPointToStart(iterWhile.getBody());
+  auto index = iterWhile.getInductionVar();
+  // Look for first non-blank from the right of the character.
+  auto c = createLoadCharAt(str, index);
+  c = builder.createConvert(loc, blank.getType(), c);
+  auto isBlank =
+      builder.create<mlir::CmpIOp>(loc, mlir::CmpIPredicate::eq, blank, c);
+  llvm::SmallVector<mlir::Value, 2> results = {isBlank, index};
+  builder.create<fir::ResultOp>(loc, results);
+  builder.restoreInsertionPoint(insPt);
+  // Compute length after iteration (zero if all blanks)
+  mlir::Value newLen =
+      builder.create<mlir::AddIOp>(loc, iterWhile.getResult(1), one);
+  auto result =
+      builder.create<SelectOp>(loc, iterWhile.getResult(0), zero, newLen);
+  return builder.createConvert(loc, getLengthType(), result);
+}
+
+mlir::Value Fortran::lower::CharacterExprHelper::createTemp(mlir::Type type,
+                                                            int len) {
+  assert(type.isa<fir::CharacterType>() && "expected fir character type");
+  assert(len >= 0 && "expected positive length");
+  fir::SequenceType::Shape shape{len};
+  auto seqType = fir::SequenceType::get(shape, type);
+  return builder.create<fir::AllocaOp>(loc, seqType);
+}
+
+// Returns integer with code for blank. The integer has the same
+// size as the character. Blank has ascii space code for all kinds.
+mlir::Value Fortran::lower::CharacterExprHelper::createBlankConstantCode(
+    fir::CharacterType type) {
+  auto bits = builder.getKindMap().getCharacterBitsize(type.getFKind());
+  auto intType = builder.getIntegerType(bits);
+  return builder.createIntegerConstant(loc, intType, ' ');
+}
+
+mlir::Value Fortran::lower::CharacterExprHelper::createBlankConstant(
+    fir::CharacterType type) {
+  return builder.createConvert(loc, type, createBlankConstantCode(type));
+}
+
+void Fortran::lower::CharacterExprHelper::createCopy(mlir::Value dest,
+                                                     mlir::Value src,
+                                                     mlir::Value count) {
+  createCopy(toDataLengthPair(dest), toDataLengthPair(src), count);
+}
+
+void Fortran::lower::CharacterExprHelper::createPadding(mlir::Value str,
+                                                        mlir::Value lower,
+                                                        mlir::Value upper) {
+  createPadding(toDataLengthPair(str), lower, upper);
+}
+
+mlir::Value Fortran::lower::CharacterExprHelper::createSubstring(
+    mlir::Value str, llvm::ArrayRef<mlir::Value> bounds) {
+  return createEmbox(createSubstring(toDataLengthPair(str), bounds));
+}
+
+void Fortran::lower::CharacterExprHelper::createAssign(mlir::Value lhs,
+                                                       mlir::Value rhs) {
+  createAssign(toDataLengthPair(lhs), toDataLengthPair(rhs));
+}
+
+mlir::Value
+Fortran::lower::CharacterExprHelper::createLenTrim(mlir::Value str) {
+  return createLenTrim(toDataLengthPair(str));
+}
+
+void Fortran::lower::CharacterExprHelper::createAssign(mlir::Value lptr,
+                                                       mlir::Value llen,
+                                                       mlir::Value rptr,
+                                                       mlir::Value rlen) {
+  createAssign(fir::CharBoxValue{lptr, llen}, fir::CharBoxValue{rptr, rlen});
+}
+
+mlir::Value
+Fortran::lower::CharacterExprHelper::createConcatenate(mlir::Value lhs,
+                                                       mlir::Value rhs) {
+  return createEmbox(
+      createConcatenate(toDataLengthPair(lhs), toDataLengthPair(rhs)));
+}
+
+mlir::Value
+Fortran::lower::CharacterExprHelper::createEmboxChar(mlir::Value addr,
+                                                     mlir::Value len) {
+  return createEmbox(fir::CharBoxValue{addr, len});
+}
+
+std::pair<mlir::Value, mlir::Value>
+Fortran::lower::CharacterExprHelper::createUnboxChar(mlir::Value boxChar) {
+  auto box = toDataLengthPair(boxChar);
+  return {box.getBuffer(), box.getLen()};
+}
+
+mlir::Value
+Fortran::lower::CharacterExprHelper::createCharacterTemp(mlir::Type type,
+                                                         mlir::Value len) {
+  return createEmbox(createTemp(type, len));
+}
+
+std::pair<mlir::Value, mlir::Value>
+Fortran::lower::CharacterExprHelper::materializeCharacter(mlir::Value str) {
+  auto box = toDataLengthPair(str);
+  if (needToMaterialize(box))
+    box = materializeValue(box);
+  return {box.getBuffer(), box.getLen()};
+}
+
+bool Fortran::lower::CharacterExprHelper::isCharacterLiteral(mlir::Type type) {
+  if (auto seqType = type.dyn_cast<fir::SequenceType>())
+    return (seqType.getShape().size() == 1) &&
+           seqType.getEleTy().isa<fir::CharacterType>();
+  return false;
+}
+
+bool Fortran::lower::CharacterExprHelper::isCharacter(mlir::Type type) {
+  if (type.isa<fir::BoxCharType>())
+    return true;
+  if (auto refType = type.dyn_cast<fir::ReferenceType>())
+    type = refType.getEleTy();
+  if (auto seqType = type.dyn_cast<fir::SequenceType>())
+    if (seqType.getShape().size() == 1)
+      type = seqType.getEleTy();
+  return type.isa<fir::CharacterType>();
+}
+
+int Fortran::lower::CharacterExprHelper::getCharacterKind(mlir::Type type) {
+  return getCharacterType(type).getFKind();
+}
diff --git a/flang/lib/Lower/CharacterRuntime.cpp b/flang/lib/Lower/CharacterRuntime.cpp
new file mode 100644
index 0000000000000..af95885f985d2
--- /dev/null
+++ b/flang/lib/Lower/CharacterRuntime.cpp
@@ -0,0 +1,129 @@
+//===-- CharacterRuntime.cpp -- runtime for CHARACTER type entities -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Lower/CharacterRuntime.h"
+#include "../../runtime/character.h"
+#include "RTBuilder.h"
+#include "flang/Lower/Bridge.h"
+#include "flang/Lower/CharacterExpr.h"
+#include "flang/Lower/FIRBuilder.h"
+#include "mlir/Dialect/StandardOps/IR/Ops.h"
+
+using namespace Fortran::runtime;
+
+#define NAMIFY_HELPER(X) #X
+#define NAMIFY(X) NAMIFY_HELPER(IONAME(X))
+#define mkRTKey(X) mkKey(RTNAME(X))
+
+namespace Fortran::lower {
+/// Static table of CHARACTER runtime calls
+///
+/// This logical map contains the name and type builder function for each
+/// runtime function listed in the tuple. This table is fully constructed at
+/// compile-time. Use the `mkRTKey` macro to access the table.
+static constexpr std::tuple<
+    mkRTKey(CharacterCompareScalar), mkRTKey(CharacterCompareScalar1),
+    mkRTKey(CharacterCompareScalar2), mkRTKey(CharacterCompareScalar4),
+    mkRTKey(CharacterCompare)>
+    newCharRTTable;
+} // namespace Fortran::lower
+
+using namespace Fortran::lower;
+
+/// Helper function to retrieve the name of the IO function given the key `A`
+template <typename A>
+static constexpr const char *getName() {
+  return std::get<A>(newCharRTTable).name;
+}
+
+/// Helper function to retrieve the type model signature builder of the IO
+/// function as defined by the key `A`
+template <typename A>
+static constexpr FuncTypeBuilderFunc getTypeModel() {
+  return std::get<A>(newCharRTTable).getTypeModel();
+}
+
+inline int64_t getLength(mlir::Type argTy) {
+  return argTy.cast<fir::SequenceType>().getShape()[0];
+}
+
+/// Get (or generate) the MLIR FuncOp for a given runtime function.
+template <typename E>
+static mlir::FuncOp getRuntimeFunc(mlir::Location loc,
+                                   Fortran::lower::FirOpBuilder &builder) {
+  auto name = getName<E>();
+  auto func = builder.getNamedFunction(name);
+  if (func)
+    return func;
+  auto funTy = getTypeModel<E>()(builder.getContext());
+  func = builder.createFunction(loc, name, funTy);
+  func.setAttr("fir.runtime", builder.getUnitAttr());
+  return func;
+}
+
+/// Helper function to recover the KIND from the FIR type.
+static int discoverKind(mlir::Type ty) {
+  if (auto charTy = ty.dyn_cast<fir::CharacterType>())
+    return charTy.getFKind();
+  if (auto eleTy = fir::dyn_cast_ptrEleTy(ty))
+    return discoverKind(eleTy);
+  if (auto arrTy = ty.dyn_cast<fir::SequenceType>())
+    return discoverKind(arrTy.getEleTy());
+  if (auto boxTy = ty.dyn_cast<fir::BoxCharType>())
+    return discoverKind(boxTy.getEleTy());
+  if (auto boxTy = ty.dyn_cast<fir::BoxType>())
+    return discoverKind(boxTy.getEleTy());
+  llvm_unreachable("unexpected character type");
+}
+
+//===----------------------------------------------------------------------===//
+// Lower character operations
+//===----------------------------------------------------------------------===//
+
+mlir::Value
+Fortran::lower::genRawCharCompare(Fortran::lower::AbstractConverter &converter,
+                                  mlir::Location loc, mlir::CmpIPredicate cmp,
+                                  mlir::Value lhsBuff, mlir::Value lhsLen,
+                                  mlir::Value rhsBuff, mlir::Value rhsLen) {
+  auto &builder = converter.getFirOpBuilder();
+  mlir::FuncOp beginFunc;
+  switch (discoverKind(lhsBuff.getType())) {
+  case 1:
+    beginFunc = getRuntimeFunc<mkRTKey(CharacterCompareScalar1)>(loc, builder);
+    break;
+  case 2:
+    beginFunc = getRuntimeFunc<mkRTKey(CharacterCompareScalar2)>(loc, builder);
+    break;
+  case 4:
+    beginFunc = getRuntimeFunc<mkRTKey(CharacterCompareScalar4)>(loc, builder);
+    break;
+  default:
+    llvm_unreachable("runtime does not support CHARACTER KIND");
+  }
+  auto fTy = beginFunc.getType();
+  auto lptr = builder.createConvert(loc, fTy.getInput(0), lhsBuff);
+  auto llen = builder.createConvert(loc, fTy.getInput(2), lhsLen);
+  auto rptr = builder.createConvert(loc, fTy.getInput(1), rhsBuff);
+  auto rlen = builder.createConvert(loc, fTy.getInput(3), rhsLen);
+  llvm::SmallVector<mlir::Value, 4> args = {lptr, rptr, llen, rlen};
+  auto tri = builder.create<mlir::CallOp>(loc, beginFunc, args).getResult(0);
+  auto zero = builder.createIntegerConstant(loc, tri.getType(), 0);
+  return builder.create<mlir::CmpIOp>(loc, cmp, tri, zero);
+}
+
+mlir::Value
+Fortran::lower::genBoxCharCompare(Fortran::lower::AbstractConverter &converter,
+                                  mlir::Location loc, mlir::CmpIPredicate cmp,
+                                  mlir::Value lhs, mlir::Value rhs) {
+  auto &builder = converter.getFirOpBuilder();
+  Fortran::lower::CharacterExprHelper helper{builder, loc};
+  auto lhsPair = helper.materializeCharacter(lhs);
+  auto rhsPair = helper.materializeCharacter(rhs);
+  return genRawCharCompare(converter, loc, cmp, lhsPair.first, lhsPair.second,
+                           rhsPair.first, rhsPair.second);
+}
diff --git a/flang/lib/Lower/Coarray.cpp b/flang/lib/Lower/Coarray.cpp
new file mode 100644
index 0000000000000..d73acbe17ce20
--- /dev/null
+++ b/flang/lib/Lower/Coarray.cpp
@@ -0,0 +1,73 @@
+//===-- Coarray.cpp -------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// Implementation of the lowering of image related constructs and expressions.
+/// Fortran images can form teams, communicate via coarrays, etc.
+///
+//===----------------------------------------------------------------------===//
+
+#include "flang/Lower/Coarray.h"
+#include "SymbolMap.h"
+#include "flang/Lower/AbstractConverter.h"
+#include "flang/Lower/FIRBuilder.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/expression.h"
+
+#undef TODO
+#define TODO(MSG)                                                              \
+  {                                                                            \
+    mlir::emitError(converter.getCurrentLocation(), "not yet implemented")     \
+        << MSG;                                                                \
+    exit(1);                                                                   \
+  }
+
+//===----------------------------------------------------------------------===//
+// TEAM statements and constructs
+//===----------------------------------------------------------------------===//
+
+void Fortran::lower::genChangeTeamConstruct(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::lower::pft::Evaluation &,
+    const Fortran::parser::ChangeTeamConstruct &) {
+  TODO("CHANGE TEAM construct");
+}
+
+void Fortran::lower::genChangeTeamStmt(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::lower::pft::Evaluation &,
+    const Fortran::parser::ChangeTeamStmt &) {
+  TODO("CHANGE TEAM stmt");
+}
+
+void Fortran::lower::genEndChangeTeamStmt(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::lower::pft::Evaluation &,
+    const Fortran::parser::EndChangeTeamStmt &) {
+  TODO("END CHANGE TEAM");
+}
+
+void Fortran::lower::genFormTeamStatement(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::lower::pft::Evaluation &, const Fortran::parser::FormTeamStmt &) {
+  TODO("FORM TEAM");
+}
+
+//===----------------------------------------------------------------------===//
+// COARRAY expressions
+//===----------------------------------------------------------------------===//
+
+fir::ExtendedValue Fortran::lower::CoarrayExprHelper::genAddr(
+    const Fortran::evaluate::CoarrayRef &expr) {
+  (void)symMap;
+  TODO("co-array address");
+}
+
+fir::ExtendedValue Fortran::lower::CoarrayExprHelper::genValue(
+    const Fortran::evaluate::CoarrayRef &expr) {
+  TODO("co-array value");
+}
diff --git a/flang/lib/Lower/ConvertType.cpp b/flang/lib/Lower/ConvertType.cpp
index 54792a51b4ebc..276df03d6288d 100644
--- a/flang/lib/Lower/ConvertType.cpp
+++ b/flang/lib/Lower/ConvertType.cpp
@@ -177,18 +177,16 @@ namespace {
 /// mlir::Type. The type returned may be an MLIR standard or FIR type.
 class TypeBuilder {
 public:
-  
   /// Constructor.
   explicit TypeBuilder(
       mlir::MLIRContext *context,
       const Fortran::common::IntrinsicTypeDefaultKinds &defaults)
       : context{context}, defaults{defaults} {}
 
-  
   //===--------------------------------------------------------------------===//
   // Generate type entry points
   //===--------------------------------------------------------------------===//
-  
+
   template <template <typename> typename A, Fortran::common::TypeCategory TC>
   mlir::Type gen(const A<Fortran::evaluate::SomeKind<TC>> &) {
     return genFIRType<TC>(context, defaultKind<TC>());
@@ -262,7 +260,7 @@ class TypeBuilder {
   //===--------------------------------------------------------------------===//
   // Generate type helpers
   //===--------------------------------------------------------------------===//
-  
+
   mlir::Type gen(const Fortran::evaluate::ImpliedDoIndex &) {
     return genFIRType<Fortran::common::TypeCategory::Integer>(
         context, defaultKind<Fortran::common::TypeCategory::Integer>());
diff --git a/flang/lib/Lower/FIRBuilder.cpp b/flang/lib/Lower/FIRBuilder.cpp
new file mode 100644
index 0000000000000..8e1df8a2e0b4d
--- /dev/null
+++ b/flang/lib/Lower/FIRBuilder.cpp
@@ -0,0 +1,186 @@
+//===-- FIRBuilder.cpp ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Lower/FIRBuilder.h"
+#include "SymbolMap.h"
+#include "flang/Lower/Bridge.h"
+#include "flang/Lower/ComplexExpr.h"
+#include "flang/Lower/ConvertType.h"
+#include "flang/Optimizer/Dialect/FIROpsSupport.h"
+#include "flang/Semantics/symbol.h"
+#include "llvm/Support/ErrorHandling.h"
+
+mlir::FuncOp Fortran::lower::FirOpBuilder::createFunction(
+    mlir::Location loc, mlir::ModuleOp module, llvm::StringRef name,
+    mlir::FunctionType ty) {
+  return fir::createFuncOp(loc, module, name, ty);
+}
+
+mlir::FuncOp
+Fortran::lower::FirOpBuilder::getNamedFunction(mlir::ModuleOp modOp,
+                                               llvm::StringRef name) {
+  return modOp.lookupSymbol<mlir::FuncOp>(name);
+}
+
+fir::GlobalOp
+Fortran::lower::FirOpBuilder::getNamedGlobal(mlir::ModuleOp modOp,
+                                             llvm::StringRef name) {
+  return modOp.lookupSymbol<fir::GlobalOp>(name);
+}
+
+mlir::Type Fortran::lower::FirOpBuilder::getRefType(mlir::Type eleTy) {
+  assert(!eleTy.isa<fir::ReferenceType>());
+  return fir::ReferenceType::get(eleTy);
+}
+
+mlir::Value
+Fortran::lower::FirOpBuilder::createNullConstant(mlir::Location loc) {
+  auto indexType = getIndexType();
+  auto zero = createIntegerConstant(loc, indexType, 0);
+  auto noneRefType = getRefType(getNoneType());
+  return createConvert(loc, noneRefType, zero);
+}
+
+mlir::Value Fortran::lower::FirOpBuilder::createIntegerConstant(
+    mlir::Location loc, mlir::Type ty, std::int64_t cst) {
+  return create<mlir::ConstantOp>(loc, ty, getIntegerAttr(ty, cst));
+}
+
+mlir::Value Fortran::lower::FirOpBuilder::createRealConstant(
+    mlir::Location loc, mlir::Type realType, const llvm::APFloat &val) {
+  return create<mlir::ConstantOp>(loc, realType, getFloatAttr(realType, val));
+}
+
+mlir::Value
+Fortran::lower::FirOpBuilder::createRealZeroConstant(mlir::Location loc,
+                                                     mlir::Type realType) {
+  mlir::Attribute attr;
+  if (auto firType = realType.dyn_cast<fir::RealType>()) {
+    attr = getFloatAttr(
+        realType,
+        llvm::APFloat(kindMap.getFloatSemantics(firType.getFKind()), 0));
+  } else { // mlir::FloatType.
+    attr = getZeroAttr(realType);
+  }
+  return create<mlir::ConstantOp>(loc, realType, attr);
+}
+
+mlir::Value Fortran::lower::FirOpBuilder::allocateLocal(
+    mlir::Location loc, mlir::Type ty, llvm::StringRef nm,
+    llvm::ArrayRef<mlir::Value> shape, bool asTarget) {
+  llvm::SmallVector<mlir::Value, 8> indices;
+  auto idxTy = getIndexType();
+  llvm::for_each(shape, [&](mlir::Value sh) {
+    indices.push_back(createConvert(loc, idxTy, sh));
+  });
+  llvm::SmallVector<mlir::NamedAttribute, 2> attrs;
+  if (asTarget)
+    attrs.emplace_back(mlir::Identifier::get("target", getContext()),
+                       getUnitAttr());
+  return create<fir::AllocaOp>(loc, ty, nm, llvm::None, indices, attrs);
+}
+
+/// Create a temporary variable on the stack. Anonymous temporaries have no
+/// `name` value.
+mlir::Value Fortran::lower::FirOpBuilder::createTemporary(
+    mlir::Location loc, mlir::Type type, llvm::StringRef name,
+    llvm::ArrayRef<mlir::Value> shape) {
+  auto insPt = saveInsertionPoint();
+  if (shape.empty())
+    setInsertionPointToStart(getEntryBlock());
+  else
+    setInsertionPointAfter(shape.back().getDefiningOp());
+  assert(!type.isa<fir::ReferenceType>() && "cannot be a reference");
+  auto ae = create<fir::AllocaOp>(loc, type, name, llvm::None, shape);
+  restoreInsertionPoint(insPt);
+  return ae;
+}
+
+/// Create a global variable in the (read-only) data section. A global variable
+/// must have a unique name to identify and reference it.
+fir::GlobalOp Fortran::lower::FirOpBuilder::createGlobal(
+    mlir::Location loc, mlir::Type type, llvm::StringRef name,
+    mlir::StringAttr linkage, mlir::Attribute value, bool isConst) {
+  auto module = getModule();
+  auto insertPt = saveInsertionPoint();
+  if (auto glob = module.lookupSymbol<fir::GlobalOp>(name))
+    return glob;
+  setInsertionPoint(module.getBody()->getTerminator());
+  auto glob = create<fir::GlobalOp>(loc, name, isConst, type, value, linkage);
+  restoreInsertionPoint(insertPt);
+  return glob;
+}
+
+fir::GlobalOp Fortran::lower::FirOpBuilder::createGlobal(
+    mlir::Location loc, mlir::Type type, llvm::StringRef name, bool isConst,
+    std::function<void(FirOpBuilder &)> bodyBuilder, mlir::StringAttr linkage) {
+  auto module = getModule();
+  auto insertPt = saveInsertionPoint();
+  if (auto glob = module.lookupSymbol<fir::GlobalOp>(name))
+    return glob;
+  setInsertionPoint(module.getBody()->getTerminator());
+  auto glob = create<fir::GlobalOp>(loc, name, isConst, type, mlir::Attribute{},
+                                    linkage);
+  auto &region = glob.getRegion();
+  region.push_back(new mlir::Block);
+  auto &block = glob.getRegion().back();
+  setInsertionPointToStart(&block);
+  bodyBuilder(*this);
+  restoreInsertionPoint(insertPt);
+  return glob;
+}
+
+mlir::Value Fortran::lower::FirOpBuilder::convertWithSemantics(
+    mlir::Location loc, mlir::Type toTy, mlir::Value val) {
+  assert(toTy && "store location must be typed");
+  auto fromTy = val.getType();
+  if (fromTy == toTy)
+    return val;
+  // FIXME: add a fir::is_integer() test
+  ComplexExprHelper helper{*this, loc};
+  if ((fir::isa_real(fromTy) || fromTy.isSignlessInteger()) &&
+      fir::isa_complex(toTy)) {
+    // imaginary part is zero
+    auto eleTy = helper.getComplexPartType(toTy);
+    auto cast = createConvert(loc, eleTy, val);
+    llvm::APFloat zero{
+        kindMap.getFloatSemantics(toTy.cast<fir::CplxType>().getFKind()), 0};
+    auto imag = createRealConstant(loc, eleTy, zero);
+    return helper.createComplex(toTy, cast, imag);
+  }
+  // FIXME: add a fir::is_integer() test
+  if (fir::isa_complex(fromTy) &&
+      (toTy.isSignlessInteger() || fir::isa_real(toTy))) {
+    // drop the imaginary part
+    auto rp = helper.extractComplexPart(val, /*isImagPart=*/false);
+    return createConvert(loc, toTy, rp);
+  }
+  return createConvert(loc, toTy, val);
+}
+
+mlir::Value Fortran::lower::FirOpBuilder::createConvert(mlir::Location loc,
+                                                        mlir::Type toTy,
+                                                        mlir::Value val) {
+  if (val.getType() != toTy)
+    return create<fir::ConvertOp>(loc, toTy, val);
+  return val;
+}
+
+fir::StringLitOp Fortran::lower::FirOpBuilder::createStringLit(
+    mlir::Location loc, mlir::Type eleTy, llvm::StringRef data) {
+  auto strAttr = mlir::StringAttr::get(data, getContext());
+  auto valTag = mlir::Identifier::get(fir::StringLitOp::value(), getContext());
+  mlir::NamedAttribute dataAttr(valTag, strAttr);
+  auto sizeTag = mlir::Identifier::get(fir::StringLitOp::size(), getContext());
+  mlir::NamedAttribute sizeAttr(sizeTag, getI64IntegerAttr(data.size()));
+  llvm::SmallVector<mlir::NamedAttribute, 2> attrs{dataAttr, sizeAttr};
+  auto arrTy =
+      fir::SequenceType::get(fir::SequenceType::Shape(1, data.size()), eleTy);
+  return create<fir::StringLitOp>(loc, llvm::ArrayRef<mlir::Type>{arrTy},
+                                  llvm::None, attrs);
+}
diff --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
new file mode 100644
index 0000000000000..50dc5c80df6ac
--- /dev/null
+++ b/flang/lib/Lower/IO.cpp
@@ -0,0 +1,1493 @@
+//===-- IO.cpp -- I/O statement lowering ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Lower/IO.h"
+#include "../../runtime/io-api.h"
+#include "RTBuilder.h"
+#include "flang/Lower/Bridge.h"
+#include "flang/Lower/CharacterExpr.h"
+#include "flang/Lower/ComplexExpr.h"
+#include "flang/Lower/FIRBuilder.h"
+#include "flang/Lower/PFTBuilder.h"
+#include "flang/Lower/Runtime.h"
+#include "flang/Lower/Utils.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/tools.h"
+#include "mlir/Dialect/StandardOps/IR/Ops.h"
+
+#define TODO() llvm_unreachable("not yet implemented")
+
+using namespace Fortran::runtime::io;
+
+#define NAMIFY_HELPER(X) #X
+#define NAMIFY(X) NAMIFY_HELPER(IONAME(X))
+#define mkIOKey(X) mkKey(IONAME(X))
+
+namespace Fortran::lower {
+/// Static table of IO runtime calls
+///
+/// This logical map contains the name and type builder function for each IO
+/// runtime function listed in the tuple. This table is fully constructed at
+/// compile-time. Use the `mkIOKey` macro to access the table.
+static constexpr std::tuple<
+    mkIOKey(BeginInternalArrayListOutput), mkIOKey(BeginInternalArrayListInput),
+    mkIOKey(BeginInternalArrayFormattedOutput),
+    mkIOKey(BeginInternalArrayFormattedInput), mkIOKey(BeginInternalListOutput),
+    mkIOKey(BeginInternalListInput), mkIOKey(BeginInternalFormattedOutput),
+    mkIOKey(BeginInternalFormattedInput), mkIOKey(BeginInternalNamelistOutput),
+    mkIOKey(BeginInternalNamelistInput), mkIOKey(BeginExternalListOutput),
+    mkIOKey(BeginExternalListInput), mkIOKey(BeginExternalFormattedOutput),
+    mkIOKey(BeginExternalFormattedInput), mkIOKey(BeginUnformattedOutput),
+    mkIOKey(BeginUnformattedInput), mkIOKey(BeginExternalNamelistOutput),
+    mkIOKey(BeginExternalNamelistInput), mkIOKey(BeginAsynchronousOutput),
+    mkIOKey(BeginAsynchronousInput), mkIOKey(BeginWait), mkIOKey(BeginWaitAll),
+    mkIOKey(BeginClose), mkIOKey(BeginFlush), mkIOKey(BeginBackspace),
+    mkIOKey(BeginEndfile), mkIOKey(BeginRewind), mkIOKey(BeginOpenUnit),
+    mkIOKey(BeginOpenNewUnit), mkIOKey(BeginInquireUnit),
+    mkIOKey(BeginInquireFile), mkIOKey(BeginInquireIoLength),
+    mkIOKey(EnableHandlers), mkIOKey(SetAdvance), mkIOKey(SetBlank),
+    mkIOKey(SetDecimal), mkIOKey(SetDelim), mkIOKey(SetPad), mkIOKey(SetPos),
+    mkIOKey(SetRec), mkIOKey(SetRound), mkIOKey(SetSign),
+    mkIOKey(OutputDescriptor), mkIOKey(InputDescriptor),
+    mkIOKey(OutputUnformattedBlock), mkIOKey(InputUnformattedBlock),
+    mkIOKey(OutputInteger64), mkIOKey(InputInteger), mkIOKey(OutputReal32),
+    mkIOKey(InputReal32), mkIOKey(OutputReal64), mkIOKey(InputReal64),
+    mkIOKey(OutputComplex64), mkIOKey(OutputComplex32), mkIOKey(OutputAscii),
+    mkIOKey(InputAscii), mkIOKey(OutputLogical), mkIOKey(InputLogical),
+    mkIOKey(SetAccess), mkIOKey(SetAction), mkIOKey(SetAsynchronous),
+    mkIOKey(SetEncoding), mkIOKey(SetForm), mkIOKey(SetPosition),
+    mkIOKey(SetRecl), mkIOKey(SetStatus), mkIOKey(SetFile), mkIOKey(GetNewUnit),
+    mkIOKey(GetSize), mkIOKey(GetIoLength), mkIOKey(GetIoMsg),
+    mkIOKey(InquireCharacter), mkIOKey(InquireLogical),
+    mkIOKey(InquirePendingId), mkIOKey(InquireInteger64),
+    mkIOKey(EndIoStatement)>
+    newIOTable;
+} // namespace Fortran::lower
+
+namespace {
+struct ConditionSpecifierInfo {
+  const Fortran::semantics::SomeExpr *ioStatExpr{};
+  const Fortran::semantics::SomeExpr *ioMsgExpr{};
+  bool hasErr{};
+  bool hasEnd{};
+  bool hasEor{};
+
+  /// Check for any condition specifier that applies to specifier processing.
+  bool hasErrorConditionSpecifier() const {
+    return ioStatExpr != nullptr || hasErr;
+  }
+  /// Check for any condition specifier that applies to data transfer items
+  /// in a PRINT, READ, WRITE, or WAIT statement.  (WAIT may be irrelevant.)
+  bool hasTransferConditionSpecifier() const {
+    return ioStatExpr != nullptr || hasErr || hasEnd || hasEor;
+  }
+  /// Check for any condition specifier, including IOMSG.
+  bool hasAnyConditionSpecifier() const {
+    return ioStatExpr != nullptr || ioMsgExpr != nullptr || hasErr || hasEnd ||
+           hasEor;
+  }
+};
+} // namespace
+
+using namespace Fortran::lower;
+
+/// Helper function to retrieve the name of the IO function given the key `A`
+template <typename A>
+static constexpr const char *getName() {
+  return std::get<A>(newIOTable).name;
+}
+
+/// Helper function to retrieve the type model signature builder of the IO
+/// function as defined by the key `A`
+template <typename A>
+static constexpr FuncTypeBuilderFunc getTypeModel() {
+  return std::get<A>(newIOTable).getTypeModel();
+}
+
+inline int64_t getLength(mlir::Type argTy) {
+  return argTy.cast<fir::SequenceType>().getShape()[0];
+}
+
+/// Get (or generate) the MLIR FuncOp for a given IO runtime function.
+template <typename E>
+static mlir::FuncOp getIORuntimeFunc(mlir::Location loc,
+                                     Fortran::lower::FirOpBuilder &builder) {
+  auto name = getName<E>();
+  auto func = builder.getNamedFunction(name);
+  if (func)
+    return func;
+  auto funTy = getTypeModel<E>()(builder.getContext());
+  func = builder.createFunction(loc, name, funTy);
+  func.setAttr("fir.runtime", builder.getUnitAttr());
+  func.setAttr("fir.io", builder.getUnitAttr());
+  return func;
+}
+
+/// Generate calls to end an IO statement.  Return the IOSTAT value, if any.
+/// It is the caller's responsibility to generate branches on that value.
+static mlir::Value genEndIO(Fortran::lower::AbstractConverter &converter,
+                            mlir::Location loc, mlir::Value cookie,
+                            const ConditionSpecifierInfo &csi) {
+  auto &builder = converter.getFirOpBuilder();
+  if (csi.ioMsgExpr) {
+    auto getIoMsg = getIORuntimeFunc<mkIOKey(GetIoMsg)>(loc, builder);
+    auto ioMsgVar =
+        Fortran::lower::CharacterExprHelper{builder, loc}.createUnboxChar(
+            converter.genExprAddr(csi.ioMsgExpr, loc));
+    llvm::SmallVector<mlir::Value, 3> args{
+        cookie,
+        builder.createConvert(loc, getIoMsg.getType().getInput(1),
+                              ioMsgVar.first),
+        builder.createConvert(loc, getIoMsg.getType().getInput(2),
+                              ioMsgVar.second)};
+    builder.create<mlir::CallOp>(loc, getIoMsg, args);
+  }
+  auto endIoStatement = getIORuntimeFunc<mkIOKey(EndIoStatement)>(loc, builder);
+  llvm::SmallVector<mlir::Value, 1> endArgs{cookie};
+  auto call = builder.create<mlir::CallOp>(loc, endIoStatement, endArgs);
+  if (csi.ioStatExpr) {
+    auto ioStatVar = converter.genExprAddr(csi.ioStatExpr, loc);
+    auto ioStatResult = builder.createConvert(
+        loc, converter.genType(*csi.ioStatExpr), call.getResult(0));
+    builder.create<fir::StoreOp>(loc, ioStatResult, ioStatVar);
+  }
+  return csi.hasTransferConditionSpecifier() ? call.getResult(0)
+                                             : mlir::Value{};
+}
+
+/// Make the next call in the IO statement conditional on runtime result `ok`.
+/// If a call returns `ok==false`, further suboperation calls for an I/O
+/// statement will be skipped.  This may generate branch heavy, deeply nested
+/// conditionals for I/O statements with a large number of suboperations.
+static void makeNextConditionalOn(Fortran::lower::FirOpBuilder &builder,
+                                  mlir::Location loc,
+                                  mlir::OpBuilder::InsertPoint &insertPt,
+                                  bool checkResult, mlir::Value ok,
+                                  bool inIterWhileLoop = false) {
+  if (!checkResult || !ok)
+    // Either I/O calls do not need to be checked, or the next I/O call is the
+    // first potentially fallable call.
+    return;
+  // A previous I/O call for a statement returned the bool `ok`.  If this call
+  // is in a fir.iterate_while loop, the result must be propagated up to the
+  // loop scope.  That is done in genIoLoop, but it is enabled here.
+  auto whereOp =
+      inIterWhileLoop
+          ? builder.create<fir::WhereOp>(loc, builder.getI1Type(), ok, true)
+          : builder.create<fir::WhereOp>(loc, ok, /*withOtherwise=*/false);
+  if (!insertPt.isSet())
+    insertPt = builder.saveInsertionPoint();
+  builder.setInsertionPointToStart(&whereOp.whereRegion().front());
+}
+
+template <typename D>
+static void genIoLoop(Fortran::lower::AbstractConverter &converter,
+                      mlir::Value cookie, const D &ioImpliedDo,
+                      bool checkResult, mlir::Value &ok, bool inIterWhileLoop);
+
+/// Get the OutputXyz routine to output a value of the given type.
+static mlir::FuncOp getOutputFunc(mlir::Location loc,
+                                  Fortran::lower::FirOpBuilder &builder,
+                                  mlir::Type type) {
+  if (auto ty = type.dyn_cast<mlir::IntegerType>())
+    return ty.getWidth() == 1
+               ? getIORuntimeFunc<mkIOKey(OutputLogical)>(loc, builder)
+               : getIORuntimeFunc<mkIOKey(OutputInteger64)>(loc, builder);
+  if (auto ty = type.dyn_cast<mlir::FloatType>())
+    return ty.getWidth() <= 32
+               ? getIORuntimeFunc<mkIOKey(OutputReal32)>(loc, builder)
+               : getIORuntimeFunc<mkIOKey(OutputReal64)>(loc, builder);
+  if (auto ty = type.dyn_cast<fir::CplxType>())
+    return ty.getFKind() <= 4
+               ? getIORuntimeFunc<mkIOKey(OutputComplex32)>(loc, builder)
+               : getIORuntimeFunc<mkIOKey(OutputComplex64)>(loc, builder);
+  if (type.isa<fir::LogicalType>())
+    return getIORuntimeFunc<mkIOKey(OutputLogical)>(loc, builder);
+  if (type.isa<fir::BoxType>())
+    return getIORuntimeFunc<mkIOKey(OutputDescriptor)>(loc, builder);
+  if (Fortran::lower::CharacterExprHelper::isCharacter(type))
+    return getIORuntimeFunc<mkIOKey(OutputAscii)>(loc, builder);
+  // TODO: handle arrays
+  mlir::emitError(loc, "output for entity type ") << type << " not implemented";
+  return {};
+}
+
+/// Generate a sequence of output data transfer calls.
+static void
+genOutputItemList(Fortran::lower::AbstractConverter &converter,
+                  mlir::Value cookie,
+                  const std::list<Fortran::parser::OutputItem> &items,
+                  mlir::OpBuilder::InsertPoint &insertPt, bool checkResult,
+                  mlir::Value &ok, bool inIterWhileLoop) {
+  auto &builder = converter.getFirOpBuilder();
+  for (auto &item : items) {
+    if (const auto &impliedDo = std::get_if<1>(&item.u)) {
+      genIoLoop(converter, cookie, impliedDo->value(), checkResult, ok,
+                inIterWhileLoop);
+      continue;
+    }
+    auto &pExpr = std::get<Fortran::parser::Expr>(item.u);
+    auto loc = converter.genLocation(pExpr.source);
+    makeNextConditionalOn(builder, loc, insertPt, checkResult, ok,
+                          inIterWhileLoop);
+    auto itemValue =
+        converter.genExprValue(Fortran::semantics::GetExpr(pExpr), loc);
+    auto itemType = itemValue.getType();
+    auto outputFunc = getOutputFunc(loc, builder, itemType);
+    auto argType = outputFunc.getType().getInput(1);
+    llvm::SmallVector<mlir::Value, 3> outputFuncArgs = {cookie};
+    Fortran::lower::CharacterExprHelper helper{builder, loc};
+    if (helper.isCharacter(itemType)) {
+      auto dataLen = helper.materializeCharacter(itemValue);
+      outputFuncArgs.push_back(builder.createConvert(
+          loc, outputFunc.getType().getInput(1), dataLen.first));
+      outputFuncArgs.push_back(builder.createConvert(
+          loc, outputFunc.getType().getInput(2), dataLen.second));
+    } else if (fir::isa_complex(itemType)) {
+      auto parts = Fortran::lower::ComplexExprHelper{builder, loc}.extractParts(
+          itemValue);
+      outputFuncArgs.push_back(parts.first);
+      outputFuncArgs.push_back(parts.second);
+    } else {
+      itemValue = builder.createConvert(loc, argType, itemValue);
+      outputFuncArgs.push_back(itemValue);
+    }
+    ok = builder.create<mlir::CallOp>(loc, outputFunc, outputFuncArgs)
+             .getResult(0);
+  }
+}
+
+/// Get the InputXyz routine to input a value of the given type.
+static mlir::FuncOp getInputFunc(mlir::Location loc,
+                                 Fortran::lower::FirOpBuilder &builder,
+                                 mlir::Type type) {
+  if (auto ty = type.dyn_cast<mlir::IntegerType>())
+    return ty.getWidth() == 1
+               ? getIORuntimeFunc<mkIOKey(InputLogical)>(loc, builder)
+               : getIORuntimeFunc<mkIOKey(InputInteger)>(loc, builder);
+  if (auto ty = type.dyn_cast<mlir::FloatType>())
+    return ty.getWidth() <= 32
+               ? getIORuntimeFunc<mkIOKey(InputReal32)>(loc, builder)
+               : getIORuntimeFunc<mkIOKey(InputReal64)>(loc, builder);
+  if (auto ty = type.dyn_cast<fir::CplxType>())
+    return ty.getFKind() <= 4
+               ? getIORuntimeFunc<mkIOKey(InputReal32)>(loc, builder)
+               : getIORuntimeFunc<mkIOKey(InputReal64)>(loc, builder);
+  if (type.isa<fir::LogicalType>())
+    return getIORuntimeFunc<mkIOKey(InputLogical)>(loc, builder);
+  if (type.isa<fir::BoxType>())
+    return getIORuntimeFunc<mkIOKey(InputDescriptor)>(loc, builder);
+  if (Fortran::lower::CharacterExprHelper::isCharacter(type))
+    return getIORuntimeFunc<mkIOKey(InputAscii)>(loc, builder);
+  // TODO: handle arrays
+  mlir::emitError(loc, "input for entity type ") << type << " not implemented";
+  return {};
+}
+
+/// Generate a sequence of input data transfer calls.
+static void genInputItemList(Fortran::lower::AbstractConverter &converter,
+                             mlir::Value cookie,
+                             const std::list<Fortran::parser::InputItem> &items,
+                             mlir::OpBuilder::InsertPoint &insertPt,
+                             bool checkResult, mlir::Value &ok,
+                             bool inIterWhileLoop) {
+  auto &builder = converter.getFirOpBuilder();
+  for (auto &item : items) {
+    if (const auto &impliedDo = std::get_if<1>(&item.u)) {
+      genIoLoop(converter, cookie, impliedDo->value(), checkResult, ok,
+                inIterWhileLoop);
+      continue;
+    }
+    auto &pVar = std::get<Fortran::parser::Variable>(item.u);
+    auto loc = converter.genLocation(pVar.GetSource());
+    makeNextConditionalOn(builder, loc, insertPt, checkResult, ok,
+                          inIterWhileLoop);
+    auto itemAddr =
+        converter.genExprAddr(Fortran::semantics::GetExpr(pVar), loc);
+    auto itemType = itemAddr.getType().cast<fir::ReferenceType>().getEleTy();
+    auto inputFunc = getInputFunc(loc, builder, itemType);
+    auto argType = inputFunc.getType().getInput(1);
+    auto originalItemAddr = itemAddr;
+    mlir::Type complexPartType;
+    if (itemType.isa<fir::CplxType>())
+      complexPartType = builder.getRefType(
+          Fortran::lower::ComplexExprHelper{builder, loc}.getComplexPartType(
+              itemType));
+    auto complexPartAddr = [&](int index) {
+      return builder.create<fir::CoordinateOp>(
+          loc, complexPartType, originalItemAddr,
+          llvm::SmallVector<mlir::Value, 1>{builder.create<mlir::ConstantOp>(
+              loc, builder.getI32IntegerAttr(index))});
+    };
+    if (complexPartType)
+      itemAddr = complexPartAddr(0); // real part
+    itemAddr = builder.createConvert(loc, argType, itemAddr);
+    llvm::SmallVector<mlir::Value, 3> inputFuncArgs = {cookie, itemAddr};
+    Fortran::lower::CharacterExprHelper helper{builder, loc};
+    if (helper.isCharacter(itemType)) {
+      auto len = helper.materializeCharacter(originalItemAddr).second;
+      inputFuncArgs.push_back(
+          builder.createConvert(loc, inputFunc.getType().getInput(2), len));
+    } else if (itemType.isa<mlir::IntegerType>()) {
+      inputFuncArgs.push_back(builder.create<mlir::ConstantOp>(
+          loc, builder.getI32IntegerAttr(
+                   itemType.cast<mlir::IntegerType>().getWidth() / 8)));
+    }
+    ok = builder.create<mlir::CallOp>(loc, inputFunc, inputFuncArgs)
+             .getResult(0);
+    if (complexPartType) { // imaginary part
+      makeNextConditionalOn(builder, loc, insertPt, checkResult, ok,
+                            inIterWhileLoop);
+      inputFuncArgs = {cookie,
+                       builder.createConvert(loc, argType, complexPartAddr(1))};
+      ok = builder.create<mlir::CallOp>(loc, inputFunc, inputFuncArgs)
+               .getResult(0);
+    }
+  }
+}
+
+/// Generate an io-implied-do loop.
+template <typename D>
+static void genIoLoop(Fortran::lower::AbstractConverter &converter,
+                      mlir::Value cookie, const D &ioImpliedDo,
+                      bool checkResult, mlir::Value &ok, bool inIterWhileLoop) {
+  mlir::OpBuilder::InsertPoint insertPt;
+  auto &builder = converter.getFirOpBuilder();
+  auto loc = converter.getCurrentLocation();
+  makeNextConditionalOn(builder, loc, insertPt, checkResult, ok,
+                        inIterWhileLoop);
+  auto parentInsertPt = builder.saveInsertionPoint();
+  const auto &itemList = std::get<0>(ioImpliedDo.t);
+  const auto &control = std::get<1>(ioImpliedDo.t);
+  const auto &loopSym = *control.name.thing.thing.symbol;
+  auto loopVar = converter.getSymbolAddress(loopSym);
+  auto genFIRLoopIndex = [&](const Fortran::parser::ScalarIntExpr &expr) {
+    return builder.createConvert(
+        loc, builder.getIndexType(),
+        converter.genExprValue(*Fortran::semantics::GetExpr(expr)));
+  };
+  auto lowerValue = genFIRLoopIndex(control.lower);
+  auto upperValue = genFIRLoopIndex(control.upper);
+  auto stepValue = control.step.has_value()
+                       ? genFIRLoopIndex(*control.step)
+                       : builder.create<mlir::ConstantIndexOp>(loc, 1);
+  auto genItemList = [&](const D &ioImpliedDo, bool inIterWhileLoop) {
+    if constexpr (std::is_same_v<D, Fortran::parser::InputImpliedDo>)
+      genInputItemList(converter, cookie, itemList, insertPt, checkResult, ok,
+                       true);
+    else
+      genOutputItemList(converter, cookie, itemList, insertPt, checkResult, ok,
+                        true);
+  };
+  if (!checkResult) {
+    // No I/O call result checks - the loop is a fir.do_loop op.
+    auto loopOp =
+        builder.create<fir::LoopOp>(loc, lowerValue, upperValue, stepValue);
+    builder.setInsertionPointToStart(loopOp.getBody());
+    auto lcv = builder.createConvert(loc, converter.genType(loopSym),
+                                     loopOp.getInductionVar());
+    builder.create<fir::StoreOp>(loc, lcv, loopVar);
+    insertPt = builder.saveInsertionPoint();
+    genItemList(ioImpliedDo, false);
+    builder.restoreInsertionPoint(parentInsertPt);
+    return;
+  }
+  // Check I/O call results - the loop is a fir.iterate_while op.
+  if (!ok)
+    ok = builder.createIntegerConstant(loc, builder.getI1Type(), 1);
+  fir::IterWhileOp iterWhileOp = builder.create<fir::IterWhileOp>(
+      loc, lowerValue, upperValue, stepValue, ok);
+  builder.setInsertionPointToStart(iterWhileOp.getBody());
+  auto lcv = builder.createConvert(loc, converter.genType(loopSym),
+                                   iterWhileOp.getInductionVar());
+  builder.create<fir::StoreOp>(loc, lcv, loopVar);
+  insertPt = builder.saveInsertionPoint();
+  ok = iterWhileOp.getIterateVar();
+  auto falseValue = builder.createIntegerConstant(loc, builder.getI1Type(), 0);
+  genItemList(ioImpliedDo, true);
+  // Unwind nested I/O call scopes, filling in true and false ResultOp's.
+  for (auto *op = builder.getBlock()->getParentOp(); isa<fir::WhereOp>(op);
+       op = op->getBlock()->getParentOp()) {
+    auto whereOp = dyn_cast<fir::WhereOp>(op);
+    auto *lastOp = &whereOp.whereRegion().front().back();
+    builder.setInsertionPointAfter(lastOp);
+    builder.create<fir::ResultOp>(loc, lastOp->getResult(0)); // runtime result
+    builder.setInsertionPointToStart(&whereOp.otherRegion().front());
+    builder.create<fir::ResultOp>(loc, falseValue); // known false result
+  }
+  builder.restoreInsertionPoint(insertPt);
+  builder.create<fir::ResultOp>(loc, builder.getBlock()->back().getResult(0));
+  ok = iterWhileOp.getResult(0);
+  builder.restoreInsertionPoint(parentInsertPt);
+}
+
+//===----------------------------------------------------------------------===//
+// Default argument generation.
+//===----------------------------------------------------------------------===//
+
+static mlir::Value getDefaultFilename(Fortran::lower::FirOpBuilder &builder,
+                                      mlir::Location loc, mlir::Type toType) {
+  mlir::Value null =
+      builder.create<mlir::ConstantOp>(loc, builder.getI64IntegerAttr(0));
+  return builder.createConvert(loc, toType, null);
+}
+
+static mlir::Value getDefaultLineNo(Fortran::lower::FirOpBuilder &builder,
+                                    mlir::Location loc, mlir::Type toType) {
+  return builder.create<mlir::ConstantOp>(loc,
+                                          builder.getIntegerAttr(toType, 0));
+}
+
+static mlir::Value getDefaultScratch(Fortran::lower::FirOpBuilder &builder,
+                                     mlir::Location loc, mlir::Type toType) {
+  mlir::Value null =
+      builder.create<mlir::ConstantOp>(loc, builder.getI64IntegerAttr(0));
+  return builder.createConvert(loc, toType, null);
+}
+
+static mlir::Value getDefaultScratchLen(Fortran::lower::FirOpBuilder &builder,
+                                        mlir::Location loc, mlir::Type toType) {
+  return builder.create<mlir::ConstantOp>(loc,
+                                          builder.getIntegerAttr(toType, 0));
+}
+
+/// Lower a string literal. Many arguments to the runtime are conveyed as
+/// Fortran CHARACTER literals.
+template <typename A>
+static std::tuple<mlir::Value, mlir::Value, mlir::Value>
+lowerStringLit(Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+               const A &syntax, mlir::Type strTy, mlir::Type lenTy,
+               mlir::Type ty2 = {}) {
+  auto &builder = converter.getFirOpBuilder();
+  auto *expr = Fortran::semantics::GetExpr(syntax);
+  auto str = converter.genExprValue(expr, loc);
+  Fortran::lower::CharacterExprHelper helper{builder, loc};
+  auto dataLen = helper.materializeCharacter(str);
+  auto buff = builder.createConvert(loc, strTy, dataLen.first);
+  auto len = builder.createConvert(loc, lenTy, dataLen.second);
+  if (ty2) {
+    auto kindVal = helper.getCharacterKind(str.getType());
+    auto kind = builder.create<mlir::ConstantOp>(
+        loc, builder.getIntegerAttr(ty2, kindVal));
+    return {buff, len, kind};
+  }
+  return {buff, len, mlir::Value{}};
+}
+
+/// Pass the body of the FORMAT statement in as if it were a CHARACTER literal
+/// constant. NB: This is the prescribed manner in which the front-end passes
+/// this information to lowering.
+static std::tuple<mlir::Value, mlir::Value, mlir::Value>
+lowerSourceTextAsStringLit(Fortran::lower::AbstractConverter &converter,
+                           mlir::Location loc, llvm::StringRef text,
+                           mlir::Type strTy, mlir::Type lenTy) {
+  text = text.drop_front(text.find('('));
+  text = text.take_front(text.rfind(')') + 1);
+  auto &builder = converter.getFirOpBuilder();
+  auto lit = builder.createStringLit(
+      loc, /*FIXME*/ fir::CharacterType::get(builder.getContext(), 1), text);
+  auto data =
+      Fortran::lower::CharacterExprHelper{builder, loc}.materializeCharacter(
+          lit);
+  auto buff = builder.createConvert(loc, strTy, data.first);
+  auto len = builder.createConvert(loc, lenTy, data.second);
+  return {buff, len, mlir::Value{}};
+}
+
+//===----------------------------------------------------------------------===//
+// Handle I/O statement specifiers.
+// These are threaded together for a single statement via the passed cookie.
+//===----------------------------------------------------------------------===//
+
+/// Generic to build an integral argument to the runtime.
+template <typename A, typename B>
+mlir::Value genIntIOOption(Fortran::lower::AbstractConverter &converter,
+                           mlir::Location loc, mlir::Value cookie,
+                           const B &spec) {
+  auto &builder = converter.getFirOpBuilder();
+  mlir::FuncOp ioFunc = getIORuntimeFunc<A>(loc, builder);
+  mlir::FunctionType ioFuncTy = ioFunc.getType();
+  auto expr = converter.genExprValue(Fortran::semantics::GetExpr(spec.v), loc);
+  auto val = builder.createConvert(loc, ioFuncTy.getInput(1), expr);
+  llvm::SmallVector<mlir::Value, 4> ioArgs = {cookie, val};
+  return builder.create<mlir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+}
+
+/// Generic to build a string argument to the runtime. This passes a CHARACTER
+/// as a pointer to the buffer and a LEN parameter.
+template <typename A, typename B>
+mlir::Value genCharIOOption(Fortran::lower::AbstractConverter &converter,
+                            mlir::Location loc, mlir::Value cookie,
+                            const B &spec) {
+  auto &builder = converter.getFirOpBuilder();
+  mlir::FuncOp ioFunc = getIORuntimeFunc<A>(loc, builder);
+  mlir::FunctionType ioFuncTy = ioFunc.getType();
+  auto tup = lowerStringLit(converter, loc, spec, ioFuncTy.getInput(1),
+                            ioFuncTy.getInput(2));
+  llvm::SmallVector<mlir::Value, 4> ioArgs = {cookie, std::get<0>(tup),
+                                              std::get<1>(tup)};
+  return builder.create<mlir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+}
+
+template <typename A>
+mlir::Value genIOOption(Fortran::lower::AbstractConverter &converter,
+                        mlir::Location loc, mlir::Value cookie, const A &spec) {
+  // default case: do nothing
+  return {};
+}
+
+template <>
+mlir::Value genIOOption<Fortran::parser::FileNameExpr>(
+    Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+    mlir::Value cookie, const Fortran::parser::FileNameExpr &spec) {
+  auto &builder = converter.getFirOpBuilder();
+  // has an extra KIND argument
+  auto ioFunc = getIORuntimeFunc<mkIOKey(SetFile)>(loc, builder);
+  mlir::FunctionType ioFuncTy = ioFunc.getType();
+  auto tup = lowerStringLit(converter, loc, spec, ioFuncTy.getInput(1),
+                            ioFuncTy.getInput(2), ioFuncTy.getInput(3));
+  llvm::SmallVector<mlir::Value, 4> ioArgs{cookie, std::get<0>(tup),
+                                           std::get<1>(tup), std::get<2>(tup)};
+  return builder.create<mlir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+}
+
+template <>
+mlir::Value genIOOption<Fortran::parser::ConnectSpec::CharExpr>(
+    Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+    mlir::Value cookie, const Fortran::parser::ConnectSpec::CharExpr &spec) {
+  auto &builder = converter.getFirOpBuilder();
+  mlir::FuncOp ioFunc;
+  switch (std::get<Fortran::parser::ConnectSpec::CharExpr::Kind>(spec.t)) {
+  case Fortran::parser::ConnectSpec::CharExpr::Kind::Access:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetAccess)>(loc, builder);
+    break;
+  case Fortran::parser::ConnectSpec::CharExpr::Kind::Action:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetAction)>(loc, builder);
+    break;
+  case Fortran::parser::ConnectSpec::CharExpr::Kind::Asynchronous:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetAsynchronous)>(loc, builder);
+    break;
+  case Fortran::parser::ConnectSpec::CharExpr::Kind::Blank:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetBlank)>(loc, builder);
+    break;
+  case Fortran::parser::ConnectSpec::CharExpr::Kind::Decimal:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetDecimal)>(loc, builder);
+    break;
+  case Fortran::parser::ConnectSpec::CharExpr::Kind::Delim:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetDelim)>(loc, builder);
+    break;
+  case Fortran::parser::ConnectSpec::CharExpr::Kind::Encoding:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetEncoding)>(loc, builder);
+    break;
+  case Fortran::parser::ConnectSpec::CharExpr::Kind::Form:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetForm)>(loc, builder);
+    break;
+  case Fortran::parser::ConnectSpec::CharExpr::Kind::Pad:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetPad)>(loc, builder);
+    break;
+  case Fortran::parser::ConnectSpec::CharExpr::Kind::Position:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetPosition)>(loc, builder);
+    break;
+  case Fortran::parser::ConnectSpec::CharExpr::Kind::Round:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetRound)>(loc, builder);
+    break;
+  case Fortran::parser::ConnectSpec::CharExpr::Kind::Sign:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetSign)>(loc, builder);
+    break;
+  case Fortran::parser::ConnectSpec::CharExpr::Kind::Convert:
+    llvm_unreachable("CONVERT not part of the runtime::io interface");
+  case Fortran::parser::ConnectSpec::CharExpr::Kind::Dispose:
+    llvm_unreachable("DISPOSE not part of the runtime::io interface");
+  }
+  mlir::FunctionType ioFuncTy = ioFunc.getType();
+  auto tup = lowerStringLit(
+      converter, loc, std::get<Fortran::parser::ScalarDefaultCharExpr>(spec.t),
+      ioFuncTy.getInput(1), ioFuncTy.getInput(2));
+  llvm::SmallVector<mlir::Value, 4> ioArgs = {cookie, std::get<0>(tup),
+                                              std::get<1>(tup)};
+  return builder.create<mlir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+}
+
+template <>
+mlir::Value genIOOption<Fortran::parser::ConnectSpec::Recl>(
+    Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+    mlir::Value cookie, const Fortran::parser::ConnectSpec::Recl &spec) {
+  return genIntIOOption<mkIOKey(SetRecl)>(converter, loc, cookie, spec);
+}
+
+template <>
+mlir::Value genIOOption<Fortran::parser::StatusExpr>(
+    Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+    mlir::Value cookie, const Fortran::parser::StatusExpr &spec) {
+  return genCharIOOption<mkIOKey(SetStatus)>(converter, loc, cookie, spec.v);
+}
+
+template <>
+mlir::Value
+genIOOption<Fortran::parser::Name>(Fortran::lower::AbstractConverter &converter,
+                                   mlir::Location loc, mlir::Value cookie,
+                                   const Fortran::parser::Name &spec) {
+  // namelist
+  llvm_unreachable("not implemented");
+}
+
+template <>
+mlir::Value genIOOption<Fortran::parser::IoControlSpec::CharExpr>(
+    Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+    mlir::Value cookie, const Fortran::parser::IoControlSpec::CharExpr &spec) {
+  auto &builder = converter.getFirOpBuilder();
+  mlir::FuncOp ioFunc;
+  switch (std::get<Fortran::parser::IoControlSpec::CharExpr::Kind>(spec.t)) {
+  case Fortran::parser::IoControlSpec::CharExpr::Kind::Advance:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetAdvance)>(loc, builder);
+    break;
+  case Fortran::parser::IoControlSpec::CharExpr::Kind::Blank:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetBlank)>(loc, builder);
+    break;
+  case Fortran::parser::IoControlSpec::CharExpr::Kind::Decimal:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetDecimal)>(loc, builder);
+    break;
+  case Fortran::parser::IoControlSpec::CharExpr::Kind::Delim:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetDelim)>(loc, builder);
+    break;
+  case Fortran::parser::IoControlSpec::CharExpr::Kind::Pad:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetPad)>(loc, builder);
+    break;
+  case Fortran::parser::IoControlSpec::CharExpr::Kind::Round:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetRound)>(loc, builder);
+    break;
+  case Fortran::parser::IoControlSpec::CharExpr::Kind::Sign:
+    ioFunc = getIORuntimeFunc<mkIOKey(SetSign)>(loc, builder);
+    break;
+  }
+  mlir::FunctionType ioFuncTy = ioFunc.getType();
+  auto tup = lowerStringLit(
+      converter, loc, std::get<Fortran::parser::ScalarDefaultCharExpr>(spec.t),
+      ioFuncTy.getInput(1), ioFuncTy.getInput(2));
+  llvm::SmallVector<mlir::Value, 4> ioArgs = {cookie, std::get<0>(tup),
+                                              std::get<1>(tup)};
+  return builder.create<mlir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+}
+
+template <>
+mlir::Value genIOOption<Fortran::parser::IoControlSpec::Asynchronous>(
+    Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+    mlir::Value cookie,
+    const Fortran::parser::IoControlSpec::Asynchronous &spec) {
+  return genCharIOOption<mkIOKey(SetAsynchronous)>(converter, loc, cookie,
+                                                   spec.v);
+}
+
+template <>
+mlir::Value genIOOption<Fortran::parser::IdVariable>(
+    Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+    mlir::Value cookie, const Fortran::parser::IdVariable &spec) {
+  llvm_unreachable("asynchronous ID not implemented");
+}
+
+template <>
+mlir::Value genIOOption<Fortran::parser::IoControlSpec::Pos>(
+    Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+    mlir::Value cookie, const Fortran::parser::IoControlSpec::Pos &spec) {
+  return genIntIOOption<mkIOKey(SetPos)>(converter, loc, cookie, spec);
+}
+template <>
+mlir::Value genIOOption<Fortran::parser::IoControlSpec::Rec>(
+    Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+    mlir::Value cookie, const Fortran::parser::IoControlSpec::Rec &spec) {
+  return genIntIOOption<mkIOKey(SetRec)>(converter, loc, cookie, spec);
+}
+
+//===----------------------------------------------------------------------===//
+// Gather I/O statement condition specifier information (if any).
+//===----------------------------------------------------------------------===//
+
+template <typename SEEK, typename A>
+static bool hasX(const A &list) {
+  for (const auto &spec : list)
+    if (std::holds_alternative<SEEK>(spec.u))
+      return true;
+  return false;
+}
+
+template <typename SEEK, typename A>
+static bool hasMem(const A &stmt) {
+  return hasX<SEEK>(stmt.v);
+}
+
+/// Get the sought expression from the specifier list.
+template <typename SEEK, typename A>
+static const Fortran::semantics::SomeExpr *getExpr(const A &stmt) {
+  for (const auto &spec : stmt.v)
+    if (auto *f = std::get_if<SEEK>(&spec.u))
+      return Fortran::semantics::GetExpr(f->v);
+  llvm_unreachable("must have a file unit");
+}
+
+/// For each specifier, build the appropriate call, threading the cookie, and
+/// returning the insertion point as to the initial context. If there are no
+/// specifiers, the insertion point is undefined.
+template <typename A>
+static mlir::OpBuilder::InsertPoint
+threadSpecs(Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+            mlir::Value cookie, const A &specList, bool checkResult,
+            mlir::Value &ok) {
+  auto &builder = converter.getFirOpBuilder();
+  mlir::OpBuilder::InsertPoint insertPt;
+  for (const auto &spec : specList) {
+    makeNextConditionalOn(builder, loc, insertPt, checkResult, ok);
+    ok = std::visit(Fortran::common::visitors{[&](const auto &x) {
+                      return genIOOption(converter, loc, cookie, x);
+                    }},
+                    spec.u);
+  }
+  return insertPt;
+}
+
+template <typename A>
+static void
+genConditionHandlerCall(Fortran::lower::AbstractConverter &converter,
+                        mlir::Location loc, mlir::Value cookie,
+                        const A &specList, ConditionSpecifierInfo &csi) {
+  for (const auto &spec : specList) {
+    std::visit(
+        Fortran::common::visitors{
+            [&](const Fortran::parser::StatVariable &msgVar) {
+              csi.ioStatExpr = Fortran::semantics::GetExpr(msgVar);
+            },
+            [&](const Fortran::parser::MsgVariable &msgVar) {
+              csi.ioMsgExpr = Fortran::semantics::GetExpr(msgVar);
+            },
+            [&](const Fortran::parser::EndLabel &) { csi.hasEnd = true; },
+            [&](const Fortran::parser::EorLabel &) { csi.hasEor = true; },
+            [&](const Fortran::parser::ErrLabel &) { csi.hasErr = true; },
+            [](const auto &) {}},
+        spec.u);
+  }
+  if (!csi.hasAnyConditionSpecifier())
+    return;
+  auto &builder = converter.getFirOpBuilder();
+  mlir::FuncOp enableHandlers =
+      getIORuntimeFunc<mkIOKey(EnableHandlers)>(loc, builder);
+  mlir::Type boolType = enableHandlers.getType().getInput(1);
+  auto boolValue = [&](bool specifierIsPresent) {
+    return builder.create<mlir::ConstantOp>(
+        loc, builder.getIntegerAttr(boolType, specifierIsPresent));
+  };
+  llvm::SmallVector<mlir::Value, 6> ioArgs = {
+      cookie,
+      boolValue(csi.ioStatExpr != nullptr),
+      boolValue(csi.hasErr),
+      boolValue(csi.hasEnd),
+      boolValue(csi.hasEor),
+      boolValue(csi.ioMsgExpr != nullptr)};
+  builder.create<mlir::CallOp>(loc, enableHandlers, ioArgs);
+}
+
+//===----------------------------------------------------------------------===//
+// Data transfer helpers
+//===----------------------------------------------------------------------===//
+
+template <typename SEEK, typename A>
+static bool hasIOControl(const A &stmt) {
+  return hasX<SEEK>(stmt.controls);
+}
+
+template <typename SEEK, typename A>
+static const auto *getIOControl(const A &stmt) {
+  for (const auto &spec : stmt.controls)
+    if (const auto *result = std::get_if<SEEK>(&spec.u))
+      return result;
+  return static_cast<const SEEK *>(nullptr);
+}
+
+/// returns true iff the expression in the parse tree is not really a format but
+/// rather a namelist variable.
+template <typename A>
+static bool formatIsActuallyNamelist(const A &format) {
+  if (auto *e = std::get_if<Fortran::parser::Expr>(&format.u)) {
+    auto *expr = Fortran::semantics::GetExpr(*e);
+    if (const Fortran::semantics::Symbol *y =
+            Fortran::evaluate::UnwrapWholeSymbolDataRef(*expr))
+      return y->has<Fortran::semantics::NamelistDetails>();
+  }
+  return false;
+}
+
+template <typename A>
+static bool isDataTransferFormatted(const A &stmt) {
+  if (stmt.format)
+    return !formatIsActuallyNamelist(*stmt.format);
+  return hasIOControl<Fortran::parser::Format>(stmt);
+}
+template <>
+constexpr bool isDataTransferFormatted<Fortran::parser::PrintStmt>(
+    const Fortran::parser::PrintStmt &) {
+  return true; // PRINT is always formatted
+}
+
+template <typename A>
+static bool isDataTransferList(const A &stmt) {
+  if (stmt.format)
+    return std::holds_alternative<Fortran::parser::Star>(stmt.format->u);
+  if (auto *mem = getIOControl<Fortran::parser::Format>(stmt))
+    return std::holds_alternative<Fortran::parser::Star>(mem->u);
+  return false;
+}
+template <>
+bool isDataTransferList<Fortran::parser::PrintStmt>(
+    const Fortran::parser::PrintStmt &stmt) {
+  return std::holds_alternative<Fortran::parser::Star>(
+      std::get<Fortran::parser::Format>(stmt.t).u);
+}
+
+template <typename A>
+static bool isDataTransferInternal(const A &stmt) {
+  if (stmt.iounit.has_value())
+    return std::holds_alternative<Fortran::parser::Variable>(stmt.iounit->u);
+  if (auto *unit = getIOControl<Fortran::parser::IoUnit>(stmt))
+    return std::holds_alternative<Fortran::parser::Variable>(unit->u);
+  return false;
+}
+template <>
+constexpr bool isDataTransferInternal<Fortran::parser::PrintStmt>(
+    const Fortran::parser::PrintStmt &) {
+  return false;
+}
+
+static bool hasNonDefaultCharKind(const Fortran::parser::Variable &var) {
+  // TODO
+  return false;
+}
+
+template <typename A>
+static bool isDataTransferInternalNotDefaultKind(const A &stmt) {
+  // same as isDataTransferInternal, but the KIND of the expression is not the
+  // default KIND.
+  if (stmt.iounit.has_value())
+    if (auto *var = std::get_if<Fortran::parser::Variable>(&stmt.iounit->u))
+      return hasNonDefaultCharKind(*var);
+  if (auto *unit = getIOControl<Fortran::parser::IoUnit>(stmt))
+    if (auto *var = std::get_if<Fortran::parser::Variable>(&unit->u))
+      return hasNonDefaultCharKind(*var);
+  return false;
+}
+template <>
+constexpr bool isDataTransferInternalNotDefaultKind<Fortran::parser::PrintStmt>(
+    const Fortran::parser::PrintStmt &) {
+  return false;
+}
+
+template <typename A>
+static bool isDataTransferAsynchronous(const A &stmt) {
+  if (auto *asynch =
+          getIOControl<Fortran::parser::IoControlSpec::Asynchronous>(stmt)) {
+    // FIXME: should contain a string of YES or NO
+    llvm_unreachable("asynchronous transfers not implemented in runtime");
+  }
+  return false;
+}
+template <>
+constexpr bool isDataTransferAsynchronous<Fortran::parser::PrintStmt>(
+    const Fortran::parser::PrintStmt &) {
+  return false;
+}
+
+template <typename A>
+static bool isDataTransferNamelist(const A &stmt) {
+  if (stmt.format)
+    return formatIsActuallyNamelist(*stmt.format);
+  return hasIOControl<Fortran::parser::Name>(stmt);
+}
+template <>
+constexpr bool isDataTransferNamelist<Fortran::parser::PrintStmt>(
+    const Fortran::parser::PrintStmt &) {
+  return false;
+}
+
+/// Generate a reference to a format string.  There are four cases - a format
+/// statement label, a character format expression, an integer that holds the
+/// label of a format statement, and the * case.  The first three are done here.
+/// The * case is done elsewhere.
+static std::tuple<mlir::Value, mlir::Value, mlir::Value>
+genFormat(Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+          const Fortran::parser::Format &format, mlir::Type strTy,
+          mlir::Type lenTy, Fortran::lower::pft::LabelEvalMap &labelMap,
+          Fortran::lower::pft::SymbolLabelMap &assignMap) {
+  if (const auto *label = std::get_if<Fortran::parser::Label>(&format.u)) {
+    // format statement label
+    auto iter = labelMap.find(*label);
+    assert(iter != labelMap.end() && "FORMAT not found in PROCEDURE");
+    return lowerSourceTextAsStringLit(
+        converter, loc, toStringRef(iter->second->position), strTy, lenTy);
+  }
+  const auto *pExpr = std::get_if<Fortran::parser::Expr>(&format.u);
+  assert(pExpr && "missing format expression");
+  auto e = Fortran::semantics::GetExpr(*pExpr);
+  if (Fortran::semantics::ExprHasTypeCategory(
+          *e, Fortran::common::TypeCategory::Character))
+    // character expression
+    return lowerStringLit(converter, loc, *pExpr, strTy, lenTy);
+  // integer variable containing an ASSIGN label
+  assert(Fortran::semantics::ExprHasTypeCategory(
+      *e, Fortran::common::TypeCategory::Integer));
+  // TODO - implement this
+  llvm::report_fatal_error(
+      "using a variable to reference a FORMAT statement; not implemented yet");
+}
+
+template <typename A>
+std::tuple<mlir::Value, mlir::Value, mlir::Value>
+getFormat(Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+          const A &stmt, mlir::Type strTy, mlir::Type lenTy,
+          Fortran::lower::pft::LabelEvalMap &labelMap,
+          Fortran::lower::pft::SymbolLabelMap &assignMap) {
+  if (stmt.format && !formatIsActuallyNamelist(*stmt.format))
+    return genFormat(converter, loc, *stmt.format, strTy, lenTy, labelMap,
+                     assignMap);
+  return genFormat(converter, loc, *getIOControl<Fortran::parser::Format>(stmt),
+                   strTy, lenTy, labelMap, assignMap);
+}
+template <>
+std::tuple<mlir::Value, mlir::Value, mlir::Value>
+getFormat<Fortran::parser::PrintStmt>(
+    Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+    const Fortran::parser::PrintStmt &stmt, mlir::Type strTy, mlir::Type lenTy,
+    Fortran::lower::pft::LabelEvalMap &labelMap,
+    Fortran::lower::pft::SymbolLabelMap &assignMap) {
+  return genFormat(converter, loc, std::get<Fortran::parser::Format>(stmt.t),
+                   strTy, lenTy, labelMap, assignMap);
+}
+
+static std::tuple<mlir::Value, mlir::Value, mlir::Value>
+genBuffer(Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+          const Fortran::parser::IoUnit &iounit, mlir::Type strTy,
+          mlir::Type lenTy) {
+  [[maybe_unused]] auto &var = std::get<Fortran::parser::Variable>(iounit.u);
+  TODO();
+}
+template <typename A>
+std::tuple<mlir::Value, mlir::Value, mlir::Value>
+getBuffer(Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+          const A &stmt, mlir::Type strTy, mlir::Type lenTy) {
+  if (stmt.iounit)
+    return genBuffer(converter, loc, *stmt.iounit, strTy, lenTy);
+  return genBuffer(converter, loc, *getIOControl<Fortran::parser::IoUnit>(stmt),
+                   strTy, lenTy);
+}
+
+template <typename A>
+mlir::Value getDescriptor(Fortran::lower::AbstractConverter &converter,
+                          mlir::Location loc, const A &stmt,
+                          mlir::Type toType) {
+  TODO();
+}
+
+static mlir::Value genIOUnit(Fortran::lower::AbstractConverter &converter,
+                             mlir::Location loc,
+                             const Fortran::parser::IoUnit &iounit,
+                             mlir::Type ty) {
+  auto &builder = converter.getFirOpBuilder();
+  if (auto *e = std::get_if<Fortran::parser::FileUnitNumber>(&iounit.u)) {
+    auto ex = converter.genExprValue(Fortran::semantics::GetExpr(*e), loc);
+    return builder.createConvert(loc, ty, ex);
+  }
+  return builder.create<mlir::ConstantOp>(
+      loc, builder.getIntegerAttr(ty, Fortran::runtime::io::DefaultUnit));
+}
+
+template <typename A>
+mlir::Value getIOUnit(Fortran::lower::AbstractConverter &converter,
+                      mlir::Location loc, const A &stmt, mlir::Type ty) {
+  if (stmt.iounit)
+    return genIOUnit(converter, loc, *stmt.iounit, ty);
+  return genIOUnit(converter, loc, *getIOControl<Fortran::parser::IoUnit>(stmt),
+                   ty);
+}
+
+//===----------------------------------------------------------------------===//
+// Generators for each I/O statement type.
+//===----------------------------------------------------------------------===//
+
+template <typename K, typename S>
+static mlir::Value genBasicIOStmt(Fortran::lower::AbstractConverter &converter,
+                                  const S &stmt) {
+  auto &builder = converter.getFirOpBuilder();
+  auto loc = converter.getCurrentLocation();
+  auto beginFunc = getIORuntimeFunc<K>(loc, builder);
+  mlir::FunctionType beginFuncTy = beginFunc.getType();
+  auto unit = converter.genExprValue(
+      getExpr<Fortran::parser::FileUnitNumber>(stmt), loc);
+  auto un = builder.createConvert(loc, beginFuncTy.getInput(0), unit);
+  auto file = getDefaultFilename(builder, loc, beginFuncTy.getInput(1));
+  auto line = getDefaultLineNo(builder, loc, beginFuncTy.getInput(2));
+  llvm::SmallVector<mlir::Value, 4> args{un, file, line};
+  auto cookie = builder.create<mlir::CallOp>(loc, beginFunc, args).getResult(0);
+  ConditionSpecifierInfo csi{};
+  genConditionHandlerCall(converter, loc, cookie, stmt.v, csi);
+  mlir::Value ok{};
+  auto insertPt = threadSpecs(converter, loc, cookie, stmt.v,
+                              csi.hasErrorConditionSpecifier(), ok);
+  if (insertPt.isSet())
+    builder.restoreInsertionPoint(insertPt);
+  return genEndIO(converter, converter.getCurrentLocation(), cookie, csi);
+}
+
+mlir::Value Fortran::lower::genBackspaceStatement(
+    Fortran::lower::AbstractConverter &converter,
+    const Fortran::parser::BackspaceStmt &stmt) {
+  return genBasicIOStmt<mkIOKey(BeginBackspace)>(converter, stmt);
+}
+
+mlir::Value Fortran::lower::genEndfileStatement(
+    Fortran::lower::AbstractConverter &converter,
+    const Fortran::parser::EndfileStmt &stmt) {
+  return genBasicIOStmt<mkIOKey(BeginEndfile)>(converter, stmt);
+}
+
+mlir::Value
+Fortran::lower::genFlushStatement(Fortran::lower::AbstractConverter &converter,
+                                  const Fortran::parser::FlushStmt &stmt) {
+  return genBasicIOStmt<mkIOKey(BeginFlush)>(converter, stmt);
+}
+
+mlir::Value
+Fortran::lower::genRewindStatement(Fortran::lower::AbstractConverter &converter,
+                                   const Fortran::parser::RewindStmt &stmt) {
+  return genBasicIOStmt<mkIOKey(BeginRewind)>(converter, stmt);
+}
+
+mlir::Value
+Fortran::lower::genOpenStatement(Fortran::lower::AbstractConverter &converter,
+                                 const Fortran::parser::OpenStmt &stmt) {
+  auto &builder = converter.getFirOpBuilder();
+  mlir::FuncOp beginFunc;
+  llvm::SmallVector<mlir::Value, 4> beginArgs;
+  auto loc = converter.getCurrentLocation();
+  if (hasMem<Fortran::parser::FileUnitNumber>(stmt)) {
+    beginFunc = getIORuntimeFunc<mkIOKey(BeginOpenUnit)>(loc, builder);
+    mlir::FunctionType beginFuncTy = beginFunc.getType();
+    auto unit = converter.genExprValue(
+        getExpr<Fortran::parser::FileUnitNumber>(stmt), loc);
+    beginArgs.push_back(
+        builder.createConvert(loc, beginFuncTy.getInput(0), unit));
+    beginArgs.push_back(
+        getDefaultFilename(builder, loc, beginFuncTy.getInput(1)));
+    beginArgs.push_back(
+        getDefaultLineNo(builder, loc, beginFuncTy.getInput(2)));
+  } else {
+    assert(hasMem<Fortran::parser::ConnectSpec::Newunit>(stmt));
+    beginFunc = getIORuntimeFunc<mkIOKey(BeginOpenNewUnit)>(loc, builder);
+    mlir::FunctionType beginFuncTy = beginFunc.getType();
+    beginArgs.push_back(
+        getDefaultFilename(builder, loc, beginFuncTy.getInput(0)));
+    beginArgs.push_back(
+        getDefaultLineNo(builder, loc, beginFuncTy.getInput(1)));
+  }
+  auto cookie =
+      builder.create<mlir::CallOp>(loc, beginFunc, beginArgs).getResult(0);
+  ConditionSpecifierInfo csi{};
+  genConditionHandlerCall(converter, loc, cookie, stmt.v, csi);
+  mlir::Value ok{};
+  auto insertPt = threadSpecs(converter, loc, cookie, stmt.v,
+                              csi.hasErrorConditionSpecifier(), ok);
+  if (insertPt.isSet())
+    builder.restoreInsertionPoint(insertPt);
+  return genEndIO(converter, loc, cookie, csi);
+}
+
+mlir::Value
+Fortran::lower::genCloseStatement(Fortran::lower::AbstractConverter &converter,
+                                  const Fortran::parser::CloseStmt &stmt) {
+  return genBasicIOStmt<mkIOKey(BeginClose)>(converter, stmt);
+}
+
+mlir::Value
+Fortran::lower::genWaitStatement(Fortran::lower::AbstractConverter &converter,
+                                 const Fortran::parser::WaitStmt &stmt) {
+  auto &builder = converter.getFirOpBuilder();
+  auto loc = converter.getCurrentLocation();
+  bool hasId = hasMem<Fortran::parser::IdExpr>(stmt);
+  mlir::FuncOp beginFunc =
+      hasId ? getIORuntimeFunc<mkIOKey(BeginWait)>(loc, builder)
+            : getIORuntimeFunc<mkIOKey(BeginWaitAll)>(loc, builder);
+  mlir::FunctionType beginFuncTy = beginFunc.getType();
+  auto unit = converter.genExprValue(
+      getExpr<Fortran::parser::FileUnitNumber>(stmt), loc);
+  auto un = builder.createConvert(loc, beginFuncTy.getInput(0), unit);
+  llvm::SmallVector<mlir::Value, 4> args{un};
+  if (hasId) {
+    auto id =
+        converter.genExprValue(getExpr<Fortran::parser::IdExpr>(stmt), loc);
+    args.push_back(builder.createConvert(loc, beginFuncTy.getInput(1), id));
+  }
+  auto cookie = builder.create<mlir::CallOp>(loc, beginFunc, args).getResult(0);
+  ConditionSpecifierInfo csi{};
+  genConditionHandlerCall(converter, loc, cookie, stmt.v, csi);
+  return genEndIO(converter, converter.getCurrentLocation(), cookie, csi);
+}
+
+//===----------------------------------------------------------------------===//
+// Data transfer statements.
+//
+// There are several dimensions to the API with regard to data transfer
+// statements that need to be considered.
+//
+//   - input (READ) vs. output (WRITE, PRINT)
+//   - formatted vs. list vs. unformatted
+//   - synchronous vs. asynchronous
+//   - namelist vs. list
+//   - external vs. internal + default KIND vs. internal + other KIND
+//===----------------------------------------------------------------------===//
+
+// Determine the correct BeginXyz{In|Out}put api to invoke.
+template <bool isInput>
+mlir::FuncOp getBeginDataTransfer(mlir::Location loc, FirOpBuilder &builder,
+                                  bool isFormatted, bool isList, bool isIntern,
+                                  bool isOtherIntern, bool isAsynch,
+                                  bool isNml) {
+  if constexpr (isInput) {
+    if (isAsynch)
+      return getIORuntimeFunc<mkIOKey(BeginAsynchronousInput)>(loc, builder);
+    if (isFormatted) {
+      if (isIntern) {
+        if (isNml)
+          return getIORuntimeFunc<mkIOKey(BeginInternalNamelistInput)>(loc,
+                                                                       builder);
+        if (isOtherIntern) {
+          if (isList)
+            return getIORuntimeFunc<mkIOKey(BeginInternalArrayListInput)>(
+                loc, builder);
+          return getIORuntimeFunc<mkIOKey(BeginInternalArrayFormattedInput)>(
+              loc, builder);
+        }
+        if (isList)
+          return getIORuntimeFunc<mkIOKey(BeginInternalListInput)>(loc,
+                                                                   builder);
+        return getIORuntimeFunc<mkIOKey(BeginInternalFormattedInput)>(loc,
+                                                                      builder);
+      }
+      if (isNml)
+        return getIORuntimeFunc<mkIOKey(BeginExternalNamelistInput)>(loc,
+                                                                     builder);
+      if (isList)
+        return getIORuntimeFunc<mkIOKey(BeginExternalListInput)>(loc, builder);
+      return getIORuntimeFunc<mkIOKey(BeginExternalFormattedInput)>(loc,
+                                                                    builder);
+    }
+    return getIORuntimeFunc<mkIOKey(BeginUnformattedInput)>(loc, builder);
+  } else {
+    if (isAsynch)
+      return getIORuntimeFunc<mkIOKey(BeginAsynchronousOutput)>(loc, builder);
+    if (isFormatted) {
+      if (isIntern) {
+        if (isNml)
+          return getIORuntimeFunc<mkIOKey(BeginInternalNamelistOutput)>(
+              loc, builder);
+        if (isOtherIntern) {
+          if (isList)
+            return getIORuntimeFunc<mkIOKey(BeginInternalArrayListOutput)>(
+                loc, builder);
+          return getIORuntimeFunc<mkIOKey(BeginInternalArrayFormattedOutput)>(
+              loc, builder);
+        }
+        if (isList)
+          return getIORuntimeFunc<mkIOKey(BeginInternalListOutput)>(loc,
+                                                                    builder);
+        return getIORuntimeFunc<mkIOKey(BeginInternalFormattedOutput)>(loc,
+                                                                       builder);
+      }
+      if (isNml)
+        return getIORuntimeFunc<mkIOKey(BeginExternalNamelistOutput)>(loc,
+                                                                      builder);
+      if (isList)
+        return getIORuntimeFunc<mkIOKey(BeginExternalListOutput)>(loc, builder);
+      return getIORuntimeFunc<mkIOKey(BeginExternalFormattedOutput)>(loc,
+                                                                     builder);
+    }
+    return getIORuntimeFunc<mkIOKey(BeginUnformattedOutput)>(loc, builder);
+  }
+}
+
+/// Generate the arguments of a BeginXyz call.
+template <bool hasIOCtrl, typename A>
+void genBeginCallArguments(llvm::SmallVector<mlir::Value, 8> &ioArgs,
+                           Fortran::lower::AbstractConverter &converter,
+                           mlir::Location loc, const A &stmt,
+                           mlir::FunctionType ioFuncTy, bool isFormatted,
+                           bool isList, bool isIntern, bool isOtherIntern,
+                           bool isAsynch, bool isNml,
+                           Fortran::lower::pft::LabelEvalMap &labelMap,
+                           Fortran::lower::pft::SymbolLabelMap &assignMap) {
+  auto &builder = converter.getFirOpBuilder();
+  if constexpr (hasIOCtrl) {
+    // READ/WRITE cases have a wide variety of argument permutations
+    if (isAsynch || !isFormatted) {
+      // unit (always first), ...
+      ioArgs.push_back(
+          getIOUnit(converter, loc, stmt, ioFuncTy.getInput(ioArgs.size())));
+      if (isAsynch) {
+        // unknown-thingy, [buff, LEN]
+        llvm_unreachable("not implemented");
+      }
+      return;
+    }
+    assert(isFormatted && "formatted data transfer");
+    if (!isIntern) {
+      if (isNml) {
+        // namelist group, ...
+        llvm_unreachable("not implemented");
+      } else if (!isList) {
+        // | [format, LEN], ...
+        auto pair = getFormat(
+            converter, loc, stmt, ioFuncTy.getInput(ioArgs.size()),
+            ioFuncTy.getInput(ioArgs.size() + 1), labelMap, assignMap);
+        ioArgs.push_back(std::get<0>(pair));
+        ioArgs.push_back(std::get<1>(pair));
+      }
+      // unit (always last)
+      ioArgs.push_back(
+          getIOUnit(converter, loc, stmt, ioFuncTy.getInput(ioArgs.size())));
+      return;
+    }
+    assert(isIntern && "internal data transfer");
+    if (isNml || isOtherIntern) {
+      // descriptor, ...
+      ioArgs.push_back(getDescriptor(converter, loc, stmt,
+                                     ioFuncTy.getInput(ioArgs.size())));
+      if (isNml) {
+        // namelist group, ...
+        llvm_unreachable("not implemented");
+      } else if (isOtherIntern && !isList) {
+        // | [format, LEN], ...
+        auto pair = getFormat(
+            converter, loc, stmt, ioFuncTy.getInput(ioArgs.size()),
+            ioFuncTy.getInput(ioArgs.size() + 1), labelMap, assignMap);
+        ioArgs.push_back(std::get<0>(pair));
+        ioArgs.push_back(std::get<1>(pair));
+      }
+    } else {
+      // | [buff, LEN], ...
+      auto pair =
+          getBuffer(converter, loc, stmt, ioFuncTy.getInput(ioArgs.size()),
+                    ioFuncTy.getInput(ioArgs.size() + 1));
+      ioArgs.push_back(std::get<0>(pair));
+      ioArgs.push_back(std::get<1>(pair));
+      if (!isList) {
+        // [format, LEN], ...
+        auto pair = getFormat(
+            converter, loc, stmt, ioFuncTy.getInput(ioArgs.size()),
+            ioFuncTy.getInput(ioArgs.size() + 1), labelMap, assignMap);
+        ioArgs.push_back(std::get<0>(pair));
+        ioArgs.push_back(std::get<1>(pair));
+      }
+    }
+    // [scratch, LEN] (always last)
+    ioArgs.push_back(
+        getDefaultScratch(builder, loc, ioFuncTy.getInput(ioArgs.size())));
+    ioArgs.push_back(
+        getDefaultScratchLen(builder, loc, ioFuncTy.getInput(ioArgs.size())));
+  } else {
+    if (!isList) {
+      // [format, LEN], ...
+      auto pair =
+          getFormat(converter, loc, stmt, ioFuncTy.getInput(ioArgs.size()),
+                    ioFuncTy.getInput(ioArgs.size() + 1), labelMap, assignMap);
+      ioArgs.push_back(std::get<0>(pair));
+      ioArgs.push_back(std::get<1>(pair));
+    }
+    // unit (always last)
+    ioArgs.push_back(builder.create<mlir::ConstantOp>(
+        loc, builder.getIntegerAttr(ioFuncTy.getInput(ioArgs.size()),
+                                    Fortran::runtime::io::DefaultUnit)));
+  }
+}
+
+template <bool isInput, bool hasIOCtrl = true, typename A>
+static mlir::Value
+genDataTransferStmt(Fortran::lower::AbstractConverter &converter, const A &stmt,
+                    Fortran::lower::pft::LabelEvalMap &labelMap,
+                    Fortran::lower::pft::SymbolLabelMap &assignMap) {
+  auto &builder = converter.getFirOpBuilder();
+  auto loc = converter.getCurrentLocation();
+  const bool isFormatted = isDataTransferFormatted(stmt);
+  const bool isList = isFormatted ? isDataTransferList(stmt) : false;
+  const bool isIntern = isDataTransferInternal(stmt);
+  const bool isOtherIntern =
+      isIntern ? isDataTransferInternalNotDefaultKind(stmt) : false;
+  const bool isAsynch = isDataTransferAsynchronous(stmt);
+  const bool isNml = isDataTransferNamelist(stmt);
+
+  // Determine which BeginXyz call to make.
+  mlir::FuncOp ioFunc =
+      getBeginDataTransfer<isInput>(loc, builder, isFormatted, isList, isIntern,
+                                    isOtherIntern, isAsynch, isNml);
+  mlir::FunctionType ioFuncTy = ioFunc.getType();
+
+  // Append BeginXyz call arguments.  File name and line number are always last.
+  llvm::SmallVector<mlir::Value, 8> ioArgs;
+  genBeginCallArguments<hasIOCtrl>(ioArgs, converter, loc, stmt, ioFuncTy,
+                                   isFormatted, isList, isIntern, isOtherIntern,
+                                   isAsynch, isNml, labelMap, assignMap);
+  ioArgs.push_back(
+      getDefaultFilename(builder, loc, ioFuncTy.getInput(ioArgs.size())));
+  ioArgs.push_back(
+      getDefaultLineNo(builder, loc, ioFuncTy.getInput(ioArgs.size())));
+
+  // Arguments are done; call the BeginXyz function.
+  mlir::Value cookie =
+      builder.create<mlir::CallOp>(loc, ioFunc, ioArgs).getResult(0);
+
+  // Generate an EnableHandlers call and remaining specifier calls.
+  ConditionSpecifierInfo csi;
+  mlir::OpBuilder::InsertPoint insertPt;
+  mlir::Value ok;
+  if constexpr (hasIOCtrl) {
+    genConditionHandlerCall(converter, loc, cookie, stmt.controls, csi);
+    insertPt = threadSpecs(converter, loc, cookie, stmt.controls,
+                           csi.hasErrorConditionSpecifier(), ok);
+  }
+
+  // Generate data transfer list calls.
+  if constexpr (isInput) // ReadStmt
+    genInputItemList(converter, cookie, stmt.items, insertPt,
+                     csi.hasTransferConditionSpecifier(), ok, false);
+  else if constexpr (std::is_same_v<A, Fortran::parser::PrintStmt>)
+    genOutputItemList(converter, cookie, std::get<1>(stmt.t), insertPt,
+                      csi.hasTransferConditionSpecifier(), ok, false);
+  else // WriteStmt
+    genOutputItemList(converter, cookie, stmt.items, insertPt,
+                      csi.hasTransferConditionSpecifier(), ok, false);
+
+  // Generate end statement call/s.
+  if (insertPt.isSet())
+    builder.restoreInsertionPoint(insertPt);
+  return genEndIO(converter, loc, cookie, csi);
+}
+
+void Fortran::lower::genPrintStatement(
+    Fortran::lower::AbstractConverter &converter,
+    const Fortran::parser::PrintStmt &stmt,
+    Fortran::lower::pft::LabelEvalMap &labelMap,
+    Fortran::lower::pft::SymbolLabelMap &assignMap) {
+  // PRINT does not take an io-control-spec. It only has a format specifier, so
+  // it is a simplified case of WRITE.
+  genDataTransferStmt</*isInput=*/false, /*ioCtrl=*/false>(converter, stmt,
+                                                           labelMap, assignMap);
+}
+
+mlir::Value Fortran::lower::genWriteStatement(
+    Fortran::lower::AbstractConverter &converter,
+    const Fortran::parser::WriteStmt &stmt,
+    Fortran::lower::pft::LabelEvalMap &labelMap,
+    Fortran::lower::pft::SymbolLabelMap &assignMap) {
+  return genDataTransferStmt</*isInput=*/false>(converter, stmt, labelMap,
+                                                assignMap);
+}
+
+mlir::Value Fortran::lower::genReadStatement(
+    Fortran::lower::AbstractConverter &converter,
+    const Fortran::parser::ReadStmt &stmt,
+    Fortran::lower::pft::LabelEvalMap &labelMap,
+    Fortran::lower::pft::SymbolLabelMap &assignMap) {
+  return genDataTransferStmt</*isInput=*/true>(converter, stmt, labelMap,
+                                               assignMap);
+}
+
+/// Get the file expression from the inquire spec list. Also return if the
+/// expression is a file name.
+static std::pair<const Fortran::semantics::SomeExpr *, bool>
+getInquireFileExpr(const std::list<Fortran::parser::InquireSpec> *stmt) {
+  if (!stmt)
+    return {nullptr, false};
+  for (const auto &spec : *stmt) {
+    if (auto *f = std::get_if<Fortran::parser::FileUnitNumber>(&spec.u))
+      return {Fortran::semantics::GetExpr(*f), false};
+    if (auto *f = std::get_if<Fortran::parser::FileNameExpr>(&spec.u))
+      return {Fortran::semantics::GetExpr(*f), true};
+  }
+  // semantics should have already caught this condition
+  llvm_unreachable("inquire spec must have a file");
+}
+
+mlir::Value Fortran::lower::genInquireStatement(
+    Fortran::lower::AbstractConverter &converter,
+    const Fortran::parser::InquireStmt &stmt) {
+  auto &builder = converter.getFirOpBuilder();
+  auto loc = converter.getCurrentLocation();
+  mlir::FuncOp beginFunc;
+  mlir::Value cookie;
+  ConditionSpecifierInfo csi{};
+  const auto *list =
+      std::get_if<std::list<Fortran::parser::InquireSpec>>(&stmt.u);
+  auto exprPair = getInquireFileExpr(list);
+  auto inquireFileUnit = [&]() -> bool {
+    return exprPair.first && !exprPair.second;
+  };
+  auto inquireFileName = [&]() -> bool {
+    return exprPair.first && exprPair.second;
+  };
+
+  // Determine which BeginInquire call to make.
+  if (inquireFileUnit()) {
+    // File unit call.
+    beginFunc = getIORuntimeFunc<mkIOKey(BeginInquireUnit)>(loc, builder);
+    mlir::FunctionType beginFuncTy = beginFunc.getType();
+    auto unit = converter.genExprValue(exprPair.first, loc);
+    auto un = builder.createConvert(loc, beginFuncTy.getInput(0), unit);
+    auto file = getDefaultFilename(builder, loc, beginFuncTy.getInput(1));
+    auto line = getDefaultLineNo(builder, loc, beginFuncTy.getInput(2));
+    llvm::SmallVector<mlir::Value, 4> beginArgs{un, file, line};
+    cookie =
+        builder.create<mlir::CallOp>(loc, beginFunc, beginArgs).getResult(0);
+    // Handle remaining arguments in specifier list.
+    genConditionHandlerCall(converter, loc, cookie, *list, csi);
+  } else if (inquireFileName()) {
+    // Filename call.
+    beginFunc = getIORuntimeFunc<mkIOKey(BeginInquireFile)>(loc, builder);
+    mlir::FunctionType beginFuncTy = beginFunc.getType();
+    auto file = converter.genExprValue(exprPair.first, loc);
+    // Helper to query [BUFFER, LEN].
+    Fortran::lower::CharacterExprHelper helper(builder, loc);
+    auto dataLen = helper.materializeCharacter(file);
+    auto buff =
+        builder.createConvert(loc, beginFuncTy.getInput(0), dataLen.first);
+    auto len =
+        builder.createConvert(loc, beginFuncTy.getInput(1), dataLen.second);
+    auto kindInt = helper.getCharacterKind(file.getType());
+    mlir::Value kindValue =
+        builder.createIntegerConstant(loc, beginFuncTy.getInput(2), kindInt);
+    auto sourceFile = getDefaultFilename(builder, loc, beginFuncTy.getInput(3));
+    auto line = getDefaultLineNo(builder, loc, beginFuncTy.getInput(4));
+    llvm::SmallVector<mlir::Value, 5> beginArgs = {
+        buff, len, kindValue, sourceFile, line,
+    };
+    cookie =
+        builder.create<mlir::CallOp>(loc, beginFunc, beginArgs).getResult(0);
+    // Handle remaining arguments in specifier list.
+    genConditionHandlerCall(converter, loc, cookie, *list, csi);
+  } else {
+    // Io length call.
+    const auto *ioLength =
+        std::get_if<Fortran::parser::InquireStmt::Iolength>(&stmt.u);
+    assert(ioLength && "must have an io length");
+    beginFunc = getIORuntimeFunc<mkIOKey(BeginInquireIoLength)>(loc, builder);
+    mlir::FunctionType beginFuncTy = beginFunc.getType();
+    auto file = getDefaultFilename(builder, loc, beginFuncTy.getInput(0));
+    auto line = getDefaultLineNo(builder, loc, beginFuncTy.getInput(1));
+    llvm::SmallVector<mlir::Value, 4> beginArgs{file, line};
+    cookie =
+        builder.create<mlir::CallOp>(loc, beginFunc, beginArgs).getResult(0);
+    // Handle remaining arguments in output list.
+    genConditionHandlerCall(
+        converter, loc, cookie,
+        std::get<std::list<Fortran::parser::OutputItem>>(ioLength->t), csi);
+  }
+  // Generate end statement call.
+  return genEndIO(converter, loc, cookie, csi);
+}
diff --git a/flang/lib/Lower/IntrinsicCall.cpp b/flang/lib/Lower/IntrinsicCall.cpp
new file mode 100644
index 0000000000000..702e85d62b2bc
--- /dev/null
+++ b/flang/lib/Lower/IntrinsicCall.cpp
@@ -0,0 +1,1380 @@
+//===-- IntrinsicCall.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Helper routines for constructing the FIR dialect of MLIR. As FIR is a
+// dialect of MLIR, it makes extensive use of MLIR interfaces and MLIR's coding
+// style (https://mlir.llvm.org/getting_started/DeveloperGuide/) is used in this
+// module.
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Lower/IntrinsicCall.h"
+#include "RTBuilder.h"
+#include "flang/Lower/CharacterExpr.h"
+#include "flang/Lower/ComplexExpr.h"
+#include "flang/Lower/ConvertType.h"
+#include "flang/Lower/FIRBuilder.h"
+#include "flang/Lower/Mangler.h"
+#include "flang/Lower/Runtime.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ErrorHandling.h"
+#include <algorithm>
+#include <utility>
+
+#define PGMATH_DECLARE
+#include "../runtime/pgmath.h.inc"
+
+/// This file implements lowering of Fortran intrinsic procedures.
+/// Intrinsics are lowered to a mix of FIR and MLIR operations as
+/// well as call to runtime functions or LLVM intrinsics.
+
+/// Lowering of intrinsic procedure calls is based on a map that associates
+/// Fortran intrinsic generic names to FIR generator functions.
+/// All generator functions are member functions of the IntrinsicLibrary class
+/// and have the same interface.
+/// If no generator is given for an intrinsic name, a math runtime library
+/// is searched for an implementation and, if a runtime function is found,
+/// a call is generated for it. LLVM intrinsics are handled as a math
+/// runtime library here.
+
+/// Enums used to templatize and share lowering of MIN and MAX.
+enum class Extremum { Min, Max };
+
+// There are different ways to deal with NaNs in MIN and MAX.
+// Known existing behaviors are listed below and can be selected for
+// f18 MIN/MAX implementation.
+enum class ExtremumBehavior {
+  // Note: the Signaling/quiet aspect of NaNs in the behaviors below are
+  // not described because there is no way to control/observe such aspect in
+  // MLIR/LLVM yet. The IEEE behaviors come with requirements regarding this
+  // aspect that are therefore currently not enforced. In the descriptions
+  // below, NaNs can be signaling or quite. Returned NaNs may be signaling
+  // if one of the input NaN was signaling but it cannot be guaranteed either.
+  // Existing compilers using an IEEE behavior (gfortran) also do not fulfill
+  // signaling/quiet requirements.
+  IeeeMinMaximumNumber,
+  // IEEE minimumNumber/maximumNumber behavior (754-2019, section 9.6):
+  // If one of the argument is and number and the other is NaN, return the
+  // number. If both arguements are NaN, return NaN.
+  // Compilers: gfortran.
+  IeeeMinMaximum,
+  // IEEE minimum/maximum behavior (754-2019, section 9.6):
+  // If one of the argument is NaN, return NaN.
+  MinMaxss,
+  // x86 minss/maxss behavior:
+  // If the second argument is a number and the other is NaN, return the number.
+  // In all other cases where at least one operand is NaN, return NaN.
+  // Compilers: xlf (only for MAX), ifort, pgfortran -nollvm, and nagfor.
+  PgfortranLlvm,
+  // "Opposite of" x86 minss/maxss behavior:
+  // If the first argument is a number and the other is NaN, return the
+  // number.
+  // In all other cases where at least one operand is NaN, return NaN.
+  // Compilers: xlf (only for MIN), and pgfortran (with llvm).
+  IeeeMinMaxNum
+  // IEEE minNum/maxNum behavior (754-2008, section 5.3.1):
+  // TODO: Not implemented.
+  // It is the only behavior where the signaling/quiet aspect of a NaN argument
+  // impacts if the result should be NaN or the argument that is a number.
+  // LLVM/MLIR do not provide ways to observe this aspect, so it is not
+  // possible to implement it without some target dependent runtime.
+};
+
+namespace {
+/// StaticMultimapView is a constexpr friendly multimap
+/// implementation over sorted constexpr arrays.
+/// As the View name suggests, it does not duplicate the
+/// sorted array but only brings range and search concepts
+/// over it. It provides compile time search and can also
+/// provide dynamic search (currently linear, can be improved to
+/// log(n) due to the sorted array property).
+
+// TODO: Find a better place for this if this is retained.
+// This is currently here because this was designed to provide
+// maps over runtime description without the burden of having to
+// instantiate these maps dynamically and to keep them somewhere.
+template <typename V>
+class StaticMultimapView {
+public:
+  using Key = typename V::Key;
+  struct Range {
+    using const_iterator = const V *;
+    constexpr const_iterator begin() const { return startPtr; }
+    constexpr const_iterator end() const { return endPtr; }
+    constexpr bool empty() const {
+      return startPtr == nullptr || endPtr == nullptr || endPtr <= startPtr;
+    }
+    constexpr std::size_t size() const {
+      return empty() ? 0 : static_cast<std::size_t>(endPtr - startPtr);
+    }
+    const V *startPtr{nullptr};
+    const V *endPtr{nullptr};
+  };
+  using const_iterator = typename Range::const_iterator;
+
+  template <std::size_t N>
+  constexpr StaticMultimapView(const V (&array)[N])
+      : range{&array[0], &array[0] + N} {}
+  template <typename Key>
+  constexpr bool verify() {
+    // TODO: sorted
+    // non empty increasing pointer direction
+    return !range.empty();
+  }
+  constexpr const_iterator begin() const { return range.begin(); }
+  constexpr const_iterator end() const { return range.end(); }
+
+  // Assume array is sorted.
+  // TODO make it a log(n) search based on sorted property
+  // std::equal_range will be constexpr in C++20 only.
+  constexpr Range getRange(const Key &key) const {
+    bool matched{false};
+    const V *start{nullptr}, *end{nullptr};
+    for (const auto &desc : range) {
+      if (desc.key == key) {
+        if (!matched) {
+          start = &desc;
+          matched = true;
+        }
+      } else if (matched) {
+        end = &desc;
+        matched = false;
+      }
+    }
+    if (matched) {
+      end = range.end();
+    }
+    return Range{start, end};
+  }
+
+  constexpr std::pair<const_iterator, const_iterator>
+  equal_range(const Key &key) const {
+    Range range{getRange(key)};
+    return {range.begin(), range.end()};
+  }
+
+  constexpr typename Range::const_iterator find(Key key) const {
+    const Range subRange{getRange(key)};
+    return subRange.size() == 1 ? subRange.begin() : end();
+  }
+
+private:
+  Range range{nullptr, nullptr};
+};
+} // namespace
+
+// TODO error handling -> return a code or directly emit messages ?
+struct IntrinsicLibrary {
+
+  // Constructors.
+  explicit IntrinsicLibrary(Fortran::lower::FirOpBuilder &builder,
+                            mlir::Location loc)
+      : builder{builder}, loc{loc} {}
+  IntrinsicLibrary() = delete;
+  IntrinsicLibrary(const IntrinsicLibrary &) = delete;
+
+  /// Generate FIR for call to Fortran intrinsic \p name with arguments \p arg
+  /// and expected result type \p resultType.
+  fir::ExtendedValue genIntrinsicCall(llvm::StringRef name,
+                                      mlir::Type resultType,
+                                      llvm::ArrayRef<fir::ExtendedValue> arg);
+
+  /// Search a runtime function that is associated to the generic intrinsic name
+  /// and whose signature matches the intrinsic arguments and result types.
+  /// If no such runtime function is found but a runtime function associated
+  /// with the Fortran generic exists and has the same number of arguments,
+  /// conversions will be inserted before and/or after the call. This is to
+  /// mainly to allow 16 bits float support even-though little or no math
+  /// runtime is currently available for it.
+  mlir::Value genRuntimeCall(llvm::StringRef name, mlir::Type,
+                             llvm::ArrayRef<mlir::Value>);
+
+  using RuntimeCallGenerator =
+      std::function<mlir::Value(Fortran::lower::FirOpBuilder &, mlir::Location,
+                                llvm::ArrayRef<mlir::Value>)>;
+  RuntimeCallGenerator
+  getRuntimeCallGenerator(llvm::StringRef name,
+                          mlir::FunctionType soughtFuncType);
+
+  mlir::Value genAbs(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAimag(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAint(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genAnint(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genCeiling(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genConjg(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genDim(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genDprod(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  template <Extremum, ExtremumBehavior>
+  mlir::Value genExtremum(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genFloor(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genIAnd(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genIchar(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genIEOr(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genIOr(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  fir::ExtendedValue genLen(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
+  fir::ExtendedValue genLenTrim(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
+  mlir::Value genMerge(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genMod(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genNint(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genSign(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  /// Implement all conversion functions like DBLE, the first argument is
+  /// the value to convert. There may be an additional KIND arguments that
+  /// is ignored because this is already reflected in the result type.
+  mlir::Value genConversion(mlir::Type, llvm::ArrayRef<mlir::Value>);
+
+  /// Define the different FIR generators that can be mapped to intrinsic to
+  /// generate the related code.
+  using ElementalGenerator = decltype(&IntrinsicLibrary::genAbs);
+  using ExtendedGenerator = decltype(&IntrinsicLibrary::genLenTrim);
+  using Generator = std::variant<ElementalGenerator, ExtendedGenerator>;
+
+  /// All generators can be outlined. This will build a function named
+  /// "fir."+ <generic name> + "." + <result type code> and generate the
+  /// intrinsic implementation inside instead of at the intrinsic call sites.
+  /// This can be used to keep the FIR more readable. Only one function will
+  /// be generated for all the similar calls in a program.
+  /// If the Generator is nullptr, the wrapper uses genRuntimeCall.
+  template <typename GeneratorType>
+  mlir::Value outlineInWrapper(GeneratorType, llvm::StringRef name,
+                               mlir::Type resultType,
+                               llvm::ArrayRef<mlir::Value> args);
+  fir::ExtendedValue outlineInWrapper(ExtendedGenerator, llvm::StringRef name,
+                                      mlir::Type resultType,
+                                      llvm::ArrayRef<fir::ExtendedValue> args);
+
+  template <typename GeneratorType>
+  mlir::FuncOp getWrapper(GeneratorType, llvm::StringRef name,
+                          mlir::FunctionType, bool loadRefArguments = false);
+
+  /// Generate calls to ElementalGenerator, handling the elemental aspects
+  template <typename GeneratorType>
+  fir::ExtendedValue
+  genElementalCall(GeneratorType, llvm::StringRef name, mlir::Type resultType,
+                   llvm::ArrayRef<fir::ExtendedValue> args, bool outline);
+
+  /// Helper to invoke code generator for the intrinsics given arguments.
+  mlir::Value invokeGenerator(ElementalGenerator generator,
+                              mlir::Type resultType,
+                              llvm::ArrayRef<mlir::Value> args);
+  mlir::Value invokeGenerator(RuntimeCallGenerator generator,
+                              mlir::Type resultType,
+                              llvm::ArrayRef<mlir::Value> args);
+  mlir::Value invokeGenerator(ExtendedGenerator generator,
+                              mlir::Type resultType,
+                              llvm::ArrayRef<mlir::Value> args);
+
+  /// Get pointer to unrestricted intrinsic. Generate the related unrestricted
+  /// intrinsic if it is not defined yet.
+  mlir::SymbolRefAttr
+  getUnrestrictedIntrinsicSymbolRefAttr(llvm::StringRef name,
+                                        mlir::FunctionType signature);
+
+  Fortran::lower::FirOpBuilder &builder;
+  mlir::Location loc;
+};
+
+/// Table that drives the fir generation depending on the intrinsic.
+/// one to one mapping with Fortran arguments. If no mapping is
+/// defined here for a generic intrinsic, genRuntimeCall will be called
+/// to look for a match in the runtime a emit a call.
+struct IntrinsicHandler {
+  const char *name;
+  IntrinsicLibrary::Generator generator;
+  bool isElemental = true;
+  /// Code heavy intrinsic can be outlined to make FIR
+  /// more readable.
+  bool outline = false;
+};
+using I = IntrinsicLibrary;
+static constexpr IntrinsicHandler handlers[]{
+    {"abs", &I::genAbs},
+    {"achar", &I::genConversion},
+    {"aimag", &I::genAimag},
+    {"aint", &I::genAint},
+    {"anint", &I::genAnint},
+    {"ceiling", &I::genCeiling},
+    {"char", &I::genConversion},
+    {"conjg", &I::genConjg},
+    {"dim", &I::genDim},
+    {"dble", &I::genConversion},
+    {"dprod", &I::genDprod},
+    {"floor", &I::genFloor},
+    {"iand", &I::genIAnd},
+    {"ichar", &I::genIchar},
+    {"ieor", &I::genIEOr},
+    {"ior", &I::genIOr},
+    {"len", &I::genLen},
+    {"len_trim", &I::genLenTrim},
+    {"max", &I::genExtremum<Extremum::Max, ExtremumBehavior::MinMaxss>},
+    {"min", &I::genExtremum<Extremum::Min, ExtremumBehavior::MinMaxss>},
+    {"merge", &I::genMerge},
+    {"mod", &I::genMod},
+    {"nint", &I::genNint},
+    {"sign", &I::genSign},
+};
+
+/// To make fir output more readable for debug, one can outline all intrinsic
+/// implementation in wrappers (overrides the IntrinsicHandler::outline flag).
+static llvm::cl::opt<bool> outlineAllIntrinsics(
+    "outline-intrinsics",
+    llvm::cl::desc(
+        "Lower all intrinsic procedure implementation in their own functions"),
+    llvm::cl::init(false));
+
+//===----------------------------------------------------------------------===//
+// Math runtime description and matching utility
+//===----------------------------------------------------------------------===//
+
+/// Command line option to modify math runtime version used to implement
+/// intrinsics.
+enum MathRuntimeVersion {
+  fastVersion,
+  relaxedVersion,
+  preciseVersion,
+  llvmOnly
+};
+llvm::cl::opt<MathRuntimeVersion> mathRuntimeVersion(
+    "math-runtime", llvm::cl::desc("Select math runtime version:"),
+    llvm::cl::values(
+        clEnumValN(fastVersion, "fast", "use pgmath fast runtime"),
+        clEnumValN(relaxedVersion, "relaxed", "use pgmath relaxed runtime"),
+        clEnumValN(preciseVersion, "precise", "use pgmath precise runtime"),
+        clEnumValN(llvmOnly, "llvm",
+                   "only use LLVM intrinsics (may be incomplete)")),
+    llvm::cl::init(fastVersion));
+
+struct RuntimeFunction {
+  using Key = llvm::StringRef;
+  Key key;
+  llvm::StringRef symbol;
+  Fortran::lower::FuncTypeBuilderFunc typeGenerator;
+};
+
+#define RUNTIME_STATIC_DESCRIPTION(name, func)                                 \
+  {#name, #func,                                                               \
+   Fortran::lower::RuntimeTableKey<decltype(func)>::getTypeModel()},
+static constexpr RuntimeFunction pgmathFast[] = {
+#define PGMATH_FAST
+#define PGMATH_USE_ALL_TYPES(name, func) RUNTIME_STATIC_DESCRIPTION(name, func)
+#include "../runtime/pgmath.h.inc"
+};
+static constexpr RuntimeFunction pgmathRelaxed[] = {
+#define PGMATH_RELAXED
+#define PGMATH_USE_ALL_TYPES(name, func) RUNTIME_STATIC_DESCRIPTION(name, func)
+#include "../runtime/pgmath.h.inc"
+};
+static constexpr RuntimeFunction pgmathPrecise[] = {
+#define PGMATH_PRECISE
+#define PGMATH_USE_ALL_TYPES(name, func) RUNTIME_STATIC_DESCRIPTION(name, func)
+#include "../runtime/pgmath.h.inc"
+};
+
+static mlir::FunctionType genF32F32FuncType(mlir::MLIRContext *context) {
+  auto t = mlir::FloatType::getF32(context);
+  return mlir::FunctionType::get({t}, {t}, context);
+}
+
+static mlir::FunctionType genF64F64FuncType(mlir::MLIRContext *context) {
+  auto t = mlir::FloatType::getF64(context);
+  return mlir::FunctionType::get({t}, {t}, context);
+}
+
+template <int Bits>
+static mlir::FunctionType genIntF64FuncType(mlir::MLIRContext *context) {
+  auto t = mlir::FloatType::getF64(context);
+  auto r = mlir::IntegerType::get(Bits, context);
+  return mlir::FunctionType::get({t}, {r}, context);
+}
+
+template <int Bits>
+static mlir::FunctionType genIntF32FuncType(mlir::MLIRContext *context) {
+  auto t = mlir::FloatType::getF32(context);
+  auto r = mlir::IntegerType::get(Bits, context);
+  return mlir::FunctionType::get({t}, {r}, context);
+}
+
+// TODO : Fill-up this table with more intrinsic.
+// Note: These are also defined as operations in LLVM dialect. See if this
+// can be use and has advantages.
+static constexpr RuntimeFunction llvmIntrinsics[] = {
+    {"abs", "llvm.fabs.f32", genF32F32FuncType},
+    {"abs", "llvm.fabs.f64", genF64F64FuncType},
+    {"aint", "llvm.trunc.f32", genF32F32FuncType},
+    {"aint", "llvm.trunc.f64", genF64F64FuncType},
+    {"anint", "llvm.round.f32", genF32F32FuncType},
+    {"anint", "llvm.round.f64", genF64F64FuncType},
+    // ceil is used for CEILING but is different, it returns a real.
+    {"ceil", "llvm.ceil.f32", genF32F32FuncType},
+    {"ceil", "llvm.ceil.f64", genF64F64FuncType},
+    {"cos", "llvm.cos.f32", genF32F32FuncType},
+    {"cos", "llvm.cos.f64", genF64F64FuncType},
+    // llvm.floor is used for FLOOR, but returns real.
+    {"floor", "llvm.floor.f32", genF32F32FuncType},
+    {"floor", "llvm.floor.f64", genF64F64FuncType},
+    {"log", "llvm.log.f32", genF32F32FuncType},
+    {"log", "llvm.log.f64", genF64F64FuncType},
+    {"log10", "llvm.log10.f32", genF32F32FuncType},
+    {"log10", "llvm.log10.f64", genF64F64FuncType},
+    {"nint", "llvm.lround.i64.f64", genIntF64FuncType<64>},
+    {"nint", "llvm.lround.i64.f32", genIntF32FuncType<64>},
+    {"nint", "llvm.lround.i32.f64", genIntF64FuncType<32>},
+    {"nint", "llvm.lround.i32.f32", genIntF32FuncType<32>},
+    {"sin", "llvm.sin.f32", genF32F32FuncType},
+    {"sin", "llvm.sin.f64", genF64F64FuncType},
+    {"sqrt", "llvm.sqrt.f32", genF32F32FuncType},
+    {"sqrt", "llvm.sqrt.f64", genF64F64FuncType},
+};
+
+// This helper class computes a "distance" between two function types.
+// The distance measures how many narrowing conversions of actual arguments
+// and result of "from" must be made in order to use "to" instead of "from".
+// For instance, the distance between ACOS(REAL(10)) and ACOS(REAL(8)) is
+// greater than the one between ACOS(REAL(10)) and ACOS(REAL(16)). This means
+// if no implementation of ACOS(REAL(10)) is available, it is better to use
+// ACOS(REAL(16)) with casts rather than ACOS(REAL(8)).
+// Note that this is not a symmetric distance and the order of "from" and "to"
+// arguments matters, d(foo, bar) may not be the same as d(bar, foo) because it
+// may be safe to replace foo by bar, but not the opposite.
+class FunctionDistance {
+public:
+  FunctionDistance() : infinite{true} {}
+
+  FunctionDistance(mlir::FunctionType from, mlir::FunctionType to) {
+    auto nInputs = from.getNumInputs();
+    auto nResults = from.getNumResults();
+    if (nResults != to.getNumResults() || nInputs != to.getNumInputs()) {
+      infinite = true;
+    } else {
+      for (decltype(nInputs) i{0}; i < nInputs && !infinite; ++i)
+        addArgumentDistance(from.getInput(i), to.getInput(i));
+      for (decltype(nResults) i{0}; i < nResults && !infinite; ++i)
+        addResultDistance(to.getResult(i), from.getResult(i));
+    }
+  }
+
+  /// Beware both d1.isSmallerThan(d2) *and* d2.isSmallerThan(d1) may be
+  /// false if both d1 and d2 are infinite. This implies that
+  ///  d1.isSmallerThan(d2) is not equivalent to !d2.isSmallerThan(d1)
+  bool isSmallerThan(const FunctionDistance &d) const {
+    return !infinite &&
+           (d.infinite || std::lexicographical_compare(
+                              conversions.begin(), conversions.end(),
+                              d.conversions.begin(), d.conversions.end()));
+  }
+
+  bool isLosingPrecision() const {
+    return conversions[narrowingArg] != 0 || conversions[extendingResult] != 0;
+  }
+
+  bool isInfinite() const { return infinite; }
+
+private:
+  enum class Conversion { Forbidden, None, Narrow, Extend };
+
+  void addArgumentDistance(mlir::Type from, mlir::Type to) {
+    switch (conversionBetweenTypes(from, to)) {
+    case Conversion::Forbidden:
+      infinite = true;
+      break;
+    case Conversion::None:
+      break;
+    case Conversion::Narrow:
+      conversions[narrowingArg]++;
+      break;
+    case Conversion::Extend:
+      conversions[nonNarrowingArg]++;
+      break;
+    }
+  }
+
+  void addResultDistance(mlir::Type from, mlir::Type to) {
+    switch (conversionBetweenTypes(from, to)) {
+    case Conversion::Forbidden:
+      infinite = true;
+      break;
+    case Conversion::None:
+      break;
+    case Conversion::Narrow:
+      conversions[nonExtendingResult]++;
+      break;
+    case Conversion::Extend:
+      conversions[extendingResult]++;
+      break;
+    }
+  }
+
+  // Floating point can be mlir::FloatType or fir::real
+  static unsigned getFloatingPointWidth(mlir::Type t) {
+    if (auto f{t.dyn_cast<mlir::FloatType>()})
+      return f.getWidth();
+    // FIXME: Get width another way for fir.real/complex
+    // - use fir/KindMapping.h and llvm::Type
+    // - or use evaluate/type.h
+    if (auto r{t.dyn_cast<fir::RealType>()})
+      return r.getFKind() * 4;
+    if (auto cplx{t.dyn_cast<fir::CplxType>()})
+      return cplx.getFKind() * 4;
+    llvm_unreachable("not a floating-point type");
+  }
+
+  static Conversion conversionBetweenTypes(mlir::Type from, mlir::Type to) {
+    if (from == to) {
+      return Conversion::None;
+    }
+    if (auto fromIntTy{from.dyn_cast<mlir::IntegerType>()}) {
+      if (auto toIntTy{to.dyn_cast<mlir::IntegerType>()}) {
+        return fromIntTy.getWidth() > toIntTy.getWidth() ? Conversion::Narrow
+                                                         : Conversion::Extend;
+      }
+    }
+    if (fir::isa_real(from) && fir::isa_real(to)) {
+      return getFloatingPointWidth(from) > getFloatingPointWidth(to)
+                 ? Conversion::Narrow
+                 : Conversion::Extend;
+    }
+    if (auto fromCplxTy{from.dyn_cast<fir::CplxType>()}) {
+      if (auto toCplxTy{to.dyn_cast<fir::CplxType>()}) {
+        return getFloatingPointWidth(fromCplxTy) >
+                       getFloatingPointWidth(toCplxTy)
+                   ? Conversion::Narrow
+                   : Conversion::Extend;
+      }
+    }
+    // Notes:
+    // - No conversion between character types, specialization of runtime
+    // functions should be made instead.
+    // - It is not clear there is a use case for automatic conversions
+    // around Logical and it may damage hidden information in the physical
+    // storage so do not do it.
+    return Conversion::Forbidden;
+  }
+
+  // Below are indexes to access data in conversions.
+  // The order in data does matter for lexicographical_compare
+  enum {
+    narrowingArg = 0,   // usually bad
+    extendingResult,    // usually bad
+    nonExtendingResult, // usually ok
+    nonNarrowingArg,    // usually ok
+    dataSize
+  };
+
+  std::array<int, dataSize> conversions{/* zero init*/};
+  bool infinite{false}; // When forbidden conversion or wrong argument number
+};
+
+/// Build mlir::FuncOp from runtime symbol description and add
+/// fir.runtime attribute.
+static mlir::FuncOp getFuncOp(mlir::Location loc,
+                              Fortran::lower::FirOpBuilder &builder,
+                              const RuntimeFunction &runtime) {
+  auto function = builder.addNamedFunction(
+      loc, runtime.symbol, runtime.typeGenerator(builder.getContext()));
+  function.setAttr("fir.runtime", builder.getUnitAttr());
+  return function;
+}
+
+/// Select runtime function that has the smallest distance to the intrinsic
+/// function type and that will not imply narrowing arguments or extending the
+/// result.
+/// If nothing is found, the mlir::FuncOp will contain a nullptr.
+template <std::size_t N>
+mlir::FuncOp searchFunctionInLibrary(mlir::Location loc,
+                                     Fortran::lower::FirOpBuilder &builder,
+                                     const RuntimeFunction (&lib)[N],
+                                     llvm::StringRef name,
+                                     mlir::FunctionType funcType,
+                                     const RuntimeFunction **bestNearMatch,
+                                     FunctionDistance &bestMatchDistance) {
+  auto map = StaticMultimapView(lib);
+  auto range = map.equal_range(name);
+  for (auto iter{range.first}; iter != range.second && iter; ++iter) {
+    const auto &impl = *iter;
+    auto implType = impl.typeGenerator(builder.getContext());
+    if (funcType == implType) {
+      return getFuncOp(loc, builder, impl); // exact match
+    } else {
+      FunctionDistance distance(funcType, implType);
+      if (distance.isSmallerThan(bestMatchDistance)) {
+        *bestNearMatch = &impl;
+        bestMatchDistance = std::move(distance);
+      }
+    }
+  }
+  return {};
+}
+
+/// Search runtime for the best runtime function given an intrinsic name
+/// and interface. The interface may not be a perfect match in which case
+/// the caller is responsible to insert argument and return value conversions.
+/// If nothing is found, the mlir::FuncOp will contain a nullptr.
+static mlir::FuncOp getRuntimeFunction(mlir::Location loc,
+                                       Fortran::lower::FirOpBuilder &builder,
+                                       llvm::StringRef name,
+                                       mlir::FunctionType funcType) {
+  const RuntimeFunction *bestNearMatch = nullptr;
+  FunctionDistance bestMatchDistance{};
+  mlir::FuncOp match;
+  if (mathRuntimeVersion == fastVersion) {
+    match = searchFunctionInLibrary(loc, builder, pgmathFast, name, funcType,
+                                    &bestNearMatch, bestMatchDistance);
+  } else if (mathRuntimeVersion == relaxedVersion) {
+    match = searchFunctionInLibrary(loc, builder, pgmathRelaxed, name, funcType,
+                                    &bestNearMatch, bestMatchDistance);
+  } else if (mathRuntimeVersion == preciseVersion) {
+    match = searchFunctionInLibrary(loc, builder, pgmathPrecise, name, funcType,
+                                    &bestNearMatch, bestMatchDistance);
+  } else {
+    assert(mathRuntimeVersion == llvmOnly && "unknown math runtime");
+  }
+  if (match)
+    return match;
+
+  // Go through llvm intrinsics if not exact match in libpgmath or if
+  // mathRuntimeVersion == llvmOnly
+  if (auto exactMatch =
+          searchFunctionInLibrary(loc, builder, llvmIntrinsics, name, funcType,
+                                  &bestNearMatch, bestMatchDistance))
+    return exactMatch;
+
+  if (bestNearMatch != nullptr) {
+    assert(!bestMatchDistance.isLosingPrecision() &&
+           "runtime selection loses precision");
+    return getFuncOp(loc, builder, *bestNearMatch);
+  }
+  return {};
+}
+
+/// Helpers to get function type from arguments and result type.
+static mlir::FunctionType
+getFunctionType(mlir::Type resultType, llvm::ArrayRef<mlir::Value> arguments,
+                Fortran::lower::FirOpBuilder &builder) {
+  llvm::SmallVector<mlir::Type, 2> argumentTypes;
+  for (auto &arg : arguments)
+    argumentTypes.push_back(arg.getType());
+  return mlir::FunctionType::get(argumentTypes, resultType,
+                                 builder.getModule().getContext());
+}
+
+/// fir::ExtendedValue to mlir::Value translation layer
+
+fir::ExtendedValue toExtendedValue(mlir::Value val,
+                                   Fortran::lower::FirOpBuilder &builder,
+                                   mlir::Location loc) {
+  assert(val && "optional unhandled here");
+  auto type = val.getType();
+  auto base = val;
+  auto indexType = builder.getIndexType();
+  llvm::SmallVector<mlir::Value, 2> extents;
+
+  Fortran::lower::CharacterExprHelper charHelper{builder, loc};
+  if (charHelper.isCharacter(type))
+    return charHelper.toExtendedValue(val);
+
+  if (auto refType = type.dyn_cast<fir::ReferenceType>())
+    type = refType.getEleTy();
+
+  if (auto arrayType = type.dyn_cast<fir::SequenceType>()) {
+    type = arrayType.getEleTy();
+    for (auto extent : arrayType.getShape()) {
+      if (extent == fir::SequenceType::getUnknownExtent())
+        break;
+      extents.emplace_back(
+          builder.createIntegerConstant(loc, indexType, extent));
+    }
+    // Last extent might be missing in case of assumed-size. If more extents
+    // could not be deduced from type, that's an error (a fir.box should
+    // have been used in the interface).
+    if (extents.size() + 1 < arrayType.getShape().size())
+      mlir::emitError(loc, "cannot retrieve array extents from type");
+  } else if (type.isa<fir::BoxType>() || type.isa<fir::RecordType>()) {
+    mlir::emitError(loc, "descriptor or derived type not yet handled");
+  }
+
+  if (!extents.empty())
+    return fir::ArrayBoxValue{base, extents};
+  return base;
+}
+
+mlir::Value toValue(const fir::ExtendedValue &val,
+                    Fortran::lower::FirOpBuilder &builder, mlir::Location loc) {
+  if (auto charBox = val.getCharBox()) {
+    auto buffer = charBox->getBuffer();
+    if (buffer.getType().isa<fir::BoxCharType>())
+      return buffer;
+    return Fortran::lower::CharacterExprHelper{builder, loc}.createEmboxChar(
+        buffer, charBox->getLen());
+  }
+
+  // FIXME: need to access other ExtendedValue variants and handle them
+  // properly.
+  return fir::getBase(val);
+}
+
+//===----------------------------------------------------------------------===//
+// IntrinsicLibrary
+//===----------------------------------------------------------------------===//
+
+template <typename GeneratorType>
+fir::ExtendedValue IntrinsicLibrary::genElementalCall(
+    GeneratorType generator, llvm::StringRef name, mlir::Type resultType,
+    llvm::ArrayRef<fir::ExtendedValue> args, bool outline) {
+  llvm::SmallVector<mlir::Value, 2> scalarArgs;
+  for (const auto &arg : args) {
+    if (arg.getUnboxed() || arg.getCharBox()) {
+      scalarArgs.emplace_back(fir::getBase(arg));
+    } else {
+      // TODO: get the result shape and create the loop...
+      mlir::emitError(loc, "array or descriptor not yet handled in elemental "
+                           "intrinsic lowering");
+      exit(1);
+    }
+  }
+  if (outline)
+    return outlineInWrapper(generator, name, resultType, scalarArgs);
+  return invokeGenerator(generator, resultType, scalarArgs);
+}
+
+/// Some ExtendedGenerator operating on characters are also elemental
+/// (e.g LEN_TRIM).
+template <>
+fir::ExtendedValue
+IntrinsicLibrary::genElementalCall<IntrinsicLibrary::ExtendedGenerator>(
+    ExtendedGenerator generator, llvm::StringRef name, mlir::Type resultType,
+    llvm::ArrayRef<fir::ExtendedValue> args, bool outline) {
+  for (const auto &arg : args)
+    if (!arg.getUnboxed() && !arg.getCharBox()) {
+      // TODO: get the result shape and create the loop...
+      mlir::emitError(loc, "array or descriptor not yet handled in elemental "
+                           "intrinsic lowering");
+      exit(1);
+    }
+  if (outline)
+    return outlineInWrapper(generator, name, resultType, args);
+  return std::invoke(generator, *this, resultType, args);
+}
+
+fir::ExtendedValue
+IntrinsicLibrary::genIntrinsicCall(llvm::StringRef name, mlir::Type resultType,
+                                   llvm::ArrayRef<fir::ExtendedValue> args) {
+  for (auto &handler : handlers)
+    if (name == handler.name) {
+      bool outline = handler.outline || outlineAllIntrinsics;
+      if (const auto *elementalGenerator =
+              std::get_if<ElementalGenerator>(&handler.generator))
+        return genElementalCall(*elementalGenerator, name, resultType, args,
+                                outline);
+      const auto &generator = std::get<ExtendedGenerator>(handler.generator);
+      if (handler.isElemental)
+        return genElementalCall(generator, name, resultType, args, outline);
+      if (outline)
+        return outlineInWrapper(generator, name, resultType, args);
+      return std::invoke(generator, *this, resultType, args);
+    }
+
+  // Try the runtime if no special handler was defined for the
+  // intrinsic being called. Maths runtime only has numerical elemental.
+  // No optional arguments are expected at this point, the code will
+  // crash if it gets absent optional.
+
+  // FIXME: using toValue to get the type won't work with array arguments.
+  llvm::SmallVector<mlir::Value, 2> mlirArgs;
+  for (const auto &extendedVal : args) {
+    auto val = toValue(extendedVal, builder, loc);
+    if (!val) {
+      // If an absent optional gets there, most likely its handler has just
+      // not yet been defined.
+      mlir::emitError(loc,
+                      "TODO: missing intrinsic lowering: " + llvm::Twine(name));
+      exit(1);
+    }
+    mlirArgs.emplace_back(val);
+  }
+  mlir::FunctionType soughtFuncType =
+      getFunctionType(resultType, mlirArgs, builder);
+
+  auto runtimeCallGenerator = getRuntimeCallGenerator(name, soughtFuncType);
+  return genElementalCall(runtimeCallGenerator, name, resultType, args,
+                          /* outline */ true);
+}
+
+mlir::Value
+IntrinsicLibrary::invokeGenerator(ElementalGenerator generator,
+                                  mlir::Type resultType,
+                                  llvm::ArrayRef<mlir::Value> args) {
+  return std::invoke(generator, *this, resultType, args);
+}
+
+mlir::Value
+IntrinsicLibrary::invokeGenerator(RuntimeCallGenerator generator,
+                                  mlir::Type resultType,
+                                  llvm::ArrayRef<mlir::Value> args) {
+  return generator(builder, loc, args);
+}
+
+mlir::Value
+IntrinsicLibrary::invokeGenerator(ExtendedGenerator generator,
+                                  mlir::Type resultType,
+                                  llvm::ArrayRef<mlir::Value> args) {
+  llvm::SmallVector<fir::ExtendedValue, 2> extendedArgs;
+  for (auto arg : args)
+    extendedArgs.emplace_back(toExtendedValue(arg, builder, loc));
+  auto extendedResult = std::invoke(generator, *this, resultType, extendedArgs);
+  return toValue(extendedResult, builder, loc);
+}
+
+template <typename GeneratorType>
+mlir::FuncOp IntrinsicLibrary::getWrapper(GeneratorType generator,
+                                          llvm::StringRef name,
+                                          mlir::FunctionType funcType,
+                                          bool loadRefArguments) {
+  assert(funcType.getNumResults() == 1 &&
+         "expect one result for intrinsic functions");
+  auto resultType = funcType.getResult(0);
+  std::string wrapperName = fir::mangleIntrinsicProcedure(name, funcType);
+  auto function = builder.getNamedFunction(wrapperName);
+  if (!function) {
+    // First time this wrapper is needed, build it.
+    function = builder.createFunction(loc, wrapperName, funcType);
+    function.setAttr("fir.intrinsic", builder.getUnitAttr());
+    function.addEntryBlock();
+
+    // Create local context to emit code into the newly created function
+    // This new function is not linked to a source file location, only
+    // its calls will be.
+    auto localBuilder = std::make_unique<Fortran::lower::FirOpBuilder>(
+        function, builder.getKindMap());
+    localBuilder->setInsertionPointToStart(&function.front());
+    // Location of code inside wrapper of the wrapper is independent from
+    // the location of the intrinsic call.
+    auto localLoc = localBuilder->getUnknownLoc();
+    llvm::SmallVector<mlir::Value, 2> localArguments;
+    for (mlir::BlockArgument bArg : function.front().getArguments()) {
+      auto refType = bArg.getType().dyn_cast<fir::ReferenceType>();
+      if (loadRefArguments && refType) {
+        auto loaded = localBuilder->create<fir::LoadOp>(localLoc, bArg);
+        localArguments.push_back(loaded);
+      } else {
+        localArguments.push_back(bArg);
+      }
+    }
+
+    IntrinsicLibrary localLib{*localBuilder, localLoc};
+    auto result =
+        localLib.invokeGenerator(generator, resultType, localArguments);
+    localBuilder->create<mlir::ReturnOp>(localLoc, result);
+  } else {
+    // Wrapper was already built, ensure it has the sought type
+    assert(function.getType() == funcType &&
+           "conflict between intrinsic wrapper types");
+  }
+  return function;
+}
+
+/// Helpers to detect absent optional (not yet supported in outlining).
+bool static hasAbsentOptional(llvm::ArrayRef<mlir::Value> args) {
+  for (const auto &arg : args)
+    if (!arg)
+      return true;
+  return false;
+}
+bool static hasAbsentOptional(llvm::ArrayRef<fir::ExtendedValue> args) {
+  for (const auto &arg : args)
+    if (!fir::getBase(arg))
+      return true;
+  return false;
+}
+
+template <typename GeneratorType>
+mlir::Value
+IntrinsicLibrary::outlineInWrapper(GeneratorType generator,
+                                   llvm::StringRef name, mlir::Type resultType,
+                                   llvm::ArrayRef<mlir::Value> args) {
+  if (hasAbsentOptional(args)) {
+    // TODO: absent optional in outlining is an issue: we cannot just ignore
+    // them. Needs a better interface here. The issue is that we cannot easily
+    // tell that a value is optional or not here if it is presents. And if it is
+    // absent, we cannot tell what it type should be.
+    mlir::emitError(loc, "todo: cannot outline call to intrinsic " +
+                             llvm::Twine(name) +
+                             " with absent optional argument");
+    exit(1);
+  }
+
+  auto funcType = getFunctionType(resultType, args, builder);
+  auto wrapper = getWrapper(generator, name, funcType);
+  return builder.create<mlir::CallOp>(loc, wrapper, args).getResult(0);
+}
+
+fir::ExtendedValue
+IntrinsicLibrary::outlineInWrapper(ExtendedGenerator generator,
+                                   llvm::StringRef name, mlir::Type resultType,
+                                   llvm::ArrayRef<fir::ExtendedValue> args) {
+  if (hasAbsentOptional(args)) {
+    // TODO
+    mlir::emitError(loc, "todo: cannot outline call to intrinsic " +
+                             llvm::Twine(name) +
+                             " with absent optional argument");
+    exit(1);
+  }
+  llvm::SmallVector<mlir::Value, 2> mlirArgs;
+  for (const auto &extendedVal : args)
+    mlirArgs.emplace_back(toValue(extendedVal, builder, loc));
+  auto funcType = getFunctionType(resultType, mlirArgs, builder);
+  auto wrapper = getWrapper(generator, name, funcType);
+  auto mlirResult =
+      builder.create<mlir::CallOp>(loc, wrapper, mlirArgs).getResult(0);
+  return toExtendedValue(mlirResult, builder, loc);
+}
+
+IntrinsicLibrary::RuntimeCallGenerator
+IntrinsicLibrary::getRuntimeCallGenerator(llvm::StringRef name,
+                                          mlir::FunctionType soughtFuncType) {
+  auto funcOp = getRuntimeFunction(loc, builder, name, soughtFuncType);
+  if (!funcOp) {
+    mlir::emitError(loc,
+                    "TODO: missing intrinsic lowering: " + llvm::Twine(name));
+    llvm::errs() << "requested type was: " << soughtFuncType << "\n";
+    exit(1);
+  }
+
+  mlir::FunctionType actualFuncType = funcOp.getType();
+  assert(actualFuncType.getNumResults() == soughtFuncType.getNumResults() &&
+         actualFuncType.getNumInputs() == soughtFuncType.getNumInputs() &&
+         actualFuncType.getNumResults() == 1 && "Bad intrinsic match");
+
+  return [funcOp, actualFuncType, soughtFuncType](
+             Fortran::lower::FirOpBuilder &builder, mlir::Location loc,
+             llvm::ArrayRef<mlir::Value> args) {
+    llvm::SmallVector<mlir::Value, 2> convertedArguments;
+    for (const auto &pair : llvm::zip(actualFuncType.getInputs(), args))
+      convertedArguments.push_back(
+          builder.createConvert(loc, std::get<0>(pair), std::get<1>(pair)));
+    auto call = builder.create<mlir::CallOp>(loc, funcOp, convertedArguments);
+    mlir::Type soughtType = soughtFuncType.getResult(0);
+    return builder.createConvert(loc, soughtType, call.getResult(0));
+  };
+}
+
+mlir::SymbolRefAttr IntrinsicLibrary::getUnrestrictedIntrinsicSymbolRefAttr(
+    llvm::StringRef name, mlir::FunctionType signature) {
+  // Unrestricted intrinsics signature follows implicit rules: argument
+  // are passed by references. But the runtime versions expect values.
+  // So instead of duplicating the runtime, just have the wrappers loading
+  // this before calling the code generators.
+  bool loadRefArguments = true;
+  mlir::FuncOp funcOp;
+  for (auto &handler : handlers)
+    if (name == handler.name)
+      funcOp = std::visit(
+          [&](auto generator) {
+            return getWrapper(generator, name, signature, loadRefArguments);
+          },
+          handler.generator);
+
+  if (!funcOp) {
+    llvm::SmallVector<mlir::Type, 2> argTypes;
+    for (auto type : signature.getInputs()) {
+      if (auto refType = type.dyn_cast<fir::ReferenceType>())
+        argTypes.push_back(refType.getEleTy());
+      else
+        argTypes.push_back(type);
+    }
+    auto soughtFuncType =
+        builder.getFunctionType(signature.getResults(), argTypes);
+    auto rtCallGenerator = getRuntimeCallGenerator(name, soughtFuncType);
+    funcOp = getWrapper(rtCallGenerator, name, signature, loadRefArguments);
+  }
+
+  return builder.getSymbolRefAttr(funcOp.getName());
+}
+
+//===----------------------------------------------------------------------===//
+// Code generators for the intrinsic
+//===----------------------------------------------------------------------===//
+
+mlir::Value IntrinsicLibrary::genRuntimeCall(llvm::StringRef name,
+                                             mlir::Type resultType,
+                                             llvm::ArrayRef<mlir::Value> args) {
+  mlir::FunctionType soughtFuncType =
+      getFunctionType(resultType, args, builder);
+  return getRuntimeCallGenerator(name, soughtFuncType)(builder, loc, args);
+}
+
+mlir::Value IntrinsicLibrary::genConversion(mlir::Type resultType,
+                                            llvm::ArrayRef<mlir::Value> args) {
+  // There can be an optional kind in second argument.
+  assert(args.size() >= 1);
+  return builder.convertWithSemantics(loc, resultType, args[0]);
+}
+
+// ABS
+mlir::Value IntrinsicLibrary::genAbs(mlir::Type resultType,
+                                     llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  auto arg = args[0];
+  auto type = arg.getType();
+  if (fir::isa_real(type)) {
+    // Runtime call to fp abs. An alternative would be to use mlir AbsFOp
+    // but it does not support all fir floating point types.
+    return genRuntimeCall("abs", resultType, args);
+  }
+  if (auto intType = type.dyn_cast<mlir::IntegerType>()) {
+    // At the time of this implementation there is no abs op in mlir.
+    // So, implement abs here without branching.
+    auto shift =
+        builder.createIntegerConstant(loc, intType, intType.getWidth() - 1);
+    auto mask = builder.create<mlir::SignedShiftRightOp>(loc, arg, shift);
+    auto xored = builder.create<mlir::XOrOp>(loc, arg, mask);
+    return builder.create<mlir::SubIOp>(loc, xored, mask);
+  }
+  if (fir::isa_complex(type)) {
+    // Use HYPOT to fulfill the no underflow/overflow requirement.
+    auto parts =
+        Fortran::lower::ComplexExprHelper{builder, loc}.extractParts(arg);
+    llvm::SmallVector<mlir::Value, 2> args = {parts.first, parts.second};
+    return genRuntimeCall("hypot", resultType, args);
+  }
+  llvm_unreachable("unexpected type in ABS argument");
+}
+
+// AIMAG
+mlir::Value IntrinsicLibrary::genAimag(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  return Fortran::lower::ComplexExprHelper{builder, loc}.extractComplexPart(
+      args[0], true /* isImagPart */);
+}
+
+// ANINT
+mlir::Value IntrinsicLibrary::genAnint(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() >= 1);
+  // Skip optional kind argument to search the runtime; it is already reflected
+  // in result type.
+  return genRuntimeCall("anint", resultType, {args[0]});
+}
+
+// AINT
+mlir::Value IntrinsicLibrary::genAint(mlir::Type resultType,
+                                      llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() >= 1);
+  // Skip optional kind argument to search the runtime; it is already reflected
+  // in result type.
+  return genRuntimeCall("aint", resultType, {args[0]});
+}
+
+// CEILING
+mlir::Value IntrinsicLibrary::genCeiling(mlir::Type resultType,
+                                         llvm::ArrayRef<mlir::Value> args) {
+  // Optional KIND argument.
+  assert(args.size() >= 1);
+  auto arg = args[0];
+  // Use ceil that is not an actual Fortran intrinsic but that is
+  // an llvm intrinsic that does the same, but return a floating
+  // point.
+  auto ceil = genRuntimeCall("ceil", arg.getType(), {arg});
+  return builder.createConvert(loc, resultType, ceil);
+}
+
+// CONJG
+mlir::Value IntrinsicLibrary::genConjg(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 1);
+  if (resultType != args[0].getType())
+    llvm_unreachable("argument type mismatch");
+
+  mlir::Value cplx = args[0];
+  auto imag =
+      Fortran::lower::ComplexExprHelper{builder, loc}.extractComplexPart(
+          cplx, /*isImagPart=*/true);
+  auto negImag = builder.create<fir::NegfOp>(loc, imag);
+  return Fortran::lower::ComplexExprHelper{builder, loc}.insertComplexPart(
+      cplx, negImag, /*isImagPart=*/true);
+}
+
+// DIM
+mlir::Value IntrinsicLibrary::genDim(mlir::Type resultType,
+                                     llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 2);
+  if (resultType.isa<mlir::IntegerType>()) {
+    auto zero = builder.createIntegerConstant(loc, resultType, 0);
+    auto diff = builder.create<mlir::SubIOp>(loc, args[0], args[1]);
+    auto cmp =
+        builder.create<mlir::CmpIOp>(loc, mlir::CmpIPredicate::sgt, diff, zero);
+    return builder.create<mlir::SelectOp>(loc, cmp, diff, zero);
+  }
+  assert(fir::isa_real(resultType) && "Only expects real and integer in DIM");
+  auto zero = builder.createRealZeroConstant(loc, resultType);
+  auto diff = builder.create<fir::SubfOp>(loc, args[0], args[1]);
+  auto cmp =
+      builder.create<fir::CmpfOp>(loc, mlir::CmpFPredicate::OGT, diff, zero);
+  return builder.create<mlir::SelectOp>(loc, cmp, diff, zero);
+}
+
+// DPROD
+mlir::Value IntrinsicLibrary::genDprod(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 2);
+  assert(fir::isa_real(resultType) &&
+         "Result must be double precision in DPROD");
+  auto a = builder.createConvert(loc, resultType, args[0]);
+  auto b = builder.createConvert(loc, resultType, args[1]);
+  return builder.create<fir::MulfOp>(loc, a, b);
+}
+
+// FLOOR
+mlir::Value IntrinsicLibrary::genFloor(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  // Optional KIND argument.
+  assert(args.size() >= 1);
+  auto arg = args[0];
+  // Use LLVM floor that returns real.
+  auto floor = genRuntimeCall("floor", arg.getType(), {arg});
+  return builder.createConvert(loc, resultType, floor);
+}
+
+// IAND
+mlir::Value IntrinsicLibrary::genIAnd(mlir::Type resultType,
+                                      llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 2);
+
+  return builder.create<mlir::AndOp>(loc, args[0], args[1]);
+}
+
+// ICHAR
+mlir::Value IntrinsicLibrary::genIchar(mlir::Type resultType,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  // There can be an optional kind in second argument.
+  assert(args.size() >= 1);
+
+  auto arg = args[0];
+  Fortran::lower::CharacterExprHelper helper{builder, loc};
+  auto dataAndLen = helper.createUnboxChar(arg);
+  auto charType = fir::CharacterType::get(
+      builder.getContext(), helper.getCharacterKind(arg.getType()));
+  auto refType = builder.getRefType(charType);
+  auto charAddr = builder.createConvert(loc, refType, dataAndLen.first);
+  auto charVal = builder.create<fir::LoadOp>(loc, charType, charAddr);
+  return builder.createConvert(loc, resultType, charVal);
+}
+
+// IEOR
+mlir::Value IntrinsicLibrary::genIEOr(mlir::Type resultType,
+                                      llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 2);
+  return builder.create<mlir::XOrOp>(loc, args[0], args[1]);
+}
+
+// IOR
+mlir::Value IntrinsicLibrary::genIOr(mlir::Type resultType,
+                                     llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 2);
+  return builder.create<mlir::OrOp>(loc, args[0], args[1]);
+}
+
+// LEN
+// Note that this is only used for unrestricted intrinsic.
+// Usage of LEN are otherwise rewritten as descriptor inquiries by the
+// front-end.
+fir::ExtendedValue
+IntrinsicLibrary::genLen(mlir::Type resultType,
+                         llvm::ArrayRef<fir::ExtendedValue> args) {
+  // Optional KIND argument reflected in result type.
+  assert(args.size() >= 1);
+  mlir::Value len;
+  if (const auto *charBox = args[0].getCharBox()) {
+    len = charBox->getLen();
+  } else if (const auto *charBoxArray = args[0].getCharBox()) {
+    len = charBoxArray->getLen();
+  } else {
+    Fortran::lower::CharacterExprHelper helper{builder, loc};
+    len = helper.createUnboxChar(fir::getBase(args[0])).second;
+  }
+
+  return builder.createConvert(loc, resultType, len);
+}
+
+// LEN_TRIM
+fir::ExtendedValue
+IntrinsicLibrary::genLenTrim(mlir::Type resultType,
+                             llvm::ArrayRef<fir::ExtendedValue> args) {
+  // Optional KIND argument reflected in result type.
+  assert(args.size() >= 1);
+  Fortran::lower::CharacterExprHelper helper{builder, loc};
+  auto len = helper.createLenTrim(fir::getBase(args[0]));
+  return builder.createConvert(loc, resultType, len);
+}
+
+// MERGE
+mlir::Value IntrinsicLibrary::genMerge(mlir::Type,
+                                       llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 3);
+
+  auto i1Type = mlir::IntegerType::get(1, builder.getContext());
+  auto mask = builder.createConvert(loc, i1Type, args[2]);
+  return builder.create<mlir::SelectOp>(loc, mask, args[0], args[1]);
+}
+
+// MOD
+mlir::Value IntrinsicLibrary::genMod(mlir::Type resultType,
+                                     llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 2);
+  if (resultType.isa<mlir::IntegerType>())
+    return builder.create<mlir::SignedRemIOp>(loc, args[0], args[1]);
+
+  // Use runtime. Note that mlir::RemFOp implements floating point
+  // remainder, but it does not work with fir::Real type.
+  // TODO: consider using mlir::RemFOp when possible, that may help folding
+  // and  optimizations.
+  return genRuntimeCall("mod", resultType, args);
+}
+
+// NINT
+mlir::Value IntrinsicLibrary::genNint(mlir::Type resultType,
+                                      llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() >= 1);
+  // Skip optional kind argument to search the runtime; it is already reflected
+  // in result type.
+  return genRuntimeCall("nint", resultType, {args[0]});
+}
+
+// SIGN
+mlir::Value IntrinsicLibrary::genSign(mlir::Type resultType,
+                                      llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 2);
+  auto abs = genAbs(resultType, {args[0]});
+  if (resultType.isa<mlir::IntegerType>()) {
+    auto zero = builder.createIntegerConstant(loc, resultType, 0);
+    auto neg = builder.create<mlir::SubIOp>(loc, zero, abs);
+    auto cmp = builder.create<mlir::CmpIOp>(loc, mlir::CmpIPredicate::slt,
+                                            args[1], zero);
+    return builder.create<mlir::SelectOp>(loc, cmp, neg, abs);
+  }
+  // TODO: Requirements when second argument is +0./0.
+  auto zeroAttr = builder.getZeroAttr(resultType);
+  auto zero = builder.create<mlir::ConstantOp>(loc, resultType, zeroAttr);
+  auto neg = builder.create<fir::NegfOp>(loc, abs);
+  auto cmp =
+      builder.create<fir::CmpfOp>(loc, mlir::CmpFPredicate::OLT, args[1], zero);
+  return builder.create<mlir::SelectOp>(loc, cmp, neg, abs);
+}
+
+// Compare two FIR values and return boolean result as i1.
+template <Extremum extremum, ExtremumBehavior behavior>
+static mlir::Value createExtremumCompare(mlir::Location loc,
+                                         Fortran::lower::FirOpBuilder &builder,
+                                         mlir::Value left, mlir::Value right) {
+  static constexpr auto integerPredicate = extremum == Extremum::Max
+                                               ? mlir::CmpIPredicate::sgt
+                                               : mlir::CmpIPredicate::slt;
+  static constexpr auto orderedCmp = extremum == Extremum::Max
+                                         ? mlir::CmpFPredicate::OGT
+                                         : mlir::CmpFPredicate::OLT;
+  auto type = left.getType();
+  mlir::Value result;
+  if (fir::isa_real(type)) {
+    // Note: the signaling/quit aspect of the result required by IEEE
+    // cannot currently be obtained with LLVM without ad-hoc runtime.
+    if constexpr (behavior == ExtremumBehavior::IeeeMinMaximumNumber) {
+      // Return the number if one of the inputs is NaN and the other is
+      // a number.
+      auto leftIsResult =
+          builder.create<fir::CmpfOp>(loc, orderedCmp, left, right);
+      auto rightIsNan = builder.create<fir::CmpfOp>(
+          loc, mlir::CmpFPredicate::UNE, right, right);
+      result = builder.create<mlir::OrOp>(loc, leftIsResult, rightIsNan);
+    } else if constexpr (behavior == ExtremumBehavior::IeeeMinMaximum) {
+      // Always return NaNs if one the input is NaNs
+      auto leftIsResult =
+          builder.create<fir::CmpfOp>(loc, orderedCmp, left, right);
+      auto leftIsNan = builder.create<fir::CmpfOp>(
+          loc, mlir::CmpFPredicate::UNE, left, left);
+      result = builder.create<mlir::OrOp>(loc, leftIsResult, leftIsNan);
+    } else if constexpr (behavior == ExtremumBehavior::MinMaxss) {
+      // If the left is a NaN, return the right whatever it is.
+      result = builder.create<fir::CmpfOp>(loc, orderedCmp, left, right);
+    } else if constexpr (behavior == ExtremumBehavior::PgfortranLlvm) {
+      // If one of the operand is a NaN, return left whatever it is.
+      static constexpr auto unorderedCmp = extremum == Extremum::Max
+                                               ? mlir::CmpFPredicate::UGT
+                                               : mlir::CmpFPredicate::ULT;
+      result = builder.create<fir::CmpfOp>(loc, unorderedCmp, left, right);
+    } else {
+      // TODO: ieeeMinNum/ieeeMaxNum
+      static_assert(behavior == ExtremumBehavior::IeeeMinMaxNum,
+                    "ieeeMinNum/ieeeMaxNum behavior not implemented");
+    }
+  } else if (fir::isa_integer(type)) {
+    result = builder.create<mlir::CmpIOp>(loc, integerPredicate, left, right);
+  } else if (type.isa<fir::CharacterType>()) {
+    // TODO: ! character min and max is tricky because the result
+    // length is the length of the longest argument!
+    // So we may need a temp.
+  }
+  assert(result);
+  return result;
+}
+
+// MIN and MAX
+template <Extremum extremum, ExtremumBehavior behavior>
+mlir::Value IntrinsicLibrary::genExtremum(mlir::Type,
+                                          llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() >= 1);
+  mlir::Value result = args[0];
+  for (auto arg : args.drop_front()) {
+    auto mask =
+        createExtremumCompare<extremum, behavior>(loc, builder, result, arg);
+    result = builder.create<mlir::SelectOp>(loc, mask, result, arg);
+  }
+  return result;
+}
+
+//===----------------------------------------------------------------------===//
+// Public intrinsic call helpers
+//===----------------------------------------------------------------------===//
+
+fir::ExtendedValue
+Fortran::lower::genIntrinsicCall(Fortran::lower::FirOpBuilder &builder,
+                                 mlir::Location loc, llvm::StringRef name,
+                                 mlir::Type resultType,
+                                 llvm::ArrayRef<fir::ExtendedValue> args) {
+  return IntrinsicLibrary{builder, loc}.genIntrinsicCall(name, resultType,
+                                                         args);
+}
+
+mlir::Value Fortran::lower::genMax(Fortran::lower::FirOpBuilder &builder,
+                                   mlir::Location loc,
+                                   llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() > 0 && "max requires at least one argument");
+  return IntrinsicLibrary{builder, loc}
+      .genExtremum<Extremum::Max, ExtremumBehavior::MinMaxss>(args[0].getType(),
+                                                              args);
+}
+
+mlir::Value Fortran::lower::genMin(Fortran::lower::FirOpBuilder &builder,
+                                   mlir::Location loc,
+                                   llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() > 0 && "min requires at least one argument");
+  return IntrinsicLibrary{builder, loc}
+      .genExtremum<Extremum::Min, ExtremumBehavior::MinMaxss>(args[0].getType(),
+                                                              args);
+}
+
+mlir::Value Fortran::lower::genPow(Fortran::lower::FirOpBuilder &builder,
+                                   mlir::Location loc, mlir::Type type,
+                                   mlir::Value x, mlir::Value y) {
+  return IntrinsicLibrary{builder, loc}.genRuntimeCall("pow", type, {x, y});
+}
+
+mlir::SymbolRefAttr Fortran::lower::getUnrestrictedIntrinsicSymbolRefAttr(
+    Fortran::lower::FirOpBuilder &builder, mlir::Location loc,
+    llvm::StringRef name, mlir::FunctionType signature) {
+  return IntrinsicLibrary{builder, loc}.getUnrestrictedIntrinsicSymbolRefAttr(
+      name, signature);
+}
diff --git a/flang/lib/Lower/Mangler.cpp b/flang/lib/Lower/Mangler.cpp
index ad20c9261771b..656a01ca0bd81 100644
--- a/flang/lib/Lower/Mangler.cpp
+++ b/flang/lib/Lower/Mangler.cpp
@@ -9,11 +9,13 @@
 #include "flang/Lower/Mangler.h"
 #include "flang/Common/reference.h"
 #include "flang/Lower/Utils.h"
+#include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/Support/InternalNames.h"
 #include "flang/Semantics/tools.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 
 // recursively build the vector of module scopes
@@ -118,3 +120,49 @@ std::string Fortran::lower::mangle::demangleName(llvm::StringRef name) {
   auto result = fir::NameUniquer::deconstruct(name);
   return result.second.name;
 }
+
+//===----------------------------------------------------------------------===//
+// Intrinsic Procedure Mangling
+//===----------------------------------------------------------------------===//
+
+/// Helper to encode type into string for intrinsic procedure names.
+/// Note: mlir has Type::dump(ostream) methods but it may add "!" that is not
+/// suitable for function names.
+static std::string typeToString(mlir::Type t) {
+  if (auto refT{t.dyn_cast<fir::ReferenceType>()})
+    return "ref_" + typeToString(refT.getEleTy());
+  if (auto i{t.dyn_cast<mlir::IntegerType>()}) {
+    return "i" + std::to_string(i.getWidth());
+  }
+  if (auto cplx{t.dyn_cast<fir::CplxType>()}) {
+    return "z" + std::to_string(cplx.getFKind());
+  }
+  if (auto real{t.dyn_cast<fir::RealType>()}) {
+    return "r" + std::to_string(real.getFKind());
+  }
+  if (auto f{t.dyn_cast<mlir::FloatType>()}) {
+    return "f" + std::to_string(f.getWidth());
+  }
+  if (auto logical{t.dyn_cast<fir::LogicalType>()}) {
+    return "l" + std::to_string(logical.getFKind());
+  }
+  if (auto character{t.dyn_cast<fir::CharacterType>()}) {
+    return "c" + std::to_string(character.getFKind());
+  }
+  if (auto boxCharacter{t.dyn_cast<fir::BoxCharType>()}) {
+    return "bc" + std::to_string(boxCharacter.getEleTy().getFKind());
+  }
+  llvm_unreachable("no mangling for type");
+}
+
+std::string fir::mangleIntrinsicProcedure(llvm::StringRef intrinsic,
+                                          mlir::FunctionType funTy) {
+  std::string name = "fir.";
+  name.append(intrinsic.str()).append(".");
+  assert(funTy.getNumResults() == 1 && "only function mangling supported");
+  name.append(typeToString(funTy.getResult(0)));
+  auto e = funTy.getNumInputs();
+  for (decltype(e) i = 0; i < e; ++i)
+    name.append(".").append(typeToString(funTy.getInput(i)));
+  return name;
+}
diff --git a/flang/lib/Lower/OpenMP.cpp b/flang/lib/Lower/OpenMP.cpp
index ad75eff0adb38..5eb6a1866d293 100644
--- a/flang/lib/Lower/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Lower/OpenMP.h"
-#include "flang/Lower/AbstractConverter.h"
+#include "flang/Lower/Bridge.h"
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Parser/parse-tree.h"
 
diff --git a/flang/lib/Lower/PFTBuilder.cpp b/flang/lib/Lower/PFTBuilder.cpp
index e23370ec9512d..7195086d8e3f2 100644
--- a/flang/lib/Lower/PFTBuilder.cpp
+++ b/flang/lib/Lower/PFTBuilder.cpp
@@ -170,13 +170,24 @@ class PFTBuilder {
 
   void exitModule() {
     parentVariantStack.pop_back();
-    resetFunctionList();
+    resetFunctionState();
+  }
+
+  /// Ensure that a function has a branch target after the last user statement.
+  void endFunctionBody() {
+    if (lastLexicalEvaluation) {
+      static const parser::ContinueStmt endTarget{};
+      addEvaluation(
+          lower::pft::Evaluation{endTarget, parentVariantStack.back(), {}, {}});
+      lastLexicalEvaluation = nullptr;
+    }
   }
 
   /// Initialize a new function-like unit and make it the builder's focus.
   template <typename A>
   bool enterFunction(const A &func,
                      const semantics::SemanticsContext &semanticsContext) {
+    endFunctionBody(); // enclosing host subprogram body, if any
     auto &unit = addFunction(lower::pft::FunctionLikeUnit{
         func, parentVariantStack.back(), semanticsContext});
     labelEvaluationMap = &unit.labelEvaluationMap;
@@ -188,17 +199,13 @@ class PFTBuilder {
   }
 
   void exitFunction() {
-    // Guarantee that there is a branch target after the last user statement.
-    static const parser::ContinueStmt endTarget{};
-    addEvaluation(
-        lower::pft::Evaluation{endTarget, parentVariantStack.back(), {}, {}});
-    lastLexicalEvaluation = nullptr;
+    endFunctionBody();
     analyzeBranches(nullptr, *evaluationListStack.back()); // add branch links
     popEvaluationList();
     labelEvaluationMap = nullptr;
     assignSymbolLabelMap = nullptr;
     parentVariantStack.pop_back();
-    resetFunctionList();
+    resetFunctionState();
   }
 
   /// Initialize a new construct and make it the builder's focus.
@@ -219,12 +226,14 @@ class PFTBuilder {
     constructAndDirectiveStack.pop_back();
   }
 
-  /// Reset functionList to an enclosing function's functionList.
-  void resetFunctionList() {
+  /// Reset function state to that of an enclosing host function.
+  void resetFunctionState() {
     if (!parentVariantStack.empty()) {
       parentVariantStack.back().visit(common::visitors{
           [&](lower::pft::FunctionLikeUnit &p) {
             functionList = &p.nestedFunctions;
+            labelEvaluationMap = &p.labelEvaluationMap;
+            assignSymbolLabelMap = &p.assignSymbolLabelMap;
           },
           [&](lower::pft::ModuleLikeUnit &p) {
             functionList = &p.nestedFunctions;
@@ -346,13 +355,10 @@ class PFTBuilder {
 
   /// Set the exit of a construct, possibly from multiple enclosing constructs.
   void setConstructExit(lower::pft::Evaluation &eval) {
-    eval.constructExit = eval.evaluationList->back().lexicalSuccessor;
-    if (eval.constructExit && eval.constructExit->isNopConstructStmt()) {
-      eval.constructExit = eval.constructExit->parentConstruct->constructExit;
-    }
-    assert(eval.constructExit && "missing construct exit");
+    eval.constructExit = &eval.evaluationList->back().nonNopSuccessor();
   }
 
+  /// Mark the target of a branch as a new block.
   void markBranchTarget(lower::pft::Evaluation &sourceEvaluation,
                         lower::pft::Evaluation &targetEvaluation) {
     sourceEvaluation.isUnstructured = true;
@@ -360,6 +366,22 @@ class PFTBuilder {
       sourceEvaluation.controlSuccessor = &targetEvaluation;
     }
     targetEvaluation.isNewBlock = true;
+    // If this is a branch into the body of a construct (usually illegal,
+    // but allowed in some legacy cases), then the targetEvaluation and its
+    // ancestors must be marked as unstructured.
+    auto *sourceConstruct = sourceEvaluation.parentConstruct;
+    auto *targetConstruct = targetEvaluation.parentConstruct;
+    if (targetEvaluation.isConstructStmt() &&
+        &targetConstruct->getFirstNestedEvaluation() == &targetEvaluation)
+      // A branch to an initial constructStmt is a branch to the construct.
+      targetConstruct = targetConstruct->parentConstruct;
+    if (targetConstruct) {
+      while (sourceConstruct && sourceConstruct != targetConstruct)
+        sourceConstruct = sourceConstruct->parentConstruct;
+      if (sourceConstruct != targetConstruct)
+        for (auto *eval = &targetEvaluation; eval; eval = eval->parentConstruct)
+          eval->isUnstructured = true;
+    }
   }
   void markBranchTarget(lower::pft::Evaluation &sourceEvaluation,
                         parser::Label label) {
@@ -370,20 +392,9 @@ class PFTBuilder {
     markBranchTarget(sourceEvaluation, *targetEvaluation);
   }
 
-  /// Return the first non-nop successor of an evaluation, possibly exiting
-  /// from one or more enclosing constructs.
-  lower::pft::Evaluation *exitSuccessor(lower::pft::Evaluation &eval) {
-    lower::pft::Evaluation *successor{eval.lexicalSuccessor};
-    if (successor && successor->isNopConstructStmt()) {
-      successor = successor->parentConstruct->constructExit;
-    }
-    assert(successor && "missing exit successor");
-    return successor;
-  }
-
-  /// Mark the exit successor of an Evaluation as a new block.
+  /// Mark the successor of an Evaluation as a new block.
   void markSuccessorAsNewBlock(lower::pft::Evaluation &eval) {
-    exitSuccessor(eval)->isNewBlock = true;
+    eval.nonNopSuccessor().isNewBlock = true;
   }
 
   template <typename A>
@@ -521,7 +532,8 @@ class PFTBuilder {
           [&](const parser::AssignedGotoStmt &) {
             // Although this statement is a branch, it doesn't have any
             // explicit control successors.  So the code at the end of the
-            // loop won't mark the exit successor.  Do that here.
+            // loop won't mark the successor.  Do that here.
+            eval.isUnstructured = true;
             markSuccessorAsNewBlock(eval);
           },
 
@@ -542,7 +554,7 @@ class PFTBuilder {
             lastConstructStmtEvaluation = &eval;
           },
           [&](const parser::EndSelectStmt &) {
-            eval.lexicalSuccessor->isNewBlock = true;
+            eval.nonNopSuccessor().isNewBlock = true;
             lastConstructStmtEvaluation = nullptr;
           },
           [&](const parser::ChangeTeamStmt &s) {
@@ -563,7 +575,7 @@ class PFTBuilder {
               eval.isUnstructured = true; // infinite loop
               return;
             }
-            eval.lexicalSuccessor->isNewBlock = true;
+            eval.nonNopSuccessor().isNewBlock = true;
             eval.controlSuccessor = &evaluationList.back();
             if (std::holds_alternative<parser::ScalarLogicalExpr>(control->u)) {
               eval.isUnstructured = true; // while loop
@@ -702,7 +714,7 @@ class PFTBuilder {
           markSuccessorAsNewBlock(eval);
           lastIfStmtEvaluation->isUnstructured = true;
         }
-        lastIfStmtEvaluation->controlSuccessor = exitSuccessor(eval);
+        lastIfStmtEvaluation->controlSuccessor = &eval.nonNopSuccessor();
         lastIfStmtEvaluation = nullptr;
       }
 
@@ -718,7 +730,7 @@ class PFTBuilder {
         parentConstruct->isUnstructured = true;
       }
 
-      // The lexical successor of a branch starts a new block.
+      // The successor of a branch starts a new block.
       if (eval.controlSuccessor && eval.isActionStmt() &&
           eval.lowerAsUnstructured()) {
         markSuccessorAsNewBlock(eval);
@@ -1041,6 +1053,16 @@ struct SymbolDependenceDepth {
 
 void Fortran::lower::pft::FunctionLikeUnit::processSymbolTable(
     const semantics::Scope &scope) {
+  // TODO: handle equivalence and common blocks
+  if (!scope.equivalenceSets().empty()) {
+    llvm::errs() << "TODO: equivalence not yet handled in lowering.\n"
+                 << "note: equivalence used in "
+                 << (scope.GetName() && !scope.GetName()->empty()
+                         ? scope.GetName()->ToString()
+                         : "unnamed program"s)
+                 << "\n";
+    exit(1);
+  }
   SymbolDependenceDepth sdd{varList};
   for (const auto &iter : scope)
     sdd.analyze(iter.second.get());
diff --git a/flang/lib/Lower/RTBuilder.h b/flang/lib/Lower/RTBuilder.h
new file mode 100644
index 0000000000000..2f66fa8efac0f
--- /dev/null
+++ b/flang/lib/Lower/RTBuilder.h
@@ -0,0 +1,268 @@
+//===-- RTBuilder.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file defines some C++17 template classes that are used to convert the
+/// signatures of plain old C functions into a model that can be used to
+/// generate MLIR calls to those functions. This can be used to autogenerate
+/// tables at compiler compile-time to call runtime support code.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_LOWER_RTBUILDER_H
+#define FORTRAN_LOWER_RTBUILDER_H
+
+#include "flang/Lower/ConvertType.h"
+#include "flang/Optimizer/Dialect/FIRType.h"
+#include "mlir/IR/MLIRContext.h"
+#include "mlir/IR/StandardTypes.h"
+#include "llvm/ADT/SmallVector.h"
+#include <functional>
+
+// List the runtime headers we want to be able to dissect
+#include "../../runtime/io-api.h"
+
+namespace Fortran::lower {
+
+using TypeBuilderFunc = mlir::Type (*)(mlir::MLIRContext *);
+using FuncTypeBuilderFunc = mlir::FunctionType (*)(mlir::MLIRContext *);
+
+//===----------------------------------------------------------------------===//
+// Type builder models
+//===----------------------------------------------------------------------===//
+
+/// Return a function that returns the type signature model for the type `T`
+/// when provided an MLIRContext*. This allows one to translate C(++) function
+/// signatures from runtime header files to MLIR signatures into a static table
+/// at compile-time.
+///
+/// For example, when `T` is `int`, return a function that returns the MLIR
+/// standard type `i32` when `sizeof(int)` is 4.
+template <typename T>
+static constexpr TypeBuilderFunc getModel();
+template <>
+constexpr TypeBuilderFunc getModel<int>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    return mlir::IntegerType::get(8 * sizeof(int), context);
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<int &>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    TypeBuilderFunc f{getModel<int>()};
+    return fir::ReferenceType::get(f(context));
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<Fortran::runtime::io::Iostat>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    return mlir::IntegerType::get(8 * sizeof(Fortran::runtime::io::Iostat),
+                                  context);
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<char *>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    return fir::ReferenceType::get(mlir::IntegerType::get(8, context));
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<const char *>() {
+  return getModel<char *>();
+}
+template <>
+constexpr TypeBuilderFunc getModel<const char16_t *>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    return fir::ReferenceType::get(mlir::IntegerType::get(16, context));
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<const char32_t *>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    return fir::ReferenceType::get(mlir::IntegerType::get(32, context));
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<void **>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    return fir::ReferenceType::get(
+        fir::PointerType::get(mlir::IntegerType::get(8, context)));
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<std::int64_t>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    return mlir::IntegerType::get(64, context);
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<std::int64_t &>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    TypeBuilderFunc f{getModel<std::int64_t>()};
+    return fir::ReferenceType::get(f(context));
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<std::size_t>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    return mlir::IntegerType::get(8 * sizeof(std::size_t), context);
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<Fortran::runtime::io::IoStatementState *>() {
+  return getModel<char *>();
+}
+template <>
+constexpr TypeBuilderFunc getModel<double>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    return mlir::FloatType::getF64(context);
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<double &>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    TypeBuilderFunc f{getModel<double>()};
+    return fir::ReferenceType::get(f(context));
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<float>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    return mlir::FloatType::getF32(context);
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<float &>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    TypeBuilderFunc f{getModel<float>()};
+    return fir::ReferenceType::get(f(context));
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<bool>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    return mlir::IntegerType::get(1, context);
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<bool &>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    TypeBuilderFunc f{getModel<bool>()};
+    return fir::ReferenceType::get(f(context));
+  };
+}
+
+template <>
+constexpr TypeBuilderFunc getModel<const Fortran::runtime::Descriptor &>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    return fir::BoxType::get(mlir::NoneType::get(context));
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<const Fortran::runtime::NamelistGroup &>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    // FIXME: a namelist group must be some well-defined data structure, use a
+    // tuple as a proxy for the moment
+    return mlir::TupleType::get(llvm::None, context);
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<void>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    return mlir::NoneType::get(context);
+  };
+}
+
+template <typename...>
+struct RuntimeTableKey;
+template <typename RT, typename... ATs>
+struct RuntimeTableKey<RT(ATs...)> {
+  static constexpr FuncTypeBuilderFunc getTypeModel() {
+    return [](mlir::MLIRContext *ctxt) {
+      TypeBuilderFunc ret = getModel<RT>();
+      std::array<TypeBuilderFunc, sizeof...(ATs)> args = {getModel<ATs>()...};
+      mlir::Type retTy = ret(ctxt);
+      llvm::SmallVector<mlir::Type, sizeof...(ATs)> argTys;
+      for (auto f : args)
+        argTys.push_back(f(ctxt));
+      return mlir::FunctionType::get(argTys, {retTy}, ctxt);
+    };
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// Runtime table building (constexpr folded)
+//===----------------------------------------------------------------------===//
+
+template <char... Cs>
+using RuntimeIdentifier = std::integer_sequence<char, Cs...>;
+
+namespace details {
+template <typename T, T... As, T... Bs>
+static constexpr std::integer_sequence<T, As..., Bs...>
+concat(std::integer_sequence<T, As...>, std::integer_sequence<T, Bs...>) {
+  return {};
+}
+template <typename T, T... As, T... Bs, typename... Cs>
+static constexpr auto concat(std::integer_sequence<T, As...>,
+                             std::integer_sequence<T, Bs...>, Cs...) {
+  return concat(std::integer_sequence<T, As..., Bs...>{}, Cs{}...);
+}
+template <typename T>
+static constexpr std::integer_sequence<T> concat(std::integer_sequence<T>) {
+  return {};
+}
+template <typename T, T a>
+static constexpr auto filterZero(std::integer_sequence<T, a>) {
+  if constexpr (a != 0) {
+    return std::integer_sequence<T, a>{};
+  } else {
+    return std::integer_sequence<T>{};
+  }
+}
+template <typename T, T... b>
+static constexpr auto filter(std::integer_sequence<T, b...>) {
+  if constexpr (sizeof...(b) > 0) {
+    return details::concat(filterZero(std::integer_sequence<T, b>{})...);
+  } else {
+    return std::integer_sequence<T>{};
+  }
+}
+} // namespace details
+
+template <typename...>
+struct RuntimeTableEntry;
+template <typename KT, char... Cs>
+struct RuntimeTableEntry<RuntimeTableKey<KT>, RuntimeIdentifier<Cs...>> {
+  static constexpr FuncTypeBuilderFunc getTypeModel() {
+    return RuntimeTableKey<KT>::getTypeModel();
+  }
+  static constexpr const char name[sizeof...(Cs) + 1] = {Cs..., '\0'};
+};
+
+#undef E
+#define E(L, I) (I < sizeof(L) / sizeof(*L) ? L[I] : 0)
+#define QuoteKey(X) #X
+#define MacroExpandKey(X)                                                      \
+  E(X, 0), E(X, 1), E(X, 2), E(X, 3), E(X, 4), E(X, 5), E(X, 6), E(X, 7),      \
+      E(X, 8), E(X, 9), E(X, 10), E(X, 11), E(X, 12), E(X, 13), E(X, 14),      \
+      E(X, 15), E(X, 16), E(X, 17), E(X, 18), E(X, 19), E(X, 20), E(X, 21),    \
+      E(X, 22), E(X, 23), E(X, 24), E(X, 25), E(X, 26), E(X, 27), E(X, 28),    \
+      E(X, 29), E(X, 30), E(X, 31), E(X, 32), E(X, 33), E(X, 34), E(X, 35),    \
+      E(X, 36), E(X, 37), E(X, 38), E(X, 39), E(X, 40), E(X, 41), E(X, 42),    \
+      E(X, 43), E(X, 44), E(X, 45), E(X, 46), E(X, 47), E(X, 48), E(X, 49)
+#define ExpandKey(X) MacroExpandKey(QuoteKey(X))
+#define FullSeq(X) std::integer_sequence<char, ExpandKey(X)>
+#define AsSequence(X) decltype(Fortran::lower::details::filter(FullSeq(X){}))
+#define mkKey(X)                                                               \
+  Fortran::lower::RuntimeTableEntry<                                           \
+      Fortran::lower::RuntimeTableKey<decltype(X)>, AsSequence(X)>
+
+} // namespace Fortran::lower
+
+#endif // FORTRAN_LOWER_RTBUILDER_H
diff --git a/flang/lib/Lower/SymbolMap.h b/flang/lib/Lower/SymbolMap.h
index a1128d1d43819..3c99febc15178 100644
--- a/flang/lib/Lower/SymbolMap.h
+++ b/flang/lib/Lower/SymbolMap.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Compiler.h"
 
 namespace Fortran::lower {
 
@@ -238,7 +239,7 @@ class SymMap {
   void clear() { symbolMap.clear(); }
 
   /// Dump the map. For debugging.
-  void dump() const;
+  LLVM_DUMP_METHOD void dump() const;
 
 private:
   /// Add `symbol` to the current map and bind a `box`.
diff --git a/flang/lib/Optimizer/.clang-tidy b/flang/lib/Optimizer/.clang-tidy
new file mode 100644
index 0000000000000..87ec2ff53af6e
--- /dev/null
+++ b/flang/lib/Optimizer/.clang-tidy
@@ -0,0 +1,19 @@
+# Almost identical to the top-level .clang-tidy, except that {Member,Parameter,Variable}Case use camelBack.
+Checks: '-*,clang-diagnostic-*,llvm-*,misc-*,-misc-unused-parameters,-misc-non-private-member-variables-in-classes,readability-identifier-naming'
+CheckOptions:
+  - key:             readability-identifier-naming.ClassCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.EnumCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.FunctionCase
+    value:           camelBack
+  - key:             readability-identifier-naming.MemberCase
+    value:           camelBack
+  - key:             readability-identifier-naming.ParameterCase
+    value:           camelBack
+  - key:             readability-identifier-naming.UnionCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.VariableCase
+    value:           camelBack
+  - key:             readability-identifier-naming.IgnoreMainLikeFunctions
+    value:           1
diff --git a/flang/lib/Optimizer/CMakeLists.txt b/flang/lib/Optimizer/CMakeLists.txt
index abeec930ae18c..1375e2c108362 100644
--- a/flang/lib/Optimizer/CMakeLists.txt
+++ b/flang/lib/Optimizer/CMakeLists.txt
@@ -10,8 +10,11 @@ add_flang_library(FIROptimizer
   Support/InternalNames.cpp
   Support/KindMapping.cpp
 
+  Transforms/Inliner.cpp
+
   DEPENDS
   FIROpsIncGen
+  FIROptTransformsPassIncGen
   ${dialect_libs}
 
   LINK_LIBS
diff --git a/flang/lib/Optimizer/Dialect/FIRType.cpp b/flang/lib/Optimizer/Dialect/FIRType.cpp
index 97d169de7e77a..c29412b86944a 100644
--- a/flang/lib/Optimizer/Dialect/FIRType.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRType.cpp
@@ -178,8 +178,10 @@ SequenceType parseSequence(mlir::DialectAsmParser &parser, mlir::Location) {
   return SequenceType::get(shape, eleTy, map);
 }
 
-static bool verifyIntegerType(mlir::Type ty) {
-  return ty.isa<mlir::IntegerType>() || ty.isa<IntType>();
+/// Is `ty` a standard or FIR integer type?
+static bool isaIntegerType(mlir::Type ty) {
+  // TODO: why aren't we using isa_integer? investigatation required.
+  return ty.isa<mlir::IntegerType>() || ty.isa<fir::IntType>();
 }
 
 bool verifyRecordMemberType(mlir::Type ty) {
@@ -205,7 +207,7 @@ RecordType verifyDerived(mlir::DialectAsmParser &parser, RecordType derivedTy,
     return {};
   }
   for (auto &p : lenPList)
-    if (!verifyIntegerType(p.second)) {
+    if (!isaIntegerType(p.second)) {
       parser.emitError(loc, "LEN parameter must be integral type");
       return {};
     }
@@ -293,11 +295,13 @@ RecordType parseDerived(mlir::DialectAsmParser &parser, mlir::Location) {
   return verifyDerived(parser, result, lenParamList, typeList);
 }
 
+#ifndef NDEBUG
 // !fir.ptr<X> and !fir.heap<X> where X is !fir.ptr, !fir.heap, or !fir.ref
 // is undefined and disallowed.
 inline bool singleIndirectionLevel(mlir::Type ty) {
   return !fir::isa_ref_type(ty);
 }
+#endif
 
 } // namespace
 
@@ -384,24 +388,22 @@ struct DimsTypeStorage : public mlir::TypeStorage {
 
   static unsigned hashKey(const KeyTy &key) { return llvm::hash_combine(key); }
 
-  bool operator==(const KeyTy &key) const {
-    return key == static_cast<unsigned>(getRank());
-  }
+  bool operator==(const KeyTy &key) const { return key == getRank(); }
 
   static DimsTypeStorage *construct(mlir::TypeStorageAllocator &allocator,
-                                    int rank) {
+                                    unsigned rank) {
     auto *storage = allocator.allocate<DimsTypeStorage>();
     return new (storage) DimsTypeStorage{rank};
   }
 
-  int getRank() const { return rank; }
+  unsigned getRank() const { return rank; }
 
 protected:
-  int rank;
+  unsigned rank;
 
 private:
   DimsTypeStorage() = delete;
-  explicit DimsTypeStorage(int rank) : rank{rank} {}
+  explicit DimsTypeStorage(unsigned rank) : rank{rank} {}
 };
 
 /// The type of a derived type part reference
@@ -832,6 +834,9 @@ bool isa_std_type(mlir::Type t) {
 }
 
 bool isa_fir_or_std_type(mlir::Type t) {
+  if (auto funcType = t.dyn_cast<mlir::FunctionType>())
+    return llvm::all_of(funcType.getInputs(), isa_fir_or_std_type) &&  
+      llvm::all_of(funcType.getResults(), isa_fir_or_std_type);
   return isa_fir_type(t) || isa_std_type(t);
 }
 
@@ -874,7 +879,7 @@ DimsType fir::DimsType::get(mlir::MLIRContext *ctxt, unsigned rank) {
   return Base::get(ctxt, FIR_DIMS, rank);
 }
 
-int fir::DimsType::getRank() const { return getImpl()->getRank(); }
+unsigned fir::DimsType::getRank() const { return getImpl()->getRank(); }
 
 // Field
 
@@ -999,10 +1004,7 @@ fir::ReferenceType::verifyConstructionInvariants(mlir::Location loc,
 // Pointer<T>
 
 PointerType fir::PointerType::get(mlir::Type elementType) {
-  if (!singleIndirectionLevel(elementType)) {
-    llvm_unreachable("FIXME: invalid element type");
-    return {};
-  }
+  assert(singleIndirectionLevel(elementType) && "invalid element type");
   return Base::get(elementType.getContext(), FIR_POINTER, elementType);
 }
 
@@ -1030,10 +1032,7 @@ fir::PointerType::verifyConstructionInvariants(mlir::Location loc,
 // Heap<T>
 
 HeapType fir::HeapType::get(mlir::Type elementType) {
-  if (!singleIndirectionLevel(elementType)) {
-    llvm_unreachable("FIXME: invalid element type");
-    return {};
-  }
+  assert(singleIndirectionLevel(elementType) && "invalid element type");
   return Base::get(elementType.getContext(), FIR_HEAP, elementType);
 }
 
@@ -1171,7 +1170,6 @@ mlir::Type fir::RecordType::getType(llvm::StringRef ident) {
   for (auto f : getTypeList())
     if (ident == f.first)
       return f.second;
-  llvm_unreachable("query for field not present in record");
   return {};
 }
 
@@ -1216,9 +1214,9 @@ llvm::SmallPtrSet<detail::RecordTypeStorage const *, 4> recordTypeVisited;
 } // namespace
 
 void fir::verifyIntegralType(mlir::Type type) {
-  if (verifyIntegerType(type) || type.isa<mlir::IndexType>())
+  if (isaIntegerType(type) || type.isa<mlir::IndexType>())
     return;
-  llvm_unreachable("expected integral type");
+  llvm::report_fatal_error("expected integral type");
 }
 
 void fir::printFirType(FIROpsDialect *, mlir::Type ty,
diff --git a/flang/lib/Optimizer/Support/KindMapping.cpp b/flang/lib/Optimizer/Support/KindMapping.cpp
index 8731c0bb087ed..e1debae9a920a 100644
--- a/flang/lib/Optimizer/Support/KindMapping.cpp
+++ b/flang/lib/Optimizer/Support/KindMapping.cpp
@@ -8,7 +8,6 @@
 
 #include "flang/Optimizer/Support/KindMapping.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
-#include "llvm/ADT/Optional.h"
 #include "llvm/Support/CommandLine.h"
 
 /// Allow the user to set the FIR intrinsic type kind value to LLVM type
@@ -219,26 +218,33 @@ MatchResult fir::KindMapping::parse(llvm::StringRef kindMap) {
   return mlir::success();
 }
 
-Bitsize fir::KindMapping::getCharacterBitsize(KindTy kind) {
+Bitsize fir::KindMapping::getCharacterBitsize(KindTy kind) const {
   return getIntegerLikeBitsize<'a'>(kind, intMap);
 }
 
-Bitsize fir::KindMapping::getIntegerBitsize(KindTy kind) {
+Bitsize fir::KindMapping::getIntegerBitsize(KindTy kind) const {
   return getIntegerLikeBitsize<'i'>(kind, intMap);
 }
 
-Bitsize fir::KindMapping::getLogicalBitsize(KindTy kind) {
+Bitsize fir::KindMapping::getLogicalBitsize(KindTy kind) const {
   return getIntegerLikeBitsize<'l'>(kind, intMap);
 }
 
-LLVMTypeID fir::KindMapping::getRealTypeID(KindTy kind) {
+LLVMTypeID fir::KindMapping::getRealTypeID(KindTy kind) const {
   return getFloatLikeTypeID<'r'>(kind, floatMap);
 }
 
-LLVMTypeID fir::KindMapping::getComplexTypeID(KindTy kind) {
+LLVMTypeID fir::KindMapping::getComplexTypeID(KindTy kind) const {
   return getFloatLikeTypeID<'c'>(kind, floatMap);
 }
 
-const llvm::fltSemantics &fir::KindMapping::getFloatSemantics(KindTy kind) {
+Bitsize fir::KindMapping::getRealBitsize(KindTy kind) const {
+  auto typeId = getFloatLikeTypeID<'r'>(kind, floatMap);
+  llvm::LLVMContext llCtxt; // FIXME
+  return llvm::Type::getPrimitiveType(llCtxt, typeId)->getPrimitiveSizeInBits();
+}
+
+const llvm::fltSemantics &
+fir::KindMapping::getFloatSemantics(KindTy kind) const {
   return getFloatSemanticsOfKind<'r'>(kind, floatMap);
 }
diff --git a/flang/lib/Optimizer/Transforms/Inliner.cpp b/flang/lib/Optimizer/Transforms/Inliner.cpp
new file mode 100644
index 0000000000000..716665a5c240c
--- /dev/null
+++ b/flang/lib/Optimizer/Transforms/Inliner.cpp
@@ -0,0 +1,24 @@
+//===-- Inliner.cpp -------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/Dialect/FIROps.h"
+#include "flang/Optimizer/Transforms/Passes.h"
+#include "mlir/Transforms/Passes.h"
+#include "llvm/Support/CommandLine.h"
+
+static llvm::cl::opt<bool>
+    aggressivelyInline("inline-all",
+                       llvm::cl::desc("aggressively inline everything"),
+                       llvm::cl::init(false));
+
+/// Should we inline the callable `op` into region `reg`?
+bool fir::canLegallyInline(mlir::Operation *op, mlir::Region *reg,
+                           mlir::BlockAndValueMapping &map) {
+  return aggressivelyInline;
+}
diff --git a/flang/lib/Parser/CMakeLists.txt b/flang/lib/Parser/CMakeLists.txt
index cc691de35b482..eb5126e1b937e 100644
--- a/flang/lib/Parser/CMakeLists.txt
+++ b/flang/lib/Parser/CMakeLists.txt
@@ -29,4 +29,7 @@ add_flang_library(FortranParser
 
   LINK_COMPONENTS
   Support
+
+  DEPENDS
+  omp_gen
 )
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 2d81f5a8197b5..41a97ff902d97 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -240,45 +240,38 @@ TYPE_PARSER(construct<OmpObjectList>(nonemptyList(Parser<OmpObject>{})))
 // Omp directives enclosing do loop
 TYPE_PARSER(sourced(construct<OmpLoopDirective>(first(
     "DISTRIBUTE PARALLEL DO SIMD" >>
-        pure(OmpLoopDirective::Directive::DistributeParallelDoSimd),
+        pure(llvm::omp::Directive::OMPD_distribute_parallel_do_simd),
     "DISTRIBUTE PARALLEL DO" >>
-        pure(OmpLoopDirective::Directive::DistributeParallelDo),
-
-    "DISTRIBUTE SIMD" >> pure(OmpLoopDirective::Directive::DistributeSimd),
-
-    "DISTRIBUTE" >> pure(OmpLoopDirective::Directive::Distribute),
-
-    "DO SIMD" >> pure(OmpLoopDirective::Directive::DoSimd),
-    "DO" >> pure(OmpLoopDirective::Directive::Do),
-    "PARALLEL DO SIMD" >> pure(OmpLoopDirective::Directive::ParallelDoSimd),
-
-    "PARALLEL DO" >> pure(OmpLoopDirective::Directive::ParallelDo),
-
-    "SIMD" >> pure(OmpLoopDirective::Directive::Simd),
+        pure(llvm::omp::Directive::OMPD_distribute_parallel_do),
+    "DISTRIBUTE SIMD" >> pure(llvm::omp::Directive::OMPD_distribute_simd),
+    "DISTRIBUTE" >> pure(llvm::omp::Directive::OMPD_distribute),
+    "DO SIMD" >> pure(llvm::omp::Directive::OMPD_do_simd),
+    "DO" >> pure(llvm::omp::Directive::OMPD_do),
+    "PARALLEL DO SIMD" >> pure(llvm::omp::Directive::OMPD_parallel_do_simd),
+    "PARALLEL DO" >> pure(llvm::omp::Directive::OMPD_parallel_do),
+    "SIMD" >> pure(llvm::omp::Directive::OMPD_simd),
     "TARGET PARALLEL DO SIMD" >>
-        pure(OmpLoopDirective::Directive::TargetParallelDoSimd),
-    "TARGET PARALLEL DO" >> pure(OmpLoopDirective::Directive::TargetParallelDo),
-
-    "TARGET SIMD" >> pure(OmpLoopDirective::Directive::TargetSimd),
+        pure(llvm::omp::Directive::OMPD_target_parallel_do_simd),
+    "TARGET PARALLEL DO" >> pure(llvm::omp::Directive::OMPD_target_parallel_do),
+    "TARGET SIMD" >> pure(llvm::omp::Directive::OMPD_target_simd),
     "TARGET TEAMS DISTRIBUTE PARALLEL DO SIMD" >>
-        pure(OmpLoopDirective::Directive::TargetTeamsDistributeParallelDoSimd),
+        pure(llvm::omp::Directive::
+                OMPD_target_teams_distribute_parallel_do_simd),
     "TARGET TEAMS DISTRIBUTE PARALLEL DO" >>
-        pure(OmpLoopDirective::Directive::TargetTeamsDistributeParallelDo),
+        pure(llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do),
     "TARGET TEAMS DISTRIBUTE SIMD" >>
-        pure(OmpLoopDirective::Directive::TargetTeamsDistributeSimd),
+        pure(llvm::omp::Directive::OMPD_target_teams_distribute_simd),
     "TARGET TEAMS DISTRIBUTE" >>
-        pure(OmpLoopDirective::Directive::TargetTeamsDistribute),
-
-    "TASKLOOP SIMD" >> pure(OmpLoopDirective::Directive::TaskloopSimd),
-
-    "TASKLOOP" >> pure(OmpLoopDirective::Directive::Taskloop),
+        pure(llvm::omp::Directive::OMPD_target_teams_distribute),
+    "TASKLOOP SIMD" >> pure(llvm::omp::Directive::OMPD_taskloop_simd),
+    "TASKLOOP" >> pure(llvm::omp::Directive::OMPD_taskloop),
     "TEAMS DISTRIBUTE PARALLEL DO SIMD" >>
-        pure(OmpLoopDirective::Directive::TeamsDistributeParallelDoSimd),
+        pure(llvm::omp::Directive::OMPD_teams_distribute_parallel_do_simd),
     "TEAMS DISTRIBUTE PARALLEL DO" >>
-        pure(OmpLoopDirective::Directive::TeamsDistributeParallelDo),
+        pure(llvm::omp::Directive::OMPD_teams_distribute_parallel_do),
     "TEAMS DISTRIBUTE SIMD" >>
-        pure(OmpLoopDirective::Directive::TeamsDistributeSimd),
-    "TEAMS DISTRIBUTE" >> pure(OmpLoopDirective::Directive::TeamsDistribute)))))
+        pure(llvm::omp::Directive::OMPD_teams_distribute_simd),
+    "TEAMS DISTRIBUTE" >> pure(llvm::omp::Directive::OMPD_teams_distribute)))))
 
 TYPE_PARSER(sourced(construct<OmpBeginLoopDirective>(
     sourced(Parser<OmpLoopDirective>{}), Parser<OmpClauseList>{})))
@@ -298,22 +291,29 @@ TYPE_PARSER(sourced(construct<OpenMPCancellationPointConstruct>(
 TYPE_PARSER(sourced(construct<OpenMPCancelConstruct>(verbatim("CANCEL"_tok),
     Parser<OmpCancelType>{}, maybe("IF" >> parenthesized(scalarLogicalExpr)))))
 
-// 2.13.7 Flush construct
-TYPE_PARSER(sourced(construct<OpenMPFlushConstruct>(
-    verbatim("FLUSH"_tok), maybe(parenthesized(Parser<OmpObjectList>{})))))
+// 2.17.8 Flush construct [OpenMP 5.0]
+//        flush -> FLUSH [memory-order-clause] [(variable-name-list)]
+//        memory-order-clause -> acq_rel
+//                               release
+//                               acquire
+TYPE_PARSER(sourced(construct<OmpFlushMemoryClause>(
+    "ACQ_REL" >> pure(OmpFlushMemoryClause::FlushMemoryOrder::AcqRel) ||
+    "RELEASE" >> pure(OmpFlushMemoryClause::FlushMemoryOrder::Release) ||
+    "ACQUIRE" >> pure(OmpFlushMemoryClause::FlushMemoryOrder::Acquire))))
+
+TYPE_PARSER(sourced(construct<OpenMPFlushConstruct>(verbatim("FLUSH"_tok),
+    maybe(Parser<OmpFlushMemoryClause>{}),
+    maybe(parenthesized(Parser<OmpObjectList>{})))))
 
 // Simple Standalone Directives
 TYPE_PARSER(sourced(construct<OmpSimpleStandaloneDirective>(first(
-    "BARRIER" >> pure(OmpSimpleStandaloneDirective::Directive::Barrier),
-    "ORDERED" >> pure(OmpSimpleStandaloneDirective::Directive::Ordered),
-    "TARGET ENTER DATA" >>
-        pure(OmpSimpleStandaloneDirective::Directive::TargetEnterData),
-    "TARGET EXIT DATA" >>
-        pure(OmpSimpleStandaloneDirective::Directive::TargetExitData),
-    "TARGET UPDATE" >>
-        pure(OmpSimpleStandaloneDirective::Directive::TargetUpdate),
-    "TASKWAIT" >> pure(OmpSimpleStandaloneDirective::Directive::Taskwait),
-    "TASKYIELD" >> pure(OmpSimpleStandaloneDirective::Directive::Taskyield)))))
+    "BARRIER" >> pure(llvm::omp::Directive::OMPD_barrier),
+    "ORDERED" >> pure(llvm::omp::Directive::OMPD_ordered),
+    "TARGET ENTER DATA" >> pure(llvm::omp::Directive::OMPD_target_enter_data),
+    "TARGET EXIT DATA" >> pure(llvm::omp::Directive::OMPD_target_exit_data),
+    "TARGET UPDATE" >> pure(llvm::omp::Directive::OMPD_target_update),
+    "TASKWAIT" >> pure(llvm::omp::Directive::OMPD_taskwait),
+    "TASKYIELD" >> pure(llvm::omp::Directive::OMPD_taskyield)))))
 
 TYPE_PARSER(sourced(construct<OpenMPSimpleStandaloneConstruct>(
     Parser<OmpSimpleStandaloneDirective>{}, Parser<OmpClauseList>{})))
@@ -329,21 +329,20 @@ TYPE_PARSER(
     endOfLine)
 
 // Directives enclosing structured-block
-TYPE_PARSER(construct<OmpBlockDirective>(
-    first("MASTER" >> pure(OmpBlockDirective::Directive::Master),
-        "ORDERED" >> pure(OmpBlockDirective::Directive::Ordered),
-        "PARALLEL WORKSHARE" >>
-            pure(OmpBlockDirective::Directive::ParallelWorkshare),
-        "PARALLEL" >> pure(OmpBlockDirective::Directive::Parallel),
-        "SINGLE" >> pure(OmpBlockDirective::Directive::Single),
-        "TARGET DATA" >> pure(OmpBlockDirective::Directive::TargetData),
-        "TARGET PARALLEL" >> pure(OmpBlockDirective::Directive::TargetParallel),
-        "TARGET TEAMS" >> pure(OmpBlockDirective::Directive::TargetTeams),
-        "TARGET" >> pure(OmpBlockDirective::Directive::Target),
-        "TASK"_id >> pure(OmpBlockDirective::Directive::Task),
-        "TASKGROUP" >> pure(OmpBlockDirective::Directive::Taskgroup),
-        "TEAMS" >> pure(OmpBlockDirective::Directive::Teams),
-        "WORKSHARE" >> pure(OmpBlockDirective::Directive::Workshare))))
+TYPE_PARSER(construct<OmpBlockDirective>(first(
+    "MASTER" >> pure(llvm::omp::Directive::OMPD_master),
+    "ORDERED" >> pure(llvm::omp::Directive::OMPD_ordered),
+    "PARALLEL WORKSHARE" >> pure(llvm::omp::Directive::OMPD_parallel_workshare),
+    "PARALLEL" >> pure(llvm::omp::Directive::OMPD_parallel),
+    "SINGLE" >> pure(llvm::omp::Directive::OMPD_single),
+    "TARGET DATA" >> pure(llvm::omp::Directive::OMPD_target_data),
+    "TARGET PARALLEL" >> pure(llvm::omp::Directive::OMPD_target_parallel),
+    "TARGET TEAMS" >> pure(llvm::omp::Directive::OMPD_target_teams),
+    "TARGET" >> pure(llvm::omp::Directive::OMPD_target),
+    "TASK"_id >> pure(llvm::omp::Directive::OMPD_task),
+    "TASKGROUP" >> pure(llvm::omp::Directive::OMPD_taskgroup),
+    "TEAMS" >> pure(llvm::omp::Directive::OMPD_teams),
+    "WORKSHARE" >> pure(llvm::omp::Directive::OMPD_workshare))))
 
 TYPE_PARSER(sourced(construct<OmpBeginBlockDirective>(
     sourced(Parser<OmpBlockDirective>{}), Parser<OmpClauseList>{})))
@@ -476,10 +475,9 @@ TYPE_PARSER(construct<OpenMPBlockConstruct>(
     Parser<OmpEndBlockDirective>{} / endOmpLine))
 
 // OMP SECTIONS Directive
-TYPE_PARSER(construct<OmpSectionsDirective>(
-    first("SECTIONS" >> pure(OmpSectionsDirective::Directive::Sections),
-        "PARALLEL SECTIONS" >>
-            pure(OmpSectionsDirective::Directive::ParallelSections))))
+TYPE_PARSER(construct<OmpSectionsDirective>(first(
+    "SECTIONS" >> pure(llvm::omp::Directive::OMPD_sections),
+    "PARALLEL SECTIONS" >> pure(llvm::omp::Directive::OMPD_parallel_sections))))
 
 // OMP BEGIN and END SECTIONS Directive
 TYPE_PARSER(sourced(construct<OmpBeginSectionsDirective>(
diff --git a/flang/lib/Parser/parse-tree.cpp b/flang/lib/Parser/parse-tree.cpp
index 650635ec97bc6..e86735c0bcb0b 100644
--- a/flang/lib/Parser/parse-tree.cpp
+++ b/flang/lib/Parser/parse-tree.cpp
@@ -14,16 +14,6 @@
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 
-// So "delete Expr;" calls an external destructor for its typedExpr.
-namespace Fortran::evaluate {
-struct GenericExprWrapper {
-  ~GenericExprWrapper();
-};
-struct GenericAssignmentWrapper {
-  ~GenericAssignmentWrapper();
-};
-} // namespace Fortran::evaluate
-
 namespace Fortran::parser {
 
 // R867
@@ -258,6 +248,3 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const Name &x) {
 }
 
 } // namespace Fortran::parser
-
-template class std::unique_ptr<Fortran::evaluate::GenericExprWrapper>;
-template class std::unique_ptr<Fortran::evaluate::GenericAssignmentWrapper>;
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index 90b71925b3897..07ddbddaa08da 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -272,6 +272,7 @@ void Prescanner::LabelField(TokenSequence &token, int outCol) {
       token.CloseToken();
     }
   }
+  SkipToNextSignificantCharacter();
 }
 
 void Prescanner::SkipToEndOfLine() {
@@ -298,6 +299,14 @@ void Prescanner::NextChar() {
     at_ += 3;
     encoding_ = Encoding::UTF_8;
   }
+  SkipToNextSignificantCharacter();
+}
+
+// Skip everything that should be ignored until the next significant
+// character is reached; handles C-style comments in preprocessing
+// directives, Fortran ! comments, stuff after the right margin in
+// fixed form, and all forms of line continuation.
+void Prescanner::SkipToNextSignificantCharacter() {
   if (inPreprocessorDirective_) {
     SkipCComments();
   } else {
diff --git a/flang/lib/Parser/prescan.h b/flang/lib/Parser/prescan.h
index f2840c3bf771b..3a08a90c3679f 100644
--- a/flang/lib/Parser/prescan.h
+++ b/flang/lib/Parser/prescan.h
@@ -147,6 +147,7 @@ class Prescanner {
   void SkipToEndOfLine();
   bool MustSkipToEndOfLine() const;
   void NextChar();
+  void SkipToNextSignificantCharacter();
   void SkipCComments();
   void SkipSpaces();
   static const char *SkipWhiteSpace(const char *);
diff --git a/flang/lib/Parser/tools.cpp b/flang/lib/Parser/tools.cpp
index 0a21e73839edd..2db832ab715fa 100644
--- a/flang/lib/Parser/tools.cpp
+++ b/flang/lib/Parser/tools.cpp
@@ -135,6 +135,30 @@ const CoindexedNamedObject *GetCoindexedNamedObject(const DataRef &base) {
       },
       base.u);
 }
+const CoindexedNamedObject *GetCoindexedNamedObject(
+    const Designator &designator) {
+  return std::visit(common::visitors{
+                        [](const DataRef &x) -> const CoindexedNamedObject * {
+                          return GetCoindexedNamedObject(x);
+                        },
+                        [](const Substring &x) -> const CoindexedNamedObject * {
+                          return GetCoindexedNamedObject(
+                              std::get<DataRef>(x.t));
+                        },
+                    },
+      designator.u);
+}
+const CoindexedNamedObject *GetCoindexedNamedObject(const Variable &variable) {
+  return std::visit(
+      common::visitors{
+          [](const common::Indirection<Designator> &designator)
+              -> const CoindexedNamedObject * {
+            return GetCoindexedNamedObject(designator.value());
+          },
+          [](const auto &) -> const CoindexedNamedObject * { return nullptr; },
+      },
+      variable.u);
+}
 const CoindexedNamedObject *GetCoindexedNamedObject(
     const AllocateObject &allocateObject) {
   return std::visit(
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index c2b5c28767daa..09acaaa37076c 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2017,141 +2017,149 @@ class UnparseVisitor {
   }
   void Unparse(const OmpLoopDirective &x) {
     switch (x.v) {
-    case OmpLoopDirective::Directive::Distribute:
+    case llvm::omp::Directive::OMPD_distribute:
       Word("DISTRIBUTE ");
       break;
-    case OmpLoopDirective::Directive::DistributeParallelDo:
+    case llvm::omp::Directive::OMPD_distribute_parallel_do:
       Word("DISTRIBUTE PARALLEL DO ");
       break;
-    case OmpLoopDirective::Directive::DistributeParallelDoSimd:
+    case llvm::omp::Directive::OMPD_distribute_parallel_do_simd:
       Word("DISTRIBUTE PARALLEL DO SIMD ");
       break;
-    case OmpLoopDirective::Directive::DistributeSimd:
+    case llvm::omp::Directive::OMPD_distribute_simd:
       Word("DISTRIBUTE SIMD ");
       break;
-    case OmpLoopDirective::Directive::Do:
+    case llvm::omp::Directive::OMPD_do:
       Word("DO ");
       break;
-    case OmpLoopDirective::Directive::DoSimd:
-      Word("Do SIMD ");
+    case llvm::omp::Directive::OMPD_do_simd:
+      Word("DO SIMD ");
       break;
-    case OmpLoopDirective::Directive::ParallelDo:
+    case llvm::omp::Directive::OMPD_parallel_do:
       Word("PARALLEL DO ");
       break;
-    case OmpLoopDirective::Directive::ParallelDoSimd:
+    case llvm::omp::Directive::OMPD_parallel_do_simd:
       Word("PARALLEL DO SIMD ");
       break;
-    case OmpLoopDirective::Directive::Simd:
+    case llvm::omp::Directive::OMPD_simd:
       Word("SIMD ");
       break;
-    case OmpLoopDirective::Directive::TargetParallelDo:
+    case llvm::omp::Directive::OMPD_target_parallel_do:
       Word("TARGET PARALLEL DO ");
       break;
-    case OmpLoopDirective::Directive::TargetParallelDoSimd:
+    case llvm::omp::Directive::OMPD_target_parallel_do_simd:
       Word("TARGET PARALLEL DO SIMD ");
       break;
-    case OmpLoopDirective::Directive::TargetTeamsDistribute:
+    case llvm::omp::Directive::OMPD_target_teams_distribute:
       Word("TARGET TEAMS DISTRIBUTE ");
       break;
-    case OmpLoopDirective::Directive::TargetTeamsDistributeParallelDo:
+    case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do:
       Word("TARGET TEAMS DISTRIBUTE PARALLEL DO ");
       break;
-    case OmpLoopDirective::Directive::TargetTeamsDistributeParallelDoSimd:
+    case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do_simd:
       Word("TARGET TEAMS DISTRIBUTE PARALLEL DO SIMD ");
       break;
-    case OmpLoopDirective::Directive::TargetTeamsDistributeSimd:
+    case llvm::omp::Directive::OMPD_target_teams_distribute_simd:
       Word("TARGET TEAMS DISTRIBUTE SIMD ");
       break;
-    case OmpLoopDirective::Directive::TargetSimd:
+    case llvm::omp::Directive::OMPD_target_simd:
       Word("TARGET SIMD ");
       break;
-    case OmpLoopDirective::Directive::Taskloop:
+    case llvm::omp::Directive::OMPD_taskloop:
       Word("TASKLOOP ");
       break;
-    case OmpLoopDirective::Directive::TaskloopSimd:
+    case llvm::omp::Directive::OMPD_taskloop_simd:
       Word("TASKLOOP SIMD ");
       break;
-    case OmpLoopDirective::Directive::TeamsDistribute:
+    case llvm::omp::Directive::OMPD_teams_distribute:
       Word("TEAMS DISTRIBUTE ");
       break;
-    case OmpLoopDirective::Directive::TeamsDistributeParallelDo:
+    case llvm::omp::Directive::OMPD_teams_distribute_parallel_do:
       Word("TEAMS DISTRIBUTE PARALLEL DO ");
       break;
-    case OmpLoopDirective::Directive::TeamsDistributeParallelDoSimd:
+    case llvm::omp::Directive::OMPD_teams_distribute_parallel_do_simd:
       Word("TEAMS DISTRIBUTE PARALLEL DO SIMD ");
       break;
-    case OmpLoopDirective::Directive::TeamsDistributeSimd:
+    case llvm::omp::Directive::OMPD_teams_distribute_simd:
       Word("TEAMS DISTRIBUTE SIMD ");
       break;
+    default:
+      break;
     }
   }
   void Unparse(const OmpObjectList &x) { Walk(x.v, ","); }
   void Unparse(const OmpSimpleStandaloneDirective &x) {
     switch (x.v) {
-    case OmpSimpleStandaloneDirective::Directive::Barrier:
+    case llvm::omp::Directive::OMPD_barrier:
       Word("BARRIER ");
       break;
-    case OmpSimpleStandaloneDirective::Directive::Taskwait:
+    case llvm::omp::Directive::OMPD_taskwait:
       Word("TASKWAIT ");
       break;
-    case OmpSimpleStandaloneDirective::Directive::Taskyield:
+    case llvm::omp::Directive::OMPD_taskyield:
       Word("TASKYIELD ");
       break;
-    case OmpSimpleStandaloneDirective::Directive::TargetEnterData:
+    case llvm::omp::Directive::OMPD_target_enter_data:
       Word("TARGET ENTER DATA ");
       break;
-    case OmpSimpleStandaloneDirective::Directive::TargetExitData:
+    case llvm::omp::Directive::OMPD_target_exit_data:
       Word("TARGET EXIT DATA ");
       break;
-    case OmpSimpleStandaloneDirective::Directive::TargetUpdate:
+    case llvm::omp::Directive::OMPD_target_update:
       Word("TARGET UPDATE ");
       break;
-    case OmpSimpleStandaloneDirective::Directive::Ordered:
+    case llvm::omp::Directive::OMPD_ordered:
       Word("ORDERED ");
       break;
+    default:
+      // Nothing to be done
+      break;
     }
   }
   void Unparse(const OmpBlockDirective &x) {
     switch (x.v) {
-    case OmpBlockDirective::Directive::Master:
+    case llvm::omp::Directive::OMPD_master:
       Word("MASTER");
       break;
-    case OmpBlockDirective::Directive::Ordered:
+    case llvm::omp::Directive::OMPD_ordered:
       Word("ORDERED ");
       break;
-    case OmpBlockDirective::Directive::ParallelWorkshare:
+    case llvm::omp::Directive::OMPD_parallel_workshare:
       Word("PARALLEL WORKSHARE ");
       break;
-    case OmpBlockDirective::Directive::Parallel:
+    case llvm::omp::Directive::OMPD_parallel:
       Word("PARALLEL ");
       break;
-    case OmpBlockDirective::Directive::Single:
+    case llvm::omp::Directive::OMPD_single:
       Word("SINGLE ");
       break;
-    case OmpBlockDirective::Directive::TargetData:
+    case llvm::omp::Directive::OMPD_target_data:
       Word("TARGET DATA ");
       break;
-    case OmpBlockDirective::Directive::TargetParallel:
+    case llvm::omp::Directive::OMPD_target_parallel:
       Word("TARGET PARALLEL ");
       break;
-    case OmpBlockDirective::Directive::TargetTeams:
+    case llvm::omp::Directive::OMPD_target_teams:
       Word("TARGET TEAMS ");
       break;
-    case OmpBlockDirective::Directive::Target:
+    case llvm::omp::Directive::OMPD_target:
       Word("TARGET ");
       break;
-    case OmpBlockDirective::Directive::Taskgroup:
+    case llvm::omp::Directive::OMPD_taskgroup:
       Word("TASKGROUP ");
       break;
-    case OmpBlockDirective::Directive::Task:
+    case llvm::omp::Directive::OMPD_task:
       Word("TASK ");
       break;
-    case OmpBlockDirective::Directive::Teams:
+    case llvm::omp::Directive::OMPD_teams:
       Word("TEAMS ");
       break;
-    case OmpBlockDirective::Directive::Workshare:
+    case llvm::omp::Directive::OMPD_workshare:
       Word("WORKSHARE ");
       break;
+    default:
+      // Nothing to be done
+      break;
     }
   }
   void Unparse(const OmpMemoryClause &x) {
@@ -2314,12 +2322,14 @@ class UnparseVisitor {
   }
   void Unparse(const OmpSectionsDirective &x) {
     switch (x.v) {
-    case OmpSectionsDirective::Directive::Sections:
+    case llvm::omp::Directive::OMPD_sections:
       Word("SECTIONS ");
       break;
-    case OmpSectionsDirective::Directive::ParallelSections:
+    case llvm::omp::Directive::OMPD_parallel_sections:
       Word("PARALLEL SECTIONS ");
       break;
+    default:
+      break;
     }
   }
   void Unparse(const OmpSectionBlocks &x) {
@@ -2359,10 +2369,24 @@ class UnparseVisitor {
     Put("\n");
     EndOpenMP();
   }
+  void Unparse(const OmpFlushMemoryClause &x) {
+    switch (x.v) {
+    case OmpFlushMemoryClause::FlushMemoryOrder::AcqRel:
+      Word("ACQ_REL ");
+      break;
+    case OmpFlushMemoryClause::FlushMemoryOrder::Release:
+      Word("RELEASE ");
+      break;
+    case OmpFlushMemoryClause::FlushMemoryOrder::Acquire:
+      Word("ACQUIRE ");
+      break;
+    }
+  }
   void Unparse(const OpenMPFlushConstruct &x) {
     BeginOpenMP();
-    Word("!$OMP FLUSH");
-    Walk("(", std::get<std::optional<OmpObjectList>>(x.t), ")");
+    Word("!$OMP FLUSH ");
+    Walk(std::get<std::optional<OmpFlushMemoryClause>>(x.t));
+    Walk(" (", std::get<std::optional<OmpObjectList>>(x.t), ")");
     Put("\n");
     EndOpenMP();
   }
diff --git a/flang/lib/Semantics/CMakeLists.txt b/flang/lib/Semantics/CMakeLists.txt
index 4fd75bc60f007..05295f5900952 100644
--- a/flang/lib/Semantics/CMakeLists.txt
+++ b/flang/lib/Semantics/CMakeLists.txt
@@ -39,10 +39,15 @@ add_flang_library(FortranSemantics
   type.cpp
   unparse-with-symbols.cpp
 
+  DEPENDS
+  omp_gen
+
   LINK_LIBS
   FortranCommon
+  FortranParser
   FortranEvaluate
 
   LINK_COMPONENTS
   Support
+  FrontendOpenMP
 )
diff --git a/flang/lib/Semantics/canonicalize-do.cpp b/flang/lib/Semantics/canonicalize-do.cpp
index 8d27d63444238..797fb27b2b14c 100644
--- a/flang/lib/Semantics/canonicalize-do.cpp
+++ b/flang/lib/Semantics/canonicalize-do.cpp
@@ -30,58 +30,62 @@ class CanonicalizationOfDoLoops {
                 // Labels on end-stmt of constructs are accepted by f18 as an
                 // extension.
                 [&](common::Indirection<AssociateConstruct> &associate) {
-          CanonicalizeIfMatch(block, stack, i,
-              std::get<Statement<EndAssociateStmt>>(associate.value().t));
+                  CanonicalizeIfMatch(block, stack, i,
+                      std::get<Statement<EndAssociateStmt>>(
+                          associate.value().t));
                 },
                 [&](common::Indirection<BlockConstruct> &blockConstruct) {
-          CanonicalizeIfMatch(block, stack, i,
-              std::get<Statement<EndBlockStmt>>(blockConstruct.value().t));
+                  CanonicalizeIfMatch(block, stack, i,
+                      std::get<Statement<EndBlockStmt>>(
+                          blockConstruct.value().t));
                 },
                 [&](common::Indirection<ChangeTeamConstruct> &changeTeam) {
-          CanonicalizeIfMatch(block, stack, i,
-              std::get<Statement<EndChangeTeamStmt>>(changeTeam.value().t));
+                  CanonicalizeIfMatch(block, stack, i,
+                      std::get<Statement<EndChangeTeamStmt>>(
+                          changeTeam.value().t));
                 },
                 [&](common::Indirection<CriticalConstruct> &critical) {
-          CanonicalizeIfMatch(block, stack, i,
-              std::get<Statement<EndCriticalStmt>>(critical.value().t));
+                  CanonicalizeIfMatch(block, stack, i,
+                      std::get<Statement<EndCriticalStmt>>(critical.value().t));
                 },
                 [&](common::Indirection<DoConstruct> &doConstruct) {
-          CanonicalizeIfMatch(block, stack, i,
-              std::get<Statement<EndDoStmt>>(doConstruct.value().t));
+                  CanonicalizeIfMatch(block, stack, i,
+                      std::get<Statement<EndDoStmt>>(doConstruct.value().t));
                 },
                 [&](common::Indirection<IfConstruct> &ifConstruct) {
-          CanonicalizeIfMatch(block, stack, i,
-              std::get<Statement<EndIfStmt>>(ifConstruct.value().t));
+                  CanonicalizeIfMatch(block, stack, i,
+                      std::get<Statement<EndIfStmt>>(ifConstruct.value().t));
                 },
                 [&](common::Indirection<CaseConstruct> &caseConstruct) {
-          CanonicalizeIfMatch(block, stack, i,
-              std::get<Statement<EndSelectStmt>>(caseConstruct.value().t));
+                  CanonicalizeIfMatch(block, stack, i,
+                      std::get<Statement<EndSelectStmt>>(
+                          caseConstruct.value().t));
                 },
                 [&](common::Indirection<SelectRankConstruct> &selectRank) {
-          CanonicalizeIfMatch(block, stack, i,
-              std::get<Statement<EndSelectStmt>>(selectRank.value().t));
+                  CanonicalizeIfMatch(block, stack, i,
+                      std::get<Statement<EndSelectStmt>>(selectRank.value().t));
                 },
                 [&](common::Indirection<SelectTypeConstruct> &selectType) {
-          CanonicalizeIfMatch(block, stack, i,
-              std::get<Statement<EndSelectStmt>>(selectType.value().t));
+                  CanonicalizeIfMatch(block, stack, i,
+                      std::get<Statement<EndSelectStmt>>(selectType.value().t));
                 },
                 [&](common::Indirection<ForallConstruct> &forall) {
-          CanonicalizeIfMatch(block, stack, i,
-              std::get<Statement<EndForallStmt>>(forall.value().t));
+                  CanonicalizeIfMatch(block, stack, i,
+                      std::get<Statement<EndForallStmt>>(forall.value().t));
                 },
                 [&](common::Indirection<WhereConstruct> &where) {
                   CanonicalizeIfMatch(block, stack, i,
                       std::get<Statement<EndWhereStmt>>(where.value().t));
                 },
                 [&](Statement<common::Indirection<LabelDoStmt>> &labelDoStmt) {
-          auto &label{std::get<Label>(labelDoStmt.statement.value().t)};
-          stack.push_back(LabelInfo{i, label});
+                  auto &label{std::get<Label>(labelDoStmt.statement.value().t)};
+                  stack.push_back(LabelInfo{i, label});
                 },
                 [&](Statement<common::Indirection<EndDoStmt>> &endDoStmt) {
-          CanonicalizeIfMatch(block, stack, i, endDoStmt);
+                  CanonicalizeIfMatch(block, stack, i, endDoStmt);
                 },
                 [&](Statement<ActionStmt> &actionStmt) {
-          CanonicalizeIfMatch(block, stack, i, actionStmt);
+                  CanonicalizeIfMatch(block, stack, i, actionStmt);
                 },
             },
             executableConstruct->u);
diff --git a/flang/lib/Semantics/check-call.cpp b/flang/lib/Semantics/check-call.cpp
index 282a8776103a6..8c3810cd9daa8 100644
--- a/flang/lib/Semantics/check-call.cpp
+++ b/flang/lib/Semantics/check-call.cpp
@@ -607,7 +607,8 @@ static void CheckExplicitInterfaceArg(evaluate::ActualArgument &arg,
                 // ok
               } else {
                 messages.Say(
-                    "Actual argument is not a variable or typed expression"_err_en_US);
+                    "Actual argument '%s' associated with %s is not a variable or typed expression"_err_en_US,
+                    expr->AsFortran(), dummyName);
               }
             } else {
               const Symbol &assumed{DEREF(arg.GetAssumedTypeDummy())};
diff --git a/flang/lib/Semantics/check-coarray.cpp b/flang/lib/Semantics/check-coarray.cpp
index a9d649ffff476..4ce286d0bca1c 100644
--- a/flang/lib/Semantics/check-coarray.cpp
+++ b/flang/lib/Semantics/check-coarray.cpp
@@ -72,6 +72,16 @@ static void CheckTeamType(SemanticsContext &context, const T &x) {
   }
 }
 
+static void CheckTeamStat(
+    SemanticsContext &context, const parser::ImageSelectorSpec::Stat &stat) {
+  const parser::Variable &var{stat.v.thing.thing.value()};
+  if (parser::GetCoindexedNamedObject(var)) {
+    context.Say(parser::FindSourceLocation(var), // C931
+        "Image selector STAT variable must not be a coindexed "
+        "object"_err_en_US);
+  }
+}
+
 void CoarrayChecker::Leave(const parser::ChangeTeamStmt &x) {
   CheckNamesAreDistinct(std::get<std::list<parser::CoarrayAssociation>>(x.t));
   CheckTeamType(context_, std::get<parser::TeamValue>(x.t));
@@ -81,9 +91,42 @@ void CoarrayChecker::Leave(const parser::SyncTeamStmt &x) {
   CheckTeamType(context_, std::get<parser::TeamValue>(x.t));
 }
 
-void CoarrayChecker::Leave(const parser::ImageSelectorSpec &x) {
-  if (const auto *team{std::get_if<parser::TeamValue>(&x.u)}) {
-    CheckTeamType(context_, *team);
+void CoarrayChecker::Leave(const parser::ImageSelector &imageSelector) {
+  haveStat_ = false;
+  haveTeam_ = false;
+  haveTeamNumber_ = false;
+  for (const auto &imageSelectorSpec :
+      std::get<std::list<parser::ImageSelectorSpec>>(imageSelector.t)) {
+    if (const auto *team{
+            std::get_if<parser::TeamValue>(&imageSelectorSpec.u)}) {
+      if (haveTeam_) {
+        context_.Say(parser::FindSourceLocation(imageSelectorSpec), // C929
+            "TEAM value can only be specified once"_err_en_US);
+      }
+      CheckTeamType(context_, *team);
+      haveTeam_ = true;
+    }
+    if (const auto *stat{std::get_if<parser::ImageSelectorSpec::Stat>(
+            &imageSelectorSpec.u)}) {
+      if (haveStat_) {
+        context_.Say(parser::FindSourceLocation(imageSelectorSpec), // C929
+            "STAT variable can only be specified once"_err_en_US);
+      }
+      CheckTeamStat(context_, *stat);
+      haveStat_ = true;
+    }
+    if (std::get_if<parser::ImageSelectorSpec::Team_Number>(
+            &imageSelectorSpec.u)) {
+      if (haveTeamNumber_) {
+        context_.Say(parser::FindSourceLocation(imageSelectorSpec), // C929
+            "TEAM_NUMBER value can only be specified once"_err_en_US);
+      }
+      haveTeamNumber_ = true;
+    }
+  }
+  if (haveTeam_ && haveTeamNumber_) {
+    context_.Say(parser::FindSourceLocation(imageSelector), // C930
+        "Cannot specify both TEAM and TEAM_NUMBER"_err_en_US);
   }
 }
 
diff --git a/flang/lib/Semantics/check-coarray.h b/flang/lib/Semantics/check-coarray.h
index 2e91c7b259d6a..5f30dda6b5474 100644
--- a/flang/lib/Semantics/check-coarray.h
+++ b/flang/lib/Semantics/check-coarray.h
@@ -18,9 +18,8 @@ class MessageFixedText;
 struct ChangeTeamStmt;
 struct CoarrayAssociation;
 struct FormTeamStmt;
-struct ImageSelectorSpec;
+struct ImageSelector;
 struct SyncTeamStmt;
-struct TeamValue;
 } // namespace Fortran::parser
 
 namespace Fortran::semantics {
@@ -30,13 +29,16 @@ class CoarrayChecker : public virtual BaseChecker {
   CoarrayChecker(SemanticsContext &context) : context_{context} {}
   void Leave(const parser::ChangeTeamStmt &);
   void Leave(const parser::SyncTeamStmt &);
-  void Leave(const parser::ImageSelectorSpec &);
+  void Leave(const parser::ImageSelector &);
   void Leave(const parser::FormTeamStmt &);
 
   void Enter(const parser::CriticalConstruct &);
 
 private:
   SemanticsContext &context_;
+  bool haveStat_;
+  bool haveTeam_;
+  bool haveTeamNumber_;
 
   void CheckNamesAreDistinct(const std::list<parser::CoarrayAssociation> &);
   void Say2(const parser::CharBlock &, parser::MessageFixedText &&,
diff --git a/flang/lib/Semantics/check-data.cpp b/flang/lib/Semantics/check-data.cpp
index 7e86790a529e3..1aa52f06f642f 100644
--- a/flang/lib/Semantics/check-data.cpp
+++ b/flang/lib/Semantics/check-data.cpp
@@ -61,18 +61,18 @@ class DataVarChecker : public evaluate::AllTraverse<DataVarChecker, true> {
     const Scope &scope{context_.FindScope(source_)};
     bool isFirstSymbol{isFirstSymbol_};
     isFirstSymbol_ = false;
-    if (const char *whyNot{IsAutomatic(symbol)          ? "Automatic variable"
-            : IsDummy(symbol)                           ? "Dummy argument"
-            : IsFunctionResult(symbol)                  ? "Function result"
-            : IsAllocatable(symbol)                     ? "Allocatable"
-            : IsInitialized(symbol, true)               ? "Default-initialized"
-            : IsInBlankCommon(symbol)                   ? "Blank COMMON object"
-            : IsProcedure(symbol) && !IsPointer(symbol) ? "Procedure"
-            // remaining checks don't apply to components
-            : !isFirstSymbol                  ? nullptr
-            : IsHostAssociated(symbol, scope) ? "Host-associated object"
-            : IsUseAssociated(symbol, scope)  ? "USE-associated object"
-                                              : nullptr}) {
+    if (const char *whyNot{IsAutomatic(symbol) ? "Automatic variable"
+                : IsDummy(symbol)              ? "Dummy argument"
+                : IsFunctionResult(symbol)     ? "Function result"
+                : IsAllocatable(symbol)        ? "Allocatable"
+                : IsInitialized(symbol, true)  ? "Default-initialized"
+                : IsInBlankCommon(symbol)      ? "Blank COMMON object"
+                : IsProcedure(symbol) && !IsPointer(symbol) ? "Procedure"
+                // remaining checks don't apply to components
+                : !isFirstSymbol                  ? nullptr
+                : IsHostAssociated(symbol, scope) ? "Host-associated object"
+                : IsUseAssociated(symbol, scope)  ? "USE-associated object"
+                                                  : nullptr}) {
       context_.Say(source_,
           "%s '%s' must not be initialized in a DATA statement"_err_en_US,
           whyNot, symbol.name());
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 77312733cd2e8..a5f65bcbc8044 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -13,55 +13,10 @@
 
 namespace Fortran::semantics {
 
-static constexpr inline OmpClauseSet doAllowedClauses{OmpClause::PRIVATE,
-    OmpClause::FIRSTPRIVATE, OmpClause::LASTPRIVATE, OmpClause::LINEAR,
-    OmpClause::REDUCTION};
-static constexpr inline OmpClauseSet doAllowedOnceClauses{
-    OmpClause::SCHEDULE, OmpClause::COLLAPSE, OmpClause::ORDERED};
-
-static constexpr inline OmpClauseSet simdAllowedClauses{OmpClause::LINEAR,
-    OmpClause::ALIGNED, OmpClause::PRIVATE, OmpClause::LASTPRIVATE,
-    OmpClause::REDUCTION};
-static constexpr inline OmpClauseSet simdAllowedOnceClauses{
-    OmpClause::COLLAPSE, OmpClause::SAFELEN, OmpClause::SIMDLEN};
-
-static constexpr inline OmpClauseSet parallelAllowedClauses{OmpClause::DEFAULT,
-    OmpClause::PRIVATE, OmpClause::FIRSTPRIVATE, OmpClause::SHARED,
-    OmpClause::COPYIN, OmpClause::REDUCTION};
-static constexpr inline OmpClauseSet parallelAllowedOnceClauses{
-    OmpClause::IF, OmpClause::NUM_THREADS, OmpClause::PROC_BIND};
-
-static constexpr inline OmpClauseSet taskloopAllowedClauses{OmpClause::SHARED,
-    OmpClause::PRIVATE, OmpClause::FIRSTPRIVATE, OmpClause::LASTPRIVATE,
-    OmpClause::DEFAULT, OmpClause::UNTIED, OmpClause::MERGEABLE,
-    OmpClause::NOGROUP};
-static constexpr inline OmpClauseSet taskloopAllowedOnceClauses{
-    OmpClause::COLLAPSE, OmpClause::IF, OmpClause::FINAL, OmpClause::PRIORITY};
-static constexpr inline OmpClauseSet taskloopAllowedExclusiveClauses{
-    OmpClause::GRAINSIZE, OmpClause::NUM_TASKS};
-
-static constexpr inline OmpClauseSet distributeAllowedClauses{
-    OmpClause::PRIVATE, OmpClause::FIRSTPRIVATE, OmpClause::LASTPRIVATE};
-static constexpr inline OmpClauseSet distributeAllowedOnceClauses{
-    OmpClause::COLLAPSE, OmpClause::DIST_SCHEDULE};
-
-static constexpr inline OmpClauseSet targetAllowedClauses{OmpClause::IF,
-    OmpClause::PRIVATE, OmpClause::FIRSTPRIVATE, OmpClause::MAP,
-    OmpClause::IS_DEVICE_PTR, OmpClause::DEPEND};
-static constexpr inline OmpClauseSet targetAllowedOnceClauses{
-    OmpClause::DEVICE, OmpClause::DEFAULTMAP, OmpClause::NOWAIT};
-
-static constexpr inline OmpClauseSet teamsAllowedClauses{OmpClause::PRIVATE,
-    OmpClause::FIRSTPRIVATE, OmpClause::SHARED, OmpClause::REDUCTION};
-static constexpr inline OmpClauseSet teamsAllowedOnceClauses{
-    OmpClause::NUM_TEAMS, OmpClause::THREAD_LIMIT, OmpClause::DEFAULT};
-
-static constexpr inline OmpClauseSet sectionsAllowedClauses{OmpClause::PRIVATE,
-    OmpClause::FIRSTPRIVATE, OmpClause::LASTPRIVATE, OmpClause::REDUCTION};
-
 std::string OmpStructureChecker::ContextDirectiveAsFortran() {
-  auto dir{EnumToString(GetContext().directive)};
-  std::replace(dir.begin(), dir.end(), '_', ' ');
+  auto dir = llvm::omp::getOpenMPDirectiveName(GetContext().directive).str();
+  std::transform(dir.begin(), dir.end(), dir.begin(),
+      [](unsigned char c) { return std::toupper(c); });
   return dir;
 }
 
@@ -87,13 +42,13 @@ bool OmpStructureChecker::HasInvalidWorksharingNesting(
   return false;
 }
 
-void OmpStructureChecker::CheckAllowed(OmpClause type) {
+void OmpStructureChecker::CheckAllowed(llvm::omp::Clause type) {
   if (!GetContext().allowedClauses.test(type) &&
       !GetContext().allowedOnceClauses.test(type) &&
       !GetContext().allowedExclusiveClauses.test(type)) {
     context_.Say(GetContext().clauseSource,
         "%s clause is not allowed on the %s directive"_err_en_US,
-        EnumToString(type),
+        parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(type).str()),
         parser::ToUpperCaseLetters(GetContext().directiveSource.ToString()));
     return;
   }
@@ -102,22 +57,25 @@ void OmpStructureChecker::CheckAllowed(OmpClause type) {
       FindClause(type)) {
     context_.Say(GetContext().clauseSource,
         "At most one %s clause can appear on the %s directive"_err_en_US,
-        EnumToString(type),
+        parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(type).str()),
         parser::ToUpperCaseLetters(GetContext().directiveSource.ToString()));
     return;
   }
   if (GetContext().allowedExclusiveClauses.test(type)) {
-    std::vector<OmpClause> others;
-    GetContext().allowedExclusiveClauses.IterateOverMembers([&](OmpClause o) {
-      if (FindClause(o)) {
-        others.emplace_back(o);
-      }
-    });
+    std::vector<llvm::omp::Clause> others;
+    GetContext().allowedExclusiveClauses.IterateOverMembers(
+        [&](llvm::omp::Clause o) {
+          if (FindClause(o)) {
+            others.emplace_back(o);
+          }
+        });
     for (const auto &e : others) {
       context_.Say(GetContext().clauseSource,
           "%s and %s are mutually exclusive and may not appear on the "
           "same %s directive"_err_en_US,
-          EnumToString(type), EnumToString(e),
+          parser::ToUpperCaseLetters(
+              llvm::omp::getOpenMPClauseName(type).str()),
+          parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(e).str()),
           parser::ToUpperCaseLetters(GetContext().directiveSource.ToString()));
     }
     if (!others.empty()) {
@@ -127,34 +85,37 @@ void OmpStructureChecker::CheckAllowed(OmpClause type) {
   SetContextClauseInfo(type);
 }
 
-void OmpStructureChecker::CheckRequired(OmpClause c) {
+void OmpStructureChecker::CheckRequired(llvm::omp::Clause c) {
   if (!FindClause(c)) {
     context_.Say(GetContext().directiveSource,
         "At least one %s clause must appear on the %s directive"_err_en_US,
-        EnumToString(c), ContextDirectiveAsFortran());
+        parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(c).str()),
+        ContextDirectiveAsFortran());
   }
 }
 
 void OmpStructureChecker::RequiresConstantPositiveParameter(
-    const OmpClause &clause, const parser::ScalarIntConstantExpr &i) {
+    const llvm::omp::Clause &clause, const parser::ScalarIntConstantExpr &i) {
   if (const auto v{GetIntValue(i)}) {
     if (*v <= 0) {
       context_.Say(GetContext().clauseSource,
           "The parameter of the %s clause must be "
           "a constant positive integer expression"_err_en_US,
-          EnumToString(clause));
+          parser::ToUpperCaseLetters(
+              llvm::omp::getOpenMPClauseName(clause).str()));
     }
   }
 }
 
 void OmpStructureChecker::RequiresPositiveParameter(
-    const OmpClause &clause, const parser::ScalarIntExpr &i) {
+    const llvm::omp::Clause &clause, const parser::ScalarIntExpr &i) {
   if (const auto v{GetIntValue(i)}) {
     if (*v <= 0) {
       context_.Say(GetContext().clauseSource,
           "The parameter of the %s clause must be "
           "a positive integer expression"_err_en_US,
-          EnumToString(clause));
+          parser::ToUpperCaseLetters(
+              llvm::omp::getOpenMPClauseName(clause).str()));
     }
   }
 }
@@ -173,259 +134,30 @@ void OmpStructureChecker::Enter(const parser::OpenMPLoopConstruct &x) {
     CheckMatching<parser::OmpLoopDirective>(beginLoopDir, *endLoopDir);
   }
 
-  switch (beginDir.v) {
-  // 2.7.1 do-clause -> private-clause |
-  //                    firstprivate-clause |
-  //                    lastprivate-clause |
-  //                    linear-clause |
-  //                    reduction-clause |
-  //                    schedule-clause |
-  //                    collapse-clause |
-  //                    ordered-clause
-  case parser::OmpLoopDirective::Directive::Do: {
+  if (beginDir.v != llvm::omp::Directive::OMPD_do) {
+    PushContextAndClauseSets(beginDir.source, beginDir.v);
+  } else {
+    // 2.7.1 do-clause -> private-clause |
+    //                    firstprivate-clause |
+    //                    lastprivate-clause |
+    //                    linear-clause |
+    //                    reduction-clause |
+    //                    schedule-clause |
+    //                    collapse-clause |
+    //                    ordered-clause
+
     // nesting check
     HasInvalidWorksharingNesting(beginDir.source,
-        {OmpDirective::DO, OmpDirective::SECTIONS, OmpDirective::SINGLE,
-            OmpDirective::WORKSHARE, OmpDirective::TASK, OmpDirective::TASKLOOP,
-            OmpDirective::CRITICAL, OmpDirective::ORDERED, OmpDirective::ATOMIC,
-            OmpDirective::MASTER});
-
-    PushContext(beginDir.source, OmpDirective::DO);
-    SetContextAllowed(doAllowedClauses);
-    SetContextAllowedOnce(doAllowedOnceClauses);
-  } break;
-
-  // 2.11.1 parallel-do-clause -> parallel-clause |
-  //                              do-clause
-  case parser::OmpLoopDirective::Directive::ParallelDo: {
-    PushContext(beginDir.source, OmpDirective::PARALLEL_DO);
-    SetContextAllowed(parallelAllowedClauses | doAllowedClauses);
-    SetContextAllowedOnce(parallelAllowedOnceClauses | doAllowedOnceClauses);
-  } break;
-
-  // 2.8.1 simd-clause -> safelen-clause |
-  //                      simdlen-clause |
-  //                      linear-clause |
-  //                      aligned-clause |
-  //                      private-clause |
-  //                      lastprivate-clause |
-  //                      reduction-clause |
-  //                      collapse-clause
-  case parser::OmpLoopDirective::Directive::Simd: {
-    PushContext(beginDir.source, OmpDirective::SIMD);
-    SetContextAllowed(simdAllowedClauses);
-    SetContextAllowedOnce(simdAllowedOnceClauses);
-  } break;
-
-  // 2.8.3 do-simd-clause -> do-clause |
-  //                         simd-clause
-  case parser::OmpLoopDirective::Directive::DoSimd: {
-    PushContext(beginDir.source, OmpDirective::DO_SIMD);
-    SetContextAllowed(doAllowedClauses | simdAllowedClauses);
-    SetContextAllowedOnce(doAllowedOnceClauses | simdAllowedOnceClauses);
-  } break;
-
-  // 2.11.4 parallel-do-simd-clause -> parallel-clause |
-  //                                   do-simd-clause
-  case parser::OmpLoopDirective::Directive::ParallelDoSimd: {
-    PushContext(beginDir.source, OmpDirective::PARALLEL_DO_SIMD);
-    SetContextAllowed(
-        parallelAllowedClauses | doAllowedClauses | simdAllowedClauses);
-    SetContextAllowedOnce(parallelAllowedOnceClauses | doAllowedOnceClauses |
-        simdAllowedOnceClauses);
-  } break;
-
-  // 2.9.2 taskloop-clause -> if-clause |
-  //                          shared-clause |
-  //                          private-clause |
-  //                          firstprivate-clause |
-  //                          lastprivate-clause |
-  //                          default-clause |
-  //                          grainsize-clause |
-  //                          num-tasks-clause |
-  //                          collapse-clause |
-  //                          final-clause |
-  //                          priority-clause |
-  //                          untied-clause |
-  //                          mergeable-clause |
-  //                          nogroup-clause
-  case parser::OmpLoopDirective::Directive::Taskloop: {
-    PushContext(beginDir.source, OmpDirective::TASKLOOP);
-    SetContextAllowed(taskloopAllowedClauses);
-    SetContextAllowedOnce(taskloopAllowedOnceClauses);
-    SetContextAllowedExclusive(taskloopAllowedExclusiveClauses);
-  } break;
-
-  // 2.9.3 taskloop-simd-clause -> taskloop-clause |
-  //                               simd-clause
-  case parser::OmpLoopDirective::Directive::TaskloopSimd: {
-    PushContext(beginDir.source, OmpDirective::TASKLOOP_SIMD);
-    SetContextAllowed(
-        (taskloopAllowedClauses | simdAllowedClauses) - OmpClause::REDUCTION);
-    SetContextAllowedOnce(taskloopAllowedOnceClauses | simdAllowedOnceClauses);
-    SetContextAllowedExclusive(taskloopAllowedExclusiveClauses);
-  } break;
-
-  // 2.10.8 distribute-clause -> private-clause |
-  //                             firstprivate-clause |
-  //                             lastprivate-clause |
-  //                             collapse-clause |
-  //                             dist-schedule-clause
-  case parser::OmpLoopDirective::Directive::Distribute: {
-    PushContext(beginDir.source, OmpDirective::DISTRIBUTE);
-    SetContextAllowed(distributeAllowedClauses);
-    SetContextAllowedOnce(distributeAllowedOnceClauses);
-  } break;
-
-    // 2.10.9 distribute-simd-clause -> distribute-clause |
-    //                                  simd-clause
-  case parser::OmpLoopDirective::Directive::DistributeSimd: {
-    PushContext(beginDir.source, OmpDirective::DISTRIBUTE_SIMD);
-    SetContextAllowed(distributeAllowedClauses | simdAllowedClauses);
-    SetContextAllowedOnce(
-        distributeAllowedOnceClauses | simdAllowedOnceClauses);
-  } break;
-
-    // 2.10.10 distribute-parallel-do-clause -> distribute-clause |
-    //                                          parallel-do-clause
-  case parser::OmpLoopDirective::Directive::DistributeParallelDo: {
-    PushContext(beginDir.source, OmpDirective::DISTRIBUTE_PARALLEL_DO);
-    SetContextAllowed(
-        distributeAllowedClauses | parallelAllowedClauses | doAllowedClauses);
-    SetContextAllowedOnce(distributeAllowedOnceClauses |
-        parallelAllowedOnceClauses | doAllowedOnceClauses);
-  } break;
-
-    // 2.10.11 distribute-parallel-do-simd-clause -> distribute-clause |
-    //                                               parallel-do-simd-clause
-  case parser::OmpLoopDirective::Directive::DistributeParallelDoSimd: {
-    PushContext(beginDir.source, OmpDirective::DISTRIBUTE_PARALLEL_DO_SIMD);
-    SetContextAllowed(distributeAllowedClauses | parallelAllowedClauses |
-        doAllowedClauses | simdAllowedClauses);
-    SetContextAllowedOnce(distributeAllowedOnceClauses |
-        parallelAllowedOnceClauses | doAllowedOnceClauses | simdAllowedClauses);
-  } break;
-
-    // 2.11.6 target-parallel-do-clause -> target-clause |
-    //                                     parallel-do-clause
-  case parser::OmpLoopDirective::Directive::TargetParallelDo: {
-    PushContext(beginDir.source, OmpDirective::TARGET_PARALLEL_DO);
-    SetContextAllowed(
-        targetAllowedClauses | parallelAllowedClauses | doAllowedClauses);
-    SetContextAllowedOnce(
-        (targetAllowedOnceClauses | parallelAllowedOnceClauses |
-            doAllowedOnceClauses) -
-        OmpClause::NOWAIT);
-  } break;
-
-    // 2.11.7 target-parallel-do-simd-clause -> target-clause |
-    //                                          parallel-do-simd-clause
-  case parser::OmpLoopDirective::Directive::TargetParallelDoSimd: {
-    PushContext(beginDir.source, OmpDirective::TARGET_PARALLEL_DO_SIMD);
-    SetContextAllowed(targetAllowedClauses | parallelAllowedClauses |
-        doAllowedClauses | simdAllowedClauses);
-    SetContextAllowedOnce(
-        (targetAllowedOnceClauses | parallelAllowedOnceClauses |
-            doAllowedOnceClauses | simdAllowedOnceClauses) -
-        OmpClause::NOWAIT);
-  } break;
-
-    // 2.11.8 target-simd-clause -> target-clause |
-    //                              simd-clause
-  case parser::OmpLoopDirective::Directive::TargetSimd: {
-    PushContext(beginDir.source, OmpDirective::TARGET_SIMD);
-    SetContextAllowed(targetAllowedClauses | simdAllowedClauses);
-    SetContextAllowedOnce(targetAllowedOnceClauses | simdAllowedOnceClauses);
-  } break;
-
-    // 2.11.10 teams-distribute-clause -> teams-clause |
-    //                                    distribute-clause
-  case parser::OmpLoopDirective::Directive::TeamsDistribute: {
-    PushContext(beginDir.source, OmpDirective::TEAMS_DISTRIBUTE);
-    SetContextAllowed(teamsAllowedClauses | distributeAllowedClauses);
-    SetContextAllowedOnce(
-        teamsAllowedOnceClauses | distributeAllowedOnceClauses);
-  } break;
-
-    // 2.11.11 teams-distribute-simd-clause -> teams-clause |
-    //                                         distribute-simd-clause
-  case parser::OmpLoopDirective::Directive::TeamsDistributeSimd: {
-    PushContext(beginDir.source, OmpDirective::TEAMS_DISTRIBUTE_SIMD);
-    SetContextAllowed(
-        teamsAllowedClauses | distributeAllowedClauses | simdAllowedClauses);
-    SetContextAllowedOnce(teamsAllowedOnceClauses |
-        distributeAllowedOnceClauses | simdAllowedOnceClauses);
-  } break;
-
-    // 2.11.12 target-teams-distribute-clause -> target-clause |
-    //                                           teams-distribute-clause
-  case parser::OmpLoopDirective::Directive::TargetTeamsDistribute: {
-    PushContext(beginDir.source, OmpDirective::TARGET_TEAMS_DISTRIBUTE);
-    SetContextAllowed(
-        targetAllowedClauses | teamsAllowedClauses | distributeAllowedClauses);
-    SetContextAllowedOnce(targetAllowedOnceClauses | teamsAllowedOnceClauses |
-        distributeAllowedOnceClauses);
-  } break;
-
-    // 2.11.13 target-teams-distribute-simd-clause -> target-clause |
-    //                                                teams-distribute-simd-clause
-  case parser::OmpLoopDirective::Directive::TargetTeamsDistributeSimd: {
-    PushContext(beginDir.source, OmpDirective::TARGET_TEAMS_DISTRIBUTE_SIMD);
-    SetContextAllowed(targetAllowedClauses | teamsAllowedClauses |
-        distributeAllowedClauses | simdAllowedClauses);
-    SetContextAllowed(targetAllowedOnceClauses | teamsAllowedOnceClauses |
-        distributeAllowedOnceClauses | simdAllowedOnceClauses);
-  } break;
-
-    // 2.11.14 teams-distribute-parallel-do-clause -> teams-clause |
-    //                                                distribute-parallel-do-clause
-  case parser::OmpLoopDirective::Directive::TeamsDistributeParallelDo: {
-    PushContext(beginDir.source, OmpDirective::TEAMS_DISTRIBUTE_PARALLEL_DO);
-    SetContextAllowed(teamsAllowedClauses | distributeAllowedClauses |
-        parallelAllowedClauses | doAllowedClauses);
-    SetContextAllowedOnce(teamsAllowedOnceClauses |
-        distributeAllowedOnceClauses | parallelAllowedOnceClauses |
-        doAllowedOnceClauses);
-  } break;
-
-    // 2.11.15 target-teams-distribute-parallel-do-clause -> target-clause |
-    //                                                       teams-distribute-parallel-do-clause
-  case parser::OmpLoopDirective::Directive::TargetTeamsDistributeParallelDo: {
-    PushContext(
-        beginDir.source, OmpDirective::TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO);
-    SetContextAllowed(targetAllowedClauses | teamsAllowedClauses |
-        distributeAllowedClauses | parallelAllowedClauses | doAllowedClauses);
-    SetContextAllowedOnce(targetAllowedOnceClauses | teamsAllowedOnceClauses |
-        distributeAllowedOnceClauses | parallelAllowedOnceClauses |
-        doAllowedOnceClauses);
-  } break;
-
-    // 2.11.16 teams-distribute-parallel-do-clause -> teams-clause |
-    //                                                distribute-parallel-do-simd-clause
-  case parser::OmpLoopDirective::Directive::TeamsDistributeParallelDoSimd: {
-    PushContext(
-        beginDir.source, OmpDirective::TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD);
-    SetContextAllowed(teamsAllowedClauses | distributeAllowedClauses |
-        parallelAllowedClauses | doAllowedClauses | simdAllowedClauses);
-    SetContextAllowedOnce(teamsAllowedOnceClauses |
-        distributeAllowedOnceClauses | parallelAllowedOnceClauses |
-        doAllowedOnceClauses | simdAllowedOnceClauses);
-  } break;
-
-  case parser::OmpLoopDirective::Directive::
-      TargetTeamsDistributeParallelDoSimd: {
-    PushContext(beginDir.source,
-        OmpDirective::TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD);
-    SetContextAllowed(targetAllowedClauses | teamsAllowedClauses |
-        distributeAllowedClauses | parallelAllowedClauses | doAllowedClauses |
-        simdAllowedClauses);
-    SetContextAllowedOnce(targetAllowedOnceClauses | teamsAllowedOnceClauses |
-        distributeAllowedOnceClauses | parallelAllowedOnceClauses |
-        doAllowedOnceClauses | simdAllowedOnceClauses);
-  }
-
-  // TODO others
-  break;
+        {llvm::omp::Directive::OMPD_do, llvm::omp::Directive::OMPD_sections,
+            llvm::omp::Directive::OMPD_single,
+            llvm::omp::Directive::OMPD_workshare,
+            llvm::omp::Directive::OMPD_task,
+            llvm::omp::Directive::OMPD_taskloop,
+            llvm::omp::Directive::OMPD_critical,
+            llvm::omp::Directive::OMPD_ordered,
+            llvm::omp::Directive::OMPD_atomic,
+            llvm::omp::Directive::OMPD_master});
+    PushContextAndClauseSets(beginDir.source, llvm::omp::Directive::OMPD_do);
   }
 }
 
@@ -439,13 +171,9 @@ void OmpStructureChecker::Enter(const parser::OmpEndLoopDirective &x) {
   switch (dir.v) {
   // 2.7.1 end-do -> END DO [nowait-clause]
   // 2.8.3 end-do-simd -> END DO SIMD [nowait-clause]
-  case parser::OmpLoopDirective::Directive::Do:
-    SetContextDirectiveEnum(OmpDirective::END_DO);
-    SetContextAllowed(OmpClauseSet{OmpClause::NOWAIT});
-    break;
-  case parser::OmpLoopDirective::Directive::DoSimd:
-    SetContextDirectiveEnum(OmpDirective::END_DO_SIMD);
-    SetContextAllowed(OmpClauseSet{OmpClause::NOWAIT});
+  case llvm::omp::Directive::OMPD_do:
+  case llvm::omp::Directive::OMPD_do_simd:
+    SetClauseSets(dir.v);
     break;
   default:
     // no clauses are allowed
@@ -459,120 +187,7 @@ void OmpStructureChecker::Enter(const parser::OpenMPBlockConstruct &x) {
   const auto &beginDir{
       CheckMatching<parser::OmpBlockDirective>(beginBlockDir, endBlockDir)};
 
-  switch (beginDir.v) {
-  // 2.5 parallel-clause -> if-clause |
-  //                        num-threads-clause |
-  //                        default-clause |
-  //                        private-clause |
-  //                        firstprivate-clause |
-  //                        shared-clause |
-  //                        copyin-clause |
-  //                        reduction-clause |
-  //                        proc-bind-clause
-  case parser::OmpBlockDirective::Directive::Parallel: {
-    // reserve for nesting check
-    PushContext(beginDir.source, OmpDirective::PARALLEL);
-    SetContextAllowed(parallelAllowedClauses);
-    SetContextAllowedOnce(parallelAllowedOnceClauses);
-  } break;
-  // 2.7.3 single-clause -> private-clause |
-  //                        firstprivate-clause
-  case parser::OmpBlockDirective::Directive::Single:
-    PushContext(beginDir.source, OmpDirective::SINGLE);
-    SetContextAllowed({OmpClause::PRIVATE, OmpClause::FIRSTPRIVATE});
-    break;
-  // 2.7.4 workshare (no clauses are allowed)
-  case parser::OmpBlockDirective::Directive::Workshare:
-    PushContext(beginDir.source, OmpDirective::WORKSHARE);
-    break;
-  // 2.11.3 parallel-workshare-clause -> parallel-clause
-  case parser::OmpBlockDirective::Directive::ParallelWorkshare: {
-    PushContext(beginDir.source, OmpDirective::PARALLEL_WORKSHARE);
-    SetContextAllowed(parallelAllowedClauses);
-    SetContextAllowedOnce(parallelAllowedOnceClauses);
-  } break;
-    // 2.9.1 task-clause -> if-clause |
-    //                      final-clause |
-    //                      untied-clause |
-    //                      default-clause |
-    //                      mergeable-clause |
-    //                      private-clause |
-    //                      firstprivate-clause |
-    //                      shared-clause |
-    //                      depend-clause |
-    //                      priority-clause
-  case parser::OmpBlockDirective::Directive::Task: {
-    PushContext(beginDir.source, OmpDirective::TASK);
-    OmpClauseSet allowed{OmpClause::UNTIED, OmpClause::DEFAULT,
-        OmpClause::MERGEABLE, OmpClause::PRIVATE, OmpClause::FIRSTPRIVATE,
-        OmpClause::SHARED, OmpClause::DEPEND};
-    SetContextAllowed(allowed);
-    OmpClauseSet allowedOnce{
-        OmpClause::IF, OmpClause::FINAL, OmpClause::PRIORITY};
-    SetContextAllowedOnce(allowedOnce);
-  } break;
-  // 2.10.4 target-clause -> if-clause |
-  //                         device-clause |
-  //                         private-clause |
-  //                         firstprivate-clause |
-  //                         map-clause |
-  //                         is-device-ptr-clause |
-  //                         defaultmap-clause |
-  //                         nowait-clause |
-  //                         depend-clause
-  case parser::OmpBlockDirective::Directive::Target: {
-    PushContext(beginDir.source, OmpDirective::TARGET);
-    SetContextAllowed(targetAllowedClauses);
-    SetContextAllowedOnce(targetAllowedOnceClauses);
-  } break;
-  // 2.10.7 teams-clause -> num-teams-clause |
-  //                        thread-limit-clause |
-  //                        default-clause |
-  //                        private-clause |
-  //                        firstprivate-clause |
-  //                        shared-clause |
-  //                        reduction-clause
-  case parser::OmpBlockDirective::Directive::Teams: {
-    PushContext(beginDir.source, OmpDirective::TEAMS);
-    SetContextAllowed(teamsAllowedClauses);
-    SetContextAllowedOnce(teamsAllowedOnceClauses);
-  } break;
-    // 2.11.9 target-teams -> target-clause |
-    //                        teams-clause
-  case parser::OmpBlockDirective::Directive::TargetTeams: {
-    PushContext(beginDir.source, OmpDirective::TARGET_TEAMS);
-    SetContextAllowed(targetAllowedClauses | teamsAllowedClauses);
-    SetContextAllowedOnce(targetAllowedOnceClauses | teamsAllowedOnceClauses);
-  } break;
-    // 2.10.1 target-data-clause -> if-clause |
-    //                              device-clause |
-    //                              map-clause |
-    //                              use-device-ptr-clause
-  case parser::OmpBlockDirective::Directive::TargetData: {
-    PushContext(beginDir.source, OmpDirective::TARGET_DATA);
-    OmpClauseSet allowed{
-        OmpClause::IF, OmpClause::MAP, OmpClause::USE_DEVICE_PTR};
-    SetContextAllowed(allowed);
-    SetContextAllowedOnce({OmpClause::DEVICE});
-    SetContextRequired({OmpClause::MAP});
-  } break;
-    // 2.13.1 master (no clauses are allowed)
-  case parser::OmpBlockDirective::Directive::Master:
-    PushContext(beginDir.source, OmpDirective::MASTER);
-    break;
-    // 2.11.5 target-parallel-clause -> target-clause |
-    //                                  parallel-clause
-  case parser::OmpBlockDirective::Directive::TargetParallel: {
-    PushContext(beginDir.source, OmpDirective::TARGET_PARALLEL);
-    SetContextAllowed(
-        (targetAllowedClauses | parallelAllowedClauses) - OmpClause::COPYIN);
-    SetContextAllowedOnce(
-        targetAllowedOnceClauses | parallelAllowedOnceClauses);
-  } break;
-  default:
-    // TODO others
-    break;
-  }
+  PushContextAndClauseSets(beginDir.source, beginDir.v);
 }
 
 void OmpStructureChecker::Leave(const parser::OpenMPBlockConstruct &) {
@@ -586,23 +201,7 @@ void OmpStructureChecker::Enter(const parser::OpenMPSectionsConstruct &x) {
   const auto &beginDir{CheckMatching<parser::OmpSectionsDirective>(
       beginSectionsDir, endSectionsDir)};
 
-  switch (beginDir.v) {
-  // 2.7.2 sections-clause -> private-clause |
-  //                          firstprivate-clause |
-  //                          lastprivate-clause |
-  //                          reduction-clause
-  case parser::OmpSectionsDirective::Directive::Sections: {
-    PushContext(beginDir.source, OmpDirective::SECTIONS);
-    SetContextAllowed(sectionsAllowedClauses);
-  } break;
-    // 2.11.2 -> parallel-sections-clause -> parallel-clause |
-    //                                       sections-clause
-  case parser::OmpSectionsDirective::Directive::ParallelSections: {
-    PushContext(beginDir.source, OmpDirective::PARALLEL_SECTIONS);
-    SetContextAllowed(parallelAllowedClauses | sectionsAllowedClauses);
-    SetContextAllowedOnce(parallelAllowedOnceClauses);
-  } break;
-  }
+  PushContextAndClauseSets(beginDir.source, beginDir.v);
 }
 
 void OmpStructureChecker::Leave(const parser::OpenMPSectionsConstruct &) {
@@ -614,9 +213,9 @@ void OmpStructureChecker::Enter(const parser::OmpEndSectionsDirective &x) {
   ResetPartialContext(dir.source);
   switch (dir.v) {
     // 2.7.2 end-sections -> END SECTIONS [nowait-clause]
-  case parser::OmpSectionsDirective::Directive::Sections:
-    SetContextDirectiveEnum(OmpDirective::END_SECTIONS);
-    SetContextAllowed(OmpClauseSet{OmpClause::NOWAIT});
+  case llvm::omp::Directive::OMPD_sections:
+    SetContextDirectiveEnum(llvm::omp::Directive::OMPD_end_sections);
+    SetContextAllowed(OmpClauseSet{llvm::omp::Clause::OMPC_nowait});
     break;
   default:
     // no clauses are allowed
@@ -626,18 +225,7 @@ void OmpStructureChecker::Enter(const parser::OmpEndSectionsDirective &x) {
 
 void OmpStructureChecker::Enter(const parser::OpenMPDeclareSimdConstruct &x) {
   const auto &dir{std::get<parser::Verbatim>(x.t)};
-  PushContext(dir.source, OmpDirective::DECLARE_SIMD);
-  // 2.8.2 declare-simd-clause -> simdlen-clause |
-  //                              linear-clause |
-  //                              aligned-clause |
-  //                              uniform-clause |
-  //                              inbranch-clause |
-  //                              notinbranch-clause
-  OmpClauseSet allowed{
-      OmpClause::LINEAR, OmpClause::ALIGNED, OmpClause::UNIFORM};
-  SetContextAllowed(allowed);
-  SetContextAllowedOnce({OmpClause::SIMDLEN});
-  SetContextAllowedExclusive({OmpClause::INBRANCH, OmpClause::NOTINBRANCH});
+  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_declare_simd);
 }
 
 void OmpStructureChecker::Leave(const parser::OpenMPDeclareSimdConstruct &) {
@@ -646,10 +234,11 @@ void OmpStructureChecker::Leave(const parser::OpenMPDeclareSimdConstruct &) {
 
 void OmpStructureChecker::Enter(const parser::OpenMPDeclareTargetConstruct &x) {
   const auto &dir{std::get<parser::Verbatim>(x.t)};
-  PushContext(dir.source, OmpDirective::DECLARE_TARGET);
+  PushContext(dir.source, llvm::omp::Directive::OMPD_declare_target);
   const auto &spec{std::get<parser::OmpDeclareTargetSpecifier>(x.t)};
   if (std::holds_alternative<parser::OmpDeclareTargetWithClause>(spec.u)) {
-    SetContextAllowed(OmpClauseSet{OmpClause::TO, OmpClause::LINK});
+    SetContextAllowed(
+        OmpClauseSet{llvm::omp::Clause::OMPC_to, llvm::omp::Clause::OMPC_link});
   }
 }
 
@@ -660,56 +249,7 @@ void OmpStructureChecker::Leave(const parser::OpenMPDeclareTargetConstruct &) {
 void OmpStructureChecker::Enter(
     const parser::OpenMPSimpleStandaloneConstruct &x) {
   const auto &dir{std::get<parser::OmpSimpleStandaloneDirective>(x.t)};
-  switch (dir.v) {
-  case parser::OmpSimpleStandaloneDirective::Directive::Barrier: {
-    // 2.13.3 barrier
-    PushContext(dir.source, OmpDirective::BARRIER);
-  } break;
-  case parser::OmpSimpleStandaloneDirective::Directive::Taskwait: {
-    // 2.13.4 taskwait
-    PushContext(dir.source, OmpDirective::TASKWAIT);
-  } break;
-  case parser::OmpSimpleStandaloneDirective::Directive::Taskyield: {
-    // 2.9.4 taskyield
-    PushContext(dir.source, OmpDirective::TASKYIELD);
-  } break;
-  case parser::OmpSimpleStandaloneDirective::Directive::TargetEnterData: {
-    // 2.10.2 target-enter-data-clause -> if-clause |
-    //                                    device-clause |
-    //                                    map-clause |
-    //                                    depend-clause |
-    //                                    nowait-clause
-    PushContext(dir.source, OmpDirective::TARGET_ENTER_DATA);
-    OmpClauseSet allowed{OmpClause::MAP, OmpClause::DEPEND, OmpClause::NOWAIT};
-    SetContextAllowed(allowed);
-    OmpClauseSet allowedOnce{OmpClause::DEVICE, OmpClause::IF};
-    SetContextAllowedOnce(allowedOnce);
-    SetContextRequired({OmpClause::MAP});
-  } break;
-  case parser::OmpSimpleStandaloneDirective::Directive::TargetExitData: {
-    // 2.10.3  target-enter-data-clause -> if-clause |
-    //                                     device-clause |
-    //                                     map-clause |
-    //                                     depend-clause |
-    //                                     nowait-clause
-    PushContext(dir.source, OmpDirective::TARGET_EXIT_DATA);
-    OmpClauseSet allowed{OmpClause::MAP, OmpClause::DEPEND, OmpClause::NOWAIT};
-    SetContextAllowed(allowed);
-    OmpClauseSet allowedOnce{OmpClause::DEVICE, OmpClause::IF};
-    SetContextAllowedOnce(allowedOnce);
-    SetContextRequired({OmpClause::MAP});
-  } break;
-  case parser::OmpSimpleStandaloneDirective::Directive::TargetUpdate: {
-    // 2.10.5 target-update
-    PushContext(dir.source, OmpDirective::TARGET_UPDATE);
-  } break;
-  case parser::OmpSimpleStandaloneDirective::Directive::Ordered: {
-    // 2.13.8 ordered-construct-clause -> depend-clause
-    PushContext(dir.source, OmpDirective::ORDERED);
-    OmpClauseSet allowed{OmpClause::DEPEND};
-    SetContextAllowed(allowed);
-  } break;
-  }
+  PushContextAndClauseSets(dir.source, dir.v);
 }
 
 void OmpStructureChecker::Leave(
@@ -719,7 +259,7 @@ void OmpStructureChecker::Leave(
 
 void OmpStructureChecker::Enter(const parser::OpenMPFlushConstruct &x) {
   const auto &dir{std::get<parser::Verbatim>(x.t)};
-  PushContext(dir.source, OmpDirective::FLUSH);
+  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_flush);
 }
 
 void OmpStructureChecker::Leave(const parser::OpenMPFlushConstruct &) {
@@ -728,7 +268,7 @@ void OmpStructureChecker::Leave(const parser::OpenMPFlushConstruct &) {
 
 void OmpStructureChecker::Enter(const parser::OpenMPCancelConstruct &x) {
   const auto &dir{std::get<parser::Verbatim>(x.t)};
-  PushContext(dir.source, OmpDirective::CANCEL);
+  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_cancel);
 }
 
 void OmpStructureChecker::Leave(const parser::OpenMPCancelConstruct &) {
@@ -738,7 +278,8 @@ void OmpStructureChecker::Leave(const parser::OpenMPCancelConstruct &) {
 void OmpStructureChecker::Enter(
     const parser::OpenMPCancellationPointConstruct &x) {
   const auto &dir{std::get<parser::Verbatim>(x.t)};
-  PushContext(dir.source, OmpDirective::CANCELLATION_POINT);
+  PushContextAndClauseSets(
+      dir.source, llvm::omp::Directive::OMPD_cancellation_point);
 }
 
 void OmpStructureChecker::Leave(
@@ -752,17 +293,17 @@ void OmpStructureChecker::Enter(const parser::OmpEndBlockDirective &x) {
   switch (dir.v) {
   // 2.7.3 end-single-clause -> copyprivate-clause |
   //                            nowait-clause
-  case parser::OmpBlockDirective::Directive::Single: {
-    SetContextDirectiveEnum(OmpDirective::END_SINGLE);
-    OmpClauseSet allowed{OmpClause::COPYPRIVATE};
+  case llvm::omp::Directive::OMPD_single: {
+    SetContextDirectiveEnum(llvm::omp::Directive::OMPD_end_single);
+    OmpClauseSet allowed{llvm::omp::Clause::OMPC_copyprivate};
     SetContextAllowed(allowed);
-    OmpClauseSet allowedOnce{OmpClause::NOWAIT};
+    OmpClauseSet allowedOnce{llvm::omp::Clause::OMPC_nowait};
     SetContextAllowedOnce(allowedOnce);
   } break;
   // 2.7.4 end-workshare -> END WORKSHARE [nowait-clause]
-  case parser::OmpBlockDirective::Directive::Workshare:
-    SetContextDirectiveEnum(OmpDirective::END_WORKSHARE);
-    SetContextAllowed(OmpClauseSet{OmpClause::NOWAIT});
+  case llvm::omp::Directive::OMPD_workshare:
+    SetContextDirectiveEnum(llvm::omp::Directive::OMPD_end_workshare);
+    SetContextAllowed(OmpClauseSet{llvm::omp::Clause::OMPC_nowait});
     break;
   default:
     // no clauses are allowed
@@ -772,13 +313,13 @@ void OmpStructureChecker::Enter(const parser::OmpEndBlockDirective &x) {
 
 void OmpStructureChecker::Leave(const parser::OmpClauseList &) {
   // 2.7 Loop Construct Restriction
-  if (doSet.test(GetContext().directive)) {
-    if (auto *clause{FindClause(OmpClause::SCHEDULE)}) {
+  if (llvm::omp::doSet.test(GetContext().directive)) {
+    if (auto *clause{FindClause(llvm::omp::Clause::OMPC_schedule)}) {
       // only one schedule clause is allowed
       const auto &schedClause{std::get<parser::OmpScheduleClause>(clause->u)};
       if (ScheduleModifierHasType(schedClause,
               parser::OmpScheduleModifierType::ModType::Nonmonotonic)) {
-        if (FindClause(OmpClause::ORDERED)) {
+        if (FindClause(llvm::omp::Clause::OMPC_ordered)) {
           context_.Say(clause->source,
               "The NONMONOTONIC modifier cannot be specified "
               "if an ORDERED clause is specified"_err_en_US);
@@ -792,19 +333,19 @@ void OmpStructureChecker::Leave(const parser::OmpClauseList &) {
       }
     }
 
-    if (auto *clause{FindClause(OmpClause::ORDERED)}) {
+    if (auto *clause{FindClause(llvm::omp::Clause::OMPC_ordered)}) {
       // only one ordered clause is allowed
       const auto &orderedClause{
           std::get<parser::OmpClause::Ordered>(clause->u)};
 
       if (orderedClause.v) {
-        if (FindClause(OmpClause::LINEAR)) {
+        if (FindClause(llvm::omp::Clause::OMPC_linear)) {
           context_.Say(clause->source,
               "A loop directive may not have both a LINEAR clause and "
               "an ORDERED clause with a parameter"_err_en_US);
         }
 
-        if (auto *clause2{FindClause(OmpClause::COLLAPSE)}) {
+        if (auto *clause2{FindClause(llvm::omp::Clause::OMPC_collapse)}) {
           const auto &collapseClause{
               std::get<parser::OmpClause::Collapse>(clause2->u)};
           // ordered and collapse both have parameters
@@ -826,9 +367,9 @@ void OmpStructureChecker::Leave(const parser::OmpClauseList &) {
   } // doSet
 
   // 2.8.1 Simd Construct Restriction
-  if (simdSet.test(GetContext().directive)) {
-    if (auto *clause{FindClause(OmpClause::SIMDLEN)}) {
-      if (auto *clause2{FindClause(OmpClause::SAFELEN)}) {
+  if (llvm::omp::simdSet.test(GetContext().directive)) {
+    if (auto *clause{FindClause(llvm::omp::Clause::OMPC_simdlen)}) {
+      if (auto *clause2{FindClause(llvm::omp::Clause::OMPC_safelen)}) {
         const auto &simdlenClause{
             std::get<parser::OmpClause::Simdlen>(clause->u)};
         const auto &safelenClause{
@@ -850,9 +391,9 @@ void OmpStructureChecker::Leave(const parser::OmpClauseList &) {
   } // SIMD
 
   // 2.7.3 Single Construct Restriction
-  if (GetContext().directive == OmpDirective::END_SINGLE) {
-    if (auto *clause{FindClause(OmpClause::COPYPRIVATE)}) {
-      if (FindClause(OmpClause::NOWAIT)) {
+  if (GetContext().directive == llvm::omp::Directive::OMPD_end_single) {
+    if (auto *clause{FindClause(llvm::omp::Clause::OMPC_copyprivate)}) {
+      if (FindClause(llvm::omp::Clause::OMPC_nowait)) {
         context_.Say(clause->source,
             "The COPYPRIVATE clause must not be used with "
             "the NOWAIT clause"_err_en_US);
@@ -861,7 +402,7 @@ void OmpStructureChecker::Leave(const parser::OmpClauseList &) {
   }
 
   GetContext().requiredClauses.IterateOverMembers(
-      [this](OmpClause c) { CheckRequired(c); });
+      [this](llvm::omp::Clause c) { CheckRequired(c); });
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause &x) {
@@ -869,80 +410,80 @@ void OmpStructureChecker::Enter(const parser::OmpClause &x) {
 }
 
 void OmpStructureChecker::Enter(const parser::OmpNowait &) {
-  CheckAllowed(OmpClause::NOWAIT);
+  CheckAllowed(llvm::omp::Clause::OMPC_nowait);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Inbranch &) {
-  CheckAllowed(OmpClause::INBRANCH);
+  CheckAllowed(llvm::omp::Clause::OMPC_inbranch);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Mergeable &) {
-  CheckAllowed(OmpClause::MERGEABLE);
+  CheckAllowed(llvm::omp::Clause::OMPC_mergeable);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Nogroup &) {
-  CheckAllowed(OmpClause::NOGROUP);
+  CheckAllowed(llvm::omp::Clause::OMPC_nogroup);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Notinbranch &) {
-  CheckAllowed(OmpClause::NOTINBRANCH);
+  CheckAllowed(llvm::omp::Clause::OMPC_notinbranch);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Untied &) {
-  CheckAllowed(OmpClause::UNTIED);
+  CheckAllowed(llvm::omp::Clause::OMPC_untied);
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Collapse &x) {
-  CheckAllowed(OmpClause::COLLAPSE);
+  CheckAllowed(llvm::omp::Clause::OMPC_collapse);
   // collapse clause must have a parameter
-  RequiresConstantPositiveParameter(OmpClause::COLLAPSE, x.v);
+  RequiresConstantPositiveParameter(llvm::omp::Clause::OMPC_collapse, x.v);
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Copyin &) {
-  CheckAllowed(OmpClause::COPYIN);
+  CheckAllowed(llvm::omp::Clause::OMPC_copyin);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Copyprivate &) {
-  CheckAllowed(OmpClause::COPYPRIVATE);
+  CheckAllowed(llvm::omp::Clause::OMPC_copyprivate);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Device &) {
-  CheckAllowed(OmpClause::DEVICE);
+  CheckAllowed(llvm::omp::Clause::OMPC_device);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::DistSchedule &) {
-  CheckAllowed(OmpClause::DIST_SCHEDULE);
+  CheckAllowed(llvm::omp::Clause::OMPC_dist_schedule);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Final &) {
-  CheckAllowed(OmpClause::FINAL);
+  CheckAllowed(llvm::omp::Clause::OMPC_final);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Firstprivate &) {
-  CheckAllowed(OmpClause::FIRSTPRIVATE);
+  CheckAllowed(llvm::omp::Clause::OMPC_firstprivate);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::From &) {
-  CheckAllowed(OmpClause::FROM);
+  CheckAllowed(llvm::omp::Clause::OMPC_from);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Grainsize &x) {
-  CheckAllowed(OmpClause::GRAINSIZE);
-  RequiresPositiveParameter(OmpClause::GRAINSIZE, x.v);
+  CheckAllowed(llvm::omp::Clause::OMPC_grainsize);
+  RequiresPositiveParameter(llvm::omp::Clause::OMPC_grainsize, x.v);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Lastprivate &) {
-  CheckAllowed(OmpClause::LASTPRIVATE);
+  CheckAllowed(llvm::omp::Clause::OMPC_lastprivate);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::NumTasks &x) {
-  CheckAllowed(OmpClause::NUM_TASKS);
-  RequiresPositiveParameter(OmpClause::NUM_TASKS, x.v);
+  CheckAllowed(llvm::omp::Clause::OMPC_num_tasks);
+  RequiresPositiveParameter(llvm::omp::Clause::OMPC_num_tasks, x.v);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::NumTeams &x) {
-  CheckAllowed(OmpClause::NUM_TEAMS);
-  RequiresPositiveParameter(OmpClause::NUM_TEAMS, x.v);
+  CheckAllowed(llvm::omp::Clause::OMPC_num_teams);
+  RequiresPositiveParameter(llvm::omp::Clause::OMPC_num_teams, x.v);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::NumThreads &x) {
-  CheckAllowed(OmpClause::NUM_THREADS);
-  RequiresPositiveParameter(OmpClause::NUM_THREADS, x.v);
+  CheckAllowed(llvm::omp::Clause::OMPC_num_threads);
+  RequiresPositiveParameter(llvm::omp::Clause::OMPC_num_threads, x.v);
   // if parameter is variable, defer to Expression Analysis
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Ordered &x) {
-  CheckAllowed(OmpClause::ORDERED);
+  CheckAllowed(llvm::omp::Clause::OMPC_ordered);
   // the parameter of ordered clause is optional
   if (const auto &expr{x.v}) {
-    RequiresConstantPositiveParameter(OmpClause::ORDERED, *expr);
+    RequiresConstantPositiveParameter(llvm::omp::Clause::OMPC_ordered, *expr);
 
     // 2.8.3 Loop SIMD Construct Restriction
-    if (doSimdSet.test(GetContext().directive)) {
+    if (llvm::omp::doSimdSet.test(GetContext().directive)) {
       context_.Say(GetContext().clauseSource,
           "No ORDERED clause with a parameter can be specified "
           "on the %s directive"_err_en_US,
@@ -951,45 +492,45 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Ordered &x) {
   }
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Priority &x) {
-  CheckAllowed(OmpClause::PRIORITY);
-  RequiresPositiveParameter(OmpClause::PRIORITY, x.v);
+  CheckAllowed(llvm::omp::Clause::OMPC_priority);
+  RequiresPositiveParameter(llvm::omp::Clause::OMPC_priority, x.v);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Private &) {
-  CheckAllowed(OmpClause::PRIVATE);
+  CheckAllowed(llvm::omp::Clause::OMPC_private);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Safelen &x) {
-  CheckAllowed(OmpClause::SAFELEN);
-  RequiresConstantPositiveParameter(OmpClause::SAFELEN, x.v);
+  CheckAllowed(llvm::omp::Clause::OMPC_safelen);
+  RequiresConstantPositiveParameter(llvm::omp::Clause::OMPC_safelen, x.v);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Shared &) {
-  CheckAllowed(OmpClause::SHARED);
+  CheckAllowed(llvm::omp::Clause::OMPC_shared);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Simdlen &x) {
-  CheckAllowed(OmpClause::SIMDLEN);
-  RequiresConstantPositiveParameter(OmpClause::SIMDLEN, x.v);
+  CheckAllowed(llvm::omp::Clause::OMPC_simdlen);
+  RequiresConstantPositiveParameter(llvm::omp::Clause::OMPC_simdlen, x.v);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::ThreadLimit &x) {
-  CheckAllowed(OmpClause::THREAD_LIMIT);
-  RequiresPositiveParameter(OmpClause::THREAD_LIMIT, x.v);
+  CheckAllowed(llvm::omp::Clause::OMPC_thread_limit);
+  RequiresPositiveParameter(llvm::omp::Clause::OMPC_thread_limit, x.v);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::To &) {
-  CheckAllowed(OmpClause::TO);
+  CheckAllowed(llvm::omp::Clause::OMPC_to);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Link &) {
-  CheckAllowed(OmpClause::LINK);
+  CheckAllowed(llvm::omp::Clause::OMPC_link);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Uniform &) {
-  CheckAllowed(OmpClause::UNIFORM);
+  CheckAllowed(llvm::omp::Clause::OMPC_uniform);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::UseDevicePtr &) {
-  CheckAllowed(OmpClause::USE_DEVICE_PTR);
+  CheckAllowed(llvm::omp::Clause::OMPC_use_device_ptr);
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::IsDevicePtr &) {
-  CheckAllowed(OmpClause::IS_DEVICE_PTR);
+  CheckAllowed(llvm::omp::Clause::OMPC_is_device_ptr);
 }
 
 void OmpStructureChecker::Enter(const parser::OmpAlignedClause &x) {
-  CheckAllowed(OmpClause::ALIGNED);
+  CheckAllowed(llvm::omp::Clause::OMPC_aligned);
 
   if (const auto &expr{
           std::get<std::optional<parser::ScalarIntConstantExpr>>(x.t)}) {
@@ -1004,10 +545,10 @@ void OmpStructureChecker::Enter(const parser::OmpAlignedClause &x) {
   // 2.8.1 TODO: list-item attribute check
 }
 void OmpStructureChecker::Enter(const parser::OmpDefaultClause &) {
-  CheckAllowed(OmpClause::DEFAULT);
+  CheckAllowed(llvm::omp::Clause::OMPC_default);
 }
 void OmpStructureChecker::Enter(const parser::OmpDefaultmapClause &x) {
-  CheckAllowed(OmpClause::DEFAULTMAP);
+  CheckAllowed(llvm::omp::Clause::OMPC_defaultmap);
   using VariableCategory = parser::OmpDefaultmapClause::VariableCategory;
   if (!std::get<std::optional<VariableCategory>>(x.t)) {
     context_.Say(GetContext().clauseSource,
@@ -1016,22 +557,26 @@ void OmpStructureChecker::Enter(const parser::OmpDefaultmapClause &x) {
   }
 }
 void OmpStructureChecker::Enter(const parser::OmpDependClause &) {
-  CheckAllowed(OmpClause::DEPEND);
+  CheckAllowed(llvm::omp::Clause::OMPC_depend);
 }
 
 void OmpStructureChecker::Enter(const parser::OmpIfClause &x) {
-  CheckAllowed(OmpClause::IF);
+  CheckAllowed(llvm::omp::Clause::OMPC_if);
 
   using dirNameModifier = parser::OmpIfClause::DirectiveNameModifier;
   static std::unordered_map<dirNameModifier, OmpDirectiveSet>
-      dirNameModifierMap{{dirNameModifier::Parallel, parallelSet},
-          {dirNameModifier::Target, targetSet},
-          {dirNameModifier::TargetEnterData, {OmpDirective::TARGET_ENTER_DATA}},
-          {dirNameModifier::TargetExitData, {OmpDirective::TARGET_EXIT_DATA}},
-          {dirNameModifier::TargetData, {OmpDirective::TARGET_DATA}},
-          {dirNameModifier::TargetUpdate, {OmpDirective::TARGET_UPDATE}},
-          {dirNameModifier::Task, {OmpDirective::TASK}},
-          {dirNameModifier::Taskloop, taskloopSet}};
+      dirNameModifierMap{{dirNameModifier::Parallel, llvm::omp::parallelSet},
+          {dirNameModifier::Target, llvm::omp::targetSet},
+          {dirNameModifier::TargetEnterData,
+              {llvm::omp::Directive::OMPD_target_enter_data}},
+          {dirNameModifier::TargetExitData,
+              {llvm::omp::Directive::OMPD_target_exit_data}},
+          {dirNameModifier::TargetData,
+              {llvm::omp::Directive::OMPD_target_data}},
+          {dirNameModifier::TargetUpdate,
+              {llvm::omp::Directive::OMPD_target_update}},
+          {dirNameModifier::Task, {llvm::omp::Directive::OMPD_task}},
+          {dirNameModifier::Taskloop, llvm::omp::taskloopSet}};
   if (const auto &directiveName{
           std::get<std::optional<dirNameModifier>>(x.t)}) {
     auto search{dirNameModifierMap.find(*directiveName)};
@@ -1049,10 +594,10 @@ void OmpStructureChecker::Enter(const parser::OmpIfClause &x) {
 }
 
 void OmpStructureChecker::Enter(const parser::OmpLinearClause &x) {
-  CheckAllowed(OmpClause::LINEAR);
+  CheckAllowed(llvm::omp::Clause::OMPC_linear);
 
   // 2.7 Loop Construct Restriction
-  if ((doSet | simdSet).test(GetContext().directive)) {
+  if ((llvm::omp::doSet | llvm::omp::simdSet).test(GetContext().directive)) {
     if (std::holds_alternative<parser::OmpLinearClause::WithModifier>(x.u)) {
       context_.Say(GetContext().clauseSource,
           "A modifier may not be specified in a LINEAR clause "
@@ -1062,18 +607,18 @@ void OmpStructureChecker::Enter(const parser::OmpLinearClause &x) {
   }
 }
 void OmpStructureChecker::Enter(const parser::OmpMapClause &x) {
-  CheckAllowed(OmpClause::MAP);
+  CheckAllowed(llvm::omp::Clause::OMPC_map);
   if (const auto &maptype{std::get<std::optional<parser::OmpMapType>>(x.t)}) {
     using Type = parser::OmpMapType::Type;
     const Type &type{std::get<Type>(maptype->t)};
     switch (GetContext().directive) {
-    case OmpDirective::TARGET:
-    case OmpDirective::TARGET_TEAMS:
-    case OmpDirective::TARGET_TEAMS_DISTRIBUTE:
-    case OmpDirective::TARGET_TEAMS_DISTRIBUTE_SIMD:
-    case OmpDirective::TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO:
-    case OmpDirective::TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD:
-    case OmpDirective::TARGET_DATA: {
+    case llvm::omp::Directive::OMPD_target:
+    case llvm::omp::Directive::OMPD_target_teams:
+    case llvm::omp::Directive::OMPD_target_teams_distribute:
+    case llvm::omp::Directive::OMPD_target_teams_distribute_simd:
+    case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do:
+    case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do_simd:
+    case llvm::omp::Directive::OMPD_target_data: {
       if (type != Type::To && type != Type::From && type != Type::Tofrom &&
           type != Type::Alloc) {
         context_.Say(GetContext().clauseSource,
@@ -1082,7 +627,7 @@ void OmpStructureChecker::Enter(const parser::OmpMapClause &x) {
             ContextDirectiveAsFortran());
       }
     } break;
-    case OmpDirective::TARGET_ENTER_DATA: {
+    case llvm::omp::Directive::OMPD_target_enter_data: {
       if (type != Type::To && type != Type::Alloc) {
         context_.Say(GetContext().clauseSource,
             "Only the TO or ALLOC map types are permitted "
@@ -1090,7 +635,7 @@ void OmpStructureChecker::Enter(const parser::OmpMapClause &x) {
             ContextDirectiveAsFortran());
       }
     } break;
-    case OmpDirective::TARGET_EXIT_DATA: {
+    case llvm::omp::Directive::OMPD_target_exit_data: {
       if (type != Type::Delete && type != Type::Release && type != Type::From) {
         context_.Say(GetContext().clauseSource,
             "Only the FROM, RELEASE, or DELETE map types are permitted "
@@ -1104,10 +649,10 @@ void OmpStructureChecker::Enter(const parser::OmpMapClause &x) {
   }
 }
 void OmpStructureChecker::Enter(const parser::OmpProcBindClause &) {
-  CheckAllowed(OmpClause::PROC_BIND);
+  CheckAllowed(llvm::omp::Clause::OMPC_proc_bind);
 }
 void OmpStructureChecker::Enter(const parser::OmpReductionClause &) {
-  CheckAllowed(OmpClause::REDUCTION);
+  CheckAllowed(llvm::omp::Clause::OMPC_reduction);
 }
 
 bool OmpStructureChecker::ScheduleModifierHasType(
@@ -1128,10 +673,10 @@ bool OmpStructureChecker::ScheduleModifierHasType(
   return false;
 }
 void OmpStructureChecker::Enter(const parser::OmpScheduleClause &x) {
-  CheckAllowed(OmpClause::SCHEDULE);
+  CheckAllowed(llvm::omp::Clause::OMPC_schedule);
 
   // 2.7 Loop Construct Restriction
-  if (doSet.test(GetContext().directive)) {
+  if (llvm::omp::doSet.test(GetContext().directive)) {
     const auto &kind{std::get<1>(x.t)};
     const auto &chunk{std::get<2>(x.t)};
     if (chunk) {
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index e952b3531f84a..eff0eb4aa76be 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -17,79 +17,65 @@
 #include "flang/Common/enum-set.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/semantics.h"
+#include "llvm/Frontend/OpenMP/OMPConstants.h"
 
-namespace Fortran::semantics {
-
-ENUM_CLASS(OmpDirective, ATOMIC, BARRIER, CANCEL, CANCELLATION_POINT, CRITICAL,
-    DECLARE_REDUCTION, DECLARE_SIMD, DECLARE_TARGET, DISTRIBUTE,
-    DISTRIBUTE_PARALLEL_DO, DISTRIBUTE_PARALLEL_DO_SIMD, DISTRIBUTE_SIMD, DO,
-    DO_SIMD, END_CRITICAL, END_DO, END_DO_SIMD, END_SECTIONS, END_SINGLE,
-    END_WORKSHARE, FLUSH, MASTER, ORDERED, PARALLEL, PARALLEL_DO,
-    PARALLEL_DO_SIMD, PARALLEL_SECTIONS, PARALLEL_WORKSHARE, SECTION, SECTIONS,
-    SIMD, SINGLE, TARGET, TARGET_DATA, TARGET_ENTER_DATA, TARGET_EXIT_DATA,
-    TARGET_PARALLEL, TARGET_PARALLEL_DO, TARGET_PARALLEL_DO_SIMD, TARGET_SIMD,
-    TARGET_TEAMS, TARGET_TEAMS_DISTRIBUTE, TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO,
-    TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD, TARGET_TEAMS_DISTRIBUTE_SIMD,
-    TARGET_UPDATE, TASK, TASKGROUP, TASKLOOP, TASKLOOP_SIMD, TASKWAIT,
-    TASKYIELD, TEAMS, TEAMS_DISTRIBUTE, TEAMS_DISTRIBUTE_PARALLEL_DO,
-    TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD, TEAMS_DISTRIBUTE_SIMD, THREADPRIVATE,
-    WORKSHARE)
+using OmpDirectiveSet = Fortran::common::EnumSet<llvm::omp::Directive,
+    llvm::omp::Directive_enumSize>;
 
-using OmpDirectiveSet = common::EnumSet<OmpDirective, OmpDirective_enumSize>;
+using OmpClauseSet =
+    Fortran::common::EnumSet<llvm::omp::Clause, llvm::omp::Clause_enumSize>;
 
-ENUM_CLASS(OmpClause, ALIGNED, COLLAPSE, COPYIN, COPYPRIVATE, DEFAULT,
-    DEFAULTMAP, DEPEND, DEVICE, DIST_SCHEDULE, FINAL, FIRSTPRIVATE, FROM,
-    GRAINSIZE, IF, INBRANCH, IS_DEVICE_PTR, LASTPRIVATE, LINEAR, LINK, MAP,
-    MERGEABLE, NOGROUP, NOTINBRANCH, NOWAIT, NUM_TASKS, NUM_TEAMS, NUM_THREADS,
-    ORDERED, PRIORITY, PRIVATE, PROC_BIND, REDUCTION, SAFELEN, SCHEDULE, SHARED,
-    SIMD, SIMDLEN, THREAD_LIMIT, THREADS, TO, UNIFORM, UNTIED, USE_DEVICE_PTR)
+#define GEN_FLANG_DIRECTIVE_CLAUSE_SETS
+#include "llvm/Frontend/OpenMP/OMP.cpp.inc"
 
-using OmpClauseSet = common::EnumSet<OmpClause, OmpClause_enumSize>;
+namespace llvm {
+namespace omp {
+static OmpDirectiveSet parallelSet{Directive::OMPD_distribute_parallel_do,
+    Directive::OMPD_distribute_parallel_do_simd, Directive::OMPD_parallel,
+    Directive::OMPD_parallel_do, Directive::OMPD_parallel_do_simd,
+    Directive::OMPD_parallel_sections, Directive::OMPD_parallel_workshare,
+    Directive::OMPD_target_parallel, Directive::OMPD_target_parallel_do,
+    Directive::OMPD_target_parallel_do_simd,
+    Directive::OMPD_target_teams_distribute_parallel_do,
+    Directive::OMPD_target_teams_distribute_parallel_do_simd,
+    Directive::OMPD_teams_distribute_parallel_do,
+    Directive::OMPD_teams_distribute_parallel_do_simd};
+static OmpDirectiveSet doSet{Directive::OMPD_distribute_parallel_do,
+    Directive::OMPD_distribute_parallel_do_simd, Directive::OMPD_parallel,
+    Directive::OMPD_parallel_do, Directive::OMPD_parallel_do_simd,
+    Directive::OMPD_do, Directive::OMPD_do_simd,
+    Directive::OMPD_target_parallel_do, Directive::OMPD_target_parallel_do_simd,
+    Directive::OMPD_target_teams_distribute_parallel_do,
+    Directive::OMPD_target_teams_distribute_parallel_do_simd,
+    Directive::OMPD_teams_distribute_parallel_do,
+    Directive::OMPD_teams_distribute_parallel_do_simd};
+static OmpDirectiveSet doSimdSet{Directive::OMPD_distribute_parallel_do_simd,
+    Directive::OMPD_parallel_do_simd, Directive::OMPD_do_simd,
+    Directive::OMPD_target_parallel_do_simd,
+    Directive::OMPD_target_teams_distribute_parallel_do_simd,
+    Directive::OMPD_teams_distribute_parallel_do_simd};
+static OmpDirectiveSet taskloopSet{
+    Directive::OMPD_taskloop, Directive::OMPD_taskloop_simd};
+static OmpDirectiveSet targetSet{Directive::OMPD_target,
+    Directive::OMPD_target_parallel, Directive::OMPD_target_parallel_do,
+    Directive::OMPD_target_parallel_do_simd, Directive::OMPD_target_simd,
+    Directive::OMPD_target_teams, Directive::OMPD_target_teams_distribute,
+    Directive::OMPD_target_teams_distribute_simd};
+static OmpDirectiveSet simdSet{Directive::OMPD_distribute_parallel_do_simd,
+    Directive::OMPD_distribute_simd, Directive::OMPD_parallel_do_simd,
+    Directive::OMPD_do_simd, Directive::OMPD_simd,
+    Directive::OMPD_target_parallel_do_simd,
+    Directive::OMPD_target_teams_distribute_parallel_do_simd,
+    Directive::OMPD_target_teams_distribute_simd, Directive::OMPD_target_simd,
+    Directive::OMPD_taskloop_simd,
+    Directive::OMPD_teams_distribute_parallel_do_simd,
+    Directive::OMPD_teams_distribute_simd};
+static OmpDirectiveSet taskGeneratingSet{
+    OmpDirectiveSet{Directive::OMPD_task} | taskloopSet};
+} // namespace omp
+} // namespace llvm
 
-static constexpr OmpDirectiveSet parallelSet{
-    OmpDirective::DISTRIBUTE_PARALLEL_DO,
-    OmpDirective::DISTRIBUTE_PARALLEL_DO_SIMD, OmpDirective::PARALLEL,
-    OmpDirective::PARALLEL_DO, OmpDirective::PARALLEL_DO_SIMD,
-    OmpDirective::PARALLEL_SECTIONS, OmpDirective::PARALLEL_WORKSHARE,
-    OmpDirective::TARGET_PARALLEL, OmpDirective::TARGET_PARALLEL_DO,
-    OmpDirective::TARGET_PARALLEL_DO_SIMD,
-    OmpDirective::TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO,
-    OmpDirective::TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD,
-    OmpDirective::TEAMS_DISTRIBUTE_PARALLEL_DO,
-    OmpDirective::TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD};
-static constexpr OmpDirectiveSet doSet{OmpDirective::DISTRIBUTE_PARALLEL_DO,
-    OmpDirective::DISTRIBUTE_PARALLEL_DO_SIMD, OmpDirective::PARALLEL_DO,
-    OmpDirective::PARALLEL_DO_SIMD, OmpDirective::DO, OmpDirective::DO_SIMD,
-    OmpDirective::TARGET_PARALLEL_DO, OmpDirective::TARGET_PARALLEL_DO_SIMD,
-    OmpDirective::TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO,
-    OmpDirective::TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD,
-    OmpDirective::TEAMS_DISTRIBUTE_PARALLEL_DO,
-    OmpDirective::TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD};
-static constexpr OmpDirectiveSet doSimdSet{
-    OmpDirective::DISTRIBUTE_PARALLEL_DO_SIMD, OmpDirective::PARALLEL_DO_SIMD,
-    OmpDirective::DO_SIMD, OmpDirective::TARGET_PARALLEL_DO_SIMD,
-    OmpDirective::TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD,
-    OmpDirective::TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD};
-static constexpr OmpDirectiveSet taskloopSet{
-    OmpDirective::TASKLOOP, OmpDirective::TASKLOOP_SIMD};
-static constexpr OmpDirectiveSet targetSet{OmpDirective::TARGET,
-    OmpDirective::TARGET_PARALLEL, OmpDirective::TARGET_PARALLEL_DO,
-    OmpDirective::TARGET_PARALLEL_DO_SIMD, OmpDirective::TARGET_SIMD,
-    OmpDirective::TARGET_TEAMS, OmpDirective::TARGET_TEAMS_DISTRIBUTE,
-    OmpDirective::TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO,
-    OmpDirective::TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD,
-    OmpDirective::TARGET_TEAMS_DISTRIBUTE_SIMD};
-static constexpr OmpDirectiveSet simdSet{
-    OmpDirective::DISTRIBUTE_PARALLEL_DO_SIMD, OmpDirective::DISTRIBUTE_SIMD,
-    OmpDirective::PARALLEL_DO_SIMD, OmpDirective::DO_SIMD, OmpDirective::SIMD,
-    OmpDirective::TARGET_PARALLEL_DO_SIMD,
-    OmpDirective::TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD,
-    OmpDirective::TARGET_TEAMS_DISTRIBUTE_SIMD, OmpDirective::TARGET_SIMD,
-    OmpDirective::TASKLOOP_SIMD,
-    OmpDirective::TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD,
-    OmpDirective::TEAMS_DISTRIBUTE_SIMD};
-static constexpr OmpDirectiveSet taskGeneratingSet{
-    OmpDirectiveSet{OmpDirective::TASK} | taskloopSet};
+namespace Fortran::semantics {
 
 class OmpStructureChecker : public virtual BaseChecker {
 public:
@@ -168,19 +154,22 @@ class OmpStructureChecker : public virtual BaseChecker {
   void Enter(const parser::OmpScheduleClause &);
 
 private:
+#define GEN_FLANG_DIRECTIVE_CLAUSE_MAP
+#include "llvm/Frontend/OpenMP/OMP.cpp.inc"
+
   struct OmpContext {
-    OmpContext(parser::CharBlock source, OmpDirective d)
+    OmpContext(parser::CharBlock source, llvm::omp::Directive d)
         : directiveSource{source}, directive{d} {}
     parser::CharBlock directiveSource{nullptr};
     parser::CharBlock clauseSource{nullptr};
-    OmpDirective directive;
+    llvm::omp::Directive directive;
     OmpClauseSet allowedClauses{};
     OmpClauseSet allowedOnceClauses{};
     OmpClauseSet allowedExclusiveClauses{};
     OmpClauseSet requiredClauses{};
 
     const parser::OmpClause *clause{nullptr};
-    std::multimap<OmpClause, const parser::OmpClause *> clauseInfo;
+    std::multimap<llvm::omp::Clause, const parser::OmpClause *> clauseInfo;
   };
   // back() is the top of the stack
   OmpContext &GetContext() {
@@ -205,7 +194,7 @@ class OmpStructureChecker : public virtual BaseChecker {
     GetContext().clauseSource = clause.source;
     GetContext().clause = &clause;
   }
-  void SetContextDirectiveEnum(OmpDirective dir) {
+  void SetContextDirectiveEnum(llvm::omp::Directive dir) {
     GetContext().directive = dir;
   }
   void SetContextAllowed(const OmpClauseSet &allowed) {
@@ -220,30 +209,43 @@ class OmpStructureChecker : public virtual BaseChecker {
   void SetContextRequired(const OmpClauseSet &required) {
     GetContext().requiredClauses = required;
   }
-  void SetContextClauseInfo(OmpClause type) {
+  void SetContextClauseInfo(llvm::omp::Clause type) {
     GetContext().clauseInfo.emplace(type, GetContext().clause);
   }
-  const parser::OmpClause *FindClause(OmpClause type) {
+  const parser::OmpClause *FindClause(llvm::omp::Clause type) {
     auto it{GetContext().clauseInfo.find(type)};
     if (it != GetContext().clauseInfo.end()) {
       return it->second;
     }
     return nullptr;
   }
-  void PushContext(const parser::CharBlock &source, OmpDirective dir) {
+  void PushContext(const parser::CharBlock &source, llvm::omp::Directive dir) {
     ompContext_.emplace_back(source, dir);
   }
-
+  void SetClauseSets(llvm::omp::Directive dir) {
+    ompContext_.back().allowedClauses = directiveClausesTable[dir].allowed;
+    ompContext_.back().allowedOnceClauses =
+        directiveClausesTable[dir].allowedOnce;
+    ompContext_.back().allowedExclusiveClauses =
+        directiveClausesTable[dir].allowedExclusive;
+    ompContext_.back().requiredClauses =
+        directiveClausesTable[dir].requiredOneOf;
+  }
+  void PushContextAndClauseSets(
+      const parser::CharBlock &source, llvm::omp::Directive dir) {
+    PushContext(source, dir);
+    SetClauseSets(dir);
+  }
   void RequiresConstantPositiveParameter(
-      const OmpClause &clause, const parser::ScalarIntConstantExpr &i);
+      const llvm::omp::Clause &clause, const parser::ScalarIntConstantExpr &i);
   void RequiresPositiveParameter(
-      const OmpClause &clause, const parser::ScalarIntExpr &i);
+      const llvm::omp::Clause &clause, const parser::ScalarIntExpr &i);
 
   bool CurrentDirectiveIsNested() { return ompContext_.size() > 0; };
   bool HasInvalidWorksharingNesting(
       const parser::CharBlock &, const OmpDirectiveSet &);
-  void CheckAllowed(OmpClause);
-  void CheckRequired(OmpClause);
+  void CheckAllowed(llvm::omp::Clause);
+  void CheckRequired(llvm::omp::Clause);
   std::string ContextDirectiveAsFortran();
   void SayNotMatching(const parser::CharBlock &, const parser::CharBlock &);
   template <typename A, typename B, typename C>
diff --git a/flang/lib/Semantics/compute-offsets.cpp b/flang/lib/Semantics/compute-offsets.cpp
index 1a03c6ee47f9e..bcc4c0caf46a5 100644
--- a/flang/lib/Semantics/compute-offsets.cpp
+++ b/flang/lib/Semantics/compute-offsets.cpp
@@ -8,6 +8,7 @@
 
 #include "compute-offsets.h"
 #include "../../runtime/descriptor.h"
+#include "flang/Evaluate/fold-designator.h"
 #include "flang/Evaluate/fold.h"
 #include "flang/Evaluate/shape.h"
 #include "flang/Evaluate/type.h"
@@ -39,19 +40,23 @@ class ComputeOffsetsHelper {
     std::size_t alignment{0};
   };
   struct SymbolAndOffset {
-    Symbol *symbol{nullptr};
-    std::size_t offset{0};
+    SymbolAndOffset(Symbol &s, std::size_t off, const EquivalenceObject &obj)
+        : symbol{&s}, offset{off}, object{&obj} {}
+    SymbolAndOffset(const SymbolAndOffset &) = default;
+    Symbol *symbol;
+    std::size_t offset;
+    const EquivalenceObject *object;
   };
 
   void Compute(Scope &);
   void DoScope(Scope &);
   void DoCommonBlock(Symbol &);
-  void DoEquivalenceSet(EquivalenceSet &);
-  std::size_t GetOffset(SymbolAndOffset &);
+  void DoEquivalenceSet(const EquivalenceSet &);
+  SymbolAndOffset Resolve(const SymbolAndOffset &);
   std::size_t ComputeOffset(const EquivalenceObject &);
   void DoSymbol(Symbol &);
   SizeAndAlignment GetSizeAndAlignment(const Symbol &);
-  SizeAndAlignment GetElementSize(const Symbol &, bool isSubstring = false);
+  SizeAndAlignment GetElementSize(const Symbol &);
   std::size_t CountElements(const Symbol &);
   static std::size_t Align(std::size_t, std::size_t);
   static SizeAndAlignment GetIntrinsicSizeAndAlignment(TypeCategory, int);
@@ -85,7 +90,7 @@ void ComputeOffsetsHelper::DoScope(Scope &scope) {
     DoCommonBlock(*pair.second);
   }
   // Build dependents_ from equivalences: symbol -> symbol+offset
-  for (EquivalenceSet &set : scope.equivalenceSets()) {
+  for (const EquivalenceSet &set : scope.equivalenceSets()) {
     DoEquivalenceSet(set);
   }
   offset_ = 0;
@@ -100,7 +105,8 @@ void ComputeOffsetsHelper::DoScope(Scope &scope) {
     if (symbol->size() == 0) {
       SizeAndAlignment s{GetSizeAndAlignment(*symbol)};
       symbol->set_size(s.size);
-      symbol->set_offset(GetOffset(dep));
+      SymbolAndOffset resolved{Resolve(dep)};
+      symbol->set_offset(dep.symbol->offset() + resolved.offset);
       offset_ = std::max(offset_, symbol->offset() + symbol->size());
     }
   }
@@ -108,12 +114,16 @@ void ComputeOffsetsHelper::DoScope(Scope &scope) {
   scope.set_alignment(alignment_);
 }
 
-std::size_t ComputeOffsetsHelper::GetOffset(SymbolAndOffset &dep) {
+auto ComputeOffsetsHelper::Resolve(const SymbolAndOffset &dep)
+    -> SymbolAndOffset {
   auto it{dependents_.find(*dep.symbol)};
   if (it == dependents_.end()) {
-    return dep.symbol->offset() + dep.offset;
+    return dep;
   } else {
-    return GetOffset(it->second) + dep.offset;
+    SymbolAndOffset result{Resolve(it->second)};
+    result.offset += dep.offset;
+    result.object = dep.object;
+    return result;
   }
 }
 
@@ -128,22 +138,51 @@ void ComputeOffsetsHelper::DoCommonBlock(Symbol &commonBlock) {
   details.set_alignment(alignment_);
 }
 
-void ComputeOffsetsHelper::DoEquivalenceSet(EquivalenceSet &set) {
+void ComputeOffsetsHelper::DoEquivalenceSet(const EquivalenceSet &set) {
   std::vector<SymbolAndOffset> symbolOffsets;
-  SymbolAndOffset max;
-  for (EquivalenceObject &object : set) {
+  std::optional<std::size_t> representative;
+  for (const EquivalenceObject &object : set) {
     std::size_t offset{ComputeOffset(object)};
-    symbolOffsets.push_back({&object.symbol, offset});
-    if (offset >= max.offset) {
-      max.offset = offset;
-      max.symbol = &object.symbol;
+    SymbolAndOffset resolved{
+        Resolve(SymbolAndOffset{object.symbol, offset, object})};
+    symbolOffsets.push_back(resolved);
+    if (!representative ||
+        resolved.offset >= symbolOffsets[*representative].offset) {
+      // The equivalenced object with the largest offset from its resolved
+      // symbol will be the representative of this set, since the offsets
+      // of the other objects will be positive relative to it.
+      representative = symbolOffsets.size() - 1;
     }
   }
-  CHECK(max.symbol);
-  for (auto &[symbol, offset] : symbolOffsets) {
-    if (symbol != max.symbol) {
-      dependents_.emplace(
-          *symbol, SymbolAndOffset{max.symbol, max.offset - offset});
+  CHECK(representative);
+  const SymbolAndOffset &base{symbolOffsets[*representative]};
+  for (const auto &[symbol, offset, object] : symbolOffsets) {
+    if (symbol == base.symbol) {
+      if (offset != base.offset) {
+        auto x{evaluate::OffsetToDesignator(
+            context_.foldingContext(), *symbol, base.offset, 1)};
+        auto y{evaluate::OffsetToDesignator(
+            context_.foldingContext(), *symbol, offset, 1)};
+        if (x && y) {
+          context_
+              .Say(base.object->source,
+                  "'%s' and '%s' cannot have the same first storage unit"_err_en_US,
+                  x->AsFortran(), y->AsFortran())
+              .Attach(object->source, "Incompatible reference to '%s'"_en_US,
+                  y->AsFortran());
+        } else { // error recovery
+          context_
+              .Say(base.object->source,
+                  "'%s' (offset %zd bytes and %zd bytes) cannot have the same first storage unit"_err_en_US,
+                  symbol->name(), base.offset, offset)
+              .Attach(object->source,
+                  "Incompatible reference to '%s' offset %zd bytes"_en_US,
+                  symbol->name(), offset);
+        }
+      }
+    } else {
+      dependents_.emplace(*symbol,
+          SymbolAndOffset{*base.symbol, base.offset - offset, *object});
     }
   }
 }
@@ -152,9 +191,6 @@ void ComputeOffsetsHelper::DoEquivalenceSet(EquivalenceSet &set) {
 std::size_t ComputeOffsetsHelper::ComputeOffset(
     const EquivalenceObject &object) {
   std::size_t offset{0};
-  if (object.substringStart) {
-    offset = *object.substringStart - 1;
-  }
   if (!object.subscripts.empty()) {
     const ArraySpec &shape{object.symbol.get<ObjectEntityDetails>().shape()};
     auto lbound{[&](std::size_t i) {
@@ -172,8 +208,17 @@ std::size_t ComputeOffsetsHelper::ComputeOffset(
       offset *= ubound(i) - lbound(i) + 1;
     }
   }
-  return offset *
-      GetElementSize(object.symbol, object.substringStart.has_value()).size;
+  auto result{offset * GetElementSize(object.symbol).size};
+  if (object.substringStart) {
+    int kind{context_.defaultKinds().GetDefaultKind(TypeCategory::Character)};
+    if (const DeclTypeSpec * type{object.symbol.GetType()}) {
+      if (const IntrinsicTypeSpec * intrinsic{type->AsIntrinsic()}) {
+        kind = ToInt64(intrinsic->kind()).value_or(kind);
+      }
+    }
+    result += kind * (*object.substringStart - 1);
+  }
+  return result;
 }
 
 void ComputeOffsetsHelper::DoSymbol(Symbol &symbol) {
@@ -203,8 +248,8 @@ auto ComputeOffsetsHelper::GetSizeAndAlignment(const Symbol &symbol)
   return result;
 }
 
-auto ComputeOffsetsHelper::GetElementSize(
-    const Symbol &symbol, bool isSubstring) -> SizeAndAlignment {
+auto ComputeOffsetsHelper::GetElementSize(const Symbol &symbol)
+    -> SizeAndAlignment {
   const DeclTypeSpec *type{symbol.GetType()};
   if (!type) {
     return {};
@@ -226,7 +271,7 @@ auto ComputeOffsetsHelper::GetElementSize(
     if (auto kind{ToInt64(intrinsic->kind())}) {
       result = GetIntrinsicSizeAndAlignment(intrinsic->category(), *kind);
     }
-    if (!isSubstring && type->category() == DeclTypeSpec::Character) {
+    if (type->category() == DeclTypeSpec::Character) {
       ParamValue length{type->characterTypeSpec().length()};
       CHECK(length.isExplicit()); // else should be descriptor
       if (MaybeIntExpr lengthExpr{length.GetExplicit()}) {
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index 60e705f0ee88f..f22f8a9669246 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -909,7 +909,10 @@ MaybeExpr ExpressionAnalyzer::Analyze(const parser::ArrayElement &ae) {
       return std::nullopt;
     } else if (baseExpr->Rank() == 0) {
       if (const Symbol * symbol{GetLastSymbol(*baseExpr)}) {
-        Say("'%s' is not an array"_err_en_US, symbol->name());
+        if (!context_.HasError(symbol)) {
+          Say("'%s' is not an array"_err_en_US, symbol->name());
+          context_.SetError(const_cast<Symbol &>(*symbol));
+        }
       }
     } else if (std::optional<DataRef> dataRef{
                    ExtractDataRef(std::move(*baseExpr))}) {
@@ -1082,12 +1085,21 @@ MaybeExpr ExpressionAnalyzer::Analyze(const parser::CoindexedNamedObject &x) {
             symbol.name(), symbol.Corank(), numCosubscripts);
       }
     }
-    // TODO: stat=/team=/team_number=
+    for (const auto &imageSelSpec :
+        std::get<std::list<parser::ImageSelectorSpec>>(x.imageSelector.t)) {
+      std::visit(
+          common::visitors{
+              [&](const auto &x) { Analyze(x.v); },
+          },
+          imageSelSpec.u);
+    }
     // Reverse the chain of symbols so that the base is first and coarray
     // ultimate component is last.
-    return Designate(
-        DataRef{CoarrayRef{SymbolVector{reversed.crbegin(), reversed.crend()},
-            std::move(subscripts), std::move(cosubscripts)}});
+    if (cosubsOk) {
+      return Designate(
+          DataRef{CoarrayRef{SymbolVector{reversed.crbegin(), reversed.crend()},
+              std::move(subscripts), std::move(cosubscripts)}});
+    }
   }
   return std::nullopt;
 }
@@ -2006,7 +2018,8 @@ void ExpressionAnalyzer::Analyze(const parser::CallStmt &callStmt) {
   const parser::Call &call{callStmt.v};
   auto restorer{GetContextualMessages().SetLocation(call.source)};
   ArgumentAnalyzer analyzer{*this, call.source, true /* allowAssumedType */};
-  for (const auto &arg : std::get<std::list<parser::ActualArgSpec>>(call.t)) {
+  const auto &actualArgList{std::get<std::list<parser::ActualArgSpec>>(call.t)};
+  for (const auto &arg : actualArgList) {
     analyzer.Analyze(arg, true /* is subroutine call */);
   }
   if (!analyzer.fatalErrors()) {
@@ -2016,8 +2029,12 @@ void ExpressionAnalyzer::Analyze(const parser::CallStmt &callStmt) {
       ProcedureDesignator *proc{std::get_if<ProcedureDesignator>(&callee->u)};
       CHECK(proc);
       if (CheckCall(call.source, *proc, callee->arguments)) {
-        callStmt.typedCall.reset(
-            new ProcedureRef{std::move(*proc), std::move(callee->arguments)});
+        bool hasAlternateReturns{
+            callee->arguments.size() < actualArgList.size()};
+        callStmt.typedCall.Reset(
+            new ProcedureRef{std::move(*proc), std::move(callee->arguments),
+                hasAlternateReturns},
+            ProcedureRef::Deleter);
       }
     }
   }
@@ -2029,7 +2046,8 @@ const Assignment *ExpressionAnalyzer::Analyze(const parser::AssignmentStmt &x) {
     analyzer.Analyze(std::get<parser::Variable>(x.t));
     analyzer.Analyze(std::get<parser::Expr>(x.t));
     if (analyzer.fatalErrors()) {
-      x.typedAssignment.reset(new GenericAssignmentWrapper{});
+      x.typedAssignment.Reset(
+          new GenericAssignmentWrapper{}, GenericAssignmentWrapper::Deleter);
     } else {
       std::optional<ProcedureRef> procRef{analyzer.TryDefinedAssignment()};
       Assignment assignment{
@@ -2037,8 +2055,9 @@ const Assignment *ExpressionAnalyzer::Analyze(const parser::AssignmentStmt &x) {
       if (procRef) {
         assignment.u = std::move(*procRef);
       }
-      x.typedAssignment.reset(
-          new GenericAssignmentWrapper{std::move(assignment)});
+      x.typedAssignment.Reset(
+          new GenericAssignmentWrapper{std::move(assignment)},
+          GenericAssignmentWrapper::Deleter);
     }
   }
   return common::GetPtrFromOptional(x.typedAssignment->v);
@@ -2050,7 +2069,8 @@ const Assignment *ExpressionAnalyzer::Analyze(
     MaybeExpr lhs{Analyze(std::get<parser::DataRef>(x.t))};
     MaybeExpr rhs{Analyze(std::get<parser::Expr>(x.t))};
     if (!lhs || !rhs) {
-      x.typedAssignment.reset(new GenericAssignmentWrapper{});
+      x.typedAssignment.Reset(
+          new GenericAssignmentWrapper{}, GenericAssignmentWrapper::Deleter);
     } else {
       Assignment assignment{std::move(*lhs), std::move(*rhs)};
       std::visit(common::visitors{
@@ -2077,8 +2097,9 @@ const Assignment *ExpressionAnalyzer::Analyze(
                      },
                  },
           std::get<parser::PointerAssignmentStmt::Bounds>(x.t).u);
-      x.typedAssignment.reset(
-          new GenericAssignmentWrapper{std::move(assignment)});
+      x.typedAssignment.Reset(
+          new GenericAssignmentWrapper{std::move(assignment)},
+          GenericAssignmentWrapper::Deleter);
     }
   }
   return common::GetPtrFromOptional(x.typedAssignment->v);
@@ -2678,6 +2699,7 @@ void ArgumentAnalyzer::Analyze(
   // be detected and represented (they're not expressions).
   // TODO: C1534: Don't allow a "restricted" specific intrinsic to be passed.
   std::optional<ActualArgument> actual;
+  bool isAltReturn{false};
   std::visit(common::visitors{
                  [&](const common::Indirection<parser::Expr> &x) {
                    // TODO: Distinguish & handle procedure name and
@@ -2690,6 +2712,7 @@ void ArgumentAnalyzer::Analyze(
                          "alternate return specification may not appear on"
                          " function reference"_err_en_US);
                    }
+                   isAltReturn = true;
                  },
                  [&](const parser::ActualArg::PercentRef &) {
                    context_.Say("TODO: %REF() argument"_err_en_US);
@@ -2704,7 +2727,7 @@ void ArgumentAnalyzer::Analyze(
       actual->set_keyword(argKW->v.source);
     }
     actuals_.emplace_back(std::move(*actual));
-  } else {
+  } else if (!isAltReturn) {
     fatalErrors_ = true;
   }
 }
@@ -2917,7 +2940,7 @@ std::optional<ActualArgument> ArgumentAnalyzer::AnalyzeExpr(
     const parser::Expr &expr) {
   source_.ExtendToCover(expr.source);
   if (const Symbol * assumedTypeDummy{AssumedTypeDummy(expr)}) {
-    expr.typedExpr.reset(new GenericExprWrapper{});
+    expr.typedExpr.Reset(new GenericExprWrapper{}, GenericExprWrapper::Deleter);
     if (allowAssumedType_) {
       return ActualArgument{ActualArgument::AssumedType{*assumedTypeDummy}};
     } else {
@@ -3023,7 +3046,7 @@ std::string ArgumentAnalyzer::TypeAsFortran(std::size_t i) {
   if (std::optional<DynamicType> type{GetType(i)}) {
     return type->category() == TypeCategory::Derived
         ? "TYPE("s + type->AsFortran() + ')'
-    : type->category() == TypeCategory::Character
+        : type->category() == TypeCategory::Character
         ? "CHARACTER(KIND="s + std::to_string(type->kind()) + ')'
         : ToUpperCase(type->AsFortran());
   } else {
diff --git a/flang/lib/Semantics/mod-file.h b/flang/lib/Semantics/mod-file.h
index 2800f315054eb..8823c5f1e4972 100644
--- a/flang/lib/Semantics/mod-file.h
+++ b/flang/lib/Semantics/mod-file.h
@@ -10,6 +10,7 @@
 #define FORTRAN_SEMANTICS_MOD_FILE_H_
 
 #include "flang/Semantics/attr.h"
+#include "llvm/Support/raw_ostream.h"
 #include <string>
 
 namespace Fortran::parser {
diff --git a/flang/lib/Semantics/resolve-names-utils.cpp b/flang/lib/Semantics/resolve-names-utils.cpp
index 5564570e263cf..d6f0302e98545 100644
--- a/flang/lib/Semantics/resolve-names-utils.cpp
+++ b/flang/lib/Semantics/resolve-names-utils.cpp
@@ -365,7 +365,8 @@ void EquivalenceSets::AddToSet(const parser::Designator &designator) {
       }
     }
     auto substringStart{currObject_.substringStart};
-    currSet_.emplace_back(symbol, subscripts, substringStart);
+    currSet_.emplace_back(
+        symbol, subscripts, substringStart, designator.source);
     PropagateSaveAttr(currSet_.back(), currSet_);
   }
   currObject_ = {};
@@ -446,16 +447,7 @@ void EquivalenceSets::MergeInto(const parser::CharBlock &source,
   EquivalenceSet &dst{sets_[dstIndex]};
   PropagateSaveAttr(dst, src);
   for (const auto &obj : src) {
-    if (const auto *obj2{Find(dst, obj.symbol)}) {
-      if (obj == *obj2) {
-        continue; // already there
-      }
-      context_.Say(source,
-          "'%s' and '%s' cannot have the same first storage unit"_err_en_US,
-          obj2->AsFortran(), obj.AsFortran());
-    } else {
-      dst.push_back(obj);
-    }
+    dst.push_back(obj);
     objectToSet_[obj] = dstIndex;
   }
   PropagateSaveAttr(src, dst);
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 2b257fce9fd6d..4d70f03dd5532 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -1150,9 +1150,9 @@ bool OmpVisitor::NeedsScope(const parser::OpenMPBlockConstruct &x) {
   const auto &beginBlockDir{std::get<parser::OmpBeginBlockDirective>(x.t)};
   const auto &beginDir{std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
   switch (beginDir.v) {
-  case parser::OmpBlockDirective::Directive::TargetData:
-  case parser::OmpBlockDirective::Directive::Master:
-  case parser::OmpBlockDirective::Directive::Ordered:
+  case llvm::omp::Directive::OMPD_target_data:
+  case llvm::omp::Directive::OMPD_master:
+  case llvm::omp::Directive::OMPD_ordered:
     return false;
   default:
     return true;
@@ -1236,10 +1236,11 @@ class OmpAttributeVisitor {
 
 private:
   struct OmpContext {
-    OmpContext(const parser::CharBlock &source, OmpDirective d, Scope &s)
+    OmpContext(
+        const parser::CharBlock &source, llvm::omp::Directive d, Scope &s)
         : directiveSource{source}, directive{d}, scope{s} {}
     parser::CharBlock directiveSource;
-    OmpDirective directive;
+    llvm::omp::Directive directive;
     Scope &scope;
     // TODO: default DSA is implicitly determined in different ways
     Symbol::Flag defaultDSA{Symbol::Flag::OmpShared};
@@ -1253,14 +1254,14 @@ class OmpAttributeVisitor {
     CHECK(!ompContext_.empty());
     return ompContext_.back();
   }
-  void PushContext(const parser::CharBlock &source, OmpDirective dir) {
+  void PushContext(const parser::CharBlock &source, llvm::omp::Directive dir) {
     ompContext_.emplace_back(source, dir, context_.FindScope(source));
   }
   void PopContext() { ompContext_.pop_back(); }
   void SetContextDirectiveSource(parser::CharBlock &dir) {
     GetContext().directiveSource = dir;
   }
-  void SetContextDirectiveEnum(OmpDirective dir) {
+  void SetContextDirectiveEnum(llvm::omp::Directive dir) {
     GetContext().directive = dir;
   }
   Scope &currScope() { return GetContext().scope; }
@@ -1436,7 +1437,7 @@ class ResolveNamesVisitor : public virtual ScopeHandler,
 
   void PreSpecificationConstruct(const parser::SpecificationConstruct &);
   void CreateGeneric(const parser::GenericSpec &);
-  void FinishSpecificationPart();
+  void FinishSpecificationPart(const std::list<parser::DeclarationConstruct> &);
   void CheckImports();
   void CheckImport(const SourceName &, const SourceName &);
   void HandleCall(Symbol::Flag, const parser::Call &);
@@ -2714,11 +2715,9 @@ bool SubprogramVisitor::HandleStmtFunction(const parser::StmtFunctionStmt &x) {
   details.set_result(result);
   const auto &parsedExpr{std::get<parser::Scalar<parser::Expr>>(x.t)};
   Walk(parsedExpr);
-  if (auto expr{AnalyzeExpr(context(), parsedExpr)}) {
-    details.set_stmtFunction(std::move(*expr));
-  } else {
-    context().SetError(symbol);
-  }
+  // The analysis of the expression that constitutes the body of the
+  // statement function is deferred to FinishSpecificationPart() so that
+  // all declarations and implicit typing are complete.
   PopScope();
   return true;
 }
@@ -5506,7 +5505,15 @@ const parser::Name *DeclarationVisitor::ResolveDataRef(
           },
           [&](const Indirection<parser::ArrayElement> &y) {
             Walk(y.value().subscripts);
-            return ResolveDataRef(y.value().base);
+            const parser::Name *name{ResolveDataRef(y.value().base)};
+            if (!name) {
+            } else if (!name->symbol->has<ProcEntityDetails>()) {
+              ConvertToObjectEntity(*name->symbol);
+            } else if (!context().HasError(*name->symbol)) {
+              SayWithDecl(*name, *name->symbol,
+                  "Cannot reference function '%s' as data"_err_en_US);
+            }
+            return name;
           },
           [&](const Indirection<parser::CoindexedNamedObject> &y) {
             Walk(y.value().imageSelector);
@@ -6011,7 +6018,7 @@ bool ResolveNamesVisitor::Pre(const parser::SpecificationPart &x) {
     }
   }
   Walk(decls);
-  FinishSpecificationPart();
+  FinishSpecificationPart(decls);
   return false;
 }
 
@@ -6068,7 +6075,8 @@ void ResolveNamesVisitor::CreateGeneric(const parser::GenericSpec &x) {
   info.Resolve(&MakeSymbol(symbolName, Attrs{}, std::move(genericDetails)));
 }
 
-void ResolveNamesVisitor::FinishSpecificationPart() {
+void ResolveNamesVisitor::FinishSpecificationPart(
+    const std::list<parser::DeclarationConstruct> &decls) {
   badStmtFuncFound_ = false;
   CheckImports();
   bool inModule{currScope().kind() == Scope::Kind::Module};
@@ -6089,6 +6097,25 @@ void ResolveNamesVisitor::FinishSpecificationPart() {
     }
   }
   currScope().InstantiateDerivedTypes(context());
+  // Analyze the bodies of statement functions now that the symbol in this
+  // specification part have been fully declared and implicitly typed.
+  for (const auto &decl : decls) {
+    if (const auto *statement{std::get_if<
+            parser::Statement<common::Indirection<parser::StmtFunctionStmt>>>(
+            &decl.u)}) {
+      const parser::StmtFunctionStmt &stmtFunc{statement->statement.value()};
+      if (Symbol * symbol{std::get<parser::Name>(stmtFunc.t).symbol}) {
+        if (auto *details{symbol->detailsIf<SubprogramDetails>()}) {
+          if (auto expr{AnalyzeExpr(context(),
+                  std::get<parser::Scalar<parser::Expr>>(stmtFunc.t))}) {
+            details->set_stmtFunction(std::move(*expr));
+          } else {
+            context().SetError(*symbol);
+          }
+        }
+      }
+    }
+  }
   // TODO: what about instantiations in BLOCK?
   CheckSaveStmts();
   CheckCommonBlocks();
@@ -6278,8 +6305,8 @@ void ResolveNamesVisitor::ResolveSpecificationParts(ProgramTree &node) {
       node.stmt());
   Walk(node.spec());
   // If this is a function, convert result to an object. This is to prevent the
-  // result to be converted later to a function symbol if it is called inside
-  // the function.
+  // result from being converted later to a function symbol if it is called
+  // inside the function.
   // If the result is function pointer, then ConvertToObjectEntity will not
   // convert the result to an object, and calling the symbol inside the function
   // will result in calls to the result pointer.
@@ -6423,41 +6450,19 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
   const auto &beginBlockDir{std::get<parser::OmpBeginBlockDirective>(x.t)};
   const auto &beginDir{std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
   switch (beginDir.v) {
-  case parser::OmpBlockDirective::Directive::Master:
-    PushContext(beginDir.source, OmpDirective::MASTER);
-    break;
-  case parser::OmpBlockDirective::Directive::Ordered:
-    PushContext(beginDir.source, OmpDirective::ORDERED);
-    break;
-  case parser::OmpBlockDirective::Directive::Parallel:
-    PushContext(beginDir.source, OmpDirective::PARALLEL);
-    break;
-  case parser::OmpBlockDirective::Directive::Single:
-    PushContext(beginDir.source, OmpDirective::SINGLE);
-    break;
-  case parser::OmpBlockDirective::Directive::Target:
-    PushContext(beginDir.source, OmpDirective::TARGET);
-    break;
-  case parser::OmpBlockDirective::Directive::TargetData:
-    PushContext(beginDir.source, OmpDirective::TARGET_DATA);
-    break;
-  case parser::OmpBlockDirective::Directive::Task:
-    PushContext(beginDir.source, OmpDirective::TASK);
-    break;
-  case parser::OmpBlockDirective::Directive::Teams:
-    PushContext(beginDir.source, OmpDirective::TEAMS);
-    break;
-  case parser::OmpBlockDirective::Directive::Workshare:
-    PushContext(beginDir.source, OmpDirective::WORKSHARE);
-    break;
-  case parser::OmpBlockDirective::Directive::ParallelWorkshare:
-    PushContext(beginDir.source, OmpDirective::PARALLEL_WORKSHARE);
-    break;
-  case parser::OmpBlockDirective::Directive::TargetTeams:
-    PushContext(beginDir.source, OmpDirective::TARGET_TEAMS);
-    break;
-  case parser::OmpBlockDirective::Directive::TargetParallel:
-    PushContext(beginDir.source, OmpDirective::TARGET_PARALLEL);
+  case llvm::omp::Directive::OMPD_master:
+  case llvm::omp::Directive::OMPD_ordered:
+  case llvm::omp::Directive::OMPD_parallel:
+  case llvm::omp::Directive::OMPD_single:
+  case llvm::omp::Directive::OMPD_target:
+  case llvm::omp::Directive::OMPD_target_data:
+  case llvm::omp::Directive::OMPD_task:
+  case llvm::omp::Directive::OMPD_teams:
+  case llvm::omp::Directive::OMPD_workshare:
+  case llvm::omp::Directive::OMPD_parallel_workshare:
+  case llvm::omp::Directive::OMPD_target_teams:
+  case llvm::omp::Directive::OMPD_target_parallel:
+    PushContext(beginDir.source, beginDir.v);
     break;
   default:
     // TODO others
@@ -6472,74 +6477,31 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPLoopConstruct &x) {
   const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
   const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
   switch (beginDir.v) {
-  case parser::OmpLoopDirective::Directive::Distribute:
-    PushContext(beginDir.source, OmpDirective::DISTRIBUTE);
+  case llvm::omp::Directive::OMPD_distribute:
+  case llvm::omp::Directive::OMPD_distribute_parallel_do:
+  case llvm::omp::Directive::OMPD_distribute_parallel_do_simd:
+  case llvm::omp::Directive::OMPD_distribute_simd:
+  case llvm::omp::Directive::OMPD_do:
+  case llvm::omp::Directive::OMPD_do_simd:
+  case llvm::omp::Directive::OMPD_parallel_do:
+  case llvm::omp::Directive::OMPD_parallel_do_simd:
+  case llvm::omp::Directive::OMPD_simd:
+  case llvm::omp::Directive::OMPD_target_parallel_do:
+  case llvm::omp::Directive::OMPD_target_parallel_do_simd:
+  case llvm::omp::Directive::OMPD_target_teams_distribute:
+  case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do:
+  case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do_simd:
+  case llvm::omp::Directive::OMPD_target_teams_distribute_simd:
+  case llvm::omp::Directive::OMPD_target_simd:
+  case llvm::omp::Directive::OMPD_taskloop:
+  case llvm::omp::Directive::OMPD_taskloop_simd:
+  case llvm::omp::Directive::OMPD_teams_distribute:
+  case llvm::omp::Directive::OMPD_teams_distribute_parallel_do:
+  case llvm::omp::Directive::OMPD_teams_distribute_parallel_do_simd:
+  case llvm::omp::Directive::OMPD_teams_distribute_simd:
+    PushContext(beginDir.source, beginDir.v);
     break;
-  case parser::OmpLoopDirective::Directive::DistributeParallelDo:
-    PushContext(beginDir.source, OmpDirective::DISTRIBUTE_PARALLEL_DO);
-    break;
-  case parser::OmpLoopDirective::Directive::DistributeParallelDoSimd:
-    PushContext(beginDir.source, OmpDirective::DISTRIBUTE_PARALLEL_DO_SIMD);
-    break;
-  case parser::OmpLoopDirective::Directive::DistributeSimd:
-    PushContext(beginDir.source, OmpDirective::DISTRIBUTE_SIMD);
-    break;
-  case parser::OmpLoopDirective::Directive::Do:
-    PushContext(beginDir.source, OmpDirective::DO);
-    break;
-  case parser::OmpLoopDirective::Directive::DoSimd:
-    PushContext(beginDir.source, OmpDirective::DO_SIMD);
-    break;
-  case parser::OmpLoopDirective::Directive::ParallelDo:
-    PushContext(beginDir.source, OmpDirective::PARALLEL_DO);
-    break;
-  case parser::OmpLoopDirective::Directive::ParallelDoSimd:
-    PushContext(beginDir.source, OmpDirective::PARALLEL_DO_SIMD);
-    break;
-  case parser::OmpLoopDirective::Directive::Simd:
-    PushContext(beginDir.source, OmpDirective::SIMD);
-    break;
-  case parser::OmpLoopDirective::Directive::TargetParallelDo:
-    PushContext(beginDir.source, OmpDirective::TARGET_PARALLEL_DO);
-    break;
-  case parser::OmpLoopDirective::Directive::TargetParallelDoSimd:
-    PushContext(beginDir.source, OmpDirective::TARGET_PARALLEL_DO_SIMD);
-    break;
-  case parser::OmpLoopDirective::Directive::TargetTeamsDistribute:
-    PushContext(beginDir.source, OmpDirective::TARGET_TEAMS_DISTRIBUTE);
-    break;
-  case parser::OmpLoopDirective::Directive::TargetTeamsDistributeParallelDo:
-    PushContext(
-        beginDir.source, OmpDirective::TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO);
-    break;
-  case parser::OmpLoopDirective::Directive::TargetTeamsDistributeParallelDoSimd:
-    PushContext(beginDir.source,
-        OmpDirective::TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD);
-    break;
-  case parser::OmpLoopDirective::Directive::TargetTeamsDistributeSimd:
-    PushContext(beginDir.source, OmpDirective::TARGET_TEAMS_DISTRIBUTE_SIMD);
-    break;
-  case parser::OmpLoopDirective::Directive::TargetSimd:
-    PushContext(beginDir.source, OmpDirective::TARGET_SIMD);
-    break;
-  case parser::OmpLoopDirective::Directive::Taskloop:
-    PushContext(beginDir.source, OmpDirective::TASKLOOP);
-    break;
-  case parser::OmpLoopDirective::Directive::TaskloopSimd:
-    PushContext(beginDir.source, OmpDirective::TASKLOOP_SIMD);
-    break;
-  case parser::OmpLoopDirective::Directive::TeamsDistribute:
-    PushContext(beginDir.source, OmpDirective::TEAMS_DISTRIBUTE);
-    break;
-  case parser::OmpLoopDirective::Directive::TeamsDistributeParallelDo:
-    PushContext(beginDir.source, OmpDirective::TEAMS_DISTRIBUTE_PARALLEL_DO);
-    break;
-  case parser::OmpLoopDirective::Directive::TeamsDistributeParallelDoSimd:
-    PushContext(
-        beginDir.source, OmpDirective::TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD);
-    break;
-  case parser::OmpLoopDirective::Directive::TeamsDistributeSimd:
-    PushContext(beginDir.source, OmpDirective::TEAMS_DISTRIBUTE_SIMD);
+  default:
     break;
   }
   ClearDataSharingAttributeObjects();
@@ -6563,8 +6525,8 @@ void OmpAttributeVisitor::ResolveSeqLoopIndexInParallelOrTaskConstruct(
     if (targetIt == ompContext_.rend()) {
       return;
     }
-    if (parallelSet.test(targetIt->directive) ||
-        taskGeneratingSet.test(targetIt->directive)) {
+    if (llvm::omp::parallelSet.test(targetIt->directive) ||
+        llvm::omp::taskGeneratingSet.test(targetIt->directive)) {
       break;
     }
   }
@@ -6651,7 +6613,7 @@ void OmpAttributeVisitor::PrivatizeAssociatedLoopIndex(
   if (level <= 0)
     return;
   Symbol::Flag ivDSA{Symbol::Flag::OmpPrivate};
-  if (simdSet.test(GetContext().directive)) {
+  if (llvm::omp::simdSet.test(GetContext().directive)) {
     if (level == 1) {
       ivDSA = Symbol::Flag::OmpLinear;
     } else {
@@ -6682,11 +6644,11 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPSectionsConstruct &x) {
   const auto &beginDir{
       std::get<parser::OmpSectionsDirective>(beginSectionsDir.t)};
   switch (beginDir.v) {
-  case parser::OmpSectionsDirective::Directive::ParallelSections:
-    PushContext(beginDir.source, OmpDirective::PARALLEL_SECTIONS);
+  case llvm::omp::Directive::OMPD_parallel_sections:
+  case llvm::omp::Directive::OMPD_sections:
+    PushContext(beginDir.source, beginDir.v);
     break;
-  case parser::OmpSectionsDirective::Directive::Sections:
-    PushContext(beginDir.source, OmpDirective::SECTIONS);
+  default:
     break;
   }
   ClearDataSharingAttributeObjects();
@@ -6694,7 +6656,7 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPSectionsConstruct &x) {
 }
 
 bool OmpAttributeVisitor::Pre(const parser::OpenMPThreadprivate &x) {
-  PushContext(x.source, OmpDirective::THREADPRIVATE);
+  PushContext(x.source, llvm::omp::Directive::OMPD_threadprivate);
   const auto &list{std::get<parser::OmpObjectList>(x.t)};
   ResolveOmpObjectList(list, Symbol::Flag::OmpThreadprivate);
   return false;
diff --git a/flang/lib/Semantics/symbol.cpp b/flang/lib/Semantics/symbol.cpp
index 9983426670039..a7e2696518ffe 100644
--- a/flang/lib/Semantics/symbol.cpp
+++ b/flang/lib/Semantics/symbol.cpp
@@ -150,9 +150,6 @@ UseErrorDetails &UseErrorDetails::add_occurrence(
   return *this;
 }
 
-GenericDetails::GenericDetails(const SymbolVector &specificProcs)
-    : specificProcs_{specificProcs} {}
-
 void GenericDetails::AddSpecificProc(
     const Symbol &proc, SourceName bindingName) {
   specificProcs_.push_back(proc);
@@ -186,6 +183,8 @@ Symbol *GenericDetails::CheckSpecific() {
 }
 
 void GenericDetails::CopyFrom(const GenericDetails &from) {
+  CHECK(specificProcs_.size() == bindingNames_.size());
+  CHECK(from.specificProcs_.size() == from.bindingNames_.size());
   if (from.specific_) {
     CHECK(!specific_ || specific_ == from.specific_);
     specific_ = from.specific_;
@@ -194,11 +193,13 @@ void GenericDetails::CopyFrom(const GenericDetails &from) {
     CHECK(!derivedType_ || derivedType_ == from.derivedType_);
     derivedType_ = from.derivedType_;
   }
-  for (const Symbol &symbol : from.specificProcs_) {
+  for (std::size_t i{0}; i < from.specificProcs_.size(); ++i) {
     if (std::find_if(specificProcs_.begin(), specificProcs_.end(),
-            [&](const Symbol &mySymbol) { return &mySymbol == &symbol; }) ==
-        specificProcs_.end()) {
-      specificProcs_.push_back(symbol);
+            [&](const Symbol &mySymbol) {
+              return &mySymbol == &*from.specificProcs_[i];
+            }) == specificProcs_.end()) {
+      specificProcs_.push_back(from.specificProcs_[i]);
+      bindingNames_.push_back(from.bindingNames_[i]);
     }
   }
 }
diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index e5ba3994b82dd..2607011333964 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -1217,13 +1217,6 @@ const Symbol *FindImmediateComponent(const DerivedTypeSpec &type,
   return nullptr;
 }
 
-bool IsFunctionResult(const Symbol &symbol) {
-  return (symbol.has<ObjectEntityDetails>() &&
-             symbol.get<ObjectEntityDetails>().isFuncResult()) ||
-      (symbol.has<ProcEntityDetails>() &&
-          symbol.get<ProcEntityDetails>().isFuncResult());
-}
-
 bool IsFunctionResultWithSameNameAsFunction(const Symbol &symbol) {
   if (IsFunctionResult(symbol)) {
     if (const Symbol * function{symbol.owner().symbol()}) {
@@ -1292,4 +1285,13 @@ void LabelEnforce::SayWithConstruct(SemanticsContext &context,
       .Attach(constructLocation, GetEnclosingConstructMsg());
 }
 
+bool HasAlternateReturns(const Symbol &subprogram) {
+  for (const auto *dummyArg : subprogram.get<SubprogramDetails>().dummyArgs()) {
+    if (!dummyArg) {
+      return true;
+    }
+  }
+  return false;
+}
+
 } // namespace Fortran::semantics
diff --git a/flang/module/.clang-format b/flang/module/.clang-format
new file mode 100644
index 0000000000000..e3845288a2aec
--- /dev/null
+++ b/flang/module/.clang-format
@@ -0,0 +1 @@
+DisableFormat: true
diff --git a/flang/runtime/connection.h b/flang/runtime/connection.h
index d45ccfb51901e..ba1e27c0cad24 100644
--- a/flang/runtime/connection.h
+++ b/flang/runtime/connection.h
@@ -26,9 +26,10 @@ inline bool IsRecordFile(Access a) { return a != Access::Stream; }
 // established in an OPEN statement.
 struct ConnectionAttributes {
   Access access{Access::Sequential}; // ACCESS='SEQUENTIAL', 'DIRECT', 'STREAM'
-  std::optional<std::int64_t> recordLength; // RECL= when fixed-length
   bool isUnformatted{false}; // FORM='UNFORMATTED'
   bool isUTF8{false}; // ENCODING='UTF-8'
+  bool isFixedRecordLength{false}; // RECL= on OPEN
+  std::optional<std::int64_t> recordLength; // RECL= or current record
 };
 
 struct ConnectionState : public ConnectionAttributes {
@@ -37,6 +38,12 @@ struct ConnectionState : public ConnectionAttributes {
   void HandleAbsolutePosition(std::int64_t);
   void HandleRelativePosition(std::int64_t);
 
+  void BeginRecord() {
+    positionInRecord = 0;
+    furthestPositionInRecord = 0;
+    leftTabLimit.reset();
+  }
+
   // Positions in a record file (sequential or direct, not stream)
   std::int64_t currentRecordNumber{1}; // 1 is first
   std::int64_t positionInRecord{0}; // offset in current record
diff --git a/flang/runtime/edit-input.cpp b/flang/runtime/edit-input.cpp
index 77bf100d2e21b..f710c1107ec16 100644
--- a/flang/runtime/edit-input.cpp
+++ b/flang/runtime/edit-input.cpp
@@ -13,14 +13,24 @@
 
 namespace Fortran::runtime::io {
 
+static std::optional<char32_t> PrepareInput(
+    IoStatementState &io, const DataEdit &edit, std::optional<int> &remaining) {
+  remaining.reset();
+  if (edit.descriptor == DataEdit::ListDirected) {
+    io.GetNextNonBlank();
+  } else {
+    if (edit.width.value_or(0) > 0) {
+      remaining = *edit.width;
+    }
+    io.SkipSpaces(remaining);
+  }
+  return io.NextInField(remaining);
+}
+
 static bool EditBOZInput(IoStatementState &io, const DataEdit &edit, void *n,
     int base, int totalBitSize) {
   std::optional<int> remaining;
-  if (edit.width) {
-    remaining = std::max(0, *edit.width);
-  }
-  io.SkipSpaces(remaining);
-  std::optional<char32_t> next{io.NextInField(remaining)};
+  std::optional<char32_t> next{PrepareInput(io, edit, remaining)};
   common::UnsignedInt128 value{0};
   for (; next; next = io.NextInField(remaining)) {
     char32_t ch{*next};
@@ -54,14 +64,7 @@ static bool EditBOZInput(IoStatementState &io, const DataEdit &edit, void *n,
 // Returns false if there's a '-' sign
 static bool ScanNumericPrefix(IoStatementState &io, const DataEdit &edit,
     std::optional<char32_t> &next, std::optional<int> &remaining) {
-  if (edit.descriptor != DataEdit::ListDirected && edit.width) {
-    remaining = std::max(0, *edit.width);
-  } else {
-    // list-directed, namelist, or (nonstandard) 0-width input editing
-    remaining.reset();
-  }
-  io.SkipSpaces(remaining);
-  next = io.NextInField(remaining);
+  next = PrepareInput(io, edit, remaining);
   bool negative{false};
   if (next) {
     negative = *next == '-';
@@ -310,11 +313,7 @@ bool EditLogicalInput(IoStatementState &io, const DataEdit &edit, bool &x) {
     return false;
   }
   std::optional<int> remaining;
-  if (edit.width) {
-    remaining = std::max(0, *edit.width);
-  }
-  io.SkipSpaces(remaining);
-  std::optional<char32_t> next{io.NextInField(remaining)};
+  std::optional<char32_t> next{PrepareInput(io, edit, remaining)};
   if (next && *next == '.') { // skip optional period
     next = io.NextInField(remaining);
   }
diff --git a/flang/runtime/edit-output.cpp b/flang/runtime/edit-output.cpp
index a162726a592bd..941c5ccf3d593 100644
--- a/flang/runtime/edit-output.cpp
+++ b/flang/runtime/edit-output.cpp
@@ -127,9 +127,9 @@ const char *RealOutputEditingBase::FormatExponent(
 bool RealOutputEditingBase::EmitPrefix(
     const DataEdit &edit, std::size_t length, std::size_t width) {
   if (edit.IsListDirected()) {
-    int prefixLength{edit.descriptor == DataEdit::ListDirectedRealPart
-            ? 2
-            : edit.descriptor == DataEdit::ListDirectedImaginaryPart ? 0 : 1};
+    int prefixLength{edit.descriptor == DataEdit::ListDirectedRealPart ? 2
+            : edit.descriptor == DataEdit::ListDirectedImaginaryPart   ? 0
+                                                                       : 1};
     int suffixLength{edit.descriptor == DataEdit::ListDirectedRealPart ||
                 edit.descriptor == DataEdit::ListDirectedImaginaryPart
             ? 1
diff --git a/flang/runtime/file.cpp b/flang/runtime/file.cpp
index 677b9675f47aa..19c86a9d4b82f 100644
--- a/flang/runtime/file.cpp
+++ b/flang/runtime/file.cpp
@@ -9,6 +9,7 @@
 #include "file.h"
 #include "magic-numbers.h"
 #include "memory.h"
+#include <algorithm>
 #include <cerrno>
 #include <cstring>
 #include <fcntl.h>
@@ -64,9 +65,11 @@ void OpenFile::Open(
     if (fd_ >= 0) {
       return;
     }
+    knownSize_.reset();
     break;
   case OpenStatus::New:
     flags |= O_CREAT | O_EXCL;
+    knownSize_ = 0;
     break;
   case OpenStatus::Scratch:
     if (path_.get()) {
@@ -74,15 +77,18 @@ void OpenFile::Open(
       path_.reset();
     }
     fd_ = openfile_mkstemp(handler);
+    knownSize_ = 0;
     return;
   case OpenStatus::Replace:
     flags |= O_CREAT | O_TRUNC;
+    knownSize_ = 0;
     break;
   case OpenStatus::Unknown:
     if (fd_ >= 0) {
       return;
     }
     flags |= O_CREAT;
+    knownSize_.reset();
     break;
   }
   // If we reach this point, we're opening a new file.
@@ -104,7 +110,6 @@ void OpenFile::Open(
     handler.SignalErrno();
   }
   pending_.reset();
-  knownSize_.reset();
   if (position == Position::Append && !RawSeekToEnd()) {
     handler.SignalErrno();
   }
@@ -152,17 +157,14 @@ std::size_t OpenFile::Read(FileOffset at, char *buffer, std::size_t minBytes,
   if (!Seek(at, handler)) {
     return 0;
   }
-  if (maxBytes < minBytes) {
-    minBytes = maxBytes;
-  }
+  minBytes = std::min(minBytes, maxBytes);
   std::size_t got{0};
   while (got < minBytes) {
     auto chunk{::read(fd_, buffer + got, maxBytes - got)};
     if (chunk == 0) {
       handler.SignalEnd();
       break;
-    }
-    if (chunk < 0) {
+    } else if (chunk < 0) {
       auto err{errno};
       if (err != EAGAIN && err != EWOULDBLOCK && err != EINTR) {
         handler.SignalError(err);
@@ -352,7 +354,9 @@ bool OpenFile::RawSeekToEnd() {
 
 int OpenFile::PendingResult(const Terminator &terminator, int iostat) {
   int id{nextId_++};
-  pending_.reset(&New<Pending>{}(terminator, id, iostat, std::move(pending_)));
+  pending_ = New<Pending>{terminator}(id, iostat, std::move(pending_));
   return id;
 }
+
+bool IsATerminal(int fd) { return ::isatty(fd); }
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/file.h b/flang/runtime/file.h
index 741a0bcffa3eb..17a5e910ecae8 100644
--- a/flang/runtime/file.h
+++ b/flang/runtime/file.h
@@ -39,6 +39,7 @@ class OpenFile {
   void set_mayPosition(bool yes) { mayPosition_ = yes; }
   FileOffset position() const { return position_; }
   bool isTerminal() const { return isTerminal_; }
+  std::optional<FileOffset> knownSize() const { return knownSize_; }
 
   bool IsOpen() const { return fd_ >= 0; }
   void Open(OpenStatus, Position, IoErrorHandler &);
@@ -94,5 +95,7 @@ class OpenFile {
   int nextId_;
   OwningPtr<Pending> pending_;
 };
+
+bool IsATerminal(int fd);
 } // namespace Fortran::runtime::io
 #endif // FORTRAN_RUNTIME_FILE_H_
diff --git a/flang/runtime/internal-unit.cpp b/flang/runtime/internal-unit.cpp
index c051100b9a185..9c29123f7ab97 100644
--- a/flang/runtime/internal-unit.cpp
+++ b/flang/runtime/internal-unit.cpp
@@ -17,6 +17,7 @@ namespace Fortran::runtime::io {
 template <Direction DIR>
 InternalDescriptorUnit<DIR>::InternalDescriptorUnit(
     Scalar scalar, std::size_t length) {
+  isFixedRecordLength = true;
   recordLength = length;
   endfileRecordNumber = 2;
   void *pointer{reinterpret_cast<void *>(const_cast<char *>(scalar))};
@@ -33,6 +34,7 @@ InternalDescriptorUnit<DIR>::InternalDescriptorUnit(
       terminator, that.SizeInBytes() <= d.SizeInBytes(maxRank, true, 0));
   new (&d) Descriptor{that};
   d.Check();
+  isFixedRecordLength = true;
   recordLength = d.ElementBytes();
   endfileRecordNumber = d.Elements() + 1;
 }
@@ -123,8 +125,7 @@ bool InternalDescriptorUnit<DIR>::AdvanceRecord(IoErrorHandler &handler) {
     }
   }
   ++currentRecordNumber;
-  positionInRecord = 0;
-  furthestPositionInRecord = 0;
+  BeginRecord();
   return true;
 }
 
@@ -132,8 +133,7 @@ template <Direction DIR>
 void InternalDescriptorUnit<DIR>::BackspaceRecord(IoErrorHandler &handler) {
   RUNTIME_CHECK(handler, currentRecordNumber > 1);
   --currentRecordNumber;
-  positionInRecord = 0;
-  furthestPositionInRecord = 0;
+  BeginRecord();
 }
 
 template class InternalDescriptorUnit<Direction::Output>;
diff --git a/flang/runtime/io-api.cpp b/flang/runtime/io-api.cpp
index db228e5bbf21c..4710a590ccab9 100644
--- a/flang/runtime/io-api.cpp
+++ b/flang/runtime/io-api.cpp
@@ -28,9 +28,10 @@ Cookie BeginInternalArrayListIO(const Descriptor &descriptor,
     void ** /*scratchArea*/, std::size_t /*scratchBytes*/,
     const char *sourceFile, int sourceLine) {
   Terminator oom{sourceFile, sourceLine};
-  return &New<InternalListIoStatementState<DIR>>{}(
-      oom, descriptor, sourceFile, sourceLine)
-              .ioStatementState();
+  return &New<InternalListIoStatementState<DIR>>{oom}(
+      descriptor, sourceFile, sourceLine)
+              .release()
+              ->ioStatementState();
 }
 
 Cookie IONAME(BeginInternalArrayListOutput)(const Descriptor &descriptor,
@@ -52,9 +53,10 @@ Cookie BeginInternalArrayFormattedIO(const Descriptor &descriptor,
     const char *format, std::size_t formatLength, void ** /*scratchArea*/,
     std::size_t /*scratchBytes*/, const char *sourceFile, int sourceLine) {
   Terminator oom{sourceFile, sourceLine};
-  return &New<InternalFormattedIoStatementState<DIR>>{}(
-      oom, descriptor, format, formatLength, sourceFile, sourceLine)
-              .ioStatementState();
+  return &New<InternalFormattedIoStatementState<DIR>>{oom}(
+      descriptor, format, formatLength, sourceFile, sourceLine)
+              .release()
+              ->ioStatementState();
 }
 
 Cookie IONAME(BeginInternalArrayFormattedOutput)(const Descriptor &descriptor,
@@ -78,9 +80,10 @@ Cookie BeginInternalFormattedIO(
     void ** /*scratchArea*/, std::size_t /*scratchBytes*/,
     const char *sourceFile, int sourceLine) {
   Terminator oom{sourceFile, sourceLine};
-  return &New<InternalFormattedIoStatementState<DIR>>{}(oom, internal,
-      internalLength, format, formatLength, sourceFile, sourceLine)
-              .ioStatementState();
+  return &New<InternalFormattedIoStatementState<DIR>>{oom}(
+      internal, internalLength, format, formatLength, sourceFile, sourceLine)
+              .release()
+              ->ioStatementState();
 }
 
 Cookie IONAME(BeginInternalFormattedOutput)(char *internal,
@@ -118,10 +121,12 @@ Cookie BeginExternalListIO(
     terminator.Crash("List-directed I/O attempted on unformatted file");
     return nullptr;
   }
+  IoErrorHandler handler{terminator};
+  unit.SetDirection(DIR, handler);
   IoStatementState &io{unit.BeginIoStatement<ExternalListIoStatementState<DIR>>(
       unit, sourceFile, sourceLine)};
   if constexpr (DIR == Direction::Input) {
-    io.AdvanceRecord();
+    unit.BeginReadingRecord(handler);
   }
   return &io;
 }
@@ -151,11 +156,13 @@ Cookie BeginExternalFormattedIO(const char *format, std::size_t formatLength,
     terminator.Crash("Formatted I/O attempted on unformatted file");
     return nullptr;
   }
+  IoErrorHandler handler{terminator};
+  unit.SetDirection(DIR, handler);
   IoStatementState &io{
       unit.BeginIoStatement<ExternalFormattedIoStatementState<DIR>>(
           unit, format, formatLength, sourceFile, sourceLine)};
   if constexpr (DIR == Direction::Input) {
-    io.AdvanceRecord();
+    unit.BeginReadingRecord(handler);
   }
   return &io;
 }
@@ -178,17 +185,19 @@ template <Direction DIR>
 Cookie BeginUnformattedIO(
     ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
   Terminator terminator{sourceFile, sourceLine};
-  ExternalFileUnit &file{
+  ExternalFileUnit &unit{
       ExternalFileUnit::LookUpOrCrash(unitNumber, terminator)};
-  if (!file.isUnformatted) {
+  if (!unit.isUnformatted) {
     terminator.Crash("Unformatted output attempted on formatted file");
   }
-  IoStatementState &io{file.BeginIoStatement<UnformattedIoStatementState<DIR>>(
-      file, sourceFile, sourceLine)};
+  IoStatementState &io{unit.BeginIoStatement<UnformattedIoStatementState<DIR>>(
+      unit, sourceFile, sourceLine)};
+  IoErrorHandler handler{terminator};
+  unit.SetDirection(DIR, handler);
   if constexpr (DIR == Direction::Input) {
-    io.AdvanceRecord();
+    unit.BeginReadingRecord(handler);
   } else {
-    if (file.access == Access::Sequential && !file.recordLength.has_value()) {
+    if (unit.access == Access::Sequential && !unit.isFixedRecordLength) {
       // Create space for (sub)record header to be completed by
       // UnformattedIoStatementState<Direction::Output>::EndIoStatement()
       io.Emit("\0\0\0\0", 4); // placeholder for record length header
@@ -222,8 +231,10 @@ Cookie IONAME(BeginOpenUnit)( // OPEN(without NEWUNIT=)
 Cookie IONAME(BeginOpenNewUnit)( // OPEN(NEWUNIT=j)
     const char *sourceFile, int sourceLine) {
   Terminator terminator{sourceFile, sourceLine};
-  return IONAME(BeginOpenUnit)(
-      ExternalFileUnit::NewUnit(terminator), sourceFile, sourceLine);
+  ExternalFileUnit &unit{ExternalFileUnit::LookUpOrCreate(
+      ExternalFileUnit::NewUnit(terminator), terminator)};
+  return &unit.BeginIoStatement<OpenStatementState>(
+      unit, false /*wasExtant*/, sourceFile, sourceLine);
 }
 
 Cookie IONAME(BeginClose)(
@@ -234,11 +245,48 @@ Cookie IONAME(BeginClose)(
   } else {
     // CLOSE(UNIT=bad unit) is just a no-op
     Terminator oom{sourceFile, sourceLine};
-    return &New<NoopCloseStatementState>{}(oom, sourceFile, sourceLine)
-                .ioStatementState();
+    return &New<NoopCloseStatementState>{oom}(sourceFile, sourceLine)
+                .release()
+                ->ioStatementState();
   }
 }
 
+Cookie IONAME(BeginFlush)(
+    ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
+  Terminator terminator{sourceFile, sourceLine};
+  ExternalFileUnit &unit{
+      ExternalFileUnit::LookUpOrCrash(unitNumber, terminator)};
+  return &unit.BeginIoStatement<ExternalMiscIoStatementState>(
+      unit, ExternalMiscIoStatementState::Flush, sourceFile, sourceLine);
+}
+
+Cookie IONAME(BeginBackspace)(
+    ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
+  Terminator terminator{sourceFile, sourceLine};
+  ExternalFileUnit &unit{
+      ExternalFileUnit::LookUpOrCrash(unitNumber, terminator)};
+  return &unit.BeginIoStatement<ExternalMiscIoStatementState>(
+      unit, ExternalMiscIoStatementState::Backspace, sourceFile, sourceLine);
+}
+
+Cookie IONAME(BeginEndfile)(
+    ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
+  Terminator terminator{sourceFile, sourceLine};
+  ExternalFileUnit &unit{
+      ExternalFileUnit::LookUpOrCrash(unitNumber, terminator)};
+  return &unit.BeginIoStatement<ExternalMiscIoStatementState>(
+      unit, ExternalMiscIoStatementState::Endfile, sourceFile, sourceLine);
+}
+
+Cookie IONAME(BeginRewind)(
+    ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
+  Terminator terminator{sourceFile, sourceLine};
+  ExternalFileUnit &unit{
+      ExternalFileUnit::LookUpOrCrash(unitNumber, terminator)};
+  return &unit.BeginIoStatement<ExternalMiscIoStatementState>(
+      unit, ExternalMiscIoStatementState::Rewind, sourceFile, sourceLine);
+}
+
 // Control list items
 
 void IONAME(EnableHandlers)(Cookie cookie, bool hasIoStat, bool hasErr,
@@ -384,7 +432,7 @@ bool IONAME(SetRec)(Cookie cookie, std::int64_t rec) {
         "REC= may not appear unless ACCESS='DIRECT'");
     return false;
   }
-  if (!connection.recordLength) {
+  if (!connection.isFixedRecordLength || !connection.recordLength) {
     io.GetIoErrorHandler().SignalError("RECL= was not specified");
     return false;
   }
@@ -636,10 +684,11 @@ bool IONAME(SetRecl)(Cookie cookie, std::size_t n) {
   if (n <= 0) {
     io.GetIoErrorHandler().SignalError("RECL= must be greater than zero");
   }
-  if (open->wasExtant() && open->unit().recordLength.has_value() &&
-      *open->unit().recordLength != static_cast<std::int64_t>(n)) {
+  if (open->wasExtant() && open->unit().isFixedRecordLength &&
+      open->unit().recordLength.value_or(n) != static_cast<std::int64_t>(n)) {
     open->SignalError("RECL= may not be changed for an open unit");
   }
+  open->unit().isFixedRecordLength = true;
   open->unit().recordLength = n;
   return true;
 }
@@ -750,7 +799,17 @@ bool IONAME(OutputUnformattedBlock)(
   if (auto *unf{io.get_if<UnformattedIoStatementState<Direction::Output>>()}) {
     return unf->Emit(x, length);
   }
-  io.GetIoErrorHandler().Crash("OutputUnformatted() called for an I/O "
+  io.GetIoErrorHandler().Crash("OutputUnformattedBlock() called for an I/O "
+                               "statement that is not unformatted output");
+  return false;
+}
+
+bool IONAME(InputUnformattedBlock)(Cookie cookie, char *x, std::size_t length) {
+  IoStatementState &io{*cookie};
+  if (auto *unf{io.get_if<UnformattedIoStatementState<Direction::Input>>()}) {
+    return unf->Receive(x, length);
+  }
+  io.GetIoErrorHandler().Crash("InputUnformattedBlock() called for an I/O "
                                "statement that is not unformatted output");
   return false;
 }
@@ -784,6 +843,35 @@ bool IONAME(InputInteger)(Cookie cookie, std::int64_t &n, int kind) {
   return false;
 }
 
+bool IONAME(OutputReal32)(Cookie cookie, float x) {
+  IoStatementState &io{*cookie};
+  if (!io.get_if<OutputStatementState>()) {
+    io.GetIoErrorHandler().Crash(
+        "OutputReal32() called for a non-output I/O statement");
+    return false;
+  }
+  if (auto edit{io.GetNextDataEdit()}) {
+    return RealOutputEditing<24>{io, x}.Edit(*edit);
+  }
+  return false;
+}
+
+bool IONAME(InputReal32)(Cookie cookie, float &x) {
+  IoStatementState &io{*cookie};
+  if (!io.get_if<InputStatementState>()) {
+    io.GetIoErrorHandler().Crash(
+        "InputReal32() called for a non-input I/O statement");
+    return false;
+  }
+  if (auto edit{io.GetNextDataEdit()}) {
+    if (edit->descriptor == DataEdit::ListDirectedNullValue) {
+      return true;
+    }
+    return EditRealInput<24>(io, *edit, reinterpret_cast<void *>(&x));
+  }
+  return false;
+}
+
 bool IONAME(OutputReal64)(Cookie cookie, double x) {
   IoStatementState &io{*cookie};
   if (!io.get_if<OutputStatementState>()) {
@@ -813,6 +901,18 @@ bool IONAME(InputReal64)(Cookie cookie, double &x) {
   return false;
 }
 
+bool IONAME(OutputComplex32)(Cookie cookie, float r, float z) {
+  IoStatementState &io{*cookie};
+  if (io.get_if<ListDirectedStatementState<Direction::Output>>()) {
+    DataEdit real, imaginary;
+    real.descriptor = DataEdit::ListDirectedRealPart;
+    imaginary.descriptor = DataEdit::ListDirectedImaginaryPart;
+    return RealOutputEditing<24>{io, r}.Edit(real) &&
+        RealOutputEditing<24>{io, z}.Edit(imaginary);
+  }
+  return IONAME(OutputReal32)(cookie, r) && IONAME(OutputReal32)(cookie, z);
+}
+
 bool IONAME(OutputComplex64)(Cookie cookie, double r, double z) {
   IoStatementState &io{*cookie};
   if (io.get_if<ListDirectedStatementState<Direction::Output>>()) {
diff --git a/flang/runtime/io-stmt.cpp b/flang/runtime/io-stmt.cpp
index 1e2e5f5ec5c80..a4d8af4f7211d 100644
--- a/flang/runtime/io-stmt.cpp
+++ b/flang/runtime/io-stmt.cpp
@@ -13,6 +13,7 @@
 #include "tools.h"
 #include "unit.h"
 #include <algorithm>
+#include <cstdio>
 #include <cstring>
 #include <limits>
 
@@ -182,10 +183,10 @@ int NoopCloseStatementState::EndIoStatement() {
 }
 
 template <Direction DIR> int ExternalIoStatementState<DIR>::EndIoStatement() {
+  if (!unit().nonAdvancing) {
+    unit().AdvanceRecord(*this);
+  }
   if constexpr (DIR == Direction::Output) {
-    if (!unit().nonAdvancing) {
-      unit().AdvanceRecord(*this);
-    }
     unit().FlushIfTerminal(*this);
   }
   return ExternalIoStatementBase::EndIoStatement();
@@ -291,7 +292,7 @@ void IoStatementState::BackspaceRecord() {
 }
 
 void IoStatementState::HandleRelativePosition(std::int64_t n) {
-  return std::visit([=](auto &x) { x.get().HandleRelativePosition(n); }, u_);
+  std::visit([=](auto &x) { x.get().HandleRelativePosition(n); }, u_);
 }
 
 int IoStatementState::EndIoStatement() {
@@ -347,15 +348,22 @@ bool IoStatementState::EmitField(
   }
 }
 
-void IoStatementState::SkipSpaces(std::optional<int> &remaining) {
-  if (!remaining || *remaining > 0) {
-    for (auto ch{GetCurrentChar()}; ch && ch == ' '; ch = GetCurrentChar()) {
+std::optional<char32_t> IoStatementState::SkipSpaces(
+    std::optional<int> &remaining) {
+  while (!remaining || *remaining > 0) {
+    if (auto ch{GetCurrentChar()}) {
+      if (*ch != ' ') {
+        return ch;
+      }
       HandleRelativePosition(1);
-      if (remaining && !--*remaining) {
-        break;
+      if (remaining) {
+        --*remaining;
       }
+    } else {
+      break;
     }
   }
+  return std::nullopt;
 }
 
 std::optional<char32_t> IoStatementState::NextInField(
@@ -372,6 +380,7 @@ std::optional<char32_t> IoStatementState::NextInField(
       case '\'':
       case '"':
       case '*':
+      case '\n': // for stream access
         break;
       default:
         HandleRelativePosition(1);
@@ -385,7 +394,8 @@ std::optional<char32_t> IoStatementState::NextInField(
       return next;
     }
     const ConnectionState &connection{GetConnectionState()};
-    if (!connection.IsAtEOF() && connection.recordLength &&
+    if (!connection.IsAtEOF() && connection.isFixedRecordLength &&
+        connection.recordLength &&
         connection.positionInRecord >= *connection.recordLength) {
       if (connection.modes.pad) { // PAD='YES'
         --*remaining;
@@ -553,29 +563,39 @@ ListDirectedStatementState<Direction::Input>::GetNextDataEdit(
   return edit;
 }
 
+template <Direction DIR>
+bool UnformattedIoStatementState<DIR>::Receive(char *data, std::size_t bytes) {
+  if constexpr (DIR == Direction::Output) {
+    this->Crash(
+        "UnformattedIoStatementState::Receive() called for output statement");
+  }
+  return this->unit().Receive(data, bytes, *this);
+}
+
 template <Direction DIR>
 int UnformattedIoStatementState<DIR>::EndIoStatement() {
-  auto &ext{static_cast<ExternalIoStatementState<DIR> &>(*this)};
-  ExternalFileUnit &unit{ext.unit()};
-  if (unit.access == Access::Sequential && !unit.recordLength.has_value()) {
-    // Overwrite the first four bytes of the record with its length,
-    // and also append the length.  These four bytes were skipped over
-    // in BeginUnformattedOutput().
-    // TODO: Break very large records up into subrecords with negative
-    // headers &/or footers
-    union {
-      std::uint32_t u;
-      char c[sizeof u];
-    } u;
-    u.u = unit.furthestPositionInRecord - sizeof u.c;
-    // TODO: Convert record length to little-endian on big-endian host?
-    if (!(ext.Emit(u.c, sizeof u.c) &&
-            (ext.HandleAbsolutePosition(0), ext.Emit(u.c, sizeof u.c)) &&
-            ext.AdvanceRecord())) {
-      return false;
+  ExternalFileUnit &unit{this->unit()};
+  if constexpr (DIR == Direction::Output) {
+    if (unit.access == Access::Sequential && !unit.isFixedRecordLength) {
+      // Append the length of a sequential unformatted variable-length record
+      // as its footer, then overwrite the reserved first four bytes of the
+      // record with its length as its header.  These four bytes were skipped
+      // over in BeginUnformattedOutput().
+      // TODO: Break very large records up into subrecords with negative
+      // headers &/or footers
+      union {
+        std::uint32_t u;
+        char c[sizeof u];
+      } u;
+      u.u = unit.furthestPositionInRecord - sizeof u;
+      // TODO: Convert record length to little-endian on big-endian host?
+      if (!(this->Emit(u.c, sizeof u) &&
+              (this->HandleAbsolutePosition(0), this->Emit(u.c, sizeof u)))) {
+        return false;
+      }
     }
   }
-  return ext.EndIoStatement();
+  return ExternalIoStatementState<DIR>::EndIoStatement();
 }
 
 template class InternalIoStatementState<Direction::Output>;
@@ -592,4 +612,25 @@ template class ExternalListIoStatementState<Direction::Output>;
 template class ExternalListIoStatementState<Direction::Input>;
 template class UnformattedIoStatementState<Direction::Output>;
 template class UnformattedIoStatementState<Direction::Input>;
+
+int ExternalMiscIoStatementState::EndIoStatement() {
+  ExternalFileUnit &ext{unit()};
+  switch (which_) {
+  case Flush:
+    ext.Flush(*this);
+    std::fflush(nullptr); // flushes C stdio output streams (12.9(2))
+    break;
+  case Backspace:
+    ext.BackspaceRecord(*this);
+    break;
+  case Endfile:
+    ext.Endfile(*this);
+    break;
+  case Rewind:
+    ext.Rewind(*this);
+    break;
+  }
+  return ExternalIoStatementBase::EndIoStatement();
+}
+
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/io-stmt.h b/flang/runtime/io-stmt.h
index b8145d79a96a2..066391bd15664 100644
--- a/flang/runtime/io-stmt.h
+++ b/flang/runtime/io-stmt.h
@@ -36,6 +36,7 @@ template <Direction, typename CHAR = char>
 class ExternalFormattedIoStatementState;
 template <Direction> class ExternalListIoStatementState;
 template <Direction> class UnformattedIoStatementState;
+class ExternalMiscIoStatementState;
 
 // The Cookie type in the I/O API is a pointer (for C) to this class.
 class IoStatementState {
@@ -73,7 +74,8 @@ class IoStatementState {
 
   bool EmitRepeated(char, std::size_t);
   bool EmitField(const char *, std::size_t length, std::size_t width);
-  void SkipSpaces(std::optional<int> &remaining);
+
+  std::optional<char32_t> SkipSpaces(std::optional<int> &remaining);
   std::optional<char32_t> NextInField(std::optional<int> &remaining);
   std::optional<char32_t> GetNextNonBlank(); // can advance record
 
@@ -94,7 +96,8 @@ class IoStatementState {
       std::reference_wrapper<ExternalListIoStatementState<Direction::Output>>,
       std::reference_wrapper<ExternalListIoStatementState<Direction::Input>>,
       std::reference_wrapper<UnformattedIoStatementState<Direction::Output>>,
-      std::reference_wrapper<UnformattedIoStatementState<Direction::Input>>>
+      std::reference_wrapper<UnformattedIoStatementState<Direction::Input>>,
+      std::reference_wrapper<ExternalMiscIoStatementState>>
       u_;
 };
 
@@ -235,7 +238,7 @@ class ExternalIoStatementState : public ExternalIoStatementBase,
 public:
   using ExternalIoStatementBase::ExternalIoStatementBase;
   int EndIoStatement();
-  bool Emit(const char *, std::size_t chars /* not bytes */);
+  bool Emit(const char *, std::size_t);
   bool Emit(const char16_t *, std::size_t chars /* not bytes */);
   bool Emit(const char32_t *, std::size_t chars /* not bytes */);
   std::optional<char32_t> GetCurrentChar();
@@ -280,6 +283,7 @@ template <Direction DIR>
 class UnformattedIoStatementState : public ExternalIoStatementState<DIR> {
 public:
   using ExternalIoStatementState<DIR>::ExternalIoStatementState;
+  bool Receive(char *, std::size_t);
   int EndIoStatement();
 };
 
@@ -353,5 +357,17 @@ extern template class FormatControl<
 extern template class FormatControl<
     ExternalFormattedIoStatementState<Direction::Input>>;
 
+class ExternalMiscIoStatementState : public ExternalIoStatementBase {
+public:
+  enum Which { Flush, Backspace, Endfile, Rewind };
+  ExternalMiscIoStatementState(ExternalFileUnit &unit, Which which,
+      const char *sourceFile = nullptr, int sourceLine = 0)
+      : ExternalIoStatementBase{unit, sourceFile, sourceLine}, which_{which} {}
+  int EndIoStatement();
+
+private:
+  Which which_;
+};
+
 } // namespace Fortran::runtime::io
 #endif // FORTRAN_RUNTIME_IO_STMT_H_
diff --git a/flang/runtime/iostat.cpp b/flang/runtime/iostat.cpp
index c86619d384f93..9fe6d22de85de 100644
--- a/flang/runtime/iostat.cpp
+++ b/flang/runtime/iostat.cpp
@@ -33,6 +33,26 @@ const char *IostatErrorString(int iostat) {
     return "Invalid FORMAT";
   case IostatErrorInKeyword:
     return "Bad keyword argument value";
+  case IostatEndfileNonSequential:
+    return "ENDFILE on non-sequential file";
+  case IostatEndfileUnwritable:
+    return "ENDFILE on read-only file";
+  case IostatOpenBadRecl:
+    return "OPEN with bad RECL= value";
+  case IostatOpenUnknownSize:
+    return "OPEN of file of unknown size";
+  case IostatOpenBadAppend:
+    return "OPEN(POSITION='APPEND') of unpositionable file";
+  case IostatWriteToReadOnly:
+    return "Attempted output to read-only file";
+  case IostatReadFromWriteOnly:
+    return "Attempted input from write-only file";
+  case IostatBackspaceNonSequential:
+    return "BACKSPACE on non-sequential file";
+  case IostatBackspaceAtFirstRecord:
+    return "BACKSPACE at first record";
+  case IostatRewindNonSequential:
+    return "REWIND on non-sequential file";
   default:
     return nullptr;
   }
diff --git a/flang/runtime/iostat.h b/flang/runtime/iostat.h
index 2ab5ac8232d2c..f51636d5f7d07 100644
--- a/flang/runtime/iostat.h
+++ b/flang/runtime/iostat.h
@@ -45,6 +45,16 @@ enum Iostat {
   IostatInternalWriteOverrun,
   IostatErrorInFormat,
   IostatErrorInKeyword,
+  IostatEndfileNonSequential,
+  IostatEndfileUnwritable,
+  IostatOpenBadRecl,
+  IostatOpenUnknownSize,
+  IostatOpenBadAppend,
+  IostatWriteToReadOnly,
+  IostatReadFromWriteOnly,
+  IostatBackspaceNonSequential,
+  IostatBackspaceAtFirstRecord,
+  IostatRewindNonSequential,
 };
 
 const char *IostatErrorString(int);
diff --git a/flang/runtime/memory.h b/flang/runtime/memory.h
index f26213f954a49..f21b237f3905b 100644
--- a/flang/runtime/memory.h
+++ b/flang/runtime/memory.h
@@ -20,7 +20,7 @@ class Terminator;
 
 [[nodiscard]] void *AllocateMemoryOrCrash(
     const Terminator &, std::size_t bytes);
-template <typename A>[[nodiscard]] A &AllocateOrCrash(const Terminator &t) {
+template <typename A> [[nodiscard]] A &AllocateOrCrash(const Terminator &t) {
   return *reinterpret_cast<A *>(AllocateMemoryOrCrash(t, sizeof(A)));
 }
 void FreeMemory(void *);
@@ -32,20 +32,32 @@ template <typename A> void FreeMemoryAndNullify(A *&p) {
   p = nullptr;
 }
 
-template <typename A> struct New {
-  template <typename... X>
-  [[nodiscard]] A &operator()(const Terminator &terminator, X &&... x) {
-    return *new (AllocateMemoryOrCrash(terminator, sizeof(A)))
-        A{std::forward<X>(x)...};
-  }
-};
-
 template <typename A> struct OwningPtrDeleter {
   void operator()(A *p) { FreeMemory(p); }
 };
 
 template <typename A> using OwningPtr = std::unique_ptr<A, OwningPtrDeleter<A>>;
 
+template <typename A> class SizedNew {
+public:
+  explicit SizedNew(const Terminator &terminator) : terminator_{terminator} {}
+  template <typename... X>
+  [[nodiscard]] OwningPtr<A> operator()(std::size_t bytes, X &&... x) {
+    return OwningPtr<A>{new (AllocateMemoryOrCrash(terminator_, bytes))
+            A{std::forward<X>(x)...}};
+  }
+
+private:
+  const Terminator &terminator_;
+};
+
+template <typename A> struct New : public SizedNew<A> {
+  using SizedNew<A>::SizedNew;
+  template <typename... X> [[nodiscard]] OwningPtr<A> operator()(X &&... x) {
+    return SizedNew<A>::operator()(sizeof(A), std::forward<X>(x)...);
+  }
+};
+
 template <typename A> struct Allocator {
   using value_type = A;
   explicit Allocator(const Terminator &t) : terminator{t} {}
diff --git a/flang/runtime/pgmath.h.inc b/flang/runtime/pgmath.h.inc
new file mode 100644
index 0000000000000..061a47431e6d4
--- /dev/null
+++ b/flang/runtime/pgmath.h.inc
@@ -0,0 +1,252 @@
+//===-- runtime/pgmath.h.inc -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// This file defines the interface of libpgmath to be used for folding
+// and code generation.
+// Usage:
+// define PGMATH_DECLARE if you simply want to declare pgmath interface.
+// Define the PGMATH_USE_S/D/C/Z(intrinsic name, function name) according
+// to what needs to be done with the runtime declaration.
+// This intrinsic will be called on all libpgmath function in the
+// intrinsic alphabetical order.
+// Define PGMATH_FAST/RELAXED/PRECISE to restrict the PGMATH_USE visit
+// to the targeted versions.
+// Define PGMATH_USE_OTHER to visit math functions that are not related to
+// floating points (e.g. int**int pow).
+
+// Control Macros
+#ifdef PGMATH_DECLARE
+#undef PGMATH_DECLARE
+#define PGMATH_DECLARE(x) extern "C" x;
+#define PGMATH_FAST
+#define PGMATH_PRECISE
+#define PGMATH_RELAXED
+#else
+#define PGMATH_DECLARE(x)
+#endif
+
+#ifdef PGMATH_USE_ALL_TYPES
+#define PGMATH_USE_S(name, func) PGMATH_USE_ALL_TYPES(name, func)
+#define PGMATH_USE_D(name, func) PGMATH_USE_ALL_TYPES(name, func)
+#define PGMATH_USE_OTHER(name, func) PGMATH_USE_ALL_TYPES(name, func)
+#endif
+
+#ifndef PGMATH_USE_S
+#define PGMATH_USE_S(name, x)
+#endif
+
+#ifndef PGMATH_USE_D
+#define PGMATH_USE_D(name, x)
+#endif
+
+#ifndef PGMATH_USE_C
+#define PGMATH_USE_C(name, x)
+#endif
+
+#ifndef PGMATH_USE_Z
+#define PGMATH_USE_Z(name, x)
+#endif
+
+#ifndef PGMATH_USE_OTHER
+#define PGMATH_USE_OTHER(name, x)
+#endif
+
+#define PGMATH_REAL_IMPL(impl, func) \
+  PGMATH_DECLARE(float __##impl##s_##func##_1(float)) \
+  PGMATH_DECLARE(double __##impl##d_##func##_1(double)) \
+  PGMATH_USE_S(func, __##impl##s_##func##_1) \
+  PGMATH_USE_D(func, __##impl##d_##func##_1)
+
+#define PGMATH_ALL_FP_IMPL(impl, func) \
+  PGMATH_REAL_IMPL(impl, func) \
+
+#define PGMATH_REAL2_IMPL(impl, func) \
+  PGMATH_DECLARE(float __##impl##s_##func##_1(float, float)) \
+  PGMATH_DECLARE(double __##impl##d_##func##_1(double, double)) \
+  PGMATH_USE_S(func, __##impl##s_##func##_1) \
+  PGMATH_USE_D(func, __##impl##d_##func##_1)
+
+#define PGMATH_ALL_FP2_IMPL(impl, func) \
+  PGMATH_REAL2_IMPL(func) \
+
+#undef PGMATH_FAST_REAL
+#undef PGMATH_FAST_COMPLEX
+#undef PGMATH_FAST_ALL_FP
+#undef PGMATH_FAST_REAL2
+#undef PGMATH_FAST_COMPLEX2
+#undef PGMATH_FAST_ALL_FP2
+#ifdef PGMATH_FAST
+#define PGMATH_FAST_REAL(func) PGMATH_REAL_IMPL(f, func)
+#define PGMATH_FAST_ALL_FP(func) PGMATH_ALL_IMPL(f, func)
+#define PGMATH_FAST_REAL2(func) PGMATH_REAL2_IMPL(f, func)
+#define PGMATH_FAST_ALL_FP2(func) PGMATH_ALL_FP2_IMPL(f, func)
+#else
+#define PGMATH_FAST_REAL(func)
+#define PGMATH_FAST_COMPLEX(func)
+#define PGMATH_FAST_ALL_FP(func)
+#define PGMATH_FAST_REAL2(func)
+#define PGMATH_FAST_COMPLEX2(func)
+#define PGMATH_FAST_ALL_FP2(func)
+#endif
+
+#undef PGMATH_RELAXED_REAL
+#undef PGMATH_RELAXED_COMPLEX
+#undef PGMATH_RELAXED_ALL_FP
+#undef PGMATH_RELAXED_REAL2
+#undef PGMATH_RELAXED_COMPLEX2
+#undef PGMATH_RELAXED_ALL_FP2
+#ifdef PGMATH_RELAXED
+#define PGMATH_RELAXED_REAL(func) PGMATH_REAL_IMPL(r, func)
+#define PGMATH_RELAXED_ALL_FP(func) PGMATH_ALL_IMPL(r, func)
+#define PGMATH_RELAXED_REAL2(func) PGMATH_REAL2_IMPL(r, func)
+#define PGMATH_RELAXED_ALL_FP2(func) PGMATH_ALL_FP2_IMPL(r, func)
+#else
+#define PGMATH_RELAXED_REAL(func)
+#define PGMATH_RELAXED_COMPLEX(func)
+#define PGMATH_RELAXED_ALL_FP(func)
+#define PGMATH_RELAXED_REAL2(func)
+#define PGMATH_RELAXED_COMPLEX2(func)
+#define PGMATH_RELAXED_ALL_FP2(func)
+#endif
+
+#undef PGMATH_PRECISE_REAL
+#undef PGMATH_PRECISE_COMPLEX
+#undef PGMATH_PRECISE_ALL_FP
+#undef PGMATH_PRECISE_REAL2
+#undef PGMATH_PRECISE_COMPLEX2
+#undef PGMATH_PRECISE_ALL_FP2
+#ifdef PGMATH_PRECISE
+#define PGMATH_PRECISE_REAL(func) PGMATH_REAL_IMPL(p, func)
+#define PGMATH_PRECISE_ALL_FP(func) PGMATH_ALL_IMPL(p, func)
+#define PGMATH_PRECISE_REAL2(func) PGMATH_REAL2_IMPL(p, func)
+#define PGMATH_PRECISE_ALL_FP2(func) PGMATH_ALL_FP2_IMPL(p, func)
+#else
+#define PGMATH_PRECISE_REAL(func)
+#define PGMATH_PRECISE_COMPLEX(func)
+#define PGMATH_PRECISE_ALL_FP(func)
+#define PGMATH_PRECISE_REAL2(func)
+#define PGMATH_PRECISE_COMPLEX2(func)
+#define PGMATH_PRECISE_ALL_FP2(func)
+#endif
+
+#define PGMATH_REAL(func) \
+  PGMATH_FAST_REAL(func) \
+  PGMATH_PRECISE_REAL(func) \
+  PGMATH_RELAXED_REAL(func)
+
+#define PGMATH_ALL(func) \
+  PGMATH_REAL(func) \
+
+#define PGMATH_REAL2(func) \
+  PGMATH_FAST_REAL2(func) \
+  PGMATH_PRECISE_REAL2(func) \
+  PGMATH_RELAXED_REAL2(func)
+
+#define PGMATH_ALL2(func) \
+  PGMATH_REAL2(func) \
+
+// Marcos to declare __mth_i libpgmath variants
+#define PGMATH_MTH_VERSION_REAL(func) \
+  PGMATH_DECLARE(float __mth_i_##func(float)) \
+  PGMATH_DECLARE(double __mth_i_d##func(double)) \
+  PGMATH_USE_S(func, __mth_i_##func) \
+  PGMATH_USE_D(func, __mth_i_d##func)
+
+// Actual libpgmath declarations
+PGMATH_ALL(acos)
+PGMATH_MTH_VERSION_REAL(acosh)
+PGMATH_ALL(asin)
+PGMATH_MTH_VERSION_REAL(asinh)
+PGMATH_ALL(atan)
+PGMATH_REAL2(atan2)
+PGMATH_MTH_VERSION_REAL(atanh)
+PGMATH_MTH_VERSION_REAL(bessel_j0)
+PGMATH_MTH_VERSION_REAL(bessel_j1)
+PGMATH_MTH_VERSION_REAL(bessel_y0)
+PGMATH_MTH_VERSION_REAL(bessel_y1)
+// bessel_jn and bessel_yn takes an int as first arg
+PGMATH_DECLARE(float __mth_i_bessel_jn(int, float))
+PGMATH_DECLARE(double __mth_i_dbessel_jn(int, double))
+PGMATH_USE_S(bessel_jn, __mth_i_bessel_jn)
+PGMATH_USE_D(bessel_jn, __mth_i_dbessel_jn)
+PGMATH_DECLARE(float __mth_i_bessel_yn(int, float))
+PGMATH_DECLARE(double __mth_i_dbessel_yn(int, double))
+PGMATH_USE_S(bessel_yn, __mth_i_bessel_yn)
+PGMATH_USE_D(bessel_yn, __mth_i_dbessel_yn)
+
+PGMATH_ALL(cos)
+PGMATH_ALL(cosh)
+PGMATH_MTH_VERSION_REAL(erf)
+PGMATH_MTH_VERSION_REAL(erfc)
+PGMATH_MTH_VERSION_REAL(erfc_scaled)
+PGMATH_ALL(exp)
+PGMATH_MTH_VERSION_REAL(gamma)
+
+PGMATH_DECLARE(float __mth_i_hypot(float, float))
+PGMATH_DECLARE(double __mth_i_dhypot(double, double))
+PGMATH_USE_S(hypot, __mth_i_hypot)
+PGMATH_USE_D(hypot, __mth_i_dhypot)
+
+PGMATH_ALL(log)
+PGMATH_REAL(log10)
+PGMATH_MTH_VERSION_REAL(log_gamma)
+// no function for modulo in libpgmath.
+// fast mod used in all versions.
+PGMATH_DECLARE(float __fs_mod_1(float, float))
+PGMATH_DECLARE(double __fd_mod_1(double, double))
+PGMATH_USE_S(mod, __fs_mod_1)
+PGMATH_USE_D(mod, __fd_mod_1)
+
+PGMATH_ALL2(pow)
+// Versions of pow with integer exponents
+#define PGMATH_DELCARE_POW(impl) \
+  PGMATH_DECLARE(float __##impl##s_powi_1(float, int)) \
+  PGMATH_DECLARE(double __##impl##d_powi_1(double, int)) \
+  PGMATH_USE_S(pow, __##impl##s_powi_1) \
+  PGMATH_USE_D(pow, __##impl##d_powi_1) \
+  PGMATH_USE_C(pow, __##impl##c_powi_1) \
+  PGMATH_USE_Z(pow, __##impl##z_powi_1) \
+  PGMATH_DECLARE(float __##impl##s_powk_1(float, int64_t)) \
+  PGMATH_DECLARE(double __##impl##d_powk_1(double, int64_t)) \
+  PGMATH_USE_S(pow, __##impl##s_powk_1) \
+  PGMATH_USE_D(pow, __##impl##d_powk_1) \
+  PGMATH_USE_C(pow, __##impl##c_powk_1) \
+  PGMATH_USE_Z(pow, __##impl##z_powk_1)
+
+#ifdef PGMATH_FAST
+PGMATH_DELCARE_POW(f)
+#endif
+#ifdef PGMATH_RELAXED
+PGMATH_DELCARE_POW(r)
+#endif
+#ifdef PGMATH_PRECISE
+PGMATH_DELCARE_POW(p)
+#endif
+
+// integer ** integer versions of pow
+PGMATH_DECLARE(int __mth_i_ipowi(int, int))
+PGMATH_DECLARE(int64_t __mth_i_kpowk(int64_t, int64_t))
+PGMATH_USE_OTHER(pow, __mth_i_ipowi)
+PGMATH_USE_OTHER(pow, __mth_i_kpowk)
+
+PGMATH_ALL(sin)
+PGMATH_ALL(sinh)
+PGMATH_MTH_VERSION_REAL(sqrt)
+PGMATH_ALL(tan)
+PGMATH_ALL(tanh)
+
+#undef PGMATH_DECLARE
+#undef PGMATH_FAST
+#undef PGMATH_PRECISE
+#undef PGMATH_RELAXED
+#undef PGMATH_USE_S
+#undef PGMATH_USE_D
+#undef PGMATH_USE_C
+#undef PGMATH_USE_Z
+#undef PGMATH_USE_OTHER
+#undef PGMATH_USE_ALL_TYPES
diff --git a/flang/runtime/stop.cpp b/flang/runtime/stop.cpp
index cd91b77cff1c0..49592b75a2ade 100644
--- a/flang/runtime/stop.cpp
+++ b/flang/runtime/stop.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "stop.h"
+#include "file.h"
 #include "io-error.h"
 #include "terminator.h"
 #include "unit.h"
@@ -71,6 +72,19 @@ static void CloseAllExternalUnits(const char *why) {
   std::exit(EXIT_FAILURE);
 }
 
+void RTNAME(PauseStatement)() {
+  if (Fortran::runtime::io::IsATerminal(0)) {
+    Fortran::runtime::io::IoErrorHandler handler{"PAUSE statement"};
+    Fortran::runtime::io::ExternalFileUnit::FlushAll(handler);
+    std::fputs("Fortran PAUSE: hit RETURN to continue:", stderr);
+    std::fflush(nullptr);
+    if (std::fgetc(stdin) == EOF) {
+      CloseAllExternalUnits("PAUSE statement");
+      std::exit(EXIT_SUCCESS);
+    }
+  }
+}
+
 [[noreturn]] void RTNAME(FailImageStatement)() {
   Fortran::runtime::NotifyOtherImagesOfFailImageStatement();
   CloseAllExternalUnits("FAIL IMAGE statement");
diff --git a/flang/runtime/stop.h b/flang/runtime/stop.h
index 249e71e0b1f00..3d5f22e5761a2 100644
--- a/flang/runtime/stop.h
+++ b/flang/runtime/stop.h
@@ -20,6 +20,7 @@ NORETURN void RTNAME(StopStatement)(int code DEFAULT_VALUE(EXIT_SUCCESS),
     bool isErrorStop DEFAULT_VALUE(false), bool quiet DEFAULT_VALUE(false));
 NORETURN void RTNAME(StopStatementText)(const char *,
     bool isErrorStop DEFAULT_VALUE(false), bool quiet DEFAULT_VALUE(false));
+void RTNAME(PauseStatement)(NO_ARGUMENTS);
 NORETURN void RTNAME(FailImageStatement)(NO_ARGUMENTS);
 NORETURN void RTNAME(ProgramEndStatement)(NO_ARGUMENTS);
 
diff --git a/flang/runtime/terminator.cpp b/flang/runtime/terminator.cpp
index cea1a8673eeb6..e2ea80d3aa55d 100644
--- a/flang/runtime/terminator.cpp
+++ b/flang/runtime/terminator.cpp
@@ -18,17 +18,18 @@ namespace Fortran::runtime {
   CrashArgs(message, ap);
 }
 
-static void (*crashHandler)(const char *, va_list &){nullptr};
+static void (*crashHandler)(const char *, int, const char *, va_list &){
+    nullptr};
 
 void Terminator::RegisterCrashHandler(
-    void (*handler)(const char *, va_list &)) {
+    void (*handler)(const char *, int, const char *, va_list &)) {
   crashHandler = handler;
 }
 
 [[noreturn]] void Terminator::CrashArgs(
     const char *message, va_list &ap) const {
   if (crashHandler) {
-    crashHandler(message, ap);
+    crashHandler(sourceFileName_, sourceLine_, message, ap);
   }
   std::fputs("\nfatal Fortran runtime error", stderr);
   if (sourceFileName_) {
diff --git a/flang/runtime/terminator.h b/flang/runtime/terminator.h
index 66bc92ae4e251..33a6d5d2c1560 100644
--- a/flang/runtime/terminator.h
+++ b/flang/runtime/terminator.h
@@ -34,7 +34,8 @@ class Terminator {
       const char *predicate, const char *file, int line) const;
 
   // For test harnessing - overrides CrashArgs().
-  static void RegisterCrashHandler(void (*)(const char *, va_list &));
+  static void RegisterCrashHandler(void (*)(const char *sourceFile,
+      int sourceLine, const char *message, va_list &ap));
 
 private:
   const char *sourceFileName_{nullptr};
diff --git a/flang/runtime/transformational.cpp b/flang/runtime/transformational.cpp
index eabb08202af7c..69de8ffc57ab7 100644
--- a/flang/runtime/transformational.cpp
+++ b/flang/runtime/transformational.cpp
@@ -9,7 +9,6 @@
 #include "transformational.h"
 #include "memory.h"
 #include "terminator.h"
-#include "flang/Evaluate/integer.h"
 #include <algorithm>
 #include <cinttypes>
 
diff --git a/flang/runtime/unit-map.cpp b/flang/runtime/unit-map.cpp
index 4e58cf590c2ac..905beb4d084fa 100644
--- a/flang/runtime/unit-map.cpp
+++ b/flang/runtime/unit-map.cpp
@@ -63,8 +63,17 @@ void UnitMap::CloseAll(IoErrorHandler &handler) {
   }
 }
 
+void UnitMap::FlushAll(IoErrorHandler &handler) {
+  CriticalSection critical{lock_};
+  for (int j{0}; j < buckets_; ++j) {
+    for (Chain *p{bucket_[j].get()}; p; p = p->next.get()) {
+      p->unit.Flush(handler);
+    }
+  }
+}
+
 ExternalFileUnit &UnitMap::Create(int n, const Terminator &terminator) {
-  Chain &chain{New<Chain>{}(terminator, n)};
+  Chain &chain{*New<Chain>{terminator}(n).release()};
   chain.next.reset(&chain);
   bucket_[Hash(n)].swap(chain.next); // pushes new node as list head
   return chain.unit;
diff --git a/flang/runtime/unit-map.h b/flang/runtime/unit-map.h
index 8b8afb3b46388..9efb2698d2233 100644
--- a/flang/runtime/unit-map.h
+++ b/flang/runtime/unit-map.h
@@ -15,6 +15,7 @@
 #include "lock.h"
 #include "memory.h"
 #include "unit.h"
+#include <cstdlib>
 
 namespace Fortran::runtime::io {
 
@@ -49,6 +50,7 @@ class UnitMap {
 
   void DestroyClosed(ExternalFileUnit &);
   void CloseAll(IoErrorHandler &);
+  void FlushAll(IoErrorHandler &);
 
 private:
   struct Chain {
@@ -58,7 +60,7 @@ class UnitMap {
   };
 
   static constexpr int buckets_{1031}; // must be prime
-  int Hash(int n) { return n % buckets_; }
+  int Hash(int n) { return std::abs(n) % buckets_; }
 
   ExternalFileUnit *Find(int n) {
     Chain *previous{nullptr};
diff --git a/flang/runtime/unit.cpp b/flang/runtime/unit.cpp
index 81035ab6fcde8..2193ee0f6aca7 100644
--- a/flang/runtime/unit.cpp
+++ b/flang/runtime/unit.cpp
@@ -17,6 +17,7 @@ namespace Fortran::runtime::io {
 // should work without a Fortran main program.
 static Lock unitMapLock;
 static UnitMap *unitMap{nullptr};
+static ExternalFileUnit *defaultInput{nullptr};
 static ExternalFileUnit *defaultOutput{nullptr};
 
 void FlushOutputOnCrash(const Terminator &terminator) {
@@ -70,14 +71,48 @@ void ExternalFileUnit::OpenUnit(OpenStatus status, Position position,
       return;
     }
     // Otherwise, OPEN on open unit with new FILE= implies CLOSE
+    DoImpliedEndfile(handler);
     Flush(handler);
     Close(CloseStatus::Keep, handler);
   }
   set_path(std::move(newPath), newPathLength);
   Open(status, position, handler);
+  auto totalBytes{knownSize()};
+  if (access == Access::Direct) {
+    if (!isFixedRecordLength || !recordLength) {
+      handler.SignalError(IostatOpenBadRecl,
+          "OPEN(UNIT=%d,ACCESS='DIRECT'): record length is not known",
+          unitNumber());
+    } else if (*recordLength <= 0) {
+      handler.SignalError(IostatOpenBadRecl,
+          "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is invalid",
+          unitNumber(), static_cast<std::intmax_t>(*recordLength));
+    } else if (!totalBytes) {
+      handler.SignalError(IostatOpenUnknownSize,
+          "OPEN(UNIT=%d,ACCESS='DIRECT'): file size is not known");
+    } else if (*totalBytes % *recordLength != 0) {
+      handler.SignalError(IostatOpenBadAppend,
+          "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is not an "
+          "even divisor of the file size %jd",
+          unitNumber(), static_cast<std::intmax_t>(*recordLength),
+          static_cast<std::intmax_t>(*totalBytes));
+    }
+  }
+  if (position == Position::Append) {
+    if (*totalBytes && recordLength && *recordLength) {
+      endfileRecordNumber = 1 + (*totalBytes / *recordLength);
+    } else {
+      // Fake it so that we can backspace relative from the end
+      endfileRecordNumber = std::numeric_limits<std::int64_t>::max() - 1;
+    }
+    currentRecordNumber = *endfileRecordNumber;
+  } else {
+    currentRecordNumber = 1;
+  }
 }
 
 void ExternalFileUnit::CloseUnit(CloseStatus status, IoErrorHandler &handler) {
+  DoImpliedEndfile(handler);
   Flush(handler);
   Close(status, handler);
 }
@@ -86,6 +121,29 @@ void ExternalFileUnit::DestroyClosed() {
   GetUnitMap().DestroyClosed(*this); // destroys *this
 }
 
+bool ExternalFileUnit::SetDirection(
+    Direction direction, IoErrorHandler &handler) {
+  if (direction == Direction::Input) {
+    if (mayRead()) {
+      direction_ = Direction::Input;
+      return true;
+    } else {
+      handler.SignalError(IostatReadFromWriteOnly,
+          "READ(UNIT=%d) with ACTION='WRITE'", unitNumber());
+      return false;
+    }
+  } else {
+    if (mayWrite()) {
+      direction_ = Direction::Output;
+      return true;
+    } else {
+      handler.SignalError(IostatWriteToReadOnly,
+          "WRITE(UNIT=%d) with ACTION='READ'", unitNumber());
+      return false;
+    }
+  }
+}
+
 UnitMap &ExternalFileUnit::GetUnitMap() {
   if (unitMap) {
     return *unitMap;
@@ -95,18 +153,22 @@ UnitMap &ExternalFileUnit::GetUnitMap() {
     return *unitMap;
   }
   Terminator terminator{__FILE__, __LINE__};
-  unitMap = &New<UnitMap>{}(terminator);
+  IoErrorHandler handler{terminator};
+  unitMap = New<UnitMap>{terminator}().release();
   ExternalFileUnit &out{ExternalFileUnit::LookUpOrCreate(6, terminator)};
   out.Predefine(1);
   out.set_mayRead(false);
   out.set_mayWrite(true);
   out.set_mayPosition(false);
+  out.SetDirection(Direction::Output, handler);
   defaultOutput = &out;
   ExternalFileUnit &in{ExternalFileUnit::LookUpOrCreate(5, terminator)};
   in.Predefine(0);
   in.set_mayRead(true);
   in.set_mayWrite(false);
   in.set_mayPosition(false);
+  in.SetDirection(Direction::Input, handler);
+  defaultInput = &in;
   // TODO: Set UTF-8 mode from the environment
   return *unitMap;
 }
@@ -120,53 +182,106 @@ void ExternalFileUnit::CloseAll(IoErrorHandler &handler) {
   defaultOutput = nullptr;
 }
 
+void ExternalFileUnit::FlushAll(IoErrorHandler &handler) {
+  CriticalSection critical{unitMapLock};
+  if (unitMap) {
+    unitMap->FlushAll(handler);
+  }
+}
+
 bool ExternalFileUnit::Emit(
     const char *data, std::size_t bytes, IoErrorHandler &handler) {
   auto furthestAfter{std::max(furthestPositionInRecord,
       positionInRecord + static_cast<std::int64_t>(bytes))};
   if (furthestAfter > recordLength.value_or(furthestAfter)) {
-    handler.SignalError(IostatRecordWriteOverrun);
+    handler.SignalError(IostatRecordWriteOverrun,
+        "Attempt to write %zd bytes to position %jd in a fixed-size record of "
+        "%jd bytes",
+        bytes, static_cast<std::intmax_t>(positionInRecord),
+        static_cast<std::intmax_t>(*recordLength));
     return false;
   }
   WriteFrame(frameOffsetInFile_, recordOffsetInFrame_ + furthestAfter, handler);
   if (positionInRecord > furthestPositionInRecord) {
-    std::memset(Frame() + furthestPositionInRecord, ' ',
+    std::memset(Frame() + recordOffsetInFrame_ + furthestPositionInRecord, ' ',
         positionInRecord - furthestPositionInRecord);
   }
-  std::memcpy(Frame() + positionInRecord, data, bytes);
+  std::memcpy(Frame() + recordOffsetInFrame_ + positionInRecord, data, bytes);
   positionInRecord += bytes;
   furthestPositionInRecord = furthestAfter;
   return true;
 }
 
+bool ExternalFileUnit::Receive(
+    char *data, std::size_t bytes, IoErrorHandler &handler) {
+  RUNTIME_CHECK(handler, direction_ == Direction::Input);
+  auto furthestAfter{std::max(furthestPositionInRecord,
+      positionInRecord + static_cast<std::int64_t>(bytes))};
+  if (furthestAfter > recordLength.value_or(furthestAfter)) {
+    handler.SignalError(IostatRecordReadOverrun,
+        "Attempt to read %zd bytes at position %jd in a record of %jd bytes",
+        bytes, static_cast<std::intmax_t>(positionInRecord),
+        static_cast<std::intmax_t>(*recordLength));
+    return false;
+  }
+  auto need{recordOffsetInFrame_ + furthestAfter};
+  auto got{ReadFrame(frameOffsetInFile_, need, handler)};
+  if (got >= need) {
+    std::memcpy(data, Frame() + recordOffsetInFrame_ + positionInRecord, bytes);
+    positionInRecord += bytes;
+    furthestPositionInRecord = furthestAfter;
+    return true;
+  } else {
+    handler.SignalEnd();
+    endfileRecordNumber = currentRecordNumber;
+    return false;
+  }
+}
+
 std::optional<char32_t> ExternalFileUnit::GetCurrentChar(
     IoErrorHandler &handler) {
-  isReading_ = true; // TODO: manage read/write transitions
-  if (isUnformatted) {
-    handler.Crash("GetCurrentChar() called for unformatted input");
-    return std::nullopt;
-  }
-  std::size_t chunk{256}; // for stream input
-  if (recordLength.has_value()) {
-    if (positionInRecord >= *recordLength) {
-      return std::nullopt;
-    }
-    chunk = *recordLength - positionInRecord;
+  RUNTIME_CHECK(handler, direction_ == Direction::Input);
+  if (const char *p{FrameNextInput(handler, 1)}) {
+    // TODO: UTF-8 decoding; may have to get more bytes in a loop
+    return *p;
   }
-  auto at{recordOffsetInFrame_ + positionInRecord};
-  std::size_t need{static_cast<std::size_t>(at + 1)};
-  std::size_t want{need + chunk};
-  auto got{ReadFrame(frameOffsetInFile_, want, handler)};
-  if (got <= need) {
-    endfileRecordNumber = currentRecordNumber;
+  return std::nullopt;
+}
+
+const char *ExternalFileUnit::FrameNextInput(
+    IoErrorHandler &handler, std::size_t bytes) {
+  RUNTIME_CHECK(handler, !isUnformatted);
+  if (static_cast<std::int64_t>(positionInRecord + bytes) <=
+      recordLength.value_or(positionInRecord + bytes)) {
+    auto at{recordOffsetInFrame_ + positionInRecord};
+    auto need{static_cast<std::size_t>(at + bytes)};
+    auto got{ReadFrame(frameOffsetInFile_, need, handler)};
+    SetSequentialVariableFormattedRecordLength();
+    if (got >= need) {
+      return Frame() + at;
+    }
     handler.SignalEnd();
-    return std::nullopt;
+    endfileRecordNumber = currentRecordNumber;
   }
-  const char *p{Frame() + at};
-  if (isUTF8) {
-    // TODO: UTF-8 decoding
+  return nullptr;
+}
+
+bool ExternalFileUnit::SetSequentialVariableFormattedRecordLength() {
+  if (recordLength || access != Access::Sequential) {
+    return true;
+  }
+  if (FrameLength() > recordOffsetInFrame_) {
+    const char *record{Frame() + recordOffsetInFrame_};
+    if (const char *nl{reinterpret_cast<const char *>(
+            std::memchr(record, '\n', FrameLength() - recordOffsetInFrame_))}) {
+      recordLength = nl - record;
+      if (*recordLength > 0 && record[*recordLength - 1] == '\r') {
+        --*recordLength;
+      }
+      return true;
+    }
   }
-  return *p;
+  return false;
 }
 
 void ExternalFileUnit::SetLeftTabLimit() {
@@ -174,55 +289,87 @@ void ExternalFileUnit::SetLeftTabLimit() {
   positionInRecord = furthestPositionInRecord;
 }
 
+void ExternalFileUnit::BeginReadingRecord(IoErrorHandler &handler) {
+  RUNTIME_CHECK(handler, direction_ == Direction::Input);
+  if (access == Access::Sequential) {
+    if (endfileRecordNumber && currentRecordNumber >= *endfileRecordNumber) {
+      handler.SignalEnd();
+    } else if (isFixedRecordLength) {
+      RUNTIME_CHECK(handler, recordLength.has_value());
+      auto need{static_cast<std::size_t>(recordOffsetInFrame_ + *recordLength)};
+      auto got{ReadFrame(frameOffsetInFile_, need, handler)};
+      if (got < need) {
+        handler.SignalEnd();
+      }
+    } else if (isUnformatted) {
+      BeginSequentialVariableUnformattedInputRecord(handler);
+    } else { // formatted
+      BeginSequentialVariableFormattedInputRecord(handler);
+    }
+  }
+}
+
 bool ExternalFileUnit::AdvanceRecord(IoErrorHandler &handler) {
   bool ok{true};
-  if (isReading_) {
+  if (direction_ == Direction::Input) {
     if (access == Access::Sequential) {
-      if (isUnformatted) {
-        NextSequentialUnformattedInputRecord(handler);
-      } else {
-        NextSequentialFormattedInputRecord(handler);
+      RUNTIME_CHECK(handler, recordLength.has_value());
+      if (isFixedRecordLength) {
+        frameOffsetInFile_ += recordOffsetInFrame_ + *recordLength;
+        recordOffsetInFrame_ = 0;
+      } else if (isUnformatted) {
+        // Retain footer in frame for more efficient BACKSPACE
+        frameOffsetInFile_ += recordOffsetInFrame_ + *recordLength;
+        recordOffsetInFrame_ = sizeof(std::uint32_t);
+        recordLength.reset();
+      } else { // formatted
+        if (Frame()[recordOffsetInFrame_ + *recordLength] == '\r') {
+          ++recordOffsetInFrame_;
+        }
+        recordOffsetInFrame_ += *recordLength + 1;
+        RUNTIME_CHECK(handler, Frame()[recordOffsetInFrame_ - 1] == '\n');
+        recordLength.reset();
       }
     }
-  } else if (!isUnformatted) {
-    if (recordLength.has_value()) {
-      // fill fixed-size record
-      if (furthestPositionInRecord < *recordLength) {
-        WriteFrame(frameOffsetInFile_, *recordLength, handler);
-        std::memset(Frame() + recordOffsetInFrame_ + furthestPositionInRecord,
-            ' ', *recordLength - furthestPositionInRecord);
+  } else { // Direction::Output
+    if (!isUnformatted) {
+      if (isFixedRecordLength && recordLength) {
+        if (furthestPositionInRecord < *recordLength) {
+          WriteFrame(frameOffsetInFile_, *recordLength, handler);
+          std::memset(Frame() + recordOffsetInFrame_ + furthestPositionInRecord,
+              ' ', *recordLength - furthestPositionInRecord);
+        }
+      } else {
+        positionInRecord = furthestPositionInRecord;
+        ok &= Emit("\n", 1, handler); // TODO: Windows CR+LF
       }
-    } else {
-      positionInRecord = furthestPositionInRecord;
-      ok &= Emit("\n", 1, handler); // TODO: Windows CR+LF
-      frameOffsetInFile_ += recordOffsetInFrame_ + furthestPositionInRecord;
-      recordOffsetInFrame_ = 0;
     }
+    frameOffsetInFile_ +=
+        recordOffsetInFrame_ + recordLength.value_or(furthestPositionInRecord);
+    recordOffsetInFrame_ = 0;
+    impliedEndfile_ = true;
   }
   ++currentRecordNumber;
-  positionInRecord = 0;
-  furthestPositionInRecord = 0;
-  leftTabLimit.reset();
+  BeginRecord();
   return ok;
 }
 
 void ExternalFileUnit::BackspaceRecord(IoErrorHandler &handler) {
-  if (!isReading_) {
-    handler.Crash("ExternalFileUnit::BackspaceRecord() called during writing");
-    // TODO: create endfile record, &c.
-  }
-  if (access == Access::Sequential) {
-    if (isUnformatted) {
-      BackspaceSequentialUnformattedRecord(handler);
+  if (access != Access::Sequential) {
+    handler.SignalError(IostatBackspaceNonSequential,
+        "BACKSPACE(UNIT=%d) on non-sequential file", unitNumber());
+  } else {
+    DoImpliedEndfile(handler);
+    --currentRecordNumber;
+    BeginRecord();
+    if (isFixedRecordLength) {
+      BackspaceFixedRecord(handler);
+    } else if (isUnformatted) {
+      BackspaceVariableUnformattedRecord(handler);
     } else {
-      BackspaceSequentialFormattedRecord(handler);
+      BackspaceVariableFormattedRecord(handler);
     }
-  } else {
-    // TODO
   }
-  positionInRecord = 0;
-  furthestPositionInRecord = 0;
-  leftTabLimit.reset();
 }
 
 void ExternalFileUnit::FlushIfTerminal(IoErrorHandler &handler) {
@@ -231,6 +378,30 @@ void ExternalFileUnit::FlushIfTerminal(IoErrorHandler &handler) {
   }
 }
 
+void ExternalFileUnit::Endfile(IoErrorHandler &handler) {
+  if (access != Access::Sequential) {
+    handler.SignalError(IostatEndfileNonSequential,
+        "ENDFILE(UNIT=%d) on non-sequential file", unitNumber());
+  } else if (!mayWrite()) {
+    handler.SignalError(IostatEndfileUnwritable,
+        "ENDFILE(UNIT=%d) on read-only file", unitNumber());
+  } else {
+    DoEndfile(handler);
+  }
+}
+
+void ExternalFileUnit::Rewind(IoErrorHandler &handler) {
+  if (access == Access::Direct) {
+    handler.SignalError(IostatRewindNonSequential,
+        "REWIND(UNIT=%d) on non-sequential file", unitNumber());
+  } else {
+    DoImpliedEndfile(handler);
+    SetPosition(0);
+    currentRecordNumber = 1;
+    // TODO: reset endfileRecordNumber?
+  }
+}
+
 void ExternalFileUnit::EndIoStatement() {
   frameOffsetInFile_ += recordOffsetInFrame_;
   recordOffsetInFrame_ = 0;
@@ -239,51 +410,36 @@ void ExternalFileUnit::EndIoStatement() {
   lock_.Drop();
 }
 
-void ExternalFileUnit::NextSequentialUnformattedInputRecord(
+void ExternalFileUnit::BeginSequentialVariableUnformattedInputRecord(
     IoErrorHandler &handler) {
   std::int32_t header{0}, footer{0};
-  // Retain previous footer (if any) in frame for more efficient BACKSPACE
-  std::size_t retain{sizeof header};
-  if (recordLength) { // not first record - advance to next
-    ++currentRecordNumber;
-    if (endfileRecordNumber && currentRecordNumber >= *endfileRecordNumber) {
-      handler.SignalEnd();
-      return;
-    }
-    frameOffsetInFile_ +=
-        recordOffsetInFrame_ + *recordLength + 2 * sizeof header;
-    recordOffsetInFrame_ = 0;
-  } else {
-    retain = 0;
-  }
-  std::size_t need{retain + sizeof header};
-  std::size_t got{ReadFrame(frameOffsetInFile_ - retain, need, handler)};
+  std::size_t need{recordOffsetInFrame_ + sizeof header};
+  std::size_t got{ReadFrame(frameOffsetInFile_, need, handler)};
   // Try to emit informative errors to help debug corrupted files.
   const char *error{nullptr};
   if (got < need) {
-    if (got == retain) {
+    if (got == recordOffsetInFrame_) {
       handler.SignalEnd();
     } else {
-      error = "Unformatted sequential file input failed at record #%jd (file "
-              "offset %jd): truncated record header";
+      error = "Unformatted variable-length sequential file input failed at "
+              "record #%jd (file offset %jd): truncated record header";
     }
   } else {
-    std::memcpy(&header, Frame() + retain, sizeof header);
-    need = retain + header + 2 * sizeof header;
-    got = ReadFrame(frameOffsetInFile_ - retain,
-        need + sizeof header /* next one */, handler);
+    std::memcpy(&header, Frame() + recordOffsetInFrame_, sizeof header);
+    recordLength = sizeof header + header; // does not include footer
+    need = recordOffsetInFrame_ + *recordLength + sizeof footer;
+    got = ReadFrame(frameOffsetInFile_, need, handler);
     if (got < need) {
-      error = "Unformatted sequential file input failed at record #%jd (file "
-              "offset %jd): hit EOF reading record with length %jd bytes";
+      error = "Unformatted variable-length sequential file input failed at "
+              "record #%jd (file offset %jd): hit EOF reading record with "
+              "length %jd bytes";
     } else {
-      const char *start{Frame() + retain + sizeof header};
-      std::memcpy(&footer, start + header, sizeof footer);
+      std::memcpy(&footer, Frame() + recordOffsetInFrame_ + *recordLength,
+          sizeof footer);
       if (footer != header) {
-        error = "Unformatted sequential file input failed at record #%jd (file "
-                "offset %jd): record header has length %jd that does not match "
-                "record footer (%jd)";
-      } else {
-        recordLength = header;
+        error = "Unformatted variable-length sequential file input failed at "
+                "record #%jd (file offset %jd): record header has length %jd "
+                "that does not match record footer (%jd)";
       }
     }
   }
@@ -291,70 +447,66 @@ void ExternalFileUnit::NextSequentialUnformattedInputRecord(
     handler.SignalError(error, static_cast<std::intmax_t>(currentRecordNumber),
         static_cast<std::intmax_t>(frameOffsetInFile_),
         static_cast<std::intmax_t>(header), static_cast<std::intmax_t>(footer));
+    // TODO: error recovery
   }
   positionInRecord = sizeof header;
 }
 
-void ExternalFileUnit::NextSequentialFormattedInputRecord(
+void ExternalFileUnit::BeginSequentialVariableFormattedInputRecord(
     IoErrorHandler &handler) {
-  static constexpr std::size_t chunk{256};
-  std::size_t length{0};
-  if (recordLength.has_value()) {
-    // not first record - advance to next
-    ++currentRecordNumber;
-    if (endfileRecordNumber && currentRecordNumber >= *endfileRecordNumber) {
-      handler.SignalEnd();
-      return;
-    }
-    if (Frame()[*recordLength] == '\r') {
-      ++*recordLength;
-    }
-    recordOffsetInFrame_ += *recordLength + 1;
+  if (this == defaultInput && defaultOutput) {
+    defaultOutput->Flush(handler);
   }
-  while (true) {
-    std::size_t got{ReadFrame(
-        frameOffsetInFile_, recordOffsetInFrame_ + length + chunk, handler)};
-    if (got <= recordOffsetInFrame_ + length) {
+  std::size_t length{0};
+  do {
+    std::size_t need{recordOffsetInFrame_ + length + 1};
+    length = ReadFrame(frameOffsetInFile_, need, handler);
+    if (length < need) {
       handler.SignalEnd();
       break;
     }
-    const char *frame{Frame() + recordOffsetInFrame_};
-    if (const char *nl{reinterpret_cast<const char *>(
-            std::memchr(frame + length, '\n', chunk))}) {
-      recordLength = nl - (frame + length) + 1;
-      if (*recordLength > 0 && frame[*recordLength - 1] == '\r') {
-        --*recordLength;
-      }
-      return;
-    }
-    length += got;
+  } while (!SetSequentialVariableFormattedRecordLength());
+}
+
+void ExternalFileUnit::BackspaceFixedRecord(IoErrorHandler &handler) {
+  RUNTIME_CHECK(handler, recordLength.has_value());
+  if (frameOffsetInFile_ < *recordLength) {
+    handler.SignalError(IostatBackspaceAtFirstRecord);
+  } else {
+    frameOffsetInFile_ -= *recordLength;
   }
 }
 
-void ExternalFileUnit::BackspaceSequentialUnformattedRecord(
+void ExternalFileUnit::BackspaceVariableUnformattedRecord(
     IoErrorHandler &handler) {
   std::int32_t header{0}, footer{0};
-  RUNTIME_CHECK(handler, currentRecordNumber > 1);
-  --currentRecordNumber;
-  int overhead{static_cast<int>(2 * sizeof header)};
+  auto headerBytes{static_cast<std::int64_t>(sizeof header)};
+  frameOffsetInFile_ += recordOffsetInFrame_;
+  recordOffsetInFrame_ = 0;
+  if (frameOffsetInFile_ <= headerBytes) {
+    handler.SignalError(IostatBackspaceAtFirstRecord);
+    return;
+  }
   // Error conditions here cause crashes, not file format errors, because the
   // validity of the file structure before the current record will have been
-  // checked informatively in NextSequentialUnformattedInputRecord().
-  RUNTIME_CHECK(handler, frameOffsetInFile_ >= overhead);
+  // checked informatively in NextSequentialVariableUnformattedInputRecord().
   std::size_t got{
-      ReadFrame(frameOffsetInFile_ - sizeof footer, sizeof footer, handler)};
+      ReadFrame(frameOffsetInFile_ - headerBytes, headerBytes, handler)};
   RUNTIME_CHECK(handler, got >= sizeof footer);
   std::memcpy(&footer, Frame(), sizeof footer);
-  RUNTIME_CHECK(handler, frameOffsetInFile_ >= footer + overhead);
-  frameOffsetInFile_ -= footer + 2 * sizeof footer;
-  auto extra{std::max<std::size_t>(sizeof footer, frameOffsetInFile_)};
-  std::size_t want{extra + footer + 2 * sizeof footer};
-  got = ReadFrame(frameOffsetInFile_ - extra, want, handler);
-  RUNTIME_CHECK(handler, got >= want);
-  std::memcpy(&header, Frame() + extra, sizeof header);
-  RUNTIME_CHECK(handler, header == footer);
-  positionInRecord = sizeof header;
   recordLength = footer;
+  RUNTIME_CHECK(handler, frameOffsetInFile_ >= *recordLength + 2 * headerBytes);
+  frameOffsetInFile_ -= *recordLength + 2 * headerBytes;
+  if (frameOffsetInFile_ >= headerBytes) {
+    frameOffsetInFile_ -= headerBytes;
+    recordOffsetInFrame_ = headerBytes;
+  }
+  auto need{static_cast<std::size_t>(
+      recordOffsetInFrame_ + sizeof header + *recordLength)};
+  got = ReadFrame(frameOffsetInFile_, need, handler);
+  RUNTIME_CHECK(handler, got >= need);
+  std::memcpy(&header, Frame() + recordOffsetInFrame_, sizeof header);
+  RUNTIME_CHECK(handler, header == *recordLength);
 }
 
 // There's no portable memrchr(), unfortunately, and strrchr() would
@@ -368,50 +520,54 @@ static const char *FindLastNewline(const char *str, std::size_t length) {
   return nullptr;
 }
 
-void ExternalFileUnit::BackspaceSequentialFormattedRecord(
+void ExternalFileUnit::BackspaceVariableFormattedRecord(
     IoErrorHandler &handler) {
-  std::int64_t start{frameOffsetInFile_ + recordOffsetInFrame_};
-  --currentRecordNumber;
-  RUNTIME_CHECK(handler, currentRecordNumber > 0);
-  if (currentRecordNumber == 1) {
-    // To simplify the code below, treat a backspace to the first record
-    // as a special case;
-    RUNTIME_CHECK(handler, start > 0);
-    *recordLength = start - 1;
-    frameOffsetInFile_ = 0;
-    recordOffsetInFrame_ = 0;
-    ReadFrame(0, *recordLength + 1, handler);
-  } else {
-    RUNTIME_CHECK(handler, start > 1);
-    std::int64_t at{start - 2}; // byte before previous record's newline
-    while (true) {
+  // File offset of previous record's newline
+  auto prevNL{
+      frameOffsetInFile_ + static_cast<std::int64_t>(recordOffsetInFrame_) - 1};
+  if (prevNL < 0) {
+    handler.SignalError(IostatBackspaceAtFirstRecord);
+    return;
+  }
+  while (true) {
+    if (frameOffsetInFile_ < prevNL) {
       if (const char *p{
-              FindLastNewline(Frame(), at - frameOffsetInFile_ + 1)}) {
-        // This is the newline that ends the record before the previous one.
+              FindLastNewline(Frame(), prevNL - 1 - frameOffsetInFile_)}) {
         recordOffsetInFrame_ = p - Frame() + 1;
-        *recordLength = start - 1 - (frameOffsetInFile_ + recordOffsetInFrame_);
+        *recordLength = prevNL - (frameOffsetInFile_ + recordOffsetInFrame_);
         break;
       }
-      RUNTIME_CHECK(handler, frameOffsetInFile_ > 0);
-      at = frameOffsetInFile_ - 1;
-      if (auto bytesBefore{BytesBufferedBeforeFrame()}) {
-        frameOffsetInFile_ = FrameAt() - bytesBefore;
-      } else {
-        static constexpr int chunk{1024};
-        frameOffsetInFile_ = std::max<std::int64_t>(0, at - chunk);
-      }
-      std::size_t want{static_cast<std::size_t>(start - frameOffsetInFile_)};
-      std::size_t got{ReadFrame(frameOffsetInFile_, want, handler)};
-      RUNTIME_CHECK(handler, got >= want);
     }
+    if (frameOffsetInFile_ == 0) {
+      recordOffsetInFrame_ = 0;
+      *recordLength = prevNL;
+      break;
+    }
+    frameOffsetInFile_ -= std::min<std::int64_t>(frameOffsetInFile_, 1024);
+    auto need{static_cast<std::size_t>(prevNL + 1 - frameOffsetInFile_)};
+    auto got{ReadFrame(frameOffsetInFile_, need, handler)};
+    RUNTIME_CHECK(handler, got >= need);
   }
-  std::size_t want{
-      static_cast<std::size_t>(recordOffsetInFrame_ + *recordLength + 1)};
-  RUNTIME_CHECK(handler, FrameLength() >= want);
   RUNTIME_CHECK(handler, Frame()[recordOffsetInFrame_ + *recordLength] == '\n');
   if (*recordLength > 0 &&
       Frame()[recordOffsetInFrame_ + *recordLength - 1] == '\r') {
     --*recordLength;
   }
 }
+
+void ExternalFileUnit::DoImpliedEndfile(IoErrorHandler &handler) {
+  if (impliedEndfile_) {
+    impliedEndfile_ = false;
+    if (access == Access::Sequential && mayPosition()) {
+      DoEndfile(handler);
+    }
+  }
+}
+
+void ExternalFileUnit::DoEndfile(IoErrorHandler &handler) {
+  endfileRecordNumber = currentRecordNumber;
+  Truncate(frameOffsetInFile_ + recordOffsetInFrame_, handler);
+  BeginRecord();
+  impliedEndfile_ = false;
+}
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/unit.h b/flang/runtime/unit.h
index cb40ddebacc54..c54625413b875 100644
--- a/flang/runtime/unit.h
+++ b/flang/runtime/unit.h
@@ -43,12 +43,15 @@ class ExternalFileUnit : public ConnectionState,
   static ExternalFileUnit *LookUpForClose(int unit);
   static int NewUnit(const Terminator &);
   static void CloseAll(IoErrorHandler &);
+  static void FlushAll(IoErrorHandler &);
 
   void OpenUnit(OpenStatus, Position, OwningPtr<char> &&path,
       std::size_t pathLength, IoErrorHandler &);
   void CloseUnit(CloseStatus, IoErrorHandler &);
   void DestroyClosed();
 
+  bool SetDirection(Direction, IoErrorHandler &);
+
   template <typename A, typename... X>
   IoStatementState &BeginIoStatement(X &&... xs) {
     // TODO: Child data transfer statements vs. locking
@@ -61,27 +64,38 @@ class ExternalFileUnit : public ConnectionState,
     return *io_;
   }
 
-  bool Emit(const char *, std::size_t bytes, IoErrorHandler &);
+  bool Emit(const char *, std::size_t, IoErrorHandler &);
+  bool Receive(char *, std::size_t, IoErrorHandler &);
   std::optional<char32_t> GetCurrentChar(IoErrorHandler &);
   void SetLeftTabLimit();
+  void BeginReadingRecord(IoErrorHandler &);
   bool AdvanceRecord(IoErrorHandler &);
   void BackspaceRecord(IoErrorHandler &);
   void FlushIfTerminal(IoErrorHandler &);
+  void Endfile(IoErrorHandler &);
+  void Rewind(IoErrorHandler &);
   void EndIoStatement();
   void SetPosition(std::int64_t pos) {
     frameOffsetInFile_ = pos;
     recordOffsetInFrame_ = 0;
+    BeginRecord();
   }
 
 private:
   static UnitMap &GetUnitMap();
-  void NextSequentialUnformattedInputRecord(IoErrorHandler &);
-  void NextSequentialFormattedInputRecord(IoErrorHandler &);
-  void BackspaceSequentialUnformattedRecord(IoErrorHandler &);
-  void BackspaceSequentialFormattedRecord(IoErrorHandler &);
+  const char *FrameNextInput(IoErrorHandler &, std::size_t);
+  void BeginSequentialVariableUnformattedInputRecord(IoErrorHandler &);
+  void BeginSequentialVariableFormattedInputRecord(IoErrorHandler &);
+  void BackspaceFixedRecord(IoErrorHandler &);
+  void BackspaceVariableUnformattedRecord(IoErrorHandler &);
+  void BackspaceVariableFormattedRecord(IoErrorHandler &);
+  bool SetSequentialVariableFormattedRecordLength();
+  void DoImpliedEndfile(IoErrorHandler &);
+  void DoEndfile(IoErrorHandler &);
 
   int unitNumber_{-1};
-  bool isReading_{false};
+  Direction direction_{Direction::Output};
+  bool impliedEndfile_{false}; // seq. output has taken place
 
   Lock lock_;
 
@@ -92,7 +106,8 @@ class ExternalFileUnit : public ConnectionState,
       ExternalListIoStatementState<Direction::Output>,
       ExternalListIoStatementState<Direction::Input>,
       UnformattedIoStatementState<Direction::Output>,
-      UnformattedIoStatementState<Direction::Input>>
+      UnformattedIoStatementState<Direction::Input>,
+      ExternalMiscIoStatementState>
       u_;
 
   // Points to the active alternative (if any) in u_ for use as a Cookie
@@ -100,9 +115,10 @@ class ExternalFileUnit : public ConnectionState,
 
   // Subtle: The beginning of the frame can't be allowed to advance
   // during a single list-directed READ due to the possibility of a
-  // multi-record CHARACTER value with a "r*" repeat count.
+  // multi-record CHARACTER value with a "r*" repeat count.  So we
+  // manage the frame and the current record therein separately.
   std::int64_t frameOffsetInFile_{0};
-  std::int64_t recordOffsetInFrame_{0}; // of currentRecordNumber
+  std::size_t recordOffsetInFrame_{0}; // of currentRecordNumber
 };
 
 } // namespace Fortran::runtime::io
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index 6f47d67aa53e2..ad47fff8c4250 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -45,7 +45,13 @@ add_lit_testsuite(check-flang "Running the Flang regression tests"
 )
 set_target_properties(check-flang PROPERTIES FOLDER "Tests")
 
-add_lit_testsuites(FLANG ${CMAKE_CURRENT_SOURCE_DIR}
-  PARAMS ${FLANG_TEST_PARAMS}
-  DEPENDS ${FLANG_TEST_DEPENDS})
-
+# In case of standalone builds.
+if (FLANG_STANDALONE_BUILD)
+  add_lit_testsuites(FLANG ${CMAKE_CURRENT_BINARY_DIR}
+    PARAMS ${FLANG_TEST_PARAMS}
+    DEPENDS ${FLANG_TEST_DEPENDS})
+else()
+  add_lit_testsuites(FLANG ${CMAKE_CURRENT_SOURCE_DIR}
+    PARAMS ${FLANG_TEST_PARAMS}
+    DEPENDS ${FLANG_TEST_DEPENDS})
+endif()
diff --git a/flang/test/Semantics/Inputs/getsymbols02-a.f90 b/flang/test/Semantics/Inputs/getsymbols02-a.f90
index 04786c0bea6d3..d4fd43cbd3b58 100644
--- a/flang/test/Semantics/Inputs/getsymbols02-a.f90
+++ b/flang/test/Semantics/Inputs/getsymbols02-a.f90
@@ -1,5 +1,3 @@
-! EXEC: ${F18} -fparse-only %s
-
 module mm2a
 implicit none
 private
diff --git a/flang/test/Semantics/Inputs/getsymbols02-b.f90 b/flang/test/Semantics/Inputs/getsymbols02-b.f90
index 1e6bb8b03a6cb..390a0c7482bfc 100644
--- a/flang/test/Semantics/Inputs/getsymbols02-b.f90
+++ b/flang/test/Semantics/Inputs/getsymbols02-b.f90
@@ -1,5 +1,3 @@
-! EXEC: ${F18} -fparse-only %s
-
 module mm2b
 use mm2a
 implicit none
diff --git a/flang/test/Semantics/Inputs/getsymbols02-c.f90 b/flang/test/Semantics/Inputs/getsymbols02-c.f90
deleted file mode 100644
index 52a210b916994..0000000000000
--- a/flang/test/Semantics/Inputs/getsymbols02-c.f90
+++ /dev/null
@@ -1,12 +0,0 @@
-! Tests -fget-symbols-sources with modules.
-
-PROGRAM helloworld
-    use mm2b
-    implicit none
-    integer::i
-    i = callget5()
-ENDPROGRAM
-
-! EXEC: ${F18} -fget-symbols-sources -fparse-only %s 2>&1 | ${FileCheck} %s
-! CHECK:callget5: mm2b
-! CHECK:get5: mm2a
diff --git a/flang/test/Semantics/call10.f90 b/flang/test/Semantics/call10.f90
index 5966b89cdebea..e7b29c7f45c5d 100644
--- a/flang/test/Semantics/call10.f90
+++ b/flang/test/Semantics/call10.f90
@@ -185,7 +185,6 @@ pure subroutine s14
                                    ! implicit sync all
     !ERROR: Procedure 'this_image' referenced in pure subprogram 's14' must be pure too
     img = this_image()
-    !ERROR: Procedure 'num_images' referenced in pure subprogram 's14' must be pure too
     nimgs = num_images()
     i = img                       ! i is ready to use
 
diff --git a/flang/test/Semantics/canondo01.f90 b/flang/test/Semantics/canondo01.f90
index a717450b953ee..3379953b12d7f 100644
--- a/flang/test/Semantics/canondo01.f90
+++ b/flang/test/Semantics/canondo01.f90
@@ -1,7 +1,5 @@
-! RUN: %S/test_any.sh %s %t %f18
-! negative test -- invalid labels, out of range
 
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK: end do
 
 SUBROUTINE sub00(a,b,n,m)
diff --git a/flang/test/Semantics/canondo02.f90 b/flang/test/Semantics/canondo02.f90
index 0125467a9f2a9..69745c4ef458c 100644
--- a/flang/test/Semantics/canondo02.f90
+++ b/flang/test/Semantics/canondo02.f90
@@ -1,7 +1,5 @@
-! RUN: %S/test_any.sh %s %t %f18
-! negative test -- invalid labels, out of range
 
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK: end do
 
 SUBROUTINE sub00(a,b,n,m)
diff --git a/flang/test/Semantics/canondo03.f90 b/flang/test/Semantics/canondo03.f90
index 94ab11cc39ee1..55b3fd2fe8f12 100644
--- a/flang/test/Semantics/canondo03.f90
+++ b/flang/test/Semantics/canondo03.f90
@@ -1,7 +1,5 @@
-! RUN: %S/test_any.sh %s %t %f18
-! negative test -- invalid labels, out of range
 
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK: 10 continue
 ! CHECK: end do
 
diff --git a/flang/test/Semantics/canondo04.f90 b/flang/test/Semantics/canondo04.f90
index 65a79d0b491d0..b252c98201595 100644
--- a/flang/test/Semantics/canondo04.f90
+++ b/flang/test/Semantics/canondo04.f90
@@ -1,5 +1,4 @@
-! RUN: %S/test_any.sh %s %t %f18
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK-NOT: do [1-9]
 
 ! Figure out how to also execute this test.
diff --git a/flang/test/Semantics/canondo05.f90 b/flang/test/Semantics/canondo05.f90
index fe23091b5964b..788dc7a5de89e 100644
--- a/flang/test/Semantics/canondo05.f90
+++ b/flang/test/Semantics/canondo05.f90
@@ -1,6 +1,5 @@
-! RUN: %S/test_any.sh %s %t %f18
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
-! XXXEXEC: ${F18} -fopenmp -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
+! RUN: %f18 -fopenmp -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK-NOT: do *[1-9]
 
 program P
diff --git a/flang/test/Semantics/canondo06.f90 b/flang/test/Semantics/canondo06.f90
index a429873a1af3f..dae6b3807bf37 100644
--- a/flang/test/Semantics/canondo06.f90
+++ b/flang/test/Semantics/canondo06.f90
@@ -1,5 +1,4 @@
-! RUN: %S/test_any.sh %s %t %f18
-! EXEC: ${F18} -fopenmp -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -fopenmp -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK-NOT: do *[1-9]
 ! CHECK: omp simd
 
diff --git a/flang/test/Semantics/canondo07.f90 b/flang/test/Semantics/canondo07.f90
index b12b8bb607206..074ea9df8f988 100644
--- a/flang/test/Semantics/canondo07.f90
+++ b/flang/test/Semantics/canondo07.f90
@@ -1,8 +1,7 @@
-! RUN: %S/test_any.sh %s %t %f18
 ! Error test -- DO loop uses obsolete loop termination statement
 ! See R1131 and C1131
 
-! EXEC: ${F18} -funparse-with-symbols -Mstandard %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols -Mstandard %s 2>&1 | FileCheck %s
 ! CHECK: A DO loop should terminate with an END DO or CONTINUE
 
 program endDo
diff --git a/flang/test/Semantics/canondo08.f90 b/flang/test/Semantics/canondo08.f90
index 465982f039392..09a277b53d5a2 100644
--- a/flang/test/Semantics/canondo08.f90
+++ b/flang/test/Semantics/canondo08.f90
@@ -1,4 +1,3 @@
-! RUN: %S/test_any.sh %s %t %f18
 ! Error test -- DO loop uses obsolete loop termination statement
 ! See R1131 and C1133
 
@@ -6,7 +5,7 @@
 ! A warning is generated with -Mstandard
 
 
-! EXEC: ${F18} -funparse-with-symbols -Mstandard %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols -Mstandard %s 2>%t.stderr | FileCheck %s
 
 ! CHECK: end do
 
@@ -15,7 +14,8 @@
 ! not want to see label do in the unparse only.
 ! CHECK-NOT: do [1-9]
 
-! CHECK: A DO loop should terminate with an END DO or CONTINUE
+! RUN: FileCheck --check-prefix=ERR --input-file=%t.stderr %s
+! ERR: A DO loop should terminate with an END DO or CONTINUE
 
 subroutine foo1()
   do 01 k=1,2
diff --git a/flang/test/Semantics/canondo09.f90 b/flang/test/Semantics/canondo09.f90
index 6c76ac2c1a688..6d9efd91f89c7 100644
--- a/flang/test/Semantics/canondo09.f90
+++ b/flang/test/Semantics/canondo09.f90
@@ -1,12 +1,11 @@
-! RUN: %S/test_any.sh %s %t %f18
-! Error test -- DO loop uses obsolete loop termination statement
+! Error test -- DO loop uses obsolete loop termination statement (warning)
 ! See R1131 and C1133
 
 ! By default, this is not an error and label do are rewritten to non-label do.
 ! A warning is generated with -Mstandard
 
 
-! EXEC: ${F18} -funparse-with-symbols -Mstandard %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols -Mstandard %s 2>%t.stderr | FileCheck %s
 
 ! CHECK: end do
 
@@ -15,7 +14,8 @@
 ! not want to see label do in the unparse only.
 ! CHECK-NOT: do [1-9]
 
-! CHECK: A DO loop should terminate with an END DO or CONTINUE
+! RUN: FileCheck --check-prefix=ERR --input-file=%t.stderr %s
+! ERR: A DO loop should terminate with an END DO or CONTINUE
 
 subroutine foo0()
   do  01 j=1,2
diff --git a/flang/test/Semantics/canondo10.f90 b/flang/test/Semantics/canondo10.f90
index adb658280e800..2733db6f7c61b 100644
--- a/flang/test/Semantics/canondo10.f90
+++ b/flang/test/Semantics/canondo10.f90
@@ -1,12 +1,11 @@
-! RUN: %S/test_any.sh %s %t %f18
-! Error test -- DO loop uses obsolete loop termination statement
+! Error test -- DO loop uses obsolete loop termination statement (warning)
 ! See R1131 and C1133
 
 ! By default, this is not an error and label do are rewritten to non-label do.
 ! A warning is generated with -Mstandard
 
 
-! EXEC: ${F18} -funparse-with-symbols -Mstandard %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols -Mstandard %s 2>%t.stderr | FileCheck %s
 
 ! CHECK: end do
 
@@ -15,7 +14,8 @@
 ! not want to see label do in the unparse only.
 ! CHECK-NOT: do [1-9]
 
-! CHECK: A DO loop should terminate with an END DO or CONTINUE
+! RUN: FileCheck --check-prefix=ERR --input-file=%t.stderr %s
+! ERR: A DO loop should terminate with an END DO or CONTINUE
 
 subroutine foo2()
   do 01 l=1,2
diff --git a/flang/test/Semantics/canondo11.f90 b/flang/test/Semantics/canondo11.f90
index 2ff61eb9682fd..267e76ab34a78 100644
--- a/flang/test/Semantics/canondo11.f90
+++ b/flang/test/Semantics/canondo11.f90
@@ -1,12 +1,11 @@
-! RUN: %S/test_any.sh %s %t %f18
-! Error test -- DO loop uses obsolete loop termination statement
+! Error test -- DO loop uses obsolete loop termination statement (warning)
 ! See R1131 and C1133
 
 ! By default, this is not an error and label do are rewritten to non-label do.
 ! A warning is generated with -Mstandard
 
 
-! EXEC: ${F18} -funparse-with-symbols -Mstandard %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols -Mstandard %s 2>%t.stderr | FileCheck %s
 
 ! CHECK: end do
 
@@ -15,7 +14,8 @@
 ! not want to see label do in the unparse only.
 ! CHECK-NOT: do [1-9]
 
-! CHECK: A DO loop should terminate with an END DO or CONTINUE
+! RUN: FileCheck --check-prefix=ERR --input-file=%t.stderr %s
+! ERR: A DO loop should terminate with an END DO or CONTINUE
 
 subroutine foo3()
   real :: a(10, 10), b(10, 10) = 1.0
diff --git a/flang/test/Semantics/canondo12.f90 b/flang/test/Semantics/canondo12.f90
index bbc73482342ba..b3bc10cba02ac 100644
--- a/flang/test/Semantics/canondo12.f90
+++ b/flang/test/Semantics/canondo12.f90
@@ -1,11 +1,10 @@
-! RUN: %S/test_any.sh %s %t %f18
-! Error test -- DO loop uses obsolete loop termination statement
+! Error test -- DO loop uses obsolete loop termination statement (warning)
 ! See R1131 and C1133
 
 ! By default, this is not an error and label do are rewritten to non-label do.
 ! A warning is generated with -Mstandard
 
-! EXEC: ${F18} -funparse-with-symbols -Mstandard %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols -Mstandard %s 2>%t.stderr | FileCheck %s
 
 ! CHECK: end do
 
@@ -14,7 +13,8 @@
 ! not want to see label do in the unparse only.
 ! CHECK-NOT: do [1-9]
 
-! CHECK: A DO loop should terminate with an END DO or CONTINUE
+! RUN: FileCheck --check-prefix=ERR --input-file=%t.stderr %s
+! ERR: A DO loop should terminate with an END DO or CONTINUE
 
 subroutine foo4()
   real :: a(10, 10), b(10, 10) = 1.0
diff --git a/flang/test/Semantics/canondo13.f90 b/flang/test/Semantics/canondo13.f90
index a1278460017e7..d75d8b96cebc9 100644
--- a/flang/test/Semantics/canondo13.f90
+++ b/flang/test/Semantics/canondo13.f90
@@ -1,11 +1,10 @@
-! RUN: %S/test_any.sh %s %t %f18
-! Error test -- DO loop uses obsolete loop termination statement
+! Error test -- DO loop uses obsolete loop termination statement (warning)
 ! See R1131 and C1133
 
 ! By default, this is not an error and label do are rewritten to non-label do.
 ! A warning is generated with -Mstandard
 
-! EXEC: ${F18} -funparse-with-symbols -Mstandard %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols -Mstandard %s 2>%t.stderr | FileCheck %s
 
 ! CHECK: end do
 
@@ -14,7 +13,8 @@
 ! not want to see label do in the unparse only.
 ! CHECK-NOT: do [1-9]
 
-! CHECK: A DO loop should terminate with an END DO or CONTINUE
+! RUN: FileCheck --check-prefix=ERR --input-file=%t.stderr %s
+! ERR: A DO loop should terminate with an END DO or CONTINUE
 
 subroutine foo5()
   real :: a(10, 10), b(10, 10) = 1.0
diff --git a/flang/test/Semantics/canondo14.f90 b/flang/test/Semantics/canondo14.f90
index c504468a0a032..99a3d25360db2 100644
--- a/flang/test/Semantics/canondo14.f90
+++ b/flang/test/Semantics/canondo14.f90
@@ -1,11 +1,10 @@
-! RUN: %S/test_any.sh %s %t %f18
-! Error test -- DO loop uses obsolete loop termination statement
+! Error test -- DO loop uses obsolete loop termination statement (warning)
 ! See R1131 and C1133
 
 ! By default, this is not an error and label do are rewritten to non-label do.
 ! A warning is generated with -Mstandard
 
-! EXEC: ${F18} -funparse-with-symbols -Mstandard %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols -Mstandard %s 2>%t.stderr | FileCheck %s
 
 ! CHECK: end do
 
@@ -14,7 +13,8 @@
 ! not want to see label do in the unparse only.
 ! CHECK-NOT: do [1-9]
 
-! CHECK: A DO loop should terminate with an END DO or CONTINUE
+! RUN: FileCheck --check-prefix=ERR --input-file=%t.stderr %s
+! ERR: A DO loop should terminate with an END DO or CONTINUE
 
 subroutine foo6(a)
   type whatever
diff --git a/flang/test/Semantics/canondo15.f90 b/flang/test/Semantics/canondo15.f90
index 6b52769d8c4ae..059f06ce64097 100644
--- a/flang/test/Semantics/canondo15.f90
+++ b/flang/test/Semantics/canondo15.f90
@@ -1,11 +1,10 @@
-! RUN: %S/test_any.sh %s %t %f18
-! Error test -- DO loop uses obsolete loop termination statement
+! Error test -- DO loop uses obsolete loop termination statement (warning)
 ! See R1131 and C1133
 
 ! By default, this is not an error and label do are rewritten to non-label do.
 ! A warning is generated with -Mstandard
 
-! EXEC: ${F18} -funparse-with-symbols -Mstandard %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols -Mstandard %s 2>%t.stderr | FileCheck %s
 
 ! CHECK: end do
 
@@ -14,7 +13,8 @@
 ! not want to see label do in the unparse only.
 ! CHECK-NOT: do [1-9]
 
-! CHECK: A DO loop should terminate with an END DO or CONTINUE
+! RUN: FileCheck --check-prefix=ERR --input-file=%t.stderr %s
+! ERR: A DO loop should terminate with an END DO or CONTINUE
 
 subroutine foo7(a)
   integer :: a(..)
diff --git a/flang/test/Semantics/canondo16.f90 b/flang/test/Semantics/canondo16.f90
index a8dfc91fa0114..e455fa32b2279 100644
--- a/flang/test/Semantics/canondo16.f90
+++ b/flang/test/Semantics/canondo16.f90
@@ -1,11 +1,10 @@
-! RUN: %S/test_any.sh %s %t %f18
-! Error test -- DO loop uses obsolete loop termination statement
+! Error test -- DO loop uses obsolete loop termination statement (warning)
 ! See R1131 and C1133
 
 ! By default, this is not an error and label do are rewritten to non-label do.
 ! A warning is generated with -Mstandard
 
-! EXEC: ${F18} -funparse-with-symbols -Mstandard %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols -Mstandard %s 2>%t.stderr | FileCheck %s
 
 ! CHECK: end do
 
@@ -14,7 +13,8 @@
 ! not want to see label do in the unparse only.
 ! CHECK-NOT: do [1-9]
 
-! CHECK: A DO loop should terminate with an END DO or CONTINUE
+! RUN: FileCheck --check-prefix=ERR --input-file=%t.stderr %s
+! ERR: A DO loop should terminate with an END DO or CONTINUE
 
 subroutine foo8()
   use iso_fortran_env, only : team_type
diff --git a/flang/test/Semantics/canondo17.f90 b/flang/test/Semantics/canondo17.f90
index c5e87e266efe7..6edc67c705b1f 100644
--- a/flang/test/Semantics/canondo17.f90
+++ b/flang/test/Semantics/canondo17.f90
@@ -1,11 +1,10 @@
-! RUN: %S/test_any.sh %s %t %f18
-! Error test -- DO loop uses obsolete loop termination statement
+! Error test -- DO loop uses obsolete loop termination statement (warning)
 ! See R1131 and C1133
 
 ! By default, this is not an error and label do are rewritten to non-label do.
 ! A warning is generated with -Mstandard
 
-! EXEC: ${F18} -funparse-with-symbols -Mstandard %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols -Mstandard %s 2>%t.stderr | FileCheck %s
 
 ! CHECK: end do
 
@@ -14,7 +13,8 @@
 ! not want to see label do in the unparse only.
 ! CHECK-NOT: do [1-9]
 
-! CHECK: A DO loop should terminate with an END DO or CONTINUE
+! RUN: FileCheck --check-prefix=ERR --input-file=%t.stderr %s
+! ERR: A DO loop should terminate with an END DO or CONTINUE
 
 subroutine foo9()
   real :: a(10, 10), b(10, 10) = 1.0
diff --git a/flang/test/Semantics/canondo18.f90 b/flang/test/Semantics/canondo18.f90
index d5b18e3e1b4d6..dd9c046e5be6c 100644
--- a/flang/test/Semantics/canondo18.f90
+++ b/flang/test/Semantics/canondo18.f90
@@ -1,11 +1,10 @@
-! RUN: %S/test_any.sh %s %t %f18
-! Error test -- DO loop uses obsolete loop termination statement
+! Error test -- DO loop uses obsolete loop termination statement (warning)
 ! See R1131 and C1133
 
 ! By default, this is not an error and label do are rewritten to non-label do.
 ! A warning is generated with -Mstandard
 
-! EXEC: ${F18} -funparse-with-symbols -Mstandard %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols -Mstandard %s 2>%t.stderr | FileCheck %s
 
 ! CHECK: end do
 
@@ -14,7 +13,8 @@
 ! not want to see label do in the unparse only.
 ! CHECK-NOT: do [1-9]
 
-! CHECK: A DO loop should terminate with an END DO or CONTINUE
+! RUN: FileCheck --check-prefix=ERR --input-file=%t.stderr %s
+! ERR: A DO loop should terminate with an END DO or CONTINUE
 
 subroutine foo10()
   real :: a(10, 10), b(10, 10) = 1.0
diff --git a/flang/test/Semantics/canondo19.f90 b/flang/test/Semantics/canondo19.f90
index 5b38a1909811e..1142640267b78 100644
--- a/flang/test/Semantics/canondo19.f90
+++ b/flang/test/Semantics/canondo19.f90
@@ -1,8 +1,7 @@
-! RUN: %S/test_any.sh %s %t %f18
 ! Check that if there is a label or a name on an label-do-stmt,
 ! then it is not lost when rewriting it to an non-label-do-stmt.
 
-! EXEC: ${F18} -funparse-with-symbols -Mstandard %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols -Mstandard %s 2>&1 | FileCheck %s
 
 ! CHECK: end do
 ! CHECK: 2 do
diff --git a/flang/test/Semantics/critical04.f90 b/flang/test/Semantics/critical04.f90
index 888461b9ebbaa..4d08597fb7cdd 100644
--- a/flang/test/Semantics/critical04.f90
+++ b/flang/test/Semantics/critical04.f90
@@ -1,5 +1,4 @@
-! RUN: %S/test_any.sh %s %t %f18
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK-NOT: Control flow escapes from CRITICAL
 
 subroutine test1(a, i)
diff --git a/flang/test/Semantics/doconcurrent02.f90 b/flang/test/Semantics/doconcurrent02.f90
index bb599bd0aee3c..2720e1356db1f 100644
--- a/flang/test/Semantics/doconcurrent02.f90
+++ b/flang/test/Semantics/doconcurrent02.f90
@@ -1,8 +1,6 @@
-! RUN: %S/test_any.sh %s %t %f18
-! negative tests: we don't want DO CONCURRENT semantics constraints checked
 ! when the loops are not DO CONCURRENT
 
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: not %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK-NOT: image control statement not allowed in DO CONCURRENT
 ! CHECK-NOT: RETURN not allowed in DO CONCURRENT
 ! CHECK-NOT: call to impure procedure in DO CONCURRENT not allowed
diff --git a/flang/test/Semantics/doconcurrent03.f90 b/flang/test/Semantics/doconcurrent03.f90
index 1ba96b5777233..d98db8dd60c82 100644
--- a/flang/test/Semantics/doconcurrent03.f90
+++ b/flang/test/Semantics/doconcurrent03.f90
@@ -1,5 +1,4 @@
-! RUN: %S/test_any.sh %s %t %f18
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: not %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK: Control flow escapes from DO CONCURRENT
 ! CHECK: branch into loop body from outside
 ! CHECK: the loop branched into
diff --git a/flang/test/Semantics/doconcurrent04.f90 b/flang/test/Semantics/doconcurrent04.f90
index cc73d3fdd8bf8..079c8d423503f 100644
--- a/flang/test/Semantics/doconcurrent04.f90
+++ b/flang/test/Semantics/doconcurrent04.f90
@@ -1,7 +1,6 @@
-! RUN: %S/test_any.sh %s %t %f18
 ! C1122 The index-name shall be a named scalar variable of type integer.
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
-! CHECK: Must have INTEGER type, but is REAL\\(4\\)
+! RUN: not %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
+! CHECK: Must have INTEGER type, but is REAL(4)
 
 subroutine do_concurrent_test1(n)
   implicit none
diff --git a/flang/test/Semantics/doconcurrent07.f90 b/flang/test/Semantics/doconcurrent07.f90
index 5882c130bb1bd..d9c494efe0ab7 100644
--- a/flang/test/Semantics/doconcurrent07.f90
+++ b/flang/test/Semantics/doconcurrent07.f90
@@ -1,5 +1,4 @@
-! RUN: %S/test_any.sh %s %t %f18
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK-NOT: exit from DO CONCURRENT construct
 
 subroutine do_concurrent_test1(n)
diff --git a/flang/test/Semantics/equivalence01.f90 b/flang/test/Semantics/equivalence01.f90
index de60e8c9b2a44..404be570d657c 100644
--- a/flang/test/Semantics/equivalence01.f90
+++ b/flang/test/Semantics/equivalence01.f90
@@ -113,7 +113,7 @@ subroutine s9
   equivalence(d(1:n), i)
   character(4) :: a(10)
   equivalence(c, a(10)(1:2))
-  !ERROR: 'a(10)' and 'a(10)(2:)' cannot have the same first storage unit
+  !ERROR: 'a(10_8)(2_8:2_8)' and 'a(10_8)(1_8:1_8)' cannot have the same first storage unit
   equivalence(c, a(10)(2:3))
 end
 
@@ -165,13 +165,20 @@ module s13
 
 module s14
   real :: a(10), b, c, d
-  !ERROR: 'a(1)' and 'a(2)' cannot have the same first storage unit
+  !ERROR: 'a(2_8)' and 'a(1_8)' cannot have the same first storage unit
   equivalence(a(1), a(2))
   equivalence(b, a(3))
-  !ERROR: 'a(3)' and 'a(4)' cannot have the same first storage unit
+  !ERROR: 'a(4_8)' and 'a(3_8)' cannot have the same first storage unit
   equivalence(a(4), b)
   equivalence(c, a(5))
+  !ERROR: 'a(6_8)' and 'a(5_8)' cannot have the same first storage unit
   equivalence(a(6), d)
-  !ERROR: 'a(5)' and 'a(6)' cannot have the same first storage unit
   equivalence(c, d)
 end
+
+module s15
+  real :: a(2), b(2)
+  equivalence(a(2),b(1))
+  !ERROR: 'a(3_8)' and 'a(1_8)' cannot have the same first storage unit
+  equivalence(b(2),a(1))
+end module
diff --git a/flang/test/Semantics/getdefinition01.f90 b/flang/test/Semantics/getdefinition01.f90
index 3130246c00226..57141b671754f 100644
--- a/flang/test/Semantics/getdefinition01.f90
+++ b/flang/test/Semantics/getdefinition01.f90
@@ -1,4 +1,3 @@
-!RUN: %S/test_any.sh %s %t %f18
 ! Tests -fget-definition returning source position of symbol definition.
 module m1
  private :: f
@@ -16,13 +15,13 @@ recursive pure function f() result(x)
  end function
 end module
 
-! EXEC: echo %t 1>&2;
-! EXEC: ${F18} -fget-definition 7 17 18 -fparse-only %s > %t;
-! EXEC: ${F18} -fget-definition 8 20 23 -fparse-only %s >> %t;
-! EXEC: ${F18} -fget-definition 15 3 4 -fparse-only %s >> %t;
-! EXEC: ${F18} -fget-definition -fparse-only %s >> %t 2>&1;
-! EXEC: cat %t | ${FileCheck} %s
-! CHECK:x:.*getdefinition01.f90, 6, 21-22
-! CHECK:yyy:.*getdefinition01.f90, 6, 24-27
-! CHECK:x:.*getdefinition01.f90, 14, 24-25
-! CHECK:Invalid argument to -fget-definitions
+! RUN and CHECK lines at the bottom as this test is sensitive to line numbers
+! RUN: %f18 -fget-definition 6 17 18 -fparse-only %s | FileCheck --check-prefix=CHECK1 %s
+! RUN: %f18 -fget-definition 7 20 23 -fparse-only %s | FileCheck --check-prefix=CHECK2 %s
+! RUN: %f18 -fget-definition 14 3 4 -fparse-only %s | FileCheck --check-prefix=CHECK3 %s
+! CHECK1: x:{{.*}}getdefinition01.f90, 5, 21-22
+! CHECK2: yyy:{{.*}}getdefinition01.f90, 5, 24-27
+! CHECK3: x:{{.*}}getdefinition01.f90, 13, 24-25
+
+! RUN: not %f18 -fget-definition -fparse-only %s 2>&1 | FileCheck --check-prefix=CHECK-ERROR %s
+! CHECK-ERROR: Invalid argument to -fget-definitions
diff --git a/flang/test/Semantics/getdefinition02.f b/flang/test/Semantics/getdefinition02.f
index 430c01e4e5efc..ee7a96750da5f 100644
--- a/flang/test/Semantics/getdefinition02.f
+++ b/flang/test/Semantics/getdefinition02.f
@@ -1,4 +1,3 @@
-!RUN: %S/test_any.sh %s %t %f18
 ! Tests -fget-definition with fixed form.
       module m2
        private :: f
@@ -17,10 +16,10 @@ recursive pure function f() result(x)
        end function
       end module
 
-! EXEC: ${F18} -fget-definition 8 9 10 -fparse-only %s > %t;
-! EXEC: ${F18} -fget-definition 9 26 29 -fparse-only %s >> %t;
-! EXEC: ${F18} -fget-definition 16 9 10 -fparse-only %s >> %t;
-! EXEC: cat %t | ${FileCheck} %s
-! CHECK:x:.*getdefinition02.f, 6, 27-28
-! CHECK:yyy:.*getdefinition02.f, 6, 30-33
-! CHECK:x:.*getdefinition02.f, 15, 30-31
+! RUN and CHECK lines here as test is sensitive to line numbers
+! RUN: %f18 -fget-definition 7 9 10 -fparse-only %s 2>&1 | FileCheck --check-prefix=CHECK1 %s
+! RUN: %f18 -fget-definition 8 26 29 -fparse-only %s 2>&1 | FileCheck --check-prefix=CHECK2 %s
+! RUN: %f18 -fget-definition 15 9 10 -fparse-only %s 2>&1 | FileCheck --check-prefix=CHECK3 %s
+! CHECK1: x:{{.*}}getdefinition02.f, 5, 27-28
+! CHECK2: yyy:{{.*}}getdefinition02.f, 5, 30-33
+! CHECK3: x:{{.*}}getdefinition02.f, 14, 30-31
diff --git a/flang/test/Semantics/getdefinition03-a.f90 b/flang/test/Semantics/getdefinition03-a.f90
index 4111166d5f8a6..ecf8c9b483890 100644
--- a/flang/test/Semantics/getdefinition03-a.f90
+++ b/flang/test/Semantics/getdefinition03-a.f90
@@ -1,5 +1,4 @@
 ! Tests -fget-definition with INCLUDE
-!RUN: %S/test_any.sh %s %t %f18
 INCLUDE "Inputs/getdefinition03-b.f90"
 
 program main
@@ -8,8 +7,7 @@ program main
  x = f
 end program
 
-! EXEC: ${F18} -fget-definition 8 6 7 -fparse-only %s > %t;
-! EXEC: ${F18} -fget-definition 8 2 3 -fparse-only %s >> %t;
-! EXEC: cat %t | ${FileCheck} %s;
-! CHECK:f:.*getdefinition03-b.f90, 2, 12-13
-! CHECK:x:.*getdefinition03-a.f90, 7, 13-14
+! RUN: %f18 -fget-definition 7 6 7 -fparse-only %s | FileCheck --check-prefix=CHECK1 %s
+! RUN: %f18 -fget-definition 7 2 3 -fparse-only %s | FileCheck --check-prefix=CHECK2 %s
+! CHECK1: f:{{.*}}getdefinition03-b.f90, 2, 12-13
+! CHECK2: x:{{.*}}getdefinition03-a.f90, 6, 13-14
diff --git a/flang/test/Semantics/getdefinition04.f90 b/flang/test/Semantics/getdefinition04.f90
index ab9564e28f4cd..72429599595b2 100644
--- a/flang/test/Semantics/getdefinition04.f90
+++ b/flang/test/Semantics/getdefinition04.f90
@@ -1,4 +1,3 @@
-!RUN: %S/test_any.sh %s %t %f18
 ! Tests -fget-definition with COMMON block with same name as variable.
 program main
   integer :: x
@@ -7,5 +6,5 @@ program main
   x = y
 end program
 
-! EXEC: ${F18} -fget-definition 7 3 4 -fparse-only %s | ${FileCheck} %s
-! CHECK:x:.*getdefinition04.f90, 4, 14-15
+! RUN: %f18 -fget-definition 6 3 4 -fparse-only %s | FileCheck %s
+! CHECK: x:{{.*}}getdefinition04.f90, 3, 14-15
diff --git a/flang/test/Semantics/getdefinition05.f90 b/flang/test/Semantics/getdefinition05.f90
index 81a5b605d0281..315e9570d3528 100644
--- a/flang/test/Semantics/getdefinition05.f90
+++ b/flang/test/Semantics/getdefinition05.f90
@@ -1,4 +1,3 @@
-!RUN: %S/test_any.sh %s %t %f18
 ! Tests -fget-symbols-sources with BLOCK that contains same variable name as 
 ! another in an outer scope.
 program main
@@ -13,9 +12,8 @@ program main
 end program
 
 !! Inner x
-! EXEC: ${F18} -fget-definition 10 5 6 -fparse-only %s > %t;
-! CHECK:x:.*getdefinition05.f90, 8, 16-17
+! RUN: %f18 -fget-definition 9 5 6 -fparse-only %s | FileCheck --check-prefix=CHECK1 %s
+! CHECK1: x:{{.*}}getdefinition05.f90, 7, 16-17
 !! Outer y
-! EXEC: ${F18} -fget-definition 12 7 8 -fparse-only %s >> %t;
-! CHECK:y:.*getdefinition05.f90, 6, 14-15
-! EXEC: cat %t | ${FileCheck} %s;
+! RUN: %f18 -fget-definition 11 7 8 -fparse-only %s | FileCheck --check-prefix=CHECK2 %s
+! CHECK2: y:{{.*}}getdefinition05.f90, 5, 14-15
diff --git a/flang/test/Semantics/getsymbols01.f90 b/flang/test/Semantics/getsymbols01.f90
index 8142e43d3a71e..8f50304825dc7 100644
--- a/flang/test/Semantics/getsymbols01.f90
+++ b/flang/test/Semantics/getsymbols01.f90
@@ -1,4 +1,3 @@
-!RUN: %S/test_any.sh %s %t %f18
 ! Tests -fget-symbols-sources finding all symbols in file.
 module mm1
  private :: f
@@ -16,11 +15,11 @@ recursive pure function f() result(x)
  end function
 end module
 
-! EXEC: ${F18} -fget-symbols-sources -fparse-only %s 2>&1 | ${FileCheck} %s
-! CHECK-ONCE:mm1:.*getsymbols01.f90, 3, 8-11
-! CHECK-ONCE:f:.*getsymbols01.f90, 13, 26-27
-! CHECK-ONCE:s:.*getsymbols01.f90, 6, 18-19
-! CHECK-ONCE:ss:.*getsymbols01.f90, 10, 19-21
-! CHECK-ONCE:x:.*getsymbols01.f90, 6, 21-22
-! CHECK-ONCE:y:.*getsymbols01.f90, 6, 24-25
-! CHECK-ONCE:x:.*getsymbols01.f90, 14, 24-25
+! RUN: %f18 -fget-symbols-sources -fparse-only %s 2>&1 | FileCheck %s
+! CHECK-COUNT-1:mm1:{{.*}}getsymbols01.f90, 2, 8-11
+! CHECK-COUNT-1:s:{{.*}}getsymbols01.f90, 5, 18-19
+! CHECK-COUNT-1:x:{{.*}}getsymbols01.f90, 5, 21-22
+! CHECK-COUNT-1:y:{{.*}}getsymbols01.f90, 5, 24-25
+! CHECK-COUNT-1:ss:{{.*}}getsymbols01.f90, 9, 19-21
+! CHECK-COUNT-1:f:{{.*}}getsymbols01.f90, 12, 26-27
+! CHECK-COUNT-1:x:{{.*}}getsymbols01.f90, 13, 24-25
diff --git a/flang/test/Semantics/getsymbols02.f90 b/flang/test/Semantics/getsymbols02.f90
index 86f10911a945a..4c8f0710eb23d 100644
--- a/flang/test/Semantics/getsymbols02.f90
+++ b/flang/test/Semantics/getsymbols02.f90
@@ -1 +1,14 @@
-!RUN: %S/test_any.sh '%S/Inputs/getsymbols02-*' %t %f18
+! Tests -fget-symbols-sources with modules.
+
+PROGRAM helloworld
+    use mm2b
+    implicit none
+    integer::i
+    i = callget5()
+ENDPROGRAM
+
+! RUN: %f18 -fparse-only %S/Inputs/getsymbols02-a.f90
+! RUN: %f18 -fparse-only %S/Inputs/getsymbols02-b.f90
+! RUN: %f18 -fget-symbols-sources -fparse-only %s 2>&1 | FileCheck %s
+! CHECK: get5: mm2a
+! CHECK: callget5: mm2b
diff --git a/flang/test/Semantics/getsymbols03-a.f90 b/flang/test/Semantics/getsymbols03-a.f90
index 51199ce24eb26..c11aee03048c9 100644
--- a/flang/test/Semantics/getsymbols03-a.f90
+++ b/flang/test/Semantics/getsymbols03-a.f90
@@ -1,5 +1,4 @@
 ! Tests -fget-symbols with INCLUDE
-!RUN: %S/test_any.sh %s %t %f18
 INCLUDE "Inputs/getsymbols03-b.f90"
 
 program main
@@ -8,8 +7,8 @@ program main
  x = f
 end program
 
-! EXEC: ${F18} -fget-symbols-sources -fparse-only %s 2>&1 | ${FileCheck} %s
-! CHECK:mm3:.*getsymbols03-b.f90, 1, 8-11
-! CHECK:f:.*getsymbols03-b.f90, 2, 12-13
-! CHECK:main:.*getsymbols03-a.f90, 5, 9-13
-! CHECK:x:.*getsymbols03-a.f90, 7, 13-14
+! RUN: %f18 -fget-symbols-sources -fparse-only %s 2>&1 | FileCheck %s
+! CHECK:mm3:{{.*}}getsymbols03-b.f90, 1, 8-11
+! CHECK:f:{{.*}}getsymbols03-b.f90, 2, 12-13
+! CHECK:main:{{.*}}getsymbols03-a.f90, 4, 9-13
+! CHECK:x:{{.*}}getsymbols03-a.f90, 6, 13-14
diff --git a/flang/test/Semantics/getsymbols04.f90 b/flang/test/Semantics/getsymbols04.f90
index e0d3a40f4e205..4decfc78560ad 100644
--- a/flang/test/Semantics/getsymbols04.f90
+++ b/flang/test/Semantics/getsymbols04.f90
@@ -1,4 +1,3 @@
-!RUN: %S/test_any.sh %s %t %f18
 ! Tests -fget-symbols-sources with COMMON.
 program main
   integer :: x
@@ -7,7 +6,7 @@ program main
   x = y
 end program
 
-! EXEC: ${F18} -fget-symbols-sources -fparse-only %s 2>&1 | ${FileCheck} %s
-! CHECK:x:.*getsymbols04.f90, 4, 14-15
-! CHECK:y:.*getsymbols04.f90, 5, 14-15
-! CHECK:x:.*getsymbols04.f90, 6, 11-12
+! RUN: %f18 -fget-symbols-sources -fparse-only %s 2>&1 | FileCheck %s
+! CHECK:x:{{.*}}getsymbols04.f90, 3, 14-15
+! CHECK:y:{{.*}}getsymbols04.f90, 4, 14-15
+! CHECK:x:{{.*}}getsymbols04.f90, 5, 11-12
diff --git a/flang/test/Semantics/getsymbols05.f90 b/flang/test/Semantics/getsymbols05.f90
index 36bf61b4e738b..30dcb2278e002 100644
--- a/flang/test/Semantics/getsymbols05.f90
+++ b/flang/test/Semantics/getsymbols05.f90
@@ -1,4 +1,3 @@
-!RUN: %S/test_any.sh %s %t %f18
 ! Tests -fget-symbols-sources with COMMON.
 program main
   integer :: x
@@ -10,7 +9,7 @@ program main
   x = y
 end program
 
-! EXEC: ${F18} -fget-symbols-sources -fparse-only %s 2>&1 | ${FileCheck} %s
-! CHECK:x:.*getsymbols05.f90, 4, 14-15
-! CHECK:y:.*getsymbols05.f90, 5, 14-15
-! CHECK:x:.*getsymbols05.f90, 7, 16-17
+! RUN: %f18 -fget-symbols-sources -fparse-only %s 2>&1 | FileCheck %s
+! CHECK:x:{{.*}}getsymbols05.f90, 3, 14-15
+! CHECK:y:{{.*}}getsymbols05.f90, 4, 14-15
+! CHECK:x:{{.*}}getsymbols05.f90, 6, 16-17
diff --git a/flang/test/Semantics/label01.F90 b/flang/test/Semantics/label01.F90
index aa1b80025d847..36a4fa56a5437 100644
--- a/flang/test/Semantics/label01.F90
+++ b/flang/test/Semantics/label01.F90
@@ -1,15 +1,12 @@
-! RUN: %S/test_any.sh %s %t %f18
-! EXEC: ${F18} -funparse-with-symbols %s -o /dev/null 2>&1 | grep -v 'procedure conflicts' | ${FileCheck} %s
-! CHECK-NOT: error:[[:space:]]
-
-! FIXME: filter out the array/function syntax issues (procedure conflicts)
-! for now...
+! RUN: %f18 -funparse-with-symbols -DSTRICT_F18 -Mstandard %s 2>&1 | FileCheck %s
+! RUN: %f18 -funparse-with-symbols -DARCHAIC_FORTRAN %s 2>&1 | FileCheck %s
+! CHECK-NOT: :{{[[:space:]]}}error:{{[[:space:]]}}
+! FIXME: the above check line does not work because diags are not emitted with error: in them.
 
 ! these are the conformance tests
 ! define STRICT_F18 to eliminate tests of features not in F18
 ! define ARCHAIC_FORTRAN to add test of feature found in Fortran before F95
 
-
 subroutine sub00(a,b,n,m)
   integer :: n, m
   real a(n)
@@ -83,6 +80,7 @@ subroutine do_loop07(a,n)
   end do loopone
 end subroutine do_loop07
 
+#ifndef STRICT_F18
 subroutine do_loop08(a,b,n,m,nn)
   integer :: n, m, nn
   real, dimension(n,n) :: a
@@ -114,6 +112,7 @@ subroutine do_loop08(a,b,n,m,nn)
   end do loopone
 111 print *, "done"
 end subroutine do_loop08
+#endif
 
 #ifndef STRICT_F18
 ! extended ranges supported by PGI, gfortran gives warnings
@@ -123,7 +122,7 @@ subroutine do_loop09(a,n,j)
   goto 400
 200 print *, "found the index", j
   print *, "value at", j, "is", a(j)
-  goto 300
+  goto 300 ! FIXME: emits diagnostic even without -Mstandard
 400  do 100 i = 1, n
      if (i .eq. j) then
         goto 200	! extension: extended GOTO ranges
@@ -168,7 +167,7 @@ subroutine arith_if12(i)
 end subroutine arith_if12
 #endif
 
-#if 0
+#ifndef STRICT_F18
 subroutine alt_return_spec13(i,*,*,*)
 9 continue
 8 labelme: if (i .lt. 42) then
@@ -192,6 +191,7 @@ subroutine alt_return_spec14(i)
 end subroutine alt_return_spec14
 #endif
 
+#ifndef STRICT_F18
 subroutine specifiers15(a,b,x)
   integer x
   OPEN (10, file="myfile.dat", err=100)
@@ -212,6 +212,7 @@ subroutine specifiers15(a,b,x)
 50 WRITE (11,30,err=100) b
   CLOSE (11)
 end subroutine specifiers15
+#endif
 
 #if !defined(STRICT_F18) && defined(ARCHAIC_FORTRAN)
 ! assigned goto was deleted in F95. PGI supports, gfortran gives warnings
diff --git a/flang/test/Semantics/label02.f90 b/flang/test/Semantics/label02.f90
index 26f98e578273e..fa4f37ec3a44c 100644
--- a/flang/test/Semantics/label02.f90
+++ b/flang/test/Semantics/label02.f90
@@ -1,7 +1,5 @@
-! RUN: %S/test_any.sh %s %t %f18
-! negative test -- invalid labels, out of range
 
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: not %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK: Label '0' is out of range
 ! CHECK: Label '100000' is out of range
 ! CHECK: Label '123456' is out of range
diff --git a/flang/test/Semantics/label03.f90 b/flang/test/Semantics/label03.f90
index 40eb6f391f184..ecc21f92b93e4 100644
--- a/flang/test/Semantics/label03.f90
+++ b/flang/test/Semantics/label03.f90
@@ -1,7 +1,5 @@
-! RUN: %S/test_any.sh %s %t %f18
-! negative test -- invalid labels, out of range
 
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: not %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK: DO loop doesn't properly nest
 ! CHECK: DO loop conflicts
 ! CHECK: Label '30' cannot be found
diff --git a/flang/test/Semantics/label04.f90 b/flang/test/Semantics/label04.f90
index 3180b0c23bd78..ea496fb95789a 100644
--- a/flang/test/Semantics/label04.f90
+++ b/flang/test/Semantics/label04.f90
@@ -1,7 +1,5 @@
-! RUN: %S/test_any.sh %s %t %f18
-! negative test -- invalid labels, out of range
 
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK: branch into loop body from outside
 ! CHECK: do 10 i = 1, m
 ! CHECK: the loop branched into
diff --git a/flang/test/Semantics/label05.f90 b/flang/test/Semantics/label05.f90
index f962b03dd7195..fbcdb4f22cb9e 100644
--- a/flang/test/Semantics/label05.f90
+++ b/flang/test/Semantics/label05.f90
@@ -1,7 +1,5 @@
-! RUN: %S/test_any.sh %s %t %f18
-! negative test -- invalid labels, out of range
 
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: not %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK: Label '50' was not found
 ! CHECK: Label '55' is not in scope
 ! CHECK: Label '70' is not a branch target
diff --git a/flang/test/Semantics/label06.f90 b/flang/test/Semantics/label06.f90
index ab948e12f086d..16be4df30cb04 100644
--- a/flang/test/Semantics/label06.f90
+++ b/flang/test/Semantics/label06.f90
@@ -1,7 +1,5 @@
-! RUN: %S/test_any.sh %s %t %f18
-! negative test -- invalid labels, out of range
 
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: not %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK: Label '10' is not in scope
 ! CHECK: Label '20' was not found
 ! CHECK: Label '30' is not a branch target
diff --git a/flang/test/Semantics/label07.f90 b/flang/test/Semantics/label07.f90
index 5a73c74bde22c..a01b58e16d329 100644
--- a/flang/test/Semantics/label07.f90
+++ b/flang/test/Semantics/label07.f90
@@ -1,7 +1,5 @@
-! RUN: %S/test_any.sh %s %t %f18
-! negative test -- invalid labels, out of range
 
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: not %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK: Label '30' is not a branch target
 ! CHECK: Control flow use of '30'
 ! CHECK: Label '10' is not in scope
diff --git a/flang/test/Semantics/label08.f90 b/flang/test/Semantics/label08.f90
index 5b5c727d0ad6c..57858022a526b 100644
--- a/flang/test/Semantics/label08.f90
+++ b/flang/test/Semantics/label08.f90
@@ -1,7 +1,5 @@
-! RUN: %S/test_any.sh %s %t %f18
-! negative test -- invalid labels, out of range
 
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: not %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK: CYCLE construct-name is not in scope
 ! CHECK: IF construct name unexpected
 ! CHECK: unnamed IF statement
diff --git a/flang/test/Semantics/label09.f90 b/flang/test/Semantics/label09.f90
index d3968e4b3d6e2..27644b671471f 100644
--- a/flang/test/Semantics/label09.f90
+++ b/flang/test/Semantics/label09.f90
@@ -1,5 +1,4 @@
-! RUN: %S/test_any.sh %s %t %f18
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: not %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK: Label '60' was not found
 
 subroutine s(a)
diff --git a/flang/test/Semantics/label10.f90 b/flang/test/Semantics/label10.f90
index dc19e5dacfc35..ad234ccb4bbb1 100644
--- a/flang/test/Semantics/label10.f90
+++ b/flang/test/Semantics/label10.f90
@@ -1,5 +1,4 @@
-! RUN: %S/test_any.sh %s %t %f18
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: not %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK: '60' not a FORMAT
 ! CHECK: data transfer use of '60'
 
diff --git a/flang/test/Semantics/label11.f90 b/flang/test/Semantics/label11.f90
index 1e477e2a250cd..3f139f3e4fb2e 100644
--- a/flang/test/Semantics/label11.f90
+++ b/flang/test/Semantics/label11.f90
@@ -1,12 +1,11 @@
-! RUN: %S/test_any.sh %s %t %f18
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: not %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK: BLOCK DATA subprogram name mismatch
 ! CHECK: should be
 ! CHECK: FUNCTION name mismatch
 ! CHECK: SUBROUTINE name mismatch
 ! CHECK: PROGRAM name mismatch
 ! CHECK: SUBMODULE name mismatch
-! CHECK: INTERFACE generic-name .t7. mismatch
+! CHECK: INTERFACE generic-name (t7) mismatch
 ! CHECK: mismatched INTERFACE
 ! CHECK: derived type definition name mismatch
 ! CHECK: MODULE PROCEDURE name mismatch
diff --git a/flang/test/Semantics/label12.f90 b/flang/test/Semantics/label12.f90
index 8dbe7e46ea041..c917a116e31e7 100644
--- a/flang/test/Semantics/label12.f90
+++ b/flang/test/Semantics/label12.f90
@@ -1,5 +1,4 @@
-! RUN: %S/test_any.sh %s %t %f18
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: not %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK: expected end of statement
 
 subroutine s
diff --git a/flang/test/Semantics/label13.f90 b/flang/test/Semantics/label13.f90
index bdd5d68e66da0..f03156aa1775d 100644
--- a/flang/test/Semantics/label13.f90
+++ b/flang/test/Semantics/label13.f90
@@ -1,5 +1,4 @@
-! RUN: %S/test_any.sh %s %t %f18
-! EXEC: ${F18} -funparse-with-symbols %s 2>&1 | ${FileCheck} %s
+! RUN: %f18 -funparse-with-symbols %s 2>&1 | FileCheck %s
 ! CHECK: branch into loop body from outside
 ! CHECK: the loop branched into
 
diff --git a/flang/test/Semantics/label14.f90 b/flang/test/Semantics/label14.f90
index 700508ab878b3..ebb865c9e85ba 100644
--- a/flang/test/Semantics/label14.f90
+++ b/flang/test/Semantics/label14.f90
@@ -1,9 +1,8 @@
-! RUN: %S/test_any.sh %s %t %f18
 ! Tests implemented for this standard
 ! 11.1.4 - 4 It is permissible to branch to and end-block-stmt only withinh its
 !            Block Construct
 
-! EXEC: ${F18} %s 2>&1 | ${FileCheck} %s
+! RUN: not %f18 %s 2>&1 | FileCheck %s
 ! CHECK: Label '20' is not in scope
 
 subroutine s1
diff --git a/flang/test/Semantics/num_images.f90 b/flang/test/Semantics/num_images.f90
new file mode 100644
index 0000000000000..788b54224d8f8
--- /dev/null
+++ b/flang/test/Semantics/num_images.f90
@@ -0,0 +1,30 @@
+! RUN: %S/test_errors.sh %s %t %f18
+! Check for semantic errors in num_images() function calls
+
+subroutine test
+
+  ! correct calls, should produce no errors
+  print *, num_images()
+  print *, num_images(team_number=1)
+  print *, num_images(1)
+
+  ! incorrectly typed argument
+  ! the error is seen as too many arguments to the num_images() call with no arguments
+  !ERROR: too many actual arguments for intrinsic 'num_images'
+  print *, num_images(3.4)
+
+  ! call with too many arguments
+  !ERROR: too many actual arguments for intrinsic 'num_images'
+  print *, num_images(1, 1)
+
+  ! keyword argument with incorrect type
+  !ERROR: unknown keyword argument to intrinsic 'num_images'
+  print *, num_images(team_number=3.4)
+
+  ! incorrect keyword argument
+  !ERROR: unknown keyword argument to intrinsic 'num_images'
+  print *, num_images(team_numbers=1)
+
+  !TODO: test num_images() calls related to team_type argument
+
+end subroutine
diff --git a/flang/test/Semantics/omp-clause-validity01.f90 b/flang/test/Semantics/omp-clause-validity01.f90
index 06fd7203763d6..77e40e323e5f9 100644
--- a/flang/test/Semantics/omp-clause-validity01.f90
+++ b/flang/test/Semantics/omp-clause-validity01.f90
@@ -403,6 +403,10 @@
   !ERROR: Internal: no symbol found for 'i'
   !$omp ordered depend(sink:i-1)
   !$omp flush (c)
+  !$omp flush acq_rel
+  !$omp flush release
+  !$omp flush acquire
+  !$omp flush release (c)
   !$omp cancel DO
   !$omp cancellation point parallel
 
@@ -454,7 +458,6 @@
   enddo
   !$omp end taskloop simd
 
-  !ERROR: REDUCTION clause is not allowed on the TASKLOOP SIMD directive
   !$omp taskloop simd reduction(+:a)
   do i = 1, N
      a = a + 3.14
diff --git a/flang/test/Semantics/resolve45.f90 b/flang/test/Semantics/resolve45.f90
index 8ab75595d4059..3e98ff662a171 100644
--- a/flang/test/Semantics/resolve45.f90
+++ b/flang/test/Semantics/resolve45.f90
@@ -15,8 +15,15 @@ function f2(x, y)
   !ERROR: SAVE attribute may not be applied to dummy argument 'x'
   complex, save :: x
   allocatable :: y
+  integer :: y
   !ERROR: SAVE attribute may not be applied to dummy argument 'y'
-  integer, save :: y
+  save :: y
+end
+
+! SAVE statement should not trigger the above errors
+function f2b(x, y)
+  real :: x, y
+  save
 end
 
 subroutine s3(x)
diff --git a/flang/test/Semantics/resolve53.f90 b/flang/test/Semantics/resolve53.f90
index 69c13dbe52eb8..acb27c8575b7d 100644
--- a/flang/test/Semantics/resolve53.f90
+++ b/flang/test/Semantics/resolve53.f90
@@ -457,3 +457,26 @@ integer function f3(i, j)
     integer :: i, j
   end
 end
+
+module m20
+  interface operator(.not.)
+    real function f(x)
+      character(*),intent(in) :: x
+    end function
+  end interface
+  interface operator(+)
+    procedure f
+  end interface
+end module
+
+subroutine s1()
+  use m20
+  interface operator(.not.)
+    !ERROR: Procedure 'f' is already specified in generic 'operator(.not.)'
+    procedure f
+  end interface
+  interface operator(+)
+    !ERROR: Procedure 'f' is already specified in generic 'operator(+)'
+    procedure f
+  end interface
+end subroutine s1
diff --git a/flang/test/Semantics/resolve93.f90 b/flang/test/Semantics/resolve93.f90
new file mode 100644
index 0000000000000..2c3764d948eee
--- /dev/null
+++ b/flang/test/Semantics/resolve93.f90
@@ -0,0 +1,44 @@
+! RUN: %S/test_errors.sh %s %t %f18
+subroutine s1()
+  character(10) str
+  character(10) str1
+  !ERROR: Cannot reference function 'str' as data
+  print *, str(1:9), str(7)
+  block
+    character(10) str2
+    character(10) str3
+    !ERROR: Cannot reference function 'str1' as data
+    print *, str1(1:9), str1(7)
+    !ERROR: 'str2' is not an array
+    print *, str2(1:9), str2(7)
+    !ERROR: Cannot reference function 'str3' as data
+    print *, str3(7), str3(1:9)
+  end block
+end subroutine s1
+
+subroutine s2()
+  character(10) func
+  !ERROR: Cannot reference function 'func' as data
+  print *, func(7), func(1:9)
+end subroutine s2
+
+subroutine s3()
+  real(8) :: func
+  !ERROR: Cannot reference function 'func' as data
+  print *, func(7), func(1:6)
+end subroutine s3
+
+subroutine s4()
+  real(8) :: local
+  real(8) :: local1
+  !ERROR: Cannot reference function 'local' as data
+  print *, local(1:6), local(7)
+  !ERROR: Cannot reference function 'local1' as data
+  print *, local1(7), local1(1:6)
+end subroutine s4
+
+subroutine s5(arg)
+  integer :: iVar
+  external :: arg
+  iVar = loc(arg)
+end subroutine s5
diff --git a/flang/test/Semantics/resolve94.f90 b/flang/test/Semantics/resolve94.f90
new file mode 100644
index 0000000000000..bf91d61514609
--- /dev/null
+++ b/flang/test/Semantics/resolve94.f90
@@ -0,0 +1,72 @@
+! RUN: %S/test_errors.sh %s %t %f18
+! C929   No specifier shall appear more than once in a given 
+!   image-selector-spec-list.
+! C930 TEAM and TEAM_NUMBER shall not both appear in the same
+!   image-selector-spec-list.
+! C931 A stat-variable in an image-selector shall not be a coindexed object.
+subroutine s1()
+  use ISO_FORTRAN_ENV
+  type(team_type) :: team1, team2
+  real :: rCoarray[10,20,*]
+  real :: rVar1, rVar2
+  integer :: iVar1, iVar2
+  integer, dimension(4) :: intArray
+  integer :: intScalarCoarray[*]
+  integer :: intCoarray[3, 4, *]
+  integer :: smallIntCoarray[4, *]
+  intCoVar = 343
+  ! OK
+  rVar1 = rCoarray[1,2,3]
+  !ERROR: 'rcoarray' has corank 3, but coindexed reference has 2 cosubscripts
+  rVar1 = rCoarray[1,2]
+  !ERROR: Must have INTEGER type, but is REAL(4)
+  rVar1 = rCoarray[1,2,3.4]
+  !ERROR: Must have INTEGER type, but is REAL(4)
+  iVar1 = smallIntCoarray[3.4]
+  !ERROR: Must be a scalar value, but is a rank-1 array
+  rVar1 = rCoarray[1,intArray,3]
+  ! OK
+  rVar1 = rCoarray[1,2,3,STAT=iVar1, TEAM=team2]
+  !ERROR: Team value must be of type TEAM_TYPE from module ISO_FORTRAN_ENV
+  rVar1 = rCoarray[1,2,3,STAT=iVar1, TEAM=2]
+  ! OK
+  rVar1 = rCoarray[1,2,3,STAT=iVar1, TEAM_NUMBER=38]
+  ! OK
+  rVar1 = rCoarray[1,2,3,STAT=iVar1]
+  ! OK
+  rVar1 = rCoarray[1,2,3,STAT=intArray(2)]
+  !ERROR: Must have INTEGER type, but is REAL(4)
+  rVar1 = rCoarray[1,2,3,STAT=rVar2]
+  !ERROR: Must be a scalar value, but is a rank-1 array
+  rVar1 = rCoarray[1,2,3,STAT=intArray]
+  ! Error on C929, no specifier can appear more than once
+  !ERROR: STAT variable can only be specified once
+  rVar1 = rCoarray[1,2,3,STAT=iVar1, STAT=iVar2]
+  ! OK
+  rVar1 = rCoarray[1,2,3,TEAM=team1]
+  ! Error on C929, no specifier can appear more than once
+  !ERROR: TEAM value can only be specified once
+  rVar1 = rCoarray[1,2,3,TEAM=team1, TEAM=team2]
+  ! OK
+  rVar1 = rCoarray[1,2,3,TEAM_NUMBER=37]
+  ! OK
+  rVar1 = rCoarray[1,2,3,TEAM_NUMBER=iVar1]
+  ! Error, team number is a scalar integer expression
+  !ERROR: Must be a scalar value, but is a rank-1 array
+  rVar1 = rCoarray[1,2,3,TEAM_NUMBER=intArray]
+  ! Error, team number is a scalar integer expression
+  !ERROR: Must have INTEGER type, but is REAL(4)
+  rVar1 = rCoarray[1,2,3,TEAM_NUMBER=3.7]
+  ! Error on C929, no specifier can appear more than once
+  !ERROR: TEAM_NUMBER value can only be specified once
+  rVar1 = rCoarray[1,2,3,TEAM_NUMBER=37, TEAM_NUMBER=37]
+  !ERROR: Cannot specify both TEAM and TEAM_NUMBER
+  rVar1 = rCoarray[1,2,3,TEAM=team1, TEAM_NUMBER=37]
+  !ERROR: Cannot specify both TEAM and TEAM_NUMBER
+  rVar1 = rCoarray[1,2,3,TEAM_number=43, TEAM=team1]
+  ! OK for a STAT variable to be a coarray integer
+  rVar1 = rCoarray[1,2,3,stat=intScalarCoarray]
+  ! Error for a STAT variable to be a coindexed object
+  !ERROR: Image selector STAT variable must not be a coindexed object
+  rVar1 = rCoarray[1,2,3,stat=intCoarray[2,3, 4]]
+end subroutine s1
diff --git a/flang/test/Semantics/test_any.sh b/flang/test/Semantics/test_any.sh
deleted file mode 100755
index 3ad960de323ab..0000000000000
--- a/flang/test/Semantics/test_any.sh
+++ /dev/null
@@ -1,65 +0,0 @@
-#/usr/bin/env bash
-# Compile a source file with '-funparse-with-symbols' and verify
-# we get the right symbols in the output, i.e. the output should be
-# the same as the input, except for the copyright comment.
-# Change the compiler by setting the F18 environment variable.
-
-srcdir=$(dirname $0)
-source $srcdir/common.sh
-
-FileCheck=internal_check
-
-function internal_check() {
-  r=true
-  linput="$1"
-  lstdin=`mktemp`
-  lcheck=`mktemp`
-  cat - > ${lstdin}
-  egrep '^[[:space:]]*![[:space:]]*CHECK:[[:space:]]*' ${linput} | sed -e 's/^[[:space:]]*![[:space:]]*CHECK:[[:space:]]*//' > ${lcheck} 2>/dev/null
-  while read p; do
-    if egrep "${p}" ${lstdin} >/dev/null 2>&1; then
-      true
-    else
-      echo "Not found: ${p}" >&2
-      r=false
-    fi
-  done < ${lcheck}
-  egrep '^[[:space:]]*![[:space:]]*CHECK-NOT:[[:space:]]*' ${linput} | sed -e 's/^[[:space:]]*![[:space:]]*CHECK-NOT:[[:space:]]*//' > ${lcheck} 2>/dev/null
-  while read p; do
-    if egrep "${p}" ${lstdin} >/dev/null 2>&1; then
-      echo "Found: ${p}" >&2
-      r=false
-    fi
-  done < ${lcheck}
-  egrep '^[[:space:]]*![[:space:]]*CHECK-ONCE:[[:space:]]*' ${linput} | sed -e 's/^[[:space:]]*![[:space:]]*CHECK-ONCE:[[:space:]]*//' > ${lcheck} 2>/dev/null
-  while read p; do
-    count=$(egrep -o -e "${p}" ${lstdin} | wc -l)
-    if [ ${count} -eq 0 ]; then
-      echo "Not found: ${p}" >&2
-      r=false
-    fi
-    if [ ${count} -gt 1 ]; then
-      echo "Found duplicates: ${p}" >&2
-      r=false
-    fi
-  done < ${lcheck}
-  rm -f ${lstdin} ${lcheck}
-  ${r}
-}
-
-gr=0
-for input in $src; do
-  [[ ! -f $input ]] && die "File not found: $input"
-  CMD=$(cat ${input} | egrep '^[[:space:]]*![[:space:]]*EXEC:[[:space:]]*' | sed -e 's/^[[:space:]]*![[:space:]]*EXEC:[[:space:]]*//')
-  CMD=$(echo ${CMD} | sed -e "s:%s:${input}:g")
-  if egrep -q -e '%t' <<< ${CMD} ; then
-    CMD=$(echo ${CMD} | sed -e "s:%t:$temp/t:g")
-  fi
-  if $(eval $CMD); then
-    echo "PASS  ${input}"
-  else
-    echo "FAIL  ${input}"
-    gr=1
-  fi
-done
-exit $gr
diff --git a/flang/tools/f18-parse-demo/CMakeLists.txt b/flang/tools/f18-parse-demo/CMakeLists.txt
index d60d7d6d5cbde..465873ca00ff6 100644
--- a/flang/tools/f18-parse-demo/CMakeLists.txt
+++ b/flang/tools/f18-parse-demo/CMakeLists.txt
@@ -1,3 +1,7 @@
+set(LLVM_LINK_COMPONENTS
+  FrontendOpenMP
+  )
+
 add_flang_tool(f18-parse-demo
   f18-parse-demo.cpp
   stub-evaluate.cpp
diff --git a/flang/tools/f18-parse-demo/stub-evaluate.cpp b/flang/tools/f18-parse-demo/stub-evaluate.cpp
index d673eca5fa504..854e5758743d2 100644
--- a/flang/tools/f18-parse-demo/stub-evaluate.cpp
+++ b/flang/tools/f18-parse-demo/stub-evaluate.cpp
@@ -9,25 +9,19 @@
 // The parse tree has slots in which pointers to the results of semantic
 // analysis may be placed.  When using the parser without the semantics
 // libraries, as here, we need to stub out the dependences on the external
-// destructors, which will never actually be called.
-
-#include "flang/Common/indirection.h"
+// deleters, which will never actually be called.
 
 namespace Fortran::evaluate {
 struct GenericExprWrapper {
-  ~GenericExprWrapper();
+  static void Deleter(GenericExprWrapper *);
 };
-GenericExprWrapper::~GenericExprWrapper() {}
+void GenericExprWrapper::Deleter(GenericExprWrapper *) {}
 struct GenericAssignmentWrapper {
-  ~GenericAssignmentWrapper();
+  static void Deleter(GenericAssignmentWrapper *);
 };
-GenericAssignmentWrapper::~GenericAssignmentWrapper() {}
+void GenericAssignmentWrapper::Deleter(GenericAssignmentWrapper *) {}
 struct ProcedureRef {
-  ~ProcedureRef();
+  static void Deleter(ProcedureRef *);
 };
-ProcedureRef::~ProcedureRef() {}
+void ProcedureRef::Deleter(ProcedureRef *) {}
 } // namespace Fortran::evaluate
-
-DEFINE_DELETER(Fortran::evaluate::GenericExprWrapper)
-DEFINE_DELETER(Fortran::evaluate::GenericAssignmentWrapper)
-DEFINE_DELETER(Fortran::evaluate::ProcedureRef)
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
index 86434b253befd..8738561fe45e7 100644
--- a/flang/tools/f18/CMakeLists.txt
+++ b/flang/tools/f18/CMakeLists.txt
@@ -1,4 +1,5 @@
 set(LLVM_LINK_COMPONENTS
+  FrontendOpenMP
   Support
   )
 add_flang_tool(f18
diff --git a/flang/unittests/Evaluate/CMakeLists.txt b/flang/unittests/Evaluate/CMakeLists.txt
index a4315b1072ce4..54676b0b5a0ad 100644
--- a/flang/unittests/Evaluate/CMakeLists.txt
+++ b/flang/unittests/Evaluate/CMakeLists.txt
@@ -4,6 +4,10 @@ add_library(FortranEvaluateTesting
   fp-testing.cpp
 )
 
+target_link_libraries(FortranEvaluateTesting
+  LLVMSupport
+)
+
 add_executable(leading-zero-bit-count-test
   leading-zero-bit-count.cpp
 )
diff --git a/flang/unittests/Evaluate/fp-testing.cpp b/flang/unittests/Evaluate/fp-testing.cpp
index 66ab7b274a824..94d8d5086d000 100644
--- a/flang/unittests/Evaluate/fp-testing.cpp
+++ b/flang/unittests/Evaluate/fp-testing.cpp
@@ -3,6 +3,9 @@
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#if __x86_64__
+#include <xmmintrin.h>
+#endif
 
 using Fortran::common::RoundingMode;
 using Fortran::evaluate::RealFlag;
@@ -20,31 +23,38 @@ ScopedHostFloatingPointEnvironment::ScopedHostFloatingPointEnvironment(
         llvm::sys::StrError(errno).c_str());
     std::abort();
   }
-  if (fegetenv(&currentFenv_) != 0) {
+  fenv_t currentFenv;
+  if (fegetenv(&currentFenv) != 0) {
     std::fprintf(
         stderr, "fegetenv() failed: %s\n", llvm::sys::StrError(errno).c_str());
     std::abort();
   }
+
 #if __x86_64__
+  originalMxcsr = _mm_getcsr();
+  unsigned int currentMxcsr{originalMxcsr};
   if (treatSubnormalOperandsAsZero) {
-    currentFenv_.__mxcsr |= 0x0040;
+    currentMxcsr |= 0x0040;
   } else {
-    currentFenv_.__mxcsr &= ~0x0040;
+    currentMxcsr &= ~0x0040;
   }
   if (flushSubnormalResultsToZero) {
-    currentFenv_.__mxcsr |= 0x8000;
+    currentMxcsr |= 0x8000;
   } else {
-    currentFenv_.__mxcsr &= ~0x8000;
+    currentMxcsr &= ~0x8000;
   }
 #else
   // TODO others
 #endif
   errno = 0;
-  if (fesetenv(&currentFenv_) != 0) {
+  if (fesetenv(&currentFenv) != 0) {
     std::fprintf(
         stderr, "fesetenv() failed: %s\n", llvm::sys::StrError(errno).c_str());
     std::abort();
   }
+#if __x86_64__
+  _mm_setcsr(currentMxcsr);
+#endif
 }
 
 ScopedHostFloatingPointEnvironment::~ScopedHostFloatingPointEnvironment() {
@@ -54,6 +64,9 @@ ScopedHostFloatingPointEnvironment::~ScopedHostFloatingPointEnvironment() {
         stderr, "fesetenv() failed: %s\n", llvm::sys::StrError(errno).c_str());
     std::abort();
   }
+#if __x86_64__
+  _mm_setcsr(originalMxcsr);
+#endif
 }
 
 void ScopedHostFloatingPointEnvironment::ClearFlags() const {
diff --git a/flang/unittests/Evaluate/fp-testing.h b/flang/unittests/Evaluate/fp-testing.h
index 760e976c4c96d..ee06765879b3d 100644
--- a/flang/unittests/Evaluate/fp-testing.h
+++ b/flang/unittests/Evaluate/fp-testing.h
@@ -19,7 +19,9 @@ class ScopedHostFloatingPointEnvironment {
 
 private:
   fenv_t originalFenv_;
-  fenv_t currentFenv_;
+#if __x86_64__
+  unsigned int originalMxcsr;
+#endif
 };
 
 #endif // FORTRAN_TEST_EVALUATE_FP_TESTING_H_
diff --git a/flang/unittests/Evaluate/intrinsics.cpp b/flang/unittests/Evaluate/intrinsics.cpp
index 7123ef63b63dd..57f71965f8d94 100644
--- a/flang/unittests/Evaluate/intrinsics.cpp
+++ b/flang/unittests/Evaluate/intrinsics.cpp
@@ -257,6 +257,40 @@ void TestIntrinsics() {
   TestCall{defaults, table, "idint"}
       .Push(Const(Scalar<Real4>{}))
       .DoCall(Int4::GetType());
+
+  TestCall{defaults, table, "num_images"}.DoCall(Int4::GetType());
+  TestCall{defaults, table, "num_images"}
+      .Push(Const(Scalar<Int1>{}))
+      .DoCall(Int4::GetType());
+  TestCall{defaults, table, "num_images"}
+      .Push(Const(Scalar<Int4>{}))
+      .DoCall(Int4::GetType());
+  TestCall{defaults, table, "num_images"}
+      .Push(Const(Scalar<Int8>{}))
+      .DoCall(Int4::GetType());
+  TestCall{defaults, table, "num_images"}
+      .Push(Named("team_number", Const(Scalar<Int4>{})))
+      .DoCall(Int4::GetType());
+  TestCall{defaults, table, "num_images"}
+      .Push(Const(Scalar<Int4>{}))
+      .Push(Const(Scalar<Int4>{}))
+      .DoCall(); // too many args
+  TestCall{defaults, table, "num_images"}
+      .Push(Named("bad", Const(Scalar<Int4>{})))
+      .DoCall(); // bad keyword
+  TestCall{defaults, table, "num_images"}
+      .Push(Const(Scalar<Char>{}))
+      .DoCall(); // bad type
+  TestCall{defaults, table, "num_images"}
+      .Push(Const(Scalar<Log4>{}))
+      .DoCall(); // bad type
+  TestCall{defaults, table, "num_images"}
+      .Push(Const(Scalar<Complex8>{}))
+      .DoCall(); // bad type
+  TestCall{defaults, table, "num_images"}
+      .Push(Const(Scalar<Real4>{}))
+      .DoCall(); // bad type
+
   // TODO: test other intrinsics
 }
 } // namespace Fortran::evaluate
diff --git a/flang/unittests/Runtime/CMakeLists.txt b/flang/unittests/Runtime/CMakeLists.txt
index e1464e54e8c55..b13c84e88a028 100644
--- a/flang/unittests/Runtime/CMakeLists.txt
+++ b/flang/unittests/Runtime/CMakeLists.txt
@@ -8,6 +8,11 @@ add_library(RuntimeTesting
 )
 llvm_update_compile_flags(RuntimeTesting)
 
+target_link_libraries(RuntimeTesting
+  FortranRuntime
+  LLVMSupport
+)
+
 add_executable(format-test
   format.cpp
 )
@@ -41,6 +46,18 @@ target_link_libraries(external-hello-world
   LLVMSupport
 )
 
+add_executable(external-io
+  external-io.cpp
+)
+
+target_link_libraries(external-io
+  RuntimeTesting
+  FortranRuntime
+  LLVMSupport
+)
+
+add_test(NAME ExternalIO COMMAND external-io)
+
 add_executable(list-input-test
   list-input.cpp
 )
diff --git a/flang/unittests/Runtime/external-hello.cpp b/flang/unittests/Runtime/external-hello.cpp
index 400d345e1b39f..eba345481db91 100644
--- a/flang/unittests/Runtime/external-hello.cpp
+++ b/flang/unittests/Runtime/external-hello.cpp
@@ -5,9 +5,8 @@
 
 using namespace Fortran::runtime::io;
 
-int main(int argc, const char *argv[], const char *envp[]) {
-  RTNAME(ProgramStart)(argc, argv, envp);
-  auto *io{IONAME(BeginExternalListOutput)()};
+void output1() {
+  auto io{IONAME(BeginExternalListOutput)()};
   const char str[]{"Hello, world!"};
   IONAME(OutputAscii)(io, str, std::strlen(str));
   IONAME(OutputInteger64)(io, 678);
@@ -21,6 +20,31 @@ int main(int argc, const char *argv[], const char *envp[]) {
   IONAME(OutputLogical)(io, false);
   IONAME(OutputLogical)(io, true);
   IONAME(EndIoStatement)(io);
+}
+
+void input1() {
+  auto io{IONAME(BeginExternalListOutput)()};
+  const char prompt[]{"Enter an integer value:"};
+  IONAME(OutputAscii)(io, prompt, std::strlen(prompt));
+  IONAME(EndIoStatement)(io);
+
+  io = IONAME(BeginExternalListInput)();
+  std::int64_t n{-666};
+  IONAME(InputInteger)(io, n);
+  IONAME(EndIoStatement)(io);
+
+  io = IONAME(BeginExternalListOutput)();
+  const char str[]{"Result:"};
+  IONAME(OutputAscii)(io, str, std::strlen(str));
+  IONAME(OutputInteger64)(io, n);
+  IONAME(EndIoStatement)(io);
+}
+
+int main(int argc, const char *argv[], const char *envp[]) {
+  RTNAME(ProgramStart)(argc, argv, envp);
+  output1();
+  input1();
+  RTNAME(PauseStatement)();
   RTNAME(ProgramEndStatement)();
   return 0;
 }
diff --git a/flang/unittests/Runtime/external-io.cpp b/flang/unittests/Runtime/external-io.cpp
new file mode 100644
index 0000000000000..63b910b239568
--- /dev/null
+++ b/flang/unittests/Runtime/external-io.cpp
@@ -0,0 +1,399 @@
+// Sanity test for all external I/O modes
+
+#include "testing.h"
+#include "../../runtime/io-api.h"
+#include "../../runtime/main.h"
+#include "../../runtime/stop.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cstring>
+
+using namespace Fortran::runtime::io;
+
+void TestDirectUnformatted() {
+  llvm::errs() << "begin TestDirectUnformatted()\n";
+  // OPEN(NEWUNIT=unit,ACCESS='DIRECT',ACTION='READWRITE',&
+  //   FORM='UNFORMATTED',RECL=8,STATUS='SCRATCH')
+  auto io{IONAME(BeginOpenNewUnit)(__FILE__, __LINE__)};
+  IONAME(SetAccess)(io, "DIRECT", 6) || (Fail() << "SetAccess(DIRECT)", 0);
+  IONAME(SetAction)
+  (io, "READWRITE", 9) || (Fail() << "SetAction(READWRITE)", 0);
+  IONAME(SetForm)
+  (io, "UNFORMATTED", 11) || (Fail() << "SetForm(UNFORMATTED)", 0);
+  std::int64_t buffer;
+  static constexpr std::size_t recl{sizeof buffer};
+  IONAME(SetRecl)(io, recl) || (Fail() << "SetRecl()", 0);
+  IONAME(SetStatus)(io, "SCRATCH", 7) || (Fail() << "SetStatus(SCRATCH)", 0);
+  int unit{-1};
+  IONAME(GetNewUnit)(io, unit) || (Fail() << "GetNewUnit()", 0);
+  llvm::errs() << "unit=" << unit << '\n';
+  IONAME(EndIoStatement)
+  (io) == IostatOk || (Fail() << "EndIoStatement() for OpenNewUnit", 0);
+  static constexpr int records{10};
+  for (int j{1}; j <= records; ++j) {
+    // WRITE(UNIT=unit,REC=j) j
+    io = IONAME(BeginUnformattedOutput)(unit, __FILE__, __LINE__);
+    IONAME(SetRec)(io, j) || (Fail() << "SetRec(" << j << ')', 0);
+    buffer = j;
+    IONAME(OutputUnformattedBlock)
+    (io, reinterpret_cast<const char *>(&buffer), recl) ||
+        (Fail() << "OutputUnformattedBlock()", 0);
+    IONAME(EndIoStatement)
+    (io) == IostatOk ||
+        (Fail() << "EndIoStatement() for OutputUnformattedBlock", 0);
+  }
+  for (int j{records}; j >= 1; --j) {
+    // READ(UNIT=unit,REC=j) n
+    io = IONAME(BeginUnformattedInput)(unit, __FILE__, __LINE__);
+    IONAME(SetRec)
+    (io, j) || (Fail() << "SetRec(" << j << ')', 0);
+    IONAME(InputUnformattedBlock)
+    (io, reinterpret_cast<char *>(&buffer), recl) ||
+        (Fail() << "InputUnformattedBlock()", 0);
+    IONAME(EndIoStatement)
+    (io) == IostatOk ||
+        (Fail() << "EndIoStatement() for InputUnformattedBlock", 0);
+    if (buffer != j) {
+      Fail() << "Read back " << buffer << " from direct unformatted record "
+             << j << ", expected " << j << '\n';
+    }
+  }
+  // CLOSE(UNIT=unit,STATUS='DELETE')
+  io = IONAME(BeginClose)(unit, __FILE__, __LINE__);
+  IONAME(SetStatus)(io, "DELETE", 6) || (Fail() << "SetStatus(DELETE)", 0);
+  IONAME(EndIoStatement)
+  (io) == IostatOk || (Fail() << "EndIoStatement() for Close", 0);
+  llvm::errs() << "end TestDirectUnformatted()\n";
+}
+
+void TestSequentialFixedUnformatted() {
+  llvm::errs() << "begin TestSequentialFixedUnformatted()\n";
+  // OPEN(NEWUNIT=unit,ACCESS='SEQUENTIAL',ACTION='READWRITE',&
+  //   FORM='UNFORMATTED',RECL=8,STATUS='SCRATCH')
+  auto io{IONAME(BeginOpenNewUnit)(__FILE__, __LINE__)};
+  IONAME(SetAccess)
+      (io, "SEQUENTIAL", 10) || (Fail() << "SetAccess(SEQUENTIAL)", 0);
+  IONAME(SetAction)
+  (io, "READWRITE", 9) || (Fail() << "SetAction(READWRITE)", 0);
+  IONAME(SetForm)
+  (io, "UNFORMATTED", 11) || (Fail() << "SetForm(UNFORMATTED)", 0);
+  std::int64_t buffer;
+  static constexpr std::size_t recl{sizeof buffer};
+  IONAME(SetRecl)(io, recl) || (Fail() << "SetRecl()", 0);
+  IONAME(SetStatus)(io, "SCRATCH", 7) || (Fail() << "SetStatus(SCRATCH)", 0);
+  int unit{-1};
+  IONAME(GetNewUnit)(io, unit) || (Fail() << "GetNewUnit()", 0);
+  llvm::errs() << "unit=" << unit << '\n';
+  IONAME(EndIoStatement)
+  (io) == IostatOk || (Fail() << "EndIoStatement() for OpenNewUnit", 0);
+  static const int records{10};
+  for (int j{1}; j <= records; ++j) {
+    // DO J=1,RECORDS; WRITE(UNIT=unit) j; END DO
+    io = IONAME(BeginUnformattedOutput)(unit, __FILE__, __LINE__);
+    buffer = j;
+    IONAME(OutputUnformattedBlock)
+    (io, reinterpret_cast<const char *>(&buffer), recl) ||
+        (Fail() << "OutputUnformattedBlock()", 0);
+    IONAME(EndIoStatement)
+    (io) == IostatOk ||
+        (Fail() << "EndIoStatement() for OutputUnformattedBlock", 0);
+  }
+  // REWIND(UNIT=unit)
+  io = IONAME(BeginRewind)(unit, __FILE__, __LINE__);
+  IONAME(EndIoStatement)
+  (io) == IostatOk || (Fail() << "EndIoStatement() for Rewind", 0);
+  for (int j{1}; j <= records; ++j) {
+    // DO J=1,RECORDS; READ(UNIT=unit) n; check n; END DO
+    io = IONAME(BeginUnformattedInput)(unit, __FILE__, __LINE__);
+    IONAME(InputUnformattedBlock)
+    (io, reinterpret_cast<char *>(&buffer), recl) ||
+        (Fail() << "InputUnformattedBlock()", 0);
+    IONAME(EndIoStatement)
+    (io) == IostatOk ||
+        (Fail() << "EndIoStatement() for InputUnformattedBlock", 0);
+    if (buffer != j) {
+      Fail() << "Read back " << buffer
+             << " from sequential fixed unformatted record " << j
+             << ", expected " << j << '\n';
+    }
+  }
+  for (int j{records}; j >= 1; --j) {
+    // BACKSPACE(UNIT=unit)
+    io = IONAME(BeginBackspace)(unit, __FILE__, __LINE__);
+    IONAME(EndIoStatement)
+    (io) == IostatOk ||
+        (Fail() << "EndIoStatement() for Backspace (before read)", 0);
+    // READ(UNIT=unit) n
+    io = IONAME(BeginUnformattedInput)(unit, __FILE__, __LINE__);
+    IONAME(InputUnformattedBlock)
+    (io, reinterpret_cast<char *>(&buffer), recl) ||
+        (Fail() << "InputUnformattedBlock()", 0);
+    IONAME(EndIoStatement)
+    (io) == IostatOk ||
+        (Fail() << "EndIoStatement() for InputUnformattedBlock", 0);
+    if (buffer != j) {
+      Fail() << "Read back " << buffer
+             << " from sequential fixed unformatted record " << j
+             << " after backspacing, expected " << j << '\n';
+    }
+    // BACKSPACE(UNIT=unit)
+    io = IONAME(BeginBackspace)(unit, __FILE__, __LINE__);
+    IONAME(EndIoStatement)
+    (io) == IostatOk ||
+        (Fail() << "EndIoStatement() for Backspace (after read)", 0);
+  }
+  // CLOSE(UNIT=unit,STATUS='DELETE')
+  io = IONAME(BeginClose)(unit, __FILE__, __LINE__);
+  IONAME(SetStatus)(io, "DELETE", 6) || (Fail() << "SetStatus(DELETE)", 0);
+  IONAME(EndIoStatement)
+  (io) == IostatOk || (Fail() << "EndIoStatement() for Close", 0);
+  llvm::errs() << "end TestSequentialFixedUnformatted()\n";
+}
+
+void TestSequentialVariableUnformatted() {
+  llvm::errs() << "begin TestSequentialVariableUnformatted()\n";
+  // OPEN(NEWUNIT=unit,ACCESS='SEQUENTIAL',ACTION='READWRITE',&
+  //   FORM='UNFORMATTED',STATUS='SCRATCH')
+  auto io{IONAME(BeginOpenNewUnit)(__FILE__, __LINE__)};
+  IONAME(SetAccess)
+  (io, "SEQUENTIAL", 10) || (Fail() << "SetAccess(SEQUENTIAL)", 0);
+  IONAME(SetAction)
+  (io, "READWRITE", 9) || (Fail() << "SetAction(READWRITE)", 0);
+  IONAME(SetForm)
+  (io, "UNFORMATTED", 11) || (Fail() << "SetForm(UNFORMATTED)", 0);
+  IONAME(SetStatus)(io, "SCRATCH", 7) || (Fail() << "SetStatus(SCRATCH)", 0);
+  int unit{-1};
+  IONAME(GetNewUnit)(io, unit) || (Fail() << "GetNewUnit()", 0);
+  llvm::errs() << "unit=" << unit << '\n';
+  IONAME(EndIoStatement)
+  (io) == IostatOk || (Fail() << "EndIoStatement() for OpenNewUnit", 0);
+  static const int records{10};
+  std::int64_t buffer[records]; // INTEGER*8 :: BUFFER(0:9) = [(j,j=0,9)]
+  for (int j{0}; j < records; ++j) {
+    buffer[j] = j;
+  }
+  for (int j{1}; j <= records; ++j) {
+    // DO J=1,RECORDS; WRITE(UNIT=unit) BUFFER(0:j); END DO
+    io = IONAME(BeginUnformattedOutput)(unit, __FILE__, __LINE__);
+    IONAME(OutputUnformattedBlock)
+    (io, reinterpret_cast<const char *>(&buffer), j * sizeof *buffer) ||
+        (Fail() << "OutputUnformattedBlock()", 0);
+    IONAME(EndIoStatement)
+    (io) == IostatOk ||
+        (Fail() << "EndIoStatement() for OutputUnformattedBlock", 0);
+  }
+  // REWIND(UNIT=unit)
+  io = IONAME(BeginRewind)(unit, __FILE__, __LINE__);
+  IONAME(EndIoStatement)
+  (io) == IostatOk || (Fail() << "EndIoStatement() for Rewind", 0);
+  for (int j{1}; j <= records; ++j) {
+    // DO J=1,RECORDS; READ(UNIT=unit) n; check n; END DO
+    io = IONAME(BeginUnformattedInput)(unit, __FILE__, __LINE__);
+    IONAME(InputUnformattedBlock)
+    (io, reinterpret_cast<char *>(&buffer), j * sizeof *buffer) ||
+        (Fail() << "InputUnformattedBlock()", 0);
+    IONAME(EndIoStatement)
+    (io) == IostatOk ||
+        (Fail() << "EndIoStatement() for InputUnformattedBlock", 0);
+    for (int k{0}; k < j; ++k) {
+      if (buffer[k] != k) {
+        Fail() << "Read back [" << k << "]=" << buffer[k]
+               << " from direct unformatted record " << j << ", expected " << k
+               << '\n';
+      }
+    }
+  }
+  for (int j{records}; j >= 1; --j) {
+    // BACKSPACE(unit)
+    io = IONAME(BeginBackspace)(unit, __FILE__, __LINE__);
+    IONAME(EndIoStatement)
+    (io) == IostatOk ||
+        (Fail() << "EndIoStatement() for Backspace (before read)", 0);
+    // READ(unit=unit) n; check
+    io = IONAME(BeginUnformattedInput)(unit, __FILE__, __LINE__);
+    IONAME(InputUnformattedBlock)
+    (io, reinterpret_cast<char *>(&buffer), j * sizeof *buffer) ||
+        (Fail() << "InputUnformattedBlock()", 0);
+    IONAME(EndIoStatement)
+    (io) == IostatOk ||
+        (Fail() << "EndIoStatement() for InputUnformattedBlock", 0);
+    for (int k{0}; k < j; ++k) {
+      if (buffer[k] != k) {
+        Fail() << "Read back [" << k << "]=" << buffer[k]
+               << " from sequential variable unformatted record " << j
+               << ", expected " << k << '\n';
+      }
+    }
+    // BACKSPACE(unit)
+    io = IONAME(BeginBackspace)(unit, __FILE__, __LINE__);
+    IONAME(EndIoStatement)
+    (io) == IostatOk ||
+        (Fail() << "EndIoStatement() for Backspace (after read)", 0);
+  }
+  // CLOSE(UNIT=unit,STATUS='DELETE')
+  io = IONAME(BeginClose)(unit, __FILE__, __LINE__);
+  IONAME(SetStatus)(io, "DELETE", 6) || (Fail() << "SetStatus(DELETE)", 0);
+  IONAME(EndIoStatement)
+  (io) == IostatOk || (Fail() << "EndIoStatement() for Close", 0);
+  llvm::errs() << "end TestSequentialVariableUnformatted()\n";
+}
+
+void TestDirectFormatted() {
+  llvm::errs() << "begin TestDirectFormatted()\n";
+  // OPEN(NEWUNIT=unit,ACCESS='DIRECT',ACTION='READWRITE',&
+  //   FORM='FORMATTED',RECL=8,STATUS='SCRATCH')
+  auto io{IONAME(BeginOpenNewUnit)(__FILE__, __LINE__)};
+  IONAME(SetAccess)(io, "DIRECT", 6) || (Fail() << "SetAccess(DIRECT)", 0);
+  IONAME(SetAction)
+  (io, "READWRITE", 9) || (Fail() << "SetAction(READWRITE)", 0);
+  IONAME(SetForm)
+  (io, "FORMATTED", 9) || (Fail() << "SetForm(FORMATTED)", 0);
+  static constexpr std::size_t recl{8};
+  IONAME(SetRecl)(io, recl) || (Fail() << "SetRecl()", 0);
+  IONAME(SetStatus)(io, "SCRATCH", 7) || (Fail() << "SetStatus(SCRATCH)", 0);
+  int unit{-1};
+  IONAME(GetNewUnit)(io, unit) || (Fail() << "GetNewUnit()", 0);
+  llvm::errs() << "unit=" << unit << '\n';
+  IONAME(EndIoStatement)
+  (io) == IostatOk || (Fail() << "EndIoStatement() for OpenNewUnit", 0);
+  static constexpr int records{10};
+  static const char fmt[]{"(I4)"};
+  for (int j{1}; j <= records; ++j) {
+    // WRITE(UNIT=unit,FMT=fmt,REC=j) j
+    io = IONAME(BeginExternalFormattedOutput)(
+        fmt, sizeof fmt - 1, unit, __FILE__, __LINE__);
+    IONAME(SetRec)(io, j) || (Fail() << "SetRec(" << j << ')', 0);
+    IONAME(OutputInteger64)(io, j) || (Fail() << "OutputInteger64()", 0);
+    IONAME(EndIoStatement)
+    (io) == IostatOk || (Fail() << "EndIoStatement() for OutputInteger64", 0);
+  }
+  for (int j{records}; j >= 1; --j) {
+    // READ(UNIT=unit,FMT=fmt,REC=j) n
+    io = IONAME(BeginExternalFormattedInput)(
+        fmt, sizeof fmt - 1, unit, __FILE__, __LINE__);
+    IONAME(SetRec)(io, j) || (Fail() << "SetRec(" << j << ')', 0);
+    std::int64_t buffer;
+    IONAME(InputInteger)(io, buffer) || (Fail() << "InputInteger()", 0);
+    IONAME(EndIoStatement)
+    (io) == IostatOk || (Fail() << "EndIoStatement() for InputInteger", 0);
+    if (buffer != j) {
+      Fail() << "Read back " << buffer << " from direct formatted record " << j
+             << ", expected " << j << '\n';
+    }
+  }
+  // CLOSE(UNIT=unit,STATUS='DELETE')
+  io = IONAME(BeginClose)(unit, __FILE__, __LINE__);
+  IONAME(SetStatus)(io, "DELETE", 6) || (Fail() << "SetStatus(DELETE)", 0);
+  IONAME(EndIoStatement)
+  (io) == IostatOk || (Fail() << "EndIoStatement() for Close", 0);
+  llvm::errs() << "end TestDirectformatted()\n";
+}
+
+void TestSequentialVariableFormatted() {
+  llvm::errs() << "begin TestSequentialVariableFormatted()\n";
+  // OPEN(NEWUNIT=unit,ACCESS='SEQUENTIAL',ACTION='READWRITE',&
+  //   FORM='FORMATTED',STATUS='SCRATCH')
+  auto io{IONAME(BeginOpenNewUnit)(__FILE__, __LINE__)};
+  IONAME(SetAccess)
+  (io, "SEQUENTIAL", 10) || (Fail() << "SetAccess(SEQUENTIAL)", 0);
+  IONAME(SetAction)
+  (io, "READWRITE", 9) || (Fail() << "SetAction(READWRITE)", 0);
+  IONAME(SetForm)
+  (io, "FORMATTED", 9) || (Fail() << "SetForm(FORMATTED)", 0);
+  IONAME(SetStatus)(io, "SCRATCH", 7) || (Fail() << "SetStatus(SCRATCH)", 0);
+  int unit{-1};
+  IONAME(GetNewUnit)(io, unit) || (Fail() << "GetNewUnit()", 0);
+  llvm::errs() << "unit=" << unit << '\n';
+  IONAME(EndIoStatement)
+  (io) == IostatOk || (Fail() << "EndIoStatement() for OpenNewUnit", 0);
+  static const int records{10};
+  std::int64_t buffer[records]; // INTEGER*8 :: BUFFER(0:9) = [(j,j=0,9)]
+  for (int j{0}; j < records; ++j) {
+    buffer[j] = j;
+  }
+  char fmt[32];
+  for (int j{1}; j <= records; ++j) {
+    std::snprintf(fmt, sizeof fmt, "(%dI4)", j);
+    // DO J=1,RECORDS; WRITE(UNIT=unit,FMT=fmt) BUFFER(0:j); END DO
+    io = IONAME(BeginExternalFormattedOutput)(
+        fmt, std::strlen(fmt), unit, __FILE__, __LINE__);
+    for (int k{0}; k < j; ++k) {
+      IONAME(OutputInteger64)
+      (io, buffer[k]) || (Fail() << "OutputInteger64()", 0);
+    }
+    IONAME(EndIoStatement)
+    (io) == IostatOk || (Fail() << "EndIoStatement() for OutputInteger64", 0);
+  }
+  // REWIND(UNIT=unit)
+  io = IONAME(BeginRewind)(unit, __FILE__, __LINE__);
+  IONAME(EndIoStatement)
+  (io) == IostatOk || (Fail() << "EndIoStatement() for Rewind", 0);
+  for (int j{1}; j <= records; ++j) {
+    std::snprintf(fmt, sizeof fmt, "(%dI4)", j);
+    // DO J=1,RECORDS; READ(UNIT=unit,FMT=fmt) n; check n; END DO
+    io = IONAME(BeginExternalFormattedInput)(
+        fmt, std::strlen(fmt), unit, __FILE__, __LINE__);
+    std::int64_t check[records];
+    for (int k{0}; k < j; ++k) {
+      IONAME(InputInteger)(io, check[k]) || (Fail() << "InputInteger()", 0);
+    }
+    IONAME(EndIoStatement)
+    (io) == IostatOk || (Fail() << "EndIoStatement() for InputInteger", 0);
+    for (int k{0}; k < j; ++k) {
+      if (buffer[k] != check[k]) {
+        Fail() << "Read back [" << k << "]=" << check[k]
+               << " from sequential variable formatted record " << j
+               << ", expected " << buffer[k] << '\n';
+      }
+    }
+  }
+  for (int j{records}; j >= 1; --j) {
+    // BACKSPACE(unit)
+    io = IONAME(BeginBackspace)(unit, __FILE__, __LINE__);
+    IONAME(EndIoStatement)
+    (io) == IostatOk ||
+        (Fail() << "EndIoStatement() for Backspace (before read)", 0);
+    std::snprintf(fmt, sizeof fmt, "(%dI4)", j);
+    // READ(UNIT=unit,FMT=fmt) n; check
+    io = IONAME(BeginExternalFormattedInput)(
+        fmt, std::strlen(fmt), unit, __FILE__, __LINE__);
+    std::int64_t check[records];
+    for (int k{0}; k < j; ++k) {
+      IONAME(InputInteger)(io, check[k]) || (Fail() << "InputInteger()", 0);
+    }
+    IONAME(EndIoStatement)
+    (io) == IostatOk || (Fail() << "EndIoStatement() for InputInteger", 0);
+    for (int k{0}; k < j; ++k) {
+      if (buffer[k] != check[k]) {
+        Fail() << "Read back [" << k << "]=" << buffer[k]
+               << " from sequential variable formatted record " << j
+               << ", expected " << buffer[k] << '\n';
+      }
+    }
+    // BACKSPACE(unit)
+    io = IONAME(BeginBackspace)(unit, __FILE__, __LINE__);
+    IONAME(EndIoStatement)
+    (io) == IostatOk ||
+        (Fail() << "EndIoStatement() for Backspace (after read)", 0);
+  }
+  // CLOSE(UNIT=unit,STATUS='DELETE')
+  io = IONAME(BeginClose)(unit, __FILE__, __LINE__);
+  IONAME(SetStatus)(io, "DELETE", 6) || (Fail() << "SetStatus(DELETE)", 0);
+  IONAME(EndIoStatement)
+  (io) == IostatOk || (Fail() << "EndIoStatement() for Close", 0);
+  llvm::errs() << "end TestSequentialVariableFormatted()\n";
+}
+
+void TestStreamUnformatted() {
+  // TODO
+}
+
+int main() {
+  StartTests();
+  TestDirectUnformatted();
+  TestSequentialFixedUnformatted();
+  TestSequentialVariableUnformatted();
+  TestDirectFormatted();
+  TestSequentialVariableFormatted();
+  TestStreamUnformatted();
+  return EndTests();
+}
diff --git a/flang/unittests/Runtime/testing.cpp b/flang/unittests/Runtime/testing.cpp
index 146b37db9a577..546a1e1da51de 100644
--- a/flang/unittests/Runtime/testing.cpp
+++ b/flang/unittests/Runtime/testing.cpp
@@ -9,10 +9,13 @@
 static int failures{0};
 
 // Override the Fortran runtime's Crash() for testing purposes
-[[noreturn]] static void CatchCrash(const char *message, va_list &ap) {
+[[noreturn]] static void CatchCrash(
+    const char *sourceFile, int sourceLine, const char *message, va_list &ap) {
   char buffer[1000];
   std::vsnprintf(buffer, sizeof buffer, message, ap);
   va_end(ap);
+  llvm::errs() << (sourceFile ? sourceFile : "unknown source file") << '('
+               << sourceLine << "): CRASH: " << buffer << '\n';
   throw std::string{buffer};
 }
 
diff --git a/flang/unittests/Runtime/testing.h b/flang/unittests/Runtime/testing.h
index 1b401aaf85433..d3551ba5ef07c 100644
--- a/flang/unittests/Runtime/testing.h
+++ b/flang/unittests/Runtime/testing.h
@@ -12,6 +12,7 @@ void StartTests();
 llvm::raw_ostream &Fail();
 int EndTests();
 
+// Defines a CHARACTER object with padding when needed
 void SetCharacter(char *, std::size_t, const char *);
 
 #endif // FORTRAN_TEST_RUNTIME_TESTING_H_
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 2b5b12926cb62..5d607b82892ff 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -9,14 +9,18 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strcpy
     libc.src.string.strcat
     libc.src.string.strlen
+    libc.src.string.memchr
+    libc.src.string.strchr
 )
 
 set(TARGET_LIBM_ENTRYPOINTS
     # math.h entrypoints
     libc.src.math.copysign
     libc.src.math.copysignf
+    libc.src.math.copysignl
     libc.src.math.ceil
     libc.src.math.ceilf
+    libc.src.math.ceill
     libc.src.math.cosf
     libc.src.math.expf
     libc.src.math.exp2f
@@ -25,14 +29,19 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fabsl
     libc.src.math.floor
     libc.src.math.floorf
+    libc.src.math.floorl
     libc.src.math.frexp
     libc.src.math.frexpf
+    libc.src.math.frexpl
     libc.src.math.logb
     libc.src.math.logbf
+    libc.src.math.logbl
     libc.src.math.modf
     libc.src.math.modff
+    libc.src.math.modfl
     libc.src.math.round
     libc.src.math.roundf
+    libc.src.math.roundl
     libc.src.math.sincosf
     libc.src.math.sinf
     libc.src.math.trunc
diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td
index 79fdb15412caf..9b9ed1b49298d 100644
--- a/libc/config/linux/api.td
+++ b/libc/config/linux/api.td
@@ -152,24 +152,31 @@ def MathAPI : PublicAPI<"math.h"> {
   let Functions = [
    "copysign",
    "copysignf",
+   "copysignl",
    "ceil",
    "ceilf",
+   "ceill",
    "cosf",
    "fabs",
    "fabsf",
    "fabsl",
    "floor",
    "floorf",
+   "floorl",
    "frexp",
    "frexpf",
+   "frexpl",
    "logb",
    "logbf",
+   "logbl",
    "modf",
    "modff",
+   "modfl",
    "expf",
    "exp2f",
    "round",
    "roundf",
+   "roundl",
    "sincosf",
    "sinf",
    "trunc",
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index a7836636c8195..fc62fbb880b41 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -27,6 +27,8 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strcat
     libc.src.string.strlen
     libc.src.string.strcmp
+    libc.src.string.memchr
+    libc.src.string.strchr
 
     # sys/mman.h entrypoints
     libc.src.sys.mman.mmap
@@ -48,8 +50,10 @@ set(TARGET_LIBM_ENTRYPOINTS
     # math.h entrypoints
     libc.src.math.copysign
     libc.src.math.copysignf
+    libc.src.math.copysignl
     libc.src.math.ceil
     libc.src.math.ceilf
+    libc.src.math.ceill
     libc.src.math.cosf
     libc.src.math.expf
     libc.src.math.exp2f
@@ -58,14 +62,19 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fabsl
     libc.src.math.floor
     libc.src.math.floorf
+    libc.src.math.floorl
     libc.src.math.frexp
     libc.src.math.frexpf
+    libc.src.math.frexpl
     libc.src.math.logb
     libc.src.math.logbf
+    libc.src.math.logbl
     libc.src.math.modf
     libc.src.math.modff
+    libc.src.math.modfl
     libc.src.math.round
     libc.src.math.roundf
+    libc.src.math.roundl
     libc.src.math.sincosf
     libc.src.math.sinf
     libc.src.math.trunc
diff --git a/libc/docs/clang_tidy_checks.rst b/libc/docs/clang_tidy_checks.rst
new file mode 100644
index 0000000000000..ffdc45bdd977c
--- /dev/null
+++ b/libc/docs/clang_tidy_checks.rst
@@ -0,0 +1,86 @@
+LLVM libc clang-tidy checks
+===========================
+These are the clang-tidy checks designed to help enforce implementation
+standards.
+The configuration file is ``src/.clang-tidy``.
+
+restrict-system-libc-header
+---------------------------
+One of libc-project’s design goals is to use kernel headers and compiler
+provided headers to prevent code duplication on a per platform basis. This
+presents a problem when writing implementations since system libc headers are
+easy to include accidentally and we can't just use the ``-nostdinc`` flag.
+Improperly included system headers can introduce runtime errors because the C
+standard outlines function prototypes and behaviors but doesn’t define
+underlying implementation details such as the layout of a struct.
+
+This check prevents accidental inclusion of system libc headers when writing a
+libc implementation.
+
+.. code-block:: c++
+
+   #include <stdio.h>            // Not allowed because it is part of system libc.
+   #include <stddef.h>           // Allowed because it is provided by the compiler.
+   #include "internal/stdio.h"   // Allowed because it is NOT part of system libc.
+
+
+implementation-in-namespace
+---------------------------
+
+It is part of our implementation standards that all implementation pieces live
+under the ``__llvm_libc`` namespace. This prevents polution of the global
+namespace. Without a formal check to ensure this, an implementation might
+compile and pass unit tests, but not produce a usable libc function.
+
+This check that ensures any function call resolves to a function within the
+``__llvm_libc`` namespace.
+
+.. code-block:: c++
+
+    // Correct: implementation inside the correct namespace.
+    namespace __llvm_libc {
+        void LLVM_LIBC_ENTRYPOINT(strcpy)(char *dest, const char *src) {}
+        // Namespaces within __llvm_libc namespace are allowed.
+        namespace inner{
+            int localVar = 0;
+        }
+        // Functions with C linkage are allowed.
+        extern "C" void str_fuzz(){}
+    }
+
+    // Incorrect: implementation not in a namespace.
+    void LLVM_LIBC_ENTRYPOINT(strcpy)(char *dest, const char *src) {}
+
+    // Incorrect: outer most namespace is not correct.
+    namespace something_else {
+        void LLVM_LIBC_ENTRYPOINT(strcpy)(char *dest, const char *src) {}
+    }
+
+
+callee-namespace
+----------------
+LLVM-libc is distinct because it is designed to maintain interoperability with
+other libc libraries, including the one that lives on the system. This feature
+creates some uncertainty about which library a call resolves to especially when
+a public header with non-namespaced functions like ``string.h`` is included.
+
+This check ensures any function call resolves to a function within the
+__llvm_libc namespace.
+
+.. code-block:: c++
+
+    namespace __llvm_libc {
+
+    // Allow calls with the fully qualified name.
+    __llvm_libc::strlen("hello");
+
+    // Allow calls to compiler provided functions.
+    (void)__builtin_abs(-1);
+
+    // Bare calls are allowed as long as they resolve to the correct namespace.
+    strlen("world");
+
+    // Disallow calling into functions in the global namespace.
+    ::strlen("!");
+
+    } // namespace __llvm_libc
diff --git a/libc/docs/integration_test.rst b/libc/docs/integration_test.rst
new file mode 100644
index 0000000000000..dd3962bdeae6e
--- /dev/null
+++ b/libc/docs/integration_test.rst
@@ -0,0 +1,19 @@
+Integration Tests
+=====================
+The implementation of libc-project is unique because our public C header files
+are generated using information from TableGen. Unit tests only exercise the
+internal C++ implementations and don't ensure the headers were generated by the
+build system. End to end testing ensures that, after building, the produced
+library and headers are usable in the C runtime. A simple solution is to have
+contributors write an integration test for each individual function as a C
+program; however, this would place a large burden on contributors and duplicates
+some effort from the unit tests.
+
+Instead we automate the generation of integration tests by modeling it from our
+generated headers. These integration tests ensure that public facing symbols are
+visible, that header files are generated as expected, and that each libc
+function has the correct function prototype.
+
+The integration test cmake rules are located in ``test/src/CMakeLists.txt`` and
+the generated integration test lives in
+``llvm/build/projects/libc/test/src/public_integration_test.cpp``
diff --git a/libc/fuzzing/string/CMakeLists.txt b/libc/fuzzing/string/CMakeLists.txt
index 326bf9c071753..db3def324fccd 100644
--- a/libc/fuzzing/string/CMakeLists.txt
+++ b/libc/fuzzing/string/CMakeLists.txt
@@ -7,3 +7,11 @@ add_libc_fuzzer(
     libc.src.string.strcpy
     libc.src.string.strlen
 )
+
+add_libc_fuzzer(
+  strcmp_fuzz
+  SRCS
+    strcmp_fuzz.cpp
+  DEPENDS
+    libc.src.string.strcmp
+)
diff --git a/libc/fuzzing/string/strcmp_fuzz.cpp b/libc/fuzzing/string/strcmp_fuzz.cpp
new file mode 100644
index 0000000000000..c1b497dfb2065
--- /dev/null
+++ b/libc/fuzzing/string/strcmp_fuzz.cpp
@@ -0,0 +1,78 @@
+//===-- strcmp_fuzz.cpp ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// Fuzzing test for llvm-libc strcmp implementation.
+///
+//===----------------------------------------------------------------------===//
+#include "src/string/strcmp.h"
+#include <stddef.h>
+#include <stdint.h>
+
+// The general structure is to take the value of the first byte, set size1 to
+// that value, and add the null terminator. size2 will then contain the rest of
+// the bytes in data.
+// For example, with inputs (data={2, 6, 4, 8, 0}, size=5):
+//         size1: data[0] = 2
+//         data1: {2, 6} + '\0' = {2, 6, '\0'}
+//         size2: size - size1 = 3
+//         data2: {4, 8, '\0'}
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  // Verify the size is at least 1 and the data is null terminated.
+  if (!size || data[size - 1] != '\0')
+    return 0;
+
+  const size_t size1 = (data[0] <= size ? data[0] : size);
+  const size_t size2 = size - size1;
+
+  // The first size will always be at least 1 since
+  // we need to append the null terminator. The second size
+  // needs to be checked since it must also contain the null
+  // terminator.
+  if (!size2)
+    return 0;
+
+  // Copy the data into new containers.
+  // Add one to data1 for null terminator.
+  uint8_t *data1 = new uint8_t[size1 + 1];
+  uint8_t *data2 = new uint8_t[size2];
+  if (!data1 || !data2)
+    __builtin_trap();
+
+  size_t i;
+  for (i = 0; i < size1; ++i)
+    data1[i] = data[i];
+  data1[size1] = '\0'; // Add null terminator to data1.
+
+  for (size_t j = 0; j < size2; ++j)
+    data2[j] = data[i++];
+
+  const char *s1 = reinterpret_cast<const char *>(data1);
+  const char *s2 = reinterpret_cast<const char *>(data2);
+  size_t k = 0;
+  // Iterate until a null terminator is hit or the character comparison is
+  // different.
+  while (s1[k] && s2[k] && s1[k] == s2[k])
+    ++k;
+
+  const unsigned char ch1 = static_cast<unsigned char>(s1[k]);
+  const unsigned char ch2 = static_cast<unsigned char>(s2[k]);
+  // The expected result should be the difference between the first non-equal
+  // characters of s1 and s2. If all characters are equal, the expected result
+  // should be '\0' - '\0' = 0.
+  if (__llvm_libc::strcmp(s1, s2) != ch1 - ch2)
+    __builtin_trap();
+
+  // Verify reversed operands. This should be the negated value of the previous
+  // result, except of course if the previous result was zero.
+  if (__llvm_libc::strcmp(s2, s1) != ch2 - ch1)
+    __builtin_trap();
+
+  delete[] data1;
+  delete[] data2;
+  return 0;
+}
diff --git a/libc/spec/spec.td b/libc/spec/spec.td
index 3dd339b435c48..bfbeb6a6fe3d1 100644
--- a/libc/spec/spec.td
+++ b/libc/spec/spec.td
@@ -49,6 +49,7 @@ def VoidPtr : PtrType<VoidType>;
 def ConstVoidPtr : ConstType<VoidPtr>;
 def SizeTType : NamedType<"size_t">;
 def FloatPtr : PtrType<FloatType>;
+def LongDoublePtr : PtrType<LongDoubleType>;
 
 // _Noreturn is really not a type, but it is convenient to treat it as a type.
 def NoReturn : NamedType<"_Noreturn void">;
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index 998a63c6a02f8..2af79373c30ce 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -191,9 +191,11 @@ def StdC : StandardSpec<"stdc"> {
       [
           FunctionSpec<"copysign", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"copysignf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
+          FunctionSpec<"copysignl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
 
           FunctionSpec<"ceil", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
           FunctionSpec<"ceilf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
+          FunctionSpec<"ceill", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
 
           FunctionSpec<"fabs", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
           FunctionSpec<"fabsf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
@@ -201,15 +203,19 @@ def StdC : StandardSpec<"stdc"> {
 
           FunctionSpec<"floor", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
           FunctionSpec<"floorf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
+          FunctionSpec<"floorl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
 
           FunctionSpec<"frexp", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntPtr>]>,
           FunctionSpec<"frexpf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntPtr>]>,
+          FunctionSpec<"frexpl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntPtr>]>,
 
           FunctionSpec<"logb", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
           FunctionSpec<"logbf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
+          FunctionSpec<"logbl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
 
           FunctionSpec<"modf", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoublePtr>]>,
           FunctionSpec<"modff", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatPtr>]>,
+          FunctionSpec<"modfl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoublePtr>]>,
 
           FunctionSpec<"cosf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
           FunctionSpec<"sinf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
@@ -219,6 +225,7 @@ def StdC : StandardSpec<"stdc"> {
 
           FunctionSpec<"round", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
           FunctionSpec<"roundf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
+          FunctionSpec<"roundl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
 
           FunctionSpec<"trunc", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
           FunctionSpec<"truncf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index 0d673e086097f..aa8680383b7ab 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -152,6 +152,18 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  ceill
+  SRCS
+    ceill.cpp
+  HDRS
+    ceill.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
 add_entrypoint_object(
   floor
   SRCS
@@ -176,6 +188,18 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  floorl
+  SRCS
+    floorl.cpp
+  HDRS
+    floorl.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
 add_entrypoint_object(
   round
   SRCS
@@ -184,6 +208,8 @@ add_entrypoint_object(
     round.h
   DEPENDS
     libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
 )
 
 add_entrypoint_object(
@@ -194,6 +220,20 @@ add_entrypoint_object(
     roundf.h
   DEPENDS
     libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  roundl
+  SRCS
+    roundl.cpp
+  HDRS
+    roundl.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
 )
 
 add_object_library(
@@ -238,6 +278,8 @@ add_entrypoint_object(
     copysign.h
   DEPENDS
     libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
 )
 
 add_entrypoint_object(
@@ -248,6 +290,20 @@ add_entrypoint_object(
     copysignf.h
   DEPENDS
     libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  copysignl
+  SRCS
+    copysignl.cpp
+  HDRS
+    copysignl.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
 )
 
 add_entrypoint_object(
@@ -258,6 +314,8 @@ add_entrypoint_object(
     frexp.h
   DEPENDS
     libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
 )
 
 add_entrypoint_object(
@@ -268,6 +326,20 @@ add_entrypoint_object(
     frexpf.h
   DEPENDS
     libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  frexpl
+  SRCS
+    frexpl.cpp
+  HDRS
+    frexpl.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
 )
 
 add_entrypoint_object(
@@ -278,6 +350,8 @@ add_entrypoint_object(
     logb.h
   DEPENDS
     libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
 )
 
 add_entrypoint_object(
@@ -288,6 +362,20 @@ add_entrypoint_object(
     logbf.h
   DEPENDS
     libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  logbl
+  SRCS
+    logbl.cpp
+  HDRS
+    logbl.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
 )
 
 add_entrypoint_object(
@@ -298,6 +386,8 @@ add_entrypoint_object(
     modf.h
   DEPENDS
     libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
 )
 
 add_entrypoint_object(
@@ -308,4 +398,18 @@ add_entrypoint_object(
     modff.h
   DEPENDS
     libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  modfl
+  SRCS
+    modfl.cpp
+  HDRS
+    modfl.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
 )
diff --git a/libcxxabi/src/cxa_unexpected.cpp b/libc/src/math/ceill.cpp
similarity index 52%
rename from libcxxabi/src/cxa_unexpected.cpp
rename to libc/src/math/ceill.cpp
index bbfa6684efa74..8ac806d730c47 100644
--- a/libcxxabi/src/cxa_unexpected.cpp
+++ b/libc/src/math/ceill.cpp
@@ -1,4 +1,4 @@
-//===------------------------- cxa_unexpected.cpp -------------------------===//
+//===-- Implementation of ceill function ----------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,17 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <exception>
-#include "cxxabi.h"
-#include "cxa_exception.h"
+#include "src/__support/common.h"
+#include "utils/FPUtil/NearestIntegerOperations.h"
 
-namespace __cxxabiv1
-{
-
-extern "C"
-{
+namespace __llvm_libc {
 
+long double LLVM_LIBC_ENTRYPOINT(ceill)(long double x) {
+  return fputil::ceil(x);
 }
 
-}  // namespace __cxxabiv1
-
+} // namespace __llvm_libc
diff --git a/libcxx/test/libcxx/selftest/test.pass.mm b/libc/src/math/ceill.h
similarity index 51%
rename from libcxx/test/libcxx/selftest/test.pass.mm
rename to libc/src/math/ceill.h
index 2475c31e18952..8bf4c565c7d3b 100644
--- a/libcxx/test/libcxx/selftest/test.pass.mm
+++ b/libc/src/math/ceill.h
@@ -1,5 +1,4 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
+//===-- Implementation header for ceill -------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -7,10 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: objective-c++
+#ifndef LLVM_LIBC_SRC_MATH_CEILL_H
+#define LLVM_LIBC_SRC_MATH_CEILL_H
+
+namespace __llvm_libc {
+
+long double ceill(long double x);
 
-#if __has_feature(objc_arc)
-#error "arc should *not* be enabled"
-#endif
+} // namespace __llvm_libc
 
-int main(int, char**) { return 0; }
+#endif // LLVM_LIBC_SRC_MATH_CEILL_H
diff --git a/libc/src/math/copysignl.cpp b/libc/src/math/copysignl.cpp
new file mode 100644
index 0000000000000..b165b8f290913
--- /dev/null
+++ b/libc/src/math/copysignl.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of copysignl function ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/ManipulationFunctions.h"
+
+namespace __llvm_libc {
+
+long double LLVM_LIBC_ENTRYPOINT(copysignl)(long double x, long double y) {
+  return fputil::copysign(x, y);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/copysignl.h b/libc/src/math/copysignl.h
new file mode 100644
index 0000000000000..4f48323c70141
--- /dev/null
+++ b/libc/src/math/copysignl.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for copysignl ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_COPYSIGNL_H
+#define LLVM_LIBC_SRC_MATH_COPYSIGNL_H
+
+namespace __llvm_libc {
+
+long double copysignl(long double x, long double y);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_COPYSIGNL_H
diff --git a/libcxx/test/libcxx/selftest/test.sh.cpp b/libc/src/math/floorl.cpp
similarity index 52%
rename from libcxx/test/libcxx/selftest/test.sh.cpp
rename to libc/src/math/floorl.cpp
index 5c4db16d84d13..1e7d7b161f20d 100644
--- a/libcxx/test/libcxx/selftest/test.sh.cpp
+++ b/libc/src/math/floorl.cpp
@@ -1,5 +1,4 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
+//===-- Implementation of floorl function ---------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -7,11 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-// RUN: %{build}
-// RUN: %{run}
+#include "src/__support/common.h"
+#include "utils/FPUtil/NearestIntegerOperations.h"
 
-int main(int, char**)
-{
+namespace __llvm_libc {
 
-  return 0;
+long double LLVM_LIBC_ENTRYPOINT(floorl)(long double x) {
+  return fputil::floor(x);
 }
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/floorl.h b/libc/src/math/floorl.h
new file mode 100644
index 0000000000000..224cc13ec6fa6
--- /dev/null
+++ b/libc/src/math/floorl.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for floorl ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FLOORL_H
+#define LLVM_LIBC_SRC_MATH_FLOORL_H
+
+namespace __llvm_libc {
+
+long double floorl(long double x);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_FLOORL_H
diff --git a/libc/src/math/frexpl.cpp b/libc/src/math/frexpl.cpp
new file mode 100644
index 0000000000000..939ac71e31d7c
--- /dev/null
+++ b/libc/src/math/frexpl.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of frexpl function ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/ManipulationFunctions.h"
+
+namespace __llvm_libc {
+
+long double LLVM_LIBC_ENTRYPOINT(frexpl)(long double x, int *exp) {
+  return fputil::frexp(x, *exp);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/frexpl.h b/libc/src/math/frexpl.h
new file mode 100644
index 0000000000000..9f75c82b9034a
--- /dev/null
+++ b/libc/src/math/frexpl.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for frexpl ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FREXPL_H
+#define LLVM_LIBC_SRC_MATH_FREXPL_H
+
+namespace __llvm_libc {
+
+long double frexpl(long double x, int *exp);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_FREXPL_H
diff --git a/libc/src/math/logbl.cpp b/libc/src/math/logbl.cpp
new file mode 100644
index 0000000000000..57255e773f1b9
--- /dev/null
+++ b/libc/src/math/logbl.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of logbl function ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/ManipulationFunctions.h"
+
+namespace __llvm_libc {
+
+long double LLVM_LIBC_ENTRYPOINT(logbl)(long double x) {
+  return fputil::logb(x);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/logbl.h b/libc/src/math/logbl.h
new file mode 100644
index 0000000000000..3630e52c92d29
--- /dev/null
+++ b/libc/src/math/logbl.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for logbl -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LOGBL_H
+#define LLVM_LIBC_SRC_MATH_LOGBL_H
+
+namespace __llvm_libc {
+
+long double logbl(long double x);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_LOGBL_H
diff --git a/libc/src/math/math_utils.h b/libc/src/math/math_utils.h
index 50851bd36f79e..69028b02bb4e2 100644
--- a/libc/src/math/math_utils.h
+++ b/libc/src/math/math_utils.h
@@ -80,7 +80,7 @@ using EnableIfFloatOrDouble = cpp::EnableIfType<IsFloatOrDouble<T>::Value, int>;
 template <typename T, EnableIfFloatOrDouble<T> = 0>
 T xflow(uint32_t sign, T y) {
   // Underflow happens when two extremely small values are multiplied.
-  // Likewise, overflow happens when two large values are mulitplied.
+  // Likewise, overflow happens when two large values are multiplied.
   y = opt_barrier(sign ? -y : y) * y;
   return with_errno(y, ERANGE);
 }
diff --git a/libc/src/math/modfl.cpp b/libc/src/math/modfl.cpp
new file mode 100644
index 0000000000000..83a0746c5c3d2
--- /dev/null
+++ b/libc/src/math/modfl.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of modfl function ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/ManipulationFunctions.h"
+
+namespace __llvm_libc {
+
+long double LLVM_LIBC_ENTRYPOINT(modfl)(long double x, long double *iptr) {
+  return fputil::modf(x, *iptr);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/modfl.h b/libc/src/math/modfl.h
new file mode 100644
index 0000000000000..ff89239c4bfde
--- /dev/null
+++ b/libc/src/math/modfl.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for modfl -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_MODFL_H
+#define LLVM_LIBC_SRC_MATH_MODFL_H
+
+namespace __llvm_libc {
+
+long double modfl(long double x, long double *iptr);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_MODFL_H
diff --git a/libcxx/test/libcxx/selftest/test.pass.cpp b/libc/src/math/roundl.cpp
similarity index 51%
rename from libcxx/test/libcxx/selftest/test.pass.cpp
rename to libc/src/math/roundl.cpp
index 779762e7e995d..afb1a895cceca 100644
--- a/libcxx/test/libcxx/selftest/test.pass.cpp
+++ b/libc/src/math/roundl.cpp
@@ -1,5 +1,4 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
+//===-- Implementation of roundl function ---------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -7,8 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-int main(int, char**)
-{
+#include "src/__support/common.h"
+#include "utils/FPUtil/NearestIntegerOperations.h"
+
+namespace __llvm_libc {
 
-  return 0;
+long double LLVM_LIBC_ENTRYPOINT(roundl)(long double x) {
+  return fputil::round(x);
 }
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/roundl.h b/libc/src/math/roundl.h
new file mode 100644
index 0000000000000..43fb63e231de4
--- /dev/null
+++ b/libc/src/math/roundl.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for roundl ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ROUNDL_H
+#define LLVM_LIBC_SRC_MATH_ROUNDL_H
+
+namespace __llvm_libc {
+
+long double roundl(long double x);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_ROUNDL_H
diff --git a/libc/src/string/CMakeLists.txt b/libc/src/string/CMakeLists.txt
index 51456e8a94c89..c8e1e4dfa39f9 100644
--- a/libc/src/string/CMakeLists.txt
+++ b/libc/src/string/CMakeLists.txt
@@ -44,6 +44,22 @@ add_entrypoint_object(
     libc.include.string
 )
 
+add_entrypoint_object(
+  memchr
+  SRCS
+    memchr.cpp
+  HDRS
+    memchr.h
+)
+
+add_entrypoint_object(
+  strchr
+  SRCS
+    strchr.cpp
+  HDRS
+    strchr.h
+)
+
 # Helper to define a function with multiple implementations
 # - Computes flags to satisfy required/rejected features and arch,
 # - Declares an entry point,
diff --git a/libc/src/string/memchr.cpp b/libc/src/string/memchr.cpp
new file mode 100644
index 0000000000000..303f78185f49c
--- /dev/null
+++ b/libc/src/string/memchr.cpp
@@ -0,0 +1,24 @@
+//===-- Implementation of memchr ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/string/memchr.h"
+#include "src/__support/common.h"
+#include <stddef.h>
+
+namespace __llvm_libc {
+
+// TODO: Look at performance benefits of comparing words.
+void *LLVM_LIBC_ENTRYPOINT(memchr)(const void *src, int c, size_t n) {
+  const unsigned char *str = reinterpret_cast<const unsigned char *>(src);
+  const unsigned char ch = c;
+  for (; n && *str != ch; --n, ++str)
+    ;
+  return n ? const_cast<unsigned char *>(str) : nullptr;
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/string/memchr.h b/libc/src/string/memchr.h
new file mode 100644
index 0000000000000..369475305236d
--- /dev/null
+++ b/libc/src/string/memchr.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for memchr ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRING_MEMCHR_H
+#define LLVM_LIBC_SRC_STRING_MEMCHR_H
+
+#include <stddef.h>
+
+namespace __llvm_libc {
+
+void *memchr(const void *src, int c, size_t n);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STRING_MEMCHR_H
diff --git a/libc/src/string/strchr.cpp b/libc/src/string/strchr.cpp
new file mode 100644
index 0000000000000..eac0e512a8fb4
--- /dev/null
+++ b/libc/src/string/strchr.cpp
@@ -0,0 +1,25 @@
+//===-- Implementation of strchr ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/string/strchr.h"
+
+#include "src/__support/common.h"
+
+namespace __llvm_libc {
+
+// TODO: Look at performance benefits of comparing words.
+char *LLVM_LIBC_ENTRYPOINT(strchr)(const char *src, int c) {
+  unsigned char *str =
+      const_cast<unsigned char *>(reinterpret_cast<const unsigned char *>(src));
+  const unsigned char ch = c;
+  for (; *str && *str != ch; ++str)
+    ;
+  return *str == ch ? reinterpret_cast<char *>(str) : nullptr;
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/string/strchr.h b/libc/src/string/strchr.h
new file mode 100644
index 0000000000000..6f106afaf695c
--- /dev/null
+++ b/libc/src/string/strchr.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for strchr ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRING_STRCHR_H
+#define LLVM_LIBC_SRC_STRING_STRCHR_H
+
+namespace __llvm_libc {
+
+char *strchr(const char *src, int c);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STRING_STRCHR_H
diff --git a/libc/test/CMakeLists.txt b/libc/test/CMakeLists.txt
index aa77dcd2727fd..500b294ad3866 100644
--- a/libc/test/CMakeLists.txt
+++ b/libc/test/CMakeLists.txt
@@ -3,3 +3,4 @@ add_custom_target(check-libc)
 add_subdirectory(config)
 add_subdirectory(loader)
 add_subdirectory(src)
+add_subdirectory(utils)
diff --git a/libc/test/src/CMakeLists.txt b/libc/test/src/CMakeLists.txt
index 432b6190e7bba..3e7bad72482be 100644
--- a/libc/test/src/CMakeLists.txt
+++ b/libc/test/src/CMakeLists.txt
@@ -91,4 +91,4 @@ endif()
 target_link_libraries(libc-integration-test
   PRIVATE
   ${library_files}
-)
+)
\ No newline at end of file
diff --git a/libc/test/src/math/CMakeLists.txt b/libc/test/src/math/CMakeLists.txt
index 1617905150e59..6c6cb9036e53c 100644
--- a/libc/test/src/math/CMakeLists.txt
+++ b/libc/test/src/math/CMakeLists.txt
@@ -175,6 +175,19 @@ add_math_unittest(
     libc.utils.FPUtil.fputil
 )
 
+add_math_unittest(
+  ceill_test
+  NEED_MPFR
+  SUITE
+    libc_math_unittests
+  SRCS
+    ceill_test.cpp
+  DEPENDS
+    libc.include.math
+    libc.src.math.ceill
+    libc.utils.FPUtil.fputil
+)
+
 add_math_unittest(
   floor_test
   NEED_MPFR
@@ -201,6 +214,19 @@ add_math_unittest(
     libc.utils.FPUtil.fputil
 )
 
+add_math_unittest(
+  floorl_test
+  NEED_MPFR
+  SUITE
+    libc_math_unittests
+  SRCS
+    floorl_test.cpp
+  DEPENDS
+    libc.include.math
+    libc.src.math.floorl
+    libc.utils.FPUtil.fputil
+)
+
 add_math_unittest(
   round_test
   NEED_MPFR
@@ -227,6 +253,19 @@ add_math_unittest(
     libc.utils.FPUtil.fputil
 )
 
+add_math_unittest(
+  roundl_test
+  NEED_MPFR
+  SUITE
+    libc_math_unittests
+  SRCS
+    roundl_test.cpp
+  DEPENDS
+    libc.include.math
+    libc.src.math.roundl
+    libc.utils.FPUtil.fputil
+)
+
 add_math_unittest(
   expf_test
   NEED_MPFR
@@ -279,6 +318,18 @@ add_math_unittest(
     libc.utils.FPUtil.fputil
 )
 
+add_math_unittest(
+  copysignl_test
+  SUITE
+    libc_math_unittests
+  SRCS
+    copysignl_test.cpp
+  DEPENDS
+    libc.include.math
+    libc.src.math.copysignl
+    libc.utils.FPUtil.fputil
+)
+
 add_math_unittest(
   frexp_test
   SUITE
@@ -303,6 +354,18 @@ add_math_unittest(
     libc.utils.FPUtil.fputil
 )
 
+add_math_unittest(
+  frexpl_test
+  SUITE
+    libc_math_unittests
+  SRCS
+    frexpl_test.cpp
+  DEPENDS
+    libc.include.math
+    libc.src.math.frexpl
+    libc.utils.FPUtil.fputil
+)
+
 add_math_unittest(
   logb_test
   SUITE
@@ -327,6 +390,18 @@ add_math_unittest(
     libc.utils.FPUtil.fputil
 )
 
+add_math_unittest(
+  logbl_test
+  SUITE
+    libc_math_unittests
+  SRCS
+    logbl_test.cpp
+  DEPENDS
+    libc.include.math
+    libc.src.math.logbl
+    libc.utils.FPUtil.fputil
+)
+
 add_math_unittest(
   modf_test
   SUITE
@@ -350,3 +425,15 @@ add_math_unittest(
     libc.src.math.modff
     libc.utils.FPUtil.fputil
 )
+
+add_math_unittest(
+  modfl_test
+  SUITE
+    libc_math_unittests
+  SRCS
+    modfl_test.cpp
+  DEPENDS
+    libc.include.math
+    libc.src.math.modfl
+    libc.utils.FPUtil.fputil
+)
diff --git a/libc/test/src/math/ceilf_test.cpp b/libc/test/src/math/ceilf_test.cpp
index 7bfb2ac15b6a1..3db004e65542b 100644
--- a/libc/test/src/math/ceilf_test.cpp
+++ b/libc/test/src/math/ceilf_test.cpp
@@ -52,7 +52,7 @@ TEST(CeilfTest, SpecialNumbers) {
                                       valueFromBits(BitPatterns::negZero))));
 }
 
-TEST(ceilfTest, RoundedNumbers) {
+TEST(CeilfTest, RoundedNumbers) {
   EXPECT_EQ(valueAsBits(1.0f), valueAsBits(__llvm_libc::ceilf(1.0f)));
   EXPECT_EQ(valueAsBits(-1.0f), valueAsBits(__llvm_libc::ceilf(-1.0f)));
   EXPECT_EQ(valueAsBits(10.0f), valueAsBits(__llvm_libc::ceilf(10.0f)));
@@ -61,7 +61,7 @@ TEST(ceilfTest, RoundedNumbers) {
   EXPECT_EQ(valueAsBits(-12345.0f), valueAsBits(__llvm_libc::ceilf(-12345.0f)));
 }
 
-TEST(ceilfTest, InFloatRange) {
+TEST(CeilfTest, InFloatRange) {
   using BitsType = Properties::BitsType;
   constexpr BitsType count = 1000000;
   constexpr BitsType step = UINT32_MAX / count;
diff --git a/libc/test/src/math/ceill_test.cpp b/libc/test/src/math/ceill_test.cpp
new file mode 100644
index 0000000000000..6be34a23db3cc
--- /dev/null
+++ b/libc/test/src/math/ceill_test.cpp
@@ -0,0 +1,72 @@
+//===-- Unittests for ceill -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "include/math.h"
+#include "src/math/ceill.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+#include "utils/UnitTest/Test.h"
+
+using FPBits = __llvm_libc::fputil::FPBits<long double>;
+
+namespace mpfr = __llvm_libc::testing::mpfr;
+
+// Zero tolerance; As in, exact match with MPFR result.
+static constexpr mpfr::Tolerance tolerance{mpfr::Tolerance::doublePrecision, 0,
+                                           0};
+TEST(CeillTest, SpecialNumbers) {
+  ASSERT_TRUE(FPBits::zero() == __llvm_libc::ceill(FPBits::zero()));
+  ASSERT_TRUE(FPBits::negZero() == __llvm_libc::ceill(FPBits::negZero()));
+
+  ASSERT_TRUE(FPBits::inf() == __llvm_libc::ceill(FPBits::inf()));
+  ASSERT_TRUE(FPBits::negInf() == __llvm_libc::ceill(FPBits::negInf()));
+
+  long double nan = FPBits::buildNaN(1);
+  ASSERT_NE(isnan(nan), 0);
+  ASSERT_NE(isnan(__llvm_libc::ceill(nan)), 0);
+}
+
+TEST(CeillTest, RoundedNumbers) {
+  ASSERT_TRUE(FPBits(1.0l) == __llvm_libc::ceill(1.0l));
+  ASSERT_TRUE(FPBits(-1.0l) == __llvm_libc::ceill(-1.0l));
+  ASSERT_TRUE(FPBits(10.0l) == __llvm_libc::ceill(10.0l));
+  ASSERT_TRUE(FPBits(-10.0l) == __llvm_libc::ceill(-10.0l));
+  ASSERT_TRUE(FPBits(1234.0l) == __llvm_libc::ceill(1234.0l));
+  ASSERT_TRUE(FPBits(-1234.0l) == __llvm_libc::ceill(-1234.0l));
+}
+
+TEST(CeillTest, Fractions) {
+  ASSERT_TRUE(FPBits(2.0l) == __llvm_libc::ceill(1.3l));
+  ASSERT_TRUE(FPBits(-1.0l) == __llvm_libc::ceill(-1.3l));
+  ASSERT_TRUE(FPBits(2.0l) == __llvm_libc::ceill(1.5l));
+  ASSERT_TRUE(FPBits(-1.0l) == __llvm_libc::ceill(-1.5l));
+  ASSERT_TRUE(FPBits(2.0l) == __llvm_libc::ceill(1.75l));
+  ASSERT_TRUE(FPBits(-1.0l) == __llvm_libc::ceill(-1.75l));
+  ASSERT_TRUE(FPBits(11.0l) == __llvm_libc::ceill(10.32l));
+  ASSERT_TRUE(FPBits(-10.0l) == __llvm_libc::ceill(-10.32l));
+  ASSERT_TRUE(FPBits(11.0l) == __llvm_libc::ceill(10.65l));
+  ASSERT_TRUE(FPBits(-10.0l) == __llvm_libc::ceill(-10.65l));
+  ASSERT_TRUE(FPBits(1235.0l) == __llvm_libc::ceill(1234.18l));
+  ASSERT_TRUE(FPBits(-1234.0l) == __llvm_libc::ceill(-1234.18l));
+  ASSERT_TRUE(FPBits(1235.0l) == __llvm_libc::ceill(1234.96l));
+  ASSERT_TRUE(FPBits(-1234.0l) == __llvm_libc::ceill(-1234.96l));
+}
+
+TEST(CeillTest, InLongDoubleRange) {
+  using UIntType = FPBits::UIntType;
+  constexpr UIntType count = 10000000;
+  constexpr UIntType step = UIntType(-1) / count;
+  for (UIntType i = 0, v = 0; i <= count; ++i, v += step) {
+    long double x = FPBits(v);
+    if (isnan(x) || isinf(x))
+      continue;
+
+    ASSERT_MPFR_MATCH(mpfr::Operation::Ceil, x, __llvm_libc::ceill(x),
+                      tolerance);
+  }
+}
diff --git a/libc/test/src/math/copysignl_test.cpp b/libc/test/src/math/copysignl_test.cpp
new file mode 100644
index 0000000000000..7ffb5fb1fb73b
--- /dev/null
+++ b/libc/test/src/math/copysignl_test.cpp
@@ -0,0 +1,47 @@
+//===-- Unittests for copysignl
+//--------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "include/math.h"
+#include "src/math/copysignl.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/UnitTest/Test.h"
+
+using FPBits = __llvm_libc::fputil::FPBits<long double>;
+
+TEST(CopysignlTest, SpecialNumbers) {
+  EXPECT_TRUE(FPBits::negZero() ==
+              __llvm_libc::copysignl(FPBits::zero(), -1.0l));
+  EXPECT_TRUE(FPBits::zero() ==
+              __llvm_libc::copysignl(FPBits::negZero(), 1.0l));
+
+  EXPECT_TRUE(FPBits::negZero() ==
+              __llvm_libc::copysignl(FPBits::zero(), -1.0l));
+  EXPECT_TRUE(FPBits::zero() ==
+              __llvm_libc::copysignl(FPBits::negZero(), 1.0l));
+
+  EXPECT_TRUE(
+      FPBits(__llvm_libc::copysignl(FPBits::buildNaN(1), -1.0l)).isNaN());
+}
+
+TEST(CopysignlTest, InDoubleRange) {
+  using UIntType = FPBits::UIntType;
+  constexpr UIntType count = 10000000;
+  constexpr UIntType step = UIntType(-1) / count;
+  for (UIntType i = 0, v = 0; i <= count; ++i, v += step) {
+    long double x = FPBits(v);
+    if (isnan(x) || isinf(x) || x == 0)
+      continue;
+
+    long double res1 = __llvm_libc::copysignl(x, -x);
+    ASSERT_TRUE(res1 == -x);
+
+    long double res2 = __llvm_libc::copysignl(x, x);
+    ASSERT_TRUE(res2 == x);
+  }
+}
diff --git a/libc/test/src/math/floorf_test.cpp b/libc/test/src/math/floorf_test.cpp
index c25014da12577..1e05dcfccd1d7 100644
--- a/libc/test/src/math/floorf_test.cpp
+++ b/libc/test/src/math/floorf_test.cpp
@@ -52,7 +52,7 @@ TEST(FloorfTest, SpecialNumbers) {
                                       valueFromBits(BitPatterns::negZero))));
 }
 
-TEST(floorfTest, RoundedNumbers) {
+TEST(FloorfTest, RoundedNumbers) {
   EXPECT_EQ(valueAsBits(1.0f), valueAsBits(__llvm_libc::floorf(1.0f)));
   EXPECT_EQ(valueAsBits(-1.0f), valueAsBits(__llvm_libc::floorf(-1.0f)));
   EXPECT_EQ(valueAsBits(10.0f), valueAsBits(__llvm_libc::floorf(10.0f)));
@@ -62,7 +62,7 @@ TEST(floorfTest, RoundedNumbers) {
             valueAsBits(__llvm_libc::floorf(-12345.0f)));
 }
 
-TEST(floorfTest, InFloatRange) {
+TEST(FloorfTest, InFloatRange) {
   using BitsType = Properties::BitsType;
   constexpr BitsType count = 1000000;
   constexpr BitsType step = UINT32_MAX / count;
diff --git a/libc/test/src/math/floorl_test.cpp b/libc/test/src/math/floorl_test.cpp
new file mode 100644
index 0000000000000..e7c9513245afc
--- /dev/null
+++ b/libc/test/src/math/floorl_test.cpp
@@ -0,0 +1,72 @@
+//===-- Unittests for floorl ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "include/math.h"
+#include "src/math/floorl.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+#include "utils/UnitTest/Test.h"
+
+using FPBits = __llvm_libc::fputil::FPBits<long double>;
+
+namespace mpfr = __llvm_libc::testing::mpfr;
+
+// Zero tolerance; As in, exact match with MPFR result.
+static constexpr mpfr::Tolerance tolerance{mpfr::Tolerance::doublePrecision, 0,
+                                           0};
+TEST(FloorlTest, SpecialNumbers) {
+  ASSERT_TRUE(FPBits::zero() == __llvm_libc::floorl(FPBits::zero()));
+  ASSERT_TRUE(FPBits::negZero() == __llvm_libc::floorl(FPBits::negZero()));
+
+  ASSERT_TRUE(FPBits::inf() == __llvm_libc::floorl(FPBits::inf()));
+  ASSERT_TRUE(FPBits::negInf() == __llvm_libc::floorl(FPBits::negInf()));
+
+  long double nan = FPBits::buildNaN(1);
+  ASSERT_NE(isnan(nan), 0);
+  ASSERT_NE(isnan(__llvm_libc::floorl(nan)), 0);
+}
+
+TEST(FloorlTest, RoundedNumbers) {
+  ASSERT_TRUE(FPBits(1.0l) == __llvm_libc::floorl(1.0l));
+  ASSERT_TRUE(FPBits(-1.0l) == __llvm_libc::floorl(-1.0l));
+  ASSERT_TRUE(FPBits(10.0l) == __llvm_libc::floorl(10.0l));
+  ASSERT_TRUE(FPBits(-10.0l) == __llvm_libc::floorl(-10.0l));
+  ASSERT_TRUE(FPBits(1234.0l) == __llvm_libc::floorl(1234.0l));
+  ASSERT_TRUE(FPBits(-1234.0l) == __llvm_libc::floorl(-1234.0l));
+}
+
+TEST(FloorlTest, Fractions) {
+  ASSERT_TRUE(FPBits(1.0l) == __llvm_libc::floorl(1.3l));
+  ASSERT_TRUE(FPBits(-2.0l) == __llvm_libc::floorl(-1.3l));
+  ASSERT_TRUE(FPBits(1.0l) == __llvm_libc::floorl(1.5l));
+  ASSERT_TRUE(FPBits(-2.0l) == __llvm_libc::floorl(-1.5l));
+  ASSERT_TRUE(FPBits(1.0l) == __llvm_libc::floorl(1.75l));
+  ASSERT_TRUE(FPBits(-2.0l) == __llvm_libc::floorl(-1.75l));
+  ASSERT_TRUE(FPBits(10.0l) == __llvm_libc::floorl(10.32l));
+  ASSERT_TRUE(FPBits(-11.0l) == __llvm_libc::floorl(-10.32l));
+  ASSERT_TRUE(FPBits(10.0l) == __llvm_libc::floorl(10.65l));
+  ASSERT_TRUE(FPBits(-11.0l) == __llvm_libc::floorl(-10.65l));
+  ASSERT_TRUE(FPBits(1234.0l) == __llvm_libc::floorl(1234.18l));
+  ASSERT_TRUE(FPBits(-1235.0l) == __llvm_libc::floorl(-1234.18l));
+  ASSERT_TRUE(FPBits(1234.0l) == __llvm_libc::floorl(1234.96l));
+  ASSERT_TRUE(FPBits(-1235.0l) == __llvm_libc::floorl(-1234.96l));
+}
+
+TEST(FloorlTest, InLongDoubleRange) {
+  using UIntType = FPBits::UIntType;
+  constexpr UIntType count = 10000000;
+  constexpr UIntType step = UIntType(-1) / count;
+  for (UIntType i = 0, v = 0; i <= count; ++i, v += step) {
+    long double x = FPBits(v);
+    if (isnan(x) || isinf(x))
+      continue;
+
+    ASSERT_MPFR_MATCH(mpfr::Operation::Floor, x, __llvm_libc::floorl(x),
+                      tolerance);
+  }
+}
diff --git a/libc/test/src/math/frexpf_test.cpp b/libc/test/src/math/frexpf_test.cpp
index e15f384ea28b2..3b82c68078ee8 100644
--- a/libc/test/src/math/frexpf_test.cpp
+++ b/libc/test/src/math/frexpf_test.cpp
@@ -1,5 +1,4 @@
-//===-- Unittests for frexpf
-//-----------------------------------------------===//
+//===-- Unittests for frexpf ----------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/libc/test/src/math/frexpl_test.cpp b/libc/test/src/math/frexpl_test.cpp
new file mode 100644
index 0000000000000..ace445f0a2de4
--- /dev/null
+++ b/libc/test/src/math/frexpl_test.cpp
@@ -0,0 +1,103 @@
+//===-- Unittests for frexpl ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "include/math.h"
+#include "src/math/frexpl.h"
+#include "utils/FPUtil/BasicOperations.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/UnitTest/Test.h"
+
+using FPBits = __llvm_libc::fputil::FPBits<long double>;
+
+TEST(FrexplTest, SpecialNumbers) {
+  int exponent;
+
+  EXPECT_TRUE(FPBits::inf() == __llvm_libc::frexpl(FPBits::inf(), &exponent));
+  EXPECT_TRUE(FPBits::negInf() ==
+              __llvm_libc::frexpl(FPBits::negInf(), &exponent));
+
+  EXPECT_TRUE(FPBits::zero() == __llvm_libc::frexpl(FPBits::zero(), &exponent));
+  EXPECT_EQ(exponent, 0);
+
+  EXPECT_TRUE(FPBits::negZero() ==
+              __llvm_libc::frexpl(FPBits::negZero(), &exponent));
+  EXPECT_EQ(exponent, 0);
+
+  EXPECT_TRUE(
+      FPBits(__llvm_libc::frexpl(FPBits::buildNaN(1), &exponent)).isNaN());
+}
+
+TEST(FrexplTest, PowersOfTwo) {
+  int exponent;
+
+  EXPECT_TRUE(0.5l == __llvm_libc::frexpl(1.0l, &exponent));
+  EXPECT_EQ(exponent, 1);
+  EXPECT_TRUE(-0.5l == __llvm_libc::frexpl(-1.0l, &exponent));
+  EXPECT_EQ(exponent, 1);
+
+  EXPECT_TRUE(0.5l == __llvm_libc::frexpl(2.0l, &exponent));
+  EXPECT_EQ(exponent, 2);
+  EXPECT_TRUE(-0.5l == __llvm_libc::frexpl(-2.0l, &exponent));
+  EXPECT_EQ(exponent, 2);
+
+  EXPECT_TRUE(0.5l == __llvm_libc::frexpl(4.0l, &exponent));
+  EXPECT_EQ(exponent, 3);
+  EXPECT_TRUE(-0.5l == __llvm_libc::frexpl(-4.0l, &exponent));
+  EXPECT_EQ(exponent, 3);
+
+  EXPECT_TRUE(0.5l == __llvm_libc::frexpl(8.0l, &exponent));
+  EXPECT_EQ(exponent, 4);
+  EXPECT_TRUE(-0.5l == __llvm_libc::frexpl(-8.0l, &exponent));
+  EXPECT_EQ(exponent, 4);
+
+  EXPECT_TRUE(0.5l == __llvm_libc::frexpl(16.0l, &exponent));
+  EXPECT_EQ(exponent, 5);
+  EXPECT_TRUE(-0.5l == __llvm_libc::frexpl(-16.0l, &exponent));
+  EXPECT_EQ(exponent, 5);
+
+  EXPECT_TRUE(0.5l == __llvm_libc::frexpl(32.0l, &exponent));
+  EXPECT_EQ(exponent, 6);
+  EXPECT_TRUE(-0.5l == __llvm_libc::frexpl(-32.0l, &exponent));
+  EXPECT_EQ(exponent, 6);
+}
+
+TEST(FrexplTest, SomeIntegers) {
+  int exponent;
+
+  EXPECT_TRUE(0.75l == __llvm_libc::frexpl(24.0l, &exponent));
+  EXPECT_EQ(exponent, 5);
+  EXPECT_TRUE(-0.75l == __llvm_libc::frexpl(-24.0l, &exponent));
+  EXPECT_EQ(exponent, 5);
+
+  EXPECT_TRUE(0.625l == __llvm_libc::frexpl(40.0l, &exponent));
+  EXPECT_EQ(exponent, 6);
+  EXPECT_TRUE(-0.625l == __llvm_libc::frexpl(-40.0l, &exponent));
+  EXPECT_EQ(exponent, 6);
+
+  EXPECT_TRUE(0.78125l == __llvm_libc::frexpl(800.0l, &exponent));
+  EXPECT_EQ(exponent, 10);
+  EXPECT_TRUE(-0.78125l == __llvm_libc::frexpl(-800.0l, &exponent));
+  EXPECT_EQ(exponent, 10);
+}
+
+TEST(FrexplTest, LongDoubleRange) {
+  using UIntType = FPBits::UIntType;
+  constexpr UIntType count = 10000000;
+  constexpr UIntType step = UIntType(-1) / count;
+  for (UIntType i = 0, v = 0; i <= count; ++i, v += step) {
+    long double x = FPBits(v);
+    if (isnan(x) || isinf(x) || x == 0.0l)
+      continue;
+
+    int exponent;
+    long double frac = __llvm_libc::frexpl(x, &exponent);
+
+    ASSERT_TRUE(__llvm_libc::fputil::abs(frac) < 1.0l);
+    ASSERT_TRUE(__llvm_libc::fputil::abs(frac) >= 0.5l);
+  }
+}
diff --git a/libc/test/src/math/logbl_test.cpp b/libc/test/src/math/logbl_test.cpp
new file mode 100644
index 0000000000000..4cdc6ce61d9a2
--- /dev/null
+++ b/libc/test/src/math/logbl_test.cpp
@@ -0,0 +1,77 @@
+//===-- Unittests for logbl -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "include/math.h"
+#include "src/math/logbl.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/FPUtil/ManipulationFunctions.h"
+#include "utils/UnitTest/Test.h"
+
+using FPBits = __llvm_libc::fputil::FPBits<long double>;
+
+TEST(logblTest, SpecialNumbers) {
+  EXPECT_TRUE(FPBits::inf() == __llvm_libc::logbl(FPBits::inf()));
+  EXPECT_TRUE(FPBits::inf() == __llvm_libc::logbl(FPBits::negInf()));
+
+  EXPECT_TRUE(FPBits::negInf() == __llvm_libc::logbl(FPBits::zero()));
+  EXPECT_TRUE(FPBits::negInf() == __llvm_libc::logbl(FPBits::negZero()));
+
+  EXPECT_TRUE(FPBits(__llvm_libc::logbl(FPBits::buildNaN(1))).isNaN());
+}
+
+TEST(logblTest, PowersOfTwo) {
+  EXPECT_TRUE(0.0l == __llvm_libc::logbl(1.0l));
+  EXPECT_TRUE(0.0l == __llvm_libc::logbl(-1.0l));
+
+  EXPECT_TRUE(1.0l == __llvm_libc::logbl(2.0l));
+  EXPECT_TRUE(1.0l == __llvm_libc::logbl(-2.0l));
+
+  EXPECT_TRUE(2.0l == __llvm_libc::logbl(4.0l));
+  EXPECT_TRUE(2.0l == __llvm_libc::logbl(-4.0l));
+
+  EXPECT_TRUE(3.0l == __llvm_libc::logbl(8.0l));
+  EXPECT_TRUE(3.0l == __llvm_libc::logbl(-8.0l));
+
+  EXPECT_TRUE(4.0l == __llvm_libc::logbl(16.0l));
+  EXPECT_TRUE(4.0l == __llvm_libc::logbl(-16.0l));
+
+  EXPECT_TRUE(5.0l == __llvm_libc::logbl(32.0l));
+  EXPECT_TRUE(5.0l == __llvm_libc::logbl(-32.0l));
+}
+
+TEST(LogbTest, SomeIntegers) {
+  EXPECT_TRUE(1.0l == __llvm_libc::logbl(3.0l));
+  EXPECT_TRUE(1.0l == __llvm_libc::logbl(-3.0l));
+
+  EXPECT_TRUE(2.0l == __llvm_libc::logbl(7.0l));
+  EXPECT_TRUE(2.0l == __llvm_libc::logbl(-7.0l));
+
+  EXPECT_TRUE(3.0l == __llvm_libc::logbl(10.0l));
+  EXPECT_TRUE(3.0l == __llvm_libc::logbl(-10.0l));
+
+  EXPECT_TRUE(4.0l == __llvm_libc::logbl(31.0l));
+  EXPECT_TRUE(4.0l == __llvm_libc::logbl(-31.0l));
+
+  EXPECT_TRUE(5.0l == __llvm_libc::logbl(55.0l));
+  EXPECT_TRUE(5.0l == __llvm_libc::logbl(-55.0l));
+}
+
+TEST(LogblTest, LongDoubleRange) {
+  using UIntType = FPBits::UIntType;
+  constexpr UIntType count = 10000000;
+  constexpr UIntType step = UIntType(-1) / count;
+  for (UIntType i = 0, v = 0; i <= count; ++i, v += step) {
+    long double x = FPBits(v);
+    if (isnan(x) || isinf(x) || x == 0.0l)
+      continue;
+
+    int exponent;
+    __llvm_libc::fputil::frexp(x, exponent);
+    ASSERT_TRUE((long double)(exponent) == __llvm_libc::logbl(x) + 1.0l);
+  }
+}
diff --git a/libc/test/src/math/modff_test.cpp b/libc/test/src/math/modff_test.cpp
index fbd3b86f163a9..66244b8ac8927 100644
--- a/libc/test/src/math/modff_test.cpp
+++ b/libc/test/src/math/modff_test.cpp
@@ -1,5 +1,4 @@
-//===-- Unittests for modfff
-//-----------------------------------------------===//
+//===-- Unittests for modff -----------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -89,7 +88,7 @@ TEST(ModffTest, Integers) {
   EXPECT_EQ(valueAsBits(integral), valueAsBits(-12345.0f));
 }
 
-TEST(ModfTest, Fractions) {
+TEST(ModffTest, Fractions) {
   float integral;
 
   EXPECT_EQ(valueAsBits(0.5f),
diff --git a/libc/test/src/math/modfl_test.cpp b/libc/test/src/math/modfl_test.cpp
new file mode 100644
index 0000000000000..2e47880d1ea89
--- /dev/null
+++ b/libc/test/src/math/modfl_test.cpp
@@ -0,0 +1,98 @@
+//===-- Unittests for modfl -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "include/math.h"
+#include "src/math/modfl.h"
+#include "utils/FPUtil/BasicOperations.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/FPUtil/NearestIntegerOperations.h"
+#include "utils/UnitTest/Test.h"
+
+using FPBits = __llvm_libc::fputil::FPBits<long double>;
+
+TEST(modflTest, SpecialNumbers) {
+  long double integral;
+
+  EXPECT_TRUE(FPBits::zero() == __llvm_libc::modfl(FPBits::inf(), &integral));
+  EXPECT_TRUE(FPBits::inf() == integral);
+
+  EXPECT_TRUE(FPBits::negZero() ==
+              __llvm_libc::modfl(FPBits::negInf(), &integral));
+  EXPECT_TRUE(FPBits::negInf() == integral);
+
+  EXPECT_TRUE(FPBits::zero() == __llvm_libc::modfl(FPBits::zero(), &integral));
+  EXPECT_TRUE(integral == 0.0l);
+
+  EXPECT_TRUE(FPBits::negZero() ==
+              __llvm_libc::modfl(FPBits::negZero(), &integral));
+  EXPECT_TRUE(integral == 0.0l);
+
+  EXPECT_TRUE(
+      FPBits(__llvm_libc::modfl(FPBits::buildNaN(1), &integral)).isNaN());
+}
+
+TEST(modflTest, Integers) {
+  long double integral;
+
+  EXPECT_TRUE(FPBits::zero() == __llvm_libc::modfl(1.0l, &integral));
+  EXPECT_TRUE(integral == 1.0l);
+
+  EXPECT_TRUE(FPBits::negZero() == __llvm_libc::modfl(-1.0l, &integral));
+  EXPECT_TRUE(integral == -1.0l);
+
+  EXPECT_TRUE(FPBits::zero() == __llvm_libc::modfl(10.0l, &integral));
+  EXPECT_TRUE(integral == 10.0l);
+
+  EXPECT_TRUE(FPBits::negZero() == __llvm_libc::modfl(-10.0l, &integral));
+  EXPECT_TRUE(integral == -10.0l);
+
+  EXPECT_TRUE(FPBits::zero() == __llvm_libc::modfl(12345.0l, &integral));
+  EXPECT_TRUE(integral == 12345.0l);
+
+  EXPECT_TRUE(FPBits::negZero() == __llvm_libc::modfl(-12345.0l, &integral));
+  EXPECT_TRUE(integral == -12345.0l);
+}
+
+TEST(ModfTest, Fractions) {
+  long double integral;
+
+  EXPECT_TRUE(0.5l == __llvm_libc::modfl(1.5l, &integral));
+  EXPECT_TRUE(integral == 1.0l);
+
+  EXPECT_TRUE(-0.5l == __llvm_libc::modfl(-1.5l, &integral));
+  EXPECT_TRUE(integral == -1.0l);
+
+  EXPECT_TRUE(0.75l == __llvm_libc::modfl(10.75l, &integral));
+  EXPECT_TRUE(integral == 10.0l);
+
+  EXPECT_TRUE(-0.75l == __llvm_libc::modfl(-10.75l, &integral));
+  EXPECT_TRUE(integral == -10.0l);
+
+  EXPECT_TRUE(0.125l == __llvm_libc::modfl(100.125l, &integral));
+  EXPECT_TRUE(integral == 100.0l);
+
+  EXPECT_TRUE(-0.125l == __llvm_libc::modfl(-100.125l, &integral));
+  EXPECT_TRUE(integral == -100.0l);
+}
+
+TEST(ModflTest, LongDoubleRange) {
+  using UIntType = FPBits::UIntType;
+  constexpr UIntType count = 10000000;
+  constexpr UIntType step = UIntType(-1) / count;
+  for (UIntType i = 0, v = 0; i <= count; ++i, v += step) {
+    long double x = FPBits(v);
+    if (isnan(x) || isinf(x) || x == 0.0l)
+      continue;
+
+    long double integral;
+    long double frac = __llvm_libc::modfl(x, &integral);
+    ASSERT_TRUE(__llvm_libc::fputil::abs(frac) < 1.0l);
+    ASSERT_TRUE(__llvm_libc::fputil::trunc(x) == integral);
+    ASSERT_TRUE(integral + frac == x);
+  }
+}
diff --git a/libc/test/src/math/roundf_test.cpp b/libc/test/src/math/roundf_test.cpp
index 88c746b126efb..ae8008a0340f6 100644
--- a/libc/test/src/math/roundf_test.cpp
+++ b/libc/test/src/math/roundf_test.cpp
@@ -62,7 +62,7 @@ TEST(RoundfTest, RoundedNumbers) {
             valueAsBits(__llvm_libc::roundf(-12345.0f)));
 }
 
-TEST(RoundTest, CloseToZeroNumbers) {
+TEST(RoundfTest, CloseToZeroNumbers) {
   EXPECT_EQ(valueAsBits(1.0f), valueAsBits(__llvm_libc::roundf(0.5f)));
   EXPECT_EQ(valueAsBits(-1.0f), valueAsBits(__llvm_libc::roundf(-0.5f)));
   EXPECT_EQ(valueAsBits(0.0f), valueAsBits(__llvm_libc::roundf(0.115f)));
diff --git a/libc/test/src/math/roundl_test.cpp b/libc/test/src/math/roundl_test.cpp
new file mode 100644
index 0000000000000..38545a842d151
--- /dev/null
+++ b/libc/test/src/math/roundl_test.cpp
@@ -0,0 +1,72 @@
+//===-- Unittests for roundl ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "include/math.h"
+#include "src/math/roundl.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+#include "utils/UnitTest/Test.h"
+
+using FPBits = __llvm_libc::fputil::FPBits<long double>;
+
+namespace mpfr = __llvm_libc::testing::mpfr;
+
+// Zero tolerance; As in, exact match with MPFR result.
+static constexpr mpfr::Tolerance tolerance{mpfr::Tolerance::floatPrecision, 0,
+                                           0};
+TEST(RoundlTest, SpecialNumbers) {
+  ASSERT_TRUE(FPBits::zero() == __llvm_libc::roundl(FPBits::zero()));
+  ASSERT_TRUE(FPBits::negZero() == __llvm_libc::roundl(FPBits::negZero()));
+
+  ASSERT_TRUE(FPBits::inf() == __llvm_libc::roundl(FPBits::inf()));
+  ASSERT_TRUE(FPBits::negInf() == __llvm_libc::roundl(FPBits::negInf()));
+
+  long double nan = FPBits::buildNaN(1);
+  ASSERT_NE(isnan(nan), 0);
+  ASSERT_NE(isnan(__llvm_libc::roundl(nan)), 0);
+}
+
+TEST(RoundlTest, RoundedNumbers) {
+  ASSERT_TRUE(FPBits(1.0l) == __llvm_libc::roundl(1.0l));
+  ASSERT_TRUE(FPBits(-1.0l) == __llvm_libc::roundl(-1.0l));
+  ASSERT_TRUE(FPBits(10.0l) == __llvm_libc::roundl(10.0l));
+  ASSERT_TRUE(FPBits(-10.0l) == __llvm_libc::roundl(-10.0l));
+  ASSERT_TRUE(FPBits(1234.0l) == __llvm_libc::roundl(1234.0l));
+  ASSERT_TRUE(FPBits(-1234.0l) == __llvm_libc::roundl(-1234.0l));
+}
+
+TEST(RoundlTest, Fractions) {
+  ASSERT_TRUE(FPBits(1.0l) == __llvm_libc::roundl(1.3l));
+  ASSERT_TRUE(FPBits(-1.0l) == __llvm_libc::roundl(-1.3l));
+  ASSERT_TRUE(FPBits(2.0l) == __llvm_libc::roundl(1.5l));
+  ASSERT_TRUE(FPBits(-2.0l) == __llvm_libc::roundl(-1.5l));
+  ASSERT_TRUE(FPBits(2.0l) == __llvm_libc::roundl(1.75l));
+  ASSERT_TRUE(FPBits(-2.0l) == __llvm_libc::roundl(-1.75l));
+  ASSERT_TRUE(FPBits(10.0l) == __llvm_libc::roundl(10.32l));
+  ASSERT_TRUE(FPBits(-10.0l) == __llvm_libc::roundl(-10.32l));
+  ASSERT_TRUE(FPBits(11.0l) == __llvm_libc::roundl(10.65l));
+  ASSERT_TRUE(FPBits(-11.0l) == __llvm_libc::roundl(-10.65l));
+  ASSERT_TRUE(FPBits(1234.0l) == __llvm_libc::roundl(1234.38l));
+  ASSERT_TRUE(FPBits(-1234.0l) == __llvm_libc::roundl(-1234.38l));
+  ASSERT_TRUE(FPBits(1235.0l) == __llvm_libc::roundl(1234.96l));
+  ASSERT_TRUE(FPBits(-1235.0l) == __llvm_libc::roundl(-1234.96l));
+}
+
+TEST(RoundlTest, InLongDoubleRange) {
+  using UIntType = FPBits::UIntType;
+  constexpr UIntType count = 10000000;
+  constexpr UIntType step = UIntType(-1) / count;
+  for (UIntType i = 0, v = 0; i <= count; ++i, v += step) {
+    long double x = FPBits(v);
+    if (isnan(x) || isinf(x))
+      continue;
+
+    ASSERT_MPFR_MATCH(mpfr::Operation::Round, x, __llvm_libc::roundl(x),
+                      tolerance);
+  }
+}
diff --git a/libc/test/src/math/truncf_test.cpp b/libc/test/src/math/truncf_test.cpp
index 90fc84a5b5f97..2ace51632af71 100644
--- a/libc/test/src/math/truncf_test.cpp
+++ b/libc/test/src/math/truncf_test.cpp
@@ -53,7 +53,7 @@ TEST(TruncfTest, SpecialNumbers) {
                                       valueFromBits(BitPatterns::negZero))));
 }
 
-TEST(TruncTest, RoundedNumbers) {
+TEST(TruncfTest, RoundedNumbers) {
   EXPECT_EQ(valueAsBits(1.0f), valueAsBits(__llvm_libc::truncf(1.0f)));
   EXPECT_EQ(valueAsBits(-1.0f), valueAsBits(__llvm_libc::truncf(-1.0f)));
   EXPECT_EQ(valueAsBits(10.0f), valueAsBits(__llvm_libc::truncf(10.0f)));
@@ -63,7 +63,7 @@ TEST(TruncTest, RoundedNumbers) {
             valueAsBits(__llvm_libc::truncf(-12345.0f)));
 }
 
-TEST(truncfTest, InFloatRange) {
+TEST(TruncfTest, InFloatRange) {
   using BitsType = Properties::BitsType;
   constexpr BitsType count = 1000000;
   constexpr BitsType step = UINT32_MAX / count;
diff --git a/libc/test/src/string/CMakeLists.txt b/libc/test/src/string/CMakeLists.txt
index 992cebe5067c2..be4836818212d 100644
--- a/libc/test/src/string/CMakeLists.txt
+++ b/libc/test/src/string/CMakeLists.txt
@@ -42,6 +42,26 @@ add_libc_unittest(
     libc.src.string.strcmp
 )
 
+add_libc_unittest(
+  memchr_test
+  SUITE
+    libc_string_unittests
+  SRCS
+    memchr_test.cpp
+  DEPENDS
+    libc.src.string.memchr
+)
+
+add_libc_unittest(
+  strchr_test
+  SUITE
+    libc_string_unittests
+  SRCS
+    strchr_test.cpp
+  DEPENDS
+    libc.src.string.strchr
+)
+
 # Tests all implementations that can run on the host.
 function(add_libc_multi_impl_test name)
   get_property(fq_implementations GLOBAL PROPERTY ${name}_implementations)
@@ -66,4 +86,3 @@ endfunction()
 add_libc_multi_impl_test(memcpy SRCS memcpy_test.cpp)
 add_libc_multi_impl_test(memset SRCS memset_test.cpp)
 add_libc_multi_impl_test(bzero SRCS bzero_test.cpp)
-
diff --git a/libc/test/src/string/memchr_test.cpp b/libc/test/src/string/memchr_test.cpp
new file mode 100644
index 0000000000000..005e02c6b3d4d
--- /dev/null
+++ b/libc/test/src/string/memchr_test.cpp
@@ -0,0 +1,122 @@
+//===-- Unittests for memchr ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/string/memchr.h"
+#include "utils/UnitTest/Test.h"
+#include <stddef.h>
+
+// A helper function that calls memchr and abstracts away the explicit cast for
+// readability purposes.
+const char *call_memchr(const void *src, int c, size_t size) {
+  return reinterpret_cast<const char *>(__llvm_libc::memchr(src, c, size));
+}
+
+TEST(MemChrTest, FindsCharacterAfterNullTerminator) {
+  // memchr should continue searching after a null terminator.
+  const size_t size = 5;
+  const unsigned char src[size] = {'a', '\0', 'b', 'c', '\0'};
+  // Should return 'b', 'c', '\0' even when after null terminator.
+  ASSERT_STREQ(call_memchr(src, 'b', size), "bc");
+}
+
+TEST(MemChrTest, FindsCharacterInNonNullTerminatedCollection) {
+  const size_t size = 3;
+  const unsigned char src[size] = {'a', 'b', 'c'};
+  // Should return 'b', 'c'.
+  const char *ret = call_memchr(src, 'b', size);
+  ASSERT_EQ(ret[0], 'b');
+  ASSERT_EQ(ret[1], 'c');
+}
+
+TEST(MemChrTest, FindsFirstCharacter) {
+  const size_t size = 6;
+  const unsigned char src[size] = {'a', 'b', 'c', 'd', 'e', '\0'};
+  // Should return original array since 'a' is the first character.
+  ASSERT_STREQ(call_memchr(src, 'a', size), "abcde");
+}
+
+TEST(MemChrTest, FindsMiddleCharacter) {
+  const size_t size = 6;
+  const unsigned char src[size] = {'a', 'b', 'c', 'd', 'e', '\0'};
+  // Should return characters after (and including) 'c'.
+  ASSERT_STREQ(call_memchr(src, 'c', size), "cde");
+}
+
+TEST(MemChrTest, FindsLastCharacterThatIsNotNullTerminator) {
+  const size_t size = 6;
+  const unsigned char src[size] = {'a', 'b', 'c', 'd', 'e', '\0'};
+  // Should return 'e' and null-terminator.
+  ASSERT_STREQ(call_memchr(src, 'e', size), "e");
+}
+
+TEST(MemChrTest, FindsNullTerminator) {
+  const size_t size = 6;
+  const unsigned char src[size] = {'a', 'b', 'c', 'd', 'e', '\0'};
+  // Should return null terminator.
+  ASSERT_STREQ(call_memchr(src, '\0', size), "");
+}
+
+TEST(MemChrTest, CharacterNotWithinStringShouldReturnNullptr) {
+  const size_t size = 4;
+  const unsigned char src[size] = {'1', '2', '3', '?'};
+  // Since 'z' is not within 'characters', should return nullptr.
+  ASSERT_STREQ(call_memchr(src, 'z', size), nullptr);
+}
+
+TEST(MemChrTest, CharacterNotWithinSizeShouldReturnNullptr) {
+  const unsigned char src[5] = {'1', '2', '3', '4', '\0'};
+  // Since '4' is not the first or second character, this should return nullptr.
+  const size_t size = 2;
+  ASSERT_STREQ(call_memchr(src, '4', size), nullptr);
+}
+
+TEST(MemChrTest, TheSourceShouldNotChange) {
+  const size_t size = 6;
+  const unsigned char src[size] = {'a', 'b', 'c', 'd', 'e', '\0'};
+  const char *src_copy = reinterpret_cast<const char *>(src);
+  // When the character is found, the source string should not change.
+  __llvm_libc::memchr(src, 'd', size);
+  ASSERT_STREQ(reinterpret_cast<const char *>(src), src_copy);
+  // Same case for when the character is not found.
+  __llvm_libc::memchr(src, 'z', size);
+  ASSERT_STREQ(reinterpret_cast<const char *>(src), src_copy);
+}
+
+TEST(MemChrTest, ShouldFindFirstOfDuplicates) {
+  const size_t size = 12; // 11 characters + null terminator.
+  const char *dups = "abc1def1ghi";
+  // 1 is duplicated in 'dups', but it should find the first copy.
+  ASSERT_STREQ(call_memchr(dups, '1', size), "1def1ghi");
+}
+
+TEST(MemChrTest, EmptyStringShouldOnlyMatchNullTerminator) {
+  const size_t size = 1; // Null terminator.
+  const char *empty_string = "";
+  // Null terminator should match.
+  ASSERT_STREQ(call_memchr(empty_string, '\0', size), "");
+  // All other characters should not match.
+  ASSERT_STREQ(call_memchr(empty_string, 'A', size), nullptr);
+  ASSERT_STREQ(call_memchr(empty_string, '9', size), nullptr);
+  ASSERT_STREQ(call_memchr(empty_string, '?', size), nullptr);
+}
+
+TEST(MemChrTest, SingleRepeatedCharacterShouldReturnFirst) {
+  const char *dups = "XXXXX";
+  const size_t size = 6; // 5 characters + null terminator.
+  // Should return original string since X is first character.
+  ASSERT_STREQ(call_memchr(dups, 'X', size), dups);
+}
+
+TEST(MemChrTest, SignedCharacterFound) {
+  char c = -1;
+  const size_t size = 1;
+  char src[size] = {c};
+  const char *actual = call_memchr(src, c, size);
+  // Should find the first character 'c'.
+  ASSERT_EQ(actual[0], c);
+}
diff --git a/libc/test/src/string/strchr_test.cpp b/libc/test/src/string/strchr_test.cpp
new file mode 100644
index 0000000000000..37b3733857997
--- /dev/null
+++ b/libc/test/src/string/strchr_test.cpp
@@ -0,0 +1,87 @@
+//===-- Unittests for strchr ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/string/strchr.h"
+#include "utils/UnitTest/Test.h"
+
+TEST(StrChrTest, FindsFirstCharacter) {
+  const char *src = "abcde";
+  const char *src_copy = src;
+
+  // Should return original string since 'a' is the first character.
+  ASSERT_STREQ(__llvm_libc::strchr(src, 'a'), "abcde");
+  // Source string should not change.
+  ASSERT_STREQ(src, src_copy);
+}
+
+TEST(StrChrTest, FindsMiddleCharacter) {
+  const char *src = "abcde";
+  const char *src_copy = src;
+
+  // Should return characters after (and including) 'c'.
+  ASSERT_STREQ(__llvm_libc::strchr(src, 'c'), "cde");
+  // Source string should not change.
+  ASSERT_STREQ(src, src_copy);
+}
+
+TEST(StrChrTest, FindsLastCharacterThatIsNotNullTerminator) {
+  const char *src = "abcde";
+  const char *src_copy = src;
+
+  // Should return 'e' and null-terminator.
+  ASSERT_STREQ(__llvm_libc::strchr(src, 'e'), "e");
+  // Source string should not change.
+  ASSERT_STREQ(src, src_copy);
+}
+
+TEST(StrChrTest, FindsNullTerminator) {
+  const char *src = "abcde";
+  const char *src_copy = src;
+
+  // Should return null terminator.
+  ASSERT_STREQ(__llvm_libc::strchr(src, '\0'), "");
+  // Source string should not change.
+  ASSERT_STREQ(src, src_copy);
+}
+
+TEST(StrChrTest, CharacterNotWithinStringShouldReturnNullptr) {
+  // Since 'z' is not within the string, should return nullptr.
+  ASSERT_STREQ(__llvm_libc::strchr("123?", 'z'), nullptr);
+}
+
+TEST(StrChrTest, TheSourceShouldNotChange) {
+  const char *src = "abcde";
+  const char *src_copy = src;
+  // When the character is found, the source string should not change.
+  __llvm_libc::strchr(src, 'd');
+  ASSERT_STREQ(src, src_copy);
+  // Same case for when the character is not found.
+  __llvm_libc::strchr(src, 'z');
+  ASSERT_STREQ(src, src_copy);
+  // Same case for when looking for nullptr.
+  __llvm_libc::strchr(src, '\0');
+  ASSERT_STREQ(src, src_copy);
+}
+
+TEST(StrChrTest, ShouldFindFirstOfDuplicates) {
+  // '1' is duplicated in the string, but it should find the first copy.
+  ASSERT_STREQ(__llvm_libc::strchr("abc1def1ghi", '1'), "1def1ghi");
+
+  const char *dups = "XXXXX";
+  // Should return original string since 'X' is the first character.
+  ASSERT_STREQ(__llvm_libc::strchr(dups, 'X'), dups);
+}
+
+TEST(StrChrTest, EmptyStringShouldOnlyMatchNullTerminator) {
+  // Null terminator should match.
+  ASSERT_STREQ(__llvm_libc::strchr("", '\0'), "");
+  // All other characters should not match.
+  ASSERT_STREQ(__llvm_libc::strchr("", 'Z'), nullptr);
+  ASSERT_STREQ(__llvm_libc::strchr("", '3'), nullptr);
+  ASSERT_STREQ(__llvm_libc::strchr("", '*'), nullptr);
+}
diff --git a/libc/test/utils/CMakeLists.txt b/libc/test/utils/CMakeLists.txt
new file mode 100644
index 0000000000000..90ff4bbdab23a
--- /dev/null
+++ b/libc/test/utils/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(FPUtil)
diff --git a/libc/test/utils/FPUtil/CMakeLists.txt b/libc/test/utils/FPUtil/CMakeLists.txt
new file mode 100644
index 0000000000000..65cb677e22d3c
--- /dev/null
+++ b/libc/test/utils/FPUtil/CMakeLists.txt
@@ -0,0 +1,10 @@
+if((${LIBC_TARGET_OS} STREQUAL "linux") AND (${LIBC_TARGET_MACHINE} MATCHES "i386|x86_64"))
+  add_libc_unittest(
+    x86_long_double_test
+    SRCS
+      x86_long_double_test.cpp
+    DEPENDS
+      libc.include.math
+      libc.utils.FPUtil.fputil
+  )
+endif()
diff --git a/libc/test/utils/FPUtil/x86_long_double_test.cpp b/libc/test/utils/FPUtil/x86_long_double_test.cpp
new file mode 100644
index 0000000000000..d4fb8707f2bf2
--- /dev/null
+++ b/libc/test/utils/FPUtil/x86_long_double_test.cpp
@@ -0,0 +1,85 @@
+//===-- Unittests for x86 long double -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "include/math.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/UnitTest/Test.h"
+
+using FPBits = __llvm_libc::fputil::FPBits<long double>;
+
+TEST(X86LongDoubleTest, isNaN) {
+  // In the nan checks below, we use the macro isnan from math.h to ensure that
+  // a number is actually a NaN. The isnan macro resolves to the compiler
+  // builtin function. Hence, matching LLVM-libc's notion of NaN with the
+  // isnan result ensures that LLVM-libc's behavior matches the compiler's
+  // behavior.
+
+  FPBits bits(0.0l);
+  bits.exponent = FPBits::maxExponent;
+  for (unsigned int i = 0; i < 1000000; ++i) {
+    // If exponent has the max value and the implicit bit is 0,
+    // then the number is a NaN for all values of mantissa.
+    bits.mantissa = i;
+    long double nan = bits;
+    ASSERT_NE(isnan(nan), 0);
+    ASSERT_TRUE(bits.isNaN());
+  }
+
+  bits.implicitBit = 1;
+  for (unsigned int i = 1; i < 1000000; ++i) {
+    // If exponent has the max value and the implicit bit is 1,
+    // then the number is a NaN for all non-zero values of mantissa.
+    // Note the initial value of |i| of 1 to avoid a zero mantissa.
+    bits.mantissa = i;
+    long double nan = bits;
+    ASSERT_NE(isnan(nan), 0);
+    ASSERT_TRUE(bits.isNaN());
+  }
+
+  bits.exponent = 1;
+  bits.implicitBit = 0;
+  for (unsigned int i = 0; i < 1000000; ++i) {
+    // If exponent is non-zero and also not max, and the implicit bit is 0,
+    // then the number is a NaN for all values of mantissa.
+    bits.mantissa = i;
+    long double nan = bits;
+    ASSERT_NE(isnan(nan), 0);
+    ASSERT_TRUE(bits.isNaN());
+  }
+
+  bits.exponent = 1;
+  bits.implicitBit = 1;
+  for (unsigned int i = 0; i < 1000000; ++i) {
+    // If exponent is non-zero and also not max, and the implicit bit is 1,
+    // then the number is normal value for all values of mantissa.
+    bits.mantissa = i;
+    long double valid = bits;
+    ASSERT_EQ(isnan(valid), 0);
+    ASSERT_FALSE(bits.isNaN());
+  }
+
+  bits.exponent = 0;
+  bits.implicitBit = 1;
+  for (unsigned int i = 0; i < 1000000; ++i) {
+    // If exponent is zero, then the number is a valid but denormal value.
+    bits.mantissa = i;
+    long double valid = bits;
+    ASSERT_EQ(isnan(valid), 0);
+    ASSERT_FALSE(bits.isNaN());
+  }
+
+  bits.exponent = 0;
+  bits.implicitBit = 0;
+  for (unsigned int i = 0; i < 1000000; ++i) {
+    // If exponent is zero, then the number is a valid but denormal value.
+    bits.mantissa = i;
+    long double valid = bits;
+    ASSERT_EQ(isnan(valid), 0);
+    ASSERT_FALSE(bits.isNaN());
+  }
+}
diff --git a/libc/utils/FPUtil/LongDoubleBitsX86.h b/libc/utils/FPUtil/LongDoubleBitsX86.h
index 1e92dba4383df..3d7f455ff22c9 100644
--- a/libc/utils/FPUtil/LongDoubleBitsX86.h
+++ b/libc/utils/FPUtil/LongDoubleBitsX86.h
@@ -62,9 +62,18 @@ template <> struct __attribute__((packed)) FPBits<long double> {
     return exponent == maxExponent && mantissa == 0 && implicitBit == 1;
   }
 
-  bool isNaN() const { return exponent == maxExponent && mantissa != 0; }
+  bool isNaN() const {
+    if (exponent == maxExponent) {
+      return (implicitBit == 0) || mantissa != 0;
+    } else if (exponent != 0) {
+      return implicitBit == 0;
+    }
+    return false;
+  }
 
-  bool isInfOrNaN() const { return exponent == maxExponent; }
+  bool isInfOrNaN() const {
+    return (exponent == maxExponent) || (exponent != 0 && implicitBit == 0);
+  }
 
   // Methods below this are used by tests.
 
diff --git a/libc/utils/FPUtil/ManipulationFunctions.h b/libc/utils/FPUtil/ManipulationFunctions.h
index b1e06f708b2f7..ce5400f5335ff 100644
--- a/libc/utils/FPUtil/ManipulationFunctions.h
+++ b/libc/utils/FPUtil/ManipulationFunctions.h
@@ -6,10 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "BitPatterns.h"
-#include "ClassificationFunctions.h"
-#include "FloatOperations.h"
-#include "FloatProperties.h"
+#include "FPBits.h"
 #include "NearestIntegerOperations.h"
 
 #include "utils/CPP/TypeTraits.h"
@@ -20,52 +17,87 @@
 namespace __llvm_libc {
 namespace fputil {
 
-template <typename T,
-          cpp::EnableIfType<cpp::IsFloatingPointType<T>::Value, int> = 0>
+#if defined(__x86_64__) || defined(__i386__)
+template <typename T> struct Standard754Type {
+  static constexpr bool Value =
+      cpp::IsSame<float, cpp::RemoveCVType<T>>::Value ||
+      cpp::IsSame<double, cpp::RemoveCVType<T>>::Value;
+};
+#else
+template <typename T> struct Standard754Type {
+  static constexpr bool Value = cpp::IsFloatingPointType<T>::Value;
+};
+#endif
+
+template <typename T> static inline T frexp_impl(FPBits<T> &bits, int &exp) {
+  exp = bits.getExponent() + 1;
+  static constexpr uint16_t resultExponent = FPBits<T>::exponentBias - 1;
+  bits.exponent = resultExponent;
+  return bits;
+}
+
+template <typename T, cpp::EnableIfType<Standard754Type<T>::Value, int> = 0>
 static inline T frexp(T x, int &exp) {
-  using Properties = FloatProperties<T>;
-  using BitsType = typename Properties::BitsType;
+  FPBits<T> bits(x);
+  if (bits.isInfOrNaN())
+    return x;
+  if (bits.isZero()) {
+    exp = 0;
+    return x;
+  }
+
+  return frexp_impl(bits, exp);
+}
 
-  auto bits = valueAsBits(x);
-  if (bitsAreInfOrNaN(bits))
+#if defined(__x86_64__) || defined(__i386__)
+static inline long double frexp(long double x, int &exp) {
+  FPBits<long double> bits(x);
+  if (bits.isInfOrNaN())
     return x;
-  if (bitsAreZero(bits)) {
+  if (bits.isZero()) {
     exp = 0;
     return x;
   }
 
-  exp = getExponentFromBits(bits) + 1;
+  if (bits.exponent != 0 || bits.implicitBit == 1)
+    return frexp_impl(bits, exp);
 
-  static constexpr BitsType resultExponent =
-      Properties::exponentOffset - BitsType(1);
-  // Capture the sign and mantissa part.
-  bits &= (Properties::mantissaMask | Properties::signMask);
-  // Insert the new exponent.
-  bits |= (resultExponent << Properties::mantissaWidth);
+  exp = bits.getExponent();
+  int shiftCount = 0;
+  uint64_t fullMantissa = *reinterpret_cast<uint64_t *>(&bits);
+  static constexpr uint64_t msBitMask = uint64_t(1) << 63;
+  for (; (fullMantissa & msBitMask) == uint64_t(0);
+       fullMantissa <<= 1, ++shiftCount) {
+    // This for loop will terminate as fullMantissa is != 0. If it were 0,
+    // then x will be NaN and handled before control reaches here.
+    // When the loop terminates, fullMantissa will represent the full mantissa
+    // of a normal long double value. That is, the implicit bit has the value
+    // of 1.
+  }
 
-  return valueFromBits(bits);
+  exp = exp - shiftCount + 1;
+  *reinterpret_cast<uint64_t *>(&bits) = fullMantissa;
+  bits.exponent = FPBits<long double>::exponentBias - 1;
+  return bits;
 }
+#endif
 
 template <typename T,
           cpp::EnableIfType<cpp::IsFloatingPointType<T>::Value, int> = 0>
 static inline T modf(T x, T &iptr) {
-  auto bits = valueAsBits(x);
-  if (bitsAreZero(bits) || bitsAreNaN(bits)) {
+  FPBits<T> bits(x);
+  if (bits.isZero() || bits.isNaN()) {
     iptr = x;
     return x;
-  } else if (bitsAreInf(bits)) {
+  } else if (bits.isInf()) {
     iptr = x;
-    return bits & FloatProperties<T>::signMask
-               ? valueFromBits(BitPatterns<T>::negZero)
-               : valueFromBits(BitPatterns<T>::zero);
+    return bits.sign ? FPBits<T>::negZero() : FPBits<T>::zero();
   } else {
     iptr = trunc(x);
     if (x == iptr) {
       // If x is already an integer value, then return zero with the right
       // sign.
-      return bits & FloatProperties<T>::signMask
-                 ? valueFromBits(BitPatterns<T>::negZero)
-                 : valueFromBits(BitPatterns<T>::zero);
+      return bits.sign ? FPBits<T>::negZero() : FPBits<T>::zero();
     } else {
       return x - iptr;
     }
@@ -75,28 +107,65 @@ static inline T modf(T x, T &iptr) {
 template <typename T,
           cpp::EnableIfType<cpp::IsFloatingPointType<T>::Value, int> = 0>
 static inline T copysign(T x, T y) {
-  constexpr auto signMask = FloatProperties<T>::signMask;
-  auto xbits = valueAsBits(x);
-  auto ybits = valueAsBits(y);
-  return valueFromBits((xbits & ~signMask) | (ybits & signMask));
+  FPBits<T> xbits(x);
+  xbits.sign = FPBits<T>(y).sign;
+  return xbits;
 }
 
-template <typename T,
-          cpp::EnableIfType<cpp::IsFloatingPointType<T>::Value, int> = 0>
+template <typename T> static inline T logb_impl(const FPBits<T> &bits) {
+  return bits.getExponent();
+}
+
+template <typename T, cpp::EnableIfType<Standard754Type<T>::Value, int> = 0>
 static inline T logb(T x) {
-  auto bits = valueAsBits(x);
-  if (bitsAreZero(bits)) {
+  FPBits<T> bits(x);
+  if (bits.isZero()) {
     // TODO(Floating point exception): Raise div-by-zero exception.
     // TODO(errno): POSIX requires setting errno to ERANGE.
-    return valueFromBits(BitPatterns<T>::negInf);
-  } else if (bitsAreInf(bits)) {
-    return valueFromBits(BitPatterns<T>::inf);
-  } else if (bitsAreNaN(bits)) {
+    return FPBits<T>::negInf();
+  } else if (bits.isNaN()) {
     return x;
-  } else {
-    return getExponentFromBits(bits);
+  } else if (bits.isInf()) {
+    // Return positive infinity.
+    return FPBits<T>::inf();
+  }
+
+  return logb_impl(bits);
+}
+
+#if defined(__x86_64__) || defined(__i386__)
+static inline long double logb(long double x) {
+  FPBits<long double> bits(x);
+  if (bits.isZero()) {
+    // TODO(Floating point exception): Raise div-by-zero exception.
+    // TODO(errno): POSIX requires setting errno to ERANGE.
+    return FPBits<long double>::negInf();
+  } else if (bits.isNaN()) {
+    return x;
+  } else if (bits.isInf()) {
+    // Return positive infinity.
+    return FPBits<long double>::inf();
   }
+
+  if (bits.exponent != 0 || bits.implicitBit == 1)
+    return logb_impl(bits);
+
+  int exp = bits.getExponent();
+  int shiftCount = 0;
+  uint64_t fullMantissa = *reinterpret_cast<uint64_t *>(&bits);
+  static constexpr uint64_t msBitMask = uint64_t(1) << 63;
+  for (; (fullMantissa & msBitMask) == uint64_t(0);
+       fullMantissa <<= 1, ++shiftCount) {
+    // This for loop will terminate as fullMantissa is != 0. If it were 0,
+    // then x will be NaN and handled before control reaches here.
+    // When the loop terminates, fullMantissa will represent the full mantissa
+    // of a normal long double value. That is, the implicit bit has the value
+    // of 1.
+  }
+
+  return exp - shiftCount;
 }
+#endif
 
 } // namespace fputil
 } // namespace __llvm_libc
diff --git a/libc/utils/FPUtil/NearestIntegerOperations.h b/libc/utils/FPUtil/NearestIntegerOperations.h
index c0535f5993654..1c8ad36274c87 100644
--- a/libc/utils/FPUtil/NearestIntegerOperations.h
+++ b/libc/utils/FPUtil/NearestIntegerOperations.h
@@ -9,10 +9,7 @@
 #ifndef LLVM_LIBC_UTILS_FPUTIL_NEAREST_INTEGER_OPERATIONS_H
 #define LLVM_LIBC_UTILS_FPUTIL_NEAREST_INTEGER_OPERATIONS_H
 
-#include "ClassificationFunctions.h"
 #include "FPBits.h"
-#include "FloatOperations.h"
-#include "FloatProperties.h"
 
 #include "utils/CPP/TypeTraits.h"
 
@@ -54,21 +51,18 @@ static inline T trunc(T x) {
 template <typename T,
           cpp::EnableIfType<cpp::IsFloatingPointType<T>::Value, int> = 0>
 static inline T ceil(T x) {
-  using Properties = FloatProperties<T>;
-  using BitsType = typename FloatProperties<T>::BitsType;
-
-  BitsType bits = valueAsBits(x);
+  FPBits<T> bits(x);
 
   // If x is infinity NaN or zero, return it.
-  if (bitsAreInfOrNaN(bits) || bitsAreZero(bits))
+  if (bits.isInfOrNaN() || bits.isZero())
     return x;
 
-  bool isNeg = bits & Properties::signMask;
-  int exponent = getExponentFromBits(bits);
+  bool isNeg = bits.sign;
+  int exponent = bits.getExponent();
 
   // If the exponent is greater than the most negative mantissa
   // exponent, then x is already an integer.
-  if (exponent >= static_cast<int>(Properties::mantissaWidth))
+  if (exponent >= static_cast<int>(MantissaWidth<T>::value))
     return x;
 
   if (exponent <= -1) {
@@ -78,14 +72,14 @@ static inline T ceil(T x) {
       return T(1.0);
   }
 
-  uint32_t trimSize = Properties::mantissaWidth - exponent;
+  uint32_t trimSize = MantissaWidth<T>::value - exponent;
+  bits.mantissa = (bits.mantissa >> trimSize) << trimSize;
+  T truncValue = T(bits);
+
   // If x is already an integer, return it.
-  if ((bits << (Properties::bitWidth - trimSize)) == 0)
+  if (truncValue == x)
     return x;
 
-  BitsType truncBits = (bits >> trimSize) << trimSize;
-  T truncValue = valueFromBits(truncBits);
-
   // If x is negative, the ceil operation is equivalent to the trunc operation.
   if (isNeg)
     return truncValue;
@@ -96,8 +90,8 @@ static inline T ceil(T x) {
 template <typename T,
           cpp::EnableIfType<cpp::IsFloatingPointType<T>::Value, int> = 0>
 static inline T floor(T x) {
-  auto bits = valueAsBits(x);
-  if (FloatProperties<T>::signMask & bits) {
+  FPBits<T> bits(x);
+  if (bits.sign) {
     return -ceil(-x);
   } else {
     return trunc(x);
@@ -107,21 +101,19 @@ static inline T floor(T x) {
 template <typename T,
           cpp::EnableIfType<cpp::IsFloatingPointType<T>::Value, int> = 0>
 static inline T round(T x) {
-  using Properties = FloatProperties<T>;
-  using BitsType = typename FloatProperties<T>::BitsType;
-
-  BitsType bits = valueAsBits(x);
+  using UIntType = typename FPBits<T>::UIntType;
+  FPBits<T> bits(x);
 
-  // If x is infinity, NaN or zero, return it.
-  if (bitsAreInfOrNaN(bits) || bitsAreZero(bits))
+  // If x is infinity NaN or zero, return it.
+  if (bits.isInfOrNaN() || bits.isZero())
     return x;
 
-  bool isNeg = bits & Properties::signMask;
-  int exponent = getExponentFromBits(bits);
+  bool isNeg = bits.sign;
+  int exponent = bits.getExponent();
 
   // If the exponent is greater than the most negative mantissa
   // exponent, then x is already an integer.
-  if (exponent >= static_cast<int>(Properties::mantissaWidth))
+  if (exponent >= static_cast<int>(MantissaWidth<T>::value))
     return x;
 
   if (exponent == -1) {
@@ -140,24 +132,22 @@ static inline T round(T x) {
       return T(0.0);
   }
 
-  uint32_t trimSize = Properties::mantissaWidth - exponent;
+  uint32_t trimSize = MantissaWidth<T>::value - exponent;
+  bool halfBitSet = bits.mantissa & (UIntType(1) << (trimSize - 1));
+  bits.mantissa = (bits.mantissa >> trimSize) << trimSize;
+  T truncValue = T(bits);
+
   // If x is already an integer, return it.
-  if ((bits << (Properties::bitWidth - trimSize)) == 0)
+  if (truncValue == x)
     return x;
 
-  BitsType truncBits = (bits >> trimSize) << trimSize;
-  T truncValue = valueFromBits(truncBits);
-
-  if ((bits & (BitsType(1) << (trimSize - 1))) == 0) {
+  if (!halfBitSet) {
     // Franctional part is less than 0.5 so round value is the
     // same as the trunc value.
     return truncValue;
+  } else {
+    return isNeg ? truncValue - T(1.0) : truncValue + T(1.0);
   }
-
-  if (isNeg)
-    return truncValue - T(1.0);
-  else
-    return truncValue + T(1.0);
 }
 
 } // namespace fputil
diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index 7d94d728f9ebe..310ca5f56c461 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -110,7 +110,9 @@ option(LIBCXX_INCLUDE_TESTS "Build the libc++ tests." ${LLVM_INCLUDE_TESTS})
 option(LIBCXX_ENABLE_PARALLEL_ALGORITHMS "Enable the parallel algorithms library. This requires the PSTL to be available." OFF)
 option(LIBCXX_TEST_GDB_PRETTY_PRINTERS "Test gdb pretty printers." OFF)
 set(LIBCXX_TEST_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/test/lit.site.cfg.in" CACHE STRING
-    "The Lit testing configuration to use when running the tests")
+    "The Lit testing configuration to use when running the tests.")
+set(LIBCXX_TEST_PARAMS "" CACHE STRING
+    "A list of parameters to run the Lit test suite with.")
 
 # Benchmark options -----------------------------------------------------------
 option(LIBCXX_INCLUDE_BENCHMARKS "Build the libc++ benchmarks and their dependencies" ON)
@@ -628,7 +630,6 @@ function(cxx_add_exception_flags target)
     # functions never throw a C++ exception.
     target_add_compile_flags_if_supported(${target} PUBLIC -EHsc)
   else()
-    target_compile_definitions(${target} PUBLIC -D_LIBCPP_NO_EXCEPTIONS)
     target_add_compile_flags_if_supported(${target} PUBLIC -EHs- -EHa-)
     target_add_compile_flags_if_supported(${target} PUBLIC -fno-exceptions)
   endif()
@@ -873,23 +874,13 @@ if (DEFINED WIN32 AND LIBCXX_ENABLE_STATIC AND NOT LIBCXX_ENABLE_SHARED)
   config_define(ON _LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS)
 endif()
 
-set(site_config_path "${LIBCXX_BINARY_DIR}/__config_site")
-if (LIBCXX_NEEDS_SITE_CONFIG)
-  configure_file("include/__config_site.in"
-                 "${site_config_path}"
-                 @ONLY)
-elseif(EXISTS "${site_config_path}")
-  message(STATUS "Removing stale site configuration ${site_config_path}")
-  file(REMOVE "${site_config_path}")
-endif()
-
+# We generate a __config_site header (see libcxx/include/CMakeLists.txt) and
+# we make sure to include it when building the library.
 function(cxx_add_config_site target)
-  if (LIBCXX_NEEDS_SITE_CONFIG)
-    if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC" OR "${CMAKE_CXX_SIMULATE_ID}" STREQUAL "MSVC")
-      target_compile_options(${target} PUBLIC /FI "${site_config_path}")
-    else()
-      target_compile_options(${target} PUBLIC -include "${site_config_path}")
-    endif()
+  if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC" OR "${CMAKE_CXX_SIMULATE_ID}" STREQUAL "MSVC")
+    target_compile_options(${target} PUBLIC /FI "${LIBCXX_BINARY_DIR}/__config_site")
+  else()
+    target_compile_options(${target} PUBLIC -include "${LIBCXX_BINARY_DIR}/__config_site")
   endif()
 endfunction()
 
@@ -926,28 +917,18 @@ if (LIBCXX_INCLUDE_BENCHMARKS)
   add_subdirectory(benchmarks)
 endif()
 
-# Create the lit.site.cfg file even when LIBCXX_INCLUDE_TESTS is OFF or
-# LLVM_FOUND is OFF. This allows users to run the tests manually using
-# LIT without requiring a full LLVM checkout.
-#
-# However, since some submission systems strip test/ subdirectories, check for
-# it before adding it.
-
-if(IS_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test")
-  add_subdirectory(test)
-endif()
 if (LIBCXX_INCLUDE_TESTS)
+  add_subdirectory(test)
   add_subdirectory(lib/abi)
-endif()
-
-if (LIBCXX_STANDALONE_BUILD AND EXISTS "${LLVM_MAIN_SRC_DIR}/utils/llvm-lit")
-  include(AddLLVM) # for get_llvm_lit_path
-  # Make sure the llvm-lit script is generated into the bin directory, and do
-  # it after adding all tests, since the generated script will only work
-  # correctly discovered tests against test locations from the source tree that
-  # have already been discovered.
-  add_subdirectory(${LLVM_MAIN_SRC_DIR}/utils/llvm-lit
-                   ${CMAKE_CURRENT_BINARY_DIR}/llvm-lit)
+  if (LIBCXX_STANDALONE_BUILD)
+    include(AddLLVM) # for get_llvm_lit_path
+    # Make sure the llvm-lit script is generated into the bin directory, and
+    # do it after adding all tests, since the generated script will only work
+    # correctly discovered tests against test locations from the source tree
+    # that have already been discovered.
+    add_subdirectory(${LLVM_MAIN_SRC_DIR}/utils/llvm-lit
+                     ${CMAKE_CURRENT_BINARY_DIR}/llvm-lit)
+  endif()
 endif()
 
 if (LIBCXX_INCLUDE_DOCS)
diff --git a/libcxx/TODO.TXT b/libcxx/TODO.TXT
index 652a38de75a52..cf489d29149d1 100644
--- a/libcxx/TODO.TXT
+++ b/libcxx/TODO.TXT
@@ -2,7 +2,6 @@ This is meant to be a general place to list things that should be done "someday"
 
 CXX Runtime Library Tasks
 =========================
-* Fix that CMake always link to /usr/lib/libc++abi.dylib on OS X.
 * Look into mirroring libsupc++'s typeinfo vtable layout when libsupc++/libstdc++
   is used as the runtime library.
 * Investigate and document interoperability between libc++ and libstdc++ on
diff --git a/libcxx/benchmarks/CMakeLists.txt b/libcxx/benchmarks/CMakeLists.txt
index bd38de97d7a95..f012cccb696e9 100644
--- a/libcxx/benchmarks/CMakeLists.txt
+++ b/libcxx/benchmarks/CMakeLists.txt
@@ -18,9 +18,7 @@ if (DEFINED LIBCXX_CXX_ABI_LIBRARY_PATH)
           -L${LIBCXX_CXX_ABI_LIBRARY_PATH}
           -Wl,-rpath,${LIBCXX_CXX_ABI_LIBRARY_PATH})
 endif()
-if (LIBCXX_NEEDS_SITE_CONFIG)
-  list(APPEND BENCHMARK_LIBCXX_COMPILE_FLAGS -include "${LIBCXX_BINARY_DIR}/__config_site")
-endif()
+list(APPEND BENCHMARK_LIBCXX_COMPILE_FLAGS -include "${LIBCXX_BINARY_DIR}/__config_site")
 split_list(BENCHMARK_LIBCXX_COMPILE_FLAGS)
 
 ExternalProject_Add(google-benchmark-libcxx
diff --git a/libcxx/benchmarks/algorithms.bench.cpp b/libcxx/benchmarks/algorithms.bench.cpp
index b259d476457b7..93383e2b9cd6a 100644
--- a/libcxx/benchmarks/algorithms.bench.cpp
+++ b/libcxx/benchmarks/algorithms.bench.cpp
@@ -14,14 +14,23 @@
 
 namespace {
 
-enum class ValueType { Uint32, String };
-struct AllValueTypes : EnumValuesAsTuple<AllValueTypes, ValueType, 2> {
-  static constexpr const char* Names[] = {"uint32", "string"};
+enum class ValueType { Uint32, Uint64, Pair, Tuple, String };
+struct AllValueTypes : EnumValuesAsTuple<AllValueTypes, ValueType, 5> {
+  static constexpr const char* Names[] = {
+      "uint32", "uint64", "pair<uint32, uint32>",
+      "tuple<uint32, uint64, uint32>", "string"};
 };
 
 template <class V>
-using Value =
-    std::conditional_t<V() == ValueType::Uint32, uint32_t, std::string>;
+using Value = std::conditional_t<
+    V() == ValueType::Uint32, uint32_t,
+    std::conditional_t<
+        V() == ValueType::Uint64, uint64_t,
+        std::conditional_t<
+            V() == ValueType::Pair, std::pair<uint32_t, uint32_t>,
+            std::conditional_t<V() == ValueType::Tuple,
+                               std::tuple<uint32_t, uint64_t, uint32_t>,
+                               std::string> > > >;
 
 enum class Order {
   Random,
@@ -37,7 +46,8 @@ struct AllOrders : EnumValuesAsTuple<AllOrders, Order, 6> {
                                           "PipeOrgan",  "Heap"};
 };
 
-void fillValues(std::vector<uint32_t>& V, size_t N, Order O) {
+template <typename T>
+void fillValues(std::vector<T>& V, size_t N, Order O) {
   if (O == Order::SingleElement) {
     V.resize(N, 0);
   } else {
@@ -46,13 +56,49 @@ void fillValues(std::vector<uint32_t>& V, size_t N, Order O) {
   }
 }
 
-void fillValues(std::vector<std::string>& V, size_t N, Order O) {
+template <typename T>
+void fillValues(std::vector<std::pair<T, T> >& V, size_t N, Order O) {
+  if (O == Order::SingleElement) {
+    V.resize(N, std::make_pair(0, 0));
+  } else {
+    while (V.size() < N)
+      // Half of array will have the same first element.
+      if (V.size() % 2) {
+        V.push_back(std::make_pair(V.size(), V.size()));
+      } else {
+        V.push_back(std::make_pair(0, V.size()));
+      }
+  }
+}
 
+template <typename T1, typename T2, typename T3>
+void fillValues(std::vector<std::tuple<T1, T2, T3> >& V, size_t N, Order O) {
+  if (O == Order::SingleElement) {
+    V.resize(N, std::make_tuple(0, 0, 0));
+  } else {
+    while (V.size() < N)
+      // One third of array will have the same first element.
+      // One third of array will have the same first element and the same second element.
+      switch (V.size() % 3) {
+      case 0:
+        V.push_back(std::make_tuple(V.size(), V.size(), V.size()));
+        break;
+      case 1:
+        V.push_back(std::make_tuple(0, V.size(), V.size()));
+        break;
+      case 2:
+        V.push_back(std::make_tuple(0, 0, V.size()));
+        break;
+      }
+  }
+}
+
+void fillValues(std::vector<std::string>& V, size_t N, Order O) {
   if (O == Order::SingleElement) {
-    V.resize(N, getRandomString(1024));
+    V.resize(N, getRandomString(64));
   } else {
     while (V.size() < N)
-      V.push_back(getRandomString(1024));
+      V.push_back(getRandomString(64));
   }
 }
 
@@ -85,21 +131,24 @@ void sortValues(T& V, Order O) {
   }
 }
 
+constexpr size_t TestSetElements =
+#if !TEST_HAS_FEATURE(memory_sanitizer)
+    1 << 18;
+#else
+    1 << 14;
+#endif
+
 template <class ValueType>
 std::vector<std::vector<Value<ValueType> > > makeOrderedValues(size_t N,
                                                                Order O) {
-  // Let's make sure that all random sequences of the same size are the same.
-  // That way we can compare the different algorithms with the same input.
-  static std::map<std::pair<size_t, Order>, std::vector<Value<ValueType> > >
-      Cached;
-
-  auto& Values = Cached[{N, O}];
-  if (Values.empty()) {
-    fillValues(Values, N, O);
-    sortValues(Values, O);
-  };
-  const size_t NumCopies = std::max(size_t{1}, 1000 / N);
-  return { NumCopies, Values };
+  std::vector<std::vector<Value<ValueType> > > Ret;
+  const size_t NumCopies = std::max(size_t{1}, TestSetElements / N);
+  Ret.resize(NumCopies);
+  for (auto& V : Ret) {
+    fillValues(V, N, O);
+    sortValues(V, O);
+  }
+  return Ret;
 }
 
 template <class T, class U>
@@ -111,19 +160,28 @@ TEST_ALWAYS_INLINE void resetCopies(benchmark::State& state, T& Copies,
   state.ResumeTiming();
 }
 
+enum class BatchSize {
+  CountElements,
+  CountBatch,
+};
+
 template <class ValueType, class F>
 void runOpOnCopies(benchmark::State& state, size_t Quantity, Order O,
-                   bool CountElements, F f) {
+                   BatchSize Count, F Body) {
   auto Copies = makeOrderedValues<ValueType>(Quantity, O);
-  const auto Orig = Copies[0];
+  auto Orig = Copies;
 
-  const size_t Batch = CountElements ? Copies.size() * Quantity : Copies.size();
+  const size_t Batch = Count == BatchSize::CountElements
+                           ? Copies.size() * Quantity
+                           : Copies.size();
   while (state.KeepRunningBatch(Batch)) {
     for (auto& Copy : Copies) {
-      f(Copy);
+      Body(Copy);
       benchmark::DoNotOptimize(Copy);
     }
-    resetCopies(state, Copies, Orig);
+    state.PauseTiming();
+    Copies = Orig;
+    state.ResumeTiming();
   }
 }
 
@@ -132,9 +190,9 @@ struct Sort {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(state, Quantity, Order(), false, [](auto& Copy) {
-      std::sort(Copy.begin(), Copy.end());
-    });
+    runOpOnCopies<ValueType>(
+        state, Quantity, Order(), BatchSize::CountElements,
+        [](auto& Copy) { std::sort(Copy.begin(), Copy.end()); });
   }
 
   bool skip() const { return Order() == ::Order::Heap; }
@@ -150,9 +208,9 @@ struct StableSort {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(state, Quantity, Order(), false, [](auto& Copy) {
-      std::stable_sort(Copy.begin(), Copy.end());
-    });
+    runOpOnCopies<ValueType>(
+        state, Quantity, Order(), BatchSize::CountElements,
+        [](auto& Copy) { std::stable_sort(Copy.begin(), Copy.end()); });
   }
 
   bool skip() const { return Order() == ::Order::Heap; }
@@ -168,9 +226,9 @@ struct MakeHeap {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(state, Quantity, Order(), false, [](auto& Copy) {
-      std::make_heap(Copy.begin(), Copy.end());
-    });
+    runOpOnCopies<ValueType>(
+        state, Quantity, Order(), BatchSize::CountElements,
+        [](auto& Copy) { std::make_heap(Copy.begin(), Copy.end()); });
   }
 
   std::string name() const {
@@ -185,7 +243,7 @@ struct SortHeap {
 
   void run(benchmark::State& state) const {
     runOpOnCopies<ValueType>(
-        state, Quantity, Order::Heap, false,
+        state, Quantity, Order::Heap, BatchSize::CountElements,
         [](auto& Copy) { std::sort_heap(Copy.begin(), Copy.end()); });
   }
 
@@ -199,10 +257,11 @@ struct MakeThenSortHeap {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(state, Quantity, Order(), false, [](auto& Copy) {
-      std::make_heap(Copy.begin(), Copy.end());
-      std::sort_heap(Copy.begin(), Copy.end());
-    });
+    runOpOnCopies<ValueType>(state, Quantity, Order(), BatchSize::CountElements,
+                             [](auto& Copy) {
+                               std::make_heap(Copy.begin(), Copy.end());
+                               std::sort_heap(Copy.begin(), Copy.end());
+                             });
   }
 
   std::string name() const {
@@ -216,11 +275,12 @@ struct PushHeap {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(state, Quantity, Order(), true, [](auto& Copy) {
-      for (auto I = Copy.begin(), E = Copy.end(); I != E; ++I) {
-        std::push_heap(Copy.begin(), I + 1);
-      }
-    });
+    runOpOnCopies<ValueType>(
+        state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
+          for (auto I = Copy.begin(), E = Copy.end(); I != E; ++I) {
+            std::push_heap(Copy.begin(), I + 1);
+          }
+        });
   }
 
   bool skip() const { return Order() == ::Order::Heap; }
@@ -236,11 +296,12 @@ struct PopHeap {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(state, Quantity, Order(), true, [](auto& Copy) {
-      for (auto B = Copy.begin(), I = Copy.end(); I != B; --I) {
-        std::pop_heap(B, I);
-      }
-    });
+    runOpOnCopies<ValueType>(
+        state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
+          for (auto B = Copy.begin(), I = Copy.end(); I != B; --I) {
+            std::pop_heap(B, I);
+          }
+        });
   }
 
   std::string name() const {
@@ -273,4 +334,4 @@ int main(int argc, char** argv) {
   makeCartesianProductBenchmark<PushHeap, AllValueTypes, AllOrders>(Quantities);
   makeCartesianProductBenchmark<PopHeap, AllValueTypes>(Quantities);
   benchmark::RunSpecifiedBenchmarks();
-}
+}
\ No newline at end of file
diff --git a/libcxx/cmake/Modules/HandleLibCXXABI.cmake b/libcxx/cmake/Modules/HandleLibCXXABI.cmake
index 5070e99887f01..c5aa26739e367 100644
--- a/libcxx/cmake/Modules/HandleLibCXXABI.cmake
+++ b/libcxx/cmake/Modules/HandleLibCXXABI.cmake
@@ -52,7 +52,7 @@ macro(setup_abi_lib abidefines abishared abistatic abifiles abidirs)
             COMMENT "Copying C++ ABI header ${fpath}...")
         list(APPEND abilib_headers "${dst}")
 
-        if (NOT LIBCXX_USING_INSTALLED_LLVM AND LIBCXX_HEADER_DIR)
+        if (LIBCXX_HEADER_DIR)
           set(dst "${LIBCXX_HEADER_DIR}/include/c++/v1/${dstdir}/${fpath}")
           add_custom_command(OUTPUT ${dst}
               DEPENDS ${src}
diff --git a/libcxx/cmake/Modules/HandleLibcxxFlags.cmake b/libcxx/cmake/Modules/HandleLibcxxFlags.cmake
index b0fb98b9866e1..a5b4df6600ee3 100644
--- a/libcxx/cmake/Modules/HandleLibcxxFlags.cmake
+++ b/libcxx/cmake/Modules/HandleLibcxxFlags.cmake
@@ -88,20 +88,17 @@ endmacro()
 macro(config_define_if condition def)
   if (${condition})
     set(${def} ON)
-    set(LIBCXX_NEEDS_SITE_CONFIG ON)
   endif()
 endmacro()
 
 macro(config_define_if_not condition def)
   if (NOT ${condition})
     set(${def} ON)
-    set(LIBCXX_NEEDS_SITE_CONFIG ON)
   endif()
 endmacro()
 
 macro(config_define value def)
   set(${def} ${value})
-  set(LIBCXX_NEEDS_SITE_CONFIG ON)
 endmacro()
 
 # Add a list of flags to all of 'CMAKE_CXX_FLAGS', 'CMAKE_C_FLAGS',
diff --git a/libcxx/cmake/Modules/HandleOutOfTreeLLVM.cmake b/libcxx/cmake/Modules/HandleOutOfTreeLLVM.cmake
index 5746afb5eb3ff..457ad41760422 100644
--- a/libcxx/cmake/Modules/HandleOutOfTreeLLVM.cmake
+++ b/libcxx/cmake/Modules/HandleOutOfTreeLLVM.cmake
@@ -1,142 +1,78 @@
-macro(find_llvm_parts)
-# Rely on llvm-config.
-  set(CONFIG_OUTPUT)
-  if(NOT LLVM_CONFIG_PATH)
-    find_program(LLVM_CONFIG_PATH "llvm-config")
-  endif()
-  if(DEFINED LLVM_PATH)
-    set(LLVM_INCLUDE_DIR ${LLVM_INCLUDE_DIR} CACHE PATH "Path to llvm/include")
-    set(LLVM_PATH ${LLVM_PATH} CACHE PATH "Path to LLVM source tree")
-    set(LLVM_MAIN_SRC_DIR ${LLVM_PATH})
-    set(LLVM_CMAKE_PATH "${LLVM_PATH}/cmake/modules")
-    if (NOT IS_DIRECTORY "${LLVM_PATH}")
-      message(FATAL_ERROR "The provided LLVM_PATH (${LLVM_PATH}) is not a valid directory")
-    endif()
-  elseif(LLVM_CONFIG_PATH)
-    message(STATUS "Found LLVM_CONFIG_PATH as ${LLVM_CONFIG_PATH}")
-    set(LIBCXX_USING_INSTALLED_LLVM 1)
-    set(CONFIG_COMMAND ${LLVM_CONFIG_PATH}
-      "--includedir"
-      "--prefix"
-      "--src-root")
-    execute_process(
-      COMMAND ${CONFIG_COMMAND}
-      RESULT_VARIABLE HAD_ERROR
-      OUTPUT_VARIABLE CONFIG_OUTPUT
-    )
-    if(NOT HAD_ERROR)
-      string(REGEX REPLACE
-        "[ \t]*[\r\n]+[ \t]*" ";"
-        CONFIG_OUTPUT ${CONFIG_OUTPUT})
-    else()
-      string(REPLACE ";" " " CONFIG_COMMAND_STR "${CONFIG_COMMAND}")
-      message(STATUS "${CONFIG_COMMAND_STR}")
-      message(FATAL_ERROR "llvm-config failed with status ${HAD_ERROR}")
-    endif()
+if (NOT DEFINED LLVM_PATH)
+  set(LLVM_PATH ${CMAKE_CURRENT_LIST_DIR}/../../../llvm CACHE PATH "" FORCE)
+endif()
 
-    list(GET CONFIG_OUTPUT 0 INCLUDE_DIR)
-    list(GET CONFIG_OUTPUT 1 LLVM_OBJ_ROOT)
-    list(GET CONFIG_OUTPUT 2 MAIN_SRC_DIR)
+if(NOT IS_DIRECTORY ${LLVM_PATH})
+  message(FATAL_ERROR
+    "The provided LLVM_PATH (${LLVM_PATH}) is not a valid directory. Note that "
+    "building libc++ outside of the monorepo is not supported anymore. Please "
+    "use a Standalone build against the monorepo, a Runtimes build or a classic "
+    "monorepo build.")
+endif()
 
-    set(LLVM_INCLUDE_DIR ${INCLUDE_DIR} CACHE PATH "Path to llvm/include")
-    set(LLVM_BINARY_DIR ${LLVM_OBJ_ROOT} CACHE PATH "Path to LLVM build tree")
-    set(LLVM_MAIN_SRC_DIR ${MAIN_SRC_DIR} CACHE PATH "Path to LLVM source tree")
+set(LLVM_INCLUDE_DIR ${LLVM_PATH}/include CACHE PATH "Path to llvm/include")
+set(LLVM_PATH ${LLVM_PATH} CACHE PATH "Path to LLVM source tree")
+set(LLVM_MAIN_SRC_DIR ${LLVM_PATH})
+set(LLVM_CMAKE_PATH "${LLVM_PATH}/cmake/modules")
 
-    # --cmakedir is supported since llvm r291218 (4.0 release)
-    execute_process(
-      COMMAND ${LLVM_CONFIG_PATH} --cmakedir
-      RESULT_VARIABLE HAD_ERROR
-      OUTPUT_VARIABLE CONFIG_OUTPUT
-      ERROR_QUIET)
-    if(NOT HAD_ERROR)
-      string(STRIP "${CONFIG_OUTPUT}" LLVM_CMAKE_PATH_FROM_LLVM_CONFIG)
-      file(TO_CMAKE_PATH "${LLVM_CMAKE_PATH_FROM_LLVM_CONFIG}" LLVM_CMAKE_PATH)
-    else()
-      file(TO_CMAKE_PATH "${LLVM_BINARY_DIR}" LLVM_BINARY_DIR_CMAKE_STYLE)
-      set(LLVM_CMAKE_PATH "${LLVM_BINARY_DIR_CMAKE_STYLE}/lib${LLVM_LIBDIR_SUFFIX}/cmake/llvm")
-    endif()
-  else()
-    set(LLVM_FOUND OFF)
-    message(WARNING "UNSUPPORTED LIBCXX CONFIGURATION DETECTED: "
-                    "llvm-config not found and LLVM_PATH not defined.\n"
-                    "Reconfigure with -DLLVM_CONFIG_PATH=path/to/llvm-config "
-                    "or -DLLVM_PATH=path/to/llvm-source-root.")
-    return()
-  endif()
+if (EXISTS "${LLVM_CMAKE_PATH}")
+  list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_PATH}")
+elseif (EXISTS "${LLVM_MAIN_SRC_DIR}/cmake/modules")
+  list(APPEND CMAKE_MODULE_PATH "${LLVM_MAIN_SRC_DIR}/cmake/modules")
+else()
+  message(FATAL_ERROR "Neither ${LLVM_CMAKE_PATH} nor ${LLVM_MAIN_SRC_DIR}/cmake/modules found. "
+                      "This is not a supported configuration.")
+endif()
 
-  if (EXISTS "${LLVM_CMAKE_PATH}")
-    list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_PATH}")
-  elseif (EXISTS "${LLVM_MAIN_SRC_DIR}/cmake/modules")
-    list(APPEND CMAKE_MODULE_PATH "${LLVM_MAIN_SRC_DIR}/cmake/modules")
-  else()
-    set(LLVM_FOUND OFF)
-    message(WARNING "Neither ${LLVM_CMAKE_PATH} nor ${LLVM_MAIN_SRC_DIR}/cmake/modules found")
-    return()
-  endif()
+message(STATUS "Configuring for standalone build.")
+set(LIBCXX_STANDALONE_BUILD 1)
 
-  set(LLVM_FOUND ON)
-endmacro(find_llvm_parts)
+# Add LLVM Functions --------------------------------------------------------
+if (WIN32)
+  set(LLVM_ON_UNIX 0)
+  set(LLVM_ON_WIN32 1)
+else()
+  set(LLVM_ON_UNIX 1)
+  set(LLVM_ON_WIN32 0)
+endif()
 
-macro(configure_out_of_tree_llvm)
-  message(STATUS "Configuring for standalone build.")
-  set(LIBCXX_STANDALONE_BUILD 1)
+include(AddLLVM OPTIONAL)
 
-  find_llvm_parts()
+# LLVM Options --------------------------------------------------------------
+if (NOT DEFINED LLVM_INCLUDE_TESTS)
+  set(LLVM_INCLUDE_TESTS ON)
+endif()
+if (NOT DEFINED LLVM_INCLUDE_DOCS)
+  set(LLVM_INCLUDE_DOCS ON)
+endif()
+if (NOT DEFINED LLVM_ENABLE_SPHINX)
+  set(LLVM_ENABLE_SPHINX OFF)
+endif()
 
-  # Add LLVM Functions --------------------------------------------------------
-  if (LLVM_FOUND AND LIBCXX_USING_INSTALLED_LLVM)
-    include(LLVMConfig) # For TARGET_TRIPLE
-  else()
-    if (WIN32)
-      set(LLVM_ON_UNIX 0)
-      set(LLVM_ON_WIN32 1)
-    else()
-      set(LLVM_ON_UNIX 1)
-      set(LLVM_ON_WIN32 0)
-    endif()
-  endif()
-  if (LLVM_FOUND)
-    include(AddLLVM OPTIONAL)
-  endif()
+# In a standalone build, we don't have llvm to automatically generate the
+# llvm-lit script for us.  So we need to provide an explicit directory that
+# the configurator should write the script into.
+set(LLVM_LIT_OUTPUT_DIR "${libcxx_BINARY_DIR}/bin")
 
-  # LLVM Options --------------------------------------------------------------
-  if (NOT DEFINED LLVM_INCLUDE_TESTS)
-    set(LLVM_INCLUDE_TESTS ${LLVM_FOUND})
-  endif()
-  if (NOT DEFINED LLVM_INCLUDE_DOCS)
-    set(LLVM_INCLUDE_DOCS ${LLVM_FOUND})
-  endif()
-  if (NOT DEFINED LLVM_ENABLE_SPHINX)
-    set(LLVM_ENABLE_SPHINX OFF)
+if (LLVM_INCLUDE_TESTS)
+  # Required LIT Configuration ------------------------------------------------
+  # Define the default arguments to use with 'lit', and an option for the user
+  # to override.
+  set(LLVM_DEFAULT_EXTERNAL_LIT "${LLVM_MAIN_SRC_DIR}/utils/lit/lit.py")
+  set(LIT_ARGS_DEFAULT "-sv --show-xfail --show-unsupported")
+  if (MSVC OR XCODE)
+    set(LIT_ARGS_DEFAULT "${LIT_ARGS_DEFAULT} --no-progress-bar")
   endif()
+  set(LLVM_LIT_ARGS "${LIT_ARGS_DEFAULT}" CACHE STRING "Default options for lit")
+endif()
 
-  # In a standalone build, we don't have llvm to automatically generate the
-  # llvm-lit script for us.  So we need to provide an explicit directory that
-  # the configurator should write the script into.
-  set(LLVM_LIT_OUTPUT_DIR "${libcxx_BINARY_DIR}/bin")
-
-  if (LLVM_INCLUDE_TESTS)
-    # Required LIT Configuration ------------------------------------------------
-    # Define the default arguments to use with 'lit', and an option for the user
-    # to override.
-    set(LLVM_DEFAULT_EXTERNAL_LIT "${LLVM_MAIN_SRC_DIR}/utils/lit/lit.py")
-    set(LIT_ARGS_DEFAULT "-sv --show-xfail --show-unsupported")
-    if (MSVC OR XCODE)
-      set(LIT_ARGS_DEFAULT "${LIT_ARGS_DEFAULT} --no-progress-bar")
-    endif()
-    set(LLVM_LIT_ARGS "${LIT_ARGS_DEFAULT}" CACHE STRING "Default options for lit")
-  endif()
-
-  # Required doc configuration
-  if (LLVM_ENABLE_SPHINX)
-    find_package(Sphinx REQUIRED)
-  endif()
-
-  if (LLVM_ON_UNIX AND NOT APPLE)
-    set(LLVM_HAVE_LINK_VERSION_SCRIPT 1)
-  else()
-    set(LLVM_HAVE_LINK_VERSION_SCRIPT 0)
-  endif()
-endmacro(configure_out_of_tree_llvm)
+# Required doc configuration
+if (LLVM_ENABLE_SPHINX)
+  find_package(Sphinx REQUIRED)
+endif()
 
-configure_out_of_tree_llvm()
+if (LLVM_ON_UNIX AND NOT APPLE)
+  set(LLVM_HAVE_LINK_VERSION_SCRIPT 1)
+else()
+  set(LLVM_HAVE_LINK_VERSION_SCRIPT 0)
+endif()
diff --git a/libcxx/cmake/caches/Generic-cxx03.cmake b/libcxx/cmake/caches/Generic-cxx03.cmake
index d1d67d86d74a9..5edb1c4cd2d16 100644
--- a/libcxx/cmake/caches/Generic-cxx03.cmake
+++ b/libcxx/cmake/caches/Generic-cxx03.cmake
@@ -1 +1,2 @@
-set(LLVM_LIT_ARGS "--param std=c++03" CACHE STRING "")
+set(LIBCXX_TEST_PARAMS "std=c++03" CACHE STRING "")
+set(LIBCXXABI_TEST_PARAMS "${LIBCXX_TEST_PARAMS}" CACHE STRING "")
diff --git a/libcxx/cmake/caches/Generic-cxx11.cmake b/libcxx/cmake/caches/Generic-cxx11.cmake
index e203c6aeaf29f..f0f89c44d40c8 100644
--- a/libcxx/cmake/caches/Generic-cxx11.cmake
+++ b/libcxx/cmake/caches/Generic-cxx11.cmake
@@ -1 +1,2 @@
-set(LLVM_LIT_ARGS "--param std=c++11" CACHE STRING "")
+set(LIBCXX_TEST_PARAMS "std=c++11" CACHE STRING "")
+set(LIBCXXABI_TEST_PARAMS "${LIBCXX_TEST_PARAMS}" CACHE STRING "")
diff --git a/libcxx/cmake/caches/Generic-cxx14.cmake b/libcxx/cmake/caches/Generic-cxx14.cmake
index b1bf1244b510a..29c688d926b4b 100644
--- a/libcxx/cmake/caches/Generic-cxx14.cmake
+++ b/libcxx/cmake/caches/Generic-cxx14.cmake
@@ -1 +1,2 @@
-set(LLVM_LIT_ARGS "--param std=c++14" CACHE STRING "")
+set(LIBCXX_TEST_PARAMS "std=c++14" CACHE STRING "")
+set(LIBCXXABI_TEST_PARAMS "${LIBCXX_TEST_PARAMS}" CACHE STRING "")
diff --git a/libcxx/cmake/caches/Generic-cxx17.cmake b/libcxx/cmake/caches/Generic-cxx17.cmake
index b23204729ced2..35bac92070b86 100644
--- a/libcxx/cmake/caches/Generic-cxx17.cmake
+++ b/libcxx/cmake/caches/Generic-cxx17.cmake
@@ -1 +1,2 @@
-set(LLVM_LIT_ARGS "--param std=c++17" CACHE STRING "")
+set(LIBCXX_TEST_PARAMS "std=c++17" CACHE STRING "")
+set(LIBCXXABI_TEST_PARAMS "${LIBCXX_TEST_PARAMS}" CACHE STRING "")
diff --git a/libcxx/cmake/caches/Generic-cxx2a.cmake b/libcxx/cmake/caches/Generic-cxx2a.cmake
index 31f1b76ab91f9..1a07a99563dcc 100644
--- a/libcxx/cmake/caches/Generic-cxx2a.cmake
+++ b/libcxx/cmake/caches/Generic-cxx2a.cmake
@@ -1 +1,2 @@
-set(LLVM_LIT_ARGS "--param std=c++2a" CACHE STRING "")
+set(LIBCXX_TEST_PARAMS "std=c++2a" CACHE STRING "")
+set(LIBCXXABI_TEST_PARAMS "${LIBCXX_TEST_PARAMS}" CACHE STRING "")
diff --git a/libcxx/docs/BuildingLibcxx.rst b/libcxx/docs/BuildingLibcxx.rst
index 0fbb6693099cc..c85c3d7fbcb6a 100644
--- a/libcxx/docs/BuildingLibcxx.rst
+++ b/libcxx/docs/BuildingLibcxx.rst
@@ -52,21 +52,19 @@ The instructions are for building libc++ on
 FreeBSD, Linux, or Mac using `libc++abi`_ as the C++ ABI library.
 On Linux, it is also possible to use :ref:`libsupc++ <libsupcxx>` or libcxxrt.
 
-It is possible to keep your LLVM and libc++ trees separate so you can avoid
-rebuilding LLVM as often. An out-of-tree build would look like this:
+It is possible to build libc++ standalone (i.e. without building other LLVM
+projects). A standalone build would look like this:
 
 .. code-block:: bash
 
-  $ cd where-you-want-libcxx-to-live
-  $ # Check out the sources (includes everything, but we'll only use libcxx)
-  $ ``git clone https://github.com/llvm/llvm-project.git``
-  $ cd where-you-want-to-build
+  $ git clone https://github.com/llvm/llvm-project.git llvm-project
+  $ cd llvm-project
   $ mkdir build && cd build
-  $ export CC=clang CXX=clang++
-  $ cmake -DLLVM_PATH=path/to/separate/llvm \
+  $ cmake -DCMAKE_C_COMPILER=clang \
+          -DCMAKE_CXX_COMPILER=clang++ \
           -DLIBCXX_CXX_ABI=libcxxabi \
           -DLIBCXX_CXX_ABI_INCLUDE_PATHS=path/to/separate/libcxxabi/include \
-          path/to/llvm-project/libcxx
+          ../libcxx
   $ make
   $ make check-cxx # optional
 
diff --git a/libcxx/docs/TestingLibcxx.rst b/libcxx/docs/TestingLibcxx.rst
index 810131f68d615..0c9c87d489bc0 100644
--- a/libcxx/docs/TestingLibcxx.rst
+++ b/libcxx/docs/TestingLibcxx.rst
@@ -109,13 +109,13 @@ default.
 
   Specify the compiler used to build the tests.
 
-.. option:: cxx_stdlib_under_test=<stdlib name>
+.. option:: stdlib=<stdlib name>
 
-  **Values**: libc++, libstdc++
+  **Values**: libc++, libstdc++, msvc
 
-  Specify the C++ standard library being tested. Unless otherwise specified
-  libc++ is used. This option is intended to allow running the libc++ test
-  suite against other standard library implementations.
+  Specify the C++ standard library being tested. The default is libc++ if this
+  option is not provided. This option is intended to allow running the libc++
+  test suite against other standard library implementations.
 
 .. option:: std=<standard version>
 
@@ -151,22 +151,6 @@ default.
   still be used to specify the path of the library to link to and run against,
   respectively.
 
-.. option:: use_lit_shell=<bool>
-
-  Enable or disable the use of LIT's internal shell in ShTests. If the
-  environment variable LIT_USE_INTERNAL_SHELL is present then that is used as
-  the default value. Otherwise the default value is True on Windows and False
-  on every other platform.
-
-.. option:: compile_flags="<list-of-args>"
-
-  Specify additional compile flags as a space delimited string.
-  Note: This options should not be used to change the standard version used.
-
-.. option:: link_flags="<list-of-args>"
-
-  Specify additional link flags as a space delimited string.
-
 .. option:: debug_level=<level>
 
   **Values**: 0, 1
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 3625dd9f62864..da6623f103b67 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -184,27 +184,24 @@ if(LIBCXX_INSTALL_SUPPORT_HEADERS)
     )
 endif()
 
-if (LIBCXX_NEEDS_SITE_CONFIG)
-  # Generate a custom __config header. The new header is created
-  # by prepending __config_site to the current __config header.
-  add_custom_command(OUTPUT ${LIBCXX_BINARY_DIR}/__generated_config
-    COMMAND ${Python3_EXECUTABLE} ${LIBCXX_SOURCE_DIR}/utils/cat_files.py
-      ${LIBCXX_BINARY_DIR}/__config_site
-      ${LIBCXX_SOURCE_DIR}/include/__config
-      -o ${LIBCXX_BINARY_DIR}/__generated_config
-    DEPENDS ${LIBCXX_SOURCE_DIR}/include/__config
-            ${LIBCXX_BINARY_DIR}/__config_site
-  )
-  # Add a target that executes the generation commands.
-  add_custom_target(cxx-generated-config ALL
-    DEPENDS ${LIBCXX_BINARY_DIR}/__generated_config)
-  set(generated_config_deps cxx-generated-config)
-else()
-  set(files
-    ${files}
-    __config
-    )
-endif()
+configure_file("__config_site.in"
+               "${LIBCXX_BINARY_DIR}/__config_site"
+               @ONLY)
+
+# Generate a custom __config header. The new header is created
+# by prepending __config_site to the current __config header.
+add_custom_command(OUTPUT ${LIBCXX_BINARY_DIR}/__generated_config
+  COMMAND ${Python3_EXECUTABLE} ${LIBCXX_SOURCE_DIR}/utils/cat_files.py
+    ${LIBCXX_BINARY_DIR}/__config_site
+    ${LIBCXX_SOURCE_DIR}/include/__config
+    -o ${LIBCXX_BINARY_DIR}/__generated_config
+  DEPENDS ${LIBCXX_SOURCE_DIR}/include/__config
+          ${LIBCXX_BINARY_DIR}/__config_site
+)
+# Add a target that executes the generation commands.
+add_custom_target(cxx-generated-config ALL
+  DEPENDS ${LIBCXX_BINARY_DIR}/__generated_config)
+set(generated_config_deps cxx-generated-config)
 
 # In some build configurations (like bootstrapping clang), we need to be able to
 # install the libcxx headers before the CMake configuration for libcxx runs. Making
@@ -214,7 +211,7 @@ endif()
 if (NOT CXX_HEADER_TARGET)
   set(CXX_HEADER_TARGET cxx-headers)
 endif()
-if(NOT LIBCXX_USING_INSTALLED_LLVM AND LIBCXX_HEADER_DIR)
+if(LIBCXX_HEADER_DIR)
   set(output_dir ${LIBCXX_HEADER_DIR}/include/c++/v1)
 
   set(out_files)
@@ -228,16 +225,14 @@ if(NOT LIBCXX_USING_INSTALLED_LLVM AND LIBCXX_HEADER_DIR)
     list(APPEND out_files ${dst})
   endforeach()
 
-  if (LIBCXX_NEEDS_SITE_CONFIG)
-    # Copy the generated header as __config into build directory.
-    set(src ${LIBCXX_BINARY_DIR}/__generated_config)
-    set(dst ${output_dir}/__config)
-    add_custom_command(OUTPUT ${dst}
-        DEPENDS ${src} ${generated_config_deps}
-        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${src} ${dst}
-        COMMENT "Copying CXX __config")
-    list(APPEND out_files ${dst})
-  endif()
+  # Copy the generated header as __config into build directory.
+  set(src ${LIBCXX_BINARY_DIR}/__generated_config)
+  set(dst ${output_dir}/__config)
+  add_custom_command(OUTPUT ${dst}
+      DEPENDS ${src} ${generated_config_deps}
+      COMMAND ${CMAKE_COMMAND} -E copy_if_different ${src} ${dst}
+      COMMENT "Copying CXX __config")
+  list(APPEND out_files ${dst})
 
   add_custom_target(${CXX_HEADER_TARGET} ALL DEPENDS ${out_files} ${LIBCXX_CXX_ABI_HEADER_TARGET})
 else()
@@ -255,14 +250,12 @@ if (LIBCXX_INSTALL_HEADERS)
     )
   endforeach()
 
-  if (LIBCXX_NEEDS_SITE_CONFIG)
-    # Install the generated header as __config.
-    install(FILES ${LIBCXX_BINARY_DIR}/__generated_config
-      DESTINATION ${LIBCXX_INSTALL_HEADER_PREFIX}include/c++/v1
-      PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
-      RENAME __config
-      COMPONENT ${CXX_HEADER_TARGET})
-  endif()
+  # Install the generated header as __config.
+  install(FILES ${LIBCXX_BINARY_DIR}/__generated_config
+    DESTINATION ${LIBCXX_INSTALL_HEADER_PREFIX}include/c++/v1
+    PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
+    RENAME __config
+    COMPONENT ${CXX_HEADER_TARGET})
 
   if (NOT CMAKE_CONFIGURATION_TYPES)
     add_custom_target(install-${CXX_HEADER_TARGET}
diff --git a/libcxx/include/__config b/libcxx/include/__config
index 26fadcff7ced4..7e4c37431ea44 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -398,12 +398,12 @@
 
 #if defined(_LIBCPP_COMPILER_CLANG)
 
-// _LIBCPP_ALTERNATE_STRING_LAYOUT is an old name for
-// _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT left here for backward compatibility.
-#if (defined(__APPLE__) && !defined(__i386__) && !defined(__x86_64__) &&       \
-     (!defined(__arm__) || __ARM_ARCH_7K__ >= 2)) ||                           \
-    defined(_LIBCPP_ALTERNATE_STRING_LAYOUT)
-#define _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
+#if defined(_LIBCPP_ALTERNATE_STRING_LAYOUT)
+#  error _LIBCPP_ALTERNATE_STRING_LAYOUT is deprecated, please use _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT instead
+#endif
+#if defined(__APPLE__) && !defined(__i386__) && !defined(__x86_64__) &&       \
+    (!defined(__arm__) || __ARM_ARCH_7K__ >= 2)
+#  define _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
 #endif
 
 #if __has_feature(cxx_alignas)
@@ -419,8 +419,8 @@ typedef __char16_t char16_t;
 typedef __char32_t char32_t;
 #endif
 
-#if !(__has_feature(cxx_exceptions)) && !defined(_LIBCPP_NO_EXCEPTIONS)
-#define _LIBCPP_NO_EXCEPTIONS
+#if !__has_feature(cxx_exceptions)
+#  define _LIBCPP_NO_EXCEPTIONS
 #endif
 
 #if !(__has_feature(cxx_rtti)) && !defined(_LIBCPP_NO_RTTI)
@@ -528,8 +528,8 @@ typedef __char32_t char32_t;
 
 #define _LIBCPP_NORETURN __attribute__((noreturn))
 
-#if !__EXCEPTIONS && !defined(_LIBCPP_NO_EXCEPTIONS)
-#define _LIBCPP_NO_EXCEPTIONS
+#if !__EXCEPTIONS
+#  define _LIBCPP_NO_EXCEPTIONS
 #endif
 
 // Determine if GCC supports relaxed constexpr
diff --git a/libcxx/include/__string b/libcxx/include/__string
index 2c038ce493d29..9060bf98ad89f 100644
--- a/libcxx/include/__string
+++ b/libcxx/include/__string
@@ -152,7 +152,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>& basic_string<_CharType>::insert(size_type, value_type const*, size_type)) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>::size_type basic_string<_CharType>::find_first_of(value_type const*, size_type, size_type) const) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>& basic_string<_CharType>::replace(size_type, size_type, size_type, value_type)) \
-  _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>& basic_string<_CharType>::assign(value_type const*, size_type)) \
+  _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>& basic_string<_CharType>::__assign_external(value_type const*, size_type)) \
+  _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>& basic_string<_CharType>::__assign_external(value_type const*)) \
   _Func(_LIBCPP_FUNC_VIS void basic_string<_CharType>::reserve(size_type)) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>& basic_string<_CharType>::append(value_type const*, size_type)) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>& basic_string<_CharType>::assign(basic_string const&, size_type, size_type)) \
@@ -176,7 +177,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD
   _Func(_LIBCPP_FUNC_VIS int basic_string<_CharType>::compare(value_type const*) const) \
   _Func(_LIBCPP_FUNC_VIS int basic_string<_CharType>::compare(size_type, size_type, value_type const*) const) \
   _Func(_LIBCPP_FUNC_VIS _CharType& basic_string<_CharType>::at(size_type)) \
-  _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>& basic_string<_CharType>::assign(value_type const*)) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>::size_type basic_string<_CharType>::find(value_type const*, size_type, size_type) const) \
   _Func(_LIBCPP_FUNC_VIS int basic_string<_CharType>::compare(size_type, size_type, basic_string const&, size_type, size_type) const) \
   _Func(_LIBCPP_FUNC_VIS int basic_string<_CharType>::compare(size_type, size_type, value_type const*, size_type) const) \
diff --git a/libcxx/include/regex b/libcxx/include/regex
index 433ba0ffe1fa1..f42f1ecd16a41 100644
--- a/libcxx/include/regex
+++ b/libcxx/include/regex
@@ -21,7 +21,7 @@ namespace std
 namespace regex_constants
 {
 
-emum syntax_option_type
+enum syntax_option_type
 {
     icase      = unspecified,
     nosubs     = unspecified,
@@ -631,7 +631,7 @@ template <class OutputIterator, class BidirectionalIterator,
                   const basic_regex<charT, traits>& e, const charT* fmt,
                   regex_constants::match_flag_type flags = regex_constants::match_default);
 
-template <class traits, class charT, class ST, class SA, class FST, class FSA>>
+template <class traits, class charT, class ST, class SA, class FST, class FSA>
     basic_string<charT, ST, SA>
     regex_replace(const basic_string<charT, ST, SA>& s,
                   const basic_regex<charT, traits>& e,
@@ -675,9 +675,9 @@ public:
     regex_iterator(BidirectionalIterator a, BidirectionalIterator b,
                    const regex_type& re,
                    regex_constants::match_flag_type m = regex_constants::match_default);
-    regex_iterator(_BidirectionalIterator __a, _BidirectionalIterator __b,
-                   const regex_type&& __re,
-                   regex_constants::match_flag_type __m
+    regex_iterator(BidirectionalIterator a, BidirectionalIterator b,
+                   const regex_type&& re,
+                   regex_constants::match_flag_type m
                                      = regex_constants::match_default) = delete; // C++14
     regex_iterator(const regex_iterator&);
     regex_iterator& operator=(const regex_iterator&);
@@ -698,7 +698,7 @@ typedef regex_iterator<string::const_iterator>  sregex_iterator;
 typedef regex_iterator<wstring::const_iterator> wsregex_iterator;
 
 template <class BidirectionalIterator,
-          class charT = typename iterator_traits< BidirectionalIterator>::value_type,
+          class charT = typename iterator_traits<BidirectionalIterator>::value_type,
           class traits = regex_traits<charT>>
 class regex_token_iterator
 {
@@ -735,8 +735,8 @@ public:
                              regex_constants::match_flag_type m = regex_constants::match_default);
     template <size_t N>
         regex_token_iterator(BidirectionalIterator a, BidirectionalIterator b,
-                             const regex_type& re, const int (&submatches)[N],
-                             regex_constants::match_flag_type m = regex_constants::match_default) = delete // C++14;
+                             const regex_type&& re, const int (&submatches)[N],
+                             regex_constants::match_flag_type m = regex_constants::match_default) = delete; // C++14
     regex_token_iterator(const regex_token_iterator&);
     regex_token_iterator& operator=(const regex_token_iterator&);
 
diff --git a/libcxx/include/string b/libcxx/include/string
index 7fc428027a1b2..2f846eda06c58 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -1659,6 +1659,19 @@ private:
         _NOEXCEPT
         {}
 
+    basic_string& __assign_external(const value_type* __s);
+    basic_string& __assign_external(const value_type* __s, size_type __n);
+
+    // Assigns the value in __s, guaranteed to be __n < __min_cap in length.
+    inline basic_string& __assign_short(const value_type* __s, size_type __n) {
+      pointer __p = __is_long()
+                        ? (__set_long_size(__n), __get_long_pointer())
+                        : (__set_short_size(__n), __get_short_pointer());
+      traits_type::move(_VSTD::__to_address(__p), __s, __n);
+      traits_type::assign(__p[__n], value_type());
+      return *this;
+    }
+
     _LIBCPP_INLINE_VISIBILITY void __invalidate_all_iterators();
     _LIBCPP_INLINE_VISIBILITY void __invalidate_iterators_past(size_type);
 
@@ -2267,26 +2280,32 @@ basic_string<_CharT, _Traits, _Allocator>::__assign_no_alias(
   return *this;
 }
 
+template <class _CharT, class _Traits, class _Allocator>
+basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::__assign_external(
+    const value_type* __s, size_type __n) {
+  size_type __cap = capacity();
+  if (__cap >= __n) {
+    value_type* __p = _VSTD::__to_address(__get_pointer());
+    traits_type::move(__p, __s, __n);
+    traits_type::assign(__p[__n], value_type());
+    __set_size(__n);
+    __invalidate_iterators_past(__n);
+  } else {
+    size_type __sz = size();
+    __grow_by_and_replace(__cap, __n - __cap, __sz, 0, __sz, __n, __s);
+  }
+  return *this;
+}
+
 template <class _CharT, class _Traits, class _Allocator>
 basic_string<_CharT, _Traits, _Allocator>&
 basic_string<_CharT, _Traits, _Allocator>::assign(const value_type* __s, size_type __n)
 {
     _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string::assign received nullptr");
-    size_type __cap = capacity();
-    if (__cap >= __n)
-    {
-        value_type* __p = _VSTD::__to_address(__get_pointer());
-        traits_type::move(__p, __s, __n);
-        traits_type::assign(__p[__n], value_type());
-        __set_size(__n);
-        __invalidate_iterators_past(__n);
-    }
-    else
-    {
-        size_type __sz = size();
-        __grow_by_and_replace(__cap, __n - __cap, __sz, 0, __sz, __n, __s);
-    }
-    return *this;
+    return (_LIBCPP_BUILTIN_CONSTANT_P(__n) && __n < __min_cap)
+               ? __assign_short(__s, __n)
+               : __assign_external(__s, __n);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -2471,14 +2490,23 @@ basic_string<_CharT, _Traits, _Allocator>::assign(const _Tp & __t, size_type __p
 }
 
 
+template <class _CharT, class _Traits, class _Allocator>
+basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::__assign_external(const value_type* __s) {
+  return __assign_external(__s, traits_type::length(__s));
+}
+
 template <class _CharT, class _Traits, class _Allocator>
 basic_string<_CharT, _Traits, _Allocator>&
 basic_string<_CharT, _Traits, _Allocator>::assign(const value_type* __s)
 {
     _LIBCPP_ASSERT(__s != nullptr, "string::assign received nullptr");
-    return assign(__s, traits_type::length(__s));
+    return _LIBCPP_BUILTIN_CONSTANT_P(*__s)
+               ? (traits_type::length(__s) < __min_cap
+                      ? __assign_short(__s, traits_type::length(__s))
+                      : __assign_external(__s, traits_type::length(__s)))
+               : __assign_external(__s);
 }
-
 // append
 
 template <class _CharT, class _Traits, class _Allocator>
diff --git a/libcxx/include/variant b/libcxx/include/variant
index 502d8d83aeab4..03557239a69e6 100644
--- a/libcxx/include/variant
+++ b/libcxx/include/variant
@@ -1063,8 +1063,7 @@ public:
       }
       __impl __tmp(_VSTD::move(*__rhs));
 #ifndef _LIBCPP_NO_EXCEPTIONS
-      if constexpr (__all<(is_nothrow_move_constructible_v<_Types> &&
-                           is_nothrow_swappable_v<_Types>)...>::value) {
+      if constexpr (__all<is_nothrow_move_constructible_v<_Types>...>::value) {
         this->__generic_construct(*__rhs, _VSTD::move(*__lhs));
       } else {
         // EXTENSION: When the move construction of `__lhs` into `__rhs` throws
diff --git a/libcxx/include/wchar.h b/libcxx/include/wchar.h
index 353c979d20724..b21a78968e23a 100644
--- a/libcxx/include/wchar.h
+++ b/libcxx/include/wchar.h
@@ -106,6 +106,7 @@ size_t wcsrtombs(char* restrict dst, const wchar_t** restrict src, size_t len,
 */
 
 #include <__config>
+#include <stddef.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #pragma GCC system_header
diff --git a/libcxx/lib/abi/x86_64-apple-darwin.v2.abilist b/libcxx/lib/abi/x86_64-apple-darwin.v2.abilist
deleted file mode 100644
index e8cb526c4a328..0000000000000
--- a/libcxx/lib/abi/x86_64-apple-darwin.v2.abilist
+++ /dev/null
@@ -1,2389 +0,0 @@
-{'is_defined': False, 'name': '__ZNKSt10bad_typeid4whatEv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNKSt10bad_typeid4whatEv', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNKSt11logic_error4whatEv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNKSt11logic_error4whatEv', 'type': 'I'}
-{'is_defined': True, 'name': '__ZNKSt12bad_any_cast4whatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt12experimental15fundamentals_v112bad_any_cast4whatEv', 'type': 'FUNC'}
-{'is_defined': False, 'name': '__ZNKSt13bad_exception4whatEv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNKSt13bad_exception4whatEv', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNKSt13runtime_error4whatEv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNKSt13runtime_error4whatEv', 'type': 'I'}
-{'is_defined': True, 'name': '__ZNKSt16nested_exception14rethrow_nestedEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt18bad_variant_access4whatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt19bad_optional_access4whatEv', 'type': 'FUNC'}
-{'is_defined': False, 'name': '__ZNKSt20bad_array_new_length4whatEv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNKSt20bad_array_new_length4whatEv', 'type': 'I'}
-{'is_defined': True, 'name': '__ZNKSt3__210__time_put8__do_putEPcRS1_PK2tmcc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210__time_put8__do_putEPwRS1_PK2tmcc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210error_code7messageEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb0EE11do_groupingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb0EE13do_neg_formatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb0EE13do_pos_formatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb0EE14do_curr_symbolEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb0EE14do_frac_digitsEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb0EE16do_decimal_pointEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb0EE16do_negative_signEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb0EE16do_positive_signEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb0EE16do_thousands_sepEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb1EE11do_groupingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb1EE13do_neg_formatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb1EE13do_pos_formatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb1EE14do_curr_symbolEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb1EE14do_frac_digitsEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb1EE16do_decimal_pointEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb1EE16do_negative_signEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb1EE16do_positive_signEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIcLb1EE16do_thousands_sepEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb0EE11do_groupingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb0EE13do_neg_formatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb0EE13do_pos_formatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb0EE14do_curr_symbolEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb0EE14do_frac_digitsEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb0EE16do_decimal_pointEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb0EE16do_negative_signEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb0EE16do_positive_signEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb0EE16do_thousands_sepEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb1EE11do_groupingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb1EE13do_neg_formatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb1EE13do_pos_formatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb1EE14do_curr_symbolEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb1EE14do_frac_digitsEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb1EE16do_decimal_pointEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb1EE16do_negative_signEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb1EE16do_positive_signEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__210moneypunctIwLb1EE16do_thousands_sepEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__211__libcpp_db15__decrementableEPKv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__211__libcpp_db15__find_c_from_iEPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__211__libcpp_db15__subscriptableEPKvl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__211__libcpp_db17__dereferenceableEPKv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__211__libcpp_db17__find_c_and_lockEPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__211__libcpp_db22__less_than_comparableEPKvS2_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__211__libcpp_db6unlockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__211__libcpp_db8__find_cEPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__211__libcpp_db9__addableEPKvl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212bad_weak_ptr4whatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE12find_last_ofEPKcmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE13find_first_ofEPKcmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE16find_last_not_ofEPKcmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE17find_first_not_ofEPKcmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE2atEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE4copyEPcmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE4findEPKcmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE4findEcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE5rfindEPKcmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE5rfindEcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7compareEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7compareEmmPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7compareEmmPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7compareEmmRKS5_mm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE12find_last_ofEPKwmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE13find_first_ofEPKwmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE16find_last_not_ofEPKwmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE17find_first_not_ofEPKwmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE2atEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE4copyEPwmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE4findEPKwmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE4findEwm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE5rfindEPKwmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE5rfindEwm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE7compareEPKw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE7compareEmmPKw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE7compareEmmPKwm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE7compareEmmRKS5_mm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212ctype_bynameIcE10do_tolowerEPcPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212ctype_bynameIcE10do_tolowerEc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212ctype_bynameIcE10do_toupperEPcPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212ctype_bynameIcE10do_toupperEc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212ctype_bynameIwE10do_scan_isEjPKwS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212ctype_bynameIwE10do_tolowerEPwPKw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212ctype_bynameIwE10do_tolowerEw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212ctype_bynameIwE10do_toupperEPwPKw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212ctype_bynameIwE10do_toupperEw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212ctype_bynameIwE11do_scan_notEjPKwS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212ctype_bynameIwE5do_isEPKwS3_Pj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212ctype_bynameIwE5do_isEjw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212ctype_bynameIwE8do_widenEPKcS3_Pw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212ctype_bynameIwE8do_widenEc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212ctype_bynameIwE9do_narrowEPKwS3_cPc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212ctype_bynameIwE9do_narrowEwc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__212strstreambuf6pcountEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__213random_device7entropyEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IDiE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IDiE11do_encodingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IDiE13do_max_lengthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IDiE16do_always_noconvEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IDiE5do_inER11__mbstate_tPKcS5_RS5_PDiS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IDiE6do_outER11__mbstate_tPKDiS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IDiE9do_lengthER11__mbstate_tPKcS5_m', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IDsE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IDsE11do_encodingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IDsE13do_max_lengthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IDsE16do_always_noconvEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IDsE5do_inER11__mbstate_tPKcS5_RS5_PDsS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IDsE6do_outER11__mbstate_tPKDsS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IDsE9do_lengthER11__mbstate_tPKcS5_m', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IwE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IwE11do_encodingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IwE13do_max_lengthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IwE16do_always_noconvEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IwE5do_inER11__mbstate_tPKcS5_RS5_PwS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IwE6do_outER11__mbstate_tPKwS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214__codecvt_utf8IwE9do_lengthER11__mbstate_tPKcS5_m', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214collate_bynameIcE10do_compareEPKcS3_S3_S3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214collate_bynameIcE12do_transformEPKcS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214collate_bynameIwE10do_compareEPKwS3_S3_S3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214collate_bynameIwE12do_transformEPKwS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214error_category10equivalentERKNS_10error_codeEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214error_category10equivalentEiRKNS_15error_conditionE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__214error_category23default_error_conditionEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDiLb0EE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDiLb0EE11do_encodingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDiLb0EE13do_max_lengthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDiLb0EE16do_always_noconvEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDiLb0EE5do_inER11__mbstate_tPKcS5_RS5_PDiS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDiLb0EE6do_outER11__mbstate_tPKDiS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDiLb0EE9do_lengthER11__mbstate_tPKcS5_m', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDiLb1EE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDiLb1EE11do_encodingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDiLb1EE13do_max_lengthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDiLb1EE16do_always_noconvEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDiLb1EE5do_inER11__mbstate_tPKcS5_RS5_PDiS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDiLb1EE6do_outER11__mbstate_tPKDiS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDiLb1EE9do_lengthER11__mbstate_tPKcS5_m', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDsLb0EE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDsLb0EE11do_encodingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDsLb0EE13do_max_lengthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDsLb0EE16do_always_noconvEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDsLb0EE5do_inER11__mbstate_tPKcS5_RS5_PDsS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDsLb0EE6do_outER11__mbstate_tPKDsS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDsLb0EE9do_lengthER11__mbstate_tPKcS5_m', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDsLb1EE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDsLb1EE11do_encodingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDsLb1EE13do_max_lengthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDsLb1EE16do_always_noconvEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDsLb1EE5do_inER11__mbstate_tPKcS5_RS5_PDsS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDsLb1EE6do_outER11__mbstate_tPKDsS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IDsLb1EE9do_lengthER11__mbstate_tPKcS5_m', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IwLb0EE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IwLb0EE11do_encodingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IwLb0EE13do_max_lengthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IwLb0EE16do_always_noconvEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IwLb0EE5do_inER11__mbstate_tPKcS5_RS5_PwS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IwLb0EE6do_outER11__mbstate_tPKwS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IwLb0EE9do_lengthER11__mbstate_tPKcS5_m', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IwLb1EE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IwLb1EE11do_encodingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IwLb1EE13do_max_lengthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IwLb1EE16do_always_noconvEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IwLb1EE5do_inER11__mbstate_tPKcS5_RS5_PwS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IwLb1EE6do_outER11__mbstate_tPKwS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215__codecvt_utf16IwLb1EE9do_lengthER11__mbstate_tPKcS5_m', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__215error_condition7messageEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217bad_function_call4whatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb0EE11do_groupingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb0EE13do_neg_formatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb0EE13do_pos_formatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb0EE14do_curr_symbolEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb0EE14do_frac_digitsEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb0EE16do_decimal_pointEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb0EE16do_negative_signEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb0EE16do_positive_signEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb0EE16do_thousands_sepEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb1EE11do_groupingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb1EE13do_neg_formatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb1EE13do_pos_formatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb1EE14do_curr_symbolEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb1EE14do_frac_digitsEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb1EE16do_decimal_pointEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb1EE16do_negative_signEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb1EE16do_positive_signEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIcLb1EE16do_thousands_sepEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb0EE11do_groupingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb0EE13do_neg_formatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb0EE13do_pos_formatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb0EE14do_curr_symbolEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb0EE14do_frac_digitsEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb0EE16do_decimal_pointEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb0EE16do_negative_signEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb0EE16do_positive_signEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb0EE16do_thousands_sepEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb1EE11do_groupingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb1EE13do_neg_formatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb1EE13do_pos_formatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb1EE14do_curr_symbolEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb1EE14do_frac_digitsEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb1EE16do_decimal_pointEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb1EE16do_negative_signEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb1EE16do_positive_signEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__217moneypunct_bynameIwLb1EE16do_thousands_sepEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__218__time_get_storageIcE15__do_date_orderEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__218__time_get_storageIwE15__do_date_orderEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__219__libcpp_debug_info4whatEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__219__shared_weak_count13__get_deleterERKSt9type_info', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IDiE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IDiE11do_encodingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IDiE13do_max_lengthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IDiE16do_always_noconvEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IDiE5do_inER11__mbstate_tPKcS5_RS5_PDiS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IDiE6do_outER11__mbstate_tPKDiS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IDiE9do_lengthER11__mbstate_tPKcS5_m', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IDsE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IDsE11do_encodingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IDsE13do_max_lengthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IDsE16do_always_noconvEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IDsE5do_inER11__mbstate_tPKcS5_RS5_PDsS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IDsE6do_outER11__mbstate_tPKDsS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IDsE9do_lengthER11__mbstate_tPKcS5_m', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IwE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IwE11do_encodingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IwE13do_max_lengthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IwE16do_always_noconvEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IwE5do_inER11__mbstate_tPKcS5_RS5_PwS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IwE6do_outER11__mbstate_tPKwS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__codecvt_utf8_utf16IwE9do_lengthER11__mbstate_tPKcS5_m', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__time_get_c_storageIcE3__XEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__time_get_c_storageIcE3__cEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__time_get_c_storageIcE3__rEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__time_get_c_storageIcE3__xEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__time_get_c_storageIcE7__am_pmEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__time_get_c_storageIcE7__weeksEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__time_get_c_storageIcE8__monthsEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__time_get_c_storageIwE3__XEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__time_get_c_storageIwE3__cEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__time_get_c_storageIwE3__rEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__time_get_c_storageIwE3__xEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__time_get_c_storageIwE7__am_pmEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__time_get_c_storageIwE7__weeksEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__time_get_c_storageIwE8__monthsEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__vector_base_commonILb1EE20__throw_length_errorEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__220__vector_base_commonILb1EE20__throw_out_of_rangeEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__221__basic_string_commonILb1EE20__throw_length_errorEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__221__basic_string_commonILb1EE20__throw_out_of_rangeEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__223__match_any_but_newlineIcE6__execERNS_7__stateIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__223__match_any_but_newlineIwE6__execERNS_7__stateIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem18directory_iterator13__dereferenceEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem28recursive_directory_iterator13__dereferenceEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem28recursive_directory_iterator5depthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem28recursive_directory_iterator7optionsEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem4path10__filenameEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem4path11__extensionEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem4path11__root_nameEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem4path13__parent_pathEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem4path15__relative_pathEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem4path15__root_path_rawEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem4path16__root_directoryEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem4path16lexically_normalEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem4path18lexically_relativeERKS2_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem4path3endEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem4path5beginEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem4path6__stemEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__24__fs10filesystem4path9__compareENS_17basic_string_viewIcNS_11char_traitsIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIcE10do_tolowerEPcPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIcE10do_tolowerEc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIcE10do_toupperEPcPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIcE10do_toupperEc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIcE8do_widenEPKcS3_Pc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIcE8do_widenEc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIcE9do_narrowEPKcS3_cPc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIcE9do_narrowEcc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIwE10do_scan_isEjPKwS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIwE10do_tolowerEPwPKw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIwE10do_tolowerEw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIwE10do_toupperEPwPKw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIwE10do_toupperEw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIwE11do_scan_notEjPKwS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIwE5do_isEPKwS3_Pj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIwE5do_isEjw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIwE8do_widenEPKcS3_Pw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIwE8do_widenEc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIwE9do_narrowEPKwS3_cPc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__25ctypeIwE9do_narrowEwc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__26locale4nameEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__26locale9has_facetERNS0_2idE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__26locale9use_facetERNS0_2idE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__26localeeqERKS0_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIDic11__mbstate_tE10do_unshiftERS1_PcS4_RS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIDic11__mbstate_tE11do_encodingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIDic11__mbstate_tE13do_max_lengthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIDic11__mbstate_tE16do_always_noconvEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIDic11__mbstate_tE5do_inERS1_PKcS5_RS5_PDiS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIDic11__mbstate_tE6do_outERS1_PKDiS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIDic11__mbstate_tE9do_lengthERS1_PKcS5_m', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIDsc11__mbstate_tE10do_unshiftERS1_PcS4_RS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIDsc11__mbstate_tE11do_encodingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIDsc11__mbstate_tE13do_max_lengthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIDsc11__mbstate_tE16do_always_noconvEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIDsc11__mbstate_tE5do_inERS1_PKcS5_RS5_PDsS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIDsc11__mbstate_tE6do_outERS1_PKDsS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIDsc11__mbstate_tE9do_lengthERS1_PKcS5_m', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIcc11__mbstate_tE10do_unshiftERS1_PcS4_RS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIcc11__mbstate_tE11do_encodingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIcc11__mbstate_tE13do_max_lengthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIcc11__mbstate_tE16do_always_noconvEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIcc11__mbstate_tE5do_inERS1_PKcS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIcc11__mbstate_tE6do_outERS1_PKcS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIcc11__mbstate_tE9do_lengthERS1_PKcS5_m', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIwc11__mbstate_tE10do_unshiftERS1_PcS4_RS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIwc11__mbstate_tE11do_encodingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIwc11__mbstate_tE13do_max_lengthEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIwc11__mbstate_tE16do_always_noconvEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIwc11__mbstate_tE5do_inERS1_PKcS5_RS5_PwS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIwc11__mbstate_tE6do_outERS1_PKwS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27codecvtIwc11__mbstate_tE9do_lengthERS1_PKcS5_m', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27collateIcE10do_compareEPKcS3_S3_S3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27collateIcE12do_transformEPKcS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27collateIcE7do_hashEPKcS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27collateIwE10do_compareEPKwS3_S3_S3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27collateIwE12do_transformEPKwS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27collateIwE7do_hashEPKwS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_getES4_S4_RNS_8ios_baseERjRPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_getES4_S4_RNS_8ios_baseERjRb', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_getES4_S4_RNS_8ios_baseERjRd', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_getES4_S4_RNS_8ios_baseERjRe', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_getES4_S4_RNS_8ios_baseERjRf', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_getES4_S4_RNS_8ios_baseERjRl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_getES4_S4_RNS_8ios_baseERjRm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_getES4_S4_RNS_8ios_baseERjRt', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_getES4_S4_RNS_8ios_baseERjRx', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_getES4_S4_RNS_8ios_baseERjRy', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_getES4_S4_RNS_8ios_baseERjS8_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_getES4_S4_RNS_8ios_baseERjRPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_getES4_S4_RNS_8ios_baseERjRb', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_getES4_S4_RNS_8ios_baseERjRd', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_getES4_S4_RNS_8ios_baseERjRe', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_getES4_S4_RNS_8ios_baseERjRf', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_getES4_S4_RNS_8ios_baseERjRl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_getES4_S4_RNS_8ios_baseERjRm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_getES4_S4_RNS_8ios_baseERjRt', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_getES4_S4_RNS_8ios_baseERjRx', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_getES4_S4_RNS_8ios_baseERjRy', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_getES4_S4_RNS_8ios_baseERjS8_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_putES4_RNS_8ios_baseEcPKv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_putES4_RNS_8ios_baseEcb', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_putES4_RNS_8ios_baseEcd', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_putES4_RNS_8ios_baseEce', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_putES4_RNS_8ios_baseEcl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_putES4_RNS_8ios_baseEcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_putES4_RNS_8ios_baseEcx', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_putES4_RNS_8ios_baseEcy', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_putES4_RNS_8ios_baseEwPKv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_putES4_RNS_8ios_baseEwb', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_putES4_RNS_8ios_baseEwd', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_putES4_RNS_8ios_baseEwe', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_putES4_RNS_8ios_baseEwl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_putES4_RNS_8ios_baseEwm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_putES4_RNS_8ios_baseEwx', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__27num_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_putES4_RNS_8ios_baseEwy', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28ios_base6getlocEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28messagesIcE6do_getEliiRKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28messagesIcE7do_openERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEERKNS_6localeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28messagesIcE8do_closeEl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28messagesIwE6do_getEliiRKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28messagesIwE7do_openERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEERKNS_6localeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28messagesIwE8do_closeEl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28numpunctIcE11do_groupingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28numpunctIcE11do_truenameEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28numpunctIcE12do_falsenameEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28numpunctIcE16do_decimal_pointEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28numpunctIcE16do_thousands_sepEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28numpunctIwE11do_groupingEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28numpunctIwE11do_truenameEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28numpunctIwE12do_falsenameEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28numpunctIwE16do_decimal_pointEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28numpunctIwE16do_thousands_sepEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE10__get_hourERiRS4_S4_RjRKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE10__get_yearERiRS4_S4_RjRKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE11__get_am_pmERiRS4_S4_RjRKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE11__get_monthERiRS4_S4_RjRKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE11__get_year4ERiRS4_S4_RjRKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE11do_get_dateES4_S4_RNS_8ios_baseERjP2tm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE11do_get_timeES4_S4_RNS_8ios_baseERjP2tm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE11do_get_yearES4_S4_RNS_8ios_baseERjP2tm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE12__get_minuteERiRS4_S4_RjRKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE12__get_secondERiRS4_S4_RjRKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE13__get_12_hourERiRS4_S4_RjRKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE13__get_percentERS4_S4_RjRKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE13__get_weekdayERiRS4_S4_RjRKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE13do_date_orderEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE14do_get_weekdayES4_S4_RNS_8ios_baseERjP2tm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE15__get_monthnameERiRS4_S4_RjRKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE16do_get_monthnameES4_S4_RNS_8ios_baseERjP2tm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE17__get_weekdaynameERiRS4_S4_RjRKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE17__get_white_spaceERS4_S4_RjRKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE18__get_day_year_numERiRS4_S4_RjRKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE3getES4_S4_RNS_8ios_baseERjP2tmPKcSC_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_getES4_S4_RNS_8ios_baseERjP2tmcc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE9__get_dayERiRS4_S4_RjRKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE10__get_hourERiRS4_S4_RjRKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE10__get_yearERiRS4_S4_RjRKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE11__get_am_pmERiRS4_S4_RjRKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE11__get_monthERiRS4_S4_RjRKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE11__get_year4ERiRS4_S4_RjRKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE11do_get_dateES4_S4_RNS_8ios_baseERjP2tm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE11do_get_timeES4_S4_RNS_8ios_baseERjP2tm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE11do_get_yearES4_S4_RNS_8ios_baseERjP2tm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE12__get_minuteERiRS4_S4_RjRKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE12__get_secondERiRS4_S4_RjRKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE13__get_12_hourERiRS4_S4_RjRKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE13__get_percentERS4_S4_RjRKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE13__get_weekdayERiRS4_S4_RjRKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE13do_date_orderEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE14do_get_weekdayES4_S4_RNS_8ios_baseERjP2tm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE15__get_monthnameERiRS4_S4_RjRKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE16do_get_monthnameES4_S4_RNS_8ios_baseERjP2tm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE17__get_weekdaynameERiRS4_S4_RjRKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE17__get_white_spaceERS4_S4_RjRKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE18__get_day_year_numERiRS4_S4_RjRKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE3getES4_S4_RNS_8ios_baseERjP2tmPKwSC_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_getES4_S4_RNS_8ios_baseERjP2tmcc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE9__get_dayERiRS4_S4_RjRKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEE3putES4_RNS_8ios_baseEcPK2tmPKcSC_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_putES4_RNS_8ios_baseEcPK2tmcc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEE3putES4_RNS_8ios_baseEwPK2tmPKwSC_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__28time_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_putES4_RNS_8ios_baseEwPK2tmcc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__29__num_getIcE10__do_widenERNS_8ios_baseEPc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__29__num_getIcE12__do_widen_pERNS_8ios_baseEPc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__29__num_getIwE10__do_widenERNS_8ios_baseEPw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__29__num_getIwE12__do_widen_pERNS_8ios_baseEPc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__29money_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_getES4_S4_bRNS_8ios_baseERjRNS_12basic_stringIcS3_NS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__29money_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_getES4_S4_bRNS_8ios_baseERjRe', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__29money_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_getES4_S4_bRNS_8ios_baseERjRNS_12basic_stringIwS3_NS_9allocatorIwEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__29money_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_getES4_S4_bRNS_8ios_baseERjRe', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__29money_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_putES4_bRNS_8ios_baseEcRKNS_12basic_stringIcS3_NS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__29money_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEE6do_putES4_bRNS_8ios_baseEce', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__29money_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_putES4_bRNS_8ios_baseEwRKNS_12basic_stringIwS3_NS_9allocatorIwEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNKSt3__29money_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEE6do_putES4_bRNS_8ios_baseEwe', 'type': 'FUNC'}
-{'is_defined': False, 'name': '__ZNKSt8bad_cast4whatEv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNKSt8bad_cast4whatEv', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNKSt9bad_alloc4whatEv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNKSt9bad_alloc4whatEv', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNKSt9exception4whatEv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNKSt9exception4whatEv', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt10bad_typeidC1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt10bad_typeidC1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt10bad_typeidC2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt10bad_typeidC2Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt10bad_typeidD0Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt10bad_typeidD0Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt10bad_typeidD1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt10bad_typeidD1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt10bad_typeidD2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt10bad_typeidD2Ev', 'type': 'I'}
-{'is_defined': True, 'name': '__ZNSt11logic_errorC1EPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt11logic_errorC1ERKNSt3__212basic_stringIcNS0_11char_traitsIcEENS0_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt11logic_errorC1ERKS_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt11logic_errorC2EPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt11logic_errorC2ERKNSt3__212basic_stringIcNS0_11char_traitsIcEENS0_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt11logic_errorC2ERKS_', 'type': 'FUNC'}
-{'is_defined': False, 'name': '__ZNSt11logic_errorD0Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt11logic_errorD0Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt11logic_errorD1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt11logic_errorD1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt11logic_errorD2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt11logic_errorD2Ev', 'type': 'I'}
-{'is_defined': True, 'name': '__ZNSt11logic_erroraSERKS_', 'type': 'FUNC'}
-{'is_defined': False, 'name': '__ZNSt11range_errorD0Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt11range_errorD0Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt11range_errorD1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt11range_errorD1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt11range_errorD2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt11range_errorD2Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt12domain_errorD0Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt12domain_errorD0Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt12domain_errorD1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt12domain_errorD1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt12domain_errorD2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt12domain_errorD2Ev', 'type': 'I'}
-{'is_defined': True, 'name': '__ZNSt12experimental19bad_optional_accessD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt12experimental19bad_optional_accessD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt12experimental19bad_optional_accessD2Ev', 'type': 'FUNC'}
-{'is_defined': False, 'name': '__ZNSt12length_errorD0Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt12length_errorD0Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt12length_errorD1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt12length_errorD1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt12length_errorD2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt12length_errorD2Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt12out_of_rangeD0Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt12out_of_rangeD0Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt12out_of_rangeD1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt12out_of_rangeD1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt12out_of_rangeD2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt12out_of_rangeD2Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt13bad_exceptionD0Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt13bad_exceptionD0Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt13bad_exceptionD1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt13bad_exceptionD1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt13bad_exceptionD2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt13bad_exceptionD2Ev', 'type': 'I'}
-{'is_defined': True, 'name': '__ZNSt13exception_ptrC1ERKS_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt13exception_ptrC2ERKS_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt13exception_ptrD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt13exception_ptrD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt13exception_ptraSERKS_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt13runtime_errorC1EPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt13runtime_errorC1ERKNSt3__212basic_stringIcNS0_11char_traitsIcEENS0_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt13runtime_errorC1ERKS_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt13runtime_errorC2EPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt13runtime_errorC2ERKNSt3__212basic_stringIcNS0_11char_traitsIcEENS0_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt13runtime_errorC2ERKS_', 'type': 'FUNC'}
-{'is_defined': False, 'name': '__ZNSt13runtime_errorD0Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt13runtime_errorD0Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt13runtime_errorD1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt13runtime_errorD1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt13runtime_errorD2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt13runtime_errorD2Ev', 'type': 'I'}
-{'is_defined': True, 'name': '__ZNSt13runtime_erroraSERKS_', 'type': 'FUNC'}
-{'is_defined': False, 'name': '__ZNSt14overflow_errorD0Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt14overflow_errorD0Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt14overflow_errorD1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt14overflow_errorD1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt14overflow_errorD2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt14overflow_errorD2Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt15underflow_errorD0Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt15underflow_errorD0Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt15underflow_errorD1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt15underflow_errorD1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt15underflow_errorD2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt15underflow_errorD2Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt16invalid_argumentD0Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt16invalid_argumentD0Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt16invalid_argumentD1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt16invalid_argumentD1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt16invalid_argumentD2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt16invalid_argumentD2Ev', 'type': 'I'}
-{'is_defined': True, 'name': '__ZNSt16nested_exceptionC1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt16nested_exceptionC2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt16nested_exceptionD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt16nested_exceptionD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt16nested_exceptionD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt19bad_optional_accessD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt19bad_optional_accessD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt19bad_optional_accessD2Ev', 'type': 'FUNC'}
-{'is_defined': False, 'name': '__ZNSt20bad_array_new_lengthC1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt20bad_array_new_lengthC1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt20bad_array_new_lengthC2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt20bad_array_new_lengthC2Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt20bad_array_new_lengthD0Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt20bad_array_new_lengthD0Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt20bad_array_new_lengthD1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt20bad_array_new_lengthD1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt20bad_array_new_lengthD2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt20bad_array_new_lengthD2Ev', 'type': 'I'}
-{'is_defined': True, 'name': '__ZNSt3__210__time_getC1EPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210__time_getC1ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210__time_getC2EPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210__time_getC2ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210__time_getD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210__time_getD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210__time_putC1EPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210__time_putC1ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210__time_putC2EPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210__time_putC2ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210__time_putD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210__time_putD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210adopt_lockE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210ctype_base5alnumE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210ctype_base5alphaE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210ctype_base5blankE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210ctype_base5cntrlE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210ctype_base5digitE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210ctype_base5graphE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210ctype_base5lowerE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210ctype_base5printE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210ctype_base5punctE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210ctype_base5spaceE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210ctype_base5upperE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210ctype_base6xdigitE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210defer_lockE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210istrstreamD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210istrstreamD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210istrstreamD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210moneypunctIcLb0EE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210moneypunctIcLb0EE4intlE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210moneypunctIcLb1EE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210moneypunctIcLb1EE4intlE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210moneypunctIwLb0EE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210moneypunctIwLb0EE4intlE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210moneypunctIwLb1EE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210moneypunctIwLb1EE4intlE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__210ostrstreamD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210ostrstreamD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210ostrstreamD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210to_wstringEd', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210to_wstringEe', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210to_wstringEf', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210to_wstringEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210to_wstringEj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210to_wstringEl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210to_wstringEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210to_wstringEx', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__210to_wstringEy', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__call_onceERVmPvPFvS2_E', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__libcpp_db10__insert_cEPvPFPNS_8__c_nodeES1_S1_S3_E', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__libcpp_db10__insert_iEPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__libcpp_db11__insert_icEPvPKv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__libcpp_db15__iterator_copyEPvPKv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__libcpp_db16__invalidate_allEPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__libcpp_db4swapEPvS1_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__libcpp_db9__erase_cEPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__libcpp_db9__erase_iEPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__libcpp_dbC1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__libcpp_dbC2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__libcpp_dbD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__libcpp_dbD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__money_getIcE13__gather_infoEbRKNS_6localeERNS_10money_base7patternERcS8_RNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEESF_SF_SF_Ri', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__money_getIwE13__gather_infoEbRKNS_6localeERNS_10money_base7patternERwS8_RNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEERNS9_IwNSA_IwEENSC_IwEEEESJ_SJ_Ri', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__money_putIcE13__gather_infoEbbRKNS_6localeERNS_10money_base7patternERcS8_RNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEESF_SF_Ri', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__money_putIcE8__formatEPcRS2_S3_jPKcS5_RKNS_5ctypeIcEEbRKNS_10money_base7patternEccRKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEESL_SL_i', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__money_putIwE13__gather_infoEbbRKNS_6localeERNS_10money_base7patternERwS8_RNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEERNS9_IwNSA_IwEENSC_IwEEEESJ_Ri', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211__money_putIwE8__formatEPwRS2_S3_jPKwS5_RKNS_5ctypeIwEEbRKNS_10money_base7patternEwwRKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEERKNSE_IwNSF_IwEENSH_IwEEEESQ_i', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211regex_errorC1ENS_15regex_constants10error_typeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211regex_errorC2ENS_15regex_constants10error_typeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211regex_errorD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211regex_errorD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211regex_errorD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211this_thread9sleep_forERKNS_6chrono8durationIxNS_5ratioILl1ELl1000000000EEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211timed_mutex4lockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211timed_mutex6unlockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211timed_mutex8try_lockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211timed_mutexC1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211timed_mutexC2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211timed_mutexD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211timed_mutexD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__211try_to_lockE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__212__do_nothingEPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212__get_sp_mutEPKv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212__next_primeEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212__rs_default4__c_E', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__212__rs_defaultC1ERKS0_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212__rs_defaultC1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212__rs_defaultC2ERKS0_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212__rs_defaultC2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212__rs_defaultD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212__rs_defaultD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212__rs_defaultclEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212bad_weak_ptrD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212bad_weak_ptrD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212bad_weak_ptrD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE21__grow_by_and_replaceEmmmmmmPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE2atEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE4nposE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE5eraseEmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6__initEPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6__initEPKcmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6__initEmc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6appendEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6appendEPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6appendERKS5_mm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6appendEmc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6assignEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6assignEPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6assignERKS5_mm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6assignEmc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6insertENS_11__wrap_iterIPKcEEc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6insertEmPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6insertEmPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6insertEmRKS5_mm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6insertEmmc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6resizeEmc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7replaceEmmPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7replaceEmmPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7replaceEmmRKS5_mm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7replaceEmmmc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7reserveEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9__grow_byEmmmmmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9push_backEc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEC1ERKS5_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEC1ERKS5_RKS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEC1ERKS5_mmRKS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEC2ERKS5_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEC2ERKS5_RKS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEC2ERKS5_mmRKS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEaSERKS5_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEaSEc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE21__grow_by_and_replaceEmmmmmmPKw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE2atEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE4nposE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE5eraseEmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6__initEPKwm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6__initEPKwmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6__initEmw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6appendEPKw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6appendEPKwm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6appendERKS5_mm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6appendEmw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6assignEPKw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6assignEPKwm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6assignERKS5_mm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6assignEmw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6insertENS_11__wrap_iterIPKwEEw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6insertEmPKw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6insertEmPKwm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6insertEmRKS5_mm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6insertEmmw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6resizeEmw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE7replaceEmmPKw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE7replaceEmmPKwm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE7replaceEmmRKS5_mm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE7replaceEmmmw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE7reserveEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE9__grow_byEmmmmmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE9push_backEw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEC1ERKS5_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEC1ERKS5_RKS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEC1ERKS5_mmRKS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEC2ERKS5_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEC2ERKS5_RKS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEC2ERKS5_mmRKS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEaSERKS5_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEaSEw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212ctype_bynameIcEC1EPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212ctype_bynameIcEC1ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212ctype_bynameIcEC2EPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212ctype_bynameIcEC2ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212ctype_bynameIcED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212ctype_bynameIcED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212ctype_bynameIcED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212ctype_bynameIwEC1EPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212ctype_bynameIwEC1ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212ctype_bynameIwEC2EPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212ctype_bynameIwEC2ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212ctype_bynameIwED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212ctype_bynameIwED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212ctype_bynameIwED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212future_errorC1ENS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212future_errorC2ENS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212future_errorD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212future_errorD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212future_errorD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212placeholders2_1E', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__212placeholders2_2E', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__212placeholders2_3E', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__212placeholders2_4E', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__212placeholders2_5E', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__212placeholders2_6E', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__212placeholders2_7E', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__212placeholders2_8E', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__212placeholders2_9E', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__212placeholders3_10E', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambuf3strEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambuf4swapERS0_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambuf6__initEPclS1_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambuf6freezeEb', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambuf7seekoffExNS_8ios_base7seekdirEj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambuf7seekposENS_4fposI11__mbstate_tEEj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambuf8overflowEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambuf9pbackfailEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambuf9underflowEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufC1EPFPvmEPFvS1_E', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufC1EPKal', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufC1EPKcl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufC1EPKhl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufC1EPalS1_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufC1EPclS1_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufC1EPhlS1_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufC1El', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufC2EPFPvmEPFvS1_E', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufC2EPKal', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufC2EPKcl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufC2EPKhl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufC2EPalS1_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufC2EPclS1_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufC2EPhlS1_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufC2El', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212strstreambufD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212system_error6__initERKNS_10error_codeENS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212system_errorC1ENS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212system_errorC1ENS_10error_codeEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212system_errorC1ENS_10error_codeERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212system_errorC1EiRKNS_14error_categoryE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212system_errorC1EiRKNS_14error_categoryEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212system_errorC1EiRKNS_14error_categoryERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212system_errorC2ENS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212system_errorC2ENS_10error_codeEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212system_errorC2ENS_10error_codeERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212system_errorC2EiRKNS_14error_categoryE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212system_errorC2EiRKNS_14error_categoryEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212system_errorC2EiRKNS_14error_categoryERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212system_errorD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212system_errorD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__212system_errorD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213allocator_argE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEE3getEPclc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEE3getERNS_15basic_streambufIcS2_EEc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEE3getEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEE4peekEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEE4readEPcl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEE4syncEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEE5seekgENS_4fposI11__mbstate_tEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEE5seekgExNS_8ios_base7seekdirE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEE5tellgEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEE5ungetEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEE6ignoreEli', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEE6sentryC1ERS3_b', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEE6sentryC2ERS3_b', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEE7getlineEPclc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEE7putbackEc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEE8readsomeEPcl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEErsEPNS_15basic_streambufIcS2_EE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEErsERPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEErsERb', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEErsERd', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEErsERe', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEErsERf', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEErsERi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEErsERj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEErsERl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEErsERm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEErsERs', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEErsERt', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEErsERx', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIcNS_11char_traitsIcEEErsERy', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEE3getEPwlw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEE3getERNS_15basic_streambufIwS2_EEw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEE3getEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEE4peekEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEE4readEPwl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEE4syncEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEE5seekgENS_4fposI11__mbstate_tEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEE5seekgExNS_8ios_base7seekdirE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEE5tellgEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEE5ungetEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEE6ignoreEli', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEE6sentryC1ERS3_b', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEE6sentryC2ERS3_b', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEE7getlineEPwlw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEE7putbackEw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEE8readsomeEPwl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEErsEPNS_15basic_streambufIwS2_EE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEErsERPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEErsERb', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEErsERd', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEErsERe', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEErsERf', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEErsERi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEErsERj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEErsERl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEErsERm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEErsERs', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEErsERt', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEErsERx', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_istreamIwNS_11char_traitsIwEEErsERy', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEE3putEc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEE5flushEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEE5writeEPKcl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEE6sentryC1ERS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEE6sentryC2ERS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEE6sentryD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEE6sentryD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEElsEPKv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEElsEPNS_15basic_streambufIcS2_EE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEElsEb', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEElsEd', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEElsEe', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEElsEf', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEElsEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEElsEj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEElsEl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEElsEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEElsEs', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEElsEt', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEElsEx', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIcNS_11char_traitsIcEEElsEy', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEE3putEw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEE5flushEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEE5writeEPKwl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEE6sentryC1ERS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEE6sentryC2ERS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEE6sentryD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEE6sentryD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEElsEPKv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEElsEPNS_15basic_streambufIwS2_EE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEElsEb', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEElsEd', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEElsEe', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEElsEf', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEElsEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEElsEj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEElsEl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEElsEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEElsEs', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEElsEt', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEElsEx', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213basic_ostreamIwNS_11char_traitsIwEEElsEy', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213random_deviceC1ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213random_deviceC2ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213random_deviceD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213random_deviceD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213random_deviceclEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213shared_futureIvED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213shared_futureIvED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__213shared_futureIvEaSERKS1_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214__get_const_dbEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214__num_get_base10__get_baseERNS_8ios_baseE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214__num_get_base5__srcE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__214__num_put_base12__format_intEPcPKcbj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214__num_put_base14__format_floatEPcPKcj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214__num_put_base18__identify_paddingEPcS1_RKNS_8ios_baseE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214__shared_countD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214__shared_countD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214__shared_countD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214basic_iostreamIcNS_11char_traitsIcEEED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214basic_iostreamIcNS_11char_traitsIcEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214basic_iostreamIcNS_11char_traitsIcEEED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214codecvt_bynameIDic11__mbstate_tED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214codecvt_bynameIDic11__mbstate_tED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214codecvt_bynameIDic11__mbstate_tED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214codecvt_bynameIDsc11__mbstate_tED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214codecvt_bynameIDsc11__mbstate_tED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214codecvt_bynameIDsc11__mbstate_tED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214codecvt_bynameIcc11__mbstate_tED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214codecvt_bynameIcc11__mbstate_tED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214codecvt_bynameIcc11__mbstate_tED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214codecvt_bynameIwc11__mbstate_tED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214codecvt_bynameIwc11__mbstate_tED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214codecvt_bynameIwc11__mbstate_tED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214collate_bynameIcEC1EPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214collate_bynameIcEC1ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214collate_bynameIcEC2EPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214collate_bynameIcEC2ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214collate_bynameIcED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214collate_bynameIcED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214collate_bynameIcED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214collate_bynameIwEC1EPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214collate_bynameIwEC1ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214collate_bynameIwEC2EPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214collate_bynameIwEC2ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214collate_bynameIwED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214collate_bynameIwED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214collate_bynameIwED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214error_categoryD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214error_categoryD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__214error_categoryD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215__get_classnameEPKcb', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215__thread_struct25notify_all_at_thread_exitEPNS_18condition_variableEPNS_5mutexE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215__thread_struct27__make_ready_at_thread_exitEPNS_17__assoc_sub_stateE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215__thread_structC1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215__thread_structC2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215__thread_structD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215__thread_structD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEE4swapERS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEE4syncEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEE5imbueERKNS_6localeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEE5uflowEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEE6setbufEPcl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEE6xsgetnEPcl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEE6xsputnEPKcl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEE7seekoffExNS_8ios_base7seekdirEj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEE7seekposENS_4fposI11__mbstate_tEEj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEE8overflowEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEE9pbackfailEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEE9showmanycEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEE9underflowEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEEC1ERKS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEEC1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEEC2ERKS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEEC2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIcNS_11char_traitsIcEEEaSERKS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEE4swapERS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEE4syncEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEE5imbueERKNS_6localeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEE5uflowEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEE6setbufEPwl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEE6xsgetnEPwl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEE6xsputnEPKwl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEE7seekoffExNS_8ios_base7seekdirEj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEE7seekposENS_4fposI11__mbstate_tEEj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEE8overflowEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEE9pbackfailEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEE9showmanycEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEE9underflowEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEEC1ERKS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEEC1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEEC2ERKS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEEC2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215basic_streambufIwNS_11char_traitsIwEEEaSERKS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215future_categoryEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215numpunct_bynameIcE6__initEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215numpunct_bynameIcEC1EPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215numpunct_bynameIcEC1ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215numpunct_bynameIcEC2EPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215numpunct_bynameIcEC2ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215numpunct_bynameIcED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215numpunct_bynameIcED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215numpunct_bynameIcED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215numpunct_bynameIwE6__initEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215numpunct_bynameIwEC1EPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215numpunct_bynameIwEC1ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215numpunct_bynameIwEC2EPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215numpunct_bynameIwEC2ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215numpunct_bynameIwED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215numpunct_bynameIwED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215numpunct_bynameIwED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215recursive_mutex4lockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215recursive_mutex6unlockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215recursive_mutex8try_lockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215recursive_mutexC1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215recursive_mutexC2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215recursive_mutexD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215recursive_mutexD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__215system_categoryEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__216__check_groupingERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPjS8_Rj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__216__narrow_to_utf8ILm16EED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__216__narrow_to_utf8ILm16EED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__216__narrow_to_utf8ILm16EED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__216__narrow_to_utf8ILm32EED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__216__narrow_to_utf8ILm32EED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__216__narrow_to_utf8ILm32EED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__216generic_categoryEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217__assoc_sub_state10__sub_waitERNS_11unique_lockINS_5mutexEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217__assoc_sub_state12__make_readyEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217__assoc_sub_state13set_exceptionESt13exception_ptr', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217__assoc_sub_state16__on_zero_sharedEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217__assoc_sub_state24set_value_at_thread_exitEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217__assoc_sub_state28set_exception_at_thread_exitESt13exception_ptr', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217__assoc_sub_state4copyEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217__assoc_sub_state4waitEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217__assoc_sub_state9__executeEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217__assoc_sub_state9set_valueEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217__widen_from_utf8ILm16EED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217__widen_from_utf8ILm16EED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217__widen_from_utf8ILm16EED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217__widen_from_utf8ILm32EED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217__widen_from_utf8ILm32EED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217__widen_from_utf8ILm32EED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217bad_function_callD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217bad_function_callD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217bad_function_callD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217declare_reachableEPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217iostream_categoryEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217moneypunct_bynameIcLb0EE4initEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217moneypunct_bynameIcLb1EE4initEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217moneypunct_bynameIwLb0EE4initEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__217moneypunct_bynameIwLb1EE4initEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218__time_get_storageIcE4initERKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218__time_get_storageIcE9__analyzeEcRKNS_5ctypeIcEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218__time_get_storageIcEC1EPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218__time_get_storageIcEC1ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218__time_get_storageIcEC2EPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218__time_get_storageIcEC2ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218__time_get_storageIwE4initERKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218__time_get_storageIwE9__analyzeEcRKNS_5ctypeIwEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218__time_get_storageIwEC1EPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218__time_get_storageIwEC1ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218__time_get_storageIwEC2EPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218__time_get_storageIwEC2ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218condition_variable10notify_allEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218condition_variable10notify_oneEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218condition_variable15__do_timed_waitERNS_11unique_lockINS_5mutexEEENS_6chrono10time_pointINS5_12system_clockENS5_8durationIxNS_5ratioILl1ELl1000000000EEEEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218condition_variable4waitERNS_11unique_lockINS_5mutexEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218condition_variableD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218condition_variableD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218shared_timed_mutex11lock_sharedEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218shared_timed_mutex13unlock_sharedEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218shared_timed_mutex15try_lock_sharedEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218shared_timed_mutex4lockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218shared_timed_mutex6unlockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218shared_timed_mutex8try_lockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218shared_timed_mutexC1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__218shared_timed_mutexC2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__219__shared_mutex_base11lock_sharedEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__219__shared_mutex_base13unlock_sharedEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__219__shared_mutex_base15try_lock_sharedEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__219__shared_mutex_base4lockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__219__shared_mutex_base6unlockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__219__shared_mutex_base8try_lockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__219__shared_mutex_baseC1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__219__shared_mutex_baseC2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__219__shared_weak_count14__release_weakEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__219__shared_weak_count4lockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__219__shared_weak_countD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__219__shared_weak_countD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__219__shared_weak_countD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__219__thread_local_dataEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__219declare_no_pointersEPcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__219piecewise_constructE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__220__get_collation_nameEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__220__throw_system_errorEiPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__221__throw_runtime_errorEPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__221__undeclare_reachableEPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__221recursive_timed_mutex4lockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__221recursive_timed_mutex6unlockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__221recursive_timed_mutex8try_lockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__221recursive_timed_mutexC1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__221recursive_timed_mutexC2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__221recursive_timed_mutexD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__221recursive_timed_mutexD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__221undeclare_no_pointersEPcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__223__libcpp_debug_functionE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__225notify_all_at_thread_exitERNS_18condition_variableENS_11unique_lockINS_5mutexEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__227__insertion_sort_incompleteIRNS_6__lessIaaEEPaEEbT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__227__insertion_sort_incompleteIRNS_6__lessIccEEPcEEbT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__227__insertion_sort_incompleteIRNS_6__lessIddEEPdEEbT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__227__insertion_sort_incompleteIRNS_6__lessIeeEEPeEEbT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__227__insertion_sort_incompleteIRNS_6__lessIffEEPfEEbT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__227__insertion_sort_incompleteIRNS_6__lessIhhEEPhEEbT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__227__insertion_sort_incompleteIRNS_6__lessIiiEEPiEEbT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__227__insertion_sort_incompleteIRNS_6__lessIjjEEPjEEbT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__227__insertion_sort_incompleteIRNS_6__lessIllEEPlEEbT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__227__insertion_sort_incompleteIRNS_6__lessImmEEPmEEbT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__227__insertion_sort_incompleteIRNS_6__lessIssEEPsEEbT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__227__insertion_sort_incompleteIRNS_6__lessIttEEPtEEbT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__227__insertion_sort_incompleteIRNS_6__lessIwwEEPwEEbT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__227__insertion_sort_incompleteIRNS_6__lessIxxEEPxEEbT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__227__insertion_sort_incompleteIRNS_6__lessIyyEEPyEEbT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__227__libcpp_set_debug_functionEPFvRKNS_19__libcpp_debug_infoEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__229__libcpp_abort_debug_functionERKNS_19__libcpp_debug_infoE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__23cinE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem10__absoluteERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem10hash_valueERKNS1_4pathE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem11__canonicalERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem11__copy_fileERKNS1_4pathES4_NS1_12copy_optionsEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem11__file_sizeERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem12__equivalentERKNS1_4pathES4_PNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem12__remove_allERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem13__fs_is_emptyERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem13__permissionsERKNS1_4pathENS1_5permsENS1_12perm_optionsEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem13__resize_fileERKNS1_4pathEmPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem14__copy_symlinkERKNS1_4pathES4_PNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem14__current_pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem14__current_pathERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem14__read_symlinkERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem15directory_entry12__do_refreshEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem16_FilesystemClock3nowEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem16_FilesystemClock9is_steadyE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem16__create_symlinkERKNS1_4pathES4_PNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem16__symlink_statusERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem16filesystem_error13__create_whatEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem16filesystem_errorD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem16filesystem_errorD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem16filesystem_errorD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem17__hard_link_countERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem17__last_write_timeERKNS1_4pathENS_6chrono10time_pointINS1_16_FilesystemClockENS5_8durationInNS_5ratioILl1ELl1000000000EEEEEEEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem17__last_write_timeERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem18__create_directoryERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem18__create_directoryERKNS1_4pathES4_PNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem18__create_hard_linkERKNS1_4pathES4_PNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem18__weakly_canonicalERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem18directory_iterator11__incrementEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem18directory_iteratorC1ERKNS1_4pathEPNS_10error_codeENS1_17directory_optionsE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem18directory_iteratorC2ERKNS1_4pathEPNS_10error_codeENS1_17directory_optionsE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem20__create_directoriesERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem21__temp_directory_pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem26__create_directory_symlinkERKNS1_4pathES4_PNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem28recursive_directory_iterator11__incrementEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem28recursive_directory_iterator15__try_recursionEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem28recursive_directory_iterator5__popEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem28recursive_directory_iterator9__advanceEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem28recursive_directory_iteratorC1ERKNS1_4pathENS1_17directory_optionsEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem28recursive_directory_iteratorC2ERKNS1_4pathENS1_17directory_optionsEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem4path17replace_extensionERKS2_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem4path19preferred_separatorE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem4path8iterator11__decrementEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem4path8iterator11__incrementEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem6__copyERKNS1_4pathES4_NS1_12copy_optionsEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem7__spaceERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem8__removeERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem8__renameERKNS1_4pathES4_PNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24__fs10filesystem8__statusERKNS1_4pathEPNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24cerrE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__24clogE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__24coutE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__24stodERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24stodERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24stofERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24stofERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24stoiERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24stoiERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24stolERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24stolERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__24wcinE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__25alignEmmRPvRm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25ctypeIcE13classic_tableEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25ctypeIcE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__25ctypeIcEC1EPKjbm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25ctypeIcEC2EPKjbm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25ctypeIcED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25ctypeIcED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25ctypeIcED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25ctypeIwE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__25ctypeIwED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25ctypeIwED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25ctypeIwED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25mutex4lockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25mutex6unlockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25mutex8try_lockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25mutexD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25mutexD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25stoldERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25stoldERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25stollERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25stollERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25stoulERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25stoulERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__25wcerrE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__25wclogE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__25wcoutE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__26__itoa8__u32toaEjPc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26__itoa8__u64toaEyPc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26__sortIRNS_6__lessIaaEEPaEEvT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26__sortIRNS_6__lessIccEEPcEEvT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26__sortIRNS_6__lessIddEEPdEEvT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26__sortIRNS_6__lessIeeEEPeEEvT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26__sortIRNS_6__lessIffEEPfEEvT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26__sortIRNS_6__lessIhhEEPhEEvT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26__sortIRNS_6__lessIiiEEPiEEvT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26__sortIRNS_6__lessIjjEEPjEEvT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26__sortIRNS_6__lessIllEEPlEEvT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26__sortIRNS_6__lessImmEEPmEEvT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26__sortIRNS_6__lessIssEEPsEEvT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26__sortIRNS_6__lessIttEEPtEEvT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26__sortIRNS_6__lessIwwEEPwEEvT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26__sortIRNS_6__lessIxxEEPxEEvT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26__sortIRNS_6__lessIyyEEPyEEvT0_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26chrono12steady_clock3nowEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26chrono12steady_clock9is_steadyE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__26chrono12system_clock11from_time_tEl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26chrono12system_clock3nowEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26chrono12system_clock9is_steadyE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__26chrono12system_clock9to_time_tERKNS0_10time_pointIS1_NS0_8durationIxNS_5ratioILl1ELl1000000EEEEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26futureIvE3getEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26futureIvEC1EPNS_17__assoc_sub_stateE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26futureIvEC2EPNS_17__assoc_sub_stateE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26futureIvED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26futureIvED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26gslice6__initEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26locale14__install_ctorERKS0_PNS0_5facetEl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26locale2id5__getEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26locale2id6__initEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26locale2id9__next_idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__26locale3allE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__26locale4noneE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__26locale4timeE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__26locale5ctypeE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__26locale5facet16__on_zero_sharedEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26locale5facetD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26locale5facetD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26locale5facetD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26locale6globalERKS0_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26locale7classicEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26locale7collateE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__26locale7numericE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__26locale8__globalEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26locale8messagesE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__26locale8monetaryE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__26localeC1EPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26localeC1ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26localeC1ERKS0_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26localeC1ERKS0_PKci', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26localeC1ERKS0_RKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26localeC1ERKS0_S2_i', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26localeC1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26localeC2EPKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26localeC2ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26localeC2ERKS0_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26localeC2ERKS0_PKci', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26localeC2ERKS0_RKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26localeC2ERKS0_S2_i', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26localeC2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26localeD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26localeD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26localeaSERKS0_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26stoullERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26stoullERKNS_12basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEEEPmi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26thread20hardware_concurrencyEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26thread4joinEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26thread6detachEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26threadD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__26threadD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27__sort5IRNS_6__lessIeeEEPeEEjT0_S5_S5_S5_S5_T_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIDic11__mbstate_tE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIDic11__mbstate_tED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIDic11__mbstate_tED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIDic11__mbstate_tED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIDsc11__mbstate_tE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIDsc11__mbstate_tED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIDsc11__mbstate_tED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIDsc11__mbstate_tED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIcc11__mbstate_tE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIcc11__mbstate_tED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIcc11__mbstate_tED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIcc11__mbstate_tED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIwc11__mbstate_tE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIwc11__mbstate_tEC1EPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIwc11__mbstate_tEC1Em', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIwc11__mbstate_tEC2EPKcm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIwc11__mbstate_tEC2Em', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIwc11__mbstate_tED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIwc11__mbstate_tED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27codecvtIwc11__mbstate_tED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27collateIcE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__27collateIcED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27collateIcED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27collateIcED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27collateIwE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__27collateIwED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27collateIwED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27collateIwED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__27num_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__27num_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__27num_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__27promiseIvE10get_futureEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27promiseIvE13set_exceptionESt13exception_ptr', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27promiseIvE24set_value_at_thread_exitEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27promiseIvE28set_exception_at_thread_exitESt13exception_ptr', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27promiseIvE9set_valueEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27promiseIvEC1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27promiseIvEC2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27promiseIvED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__27promiseIvED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28__c_node5__addEPNS_8__i_nodeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28__c_nodeD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28__c_nodeD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28__c_nodeD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28__get_dbEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28__i_nodeD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28__i_nodeD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28__rs_getEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28__sp_mut4lockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28__sp_mut6unlockEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base10floatfieldE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base10scientificE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base11adjustfieldE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base15sync_with_stdioEb', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base16__call_callbacksENS0_5eventE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base17register_callbackEPFvNS0_5eventERS0_iEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base2inE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base33__set_badbit_and_consider_rethrowEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base34__set_failbit_and_consider_rethrowEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base3appE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base3ateE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base3decE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base3hexE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base3octE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base3outE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base4InitC1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base4InitC2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base4InitD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base4InitD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base4initEPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base4leftE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base4moveERS0_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base4swapERS0_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base5clearEj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base5fixedE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base5imbueERKNS_6localeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base5iwordEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base5pwordEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base5rightE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base5truncE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base6badbitE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base6binaryE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base6eofbitE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base6skipwsE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base6xallocEv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base7copyfmtERKS0_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base7failbitE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base7failureC1EPKcRKNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base7failureC1ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEERKNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base7failureC2EPKcRKNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base7failureC2ERKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEERKNS_10error_codeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base7failureD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base7failureD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base7failureD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base7goodbitE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base7showposE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base7unitbufE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base8internalE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base8showbaseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base9__xindex_E', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base9basefieldE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base9boolalphaE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base9showpointE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_base9uppercaseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_baseD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_baseD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28ios_baseD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28messagesIcE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28messagesIwE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28numpunctIcE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28numpunctIcEC1Em', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28numpunctIcEC2Em', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28numpunctIcED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28numpunctIcED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28numpunctIcED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28numpunctIwE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28numpunctIwEC1Em', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28numpunctIwEC2Em', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28numpunctIwED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28numpunctIwED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28numpunctIwED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28time_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28time_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__28valarrayImE6resizeEmm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29__num_getIcE17__stage2_int_loopEciPcRS2_RjcRKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPjRSD_PKc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29__num_getIcE17__stage2_int_prepERNS_8ios_baseERc', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29__num_getIcE19__stage2_float_loopEcRbRcPcRS4_ccRKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPjRSE_RjS4_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29__num_getIcE19__stage2_float_prepERNS_8ios_baseEPcRcS5_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29__num_getIwE17__stage2_int_loopEwiPcRS2_RjwRKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPjRSD_PKw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29__num_getIwE17__stage2_int_prepERNS_8ios_baseERw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29__num_getIwE19__stage2_float_loopEwRbRcPcRS4_wwRKNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPjRSE_RjPw', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29__num_getIwE19__stage2_float_prepERNS_8ios_baseEPwRwS5_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29__num_putIcE21__widen_and_group_intEPcS2_S2_S2_RS2_S3_RKNS_6localeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29__num_putIcE23__widen_and_group_floatEPcS2_S2_S2_RS2_S3_RKNS_6localeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29__num_putIwE21__widen_and_group_intEPcS2_S2_PwRS3_S4_RKNS_6localeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29__num_putIwE23__widen_and_group_floatEPcS2_S2_PwRS3_S4_RKNS_6localeE', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29basic_iosIcNS_11char_traitsIcEEE7copyfmtERKS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29basic_iosIcNS_11char_traitsIcEEED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29basic_iosIcNS_11char_traitsIcEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29basic_iosIcNS_11char_traitsIcEEED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29basic_iosIwNS_11char_traitsIwEEE7copyfmtERKS3_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29basic_iosIwNS_11char_traitsIwEEED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29basic_iosIwNS_11char_traitsIwEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29basic_iosIwNS_11char_traitsIwEEED2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29money_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__29money_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEE8__do_getERS4_S4_bRKNS_6localeEjRjRbRKNS_5ctypeIcEERNS_10unique_ptrIcPFvPvEEERPcSM_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29money_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__29money_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEE8__do_getERS4_S4_bRKNS_6localeEjRjRbRKNS_5ctypeIwEERNS_10unique_ptrIwPFvPvEEERPwSM_', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29money_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__29money_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEE2idE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZNSt3__29strstreamD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29strstreamD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29strstreamD2Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29to_stringEd', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29to_stringEe', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29to_stringEf', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29to_stringEi', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29to_stringEj', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29to_stringEl', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29to_stringEm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29to_stringEx', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__29to_stringEy', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZNSt3__2plIcNS_11char_traitsIcEENS_9allocatorIcEEEENS_12basic_stringIT_T0_T1_EEPKS6_RKS9_', 'type': 'FUNC'}
-{'is_defined': False, 'name': '__ZNSt8bad_castC1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt8bad_castC1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt8bad_castC2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt8bad_castC2Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt8bad_castD0Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt8bad_castD0Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt8bad_castD1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt8bad_castD1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt8bad_castD2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt8bad_castD2Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt9bad_allocC1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt9bad_allocC1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt9bad_allocC2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt9bad_allocC2Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt9bad_allocD0Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt9bad_allocD0Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt9bad_allocD1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt9bad_allocD1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt9bad_allocD2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt9bad_allocD2Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt9exceptionD0Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt9exceptionD0Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt9exceptionD1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt9exceptionD1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt9exceptionD2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt9exceptionD2Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt9type_infoD0Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt9type_infoD0Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt9type_infoD1Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt9type_infoD1Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZNSt9type_infoD2Ev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZNSt9type_infoD2Ev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZSt10unexpectedv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZSt10unexpectedv', 'type': 'I'}
-{'is_defined': False, 'name': '__ZSt13get_terminatev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZSt13get_terminatev', 'type': 'I'}
-{'is_defined': False, 'name': '__ZSt13set_terminatePFvvE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZSt13set_terminatePFvvE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZSt14get_unexpectedv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZSt14get_unexpectedv', 'type': 'I'}
-{'is_defined': False, 'name': '__ZSt14set_unexpectedPFvvE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZSt14set_unexpectedPFvvE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZSt15get_new_handlerv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZSt15get_new_handlerv', 'type': 'I'}
-{'is_defined': False, 'name': '__ZSt15set_new_handlerPFvvE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZSt15set_new_handlerPFvvE', 'type': 'I'}
-{'is_defined': True, 'name': '__ZSt17__throw_bad_allocv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZSt17current_exceptionv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZSt17rethrow_exceptionSt13exception_ptr', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZSt18uncaught_exceptionv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZSt19uncaught_exceptionsv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZSt7nothrow', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': False, 'name': '__ZSt9terminatev', 'type': 'U'}
-{'is_defined': True, 'name': '__ZSt9terminatev', 'type': 'I'}
-{'is_defined': True, 'name': '__ZTCNSt3__210istrstreamE0_NS_13basic_istreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTCNSt3__210ostrstreamE0_NS_13basic_ostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTCNSt3__214basic_iostreamIcNS_11char_traitsIcEEEE0_NS_13basic_istreamIcS2_EE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTCNSt3__214basic_iostreamIcNS_11char_traitsIcEEEE16_NS_13basic_ostreamIcS2_EE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTCNSt3__29strstreamE0_NS_13basic_istreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTCNSt3__29strstreamE0_NS_14basic_iostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTCNSt3__29strstreamE16_NS_13basic_ostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': False, 'name': '__ZTIDi', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIDi', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIDn', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIDn', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIDs', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIDs', 'type': 'I'}
-{'is_defined': True, 'name': '__ZTINSt12experimental15fundamentals_v112bad_any_castE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt12experimental19bad_optional_accessE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__210__time_getE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__210__time_putE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__210ctype_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__210istrstreamE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__210money_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__210moneypunctIcLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__210moneypunctIcLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__210moneypunctIwLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__210moneypunctIwLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__210ostrstreamE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__211__money_getIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__211__money_getIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__211__money_putIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__211__money_putIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__211regex_errorE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__212bad_weak_ptrE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__212codecvt_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__212ctype_bynameIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__212ctype_bynameIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__212future_errorE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__212strstreambufE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__212system_errorE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__213basic_istreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__213basic_istreamIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__213basic_ostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__213basic_ostreamIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__213messages_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__214__codecvt_utf8IDiEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__214__codecvt_utf8IDsEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__214__codecvt_utf8IwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__214__num_get_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__214__num_put_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__214__shared_countE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__214basic_iostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__214codecvt_bynameIDic11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__214codecvt_bynameIDsc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__214codecvt_bynameIcc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__214codecvt_bynameIwc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__214collate_bynameIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__214collate_bynameIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__214error_categoryE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__215__codecvt_utf16IDiLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__215__codecvt_utf16IDiLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__215__codecvt_utf16IDsLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__215__codecvt_utf16IDsLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__215__codecvt_utf16IwLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__215__codecvt_utf16IwLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__215basic_streambufIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__215basic_streambufIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__215messages_bynameIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__215messages_bynameIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__215numpunct_bynameIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__215numpunct_bynameIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__215time_get_bynameIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__215time_get_bynameIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__215time_put_bynameIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__215time_put_bynameIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__216__narrow_to_utf8ILm16EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__216__narrow_to_utf8ILm32EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__217__assoc_sub_stateE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__217__widen_from_utf8ILm16EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__217__widen_from_utf8ILm32EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__217bad_function_callE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__217moneypunct_bynameIcLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__217moneypunct_bynameIcLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__217moneypunct_bynameIwLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__217moneypunct_bynameIwLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__218__time_get_storageIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__218__time_get_storageIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__219__shared_weak_countE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__220__codecvt_utf8_utf16IDiEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__220__codecvt_utf8_utf16IDsEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__220__codecvt_utf8_utf16IwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__220__time_get_c_storageIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__220__time_get_c_storageIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__24__fs10filesystem16filesystem_errorE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__25ctypeIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__25ctypeIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__26locale5facetE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__27codecvtIDic11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__27codecvtIDsc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__27codecvtIcc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__27codecvtIwc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__27collateIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__27collateIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__27num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__27num_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__27num_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__27num_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__28__c_nodeE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__28ios_base7failureE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__28ios_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__28messagesIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__28messagesIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__28numpunctIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__28numpunctIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__28time_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__28time_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__29__num_getIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__29__num_getIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__29__num_putIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__29__num_putIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__29basic_iosIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__29basic_iosIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__29money_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__29money_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__29money_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__29money_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__29strstreamE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTINSt3__29time_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': False, 'name': '__ZTIPDi', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPDi', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPDn', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPDn', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPDs', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPDs', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKDi', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKDi', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKDn', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKDn', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKDs', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKDs', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKa', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKa', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKb', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKb', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKc', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKc', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKd', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKd', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKe', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKe', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKf', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKf', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKh', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKh', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKi', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKi', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKj', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKj', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKl', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKl', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKm', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKm', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKs', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKs', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKt', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKt', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKv', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKw', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKw', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKx', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKx', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPKy', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPKy', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPa', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPa', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPb', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPb', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPc', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPc', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPd', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPd', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPe', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPe', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPf', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPf', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPh', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPh', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPi', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPi', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPj', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPj', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPl', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPl', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPm', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPm', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPs', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPs', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPt', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPt', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPv', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPw', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPw', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPx', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPx', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIPy', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIPy', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTISt10bad_typeid', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTISt10bad_typeid', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTISt11logic_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTISt11logic_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTISt11range_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTISt11range_error', 'type': 'I'}
-{'is_defined': True, 'name': '__ZTISt12bad_any_cast', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': False, 'name': '__ZTISt12domain_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTISt12domain_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTISt12length_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTISt12length_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTISt12out_of_range', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTISt12out_of_range', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTISt13bad_exception', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTISt13bad_exception', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTISt13runtime_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTISt13runtime_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTISt14overflow_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTISt14overflow_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTISt15underflow_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTISt15underflow_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTISt16invalid_argument', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTISt16invalid_argument', 'type': 'I'}
-{'is_defined': True, 'name': '__ZTISt16nested_exception', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTISt18bad_variant_access', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTISt19bad_optional_access', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': False, 'name': '__ZTISt20bad_array_new_length', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTISt20bad_array_new_length', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTISt8bad_cast', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTISt8bad_cast', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTISt9bad_alloc', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTISt9bad_alloc', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTISt9exception', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTISt9exception', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTISt9type_info', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTISt9type_info', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIa', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIa', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIb', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIb', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIc', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIc', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTId', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTId', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIe', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIe', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIf', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIf', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIh', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIh', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIi', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIi', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIj', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIj', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIl', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIl', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIm', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIm', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIs', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIs', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIt', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIt', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIv', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIw', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIw', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIx', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIx', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTIy', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTIy', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSDi', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSDi', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSDn', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSDn', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSDs', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSDs', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSN10__cxxabiv116__enum_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSN10__cxxabiv116__enum_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSN10__cxxabiv117__array_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSN10__cxxabiv117__array_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSN10__cxxabiv117__class_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSN10__cxxabiv117__class_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSN10__cxxabiv117__pbase_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSN10__cxxabiv117__pbase_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSN10__cxxabiv119__pointer_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSN10__cxxabiv119__pointer_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSN10__cxxabiv120__function_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSN10__cxxabiv120__function_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSN10__cxxabiv120__si_class_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSN10__cxxabiv120__si_class_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSN10__cxxabiv121__vmi_class_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSN10__cxxabiv121__vmi_class_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSN10__cxxabiv123__fundamental_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSN10__cxxabiv123__fundamental_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSN10__cxxabiv129__pointer_to_member_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSN10__cxxabiv129__pointer_to_member_type_infoE', 'type': 'I'}
-{'is_defined': True, 'name': '__ZTSNSt12experimental15fundamentals_v112bad_any_castE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt12experimental19bad_optional_accessE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__210__time_getE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__210__time_putE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__210ctype_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__210istrstreamE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__210money_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__210moneypunctIcLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__210moneypunctIcLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__210moneypunctIwLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__210moneypunctIwLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__210ostrstreamE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__211__money_getIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__211__money_getIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__211__money_putIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__211__money_putIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__211regex_errorE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__212bad_weak_ptrE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__212codecvt_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__212ctype_bynameIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__212ctype_bynameIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__212future_errorE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__212strstreambufE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__212system_errorE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__213basic_istreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__213basic_istreamIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__213basic_ostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__213basic_ostreamIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__213messages_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__214__codecvt_utf8IDiEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__214__codecvt_utf8IDsEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__214__codecvt_utf8IwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__214__num_get_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__214__num_put_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__214__shared_countE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__214basic_iostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__214codecvt_bynameIDic11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__214codecvt_bynameIDsc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__214codecvt_bynameIcc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__214codecvt_bynameIwc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__214collate_bynameIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__214collate_bynameIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__214error_categoryE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__215__codecvt_utf16IDiLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__215__codecvt_utf16IDiLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__215__codecvt_utf16IDsLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__215__codecvt_utf16IDsLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__215__codecvt_utf16IwLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__215__codecvt_utf16IwLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__215basic_streambufIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__215basic_streambufIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__215messages_bynameIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__215messages_bynameIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__215numpunct_bynameIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__215numpunct_bynameIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__215time_get_bynameIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__215time_get_bynameIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__215time_put_bynameIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__215time_put_bynameIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__216__narrow_to_utf8ILm16EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__216__narrow_to_utf8ILm32EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__217__assoc_sub_stateE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__217__widen_from_utf8ILm16EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__217__widen_from_utf8ILm32EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__217bad_function_callE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__217moneypunct_bynameIcLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__217moneypunct_bynameIcLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__217moneypunct_bynameIwLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__217moneypunct_bynameIwLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__218__time_get_storageIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__218__time_get_storageIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__219__shared_weak_countE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__220__codecvt_utf8_utf16IDiEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__220__codecvt_utf8_utf16IDsEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__220__codecvt_utf8_utf16IwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__220__time_get_c_storageIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__220__time_get_c_storageIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__24__fs10filesystem16filesystem_errorE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__25ctypeIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__25ctypeIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__26locale5facetE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__27codecvtIDic11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__27codecvtIDsc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__27codecvtIcc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__27codecvtIwc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__27collateIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__27collateIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__27num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__27num_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__27num_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__27num_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__28__c_nodeE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__28ios_base7failureE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__28ios_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__28messagesIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__28messagesIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__28numpunctIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__28numpunctIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__28time_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__28time_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__29__num_getIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__29__num_getIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__29__num_putIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__29__num_putIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__29basic_iosIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__29basic_iosIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__29money_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__29money_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__29money_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__29money_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__29strstreamE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSNSt3__29time_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': False, 'name': '__ZTSPDi', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPDi', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPDn', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPDn', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPDs', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPDs', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKDi', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKDi', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKDn', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKDn', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKDs', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKDs', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKa', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKa', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKb', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKb', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKc', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKc', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKd', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKd', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKe', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKe', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKf', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKf', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKh', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKh', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKi', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKi', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKj', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKj', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKl', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKl', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKm', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKm', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKs', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKs', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKt', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKt', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKv', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKw', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKw', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKx', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKx', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPKy', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPKy', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPa', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPa', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPb', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPb', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPc', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPc', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPd', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPd', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPe', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPe', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPf', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPf', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPh', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPh', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPi', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPi', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPj', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPj', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPl', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPl', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPm', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPm', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPs', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPs', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPt', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPt', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPv', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPw', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPw', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPx', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPx', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSPy', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSPy', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSSt10bad_typeid', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSSt10bad_typeid', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSSt11logic_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSSt11logic_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSSt11range_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSSt11range_error', 'type': 'I'}
-{'is_defined': True, 'name': '__ZTSSt12bad_any_cast', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': False, 'name': '__ZTSSt12domain_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSSt12domain_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSSt12length_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSSt12length_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSSt12out_of_range', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSSt12out_of_range', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSSt13bad_exception', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSSt13bad_exception', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSSt13runtime_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSSt13runtime_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSSt14overflow_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSSt14overflow_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSSt15underflow_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSSt15underflow_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSSt16invalid_argument', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSSt16invalid_argument', 'type': 'I'}
-{'is_defined': True, 'name': '__ZTSSt16nested_exception', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSSt18bad_variant_access', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTSSt19bad_optional_access', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': False, 'name': '__ZTSSt20bad_array_new_length', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSSt20bad_array_new_length', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSSt8bad_cast', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSSt8bad_cast', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSSt9bad_alloc', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSSt9bad_alloc', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSSt9exception', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSSt9exception', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSSt9type_info', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSSt9type_info', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSa', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSa', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSb', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSb', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSc', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSc', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSd', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSd', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSe', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSe', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSf', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSf', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSh', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSh', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSi', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSi', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSj', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSj', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSl', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSl', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSm', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSm', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSs', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSs', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSt', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSt', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSv', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSv', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSw', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSw', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSx', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSx', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTSy', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTSy', 'type': 'I'}
-{'is_defined': True, 'name': '__ZTTNSt3__210istrstreamE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTTNSt3__210ostrstreamE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTTNSt3__213basic_istreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTTNSt3__213basic_istreamIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTTNSt3__213basic_ostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTTNSt3__213basic_ostreamIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTTNSt3__214basic_iostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTTNSt3__29strstreamE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': False, 'name': '__ZTVN10__cxxabiv116__enum_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVN10__cxxabiv116__enum_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVN10__cxxabiv117__array_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVN10__cxxabiv117__array_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVN10__cxxabiv117__class_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVN10__cxxabiv117__class_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVN10__cxxabiv117__pbase_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVN10__cxxabiv117__pbase_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVN10__cxxabiv119__pointer_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVN10__cxxabiv119__pointer_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVN10__cxxabiv120__function_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVN10__cxxabiv120__function_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVN10__cxxabiv120__si_class_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVN10__cxxabiv120__si_class_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVN10__cxxabiv121__vmi_class_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVN10__cxxabiv121__vmi_class_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVN10__cxxabiv123__fundamental_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVN10__cxxabiv123__fundamental_type_infoE', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVN10__cxxabiv129__pointer_to_member_type_infoE', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVN10__cxxabiv129__pointer_to_member_type_infoE', 'type': 'I'}
-{'is_defined': True, 'name': '__ZTVNSt12experimental15fundamentals_v112bad_any_castE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt12experimental19bad_optional_accessE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__210istrstreamE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__210moneypunctIcLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__210moneypunctIcLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__210moneypunctIwLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__210moneypunctIwLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__210ostrstreamE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__211regex_errorE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__212bad_weak_ptrE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__212ctype_bynameIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__212ctype_bynameIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__212future_errorE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__212strstreambufE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__212system_errorE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__213basic_istreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__213basic_istreamIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__213basic_ostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__213basic_ostreamIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__214__codecvt_utf8IDiEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__214__codecvt_utf8IDsEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__214__codecvt_utf8IwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__214__shared_countE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__214basic_iostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__214codecvt_bynameIDic11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__214codecvt_bynameIDsc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__214codecvt_bynameIcc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__214codecvt_bynameIwc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__214collate_bynameIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__214collate_bynameIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__214error_categoryE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__215__codecvt_utf16IDiLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__215__codecvt_utf16IDiLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__215__codecvt_utf16IDsLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__215__codecvt_utf16IDsLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__215__codecvt_utf16IwLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__215__codecvt_utf16IwLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__215basic_streambufIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__215basic_streambufIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__215messages_bynameIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__215messages_bynameIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__215numpunct_bynameIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__215numpunct_bynameIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__215time_get_bynameIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__215time_get_bynameIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__215time_put_bynameIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__215time_put_bynameIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__216__narrow_to_utf8ILm16EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__216__narrow_to_utf8ILm32EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__217__assoc_sub_stateE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__217__widen_from_utf8ILm16EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__217__widen_from_utf8ILm32EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__217bad_function_callE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__217moneypunct_bynameIcLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__217moneypunct_bynameIcLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__217moneypunct_bynameIwLb0EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__217moneypunct_bynameIwLb1EEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__219__shared_weak_countE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__220__codecvt_utf8_utf16IDiEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__220__codecvt_utf8_utf16IDsEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__220__codecvt_utf8_utf16IwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__24__fs10filesystem16filesystem_errorE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__25ctypeIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__25ctypeIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__26locale5facetE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__27codecvtIDic11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__27codecvtIDsc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__27codecvtIcc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__27codecvtIwc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__27collateIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__27collateIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__27num_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__27num_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__27num_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__27num_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__28__c_nodeE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__28ios_base7failureE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__28ios_baseE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__28messagesIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__28messagesIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__28numpunctIcEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__28numpunctIwEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__28time_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__28time_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__28time_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__28time_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__29basic_iosIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__29basic_iosIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__29money_getIcNS_19istreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__29money_getIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__29money_putIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__29money_putIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVNSt3__29strstreamE', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': False, 'name': '__ZTVSt10bad_typeid', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVSt10bad_typeid', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVSt11logic_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVSt11logic_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVSt11range_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVSt11range_error', 'type': 'I'}
-{'is_defined': True, 'name': '__ZTVSt12bad_any_cast', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': False, 'name': '__ZTVSt12domain_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVSt12domain_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVSt12length_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVSt12length_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVSt12out_of_range', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVSt12out_of_range', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVSt13bad_exception', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVSt13bad_exception', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVSt13runtime_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVSt13runtime_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVSt14overflow_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVSt14overflow_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVSt15underflow_error', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVSt15underflow_error', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVSt16invalid_argument', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVSt16invalid_argument', 'type': 'I'}
-{'is_defined': True, 'name': '__ZTVSt16nested_exception', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVSt18bad_variant_access', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': True, 'name': '__ZTVSt19bad_optional_access', 'size': 0, 'type': 'OBJECT'}
-{'is_defined': False, 'name': '__ZTVSt20bad_array_new_length', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVSt20bad_array_new_length', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVSt8bad_cast', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVSt8bad_cast', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVSt9bad_alloc', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVSt9bad_alloc', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVSt9exception', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVSt9exception', 'type': 'I'}
-{'is_defined': False, 'name': '__ZTVSt9type_info', 'type': 'U'}
-{'is_defined': True, 'name': '__ZTVSt9type_info', 'type': 'I'}
-{'is_defined': True, 'name': '__ZThn16_NSt3__214basic_iostreamIcNS_11char_traitsIcEEED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZThn16_NSt3__214basic_iostreamIcNS_11char_traitsIcEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZThn16_NSt3__29strstreamD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZThn16_NSt3__29strstreamD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZTv0_n24_NSt3__210istrstreamD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZTv0_n24_NSt3__210istrstreamD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZTv0_n24_NSt3__210ostrstreamD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZTv0_n24_NSt3__210ostrstreamD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZTv0_n24_NSt3__213basic_istreamIcNS_11char_traitsIcEEED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZTv0_n24_NSt3__213basic_istreamIcNS_11char_traitsIcEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZTv0_n24_NSt3__213basic_istreamIwNS_11char_traitsIwEEED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZTv0_n24_NSt3__213basic_istreamIwNS_11char_traitsIwEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZTv0_n24_NSt3__213basic_ostreamIcNS_11char_traitsIcEEED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZTv0_n24_NSt3__213basic_ostreamIcNS_11char_traitsIcEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZTv0_n24_NSt3__213basic_ostreamIwNS_11char_traitsIwEEED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZTv0_n24_NSt3__213basic_ostreamIwNS_11char_traitsIwEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZTv0_n24_NSt3__214basic_iostreamIcNS_11char_traitsIcEEED0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZTv0_n24_NSt3__214basic_iostreamIcNS_11char_traitsIcEEED1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZTv0_n24_NSt3__29strstreamD0Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZTv0_n24_NSt3__29strstreamD1Ev', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZdaPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZdaPvRKSt9nothrow_t', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZdaPvSt11align_val_t', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZdaPvSt11align_val_tRKSt9nothrow_t', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZdaPvm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZdaPvmSt11align_val_t', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZdlPv', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZdlPvRKSt9nothrow_t', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZdlPvSt11align_val_t', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZdlPvSt11align_val_tRKSt9nothrow_t', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZdlPvm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZdlPvmSt11align_val_t', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__Znam', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZnamRKSt9nothrow_t', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZnamSt11align_val_t', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZnamSt11align_val_tRKSt9nothrow_t', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__Znwm', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZnwmRKSt9nothrow_t', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZnwmSt11align_val_t', 'type': 'FUNC'}
-{'is_defined': True, 'name': '__ZnwmSt11align_val_tRKSt9nothrow_t', 'type': 'FUNC'}
-{'is_defined': False, 'name': '___cxa_allocate_exception', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_allocate_exception', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_atexit', 'type': 'U'}
-{'is_defined': False, 'name': '___cxa_bad_cast', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_bad_cast', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_bad_typeid', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_bad_typeid', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_begin_catch', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_begin_catch', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_call_unexpected', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_call_unexpected', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_current_exception_type', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_current_exception_type', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_current_primary_exception', 'type': 'U'}
-{'is_defined': False, 'name': '___cxa_decrement_exception_refcount', 'type': 'U'}
-{'is_defined': False, 'name': '___cxa_deleted_virtual', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_deleted_virtual', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_demangle', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_demangle', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_end_catch', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_end_catch', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_free_exception', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_free_exception', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_get_exception_ptr', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_get_exception_ptr', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_get_globals', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_get_globals', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_get_globals_fast', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_get_globals_fast', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_guard_abort', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_guard_abort', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_guard_acquire', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_guard_acquire', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_guard_release', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_guard_release', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_increment_exception_refcount', 'type': 'U'}
-{'is_defined': False, 'name': '___cxa_pure_virtual', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_pure_virtual', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_rethrow', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_rethrow', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_rethrow_primary_exception', 'type': 'U'}
-{'is_defined': False, 'name': '___cxa_throw', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_throw', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_throw_bad_array_new_length', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_throw_bad_array_new_length', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_uncaught_exceptions', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_uncaught_exceptions', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_vec_cctor', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_vec_cctor', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_vec_cleanup', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_vec_cleanup', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_vec_ctor', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_vec_ctor', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_vec_delete', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_vec_delete', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_vec_delete2', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_vec_delete2', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_vec_delete3', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_vec_delete3', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_vec_dtor', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_vec_dtor', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_vec_new', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_vec_new', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_vec_new2', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_vec_new2', 'type': 'I'}
-{'is_defined': False, 'name': '___cxa_vec_new3', 'type': 'U'}
-{'is_defined': True, 'name': '___cxa_vec_new3', 'type': 'I'}
-{'is_defined': False, 'name': '___dynamic_cast', 'type': 'U'}
-{'is_defined': True, 'name': '___dynamic_cast', 'type': 'I'}
-{'is_defined': False, 'name': '___gxx_personality_v0', 'type': 'U'}
-{'is_defined': True, 'name': '___gxx_personality_v0', 'type': 'I'}
-{'type': 'FUNC', 'is_defined': True, 'name': '__ZNSt3__220__libcpp_atomic_waitEPVKNS_17__cxx_atomic_implIxNS_22__cxx_atomic_base_implIxEEEEx'}
-{'type': 'FUNC', 'is_defined': True, 'name': '__ZNSt3__220__libcpp_atomic_waitEPVKvx'}
-{'type': 'FUNC', 'is_defined': True, 'name': '__ZNSt3__223__cxx_atomic_notify_allEPVKNS_17__cxx_atomic_implIxNS_22__cxx_atomic_base_implIxEEEE'}
-{'type': 'FUNC', 'is_defined': True, 'name': '__ZNSt3__223__cxx_atomic_notify_allEPVKv'}
-{'type': 'FUNC', 'is_defined': True, 'name': '__ZNSt3__223__cxx_atomic_notify_oneEPVKNS_17__cxx_atomic_implIxNS_22__cxx_atomic_base_implIxEEEE'}
-{'type': 'FUNC', 'is_defined': True, 'name': '__ZNSt3__223__cxx_atomic_notify_oneEPVKv'}
-{'type': 'FUNC', 'is_defined': True, 'name': '__ZNSt3__223__libcpp_atomic_monitorEPVKNS_17__cxx_atomic_implIxNS_22__cxx_atomic_base_implIxEEEE'}
-{'type': 'FUNC', 'is_defined': True, 'name': '__ZNSt3__223__libcpp_atomic_monitorEPVKv'}
-{'type': 'FUNC', 'is_defined': True, 'name': '__ZNSt3__231__arrive_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseEh'}
-{'type': 'FUNC', 'is_defined': True, 'name': '__ZNSt3__232__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE'}
-{'type': 'FUNC', 'is_defined': True, 'name': '__ZNSt3__234__construct_barrier_algorithm_baseERl'}
\ No newline at end of file
diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt
index 898ec027f3265..9c2db48b66b7f 100644
--- a/libcxx/src/CMakeLists.txt
+++ b/libcxx/src/CMakeLists.txt
@@ -29,6 +29,7 @@ set(LIBCXX_SOURCES
   new.cpp
   optional.cpp
   random.cpp
+  random_shuffle.cpp
   regex.cpp
   shared_mutex.cpp
   stdexcept.cpp
@@ -380,10 +381,14 @@ if (NOT CMAKE_CONFIGURATION_TYPES AND (LIBCXX_INSTALL_LIBRARY OR
     if(LIBCXX_INSTALL_HEADERS)
       set(header_install_target install-cxx-headers)
     endif()
+    if (LIBCXX_ENABLE_PARALLEL_ALGORITHMS)
+      set(pstl_install_target install-pstl)
+    endif()
     add_custom_target(install-cxx
                       DEPENDS ${lib_install_target}
                               ${experimental_lib_install_target}
                               ${header_install_target}
+                              ${pstl_install_target}
                       COMMAND "${CMAKE_COMMAND}"
                       -DCMAKE_INSTALL_COMPONENT=cxx
                       -P "${LIBCXX_BINARY_DIR}/cmake_install.cmake")
@@ -391,6 +396,7 @@ if (NOT CMAKE_CONFIGURATION_TYPES AND (LIBCXX_INSTALL_LIBRARY OR
                       DEPENDS ${lib_install_target}
                               ${experimental_lib_install_target}
                               ${header_install_target}
+                              ${pstl_install_target}
                       COMMAND "${CMAKE_COMMAND}"
                       -DCMAKE_INSTALL_COMPONENT=cxx
                       -DCMAKE_INSTALL_DO_STRIP=1
diff --git a/libcxx/src/algorithm.cpp b/libcxx/src/algorithm.cpp
index 40669fb9e7561..9f11dc38b15b4 100644
--- a/libcxx/src/algorithm.cpp
+++ b/libcxx/src/algorithm.cpp
@@ -7,13 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "algorithm"
-#include "random"
-#ifndef _LIBCPP_HAS_NO_THREADS
-#include "mutex"
-#if defined(__ELF__) && defined(_LIBCPP_LINK_PTHREAD_LIB)
-#pragma comment(lib, "pthread")
-#endif
-#endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -51,45 +44,4 @@ template bool __insertion_sort_incomplete<__less<long double>&, long double*>(lo
 
 template unsigned __sort5<__less<long double>&, long double*>(long double*, long double*, long double*, long double*, long double*, __less<long double>&);
 
-#ifndef _LIBCPP_HAS_NO_THREADS
-_LIBCPP_SAFE_STATIC static __libcpp_mutex_t __rs_mut = _LIBCPP_MUTEX_INITIALIZER;
-#endif
-unsigned __rs_default::__c_ = 0;
-
-__rs_default::__rs_default()
-{
-#ifndef _LIBCPP_HAS_NO_THREADS
-    __libcpp_mutex_lock(&__rs_mut);
-#endif
-    __c_ = 1;
-}
-
-__rs_default::__rs_default(const __rs_default&)
-{
-    ++__c_;
-}
-
-__rs_default::~__rs_default()
-{
-#ifndef _LIBCPP_HAS_NO_THREADS
-    if (--__c_ == 0)
-       __libcpp_mutex_unlock(&__rs_mut);
-#else
-    --__c_;
-#endif
-}
-
-__rs_default::result_type
-__rs_default::operator()()
-{
-    static mt19937 __rs_g;
-    return __rs_g();
-}
-
-__rs_default
-__rs_get()
-{
-    return __rs_default();
-}
-
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/src/random_shuffle.cpp b/libcxx/src/random_shuffle.cpp
new file mode 100644
index 0000000000000..be2c47fa0defb
--- /dev/null
+++ b/libcxx/src/random_shuffle.cpp
@@ -0,0 +1,61 @@
+//===----------------------- random_shuffle.cpp ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "algorithm"
+#include "random"
+#ifndef _LIBCPP_HAS_NO_THREADS
+#   include "mutex"
+#   if defined(__ELF__) && defined(_LIBCPP_LINK_PTHREAD_LIB)
+#       pragma comment(lib, "pthread")
+#   endif
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#ifndef _LIBCPP_HAS_NO_THREADS
+_LIBCPP_SAFE_STATIC static __libcpp_mutex_t __rs_mut = _LIBCPP_MUTEX_INITIALIZER;
+#endif
+unsigned __rs_default::__c_ = 0;
+
+__rs_default::__rs_default()
+{
+#ifndef _LIBCPP_HAS_NO_THREADS
+    __libcpp_mutex_lock(&__rs_mut);
+#endif
+    __c_ = 1;
+}
+
+__rs_default::__rs_default(const __rs_default&)
+{
+    ++__c_;
+}
+
+__rs_default::~__rs_default()
+{
+#ifndef _LIBCPP_HAS_NO_THREADS
+    if (--__c_ == 0)
+       __libcpp_mutex_unlock(&__rs_mut);
+#else
+    --__c_;
+#endif
+}
+
+__rs_default::result_type
+__rs_default::operator()()
+{
+    static mt19937 __rs_g;
+    return __rs_g();
+}
+
+__rs_default
+__rs_get()
+{
+    return __rs_default();
+}
+
+_LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/test/CMakeLists.txt b/libcxx/test/CMakeLists.txt
index d4b84b85ac5de..b06984fc0ba93 100644
--- a/libcxx/test/CMakeLists.txt
+++ b/libcxx/test/CMakeLists.txt
@@ -103,7 +103,7 @@ if(LIBCXX_TEST_GDB_PRETTY_PRINTERS)
 endif()
 
 if (LIBCXX_INCLUDE_TESTS)
-  include(AddLLVM) # for configure_lit_site_cfg and add_lit_testsuite
+  include(AddLLVM) # for configure_lit_site_cfg and add_lit_target
 
   configure_lit_site_cfg(
     "${LIBCXX_TEST_CONFIG}"
@@ -114,10 +114,11 @@ if (LIBCXX_INCLUDE_TESTS)
     DEPENDS cxx ${LIBCXX_TEST_DEPS}
     COMMENT "Builds dependencies required to run the test suite.")
 
-  add_lit_testsuite(check-cxx
+  add_lit_target(check-cxx
     "Running libcxx tests"
     ${CMAKE_CURRENT_BINARY_DIR}
-    DEPENDS check-cxx-deps)
+    DEPENDS check-cxx-deps
+    PARAMS "${LIBCXX_TEST_PARAMS}")
 endif()
 
 if (LIBCXX_GENERATE_COVERAGE)
diff --git a/libcxx/test/libcxx/containers/associative/non_const_comparator.verify.cpp b/libcxx/test/libcxx/containers/associative/non_const_comparator.verify.cpp
index d60b3be315738..61cdc2f10bd99 100644
--- a/libcxx/test/libcxx/containers/associative/non_const_comparator.verify.cpp
+++ b/libcxx/test/libcxx/containers/associative/non_const_comparator.verify.cpp
@@ -12,8 +12,9 @@
 // Test that libc++ generates a warning diagnostic when the container is
 // provided a non-const callable comparator.
 
-#include <set>
 #include <map>
+#include <set>
+#include <type_traits> // for __invokable
 
 struct BadCompare {
   template <class T, class U>
diff --git a/libcxx/test/libcxx/debug/containers/db_sequence_container_iterators.multithread.pass.cpp b/libcxx/test/libcxx/debug/containers/db_sequence_container_iterators.multithread.pass.cpp
index 83d800b31d7dd..bdfe3115a0c54 100644
--- a/libcxx/test/libcxx/debug/containers/db_sequence_container_iterators.multithread.pass.cpp
+++ b/libcxx/test/libcxx/debug/containers/db_sequence_container_iterators.multithread.pass.cpp
@@ -9,6 +9,7 @@
 // UNSUPPORTED: c++03, c++11, c++14
 // UNSUPPORTED: windows
 // UNSUPPORTED: libcpp-has-no-threads
+// UNSUPPORTED: libcpp-no-if-constexpr
 // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DEBUG=1
 
 // Can't test the system lib because this test enables debug mode
diff --git a/libcxx/test/libcxx/diagnostics/nodiscard_extensions.pass.cpp b/libcxx/test/libcxx/diagnostics/nodiscard_extensions.pass.cpp
index c4d6016d7e1c5..cfb7054d3e2d4 100644
--- a/libcxx/test/libcxx/diagnostics/nodiscard_extensions.pass.cpp
+++ b/libcxx/test/libcxx/diagnostics/nodiscard_extensions.pass.cpp
@@ -13,9 +13,10 @@
 // This test intentionally leaks memory, so it is unsupported under ASAN.
 // UNSUPPORTED: asan
 
-// AppleClang9 doesn't yet support C++17's implicitly synthesized deduction
-// guides from existing ctors, needed by default_searcher() below.
+// AppleClang9 and GCC 5 don't support C++17's implicitly synthesized
+// deduction guides from existing ctors, needed by default_searcher() below.
 // UNSUPPORTED: apple-clang-9
+// UNSUPPORTED: gcc-5
 
 // All entities to which libc++ applies [[nodiscard]] as an extension should
 // be tested here and in nodiscard_extensions.fail.cpp. They should also
diff --git a/libcxx/test/libcxx/double_include.sh.cpp b/libcxx/test/libcxx/double_include.sh.cpp
index 9fdd82f06e919..42a437bbdbe1c 100644
--- a/libcxx/test/libcxx/double_include.sh.cpp
+++ b/libcxx/test/libcxx/double_include.sh.cpp
@@ -14,6 +14,11 @@
 // RUN: %{cxx} -o %t.exe %t.first.o %t.second.o %{flags} %{link_flags}
 // RUN: %{run}
 
+// GCC 5 pretends it supports C++17 features, but some features like static_assert
+// without a message are not actually supported. This causes some headers to fail
+// when included.
+// UNSUPPORTED: gcc-5 && c++17
+
 // Prevent <ext/hash_map> from generating deprecated warnings for this test.
 #if defined(__DEPRECATED)
 #undef __DEPRECATED
diff --git a/libcxx/test/libcxx/experimental/language.support/support.coroutines/dialect_support.pass.cpp b/libcxx/test/libcxx/experimental/language.support/support.coroutines/dialect_support.pass.cpp
index 1699d6714b22b..a168ee516b6c3 100644
--- a/libcxx/test/libcxx/experimental/language.support/support.coroutines/dialect_support.pass.cpp
+++ b/libcxx/test/libcxx/experimental/language.support/support.coroutines/dialect_support.pass.cpp
@@ -16,6 +16,8 @@
 
 #include <experimental/coroutine>
 
+#include "test_macros.h"
+
 namespace coro = std::experimental::coroutines_v1;
 
 coro::suspend_always sa;
@@ -25,7 +27,7 @@ struct MyFuture {
   struct promise_type {
     typedef coro::coroutine_handle<promise_type> HandleT;
     coro::suspend_never initial_suspend() { return sn; }
-    coro::suspend_always final_suspend() { return sa; }
+    coro::suspend_always final_suspend() TEST_NOEXCEPT { return sa; }
     coro::suspend_never yield_value(int) { return sn; }
     MyFuture get_return_object() {
       MyFuture f(HandleT::from_promise(*this));
diff --git a/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp b/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp
index b7f2f07b31a42..5a5735934e95f 100644
--- a/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp
+++ b/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03
-// ADDITIONAL_COMPILE_FLAGS: -I%{libcxx_src_root}/src/filesystem
+// ADDITIONAL_COMPILE_FLAGS: -I %S/../../../../../../src/filesystem
 
 // <filesystem>
 
diff --git a/libcxx/test/libcxx/input.output/filesystems/convert_file_time.pass.cpp b/libcxx/test/libcxx/input.output/filesystems/convert_file_time.pass.cpp
index 3f99ae2d11957..9e406b00f5c1d 100644
--- a/libcxx/test/libcxx/input.output/filesystems/convert_file_time.pass.cpp
+++ b/libcxx/test/libcxx/input.output/filesystems/convert_file_time.pass.cpp
@@ -7,12 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11
+// UNSUPPORTED: gcc-5
 
 // <filesystem>
 
 // typedef TrivialClock file_time_type;
 
-// ADDITIONAL_COMPILE_FLAGS: -I%{libcxx_src_root}/src/filesystem
+// ADDITIONAL_COMPILE_FLAGS: -I %S/../../../../src/filesystem
 
 #include <filesystem>
 #include <chrono>
diff --git a/libcxx/test/libcxx/memory/aligned_allocation_macro.pass.cpp b/libcxx/test/libcxx/memory/aligned_allocation_macro.pass.cpp
index 61c73ef058c16..749c9470c3063 100644
--- a/libcxx/test/libcxx/memory/aligned_allocation_macro.pass.cpp
+++ b/libcxx/test/libcxx/memory/aligned_allocation_macro.pass.cpp
@@ -12,6 +12,9 @@
 // target, so this test would fail.
 // UNSUPPORTED: apple-clang-9, apple-clang-10
 
+// GCC 5 doesn't support aligned allocation
+// UNSUPPORTED: gcc-5
+
 // XFAIL: availability=macosx10.13
 // XFAIL: availability=macosx10.12
 // XFAIL: availability=macosx10.11
diff --git a/libcxx/test/libcxx/min_max_macros.compile.pass.cpp b/libcxx/test/libcxx/min_max_macros.compile.pass.cpp
index 03b79bed2829d..461fe9e5964b7 100644
--- a/libcxx/test/libcxx/min_max_macros.compile.pass.cpp
+++ b/libcxx/test/libcxx/min_max_macros.compile.pass.cpp
@@ -10,6 +10,9 @@
 // Test that headers are not tripped up by the surrounding code defining the
 // min() and max() macros.
 
+// GCC 5 has incomplete support for C++17, so some headers fail when included.
+// UNSUPPORTED: gcc-5 && c++17
+
 // Prevent <ext/hash_map> from generating deprecated warnings for this test.
 #if defined(__DEPRECATED)
 #undef __DEPRECATED
diff --git a/libcxx/test/libcxx/no_assert_include.compile.pass.cpp b/libcxx/test/libcxx/no_assert_include.compile.pass.cpp
index 79f3536a5b0d7..fc83df6b4fafc 100644
--- a/libcxx/test/libcxx/no_assert_include.compile.pass.cpp
+++ b/libcxx/test/libcxx/no_assert_include.compile.pass.cpp
@@ -10,6 +10,9 @@
 // Ensure that none of the standard C++ headers implicitly include cassert or
 // assert.h (because assert() is implemented as a macro).
 
+// GCC 5 has incomplete support for C++17, so some headers fail when included.
+// UNSUPPORTED: gcc-5 && c++17
+
 // Prevent <ext/hash_map> from generating deprecated warnings for this test.
 #if defined(__DEPRECATED)
 #undef __DEPRECATED
diff --git a/libcxx/test/libcxx/selftest/newformat/additional_compile_flags/substitutes-in-compile-flags.sh.cpp b/libcxx/test/libcxx/selftest/additional_compile_flags/substitutes-in-compile-flags.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/additional_compile_flags/substitutes-in-compile-flags.sh.cpp
rename to libcxx/test/libcxx/selftest/additional_compile_flags/substitutes-in-compile-flags.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/additional_compile_flags/substitutes-in-run.sh.cpp b/libcxx/test/libcxx/selftest/additional_compile_flags/substitutes-in-run.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/additional_compile_flags/substitutes-in-run.sh.cpp
rename to libcxx/test/libcxx/selftest/additional_compile_flags/substitutes-in-run.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/compile.fail.cpp/compile-error.compile.fail.cpp b/libcxx/test/libcxx/selftest/compile.fail.cpp/compile-error.compile.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/compile.fail.cpp/compile-error.compile.fail.cpp
rename to libcxx/test/libcxx/selftest/compile.fail.cpp/compile-error.compile.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/compile.fail.cpp/compile-success.compile.fail.cpp b/libcxx/test/libcxx/selftest/compile.fail.cpp/compile-success.compile.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/compile.fail.cpp/compile-success.compile.fail.cpp
rename to libcxx/test/libcxx/selftest/compile.fail.cpp/compile-success.compile.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/compile-error.compile.pass.cpp b/libcxx/test/libcxx/selftest/compile.pass.cpp/compile-error.compile.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/compile-error.compile.pass.cpp
rename to libcxx/test/libcxx/selftest/compile.pass.cpp/compile-error.compile.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/compile-success.compile.pass.cpp b/libcxx/test/libcxx/selftest/compile.pass.cpp/compile-success.compile.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/compile-success.compile.pass.cpp
rename to libcxx/test/libcxx/selftest/compile.pass.cpp/compile-success.compile.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/link-error.compile.pass.cpp b/libcxx/test/libcxx/selftest/compile.pass.cpp/link-error.compile.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/link-error.compile.pass.cpp
rename to libcxx/test/libcxx/selftest/compile.pass.cpp/link-error.compile.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/run-error.compile.pass.cpp b/libcxx/test/libcxx/selftest/compile.pass.cpp/run-error.compile.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/run-error.compile.pass.cpp
rename to libcxx/test/libcxx/selftest/compile.pass.cpp/run-error.compile.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/convenience_substitutions/build_run.sh.cpp b/libcxx/test/libcxx/selftest/convenience_substitutions/build_run.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/convenience_substitutions/build_run.sh.cpp
rename to libcxx/test/libcxx/selftest/convenience_substitutions/build_run.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/dsl/dsl.sh.py b/libcxx/test/libcxx/selftest/dsl/dsl.sh.py
index 7205f35e7dc55..ff4ac2147bf68 100644
--- a/libcxx/test/libcxx/selftest/dsl/dsl.sh.py
+++ b/libcxx/test/libcxx/selftest/dsl/dsl.sh.py
@@ -6,11 +6,15 @@
 #
 #===----------------------------------------------------------------------===##
 
-# RUN: %{python} %s %S %T %{escaped_exec} \
-# RUN:                    %{escaped_cxx} \
-# RUN:                    %{escaped_flags} \
-# RUN:                    %{escaped_compile_flags} \
-# RUN:                    %{escaped_link_flags}
+# Note: We prepend arguments with 'x' to avoid thinking there are too few
+#       arguments in case an argument is an empty string.
+# RUN: %{python} %s x%S \
+# RUN:              x%T \
+# RUN:              x%{escaped_exec} \
+# RUN:              x%{escaped_cxx} \
+# RUN:              x%{escaped_flags} \
+# RUN:              x%{escaped_compile_flags} \
+# RUN:              x%{escaped_link_flags}
 # END.
 
 import base64
@@ -33,7 +37,8 @@
 
 # Steal some parameters from the config running this test so that we can
 # bootstrap our own TestingConfig.
-SOURCE_ROOT, EXEC_PATH, EXEC, CXX, FLAGS, COMPILE_FLAGS, LINK_FLAGS = sys.argv[1:8]
+args = list(map(lambda s: s[1:], sys.argv[1:8])) # Remove the leading 'x'
+SOURCE_ROOT, EXEC_PATH, EXEC, CXX, FLAGS, COMPILE_FLAGS, LINK_FLAGS = args
 sys.argv[1:8] = []
 
 class SetupConfigs(unittest.TestCase):
diff --git a/libcxx/test/libcxx/selftest/newformat/fail.cpp/compile-failure.fail.cpp b/libcxx/test/libcxx/selftest/fail.cpp/compile-failure.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/fail.cpp/compile-failure.fail.cpp
rename to libcxx/test/libcxx/selftest/fail.cpp/compile-failure.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/fail.cpp/compile-success.fail.cpp b/libcxx/test/libcxx/selftest/fail.cpp/compile-success.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/fail.cpp/compile-success.fail.cpp
rename to libcxx/test/libcxx/selftest/fail.cpp/compile-success.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/fail.cpp/lit.local.cfg b/libcxx/test/libcxx/selftest/fail.cpp/lit.local.cfg
similarity index 65%
rename from libcxx/test/libcxx/selftest/newformat/fail.cpp/lit.local.cfg
rename to libcxx/test/libcxx/selftest/fail.cpp/lit.local.cfg
index 2cd7606f4c4e3..b8ef6cb95c96c 100644
--- a/libcxx/test/libcxx/selftest/newformat/fail.cpp/lit.local.cfg
+++ b/libcxx/test/libcxx/selftest/fail.cpp/lit.local.cfg
@@ -1,6 +1,6 @@
-import libcxx.test.newformat
+import libcxx.test.format
 
 # The tests in this directory need to know whether Clang-verify is supported
 # to work properly.
-if libcxx.test.newformat._supportsVerify(config):
+if libcxx.test.format._supportsVerify(config):
     config.available_features.add('verify-support')
diff --git a/libcxx/test/libcxx/selftest/newformat/fail.cpp/no-diagnostics-unmarked.fail.cpp b/libcxx/test/libcxx/selftest/fail.cpp/no-diagnostics-unmarked.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/fail.cpp/no-diagnostics-unmarked.fail.cpp
rename to libcxx/test/libcxx/selftest/fail.cpp/no-diagnostics-unmarked.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/fail.cpp/no-diagnostics.fail.cpp b/libcxx/test/libcxx/selftest/fail.cpp/no-diagnostics.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/fail.cpp/no-diagnostics.fail.cpp
rename to libcxx/test/libcxx/selftest/fail.cpp/no-diagnostics.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/fail.cpp/right-diagnostic.fail.cpp b/libcxx/test/libcxx/selftest/fail.cpp/right-diagnostic.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/fail.cpp/right-diagnostic.fail.cpp
rename to libcxx/test/libcxx/selftest/fail.cpp/right-diagnostic.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/fail.cpp/wrong-diagnostic.fail.cpp b/libcxx/test/libcxx/selftest/fail.cpp/wrong-diagnostic.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/fail.cpp/wrong-diagnostic.fail.cpp
rename to libcxx/test/libcxx/selftest/fail.cpp/wrong-diagnostic.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/file_dependencies/a.txt b/libcxx/test/libcxx/selftest/file_dependencies/a.txt
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/file_dependencies/a.txt
rename to libcxx/test/libcxx/selftest/file_dependencies/a.txt
diff --git a/libcxx/test/libcxx/selftest/newformat/file_dependencies/absolute-and-relative-paths.sh.cpp b/libcxx/test/libcxx/selftest/file_dependencies/absolute-and-relative-paths.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/file_dependencies/absolute-and-relative-paths.sh.cpp
rename to libcxx/test/libcxx/selftest/file_dependencies/absolute-and-relative-paths.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/file_dependencies/dir/b.txt b/libcxx/test/libcxx/selftest/file_dependencies/dir/b.txt
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/file_dependencies/dir/b.txt
rename to libcxx/test/libcxx/selftest/file_dependencies/dir/b.txt
diff --git a/libcxx/test/libcxx/selftest/newformat/file_dependencies/substitute-in-dependencies.sh.cpp b/libcxx/test/libcxx/selftest/file_dependencies/substitute-in-dependencies.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/file_dependencies/substitute-in-dependencies.sh.cpp
rename to libcxx/test/libcxx/selftest/file_dependencies/substitute-in-dependencies.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/link.fail.cpp/compile-error.link.fail.cpp b/libcxx/test/libcxx/selftest/link.fail.cpp/compile-error.link.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/link.fail.cpp/compile-error.link.fail.cpp
rename to libcxx/test/libcxx/selftest/link.fail.cpp/compile-error.link.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/link.fail.cpp/link-error.link.fail.cpp b/libcxx/test/libcxx/selftest/link.fail.cpp/link-error.link.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/link.fail.cpp/link-error.link.fail.cpp
rename to libcxx/test/libcxx/selftest/link.fail.cpp/link-error.link.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/link.fail.cpp/link-success.link.fail.cpp b/libcxx/test/libcxx/selftest/link.fail.cpp/link-success.link.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/link.fail.cpp/link-success.link.fail.cpp
rename to libcxx/test/libcxx/selftest/link.fail.cpp/link-success.link.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/link.pass.cpp/compile-error.link.pass.cpp b/libcxx/test/libcxx/selftest/link.pass.cpp/compile-error.link.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/link.pass.cpp/compile-error.link.pass.cpp
rename to libcxx/test/libcxx/selftest/link.pass.cpp/compile-error.link.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/link.pass.cpp/link-error.link.pass.cpp b/libcxx/test/libcxx/selftest/link.pass.cpp/link-error.link.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/link.pass.cpp/link-error.link.pass.cpp
rename to libcxx/test/libcxx/selftest/link.pass.cpp/link-error.link.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/link.pass.cpp/link-success.link.pass.cpp b/libcxx/test/libcxx/selftest/link.pass.cpp/link-success.link.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/link.pass.cpp/link-success.link.pass.cpp
rename to libcxx/test/libcxx/selftest/link.pass.cpp/link-success.link.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/link.pass.cpp/run-error.link.pass.cpp b/libcxx/test/libcxx/selftest/link.pass.cpp/run-error.link.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/link.pass.cpp/run-error.link.pass.cpp
rename to libcxx/test/libcxx/selftest/link.pass.cpp/run-error.link.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/lit.local.cfg b/libcxx/test/libcxx/selftest/newformat/lit.local.cfg
deleted file mode 100644
index 0f75e02dc6614..0000000000000
--- a/libcxx/test/libcxx/selftest/newformat/lit.local.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-import os
-import site
-site.addsitedir(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'utils'))
-import libcxx.test.newformat
-
-if not isinstance(config.test_format, libcxx.test.newformat.CxxStandardLibraryTest):
-    config.unsupported = True
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.cpp/compile-error.pass.cpp b/libcxx/test/libcxx/selftest/pass.cpp/compile-error.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/pass.cpp/compile-error.pass.cpp
rename to libcxx/test/libcxx/selftest/pass.cpp/compile-error.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.cpp/link-error.pass.cpp b/libcxx/test/libcxx/selftest/pass.cpp/link-error.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/pass.cpp/link-error.pass.cpp
rename to libcxx/test/libcxx/selftest/pass.cpp/link-error.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.cpp/run-error.pass.cpp b/libcxx/test/libcxx/selftest/pass.cpp/run-error.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/pass.cpp/run-error.pass.cpp
rename to libcxx/test/libcxx/selftest/pass.cpp/run-error.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.cpp/run-success.pass.cpp b/libcxx/test/libcxx/selftest/pass.cpp/run-success.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/pass.cpp/run-success.pass.cpp
rename to libcxx/test/libcxx/selftest/pass.cpp/run-success.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.cpp/werror.pass.cpp b/libcxx/test/libcxx/selftest/pass.cpp/werror.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/pass.cpp/werror.pass.cpp
rename to libcxx/test/libcxx/selftest/pass.cpp/werror.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.mm/compile-error.pass.mm b/libcxx/test/libcxx/selftest/pass.mm/compile-error.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/pass.mm/compile-error.pass.mm
rename to libcxx/test/libcxx/selftest/pass.mm/compile-error.pass.mm
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.mm/link-error.pass.mm b/libcxx/test/libcxx/selftest/pass.mm/link-error.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/pass.mm/link-error.pass.mm
rename to libcxx/test/libcxx/selftest/pass.mm/link-error.pass.mm
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.mm/no-arc.pass.mm b/libcxx/test/libcxx/selftest/pass.mm/no-arc.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/pass.mm/no-arc.pass.mm
rename to libcxx/test/libcxx/selftest/pass.mm/no-arc.pass.mm
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.mm/run-error.pass.mm b/libcxx/test/libcxx/selftest/pass.mm/run-error.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/pass.mm/run-error.pass.mm
rename to libcxx/test/libcxx/selftest/pass.mm/run-error.pass.mm
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.mm/run-success.pass.mm b/libcxx/test/libcxx/selftest/pass.mm/run-success.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/pass.mm/run-success.pass.mm
rename to libcxx/test/libcxx/selftest/pass.mm/run-success.pass.mm
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.mm/use-objective-cxx.pass.mm b/libcxx/test/libcxx/selftest/pass.mm/use-objective-cxx.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/pass.mm/use-objective-cxx.pass.mm
rename to libcxx/test/libcxx/selftest/pass.mm/use-objective-cxx.pass.mm
diff --git a/libcxx/test/libcxx/selftest/newformat/remote-substitutions.sh.cpp b/libcxx/test/libcxx/selftest/remote-substitutions.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/remote-substitutions.sh.cpp
rename to libcxx/test/libcxx/selftest/remote-substitutions.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/run.fail.cpp/compile-error.run.fail.cpp b/libcxx/test/libcxx/selftest/run.fail.cpp/compile-error.run.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/run.fail.cpp/compile-error.run.fail.cpp
rename to libcxx/test/libcxx/selftest/run.fail.cpp/compile-error.run.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/run.fail.cpp/link-error.run.fail.cpp b/libcxx/test/libcxx/selftest/run.fail.cpp/link-error.run.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/run.fail.cpp/link-error.run.fail.cpp
rename to libcxx/test/libcxx/selftest/run.fail.cpp/link-error.run.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/run.fail.cpp/run-error.run.fail.cpp b/libcxx/test/libcxx/selftest/run.fail.cpp/run-error.run.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/run.fail.cpp/run-error.run.fail.cpp
rename to libcxx/test/libcxx/selftest/run.fail.cpp/run-error.run.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/run.fail.cpp/run-success.run.fail.cpp b/libcxx/test/libcxx/selftest/run.fail.cpp/run-success.run.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/run.fail.cpp/run-success.run.fail.cpp
rename to libcxx/test/libcxx/selftest/run.fail.cpp/run-success.run.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/sh.cpp/empty.sh.cpp b/libcxx/test/libcxx/selftest/sh.cpp/empty.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/sh.cpp/empty.sh.cpp
rename to libcxx/test/libcxx/selftest/sh.cpp/empty.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/sh.cpp/run-error.sh.cpp b/libcxx/test/libcxx/selftest/sh.cpp/run-error.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/sh.cpp/run-error.sh.cpp
rename to libcxx/test/libcxx/selftest/sh.cpp/run-error.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/sh.cpp/run-success.sh.cpp b/libcxx/test/libcxx/selftest/sh.cpp/run-success.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/sh.cpp/run-success.sh.cpp
rename to libcxx/test/libcxx/selftest/sh.cpp/run-success.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/sh.cpp/substitutions.sh.cpp b/libcxx/test/libcxx/selftest/sh.cpp/substitutions.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/sh.cpp/substitutions.sh.cpp
rename to libcxx/test/libcxx/selftest/sh.cpp/substitutions.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/sh.cpp/werror.sh.cpp b/libcxx/test/libcxx/selftest/sh.cpp/werror.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/sh.cpp/werror.sh.cpp
rename to libcxx/test/libcxx/selftest/sh.cpp/werror.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/shell-escape-pipes.sh.cpp b/libcxx/test/libcxx/selftest/shell-escape-pipes.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/shell-escape-pipes.sh.cpp
rename to libcxx/test/libcxx/selftest/shell-escape-pipes.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/shell-escape.sh.cpp b/libcxx/test/libcxx/selftest/shell-escape.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/shell-escape.sh.cpp
rename to libcxx/test/libcxx/selftest/shell-escape.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/tmpdir-exists.sh.cpp b/libcxx/test/libcxx/selftest/tmpdir-exists.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/tmpdir-exists.sh.cpp
rename to libcxx/test/libcxx/selftest/tmpdir-exists.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/verify.cpp/no-diagnostics-unmarked.verify.cpp b/libcxx/test/libcxx/selftest/verify.cpp/no-diagnostics-unmarked.verify.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/verify.cpp/no-diagnostics-unmarked.verify.cpp
rename to libcxx/test/libcxx/selftest/verify.cpp/no-diagnostics-unmarked.verify.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/verify.cpp/no-diagnostics.verify.cpp b/libcxx/test/libcxx/selftest/verify.cpp/no-diagnostics.verify.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/verify.cpp/no-diagnostics.verify.cpp
rename to libcxx/test/libcxx/selftest/verify.cpp/no-diagnostics.verify.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/verify.cpp/no-werror.verify.cpp b/libcxx/test/libcxx/selftest/verify.cpp/no-werror.verify.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/verify.cpp/no-werror.verify.cpp
rename to libcxx/test/libcxx/selftest/verify.cpp/no-werror.verify.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/verify.cpp/right-diagnostic.verify.cpp b/libcxx/test/libcxx/selftest/verify.cpp/right-diagnostic.verify.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/verify.cpp/right-diagnostic.verify.cpp
rename to libcxx/test/libcxx/selftest/verify.cpp/right-diagnostic.verify.cpp
diff --git a/libcxx/test/libcxx/selftest/newformat/verify.cpp/wrong-diagnostic.verify.cpp b/libcxx/test/libcxx/selftest/verify.cpp/wrong-diagnostic.verify.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/newformat/verify.cpp/wrong-diagnostic.verify.cpp
rename to libcxx/test/libcxx/selftest/verify.cpp/wrong-diagnostic.verify.cpp
diff --git a/libcxx/test/libcxx/strings/iterators.noexcept.pass.cpp b/libcxx/test/libcxx/strings/iterators.noexcept.pass.cpp
index 787b86ad95b99..78f8429598c0c 100644
--- a/libcxx/test/libcxx/strings/iterators.noexcept.pass.cpp
+++ b/libcxx/test/libcxx/strings/iterators.noexcept.pass.cpp
@@ -18,7 +18,7 @@
 // When exceptions are disabled, all iterators should get this "fast path"
 //
 
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_NO_EXCEPTIONS
+// ADDITIONAL_COMPILE_FLAGS: -fno-exceptions
 
 #include <iterator>
 #include <cassert>
diff --git a/libcxx/test/libcxx/utilities/optional/optional.object/optional.object.assign/copy.pass.cpp b/libcxx/test/libcxx/utilities/optional/optional.object/optional.object.assign/copy.pass.cpp
index d1262677336f7..06ddb04c614f4 100644
--- a/libcxx/test/libcxx/utilities/optional/optional.object/optional.object.assign/copy.pass.cpp
+++ b/libcxx/test/libcxx/utilities/optional/optional.object/optional.object.assign/copy.pass.cpp
@@ -7,6 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11, c++14
+
+// This test segfaults GCC 5
+// UNSUPPORTED: gcc-5
+
 // <optional>
 
 // optional<T>& operator=(const optional<T>& rhs);
diff --git a/libcxx/test/libcxx/utilities/optional/optional.object/optional.object.assign/move.pass.cpp b/libcxx/test/libcxx/utilities/optional/optional.object/optional.object.assign/move.pass.cpp
index 2e4b7bd435a27..4f3f0c683d8bd 100644
--- a/libcxx/test/libcxx/utilities/optional/optional.object/optional.object.assign/move.pass.cpp
+++ b/libcxx/test/libcxx/utilities/optional/optional.object/optional.object.assign/move.pass.cpp
@@ -7,6 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11, c++14
+
+// This test segfaults GCC 5
+// UNSUPPORTED: gcc-5
+
 // <optional>
 
 // optional<T>& operator=(optional<T>&& rhs);
diff --git a/libcxx/test/libcxx/utilities/variant/variant.variant/variant_size.pass.cpp b/libcxx/test/libcxx/utilities/variant/variant.variant/variant_size.pass.cpp
index dca0e7c14b5d9..8b66e311dae10 100644
--- a/libcxx/test/libcxx/utilities/variant/variant.variant/variant_size.pass.cpp
+++ b/libcxx/test/libcxx/utilities/variant/variant.variant/variant_size.pass.cpp
@@ -9,6 +9,9 @@
 
 // UNSUPPORTED: c++03, c++11, c++14
 
+// GCC 5 pretends it supports C++17, but it doesn't properly support it
+// UNSUPPORTED: gcc-5
+
 // <variant>
 
 // template <class ...Types> class variant;
diff --git a/libcxx/test/libcxx/utilities/variant/version.pass.cpp b/libcxx/test/libcxx/utilities/variant/version.pass.cpp
index 1dcb75d180d8a..2a4ebe52dc180 100644
--- a/libcxx/test/libcxx/utilities/variant/version.pass.cpp
+++ b/libcxx/test/libcxx/utilities/variant/version.pass.cpp
@@ -8,6 +8,10 @@
 
 // <variant>
 
+// GCC 5 pretends it supports C++17, but it doesn't properly support it, and
+// <variant> fails.
+// UNSUPPORTED: gcc-5
+
 #include <variant>
 
 #include "test_macros.h"
diff --git a/libcxx/test/lit.site.cfg.in b/libcxx/test/lit.site.cfg.in
index 8692217fddfa6..939776f2287da 100644
--- a/libcxx/test/lit.site.cfg.in
+++ b/libcxx/test/lit.site.cfg.in
@@ -60,15 +60,11 @@ config.recursiveExpansionLimit = 10
 # Infer the test_exec_root from the libcxx_object root.
 config.test_exec_root = os.path.join(config.libcxx_obj_root, 'test')
 
+import libcxx.test.format
+config.test_format = libcxx.test.format.CxxStandardLibraryTest()
+
 lit_config.note('Using configuration variant: {}'.format(config.configuration_variant))
 import libcxx.test.config
 configuration = libcxx.test.config.Configuration(lit_config, config)
 configuration.configure()
 configuration.print_config_info()
-if lit_config.params.get('use_old_format', False):
-    lit_config.note("Using the old libc++ testing format")
-    config.test_format = configuration.get_test_format()
-else:
-    lit_config.note("Using the new libc++ testing format")
-    import libcxx.test.newformat
-    config.test_format = libcxx.test.newformat.CxxStandardLibraryTest()
diff --git a/libcxx/test/pretty_printers/gdb_pretty_printer_test.sh.cpp b/libcxx/test/pretty_printers/gdb_pretty_printer_test.sh.cpp
index 96cdd2016a692..081e778540a0b 100644
--- a/libcxx/test/pretty_printers/gdb_pretty_printer_test.sh.cpp
+++ b/libcxx/test/pretty_printers/gdb_pretty_printer_test.sh.cpp
@@ -10,7 +10,7 @@
 //
 // RUN: %{cxx} %{flags} %s -o %t.exe %{compile_flags} -g %{link_flags}
 // Ensure locale-independence for unicode tests.
-// RUN: %{libcxx_gdb} -nx -batch -iex "set autoload off" -ex "source %{libcxx_src_root}/utils/gdb/libcxx/printers.py" -ex "python register_libcxx_printer_loader()" -ex "source %{libcxx_src_root}/test/pretty_printers/gdb_pretty_printer_test.py" %t.exe
+// RUN: %{libcxx_gdb} -nx -batch -iex "set autoload off" -ex "source %S/../../utils/gdb/libcxx/printers.py" -ex "python register_libcxx_printer_loader()" -ex "source %S/gdb_pretty_printer_test.py" %t.exe
 
 #include <bitset>
 #include <deque>
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.random.sample/sample.fail.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.random.sample/sample.fail.cpp
index 551d4c8d2d61c..4d03646d83feb 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.random.sample/sample.fail.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.random.sample/sample.fail.cpp
@@ -8,6 +8,9 @@
 
 // UNSUPPORTED: c++03, c++11, c++14
 
+// Test failure not investigated on GCC 5.
+// UNSUPPORTED: gcc-5
+
 // <algorithm>
 
 // template <class PopulationIterator, class SampleIterator, class Distance,
diff --git a/libcxx/test/std/containers/sequences/array/aggregate.pass.cpp b/libcxx/test/std/containers/sequences/array/aggregate.pass.cpp
index 9db25a9231bca..9f3ab23884d52 100644
--- a/libcxx/test/std/containers/sequences/array/aggregate.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/aggregate.pass.cpp
@@ -11,6 +11,10 @@
 // trait before that.
 // UNSUPPORTED: c++03, c++11, c++14
 
+// libc++ doesn't implement std::is_aggregate on GCC 5 and GCC 6.
+// UNSUPPORTED: libc++ && gcc-5
+// UNSUPPORTED: libc++ && gcc-6
+
 #include <array>
 #include <type_traits>
 
diff --git a/libcxx/test/std/containers/sequences/array/at.pass.cpp b/libcxx/test/std/containers/sequences/array/at.pass.cpp
index 77474d6090e2a..5af4549df174a 100644
--- a/libcxx/test/std/containers/sequences/array/at.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/at.pass.cpp
@@ -10,6 +10,9 @@
 
 // reference at (size_type); // constexpr in C++17
 
+// GCC 5 doesn't implement the required constexpr support
+// UNSUPPORTED: gcc-5
+
 #include <array>
 #include <cassert>
 
diff --git a/libcxx/test/std/containers/sequences/array/at_const.pass.cpp b/libcxx/test/std/containers/sequences/array/at_const.pass.cpp
index b8d6b4c24c12e..8d625097cf2e3 100644
--- a/libcxx/test/std/containers/sequences/array/at_const.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/at_const.pass.cpp
@@ -10,6 +10,9 @@
 
 // const_reference at (size_type) const; // constexpr in C++14
 
+// GCC 5 doesn't implement the required constexpr support
+// UNSUPPORTED: gcc-5
+
 #include <array>
 #include <cassert>
 
diff --git a/libcxx/test/std/containers/sequences/array/contiguous.pass.cpp b/libcxx/test/std/containers/sequences/array/contiguous.pass.cpp
index 9589e63643941..d61a09c688c80 100644
--- a/libcxx/test/std/containers/sequences/array/contiguous.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/contiguous.pass.cpp
@@ -8,6 +8,9 @@
 
 // <array>
 
+// GCC 5 doesn't implement the required constexpr support
+// UNSUPPORTED: gcc-5
+
 // An array is a contiguous container
 
 #include <array>
diff --git a/libcxx/test/std/experimental/language.support/support.coroutines/coroutine.handle/coroutine.handle.prom/promise.pass.cpp b/libcxx/test/std/experimental/language.support/support.coroutines/coroutine.handle/coroutine.handle.prom/promise.pass.cpp
index 200bb96a6b180..b6dc79e287952 100644
--- a/libcxx/test/std/experimental/language.support/support.coroutines/coroutine.handle/coroutine.handle.prom/promise.pass.cpp
+++ b/libcxx/test/std/experimental/language.support/support.coroutines/coroutine.handle/coroutine.handle.prom/promise.pass.cpp
@@ -32,7 +32,7 @@ struct MyCoro {
     void unhandled_exception() {}
     void return_void() {}
     coro::suspend_never initial_suspend() { return {}; }
-    coro::suspend_never final_suspend() { return {}; }
+    coro::suspend_never final_suspend() noexcept { return {}; }
     MyCoro get_return_object() {
       do_runtime_test();
       return {};
diff --git a/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/await_result.pass.cpp b/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/await_result.pass.cpp
index 86746f556c4d1..43c71f34c8e62 100644
--- a/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/await_result.pass.cpp
+++ b/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/await_result.pass.cpp
@@ -23,7 +23,7 @@ struct coro_t {
       return {};
     }
     suspend_never initial_suspend() { return {}; }
-    suspend_never final_suspend() { return {}; }
+    suspend_never final_suspend() noexcept { return {}; }
     void return_void() {}
     static void unhandled_exception() {}
   };
diff --git a/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/bool_await_suspend.pass.cpp b/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/bool_await_suspend.pass.cpp
index 679fccbf17fea..f9f9f7bc808bd 100644
--- a/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/bool_await_suspend.pass.cpp
+++ b/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/bool_await_suspend.pass.cpp
@@ -25,7 +25,7 @@ struct coro_t {
       return coroutine_handle<promise_type>::from_promise(*this);
     }
     suspend_never initial_suspend() { return {}; }
-    suspend_never final_suspend() { return {}; }
+    suspend_never final_suspend() noexcept { return {}; }
     void return_void() {}
     void unhandled_exception() {}
   };
diff --git a/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/expected.pass.cpp b/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/expected.pass.cpp
index 3b227cf19e2a9..a0c31b6a6edaa 100644
--- a/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/expected.pass.cpp
+++ b/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/expected.pass.cpp
@@ -40,7 +40,7 @@ struct expected {
     Data* data;
     DataPtr get_return_object() { data = new Data{}; return {data}; }
     suspend_never initial_suspend() { return {}; }
-    suspend_never final_suspend() { return {}; }
+    suspend_never final_suspend() noexcept { return {}; }
     void return_value(T v) { data->val = std::move(v); data->error = {};}
     void unhandled_exception() {}
   };
diff --git a/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/fullexpr-dtor.pass.cpp b/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/fullexpr-dtor.pass.cpp
index bb6fce517df37..047e324dd768c 100644
--- a/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/fullexpr-dtor.pass.cpp
+++ b/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/fullexpr-dtor.pass.cpp
@@ -45,7 +45,7 @@ struct Bug {
 struct coro2 {
   struct promise_type {
     suspend_never initial_suspend() { return{}; }
-    suspend_never final_suspend() { return{}; }
+    suspend_never final_suspend() noexcept { return {}; }
     coro2 get_return_object() { return{}; }
     void return_void() {}
     Bug yield_value(int) { return {}; }
diff --git a/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/generator.pass.cpp b/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/generator.pass.cpp
index 08685b962b735..2c1e2295b06e0 100644
--- a/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/generator.pass.cpp
+++ b/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/generator.pass.cpp
@@ -29,7 +29,7 @@ struct minig {
       return {};
     }
     suspend_always initial_suspend() { return {}; }
-    suspend_always final_suspend() { return {}; }
+    suspend_always final_suspend() noexcept { return {}; }
     minig get_return_object() { return minig{this}; };
     void return_void() {}
     void unhandled_exception() {}
diff --git a/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/go.pass.cpp b/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/go.pass.cpp
index b825fee4bf16c..9a8155a81ad6a 100644
--- a/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/go.pass.cpp
+++ b/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/go.pass.cpp
@@ -46,9 +46,7 @@ struct goroutine
     suspend_never initial_suspend() {
       return {};
     }
-    suspend_never final_suspend() {
-      return {};
-    }
+    suspend_never final_suspend() noexcept { return {}; }
     void return_void() {}
     goroutine get_return_object() {
       return{};
diff --git a/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/multishot_func.pass.cpp b/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/multishot_func.pass.cpp
index 89dfa13695e27..e0ef88810000f 100644
--- a/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/multishot_func.pass.cpp
+++ b/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/multishot_func.pass.cpp
@@ -26,7 +26,7 @@ template <typename R> struct func {
     R result;
     func get_return_object() { return {this}; }
     suspend_always initial_suspend() { return {}; }
-    suspend_never final_suspend() { return {}; }
+    suspend_never final_suspend() noexcept { return {}; }
     void return_void() {}
     template <typename F>
     suspend_always yield_value(F&& f) {
diff --git a/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/oneshot_func.pass.cpp b/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/oneshot_func.pass.cpp
index 352c4abe09993..d19c49d739ea0 100644
--- a/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/oneshot_func.pass.cpp
+++ b/libcxx/test/std/experimental/language.support/support.coroutines/end.to.end/oneshot_func.pass.cpp
@@ -25,7 +25,7 @@ template <typename R> struct func {
     R result;
     func get_return_object() { return {this}; }
     suspend_always initial_suspend() { return {}; }
-    suspend_always final_suspend() { return {}; }
+    suspend_always final_suspend() noexcept { return {}; }
     void return_value(R v) { result = v; }
     void unhandled_exception() {}
   };
diff --git a/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.nonmember/make_move_iterator.pass.cpp b/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.nonmember/make_move_iterator.pass.cpp
index 3a9b467e06f57..b7c6b6bd43108 100644
--- a/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.nonmember/make_move_iterator.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.nonmember/make_move_iterator.pass.cpp
@@ -50,7 +50,7 @@ int main(int, char**)
     {
     constexpr const char *p = "123456789";
     constexpr auto iter = std::make_move_iterator<const char *>(p);
-    static_assert(iter.base() == p);
+    static_assert(iter.base() == p, "");
     }
 #endif
 
diff --git a/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.op.comp/op_gt.pass.cpp b/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.op.comp/op_gt.pass.cpp
index 3da38b3766641..d4a5273ae5da4 100644
--- a/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.op.comp/op_gt.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.op.comp/op_gt.pass.cpp
@@ -17,6 +17,9 @@
 //
 //  constexpr in C++17
 
+// GCC 5 doesn't implement the required constexpr support
+// UNSUPPORTED: gcc-5
+
 #include <iterator>
 #include <cassert>
 
diff --git a/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.op.comp/op_gte.pass.cpp b/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.op.comp/op_gte.pass.cpp
index dceb411515252..7b62b125ab3e4 100644
--- a/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.op.comp/op_gte.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.op.comp/op_gte.pass.cpp
@@ -17,6 +17,9 @@
 //
 //  constexpr in C++17
 
+// GCC 5 doesn't implement the required constexpr support
+// UNSUPPORTED: gcc-5
+
 #include <iterator>
 #include <cassert>
 
diff --git a/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.op.comp/op_lte.pass.cpp b/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.op.comp/op_lte.pass.cpp
index a2ac0b7a6f7bb..5417481271358 100644
--- a/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.op.comp/op_lte.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.op.comp/op_lte.pass.cpp
@@ -17,6 +17,9 @@
 //
 //  constexpr in C++17
 
+// GCC 5 doesn't implement the required constexpr support
+// UNSUPPORTED: gcc-5
+
 #include <iterator>
 #include <cassert>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_duration.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_duration.pass.cpp
index 205c4a9f170c1..45cdf646340a9 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_duration.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_duration.pass.cpp
@@ -10,8 +10,6 @@
 // UNSUPPORTED: c++03, c++11
 // XFAIL: dylib-has-no-shared_mutex
 
-// ALLOW_RETRIES: 2
-
 // <shared_mutex>
 
 // class timed_mutex;
@@ -35,60 +33,63 @@ typedef Clock::duration duration;
 typedef std::chrono::milliseconds ms;
 typedef std::chrono::nanoseconds ns;
 
-ms WaitTime = ms(250);
+ms LongTime = ms(5000);
+ms ShortTime = ms(50);
 
-// Thread sanitizer causes more overhead and will sometimes cause this test
-// to fail. To prevent this we give Thread sanitizer more time to complete the
-// test.
-#if !defined(TEST_HAS_SANITIZERS)
-ms Tolerance = ms(50);
-#else
-ms Tolerance = ms(50 * 5);
-#endif
+static const unsigned Threads = 5;
 
+std::atomic<unsigned> CountDown(Threads);
 
 void f1()
 {
-    time_point t0 = Clock::now();
-    std::shared_lock<std::shared_timed_mutex> lk(m, WaitTime + Tolerance);
-    assert(lk.owns_lock() == true);
-    time_point t1 = Clock::now();
-    ns d = t1 - t0 - WaitTime;
-    assert(d < Tolerance);  // within 50ms
+  // Preemptive scheduling means that one cannot make assumptions about when
+  // code executes and therefore we cannot assume anthing about when the mutex
+  // starts waiting relative to code in the main thread. We can however prove
+  // that a timeout occured and that implies that this code is waiting.
+  // See f2() below.
+  //
+  // Nevertheless, we should at least try to ensure that the mutex waits and
+  // therefore we use an atomic variable to signal to the main thread that this
+  // code is just a few instructions away from waiting.
+  --CountDown;
+  std::shared_lock<std::shared_timed_mutex> lk(m, LongTime);
+  assert(lk.owns_lock() == true);
 }
 
 void f2()
 {
-    time_point t0 = Clock::now();
-    std::shared_lock<std::shared_timed_mutex> lk(m, WaitTime);
-    assert(lk.owns_lock() == false);
-    time_point t1 = Clock::now();
-    ns d = t1 - t0 - WaitTime;
-    assert(d < Tolerance);  // within 50ms
+  time_point t0 = Clock::now();
+  std::shared_lock<std::shared_timed_mutex> lk(m, ShortTime);
+  time_point t1 = Clock::now();
+  assert(lk.owns_lock() == false);
+  assert(t1 - t0 >= ShortTime);
 }
 
 int main(int, char**)
 {
-    {
-        m.lock();
-        std::vector<std::thread> v;
-        for (int i = 0; i < 5; ++i)
-            v.push_back(std::thread(f1));
-        std::this_thread::sleep_for(WaitTime);
-        m.unlock();
-        for (auto& t : v)
-            t.join();
-    }
-    {
-        m.lock();
-        std::vector<std::thread> v;
-        for (int i = 0; i < 5; ++i)
-            v.push_back(std::thread(f2));
-        std::this_thread::sleep_for(WaitTime + Tolerance);
-        m.unlock();
-        for (auto& t : v)
-            t.join();
-    }
+  {
+    m.lock();
+    std::vector<std::thread> v;
+    for (unsigned i = 0; i < Threads; ++i)
+      v.push_back(std::thread(f1));
+    while (CountDown > 0)
+      std::this_thread::yield();
+    // Give one more chance for threads to block and wait for the mutex.
+    std::this_thread::yield();
+    std::this_thread::sleep_for(ShortTime);
+    m.unlock();
+    for (auto& t : v)
+      t.join();
+  }
+  {
+    m.lock();
+    std::vector<std::thread> v;
+    for (unsigned i = 0; i < Threads; ++i)
+      v.push_back(std::thread(f2));
+    for (auto& t : v)
+      t.join();
+    m.unlock();
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_time_point.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_time_point.pass.cpp
index 24c033d8a2878..9a3e222e30a06 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_time_point.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_time_point.pass.cpp
@@ -10,8 +10,6 @@
 // UNSUPPORTED: c++03, c++11
 // XFAIL: dylib-has-no-shared_mutex
 
-// ALLOW_RETRIES: 2
-
 // <shared_mutex>
 
 // class shared_timed_mutex;
@@ -35,59 +33,55 @@ typedef Clock::duration duration;
 typedef std::chrono::milliseconds ms;
 typedef std::chrono::nanoseconds ns;
 
-ms WaitTime = ms(250);
+ms LongTime = ms(5000);
+ms ShortTime = ms(50);
+
+static constexpr unsigned Threads = 5;
 
-// Thread sanitizer causes more overhead and will sometimes cause this test
-// to fail. To prevent this we give Thread sanitizer more time to complete the
-// test.
-#if !defined(TEST_HAS_SANITIZERS)
-ms Tolerance = ms(50);
-#else
-ms Tolerance = ms(50 * 5);
-#endif
+std::atomic<unsigned> CountDown(Threads);
 
 void f1()
 {
-    time_point t0 = Clock::now();
-    std::shared_lock<std::shared_timed_mutex> lk(m, Clock::now() + WaitTime + Tolerance);
-    assert(lk.owns_lock() == true);
-    time_point t1 = Clock::now();
-    ns d = t1 - t0 - WaitTime;
-    assert(d < Tolerance);  // within 50ms
+  --CountDown;
+  time_point t0 = Clock::now();
+  std::shared_lock<std::shared_timed_mutex> lk(m, t0 + LongTime);
+  time_point t1 = Clock::now();
+  assert(lk.owns_lock() == true);
+  assert(t1 - t0 <= LongTime);
 }
 
 void f2()
 {
-    time_point t0 = Clock::now();
-    std::shared_lock<std::shared_timed_mutex> lk(m, Clock::now() + WaitTime);
-    assert(lk.owns_lock() == false);
-    time_point t1 = Clock::now();
-    ns d = t1 - t0 - WaitTime;
-    assert(d < Tolerance);  // within 50ms
+  time_point t0 = Clock::now();
+  std::shared_lock<std::shared_timed_mutex> lk(m, t0 + ShortTime);
+  time_point t1 = Clock::now();
+  assert(lk.owns_lock() == false);
+  assert(t1 - t0 >= ShortTime);
 }
 
 int main(int, char**)
 {
-    {
-        m.lock();
-        std::vector<std::thread> v;
-        for (int i = 0; i < 5; ++i)
-            v.push_back(std::thread(f1));
-        std::this_thread::sleep_for(WaitTime);
-        m.unlock();
-        for (auto& t : v)
-            t.join();
-    }
-    {
-        m.lock();
-        std::vector<std::thread> v;
-        for (int i = 0; i < 5; ++i)
-            v.push_back(std::thread(f2));
-        std::this_thread::sleep_for(WaitTime + Tolerance);
-        m.unlock();
-        for (auto& t : v)
-            t.join();
-    }
+  {
+    m.lock();
+    std::vector<std::thread> v;
+    for (unsigned i = 0; i < Threads; ++i)
+      v.push_back(std::thread(f1));
+    while (CountDown > 0)
+      std::this_thread::yield();
+    std::this_thread::sleep_for(ShortTime);
+    m.unlock();
+    for (auto& t : v)
+      t.join();
+  }
+  {
+    m.lock();
+    std::vector<std::thread> v;
+    for (unsigned i = 0; i < Threads; ++i)
+      v.push_back(std::thread(f2));
+    for (auto& t : v)
+      t.join();
+    m.unlock();
+  }
 
   return 0;
 }
diff --git a/libcxx/test/support/coroutine_types.h b/libcxx/test/support/coroutine_types.h
index e5d40b1fc530c..04b27a161eb34 100644
--- a/libcxx/test/support/coroutine_types.h
+++ b/libcxx/test/support/coroutine_types.h
@@ -20,7 +20,7 @@ template <typename Ty> struct generator {
       return {};
     }
     std::experimental::suspend_always initial_suspend() { return {}; }
-    std::experimental::suspend_always final_suspend() { return {}; }
+    std::experimental::suspend_always final_suspend() noexcept { return {}; }
     generator get_return_object() { return generator{this}; };
     void return_void() {}
     void unhandled_exception() {}
diff --git a/libcxx/utils/ci/apple-install-libcxx.sh b/libcxx/utils/ci/apple-install-libcxx.sh
index ae20bf3cd1194..7c3517d7c88dd 100755
--- a/libcxx/utils/ci/apple-install-libcxx.sh
+++ b/libcxx/utils/ci/apple-install-libcxx.sh
@@ -118,7 +118,6 @@ for arch in ${architectures}; do
             -GNinja \
             -DCMAKE_MAKE_PROGRAM="$(xcrun --sdk "${sdk}" --find ninja)" \
             -DLLVM_ENABLE_PROJECTS="libcxx;libcxxabi" \
-            -DLLVM_INCLUDE_BENCHMARKS=OFF \
             -C "${cache}" \
             -DCMAKE_INSTALL_PREFIX="${build_dir}/${arch}-install" \
             -DCMAKE_INSTALL_NAME_DIR="${install_name_dir}" \
diff --git a/libcxx/utils/ci/macos-backdeployment.sh b/libcxx/utils/ci/macos-backdeployment.sh
index 839c613868245..d86337e4db376 100755
--- a/libcxx/utils/ci/macos-backdeployment.sh
+++ b/libcxx/utils/ci/macos-backdeployment.sh
@@ -120,8 +120,10 @@ LIBCXX_ROOT_ON_DEPLOYMENT_TARGET="${PREVIOUS_DYLIBS_DIR}/macOS/libc++/${DEPLOYME
 LIBCXXABI_ROOT_ON_DEPLOYMENT_TARGET="${PREVIOUS_DYLIBS_DIR}/macOS/libc++abi/${DEPLOYMENT_TARGET}"
 
 # Filesystem is supported on Apple platforms starting with macosx10.15.
-if [[ ${DEPLOYMENT_TARGET} =~ "^10.9|10.10|10.11|10.12|10.13|10.14$" ]]; then
-    ENABLE_FILESYSTEM="--param enable_filesystem=False"
+if [[ ${DEPLOYMENT_TARGET} =~ ^10.9|10.10|10.11|10.12|10.13|10.14$ ]]; then
+    ENABLE_FILESYSTEM="--param enable_filesystem=false"
+else
+    ENABLE_FILESYSTEM="--param enable_filesystem=true"
 fi
 
 # TODO: We need to also run the tests for libc++abi.
diff --git a/libcxx/utils/libcxx/test/config.py b/libcxx/utils/libcxx/test/config.py
index 41c0b084393d9..99a966dbfdfde 100644
--- a/libcxx/utils/libcxx/test/config.py
+++ b/libcxx/utils/libcxx/test/config.py
@@ -18,8 +18,6 @@
 
 from libcxx.compiler import CXXCompiler
 from libcxx.test.target_info import make_target_info
-from libcxx.test.executor import *
-from libcxx.test.tracing import *
 import libcxx.util
 import libcxx.test.features
 import libcxx.test.params
@@ -76,7 +74,6 @@ def __init__(self, lit_config, config):
         self.use_target = False
         self.use_system_cxx_lib = self.get_lit_bool('use_system_cxx_lib', False)
         self.use_clang_verify = False
-        self.long_tests = None
 
     def get_lit_conf(self, name, default=None):
         val = self.lit_config.params.get(name, None)
@@ -128,7 +125,7 @@ def configure(self):
         self.configure_deployment()
         self.configure_src_root()
         self.configure_obj_root()
-        self.configure_cxx_stdlib_under_test()
+        self.cxx_stdlib_under_test = self.get_lit_conf('cxx_stdlib_under_test', 'libc++')
         self.cxx_library_root = self.get_lit_conf('cxx_library_root', self.libcxx_obj_root)
         self.abi_library_root = self.get_lit_conf('abi_library_path', None)
         self.cxx_runtime_root = self.get_lit_conf('cxx_runtime_root', self.cxx_library_root)
@@ -189,22 +186,8 @@ def get_test_format(self):
             exec_env=self.exec_env)
 
     def configure_executor(self):
-        if self.get_lit_conf('use_old_format'):
-            exec_str = self.get_lit_conf('executor', "None")
-            te = eval(exec_str)
-            if te:
-                self.lit_config.note("Using executor: %r" % exec_str)
-                if self.lit_config.useValgrind:
-                    self.lit_config.fatal("The libc++ test suite can't run under Valgrind with a custom executor")
-            else:
-                te = LocalExecutor()
-
-            te.target_info = self.target_info
-            self.target_info.executor = te
-            self.executor = te
-        else:
-            self.executor = self.get_lit_conf('executor')
-            self.lit_config.note("Using executor: {}".format(self.executor))
+        self.executor = self.get_lit_conf('executor')
+        self.lit_config.note("Using executor: {}".format(self.executor))
 
     def configure_target_info(self):
         self.target_info = make_target_info(self)
@@ -273,22 +256,6 @@ def configure_obj_root(self):
             else:
                 self.libcxx_obj_root = self.project_obj_root
 
-    def configure_cxx_stdlib_under_test(self):
-        self.cxx_stdlib_under_test = self.get_lit_conf(
-            'cxx_stdlib_under_test', 'libc++')
-        if self.cxx_stdlib_under_test not in \
-                ['libc++', 'libstdc++', 'msvc', 'cxx_default']:
-            self.lit_config.fatal(
-                'unsupported value for "cxx_stdlib_under_test": %s'
-                % self.cxx_stdlib_under_test)
-        self.config.available_features.add(self.cxx_stdlib_under_test)
-        if self.cxx_stdlib_under_test == 'libstdc++':
-            # Manually enable the experimental and filesystem tests for libstdc++
-            # if the options aren't present.
-            # FIXME this is a hack.
-            if self.get_lit_conf('enable_experimental') is None:
-                self.config.enable_experimental = 'true'
-
     def configure_features(self):
         additional_features = self.get_lit_conf('additional_features')
         if additional_features:
@@ -317,18 +284,6 @@ def configure_features(self):
             self.config.available_features.add('availability=%s' % name)
             self.config.available_features.add('availability=%s%s' % (name, version))
 
-        # Simulator testing can take a really long time for some of these tests
-        # so add a feature check so we can REQUIRES: long_tests in them
-        self.long_tests = self.get_lit_bool('long_tests')
-        if self.long_tests is None:
-            # Default to running long tests.
-            self.long_tests = True
-            self.lit_config.note(
-                "inferred long_tests as: %r" % self.long_tests)
-
-        if self.long_tests:
-            self.config.available_features.add('long_tests')
-
         if self.target_info.is_windows():
             if self.cxx_stdlib_under_test == 'libc++':
                 # LIBCXX-WINDOWS-FIXME is the feature name used to XFAIL the
@@ -360,38 +315,6 @@ def configure_compile_flags(self):
             self.cxx.compile_flags += shlex.split(additional_flags)
 
     def configure_default_compile_flags(self):
-        # Try and get the std version from the command line. Fall back to
-        # default given in lit.site.cfg is not present. If default is not
-        # present then force c++11.
-        std = self.get_lit_conf('std')
-        if not std:
-            # Choose the newest possible language dialect if none is given.
-            possible_stds = ['c++2a', 'c++17', 'c++1z', 'c++14', 'c++11',
-                             'c++03']
-            if self.cxx.type == 'gcc':
-                maj_v, _, _ = self.cxx.version
-                maj_v = int(maj_v)
-                if maj_v < 7:
-                    possible_stds.remove('c++1z')
-                    possible_stds.remove('c++17')
-                # FIXME: How many C++14 tests actually fail under GCC 5 and 6?
-                # Should we XFAIL them individually instead?
-                if maj_v <= 6:
-                    possible_stds.remove('c++14')
-            for s in possible_stds:
-                if self.cxx.hasCompileFlag('-std=%s' % s):
-                    std = s
-                    self.lit_config.note(
-                        'inferred language dialect as: %s' % std)
-                    break
-            if not std:
-                self.lit_config.fatal(
-                    'Failed to infer a supported language dialect from one of %r'
-                    % possible_stds)
-        self.cxx.compile_flags += ['-std={0}'.format(std)]
-        std_feature = std.replace('gnu++', 'c++')
-        std_feature = std.replace('1z', '17')
-        self.config.available_features.add(std_feature)
         # Configure include paths
         self.configure_compile_flags_header_includes()
         self.target_info.add_cxx_compile_flags(self.cxx.compile_flags)
@@ -504,7 +427,6 @@ def configure_link_flags(self):
             self.configure_link_flags_abi_library()
             self.configure_extra_library_flags()
         elif self.cxx_stdlib_under_test == 'libstdc++':
-            self.config.available_features.add('c++experimental')
             self.cxx.link_flags += ['-lstdc++fs', '-lm', '-pthread']
         elif self.cxx_stdlib_under_test == 'msvc':
             # FIXME: Correctly setup debug/release flags here.
@@ -542,10 +464,6 @@ def configure_link_flags_abi_library_path(self):
                 self.add_path(self.exec_env, self.abi_library_root)
 
     def configure_link_flags_cxx_library(self):
-        libcxx_experimental = self.get_lit_bool('enable_experimental', default=False)
-        if libcxx_experimental:
-            self.config.available_features.add('c++experimental')
-            self.cxx.link_flags += ['-lc++experimental']
         if self.link_shared:
             self.cxx.link_flags += ['-lc++']
         else:
@@ -731,19 +649,13 @@ def configure_modules(self):
 
     def configure_substitutions(self):
         sub = self.config.substitutions
-        # Configure compiler substitutions
         sub.append(('%{cxx}', pipes.quote(self.cxx.path)))
-        sub.append(('%{libcxx_src_root}', self.libcxx_src_root))
-        # Configure flags substitutions
         flags = self.cxx.flags + (self.cxx.modules_flags if self.cxx.use_modules else [])
         compile_flags = self.cxx.compile_flags + (self.cxx.warning_flags if self.cxx.use_warnings else [])
         sub.append(('%{flags}',         ' '.join(map(pipes.quote, flags))))
         sub.append(('%{compile_flags}', ' '.join(map(pipes.quote, compile_flags))))
         sub.append(('%{link_flags}',    ' '.join(map(pipes.quote, self.cxx.link_flags))))
         sub.append(('%{link_libcxxabi}', pipes.quote(self.cxx.link_libcxxabi_flag)))
-
-        # Configure exec prefix substitutions.
-        # Configure run env substitution.
         codesign_ident = self.get_lit_conf('llvm_codesign_identity', '')
         env_vars = ' '.join('%s=%s' % (k, pipes.quote(v)) for (k, v) in self.exec_env.items())
         exec_args = [
@@ -751,8 +663,7 @@ def configure_substitutions(self):
             '--codesign_identity "{}"'.format(codesign_ident),
             '--env {}'.format(env_vars)
         ]
-        if not self.get_lit_conf('use_old_format'):
-            sub.append(('%{exec}', '{} {} -- '.format(self.executor, ' '.join(exec_args))))
+        sub.append(('%{exec}', '{} {} -- '.format(self.executor, ' '.join(exec_args))))
         if self.get_lit_conf('libcxx_gdb'):
             sub.append(('%{libcxx_gdb}', self.get_lit_conf('libcxx_gdb')))
 
diff --git a/libcxx/utils/libcxx/test/dsl.py b/libcxx/utils/libcxx/test/dsl.py
index 9f5152d35fe49..e0e09e6bcac47 100644
--- a/libcxx/utils/libcxx/test/dsl.py
+++ b/libcxx/utils/libcxx/test/dsl.py
@@ -6,7 +6,7 @@
 #
 #===----------------------------------------------------------------------===##
 
-import libcxx.test.newformat
+import libcxx.test.format
 import lit
 import lit.util
 import os
@@ -29,7 +29,7 @@ def _executeScriptInternal(test, commands):
 
   TODO: This really should be easier to access from Lit itself
   """
-  parsedCommands = libcxx.test.newformat.parseScript(test, preamble=commands)
+  parsedCommands = libcxx.test.format.parseScript(test, preamble=commands)
 
   litConfig = lit.LitConfig.LitConfig(
     progname='lit',
@@ -42,7 +42,7 @@ def _executeScriptInternal(test, commands):
     debug=False,
     isWindows=platform.system() == 'Windows',
     params={})
-  _, tmpBase = libcxx.test.newformat._getTempPaths(test)
+  _, tmpBase = libcxx.test.format._getTempPaths(test)
   execDir = os.path.dirname(test.getExecPath())
   for d in (execDir, os.path.dirname(tmpBase)):
     if not os.path.exists(d):
diff --git a/libcxx/utils/libcxx/test/executor.py b/libcxx/utils/libcxx/test/executor.py
deleted file mode 100644
index c34310cdd2e22..0000000000000
--- a/libcxx/utils/libcxx/test/executor.py
+++ /dev/null
@@ -1,258 +0,0 @@
-#===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===----------------------------------------------------------------------===##
-
-import platform
-import os
-import posixpath
-import ntpath
-import shutil
-
-from libcxx.test import tracing
-from libcxx.util import executeCommand
-
-class Executor(object):
-    def __init__(self):
-        self.target_info = None
-
-    def run(self, exe_path, cmd, local_cwd, file_deps=None, env=None):
-        """Execute a command.
-            Be very careful not to change shared state in this function.
-            Executor objects are shared between python processes in `lit -jN`.
-        Args:
-            exe_path: str:    Local path to the executable to be run
-            cmd: [str]:       subprocess.call style command
-            local_cwd: str:   Local path to the working directory
-            file_deps: [str]: Files required by the test
-            env: {str: str}:  Environment variables to execute under
-        Returns:
-            cmd, out, err, exitCode
-        """
-        raise NotImplementedError
-
-    def merge_environments(self, current_env, updated_env):
-        """Merges two execution environments.
-
-        If both environments contain the PATH variables, they are also merged
-        using the proper separator.
-        """
-        result_env = dict(current_env)
-        for k, v in updated_env.items():
-            if k == 'PATH' and self.target_info:
-                self.target_info.add_path(result_env, v)
-            else:
-                result_env[k] = v
-        return result_env
-
-
-class LocalExecutor(Executor):
-    def __init__(self):
-        super(LocalExecutor, self).__init__()
-        self.is_windows = platform.system() == 'Windows'
-
-    def run(self, exe_path, cmd=None, work_dir='.', file_deps=None, env=None):
-        cmd = cmd or [exe_path]
-        if work_dir == '.':
-            work_dir = os.getcwd()
-
-        if env:
-            env = self.merge_environments(os.environ, env)
-
-        for dep in file_deps:
-            if os.path.isdir(dep):
-                shutil.copytree(dep, os.path.join(work_dir, os.path.basename(dep)), symlinks=True)
-            else:
-                shutil.copy2(dep, work_dir)
-
-        out, err, rc = executeCommand(cmd, cwd=work_dir, env=env)
-        return (cmd, out, err, rc)
-
-
-class PrefixExecutor(Executor):
-    """Prefix an executor with some other command wrapper.
-
-    Most useful for setting ulimits on commands, or running an emulator like
-    qemu and valgrind.
-    """
-    def __init__(self, commandPrefix, chain):
-        super(PrefixExecutor, self).__init__()
-
-        self.commandPrefix = commandPrefix
-        self.chain = chain
-
-    def run(self, exe_path, cmd=None, work_dir='.', file_deps=None, env=None):
-        cmd = cmd or [exe_path]
-        return self.chain.run(exe_path, self.commandPrefix + cmd, work_dir,
-                              file_deps, env=env)
-
-
-class PostfixExecutor(Executor):
-    """Postfix an executor with some args."""
-    def __init__(self, commandPostfix, chain):
-        super(PostfixExecutor, self).__init__()
-
-        self.commandPostfix = commandPostfix
-        self.chain = chain
-
-    def run(self, exe_path, cmd=None, work_dir='.', file_deps=None, env=None):
-        cmd = cmd or [exe_path]
-        return self.chain.run(cmd + self.commandPostfix, work_dir, file_deps,
-                              env=env)
-
-
-
-class TimeoutExecutor(PrefixExecutor):
-    """Execute another action under a timeout.
-
-    Deprecated. http://reviews.llvm.org/D6584 adds timeouts to LIT.
-    """
-    def __init__(self, duration, chain):
-        super(TimeoutExecutor, self).__init__(
-            ['timeout', duration], chain)
-
-
-class RemoteExecutor(Executor):
-    def __init__(self):
-        super(RemoteExecutor, self).__init__()
-        self.local_run = executeCommand
-
-    def remote_temp_dir(self):
-        return self._remote_temp(True)
-
-    def remote_temp_file(self):
-        return self._remote_temp(False)
-
-    def _remote_temp(self, is_dir):
-        raise NotImplementedError()
-
-    def copy_in(self, local_srcs, remote_dsts):
-        # This could be wrapped up in a tar->scp->untar for performance
-        # if there are lots of files to be copied/moved
-        for src, dst in zip(local_srcs, remote_dsts):
-            self._copy_in_file(src, dst)
-
-    def _copy_in_file(self, src, dst):
-        raise NotImplementedError()
-
-    def delete_remote(self, remote):
-        try:
-            self.execute_command_remote(['rm', '-rf', remote])
-        except OSError:
-            # TODO: Log failure to delete?
-            pass
-
-    def run(self, exe_path, cmd=None, work_dir='.', file_deps=None, env=None):
-        target_exe_path = None
-        target_cwd = None
-        try:
-            target_cwd = self.remote_temp_dir()
-            executable_name = 'libcxx_test.exe'
-            if self.target_info.is_windows():
-                target_exe_path = ntpath.join(target_cwd, executable_name)
-            else:
-                target_exe_path = posixpath.join(target_cwd, executable_name)
-
-            if cmd:
-                # Replace exe_path with target_exe_path.
-                cmd = [c if c != exe_path else target_exe_path for c in cmd]
-            else:
-                cmd = [target_exe_path]
-
-            srcs = [exe_path]
-            dsts = [target_exe_path]
-            if file_deps is not None:
-                dev_paths = [os.path.join(target_cwd, os.path.basename(f))
-                             for f in file_deps]
-                srcs.extend(file_deps)
-                dsts.extend(dev_paths)
-            self.copy_in(srcs, dsts)
-
-            # When testing executables that were cross-compiled on Windows for
-            # Linux, we may need to explicitly set the execution permission to
-            # avoid the 'Permission denied' error:
-            chmod_cmd = ['chmod', '+x', target_exe_path]
-
-            # TODO(jroelofs): capture the copy_in and delete_remote commands,
-            # and conjugate them with '&&'s around the first tuple element
-            # returned here:
-            return self.execute_command_remote(chmod_cmd + ['&&'] + cmd,
-                                               target_cwd,
-                                               env)
-        finally:
-            if target_cwd:
-                self.delete_remote(target_cwd)
-
-    def execute_command_remote(self, cmd, remote_work_dir='.', env=None):
-        raise NotImplementedError()
-
-
-class SSHExecutor(RemoteExecutor):
-    def __init__(self, host, username=None):
-        super(SSHExecutor, self).__init__()
-
-        self.user_prefix = username + '@' if username else ''
-        self.host = host
-        self.scp_command = 'scp'
-        self.ssh_command = 'ssh'
-
-        # TODO(jroelofs): switch this on some -super-verbose-debug config flag
-        if False:
-            self.local_run = tracing.trace_function(
-                self.local_run, log_calls=True, log_results=True,
-                label='ssh_local')
-
-    def _remote_temp(self, is_dir):
-        # TODO: detect what the target system is, and use the correct
-        # mktemp command for it. (linux and darwin differ here, and I'm
-        # sure windows has another way to do it)
-
-        # Not sure how to do suffix on osx yet
-        dir_arg = '-d' if is_dir else ''
-        cmd = 'mktemp -q {} /tmp/libcxx.XXXXXXXXXX'.format(dir_arg)
-        _, temp_path, err, exitCode = self.execute_command_remote([cmd])
-        temp_path = temp_path.strip()
-        if exitCode != 0:
-            raise RuntimeError(err)
-        return temp_path
-
-    def _copy_in_file(self, src, dst):
-        scp = self.scp_command
-        remote = self.host
-        remote = self.user_prefix + remote
-        cmd = [scp, '-p', src, remote + ':' + dst]
-        self.local_run(cmd)
-
-    def _export_command(self, env):
-        if not env:
-            return []
-
-        export_cmd = ['export']
-
-        for k, v in env.items():
-            v = v.replace('\\', '\\\\')
-            if k == 'PATH':
-                # Pick up the existing paths, so we don't lose any commands
-                if self.target_info and self.target_info.is_windows():
-                    export_cmd.append('PATH="%s;%PATH%"' % v)
-                else:
-                    export_cmd.append('PATH="%s:$PATH"' % v)
-            else:
-                export_cmd.append('"%s"="%s"' % (k, v))
-
-        return export_cmd
-
-    def execute_command_remote(self, cmd, remote_work_dir='.', env=None):
-        remote = self.user_prefix + self.host
-        ssh_cmd = [self.ssh_command, '-oBatchMode=yes', remote]
-        export_cmd = self._export_command(env)
-        remote_cmd = ' '.join(cmd)
-        if export_cmd:
-            remote_cmd = ' '.join(export_cmd) + ' && ' + remote_cmd
-        if remote_work_dir != '.':
-            remote_cmd = 'cd ' + remote_work_dir + ' && ' + remote_cmd
-        out, err, rc = self.local_run(ssh_cmd + [remote_cmd])
-        return (remote_cmd, out, err, rc)
diff --git a/libcxx/utils/libcxx/test/features.py b/libcxx/utils/libcxx/test/features.py
index 5229cf67b9596..f3d8e782be8e1 100644
--- a/libcxx/utils/libcxx/test/features.py
+++ b/libcxx/utils/libcxx/test/features.py
@@ -110,22 +110,10 @@
   ]
 
 
-# Add a feature representing the platform name: darwin, linux, windows, etc...
+# Add features representing the platform name: darwin, linux, windows, etc...
 features += [
-  Feature(name=lambda cfg: programOutput(cfg, """
-    #include <cstdio>
-    int main() {
-    #if defined(__APPLE__)
-      std::printf("darwin");
-    #elif defined(_WIN32)
-      std::printf("windows");
-    #elif defined(__NetBSD__)
-      std::printf("netbsd");
-    #elif defined(__linux__)
-      std::printf("linux");
-    #else
-      std::printf("unknown-platform");
-    #endif
-    }
-  """))
+  Feature(name='darwin', when=lambda cfg: '__APPLE__' in compilerMacros(cfg)),
+  Feature(name='windows', when=lambda cfg: '_WIN32' in compilerMacros(cfg)),
+  Feature(name='linux', when=lambda cfg: '__linux__' in compilerMacros(cfg)),
+  Feature(name='netbsd', when=lambda cfg: '__NetBSD__' in compilerMacros(cfg))
 ]
diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py
index 207df2322be9b..4390166ca6bf5 100644
--- a/libcxx/utils/libcxx/test/format.py
+++ b/libcxx/utils/libcxx/test/format.py
@@ -6,258 +6,311 @@
 #
 #===----------------------------------------------------------------------===##
 
-import copy
-import errno
+import lit
+import lit.formats
 import os
+import pipes
+import re
 import shutil
-import tempfile
-import time
-import random
+import subprocess
 
-import lit.Test        # pylint: disable=import-error
-import lit.TestRunner  # pylint: disable=import-error
-from lit.TestRunner import ParserKind, IntegratedTestKeywordParser  \
-    # pylint: disable=import-error
+def _supportsVerify(config):
+    """
+    Determine whether clang-verify is supported by the given configuration.
+
+    This is done by checking whether the %{cxx} substitution in that
+    configuration supports certain compiler flags.
+    """
+    command = "%{{cxx}} -xc++ {} -Werror -fsyntax-only -Xclang -verify-ignore-unexpected".format(os.devnull)
+    command = lit.TestRunner.applySubstitutions([command], config.substitutions,
+                                                recursion_limit=config.recursiveExpansionLimit)[0]
+    devNull = open(os.devnull, 'w')
+    result = subprocess.call(command, shell=True, stdout=devNull, stderr=devNull)
+    return result == 0
+
+def _getTempPaths(test):
+    """
+    Return the values to use for the %T and %t substitutions, respectively.
+
+    The difference between this and Lit's default behavior is that we guarantee
+    that %T is a path unique to the test being run.
+    """
+    tmpDir, _ = lit.TestRunner.getTempPaths(test)
+    _, testName = os.path.split(test.getExecPath())
+    tmpDir = os.path.join(tmpDir, testName + '.dir')
+    tmpBase = os.path.join(tmpDir, 't')
+    return tmpDir, tmpBase
+
+def parseScript(test, preamble):
+    """
+    Extract the script from a test, with substitutions applied.
 
-from libcxx.test.executor import LocalExecutor as LocalExecutor
-from libcxx.test.executor import SSHExecutor as SSHExecutor
-import libcxx.util
+    Returns a list of commands ready to be executed.
 
+    - test
+        The lit.Test to parse.
 
-class LibcxxTestFormat(object):
+    - preamble
+        A list of commands to perform before any command in the test.
+        These commands can contain unexpanded substitutions, but they
+        must not be of the form 'RUN:' -- they must be proper commands
+        once substituted.
     """
-    Custom test format handler for use with the test format use by libc++.
 
-    Tests fall into two categories:
-      FOO.pass.cpp - Executable test which should compile, run, and exit with
-                     code 0.
-      FOO.fail.cpp - Negative test case which is expected to fail compilation.
-      FOO.sh.cpp   - A test that uses LIT's ShTest format.
+    # Get the default substitutions
+    tmpDir, tmpBase = _getTempPaths(test)
+    useExternalSh = True
+    substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase,
+                                                           normalize_slashes=useExternalSh)
+
+    # Add the %{build} and %{run} convenience substitutions
+    substitutions.append(('%{build}', '%{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe'))
+    substitutions.append(('%{run}', '%{exec} %t.exe'))
+
+    # Parse the test file, including custom directives
+    additionalCompileFlags = []
+    fileDependencies = []
+    parsers = [
+        lit.TestRunner.IntegratedTestKeywordParser('FILE_DEPENDENCIES:',
+                                                   lit.TestRunner.ParserKind.LIST,
+                                                   initial_value=fileDependencies),
+        lit.TestRunner.IntegratedTestKeywordParser('ADDITIONAL_COMPILE_FLAGS:',
+                                                   lit.TestRunner.ParserKind.LIST,
+                                                   initial_value=additionalCompileFlags)
+    ]
+
+    scriptInTest = lit.TestRunner.parseIntegratedTestScript(test, additional_parsers=parsers,
+                                                            require_script=not preamble)
+    if isinstance(scriptInTest, lit.Test.Result):
+        return scriptInTest
+
+    script = []
+
+    # For each file dependency in FILE_DEPENDENCIES, inject a command to copy
+    # that file to the execution directory. Execute the copy from %S to allow
+    # relative paths from the test directory.
+    for dep in fileDependencies:
+        script += ['%dbg(SETUP) cd %S && cp {} %T'.format(dep)]
+    script += preamble
+    script += scriptInTest
+
+    # Add compile flags specified with ADDITIONAL_COMPILE_FLAGS.
+    substitutions = [(s, x + ' ' + ' '.join(additionalCompileFlags)) if s == '%{compile_flags}'
+                            else (s, x) for (s, x) in substitutions]
+
+    # Perform substitutions in the script itself.
+    script = lit.TestRunner.applySubstitutions(script, substitutions,
+                                               recursion_limit=test.config.recursiveExpansionLimit)
+
+    return script
+
+
+class CxxStandardLibraryTest(lit.formats.TestFormat):
     """
+    Lit test format for the C++ Standard Library conformance test suite.
 
-    def __init__(self, cxx, use_verify_for_fail,
-                 executor, exec_env):
-        self.cxx = copy.deepcopy(cxx)
-        self.use_verify_for_fail = use_verify_for_fail
-        self.executor = executor
-        self.exec_env = dict(exec_env)
-
-    @staticmethod
-    def _make_custom_parsers(test):
-        return [
-            IntegratedTestKeywordParser('FILE_DEPENDENCIES:', ParserKind.LIST,
-                                        initial_value=test.file_dependencies),
-            IntegratedTestKeywordParser('ADDITIONAL_COMPILE_FLAGS:', ParserKind.LIST,
-                                        initial_value=[])
-        ]
+    This test format is based on top of the ShTest format -- it basically
+    creates a shell script performing the right operations (compile/link/run)
+    based on the extension of the test file it encounters. It supports files
+    with the following extensions:
 
-    # Utility function to add compile flags in lit.local.cfg files.
-    def addCompileFlags(self, config, *flags):
-        self.cxx = copy.deepcopy(self.cxx)
-        self.cxx.compile_flags += flags
-
-    @staticmethod
-    def _get_parser(key, parsers):
-        for p in parsers:
-            if p.keyword == key:
-                return p
-        assert False and "parser not found"
-
-    # TODO: Move this into lit's FileBasedTest
-    def getTestsInDirectory(self, testSuite, path_in_suite,
-                            litConfig, localConfig):
-        source_path = testSuite.getSourcePath(path_in_suite)
-        for filename in os.listdir(source_path):
+    FOO.pass.cpp            - Compiles, links and runs successfully
+    FOO.pass.mm             - Same as .pass.cpp, but for Objective-C++
+    FOO.run.fail.cpp        - Compiles and links successfully, but fails at runtime
+
+    FOO.compile.pass.cpp    - Compiles successfully, link and run not attempted
+    FOO.compile.fail.cpp    - Does not compile successfully
+
+    FOO.link.pass.cpp       - Compiles and links successfully, run not attempted
+    FOO.link.fail.cpp       - Compiles successfully, but fails to link
+
+    FOO.sh.<anything>       - A builtin Lit Shell test
+
+    FOO.verify.cpp          - Compiles with clang-verify. This type of test is
+                              automatically marked as UNSUPPORTED if the compiler
+                              does not support Clang-verify.
+
+    FOO.fail.cpp            - Compiled with clang-verify if clang-verify is
+                              supported, and equivalent to a .compile.fail.cpp
+                              test otherwise. This is supported only for backwards
+                              compatibility with the test suite.
+
+
+    Substitution requirements
+    ===============================
+    The test format operates by assuming that each test's configuration provides
+    the following substitutions, which it will reuse in the shell scripts it
+    constructs:
+        %{cxx}           - A command that can be used to invoke the compiler
+        %{compile_flags} - Flags to use when compiling a test case
+        %{link_flags}    - Flags to use when linking a test case
+        %{flags}         - Flags to use either when compiling or linking a test case
+        %{exec}          - A command to prefix the execution of executables
+
+    Note that when building an executable (as opposed to only compiling a source
+    file), all three of %{flags}, %{compile_flags} and %{link_flags} will be used
+    in the same command line. In other words, the test format doesn't perform
+    separate compilation and linking steps in this case.
+
+
+    Additional supported directives
+    ===============================
+    In addition to everything that's supported in Lit ShTests, this test format
+    also understands the following directives inside test files:
+
+        // FILE_DEPENDENCIES: file, directory, /path/to/file
+
+            This directive expresses that the test requires the provided files
+            or directories in order to run. An example is a test that requires
+            some test input stored in a data file. When a test file contains
+            such a directive, this test format will collect them and copy them
+            to the directory represented by %T. The intent is that %T contains
+            all the inputs necessary to run the test, such that e.g. execution
+            on a remote host can be done by simply copying %T to the host.
+
+        // ADDITIONAL_COMPILE_FLAGS: flag1, flag2, flag3
+
+            This directive will cause the provided flags to be added to the
+            %{compile_flags} substitution for the test that contains it. This
+            allows adding special compilation flags without having to use a
+            .sh.cpp test, which would be more powerful but perhaps overkill.
+
+
+    Additional provided substitutions and features
+    ==============================================
+    The test format will define the following substitutions for use inside tests:
+
+        %{build}
+            Expands to a command-line that builds the current source
+            file with the %{flags}, %{compile_flags} and %{link_flags}
+            substitutions, and that produces an executable named %t.exe.
+
+        %{run}
+            Equivalent to `%{exec} %t.exe`. This is intended to be used
+            in conjunction with the %{build} substitution.
+    """
+    def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig):
+        SUPPORTED_SUFFIXES = ['[.]pass[.]cpp$', '[.]pass[.]mm$', '[.]run[.]fail[.]cpp$',
+                              '[.]compile[.]pass[.]cpp$', '[.]compile[.]fail[.]cpp$',
+                              '[.]link[.]pass[.]cpp$', '[.]link[.]fail[.]cpp$',
+                              '[.]sh[.][^.]+$',
+                              '[.]verify[.]cpp$',
+                              '[.]fail[.]cpp$']
+        sourcePath = testSuite.getSourcePath(pathInSuite)
+        for filename in os.listdir(sourcePath):
             # Ignore dot files and excluded tests.
             if filename.startswith('.') or filename in localConfig.excludes:
                 continue
 
-            filepath = os.path.join(source_path, filename)
+            filepath = os.path.join(sourcePath, filename)
             if not os.path.isdir(filepath):
-                if any([filename.endswith(ext)
-                        for ext in localConfig.suffixes]):
-                    yield lit.Test.Test(testSuite, path_in_suite + (filename,),
-                                        localConfig)
-
-    def execute(self, test, lit_config):
-        while True:
-            try:
-                return self._execute(test, lit_config)
-            except OSError as oe:
-                if oe.errno != errno.ETXTBSY:
-                    raise
-                time.sleep(0.1)
-
-    def _execute(self, test, lit_config):
-        name = test.path_in_suite[-1]
-        name_root, name_ext = os.path.splitext(name)
-        is_libcxx_test = test.path_in_suite[0] == 'libcxx'
-        is_sh_test = name_root.endswith('.sh')
-        is_pass_test = name.endswith('.pass.cpp') or name.endswith('.pass.mm')
-        is_fail_test = name.endswith('.fail.cpp')
-        is_objcxx_test = name.endswith('.mm')
-        assert is_sh_test or name_ext == '.cpp' or name_ext == '.mm', \
-            'non-cpp file must be sh test'
+                if any([re.search(ext, filename) for ext in SUPPORTED_SUFFIXES]):
+                    yield lit.Test.Test(testSuite, pathInSuite + (filename,), localConfig)
 
-        if test.config.unsupported:
-            return (lit.Test.UNSUPPORTED,
-                    "A lit.local.cfg marked this unsupported")
+    def _checkBaseSubstitutions(self, substitutions):
+        substitutions = [s for (s, _) in substitutions]
+        for s in ['%{cxx}', '%{compile_flags}', '%{link_flags}', '%{flags}', '%{exec}']:
+            assert s in substitutions, "Required substitution {} was not provided".format(s)
 
-        if is_objcxx_test and not \
-           'objective-c++' in test.config.available_features:
-            return (lit.Test.UNSUPPORTED, "Objective-C++ is not supported")
+    def _disableWithModules(self, test):
+        with open(test.getSourcePath(), 'rb') as f:
+            contents = f.read()
+        return b'#define _LIBCPP_ASSERT' in contents
+
+    def execute(self, test, litConfig):
+        self._checkBaseSubstitutions(test.config.substitutions)
+        VERIFY_FLAGS = '-Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0'
+        supportsVerify = _supportsVerify(test.config)
+        filename = test.path_in_suite[-1]
+
+        # TODO(ldionne): We currently disable tests that re-define _LIBCPP_ASSERT
+        #                when we run with modules enabled. Instead, we should
+        #                split the part that does a death test outside of the
+        #                test, and only disable that part when modules are
+        #                enabled.
+        if '-fmodules' in test.config.available_features and self._disableWithModules(test):
+            return lit.Test.Result(lit.Test.UNSUPPORTED, 'Test {} is unsupported when modules are enabled')
+
+        if re.search('[.]sh[.][^.]+$', filename):
+            steps = [ ] # The steps are already in the script
+            return self._executeShTest(test, litConfig, steps)
+        elif filename.endswith('.compile.pass.cpp'):
+            steps = [
+                "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -fsyntax-only"
+            ]
+            return self._executeShTest(test, litConfig, steps)
+        elif filename.endswith('.compile.fail.cpp'):
+            steps = [
+                "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only"
+            ]
+            return self._executeShTest(test, litConfig, steps)
+        elif filename.endswith('.link.pass.cpp'):
+            steps = [
+                "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe"
+            ]
+            return self._executeShTest(test, litConfig, steps)
+        elif filename.endswith('.link.fail.cpp'):
+            steps = [
+                "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -c -o %t.o",
+                "%dbg(LINKED WITH) ! %{cxx} %t.o %{flags} %{link_flags} -o %t.exe"
+            ]
+            return self._executeShTest(test, litConfig, steps)
+        elif filename.endswith('.run.fail.cpp'):
+            steps = [
+                "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe",
+                "%dbg(EXECUTED AS) %{exec} ! %t.exe"
+            ]
+            return self._executeShTest(test, litConfig, steps)
+        elif filename.endswith('.verify.cpp'):
+            if not supportsVerify:
+                return lit.Test.Result(lit.Test.UNSUPPORTED,
+                    "Test {} requires support for Clang-verify, which isn't supported by the compiler".format(test.getFullName()))
+            steps = [
+                # Note: Use -Wno-error to make sure all diagnostics are not treated as errors,
+                #       which doesn't make sense for clang-verify tests.
+                "%dbg(COMPILED WITH) %{{cxx}} %s %{{flags}} %{{compile_flags}} -fsyntax-only -Wno-error {}".format(VERIFY_FLAGS)
+            ]
+            return self._executeShTest(test, litConfig, steps)
+        # Make sure to check these ones last, since they will match other
+        # suffixes above too.
+        elif filename.endswith('.pass.cpp') or filename.endswith('.pass.mm'):
+            steps = [
+                "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe",
+                "%dbg(EXECUTED AS) %{exec} %t.exe"
+            ]
+            return self._executeShTest(test, litConfig, steps)
+        # This is like a .verify.cpp test when clang-verify is supported,
+        # otherwise it's like a .compile.fail.cpp test. This is only provided
+        # for backwards compatibility with the test suite.
+        elif filename.endswith('.fail.cpp'):
+            if supportsVerify:
+                steps = [
+                    "%dbg(COMPILED WITH) %{{cxx}} %s %{{flags}} %{{compile_flags}} -fsyntax-only -Wno-error {}".format(VERIFY_FLAGS)
+                ]
+            else:
+                steps = [
+                    "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only"
+                ]
+            return self._executeShTest(test, litConfig, steps)
+        else:
+            return lit.Test.Result(lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename))
 
-        setattr(test, 'file_dependencies', [])
-        parsers = self._make_custom_parsers(test)
-        script = lit.TestRunner.parseIntegratedTestScript(
-            test, additional_parsers=parsers, require_script=is_sh_test)
+    # Utility function to add compile flags in lit.local.cfg files.
+    def addCompileFlags(self, config, *flags):
+        string = ' '.join(flags)
+        config.substitutions = [(s, x + ' ' + string) if s == '%{compile_flags}' else (s, x) for (s, x) in config.substitutions]
 
-        # Check if a result for the test was returned. If so return that
-        # result.
+    def _executeShTest(self, test, litConfig, steps):
+        if test.config.unsupported:
+            return lit.Test.Result(lit.Test.UNSUPPORTED, 'Test is unsupported')
+
+        script = parseScript(test, steps)
         if isinstance(script, lit.Test.Result):
             return script
-        if lit_config.noExecute:
-            return lit.Test.Result(lit.Test.PASS)
-
-        # Check that we don't have run lines on tests that don't support them.
-        if not is_sh_test and len(script) != 0:
-            lit_config.fatal('Unsupported RUN line found in test %s' % name)
-
-        tmpDir, tmpBase = lit.TestRunner.getTempPaths(test)
-        substitutions = lit.TestRunner.getDefaultSubstitutions(
-            test, tmpDir, tmpBase, normalize_slashes=True)
-
-        # Apply substitutions in FILE_DEPENDENCIES markup
-        data_files = lit.TestRunner.applySubstitutions(test.file_dependencies, substitutions,
-                                                       recursion_limit=test.config.recursiveExpansionLimit)
-        local_cwd = os.path.dirname(test.getSourcePath())
-        data_files = [f if os.path.isabs(f) else os.path.join(local_cwd, f) for f in data_files]
-        substitutions.append(('%{file_dependencies}', ' '.join(data_files)))
-
-        # Add other convenience substitutions
-        substitutions.append(('%{build}', '%{cxx} -o %t.exe %s %{flags} %{compile_flags} %{link_flags}'))
-        substitutions.append(('%{run}', '%{exec} %t.exe'))
-
-        script = lit.TestRunner.applySubstitutions(script, substitutions,
-                                                   recursion_limit=test.config.recursiveExpansionLimit)
-
-        test_cxx = copy.deepcopy(self.cxx)
-        if is_fail_test:
-            test_cxx.useCCache(False)
-            test_cxx.useWarnings(False)
-        if '-fmodules' in test.config.available_features:
-            test_cxx.addWarningFlagIfSupported('-Wno-macro-redefined')
-            # FIXME: libc++ debug tests #define _LIBCPP_ASSERT to override it
-            # If we see this we need to build the test against uniquely built
-            # modules.
-            if is_libcxx_test:
-                with open(test.getSourcePath(), 'rb') as f:
-                    contents = f.read()
-                if b'#define _LIBCPP_ASSERT' in contents:
-                    test_cxx.useModules(False)
-
-        # Handle ADDITIONAL_COMPILE_FLAGS keywords by adding those compilation
-        # flags, but first perform substitutions in those flags.
-        extra_compile_flags = self._get_parser('ADDITIONAL_COMPILE_FLAGS:', parsers).getValue()
-        extra_compile_flags = lit.TestRunner.applySubstitutions(extra_compile_flags, substitutions)
-        test_cxx.compile_flags.extend(extra_compile_flags)
-
-        if is_objcxx_test:
-            test_cxx.source_lang = 'objective-c++'
-            test_cxx.link_flags += ['-framework', 'Foundation']
-
-        # Dispatch the test based on its suffix.
-        if is_sh_test:
-            if not isinstance(self.executor, LocalExecutor) and not isinstance(self.executor, SSHExecutor):
-                # We can't run ShTest tests with other executors than
-                # LocalExecutor and SSHExecutor yet.
-                # For now, bail on trying to run them
-                return lit.Test.UNSUPPORTED, 'ShTest format not yet supported'
-            test.config.environment = self.executor.merge_environments(os.environ, self.exec_env)
-            return lit.TestRunner._runShTest(test, lit_config,
-                                             True, script,
-                                             tmpBase)
-        elif is_fail_test:
-            return self._evaluate_fail_test(test, test_cxx, parsers)
-        elif is_pass_test:
-            return self._evaluate_pass_test(test, tmpBase, lit_config,
-                                            test_cxx, parsers, data_files)
-        else:
-            # No other test type is supported
-            assert False
-
-    def _clean(self, exec_path):  # pylint: disable=no-self-use
-        libcxx.util.cleanFile(exec_path)
-
-    def _evaluate_pass_test(self, test, tmpBase, lit_config,
-                            test_cxx, parsers, data_files):
-        execDir = os.path.dirname(test.getExecPath())
-        source_path = test.getSourcePath()
-        exec_path = tmpBase + '.exe'
-        object_path = tmpBase + '.o'
-        # Create the output directory if it does not already exist.
-        libcxx.util.mkdir_p(os.path.dirname(tmpBase))
-        try:
-            # Compile the test
-            cmd, out, err, rc = test_cxx.compileLinkTwoSteps(
-                source_path, out=exec_path, object_file=object_path,
-                cwd=execDir)
-            compile_cmd = cmd
-            if rc != 0:
-                report = libcxx.util.makeReport(cmd, out, err, rc)
-                report += "Compilation failed unexpectedly!"
-                return lit.Test.Result(lit.Test.FAIL, report)
-            # Run the test
-            env = None
-            if self.exec_env:
-                env = self.exec_env
-
-            max_retry = test.allowed_retries + 1
-            for retry_count in range(max_retry):
-                # Create a temporary directory just for that test and run the
-                # test in that directory
-                try:
-                    execDirTmp = tempfile.mkdtemp(dir=execDir)
-                    cmd, out, err, rc = self.executor.run(exec_path, [exec_path],
-                                                          execDirTmp, data_files,
-                                                          env)
-                finally:
-                    shutil.rmtree(execDirTmp)
-                report = "Compiled With: '%s'\n" % ' '.join(compile_cmd)
-                report += libcxx.util.makeReport(cmd, out, err, rc)
-                if rc == 0:
-                    res = lit.Test.PASS if retry_count == 0 else lit.Test.FLAKYPASS
-                    return lit.Test.Result(res, report)
-                elif rc != 0 and retry_count + 1 == max_retry:
-                    report += "Compiled test failed unexpectedly!"
-                    return lit.Test.Result(lit.Test.FAIL, report)
-
-            assert False # Unreachable
-        finally:
-            # Note that cleanup of exec_file happens in `_clean()`. If you
-            # override this, cleanup is your reponsibility.
-            libcxx.util.cleanFile(object_path)
-            self._clean(exec_path)
-
-    def _evaluate_fail_test(self, test, test_cxx, parsers):
-        source_path = test.getSourcePath()
-        # FIXME: lift this detection into LLVM/LIT.
-        with open(source_path, 'rb') as f:
-            contents = f.read()
-        verify_tags = [b'expected-note', b'expected-remark',
-                       b'expected-warning', b'expected-error',
-                       b'expected-no-diagnostics']
-        use_verify = self.use_verify_for_fail and \
-                     any([tag in contents for tag in verify_tags])
-        test_cxx.flags += ['-fsyntax-only']
-        if use_verify:
-            test_cxx.useVerify()
-        cmd, out, err, rc = test_cxx.compile(source_path, out=os.devnull)
-        check_rc = lambda rc: rc == 0 if use_verify else rc != 0
-        report = libcxx.util.makeReport(cmd, out, err, rc)
-        if check_rc(rc):
-            return lit.Test.Result(lit.Test.PASS, report)
+
+        if litConfig.noExecute:
+            return lit.Test.Result(lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS)
         else:
-            report += ('Expected compilation to fail!\n' if not use_verify else
-                       'Expected compilation using verify to pass!\n')
-            return lit.Test.Result(lit.Test.FAIL, report)
+            _, tmpBase = _getTempPaths(test)
+            useExternalSh = True
+            return lit.TestRunner._runShTest(test, litConfig, useExternalSh, script, tmpBase)
diff --git a/libcxx/utils/libcxx/test/newformat.py b/libcxx/utils/libcxx/test/newformat.py
deleted file mode 100644
index 4390166ca6bf5..0000000000000
--- a/libcxx/utils/libcxx/test/newformat.py
+++ /dev/null
@@ -1,316 +0,0 @@
-#===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===----------------------------------------------------------------------===##
-
-import lit
-import lit.formats
-import os
-import pipes
-import re
-import shutil
-import subprocess
-
-def _supportsVerify(config):
-    """
-    Determine whether clang-verify is supported by the given configuration.
-
-    This is done by checking whether the %{cxx} substitution in that
-    configuration supports certain compiler flags.
-    """
-    command = "%{{cxx}} -xc++ {} -Werror -fsyntax-only -Xclang -verify-ignore-unexpected".format(os.devnull)
-    command = lit.TestRunner.applySubstitutions([command], config.substitutions,
-                                                recursion_limit=config.recursiveExpansionLimit)[0]
-    devNull = open(os.devnull, 'w')
-    result = subprocess.call(command, shell=True, stdout=devNull, stderr=devNull)
-    return result == 0
-
-def _getTempPaths(test):
-    """
-    Return the values to use for the %T and %t substitutions, respectively.
-
-    The difference between this and Lit's default behavior is that we guarantee
-    that %T is a path unique to the test being run.
-    """
-    tmpDir, _ = lit.TestRunner.getTempPaths(test)
-    _, testName = os.path.split(test.getExecPath())
-    tmpDir = os.path.join(tmpDir, testName + '.dir')
-    tmpBase = os.path.join(tmpDir, 't')
-    return tmpDir, tmpBase
-
-def parseScript(test, preamble):
-    """
-    Extract the script from a test, with substitutions applied.
-
-    Returns a list of commands ready to be executed.
-
-    - test
-        The lit.Test to parse.
-
-    - preamble
-        A list of commands to perform before any command in the test.
-        These commands can contain unexpanded substitutions, but they
-        must not be of the form 'RUN:' -- they must be proper commands
-        once substituted.
-    """
-
-    # Get the default substitutions
-    tmpDir, tmpBase = _getTempPaths(test)
-    useExternalSh = True
-    substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase,
-                                                           normalize_slashes=useExternalSh)
-
-    # Add the %{build} and %{run} convenience substitutions
-    substitutions.append(('%{build}', '%{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe'))
-    substitutions.append(('%{run}', '%{exec} %t.exe'))
-
-    # Parse the test file, including custom directives
-    additionalCompileFlags = []
-    fileDependencies = []
-    parsers = [
-        lit.TestRunner.IntegratedTestKeywordParser('FILE_DEPENDENCIES:',
-                                                   lit.TestRunner.ParserKind.LIST,
-                                                   initial_value=fileDependencies),
-        lit.TestRunner.IntegratedTestKeywordParser('ADDITIONAL_COMPILE_FLAGS:',
-                                                   lit.TestRunner.ParserKind.LIST,
-                                                   initial_value=additionalCompileFlags)
-    ]
-
-    scriptInTest = lit.TestRunner.parseIntegratedTestScript(test, additional_parsers=parsers,
-                                                            require_script=not preamble)
-    if isinstance(scriptInTest, lit.Test.Result):
-        return scriptInTest
-
-    script = []
-
-    # For each file dependency in FILE_DEPENDENCIES, inject a command to copy
-    # that file to the execution directory. Execute the copy from %S to allow
-    # relative paths from the test directory.
-    for dep in fileDependencies:
-        script += ['%dbg(SETUP) cd %S && cp {} %T'.format(dep)]
-    script += preamble
-    script += scriptInTest
-
-    # Add compile flags specified with ADDITIONAL_COMPILE_FLAGS.
-    substitutions = [(s, x + ' ' + ' '.join(additionalCompileFlags)) if s == '%{compile_flags}'
-                            else (s, x) for (s, x) in substitutions]
-
-    # Perform substitutions in the script itself.
-    script = lit.TestRunner.applySubstitutions(script, substitutions,
-                                               recursion_limit=test.config.recursiveExpansionLimit)
-
-    return script
-
-
-class CxxStandardLibraryTest(lit.formats.TestFormat):
-    """
-    Lit test format for the C++ Standard Library conformance test suite.
-
-    This test format is based on top of the ShTest format -- it basically
-    creates a shell script performing the right operations (compile/link/run)
-    based on the extension of the test file it encounters. It supports files
-    with the following extensions:
-
-    FOO.pass.cpp            - Compiles, links and runs successfully
-    FOO.pass.mm             - Same as .pass.cpp, but for Objective-C++
-    FOO.run.fail.cpp        - Compiles and links successfully, but fails at runtime
-
-    FOO.compile.pass.cpp    - Compiles successfully, link and run not attempted
-    FOO.compile.fail.cpp    - Does not compile successfully
-
-    FOO.link.pass.cpp       - Compiles and links successfully, run not attempted
-    FOO.link.fail.cpp       - Compiles successfully, but fails to link
-
-    FOO.sh.<anything>       - A builtin Lit Shell test
-
-    FOO.verify.cpp          - Compiles with clang-verify. This type of test is
-                              automatically marked as UNSUPPORTED if the compiler
-                              does not support Clang-verify.
-
-    FOO.fail.cpp            - Compiled with clang-verify if clang-verify is
-                              supported, and equivalent to a .compile.fail.cpp
-                              test otherwise. This is supported only for backwards
-                              compatibility with the test suite.
-
-
-    Substitution requirements
-    ===============================
-    The test format operates by assuming that each test's configuration provides
-    the following substitutions, which it will reuse in the shell scripts it
-    constructs:
-        %{cxx}           - A command that can be used to invoke the compiler
-        %{compile_flags} - Flags to use when compiling a test case
-        %{link_flags}    - Flags to use when linking a test case
-        %{flags}         - Flags to use either when compiling or linking a test case
-        %{exec}          - A command to prefix the execution of executables
-
-    Note that when building an executable (as opposed to only compiling a source
-    file), all three of %{flags}, %{compile_flags} and %{link_flags} will be used
-    in the same command line. In other words, the test format doesn't perform
-    separate compilation and linking steps in this case.
-
-
-    Additional supported directives
-    ===============================
-    In addition to everything that's supported in Lit ShTests, this test format
-    also understands the following directives inside test files:
-
-        // FILE_DEPENDENCIES: file, directory, /path/to/file
-
-            This directive expresses that the test requires the provided files
-            or directories in order to run. An example is a test that requires
-            some test input stored in a data file. When a test file contains
-            such a directive, this test format will collect them and copy them
-            to the directory represented by %T. The intent is that %T contains
-            all the inputs necessary to run the test, such that e.g. execution
-            on a remote host can be done by simply copying %T to the host.
-
-        // ADDITIONAL_COMPILE_FLAGS: flag1, flag2, flag3
-
-            This directive will cause the provided flags to be added to the
-            %{compile_flags} substitution for the test that contains it. This
-            allows adding special compilation flags without having to use a
-            .sh.cpp test, which would be more powerful but perhaps overkill.
-
-
-    Additional provided substitutions and features
-    ==============================================
-    The test format will define the following substitutions for use inside tests:
-
-        %{build}
-            Expands to a command-line that builds the current source
-            file with the %{flags}, %{compile_flags} and %{link_flags}
-            substitutions, and that produces an executable named %t.exe.
-
-        %{run}
-            Equivalent to `%{exec} %t.exe`. This is intended to be used
-            in conjunction with the %{build} substitution.
-    """
-    def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig):
-        SUPPORTED_SUFFIXES = ['[.]pass[.]cpp$', '[.]pass[.]mm$', '[.]run[.]fail[.]cpp$',
-                              '[.]compile[.]pass[.]cpp$', '[.]compile[.]fail[.]cpp$',
-                              '[.]link[.]pass[.]cpp$', '[.]link[.]fail[.]cpp$',
-                              '[.]sh[.][^.]+$',
-                              '[.]verify[.]cpp$',
-                              '[.]fail[.]cpp$']
-        sourcePath = testSuite.getSourcePath(pathInSuite)
-        for filename in os.listdir(sourcePath):
-            # Ignore dot files and excluded tests.
-            if filename.startswith('.') or filename in localConfig.excludes:
-                continue
-
-            filepath = os.path.join(sourcePath, filename)
-            if not os.path.isdir(filepath):
-                if any([re.search(ext, filename) for ext in SUPPORTED_SUFFIXES]):
-                    yield lit.Test.Test(testSuite, pathInSuite + (filename,), localConfig)
-
-    def _checkBaseSubstitutions(self, substitutions):
-        substitutions = [s for (s, _) in substitutions]
-        for s in ['%{cxx}', '%{compile_flags}', '%{link_flags}', '%{flags}', '%{exec}']:
-            assert s in substitutions, "Required substitution {} was not provided".format(s)
-
-    def _disableWithModules(self, test):
-        with open(test.getSourcePath(), 'rb') as f:
-            contents = f.read()
-        return b'#define _LIBCPP_ASSERT' in contents
-
-    def execute(self, test, litConfig):
-        self._checkBaseSubstitutions(test.config.substitutions)
-        VERIFY_FLAGS = '-Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0'
-        supportsVerify = _supportsVerify(test.config)
-        filename = test.path_in_suite[-1]
-
-        # TODO(ldionne): We currently disable tests that re-define _LIBCPP_ASSERT
-        #                when we run with modules enabled. Instead, we should
-        #                split the part that does a death test outside of the
-        #                test, and only disable that part when modules are
-        #                enabled.
-        if '-fmodules' in test.config.available_features and self._disableWithModules(test):
-            return lit.Test.Result(lit.Test.UNSUPPORTED, 'Test {} is unsupported when modules are enabled')
-
-        if re.search('[.]sh[.][^.]+$', filename):
-            steps = [ ] # The steps are already in the script
-            return self._executeShTest(test, litConfig, steps)
-        elif filename.endswith('.compile.pass.cpp'):
-            steps = [
-                "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -fsyntax-only"
-            ]
-            return self._executeShTest(test, litConfig, steps)
-        elif filename.endswith('.compile.fail.cpp'):
-            steps = [
-                "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only"
-            ]
-            return self._executeShTest(test, litConfig, steps)
-        elif filename.endswith('.link.pass.cpp'):
-            steps = [
-                "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe"
-            ]
-            return self._executeShTest(test, litConfig, steps)
-        elif filename.endswith('.link.fail.cpp'):
-            steps = [
-                "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -c -o %t.o",
-                "%dbg(LINKED WITH) ! %{cxx} %t.o %{flags} %{link_flags} -o %t.exe"
-            ]
-            return self._executeShTest(test, litConfig, steps)
-        elif filename.endswith('.run.fail.cpp'):
-            steps = [
-                "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe",
-                "%dbg(EXECUTED AS) %{exec} ! %t.exe"
-            ]
-            return self._executeShTest(test, litConfig, steps)
-        elif filename.endswith('.verify.cpp'):
-            if not supportsVerify:
-                return lit.Test.Result(lit.Test.UNSUPPORTED,
-                    "Test {} requires support for Clang-verify, which isn't supported by the compiler".format(test.getFullName()))
-            steps = [
-                # Note: Use -Wno-error to make sure all diagnostics are not treated as errors,
-                #       which doesn't make sense for clang-verify tests.
-                "%dbg(COMPILED WITH) %{{cxx}} %s %{{flags}} %{{compile_flags}} -fsyntax-only -Wno-error {}".format(VERIFY_FLAGS)
-            ]
-            return self._executeShTest(test, litConfig, steps)
-        # Make sure to check these ones last, since they will match other
-        # suffixes above too.
-        elif filename.endswith('.pass.cpp') or filename.endswith('.pass.mm'):
-            steps = [
-                "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe",
-                "%dbg(EXECUTED AS) %{exec} %t.exe"
-            ]
-            return self._executeShTest(test, litConfig, steps)
-        # This is like a .verify.cpp test when clang-verify is supported,
-        # otherwise it's like a .compile.fail.cpp test. This is only provided
-        # for backwards compatibility with the test suite.
-        elif filename.endswith('.fail.cpp'):
-            if supportsVerify:
-                steps = [
-                    "%dbg(COMPILED WITH) %{{cxx}} %s %{{flags}} %{{compile_flags}} -fsyntax-only -Wno-error {}".format(VERIFY_FLAGS)
-                ]
-            else:
-                steps = [
-                    "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only"
-                ]
-            return self._executeShTest(test, litConfig, steps)
-        else:
-            return lit.Test.Result(lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename))
-
-    # Utility function to add compile flags in lit.local.cfg files.
-    def addCompileFlags(self, config, *flags):
-        string = ' '.join(flags)
-        config.substitutions = [(s, x + ' ' + string) if s == '%{compile_flags}' else (s, x) for (s, x) in config.substitutions]
-
-    def _executeShTest(self, test, litConfig, steps):
-        if test.config.unsupported:
-            return lit.Test.Result(lit.Test.UNSUPPORTED, 'Test is unsupported')
-
-        script = parseScript(test, steps)
-        if isinstance(script, lit.Test.Result):
-            return script
-
-        if litConfig.noExecute:
-            return lit.Test.Result(lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS)
-        else:
-            _, tmpBase = _getTempPaths(test)
-            useExternalSh = True
-            return lit.TestRunner._runShTest(test, litConfig, useExternalSh, script, tmpBase)
diff --git a/libcxx/utils/libcxx/test/params.py b/libcxx/utils/libcxx/test/params.py
index 43674de2c213f..864a5108fc098 100644
--- a/libcxx/utils/libcxx/test/params.py
+++ b/libcxx/utils/libcxx/test/params.py
@@ -8,15 +8,38 @@
 
 from libcxx.test.dsl import *
 
+_allStandards = ['c++98', 'c++03', 'c++11', 'c++14', 'c++17', 'c++2a']
+
 parameters = [
+  # Core parameters of the test suite
+  Parameter(name='std', choices=_allStandards, type=str,
+            help="The version of the standard to compile the test suite with.",
+            default=lambda cfg: next(s for s in reversed(_allStandards) if hasCompileFlag(cfg, '-std='+s)),
+            feature=lambda std:
+              Feature(name=std, compileFlag='-std={}'.format(std),
+                      when=lambda cfg: hasCompileFlag(cfg, '-std={}'.format(std)))),
+
   Parameter(name='enable_exceptions', choices=[True, False], type=bool, default=True,
             help="Whether to enable exceptions when compiling the test suite.",
             feature=lambda exceptions: None if exceptions else
               Feature(name='no-exceptions', compileFlag='-fno-exceptions')),
 
+  Parameter(name='stdlib', choices=['libc++', 'libstdc++', 'msvc'], type=str, default='libc++',
+            help="The C++ Standard Library implementation being tested.",
+            feature=lambda stdlib: Feature(name=stdlib)),
+
   # Parameters to enable or disable parts of the test suite
   Parameter(name='enable_filesystem', choices=[True, False], type=bool, default=True,
             help="Whether to enable tests for the C++ <filesystem> library.",
             feature=lambda filesystem: None if filesystem else
               Feature(name='c++filesystem-disabled')),
+
+  Parameter(name='enable_experimental', choices=[True, False], type=bool, default=False,
+          help="Whether to enable tests for experimental C++ libraries (typically Library Fundamentals TSes).",
+          feature=lambda experimental: None if not experimental else
+            Feature(name='c++experimental', linkFlag='-lc++experimental')),
+
+  Parameter(name='long_tests', choices=[True, False], type=bool, default=True,
+            help="Whether to tests that take longer to run. This can be useful when running on a very slow device.",
+            feature=lambda enabled: Feature(name='long_tests') if enabled else None),
 ]
diff --git a/libcxx/utils/libcxx/test/target_info.py b/libcxx/utils/libcxx/test/target_info.py
index 5782fc53dca0f..3197276ffa5b5 100644
--- a/libcxx/utils/libcxx/test/target_info.py
+++ b/libcxx/utils/libcxx/test/target_info.py
@@ -33,7 +33,6 @@ def is_darwin(self):
     def add_cxx_compile_flags(self, flags): pass
     def add_cxx_link_flags(self, flags): pass
     def allow_cxxabi_link(self): return True
-    def use_lit_shell_default(self): return False
 
     def add_path(self, dest_env, new_path):
         if not new_path:
@@ -210,12 +209,6 @@ class WindowsLocalTI(DefaultTargetInfo):
     def __init__(self, full_config):
         super(WindowsLocalTI, self).__init__(full_config)
 
-    def use_lit_shell_default(self):
-        # Default to the internal shell on Windows, as bash on Windows is
-        # usually very slow.
-        return True
-
-
 def make_target_info(full_config):
     default = "libcxx.test.target_info.LocalTI"
     info_str = full_config.get_lit_conf('target_info', default)
diff --git a/libcxx/utils/libcxx/test/tracing.py b/libcxx/utils/libcxx/test/tracing.py
deleted file mode 100644
index 0bd339273b683..0000000000000
--- a/libcxx/utils/libcxx/test/tracing.py
+++ /dev/null
@@ -1,42 +0,0 @@
-#===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===----------------------------------------------------------------------===##
-
-import os
-import inspect
-
-
-def trace_function(function, log_calls, log_results, label=''):
-    def wrapper(*args, **kwargs):
-        kwarg_strs = ['{}={}'.format(k, v) for (k, v) in kwargs]
-        arg_str = ', '.join([str(a) for a in args] + kwarg_strs)
-        call_str = '{}({})'.format(function.__name__, arg_str)
-
-        # Perform the call itself, logging before, after, and anything thrown.
-        try:
-            if log_calls:
-                print('{}: Calling {}'.format(label, call_str))
-            res = function(*args, **kwargs)
-            if log_results:
-                print('{}: {} -> {}'.format(label, call_str, res))
-            return res
-        except Exception as ex:
-            if log_results:
-                print('{}: {} raised {}'.format(label, call_str, type(ex)))
-            raise ex
-
-    return wrapper
-
-
-def trace_object(obj, log_calls, log_results, label=''):
-    for name, member in inspect.getmembers(obj):
-        if inspect.ismethod(member):
-            # Skip meta-functions, decorate everything else
-            if not member.__name__.startswith('__'):
-                setattr(obj, name, trace_function(member, log_calls,
-                                                  log_results, label))
-    return obj
diff --git a/libcxxabi/CMakeLists.txt b/libcxxabi/CMakeLists.txt
index 08bc8dcee65a1..8881a5018dc43 100644
--- a/libcxxabi/CMakeLists.txt
+++ b/libcxxabi/CMakeLists.txt
@@ -135,6 +135,11 @@ message(STATUS "Libc++abi will be using libc++ includes from ${LIBCXXABI_LIBCXX_
 option(LIBCXXABI_HERMETIC_STATIC_LIBRARY
   "Do not export any symbols from the static library." OFF)
 
+set(LIBCXXABI_TEST_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/test/lit.site.cfg.in" CACHE STRING
+    "The Lit testing configuration to use when running the tests.")
+set(LIBCXXABI_TEST_PARAMS "" CACHE STRING
+    "A list of parameters to run the Lit test suite with.")
+
 #===============================================================================
 # Configure System
 #===============================================================================
@@ -304,7 +309,6 @@ if (LIBCXXABI_ENABLE_EXCEPTIONS)
   # Do we really need to be run through the C compiler ?
   add_c_compile_flags_if_supported(-funwind-tables)
 else()
-  add_definitions(-D_LIBCXXABI_NO_EXCEPTIONS)
   add_compile_flags_if_supported(-fno-exceptions)
   add_compile_flags_if_supported(-EHs-)
   add_compile_flags_if_supported(-EHa-)
diff --git a/libcxxabi/include/__cxxabi_config.h b/libcxxabi/include/__cxxabi_config.h
index cbbb1c10c1a29..b5444d67ee44b 100644
--- a/libcxxabi/include/__cxxabi_config.h
+++ b/libcxxabi/include/__cxxabi_config.h
@@ -76,4 +76,12 @@
 #  define _LIBCXXABI_GUARD_ABI_ARM
 #endif
 
+#if defined(_LIBCXXABI_COMPILER_CLANG)
+#  if !__has_feature(cxx_exceptions)
+#    define _LIBCXXABI_NO_EXCEPTIONS
+#  endif
+#elif defined(_LIBCXXABI_COMPILER_GCC) && !__EXCEPTIONS
+#  define _LIBCXXABI_NO_EXCEPTIONS
+#endif
+
 #endif // ____CXXABI_CONFIG_H
diff --git a/libcxxabi/src/CMakeLists.txt b/libcxxabi/src/CMakeLists.txt
index d5251acdca380..42bec421d2be7 100644
--- a/libcxxabi/src/CMakeLists.txt
+++ b/libcxxabi/src/CMakeLists.txt
@@ -7,7 +7,6 @@ set(LIBCXXABI_SOURCES
   cxa_exception_storage.cpp
   cxa_guard.cpp
   cxa_handlers.cpp
-  cxa_unexpected.cpp
   cxa_vector.cpp
   cxa_virtual.cpp
   # C++ STL files
diff --git a/libcxxabi/src/abort_message.cpp b/libcxxabi/src/abort_message.cpp
index d556af1c9037d..ad44063facb71 100644
--- a/libcxxabi/src/abort_message.cpp
+++ b/libcxxabi/src/abort_message.cpp
@@ -12,52 +12,54 @@
 #include "abort_message.h"
 
 #ifdef __BIONIC__
-#include <android/api-level.h>
-#if __ANDROID_API__ >= 21
-#include <syslog.h>
-extern "C" void android_set_abort_message(const char* msg);
-#else
-#include <assert.h>
-#endif // __ANDROID_API__ >= 21
+#   include <android/api-level.h>
+#   if __ANDROID_API__ >= 21
+#       include <syslog.h>
+        extern "C" void android_set_abort_message(const char* msg);
+#   else
+#       include <assert.h>
+#   endif // __ANDROID_API__ >= 21
 #endif // __BIONIC__
 
-#ifdef __APPLE__
-#   if defined(__has_include) && __has_include(<CrashReporterClient.h>)
-#       define HAVE_CRASHREPORTERCLIENT_H
-#       include <CrashReporterClient.h>
-#   endif
+#if defined(__APPLE__) && __has_include(<CrashReporterClient.h>)
+#   include <CrashReporterClient.h>
+#   define _LIBCXXABI_USE_CRASHREPORTER_CLIENT
 #endif
 
 void abort_message(const char* format, ...)
 {
-    // write message to stderr
+    // Write message to stderr. We do this before formatting into a
+    // variable-size buffer so that we still get some information if
+    // formatting into the variable-sized buffer fails.
 #if !defined(NDEBUG) || !defined(LIBCXXABI_BAREMETAL)
-#ifdef __APPLE__
-    fprintf(stderr, "libc++abi.dylib: ");
+    {
+        fprintf(stderr, "libc++abi: ");
+        va_list list;
+        va_start(list, format);
+        vfprintf(stderr, format, list);
+        va_end(list);
+        fprintf(stderr, "\n");
+    }
 #endif
+
+    // Format the arguments into an allocated buffer. We leak the buffer on
+    // purpose, since we're about to abort() anyway.
+#if defined(_LIBCXXABI_USE_CRASHREPORTER_CLIENT)
+    char* buffer;
     va_list list;
     va_start(list, format);
-    vfprintf(stderr, format, list);
+    vasprintf(&buffer, format, list);
     va_end(list);
-    fprintf(stderr, "\n");
-#endif
 
-#if defined(__APPLE__) && defined(HAVE_CRASHREPORTERCLIENT_H)
-    // record message in crash report
-    char* buffer;
-    va_list list2;
-    va_start(list2, format);
-    vasprintf(&buffer, format, list2);
-    va_end(list2);
     CRSetCrashLogMessage(buffer);
 #elif defined(__BIONIC__)
     char* buffer;
-    va_list list2;
-    va_start(list2, format);
-    vasprintf(&buffer, format, list2);
-    va_end(list2);
+    va_list list;
+    va_start(list, format);
+    vasprintf(&buffer, format, list);
+    va_end(list);
 
-#if __ANDROID_API__ >= 21
+#   if __ANDROID_API__ >= 21
     // Show error in tombstone.
     android_set_abort_message(buffer);
 
@@ -65,12 +67,12 @@ void abort_message(const char* format, ...)
     openlog("libc++abi", 0, 0);
     syslog(LOG_CRIT, "%s", buffer);
     closelog();
-#else
+#   else
     // The good error reporting wasn't available in Android until L. Since we're
     // about to abort anyway, just call __assert2, which will log _somewhere_
     // (tombstone and/or logcat) in older releases.
     __assert2(__FILE__, __LINE__, __func__, buffer);
-#endif // __ANDROID_API__ >= 21
+#   endif // __ANDROID_API__ >= 21
 #endif // __BIONIC__
 
     abort();
diff --git a/libcxxabi/src/cxa_guard_impl.h b/libcxxabi/src/cxa_guard_impl.h
index dfb26cf687e41..f6a698e23acb2 100644
--- a/libcxxabi/src/cxa_guard_impl.h
+++ b/libcxxabi/src/cxa_guard_impl.h
@@ -40,7 +40,6 @@
 #include "__cxxabi_config.h"
 #include "include/atomic_support.h"
 #include <unistd.h>
-#include <sys/types.h>
 #if defined(__has_include)
 # if __has_include(<sys/syscall.h>)
 #   include <sys/syscall.h>
diff --git a/libcxxabi/src/demangle/ItaniumDemangle.h b/libcxxabi/src/demangle/ItaniumDemangle.h
index dcece38992138..6ab873218386b 100644
--- a/libcxxabi/src/demangle/ItaniumDemangle.h
+++ b/libcxxabi/src/demangle/ItaniumDemangle.h
@@ -5096,6 +5096,22 @@ Node *AbstractManglingParser<Derived, Alloc>::parseSpecialName() {
 //            ::= <special-name>
 template <typename Derived, typename Alloc>
 Node *AbstractManglingParser<Derived, Alloc>::parseEncoding() {
+  // The template parameters of an encoding are unrelated to those of the
+  // enclosing context.
+  class SaveTemplateParams {
+    AbstractManglingParser *Parser;
+    decltype(TemplateParams) OldParams;
+
+  public:
+    SaveTemplateParams(AbstractManglingParser *Parser) : Parser(Parser) {
+      OldParams = std::move(Parser->TemplateParams);
+      Parser->TemplateParams.clear();
+    }
+    ~SaveTemplateParams() {
+      Parser->TemplateParams = std::move(OldParams);
+    }
+  } SaveTemplateParams(this);
+
   if (look() == 'G' || look() == 'T')
     return getDerived().parseSpecialName();
 
diff --git a/libcxxabi/test/CMakeLists.txt b/libcxxabi/test/CMakeLists.txt
index 2160f52c33504..8dcd65d9b4b4b 100644
--- a/libcxxabi/test/CMakeLists.txt
+++ b/libcxxabi/test/CMakeLists.txt
@@ -1,4 +1,4 @@
-include(AddLLVM) # for add_lit_testsuite
+include(AddLLVM) # for configure_lit_site_cfg and add_lit_target
 macro(pythonize_bool var)
   if (${var})
     set(${var} True)
@@ -53,8 +53,6 @@ set(LIBCXXABI_TARGET_INFO "libcxx.test.target_info.LocalTI" CACHE STRING
 set(LIBCXXABI_EXECUTOR "${Python3_EXECUTABLE} ${LIBCXXABI_LIBCXX_PATH}/utils/run.py" CACHE STRING
     "Executor to use when running tests.")
 
-set(AUTO_GEN_COMMENT "## Autogenerated by libcxxabi configuration.\n# Do not edit!")
-
 if (LIBCXXABI_ENABLE_SHARED)
   set(LIBCXXABI_TEST_DEPS cxxabi_shared)
 else()
@@ -72,12 +70,13 @@ if (NOT LIBCXXABI_STANDALONE_BUILD)
   endif()
 endif()
 
-
+set(AUTO_GEN_COMMENT "## Autogenerated by libcxxabi configuration.\n# Do not edit!")
 configure_lit_site_cfg(
-  ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
-  ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg)
+  "${LIBCXXABI_TEST_CONFIG}"
+  ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
+  MAIN_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py")
 
-add_lit_testsuite(check-cxxabi "Running libcxxabi tests"
+add_lit_target(check-cxxabi "Running libcxxabi tests"
   ${CMAKE_CURRENT_BINARY_DIR}
   DEPENDS ${LIBCXXABI_TEST_DEPS}
-  )
+  PARAMS "${LIBCXXABI_TEST_PARAMS}")
diff --git a/libcxxabi/test/lit.cfg b/libcxxabi/test/lit.cfg
deleted file mode 100644
index 4c4aa3b2dd583..0000000000000
--- a/libcxxabi/test/lit.cfg
+++ /dev/null
@@ -1,79 +0,0 @@
-# -*- Python -*- vim: set ft=python ts=4 sw=4 expandtab tw=79:
-
-# Configuration file for the 'lit' test runner.
-
-
-import os
-import site
-
-site.addsitedir(os.path.dirname(__file__))
-
-
-# Tell pylint that we know config and lit_config exist somewhere.
-if 'PYLINT_IMPORT' in os.environ:
-    config = object()
-    lit_config = object()
-
-# name: The name of this test suite.
-config.name = 'libc++abi'
-
-# suffixes: A list of file extensions to treat as test files.
-config.suffixes = ['.cpp', '.s']
-
-# Allow expanding substitutions that are based on other substitutions
-config.recursiveExpansionLimit = 10
-
-# test_source_root: The root path where tests are located.
-config.test_source_root = os.path.dirname(__file__)
-
-# Allow expanding substitutions that are based on other substitutions
-config.recursiveExpansionLimit = 10
-
-# Infer the libcxx_test_source_root for configuration import.
-# If libcxx_source_root isn't specified in the config, assume that the libcxx
-# and libcxxabi source directories are sibling directories.
-libcxx_src_root = getattr(config, 'libcxx_src_root', None)
-if not libcxx_src_root:
-    libcxx_src_root = os.path.join(config.test_source_root, '../../libcxx')
-libcxx_test_src_root = os.path.join(libcxx_src_root, 'utils')
-if os.path.isfile(os.path.join(libcxx_test_src_root, 'libcxx', '__init__.py')):
-    site.addsitedir(libcxx_test_src_root)
-else:
-    lit_config.fatal('Could not find libcxx test directory for test imports'
-                     ' in: %s' % libcxx_test_src_root)
-
-# Infer the test_exec_root from the libcxx_object root.
-obj_root = getattr(config, 'libcxxabi_obj_root', None)
-
-# Check that the test exec root is known.
-if obj_root is None:
-    import libcxx.test.config
-    libcxx.test.config.loadSiteConfig(
-        lit_config, config, 'libcxxabi_site_config', 'LIBCXXABI_SITE_CONFIG')
-    obj_root = getattr(config, 'libcxxabi_obj_root', None)
-    if obj_root is None:
-        import tempfile
-        obj_root = tempfile.mkdtemp(prefix='libcxxabi-testsuite-')
-        lit_config.warning('Creating temporary directory for object root: %s' %
-                           obj_root)
-
-config.test_exec_root = os.path.join(obj_root, 'test')
-
-cfg_variant = getattr(config, 'configuration_variant', 'libcxxabi')
-if cfg_variant:
-    lit_config.note('Using configuration variant: %s' % cfg_variant)
-
-# Load the Configuration class from the module name <cfg_variant>.test.config.
-config_module_name = '.'.join([cfg_variant, 'test', 'config'])
-config_module = __import__(config_module_name, fromlist=['Configuration'])
-
-configuration = config_module.Configuration(lit_config, config)
-configuration.configure()
-configuration.print_config_info()
-if lit_config.params.get('use_old_format', False):
-    lit_config.note("Using the old libc++ testing format")
-    config.test_format = configuration.get_test_format()
-else:
-    lit_config.note("Using the new libc++ testing format")
-    import libcxx.test.newformat
-    config.test_format = libcxx.test.newformat.CxxStandardLibraryTest()
diff --git a/libcxxabi/test/lit.cfg.py b/libcxxabi/test/lit.cfg.py
new file mode 100644
index 0000000000000..647464abe22d9
--- /dev/null
+++ b/libcxxabi/test/lit.cfg.py
@@ -0,0 +1,10 @@
+# All the Lit configuration is handled in the site configs -- this file is only
+# left as a canary to catch invocations of Lit that do not go through llvm-lit.
+#
+# Invocations that go through llvm-lit will automatically use the right Lit
+# site configuration inside the build directory.
+
+lit_config.fatal(
+    "You seem to be running Lit directly -- you should be running Lit through "
+    "<build>/bin/llvm-lit, which will ensure that the right Lit configuration "
+    "file is used.")
diff --git a/libcxxabi/test/lit.site.cfg.in b/libcxxabi/test/lit.site.cfg.in
index 8125e89a4533c..75fde7ee92507 100644
--- a/libcxxabi/test/lit.site.cfg.in
+++ b/libcxxabi/test/lit.site.cfg.in
@@ -1,4 +1,8 @@
 @AUTO_GEN_COMMENT@
+
+import os
+import site
+
 config.cxx_under_test           = "@LIBCXXABI_COMPILER@"
 config.project_obj_root         = "@CMAKE_BINARY_DIR@"
 config.libcxxabi_src_root       = "@LIBCXXABI_SOURCE_DIR@"
@@ -29,5 +33,32 @@ config.cxx_ext_threads          = @LIBCXXABI_BUILD_EXTERNAL_THREAD_LIBRARY@
 config.pstl_src_root            = "@ParallelSTL_SOURCE_DIR@" if @LIBCXX_ENABLE_PARALLEL_ALGORITHMS@ else None
 config.pstl_obj_root            = "@ParallelSTL_BINARY_DIR@" if @LIBCXX_ENABLE_PARALLEL_ALGORITHMS@ else None
 
-# Let the main config do the real work.
-lit_config.load_config(config, "@LIBCXXABI_SOURCE_DIR@/test/lit.cfg")
+# Code signing
+config.llvm_codesign_identity   = "@LLVM_CODESIGNING_IDENTITY@"
+
+site.addsitedir(os.path.join(config.libcxxabi_src_root, 'test'))
+site.addsitedir(os.path.join(config.libcxx_src_root, 'utils'))
+
+# name: The name of this test suite.
+config.name = 'libc++abi'
+
+# suffixes: A list of file extensions to treat as test files.
+config.suffixes = ['.cpp', '.s']
+
+# test_source_root: The root path where tests are located.
+config.test_source_root = os.path.join(config.libcxxabi_src_root, 'test')
+
+# Allow expanding substitutions that are based on other substitutions
+config.recursiveExpansionLimit = 10
+
+# Infer the test_exec_root from the build directory.
+config.test_exec_root = os.path.join(config.libcxxabi_obj_root, 'test')
+
+import libcxx.test.format
+config.test_format = libcxx.test.format.CxxStandardLibraryTest()
+
+lit_config.note('Using configuration variant: libcxxabi')
+import libcxxabi.test.config
+configuration = libcxxabi.test.config.Configuration(lit_config, config)
+configuration.configure()
+configuration.print_config_info()
diff --git a/libcxxabi/test/test_demangle.pass.cpp b/libcxxabi/test/test_demangle.pass.cpp
index c75b6974881e1..ef75b61a94af2 100644
--- a/libcxxabi/test/test_demangle.pass.cpp
+++ b/libcxxabi/test/test_demangle.pass.cpp
@@ -29792,6 +29792,17 @@ const char* cases[][2] =
     //   "auto inline_func()::'lambda'<int, int>(int, int) const"
     {"_ZZ11inline_funcvENKUlTyTyT_T0_E_clIiiEEDaS_S0_", "auto inline_func()::'lambda'<typename $T, typename $T0>($T, $T0)::operator()<int, int>($T, $T0) const"},
     {"_ZZ11inline_funcvENKUlTyTyT_T1_T0_E_clIiiiEEDaS_S0_S1_", "auto inline_func()::'lambda'<typename $T, typename $T0>($T, auto, $T0)::operator()<int, int, int>($T, auto, $T0) const"},
+    {"_ZN1XIZ1fIiEvOT_EUlOT_DpT0_E_EclIJEEEvDpT_", "void X<void f<int>(int&&)::'lambda'(auto&&, auto...)>::operator()<>()"},
+    // FIXME: This is wrong, should demangle to the same as the previous entry.
+    // See https://github.com/itanium-cxx-abi/cxx-abi/issues/106.
+    {"_ZN1XIZ1fIiEvOT_EUlS2_DpT0_E_EclIJEEEvDpT_", "void X<void f<int>(int&&)::'lambda'(int&&, auto...)>::operator()<>()"},
+    {"_ZZZZN6abcdef9abcdefghi29abcdefabcdefabcdefabcefabcdef27xxxxxxxxxxxxxxxxxxxxxxxxxxxEN4absl8DurationERKNSt3__u12basic_stringIcNS4_11char_traitsIcEENS4_9allocatorIcEEEEPNS1_19yyyyyyyyyyyyyyyyyyyEENK3$_5clEvENKUlvE_clEvE6zzzzzz", "abcdef::abcdefghi::abcdefabcdefabcdefabcefabcdef::xxxxxxxxxxxxxxxxxxxxxxxxxxx(absl::Duration, std::__u::basic_string<char, std::__u::char_traits<char>, std::__u::allocator<char> > const&, abcdef::abcdefghi::abcdefabcdefabcdefabcefabcdef::yyyyyyyyyyyyyyyyyyy*)::$_5::operator()() const::'lambda'()::operator()() const::zzzzzz"},
+
+    // FIXME: This is wrong; the S2_ backref should expand to OT_ and then to
+    // "double&&". But we can't cope with a substitution that represents a
+    // different type the node it is a substitute for.
+    // See https://github.com/itanium-cxx-abi/cxx-abi/issues/106.
+    {"_Z1h1XIJZ1fIiEDaOT_E1AZ1gIdEDaS2_E1BEE", "h(X<auto f<int>(int&&)::A, auto g<double>(int&&)::B>)"},
 
     {"_Z1fIL4Enumn1EEvv", "void f<(Enum)-1>()"},
 
diff --git a/libunwind/CMakeLists.txt b/libunwind/CMakeLists.txt
index d3f791e17f108..7065112627a25 100644
--- a/libunwind/CMakeLists.txt
+++ b/libunwind/CMakeLists.txt
@@ -154,6 +154,8 @@ set(LIBUNWIND_TEST_LINKER_FLAGS "" CACHE STRING
     "Additional linker flags for test programs.")
 set(LIBUNWIND_TEST_COMPILER_FLAGS "" CACHE STRING
     "Additional compiler flags for test programs.")
+set(LIBUNWIND_TEST_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/test/lit.site.cfg.in" CACHE STRING
+    "The Lit testing configuration to use when running the tests.")
 
 if (NOT LIBUNWIND_ENABLE_SHARED AND NOT LIBUNWIND_ENABLE_STATIC)
   message(FATAL_ERROR "libunwind must be built as either a shared or static library.")
diff --git a/libunwind/test/CMakeLists.txt b/libunwind/test/CMakeLists.txt
index 26e7842b7a1fb..794a59f58f84a 100644
--- a/libunwind/test/CMakeLists.txt
+++ b/libunwind/test/CMakeLists.txt
@@ -25,12 +25,11 @@ set(LIBUNWIND_EXECUTOR "${Python3_EXECUTABLE} ${LIBUNWIND_LIBCXX_PATH}/utils/run
     "Executor to use when running tests.")
 
 set(AUTO_GEN_COMMENT "## Autogenerated by libunwind configuration.\n# Do not edit!")
-configure_file(
-  ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
+configure_lit_site_cfg(
+  "${LIBUNWIND_TEST_CONFIG}"
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
-  @ONLY)
+  MAIN_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py")
 
 add_lit_testsuite(check-unwind "Running libunwind tests"
   ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS ${LIBUNWIND_TEST_DEPS}
-  )
+  DEPENDS ${LIBUNWIND_TEST_DEPS})
diff --git a/libunwind/test/lit.cfg b/libunwind/test/lit.cfg
deleted file mode 100644
index 7f74bd6e4afb4..0000000000000
--- a/libunwind/test/lit.cfg
+++ /dev/null
@@ -1,73 +0,0 @@
-# -*- Python -*- vim: set ft=python ts=4 sw=4 expandtab tw=79:
-
-# Configuration file for the 'lit' test runner.
-
-
-import os
-import site
-
-site.addsitedir(os.path.dirname(__file__))
-
-
-# Tell pylint that we know config and lit_config exist somewhere.
-if 'PYLINT_IMPORT' in os.environ:
-    config = object()
-    lit_config = object()
-
-# name: The name of this test suite.
-config.name = 'libunwind'
-
-# suffixes: A list of file extensions to treat as test files.
-config.suffixes = ['.cpp', '.s']
-
-# test_source_root: The root path where tests are located.
-config.test_source_root = os.path.dirname(__file__)
-
-# Infer the libcxx_test_source_root for configuration import.
-# If libcxx_source_root isn't specified in the config, assume that the libcxx
-# and libunwind source directories are sibling directories.
-libcxx_src_root = getattr(config, 'libcxx_src_root', None)
-if not libcxx_src_root:
-    libcxx_src_root = os.path.join(config.test_source_root, '../../libcxx')
-libcxx_test_src_root = os.path.join(libcxx_src_root, 'utils')
-if os.path.isfile(os.path.join(libcxx_test_src_root, 'libcxx', '__init__.py')):
-    site.addsitedir(libcxx_test_src_root)
-else:
-    lit_config.fatal('Could not find libcxx test directory for test imports'
-                     ' in: %s' % libcxx_test_src_root)
-
-# Infer the test_exec_root from the libcxx_object root.
-obj_root = getattr(config, 'libunwind_obj_root', None)
-
-# Check that the test exec root is known.
-if obj_root is None:
-    import libcxx.test.config
-    libcxx.test.config.loadSiteConfig(
-        lit_config, config, 'libunwind_site_config', 'LIBUNWIND_SITE_CONFIG')
-    obj_root = getattr(config, 'libunwind_obj_root', None)
-    if obj_root is None:
-        import tempfile
-        obj_root = tempfile.mkdtemp(prefix='libunwind-testsuite-')
-        lit_config.warning('Creating temporary directory for object root: %s' %
-                           obj_root)
-
-config.test_exec_root = os.path.join(obj_root, 'test')
-
-cfg_variant = getattr(config, 'configuration_variant', 'libunwind')
-if cfg_variant:
-    lit_config.note('Using configuration variant: %s' % cfg_variant)
-
-# Load the Configuration class from the module name <cfg_variant>.test.config.
-config_module_name = '.'.join([cfg_variant, 'test', 'config'])
-config_module = __import__(config_module_name, fromlist=['Configuration'])
-
-configuration = config_module.Configuration(lit_config, config)
-configuration.configure()
-configuration.print_config_info()
-if lit_config.params.get('use_old_format', False):
-    lit_config.note("Using the old libc++ testing format")
-    config.test_format = configuration.get_test_format()
-else:
-    lit_config.note("Using the new libc++ testing format")
-    import libcxx.test.newformat
-    config.test_format = libcxx.test.newformat.CxxStandardLibraryTest()
diff --git a/libunwind/test/lit.cfg.py b/libunwind/test/lit.cfg.py
new file mode 100644
index 0000000000000..647464abe22d9
--- /dev/null
+++ b/libunwind/test/lit.cfg.py
@@ -0,0 +1,10 @@
+# All the Lit configuration is handled in the site configs -- this file is only
+# left as a canary to catch invocations of Lit that do not go through llvm-lit.
+#
+# Invocations that go through llvm-lit will automatically use the right Lit
+# site configuration inside the build directory.
+
+lit_config.fatal(
+    "You seem to be running Lit directly -- you should be running Lit through "
+    "<build>/bin/llvm-lit, which will ensure that the right Lit configuration "
+    "file is used.")
diff --git a/libunwind/test/lit.site.cfg.in b/libunwind/test/lit.site.cfg.in
index 809ad1009f4bd..d0f0e08fc9263 100644
--- a/libunwind/test/lit.site.cfg.in
+++ b/libunwind/test/lit.site.cfg.in
@@ -1,4 +1,8 @@
 @AUTO_GEN_COMMENT@
+
+import os
+import site
+
 config.cxx_under_test           = "@LIBUNWIND_COMPILER@"
 config.project_obj_root         = "@CMAKE_BINARY_DIR@"
 config.libunwind_src_root       = "@LIBUNWIND_SOURCE_DIR@"
@@ -26,5 +30,29 @@ config.sysroot                  = "@LIBUNWIND_SYSROOT@"
 config.gcc_toolchain            = "@LIBUNWIND_GCC_TOOLCHAIN@"
 config.cxx_ext_threads          = @LIBUNWIND_BUILD_EXTERNAL_THREAD_LIBRARY@
 
-# Let the main config do the real work.
-lit_config.load_config(config, "@LIBUNWIND_SOURCE_DIR@/test/lit.cfg")
+site.addsitedir(os.path.join(config.libunwind_src_root, 'test'))
+site.addsitedir(os.path.join(config.libcxx_src_root, 'utils'))
+
+# name: The name of this test suite.
+config.name = 'libunwind'
+
+# suffixes: A list of file extensions to treat as test files.
+config.suffixes = ['.cpp', '.s']
+
+# test_source_root: The root path where tests are located.
+config.test_source_root = os.path.join(config.libunwind_src_root, 'test')
+
+# Allow expanding substitutions that are based on other substitutions
+config.recursiveExpansionLimit = 10
+
+# Infer the test_exec_root from the build directory.
+config.test_exec_root = os.path.join(config.libunwind_obj_root, 'test')
+
+import libcxx.test.format
+config.test_format = libcxx.test.format.CxxStandardLibraryTest()
+
+lit_config.note('Using configuration variant: libunwind')
+import libunwind.test.config
+configuration = libunwind.test.config.Configuration(lit_config, config)
+configuration.configure()
+configuration.print_config_info()
diff --git a/lld/COFF/PDB.cpp b/lld/COFF/PDB.cpp
index be6af13f36479..49d04add5be04 100644
--- a/lld/COFF/PDB.cpp
+++ b/lld/COFF/PDB.cpp
@@ -946,12 +946,12 @@ static pdb::BulkPublic createPublic(Defined *def) {
   } else if (isa<DefinedImportThunk>(def)) {
     flags = PublicSymFlags::Function;
   }
-  pub.Flags = static_cast<uint16_t>(flags);
+  pub.setFlags(flags);
 
   OutputSection *os = def->getChunk()->getOutputSection();
   assert(os && "all publics should be in final image");
   pub.Offset = def->getRVA() - os->getRVA();
-  pub.U.Segment = os->sectionIndex;
+  pub.Segment = os->sectionIndex;
   return pub;
 }
 
diff --git a/lld/Common/Filesystem.cpp b/lld/Common/Filesystem.cpp
index 21b387ea26f70..206b892f0a69c 100644
--- a/lld/Common/Filesystem.cpp
+++ b/lld/Common/Filesystem.cpp
@@ -15,6 +15,7 @@
 #include "llvm/Support/FileOutputBuffer.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Parallel.h"
+#include "llvm/Support/Path.h"
 #if LLVM_ON_UNIX
 #include <unistd.h>
 #endif
@@ -41,6 +42,33 @@ using namespace lld;
 void lld::unlinkAsync(StringRef path) {
 // Removing a file is async on windows.
 #if defined(_WIN32)
+  // On Windows co-operative programs can be expected to open LLD's
+  // output in FILE_SHARE_DELETE mode. This allows us to delete the
+  // file (by moving it to a temporary filename and then deleting
+  // it) so that we can link another output file that overwrites
+  // the existing file, even if the current file is in use.
+  //
+  // This is done on a best effort basis - we do not error if the
+  // operation fails. The consequence is merely that the user
+  // experiences an inconvenient work-flow.
+  //
+  // The code here allows LLD to work on all versions of Windows.
+  // However, at Windows 10 1903 it seems that the behavior of
+  // Windows has changed, so that we could simply delete the output 
+  // file. This code should be simplified once support for older
+  // versions of Windows is dropped.
+  //
+  // Warning: It seems that the WINVER and _WIN32_WINNT preprocessor
+  // defines affect the behavior of the Windows versions of the calls
+  // we are using here. If this code stops working this is worth
+  // bearing in mind.
+  SmallString<128> tmpName;
+  if (!sys::fs::createUniqueFile(path + "%%%%%%%%.tmp", tmpName)) {
+    if (!sys::fs::rename(path, tmpName))
+      path = tmpName;
+    else
+      sys::fs::remove(tmpName);
+  }
   sys::fs::remove(path);
 #else
   if (parallel::strategy.ThreadsRequested == 1 || !sys::fs::exists(path) ||
diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp
index 704ef1a840122..637046e90bbd3 100644
--- a/lld/ELF/Arch/AArch64.cpp
+++ b/lld/ELF/Arch/AArch64.cpp
@@ -126,6 +126,7 @@ RelExpr AArch64::getRelExpr(RelType type, const Symbol &s,
   case R_AARCH64_CONDBR19:
   case R_AARCH64_JUMP26:
   case R_AARCH64_TSTBR14:
+  case R_AARCH64_PLT32:
     return R_PLT_PC;
   case R_AARCH64_PREL16:
   case R_AARCH64_PREL32:
@@ -244,7 +245,8 @@ bool AArch64::needsThunk(RelExpr expr, RelType type, const InputFile *file,
   // ELF for the ARM 64-bit architecture, section Call and Jump relocations
   // only permits range extension thunks for R_AARCH64_CALL26 and
   // R_AARCH64_JUMP26 relocation types.
-  if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26)
+  if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 &&
+      type != R_AARCH64_PLT32)
     return false;
   uint64_t dst = expr == R_PLT_PC ? s.getPltVA() : s.getVA(a);
   return !inBranchRange(type, branchAddr, dst);
@@ -258,11 +260,13 @@ uint32_t AArch64::getThunkSectionSpacing() const {
 }
 
 bool AArch64::inBranchRange(RelType type, uint64_t src, uint64_t dst) const {
-  if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26)
+  if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 &&
+      type != R_AARCH64_PLT32)
     return true;
   // The AArch64 call and unconditional branch instructions have a range of
-  // +/- 128 MiB.
-  uint64_t range = 128 * 1024 * 1024;
+  // +/- 128 MiB. The PLT32 relocation supports a range up to +/- 2 GiB.
+  uint64_t range =
+      type == R_AARCH64_PLT32 ? (UINT64_C(1) << 31) : (128 * 1024 * 1024);
   if (dst > src) {
     // Immediate of branch is signed.
     range -= 4;
@@ -325,6 +329,10 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
     checkIntUInt(loc, val, 32, rel);
     write32le(loc, val);
     break;
+  case R_AARCH64_PLT32:
+    checkInt(loc, val, 32, rel);
+    write32le(loc, val);
+    break;
   case R_AARCH64_ABS64:
   case R_AARCH64_PREL64:
     write64le(loc, val);
diff --git a/lld/ELF/Arch/ARM.cpp b/lld/ELF/Arch/ARM.cpp
index 0dfdbf3d01e20..fd90557cc4f68 100644
--- a/lld/ELF/Arch/ARM.cpp
+++ b/lld/ELF/Arch/ARM.cpp
@@ -121,6 +121,8 @@ RelExpr ARM::getRelExpr(RelType type, const Symbol &s,
     return R_TLSGD_PC;
   case R_ARM_TLS_LDM32:
     return R_TLSLD_PC;
+  case R_ARM_TLS_LDO32:
+    return R_DTPREL;
   case R_ARM_BASE_PREL:
     // B(S) + A - P
     // FIXME: currently B(S) assumed to be .got, this may not hold for all
diff --git a/lld/ELF/Arch/AVR.cpp b/lld/ELF/Arch/AVR.cpp
index 9b733837dd5d0..4513a970b32d7 100644
--- a/lld/ELF/Arch/AVR.cpp
+++ b/lld/ELF/Arch/AVR.cpp
@@ -54,11 +54,131 @@ AVR::AVR() { noneRel = R_AVR_NONE; }
 
 RelExpr AVR::getRelExpr(RelType type, const Symbol &s,
                         const uint8_t *loc) const {
-  return R_ABS;
+  switch (type) {
+  case R_AVR_7_PCREL:
+  case R_AVR_13_PCREL:
+    return R_PC;
+  default:
+    return R_ABS;
+  }
+}
+
+static void writeLDI(uint8_t *loc, uint64_t val) {
+  write16le(loc, (read16le(loc) & 0xf0f0) | (val & 0xf0) << 4 | (val & 0x0f));
 }
 
 void AVR::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   switch (rel.type) {
+  case R_AVR_8:
+    checkUInt(loc, val, 8, rel);
+    *loc = val;
+    break;
+  case R_AVR_16:
+    // Note: this relocation is often used between code and data space, which
+    // are 0x800000 apart in the output ELF file. The bitmask cuts off the high
+    // bit.
+    write16le(loc, val & 0xffff);
+    break;
+  case R_AVR_16_PM:
+    checkAlignment(loc, val, 2, rel);
+    checkUInt(loc, val >> 1, 16, rel);
+    write16le(loc, val >> 1);
+    break;
+  case R_AVR_32:
+    checkUInt(loc, val, 32, rel);
+    write32le(loc, val);
+    break;
+
+  case R_AVR_LDI:
+    checkUInt(loc, val, 8, rel);
+    writeLDI(loc, val & 0xff);
+    break;
+
+  case R_AVR_LO8_LDI_NEG:
+    writeLDI(loc, -val & 0xff);
+    break;
+  case R_AVR_LO8_LDI:
+    writeLDI(loc, val & 0xff);
+    break;
+  case R_AVR_HI8_LDI_NEG:
+    writeLDI(loc, (-val >> 8) & 0xff);
+    break;
+  case R_AVR_HI8_LDI:
+    writeLDI(loc, (val >> 8) & 0xff);
+    break;
+  case R_AVR_HH8_LDI_NEG:
+    writeLDI(loc, (-val >> 16) & 0xff);
+    break;
+  case R_AVR_HH8_LDI:
+    writeLDI(loc, (val >> 16) & 0xff);
+    break;
+  case R_AVR_MS8_LDI_NEG:
+    writeLDI(loc, (-val >> 24) & 0xff);
+    break;
+  case R_AVR_MS8_LDI:
+    writeLDI(loc, (val >> 24) & 0xff);
+    break;
+
+  case R_AVR_LO8_LDI_PM:
+    checkAlignment(loc, val, 2, rel);
+    writeLDI(loc, (val >> 1) & 0xff);
+    break;
+  case R_AVR_HI8_LDI_PM:
+    checkAlignment(loc, val, 2, rel);
+    writeLDI(loc, (val >> 9) & 0xff);
+    break;
+  case R_AVR_HH8_LDI_PM:
+    checkAlignment(loc, val, 2, rel);
+    writeLDI(loc, (val >> 17) & 0xff);
+    break;
+
+  case R_AVR_LO8_LDI_PM_NEG:
+    checkAlignment(loc, val, 2, rel);
+    writeLDI(loc, (-val >> 1) & 0xff);
+    break;
+  case R_AVR_HI8_LDI_PM_NEG:
+    checkAlignment(loc, val, 2, rel);
+    writeLDI(loc, (-val >> 9) & 0xff);
+    break;
+  case R_AVR_HH8_LDI_PM_NEG:
+    checkAlignment(loc, val, 2, rel);
+    writeLDI(loc, (-val >> 17) & 0xff);
+    break;
+
+  case R_AVR_PORT5:
+    checkUInt(loc, val, 5, rel);
+    write16le(loc, (read16le(loc) & 0xff07) | (val << 3));
+    break;
+  case R_AVR_PORT6:
+    checkUInt(loc, val, 6, rel);
+    write16le(loc, (read16le(loc) & 0xf9f0) | (val & 0x30) << 5 | (val & 0x0f));
+    break;
+
+  // Since every jump destination is word aligned we gain an extra bit
+  case R_AVR_7_PCREL: {
+    checkInt(loc, val, 7, rel);
+    checkAlignment(loc, val, 2, rel);
+    const uint16_t target = (val - 2) >> 1;
+    write16le(loc, (read16le(loc) & 0xfc07) | ((target & 0x7f) << 3));
+    break;
+  }
+  case R_AVR_13_PCREL: {
+    checkAlignment(loc, val, 2, rel);
+    const uint16_t target = (val - 2) >> 1;
+    write16le(loc, (read16le(loc) & 0xf000) | (target & 0xfff));
+    break;
+  }
+
+  case R_AVR_6:
+    checkInt(loc, val, 6, rel);
+    write16le(loc, (read16le(loc) & 0xd3f8) | (val & 0x20) << 8 |
+                       (val & 0x18) << 7 | (val & 0x07));
+    break;
+  case R_AVR_6_ADIW:
+    checkInt(loc, val, 6, rel);
+    write16le(loc, (read16le(loc) & 0xff30) | (val & 0x30) << 2 | (val & 0x0F));
+    break;
+
   case R_AVR_CALL: {
     uint16_t hi = val >> 17;
     uint16_t lo = val >> 1;
diff --git a/lld/ELF/Arch/PPC64.cpp b/lld/ELF/Arch/PPC64.cpp
index a182c77209aef..71c568088fb9f 100644
--- a/lld/ELF/Arch/PPC64.cpp
+++ b/lld/ELF/Arch/PPC64.cpp
@@ -376,6 +376,20 @@ static uint32_t readFromHalf16(const uint8_t *loc) {
   return read32(config->isLE ? loc : loc - 2);
 }
 
+// The prefixed instruction is always a 4 byte prefix followed by a 4 byte
+// instruction. Therefore, the prefix is always in lower memory than the
+// instruction (regardless of endianness).
+// As a result, we need to shift the pieces around on little endian machines.
+static void writePrefixedInstruction(uint8_t *loc, uint64_t insn) {
+  insn = config->isLE ? insn << 32 | insn >> 32 : insn;
+  write64(loc, insn);
+}
+
+static uint64_t readPrefixedInstruction(const uint8_t *loc) {
+  uint64_t fullInstr = read64(loc);
+  return config->isLE ? (fullInstr << 32 | fullInstr >> 32) : fullInstr;
+}
+
 PPC64::PPC64() {
   copyRel = R_PPC64_COPY;
   gotRel = R_PPC64_GLOB_DAT;
@@ -657,6 +671,8 @@ RelExpr PPC64::getRelExpr(RelType type, const Symbol &s,
   case R_PPC64_TOC16_HI:
   case R_PPC64_TOC16_LO:
     return R_GOTREL;
+  case R_PPC64_GOT_PCREL34:
+    return R_GOT_PC;
   case R_PPC64_TOC16_HA:
   case R_PPC64_TOC16_LO_DS:
     return config->tocOptimize ? R_PPC64_RELAX_TOC : R_GOTREL;
@@ -665,11 +681,14 @@ RelExpr PPC64::getRelExpr(RelType type, const Symbol &s,
   case R_PPC64_REL14:
   case R_PPC64_REL24:
     return R_PPC64_CALL_PLT;
+  case R_PPC64_REL24_NOTOC:
+    return R_PLT_PC;
   case R_PPC64_REL16_LO:
   case R_PPC64_REL16_HA:
   case R_PPC64_REL16_HI:
   case R_PPC64_REL32:
   case R_PPC64_REL64:
+  case R_PPC64_PCREL34:
     return R_PC;
   case R_PPC64_GOT_TLSGD16:
   case R_PPC64_GOT_TLSGD16_HA:
@@ -976,7 +995,8 @@ void PPC64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     write32(loc, (read32(loc) & ~mask) | (val & mask));
     break;
   }
-  case R_PPC64_REL24: {
+  case R_PPC64_REL24:
+  case R_PPC64_REL24_NOTOC: {
     uint32_t mask = 0x03FFFFFC;
     checkInt(loc, val, 26, rel);
     checkAlignment(loc, val, 4, rel);
@@ -986,6 +1006,28 @@ void PPC64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   case R_PPC64_DTPREL64:
     write64(loc, val - dynamicThreadPointerOffset);
     break;
+  case R_PPC64_PCREL34: {
+    const uint64_t si0Mask = 0x00000003ffff0000;
+    const uint64_t si1Mask = 0x000000000000ffff;
+    const uint64_t fullMask = 0x0003ffff0000ffff;
+    checkInt(loc, val, 34, rel);
+
+    uint64_t instr = readPrefixedInstruction(loc) & ~fullMask;
+    writePrefixedInstruction(loc, instr | ((val & si0Mask) << 16) |
+                             (val & si1Mask));
+    break;
+  }
+  case R_PPC64_GOT_PCREL34: {
+    const uint64_t si0Mask = 0x00000003ffff0000;
+    const uint64_t si1Mask = 0x000000000000ffff;
+    const uint64_t fullMask = 0x0003ffff0000ffff;
+    checkInt(loc, val, 34, rel);
+
+    uint64_t instr = readPrefixedInstruction(loc) & ~fullMask;
+    writePrefixedInstruction(loc, instr | ((val & si0Mask) << 16) |
+                             (val & si1Mask));
+    break;
+  }
   default:
     llvm_unreachable("unknown relocation");
   }
@@ -993,13 +1035,30 @@ void PPC64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
 
 bool PPC64::needsThunk(RelExpr expr, RelType type, const InputFile *file,
                        uint64_t branchAddr, const Symbol &s, int64_t a) const {
-  if (type != R_PPC64_REL14 && type != R_PPC64_REL24)
+  if (type != R_PPC64_REL14 && type != R_PPC64_REL24 &&
+      type != R_PPC64_REL24_NOTOC)
     return false;
 
+  // FIXME: Remove the fatal error once the call protocol is implemented.
+  if (type == R_PPC64_REL24_NOTOC && s.isInPlt())
+    fatal("unimplemented feature: external function call with the reltype"
+          " R_PPC64_REL24_NOTOC");
+
   // If a function is in the Plt it needs to be called with a call-stub.
   if (s.isInPlt())
     return true;
 
+  // FIXME: Remove the fatal error once the call protocol is implemented.
+  if (type == R_PPC64_REL24_NOTOC && (s.stOther >> 5) > 1)
+    fatal("unimplemented feature: local function call with the reltype"
+          " R_PPC64_REL24_NOTOC and the callee needs toc-pointer setup");
+
+  // This check looks at the st_other bits of the callee with relocation
+  // R_PPC64_REL14 or R_PPC64_REL24. If the value is 1, then the callee
+  // clobbers the TOC and we need an R2 save stub.
+  if (type != R_PPC64_REL24_NOTOC && (s.stOther >> 5) == 1)
+    return true;
+
   // If a symbol is a weak undefined and we are compiling an executable
   // it doesn't need a range-extending thunk since it can't be called.
   if (s.isUndefWeak() && !config->shared)
@@ -1025,7 +1084,7 @@ bool PPC64::inBranchRange(RelType type, uint64_t src, uint64_t dst) const {
   int64_t offset = dst - src;
   if (type == R_PPC64_REL14)
     return isInt<16>(offset);
-  if (type == R_PPC64_REL24)
+  if (type == R_PPC64_REL24 || type == R_PPC64_REL24_NOTOC)
     return isInt<26>(offset);
   llvm_unreachable("unsupported relocation type used in branch");
 }
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index cfdd1ebf0b038..e74a4a0c5b223 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -145,6 +145,7 @@ struct Configuration {
   bool checkSections;
   bool compressDebugSections;
   bool cref;
+  std::vector<std::pair<llvm::GlobPattern, uint64_t>> deadRelocInNonAlloc;
   bool defineCommon;
   bool demangle = true;
   bool dependentLibraries;
@@ -230,6 +231,7 @@ struct Configuration {
   bool zRelro;
   bool zRodynamic;
   bool zShstk;
+  uint8_t zStartStopVisibility;
   bool zText;
   bool zRetpolineplt;
   bool zWxneeded;
diff --git a/lld/ELF/DWARF.h b/lld/ELF/DWARF.h
index 8609e35faf953..a12dae6e99600 100644
--- a/lld/ELF/DWARF.h
+++ b/lld/ELF/DWARF.h
@@ -56,11 +56,11 @@ template <class ELFT> class LLDDwarfObj final : public llvm::DWARFObject {
     return addrSection;
   }
 
-  const llvm::DWARFSection &getGnuPubnamesSection() const override {
+  const LLDDWARFSection &getGnuPubnamesSection() const override {
     return gnuPubnamesSection;
   }
 
-  const llvm::DWARFSection &getGnuPubtypesSection() const override {
+  const LLDDWARFSection &getGnuPubtypesSection() const override {
     return gnuPubtypesSection;
   }
 
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 7430e0d1099d3..301f11359823b 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -411,6 +411,24 @@ static GnuStackKind getZGnuStack(opt::InputArgList &args) {
   return GnuStackKind::NoExec;
 }
 
+static uint8_t getZStartStopVisibility(opt::InputArgList &args) {
+  for (auto *arg : args.filtered_reverse(OPT_z)) {
+    std::pair<StringRef, StringRef> kv = StringRef(arg->getValue()).split('=');
+    if (kv.first == "start-stop-visibility") {
+      if (kv.second == "default")
+        return STV_DEFAULT;
+      else if (kv.second == "internal")
+        return STV_INTERNAL;
+      else if (kv.second == "hidden")
+        return STV_HIDDEN;
+      else if (kv.second == "protected")
+        return STV_PROTECTED;
+      error("unknown -z start-stop-visibility= value: " + StringRef(kv.second));
+    }
+  }
+  return STV_PROTECTED;
+}
+
 static bool isKnownZFlag(StringRef s) {
   return s == "combreloc" || s == "copyreloc" || s == "defs" ||
          s == "execstack" || s == "force-bti" || s == "force-ibt" ||
@@ -426,7 +444,9 @@ static bool isKnownZFlag(StringRef s) {
          s == "rela" || s == "relro" || s == "retpolineplt" ||
          s == "rodynamic" || s == "shstk" || s == "text" || s == "undefs" ||
          s == "wxneeded" || s.startswith("common-page-size=") ||
-         s.startswith("max-page-size=") || s.startswith("stack-size=");
+         s.startswith("dead-reloc-in-nonalloc=") ||
+         s.startswith("max-page-size=") || s.startswith("stack-size=") ||
+         s.startswith("start-stop-visibility=");
 }
 
 // Report an error for an unknown -z option.
@@ -1046,9 +1066,31 @@ static void readConfigs(opt::InputArgList &args) {
   config->zSeparate = getZSeparate(args);
   config->zShstk = hasZOption(args, "shstk");
   config->zStackSize = args::getZOptionValue(args, OPT_z, "stack-size", 0);
+  config->zStartStopVisibility = getZStartStopVisibility(args);
   config->zText = getZFlag(args, "text", "notext", true);
   config->zWxneeded = hasZOption(args, "wxneeded");
 
+  for (opt::Arg *arg : args.filtered(OPT_z)) {
+    std::pair<StringRef, StringRef> option =
+        StringRef(arg->getValue()).split('=');
+    if (option.first != "dead-reloc-in-nonalloc")
+      continue;
+    constexpr StringRef errPrefix = "-z dead-reloc-in-nonalloc=: ";
+    std::pair<StringRef, StringRef> kv = option.second.split('=');
+    if (kv.first.empty() || kv.second.empty()) {
+      error(errPrefix + "expected <section_glob>=<value>");
+      continue;
+    }
+    uint64_t v;
+    if (!to_integer(kv.second, v))
+      error(errPrefix + "expected a non-negative integer, but got '" +
+            kv.second + "'");
+    else if (Expected<GlobPattern> pat = GlobPattern::create(kv.first))
+      config->deadRelocInNonAlloc.emplace_back(std::move(*pat), v);
+    else
+      error(errPrefix + toString(pat.takeError()));
+  }
+
   // Parse LTO options.
   if (auto *arg = args.getLastArg(OPT_plugin_opt_mcpu_eq))
     parseClangOption(saver.save("-mcpu=" + StringRef(arg->getValue())),
@@ -1697,8 +1739,11 @@ template <class ELFT> void LinkerDriver::compileBitcodeFiles() {
   for (InputFile *file : lto->compile()) {
     auto *obj = cast<ObjFile<ELFT>>(file);
     obj->parse(/*ignoreComdats=*/true);
-    for (Symbol *sym : obj->getGlobalSymbols())
-      sym->parseSymbolVersion();
+
+    // Parse '@' in symbol names for non-relocatable output.
+    if (!config->relocatable)
+      for (Symbol *sym : obj->getGlobalSymbols())
+        sym->parseSymbolVersion();
     objectFiles.push_back(file);
   }
 }
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index b0bec7349eb8e..e59bf626be501 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -1711,8 +1711,9 @@ template <class ELFT> void LazyObjFile::parse() {
         continue;
       sym->resolve(LazyObject{*this, sym->getName()});
 
-      // MemoryBuffer is emptied if this file is instantiated as ObjFile.
-      if (mb.getBuffer().empty())
+      // If fetched, stop iterating because this->symbols has been transferred
+      // to the instantiated ObjFile.
+      if (fetched)
         return;
     }
     return;
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index d5e1370eeda09..7a7ebd9749099 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -556,6 +556,7 @@ static uint64_t getAArch64UndefinedRelativeWeakVA(uint64_t type, uint64_t a,
   case R_AARCH64_PREL64:
   case R_AARCH64_ADR_PREL_LO21:
   case R_AARCH64_LD_PREL_LO19:
+  case R_AARCH64_PLT32:
     return p + a;
   }
   llvm_unreachable("AArch64 pc-relative relocation expected\n");
@@ -852,6 +853,16 @@ uint64_t InputSectionBase::getRelocTargetVA(const InputFile *file, RelType type,
 template <class ELFT, class RelTy>
 void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef<RelTy> rels) {
   const unsigned bits = sizeof(typename ELFT::uint) * 8;
+  const bool isDebug = isDebugSection(*this);
+  const bool isDebugLocOrRanges =
+      isDebug && (name == ".debug_loc" || name == ".debug_ranges");
+  const bool isDebugLine = isDebug && name == ".debug_line";
+  Optional<uint64_t> tombstone;
+  for (const auto &patAndValue : llvm::reverse(config->deadRelocInNonAlloc))
+    if (patAndValue.first.match(this->name)) {
+      tombstone = patAndValue.second;
+      break;
+    }
 
   for (const RelTy &rel : rels) {
     RelType type = rel.getType(config->isMips64EL);
@@ -902,11 +913,44 @@ void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef<RelTy> rels) {
       continue;
     }
 
-    if (sym.isTls() && !Out::tlsPhdr)
-      target->relocateNoSym(bufLoc, type, 0);
-    else
-      target->relocateNoSym(bufLoc, type,
-                            SignExtend64<bits>(sym.getVA(addend)));
+    if (tombstone ||
+        (isDebug && (type == target->symbolicRel || expr == R_DTPREL))) {
+      // Resolve relocations in .debug_* referencing (discarded symbols or ICF
+      // folded section symbols) to a tombstone value. Resolving to addend is
+      // unsatisfactory because the result address range may collide with a
+      // valid range of low address, or leave multiple CUs claiming ownership of
+      // the same range of code, which may confuse consumers.
+      //
+      // To address the problems, we use -1 as a tombstone value for most
+      // .debug_* sections. We have to ignore the addend because we don't want
+      // to resolve an address attribute (which may have a non-zero addend) to
+      // -1+addend (wrap around to a low address).
+      //
+      // R_DTPREL type relocations represent an offset into the dynamic thread
+      // vector. The computed value is st_value plus a non-negative offset.
+      // Negative values are invalid, so -1 can be used as the tombstone value.
+      //
+      // If the referenced symbol is discarded (made Undefined), or the
+      // section defining the referenced symbol is garbage collected,
+      // sym.getOutputSection() is nullptr. `ds->section->repl != ds->section`
+      // catches the ICF folded case. However, resolving a relocation in
+      // .debug_line to -1 would stop debugger users from setting breakpoints on
+      // the folded-in function, so exclude .debug_line.
+      //
+      // For pre-DWARF-v5 .debug_loc and .debug_ranges, -1 is a reserved value
+      // (base address selection entry), so -2 is used.
+      auto *ds = dyn_cast<Defined>(&sym);
+      if (!sym.getOutputSection() ||
+          (ds && ds->section->repl != ds->section && !isDebugLine)) {
+        // If -z dead-reloc-in-nonalloc= is specified, respect it.
+        const uint64_t value =
+            tombstone ? SignExtend64<bits>(*tombstone)
+                      : (isDebugLocOrRanges ? UINT64_MAX - 1 : UINT64_MAX);
+        target->relocateNoSym(bufLoc, type, value);
+        continue;
+      }
+    }
+    target->relocateNoSym(bufLoc, type, SignExtend64<bits>(sym.getVA(addend)));
   }
 }
 
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 0a16faa2b8fe9..c3c1309aca1a7 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -436,7 +436,7 @@ defm warn_backrefs: BB<"warn-backrefs",
     "Do not warn about backward symbol references to fetch archive members (default)">;
 
 defm warn_backrefs_exclude
-    : Eq<"warn-backrefs-exclude",
+    : EEq<"warn-backrefs-exclude",
          "Glob describing an archive (or an object file within --start-lib) "
          "which should be ignored for --warn-backrefs.">,
       MetaVarName<"<glob>">;
@@ -635,6 +635,7 @@ def: F<"no-copy-dt-needed-entries">;
 def: F<"no-ctors-in-init-array">;
 def: F<"no-keep-memory">;
 def: F<"no-pipeline-knowledge">;
+def: F<"no-relax">;
 def: F<"no-warn-mismatch">;
 def: Flag<["-"], "p">;
 def: Separate<["--", "-"], "rpath-link">;
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index dfae234fd60ce..751ded3977684 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -197,9 +197,9 @@ handleTlsRelocation(RelType type, Symbol &sym, InputSectionBase &c,
     return 1;
   }
 
-  bool canRelax = config->emachine != EM_ARM &&
-                  config->emachine != EM_HEXAGON &&
-                  config->emachine != EM_RISCV;
+  bool toExecRelax = !config->shared && config->emachine != EM_ARM &&
+                     config->emachine != EM_HEXAGON &&
+                     config->emachine != EM_RISCV;
 
   // If we are producing an executable and the symbol is non-preemptable, it
   // must be defined and the code sequence can be relaxed to use Local-Exec.
@@ -217,7 +217,7 @@ handleTlsRelocation(RelType type, Symbol &sym, InputSectionBase &c,
   if (oneof<R_TLSLD_GOT, R_TLSLD_GOTPLT, R_TLSLD_PC, R_TLSLD_HINT>(
           expr)) {
     // Local-Dynamic relocs can be relaxed to Local-Exec.
-    if (canRelax && !config->shared) {
+    if (toExecRelax) {
       c.relocations.push_back(
           {target->adjustRelaxExpr(type, nullptr, R_RELAX_TLS_LD_TO_LE), type,
            offset, addend, &sym});
@@ -238,7 +238,7 @@ handleTlsRelocation(RelType type, Symbol &sym, InputSectionBase &c,
   }
 
   // Local-Dynamic relocs can be relaxed to Local-Exec.
-  if (expr == R_DTPREL && !config->shared) {
+  if (expr == R_DTPREL && toExecRelax) {
     c.relocations.push_back(
         {target->adjustRelaxExpr(type, nullptr, R_RELAX_TLS_LD_TO_LE), type,
          offset, addend, &sym});
@@ -260,7 +260,7 @@ handleTlsRelocation(RelType type, Symbol &sym, InputSectionBase &c,
 
   if (oneof<R_AARCH64_TLSDESC_PAGE, R_TLSDESC, R_TLSDESC_CALL, R_TLSDESC_PC,
             R_TLSGD_GOT, R_TLSGD_GOTPLT, R_TLSGD_PC>(expr)) {
-    if (!canRelax || config->shared) {
+    if (!toExecRelax) {
       if (in.got->addDynTlsEntry(sym)) {
         uint64_t off = in.got->getGlobalDynOffset(sym);
 
@@ -308,7 +308,7 @@ handleTlsRelocation(RelType type, Symbol &sym, InputSectionBase &c,
   // defined.
   if (oneof<R_GOT, R_GOTPLT, R_GOT_PC, R_AARCH64_GOT_PAGE_PC, R_GOT_OFF,
             R_TLSIE_HINT>(expr) &&
-      canRelax && isLocalInExecutable) {
+      toExecRelax && isLocalInExecutable) {
     c.relocations.push_back({R_RELAX_TLS_IE_TO_LE, type, offset, addend, &sym});
     return 1;
   }
diff --git a/lld/ELF/SymbolTable.cpp b/lld/ELF/SymbolTable.cpp
index f0a6af437c5fa..afc8b05f8767e 100644
--- a/lld/ELF/SymbolTable.cpp
+++ b/lld/ELF/SymbolTable.cpp
@@ -94,7 +94,7 @@ Symbol *SymbolTable::insert(StringRef name) {
 }
 
 Symbol *SymbolTable::addSymbol(const Symbol &newSym) {
-  Symbol *sym = symtab->insert(newSym.getName());
+  Symbol *sym = insert(newSym.getName());
   sym->resolve(newSym);
   return sym;
 }
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index f6d66fff6d4b8..731b9f6580609 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -2705,12 +2705,16 @@ template <class ELFT>
 static std::vector<GdbIndexSection::NameAttrEntry>
 readPubNamesAndTypes(const LLDDwarfObj<ELFT> &obj,
                      const std::vector<GdbIndexSection::CuEntry> &cus) {
-  const DWARFSection &pubNames = obj.getGnuPubnamesSection();
-  const DWARFSection &pubTypes = obj.getGnuPubtypesSection();
+  const LLDDWARFSection &pubNames = obj.getGnuPubnamesSection();
+  const LLDDWARFSection &pubTypes = obj.getGnuPubtypesSection();
 
   std::vector<GdbIndexSection::NameAttrEntry> ret;
-  for (const DWARFSection *pub : {&pubNames, &pubTypes}) {
-    DWARFDebugPubTable table(obj, *pub, config->isLE, true);
+  for (const LLDDWARFSection *pub : {&pubNames, &pubTypes}) {
+    DWARFDataExtractor data(obj, *pub, config->isLE, config->wordsize);
+    DWARFDebugPubTable table;
+    table.extract(data, /*GnuStyle=*/true, [&](Error e) {
+      warn(toString(pub->sec) + ": " + toString(std::move(e)));
+    });
     for (const DWARFDebugPubTable::Set &set : table.getData()) {
       // The value written into the constant pool is kind << 24 | cuIndex. As we
       // don't know how many compilation units precede this object to compute
diff --git a/lld/ELF/Thunks.cpp b/lld/ELF/Thunks.cpp
index 6f0d16ae6796a..ea74d343ebb26 100644
--- a/lld/ELF/Thunks.cpp
+++ b/lld/ELF/Thunks.cpp
@@ -279,6 +279,20 @@ class PPC64PltCallStub final : public Thunk {
   void addSymbols(ThunkSection &isec) override;
 };
 
+// PPC64 R2 Save Stub
+// When the caller requires a valid R2 TOC pointer but the callee does not
+// require a TOC pointer and the callee cannot guarantee that it doesn't
+// clobber R2 then we need to save R2. This stub:
+// 1) Saves the TOC pointer to the stack.
+// 2) Tail calls the callee.
+class PPC64R2SaveStub final : public Thunk {
+public:
+  PPC64R2SaveStub(Symbol &dest) : Thunk(dest, 0) {}
+  uint32_t size() override { return 8; }
+  void writeTo(uint8_t *buf) override;
+  void addSymbols(ThunkSection &isec) override;
+};
+
 // A bl instruction uses a signed 24 bit offset, with an implicit 4 byte
 // alignment. This gives a possible 26 bits of 'reach'. If the call offset is
 // larger then that we need to emit a long-branch thunk. The target address
@@ -822,6 +836,21 @@ void PPC64PltCallStub::addSymbols(ThunkSection &isec) {
   s->file = destination.file;
 }
 
+void PPC64R2SaveStub::writeTo(uint8_t *buf) {
+  int64_t offset = destination.getVA() - (getThunkTargetSym()->getVA() + 4);
+  // The branch offset needs to fit in 26 bits.
+  if (!isInt<26>(offset))
+    fatal("R2 save stub branch offset is too large: " + Twine(offset));
+  write32(buf + 0, 0xf8410018);                         // std  r2,24(r1)
+  write32(buf + 4, 0x48000000 | (offset & 0x03fffffc)); // b    <offset>
+}
+
+void PPC64R2SaveStub::addSymbols(ThunkSection &isec) {
+  Defined *s = addSymbol(saver.save("__toc_save_" + destination.getName()),
+                         STT_FUNC, 0, isec);
+  s->needsTocRestore = true;
+}
+
 void PPC64LongBranchThunk::writeTo(uint8_t *buf) {
   int64_t offset = in.ppc64LongBranchTarget->getEntryVA(&destination, addend) -
                    getPPC64TocBase();
@@ -838,7 +867,8 @@ Thunk::Thunk(Symbol &d, int64_t a) : destination(d), addend(a), offset(0) {}
 Thunk::~Thunk() = default;
 
 static Thunk *addThunkAArch64(RelType type, Symbol &s, int64_t a) {
-  if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26)
+  if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 &&
+      type != R_AARCH64_PLT32)
     fatal("unrecognized relocation type");
   if (config->picThunk)
     return make<AArch64ADRPThunk>(s, a);
@@ -949,6 +979,11 @@ static Thunk *addThunkPPC64(RelType type, Symbol &s, int64_t a) {
   if (s.isInPlt())
     return make<PPC64PltCallStub>(s);
 
+  // This check looks at the st_other bits of the callee. If the value is 1
+  // then the callee clobbers the TOC and we need an R2 save stub.
+  if ((s.stOther >> 5) == 1)
+    return make<PPC64R2SaveStub>(s);
+
   if (config->picThunk)
     return make<PPC64PILongBranchThunk>(s, a);
 
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index 57e0066892526..b9fd03bc2eda0 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -2229,8 +2229,10 @@ void Writer<ELFT>::addStartStopSymbols(OutputSection *sec) {
   StringRef s = sec->name;
   if (!isValidCIdentifier(s))
     return;
-  addOptionalRegular(saver.save("__start_" + s), sec, 0, STV_PROTECTED);
-  addOptionalRegular(saver.save("__stop_" + s), sec, -1, STV_PROTECTED);
+  addOptionalRegular(saver.save("__start_" + s), sec, 0,
+                     config->zStartStopVisibility);
+  addOptionalRegular(saver.save("__stop_" + s), sec, -1,
+                     config->zStartStopVisibility);
 }
 
 static bool needsPtLoad(OutputSection *sec) {
diff --git a/lld/MachO/Arch/X86_64.cpp b/lld/MachO/Arch/X86_64.cpp
index 3ce65ad4e22f1..36f686ca2f1d8 100644
--- a/lld/MachO/Arch/X86_64.cpp
+++ b/lld/MachO/Arch/X86_64.cpp
@@ -34,7 +34,8 @@ struct X86_64 : TargetInfo {
   void writeStubHelperEntry(uint8_t *buf, const DylibSymbol &,
                             uint64_t entryAddr) const override;
 
-  void prepareSymbolRelocation(lld::macho::Symbol &, uint8_t type) override;
+  void prepareSymbolRelocation(lld::macho::Symbol &, const InputSection *,
+                               const Reloc &) override;
   uint64_t getSymbolVA(const lld::macho::Symbol &, uint8_t type) const override;
 };
 
@@ -208,8 +209,9 @@ void X86_64::writeStubHelperEntry(uint8_t *buf, const DylibSymbol &sym,
                    in.stubHelper->addr);
 }
 
-void X86_64::prepareSymbolRelocation(lld::macho::Symbol &sym, uint8_t type) {
-  switch (type) {
+void X86_64::prepareSymbolRelocation(lld::macho::Symbol &sym,
+                                     const InputSection *isec, const Reloc &r) {
+  switch (r.type) {
   case X86_64_RELOC_GOT_LOAD:
     // TODO: implement mov -> lea relaxation for non-dynamic symbols
   case X86_64_RELOC_GOT:
@@ -220,7 +222,17 @@ void X86_64::prepareSymbolRelocation(lld::macho::Symbol &sym, uint8_t type) {
       in.stubs->addEntry(*dysym);
     break;
   }
-  case X86_64_RELOC_UNSIGNED:
+  case X86_64_RELOC_UNSIGNED: {
+    if (auto *dysym = dyn_cast<DylibSymbol>(&sym)) {
+      if (r.length != 3) {
+        error("X86_64_RELOC_UNSIGNED referencing the dynamic symbol " +
+              dysym->getName() + " must have r_length = 3");
+        return;
+      }
+      in.binding->addEntry(dysym, isec, r.offset, r.addend);
+    }
+    break;
+  }
   case X86_64_RELOC_SIGNED:
   case X86_64_RELOC_SIGNED_1:
   case X86_64_RELOC_SIGNED_2:
@@ -228,7 +240,7 @@ void X86_64::prepareSymbolRelocation(lld::macho::Symbol &sym, uint8_t type) {
     break;
   case X86_64_RELOC_SUBTRACTOR:
   case X86_64_RELOC_TLV:
-    fatal("TODO: handle relocation type " + std::to_string(type));
+    fatal("TODO: handle relocation type " + std::to_string(r.type));
     break;
   default:
     llvm_unreachable("unexpected relocation type");
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index f4531e42ad13b..2a3b0042162eb 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -178,8 +178,7 @@ static void addFile(StringRef path) {
     if (!result)
       return;
 
-    std::unique_ptr<llvm::MachO::InterfaceFile> interface{std::move(*result)};
-    inputFiles.push_back(make<DylibFile>(std::move(interface)));
+    inputFiles.push_back(make<DylibFile>(std::move(*result)));
     break;
   }
   default:
diff --git a/lld/MachO/ExportTrie.cpp b/lld/MachO/ExportTrie.cpp
index de8b42d732814..7cc81bcfd5f18 100644
--- a/lld/MachO/ExportTrie.cpp
+++ b/lld/MachO/ExportTrie.cpp
@@ -64,10 +64,7 @@ struct ExportInfo {
 
 } // namespace
 
-namespace lld {
-namespace macho {
-
-struct TrieNode {
+struct macho::TrieNode {
   std::vector<Edge> edges;
   Optional<ExportInfo> info;
   // Estimated offset from the start of the serialized trie to the current node.
@@ -277,13 +274,10 @@ void TrieParser::parse(const uint8_t *buf, const Twine &cumulativeString) {
   }
 }
 
-void parseTrie(const uint8_t *buf, size_t size,
-               const TrieEntryCallback &callback) {
+void macho::parseTrie(const uint8_t *buf, size_t size,
+                      const TrieEntryCallback &callback) {
   if (size == 0)
     return;
 
   TrieParser(buf, size, callback).parse();
 }
-
-} // namespace macho
-} // namespace lld
diff --git a/lld/MachO/MergedOutputSection.h b/lld/MachO/MergedOutputSection.h
index 5658fe742c557..279a7e0f75cd7 100644
--- a/lld/MachO/MergedOutputSection.h
+++ b/lld/MachO/MergedOutputSection.h
@@ -12,6 +12,7 @@
 #include "InputSection.h"
 #include "OutputSection.h"
 #include "lld/Common/LLVM.h"
+#include "llvm/ADT/MapVector.h"
 
 namespace lld {
 namespace macho {
@@ -31,7 +32,7 @@ class MergedOutputSection : public OutputSection {
   uint64_t getSize() const override { return size; }
   uint64_t getFileSize() const override { return fileSize; }
 
-  void mergeInput(InputSection *input) override;
+  void mergeInput(InputSection *input);
   void finalize() override;
 
   void writeTo(uint8_t *buf) const override;
diff --git a/lld/MachO/OutputSection.cpp b/lld/MachO/OutputSection.cpp
index d4c24f6b73455..c006828267cff 100644
--- a/lld/MachO/OutputSection.cpp
+++ b/lld/MachO/OutputSection.cpp
@@ -8,7 +8,6 @@
 
 #include "OutputSection.h"
 #include "OutputSegment.h"
-#include "lld/Common/ErrorHandler.h"
 
 using namespace llvm;
 using namespace lld;
@@ -17,7 +16,3 @@ using namespace lld::macho;
 uint64_t OutputSection::getSegmentOffset() const {
   return addr - parent->firstSection()->addr;
 }
-
-void OutputSection::mergeInput(InputSection *input) {
-  llvm_unreachable("Cannot merge input section into unmergable output section");
-}
diff --git a/lld/MachO/OutputSection.h b/lld/MachO/OutputSection.h
index d9ffb1eb2ccb8..07b53a04639f4 100644
--- a/lld/MachO/OutputSection.h
+++ b/lld/MachO/OutputSection.h
@@ -47,9 +47,6 @@ class OutputSection {
   // Unneeded sections are omitted entirely (header and body).
   virtual bool isNeeded() const { return true; }
 
-  // Some sections may allow coalescing other raw input sections.
-  virtual void mergeInput(InputSection *input);
-
   // Specifically finalizes addresses and section size, not content.
   virtual void finalize() {
     // TODO investigate refactoring synthetic section finalization logic into
@@ -71,38 +68,6 @@ class OutputSection {
   Kind sectionKind;
 };
 
-class OutputSectionComparator {
-public:
-  OutputSectionComparator(uint32_t segmentOrder,
-                          const std::vector<StringRef> &sectOrdering)
-      : segmentOrder(segmentOrder) {
-    for (uint32_t j = 0, m = sectOrdering.size(); j < m; ++j)
-      sectionOrdering[sectOrdering[j]] = j;
-  }
-
-  uint32_t sectionOrder(StringRef secname) {
-    auto sectIt = sectionOrdering.find(secname);
-    if (sectIt != sectionOrdering.end())
-      return sectIt->second;
-    return sectionOrdering.size();
-  }
-
-  // Sort sections within a common segment, which stores them in
-  // a MapVector of section name -> section
-  bool operator()(const std::pair<StringRef, OutputSection *> &a,
-                  const std::pair<StringRef, OutputSection *> &b) {
-    return sectionOrder(a.first) < sectionOrder(b.first);
-  }
-
-  bool operator<(const OutputSectionComparator &b) {
-    return segmentOrder < b.segmentOrder;
-  }
-
-private:
-  uint32_t segmentOrder;
-  llvm::DenseMap<StringRef, uint32_t> sectionOrdering;
-};
-
 } // namespace macho
 } // namespace lld
 
diff --git a/lld/MachO/OutputSegment.cpp b/lld/MachO/OutputSegment.cpp
index bc89ff65a7456..5e57c49f5c0a3 100644
--- a/lld/MachO/OutputSegment.cpp
+++ b/lld/MachO/OutputSegment.cpp
@@ -38,84 +38,20 @@ static uint32_t maxProt(StringRef name) {
 
 size_t OutputSegment::numNonHiddenSections() const {
   size_t count = 0;
-  for (const OutputSegment::SectionMapEntry &i : sections) {
-    OutputSection *os = i.second;
-    count += (!os->isHidden() ? 1 : 0);
+  for (const OutputSection *osec : sections) {
+    count += (!osec->isHidden() ? 1 : 0);
   }
   return count;
 }
 
-void OutputSegment::addOutputSection(OutputSection *os) {
-  os->parent = this;
-  std::pair<SectionMap::iterator, bool> result =
-      sections.insert(SectionMapEntry(os->name, os));
-  if (!result.second) {
-    llvm_unreachable("Attempted to set section, but a section with the same "
-                     "name already exists");
-  }
-}
-
-OutputSection *OutputSegment::getOrCreateOutputSection(StringRef name) {
-  OutputSegment::SectionMap::iterator i = sections.find(name);
-  if (i != sections.end()) {
-    return i->second;
-  }
-
-  auto *os = make<MergedOutputSection>(name);
-  addOutputSection(os);
-  return os;
-}
-
-void OutputSegment::sortOutputSections(OutputSegmentComparator *comparator) {
-  llvm::stable_sort(sections, *comparator->sectionComparator(this));
-}
-
-void OutputSegment::removeUnneededSections() {
-  sections.remove_if([](const std::pair<StringRef, OutputSection *> &p) {
-    return !p.second->isNeeded();
-  });
-}
-
-OutputSegmentComparator::OutputSegmentComparator() {
-  // This defines the order of segments and the sections within each segment.
-  // Segments that are not mentioned here will end up at defaultPosition;
-  // sections that are not mentioned will end up at the end of the section
-  // list for their given segment.
-  std::vector<std::pair<StringRef, std::vector<StringRef>>> ordering{
-      {segment_names::pageZero, {}},
-      {segment_names::text, {section_names::header}},
-      {defaultPosition, {}},
-      // Make sure __LINKEDIT is the last segment (i.e. all its hidden
-      // sections must be ordered after other sections).
-      {segment_names::linkEdit,
-       {
-           section_names::binding,
-           section_names::export_,
-           section_names::symbolTable,
-           section_names::stringTable,
-       }},
-  };
-
-  for (uint32_t i = 0, n = ordering.size(); i < n; ++i) {
-    auto &p = ordering[i];
-    StringRef segname = p.first;
-    const std::vector<StringRef> &sectOrdering = p.second;
-    orderMap.insert(std::pair<StringRef, OutputSectionComparator>(
-        segname, OutputSectionComparator(i, sectOrdering)));
-  }
-
-  // Cache the position for the default comparator since this is the likely
-  // scenario.
-  defaultPositionComparator = &orderMap.find(defaultPosition)->second;
+void OutputSegment::addOutputSection(OutputSection *osec) {
+  osec->parent = this;
+  sections.push_back(osec);
 }
 
 static llvm::DenseMap<StringRef, OutputSegment *> nameToOutputSegment;
 std::vector<OutputSegment *> macho::outputSegments;
 
-OutputSegment *macho::getOutputSegment(StringRef name) {
-  return nameToOutputSegment.lookup(name);
-}
-
 OutputSegment *macho::getOrCreateOutputSegment(StringRef name) {
   OutputSegment *&segRef = nameToOutputSegment[name];
   if (segRef != nullptr)
diff --git a/lld/MachO/OutputSegment.h b/lld/MachO/OutputSegment.h
index 346f1a8506167..d977c281272f2 100644
--- a/lld/MachO/OutputSegment.h
+++ b/lld/MachO/OutputSegment.h
@@ -11,50 +11,35 @@
 
 #include "OutputSection.h"
 #include "lld/Common/LLVM.h"
-#include "llvm/ADT/MapVector.h"
 
 namespace lld {
 namespace macho {
 
 namespace segment_names {
 
-constexpr const char *pageZero = "__PAGEZERO";
-constexpr const char *text = "__TEXT";
-constexpr const char *data = "__DATA";
-constexpr const char *linkEdit = "__LINKEDIT";
-constexpr const char *dataConst = "__DATA_CONST";
+constexpr const char pageZero[] = "__PAGEZERO";
+constexpr const char text[] = "__TEXT";
+constexpr const char data[] = "__DATA";
+constexpr const char linkEdit[] = "__LINKEDIT";
+constexpr const char dataConst[] = "__DATA_CONST";
 
 } // namespace segment_names
 
 class OutputSection;
-class OutputSegmentComparator;
 class InputSection;
 
 class OutputSegment {
 public:
-  using SectionMap = typename llvm::MapVector<StringRef, OutputSection *>;
-  using SectionMapEntry = typename std::pair<StringRef, OutputSection *>;
-
-  const OutputSection *firstSection() const { return sections.front().second; }
-  const OutputSection *lastSection() const { return sections.back().second; }
-
-  bool isNeeded() const {
-    if (name == segment_names::linkEdit)
-      return true;
-    for (const SectionMapEntry &i : sections) {
-      OutputSection *os = i.second;
-      if (os->isNeeded())
-        return true;
-    }
-    return false;
-  }
+  const OutputSection *firstSection() const { return sections.front(); }
+  const OutputSection *lastSection() const { return sections.back(); }
 
-  OutputSection *getOrCreateOutputSection(StringRef name);
   void addOutputSection(OutputSection *os);
-  void sortOutputSections(OutputSegmentComparator *comparator);
-  void removeUnneededSections();
+  void sortOutputSections(
+      llvm::function_ref<bool(OutputSection *, OutputSection *)> comparator) {
+    llvm::stable_sort(sections, comparator);
+  }
 
-  const SectionMap &getSections() const { return sections; }
+  const std::vector<OutputSection *> &getSections() const { return sections; }
   size_t numNonHiddenSections() const;
 
   uint64_t fileOff = 0;
@@ -64,34 +49,11 @@ class OutputSegment {
   uint8_t index;
 
 private:
-  SectionMap sections;
-};
-
-class OutputSegmentComparator {
-public:
-  OutputSegmentComparator();
-
-  OutputSectionComparator *sectionComparator(const OutputSegment *os) {
-    auto it = orderMap.find(os->name);
-    if (it == orderMap.end()) {
-      return defaultPositionComparator;
-    }
-    return &it->second;
-  }
-
-  bool operator()(const OutputSegment *a, const OutputSegment *b) {
-    return *sectionComparator(a) < *sectionComparator(b);
-  }
-
-private:
-  const StringRef defaultPosition = StringRef();
-  llvm::DenseMap<StringRef, OutputSectionComparator> orderMap;
-  OutputSectionComparator *defaultPositionComparator;
+  std::vector<OutputSection *> sections;
 };
 
 extern std::vector<OutputSegment *> outputSegments;
 
-OutputSegment *getOutputSegment(StringRef name);
 OutputSegment *getOrCreateOutputSegment(StringRef name);
 
 } // namespace macho
diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp
index f84a603f521e1..cc0d5a93c40de 100644
--- a/lld/MachO/SyntheticSections.cpp
+++ b/lld/MachO/SyntheticSections.cpp
@@ -11,6 +11,7 @@
 #include "ExportTrie.h"
 #include "InputFiles.h"
 #include "MachOStructs.h"
+#include "MergedOutputSection.h"
 #include "OutputSegment.h"
 #include "SymbolTable.h"
 #include "Symbols.h"
@@ -22,18 +23,17 @@
 #include "llvm/Support/LEB128.h"
 
 using namespace llvm;
-using namespace llvm::MachO;
 using namespace llvm::support;
 using namespace llvm::support::endian;
+using namespace lld;
+using namespace lld::macho;
 
-namespace lld {
-namespace macho {
+InStruct macho::in;
+std::vector<SyntheticSection *> macho::syntheticSections;
 
 SyntheticSection::SyntheticSection(const char *segname, const char *name)
-    : OutputSection(SyntheticKind, name) {
-  // Synthetic sections always know which segment they belong to so hook
-  // them up when they're made
-  getOrCreateOutputSegment(segname)->addOutputSection(this);
+    : OutputSection(SyntheticKind, name), segname(segname) {
+  syntheticSections.push_back(this);
 }
 
 // dyld3's MachOLoaded::getSlide() assumes that the __TEXT segment starts
@@ -47,20 +47,20 @@ void MachHeaderSection::addLoadCommand(LoadCommand *lc) {
 }
 
 uint64_t MachHeaderSection::getSize() const {
-  return sizeof(mach_header_64) + sizeOfCmds;
+  return sizeof(MachO::mach_header_64) + sizeOfCmds;
 }
 
 void MachHeaderSection::writeTo(uint8_t *buf) const {
-  auto *hdr = reinterpret_cast<mach_header_64 *>(buf);
-  hdr->magic = MH_MAGIC_64;
-  hdr->cputype = CPU_TYPE_X86_64;
-  hdr->cpusubtype = CPU_SUBTYPE_X86_64_ALL | CPU_SUBTYPE_LIB64;
+  auto *hdr = reinterpret_cast<MachO::mach_header_64 *>(buf);
+  hdr->magic = MachO::MH_MAGIC_64;
+  hdr->cputype = MachO::CPU_TYPE_X86_64;
+  hdr->cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL | MachO::CPU_SUBTYPE_LIB64;
   hdr->filetype = config->outputType;
   hdr->ncmds = loadCommands.size();
   hdr->sizeofcmds = sizeOfCmds;
-  hdr->flags = MH_NOUNDEFS | MH_DYLDLINK | MH_TWOLEVEL;
-  if (config->outputType == MH_DYLIB && !config->hasReexports)
-    hdr->flags |= MH_NO_REEXPORTED_DYLIBS;
+  hdr->flags = MachO::MH_NOUNDEFS | MachO::MH_DYLDLINK | MachO::MH_TWOLEVEL;
+  if (config->outputType == MachO::MH_DYLIB && !config->hasReexports)
+    hdr->flags |= MachO::MH_NO_REEXPORTED_DYLIBS;
 
   uint8_t *p = reinterpret_cast<uint8_t *>(hdr + 1);
   for (LoadCommand *lc : loadCommands) {
@@ -75,7 +75,7 @@ PageZeroSection::PageZeroSection()
 GotSection::GotSection()
     : SyntheticSection(segment_names::dataConst, section_names::got) {
   align = 8;
-  flags = S_NON_LAZY_SYMBOL_POINTERS;
+  flags = MachO::S_NON_LAZY_SYMBOL_POINTERS;
 
   // TODO: section_64::reserved1 should be an index into the indirect symbol
   // table, which we do not currently emit
@@ -96,7 +96,68 @@ void GotSection::writeTo(uint8_t *buf) const {
 BindingSection::BindingSection()
     : SyntheticSection(segment_names::linkEdit, section_names::binding) {}
 
-bool BindingSection::isNeeded() const { return in.got->isNeeded(); }
+bool BindingSection::isNeeded() const {
+  return bindings.size() != 0 || in.got->isNeeded();
+}
+
+namespace {
+struct Binding {
+  OutputSegment *segment = nullptr;
+  uint64_t offset = 0;
+  int64_t addend = 0;
+  uint8_t ordinal = 0;
+};
+} // namespace
+
+// Encode a sequence of opcodes that tell dyld to write the address of dysym +
+// addend at osec->addr + outSecOff.
+//
+// The bind opcode "interpreter" remembers the values of each binding field, so
+// we only need to encode the differences between bindings. Hence the use of
+// lastBinding.
+static void encodeBinding(const DylibSymbol &dysym, const OutputSection *osec,
+                          uint64_t outSecOff, int64_t addend,
+                          Binding &lastBinding, raw_svector_ostream &os) {
+  using namespace llvm::MachO;
+  OutputSegment *seg = osec->parent;
+  uint64_t offset = osec->getSegmentOffset() + outSecOff;
+  if (lastBinding.segment != seg) {
+    os << static_cast<uint8_t>(BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB |
+                               seg->index);
+    encodeULEB128(offset, os);
+    lastBinding.segment = seg;
+    lastBinding.offset = offset;
+  } else if (lastBinding.offset != offset) {
+    assert(lastBinding.offset <= offset);
+    os << static_cast<uint8_t>(BIND_OPCODE_ADD_ADDR_ULEB);
+    encodeULEB128(offset - lastBinding.offset, os);
+    lastBinding.offset = offset;
+  }
+
+  if (lastBinding.ordinal != dysym.file->ordinal) {
+    if (dysym.file->ordinal <= BIND_IMMEDIATE_MASK) {
+      os << static_cast<uint8_t>(BIND_OPCODE_SET_DYLIB_ORDINAL_IMM |
+                                 dysym.file->ordinal);
+    } else {
+      error("TODO: Support larger dylib symbol ordinals");
+      return;
+    }
+    lastBinding.ordinal = dysym.file->ordinal;
+  }
+
+  if (lastBinding.addend != addend) {
+    os << static_cast<uint8_t>(BIND_OPCODE_SET_ADDEND_SLEB);
+    encodeSLEB128(addend, os);
+    lastBinding.addend = addend;
+  }
+
+  os << static_cast<uint8_t>(BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM)
+     << dysym.getName() << '\0'
+     << static_cast<uint8_t>(BIND_OPCODE_SET_TYPE_IMM | BIND_TYPE_POINTER)
+     << static_cast<uint8_t>(BIND_OPCODE_DO_BIND);
+  // DO_BIND causes dyld to both perform the binding and increment the offset
+  lastBinding.offset += WordSize;
+}
 
 // Emit bind opcodes, which are a stream of byte-sized opcodes that dyld
 // interprets to update a record with the following fields:
@@ -112,42 +173,40 @@ bool BindingSection::isNeeded() const { return in.got->isNeeded(); }
 // entry. It does *not* clear the record state after doing the bind, so
 // subsequent opcodes only need to encode the differences between bindings.
 void BindingSection::finalizeContents() {
-  if (!isNeeded())
-    return;
-
   raw_svector_ostream os{contents};
-  os << static_cast<uint8_t>(BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB |
-                             in.got->parent->index);
-  encodeULEB128(in.got->getSegmentOffset(), os);
-  uint32_t entries_to_skip = 0;
+  Binding lastBinding;
+  bool didEncode = false;
+  size_t gotIdx = 0;
   for (const Symbol *sym : in.got->getEntries()) {
     if (const auto *dysym = dyn_cast<DylibSymbol>(sym)) {
-      if (entries_to_skip != 0) {
-        os << static_cast<uint8_t>(BIND_OPCODE_ADD_ADDR_ULEB);
-        encodeULEB128(WordSize * entries_to_skip, os);
-        entries_to_skip = 0;
-      }
-
-      // TODO: Implement compact encoding -- we only need to encode the
-      // differences between consecutive symbol entries.
-      if (dysym->file->ordinal <= BIND_IMMEDIATE_MASK) {
-        os << static_cast<uint8_t>(BIND_OPCODE_SET_DYLIB_ORDINAL_IMM |
-                                   dysym->file->ordinal);
-      } else {
-        error("TODO: Support larger dylib symbol ordinals");
-        continue;
-      }
-      os << static_cast<uint8_t>(BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM)
-         << dysym->getName() << '\0'
-         << static_cast<uint8_t>(BIND_OPCODE_SET_TYPE_IMM | BIND_TYPE_POINTER)
-         << static_cast<uint8_t>(BIND_OPCODE_DO_BIND);
-    } else {
-      // We have a defined symbol with a pre-populated address; skip over it.
-      ++entries_to_skip;
+      didEncode = true;
+      encodeBinding(*dysym, in.got, gotIdx * WordSize, 0, lastBinding, os);
     }
+    ++gotIdx;
   }
 
-  os << static_cast<uint8_t>(BIND_OPCODE_DONE);
+  // Sorting the relocations by segment and address allows us to encode them
+  // more compactly.
+  llvm::sort(bindings, [](const BindingEntry &a, const BindingEntry &b) {
+    OutputSegment *segA = a.isec->parent->parent;
+    OutputSegment *segB = b.isec->parent->parent;
+    if (segA != segB)
+      return segA->fileOff < segB->fileOff;
+    OutputSection *osecA = a.isec->parent;
+    OutputSection *osecB = b.isec->parent;
+    if (osecA != osecB)
+      return osecA->addr < osecB->addr;
+    if (a.isec != b.isec)
+      return a.isec->outSecOff < b.isec->outSecOff;
+    return a.offset < b.offset;
+  });
+  for (const BindingEntry &b : bindings) {
+    didEncode = true;
+    encodeBinding(*b.dysym, b.isec->parent, b.isec->outSecOff + b.offset,
+                  b.addend, lastBinding, os);
+  }
+  if (didEncode)
+    os << static_cast<uint8_t>(MachO::BIND_OPCODE_DONE);
 }
 
 void BindingSection::writeTo(uint8_t *buf) const {
@@ -219,7 +278,7 @@ ImageLoaderCacheSection::ImageLoaderCacheSection() {
 LazyPointerSection::LazyPointerSection()
     : SyntheticSection(segment_names::data, "__la_symbol_ptr") {
   align = 8;
-  flags = S_LAZY_SYMBOL_POINTERS;
+  flags = MachO::S_LAZY_SYMBOL_POINTERS;
 }
 
 uint64_t LazyPointerSection::getSize() const {
@@ -265,20 +324,21 @@ void LazyBindingSection::writeTo(uint8_t *buf) const {
 uint32_t LazyBindingSection::encode(const DylibSymbol &sym) {
   uint32_t opstreamOffset = contents.size();
   OutputSegment *dataSeg = in.lazyPointers->parent;
-  os << static_cast<uint8_t>(BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB |
+  os << static_cast<uint8_t>(MachO::BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB |
                              dataSeg->index);
   uint64_t offset = in.lazyPointers->addr - dataSeg->firstSection()->addr +
                     sym.stubsIndex * WordSize;
   encodeULEB128(offset, os);
-  if (sym.file->ordinal <= BIND_IMMEDIATE_MASK)
-    os << static_cast<uint8_t>(BIND_OPCODE_SET_DYLIB_ORDINAL_IMM |
+  if (sym.file->ordinal <= MachO::BIND_IMMEDIATE_MASK)
+    os << static_cast<uint8_t>(MachO::BIND_OPCODE_SET_DYLIB_ORDINAL_IMM |
                                sym.file->ordinal);
   else
     fatal("TODO: Support larger dylib symbol ordinals");
 
-  os << static_cast<uint8_t>(BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM)
-     << sym.getName() << '\0' << static_cast<uint8_t>(BIND_OPCODE_DO_BIND)
-     << static_cast<uint8_t>(BIND_OPCODE_DONE);
+  os << static_cast<uint8_t>(MachO::BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM)
+     << sym.getName() << '\0'
+     << static_cast<uint8_t>(MachO::BIND_OPCODE_DO_BIND)
+     << static_cast<uint8_t>(MachO::BIND_OPCODE_DONE);
   return opstreamOffset;
 }
 
@@ -321,7 +381,7 @@ void SymtabSection::writeTo(uint8_t *buf) const {
     // TODO support other symbol types
     // TODO populate n_desc
     if (auto *defined = dyn_cast<Defined>(entry.sym)) {
-      nList->n_type = N_EXT | N_SECT;
+      nList->n_type = MachO::N_EXT | MachO::N_SECT;
       nList->n_sect = defined->isec->parent->index;
       // For the N_SECT symbol type, n_value is the address of the symbol
       nList->n_value = defined->value + defined->isec->getVA();
@@ -347,8 +407,3 @@ void StringTableSection::writeTo(uint8_t *buf) const {
     off += str.size() + 1; // account for null terminator
   }
 }
-
-InStruct in;
-
-} // namespace macho
-} // namespace lld
diff --git a/lld/MachO/SyntheticSections.h b/lld/MachO/SyntheticSections.h
index 95452538c445c..a8fbf6c8a2657 100644
--- a/lld/MachO/SyntheticSections.h
+++ b/lld/MachO/SyntheticSections.h
@@ -23,14 +23,14 @@ namespace macho {
 
 namespace section_names {
 
-constexpr const char *pageZero = "__pagezero";
-constexpr const char *header = "__mach_header";
-constexpr const char *binding = "__binding";
-constexpr const char *lazyBinding = "__lazy_binding";
-constexpr const char *export_ = "__export";
-constexpr const char *symbolTable = "__symbol_table";
-constexpr const char *stringTable = "__string_table";
-constexpr const char *got = "__got";
+constexpr const char pageZero[] = "__pagezero";
+constexpr const char header[] = "__mach_header";
+constexpr const char binding[] = "__binding";
+constexpr const char lazyBinding[] = "__lazy_binding";
+constexpr const char export_[] = "__export";
+constexpr const char symbolTable[] = "__symbol_table";
+constexpr const char stringTable[] = "__string_table";
+constexpr const char got[] = "__got";
 
 } // namespace section_names
 
@@ -45,6 +45,8 @@ class SyntheticSection : public OutputSection {
   static bool classof(const OutputSection *sec) {
     return sec->kind() == SyntheticKind;
   }
+
+  const StringRef segname;
 };
 
 // The header of the Mach-O file, which must have a file offset of zero.
@@ -92,6 +94,16 @@ class GotSection : public SyntheticSection {
   llvm::SetVector<const Symbol *> entries;
 };
 
+struct BindingEntry {
+  const DylibSymbol *dysym;
+  const InputSection *isec;
+  uint64_t offset;
+  int64_t addend;
+  BindingEntry(const DylibSymbol *dysym, const InputSection *isec,
+               uint64_t offset, int64_t addend)
+      : dysym(dysym), isec(isec), offset(offset), addend(addend) {}
+};
+
 // Stores bind opcodes for telling dyld which symbols to load non-lazily.
 class BindingSection : public SyntheticSection {
 public:
@@ -105,6 +117,13 @@ class BindingSection : public SyntheticSection {
   bool isNeeded() const override;
   void writeTo(uint8_t *buf) const override;
 
+  void addEntry(const DylibSymbol *dysym, const InputSection *isec,
+                uint64_t offset, int64_t addend) {
+    bindings.emplace_back(dysym, isec, offset, addend);
+  }
+
+private:
+  std::vector<BindingEntry> bindings;
   SmallVector<char, 128> contents;
 };
 
@@ -254,6 +273,7 @@ class SymtabSection : public SyntheticSection {
 };
 
 struct InStruct {
+  BindingSection *binding = nullptr;
   GotSection *got = nullptr;
   LazyPointerSection *lazyPointers = nullptr;
   StubsSection *stubs = nullptr;
@@ -262,6 +282,7 @@ struct InStruct {
 };
 
 extern InStruct in;
+extern std::vector<SyntheticSection *> syntheticSections;
 
 } // namespace macho
 } // namespace lld
diff --git a/lld/MachO/Target.h b/lld/MachO/Target.h
index 7687fcdc66eae..8ea1bde12307f 100644
--- a/lld/MachO/Target.h
+++ b/lld/MachO/Target.h
@@ -53,7 +53,8 @@ class TargetInfo {
   // depending on the relocation type. prepareSymbolRelocation() will set up the
   // GOT/stubs entries, and getSymbolVA() will return the addresses of those
   // entries.
-  virtual void prepareSymbolRelocation(Symbol &, uint8_t type) = 0;
+  virtual void prepareSymbolRelocation(Symbol &, const InputSection *,
+                                       const Reloc &) = 0;
   virtual uint64_t getSymbolVA(const Symbol &, uint8_t type) const = 0;
 
   uint32_t cpuType;
diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp
index 0fcde15699c3e..03000a7f437e0 100644
--- a/lld/MachO/Writer.cpp
+++ b/lld/MachO/Writer.cpp
@@ -54,7 +54,6 @@ class Writer {
   uint64_t addr = 0;
   uint64_t fileOff = 0;
   MachHeaderSection *headerSection = nullptr;
-  BindingSection *bindingSection = nullptr;
   LazyBindingSection *lazyBindingSection = nullptr;
   ExportSection *exportSection = nullptr;
   StringTableSection *stringTableSection = nullptr;
@@ -134,25 +133,23 @@ class LCSegment : public LoadCommand {
         seg->lastSection()->addr + seg->lastSection()->getSize() - c->vmaddr;
     c->nsects = seg->numNonHiddenSections();
 
-    for (auto &p : seg->getSections()) {
-      StringRef s = p.first;
-      OutputSection *section = p.second;
-      c->filesize += section->getFileSize();
+    for (OutputSection *osec : seg->getSections()) {
+      c->filesize += osec->getFileSize();
 
-      if (section->isHidden())
+      if (osec->isHidden())
         continue;
 
       auto *sectHdr = reinterpret_cast<section_64 *>(buf);
       buf += sizeof(section_64);
 
-      memcpy(sectHdr->sectname, s.data(), s.size());
+      memcpy(sectHdr->sectname, osec->name.data(), osec->name.size());
       memcpy(sectHdr->segname, name.data(), name.size());
 
-      sectHdr->addr = section->addr;
-      sectHdr->offset = section->fileOff;
-      sectHdr->align = Log2_32(section->align);
-      sectHdr->flags = section->flags;
-      sectHdr->size = section->getSize();
+      sectHdr->addr = osec->addr;
+      sectHdr->offset = osec->fileOff;
+      sectHdr->align = Log2_32(osec->align);
+      sectHdr->flags = osec->flags;
+      sectHdr->size = osec->getSize();
     }
   }
 
@@ -256,7 +253,7 @@ void Writer::scanRelocations() {
           error("undefined symbol " + s->getName() + ", referenced from " +
                 sys::path::filename(isec->file->getName()));
         else
-          target->prepareSymbolRelocation(*s, r.type);
+          target->prepareSymbolRelocation(*s, isec, r);
       }
     }
   }
@@ -264,7 +261,7 @@ void Writer::scanRelocations() {
 
 void Writer::createLoadCommands() {
   headerSection->addLoadCommand(
-      make<LCDyldInfo>(bindingSection, lazyBindingSection, exportSection));
+      make<LCDyldInfo>(in.binding, lazyBindingSection, exportSection));
   headerSection->addLoadCommand(
       make<LCSymtab>(symtabSection, stringTableSection));
   headerSection->addLoadCommand(make<LCDysymtab>());
@@ -336,28 +333,63 @@ static DenseMap<const InputSection *, size_t> buildInputSectionPriorities() {
   return sectionPriorities;
 }
 
+static int segmentOrder(OutputSegment *seg) {
+  return StringSwitch<int>(seg->name)
+      .Case(segment_names::pageZero, -2)
+      .Case(segment_names::text, -1)
+      // Make sure __LINKEDIT is the last segment (i.e. all its hidden
+      // sections must be ordered after other sections).
+      .Case(segment_names::linkEdit, std::numeric_limits<int>::max())
+      .Default(0);
+}
+
+static int sectionOrder(OutputSection *osec) {
+  StringRef segname = osec->parent->name;
+  // Sections are uniquely identified by their segment + section name.
+  if (segname == segment_names::text) {
+    if (osec->name == section_names::header)
+      return -1;
+  } else if (segname == segment_names::linkEdit) {
+    return StringSwitch<int>(osec->name)
+        .Case(section_names::binding, -4)
+        .Case(section_names::export_, -3)
+        .Case(section_names::symbolTable, -2)
+        .Case(section_names::stringTable, -1)
+        .Default(0);
+  }
+  // ZeroFill sections must always be the at the end of their segments,
+  // otherwise subsequent sections may get overwritten with zeroes at runtime.
+  if (isZeroFill(osec->flags))
+    return std::numeric_limits<int>::max();
+  return 0;
+}
+
+template <typename T, typename F>
+static std::function<bool(T, T)> compareByOrder(F ord) {
+  return [=](T a, T b) { return ord(a) < ord(b); };
+}
+
 // Sorting only can happen once all outputs have been collected. Here we sort
 // segments, output sections within each segment, and input sections within each
 // output segment.
 static void sortSegmentsAndSections() {
-  auto comparator = OutputSegmentComparator();
-  llvm::stable_sort(outputSegments, comparator);
+  llvm::stable_sort(outputSegments,
+                    compareByOrder<OutputSegment *>(segmentOrder));
 
   DenseMap<const InputSection *, size_t> isecPriorities =
       buildInputSectionPriorities();
 
   uint32_t sectionIndex = 0;
   for (OutputSegment *seg : outputSegments) {
-    seg->sortOutputSections(&comparator);
-    for (auto &p : seg->getSections()) {
-      OutputSection *section = p.second;
+    seg->sortOutputSections(compareByOrder<OutputSection *>(sectionOrder));
+    for (auto *osec : seg->getSections()) {
       // Now that the output sections are sorted, assign the final
       // output section indices.
-      if (!section->isHidden())
-        section->index = ++sectionIndex;
+      if (!osec->isHidden())
+        osec->index = ++sectionIndex;
 
       if (!isecPriorities.empty()) {
-        if (auto *merged = dyn_cast<MergedOutputSection>(section)) {
+        if (auto *merged = dyn_cast<MergedOutputSection>(osec)) {
           llvm::stable_sort(merged->inputs,
                             [&](InputSection *a, InputSection *b) {
                               return isecPriorities[a] > isecPriorities[b];
@@ -371,7 +403,6 @@ static void sortSegmentsAndSections() {
 void Writer::createOutputSections() {
   // First, create hidden sections
   headerSection = make<MachHeaderSection>();
-  bindingSection = make<BindingSection>();
   lazyBindingSection = make<LazyBindingSection>();
   stringTableSection = make<StringTableSection>();
   symtabSection = make<SymtabSection>(*stringTableSection);
@@ -387,22 +418,33 @@ void Writer::createOutputSections() {
     llvm_unreachable("unhandled output file type");
   }
 
-  // Then merge input sections into output sections/segments.
+  // Then merge input sections into output sections.
+  MapVector<std::pair<StringRef, StringRef>, MergedOutputSection *>
+      mergedOutputSections;
   for (InputSection *isec : inputSections) {
-    getOrCreateOutputSegment(isec->segname)
-        ->getOrCreateOutputSection(isec->name)
-        ->mergeInput(isec);
+    MergedOutputSection *&osec =
+        mergedOutputSections[{isec->segname, isec->name}];
+    if (osec == nullptr)
+      osec = make<MergedOutputSection>(isec->name);
+    osec->mergeInput(isec);
   }
 
-  // Remove unneeded segments and sections.
-  // TODO: Avoid creating unneeded segments in the first place
-  for (auto it = outputSegments.begin(); it != outputSegments.end();) {
-    OutputSegment *seg = *it;
-    seg->removeUnneededSections();
-    if (!seg->isNeeded())
-      it = outputSegments.erase(it);
-    else
-      ++it;
+  for (const auto &it : mergedOutputSections) {
+    StringRef segname = it.first.first;
+    MergedOutputSection *osec = it.second;
+    getOrCreateOutputSegment(segname)->addOutputSection(osec);
+  }
+
+  for (SyntheticSection *ssec : syntheticSections) {
+    auto it = mergedOutputSections.find({ssec->segname, ssec->name});
+    if (it == mergedOutputSections.end()) {
+      if (ssec->isNeeded())
+        getOrCreateOutputSegment(ssec->segname)->addOutputSection(ssec);
+    } else {
+      error("section from " + it->second->firstSection()->file->getName() +
+            " conflicts with synthetic section " + ssec->segname + "," +
+            ssec->name);
+    }
   }
 }
 
@@ -411,16 +453,15 @@ void Writer::assignAddresses(OutputSegment *seg) {
   fileOff = alignTo(fileOff, PageSize);
   seg->fileOff = fileOff;
 
-  for (auto &p : seg->getSections()) {
-    OutputSection *section = p.second;
-    addr = alignTo(addr, section->align);
-    fileOff = alignTo(fileOff, section->align);
-    section->addr = addr;
-    section->fileOff = isZeroFill(section->flags) ? 0 : fileOff;
-    section->finalize();
+  for (auto *osec : seg->getSections()) {
+    addr = alignTo(addr, osec->align);
+    fileOff = alignTo(fileOff, osec->align);
+    osec->addr = addr;
+    osec->fileOff = isZeroFill(osec->flags) ? 0 : fileOff;
+    osec->finalize();
 
-    addr += section->getSize();
-    fileOff += section->getFileSize();
+    addr += osec->getSize();
+    fileOff += osec->getFileSize();
   }
 }
 
@@ -438,12 +479,9 @@ void Writer::openFile() {
 
 void Writer::writeSections() {
   uint8_t *buf = buffer->getBufferStart();
-  for (OutputSegment *seg : outputSegments) {
-    for (auto &p : seg->getSections()) {
-      OutputSection *section = p.second;
-      section->writeTo(buf + section->fileOff);
-    }
-  }
+  for (OutputSegment *seg : outputSegments)
+    for (OutputSection *osec : seg->getSections())
+      osec->writeTo(buf + osec->fileOff);
 }
 
 void Writer::run() {
@@ -473,7 +511,7 @@ void Writer::run() {
       assignAddresses(seg);
 
   // Fill __LINKEDIT contents.
-  bindingSection->finalizeContents();
+  in.binding->finalizeContents();
   lazyBindingSection->finalizeContents();
   exportSection->finalizeContents();
   symtabSection->finalizeContents();
@@ -495,6 +533,7 @@ void Writer::run() {
 void macho::writeResult() { Writer().run(); }
 
 void macho::createSyntheticSections() {
+  in.binding = make<BindingSection>();
   in.got = make<GotSection>();
   in.lazyPointers = make<LazyPointerSection>();
   in.stubs = make<StubsSection>();
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index 0522feb145f1a..5edeaf85f93f1 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -308,6 +308,8 @@ Disable merging .ARM.exidx entries.
 Page align sections.
 .It Fl -no-omagic
 Do not set the text data sections to be writable, page align sections.
+.It Fl -no-relax
+Disable target-specific relaxations. This is currently a no-op.
 .It Fl -no-rosegment
 Do not put read-only non-executable sections in their own segment.
 .It Fl -no-undefined-version
@@ -623,6 +625,13 @@ Use wrapper functions for symbol.
 Linker option extensions.
 .Bl -tag -width indent -compact
 .Pp
+.It Cm dead-reloc-in-nonalloc Ns = Ns Ar section_glob=value
+Resolve a relocation in a matched non-SHF_ALLOC section referencing a discarded symbol to
+.Ar value
+Accepts globs, in the event of a section matching more than one option, the last
+option takes precedence. An order of least specific to most specific match is
+recommended.
+.Pp
 .It Cm execstack
 Make the main stack executable.
 Stack permissions are recorded in the
diff --git a/lld/lib/ReaderWriter/MachO/CMakeLists.txt b/lld/lib/ReaderWriter/MachO/CMakeLists.txt
index 37d1de432c0f1..bdbfbee746863 100644
--- a/lld/lib/ReaderWriter/MachO/CMakeLists.txt
+++ b/lld/lib/ReaderWriter/MachO/CMakeLists.txt
@@ -24,6 +24,7 @@ add_lld_library(lldMachO
     Demangle
     Object
     Support
+    TextAPI
 
   LINK_LIBS
     lldCommon
diff --git a/lld/lib/ReaderWriter/MachO/File.h b/lld/lib/ReaderWriter/MachO/File.h
index 072702973f81a..8d59656beab58 100644
--- a/lld/lib/ReaderWriter/MachO/File.h
+++ b/lld/lib/ReaderWriter/MachO/File.h
@@ -17,6 +17,8 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/Support/Format.h"
+#include "llvm/TextAPI/MachO/InterfaceFile.h"
+#include "llvm/TextAPI/MachO/TextAPIReader.h"
 #include <unordered_map>
 
 namespace lld {
@@ -322,7 +324,8 @@ class MachODylibFile : public SharedLibraryFile {
 
   void loadReExportedDylibs(FindDylib find) {
     for (ReExportedDylib &entry : _reExportedDylibs) {
-      entry.file = find(entry.path);
+      if (!entry.file)
+        entry.file = find(entry.path);
     }
   }
 
@@ -339,7 +342,7 @@ class MachODylibFile : public SharedLibraryFile {
     return std::error_code();
   }
 
-private:
+protected:
   OwningAtomPtr<SharedLibraryAtom> exports(StringRef name,
                                    StringRef installName) const {
     // First, check if requested symbol is directly implemented by this dylib.
@@ -373,6 +376,7 @@ class MachODylibFile : public SharedLibraryFile {
 
   struct ReExportedDylib {
     ReExportedDylib(StringRef p) : path(p), file(nullptr) { }
+    ReExportedDylib(StringRef p, MachODylibFile *file) : path(p), file(file) { }
     StringRef       path;
     MachODylibFile *file;
   };
@@ -393,6 +397,70 @@ class MachODylibFile : public SharedLibraryFile {
   mutable std::unordered_map<StringRef, AtomAndFlags> _nameToAtom;
 };
 
+class TAPIFile : public MachODylibFile {
+public:
+
+  TAPIFile(std::unique_ptr<MemoryBuffer> mb, MachOLinkingContext *ctx)
+      : MachODylibFile(std::move(mb), ctx) {}
+
+  std::error_code doParse() override {
+
+    llvm::Expected<std::unique_ptr<llvm::MachO::InterfaceFile>> result =
+        llvm::MachO::TextAPIReader::get(*_mb);
+    if (!result)
+      return std::make_error_code(std::errc::invalid_argument);
+
+    std::unique_ptr<llvm::MachO::InterfaceFile> interface{std::move(*result)};
+    return loadFromInterface(*interface);
+  }
+
+private:
+  std::error_code loadFromInterface(llvm::MachO::InterfaceFile &interface) {
+    llvm::MachO::Architecture arch;
+    switch(_ctx->arch()) {
+    case MachOLinkingContext::arch_x86:
+      arch = llvm::MachO::AK_i386;
+      break;
+    case MachOLinkingContext::arch_x86_64:
+      arch = llvm::MachO::AK_x86_64;
+      break;
+    case MachOLinkingContext::arch_arm64:
+      arch = llvm::MachO::AK_arm64;
+      break;
+    default:
+      return std::make_error_code(std::errc::invalid_argument);
+    }
+
+    setInstallName(interface.getInstallName().copy(allocator()));
+    // TODO(compnerd) filter out symbols based on the target platform
+    for (const auto symbol : interface.symbols())
+      if (symbol->getArchitectures().has(arch))
+        addExportedSymbol(symbol->getName(), symbol->isWeakDefined(), true);
+
+    for (const llvm::MachO::InterfaceFileRef &reexport :
+         interface.reexportedLibraries())
+      addReExportedDylib(reexport.getInstallName().copy(allocator()));
+
+    for (const auto& document : interface.documents()) {
+      for (auto& reexport : _reExportedDylibs) {
+        if (reexport.path != document->getInstallName())
+          continue;
+        assert(!reexport.file);
+        _ownedFiles.push_back(std::make_unique<TAPIFile>(
+            MemoryBuffer::getMemBuffer("", _mb->getBufferIdentifier()), _ctx));
+        reexport.file = _ownedFiles.back().get();
+        std::error_code err = _ownedFiles.back()->loadFromInterface(*document);
+        if (err)
+          return err;
+      }
+    }
+
+    return std::error_code();
+  }
+
+  std::vector<std::unique_ptr<TAPIFile>> _ownedFiles;
+};
+
 } // end namespace mach_o
 } // end namespace lld
 
diff --git a/lld/lib/ReaderWriter/MachO/MachOLinkingContext.cpp b/lld/lib/ReaderWriter/MachO/MachOLinkingContext.cpp
index 6fe9cde544d68..acd919e4d411b 100644
--- a/lld/lib/ReaderWriter/MachO/MachOLinkingContext.cpp
+++ b/lld/lib/ReaderWriter/MachO/MachOLinkingContext.cpp
@@ -540,6 +540,12 @@ MachOLinkingContext::searchDirForLibrary(StringRef path,
     return llvm::None;
   }
 
+  // Search for stub library
+  fullPath.assign(path);
+  llvm::sys::path::append(fullPath, Twine("lib") + libName + ".tbd");
+  if (fileExists(fullPath))
+    return fullPath.str().copy(_allocator);
+
   // Search for dynamic library
   fullPath.assign(path);
   llvm::sys::path::append(fullPath, Twine("lib") + libName + ".dylib");
diff --git a/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryReader.cpp b/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryReader.cpp
index 963f1227fa448..87601ca1be8b5 100644
--- a/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryReader.cpp
+++ b/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryReader.cpp
@@ -576,6 +576,26 @@ class MachODylibReader : public Reader {
   MachOLinkingContext &_ctx;
 };
 
+class MachOTAPIReader : public Reader {
+public:
+  MachOTAPIReader(MachOLinkingContext &ctx) : _ctx(ctx) {}
+
+  bool canParse(file_magic magic, MemoryBufferRef mb) const override {
+    return magic == file_magic::tapi_file;
+  }
+
+  ErrorOr<std::unique_ptr<File>>
+  loadFile(std::unique_ptr<MemoryBuffer> mb,
+           const Registry &registry) const override {
+    std::unique_ptr<File> ret =
+        std::make_unique<TAPIFile>(std::move(mb), &_ctx);
+    return std::move(ret);
+  }
+
+private:
+  MachOLinkingContext &_ctx;
+};
+
 } // namespace normalized
 } // namespace mach_o
 
@@ -583,6 +603,7 @@ void Registry::addSupportMachOObjects(MachOLinkingContext &ctx) {
   MachOLinkingContext::Arch arch = ctx.arch();
   add(std::unique_ptr<Reader>(new mach_o::normalized::MachOObjectReader(ctx)));
   add(std::unique_ptr<Reader>(new mach_o::normalized::MachODylibReader(ctx)));
+  add(std::unique_ptr<Reader>(new mach_o::normalized::MachOTAPIReader(ctx)));
   addKindTable(Reference::KindNamespace::mach_o, ctx.archHandler().kindArch(),
                ctx.archHandler().kindStrings());
   add(std::unique_ptr<YamlIOTaggedDocumentHandler>(
diff --git a/lld/lib/ReaderWriter/MachO/MachONormalizedFileFromAtoms.cpp b/lld/lib/ReaderWriter/MachO/MachONormalizedFileFromAtoms.cpp
index 55dcad955e859..42ac711bc9dc3 100644
--- a/lld/lib/ReaderWriter/MachO/MachONormalizedFileFromAtoms.cpp
+++ b/lld/lib/ReaderWriter/MachO/MachONormalizedFileFromAtoms.cpp
@@ -1037,7 +1037,7 @@ llvm::Error Util::addSymbols(const lld::File &atomFile,
 
   // Add all stabs.
   for (auto &stab : _stabs) {
-    Symbol sym;
+    lld::mach_o::normalized::Symbol sym;
     sym.type = static_cast<NListType>(stab.type);
     sym.scope = 0;
     sym.sect = stab.other;
@@ -1066,7 +1066,7 @@ llvm::Error Util::addSymbols(const lld::File &atomFile,
           AtomAndIndex ai = { atom, sect->finalSectionIndex, symbolScope };
           globals.push_back(ai);
         } else {
-          Symbol sym;
+          lld::mach_o::normalized::Symbol sym;
           sym.name  = atom->name();
           sym.type  = N_SECT;
           sym.scope = symbolScope;
@@ -1082,7 +1082,7 @@ llvm::Error Util::addSymbols(const lld::File &atomFile,
         char tmpName[16];
         sprintf(tmpName, "L%04u", tempNum++);
         StringRef tempRef(tmpName);
-        Symbol sym;
+        lld::mach_o::normalized::Symbol sym;
         sym.name  = tempRef.copy(file.ownedAllocations);
         sym.type  = N_SECT;
         sym.scope = 0;
@@ -1099,7 +1099,7 @@ llvm::Error Util::addSymbols(const lld::File &atomFile,
   std::sort(globals.begin(), globals.end(), AtomSorter());
   const uint32_t globalStartIndex = file.localSymbols.size();
   for (AtomAndIndex &ai : globals) {
-    Symbol sym;
+    lld::mach_o::normalized::Symbol sym;
     sym.name  = ai.atom->name();
     sym.type  = N_SECT;
     sym.scope = ai.scope;
@@ -1124,7 +1124,7 @@ llvm::Error Util::addSymbols(const lld::File &atomFile,
   std::sort(undefs.begin(), undefs.end(), AtomSorter());
   const uint32_t start = file.globalSymbols.size() + file.localSymbols.size();
   for (AtomAndIndex &ai : undefs) {
-    Symbol sym;
+    lld::mach_o::normalized::Symbol sym;
     uint16_t desc = 0;
     if (!rMode) {
       uint8_t ordinal = 0;
diff --git a/lld/lib/ReaderWriter/MachO/MachONormalizedFileToAtoms.cpp b/lld/lib/ReaderWriter/MachO/MachONormalizedFileToAtoms.cpp
index 80a1bf00a70d7..0a59e24c47a80 100644
--- a/lld/lib/ReaderWriter/MachO/MachONormalizedFileToAtoms.cpp
+++ b/lld/lib/ReaderWriter/MachO/MachONormalizedFileToAtoms.cpp
@@ -223,10 +223,11 @@ Atom::Scope atomScope(uint8_t scope) {
   llvm_unreachable("unknown scope value!");
 }
 
-void appendSymbolsInSection(const std::vector<Symbol> &inSymbols,
-                            uint32_t sectionIndex,
-                            SmallVector<const Symbol *, 64> &outSyms) {
-  for (const Symbol &sym : inSymbols) {
+void appendSymbolsInSection(
+    const std::vector<lld::mach_o::normalized::Symbol> &inSymbols,
+    uint32_t sectionIndex,
+    SmallVector<const lld::mach_o::normalized::Symbol *, 64> &outSyms) {
+  for (const lld::mach_o::normalized::Symbol &sym : inSymbols) {
     // Only look at definition symbols.
     if ((sym.type & N_TYPE) != N_SECT)
       continue;
@@ -286,13 +287,14 @@ llvm::Error processSymboledSection(DefinedAtom::ContentType atomType,
   }
 
   // Find all symbols in this section.
-  SmallVector<const Symbol *, 64> symbols;
+  SmallVector<const lld::mach_o::normalized::Symbol *, 64> symbols;
   appendSymbolsInSection(normalizedFile.globalSymbols, sectIndex, symbols);
   appendSymbolsInSection(normalizedFile.localSymbols,  sectIndex, symbols);
 
   // Sort symbols.
   std::sort(symbols.begin(), symbols.end(),
-            [](const Symbol *lhs, const Symbol *rhs) -> bool {
+            [](const lld::mach_o::normalized::Symbol *lhs,
+               const lld::mach_o::normalized::Symbol *rhs) -> bool {
               if (lhs == rhs)
                 return false;
               // First by address.
@@ -300,7 +302,7 @@ llvm::Error processSymboledSection(DefinedAtom::ContentType atomType,
               uint64_t rhsAddr = rhs->value;
               if (lhsAddr != rhsAddr)
                 return lhsAddr < rhsAddr;
-               // If same address, one is an alias so sort by scope.
+              // If same address, one is an alias so sort by scope.
               Atom::Scope lScope = atomScope(lhs->scope);
               Atom::Scope rScope = atomScope(rhs->scope);
               if (lScope != rScope)
@@ -339,8 +341,8 @@ llvm::Error processSymboledSection(DefinedAtom::ContentType atomType,
                    scatterable, copyRefs);
   }
 
-  const Symbol *lastSym = nullptr;
-  for (const Symbol *sym : symbols) {
+  const lld::mach_o::normalized::Symbol *lastSym = nullptr;
+  for (const lld::mach_o::normalized::Symbol *sym : symbols) {
     if (lastSym != nullptr) {
       // Ignore any assembler added "ltmpNNN" symbol at start of section
       // if there is another symbol at the start.
@@ -550,7 +552,7 @@ llvm::Error convertRelocs(const Section &section,
   auto atomBySymbol = [&] (uint32_t symbolIndex, const lld::Atom **result)
                            -> llvm::Error {
     // Find symbol from index.
-    const Symbol *sym = nullptr;
+    const lld::mach_o::normalized::Symbol *sym = nullptr;
     uint32_t numStabs  = normalizedFile.stabsSymbols.size();
     uint32_t numLocal  = normalizedFile.localSymbols.size();
     uint32_t numGlobal = normalizedFile.globalSymbols.size();
diff --git a/lld/test/COFF/lto.ll b/lld/test/COFF/lto.ll
index d8ccf566939eb..b23609cc86a0f 100644
--- a/lld/test/COFF/lto.ll
+++ b/lld/test/COFF/lto.ll
@@ -86,21 +86,8 @@
 ; TEXT-10-EMPTY:
 ; TEXT-10-NEXT: <.text>:
 ; TEXT-10-NEXT: retq
-; TEXT-10-NEXT: nop
-; TEXT-10-NEXT: nop
-; TEXT-10-NEXT: nop
-; TEXT-10-NEXT: nop
-; TEXT-10-NEXT: nop
-; TEXT-10-NEXT: nop
-; TEXT-10-NEXT: nop
-; TEXT-10-NEXT: nop
-; TEXT-10-NEXT: nop
-; TEXT-10-NEXT: nop
-; TEXT-10-NEXT: nop
-; TEXT-10-NEXT: nop
-; TEXT-10-NEXT: nop
-; TEXT-10-NEXT: nop
-; TEXT-10-NEXT: nop
+; TEXT-10-NEXT: nopw %cs:(%rax,%rax)
+; TEXT-10-NEXT: nopl (%rax,%rax)
 ; TEXT-10-NEXT: retq
 ; TEXT-10-NEXT: int3
 ; TEXT-10-NEXT: int3
diff --git a/lld/test/ELF/Inputs/gdb-index.s b/lld/test/ELF/Inputs/gdb-index.s
index 794995c150f94..88474e1fdb126 100644
--- a/lld/test/ELF/Inputs/gdb-index.s
+++ b/lld/test/ELF/Inputs/gdb-index.s
@@ -53,7 +53,7 @@ aaaaaaaaaaaaaaaa:
 .byte 0
 
 .section .debug_gnu_pubnames,"",@progbits
-.long 0x18
+.long 0x24
 .value 0x2
 .long 0
 .long 0x33
diff --git a/lld/test/ELF/Inputs/ppc64-callee-global-hidden.s b/lld/test/ELF/Inputs/ppc64-callee-global-hidden.s
new file mode 100644
index 0000000000000..e33d191f226de
--- /dev/null
+++ b/lld/test/ELF/Inputs/ppc64-callee-global-hidden.s
@@ -0,0 +1,15 @@
+func_extern:
+  blr
+
+.hidden callee3_stother0_hidden
+.globl callee3_stother0_hidden
+callee3_stother0_hidden:
+  blr
+
+.hidden callee4_stother1_hidden
+.globl callee4_stother1_hidden
+callee4_stother1_hidden:
+  .localentry callee4_stother1_hidden, 1
+  ## nop is not needed after bl for R_PPC64_REL24_NOTOC
+  bl func_extern@notoc
+  blr
diff --git a/lld/test/ELF/aarch64-range-thunk-extension-plt32.s b/lld/test/ELF/aarch64-range-thunk-extension-plt32.s
new file mode 100644
index 0000000000000..20b497fcb67ee
--- /dev/null
+++ b/lld/test/ELF/aarch64-range-thunk-extension-plt32.s
@@ -0,0 +1,36 @@
+// REQUIRES: aarch64
+// RUN: llvm-mc -filetype=obj -triple=aarch64 %s -o %t.o
+// RUN: echo 'SECTIONS { \
+// RUN:         .text.1 0x10000 : { *(.text.1) } \
+// RUN:         .text.2 0x200000000 : AT(0x20000) { *(.text.2) } \
+// RUN:       } ' > %t.script
+// RUN: ld.lld --script %t.script %t.o -o %t
+// RUN: llvm-objdump -d --no-show-raw-insn --print-imm-hex %t | FileCheck %s
+
+// The word should be an offset to the range extension thunk.
+// CHECK-LABEL: <_start>:
+// CHECK-NEXT:    10000:       04 00 00 00     .word   0x00000004
+
+// The thunk redirects to the address of callee.
+// CHECK-LABEL: <__AArch64AbsLongThunk_callee>:
+// CHECK-NEXT:    10004:       ldr     x16, 0x1000c <$d>
+// CHECK-NEXT:    10008:       br      x16
+
+// CHECK-LABEL: <$d>:
+// CHECK-NEXT:    1000c:       00 00 00 00     .word   0x00000000
+// CHECK-NEXT:    10010:       02 00 00 00     .word   0x00000002
+
+// CHECK-LABEL: <callee>:
+// CHECK-NEXT:    200000000:      ret
+
+  .section .text.1, "ax", %progbits
+  .global _start
+  .type _start, %function
+_start:
+  .word callee@PLT - .
+
+  .section .text.2, "ax", %progbits
+  .global callee
+  .type callee, %function
+callee:
+  ret
diff --git a/lld/test/ELF/aarch64-reloc-plt32.s b/lld/test/ELF/aarch64-reloc-plt32.s
new file mode 100644
index 0000000000000..a7467562b2dbb
--- /dev/null
+++ b/lld/test/ELF/aarch64-reloc-plt32.s
@@ -0,0 +1,32 @@
+// REQUIRES: aarch64
+// RUN: llvm-mc -filetype=obj -triple=aarch64 %s -o %t.o
+// RUN: llvm-mc -filetype=obj -triple=aarch64 %S/Inputs/abs255.s -o %t255.o
+// RUN: llvm-mc -filetype=obj -triple=aarch64 %S/Inputs/abs256.s -o %t256.o
+// RUN: llvm-mc -filetype=obj -triple=aarch64 %S/Inputs/abs257.s -o %t257.o
+
+/// Check for overflow with a R_AACH64_PLT32 relocation.
+
+// RUN: ld.lld -z max-page-size=4096 %t.o %t256.o -o %t2
+// RUN: llvm-objdump -s --section=.data %t2 | FileCheck %s
+
+// CHECK: Contents of section .data:
+/// 202158: S = 0x100, A = 0x80202057, P = 0x202158
+///         S + A - P = 0xffffff7f
+/// 20215c: S = 0x100, A = -0x7fdfdfa4, P = 0x20215c
+///         S + A - P = 0x80000000
+/// 202160: S = 0x100, A = 0, P = 0x202160
+///         S + A - P = 0xffdfdfa0
+// CHECK-NEXT: 202158 ffffff7f 00000080 a0dfdfff
+
+// RUN: not ld.lld -z max-page-size=4096 %t.o %t255.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=OVERFLOW1
+// OVERFLOW1: relocation R_AARCH64_PLT32 out of range: -2147483649 is not in [-2147483648, 2147483647]; references foo
+
+// RUN: not ld.lld -z max-page-size=4096 %t.o %t257.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=OVERFLOW2
+// OVERFLOW2: relocation R_AARCH64_PLT32 out of range: 2147483648 is not in [-2147483648, 2147483647]; references foo
+
+  .globl _start
+  _start:
+.data
+  .word foo@PLT - . + 2149589079
+  .word foo@PLT - . - 2145378212
+  .word foo@PLT - .
diff --git a/lld/test/ELF/aarch64-thunk-script.s b/lld/test/ELF/aarch64-thunk-script.s
index b1a7aab18e1ef..0c376b1553a25 100644
--- a/lld/test/ELF/aarch64-thunk-script.s
+++ b/lld/test/ELF/aarch64-thunk-script.s
@@ -6,7 +6,7 @@
 // RUN:       } " > %t.script
 // RUN: ld.lld --script %t.script %t.o -o %t
 // RUN: llvm-objdump -d --no-show-raw-insn --print-imm-hex %t | FileCheck %s
-// RUN: llvm-nm --no-sort %t | FileCheck --check-prefix=NM %s
+// RUN: llvm-nm --no-sort --special-syms %t | FileCheck --check-prefix=NM %s
 
 // Check that we have the out of branch range calculation right. The immediate
 // field is signed so we have a slightly higher negative displacement.
diff --git a/lld/test/ELF/aarch64-undefined-weak.s b/lld/test/ELF/aarch64-undefined-weak.s
index 839dd8d14d61a..9a4b7c23aa7a1 100644
--- a/lld/test/ELF/aarch64-undefined-weak.s
+++ b/lld/test/ELF/aarch64-undefined-weak.s
@@ -33,6 +33,8 @@ _start:
  .xword target - .
 // R_AARCH64_PREL16
  .hword target - .
+// R_AARCH64_PLT32
+ .word target@PLT - .
 
 // CHECK: Disassembly of section .text:
 // CHECK-EMPTY:
@@ -47,4 +49,5 @@ _start:
 // CHECK:      1001013c: 00 00 00 00     .word   0x00000000
 // CHECK-NEXT: 10010140: 00 00 00 00     .word   0x00000000
 // CHECK-NEXT: 10010144: 00 00 00 00     .word   0x00000000
-// CHECK-NEXT: 10010148: 00 00           .short  0x0000
+// CHECK-NEXT: 10010148: 00 00 00 00     .word   0x00000000
+// CHECK-NEXT: 1001014c: 00 00           .short  0x0000
diff --git a/lld/test/ELF/avr-reloc.s b/lld/test/ELF/avr-reloc.s
new file mode 100644
index 0000000000000..49f78044068b9
--- /dev/null
+++ b/lld/test/ELF/avr-reloc.s
@@ -0,0 +1,84 @@
+; REQUIRES: avr
+; RUN: llvm-mc -filetype=obj -triple=avr -mcpu=atmega328p %s -o %t.o
+; RUN: ld.lld %t.o --defsym=a=0x12345678 --defsym=b=30 -o %t
+; RUN: llvm-objdump -d --print-imm-hex %t | FileCheck %s
+; RUN: llvm-objdump -s %t | FileCheck --check-prefix=HEX %s
+
+.section .LDI,"ax",@progbits
+; CHECK-LABEL: section .LDI:
+; CHECK:       ldi     r20, 0x78
+; CHECK-NEXT:  ldi     r20, 0x56
+; CHECK-NEXT:  ldi     r20, 0x34
+; CHECK-NEXT:  ldi     r20, 0x12
+; CHECK-NEXT:  ldi     r20, 0x3c
+; CHECK-NEXT:  ldi     r20, 0x2b
+; CHECK-NEXT:  ldi     r20, 0x1a
+; CHECK-NEXT:  ldi     r20, 0xff
+ldi r20, lo8(a)     ; R_AVR_LO8_LDI
+ldi r20, hi8(a)     ; R_AVR_HI8_LDI
+ldi r20, hh8(a)     ; R_AVR_HH8_LDI
+ldi r20, hhi8(a)    ; R_AVR_MS8_LDI
+
+ldi r20, pm_lo8(a)  ; R_AVR_LO8_LDI_PM
+ldi r20, pm_hi8(a)  ; R_AVR_HI8_LDI_PM
+ldi r20, pm_hh8(a)  ; R_AVR_HH8_LDI_PM
+
+ldi r20, b+225
+
+.section .LDI_NEG,"ax",@progbits
+; CHECK-LABEL: section .LDI_NEG:
+; CHECK:       ldi     r20, 0x88
+; CHECK-NEXT:  ldi     r20, 0xa9
+; CHECK-NEXT:  ldi     r20, 0xcb
+; CHECK-NEXT:  ldi     r20, 0xed
+; CHECK-NEXT:  ldi     r20, 0xc4
+; CHECK-NEXT:  ldi     r20, 0xd4
+; CHECK-NEXT:  ldi     r20, 0xe5
+ldi r20, lo8(-(a))     ; R_AVR_LO8_LDI_NEG
+ldi r20, hi8(-(a))     ; R_AVR_HI8_LDI_NEG
+ldi r20, hh8(-(a))     ; R_AVR_HH8_LDI_NEG
+ldi r20, hhi8(-(a))    ; R_AVR_MS8_LDI_NEG
+
+ldi r20, pm_lo8(-(a))  ; R_AVR_LO8_LDI_PM_NEG
+ldi r20, pm_hi8(-(a))  ; R_AVR_HI8_LDI_PM_NEG
+ldi r20, pm_hh8(-(a))  ; R_AVR_HH8_LDI_PM_NEG
+
+;; The disassembler is not yet able to decode those opcodes
+;; 9e 8e    std    Y+30, r9
+;; 9e 8c    ldd    r9, Y+30
+;; 4e 96    adiw   r24, 0x1e
+.section .SIX,"ax",@progbits
+; HEX-LABEL: section .SIX:
+; HEX-NEXT:  9e8e9e8c 4e96
+std Y+b, r9   ; R_AVR_6
+ldd r9, Y+b   ; R_AVR_6
+adiw r24, b   ; R_AVR_6_ADIW
+
+.section .PORT,"ax",@progbits
+; CHECK-LABEL: section .PORT:
+; CHECK:       in     r20, 0x1e
+; CHECK-NEXT:  sbic   0x1e, 0x1
+in    r20, b  ; R_AVR_PORT6
+sbic  b, 1    ; R_AVR_PORT5
+
+;; The disassembler is not yet able to decode those opcodes
+;; 0f c0    rjmp   .+30
+;; ee cf    rjmp   .-36
+;; 69 f0    breq   .+26
+;; 61 f3    breq   .-40
+.section .PCREL,"ax",@progbits
+; HEX-LABEL: section .PCREL:
+; HEX-NEXT:  0fc0eecf 69f061f3
+foo:
+rjmp foo + 32  ; R_AVR_13_PCREL
+rjmp foo - 32  ; R_AVR_13_PCREL
+breq foo + 32  ; R_AVR_7_PCREL
+breq foo - 32  ; R_AVR_7_PCREL
+
+.section .DATA,"ax",@progbits
+; HEX-LABEL: section .DATA:
+; HEX-NEXT:  {{.*}} 1e1e000f 00785634 12
+.byte b        ; R_AVR_8
+.short b       ; R_AVR_16
+.short gs(b)   ; R_AVR_16_PM
+.long a        ; R_AVR_32
diff --git a/lld/test/ELF/dead-reloc-in-nonalloc.s b/lld/test/ELF/dead-reloc-in-nonalloc.s
new file mode 100644
index 0000000000000..00d3d2cbc4a86
--- /dev/null
+++ b/lld/test/ELF/dead-reloc-in-nonalloc.s
@@ -0,0 +1,69 @@
+# REQUIRES: x86
+## Test that -z dead-reloc-in-nonalloc= can customize the tombstone value we
+## use for an absolute relocation referencing a discarded symbol.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+# RUN: ld.lld --icf=all -z dead-reloc-in-nonalloc=.debug_info=0xaaaaaaaa \
+# RUN:   -z dead-reloc-in-nonalloc=.not_debug=0xbbbbbbbb %t.o -o %t
+# RUN: llvm-objdump -s %t | FileCheck %s --check-prefixes=COMMON,AA
+## 0xaaaaaaaa == 2863311530
+# RUN: ld.lld --icf=all -z dead-reloc-in-nonalloc=.debug_info=2863311530 \
+# RUN:   -z dead-reloc-in-nonalloc=.not_debug=0xbbbbbbbb %t.o -o - | cmp %t -
+
+# COMMON:      Contents of section .debug_addr:
+# COMMON-NEXT:  0000 [[ADDR:[0-9a-f]+]] 00000000 ffffffff ffffffff
+
+# AA:          Contents of section .debug_info:
+# AA-NEXT:      0000 [[ADDR]] 00000000 aaaaaaaa 00000000
+# AA:          Contents of section .not_debug:
+# AA-NEXT:      0000 bbbbbbbb
+
+## Specifying zero can get a behavior similar to GNU ld.
+# RUN: ld.lld --icf=all -z dead-reloc-in-nonalloc=.debug_info=0 %t.o -o %tzero
+# RUN: llvm-objdump -s %tzero | FileCheck %s --check-prefixes=COMMON,ZERO
+
+# ZERO:        Contents of section .debug_info:
+# ZERO-NEXT:    0000 {{[0-9a-f]+}}000 00000000 00000000 00000000
+
+## Glob works.
+# RUN: ld.lld --icf=all -z dead-reloc-in-nonalloc='.debug_i*=0xaaaaaaaa' \
+# RUN:   -z dead-reloc-in-nonalloc='[.]not_debug=0xbbbbbbbb' %t.o -o - | cmp %t -
+
+## If a section matches multiple option. The last option wins.
+# RUN: ld.lld --icf=all -z dead-reloc-in-nonalloc='.debug_info=1' \
+# RUN:   -z dead-reloc-in-nonalloc='.debug_i*=0' %t.o -o - | cmp %tzero -
+
+## Test all possible invalid cases.
+# RUN: not ld.lld -z dead-reloc-in-nonalloc= 2>&1 | FileCheck %s --check-prefix=USAGE
+# RUN: not ld.lld -z dead-reloc-in-nonalloc=a= 2>&1 | FileCheck %s --check-prefix=USAGE
+# RUN: not ld.lld -z dead-reloc-in-nonalloc==0 2>&1 | FileCheck %s --check-prefix=USAGE
+
+# USAGE: error: -z dead-reloc-in-nonalloc=: expected <section_glob>=<value>
+
+# RUN: not ld.lld -z dead-reloc-in-nonalloc=a=-1 2>&1 | FileCheck %s --check-prefix=NON-INTEGER
+
+# NON-INTEGER: error: -z dead-reloc-in-nonalloc=: expected a non-negative integer, but got '-1'
+
+# RUN: not ld.lld -z dead-reloc-in-nonalloc='['=0 2>&1 | FileCheck %s --check-prefix=INVALID
+
+# INVALID: error: -z dead-reloc-in-nonalloc=: invalid glob pattern: [
+
+.globl _start
+_start:
+  ret
+
+## .text.1 will be folded by ICF.
+.section .text.1,"ax"
+  ret
+
+.section .debug_addr
+  .quad .text+8
+  .quad .text.1+8
+
+.section .debug_info
+  .quad .text+8
+  .quad .text.1+8
+
+## Test a non-.debug_ section.
+.section .not_debug
+  .long .text.1+8
diff --git a/lld/test/ELF/debug-dead-reloc-32.s b/lld/test/ELF/debug-dead-reloc-32.s
new file mode 100644
index 0000000000000..a7496798c580e
--- /dev/null
+++ b/lld/test/ELF/debug-dead-reloc-32.s
@@ -0,0 +1,29 @@
+# REQUIRES: x86
+## Test we resolve symbolic relocations in .debug_* sections to a tombstone
+## value if the referenced symbol is discarded (--gc-sections, non-prevailing
+## section group, SHF_EXCLUDE, /DISCARD/, etc).
+
+# RUN: llvm-mc -filetype=obj -triple=i386 %s -o %t.o
+# RUN: ld.lld %t.o -o %t
+# RUN: llvm-objdump -s %t | FileCheck %s
+
+# CHECK:      Contents of section .debug_loc:
+# CHECK-NEXT:  0000 feffffff
+# CHECK-NEXT: Contents of section .debug_ranges:
+# CHECK-NEXT:  0000 feffffff
+# CHECK-NEXT: Contents of section .debug_addr:
+# CHECK-NEXT:  0000 ffffffff
+
+.section .text.1,"axe"
+  .byte 0
+
+## Resolved to UINT32_C(-2), with the addend ignored.
+## UINT32_C(-1) is a reserved value (base address selection entry) which can't be used.
+.section .debug_loc
+  .long .text.1+8
+.section .debug_ranges
+  .long .text.1+16
+
+## Resolved to UINT32_C(-1), with the addend ignored.
+.section .debug_addr
+  .long .text.1+8
diff --git a/lld/test/ELF/debug-dead-reloc-icf.s b/lld/test/ELF/debug-dead-reloc-icf.s
new file mode 100644
index 0000000000000..716e245c12c90
--- /dev/null
+++ b/lld/test/ELF/debug-dead-reloc-icf.s
@@ -0,0 +1,35 @@
+# REQUIRES: x86
+## Test we resolve symbolic relocations in .debug_* sections to a tombstone
+## value if the referenced section symbol is folded into another section by ICF.
+## Otherwise, we would leave entries in multiple CUs claiming ownership of the
+## same range of code, which can confuse consumers.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+# RUN: ld.lld --icf=all %t.o -o %t
+# RUN: llvm-objdump -s %t | FileCheck %s
+
+# CHECK:      Contents of section .debug_info:
+# CHECK-NEXT:  0000 {{[0-9a-f]+}}000 00000000 ffffffff ffffffff
+# CHECK:      Contents of section .debug_line:
+# CHECK-NEXT:  0000 [[ADDR:[0-9a-f]+]] 00000000
+# CHECK-SAME:                                   [[ADDR]] 00000000
+
+.globl _start
+_start:
+  ret
+
+## .text.1 will be folded by ICF.
+.section .text.1,"ax"
+  ret
+
+.section .debug_info
+  .quad .text+8
+  .quad .text.1+8
+
+## .debug_line contributions associated with folded-in functions will describe
+## different lines to the canonical function. Leaving a tombstone value would
+## prevent users from setting breakpoints on the folded-in functions.
+## Instead resolve the relocation to the folded .text.1 to .text
+.section .debug_line
+  .quad .text
+  .quad .text.1
diff --git a/lld/test/ELF/debug-dead-reloc-tls-arm.s b/lld/test/ELF/debug-dead-reloc-tls-arm.s
new file mode 100644
index 0000000000000..7fa5bcaae19e7
--- /dev/null
+++ b/lld/test/ELF/debug-dead-reloc-tls-arm.s
@@ -0,0 +1,22 @@
+# REQUIRES: arm
+## Test we resolve relocations referencing TLS symbols in .debug_* sections to
+## a tombstone value if the referenced TLS symbol is discarded.
+
+# RUN: llvm-mc -filetype=obj -triple=armv7a-none-linux-gnueabi %s -o %t.o
+# RUN: ld.lld --gc-sections %t.o -o %t
+# RUN: llvm-objdump -s %t | FileCheck %s
+
+# CHECK:      Contents of section .debug_info:
+# CHECK-NEXT:  0000 ffffffff
+
+.globl _start
+_start:
+  bx lr
+
+.section .tbss,"awT",%nobits
+.globl tls
+  .long 0
+
+.section .debug_info
+## R_ARM_TLS_LDO32
+  .long tls(tlsldo)
diff --git a/lld/test/ELF/debug-dead-reloc-tls.s b/lld/test/ELF/debug-dead-reloc-tls.s
new file mode 100644
index 0000000000000..1b26a920d3dd4
--- /dev/null
+++ b/lld/test/ELF/debug-dead-reloc-tls.s
@@ -0,0 +1,32 @@
+# REQUIRES: x86
+## Test we resolve relocations referencing TLS symbols in .debug_* sections to
+## a tombstone value if the referenced TLS symbol is discarded.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+# RUN: ld.lld --gc-sections %t.o -o %t
+# RUN: llvm-objdump -s %t | FileCheck %s
+
+# CHECK:      Contents of section .debug_info:
+# CHECK-NEXT:  0000 ffffffff ffffffff ffffffff ffffffff
+# CHECK-NEXT:  0010 ffffffff ffffffff
+
+.globl _start
+_start:
+  ret
+
+.section .tbss,"awT",@nobits
+.globl global
+local:
+global:
+  .quad 0
+
+.section .debug_info
+## On ppc64, .quad local@dtprel+0x8000 (st_value 0 is supposed to point to
+## 0x8000 bytes past the start of ## the dynamic TLS vector. References usually
+## have an addend of 0x8000). MIPS is similar. RISC-V uses 0x800.
+  .quad local@dtpoff+0x8000
+  .quad global@dtpoff+0x8000
+
+## Many other architectures don't use an offset. GCC x86-64 uses a 32-bit value.
+  .long global@dtpoff
+  .long -1
diff --git a/lld/test/ELF/debug-dead-reloc.s b/lld/test/ELF/debug-dead-reloc.s
new file mode 100644
index 0000000000000..d784519e9af4c
--- /dev/null
+++ b/lld/test/ELF/debug-dead-reloc.s
@@ -0,0 +1,60 @@
+# REQUIRES: x86
+## Test we resolve symbolic relocations in .debug_* sections to a tombstone
+## value if the referenced symbol is discarded (--gc-sections, non-prevailing
+## section group, SHF_EXCLUDE, /DISCARD/, etc).
+
+# RUN: echo '.globl _start; _start: call group' | llvm-mc -filetype=obj -triple=x86_64 - -o %t.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t1.o
+# RUN: ld.lld --gc-sections %t.o %t1.o %t1.o -o %t
+# RUN: llvm-objdump -s %t | FileCheck %s
+
+# CHECK:      Contents of section .debug_loc:
+# CHECK-NEXT:  0000 feffffff ffffffff feffffff ffffffff
+# CHECK-NEXT: Contents of section .debug_ranges:
+# CHECK-NEXT:  0000 feffffff ffffffff feffffff ffffffff
+# CHECK-NEXT: Contents of section .debug_addr:
+# CHECK-NEXT:  0000 {{.*}}000 00000000 {{.*}}000 00000000
+# CHECK-NEXT:  0010 ffffffff  ffffffff {{.*}}000 00000000
+# CHECK-NEXT: Contents of section .debug_foo:
+# CHECK-NEXT:  0000 ffffffff ffffffff 08000000 00000000
+# CHECK-NEXT:  0010 ffffffff ffffffff 08000000 00000000
+
+## -z dead-reloc-in-nonalloc= can override the tombstone value.
+# RUN: ld.lld --gc-sections -z dead-reloc-in-nonalloc=.debug_loc=42 %t.o %t1.o %t1.o -o %t42
+# RUN: llvm-objdump -s %t42 | FileCheck %s --check-prefix=OVERRIDE
+
+# OVERRIDE:      Contents of section .debug_loc:
+# OVERRIDE-NEXT:  0000 2a000000 00000000 2a000000 00000000
+
+.section .text.1,"ax"
+  .byte 0
+.section .text.2,"axe"
+  .byte 0
+.section .text.3,"axG",@progbits,group,comdat
+.globl group
+group:
+  .byte 0
+
+## Resolved to UINT64_C(-2), with the addend ignored.
+## UINT64_C(-1) is a reserved value (base address selection entry) which can't be used.
+.section .debug_loc
+  .quad .text.1+8
+.section .debug_ranges
+  .quad .text.2+16
+
+.section .debug_addr
+## .text.3 is a local symbol. The symbol defined in a non-prevailing group is
+## discarded. Resolved to UINT64_C(-1).
+  .quad .text.3+24
+## group is a non-local symbol. The relocation from the second %t1.o gets
+## resolved to the prevailing copy.
+  .quad group+32
+
+.section .debug_foo
+  .quad .text.1+8
+
+## We only deal with DW_FORM_addr. Don't special case short-range absolute
+## relocations. Treat them like regular absolute relocations referencing
+## discarded symbols, which are resolved to the addend.
+  .long .text.1+8
+  .long 0
diff --git a/lld/test/ELF/eh-frame-hdr-comdat.s b/lld/test/ELF/eh-frame-hdr-comdat.s
new file mode 100644
index 0000000000000..e28d3c7dd229d
--- /dev/null
+++ b/lld/test/ELF/eh-frame-hdr-comdat.s
@@ -0,0 +1,37 @@
+# REQUIRES: x86
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t
+# RUN: ld.lld %t %t -o %t2 --eh-frame-hdr
+# RUN: llvm-readelf -u --symbols %t2 | FileCheck %s
+
+# CHECK:      EHFrameHeader {
+# CHECK-NEXT:   Address:
+# CHECK-NEXT:   Offset:
+# CHECK-NEXT:   Size: 0x14
+# CHECK-NEXT:   Corresponding Section: .eh_frame_hdr
+# CHECK-NEXT:   Header {
+# CHECK-NEXT:     version: 1
+# CHECK-NEXT:     eh_frame_ptr_enc:
+# CHECK-NEXT:     fde_count_enc:
+# CHECK-NEXT:     table_enc:
+# CHECK-NEXT:     eh_frame_ptr:
+# CHECK-NEXT:     fde_count: 1
+# CHECK-NEXT:     entry 0 {
+# CHECK-NEXT:       initial_location: 0x[[# %x, SYM:]]
+# CHECK-NEXT:       address: 0x[[# %x, FDE:]]
+# CHECK-NEXT:     }
+# CHECK-NEXT:   }
+# CHECK-NEXT: }
+
+# CHECK:      [0x[[# FDE]]] FDE
+
+# CHECK:      Symbol table
+# CHECK-NEXT:    Num:    Value        {{.*}} Name
+# CHECK:              {{0+}}[[# SYM]] {{.*}} bar
+
+.section .text.bar,"axG",@progbits,bar,comdat
+.global bar
+bar:
+.cfi_startproc
+    ret
+.cfi_endproc
diff --git a/lld/test/ELF/eh-frame-merge.s b/lld/test/ELF/eh-frame-merge.s
index ddfb8ec072607..b60a5816f20ce 100644
--- a/lld/test/ELF/eh-frame-merge.s
+++ b/lld/test/ELF/eh-frame-merge.s
@@ -3,6 +3,11 @@
 // RUN: ld.lld --hash-style=sysv %t.o %t.o -o %t -shared
 // RUN: llvm-readobj -S --section-data %t | FileCheck %s
 
+/// Also show that the merging happens when going via a -r link.
+// RUN: ld.lld -r %t.o %t.o -o %t.r.o
+// RUN: ld.lld --hash-style=sysv %t.r.o -o %t2 -shared
+// RUN: llvm-readobj -S --section-data %t2 | FileCheck %s
+
         .section	foo,"ax",@progbits
 	.cfi_startproc
         nop
diff --git a/lld/test/ELF/eh-frame-type.test b/lld/test/ELF/eh-frame-type.test
index 9649921787e3a..22b802a9e528f 100644
--- a/lld/test/ELF/eh-frame-type.test
+++ b/lld/test/ELF/eh-frame-type.test
@@ -1,17 +1,26 @@
-# RUN: yaml2obj %s -o %t.o
-# RUN: ld.lld %t.o -o %t -shared
-# RUN: llvm-readobj -S %t | FileCheck %s
+## Show that LLD can handle .eh_frame sections of different types.
+
+# RUN: yaml2obj %s -o %t1.o -D TYPE=SHT_PROGBITS
+# RUN: ld.lld %t1.o -o %t1
+# RUN: llvm-readobj -S %t1 | FileCheck %s
+
+# RUN: yaml2obj %s -o %t2.o -D TYPE=SHT_X86_64_UNWIND
+# RUN: ld.lld %t2.o -o %t2
+# RUN: llvm-readobj -S %t2 | FileCheck %s
+
+# RUN: ld.lld %t1.o %t2.o -o %tboth
+# RUN: llvm-readobj -S %tboth | FileCheck %s
 
 # CHECK:      Name: .eh_frame
 # CHECK-NEXT: Type: SHT_PROGBITS
 
-!ELF
+--- !ELF
 FileHeader:
-  Class:           ELFCLASS64
-  Data:            ELFDATA2LSB
-  Type:            ET_REL
-  Machine:         EM_X86_64
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
 Sections:
-  - Name:            .eh_frame
-    Type:            SHT_PROGBITS
-    Flags:           [ SHF_ALLOC ]
+  - Name:  .eh_frame
+    Type:  [[TYPE]]
+    Flags: [ SHF_ALLOC ]
diff --git a/lld/test/ELF/gc-debuginfo-tls.s b/lld/test/ELF/gc-debuginfo-tls.s
deleted file mode 100644
index 2fe6a32965d44..0000000000000
--- a/lld/test/ELF/gc-debuginfo-tls.s
+++ /dev/null
@@ -1,24 +0,0 @@
-# REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o
-# RUN: ld.lld %t.o --gc-sections -shared -o %t1
-# RUN: ld.lld %t.o -shared -o %t2
-# RUN: llvm-readobj --symbols %t1 | FileCheck %s --check-prefix=GC
-# RUN: llvm-readobj --symbols %t2 | FileCheck %s --check-prefix=NOGC
-
-# NOGC:      Symbol {
-# NOGC:        Name: patatino
-# NOGC-NEXT:   Value: 0x0
-# NOGC-NEXT:   Size: 0
-# NOGC-NEXT:   Binding: Local
-# NOGC-NEXT:   Type: TLS
-# NOGC-NEXT:   Other: 0
-# NOGC-NEXT:   Section: .tbss
-# NOGC-NEXT: }
-
-# GC-NOT: tbss
-
-.section .tbss,"awT",@nobits
-patatino:
-  .long 0
-  .section .noalloc,""
-  .quad patatino
diff --git a/lld/test/ELF/gc-sections-tls.s b/lld/test/ELF/gc-sections-tls.s
new file mode 100644
index 0000000000000..ccc9ac3c74e56
--- /dev/null
+++ b/lld/test/ELF/gc-sections-tls.s
@@ -0,0 +1,25 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+
+## Relocation in a non .debug_* referencing a discarded TLS symbol is invalid.
+## If we happen to have no PT_TLS, we will emit an error.
+# RUN: not ld.lld %t.o --gc-sections -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR
+
+# ERR: error: {{.*}}.o has an STT_TLS symbol but doesn't have an SHF_TLS section
+
+## If we happen to have a PT_TLS, we will resolve the relocation to
+## an arbitrary value (current implementation uses a negative value).
+# RUN: echo '.section .tbss,"awT"; .globl root; root: .long 0' | \
+# RUN:   llvm-mc -filetype=obj -triple=x86_64 - -o %t1.o
+# RUN: ld.lld --gc-sections -u root %t.o %t1.o -o %t
+# RUN: llvm-readelf -x .noalloc %t | FileCheck %s
+
+# CHECK:      Hex dump of section '.noalloc':
+# CHECK-NEXT: 0x00000000 {{[0-9a-f]+}} ffffffff
+
+.section .tbss,"awT",@nobits
+tls:
+  .long 0
+
+.section .noalloc,""
+  .quad tls+0x8000
diff --git a/lld/test/ELF/gdb-index-invalid-pubnames.s b/lld/test/ELF/gdb-index-invalid-pubnames.s
new file mode 100644
index 0000000000000..fc10dac487c52
--- /dev/null
+++ b/lld/test/ELF/gdb-index-invalid-pubnames.s
@@ -0,0 +1,26 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t
+# RUN: ld.lld --gdb-index %t -o /dev/null 2>&1 | FileCheck %s
+
+# CHECK: warning: {{.*}}(.debug_gnu_pubnames): name lookup table at offset 0x0 parsing failed: unexpected end of data at offset 0x1 while reading [0x0, 0x4)
+
+    .section .debug_abbrev,"",@progbits
+    .byte 1                         # Abbreviation Code
+    .byte 17                        # DW_TAG_compile_unit
+    .byte 0                         # DW_CHILDREN_no
+    .byte 0                         # EOM(1)
+    .byte 0                         # EOM(2)
+    .byte 0                         # EOM(3)
+
+    .section .debug_info,"",@progbits
+.LCUBegin:
+    .long .LUnitEnd-.LUnitBegin     # Length of Unit
+.LUnitBegin:
+    .short 4                        # DWARF version number
+    .long .debug_abbrev             # Offset Into Abbrev. Section
+    .byte 8                         # Address Size (in bytes)
+    .byte 1                         # Abbrev [1] DW_TAG_compile_unit
+.LUnitEnd:
+
+    .section .debug_gnu_pubnames,"",@progbits
+    .byte 0
diff --git a/lld/test/ELF/gdb-index.s b/lld/test/ELF/gdb-index.s
index bb8ecf34bb6a4..546590ab359e6 100644
--- a/lld/test/ELF/gdb-index.s
+++ b/lld/test/ELF/gdb-index.s
@@ -109,7 +109,7 @@ entrypoint:
 .byte 0
 
 .section .debug_gnu_pubnames,"",@progbits
-.long 0x18
+.long 0x1e
 .value 0x2
 .long 0
 .long 0x33
diff --git a/lld/test/ELF/link-open-file.test b/lld/test/ELF/link-open-file.test
new file mode 100644
index 0000000000000..f1239308324c7
--- /dev/null
+++ b/lld/test/ELF/link-open-file.test
@@ -0,0 +1,71 @@
+## On Windows co-operative applications can be expected to open LLD's output
+## with FILE_SHARE_DELETE included in the sharing mode. This allows us to link
+## over the top of an existing file even if it is in use by another application.
+
+# REQUIRES: system-windows, x86
+# RUN: echo '.globl _start; _start:' > %t.s
+# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-unknown %t.s -o %t.o
+
+## FILE_SHARE_READ   = 1
+## FILE_SHARE_WRITE  = 2
+## FILE_SHARE_DELETE = 4
+
+# RUN:     %python %s %t.o 7
+# RUN: not %python %s %t.o 3 2>&1 | FileCheck %s
+# CHECK: error: failed to write to the output file
+
+import contextlib
+import ctypes
+from ctypes import wintypes as w
+import os
+import shutil
+import subprocess
+import platform
+import sys
+import time
+
+object_file = sys.argv[1]
+share_flags = int(sys.argv[2])
+
+@contextlib.contextmanager
+def open_with_share_flags(filename, share_flags):
+    GENERIC_READ          = 0x80000000
+    FILE_ATTRIBUTE_NORMAL = 0x80
+    OPEN_EXISTING         = 0x3
+    INVALID_HANDLE_VALUE = w.HANDLE(-1).value
+
+    CreateFileA = ctypes.windll.kernel32.CreateFileA
+    CreateFileA.restype = w.HANDLE
+    h = CreateFileA(filename.encode('mbcs'), GENERIC_READ, share_flags,
+                    None, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, None)
+
+    assert h != INVALID_HANDLE_VALUE, 'Failed to open ' + filename
+    try:
+        yield
+    finally:
+        ctypes.windll.kernel32.CloseHandle(h)
+
+## Ensure we have an empty directory for the output.
+outdir = os.path.basename(__file__) + '.dir'
+if os.path.exists(outdir):
+    shutil.rmtree(outdir)
+os.makedirs(outdir)
+
+## Link on top of an open file.
+elf = os.path.join(outdir, 'output_file.elf')
+open(elf, 'wb').close()
+with open_with_share_flags(elf, share_flags):
+    subprocess.check_call(['ld.lld.exe', object_file, '-o', elf])
+
+## Check the linker wrote the output file.
+with open(elf, 'rb') as f:
+    assert f.read(4) == b'\x7fELF', "linker did not write output file correctly"
+
+## Check no temp files are left around.
+## It might take a while for Windows to remove them, so loop.
+deleted = lambda: len(os.listdir(outdir)) == 1
+for _ in range(10):
+    if not deleted():
+        time.sleep (1)
+
+assert deleted(), "temp file(s) not deleted after grace period"
diff --git a/lld/test/ELF/lto/version-script2.ll b/lld/test/ELF/lto/version-script2.ll
index cd1da747a7b4b..b8c8b56b4317a 100644
--- a/lld/test/ELF/lto/version-script2.ll
+++ b/lld/test/ELF/lto/version-script2.ll
@@ -1,18 +1,30 @@
 ; REQUIRES: x86
 
-;; Test we parse symbol versions before LTO, otherwise we may get a symbol
-;; named "foo@@VER1", but not "foo" with the version VER1.
+;; For non-relocatable output, test we parse symbol versions after LTO,
+;; otherwise we may get a symbol named "foo@@VER1", but not "foo" with the
+;; version VER1.
 
 ; RUN: llvm-as %s -o %t.o
 ; RUN: echo "VER1 {};" > %t.script
 ; RUN: ld.lld %t.o -o %t.so -shared --version-script %t.script
-; RUN: llvm-readobj --dyn-syms %t.so | FileCheck %s
+; RUN: llvm-readelf --dyn-syms %t.so | FileCheck %s
 
-; CHECK: Name: foo@@VER1 (
+;; For non-relocatable output, @ in symbol names has no effect on undefined symbols.
+; CHECK:      UND       bar{{$}}
+; CHECK-NEXT: {{[1-9]}} foo@@VER1
+
+;; For relocatable output, @ should be retained in the symbol name.
+;; Don't parse and drop `@VER1`. Also check that --version-script is ignored.
+; RUN: ld.lld %t.o -o %t.ro -r --version-script %t.script
+; RUN: llvm-readelf -s %t.ro | FileCheck %s --check-prefix=RELOCATABLE
+
+; RELOCATABLE:      {{[1-9]}} foo@@VER1
+; RELOCATABLE-NEXT: UND       bar@VER1
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 module asm ".global foo"
-module asm "foo:"
+module asm "foo: call bar"
 module asm ".symver foo,foo@@@VER1"
+module asm ".symver bar,bar@@@VER1"
diff --git a/lld/test/ELF/ppc32-gnu-ifunc.s b/lld/test/ELF/ppc32-gnu-ifunc.s
index 4770fefcfa3e7..b85b9ebb8d37a 100644
--- a/lld/test/ELF/ppc32-gnu-ifunc.s
+++ b/lld/test/ELF/ppc32-gnu-ifunc.s
@@ -1,5 +1,4 @@
 # REQUIRES: ppc
-# XFAIL: *
 # RUN: llvm-mc -filetype=obj -triple=powerpc %s -o %t.o
 # RUN: ld.lld %t.o -o %t
 # RUN: llvm-readobj -r %t | FileCheck --check-prefix=RELOC %s
@@ -7,22 +6,22 @@
 # RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck %s
 
 # RELOC:      .rela.dyn {
-# RELOC-NEXT:   0x10020114 R_PPC_IRELATIVE - 0x100100E0
+# RELOC-NEXT:   0x10020118 R_PPC_IRELATIVE - 0x100100E0
 # RELOC-NEXT: }
 
 # SEC: .rela.dyn RELA 100000d4 0000d4 00000c
 # SYM: 100100e0 0 FUNC GLOBAL DEFAULT {{.*}} func
 
-# CHECK:      func_resolver:
-# CHECK-NEXT:   100100e0:
-# CHECK:      _start:
-# CHECK-NEXT:   bl .+20
+# CHECK-LABEL: <func_resolver>:
+# CHECK-NEXT:    100100e0:
+# CHECK-LABEL: <_start>:
+# CHECK-NEXT:    bl 0x100100f8
 ## .rela.plt = 0x100000d4 = 65536*4096+212
 ## end(.rela.plt) = 0x100000d4+0xc = 65536*4096+224
-# CHECK-NEXT:   lis 9, 4096
-# CHECK-NEXT:   lis 8, 4096
-# CHECK-NEXT:   addi 9, 9, 212
-# CHECK-NEXT:   addi 8, 8, 224
+# CHECK-NEXT:    lis 9, 4096
+# CHECK-NEXT:    lis 8, 4096
+# CHECK-NEXT:    addi 9, 9, 212
+# CHECK-NEXT:    addi 8, 8, 224
 
 .globl func
 .type func, @gnu_indirect_function
diff --git a/lld/test/ELF/ppc64-error-toc-local-call.s b/lld/test/ELF/ppc64-error-toc-local-call.s
new file mode 100644
index 0000000000000..606f6ead54637
--- /dev/null
+++ b/lld/test/ELF/ppc64-error-toc-local-call.s
@@ -0,0 +1,33 @@
+# REQUIRES: ppc
+# RUN: llvm-mc -filetype=obj -triple=powerpc64le %s -o %t.o
+# RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64 %s -o %t.o
+# RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
+
+## This test checks that the linker produces errors when it is missing the nop
+## after a local call to a callee with st_other=1.
+
+# CHECK: (.text+0xC): call to save_callee lacks nop, can't restore toc
+# CHECK: (.text+0x1C): call to save_callee lacks nop, can't restore toc
+
+callee:
+	.localentry	callee, 1
+	blr                                                # 0x0
+
+caller:
+.Lfunc_gep1:
+	addis 2, 12, .TOC.-.Lfunc_gep1@ha
+	addi 2, 2, .TOC.-.Lfunc_gep1@l
+.Lfunc_lep1:
+	.localentry	caller, .Lfunc_lep1-.Lfunc_gep1
+	bl callee                                          # 0xC
+	blr
+
+caller_tail:
+.Lfunc_gep2:
+	addis 2, 12, .TOC.-.Lfunc_gep2@ha
+	addi 2, 2, .TOC.-.Lfunc_gep2@l
+.Lfunc_lep2:
+	.localentry	caller_tail, .Lfunc_lep2-.Lfunc_gep2
+	b callee                                           # 0x1C
diff --git a/lld/test/ELF/ppc64-pcrel-call-to-pcrel.s b/lld/test/ELF/ppc64-pcrel-call-to-pcrel.s
new file mode 100644
index 0000000000000..3b0d9e31fb7b5
--- /dev/null
+++ b/lld/test/ELF/ppc64-pcrel-call-to-pcrel.s
@@ -0,0 +1,124 @@
+# REQUIRES: ppc
+# RUN: echo 'SECTIONS { \
+# RUN:   .text_default_stother0 0x10010000: { *(.text_default_stother0) } \
+# RUN:   .text_default_stother1 0x10020000: { *(.text_default_stother1) } \
+# RUN:   .text_hidden_stother0 0x10030000: { *(.text_hidden_stother0) } \
+# RUN:   .text_hidden_stother1 0x10040000: { *(.text_hidden_stother1) } \
+# RUN:   }' > %t.script
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64le -defsym HIDDEN=1 %s -o %t1.o
+# RUN: llvm-mc -filetype=obj -triple=powerpc64le %p/Inputs/ppc64-callee-global-hidden.s -o %t2.o
+# RUN: ld.lld -T %t.script -shared %t1.o %t2.o -o %t.so
+# RUN: llvm-readelf -s %t.so | FileCheck %s --check-prefix=SYMBOL
+# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=pwr10 %t.so | FileCheck --check-prefix=CHECK --check-prefix=CHECK-HIDDEN %s
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64le -defsym GLOBAL=1 %s -o %t3.o
+# RUN: ld.lld -T %t.script %t3.o -o %t
+# RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYMBOL-GLOBAL
+# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=pwr10 %t | FileCheck %s
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64 -defsym HIDDEN=1 %s -o %t1.o
+# RUN: llvm-mc -filetype=obj -triple=powerpc64 %p/Inputs/ppc64-callee-global-hidden.s -o %t2.o
+# RUN: ld.lld -T %t.script -shared %t1.o %t2.o -o %t.so
+# RUN: llvm-readelf -s %t.so | FileCheck %s --check-prefix=SYMBOL
+# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=pwr10 %t.so | FileCheck --check-prefix=CHECK --check-prefix=CHECK-HIDDEN %s
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64 -defsym GLOBAL=1 %s -o %t3.o
+# RUN: ld.lld -T %t.script %t3.o -o %t
+# RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYMBOL-GLOBAL
+# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=pwr10 %t | FileCheck %s
+
+# SYMBOL:      2: 0000000010010000 0 NOTYPE LOCAL DEFAULT 5 callee1_stother0_default
+# SYMBOL-NEXT: 3: 0000000010020004 0 NOTYPE LOCAL DEFAULT [<other: 0x20>] 6 callee2_stother1_default
+# SYMBOL-NEXT: 4: 0000000010010004 0 NOTYPE LOCAL DEFAULT [<other: 0x20>] 5 caller1
+# SYMBOL-NEXT: 5: 000000001002000c 0 NOTYPE LOCAL DEFAULT [<other: 0x20>] 6 caller2
+# SYMBOL-NEXT: 6: 0000000010030000 0 NOTYPE LOCAL DEFAULT [<other: 0x20>] 7 caller3
+# SYMBOL-NEXT: 7: 0000000010040000 0 NOTYPE LOCAL DEFAULT [<other: 0x20>] 8 caller4
+# SYMBOL-NEXT: 8: 0000000010020000 0 NOTYPE LOCAL DEFAULT 6 func_local
+# SYMBOL-NEXT: 9: 0000000010040008 0 NOTYPE LOCAL DEFAULT 9 func_extern
+# SYMBOL-NEXT: 10: 000000001004000c 0 NOTYPE LOCAL HIDDEN 9 callee3_stother0_hidden
+# SYMBOL-NEXT: 11: 0000000010040010 0 NOTYPE LOCAL HIDDEN [<other: 0x22>] 9 callee4_stother1_hidden
+
+# SYMBOL-GLOBAL:      2: 0000000010010004 0 NOTYPE LOCAL DEFAULT [<other: 0x20>] 1 caller1
+# SYMBOL-GLOBAL-NEXT: 3: 000000001002000c 0 NOTYPE LOCAL DEFAULT [<other: 0x20>] 2 caller2
+# SYMBOL-GLOBAL-NEXT: 4: 0000000010020000 0 NOTYPE LOCAL DEFAULT 2 func_local
+# SYMBOL-GLOBAL-NEXT: 5: 0000000010010000 0 NOTYPE GLOBAL DEFAULT 1 callee1_stother0_default
+# SYMBOL-GLOBAL-NEXT: 6: 0000000010020004 0 NOTYPE GLOBAL DEFAULT [<other: 0x20>] 2 callee2_stother1_default
+
+# CHECK-LABEL: <callee1_stother0_default>:
+# CHECK-NEXT:  10010000: blr
+
+# CHECK-LABEL: <caller1>:
+# CHECK:       10010004: bl 0x10010000
+# CHECK-NEXT:  10010008: b 0x10010000
+.section .text_default_stother0, "ax", %progbits
+.ifdef GLOBAL
+.globl callee1_stother0_default
+.endif
+callee1_stother0_default:
+  blr
+caller1:
+  .localentry caller1, 1
+  ## nop is not needed after bl for R_PPC64_REL24_NOTOC
+  bl callee1_stother0_default@notoc
+  b callee1_stother0_default@notoc
+
+# CHECK-LABEL: <func_local>:
+# CHECK-NEXT:  10020000: blr
+
+# CHECK-LABEL: <callee2_stother1_default>:
+# CHECK-NEXT:  10020004: bl 0x10020000
+# CHECK-NEXT:  10020008: blr
+
+# CHECK-LABEL: <caller2>:
+# CHECK:       1002000c: bl 0x10020004
+# CHECK-NEXT:  10020010: b 0x10020004
+.section .text_default_stother1, "ax", %progbits
+func_local:
+  blr
+.ifdef GLOBAL
+.globl callee2_stother1_default
+.endif
+callee2_stother1_default:
+  .localentry callee2_stother1_default, 1
+  ## nop is not needed after bl for R_PPC64_REL24_NOTOC
+  bl func_local@notoc
+  blr
+caller2:
+  .localentry caller2, 1
+  ## nop is not needed after bl for R_PPC64_REL24_NOTOC
+  bl callee2_stother1_default@notoc
+  b callee2_stother1_default@notoc
+
+# CHECK-HIDDEN-LABEL: <caller3>:
+# CHECK-HIDDEN-NEXT:  10030000: bl 0x1004000c
+# CHECK-HIDDEN-NEXT:  10030004: b 0x1004000c
+
+# CHECK-HIDDEN-LABEL: <caller4>:
+# CHECK-HIDDEN-NEXT:  10040000: bl 0x10040010
+# CHECK-HIDDEN-NEXT:  10040004: b 0x10040010
+
+# CHECK-HIDDEN-LABEL: <func_extern>:
+# CHECK-HIDDEN-NEXT:  10040008: blr
+
+# CHECK-HIDDEN-LABEL: <callee3_stother0_hidden>:
+# CHECK-HIDDEN-NEXT:  1004000c: blr
+
+# CHECK-HIDDEN-LABEL: <callee4_stother1_hidden>:
+# CHECK-HIDDEN-NEXT:  10040010: bl 0x10040008
+# CHECK-HIDDEN-NEXT:  10040014: blr
+.ifdef HIDDEN
+.section .text_hidden_stother0, "ax", %progbits
+caller3:
+  .localentry caller3, 1
+  ## nop is not needed after bl for R_PPC64_REL24_NOTOC
+  bl callee3_stother0_hidden@notoc
+  b callee3_stother0_hidden@notoc
+
+.section .text_hidden_stother1, "ax", %progbits
+caller4:
+  .localentry caller4, 1
+  ## nop is not needed after bl for R_PPC64_REL24_NOTOC
+  bl callee4_stother1_hidden@notoc
+  b callee4_stother1_hidden@notoc
+.endif
diff --git a/lld/test/ELF/ppc64-reloc-got-pcrel34.s b/lld/test/ELF/ppc64-reloc-got-pcrel34.s
new file mode 100644
index 0000000000000..4e5cf97b7ae90
--- /dev/null
+++ b/lld/test/ELF/ppc64-reloc-got-pcrel34.s
@@ -0,0 +1,51 @@
+# REQUIRES: ppc
+# RUN: echo 'SECTIONS { \
+# RUN:       .text_low 0x10010000: { *(.text_low) } \
+# RUN:       .text_high 0x10080000 : { *(.text_high) } \
+# RUN:       }' > %t.script
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64le %s -o %t.o
+# RUN: ld.lld -T %t.script --shared %t.o -o %t
+# RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYMBOL
+# RUN: llvm-readelf -r %t | FileCheck %s --check-prefix=RELA
+# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=future %t | FileCheck %s
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64 %s -o %t.o
+# RUN: ld.lld -T %t.script --shared %t.o -o %t
+# RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYMBOL
+# RUN: llvm-readelf -r %t | FileCheck %s --check-prefix=RELA
+# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=future %t | FileCheck %s
+
+.text
+.section .text_low, "ax", %progbits
+# CHECK-LABEL: <GlobIntPCRel>:
+# CHECK-NEXT:    pld 3, 458928(0), 1
+# CHECK-NEXT:    lwa 3, 0(3)
+# SYMBOL: Symbol table '.dynsym' contains 4 entries:
+# SYMBOL:      00000000     0 NOTYPE  GLOBAL  DEFAULT     UND glob_int
+# RELA:        100800b0  0000000100000014 R_PPC64_GLOB_DAT       0000000000000000 glob_int + 0
+GlobIntPCRel:
+  pld 3, glob_int@got@PCREL(0), 1
+  lwa 3, 0(3)
+  blr
+
+# CHECK-LABEL: <GlobIntPCRelOffset>:
+# CHECK-NEXT:    pld 3, 458920(0), 1
+# CHECK-NEXT:    lwa 3, 8(3)
+# SYMBOL:      00000000     0 NOTYPE  GLOBAL  DEFAULT     UND glob_int8
+# RELA:        100800b8  0000000200000014 R_PPC64_GLOB_DAT       0000000000000000 glob_int8 + 0
+GlobIntPCRelOffset:
+  pld 3, glob_int8@got@PCREL(0), 1
+  lwa 3, 8(3)
+  blr
+
+# CHECK-LABEL: <GlobIntPCRelBigOffset>:
+# CHECK-NEXT:    pld 3, 192(0), 1
+# CHECK-NEXT:    lwa 3, 64(3)
+# SYMBOL:      00000000     0 NOTYPE  GLOBAL  DEFAULT     UND glob_int8_big
+# RELA:        100800c0  0000000300000014 R_PPC64_GLOB_DAT       0000000000000000 glob_int8_big + 0
+.section .text_high, "ax", %progbits
+GlobIntPCRelBigOffset:
+  pld 3, glob_int8_big@got@PCREL(0), 1
+  lwa 3, 64(3)
+  blr
diff --git a/lld/test/ELF/ppc64-reloc-pcrel34-overflow.s b/lld/test/ELF/ppc64-reloc-pcrel34-overflow.s
new file mode 100644
index 0000000000000..a97160a430dd0
--- /dev/null
+++ b/lld/test/ELF/ppc64-reloc-pcrel34-overflow.s
@@ -0,0 +1,21 @@
+# REQUIRES: ppc
+# RUN: echo 'SECTIONS { \
+# RUN:       .text_low 0x10010000: { *(.text_low) } \
+# RUN:       .text_overflow 0x1000000000 : { *(.text_overflow) } \
+# RUN:       }' > %t.script
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64le %s -o %t.o
+# RUN: not ld.lld -T %t.script %t.o -o %t
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64 %s -o %t.o
+# RUN: not ld.lld -T %t.script %t.o -o %t
+
+.section .text_low, "ax", %progbits
+# CHECK: relocation R_PPC64_PCREL34 out of range
+GlobIntOverflow:
+	plwa 3, glob_overflow@PCREL(0), 1
+	blr
+.section .text_overflow, "ax", %progbits
+glob_overflow:
+	.long	0
+	.size	glob_overflow, 4
diff --git a/lld/test/ELF/ppc64-reloc-pcrel34.s b/lld/test/ELF/ppc64-reloc-pcrel34.s
new file mode 100644
index 0000000000000..1bc2adf8dcdd6
--- /dev/null
+++ b/lld/test/ELF/ppc64-reloc-pcrel34.s
@@ -0,0 +1,49 @@
+# REQUIRES: ppc
+# RUN: echo 'SECTIONS { \
+# RUN:       .text_low 0x10010000: { *(.text_low) } \
+# RUN:       .text_high 0x10080000 : { *(.text_high) } \
+# RUN:       }' > %t.script
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64le %s -o %t.o
+# RUN: ld.lld -T %t.script %t.o -o %t
+# RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYMBOL
+# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=future %t | FileCheck %s
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64 %s -o %t.o
+# RUN: ld.lld -T %t.script %t.o -o %t
+# RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYMBOL
+# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=future %t | FileCheck %s
+
+.section .text_low, "ax", %progbits
+# CHECK-LABEL: <GlobIntPCRel>:
+# CHECK-NEXT:    10010000:        plwa 3, 12(0), 1
+# SYMBOL:        1001000c     4 NOTYPE  LOCAL  DEFAULT     1 glob_int
+GlobIntPCRel:
+	plwa 3, glob_int@PCREL(0), 1
+	blr
+glob_int:
+	.long	0
+	.size	glob_int, 4
+
+
+# CHECK-LABEL: <GlobIntPCRelOffset>:
+# CHECK-NEXT:    10010010:        plwa 3, 16(0), 1
+# SYMBOL:        1001001c     8 NOTYPE  LOCAL  DEFAULT     1 glob_int8
+GlobIntPCRelOffset:
+	plwa 3, glob_int8@PCREL+4(0), 1
+	blr
+glob_int8:
+	.quad	0
+	.size	glob_int8, 8
+
+
+# CHECK-LABEL: <GlobIntPCRelBigOffset>:
+# CHECK-NEXT:    10010024:        plwa 3, 458720(0), 1
+# SYMBOL:        10080000     8 NOTYPE  LOCAL  DEFAULT     2 glob_int8_big
+GlobIntPCRelBigOffset:
+	plwa 3, glob_int8_big@PCREL+4(0), 1
+	blr
+.section .text_high, "ax", %progbits
+glob_int8_big:
+	.quad	0
+	.size	glob_int8_big, 8
diff --git a/lld/test/ELF/ppc64-toc-call-to-pcrel-long-jump.s b/lld/test/ELF/ppc64-toc-call-to-pcrel-long-jump.s
new file mode 100644
index 0000000000000..89f62c7de7ce3
--- /dev/null
+++ b/lld/test/ELF/ppc64-toc-call-to-pcrel-long-jump.s
@@ -0,0 +1,33 @@
+# REQUIRES: ppc
+# RUN: echo 'SECTIONS { \
+# RUN:       .text_callee 0x10010000 : { *(.text_callee) } \
+# RUN:       .text_caller 0x20020000 : { *(.text_caller) } \
+# RUN:       }' > %t.script
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64le %s -o %t.o
+# RUN: not ld.lld -T %t.script %t.o -o %t 2>&1 >/dev/null | FileCheck %s
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64 %s -o %t.o
+# RUN: not ld.lld -T %t.script %t.o -o %t 2>&1 >/dev/null | FileCheck %s
+
+# CHECK: error: R2 save stub branch offset is too large: -268501028
+
+.section .text_callee, "ax", %progbits
+callee:
+  .localentry callee, 1
+  blr
+
+.section .text_caller, "ax", %progbits
+caller:
+.Lfunc_gep1:
+  addis 2, 12, .TOC.-.Lfunc_gep1@ha
+  addi 2, 2, .TOC.-.Lfunc_gep1@l
+.Lfunc_lep1:
+  .localentry caller, .Lfunc_lep1-.Lfunc_gep1
+  addis 30, 2, global@toc@ha
+  lwz 3, global@toc@l(30)
+  bl callee
+  nop
+  blr
+global:
+  .long	0
diff --git a/lld/test/ELF/ppc64-toc-call-to-pcrel.s b/lld/test/ELF/ppc64-toc-call-to-pcrel.s
new file mode 100644
index 0000000000000..1807895a1914b
--- /dev/null
+++ b/lld/test/ELF/ppc64-toc-call-to-pcrel.s
@@ -0,0 +1,74 @@
+# REQUIRES: ppc
+# RUN: echo 'SECTIONS { \
+# RUN:       .text_callee 0x10010000 : { *(.text_callee) } \
+# RUN:       .text_caller 0x10020000 : { *(.text_caller) } \
+# RUN:       }' > %t.script
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64le %s -o %t.o
+# RUN: ld.lld -T %t.script %t.o -o %t
+# RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYMBOL
+# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=future %t | FileCheck %s
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64 %s -o %t.o
+# RUN: ld.lld -T %t.script %t.o -o %t
+# RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYMBOL
+# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=future %t | FileCheck %s
+
+# The point of this test is to make sure that when a function with TOC access
+# a local function with st_other=1, a TOC save stub is inserted.
+
+# SYMBOL: Symbol table '.symtab' contains 7 entries
+# SYMBOL: 10010000     0 NOTYPE  LOCAL  DEFAULT [<other: 0x20>]   1 callee
+# SYMBOL: 10020000     0 NOTYPE  LOCAL  DEFAULT [<other: 0x60>]   2 caller
+# SYMBOL: 10020020     0 NOTYPE  LOCAL  DEFAULT [<other: 0x60>]   2 caller_14
+# SYMBOL: 1002003c     8 FUNC    LOCAL  DEFAULT                   2 __toc_save_callee
+
+# CHECK-LABEL: callee
+# CHECK:       blr
+
+# CHECK-LABEL: caller
+# CHECK:       bl 0x1002003c
+# CHECK-NEXT:  ld 2, 24(1)
+# CHECK-NEXT:  blr
+
+# CHECK-LABEL: caller_14
+# CHECK:       bfl 0, 0x1002003c
+# CHECK-NEXT:  ld 2, 24(1)
+# CHECK-NEXT:  blr
+
+# CHECK-LABEL: __toc_save_callee
+# CHECK-NEXT:  std 2, 24(1)
+# CHECK-NEXT:  b 0x10010000
+
+
+.section .text_callee, "ax", %progbits
+callee:
+  .localentry callee, 1
+  blr
+
+.section .text_caller, "ax", %progbits
+caller:
+.Lfunc_gep1:
+  addis 2, 12, .TOC.-.Lfunc_gep1@ha
+  addi 2, 2, .TOC.-.Lfunc_gep1@l
+.Lfunc_lep1:
+  .localentry caller, .Lfunc_lep1-.Lfunc_gep1
+  addis 30, 2, global@toc@ha
+  lwz 3, global@toc@l(30)
+  bl callee
+  nop
+  blr
+global:
+  .long	0
+
+caller_14:
+.Lfunc_gep2:
+  addis 2, 12, .TOC.-.Lfunc_gep1@ha
+  addi 2, 2, .TOC.-.Lfunc_gep1@l
+.Lfunc_lep2:
+  .localentry caller_14, .Lfunc_lep2-.Lfunc_gep2
+  addis 30, 2, global@toc@ha
+  lwz 3, global@toc@l(30)
+  bcl 4, 0, callee
+  nop
+  blr
diff --git a/lld/test/ELF/silent-ignore.test b/lld/test/ELF/silent-ignore.test
index 600c9f86b18a5..6f76809d0e119 100644
--- a/lld/test/ELF/silent-ignore.test
+++ b/lld/test/ELF/silent-ignore.test
@@ -7,6 +7,7 @@ RUN:   -no-copy-dt-needed-entries \
 RUN:   -no-ctors-in-init-array \
 RUN:   -no-keep-memory \
 RUN:   -no-pipeline-knowledge \
+RUN:   --no-relax \
 RUN:   -no-warn-mismatch \
 RUN:   -p \
 RUN:   -rpath-link . \
diff --git a/lld/test/ELF/startstop-visibility.s b/lld/test/ELF/startstop-visibility.s
new file mode 100644
index 0000000000000..e6c5b1b707c08
--- /dev/null
+++ b/lld/test/ELF/startstop-visibility.s
@@ -0,0 +1,45 @@
+# REQUIRES: x86
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+
+# RUN: ld.lld %t.o -o %t
+# RUN: llvm-readelf -s %t | FileCheck %s
+
+# CHECK: 0 NOTYPE GLOBAL PROTECTED 2 __start_aaa
+# CHECK: 0 NOTYPE GLOBAL PROTECTED 2 __stop_aaa
+
+# RUN: ld.lld -z start-stop-visibility=default %t.o -o %t1
+# RUN: llvm-readelf -s %t1 | FileCheck --check-prefix=CHECK-DEFAULT %s
+
+# CHECK-DEFAULT: 0 NOTYPE GLOBAL DEFAULT 2 __start_aaa
+# CHECK-DEFAULT: 0 NOTYPE GLOBAL DEFAULT 2 __stop_aaa
+
+# RUN: ld.lld -z start-stop-visibility=internal %t.o -o %t2
+# RUN: llvm-readelf -s %t2 | FileCheck --check-prefix=CHECK-INTERNAL %s
+
+# CHECK-INTERNAL: 0 NOTYPE LOCAL INTERNAL 2 __start_aaa
+# CHECK-INTERNAL: 0 NOTYPE LOCAL INTERNAL 2 __stop_aaa
+
+# RUN: ld.lld -z start-stop-visibility=hidden %t.o -o %t3
+# RUN: llvm-readelf -s %t3 | FileCheck --check-prefix=CHECK-HIDDEN %s
+
+# CHECK-HIDDEN: 0 NOTYPE LOCAL HIDDEN 2 __start_aaa
+# CHECK-HIDDEN: 0 NOTYPE LOCAL HIDDEN 2 __stop_aaa
+
+# RUN: ld.lld -z start-stop-visibility=protected %t.o -o %t4
+# RUN: llvm-readelf -s %t4 | FileCheck --check-prefix=CHECK-PROTECTED %s
+
+# CHECK-PROTECTED: 0 NOTYPE GLOBAL PROTECTED 2 __start_aaa
+# CHECK-PROTECTED: 0 NOTYPE GLOBAL PROTECTED 2 __stop_aaa
+
+# RUN: not ld.lld -z start-stop-visibility=aaa %t.o -o /dev/null
+# CHECK-ERROR: error: unknown -z start-stop-visibility= value: aaa
+
+.quad __start_aaa
+.quad __stop_aaa
+.section aaa,"ax"
+
+.global _start
+.text
+_start:
+  nop
diff --git a/lld/test/ELF/warn-backrefs.s b/lld/test/ELF/warn-backrefs.s
index 34d105d3036fd..a1b186fc98b1c 100644
--- a/lld/test/ELF/warn-backrefs.s
+++ b/lld/test/ELF/warn-backrefs.s
@@ -51,6 +51,12 @@
 
 # OBJECT: warning: backward reference detected: foo in {{.*}}1.o refers to {{.*}}2.o
 
+## Back reference from an fetched --start-lib to a previous --start-lib.
+# RUN: ld.lld -m elf_x86_64 -u _start --warn-backrefs --start-lib %/t2.o --end-lib \
+# RUN:   --start-lib %t1.o --end-lib -o /dev/null 2>&1 | FileCheck --check-prefix=OBJECT %s
+## --warn-backrefs-exclude=%/t2.o can be used for a fetched --start-lib.
+# RUN: ld.lld --fatal-warnings -m elf_x86_64 -u _start --warn-backrefs --warn-backrefs-exclude=%/t2.o --start-lib %/t2.o --end-lib --start-lib %t1.o --end-lib -o /dev/null
+
 ## Don't warn if the definition and the backward reference are in a group.
 # RUN: echo '.globl bar; bar:' | llvm-mc -filetype=obj -triple=x86_64 - -o %t3.o
 # RUN: echo '.globl foo; foo: call bar' | llvm-mc -filetype=obj -triple=x86_64 - -o %t4.o
diff --git a/lld/test/MachO/Inputs/MacOSX.sdk/usr/lib/libSystem.tbd b/lld/test/MachO/Inputs/MacOSX.sdk/usr/lib/libSystem.tbd
index 1a90cff2978b7..fddd192630d18 100644
--- a/lld/test/MachO/Inputs/MacOSX.sdk/usr/lib/libSystem.tbd
+++ b/lld/test/MachO/Inputs/MacOSX.sdk/usr/lib/libSystem.tbd
@@ -13,7 +13,7 @@ exports:
 archs:            [ x86_64 ]
 uuids:            [ 'x86_64: 00000000-0000-0000-0000-000000000001' ]
 platform:         macosx
-install-name:     '/usr/lib/libdyld.dylib'
+install-name:     '/usr/lib/system/libdyld.dylib'
 current-version:  0001.001.1
 parent-umbrella:  System
 exports:
@@ -23,7 +23,7 @@ exports:
 archs:            [ x86_64 ]
 uuids:            [ 'x86_64: 00000000-0000-0000-0000-000000000002' ]
 platform:         macosx
-install-name:     '/usr/lib/libsystem_c.dylib'
+install-name:     '/usr/lib/system/libsystem_c.dylib'
 current-version:  0001.001.1
 parent-umbrella:  System
 exports:
@@ -33,7 +33,7 @@ exports:
 archs:            [ x86_64 ]
 uuids:            [ 'x86_64: 00000000-0000-0000-0000-000000000003' ]
 platform:         macosx
-install-name:     '/usr/lib/libsystem_m.dylib'
+install-name:     '/usr/lib/system/libsystem_m.dylib'
 current-version:  0001.001.1
 parent-umbrella:  System
 exports:
diff --git a/lld/test/MachO/bss.s b/lld/test/MachO/bss.s
index 7e7af12175f4b..b56f02d2bcb53 100644
--- a/lld/test/MachO/bss.s
+++ b/lld/test/MachO/bss.s
@@ -3,9 +3,27 @@
 # RUN: lld -flavor darwinnew -o %t %t.o
 # RUN: llvm-readobj --section-headers --macho-segment %t | FileCheck %s
 
-## Check that __bss takes up zero file size and is at file offset zero.
+## Check that __bss takes up zero file size, is at file offset zero, and
+## appears at the end of its segment.
 
-# CHECK:        Name: __bss
+# CHECK:        Index: 1
+# CHECK-NEXT:   Name: __data
+# CHECK-NEXT:   Segment: __DATA
+# CHECK-NEXT:   Address:
+# CHECK-NEXT:   Size: 0x8
+# CHECK-NEXT:   Offset: 4096
+# CHECK-NEXT:   Alignment: 0
+# CHECK-NEXT:   RelocationOffset: 0x0
+# CHECK-NEXT:   RelocationCount: 0
+# CHECK-NEXT:   Type: Regular (0x0)
+# CHECK-NEXT:   Attributes [ (0x0)
+# CHECK-NEXT:   ]
+# CHECK-NEXT:   Reserved1: 0x0
+# CHECK-NEXT:   Reserved2: 0x0
+# CHECK-NEXT:   Reserved3: 0x0
+
+# CHECK:        Index: 2
+# CHECK-NEXT:   Name: __bss
 # CHECK-NEXT:   Segment: __DATA
 # CHECK-NEXT:   Address:
 # CHECK-NEXT:   Size: 0x4
@@ -23,9 +41,9 @@
 # CHECK:      Name: __DATA
 # CHECK-NEXT: Size:
 # CHECK-NEXT: vmaddr:
-# CHECK-NEXT: vmsize: 0x4
+# CHECK-NEXT: vmsize: 0xC
 # CHECK-NEXT: fileoff:
-# CHECK-NEXT: filesize: 0
+# CHECK-NEXT: filesize: 8
 
 .globl _main
 
@@ -36,3 +54,6 @@ _main:
 
 .bss
 .zero 4
+
+.data
+.quad 0x1234
diff --git a/lld/test/MachO/dylink.s b/lld/test/MachO/dylink.s
index 1909e380fc6ad..0549aab982926 100644
--- a/lld/test/MachO/dylink.s
+++ b/lld/test/MachO/dylink.s
@@ -31,9 +31,12 @@
 # CHECK-NEXT: [[#%x, GOODBYE_RIP:]]: popq %rsi
 
 # CHECK-LABEL: Bind table:
-# CHECK-DAG: __DATA_CONST __got 0x{{0*}}[[#%x, HELLO_RIP + HELLO_OFF]]               pointer 0 libhello   _hello_world
-# CHECK-DAG: __DATA_CONST __got 0x{{0*}}[[#%x, HELLO_ITS_ME_RIP + HELLO_ITS_ME_OFF]] pointer 0 libhello   _hello_its_me
-# CHECK-DAG: __DATA_CONST __got 0x{{0*}}[[#%x, GOODBYE_RIP + GOODBYE_OFF]]           pointer 0 libgoodbye _goodbye_world
+# CHECK-DAG: __DATA_CONST __got  0x{{0*}}[[#%x, HELLO_RIP + HELLO_OFF]]               pointer 0   libhello   _hello_world
+# CHECK-DAG: __DATA_CONST __got  0x{{0*}}[[#%x, HELLO_ITS_ME_RIP + HELLO_ITS_ME_OFF]] pointer 0   libhello   _hello_its_me
+# CHECK-DAG: __DATA_CONST __got  0x{{0*}}[[#%x, GOODBYE_RIP + GOODBYE_OFF]]           pointer 0   libgoodbye _goodbye_world
+# CHECK-DAG: __DATA       __data 0x[[#%x, DATA_ADDR:]]                                pointer 0   libhello   _hello_world
+# CHECK-DAG: __DATA       __data 0x{{0*}}[[#%x, DATA_ADDR + 8]]                       pointer 8   libhello   _hello_its_me
+# CHECK-DAG: __DATA       __data 0x{{0*}}[[#%x, DATA_ADDR + 16]]                      pointer -15 libgoodbye _goodbye_world
 
 .section __TEXT,__text
 .globl _main
@@ -59,3 +62,8 @@ _main:
   syscall
   mov $0, %rax
   ret
+
+.data
+.quad _hello_world
+.quad _hello_its_me + 0x8
+.quad _goodbye_world - 0xf
diff --git a/lld/test/MachO/invalid/reserved-section-name.s b/lld/test/MachO/invalid/reserved-section-name.s
new file mode 100644
index 0000000000000..8d767bcf29298
--- /dev/null
+++ b/lld/test/MachO/invalid/reserved-section-name.s
@@ -0,0 +1,14 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
+# RUN: not lld -flavor darwinnew -o %t %t.o 2>&1 | FileCheck %s -DFILE=%t.o
+# CHECK: error: section from [[FILE]] conflicts with synthetic section __DATA_CONST,__got
+
+.globl _main
+
+.section __DATA_CONST,__got
+.space 1
+
+.text
+_main:
+  mov $0, %rax
+  ret
diff --git a/lld/test/mach-o/Inputs/MacOSX.sdk/usr/lib/libSystem.tbd b/lld/test/mach-o/Inputs/MacOSX.sdk/usr/lib/libSystem.tbd
new file mode 100644
index 0000000000000..fddd192630d18
--- /dev/null
+++ b/lld/test/mach-o/Inputs/MacOSX.sdk/usr/lib/libSystem.tbd
@@ -0,0 +1,42 @@
+--- !tapi-tbd-v3
+archs:            [ x86_64 ]
+uuids:            [ 'x86_64: 00000000-0000-0000-0000-000000000000' ]
+platform:         macosx
+install-name:     '/usr/lib/libSystem.B.dylib'
+current-version:  0001.001.1
+exports:
+  - archs:        [ 'x86_64' ]
+    re-exports:   [ '/usr/lib/system/libdyld.dylib',
+                    '/usr/lib/system/libsystem_c.dylib',
+                    '/usr/lib/system/libsystem_m.dylib' ]
+--- !tapi-tbd-v3
+archs:            [ x86_64 ]
+uuids:            [ 'x86_64: 00000000-0000-0000-0000-000000000001' ]
+platform:         macosx
+install-name:     '/usr/lib/system/libdyld.dylib'
+current-version:  0001.001.1
+parent-umbrella:  System
+exports:
+  - archs:        [ 'x86_64' ]
+    symbols:      [ dyld_stub_binder ]
+--- !tapi-tbd-v3
+archs:            [ x86_64 ]
+uuids:            [ 'x86_64: 00000000-0000-0000-0000-000000000002' ]
+platform:         macosx
+install-name:     '/usr/lib/system/libsystem_c.dylib'
+current-version:  0001.001.1
+parent-umbrella:  System
+exports:
+  - archs:        [ 'x86_64' ]
+    symbols:      [ ]
+--- !tapi-tbd-v3
+archs:            [ x86_64 ]
+uuids:            [ 'x86_64: 00000000-0000-0000-0000-000000000003' ]
+platform:         macosx
+install-name:     '/usr/lib/system/libsystem_m.dylib'
+current-version:  0001.001.1
+parent-umbrella:  System
+exports:
+  - archs:        [ 'x86_64' ]
+    symbols:      [ ___nan ]
+...
diff --git a/lld/test/mach-o/stub-link.s b/lld/test/mach-o/stub-link.s
new file mode 100644
index 0000000000000..fb12b76f44d90
--- /dev/null
+++ b/lld/test/mach-o/stub-link.s
@@ -0,0 +1,21 @@
+# REQUIRES: x86
+
+# RUN: mkdir -p %t
+#
+# RUN: llvm-mc -filetype obj -triple x86_64-apple-darwin %s -o %t/test.o
+# RUN: ld64.lld -o %t/test -Z -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem %t/test.o
+#
+# RUN: llvm-objdump --bind --no-show-raw-insn -d -r %t/test | FileCheck %s
+
+# CHECK: Disassembly of section __TEXT,__text:
+# CHECK: movq {{.*}} # [[ADDR:[0-9a-f]+]]
+
+# CHECK: Bind table:
+# CHECK: __DATA __got 0x[[ADDR]] pointer 0 libSystem ___nan
+
+.section __TEXT,__text
+.global _main
+
+_main:
+  movq ___nan@GOTPCREL(%rip), %rax
+  ret
diff --git a/lld/test/wasm/data-layout.ll b/lld/test/wasm/data-layout.ll
index 759c5440fe99a..863447cfd26bf 100644
--- a/lld/test/wasm/data-layout.ll
+++ b/lld/test/wasm/data-layout.ll
@@ -1,7 +1,12 @@
-; RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown %p/Inputs/hello.s -o %t.hello.o
-; RUN: llc -filetype=obj %s -o %t.o
+; RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown %p/Inputs/hello.s -o %t.hello32.o
+; RUN: llc -mtriple=wasm32-unknown-unknown -filetype=obj %s -o %t32.o
+; RUN: wasm-ld -m wasm32 -no-gc-sections --export=__data_end --export=__heap_base --allow-undefined --no-entry -o %t32.wasm %t32.o %t.hello32.o
+; RUN: obj2yaml %t32.wasm | FileCheck --check-prefixes CHECK,CHK32 %s
 
-target triple = "wasm32-unknown-unknown"
+; RUN: llvm-mc -filetype=obj -triple=wasm64-unknown-unknown %p/Inputs/hello.s -o %t.hello64.o
+; RUN: llc -mtriple=wasm64-unknown-unknown -filetype=obj %s -o %t64.o
+; RUN: wasm-ld -m wasm64 -no-gc-sections --export=__data_end --export=__heap_base --allow-undefined --no-entry -o %t64.wasm %t64.o %t.hello64.o
+; RUN: obj2yaml %t64.wasm | FileCheck --check-prefixes CHECK,CHK64 %s
 
 @foo = hidden global i32 1, align 4
 @aligned_bar = hidden global i32 3, align 16
@@ -13,26 +18,28 @@ target triple = "wasm32-unknown-unknown"
 @local_struct = hidden global %struct.s zeroinitializer, align 4
 @local_struct_internal_ptr = hidden local_unnamed_addr global i32* getelementptr inbounds (%struct.s, %struct.s* @local_struct, i32 0, i32 1), align 4
 
-; RUN: wasm-ld -no-gc-sections --export=__data_end --export=__heap_base --allow-undefined --no-entry -o %t.wasm %t.o %t.hello.o
-; RUN: obj2yaml %t.wasm | FileCheck %s
-
 ; CHECK:        - Type:            MEMORY
 ; CHECK-NEXT:     Memories:
-; CHECK-NEXT:       - Initial:         0x00000002
+; CHK32-NEXT:       - Initial:         0x00000002
+; CHK64-NEXT:       - Flags:           [ IS_64 ]
+; CHK64-NEXT:         Initial:         0x00000002
 ; CHECK-NEXT:   - Type:            GLOBAL
 ; CHECK-NEXT:     Globals:
 ; CHECK-NEXT:       - Index:           0
-; CHECK-NEXT:         Type:            I32
+; CHK32-NEXT:         Type:            I32
+; CHK64-NEXT:         Type:            I64
 ; CHECK-NEXT:         Mutable:         true
 ; CHECK-NEXT:         InitExpr:
-; CHECK-NEXT:           Opcode:          I32_CONST
+; CHK32-NEXT:           Opcode:          I32_CONST
+; CHK64-NEXT:           Opcode:          I64_CONST
 ; CHECK-NEXT:           Value:           66624
 ; CHECK-NEXT:       - Index:           1
 ; CHECK-NEXT:         Type:            I32
 ; CHECK-NEXT:         Mutable:         false
 ; CHECK-NEXT:         InitExpr:
 ; CHECK-NEXT:           Opcode:          I32_CONST
-; CHECK-NEXT:           Value:           1080
+; CHK32-NEXT:           Value:           1080
+; CHK64-NEXT:           Value:           1088
 ; CHECK-NEXT:       - Index:           2
 ; CHECK-NEXT:         Type:            I32
 ; CHECK-NEXT:         Mutable:         false
@@ -53,13 +60,11 @@ target triple = "wasm32-unknown-unknown"
 ; CHECK-NEXT:         Offset:
 ; CHECK-NEXT:           Opcode:          I32_CONST
 ; CHECK-NEXT:           Value:           1040
-; CHECK-NEXT:         Content:         '0100000000000000000000000000000003000000000000000004000034040000'
-; CHECK-NEXT:    - Type:            CUSTOM
 
 
 ; RUN: wasm-ld -no-gc-sections --allow-undefined --no-entry \
-; RUN:     --initial-memory=131072 --max-memory=131072 -o %t_max.wasm %t.o \
-; RUN:     %t.hello.o
+; RUN:     --initial-memory=131072 --max-memory=131072 -o %t_max.wasm %t32.o \
+; RUN:     %t.hello32.o
 ; RUN: obj2yaml %t_max.wasm | FileCheck %s -check-prefix=CHECK-MAX
 
 ; CHECK-MAX:        - Type:            MEMORY
@@ -70,7 +75,7 @@ target triple = "wasm32-unknown-unknown"
 
 ; RUN: wasm-ld -no-gc-sections --allow-undefined --no-entry --shared-memory \
 ; RUN:     --features=atomics,bulk-memory --initial-memory=131072 \
-; RUN:     --max-memory=131072 -o %t_max.wasm %t.o %t.hello.o
+; RUN:     --max-memory=131072 -o %t_max.wasm %t32.o %t.hello32.o
 ; RUN: obj2yaml %t_max.wasm | FileCheck %s -check-prefix=CHECK-SHARED
 
 ; CHECK-SHARED:        - Type:            MEMORY
@@ -79,7 +84,7 @@ target triple = "wasm32-unknown-unknown"
 ; CHECK-SHARED-NEXT:         Initial:         0x00000002
 ; CHECK-SHARED-NEXT:         Maximum:         0x00000002
 
-; RUN: wasm-ld --relocatable -o %t_reloc.wasm %t.o %t.hello.o
+; RUN: wasm-ld --relocatable -o %t_reloc.wasm %t32.o %t.hello32.o
 ; RUN: obj2yaml %t_reloc.wasm | FileCheck %s -check-prefix=RELOC
 
 ; RELOC:       - Type:            DATA
diff --git a/lld/test/wasm/externref.s b/lld/test/wasm/externref.s
new file mode 100644
index 0000000000000..ffc63a6d3d0be
--- /dev/null
+++ b/lld/test/wasm/externref.s
@@ -0,0 +1,44 @@
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -filetype=obj -o %t.o %s
+# RUN: wasm-ld %t.o -o %t.wasm
+# RUN: obj2yaml %t.wasm | FileCheck %s
+
+# Tests use of externref type, including storing it in a global
+
+.globaltype my_global, externref
+
+.globl read_externref
+read_externref:
+  .functype read_externref () -> (externref)
+  global.get my_global
+  end_function
+
+.globl write_externref
+write_externref:
+  .functype write_externref (externref) -> ()
+  local.get 0
+  global.set my_global
+  end_function
+
+my_global:
+
+.globl _start
+_start:
+  .functype _start () -> ()
+  call read_externref
+  call write_externref
+  end_function
+
+#      CHECK:  - Type:            GLOBAL
+# CHECK-NEXT:    Globals:
+# CHECK-NEXT:      - Index:           0
+# CHECK-NEXT:        Type:            I32
+# CHECK-NEXT:        Mutable:         true
+# CHECK-NEXT:        InitExpr:
+# CHECK-NEXT:          Opcode:          I32_CONST
+# CHECK-NEXT:          Value:           66560
+# CHECK-NEXT:      - Index:           1
+# CHECK-NEXT:        Type:            EXTERNREF
+# CHECK-NEXT:        Mutable:         true
+# CHECK-NEXT:        InitExpr:
+# CHECK-NEXT:          Opcode:          REF_NULL
+# CHECK-NEXT:          Type:            EXTERNREF
diff --git a/lld/test/wasm/lto/incompatible.ll b/lld/test/wasm/lto/incompatible.ll
index 335756420639f..46d5c73dde66c 100644
--- a/lld/test/wasm/lto/incompatible.ll
+++ b/lld/test/wasm/lto/incompatible.ll
@@ -2,7 +2,11 @@
 ; RUN: llvm-as %s -o %t.bc
 ; RUN: not wasm-ld %t.bc -o %t.wasm 2>&1 | FileCheck %s
 
+; RUN: llvm-ar rc %t.a %t.bc
+; RUN: not wasm-ld --whole-archive %t.a -o %t.wasm 2>&1 | FileCheck %s --check-prefix=CHECK-ARCHIVE
+
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; CHECK: {{.*}}incompatible.ll.tmp.bc: machine type must be wasm32
+; CHECK-ARCHIVE: wasm-ld: error: {{.*}}.a(incompatible.ll.tmp.bc): machine type must be wasm32
diff --git a/lld/test/wasm/lto/libcall-truncsfhf2.ll b/lld/test/wasm/lto/libcall-truncsfhf2.ll
index a427acd139d8e..a60af3b52f79c 100644
--- a/lld/test/wasm/lto/libcall-truncsfhf2.ll
+++ b/lld/test/wasm/lto/libcall-truncsfhf2.ll
@@ -17,4 +17,4 @@ define void @_start() {
   ret void
 }
 
-; CHECK: wasm-ld: error: {{.*}}truncsfhf2.o: attempt to add bitcode file after LTO.
+; CHECK: wasm-ld: error: {{.*}}truncsfhf2.o): attempt to add bitcode file after LTO.
diff --git a/lld/unittests/MachOTests/MachONormalizedFileBinaryReaderTests.cpp b/lld/unittests/MachOTests/MachONormalizedFileBinaryReaderTests.cpp
index aad5f8afcfdc3..fbf18a8d9e007 100644
--- a/lld/unittests/MachOTests/MachONormalizedFileBinaryReaderTests.cpp
+++ b/lld/unittests/MachOTests/MachONormalizedFileBinaryReaderTests.cpp
@@ -75,7 +75,7 @@ TEST(BinaryReaderTest, empty_obj_x86_64) {
       fromBinary(fileBytes, sizeof(fileBytes), "x86_64");
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_x86_64);
   EXPECT_EQ((int)(f->fileType), MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_TRUE(f->localSymbols.empty());
   EXPECT_TRUE(f->globalSymbols.empty());
   EXPECT_TRUE(f->undefinedSymbols.empty());
@@ -106,7 +106,7 @@ TEST(BinaryReaderTest, empty_obj_x86) {
       fromBinary(fileBytes, sizeof(fileBytes), "i386");
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_x86);
   EXPECT_EQ((int)(f->fileType), MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_TRUE(f->localSymbols.empty());
   EXPECT_TRUE(f->globalSymbols.empty());
   EXPECT_TRUE(f->undefinedSymbols.empty());
@@ -137,7 +137,7 @@ TEST(BinaryReaderTest, empty_obj_ppc) {
       fromBinary(fileBytes, sizeof(fileBytes), "ppc");
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_ppc);
   EXPECT_EQ((int)(f->fileType), MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_TRUE(f->localSymbols.empty());
   EXPECT_TRUE(f->globalSymbols.empty());
   EXPECT_TRUE(f->undefinedSymbols.empty());
@@ -168,7 +168,7 @@ TEST(BinaryReaderTest, empty_obj_armv7) {
       fromBinary(fileBytes, sizeof(fileBytes), "armv7");
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_armv7);
   EXPECT_EQ((int)(f->fileType), MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_TRUE(f->localSymbols.empty());
   EXPECT_TRUE(f->globalSymbols.empty());
   EXPECT_TRUE(f->undefinedSymbols.empty());
@@ -182,7 +182,7 @@ TEST(BinaryReaderTest, empty_obj_x86_64_arm7) {
       fromBinary(fileBytes, sizeof(fileBytes), "x86_64");
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_x86_64);
   EXPECT_EQ((int)(f->fileType), MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_TRUE(f->localSymbols.empty());
   EXPECT_TRUE(f->globalSymbols.empty());
   EXPECT_TRUE(f->undefinedSymbols.empty());
@@ -191,7 +191,7 @@ TEST(BinaryReaderTest, empty_obj_x86_64_arm7) {
       fromBinary(fileBytes, sizeof(fileBytes), "armv7");
   EXPECT_EQ(f2->arch, lld::MachOLinkingContext::arch_armv7);
   EXPECT_EQ((int)(f2->fileType), MH_OBJECT);
-  EXPECT_EQ((int)(f2->flags), MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f2->flags), (int)MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_TRUE(f2->localSymbols.empty());
   EXPECT_TRUE(f2->globalSymbols.empty());
   EXPECT_TRUE(f2->undefinedSymbols.empty());
@@ -268,7 +268,7 @@ TEST(BinaryReaderTest, hello_obj_x86_64) {
 
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_x86_64);
   EXPECT_EQ((int)(f->fileType), MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_EQ(f->sections.size(), 2UL);
   const Section& text = f->sections[0];
   EXPECT_TRUE(text.segmentName.equals("__TEXT"));
@@ -393,7 +393,7 @@ TEST(BinaryReaderTest, hello_obj_x86) {
 
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_x86);
   EXPECT_EQ((int)(f->fileType), MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_EQ(f->sections.size(), 2UL);
   const Section& text = f->sections[0];
   EXPECT_TRUE(text.segmentName.equals("__TEXT"));
@@ -525,7 +525,7 @@ TEST(BinaryReaderTest, hello_obj_armv7) {
 
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_armv7);
   EXPECT_EQ((int)(f->fileType), MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_EQ(f->sections.size(), 2UL);
   const Section& text = f->sections[0];
   EXPECT_TRUE(text.segmentName.equals("__TEXT"));
@@ -669,7 +669,7 @@ TEST(BinaryReaderTest, hello_obj_ppc) {
 
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_ppc);
   EXPECT_EQ((int)(f->fileType), MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_EQ(f->sections.size(), 2UL);
   const Section& text = f->sections[0];
   EXPECT_TRUE(text.segmentName.equals("__TEXT"));
diff --git a/lld/unittests/MachOTests/MachONormalizedFileYAMLTests.cpp b/lld/unittests/MachOTests/MachONormalizedFileYAMLTests.cpp
index 6ceb197b4b84a..dbfe3a051811a 100644
--- a/lld/unittests/MachOTests/MachONormalizedFileYAMLTests.cpp
+++ b/lld/unittests/MachOTests/MachONormalizedFileYAMLTests.cpp
@@ -50,7 +50,7 @@ TEST(ObjectFileYAML, empty_ppc) {
     "...\n");
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_ppc);
   EXPECT_EQ(f->fileType, llvm::MachO::MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)(int)llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_TRUE(f->sections.empty());
   EXPECT_TRUE(f->localSymbols.empty());
   EXPECT_TRUE(f->globalSymbols.empty());
@@ -66,7 +66,7 @@ TEST(ObjectFileYAML, empty_x86_64) {
     "...\n");
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_x86_64);
   EXPECT_EQ(f->fileType, llvm::MachO::MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)(int)llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_TRUE(f->sections.empty());
   EXPECT_TRUE(f->localSymbols.empty());
   EXPECT_TRUE(f->globalSymbols.empty());
@@ -82,7 +82,7 @@ TEST(ObjectFileYAML, empty_x86) {
     "...\n");
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_x86);
   EXPECT_EQ(f->fileType, llvm::MachO::MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_TRUE(f->sections.empty());
   EXPECT_TRUE(f->localSymbols.empty());
   EXPECT_TRUE(f->globalSymbols.empty());
@@ -98,7 +98,7 @@ TEST(ObjectFileYAML, empty_armv6) {
     "...\n");
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_armv6);
   EXPECT_EQ(f->fileType, llvm::MachO::MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_TRUE(f->sections.empty());
   EXPECT_TRUE(f->localSymbols.empty());
   EXPECT_TRUE(f->globalSymbols.empty());
@@ -114,7 +114,7 @@ TEST(ObjectFileYAML, empty_armv7) {
     "...\n");
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_armv7);
   EXPECT_EQ(f->fileType, llvm::MachO::MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_TRUE(f->sections.empty());
   EXPECT_TRUE(f->localSymbols.empty());
   EXPECT_TRUE(f->globalSymbols.empty());
@@ -130,7 +130,7 @@ TEST(ObjectFileYAML, empty_armv7s) {
     "...\n");
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_armv7s);
   EXPECT_EQ(f->fileType, llvm::MachO::MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_TRUE(f->sections.empty());
   EXPECT_TRUE(f->localSymbols.empty());
   EXPECT_TRUE(f->globalSymbols.empty());
@@ -143,7 +143,7 @@ TEST(ObjectFileYAML, roundTrip) {
     NormalizedFile f;
     f.arch = lld::MachOLinkingContext::arch_x86_64;
     f.fileType = llvm::MachO::MH_OBJECT;
-    f.flags = llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS;
+    f.flags = (int)llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS;
     f.os = lld::MachOLinkingContext::OS::macOSX;
     toYAML(f, intermediate);
   }
@@ -151,7 +151,7 @@ TEST(ObjectFileYAML, roundTrip) {
     std::unique_ptr<NormalizedFile> f2 = fromYAML(intermediate);
     EXPECT_EQ(f2->arch, lld::MachOLinkingContext::arch_x86_64);
     EXPECT_EQ((int)(f2->fileType), llvm::MachO::MH_OBJECT);
-    EXPECT_EQ((int)(f2->flags), llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
+    EXPECT_EQ((int)(f2->flags), (int)llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
     EXPECT_TRUE(f2->sections.empty());
     EXPECT_TRUE(f2->localSymbols.empty());
     EXPECT_TRUE(f2->globalSymbols.empty());
@@ -275,7 +275,7 @@ TEST(ObjectFileYAML, hello_x86_64) {
     "...\n");
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_x86_64);
   EXPECT_EQ(f->fileType, llvm::MachO::MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_EQ(f->sections.size(), 2UL);
 
   const Section& sect1 = f->sections[0];
@@ -405,7 +405,7 @@ TEST(ObjectFileYAML, hello_x86) {
     "...\n");
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_x86);
   EXPECT_EQ(f->fileType, llvm::MachO::MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_EQ(f->sections.size(), 2UL);
 
   const Section& sect1 = f->sections[0];
@@ -533,7 +533,7 @@ TEST(ObjectFileYAML, hello_armv6) {
     "...\n");
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_armv6);
   EXPECT_EQ(f->fileType, llvm::MachO::MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_EQ(f->sections.size(), 2UL);
 
   const Section& sect1 = f->sections[0];
@@ -673,7 +673,7 @@ TEST(ObjectFileYAML, hello_armv7) {
     "...\n");
   EXPECT_EQ(f->arch, lld::MachOLinkingContext::arch_armv7);
   EXPECT_EQ(f->fileType, llvm::MachO::MH_OBJECT);
-  EXPECT_EQ((int)(f->flags), llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
+  EXPECT_EQ((int)(f->flags), (int)llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
   EXPECT_EQ(f->sections.size(), 2UL);
 
   const Section& sect1 = f->sections[0];
diff --git a/lld/wasm/Config.h b/lld/wasm/Config.h
index 1ba28899e64db..cae2852baf86b 100644
--- a/lld/wasm/Config.h
+++ b/lld/wasm/Config.h
@@ -27,6 +27,7 @@ struct Configuration {
   bool compressRelocations;
   bool demangle;
   bool disableVerify;
+  bool experimentalPic;
   bool emitRelocs;
   bool exportAll;
   bool exportDynamic;
@@ -36,6 +37,7 @@ struct Configuration {
   bool importMemory;
   bool sharedMemory;
   bool importTable;
+  bool is64;
   bool mergeDataSegments;
   bool pie;
   bool printGcSections;
diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp
index c6036111ec79b..d0805bf3b3036 100644
--- a/lld/wasm/Driver.cpp
+++ b/lld/wasm/Driver.cpp
@@ -332,6 +332,7 @@ static void readConfigs(opt::InputArgList &args) {
   config->demangle = args.hasFlag(OPT_demangle, OPT_no_demangle, true);
   config->disableVerify = args.hasArg(OPT_disable_verify);
   config->emitRelocs = args.hasArg(OPT_emit_relocs);
+  config->experimentalPic = args.hasArg(OPT_experimental_pic);
   config->entry = getEntry(args);
   config->exportAll = args.hasArg(OPT_export_all);
   config->exportTable = args.hasArg(OPT_export_table);
@@ -378,6 +379,18 @@ static void readConfigs(opt::InputArgList &args) {
   config->exportDynamic =
       args.hasFlag(OPT_export_dynamic, OPT_no_export_dynamic, config->shared);
 
+  // Parse wasm32/64.
+  config->is64 = false;
+  if (auto *arg = args.getLastArg(OPT_m)) {
+    StringRef s = arg->getValue();
+    if (s == "wasm32")
+      config->is64 = false;
+    else if (s == "wasm64")
+      config->is64 = true;
+    else
+      error("invalid target architecture: " + s);
+  }
+
   // --threads= takes a positive integer and provides the default value for
   // --thinlto-jobs=.
   if (auto *arg = args.getLastArg(OPT_threads)) {
@@ -456,6 +469,23 @@ static void checkOptions(opt::InputArgList &args) {
     if (config->sharedMemory)
       error("-r and --shared-memory may not be used together");
   }
+
+  // To begin to prepare for Module Linking-style shared libraries, start
+  // warning about uses of `-shared` and related flags outside of Experimental
+  // mode, to give anyone using them a heads-up that they will be changing.
+  //
+  // Also, warn about flags which request explicit exports.
+  if (!config->experimentalPic) {
+    // -shared will change meaning when Module Linking is implemented.
+    if (config->shared) {
+      warn("creating shared libraries, with -shared, is not yet stable");
+    }
+
+    // -pie will change meaning when Module Linking is implemented.
+    if (config->pie) {
+      warn("creating PIEs, with -pie, is not yet stable");
+    }
+  }
 }
 
 // Force Sym to be entered in the output. Used for -u or equivalent.
@@ -498,9 +528,15 @@ createUndefinedGlobal(StringRef name, llvm::wasm::WasmGlobalType *type) {
 static GlobalSymbol *createGlobalVariable(StringRef name, bool isMutable,
                                           int value) {
   llvm::wasm::WasmGlobal wasmGlobal;
-  wasmGlobal.Type = {WASM_TYPE_I32, isMutable};
-  wasmGlobal.InitExpr.Value.Int32 = value;
-  wasmGlobal.InitExpr.Opcode = WASM_OPCODE_I32_CONST;
+  if (config->is64) {
+    wasmGlobal.Type = {WASM_TYPE_I64, isMutable};
+    wasmGlobal.InitExpr.Value.Int64 = value;
+    wasmGlobal.InitExpr.Opcode = WASM_OPCODE_I64_CONST;
+  } else {
+    wasmGlobal.Type = {WASM_TYPE_I32, isMutable};
+    wasmGlobal.InitExpr.Value.Int32 = value;
+    wasmGlobal.InitExpr.Opcode = WASM_OPCODE_I32_CONST;
+  }
   wasmGlobal.SymbolName = name;
   return symtab->addSyntheticGlobal(name, WASM_SYMBOL_VISIBILITY_HIDDEN,
                                     make<InputGlobal>(wasmGlobal, nullptr));
@@ -513,9 +549,13 @@ static void createSyntheticSymbols() {
 
   static WasmSignature nullSignature = {{}, {}};
   static WasmSignature i32ArgSignature = {{}, {ValType::I32}};
+  static WasmSignature i64ArgSignature = {{}, {ValType::I64}};
   static llvm::wasm::WasmGlobalType globalTypeI32 = {WASM_TYPE_I32, false};
+  static llvm::wasm::WasmGlobalType globalTypeI64 = {WASM_TYPE_I64, false};
   static llvm::wasm::WasmGlobalType mutableGlobalTypeI32 = {WASM_TYPE_I32,
                                                             true};
+  static llvm::wasm::WasmGlobalType mutableGlobalTypeI64 = {WASM_TYPE_I64,
+                                                            true};
   WasmSym::callCtors = symtab->addSyntheticFunction(
       "__wasm_call_ctors", WASM_SYMBOL_VISIBILITY_HIDDEN,
       make<SyntheticFunction>(nullSignature, "__wasm_call_ctors"));
@@ -530,15 +570,16 @@ static void createSyntheticSymbols() {
 
 
   if (config->isPic) {
-    WasmSym::stackPointer =
-        createUndefinedGlobal("__stack_pointer", &mutableGlobalTypeI32);
+    WasmSym::stackPointer = createUndefinedGlobal(
+        "__stack_pointer",
+        config->is64 ? &mutableGlobalTypeI64 : &mutableGlobalTypeI32);
     // For PIC code, we import two global variables (__memory_base and
     // __table_base) from the environment and use these as the offset at
     // which to load our static data and function table.
     // See:
     // https://github.com/WebAssembly/tool-conventions/blob/master/DynamicLinking.md
-    WasmSym::memoryBase =
-        createUndefinedGlobal("__memory_base", &globalTypeI32);
+    WasmSym::memoryBase = createUndefinedGlobal(
+        "__memory_base", config->is64 ? &globalTypeI64 : &globalTypeI32);
     WasmSym::tableBase = createUndefinedGlobal("__table_base", &globalTypeI32);
     WasmSym::memoryBase->markLive();
     WasmSym::tableBase->markLive();
@@ -563,7 +604,9 @@ static void createSyntheticSymbols() {
     WasmSym::tlsAlign = createGlobalVariable("__tls_align", false, 1);
     WasmSym::initTLS = symtab->addSyntheticFunction(
         "__wasm_init_tls", WASM_SYMBOL_VISIBILITY_HIDDEN,
-        make<SyntheticFunction>(i32ArgSignature, "__wasm_init_tls"));
+        make<SyntheticFunction>(config->is64 ? i64ArgSignature
+                                             : i32ArgSignature,
+                                "__wasm_init_tls"));
   }
 }
 
diff --git a/lld/wasm/InputChunks.cpp b/lld/wasm/InputChunks.cpp
index abea11bd805b1..7f06e61a4b5ae 100644
--- a/lld/wasm/InputChunks.cpp
+++ b/lld/wasm/InputChunks.cpp
@@ -32,6 +32,18 @@ StringRef relocTypeToString(uint8_t relocType) {
   llvm_unreachable("unknown reloc type");
 }
 
+bool relocIs64(uint8_t relocType) {
+  switch (relocType) {
+  case R_WASM_MEMORY_ADDR_LEB64:
+  case R_WASM_MEMORY_ADDR_SLEB64:
+  case R_WASM_MEMORY_ADDR_REL_SLEB64:
+  case R_WASM_MEMORY_ADDR_I64:
+    return true;
+  default:
+    return false;
+  }
+}
+
 std::string toString(const wasm::InputChunk *c) {
   return (toString(c->file) + ":(" + c->getName() + ")").str();
 }
@@ -323,12 +335,17 @@ void InputSegment::generateRelocationCode(raw_ostream &os) const {
   LLVM_DEBUG(dbgs() << "generating runtime relocations: " << getName()
                     << " count=" << relocations.size() << "\n");
 
+  unsigned opcode_ptr_const =
+      config->is64 ? WASM_OPCODE_I64_CONST : WASM_OPCODE_I32_CONST;
+  unsigned opcode_ptr_add =
+      config->is64 ? WASM_OPCODE_I64_ADD : WASM_OPCODE_I32_ADD;
+
   // TODO(sbc): Encode the relocations in the data section and write a loop
   // here to apply them.
   uint32_t segmentVA = outputSeg->startVA + outputSegmentOffset;
   for (const WasmRelocation &rel : relocations) {
-    uint32_t offset = rel.Offset - getInputSectionOffset();
-    uint32_t outputOffset = segmentVA + offset;
+    uint64_t offset = rel.Offset - getInputSectionOffset();
+    uint64_t outputOffset = segmentVA + offset;
 
     LLVM_DEBUG(dbgs() << "gen reloc: type=" << relocTypeToString(rel.Type)
                       << " addend=" << rel.Addend << " index=" << rel.Index
@@ -339,9 +356,17 @@ void InputSegment::generateRelocationCode(raw_ostream &os) const {
     writeUleb128(os, WasmSym::memoryBase->getGlobalIndex(), "memory_base");
 
     // Add the offset of the relocation
-    writeU8(os, WASM_OPCODE_I32_CONST, "I32_CONST");
+    writeU8(os, opcode_ptr_const, "CONST");
     writeSleb128(os, outputOffset, "offset");
-    writeU8(os, WASM_OPCODE_I32_ADD, "ADD");
+    writeU8(os, opcode_ptr_add, "ADD");
+
+    bool is64 = relocIs64(rel.Type);
+    unsigned opcode_reloc_const =
+        is64 ? WASM_OPCODE_I64_CONST : WASM_OPCODE_I32_CONST;
+    unsigned opcode_reloc_add =
+        is64 ? WASM_OPCODE_I64_ADD : WASM_OPCODE_I32_ADD;
+    unsigned opcode_reloc_store =
+        is64 ? WASM_OPCODE_I64_STORE : WASM_OPCODE_I32_STORE;
 
     Symbol *sym = file->getSymbol(rel);
     // Now figure out what we want to store
@@ -349,9 +374,9 @@ void InputSegment::generateRelocationCode(raw_ostream &os) const {
       writeU8(os, WASM_OPCODE_GLOBAL_GET, "GLOBAL_GET");
       writeUleb128(os, sym->getGOTIndex(), "global index");
       if (rel.Addend) {
-        writeU8(os, WASM_OPCODE_I32_CONST, "CONST");
+        writeU8(os, opcode_reloc_const, "CONST");
         writeSleb128(os, rel.Addend, "addend");
-        writeU8(os, WASM_OPCODE_I32_ADD, "ADD");
+        writeU8(os, opcode_reloc_add, "ADD");
       }
     } else {
       const GlobalSymbol* baseSymbol = WasmSym::memoryBase;
@@ -359,13 +384,13 @@ void InputSegment::generateRelocationCode(raw_ostream &os) const {
         baseSymbol = WasmSym::tableBase;
       writeU8(os, WASM_OPCODE_GLOBAL_GET, "GLOBAL_GET");
       writeUleb128(os, baseSymbol->getGlobalIndex(), "base");
-      writeU8(os, WASM_OPCODE_I32_CONST, "CONST");
+      writeU8(os, opcode_reloc_const, "CONST");
       writeSleb128(os, file->calcNewValue(rel), "offset");
-      writeU8(os, WASM_OPCODE_I32_ADD, "ADD");
+      writeU8(os, opcode_reloc_add, "ADD");
     }
 
     // Store that value at the virtual address
-    writeU8(os, WASM_OPCODE_I32_STORE, "I32_STORE");
+    writeU8(os, opcode_reloc_store, "I32_STORE");
     writeUleb128(os, 2, "align");
     writeUleb128(os, 0, "offset");
   }
diff --git a/lld/wasm/InputFiles.cpp b/lld/wasm/InputFiles.cpp
index d6f01578465ff..93d390a5457a9 100644
--- a/lld/wasm/InputFiles.cpp
+++ b/lld/wasm/InputFiles.cpp
@@ -569,8 +569,7 @@ bool BitcodeFile::doneLTO = false;
 
 void BitcodeFile::parse() {
   if (doneLTO) {
-    error(toString(mb.getBufferIdentifier()) +
-          ": attempt to add bitcode file after LTO.");
+    error(toString(this) + ": attempt to add bitcode file after LTO.");
     return;
   }
 
@@ -578,7 +577,7 @@ void BitcodeFile::parse() {
       mb.getBuffer(), saver.save(archiveName + mb.getBufferIdentifier()))));
   Triple t(obj->getTargetTriple());
   if (t.getArch() != Triple::wasm32) {
-    error(toString(mb.getBufferIdentifier()) + ": machine type must be wasm32");
+    error(toString(this) + ": machine type must be wasm32");
     return;
   }
   std::vector<bool> keptComdats;
diff --git a/lld/wasm/Options.td b/lld/wasm/Options.td
index 0910a74660f3c..16c784f74828a 100644
--- a/lld/wasm/Options.td
+++ b/lld/wasm/Options.td
@@ -62,6 +62,8 @@ def l: JoinedOrSeparate<["-"], "l">, MetaVarName<"<libName>">,
 def L: JoinedOrSeparate<["-"], "L">, MetaVarName<"<dir>">,
   HelpText<"Add a directory to the library search path">;
 
+def m: JoinedOrSeparate<["-"], "m">, HelpText<"Set target emulation">;
+
 def mllvm: S<"mllvm">, HelpText<"Options to pass to LLVM">;
 
 def no_color_diagnostics: F<"no-color-diagnostics">,
@@ -179,7 +181,6 @@ def: JoinedOrSeparate<["-"], "e">, Alias<entry>;
 def: J<"entry=">, Alias<entry>;
 def: Flag<["-"], "E">, Alias<export_dynamic>, HelpText<"Alias for --export-dynamic">;
 def: Flag<["-"], "i">, Alias<initial_memory>;
-def: Flag<["-"], "m">, Alias<max_memory>;
 def: Flag<["-"], "r">, Alias<relocatable>;
 def: Flag<["-"], "s">, Alias<strip_all>, HelpText<"Alias for --strip-all">;
 def: Flag<["-"], "S">, Alias<strip_debug>, HelpText<"Alias for --strip-debug">;
@@ -199,3 +200,7 @@ def thinlto_cache_dir: J<"thinlto-cache-dir=">,
 defm thinlto_cache_policy: Eq<"thinlto-cache-policy", "Pruning policy for the ThinLTO cache">;
 def thinlto_jobs: J<"thinlto-jobs=">,
   HelpText<"Number of ThinLTO jobs. Default to --threads=">;
+
+// Experimental PIC mode.
+def experimental_pic: F<"experimental-pic">,
+  HelpText<"Enable Experimental PIC">;
diff --git a/lld/wasm/SyntheticSections.cpp b/lld/wasm/SyntheticSections.cpp
index 0ed3ea273d703..70d6a10200c6e 100644
--- a/lld/wasm/SyntheticSections.cpp
+++ b/lld/wasm/SyntheticSections.cpp
@@ -139,6 +139,8 @@ void ImportSection::writeBody() {
     }
     if (config->sharedMemory)
       import.Memory.Flags |= WASM_LIMITS_FLAG_IS_SHARED;
+    if (config->is64)
+      import.Memory.Flags |= WASM_LIMITS_FLAG_IS_64;
     writeImport(os, import);
   }
 
@@ -234,6 +236,8 @@ void MemorySection::writeBody() {
     flags |= WASM_LIMITS_FLAG_HAS_MAX;
   if (config->sharedMemory)
     flags |= WASM_LIMITS_FLAG_IS_SHARED;
+  if (config->is64)
+    flags |= WASM_LIMITS_FLAG_IS_64;
   writeUleb128(os, flags, "memory limits flags");
   writeUleb128(os, numMemoryPages, "initial pages");
   if (hasMax)
diff --git a/lld/wasm/SyntheticSections.h b/lld/wasm/SyntheticSections.h
index 6cf593cf016ce..3e125ca84e401 100644
--- a/lld/wasm/SyntheticSections.h
+++ b/lld/wasm/SyntheticSections.h
@@ -167,8 +167,8 @@ class MemorySection : public SyntheticSection {
   bool isNeeded() const override { return !config->importMemory; }
   void writeBody() override;
 
-  uint32_t numMemoryPages = 0;
-  uint32_t maxMemoryPages = 0;
+  uint64_t numMemoryPages = 0;
+  uint64_t maxMemoryPages = 0;
 };
 
 // The event section contains a list of declared wasm events associated with the
diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp
index 0434a4cf02938..1401dc50931b3 100644
--- a/lld/wasm/Writer.cpp
+++ b/lld/wasm/Writer.cpp
@@ -224,8 +224,16 @@ void Writer::layoutMemory() {
     log("mem: stack base  = " + Twine(memoryPtr));
     memoryPtr += config->zStackSize;
     auto *sp = cast<DefinedGlobal>(WasmSym::stackPointer);
-    assert(sp->global->global.InitExpr.Opcode == WASM_OPCODE_I32_CONST);
-    sp->global->global.InitExpr.Value.Int32 = memoryPtr;
+    switch (sp->global->global.InitExpr.Opcode) {
+    case WASM_OPCODE_I32_CONST:
+      sp->global->global.InitExpr.Value.Int32 = memoryPtr;
+      break;
+    case WASM_OPCODE_I64_CONST:
+      sp->global->global.InitExpr.Value.Int64 = memoryPtr;
+      break;
+    default:
+      llvm_unreachable("init expr must be i32/i64.const");
+    }
     log("mem: stack top   = " + Twine(memoryPtr));
   };
 
@@ -296,13 +304,16 @@ void Writer::layoutMemory() {
   if (WasmSym::heapBase)
     WasmSym::heapBase->setVirtualAddress(memoryPtr);
 
+  uint64_t maxMemorySetting = 1ULL << (config->is64 ? 48 : 32);
+
   if (config->initialMemory != 0) {
     if (config->initialMemory != alignTo(config->initialMemory, WasmPageSize))
       error("initial memory must be " + Twine(WasmPageSize) + "-byte aligned");
     if (memoryPtr > config->initialMemory)
       error("initial memory too small, " + Twine(memoryPtr) + " bytes needed");
-    if (config->initialMemory > (1ULL << 32))
-      error("initial memory too large, cannot be greater than 4294967296");
+    if (config->initialMemory > maxMemorySetting)
+      error("initial memory too large, cannot be greater than " +
+            Twine(maxMemorySetting));
     memoryPtr = config->initialMemory;
   }
   out.dylinkSec->memSize = memoryPtr;
@@ -316,8 +327,9 @@ void Writer::layoutMemory() {
       error("maximum memory must be " + Twine(WasmPageSize) + "-byte aligned");
     if (memoryPtr > config->maxMemory)
       error("maximum memory too small, " + Twine(memoryPtr) + " bytes needed");
-    if (config->maxMemory > (1ULL << 32))
-      error("maximum memory too large, cannot be greater than 4294967296");
+    if (config->maxMemory > maxMemorySetting)
+      error("maximum memory too large, cannot be greater than " +
+            Twine(maxMemorySetting));
     out.memorySec->maxMemoryPages = config->maxMemory / WasmPageSize;
     log("mem: max pages   = " + Twine(out.memorySec->maxMemoryPages));
   }
diff --git a/lld/wasm/WriterUtils.cpp b/lld/wasm/WriterUtils.cpp
index 9b14da8e9f2e4..436d34e3439a1 100644
--- a/lld/wasm/WriterUtils.cpp
+++ b/lld/wasm/WriterUtils.cpp
@@ -32,6 +32,8 @@ std::string toString(ValType type) {
     return "v128";
   case ValType::EXNREF:
     return "exnref";
+  case ValType::EXTERNREF:
+    return "externref";
   }
   llvm_unreachable("Invalid wasm::ValType");
 }
@@ -155,6 +157,9 @@ void writeInitExpr(raw_ostream &os, const WasmInitExpr &initExpr) {
   case WASM_OPCODE_GLOBAL_GET:
     writeUleb128(os, initExpr.Value.Global, "literal (global index)");
     break;
+  case WASM_OPCODE_REF_NULL:
+    writeValueType(os, ValType::EXTERNREF, "literal (externref type)");
+    break;
   default:
     fatal("unknown opcode in init expr: " + Twine(initExpr.Opcode));
   }
diff --git a/lldb/bindings/interface/SBBreakpoint.i b/lldb/bindings/interface/SBBreakpoint.i
index 20354346be900..a2d747db0bf6d 100644
--- a/lldb/bindings/interface/SBBreakpoint.i
+++ b/lldb/bindings/interface/SBBreakpoint.i
@@ -206,6 +206,9 @@ public:
     bool
     AddName (const char *new_name);
 
+    SBError
+    AddNameWithErrorHandling (const char *new_name);
+
     void
     RemoveName (const char *name_to_remove);
 
diff --git a/lldb/cmake/modules/FindLuaAndSwig.cmake b/lldb/cmake/modules/FindLuaAndSwig.cmake
index cc9083ce049a7..968f905ef2d33 100644
--- a/lldb/cmake/modules/FindLuaAndSwig.cmake
+++ b/lldb/cmake/modules/FindLuaAndSwig.cmake
@@ -9,7 +9,7 @@ if(LUA_LIBRARIES AND LUA_INCLUDE_DIR AND SWIG_EXECUTABLE)
 else()
   find_package(SWIG 2.0)
   if (SWIG_FOUND)
-    find_package(Lua)
+    find_package(Lua 5.3)
     if(LUA_FOUND AND SWIG_FOUND)
       mark_as_advanced(
         LUA_LIBRARIES
diff --git a/lldb/docs/index.rst b/lldb/docs/index.rst
index 29f63b3283811..50441f5be9107 100644
--- a/lldb/docs/index.rst
+++ b/lldb/docs/index.rst
@@ -69,12 +69,12 @@ Platform Support
 LLDB is known to work on the following platforms, but ports to new platforms
 are welcome:
 
-* macOS desktop user space debugging for i386 and x86_64
-* iOS, tvOS, and watchOS simulator debugging on i386 and x86_64
+* macOS debugging for i386, x86_64 and AArch64
+* iOS, tvOS, and watchOS simulator debugging on i386, x86_64 and AArch64
 * iOS, tvOS, and watchOS device debugging on ARM and AArch64
-* Linux local user-space debugging for i386, x86_64 and PPC64le
-* FreeBSD local user-space debugging for i386 and x86_64
-* Windows local user-space debugging for i386 (*)
+* Linux user-space debugging for i386, x86_64 and PPC64le
+* FreeBSD user-space debugging for i386 and x86_64
+* Windows user-space debugging for i386 (*)
 
 (*) Support for Windows is under active development. Basic functionality is
 expected to work, with functionality improving rapidly.
diff --git a/lldb/docs/resources/caveats.rst b/lldb/docs/resources/caveats.rst
index 780fb349051bc..2f37a6821ca5c 100644
--- a/lldb/docs/resources/caveats.rst
+++ b/lldb/docs/resources/caveats.rst
@@ -10,7 +10,7 @@ Python
 ------
 
 LLDB has a powerful scripting interface which is accessible through Python.
-Python is available either from withing LLDB through a (interactive) script
+Python is available either from within LLDB through a (interactive) script
 interpreter, or as a Python module which you can import from the Python
 interpreter.
 
@@ -31,7 +31,41 @@ against Python comes with some constraints to be aware of.
     the one used to build and link LLDB.
 
 The previous considerations are especially important during development, but
-apply to binary distributions of LLDB as well. For example, the LLDB that comes
-with Xcode links against the Python 3 that's part of Xcode. Therefore you
-should always use the Python in Xcode (through ``xcrun python3`` or
-``/usr/bin/python3``) to import the lldb module or install packages.
+apply to binary distributions of LLDB as well.
+
+LLDB in Xcode on macOS
+``````````````````````
+
+Users of lldb in Xcode on macOS commonly run into these issues when they
+install Python, often unknowingly as a dependency pulled in by Homebrew or
+other package managers. The problem is the symlinks that get created in
+``/usr/local/bin``, which comes before ``/usr/bin`` in your path. You can use
+``which python3`` to check to what it resolves.
+
+To be sure you use the Python that matches with the lldb in Xcode use ``xcrun``
+or use the absolute path to the shims in ``/usr/bin``.
+
+::
+
+   $ xcrun python3
+   $ /usr/bin/python3
+
+Similarly, to install packages and be able to use them from within lldb, you'll
+need to install them with the matching ``pip3``.
+
+::
+
+   $ xcrun pip3
+   $ /usr/bin/pip3
+
+The same is true for Python 2. Although Python 2 comes with the operating
+system rather than Xcode, you can still use ``xcrun`` to launch the system
+variant.
+
+::
+
+   $ xcrun python
+   $ /usr/bin/python
+
+Keep in mind that Python 2 is deprecated and no longer maintained. Future
+versions of macOS will not include Python 2.7.
diff --git a/lldb/include/lldb/API/SBBreakpoint.h b/lldb/include/lldb/API/SBBreakpoint.h
index 5a94297b888ec..c9a52fcacf1a4 100644
--- a/lldb/include/lldb/API/SBBreakpoint.h
+++ b/lldb/include/lldb/API/SBBreakpoint.h
@@ -105,6 +105,8 @@ class LLDB_API SBBreakpoint {
 
   bool AddName(const char *new_name);
 
+  SBError AddNameWithErrorHandling(const char *new_name);
+
   void RemoveName(const char *name_to_remove);
 
   bool MatchesName(const char *name);
diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h
index 7c8a01beea56d..0080368fd9965 100644
--- a/lldb/include/lldb/Core/ValueObject.h
+++ b/lldb/include/lldb/Core/ValueObject.h
@@ -963,9 +963,14 @@ class ValueObject : public UserID {
 
   void SetPreferredDisplayLanguageIfNeeded(lldb::LanguageType);
 
+protected:
+  virtual void DoUpdateChildrenAddressType(ValueObject &valobj) { return; };
+
 private:
   virtual CompilerType MaybeCalculateCompleteType();
-  void UpdateChildrenAddressType();
+  void UpdateChildrenAddressType() {
+    GetRoot()->DoUpdateChildrenAddressType(*this);
+  }
 
   lldb::ValueObjectSP GetValueForExpressionPath_Impl(
       llvm::StringRef expression_cstr,
diff --git a/lldb/include/lldb/Core/ValueObjectVariable.h b/lldb/include/lldb/Core/ValueObjectVariable.h
index a0417e5e7d85a..b7e262574a14d 100644
--- a/lldb/include/lldb/Core/ValueObjectVariable.h
+++ b/lldb/include/lldb/Core/ValueObjectVariable.h
@@ -67,6 +67,8 @@ class ValueObjectVariable : public ValueObject {
 
 protected:
   bool UpdateValue() override;
+  
+  void DoUpdateChildrenAddressType(ValueObject &valobj) override;
 
   CompilerType GetCompilerTypeImpl() override;
 
diff --git a/lldb/include/lldb/Interpreter/ScriptInterpreter.h b/lldb/include/lldb/Interpreter/ScriptInterpreter.h
index d008e18e7b6f5..491923e6a6c4f 100644
--- a/lldb/include/lldb/Interpreter/ScriptInterpreter.h
+++ b/lldb/include/lldb/Interpreter/ScriptInterpreter.h
@@ -9,16 +9,16 @@
 #ifndef LLDB_INTERPRETER_SCRIPTINTERPRETER_H
 #define LLDB_INTERPRETER_SCRIPTINTERPRETER_H
 
-#include "lldb/lldb-private.h"
-
 #include "lldb/Breakpoint/BreakpointOptions.h"
+#include "lldb/Core/Communication.h"
 #include "lldb/Core/PluginInterface.h"
 #include "lldb/Core/SearchFilter.h"
+#include "lldb/Core/StreamFile.h"
+#include "lldb/Host/PseudoTerminal.h"
 #include "lldb/Utility/Broadcaster.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/StructuredData.h"
-
-#include "lldb/Host/PseudoTerminal.h"
+#include "lldb/lldb-private.h"
 
 namespace lldb_private {
 
@@ -34,6 +34,35 @@ class ScriptInterpreterLocker {
   operator=(const ScriptInterpreterLocker &) = delete;
 };
 
+class ScriptInterpreterIORedirect {
+public:
+  /// Create an IO redirect. If IO is enabled, this will redirects the output
+  /// to the command return object if set or to the debugger otherwise. If IO
+  /// is disabled, it will redirect all IO to /dev/null.
+  static llvm::Expected<std::unique_ptr<ScriptInterpreterIORedirect>>
+  Create(bool enable_io, Debugger &debugger, CommandReturnObject *result);
+
+  ~ScriptInterpreterIORedirect();
+
+  lldb::FileSP GetInputFile() const { return m_input_file_sp; }
+  lldb::FileSP GetOutputFile() const { return m_output_file_sp->GetFileSP(); }
+  lldb::FileSP GetErrorFile() const { return m_error_file_sp->GetFileSP(); }
+
+  /// Flush our output and error file handles.
+  void Flush();
+
+private:
+  ScriptInterpreterIORedirect(std::unique_ptr<File> input,
+                              std::unique_ptr<File> output);
+  ScriptInterpreterIORedirect(Debugger &debugger, CommandReturnObject *result);
+
+  lldb::FileSP m_input_file_sp;
+  lldb::StreamFileSP m_output_file_sp;
+  lldb::StreamFileSP m_error_file_sp;
+  Communication m_communication;
+  bool m_disconnect;
+};
+
 class ScriptInterpreter : public PluginInterface {
 public:
   enum ScriptReturnType {
diff --git a/lldb/include/lldb/Symbol/Function.h b/lldb/include/lldb/Symbol/Function.h
index e548b8d626a3a..300d829219d4d 100644
--- a/lldb/include/lldb/Symbol/Function.h
+++ b/lldb/include/lldb/Symbol/Function.h
@@ -17,6 +17,8 @@
 #include "lldb/Utility/UserID.h"
 #include "llvm/ADT/ArrayRef.h"
 
+#include <mutex>
+
 namespace lldb_private {
 
 class ExecutionContext;
@@ -113,10 +115,11 @@ class FunctionInfo {
   virtual size_t MemorySize() const;
 
 protected:
-  // Member variables.
-  ConstString m_name;        ///< Function method name (not a mangled name).
-  Declaration m_declaration; ///< Information describing where this function
-                             ///information was defined.
+  /// Function method name (not a mangled name).
+  ConstString m_name;
+
+  /// Information describing where this function information was defined.
+  Declaration m_declaration;
 };
 
 /// \class InlineFunctionInfo Function.h "lldb/Symbol/Function.h"
@@ -240,9 +243,10 @@ class InlineFunctionInfo : public FunctionInfo {
   size_t MemorySize() const override;
 
 private:
-  // Member variables.
-  Mangled m_mangled; ///< Mangled inlined function name (can be empty if there
-                     ///is no mangled information).
+  /// Mangled inlined function name (can be empty if there is no mangled
+  /// information).
+  Mangled m_mangled;
+
   Declaration m_call_decl;
 };
 
@@ -634,30 +638,50 @@ class Function : public UserID, public SymbolContextScope {
 
 protected:
   enum {
-    flagsCalculatedPrologueSize =
-        (1 << 0) ///< Have we already tried to calculate the prologue size?
+    /// Whether we already tried to calculate the prologue size.
+    flagsCalculatedPrologueSize = (1 << 0)
   };
 
-  // Member variables.
-  CompileUnit *m_comp_unit; ///< The compile unit that owns this function.
-  lldb::user_id_t
-      m_type_uid; ///< The user ID of for the prototype Type for this function.
-  Type *m_type; ///< The function prototype type for this function that include
-                ///the function info (FunctionInfo), return type and parameters.
-  Mangled m_mangled; ///< The mangled function name if any, if empty, there is
-                     ///no mangled information.
-  Block m_block;     ///< All lexical blocks contained in this function.
-  AddressRange m_range; ///< The function address range that covers the widest
-                        ///range needed to contain all blocks
-  DWARFExpression m_frame_base; ///< The frame base expression for variables
-                                ///that are relative to the frame pointer.
+  /// The compile unit that owns this function.
+  CompileUnit *m_comp_unit;
+
+  /// The user ID of for the prototype Type for this function.
+  lldb::user_id_t m_type_uid;
+
+  /// The function prototype type for this function that includes the function
+  /// info (FunctionInfo), return type and parameters.
+  Type *m_type;
+
+  /// The mangled function name if any. If empty, there is no mangled
+  /// information.
+  Mangled m_mangled;
+
+  /// All lexical blocks contained in this function.
+  Block m_block;
+
+  /// The function address range that covers the widest range needed to contain
+  /// all blocks
+  AddressRange m_range;
+
+  /// The frame base expression for variables that are relative to the frame
+  /// pointer.
+  DWARFExpression m_frame_base;
+
   Flags m_flags;
-  uint32_t
-      m_prologue_byte_size; ///< Compute the prologue size once and cache it
 
-  bool m_call_edges_resolved = false; ///< Whether call site info has been
-                                      ///  parsed.
-  std::vector<std::unique_ptr<CallEdge>> m_call_edges; ///< Outgoing call edges.
+  /// Compute the prologue size once and cache it.
+  uint32_t m_prologue_byte_size;
+
+  /// Exclusive lock that controls read/write access to m_call_edges and
+  /// m_call_edges_resolved.
+  std::mutex m_call_edges_lock;
+
+  /// Whether call site info has been parsed.
+  bool m_call_edges_resolved = false;
+
+  /// Outgoing call edges.
+  std::vector<std::unique_ptr<CallEdge>> m_call_edges;
+
 private:
   Function(const Function &) = delete;
   const Function &operator=(const Function &) = delete;
diff --git a/lldb/include/lldb/Utility/DataExtractor.h b/lldb/include/lldb/Utility/DataExtractor.h
index 8f82ae5af42da..0210af5cf6d18 100644
--- a/lldb/include/lldb/Utility/DataExtractor.h
+++ b/lldb/include/lldb/Utility/DataExtractor.h
@@ -9,12 +9,14 @@
 #ifndef LLDB_UTILITY_DATAEXTRACTOR_H
 #define LLDB_UTILITY_DATAEXTRACTOR_H
 
+#include "lldb/Utility/Endian.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-types.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/DataExtractor.h"
+#include "llvm/Support/SwapByteOrder.h"
 
 #include <cassert>
 #include <stdint.h>
@@ -979,6 +981,21 @@ class DataExtractor {
   }
 
 protected:
+  template <typename T> T Get(lldb::offset_t *offset_ptr, T fail_value) const {
+    constexpr size_t src_size = sizeof(T);
+    T val = fail_value;
+
+    const T *src = static_cast<const T *>(GetData(offset_ptr, src_size));
+    if (!src)
+      return val;
+
+    memcpy(&val, src, src_size);
+    if (m_byte_order != endian::InlHostByteOrder())
+      llvm::sys::swapByteOrder(val);
+
+    return val;
+  }
+
   // Member variables
   const uint8_t *m_start; ///< A pointer to the first byte of data.
   const uint8_t
diff --git a/lldb/include/lldb/Utility/RegisterValue.h b/lldb/include/lldb/Utility/RegisterValue.h
index 494f8be5391c0..c9f295a8d95ae 100644
--- a/lldb/include/lldb/Utility/RegisterValue.h
+++ b/lldb/include/lldb/Utility/RegisterValue.h
@@ -260,8 +260,9 @@ class RegisterValue {
   Scalar m_scalar;
 
   struct {
-    uint8_t bytes[kMaxRegisterByteSize]; // This must be big enough to hold any
-                                         // register for any supported target.
+    mutable uint8_t
+        bytes[kMaxRegisterByteSize]; // This must be big enough to hold any
+                                     // register for any supported target.
     uint16_t length;
     lldb::ByteOrder byte_order;
   } buffer;
diff --git a/lldb/include/lldb/Utility/Reproducer.h b/lldb/include/lldb/Utility/Reproducer.h
index 5d810460a0c5b..ab673e5e00192 100644
--- a/lldb/include/lldb/Utility/Reproducer.h
+++ b/lldb/include/lldb/Utility/Reproducer.h
@@ -159,6 +159,9 @@ class WorkingDirectoryProvider : public Provider<WorkingDirectoryProvider> {
   static char ID;
 };
 
+/// The recorder is a small object handed out by a provider to record data. It
+/// is commonly used in combination with a MultiProvider which is meant to
+/// record information for multiple instances of the same source of data.
 class AbstractRecorder {
 protected:
   AbstractRecorder(const FileSpec &filename, std::error_code &ec)
@@ -181,6 +184,7 @@ class AbstractRecorder {
   bool m_record;
 };
 
+/// Recorder that records its data as text to a file.
 class DataRecorder : public AbstractRecorder {
 public:
   DataRecorder(const FileSpec &filename, std::error_code &ec)
@@ -199,24 +203,88 @@ class DataRecorder : public AbstractRecorder {
   }
 };
 
-class CommandProvider : public Provider<CommandProvider> {
+/// Recorder that records its data as YAML to a file.
+class YamlRecorder : public AbstractRecorder {
+public:
+  YamlRecorder(const FileSpec &filename, std::error_code &ec)
+      : AbstractRecorder(filename, ec) {}
+
+  static llvm::Expected<std::unique_ptr<YamlRecorder>>
+  Create(const FileSpec &filename);
+
+  template <typename T> void Record(const T &t) {
+    if (!m_record)
+      return;
+    llvm::yaml::Output yout(m_os);
+    // The YAML traits are defined as non-const because they are used for
+    // serialization and deserialization. The cast is safe because
+    // serialization doesn't modify the object.
+    yout << const_cast<T &>(t);
+    m_os.flush();
+  }
+};
+
+/// The MultiProvider is a provider that hands out recorder which can be used
+/// to capture data for different instances of the same object. The recorders
+/// can be passed around or stored as an instance member.
+///
+/// The Info::file for the MultiProvider contains an index of files for every
+/// recorder. Use the MultiLoader to read the index and get the individual
+/// files.
+template <typename T, typename V>
+class MultiProvider : public repro::Provider<V> {
+public:
+  MultiProvider(const FileSpec &directory) : Provider<V>(directory) {}
+
+  T *GetNewRecorder() {
+    std::size_t i = m_recorders.size() + 1;
+    std::string filename = (llvm::Twine(V::Info::name) + llvm::Twine("-") +
+                            llvm::Twine(i) + llvm::Twine(".yaml"))
+                               .str();
+    auto recorder_or_error =
+        T::Create(this->GetRoot().CopyByAppendingPathComponent(filename));
+    if (!recorder_or_error) {
+      llvm::consumeError(recorder_or_error.takeError());
+      return nullptr;
+    }
+
+    m_recorders.push_back(std::move(*recorder_or_error));
+    return m_recorders.back().get();
+  }
+
+  void Keep() override {
+    std::vector<std::string> files;
+    for (auto &recorder : m_recorders) {
+      recorder->Stop();
+      files.push_back(recorder->GetFilename().GetPath());
+    }
+
+    FileSpec file = this->GetRoot().CopyByAppendingPathComponent(V::Info::file);
+    std::error_code ec;
+    llvm::raw_fd_ostream os(file.GetPath(), ec, llvm::sys::fs::OF_Text);
+    if (ec)
+      return;
+    llvm::yaml::Output yout(os);
+    yout << files;
+  }
+
+  void Discard() override { m_recorders.clear(); }
+
+private:
+  std::vector<std::unique_ptr<T>> m_recorders;
+};
+
+class CommandProvider : public MultiProvider<DataRecorder, CommandProvider> {
 public:
   struct Info {
     static const char *name;
     static const char *file;
   };
 
-  CommandProvider(const FileSpec &directory) : Provider(directory) {}
-
-  DataRecorder *GetNewDataRecorder();
-
-  void Keep() override;
-  void Discard() override;
+  CommandProvider(const FileSpec &directory)
+      : MultiProvider<DataRecorder, CommandProvider>(directory) {}
 
   static char ID;
-
-private:
-  std::vector<std::unique_ptr<DataRecorder>> m_data_recorders;
 };
 
 /// The generator is responsible for the logic needed to generate a
@@ -360,6 +428,8 @@ class Reproducer {
   mutable std::mutex m_mutex;
 };
 
+/// Loader for data captured with the MultiProvider. It will read the index and
+/// return the path to the files in the index.
 template <typename T> class MultiLoader {
 public:
   MultiLoader(std::vector<std::string> files) : m_files(files) {}
diff --git a/lldb/include/lldb/Utility/Scalar.h b/lldb/include/lldb/Utility/Scalar.h
index 1865a34775ee8..f215fa71c84c5 100644
--- a/lldb/include/lldb/Utility/Scalar.h
+++ b/lldb/include/lldb/Utility/Scalar.h
@@ -106,7 +106,10 @@ class Scalar {
 
   bool ClearBit(uint32_t bit);
 
-  const void *GetBytes() const;
+  /// Store the binary representation of this value into the given storage.
+  /// Exactly GetByteSize() bytes will be stored, and the buffer must be large
+  /// enough to hold this data.
+  void GetBytes(llvm::MutableArrayRef<uint8_t> storage) const;
 
   size_t GetByteSize() const;
 
@@ -122,7 +125,7 @@ class Scalar {
     m_integer.clearAllBits();
   }
 
-  const char *GetTypeAsCString() const;
+  const char *GetTypeAsCString() const { return GetValueTypeAsCString(m_type); }
 
   void GetValue(Stream *s, bool show_type) const;
 
@@ -130,8 +133,8 @@ class Scalar {
     return (m_type >= e_sint) && (m_type <= e_long_double);
   }
 
-  /// Convert integer to \p type, limited to \p bits size.
-  void TruncOrExtendTo(Scalar::Type type, uint16_t bits);
+  /// Convert to an integer with \p bits and the given signedness.
+  void TruncOrExtendTo(uint16_t bits, bool sign);
 
   bool Promote(Scalar::Type type);
 
@@ -185,7 +188,7 @@ class Scalar {
 
   unsigned char UChar(unsigned char fail_value = 0) const;
 
-  signed char SChar(char fail_value = 0) const;
+  signed char SChar(signed char fail_value = 0) const;
 
   unsigned short UShort(unsigned short fail_value = 0) const;
 
@@ -201,7 +204,7 @@ class Scalar {
 
   unsigned long long ULongLong(unsigned long long fail_value = 0) const;
 
-  llvm::APInt SInt128(llvm::APInt &fail_value) const;
+  llvm::APInt SInt128(const llvm::APInt &fail_value) const;
 
   llvm::APInt UInt128(const llvm::APInt &fail_value) const;
 
@@ -264,6 +267,8 @@ class Scalar {
   llvm::APInt m_integer;
   llvm::APFloat m_float;
 
+  template <typename T> T GetAs(T fail_value) const;
+
 private:
   friend const Scalar operator+(const Scalar &lhs, const Scalar &rhs);
   friend const Scalar operator-(const Scalar &lhs, const Scalar &rhs);
diff --git a/lldb/packages/Python/lldbsuite/test/configuration.py b/lldb/packages/Python/lldbsuite/test/configuration.py
index f05152253c75f..ca27864463002 100644
--- a/lldb/packages/Python/lldbsuite/test/configuration.py
+++ b/lldb/packages/Python/lldbsuite/test/configuration.py
@@ -57,6 +57,9 @@
 # Path to the FileCheck testing tool. Not optional.
 filecheck = None
 
+# Path to the yaml2obj tool. Not optional.
+yaml2obj = None
+
 # The arch might dictate some specific CFLAGS to be passed to the toolchain to build
 # the inferior programs.  The global variable cflags_extras provides a hook to do
 # just that.
@@ -163,6 +166,13 @@ def get_filecheck_path():
     if filecheck and os.path.lexists(filecheck):
         return filecheck
 
+def get_yaml2obj_path():
+    """
+    Get the path to the yaml2obj tool.
+    """
+    if yaml2obj and os.path.lexists(yaml2obj):
+        return yaml2obj
+
 def is_reproducer_replay():
     """
     Returns true when dotest is being replayed from a reproducer. Never use
diff --git a/lldb/packages/Python/lldbsuite/test/decorators.py b/lldb/packages/Python/lldbsuite/test/decorators.py
index 0ef2f931542c6..ec084184cd656 100644
--- a/lldb/packages/Python/lldbsuite/test/decorators.py
+++ b/lldb/packages/Python/lldbsuite/test/decorators.py
@@ -411,14 +411,15 @@ def expectedFailureOS(
         debug_info=debug_info)
 
 
-def expectedFailureDarwin(bugnumber=None, compilers=None, debug_info=None):
+def expectedFailureDarwin(bugnumber=None, compilers=None, debug_info=None, archs=None):
     # For legacy reasons, we support both "darwin" and "macosx" as OS X
     # triples.
     return expectedFailureOS(
         lldbplatform.darwin_all,
         bugnumber,
         compilers,
-        debug_info=debug_info)
+        debug_info=debug_info,
+        archs=archs)
 
 
 def expectedFailureAndroid(bugnumber=None, api_levels=None, archs=None):
diff --git a/lldb/packages/Python/lldbsuite/test/dotest.py b/lldb/packages/Python/lldbsuite/test/dotest.py
index 2b8185a18b77e..8238168d0fb6d 100644
--- a/lldb/packages/Python/lldbsuite/test/dotest.py
+++ b/lldb/packages/Python/lldbsuite/test/dotest.py
@@ -272,13 +272,17 @@ def parseOptionsAndInitTestdirs():
         configuration.dsymutil = seven.get_command_output(
             'xcrun -find -toolchain default dsymutil')
 
+
+    # The lldb-dotest script produced by the CMake build passes in a path to a
+    # working FileCheck and yaml2obj binary. So does one specific Xcode
+    # project target. However, when invoking dotest.py directly, a valid
+    # --filecheck and --yaml2obj option needs to be given.
     if args.filecheck:
-        # The lldb-dotest script produced by the CMake build passes in a path
-        # to a working FileCheck binary. So does one specific Xcode project
-        # target. However, when invoking dotest.py directly, a valid --filecheck
-        # option needs to be given.
         configuration.filecheck = os.path.abspath(args.filecheck)
 
+    if args.yaml2obj:
+        configuration.yaml2obj = os.path.abspath(args.yaml2obj)
+
     if not configuration.get_filecheck_path():
         logging.warning('No valid FileCheck executable; some tests may fail...')
         logging.warning('(Double-check the --filecheck argument to dotest.py)')
@@ -363,7 +367,10 @@ def parseOptionsAndInitTestdirs():
                     args.executable)
             sys.exit(-1)
 
-    if args.server:
+    if args.server and args.out_of_tree_debugserver:
+        logging.warning('Both --server and --out-of-tree-debugserver are set')
+
+    if args.server and not args.out_of_tree_debugserver:
         os.environ['LLDB_DEBUGSERVER_PATH'] = args.server
 
     if args.excluded:
diff --git a/lldb/packages/Python/lldbsuite/test/dotest_args.py b/lldb/packages/Python/lldbsuite/test/dotest_args.py
index ff2ac5a47ea50..d6f59efdf28b4 100644
--- a/lldb/packages/Python/lldbsuite/test/dotest_args.py
+++ b/lldb/packages/Python/lldbsuite/test/dotest_args.py
@@ -51,7 +51,7 @@ def create_parser():
                                                            suggestions: do not lump the "-A arch1 -A arch2" together such that the -E option applies to only one of the architectures'''))
 
     group.add_argument('--dsymutil', metavar='dsymutil', dest='dsymutil', help=textwrap.dedent('Specify which dsymutil to use.'))
-
+    group.add_argument('--yaml2obj', metavar='yaml2obj', dest='yaml2obj', help=textwrap.dedent('Specify which yaml2obj binary to use.'))
     group.add_argument('--filecheck', metavar='filecheck', dest='filecheck', help=textwrap.dedent('Specify which FileCheck binary to use.'))
 
     # Test filtering options
diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py
index 481c236712282..29561d4794beb 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbtest.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py
@@ -1633,20 +1633,6 @@ def findBuiltClang(self):
 
         return os.environ["CC"]
 
-    def findYaml2obj(self):
-        """
-        Get the path to the yaml2obj executable, which can be used to create
-        test object files from easy to write yaml instructions.
-
-        Throws an Exception if the executable cannot be found.
-        """
-        # Tries to find yaml2obj at the same folder as clang
-        clang_dir = os.path.dirname(self.findBuiltClang())
-        path = distutils.spawn.find_executable("yaml2obj", clang_dir)
-        if path is not None:
-            return path
-        raise Exception("yaml2obj executable not found")
-
 
     def yaml2obj(self, yaml_path, obj_path):
         """
@@ -1654,8 +1640,10 @@ def yaml2obj(self, yaml_path, obj_path):
 
         Throws subprocess.CalledProcessError if the object could not be created.
         """
-        yaml2obj = self.findYaml2obj()
-        command = [yaml2obj, "-o=%s" % obj_path, yaml_path]
+        yaml2obj_bin = configuration.get_yaml2obj_path()
+        if not yaml2obj_bin:
+            self.assertTrue(False, "No valid FileCheck executable specified")
+        command = [yaml2obj_bin, "-o=%s" % obj_path, yaml_path]
         system([command])
 
     def getBuildFlags(
@@ -2476,14 +2464,12 @@ def expect_expr(
         options.SetIgnoreBreakpoints(True)
 
         if self.frame().IsValid():
-          options.SetLanguage(frame.GuessLanguage())
-          eval_result = self.frame().EvaluateExpression(expr, options)
+            options.SetLanguage(frame.GuessLanguage())
+            eval_result = self.frame().EvaluateExpression(expr, options)
         else:
-          eval_result = self.target().EvaluateExpression(expr, options)
+            eval_result = self.target().EvaluateExpression(expr, options)
 
-        if not eval_result.GetError().Success():
-            self.assertTrue(eval_result.GetError().Success(),
-                "Unexpected failure with msg: " + eval_result.GetError().GetCString())
+        self.assertSuccess(eval_result.GetError())
 
         if result_type:
             self.assertEqual(result_type, eval_result.GetDisplayTypeName())
@@ -2535,6 +2521,13 @@ def run_platform_command(self, cmd):
         err = platform.Run(shell_command)
         return (err, shell_command.GetStatus(), shell_command.GetOutput())
 
+    """Assert that an lldb.SBError is in the "success" state."""
+    def assertSuccess(self, obj, msg=None):
+        if not obj.Success():
+            error = obj.GetCString()
+            self.fail(self._formatMessage(msg,
+                "'{}' is not success".format(error)))
+
     # =================================================
     # Misc. helper methods for debugging test execution
     # =================================================
diff --git a/lldb/packages/Python/lldbsuite/test/lldbutil.py b/lldb/packages/Python/lldbsuite/test/lldbutil.py
index ef0df90c416ea..25fcd5f29ee21 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbutil.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbutil.py
@@ -775,7 +775,7 @@ def run_to_breakpoint_do_run(test, target, bkpt, launch_info = None,
         launch_info = target.GetLaunchInfo()
         launch_info.SetWorkingDirectory(test.get_process_working_directory())
 
-    if extra_images and lldb.remote_platform:
+    if extra_images:
         environ = test.registerSharedLibrariesWithTarget(target, extra_images)
         launch_info.SetEnvironmentEntries(environ, True)
 
@@ -788,6 +788,8 @@ def run_to_breakpoint_do_run(test, target, bkpt, launch_info = None,
     test.assertFalse(error.Fail(),
                      "Process launch failed: %s" % (error.GetCString()))
 
+    test.assertEqual(process.GetState(), lldb.eStateStopped)
+
     # Frame #0 should be at our breakpoint.
     threads = get_threads_stopped_at_breakpoint(
                 process, bkpt)
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py
index 790628d2b0fde..676e08d5a38cc 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py
@@ -179,6 +179,9 @@ def get_stdout(self, timeout=0.0):
     def get_console(self, timeout=0.0):
         return self.vscode.get_output('console', timeout=timeout)
 
+    def collect_console(self, duration):
+        return self.vscode.collect_output('console', duration=duration)
+
     def get_local_as_int(self, name, threadId=None):
         value = self.vscode.get_local_variable_value(name, threadId=threadId)
         if value.startswith('0x'):
@@ -239,14 +242,16 @@ def continue_to_exit(self, exitCode=0):
 
     def attach(self, program=None, pid=None, waitFor=None, trace=None,
                initCommands=None, preRunCommands=None, stopCommands=None,
-               exitCommands=None, attachCommands=None, coreFile=None):
+               exitCommands=None, attachCommands=None, coreFile=None,
+               disconnectAutomatically=True, terminateCommands=None):
         '''Build the default Makefile target, create the VSCode debug adaptor,
            and attach to the process.
         '''
         # Make sure we disconnect and terminate the VSCode debug adaptor even
         # if we throw an exception during the test case.
         def cleanup():
-            self.vscode.request_disconnect(terminateDebuggee=True)
+            if disconnectAutomatically:
+                self.vscode.request_disconnect(terminateDebuggee=True)
             self.vscode.terminate()
 
         # Execute the cleanup function during test case tear down.
@@ -257,7 +262,8 @@ def cleanup():
             program=program, pid=pid, waitFor=waitFor, trace=trace,
             initCommands=initCommands, preRunCommands=preRunCommands,
             stopCommands=stopCommands, exitCommands=exitCommands,
-            attachCommands=attachCommands, coreFile=coreFile)
+            attachCommands=attachCommands, terminateCommands=terminateCommands,
+            coreFile=coreFile)
         if not (response and response['success']):
             self.assertTrue(response['success'],
                             'attach failed (%s)' % (response['message']))
@@ -266,15 +272,17 @@ def launch(self, program=None, args=None, cwd=None, env=None,
                stopOnEntry=False, disableASLR=True,
                disableSTDIO=False, shellExpandArguments=False,
                trace=False, initCommands=None, preRunCommands=None,
-               stopCommands=None, exitCommands=None,sourcePath=None,
-               debuggerRoot=None, launchCommands=None, sourceMap=None):
+               stopCommands=None, exitCommands=None, terminateCommands=None,
+               sourcePath=None, debuggerRoot=None, launchCommands=None,
+               sourceMap=None, disconnectAutomatically=True):
         '''Sending launch request to vscode
         '''
 
         # Make sure we disconnect and terminate the VSCode debug adapter,
         # if we throw an exception during the test case
         def cleanup():
-            self.vscode.request_disconnect(terminateDebuggee=True)
+            if disconnectAutomatically:
+                self.vscode.request_disconnect(terminateDebuggee=True)
             self.vscode.terminate()
 
         # Execute the cleanup function during test case tear down.
@@ -296,6 +304,7 @@ def cleanup():
             preRunCommands=preRunCommands,
             stopCommands=stopCommands,
             exitCommands=exitCommands,
+            terminateCommands=terminateCommands,
             sourcePath=sourcePath,
             debuggerRoot=debuggerRoot,
             launchCommands=launchCommands,
@@ -309,7 +318,8 @@ def build_and_launch(self, program, args=None, cwd=None, env=None,
                          disableSTDIO=False, shellExpandArguments=False,
                          trace=False, initCommands=None, preRunCommands=None,
                          stopCommands=None, exitCommands=None,
-                         sourcePath=None, debuggerRoot=None):
+                         terminateCommands=None, sourcePath=None,
+                         debuggerRoot=None):
         '''Build the default Makefile target, create the VSCode debug adaptor,
            and launch the process.
         '''
@@ -319,4 +329,4 @@ def build_and_launch(self, program, args=None, cwd=None, env=None,
         self.launch(program, args, cwd, env, stopOnEntry, disableASLR,
                     disableSTDIO, shellExpandArguments, trace,
                     initCommands, preRunCommands, stopCommands, exitCommands,
-                    sourcePath, debuggerRoot)
+                    terminateCommands, sourcePath, debuggerRoot)
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/vscode.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/vscode.py
index 643a313dca1cb..1ad168e794cff 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/vscode.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/vscode.py
@@ -10,6 +10,7 @@
 import subprocess
 import sys
 import threading
+import time
 
 
 def dump_memory(base_addr, data, num_per_line, outfile):
@@ -148,6 +149,15 @@ def get_output(self, category, timeout=0.0, clear=True):
         self.output_condition.release()
         return output
 
+    def collect_output(self, category, duration, clear=True):
+        end_time = time.time() + duration
+        collected_output = ""
+        while end_time > time.time():
+            output = self.get_output(category, timeout=0.25, clear=clear)
+            if output:
+                collected_output += output
+        return collected_output if collected_output else None
+
     def enqueue_recv_packet(self, packet):
         self.recv_condition.acquire()
         self.recv_packets.append(packet)
@@ -450,7 +460,8 @@ def replay_packets(self, replay_file_path):
     def request_attach(self, program=None, pid=None, waitFor=None, trace=None,
                        initCommands=None, preRunCommands=None,
                        stopCommands=None, exitCommands=None,
-                       attachCommands=None, coreFile=None):
+                       attachCommands=None, terminateCommands=None,
+                       coreFile=None):
         args_dict = {}
         if pid is not None:
             args_dict['pid'] = pid
@@ -469,6 +480,8 @@ def request_attach(self, program=None, pid=None, waitFor=None, trace=None,
             args_dict['stopCommands'] = stopCommands
         if exitCommands:
             args_dict['exitCommands'] = exitCommands
+        if terminateCommands:
+            args_dict['terminateCommands'] = terminateCommands
         if attachCommands:
             args_dict['attachCommands'] = attachCommands
         if coreFile:
@@ -571,7 +584,8 @@ def request_launch(self, program, args=None, cwd=None, env=None,
                        stopOnEntry=False, disableASLR=True,
                        disableSTDIO=False, shellExpandArguments=False,
                        trace=False, initCommands=None, preRunCommands=None,
-                       stopCommands=None, exitCommands=None, sourcePath=None,
+                       stopCommands=None, exitCommands=None,
+                       terminateCommands=None ,sourcePath=None,
                        debuggerRoot=None, launchCommands=None, sourceMap=None):
         args_dict = {
             'program': program
@@ -601,6 +615,8 @@ def request_launch(self, program, args=None, cwd=None, env=None,
             args_dict['stopCommands'] = stopCommands
         if exitCommands:
             args_dict['exitCommands'] = exitCommands
+        if terminateCommands:
+            args_dict['terminateCommands'] = terminateCommands
         if sourcePath:
             args_dict['sourcePath'] = sourcePath
         if debuggerRoot:
@@ -905,7 +921,8 @@ def run_vscode(dbg, args, options):
                                       initCommands=options.initCmds,
                                       preRunCommands=options.preRunCmds,
                                       stopCommands=options.stopCmds,
-                                      exitCommands=options.exitCmds)
+                                      exitCommands=options.exitCmds,
+                                      terminateCommands=options.terminateCmds)
     else:
         response = dbg.request_launch(options.program,
                                       args=args,
@@ -916,7 +933,8 @@ def run_vscode(dbg, args, options):
                                       initCommands=options.initCmds,
                                       preRunCommands=options.preRunCmds,
                                       stopCommands=options.stopCmds,
-                                      exitCommands=options.exitCmds)
+                                      exitCommands=options.exitCmds,
+                                      terminateCommands=options.terminateCmds)
 
     if response['success']:
         if options.sourceBreakpoints:
@@ -1089,6 +1107,15 @@ def main():
         help=('Specify a LLDB command that will be executed when the process '
               'exits. Can be specified more than once.'))
 
+    parser.add_option(
+        '--terminateCommand',
+        type='string',
+        action='append',
+        dest='terminateCmds',
+        default=[],
+        help=('Specify a LLDB command that will be executed when the debugging '
+              'session is terminated. Can be specified more than once.'))
+
     parser.add_option(
         '--env',
         type='string',
diff --git a/lldb/scripts/analyze-project-deps.py b/lldb/scripts/analyze-project-deps.py
index c6e3263a2f26a..a120260abfe2c 100755
--- a/lldb/scripts/analyze-project-deps.py
+++ b/lldb/scripts/analyze-project-deps.py
@@ -109,7 +109,7 @@ def expand(path_queue, path_lengths, cycles, src_map):
         next_len = path_lengths.pop(0) + 1
         last_component = cur_path[-1]
 
-        for item in src_map[last_component]:
+        for item in src_map.get(last_component, []):
             if item.startswith("clang"):
                 continue
 
diff --git a/lldb/source/API/SBAddress.cpp b/lldb/source/API/SBAddress.cpp
index d2b9b80973c8e..6444a006c0ff4 100644
--- a/lldb/source/API/SBAddress.cpp
+++ b/lldb/source/API/SBAddress.cpp
@@ -89,7 +89,7 @@ SBAddress::operator bool() const {
 void SBAddress::Clear() {
   LLDB_RECORD_METHOD_NO_ARGS(void, SBAddress, Clear);
 
-  m_opaque_up.reset(new Address());
+  m_opaque_up = std::make_unique<Address>();
 }
 
 void SBAddress::SetAddress(lldb::SBSection section, lldb::addr_t offset) {
@@ -105,7 +105,7 @@ void SBAddress::SetAddress(const Address *lldb_object_ptr) {
   if (lldb_object_ptr)
     ref() = *lldb_object_ptr;
   else
-    m_opaque_up.reset(new Address());
+    m_opaque_up = std::make_unique<Address>();
 }
 
 lldb::addr_t SBAddress::GetFileAddress() const {
@@ -187,7 +187,7 @@ const Address *SBAddress::operator->() const { return m_opaque_up.get(); }
 
 Address &SBAddress::ref() {
   if (m_opaque_up == nullptr)
-    m_opaque_up.reset(new Address());
+    m_opaque_up = std::make_unique<Address>();
   return *m_opaque_up;
 }
 
diff --git a/lldb/source/API/SBBreakpoint.cpp b/lldb/source/API/SBBreakpoint.cpp
index 283dd7ea82538..eb75bf8b33f43 100644
--- a/lldb/source/API/SBBreakpoint.cpp
+++ b/lldb/source/API/SBBreakpoint.cpp
@@ -652,19 +652,28 @@ SBError SBBreakpoint::SetScriptCallbackBody(const char *callback_body_text) {
 bool SBBreakpoint::AddName(const char *new_name) {
   LLDB_RECORD_METHOD(bool, SBBreakpoint, AddName, (const char *), new_name);
 
+  SBError status = AddNameWithErrorHandling(new_name);
+  return status.Success();
+}
+
+SBError SBBreakpoint::AddNameWithErrorHandling(const char *new_name) {
+  LLDB_RECORD_METHOD(SBError, SBBreakpoint, AddNameWithErrorHandling,
+                     (const char *), new_name);
+
   BreakpointSP bkpt_sp = GetSP();
 
+  SBError status;
   if (bkpt_sp) {
     std::lock_guard<std::recursive_mutex> guard(
         bkpt_sp->GetTarget().GetAPIMutex());
-    Status error; // Think I'm just going to swallow the error here, it's
-                  // probably more annoying to have to provide it.
+    Status error;
     bkpt_sp->GetTarget().AddNameToBreakpoint(bkpt_sp, new_name, error);
-    if (error.Fail())
-      return false;
+    status.SetError(error);
+  } else {
+    status.SetErrorString("invalid breakpoint");
   }
 
-  return true;
+  return LLDB_RECORD_RESULT(status);
 }
 
 void SBBreakpoint::RemoveName(const char *name_to_remove) {
@@ -1015,6 +1024,8 @@ void RegisterMethods<SBBreakpoint>(Registry &R) {
   LLDB_REGISTER_METHOD(lldb::SBError, SBBreakpoint, SetScriptCallbackBody,
                        (const char *));
   LLDB_REGISTER_METHOD(bool, SBBreakpoint, AddName, (const char *));
+  LLDB_REGISTER_METHOD(lldb::SBError, SBBreakpoint, AddNameWithErrorHandling,
+                       (const char *));
   LLDB_REGISTER_METHOD(void, SBBreakpoint, RemoveName, (const char *));
   LLDB_REGISTER_METHOD(bool, SBBreakpoint, MatchesName, (const char *));
   LLDB_REGISTER_METHOD(void, SBBreakpoint, GetNames, (lldb::SBStringList &));
diff --git a/lldb/source/API/SBBreakpointName.cpp b/lldb/source/API/SBBreakpointName.cpp
index e5a0dbc9ef8ef..3995defcf97c2 100644
--- a/lldb/source/API/SBBreakpointName.cpp
+++ b/lldb/source/API/SBBreakpointName.cpp
@@ -115,7 +115,7 @@ SBBreakpointName::SBBreakpointName(SBTarget &sb_target, const char *name) {
   LLDB_RECORD_CONSTRUCTOR(SBBreakpointName, (lldb::SBTarget &, const char *),
                           sb_target, name);
 
-  m_impl_up.reset(new SBBreakpointNameImpl(sb_target, name));
+  m_impl_up = std::make_unique<SBBreakpointNameImpl>(sb_target, name);
   // Call FindBreakpointName here to make sure the name is valid, reset if not:
   BreakpointName *bp_name = GetBreakpointName();
   if (!bp_name)
@@ -133,7 +133,8 @@ SBBreakpointName::SBBreakpointName(SBBreakpoint &sb_bkpt, const char *name) {
   BreakpointSP bkpt_sp = sb_bkpt.GetSP();
   Target &target = bkpt_sp->GetTarget();
 
-  m_impl_up.reset(new SBBreakpointNameImpl(target.shared_from_this(), name));
+  m_impl_up =
+      std::make_unique<SBBreakpointNameImpl>(target.shared_from_this(), name);
 
   // Call FindBreakpointName here to make sure the name is valid, reset if not:
   BreakpointName *bp_name = GetBreakpointName();
@@ -154,8 +155,8 @@ SBBreakpointName::SBBreakpointName(const SBBreakpointName &rhs) {
   if (!rhs.m_impl_up)
     return;
   else
-    m_impl_up.reset(new SBBreakpointNameImpl(rhs.m_impl_up->GetTarget(),
-                                             rhs.m_impl_up->GetName()));
+    m_impl_up = std::make_unique<SBBreakpointNameImpl>(
+        rhs.m_impl_up->GetTarget(), rhs.m_impl_up->GetName());
 }
 
 SBBreakpointName::~SBBreakpointName() = default;
@@ -171,8 +172,8 @@ operator=(const SBBreakpointName &rhs) {
     return LLDB_RECORD_RESULT(*this);
   }
 
-  m_impl_up.reset(new SBBreakpointNameImpl(rhs.m_impl_up->GetTarget(),
-                                           rhs.m_impl_up->GetName()));
+  m_impl_up = std::make_unique<SBBreakpointNameImpl>(rhs.m_impl_up->GetTarget(),
+                                                     rhs.m_impl_up->GetName());
   return LLDB_RECORD_RESULT(*this);
 }
 
diff --git a/lldb/source/API/SBCommandInterpreter.cpp b/lldb/source/API/SBCommandInterpreter.cpp
index d06b33c760fcd..f4f19577b36c5 100644
--- a/lldb/source/API/SBCommandInterpreter.cpp
+++ b/lldb/source/API/SBCommandInterpreter.cpp
@@ -69,7 +69,7 @@ class CommandPluginInterfaceImplementation : public CommandObjectParsed {
     SBCommandInterpreter sb_interpreter(&m_interpreter);
     SBDebugger debugger_sb(m_interpreter.GetDebugger().shared_from_this());
     bool ret = m_backend->DoExecute(
-        debugger_sb, (char **)command.GetArgumentVector(), sb_return);
+        debugger_sb, command.GetArgumentVector(), sb_return);
     return ret;
   }
   std::shared_ptr<lldb::SBCommandPluginInterface> m_backend;
diff --git a/lldb/source/API/SBCommandInterpreterRunOptions.cpp b/lldb/source/API/SBCommandInterpreterRunOptions.cpp
index 7f880dc4605b2..fcfbf5e5401ae 100644
--- a/lldb/source/API/SBCommandInterpreterRunOptions.cpp
+++ b/lldb/source/API/SBCommandInterpreterRunOptions.cpp
@@ -21,7 +21,7 @@ using namespace lldb_private;
 SBCommandInterpreterRunOptions::SBCommandInterpreterRunOptions() {
   LLDB_RECORD_CONSTRUCTOR_NO_ARGS(SBCommandInterpreterRunOptions);
 
-  m_opaque_up.reset(new CommandInterpreterRunOptions());
+  m_opaque_up = std::make_unique<CommandInterpreterRunOptions>();
 }
 
 SBCommandInterpreterRunOptions::~SBCommandInterpreterRunOptions() = default;
@@ -182,7 +182,7 @@ SBCommandInterpreterRunResult::SBCommandInterpreterRunResult(
 SBCommandInterpreterRunResult::SBCommandInterpreterRunResult(
     const CommandInterpreterRunResult &rhs)
     : m_opaque_up() {
-  m_opaque_up.reset(new CommandInterpreterRunResult(rhs));
+  m_opaque_up = std::make_unique<CommandInterpreterRunResult>(rhs);
 }
 
 SBCommandInterpreterRunResult::~SBCommandInterpreterRunResult() = default;
diff --git a/lldb/source/API/SBDebugger.cpp b/lldb/source/API/SBDebugger.cpp
index 978d0befa5ff5..5f62987f37dad 100644
--- a/lldb/source/API/SBDebugger.cpp
+++ b/lldb/source/API/SBDebugger.cpp
@@ -313,7 +313,7 @@ SBError SBDebugger::SetInputFile(SBFile file) {
 
   repro::DataRecorder *recorder = nullptr;
   if (repro::Generator *g = repro::Reproducer::Instance().GetGenerator())
-    recorder = g->GetOrCreate<repro::CommandProvider>().GetNewDataRecorder();
+    recorder = g->GetOrCreate<repro::CommandProvider>().GetNewRecorder();
 
   FileSP file_sp = file.m_opaque_sp;
 
diff --git a/lldb/source/API/SBDeclaration.cpp b/lldb/source/API/SBDeclaration.cpp
index 0377efc338a41..f1066d63c06ab 100644
--- a/lldb/source/API/SBDeclaration.cpp
+++ b/lldb/source/API/SBDeclaration.cpp
@@ -148,7 +148,7 @@ const lldb_private::Declaration *SBDeclaration::operator->() const {
 
 lldb_private::Declaration &SBDeclaration::ref() {
   if (m_opaque_up == nullptr)
-    m_opaque_up.reset(new lldb_private::Declaration());
+    m_opaque_up = std::make_unique<lldb_private::Declaration>();
   return *m_opaque_up;
 }
 
diff --git a/lldb/source/API/SBError.cpp b/lldb/source/API/SBError.cpp
index 9efabe40ce93c..67c7663d35831 100644
--- a/lldb/source/API/SBError.cpp
+++ b/lldb/source/API/SBError.cpp
@@ -149,7 +149,7 @@ SBError::operator bool() const {
 
 void SBError::CreateIfNeeded() {
   if (m_opaque_up == nullptr)
-    m_opaque_up.reset(new Status());
+    m_opaque_up = std::make_unique<Status>();
 }
 
 lldb_private::Status *SBError::operator->() { return m_opaque_up.get(); }
diff --git a/lldb/source/API/SBFrame.cpp b/lldb/source/API/SBFrame.cpp
index 5ecf1c537536e..81782dbf838f3 100644
--- a/lldb/source/API/SBFrame.cpp
+++ b/lldb/source/API/SBFrame.cpp
@@ -354,15 +354,15 @@ bool SBFrame::SetPC(addr_t new_pc) {
   std::unique_lock<std::recursive_mutex> lock;
   ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
 
-  StackFrame *frame = nullptr;
   Target *target = exe_ctx.GetTargetPtr();
   Process *process = exe_ctx.GetProcessPtr();
   if (target && process) {
     Process::StopLocker stop_locker;
     if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
-        ret_val = frame->GetRegisterContext()->SetPC(new_pc);
+      if (StackFrame *frame = exe_ctx.GetFramePtr()) {
+        if (RegisterContextSP reg_ctx_sp = frame->GetRegisterContext()) {
+          ret_val = reg_ctx_sp->SetPC(new_pc);
+        }
       }
     }
   }
@@ -377,15 +377,15 @@ addr_t SBFrame::GetSP() const {
   std::unique_lock<std::recursive_mutex> lock;
   ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
 
-  StackFrame *frame = nullptr;
   Target *target = exe_ctx.GetTargetPtr();
   Process *process = exe_ctx.GetProcessPtr();
   if (target && process) {
     Process::StopLocker stop_locker;
     if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
-        addr = frame->GetRegisterContext()->GetSP();
+      if (StackFrame *frame = exe_ctx.GetFramePtr()) {
+        if (RegisterContextSP reg_ctx_sp = frame->GetRegisterContext()) {
+          addr = reg_ctx_sp->GetSP();
+        }
       }
     }
   }
@@ -400,15 +400,16 @@ addr_t SBFrame::GetFP() const {
   std::unique_lock<std::recursive_mutex> lock;
   ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
 
-  StackFrame *frame = nullptr;
   Target *target = exe_ctx.GetTargetPtr();
   Process *process = exe_ctx.GetProcessPtr();
   if (target && process) {
     Process::StopLocker stop_locker;
     if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame)
-        addr = frame->GetRegisterContext()->GetFP();
+      if (StackFrame *frame = exe_ctx.GetFramePtr()) {
+        if (RegisterContextSP reg_ctx_sp = frame->GetRegisterContext()) {
+          addr = reg_ctx_sp->GetFP();
+        }
+      }
     }
   }
 
diff --git a/lldb/source/API/SBLineEntry.cpp b/lldb/source/API/SBLineEntry.cpp
index 44168dfd5eb9a..cefbe3ee1a1eb 100644
--- a/lldb/source/API/SBLineEntry.cpp
+++ b/lldb/source/API/SBLineEntry.cpp
@@ -163,7 +163,7 @@ const lldb_private::LineEntry *SBLineEntry::operator->() const {
 
 lldb_private::LineEntry &SBLineEntry::ref() {
   if (m_opaque_up == nullptr)
-    m_opaque_up.reset(new lldb_private::LineEntry());
+    m_opaque_up = std::make_unique<lldb_private::LineEntry>();
   return *m_opaque_up;
 }
 
diff --git a/lldb/source/API/SBProcessInfo.cpp b/lldb/source/API/SBProcessInfo.cpp
index aa97b5af4d207..29a9c7b24b5a7 100644
--- a/lldb/source/API/SBProcessInfo.cpp
+++ b/lldb/source/API/SBProcessInfo.cpp
@@ -39,7 +39,7 @@ SBProcessInfo &SBProcessInfo::operator=(const SBProcessInfo &rhs) {
 
 ProcessInstanceInfo &SBProcessInfo::ref() {
   if (m_opaque_up == nullptr) {
-    m_opaque_up.reset(new ProcessInstanceInfo());
+    m_opaque_up = std::make_unique<ProcessInstanceInfo>();
   }
   return *m_opaque_up;
 }
diff --git a/lldb/source/API/SBSourceManager.cpp b/lldb/source/API/SBSourceManager.cpp
index 6fe3dac7767dc..43c3443672f73 100644
--- a/lldb/source/API/SBSourceManager.cpp
+++ b/lldb/source/API/SBSourceManager.cpp
@@ -75,13 +75,13 @@ SBSourceManager::SBSourceManager(const SBDebugger &debugger) {
   LLDB_RECORD_CONSTRUCTOR(SBSourceManager, (const lldb::SBDebugger &),
                           debugger);
 
-  m_opaque_up.reset(new SourceManagerImpl(debugger.get_sp()));
+  m_opaque_up = std::make_unique<SourceManagerImpl>(debugger.get_sp());
 }
 
 SBSourceManager::SBSourceManager(const SBTarget &target) {
   LLDB_RECORD_CONSTRUCTOR(SBSourceManager, (const lldb::SBTarget &), target);
 
-  m_opaque_up.reset(new SourceManagerImpl(target.GetSP()));
+  m_opaque_up = std::make_unique<SourceManagerImpl>(target.GetSP());
 }
 
 SBSourceManager::SBSourceManager(const SBSourceManager &rhs) {
@@ -91,7 +91,7 @@ SBSourceManager::SBSourceManager(const SBSourceManager &rhs) {
   if (&rhs == this)
     return;
 
-  m_opaque_up.reset(new SourceManagerImpl(*(rhs.m_opaque_up.get())));
+  m_opaque_up = std::make_unique<SourceManagerImpl>(*(rhs.m_opaque_up.get()));
 }
 
 const lldb::SBSourceManager &SBSourceManager::
@@ -100,7 +100,7 @@ operator=(const lldb::SBSourceManager &rhs) {
                      SBSourceManager, operator=,(const lldb::SBSourceManager &),
                      rhs);
 
-  m_opaque_up.reset(new SourceManagerImpl(*(rhs.m_opaque_up.get())));
+  m_opaque_up = std::make_unique<SourceManagerImpl>(*(rhs.m_opaque_up.get()));
   return LLDB_RECORD_RESULT(*this);
 }
 
diff --git a/lldb/source/API/SBStream.cpp b/lldb/source/API/SBStream.cpp
index 0f49c5111f289..eb81153084e8c 100644
--- a/lldb/source/API/SBStream.cpp
+++ b/lldb/source/API/SBStream.cpp
@@ -177,7 +177,7 @@ lldb_private::Stream *SBStream::get() { return m_opaque_up.get(); }
 
 lldb_private::Stream &SBStream::ref() {
   if (m_opaque_up == nullptr)
-    m_opaque_up.reset(new StreamString());
+    m_opaque_up = std::make_unique<StreamString>();
   return *m_opaque_up;
 }
 
diff --git a/lldb/source/API/SBStringList.cpp b/lldb/source/API/SBStringList.cpp
index 5364a4d61c9f9..d9b03692ec0e4 100644
--- a/lldb/source/API/SBStringList.cpp
+++ b/lldb/source/API/SBStringList.cpp
@@ -66,7 +66,7 @@ void SBStringList::AppendString(const char *str) {
     if (IsValid())
       m_opaque_up->AppendString(str);
     else
-      m_opaque_up.reset(new lldb_private::StringList(str));
+      m_opaque_up = std::make_unique<lldb_private::StringList>(str);
   }
 }
 
@@ -78,7 +78,7 @@ void SBStringList::AppendList(const char **strv, int strc) {
     if (IsValid())
       m_opaque_up->AppendList(strv, strc);
     else
-      m_opaque_up.reset(new lldb_private::StringList(strv, strc));
+      m_opaque_up = std::make_unique<lldb_private::StringList>(strv, strc);
   }
 }
 
@@ -88,14 +88,14 @@ void SBStringList::AppendList(const SBStringList &strings) {
 
   if (strings.IsValid()) {
     if (!IsValid())
-      m_opaque_up.reset(new lldb_private::StringList());
+      m_opaque_up = std::make_unique<lldb_private::StringList>();
     m_opaque_up->AppendList(*(strings.m_opaque_up));
   }
 }
 
 void SBStringList::AppendList(const StringList &strings) {
   if (!IsValid())
-    m_opaque_up.reset(new lldb_private::StringList());
+    m_opaque_up = std::make_unique<lldb_private::StringList>();
   m_opaque_up->AppendList(strings);
 }
 
diff --git a/lldb/source/API/SBSymbolContext.cpp b/lldb/source/API/SBSymbolContext.cpp
index 4878bfe2e32ac..488d498849038 100644
--- a/lldb/source/API/SBSymbolContext.cpp
+++ b/lldb/source/API/SBSymbolContext.cpp
@@ -185,13 +185,13 @@ const lldb_private::SymbolContext &SBSymbolContext::operator*() const {
 
 lldb_private::SymbolContext &SBSymbolContext::operator*() {
   if (m_opaque_up == nullptr)
-    m_opaque_up.reset(new SymbolContext);
+    m_opaque_up = std::make_unique<SymbolContext>();
   return *m_opaque_up;
 }
 
 lldb_private::SymbolContext &SBSymbolContext::ref() {
   if (m_opaque_up == nullptr)
-    m_opaque_up.reset(new SymbolContext);
+    m_opaque_up = std::make_unique<SymbolContext>();
   return *m_opaque_up;
 }
 
diff --git a/lldb/source/API/SBType.cpp b/lldb/source/API/SBType.cpp
index 64f9103e411a3..852630f2d01a6 100644
--- a/lldb/source/API/SBType.cpp
+++ b/lldb/source/API/SBType.cpp
@@ -589,7 +589,7 @@ SBTypeList &SBTypeList::operator=(const SBTypeList &rhs) {
                      SBTypeList, operator=,(const lldb::SBTypeList &), rhs);
 
   if (this != &rhs) {
-    m_opaque_up.reset(new TypeListImpl());
+    m_opaque_up = std::make_unique<TypeListImpl>();
     for (uint32_t i = 0, rhs_size = const_cast<SBTypeList &>(rhs).GetSize();
          i < rhs_size; i++)
       Append(const_cast<SBTypeList &>(rhs).GetTypeAtIndex(i));
@@ -632,7 +632,7 @@ SBTypeMember::SBTypeMember(const SBTypeMember &rhs) : m_opaque_up() {
 
   if (this != &rhs) {
     if (rhs.IsValid())
-      m_opaque_up.reset(new TypeMemberImpl(rhs.ref()));
+      m_opaque_up = std::make_unique<TypeMemberImpl>(rhs.ref());
   }
 }
 
@@ -642,7 +642,7 @@ lldb::SBTypeMember &SBTypeMember::operator=(const lldb::SBTypeMember &rhs) {
 
   if (this != &rhs) {
     if (rhs.IsValid())
-      m_opaque_up.reset(new TypeMemberImpl(rhs.ref()));
+      m_opaque_up = std::make_unique<TypeMemberImpl>(rhs.ref());
   }
   return LLDB_RECORD_RESULT(*this);
 }
@@ -746,7 +746,7 @@ void SBTypeMember::reset(TypeMemberImpl *type_member_impl) {
 
 TypeMemberImpl &SBTypeMember::ref() {
   if (m_opaque_up == nullptr)
-    m_opaque_up.reset(new TypeMemberImpl());
+    m_opaque_up = std::make_unique<TypeMemberImpl>();
   return *m_opaque_up;
 }
 
diff --git a/lldb/source/API/SBTypeEnumMember.cpp b/lldb/source/API/SBTypeEnumMember.cpp
index fd95d6e34491e..43a4891b54b10 100644
--- a/lldb/source/API/SBTypeEnumMember.cpp
+++ b/lldb/source/API/SBTypeEnumMember.cpp
@@ -141,7 +141,7 @@ operator=(const SBTypeEnumMemberList &rhs) {
       rhs);
 
   if (this != &rhs) {
-    m_opaque_up.reset(new TypeEnumMemberListImpl());
+    m_opaque_up = std::make_unique<TypeEnumMemberListImpl>();
     for (uint32_t i = 0,
                   rhs_size = const_cast<SBTypeEnumMemberList &>(rhs).GetSize();
          i < rhs_size; i++)
diff --git a/lldb/source/API/SBTypeSummary.cpp b/lldb/source/API/SBTypeSummary.cpp
index e8a6f4c130889..3800ae940c703 100644
--- a/lldb/source/API/SBTypeSummary.cpp
+++ b/lldb/source/API/SBTypeSummary.cpp
@@ -21,7 +21,7 @@ using namespace lldb_private;
 SBTypeSummaryOptions::SBTypeSummaryOptions() {
   LLDB_RECORD_CONSTRUCTOR_NO_ARGS(SBTypeSummaryOptions);
 
-  m_opaque_up.reset(new TypeSummaryOptions());
+  m_opaque_up = std::make_unique<TypeSummaryOptions>();
 }
 
 SBTypeSummaryOptions::SBTypeSummaryOptions(
@@ -111,9 +111,9 @@ SBTypeSummaryOptions::SBTypeSummaryOptions(
 void SBTypeSummaryOptions::SetOptions(
     const lldb_private::TypeSummaryOptions *lldb_object_ptr) {
   if (lldb_object_ptr)
-    m_opaque_up.reset(new TypeSummaryOptions(*lldb_object_ptr));
+    m_opaque_up = std::make_unique<TypeSummaryOptions>(*lldb_object_ptr);
   else
-    m_opaque_up.reset(new TypeSummaryOptions());
+    m_opaque_up = std::make_unique<TypeSummaryOptions>();
 }
 
 SBTypeSummary::SBTypeSummary() : m_opaque_sp() {
diff --git a/lldb/source/API/SBValueList.cpp b/lldb/source/API/SBValueList.cpp
index d8d40b155b618..0fd2a591c321c 100644
--- a/lldb/source/API/SBValueList.cpp
+++ b/lldb/source/API/SBValueList.cpp
@@ -75,12 +75,12 @@ SBValueList::SBValueList(const SBValueList &rhs) : m_opaque_up() {
   LLDB_RECORD_CONSTRUCTOR(SBValueList, (const lldb::SBValueList &), rhs);
 
   if (rhs.IsValid())
-    m_opaque_up.reset(new ValueListImpl(*rhs));
+    m_opaque_up = std::make_unique<ValueListImpl>(*rhs);
 }
 
 SBValueList::SBValueList(const ValueListImpl *lldb_object_ptr) : m_opaque_up() {
   if (lldb_object_ptr)
-    m_opaque_up.reset(new ValueListImpl(*lldb_object_ptr));
+    m_opaque_up = std::make_unique<ValueListImpl>(*lldb_object_ptr);
 }
 
 SBValueList::~SBValueList() = default;
@@ -107,7 +107,7 @@ const SBValueList &SBValueList::operator=(const SBValueList &rhs) {
 
   if (this != &rhs) {
     if (rhs.IsValid())
-      m_opaque_up.reset(new ValueListImpl(*rhs));
+      m_opaque_up = std::make_unique<ValueListImpl>(*rhs);
     else
       m_opaque_up.reset();
   }
@@ -173,7 +173,7 @@ uint32_t SBValueList::GetSize() const {
 
 void SBValueList::CreateIfNeeded() {
   if (m_opaque_up == nullptr)
-    m_opaque_up.reset(new ValueListImpl());
+    m_opaque_up = std::make_unique<ValueListImpl>();
 }
 
 SBValue SBValueList::FindValueObjectByUID(lldb::user_id_t uid) {
diff --git a/lldb/source/API/SBVariablesOptions.cpp b/lldb/source/API/SBVariablesOptions.cpp
index 4156377e36546..4ef16364e6280 100644
--- a/lldb/source/API/SBVariablesOptions.cpp
+++ b/lldb/source/API/SBVariablesOptions.cpp
@@ -97,7 +97,7 @@ operator=(const SBVariablesOptions &options) {
       SBVariablesOptions, operator=,(const lldb::SBVariablesOptions &),
       options);
 
-  m_opaque_up.reset(new VariablesOptionsImpl(options.ref()));
+  m_opaque_up = std::make_unique<VariablesOptionsImpl>(options.ref());
   return LLDB_RECORD_RESULT(*this);
 }
 
diff --git a/lldb/source/Breakpoint/BreakpointID.cpp b/lldb/source/Breakpoint/BreakpointID.cpp
index a8fadbb8b5f3f..293baf4ad1c77 100644
--- a/lldb/source/Breakpoint/BreakpointID.cpp
+++ b/lldb/source/Breakpoint/BreakpointID.cpp
@@ -112,7 +112,7 @@ bool BreakpointID::StringIsBreakpointName(llvm::StringRef str, Status &error) {
   // Cannot contain ., -, or space.
   if (str.find_first_of(".- ") != llvm::StringRef::npos) {
     error.SetErrorStringWithFormat("Breakpoint names cannot contain "
-                                   "'.' or '-': \"%s\"",
+                                   "'.' or '-' or spaces: \"%s\"",
                                    str.str().c_str());
     return false;
   }
diff --git a/lldb/source/Breakpoint/BreakpointLocation.cpp b/lldb/source/Breakpoint/BreakpointLocation.cpp
index 646c2e3c628bd..93d54c051ee5f 100644
--- a/lldb/source/Breakpoint/BreakpointLocation.cpp
+++ b/lldb/source/Breakpoint/BreakpointLocation.cpp
@@ -369,7 +369,7 @@ BreakpointOptions *BreakpointLocation::GetLocationOptions() {
   // potentially expensive and we don't want to do that for the simple case
   // where someone is just disabling the location.
   if (m_options_up == nullptr)
-    m_options_up.reset(new BreakpointOptions(false));
+    m_options_up = std::make_unique<BreakpointOptions>(false);
 
   return m_options_up.get();
 }
diff --git a/lldb/source/Breakpoint/BreakpointOptions.cpp b/lldb/source/Breakpoint/BreakpointOptions.cpp
index 3886d03290628..0ce7b0f852e8c 100644
--- a/lldb/source/Breakpoint/BreakpointOptions.cpp
+++ b/lldb/source/Breakpoint/BreakpointOptions.cpp
@@ -154,7 +154,7 @@ BreakpointOptions::BreakpointOptions(const BreakpointOptions &rhs)
       m_ignore_count(rhs.m_ignore_count), m_thread_spec_up(),
       m_auto_continue(rhs.m_auto_continue), m_set_flags(rhs.m_set_flags) {
   if (rhs.m_thread_spec_up != nullptr)
-    m_thread_spec_up.reset(new ThreadSpec(*rhs.m_thread_spec_up));
+    m_thread_spec_up = std::make_unique<ThreadSpec>(*rhs.m_thread_spec_up);
   m_condition_text = rhs.m_condition_text;
   m_condition_text_hash = rhs.m_condition_text_hash;
 }
@@ -170,7 +170,7 @@ operator=(const BreakpointOptions &rhs) {
   m_one_shot = rhs.m_one_shot;
   m_ignore_count = rhs.m_ignore_count;
   if (rhs.m_thread_spec_up != nullptr)
-    m_thread_spec_up.reset(new ThreadSpec(*rhs.m_thread_spec_up));
+    m_thread_spec_up = std::make_unique<ThreadSpec>(*rhs.m_thread_spec_up);
   m_condition_text = rhs.m_condition_text;
   m_condition_text_hash = rhs.m_condition_text_hash;
   m_auto_continue = rhs.m_auto_continue;
@@ -223,7 +223,8 @@ void BreakpointOptions::CopyOverSetOptions(const BreakpointOptions &incoming)
   }
   if (incoming.m_set_flags.Test(eThreadSpec) && incoming.m_thread_spec_up) {
     if (!m_thread_spec_up)
-      m_thread_spec_up.reset(new ThreadSpec(*incoming.m_thread_spec_up));
+      m_thread_spec_up =
+          std::make_unique<ThreadSpec>(*incoming.m_thread_spec_up);
     else
       *m_thread_spec_up = *incoming.m_thread_spec_up;
     m_set_flags.Set(eThreadSpec);
@@ -509,7 +510,7 @@ const ThreadSpec *BreakpointOptions::GetThreadSpecNoCreate() const {
 ThreadSpec *BreakpointOptions::GetThreadSpec() {
   if (m_thread_spec_up == nullptr) {
     m_set_flags.Set(eThreadSpec);
-    m_thread_spec_up.reset(new ThreadSpec());
+    m_thread_spec_up = std::make_unique<ThreadSpec>();
   }
 
   return m_thread_spec_up.get();
diff --git a/lldb/source/Breakpoint/WatchpointOptions.cpp b/lldb/source/Breakpoint/WatchpointOptions.cpp
index 609aacf0718b3..f01f5ad3dd271 100644
--- a/lldb/source/Breakpoint/WatchpointOptions.cpp
+++ b/lldb/source/Breakpoint/WatchpointOptions.cpp
@@ -36,7 +36,7 @@ WatchpointOptions::WatchpointOptions(const WatchpointOptions &rhs)
       m_callback_is_synchronous(rhs.m_callback_is_synchronous),
       m_thread_spec_up() {
   if (rhs.m_thread_spec_up != nullptr)
-    m_thread_spec_up.reset(new ThreadSpec(*rhs.m_thread_spec_up));
+    m_thread_spec_up = std::make_unique<ThreadSpec>(*rhs.m_thread_spec_up);
 }
 
 // WatchpointOptions assignment operator
@@ -46,7 +46,7 @@ operator=(const WatchpointOptions &rhs) {
   m_callback_baton_sp = rhs.m_callback_baton_sp;
   m_callback_is_synchronous = rhs.m_callback_is_synchronous;
   if (rhs.m_thread_spec_up != nullptr)
-    m_thread_spec_up.reset(new ThreadSpec(*rhs.m_thread_spec_up));
+    m_thread_spec_up = std::make_unique<ThreadSpec>(*rhs.m_thread_spec_up);
   return *this;
 }
 
@@ -108,7 +108,7 @@ const ThreadSpec *WatchpointOptions::GetThreadSpecNoCreate() const {
 
 ThreadSpec *WatchpointOptions::GetThreadSpec() {
   if (m_thread_spec_up == nullptr)
-    m_thread_spec_up.reset(new ThreadSpec());
+    m_thread_spec_up = std::make_unique<ThreadSpec>();
 
   return m_thread_spec_up.get();
 }
diff --git a/lldb/source/Commands/CommandObjectFrame.cpp b/lldb/source/Commands/CommandObjectFrame.cpp
index 5a61fc40e2e77..6ebad9b5c488d 100644
--- a/lldb/source/Commands/CommandObjectFrame.cpp
+++ b/lldb/source/Commands/CommandObjectFrame.cpp
@@ -12,7 +12,6 @@
 #include "lldb/DataFormatters/ValueObjectPrinter.h"
 #include "lldb/Host/Config.h"
 #include "lldb/Host/OptionParser.h"
-#include "lldb/Host/StringConvert.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
 #include "lldb/Interpreter/OptionGroupFormat.h"
@@ -289,6 +288,22 @@ class CommandObjectFrameSelect : public CommandObjectParsed {
 
   ~CommandObjectFrameSelect() override = default;
 
+  void
+  HandleArgumentCompletion(CompletionRequest &request,
+                           OptionElementVector &opt_element_vector) override {
+    if (!m_exe_ctx.HasProcessScope() || request.GetCursorIndex() != 0)
+      return;
+
+    lldb::ThreadSP thread_sp = m_exe_ctx.GetThreadSP();
+    const uint32_t frame_num = thread_sp->GetStackFrameCount();
+    for (uint32_t i = 0; i < frame_num; ++i) {
+      lldb::StackFrameSP frame_sp = thread_sp->GetStackFrameAtIndex(i);
+      StreamString strm;
+      frame_sp->Dump(&strm, false, true);
+      request.TryCompleteCurrentArg(std::to_string(i), strm.GetString());
+    }
+  }
+
   Options *GetOptions() override { return &m_options; }
 
 protected:
@@ -920,6 +935,33 @@ class CommandObjectFrameRecognizerDelete : public CommandObjectParsed {
 
   ~CommandObjectFrameRecognizerDelete() override = default;
 
+  void
+  HandleArgumentCompletion(CompletionRequest &request,
+                           OptionElementVector &opt_element_vector) override {
+    if (request.GetCursorIndex() != 0)
+      return;
+
+    StackFrameRecognizerManager::ForEach(
+        [&request](uint32_t rid, std::string rname, std::string module,
+                   llvm::ArrayRef<lldb_private::ConstString> symbols,
+                   bool regexp) {
+          StreamString strm;
+          if (rname.empty())
+            rname = "(internal)";
+
+          strm << rname;
+          if (!module.empty())
+            strm << ", module " << module;
+          if (!symbols.empty())
+            for (auto &symbol : symbols)
+              strm << ", symbol " << symbol;
+          if (regexp)
+            strm << " (regexp)";
+
+          request.TryCompleteCurrentArg(std::to_string(rid), strm.GetString());
+        });
+  }
+
 protected:
   bool DoExecute(Args &command, CommandReturnObject &result) override {
     if (command.GetArgumentCount() == 0) {
@@ -943,8 +985,13 @@ class CommandObjectFrameRecognizerDelete : public CommandObjectParsed {
       return false;
     }
 
-    uint32_t recognizer_id =
-        StringConvert::ToUInt32(command.GetArgumentAtIndex(0), 0, 0);
+    uint32_t recognizer_id;
+    if (!llvm::to_integer(command.GetArgumentAtIndex(0), recognizer_id)) {
+      result.AppendErrorWithFormat("'%s' is not a valid recognizer id.\n",
+                                   command.GetArgumentAtIndex(0));
+      result.SetStatus(eReturnStatusFailed);
+      return false;
+    }
 
     StackFrameRecognizerManager::RemoveRecognizerWithID(recognizer_id);
     result.SetStatus(eReturnStatusSuccessFinishResult);
@@ -1024,6 +1071,15 @@ class CommandObjectFrameRecognizerInfo : public CommandObjectParsed {
 
 protected:
   bool DoExecute(Args &command, CommandReturnObject &result) override {
+    const char *frame_index_str = command.GetArgumentAtIndex(0);
+    uint32_t frame_index;
+    if (!llvm::to_integer(frame_index_str, frame_index)) {
+      result.AppendErrorWithFormat("'%s' is not a valid frame index.",
+                                   frame_index_str);
+      result.SetStatus(eReturnStatusFailed);
+      return false;
+    }
+
     Process *process = m_exe_ctx.GetProcessPtr();
     if (process == nullptr) {
       result.AppendError("no process");
@@ -1043,8 +1099,6 @@ class CommandObjectFrameRecognizerInfo : public CommandObjectParsed {
       return false;
     }
 
-    uint32_t frame_index =
-        StringConvert::ToUInt32(command.GetArgumentAtIndex(0), 0, 0);
     StackFrameSP frame_sp = thread->GetStackFrameAtIndex(frame_index);
     if (!frame_sp) {
       result.AppendErrorWithFormat("no frame with index %u", frame_index);
diff --git a/lldb/source/Commands/CommandObjectPlatform.cpp b/lldb/source/Commands/CommandObjectPlatform.cpp
index 1fa9c2c23570a..fcc8af6f915ce 100644
--- a/lldb/source/Commands/CommandObjectPlatform.cpp
+++ b/lldb/source/Commands/CommandObjectPlatform.cpp
@@ -11,7 +11,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Host/OptionParser.h"
-#include "lldb/Host/StringConvert.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Interpreter/CommandOptionValidators.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
@@ -546,8 +545,13 @@ class CommandObjectPlatformFClose : public CommandObjectParsed {
     if (platform_sp) {
       std::string cmd_line;
       args.GetCommandString(cmd_line);
-      const lldb::user_id_t fd =
-          StringConvert::ToUInt64(cmd_line.c_str(), UINT64_MAX);
+      lldb::user_id_t fd;
+      if (!llvm::to_integer(cmd_line, fd)) {
+        result.AppendErrorWithFormatv("'{0}' is not a valid file descriptor.\n",
+                                      cmd_line);
+        result.SetStatus(eReturnStatusFailed);
+        return result.Succeeded();
+      }
       Status error;
       bool success = platform_sp->CloseFile(fd, error);
       if (success) {
@@ -586,8 +590,13 @@ class CommandObjectPlatformFRead : public CommandObjectParsed {
     if (platform_sp) {
       std::string cmd_line;
       args.GetCommandString(cmd_line);
-      const lldb::user_id_t fd =
-          StringConvert::ToUInt64(cmd_line.c_str(), UINT64_MAX);
+      lldb::user_id_t fd;
+      if (!llvm::to_integer(cmd_line, fd)) {
+        result.AppendErrorWithFormatv("'{0}' is not a valid file descriptor.\n",
+                                      cmd_line);
+        result.SetStatus(eReturnStatusFailed);
+        return result.Succeeded();
+      }
       std::string buffer(m_options.m_count, 0);
       Status error;
       uint32_t retcode = platform_sp->ReadFile(
@@ -674,8 +683,13 @@ class CommandObjectPlatformFWrite : public CommandObjectParsed {
       std::string cmd_line;
       args.GetCommandString(cmd_line);
       Status error;
-      const lldb::user_id_t fd =
-          StringConvert::ToUInt64(cmd_line.c_str(), UINT64_MAX);
+      lldb::user_id_t fd;
+      if (!llvm::to_integer(cmd_line, fd)) {
+        result.AppendErrorWithFormatv("'{0}' is not a valid file descriptor.",
+                                      cmd_line);
+        result.SetStatus(eReturnStatusFailed);
+        return result.Succeeded();
+      }
       uint32_t retcode =
           platform_sp->WriteFile(fd, m_options.m_offset, &m_options.m_data[0],
                                  m_options.m_data.size(), error);
diff --git a/lldb/source/Commands/CommandObjectProcess.cpp b/lldb/source/Commands/CommandObjectProcess.cpp
index aa90ad61225b3..3659f0db832ce 100644
--- a/lldb/source/Commands/CommandObjectProcess.cpp
+++ b/lldb/source/Commands/CommandObjectProcess.cpp
@@ -13,7 +13,6 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Host/OptionParser.h"
-#include "lldb/Host/StringConvert.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
 #include "lldb/Interpreter/OptionArgParser.h"
@@ -1059,10 +1058,10 @@ class CommandObjectProcessSignal : public CommandObjectParsed {
       int signo = LLDB_INVALID_SIGNAL_NUMBER;
 
       const char *signal_name = command.GetArgumentAtIndex(0);
-      if (::isxdigit(signal_name[0]))
-        signo =
-            StringConvert::ToSInt32(signal_name, LLDB_INVALID_SIGNAL_NUMBER, 0);
-      else
+      if (::isxdigit(signal_name[0])) {
+        if (!llvm::to_integer(signal_name, signo))
+          signo = LLDB_INVALID_SIGNAL_NUMBER;
+      } else
         signo = process->GetUnixSignals()->GetSignalNumberFromName(signal_name);
 
       if (signo == LLDB_INVALID_SIGNAL_NUMBER) {
@@ -1410,7 +1409,8 @@ class CommandObjectProcessHandle : public CommandObjectParsed {
       real_value = 0;
     else {
       // If the value isn't 'true' or 'false', it had better be 0 or 1.
-      real_value = StringConvert::ToUInt32(option.c_str(), 3);
+      if (!llvm::to_integer(option, real_value))
+        real_value = 3;
       if (real_value != 0 && real_value != 1)
         okay = false;
     }
diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp
index ce2fb99fba7da..7bb71f4d518cc 100644
--- a/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/lldb/source/Commands/CommandObjectTarget.cpp
@@ -16,7 +16,6 @@
 #include "lldb/Core/ValueObjectVariable.h"
 #include "lldb/DataFormatters/ValueObjectPrinter.h"
 #include "lldb/Host/OptionParser.h"
-#include "lldb/Host/StringConvert.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
 #include "lldb/Interpreter/OptionArgParser.h"
@@ -506,11 +505,9 @@ class CommandObjectTargetSelect : public CommandObjectParsed {
 protected:
   bool DoExecute(Args &args, CommandReturnObject &result) override {
     if (args.GetArgumentCount() == 1) {
-      bool success = false;
       const char *target_idx_arg = args.GetArgumentAtIndex(0);
-      uint32_t target_idx =
-          StringConvert::ToUInt32(target_idx_arg, UINT32_MAX, 0, &success);
-      if (success) {
+      uint32_t target_idx;
+      if (llvm::to_integer(target_idx_arg, target_idx)) {
         TargetList &target_list = GetDebugger().GetTargetList();
         const uint32_t num_targets = target_list.GetNumTargets();
         if (target_idx < num_targets) {
@@ -1176,12 +1173,9 @@ class CommandObjectTargetModulesSearchPathsInsert : public CommandObjectParsed {
     size_t argc = command.GetArgumentCount();
     // check for at least 3 arguments and an odd number of parameters
     if (argc >= 3 && argc & 1) {
-      bool success = false;
+      uint32_t insert_idx;
 
-      uint32_t insert_idx = StringConvert::ToUInt32(
-          command.GetArgumentAtIndex(0), UINT32_MAX, 0, &success);
-
-      if (!success) {
+      if (!llvm::to_integer(command.GetArgumentAtIndex(0), insert_idx)) {
         result.AppendErrorWithFormat(
             "<index> parameter is not an integer: '%s'.\n",
             command.GetArgumentAtIndex(0));
@@ -2748,10 +2742,8 @@ class CommandObjectTargetModulesLoad
                   const char *load_addr_cstr = args.GetArgumentAtIndex(i + 1);
                   if (sect_name && load_addr_cstr) {
                     ConstString const_sect_name(sect_name);
-                    bool success = false;
-                    addr_t load_addr = StringConvert::ToUInt64(
-                        load_addr_cstr, LLDB_INVALID_ADDRESS, 0, &success);
-                    if (success) {
+                    addr_t load_addr;
+                    if (llvm::to_integer(load_addr_cstr, load_addr)) {
                       SectionSP section_sp(
                           section_list->FindSectionByName(const_sect_name));
                       if (section_sp) {
@@ -4622,8 +4614,8 @@ class CommandObjectTargetStopHookAdd : public CommandObjectParsed,
     //  First step, make the specifier.
     std::unique_ptr<SymbolContextSpecifier> specifier_up;
     if (m_options.m_sym_ctx_specified) {
-      specifier_up.reset(
-          new SymbolContextSpecifier(GetDebugger().GetSelectedTarget()));
+      specifier_up = std::make_unique<SymbolContextSpecifier>(
+          GetDebugger().GetSelectedTarget());
 
       if (!m_options.m_module_name.empty()) {
         specifier_up->AddSpecification(
@@ -4732,18 +4724,15 @@ class CommandObjectTargetStopHookDelete : public CommandObjectParsed {
         target.RemoveAllStopHooks();
       }
     } else {
-      bool success;
       for (size_t i = 0; i < num_args; i++) {
-        lldb::user_id_t user_id = StringConvert::ToUInt32(
-            command.GetArgumentAtIndex(i), 0, 0, &success);
-        if (!success) {
+        lldb::user_id_t user_id;
+        if (!llvm::to_integer(command.GetArgumentAtIndex(i), user_id)) {
           result.AppendErrorWithFormat("invalid stop hook id: \"%s\".\n",
                                        command.GetArgumentAtIndex(i));
           result.SetStatus(eReturnStatusFailed);
           return false;
         }
-        success = target.RemoveStopHookByID(user_id);
-        if (!success) {
+        if (!target.RemoveStopHookByID(user_id)) {
           result.AppendErrorWithFormat("unknown stop hook id: \"%s\".\n",
                                        command.GetArgumentAtIndex(i));
           result.SetStatus(eReturnStatusFailed);
@@ -4781,9 +4770,8 @@ class CommandObjectTargetStopHookEnableDisable : public CommandObjectParsed {
       target.SetAllStopHooksActiveState(m_enable);
     } else {
       for (size_t i = 0; i < num_args; i++) {
-        lldb::user_id_t user_id = StringConvert::ToUInt32(
-            command.GetArgumentAtIndex(i), 0, 0, &success);
-        if (!success) {
+        lldb::user_id_t user_id;
+        if (!llvm::to_integer(command.GetArgumentAtIndex(i), user_id)) {
           result.AppendErrorWithFormat("invalid stop hook id: \"%s\".\n",
                                        command.GetArgumentAtIndex(i));
           result.SetStatus(eReturnStatusFailed);
diff --git a/lldb/source/Commands/CommandObjectThread.cpp b/lldb/source/Commands/CommandObjectThread.cpp
index 8e599ed69880f..f0ad1798fec6a 100644
--- a/lldb/source/Commands/CommandObjectThread.cpp
+++ b/lldb/source/Commands/CommandObjectThread.cpp
@@ -10,7 +10,6 @@
 
 #include "lldb/Core/ValueObject.h"
 #include "lldb/Host/OptionParser.h"
-#include "lldb/Host/StringConvert.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
 #include "lldb/Interpreter/OptionArgParser.h"
@@ -109,11 +108,8 @@ class CommandObjectIterateOverThreads : public CommandObjectParsed {
           process->GetThreadList().GetMutex());
 
       for (size_t i = 0; i < num_args; i++) {
-        bool success;
-
-        uint32_t thread_idx = StringConvert::ToUInt32(
-            command.GetArgumentAtIndex(i), 0, 0, &success);
-        if (!success) {
+        uint32_t thread_idx;
+        if (!llvm::to_integer(command.GetArgumentAtIndex(i), thread_idx)) {
           result.AppendErrorWithFormat("invalid thread specification: \"%s\"\n",
                                        command.GetArgumentAtIndex(i));
           result.SetStatus(eReturnStatusFailed);
@@ -565,9 +561,9 @@ class CommandObjectThreadStepWithTypeAndScope : public CommandObjectParsed {
       }
     } else {
       const char *thread_idx_cstr = command.GetArgumentAtIndex(0);
-      uint32_t step_thread_idx =
-          StringConvert::ToUInt32(thread_idx_cstr, LLDB_INVALID_INDEX32);
-      if (step_thread_idx == LLDB_INVALID_INDEX32) {
+      uint32_t step_thread_idx;
+
+      if (!llvm::to_integer(thread_idx_cstr, step_thread_idx)) {
         result.AppendErrorWithFormat("invalid thread index '%s'.\n",
                                      thread_idx_cstr);
         result.SetStatus(eReturnStatusFailed);
@@ -1095,9 +1091,7 @@ class CommandObjectThreadUntil : public CommandObjectParsed {
         size_t num_args = command.GetArgumentCount();
         for (size_t i = 0; i < num_args; i++) {
           uint32_t line_number;
-          line_number = StringConvert::ToUInt32(command.GetArgumentAtIndex(i),
-                                                UINT32_MAX);
-          if (line_number == UINT32_MAX) {
+          if (!llvm::to_integer(command.GetArgumentAtIndex(i), line_number)) {
             result.AppendErrorWithFormat("invalid line number: '%s'.\n",
                                          command.GetArgumentAtIndex(i));
             result.SetStatus(eReturnStatusFailed);
@@ -1321,8 +1315,13 @@ class CommandObjectThreadSelect : public CommandObjectParsed {
       return false;
     }
 
-    uint32_t index_id =
-        StringConvert::ToUInt32(command.GetArgumentAtIndex(0), 0, 0);
+    uint32_t index_id;
+    if (!llvm::to_integer(command.GetArgumentAtIndex(0), index_id)) {
+      result.AppendErrorWithFormat("Invalid thread index '%s'",
+                                   command.GetArgumentAtIndex(0));
+      result.SetStatus(eReturnStatusFailed);
+      return false;
+    }
 
     Thread *new_thread =
         process->GetThreadList().FindThreadByIndexID(index_id).get();
@@ -1989,10 +1988,8 @@ class CommandObjectThreadPlanDiscard : public CommandObjectParsed {
       return false;
     }
 
-    bool success;
-    uint32_t thread_plan_idx =
-        StringConvert::ToUInt32(args.GetArgumentAtIndex(0), 0, 0, &success);
-    if (!success) {
+    uint32_t thread_plan_idx;
+    if (!llvm::to_integer(args.GetArgumentAtIndex(0), thread_plan_idx)) {
       result.AppendErrorWithFormat(
           "Invalid thread index: \"%s\" - should be unsigned int.",
           args.GetArgumentAtIndex(0));
@@ -2066,11 +2063,8 @@ class CommandObjectThreadPlanPrune : public CommandObjectParsed {
         process->GetThreadList().GetMutex());
 
     for (size_t i = 0; i < num_args; i++) {
-      bool success;
-
-      lldb::tid_t tid = StringConvert::ToUInt64(
-          args.GetArgumentAtIndex(i), 0, 0, &success);
-      if (!success) {
+      lldb::tid_t tid;
+      if (!llvm::to_integer(args.GetArgumentAtIndex(i), tid)) {
         result.AppendErrorWithFormat("invalid thread specification: \"%s\"\n",
                                      args.GetArgumentAtIndex(i));
         result.SetStatus(eReturnStatusFailed);
diff --git a/lldb/source/Commands/CommandObjectType.cpp b/lldb/source/Commands/CommandObjectType.cpp
index d6dd4067a3edc..b2020f26621f8 100644
--- a/lldb/source/Commands/CommandObjectType.cpp
+++ b/lldb/source/Commands/CommandObjectType.cpp
@@ -1031,8 +1031,8 @@ class CommandObjectTypeFormatterList : public CommandObjectParsed {
     std::unique_ptr<RegularExpression> formatter_regex;
 
     if (m_options.m_category_regex.OptionWasSet()) {
-      category_regex.reset(new RegularExpression(
-          m_options.m_category_regex.GetCurrentValueAsRef()));
+      category_regex = std::make_unique<RegularExpression>(
+          m_options.m_category_regex.GetCurrentValueAsRef());
       if (!category_regex->IsValid()) {
         result.AppendErrorWithFormat(
             "syntax error in category regular expression '%s'",
@@ -1044,8 +1044,8 @@ class CommandObjectTypeFormatterList : public CommandObjectParsed {
 
     if (argc == 1) {
       const char *arg = command.GetArgumentAtIndex(0);
-      formatter_regex.reset(
-          new RegularExpression(llvm::StringRef::withNullAsEmpty(arg)));
+      formatter_regex = std::make_unique<RegularExpression>(
+          llvm::StringRef::withNullAsEmpty(arg));
       if (!formatter_regex->IsValid()) {
         result.AppendErrorWithFormat("syntax error in regular expression '%s'",
                                      arg);
@@ -2092,7 +2092,8 @@ class CommandObjectTypeCategoryList : public CommandObjectParsed {
 
     if (argc == 1) {
       const char *arg = command.GetArgumentAtIndex(0);
-      regex.reset(new RegularExpression(llvm::StringRef::withNullAsEmpty(arg)));
+      regex = std::make_unique<RegularExpression>(
+          llvm::StringRef::withNullAsEmpty(arg));
       if (!regex->IsValid()) {
         result.AppendErrorWithFormat(
             "syntax error in category regular expression '%s'", arg);
diff --git a/lldb/source/Core/IOHandler.cpp b/lldb/source/Core/IOHandler.cpp
index cd9531b17a925..6cf09aaa7f9d1 100644
--- a/lldb/source/Core/IOHandler.cpp
+++ b/lldb/source/Core/IOHandler.cpp
@@ -253,9 +253,9 @@ IOHandlerEditline::IOHandlerEditline(
                  m_input_sp && m_input_sp->GetIsRealTerminal();
 
   if (use_editline) {
-    m_editline_up.reset(new Editline(editline_name, GetInputFILE(),
-                                     GetOutputFILE(), GetErrorFILE(),
-                                     m_color_prompts));
+    m_editline_up = std::make_unique<Editline>(editline_name, GetInputFILE(),
+                                               GetOutputFILE(), GetErrorFILE(),
+                                               m_color_prompts);
     m_editline_up->SetIsInputCompleteCallback(IsInputCompleteCallback, this);
     m_editline_up->SetAutoCompleteCallback(AutoCompleteCallback, this);
     // See if the delegate supports fixing indentation
diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp
index 49e93cd11dd1d..f8fc917721983 100644
--- a/lldb/source/Core/IOHandlerCursesGUI.cpp
+++ b/lldb/source/Core/IOHandlerCursesGUI.cpp
@@ -3910,7 +3910,7 @@ IOHandlerCursesGUI::IOHandlerCursesGUI(Debugger &debugger)
 void IOHandlerCursesGUI::Activate() {
   IOHandler::Activate();
   if (!m_app_ap) {
-    m_app_ap.reset(new Application(GetInputFILE(), GetOutputFILE()));
+    m_app_ap = std::make_unique<Application>(GetInputFILE(), GetOutputFILE());
 
     // This is both a window and a menu delegate
     std::shared_ptr<ApplicationDelegate> app_delegate_sp(
diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp
index ba3d2c509be84..3a775b07e5e1f 100644
--- a/lldb/source/Core/ValueObject.cpp
+++ b/lldb/source/Core/ValueObject.cpp
@@ -140,58 +140,6 @@ ValueObject::ValueObject(ExecutionContextScope *exe_scope,
 // Destructor
 ValueObject::~ValueObject() {}
 
-void ValueObject::UpdateChildrenAddressType() {
-  Value::ValueType value_type = m_value.GetValueType();
-  ExecutionContext exe_ctx(GetExecutionContextRef());
-  Process *process = exe_ctx.GetProcessPtr();
-  const bool process_is_alive = process && process->IsAlive();
-  const uint32_t type_info = GetCompilerType().GetTypeInfo();
-  const bool is_pointer_or_ref =
-      (type_info & (lldb::eTypeIsPointer | lldb::eTypeIsReference)) != 0;
-
-  switch (value_type) {
-  case Value::eValueTypeFileAddress:
-    // If this type is a pointer, then its children will be considered load
-    // addresses if the pointer or reference is dereferenced, but only if
-    // the process is alive.
-    //
-    // There could be global variables like in the following code:
-    // struct LinkedListNode { Foo* foo; LinkedListNode* next; };
-    // Foo g_foo1;
-    // Foo g_foo2;
-    // LinkedListNode g_second_node = { &g_foo2, NULL };
-    // LinkedListNode g_first_node = { &g_foo1, &g_second_node };
-    //
-    // When we aren't running, we should be able to look at these variables
-    // using the "target variable" command. Children of the "g_first_node"
-    // always will be of the same address type as the parent. But children
-    // of the "next" member of LinkedListNode will become load addresses if
-    // we have a live process, or remain a file address if it was a file
-    // address.
-    if (process_is_alive && is_pointer_or_ref)
-      SetAddressTypeOfChildren(eAddressTypeLoad);
-    else
-      SetAddressTypeOfChildren(eAddressTypeFile);
-    break;
-  case Value::eValueTypeHostAddress:
-    // Same as above for load addresses, except children of pointer or refs
-    // are always load addresses. Host addresses are used to store freeze
-    // dried variables. If this type is a struct, the entire struct
-    // contents will be copied into the heap of the
-    // LLDB process, but we do not currently follow any pointers.
-    if (is_pointer_or_ref)
-      SetAddressTypeOfChildren(eAddressTypeLoad);
-    else
-      SetAddressTypeOfChildren(eAddressTypeHost);
-    break;
-  case Value::eValueTypeLoadAddress:
-  case Value::eValueTypeScalar:
-  case Value::eValueTypeVector:
-    SetAddressTypeOfChildren(eAddressTypeLoad);
-    break;
-  }
-}
-
 bool ValueObject::UpdateValueIfNeeded(bool update_format) {
 
   bool did_change_formats = false;
@@ -574,15 +522,13 @@ size_t ValueObject::GetIndexOfChildWithName(ConstString name) {
 
 ValueObjectSP ValueObject::GetChildMemberWithName(ConstString name,
                                                   bool can_create) {
-  // when getting a child by name, it could be buried inside some base classes
-  // (which really aren't part of the expression path), so we need a vector of
-  // indexes that can get us down to the correct child
-  ValueObjectSP child_sp;
-
-  // We may need to update our value if we are dynamic
+  // We may need to update our value if we are dynamic.
   if (IsPossibleDynamicType())
     UpdateValueIfNeeded(false);
 
+  // When getting a child by name, it could be buried inside some base classes
+  // (which really aren't part of the expression path), so we need a vector of
+  // indexes that can get us down to the correct child.
   std::vector<uint32_t> child_indexes;
   bool omit_empty_base_classes = true;
 
@@ -592,20 +538,13 @@ ValueObjectSP ValueObject::GetChildMemberWithName(ConstString name,
   const size_t num_child_indexes =
       GetCompilerType().GetIndexOfChildMemberWithName(
           name.GetCString(), omit_empty_base_classes, child_indexes);
-  if (num_child_indexes > 0) {
-    std::vector<uint32_t>::const_iterator pos = child_indexes.begin();
-    std::vector<uint32_t>::const_iterator end = child_indexes.end();
-
-    child_sp = GetChildAtIndex(*pos, can_create);
-    for (++pos; pos != end; ++pos) {
-      if (child_sp) {
-        ValueObjectSP new_child_sp(child_sp->GetChildAtIndex(*pos, can_create));
-        child_sp = new_child_sp;
-      } else {
-        child_sp.reset();
-      }
-    }
-  }
+  if (num_child_indexes == 0)
+    return nullptr;
+
+  ValueObjectSP child_sp = GetSP();
+  for (uint32_t idx : child_indexes)
+    if (child_sp)
+      child_sp = child_sp->GetChildAtIndex(idx, can_create);
   return child_sp;
 }
 
@@ -687,10 +626,15 @@ ValueObject *ValueObject::CreateChildAtIndex(size_t idx,
         language_flags);
   }
 
-  if (!valobj && synthetic_array_member)
-    valobj = GetSyntheticValue()
-                 ->GetChildAtIndex(synthetic_index, synthetic_array_member)
-                 .get();
+  // In case of an incomplete type, try to use the ValueObject's
+  // synthetic value to create the child ValueObject.
+  if (!valobj && synthetic_array_member) {
+    if (ValueObjectSP synth_valobj_sp = GetSyntheticValue()) {
+      valobj = synth_valobj_sp
+                   ->GetChildAtIndex(synthetic_index, synthetic_array_member)
+                   .get();
+    }
+  }
 
   return valobj;
 }
@@ -1203,6 +1147,7 @@ uint64_t ValueObject::GetValueAsUnsigned(uint64_t fail_value, bool *success) {
     if (ResolveValue(scalar)) {
       if (success)
         *success = true;
+      scalar.MakeUnsigned();
       return scalar.ULongLong(fail_value);
     }
     // fallthrough, otherwise...
@@ -1220,6 +1165,7 @@ int64_t ValueObject::GetValueAsSigned(int64_t fail_value, bool *success) {
     if (ResolveValue(scalar)) {
       if (success)
         *success = true;
+      scalar.MakeSigned();
       return scalar.SLongLong(fail_value);
     }
     // fallthrough, otherwise...
diff --git a/lldb/source/Core/ValueObjectVariable.cpp b/lldb/source/Core/ValueObjectVariable.cpp
index 8199c6101395f..0d1e7b047a0ac 100644
--- a/lldb/source/Core/ValueObjectVariable.cpp
+++ b/lldb/source/Core/ValueObjectVariable.cpp
@@ -242,9 +242,64 @@ bool ValueObjectVariable::UpdateValue() {
       m_resolved_value.SetContext(Value::eContextTypeInvalid, nullptr);
     }
   }
+  
   return m_error.Success();
 }
 
+void ValueObjectVariable::DoUpdateChildrenAddressType(ValueObject &valobj) {
+  Value::ValueType value_type = valobj.GetValue().GetValueType();
+  ExecutionContext exe_ctx(GetExecutionContextRef());
+  Process *process = exe_ctx.GetProcessPtr();
+  const bool process_is_alive = process && process->IsAlive();
+  const uint32_t type_info = valobj.GetCompilerType().GetTypeInfo();
+  const bool is_pointer_or_ref =
+      (type_info & (lldb::eTypeIsPointer | lldb::eTypeIsReference)) != 0;
+
+  switch (value_type) {
+  case Value::eValueTypeFileAddress:
+    // If this type is a pointer, then its children will be considered load
+    // addresses if the pointer or reference is dereferenced, but only if
+    // the process is alive.
+    //
+    // There could be global variables like in the following code:
+    // struct LinkedListNode { Foo* foo; LinkedListNode* next; };
+    // Foo g_foo1;
+    // Foo g_foo2;
+    // LinkedListNode g_second_node = { &g_foo2, NULL };
+    // LinkedListNode g_first_node = { &g_foo1, &g_second_node };
+    //
+    // When we aren't running, we should be able to look at these variables
+    // using the "target variable" command. Children of the "g_first_node"
+    // always will be of the same address type as the parent. But children
+    // of the "next" member of LinkedListNode will become load addresses if
+    // we have a live process, or remain a file address if it was a file
+    // address.
+    if (process_is_alive && is_pointer_or_ref)
+      valobj.SetAddressTypeOfChildren(eAddressTypeLoad);
+    else
+      valobj.SetAddressTypeOfChildren(eAddressTypeFile);
+    break;
+  case Value::eValueTypeHostAddress:
+    // Same as above for load addresses, except children of pointer or refs
+    // are always load addresses. Host addresses are used to store freeze
+    // dried variables. If this type is a struct, the entire struct
+    // contents will be copied into the heap of the
+    // LLDB process, but we do not currently follow any pointers.
+    if (is_pointer_or_ref)
+      valobj.SetAddressTypeOfChildren(eAddressTypeLoad);
+    else
+      valobj.SetAddressTypeOfChildren(eAddressTypeHost);
+    break;
+  case Value::eValueTypeLoadAddress:
+  case Value::eValueTypeScalar:
+  case Value::eValueTypeVector:
+    valobj.SetAddressTypeOfChildren(eAddressTypeLoad);
+    break;
+  }
+}
+
+
+
 bool ValueObjectVariable::IsInScope() {
   const ExecutionContextRef &exe_ctx_ref = GetExecutionContextRef();
   if (exe_ctx_ref.HasFrameRef()) {
diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp
index 3bcadbe9ed4a2..6050c1922564c 100644
--- a/lldb/source/Expression/DWARFExpression.cpp
+++ b/lldb/source/Expression/DWARFExpression.cpp
@@ -2346,8 +2346,8 @@ bool DWARFExpression::Evaluate(
         return false;
       }
       const uint64_t die_offset = opcodes.GetULEB128(&offset);
-      Scalar::Type type = Scalar::e_void;
       uint64_t bit_size;
+      bool sign;
       if (die_offset == 0) {
         // The generic type has the size of an address on the target
         // machine and an unspecified signedness. Scalar has no
@@ -2357,13 +2357,13 @@ bool DWARFExpression::Evaluate(
             error_ptr->SetErrorString("No module");
           return false;
         }
+        sign = false;
         bit_size = module_sp->GetArchitecture().GetAddressByteSize() * 8;
         if (!bit_size) {
           if (error_ptr)
             error_ptr->SetErrorString("unspecified architecture");
           return false;
         }
-        type = Scalar::GetBestTypeForBitSize(bit_size, false);
       } else {
         // Retrieve the type DIE that the value is being converted to.
         // FIXME: the constness has annoying ripple effects.
@@ -2386,11 +2386,11 @@ bool DWARFExpression::Evaluate(
         switch (encoding) {
         case DW_ATE_signed:
         case DW_ATE_signed_char:
-          type = Scalar::GetBestTypeForBitSize(bit_size, true);
+          sign = true;
           break;
         case DW_ATE_unsigned:
         case DW_ATE_unsigned_char:
-          type = Scalar::GetBestTypeForBitSize(bit_size, false);
+          sign = false;
           break;
         default:
           if (error_ptr)
@@ -2398,13 +2398,8 @@ bool DWARFExpression::Evaluate(
           return false;
         }
       }
-      if (type == Scalar::e_void) {
-        if (error_ptr)
-          error_ptr->SetErrorString("Unsupported pointer size");
-        return false;
-      }
       Scalar &top = stack.back().ResolveValue(exe_ctx);
-      top.TruncOrExtendTo(type, bit_size);
+      top.TruncOrExtendTo(bit_size, sign);
       break;
     }
 
diff --git a/lldb/source/Expression/IRExecutionUnit.cpp b/lldb/source/Expression/IRExecutionUnit.cpp
index 1bcd6aa918e9b..e3c9c1d7fdf54 100644
--- a/lldb/source/Expression/IRExecutionUnit.cpp
+++ b/lldb/source/Expression/IRExecutionUnit.cpp
@@ -265,11 +265,9 @@ void IRExecutionUnit::GetRunnableInfo(Status &error, lldb::addr_t &func_addr,
 
   builder.setEngineKind(llvm::EngineKind::JIT)
       .setErrorStr(&error_string)
-      .setRelocationModel(triple.isOSBinFormatMachO()
-                              ? llvm::Reloc::PIC_
-                              : llvm::Reloc::Static)
-      .setMCJITMemoryManager(
-          std::unique_ptr<MemoryManager>(new MemoryManager(*this)))
+      .setRelocationModel(triple.isOSBinFormatMachO() ? llvm::Reloc::PIC_
+                                                      : llvm::Reloc::Static)
+      .setMCJITMemoryManager(std::make_unique<MemoryManager>(*this))
       .setOptLevel(llvm::CodeGenOpt::Less);
 
   llvm::StringRef mArch;
diff --git a/lldb/source/Expression/IRInterpreter.cpp b/lldb/source/Expression/IRInterpreter.cpp
index 04d7cb3e74b6e..4c7a656265981 100644
--- a/lldb/source/Expression/IRInterpreter.cpp
+++ b/lldb/source/Expression/IRInterpreter.cpp
@@ -147,7 +147,7 @@ class InterpreterStackFrame {
     return std::string(ss.GetString());
   }
 
-  bool AssignToMatchType(lldb_private::Scalar &scalar, uint64_t u64value,
+  bool AssignToMatchType(lldb_private::Scalar &scalar, llvm::APInt value,
                          Type *type) {
     size_t type_size = m_target_data.getTypeStoreSize(type);
 
@@ -157,7 +157,7 @@ class InterpreterStackFrame {
     if (type_size != 1)
       type_size = PowerOf2Ceil(type_size);
 
-    scalar = llvm::APInt(type_size*8, u64value);
+    scalar = value.zextOrTrunc(type_size * 8);
     return true;
   }
 
@@ -171,8 +171,7 @@ class InterpreterStackFrame {
       if (!ResolveConstantValue(value_apint, constant))
         return false;
 
-      return AssignToMatchType(scalar, value_apint.getLimitedValue(),
-                               value->getType());
+      return AssignToMatchType(scalar, value_apint, value->getType());
     }
 
     lldb::addr_t process_address = ResolveValue(value, module);
@@ -190,13 +189,14 @@ class InterpreterStackFrame {
     lldb::offset_t offset = 0;
     if (value_size <= 8) {
       uint64_t u64value = value_extractor.GetMaxU64(&offset, value_size);
-      return AssignToMatchType(scalar, u64value, value->getType());
+      return AssignToMatchType(scalar, llvm::APInt(64, u64value),
+                               value->getType());
     }
 
     return false;
   }
 
-  bool AssignValue(const Value *value, lldb_private::Scalar &scalar,
+  bool AssignValue(const Value *value, lldb_private::Scalar scalar,
                    Module &module) {
     lldb::addr_t process_address = ResolveValue(value, module);
 
@@ -205,7 +205,9 @@ class InterpreterStackFrame {
 
     lldb_private::Scalar cast_scalar;
 
-    if (!AssignToMatchType(cast_scalar, scalar.ULongLong(), value->getType()))
+    scalar.MakeUnsigned();
+    if (!AssignToMatchType(cast_scalar, scalar.UInt128(llvm::APInt()),
+                           value->getType()))
       return false;
 
     size_t value_byte_size = m_target_data.getTypeStoreSize(value->getType());
diff --git a/lldb/source/Expression/Materializer.cpp b/lldb/source/Expression/Materializer.cpp
index 8e96891257e4d..f33462053f22a 100644
--- a/lldb/source/Expression/Materializer.cpp
+++ b/lldb/source/Expression/Materializer.cpp
@@ -399,7 +399,8 @@ uint32_t Materializer::AddPersistentVariable(
     lldb::ExpressionVariableSP &persistent_variable_sp,
     PersistentVariableDelegate *delegate, Status &err) {
   EntityVector::iterator iter = m_entities.insert(m_entities.end(), EntityUP());
-  iter->reset(new EntityPersistentVariable(persistent_variable_sp, delegate));
+  *iter = std::make_unique<EntityPersistentVariable>(persistent_variable_sp,
+                                                     delegate);
   uint32_t ret = AddStructMember(**iter);
   (*iter)->SetOffset(ret);
   return ret;
@@ -752,7 +753,7 @@ class EntityVariable : public Materializer::Entity {
 
 uint32_t Materializer::AddVariable(lldb::VariableSP &variable_sp, Status &err) {
   EntityVector::iterator iter = m_entities.insert(m_entities.end(), EntityUP());
-  iter->reset(new EntityVariable(variable_sp));
+  *iter = std::make_unique<EntityVariable>(variable_sp);
   uint32_t ret = AddStructMember(**iter);
   (*iter)->SetOffset(ret);
   return ret;
@@ -784,7 +785,9 @@ class EntityResultVariable : public Materializer::Entity {
 
       const lldb::addr_t load_addr = process_address + m_offset;
 
-      ExecutionContextScope *exe_scope = map.GetBestExecutionContextScope();
+      ExecutionContextScope *exe_scope = frame_sp.get();
+      if (!exe_scope)
+        exe_scope = map.GetBestExecutionContextScope();
 
       llvm::Optional<uint64_t> byte_size = m_type.GetByteSize(exe_scope);
       if (!byte_size) {
@@ -834,7 +837,9 @@ class EntityResultVariable : public Materializer::Entity {
                      lldb::addr_t frame_bottom, Status &err) override {
     err.Clear();
 
-    ExecutionContextScope *exe_scope = map.GetBestExecutionContextScope();
+    ExecutionContextScope *exe_scope = frame_sp.get();
+    if (!exe_scope)
+      exe_scope = map.GetBestExecutionContextScope();
 
     if (!exe_scope) {
       err.SetErrorString("Couldn't dematerialize a result variable: invalid "
@@ -1030,8 +1035,8 @@ uint32_t Materializer::AddResultVariable(const CompilerType &type,
                                          PersistentVariableDelegate *delegate,
                                          Status &err) {
   EntityVector::iterator iter = m_entities.insert(m_entities.end(), EntityUP());
-  iter->reset(new EntityResultVariable(type, is_program_reference,
-                                       keep_in_memory, delegate));
+  *iter = std::make_unique<EntityResultVariable>(type, is_program_reference,
+                                                 keep_in_memory, delegate);
   uint32_t ret = AddStructMember(**iter);
   (*iter)->SetOffset(ret);
   return ret;
@@ -1060,7 +1065,9 @@ class EntitySymbol : public Materializer::Entity {
 
     const Address sym_address = m_symbol.GetAddress();
 
-    ExecutionContextScope *exe_scope = map.GetBestExecutionContextScope();
+    ExecutionContextScope *exe_scope = frame_sp.get();
+    if (!exe_scope)
+      exe_scope = map.GetBestExecutionContextScope();
 
     lldb::TargetSP target_sp;
 
@@ -1147,7 +1154,7 @@ class EntitySymbol : public Materializer::Entity {
 
 uint32_t Materializer::AddSymbol(const Symbol &symbol_sp, Status &err) {
   EntityVector::iterator iter = m_entities.insert(m_entities.end(), EntityUP());
-  iter->reset(new EntitySymbol(symbol_sp));
+  *iter = std::make_unique<EntitySymbol>(symbol_sp);
   uint32_t ret = AddStructMember(**iter);
   (*iter)->SetOffset(ret);
   return ret;
@@ -1324,7 +1331,7 @@ class EntityRegister : public Materializer::Entity {
 uint32_t Materializer::AddRegister(const RegisterInfo &register_info,
                                    Status &err) {
   EntityVector::iterator iter = m_entities.insert(m_entities.end(), EntityUP());
-  iter->reset(new EntityRegister(register_info));
+  *iter = std::make_unique<EntityRegister>(register_info);
   uint32_t ret = AddStructMember(**iter);
   (*iter)->SetOffset(ret);
   return ret;
@@ -1341,7 +1348,6 @@ Materializer::DematerializerSP
 Materializer::Materialize(lldb::StackFrameSP &frame_sp, IRMemoryMap &map,
                           lldb::addr_t process_address, Status &error) {
   ExecutionContextScope *exe_scope = frame_sp.get();
-
   if (!exe_scope)
     exe_scope = map.GetBestExecutionContextScope();
 
@@ -1392,7 +1398,9 @@ void Materializer::Dematerializer::Dematerialize(Status &error,
   if (thread_sp)
     frame_sp = thread_sp->GetFrameWithStackID(m_stack_id);
 
-  ExecutionContextScope *exe_scope = m_map->GetBestExecutionContextScope();
+  ExecutionContextScope *exe_scope = frame_sp.get();
+  if (!exe_scope)
+    exe_scope = m_map->GetBestExecutionContextScope();
 
   if (!IsValid()) {
     error.SetErrorToGenericError();
diff --git a/lldb/source/Expression/REPL.cpp b/lldb/source/Expression/REPL.cpp
index b49b304b3f742..fd7c39686921d 100644
--- a/lldb/source/Expression/REPL.cpp
+++ b/lldb/source/Expression/REPL.cpp
@@ -457,6 +457,10 @@ void REPL::IOHandlerComplete(IOHandler &io_handler,
     debugger.GetCommandInterpreter().HandleCompletion(sub_request);
     StringList matches, descriptions;
     sub_result.GetMatches(matches);
+    // Prepend command prefix that was excluded in the completion request.
+    if (request.GetCursorIndex() == 0)
+      for (auto &match : matches)
+        match.insert(0, 1, ':');
     sub_result.GetDescriptions(descriptions);
     request.AddCompletions(matches, descriptions);
     return;
diff --git a/lldb/source/Host/macosx/objcxx/Host.mm b/lldb/source/Host/macosx/objcxx/Host.mm
index 045ba7f3671f0..1635132a154e5 100644
--- a/lldb/source/Host/macosx/objcxx/Host.mm
+++ b/lldb/source/Host/macosx/objcxx/Host.mm
@@ -631,8 +631,7 @@ static bool GetMacOSXProcessUserAndGroup(ProcessInstanceInfo &process_info) {
         kinfo.kp_proc.p_pid == 0 ||      // Skip kernel (kernel pid is zero)
         kinfo.kp_proc.p_stat == SZOMB || // Zombies are bad, they like brains...
         kinfo.kp_proc.p_flag & P_TRACED ||   // Being debugged?
-        kinfo.kp_proc.p_flag & P_WEXIT ||    // Working on exiting?
-        kinfo.kp_proc.p_flag & P_TRANSLATED) // Skip translated ppc (Rosetta)
+        kinfo.kp_proc.p_flag & P_WEXIT)
       continue;
 
     ProcessInstanceInfo process_info;
diff --git a/lldb/source/Host/macosx/objcxx/HostInfoMacOSX.mm b/lldb/source/Host/macosx/objcxx/HostInfoMacOSX.mm
index fd88d0c31de6a..60eacb1e49b2c 100644
--- a/lldb/source/Host/macosx/objcxx/HostInfoMacOSX.mm
+++ b/lldb/source/Host/macosx/objcxx/HostInfoMacOSX.mm
@@ -136,7 +136,7 @@ static void ParseOSVersion(llvm::VersionTuple &version, NSString *Key) {
   size_t framework_pos = raw_path.find("LLDB.framework");
   if (framework_pos != std::string::npos) {
     framework_pos += strlen("LLDB.framework");
-#if defined(__arm__) || defined(__arm64__) || defined(__aarch64__)
+#if TARGET_OS_EMBEDDED
     // Shallow bundle
     raw_path.resize(framework_pos);
 #else
@@ -276,6 +276,9 @@ static void ParseOSVersion(llvm::VersionTuple &version, NSString *Key) {
 #elif defined(TARGET_OS_WATCHOS) && TARGET_OS_WATCHOS == 1
         arch_32.GetTriple().setOS(llvm::Triple::WatchOS);
         arch_64.GetTriple().setOS(llvm::Triple::WatchOS);
+#elif defined(TARGET_OS_OSX) && TARGET_OS_OSX == 1
+        arch_32.GetTriple().setOS(llvm::Triple::MacOSX);
+        arch_64.GetTriple().setOS(llvm::Triple::MacOSX);
 #else
         arch_32.GetTriple().setOS(llvm::Triple::IOS);
         arch_64.GetTriple().setOS(llvm::Triple::IOS);
diff --git a/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp b/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp
index 5683330a75c8e..3a547ce128d8f 100644
--- a/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp
+++ b/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp
@@ -209,7 +209,7 @@ ConnectionStatus ConnectionFileDescriptor::Connect(llvm::StringRef path,
           // this. For now, we assume we must assume we don't own it.
 
           std::unique_ptr<TCPSocket> tcp_socket;
-          tcp_socket.reset(new TCPSocket(fd, false, false));
+          tcp_socket = std::make_unique<TCPSocket>(fd, false, false);
           // Try and get a socket option from this file descriptor to see if
           // this is a socket and set m_is_socket accordingly.
           int resuse;
diff --git a/lldb/source/Interpreter/Options.cpp b/lldb/source/Interpreter/Options.cpp
index fb13bd48e478f..c14b6b6ce66a9 100644
--- a/lldb/source/Interpreter/Options.cpp
+++ b/lldb/source/Interpreter/Options.cpp
@@ -798,7 +798,8 @@ void Options::HandleOptionArgumentCompletion(
               interpreter.GetDebugger().GetSelectedTarget();
           // Search filters require a target...
           if (target_sp)
-            filter_up.reset(new SearchFilterByModule(target_sp, module_spec));
+            filter_up =
+                std::make_unique<SearchFilterByModule>(target_sp, module_spec);
         }
         break;
       }
diff --git a/lldb/source/Interpreter/ScriptInterpreter.cpp b/lldb/source/Interpreter/ScriptInterpreter.cpp
index 5013d993dbeed..86620449f2f4d 100644
--- a/lldb/source/Interpreter/ScriptInterpreter.cpp
+++ b/lldb/source/Interpreter/ScriptInterpreter.cpp
@@ -7,16 +7,21 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Interpreter/ScriptInterpreter.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string>
-
+#include "lldb/Core/Debugger.h"
+#include "lldb/Host/ConnectionFileDescriptor.h"
+#include "lldb/Host/Pipe.h"
 #include "lldb/Host/PseudoTerminal.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StringList.h"
+#if defined(_WIN32)
+#include "lldb/Host/windows/ConnectionGenericFileWindows.h"
+#endif
+#include <memory>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
 
 using namespace lldb;
 using namespace lldb_private;
@@ -102,6 +107,113 @@ Status ScriptInterpreter::SetBreakpointCommandCallbackFunction(
 
 std::unique_ptr<ScriptInterpreterLocker>
 ScriptInterpreter::AcquireInterpreterLock() {
-  return std::unique_ptr<ScriptInterpreterLocker>(
-      new ScriptInterpreterLocker());
+  return std::make_unique<ScriptInterpreterLocker>();
+}
+
+static void ReadThreadBytesReceived(void *baton, const void *src,
+                                    size_t src_len) {
+  if (src && src_len) {
+    Stream *strm = (Stream *)baton;
+    strm->Write(src, src_len);
+    strm->Flush();
+  }
+}
+
+llvm::Expected<std::unique_ptr<ScriptInterpreterIORedirect>>
+ScriptInterpreterIORedirect::Create(bool enable_io, Debugger &debugger,
+                                    CommandReturnObject *result) {
+  if (enable_io)
+    return std::unique_ptr<ScriptInterpreterIORedirect>(
+        new ScriptInterpreterIORedirect(debugger, result));
+
+  auto nullin = FileSystem::Instance().Open(FileSpec(FileSystem::DEV_NULL),
+                                            File::eOpenOptionRead);
+  if (!nullin)
+    return nullin.takeError();
+
+  auto nullout = FileSystem::Instance().Open(FileSpec(FileSystem::DEV_NULL),
+                                             File::eOpenOptionWrite);
+  if (!nullout)
+    return nullin.takeError();
+
+  return std::unique_ptr<ScriptInterpreterIORedirect>(
+      new ScriptInterpreterIORedirect(std::move(*nullin), std::move(*nullout)));
+}
+
+ScriptInterpreterIORedirect::ScriptInterpreterIORedirect(
+    std::unique_ptr<File> input, std::unique_ptr<File> output)
+    : m_input_file_sp(std::move(input)),
+      m_output_file_sp(std::make_shared<StreamFile>(std::move(output))),
+      m_error_file_sp(m_output_file_sp),
+      m_communication("lldb.ScriptInterpreterIORedirect.comm"),
+      m_disconnect(false) {}
+
+ScriptInterpreterIORedirect::ScriptInterpreterIORedirect(
+    Debugger &debugger, CommandReturnObject *result)
+    : m_communication("lldb.ScriptInterpreterIORedirect.comm"),
+      m_disconnect(false) {
+
+  if (result) {
+    m_input_file_sp = debugger.GetInputFileSP();
+
+    Pipe pipe;
+    Status pipe_result = pipe.CreateNew(false);
+#if defined(_WIN32)
+    lldb::file_t read_file = pipe.GetReadNativeHandle();
+    pipe.ReleaseReadFileDescriptor();
+    std::unique_ptr<ConnectionGenericFile> conn_up =
+        std::make_unique<ConnectionGenericFile>(read_file, true);
+#else
+    std::unique_ptr<ConnectionFileDescriptor> conn_up =
+        std::make_unique<ConnectionFileDescriptor>(
+            pipe.ReleaseReadFileDescriptor(), true);
+#endif
+
+    if (conn_up->IsConnected()) {
+      m_communication.SetConnection(std::move(conn_up));
+      m_communication.SetReadThreadBytesReceivedCallback(
+          ReadThreadBytesReceived, &result->GetOutputStream());
+      m_communication.StartReadThread();
+      m_disconnect = true;
+
+      FILE *outfile_handle = fdopen(pipe.ReleaseWriteFileDescriptor(), "w");
+      m_output_file_sp = std::make_shared<StreamFile>(outfile_handle, true);
+      m_error_file_sp = m_output_file_sp;
+      if (outfile_handle)
+        ::setbuf(outfile_handle, nullptr);
+
+      result->SetImmediateOutputFile(debugger.GetOutputStream().GetFileSP());
+      result->SetImmediateErrorFile(debugger.GetErrorStream().GetFileSP());
+    }
+  }
+
+  if (!m_input_file_sp || !m_output_file_sp || !m_error_file_sp)
+    debugger.AdoptTopIOHandlerFilesIfInvalid(m_input_file_sp, m_output_file_sp,
+                                             m_error_file_sp);
+}
+
+void ScriptInterpreterIORedirect::Flush() {
+  if (m_output_file_sp)
+    m_output_file_sp->Flush();
+  if (m_error_file_sp)
+    m_error_file_sp->Flush();
+}
+
+ScriptInterpreterIORedirect::~ScriptInterpreterIORedirect() {
+  if (!m_disconnect)
+    return;
+
+  assert(m_output_file_sp);
+  assert(m_error_file_sp);
+  assert(m_output_file_sp == m_error_file_sp);
+
+  // Close the write end of the pipe since we are done with our one line
+  // script. This should cause the read thread that output_comm is using to
+  // exit.
+  m_output_file_sp->GetFile().Close();
+  // The close above should cause this thread to exit when it gets to the end
+  // of file, so let it get all its data.
+  m_communication.JoinReadThread();
+  // Now we can close the read end of the pipe.
+  m_communication.Disconnect();
 }
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
index 6f3e8b637cf29..7b0d6f343c030 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
@@ -681,11 +681,16 @@ bool DynamicLoaderDarwin::AddModulesUsingImageInfos(
         loaded_module_list.AppendIfNeeded(image_module_sp);
       }
 
-      // macCatalyst support:
-      // Update the module's platform with the DYLD info.
+      // To support macCatalyst and legacy iOS simulator,
+      // update the module's platform with the DYLD info.
       ArchSpec dyld_spec = image_infos[idx].GetArchitecture();
-      if (dyld_spec.GetTriple().getOS() == llvm::Triple::IOS &&
-          dyld_spec.GetTriple().getEnvironment() == llvm::Triple::MacABI)
+      auto &dyld_triple = dyld_spec.GetTriple();
+      if ((dyld_triple.getEnvironment() == llvm::Triple::MacABI &&
+           dyld_triple.getOS() == llvm::Triple::IOS) ||
+          (dyld_triple.getEnvironment() == llvm::Triple::Simulator &&
+           (dyld_triple.getOS() == llvm::Triple::IOS ||
+            dyld_triple.getOS() == llvm::Triple::TvOS ||
+            dyld_triple.getOS() == llvm::Triple::WatchOS)))
         image_module_sp->MergeArchitecture(dyld_spec);
     }
   }
@@ -748,13 +753,23 @@ lldb_private::ArchSpec DynamicLoaderDarwin::ImageInfo::GetArchitecture() const {
   // Update the module's platform with the DYLD info.
   lldb_private::ArchSpec arch_spec(lldb_private::eArchTypeMachO, header.cputype,
                                    header.cpusubtype);
-  if (os_type == llvm::Triple::IOS && os_env == llvm::Triple::MacABI) {
-    llvm::Triple triple(llvm::Twine("x86_64-apple-ios") + min_version_os_sdk +
-                        "-macabi");
+  if (os_env == llvm::Triple::MacABI && os_type == llvm::Triple::IOS) {
+    llvm::Triple triple(llvm::Twine(arch_spec.GetArchitectureName()) +
+                        "-apple-ios" + min_version_os_sdk + "-macabi");
     ArchSpec maccatalyst_spec(triple);
     if (arch_spec.IsCompatibleMatch(maccatalyst_spec))
       arch_spec.MergeFrom(maccatalyst_spec);
   }
+  if (os_env == llvm::Triple::Simulator &&
+      (os_type == llvm::Triple::IOS || os_type == llvm::Triple::TvOS ||
+       os_type == llvm::Triple::WatchOS)) {
+    llvm::Triple triple(llvm::Twine(arch_spec.GetArchitectureName()) +
+                        "-apple-" + llvm::Triple::getOSTypeName(os_type) +
+                        min_version_os_sdk + "-simulator");
+    ArchSpec sim_spec(triple);
+    if (arch_spec.IsCompatibleMatch(sim_spec))
+      arch_spec.MergeFrom(sim_spec);
+  }
   return arch_spec;
 }
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
index 27efad2144765..ac16738933ac2 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
@@ -18,6 +18,7 @@
 
 #include "Plugins/ExpressionParser/Clang/ClangASTImporter.h"
 #include "Plugins/ExpressionParser/Clang/ClangASTMetadata.h"
+#include "Plugins/ExpressionParser/Clang/ClangASTSource.h"
 #include "Plugins/ExpressionParser/Clang/ClangExternalASTSourceCallbacks.h"
 #include "Plugins/ExpressionParser/Clang/ClangUtil.h"
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
@@ -876,6 +877,33 @@ ClangASTImporter::ASTImporterDelegate::ImportImpl(Decl *From) {
     }
   }
 
+  // If we have a forcefully completed type, try to find an actual definition
+  // for it in other modules.
+  const ClangASTMetadata *md = m_master.GetDeclMetadata(From);
+  auto *td = dyn_cast<TagDecl>(From);
+  if (td && md && md->IsForcefullyCompleted()) {
+    Log *log = GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS);
+    LLDB_LOG(log,
+             "[ClangASTImporter] Searching for a complete definition of {0} in "
+             "other modules",
+             td->getName());
+    Expected<DeclContext *> dc_or_err = ImportContext(td->getDeclContext());
+    if (!dc_or_err)
+      return dc_or_err.takeError();
+    Expected<DeclarationName> dn_or_err = Import(td->getDeclName());
+    if (!dn_or_err)
+      return dn_or_err.takeError();
+    DeclContext *dc = *dc_or_err;
+    DeclContext::lookup_result lr = dc->lookup(*dn_or_err);
+    if (lr.size()) {
+      clang::Decl *lookup_found = lr.front();
+      RegisterImportedDecl(From, lookup_found);
+      m_decls_to_ignore.insert(lookup_found);
+      return lookup_found;
+    } else
+      LLDB_LOG(log, "[ClangASTImporter] Complete definition not found");
+  }
+
   return ASTImporter::ImportImpl(From);
 }
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTMetadata.h b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTMetadata.h
index 6cef11b521104..d3bcde2ced799 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTMetadata.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTMetadata.h
@@ -19,7 +19,8 @@ class ClangASTMetadata {
 public:
   ClangASTMetadata()
       : m_user_id(0), m_union_is_user_id(false), m_union_is_isa_ptr(false),
-        m_has_object_ptr(false), m_is_self(false), m_is_dynamic_cxx(true) {}
+        m_has_object_ptr(false), m_is_self(false), m_is_dynamic_cxx(true),
+        m_is_forcefully_completed(false) {}
 
   bool GetIsDynamicCXXType() const { return m_is_dynamic_cxx; }
 
@@ -83,6 +84,15 @@ class ClangASTMetadata {
 
   bool HasObjectPtr() const { return m_has_object_ptr; }
 
+  /// A type is "forcefully completed" if it was declared complete to satisfy an
+  /// AST invariant (e.g. base classes must be complete types), but in fact we
+  /// were not able to find a actual definition for it.
+  bool IsForcefullyCompleted() const { return m_is_forcefully_completed; }
+
+  void SetIsForcefullyCompleted(bool value = true) {
+    m_is_forcefully_completed = true;
+  }
+
   void Dump(Stream *s);
 
 private:
@@ -92,7 +102,7 @@ class ClangASTMetadata {
   };
 
   bool m_union_is_user_id : 1, m_union_is_isa_ptr : 1, m_has_object_ptr : 1,
-      m_is_self : 1, m_is_dynamic_cxx : 1;
+      m_is_self : 1, m_is_dynamic_cxx : 1, m_is_forcefully_completed : 1;
 };
 
 } // namespace lldb_private
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
index da25fa3ebcb56..6fe85a1298fcb 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
@@ -53,9 +53,9 @@ class ScopedLexicalDeclEraser {
 ClangASTSource::ClangASTSource(
     const lldb::TargetSP &target,
     const std::shared_ptr<ClangASTImporter> &importer)
-    : m_import_in_progress(false), m_lookups_enabled(false), m_target(target),
-      m_ast_context(nullptr), m_ast_importer_sp(importer),
-      m_active_lexical_decls(), m_active_lookups() {
+    : m_lookups_enabled(false), m_target(target), m_ast_context(nullptr),
+      m_ast_importer_sp(importer), m_active_lexical_decls(),
+      m_active_lookups() {
   assert(m_ast_importer_sp && "No ClangASTImporter passed to ClangASTSource?");
 }
 
@@ -103,11 +103,6 @@ bool ClangASTSource::FindExternalVisibleDeclsByName(
     return false;
   }
 
-  if (GetImportInProgress()) {
-    SetNoExternalVisibleDeclsForName(decl_ctx, clang_decl_name);
-    return false;
-  }
-
   std::string decl_name(clang_decl_name.getAsString());
 
   switch (clang_decl_name.getNameKind()) {
@@ -185,127 +180,125 @@ bool ClangASTSource::FindExternalVisibleDeclsByName(
   return (name_decls.size() != 0);
 }
 
-void ClangASTSource::CompleteType(TagDecl *tag_decl) {
+TagDecl *ClangASTSource::FindCompleteType(const TagDecl *decl) {
   Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS));
 
-  if (log) {
-    LLDB_LOG(log,
-             "    CompleteTagDecl on (ASTContext*){0} Completing "
-             "(TagDecl*){1} named {2}",
-             m_clang_ast_context->getDisplayName(), tag_decl,
-             tag_decl->getName());
-
-    LLDB_LOG(log, "      CTD Before:\n{0}", ClangUtil::DumpDecl(tag_decl));
-  }
+  if (const NamespaceDecl *namespace_context =
+          dyn_cast<NamespaceDecl>(decl->getDeclContext())) {
+    ClangASTImporter::NamespaceMapSP namespace_map =
+        m_ast_importer_sp->GetNamespaceMap(namespace_context);
 
-  auto iter = m_active_lexical_decls.find(tag_decl);
-  if (iter != m_active_lexical_decls.end())
-    return;
-  m_active_lexical_decls.insert(tag_decl);
-  ScopedLexicalDeclEraser eraser(m_active_lexical_decls, tag_decl);
+    LLDB_LOGV(log, "      CTD Inspecting namespace map{0} ({1} entries)",
+              namespace_map.get(), namespace_map->size());
 
-  if (!m_ast_importer_sp->CompleteTagDecl(tag_decl)) {
-    // We couldn't complete the type.  Maybe there's a definition somewhere
-    // else that can be completed.
+    if (!namespace_map)
+      return nullptr;
 
-    LLDB_LOG(log, "      CTD Type could not be completed in the module in "
-                  "which it was first found.");
+    for (const ClangASTImporter::NamespaceMapItem &item : *namespace_map) {
+      LLDB_LOG(log, "      CTD Searching namespace {0} in module {1}",
+               item.second.GetName(), item.first->GetFileSpec().GetFilename());
 
-    bool found = false;
-
-    DeclContext *decl_ctx = tag_decl->getDeclContext();
+      TypeList types;
 
-    if (const NamespaceDecl *namespace_context =
-            dyn_cast<NamespaceDecl>(decl_ctx)) {
-      ClangASTImporter::NamespaceMapSP namespace_map =
-          m_ast_importer_sp->GetNamespaceMap(namespace_context);
+      ConstString name(decl->getName());
 
-      LLDB_LOGV(log, "      CTD Inspecting namespace map{0} ({1} entries)",
-                namespace_map.get(), namespace_map->size());
+      item.first->FindTypesInNamespace(name, item.second, UINT32_MAX, types);
 
-      if (!namespace_map)
-        return;
+      for (uint32_t ti = 0, te = types.GetSize(); ti != te; ++ti) {
+        lldb::TypeSP type = types.GetTypeAtIndex(ti);
 
-      for (ClangASTImporter::NamespaceMap::iterator i = namespace_map->begin(),
-                                                    e = namespace_map->end();
-           i != e && !found; ++i) {
-        LLDB_LOG(log, "      CTD Searching namespace {0} in module {1}",
-                 i->second.GetName(), i->first->GetFileSpec().GetFilename());
+        if (!type)
+          continue;
 
-        TypeList types;
+        CompilerType clang_type(type->GetFullCompilerType());
 
-        ConstString name(tag_decl->getName().str().c_str());
+        if (!ClangUtil::IsClangType(clang_type))
+          continue;
 
-        i->first->FindTypesInNamespace(name, i->second, UINT32_MAX, types);
+        const TagType *tag_type =
+            ClangUtil::GetQualType(clang_type)->getAs<TagType>();
 
-        for (uint32_t ti = 0, te = types.GetSize(); ti != te && !found; ++ti) {
-          lldb::TypeSP type = types.GetTypeAtIndex(ti);
+        if (!tag_type)
+          continue;
 
-          if (!type)
-            continue;
+        TagDecl *candidate_tag_decl =
+            const_cast<TagDecl *>(tag_type->getDecl());
 
-          CompilerType clang_type(type->GetFullCompilerType());
+        if (TypeSystemClang::GetCompleteDecl(
+                &candidate_tag_decl->getASTContext(), candidate_tag_decl))
+          return candidate_tag_decl;
+      }
+    }
+  } else {
+    TypeList types;
 
-          if (!ClangUtil::IsClangType(clang_type))
-            continue;
+    ConstString name(decl->getName());
 
-          const TagType *tag_type =
-              ClangUtil::GetQualType(clang_type)->getAs<TagType>();
+    const ModuleList &module_list = m_target->GetImages();
 
-          if (!tag_type)
-            continue;
+    bool exact_match = false;
+    llvm::DenseSet<SymbolFile *> searched_symbol_files;
+    module_list.FindTypes(nullptr, name, exact_match, UINT32_MAX,
+                          searched_symbol_files, types);
 
-          TagDecl *candidate_tag_decl =
-              const_cast<TagDecl *>(tag_type->getDecl());
+    for (uint32_t ti = 0, te = types.GetSize(); ti != te; ++ti) {
+      lldb::TypeSP type = types.GetTypeAtIndex(ti);
 
-          if (m_ast_importer_sp->CompleteTagDeclWithOrigin(tag_decl,
-                                                           candidate_tag_decl))
-            found = true;
-        }
-      }
-    } else {
-      TypeList types;
+      if (!type)
+        continue;
 
-      ConstString name(tag_decl->getName().str().c_str());
+      CompilerType clang_type(type->GetFullCompilerType());
 
-      const ModuleList &module_list = m_target->GetImages();
+      if (!ClangUtil::IsClangType(clang_type))
+        continue;
 
-      bool exact_match = false;
-      llvm::DenseSet<SymbolFile *> searched_symbol_files;
-      module_list.FindTypes(nullptr, name, exact_match, UINT32_MAX,
-                            searched_symbol_files, types);
+      const TagType *tag_type =
+          ClangUtil::GetQualType(clang_type)->getAs<TagType>();
 
-      for (uint32_t ti = 0, te = types.GetSize(); ti != te && !found; ++ti) {
-        lldb::TypeSP type = types.GetTypeAtIndex(ti);
+      if (!tag_type)
+        continue;
 
-        if (!type)
-          continue;
+      TagDecl *candidate_tag_decl = const_cast<TagDecl *>(tag_type->getDecl());
 
-        CompilerType clang_type(type->GetFullCompilerType());
+      // We have found a type by basename and we need to make sure the decl
+      // contexts are the same before we can try to complete this type with
+      // another
+      if (!TypeSystemClang::DeclsAreEquivalent(const_cast<TagDecl *>(decl),
+                                               candidate_tag_decl))
+        continue;
 
-        if (!ClangUtil::IsClangType(clang_type))
-          continue;
+      if (TypeSystemClang::GetCompleteDecl(&candidate_tag_decl->getASTContext(),
+                                           candidate_tag_decl))
+        return candidate_tag_decl;
+    }
+  }
+  return nullptr;
+}
 
-        const TagType *tag_type =
-            ClangUtil::GetQualType(clang_type)->getAs<TagType>();
+void ClangASTSource::CompleteType(TagDecl *tag_decl) {
+  Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS));
 
-        if (!tag_type)
-          continue;
+  if (log) {
+    LLDB_LOG(log,
+             "    CompleteTagDecl on (ASTContext*){0} Completing "
+             "(TagDecl*){1} named {2}",
+             m_clang_ast_context->getDisplayName(), tag_decl,
+             tag_decl->getName());
 
-        TagDecl *candidate_tag_decl =
-            const_cast<TagDecl *>(tag_type->getDecl());
+    LLDB_LOG(log, "      CTD Before:\n{0}", ClangUtil::DumpDecl(tag_decl));
+  }
 
-        // We have found a type by basename and we need to make sure the decl
-        // contexts are the same before we can try to complete this type with
-        // another
-        if (!TypeSystemClang::DeclsAreEquivalent(tag_decl, candidate_tag_decl))
-          continue;
+  auto iter = m_active_lexical_decls.find(tag_decl);
+  if (iter != m_active_lexical_decls.end())
+    return;
+  m_active_lexical_decls.insert(tag_decl);
+  ScopedLexicalDeclEraser eraser(m_active_lexical_decls, tag_decl);
 
-        if (m_ast_importer_sp->CompleteTagDeclWithOrigin(tag_decl,
-                                                         candidate_tag_decl))
-          found = true;
-      }
-    }
+  if (!m_ast_importer_sp->CompleteTagDecl(tag_decl)) {
+    // We couldn't complete the type.  Maybe there's a definition somewhere
+    // else that can be completed.
+    if (TagDecl *alternate = FindCompleteType(tag_decl))
+      m_ast_importer_sp->CompleteTagDeclWithOrigin(tag_decl, alternate);
   }
 
   LLDB_LOG(log, "      [CTD] After:\n{0}", ClangUtil::DumpDecl(tag_decl));
@@ -1746,13 +1739,9 @@ CompilerType ClangASTSource::GuardedCopyType(const CompilerType &src_type) {
   if (src_ast == nullptr)
     return CompilerType();
 
-  SetImportInProgress(true);
-
   QualType copied_qual_type = ClangUtil::GetQualType(
       m_ast_importer_sp->CopyType(*m_clang_ast_context, src_type));
 
-  SetImportInProgress(false);
-
   if (copied_qual_type.getAsOpaquePtr() &&
       copied_qual_type->getCanonicalTypeInternal().isNull())
     // this shouldn't happen, but we're hardening because the AST importer
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.h b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.h
index 2563c6499de3b..14761fbeb26b4 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.h
@@ -197,11 +197,6 @@ class ClangASTSource : public clang::ExternalASTSource,
 
   clang::Sema *getSema();
 
-  void SetImportInProgress(bool import_in_progress) {
-    m_import_in_progress = import_in_progress;
-  }
-  bool GetImportInProgress() { return m_import_in_progress; }
-
   void SetLookupsEnabled(bool lookups_enabled) {
     m_lookups_enabled = lookups_enabled;
   }
@@ -372,9 +367,10 @@ class ClangASTSource : public clang::ExternalASTSource,
   void FillNamespaceMap(NameSearchContext &context, lldb::ModuleSP module_sp,
                         const CompilerDeclContext &namespace_decl);
 
+  clang::TagDecl *FindCompleteType(const clang::TagDecl *decl);
+
   friend struct NameSearchContext;
 
-  bool m_import_in_progress;
   bool m_lookups_enabled;
 
   /// The target to use in finding variables and types.
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
index d71036f793b1d..8c49898e1d6ca 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
@@ -632,11 +632,6 @@ void ClangExpressionDeclMap::FindExternalVisibleDecls(
 
   Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS));
 
-  if (GetImportInProgress()) {
-    LLDB_LOGV(log, "Ignoring a query during an import");
-    return;
-  }
-
   if (log) {
     if (!context.m_decl_context)
       LLDB_LOG(log,
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
index 08b1ae0ccde2b..6ff028cf6980f 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
@@ -158,12 +158,12 @@ static void AddAllFixIts(ClangDiagnostic *diag, const clang::Diagnostic &Info) {
 class ClangDiagnosticManagerAdapter : public clang::DiagnosticConsumer {
 public:
   ClangDiagnosticManagerAdapter(DiagnosticOptions &opts) {
-    DiagnosticOptions *m_options = new DiagnosticOptions(opts);
-    m_options->ShowPresumedLoc = true;
-    m_options->ShowLevel = false;
+    DiagnosticOptions *options = new DiagnosticOptions(opts);
+    options->ShowPresumedLoc = true;
+    options->ShowLevel = false;
     m_os = std::make_shared<llvm::raw_string_ostream>(m_output);
     m_passthrough =
-        std::make_shared<clang::TextDiagnosticPrinter>(*m_os, m_options);
+        std::make_shared<clang::TextDiagnosticPrinter>(*m_os, options);
   }
 
   void ResetManager(DiagnosticManager *manager = nullptr) {
@@ -200,6 +200,9 @@ class ClangDiagnosticManagerAdapter : public clang::DiagnosticConsumer {
       return;
     }
 
+    // Update error/warning counters.
+    DiagnosticConsumer::HandleDiagnostic(DiagLevel, Info);
+
     // Render diagnostic message to m_output.
     m_output.clear();
     m_passthrough->HandleDiagnostic(DiagLevel, Info);
@@ -261,7 +264,11 @@ class ClangDiagnosticManagerAdapter : public clang::DiagnosticConsumer {
     }
   }
 
-  clang::TextDiagnosticPrinter *GetPassthrough() { return m_passthrough.get(); }
+  void BeginSourceFile(const LangOptions &LO, const Preprocessor *PP) override {
+    m_passthrough->BeginSourceFile(LO, PP);
+  }
+
+  void EndSourceFile() override { m_passthrough->EndSourceFile(); }
 
 private:
   DiagnosticManager *m_manager = nullptr;
@@ -331,7 +338,7 @@ ClangExpressionParser::ClangExpressionParser(
   }
 
   // 1. Create a new compiler instance.
-  m_compiler.reset(new CompilerInstance());
+  m_compiler = std::make_unique<CompilerInstance>();
 
   // When capturing a reproducer, hook up the file collector with clang to
   // collector modules and headers.
@@ -641,12 +648,12 @@ ClangExpressionParser::ClangExpressionParser(
   m_compiler->createASTContext();
   clang::ASTContext &ast_context = m_compiler->getASTContext();
 
-  m_ast_context.reset(new TypeSystemClang(
-      "Expression ASTContext for '" + m_filename + "'", ast_context));
+  m_ast_context = std::make_unique<TypeSystemClang>(
+      "Expression ASTContext for '" + m_filename + "'", ast_context);
 
   std::string module_name("$__lldb_module");
 
-  m_llvm_context.reset(new LLVMContext());
+  m_llvm_context = std::make_unique<LLVMContext>();
   m_code_generator.reset(CreateLLVMCodeGen(
       m_compiler->getDiagnostics(), module_name,
       m_compiler->getHeaderSearchOpts(), m_compiler->getPreprocessorOpts(),
@@ -967,7 +974,6 @@ ClangExpressionParser::ParseInternal(DiagnosticManager &diagnostic_manager,
   ClangDiagnosticManagerAdapter *adapter =
       static_cast<ClangDiagnosticManagerAdapter *>(
           m_compiler->getDiagnostics().getClient());
-  auto diag_buf = adapter->GetPassthrough();
 
   adapter->ResetManager(&diagnostic_manager);
 
@@ -1022,8 +1028,8 @@ ClangExpressionParser::ParseInternal(DiagnosticManager &diagnostic_manager,
     source_mgr.setMainFileID(source_mgr.createFileID(std::move(memory_buffer)));
   }
 
-  diag_buf->BeginSourceFile(m_compiler->getLangOpts(),
-                            &m_compiler->getPreprocessor());
+  adapter->BeginSourceFile(m_compiler->getLangOpts(),
+                           &m_compiler->getPreprocessor());
 
   ClangExpressionHelper *type_system_helper =
       dyn_cast<ClangExpressionHelper>(m_expr.GetTypeSystemHelper());
@@ -1047,11 +1053,11 @@ ClangExpressionParser::ParseInternal(DiagnosticManager &diagnostic_manager,
 
   std::unique_ptr<clang::ASTConsumer> Consumer;
   if (ast_transformer) {
-    Consumer.reset(new ASTConsumerForwarder(ast_transformer));
+    Consumer = std::make_unique<ASTConsumerForwarder>(ast_transformer);
   } else if (m_code_generator) {
-    Consumer.reset(new ASTConsumerForwarder(m_code_generator.get()));
+    Consumer = std::make_unique<ASTConsumerForwarder>(m_code_generator.get());
   } else {
-    Consumer.reset(new ASTConsumer());
+    Consumer = std::make_unique<ASTConsumer>();
   }
 
   clang::ASTContext &ast_context = m_compiler->getASTContext();
@@ -1108,9 +1114,9 @@ ClangExpressionParser::ParseInternal(DiagnosticManager &diagnostic_manager,
   // original behavior of ParseAST (which also destroys the Sema after parsing).
   m_compiler->setSema(nullptr);
 
-  diag_buf->EndSourceFile();
+  adapter->EndSourceFile();
 
-  unsigned num_errors = diag_buf->getNumErrors();
+  unsigned num_errors = adapter->getNumErrors();
 
   if (m_pp_callbacks && m_pp_callbacks->hasErrors()) {
     num_errors++;
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangFunctionCaller.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangFunctionCaller.cpp
index ea56e429a9d50..0c9ad20210357 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangFunctionCaller.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangFunctionCaller.cpp
@@ -209,8 +209,8 @@ ClangFunctionCaller::CompileFunction(lldb::ThreadSP thread_to_use_sp,
 clang::ASTConsumer *
 ClangFunctionCaller::ClangFunctionCallerHelper::ASTTransformer(
     clang::ASTConsumer *passthrough) {
-  m_struct_extractor.reset(new ASTStructExtractor(
-      passthrough, m_owner.GetWrapperStructName(), m_owner));
+  m_struct_extractor = std::make_unique<ASTStructExtractor>(
+      passthrough, m_owner.GetWrapperStructName(), m_owner);
 
   return m_struct_extractor.get();
 }
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp
index f915a699a8e2b..95acb883774d8 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp
@@ -187,9 +187,9 @@ ClangModulesDeclVendorImpl::ClangModulesDeclVendorImpl(
       m_parser(std::move(parser)) {
 
   // Initialize our TypeSystemClang.
-  m_ast_context.reset(
-      new TypeSystemClang("ClangModulesDeclVendor ASTContext",
-                          m_compiler_instance->getASTContext()));
+  m_ast_context =
+      std::make_unique<TypeSystemClang>("ClangModulesDeclVendor ASTContext",
+                                        m_compiler_instance->getASTContext());
 }
 
 void ClangModulesDeclVendorImpl::ReportModuleExportsHelper(
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
index 2d6de782ced2f..a28b4a7fb42cc 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
@@ -595,7 +595,7 @@ bool ClangUserExpression::Parse(DiagnosticManager &diagnostic_manager,
   // Parse the expression
   //
 
-  m_materializer_up.reset(new Materializer());
+  m_materializer_up = std::make_unique<Materializer>();
 
   ResetDeclMap(exe_ctx, m_result_delegate, keep_result_in_memory);
 
@@ -783,7 +783,7 @@ bool ClangUserExpression::Complete(ExecutionContext &exe_ctx,
   // Parse the expression
   //
 
-  m_materializer_up.reset(new Materializer());
+  m_materializer_up = std::make_unique<Materializer>();
 
   ResetDeclMap(exe_ctx, m_result_delegate, /*keep result in memory*/ true);
 
@@ -919,16 +919,16 @@ void ClangUserExpression::ClangUserExpressionHelper::ResetDeclMap(
     auto *persistent_vars = llvm::cast<ClangPersistentVariables>(state);
     ast_importer = persistent_vars->GetClangASTImporter();
   }
-  m_expr_decl_map_up.reset(
-      new ClangExpressionDeclMap(keep_result_in_memory, &delegate,
-                                 exe_ctx.GetTargetSP(), ast_importer, ctx_obj));
+  m_expr_decl_map_up = std::make_unique<ClangExpressionDeclMap>(
+      keep_result_in_memory, &delegate, exe_ctx.GetTargetSP(), ast_importer,
+      ctx_obj);
 }
 
 clang::ASTConsumer *
 ClangUserExpression::ClangUserExpressionHelper::ASTTransformer(
     clang::ASTConsumer *passthrough) {
-  m_result_synthesizer_up.reset(
-      new ASTResultSynthesizer(passthrough, m_top_level, m_target));
+  m_result_synthesizer_up = std::make_unique<ASTResultSynthesizer>(
+      passthrough, m_top_level, m_target);
 
   return m_result_synthesizer_up.get();
 }
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangUtilityFunction.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangUtilityFunction.cpp
index ae519658e5019..25ec982220a07 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangUtilityFunction.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangUtilityFunction.cpp
@@ -167,7 +167,7 @@ void ClangUtilityFunction::ClangUtilityFunctionHelper::ResetDeclMap(
     auto *persistent_vars = llvm::cast<ClangPersistentVariables>(state);
     ast_importer = persistent_vars->GetClangASTImporter();
   }
-  m_expr_decl_map_up.reset(
-      new ClangExpressionDeclMap(keep_result_in_memory, nullptr,
-                                 exe_ctx.GetTargetSP(), ast_importer, nullptr));
+  m_expr_decl_map_up = std::make_unique<ClangExpressionDeclMap>(
+      keep_result_in_memory, nullptr, exe_ctx.GetTargetSP(), ast_importer,
+      nullptr);
 }
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/CxxModuleHandler.cpp b/lldb/source/Plugins/ExpressionParser/Clang/CxxModuleHandler.cpp
index 8fdce858a57cf..2f8cf1846ee77 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/CxxModuleHandler.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/CxxModuleHandler.cpp
@@ -66,9 +66,9 @@ emulateLookupInCtxt(Sema &sema, llvm::StringRef name, DeclContext *ctxt) {
   IdentifierInfo &ident = sema.getASTContext().Idents.get(name);
 
   std::unique_ptr<LookupResult> lookup_result;
-  lookup_result.reset(new LookupResult(sema, DeclarationName(&ident),
-                                       SourceLocation(),
-                                       Sema::LookupOrdinaryName));
+  lookup_result = std::make_unique<LookupResult>(sema, DeclarationName(&ident),
+                                                 SourceLocation(),
+                                                 Sema::LookupOrdinaryName);
 
   // Usually during parsing we already encountered the scopes we would use. But
   // here don't have these scopes so we have to emulate the behavior of the
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp b/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
index 8819b8a773f59..8511e554509ac 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
@@ -1842,7 +1842,7 @@ bool IRForTarget::runOnModule(Module &llvm_module) {
       lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS));
 
   m_module = &llvm_module;
-  m_target_data.reset(new DataLayout(m_module));
+  m_target_data = std::make_unique<DataLayout>(m_module);
   m_intptr_ty = llvm::Type::getIntNTy(m_module->getContext(),
                                       m_target_data->getPointerSizeInBits());
 
diff --git a/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.cpp b/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.cpp
index 9837e33773d96..555912780df96 100644
--- a/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.cpp
+++ b/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.cpp
@@ -14361,7 +14361,7 @@ bool EmulateInstructionARM::EvaluateInstruction(uint32_t evaluate_options) {
     if (!success)
       return false;
 
-    if (auto_advance_pc && (after_pc_value == orig_pc_value)) {
+    if (after_pc_value == orig_pc_value) {
       after_pc_value += m_opcode.GetByteSize();
 
       EmulateInstruction::Context context;
diff --git a/lldb/source/Plugins/Instruction/ARM64/EmulateInstructionARM64.cpp b/lldb/source/Plugins/Instruction/ARM64/EmulateInstructionARM64.cpp
index 35fb51759ca86..9b0c06bcccab0 100644
--- a/lldb/source/Plugins/Instruction/ARM64/EmulateInstructionARM64.cpp
+++ b/lldb/source/Plugins/Instruction/ARM64/EmulateInstructionARM64.cpp
@@ -433,7 +433,7 @@ bool EmulateInstructionARM64::EvaluateInstruction(uint32_t evaluate_options) {
     if (!success)
       return false;
 
-    if (auto_advance_pc && (new_pc_value == orig_pc_value)) {
+    if (new_pc_value == orig_pc_value) {
       EmulateInstruction::Context context;
       context.type = eContextAdvancePC;
       context.SetNoArgs();
diff --git a/lldb/source/Plugins/Instruction/MIPS/EmulateInstructionMIPS.cpp b/lldb/source/Plugins/Instruction/MIPS/EmulateInstructionMIPS.cpp
index 61eb8912295f9..d4cb726fc7e53 100644
--- a/lldb/source/Plugins/Instruction/MIPS/EmulateInstructionMIPS.cpp
+++ b/lldb/source/Plugins/Instruction/MIPS/EmulateInstructionMIPS.cpp
@@ -159,8 +159,8 @@ EmulateInstructionMIPS::EmulateInstructionMIPS(
       target->createMCSubtargetInfo(triple.getTriple(), cpu, features));
   assert(m_asm_info.get() && m_subtype_info.get());
 
-  m_context.reset(
-      new llvm::MCContext(m_asm_info.get(), m_reg_info.get(), nullptr));
+  m_context = std::make_unique<llvm::MCContext>(m_asm_info.get(),
+                                                m_reg_info.get(), nullptr);
   assert(m_context.get());
 
   m_disasm.reset(target->createMCDisassembler(*m_subtype_info, *m_context));
diff --git a/lldb/source/Plugins/Instruction/MIPS64/EmulateInstructionMIPS64.cpp b/lldb/source/Plugins/Instruction/MIPS64/EmulateInstructionMIPS64.cpp
index b8305591749b2..4ccaf0de0758b 100644
--- a/lldb/source/Plugins/Instruction/MIPS64/EmulateInstructionMIPS64.cpp
+++ b/lldb/source/Plugins/Instruction/MIPS64/EmulateInstructionMIPS64.cpp
@@ -163,8 +163,8 @@ EmulateInstructionMIPS64::EmulateInstructionMIPS64(
       target->createMCSubtargetInfo(triple.getTriple(), cpu, features));
   assert(m_asm_info.get() && m_subtype_info.get());
 
-  m_context.reset(
-      new llvm::MCContext(m_asm_info.get(), m_reg_info.get(), nullptr));
+  m_context = std::make_unique<llvm::MCContext>(m_asm_info.get(),
+                                                m_reg_info.get(), nullptr);
   assert(m_context.get());
 
   m_disasm.reset(target->createMCDisassembler(*m_subtype_info, *m_context));
diff --git a/lldb/source/Plugins/Instruction/PPC64/EmulateInstructionPPC64.cpp b/lldb/source/Plugins/Instruction/PPC64/EmulateInstructionPPC64.cpp
index 1cab8fdd6e2ac..5d97513c0be73 100644
--- a/lldb/source/Plugins/Instruction/PPC64/EmulateInstructionPPC64.cpp
+++ b/lldb/source/Plugins/Instruction/PPC64/EmulateInstructionPPC64.cpp
@@ -198,7 +198,7 @@ bool EmulateInstructionPPC64::EvaluateInstruction(uint32_t evaluate_options) {
     if (!success)
       return false;
 
-    if (auto_advance_pc && (new_pc_value == orig_pc_value)) {
+    if (new_pc_value == orig_pc_value) {
       EmulateInstruction::Context context;
       context.type = eContextAdvancePC;
       context.SetNoArgs();
diff --git a/lldb/source/Plugins/Language/ObjC/CF.cpp b/lldb/source/Plugins/Language/ObjC/CF.cpp
index 9e4eb9d53d477..2610468b17fd9 100644
--- a/lldb/source/Plugins/Language/ObjC/CF.cpp
+++ b/lldb/source/Plugins/Language/ObjC/CF.cpp
@@ -29,7 +29,7 @@ using namespace lldb_private::formatters;
 bool lldb_private::formatters::CFAbsoluteTimeSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
   time_t epoch = GetOSXEpoch();
-  epoch = epoch + (time_t)valobj.GetValueAsUnsigned(0);
+  epoch = epoch + (time_t)valobj.GetValueAsSigned(0);
   tm *tm_date = localtime(&epoch);
   if (!tm_date)
     return false;
diff --git a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
index 8a44811dd36b9..da910f48e59a2 100644
--- a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
+++ b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
@@ -867,7 +867,7 @@ bool lldb_private::formatters::NSDateSummaryProvider(
   // is generally true and POSIXly happy, but might break if a library vendor
   // decides to get creative
   time_t epoch = GetOSXEpoch();
-  epoch = epoch + (time_t)date_value;
+  epoch = epoch + static_cast<time_t>(std::floor(date_value));
   tm *tm_date = gmtime(&epoch);
   if (!tm_date)
     return false;
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCClassDescriptorV2.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCClassDescriptorV2.cpp
index 2cf98620687c7..bdd5c29db8484 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCClassDescriptorV2.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCClassDescriptorV2.cpp
@@ -16,7 +16,7 @@ using namespace lldb_private;
 
 bool ClassDescriptorV2::Read_objc_class(
     Process *process, std::unique_ptr<objc_class_t> &objc_class) const {
-  objc_class.reset(new objc_class_t);
+  objc_class = std::make_unique<objc_class_t>();
 
   bool ret = objc_class->Read(process, m_objc_class_ptr);
 
@@ -197,14 +197,14 @@ bool ClassDescriptorV2::Read_class_row(
     return false;
 
   if (class_row_t_flags & RW_REALIZED) {
-    class_rw.reset(new class_rw_t);
+    class_rw = std::make_unique<class_rw_t>();
 
     if (!class_rw->Read(process, objc_class.m_data_ptr)) {
       class_rw.reset();
       return false;
     }
 
-    class_ro.reset(new class_ro_t);
+    class_ro = std::make_unique<class_ro_t>();
 
     if (!class_ro->Read(process, class_rw->m_ro_ptr)) {
       class_rw.reset();
@@ -212,7 +212,7 @@ bool ClassDescriptorV2::Read_class_row(
       return false;
     }
   } else {
-    class_ro.reset(new class_ro_t);
+    class_ro = std::make_unique<class_ro_t>();
 
     if (!class_ro->Read(process, objc_class.m_data_ptr)) {
       class_ro.reset();
@@ -241,15 +241,20 @@ bool ClassDescriptorV2::method_list_t::Read(Process *process,
 
   lldb::offset_t cursor = 0;
 
-  m_entsize = extractor.GetU32_unchecked(&cursor) & ~(uint32_t)3;
+  uint32_t entsize = extractor.GetU32_unchecked(&cursor);
+  m_is_small = (entsize & 0x80000000) != 0;
+  m_has_direct_selector = (entsize & 0x40000000) != 0;
+  m_entsize = entsize & 0xfffc;
   m_count = extractor.GetU32_unchecked(&cursor);
   m_first_ptr = addr + cursor;
 
   return true;
 }
 
-bool ClassDescriptorV2::method_t::Read(Process *process, lldb::addr_t addr) {
-  size_t size = GetSize(process);
+bool ClassDescriptorV2::method_t::Read(Process *process, lldb::addr_t addr,
+                                       bool is_small, bool has_direct_sel) {
+  size_t ptr_size = process->GetAddressByteSize();
+  size_t size = GetSize(process, is_small);
 
   DataBufferHeap buffer(size, '\0');
   Status error;
@@ -260,13 +265,30 @@ bool ClassDescriptorV2::method_t::Read(Process *process, lldb::addr_t addr) {
   }
 
   DataExtractor extractor(buffer.GetBytes(), size, process->GetByteOrder(),
-                          process->GetAddressByteSize());
-
+                          ptr_size);
   lldb::offset_t cursor = 0;
 
-  m_name_ptr = extractor.GetAddress_unchecked(&cursor);
-  m_types_ptr = extractor.GetAddress_unchecked(&cursor);
-  m_imp_ptr = extractor.GetAddress_unchecked(&cursor);
+  if (is_small) {
+    uint32_t nameref_offset = extractor.GetU32_unchecked(&cursor);
+    uint32_t types_offset = extractor.GetU32_unchecked(&cursor);
+    uint32_t imp_offset = extractor.GetU32_unchecked(&cursor);
+
+    m_name_ptr = addr + nameref_offset;
+
+    if (!has_direct_sel) {
+      // The SEL offset points to a SELRef. We need to dereference twice.
+      m_name_ptr = process->ReadUnsignedIntegerFromMemory(m_name_ptr, ptr_size,
+                                                          0, error);
+      if (!error.Success())
+        return false;
+    }
+    m_types_ptr = addr + 4 + types_offset;
+    m_imp_ptr = addr + 8 + imp_offset;
+  } else {
+    m_name_ptr = extractor.GetAddress_unchecked(&cursor);
+    m_types_ptr = extractor.GetAddress_unchecked(&cursor);
+    m_imp_ptr = extractor.GetAddress_unchecked(&cursor);
+  }
 
   process->ReadCStringFromMemory(m_name_ptr, m_name, error);
   if (error.Fail()) {
@@ -357,19 +379,24 @@ bool ClassDescriptorV2::Describe(
   if (instance_method_func) {
     std::unique_ptr<method_list_t> base_method_list;
 
-    base_method_list.reset(new method_list_t);
+    base_method_list = std::make_unique<method_list_t>();
     if (!base_method_list->Read(process, class_ro->m_baseMethods_ptr))
       return false;
 
-    if (base_method_list->m_entsize != method_t::GetSize(process))
+    bool is_small = base_method_list->m_is_small;
+    bool has_direct_selector = base_method_list->m_has_direct_selector;
+
+    if (base_method_list->m_entsize != method_t::GetSize(process, is_small))
       return false;
 
     std::unique_ptr<method_t> method;
-    method.reset(new method_t);
+    method = std::make_unique<method_t>();
 
     for (uint32_t i = 0, e = base_method_list->m_count; i < e; ++i) {
-      method->Read(process, base_method_list->m_first_ptr +
-                                (i * base_method_list->m_entsize));
+      method->Read(process,
+                   base_method_list->m_first_ptr +
+                       (i * base_method_list->m_entsize),
+                   is_small, has_direct_selector);
 
       if (instance_method_func(method->m_name.c_str(), method->m_types.c_str()))
         break;
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCClassDescriptorV2.h b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCClassDescriptorV2.h
index 56728cff5b3f1..9ef21c6e72081 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCClassDescriptorV2.h
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCClassDescriptorV2.h
@@ -133,7 +133,9 @@ class ClassDescriptorV2 : public ObjCLanguageRuntime::ClassDescriptor {
   };
 
   struct method_list_t {
-    uint32_t m_entsize;
+    uint16_t m_entsize;
+    bool m_is_small;
+    bool m_has_direct_selector;
     uint32_t m_count;
     lldb::addr_t m_first_ptr;
 
@@ -148,15 +150,19 @@ class ClassDescriptorV2 : public ObjCLanguageRuntime::ClassDescriptor {
     std::string m_name;
     std::string m_types;
 
-    static size_t GetSize(Process *process) {
-      size_t ptr_size = process->GetAddressByteSize();
+    static size_t GetSize(Process *process, bool is_small) {
+      size_t field_size;
+      if (is_small)
+        field_size = 4; // uint32_t relative indirect fields
+      else
+        field_size = process->GetAddressByteSize();
 
-      return ptr_size    // SEL name;
-             + ptr_size  // const char *types;
-             + ptr_size; // IMP imp;
+      return field_size    // SEL name;
+             + field_size  // const char *types;
+             + field_size; // IMP imp;
     }
 
-    bool Read(Process *process, lldb::addr_t addr);
+    bool Read(Process *process, lldb::addr_t addr, bool, bool);
   };
 
   struct ivar_list_t {
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
index e00ce9d019dfb..22ea83a57beb7 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
@@ -250,7 +250,8 @@ Address *AppleObjCRuntime::GetPrintForDebuggerAddr() {
 
     contexts.GetContextAtIndex(0, context);
 
-    m_PrintForDebugger_addr.reset(new Address(context.symbol->GetAddress()));
+    m_PrintForDebugger_addr =
+        std::make_unique<Address>(context.symbol->GetAddress());
   }
 
   return m_PrintForDebugger_addr.get();
@@ -346,8 +347,8 @@ bool AppleObjCRuntime::ReadObjCLibrary(const ModuleSP &module_sp) {
   // Maybe check here and if we have a handler already, and the UUID of this
   // module is the same as the one in the current module, then we don't have to
   // reread it?
-  m_objc_trampoline_handler_up.reset(
-      new AppleObjCTrampolineHandler(m_process->shared_from_this(), module_sp));
+  m_objc_trampoline_handler_up = std::make_unique<AppleObjCTrampolineHandler>(
+      m_process->shared_from_this(), module_sp);
   if (m_objc_trampoline_handler_up != nullptr) {
     m_read_objc_library = true;
     return true;
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
index 4a07c792eebba..ac9a093940217 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
@@ -581,8 +581,8 @@ class CommandObjectObjC_ClassTable_Dump : public CommandObjectParsed {
     case 0:
       break;
     case 1: {
-      regex_up.reset(new RegularExpression(
-          llvm::StringRef::withNullAsEmpty(command.GetArgumentAtIndex(0))));
+      regex_up = std::make_unique<RegularExpression>(
+          llvm::StringRef::withNullAsEmpty(command.GetArgumentAtIndex(0)));
       if (!regex_up->IsValid()) {
         result.AppendError(
             "invalid argument - please provide a valid regular expression");
@@ -2006,7 +2006,7 @@ void AppleObjCRuntimeV2::WarnIfNoClassesCached(
 
 DeclVendor *AppleObjCRuntimeV2::GetDeclVendor() {
   if (!m_decl_vendor_up)
-    m_decl_vendor_up.reset(new AppleObjCDeclVendor(*this));
+    m_decl_vendor_up = std::make_unique<AppleObjCDeclVendor>(*this);
 
   return m_decl_vendor_up.get();
 }
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTrampolineHandler.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTrampolineHandler.cpp
index 0fe9dc35a9a75..c96768c9f585f 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTrampolineHandler.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTrampolineHandler.cpp
@@ -787,7 +787,8 @@ AppleObjCTrampolineHandler::AppleObjCTrampolineHandler(
   }
 
   // Build our vtable dispatch handler here:
-  m_vtables_up.reset(new AppleObjCVTables(process_sp, m_objc_module_sp));
+  m_vtables_up =
+      std::make_unique<AppleObjCVTables>(process_sp, m_objc_module_sp);
   if (m_vtables_up)
     m_vtables_up->ReadRegions();
 }
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTypeEncodingParser.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTypeEncodingParser.cpp
index ba76f68f3f54c..b19d3d90d4b72 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTypeEncodingParser.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTypeEncodingParser.cpp
@@ -25,9 +25,9 @@ AppleObjCTypeEncodingParser::AppleObjCTypeEncodingParser(
     ObjCLanguageRuntime &runtime)
     : ObjCLanguageRuntime::EncodingToType(), m_runtime(runtime) {
   if (!m_scratch_ast_ctx_up)
-    m_scratch_ast_ctx_up.reset(new TypeSystemClang(
+    m_scratch_ast_ctx_up = std::make_unique<TypeSystemClang>(
         "AppleObjCTypeEncodingParser ASTContext",
-        runtime.GetProcess()->GetTarget().GetArchitecture().GetTriple()));
+        runtime.GetProcess()->GetTarget().GetArchitecture().GetTriple());
 }
 
 std::string AppleObjCTypeEncodingParser::ReadStructName(StringLexer &type) {
diff --git a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
index 6b8a39021cc9c..beee2f5b6df79 100644
--- a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
+++ b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
@@ -903,7 +903,7 @@ size_t ObjectFileELF::ParseDependentModules() {
   if (m_filespec_up)
     return m_filespec_up->GetSize();
 
-  m_filespec_up.reset(new FileSpecList());
+  m_filespec_up = std::make_unique<FileSpecList>();
 
   if (!ParseSectionHeaders())
     return 0;
@@ -2717,7 +2717,7 @@ Symtab *ObjectFileELF::GetSymtab() {
     Section *symtab =
         section_list->FindSectionByType(eSectionTypeELFSymbolTable, true).get();
     if (symtab) {
-      m_symtab_up.reset(new Symtab(symtab->GetObjectFile()));
+      m_symtab_up = std::make_unique<Symtab>(symtab->GetObjectFile());
       symbol_id += ParseSymbolTable(m_symtab_up.get(), symbol_id, symtab);
     }
 
@@ -2734,7 +2734,7 @@ Symtab *ObjectFileELF::GetSymtab() {
               .get();
       if (dynsym) {
         if (!m_symtab_up)
-          m_symtab_up.reset(new Symtab(dynsym->GetObjectFile()));
+          m_symtab_up = std::make_unique<Symtab>(dynsym->GetObjectFile());
         symbol_id += ParseSymbolTable(m_symtab_up.get(), symbol_id, dynsym);
       }
     }
@@ -2761,7 +2761,8 @@ Symtab *ObjectFileELF::GetSymtab() {
         assert(reloc_header);
 
         if (m_symtab_up == nullptr)
-          m_symtab_up.reset(new Symtab(reloc_section->GetObjectFile()));
+          m_symtab_up =
+              std::make_unique<Symtab>(reloc_section->GetObjectFile());
 
         ParseTrampolineSymbols(m_symtab_up.get(), symbol_id, reloc_header,
                                reloc_id);
@@ -2771,14 +2772,14 @@ Symtab *ObjectFileELF::GetSymtab() {
     if (DWARFCallFrameInfo *eh_frame =
             GetModule()->GetUnwindTable().GetEHFrameInfo()) {
       if (m_symtab_up == nullptr)
-        m_symtab_up.reset(new Symtab(this));
+        m_symtab_up = std::make_unique<Symtab>(this);
       ParseUnwindSymbols(m_symtab_up.get(), eh_frame);
     }
 
     // If we still don't have any symtab then create an empty instance to avoid
     // do the section lookup next time.
     if (m_symtab_up == nullptr)
-      m_symtab_up.reset(new Symtab(this));
+      m_symtab_up = std::make_unique<Symtab>(this);
 
     // In the event that there's no symbol entry for the entry point we'll
     // artificially create one. We delegate to the symtab object the figuring
diff --git a/lldb/source/Plugins/ObjectFile/JIT/ObjectFileJIT.cpp b/lldb/source/Plugins/ObjectFile/JIT/ObjectFileJIT.cpp
index 93cef12fef495..93c2c9f945fe9 100644
--- a/lldb/source/Plugins/ObjectFile/JIT/ObjectFileJIT.cpp
+++ b/lldb/source/Plugins/ObjectFile/JIT/ObjectFileJIT.cpp
@@ -120,7 +120,7 @@ Symtab *ObjectFileJIT::GetSymtab() {
   if (module_sp) {
     std::lock_guard<std::recursive_mutex> guard(module_sp->GetMutex());
     if (m_symtab_up == nullptr) {
-      m_symtab_up.reset(new Symtab(this));
+      m_symtab_up = std::make_unique<Symtab>(this);
       std::lock_guard<std::recursive_mutex> symtab_guard(
           m_symtab_up->GetMutex());
       ObjectFileJITDelegateSP delegate_sp(m_delegate_wp.lock());
@@ -139,7 +139,7 @@ bool ObjectFileJIT::IsStripped() {
 
 void ObjectFileJIT::CreateSections(SectionList &unified_section_list) {
   if (!m_sections_up) {
-    m_sections_up.reset(new SectionList());
+    m_sections_up = std::make_unique<SectionList>();
     ObjectFileJITDelegateSP delegate_sp(m_delegate_wp.lock());
     if (delegate_sp) {
       delegate_sp->PopulateSectionList(this, *m_sections_up);
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index bc4017b86f072..2bb4b21adeaec 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -1253,7 +1253,7 @@ Symtab *ObjectFileMachO::GetSymtab() {
   if (module_sp) {
     std::lock_guard<std::recursive_mutex> guard(module_sp->GetMutex());
     if (m_symtab_up == nullptr) {
-      m_symtab_up.reset(new Symtab(this));
+      m_symtab_up = std::make_unique<Symtab>(this);
       std::lock_guard<std::recursive_mutex> symtab_guard(
           m_symtab_up->GetMutex());
       ParseSymtab();
@@ -1820,7 +1820,7 @@ void ObjectFileMachO::CreateSections(SectionList &unified_section_list) {
   if (m_sections_up)
     return;
 
-  m_sections_up.reset(new SectionList());
+  m_sections_up = std::make_unique<SectionList>();
 
   lldb::offset_t offset = MachHeaderSizeFromMagic(m_header.magic);
   // bool dump_sections = false;
@@ -2297,7 +2297,7 @@ size_t ObjectFileMachO::ParseSymtab() {
 
 #if defined(__APPLE__) &&                                                      \
     (defined(__arm__) || defined(__arm64__) || defined(__aarch64__))
-      if (m_header.flags & 0x80000000u &&
+      if (m_header.flags & MH_DYLIB_IN_CACHE &&
           process->GetAddressByteSize() == sizeof(void *)) {
         // This mach-o memory file is in the dyld shared cache. If this
         // program is not remote and this is iOS, then this process will
@@ -2379,7 +2379,7 @@ size_t ObjectFileMachO::ParseSymtab() {
             // problem. For binaries outside the shared cache, it's faster to
             // read the entire strtab at once instead of piece-by-piece as we
             // process the nlist records.
-            if ((m_header.flags & 0x80000000u) == 0) {
+            if ((m_header.flags & MH_DYLIB_IN_CACHE) == 0) {
               DataBufferSP strtab_data_sp(
                   ReadMemory(process_sp, strtab_addr, strtab_data_byte_size));
               if (strtab_data_sp) {
@@ -2596,8 +2596,7 @@ size_t ObjectFileMachO::ParseSymtab() {
   typedef std::set<ConstString> IndirectSymbols;
   IndirectSymbols indirect_symbol_names;
 
-#if defined(__APPLE__) &&                                                      \
-    (defined(__arm__) || defined(__arm64__) || defined(__aarch64__))
+#if defined(__APPLE__) && TARGET_OS_EMBEDDED
 
   // Some recent builds of the dyld_shared_cache (hereafter: DSC) have been
   // optimized by moving LOCAL symbols out of the memory mapped portion of
@@ -2609,7 +2608,7 @@ size_t ObjectFileMachO::ParseSymtab() {
   // to parse any DSC unmapped symbol information. If we find any, we set a
   // flag that tells the normal nlist parser to ignore all LOCAL symbols.
 
-  if (m_header.flags & 0x80000000u) {
+  if (m_header.flags & MH_DYLIB_IN_CACHE) {
     // Before we can start mapping the DSC, we need to make certain the
     // target process is actually using the cache we can find.
 
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
index 0a5a1d318fbe7..d2227bde47e92 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
@@ -55,15 +55,12 @@ struct CVInfoPdb70 {
   llvm::support::ulittle32_t Age;
 };
 
-static UUID GetCoffUUID(llvm::object::COFFObjectFile *coff_obj) {
-  if (!coff_obj)
-    return UUID();
-
+static UUID GetCoffUUID(llvm::object::COFFObjectFile &coff_obj) {
   const llvm::codeview::DebugInfo *pdb_info = nullptr;
   llvm::StringRef pdb_file;
 
   // This part is similar with what has done in minidump parser.
-  if (!coff_obj->getDebugPDBInfo(pdb_info, pdb_file) && pdb_info) {
+  if (!coff_obj.getDebugPDBInfo(pdb_info, pdb_file) && pdb_info) {
     if (pdb_info->PDB70.CVSignature == llvm::OMF::Signature::PDB70) {
       using llvm::support::endian::read16be;
       using llvm::support::endian::read32be;
@@ -172,7 +169,10 @@ size_t ObjectFilePECOFF::GetModuleSpecifications(
 
   Log *log(GetLogIfAllCategoriesSet(LIBLLDB_LOG_OBJECT));
 
-  auto binary = llvm::object::createBinary(file.GetPath());
+  if (DataBufferSP full_sp = MapFileData(file, -1, file_offset))
+    data_sp = std::move(full_sp);
+  auto binary = llvm::object::createBinary(llvm::MemoryBufferRef(
+      toStringRef(data_sp->GetData()), file.GetFilename().GetStringRef()));
 
   if (!binary) {
     LLDB_LOG_ERROR(log, binary.takeError(),
@@ -180,18 +180,15 @@ size_t ObjectFilePECOFF::GetModuleSpecifications(
     return initial_count;
   }
 
-  if (!binary->getBinary()->isCOFF() &&
-      !binary->getBinary()->isCOFFImportFile())
+  auto *COFFObj = llvm::dyn_cast<llvm::object::COFFObjectFile>(binary->get());
+  if (!COFFObj)
     return initial_count;
 
-  auto COFFObj =
-    llvm::cast<llvm::object::COFFObjectFile>(binary->getBinary());
-
   ModuleSpec module_spec(file);
   ArchSpec &spec = module_spec.GetArchitecture();
   lldb_private::UUID &uuid = module_spec.GetUUID();
   if (!uuid.IsValid())
-    uuid = GetCoffUUID(COFFObj);
+    uuid = GetCoffUUID(*COFFObj);
 
   switch (COFFObj->getMachine()) {
   case MachineAmd64:
@@ -244,12 +241,13 @@ lldb::SymbolType ObjectFilePECOFF::MapSymbolType(uint16_t coff_symbol_type) {
 }
 
 bool ObjectFilePECOFF::CreateBinary() {
-  if (m_owningbin)
+  if (m_binary)
     return true;
 
   Log *log(GetLogIfAllCategoriesSet(LIBLLDB_LOG_OBJECT));
 
-  auto binary = llvm::object::createBinary(m_file.GetPath());
+  auto binary = llvm::object::createBinary(llvm::MemoryBufferRef(
+      toStringRef(m_data.GetData()), m_file.GetFilename().GetStringRef()));
   if (!binary) {
     LLDB_LOG_ERROR(log, binary.takeError(),
                    "Failed to create binary for file ({1}): {0}", m_file);
@@ -257,19 +255,14 @@ bool ObjectFilePECOFF::CreateBinary() {
   }
 
   // Make sure we only handle COFF format.
-  if (!binary->getBinary()->isCOFF() &&
-      !binary->getBinary()->isCOFFImportFile())
+  m_binary =
+      llvm::unique_dyn_cast<llvm::object::COFFObjectFile>(std::move(*binary));
+  if (!m_binary)
     return false;
 
-  m_owningbin = OWNBINType(std::move(*binary));
-  LLDB_LOGF(log,
-            "%p ObjectFilePECOFF::CreateBinary() module = %p (%s), file = "
-            "%s, binary = %p (Bin = %p)",
-            static_cast<void *>(this), static_cast<void *>(GetModule().get()),
-            GetModule()->GetSpecificationDescription().c_str(),
-            m_file ? m_file.GetPath().c_str() : "<NULL>",
-            static_cast<void *>(m_owningbin.getPointer()),
-            static_cast<void *>(m_owningbin->getBinary()));
+  LLDB_LOG(log, "this = {0}, module = {1} ({2}), file = {3}, binary = {4}",
+           this, GetModule().get(), GetModule()->GetSpecificationDescription(),
+           m_file.GetPath(), m_binary.get());
   return true;
 }
 
@@ -281,7 +274,7 @@ ObjectFilePECOFF::ObjectFilePECOFF(const lldb::ModuleSP &module_sp,
                                    lldb::offset_t length)
     : ObjectFile(module_sp, file, file_offset, length, data_sp, data_offset),
       m_dos_header(), m_coff_header(), m_sect_headers(),
-      m_entry_point_address(), m_deps_filespec(), m_owningbin() {
+      m_entry_point_address(), m_deps_filespec() {
   ::memset(&m_dos_header, 0, sizeof(m_dos_header));
   ::memset(&m_coff_header, 0, sizeof(m_coff_header));
 }
@@ -292,7 +285,7 @@ ObjectFilePECOFF::ObjectFilePECOFF(const lldb::ModuleSP &module_sp,
                                    addr_t header_addr)
     : ObjectFile(module_sp, process_sp, header_addr, header_data_sp),
       m_dos_header(), m_coff_header(), m_sect_headers(),
-      m_entry_point_address(), m_deps_filespec(), m_owningbin() {
+      m_entry_point_address(), m_deps_filespec() {
   ::memset(&m_dos_header, 0, sizeof(m_dos_header));
   ::memset(&m_coff_header, 0, sizeof(m_coff_header));
 }
@@ -640,7 +633,7 @@ Symtab *ObjectFilePECOFF::GetSymtab() {
     std::lock_guard<std::recursive_mutex> guard(module_sp->GetMutex());
     if (m_symtab_up == nullptr) {
       SectionList *sect_list = GetSectionList();
-      m_symtab_up.reset(new Symtab(this));
+      m_symtab_up = std::make_unique<Symtab>(this);
       std::lock_guard<std::recursive_mutex> guard(m_symtab_up->GetMutex());
 
       const uint32_t num_syms = m_coff_header.nsyms;
@@ -868,7 +861,7 @@ SectionType ObjectFilePECOFF::GetSectionType(llvm::StringRef sect_name,
 void ObjectFilePECOFF::CreateSections(SectionList &unified_section_list) {
   if (m_sections_up)
     return;
-  m_sections_up.reset(new SectionList());
+  m_sections_up = std::make_unique<SectionList>();
 
   ModuleSP module_sp(GetModule());
   if (module_sp) {
@@ -931,10 +924,7 @@ UUID ObjectFilePECOFF::GetUUID() {
   if (!CreateBinary())
     return UUID();
 
-  auto COFFObj =
-    llvm::cast<llvm::object::COFFObjectFile>(m_owningbin->getBinary());
-
-  m_uuid = GetCoffUUID(COFFObj);
+  m_uuid = GetCoffUUID(*m_binary);
   return m_uuid;
 }
 
@@ -952,22 +942,13 @@ uint32_t ObjectFilePECOFF::ParseDependentModules() {
     return 0;
 
   Log *log(GetLogIfAllCategoriesSet(LIBLLDB_LOG_OBJECT));
-  LLDB_LOGF(log,
-            "%p ObjectFilePECOFF::ParseDependentModules() module = %p "
-            "(%s), binary = %p (Bin = %p)",
-            static_cast<void *>(this), static_cast<void *>(module_sp.get()),
-            module_sp->GetSpecificationDescription().c_str(),
-            static_cast<void *>(m_owningbin.getPointer()),
-            static_cast<void *>(m_owningbin->getBinary()));
-
-  auto COFFObj =
-      llvm::dyn_cast<llvm::object::COFFObjectFile>(m_owningbin->getBinary());
-  if (!COFFObj)
-    return 0;
+  LLDB_LOG(log, "this = {0}, module = {1} ({2}), file = {3}, binary = {4}",
+           this, GetModule().get(), GetModule()->GetSpecificationDescription(),
+           m_file.GetPath(), m_binary.get());
 
   m_deps_filespec = FileSpecList();
 
-  for (const auto &entry : COFFObj->import_directories()) {
+  for (const auto &entry : m_binary->import_directories()) {
     llvm::StringRef dll_name;
     // Report a bogus entry.
     if (llvm::Error e = entry.getName(dll_name)) {
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h
index c6cd1d407b9ad..fdcacbeab1eed 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h
@@ -12,7 +12,7 @@
 #include <vector>
 
 #include "lldb/Symbol/ObjectFile.h"
-#include "llvm/Object/Binary.h"
+#include "llvm/Object/COFF.h"
 
 class ObjectFilePECOFF : public lldb_private::ObjectFile {
 public:
@@ -300,8 +300,7 @@ class ObjectFilePECOFF : public lldb_private::ObjectFile {
   lldb::addr_t m_image_base;
   lldb_private::Address m_entry_point_address;
   llvm::Optional<lldb_private::FileSpecList> m_deps_filespec;
-  typedef llvm::object::OwningBinary<llvm::object::Binary> OWNBINType;
-  llvm::Optional<OWNBINType> m_owningbin;
+  std::unique_ptr<llvm::object::COFFObjectFile> m_binary;
   lldb_private::UUID m_uuid;
 };
 
diff --git a/lldb/source/Plugins/Platform/Android/AdbClient.cpp b/lldb/source/Plugins/Platform/Android/AdbClient.cpp
index 17707118d9c91..ffccd6d628aad 100644
--- a/lldb/source/Plugins/Platform/Android/AdbClient.cpp
+++ b/lldb/source/Plugins/Platform/Android/AdbClient.cpp
@@ -132,7 +132,7 @@ const std::string &AdbClient::GetDeviceID() const { return m_device_id; }
 
 Status AdbClient::Connect() {
   Status error;
-  m_conn.reset(new ConnectionFileDescriptor);
+  m_conn = std::make_unique<ConnectionFileDescriptor>();
   std::string port = "5037";
   if (const char *env_port = std::getenv("ANDROID_ADB_SERVER_PORT")) {
     port = env_port;
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
index 85cd000d8787c..f5ec08a1a199c 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
@@ -546,6 +546,8 @@ bool PlatformDarwin::ARMGetSupportedArchitectureAtIndex(uint32_t idx,
 #define OSNAME "watchos"
 #elif defined(TARGET_OS_BRIDGE) && TARGET_OS_BRIDGE == 1
 #define OSNAME "bridgeos"
+#elif defined(TARGET_OS_OSX) && TARGET_OS_OSX == 1
+#define OSNAME "macosx"
 #else
 #define OSNAME "ios"
 #endif
diff --git a/lldb/source/Plugins/Process/FreeBSD/FreeBSDThread.cpp b/lldb/source/Plugins/Process/FreeBSD/FreeBSDThread.cpp
index 32c6a440db590..48dbddb86cca3 100644
--- a/lldb/source/Plugins/Process/FreeBSD/FreeBSDThread.cpp
+++ b/lldb/source/Plugins/Process/FreeBSD/FreeBSDThread.cpp
@@ -164,7 +164,6 @@ lldb::RegisterContextSP FreeBSDThread::GetRegisterContext() {
     assert(target_arch.GetTriple().getOS() == llvm::Triple::FreeBSD);
     switch (target_arch.GetMachine()) {
     case llvm::Triple::aarch64:
-      reg_interface = new RegisterInfoPOSIX_arm64(target_arch);
       break;
     case llvm::Triple::arm:
       reg_interface = new RegisterInfoPOSIX_arm(target_arch);
@@ -193,7 +192,8 @@ lldb::RegisterContextSP FreeBSDThread::GetRegisterContext() {
     switch (target_arch.GetMachine()) {
     case llvm::Triple::aarch64: {
       RegisterContextPOSIXProcessMonitor_arm64 *reg_ctx =
-          new RegisterContextPOSIXProcessMonitor_arm64(*this, 0, reg_interface);
+          new RegisterContextPOSIXProcessMonitor_arm64(
+              *this, std::make_unique<RegisterInfoPOSIX_arm64>(target_arch));
       m_posix_thread = reg_ctx;
       m_reg_context_sp.reset(reg_ctx);
       break;
diff --git a/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.cpp b/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.cpp
index 6403d98a74cb7..d3eafae1e6de4 100644
--- a/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.cpp
+++ b/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.cpp
@@ -22,9 +22,9 @@ using namespace lldb_private;
 
 RegisterContextPOSIXProcessMonitor_arm64::
     RegisterContextPOSIXProcessMonitor_arm64(
-        lldb_private::Thread &thread, uint32_t concrete_frame_idx,
-        lldb_private::RegisterInfoInterface *register_info)
-    : RegisterContextPOSIX_arm64(thread, concrete_frame_idx, register_info) {}
+        lldb_private::Thread &thread,
+        std::unique_ptr<RegisterInfoPOSIX_arm64> register_info)
+    : RegisterContextPOSIX_arm64(thread, std::move(register_info)) {}
 
 ProcessMonitor &RegisterContextPOSIXProcessMonitor_arm64::GetMonitor() {
   lldb::ProcessSP base = CalculateProcess();
diff --git a/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.h b/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.h
index d54d34e89cad2..f100d905e28fd 100644
--- a/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.h
+++ b/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.h
@@ -17,8 +17,8 @@ class RegisterContextPOSIXProcessMonitor_arm64
       public POSIXBreakpointProtocol {
 public:
   RegisterContextPOSIXProcessMonitor_arm64(
-      lldb_private::Thread &thread, uint32_t concrete_frame_idx,
-      lldb_private::RegisterInfoInterface *register_info);
+      lldb_private::Thread &thread,
+      std::unique_ptr<RegisterInfoPOSIX_arm64> register_info);
 
 protected:
   bool ReadGPR();
diff --git a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux.h b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux.h
index 9a89a13f5f5b5..da7a762419f87 100644
--- a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux.h
+++ b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux.h
@@ -55,7 +55,7 @@ class NativeRegisterContextLinux : public NativeRegisterContextRegisterInfo {
 
   virtual void *GetGPRBuffer() = 0;
 
-  virtual size_t GetGPRSize() {
+  virtual size_t GetGPRSize() const {
     return GetRegisterInfoInterface().GetGPRSize();
   }
 
diff --git a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
index d39044f6bdaf4..beae3aef7c07d 100644
--- a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
+++ b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
@@ -21,7 +21,6 @@
 #include "Plugins/Process/Linux/NativeProcessLinux.h"
 #include "Plugins/Process/Linux/Procfs.h"
 #include "Plugins/Process/POSIX/ProcessPOSIXLog.h"
-#include "Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h"
 
 // System includes - They have to be included after framework includes because
 // they define some macros which collide with variable names in other modules
@@ -37,76 +36,6 @@ using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_private::process_linux;
 
-// ARM64 general purpose registers.
-static const uint32_t g_gpr_regnums_arm64[] = {
-    gpr_x0_arm64,       gpr_x1_arm64,   gpr_x2_arm64,  gpr_x3_arm64,
-    gpr_x4_arm64,       gpr_x5_arm64,   gpr_x6_arm64,  gpr_x7_arm64,
-    gpr_x8_arm64,       gpr_x9_arm64,   gpr_x10_arm64, gpr_x11_arm64,
-    gpr_x12_arm64,      gpr_x13_arm64,  gpr_x14_arm64, gpr_x15_arm64,
-    gpr_x16_arm64,      gpr_x17_arm64,  gpr_x18_arm64, gpr_x19_arm64,
-    gpr_x20_arm64,      gpr_x21_arm64,  gpr_x22_arm64, gpr_x23_arm64,
-    gpr_x24_arm64,      gpr_x25_arm64,  gpr_x26_arm64, gpr_x27_arm64,
-    gpr_x28_arm64,      gpr_fp_arm64,   gpr_lr_arm64,  gpr_sp_arm64,
-    gpr_pc_arm64,       gpr_cpsr_arm64, gpr_w0_arm64,  gpr_w1_arm64,
-    gpr_w2_arm64,       gpr_w3_arm64,   gpr_w4_arm64,  gpr_w5_arm64,
-    gpr_w6_arm64,       gpr_w7_arm64,   gpr_w8_arm64,  gpr_w9_arm64,
-    gpr_w10_arm64,      gpr_w11_arm64,  gpr_w12_arm64, gpr_w13_arm64,
-    gpr_w14_arm64,      gpr_w15_arm64,  gpr_w16_arm64, gpr_w17_arm64,
-    gpr_w18_arm64,      gpr_w19_arm64,  gpr_w20_arm64, gpr_w21_arm64,
-    gpr_w22_arm64,      gpr_w23_arm64,  gpr_w24_arm64, gpr_w25_arm64,
-    gpr_w26_arm64,      gpr_w27_arm64,  gpr_w28_arm64,
-    LLDB_INVALID_REGNUM // register sets need to end with this flag
-};
-static_assert(((sizeof g_gpr_regnums_arm64 / sizeof g_gpr_regnums_arm64[0]) -
-               1) == k_num_gpr_registers_arm64,
-              "g_gpr_regnums_arm64 has wrong number of register infos");
-
-// ARM64 floating point registers.
-static const uint32_t g_fpu_regnums_arm64[] = {
-    fpu_v0_arm64,       fpu_v1_arm64,   fpu_v2_arm64,  fpu_v3_arm64,
-    fpu_v4_arm64,       fpu_v5_arm64,   fpu_v6_arm64,  fpu_v7_arm64,
-    fpu_v8_arm64,       fpu_v9_arm64,   fpu_v10_arm64, fpu_v11_arm64,
-    fpu_v12_arm64,      fpu_v13_arm64,  fpu_v14_arm64, fpu_v15_arm64,
-    fpu_v16_arm64,      fpu_v17_arm64,  fpu_v18_arm64, fpu_v19_arm64,
-    fpu_v20_arm64,      fpu_v21_arm64,  fpu_v22_arm64, fpu_v23_arm64,
-    fpu_v24_arm64,      fpu_v25_arm64,  fpu_v26_arm64, fpu_v27_arm64,
-    fpu_v28_arm64,      fpu_v29_arm64,  fpu_v30_arm64, fpu_v31_arm64,
-    fpu_s0_arm64,       fpu_s1_arm64,   fpu_s2_arm64,  fpu_s3_arm64,
-    fpu_s4_arm64,       fpu_s5_arm64,   fpu_s6_arm64,  fpu_s7_arm64,
-    fpu_s8_arm64,       fpu_s9_arm64,   fpu_s10_arm64, fpu_s11_arm64,
-    fpu_s12_arm64,      fpu_s13_arm64,  fpu_s14_arm64, fpu_s15_arm64,
-    fpu_s16_arm64,      fpu_s17_arm64,  fpu_s18_arm64, fpu_s19_arm64,
-    fpu_s20_arm64,      fpu_s21_arm64,  fpu_s22_arm64, fpu_s23_arm64,
-    fpu_s24_arm64,      fpu_s25_arm64,  fpu_s26_arm64, fpu_s27_arm64,
-    fpu_s28_arm64,      fpu_s29_arm64,  fpu_s30_arm64, fpu_s31_arm64,
-
-    fpu_d0_arm64,       fpu_d1_arm64,   fpu_d2_arm64,  fpu_d3_arm64,
-    fpu_d4_arm64,       fpu_d5_arm64,   fpu_d6_arm64,  fpu_d7_arm64,
-    fpu_d8_arm64,       fpu_d9_arm64,   fpu_d10_arm64, fpu_d11_arm64,
-    fpu_d12_arm64,      fpu_d13_arm64,  fpu_d14_arm64, fpu_d15_arm64,
-    fpu_d16_arm64,      fpu_d17_arm64,  fpu_d18_arm64, fpu_d19_arm64,
-    fpu_d20_arm64,      fpu_d21_arm64,  fpu_d22_arm64, fpu_d23_arm64,
-    fpu_d24_arm64,      fpu_d25_arm64,  fpu_d26_arm64, fpu_d27_arm64,
-    fpu_d28_arm64,      fpu_d29_arm64,  fpu_d30_arm64, fpu_d31_arm64,
-    fpu_fpsr_arm64,     fpu_fpcr_arm64,
-    LLDB_INVALID_REGNUM // register sets need to end with this flag
-};
-static_assert(((sizeof g_fpu_regnums_arm64 / sizeof g_fpu_regnums_arm64[0]) -
-               1) == k_num_fpr_registers_arm64,
-              "g_fpu_regnums_arm64 has wrong number of register infos");
-
-namespace {
-// Number of register sets provided by this context.
-enum { k_num_register_sets = 2 };
-}
-
-// Register sets for ARM64.
-static const RegisterSet g_reg_sets_arm64[k_num_register_sets] = {
-    {"General Purpose Registers", "gpr", k_num_gpr_registers_arm64,
-     g_gpr_regnums_arm64},
-    {"Floating Point Registers", "fpu", k_num_fpr_registers_arm64,
-     g_fpu_regnums_arm64}};
-
 std::unique_ptr<NativeRegisterContextLinux>
 NativeRegisterContextLinux::CreateHostNativeRegisterContextLinux(
     const ArchSpec &target_arch, NativeThreadProtocol &native_thread) {
@@ -126,23 +55,6 @@ NativeRegisterContextLinux_arm64::NativeRegisterContextLinux_arm64(
     const ArchSpec &target_arch, NativeThreadProtocol &native_thread)
     : NativeRegisterContextLinux(native_thread,
                                  new RegisterInfoPOSIX_arm64(target_arch)) {
-  switch (target_arch.GetMachine()) {
-  case llvm::Triple::aarch64:
-    m_reg_info.num_registers = k_num_registers_arm64;
-    m_reg_info.num_gpr_registers = k_num_gpr_registers_arm64;
-    m_reg_info.num_fpr_registers = k_num_fpr_registers_arm64;
-    m_reg_info.last_gpr = k_last_gpr_arm64;
-    m_reg_info.first_fpr = k_first_fpr_arm64;
-    m_reg_info.last_fpr = k_last_fpr_arm64;
-    m_reg_info.first_fpr_v = fpu_v0_arm64;
-    m_reg_info.last_fpr_v = fpu_v31_arm64;
-    m_reg_info.gpr_flags = gpr_cpsr_arm64;
-    break;
-  default:
-    llvm_unreachable("Unhandled target architecture.");
-    break;
-  }
-
   ::memset(&m_fpr, 0, sizeof(m_fpr));
   ::memset(&m_gpr_arm64, 0, sizeof(m_gpr_arm64));
   ::memset(&m_hwp_regs, 0, sizeof(m_hwp_regs));
@@ -157,22 +69,24 @@ NativeRegisterContextLinux_arm64::NativeRegisterContextLinux_arm64(
   m_fpu_is_valid = false;
 }
 
+RegisterInfoPOSIX_arm64 &
+NativeRegisterContextLinux_arm64::GetRegisterInfo() const {
+  return static_cast<RegisterInfoPOSIX_arm64 &>(*m_register_info_interface_up);
+}
+
 uint32_t NativeRegisterContextLinux_arm64::GetRegisterSetCount() const {
-  return k_num_register_sets;
+  return GetRegisterInfo().GetRegisterSetCount();
 }
 
 const RegisterSet *
 NativeRegisterContextLinux_arm64::GetRegisterSet(uint32_t set_index) const {
-  if (set_index < k_num_register_sets)
-    return &g_reg_sets_arm64[set_index];
-
-  return nullptr;
+  return GetRegisterInfo().GetRegisterSet(set_index);
 }
 
 uint32_t NativeRegisterContextLinux_arm64::GetUserRegisterCount() const {
   uint32_t count = 0;
-  for (uint32_t set_index = 0; set_index < k_num_register_sets; ++set_index)
-    count += g_reg_sets_arm64[set_index].num_registers;
+  for (uint32_t set_index = 0; set_index < GetRegisterSetCount(); ++set_index)
+    count += GetRegisterSet(set_index)->num_registers;
   return count;
 }
 
@@ -344,11 +258,17 @@ Status NativeRegisterContextLinux_arm64::WriteAllRegisterValues(
 }
 
 bool NativeRegisterContextLinux_arm64::IsGPR(unsigned reg) const {
-  return reg <= m_reg_info.last_gpr; // GPR's come first.
+  if (GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg) ==
+      RegisterInfoPOSIX_arm64::GPRegSet)
+    return true;
+  return false;
 }
 
 bool NativeRegisterContextLinux_arm64::IsFPR(unsigned reg) const {
-  return (m_reg_info.first_fpr <= reg && reg <= m_reg_info.last_fpr);
+  if (GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg) ==
+      RegisterInfoPOSIX_arm64::FPRegSet)
+    return true;
+  return false;
 }
 
 uint32_t NativeRegisterContextLinux_arm64::NumSupportedHardwareBreakpoints() {
@@ -900,8 +820,7 @@ void NativeRegisterContextLinux_arm64::InvalidateAllRegisters() {
 
 uint32_t NativeRegisterContextLinux_arm64::CalculateFprOffset(
     const RegisterInfo *reg_info) const {
-  return reg_info->byte_offset -
-         GetRegisterInfoAtIndex(m_reg_info.first_fpr)->byte_offset;
+  return reg_info->byte_offset - GetGPRSize();
 }
 
 #endif // defined (__arm64__) || defined (__aarch64__)
diff --git a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h
index f7c501a9fdf6a..0fbbea51d2172 100644
--- a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h
+++ b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h
@@ -12,7 +12,7 @@
 #define lldb_NativeRegisterContextLinux_arm64_h
 
 #include "Plugins/Process/Linux/NativeRegisterContextLinux.h"
-#include "Plugins/Process/Utility/lldb-arm64-register-enums.h"
+#include "Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h"
 
 namespace lldb_private {
 namespace process_linux {
@@ -95,47 +95,13 @@ class NativeRegisterContextLinux_arm64 : public NativeRegisterContextLinux {
   size_t GetFPRSize() override { return sizeof(m_fpr); }
 
 private:
-  struct RegInfo {
-    uint32_t num_registers;
-    uint32_t num_gpr_registers;
-    uint32_t num_fpr_registers;
-
-    uint32_t last_gpr;
-    uint32_t first_fpr;
-    uint32_t last_fpr;
-
-    uint32_t first_fpr_v;
-    uint32_t last_fpr_v;
-
-    uint32_t gpr_flags;
-  };
-
-  // based on RegisterContextDarwin_arm64.h
-  struct VReg {
-    uint8_t bytes[16];
-  };
-
-  // based on RegisterContextDarwin_arm64.h
-  struct FPU {
-    VReg v[32];
-    uint32_t fpsr;
-    uint32_t fpcr;
-  };
-
-  struct GPR {
-    uint64_t x[31];
-    uint64_t sp;
-    uint64_t pc;
-    uint64_t pstate;
-  };
-
   bool m_gpr_is_valid;
   bool m_fpu_is_valid;
 
-  GPR m_gpr_arm64; // 64-bit general purpose registers.
-  RegInfo m_reg_info;
-  FPU m_fpr; // floating-point registers including extended register sets.
+  RegisterInfoPOSIX_arm64::GPR m_gpr_arm64; // 64-bit general purpose registers.
 
+  RegisterInfoPOSIX_arm64::FPU
+      m_fpr; // floating-point registers including extended register sets.
   // Debug register info for hardware breakpoints and watchpoints management.
   struct DREG {
     lldb::addr_t address;  // Breakpoint/watchpoint address value.
@@ -162,6 +128,8 @@ class NativeRegisterContextLinux_arm64 : public NativeRegisterContextLinux {
   Status WriteHardwareDebugRegs(int hwbType);
 
   uint32_t CalculateFprOffset(const RegisterInfo *reg_info) const;
+
+  RegisterInfoPOSIX_arm64 &GetRegisterInfo() const;
 };
 
 } // namespace process_linux
diff --git a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_s390x.h b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_s390x.h
index 69714e400113d..5ed78810a18bd 100644
--- a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_s390x.h
+++ b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_s390x.h
@@ -76,7 +76,7 @@ class NativeRegisterContextLinux_s390x : public NativeRegisterContextLinux {
   Status WriteFPR() override;
 
   void *GetGPRBuffer() override { return &m_regs; }
-  size_t GetGPRSize() override { return sizeof(m_regs); }
+  size_t GetGPRSize() const override { return sizeof(m_regs); }
   void *GetFPRBuffer() override { return &m_fp_regs; }
   size_t GetFPRSize() override { return sizeof(m_fp_regs); }
 
diff --git a/lldb/source/Plugins/Process/POSIX/NativeProcessELF.cpp b/lldb/source/Plugins/Process/POSIX/NativeProcessELF.cpp
index 84798bd004b5b..117d12101a0be 100644
--- a/lldb/source/Plugins/Process/POSIX/NativeProcessELF.cpp
+++ b/lldb/source/Plugins/Process/POSIX/NativeProcessELF.cpp
@@ -107,6 +107,11 @@ lldb::addr_t NativeProcessELF::GetELFImageInfoAddress() {
   return LLDB_INVALID_ADDRESS;
 }
 
+template lldb::addr_t NativeProcessELF::GetELFImageInfoAddress<
+    llvm::ELF::Elf32_Ehdr, llvm::ELF::Elf32_Phdr, llvm::ELF::Elf32_Dyn>();
+template lldb::addr_t NativeProcessELF::GetELFImageInfoAddress<
+    llvm::ELF::Elf64_Ehdr, llvm::ELF::Elf64_Phdr, llvm::ELF::Elf64_Dyn>();
+
 template <typename T>
 llvm::Expected<SVR4LibraryInfo>
 NativeProcessELF::ReadSVR4LibraryInfo(lldb::addr_t link_map_addr) {
diff --git a/lldb/source/Plugins/Process/POSIX/NativeProcessELF.h b/lldb/source/Plugins/Process/POSIX/NativeProcessELF.h
index 4fb513baebf08..dcfa9290ff507 100644
--- a/lldb/source/Plugins/Process/POSIX/NativeProcessELF.h
+++ b/lldb/source/Plugins/Process/POSIX/NativeProcessELF.h
@@ -48,6 +48,14 @@ class NativeProcessELF : public NativeProcessProtocol {
   llvm::Optional<lldb::addr_t> m_shared_library_info_addr;
 };
 
+// Explicitly declare the two 32/64 bit templates that NativeProcessELF.cpp will
+// define. This allows us to keep the template definition here and usable
+// elsewhere.
+extern template lldb::addr_t NativeProcessELF::GetELFImageInfoAddress<
+    llvm::ELF::Elf32_Ehdr, llvm::ELF::Elf32_Phdr, llvm::ELF::Elf32_Dyn>();
+extern template lldb::addr_t NativeProcessELF::GetELFImageInfoAddress<
+    llvm::ELF::Elf64_Ehdr, llvm::ELF::Elf64_Phdr, llvm::ELF::Elf64_Dyn>();
+
 } // namespace lldb_private
 
-#endif
\ No newline at end of file
+#endif
diff --git a/lldb/source/Plugins/Process/Utility/HistoryThread.cpp b/lldb/source/Plugins/Process/Utility/HistoryThread.cpp
index 0649cd2f07de1..d73b132539f19 100644
--- a/lldb/source/Plugins/Process/Utility/HistoryThread.cpp
+++ b/lldb/source/Plugins/Process/Utility/HistoryThread.cpp
@@ -31,7 +31,8 @@ HistoryThread::HistoryThread(lldb_private::Process &process, lldb::tid_t tid,
       m_pcs(pcs), m_extended_unwind_token(LLDB_INVALID_ADDRESS), m_queue_name(),
       m_thread_name(), m_originating_unique_thread_id(tid),
       m_queue_id(LLDB_INVALID_QUEUE_ID) {
-  m_unwinder_up.reset(new HistoryUnwind(*this, pcs, pcs_are_call_addresses));
+  m_unwinder_up =
+      std::make_unique<HistoryUnwind>(*this, pcs, pcs_are_call_addresses);
   Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_OBJECT));
   LLDB_LOGF(log, "%p HistoryThread::HistoryThread", static_cast<void *>(this));
 }
diff --git a/lldb/source/Plugins/Process/Utility/NativeRegisterContextRegisterInfo.h b/lldb/source/Plugins/Process/Utility/NativeRegisterContextRegisterInfo.h
index ae87130cd9de9..0e96841fd9091 100644
--- a/lldb/source/Plugins/Process/Utility/NativeRegisterContextRegisterInfo.h
+++ b/lldb/source/Plugins/Process/Utility/NativeRegisterContextRegisterInfo.h
@@ -33,7 +33,7 @@ class NativeRegisterContextRegisterInfo : public NativeRegisterContext {
 
   const RegisterInfoInterface &GetRegisterInfoInterface() const;
 
-private:
+protected:
   std::unique_ptr<RegisterInfoInterface> m_register_info_interface_up;
 };
 }
diff --git a/lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.cpp b/lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.cpp
index 63f22620beda4..8ef587f13e3ae 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.cpp
+++ b/lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.cpp
@@ -25,105 +25,25 @@
 using namespace lldb;
 using namespace lldb_private;
 
-// ARM64 general purpose registers.
-const uint32_t g_gpr_regnums_arm64[] = {
-    gpr_x0_arm64,       gpr_x1_arm64,   gpr_x2_arm64,  gpr_x3_arm64,
-    gpr_x4_arm64,       gpr_x5_arm64,   gpr_x6_arm64,  gpr_x7_arm64,
-    gpr_x8_arm64,       gpr_x9_arm64,   gpr_x10_arm64, gpr_x11_arm64,
-    gpr_x12_arm64,      gpr_x13_arm64,  gpr_x14_arm64, gpr_x15_arm64,
-    gpr_x16_arm64,      gpr_x17_arm64,  gpr_x18_arm64, gpr_x19_arm64,
-    gpr_x20_arm64,      gpr_x21_arm64,  gpr_x22_arm64, gpr_x23_arm64,
-    gpr_x24_arm64,      gpr_x25_arm64,  gpr_x26_arm64, gpr_x27_arm64,
-    gpr_x28_arm64,      gpr_fp_arm64,   gpr_lr_arm64,  gpr_sp_arm64,
-    gpr_pc_arm64,       gpr_cpsr_arm64, gpr_w0_arm64,  gpr_w1_arm64,
-    gpr_w2_arm64,       gpr_w3_arm64,   gpr_w4_arm64,  gpr_w5_arm64,
-    gpr_w6_arm64,       gpr_w7_arm64,   gpr_w8_arm64,  gpr_w9_arm64,
-    gpr_w10_arm64,      gpr_w11_arm64,  gpr_w12_arm64, gpr_w13_arm64,
-    gpr_w14_arm64,      gpr_w15_arm64,  gpr_w16_arm64, gpr_w17_arm64,
-    gpr_w18_arm64,      gpr_w19_arm64,  gpr_w20_arm64, gpr_w21_arm64,
-    gpr_w22_arm64,      gpr_w23_arm64,  gpr_w24_arm64, gpr_w25_arm64,
-    gpr_w26_arm64,      gpr_w27_arm64,  gpr_w28_arm64,
-    LLDB_INVALID_REGNUM // register sets need to end with this flag
-};
-static_assert(((sizeof g_gpr_regnums_arm64 / sizeof g_gpr_regnums_arm64[0]) -
-               1) == k_num_gpr_registers_arm64,
-              "g_gpr_regnums_arm64 has wrong number of register infos");
-
-// ARM64 floating point registers.
-static const uint32_t g_fpu_regnums_arm64[] = {
-    fpu_v0_arm64,       fpu_v1_arm64,   fpu_v2_arm64,  fpu_v3_arm64,
-    fpu_v4_arm64,       fpu_v5_arm64,   fpu_v6_arm64,  fpu_v7_arm64,
-    fpu_v8_arm64,       fpu_v9_arm64,   fpu_v10_arm64, fpu_v11_arm64,
-    fpu_v12_arm64,      fpu_v13_arm64,  fpu_v14_arm64, fpu_v15_arm64,
-    fpu_v16_arm64,      fpu_v17_arm64,  fpu_v18_arm64, fpu_v19_arm64,
-    fpu_v20_arm64,      fpu_v21_arm64,  fpu_v22_arm64, fpu_v23_arm64,
-    fpu_v24_arm64,      fpu_v25_arm64,  fpu_v26_arm64, fpu_v27_arm64,
-    fpu_v28_arm64,      fpu_v29_arm64,  fpu_v30_arm64, fpu_v31_arm64,
-    fpu_s0_arm64,       fpu_s1_arm64,   fpu_s2_arm64,  fpu_s3_arm64,
-    fpu_s4_arm64,       fpu_s5_arm64,   fpu_s6_arm64,  fpu_s7_arm64,
-    fpu_s8_arm64,       fpu_s9_arm64,   fpu_s10_arm64, fpu_s11_arm64,
-    fpu_s12_arm64,      fpu_s13_arm64,  fpu_s14_arm64, fpu_s15_arm64,
-    fpu_s16_arm64,      fpu_s17_arm64,  fpu_s18_arm64, fpu_s19_arm64,
-    fpu_s20_arm64,      fpu_s21_arm64,  fpu_s22_arm64, fpu_s23_arm64,
-    fpu_s24_arm64,      fpu_s25_arm64,  fpu_s26_arm64, fpu_s27_arm64,
-    fpu_s28_arm64,      fpu_s29_arm64,  fpu_s30_arm64, fpu_s31_arm64,
-
-    fpu_d0_arm64,       fpu_d1_arm64,   fpu_d2_arm64,  fpu_d3_arm64,
-    fpu_d4_arm64,       fpu_d5_arm64,   fpu_d6_arm64,  fpu_d7_arm64,
-    fpu_d8_arm64,       fpu_d9_arm64,   fpu_d10_arm64, fpu_d11_arm64,
-    fpu_d12_arm64,      fpu_d13_arm64,  fpu_d14_arm64, fpu_d15_arm64,
-    fpu_d16_arm64,      fpu_d17_arm64,  fpu_d18_arm64, fpu_d19_arm64,
-    fpu_d20_arm64,      fpu_d21_arm64,  fpu_d22_arm64, fpu_d23_arm64,
-    fpu_d24_arm64,      fpu_d25_arm64,  fpu_d26_arm64, fpu_d27_arm64,
-    fpu_d28_arm64,      fpu_d29_arm64,  fpu_d30_arm64, fpu_d31_arm64,
-    fpu_fpsr_arm64,     fpu_fpcr_arm64,
-    LLDB_INVALID_REGNUM // register sets need to end with this flag
-};
-static_assert(((sizeof g_fpu_regnums_arm64 / sizeof g_fpu_regnums_arm64[0]) -
-               1) == k_num_fpr_registers_arm64,
-              "g_fpu_regnums_arm64 has wrong number of register infos");
-
-// Number of register sets provided by this context.
-enum { k_num_register_sets = 2 };
-
-// Register sets for ARM64.
-static const lldb_private::RegisterSet g_reg_sets_arm64[k_num_register_sets] = {
-    {"General Purpose Registers", "gpr", k_num_gpr_registers_arm64,
-     g_gpr_regnums_arm64},
-    {"Floating Point Registers", "fpu", k_num_fpr_registers_arm64,
-     g_fpu_regnums_arm64}};
-
 bool RegisterContextPOSIX_arm64::IsGPR(unsigned reg) {
-  return reg <= m_reg_info.last_gpr; // GPR's come first.
+  if (m_register_info_up->GetRegisterSetFromRegisterIndex(reg) ==
+      RegisterInfoPOSIX_arm64::GPRegSet)
+    return true;
+  return false;
 }
 
 bool RegisterContextPOSIX_arm64::IsFPR(unsigned reg) {
-  return (m_reg_info.first_fpr <= reg && reg <= m_reg_info.last_fpr);
+  if (m_register_info_up->GetRegisterSetFromRegisterIndex(reg) ==
+      RegisterInfoPOSIX_arm64::FPRegSet)
+    return true;
+  return false;
 }
 
 RegisterContextPOSIX_arm64::RegisterContextPOSIX_arm64(
-    lldb_private::Thread &thread, uint32_t concrete_frame_idx,
-    lldb_private::RegisterInfoInterface *register_info)
-    : lldb_private::RegisterContext(thread, concrete_frame_idx) {
-  m_register_info_up.reset(register_info);
-
-  switch (register_info->m_target_arch.GetMachine()) {
-  case llvm::Triple::aarch64:
-  case llvm::Triple::aarch64_32:
-    m_reg_info.num_registers = k_num_registers_arm64;
-    m_reg_info.num_gpr_registers = k_num_gpr_registers_arm64;
-    m_reg_info.num_fpr_registers = k_num_fpr_registers_arm64;
-    m_reg_info.last_gpr = k_last_gpr_arm64;
-    m_reg_info.first_fpr = k_first_fpr_arm64;
-    m_reg_info.last_fpr = k_last_fpr_arm64;
-    m_reg_info.first_fpr_v = fpu_v0_arm64;
-    m_reg_info.last_fpr_v = fpu_v31_arm64;
-    m_reg_info.gpr_flags = gpr_cpsr_arm64;
-    break;
-  default:
-    assert(false && "Unhandled target architecture.");
-    break;
-  }
+    lldb_private::Thread &thread,
+    std::unique_ptr<RegisterInfoPOSIX_arm64> register_info)
+    : lldb_private::RegisterContext(thread, 0),
+      m_register_info_up(std::move(register_info)) {
 
   ::memset(&m_fpr, 0, sizeof m_fpr);
 }
@@ -135,19 +55,15 @@ void RegisterContextPOSIX_arm64::Invalidate() {}
 void RegisterContextPOSIX_arm64::InvalidateAllRegisters() {}
 
 unsigned RegisterContextPOSIX_arm64::GetRegisterOffset(unsigned reg) {
-  assert(reg < m_reg_info.num_registers && "Invalid register number.");
-  return GetRegisterInfo()[reg].byte_offset;
+  return m_register_info_up->GetRegisterInfo()[reg].byte_offset;
 }
 
 unsigned RegisterContextPOSIX_arm64::GetRegisterSize(unsigned reg) {
-  assert(reg < m_reg_info.num_registers && "Invalid register number.");
-  return GetRegisterInfo()[reg].byte_size;
+  return m_register_info_up->GetRegisterInfo()[reg].byte_size;
 }
 
 size_t RegisterContextPOSIX_arm64::GetRegisterCount() {
-  size_t num_registers =
-      m_reg_info.num_gpr_registers + m_reg_info.num_fpr_registers;
-  return num_registers;
+  return m_register_info_up->GetRegisterCount();
 }
 
 size_t RegisterContextPOSIX_arm64::GetGPRSize() {
@@ -164,42 +80,23 @@ RegisterContextPOSIX_arm64::GetRegisterInfo() {
 
 const lldb_private::RegisterInfo *
 RegisterContextPOSIX_arm64::GetRegisterInfoAtIndex(size_t reg) {
-  if (reg < m_reg_info.num_registers)
+  if (reg < GetRegisterCount())
     return &GetRegisterInfo()[reg];
   else
     return nullptr;
 }
 
 size_t RegisterContextPOSIX_arm64::GetRegisterSetCount() {
-  size_t sets = 0;
-  for (size_t set = 0; set < k_num_register_sets; ++set) {
-    if (IsRegisterSetAvailable(set))
-      ++sets;
-  }
-
-  return sets;
-}
-
-bool RegisterContextPOSIX_arm64::IsRegisterSetAvailable(size_t set_index) {
-  return set_index < k_num_register_sets;
+  return m_register_info_up->GetRegisterSetCount();
 }
 
 const lldb_private::RegisterSet *
 RegisterContextPOSIX_arm64::GetRegisterSet(size_t set) {
-  if (IsRegisterSetAvailable(set)) {
-    switch (m_register_info_up->m_target_arch.GetMachine()) {
-    case llvm::Triple::aarch64:
-    case llvm::Triple::aarch64_32:
-      return &g_reg_sets_arm64[set];
-    default:
-      assert(false && "Unhandled target architecture.");
-      return nullptr;
-    }
-  }
-  return nullptr;
+  return m_register_info_up->GetRegisterSet(set);
 }
 
 const char *RegisterContextPOSIX_arm64::GetRegisterName(unsigned reg) {
-  assert(reg < m_reg_info.num_registers && "Invalid register offset.");
-  return GetRegisterInfo()[reg].name;
+  if (reg < GetRegisterCount())
+    return GetRegisterInfo()[reg].name;
+  return nullptr;
 }
diff --git a/lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.h b/lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.h
index 5b084c9253b49..c2d5aee7f73c9 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.h
+++ b/lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.h
@@ -10,6 +10,7 @@
 #define LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERCONTEXTPOSIX_ARM64_H
 
 #include "RegisterInfoInterface.h"
+#include "RegisterInfoPOSIX_arm64.h"
 #include "lldb-arm64-register-enums.h"
 #include "lldb/Target/RegisterContext.h"
 #include "lldb/Utility/Log.h"
@@ -19,8 +20,8 @@ class ProcessMonitor;
 class RegisterContextPOSIX_arm64 : public lldb_private::RegisterContext {
 public:
   RegisterContextPOSIX_arm64(
-      lldb_private::Thread &thread, uint32_t concrete_frame_idx,
-      lldb_private::RegisterInfoInterface *register_info);
+      lldb_private::Thread &thread,
+      std::unique_ptr<RegisterInfoPOSIX_arm64> register_info);
 
   ~RegisterContextPOSIX_arm64() override;
 
@@ -45,46 +46,14 @@ class RegisterContextPOSIX_arm64 : public lldb_private::RegisterContext {
   const char *GetRegisterName(unsigned reg);
 
 protected:
-  struct RegInfo {
-    uint32_t num_registers;
-    uint32_t num_gpr_registers;
-    uint32_t num_fpr_registers;
-
-    uint32_t last_gpr;
-    uint32_t first_fpr;
-    uint32_t last_fpr;
-
-    uint32_t first_fpr_v;
-    uint32_t last_fpr_v;
-
-    uint32_t gpr_flags;
-  };
-
-  // based on RegisterContextDarwin_arm64.h
-  struct VReg {
-    uint8_t bytes[16];
-  };
-
-  // based on RegisterContextDarwin_arm64.h
-  struct FPU {
-    VReg v[32];
-    uint32_t fpsr;
-    uint32_t fpcr;
-  };
-
   uint64_t m_gpr_arm64[lldb_private::k_num_gpr_registers_arm64]; // 64-bit
                                                                  // general
                                                                  // purpose
                                                                  // registers.
-  RegInfo m_reg_info;
-  struct RegisterContextPOSIX_arm64::FPU
-      m_fpr; // floating-point registers including extended register sets.
-  std::unique_ptr<lldb_private::RegisterInfoInterface>
-      m_register_info_up; // Register Info Interface (FreeBSD or Linux)
 
-  // Determines if an extended register set is supported on the processor
-  // running the inferior process.
-  virtual bool IsRegisterSetAvailable(size_t set_index);
+  struct RegisterInfoPOSIX_arm64::FPU
+      m_fpr; // floating-point registers including extended register sets.
+  std::unique_ptr<RegisterInfoPOSIX_arm64> m_register_info_up;
 
   virtual const lldb_private::RegisterInfo *GetRegisterInfo();
 
@@ -92,6 +61,8 @@ class RegisterContextPOSIX_arm64 : public lldb_private::RegisterContext {
 
   bool IsFPR(unsigned reg);
 
+  size_t GetFPUSize() { return sizeof(RegisterInfoPOSIX_arm64::FPU); }
+
   virtual bool ReadGPR() = 0;
   virtual bool ReadFPR() = 0;
   virtual bool WriteGPR() = 0;
diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfoAndSetInterface.h b/lldb/source/Plugins/Process/Utility/RegisterInfoAndSetInterface.h
new file mode 100644
index 0000000000000..7e569dc9ba784
--- /dev/null
+++ b/lldb/source/Plugins/Process/Utility/RegisterInfoAndSetInterface.h
@@ -0,0 +1,36 @@
+//===-- RegisterInfoAndSetInterface.h ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERINFOANDSETINTERFACE_H
+#define LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERINFOANDSETINTERFACE_H
+
+#include "RegisterInfoInterface.h"
+
+#include "lldb/Utility/ArchSpec.h"
+#include "lldb/lldb-private-types.h"
+#include <vector>
+
+namespace lldb_private {
+
+class RegisterInfoAndSetInterface : public RegisterInfoInterface {
+public:
+  RegisterInfoAndSetInterface(const lldb_private::ArchSpec &target_arch)
+      : RegisterInfoInterface(target_arch) {}
+
+  virtual size_t GetFPRSize() const = 0;
+
+  virtual const lldb_private::RegisterSet *
+  GetRegisterSet(size_t reg_set) const = 0;
+
+  virtual size_t GetRegisterSetCount() const = 0;
+
+  virtual size_t GetRegisterSetFromRegisterIndex(uint32_t reg_index) const = 0;
+};
+} // namespace lldb_private
+
+#endif
diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp
index dccd7ac7d52d1..4537cee42ad96 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp
+++ b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp
@@ -65,6 +65,82 @@ GetRegisterInfoPtr(const lldb_private::ArchSpec &target_arch) {
   }
 }
 
+// Number of register sets provided by this context.
+enum {
+  k_num_gpr_registers = gpr_w28 - gpr_x0 + 1,
+  k_num_fpr_registers = fpu_fpcr - fpu_v0 + 1,
+  k_num_register_sets = 2
+};
+
+// ARM64 general purpose registers.
+static const uint32_t g_gpr_regnums_arm64[] = {
+    gpr_x0,  gpr_x1,   gpr_x2,  gpr_x3,
+    gpr_x4,  gpr_x5,   gpr_x6,  gpr_x7,
+    gpr_x8,  gpr_x9,   gpr_x10, gpr_x11,
+    gpr_x12, gpr_x13,  gpr_x14, gpr_x15,
+    gpr_x16, gpr_x17,  gpr_x18, gpr_x19,
+    gpr_x20, gpr_x21,  gpr_x22, gpr_x23,
+    gpr_x24, gpr_x25,  gpr_x26, gpr_x27,
+    gpr_x28, gpr_fp,   gpr_lr,  gpr_sp,
+    gpr_pc,  gpr_cpsr, gpr_w0,  gpr_w1,
+    gpr_w2,  gpr_w3,   gpr_w4,  gpr_w5,
+    gpr_w6,  gpr_w7,   gpr_w8,  gpr_w9,
+    gpr_w10, gpr_w11,  gpr_w12, gpr_w13,
+    gpr_w14, gpr_w15,  gpr_w16, gpr_w17,
+    gpr_w18, gpr_w19,  gpr_w20, gpr_w21,
+    gpr_w22, gpr_w23,  gpr_w24, gpr_w25,
+    gpr_w26, gpr_w27,  gpr_w28, LLDB_INVALID_REGNUM};
+
+static_assert(((sizeof g_gpr_regnums_arm64 / sizeof g_gpr_regnums_arm64[0]) -
+               1) == k_num_gpr_registers,
+              "g_gpr_regnums_arm64 has wrong number of register infos");
+
+// ARM64 floating point registers.
+static const uint32_t g_fpu_regnums_arm64[] = {
+    fpu_v0,   fpu_v1,   fpu_v2,
+    fpu_v3,   fpu_v4,   fpu_v5,
+    fpu_v6,   fpu_v7,   fpu_v8,
+    fpu_v9,   fpu_v10,  fpu_v11,
+    fpu_v12,  fpu_v13,  fpu_v14,
+    fpu_v15,  fpu_v16,  fpu_v17,
+    fpu_v18,  fpu_v19,  fpu_v20,
+    fpu_v21,  fpu_v22,  fpu_v23,
+    fpu_v24,  fpu_v25,  fpu_v26,
+    fpu_v27,  fpu_v28,  fpu_v29,
+    fpu_v30,  fpu_v31,  fpu_s0,
+    fpu_s1,   fpu_s2,   fpu_s3,
+    fpu_s4,   fpu_s5,   fpu_s6,
+    fpu_s7,   fpu_s8,   fpu_s9,
+    fpu_s10,  fpu_s11,  fpu_s12,
+    fpu_s13,  fpu_s14,  fpu_s15,
+    fpu_s16,  fpu_s17,  fpu_s18,
+    fpu_s19,  fpu_s20,  fpu_s21,
+    fpu_s22,  fpu_s23,  fpu_s24,
+    fpu_s25,  fpu_s26,  fpu_s27,
+    fpu_s28,  fpu_s29,  fpu_s30,
+    fpu_s31,  fpu_d0,   fpu_d1,
+    fpu_d2,   fpu_d3,   fpu_d4,
+    fpu_d5,   fpu_d6,   fpu_d7,
+    fpu_d8,   fpu_d9,   fpu_d10,
+    fpu_d11,  fpu_d12,  fpu_d13,
+    fpu_d14,  fpu_d15,  fpu_d16,
+    fpu_d17,  fpu_d18,  fpu_d19,
+    fpu_d20,  fpu_d21,  fpu_d22,
+    fpu_d23,  fpu_d24,  fpu_d25,
+    fpu_d26,  fpu_d27,  fpu_d28,
+    fpu_d29,  fpu_d30,  fpu_d31,
+    fpu_fpsr, fpu_fpcr, LLDB_INVALID_REGNUM};
+static_assert(((sizeof g_fpu_regnums_arm64 / sizeof g_fpu_regnums_arm64[0]) -
+               1) == k_num_fpr_registers,
+              "g_fpu_regnums_arm64 has wrong number of register infos");
+// clang-format on
+// Register sets for ARM64.
+static const lldb_private::RegisterSet g_reg_sets_arm64[k_num_register_sets] = {
+    {"General Purpose Registers", "gpr", k_num_gpr_registers,
+     g_gpr_regnums_arm64},
+    {"Floating Point Registers", "fpu", k_num_fpr_registers,
+     g_fpu_regnums_arm64}};
+
 static uint32_t
 GetRegisterInfoCount(const lldb_private::ArchSpec &target_arch) {
   switch (target_arch.GetMachine()) {
@@ -80,19 +156,60 @@ GetRegisterInfoCount(const lldb_private::ArchSpec &target_arch) {
 
 RegisterInfoPOSIX_arm64::RegisterInfoPOSIX_arm64(
     const lldb_private::ArchSpec &target_arch)
-    : lldb_private::RegisterInfoInterface(target_arch),
+    : lldb_private::RegisterInfoAndSetInterface(target_arch),
       m_register_info_p(GetRegisterInfoPtr(target_arch)),
-      m_register_info_count(GetRegisterInfoCount(target_arch)) {}
+      m_register_info_count(GetRegisterInfoCount(target_arch)) {
+
+  switch (target_arch.GetMachine()) {
+  case llvm::Triple::aarch64:
+  case llvm::Triple::aarch64_32:
+    num_registers = k_num_gpr_registers + k_num_fpr_registers;
+    num_gpr_registers = k_num_gpr_registers;
+    num_fpr_registers = k_num_fpr_registers;
+    last_gpr = gpr_w28;
+    first_fpr = fpu_v0;
+    last_fpr = fpu_fpcr;
+    break;
+  default:
+    assert(false && "Unhandled target architecture.");
+    break;
+  }
+}
+
+uint32_t RegisterInfoPOSIX_arm64::GetRegisterCount() const {
+  return num_gpr_registers + num_fpr_registers;
+}
 
 size_t RegisterInfoPOSIX_arm64::GetGPRSize() const {
   return sizeof(struct RegisterInfoPOSIX_arm64::GPR);
 }
 
+size_t RegisterInfoPOSIX_arm64::GetFPRSize() const {
+  return sizeof(struct RegisterInfoPOSIX_arm64::FPU);
+}
+
 const lldb_private::RegisterInfo *
 RegisterInfoPOSIX_arm64::GetRegisterInfo() const {
   return m_register_info_p;
 }
 
-uint32_t RegisterInfoPOSIX_arm64::GetRegisterCount() const {
-  return m_register_info_count;
+size_t RegisterInfoPOSIX_arm64::GetRegisterSetCount() const {
+  return k_num_register_sets;
+}
+
+size_t RegisterInfoPOSIX_arm64::GetRegisterSetFromRegisterIndex(
+    uint32_t reg_index) const {
+  if (reg_index <= last_gpr)
+    return GPRegSet;
+  else if (reg_index <= last_fpr)
+    return FPRegSet;
+  return LLDB_INVALID_REGNUM;
+}
+
+const lldb_private::RegisterSet *
+RegisterInfoPOSIX_arm64::GetRegisterSet(size_t set_index) const {
+  if (set_index < k_num_register_sets)
+    return &g_reg_sets_arm64[set_index];
+
+  return nullptr;
 }
diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
index 2a27a655d1976..2da6a531a6b60 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
+++ b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
@@ -9,12 +9,15 @@
 #ifndef LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERINFOPOSIX_ARM64_H
 #define LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERINFOPOSIX_ARM64_H
 
-#include "RegisterInfoInterface.h"
+#include "RegisterInfoAndSetInterface.h"
 #include "lldb/Target/RegisterContext.h"
 #include "lldb/lldb-private.h"
 
-class RegisterInfoPOSIX_arm64 : public lldb_private::RegisterInfoInterface {
+class RegisterInfoPOSIX_arm64
+    : public lldb_private::RegisterInfoAndSetInterface {
 public:
+  enum { GPRegSet = 0, FPRegSet };
+
   // based on RegisterContextDarwin_arm64.h
   struct GPR {
     uint64_t x[29]; // x0-x28
@@ -57,11 +60,28 @@ class RegisterInfoPOSIX_arm64 : public lldb_private::RegisterInfoInterface {
 
   size_t GetGPRSize() const override;
 
+  size_t GetFPRSize() const override;
+
   const lldb_private::RegisterInfo *GetRegisterInfo() const override;
 
   uint32_t GetRegisterCount() const override;
 
+  const lldb_private::RegisterSet *
+  GetRegisterSet(size_t reg_set) const override;
+
+  size_t GetRegisterSetCount() const override;
+
+  size_t GetRegisterSetFromRegisterIndex(uint32_t reg_index) const override;
+
 private:
+  uint32_t num_registers;
+  uint32_t num_gpr_registers;
+  uint32_t num_fpr_registers;
+
+  uint32_t last_gpr;
+  uint32_t first_fpr;
+  uint32_t last_fpr;
+
   const lldb_private::RegisterInfo *m_register_info_p;
   uint32_t m_register_info_count;
 };
diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfos_arm64.h b/lldb/source/Plugins/Process/Utility/RegisterInfos_arm64.h
index 0a0979c920715..4aee55e7afba0 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterInfos_arm64.h
+++ b/lldb/source/Plugins/Process/Utility/RegisterInfos_arm64.h
@@ -456,37 +456,26 @@ static uint32_t g_d29_invalidates[] = {fpu_v29, fpu_s29, LLDB_INVALID_REGNUM};
 static uint32_t g_d30_invalidates[] = {fpu_v30, fpu_s30, LLDB_INVALID_REGNUM};
 static uint32_t g_d31_invalidates[] = {fpu_v31, fpu_s31, LLDB_INVALID_REGNUM};
 
-// Generates register kinds array for 64-bit general purpose registers
-#define GPR64_KIND(reg, generic_kind)                                          \
+// Generates register kinds array with DWARF, EH frame and generic kind
+#define MISC_KIND(reg, type, generic_kind)                                     \
   {                                                                            \
     arm64_ehframe::reg, arm64_dwarf::reg, generic_kind, LLDB_INVALID_REGNUM,   \
-        gpr_##reg                                                              \
+        type##_##reg                                                           \
   }
 
-// Generates register kinds array for registers with lldb kind
-#define MISC_KIND(lldb_kind)                                                   \
+// Generates register kinds array for registers with only lldb kind
+#define LLDB_KIND(lldb_kind)                                                   \
   {                                                                            \
     LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,             \
         LLDB_INVALID_REGNUM, lldb_kind                                         \
   }
 
 // Generates register kinds array for vector registers
-#define VREG_KIND(reg)                                                         \
-  {                                                                            \
-    LLDB_INVALID_REGNUM, arm64_dwarf::reg, LLDB_INVALID_REGNUM,                \
-        LLDB_INVALID_REGNUM, fpu_##reg                                         \
-  }
-
-// Generates register kinds array for cpsr
-#define CPSR_KIND(lldb_kind)                                                   \
-  {                                                                            \
-    arm64_ehframe::cpsr, arm64_dwarf::cpsr, LLDB_REGNUM_GENERIC_FLAGS,         \
-        LLDB_INVALID_REGNUM, lldb_kind                                         \
-  }
-
-#define MISC_GPR_KIND(lldb_kind) CPSR_KIND(lldb_kind)
-#define MISC_FPU_KIND(lldb_kind) MISC_KIND(lldb_kind)
-#define MISC_EXC_KIND(lldb_kind) MISC_KIND(lldb_kind)
+#define GPR64_KIND(reg, generic_kind) MISC_KIND(reg, gpr, generic_kind)
+#define VREG_KIND(reg) MISC_KIND(reg, fpu, LLDB_INVALID_REGNUM)
+#define MISC_GPR_KIND(lldb_kind) MISC_KIND(cpsr, gpr, LLDB_REGNUM_GENERIC_FLAGS)
+#define MISC_FPU_KIND(lldb_kind) LLDB_KIND(lldb_kind)
+#define MISC_EXC_KIND(lldb_kind) LLDB_KIND(lldb_kind)
 
 // Defines a 64-bit general purpose register
 #define DEFINE_GPR64(reg, generic_kind)                                        \
@@ -509,7 +498,7 @@ static uint32_t g_d31_invalidates[] = {fpu_v31, fpu_s31, LLDB_INVALID_REGNUM};
   {                                                                            \
     #wreg, nullptr, 4,                                                         \
         GPR_OFFSET(gpr_##xreg) + GPR_W_PSEUDO_REG_ENDIAN_OFFSET,               \
-        lldb::eEncodingUint, lldb::eFormatHex, MISC_KIND(gpr_##wreg),          \
+        lldb::eEncodingUint, lldb::eFormatHex, LLDB_KIND(gpr_##wreg),          \
         g_contained_##xreg, g_##wreg##_invalidates, nullptr, 0                 \
   }
 
@@ -525,7 +514,7 @@ static uint32_t g_d31_invalidates[] = {fpu_v31, fpu_s31, LLDB_INVALID_REGNUM};
 #define DEFINE_FPU_PSEUDO(reg, size, offset, vreg)                             \
   {                                                                            \
     #reg, nullptr, size, FPU_OFFSET(fpu_##vreg - fpu_v0) + offset,             \
-        lldb::eEncodingIEEE754, lldb::eFormatFloat, MISC_KIND(fpu_##reg),      \
+        lldb::eEncodingIEEE754, lldb::eFormatFloat, LLDB_KIND(fpu_##reg),      \
         g_contained_##vreg, g_##reg##_invalidates, nullptr, 0                  \
   }
 
diff --git a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp
index 95c419b053fa9..685567416983c 100644
--- a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp
+++ b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp
@@ -17,16 +17,16 @@
 using namespace lldb_private;
 
 RegisterContextCorePOSIX_arm64::RegisterContextCorePOSIX_arm64(
-    Thread &thread, RegisterInfoInterface *register_info,
+    Thread &thread, std::unique_ptr<RegisterInfoPOSIX_arm64> register_info,
     const DataExtractor &gpregset, llvm::ArrayRef<CoreNote> notes)
-    : RegisterContextPOSIX_arm64(thread, 0, register_info) {
+    : RegisterContextPOSIX_arm64(thread, std::move(register_info)) {
   m_gpr_buffer = std::make_shared<DataBufferHeap>(gpregset.GetDataStart(),
                                                   gpregset.GetByteSize());
   m_gpr.SetData(m_gpr_buffer);
   m_gpr.SetByteOrder(gpregset.GetByteOrder());
 
   m_fpregset = getRegset(
-      notes, register_info->GetTargetArchitecture().GetTriple(), FPR_Desc);
+      notes, m_register_info_up->GetTargetArchitecture().GetTriple(), FPR_Desc);
 }
 
 RegisterContextCorePOSIX_arm64::~RegisterContextCorePOSIX_arm64() {}
@@ -61,7 +61,7 @@ bool RegisterContextCorePOSIX_arm64::ReadRegister(const RegisterInfo *reg_info,
     return false;
 
   offset -= GetGPRSize();
-  if (IsFPR(reg) && offset + reg_info->byte_size <= sizeof(FPU)) {
+  if (IsFPR(reg) && offset + reg_info->byte_size <= GetFPUSize()) {
     Status error;
     value.SetFromMemoryData(reg_info, m_fpregset.GetDataStart() + offset,
                             reg_info->byte_size, lldb::eByteOrderLittle, error);
diff --git a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h
index 5bbcdf5677f6f..830e0ff91e4c6 100644
--- a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h
+++ b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h
@@ -18,7 +18,7 @@ class RegisterContextCorePOSIX_arm64 : public RegisterContextPOSIX_arm64 {
 public:
   RegisterContextCorePOSIX_arm64(
       lldb_private::Thread &thread,
-      lldb_private::RegisterInfoInterface *register_info,
+      std::unique_ptr<RegisterInfoPOSIX_arm64> register_info,
       const lldb_private::DataExtractor &gpregset,
       llvm::ArrayRef<lldb_private::CoreNote> notes);
 
diff --git a/lldb/source/Plugins/Process/elf-core/ThreadElfCore.cpp b/lldb/source/Plugins/Process/elf-core/ThreadElfCore.cpp
index 714090459cbbd..6b5acfa4bc1bb 100644
--- a/lldb/source/Plugins/Process/elf-core/ThreadElfCore.cpp
+++ b/lldb/source/Plugins/Process/elf-core/ThreadElfCore.cpp
@@ -82,7 +82,6 @@ ThreadElfCore::CreateRegisterContextForFrame(StackFrame *frame) {
     case llvm::Triple::FreeBSD: {
       switch (arch.GetMachine()) {
       case llvm::Triple::aarch64:
-        reg_interface = new RegisterInfoPOSIX_arm64(arch);
         break;
       case llvm::Triple::arm:
         reg_interface = new RegisterInfoPOSIX_arm(arch);
@@ -111,7 +110,6 @@ ThreadElfCore::CreateRegisterContextForFrame(StackFrame *frame) {
     case llvm::Triple::NetBSD: {
       switch (arch.GetMachine()) {
       case llvm::Triple::aarch64:
-        reg_interface = new RegisterInfoPOSIX_arm64(arch);
         break;
       case llvm::Triple::x86_64:
         reg_interface = new RegisterContextNetBSD_x86_64(arch);
@@ -128,7 +126,6 @@ ThreadElfCore::CreateRegisterContextForFrame(StackFrame *frame) {
         reg_interface = new RegisterInfoPOSIX_arm(arch);
         break;
       case llvm::Triple::aarch64:
-        reg_interface = new RegisterInfoPOSIX_arm64(arch);
         break;
       case llvm::Triple::mipsel:
       case llvm::Triple::mips:
@@ -159,7 +156,6 @@ ThreadElfCore::CreateRegisterContextForFrame(StackFrame *frame) {
     case llvm::Triple::OpenBSD: {
       switch (arch.GetMachine()) {
       case llvm::Triple::aarch64:
-        reg_interface = new RegisterInfoPOSIX_arm64(arch);
         break;
       case llvm::Triple::arm:
         reg_interface = new RegisterInfoPOSIX_arm(arch);
@@ -180,7 +176,7 @@ ThreadElfCore::CreateRegisterContextForFrame(StackFrame *frame) {
       break;
     }
 
-    if (!reg_interface) {
+    if (!reg_interface && arch.GetMachine() != llvm::Triple::aarch64) {
       LLDB_LOGF(log, "elf-core::%s:: Architecture(%d) or OS(%d) not supported",
                 __FUNCTION__, arch.GetMachine(), arch.GetTriple().getOS());
       assert(false && "Architecture or OS not supported");
@@ -189,7 +185,8 @@ ThreadElfCore::CreateRegisterContextForFrame(StackFrame *frame) {
     switch (arch.GetMachine()) {
     case llvm::Triple::aarch64:
       m_thread_reg_ctx_sp = std::make_shared<RegisterContextCorePOSIX_arm64>(
-          *this, reg_interface, m_gpregset_data, m_notes);
+          *this, std::make_unique<RegisterInfoPOSIX_arm64>(arch),
+          m_gpregset_data, m_notes);
       break;
     case llvm::Triple::arm:
       m_thread_reg_ctx_sp = std::make_shared<RegisterContextCorePOSIX_arm>(
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index c673a16b348c3..ff263fa162587 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -17,6 +17,9 @@
 #include <unistd.h>
 #endif
 #include <sys/stat.h>
+#if defined(__APPLE__)
+#include <sys/sysctl.h>
+#endif
 #include <sys/types.h>
 #include <time.h>
 
@@ -186,21 +189,6 @@ static const ProcessKDPPropertiesSP &GetGlobalPluginProperties() {
 #define HIGH_PORT (49151u)
 #endif
 
-#if defined(__APPLE__) &&                                                      \
-    (defined(__arm__) || defined(__arm64__) || defined(__aarch64__))
-static bool rand_initialized = false;
-
-static inline uint16_t get_random_port() {
-  if (!rand_initialized) {
-    time_t seed = time(NULL);
-
-    rand_initialized = true;
-    srand(seed);
-  }
-  return (rand() % (HIGH_PORT - LOW_PORT)) + LOW_PORT;
-}
-#endif
-
 ConstString ProcessGDBRemote::GetPluginNameStatic() {
   static ConstString g_name("gdb-remote");
   return g_name;
@@ -3432,6 +3420,23 @@ Status ProcessGDBRemote::LaunchAndConnectToDebugserver(
         std::bind(MonitorDebugserverProcess, this_wp, _1, _2, _3, _4), false);
     debugserver_launch_info.SetUserID(process_info.GetUserID());
 
+#if defined(__APPLE__)
+    // On macOS 11, we need to support x86_64 applications translated to
+    // arm64. We check whether a binary is translated and spawn the correct
+    // debugserver accordingly.
+    int mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_PID,
+                  static_cast<int>(process_info.GetProcessID()) };
+    struct kinfo_proc processInfo;
+    size_t bufsize = sizeof(processInfo);
+    if (sysctl(mib, (unsigned)(sizeof(mib)/sizeof(int)), &processInfo,
+               &bufsize, NULL, 0) == 0 && bufsize > 0) {
+      if (processInfo.kp_proc.p_flag & P_TRANSLATED) {
+        FileSpec rosetta_debugserver("/Library/Apple/usr/libexec/oah/debugserver");
+        debugserver_launch_info.SetExecutableFile(rosetta_debugserver, false);
+      }
+    }
+#endif
+
     int communication_fd = -1;
 #ifdef USE_SOCKETPAIR_FOR_LOCAL_CONNECTION
     // Use a socketpair on non-Windows systems for security and performance
diff --git a/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp b/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp
index ecee8cc674f81..acd6128d84c5a 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp
@@ -57,3 +57,35 @@ llvm::Error Lua::LoadModule(llvm::StringRef filename) {
   lua_setglobal(m_lua_state, module_name.GetCString());
   return llvm::Error::success();
 }
+
+llvm::Error Lua::ChangeIO(FILE *out, FILE *err) {
+  assert(out != nullptr);
+  assert(err != nullptr);
+
+  lua_getglobal(m_lua_state, "io");
+
+  lua_getfield(m_lua_state, -1, "stdout");
+  if (luaL_Stream *s = static_cast<luaL_Stream *>(
+          luaL_testudata(m_lua_state, -1, LUA_FILEHANDLE))) {
+    s->f = out;
+    lua_pop(m_lua_state, 1);
+  } else {
+    lua_pop(m_lua_state, 2);
+    return llvm::make_error<llvm::StringError>("could not get stdout",
+                                               llvm::inconvertibleErrorCode());
+  }
+
+  lua_getfield(m_lua_state, -1, "stderr");
+  if (luaL_Stream *s = static_cast<luaL_Stream *>(
+          luaL_testudata(m_lua_state, -1, LUA_FILEHANDLE))) {
+    s->f = out;
+    lua_pop(m_lua_state, 1);
+  } else {
+    lua_pop(m_lua_state, 2);
+    return llvm::make_error<llvm::StringError>("could not get stderr",
+                                               llvm::inconvertibleErrorCode());
+  }
+
+  lua_pop(m_lua_state, 1);
+  return llvm::Error::success();
+}
diff --git a/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.h b/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.h
index f2984a925dfe1..300115aac8a7d 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.h
@@ -38,6 +38,7 @@ class Lua {
 
   llvm::Error Run(llvm::StringRef buffer);
   llvm::Error LoadModule(llvm::StringRef filename);
+  llvm::Error ChangeIO(FILE *out, FILE *err);
 
 private:
   lua_State *m_lua_state;
diff --git a/lldb/source/Plugins/ScriptInterpreter/Lua/ScriptInterpreterLua.cpp b/lldb/source/Plugins/ScriptInterpreter/Lua/ScriptInterpreterLua.cpp
index cbd639417990f..8cbeac4563c30 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Lua/ScriptInterpreterLua.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Lua/ScriptInterpreterLua.cpp
@@ -15,6 +15,7 @@
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StringList.h"
 #include "lldb/Utility/Timer.h"
+#include "llvm/Support/FormatAdapters.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -30,6 +31,9 @@ class IOHandlerLuaInterpreter : public IOHandlerDelegate,
                           ">>> ", "..> ", true, debugger.GetUseColor(), 0,
                           *this, nullptr),
         m_script_interpreter(script_interpreter) {
+    llvm::cantFail(m_script_interpreter.GetLua().ChangeIO(
+        debugger.GetOutputFile().GetStream(),
+        debugger.GetErrorFile().GetStream()));
     llvm::cantFail(m_script_interpreter.EnterSession(debugger.GetID()));
   }
 
@@ -39,6 +43,11 @@ class IOHandlerLuaInterpreter : public IOHandlerDelegate,
 
   void IOHandlerInputComplete(IOHandler &io_handler,
                               std::string &data) override {
+    if (llvm::StringRef(data).rtrim() == "quit") {
+      io_handler.SetIsDone(true);
+      return;
+    }
+
     if (llvm::Error error = m_script_interpreter.GetLua().Run(data)) {
       *GetOutputStreamFileSP() << llvm::toString(std::move(error));
     }
@@ -57,12 +66,43 @@ ScriptInterpreterLua::~ScriptInterpreterLua() {}
 bool ScriptInterpreterLua::ExecuteOneLine(llvm::StringRef command,
                                           CommandReturnObject *result,
                                           const ExecuteScriptOptions &options) {
+  if (command.empty()) {
+    if (result)
+      result->AppendError("empty command passed to lua\n");
+    return false;
+  }
+
+  llvm::Expected<std::unique_ptr<ScriptInterpreterIORedirect>>
+      io_redirect_or_error = ScriptInterpreterIORedirect::Create(
+          options.GetEnableIO(), m_debugger, result);
+  if (!io_redirect_or_error) {
+    if (result)
+      result->AppendErrorWithFormatv(
+          "failed to redirect I/O: {0}\n",
+          llvm::fmt_consume(io_redirect_or_error.takeError()));
+    else
+      llvm::consumeError(io_redirect_or_error.takeError());
+    return false;
+  }
+
+  ScriptInterpreterIORedirect &io_redirect = **io_redirect_or_error;
+
+  if (llvm::Error e =
+          m_lua->ChangeIO(io_redirect.GetOutputFile()->GetStream(),
+                          io_redirect.GetErrorFile()->GetStream())) {
+    result->AppendErrorWithFormatv("lua failed to redirect I/O: {0}\n",
+                                   llvm::toString(std::move(e)));
+    return false;
+  }
+
   if (llvm::Error e = m_lua->Run(command)) {
     result->AppendErrorWithFormatv(
         "lua failed attempting to evaluate '{0}': {1}\n", command,
         llvm::toString(std::move(e)));
     return false;
   }
+
+  io_redirect.Flush();
   return true;
 }
 
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
index 39f76e7c5bb58..9f56b4fa60a50 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -16,7 +16,6 @@
 #include "PythonDataObjects.h"
 #include "PythonReadline.h"
 #include "ScriptInterpreterPythonImpl.h"
-
 #include "lldb/API/SBFrame.h"
 #include "lldb/API/SBValue.h"
 #include "lldb/Breakpoint/StoppointCallbackContext.h"
@@ -26,7 +25,6 @@
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSummary.h"
-#include "lldb/Host/ConnectionFileDescriptor.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/HostInfo.h"
 #include "lldb/Host/Pipe.h"
@@ -35,13 +33,9 @@
 #include "lldb/Target/Thread.h"
 #include "lldb/Target/ThreadPlan.h"
 #include "lldb/Utility/Timer.h"
-
-#if defined(_WIN32)
-#include "lldb/Host/windows/ConnectionGenericFileWindows.h"
-#endif
-
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FormatAdapters.h"
 
@@ -895,15 +889,6 @@ bool ScriptInterpreterPythonImpl::GetEmbeddedInterpreterModuleObjects() {
   return m_run_one_line_function.IsValid();
 }
 
-static void ReadThreadBytesReceived(void *baton, const void *src,
-                                    size_t src_len) {
-  if (src && src_len) {
-    Stream *strm = (Stream *)baton;
-    strm->Write(src, src_len);
-    strm->Flush();
-  }
-}
-
 bool ScriptInterpreterPythonImpl::ExecuteOneLine(
     llvm::StringRef command, CommandReturnObject *result,
     const ExecuteScriptOptions &options) {
@@ -919,77 +904,21 @@ bool ScriptInterpreterPythonImpl::ExecuteOneLine(
     // another string to pass to PyRun_SimpleString messes up the escaping.  So
     // we use the following more complicated method to pass the command string
     // directly down to Python.
-    Debugger &debugger = m_debugger;
-
-    FileSP input_file_sp;
-    StreamFileSP output_file_sp;
-    StreamFileSP error_file_sp;
-    Communication output_comm(
-        "lldb.ScriptInterpreterPythonImpl.ExecuteOneLine.comm");
-    bool join_read_thread = false;
-    if (options.GetEnableIO()) {
-      if (result) {
-        input_file_sp = debugger.GetInputFileSP();
-        // Set output to a temporary file so we can forward the results on to
-        // the result object
-
-        Pipe pipe;
-        Status pipe_result = pipe.CreateNew(false);
-        if (pipe_result.Success()) {
-#if defined(_WIN32)
-          lldb::file_t read_file = pipe.GetReadNativeHandle();
-          pipe.ReleaseReadFileDescriptor();
-          std::unique_ptr<ConnectionGenericFile> conn_up(
-              new ConnectionGenericFile(read_file, true));
-#else
-          std::unique_ptr<ConnectionFileDescriptor> conn_up(
-              new ConnectionFileDescriptor(pipe.ReleaseReadFileDescriptor(),
-                                           true));
-#endif
-          if (conn_up->IsConnected()) {
-            output_comm.SetConnection(std::move(conn_up));
-            output_comm.SetReadThreadBytesReceivedCallback(
-                ReadThreadBytesReceived, &result->GetOutputStream());
-            output_comm.StartReadThread();
-            join_read_thread = true;
-            FILE *outfile_handle =
-                fdopen(pipe.ReleaseWriteFileDescriptor(), "w");
-            output_file_sp = std::make_shared<StreamFile>(outfile_handle, true);
-            error_file_sp = output_file_sp;
-            if (outfile_handle)
-              ::setbuf(outfile_handle, nullptr);
-
-            result->SetImmediateOutputFile(
-                debugger.GetOutputStream().GetFileSP());
-            result->SetImmediateErrorFile(
-                debugger.GetErrorStream().GetFileSP());
-          }
-        }
-      }
-      if (!input_file_sp || !output_file_sp || !error_file_sp)
-        debugger.AdoptTopIOHandlerFilesIfInvalid(input_file_sp, output_file_sp,
-                                                 error_file_sp);
-    } else {
-      auto nullin = FileSystem::Instance().Open(
-                                  FileSpec(FileSystem::DEV_NULL),
-                                  File::eOpenOptionRead);
-      auto nullout = FileSystem::Instance().Open(
-                                  FileSpec(FileSystem::DEV_NULL),
-                                  File::eOpenOptionWrite);
-      if (!nullin) {
-        result->AppendErrorWithFormatv("failed to open /dev/null: {0}\n",
-                                       llvm::fmt_consume(nullin.takeError()));
-        return false;
-      }
-      if (!nullout) {
-        result->AppendErrorWithFormatv("failed to open /dev/null: {0}\n",
-                                       llvm::fmt_consume(nullout.takeError()));
-        return false;
-      }
-      input_file_sp = std::move(nullin.get());
-      error_file_sp = output_file_sp = std::make_shared<StreamFile>(std::move(nullout.get()));
+    llvm::Expected<std::unique_ptr<ScriptInterpreterIORedirect>>
+        io_redirect_or_error = ScriptInterpreterIORedirect::Create(
+            options.GetEnableIO(), m_debugger, result);
+    if (!io_redirect_or_error) {
+      if (result)
+        result->AppendErrorWithFormatv(
+            "failed to redirect I/O: {0}\n",
+            llvm::fmt_consume(io_redirect_or_error.takeError()));
+      else
+        llvm::consumeError(io_redirect_or_error.takeError());
+      return false;
     }
 
+    ScriptInterpreterIORedirect &io_redirect = **io_redirect_or_error;
+
     bool success = false;
     {
       // WARNING!  It's imperative that this RAII scope be as tight as
@@ -1005,8 +934,9 @@ bool ScriptInterpreterPythonImpl::ExecuteOneLine(
           Locker::AcquireLock | Locker::InitSession |
               (options.GetSetLLDBGlobals() ? Locker::InitGlobals : 0) |
               ((result && result->GetInteractive()) ? 0 : Locker::NoSTDIN),
-          Locker::FreeAcquiredLock | Locker::TearDownSession, input_file_sp,
-          output_file_sp->GetFileSP(), error_file_sp->GetFileSP());
+          Locker::FreeAcquiredLock | Locker::TearDownSession,
+          io_redirect.GetInputFile(), io_redirect.GetOutputFile(),
+          io_redirect.GetErrorFile());
 
       // Find the correct script interpreter dictionary in the main module.
       PythonDictionary &session_dict = GetSessionDictionary();
@@ -1032,21 +962,7 @@ bool ScriptInterpreterPythonImpl::ExecuteOneLine(
         }
       }
 
-      // Flush our output and error file handles
-      output_file_sp->Flush();
-      error_file_sp->Flush();
-    }
-
-    if (join_read_thread) {
-      // Close the write end of the pipe since we are done with our one line
-      // script. This should cause the read thread that output_comm is using to
-      // exit
-      output_file_sp->GetFile().Close();
-      // The close above should cause this thread to exit when it gets to the
-      // end of file, so let it get all its data
-      output_comm.JoinReadThread();
-      // Now we can close the read end of the pipe
-      output_comm.Disconnect();
+      io_redirect.Flush();
     }
 
     if (success)
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 0fc3e9c4928be..7de88274ccf6e 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -90,7 +90,7 @@ static bool DeclKindIsCXXClass(clang::Decl::Kind decl_kind) {
 
 ClangASTImporter &DWARFASTParserClang::GetClangASTImporter() {
   if (!m_clang_ast_importer_up) {
-    m_clang_ast_importer_up.reset(new ClangASTImporter);
+    m_clang_ast_importer_up = std::make_unique<ClangASTImporter>();
   }
   return *m_clang_ast_importer_up;
 }
@@ -1263,44 +1263,7 @@ TypeSP DWARFASTParserClang::ParseArrayType(const DWARFDIE &die,
   if (attrs.byte_stride == 0 && attrs.bit_stride == 0)
     attrs.byte_stride = element_type->GetByteSize().getValueOr(0);
   CompilerType array_element_type = element_type->GetForwardCompilerType();
-
-  if (TypeSystemClang::IsCXXClassType(array_element_type) &&
-      !array_element_type.GetCompleteType()) {
-    ModuleSP module_sp = die.GetModule();
-    if (module_sp) {
-      if (die.GetCU()->GetProducer() == eProducerClang)
-        module_sp->ReportError(
-            "DWARF DW_TAG_array_type DIE at 0x%8.8x has a "
-            "class/union/struct element type DIE 0x%8.8x that is a "
-            "forward declaration, not a complete definition.\nTry "
-            "compiling the source file with -fstandalone-debug or "
-            "disable -gmodules",
-            die.GetOffset(), type_die.GetOffset());
-      else
-        module_sp->ReportError(
-            "DWARF DW_TAG_array_type DIE at 0x%8.8x has a "
-            "class/union/struct element type DIE 0x%8.8x that is a "
-            "forward declaration, not a complete definition.\nPlease "
-            "file a bug against the compiler and include the "
-            "preprocessed output for %s",
-            die.GetOffset(), type_die.GetOffset(), GetUnitName(die).c_str());
-    }
-
-    // We have no choice other than to pretend that the element class
-    // type is complete. If we don't do this, clang will crash when
-    // trying to layout the class. Since we provide layout
-    // assistance, all ivars in this class and other classes will be
-    // fine, this is the best we can do short of crashing.
-    if (TypeSystemClang::StartTagDeclarationDefinition(array_element_type)) {
-      TypeSystemClang::CompleteTagDeclarationDefinition(array_element_type);
-    } else {
-      module_sp->ReportError("DWARF DIE at 0x%8.8x was not able to "
-                             "start its definition.\nPlease file a "
-                             "bug and attach the file at the start "
-                             "of this error message",
-                             type_die.GetOffset());
-    }
-  }
+  CompleteType(array_element_type);
 
   uint64_t array_element_bit_stride =
       attrs.byte_stride * 8 + attrs.bit_stride;
@@ -1355,6 +1318,28 @@ TypeSP DWARFASTParserClang::ParsePointerToMemberType(
   return nullptr;
 }
 
+void DWARFASTParserClang::CompleteType(CompilerType type) {
+  // Technically, enums can be incomplete too, but we don't handle those as they
+  // are emitted even under -flimit-debug-info.
+  if (!TypeSystemClang::IsCXXClassType(type))
+    return;
+
+  if (type.GetCompleteType())
+    return;
+
+  // No complete definition in this module.  Mark the class as complete to
+  // satisfy local ast invariants, but make a note of the fact that
+  // it is not _really_ complete so we can later search for a definition in a
+  // different module.
+  // Since we provide layout assistance, layouts of types containing this class
+  // will be correct even if we  are not able to find the definition elsewhere.
+  bool started = TypeSystemClang::StartTagDeclarationDefinition(type);
+  lldbassert(started && "Unable to start a class type definition.");
+  TypeSystemClang::CompleteTagDeclarationDefinition(type);
+  const clang::TagDecl *td = ClangUtil::GetAsTagDecl(type);
+  m_ast.GetMetadata(td)->SetIsForcefullyCompleted();
+}
+
 TypeSP DWARFASTParserClang::UpdateSymbolContextScopeForType(
     const SymbolContext &sc, const DWARFDIE &die, TypeSP type_sp) {
   if (!type_sp)
@@ -1783,7 +1768,7 @@ class DWARFASTParserClang::DelayedAddObjCClassProperty {
         m_property_getter_name(property_getter_name),
         m_property_attributes(property_attributes) {
     if (metadata != nullptr) {
-      m_metadata_up.reset(new ClangASTMetadata());
+      m_metadata_up = std::make_unique<ClangASTMetadata>();
       *m_metadata_up = *metadata;
     }
   }
@@ -1803,7 +1788,7 @@ class DWARFASTParserClang::DelayedAddObjCClassProperty {
     m_property_attributes = rhs.m_property_attributes;
 
     if (rhs.m_metadata_up) {
-      m_metadata_up.reset(new ClangASTMetadata());
+      m_metadata_up = std::make_unique<ClangASTMetadata>();
       *m_metadata_up = *rhs.m_metadata_up;
     }
     return *this;
@@ -1835,8 +1820,8 @@ bool DWARFASTParserClang::ParseTemplateDIE(
 
   switch (tag) {
   case DW_TAG_GNU_template_parameter_pack: {
-    template_param_infos.packed_args.reset(
-      new TypeSystemClang::TemplateParameterInfos);
+    template_param_infos.packed_args =
+        std::make_unique<TypeSystemClang::TemplateParameterInfos>();
     for (DWARFDIE child_die = die.GetFirstChild(); child_die.IsValid();
          child_die = child_die.GetSibling()) {
       if (!ParseTemplateDIE(child_die, *template_param_infos.packed_args))
@@ -2057,33 +2042,8 @@ bool DWARFASTParserClang::CompleteRecordType(const DWARFDIE &die,
       for (const auto &base_class : bases) {
         clang::TypeSourceInfo *type_source_info =
             base_class->getTypeSourceInfo();
-        if (type_source_info) {
-          CompilerType base_class_type =
-              m_ast.GetType(type_source_info->getType());
-          if (!base_class_type.GetCompleteType()) {
-            auto module = dwarf->GetObjectFile()->GetModule();
-            module->ReportError(":: Class '%s' has a base class '%s' which "
-                                "does not have a complete definition.",
-                                die.GetName(),
-                                base_class_type.GetTypeName().GetCString());
-            if (die.GetCU()->GetProducer() == eProducerClang)
-              module->ReportError(":: Try compiling the source file with "
-                                  "-fstandalone-debug.");
-
-            // We have no choice other than to pretend that the base class
-            // is complete. If we don't do this, clang will crash when we
-            // call setBases() inside of
-            // "clang_type.TransferBaseClasses()" below. Since we
-            // provide layout assistance, all ivars in this class and other
-            // classes will be fine, this is the best we can do short of
-            // crashing.
-            if (TypeSystemClang::StartTagDeclarationDefinition(
-                    base_class_type)) {
-              TypeSystemClang::CompleteTagDeclarationDefinition(
-                  base_class_type);
-            }
-          }
-        }
+        if (type_source_info)
+          CompleteType(m_ast.GetType(type_source_info->getType()));
       }
 
       m_ast.TransferBaseClasses(clang_type.GetOpaqueQualType(),
@@ -2354,8 +2314,8 @@ Function *DWARFASTParserClang::ParseFunctionFromDWARF(CompileUnit &comp_unit,
       FunctionSP func_sp;
       std::unique_ptr<Declaration> decl_up;
       if (decl_file != 0 || decl_line != 0 || decl_column != 0)
-        decl_up.reset(new Declaration(die.GetCU()->GetFile(decl_file),
-                                      decl_line, decl_column));
+        decl_up = std::make_unique<Declaration>(die.GetCU()->GetFile(decl_file),
+                                                decl_line, decl_column);
 
       SymbolFileDWARF *dwarf = die.GetDWARF();
       // Supply the type _only_ if it has already been parsed
@@ -2746,44 +2706,7 @@ void DWARFASTParserClang::ParseSingleMember(
             }
           }
 
-          if (TypeSystemClang::IsCXXClassType(member_clang_type) &&
-              !member_clang_type.GetCompleteType()) {
-            if (die.GetCU()->GetProducer() == eProducerClang)
-              module_sp->ReportError(
-                  "DWARF DIE at 0x%8.8x (class %s) has a member variable "
-                  "0x%8.8x (%s) whose type is a forward declaration, not a "
-                  "complete definition.\nTry compiling the source file "
-                  "with -fstandalone-debug",
-                  parent_die.GetOffset(), parent_die.GetName(), die.GetOffset(),
-                  name);
-            else
-              module_sp->ReportError(
-                  "DWARF DIE at 0x%8.8x (class %s) has a member variable "
-                  "0x%8.8x (%s) whose type is a forward declaration, not a "
-                  "complete definition.\nPlease file a bug against the "
-                  "compiler and include the preprocessed output for %s",
-                  parent_die.GetOffset(), parent_die.GetName(), die.GetOffset(),
-                  name, GetUnitName(parent_die).c_str());
-            // We have no choice other than to pretend that the member
-            // class is complete. If we don't do this, clang will crash
-            // when trying to layout the class. Since we provide layout
-            // assistance, all ivars in this class and other classes will
-            // be fine, this is the best we can do short of crashing.
-            if (TypeSystemClang::StartTagDeclarationDefinition(
-                    member_clang_type)) {
-              TypeSystemClang::CompleteTagDeclarationDefinition(
-                  member_clang_type);
-            } else {
-              module_sp->ReportError(
-                  "DWARF DIE at 0x%8.8x (class %s) has a member variable "
-                  "0x%8.8x (%s) whose type claims to be a C++ class but we "
-                  "were not able to start its definition.\nPlease file a "
-                  "bug and attach the file at the start of this error "
-                  "message",
-                  parent_die.GetOffset(), parent_die.GetName(), die.GetOffset(),
-                  name);
-            }
-          }
+          CompleteType(member_clang_type);
 
           field_decl = TypeSystemClang::AddFieldToRecordType(
               class_clang_type, name, member_clang_type, accessibility,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
index 71d978536f8f2..cb718a207d2d4 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
@@ -219,6 +219,12 @@ class DWARFASTParserClang : public DWARFASTParser {
                               ParsedDWARFTypeAttributes &attrs);
   lldb::TypeSP ParsePointerToMemberType(const DWARFDIE &die,
                                         const ParsedDWARFTypeAttributes &attrs);
+
+  /// Complete a type from debug info, or mark it as forcefully completed if
+  /// there is no of the type in the current Module. Call this function in
+  /// contexts where the usual C++ rules require a type to be complete (base
+  /// class, member, etc.).
+  void CompleteType(lldb_private::CompilerType type);
 };
 
 /// Parsed form of all attributes that are relevant for type reconstruction.
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
index b8d16758246f6..dfa40759a7ff0 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
@@ -769,7 +769,7 @@ SymbolFileDWARFDwo *DWARFUnit::GetDwoSymbolFile() {
 
 const DWARFDebugAranges &DWARFUnit::GetFunctionAranges() {
   if (m_func_aranges_up == nullptr) {
-    m_func_aranges_up.reset(new DWARFDebugAranges());
+    m_func_aranges_up = std::make_unique<DWARFDebugAranges>();
     const DWARFDebugInfoEntry *die = DIEPtr();
     if (die)
       die->BuildFunctionAddressRangeTable(this, m_func_aranges_up.get());
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index d662dcba4cddf..9f64e5255fd5c 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -651,7 +651,7 @@ DWARFDebugRanges *SymbolFileDWARF::GetDebugRanges() {
                        static_cast<void *>(this));
 
     if (m_context.getOrLoadRangesData().GetByteSize() > 0)
-      m_ranges.reset(new DWARFDebugRanges());
+      m_ranges = std::make_unique<DWARFDebugRanges>();
 
     if (m_ranges)
       m_ranges->Extract(m_context);
@@ -1192,15 +1192,15 @@ size_t SymbolFileDWARF::ParseBlocksRecursive(
             (name != nullptr || mangled_name != nullptr)) {
           std::unique_ptr<Declaration> decl_up;
           if (decl_file != 0 || decl_line != 0 || decl_column != 0)
-            decl_up.reset(new Declaration(
+            decl_up = std::make_unique<Declaration>(
                 comp_unit.GetSupportFiles().GetFileSpecAtIndex(decl_file),
-                decl_line, decl_column));
+                decl_line, decl_column);
 
           std::unique_ptr<Declaration> call_up;
           if (call_file != 0 || call_line != 0 || call_column != 0)
-            call_up.reset(new Declaration(
+            call_up = std::make_unique<Declaration>(
                 comp_unit.GetSupportFiles().GetFileSpecAtIndex(call_file),
-                call_line, call_column));
+                call_line, call_column);
 
           block->SetInlinedFunctionInfo(name, mangled_name, decl_up.get(),
                                         call_up.get());
@@ -1763,7 +1763,7 @@ void SymbolFileDWARF::UpdateExternalModuleListIfNeeded() {
 
 SymbolFileDWARF::GlobalVariableMap &SymbolFileDWARF::GetGlobalAranges() {
   if (!m_global_aranges_up) {
-    m_global_aranges_up.reset(new GlobalVariableMap());
+    m_global_aranges_up = std::make_unique<GlobalVariableMap>();
 
     ModuleSP module_sp = GetObjectFile()->GetModule();
     if (module_sp) {
@@ -3844,6 +3844,11 @@ SymbolFileDWARF::CollectCallEdges(ModuleSP module, DWARFDIE function_die) {
 
 std::vector<std::unique_ptr<lldb_private::CallEdge>>
 SymbolFileDWARF::ParseCallEdgesInFunction(UserID func_id) {
+  // ParseCallEdgesInFunction must be called at the behest of an exclusively
+  // locked lldb::Function instance. Storage for parsed call edges is owned by
+  // the lldb::Function instance: locking at the SymbolFile level would be too
+  // late, because the act of storing results from ParseCallEdgesInFunction
+  // would be racy.
   DWARFDIE func_die = GetDIE(func_id.GetID());
   if (func_die.IsValid())
     return CollectCallEdges(GetObjectFile()->GetModule(), func_die);
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp
index b88f87527d8c6..0acc77d7c67ff 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp
@@ -681,7 +681,7 @@ bool PdbAstBuilder::CompleteTagDecl(clang::TagDecl &tag) {
   // Visit all members of this class, then perform any finalization necessary
   // to complete the class.
   CompilerType ct = ToCompilerType(tag_qt);
-  UdtRecordCompleter completer(best_ti, ct, tag, *this, m_index.tpi());
+  UdtRecordCompleter completer(best_ti, ct, tag, *this, m_index);
   auto error =
       llvm::codeview::visitMemberRecordStream(field_list_cvt.data(), completer);
   completer.complete();
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/UdtRecordCompleter.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/UdtRecordCompleter.cpp
index 308ba9e3dec0b..c8fb46c750342 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/UdtRecordCompleter.cpp
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/UdtRecordCompleter.cpp
@@ -6,14 +6,17 @@
 #include "PdbUtil.h"
 
 #include "Plugins/ExpressionParser/Clang/ClangASTImporter.h"
+#include "Plugins/ExpressionParser/Clang/ClangUtil.h"
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 #include "lldb/Symbol/Type.h"
 #include "lldb/Utility/LLDBAssert.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
 
+#include "llvm/DebugInfo/CodeView/SymbolDeserializer.h"
 #include "llvm/DebugInfo/CodeView/TypeDeserializer.h"
 #include "llvm/DebugInfo/CodeView/TypeIndex.h"
+#include "llvm/DebugInfo/PDB/Native/GlobalsStream.h"
 #include "llvm/DebugInfo/PDB/Native/TpiStream.h"
 #include "llvm/DebugInfo/PDB/PDBTypes.h"
 
@@ -29,10 +32,10 @@ UdtRecordCompleter::UdtRecordCompleter(PdbTypeSymId id,
                                        CompilerType &derived_ct,
                                        clang::TagDecl &tag_decl,
                                        PdbAstBuilder &ast_builder,
-                                       TpiStream &tpi)
+                                       PdbIndex &index)
     : m_id(id), m_derived_ct(derived_ct), m_tag_decl(tag_decl),
-      m_ast_builder(ast_builder), m_tpi(tpi) {
-  CVType cvt = m_tpi.getType(m_id.index);
+      m_ast_builder(ast_builder), m_index(index) {
+  CVType cvt = m_index.tpi().getType(m_id.index);
   switch (cvt.kind()) {
   case LF_ENUM:
     llvm::cantFail(TypeDeserializer::deserializeAs<EnumRecord>(cvt, m_cvr.er));
@@ -55,7 +58,7 @@ clang::QualType UdtRecordCompleter::AddBaseClassForTypeIndex(
   PdbTypeSymId type_id(ti);
   clang::QualType qt = m_ast_builder.GetOrCreateType(type_id);
 
-  CVType udt_cvt = m_tpi.getType(ti);
+  CVType udt_cvt = m_index.tpi().getType(ti);
 
   std::unique_ptr<clang::CXXBaseSpecifier> base_spec =
       m_ast_builder.clang().CreateBaseClassSpecifier(
@@ -128,9 +131,70 @@ Error UdtRecordCompleter::visitKnownMember(
 
   lldb::AccessType access =
       TranslateMemberAccess(static_data_member.getAccess());
-  TypeSystemClang::AddVariableToRecordType(
+  auto decl = TypeSystemClang::AddVariableToRecordType(
       m_derived_ct, static_data_member.Name, member_ct, access);
 
+  // Static constant members may be a const[expr] declaration.
+  // Query the symbol's value as the variable initializer if valid.
+  if (member_ct.IsConst()) {
+    std::string qual_name = decl->getQualifiedNameAsString();
+
+    auto results =
+        m_index.globals().findRecordsByName(qual_name, m_index.symrecords());
+
+    for (const auto &result : results) {
+      if (result.second.kind() == SymbolKind::S_CONSTANT) {
+        ConstantSym constant(SymbolRecordKind::ConstantSym);
+        cantFail(SymbolDeserializer::deserializeAs<ConstantSym>(result.second,
+                                                                constant));
+
+        clang::QualType qual_type = decl->getType();
+        unsigned type_width = decl->getASTContext().getIntWidth(qual_type);
+        unsigned constant_width = constant.Value.getBitWidth();
+
+        if (qual_type->isIntegralOrEnumerationType()) {
+          if (type_width >= constant_width) {
+            TypeSystemClang::SetIntegerInitializerForVariable(
+                decl, constant.Value.extOrTrunc(type_width));
+          } else {
+            LLDB_LOG(GetLogIfAllCategoriesSet(LIBLLDB_LOG_AST),
+                     "Class '{0}' has a member '{1}' of type '{2}' ({3} bits) "
+                     "which resolves to a wider constant value ({4} bits). "
+                     "Ignoring constant.",
+                     m_derived_ct.GetTypeName(), static_data_member.Name,
+                     member_ct.GetTypeName(), type_width, constant_width);
+          }
+        } else {
+          lldb::BasicType basic_type_enum = member_ct.GetBasicTypeEnumeration();
+          switch (basic_type_enum) {
+          case lldb::eBasicTypeFloat:
+          case lldb::eBasicTypeDouble:
+          case lldb::eBasicTypeLongDouble:
+            if (type_width == constant_width) {
+              TypeSystemClang::SetFloatingInitializerForVariable(
+                  decl, basic_type_enum == lldb::eBasicTypeFloat
+                            ? llvm::APFloat(constant.Value.bitsToFloat())
+                            : llvm::APFloat(constant.Value.bitsToDouble()));
+              decl->setConstexpr(true);
+            } else {
+              LLDB_LOG(
+                  GetLogIfAllCategoriesSet(LIBLLDB_LOG_AST),
+                  "Class '{0}' has a member '{1}' of type '{2}' ({3} bits) "
+                  "which resolves to a constant value of mismatched width "
+                  "({4} bits). Ignoring constant.",
+                  m_derived_ct.GetTypeName(), static_data_member.Name,
+                  member_ct.GetTypeName(), type_width, constant_width);
+            }
+            break;
+          default:
+            break;
+          }
+        }
+        break;
+      }
+    }
+  }
+
   // FIXME: Add a PdbSymUid namespace for field list members and update
   // the m_uid_to_decl map with this decl.
   return Error::success();
@@ -149,7 +213,7 @@ Error UdtRecordCompleter::visitKnownMember(CVMemberRecord &cvr,
 
   TypeIndex ti(data_member.Type);
   if (!ti.isSimple()) {
-    CVType cvt = m_tpi.getType(ti);
+    CVType cvt = m_index.tpi().getType(ti);
     if (cvt.kind() == LF_BITFIELD) {
       BitFieldRecord bfr;
       llvm::cantFail(TypeDeserializer::deserializeAs<BitFieldRecord>(cvt, bfr));
@@ -187,7 +251,7 @@ Error UdtRecordCompleter::visitKnownMember(CVMemberRecord &cvr,
                                            OverloadedMethodRecord &overloaded) {
   TypeIndex method_list_idx = overloaded.MethodList;
 
-  CVType method_list_type = m_tpi.getType(method_list_idx);
+  CVType method_list_type = m_index.tpi().getType(method_list_idx);
   assert(method_list_type.kind() == LF_METHODLIST);
 
   MethodOverloadListRecord method_list;
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/UdtRecordCompleter.h b/lldb/source/Plugins/SymbolFile/NativePDB/UdtRecordCompleter.h
index ed1f8e4e05b16..ae7e47c82fe55 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/UdtRecordCompleter.h
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/UdtRecordCompleter.h
@@ -25,6 +25,7 @@ class TagDecl;
 namespace llvm {
 namespace pdb {
 class TpiStream;
+class GlobalsStream;
 }
 } // namespace llvm
 
@@ -33,6 +34,7 @@ class Type;
 class CompilerType;
 namespace npdb {
 class PdbAstBuilder;
+class PdbIndex;
 
 class UdtRecordCompleter : public llvm::codeview::TypeVisitorCallbacks {
   using IndexedBase =
@@ -49,14 +51,14 @@ class UdtRecordCompleter : public llvm::codeview::TypeVisitorCallbacks {
   CompilerType &m_derived_ct;
   clang::TagDecl &m_tag_decl;
   PdbAstBuilder &m_ast_builder;
-  llvm::pdb::TpiStream &m_tpi;
+  PdbIndex &m_index;
   std::vector<IndexedBase> m_bases;
   ClangASTImporter::LayoutInfo m_layout;
 
 public:
   UdtRecordCompleter(PdbTypeSymId id, CompilerType &derived_ct,
                      clang::TagDecl &tag_decl, PdbAstBuilder &ast_builder,
-                     llvm::pdb::TpiStream &tpi);
+                     PdbIndex &index);
 
 #define MEMBER_RECORD(EnumName, EnumVal, Name)                                 \
   llvm::Error visitKnownMember(llvm::codeview::CVMemberRecord &CVR,            \
diff --git a/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp b/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
index 112bec41af8ba..d87926a6588f6 100644
--- a/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
+++ b/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
@@ -1265,6 +1265,52 @@ void PDBASTParser::AddRecordMembers(
       if (!decl)
         continue;
 
+      // Static constant members may be a const[expr] declaration.
+      // Query the symbol's value as the variable initializer if valid.
+      if (member_comp_type.IsConst()) {
+        auto value = member->getValue();
+        clang::QualType qual_type = decl->getType();
+        unsigned type_width = m_ast.getASTContext().getIntWidth(qual_type);
+        unsigned constant_width = value.getBitWidth();
+
+        if (qual_type->isIntegralOrEnumerationType()) {
+          if (type_width >= constant_width) {
+            TypeSystemClang::SetIntegerInitializerForVariable(
+                decl, value.toAPSInt().extOrTrunc(type_width));
+          } else {
+            LLDB_LOG(GetLogIfAllCategoriesSet(LIBLLDB_LOG_AST),
+                     "Class '{0}' has a member '{1}' of type '{2}' ({3} bits) "
+                     "which resolves to a wider constant value ({4} bits). "
+                     "Ignoring constant.",
+                     record_type.GetTypeName(), member_name,
+                     member_comp_type.GetTypeName(), type_width,
+                     constant_width);
+          }
+        } else {
+          switch (member_comp_type.GetBasicTypeEnumeration()) {
+          case lldb::eBasicTypeFloat:
+          case lldb::eBasicTypeDouble:
+          case lldb::eBasicTypeLongDouble:
+            if (type_width == constant_width) {
+              TypeSystemClang::SetFloatingInitializerForVariable(
+                  decl, value.toAPFloat());
+              decl->setConstexpr(true);
+            } else {
+              LLDB_LOG(GetLogIfAllCategoriesSet(LIBLLDB_LOG_AST),
+                       "Class '{0}' has a member '{1}' of type '{2}' ({3} "
+                       "bits) which resolves to a constant value of mismatched "
+                       "width ({4} bits). Ignoring constant.",
+                       record_type.GetTypeName(), member_name,
+                       member_comp_type.GetTypeName(), type_width,
+                       constant_width);
+            }
+            break;
+          default:
+            break;
+          }
+        }
+      }
+
       m_uid_to_decl[member->getSymIndexId()] = decl;
 
       break;
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index d63754d127e31..bc06ea8164d43 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -720,31 +720,31 @@ void TypeSystemClang::CreateASTContext() {
   assert(!m_ast_up);
   m_ast_owned = true;
 
-  m_language_options_up.reset(new LangOptions());
+  m_language_options_up = std::make_unique<LangOptions>();
   ParseLangArgs(*m_language_options_up, clang::Language::ObjCXX,
                 GetTargetTriple());
 
-  m_identifier_table_up.reset(
-      new IdentifierTable(*m_language_options_up, nullptr));
-  m_builtins_up.reset(new Builtin::Context());
+  m_identifier_table_up =
+      std::make_unique<IdentifierTable>(*m_language_options_up, nullptr);
+  m_builtins_up = std::make_unique<Builtin::Context>();
 
-  m_selector_table_up.reset(new SelectorTable());
+  m_selector_table_up = std::make_unique<SelectorTable>();
 
   clang::FileSystemOptions file_system_options;
-  m_file_manager_up.reset(new clang::FileManager(
-      file_system_options, FileSystem::Instance().GetVirtualFileSystem()));
+  m_file_manager_up = std::make_unique<clang::FileManager>(
+      file_system_options, FileSystem::Instance().GetVirtualFileSystem());
 
   llvm::IntrusiveRefCntPtr<DiagnosticIDs> diag_id_sp(new DiagnosticIDs());
-  m_diagnostics_engine_up.reset(
-      new DiagnosticsEngine(diag_id_sp, new DiagnosticOptions()));
+  m_diagnostics_engine_up =
+      std::make_unique<DiagnosticsEngine>(diag_id_sp, new DiagnosticOptions());
 
-  m_source_manager_up.reset(
-      new clang::SourceManager(*m_diagnostics_engine_up, *m_file_manager_up));
-  m_ast_up.reset(new ASTContext(*m_language_options_up, *m_source_manager_up,
-                                *m_identifier_table_up, *m_selector_table_up,
-                                *m_builtins_up));
+  m_source_manager_up = std::make_unique<clang::SourceManager>(
+      *m_diagnostics_engine_up, *m_file_manager_up);
+  m_ast_up = std::make_unique<ASTContext>(
+      *m_language_options_up, *m_source_manager_up, *m_identifier_table_up,
+      *m_selector_table_up, *m_builtins_up);
 
-  m_diagnostic_consumer_up.reset(new NullDiagnosticConsumer);
+  m_diagnostic_consumer_up = std::make_unique<NullDiagnosticConsumer>();
   m_ast_up->getDiagnostics().setClient(m_diagnostic_consumer_up.get(), false);
 
   // This can be NULL if we don't know anything about the architecture or if
@@ -1729,10 +1729,8 @@ TypeSystemClang::GetNumBaseClasses(const CXXRecordDecl *cxx_record_decl,
           base_class_end = cxx_record_decl->bases_end();
            base_class != base_class_end; ++base_class) {
         // Skip empty base classes
-        if (omit_empty_base_classes) {
-          if (BaseSpecifierIsEmpty(base_class))
-            continue;
-        }
+        if (BaseSpecifierIsEmpty(base_class))
+          continue;
         ++num_bases;
       }
     } else
@@ -4853,6 +4851,9 @@ lldb::Encoding TypeSystemClang::GetEncoding(lldb::opaque_compiler_type_t type,
     case clang::BuiltinType::SveUint64x4:
     case clang::BuiltinType::SveFloat16:
     case clang::BuiltinType::SveBFloat16:
+    case clang::BuiltinType::SveBFloat16x2:
+    case clang::BuiltinType::SveBFloat16x3:
+    case clang::BuiltinType::SveBFloat16x4:
     case clang::BuiltinType::SveFloat16x2:
     case clang::BuiltinType::SveFloat16x3:
     case clang::BuiltinType::SveFloat16x4:
@@ -7313,6 +7314,35 @@ clang::VarDecl *TypeSystemClang::AddVariableToRecordType(
   return var_decl;
 }
 
+void TypeSystemClang::SetIntegerInitializerForVariable(
+    VarDecl *var, const llvm::APInt &init_value) {
+  assert(!var->hasInit() && "variable already initialized");
+
+  clang::ASTContext &ast = var->getASTContext();
+  QualType qt = var->getType();
+  assert(qt->isIntegralOrEnumerationType() &&
+         "only integer or enum types supported");
+  // If the variable is an enum type, take the underlying integer type as
+  // the type of the integer literal.
+  if (const EnumType *enum_type = llvm::dyn_cast<EnumType>(qt.getTypePtr())) {
+    const EnumDecl *enum_decl = enum_type->getDecl();
+    qt = enum_decl->getIntegerType();
+  }
+  var->setInit(IntegerLiteral::Create(ast, init_value, qt.getUnqualifiedType(),
+                                      SourceLocation()));
+}
+
+void TypeSystemClang::SetFloatingInitializerForVariable(
+    clang::VarDecl *var, const llvm::APFloat &init_value) {
+  assert(!var->hasInit() && "variable already initialized");
+
+  clang::ASTContext &ast = var->getASTContext();
+  QualType qt = var->getType();
+  assert(qt->isFloatingType() && "only floating point types supported");
+  var->setInit(FloatingLiteral::Create(
+      ast, init_value, true, qt.getUnqualifiedType(), SourceLocation()));
+}
+
 clang::CXXMethodDecl *TypeSystemClang::AddMethodToCXXRecordType(
     lldb::opaque_compiler_type_t type, llvm::StringRef name,
     const char *mangled_name, const CompilerType &method_clang_type,
@@ -8214,7 +8244,8 @@ void TypeSystemClang::DumpFromSymbolFile(Stream &s,
         continue;
       }
     }
-    GetCanonicalQualType(full_type.GetOpaqueQualType()).dump(s.AsRawOstream());
+    GetCanonicalQualType(full_type.GetOpaqueQualType())
+        .dump(s.AsRawOstream(), getASTContext());
   }
 }
 
@@ -8933,7 +8964,7 @@ void TypeSystemClang::DumpTypeDescription(lldb::opaque_compiler_type_t type,
         }
       } else {
         if (level == eDescriptionLevelVerbose)
-          qual_type->dump(llvm_ostrm);
+          qual_type->dump(llvm_ostrm, getASTContext());
         else {
           std::string clang_type_name(qual_type.getAsString());
           if (!clang_type_name.empty())
@@ -9057,13 +9088,13 @@ void TypeSystemClang::CompleteObjCInterfaceDecl(
 
 DWARFASTParser *TypeSystemClang::GetDWARFParser() {
   if (!m_dwarf_ast_parser_up)
-    m_dwarf_ast_parser_up.reset(new DWARFASTParserClang(*this));
+    m_dwarf_ast_parser_up = std::make_unique<DWARFASTParserClang>(*this);
   return m_dwarf_ast_parser_up.get();
 }
 
 PDBASTParser *TypeSystemClang::GetPDBParser() {
   if (!m_pdb_ast_parser_up)
-    m_pdb_ast_parser_up.reset(new PDBASTParser(*this));
+    m_pdb_ast_parser_up = std::make_unique<PDBASTParser>(*this);
   return m_pdb_ast_parser_up.get();
 }
 
@@ -9499,8 +9530,8 @@ TypeSystemClangForExpressions::TypeSystemClangForExpressions(
     : TypeSystemClang("scratch ASTContext", triple),
       m_target_wp(target.shared_from_this()),
       m_persistent_variables(new ClangPersistentVariables) {
-  m_scratch_ast_source_up.reset(new ClangASTSource(
-      target.shared_from_this(), m_persistent_variables->GetClangASTImporter()));
+  m_scratch_ast_source_up = std::make_unique<ClangASTSource>(
+      target.shared_from_this(), m_persistent_variables->GetClangASTImporter());
   m_scratch_ast_source_up->InstallASTContext(*this);
   llvm::IntrusiveRefCntPtr<clang::ExternalASTSource> proxy_ast_source(
       m_scratch_ast_source_up->CreateProxy());
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index 920d7cb4c23dd..9475e4d9f4424 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -873,6 +873,24 @@ class TypeSystemClang : public TypeSystem {
                                                  const CompilerType &var_type,
                                                  lldb::AccessType access);
 
+  /// Initializes a variable with an integer value.
+  /// \param var The variable to initialize. Must not already have an
+  ///            initializer and must have an integer or enum type.
+  /// \param init_value The integer value that the variable should be
+  ///                   initialized to. Has to match the bit width of the
+  ///                   variable type.
+  static void SetIntegerInitializerForVariable(clang::VarDecl *var,
+                                               const llvm::APInt &init_value);
+
+  /// Initializes a variable with a floating point value.
+  /// \param var The variable to initialize. Must not already have an
+  ///            initializer and must have a floating point type.
+  /// \param init_value The float value that the variable should be
+  ///                   initialized to.
+  static void
+  SetFloatingInitializerForVariable(clang::VarDecl *var,
+                                    const llvm::APFloat &init_value);
+
   clang::CXXMethodDecl *AddMethodToCXXRecordType(
       lldb::opaque_compiler_type_t type, llvm::StringRef name,
       const char *mangled_name, const CompilerType &method_type,
diff --git a/lldb/source/Plugins/UnwindAssembly/x86/UnwindAssembly-x86.cpp b/lldb/source/Plugins/UnwindAssembly/x86/UnwindAssembly-x86.cpp
index dc62e957c1a15..fe1275d5b0cf6 100644
--- a/lldb/source/Plugins/UnwindAssembly/x86/UnwindAssembly-x86.cpp
+++ b/lldb/source/Plugins/UnwindAssembly/x86/UnwindAssembly-x86.cpp
@@ -141,7 +141,7 @@ bool UnwindAssembly_x86::AugmentUnwindPlanFromCallSite(
           // and we don't need to modify it at all.
 
           if (first_row_pc_loc.GetOffset() == -wordsize) {
-            do_augment_unwindplan = false;
+            return true;
           }
         }
       }
diff --git a/lldb/source/Symbol/Function.cpp b/lldb/source/Symbol/Function.cpp
index 533fdd465c42d..67013f6dd8b1d 100644
--- a/lldb/source/Symbol/Function.cpp
+++ b/lldb/source/Symbol/Function.cpp
@@ -290,6 +290,8 @@ void Function::GetEndLineSourceInfo(FileSpec &source_file, uint32_t &line_no) {
 }
 
 llvm::ArrayRef<std::unique_ptr<CallEdge>> Function::GetCallEdges() {
+  std::lock_guard<std::mutex> guard(m_call_edges_lock);
+
   if (m_call_edges_resolved)
     return m_call_edges;
 
diff --git a/lldb/source/Symbol/SymbolContext.cpp b/lldb/source/Symbol/SymbolContext.cpp
index 3b120fdf5e02d..12c2077154b91 100644
--- a/lldb/source/Symbol/SymbolContext.cpp
+++ b/lldb/source/Symbol/SymbolContext.cpp
@@ -968,7 +968,7 @@ bool SymbolContextSpecifier::AddSpecification(const char *spec_string,
     // CompUnits can't necessarily be resolved here, since an inlined function
     // might show up in a number of CompUnits.  Instead we just convert to a
     // FileSpec and store it away.
-    m_file_spec_up.reset(new FileSpec(spec_string));
+    m_file_spec_up = std::make_unique<FileSpec>(spec_string);
     m_type |= eFileSpecified;
     break;
   case eLineStartSpecified:
diff --git a/lldb/source/Symbol/SymbolVendor.cpp b/lldb/source/Symbol/SymbolVendor.cpp
index 403ca7a40e898..0a5acbc48eb4c 100644
--- a/lldb/source/Symbol/SymbolVendor.cpp
+++ b/lldb/source/Symbol/SymbolVendor.cpp
@@ -51,7 +51,7 @@ SymbolVendor *SymbolVendor::FindPlugin(const lldb::ModuleSP &module_sp,
   }
   if (!sym_objfile_sp)
     sym_objfile_sp = module_sp->GetObjectFile()->shared_from_this();
-  instance_up.reset(new SymbolVendor(module_sp));
+  instance_up = std::make_unique<SymbolVendor>(module_sp);
   instance_up->AddSymbolFileRepresentation(sym_objfile_sp);
   return instance_up.release();
 }
diff --git a/lldb/source/Symbol/UnwindTable.cpp b/lldb/source/Symbol/UnwindTable.cpp
index ef6e848e8259c..3faa528a339f9 100644
--- a/lldb/source/Symbol/UnwindTable.cpp
+++ b/lldb/source/Symbol/UnwindTable.cpp
@@ -57,26 +57,28 @@ void UnwindTable::Initialize() {
 
   SectionSP sect = sl->FindSectionByType(eSectionTypeEHFrame, true);
   if (sect.get()) {
-    m_eh_frame_up.reset(
-        new DWARFCallFrameInfo(*object_file, sect, DWARFCallFrameInfo::EH));
+    m_eh_frame_up = std::make_unique<DWARFCallFrameInfo>(
+        *object_file, sect, DWARFCallFrameInfo::EH);
   }
 
   sect = sl->FindSectionByType(eSectionTypeDWARFDebugFrame, true);
   if (sect) {
-    m_debug_frame_up.reset(
-        new DWARFCallFrameInfo(*object_file, sect, DWARFCallFrameInfo::DWARF));
+    m_debug_frame_up = std::make_unique<DWARFCallFrameInfo>(
+        *object_file, sect, DWARFCallFrameInfo::DWARF);
   }
 
   sect = sl->FindSectionByType(eSectionTypeCompactUnwind, true);
   if (sect) {
-    m_compact_unwind_up.reset(new CompactUnwindInfo(*object_file, sect));
+    m_compact_unwind_up =
+        std::make_unique<CompactUnwindInfo>(*object_file, sect);
   }
 
   sect = sl->FindSectionByType(eSectionTypeARMexidx, true);
   if (sect) {
     SectionSP sect_extab = sl->FindSectionByType(eSectionTypeARMextab, true);
     if (sect_extab.get()) {
-      m_arm_unwind_up.reset(new ArmUnwindInfo(*object_file, sect, sect_extab));
+      m_arm_unwind_up =
+          std::make_unique<ArmUnwindInfo>(*object_file, sect, sect_extab);
     }
   }
 }
diff --git a/lldb/source/Target/ModuleCache.cpp b/lldb/source/Target/ModuleCache.cpp
index cdd33f512c095..20661a7b7a25e 100644
--- a/lldb/source/Target/ModuleCache.cpp
+++ b/lldb/source/Target/ModuleCache.cpp
@@ -169,7 +169,7 @@ ModuleLock::ModuleLock(const FileSpec &root_dir_spec, const UUID &uuid,
     return;
   }
 
-  m_lock.reset(new lldb_private::LockFile(m_file_up->GetDescriptor()));
+  m_lock = std::make_unique<lldb_private::LockFile>(m_file_up->GetDescriptor());
   error = m_lock->WriteLock(0, 1);
   if (error.Fail())
     error.SetErrorStringWithFormat("Failed to lock file: %s",
diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp
index 9cb38556a6ec7..78f75981a94dd 100644
--- a/lldb/source/Target/Process.cpp
+++ b/lldb/source/Target/Process.cpp
@@ -164,7 +164,8 @@ ProcessProperties::ProcessProperties(lldb_private::Process *process)
         [this] { m_process->LoadOperatingSystemPlugin(true); });
   }
 
-  m_experimental_properties_up.reset(new ProcessExperimentalProperties());
+  m_experimental_properties_up =
+      std::make_unique<ProcessExperimentalProperties>();
   m_collection_sp->AppendProperty(
       ConstString(Properties::GetExperimentalSettingsName()),
       ConstString("Experimental settings - setting these won't produce "
@@ -2748,7 +2749,7 @@ DataExtractor Process::GetAuxvData() { return DataExtractor(); }
 
 JITLoaderList &Process::GetJITLoaders() {
   if (!m_jit_loaders_up) {
-    m_jit_loaders_up.reset(new JITLoaderList());
+    m_jit_loaders_up = std::make_unique<JITLoaderList>();
     JITLoader::LoadPlugins(this, *m_jit_loaders_up);
   }
   return *m_jit_loaders_up;
diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index 21d2957baf0e9..dad56376005c6 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -2458,7 +2458,7 @@ lldb::addr_t Target::GetBreakableLoadAddress(lldb::addr_t addr) {
 
 SourceManager &Target::GetSourceManager() {
   if (!m_source_manager_up)
-    m_source_manager_up.reset(new SourceManager(shared_from_this()));
+    m_source_manager_up = std::make_unique<SourceManager>(shared_from_this());
   return *m_source_manager_up;
 }
 
@@ -3137,7 +3137,7 @@ Target::StopHook::StopHook(const StopHook &rhs)
       m_thread_spec_up(), m_active(rhs.m_active),
       m_auto_continue(rhs.m_auto_continue) {
   if (rhs.m_thread_spec_up)
-    m_thread_spec_up.reset(new ThreadSpec(*rhs.m_thread_spec_up));
+    m_thread_spec_up = std::make_unique<ThreadSpec>(*rhs.m_thread_spec_up);
 }
 
 Target::StopHook::~StopHook() = default;
@@ -3427,7 +3427,8 @@ TargetProperties::TargetProperties(Target *target)
     m_collection_sp->SetValueChangedCallback(
         ePropertyDisableSTDIO, [this] { DisableSTDIOValueChangedCallback(); });
 
-    m_experimental_properties_up.reset(new TargetExperimentalProperties());
+    m_experimental_properties_up =
+        std::make_unique<TargetExperimentalProperties>();
     m_collection_sp->AppendProperty(
         ConstString(Properties::GetExperimentalSettingsName()),
         ConstString("Experimental settings - setting these won't produce "
@@ -3437,7 +3438,8 @@ TargetProperties::TargetProperties(Target *target)
     m_collection_sp =
         std::make_shared<TargetOptionValueProperties>(ConstString("target"));
     m_collection_sp->Initialize(g_target_properties);
-    m_experimental_properties_up.reset(new TargetExperimentalProperties());
+    m_experimental_properties_up =
+        std::make_unique<TargetExperimentalProperties>();
     m_collection_sp->AppendProperty(
         ConstString(Properties::GetExperimentalSettingsName()),
         ConstString("Experimental settings - setting these won't produce "
diff --git a/lldb/source/Target/Thread.cpp b/lldb/source/Target/Thread.cpp
index 35cde4015986c..24cf4bf3ee1ed 100644
--- a/lldb/source/Target/Thread.cpp
+++ b/lldb/source/Target/Thread.cpp
@@ -1066,7 +1066,7 @@ ThreadPlanStack &Thread::GetPlans() const {
   // ThreadPlanNull as its base plan.  That will give the right answers to the
   // queries GetDescription makes, and only assert if you try to run the thread.
   if (!m_null_plan_stack_up)
-    m_null_plan_stack_up.reset(new ThreadPlanStack(*this, true));
+    m_null_plan_stack_up = std::make_unique<ThreadPlanStack>(*this, true);
   return *(m_null_plan_stack_up.get());
 }
 
@@ -1859,7 +1859,7 @@ size_t Thread::GetStackFrameStatus(Stream &strm, uint32_t first_frame,
 
 Unwind &Thread::GetUnwinder() {
   if (!m_unwinder_up)
-    m_unwinder_up.reset(new UnwindLLDB(*this));
+    m_unwinder_up = std::make_unique<UnwindLLDB>(*this);
   return *m_unwinder_up;
 }
 
diff --git a/lldb/source/Target/ThreadPlanStepInRange.cpp b/lldb/source/Target/ThreadPlanStepInRange.cpp
index c9a5697e2c6de..c5f81d6665a12 100644
--- a/lldb/source/Target/ThreadPlanStepInRange.cpp
+++ b/lldb/source/Target/ThreadPlanStepInRange.cpp
@@ -311,7 +311,7 @@ void ThreadPlanStepInRange::SetAvoidRegexp(const char *name) {
   if (m_avoid_regexp_up)
     *m_avoid_regexp_up = RegularExpression(name_ref);
   else
-    m_avoid_regexp_up.reset(new RegularExpression(name_ref));
+    m_avoid_regexp_up = std::make_unique<RegularExpression>(name_ref);
 }
 
 void ThreadPlanStepInRange::SetDefaultFlagValue(uint32_t new_value) {
diff --git a/lldb/source/Utility/ARM64_DWARF_Registers.h b/lldb/source/Utility/ARM64_DWARF_Registers.h
index 1bb2ba071f956..ed8ff722088dc 100644
--- a/lldb/source/Utility/ARM64_DWARF_Registers.h
+++ b/lldb/source/Utility/ARM64_DWARF_Registers.h
@@ -51,7 +51,31 @@ enum {
   sp = x31,
   pc = 32,
   cpsr = 33,
-  // 34-63 reserved
+  // 34-45 reserved
+
+  // 64-bit SVE Vector granule pseudo register
+  vg = 46,
+
+  // VG ́8-bit SVE first fault register
+  ffr = 47,
+
+  // VG x ́8-bit SVE predicate registers
+  p0 = 48,
+  p1,
+  p2,
+  p3,
+  p4,
+  p5,
+  p6,
+  p7,
+  p8,
+  p9,
+  p10,
+  p11,
+  p12,
+  p13,
+  p14,
+  p15,
 
   // V0-V31 (128 bit vector registers)
   v0 = 64,
@@ -85,9 +109,41 @@ enum {
   v28,
   v29,
   v30,
-  v31
+  v31,
 
-  // 96-127 reserved
+  // VG ́64-bit SVE vector registers
+  z0 = 96,
+  z1,
+  z2,
+  z3,
+  z4,
+  z5,
+  z6,
+  z7,
+  z8,
+  z9,
+  z10,
+  z11,
+  z12,
+  z13,
+  z14,
+  z15,
+  z16,
+  z17,
+  z18,
+  z19,
+  z20,
+  z21,
+  z22,
+  z23,
+  z24,
+  z25,
+  z26,
+  z27,
+  z28,
+  z29,
+  z30,
+  z31
 };
 
 } // namespace arm64_dwarf
diff --git a/lldb/source/Utility/ARM64_ehframe_Registers.h b/lldb/source/Utility/ARM64_ehframe_Registers.h
index 3e7baf98e1480..c235891ec015e 100644
--- a/lldb/source/Utility/ARM64_ehframe_Registers.h
+++ b/lldb/source/Utility/ARM64_ehframe_Registers.h
@@ -49,10 +49,34 @@ enum {
   lr, // aka x30
   sp, // aka x31 aka wzr
   pc, // value is 32
-  cpsr
-};
+  cpsr,
+  // 34-45 reserved
 
-enum {
+  // 64-bit SVE Vector granule pseudo register
+  vg = 46,
+
+  // VG ́8-bit SVE first fault register
+  ffr = 47,
+
+  // VG x ́8-bit SVE predicate registers
+  p0 = 48,
+  p1,
+  p2,
+  p3,
+  p4,
+  p5,
+  p6,
+  p7,
+  p8,
+  p9,
+  p10,
+  p11,
+  p12,
+  p13,
+  p14,
+  p15,
+
+  // V0-V31 (128 bit vector registers)
   v0 = 64,
   v1,
   v2,
@@ -84,7 +108,41 @@ enum {
   v28,
   v29,
   v30,
-  v31 // 95
+  v31,
+
+  // VG ́64-bit SVE vector registers
+  z0 = 96,
+  z1,
+  z2,
+  z3,
+  z4,
+  z5,
+  z6,
+  z7,
+  z8,
+  z9,
+  z10,
+  z11,
+  z12,
+  z13,
+  z14,
+  z15,
+  z16,
+  z17,
+  z18,
+  z19,
+  z20,
+  z21,
+  z22,
+  z23,
+  z24,
+  z25,
+  z26,
+  z27,
+  z28,
+  z29,
+  z30,
+  z31
 };
 }
 
diff --git a/lldb/source/Utility/DataExtractor.cpp b/lldb/source/Utility/DataExtractor.cpp
index ac3662a8e3c83..64b878d7dee74 100644
--- a/lldb/source/Utility/DataExtractor.cpp
+++ b/lldb/source/Utility/DataExtractor.cpp
@@ -15,7 +15,6 @@
 
 #include "lldb/Utility/DataBuffer.h"
 #include "lldb/Utility/DataBufferHeap.h"
-#include "lldb/Utility/Endian.h"
 #include "lldb/Utility/LLDBAssert.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Stream.h"
@@ -624,41 +623,11 @@ int64_t DataExtractor::GetMaxS64Bitfield(offset_t *offset_ptr, size_t size,
 }
 
 float DataExtractor::GetFloat(offset_t *offset_ptr) const {
-  typedef float float_type;
-  float_type val = 0.0;
-  const size_t src_size = sizeof(float_type);
-  const float_type *src =
-      static_cast<const float_type *>(GetData(offset_ptr, src_size));
-  if (src) {
-    if (m_byte_order != endian::InlHostByteOrder()) {
-      const uint8_t *src_data = reinterpret_cast<const uint8_t *>(src);
-      uint8_t *dst_data = reinterpret_cast<uint8_t *>(&val);
-      for (size_t i = 0; i < sizeof(float_type); ++i)
-        dst_data[sizeof(float_type) - 1 - i] = src_data[i];
-    } else {
-      val = *src;
-    }
-  }
-  return val;
+  return Get<float>(offset_ptr, 0.0f);
 }
 
 double DataExtractor::GetDouble(offset_t *offset_ptr) const {
-  typedef double float_type;
-  float_type val = 0.0;
-  const size_t src_size = sizeof(float_type);
-  const float_type *src =
-      static_cast<const float_type *>(GetData(offset_ptr, src_size));
-  if (src) {
-    if (m_byte_order != endian::InlHostByteOrder()) {
-      const uint8_t *src_data = reinterpret_cast<const uint8_t *>(src);
-      uint8_t *dst_data = reinterpret_cast<uint8_t *>(&val);
-      for (size_t i = 0; i < sizeof(float_type); ++i)
-        dst_data[sizeof(float_type) - 1 - i] = src_data[i];
-    } else {
-      val = *src;
-    }
-  }
-  return val;
+  return Get<double>(offset_ptr, 0.0);
 }
 
 long double DataExtractor::GetLongDouble(offset_t *offset_ptr) const {
diff --git a/lldb/source/Utility/RegisterValue.cpp b/lldb/source/Utility/RegisterValue.cpp
index 91f4025c923cd..7f545c214a4c6 100644
--- a/lldb/source/Utility/RegisterValue.cpp
+++ b/lldb/source/Utility/RegisterValue.cpp
@@ -728,7 +728,8 @@ const void *RegisterValue::GetBytes() const {
   case eTypeFloat:
   case eTypeDouble:
   case eTypeLongDouble:
-    return m_scalar.GetBytes();
+    m_scalar.GetBytes(buffer.bytes);
+    return buffer.bytes;
   case eTypeBytes:
     return buffer.bytes;
   }
diff --git a/lldb/source/Utility/Reproducer.cpp b/lldb/source/Utility/Reproducer.cpp
index eb7d96baab572..7620ab2c389d6 100644
--- a/lldb/source/Utility/Reproducer.cpp
+++ b/lldb/source/Utility/Reproducer.cpp
@@ -272,40 +272,15 @@ DataRecorder::Create(const FileSpec &filename) {
   return std::move(recorder);
 }
 
-DataRecorder *CommandProvider::GetNewDataRecorder() {
-  std::size_t i = m_data_recorders.size() + 1;
-  std::string filename = (llvm::Twine(Info::name) + llvm::Twine("-") +
-                          llvm::Twine(i) + llvm::Twine(".yaml"))
-                             .str();
-  auto recorder_or_error =
-      DataRecorder::Create(GetRoot().CopyByAppendingPathComponent(filename));
-  if (!recorder_or_error) {
-    llvm::consumeError(recorder_or_error.takeError());
-    return nullptr;
-  }
-
-  m_data_recorders.push_back(std::move(*recorder_or_error));
-  return m_data_recorders.back().get();
-}
-
-void CommandProvider::Keep() {
-  std::vector<std::string> files;
-  for (auto &recorder : m_data_recorders) {
-    recorder->Stop();
-    files.push_back(recorder->GetFilename().GetPath());
-  }
-
-  FileSpec file = GetRoot().CopyByAppendingPathComponent(Info::file);
+llvm::Expected<std::unique_ptr<YamlRecorder>>
+YamlRecorder::Create(const FileSpec &filename) {
   std::error_code ec;
-  llvm::raw_fd_ostream os(file.GetPath(), ec, llvm::sys::fs::OF_Text);
+  auto recorder = std::make_unique<YamlRecorder>(std::move(filename), ec);
   if (ec)
-    return;
-  yaml::Output yout(os);
-  yout << files;
+    return llvm::errorCodeToError(ec);
+  return std::move(recorder);
 }
 
-void CommandProvider::Discard() { m_data_recorders.clear(); }
-
 void VersionProvider::Keep() {
   FileSpec file = GetRoot().CopyByAppendingPathComponent(Info::file);
   std::error_code ec;
diff --git a/lldb/source/Utility/Scalar.cpp b/lldb/source/Utility/Scalar.cpp
index 759e2a7b56fc5..87ac6c23892d8 100644
--- a/lldb/source/Utility/Scalar.cpp
+++ b/lldb/source/Utility/Scalar.cpp
@@ -7,14 +7,14 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Utility/Scalar.h"
-
+#include "lldb/Utility/DataBufferHeap.h"
 #include "lldb/Utility/DataExtractor.h"
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/lldb-types.h"
-
+#include "llvm/ADT/APSInt.h"
 #include "llvm/ADT/SmallString.h"
 
 #include <cinttypes>
@@ -23,6 +23,62 @@
 using namespace lldb;
 using namespace lldb_private;
 
+using llvm::APFloat;
+
+namespace {
+enum class Category { Void, Integral, Float };
+}
+
+static Category GetCategory(Scalar::Type type) {
+  switch (type) {
+  case Scalar::e_void:
+    return Category::Void;
+  case Scalar::e_float:
+  case Scalar::e_double:
+  case Scalar::e_long_double:
+    return Category::Float;
+  case Scalar::e_sint:
+  case Scalar::e_slong:
+  case Scalar::e_slonglong:
+  case Scalar::e_sint128:
+  case Scalar::e_sint256:
+  case Scalar::e_sint512:
+  case Scalar::e_uint:
+  case Scalar::e_ulong:
+  case Scalar::e_ulonglong:
+  case Scalar::e_uint128:
+  case Scalar::e_uint256:
+  case Scalar::e_uint512:
+    return Category::Integral;
+  }
+  llvm_unreachable("Unhandled type!");
+}
+
+static bool IsSigned(Scalar::Type type) {
+  switch (type) {
+  case Scalar::e_void:
+  case Scalar::e_uint:
+  case Scalar::e_ulong:
+  case Scalar::e_ulonglong:
+  case Scalar::e_uint128:
+  case Scalar::e_uint256:
+  case Scalar::e_uint512:
+    return false;
+  case Scalar::e_sint:
+  case Scalar::e_slong:
+  case Scalar::e_slonglong:
+  case Scalar::e_sint128:
+  case Scalar::e_sint256:
+  case Scalar::e_sint512:
+  case Scalar::e_float:
+  case Scalar::e_double:
+  case Scalar::e_long_double:
+    return true;
+  }
+  llvm_unreachable("Unhandled type!");
+}
+
+
 // Promote to max type currently follows the ANSI C rule for type promotion in
 // expressions.
 static Scalar::Type PromoteToMaxType(
@@ -73,112 +129,49 @@ Scalar::Scalar() : m_type(e_void), m_float(static_cast<float>(0)) {}
 
 bool Scalar::GetData(DataExtractor &data, size_t limit_byte_size) const {
   size_t byte_size = GetByteSize();
-  if (byte_size > 0) {
-    const uint8_t *bytes = static_cast<const uint8_t *>(GetBytes());
-
-    if (limit_byte_size < byte_size) {
-      if (endian::InlHostByteOrder() == eByteOrderLittle) {
-        // On little endian systems if we want fewer bytes from the current
-        // type we just specify fewer bytes since the LSByte is first...
-        byte_size = limit_byte_size;
-      } else if (endian::InlHostByteOrder() == eByteOrderBig) {
-        // On big endian systems if we want fewer bytes from the current type
-        // have to advance our initial byte pointer and trim down the number of
-        // bytes since the MSByte is first
-        bytes += byte_size - limit_byte_size;
-        byte_size = limit_byte_size;
-      }
+  if (byte_size == 0) {
+    data.Clear();
+    return false;
+  }
+  auto buffer_up = std::make_unique<DataBufferHeap>(byte_size, 0);
+  GetBytes(buffer_up->GetData());
+  lldb::offset_t offset = 0;
+
+  if (limit_byte_size < byte_size) {
+    if (endian::InlHostByteOrder() == eByteOrderLittle) {
+      // On little endian systems if we want fewer bytes from the current
+      // type we just specify fewer bytes since the LSByte is first...
+      byte_size = limit_byte_size;
+    } else if (endian::InlHostByteOrder() == eByteOrderBig) {
+      // On big endian systems if we want fewer bytes from the current type
+      // have to advance our initial byte pointer and trim down the number of
+      // bytes since the MSByte is first
+      offset = byte_size - limit_byte_size;
+      byte_size = limit_byte_size;
     }
-
-    data.SetData(bytes, byte_size, endian::InlHostByteOrder());
-    return true;
   }
-  data.Clear();
-  return false;
+
+  data.SetData(std::move(buffer_up), offset, byte_size);
+  data.SetByteOrder(endian::InlHostByteOrder());
+  return true;
 }
 
-const void *Scalar::GetBytes() const {
-  const uint64_t *apint_words;
-  const uint8_t *bytes;
-  static float_t flt_val;
-  static double_t dbl_val;
-  static uint64_t swapped_words[8];
-  switch (m_type) {
-  case e_void:
+void Scalar::GetBytes(llvm::MutableArrayRef<uint8_t> storage) const {
+  assert(storage.size() >= GetByteSize());
+
+  const auto &store = [&](const llvm::APInt val) {
+    StoreIntToMemory(val, storage.data(), (val.getBitWidth() + 7) / 8);
+  };
+  switch (GetCategory(m_type)) {
+  case Category::Void:
+    break;
+  case Category::Integral:
+    store(m_integer);
+    break;
+  case Category::Float:
+    store(m_float.bitcastToAPInt());
     break;
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-    bytes = reinterpret_cast<const uint8_t *>(m_integer.getRawData());
-    // getRawData always returns a pointer to an uint64_t.  If we have a
-    // smaller type, we need to update the pointer on big-endian systems.
-    if (endian::InlHostByteOrder() == eByteOrderBig) {
-      size_t byte_size = m_integer.getBitWidth() / 8;
-      if (byte_size < 8)
-        bytes += 8 - byte_size;
-    }
-    return bytes;
-  // getRawData always returns a pointer to an array of uint64_t values,
-  // where the least-significant word always comes first.  On big-endian
-  // systems we need to swap the words.
-  case e_sint128:
-  case e_uint128:
-    apint_words = m_integer.getRawData();
-    if (endian::InlHostByteOrder() == eByteOrderBig) {
-      swapped_words[0] = apint_words[1];
-      swapped_words[1] = apint_words[0];
-      apint_words = swapped_words;
-    }
-    return static_cast<const void *>(apint_words);
-  case e_sint256:
-  case e_uint256:
-    apint_words = m_integer.getRawData();
-    if (endian::InlHostByteOrder() == eByteOrderBig) {
-      swapped_words[0] = apint_words[3];
-      swapped_words[1] = apint_words[2];
-      swapped_words[2] = apint_words[1];
-      swapped_words[3] = apint_words[0];
-      apint_words = swapped_words;
-    }
-    return static_cast<const void *>(apint_words);
-  case e_sint512:
-  case e_uint512:
-    apint_words = m_integer.getRawData();
-    if (endian::InlHostByteOrder() == eByteOrderBig) {
-      swapped_words[0] = apint_words[7];
-      swapped_words[1] = apint_words[6];
-      swapped_words[2] = apint_words[5];
-      swapped_words[3] = apint_words[4];
-      swapped_words[4] = apint_words[3];
-      swapped_words[5] = apint_words[2];
-      swapped_words[6] = apint_words[1];
-      swapped_words[7] = apint_words[0];
-      apint_words = swapped_words;
-    }
-    return static_cast<const void *>(apint_words);
-  case e_float:
-    flt_val = m_float.convertToFloat();
-    return static_cast<const void *>(&flt_val);
-  case e_double:
-    dbl_val = m_float.convertToDouble();
-    return static_cast<const void *>(&dbl_val);
-  case e_long_double:
-    llvm::APInt ldbl_val = m_float.bitcastToAPInt();
-    apint_words = ldbl_val.getRawData();
-    // getRawData always returns a pointer to an array of two uint64_t values,
-    // where the least-significant word always comes first.  On big-endian
-    // systems we need to swap the two words.
-    if (endian::InlHostByteOrder() == eByteOrderBig) {
-      swapped_words[0] = apint_words[1];
-      swapped_words[1] = apint_words[0];
-      apint_words = swapped_words;
-    }
-    return static_cast<const void *>(apint_words);
   }
-  return nullptr;
 }
 
 size_t Scalar::GetByteSize() const {
@@ -209,26 +202,12 @@ size_t Scalar::GetByteSize() const {
 }
 
 bool Scalar::IsZero() const {
-  llvm::APInt zero_int = llvm::APInt::getNullValue(m_integer.getBitWidth() / 8);
-  switch (m_type) {
-  case e_void:
+  switch (GetCategory(m_type)) {
+  case Category::Void:
     break;
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_uint512:
-  case e_sint512:
-    return llvm::APInt::isSameValue(zero_int, m_integer);
-  case e_float:
-  case e_double:
-  case e_long_double:
+  case Category::Integral:
+    return m_integer.isNullValue();
+  case Category::Float:
     return m_float.isZero();
   }
   return false;
@@ -238,73 +217,20 @@ void Scalar::GetValue(Stream *s, bool show_type) const {
   if (show_type)
     s->Printf("(%s) ", GetTypeAsCString());
 
-  switch (m_type) {
-  case e_void:
-    break;
-  case e_sint:
-  case e_slong:
-  case e_slonglong:
-  case e_sint128:
-  case e_sint256:
-  case e_sint512:
-    s->PutCString(m_integer.toString(10, true));
+  switch (GetCategory(m_type)) {
+  case Category::Void:
     break;
-  case e_uint:
-  case e_ulong:
-  case e_ulonglong:
-  case e_uint128:
-  case e_uint256:
-  case e_uint512:
-    s->PutCString(m_integer.toString(10, false));
+  case Category::Integral:
+    s->PutCString(m_integer.toString(10, IsSigned(m_type)));
     break;
-  case e_float:
-  case e_double:
-  case e_long_double:
+  case Category::Float:
     llvm::SmallString<24> string;
     m_float.toString(string);
-    s->Printf("%s", string.c_str());
+    s->PutCString(string);
     break;
   }
 }
 
-const char *Scalar::GetTypeAsCString() const {
-  switch (m_type) {
-  case e_void:
-    return "void";
-  case e_sint:
-    return "int";
-  case e_uint:
-    return "unsigned int";
-  case e_slong:
-    return "long";
-  case e_ulong:
-    return "unsigned long";
-  case e_slonglong:
-    return "long long";
-  case e_ulonglong:
-    return "unsigned long long";
-  case e_sint128:
-    return "int128_t";
-  case e_uint128:
-    return "unsigned int128_t";
-  case e_sint256:
-    return "int256_t";
-  case e_uint256:
-    return "unsigned int256_t";
-  case e_sint512:
-    return "int512_t";
-  case e_uint512:
-    return "unsigned int512_t";
-  case e_float:
-    return "float";
-  case e_double:
-    return "double";
-  case e_long_double:
-    return "long double";
-  }
-  return "<invalid Scalar type>";
-}
-
 Scalar::~Scalar() = default;
 
 Scalar::Type Scalar::GetBestTypeForBitSize(size_t bit_size, bool sign) {
@@ -327,1473 +253,478 @@ Scalar::Type Scalar::GetBestTypeForBitSize(size_t bit_size, bool sign) {
   return Scalar::e_void;
 }
 
-void Scalar::TruncOrExtendTo(Scalar::Type type, uint16_t bits) {
+void Scalar::TruncOrExtendTo(uint16_t bits, bool sign) {
+  m_integer = sign ? m_integer.sextOrTrunc(bits) : m_integer.zextOrTrunc(bits);
+  m_type = GetBestTypeForBitSize(bits, sign);
+}
+
+static size_t GetBitSize(Scalar::Type type) {
   switch (type) {
-  case e_sint:
-  case e_slong:
-  case e_slonglong:
-  case e_sint128:
-  case e_sint256:
-  case e_sint512:
-    m_integer = m_integer.sextOrTrunc(bits);
-    break;
-  case e_uint:
-  case e_ulong:
-  case e_ulonglong:
-  case e_uint128:
-  case e_uint256:
-  case e_uint512:
-    m_integer = m_integer.zextOrTrunc(bits);
-    break;
-  default:
-    llvm_unreachable("Promoting a Scalar to a specific number of bits is only "
-                     "supported for integer types.");
+  case Scalar::e_void:
+    return 0;
+  case Scalar::e_sint:
+    return 8 * sizeof(int);
+  case Scalar::e_uint:
+    return 8 * sizeof(unsigned int);
+  case Scalar::e_slong:
+    return 8 * sizeof(long);
+  case Scalar::e_ulong:
+    return 8 * sizeof(unsigned long);
+  case Scalar::e_slonglong:
+    return 8 * sizeof(long long);
+  case Scalar::e_ulonglong:
+    return 8 * sizeof(unsigned long long);
+  case Scalar::e_sint128:
+  case Scalar::e_uint128:
+    return BITWIDTH_INT128;
+  case Scalar::e_sint256:
+  case Scalar::e_uint256:
+    return BITWIDTH_INT256;
+  case Scalar::e_sint512:
+  case Scalar::e_uint512:
+    return BITWIDTH_INT512;
+  case Scalar::e_float:
+    return 8 * sizeof(float);
+  case Scalar::e_double:
+    return 8 * sizeof(double);
+  case Scalar::e_long_double:
+    return 8 * sizeof(long double);
+  }
+  llvm_unreachable("Unhandled type!");
+}
+
+static const llvm::fltSemantics &GetFltSemantics(Scalar::Type type) {
+  switch (type) {
+  case Scalar::e_void:
+  case Scalar::e_sint:
+  case Scalar::e_slong:
+  case Scalar::e_slonglong:
+  case Scalar::e_sint128:
+  case Scalar::e_sint256:
+  case Scalar::e_sint512:
+  case Scalar::e_uint:
+  case Scalar::e_ulong:
+  case Scalar::e_ulonglong:
+  case Scalar::e_uint128:
+  case Scalar::e_uint256:
+  case Scalar::e_uint512:
+    llvm_unreachable("Only floating point types supported!");
+  case Scalar::e_float:
+    return llvm::APFloat::IEEEsingle();
+  case Scalar::e_double:
+    return llvm::APFloat::IEEEdouble();
+  case Scalar::e_long_double:
+    return llvm::APFloat::x87DoubleExtended();
   }
-  m_type = type;
+  llvm_unreachable("Unhandled type!");
 }
 
 bool Scalar::Promote(Scalar::Type type) {
   bool success = false;
-  switch (m_type) {
-  case e_void:
+  switch (GetCategory(m_type)) {
+  case Category::Void:
     break;
-
-  case e_sint:
-    switch (type) {
-    case e_void:
-      break;
-    case e_sint:
-      success = true;
-      break;
-    case e_uint:
-      m_integer = m_integer.sextOrTrunc(sizeof(uint_t) * 8);
-      success = true;
-      break;
-
-    case e_slong:
-      m_integer = m_integer.sextOrTrunc(sizeof(slong_t) * 8);
-      success = true;
-      break;
-
-    case e_ulong:
-      m_integer = m_integer.sextOrTrunc(sizeof(ulong_t) * 8);
-      success = true;
-      break;
-
-    case e_slonglong:
-      m_integer = m_integer.sextOrTrunc(sizeof(slonglong_t) * 8);
-      success = true;
+  case Category::Integral:
+    switch (GetCategory(type)) {
+    case Category::Void:
       break;
-
-    case e_ulonglong:
-      m_integer = m_integer.sextOrTrunc(sizeof(ulonglong_t) * 8);
-      success = true;
-      break;
-
-    case e_sint128:
-    case e_uint128:
-      m_integer = m_integer.sextOrTrunc(BITWIDTH_INT128);
-      success = true;
-      break;
-
-    case e_sint256:
-    case e_uint256:
-      m_integer = m_integer.sextOrTrunc(BITWIDTH_INT256);
-      success = true;
-      break;
-
-    case e_sint512:
-    case e_uint512:
-      m_integer = m_integer.sextOrTrunc(BITWIDTH_INT512);
+    case Category::Integral:
+      if (type < m_type)
+        break;
       success = true;
+      if (IsSigned(m_type))
+        m_integer = m_integer.sextOrTrunc(GetBitSize(type));
+      else
+        m_integer = m_integer.zextOrTrunc(GetBitSize(type));
       break;
-
-    case e_float:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEsingle());
-      m_float.convertFromAPInt(m_integer, true,
+    case Category::Float:
+      m_float = llvm::APFloat(GetFltSemantics(type));
+      m_float.convertFromAPInt(m_integer, IsSigned(m_type),
                                llvm::APFloat::rmNearestTiesToEven);
       success = true;
       break;
-
-    case e_double:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEdouble());
-      m_float.convertFromAPInt(m_integer, true,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
+    }
+    break;
+  case Category::Float:
+    switch (GetCategory(type)) {
+    case Category::Void:
+    case Category::Integral:
       break;
-
-    case e_long_double:
-      m_float = llvm::APFloat(llvm::APFloat::x87DoubleExtended());
-      m_float.convertFromAPInt(m_integer, true,
-                               llvm::APFloat::rmNearestTiesToEven);
+    case Category::Float:
+      if (type < m_type)
+        break;
+      bool ignore;
       success = true;
-      break;
+      m_float.convert(GetFltSemantics(type), llvm::APFloat::rmNearestTiesToEven,
+                      &ignore);
     }
-    break;
+  }
 
-  case e_uint:
-    switch (type) {
-    case e_void:
-    case e_sint:
-      break;
-    case e_uint:
-      success = true;
-      break;
-    case e_slong:
-      m_integer = m_integer.zextOrTrunc(sizeof(slong_t) * 8);
-      success = true;
-      break;
-
-    case e_ulong:
-      m_integer = m_integer.zextOrTrunc(sizeof(ulong_t) * 8);
-      success = true;
-      break;
-
-    case e_slonglong:
-      m_integer = m_integer.zextOrTrunc(sizeof(slonglong_t) * 8);
-      success = true;
-      break;
-
-    case e_ulonglong:
-      m_integer = m_integer.zextOrTrunc(sizeof(ulonglong_t) * 8);
-      success = true;
-      break;
-
-    case e_sint128:
-    case e_uint128:
-      m_integer = m_integer.zextOrTrunc(BITWIDTH_INT128);
-      success = true;
-      break;
-
-    case e_sint256:
-    case e_uint256:
-      m_integer = m_integer.zextOrTrunc(BITWIDTH_INT256);
-      success = true;
-      break;
-
-    case e_sint512:
-    case e_uint512:
-      m_integer = m_integer.zextOrTrunc(BITWIDTH_INT512);
-      success = true;
-      break;
-
-    case e_float:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEsingle());
-      m_float.convertFromAPInt(m_integer, false,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_double:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEdouble());
-      m_float.convertFromAPInt(m_integer, false,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_long_double:
-      m_float = llvm::APFloat(llvm::APFloat::x87DoubleExtended());
-      m_float.convertFromAPInt(m_integer, false,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-    }
-    break;
-
-  case e_slong:
-    switch (type) {
-    case e_void:
-    case e_sint:
-    case e_uint:
-      break;
-    case e_slong:
-      success = true;
-      break;
-    case e_ulong:
-      m_integer = m_integer.sextOrTrunc(sizeof(ulong_t) * 8);
-      success = true;
-      break;
-
-    case e_slonglong:
-      m_integer = m_integer.sextOrTrunc(sizeof(slonglong_t) * 8);
-      success = true;
-      break;
-
-    case e_ulonglong:
-      m_integer = m_integer.sextOrTrunc(sizeof(ulonglong_t) * 8);
-      success = true;
-      break;
-
-    case e_sint128:
-    case e_uint128:
-      m_integer = m_integer.sextOrTrunc(BITWIDTH_INT128);
-      success = true;
-      break;
-
-    case e_sint256:
-    case e_uint256:
-      m_integer = m_integer.sextOrTrunc(BITWIDTH_INT256);
-      success = true;
-      break;
-
-    case e_sint512:
-    case e_uint512:
-      m_integer = m_integer.sextOrTrunc(BITWIDTH_INT512);
-      success = true;
-      break;
-
-    case e_float:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEsingle());
-      m_float.convertFromAPInt(m_integer, true,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_double:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEdouble());
-      m_float.convertFromAPInt(m_integer, true,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_long_double:
-      m_float = llvm::APFloat(llvm::APFloat::x87DoubleExtended());
-      m_float.convertFromAPInt(m_integer, true,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-    }
-    break;
-
-  case e_ulong:
-    switch (type) {
-    case e_void:
-    case e_sint:
-    case e_uint:
-    case e_slong:
-      break;
-    case e_ulong:
-      success = true;
-      break;
-    case e_slonglong:
-      m_integer = m_integer.zextOrTrunc(sizeof(slonglong_t) * 8);
-      success = true;
-      break;
-
-    case e_ulonglong:
-      m_integer = m_integer.zextOrTrunc(sizeof(ulonglong_t) * 8);
-      success = true;
-      break;
-
-    case e_sint128:
-    case e_uint128:
-      m_integer = m_integer.zextOrTrunc(BITWIDTH_INT128);
-      success = true;
-      break;
-
-    case e_sint256:
-    case e_uint256:
-      m_integer = m_integer.zextOrTrunc(BITWIDTH_INT256);
-      success = true;
-      break;
-
-    case e_sint512:
-    case e_uint512:
-      m_integer = m_integer.zextOrTrunc(BITWIDTH_INT512);
-      success = true;
-      break;
-
-    case e_float:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEsingle());
-      m_float.convertFromAPInt(m_integer, false,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_double:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEdouble());
-      m_float.convertFromAPInt(m_integer, false,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_long_double:
-      m_float = llvm::APFloat(llvm::APFloat::x87DoubleExtended());
-      m_float.convertFromAPInt(m_integer, false,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-    }
-    break;
-
-  case e_slonglong:
-    switch (type) {
-    case e_void:
-    case e_sint:
-    case e_uint:
-    case e_slong:
-    case e_ulong:
-      break;
-    case e_slonglong:
-      success = true;
-      break;
-    case e_ulonglong:
-      m_integer = m_integer.sextOrTrunc(sizeof(ulonglong_t) * 8);
-      success = true;
-      break;
-
-    case e_sint128:
-    case e_uint128:
-      m_integer = m_integer.sextOrTrunc(BITWIDTH_INT128);
-      success = true;
-      break;
-
-    case e_sint256:
-    case e_uint256:
-      m_integer = m_integer.sextOrTrunc(BITWIDTH_INT256);
-      success = true;
-      break;
-
-    case e_sint512:
-    case e_uint512:
-      m_integer = m_integer.sextOrTrunc(BITWIDTH_INT512);
-      success = true;
-      break;
-
-    case e_float:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEsingle());
-      m_float.convertFromAPInt(m_integer, true,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_double:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEdouble());
-      m_float.convertFromAPInt(m_integer, true,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_long_double:
-      m_float = llvm::APFloat(llvm::APFloat::x87DoubleExtended());
-      m_float.convertFromAPInt(m_integer, true,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-    }
-    break;
-
-  case e_ulonglong:
-    switch (type) {
-    case e_void:
-    case e_sint:
-    case e_uint:
-    case e_slong:
-    case e_ulong:
-    case e_slonglong:
-      break;
-    case e_ulonglong:
-      success = true;
-      break;
-    case e_sint128:
-    case e_uint128:
-      m_integer = m_integer.zextOrTrunc(BITWIDTH_INT128);
-      success = true;
-      break;
-
-    case e_sint256:
-    case e_uint256:
-      m_integer = m_integer.zextOrTrunc(BITWIDTH_INT256);
-      success = true;
-      break;
-
-    case e_sint512:
-    case e_uint512:
-      m_integer = m_integer.zextOrTrunc(BITWIDTH_INT512);
-      success = true;
-      break;
-
-    case e_float:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEsingle());
-      m_float.convertFromAPInt(m_integer, false,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_double:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEdouble());
-      m_float.convertFromAPInt(m_integer, false,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_long_double:
-      m_float = llvm::APFloat(llvm::APFloat::x87DoubleExtended());
-      m_float.convertFromAPInt(m_integer, false,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-    }
-    break;
-
-  case e_sint128:
-    switch (type) {
-    case e_void:
-    case e_sint:
-    case e_uint:
-    case e_slong:
-    case e_ulong:
-    case e_slonglong:
-    case e_ulonglong:
-      break;
-    case e_sint128:
-      success = true;
-      break;
-    case e_uint128:
-      m_integer = m_integer.sextOrTrunc(BITWIDTH_INT128);
-      success = true;
-      break;
-
-    case e_sint256:
-    case e_uint256:
-      m_integer = m_integer.sextOrTrunc(BITWIDTH_INT256);
-      success = true;
-      break;
-
-    case e_sint512:
-    case e_uint512:
-      m_integer = m_integer.sextOrTrunc(BITWIDTH_INT512);
-      success = true;
-      break;
-
-    case e_float:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEsingle());
-      m_float.convertFromAPInt(m_integer, true,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_double:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEdouble());
-      m_float.convertFromAPInt(m_integer, true,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_long_double:
-      m_float = llvm::APFloat(llvm::APFloat::x87DoubleExtended());
-      m_float.convertFromAPInt(m_integer, true,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-    }
-    break;
-
-  case e_uint128:
-    switch (type) {
-    case e_void:
-    case e_sint:
-    case e_uint:
-    case e_slong:
-    case e_ulong:
-    case e_slonglong:
-    case e_ulonglong:
-    case e_sint128:
-      break;
-    case e_uint128:
-      success = true;
-      break;
-    case e_sint256:
-    case e_uint256:
-      m_integer = m_integer.zextOrTrunc(BITWIDTH_INT256);
-      success = true;
-      break;
-
-    case e_sint512:
-    case e_uint512:
-      m_integer = m_integer.zextOrTrunc(BITWIDTH_INT512);
-      success = true;
-      break;
-
-    case e_float:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEsingle());
-      m_float.convertFromAPInt(m_integer, false,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_double:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEdouble());
-      m_float.convertFromAPInt(m_integer, false,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_long_double:
-      m_float = llvm::APFloat(llvm::APFloat::x87DoubleExtended());
-      m_float.convertFromAPInt(m_integer, false,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-    }
-    break;
-
-  case e_sint256:
-    switch (type) {
-    case e_void:
-    case e_sint:
-    case e_uint:
-    case e_slong:
-    case e_ulong:
-    case e_slonglong:
-    case e_ulonglong:
-    case e_sint128:
-    case e_uint128:
-      break;
-    case e_sint256:
-      success = true;
-      break;
-    case e_uint256:
-      m_integer = m_integer.sextOrTrunc(BITWIDTH_INT256);
-      success = true;
-      break;
-
-    case e_sint512:
-    case e_uint512:
-      m_integer = m_integer.zextOrTrunc(BITWIDTH_INT512);
-      success = true;
-      break;
-
-    case e_float:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEsingle());
-      m_float.convertFromAPInt(m_integer, true,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_double:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEdouble());
-      m_float.convertFromAPInt(m_integer, true,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_long_double:
-      m_float = llvm::APFloat(llvm::APFloat::x87DoubleExtended());
-      m_float.convertFromAPInt(m_integer, true,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-    }
-    break;
-
-  case e_uint256:
-    switch (type) {
-    case e_void:
-    case e_sint:
-    case e_uint:
-    case e_slong:
-    case e_ulong:
-    case e_slonglong:
-    case e_ulonglong:
-    case e_sint128:
-    case e_uint128:
-    case e_sint256:
-      break;
-    case e_uint256:
-      success = true;
-      break;
-
-    case e_sint512:
-    case e_uint512:
-      m_integer = m_integer.zextOrTrunc(BITWIDTH_INT512);
-      success = true;
-      break;
-
-    case e_float:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEsingle());
-      m_float.convertFromAPInt(m_integer, false,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_double:
-      m_float = llvm::APFloat(llvm::APFloat::IEEEdouble());
-      m_float.convertFromAPInt(m_integer, false,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-
-    case e_long_double:
-      m_float = llvm::APFloat(llvm::APFloat::x87DoubleExtended());
-      m_float.convertFromAPInt(m_integer, false,
-                               llvm::APFloat::rmNearestTiesToEven);
-      success = true;
-      break;
-    }
-    break;
-
-  case e_sint512:
-  case e_uint512:
-    lldbassert(false && "unimplemented");
-    break;
-
-  case e_float:
-    switch (type) {
-    case e_void:
-    case e_sint:
-    case e_uint:
-    case e_slong:
-    case e_ulong:
-    case e_slonglong:
-    case e_ulonglong:
-    case e_sint128:
-    case e_uint128:
-    case e_sint256:
-    case e_uint256:
-    case e_uint512:
-    case e_sint512:
-      break;
-    case e_float:
-      success = true;
-      break;
-    case e_double:
-      m_float = llvm::APFloat(static_cast<double_t>(m_float.convertToFloat()));
-      success = true;
-      break;
-
-    case e_long_double: {
-      bool ignore;
-      m_float.convert(llvm::APFloat::x87DoubleExtended(),
-                      llvm::APFloat::rmNearestTiesToEven, &ignore);
-      success = true;
-      break;
-    }
-    }
-    break;
-
-  case e_double:
-    switch (type) {
-    case e_void:
-    case e_sint:
-    case e_uint:
-    case e_slong:
-    case e_ulong:
-    case e_slonglong:
-    case e_ulonglong:
-    case e_sint128:
-    case e_uint128:
-    case e_sint256:
-    case e_uint256:
-    case e_sint512:
-    case e_uint512:
-    case e_float:
-      break;
-    case e_double:
-      success = true;
-      break;
-    case e_long_double: {
-      bool ignore;
-      m_float.convert(llvm::APFloat::x87DoubleExtended(),
-                      llvm::APFloat::rmNearestTiesToEven, &ignore);
-      success = true;
-      break;
-    }
-    }
-    break;
-
-  case e_long_double:
-    switch (type) {
-    case e_void:
-    case e_sint:
-    case e_uint:
-    case e_slong:
-    case e_ulong:
-    case e_slonglong:
-    case e_ulonglong:
-    case e_sint128:
-    case e_uint128:
-    case e_sint256:
-    case e_uint256:
-    case e_sint512:
-    case e_uint512:
-    case e_float:
-    case e_double:
-      break;
-    case e_long_double:
-      success = true;
-      break;
-    }
-    break;
-  }
-
-  if (success)
-    m_type = type;
-  return success;
-}
-
-const char *Scalar::GetValueTypeAsCString(Scalar::Type type) {
-  switch (type) {
-  case e_void:
-    return "void";
-  case e_sint:
-    return "int";
-  case e_uint:
-    return "unsigned int";
-  case e_slong:
-    return "long";
-  case e_ulong:
-    return "unsigned long";
-  case e_slonglong:
-    return "long long";
-  case e_ulonglong:
-    return "unsigned long long";
-  case e_float:
-    return "float";
-  case e_double:
-    return "double";
-  case e_long_double:
-    return "long double";
-  case e_sint128:
-    return "int128_t";
-  case e_uint128:
-    return "uint128_t";
-  case e_sint256:
-    return "int256_t";
-  case e_uint256:
-    return "uint256_t";
-  case e_sint512:
-    return "int512_t";
-  case e_uint512:
-    return "uint512_t";
-  }
-  return "???";
-}
-
-Scalar::Type
-Scalar::GetValueTypeForSignedIntegerWithByteSize(size_t byte_size) {
-  if (byte_size <= sizeof(sint_t))
-    return e_sint;
-  if (byte_size <= sizeof(slong_t))
-    return e_slong;
-  if (byte_size <= sizeof(slonglong_t))
-    return e_slonglong;
-  return e_void;
-}
-
-Scalar::Type
-Scalar::GetValueTypeForUnsignedIntegerWithByteSize(size_t byte_size) {
-  if (byte_size <= sizeof(uint_t))
-    return e_uint;
-  if (byte_size <= sizeof(ulong_t))
-    return e_ulong;
-  if (byte_size <= sizeof(ulonglong_t))
-    return e_ulonglong;
-  return e_void;
-}
-
-Scalar::Type Scalar::GetValueTypeForFloatWithByteSize(size_t byte_size) {
-  if (byte_size == sizeof(float_t))
-    return e_float;
-  if (byte_size == sizeof(double_t))
-    return e_double;
-  if (byte_size == sizeof(long_double_t))
-    return e_long_double;
-  return e_void;
-}
-
-bool Scalar::MakeSigned() {
-  bool success = false;
-
-  switch (m_type) {
-  case e_void:
-    break;
-  case e_sint:
-    success = true;
-    break;
-  case e_uint:
-    m_type = e_sint;
-    success = true;
-    break;
-  case e_slong:
-    success = true;
-    break;
-  case e_ulong:
-    m_type = e_slong;
-    success = true;
-    break;
-  case e_slonglong:
-    success = true;
-    break;
-  case e_ulonglong:
-    m_type = e_slonglong;
-    success = true;
-    break;
-  case e_sint128:
-    success = true;
-    break;
-  case e_uint128:
-    m_type = e_sint128;
-    success = true;
-    break;
-  case e_sint256:
-    success = true;
-    break;
-  case e_uint256:
-    m_type = e_sint256;
-    success = true;
-    break;
-  case e_sint512:
-    success = true;
-    break;
-  case e_uint512:
-    m_type = e_sint512;
-    success = true;
-    break;
-  case e_float:
-    success = true;
-    break;
-  case e_double:
-    success = true;
-    break;
-  case e_long_double:
-    success = true;
-    break;
-  }
-
-  return success;
-}
-
-bool Scalar::MakeUnsigned() {
-  bool success = false;
-
-  switch (m_type) {
-  case e_void:
-    break;
-  case e_sint:
-    m_type = e_uint;
-    success = true;
-    break;
-  case e_uint:
-    success = true;
-    break;
-  case e_slong:
-    m_type = e_ulong;
-    success = true;
-    break;
-  case e_ulong:
-    success = true;
-    break;
-  case e_slonglong:
-    m_type = e_ulonglong;
-    success = true;
-    break;
-  case e_ulonglong:
-    success = true;
-    break;
-  case e_sint128:
-    m_type = e_uint128;
-    success = true;
-    break;
-  case e_uint128:
-    success = true;
-    break;
-  case e_sint256:
-    m_type = e_uint256;
-    success = true;
-    break;
-  case e_uint256:
-    success = true;
-    break;
-  case e_sint512:
-    m_type = e_uint512;
-    success = true;
-    break;
-  case e_uint512:
-    success = true;
-    break;
-  case e_float:
-    success = true;
-    break;
-  case e_double:
-    success = true;
-    break;
-  case e_long_double:
-    success = true;
-    break;
-  }
-
-  return success;
-}
-
-signed char Scalar::SChar(char fail_value) const {
-  switch (m_type) {
-  case e_void:
-    break;
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
-    return static_cast<schar_t>(
-        (m_integer.sextOrTrunc(sizeof(schar_t) * 8)).getSExtValue());
-  case e_float:
-    return static_cast<schar_t>(m_float.convertToFloat());
-  case e_double:
-    return static_cast<schar_t>(m_float.convertToDouble());
-  case e_long_double:
-    llvm::APInt ldbl_val = m_float.bitcastToAPInt();
-    return static_cast<schar_t>(
-        (ldbl_val.sextOrTrunc(sizeof(schar_t) * 8)).getSExtValue());
-  }
-  return fail_value;
-}
-
-unsigned char Scalar::UChar(unsigned char fail_value) const {
-  switch (m_type) {
-  case e_void:
-    break;
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
-    return static_cast<uchar_t>(
-        (m_integer.zextOrTrunc(sizeof(uchar_t) * 8)).getZExtValue());
-  case e_float:
-    return static_cast<uchar_t>(m_float.convertToFloat());
-  case e_double:
-    return static_cast<uchar_t>(m_float.convertToDouble());
-  case e_long_double:
-    llvm::APInt ldbl_val = m_float.bitcastToAPInt();
-    return static_cast<uchar_t>(
-        (ldbl_val.zextOrTrunc(sizeof(uchar_t) * 8)).getZExtValue());
-  }
-  return fail_value;
-}
-
-short Scalar::SShort(short fail_value) const {
-  switch (m_type) {
-  case e_void:
-    break;
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
-    return static_cast<sshort_t>(
-        (m_integer.sextOrTrunc(sizeof(sshort_t) * 8)).getSExtValue());
-  case e_float:
-    return static_cast<sshort_t>(m_float.convertToFloat());
-  case e_double:
-    return static_cast<sshort_t>(m_float.convertToDouble());
-  case e_long_double:
-    llvm::APInt ldbl_val = m_float.bitcastToAPInt();
-    return static_cast<sshort_t>(
-        (ldbl_val.sextOrTrunc(sizeof(sshort_t) * 8)).getSExtValue());
-  }
-  return fail_value;
-}
-
-unsigned short Scalar::UShort(unsigned short fail_value) const {
-  switch (m_type) {
-  case e_void:
-    break;
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
-    return static_cast<ushort_t>(
-        (m_integer.zextOrTrunc(sizeof(ushort_t) * 8)).getZExtValue());
-  case e_float:
-    return static_cast<ushort_t>(m_float.convertToFloat());
-  case e_double:
-    return static_cast<ushort_t>(m_float.convertToDouble());
-  case e_long_double:
-    llvm::APInt ldbl_val = m_float.bitcastToAPInt();
-    return static_cast<ushort_t>(
-        (ldbl_val.zextOrTrunc(sizeof(ushort_t) * 8)).getZExtValue());
-  }
-  return fail_value;
-}
-
-int Scalar::SInt(int fail_value) const {
-  switch (m_type) {
-  case e_void:
-    break;
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
-    return static_cast<sint_t>(
-        (m_integer.sextOrTrunc(sizeof(sint_t) * 8)).getSExtValue());
-  case e_float:
-    return static_cast<sint_t>(m_float.convertToFloat());
-  case e_double:
-    return static_cast<sint_t>(m_float.convertToDouble());
-  case e_long_double:
-    llvm::APInt ldbl_val = m_float.bitcastToAPInt();
-    return static_cast<sint_t>(
-        (ldbl_val.sextOrTrunc(sizeof(sint_t) * 8)).getSExtValue());
-  }
-  return fail_value;
-}
-
-unsigned int Scalar::UInt(unsigned int fail_value) const {
-  switch (m_type) {
-  case e_void:
-    break;
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
-    return static_cast<uint_t>(
-        (m_integer.zextOrTrunc(sizeof(uint_t) * 8)).getZExtValue());
-  case e_float:
-    return static_cast<uint_t>(m_float.convertToFloat());
-  case e_double:
-    return static_cast<uint_t>(m_float.convertToDouble());
-  case e_long_double:
-    llvm::APInt ldbl_val = m_float.bitcastToAPInt();
-    return static_cast<uint_t>(
-        (ldbl_val.zextOrTrunc(sizeof(uint_t) * 8)).getZExtValue());
-  }
-  return fail_value;
-}
-
-long Scalar::SLong(long fail_value) const {
-  switch (m_type) {
-  case e_void:
-    break;
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
-    return static_cast<slong_t>(
-        (m_integer.sextOrTrunc(sizeof(slong_t) * 8)).getSExtValue());
-  case e_float:
-    return static_cast<slong_t>(m_float.convertToFloat());
-  case e_double:
-    return static_cast<slong_t>(m_float.convertToDouble());
-  case e_long_double:
-    llvm::APInt ldbl_val = m_float.bitcastToAPInt();
-    return static_cast<slong_t>(
-        (ldbl_val.sextOrTrunc(sizeof(slong_t) * 8)).getSExtValue());
-  }
-  return fail_value;
+  if (success)
+    m_type = type;
+  return success;
 }
 
-unsigned long Scalar::ULong(unsigned long fail_value) const {
-  switch (m_type) {
+const char *Scalar::GetValueTypeAsCString(Scalar::Type type) {
+  switch (type) {
   case e_void:
-    break;
+    return "void";
   case e_sint:
+    return "int";
   case e_uint:
+    return "unsigned int";
   case e_slong:
+    return "long";
   case e_ulong:
+    return "unsigned long";
   case e_slonglong:
+    return "long long";
   case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
-    return static_cast<ulong_t>(
-        (m_integer.zextOrTrunc(sizeof(ulong_t) * 8)).getZExtValue());
+    return "unsigned long long";
   case e_float:
-    return static_cast<ulong_t>(m_float.convertToFloat());
+    return "float";
   case e_double:
-    return static_cast<ulong_t>(m_float.convertToDouble());
+    return "double";
   case e_long_double:
-    llvm::APInt ldbl_val = m_float.bitcastToAPInt();
-    return static_cast<ulong_t>(
-        (ldbl_val.zextOrTrunc(sizeof(ulong_t) * 8)).getZExtValue());
-  }
-  return fail_value;
-}
-
-long long Scalar::SLongLong(long long fail_value) const {
-  switch (m_type) {
-  case e_void:
-    break;
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
+    return "long double";
   case e_sint128:
+    return "int128_t";
   case e_uint128:
+    return "uint128_t";
   case e_sint256:
+    return "int256_t";
   case e_uint256:
+    return "uint256_t";
   case e_sint512:
+    return "int512_t";
   case e_uint512:
-    return static_cast<slonglong_t>(
-        (m_integer.sextOrTrunc(sizeof(slonglong_t) * 8)).getSExtValue());
-  case e_float:
-    return static_cast<slonglong_t>(m_float.convertToFloat());
-  case e_double:
-    return static_cast<slonglong_t>(m_float.convertToDouble());
-  case e_long_double:
-    llvm::APInt ldbl_val = m_float.bitcastToAPInt();
-    return static_cast<slonglong_t>(
-        (ldbl_val.sextOrTrunc(sizeof(slonglong_t) * 8)).getSExtValue());
+    return "uint512_t";
   }
-  return fail_value;
+  return "???";
 }
 
-unsigned long long Scalar::ULongLong(unsigned long long fail_value) const {
-  switch (m_type) {
-  case e_void:
-    break;
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
-    return static_cast<ulonglong_t>(
-        (m_integer.zextOrTrunc(sizeof(ulonglong_t) * 8)).getZExtValue());
-  case e_float:
-    return static_cast<ulonglong_t>(m_float.convertToFloat());
-  case e_double: {
-    double d_val = m_float.convertToDouble();
-    llvm::APInt rounded_double =
-        llvm::APIntOps::RoundDoubleToAPInt(d_val, sizeof(ulonglong_t) * 8);
-    return static_cast<ulonglong_t>(
-        (rounded_double.zextOrTrunc(sizeof(ulonglong_t) * 8)).getZExtValue());
-  }
-  case e_long_double:
-    llvm::APInt ldbl_val = m_float.bitcastToAPInt();
-    return static_cast<ulonglong_t>(
-        (ldbl_val.zextOrTrunc(sizeof(ulonglong_t) * 8)).getZExtValue());
-  }
-  return fail_value;
+Scalar::Type
+Scalar::GetValueTypeForSignedIntegerWithByteSize(size_t byte_size) {
+  if (byte_size <= sizeof(sint_t))
+    return e_sint;
+  if (byte_size <= sizeof(slong_t))
+    return e_slong;
+  if (byte_size <= sizeof(slonglong_t))
+    return e_slonglong;
+  return e_void;
 }
 
-llvm::APInt Scalar::SInt128(llvm::APInt &fail_value) const {
-  switch (m_type) {
-  case e_void:
-    break;
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
-    return m_integer;
-  case e_float:
-  case e_double:
-  case e_long_double:
-    return m_float.bitcastToAPInt();
-  }
-  return fail_value;
+Scalar::Type
+Scalar::GetValueTypeForUnsignedIntegerWithByteSize(size_t byte_size) {
+  if (byte_size <= sizeof(uint_t))
+    return e_uint;
+  if (byte_size <= sizeof(ulong_t))
+    return e_ulong;
+  if (byte_size <= sizeof(ulonglong_t))
+    return e_ulonglong;
+  return e_void;
 }
 
-llvm::APInt Scalar::UInt128(const llvm::APInt &fail_value) const {
-  switch (m_type) {
-  case e_void:
-    break;
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
-    return m_integer;
-  case e_float:
-  case e_double:
-  case e_long_double:
-    return m_float.bitcastToAPInt();
-  }
-  return fail_value;
+Scalar::Type Scalar::GetValueTypeForFloatWithByteSize(size_t byte_size) {
+  if (byte_size == sizeof(float_t))
+    return e_float;
+  if (byte_size == sizeof(double_t))
+    return e_double;
+  if (byte_size == sizeof(long_double_t))
+    return e_long_double;
+  return e_void;
 }
 
-float Scalar::Float(float fail_value) const {
-  switch (m_type) {
-  case e_void:
-    break;
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
-    return llvm::APIntOps::RoundAPIntToFloat(m_integer);
-  case e_float:
-    return m_float.convertToFloat();
-  case e_double:
-    return static_cast<float_t>(m_float.convertToDouble());
-  case e_long_double:
-    llvm::APInt ldbl_val = m_float.bitcastToAPInt();
-    return ldbl_val.bitsToFloat();
-  }
-  return fail_value;
-}
+bool Scalar::MakeSigned() {
+  bool success = false;
 
-double Scalar::Double(double fail_value) const {
   switch (m_type) {
   case e_void:
     break;
   case e_sint:
+    success = true;
+    break;
   case e_uint:
+    m_type = e_sint;
+    success = true;
+    break;
   case e_slong:
+    success = true;
+    break;
   case e_ulong:
+    m_type = e_slong;
+    success = true;
+    break;
   case e_slonglong:
+    success = true;
+    break;
   case e_ulonglong:
+    m_type = e_slonglong;
+    success = true;
+    break;
   case e_sint128:
+    success = true;
+    break;
   case e_uint128:
+    m_type = e_sint128;
+    success = true;
+    break;
   case e_sint256:
+    success = true;
+    break;
   case e_uint256:
+    m_type = e_sint256;
+    success = true;
+    break;
   case e_sint512:
+    success = true;
+    break;
   case e_uint512:
-    return llvm::APIntOps::RoundAPIntToDouble(m_integer);
+    m_type = e_sint512;
+    success = true;
+    break;
   case e_float:
-    return static_cast<double_t>(m_float.convertToFloat());
+    success = true;
+    break;
   case e_double:
-    return m_float.convertToDouble();
+    success = true;
+    break;
   case e_long_double:
-    llvm::APInt ldbl_val = m_float.bitcastToAPInt();
-    return ldbl_val.bitsToFloat();
+    success = true;
+    break;
   }
-  return fail_value;
+
+  return success;
 }
 
-long double Scalar::LongDouble(long double fail_value) const {
+bool Scalar::MakeUnsigned() {
+  bool success = false;
+
   switch (m_type) {
   case e_void:
     break;
   case e_sint:
+    m_type = e_uint;
+    success = true;
+    break;
   case e_uint:
+    success = true;
+    break;
   case e_slong:
+    m_type = e_ulong;
+    success = true;
+    break;
   case e_ulong:
+    success = true;
+    break;
   case e_slonglong:
+    m_type = e_ulonglong;
+    success = true;
+    break;
   case e_ulonglong:
+    success = true;
+    break;
   case e_sint128:
+    m_type = e_uint128;
+    success = true;
+    break;
   case e_uint128:
+    success = true;
+    break;
   case e_sint256:
+    m_type = e_uint256;
+    success = true;
+    break;
   case e_uint256:
+    success = true;
+    break;
   case e_sint512:
+    m_type = e_uint512;
+    success = true;
+    break;
   case e_uint512:
-    return static_cast<long_double_t>(
-        llvm::APIntOps::RoundAPIntToDouble(m_integer));
+    success = true;
+    break;
   case e_float:
-    return static_cast<long_double_t>(m_float.convertToFloat());
+    success = true;
+    break;
   case e_double:
-    return static_cast<long_double_t>(m_float.convertToDouble());
+    success = true;
+    break;
   case e_long_double:
-    llvm::APInt ldbl_val = m_float.bitcastToAPInt();
-    return static_cast<long_double_t>(ldbl_val.bitsToDouble());
+    success = true;
+    break;
+  }
+
+  return success;
+}
+
+static llvm::APInt ToAPInt(const llvm::APFloat &f, unsigned bits,
+                           bool is_unsigned) {
+  llvm::APSInt result(bits, is_unsigned);
+  bool isExact;
+  f.convertToInteger(result, llvm::APFloat::rmTowardZero, &isExact);
+  return std::move(result);
+}
+
+template <typename T> T Scalar::GetAs(T fail_value) const {
+  switch (GetCategory(m_type)) {
+  case Category::Void:
+    break;
+  case Category::Integral:
+    if (IsSigned(m_type))
+      return m_integer.sextOrTrunc(sizeof(T) * 8).getSExtValue();
+    return m_integer.zextOrTrunc(sizeof(T) * 8).getZExtValue();
+  case Category::Float:
+    return ToAPInt(m_float, sizeof(T) * 8, std::is_unsigned<T>::value)
+        .getSExtValue();
   }
   return fail_value;
 }
 
-Scalar &Scalar::operator+=(const Scalar &rhs) {
-  Scalar temp_value;
-  const Scalar *a;
-  const Scalar *b;
-  if ((m_type = PromoteToMaxType(*this, rhs, temp_value, a, b)) !=
-      Scalar::e_void) {
-    switch (m_type) {
-    case e_void:
-      break;
-    case e_sint:
-    case e_uint:
-    case e_slong:
-    case e_ulong:
-    case e_slonglong:
-    case e_ulonglong:
-    case e_sint128:
-    case e_uint128:
-    case e_sint256:
-    case e_uint256:
-    case e_sint512:
-    case e_uint512:
-      m_integer = a->m_integer + b->m_integer;
-      break;
+signed char Scalar::SChar(signed char fail_value) const {
+  return GetAs<signed char>(fail_value);
+}
 
-    case e_float:
-    case e_double:
-    case e_long_double:
-      m_float = a->m_float + b->m_float;
-      break;
-    }
+unsigned char Scalar::UChar(unsigned char fail_value) const {
+  return GetAs<unsigned char>(fail_value);
+}
+
+short Scalar::SShort(short fail_value) const {
+  return GetAs<short>(fail_value);
+}
+
+unsigned short Scalar::UShort(unsigned short fail_value) const {
+  return GetAs<unsigned short>(fail_value);
+}
+
+int Scalar::SInt(int fail_value) const { return GetAs<int>(fail_value); }
+
+unsigned int Scalar::UInt(unsigned int fail_value) const {
+  return GetAs<unsigned int>(fail_value);
+}
+
+long Scalar::SLong(long fail_value) const { return GetAs<long>(fail_value); }
+
+unsigned long Scalar::ULong(unsigned long fail_value) const {
+  return GetAs<unsigned long>(fail_value);
+}
+
+long long Scalar::SLongLong(long long fail_value) const {
+  return GetAs<long long>(fail_value);
+}
+
+unsigned long long Scalar::ULongLong(unsigned long long fail_value) const {
+  return GetAs<unsigned long long>(fail_value);
+}
+
+llvm::APInt Scalar::SInt128(const llvm::APInt &fail_value) const {
+  switch (GetCategory(m_type)) {
+  case Category::Void:
+    break;
+  case Category::Integral:
+    return m_integer;
+  case Category::Float:
+    return ToAPInt(m_float, 128, /*is_unsigned=*/false);
   }
-  return *this;
+  return fail_value;
 }
 
-Scalar &Scalar::operator<<=(const Scalar &rhs) {
-  switch (m_type) {
-  case e_void:
-  case e_float:
-  case e_double:
-  case e_long_double:
-    m_type = e_void;
+llvm::APInt Scalar::UInt128(const llvm::APInt &fail_value) const {
+  switch (GetCategory(m_type)) {
+  case Category::Void:
     break;
+  case Category::Integral:
+    return m_integer;
+  case Category::Float:
+    return ToAPInt(m_float, 128, /*is_unsigned=*/true);
+  }
+  return fail_value;
+}
 
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
-    switch (rhs.m_type) {
-    case e_void:
-    case e_float:
-    case e_double:
-    case e_long_double:
-      m_type = e_void;
-      break;
-    case e_sint:
-    case e_uint:
-    case e_slong:
-    case e_ulong:
-    case e_slonglong:
-    case e_ulonglong:
-    case e_sint128:
-    case e_uint128:
-    case e_sint256:
-    case e_uint256:
-    case e_sint512:
-    case e_uint512:
-      m_integer = m_integer << rhs.m_integer;
-      break;
-    }
+float Scalar::Float(float fail_value) const {
+  switch (GetCategory(m_type)) {
+  case Category::Void:
     break;
+  case Category::Integral:
+    if (IsSigned(m_type))
+      return llvm::APIntOps::RoundSignedAPIntToFloat(m_integer);
+    return llvm::APIntOps::RoundAPIntToFloat(m_integer);
+
+  case Category::Float: {
+    APFloat result = m_float;
+    bool losesInfo;
+    result.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven,
+                   &losesInfo);
+    return result.convertToFloat();
   }
-  return *this;
+  }
+  return fail_value;
 }
 
-bool Scalar::ShiftRightLogical(const Scalar &rhs) {
-  switch (m_type) {
-  case e_void:
-  case e_float:
-  case e_double:
-  case e_long_double:
-    m_type = e_void;
+double Scalar::Double(double fail_value) const {
+  switch (GetCategory(m_type)) {
+  case Category::Void:
     break;
+  case Category::Integral:
+    if (IsSigned(m_type))
+      return llvm::APIntOps::RoundSignedAPIntToDouble(m_integer);
+    return llvm::APIntOps::RoundAPIntToDouble(m_integer);
 
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
-    switch (rhs.m_type) {
-    case e_void:
-    case e_float:
-    case e_double:
-    case e_long_double:
-      m_type = e_void;
+  case Category::Float: {
+    APFloat result = m_float;
+    bool losesInfo;
+    result.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
+                   &losesInfo);
+    return result.convertToDouble();
+  }
+  }
+  return fail_value;
+}
+
+long double Scalar::LongDouble(long double fail_value) const {
+  /// No way to get more precision at the moment.
+  return static_cast<long double>(Double(fail_value));
+}
+
+Scalar &Scalar::operator+=(const Scalar &rhs) {
+  Scalar temp_value;
+  const Scalar *a;
+  const Scalar *b;
+  if ((m_type = PromoteToMaxType(*this, rhs, temp_value, a, b)) !=
+      Scalar::e_void) {
+    switch (GetCategory(m_type)) {
+    case Category::Void:
       break;
-    case e_sint:
-    case e_uint:
-    case e_slong:
-    case e_ulong:
-    case e_slonglong:
-    case e_ulonglong:
-    case e_sint128:
-    case e_uint128:
-    case e_sint256:
-    case e_uint256:
-    case e_sint512:
-    case e_uint512:
-      m_integer = m_integer.lshr(rhs.m_integer);
+    case Category::Integral:
+      m_integer = a->m_integer + b->m_integer;
+      break;
+
+    case Category::Float:
+      m_float = a->m_float + b->m_float;
       break;
     }
-    break;
   }
-  return m_type != e_void;
+  return *this;
+}
+
+Scalar &Scalar::operator<<=(const Scalar &rhs) {
+  if (GetCategory(m_type) == Category::Integral &&
+      GetCategory(rhs.m_type) == Category::Integral)
+    m_integer <<= rhs.m_integer;
+  else
+    m_type = e_void;
+  return *this;
+}
+
+bool Scalar::ShiftRightLogical(const Scalar &rhs) {
+  if (GetCategory(m_type) == Category::Integral &&
+      GetCategory(rhs.m_type) == Category::Integral) {
+    m_integer = m_integer.lshr(rhs.m_integer);
+    return true;
+  }
+  m_type = e_void;
+  return false;
 }
 
 Scalar &Scalar::operator>>=(const Scalar &rhs) {
@@ -1845,50 +776,11 @@ Scalar &Scalar::operator>>=(const Scalar &rhs) {
 }
 
 Scalar &Scalar::operator&=(const Scalar &rhs) {
-  switch (m_type) {
-  case e_void:
-  case e_float:
-  case e_double:
-  case e_long_double:
+  if (GetCategory(m_type) == Category::Integral &&
+      GetCategory(rhs.m_type) == Category::Integral)
+    m_integer &= rhs.m_integer;
+  else
     m_type = e_void;
-    break;
-
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
-    switch (rhs.m_type) {
-    case e_void:
-    case e_float:
-    case e_double:
-    case e_long_double:
-      m_type = e_void;
-      break;
-    case e_sint:
-    case e_uint:
-    case e_slong:
-    case e_ulong:
-    case e_slonglong:
-    case e_ulonglong:
-    case e_sint128:
-    case e_uint128:
-    case e_sint256:
-    case e_uint256:
-    case e_sint512:
-    case e_uint512:
-      m_integer &= rhs.m_integer;
-      break;
-    }
-    break;
-  }
   return *this;
 }
 
@@ -1924,26 +816,13 @@ bool Scalar::AbsoluteValue() {
 }
 
 bool Scalar::UnaryNegate() {
-  switch (m_type) {
-  case e_void:
+  switch (GetCategory(m_type)) {
+  case Category::Void:
     break;
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
+  case Category::Integral:
     m_integer = -m_integer;
     return true;
-  case e_float:
-  case e_double:
-  case e_long_double:
+  case Category::Float:
     m_float.changeSign();
     return true;
   }
@@ -1951,62 +830,17 @@ bool Scalar::UnaryNegate() {
 }
 
 bool Scalar::OnesComplement() {
-  switch (m_type) {
-  case e_sint:
-  case e_uint:
-  case e_slong:
-  case e_ulong:
-  case e_slonglong:
-  case e_ulonglong:
-  case e_sint128:
-  case e_uint128:
-  case e_sint256:
-  case e_uint256:
-  case e_sint512:
-  case e_uint512:
+  if (GetCategory(m_type) == Category::Integral) {
     m_integer = ~m_integer;
     return true;
-
-  case e_void:
-  case e_float:
-  case e_double:
-  case e_long_double:
-    break;
   }
+
   return false;
 }
 
 const Scalar lldb_private::operator+(const Scalar &lhs, const Scalar &rhs) {
-  Scalar result;
-  Scalar temp_value;
-  const Scalar *a;
-  const Scalar *b;
-  if ((result.m_type = PromoteToMaxType(lhs, rhs, temp_value, a, b)) !=
-      Scalar::e_void) {
-    switch (result.m_type) {
-    case Scalar::e_void:
-      break;
-    case Scalar::e_sint:
-    case Scalar::e_uint:
-    case Scalar::e_slong:
-    case Scalar::e_ulong:
-    case Scalar::e_slonglong:
-    case Scalar::e_ulonglong:
-    case Scalar::e_sint128:
-    case Scalar::e_uint128:
-    case Scalar::e_sint256:
-    case Scalar::e_uint256:
-    case Scalar::e_sint512:
-    case Scalar::e_uint512:
-      result.m_integer = a->m_integer + b->m_integer;
-      break;
-    case Scalar::e_float:
-    case Scalar::e_double:
-    case Scalar::e_long_double:
-      result.m_float = a->m_float + b->m_float;
-      break;
-    }
-  }
+  Scalar result = lhs;
+  result += rhs;
   return result;
 }
 
@@ -2017,26 +851,13 @@ const Scalar lldb_private::operator-(const Scalar &lhs, const Scalar &rhs) {
   const Scalar *b;
   if ((result.m_type = PromoteToMaxType(lhs, rhs, temp_value, a, b)) !=
       Scalar::e_void) {
-    switch (result.m_type) {
-    case Scalar::e_void:
+    switch (GetCategory(result.m_type)) {
+    case Category::Void:
       break;
-    case Scalar::e_sint:
-    case Scalar::e_uint:
-    case Scalar::e_slong:
-    case Scalar::e_ulong:
-    case Scalar::e_slonglong:
-    case Scalar::e_ulonglong:
-    case Scalar::e_sint128:
-    case Scalar::e_uint128:
-    case Scalar::e_sint256:
-    case Scalar::e_uint256:
-    case Scalar::e_sint512:
-    case Scalar::e_uint512:
+    case Category::Integral:
       result.m_integer = a->m_integer - b->m_integer;
       break;
-    case Scalar::e_float:
-    case Scalar::e_double:
-    case Scalar::e_long_double:
+    case Category::Float:
       result.m_float = a->m_float - b->m_float;
       break;
     }
@@ -2050,40 +871,20 @@ const Scalar lldb_private::operator/(const Scalar &lhs, const Scalar &rhs) {
   const Scalar *a;
   const Scalar *b;
   if ((result.m_type = PromoteToMaxType(lhs, rhs, temp_value, a, b)) !=
-      Scalar::e_void) {
-    switch (result.m_type) {
-    case Scalar::e_void:
+          Scalar::e_void &&
+      !b->IsZero()) {
+    switch (GetCategory(result.m_type)) {
+    case Category::Void:
       break;
-    case Scalar::e_sint:
-    case Scalar::e_slong:
-    case Scalar::e_slonglong:
-    case Scalar::e_sint128:
-    case Scalar::e_sint256:
-    case Scalar::e_sint512:
-      if (b->m_integer != 0) {
+    case Category::Integral:
+      if (IsSigned(result.m_type))
         result.m_integer = a->m_integer.sdiv(b->m_integer);
-        return result;
-      }
-      break;
-    case Scalar::e_uint:
-    case Scalar::e_ulong:
-    case Scalar::e_ulonglong:
-    case Scalar::e_uint128:
-    case Scalar::e_uint256:
-    case Scalar::e_uint512:
-      if (b->m_integer != 0) {
+      else 
         result.m_integer = a->m_integer.udiv(b->m_integer);
-        return result;
-      }
-      break;
-    case Scalar::e_float:
-    case Scalar::e_double:
-    case Scalar::e_long_double:
-      if (!b->m_float.isZero()) {
-        result.m_float = a->m_float / b->m_float;
-        return result;
-      }
-      break;
+      return result;
+    case Category::Float:
+      result.m_float = a->m_float / b->m_float;
+      return result;
     }
   }
   // For division only, the only way it should make it here is if a promotion
@@ -2099,26 +900,13 @@ const Scalar lldb_private::operator*(const Scalar &lhs, const Scalar &rhs) {
   const Scalar *b;
   if ((result.m_type = PromoteToMaxType(lhs, rhs, temp_value, a, b)) !=
       Scalar::e_void) {
-    switch (result.m_type) {
-    case Scalar::e_void:
+    switch (GetCategory(result.m_type)) {
+    case Category::Void:
       break;
-    case Scalar::e_sint:
-    case Scalar::e_uint:
-    case Scalar::e_slong:
-    case Scalar::e_ulong:
-    case Scalar::e_slonglong:
-    case Scalar::e_ulonglong:
-    case Scalar::e_sint128:
-    case Scalar::e_uint128:
-    case Scalar::e_sint256:
-    case Scalar::e_uint256:
-    case Scalar::e_sint512:
-    case Scalar::e_uint512:
+    case Category::Integral:
       result.m_integer = a->m_integer * b->m_integer;
       break;
-    case Scalar::e_float:
-    case Scalar::e_double:
-    case Scalar::e_long_double:
+    case Category::Float:
       result.m_float = a->m_float * b->m_float;
       break;
     }
@@ -2133,29 +921,10 @@ const Scalar lldb_private::operator&(const Scalar &lhs, const Scalar &rhs) {
   const Scalar *b;
   if ((result.m_type = PromoteToMaxType(lhs, rhs, temp_value, a, b)) !=
       Scalar::e_void) {
-    switch (result.m_type) {
-    case Scalar::e_sint:
-    case Scalar::e_uint:
-    case Scalar::e_slong:
-    case Scalar::e_ulong:
-    case Scalar::e_slonglong:
-    case Scalar::e_ulonglong:
-    case Scalar::e_sint128:
-    case Scalar::e_uint128:
-    case Scalar::e_sint256:
-    case Scalar::e_uint256:
-    case Scalar::e_sint512:
-    case Scalar::e_uint512:
+    if (GetCategory(result.m_type) == Category::Integral)
       result.m_integer = a->m_integer & b->m_integer;
-      break;
-    case Scalar::e_void:
-    case Scalar::e_float:
-    case Scalar::e_double:
-    case Scalar::e_long_double:
-      // No bitwise AND on floats, doubles of long doubles
+    else
       result.m_type = Scalar::e_void;
-      break;
-    }
   }
   return result;
 }
@@ -2167,30 +936,10 @@ const Scalar lldb_private::operator|(const Scalar &lhs, const Scalar &rhs) {
   const Scalar *b;
   if ((result.m_type = PromoteToMaxType(lhs, rhs, temp_value, a, b)) !=
       Scalar::e_void) {
-    switch (result.m_type) {
-    case Scalar::e_sint:
-    case Scalar::e_uint:
-    case Scalar::e_slong:
-    case Scalar::e_ulong:
-    case Scalar::e_slonglong:
-    case Scalar::e_ulonglong:
-    case Scalar::e_sint128:
-    case Scalar::e_uint128:
-    case Scalar::e_sint256:
-    case Scalar::e_uint256:
-    case Scalar::e_sint512:
-    case Scalar::e_uint512:
+    if (GetCategory(result.m_type) == Category::Integral)
       result.m_integer = a->m_integer | b->m_integer;
-      break;
-
-    case Scalar::e_void:
-    case Scalar::e_float:
-    case Scalar::e_double:
-    case Scalar::e_long_double:
-      // No bitwise AND on floats, doubles of long doubles
+    else
       result.m_type = Scalar::e_void;
-      break;
-    }
   }
   return result;
 }
@@ -2202,33 +951,12 @@ const Scalar lldb_private::operator%(const Scalar &lhs, const Scalar &rhs) {
   const Scalar *b;
   if ((result.m_type = PromoteToMaxType(lhs, rhs, temp_value, a, b)) !=
       Scalar::e_void) {
-    switch (result.m_type) {
-    default:
-      break;
-    case Scalar::e_void:
-      break;
-    case Scalar::e_sint:
-    case Scalar::e_slong:
-    case Scalar::e_slonglong:
-    case Scalar::e_sint128:
-    case Scalar::e_sint256:
-    case Scalar::e_sint512:
-      if (b->m_integer != 0) {
+    if (!b->IsZero() && GetCategory(result.m_type) == Category::Integral) {
+      if (IsSigned(result.m_type))
         result.m_integer = a->m_integer.srem(b->m_integer);
-        return result;
-      }
-      break;
-    case Scalar::e_uint:
-    case Scalar::e_ulong:
-    case Scalar::e_ulonglong:
-    case Scalar::e_uint128:
-    case Scalar::e_uint256:
-    case Scalar::e_uint512:
-      if (b->m_integer != 0) {
+      else
         result.m_integer = a->m_integer.urem(b->m_integer);
-        return result;
-      }
-      break;
+      return result;
     }
   }
   result.m_type = Scalar::e_void;
@@ -2242,30 +970,10 @@ const Scalar lldb_private::operator^(const Scalar &lhs, const Scalar &rhs) {
   const Scalar *b;
   if ((result.m_type = PromoteToMaxType(lhs, rhs, temp_value, a, b)) !=
       Scalar::e_void) {
-    switch (result.m_type) {
-    case Scalar::e_sint:
-    case Scalar::e_uint:
-    case Scalar::e_slong:
-    case Scalar::e_ulong:
-    case Scalar::e_slonglong:
-    case Scalar::e_ulonglong:
-    case Scalar::e_sint128:
-    case Scalar::e_uint128:
-    case Scalar::e_sint256:
-    case Scalar::e_uint256:
-    case Scalar::e_sint512:
-    case Scalar::e_uint512:
+    if (GetCategory(result.m_type) == Category::Integral)
       result.m_integer = a->m_integer ^ b->m_integer;
-      break;
-
-    case Scalar::e_void:
-    case Scalar::e_float:
-    case Scalar::e_double:
-    case Scalar::e_long_double:
-      // No bitwise AND on floats, doubles of long doubles
+    else
       result.m_type = Scalar::e_void;
-      break;
-    }
   }
   return result;
 }
diff --git a/lldb/test/API/CMakeLists.txt b/lldb/test/API/CMakeLists.txt
index 9aad9fc750cae..34f3522c8dfec 100644
--- a/lldb/test/API/CMakeLists.txt
+++ b/lldb/test/API/CMakeLists.txt
@@ -49,6 +49,7 @@ set(LLDB_DEFAULT_TEST_EXECUTABLE "${LLVM_RUNTIME_OUTPUT_INTDIR}/lldb${CMAKE_EXEC
 # Set the paths to default llvm tools.
 set(LLDB_DEFAULT_TEST_DSYMUTIL "${LLVM_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin/dsymutil${CMAKE_EXECUTABLE_SUFFIX}")
 set(LLDB_DEFAULT_TEST_FILECHECK "${LLVM_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin/FileCheck${CMAKE_EXECUTABLE_SUFFIX}")
+set(LLDB_DEFAULT_TEST_YAML2OBJ "${LLVM_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin/yaml2obj${CMAKE_EXECUTABLE_SUFFIX}")
 
 if (TARGET clang)
   set(LLDB_DEFAULT_TEST_COMPILER "${LLVM_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin/clang${CMAKE_EXECUTABLE_SUFFIX}")
@@ -60,6 +61,7 @@ set(LLDB_TEST_EXECUTABLE "${LLDB_DEFAULT_TEST_EXECUTABLE}" CACHE PATH "lldb exec
 set(LLDB_TEST_COMPILER "${LLDB_DEFAULT_TEST_COMPILER}" CACHE PATH "C Compiler to use for building LLDB test inferiors")
 set(LLDB_TEST_DSYMUTIL "${LLDB_DEFAULT_TEST_DSYMUTIL}" CACHE PATH "dsymutil used for generating dSYM bundles")
 set(LLDB_TEST_FILECHECK "${LLDB_DEFAULT_TEST_FILECHECK}" CACHE PATH "FileCheck used for testing purposes")
+set(LLDB_TEST_YAML2OBJ "${LLDB_DEFAULT_TEST_YAML2OBJ}" CACHE PATH "yaml2obj used for testing purposes")
 
 if ("${LLDB_TEST_COMPILER}" STREQUAL "")
   message(FATAL_ERROR "LLDB test compiler not specified. Tests will not run.")
@@ -145,6 +147,7 @@ if(LLDB_BUILT_STANDALONE)
   string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_COMPILER "${LLDB_TEST_COMPILER}")
   string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_DSYMUTIL "${LLDB_TEST_DSYMUTIL}")
   string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_FILECHECK "${LLDB_TEST_FILECHECK}")
+  string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_YAML2OBJ "${LLDB_TEST_YAML2OBJ}")
 
   # Remaining ones must be paths to the provided LLVM build-tree.
   if(LLVM_CONFIGURATION_TYPES)
@@ -172,6 +175,7 @@ string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_EXECUTAB
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_COMPILER "${LLDB_TEST_COMPILER}")
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_DSYMUTIL "${LLDB_TEST_DSYMUTIL}")
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_FILECHECK "${LLDB_TEST_FILECHECK}")
+string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_YAML2OBJ "${LLDB_TEST_YAML2OBJ}")
 
 # Configure the API test suite.
 configure_lit_site_cfg(
diff --git a/lldb/test/API/arm/breakpoint-thumb-codesection/TestBreakpointThumbCodesection.py b/lldb/test/API/arm/breakpoint-thumb-codesection/TestBreakpointThumbCodesection.py
index 0726369b6765c..d472b79fb87a2 100644
--- a/lldb/test/API/arm/breakpoint-thumb-codesection/TestBreakpointThumbCodesection.py
+++ b/lldb/test/API/arm/breakpoint-thumb-codesection/TestBreakpointThumbCodesection.py
@@ -15,6 +15,7 @@ class TestBreakpointThumbCodesection(TestBase):
     mydir = TestBase.compute_mydir(__file__)
 
     @skipIf(archs=no_match(["arm"]))
+    @skipIf(archs=["arm64"])
     @skipIfDarwinEmbedded   # codegen on darwin always defaults to thumb for armv7/armv7k targets
     def test_breakpoint(self):
         self.build()
diff --git a/lldb/test/API/commands/expression/call-restarts/TestCallThatRestarts.py b/lldb/test/API/commands/expression/call-restarts/TestCallThatRestarts.py
index f48cc5e6332b6..a7e1a8c800ccb 100644
--- a/lldb/test/API/commands/expression/call-restarts/TestCallThatRestarts.py
+++ b/lldb/test/API/commands/expression/call-restarts/TestCallThatRestarts.py
@@ -52,10 +52,7 @@ def call_function(self):
                                       'Stop here in main.', self.main_source_spec)
 
         # Make sure the SIGCHLD behavior is pass/no-stop/no-notify:
-        return_obj = lldb.SBCommandReturnObject()
-        self.dbg.GetCommandInterpreter().HandleCommand(
-            "process handle SIGCHLD -s 0 -p 1 -n 0", return_obj)
-        self.assertTrue(return_obj.Succeeded(), "Set SIGCHLD to pass, no-stop")
+        self.runCmd("process handle SIGCHLD -s 0 -p 1 -n 0")
 
         # The sigchld_no variable should be 0 at this point.
         self.sigchld_no = target.FindFirstGlobalVariable("sigchld_no")
@@ -84,7 +81,7 @@ def call_function(self):
             "call_me (%d)" %
             (num_sigchld), options)
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
         self.assertEquals(value.GetValueAsSigned(-1), num_sigchld)
 
         self.check_after_call(num_sigchld)
@@ -101,23 +98,21 @@ def call_function(self):
             "call_me (%d)" %
             (num_sigchld), options)
 
-        self.assertTrue(value.IsValid() and value.GetError().Success())
+        self.assertTrue(value.IsValid())
+        self.assertSuccess(value.GetError())
         self.assertEquals(value.GetValueAsSigned(-1), num_sigchld)
         self.check_after_call(num_sigchld)
 
         # Now set the signal to print but not stop and make sure that calling
         # still works:
-        self.dbg.GetCommandInterpreter().HandleCommand(
-            "process handle SIGCHLD -s 0 -p 1 -n 1", return_obj)
-        self.assertTrue(
-            return_obj.Succeeded(),
-            "Set SIGCHLD to pass, no-stop, notify")
+        self.runCmd("process handle SIGCHLD -s 0 -p 1 -n 1")
 
         value = frame.EvaluateExpression(
             "call_me (%d)" %
             (num_sigchld), options)
 
-        self.assertTrue(value.IsValid() and value.GetError().Success())
+        self.assertTrue(value.IsValid())
+        self.assertSuccess(value.GetError())
         self.assertEquals(value.GetValueAsSigned(-1), num_sigchld)
         self.check_after_call(num_sigchld)
 
@@ -128,36 +123,28 @@ def call_function(self):
             "call_me (%d)" %
             (num_sigchld), options)
 
-        self.assertTrue(value.IsValid() and value.GetError().Success())
+        self.assertTrue(value.IsValid())
+        self.assertSuccess(value.GetError())
         self.assertEquals(value.GetValueAsSigned(-1), num_sigchld)
         self.check_after_call(num_sigchld)
 
         # Okay, now set UnwindOnError to true, and then make the signal behavior to stop
         # and see that now we do stop at the signal point:
 
-        self.dbg.GetCommandInterpreter().HandleCommand(
-            "process handle SIGCHLD -s 1 -p 1 -n 1", return_obj)
-        self.assertTrue(
-            return_obj.Succeeded(),
-            "Set SIGCHLD to pass, stop, notify")
+        self.runCmd("process handle SIGCHLD -s 1 -p 1 -n 1")
 
         value = frame.EvaluateExpression(
             "call_me (%d)" %
             (num_sigchld), options)
-        self.assertTrue(
-            value.IsValid() and value.GetError().Success() == False)
+        self.assertTrue(value.IsValid())
+        self.assertFalse(value.GetError().Success())
 
         # Set signal handling back to no-stop, and continue and we should end
         # up back in out starting frame:
-        self.dbg.GetCommandInterpreter().HandleCommand(
-            "process handle SIGCHLD -s 0 -p 1 -n 1", return_obj)
-        self.assertTrue(
-            return_obj.Succeeded(),
-            "Set SIGCHLD to pass, no-stop, notify")
+        self.runCmd("process handle SIGCHLD -s 0 -p 1 -n 1")
 
         error = process.Continue()
-        self.assertTrue(
-            error.Success(),
+        self.assertSuccess(error,
             "Continuing after stopping for signal succeeds.")
 
         frame = self.thread.GetFrameAtIndex(0)
diff --git a/lldb/test/API/commands/expression/call-throws/TestCallThatThrows.py b/lldb/test/API/commands/expression/call-throws/TestCallThatThrows.py
index 9fe1405c3d720..eb84b7b54eb61 100644
--- a/lldb/test/API/commands/expression/call-throws/TestCallThatThrows.py
+++ b/lldb/test/API/commands/expression/call-throws/TestCallThatThrows.py
@@ -88,7 +88,7 @@ def call_function(self):
         options.SetTrapExceptions(False)
         value = frame.EvaluateExpression("[my_class iCatchMyself]", options)
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
         self.assertEquals(value.GetValueAsUnsigned(), 57)
         self.check_after_call()
         options.SetTrapExceptions(True)
diff --git a/lldb/test/API/commands/expression/context-object-objc/TestContextObjectObjc.py b/lldb/test/API/commands/expression/context-object-objc/TestContextObjectObjc.py
index 4ae4fd8680d81..36384b7527510 100644
--- a/lldb/test/API/commands/expression/context-object-objc/TestContextObjectObjc.py
+++ b/lldb/test/API/commands/expression/context-object-objc/TestContextObjectObjc.py
@@ -44,13 +44,13 @@ def test_context_object_objc(self):
         # Test retrieving of an objcClass's property through the self pointer
         value = obj_val.EvaluateExpression("self.property")
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
         self.assertEqual(value.GetValueAsSigned(), 2222)
 
         # Test objcClass's methods evaluation through the self pointer
         value = obj_val.EvaluateExpression("[self method]")
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
         self.assertEqual(value.GetValueAsSigned(), 3333)
 
         # Test if we can use a computation result reference object correctly
@@ -63,12 +63,12 @@ def test_context_object_objc(self):
         # Test an expression evaluation on it
         value = obj_val.EvaluateExpression("1")
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
 
         # Test retrieving of a field on it
         value = obj_val.EvaluateExpression("field")
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
         self.assertEqual(value.GetValueAsSigned(), 1111)
 
     def setUp(self):
diff --git a/lldb/test/API/commands/expression/context-object/TestContextObject.py b/lldb/test/API/commands/expression/context-object/TestContextObject.py
index 02b8162aad656..b7dc6d53dc47c 100644
--- a/lldb/test/API/commands/expression/context-object/TestContextObject.py
+++ b/lldb/test/API/commands/expression/context-object/TestContextObject.py
@@ -32,19 +32,19 @@ def test_context_object(self):
         # Test retrieveing of a field (not a local with the same name)
         value = obj_val.EvaluateExpression("field")
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
         self.assertEqual(value.GetValueAsSigned(), 1111)
 
         # Test functions evaluation
         value = obj_val.EvaluateExpression("function()")
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
         self.assertEqual(value.GetValueAsSigned(), 2222)
 
         # Test that we retrieve the right global
         value = obj_val.EvaluateExpression("global.field")
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
         self.assertEqual(value.GetValueAsSigned(), 1111)
 
         #
@@ -57,7 +57,7 @@ def test_context_object(self):
         # Test retrieveing of a field
         value = obj_val.EvaluateExpression("field_int")
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
         self.assertEqual(value.GetValueAsSigned(), 5555)
 
         #
@@ -87,7 +87,7 @@ def test_context_object(self):
         # Test retrieveing of an element's field
         value = obj_val.GetValueForExpressionPath("[7]").EvaluateExpression("field")
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
         self.assertEqual(value.GetValueAsSigned(), 1111)
 
         #
@@ -105,7 +105,7 @@ def test_context_object(self):
         # Test retrieveing of a dereferenced object's field
         value = obj_val.Dereference().EvaluateExpression("field")
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
         self.assertEqual(value.GetValueAsSigned(), 1111)
 
         #
@@ -135,7 +135,7 @@ def test_context_object(self):
         # Test retrieveing of a dereferenced object's field
         value = obj_val.Dereference().EvaluateExpression("field")
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
         self.assertEqual(value.GetValueAsSigned(), 1111)
 
     def setUp(self):
diff --git a/lldb/test/API/commands/expression/dollar-in-variable/Makefile b/lldb/test/API/commands/expression/dollar-in-variable/Makefile
new file mode 100644
index 0000000000000..10495940055b6
--- /dev/null
+++ b/lldb/test/API/commands/expression/dollar-in-variable/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/commands/expression/dollar-in-variable/TestDollarInVariable.py b/lldb/test/API/commands/expression/dollar-in-variable/TestDollarInVariable.py
index 7458867527df7..d15eae304a4ca 100644
--- a/lldb/test/API/commands/expression/dollar-in-variable/TestDollarInVariable.py
+++ b/lldb/test/API/commands/expression/dollar-in-variable/TestDollarInVariable.py
@@ -1,5 +1,19 @@
-from lldbsuite.test import lldbinline
-from lldbsuite.test import decorators
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
 
-lldbinline.MakeInlineTest(__file__, globals(),
-  [lldbinline.expectedFailureAll(oslist=["windows"])])
+class TestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @expectedFailureAll(oslist=["windows"])
+    def test(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(self, "// break here", lldb.SBFileSpec("main.c"))
+
+        self.expect_expr("$__lldb_expr_result", result_type="int", result_value="11")
+        self.expect_expr("$foo", result_type="int", result_value="12")
+        self.expect_expr("$R0", result_type="int", result_value="13")
+        self.expect("expr int $foo = 123", error=True, substrs=["declaration conflicts"])
+        self.expect_expr("$0", result_type="int", result_value="11")
diff --git a/lldb/test/API/commands/expression/dollar-in-variable/main.c b/lldb/test/API/commands/expression/dollar-in-variable/main.c
index 97c939b68edfa..d1a16a7b88ddb 100644
--- a/lldb/test/API/commands/expression/dollar-in-variable/main.c
+++ b/lldb/test/API/commands/expression/dollar-in-variable/main.c
@@ -13,10 +13,5 @@ int main() {
   int $R0 = 13;
   int $0 = 14;
 
-  //%self.expect("expr $__lldb_expr_result", substrs=['(int) $0 = 11'])
-  //%self.expect("expr $foo", substrs=['(int)', ' = 12'])
-  //%self.expect("expr $R0", substrs=['(int)', ' = 13'])
-  //%self.expect("expr int $foo = 123", error=True, substrs=["declaration conflicts"])
-  //%self.expect("expr $0", substrs=['(int)', ' = 11'])
-  return 0;
+  return 0; // break here
 }
diff --git a/lldb/test/API/commands/expression/dont_allow_jit/TestAllowJIT.py b/lldb/test/API/commands/expression/dont_allow_jit/TestAllowJIT.py
index c70e90e7a26f7..083d97adf1b26 100644
--- a/lldb/test/API/commands/expression/dont_allow_jit/TestAllowJIT.py
+++ b/lldb/test/API/commands/expression/dont_allow_jit/TestAllowJIT.py
@@ -58,7 +58,7 @@ def expr_options_test(self):
 
         # Now use the options:
         result = frame.EvaluateExpression("call_me(10)", options)
-        self.assertTrue(result.GetError().Success(), "expression succeeded")
+        self.assertSuccess(result.GetError())
         self.assertEqual(result.GetValueAsSigned(), 18, "got the right value.")
 
         # Now disallow JIT and make sure it fails:
@@ -77,6 +77,6 @@ def expr_options_test(self):
 
         # And again, make sure this works:
         result = frame.EvaluateExpression("call_me(10)", options)
-        self.assertTrue(result.GetError().Success(), "expression succeeded")
+        self.assertSuccess(result.GetError())
         self.assertEqual(result.GetValueAsSigned(), 18, "got the right value.")
 
diff --git a/lldb/test/API/commands/expression/fixits/TestFixIts.py b/lldb/test/API/commands/expression/fixits/TestFixIts.py
index 9907c25577883..59a1eb2d2abf3 100644
--- a/lldb/test/API/commands/expression/fixits/TestFixIts.py
+++ b/lldb/test/API/commands/expression/fixits/TestFixIts.py
@@ -42,7 +42,7 @@ def test_with_target(self):
         # Try with one error:
         value = frame.EvaluateExpression("my_pointer.first", options)
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
         self.assertEquals(value.GetValueAsUnsigned(), 10)
 
         # Try with one error in a top-level expression.
@@ -58,7 +58,7 @@ def test_with_target(self):
         two_error_expression = "my_pointer.second->a"
         value = frame.EvaluateExpression(two_error_expression, options)
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
         self.assertEquals(value.GetValueAsUnsigned(), 20)
 
         # Try a Fix-It that is stored in the 'note:' diagnostic of an error.
@@ -66,7 +66,7 @@ def test_with_target(self):
         fixit_in_note_expr ="#define ToStr(x) #x\nToStr(0 {, })"
         value = frame.EvaluateExpression(fixit_in_note_expr, options)
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success(), value.GetError())
+        self.assertSuccess(value.GetError())
         self.assertEquals(value.GetSummary(), '"(0 {, })"')
 
         # Now turn off the fixits, and the expression should fail:
diff --git a/lldb/test/API/commands/expression/ir-interpreter/TestIRInterpreter.py b/lldb/test/API/commands/expression/ir-interpreter/TestIRInterpreter.py
index 033f5bc5fa1d1..5efbba90ccbab 100644
--- a/lldb/test/API/commands/expression/ir-interpreter/TestIRInterpreter.py
+++ b/lldb/test/API/commands/expression/ir-interpreter/TestIRInterpreter.py
@@ -51,7 +51,8 @@ def test_ir_interpreter(self):
         options = lldb.SBExpressionOptions()
         options.SetLanguage(lldb.eLanguageTypeC_plus_plus)
 
-        set_up_expressions = ["int $i = 9", "int $j = 3", "int $k = 5"]
+        set_up_expressions = ["int $i = 9", "int $j = 3", "int $k = 5",
+            "unsigned long long $ull = -1", "unsigned $u = -1"]
 
         expressions = ["$i + $j",
                        "$i - $j",
@@ -61,7 +62,8 @@ def test_ir_interpreter(self):
                        "$i << $j",
                        "$i & $j",
                        "$i | $j",
-                       "$i ^ $j"]
+                       "$i ^ $j",
+                       "($ull & -1) == $u"]
 
         for expression in set_up_expressions:
             self.frame().EvaluateExpression(expression, options)
diff --git a/lldb/test/API/commands/expression/options/TestExprOptions.py b/lldb/test/API/commands/expression/options/TestExprOptions.py
index 833833a866f83..1f56e5fb09719 100644
--- a/lldb/test/API/commands/expression/options/TestExprOptions.py
+++ b/lldb/test/API/commands/expression/options/TestExprOptions.py
@@ -46,14 +46,14 @@ def test_expr_options(self):
         # Make sure we can evaluate a C++11 expression.
         val = frame.EvaluateExpression('foo != nullptr')
         self.assertTrue(val.IsValid())
-        self.assertTrue(val.GetError().Success())
+        self.assertSuccess(val.GetError())
         self.DebugSBValue(val)
 
         # Make sure it still works if language is set to C++11:
         options.SetLanguage(lldb.eLanguageTypeC_plus_plus_11)
         val = frame.EvaluateExpression('foo != nullptr', options)
         self.assertTrue(val.IsValid())
-        self.assertTrue(val.GetError().Success())
+        self.assertSuccess(val.GetError())
         self.DebugSBValue(val)
 
         # Make sure it fails if language is set to C:
@@ -80,7 +80,7 @@ def test_expr_options_lang(self):
         options.SetLanguage(lldb.eLanguageTypeC_plus_plus_11)
         val = frame.EvaluateExpression('id == 0', options)
         self.assertTrue(val.IsValid())
-        self.assertTrue(val.GetError().Success())
+        self.assertSuccess(val.GetError())
         self.DebugSBValue(val)
 
         # Make sure we can't retrieve `id` variable if language is set to ObjC:
diff --git a/lldb/test/API/commands/expression/pr35310/TestExprsBug35310.py b/lldb/test/API/commands/expression/pr35310/TestExprsBug35310.py
index 23dbce9227e76..a9987ce3ed523 100644
--- a/lldb/test/API/commands/expression/pr35310/TestExprsBug35310.py
+++ b/lldb/test/API/commands/expression/pr35310/TestExprsBug35310.py
@@ -23,16 +23,8 @@ def test_issue35310(self):
         """
         self.build()
 
-        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(self,
+        lldbutil.run_to_source_breakpoint(self,
                                           '// Break here', self.main_source_spec)
-        frame = thread.GetFrameAtIndex(0)
 
-        value = frame.EvaluateExpression("a.test_abi_tag()")
-        self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
-        self.assertEqual(value.GetValueAsSigned(0), 1)
-
-        value = frame.EvaluateExpression("a.test_asm_name()")
-        self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
-        self.assertEqual(value.GetValueAsSigned(0), 2)
+        self.expect_expr("a.test_abi_tag()", result_value='1')
+        self.expect_expr("a.test_asm_name()", result_value='2')
diff --git a/lldb/test/API/commands/expression/result_numbering/TestResultNumbering.py b/lldb/test/API/commands/expression/result_numbering/TestResultNumbering.py
index cd6b9c43775c7..11429516d1f1e 100644
--- a/lldb/test/API/commands/expression/result_numbering/TestResultNumbering.py
+++ b/lldb/test/API/commands/expression/result_numbering/TestResultNumbering.py
@@ -33,7 +33,7 @@ def do_numbering_test(self):
 
         # Get the number of the last expression:
         result = thread.frames[0].EvaluateExpression("call_me(200)")
-        self.assertTrue(result.GetError().Success(), "Our expression succeeded")
+        self.assertSuccess(result.GetError(), "Our expression succeeded")
         name = result.GetName()
         ordinal = int(name[1:])
         
@@ -42,7 +42,7 @@ def do_numbering_test(self):
         # The condition evaluation had to run a 4 expressions, but we haven't
         # run any user expressions.
         result = thread.frames[0].EvaluateExpression("call_me(200)")
-        self.assertTrue(result.GetError().Success(), "Our expression succeeded the second time")
+        self.assertSuccess(result.GetError(), "Our expression succeeded the second time")
         after_name = result.GetName()
         after_ordinal = int(after_name[1:])
         self.assertEqual(ordinal + 1, after_ordinal) 
diff --git a/lldb/test/API/commands/expression/scoped_enums/TestScopedEnumType.py b/lldb/test/API/commands/expression/scoped_enums/TestScopedEnumType.py
index dd40e87d68d33..2c45d213e4df0 100644
--- a/lldb/test/API/commands/expression/scoped_enums/TestScopedEnumType.py
+++ b/lldb/test/API/commands/expression/scoped_enums/TestScopedEnumType.py
@@ -23,15 +23,8 @@ def test(self):
         self.expect("expr f == Foo::FooBar",
                 substrs=['(bool) $0 = true'])
 
-        value = frame.EvaluateExpression("f == Foo::FooBar")
-        self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
-        self.assertEqual(value.GetValueAsUnsigned(), 1)
-
-        value = frame.EvaluateExpression("b == BarBar")
-        self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
-        self.assertEqual(value.GetValueAsUnsigned(), 1)
+        self.expect_expr("f == Foo::FooBar", result_value='true')
+        self.expect_expr("b == BarBar", result_value='true')
 
         ## b is not a Foo
         value = frame.EvaluateExpression("b == Foo::FooBar")
diff --git a/lldb/test/API/commands/expression/timeout/TestCallWithTimeout.py b/lldb/test/API/commands/expression/timeout/TestCallWithTimeout.py
index 36ed7ce26de13..733c9921487eb 100644
--- a/lldb/test/API/commands/expression/timeout/TestCallWithTimeout.py
+++ b/lldb/test/API/commands/expression/timeout/TestCallWithTimeout.py
@@ -59,7 +59,7 @@ def test(self):
         options.SetTimeoutInMicroSeconds(1000000)
         value = frame.EvaluateExpression("wait_a_while (1000)", options)
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
 
         # Now do the same thingwith the command line command, and make sure it
         # works too.
@@ -77,4 +77,4 @@ def test(self):
         options.SetOneThreadTimeoutInMicroSeconds(500000)
         value = frame.EvaluateExpression("wait_a_while (1000)", options)
         self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
+        self.assertSuccess(value.GetError())
diff --git a/lldb/test/API/commands/expression/unwind_expression/TestUnwindExpression.py b/lldb/test/API/commands/expression/unwind_expression/TestUnwindExpression.py
index 3839f7d89235a..17424551fd834 100644
--- a/lldb/test/API/commands/expression/unwind_expression/TestUnwindExpression.py
+++ b/lldb/test/API/commands/expression/unwind_expression/TestUnwindExpression.py
@@ -45,9 +45,7 @@ def test_conditional_bktp(self):
 
         main_frame = self.thread.GetFrameAtIndex(0)
         val = main_frame.EvaluateExpression("second_function(47)", options)
-        self.assertTrue(
-            val.GetError().Success(),
-            "We did complete the execution.")
+        self.assertSuccess(val.GetError(), "We did complete the execution.")
         self.assertEquals(47, val.GetValueAsSigned())
 
 
@@ -92,8 +90,8 @@ def do_unwind_test(self, thread, bkpt, timeout):
 
         # Now unwind the expression, and make sure we got back to where we
         # started.
-        error = thread.UnwindInnermostExpression()
-        self.assertTrue(error.Success(), "We succeeded in unwinding")
+        self.assertSuccess(thread.UnwindInnermostExpression(),
+                "We succeeded in unwinding")
 
         cur_frame = thread.GetFrameAtIndex(0)
         self.assertTrue(
diff --git a/lldb/test/API/commands/expression/weak_symbols/TestWeakSymbols.py b/lldb/test/API/commands/expression/weak_symbols/TestWeakSymbols.py
index b58d838b14b70..b67b5b4efe0bd 100644
--- a/lldb/test/API/commands/expression/weak_symbols/TestWeakSymbols.py
+++ b/lldb/test/API/commands/expression/weak_symbols/TestWeakSymbols.py
@@ -39,7 +39,7 @@ def run_weak_var_check (self, weak_varname, present):
         # the bug that expressions with no result currently return False for Success()...
         expr = "if (&" + weak_varname + " != NULL) { present_weak_int = 10; } else { present_weak_int = 20;}; 10"
         result = self.frame.EvaluateExpression(expr)
-        self.assertTrue(result.GetError().Success(), "absent_weak_int expr failed: %s"%(result.GetError().GetCString()))
+        self.assertSuccess(result.GetError(), "absent_weak_int expr failed")
         self.assertEqual(value.GetValueAsSigned(), correct_value, "Didn't change present_weak_int correctly.")
         
     def do_test(self):
diff --git a/lldb/test/API/commands/frame/recognizer/TestFrameRecognizer.py b/lldb/test/API/commands/frame/recognizer/TestFrameRecognizer.py
index 04b940049496e..218c7e85aae74 100644
--- a/lldb/test/API/commands/frame/recognizer/TestFrameRecognizer.py
+++ b/lldb/test/API/commands/frame/recognizer/TestFrameRecognizer.py
@@ -144,3 +144,25 @@ def test_frame_recognizer_multi_symbol(self):
 
         self.expect("frame recognizer info 0",
                     substrs=['frame 0 is recognized by recognizer.MyFrameRecognizer'])
+
+    @no_debug_info_test
+    def test_frame_recognizer_delete_invalid_arg(self):
+        self.expect("frame recognizer delete a", error=True,
+                    substrs=["error: 'a' is not a valid recognizer id."])
+        self.expect("frame recognizer delete \"\"", error=True,
+                    substrs=["error: '' is not a valid recognizer id."])
+        self.expect("frame recognizer delete -1", error=True,
+                    substrs=["error: '-1' is not a valid recognizer id."])
+        self.expect("frame recognizer delete 4294967297", error=True,
+                    substrs=["error: '4294967297' is not a valid recognizer id."])
+
+    @no_debug_info_test
+    def test_frame_recognizer_info_invalid_arg(self):
+        self.expect("frame recognizer info a", error=True,
+                    substrs=["error: 'a' is not a valid frame index."])
+        self.expect("frame recognizer info \"\"", error=True,
+                    substrs=["error: '' is not a valid frame index."])
+        self.expect("frame recognizer info -1", error=True,
+                    substrs=["error: '-1' is not a valid frame index."])
+        self.expect("frame recognizer info 4294967297", error=True,
+                    substrs=["error: '4294967297' is not a valid frame index."])
diff --git a/lldb/test/API/commands/platform/file/close/TestPlatformFileClose.py b/lldb/test/API/commands/platform/file/close/TestPlatformFileClose.py
new file mode 100644
index 0000000000000..f440b674517e9
--- /dev/null
+++ b/lldb/test/API/commands/platform/file/close/TestPlatformFileClose.py
@@ -0,0 +1,15 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @no_debug_info_test
+    def test_file_close_invalid_arg(self):
+        self.expect("platform file close y", error=True,
+                    substrs=["'y' is not a valid file descriptor."])
+        self.expect("platform file close -1", error=True,
+                    substrs=["'-1' is not a valid file descriptor."])
diff --git a/lldb/test/API/commands/platform/file/read/TestPlatformFileRead.py b/lldb/test/API/commands/platform/file/read/TestPlatformFileRead.py
new file mode 100644
index 0000000000000..a1f963a0e065d
--- /dev/null
+++ b/lldb/test/API/commands/platform/file/read/TestPlatformFileRead.py
@@ -0,0 +1,16 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @no_debug_info_test
+    def test_file_close_invalid_arg(self):
+        self.expect("platform file read y", error=True,
+                    substrs=["'y' is not a valid file descriptor."])
+        # 'file read' takes an option, so this will be treated as an option.
+        self.expect("platform file read -1", error=True,
+                    substrs=["unknown or ambiguous option"])
diff --git a/lldb/test/API/commands/process/signal/Makefile b/lldb/test/API/commands/process/signal/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/commands/process/signal/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/commands/process/signal/TestProcessSignal.py b/lldb/test/API/commands/process/signal/TestProcessSignal.py
new file mode 100644
index 0000000000000..389732385c5f1
--- /dev/null
+++ b/lldb/test/API/commands/process/signal/TestProcessSignal.py
@@ -0,0 +1,17 @@
+import lldb
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+from lldbsuite.test.decorators import *
+
+class TestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @no_debug_info_test
+    def test_invalid_arg(self):
+        self.build()
+
+        lldbutil.run_to_source_breakpoint(self, '// break here', lldb.SBFileSpec("main.cpp"))
+        self.expect("process signal az", error=True, startstr="error: Invalid signal argument 'az'.")
+        self.expect("process signal 0x1ffffffff", error=True, startstr="error: Invalid signal argument '0x1ffffffff'.")
+        self.expect("process signal 0xffffffff", error=True, startstr="error: Invalid signal argument '0xffffffff'.")
diff --git a/lldb/test/API/commands/process/signal/main.cpp b/lldb/test/API/commands/process/signal/main.cpp
new file mode 100644
index 0000000000000..ba45ee316cd42
--- /dev/null
+++ b/lldb/test/API/commands/process/signal/main.cpp
@@ -0,0 +1,3 @@
+int main() {
+  return 0; // break here
+}
diff --git a/lldb/test/API/commands/register/register/register_command/TestRegisters.py b/lldb/test/API/commands/register/register/register_command/TestRegisters.py
index 01d2367c85502..b0931a7d6977c 100644
--- a/lldb/test/API/commands/register/register/register_command/TestRegisters.py
+++ b/lldb/test/API/commands/register/register/register_command/TestRegisters.py
@@ -235,12 +235,12 @@ def fp_special_purpose_register_read(self):
         error = lldb.SBError()
         reg_value_fstat_initial = value.GetValueAsUnsigned(error, 0)
 
-        self.assertTrue(error.Success(), "reading a value for fstat")
+        self.assertSuccess(error, "reading a value for fstat")
         value = currentFrame.FindValue("ftag", lldb.eValueTypeRegister)
         error = lldb.SBError()
         reg_value_ftag_initial = value.GetValueAsUnsigned(error, 0)
 
-        self.assertTrue(error.Success(), "reading a value for ftag")
+        self.assertSuccess(error, "reading a value for ftag")
         fstat_top_pointer_initial = (reg_value_fstat_initial & 0x3800) >> 11
 
         # Execute 'si' aka 'thread step-inst' instruction 5 times and with
@@ -292,7 +292,7 @@ def fp_register_write(self):
                 0, # launch flags
                 True, # stop at entry
                 error)
-        self.assertTrue(error.Success(), "Launch succeeds. Error is :" + str(error))
+        self.assertSuccess(error, "Launch succeeds")
 
         self.assertTrue(
             process.GetState() == lldb.eStateStopped,
diff --git a/lldb/test/API/commands/target/auto-install-main-executable/TestAutoInstallMainExecutable.py b/lldb/test/API/commands/target/auto-install-main-executable/TestAutoInstallMainExecutable.py
index 874a90853c2aa..ae5822750d694 100644
--- a/lldb/test/API/commands/target/auto-install-main-executable/TestAutoInstallMainExecutable.py
+++ b/lldb/test/API/commands/target/auto-install-main-executable/TestAutoInstallMainExecutable.py
@@ -57,47 +57,26 @@ def test_target_auto_install_main_executable(self):
 
         new_platform = lldb.SBPlatform(lldb.remote_platform.GetName())
         self.dbg.SetSelectedPlatform(new_platform)
-        interpreter = self.dbg.GetCommandInterpreter()
 
         connect_url = "%s://%s:%s" % (protocol, hostname, str(hostport+1))
 
-        command = "platform connect %s" % (connect_url)
-
-        result = lldb.SBCommandReturnObject()
-
         # Test the default setting.
-        interpreter.HandleCommand("settings show target.auto-install-main-executable", result)
-        self.assertTrue(
-            result.Succeeded() and
-            "target.auto-install-main-executable (boolean) = true" in result.GetOutput(),
-            "Default settings for target.auto-install-main-executable failed.: %s - %s" %
-            (result.GetOutput(), result.GetError()))
+        self.expect("settings show target.auto-install-main-executable",
+                substrs=["target.auto-install-main-executable (boolean) = true"],
+                msg="Default settings for target.auto-install-main-executable failed.")
 
         # Disable the auto install.
-        interpreter.HandleCommand("settings set target.auto-install-main-executable false", result)
-        interpreter.HandleCommand("settings show target.auto-install-main-executable", result)
-        self.assertTrue(
-            result.Succeeded() and
-            "target.auto-install-main-executable (boolean) = false" in result.GetOutput(),
-            "Default settings for target.auto-install-main-executable failed.: %s - %s" %
-            (result.GetOutput(), result.GetError()))
+        self.runCmd("settings set target.auto-install-main-executable false")
+        self.expect("settings show target.auto-install-main-executable", 
+            substrs=["target.auto-install-main-executable (boolean) = false"])
 
-        interpreter.HandleCommand("platform select %s"%configuration.lldb_platform_name, result)
-        interpreter.HandleCommand(command, result)
-
-        self.assertTrue(
-            result.Succeeded(),
-            "platform process connect failed: %s - %s" %
-            (result.GetOutput(),result.GetError()))
+        self.runCmd("platform select %s"%configuration.lldb_platform_name)
+        self.runCmd("platform connect %s" % (connect_url))
 
         # Create the target with the original file.
-        interpreter.HandleCommand("target create --remote-file %s %s "%
-                                        (os.path.join(working_dir,dest.GetFilename()), self.getBuildArtifact("a.out")),
-                                      result)
-        self.assertTrue(
-            result.Succeeded(),
-            "platform create failed: %s - %s" %
-            (result.GetOutput(),result.GetError()))
+        self.runCmd("target create --remote-file %s %s "%
+                                        (os.path.join(working_dir,dest.GetFilename()),
+                                            self.getBuildArtifact("a.out")))
 
         target = new_debugger.GetSelectedTarget()
         breakpoint = target.BreakpointCreateByName("main")
@@ -115,10 +94,7 @@ def test_target_auto_install_main_executable(self):
         frame = thread.GetFrameAtIndex(0)
         self.assertEqual(frame.GetFunction().GetName(), "main")
 
-        interpreter.HandleCommand("target variable build", result)
-        self.assertTrue(
-            result.Succeeded() and
-            '"device"' in result.GetOutput(),
-            "Magic in the binary is wrong: %s " % result.GetOutput())
+        self.expect("target variable build", substrs=['"device"'],
+                msg="Magic in the binary is wrong")
 
         process.Continue()
diff --git a/lldb/test/API/commands/target/modules/search-paths/insert/Makefile b/lldb/test/API/commands/target/modules/search-paths/insert/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/commands/target/modules/search-paths/insert/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/commands/target/modules/search-paths/insert/TestTargetModulesSearchpathsInsert.py b/lldb/test/API/commands/target/modules/search-paths/insert/TestTargetModulesSearchpathsInsert.py
new file mode 100644
index 0000000000000..169b32b974420
--- /dev/null
+++ b/lldb/test/API/commands/target/modules/search-paths/insert/TestTargetModulesSearchpathsInsert.py
@@ -0,0 +1,20 @@
+import lldb
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test.decorators import *
+import lldbsuite.test.lldbutil as lldbutil
+
+class TestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @no_debug_info_test
+    def test_invalid_arg(self):
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+
+        self.expect("target create %s" % (exe))
+        self.expect("target modules search-paths insert -1 a b", error=True,
+                    startstr="error: <index> parameter is not an integer: '-1'.")
+
+        self.expect("target modules search-paths insert abcdefx a b", error=True,
+                    startstr="error: <index> parameter is not an integer: 'abcdefx'.")
diff --git a/lldb/test/API/commands/target/modules/search-paths/insert/main.cpp b/lldb/test/API/commands/target/modules/search-paths/insert/main.cpp
new file mode 100644
index 0000000000000..76e8197013aab
--- /dev/null
+++ b/lldb/test/API/commands/target/modules/search-paths/insert/main.cpp
@@ -0,0 +1 @@
+int main() { return 0; }
diff --git a/lldb/test/API/commands/target/select/TestTargetSelect.py b/lldb/test/API/commands/target/select/TestTargetSelect.py
new file mode 100644
index 0000000000000..e53745ee21637
--- /dev/null
+++ b/lldb/test/API/commands/target/select/TestTargetSelect.py
@@ -0,0 +1,15 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @no_debug_info_test
+    def test_invalid_arg(self):
+        self.expect("target select -1", error=True,
+                    startstr="error: invalid index string value '-1'")
+        self.expect("target select abcdfx", error=True,
+                    startstr="error: invalid index string value 'abcdfx'")
diff --git a/lldb/test/API/commands/target/stop-hook/delete/TestTargetStopHookDelete.py b/lldb/test/API/commands/target/stop-hook/delete/TestTargetStopHookDelete.py
new file mode 100644
index 0000000000000..35b718b93f617
--- /dev/null
+++ b/lldb/test/API/commands/target/stop-hook/delete/TestTargetStopHookDelete.py
@@ -0,0 +1,15 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @no_debug_info_test
+    def test_invalid_arg(self):
+        self.expect("target stop-hook delete -1", error=True,
+                    startstr="error: invalid stop hook id: \"-1\".")
+        self.expect("target stop-hook delete abcdfx", error=True,
+                    startstr="error: invalid stop hook id: \"abcdfx\".")
diff --git a/lldb/test/API/commands/target/stop-hook/disable/TestTargetStopHookDisable.py b/lldb/test/API/commands/target/stop-hook/disable/TestTargetStopHookDisable.py
new file mode 100644
index 0000000000000..5e665a87c344d
--- /dev/null
+++ b/lldb/test/API/commands/target/stop-hook/disable/TestTargetStopHookDisable.py
@@ -0,0 +1,15 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @no_debug_info_test
+    def test_invalid_arg(self):
+        self.expect("target stop-hook disable -1", error=True,
+                    startstr="error: invalid stop hook id: \"-1\".")
+        self.expect("target stop-hook disable abcdfx", error=True,
+                    startstr="error: invalid stop hook id: \"abcdfx\".")
diff --git a/lldb/test/API/commands/target/stop-hook/enable/TestTargetStopHookEnable.py b/lldb/test/API/commands/target/stop-hook/enable/TestTargetStopHookEnable.py
new file mode 100644
index 0000000000000..5c1dcf4b2c00a
--- /dev/null
+++ b/lldb/test/API/commands/target/stop-hook/enable/TestTargetStopHookEnable.py
@@ -0,0 +1,15 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @no_debug_info_test
+    def test_invalid_arg(self):
+        self.expect("target stop-hook enable -1", error=True,
+                    startstr="error: invalid stop hook id: \"-1\".")
+        self.expect("target stop-hook enable abcdfx", error=True,
+                    startstr="error: invalid stop hook id: \"abcdfx\".")
diff --git a/lldb/test/API/commands/thread/select/Makefile b/lldb/test/API/commands/thread/select/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/commands/thread/select/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/commands/thread/select/TestThreadSelect.py b/lldb/test/API/commands/thread/select/TestThreadSelect.py
new file mode 100644
index 0000000000000..4650fae518ddd
--- /dev/null
+++ b/lldb/test/API/commands/thread/select/TestThreadSelect.py
@@ -0,0 +1,18 @@
+import lldb
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+from lldbsuite.test.decorators import *
+
+class TestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    def test_invalid_arg(self):
+        self.build()
+
+        lldbutil.run_to_source_breakpoint(self, '// break here', lldb.SBFileSpec("main.cpp"))
+
+        self.expect("thread select -1", error=True, startstr="error: Invalid thread index '-1'")
+        self.expect("thread select 0x1ffffffff", error=True, startstr="error: Invalid thread index '0x1ffffffff'")
+        # Parses but not a valid thread id.
+        self.expect("thread select 0xffffffff", error=True, startstr="error: invalid thread #0xffffffff.")
diff --git a/lldb/test/API/commands/thread/select/main.cpp b/lldb/test/API/commands/thread/select/main.cpp
new file mode 100644
index 0000000000000..ba45ee316cd42
--- /dev/null
+++ b/lldb/test/API/commands/thread/select/main.cpp
@@ -0,0 +1,3 @@
+int main() {
+  return 0; // break here
+}
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_names/TestBreakpointNames.py b/lldb/test/API/functionalities/breakpoint/breakpoint_names/TestBreakpointNames.py
index e421140e0182b..3c3c64adf7fd6 100644
--- a/lldb/test/API/functionalities/breakpoint/breakpoint_names/TestBreakpointNames.py
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_names/TestBreakpointNames.py
@@ -99,8 +99,8 @@ def do_check_names(self):
         other_bkpt_name = "_AnotherBreakpoint"
 
         # Add a name and make sure we match it:
-        success = bkpt.AddName(bkpt_name)
-        self.assertTrue(success, "We couldn't add a legal name to a breakpoint.")
+        success = bkpt.AddNameWithErrorHandling(bkpt_name)
+        self.assertSuccess(success, "We couldn't add a legal name to a breakpoint.")
 
         matches = bkpt.MatchesName(bkpt_name)
         self.assertTrue(matches, "We didn't match the name we just set")
@@ -113,7 +113,7 @@ def do_check_names(self):
         self.check_name_in_target(bkpt_name)
 
         # Add another name, make sure that works too:
-        bkpt.AddName(other_bkpt_name)
+        bkpt.AddNameWithErrorHandling(other_bkpt_name)
 
         matches = bkpt.MatchesName(bkpt_name)
         self.assertTrue(matches, "Adding a name means we didn't match the name we just set")
@@ -142,8 +142,8 @@ def do_check_illegal_names(self):
                      "CantHave-ADash",
                      "Cant Have Spaces"]
         for bad_name in bad_names:
-            success = bkpt.AddName(bad_name)
-            self.assertTrue(not success,"We allowed an illegal name: %s"%(bad_name))
+            success = bkpt.AddNameWithErrorHandling(bad_name)
+            self.assertTrue(success.Fail(), "We allowed an illegal name: %s"%(bad_name))
             bp_name = lldb.SBBreakpointName(self.target, bad_name)
             self.assertFalse(bp_name.IsValid(), "We made a breakpoint name with an illegal name: %s"%(bad_name));
 
@@ -164,8 +164,8 @@ def do_check_using_names(self):
         other_bkpt_name= "_AnotherBreakpoint"
 
         # Add a name and make sure we match it:
-        success = bkpt.AddName(bkpt_name)
-        self.assertTrue(success, "We couldn't add a legal name to a breakpoint.")
+        success = bkpt.AddNameWithErrorHandling(bkpt_name)
+        self.assertSuccess(success, "We couldn't add a legal name to a breakpoint.")
 
         bkpts = lldb.SBBreakpointList(self.target)
         self.target.FindBreakpointsByName(bkpt_name, bkpts)
@@ -243,8 +243,8 @@ def do_check_configuring_names(self):
 
         # Now add this name to a breakpoint, and make sure it gets configured properly
         bkpt = self.target.BreakpointCreateByLocation(self.main_file_spec, 10)
-        success = bkpt.AddName(self.bp_name_string)
-        self.assertTrue(success, "Couldn't add this name to the breakpoint")
+        success = bkpt.AddNameWithErrorHandling(self.bp_name_string)
+        self.assertSuccess(success, "Couldn't add this name to the breakpoint")
         self.check_option_values(bkpt)
 
         # Now make a name from this breakpoint, and make sure the new name is properly configured:
@@ -317,8 +317,8 @@ def check_permission_results(self, bp_name):
         unprotected_bkpt = self.target.BreakpointCreateByLocation(self.main_file_spec, 10)
         unprotected_id = unprotected_bkpt.GetID()
 
-        success = protected_bkpt.AddName(self.bp_name_string)
-        self.assertTrue(success, "Couldn't add this name to the breakpoint")
+        success = protected_bkpt.AddNameWithErrorHandling(self.bp_name_string)
+        self.assertSuccess(success, "Couldn't add this name to the breakpoint")
 
         self.target.DisableAllBreakpoints()
         self.assertEqual(protected_bkpt.IsEnabled(), True, "Didnt' keep breakpoint from being disabled")
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
index a4a1d9effbe19..bca7b278631fb 100644
--- a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
@@ -36,7 +36,7 @@ def test_hw_break_set_disable_multi_thread_linux(self):
     # architectures.
     @skipUnlessDarwin
     @skipIfOutOfTreeDebugserver
-    @skipIfDarwinEmbedded
+    @expectedFailureAll(archs=["arm64"])
     def test_hw_break_set_delete_multi_thread_macos(self):
         self.build()
         self.setTearDownCleanup()
@@ -46,7 +46,7 @@ def test_hw_break_set_delete_multi_thread_macos(self):
     # architectures.
     @skipUnlessDarwin
     @skipIfOutOfTreeDebugserver
-    @skipIfDarwinEmbedded
+    @expectedFailureAll(archs=["arm64"])
     def test_hw_break_set_disable_multi_thread_macos(self):
         self.build()
         self.setTearDownCleanup()
diff --git a/lldb/test/API/functionalities/completion/TestCompletion.py b/lldb/test/API/functionalities/completion/TestCompletion.py
index 4a548ad770838..0cec206025d1f 100644
--- a/lldb/test/API/functionalities/completion/TestCompletion.py
+++ b/lldb/test/API/functionalities/completion/TestCompletion.py
@@ -415,6 +415,17 @@ def test_completion_description_command_options(self):
         self.check_completion_with_desc("breakpoint set --Z", [
         ])
 
+    def test_frame_select(self):
+        self.build()
+        self.main_source_spec = lldb.SBFileSpec("main.cpp")
+        lldbutil.run_to_source_breakpoint(self, '// Break here', self.main_source_spec)
+
+        self.complete_from_to('frame select ', ['0'])
+    
+    def test_frame_recognizer_delete(self):
+        self.runCmd("frame recognizer add -l py_class -s module_name -n recognizer_name")
+        self.check_completion_with_desc('frame recognizer delete ', [['0', 'py_class, module module_name, symbol recognizer_name']])
+
     @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24489")
     def test_symbol_name(self):
         self.build()
diff --git a/lldb/test/API/functionalities/data-formatter/boolreference/TestFormattersBoolRefPtr.py b/lldb/test/API/functionalities/data-formatter/boolreference/TestFormattersBoolRefPtr.py
index 25ecdef5948c4..7435409939e44 100644
--- a/lldb/test/API/functionalities/data-formatter/boolreference/TestFormattersBoolRefPtr.py
+++ b/lldb/test/API/functionalities/data-formatter/boolreference/TestFormattersBoolRefPtr.py
@@ -50,14 +50,14 @@ def cleanup():
         # Execute the cleanup function during test case tear down.
         self.addTearDownHook(cleanup)
 
-        isiOS = (lldbplatformutil.getPlatform() == 'ios' or lldbplatformutil.getPlatform() == 'watchos')
+        isArm = 'arm' in self.getArchitecture()
 
         # Now check that we use the right summary for BOOL&
         self.expect('frame variable yes_ref',
                     substrs=['YES'])
         self.expect('frame variable no_ref',
                     substrs=['NO'])
-        if not(isiOS):
+        if not(isArm):
             self.expect('frame variable unset_ref', substrs=['12'])
 
         # Now check that we use the right summary for BOOL*
@@ -65,7 +65,7 @@ def cleanup():
                     substrs=['YES'])
         self.expect('frame variable no_ptr',
                     substrs=['NO'])
-        if not(isiOS):
+        if not(isArm):
             self.expect('frame variable unset_ptr', substrs=['12'])
 
         # Now check that we use the right summary for BOOL
@@ -73,5 +73,5 @@ def cleanup():
                     substrs=['YES'])
         self.expect('frame variable no',
                     substrs=['NO'])
-        if not(isiOS):
+        if not(isArm):
             self.expect('frame variable unset', substrs=['12'])
diff --git a/lldb/test/API/functionalities/data-formatter/builtin-formats/TestBuiltinFormats.py b/lldb/test/API/functionalities/data-formatter/builtin-formats/TestBuiltinFormats.py
index 9fefae6bbf5c6..741acd0431bd0 100644
--- a/lldb/test/API/functionalities/data-formatter/builtin-formats/TestBuiltinFormats.py
+++ b/lldb/test/API/functionalities/data-formatter/builtin-formats/TestBuiltinFormats.py
@@ -42,6 +42,9 @@ def test(self):
         self.assertIn("= 0\n", self.getFormatted("float", "0"))
         self.assertIn("= 2\n", self.getFormatted("float", "0x40000000"))
         self.assertIn("= NaN\n", self.getFormatted("float", "-1"))
+        # Checks the float16 code.
+        self.assertIn("= 2\n", self.getFormatted("float", "(__UINT16_TYPE__)0x4000"))
+        self.assertIn("= error: unsupported byte size (1) for float format\n", self.getFormatted("float", "'a'"))
 
         # enumeration
         self.assertIn("= 0\n", self.getFormatted("enumeration", "0"))
@@ -59,6 +62,13 @@ def test(self):
 
         # octal
         self.assertIn("= 04553207\n", self.getFormatted("octal", "1234567"))
+        self.assertIn("= 0221505317046536757\n", self.getFormatted("octal", "(__uint128_t)0x123456789ABDEFull"))
+
+        # complex float
+        self.assertIn("= error: unsupported byte size (1) for complex float format\n", self.getFormatted("complex float", "'a'"))
+
+        # complex integer
+        self.assertIn("= error: unsupported byte size (1) for complex integer format\n", self.getFormatted("complex integer", "'a'"))
 
         # hex
         self.assertIn("= 0x00abc123\n", self.getFormatted("hex", "0xABC123"))
@@ -79,28 +89,47 @@ def test(self):
         self.assertIn(" = 0b10010001101000101011001111000\n", self.getFormatted("binary", "(__uint128_t)0x12345678ll"))
 
         # Different character arrays.
+        # FIXME: Passing a 'const char *' will ignore any given format,
         self.assertIn('= " \\U0000001b\\a\\b\\f\\n\\r\\t\\vaA09"\n', self.getFormatted("character array", "cstring"))
-        self.assertIn('= " \\U0000001b\\a\\b\\f\\n\\r\\t\\vaA09"\n', self.getFormatted("character", "cstring"))
         self.assertIn('= " \\U0000001b\\a\\b\\f\\n\\r\\t\\vaA09"\n', self.getFormatted("c-string", "cstring"))
-        self.assertIn('= " \\U0000001b\\a\\b\\f\\n\\r\\t\\vaA09"\n', self.getFormatted("printable character", "cstring"))
-        self.assertIn('= " \\U0000001b\\a\\b\\f\\n\\r\\t\\vaA09"\n', self.getFormatted("OSType", "cstring"))
-        self.assertIn('= " \\U0000001b\\a\\b\\f\\n\\r\\t\\vaA09"\n', self.getFormatted("unicode8", "cstring"))
-
-        self.assertIn('= \\0\\0\\0\\0\\0\\0\\0\\0\n', self.getFormatted("character array", "(__UINT64_TYPE__)0"))
-        self.assertIn('= \\0\\0\\0\\0\\0\\0\\0\\0\n', self.getFormatted("character", "(__UINT64_TYPE__)0"))
+        self.assertIn(' = " \\e\\a\\b\\f\\n\\r\\t\\vaA09" " \\U0000001b\\a\\b\\f\\n\\r\\t\\vaA09"\n',
+                      self.getFormatted("c-string", "(char *)cstring"))
         self.assertIn('=\n', self.getFormatted("c-string", "(__UINT64_TYPE__)0"))
-        self.assertIn('= ........\n', self.getFormatted("printable character", "(__UINT64_TYPE__)0"))
-        self.assertIn('= \'\\0\\0\\0\\0\\0\\0\\0\\0\'\n', self.getFormatted("OSType", "(__UINT64_TYPE__)0"))
-        # FIXME: This seems wrong.
-        self.assertIn('= 0x0000000000000000\n', self.getFormatted("unicode8", "(__UINT64_TYPE__)0"))
-
-        self.assertIn('= xV4\\x12\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\n', self.getFormatted("character array", "(__uint128_t)0x12345678ll"))
-        self.assertIn('= xV4\\x12\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\n', self.getFormatted("character", "(__uint128_t)0x12345678ll"))
-        self.assertIn('=\n', self.getFormatted("c-string", "(__uint128_t)0x12345678ll"))
-        self.assertIn('= xV4.............\n', self.getFormatted("printable character", "(__uint128_t)0x12345678ll"))
-        self.assertIn('= 0x00000000000000000000000012345678\n', self.getFormatted("OSType", "(__uint128_t)0x12345678ll"))
-        # FIXME: This seems wrong.
-        self.assertIn('= 0x00000000000000000000000012345678\n', self.getFormatted("unicode8", "(__uint128_t)0x12345678ll"))
+
+        # Build a uint128_t that contains a series of characters in each byte.
+        # First 8 byte of the uint128_t.
+        cstring_chars1 = " \a\b\f\n\r\t\v"
+        # Last 8 byte of the uint128_t.
+        cstring_chars2 = "AZaz09\033\0"
+
+        # Build a uint128_t value with the hex encoded characters.
+        string_expr = "((__uint128_t)0x"
+        for c in cstring_chars1:
+             string_expr += format(ord(c), "x").zfill(2)
+        string_expr += "ull << 64) | (__uint128_t)0x"
+        for c in cstring_chars2:
+             string_expr += format(ord(c), "x").zfill(2)
+        string_expr += "ull"
+
+        # Try to print that uint128_t with the different char formatters.
+        self.assertIn('= \\0\\e90zaZA\\v\\t\\r\\n\\f\\b\\a \n', self.getFormatted("character array", string_expr))
+        self.assertIn('= \\0\\e90zaZA\\v\\t\\r\\n\\f\\b\\a \n', self.getFormatted("character", string_expr))
+        self.assertIn('= ..90zaZA....... \n', self.getFormatted("printable character", string_expr))
+        # FIXME: This should probably print the characters in the uint128_t.
+        self.assertIn('= 0x2007080c0a0d090b415a617a30391b00\n', self.getFormatted("unicode8", string_expr))
+
+        # OSType
+        ostype_expr = "(__UINT64_TYPE__)0x"
+        for c in cstring_chars1:
+             ostype_expr += format(ord(c), "x").zfill(2)
+        self.assertIn("= ' \\a\\b\\f\\n\\r\\t\\v'\n", self.getFormatted("OSType", ostype_expr))
+
+        ostype_expr = "(__UINT64_TYPE__)0x"
+        for c in cstring_chars2:
+             ostype_expr += format(ord(c), "x").zfill(2)
+        self.assertIn("= 'AZaz09\\e\\0'\n", self.getFormatted("OSType", ostype_expr))
+
+        self.assertIn('= 0x2007080c0a0d090b415a617a30391b00\n', self.getFormatted("OSType", string_expr))
 
         # bytes
         self.assertIn('= " \\U0000001b\\a\\b\\f\\n\\r\\t\\vaA09"\n', self.getFormatted("bytes", "cstring"))
@@ -137,6 +166,11 @@ def test(self):
         # There is not int128_t[] style, so this only tests uint128_t[].
         self.assertIn("= {0x00000000000000000123456789abcdef}\n", self.getFormatted("uint128_t[]", "__uint128_t i = 0x123456789ABCDEF; i"))
 
+        # Different fixed-width float type arrays.
+        self.assertIn("{2 2}\n", self.getFormatted("float16[]", "0x40004000"))
+        self.assertIn("{2 2}\n", self.getFormatted("float32[]", "0x4000000040000000ll"))
+        self.assertIn("{2 0}\n", self.getFormatted("float64[]", "__uint128_t i = 0x4000000000000000ll; i"))
+
         # Invalid format string
         self.expect("expr --format invalid_format_string -- 1", error=True,
                     substrs=["error: Invalid format character or name 'invalid_format_string'. Valid values are:"])
@@ -144,10 +178,12 @@ def test(self):
     # Extends to host target pointer width.
     @skipIf(archs=no_match(['x86_64']))
     @no_debug_info_test
-    def test_instruction(self):
+    def test_pointer(self):
         # pointer
         self.assertIn("= 0x000000000012d687\n", self.getFormatted("pointer", "1234567"))
         self.assertIn("= 0x0000000000000000\n", self.getFormatted("pointer", "0"))
+        # FIXME: Just ignores the input value as it's not pointer sized.
+        self.assertIn("= 0x0000000000000000\n", self.getFormatted("pointer", "'a'"))
 
     # Depends on the host target for decoding.
     @skipIf(archs=no_match(['x86_64']))
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSDate.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSDate.py
index 5cfaa892bb626..61394c05f5d56 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSDate.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSDate.py
@@ -43,6 +43,15 @@ def nsdate_data_formatter_commands(self):
         self.expect('frame variable date4_abs', substrs=['1970'])
         self.expect('frame variable date5_abs', substrs=[now_year])
 
+        # Check that LLDB always follow's NSDate's rounding behavior (which
+        # is always rounding down).
+        self.expect_expr("date_1970_minus_06", result_summary="1969-12-31 23:59:59 UTC")
+        self.expect_expr("date_1970_minus_05", result_summary="1969-12-31 23:59:59 UTC")
+        self.expect_expr("date_1970_minus_04", result_summary="1969-12-31 23:59:59 UTC")
+        self.expect_expr("date_1970_plus_06", result_summary="1970-01-01 00:00:00 UTC")
+        self.expect_expr("date_1970_plus_05", result_summary="1970-01-01 00:00:00 UTC")
+        self.expect_expr("date_1970_plus_04", result_summary="1970-01-01 00:00:00 UTC")
+
         self.expect('frame variable cupertino home europe',
                     substrs=['@"America/Los_Angeles"',
                              '@"Europe/Rome"',
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-objc/main.m b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/main.m
index df96a5e59b5b5..a44a7837f7713 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-objc/main.m
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/main.m
@@ -655,6 +655,14 @@ int main(int argc, const char *argv[]) {
       [NSDate dateWithTimeIntervalSinceReferenceDate:
                   floor([[NSDate date] timeIntervalSinceReferenceDate])];
 
+  NSDate *date_1970_minus_06 = [NSDate dateWithTimeIntervalSince1970:-0.6];
+  NSDate *date_1970_minus_05 = [NSDate dateWithTimeIntervalSince1970:-0.5];
+  NSDate *date_1970_minus_04 = [NSDate dateWithTimeIntervalSince1970:-0.4];
+
+  NSDate *date_1970_plus_06 = [NSDate dateWithTimeIntervalSince1970:0.6];
+  NSDate *date_1970_plus_05 = [NSDate dateWithTimeIntervalSince1970:0.5];
+  NSDate *date_1970_plus_04 = [NSDate dateWithTimeIntervalSince1970:0.4];
+
   CFAbsoluteTime date1_abs = CFDateGetAbsoluteTime(date1);
   CFAbsoluteTime date2_abs = CFDateGetAbsoluteTime(date2);
   CFAbsoluteTime date3_abs = CFDateGetAbsoluteTime(date3);
diff --git a/lldb/test/API/functionalities/limit-debug-info/Makefile b/lldb/test/API/functionalities/limit-debug-info/Makefile
new file mode 100644
index 0000000000000..874b3a15e0fee
--- /dev/null
+++ b/lldb/test/API/functionalities/limit-debug-info/Makefile
@@ -0,0 +1,27 @@
+CFLAGS_EXTRAS = $(LIMIT_DEBUG_INFO_FLAGS)
+LD_EXTRAS = -L. -lone -ltwo
+CXX_SOURCES = main.cpp
+
+ONE_CXXFLAGS = $(LIMIT_DEBUG_INFO_FLAGS)
+ifdef STRIP_ONE
+  ONE_CXXFLAGS += -g0
+endif
+
+TWO_CXXFLAGS = $(LIMIT_DEBUG_INFO_FLAGS)
+ifdef STRIP_TWO
+  TWO_CXXFLAGS += -g0
+endif
+
+include Makefile.rules
+
+a.out: libone libtwo
+
+libone:
+	$(MAKE) -f $(MAKEFILE_RULES) \
+	  DYLIB_ONLY=YES DYLIB_CXX_SOURCES=one.cpp DYLIB_NAME=one \
+	  CFLAGS_EXTRAS="$(ONE_CXXFLAGS)"
+
+libtwo: libone
+	$(MAKE) -f $(MAKEFILE_RULES) \
+	  DYLIB_ONLY=YES DYLIB_CXX_SOURCES=two.cpp DYLIB_NAME=two \
+	  CFLAGS_EXTRAS="$(TWO_CXXFLAGS)" LD_EXTRAS="-L. -lone"
diff --git a/lldb/test/API/functionalities/limit-debug-info/TestLimitDebugInfo.py b/lldb/test/API/functionalities/limit-debug-info/TestLimitDebugInfo.py
new file mode 100644
index 0000000000000..9408ad6eee1d1
--- /dev/null
+++ b/lldb/test/API/functionalities/limit-debug-info/TestLimitDebugInfo.py
@@ -0,0 +1,128 @@
+"""
+Test completing types using information from other shared libraries.
+"""
+
+import os
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class LimitDebugInfoTestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    def _check_type(self, target, name):
+        exe = target.FindModule(lldb.SBFileSpec("a.out"))
+        type_ = exe.FindFirstType(name)
+        self.trace("type_: %s"%type_)
+        self.assertTrue(type_)
+        base = type_.GetDirectBaseClassAtIndex(0).GetType()
+        self.trace("base:%s"%base)
+        self.assertTrue(base)
+        self.assertEquals(base.GetNumberOfFields(), 0)
+
+    def _check_debug_info_is_limited(self, target):
+        # Without other shared libraries we should only see the member declared
+        # in the derived class. This serves as a sanity check that we are truly
+        # building with limited debug info.
+        self._check_type(target, "InheritsFromOne")
+        self._check_type(target, "InheritsFromTwo")
+
+    @skipIf(bugnumber="pr46284", debug_info="gmodules")
+    @skipIfWindows # Clang emits type info even with -flimit-debug-info
+    def test_one_and_two_debug(self):
+        self.build()
+        target = self.dbg.CreateTarget(self.getBuildArtifact("a.out"))
+
+        self._check_debug_info_is_limited(target)
+
+        self.registerSharedLibrariesWithTarget(target, ["one", "two"])
+
+        # But when other shared libraries are loaded, we should be able to see
+        # all members.
+        self.expect_expr("inherits_from_one.member", result_value="47")
+        self.expect_expr("inherits_from_one.one", result_value="142")
+        self.expect_expr("inherits_from_two.member", result_value="47")
+        self.expect_expr("inherits_from_two.one", result_value="142")
+        self.expect_expr("inherits_from_two.two", result_value="242")
+
+        self.expect_expr("one_as_member.member", result_value="47")
+        self.expect_expr("one_as_member.one.member", result_value="147")
+        self.expect_expr("two_as_member.member", result_value="47")
+        self.expect_expr("two_as_member.two.one.member", result_value="147")
+        self.expect_expr("two_as_member.two.member", result_value="247")
+
+        self.expect_expr("array_of_one[2].member", result_value="174")
+        self.expect_expr("array_of_two[2].one[2].member", result_value="174")
+        self.expect_expr("array_of_two[2].member", result_value="274")
+
+    @skipIf(bugnumber="pr46284", debug_info="gmodules")
+    @skipIfWindows # Clang emits type info even with -flimit-debug-info
+    def test_two_debug(self):
+        self.build(dictionary=dict(STRIP_ONE="1"))
+        target = self.dbg.CreateTarget(self.getBuildArtifact("a.out"))
+
+        self._check_debug_info_is_limited(target)
+
+        self.registerSharedLibrariesWithTarget(target, ["one", "two"])
+
+        # This time, we should only see the members from the second library.
+        self.expect_expr("inherits_from_one.member", result_value="47")
+        self.expect("expr inherits_from_one.one", error=True,
+            substrs=["no member named 'one' in 'InheritsFromOne'"])
+        self.expect_expr("inherits_from_two.member", result_value="47")
+        self.expect("expr inherits_from_two.one", error=True,
+            substrs=["no member named 'one' in 'InheritsFromTwo'"])
+        self.expect_expr("inherits_from_two.two", result_value="242")
+
+        self.expect_expr("one_as_member.member", result_value="47")
+        self.expect("expr one_as_member.one.member", error=True,
+                substrs=["no member named 'member' in 'member::One'"])
+        self.expect_expr("two_as_member.member", result_value="47")
+        self.expect("expr two_as_member.two.one.member", error=True,
+                substrs=["no member named 'member' in 'member::One'"])
+        self.expect_expr("two_as_member.two.member", result_value="247")
+
+        self.expect("expr array_of_one[2].member", error=True,
+                substrs=["no member named 'member' in 'array::One'"])
+        self.expect("expr array_of_two[2].one[2].member", error=True,
+                substrs=["no member named 'member' in 'array::One'"])
+        self.expect_expr("array_of_two[2].member", result_value="274")
+
+    @skipIf(bugnumber="pr46284", debug_info="gmodules")
+    @skipIfWindows # Clang emits type info even with -flimit-debug-info
+    def test_one_debug(self):
+        self.build(dictionary=dict(STRIP_TWO="1"))
+        target = self.dbg.CreateTarget(self.getBuildArtifact("a.out"))
+
+        self._check_debug_info_is_limited(target)
+
+        self.registerSharedLibrariesWithTarget(target, ["one", "two"])
+
+        # In this case we should only see the members from the second library.
+        # Note that we cannot see inherits_from_two.one because without debug
+        # info for "Two", we cannot determine that it in fact inherits from
+        # "One".
+        self.expect_expr("inherits_from_one.member", result_value="47")
+        self.expect_expr("inherits_from_one.one", result_value="142")
+        self.expect_expr("inherits_from_two.member", result_value="47")
+        self.expect("expr inherits_from_two.one", error=True,
+            substrs=["no member named 'one' in 'InheritsFromTwo'"])
+        self.expect("expr inherits_from_two.two", error=True,
+            substrs=["no member named 'two' in 'InheritsFromTwo'"])
+
+        self.expect_expr("one_as_member.member", result_value="47")
+        self.expect_expr("one_as_member.one.member", result_value="147")
+        self.expect_expr("two_as_member.member", result_value="47")
+        self.expect("expr two_as_member.two.one.member", error=True,
+                substrs=["no member named 'one' in 'member::Two'"])
+        self.expect("expr two_as_member.two.member", error=True,
+                substrs=["no member named 'member' in 'member::Two'"])
+
+        self.expect_expr("array_of_one[2].member", result_value="174")
+        self.expect("expr array_of_two[2].one[2].member", error=True,
+                substrs=["no member named 'one' in 'array::Two'"])
+        self.expect("expr array_of_two[2].member", error=True,
+                substrs=["no member named 'member' in 'array::Two'"])
diff --git a/lldb/test/API/functionalities/limit-debug-info/foo.cpp b/lldb/test/API/functionalities/limit-debug-info/foo.cpp
new file mode 100644
index 0000000000000..69f2f7996f32a
--- /dev/null
+++ b/lldb/test/API/functionalities/limit-debug-info/foo.cpp
@@ -0,0 +1,6 @@
+struct A {
+  int a = 47;
+  virtual ~A();
+};
+
+A::~A() = default;
diff --git a/lldb/test/API/functionalities/limit-debug-info/main.cpp b/lldb/test/API/functionalities/limit-debug-info/main.cpp
new file mode 100644
index 0000000000000..0a25de13d9fb2
--- /dev/null
+++ b/lldb/test/API/functionalities/limit-debug-info/main.cpp
@@ -0,0 +1,28 @@
+#include "onetwo.h"
+
+struct InheritsFromOne : One {
+  constexpr InheritsFromOne() = default;
+  int member = 47;
+} inherits_from_one;
+
+struct InheritsFromTwo : Two {
+  constexpr InheritsFromTwo() = default;
+  int member = 47;
+} inherits_from_two;
+
+struct OneAsMember {
+  constexpr OneAsMember() = default;
+  member::One one;
+  int member = 47;
+} one_as_member;
+
+struct TwoAsMember {
+  constexpr TwoAsMember() = default;
+  member::Two two;
+  int member = 47;
+} two_as_member;
+
+array::One array_of_one[3];
+array::Two array_of_two[3];
+
+int main() { return 0; }
diff --git a/lldb/test/API/functionalities/limit-debug-info/one.cpp b/lldb/test/API/functionalities/limit-debug-info/one.cpp
new file mode 100644
index 0000000000000..c1eb6310dd679
--- /dev/null
+++ b/lldb/test/API/functionalities/limit-debug-info/one.cpp
@@ -0,0 +1,5 @@
+#include "onetwo.h"
+
+One::~One() = default;
+member::One::~One() = default;
+array::One::~One() = default;
diff --git a/lldb/test/API/functionalities/limit-debug-info/onetwo.h b/lldb/test/API/functionalities/limit-debug-info/onetwo.h
new file mode 100644
index 0000000000000..67609dd7ff61c
--- /dev/null
+++ b/lldb/test/API/functionalities/limit-debug-info/onetwo.h
@@ -0,0 +1,41 @@
+struct One {
+  int one = 142;
+  constexpr One() = default;
+  virtual ~One();
+};
+
+struct Two : One {
+  int two = 242;
+  constexpr Two() = default;
+  ~Two() override;
+};
+
+namespace member {
+struct One {
+  int member = 147;
+  constexpr One() = default;
+  virtual ~One();
+};
+
+struct Two {
+  One one;
+  int member = 247;
+  constexpr Two() = default;
+  virtual ~Two();
+};
+} // namespace member
+
+namespace array {
+struct One {
+  int member = 174;
+  constexpr One() = default;
+  virtual ~One();
+};
+
+struct Two {
+  One one[3];
+  int member = 274;
+  constexpr Two() = default;
+  virtual ~Two();
+};
+} // namespace array
diff --git a/lldb/test/API/functionalities/limit-debug-info/two.cpp b/lldb/test/API/functionalities/limit-debug-info/two.cpp
new file mode 100644
index 0000000000000..04683da6e9cce
--- /dev/null
+++ b/lldb/test/API/functionalities/limit-debug-info/two.cpp
@@ -0,0 +1,5 @@
+#include "onetwo.h"
+
+Two::~Two() = default;
+member::Two::~Two() = default;
+array::Two::~Two() = default;
diff --git a/lldb/test/API/functionalities/return-value/TestReturnValue.py b/lldb/test/API/functionalities/return-value/TestReturnValue.py
index 24aa504a593c6..08293ca48b95b 100644
--- a/lldb/test/API/functionalities/return-value/TestReturnValue.py
+++ b/lldb/test/API/functionalities/return-value/TestReturnValue.py
@@ -165,7 +165,7 @@ def test_with_python(self):
         archs=["i386"])
     @expectedFailureAll(compiler=["gcc"], archs=["x86_64", "i386"])
     @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24778")
-    @skipIfDarwinEmbedded # <rdar://problem/33976032> ABIMacOSX_arm64 doesn't get structs this big correctly
+    @expectedFailureDarwin(archs=["arm64"]) # <rdar://problem/33976032> ABIMacOSX_arm64 doesn't get structs this big correctly
     def test_vector_values(self):
         self.build()
         exe = self.getBuildArtifact("a.out")
diff --git a/lldb/test/API/functionalities/source-map/a.yaml b/lldb/test/API/functionalities/source-map/a.yaml
index 08fd9895d9125..cf2fe7b75c4e2 100644
--- a/lldb/test/API/functionalities/source-map/a.yaml
+++ b/lldb/test/API/functionalities/source-map/a.yaml
@@ -296,8 +296,7 @@ DWARF:
         - Attribute:       DW_AT_type
           Form:            DW_FORM_ref4
   debug_info:      
-    - Length:          
-        TotalLength:     122
+    - Length:          122
       Version:         4
       AbbrOffset:      0
       AddrSize:        8
diff --git a/lldb/test/API/functionalities/step-avoids-no-debug/TestStepNoDebug.py b/lldb/test/API/functionalities/step-avoids-no-debug/TestStepNoDebug.py
index e166848f853ff..92036b2f215a3 100644
--- a/lldb/test/API/functionalities/step-avoids-no-debug/TestStepNoDebug.py
+++ b/lldb/test/API/functionalities/step-avoids-no-debug/TestStepNoDebug.py
@@ -48,7 +48,7 @@ def test_step_over_with_python(self):
             "3.9"],
         archs=["i386"],
         bugnumber="llvm.org/pr28549")
-    @expectedFailureAll(oslist=["ios", "tvos", "bridgeos"], bugnumber="<rdar://problem/34026777>")  # lldb doesn't step past last source line in function on arm64
+    @expectedFailureAll(archs=["arm64"], bugnumber="<rdar://problem/34026777>")  # lldb doesn't step past last source line in function on arm64
     @expectedFailureAll(archs=["aarch64"], oslist=["linux"],
                         bugnumber="llvm.org/pr44057")
     def test_step_in_with_python(self):
diff --git a/lldb/test/API/functionalities/target_var/Makefile b/lldb/test/API/functionalities/target_var/Makefile
index e51de3a02a46c..10495940055b6 100644
--- a/lldb/test/API/functionalities/target_var/Makefile
+++ b/lldb/test/API/functionalities/target_var/Makefile
@@ -1,5 +1,3 @@
-include Makefile.rules
+C_SOURCES := main.c
 
-a.out: globals.ll
-	$(CC) $(CFLAGS) -g -c $^ -o globals.o
-	$(LD) $(LDFLAGS) -g globals.o -o $@
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/target_var/TestTargetVar.py b/lldb/test/API/functionalities/target_var/TestTargetVar.py
index f8c2a69014720..4eee0e61816e4 100644
--- a/lldb/test/API/functionalities/target_var/TestTargetVar.py
+++ b/lldb/test/API/functionalities/target_var/TestTargetVar.py
@@ -20,3 +20,5 @@ def testTargetVarExpr(self):
         self.build()
         lldbutil.run_to_name_breakpoint(self, 'main')
         self.expect("target variable i", substrs=['i', '42'])
+        self.expect("target variable var", patterns=['\(incomplete \*\) var = 0[xX](0)*dead'])
+        self.expect("target variable var[0]", error=True, substrs=["can't find global variable 'var[0]'"])
diff --git a/lldb/test/API/functionalities/target_var/globals.c b/lldb/test/API/functionalities/target_var/globals.c
deleted file mode 100644
index 2661928496416..0000000000000
--- a/lldb/test/API/functionalities/target_var/globals.c
+++ /dev/null
@@ -1,6 +0,0 @@
-int i = 42;
-int *p = &i;
-
-int main() {
-  return *p;
-}
diff --git a/lldb/test/API/functionalities/target_var/globals.ll b/lldb/test/API/functionalities/target_var/globals.ll
deleted file mode 100644
index 192d4e126981f..0000000000000
--- a/lldb/test/API/functionalities/target_var/globals.ll
+++ /dev/null
@@ -1,42 +0,0 @@
-source_filename = "globals.c"
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-apple-macosx10.14.0"
-
-@i = global i32 42, align 4
-@p = global i32* @i, align 8, !dbg !0, !dbg !6
-
-; Function Attrs: noinline nounwind optnone ssp uwtable
-define i32 @main() #0 !dbg !15 {
-entry:
-  %retval = alloca i32, align 4
-  store i32 0, i32* %retval, align 4
-  %0 = load i32*, i32** @p, align 8, !dbg !18
-  %1 = load i32, i32* %0, align 4, !dbg !18
-  ret i32 %1, !dbg !18
-}
-
-attributes #0 = { noinline nounwind optnone ssp uwtable }
-
-!llvm.dbg.cu = !{!2}
-!llvm.module.flags = !{!10, !11, !12, !13}
-!llvm.ident = !{!14}
-
-!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression(DW_OP_deref))
-!1 = distinct !DIGlobalVariable(name: "i", scope: !2, file: !3, line: 1, type: !9, isLocal: false, isDefinition: true)
-!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, emissionKind: FullDebug, globals: !5)
-!3 = !DIFile(filename: "globals.c", directory: "/")
-!4 = !{}
-!5 = !{!0, !6}
-!6 = !DIGlobalVariableExpression(var: !7, expr: !DIExpression())
-!7 = distinct !DIGlobalVariable(name: "p", scope: !2, file: !3, line: 2, type: !8, isLocal: false, isDefinition: true)
-!8 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !9, size: 64)
-!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!10 = !{i32 2, !"Dwarf Version", i32 4}
-!11 = !{i32 2, !"Debug Info Version", i32 3}
-!12 = !{i32 1, !"wchar_size", i32 4}
-!13 = !{i32 7, !"PIC Level", i32 2}
-!14 = !{!"clang version 8.0.0 (trunk 340838) (llvm/trunk 340843)"}
-!15 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 4, type: !16, isLocal: false, isDefinition: true, scopeLine: 4, isOptimized: false, unit: !2, retainedNodes: !4)
-!16 = !DISubroutineType(types: !17)
-!17 = !{!9}
-!18 = !DILocation(line: 5, scope: !15)
diff --git a/lldb/test/API/functionalities/target_var/main.c b/lldb/test/API/functionalities/target_var/main.c
new file mode 100644
index 0000000000000..cbc379d9f0933
--- /dev/null
+++ b/lldb/test/API/functionalities/target_var/main.c
@@ -0,0 +1,7 @@
+int i = 42;
+int *p = &i;
+
+struct incomplete;
+struct incomplete *var = (struct incomplete *)0xdead;
+
+int main() { return *p; }
diff --git a/lldb/test/API/functionalities/unwind/sigtramp/TestSigtrampUnwind.py b/lldb/test/API/functionalities/unwind/sigtramp/TestSigtrampUnwind.py
index bf19cacac06d5..3ee86f786133f 100644
--- a/lldb/test/API/functionalities/unwind/sigtramp/TestSigtrampUnwind.py
+++ b/lldb/test/API/functionalities/unwind/sigtramp/TestSigtrampUnwind.py
@@ -17,7 +17,7 @@ class SigtrampUnwind(TestBase):
     # On different platforms the "_sigtramp" and "__kill" frames are likely to be different.
     # This test could probably be adapted to run on linux/*bsd easily enough.
     @skipUnlessDarwin
-    @expectedFailureAll(oslist=["ios", "watchos", "tvos", "bridgeos"], bugnumber="<rdar://problem/34006863>")  # lldb skips 1 frame on arm64 above _sigtramp
+    @expectedFailureAll(archs=["arm64"], bugnumber="<rdar://problem/34006863>")  # lldb skips 1 frame on arm64 above _sigtramp
     def test(self):
         """Test that we can backtrace correctly with _sigtramp on the stack"""
         self.build()
diff --git a/lldb/test/API/iohandler/resize/TestIOHandlerResizeNoEditline.py b/lldb/test/API/iohandler/resize/TestIOHandlerResizeNoEditline.py
index 4a99ff6d0ead2..9f0d562e8afcf 100644
--- a/lldb/test/API/iohandler/resize/TestIOHandlerResizeNoEditline.py
+++ b/lldb/test/API/iohandler/resize/TestIOHandlerResizeNoEditline.py
@@ -8,12 +8,14 @@ class TestCase(TestBase):
     mydir = TestBase.compute_mydir(__file__)
 
     @no_debug_info_test
+    @skipIfReproducer
+    @skipIfWindows
     def test_resize_no_editline(self):
         """ Tests terminal resizing if the editline isn't used. """
         dbg = lldb.SBDebugger.Create(False)
         # Set the input handle to some stream so that we don't start the
         # editline interface.
-        dbg.SetInputFileHandle(io.BytesIO(b""), True)
+        dbg.SetInputFileHandle(open("input_file"), True)
         opts = lldb.SBCommandInterpreterRunOptions()
         # Launch the command interpreter now.
         dbg.RunCommandInterpreter(True, True, opts, 0, False, False)
diff --git a/lldb/test/API/iohandler/resize/input_file b/lldb/test/API/iohandler/resize/input_file
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/lldb/test/API/lang/c/struct_types/main.c b/lldb/test/API/lang/c/struct_types/main.c
index 0b87bb1930324..08ef0a703f22b 100644
--- a/lldb/test/API/lang/c/struct_types/main.c
+++ b/lldb/test/API/lang/c/struct_types/main.c
@@ -17,7 +17,7 @@ int main (int argc, char const *argv[])
         char padding[0];
     }; //% self.expect("frame variable pt.padding[0]", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["pt.padding[0] = "])
        //% self.expect("frame variable pt.padding[1]", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["pt.padding[1] = "])
-       //% self.expect("expression -- (pt.padding[0])", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["(char)", " = "])
+       //% self.expect_expr("pt.padding[0]", result_type="char")
        //% self.expect("image lookup -t point_tag", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ['padding[]'])
 
     struct {} empty;
@@ -32,7 +32,7 @@ int main (int argc, char const *argv[])
     struct rect_tag rect = {{1, 2, {}}, {3, 4, {}}};
     struct things_to_sum tts = { 2, 3, 4 };
 
-    int sum = sum_things(tts); //% self.expect("expression -- &pt == (struct point_tag*)0", substrs = ['false'])
-                               //% self.expect("expression -- sum_things(tts)", substrs = ['9'])
+    int sum = sum_things(tts); //% self.expect_expr("&pt == (struct point_tag*)0", result_value="false")
+                               //% self.expect_expr("sum_things(tts)", result_value="9")
     return 0;
 }
diff --git a/lldb/test/API/lang/cpp/alignas_base_class/Makefile b/lldb/test/API/lang/cpp/alignas_base_class/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/lang/cpp/alignas_base_class/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/lang/cpp/alignas_base_class/TestAlignAsBaseClass.py b/lldb/test/API/lang/cpp/alignas_base_class/TestAlignAsBaseClass.py
new file mode 100644
index 0000000000000..25f37ab7fe5fa
--- /dev/null
+++ b/lldb/test/API/lang/cpp/alignas_base_class/TestAlignAsBaseClass.py
@@ -0,0 +1,16 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @no_debug_info_test
+    def test(self):
+        self.build()
+        self.dbg.CreateTarget(self.getBuildArtifact("a.out"))
+
+        # The offset of f2 should be 8 because of `alignas(8)`.
+        self.expect_expr("(intptr_t)&d3g.f2 - (intptr_t)&d3g", result_value="8")
diff --git a/lldb/test/API/lang/cpp/alignas_base_class/main.cpp b/lldb/test/API/lang/cpp/alignas_base_class/main.cpp
new file mode 100644
index 0000000000000..8dfced6c784e1
--- /dev/null
+++ b/lldb/test/API/lang/cpp/alignas_base_class/main.cpp
@@ -0,0 +1,13 @@
+struct B1 {
+  char f1;
+};
+
+struct alignas(8) B2 {
+  char f2;
+};
+
+struct D : B1, B2 {};
+
+D d3g;
+
+int main() {}
diff --git a/lldb/test/API/lang/cpp/operator-overload/TestOperatorOverload.py b/lldb/test/API/lang/cpp/operator-overload/TestOperatorOverload.py
index f541a6617e3a4..c361a3916fb21 100644
--- a/lldb/test/API/lang/cpp/operator-overload/TestOperatorOverload.py
+++ b/lldb/test/API/lang/cpp/operator-overload/TestOperatorOverload.py
@@ -13,10 +13,6 @@ def test_overload(self):
         "break here", lldb.SBFileSpec("b.cpp"))
     frame = thread.GetSelectedFrame()
     value = frame.EvaluateExpression("x == nil")
-    self.assertTrue(str(value.GetError())
-      .find("comparison between NULL and non-pointer ('Tinky' and NULL)")
-        != -1)
-    self.assertTrue(str(value.GetError())
-      .find("invalid operands to binary expression ('Tinky' and")
-        != -1)
     self.assertFalse(value.GetError().Success())
+    self.assertIn("comparison between NULL and non-pointer ('Tinky' and NULL)", str(value.GetError()))
+    self.assertIn("invalid operands to binary expression ('Tinky' and", str(value.GetError()))
diff --git a/lldb/test/API/linux/thread/create_during_instruction_step/TestCreateDuringInstructionStep.py b/lldb/test/API/linux/thread/create_during_instruction_step/TestCreateDuringInstructionStep.py
index 6e19aa830cd3a..92384e1c50789 100644
--- a/lldb/test/API/linux/thread/create_during_instruction_step/TestCreateDuringInstructionStep.py
+++ b/lldb/test/API/linux/thread/create_during_instruction_step/TestCreateDuringInstructionStep.py
@@ -4,7 +4,6 @@
 """
 
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -18,7 +17,7 @@ class CreateDuringInstructionStepTestCase(TestBase):
 
     @skipUnlessPlatform(['linux'])
     @expectedFailureAndroid('llvm.org/pr24737', archs=['arm'])
-    @skipIf(oslist=["linux"], archs=["arm"], bugnumber="llvm.org/pr24737")
+    @skipIf(oslist=["linux"], archs=["arm", "aarch64"], bugnumber="llvm.org/pr24737")
     def test_step_inst(self):
         self.build(dictionary=self.getBuildFlags())
         exe = self.getBuildArtifact("a.out")
diff --git a/lldb/test/API/lit.cfg.py b/lldb/test/API/lit.cfg.py
index 632d883e0da9a..57e7509a9f7e5 100644
--- a/lldb/test/API/lit.cfg.py
+++ b/lldb/test/API/lit.cfg.py
@@ -182,6 +182,9 @@ def find_python_interpreter():
 if config.filecheck:
   dotest_cmd += ['--filecheck', config.filecheck]
 
+if config.yaml2obj:
+  dotest_cmd += ['--yaml2obj', config.yaml2obj]
+
 if config.lldb_libs_dir:
   dotest_cmd += ['--lldb-libs-dir', config.lldb_libs_dir]
 
diff --git a/lldb/test/API/lit.site.cfg.py.in b/lldb/test/API/lit.site.cfg.py.in
index e97f867b265b8..866dc1675e7cf 100644
--- a/lldb/test/API/lit.site.cfg.py.in
+++ b/lldb/test/API/lit.site.cfg.py.in
@@ -30,6 +30,7 @@ config.test_arch = '@LLDB_TEST_ARCH@'
 config.test_compiler = '@LLDB_TEST_COMPILER@'
 config.dsymutil = '@LLDB_TEST_DSYMUTIL@'
 config.filecheck = '@LLDB_TEST_FILECHECK@'
+config.yaml2obj = '@LLDB_TEST_YAML2OBJ@'
 # The API tests use their own module caches.
 config.lldb_module_cache = os.path.join("@LLDB_TEST_MODULE_CACHE_LLDB@", "lldb-api")
 config.clang_module_cache = os.path.join("@LLDB_TEST_MODULE_CACHE_CLANG@", "lldb-api")
diff --git a/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py b/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py
index dce37aa6d706d..6e67fdc879bef 100644
--- a/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py
+++ b/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py
@@ -6,17 +6,60 @@
 import unittest2
 
 
+@skipIfDarwin # rdar://problem/64552748
 class TestSimulatorPlatformLaunching(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
     NO_DEBUG_INFO_TESTCASE = True
 
-    def run_with(self, arch, platform, os, env):
-        self.build(dictionary={'TRIPLE': arch+'-apple-'+os+'-'+env, 'ARCH': arch})
+    def check_load_commands(self, expected_load_command):
+        """sanity check the built binary for the expected number of load commands"""
+        load_cmds = subprocess.check_output(
+            ['otool', '-l', self.getBuildArtifact()]
+        ).decode("utf-8")
+        found = 0
+        for line in load_cmds.split('\n'):
+            if expected_load_command in line:
+              found += 1
+        self.assertEquals(found, 1, "wrong load command")
+
+    def check_debugserver(self, log, expected_platform, expected_version):
+        """scan the debugserver packet log"""
+        logfile = open(log, "r")
+        dylib_info = None
+        response = False
+        for line in logfile:
+            if response:
+                while line[0] != '$':
+                    line = line[1:]
+                line = line[1:]
+                # Unescape '}'.
+                dylib_info = json.loads(line.replace('}]','}')[:-4])
+                response = False
+            if 'send packet: $jGetLoadedDynamicLibrariesInfos:{' in line:
+                response = True
+
+        self.assertTrue(dylib_info)
+        aout_info = None
+        for image in dylib_info['images']:
+            if image['pathname'].endswith('a.out'):
+                aout_info = image
+        self.assertTrue(aout_info)
+        self.assertEquals(aout_info['min_version_os_name'], expected_platform)
+        if expected_version:
+            self.assertEquals(aout_info['min_version_os_sdk'], expected_version)
+
+
+    def run_with(self, arch, os, vers, env, expected_load_command):
+        self.build(dictionary={'TRIPLE': arch+'-apple-'+os+vers+'-'+env})
+        self.check_load_commands(expected_load_command)
+        log = self.getBuildArtifact('packets.log')
+        self.expect("log enable gdb-remote packets -f "+log)
         lldbutil.run_to_source_breakpoint(self, "break here",
                                           lldb.SBFileSpec("hello.c"))
         self.expect('image list -b -t',
-                    patterns=['a\.out '+arch+'-apple-'+os+'.*-'+env])
+                    patterns=['a\.out '+arch+'-apple-'+os+vers+'.*-'+env])
+        self.check_debugserver(log, os+env, vers)
 
     @skipUnlessDarwin
     @skipIfDarwinEmbedded
@@ -24,8 +67,8 @@ def run_with(self, arch, platform, os, env):
     def test_ios(self):
         """Test running an iOS simulator binary"""
         self.run_with(arch=self.getArchitecture(),
-                      os='ios', env='simulator',
-                      platform='iphonesimulator')
+                      os='ios', vers='', env='simulator',
+                      expected_load_command='LC_BUILD_VERSION')
 
     @skipUnlessDarwin
     @skipIfDarwinEmbedded
@@ -33,15 +76,113 @@ def test_ios(self):
     def test_tvos(self):
         """Test running an tvOS simulator binary"""
         self.run_with(arch=self.getArchitecture(),
-                      os='tvos', env='simulator',
-                      platform='appletvsimulator')
+                      os='tvos', vers='', env='simulator',
+                      expected_load_command='LC_BUILD_VERSION')
 
     @skipUnlessDarwin
     @skipIfDarwinEmbedded
     @apple_simulator_test('watch')
     @skipIfDarwin # rdar://problem/64552748
-    def test_watchos(self):
+    @skipIf(archs=['arm64','arm64e'])
+    def test_watchos_i386(self):
         """Test running a 32-bit watchOS simulator binary"""
         self.run_with(arch='i386',
-                      os='watchos', env='simulator',
-                      platform='watchsimulator')
+                      os='watchos', vers='', env='simulator',
+                      expected_load_command='LC_BUILD_VERSION')
+
+    @skipUnlessDarwin
+    @skipIfDarwinEmbedded
+    @apple_simulator_test('watch')
+    @skipIfDarwin # rdar://problem/64552748
+    @skipIf(archs=['i386','x86_64'])
+    def test_watchos_armv7k(self):
+        """Test running a 32-bit watchOS simulator binary"""
+        self.run_with(arch='armv7k',
+                      os='watchos', vers='', env='simulator',
+                      expected_load_command='LC_BUILD_VERSION')
+
+
+    #
+    # Back-deployment tests.
+    #
+    # Older Mach-O versions used less expressive load commands, such
+    # as LC_VERSION_MIN_IPHONEOS that wouldn't distinguish between ios
+    # and ios-simulator.  When targeting a simulator on Apple Silicon
+    # macOS, however, these legacy load commands are never generated.
+    #
+        
+    @skipUnlessDarwin
+    @skipIfDarwinEmbedded
+    @apple_simulator_test('iphone')
+    @skipIf(archs=['arm64','arm64e'])
+    def test_lc_version_min_iphoneos(self):
+        """Test running a back-deploying iOS simulator binary
+           with a legacy iOS load command"""
+        self.run_with(arch=self.getArchitecture(),
+                      os='ios', vers='11.0', env='simulator',
+                      expected_load_command='LC_VERSION_MIN_IPHONEOS')
+
+    @skipUnlessDarwin
+    @skipIfDarwinEmbedded
+    @apple_simulator_test('iphone')
+    @skipIf(archs=['arm64','arm64e'])
+    def test_ios_backdeploy_x86(self):
+        """Test running a back-deploying iOS simulator binary
+           with a legacy iOS load command"""
+        self.run_with(arch=self.getArchitecture(),
+                      os='ios', vers='13.0', env='simulator',
+                      expected_load_command='LC_BUILD_VERSION')
+
+    @skipUnlessDarwin
+    @skipIfDarwinEmbedded
+    @apple_simulator_test('iphone')
+    @skipIf(archs=['i386','x86_64'])
+    def test_ios_backdeploy_apple_silicon(self):
+        """Test running a back-deploying iOS simulator binary"""
+        self.run_with(arch=self.getArchitecture(),
+                      os='ios', vers='11.0', env='simulator',
+                      expected_load_command='LC_BUILD_VERSION')
+
+    @skipUnlessDarwin
+    @skipIfDarwinEmbedded
+    @apple_simulator_test('appletv')
+    @skipIf(archs=['arm64','arm64e'])
+    def test_lc_version_min_tvos(self):
+        """Test running a back-deploying tvOS simulator binary
+           with a legacy tvOS load command"""
+        self.run_with(arch=self.getArchitecture(),
+                      os='tvos', vers='11.0', env='simulator',
+                      expected_load_command='LC_VERSION_MIN_TVOS')
+
+    @skipUnlessDarwin
+    @skipIfDarwinEmbedded
+    @apple_simulator_test('appletv')
+    @skipIf(archs=['i386','x86_64'])
+    def test_tvos_backdeploy_apple_silicon(self):
+        """Test running a back-deploying tvOS simulator binary"""
+        self.run_with(arch=self.getArchitecture(),
+                      os='tvos', vers='11.0', env='simulator',
+                      expected_load_command='LC_BUILD_VERSION')
+
+    @skipUnlessDarwin
+    @skipIfDarwinEmbedded
+    @apple_simulator_test('watch')
+    @skipIf(archs=['arm64','arm64e'])
+    @skipIfDarwin # rdar://problem/64552748
+    def test_lc_version_min_watchos(self):
+        """Test running a back-deploying watchOS simulator binary
+           with a legacy watchOS load command"""
+        self.run_with(arch='i386',
+                      os='watchos', vers='4.0', env='simulator',
+                      expected_load_command='LC_VERSION_MIN_WATCHOS')
+
+    @skipUnlessDarwin
+    @skipIfDarwinEmbedded
+    @apple_simulator_test('watch')
+    @skipIf(archs=['arm64','arm64e'])
+    @skipIfDarwin # rdar://problem/64552748
+    def test_watchos_backdeploy_apple_silicon(self):
+        """Test running a back-deploying watchOS simulator binary"""
+        self.run_with(arch='armv7k',
+                      os='watchos', vers='4.0', env='simulator',
+                      expected_load_command='LC_BUILD_VERSION')
diff --git a/lldb/test/API/python_api/hello_world/TestHelloWorld.py b/lldb/test/API/python_api/hello_world/TestHelloWorld.py
index 75a55ab1f44dc..c1155cf298145 100644
--- a/lldb/test/API/python_api/hello_world/TestHelloWorld.py
+++ b/lldb/test/API/python_api/hello_world/TestHelloWorld.py
@@ -142,7 +142,8 @@ def test_with_attach_to_process_with_name_api(self):
         target.ConnectRemote(listener, None, None, error)
 
         process = target.AttachToProcessWithName(listener, name, False, error)
-        self.assertTrue(error.Success() and process, PROCESS_IS_VALID)
+        self.assertSuccess(error)
+        self.assertTrue(process, PROCESS_IS_VALID)
 
         # Verify that after attach, our selected target indeed matches name.
         self.expect(
diff --git a/lldb/test/API/tools/lldb-vscode/attach/TestVSCode_attach.py b/lldb/test/API/tools/lldb-vscode/attach/TestVSCode_attach.py
index bd66f98270ce6..e49c9267d971a 100644
--- a/lldb/test/API/tools/lldb-vscode/attach/TestVSCode_attach.py
+++ b/lldb/test/API/tools/lldb-vscode/attach/TestVSCode_attach.py
@@ -43,7 +43,6 @@ def set_and_hit_breakpoint(self, continueToExit=True):
         if continueToExit:
             self.continue_to_exit()
 
-
     @skipIfWindows
     @skipIfNetBSD # Hangs on NetBSD as well
     @skipIfRemote
@@ -118,11 +117,12 @@ def test_by_name_waitFor(self):
     @skipIfWindows
     @skipIfDarwin
     @skipIfNetBSD # Hangs on NetBSD as well
+    @skipIf(archs=["arm", "aarch64"]) # Example of a flaky run http://lab.llvm.org:8011/builders/lldb-aarch64-ubuntu/builds/5527/steps/test/logs/stdio
     def test_commands(self):
         '''
             Tests the "initCommands", "preRunCommands", "stopCommands",
-            "exitCommands", and "attachCommands" that can be passed during
-            attach.
+            "exitCommands", "terminateCommands" and "attachCommands"
+            that can be passed during attach.
 
             "initCommands" are a list of LLDB commands that get executed
             before the targt is created.
@@ -136,6 +136,8 @@ def test_commands(self):
             must have a valid process in the selected target in LLDB after
             they are done executing. This allows custom commands to create any
             kind of debug session.
+            "terminateCommands" are a list of LLDB commands that get executed when
+            the debugger session terminates.
         '''
         self.build_and_create_debug_adaptor()
         program = self.getBuildArtifact("a.out")
@@ -150,13 +152,14 @@ def test_commands(self):
         preRunCommands = ['image list a.out', 'image dump sections a.out']
         stopCommands = ['frame variable', 'bt']
         exitCommands = ['expr 2+3', 'expr 3+4']
+        terminateCommands = ['expr 4+2']
         self.attach(program=program,
                     attachCommands=attachCommands,
                     initCommands=initCommands,
                     preRunCommands=preRunCommands,
                     stopCommands=stopCommands,
-                    exitCommands=exitCommands)
-
+                    exitCommands=exitCommands,
+                    terminateCommands=terminateCommands)
         # Get output from the console. This should contain both the
         # "initCommands" and the "preRunCommands".
         output = self.get_console()
@@ -187,5 +190,37 @@ def test_commands(self):
         self.continue_to_exit()
         # Get output from the console. This should contain both the
         # "exitCommands" that were run after the second breakpoint was hit
-        output = self.get_console(timeout=1.0)
+        # and the "terminateCommands" due to the debugging session ending
+        output = self.collect_console(duration=1.0)
         self.verify_commands('exitCommands', output, exitCommands)
+        self.verify_commands('terminateCommands', output, terminateCommands)
+
+    @skipIfWindows
+    @skipIfDarwin
+    @skipIfNetBSD # Hangs on NetBSD as well
+    @skipIf(archs=["arm", "aarch64"]) # Example of a flaky run http://lab.llvm.org:8011/builders/lldb-aarch64-ubuntu/builds/5517/steps/test/logs/stdio
+    def test_terminate_commands(self):
+        '''
+            Tests that the "terminateCommands", that can be passed during
+            attach, are run when the debugger is disconnected.
+        '''
+        self.build_and_create_debug_adaptor()
+        program = self.getBuildArtifact("a.out")
+        # Here we just create a target and launch the process as a way to test
+        # if we are able to use attach commands to create any kind of a target
+        # and use it for debugging
+        attachCommands = [
+            'target create -d "%s"' % (program),
+            'process launch'
+        ]
+        terminateCommands = ['expr 4+2']
+        self.attach(program=program,
+                    attachCommands=attachCommands,
+                    terminateCommands=terminateCommands,
+                    disconnectAutomatically=False)
+        self.get_console()
+        # Once it's disconnected the console should contain the
+        # "terminateCommands"
+        self.vscode.request_disconnect(terminateDebuggee=True)
+        output = self.collect_console(duration=1.0)
+        self.verify_commands('terminateCommands', output, terminateCommands)
diff --git a/lldb/test/API/tools/lldb-vscode/disconnect/Makefile b/lldb/test/API/tools/lldb-vscode/disconnect/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/tools/lldb-vscode/disconnect/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/tools/lldb-vscode/disconnect/TestVSCode_disconnect.py b/lldb/test/API/tools/lldb-vscode/disconnect/TestVSCode_disconnect.py
new file mode 100644
index 0000000000000..cc001e89196b9
--- /dev/null
+++ b/lldb/test/API/tools/lldb-vscode/disconnect/TestVSCode_disconnect.py
@@ -0,0 +1,84 @@
+"""
+Test lldb-vscode disconnect request
+"""
+
+
+import unittest2
+import vscode
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+import lldbvscode_testcase
+import subprocess
+import time
+import os
+
+
+class TestVSCode_launch(lldbvscode_testcase.VSCodeTestCaseBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+    source = 'main.cpp'
+
+    def disconnect_and_assert_no_output_printed(self):
+        self.vscode.request_disconnect()
+        # verify we didn't get any input after disconnect
+        time.sleep(2)
+        output = self.get_stdout()
+        self.assertTrue(output is None or len(output) == 0)
+
+    @skipIfDarwin
+    @skipIfWindows
+    @skipIfRemote
+    def test_launch(self):
+        """
+            This test launches a process that would creates a file, but we disconnect
+            before the file is created, which terminates the process and thus the file is not
+            created.
+        """
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program)
+
+        # We set a breakpoint right before the side effect file is created
+        self.set_source_breakpoints(self.source, [line_number(self.source, '// breakpoint')])
+        self.continue_to_next_stop()
+
+        self.vscode.request_disconnect()
+        # verify we didn't produce the side effect file
+        time.sleep(1)
+        self.assertFalse(os.path.exists(program + ".side_effect"))
+
+
+    @skipIfDarwin
+    @skipIfWindows
+    @skipIfRemote
+    def test_attach(self):
+        """
+            This test attaches to a process that creates a file. We attach and disconnect
+            before the file is created, and as the process is not terminated upon disconnection,
+            the file is created anyway.
+        """
+        self.build_and_create_debug_adaptor()
+        program = self.getBuildArtifact("a.out")
+
+        # Use a file as a synchronization point between test and inferior.
+        sync_file_path = lldbutil.append_to_process_working_directory(self,
+            "sync_file_%d" % (int(time.time())))
+        self.addTearDownHook(
+            lambda: self.run_platform_command(
+                "rm %s" %
+                (sync_file_path)))
+
+        self.process = subprocess.Popen([program, sync_file_path])
+        lldbutil.wait_for_file_on_target(self, sync_file_path)
+
+        self.attach(pid=self.process.pid, disconnectAutomatically=False)
+        response = self.vscode.request_evaluate("wait_for_attach = false;")
+        self.assertTrue(response['success'])
+
+        # verify we haven't produced the side effect file yet
+        self.assertFalse(os.path.exists(program + ".side_effect"))
+
+        self.vscode.request_disconnect()
+        time.sleep(2)
+        # verify we produced the side effect file, as the program continued after disconnecting
+        self.assertTrue(os.path.exists(program + ".side_effect"))
diff --git a/lldb/test/API/tools/lldb-vscode/disconnect/main.cpp b/lldb/test/API/tools/lldb-vscode/disconnect/main.cpp
new file mode 100644
index 0000000000000..d3d7a4b7338a8
--- /dev/null
+++ b/lldb/test/API/tools/lldb-vscode/disconnect/main.cpp
@@ -0,0 +1,33 @@
+#include <chrono>
+#include <cstdio>
+#include <fstream>
+#include <string>
+#include <thread>
+
+volatile bool wait_for_attach = true;
+
+void handle_attach(char *sync_file_path) {
+  lldb_enable_attach();
+
+  {
+    // Create a file to signal that this process has started up.
+    std::ofstream sync_file;
+    sync_file.open(sync_file_path);
+  }
+
+  while (wait_for_attach)
+    std::this_thread::sleep_for(std::chrono::milliseconds(10));
+}
+
+int main(int argc, char **args) {
+  if (argc == 2)
+    handle_attach(args[1]);
+
+  // We let the binary live a little bit to see if it executed after detaching
+  // from // breakpoint
+
+  // Create a file to signal that this process has started up.
+  std::ofstream out_file; // breakpoint
+  out_file.open(std::string(args[0]) + ".side_effect");
+  return 0;
+}
diff --git a/lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py b/lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py
index f7f314d76f252..fb7d71872a16d 100644
--- a/lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py
+++ b/lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py
@@ -294,8 +294,9 @@ def test_environment(self):
     @skipIfRemote
     def test_commands(self):
         '''
-            Tests the "initCommands", "preRunCommands", "stopCommands" and
-            "exitCommands" that can be passed during launch.
+            Tests the "initCommands", "preRunCommands", "stopCommands",
+            "terminateCommands" and "exitCommands" that can be passed during
+            launch.
 
             "initCommands" are a list of LLDB commands that get executed
             before the targt is created.
@@ -305,17 +306,21 @@ def test_commands(self):
             time the program stops.
             "exitCommands" are a list of LLDB commands that get executed when
             the process exits
+            "terminateCommands" are a list of LLDB commands that get executed when
+            the debugger session terminates.
         '''
         program = self.getBuildArtifact("a.out")
         initCommands = ['target list', 'platform list']
         preRunCommands = ['image list a.out', 'image dump sections a.out']
         stopCommands = ['frame variable', 'bt']
         exitCommands = ['expr 2+3', 'expr 3+4']
+        terminateCommands = ['expr 4+2']
         self.build_and_launch(program,
                               initCommands=initCommands,
                               preRunCommands=preRunCommands,
                               stopCommands=stopCommands,
-                              exitCommands=exitCommands)
+                              exitCommands=exitCommands,
+                              terminateCommands=terminateCommands)
 
         # Get output from the console. This should contain both the
         # "initCommands" and the "preRunCommands".
@@ -354,8 +359,10 @@ def test_commands(self):
         self.continue_to_exit()
         # Get output from the console. This should contain both the
         # "exitCommands" that were run after the second breakpoint was hit
-        output = self.get_console(timeout=1.0)
+        # and the "terminateCommands" due to the debugging session ending
+        output = self.collect_console(duration=1.0)
         self.verify_commands('exitCommands', output, exitCommands)
+        self.verify_commands('terminateCommands', output, terminateCommands)
 
     @skipIfWindows
     @skipIfRemote
@@ -420,3 +427,25 @@ def test_extra_launch_commands(self):
         # "exitCommands" that were run after the second breakpoint was hit
         output = self.get_console(timeout=1.0)
         self.verify_commands('exitCommands', output, exitCommands)
+
+    @skipIfWindows
+    @skipIfNetBSD # Hangs on NetBSD as well
+    @skipIfDarwin
+    @skipIf(archs="aarch64") # Example of a flaky run http://lab.llvm.org:8011/builders/lldb-aarch64-ubuntu/builds/5540/steps/test/logs/stdio
+    def test_terminate_commands(self):
+        '''
+            Tests that the "terminateCommands", that can be passed during
+            launch, are run when the debugger is disconnected.
+        '''
+        self.build_and_create_debug_adaptor()
+        program = self.getBuildArtifact("a.out")
+        
+        terminateCommands = ['expr 4+2']
+        self.launch(program=program,
+                    terminateCommands=terminateCommands)
+        self.get_console()
+        # Once it's disconnected the console should contain the
+        # "terminateCommands"
+        self.vscode.request_disconnect(terminateDebuggee=True)
+        output = self.collect_console(duration=1.0)
+        self.verify_commands('terminateCommands', output, terminateCommands)
diff --git a/lldb/test/Shell/Recognizer/lit.local.cfg b/lldb/test/Shell/Recognizer/lit.local.cfg
new file mode 100644
index 0000000000000..945a5615c4219
--- /dev/null
+++ b/lldb/test/Shell/Recognizer/lit.local.cfg
@@ -0,0 +1,2 @@
+if 'python' not in config.available_features:
+  config.unsupported = True
diff --git a/lldb/test/Shell/Register/arm-fp-read.test b/lldb/test/Shell/Register/arm-fp-read.test
index 538d6af54dcf2..116fa7aa0df5e 100644
--- a/lldb/test/Shell/Register/arm-fp-read.test
+++ b/lldb/test/Shell/Register/arm-fp-read.test
@@ -1,4 +1,4 @@
-# REQUIRES: native && target-arm
+# REQUIRES: native && target-arm && !system-darwin
 # RUN: %clangxx_host -fomit-frame-pointer %p/Inputs/arm-fp-read.cpp -o %t
 # RUN: %lldb -b -s %s %t | FileCheck %s
 process launch
diff --git a/lldb/test/Shell/Register/arm-gp-read.test b/lldb/test/Shell/Register/arm-gp-read.test
index bcb289b880a75..9f8461842e9ba 100644
--- a/lldb/test/Shell/Register/arm-gp-read.test
+++ b/lldb/test/Shell/Register/arm-gp-read.test
@@ -1,4 +1,4 @@
-# REQUIRES: native && target-arm
+# REQUIRES: native && target-arm && !system-darwin
 # RUN: %clangxx_host -fomit-frame-pointer %p/Inputs/arm-gp-read.cpp -o %t
 # RUN: %lldb -b -s %s %t | FileCheck %s
 process launch
diff --git a/lldb/test/Shell/Reproducer/TestCrash.test b/lldb/test/Shell/Reproducer/TestCrash.test
index 1389a9b76ad39..530dd8ad398d8 100644
--- a/lldb/test/Shell/Reproducer/TestCrash.test
+++ b/lldb/test/Shell/Reproducer/TestCrash.test
@@ -10,3 +10,8 @@
 # CHECK: Crash reproducer for
 # CHECK: Reproducer written to
 # CHECK: ********************
+
+# RUN: %lldb -b --capture --capture-path %t.repro --reproducer-no-generate-on-signal -o 'reproducer xcrash -s SIGSEGV' | FileCheck %s --check-prefix NOHANDLER
+
+# NOHANDLER-NOT: Crash reproducer
+# NOHANDLER-NOT: Reproducer written
diff --git a/lldb/test/Shell/Reproducer/TestDriverOptions.test b/lldb/test/Shell/Reproducer/TestDriverOptions.test
index 8bc8288e2e5f5..ccdccb3c9411c 100644
--- a/lldb/test/Shell/Reproducer/TestDriverOptions.test
+++ b/lldb/test/Shell/Reproducer/TestDriverOptions.test
@@ -11,16 +11,16 @@
 # RUN: %lldb --capture --capture-path %t.repro -b -o 'reproducer status' 2>&1 | FileCheck %s --check-prefix NO-WARNING --check-prefix STATUS-CAPTURE
 # RUN: %lldb --capture -b -o 'reproducer status' 2>&1 | FileCheck %s --check-prefix NO-WARNING --check-prefix STATUS-CAPTURE
 # RUN: %lldb --capture-path %t.repro -b -o 'reproducer status' 2>&1 | FileCheck %s --check-prefix WARNING --check-prefix STATUS-CAPTURE --check-prefix NOAUTOGEN
-# RUN: %lldb --capture-path %t.repro -b -o 'reproducer status' --reproducer-auto-generate  2>&1 | FileCheck %s --check-prefix WARNING2
+# RUN: %lldb --capture-path %t.repro -b -o 'reproducer status' --reproducer-generate-on-exit  2>&1 | FileCheck %s --check-prefix WARNING2
 #
 # NO-WARNING-NOT: warning: -capture-path specified without -capture
 # WARNING: warning: -capture-path specified without -capture
-# WARNING2: warning: -reproducer-auto-generate specified without -capture
+# WARNING2: warning: -reproducer-generate-on-exit specified without -capture
 # STATUS-CAPTURE: Reproducer is in capture mode.
 # NOAUTOGEN-NOT: Auto generate
 
 # Check auto generate.
 # RUN: rm -rf %t.repro
-# RUN: %lldb --capture --capture-path %t.repro -b --reproducer-auto-generate -o 'reproducer status' 2>&1 | FileCheck %s --check-prefix NO-WARNING --check-prefix AUTOGEN
+# RUN: %lldb --capture --capture-path %t.repro -b --reproducer-generate-on-exit -o 'reproducer status' 2>&1 | FileCheck %s --check-prefix NO-WARNING --check-prefix AUTOGEN
 # RUN: cat %t.repro/index.yaml
 # AUTOGEN: Auto generate: on
diff --git a/lldb/test/Shell/Reproducer/TestVersionCheck.test b/lldb/test/Shell/Reproducer/TestVersionCheck.test
index e3fb60367cec2..d37059be6db82 100644
--- a/lldb/test/Shell/Reproducer/TestVersionCheck.test
+++ b/lldb/test/Shell/Reproducer/TestVersionCheck.test
@@ -15,8 +15,8 @@
 # Make sure that replay works.
 # RUN: not %lldb --replay %t.repro 2>&1 | FileCheck %s --check-prefix ERROR
 
-# Make sure that we can circumvent the version check with -reproducer-skip-version-check.
-# RUN: %lldb --replay %t.repro -reproducer-skip-version-check | FileCheck %s --check-prefix CHECK --check-prefix REPLAY
+# Make sure that we can circumvent the version check with -reproducer-no-version-check.
+# RUN: %lldb --replay %t.repro -reproducer-no-version-check | FileCheck %s --check-prefix CHECK --check-prefix REPLAY
 
 # CAPTURE: testing
 # REPLAY-NOT: testing
diff --git a/lldb/test/Shell/ScriptInterpreter/Lua/io.test b/lldb/test/Shell/ScriptInterpreter/Lua/io.test
new file mode 100644
index 0000000000000..af86253f90f25
--- /dev/null
+++ b/lldb/test/Shell/ScriptInterpreter/Lua/io.test
@@ -0,0 +1,22 @@
+# REQUIRES: lua
+# UNSUPPORTED: lldb-repro
+#
+# RUN: rm -rf %t.stderr %t.stdout
+# RUN: cat %s | %lldb --script-language lua 2> %t.stderr > %t.stdout
+# RUN: cat %t.stdout | FileCheck %s --check-prefix STDOUT
+# RUN: cat %t.stderr | FileCheck %s --check-prefix STDERR
+script
+file = lldb.SBFile(2, "w", false)
+lldb.debugger:SetOutputFile(file)
+io.write(95000 + 126, "\n")
+quit
+script
+io.write(95000 + 14, "\n")
+
+# STDOUT: 95126
+# STDOUT-NOT: 95014
+# STDERR: 95014
+
+# RUN: rm -rf %t.stderr %t.stdout
+# RUN: %lldb --script-language lua -o 'script io.stderr:write(95000 + 126, "\n")' 2> %t.stderr > %t.stdout
+# RUN: cat %t.stdout | FileCheck %s --check-prefix STDOUT
diff --git a/lldb/test/Shell/ScriptInterpreter/Lua/quit.test b/lldb/test/Shell/ScriptInterpreter/Lua/quit.test
new file mode 100644
index 0000000000000..e47c66c5a10ac
--- /dev/null
+++ b/lldb/test/Shell/ScriptInterpreter/Lua/quit.test
@@ -0,0 +1,10 @@
+# REQUIRES: lua
+# UNSUPPORTED: lldb-repro
+#
+# RUN: cat %s | %lldb --script-language lua 2>&1 | FileCheck %s
+script
+print(95000 + 126)
+quit
+target list
+# CHECK: 95126
+# CHECK: No targets
diff --git a/lldb/test/Shell/SymbolFile/DWARF/forward-declarations.s b/lldb/test/Shell/SymbolFile/DWARF/forward-declarations.s
deleted file mode 100644
index 952c4ee7f87ed..0000000000000
--- a/lldb/test/Shell/SymbolFile/DWARF/forward-declarations.s
+++ /dev/null
@@ -1,111 +0,0 @@
-# Test handling of the situation (including the error message) where a structure
-# has a incomplete member.
-
-# REQUIRES: x86
-
-# RUN: llvm-mc -triple x86_64-pc-linux -filetype=obj %s -o %t
-# RUN: %lldb %t -o "target var b" -b 2>&1 | FileCheck %s
-
-# CHECK: error: {{.*}} DWARF DIE at 0x0000002b (class B) has a member variable 0x00000030 (a) whose type is a forward declaration, not a complete definition.
-# CHECK-NEXT: Please file a bug against the compiler and include the preprocessed output for /tmp/a.cc
-
-# CHECK: b = (a = A @ 0x0000000000000001)
-
-        .type   b,@object               # @b
-        .comm   b,1,1
-        .section        .debug_str,"MS",@progbits,1
-.Linfo_string0:
-        .asciz  "Hand-written DWARF"
-.Lcu_name:
-        .asciz  "/tmp/a.cc"
-.Lcu_compdir:
-        .asciz  "/foo/bar"
-.Lb:
-        .asciz  "b"
-.La:
-        .asciz  "a"
-.LA:
-        .asciz  "A"
-.LB:
-        .asciz  "B"
-
-        .section        .debug_abbrev,"",@progbits
-        .byte   1                       # Abbreviation Code
-        .byte   17                      # DW_TAG_compile_unit
-        .byte   1                       # DW_CHILDREN_yes
-        .byte   37                      # DW_AT_producer
-        .byte   14                      # DW_FORM_strp
-        .byte   3                       # DW_AT_name
-        .byte   14                      # DW_FORM_strp
-        .byte   27                      # DW_AT_comp_dir
-        .byte   14                      # DW_FORM_strp
-        .byte   0                       # EOM(1)
-        .byte   0                       # EOM(2)
-        .byte   2                       # Abbreviation Code
-        .byte   52                      # DW_TAG_variable
-        .byte   0                       # DW_CHILDREN_no
-        .byte   3                       # DW_AT_name
-        .byte   14                      # DW_FORM_strp
-        .byte   73                      # DW_AT_type
-        .byte   19                      # DW_FORM_ref4
-        .byte   2                       # DW_AT_location
-        .byte   24                      # DW_FORM_exprloc
-        .byte   0                       # EOM(1)
-        .byte   0                       # EOM(2)
-        .byte   3                       # Abbreviation Code
-        .byte   19                      # DW_TAG_structure_type
-        .byte   1                       # DW_CHILDREN_yes
-        .byte   3                       # DW_AT_name
-        .byte   14                      # DW_FORM_strp
-        .byte   0                       # EOM(1)
-        .byte   0                       # EOM(2)
-        .byte   4                       # Abbreviation Code
-        .byte   13                      # DW_TAG_member
-        .byte   0                       # DW_CHILDREN_no
-        .byte   3                       # DW_AT_name
-        .byte   14                      # DW_FORM_strp
-        .byte   73                      # DW_AT_type
-        .byte   19                      # DW_FORM_ref4
-        .byte   0                       # EOM(1)
-        .byte   0                       # EOM(2)
-        .byte   5                       # Abbreviation Code
-        .byte   19                      # DW_TAG_structure_type
-        .byte   0                       # DW_CHILDREN_no
-        .byte   60                      # DW_AT_declaration
-        .byte   25                      # DW_FORM_flag_present
-        .byte   3                       # DW_AT_name
-        .byte   14                      # DW_FORM_strp
-        .byte   0                       # EOM(1)
-        .byte   0                       # EOM(2)
-        .byte   0                       # EOM(3)
-
-        .section        .debug_info,"",@progbits
-.Lcu_begin0:
-        .long   .Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
-.Ldebug_info_start0:
-        .short  4                       # DWARF version number
-        .long   .debug_abbrev           # Offset Into Abbrev. Section
-        .byte   8                       # Address Size (in bytes)
-        .byte   1                       # Abbrev [1] 0xb:0x46 DW_TAG_compile_unit
-        .long   .Linfo_string0          # DW_AT_producer
-        .long   .Lcu_name               # DW_AT_name
-        .long   .Lcu_compdir            # DW_AT_comp_dir
-        .byte   2                       # Abbrev [2] 0x1e:0x15 DW_TAG_variable
-        .long   .Lb                     # DW_AT_name
-        .long   .LB_die-.Lcu_begin0     # DW_AT_type
-        .byte   9                       # DW_AT_location
-        .byte   3
-        .quad   b
-.LB_die:
-        .byte   3                       # Abbrev [3] 0x33:0x15 DW_TAG_structure_type
-        .long   .LB                     # DW_AT_name
-        .byte   4                       # Abbrev [4] 0x3b:0xc DW_TAG_member
-        .long   .La                     # DW_AT_name
-        .long   .LA_die-.Lcu_begin0     # DW_AT_type
-        .byte   0                       # End Of Children Mark
-.LA_die:
-        .byte   5                       # Abbrev [5] 0x48:0x8 DW_TAG_structure_type
-                                        # DW_AT_declaration
-        .long   .LA                     # DW_AT_name
-        .byte   0                       # End Of Children Mark
-.Ldebug_info_end0:
diff --git a/lldb/test/Shell/SymbolFile/DWARF/limit-debug-info.s b/lldb/test/Shell/SymbolFile/DWARF/limit-debug-info.s
new file mode 100644
index 0000000000000..b14d6106cf06a
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/DWARF/limit-debug-info.s
@@ -0,0 +1,131 @@
+# REQUIRES: lld, x86
+# RUN: llvm-mc --triple=x86_64-pc-windows --filetype=obj --defsym DLL=0 %s >%t.dll.o
+# RUN: llvm-mc --triple=x86_64-pc-windows --filetype=obj --defsym EXE=0 %s >%t.exe.o
+# RUN: lld-link /OUT:%t.dll %t.dll.o /SUBSYSTEM:console /dll /noentry /debug
+# RUN: lld-link /OUT:%t.exe %t.exe.o /SUBSYSTEM:console /debug /force
+# RUN: %lldb %t.exe -o "target modules add %t.dll" -o "p var" \
+# RUN:   -o exit 2>&1 | FileCheck %s
+
+# CHECK: (lldb) p var
+# CHECK: (A) $0 = (member = 47)
+
+        .section        .debug_abbrev,"dr"
+.Lsection_abbrev:
+        .byte   1                       # Abbreviation Code
+        .byte   17                      # DW_TAG_compile_unit
+        .byte   1                       # DW_CHILDREN_yes
+        .byte   37                      # DW_AT_producer
+        .byte   8                       # DW_FORM_string
+        .byte   0                       # EOM(1)
+        .byte   0                       # EOM(2)
+        .byte   2                       # Abbreviation Code
+        .byte   52                      # DW_TAG_variable
+        .byte   0                       # DW_CHILDREN_no
+        .byte   3                       # DW_AT_name
+        .byte   8                       # DW_FORM_string
+        .byte   73                      # DW_AT_type
+        .byte   19                      # DW_FORM_ref4
+        .byte   2                       # DW_AT_location
+        .byte   24                      # DW_FORM_exprloc
+        .byte   0                       # EOM(1)
+        .byte   0                       # EOM(2)
+        .byte   3                       # Abbreviation Code
+        .byte   19                      # DW_TAG_structure_type
+        .byte   1                       # DW_CHILDREN_yes
+        .byte   3                       # DW_AT_name
+        .byte   8                       # DW_FORM_string
+        .byte   11                      # DW_AT_byte_size
+        .byte   11                      # DW_FORM_data1
+        .byte   0                       # EOM(1)
+        .byte   0                       # EOM(2)
+        .byte   4                       # Abbreviation Code
+        .byte   13                      # DW_TAG_member
+        .byte   0                       # DW_CHILDREN_no
+        .byte   3                       # DW_AT_name
+        .byte   8                       # DW_FORM_string
+        .byte   73                      # DW_AT_type
+        .byte   19                      # DW_FORM_ref4
+        .byte   56                      # DW_AT_data_member_location
+        .byte   11                      # DW_FORM_data1
+        .byte   0                       # EOM(1)
+        .byte   0                       # EOM(2)
+        .byte   5                       # Abbreviation Code
+        .byte   36                      # DW_TAG_base_type
+        .byte   0                       # DW_CHILDREN_no
+        .byte   3                       # DW_AT_name
+        .byte   8                       # DW_FORM_string
+        .byte   62                      # DW_AT_encoding
+        .byte   11                      # DW_FORM_data1
+        .byte   11                      # DW_AT_byte_size
+        .byte   11                      # DW_FORM_data1
+        .byte   0                       # EOM(1)
+        .byte   0                       # EOM(2)
+        .byte   6                       # Abbreviation Code
+        .byte   19                      # DW_TAG_structure_type
+        .byte   0                       # DW_CHILDREN_no
+        .byte   3                       # DW_AT_name
+        .byte   8                       # DW_FORM_string
+        .byte   60                      # DW_AT_declaration
+        .byte   25                      # DW_FORM_flag_present
+        .byte   0                       # EOM(1)
+        .byte   0                       # EOM(2)
+        .byte   0                       # EOM(3)
+
+.ifdef DLL
+        .section        .debug_info,"dr"
+.Lsection_info:
+.Lcu_begin0:
+        .long   .Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+        .short  4                       # DWARF version number
+        .secrel32       .Lsection_abbrev # Offset Into Abbrev. Section
+        .byte   8                       # Address Size (in bytes)
+        .byte   1                       # Abbrev [1] 0xb:0x4a DW_TAG_compile_unit
+        .asciz  "Hand-written DWARF"    # DW_AT_producer
+        .byte   3                       # Abbrev [3] 0x37:0x16 DW_TAG_structure_type
+        .asciz  "A"                     # DW_AT_name
+        .byte   4                       # DW_AT_byte_size
+        .byte   4                       # Abbrev [4] 0x40:0xc DW_TAG_member
+        .asciz  "member"                # DW_AT_name
+        .long   .Lint-.Lsection_info    # DW_AT_type
+        .byte   0                       # DW_AT_data_member_location
+        .byte   0                       # End Of Children Mark
+.Lint:
+        .byte   5                       # Abbrev [5] 0x4d:0x7 DW_TAG_base_type
+        .asciz  "int"                   # DW_AT_name
+        .byte   5                       # DW_AT_encoding
+        .byte   4                       # DW_AT_byte_size
+        .byte   0                       # End Of Children Mark
+.Ldebug_info_end0:
+.endif
+
+.ifdef EXE
+        .data
+        .globl  var
+        .p2align        2
+var:
+        .long   47
+
+        .section        .debug_info,"dr"
+.Lsection_info:
+.Lcu_begin0:
+        .long   .Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+        .short  4                       # DWARF version number
+        .secrel32       .Lsection_abbrev # Offset Into Abbrev. Section
+        .byte   8                       # Address Size (in bytes)
+        .byte   1                       # Abbrev [1] 0xb:0x4a DW_TAG_compile_unit
+        .asciz  "Hand-written DWARF"    # DW_AT_producer
+        .byte   2                       # Abbrev [2] 0x1e:0x19 DW_TAG_variable
+        .asciz  "var"                   # DW_AT_name
+        .long   .LA-.Lsection_info      # DW_AT_type
+        .byte   9                       # DW_AT_location
+        .byte   3
+        .quad   var
+.LA:
+        .byte   6                       # Abbrev [6] 0x37:0x16 DW_TAG_structure_type
+        .asciz  "A"                     # DW_AT_name
+                                        # DW_AT_declaration
+        .byte   0                       # End Of Children Mark
+.Ldebug_info_end0:
+.endif
diff --git a/lldb/test/Shell/SymbolFile/PDB/Inputs/AstRestoreTest.cpp b/lldb/test/Shell/SymbolFile/PDB/Inputs/AstRestoreTest.cpp
index 8c9e26744d5f3..fb20c9d734bd2 100644
--- a/lldb/test/Shell/SymbolFile/PDB/Inputs/AstRestoreTest.cpp
+++ b/lldb/test/Shell/SymbolFile/PDB/Inputs/AstRestoreTest.cpp
@@ -3,6 +3,7 @@ namespace N1 {
 
 namespace {
 enum Enum { Enum_0 = 1, Enum_1 = 2, Enum_2 = 4, Enum_3 = 8 };
+enum class ScopedEnum { Enum_0 = 1, Enum_1 = 2, Enum_2 = 4, Enum_3 = 8 };
 }
 
 Enum Global = Enum_3;
@@ -22,6 +23,13 @@ class Class : public Base {
   const Enum m_ce;
 
   static int ClassStatic;
+  static const int ClassStaticConst = 8;
+  static constexpr int ClassStaticConstexpr = 9;
+  static constexpr float ClassStaticConstexprFloat = 10.f;
+  static constexpr double ClassStaticConstexprDouble = 11.0;
+  static constexpr long double ClassStaticConstexprLongDouble = 12.0;
+  static const Enum ClassStaticConstEnum = Enum_3;
+  static const ScopedEnum ClassStaticConstScopedEnum = ScopedEnum::Enum_2;
 
 private:
   struct Inner {
diff --git a/lldb/test/Shell/SymbolFile/PDB/Inputs/ClassLayoutTest.cpp b/lldb/test/Shell/SymbolFile/PDB/Inputs/ClassLayoutTest.cpp
index 3c4b005cdf1be..503939680c500 100644
--- a/lldb/test/Shell/SymbolFile/PDB/Inputs/ClassLayoutTest.cpp
+++ b/lldb/test/Shell/SymbolFile/PDB/Inputs/ClassLayoutTest.cpp
@@ -106,6 +106,7 @@ class Class : public Base { // Test base class.
 int main() {
   MemberTest::Base B1;
   B1.Get();
+  MemberTest::Class C1;
   MemberTest::Class::StaticMemberFunc(1, 10, 2);
   return 0;
 }
diff --git a/lldb/test/Shell/SymbolFile/PDB/ast-restore.test b/lldb/test/Shell/SymbolFile/PDB/ast-restore.test
index 14ec0d52f2ed6..2473bb1d66f57 100644
--- a/lldb/test/Shell/SymbolFile/PDB/ast-restore.test
+++ b/lldb/test/Shell/SymbolFile/PDB/ast-restore.test
@@ -4,7 +4,8 @@ RUN: env LLDB_USE_NATIVE_PDB_READER=0 lldb-test symbols -dump-ast %t.exe | FileC
 RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=ENUM %s
 RUN: lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=GLOBAL %s
 RUN: lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=BASE %s
-RUN: lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=CLASS %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=0 lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=CLASS %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=CLASS %s
 RUN: lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=INNER %s
 RUN: lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=TEMPLATE %s
 RUN: lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=FOO %s
@@ -46,6 +47,13 @@ CLASS:     namespace N1 {
 CLASS:         class Class : public N0::N1::Base {
 CLASS-DAG:             const N0::N1::(anonymous namespace)::Enum m_ce;
 CLASS-DAG:             static int ClassStatic;
+CLASS-DAG:             static const int ClassStaticConst = 8;
+CLASS-DAG:             static const int ClassStaticConstexpr = 9;
+CLASS-DAG:             static constexpr float ClassStaticConstexprFloat = 10.F;
+CLASS-DAG:             static constexpr double ClassStaticConstexprDouble = 11.;
+CLASS-DAG:             static constexpr double ClassStaticConstexprLongDouble = 12.;
+CLASS-DAG:             static const N0::N1::(anonymous namespace)::Enum ClassStaticConstEnum = 8;
+CLASS-DAG:             static const N0::N1::(anonymous namespace)::ScopedEnum ClassStaticConstScopedEnum = 4;
 CLASS-DAG:             N0::N1::Class::Inner m_inner;
 CLASS-DAG:             {{(inline )?}}Class(N0::N1::(anonymous namespace)::Enum);
 CLASS-DAG:             static {{(inline )?}}int StaticFunc(const N0::N1::Class &);
diff --git a/lldb/test/Shell/Unwind/Inputs/eh-frame-augment-noop.s b/lldb/test/Shell/Unwind/Inputs/eh-frame-augment-noop.s
new file mode 100644
index 0000000000000..2bcf534f04fcf
--- /dev/null
+++ b/lldb/test/Shell/Unwind/Inputs/eh-frame-augment-noop.s
@@ -0,0 +1,22 @@
+# A function whose prologue and epilogue are described perfectly.  eh_frame
+# augmentation machinery should detect that no augmentation is needed and use
+# eh_frame directly.
+
+        .text
+        .globl  foo
+foo:
+        .cfi_startproc
+        pushq   %rax
+        .cfi_def_cfa_offset 16
+        int3
+        pop %rcx
+        .cfi_def_cfa_offset 8
+        retq
+        .cfi_endproc
+
+        .globl asm_main
+asm_main:
+        .cfi_startproc
+        callq foo
+        retq
+        .cfi_endproc
diff --git a/lldb/test/Shell/Unwind/eh-frame-augment-noop.test b/lldb/test/Shell/Unwind/eh-frame-augment-noop.test
new file mode 100644
index 0000000000000..900a3d21bc7a7
--- /dev/null
+++ b/lldb/test/Shell/Unwind/eh-frame-augment-noop.test
@@ -0,0 +1,23 @@
+# Test handing of dwarf expressions specifying the location of registers, if
+# those expressions refer to the frame's CFA value.
+
+# UNSUPPORTED: system-windows
+# UNSUPPORTED: system-darwin
+# REQUIRES: target-x86_64, native
+
+# RUN: %clang_host %p/Inputs/call-asm.c %p/Inputs/eh-frame-augment-noop.s -o %t
+# RUN: %lldb %t -s %s -o exit | FileCheck %s
+
+process launch
+# CHECK: stop reason
+
+thread backtrace
+# CHECK: frame #0: {{.*}}`foo + 2
+# CHECK: frame #1: {{.*}}`asm_main + 5
+
+target modules show-unwind -n foo
+# CHECK: Asynchronous (not restricted to call-sites) UnwindPlan is 'eh_frame CFI'
+# CHECK: eh_frame augmented UnwindPlan:
+# CHECK:      row[0]:  0: CFA=rsp +8 => rip=[CFA-8]
+# CHECK-NEXT: row[1]:  1: CFA=rsp+16 => rip=[CFA-8]
+# CHECK-NEXT: row[2]:  3: CFA=rsp +8 => rip=[CFA-8]
diff --git a/lldb/test/Shell/helper/build.py b/lldb/test/Shell/helper/build.py
index d2cb52f00e553..3de2f33503185 100755
--- a/lldb/test/Shell/helper/build.py
+++ b/lldb/test/Shell/helper/build.py
@@ -623,6 +623,9 @@ def output_files(self):
 class GccBuilder(Builder):
     def __init__(self, toolchain_type, args):
         Builder.__init__(self, toolchain_type, args, '.o')
+        if sys.platform == 'darwin':
+            cmd = ['xcrun', '--sdk', args.apple_sdk, '--show-sdk-path']
+            self.apple_sdk = subprocess.check_output(cmd).strip().decode('utf-8')
 
     def _get_compilation_command(self, source, obj):
         args = []
@@ -645,6 +648,9 @@ def _get_compilation_command(self, source, obj):
         args.extend(['-o', obj])
         args.append(source)
 
+        if sys.platform == 'darwin':
+            args.extend(['-isysroot', self.apple_sdk])
+
         return ('compiling', [source], obj, None, args)
 
     def _get_link_command(self):
@@ -664,6 +670,9 @@ def _get_link_command(self):
         args.extend(['-o', self._exe_file_name()])
         args.extend(self._obj_file_names())
 
+        if sys.platform == 'darwin':
+            args.extend(['-isysroot', self.apple_sdk])
+
         return ('linking', self._obj_file_names(), self._exe_file_name(), None, args)
 
 
diff --git a/lldb/tools/debugserver/source/CMakeLists.txt b/lldb/tools/debugserver/source/CMakeLists.txt
index 6c4fa026092b6..9a7e2eb9a1a0f 100644
--- a/lldb/tools/debugserver/source/CMakeLists.txt
+++ b/lldb/tools/debugserver/source/CMakeLists.txt
@@ -129,7 +129,7 @@ if(LLDB_USE_ENTITLEMENTS)
   endif()
 endif()
 
-if($ENV{SDKROOT} MATCHES ".Internal.sdk$")
+if(${CMAKE_OSX_SYSROOT} MATCHES ".Internal.sdk$")
   message(STATUS "LLDB debugserver energy support is enabled")
   add_definitions(-DLLDB_ENERGY)
   set(ENERGY_LIBRARY -lpmenergy -lpmsample)
diff --git a/lldb/tools/debugserver/source/DNB.cpp b/lldb/tools/debugserver/source/DNB.cpp
index b87ef5768a96e..af13a8f8208ba 100644
--- a/lldb/tools/debugserver/source/DNB.cpp
+++ b/lldb/tools/debugserver/source/DNB.cpp
@@ -1392,10 +1392,14 @@ const char *DNBGetDeploymentInfo(nub_process_t pid,
                                  uint32_t& minor_version,
                                  uint32_t& patch_version) {
   MachProcessSP procSP;
-  if (GetProcessSP(pid, procSP))
-    return procSP->GetDeploymentInfo(lc, load_command_address,
-                                     major_version, minor_version,
-                                     patch_version);
+  if (GetProcessSP(pid, procSP)) {
+    // FIXME: This doesn't correct for older ios simulator and macCatalyst.
+    auto info = procSP->GetDeploymentInfo(lc, load_command_address);
+    major_version = info.major_version;
+    minor_version = info.minor_version;
+    patch_version = info.patch_version;
+    return procSP->GetPlatformString(info.platform);
+  }
   return nullptr;
 }
 
diff --git a/lldb/tools/debugserver/source/MacOSX/MachProcess.h b/lldb/tools/debugserver/source/MacOSX/MachProcess.h
index e62051fbe0111..c749dd8426c5d 100644
--- a/lldb/tools/debugserver/source/MacOSX/MachProcess.h
+++ b/lldb/tools/debugserver/source/MacOSX/MachProcess.h
@@ -230,10 +230,22 @@ class MachProcess {
                          uint64_t plo_pthread_tsd_base_address_offset,
                          uint64_t plo_pthread_tsd_base_offset,
                          uint64_t plo_pthread_tsd_entry_size);
-  const char *
-  GetDeploymentInfo(const struct load_command&, uint64_t load_command_address,
-                    uint32_t& major_version, uint32_t& minor_version,
-                    uint32_t& patch_version);
+
+  struct DeploymentInfo {
+    DeploymentInfo() = default;
+    operator bool() { return platform > 0; }
+    /// The Mach-O platform type;
+    unsigned char platform = 0;
+    /// Pre-LC_BUILD_VERSION files don't disambiguate between ios and ios
+    /// simulator.
+    bool maybe_simulator = false;
+    uint32_t major_version = 0;
+    uint32_t minor_version = 0;
+    uint32_t patch_version = 0;
+  };
+  DeploymentInfo GetDeploymentInfo(const struct load_command &,
+                                   uint64_t load_command_address);
+  static const char *GetPlatformString(unsigned char platform);
   bool GetMachOInformationFromMemory(uint32_t platform,
                                      nub_addr_t mach_o_header_addr,
                                      int wordsize,
diff --git a/lldb/tools/debugserver/source/MacOSX/MachProcess.mm b/lldb/tools/debugserver/source/MacOSX/MachProcess.mm
index 74d20be42e315..8a35f605daa38 100644
--- a/lldb/tools/debugserver/source/MacOSX/MachProcess.mm
+++ b/lldb/tools/debugserver/source/MacOSX/MachProcess.mm
@@ -601,87 +601,77 @@ static bool FBSAddEventDataToOptions(NSMutableDictionary *options,
       plo_pthread_tsd_entry_size);
 }
 
-/// Determine whether this is running on macOS.
-/// Since debugserver runs on the same machine as the process, we can
-/// just look at the compilation target.
-static bool IsMacOSHost() {
-#if TARGET_OS_OSX == 1
-  return true;
-#else
-  return false;
-#endif
-}
-
-const char *MachProcess::GetDeploymentInfo(const struct load_command& lc,
-                                           uint64_t load_command_address,
-                                           uint32_t& major_version,
-                                           uint32_t& minor_version,
-                                           uint32_t& patch_version) {
+MachProcess::DeploymentInfo
+MachProcess::GetDeploymentInfo(const struct load_command &lc,
+                               uint64_t load_command_address) {
+  DeploymentInfo info;
   uint32_t cmd = lc.cmd & ~LC_REQ_DYLD;
-  bool lc_cmd_known =
-    cmd == LC_VERSION_MIN_IPHONEOS || cmd == LC_VERSION_MIN_MACOSX ||
-    cmd == LC_VERSION_MIN_TVOS || cmd == LC_VERSION_MIN_WATCHOS;
-
-  if (lc_cmd_known) {
+  // Handle the older LC_VERSION load commands, which don't
+  // distinguish between simulator and real hardware.
+  auto handle_version_min = [&](char platform) {
     struct version_min_command vers_cmd;
     if (ReadMemory(load_command_address, sizeof(struct version_min_command),
-                   &vers_cmd) != sizeof(struct version_min_command)) {
-      return nullptr;
-    }
-    major_version = vers_cmd.sdk >> 16;
-    minor_version = (vers_cmd.sdk >> 8) & 0xffu;
-    patch_version = vers_cmd.sdk & 0xffu;
-
-    // Handle the older LC_VERSION load commands, which don't
-    // distinguish between simulator and real hardware.
-    switch (cmd) {
-    case LC_VERSION_MIN_IPHONEOS:
-      return IsMacOSHost() ? "iossimulator": "ios";
-    case LC_VERSION_MIN_MACOSX:
-      return "macosx";
-    case LC_VERSION_MIN_TVOS:
-      return IsMacOSHost() ? "tvossimulator": "tvos";
-    case LC_VERSION_MIN_WATCHOS:
-      return IsMacOSHost() ? "watchossimulator" : "watchos";
-    default:
-      return nullptr;
-    }
-  }
-#if defined (LC_BUILD_VERSION)
-  if (cmd == LC_BUILD_VERSION) {
+                   &vers_cmd) != sizeof(struct version_min_command))
+      return;
+    info.platform = platform;
+    info.major_version = vers_cmd.version >> 16;
+    info.minor_version = (vers_cmd.version >> 8) & 0xffu;
+    info.patch_version = vers_cmd.version & 0xffu;
+    info.maybe_simulator = true;
+  };
+  switch (cmd) {
+  case LC_VERSION_MIN_IPHONEOS:
+    handle_version_min(PLATFORM_IOS);
+    break;
+  case LC_VERSION_MIN_MACOSX:
+    handle_version_min(PLATFORM_MACOS);
+    break;
+  case LC_VERSION_MIN_TVOS:
+    handle_version_min(PLATFORM_TVOS);
+    break;
+  case LC_VERSION_MIN_WATCHOS:
+    handle_version_min(PLATFORM_WATCHOS);
+    break;
+#if defined(LC_BUILD_VERSION)
+  case LC_BUILD_VERSION: {
     struct build_version_command build_vers;
     if (ReadMemory(load_command_address, sizeof(struct build_version_command),
-                   &build_vers) != sizeof(struct build_version_command)) {
-      return nullptr;
-    }
-    major_version = build_vers.sdk >> 16;;
-    minor_version = (build_vers.sdk >> 8) & 0xffu;
-    patch_version = build_vers.sdk & 0xffu;
-
-    switch (build_vers.platform) {
-    case PLATFORM_MACOS:
-      return "macosx";
-    case PLATFORM_MACCATALYST:
-      return "maccatalyst";
-    case PLATFORM_IOS:
-      return "ios";
-    case PLATFORM_IOSSIMULATOR:
-      return "iossimulator";
-    case PLATFORM_TVOS:
-      return "tvos";
-    case PLATFORM_TVOSSIMULATOR:
-      return "tvossimulator";
-    case PLATFORM_WATCHOS:
-      return "watchos";
-    case PLATFORM_WATCHOSSIMULATOR:
-      return "watchossimulator";
-    case PLATFORM_BRIDGEOS:
-      return "bridgeos";
-    case PLATFORM_DRIVERKIT:
-      return "driverkit";
-    }
+                   &build_vers) != sizeof(struct build_version_command))
+      break;
+    info.platform = build_vers.platform;
+    info.major_version = build_vers.minos >> 16;
+    info.minor_version = (build_vers.minos >> 8) & 0xffu;
+    info.patch_version = build_vers.minos & 0xffu;
+    break;
   }
 #endif
+  }
+  return info;
+}
+
+const char *MachProcess::GetPlatformString(unsigned char platform) {
+  switch (platform) {
+  case PLATFORM_MACOS:
+    return "macosx";
+  case PLATFORM_MACCATALYST:
+    return "maccatalyst";
+  case PLATFORM_IOS:
+    return "ios";
+  case PLATFORM_IOSSIMULATOR:
+    return "iossimulator";
+  case PLATFORM_TVOS:
+    return "tvos";
+  case PLATFORM_TVOSSIMULATOR:
+    return "tvossimulator";
+  case PLATFORM_WATCHOS:
+    return "watchos";
+  case PLATFORM_WATCHOSSIMULATOR:
+    return "watchossimulator";
+  case PLATFORM_BRIDGEOS:
+    return "bridgeos";
+  case PLATFORM_DRIVERKIT:
+    return "driverkit";
+  }
   return nullptr;
 }
 
@@ -787,10 +777,36 @@ static bool IsMacOSHost() {
           sizeof(struct uuid_command))
         uuid_copy(inf.uuid, uuidcmd.uuid);
     }
-
-    uint32_t major_version, minor_version, patch_version;
-    if (const char *lc_platform = GetDeploymentInfo(
-            lc, load_cmds_p, major_version, minor_version, patch_version)) {
+    if (DeploymentInfo deployment_info = GetDeploymentInfo(lc, load_cmds_p)) {
+      // Simulator support. If the platform is ambiguous, use the dyld info.
+      if (deployment_info.maybe_simulator) {
+        if (deployment_info.maybe_simulator) {
+#if (defined(__x86_64__) || defined(__i386__))
+          // If dyld doesn't return a platform, use a heuristic.
+          // If we are running on Intel macOS, it is safe to assume
+          // this is really a back-deploying simulator binary.
+          switch (deployment_info.platform) {
+          case PLATFORM_IOS:
+            deployment_info.platform = PLATFORM_IOSSIMULATOR;
+            break;
+          case PLATFORM_TVOS:
+            deployment_info.platform = PLATFORM_TVOSSIMULATOR;
+            break;
+          case PLATFORM_WATCHOS:
+            deployment_info.platform = PLATFORM_WATCHOSSIMULATOR;
+            break;
+          }
+#else
+          // On an Apple Silicon macOS host, there is no
+          // ambiguity. The only binaries that use legacy load
+          // commands are back-deploying native iOS binaries. All
+          // simulator binaries use the newer, unambiguous
+          // LC_BUILD_VERSION load commands.
+          deployment_info.maybe_simulator = false;
+#endif
+        }
+      }
+      const char *lc_platform = GetPlatformString(deployment_info.platform);
       // macCatalyst support.
       //
       // This handles two special cases:
@@ -824,12 +840,15 @@ static bool IsMacOSHost() {
       } else {
         inf.min_version_os_name = lc_platform;
         inf.min_version_os_version = "";
-        inf.min_version_os_version += std::to_string(major_version);
+        inf.min_version_os_version +=
+            std::to_string(deployment_info.major_version);
         inf.min_version_os_version += ".";
-        inf.min_version_os_version += std::to_string(minor_version);
-        if (patch_version != 0) {
+        inf.min_version_os_version +=
+            std::to_string(deployment_info.minor_version);
+        if (deployment_info.patch_version != 0) {
           inf.min_version_os_version += ".";
-          inf.min_version_os_version += std::to_string(patch_version);
+          inf.min_version_os_version +=
+              std::to_string(deployment_info.patch_version);
         }
       }
     }
diff --git a/lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.cpp b/lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.cpp
index 3e7bda88e6af3..7e1af7a750215 100644
--- a/lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.cpp
+++ b/lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.cpp
@@ -590,23 +590,21 @@ kern_return_t DNBArchMachARM64::EnableHardwareSingleStep(bool enable) {
     return err.Status();
   }
 
-  if (enable) {
-    DNBLogThreadedIf(LOG_STEP,
-                     "%s: Setting MDSCR_EL1 Single Step bit at pc 0x%llx",
 #if defined(__LP64__)
-                     __FUNCTION__, (uint64_t)arm_thread_state64_get_pc (m_state.context.gpr));
+  uint64_t pc = arm_thread_state64_get_pc (m_state.context.gpr);
 #else
-                     __FUNCTION__, (uint64_t)m_state.context.gpr.__pc);
+  uint64_t pc = m_state.context.gpr.__pc;
 #endif
+
+  if (enable) {
+    DNBLogThreadedIf(LOG_STEP,
+                     "%s: Setting MDSCR_EL1 Single Step bit at pc 0x%llx",
+                     __FUNCTION__, pc);
     m_state.dbg.__mdscr_el1 |= SS_ENABLE;
   } else {
     DNBLogThreadedIf(LOG_STEP,
                      "%s: Clearing MDSCR_EL1 Single Step bit at pc 0x%llx",
-#if defined(__LP64__)
-                     __FUNCTION__, (uint64_t)arm_thread_state64_get_pc (m_state.context.gpr));
-#else
-                     __FUNCTION__, (uint64_t)m_state.context.gpr.__pc);
-#endif
+                     __FUNCTION__, pc);
     m_state.dbg.__mdscr_el1 &= ~(SS_ENABLE);
   }
 
diff --git a/lldb/tools/debugserver/source/RNBRemote.cpp b/lldb/tools/debugserver/source/RNBRemote.cpp
index 5f066bda42275..8a3045564aaf1 100644
--- a/lldb/tools/debugserver/source/RNBRemote.cpp
+++ b/lldb/tools/debugserver/source/RNBRemote.cpp
@@ -4908,6 +4908,8 @@ rnb_err_t RNBRemote::HandlePacket_qHostInfo(const char *p) {
     strm << "ostype:watchos;";
 #elif defined(TARGET_OS_BRIDGE) && TARGET_OS_BRIDGE == 1
     strm << "ostype:bridgeos;";
+#elif defined(TARGET_OS_OSX) && TARGET_OS_OSX == 1
+    strm << "ostype:macosx;";
 #else
     strm << "ostype:ios;";
 #endif
@@ -6336,7 +6338,7 @@ rnb_err_t RNBRemote::HandlePacket_qProcessInfo(const char *p) {
   if (addr_size > 0) {
     rep << "ptrsize:" << std::dec << addr_size << ';';
 
-#if (defined(__x86_64__) || defined(__i386__))
+#if defined(TARGET_OS_OSX) && TARGET_OS_OSX == 1
     // Try and get the OS type by looking at the load commands in the main
     // executable and looking for a LC_VERSION_MIN load command. This is the
     // most reliable way to determine the "ostype" value when on desktop.
@@ -6366,7 +6368,7 @@ rnb_err_t RNBRemote::HandlePacket_qProcessInfo(const char *p) {
         load_command_addr = load_command_addr + lc.cmdsize;
       }
     }
-#endif // when compiling this on x86 targets
+#endif // TARGET_OS_OSX
   }
 
   // If we weren't able to find the OS in a LC_VERSION_MIN load command, try
@@ -6383,6 +6385,8 @@ rnb_err_t RNBRemote::HandlePacket_qProcessInfo(const char *p) {
       rep << "ostype:watchos;";
 #elif defined(TARGET_OS_BRIDGE) && TARGET_OS_BRIDGE == 1
       rep << "ostype:bridgeos;";
+#elif defined(TARGET_OS_OSX) && TARGET_OS_OSX == 1
+      rep << "ostype:macosx;";
 #else
       rep << "ostype:ios;";
 #endif
diff --git a/lldb/tools/driver/Driver.cpp b/lldb/tools/driver/Driver.cpp
index 1e639e9a1deab..cea9e5a44aa80 100644
--- a/lldb/tools/driver/Driver.cpp
+++ b/lldb/tools/driver/Driver.cpp
@@ -797,11 +797,12 @@ static void printHelp(LLDBOptTable &table, llvm::StringRef tool_name) {
   llvm::outs() << examples << '\n';
 }
 
-llvm::Optional<int> InitializeReproducer(opt::InputArgList &input_args) {
+llvm::Optional<int> InitializeReproducer(llvm::StringRef argv0,
+                                         opt::InputArgList &input_args) {
   if (auto *replay_path = input_args.getLastArg(OPT_replay)) {
-    const bool skip_version_check = input_args.hasArg(OPT_skip_version_check);
+    const bool no_version_check = input_args.hasArg(OPT_no_version_check);
     if (const char *error =
-            SBReproducer::Replay(replay_path->getValue(), skip_version_check)) {
+            SBReproducer::Replay(replay_path->getValue(), no_version_check)) {
       WithColor::error() << "reproducer replay failed: " << error << '\n';
       return 1;
     }
@@ -809,15 +810,21 @@ llvm::Optional<int> InitializeReproducer(opt::InputArgList &input_args) {
   }
 
   bool capture = input_args.hasArg(OPT_capture);
-  bool auto_generate = input_args.hasArg(OPT_auto_generate);
+  bool generate_on_exit = input_args.hasArg(OPT_generate_on_exit);
   auto *capture_path = input_args.getLastArg(OPT_capture_path);
 
-  if (auto_generate && !capture) {
+  if (generate_on_exit && !capture) {
     WithColor::warning()
-        << "-reproducer-auto-generate specified without -capture\n";
+        << "-reproducer-generate-on-exit specified without -capture\n";
   }
 
   if (capture || capture_path) {
+    // Register the reproducer signal handler.
+    if (!input_args.hasArg(OPT_no_generate_on_signal)) {
+      llvm::sys::AddSignalHandler(reproducer_handler,
+                                  const_cast<char *>(argv0.data()));
+    }
+
     if (capture_path) {
       if (!capture)
         WithColor::warning() << "-capture-path specified without -capture\n";
@@ -832,7 +839,7 @@ llvm::Optional<int> InitializeReproducer(opt::InputArgList &input_args) {
         return 1;
       }
     }
-    if (auto_generate)
+    if (generate_on_exit)
       SBReproducer::SetAutoGenerate(true);
   }
 
@@ -867,13 +874,10 @@ int main(int argc, char const *argv[]) {
     return 1;
   }
 
-  if (auto exit_code = InitializeReproducer(input_args)) {
+  if (auto exit_code = InitializeReproducer(argv[0], input_args)) {
     return *exit_code;
   }
 
-  // Register the reproducer signal handler.
-  llvm::sys::AddSignalHandler(reproducer_handler, const_cast<char *>(argv[0]));
-
   SBError error = SBDebugger::InitializeWithErrorHandling();
   if (error.Fail()) {
     WithColor::error() << "initialization failed: " << error.GetCString()
diff --git a/lldb/tools/driver/Options.td b/lldb/tools/driver/Options.td
index e459a153d6185..96f696ec3ca6e 100644
--- a/lldb/tools/driver/Options.td
+++ b/lldb/tools/driver/Options.td
@@ -232,9 +232,11 @@ def capture_path: Separate<["--", "-"], "capture-path">,
 def replay: Separate<["--", "-"], "replay">,
   MetaVarName<"<filename>">,
   HelpText<"Tells the debugger to replay a reproducer from <filename>.">;
-def skip_version_check: F<"reproducer-skip-version-check">,
-  HelpText<"Skip the reproducer version check.">;
-def auto_generate: F<"reproducer-auto-generate">,
+def no_version_check: F<"reproducer-no-version-check">,
+  HelpText<"Disable the reproducer version check.">;
+def no_generate_on_signal: F<"reproducer-no-generate-on-signal">,
+  HelpText<"Don't generate reproducer when a signal is received.">;
+def generate_on_exit: F<"reproducer-generate-on-exit">,
   HelpText<"Generate reproducer on exit.">;
 
 def REM : R<["--"], "">;
diff --git a/lldb/tools/intel-features/intel-mpx/cli-wrapper-mpxtable.cpp b/lldb/tools/intel-features/intel-mpx/cli-wrapper-mpxtable.cpp
index 5bffd27409e8c..d2ef0445bc671 100644
--- a/lldb/tools/intel-features/intel-mpx/cli-wrapper-mpxtable.cpp
+++ b/lldb/tools/intel-features/intel-mpx/cli-wrapper-mpxtable.cpp
@@ -19,6 +19,7 @@
 #include "lldb/API/SBThread.h"
 
 #include "llvm/ADT/Triple.h"
+#include "llvm/ADT/Twine.h"
 
 static bool GetPtr(char *cptr, uint64_t &ptr, lldb::SBFrame &frame,
                    lldb::SBCommandReturnObject &result) {
diff --git a/lldb/tools/lldb-vscode/README.md b/lldb/tools/lldb-vscode/README.md
index 20c713164c293..70ca245f85b92 100644
--- a/lldb/tools/lldb-vscode/README.md
+++ b/lldb/tools/lldb-vscode/README.md
@@ -16,14 +16,14 @@
 
 The `lldb-vscode` tool creates a command line tool that implements the [Visual
 Studio Code Debug API](https://code.visualstudio.com/docs/extensionAPI/api-debugging).
-It can be installed as an extension for the Visual Studio Code and Nuclide IDE. 
+It can be installed as an extension for the Visual Studio Code and Nuclide IDE.
 The protocol is easy to run remotely and also can allow other tools and IDEs to
-get a full featured debugger with a well defined protocol. 
+get a full featured debugger with a well defined protocol.
 
 # Installation for Visual Studio Code
 
 Installing the plug-in involves creating a directory in the `~/.vscode/extensions` folder and copying the package.json file that is in the same directory as this
-documentation into it, and copying to symlinking a lldb-vscode binary into 
+documentation into it, and copying to symlinking a lldb-vscode binary into
 the `bin` directory inside the plug-in directory.
 
 If you want to make a stand alone plug-in that you can send to others on unix systems:
@@ -86,6 +86,7 @@ file that defines how your program will be run. The JSON configuration file can
 |**preRunCommands** |[string]| | LLDB commands executed just before launching after the LLDB target has been created. Commands and command output will be sent to the debugger console when they are executed.
 |**stopCommands**   |[string]| | LLDB commands executed just after each stop. Commands and command output will be sent to the debugger console when they are executed.
 |**exitCommands**   |[string]| | LLDB commands executed when the program exits. Commands and command output will be sent to the debugger console when they are executed.
+|**terminateCommands** |[string]| | LLDB commands executed when the debugging session ends. Commands and command output will be sent to the debugger console when they are executed.
 |**sourceMap**      |[string[2]]| | Specify an array of path re-mappings. Each element in the array must be a two element array containing a source and destination pathname.
 |**debuggerRoot**   | string| |Specify a working directory to use when launching lldb-vscode. If the debug information in your executable contains relative paths, this option can be used so that `lldb-vscode` can find source files and object files that have relative paths.
 
@@ -112,6 +113,7 @@ The JSON configuration file can contain the following `lldb-vscode` specific lau
 |**preRunCommands** |[string]| | LLDB commands executed just before launching after the LLDB target has been created. Commands and command output will be sent to the debugger console when they are executed.
 |**stopCommands**   |[string]| | LLDB commands executed just after each stop. Commands and command output will be sent to the debugger console when they are executed.
 |**exitCommands**   |[string]| | LLDB commands executed when the program exits. Commands and command output will be sent to the debugger console when they are executed.
+|**terminateCommands** |[string]| | LLDB commands executed when the debugging session ends. Commands and command output will be sent to the debugger console when they are executed.
 |**attachCommands** |[string]| | LLDB commands that will be executed after **preRunCommands** which take place of the code that normally does the attach. The commands can create a new target and attach or launch it however desired. This allows custom launch and attach configurations. Core files can use `target create --core /path/to/core` to attach to core files.
 
 
diff --git a/lldb/tools/lldb-vscode/VSCode.cpp b/lldb/tools/lldb-vscode/VSCode.cpp
index 36bc8ec8ebfd1..b2d16f96d1f2b 100644
--- a/lldb/tools/lldb-vscode/VSCode.cpp
+++ b/lldb/tools/lldb-vscode/VSCode.cpp
@@ -38,7 +38,7 @@ VSCode::VSCode()
            {"swift_catch", "Swift Catch", lldb::eLanguageTypeSwift},
            {"swift_throw", "Swift Throw", lldb::eLanguageTypeSwift}}),
       focus_tid(LLDB_INVALID_THREAD_ID), sent_terminated_event(false),
-      stop_at_entry(false) {
+      stop_at_entry(false), is_attach(false) {
   const char *log_file_path = getenv("LLDBVSCODE_LOG");
 #if defined(_WIN32)
 // Windows opens stdout and stdin in text mode which converts \n to 13,10
@@ -309,6 +309,10 @@ void VSCode::RunExitCommands() {
   RunLLDBCommands("Running exitCommands:", exit_commands);
 }
 
+void VSCode::RunTerminateCommands() {
+  RunLLDBCommands("Running terminateCommands:", terminate_commands);
+}
+
 lldb::SBTarget VSCode::CreateTargetFromArguments(
     const llvm::json::Object &arguments,
     lldb::SBError &error) {
diff --git a/lldb/tools/lldb-vscode/VSCode.h b/lldb/tools/lldb-vscode/VSCode.h
index 5298d7554f4de..ddb4c09a41743 100644
--- a/lldb/tools/lldb-vscode/VSCode.h
+++ b/lldb/tools/lldb-vscode/VSCode.h
@@ -86,9 +86,11 @@ struct VSCode {
   std::vector<std::string> pre_run_commands;
   std::vector<std::string> exit_commands;
   std::vector<std::string> stop_commands;
+  std::vector<std::string> terminate_commands;
   lldb::tid_t focus_tid;
   bool sent_terminated_event;
   bool stop_at_entry;
+  bool is_attach;
   // Keep track of the last stop thread index IDs as threads won't go away
   // unless we send a "thread" event to indicate the thread exited.
   llvm::DenseSet<lldb::tid_t> thread_ids;
@@ -132,6 +134,7 @@ struct VSCode {
   void RunPreRunCommands();
   void RunStopCommands();
   void RunExitCommands();
+  void RunTerminateCommands();
 
   /// Create a new SBTarget object from the given request arguments.
   /// \param[in] arguments
diff --git a/lldb/tools/lldb-vscode/lldb-vscode.cpp b/lldb/tools/lldb-vscode/lldb-vscode.cpp
index f1620d945fbc2..168873f827527 100644
--- a/lldb/tools/lldb-vscode/lldb-vscode.cpp
+++ b/lldb/tools/lldb-vscode/lldb-vscode.cpp
@@ -174,6 +174,7 @@ void SendThreadExitedEvent(lldb::tid_t tid) {
 void SendTerminatedEvent() {
   if (!g_vsc.sent_terminated_event) {
     g_vsc.sent_terminated_event = true;
+    g_vsc.RunTerminateCommands();
     // Send a "terminated" event
     llvm::json::Object event(CreateEventObject("terminated"));
     g_vsc.SendJSON(llvm::json::Value(std::move(event)));
@@ -514,6 +515,7 @@ void SetSourceMapFromArguments(const llvm::json::Object &arguments) {
 //   }]
 // }
 void request_attach(const llvm::json::Object &request) {
+  g_vsc.is_attach = true;
   llvm::json::Object response;
   lldb::SBError error;
   FillResponse(request, response);
@@ -529,6 +531,7 @@ void request_attach(const llvm::json::Object &request) {
   g_vsc.pre_run_commands = GetStrings(arguments, "preRunCommands");
   g_vsc.stop_commands = GetStrings(arguments, "stopCommands");
   g_vsc.exit_commands = GetStrings(arguments, "exitCommands");
+  g_vsc.terminate_commands = GetStrings(arguments, "terminateCommands");
   auto attachCommands = GetStrings(arguments, "attachCommands");
   llvm::StringRef core_file = GetString(arguments, "coreFile");
   g_vsc.stop_at_entry =
@@ -769,10 +772,11 @@ void request_disconnect(const llvm::json::Object &request) {
   FillResponse(request, response);
   auto arguments = request.getObject("arguments");
 
-  bool terminateDebuggee = GetBoolean(arguments, "terminateDebuggee", false);
+  bool defaultTerminateDebuggee = g_vsc.is_attach ? false : true;
+  bool terminateDebuggee =
+      GetBoolean(arguments, "terminateDebuggee", defaultTerminateDebuggee);
   lldb::SBProcess process = g_vsc.target.GetProcess();
   auto state = process.GetState();
-
   switch (state) {
   case lldb::eStateInvalid:
   case lldb::eStateUnloaded:
@@ -788,15 +792,14 @@ void request_disconnect(const llvm::json::Object &request) {
   case lldb::eStateStopped:
   case lldb::eStateRunning:
     g_vsc.debugger.SetAsync(false);
-    if (terminateDebuggee)
-      process.Kill();
-    else
-      process.Detach();
+    lldb::SBError error = terminateDebuggee ? process.Kill() : process.Detach();
+    if (!error.Success())
+      response.try_emplace("error", error.GetCString());
     g_vsc.debugger.SetAsync(true);
     break;
   }
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
   SendTerminatedEvent();
+  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
   if (g_vsc.event_thread.joinable()) {
     g_vsc.broadcaster.BroadcastEventByType(eBroadcastBitStopEventThread);
     g_vsc.event_thread.join();
@@ -1357,6 +1360,7 @@ void request_initialize(const llvm::json::Object &request) {
 //   }]
 // }
 void request_launch(const llvm::json::Object &request) {
+  g_vsc.is_attach = false;
   llvm::json::Object response;
   lldb::SBError error;
   FillResponse(request, response);
@@ -1365,6 +1369,7 @@ void request_launch(const llvm::json::Object &request) {
   g_vsc.pre_run_commands = GetStrings(arguments, "preRunCommands");
   g_vsc.stop_commands = GetStrings(arguments, "stopCommands");
   g_vsc.exit_commands = GetStrings(arguments, "exitCommands");
+  g_vsc.terminate_commands = GetStrings(arguments, "terminateCommands");
   auto launchCommands = GetStrings(arguments, "launchCommands");
   g_vsc.stop_at_entry = GetBoolean(arguments, "stopOnEntry", false);
   const llvm::StringRef debuggerRoot = GetString(arguments, "debuggerRoot");
diff --git a/lldb/unittests/Expression/DWARFExpressionTest.cpp b/lldb/unittests/Expression/DWARFExpressionTest.cpp
index 590ac2fa8338b..78c400c33d4e3 100644
--- a/lldb/unittests/Expression/DWARFExpressionTest.cpp
+++ b/lldb/unittests/Expression/DWARFExpressionTest.cpp
@@ -61,35 +61,8 @@ class DWARFExpressionTester : public YAMLModuleTester {
 
 /// Unfortunately Scalar's operator==() is really picky.
 static Scalar GetScalar(unsigned bits, uint64_t value, bool sign) {
-  Scalar scalar;
-  auto type = Scalar::GetBestTypeForBitSize(bits, sign);
-  switch (type) {
-  case Scalar::e_sint:
-    scalar = Scalar((int)value);
-    break;
-  case Scalar::e_slong:
-    scalar = Scalar((long)value);
-    break;
-  case Scalar::e_slonglong:
-    scalar = Scalar((long long)value);
-    break;
-  case Scalar::e_uint:
-    scalar = Scalar((unsigned int)value);
-    break;
-  case Scalar::e_ulong:
-    scalar = Scalar((unsigned long)value);
-    break;
-  case Scalar::e_ulonglong:
-    scalar = Scalar((unsigned long long)value);
-    break;
-  default:
-    llvm_unreachable("not implemented");
-  }
-  scalar.TruncOrExtendTo(type, bits);
-  if (sign)
-    scalar.MakeSigned();
-  else
-    scalar.MakeUnsigned();
+  Scalar scalar(value);
+  scalar.TruncOrExtendTo(bits, sign);
   return scalar;
 }
 
@@ -121,8 +94,7 @@ TEST(DWARFExpression, DW_OP_convert) {
       "      - Attribute:       DW_AT_byte_size\n"
       "        Form:            DW_FORM_data1\n"
       "debug_info:\n"
-      "  - Length:\n"
-      "      TotalLength:     0\n"
+      "  - Length:          0\n"
       "    Version:         4\n"
       "    AbbrOffset:      0\n"
       "    AddrSize:        8\n"
diff --git a/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp b/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp
index 78234612500b1..f661835d191b1 100644
--- a/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp
+++ b/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp
@@ -8,9 +8,10 @@
 
 #include "gtest/gtest.h"
 
+#include "Plugins/ScriptInterpreter/Python/lldb-python.h"
+
 #include "Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.h"
 #include "Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h"
-#include "Plugins/ScriptInterpreter/Python/lldb-python.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/HostInfo.h"
 
diff --git a/lldb/unittests/Symbol/Inputs/inlined-functions.yaml b/lldb/unittests/Symbol/Inputs/inlined-functions.yaml
index 14573da3498d4..8ce92115a0b56 100644
--- a/lldb/unittests/Symbol/Inputs/inlined-functions.yaml
+++ b/lldb/unittests/Symbol/Inputs/inlined-functions.yaml
@@ -342,8 +342,7 @@ DWARF:
         - Attribute:       DW_AT_type
           Form:            DW_FORM_ref4
   debug_info:      
-    - Length:          
-        TotalLength:     579
+    - Length:          579
       Version:         4
       AbbrOffset:      0
       AddrSize:        8
diff --git a/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp b/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
index 44251e944df62..858aa61b19bb8 100644
--- a/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
+++ b/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
@@ -56,8 +56,7 @@ TEST_F(DWARFASTParserClangTests,
       "      - Attribute:       DW_AT_byte_size\n"
       "        Form:            DW_FORM_data1\n"
       "debug_info:\n"
-      "  - Length:\n"
-      "      TotalLength:     0\n"
+      "  - Length:          0\n"
       "    Version:         4\n"
       "    AbbrOffset:      0\n"
       "    AddrSize:        8\n"
diff --git a/lldb/unittests/SymbolFile/DWARF/XcodeSDKModuleTests.cpp b/lldb/unittests/SymbolFile/DWARF/XcodeSDKModuleTests.cpp
index ac2c9fed94456..399eaf7aa6484 100644
--- a/lldb/unittests/SymbolFile/DWARF/XcodeSDKModuleTests.cpp
+++ b/lldb/unittests/SymbolFile/DWARF/XcodeSDKModuleTests.cpp
@@ -47,8 +47,7 @@ TEST_F(XcodeSDKModuleTests, TestModuleGetXcodeSDK) {
       - Attribute:       DW_AT_APPLE_sdk
         Form:            DW_FORM_strp
 debug_info:
-  - Length:
-      TotalLength:     8
+  - Length:          8
     Version:         2
     AbbrOffset:      0
     AddrSize:        8
diff --git a/lldb/unittests/Utility/DataExtractorTest.cpp b/lldb/unittests/Utility/DataExtractorTest.cpp
index 109c15297b165..536e69755d17b 100644
--- a/lldb/unittests/Utility/DataExtractorTest.cpp
+++ b/lldb/unittests/Utility/DataExtractorTest.cpp
@@ -299,3 +299,105 @@ TEST(DataExtractorTest, GetULEB128_bit63) {
   EXPECT_EQ(expected, BE.GetULEB128(&offset));
   EXPECT_EQ(9U, offset);
 }
+
+TEST(DataExtractorTest, GetFloat) {
+  float expected = 4.0f;
+  lldb::offset_t offset;
+
+  {
+    uint8_t buffer[] = {0x00, 0x00, 0x80, 0x40};
+    DataExtractor LE(buffer, sizeof(buffer), lldb::eByteOrderLittle,
+                     sizeof(void *));
+
+    offset = 0;
+    EXPECT_DOUBLE_EQ(expected, LE.GetFloat(&offset));
+    EXPECT_EQ(4U, offset);
+  }
+
+  {
+    uint8_t buffer[] = {0x40, 0x80, 0x00, 0x00};
+    DataExtractor BE(buffer, sizeof(buffer), lldb::eByteOrderBig,
+                     sizeof(void *));
+    offset = 0;
+    EXPECT_DOUBLE_EQ(expected, BE.GetFloat(&offset));
+    EXPECT_EQ(4U, offset);
+  }
+}
+
+TEST(DataExtractorTest, GetFloatUnaligned) {
+  float expected = 4.0f;
+  lldb::offset_t offset;
+
+  {
+    uint8_t buffer[] = {0x00, 0x00, 0x00, 0x80, 0x40};
+    DataExtractor LE(buffer, sizeof(buffer), lldb::eByteOrderLittle,
+                     sizeof(void *));
+
+    offset = 1;
+    EXPECT_DOUBLE_EQ(expected, LE.GetFloat(&offset));
+    EXPECT_EQ(5U, offset);
+  }
+
+  {
+    uint8_t buffer[] = {0x00, 0x40, 0x80, 0x00, 0x00};
+    DataExtractor BE(buffer, sizeof(buffer), lldb::eByteOrderBig,
+                     sizeof(void *));
+    offset = 1;
+    EXPECT_DOUBLE_EQ(expected, BE.GetFloat(&offset));
+    EXPECT_EQ(5U, offset);
+  }
+}
+
+TEST(DataExtractorTest, GetDouble) {
+  if (sizeof(double) != 8)
+    return;
+
+  double expected = 4.0f;
+  lldb::offset_t offset;
+
+  {
+    uint8_t buffer[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x40};
+    DataExtractor LE(buffer, sizeof(buffer), lldb::eByteOrderLittle,
+                     sizeof(void *));
+
+    offset = 0;
+    EXPECT_DOUBLE_EQ(expected, LE.GetDouble(&offset));
+    EXPECT_EQ(8U, offset);
+  }
+
+  {
+    uint8_t buffer[] = {0x40, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+    DataExtractor BE(buffer, sizeof(buffer), lldb::eByteOrderBig,
+                     sizeof(void *));
+    offset = 0;
+    EXPECT_DOUBLE_EQ(expected, BE.GetDouble(&offset));
+    EXPECT_EQ(8U, offset);
+  }
+}
+
+TEST(DataExtractorTest, GetDoubleUnaligned) {
+  if (sizeof(double) != 8)
+    return;
+
+  float expected = 4.0f;
+  lldb::offset_t offset;
+
+  {
+    uint8_t buffer[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x40};
+    DataExtractor LE(buffer, sizeof(buffer), lldb::eByteOrderLittle,
+                     sizeof(void *));
+
+    offset = 1;
+    EXPECT_DOUBLE_EQ(expected, LE.GetDouble(&offset));
+    EXPECT_EQ(9U, offset);
+  }
+
+  {
+    uint8_t buffer[] = {0x00, 0x40, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+    DataExtractor BE(buffer, sizeof(buffer), lldb::eByteOrderBig,
+                     sizeof(void *));
+    offset = 1;
+    EXPECT_DOUBLE_EQ(expected, BE.GetDouble(&offset));
+    EXPECT_EQ(9U, offset);
+  }
+}
diff --git a/lldb/unittests/Utility/ReproducerTest.cpp b/lldb/unittests/Utility/ReproducerTest.cpp
index 1ada0b32827cd..5a9dea3450f0e 100644
--- a/lldb/unittests/Utility/ReproducerTest.cpp
+++ b/lldb/unittests/Utility/ReproducerTest.cpp
@@ -9,6 +9,7 @@
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Testing/Support/Error.h"
 
@@ -31,8 +32,25 @@ class DummyProvider : public repro::Provider<DummyProvider> {
   static char ID;
 };
 
+class YamlMultiProvider
+    : public MultiProvider<YamlRecorder, YamlMultiProvider> {
+public:
+  struct Info {
+    static const char *name;
+    static const char *file;
+  };
+
+  YamlMultiProvider(const FileSpec &directory) : MultiProvider(directory) {}
+
+  static char ID;
+};
+
 const char *DummyProvider::Info::name = "dummy";
 const char *DummyProvider::Info::file = "dummy.yaml";
+const char *YamlMultiProvider::Info::name = "mutli";
+const char *YamlMultiProvider::Info::file = "mutli.yaml";
+char DummyProvider::ID = 0;
+char YamlMultiProvider::ID = 0;
 
 class DummyReproducer : public Reproducer {
 public:
@@ -42,7 +60,25 @@ class DummyReproducer : public Reproducer {
   using Reproducer::SetReplay;
 };
 
-char DummyProvider::ID = 0;
+struct YamlData {
+  YamlData() : i(-1) {}
+  YamlData(int i) : i(i) {}
+  int i;
+};
+
+inline bool operator==(const YamlData &LHS, const YamlData &RHS) {
+  return LHS.i == RHS.i;
+}
+
+LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(YamlData)
+
+namespace llvm {
+namespace yaml {
+template <> struct MappingTraits<YamlData> {
+  static void mapping(IO &io, YamlData &Y) { io.mapRequired("i", Y.i); };
+};
+} // namespace yaml
+} // namespace llvm
 
 TEST(ReproducerTest, SetCapture) {
   DummyReproducer reproducer;
@@ -144,3 +180,83 @@ TEST(GeneratorTest, GetOrCreate) {
   auto &provider_alt = generator.GetOrCreate<DummyProvider>();
   EXPECT_EQ(&provider, &provider_alt);
 }
+
+TEST(GeneratorTest, YamlMultiProvider) {
+  SmallString<128> root;
+  std::error_code ec = llvm::sys::fs::createUniqueDirectory("reproducer", root);
+  ASSERT_FALSE(static_cast<bool>(ec));
+
+  auto cleanup = llvm::make_scope_exit(
+      [&] { EXPECT_FALSE(llvm::sys::fs::remove_directories(root.str())); });
+
+  YamlData data0(0);
+  YamlData data1(1);
+  YamlData data2(2);
+  YamlData data3(3);
+
+  {
+    DummyReproducer reproducer;
+    EXPECT_THAT_ERROR(reproducer.SetCapture(FileSpec(root.str())), Succeeded());
+
+    auto &generator = *reproducer.GetGenerator();
+    auto *provider = generator.Create<YamlMultiProvider>();
+    ASSERT_NE(nullptr, provider);
+
+    auto *recorder = provider->GetNewRecorder();
+    ASSERT_NE(nullptr, recorder);
+    recorder->Record(data0);
+    recorder->Record(data1);
+
+    recorder = provider->GetNewRecorder();
+    ASSERT_NE(nullptr, recorder);
+    recorder->Record(data2);
+    recorder->Record(data3);
+
+    generator.Keep();
+  }
+
+  {
+    DummyReproducer reproducer;
+    EXPECT_THAT_ERROR(reproducer.SetReplay(FileSpec(root.str())), Succeeded());
+
+    auto &loader = *reproducer.GetLoader();
+    std::unique_ptr<repro::MultiLoader<YamlMultiProvider>> multi_loader =
+        repro::MultiLoader<YamlMultiProvider>::Create(&loader);
+
+    // Read the first file.
+    {
+      llvm::Optional<std::string> file = multi_loader->GetNextFile();
+      EXPECT_TRUE(static_cast<bool>(file));
+
+      auto buffer = llvm::MemoryBuffer::getFile(*file);
+      EXPECT_TRUE(static_cast<bool>(buffer));
+
+      yaml::Input yin((*buffer)->getBuffer());
+      std::vector<YamlData> data;
+      yin >> data;
+
+      ASSERT_EQ(data.size(), 2U);
+      EXPECT_THAT(data, testing::ElementsAre(data0, data1));
+    }
+
+    // Read the second file.
+    {
+      llvm::Optional<std::string> file = multi_loader->GetNextFile();
+      EXPECT_TRUE(static_cast<bool>(file));
+
+      auto buffer = llvm::MemoryBuffer::getFile(*file);
+      EXPECT_TRUE(static_cast<bool>(buffer));
+
+      yaml::Input yin((*buffer)->getBuffer());
+      std::vector<YamlData> data;
+      yin >> data;
+
+      ASSERT_EQ(data.size(), 2U);
+      EXPECT_THAT(data, testing::ElementsAre(data2, data3));
+    }
+
+    // There is no third file.
+    llvm::Optional<std::string> file = multi_loader->GetNextFile();
+    EXPECT_FALSE(static_cast<bool>(file));
+  }
+}
diff --git a/lldb/unittests/Utility/ScalarTest.cpp b/lldb/unittests/Utility/ScalarTest.cpp
index baf1de98c121f..42a2f2aaebf2f 100644
--- a/lldb/unittests/Utility/ScalarTest.cpp
+++ b/lldb/unittests/Utility/ScalarTest.cpp
@@ -66,6 +66,47 @@ TEST(ScalarTest, ComparisonFloat) {
   ASSERT_TRUE(s2 >= s1);
 }
 
+template <typename T> static void CheckConversion(T val) {
+  SCOPED_TRACE("val = " + std::to_string(val));
+  EXPECT_EQ((signed char)val, Scalar(val).SChar());
+  EXPECT_EQ((unsigned char)val, Scalar(val).UChar());
+  EXPECT_EQ((short)val, Scalar(val).SShort());
+  EXPECT_EQ((unsigned short)val, Scalar(val).UShort());
+  EXPECT_EQ((int)val, Scalar(val).SInt());
+  EXPECT_EQ((unsigned)val, Scalar(val).UInt());
+  EXPECT_EQ((long)val, Scalar(val).SLong());
+  EXPECT_EQ((unsigned long)val, Scalar(val).ULong());
+  EXPECT_EQ((long long)val, Scalar(val).SLongLong());
+  EXPECT_EQ((unsigned long long)val, Scalar(val).ULongLong());
+  EXPECT_NEAR((float)val, Scalar(val).Float(), std::abs(val / 1e6));
+  EXPECT_NEAR((double)val, Scalar(val).Double(), std::abs(val / 1e12));
+  EXPECT_NEAR((long double)val, Scalar(val).LongDouble(), std::abs(val / 1e12));
+}
+
+TEST(ScalarTest, Getters) {
+  CheckConversion<int>(0x87654321);
+  CheckConversion<unsigned int>(0x87654321u);
+  CheckConversion<long>(0x87654321l);
+  CheckConversion<unsigned long>(0x87654321ul);
+  CheckConversion<long long>(0x8765432112345678ll);
+  CheckConversion<unsigned long long>(0x8765432112345678ull);
+  CheckConversion<float>(42.25f);
+  CheckConversion<double>(42.25);
+
+  EXPECT_EQ(APInt(128, 1) << 70, Scalar(std::pow(2.0f, 70.0f)).SInt128(APInt()));
+  EXPECT_EQ(APInt(128, -1, true) << 70,
+            Scalar(-std::pow(2.0f, 70.0f)).SInt128(APInt()));
+  EXPECT_EQ(APInt(128, 1) << 70,
+            Scalar(std::pow(2.0f, 70.0f)).UInt128(APInt()));
+  EXPECT_EQ(APInt(128, 0), Scalar(-std::pow(2.0f, 70.0f)).UInt128(APInt()));
+
+  EXPECT_EQ(APInt(128, 1) << 70, Scalar(std::pow(2.0, 70.0)).SInt128(APInt()));
+  EXPECT_EQ(APInt(128, -1, true) << 70,
+            Scalar(-std::pow(2.0, 70.0)).SInt128(APInt()));
+  EXPECT_EQ(APInt(128, 1) << 70, Scalar(std::pow(2.0, 70.0)).UInt128(APInt()));
+  EXPECT_EQ(APInt(128, 0), Scalar(-std::pow(2.0, 70.0)).UInt128(APInt()));
+}
+
 TEST(ScalarTest, RightShiftOperator) {
   int a = 0x00001000;
   int b = 0xFFFFFFFF;
@@ -78,6 +119,7 @@ TEST(ScalarTest, RightShiftOperator) {
 }
 
 TEST(ScalarTest, GetBytes) {
+  uint8_t Storage[256];
   int a = 0x01020304;
   long long b = 0x0102030405060708LL;
   float c = 1234567.89e32f;
@@ -99,35 +141,53 @@ TEST(ScalarTest, GetBytes) {
                        sizeof(void *));
   Status f_error =
       f_scalar.SetValueFromData(f_data, lldb::eEncodingUint, sizeof(f));
-  ASSERT_EQ(0, memcmp(&a, a_scalar.GetBytes(), sizeof(a)));
-  ASSERT_EQ(0, memcmp(&b, b_scalar.GetBytes(), sizeof(b)));
-  ASSERT_EQ(0, memcmp(&c, c_scalar.GetBytes(), sizeof(c)));
-  ASSERT_EQ(0, memcmp(&d, d_scalar.GetBytes(), sizeof(d)));
+  a_scalar.GetBytes(Storage);
+  ASSERT_EQ(0, memcmp(&a, Storage, sizeof(a)));
+  b_scalar.GetBytes(Storage);
+  ASSERT_EQ(0, memcmp(&b, Storage, sizeof(b)));
+  c_scalar.GetBytes(Storage);
+  ASSERT_EQ(0, memcmp(&c, Storage, sizeof(c)));
+  d_scalar.GetBytes(Storage);
+  ASSERT_EQ(0, memcmp(&d, Storage, sizeof(d)));
   ASSERT_EQ(0, e_error.Fail());
-  ASSERT_EQ(0, memcmp(e, e_scalar.GetBytes(), sizeof(e)));
+  e_scalar.GetBytes(Storage);
+  ASSERT_EQ(0, memcmp(e, Storage, sizeof(e)));
   ASSERT_EQ(0, f_error.Fail());
-  ASSERT_EQ(0, memcmp(f, f_scalar.GetBytes(), sizeof(f)));
+  f_scalar.GetBytes(Storage);
+  ASSERT_EQ(0, memcmp(f, Storage, sizeof(f)));
 }
 
 TEST(ScalarTest, CastOperations) {
   long long a = 0xf1f2f3f4f5f6f7f8LL;
   Scalar a_scalar(a);
-  ASSERT_EQ((signed char)a, a_scalar.SChar());
-  ASSERT_EQ((unsigned char)a, a_scalar.UChar());
-  ASSERT_EQ((signed short)a, a_scalar.SShort());
-  ASSERT_EQ((unsigned short)a, a_scalar.UShort());
-  ASSERT_EQ((signed int)a, a_scalar.SInt());
-  ASSERT_EQ((unsigned int)a, a_scalar.UInt());
-  ASSERT_EQ((signed long)a, a_scalar.SLong());
-  ASSERT_EQ((unsigned long)a, a_scalar.ULong());
-  ASSERT_EQ((signed long long)a, a_scalar.SLongLong());
-  ASSERT_EQ((unsigned long long)a, a_scalar.ULongLong());
+  EXPECT_EQ((signed char)a, a_scalar.SChar());
+  EXPECT_EQ((unsigned char)a, a_scalar.UChar());
+  EXPECT_EQ((signed short)a, a_scalar.SShort());
+  EXPECT_EQ((unsigned short)a, a_scalar.UShort());
+  EXPECT_EQ((signed int)a, a_scalar.SInt());
+  EXPECT_EQ((unsigned int)a, a_scalar.UInt());
+  EXPECT_EQ((signed long)a, a_scalar.SLong());
+  EXPECT_EQ((unsigned long)a, a_scalar.ULong());
+  EXPECT_EQ((signed long long)a, a_scalar.SLongLong());
+  EXPECT_EQ((unsigned long long)a, a_scalar.ULongLong());
 
   int a2 = 23;
   Scalar a2_scalar(a2);
-  ASSERT_EQ((float)a2, a2_scalar.Float());
-  ASSERT_EQ((double)a2, a2_scalar.Double());
-  ASSERT_EQ((long double)a2, a2_scalar.LongDouble());
+  EXPECT_EQ((float)a2, a2_scalar.Float());
+  EXPECT_EQ((double)a2, a2_scalar.Double());
+  EXPECT_EQ((long double)a2, a2_scalar.LongDouble());
+
+  EXPECT_EQ(std::numeric_limits<unsigned int>::min(), Scalar(-1.0f).UInt());
+  EXPECT_EQ(std::numeric_limits<unsigned int>::max(), Scalar(1e11f).UInt());
+  EXPECT_EQ(std::numeric_limits<unsigned long long>::min(),
+            Scalar(-1.0).ULongLong());
+  EXPECT_EQ(std::numeric_limits<unsigned long long>::max(),
+            Scalar(1e22).ULongLong());
+
+  EXPECT_EQ(std::numeric_limits<int>::min(), Scalar(-1e11f).SInt());
+  EXPECT_EQ(std::numeric_limits<int>::max(), Scalar(1e11f).SInt());
+  EXPECT_EQ(std::numeric_limits<long long>::min(), Scalar(-1e22).SLongLong());
+  EXPECT_EQ(std::numeric_limits<long long>::max(), Scalar(1e22).SLongLong());
 }
 
 TEST(ScalarTest, ExtractBitfield) {
@@ -137,21 +197,21 @@ TEST(ScalarTest, ExtractBitfield) {
   long long b1 = 0xff1f2f3f4f5f6f7fLL;
   Scalar s_scalar(a1);
   ASSERT_TRUE(s_scalar.ExtractBitfield(0, 0));
-  ASSERT_EQ(0, memcmp(&a1, s_scalar.GetBytes(), sizeof(a1)));
+  EXPECT_EQ(s_scalar, a1);
   ASSERT_TRUE(s_scalar.ExtractBitfield(len, 0));
-  ASSERT_EQ(0, memcmp(&a1, s_scalar.GetBytes(), sizeof(a1)));
+  EXPECT_EQ(s_scalar, a1);
   ASSERT_TRUE(s_scalar.ExtractBitfield(len - 4, 4));
-  ASSERT_EQ(0, memcmp(&b1, s_scalar.GetBytes(), sizeof(b1)));
+  EXPECT_EQ(s_scalar, b1);
 
   unsigned long long a2 = 0xf1f2f3f4f5f6f7f8ULL;
   unsigned long long b2 = 0x0f1f2f3f4f5f6f7fULL;
   Scalar u_scalar(a2);
   ASSERT_TRUE(u_scalar.ExtractBitfield(0, 0));
-  ASSERT_EQ(0, memcmp(&a2, u_scalar.GetBytes(), sizeof(a2)));
+  EXPECT_EQ(u_scalar, a2);
   ASSERT_TRUE(u_scalar.ExtractBitfield(len, 0));
-  ASSERT_EQ(0, memcmp(&a2, u_scalar.GetBytes(), sizeof(a2)));
+  EXPECT_EQ(u_scalar, a2);
   ASSERT_TRUE(u_scalar.ExtractBitfield(len - 4, 4));
-  ASSERT_EQ(0, memcmp(&b2, u_scalar.GetBytes(), sizeof(b2)));
+  EXPECT_EQ(u_scalar, b2);
 }
 
 template <typename T> static std::string ScalarGetValue(T value) {
@@ -305,7 +365,7 @@ TEST(ScalarTest, Scalar_512) {
 
   ASSERT_TRUE(S.MakeUnsigned());
   EXPECT_EQ(S.GetType(), Scalar::e_uint512);
-  ASSERT_STREQ(S.GetTypeAsCString(), "unsigned int512_t");
+  ASSERT_STREQ(S.GetTypeAsCString(), "uint512_t");
   ASSERT_STREQ(S.GetValueTypeAsCString(Scalar::e_uint512), "uint512_t");
   EXPECT_EQ(S.GetByteSize(), 64U);
 
@@ -313,3 +373,15 @@ TEST(ScalarTest, Scalar_512) {
   EXPECT_EQ(S.GetType(), Scalar::e_sint512);
   EXPECT_EQ(S.GetByteSize(), 64U);
 }
+
+TEST(ScalarTest, TruncOrExtendTo) {
+  Scalar S(0xffff);
+  S.TruncOrExtendTo(12, true);
+  EXPECT_EQ(S.UInt128(APInt()), APInt(12, 0xfffu));
+  S.TruncOrExtendTo(20, true);
+  EXPECT_EQ(S.UInt128(APInt()), APInt(20, 0xfffffu));
+  S.TruncOrExtendTo(24, false);
+  EXPECT_EQ(S.UInt128(APInt()), APInt(24, 0x0fffffu));
+  S.TruncOrExtendTo(16, false);
+  EXPECT_EQ(S.UInt128(APInt()), APInt(16, 0xffffu));
+}
diff --git a/lldb/utils/lldb-dotest/CMakeLists.txt b/lldb/utils/lldb-dotest/CMakeLists.txt
index 0278c370f7fe1..0ef60c1427610 100644
--- a/lldb/utils/lldb-dotest/CMakeLists.txt
+++ b/lldb/utils/lldb-dotest/CMakeLists.txt
@@ -26,6 +26,7 @@ if(LLDB_BUILT_STANDALONE)
     string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_COMPILER_CONFIGURED "${LLDB_TEST_COMPILER}")
     string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_DSYMUTIL_CONFIGURED "${LLDB_TEST_DSYMUTIL}")
     string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_FILECHECK_CONFIGURED "${LLDB_TEST_FILECHECK}")
+    string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_YAML2OBJ_CONFIGURED "${LLDB_TEST_YAML2OBJ}")
 
     # Remaining ones must be paths to the provided LLVM build-tree.
     if(${config_type} IN_LIST LLVM_CONFIGURATION_TYPES)
@@ -37,6 +38,7 @@ if(LLDB_BUILT_STANDALONE)
       string(REPLACE ${CMAKE_CFG_INTDIR} ${config_type} LLDB_TEST_COMPILER_CONFIGURED "${LLDB_TEST_COMPILER}")
       string(REPLACE ${CMAKE_CFG_INTDIR} ${config_type} LLDB_TEST_DSYMUTIL_CONFIGURED "${LLDB_TEST_DSYMUTIL}")
       string(REPLACE ${CMAKE_CFG_INTDIR} ${config_type} LLDB_TEST_FILECHECK_CONFIGURED "${LLDB_TEST_FILECHECK}")
+      string(REPLACE ${CMAKE_CFG_INTDIR} ${config_type} LLDB_TEST_YAML2OBJ_CONFIGURED "${LLDB_TEST_YAML2OBJ}")
       string(REPLACE ${CMAKE_CFG_INTDIR} ${config_type} LLDB_LIBS_DIR_CONFIGURED "${LLDB_LIBS_DIR}")
     else()
       # Single-configuration generator like Ninja.
@@ -47,6 +49,7 @@ if(LLDB_BUILT_STANDALONE)
       string(REPLACE ${CMAKE_CFG_INTDIR} "." LLDB_TEST_COMPILER_CONFIGURED "${LLDB_TEST_COMPILER}")
       string(REPLACE ${CMAKE_CFG_INTDIR} "." LLDB_TEST_DSYMUTIL_CONFIGURED "${LLDB_TEST_DSYMUTIL}")
       string(REPLACE ${CMAKE_CFG_INTDIR} "." LLDB_TEST_FILECHECK_CONFIGURED "${LLDB_TEST_FILECHECK}")
+      string(REPLACE ${CMAKE_CFG_INTDIR} "." LLDB_TEST_YAML2OBJ_CONFIGURED "${LLDB_TEST_YAML2OBJ_CONFIGURED}")
       string(REPLACE ${CMAKE_CFG_INTDIR} "." LLDB_LIBS_DIR_CONFIGURED "${LLDB_LIBS_DIR}")
     endif()
 
@@ -65,6 +68,7 @@ elseif(NOT "${CMAKE_CFG_INTDIR}" STREQUAL ".")
     string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} LLDB_TEST_COMPILER_CONFIGURED "${LLDB_TEST_COMPILER}")
     string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} LLDB_TEST_DSYMUTIL_CONFIGURED "${LLDB_TEST_DSYMUTIL}")
     string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} LLDB_TEST_FILECHECK_CONFIGURED "${LLDB_TEST_FILECHECK}")
+    string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} LLDB_TEST_YAML2OBJ_CONFIGURED "${LLDB_TEST_YAML2OBJ}")
     string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} LLDB_LIBS_DIR_CONFIGURED "${LLDB_LIBS_DIR}")
 
     configure_file(
@@ -80,6 +84,7 @@ else()
   set(LLDB_TEST_COMPILER_CONFIGURED "${LLDB_TEST_COMPILER}")
   set(LLDB_TEST_DSYMUTIL_CONFIGURED "${LLDB_TEST_DSYMUTIL}")
   set(LLDB_TEST_FILECHECK_CONFIGURED "${LLDB_TEST_FILECHECK}")
+  set(LLDB_TEST_YAML2OBJ_CONFIGURED "${LLDB_TEST_YAML2OBJ}")
   set(LLDB_LIBS_DIR_CONFIGURED "${LLDB_LIBS_DIR}")
 
   configure_file(
diff --git a/lldb/utils/lldb-dotest/lldb-dotest.in b/lldb/utils/lldb-dotest/lldb-dotest.in
index 36d5fd38cc6d5..ee0ea6dff748c 100755
--- a/lldb/utils/lldb-dotest/lldb-dotest.in
+++ b/lldb/utils/lldb-dotest/lldb-dotest.in
@@ -10,6 +10,7 @@ executable = '@LLDB_TEST_EXECUTABLE_CONFIGURED@'
 compiler = '@LLDB_TEST_COMPILER_CONFIGURED@'
 dsymutil = '@LLDB_TEST_DSYMUTIL_CONFIGURED@'
 filecheck = '@LLDB_TEST_FILECHECK_CONFIGURED@'
+yaml2obj = '@LLDB_TEST_YAML2OBJ_CONFIGURED@'
 lldb_libs_dir = "@LLDB_LIBS_DIR_CONFIGURED@"
 lldb_build_intel_pt = "@LLDB_BUILD_INTEL_PT@"
 
@@ -24,6 +25,7 @@ if __name__ == '__main__':
     cmd.extend(['--executable', executable])
     cmd.extend(['--compiler', compiler])
     cmd.extend(['--dsymutil', dsymutil])
+    cmd.extend(['--yaml2obj', yaml2obj])
     cmd.extend(['--filecheck', filecheck])
     cmd.extend(['--lldb-libs-dir', lldb_libs_dir])
     if lldb_build_intel_pt == "1":
diff --git a/lldb/utils/lldb-repro/lldb-repro.py b/lldb/utils/lldb-repro/lldb-repro.py
index 2244e97a0ff15..2b7b23f8cf29d 100755
--- a/lldb/utils/lldb-repro/lldb-repro.py
+++ b/lldb/utils/lldb-repro/lldb-repro.py
@@ -49,7 +49,7 @@ def main():
     elif sys.argv[1] == "capture":
         new_args.extend([
             '--capture', '--capture-path', reproducer_path,
-            '--reproducer-auto-generate'
+            '--reproducer-generate-on-exit'
         ])
         new_args.extend(sys.argv[2:])
     else:
diff --git a/llvm/.gitignore b/llvm/.gitignore
index f2b343ade4f32..d6021ff18e1fa 100644
--- a/llvm/.gitignore
+++ b/llvm/.gitignore
@@ -59,8 +59,6 @@ docs/_build
 # VS2017 and VSCode config files.
 .vscode
 .vs
-# clangd index
-.clangd
 
 #==============================================================================#
 # Files created in tree by the Go bindings.
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index b224393ac16a6..de2887b64c2a9 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -368,22 +368,20 @@ set(LLVM_ENABLE_ZLIB "ON" CACHE STRING "Use zlib for compression/decompression i
 
 set(LLVM_Z3_INSTALL_DIR "" CACHE STRING "Install directory of the Z3 solver.")
 
-find_package(Z3 4.7.1)
-
-if (LLVM_Z3_INSTALL_DIR)
-  if (NOT Z3_FOUND)
-    message(FATAL_ERROR "Z3 >= 4.7.1 has not been found in LLVM_Z3_INSTALL_DIR: ${LLVM_Z3_INSTALL_DIR}.")
-  endif()
-endif()
-
-set(LLVM_ENABLE_Z3_SOLVER_DEFAULT "${Z3_FOUND}")
-
 option(LLVM_ENABLE_Z3_SOLVER
   "Enable Support for the Z3 constraint solver in LLVM."
   ${LLVM_ENABLE_Z3_SOLVER_DEFAULT}
 )
 
 if (LLVM_ENABLE_Z3_SOLVER)
+  find_package(Z3 4.7.1)
+  
+  if (LLVM_Z3_INSTALL_DIR)
+    if (NOT Z3_FOUND)
+      message(FATAL_ERROR "Z3 >= 4.7.1 has not been found in LLVM_Z3_INSTALL_DIR: ${LLVM_Z3_INSTALL_DIR}.")
+    endif()
+  endif()
+  
   if (NOT Z3_FOUND)
     message(FATAL_ERROR "LLVM_ENABLE_Z3_SOLVER cannot be enabled when Z3 is not available.")
   endif()
@@ -391,6 +389,9 @@ if (LLVM_ENABLE_Z3_SOLVER)
   set(LLVM_WITH_Z3 1)
 endif()
 
+set(LLVM_ENABLE_Z3_SOLVER_DEFAULT "${Z3_FOUND}")
+
+
 if( LLVM_TARGETS_TO_BUILD STREQUAL "all" )
   set( LLVM_TARGETS_TO_BUILD ${LLVM_ALL_TARGETS} )
 endif()
@@ -961,6 +962,25 @@ if( MINGW AND NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" )
   llvm_replace_compiler_option(CMAKE_CXX_FLAGS_RELEASE "-O3" "-O2")
 endif()
 
+# For up-to-date instructions for installing the Tensorflow dependency, refer to
+# the bot setup script: https://github.com/google/ml-compiler-opt/blob/master/buildbot/buildbot_init.sh
+# Specifically, assuming python3 is installed:
+# python3 -m pip install --upgrade pip && python3 -m pip install --user tf_nightly==2.3.0.dev20200528
+# Then set TENSORFLOW_AOT_PATH to the package install - usually it's ~/.local/lib/python3.7/site-packages/tensorflow
+#
+set(TENSORFLOW_AOT_PATH "" CACHE PATH "Path to TensorFlow pip install dir")
+
+if (NOT TENSORFLOW_AOT_PATH STREQUAL "")
+  set(LLVM_HAVE_TF_AOT "ON" CACHE BOOL "Tensorflow AOT available")
+  set(TENSORFLOW_AOT_COMPILER 
+    "${TENSORFLOW_AOT_PATH}/../../../../bin/saved_model_cli" 
+      CACHE PATH "Path to the Tensorflow AOT compiler")
+  add_definitions("-DLLVM_HAVE_TF_AOT")
+  include_directories(${TENSORFLOW_AOT_PATH}/include)
+  add_subdirectory(${TENSORFLOW_AOT_PATH}/xla_aot_runtime_src
+    ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/tf_runtime)
+endif()
+
 # Put this before tblgen. Else we have a circular dependence.
 add_subdirectory(lib/Demangle)
 add_subdirectory(lib/Support)
@@ -1111,7 +1131,7 @@ if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
 
   # Installing the headers needs to depend on generating any public
   # tablegen'd headers.
-  add_custom_target(llvm-headers DEPENDS intrinsics_gen)
+  add_custom_target(llvm-headers DEPENDS intrinsics_gen omp_gen)
   set_target_properties(llvm-headers PROPERTIES FOLDER "Misc")
 
   if (NOT LLVM_ENABLE_IDE)
diff --git a/llvm/cmake/config.guess b/llvm/cmake/config.guess
index 26d120601e494..9fdfcce8d033c 100644
--- a/llvm/cmake/config.guess
+++ b/llvm/cmake/config.guess
@@ -1263,6 +1263,23 @@ EOF
 		      UNAME_PROCESSOR="x86_64"
 		  fi
 		fi ;;
+		arm)
+		eval $set_cc_for_build
+		if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
+			if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
+				(CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
+				grep IS_64BIT_ARCH >/dev/null
+			then
+				if (echo '#ifdef __PTRAUTH_INTRINSICS__'; echo HAS_AUTH; echo '#endif') | \
+					(CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
+					grep HAS_AUTH >/dev/null
+				then
+					UNAME_PROCESSOR="arm64e"
+				else
+					UNAME_PROCESSOR="arm64"
+				fi
+			fi
+		fi ;;
 	    unknown) UNAME_PROCESSOR=powerpc ;;
 	esac
 	echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE}
diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake
index e25c72f3cae35..744c52e82f3b3 100644
--- a/llvm/cmake/modules/AddLLVM.cmake
+++ b/llvm/cmake/modules/AddLLVM.cmake
@@ -925,6 +925,9 @@ function(add_llvm_pass_plugin name)
     if (TARGET intrinsics_gen)
       add_dependencies(obj.${name} intrinsics_gen)
     endif()
+    if (TARGET omp_gen)
+      add_dependencies(obj.${name} omp_gen)
+    endif()
     set_property(GLOBAL APPEND PROPERTY LLVM_STATIC_EXTENSIONS ${name})
   elseif(NOT ARG_NO_MODULE)
     add_llvm_library(${name} MODULE ${ARG_UNPARSED_ARGUMENTS})
@@ -1428,7 +1431,7 @@ function(add_unittest test_suite test_name)
 
   add_dependencies(${test_suite} ${test_name})
   get_target_property(test_suite_folder ${test_suite} FOLDER)
-  if (NOT ${test_suite_folder} STREQUAL "NOTFOUND")
+  if (test_suite_folder)
     set_property(TARGET ${test_name} PROPERTY FOLDER "${test_suite_folder}")
   endif ()
 endfunction()
diff --git a/llvm/cmake/modules/AddSphinxTarget.cmake b/llvm/cmake/modules/AddSphinxTarget.cmake
index f053d8084da42..b5babb30abcf1 100644
--- a/llvm/cmake/modules/AddSphinxTarget.cmake
+++ b/llvm/cmake/modules/AddSphinxTarget.cmake
@@ -38,6 +38,7 @@ function (add_sphinx_target builder project)
                             -b ${builder}
                             -d "${SPHINX_DOC_TREE_DIR}"
                             -q # Quiet: no output other than errors and warnings.
+                            -t builder-${builder} # tag for builder
                             ${SPHINX_WARNINGS_AS_ERRORS_FLAG} # Treat warnings as errors if requested
                             "${ARG_SOURCE_DIR}" # Source
                             "${SPHINX_BUILD_DIR}" # Output
diff --git a/llvm/cmake/modules/CMakeLists.txt b/llvm/cmake/modules/CMakeLists.txt
index 1cef87a5e18de..4b8879f65fe47 100644
--- a/llvm/cmake/modules/CMakeLists.txt
+++ b/llvm/cmake/modules/CMakeLists.txt
@@ -28,6 +28,9 @@ endforeach(lib)
 if(intrinsics_gen IN_LIST LLVM_COMMON_DEPENDS)
   list(REMOVE_ITEM LLVM_COMMON_DEPENDS intrinsics_gen)
 endif()
+if(omp_gen IN_LIST LLVM_COMMON_DEPENDS)
+  list(REMOVE_ITEM LLVM_COMMON_DEPENDS omp_gen)
+endif()
 
 # Generate LLVMConfig.cmake for the build tree.
 set(LLVM_CONFIG_CODE "
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index d5c924ca1a2c8..2e249593e12f1 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -27,7 +27,7 @@ string(TOUPPER "${LLVM_ENABLE_LTO}" uppercase_LLVM_ENABLE_LTO)
 set(LLVM_PARALLEL_COMPILE_JOBS "" CACHE STRING
   "Define the maximum number of concurrent compilation jobs (Ninja only).")
 if(LLVM_PARALLEL_COMPILE_JOBS)
-  if(NOT CMAKE_MAKE_PROGRAM MATCHES "ninja")
+  if(NOT CMAKE_GENERATOR STREQUAL "Ninja")
     message(WARNING "Job pooling is only available with Ninja generators.")
   else()
     set_property(GLOBAL APPEND PROPERTY JOB_POOLS compile_job_pool=${LLVM_PARALLEL_COMPILE_JOBS})
@@ -37,7 +37,7 @@ endif()
 
 set(LLVM_PARALLEL_LINK_JOBS "" CACHE STRING
   "Define the maximum number of concurrent link jobs (Ninja only).")
-if(CMAKE_MAKE_PROGRAM MATCHES "ninja")
+if(CMAKE_GENERATOR STREQUAL "Ninja")
   if(NOT LLVM_PARALLEL_LINK_JOBS AND uppercase_LLVM_ENABLE_LTO STREQUAL "THIN")
     message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 2.")
     set(LLVM_PARALLEL_LINK_JOBS "2")
@@ -911,6 +911,7 @@ if (CLANG_CL AND (LLVM_BUILD_INSTRUMENTED OR LLVM_USE_SANITIZER))
   file(TO_CMAKE_PATH "${clang_resource_dir}" clang_resource_dir)
   append("/libpath:${clang_resource_dir}/lib/windows"
     CMAKE_EXE_LINKER_FLAGS
+    CMAKE_MODULE_LINKER_FLAGS
     CMAKE_SHARED_LINKER_FLAGS)
 endif()
 
diff --git a/llvm/cmake/modules/LLVMConfig.cmake.in b/llvm/cmake/modules/LLVMConfig.cmake.in
index fa6082c7e3977..c9a1c8131d2d6 100644
--- a/llvm/cmake/modules/LLVMConfig.cmake.in
+++ b/llvm/cmake/modules/LLVMConfig.cmake.in
@@ -104,12 +104,15 @@ if(NOT TARGET LLVMSupport)
   @llvm_config_include_buildtree_only_exports@
 endif()
 
-# By creating intrinsics_gen here, subprojects that depend on LLVM's
+# By creating intrinsics_gen and omp_gen here, subprojects that depend on LLVM's
 # tablegen-generated headers can always depend on this target whether building
 # in-tree with LLVM or not.
 if(NOT TARGET intrinsics_gen)
   add_custom_target(intrinsics_gen)
 endif()
+if(NOT TARGET omp_gen)
+  add_custom_target(omp_gen)
+endif()
 
 set_property(GLOBAL PROPERTY LLVM_TARGETS_CONFIGURED On)
 include(${LLVM_CMAKE_DIR}/LLVM-Config.cmake)
diff --git a/llvm/cmake/modules/TensorFlowCompile.cmake b/llvm/cmake/modules/TensorFlowCompile.cmake
new file mode 100644
index 0000000000000..a8ba56e67d1f6
--- /dev/null
+++ b/llvm/cmake/modules/TensorFlowCompile.cmake
@@ -0,0 +1,38 @@
+# Run the tensorflow compiler (saved_model_cli) on the saved model in the 
+# ${model} directory, looking for the ${tag_set} tag set, and the SignatureDef
+# ${signature_def_key}.
+# Produce a pair of files called ${fname}.h and  ${fname}.o in the 
+# ${CMAKE_CURRENT_BINARY_DIR}. The generated header will define a C++ class
+# called ${cpp_class} - which may be a namespace-qualified class name.
+function(tfcompile model tag_set signature_def_key fname cpp_class)
+  if (IS_ABSOLUTE ${model})
+    set(LLVM_ML_MODELS_ABSOLUTE ${model})
+  else()
+    set(LLVM_ML_MODELS_ABSOLUTE
+      ${CMAKE_CURRENT_SOURCE_DIR}/${model})
+  endif()
+
+  set(prefix ${CMAKE_CURRENT_BINARY_DIR}/${fname})
+  set(obj_file ${prefix}.o)
+  set(hdr_file ${prefix}.h)
+  add_custom_command(OUTPUT ${obj_file} ${hdr_file}
+    COMMAND "XLA_FLAGS=\"--xla_cpu_multi_thread_eigen=false\"" ${TENSORFLOW_AOT_COMPILER} aot_compile_cpu
+          --dir ${LLVM_ML_MODELS_ABSOLUTE}
+          --tag_set ${tag_set}
+          --signature_def_key ${signature_def_key}
+          --output_prefix ${prefix}
+          --cpp_class ${cpp_class}
+          --target_triple ${LLVM_HOST_TRIPLE}
+  )
+
+  # Aggregate the objects so that results of different tfcompile calls may be
+  # grouped into one target.
+  set(GENERATED_OBJS ${GENERATED_OBJS} ${obj_file} PARENT_SCOPE)
+  set_source_files_properties(${obj_file} PROPERTIES
+    GENERATED 1 EXTERNAL_OBJECT 1)
+
+  set(GENERATED_HEADERS ${GENERATED_HEADERS} ${hdr_file} PARENT_SCOPE)
+  set_source_files_properties(${hdr_file} PROPERTIES
+    GENERATED 1)
+  
+endfunction()
diff --git a/llvm/cmake/modules/VersionFromVCS.cmake b/llvm/cmake/modules/VersionFromVCS.cmake
index 1b6519b4b7c4e..18edbeabe3e4b 100644
--- a/llvm/cmake/modules/VersionFromVCS.cmake
+++ b/llvm/cmake/modules/VersionFromVCS.cmake
@@ -4,7 +4,7 @@
 # extra argument, otherwise uses CMAKE_CURRENT_SOURCE_DIR).
 
 function(get_source_info path revision repository)
-  find_package(Git)
+  find_package(Git QUIET)
   if(GIT_FOUND)
     execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse --git-dir
       WORKING_DIRECTORY ${path}
@@ -45,5 +45,7 @@ function(get_source_info path revision repository)
         set(${repository} ${path} PARENT_SCOPE)
       endif()
     endif()
+  else()
+    message(WARNING "Git not found. Version cannot be determined.")
   endif()
 endfunction()
diff --git a/llvm/docs/AMDGPUDwarfProposalForHeterogeneousDebugging.rst b/llvm/docs/AMDGPUDwarfProposalForHeterogeneousDebugging.rst
index c6868d675a88e..a2c48f97717a4 100644
--- a/llvm/docs/AMDGPUDwarfProposalForHeterogeneousDebugging.rst
+++ b/llvm/docs/AMDGPUDwarfProposalForHeterogeneousDebugging.rst
@@ -671,8 +671,9 @@ result of the evaluation is:
     empty operation expression for this purpose.*
 
   * If the top stack entry is a location description, or can be converted
-    to one, then the result is that, possibly converted, location description.
-    Any other entries on the stack are discarded.
+    to one (see :ref:`amdgpu-dwarf-memory-location-description-operations`),
+    then the result is that, possibly converted, location description. Any other
+    entries on the stack are discarded.
 
   * Otherwise the DWARF expression is ill-formed.
 
@@ -683,9 +684,10 @@ result of the evaluation is:
 
 * If evaluation of the DWARF expression requires a value, then:
 
-  * If the top stack entry is a value, or can be converted to one, then the
-    result is that, possibly converted, value. Any other entries on the stack
-    are discarded.
+  * If the top stack entry is a value, or can be converted to one (see
+    :ref:`amdgpu-dwarf-memory-location-description-operations`), then the result
+    is that, possibly converted, value. Any other entries on the stack are
+    discarded.
 
   * Otherwise the DWARF expression is ill-formed.
 
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index f97d6d75b9f8d..af764c0856000 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -225,47 +225,41 @@ names from both the *Processor* and *Alternative Processor* can be used.
                                                                                  names.
      ``gfx906``                  ``amdgcn``   dGPU  - xnack                   - Radeon Instinct MI50
                                                       [off]                   - Radeon Instinct MI60
+                                                                              - Radeon VII
+                                                                              - Radeon Pro VII
      ``gfx908``                  ``amdgcn``   dGPU  - xnack                   *TBA*
                                                       [off]
                                                       sram-ecc
                                                       [on]
-     ``gfx909``                  ``amdgcn``   APU   - xnack                   *TBA* (Raven Ridge 2)
+                                                                              .. TODO::
+                                                                                 Add product
+                                                                                 names.
+     ``gfx909``                  ``amdgcn``   APU   - xnack                   *TBA*
                                                       [on]
                                                                               .. TODO::
                                                                                  Add product
                                                                                  names.
      **GCN GFX10** [AMD-GCN-GFX10]_
      -----------------------------------------------------------------------------------------------
-     ``gfx1010``                 ``amdgcn``   dGPU  - xnack                   *TBA*
-                                                      [off]
-                                                    - wavefrontsize64
+     ``gfx1010``                 ``amdgcn``   dGPU  - xnack                   - Radeon RX 5700
+                                                      [off]                   - Radeon RX 5700 XT
+                                                    - wavefrontsize64         - Radeon Pro 5600 XT
                                                       [off]
                                                     - cumode
                                                       [off]
-                                                                              .. TODO::
-                                                                                 Add product
-                                                                                 names.
-     ``gfx1011``                 ``amdgcn``   dGPU  - xnack                   *TBA*
+     ``gfx1011``                 ``amdgcn``   dGPU  - xnack                   - Radeon Pro 5600M
                                                       [off]
                                                     - wavefrontsize64
                                                       [off]
                                                     - cumode
                                                       [off]
-                                                                              .. TODO::
-                                                                                 Add product
-                                                                                 names.
-     ``gfx1012``                 ``amdgcn``   dGPU  - xnack                   *TBA*
-                                                      [off]
+     ``gfx1012``                 ``amdgcn``   dGPU  - xnack                   - Radeon RX 5500
+                                                      [off]                   - Radeon RX 5500 XT
                                                     - wavefrontsize64
                                                       [off]
                                                     - cumode
                                                       [off]
-                                                                              .. TODO::
-                                                                                 Add product
-                                                                                 names.
-     ``gfx1030``                 ``amdgcn``   dGPU  - xnack                   *TBA*
-                                                      [off]
-                                                    - wavefrontsize64
+     ``gfx1030``                 ``amdgcn``   dGPU  - wavefrontsize64         *TBA*
                                                       [off]
                                                     - cumode
                                                       [off]
@@ -1218,7 +1212,8 @@ mapping.
                                              frame.
    1              EXEC_MASK_32      32       Execution Mask Register when
                                              executing in wavefront 32 mode.
-   2-15           *Reserved*
+   2-15           *Reserved*                 *Reserved for highly accessed
+                                             registers using DWARF shortcut.*
    16             PC_64             64       Program Counter (PC) when
                                              executing in a 64-bit process
                                              address space. Used in the CFI to
@@ -1226,31 +1221,55 @@ mapping.
                                              frame.
    17             EXEC_MASK_64      64       Execution Mask Register when
                                              executing in wavefront 64 mode.
-   18-31          *Reserved*
+   18-31          *Reserved*                 *Reserved for highly accessed
+                                             registers using DWARF shortcut.*
    32-95          SGPR0-SGPR63      32       Scalar General Purpose
                                              Registers.
-   96-127         *Reserved*
-   128-511        *Reserved*
-   512-1023       *Reserved*
-   1024-1087      *Reserved*
-   1088-1129      SGPR64-SGPR105    32       Scalar General Purpose Registers
-   1130-1535      *Reserved*
+   96-127         *Reserved*                 *Reserved for frequently accessed
+                                             registers using DWARF 1-byte ULEB.*
+   128            SCC               32       Scalar Condition Code Register.
+   129-511        *Reserved*                 *Reserved for future Scalar
+                                             Architectural Registers.*
+   512            VCC_32            32       Vector Condition Code Register
+                                             when executing in wavefront 32
+                                             mode.
+   513-1023       *Reserved*                 *Reserved for future Vector
+                                             Architectural Registers when
+                                             executing in wavefront 32 mode.*
+   768            VCC_64            32       Vector Condition Code Register
+                                             when executing in wavefront 64
+                                             mode.
+   769-1023       *Reserved*                 *Reserved for future Vector
+                                             Architectural Registers when
+                                             executing in wavefront 64 mode.*
+   1024-1087      *Reserved*                 *Reserved for padding.*
+   1088-1129      SGPR64-SGPR105    32       Scalar General Purpose Registers.
+   1130-1535      *Reserved*                 *Reserved for future Scalar
+                                             General Purpose Registers.*
    1536-1791      VGPR0-VGPR255     32*32    Vector General Purpose Registers
                                              when executing in wavefront 32
                                              mode.
-   1792-2047      *Reserved*
+   1792-2047      *Reserved*                 *Reserved for future Vector
+                                             General Purpose Registers when
+                                             executing in wavefront 32 mode.*
    2048-2303      AGPR0-AGPR255     32*32    Vector Accumulation Registers
                                              when executing in wavefront 32
-                                             ode.
-   2304-2559      *Reserved*
+                                             mode.
+   2304-2559      *Reserved*                 *Reserved for future Vector
+                                             Accumulation Registers when
+                                             executing in wavefront 32 mode.*
    2560-2815      VGPR0-VGPR255     64*32    Vector General Purpose Registers
                                              when executing in wavefront 64
                                              mode.
-   2816-3071      *Reserved*
+   2816-3071      *Reserved*                 *Reserved for future Vector
+                                             General Purpose Registers when
+                                             executing in wavefront 64 mode.*
    3072-3327      AGPR0-AGPR255     64*32    Vector Accumulation Registers
                                              when executing in wavefront 64
                                              mode.
-   3328-3583      *Reserved*
+   3328-3583      *Reserved*                 *Reserved for future Vector
+                                             Accumulation Registers when
+                                             executing in wavefront 64 mode.*
    ============== ================= ======== ==================================
 
 The vector registers are represented as the full size for the wavefront. They
@@ -1637,7 +1656,7 @@ The following provides an example using pseudo LLVM MIR.
         DW_AT_name = "__divergent_lane_pc_1_then";
         DW_AT_location = DIExpression[
           DW_OP_call_ref %__divergent_lane_pc;
-          DW_OP_xaddr &lex_1_start;
+          DW_OP_addrx &lex_1_start;
           DW_OP_stack_value;
           DW_OP_LLVM_extend 64, 64;
           DW_OP_call_ref %__lex_1_save_exec;
@@ -1660,7 +1679,7 @@ The following provides an example using pseudo LLVM MIR.
           DW_AT_name = "__divergent_lane_pc_1_1_then";
           DW_AT_location = DIExpression[
             DW_OP_call_ref %__divergent_lane_pc_1_then;
-            DW_OP_xaddr &lex_1_1_start;
+            DW_OP_addrx &lex_1_1_start;
             DW_OP_stack_value;
             DW_OP_LLVM_extend 64, 64;
             DW_OP_call_ref %__lex_1_1_save_exec;
@@ -1679,7 +1698,7 @@ The following provides an example using pseudo LLVM MIR.
           DW_AT_name = "__divergent_lane_pc_1_1_else";
           DW_AT_location = DIExpression[
             DW_OP_call_ref %__divergent_lane_pc_1_then;
-            DW_OP_xaddr &lex_1_1_end;
+            DW_OP_addrx &lex_1_1_end;
             DW_OP_stack_value;
             DW_OP_LLVM_extend 64, 64;
             DW_OP_call_ref %__lex_1_1_save_exec;
@@ -1705,7 +1724,7 @@ The following provides an example using pseudo LLVM MIR.
         DW_AT_name = "__divergent_lane_pc_1_else";
         DW_AT_location = DIExpression[
           DW_OP_call_ref %__divergent_lane_pc;
-          DW_OP_xaddr &lex_1_end;
+          DW_OP_addrx &lex_1_end;
           DW_OP_stack_value;
           DW_OP_LLVM_extend 64, 64;
           DW_OP_call_ref %__lex_1_save_exec;
@@ -2299,29 +2318,10 @@ non-AMD key names should be prefixed by "*vendor-name*.".
                                                   multi-grid synchronization is
                                                   passed in the kernarg.
 
-     "ValueType"       string         Required  Kernel argument value type. Only
-                                                present if "ValueKind" is
-                                                "ByValue". For vector data
-                                                types, the value is for the
-                                                element type. Values include:
-
-                                                - "Struct"
-                                                - "I8"
-                                                - "U8"
-                                                - "I16"
-                                                - "U16"
-                                                - "F16"
-                                                - "I32"
-                                                - "U32"
-                                                - "F32"
-                                                - "I64"
-                                                - "U64"
-                                                - "F64"
+     "ValueType"       string                   Unused and deprecated. This should no longer
+                                                be emitted, but is accepted for compatibility.
+
 
-                                                .. TODO::
-                                                   How can it be determined if a
-                                                   vector type, and what size
-                                                   vector?
      "PointeeAlign"    integer                  Alignment in bytes of pointee
                                                 type for pointer type kernel
                                                 argument. Must be a power
@@ -2798,29 +2798,9 @@ same *vendor-name*.
                                                        multi-grid synchronization is
                                                        passed in the kernarg.
 
-     ".value_type"          string         Required  Kernel argument value type. Only
-                                                     present if ".value_kind" is
-                                                     "by_value". For vector data
-                                                     types, the value is for the
-                                                     element type. Values include:
-
-                                                     - "struct"
-                                                     - "i8"
-                                                     - "u8"
-                                                     - "i16"
-                                                     - "u16"
-                                                     - "f16"
-                                                     - "i32"
-                                                     - "u32"
-                                                     - "f32"
-                                                     - "i64"
-                                                     - "u64"
-                                                     - "f64"
+     ".value_type"          string                    Unused and deprecated. This should no longer
+                                                      be emitted, but is accepted for compatibility.
 
-                                                     .. TODO::
-                                                        How can it be determined if a
-                                                        vector type, and what size
-                                                        vector?
      ".pointee_align"       integer                  Alignment in bytes of pointee
                                                      type for pointer type kernel
                                                      argument. Must be a power
diff --git a/llvm/docs/AliasAnalysis.rst b/llvm/docs/AliasAnalysis.rst
index 23d374a42ddbd..191addca79ec8 100644
--- a/llvm/docs/AliasAnalysis.rst
+++ b/llvm/docs/AliasAnalysis.rst
@@ -77,7 +77,7 @@ possible) C code:
     C[1] = A[9-i];        /* One byte store */
   }
 
-In this case, the ``basicaa`` pass will disambiguate the stores to ``C[0]`` and
+In this case, the ``basic-aa`` pass will disambiguate the stores to ``C[0]`` and
 ``C[1]`` because they are accesses to two distinct locations one byte apart, and
 the accesses are each one byte.  In this case, the Loop Invariant Code Motion
 (LICM) pass can use store motion to remove the stores from the loop.  In
@@ -278,7 +278,7 @@ implementing, you just override the interfaces you can improve.
 
 With only one special exception (the :ref:`-no-aa <aliasanalysis-no-aa>` pass)
 every alias analysis pass chains to another alias analysis implementation (for
-example, the user can specify "``-basicaa -ds-aa -licm``" to get the maximum
+example, the user can specify "``-basic-aa -ds-aa -licm``" to get the maximum
 benefit from both alias analyses).  The alias analysis class automatically
 takes care of most of this for methods that you don't override.  For methods
 that you do override, in code paths that return a conservative MayAlias or
@@ -515,10 +515,10 @@ The ``-no-aa`` pass is just like what it sounds: an alias analysis that never
 returns any useful information.  This pass can be useful if you think that alias
 analysis is doing something wrong and are trying to narrow down a problem.
 
-The ``-basicaa`` pass
-^^^^^^^^^^^^^^^^^^^^^
+The ``-basic-aa`` pass
+^^^^^^^^^^^^^^^^^^^^^^
 
-The ``-basicaa`` pass is an aggressive local analysis that *knows* many
+The ``-basic-aa`` pass is an aggressive local analysis that *knows* many
 important facts:
 
 * Distinct globals, stack allocations, and heap allocations can never alias.
diff --git a/llvm/docs/BitCodeFormat.rst b/llvm/docs/BitCodeFormat.rst
index c8b99c60f013d..6e491c6e28547 100644
--- a/llvm/docs/BitCodeFormat.rst
+++ b/llvm/docs/BitCodeFormat.rst
@@ -1058,7 +1058,16 @@ The integer codes are mapped to well-known attributes as follows.
 * code 56: ``nocf_check``
 * code 57: ``optforfuzzing``
 * code 58: ``shadowcallstack``
+* code 59: ``speculative_load_hardening``
+* code 60: ``immarg``
+* code 61: ``willreturn``
+* code 62: ``nofree``
+* code 63: ``nosync``
 * code 64: ``sanitize_memtag``
+* code 65: ``preallocated``
+* code 66: ``no_merge``
+* code 67: ``null_pointer_is_valid``
+* code 68: ``noundef``
 
 .. note::
   The ``allocsize`` attribute has a special encoding for its arguments. Its two
diff --git a/llvm/docs/BranchWeightMetadata.rst b/llvm/docs/BranchWeightMetadata.rst
index 2a1083215987f..902ba87316edc 100644
--- a/llvm/docs/BranchWeightMetadata.rst
+++ b/llvm/docs/BranchWeightMetadata.rst
@@ -15,7 +15,7 @@ The first operator is always a ``MDString`` node with the string
 "branch_weights".  Number of operators depends on the terminator type.
 
 Branch weights might be fetch from the profiling file, or generated based on
-`__builtin_expect`_ instruction.
+`__builtin_expect`_ and `__builtin_expect_with_probability`_ instruction.
 
 All weights are represented as an unsigned 32-bit values, where higher value
 indicates greater chance to be taken.
@@ -144,6 +144,47 @@ case is assumed to be likely taken.
   case 5:  // This case is likely to be taken.
   }
 
+.. _\__builtin_expect_with_probability:
+
+Built-in ``expect.with.probability`` Instruction
+================================================
+
+``__builtin_expect_with_probability(long exp, long c, double probability)`` has
+the same semantics as ``__builtin_expect``, but the caller provides the
+probability that ``exp == c``. The last argument ``probability`` must be
+constant floating-point expression and be in the range [0.0, 1.0] inclusive.
+The usage is also similar as ``__builtin_expect``, for example:
+
+``if`` statement
+^^^^^^^^^^^^^^^^
+
+If the expect comparison value ``c`` is equal to 1(true), and probability
+value ``probability`` is set to 0.8, that means the probability of condition
+to be true is 80% while that of false is 20%.
+
+.. code-block:: c++
+
+  if (__builtin_expect_with_probability(x > 0, 1, 0.8)) {
+    // This block is likely to be taken with probability 80%.
+  }
+
+``switch`` statement
+^^^^^^^^^^^^^^^^^^^^
+
+This is basically the same as ``switch`` statement in ``__builtin_expect``.
+The probability that ``exp`` is equal to the expect value is given in
+the third argument ``probability``, while the probability of other value is
+the average of remaining probability(``1.0 - probability``). For example:
+
+.. code-block:: c++
+
+  switch (__builtin_expect_with_probability(x, 5, 0.7)) {
+  default: break;  // Take this case with probability 10%
+  case 0:  break;  // Take this case with probability 10%
+  case 3:  break;  // Take this case with probability 10%
+  case 5:  break;  // This case is likely to be taken with probability 70%
+  }
+
 CFG Modifications
 =================
 
diff --git a/llvm/docs/CodingStandards.rst b/llvm/docs/CodingStandards.rst
index 861ab05420fb6..99fb6af02a282 100644
--- a/llvm/docs/CodingStandards.rst
+++ b/llvm/docs/CodingStandards.rst
@@ -1302,6 +1302,9 @@ loops wherever possible for all newly added code. For example:
   for (Instruction &I : *BB)
     ... use I ...
 
+Usage of ``std::for_each()``/``llvm::for_each()`` functions is discouraged,
+unless the the callable object already exists.
+
 Don't evaluate ``end()`` every time through a loop
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/llvm/docs/CommandGuide/FileCheck.rst b/llvm/docs/CommandGuide/FileCheck.rst
index 1e69c76d2c015..0a0c2c5dd25d4 100644
--- a/llvm/docs/CommandGuide/FileCheck.rst
+++ b/llvm/docs/CommandGuide/FileCheck.rst
@@ -103,11 +103,37 @@ and from the command line.
   -verify``. With this option FileCheck will verify that input does not contain
   warnings not covered by any ``CHECK:`` patterns.
 
-.. option:: --dump-input <mode>
+.. option:: --dump-input <value>
 
   Dump input to stderr, adding annotations representing currently enabled
-  diagnostics.  Do this either 'always', on 'fail' (default), or 'never'.
-  Specify 'help' to explain the dump format and quit.
+  diagnostics.  When there are multiple occurrences of this option, the
+  ``<value>`` that appears earliest in the list below has precedence.  The
+  default is ``fail``.
+
+  * ``help``   - Explain input dump and quit
+  * ``always`` - Always dump input
+  * ``fail``   - Dump input on failure
+  * ``never``  - Never dump input
+
+.. option:: --dump-input-context <N>
+
+  In the dump requested by ``--dump-input``, print ``<N>`` input lines before
+  and ``<N>`` input lines after any lines specified by ``--dump-input-filter``.
+  When there are multiple occurrences of this option, the largest specified
+  ``<N>`` has precedence.  The default is 5.
+
+.. option:: --dump-input-filter <value>
+
+  In the dump requested by ``--dump-input``, print only input lines of kind
+  ``<value>`` plus any context specified by ``--dump-input-context``.  When
+  there are multiple occurrences of this option, the ``<value>`` that appears
+  earliest in the list below has precedence.  The default is ``error`` when
+  ``--dump-input=fail``, and it's ``all`` when ``--dump-input=always``.
+
+  * ``all``             - All input lines
+  * ``annotation-full`` - Input lines with annotations
+  * ``annotation``      - Input lines with starting points of annotations
+  * ``error``           - Input lines with starting points of error annotations
 
 .. option:: --enable-var-scope
 
@@ -137,15 +163,15 @@ and from the command line.
 
 .. option:: -v
 
-  Print good directive pattern matches.  However, if ``-input-dump=fail`` or
-  ``-input-dump=always``, add those matches as input annotations instead.
+  Print good directive pattern matches.  However, if ``-dump-input=fail`` or
+  ``-dump-input=always``, add those matches as input annotations instead.
 
 .. option:: -vv
 
   Print information helpful in diagnosing internal FileCheck issues, such as
   discarded overlapping ``CHECK-DAG:`` matches, implicit EOF pattern matches,
   and ``CHECK-NOT:`` patterns that do not have matches.  Implies ``-v``.
-  However, if ``-input-dump=fail`` or ``-input-dump=always``, just add that
+  However, if ``-dump-input=fail`` or ``-dump-input=always``, just add that
   information as input annotations instead.
 
 .. option:: --allow-deprecated-dag-overlap
@@ -696,7 +722,7 @@ The syntax of a numeric substitution is
   A numeric operand is a previously defined numeric variable, an integer
   literal, or a function. Spaces are accepted before, after and between any of
   these elements. Numeric operands have 64-bit precision. Overflow and underflow
-  are rejected. There is no support for operator precendence, but parentheses
+  are rejected. There is no support for operator precedence, but parentheses
   can be used to change the evaluation order.
 
 The supported operators are:
@@ -715,7 +741,7 @@ The syntax of a function call is ``<name>(<arguments>)`` where:
   * mul - Returns the product of its two operands.
   * sub - Returns the difference of its two operands.
 
-* ``<arguments>`` is a comma seperated list of expressions.
+* ``<arguments>`` is a comma separated list of expressions.
 
 For example:
 
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index dfc079e9974c2..d29d293f32d8d 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -158,11 +158,10 @@ EXECUTION OPTIONS
 SELECTION OPTIONS
 -----------------
 
-.. option:: --max-failures MAX_FAILURES
+.. option:: --max-failures N
 
- Stop execution after the given number of failures.
- An integer argument may be passed on the command line
- or the environment vairable MAX_FAILURES may be set
+ Stop execution after the given number ``N`` of failures.
+ An integer argument should be passed on the command line
  prior to execution.
 
 .. option:: --max-tests=N
diff --git a/llvm/docs/CommandGuide/llvm-dwarfdump.rst b/llvm/docs/CommandGuide/llvm-dwarfdump.rst
index d31af50e63f7c..d0c7be4421365 100644
--- a/llvm/docs/CommandGuide/llvm-dwarfdump.rst
+++ b/llvm/docs/CommandGuide/llvm-dwarfdump.rst
@@ -149,7 +149,7 @@ OPTIONS
 
             Display the version of the tool.
 
-.. option:: --debug-abbrev, --debug-addr, --debug-aranges, --debug-cu-index, --debug-frame[=<offset>], --debug-gnu-pubnames, --debug-gnu-pubtypes, --debug-info[=<offset>], --debug-line[=<offset>], --debug-line-str, --debug-loc[=<offset>], --debug-loclists[=<offset>], --debug-macro, --debug-names, --debug-pubnames, --debug-pubtypes, --debug-ranges, --debug-rnglists, --debug-str, --debug-str-offsets, --debug-tu-index, --debug-types[=<offset>], --eh-frame[=<offset>], --gdb-index, --apple-names, --apple-types, --apple-namespaces, --apple-objc
+.. option:: --debug-abbrev, --debug-addr, --debug-aranges, --debug-cu-index, --debug-frame[=<offset>], --debug-gnu-pubnames, --debug-gnu-pubtypes, --debug-info [=<offset>], --debug-line [=<offset>], --debug-line-str, --debug-loc [=<offset>], --debug-loclists [=<offset>], --debug-macro, --debug-names, --debug-pubnames, --debug-pubtypes, --debug-ranges, --debug-rnglists, --debug-str, --debug-str-offsets, --debug-tu-index, --debug-types [=<offset>], --eh-frame [=<offset>], --gdb-index, --apple-names, --apple-types, --apple-namespaces, --apple-objc
 
             Dump the specified DWARF section by name. Only the
             `.debug_info` section is shown by default. Some entries
diff --git a/llvm/docs/CommandGuide/llvm-nm.rst b/llvm/docs/CommandGuide/llvm-nm.rst
index c47be8034efc2..607c871ff7a1b 100644
--- a/llvm/docs/CommandGuide/llvm-nm.rst
+++ b/llvm/docs/CommandGuide/llvm-nm.rst
@@ -219,7 +219,7 @@ OPTIONS
 
 .. option:: --special-syms
 
- Ignored. For GNU compatibility only.
+ Do not filter special symbols from the output.
 
 .. option:: --undefined-only, -u
 
diff --git a/llvm/docs/CommandGuide/llvm-objdump.rst b/llvm/docs/CommandGuide/llvm-objdump.rst
index 2de48d66536fe..df4cf746abbb9 100644
--- a/llvm/docs/CommandGuide/llvm-objdump.rst
+++ b/llvm/docs/CommandGuide/llvm-objdump.rst
@@ -123,6 +123,17 @@ OPTIONS
 
   Demangle symbol names in the output.
 
+.. option:: --debug-vars=<format>
+
+  Print the locations (in registers or memory) of source-level variables
+  alongside disassembly. ``format`` may be ``unicode`` or ``ascii``, defaulting
+  to ``unicode`` if omitted.
+
+.. option:: --debug-vars-indent=<width>
+
+  Distance to indent the source-level variable display, relative to the start
+  of the disassembly. Defaults to 40 characters.
+
 .. option:: -j, --section=<section1[,section2,...]>
 
   Perform commands on the specified sections only. For Mach-O use
diff --git a/llvm/docs/Contributing.rst b/llvm/docs/Contributing.rst
index 9185bfdf49177..45d4d7a27fa71 100644
--- a/llvm/docs/Contributing.rst
+++ b/llvm/docs/Contributing.rst
@@ -37,6 +37,11 @@ and use the built binaries to reproduce the failure described in the bug. Use
 a debug build (`-DCMAKE_BUILD_TYPE=Debug`) or a build with assertions
 (`-DLLVM_ENABLE_ASSERTIONS=On`, enabled for Debug builds).
 
+Reporting a Security Issue
+--------------------------
+
+There is a separate process to submit security-related bugs, see :ref:`report-security-issue`.
+
 Bigger Pieces of Work
 ---------------------
 In case you are interested in taking on a bigger piece of work, a list of
diff --git a/llvm/docs/DeveloperPolicy.rst b/llvm/docs/DeveloperPolicy.rst
index 22f29a06b2893..b6e43ce0534e0 100644
--- a/llvm/docs/DeveloperPolicy.rst
+++ b/llvm/docs/DeveloperPolicy.rst
@@ -521,13 +521,74 @@ C API Changes
   release notes so that it's clear to external users who do not follow the
   project how the C API is changing and evolving.
 
-New Targets
------------
+.. _toolchain:
+
+Updating Toolchain Requirements
+-------------------------------
+
+We intend to require newer toolchains as time goes by. This means LLVM's
+codebase can use newer versions of C++ as they get standardized. Requiring newer
+toolchains to build LLVM can be painful for those building LLVM; therefore, it
+will only be done through the following process:
+
+  * It is a general goal to support LLVM and GCC versions from the last 3 years
+    at a minimum. This time-based guideline is not strict: we may support much
+    older compilers, or decide to support fewer versions.
+
+  * An RFC is sent to the `llvm-dev mailing list`_
+
+    - Detail upsides of the version increase (e.g. which newer C++ language or
+      library features LLVM should use; avoid miscompiles in particular compiler
+      versions, etc).
+    - Detail downsides on important platforms (e.g. Ubuntu LTS status).
+
+  * Once the RFC reaches consensus, update the CMake toolchain version checks as
+    well as the :doc:`getting started<GettingStarted>` guide.  This provides a
+    softer transition path for developers compiling LLVM, because the
+    error can be turned into a warning using a CMake flag. This is an important
+    step: LLVM still doesn't have code which requires the new toolchains, but it
+    soon will. If you compile LLVM but don't read the mailing list, we should
+    tell you!
+
+  * Ensure that at least one LLVM release has had this soft-error. Not all
+    developers compile LLVM top-of-tree. These release-bound developers should
+    also be told about upcoming changes.
+
+  * Turn the soft-error into a hard-error after said LLVM release has branched.
+
+  * Update the :doc:`coding standards<CodingStandards>` to allow the new
+    features we've explicitly approved in the RFC.
+
+  * Start using the new features in LLVM's codebase.
+
+Here's a `sample RFC
+<http://lists.llvm.org/pipermail/llvm-dev/2019-January/129452.html>`_ and the
+`corresponding change <https://reviews.llvm.org/D57264>`_.
+
+.. _new-llvm-components:
+
+Introducing New Components into LLVM
+====================================
+
+The LLVM community is a vibrant and exciting place to be, and we look to be
+inclusive of new projects and foster new communities, and increase
+collaboration across industry and academia.
+
+That said, we need to strike a balance between being inclusive of new ideas and
+people and the cost of ongoing maintenance that new code requires.  As such, we
+have the following general policies for introducing major new components into
+the LLVM world.  However, this is really only intended to cover common cases
+that we have seen arise: different situations are different, and we are open
+to discussing unusual cases as well - just start an RFC thread on the
+`llvm-dev mailing list`_.
+
+Adding a New Target
+-------------------
 
 LLVM is very receptive to new targets, even experimental ones, but a number of
 problems can appear when adding new large portions of code, and back-ends are
-normally added in bulk.  We have found that landing large pieces of new code 
-and then trying to fix emergent problems in-tree is problematic for a variety 
+normally added in bulk.  We have found that landing large pieces of new code
+and then trying to fix emergent problems in-tree is problematic for a variety
 of reasons.
 
 For these reasons, new targets are *always* added as *experimental* until
@@ -566,8 +627,8 @@ The basic rules for a back-end to be upstreamed in **experimental** mode are:
 * The target should have either reasonable documentation on how it
   works (ISA, ABI, etc.) or a publicly available simulator/hardware
   (either free or cheap enough) - preferably both.  This allows
-  developers to validate assumptions, understand constraints and review code 
-  that can affect the target. 
+  developers to validate assumptions, understand constraints and review code
+  that can affect the target.
 
 In addition, the rules for a back-end to be promoted to **official** are:
 
@@ -606,49 +667,110 @@ In essences, these rules are necessary for targets to gain and retain their
 status, but also markers to define bit-rot, and will be used to clean up the
 tree from unmaintained targets.
 
-.. _toolchain:
-
-Updating Toolchain Requirements
--------------------------------
-
-We intend to require newer toolchains as time goes by. This means LLVM's
-codebase can use newer versions of C++ as they get standardized. Requiring newer
-toolchains to build LLVM can be painful for those building LLVM; therefore, it
-will only be done through the following process:
-
-  * Generally, try to support LLVM and GCC versions from the last 3 years at a
-    minimum. This time-based guideline is not strict: we may support much older
-    compilers, or decide to support fewer versions.
-
-  * An RFC is sent to the `llvm-dev mailing list <http://lists.llvm.org/mailman/listinfo/llvm-dev>`_
-
-    - Detail upsides of the version increase (e.g. which newer C++ language or
-      library features LLVM should use; avoid miscompiles in particular compiler
-      versions, etc).
-    - Detail downsides on important platforms (e.g. Ubuntu LTS status).
-
-  * Once the RFC reaches consensus, update the CMake toolchain version checks as
-    well as the :doc:`getting started<GettingStarted>` guide. We want to
-    soft-error when developers compile LLVM. We say "soft-error" because the
-    error can be turned into a warning using a CMake flag. This is an important
-    step: LLVM still doesn't have code which requires the new toolchains, but it
-    soon will. If you compile LLVM but don't read the mailing list, we should
-    tell you!
-
-  * Ensure that at least one LLVM release has had this soft-error. Not all
-    developers compile LLVM top-of-tree. These release-bound developers should
-    also be told about upcoming changes.
-
-  * Turn the soft-error into a hard-error after said LLVM release has branched.
+Adding an Established Project To the LLVM Monorepo
+--------------------------------------------------
+
+The `LLVM monorepo <https://github.com/llvm/llvm-project>`_ is the centerpoint
+of development in the LLVM world, and has all of the primary LLVM components,
+including the LLVM optimizer and code generators, Clang, LLDB, etc.  `Monorepos
+in general <https://en.wikipedia.org/wiki/Monorepo>`_ are great because they
+allow atomic commits to the project, simplify CI, and make it easier for
+subcommunities to collaborate.
+
+That said, the burden to add things to the LLVM monorepo needs to be very high -
+code that is added to this repository is checked out by everyone in the
+community.  As such, we hold subprojects to a high bar similar to "official
+targets", they:
+
+ * Must be generally aligned with the mission of the LLVM project to advance
+   compilers, languages, tools, runtimes, etc.
+ * Must conform to all of the policies laid out in this developer policy
+   document, including license, patent, coding standards, and code of conduct.
+ * Must have an active community that maintains the code, including established
+   code owners.
+ * Should have reasonable documentation about how it works, including a high
+   quality README file.
+ * Should have CI to catch breakage within the project itself or due to
+   underlying LLVM dependencies.
+ * Should have code free of issues the community finds contentious, or be on a
+   clear path to resolving them.
+ * Must be proposed through the LLVM RFC process, and have its addition approved
+   by the LLVM community - this ultimately mediates the resolution of the
+   "should" concerns above.
+
+If you have a project that you think would make sense to add to the LLVM
+monorepo, please start an RFC thread on the `llvm-dev mailing list`_ to kick off
+the discussion.  This process can take some time and iteration - please don’t
+be discouraged or intimidated by that!
+
+If you have an earlier stage project that you think is aligned with LLVM, please
+see the "Incubating New Projects" section.
+
+Incubating New Projects
+-----------------------
 
-  * Update the :doc:`coding standards<CodingStandards>` to allow the new
-    features we've explicitly approved in the RFC.
+The burden to add a new project to the LLVM monorepo is intentionally very high,
+but that can have a chilling effect on new and innovative projects.  To help
+foster these sorts of projects, LLVM supports an "incubator" process that is
+much easier to get started with.  It provides space for potentially valuable,
+new top-level and sub-projects to reach a critical mass before they have enough
+code to prove their utility and grow a community.  This also allows
+collaboration between teams that already have permissions to make contributions
+to projects under the LLVM umbrella.
+
+Projects which can be considered for the LLVM incubator meet the following
+criteria:
+
+ * Must be generally aligned with the mission of the LLVM project to advance
+   compilers, languages, tools, runtimes, etc.
+ * Must conform to the license, patent, and code of conduct policies laid out
+   in this developer policy document.
+ * Must have a documented charter and development plan, e.g. in the form of a
+   README file, mission statement, and/or manifesto.
+ * Should conform to coding standards, incremental development process, and
+   other expectations.
+ * Should have a sense of the community that it hopes to eventually foster, and
+   there should be interest from members with different affiliations /
+   organizations.
+ * Should have a feasible path to eventually graduate as a dedicated top-level
+   or sub-project within the `LLVM monorepo
+   <https://github.com/llvm/llvm-project>`_.
+ * Should include a notice (e.g. in the project README or web page) that the
+   project is in ‘incubation status’ and is not included in LLVM releases (see
+   suggested wording below).
+ * Must be proposed through the LLVM RFC process, and have its addition
+   approved by the LLVM community - this ultimately mediates the resolution of
+   the "should" concerns above.
+
+That said, the project need not have any code to get started, and need not have
+an established community at all!  Furthermore, incubating projects may pass
+through transient states that violate the "Should" guidelines above, or would
+otherwise make them unsuitable for direct inclusion in the monorepo (e.g.
+dependencies that have not yet been factored appropriately, leveraging
+experimental components or APIs that are not yet upstream, etc).
+
+When approved, the llvm-admin group can grant the new project:
+ * A new repository in the LLVM Github Organization - but not the LLVM monorepo.
+ * New mailing list, discourse forum, and/or discord chat hosted with other LLVM
+   forums.
+ * Other infrastructure integration can be discussed on a case-by-case basis.
+
+Graduation to the mono-repo would follow existing processes and standards for
+becoming a first-class part of the monorepo.  Similarly, an incubating project
+may be eventually retired, but no process has been established for that yet.  If
+and when this comes up, please start an RFC discussion on llvm-dev.
+
+This process is very new - please expect the details to change, it is always
+safe to ask on the `llvm-dev mailing list`_ about this.
+
+Suggested disclaimer for the project README and the main project web page:
 
-  * Start using the new features in LLVM's codebase.
+::
 
-Here's a `sample RFC
-<http://lists.llvm.org/pipermail/llvm-dev/2019-January/129452.html>`_ and the
-`corresponding change <https://reviews.llvm.org/D57264>`_.
+   This project is participating in the LLVM Incubator process: as such, it is
+   not part of any official LLVM release.  While incubation status is not
+   necessarily a reflection of the completeness or stability of the code, it
+   does indicate that the project is not yet endorsed as a component of LLVM.
 
 .. _copyright-license-patents:
 
@@ -910,3 +1032,5 @@ applications to the binary redistribution clause. This also means that it is ok
 to move code from (e.g.)  libc++ to the LLVM core without concern, but that code
 cannot be moved from the LLVM core to libc++ without the copyright owner's
 permission.
+
+.. _llvm-dev mailing list: http://lists.llvm.org/mailman/listinfo/llvm-dev
diff --git a/llvm/docs/GettingStarted.rst b/llvm/docs/GettingStarted.rst
index 5cce01b72c116..478571bd4c238 100644
--- a/llvm/docs/GettingStarted.rst
+++ b/llvm/docs/GettingStarted.rst
@@ -528,6 +528,17 @@ that include merges, so the `git rebase` step above is required.
 
 Please ask for help if you're having trouble with your particular git workflow.
 
+Git pre-push hook
+^^^^^^^^^^^^^^^^^
+
+We include an optional pre-push hook that run some sanity checks on the revisions
+you are about to push and ask confirmation if you push multiple commits at once.
+You can set it up (on Unix systems) by running from the repository root:
+
+.. code-block:: console
+
+  % ln -sf ../../llvm/utils/git/pre-push.py .git/hooks/pre-push
+
 Bisecting commits
 ^^^^^^^^^^^^^^^^^
 
diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst
index 33611ea31a357..5c0be7ef70b52 100644
--- a/llvm/docs/GlobalISel/GenericOpcode.rst
+++ b/llvm/docs/GlobalISel/GenericOpcode.rst
@@ -294,7 +294,7 @@ G_PTRMASK
 
 Zero out an arbitrary mask of bits of a pointer. The mask type must be
 an integer, and the number of vector elements must match for all
-operands. This corresponds to :ref:`i_intr_llvm_ptrmask`.
+operands. This corresponds to `i_intr_llvm_ptrmask`.
 
 .. code-block:: none
 
diff --git a/llvm/docs/HowToSubmitABug.rst b/llvm/docs/HowToSubmitABug.rst
index ac28f290bbdd1..58c020fbaedb4 100644
--- a/llvm/docs/HowToSubmitABug.rst
+++ b/llvm/docs/HowToSubmitABug.rst
@@ -10,6 +10,8 @@ If you're working with LLVM and run into a bug, we definitely want to know
 about it.  This document describes what you can do to increase the odds of
 getting it fixed quickly.
 
+🔒 If you believe that the bug is security related, please follow :ref:`report-security-issue`. 🔒
+
 Basically you have to do two things at a minimum.  First, decide whether
 the bug `crashes the compiler`_ (or an LLVM pass), or if the
 compiler is `miscompiling`_ the program (i.e., the
diff --git a/llvm/docs/HowToUpdateDebugInfo.rst b/llvm/docs/HowToUpdateDebugInfo.rst
index f08722fb8eb93..3283bfd893393 100644
--- a/llvm/docs/HowToUpdateDebugInfo.rst
+++ b/llvm/docs/HowToUpdateDebugInfo.rst
@@ -139,7 +139,7 @@ Deleting an IR-level Instruction
 --------------------------------
 
 When an ``Instruction`` is deleted, its debug uses change to ``undef``. This is
-a loss of debug info: the value of a one or more source variables becomes
+a loss of debug info: the value of one or more source variables becomes
 unavailable, starting with the ``llvm.dbg.value(undef, ...)``. When there is no
 way to reconstitute the value of the lost instruction, this is the best
 possible outcome. However, it's often possible to do better:
diff --git a/llvm/docs/HowToUseAttributes.rst b/llvm/docs/HowToUseAttributes.rst
index 1d05e23858740..cc8a00ec5454b 100644
--- a/llvm/docs/HowToUseAttributes.rst
+++ b/llvm/docs/HowToUseAttributes.rst
@@ -57,7 +57,7 @@ An ``AttributeList`` object is designed to be passed around by value.
 
 Note: It is advised that you do *not* use the ``AttributeList`` "introspection"
 methods (e.g. ``Raw``, ``getRawPointer``, etc.).  These methods break
-encapsulation, and may be removed in a future release (i.e. LLVM 4.0).
+encapsulation, and may be removed in a future release.
 
 ``AttrBuilder``
 ===============
@@ -73,7 +73,7 @@ should be passed by reference.
 
 Note: It is advised that you do *not* use the ``AttrBuilder::addRawValue()``
 method or the ``AttrBuilder(uint64_t Val)`` constructor.  These are for
-backwards compatibility and may be removed in a future release (i.e. LLVM 4.0).
+backwards compatibility and may be removed in a future release.
 
 And that's basically it! A lot of functionality is hidden behind these classes,
 but the interfaces are pretty straight forward.
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index e8e481a0a411c..86d315be74bcf 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1130,7 +1130,7 @@ Currently, only the following parameter attributes are defined:
 
 .. _attr_align:
 
-``align <n>``
+``align <n>`` or ``align(<n>)``
     This indicates that the pointer value may be assumed by the optimizer to
     have the specified alignment.  If the pointer value does not have the
     specified alignment, behavior is undefined.
@@ -1252,6 +1252,12 @@ Currently, only the following parameter attributes are defined:
     only valid on intrinsic declarations and cannot be applied to a
     call site or arbitrary function.
 
+``noundef``
+    This attribute applies to parameters and return values. If the value
+    representation contains any undefined or poison bits, the behavior is
+    undefined. Note that this does not refer to padding introduced by the
+    type's storage representation.
+
 .. _gc:
 
 Garbage Collector Strategy Names
@@ -2366,6 +2372,7 @@ as follows:
       starting with ``?`` are not mangled in any way.
     * ``w``: Windows COFF mangling: Similar to ``x``, except that normal C
       symbols do not receive a ``_`` prefix.
+    * ``a``: XCOFF mangling: Private symbols get a ``L..`` prefix.
 ``n<size1>:<size2>:<size3>...``
     This specifies a set of native integer widths for the target CPU in
     bits. For example, it might contain ``n32`` for 32-bit PowerPC,
@@ -2771,7 +2778,9 @@ floating-point transformations.
 
 ``contract``
    Allow floating-point contraction (e.g. fusing a multiply followed by an
-   addition into a fused multiply-and-add).
+   addition into a fused multiply-and-add). This does not enable reassociating
+   to form arbitrary contractions. For example, ``(a*b) + (c*d) + e`` can not
+   be transformed into ``(a*b) + ((c*d) + e)`` to create two fma operations.
 
 ``afn``
    Approximate functions - Allow substitution of approximate calculations for
@@ -3656,6 +3665,11 @@ behavior. Notably this includes (but is not limited to):
 -  The condition operand of a :ref:`br <i_br>` instruction.
 -  The callee operand of a :ref:`call <i_call>` or :ref:`invoke <i_invoke>`
    instruction.
+-  The parameter operand of a :ref:`call <i_call>` or :ref:`invoke <i_invoke>`
+   instruction, when the function or invoking call site has a ``noundef``
+   attribute in the corresponding position.
+-  The operand of a :ref:`ret <i_ret>` instruction if the function or invoking
+   call site has a `noundef` attribute in the return value position.
 
 Here are some examples:
 
@@ -4146,7 +4160,13 @@ AMDGPU:
 - ``[0-9]v``: The 32-bit VGPR register, number 0-9.
 - ``[0-9]s``: The 32-bit SGPR register, number 0-9.
 - ``[0-9]a``: The 32-bit AGPR register, number 0-9.
+- ``I``: An integer inline constant in the range from -16 to 64.
+- ``J``: A 16-bit signed integer constant.
 - ``A``: An integer or a floating-point inline constant.
+- ``B``: A 32-bit signed integer constant.
+- ``C``: A 32-bit unsigned integer constant or an integer inline constant in the range from -16 to 64.
+- ``DA``: A 64-bit constant that can be split into two "A" constants.
+- ``DB``: A 64-bit constant that can be split into two "B" constants.
 
 All ARM modes:
 
@@ -4619,7 +4639,7 @@ to the ``add`` instruction using the ``!dbg`` identifier:
     %indvar.next = add i64 %indvar, 1, !dbg !21
 
 Metadata can also be attached to a function or a global variable. Here metadata
-``!22`` is attached to the ``f1`` and ``f2 functions, and the globals ``g1``
+``!22`` is attached to the ``f1`` and ``f2`` functions, and the globals ``g1``
 and ``g2`` using the ``!dbg`` identifier:
 
 .. code-block:: llvm
@@ -6825,7 +6845,9 @@ function and looks like:
     param: 4, offset: [0, 5][, calls: ((Callee)[, (Callee)]*)]?
 
 where the first ``param`` is the number of the parameter it describes,
-``offset`` is the known access range of the paramenter inside of the function.
+``offset`` is the inclusive range of offsets from the pointer parameter to bytes
+which can be accessed by the function. This range does not include accesses by
+function calls from ``calls`` list.
 
 where each ``Callee`` decribes how parameter is forwared into other
 functions and looks like:
@@ -6836,7 +6858,44 @@ functions and looks like:
 
 The ``callee`` refers to the summary entry id of the callee,  ``param`` is
 the number of the callee parameter which points into the callers parameter
-with offset known to be inside of the ``offset`` range.
+with offset known to be inside of the ``offset`` range. ``calls`` will be
+consumed and removed by thin link stage to update ``Param::offset`` so it
+covers all accesses possible by ``calls``.
+
+Pointer parameter without corresponding ``Param`` is considered unsafe and we
+assume that access with any offset is possible.
+
+Example:
+
+If we have the following function:
+
+.. code-block:: text
+
+    define i64 @foo(i64* %0, i32* %1, i8* %2, i8 %3) {
+      store i32* %1, i32** @x
+      %5 = getelementptr inbounds i8, i8* %2, i64 5
+      %6 = load i8, i8* %5
+      %7 = getelementptr inbounds i8, i8* %2, i8 %3
+      tail call void @bar(i8 %3, i8* %7)
+      %8 = load i64, i64* %0
+      ret i64 %8
+    }
+
+We can expect the record like this:
+
+.. code-block:: text
+
+    params: ((param: 0, offset: [0, 7]),(param: 2, offset: [5, 5], calls: ((callee: ^3, param: 1, offset: [-128, 127]))))
+
+The function may access just 8 bytes of the paramenter %0 . ``calls`` is empty,
+so the parameter is either not used for function calls or ``offset`` already
+covers all accesses from nested function calls.
+Parameter %1 escapes, so access is unknown.
+The function itself can access just a single byte of the parameter %2. Additional
+access is possible inside of the ``@bar`` or ``^3``. The function adds signed
+offset to the pointer and passes the result as the argument %1 into ``^3``.
+This record itself does not tell us how ``^3`` will access the parameter.
+Parameter %3 is not a pointer.
 
 .. _refs_summary:
 
@@ -12096,6 +12155,65 @@ It is undefined behavior if this is called with a token from an
 preallocated call corresponding to the '``llvm.call.preallocated.setup``'
 has already been called.
 
+.. _int_call_preallocated_teardown:
+
+'``llvm.call.preallocated.teardown``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+      declare i8* @llvm.call.preallocated.teardown(token %setup_token)
+
+Overview:
+"""""""""
+
+The '``llvm.call.preallocated.teardown``' intrinsic cleans up the stack
+created by a '``llvm.call.preallocated.setup``'.
+
+Semantics:
+""""""""""
+
+The token argument must be a '``llvm.call.preallocated.setup``'.
+
+The '``llvm.call.preallocated.teardown``' intrinsic cleans up the stack
+allocated by the corresponding '``llvm.call.preallocated.setup``'. Exactly
+one of this or the preallocated call must be called to prevent stack leaks.
+It is undefined behavior to call both a '``llvm.call.preallocated.teardown``'
+and the preallocated call for a given '``llvm.call.preallocated.setup``'.
+
+For example, if the stack is allocated for a preallocated call by a
+'``llvm.call.preallocated.setup``', then an initializer function called on an
+allocated argument throws an exception, there should be a
+'``llvm.call.preallocated.teardown``' in the exception handler to prevent
+stack leaks.
+
+Following the nesting rules in '``llvm.call.preallocated.setup``', nested
+calls to '``llvm.call.preallocated.setup``' and
+'``llvm.call.preallocated.teardown``' are allowed but must be properly
+nested.
+
+Example:
+""""""""
+
+.. code-block:: llvm
+
+        %cs = call token @llvm.call.preallocated.setup(i32 1)
+        %x = call i8* @llvm.call.preallocated.arg(token %cs, i32 0) preallocated(i32)
+        %y = bitcast i8* %x to i32*
+        invoke void @constructor(i32* %y) to label %conta unwind label %contb
+    conta:
+        call void @foo1(i32* preallocated(i32) %y) ["preallocated"(token %cs)]
+        ret void
+    contb:
+        %s = catchswitch within none [label %catch] unwind to caller
+    catch:
+        %p = catchpad within %s []
+        call void @llvm.call.preallocated.teardown(token %cs)
+        ret void
+
 Standard C Library Intrinsics
 -----------------------------
 
@@ -19063,6 +19181,40 @@ Semantics:
 
 This intrinsic is lowered to the ``val``.
 
+'``llvm.expect.with.probability``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+This intrinsic is similar to ``llvm.expect``. This is an overloaded intrinsic.
+You can use ``llvm.expect.with.probability`` on any integer bit width.
+
+::
+
+      declare i1 @llvm.expect.with.probability.i1(i1 <val>, i1 <expected_val>, double <prob>)
+      declare i32 @llvm.expect.with.probability.i32(i32 <val>, i32 <expected_val>, double <prob>)
+      declare i64 @llvm.expect.with.probability.i64(i64 <val>, i64 <expected_val>, double <prob>)
+
+Overview:
+"""""""""
+
+The ``llvm.expect.with.probability`` intrinsic provides information about
+expected value of ``val`` with probability(or confidence) ``prob``, which can
+be used by optimizers.
+
+Arguments:
+""""""""""
+
+The ``llvm.expect.with.probability`` intrinsic takes three arguments. The first
+argument is a value. The second argument is an expected value. The third
+argument is a probability.
+
+Semantics:
+""""""""""
+
+This intrinsic is lowered to the ``val``.
+
 .. _int_assume:
 
 '``llvm.assume``' Intrinsic
diff --git a/llvm/docs/Passes.rst b/llvm/docs/Passes.rst
index abb7b9d820e0e..9a6c6944b96ef 100644
--- a/llvm/docs/Passes.rst
+++ b/llvm/docs/Passes.rst
@@ -66,8 +66,8 @@ function.
 This is inspired and adapted from code by: Naveen Neelakantam, Francesco
 Spadini, and Wojciech Stryjewski.
 
-``-basicaa``: Basic Alias Analysis (stateless AA impl)
-------------------------------------------------------
+``-basic-aa``: Basic Alias Analysis (stateless AA impl)
+-------------------------------------------------------
 
 A basic alias analysis pass that implements identities (two different globals
 cannot alias, etc), but does no stateful analysis.
diff --git a/llvm/docs/Phabricator.rst b/llvm/docs/Phabricator.rst
index 54789df71dd21..2493d4eea17d8 100644
--- a/llvm/docs/Phabricator.rst
+++ b/llvm/docs/Phabricator.rst
@@ -82,8 +82,7 @@ To upload a new patch:
 * Add reviewers (see below for advice). (If you set the Repository field
   correctly, llvm-commits or cfe-commits will be subscribed automatically;
   otherwise, you will have to manually subscribe them.)
-* In the Repository field, enter the name of the project (LLVM, Clang,
-  etc.) to which the review should be sent.
+* In the Repository field, enter "rG LLVM Github Monorepo".
 * Click *Save*.
 
 To submit an updated patch:
@@ -183,6 +182,13 @@ that you close the review manually. In the web UI, under "Leap Into Action" put
 the git revision number in the Comment, set the Action to "Close Revision" and
 click Submit.  Note the review must have been Accepted first.
 
+Arcanist also adds extra tags that are mostly noise in the commit message, for
+this reason avoid using `arc land` and push commits to master directly with git
+after removing tags other than "Reviewed by" and "Differential Revision".
+You can run `llvm/utils/git/arcfilter.sh` to clean the commit message of the
+current "HEAD" commit automatically. You can also setup a git hook to catch this
+for you (see `Getting Started <GettingStarted.html#git-pre-push-hook>`).
+
 
 Committing someone's change from Phabricator
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -217,12 +223,6 @@ the following:
   ninja check-$whatever # Rerun the appropriate tests if needed.
   git push https://github.com/llvm/llvm-project.git HEAD:master
 
-Or
-
-::
-
-  arc land D<Revision>
-
 
 Abandoning a change
 -------------------
diff --git a/llvm/docs/Reference.rst b/llvm/docs/Reference.rst
index d116edafb9bfb..3911094fbfd65 100644
--- a/llvm/docs/Reference.rst
+++ b/llvm/docs/Reference.rst
@@ -37,6 +37,7 @@ LLVM and API reference documentation.
    PDB/index
    ScudoHardenedAllocator
    MemTagSanitizer
+   Security
    SegmentedStacks
    StackMaps
    SpeculativeLoadHardening
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 7d71d0c58a4c1..8d8da954ece39 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -187,6 +187,10 @@ Changes to the LLVM tools
 * Added an option (--show-section-sizes) to llvm-dwarfdump to show the sizes
   of all debug sections within a file.
 
+* llvm-nm now implements the flag ``--special-syms`` and will filter out special
+  symbols, i.e. mapping symbols on ARM and AArch64, by default. This matches
+  the GNU nm behavior.
+
 Changes to LLDB
 ===============
 
diff --git a/llvm/docs/Security.rst b/llvm/docs/Security.rst
new file mode 100644
index 0000000000000..62085cc1c2028
--- /dev/null
+++ b/llvm/docs/Security.rst
@@ -0,0 +1,220 @@
+===================
+LLVM Security Group
+===================
+
+The LLVM Security Group has the following goals:
+
+1. Allow LLVM contributors and security researchers to disclose security-related issues affecting the LLVM project to members of the LLVM community.
+2. Organize fixes, code reviews, and release management for said issues.
+3. Allow distributors time to investigate and deploy fixes before wide dissemination of vulnerabilities or mitigation shortcomings.
+4. Ensure timely notification and release to vendors who package and distribute LLVM-based toolchains and projects.
+5. Ensure timely notification to users of LLVM-based toolchains whose compiled code is security-sensitive, through the `CVE process`_.
+6. Strive to improve security over time, for example by adding additional testing, fuzzing, and hardening after fixing issues.
+
+*Note*: these goals ensure timely action, provide disclosure timing when issues are reported, and respect vendors' / packagers' / users' constraints.
+
+The LLVM Security Group is private. It is composed of trusted LLVM contributors. Its discussions remain within the Security Group (plus issue reporter and key experts) while an issue is being investigated. After an issue becomes public, the entirety of the group’s discussions pertaining to that issue also become public.
+
+
+Group Composition
+=================
+
+Security Group Members
+----------------------
+
+The members of the group represent a wide cross-section of the community, and meet the criteria for inclusion below.
+
+* Akila Srinivasan (Apple)
+* Dimitry Andric (invidual; FreeBSD)
+* Ed Maste (individual; FreeBSD)
+* JF Bastien (Apple)
+* Josh Eads (Sony)
+* Kristof Beyls (ARM)
+* Matthew Riley (Google)
+* Oliver Hunt (Apple)
+* Paul Robinson (Sony)
+* Peter Smith (ARM)
+* Philip Reames (Azul Systems Inc)
+* Pietro Albini (individual; Rust)
+* Serge Guelton (RedHat)
+* Shayne Hiet-Block (Microsoft)
+* Steve Klabnik (Oxide Computer Company; Rust)
+
+Criteria
+--------
+
+* Nominees for LLVM Security Group membership should fall in one of these groups:
+
+  - Individual contributors:
+
+    + Specializes in fixing compiler-based security related issues or often participates in their exploration and resolution.
+    + Has a track record of finding security vulnerabilities and responsible disclosure of those vulnerabilities.
+    + Is a compiler expert who has specific interests in knowing about, resolving, and preventing future security vulnerabilities.
+    + Has actively contributed non-trivial code to the LLVM project in the last year.
+
+  - Researchers:
+
+    + Has a track record of finding security vulnerabilities and responsible disclosure of those vulnerabilities.
+    + Is a compiler expert who has specific interests in knowing about, resolving, and preventing future security vulnerabilities.
+
+  - Vendor contacts:
+
+    + Represents an organization or company which ships products that include their own copy of LLVM. Due to their position in the organization, the nominee has a reasonable need to know about security issues and disclosure embargoes.
+
+* Additionally, the following are necessary but not sufficient criteria for membership in the LLVM Security Group:
+
+  - If already in the LLVM Security Group, has actively participated in one (if any) security issue in the last year.
+  - If already in the LLVM Security Group, has actively participated in most membership discussions in the last year.
+  - If already in the LLVM Security Group, has actively participated in writing or reviewing a transparency report in the last year.
+  - When employed by a company or other entity, the parent entity has no more than three members already in the LLVM Security Group.
+  - When nominated as a vendor contact, their position with that vendor remains the same as when originally nominated.
+  - Nominees are trusted by existing Security Group members to keep communications embargoed while still active.
+
+Nomination process
+------------------
+
+Anyone who feels they meet these criteria can nominate themselves, or may be nominated by a third party such as an existing LLVM Security Group member. The nomination should state whether the nominee is nominated as an individual, researcher, or as a vendor contact. It should clearly describe the grounds for nomination.
+
+*FUTURE*: where nomination occurs (mailing list, GitHub, etc), can be decided later. See `Discussion Medium`_ below.
+
+
+Choosing new members
+--------------------
+
+If a nomination for LLVM Security Group membership is supported by a majority of existing LLVM Security Group members, then it carries within five business days unless an existing member of the Security Group objects. If an objection is raised, the LLVM Security Group members should discuss the matter and try to come to consensus; failing this, the nomination will succeed only by a two-thirds supermajority vote of the LLVM Security Group.
+
+Accepting membership
+--------------------
+
+Before new LLVM Security Group membership is finalized, the successful nominee should accept membership and agree to abide by this security policy, particularly `Privileges and Responsibilities of LLVM Security Group Members`_ below.
+
+Keeping Membership Current
+--------------------------
+
+* At least every six months, the LLVM Security Group applies the above criteria. The membership list is pruned accordingly.
+* Any Security Group member can ask that the criteria be applied within the next five business days.
+* If a member of the LLVM Security Group does not act in accordance with the letter and spirit of this policy, then their LLVM Security Group membership can be revoked by a majority vote of the members, not including the person under consideration for revocation. After a member calls for a revocation vote, voting will be open for five business days.
+* Emergency suspension: an LLVM Security Group member who blatantly disregards the LLVM Security Policy may have their membership temporarily suspended on the request of any two members. In such a case, the requesting members should notify the Security Group with a description of the offense. At this point, membership will be temporarily suspended for five business days, pending outcome of the vote for permanent revocation.
+* The LLVM Board may remove any member from the LLVM Security Group.
+
+Transparency Report
+-------------------
+
+Every year, the LLVM Security Group must publish a transparency report. The intent of this report is to keep the community informed by summarizing the disclosures that have been made public in the last year. It shall contain a list of all public disclosures, as well as statistics on time to fix issues, length of embargo periods, and so on.
+
+
+Privileges and Responsibilities of LLVM Security Group Members
+==============================================================
+
+Access
+------
+
+LLVM Security Group members will be subscribed to a private `Discussion Medium`_ (*FUTURE*: see section below). It will be used for technical discussions of security issues, as well as process discussions about matters such as disclosure timelines and group membership. Members have access to all security issues.
+
+Confidentiality
+---------------
+
+Members of the LLVM Security Group will be expected to treat LLVM security issue information shared with the group as confidential until publicly disclosed:
+
+* Members should not disclose security issue information to non-members unless both members are employed by the same vendor of a LLVM based product, in which case information can be shared within that organization on a need-to-know basis and handled as confidential information normally is within that organization.
+* If the LLVM Security Group agrees, designated members may share issues with vendors of non-LLVM based products if their product suffers from the same issue. The non-LLVM vendor should be asked to respect the issue’s embargo date, and to not share the information beyond the need-to-know people within their organization.
+* If the LLVM Security Group agrees, key experts can be brought in to help address particular issues. The key expert should be asked to respect the issue’s embargo date, and to not share the information.
+
+Disclosure
+----------
+
+Following the process below, the LLVM Security Group decides on embargo date for public disclosure for each Security issue. An embargo may be lifted before the agreed-upon date if all vendors planning to ship a fix have already done so, and if the reporter does not object.
+
+Collaboration
+-------------
+
+Members of the LLVM Security Group are expected to:
+
+* Promptly share any LLVM vulnerabilities they become aware of.
+* Volunteer to drive issues forward.
+* Help evaluate the severity of incoming issues.
+* Help write and review patches to address security issues.
+* Participate in the member nomination and removal processes.
+
+
+Discussion Medium
+=================
+
+*FUTURE*: this section needs more work! Where discussions occur is influenced by other factors that are still open in this document. We can figure it out later.
+See other existing systems: `chromium issue tracker`_, tentative `GitHub security`_. It seems like bugzilla and email don’t meet security requirements.
+
+The medium used to host LLVM Security Group discussions is security-sensitive. It should therefore run on infrastructure which can meet our security expectations.
+
+This is where all security discussions occur:
+
+* File security issues.
+* Nominate new members.
+* Propose member removal.
+* Suggest policy changes.
+* Discuss security improvements to LLVM.
+
+
+When a new issue is filed, a template is provided to help issue reporters provide all relevant information.
+
+
+Process
+=======
+
+The following process occurs on the discussion medium for each reported issue:
+
+* A security issue reporter (not necessarily an LLVM contributor) reports an issue.
+* Within two business days, a member of the Security Group is put in charge of driving the issue to an acceptable resolution. This champion doesn’t need to be the same person for each issue. This person can self-nominate.
+* Members of the Security Group discuss in which circumstances (if any) an issue is relevant to security, and determine if it is a security issue.
+* Negotiate an embargo date for public disclosure, with a default minimum time limit of ninety days.
+* Security Group members can recommend that key experts be pulled in to specific issue discussions. The key expert can be pulled in unless there are objections from other Security Group members.
+* Patches are written and reviewed.
+* Backporting security patches from recent versions to old versions cannot always work. It is up to the Security Group to decide if such backporting should be done, and how far back.
+* The Security Group figures out how the LLVM project’s own releases, as well as individual vendors’ releases, can be timed to patch the issue simultaneously.
+* Embargo date can be delayed or pulled forward at the Security Group’s discretion.
+* The issue champion obtains a CVE entry from MITRE_.
+* Once the embargo expires, the patch is posted publicly according to LLVM’s usual code review process.
+* All security issues (as well as nomination / removal discussions) become public within approximately fourteen weeks of the fix landing in the LLVM repository. Precautions should be taken to avoid disclosing particularly sensitive data included in the report (e.g. username and password pairs).
+
+
+Changes to the Policy
+=====================
+
+The LLVM Security Policy may be changed by majority vote of the LLVM Security Group. Such changes also need to be approved by the LLVM Board.
+
+
+What is considered a security issue?
+====================================
+
+*FUTURE*: this section will be expanded once the Security Group is formed, and it agrees on an initial security surface area.
+
+The LLVM Project has a significant amount of code, and not all of it is considered security-sensitive. This is particularly true because LLVM is used in a wide variety of circumstances: there are different threat models, untrusted inputs differ, and the environment LLVM runs in is varied. Therefore, what the LLVM Project considers a security issue is what its members have signed up to maintain securely.
+
+As this security process matures, members of the LLVM community can propose that a part of the codebase be designated as security-sensitive (or no longer security-sensitive). This requires a rationale, and buy-in from the LLVM community as for any RFC. In some cases, parts of the codebase could be handled as security-sensitive but need significant work to get to the stage where that's manageable. The LLVM community will need to decide whether it wants to invest in making these parts of the code secure-able, and maintain these security properties over time. In all cases the LLVM Security Group should be consulted, since they'll be responding to security issues filed against these parts of the codebase.
+
+If you're not sure whether an issue is in-scope for this security process or not, err towards assuming that it is. The Security Group might agree or disagree and will explain its rationale in the report, as well as  update this document through the above process.
+
+The security-sensitive parts of the LLVM Project currently are:
+
+* None (this process is new, the list hasn't been populated yet)
+* *FUTURE*: this section will be expanded.
+
+The parts of the LLVM Project which are currently treated as non-security sensitive are:
+
+* Language front-ends, such as clang, for which a malicious input file can cause undesirable behavior. For example, a maliciously-crafter C or Rust source file can cause arbitrary code to execute in LLVM. These parts of LLVM haven't been hardened, and compiling untrusted code usually also includes running utilities such as `make` which can more readily perform malicious things.
+* *FUTURE*: this section will be expanded.
+
+.. _report-security-issue:
+
+How to report a security issue?
+===============================
+
+*FUTURE*: this section will be expanded once we’ve figured out other details above.
+
+Not everyone who wants to report a security issue will be familiar with LLVM, its community, and processes. Therefore, this needs to be easy to find on the LLVM website, and set clear expectations to issue reporters.
+
+
+
+.. _CVE process: https://cve.mitre.org
+.. _chromium issue tracker: https://crbug.com
+.. _GitHub security: https://help.github.com/en/articles/about-maintainer-security-advisories
+.. _MITRE: https://cve.mitre.org
diff --git a/llvm/docs/WritingAnLLVMPass.rst b/llvm/docs/WritingAnLLVMPass.rst
index b9ba13f302f12..88f481ba6b076 100644
--- a/llvm/docs/WritingAnLLVMPass.rst
+++ b/llvm/docs/WritingAnLLVMPass.rst
@@ -916,7 +916,7 @@ be registered with :ref:`RegisterAnalysisGroup
 As a concrete example of an Analysis Group in action, consider the
 `AliasAnalysis <https://llvm.org/doxygen/classllvm_1_1AliasAnalysis.html>`_
 analysis group.  The default implementation of the alias analysis interface
-(the `basicaa <https://llvm.org/doxygen/structBasicAliasAnalysis.html>`_ pass)
+(the `basic-aa <https://llvm.org/doxygen/structBasicAliasAnalysis.html>`_ pass)
 just does a few simple checks that don't require significant analysis to
 compute (such as: two different globals can never alias each other, etc).
 Passes that use the `AliasAnalysis
@@ -926,7 +926,7 @@ care which implementation of alias analysis is actually provided, they just use
 the designated interface.
 
 From the user's perspective, commands work just like normal.  Issuing the
-command ``opt -gvn ...`` will cause the ``basicaa`` class to be instantiated
+command ``opt -gvn ...`` will cause the ``basic-aa`` class to be instantiated
 and added to the pass sequence.  Issuing the command ``opt -somefancyaa -gvn
 ...`` will cause the ``gvn`` pass to use the ``somefancyaa`` alias analysis
 (which doesn't actually exist, it's just a hypothetical example) instead.
@@ -970,7 +970,7 @@ Every implementation of an analysis group should join using this macro.
 
   namespace {
     // Declare that we implement the AliasAnalysis interface
-    INITIALIZE_AG_PASS(BasicAA, AliasAnalysis, "basicaa",
+    INITIALIZE_AG_PASS(BasicAA, AliasAnalysis, "basic-aa",
         "Basic Alias Analysis (default AA impl)",
         false, // Is CFG Only?
         true,  // Is Analysis?
diff --git a/llvm/docs/conf.py b/llvm/docs/conf.py
index 13bbc826de863..aed5e06b6f508 100644
--- a/llvm/docs/conf.py
+++ b/llvm/docs/conf.py
@@ -26,7 +26,7 @@
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.intersphinx', 'sphinx.ext.todo', 'recommonmark']
+extensions = ['sphinx.ext.intersphinx', 'sphinx.ext.todo']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -34,9 +34,23 @@
 # The suffix of source filenames.
 source_suffix = {
     '.rst': 'restructuredtext',
-    '.md': 'markdown',
 }
 
+try:
+  import recommonmark
+except ImportError:
+  # manpages do not use any .md sources
+  if not tags.has('builder-man'):
+    raise
+else:
+  import sphinx
+  if sphinx.version_info >= (3, 0):
+    # This requires 0.5 or later.
+    extensions.append('recommonmark')
+  else:
+    source_parsers = {'.md': 'recommonmark.parser.CommonMarkParser'}
+  source_suffix['.md'] = 'markdown'
+
 # The encoding of source files.
 #source_encoding = 'utf-8-sig'
 
diff --git a/llvm/docs/index.rst b/llvm/docs/index.rst
index 7315d7278f8b3..7e0bc8c4552b9 100644
--- a/llvm/docs/index.rst
+++ b/llvm/docs/index.rst
@@ -83,6 +83,10 @@ LLVM welcomes contributions of all kinds. To learn more, see the following artic
 * :ref:`meetups-social-events`
 * :ref:`community-proposals`
 
+  Reporting a security issue
+
+* :ref:`report-security-issue`
+
 Indices and tables
 ==================
 
diff --git a/llvm/examples/ExceptionDemo/ExceptionDemo.cpp b/llvm/examples/ExceptionDemo/ExceptionDemo.cpp
index 0ecb527f4ec05..1b3ec7c91ddee 100644
--- a/llvm/examples/ExceptionDemo/ExceptionDemo.cpp
+++ b/llvm/examples/ExceptionDemo/ExceptionDemo.cpp
@@ -792,7 +792,7 @@ _Unwind_Reason_Code ourPersonality(int version, _Unwind_Action actions,
   }
 #endif
 
-  const uint8_t *lsda = _Unwind_GetLanguageSpecificData(context);
+  const uint8_t *lsda = (const uint8_t *)_Unwind_GetLanguageSpecificData(context);
 
 #ifdef DEBUG
   fprintf(stderr,
@@ -1959,11 +1959,13 @@ int main(int argc, char *argv[]) {
 
     executionEngine->finalizeObject();
 
+#ifndef NDEBUG
     fprintf(stderr, "\nBegin module dump:\n\n");
 
     module->dump();
 
     fprintf(stderr, "\nEnd module dump:\n");
+#endif
 
     fprintf(stderr, "\n\nBegin Test:\n");
 
diff --git a/llvm/include/llvm/ADT/Bitfields.h b/llvm/include/llvm/ADT/Bitfields.h
new file mode 100644
index 0000000000000..d93f6483fa527
--- /dev/null
+++ b/llvm/include/llvm/ADT/Bitfields.h
@@ -0,0 +1,289 @@
+//===-- llvm/ADT/Bitfield.h - Get and Set bits in an integer ---*- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements methods to test, set and extract typed bits from packed
+/// unsigned integers.
+///
+/// Why not C++ bitfields?
+/// ----------------------
+/// C++ bitfields do not offer control over the bit layout nor consistent
+/// behavior when it comes to out of range values.
+/// For instance, the layout is implementation defined and adjacent bits may be
+/// packed together but are not required to. This is problematic when storage is
+/// sparse and data must be stored in a particular integer type.
+///
+/// The methods provided in this file ensure precise control over the
+/// layout/storage as well as protection against out of range values.
+///
+/// Usage example
+/// -------------
+/// \code{.cpp}
+///  uint8_t Storage = 0;
+///
+///  // Store and retrieve a single bit as bool.
+///  using Bool = Bitfield::Element<bool, 0, 1>;
+///  Bitfield::set<Bool>(Storage, true);
+///  EXPECT_EQ(Storage, 0b00000001);
+///  //                          ^
+///  EXPECT_EQ(Bitfield::get<Bool>(Storage), true);
+///
+///  // Store and retrieve a 2 bit typed enum.
+///  // Note: enum underlying type must be unsigned.
+///  enum class SuitEnum : uint8_t { CLUBS, DIAMONDS, HEARTS, SPADES };
+///  // Note: enum maximum value needs to be passed in as last parameter.
+///  using Suit = Bitfield::Element<SuitEnum, 1, 2, SuitEnum::SPADES>;
+///  Bitfield::set<Suit>(Storage, SuitEnum::HEARTS);
+///  EXPECT_EQ(Storage, 0b00000101);
+///  //                        ^^
+///  EXPECT_EQ(Bitfield::get<Suit>(Storage), SuitEnum::HEARTS);
+///
+///  // Store and retrieve a 5 bit value as unsigned.
+///  using Value = Bitfield::Element<unsigned, 3, 5>;
+///  Bitfield::set<Value>(Storage, 10);
+///  EXPECT_EQ(Storage, 0b01010101);
+///  //                   ^^^^^
+///  EXPECT_EQ(Bitfield::get<Value>(Storage), 10U);
+///
+///  // Interpret the same 5 bit value as signed.
+///  using SignedValue = Bitfield::Element<int, 3, 5>;
+///  Bitfield::set<SignedValue>(Storage, -2);
+///  EXPECT_EQ(Storage, 0b11110101);
+///  //                   ^^^^^
+///  EXPECT_EQ(Bitfield::get<SignedValue>(Storage), -2);
+///
+///  // Ability to efficiently test if a field is non zero.
+///  EXPECT_TRUE(Bitfield::test<Value>(Storage));
+///
+///  // Alter Storage changes value.
+///  Storage = 0;
+///  EXPECT_EQ(Bitfield::get<Bool>(Storage), false);
+///  EXPECT_EQ(Bitfield::get<Suit>(Storage), SuitEnum::CLUBS);
+///  EXPECT_EQ(Bitfield::get<Value>(Storage), 0U);
+///  EXPECT_EQ(Bitfield::get<SignedValue>(Storage), 0);
+///
+///  Storage = 255;
+///  EXPECT_EQ(Bitfield::get<Bool>(Storage), true);
+///  EXPECT_EQ(Bitfield::get<Suit>(Storage), SuitEnum::SPADES);
+///  EXPECT_EQ(Bitfield::get<Value>(Storage), 31U);
+///  EXPECT_EQ(Bitfield::get<SignedValue>(Storage), -1);
+/// \endcode
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_BITFIELDS_H
+#define LLVM_ADT_BITFIELDS_H
+
+#include <cassert>
+#include <climits> // CHAR_BIT
+#include <cstddef> // size_t
+#include <cstdint> // uintXX_t
+#include <limits>  // numeric_limits
+#include <type_traits>
+
+namespace llvm {
+
+namespace bitfields_details {
+
+/// A struct defining useful bit patterns for n-bits integer types.
+template <typename T, unsigned Bits> struct BitPatterns {
+  /// Bit patterns are forged using the equivalent `Unsigned` type because of
+  /// undefined operations over signed types (e.g. Bitwise shift operators).
+  /// Moreover same size casting from unsigned to signed is well defined but not
+  /// the other way around.
+  using Unsigned = typename std::make_unsigned<T>::type;
+  static_assert(sizeof(Unsigned) == sizeof(T), "Types must have same size");
+
+  static constexpr unsigned TypeBits = sizeof(Unsigned) * CHAR_BIT;
+  static_assert(TypeBits >= Bits, "n-bit must fit in T");
+
+  /// e.g. with TypeBits == 8 and Bits == 6.
+  static constexpr Unsigned AllZeros = Unsigned(0);                  // 00000000
+  static constexpr Unsigned AllOnes = ~Unsigned(0);                  // 11111111
+  static constexpr Unsigned Umin = AllZeros;                         // 00000000
+  static constexpr Unsigned Umax = AllOnes >> (TypeBits - Bits);     // 00111111
+  static constexpr Unsigned SignBitMask = Unsigned(1) << (Bits - 1); // 00100000
+  static constexpr Unsigned Smax = Umax >> 1U;                       // 00011111
+  static constexpr Unsigned Smin = ~Smax;                            // 11100000
+  static constexpr Unsigned SignExtend = Unsigned(Smin << 1U);       // 11000000
+};
+
+/// `Compressor` is used to manipulate the bits of a (possibly signed) integer
+/// type so it can be packed and unpacked into a `bits` sized integer,
+/// `Compressor` is specialized on signed-ness so no runtime cost is incurred.
+/// The `pack` method also checks that the passed in `UserValue` is valid.
+template <typename T, unsigned Bits, bool = std::is_unsigned<T>::value>
+struct Compressor {
+  static_assert(std::is_unsigned<T>::value, "T is unsigned");
+  using BP = BitPatterns<T, Bits>;
+
+  static T pack(T UserValue, T UserMaxValue) {
+    assert(UserValue <= UserMaxValue && "value is too big");
+    assert(UserValue <= BP::Umax && "value is too big");
+    return UserValue;
+  }
+
+  static T unpack(T StorageValue) { return StorageValue; }
+};
+
+template <typename T, unsigned Bits> struct Compressor<T, Bits, false> {
+  static_assert(std::is_signed<T>::value, "T is signed");
+  using BP = BitPatterns<T, Bits>;
+
+  static T pack(T UserValue, T UserMaxValue) {
+    assert(UserValue <= UserMaxValue && "value is too big");
+    assert(UserValue <= T(BP::Smax) && "value is too big");
+    assert(UserValue >= T(BP::Smin) && "value is too small");
+    if (UserValue < 0)
+      UserValue &= ~BP::SignExtend;
+    return UserValue;
+  }
+
+  static T unpack(T StorageValue) {
+    if (StorageValue >= T(BP::SignBitMask))
+      StorageValue |= BP::SignExtend;
+    return StorageValue;
+  }
+};
+
+/// Impl is where Bifield description and Storage are put together to interact
+/// with values.
+template <typename Bitfield, typename StorageType> struct Impl {
+  static_assert(std::is_unsigned<StorageType>::value,
+                "Storage must be unsigned");
+  using IntegerType = typename Bitfield::IntegerType;
+  using C = Compressor<IntegerType, Bitfield::Bits>;
+  using BP = BitPatterns<StorageType, Bitfield::Bits>;
+
+  static constexpr size_t StorageBits = sizeof(StorageType) * CHAR_BIT;
+  static_assert(Bitfield::FirstBit <= StorageBits, "Data must fit in mask");
+  static_assert(Bitfield::LastBit <= StorageBits, "Data must fit in mask");
+  static constexpr StorageType Mask = BP::Umax << Bitfield::Shift;
+
+  /// Checks `UserValue` is within bounds and packs it between `FirstBit` and
+  /// `LastBit` of `Packed` leaving the rest unchanged.
+  static void update(StorageType &Packed, IntegerType UserValue) {
+    const StorageType StorageValue = C::pack(UserValue, Bitfield::UserMaxValue);
+    Packed &= ~Mask;
+    Packed |= StorageValue << Bitfield::Shift;
+  }
+
+  /// Interprets bits between `FirstBit` and `LastBit` of `Packed` as
+  /// an`IntegerType`.
+  static IntegerType extract(StorageType Packed) {
+    const StorageType StorageValue = (Packed & Mask) >> Bitfield::Shift;
+    return C::unpack(StorageValue);
+  }
+
+  /// Interprets bits between `FirstBit` and `LastBit` of `Packed` as
+  /// an`IntegerType`.
+  static StorageType test(StorageType Packed) { return Packed & Mask; }
+};
+
+/// `Bitfield` deals with the following type:
+/// - unsigned enums
+/// - signed and unsigned integer
+/// - `bool`
+/// Internally though we only manipulate integer with well defined and
+/// consistent semantics, this excludes typed enums and `bool` that are replaced
+/// with their unsigned counterparts. The correct type is restored in the public
+/// API.
+template <typename T, bool = std::is_enum<T>::value>
+struct ResolveUnderlyingType {
+  using type = typename std::underlying_type<T>::type;
+};
+template <typename T> struct ResolveUnderlyingType<T, false> {
+  using type = T;
+};
+template <> struct ResolveUnderlyingType<bool, false> {
+  /// In case sizeof(bool) != 1, replace `void` by an additionnal
+  /// std::conditional.
+  using type = std::conditional<sizeof(bool) == 1, uint8_t, void>::type;
+};
+
+} // namespace bitfields_details
+
+/// Holds functions to get, set or test bitfields.
+struct Bitfield {
+  /// Describes an element of a Bitfield. This type is then used with the
+  /// Bitfield static member functions.
+  /// \tparam T         The type of the field once in unpacked form.
+  /// \tparam Offset    The position of the first bit.
+  /// \tparam Size      The size of the field.
+  /// \tparam MaxValue  For enums the maximum enum allowed.
+  template <typename T, unsigned Offset, unsigned Size,
+            T MaxValue = std::is_enum<T>::value
+                             ? T(0) // coupled with static_assert below
+                             : std::numeric_limits<T>::max()>
+  struct Element {
+    using Type = T;
+    using IntegerType =
+        typename bitfields_details::ResolveUnderlyingType<T>::type;
+    static constexpr unsigned Shift = Offset;
+    static constexpr unsigned Bits = Size;
+    static constexpr unsigned FirstBit = Offset;
+    static constexpr unsigned LastBit = Shift + Bits - 1;
+    static constexpr unsigned NextBit = Shift + Bits;
+
+  private:
+    template <typename, typename> friend struct bitfields_details::Impl;
+
+    static_assert(Bits > 0, "Bits must be non zero");
+    static constexpr size_t TypeBits = sizeof(IntegerType) * CHAR_BIT;
+    static_assert(Bits <= TypeBits, "Bits may not be greater than T size");
+    static_assert(!std::is_enum<T>::value || MaxValue != T(0),
+                  "Enum Bitfields must provide a MaxValue");
+    static_assert(!std::is_enum<T>::value ||
+                      std::is_unsigned<IntegerType>::value,
+                  "Enum must be unsigned");
+    static_assert(std::is_integral<IntegerType>::value &&
+                      std::numeric_limits<IntegerType>::is_integer,
+                  "IntegerType must be an integer type");
+
+    static constexpr IntegerType UserMaxValue =
+        static_cast<IntegerType>(MaxValue);
+  };
+
+  /// Unpacks the field from the `Packed` value.
+  template <typename Bitfield, typename StorageType>
+  static typename Bitfield::Type get(StorageType Packed) {
+    using I = bitfields_details::Impl<Bitfield, StorageType>;
+    return static_cast<typename Bitfield::Type>(I::extract(Packed));
+  }
+
+  /// Return a non-zero value if the field is non-zero.
+  /// It is more efficient than `getField`.
+  template <typename Bitfield, typename StorageType>
+  static StorageType test(StorageType Packed) {
+    using I = bitfields_details::Impl<Bitfield, StorageType>;
+    return I::test(Packed);
+  }
+
+  /// Sets the typed value in the provided `Packed` value.
+  /// The method will asserts if the provided value is too big to fit in.
+  template <typename Bitfield, typename StorageType>
+  static void set(StorageType &Packed, typename Bitfield::Type Value) {
+    using I = bitfields_details::Impl<Bitfield, StorageType>;
+    I::update(Packed, static_cast<typename Bitfield::IntegerType>(Value));
+  }
+
+  /// Returns whether the two bitfields share common bits.
+  template <typename A, typename B> static constexpr bool isOverlapping() {
+    return A::LastBit >= B::FirstBit && B::LastBit >= A::FirstBit;
+  }
+
+  template <typename A> static constexpr bool areContiguous() { return true; }
+  template <typename A, typename B, typename... Others>
+  static constexpr bool areContiguous() {
+    return A::NextBit == B::FirstBit && areContiguous<B, Others...>();
+  }
+};
+
+} // namespace llvm
+
+#endif // LLVM_ADT_BITFIELDS_H
diff --git a/llvm/include/llvm/ADT/EnumeratedArray.h b/llvm/include/llvm/ADT/EnumeratedArray.h
index a9528115618cf..a66ec9d08c37e 100644
--- a/llvm/include/llvm/ADT/EnumeratedArray.h
+++ b/llvm/include/llvm/ADT/EnumeratedArray.h
@@ -38,6 +38,7 @@ class EnumeratedArray {
         static_cast<const EnumeratedArray<ValueType, Enumeration, LargestEnum,
                                           IndexType, Size> &>(*this)[Index]);
   }
+  inline IndexType size() { return Size; }
 
 private:
   ValueType Underlying[Size];
diff --git a/llvm/include/llvm/ADT/FunctionExtras.h b/llvm/include/llvm/ADT/FunctionExtras.h
index ad84bbc35b787..4c75e4d2547be 100644
--- a/llvm/include/llvm/ADT/FunctionExtras.h
+++ b/llvm/include/llvm/ADT/FunctionExtras.h
@@ -11,11 +11,11 @@
 /// in `<function>`.
 ///
 /// It provides `unique_function`, which works like `std::function` but supports
-/// move-only callable objects.
+/// move-only callable objects and const-qualification.
 ///
 /// Future plans:
-/// - Add a `function` that provides const, volatile, and ref-qualified support,
-///   which doesn't work with `std::function`.
+/// - Add a `function` that provides ref-qualified support, which doesn't work
+///   with `std::function`.
 /// - Provide support for specifying multiple signatures to type erase callable
 ///   objects with an overload set, such as those produced by generic lambdas.
 /// - Expand to include a copyable utility that directly replaces std::function
@@ -37,13 +37,31 @@
 #include "llvm/Support/MemAlloc.h"
 #include "llvm/Support/type_traits.h"
 #include <memory>
+#include <type_traits>
 
 namespace llvm {
 
+/// unique_function is a type-erasing functor similar to std::function.
+///
+/// It can hold move-only function objects, like lambdas capturing unique_ptrs.
+/// Accordingly, it is movable but not copyable.
+///
+/// It supports const-qualification:
+/// - unique_function<int() const> has a const operator().
+///   It can only hold functions which themselves have a const operator().
+/// - unique_function<int()> has a non-const operator().
+///   It can hold functions with a non-const operator(), like mutable lambdas.
 template <typename FunctionT> class unique_function;
 
-template <typename ReturnT, typename... ParamTs>
-class unique_function<ReturnT(ParamTs...)> {
+namespace detail {
+
+template <typename T>
+using EnableIfTrivial =
+    std::enable_if_t<llvm::is_trivially_move_constructible<T>::value &&
+                     std::is_trivially_destructible<T>::value>;
+
+template <typename ReturnT, typename... ParamTs> class UniqueFunctionBase {
+protected:
   static constexpr size_t InlineStorageSize = sizeof(void *) * 3;
 
   // MSVC has a bug and ICEs if we give it a particular dependent value
@@ -113,8 +131,11 @@ class unique_function<ReturnT(ParamTs...)> {
 
     // For in-line storage, we just provide an aligned character buffer. We
     // provide three pointers worth of storage here.
-    typename std::aligned_storage<InlineStorageSize, alignof(void *)>::type
-        InlineStorage;
+    // This is mutable as an inlined `const unique_function<void() const>` may
+    // still modify its own mutable members.
+    mutable
+        typename std::aligned_storage<InlineStorageSize, alignof(void *)>::type
+            InlineStorage;
   } StorageUnion;
 
   // A compressed pointer to either our dispatching callback or our table of
@@ -137,11 +158,25 @@ class unique_function<ReturnT(ParamTs...)> {
         .template get<NonTrivialCallbacks *>();
   }
 
-  void *getInlineStorage() { return &StorageUnion.InlineStorage; }
+  CallPtrT getCallPtr() const {
+    return isTrivialCallback() ? getTrivialCallback()
+                               : getNonTrivialCallbacks()->CallPtr;
+  }
 
-  void *getOutOfLineStorage() {
+  // These three functions are only const in the narrow sense. They return
+  // mutable pointers to function state.
+  // This allows unique_function<T const>::operator() to be const, even if the
+  // underlying functor may be internally mutable.
+  //
+  // const callers must ensure they're only used in const-correct ways.
+  void *getCalleePtr() const {
+    return isInlineStorage() ? getInlineStorage() : getOutOfLineStorage();
+  }
+  void *getInlineStorage() const { return &StorageUnion.InlineStorage; }
+  void *getOutOfLineStorage() const {
     return StorageUnion.OutOfLineStorage.StoragePtr;
   }
+
   size_t getOutOfLineStorageSize() const {
     return StorageUnion.OutOfLineStorage.Size;
   }
@@ -153,10 +188,11 @@ class unique_function<ReturnT(ParamTs...)> {
     StorageUnion.OutOfLineStorage = {Ptr, Size, Alignment};
   }
 
-  template <typename CallableT>
-  static ReturnT CallImpl(void *CallableAddr, AdjustedParamT<ParamTs>... Params) {
-    return (*reinterpret_cast<CallableT *>(CallableAddr))(
-        std::forward<ParamTs>(Params)...);
+  template <typename CalledAsT>
+  static ReturnT CallImpl(void *CallableAddr,
+                          AdjustedParamT<ParamTs>... Params) {
+    auto &Func = *reinterpret_cast<CalledAsT *>(CallableAddr);
+    return Func(std::forward<ParamTs>(Params)...);
   }
 
   template <typename CallableT>
@@ -170,11 +206,54 @@ class unique_function<ReturnT(ParamTs...)> {
     reinterpret_cast<CallableT *>(CallableAddr)->~CallableT();
   }
 
-public:
-  unique_function() = default;
-  unique_function(std::nullptr_t /*null_callable*/) {}
+  // The pointers to call/move/destroy functions are determined for each
+  // callable type (and called-as type, which determines the overload chosen).
+  // (definitions are out-of-line).
+
+  // By default, we need an object that contains all the different
+  // type erased behaviors needed. Create a static instance of the struct type
+  // here and each instance will contain a pointer to it.
+  // Wrap in a struct to avoid https://gcc.gnu.org/PR71954
+  template <typename CallableT, typename CalledAs, typename Enable = void>
+  struct CallbacksHolder {
+    static NonTrivialCallbacks Callbacks;
+  };
+  // See if we can create a trivial callback. We need the callable to be
+  // trivially moved and trivially destroyed so that we don't have to store
+  // type erased callbacks for those operations.
+  template <typename CallableT, typename CalledAs>
+  struct CallbacksHolder<CallableT, CalledAs, EnableIfTrivial<CallableT>> {
+    static TrivialCallback Callbacks;
+  };
+
+  // A simple tag type so the call-as type to be passed to the constructor.
+  template <typename T> struct CalledAs {};
+
+  // Essentially the "main" unique_function constructor, but subclasses
+  // provide the qualified type to be used for the call.
+  // (We always store a T, even if the call will use a pointer to const T).
+  template <typename CallableT, typename CalledAsT>
+  UniqueFunctionBase(CallableT Callable, CalledAs<CalledAsT>) {
+    bool IsInlineStorage = true;
+    void *CallableAddr = getInlineStorage();
+    if (sizeof(CallableT) > InlineStorageSize ||
+        alignof(CallableT) > alignof(decltype(StorageUnion.InlineStorage))) {
+      IsInlineStorage = false;
+      // Allocate out-of-line storage. FIXME: Use an explicit alignment
+      // parameter in C++17 mode.
+      auto Size = sizeof(CallableT);
+      auto Alignment = alignof(CallableT);
+      CallableAddr = allocate_buffer(Size, Alignment);
+      setOutOfLineStorage(CallableAddr, Size, Alignment);
+    }
+
+    // Now move into the storage.
+    new (CallableAddr) CallableT(std::move(Callable));
+    CallbackAndInlineFlag.setPointerAndInt(
+        &CallbacksHolder<CallableT, CalledAsT>::Callbacks, IsInlineStorage);
+  }
 
-  ~unique_function() {
+  ~UniqueFunctionBase() {
     if (!CallbackAndInlineFlag.getPointer())
       return;
 
@@ -190,7 +269,7 @@ class unique_function<ReturnT(ParamTs...)> {
                         getOutOfLineStorageAlignment());
   }
 
-  unique_function(unique_function &&RHS) noexcept {
+  UniqueFunctionBase(UniqueFunctionBase &&RHS) noexcept {
     // Copy the callback and inline flag.
     CallbackAndInlineFlag = RHS.CallbackAndInlineFlag;
 
@@ -219,72 +298,83 @@ class unique_function<ReturnT(ParamTs...)> {
 #endif
   }
 
-  unique_function &operator=(unique_function &&RHS) noexcept {
+  UniqueFunctionBase &operator=(UniqueFunctionBase &&RHS) noexcept {
     if (this == &RHS)
       return *this;
 
     // Because we don't try to provide any exception safety guarantees we can
     // implement move assignment very simply by first destroying the current
     // object and then move-constructing over top of it.
-    this->~unique_function();
-    new (this) unique_function(std::move(RHS));
+    this->~UniqueFunctionBase();
+    new (this) UniqueFunctionBase(std::move(RHS));
     return *this;
   }
 
-  template <typename CallableT> unique_function(CallableT Callable) {
-    bool IsInlineStorage = true;
-    void *CallableAddr = getInlineStorage();
-    if (sizeof(CallableT) > InlineStorageSize ||
-        alignof(CallableT) > alignof(decltype(StorageUnion.InlineStorage))) {
-      IsInlineStorage = false;
-      // Allocate out-of-line storage. FIXME: Use an explicit alignment
-      // parameter in C++17 mode.
-      auto Size = sizeof(CallableT);
-      auto Alignment = alignof(CallableT);
-      CallableAddr = allocate_buffer(Size, Alignment);
-      setOutOfLineStorage(CallableAddr, Size, Alignment);
-    }
+  UniqueFunctionBase() = default;
 
-    // Now move into the storage.
-    new (CallableAddr) CallableT(std::move(Callable));
+public:
+  explicit operator bool() const {
+    return (bool)CallbackAndInlineFlag.getPointer();
+  }
+};
 
-    // See if we can create a trivial callback. We need the callable to be
-    // trivially moved and trivially destroyed so that we don't have to store
-    // type erased callbacks for those operations.
-    //
-    // FIXME: We should use constexpr if here and below to avoid instantiating
-    // the non-trivial static objects when unnecessary. While the linker should
-    // remove them, it is still wasteful.
-    if (llvm::is_trivially_move_constructible<CallableT>::value &&
-        std::is_trivially_destructible<CallableT>::value) {
-      // We need to create a nicely aligned object. We use a static variable
-      // for this because it is a trivial struct.
-      static TrivialCallback Callback = { &CallImpl<CallableT> };
-
-      CallbackAndInlineFlag = {&Callback, IsInlineStorage};
-      return;
-    }
+template <typename R, typename... P>
+template <typename CallableT, typename CalledAsT, typename Enable>
+typename UniqueFunctionBase<R, P...>::NonTrivialCallbacks UniqueFunctionBase<
+    R, P...>::CallbacksHolder<CallableT, CalledAsT, Enable>::Callbacks = {
+    &CallImpl<CalledAsT>, &MoveImpl<CallableT>, &DestroyImpl<CallableT>};
 
-    // Otherwise, we need to point at an object that contains all the different
-    // type erased behaviors needed. Create a static instance of the struct type
-    // here and then use a pointer to that.
-    static NonTrivialCallbacks Callbacks = {
-        &CallImpl<CallableT>, &MoveImpl<CallableT>, &DestroyImpl<CallableT>};
+template <typename R, typename... P>
+template <typename CallableT, typename CalledAsT>
+typename UniqueFunctionBase<R, P...>::TrivialCallback
+    UniqueFunctionBase<R, P...>::CallbacksHolder<
+        CallableT, CalledAsT, EnableIfTrivial<CallableT>>::Callbacks{
+        &CallImpl<CalledAsT>};
 
-    CallbackAndInlineFlag = {&Callbacks, IsInlineStorage};
-  }
+} // namespace detail
+
+template <typename R, typename... P>
+class unique_function<R(P...)> : public detail::UniqueFunctionBase<R, P...> {
+  using Base = detail::UniqueFunctionBase<R, P...>;
+
+public:
+  unique_function() = default;
+  unique_function(std::nullptr_t) {}
+  unique_function(unique_function &&) = default;
+  unique_function(const unique_function &) = delete;
+  unique_function &operator=(unique_function &&) = default;
+  unique_function &operator=(const unique_function &) = delete;
 
-  ReturnT operator()(ParamTs... Params) {
-    void *CallableAddr =
-        isInlineStorage() ? getInlineStorage() : getOutOfLineStorage();
+  template <typename CallableT>
+  unique_function(CallableT Callable)
+      : Base(std::forward<CallableT>(Callable),
+             typename Base::template CalledAs<CallableT>{}) {}
 
-    return (isTrivialCallback()
-                ? getTrivialCallback()
-                : getNonTrivialCallbacks()->CallPtr)(CallableAddr, Params...);
+  R operator()(P... Params) {
+    return this->getCallPtr()(this->getCalleePtr(), Params...);
   }
+};
 
-  explicit operator bool() const {
-    return (bool)CallbackAndInlineFlag.getPointer();
+template <typename R, typename... P>
+class unique_function<R(P...) const>
+    : public detail::UniqueFunctionBase<R, P...> {
+  using Base = detail::UniqueFunctionBase<R, P...>;
+
+public:
+  unique_function() = default;
+  unique_function(std::nullptr_t) {}
+  unique_function(unique_function &&) = default;
+  unique_function(const unique_function &) = delete;
+  unique_function &operator=(unique_function &&) = default;
+  unique_function &operator=(const unique_function &) = delete;
+
+  template <typename CallableT>
+  unique_function(CallableT Callable)
+      : Base(std::forward<CallableT>(Callable),
+             typename Base::template CalledAs<const CallableT>{}) {}
+
+  R operator()(P... Params) const {
+    return this->getCallPtr()(this->getCalleePtr(), Params...);
   }
 };
 
diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index ed8d99ea83983..50b688b366489 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -267,10 +267,10 @@ constexpr bool empty(const T &RangeOrContainer) {
   return adl_begin(RangeOrContainer) == adl_end(RangeOrContainer);
 }
 
-/// Returns true of the given range only contains a single element.
-template <typename ContainerTy> bool hasSingleElement(ContainerTy &&c) {
-  auto it = std::begin(c), e = std::end(c);
-  return it != e && std::next(it) == e;
+/// Returns true if the given container only contains a single element.
+template <typename ContainerTy> bool hasSingleElement(ContainerTy &&C) {
+  auto B = std::begin(C), E = std::end(C);
+  return B != E && std::next(B) == E;
 }
 
 /// Return a range covering \p RangeOrContainer with the first N elements
@@ -745,14 +745,14 @@ detail::zippy<detail::zip_first, T, U, Args...> zip_first(T &&t, U &&u,
 
 namespace detail {
 template <typename Iter>
-static Iter next_or_end(const Iter &I, const Iter &End) {
+Iter next_or_end(const Iter &I, const Iter &End) {
   if (I == End)
     return End;
   return std::next(I);
 }
 
 template <typename Iter>
-static auto deref_or_none(const Iter &I, const Iter &End) -> llvm::Optional<
+auto deref_or_none(const Iter &I, const Iter &End) -> llvm::Optional<
     std::remove_const_t<std::remove_reference_t<decltype(*I)>>> {
   if (I == End)
     return None;
@@ -1121,6 +1121,14 @@ class indexed_accessor_range_base {
     assert(index < size() && "invalid index for value range");
     return DerivedT::dereference_iterator(base, index);
   }
+  ReferenceT front() const {
+    assert(!empty() && "expected non-empty range");
+    return (*this)[0];
+  }
+  ReferenceT back() const {
+    assert(!empty() && "expected non-empty range");
+    return (*this)[size() - 1];
+  }
 
   /// Compare this range with another.
   template <typename OtherT> bool operator==(const OtherT &other) const {
@@ -1919,12 +1927,15 @@ bool hasNItems(
       return false; // Too few.
     N -= ShouldBeCounted(*Begin);
   }
-  return Begin == End;
+  for (; Begin != End; ++Begin)
+    if (ShouldBeCounted(*Begin))
+      return false; // Too many.
+  return true;
 }
 
 /// Return true if the sequence [Begin, End) has N or more items. Runs in O(N)
 /// time. Not meant for use with random-access iterators.
-/// Can optionally take a predicate to filter lazily some items.
+/// Can optionally take a predicate to lazily filter some items.
 template<typename IterTy,
          typename Pred = bool (*)(const decltype(*std::declval<IterTy>()) &)>
 bool hasNItemsOrMore(
@@ -1944,6 +1955,36 @@ bool hasNItemsOrMore(
   return true;
 }
 
+/// Returns true if the sequence [Begin, End) has N or less items. Can
+/// optionally take a predicate to lazily filter some items.
+template <typename IterTy,
+          typename Pred = bool (*)(const decltype(*std::declval<IterTy>()) &)>
+bool hasNItemsOrLess(
+    IterTy &&Begin, IterTy &&End, unsigned N,
+    Pred &&ShouldBeCounted = [](const decltype(*std::declval<IterTy>()) &) {
+      return true;
+    }) {
+  assert(N != std::numeric_limits<unsigned>::max());
+  return !hasNItemsOrMore(Begin, End, N + 1, ShouldBeCounted);
+}
+
+/// Returns true if the given container has exactly N items
+template <typename ContainerTy> bool hasNItems(ContainerTy &&C, unsigned N) {
+  return hasNItems(std::begin(C), std::end(C), N);
+}
+
+/// Returns true if the given container has N or more items
+template <typename ContainerTy>
+bool hasNItemsOrMore(ContainerTy &&C, unsigned N) {
+  return hasNItemsOrMore(std::begin(C), std::end(C), N);
+}
+
+/// Returns true if the given container has N or less items
+template <typename ContainerTy>
+bool hasNItemsOrLess(ContainerTy &&C, unsigned N) {
+  return hasNItemsOrLess(std::begin(C), std::end(C), N);
+}
+
 /// Returns a raw pointer that represents the same address as the argument.
 ///
 /// This implementation can be removed once we move to C++20 where it's defined
diff --git a/llvm/include/llvm/ADT/Triple.h b/llvm/include/llvm/ADT/Triple.h
index 8e46265c7f71a..6bad18f19244e 100644
--- a/llvm/include/llvm/ADT/Triple.h
+++ b/llvm/include/llvm/ADT/Triple.h
@@ -19,6 +19,8 @@
 
 namespace llvm {
 
+class VersionTuple;
+
 /// Triple - Helper class for working with autoconf configuration names. For
 /// historical reasons, we also call these 'triples' (they used to contain
 /// exactly three fields).
@@ -438,17 +440,7 @@ class Triple {
   /// compatibility, which handles supporting skewed version numbering schemes
   /// used by the "darwin" triples.
   bool isMacOSXVersionLT(unsigned Major, unsigned Minor = 0,
-                         unsigned Micro = 0) const {
-    assert(isMacOSX() && "Not an OS X triple!");
-
-    // If this is OS X, expect a sane version number.
-    if (getOS() == Triple::MacOSX)
-      return isOSVersionLT(Major, Minor, Micro);
-
-    // Otherwise, compare to the "Darwin" number.
-    assert(Major == 10 && "Unexpected major version");
-    return isOSVersionLT(Minor + 4, Micro, 0);
-  }
+                         unsigned Micro = 0) const;
 
   /// isMacOSX - Is this a Mac OS X triple. For legacy reasons, we support both
   /// "darwin" and "osx" as OS X triples.
@@ -868,6 +860,12 @@ class Triple {
   /// Merge target triples.
   std::string merge(const Triple &Other) const;
 
+  /// Some platforms have different minimum supported OS versions that
+  /// varies by the architecture specified in the triple. This function
+  /// returns the minimum supported OS version for this triple if one an exists,
+  /// or an invalid version tuple if this triple doesn't have one.
+  VersionTuple getMinimumSupportedOSVersion() const;
+
   /// @}
   /// @name Static helpers for IDs.
   /// @{
@@ -902,6 +900,10 @@ class Triple {
   static ArchType getArchTypeForLLVMName(StringRef Str);
 
   /// @}
+
+  /// Returns a canonicalized OS version number for the specified OS.
+  static VersionTuple getCanonicalVersionForOS(OSType OSKind,
+                                               const VersionTuple &Version);
 };
 
 } // End llvm namespace
diff --git a/llvm/include/llvm/Analysis/AliasSetTracker.h b/llvm/include/llvm/Analysis/AliasSetTracker.h
index e94a758b06bab..690a94d9cf2ce 100644
--- a/llvm/include/llvm/Analysis/AliasSetTracker.h
+++ b/llvm/include/llvm/Analysis/AliasSetTracker.h
@@ -87,12 +87,7 @@ class AliasSet : public ilist_node<AliasSet> {
         AAInfo = NewAAInfo;
       else {
         AAMDNodes Intersection(AAInfo.intersect(NewAAInfo));
-        if (!Intersection.TBAA || !Intersection.Scope ||
-            !Intersection.NoAlias) {
-          // NewAAInfo conflicts with AAInfo.
-          AAInfo = DenseMapInfo<AAMDNodes>::getTombstoneKey();
-          SizeChanged = true;
-        }
+        SizeChanged |= Intersection != AAInfo;
         AAInfo = Intersection;
       }
       return SizeChanged;
diff --git a/llvm/include/llvm/Analysis/BasicAliasAnalysis.h b/llvm/include/llvm/Analysis/BasicAliasAnalysis.h
index 403510dbb3a3a..9214bfcd7a242 100644
--- a/llvm/include/llvm/Analysis/BasicAliasAnalysis.h
+++ b/llvm/include/llvm/Analysis/BasicAliasAnalysis.h
@@ -189,7 +189,7 @@ class BasicAAResult : public AAResultBase<BasicAAResult> {
   bool
   constantOffsetHeuristic(const SmallVectorImpl<VariableGEPIndex> &VarIndices,
                           LocationSize V1Size, LocationSize V2Size,
-                          APInt BaseOffset, AssumptionCache *AC,
+                          const APInt &BaseOffset, AssumptionCache *AC,
                           DominatorTree *DT);
 
   bool isValueEqualInPotentialCycles(const Value *V1, const Value *V2);
diff --git a/llvm/include/llvm/Analysis/CallGraph.h b/llvm/include/llvm/Analysis/CallGraph.h
index 4cb4cfa60ec77..98f9b0683fd45 100644
--- a/llvm/include/llvm/Analysis/CallGraph.h
+++ b/llvm/include/llvm/Analysis/CallGraph.h
@@ -175,13 +175,21 @@ class CallGraphNode {
 public:
   /// A pair of the calling instruction (a call or invoke)
   /// and the call graph node being called.
-  using CallRecord = std::pair<WeakTrackingVH, CallGraphNode *>;
+  /// Call graph node may have two types of call records which represent an edge
+  /// in the call graph - reference or a call edge. Reference edges are not
+  /// associated with any call instruction and are created with the first field
+  /// set to `None`, while real call edges have instruction address in this
+  /// field. Therefore, all real call edges are expected to have a value in the
+  /// first field and it is not supposed to be `nullptr`.
+  /// Reference edges, for example, are used for connecting broker function
+  /// caller to the callback function for callback call sites.
+  using CallRecord = std::pair<Optional<WeakTrackingVH>, CallGraphNode *>;
 
 public:
   using CalledFunctionsVector = std::vector<CallRecord>;
 
   /// Creates a node for the specified function.
-  inline CallGraphNode(Function *F) : F(F) {}
+  inline CallGraphNode(CallGraph *CG, Function *F) : CG(CG), F(F) {}
 
   CallGraphNode(const CallGraphNode &) = delete;
   CallGraphNode &operator=(const CallGraphNode &) = delete;
@@ -243,7 +251,8 @@ class CallGraphNode {
     assert(!Call || !Call->getCalledFunction() ||
            !Call->getCalledFunction()->isIntrinsic() ||
            !Intrinsic::isLeaf(Call->getCalledFunction()->getIntrinsicID()));
-    CalledFunctions.emplace_back(Call, M);
+    CalledFunctions.emplace_back(
+        Call ? Optional<WeakTrackingVH>(Call) : Optional<WeakTrackingVH>(), M);
     M->AddRef();
   }
 
@@ -279,6 +288,7 @@ class CallGraphNode {
 private:
   friend class CallGraph;
 
+  CallGraph *CG;
   Function *F;
 
   std::vector<CallRecord> CalledFunctions;
@@ -412,7 +422,7 @@ class CallGraphWrapperPass : public ModulePass {
 // graphs by the generic graph algorithms.
 //
 
-// Provide graph traits for tranversing call graphs using standard graph
+// Provide graph traits for traversing call graphs using standard graph
 // traversals.
 template <> struct GraphTraits<CallGraphNode *> {
   using NodeRef = CallGraphNode *;
diff --git a/llvm/include/llvm/Analysis/DominanceFrontier.h b/llvm/include/llvm/Analysis/DominanceFrontier.h
index c0bf30e162ddd..f67929c997f93 100644
--- a/llvm/include/llvm/Analysis/DominanceFrontier.h
+++ b/llvm/include/llvm/Analysis/DominanceFrontier.h
@@ -130,7 +130,7 @@ class ForwardDominanceFrontierBase
   using DomSetType = typename DominanceFrontierBase<BlockT, false>::DomSetType;
 
   void analyze(DomTreeT &DT) {
-    assert(DT.getRoots().size() == 1 &&
+    assert(DT.root_size() == 1 &&
            "Only one entry block for forward domfronts!");
     this->Roots = {DT.getRoot()};
     calculate(DT, DT[this->Roots[0]]);
diff --git a/llvm/include/llvm/Analysis/GlobalsModRef.h b/llvm/include/llvm/Analysis/GlobalsModRef.h
index 54cc0d4af6294..7daaa7f484ded 100644
--- a/llvm/include/llvm/Analysis/GlobalsModRef.h
+++ b/llvm/include/llvm/Analysis/GlobalsModRef.h
@@ -14,7 +14,6 @@
 #define LLVM_ANALYSIS_GLOBALSMODREF_H
 
 #include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Analysis/CallGraph.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Module.h"
@@ -23,6 +22,7 @@
 #include <list>
 
 namespace llvm {
+class CallGraph;
 
 /// An alias analysis result set for globals.
 ///
diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h
index 66a848e865e19..3480d93385a8e 100644
--- a/llvm/include/llvm/Analysis/InlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/InlineAdvisor.h
@@ -203,6 +203,11 @@ class InlineAdvisorAnalysis : public AnalysisInfoMixin<InlineAdvisorAnalysis> {
   Result run(Module &M, ModuleAnalysisManager &MAM) { return Result(M, MAM); }
 };
 
+#ifdef LLVM_HAVE_TF_AOT
+std::unique_ptr<InlineAdvisor>
+getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM);
+#endif
+
 // Default (manual policy) decision making helper APIs. Shared with the legacy
 // pass manager inliner.
 
diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h
index a1f49d45789c1..7f04a8ce8f5fa 100644
--- a/llvm/include/llvm/Analysis/InlineCost.h
+++ b/llvm/include/llvm/Analysis/InlineCost.h
@@ -49,6 +49,9 @@ const int ColdccPenalty = 2000;
 /// Do not inline functions which allocate this many bytes on the stack
 /// when the caller is recursive.
 const unsigned TotalAllocaSizeRecursiveCaller = 1024;
+/// Do not inline dynamic allocas that have been constant propagated to be
+/// static allocas above this amount in bytes.
+const uint64_t MaxSimplifiedDynamicAllocaToInline = 65536;
 } // namespace InlineConstants
 
 /// Represents the cost of inlining a function.
@@ -269,6 +272,18 @@ Optional<int> getInliningCostEstimate(
 
 /// Minimal filter to detect invalid constructs for inlining.
 InlineResult isInlineViable(Function &Callee);
+
+// This pass is used to annotate instructions during the inline process for
+// debugging and analysis. The main purpose of the pass is to see and test
+// inliner's decisions when creating new optimizations to InlineCost.
+struct InlineCostAnnotationPrinterPass
+    : PassInfoMixin<InlineCostAnnotationPrinterPass> {
+  raw_ostream &OS;
+
+public:
+  explicit InlineCostAnnotationPrinterPass(raw_ostream &OS) : OS(OS) {}
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+};
 } // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h b/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h
new file mode 100644
index 0000000000000..8da442cc4a53a
--- /dev/null
+++ b/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h
@@ -0,0 +1,70 @@
+//===- InlineModelFeatureMaps.h - common model runner defs ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+
+#ifndef LLVM_ANALYSIS_INLINEMODELFEATUREMAPS_H
+#define LLVM_ANALYSIS_INLINEMODELFEATUREMAPS_H
+
+#include <array>
+#include <string>
+#include <vector>
+
+namespace llvm {
+
+// List of features. Each feature is defined through a triple:
+// - the name of an enum member, which will be the feature index
+// - a textual name, used for Tensorflow model binding (so it needs to match the
+// names used by the Tensorflow model)
+// - a documentation description. Currently, that is not used anywhere
+// programmatically, and serves as workaround to inability of inserting comments
+// in macros.
+#define INLINE_FEATURE_ITERATOR(M)                                             \
+  M(CalleeBasicBlockCount, "callee_basic_block_count",                         \
+    "number of basic blocks of the callee")                                    \
+  M(CallSiteHeight, "callsite_height",                                         \
+    "position of the call site in the original call graph - measured from "    \
+    "the farthest SCC")                                                        \
+  M(NodeCount, "node_count",                                                   \
+    "total current number of defined functions in the module")                 \
+  M(NrCtantParams, "nr_ctant_params",                                          \
+    "number of parameters in the call site that are constants")                \
+  M(CostEstimate, "cost_estimate", "total cost estimate (threshold - free)")   \
+  M(EdgeCount, "edge_count",                                                   \
+    "number of module-internal users of the caller, +1 if the caller is "      \
+    "exposed externally")                                                      \
+  M(CallerUsers, "caller_users",                                               \
+    "number of blocks reached from a conditional instruction, in the caller")  \
+  M(CallerConditionallyExecutedBlocks, "caller_conditionally_executed_blocks", \
+    "number of blocks reached from a conditional instruction, in the caller")  \
+  M(CallerBasicBlockCount, "caller_basic_block_count",                         \
+    "number of basic blocks in the caller")                                    \
+  M(CalleeConditionallyExecutedBlocks, "callee_conditionally_executed_blocks", \
+    "number of blocks reached from a conditional instruction, in the callee")  \
+  M(CalleeUsers, "callee_users",                                               \
+    "number of blocks reached from a conditional instruction, in the callee")
+
+enum class FeatureIndex : size_t {
+#define POPULATE_INDICES(INDEX_NAME, NAME, COMMENT) INDEX_NAME,
+  INLINE_FEATURE_ITERATOR(POPULATE_INDICES)
+#undef POPULATE_INDICES
+      NumberOfFeatures
+};
+
+constexpr size_t NumberOfFeatures =
+    static_cast<size_t>(FeatureIndex::NumberOfFeatures);
+
+extern const std::array<std::string, NumberOfFeatures> FeatureNameMap;
+
+extern const char *const DecisionName;
+extern const char *const DefaultDecisionName;
+extern const char *const RewardName;
+
+using InlineFeatures = std::vector<int64_t>;
+
+} // namespace llvm
+#endif // LLVM_ANALYSIS_INLINEMODELFEATUREMAPS_H
diff --git a/llvm/include/llvm/Analysis/Loads.h b/llvm/include/llvm/Analysis/Loads.h
index 2fee7d646b6ba..5665a802942d5 100644
--- a/llvm/include/llvm/Analysis/Loads.h
+++ b/llvm/include/llvm/Analysis/Loads.h
@@ -13,13 +13,16 @@
 #ifndef LLVM_ANALYSIS_LOADS_H
 #define LLVM_ANALYSIS_LOADS_H
 
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/Support/CommandLine.h"
 
 namespace llvm {
 
+class AAResults;
 class DataLayout;
+class DominatorTree;
+class Instruction;
+class LoadInst;
 class Loop;
 class MDNode;
 class ScalarEvolution;
@@ -120,7 +123,7 @@ Value *FindAvailableLoadedValue(LoadInst *Load,
                                 BasicBlock *ScanBB,
                                 BasicBlock::iterator &ScanFrom,
                                 unsigned MaxInstsToScan = DefMaxInstsToScan,
-                                AliasAnalysis *AA = nullptr,
+                                AAResults *AA = nullptr,
                                 bool *IsLoadCSE = nullptr,
                                 unsigned *NumScanedInst = nullptr);
 
@@ -143,15 +146,15 @@ Value *FindAvailableLoadedValue(LoadInst *Load,
 /// is zero, the whole block will be scanned.
 /// \param AA Optional pointer to alias analysis, to make the scan more
 /// precise.
-/// \param [out] IsLoad Whether the returned value is a load from the same
+/// \param [out] IsLoadCSE Whether the returned value is a load from the same
 /// location in memory, as opposed to the value operand of a store.
 ///
 /// \returns The found value, or nullptr if no value is found.
 Value *FindAvailablePtrLoadStore(Value *Ptr, Type *AccessTy, bool AtLeastAtomic,
                                  BasicBlock *ScanBB,
                                  BasicBlock::iterator &ScanFrom,
-                                 unsigned MaxInstsToScan, AliasAnalysis *AA,
-                                 bool *IsLoad, unsigned *NumScanedInst);
+                                 unsigned MaxInstsToScan, AAResults *AA,
+                                 bool *IsLoadCSE, unsigned *NumScanedInst);
 }
 
 #endif
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 0ca4013f7d5ca..a5237e9ba59e3 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -15,7 +15,6 @@
 #define LLVM_ANALYSIS_LOOPACCESSANALYSIS_H
 
 #include "llvm/ADT/EquivalenceClasses.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/LoopAnalysisManager.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/IR/DiagnosticInfo.h"
@@ -23,15 +22,15 @@
 
 namespace llvm {
 
-class Value;
+class AAResults;
 class DataLayout;
-class ScalarEvolution;
 class Loop;
-class SCEV;
-class SCEVUnionPredicate;
 class LoopAccessInfo;
 class OptimizationRemarkEmitter;
-class raw_osstream;
+class raw_ostream;
+class SCEV;
+class SCEVUnionPredicate;
+class Value;
 
 /// Collection of parameters shared beetween the Loop Vectorizer and the
 /// Loop Access Analysis.
@@ -511,7 +510,7 @@ class RuntimePointerChecking {
 class LoopAccessInfo {
 public:
   LoopAccessInfo(Loop *L, ScalarEvolution *SE, const TargetLibraryInfo *TLI,
-                 AliasAnalysis *AA, DominatorTree *DT, LoopInfo *LI);
+                 AAResults *AA, DominatorTree *DT, LoopInfo *LI);
 
   /// Return true we can analyze the memory accesses in the loop and there are
   /// no memory dependence cycles.
@@ -584,7 +583,7 @@ class LoopAccessInfo {
 
 private:
   /// Analyze the loop.
-  void analyzeLoop(AliasAnalysis *AA, LoopInfo *LI,
+  void analyzeLoop(AAResults *AA, LoopInfo *LI,
                    const TargetLibraryInfo *TLI, DominatorTree *DT);
 
   /// Check if the structure of the loop allows it to be analyzed by this
@@ -727,7 +726,7 @@ class LoopAccessLegacyAnalysis : public FunctionPass {
   // The used analysis passes.
   ScalarEvolution *SE = nullptr;
   const TargetLibraryInfo *TLI = nullptr;
-  AliasAnalysis *AA = nullptr;
+  AAResults *AA = nullptr;
   DominatorTree *DT = nullptr;
   LoopInfo *LI = nullptr;
 };
diff --git a/llvm/include/llvm/Analysis/MLInlineAdvisor.h b/llvm/include/llvm/Analysis/MLInlineAdvisor.h
new file mode 100644
index 0000000000000..cbe3b1f1f4e6e
--- /dev/null
+++ b/llvm/include/llvm/Analysis/MLInlineAdvisor.h
@@ -0,0 +1,107 @@
+//===- MLInlineAdvisor.h - ML - based InlineAdvisor factories ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_MLINLINEADVISOR_H
+#define LLVM_ANALYSIS_MLINLINEADVISOR_H
+
+#include "llvm/Analysis/CallGraph.h"
+#include "llvm/Analysis/InlineAdvisor.h"
+#include "llvm/Analysis/MLModelRunner.h"
+#include "llvm/IR/PassManager.h"
+
+#include <memory>
+#include <unordered_map>
+
+namespace llvm {
+class Module;
+class MLInlineAdvice;
+
+class MLInlineAdvisor : public InlineAdvisor {
+public:
+  MLInlineAdvisor(Module &M, ModuleAnalysisManager &MAM,
+                  std::unique_ptr<MLModelRunner> ModelRunner);
+
+  CallGraph *callGraph() const { return CG.get(); }
+  virtual ~MLInlineAdvisor() = default;
+
+  void onPassEntry() override;
+
+  std::unique_ptr<InlineAdvice> getAdvice(CallBase &CB) override;
+
+  int64_t getIRSize(const Function &F) const { return F.getInstructionCount(); }
+  void onSuccessfulInlining(const MLInlineAdvice &Advice,
+                            bool CalleeWasDeleted);
+
+  bool isForcedToStop() const { return ForceStop; }
+  int64_t getLocalCalls(Function &F);
+  const MLModelRunner &getModelRunner() const { return *ModelRunner.get(); }
+
+protected:
+  virtual std::unique_ptr<MLInlineAdvice>
+  getMandatoryAdvice(CallBase &CB, OptimizationRemarkEmitter &ORE);
+
+  virtual std::unique_ptr<MLInlineAdvice>
+  getAdviceFromModel(CallBase &CB, OptimizationRemarkEmitter &ORE);
+
+  Module &M;
+  std::unique_ptr<MLModelRunner> ModelRunner;
+
+private:
+  int64_t getModuleIRSize() const;
+
+  std::unique_ptr<CallGraph> CG;
+
+  int64_t NodeCount = 0;
+  int64_t EdgeCount = 0;
+  std::map<const Function *, unsigned> FunctionLevels;
+  const int32_t InitialIRSize = 0;
+  int32_t CurrentIRSize = 0;
+
+  bool ForceStop = false;
+};
+
+/// InlineAdvice that tracks changes post inlining. For that reason, it only
+/// overrides the "successful inlining" extension points.
+class MLInlineAdvice : public InlineAdvice {
+public:
+  MLInlineAdvice(MLInlineAdvisor *Advisor, CallBase &CB,
+                 OptimizationRemarkEmitter &ORE, bool Recommendation)
+      : InlineAdvice(Advisor, CB, ORE, Recommendation),
+        CallerIRSize(Advisor->isForcedToStop() ? 0
+                                               : Advisor->getIRSize(*Caller)),
+        CalleeIRSize(Advisor->isForcedToStop() ? 0
+                                               : Advisor->getIRSize(*Callee)),
+        CallerAndCalleeEdges(Advisor->isForcedToStop()
+                                 ? 0
+                                 : (Advisor->getLocalCalls(*Caller) +
+                                    Advisor->getLocalCalls(*Callee))) {}
+  virtual ~MLInlineAdvice() = default;
+
+  void recordInliningImpl() override;
+  void recordInliningWithCalleeDeletedImpl() override;
+  void recordUnsuccessfulInliningImpl(const InlineResult &Result) override;
+  void recordUnattemptedInliningImpl() override;
+
+  Function *getCaller() const { return Caller; }
+  Function *getCallee() const { return Callee; }
+
+  const int64_t CallerIRSize;
+  const int64_t CalleeIRSize;
+  const int64_t CallerAndCalleeEdges;
+
+private:
+  void reportContextForRemark(DiagnosticInfoOptimizationBase &OR);
+
+  MLInlineAdvisor *getAdvisor() const {
+    return static_cast<MLInlineAdvisor *>(Advisor);
+  };
+};
+
+} // namespace llvm
+
+#endif // LLVM_ANALYSIS_MLINLINEADVISOR_H
\ No newline at end of file
diff --git a/llvm/include/llvm/Analysis/MLModelRunner.h b/llvm/include/llvm/Analysis/MLModelRunner.h
new file mode 100644
index 0000000000000..7cfa6efedf108
--- /dev/null
+++ b/llvm/include/llvm/Analysis/MLModelRunner.h
@@ -0,0 +1,39 @@
+//===- MLModelRunner.h ---- ML model runner interface -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+
+#ifndef LLVM_ANALYSIS_MLMODELRUNNER_H
+#define LLVM_ANALYSIS_MLMODELRUNNER_H
+
+#include "llvm/Analysis/InlineModelFeatureMaps.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+/// MLModelRunner interface: abstraction of a mechanism for evaluating a
+/// tensorflow "saved model".
+class MLModelRunner {
+public:
+  // Disallows copy and assign.
+  MLModelRunner(const MLModelRunner &) = delete;
+  MLModelRunner &operator=(const MLModelRunner &) = delete;
+  virtual ~MLModelRunner() = default;
+
+  virtual bool run() = 0;
+  virtual void setFeature(FeatureIndex Index, int64_t Value) = 0;
+  virtual int64_t getFeature(int Index) const = 0;
+
+protected:
+  MLModelRunner(LLVMContext &Ctx) : Ctx(Ctx) {}
+
+  LLVMContext &Ctx;
+};
+} // namespace llvm
+
+#endif // LLVM_ANALYSIS_MLMODELRUNNER_H
diff --git a/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h b/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
index f69adc2d9b77e..0777dc7d7862b 100644
--- a/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
+++ b/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
@@ -19,7 +19,6 @@
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/PointerSumType.h"
 #include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Metadata.h"
@@ -35,6 +34,7 @@
 
 namespace llvm {
 
+class AAResults;
 class AssumptionCache;
 class DominatorTree;
 class Function;
@@ -355,7 +355,7 @@ class MemoryDependenceResults {
   ReverseDepMapType ReverseNonLocalDeps;
 
   /// Current AA implementation, just a cache.
-  AliasAnalysis &AA;
+  AAResults &AA;
   AssumptionCache &AC;
   const TargetLibraryInfo &TLI;
   DominatorTree &DT;
@@ -365,7 +365,7 @@ class MemoryDependenceResults {
   unsigned DefaultBlockScanLimit;
 
 public:
-  MemoryDependenceResults(AliasAnalysis &AA, AssumptionCache &AC,
+  MemoryDependenceResults(AAResults &AA, AssumptionCache &AC,
                           const TargetLibraryInfo &TLI, DominatorTree &DT,
                           PhiValues &PV, unsigned DefaultBlockScanLimit)
       : AA(AA), AC(AC), TLI(TLI), DT(DT), PV(PV),
diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h
index f7bb15d256fde..d01ac7da85cd0 100644
--- a/llvm/include/llvm/Analysis/MemoryLocation.h
+++ b/llvm/include/llvm/Analysis/MemoryLocation.h
@@ -17,13 +17,13 @@
 
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/Optional.h"
-#include "llvm/IR/Instruction.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/Support/TypeSize.h"
 
 namespace llvm {
 
 class CallBase;
+class Instruction;
 class LoadInst;
 class StoreInst;
 class MemTransferInst;
diff --git a/llvm/include/llvm/Analysis/MemorySSAUpdater.h b/llvm/include/llvm/Analysis/MemorySSAUpdater.h
index 141c0e57e6330..20588ef083c59 100644
--- a/llvm/include/llvm/Analysis/MemorySSAUpdater.h
+++ b/llvm/include/llvm/Analysis/MemorySSAUpdater.h
@@ -48,7 +48,6 @@ class BranchInst;
 class DominatorTree;
 class Instruction;
 class LoopBlocksRPO;
-class WeakVH;
 
 using ValueToValueMapTy = ValueMap<const Value *, WeakTrackingVH>;
 using PhiToDefMap = SmallDenseMap<MemoryPhi *, MemoryAccess *>;
diff --git a/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h b/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h
index b89ea496cd8a3..ab97d5b8504e1 100644
--- a/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h
+++ b/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h
@@ -17,11 +17,11 @@
 #include "llvm/ADT/Optional.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/IR/DiagnosticInfo.h"
-#include "llvm/IR/Function.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
 
 namespace llvm {
+class Function;
 class Value;
 
 /// The optimization diagnostic interface.
diff --git a/llvm/include/llvm/Analysis/OrderedInstructions.h b/llvm/include/llvm/Analysis/OrderedInstructions.h
deleted file mode 100644
index b2bf857502280..0000000000000
--- a/llvm/include/llvm/Analysis/OrderedInstructions.h
+++ /dev/null
@@ -1,57 +0,0 @@
-//===- llvm/Transforms/Utils/OrderedInstructions.h -------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines an efficient way to check for dominance relation between 2
-// instructions.
-//
-// FIXME: This is really just a convenience wrapper to check dominance between
-// two arbitrary instructions in different basic blocks. We should fold it into
-// DominatorTree, which is the more widely used interface.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_ANALYSIS_ORDEREDINSTRUCTIONS_H
-#define LLVM_ANALYSIS_ORDEREDINSTRUCTIONS_H
-
-namespace llvm {
-
-class DominatorTree;
-class Instruction;
-
-class OrderedInstructions {
-  /// The dominator tree of the parent function.
-  DominatorTree *DT;
-
-  /// Return true if the first instruction comes before the second in the
-  /// same basic block. It will create an ordered basic block, if it does
-  /// not yet exist in OBBMap.
-  bool localDominates(const Instruction *, const Instruction *) const;
-
-public:
-  /// Constructor.
-  OrderedInstructions(DominatorTree *DT) : DT(DT) {}
-
-  /// Return true if first instruction dominates the second.
-  bool dominates(const Instruction *, const Instruction *) const;
-
-  /// Return true if the first instruction comes before the second in the
-  /// dominator tree DFS traversal if they are in different basic blocks,
-  /// or if the first instruction comes before the second in the same basic
-  /// block.
-  bool dfsBefore(const Instruction *, const Instruction *) const;
-
-  // Return true if the first instruction comes before the second in the
-  // dominator tree BFS traversal based on the level number of nodes in
-  // dominator tree if they are in different basic blocks else if the first
-  // instruction comes before the second in the same basic block.
-  bool domTreeLevelBefore(const Instruction *, const Instruction *) const;
-};
-
-} // end namespace llvm
-
-#endif // LLVM_ANALYSIS_ORDEREDINSTRUCTIONS_H
diff --git a/llvm/include/llvm/Analysis/PostDominators.h b/llvm/include/llvm/Analysis/PostDominators.h
index 801eb3d59673f..296110d8d03be 100644
--- a/llvm/include/llvm/Analysis/PostDominators.h
+++ b/llvm/include/llvm/Analysis/PostDominators.h
@@ -88,9 +88,7 @@ struct PostDominatorTreeWrapperPass : public FunctionPass {
     AU.setPreservesAll();
   }
 
-  void releaseMemory() override {
-    DT.releaseMemory();
-  }
+  void releaseMemory() override { DT.reset(); }
 
   void print(raw_ostream &OS, const Module*) const override;
 };
diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionDivision.h b/llvm/include/llvm/Analysis/ScalarEvolutionDivision.h
new file mode 100644
index 0000000000000..480f92c117a01
--- /dev/null
+++ b/llvm/include/llvm/Analysis/ScalarEvolutionDivision.h
@@ -0,0 +1,69 @@
+//===- llvm/Analysis/ScalarEvolutionDivision.h - See below ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the class that knows how to divide SCEV's.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_SCALAREVOLUTIONDIVISION_H
+#define LLVM_ANALYSIS_SCALAREVOLUTIONDIVISION_H
+
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+
+namespace llvm {
+
+class SCEV;
+
+class ScalarEvolution;
+
+struct SCEVCouldNotCompute;
+
+struct SCEVDivision : public SCEVVisitor<SCEVDivision, void> {
+public:
+  // Computes the Quotient and Remainder of the division of Numerator by
+  // Denominator.
+  static void divide(ScalarEvolution &SE, const SCEV *Numerator,
+                     const SCEV *Denominator, const SCEV **Quotient,
+                     const SCEV **Remainder);
+
+  // Except in the trivial case described above, we do not know how to divide
+  // Expr by Denominator for the following functions with empty implementation.
+  void visitTruncateExpr(const SCEVTruncateExpr *Numerator) {}
+  void visitZeroExtendExpr(const SCEVZeroExtendExpr *Numerator) {}
+  void visitSignExtendExpr(const SCEVSignExtendExpr *Numerator) {}
+  void visitUDivExpr(const SCEVUDivExpr *Numerator) {}
+  void visitSMaxExpr(const SCEVSMaxExpr *Numerator) {}
+  void visitUMaxExpr(const SCEVUMaxExpr *Numerator) {}
+  void visitSMinExpr(const SCEVSMinExpr *Numerator) {}
+  void visitUMinExpr(const SCEVUMinExpr *Numerator) {}
+  void visitUnknown(const SCEVUnknown *Numerator) {}
+  void visitCouldNotCompute(const SCEVCouldNotCompute *Numerator) {}
+
+  void visitConstant(const SCEVConstant *Numerator);
+
+  void visitAddRecExpr(const SCEVAddRecExpr *Numerator);
+
+  void visitAddExpr(const SCEVAddExpr *Numerator);
+
+  void visitMulExpr(const SCEVMulExpr *Numerator);
+
+private:
+  SCEVDivision(ScalarEvolution &S, const SCEV *Numerator,
+               const SCEV *Denominator);
+
+  // Convenience function for giving up on the division. We set the quotient to
+  // be equal to zero and the remainder to be equal to the numerator.
+  void cannotDivide(const SCEV *Numerator);
+
+  ScalarEvolution &SE;
+  const SCEV *Denominator, *Quotient, *Remainder, *Zero, *One;
+};
+
+} // end namespace llvm
+
+#endif // LLVM_ANALYSIS_SCALAREVOLUTIONDIVISION_H
diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
index 876d68438ef95..0076e02ae1bf7 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
@@ -222,18 +222,22 @@ class Type;
   class SCEVAddExpr : public SCEVCommutativeExpr {
     friend class ScalarEvolution;
 
-    SCEVAddExpr(const FoldingSetNodeIDRef ID,
-                const SCEV *const *O, size_t N)
-      : SCEVCommutativeExpr(ID, scAddExpr, O, N) {}
+    Type *Ty;
 
-  public:
-    Type *getType() const {
-      // Use the type of the last operand, which is likely to be a pointer
-      // type, if there is one. This doesn't usually matter, but it can help
-      // reduce casts when the expressions are expanded.
-      return getOperand(getNumOperands() - 1)->getType();
+    SCEVAddExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N)
+        : SCEVCommutativeExpr(ID, scAddExpr, O, N) {
+      auto *FirstPointerTypedOp = find_if(operands(), [](const SCEV *Op) {
+        return Op->getType()->isPointerTy();
+      });
+      if (FirstPointerTypedOp != operands().end())
+        Ty = (*FirstPointerTypedOp)->getType();
+      else
+        Ty = getOperand(0)->getType();
     }
 
+  public:
+    Type *getType() const { return Ty; }
+
     /// Methods for support type inquiry through isa, cast, and dyn_cast:
     static bool classof(const SCEV *S) {
       return S->getSCEVType() == scAddExpr;
diff --git a/llvm/include/llvm/Analysis/StackLifetime.h b/llvm/include/llvm/Analysis/StackLifetime.h
index cead60020b193..8abc6cc1ce00f 100644
--- a/llvm/include/llvm/Analysis/StackLifetime.h
+++ b/llvm/include/llvm/Analysis/StackLifetime.h
@@ -13,7 +13,6 @@
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/llvm/include/llvm/Analysis/StackSafetyAnalysis.h b/llvm/include/llvm/Analysis/StackSafetyAnalysis.h
index 3ee520eb04112..846c2e6f7e914 100644
--- a/llvm/include/llvm/Analysis/StackSafetyAnalysis.h
+++ b/llvm/include/llvm/Analysis/StackSafetyAnalysis.h
@@ -45,6 +45,12 @@ class StackSafetyInfo {
   void print(raw_ostream &O) const;
 
   /// Parameters use for a FunctionSummary.
+  /// Function collects access information of all pointer parameters.
+  /// Information includes a range of direct access of parameters by the
+  /// functions and all call sites accepting the parameter.
+  /// StackSafety assumes that missing parameter information means possibility
+  /// of access to the parameter with any offset, so we can correctly link
+  /// code without StackSafety information, e.g. non-ThinLTO.
   std::vector<FunctionSummary::ParamAccess> getParamAccesses() const;
 };
 
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index cc6426b86f3cf..b6698eefdb011 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -112,7 +112,7 @@ class IntrinsicCostAttributes {
   Type *RetTy = nullptr;
   Intrinsic::ID IID;
   SmallVector<Type *, 4> ParamTys;
-  SmallVector<Value *, 4> Arguments;
+  SmallVector<const Value *, 4> Arguments;
   FastMathFlags FMF;
   unsigned VF = 1;
   // If ScalarizationCost is UINT_MAX, the cost of scalarizing the
@@ -146,7 +146,7 @@ class IntrinsicCostAttributes {
                           ArrayRef<Type *> Tys);
 
   IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
-                          ArrayRef<Value *> Args);
+                          ArrayRef<const Value *> Args);
 
   Intrinsic::ID getID() const { return IID; }
   const IntrinsicInst *getInst() const { return II; }
@@ -154,7 +154,7 @@ class IntrinsicCostAttributes {
   unsigned getVectorFactor() const { return VF; }
   FastMathFlags getFlags() const { return FMF; }
   unsigned getScalarizationCost() const { return ScalarizationCost; }
-  const SmallVectorImpl<Value *> &getArgs() const { return Arguments; }
+  const SmallVectorImpl<const Value *> &getArgs() const { return Arguments; }
   const SmallVectorImpl<Type *> &getArgTypes() const { return ParamTys; }
 
   bool isTypeBasedOnly() const {
@@ -450,11 +450,6 @@ class TargetTransformInfo {
     /// transformation will select an unrolling factor based on the current cost
     /// threshold and other factors.
     unsigned Count;
-    /// A forced peeling factor (the number of bodied of the original loop
-    /// that should be peeled off before the loop body). When set to 0, the
-    /// unrolling transformation will select a peeling factor based on profile
-    /// information and other factors.
-    unsigned PeelCount;
     /// Default unroll count for loops with run-time trip count.
     unsigned DefaultUnrollRuntimeCount;
     // Set the maximum unrolling factor. The unrolling factor may be selected
@@ -488,19 +483,10 @@ class TargetTransformInfo {
     bool Force;
     /// Allow using trip count upper bound to unroll loops.
     bool UpperBound;
-    /// Allow peeling off loop iterations.
-    bool AllowPeeling;
-    /// Allow peeling off loop iterations for loop nests.
-    bool AllowLoopNestsPeeling;
     /// Allow unrolling of all the iterations of the runtime loop remainder.
     bool UnrollRemainder;
     /// Allow unroll and jam. Used to enable unroll and jam for the target.
     bool UnrollAndJam;
-    /// Allow peeling basing on profile. Uses to enable peeling off all
-    /// iterations basing on provided profile.
-    /// If the value is true the peeling cost model can decide to peel only
-    /// some iterations and in this case it will set this to false.
-    bool PeelProfiledIterations;
     /// Threshold for unroll and jam, for inner loop size. The 'Threshold'
     /// value above is used during unroll and jam for the outer loop size.
     /// This value is used in the same manner to limit the size of the inner
@@ -534,6 +520,28 @@ class TargetTransformInfo {
   /// intrinsic is supported.
   bool emitGetActiveLaneMask() const;
 
+  // Parameters that control the loop peeling transformation
+  struct PeelingPreferences {
+    /// A forced peeling factor (the number of bodied of the original loop
+    /// that should be peeled off before the loop body). When set to 0, the
+    /// a peeling factor based on profile information and other factors.
+    unsigned PeelCount;
+    /// Allow peeling off loop iterations.
+    bool AllowPeeling;
+    /// Allow peeling off loop iterations for loop nests.
+    bool AllowLoopNestsPeeling;
+    /// Allow peeling basing on profile. Uses to enable peeling off all
+    /// iterations basing on provided profile.
+    /// If the value is true the peeling cost model can decide to peel only
+    /// some iterations and in this case it will set this to false.
+    bool PeelProfiledIterations;
+  };
+
+  /// Get target-customized preferences for the generic loop peeling
+  /// transformation. The caller will initialize \p PP with the current
+  /// target-independent defaults with information from \p L and \p SE.
+  void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                             PeelingPreferences &PP) const;
   /// @}
 
   /// \name Scalar Target Information
@@ -951,7 +959,7 @@ class TargetTransformInfo {
   unsigned getMaxInterleaveFactor(unsigned VF) const;
 
   /// Collect properties of V used in cost analysis, e.g. OP_PowerOf2.
-  static OperandValueKind getOperandInfo(Value *V,
+  static OperandValueKind getOperandInfo(const Value *V,
                                          OperandValueProperties &OpProps);
 
   /// This is an approximation of reciprocal throughput of a math/logic op.
@@ -1024,8 +1032,8 @@ class TargetTransformInfo {
 
   /// \return The cost of masked Load and Store instructions.
   int getMaskedMemoryOpCost(
-    unsigned Opcode, Type *Src, unsigned Alignment, unsigned AddressSpace,
-    TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const;
+      unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const;
 
   /// \return The cost of Gather or Scatter operation
   /// \p Opcode - is a type of memory access Load or Store
@@ -1037,9 +1045,9 @@ class TargetTransformInfo {
   /// \p I - the optional original context instruction, if one exists, e.g. the
   ///        load/store to transform or the call to the gather/scatter intrinsic
   int getGatherScatterOpCost(
-    unsigned Opcode, Type *DataTy, Value *Ptr, bool VariableMask,
-    unsigned Alignment, TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
-    const Instruction *I = nullptr) const;
+      unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
+      Align Alignment, TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
+      const Instruction *I = nullptr) const;
 
   /// \return The cost of the interleaved memory operation.
   /// \p Opcode is the memory operation code
@@ -1052,10 +1060,10 @@ class TargetTransformInfo {
   /// \p UseMaskForCond indicates if the memory access is predicated.
   /// \p UseMaskForGaps indicates if gaps should be masked.
   int getInterleavedMemoryOpCost(
-    unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
-    unsigned Alignment, unsigned AddressSpace,
-    TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
-    bool UseMaskForCond = false, bool UseMaskForGaps = false) const;
+      unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+      Align Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
+      bool UseMaskForCond = false, bool UseMaskForGaps = false) const;
 
   /// Calculate the cost of performing a vector reduction.
   ///
@@ -1181,13 +1189,11 @@ class TargetTransformInfo {
   bool isLegalToVectorizeStore(StoreInst *SI) const;
 
   /// \returns True if it is legal to vectorize the given load chain.
-  bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
-                                   unsigned Alignment,
+  bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, Align Alignment,
                                    unsigned AddrSpace) const;
 
   /// \returns True if it is legal to vectorize the given store chain.
-  bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
-                                    unsigned Alignment,
+  bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, Align Alignment,
                                     unsigned AddrSpace) const;
 
   /// \returns The new vector factor value if the target doesn't support \p
@@ -1284,6 +1290,8 @@ class TargetTransformInfo::Concept {
   virtual bool isLoweredToCall(const Function *F) = 0;
   virtual void getUnrollingPreferences(Loop *L, ScalarEvolution &,
                                        UnrollingPreferences &UP) = 0;
+  virtual void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                                     PeelingPreferences &PP) = 0;
   virtual bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
                                         AssumptionCache &AC,
                                         TargetLibraryInfo *LibInfo,
@@ -1425,22 +1433,19 @@ class TargetTransformInfo::Concept {
                               unsigned AddressSpace,
                               TTI::TargetCostKind CostKind,
                               const Instruction *I) = 0;
-  virtual int getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
-                                    unsigned Alignment,
+  virtual int getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
                                     unsigned AddressSpace,
                                     TTI::TargetCostKind CostKind) = 0;
-  virtual int getGatherScatterOpCost(
-    unsigned Opcode, Type *DataTy, Value *Ptr, bool VariableMask,
-    unsigned Alignment, TTI::TargetCostKind CostKind,
-    const Instruction *I = nullptr) = 0;
-
-  virtual int
-  getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor,
-                             ArrayRef<unsigned> Indices, unsigned Alignment,
-                             unsigned AddressSpace,
-                             TTI::TargetCostKind CostKind,
-                             bool UseMaskForCond = false,
-                             bool UseMaskForGaps = false) = 0;
+  virtual int getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
+                                     const Value *Ptr, bool VariableMask,
+                                     Align Alignment,
+                                     TTI::TargetCostKind CostKind,
+                                     const Instruction *I = nullptr) = 0;
+
+  virtual int getInterleavedMemoryOpCost(
+      unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+      Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
+      bool UseMaskForCond = false, bool UseMaskForGaps = false) = 0;
   virtual int getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
                                          bool IsPairwiseForm,
                                          TTI::TargetCostKind CostKind) = 0;
@@ -1481,10 +1486,10 @@ class TargetTransformInfo::Concept {
   virtual bool isLegalToVectorizeLoad(LoadInst *LI) const = 0;
   virtual bool isLegalToVectorizeStore(StoreInst *SI) const = 0;
   virtual bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
-                                           unsigned Alignment,
+                                           Align Alignment,
                                            unsigned AddrSpace) const = 0;
   virtual bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
-                                            unsigned Alignment,
+                                            Align Alignment,
                                             unsigned AddrSpace) const = 0;
   virtual unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize,
                                        unsigned ChainSizeInBytes,
@@ -1565,6 +1570,10 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
                                UnrollingPreferences &UP) override {
     return Impl.getUnrollingPreferences(L, SE, UP);
   }
+  void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                             PeelingPreferences &PP) override {
+    return Impl.getPeelingPreferences(L, SE, PP);
+  }
   bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
                                 AssumptionCache &AC, TargetLibraryInfo *LibInfo,
                                 HardwareLoopInfo &HWLoopInfo) override {
@@ -1842,21 +1851,21 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
     return Impl.getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
                                 CostKind, I);
   }
-  int getMaskedMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment,
+  int getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
                             unsigned AddressSpace,
                             TTI::TargetCostKind CostKind) override {
     return Impl.getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
                                       CostKind);
   }
-  int getGatherScatterOpCost(
-      unsigned Opcode, Type *DataTy, Value *Ptr, bool VariableMask,
-      unsigned Alignment, TTI::TargetCostKind CostKind,
-      const Instruction *I = nullptr) override {
+  int getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
+                             bool VariableMask, Align Alignment,
+                             TTI::TargetCostKind CostKind,
+                             const Instruction *I = nullptr) override {
     return Impl.getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
                                        Alignment, CostKind, I);
   }
   int getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor,
-                                 ArrayRef<unsigned> Indices, unsigned Alignment,
+                                 ArrayRef<unsigned> Indices, Align Alignment,
                                  unsigned AddressSpace,
                                  TTI::TargetCostKind CostKind,
                                  bool UseMaskForCond,
@@ -1946,14 +1955,12 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   bool isLegalToVectorizeStore(StoreInst *SI) const override {
     return Impl.isLegalToVectorizeStore(SI);
   }
-  bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
-                                   unsigned Alignment,
+  bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, Align Alignment,
                                    unsigned AddrSpace) const override {
     return Impl.isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment,
                                             AddrSpace);
   }
-  bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
-                                    unsigned Alignment,
+  bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, Align Alignment,
                                     unsigned AddrSpace) const override {
     return Impl.isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment,
                                              AddrSpace);
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index f0a07c0aa0182..0ce975d6d4b5d 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -150,6 +150,9 @@ class TargetTransformInfoImplBase {
   void getUnrollingPreferences(Loop *, ScalarEvolution &,
                                TTI::UnrollingPreferences &) {}
 
+  void getPeelingPreferences(Loop *, ScalarEvolution &,
+                             TTI::PeelingPreferences &) {}
+
   bool isLegalAddImmediate(int64_t Imm) { return false; }
 
   bool isLegalICmpImmediate(int64_t Imm) { return false; }
@@ -464,26 +467,23 @@ class TargetTransformInfoImplBase {
     return 1;
   }
 
-  unsigned getMaskedMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment,
+  unsigned getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
                                  unsigned AddressSpace,
                                  TTI::TargetCostKind CostKind) {
     return 1;
   }
 
-  unsigned getGatherScatterOpCost(
-      unsigned Opcode, Type *DataTy, Value *Ptr, bool VariableMask,
-      unsigned Alignment, TTI::TargetCostKind CostKind,
-      const Instruction *I = nullptr) {
+  unsigned getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
+                                  const Value *Ptr, bool VariableMask,
+                                  Align Alignment, TTI::TargetCostKind CostKind,
+                                  const Instruction *I = nullptr) {
     return 1;
   }
 
-  unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
-                                      unsigned Factor,
-                                      ArrayRef<unsigned> Indices,
-                                      unsigned Alignment, unsigned AddressSpace,
-                                      TTI::TargetCostKind CostKind,
-                                      bool UseMaskForCond,
-                                      bool UseMaskForGaps) {
+  unsigned getInterleavedMemoryOpCost(
+      unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+      Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
+      bool UseMaskForCond, bool UseMaskForGaps) {
     return 1;
   }
 
@@ -610,14 +610,12 @@ class TargetTransformInfoImplBase {
 
   bool isLegalToVectorizeStore(StoreInst *SI) const { return true; }
 
-  bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
-                                   unsigned Alignment,
+  bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, Align Alignment,
                                    unsigned AddrSpace) const {
     return true;
   }
 
-  bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
-                                    unsigned Alignment,
+  bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, Align Alignment,
                                     unsigned AddrSpace) const {
     return true;
   }
@@ -923,13 +921,17 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
                                            CostKind, I);
     }
     case Instruction::InsertElement: {
-      auto *IE = cast<InsertElementInst>(U);
+      auto *IE = dyn_cast<InsertElementInst>(U);
+      if (!IE)
+        return TTI::TCC_Basic; // FIXME
       auto *CI = dyn_cast<ConstantInt>(IE->getOperand(2));
       unsigned Idx = CI ? CI->getZExtValue() : -1;
       return TargetTTI->getVectorInstrCost(Opcode, Ty, Idx);
     }
     case Instruction::ShuffleVector: {
-      auto *Shuffle = cast<ShuffleVectorInst>(U);
+      auto *Shuffle = dyn_cast<ShuffleVectorInst>(U);
+      if (!Shuffle)
+        return TTI::TCC_Basic; // FIXME
       auto *VecTy = cast<VectorType>(U->getType());
       auto *VecSrcTy = cast<VectorType>(U->getOperand(0)->getType());
 
@@ -959,7 +961,10 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
     }
     case Instruction::ExtractElement: {
       unsigned Idx = -1;
-      auto *EEI = cast<ExtractElementInst>(U);
+      auto *EEI = dyn_cast<ExtractElementInst>(U);
+      if (!EEI)
+        return TTI::TCC_Basic; // FIXME
+
       auto *CI = dyn_cast<ConstantInt>(EEI->getOperand(1));
       if (CI)
         Idx = CI->getZExtValue();
diff --git a/llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h b/llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h
index 93ef9103df000..345f11a3aad5c 100644
--- a/llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h
+++ b/llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h
@@ -53,7 +53,6 @@ class TypeBasedAAResult : public AAResultBase<TypeBasedAAResult> {
 
 private:
   bool Aliases(const MDNode *A, const MDNode *B) const;
-  bool PathAliases(const MDNode *A, const MDNode *B) const;
 };
 
 /// Analysis pass providing a never-invalidated alias analysis result.
diff --git a/llvm/include/llvm/Analysis/Utils/Local.h b/llvm/include/llvm/Analysis/Utils/Local.h
index c21be2301ab1f..f31b563454244 100644
--- a/llvm/include/llvm/Analysis/Utils/Local.h
+++ b/llvm/include/llvm/Analysis/Utils/Local.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_ANALYSIS_UTILS_LOCAL_H
 #define LLVM_ANALYSIS_UTILS_LOCAL_H
 
+#include "llvm/ADT/StringRef.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/GetElementPtrTypeIterator.h"
 
@@ -56,7 +57,7 @@ Value *EmitGEPOffset(IRBuilderTy *Builder, const DataLayout &DL, User *GEP,
 
         if (Size)
           Result = Builder->CreateAdd(Result, ConstantInt::get(IntIdxTy, Size),
-                                      GEP->getName()+".offs");
+                                      GEP->getName().str()+".offs");
         continue;
       }
 
@@ -70,7 +71,7 @@ Value *EmitGEPOffset(IRBuilderTy *Builder, const DataLayout &DL, User *GEP,
       Scale =
           ConstantExpr::getMul(OC, Scale, false /*NUW*/, isInBounds /*NSW*/);
       // Emit an add instruction.
-      Result = Builder->CreateAdd(Result, Scale, GEP->getName()+".offs");
+      Result = Builder->CreateAdd(Result, Scale, GEP->getName().str()+".offs");
       continue;
     }
 
@@ -81,16 +82,16 @@ Value *EmitGEPOffset(IRBuilderTy *Builder, const DataLayout &DL, User *GEP,
 
     // Convert to correct type.
     if (Op->getType() != IntIdxTy)
-      Op = Builder->CreateIntCast(Op, IntIdxTy, true, Op->getName()+".c");
+      Op = Builder->CreateIntCast(Op, IntIdxTy, true, Op->getName().str()+".c");
     if (Size != 1) {
       // We'll let instcombine(mul) convert this to a shl if possible.
       Op = Builder->CreateMul(Op, ConstantInt::get(IntIdxTy, Size),
-                              GEP->getName() + ".idx", false /*NUW*/,
+                              GEP->getName().str() + ".idx", false /*NUW*/,
                               isInBounds /*NSW*/);
     }
 
     // Emit an add instruction.
-    Result = Builder->CreateAdd(Op, Result, GEP->getName()+".offs");
+    Result = Builder->CreateAdd(Op, Result, GEP->getName().str()+".offs");
   }
   return Result;
 }
diff --git a/llvm/include/llvm/BinaryFormat/MachO.h b/llvm/include/llvm/BinaryFormat/MachO.h
index 0010f36e8b897..e43fea0a2465e 100644
--- a/llvm/include/llvm/BinaryFormat/MachO.h
+++ b/llvm/include/llvm/BinaryFormat/MachO.h
@@ -82,7 +82,8 @@ enum {
   MH_HAS_TLV_DESCRIPTORS = 0x00800000u,
   MH_NO_HEAP_EXECUTION = 0x01000000u,
   MH_APP_EXTENSION_SAFE = 0x02000000u,
-  MH_NLIST_OUTOFSYNC_WITH_DYLDINFO = 0x04000000u
+  MH_NLIST_OUTOFSYNC_WITH_DYLDINFO = 0x04000000u,
+  MH_DYLIB_IN_CACHE = 0x80000000u,
 };
 
 enum : uint32_t {
diff --git a/llvm/include/llvm/BinaryFormat/Wasm.h b/llvm/include/llvm/BinaryFormat/Wasm.h
index 28635d79106ff..1aca692e30a79 100644
--- a/llvm/include/llvm/BinaryFormat/Wasm.h
+++ b/llvm/include/llvm/BinaryFormat/Wasm.h
@@ -63,8 +63,8 @@ struct WasmExport {
 
 struct WasmLimits {
   uint8_t Flags;
-  uint32_t Initial;
-  uint32_t Maximum;
+  uint64_t Initial;
+  uint64_t Maximum;
 };
 
 struct WasmTable {
@@ -232,6 +232,7 @@ enum : unsigned {
   WASM_TYPE_V128 = 0x7B,
   WASM_TYPE_FUNCREF = 0x70,
   WASM_TYPE_EXNREF = 0x68,
+  WASM_TYPE_EXTERNREF = 0x6F,
   WASM_TYPE_FUNC = 0x60,
   WASM_TYPE_NORESULT = 0x40, // for blocks with no result values
 };
@@ -253,11 +254,14 @@ enum : unsigned {
   WASM_OPCODE_GLOBAL_GET = 0x23,
   WASM_OPCODE_GLOBAL_SET = 0x24,
   WASM_OPCODE_I32_STORE = 0x36,
+  WASM_OPCODE_I64_STORE = 0x37,
   WASM_OPCODE_I32_CONST = 0x41,
   WASM_OPCODE_I64_CONST = 0x42,
   WASM_OPCODE_F32_CONST = 0x43,
   WASM_OPCODE_F64_CONST = 0x44,
   WASM_OPCODE_I32_ADD = 0x6a,
+  WASM_OPCODE_I64_ADD = 0x7c,
+  WASM_OPCODE_REF_NULL = 0xd0,
 };
 
 // Opcodes used in synthetic functions.
@@ -276,8 +280,10 @@ enum : unsigned {
 };
 
 enum : unsigned {
+  WASM_LIMITS_FLAG_NONE = 0x0,
   WASM_LIMITS_FLAG_HAS_MAX = 0x1,
   WASM_LIMITS_FLAG_IS_SHARED = 0x2,
+  WASM_LIMITS_FLAG_IS_64 = 0x4,
 };
 
 enum : unsigned {
@@ -355,6 +361,7 @@ enum class ValType {
   F64 = WASM_TYPE_F64,
   V128 = WASM_TYPE_V128,
   EXNREF = WASM_TYPE_EXNREF,
+  EXTERNREF = WASM_TYPE_EXTERNREF,
 };
 
 struct WasmSignature {
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index b370fa48718bd..de4fe66303246 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -85,7 +85,7 @@ enum ModuleCodes {
   MODULE_CODE_ASM = 4,         // ASM:         [strchr x N]
   MODULE_CODE_SECTIONNAME = 5, // SECTIONNAME: [strchr x N]
 
-  // FIXME: Remove DEPLIB in 4.0.
+  // Deprecated, but still needed to read old bitcode files.
   MODULE_CODE_DEPLIB = 6, // DEPLIB:      [strchr x N]
 
   // GLOBALVAR: [pointer type, isconst, initid,
@@ -121,7 +121,7 @@ enum ModuleCodes {
 
 /// PARAMATTR blocks have code for defining a parameter attribute set.
 enum AttributeCodes {
-  // FIXME: Remove `PARAMATTR_CODE_ENTRY_OLD' in 4.0
+  // Deprecated, but still needed to read old bitcode files.
   PARAMATTR_CODE_ENTRY_OLD = 1, // ENTRY: [paramidx0, attr0,
                                 //         paramidx1, attr1...]
   PARAMATTR_CODE_ENTRY = 2,     // ENTRY: [attrgrp0, attrgrp1, ...]
@@ -643,6 +643,7 @@ enum AttributeKindCodes {
   ATTR_KIND_PREALLOCATED = 65,
   ATTR_KIND_NO_MERGE = 66,
   ATTR_KIND_NULL_POINTER_IS_VALID = 67,
+  ATTR_KIND_NOUNDEF = 68,
 };
 
 enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/CMakeLists.txt b/llvm/include/llvm/CMakeLists.txt
index 1d5ca3ba92b0e..7cf8699aa21e4 100644
--- a/llvm/include/llvm/CMakeLists.txt
+++ b/llvm/include/llvm/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_subdirectory(IR)
 add_subdirectory(Support)
+add_subdirectory(Frontend/OpenMP)
 
 # If we're doing an out-of-tree build, copy a module map for generated
 # header files into the build area.
diff --git a/llvm/include/llvm/CodeGen/AntiDepBreaker.h b/llvm/include/llvm/CodeGen/AntiDepBreaker.h
index d9dfcd34ded1b..d75c13e2dd756 100644
--- a/llvm/include/llvm/CodeGen/AntiDepBreaker.h
+++ b/llvm/include/llvm/CodeGen/AntiDepBreaker.h
@@ -60,8 +60,9 @@ class AntiDepBreaker {
   /// other machine instruction to use NewReg.
   void UpdateDbgValue(MachineInstr &MI, unsigned OldReg, unsigned NewReg) {
     assert(MI.isDebugValue() && "MI is not DBG_VALUE!");
-    if (MI.getOperand(0).isReg() && MI.getOperand(0).getReg() == OldReg)
-      MI.getOperand(0).setReg(NewReg);
+    if (MI.getDebugOperand(0).isReg() &&
+        MI.getDebugOperand(0).getReg() == OldReg)
+      MI.getDebugOperand(0).setReg(NewReg);
   }
 
   /// Update all DBG_VALUE instructions that may be affected by the dependency
diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h
index 939a5cb08a378..0eb950861af6e 100644
--- a/llvm/include/llvm/CodeGen/AsmPrinter.h
+++ b/llvm/include/llvm/CodeGen/AsmPrinter.h
@@ -17,8 +17,6 @@
 
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/Twine.h"
 #include "llvm/CodeGen/AsmPrinterHandler.h"
 #include "llvm/CodeGen/DwarfStringPoolEntry.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -70,8 +68,10 @@ class MDNode;
 class Module;
 class raw_ostream;
 class StackMaps;
+class StringRef;
 class TargetLoweringObjectFile;
 class TargetMachine;
+class Twine;
 
 namespace remarks {
 class RemarkStreamer;
@@ -126,6 +126,14 @@ class AsmPrinter : public MachineFunctionPass {
   /// default, this is equal to CurrentFnSym.
   MCSymbol *CurrentFnSymForSize = nullptr;
 
+  /// Map a basic block section ID to the begin and end symbols of that section
+  ///  which determine the section's range.
+  struct MBBSectionRange {
+    MCSymbol *BeginLabel, *EndLabel;
+  };
+
+  MapVector<unsigned, MBBSectionRange> MBBSectionRanges;
+
   /// Map global GOT equivalent MCSymbols to GlobalVariables and keep track of
   /// its number of uses by other globals.
   using GOTEquivUsePair = std::pair<const GlobalVariable *, unsigned>;
@@ -662,7 +670,7 @@ class AsmPrinter : public MachineFunctionPass {
   virtual void emitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const;
 
   /// Return the alignment for the specified \p GV.
-  static Align getGVAlignment(const GlobalValue *GV, const DataLayout &DL,
+  static Align getGVAlignment(const GlobalObject *GV, const DataLayout &DL,
                               Align InAlign = Align(1));
 
 private:
diff --git a/llvm/include/llvm/CodeGen/AsmPrinterHandler.h b/llvm/include/llvm/CodeGen/AsmPrinterHandler.h
index affb558f2fa66..899d067d03f00 100644
--- a/llvm/include/llvm/CodeGen/AsmPrinterHandler.h
+++ b/llvm/include/llvm/CodeGen/AsmPrinterHandler.h
@@ -67,6 +67,12 @@ class AsmPrinterHandler {
 
   /// Process end of an instruction.
   virtual void endInstruction() = 0;
+
+  /// Process beginning of a basic block during basic block sections.
+  virtual void beginBasicBlock(const MachineBasicBlock &MBB) {}
+
+  /// Process end of a basic block during basic block sections.
+  virtual void endBasicBlock(const MachineBasicBlock &MBB) {}
 };
 } // End of namespace llvm
 
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 5a52f273a566b..f9d32eadd23e2 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -78,18 +78,19 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   using BaseT = TargetTransformInfoImplCRTPBase<T>;
   using TTI = TargetTransformInfo;
 
+  /// Helper function to access this as a T.
+  T *thisT() { return static_cast<T *>(this); }
+
   /// Estimate a cost of Broadcast as an extract and sequence of insert
   /// operations.
   unsigned getBroadcastShuffleOverhead(FixedVectorType *VTy) {
     unsigned Cost = 0;
     // Broadcast cost is equal to the cost of extracting the zero'th element
     // plus the cost of inserting it into every element of the result vector.
-    Cost += static_cast<T *>(this)->getVectorInstrCost(
-        Instruction::ExtractElement, VTy, 0);
+    Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, VTy, 0);
 
     for (int i = 0, e = VTy->getNumElements(); i < e; ++i) {
-      Cost += static_cast<T *>(this)->getVectorInstrCost(
-          Instruction::InsertElement, VTy, i);
+      Cost += thisT()->getVectorInstrCost(Instruction::InsertElement, VTy, i);
     }
     return Cost;
   }
@@ -106,10 +107,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     // vector and finally index 3 of second vector and insert them at index
     // <0,1,2,3> of result vector.
     for (int i = 0, e = VTy->getNumElements(); i < e; ++i) {
-      Cost += static_cast<T *>(this)->getVectorInstrCost(
-          Instruction::InsertElement, VTy, i);
-      Cost += static_cast<T *>(this)->getVectorInstrCost(
-          Instruction::ExtractElement, VTy, i);
+      Cost += thisT()->getVectorInstrCost(Instruction::InsertElement, VTy, i);
+      Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, VTy, i);
     }
     return Cost;
   }
@@ -129,10 +128,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     // the source type plus the cost of inserting them into the result vector
     // type.
     for (int i = 0; i != NumSubElts; ++i) {
-      Cost += static_cast<T *>(this)->getVectorInstrCost(
-          Instruction::ExtractElement, VTy, i + Index);
-      Cost += static_cast<T *>(this)->getVectorInstrCost(
-          Instruction::InsertElement, SubVTy, i);
+      Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, VTy,
+                                          i + Index);
+      Cost +=
+          thisT()->getVectorInstrCost(Instruction::InsertElement, SubVTy, i);
     }
     return Cost;
   }
@@ -152,10 +151,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     // the source type plus the cost of inserting them into the result vector
     // type.
     for (int i = 0; i != NumSubElts; ++i) {
-      Cost += static_cast<T *>(this)->getVectorInstrCost(
-          Instruction::ExtractElement, SubVTy, i);
-      Cost += static_cast<T *>(this)->getVectorInstrCost(
-          Instruction::InsertElement, VTy, i + Index);
+      Cost +=
+          thisT()->getVectorInstrCost(Instruction::ExtractElement, SubVTy, i);
+      Cost += thisT()->getVectorInstrCost(Instruction::InsertElement, VTy,
+                                          i + Index);
     }
     return Cost;
   }
@@ -429,7 +428,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       for (Instruction &I : *BB) {
         if (isa<CallInst>(I) || isa<InvokeInst>(I)) {
           if (const Function *F = cast<CallBase>(I).getCalledFunction()) {
-            if (!static_cast<T *>(this)->isLoweredToCall(F))
+            if (!thisT()->isLoweredToCall(F))
               continue;
           }
 
@@ -452,6 +451,14 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     UP.BEInsns = 2;
   }
 
+  void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                             TTI::PeelingPreferences &PP) {
+    PP.PeelCount = 0;
+    PP.AllowPeeling = true;
+    PP.AllowLoopNestsPeeling = false;
+    PP.PeelProfiledIterations = true;
+  }
+
   bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
                                 AssumptionCache &AC,
                                 TargetLibraryInfo *LibInfo,
@@ -543,11 +550,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       if (!DemandedElts[i])
         continue;
       if (Insert)
-        Cost += static_cast<T *>(this)->getVectorInstrCost(
-            Instruction::InsertElement, Ty, i);
+        Cost += thisT()->getVectorInstrCost(Instruction::InsertElement, Ty, i);
       if (Extract)
-        Cost += static_cast<T *>(this)->getVectorInstrCost(
-            Instruction::ExtractElement, Ty, i);
+        Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, Ty, i);
     }
 
     return Cost;
@@ -559,8 +564,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     auto *Ty = cast<FixedVectorType>(InTy);
 
     APInt DemandedElts = APInt::getAllOnesValue(Ty->getNumElements());
-    return static_cast<T *>(this)->getScalarizationOverhead(Ty, DemandedElts,
-                                                            Insert, Extract);
+    return thisT()->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract);
   }
 
   /// Estimate the overhead of scalarizing an instructions unique
@@ -653,7 +657,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     // similarly to what getCastInstrCost() does.
     if (auto *VTy = dyn_cast<VectorType>(Ty)) {
       unsigned Num = cast<FixedVectorType>(VTy)->getNumElements();
-      unsigned Cost = static_cast<T *>(this)->getArithmeticInstrCost(
+      unsigned Cost = thisT()->getArithmeticInstrCost(
           Opcode, VTy->getScalarType(), CostKind);
       // Return the cost of multiple scalar invocation plus the cost of
       // inserting and extracting the values.
@@ -818,9 +822,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       // In other cases where the source or destination are illegal, assume
       // the operation will get scalarized.
       unsigned Num = cast<FixedVectorType>(DstVTy)->getNumElements();
-      unsigned Cost = static_cast<T *>(this)->getCastInstrCost(
-          Opcode, Dst->getScalarType(), Src->getScalarType(),
-          CostKind, I);
+      unsigned Cost = thisT()->getCastInstrCost(
+          Opcode, Dst->getScalarType(), Src->getScalarType(), CostKind, I);
 
       // Return the cost of multiple scalar invocation plus the cost of
       // inserting and extracting the values.
@@ -842,11 +845,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
   unsigned getExtractWithExtendCost(unsigned Opcode, Type *Dst,
                                     VectorType *VecTy, unsigned Index) {
-    return static_cast<T *>(this)->getVectorInstrCost(
-               Instruction::ExtractElement, VecTy, Index) +
-           static_cast<T *>(this)->getCastInstrCost(Opcode, Dst,
-                                                    VecTy->getElementType(),
-                                                    TTI::TCK_RecipThroughput);
+    return thisT()->getVectorInstrCost(Instruction::ExtractElement, VecTy,
+                                       Index) +
+           thisT()->getCastInstrCost(Opcode, Dst, VecTy->getElementType(),
+                                     TTI::TCK_RecipThroughput);
   }
 
   unsigned getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind) {
@@ -886,7 +888,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       unsigned Num = cast<FixedVectorType>(ValVTy)->getNumElements();
       if (CondTy)
         CondTy = CondTy->getScalarType();
-      unsigned Cost = static_cast<T *>(this)->getCmpSelInstrCost(
+      unsigned Cost = thisT()->getCmpSelInstrCost(
           Opcode, ValVTy->getScalarType(), CondTy, CostKind, I);
 
       // Return the cost of multiple scalar invocation plus the cost of
@@ -944,13 +946,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return Cost;
   }
 
-  unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
-                                      unsigned Factor,
-                                      ArrayRef<unsigned> Indices,
-                                      unsigned Alignment, unsigned AddressSpace,
-                                      TTI::TargetCostKind CostKind,
-                                      bool UseMaskForCond = false,
-                                      bool UseMaskForGaps = false) {
+  unsigned getInterleavedMemoryOpCost(
+      unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+      Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
+      bool UseMaskForCond = false, bool UseMaskForGaps = false) {
     auto *VT = cast<FixedVectorType>(VecTy);
 
     unsigned NumElts = VT->getNumElements();
@@ -962,17 +961,16 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     // Firstly, the cost of load/store operation.
     unsigned Cost;
     if (UseMaskForCond || UseMaskForGaps)
-      Cost = static_cast<T *>(this)->getMaskedMemoryOpCost(
-          Opcode, VecTy, Alignment, AddressSpace, CostKind);
+      Cost = thisT()->getMaskedMemoryOpCost(Opcode, VecTy, Alignment,
+                                            AddressSpace, CostKind);
     else
-      Cost = static_cast<T *>(this)->getMemoryOpCost(
-          Opcode, VecTy, MaybeAlign(Alignment), AddressSpace, CostKind);
+      Cost = thisT()->getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace,
+                                      CostKind);
 
     // Legalize the vector type, and get the legalized and unlegalized type
     // sizes.
     MVT VecTyLT = getTLI()->getTypeLegalizationCost(DL, VecTy).second;
-    unsigned VecTySize =
-        static_cast<T *>(this)->getDataLayout().getTypeStoreSize(VecTy);
+    unsigned VecTySize = thisT()->getDataLayout().getTypeStoreSize(VecTy);
     unsigned VecTyLTSize = VecTyLT.getStoreSize();
 
     // Return the ceiling of dividing A by B.
@@ -1031,14 +1029,14 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
         // Extract elements from loaded vector for each sub vector.
         for (unsigned i = 0; i < NumSubElts; i++)
-          Cost += static_cast<T *>(this)->getVectorInstrCost(
-              Instruction::ExtractElement, VT, Index + i * Factor);
+          Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, VT,
+                                              Index + i * Factor);
       }
 
       unsigned InsSubCost = 0;
       for (unsigned i = 0; i < NumSubElts; i++)
-        InsSubCost += static_cast<T *>(this)->getVectorInstrCost(
-            Instruction::InsertElement, SubVT, i);
+        InsSubCost +=
+            thisT()->getVectorInstrCost(Instruction::InsertElement, SubVT, i);
 
       Cost += Indices.size() * InsSubCost;
     } else {
@@ -1053,8 +1051,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
       unsigned ExtSubCost = 0;
       for (unsigned i = 0; i < NumSubElts; i++)
-        ExtSubCost += static_cast<T *>(this)->getVectorInstrCost(
-            Instruction::ExtractElement, SubVT, i);
+        ExtSubCost +=
+            thisT()->getVectorInstrCost(Instruction::ExtractElement, SubVT, i);
       Cost += ExtSubCost * Factor;
 
       for (unsigned i = 0; i < NumElts; i++)
@@ -1080,12 +1078,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     // vector and insert them factor times into the <24xi1> shuffled mask
     // vector.
     for (unsigned i = 0; i < NumSubElts; i++)
-      Cost += static_cast<T *>(this)->getVectorInstrCost(
-          Instruction::ExtractElement, SubVT, i);
+      Cost +=
+          thisT()->getVectorInstrCost(Instruction::ExtractElement, SubVT, i);
 
     for (unsigned i = 0; i < NumElts; i++)
-      Cost += static_cast<T *>(this)->getVectorInstrCost(
-          Instruction::InsertElement, MaskVT, i);
+      Cost +=
+          thisT()->getVectorInstrCost(Instruction::InsertElement, MaskVT, i);
 
     // The Gaps mask is invariant and created outside the loop, therefore the
     // cost of creating it is not accounted for here. However if we have both
@@ -1093,8 +1091,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     // memory access, we need to account for the cost of And-ing the two masks
     // inside the loop.
     if (UseMaskForGaps)
-      Cost += static_cast<T *>(this)->getArithmeticInstrCost(
-          BinaryOperator::And, MaskVT, CostKind);
+      Cost += thisT()->getArithmeticInstrCost(BinaryOperator::And, MaskVT,
+                                              CostKind);
 
     return Cost;
   }
@@ -1103,7 +1101,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   unsigned getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                  TTI::TargetCostKind CostKind) {
     Intrinsic::ID IID = ICA.getID();
-    auto *ConcreteTTI = static_cast<T *>(this);
 
     // Special case some scalar intrinsics.
     if (CostKind != TTI::TCK_RecipThroughput) {
@@ -1119,8 +1116,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
           return TargetTransformInfo::TCC_Basic;
         break;
       case Intrinsic::memcpy:
-        return ConcreteTTI->getMemcpyCost(ICA.getInst());
-      // TODO: other libc intrinsics.
+        return thisT()->getMemcpyCost(ICA.getInst());
+        // TODO: other libc intrinsics.
       }
       return BaseT::getIntrinsicInstrCost(ICA, CostKind);
     }
@@ -1139,14 +1136,14 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
                              : 1);
     assert((RetVF == 1 || VF == 1) && "VF > 1 and RetVF is a vector type");
     const IntrinsicInst *I = ICA.getInst();
-    const SmallVectorImpl<Value *> &Args = ICA.getArgs();
+    const SmallVectorImpl<const Value *> &Args = ICA.getArgs();
     FastMathFlags FMF = ICA.getFlags();
 
     switch (IID) {
     default: {
       // Assume that we need to scalarize this intrinsic.
       SmallVector<Type *, 4> Types;
-      for (Value *Op : Args) {
+      for (const Value *Op : Args) {
         Type *OpTy = Op->getType();
         assert(VF == 1 || !OpTy->isVectorTy());
         Types.push_back(VF == 1 ? OpTy : FixedVectorType::get(OpTy, VF));
@@ -1169,25 +1166,24 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
       IntrinsicCostAttributes Attrs(IID, RetTy, Types, FMF,
                                     ScalarizationCost, I);
-      return ConcreteTTI->getIntrinsicInstrCost(Attrs, CostKind);
+      return thisT()->getIntrinsicInstrCost(Attrs, CostKind);
     }
     case Intrinsic::masked_scatter: {
       assert(VF == 1 && "Can't vectorize types here.");
-      Value *Mask = Args[3];
+      const Value *Mask = Args[3];
       bool VarMask = !isa<Constant>(Mask);
-      unsigned Alignment = cast<ConstantInt>(Args[2])->getZExtValue();
-      return ConcreteTTI->getGatherScatterOpCost(Instruction::Store,
-                                                 Args[0]->getType(), Args[1],
-                                                 VarMask, Alignment, CostKind,
-                                                 I);
+      Align Alignment = cast<ConstantInt>(Args[2])->getAlignValue();
+      return thisT()->getGatherScatterOpCost(Instruction::Store,
+                                             Args[0]->getType(), Args[1],
+                                             VarMask, Alignment, CostKind, I);
     }
     case Intrinsic::masked_gather: {
       assert(VF == 1 && "Can't vectorize types here.");
-      Value *Mask = Args[2];
+      const Value *Mask = Args[2];
       bool VarMask = !isa<Constant>(Mask);
-      unsigned Alignment = cast<ConstantInt>(Args[1])->getZExtValue();
-      return ConcreteTTI->getGatherScatterOpCost(
-          Instruction::Load, RetTy, Args[0], VarMask, Alignment, CostKind, I);
+      Align Alignment = cast<ConstantInt>(Args[1])->getAlignValue();
+      return thisT()->getGatherScatterOpCost(Instruction::Load, RetTy, Args[0],
+                                             VarMask, Alignment, CostKind, I);
     }
     case Intrinsic::experimental_vector_reduce_add:
     case Intrinsic::experimental_vector_reduce_mul:
@@ -1207,9 +1203,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     }
     case Intrinsic::fshl:
     case Intrinsic::fshr: {
-      Value *X = Args[0];
-      Value *Y = Args[1];
-      Value *Z = Args[2];
+      const Value *X = Args[0];
+      const Value *Y = Args[1];
+      const Value *Z = Args[2];
       TTI::OperandValueProperties OpPropsX, OpPropsY, OpPropsZ, OpPropsBW;
       TTI::OperandValueKind OpKindX = TTI::getOperandInfo(X, OpPropsX);
       TTI::OperandValueKind OpKindY = TTI::getOperandInfo(Y, OpPropsY);
@@ -1220,30 +1216,27 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
       // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
       unsigned Cost = 0;
-      Cost += ConcreteTTI->getArithmeticInstrCost(BinaryOperator::Or, RetTy,
-                                                  CostKind);
-      Cost += ConcreteTTI->getArithmeticInstrCost(BinaryOperator::Sub, RetTy,
-                                                  CostKind);
-      Cost += ConcreteTTI->getArithmeticInstrCost(BinaryOperator::Shl, RetTy,
-                                                  CostKind,
-                                                  OpKindX, OpKindZ, OpPropsX);
-      Cost += ConcreteTTI->getArithmeticInstrCost(BinaryOperator::LShr, RetTy,
-                                                  CostKind,
-                                                  OpKindY, OpKindZ, OpPropsY);
+      Cost +=
+          thisT()->getArithmeticInstrCost(BinaryOperator::Or, RetTy, CostKind);
+      Cost +=
+          thisT()->getArithmeticInstrCost(BinaryOperator::Sub, RetTy, CostKind);
+      Cost += thisT()->getArithmeticInstrCost(
+          BinaryOperator::Shl, RetTy, CostKind, OpKindX, OpKindZ, OpPropsX);
+      Cost += thisT()->getArithmeticInstrCost(
+          BinaryOperator::LShr, RetTy, CostKind, OpKindY, OpKindZ, OpPropsY);
       // Non-constant shift amounts requires a modulo.
       if (OpKindZ != TTI::OK_UniformConstantValue &&
           OpKindZ != TTI::OK_NonUniformConstantValue)
-        Cost += ConcreteTTI->getArithmeticInstrCost(BinaryOperator::URem, RetTy,
-                                                    CostKind,
-                                                    OpKindZ, OpKindBW, OpPropsZ,
-                                                    OpPropsBW);
+        Cost += thisT()->getArithmeticInstrCost(BinaryOperator::URem, RetTy,
+                                                CostKind, OpKindZ, OpKindBW,
+                                                OpPropsZ, OpPropsBW);
       // For non-rotates (X != Y) we must add shift-by-zero handling costs.
       if (X != Y) {
         Type *CondTy = RetTy->getWithNewBitWidth(1);
-        Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy,
-                                                CondTy, CostKind);
-        Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::Select, RetTy,
-                                                CondTy, CostKind);
+        Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy, CondTy,
+                                            CostKind);
+        Cost += thisT()->getCmpSelInstrCost(BinaryOperator::Select, RetTy,
+                                            CondTy, CostKind);
       }
       return Cost;
     }
@@ -1256,9 +1249,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   /// based on types.
   unsigned getTypeBasedIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                           TTI::TargetCostKind CostKind) {
-
-    auto *ConcreteTTI = static_cast<T *>(this);
-
     Intrinsic::ID IID = ICA.getID();
     Type *RetTy = ICA.getReturnType();
     const SmallVectorImpl<Type *> &Tys = ICA.getArgTypes();
@@ -1300,7 +1290,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
       IntrinsicCostAttributes ScalarAttrs(IID, ScalarRetTy, ScalarTys, FMF);
       unsigned ScalarCost =
-          ConcreteTTI->getIntrinsicInstrCost(ScalarAttrs, CostKind);
+          thisT()->getIntrinsicInstrCost(ScalarAttrs, CostKind);
 
       return ScalarCalls * ScalarCost + ScalarizationCost;
     }
@@ -1387,53 +1377,59 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     case Intrinsic::lifetime_end:
     case Intrinsic::sideeffect:
       return 0;
-    case Intrinsic::masked_store:
-      return ConcreteTTI->getMaskedMemoryOpCost(Instruction::Store, Tys[0], 0,
-                                                0, CostKind);
-    case Intrinsic::masked_load:
-      return ConcreteTTI->getMaskedMemoryOpCost(Instruction::Load, RetTy, 0, 0,
-                                                CostKind);
+    case Intrinsic::masked_store: {
+      Type *Ty = Tys[0];
+      Align TyAlign = thisT()->DL.getABITypeAlign(Ty);
+      return thisT()->getMaskedMemoryOpCost(Instruction::Store, Ty, TyAlign, 0,
+                                            CostKind);
+    }
+    case Intrinsic::masked_load: {
+      Type *Ty = RetTy;
+      Align TyAlign = thisT()->DL.getABITypeAlign(Ty);
+      return thisT()->getMaskedMemoryOpCost(Instruction::Load, Ty, TyAlign, 0,
+                                            CostKind);
+    }
     case Intrinsic::experimental_vector_reduce_add:
-      return ConcreteTTI->getArithmeticReductionCost(Instruction::Add, VecOpTy,
-                                                     /*IsPairwiseForm=*/false,
-                                                     CostKind);
+      return thisT()->getArithmeticReductionCost(Instruction::Add, VecOpTy,
+                                                 /*IsPairwiseForm=*/false,
+                                                 CostKind);
     case Intrinsic::experimental_vector_reduce_mul:
-      return ConcreteTTI->getArithmeticReductionCost(Instruction::Mul, VecOpTy,
-                                                     /*IsPairwiseForm=*/false,
-                                                     CostKind);
+      return thisT()->getArithmeticReductionCost(Instruction::Mul, VecOpTy,
+                                                 /*IsPairwiseForm=*/false,
+                                                 CostKind);
     case Intrinsic::experimental_vector_reduce_and:
-      return ConcreteTTI->getArithmeticReductionCost(Instruction::And, VecOpTy,
-                                                     /*IsPairwiseForm=*/false,
-                                                     CostKind);
+      return thisT()->getArithmeticReductionCost(Instruction::And, VecOpTy,
+                                                 /*IsPairwiseForm=*/false,
+                                                 CostKind);
     case Intrinsic::experimental_vector_reduce_or:
-      return ConcreteTTI->getArithmeticReductionCost(Instruction::Or, VecOpTy,
-                                                     /*IsPairwiseForm=*/false,
-                                                     CostKind);
+      return thisT()->getArithmeticReductionCost(Instruction::Or, VecOpTy,
+                                                 /*IsPairwiseForm=*/false,
+                                                 CostKind);
     case Intrinsic::experimental_vector_reduce_xor:
-      return ConcreteTTI->getArithmeticReductionCost(Instruction::Xor, VecOpTy,
-                                                     /*IsPairwiseForm=*/false,
-                                                     CostKind);
+      return thisT()->getArithmeticReductionCost(Instruction::Xor, VecOpTy,
+                                                 /*IsPairwiseForm=*/false,
+                                                 CostKind);
     case Intrinsic::experimental_vector_reduce_v2_fadd:
       // FIXME: Add new flag for cost of strict reductions.
-      return ConcreteTTI->getArithmeticReductionCost(
-          Instruction::FAdd, VecOpTy,
-          /*IsPairwiseForm=*/false, CostKind);
+      return thisT()->getArithmeticReductionCost(Instruction::FAdd, VecOpTy,
+                                                 /*IsPairwiseForm=*/false,
+                                                 CostKind);
     case Intrinsic::experimental_vector_reduce_v2_fmul:
       // FIXME: Add new flag for cost of strict reductions.
-      return ConcreteTTI->getArithmeticReductionCost(
-          Instruction::FMul, VecOpTy,
-          /*IsPairwiseForm=*/false, CostKind);
+      return thisT()->getArithmeticReductionCost(Instruction::FMul, VecOpTy,
+                                                 /*IsPairwiseForm=*/false,
+                                                 CostKind);
     case Intrinsic::experimental_vector_reduce_smax:
     case Intrinsic::experimental_vector_reduce_smin:
     case Intrinsic::experimental_vector_reduce_fmax:
     case Intrinsic::experimental_vector_reduce_fmin:
-      return ConcreteTTI->getMinMaxReductionCost(
+      return thisT()->getMinMaxReductionCost(
           VecOpTy, cast<VectorType>(CmpInst::makeCmpResultType(VecOpTy)),
           /*IsPairwiseForm=*/false,
           /*IsUnsigned=*/false, CostKind);
     case Intrinsic::experimental_vector_reduce_umax:
     case Intrinsic::experimental_vector_reduce_umin:
-      return ConcreteTTI->getMinMaxReductionCost(
+      return thisT()->getMinMaxReductionCost(
           VecOpTy, cast<VectorType>(CmpInst::makeCmpResultType(VecOpTy)),
           /*IsPairwiseForm=*/false,
           /*IsUnsigned=*/true, CostKind);
@@ -1451,11 +1447,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       unsigned Cost = 0;
       IntrinsicCostAttributes Attrs(OverflowOp, OpTy, {RetTy, RetTy}, FMF,
                                     ScalarizationCostPassed);
-      Cost += ConcreteTTI->getIntrinsicInstrCost(Attrs, CostKind);
-      Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy,
+      Cost += thisT()->getIntrinsicInstrCost(Attrs, CostKind);
+      Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy, CondTy,
+                                          CostKind);
+      Cost += 2 * thisT()->getCmpSelInstrCost(BinaryOperator::Select, RetTy,
                                               CondTy, CostKind);
-      Cost += 2 * ConcreteTTI->getCmpSelInstrCost(BinaryOperator::Select, RetTy,
-                                                  CondTy, CostKind);
       return Cost;
     }
     case Intrinsic::uadd_sat:
@@ -1470,9 +1466,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       unsigned Cost = 0;
       IntrinsicCostAttributes Attrs(OverflowOp, OpTy, {RetTy, RetTy}, FMF,
                                     ScalarizationCostPassed);
-      Cost += ConcreteTTI->getIntrinsicInstrCost(Attrs, CostKind);
-      Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::Select, RetTy,
-                                              CondTy, CostKind);
+      Cost += thisT()->getIntrinsicInstrCost(Attrs, CostKind);
+      Cost += thisT()->getCmpSelInstrCost(BinaryOperator::Select, RetTy, CondTy,
+                                          CostKind);
       return Cost;
     }
     case Intrinsic::smul_fix:
@@ -1484,22 +1480,18 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
           IID == Intrinsic::smul_fix ? Instruction::SExt : Instruction::ZExt;
 
       unsigned Cost = 0;
-      Cost += 2 * ConcreteTTI->getCastInstrCost(ExtOp, ExtTy, RetTy, CostKind);
-      Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::Mul, ExtTy,
-                                                  CostKind);
+      Cost += 2 * thisT()->getCastInstrCost(ExtOp, ExtTy, RetTy, CostKind);
       Cost +=
-          2 * ConcreteTTI->getCastInstrCost(Instruction::Trunc, RetTy, ExtTy,
+          thisT()->getArithmeticInstrCost(Instruction::Mul, ExtTy, CostKind);
+      Cost += 2 * thisT()->getCastInstrCost(Instruction::Trunc, RetTy, ExtTy,
                                             CostKind);
-      Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::LShr, RetTy,
-                                                  CostKind,
-                                                  TTI::OK_AnyValue,
-                                                  TTI::OK_UniformConstantValue);
-      Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::Shl, RetTy,
-                                                  CostKind,
-                                                  TTI::OK_AnyValue,
-                                                  TTI::OK_UniformConstantValue);
-      Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::Or, RetTy,
-                                                  CostKind);
+      Cost += thisT()->getArithmeticInstrCost(Instruction::LShr, RetTy,
+                                              CostKind, TTI::OK_AnyValue,
+                                              TTI::OK_UniformConstantValue);
+      Cost += thisT()->getArithmeticInstrCost(Instruction::Shl, RetTy, CostKind,
+                                              TTI::OK_AnyValue,
+                                              TTI::OK_UniformConstantValue);
+      Cost += thisT()->getArithmeticInstrCost(Instruction::Or, RetTy, CostKind);
       return Cost;
     }
     case Intrinsic::sadd_with_overflow:
@@ -1519,13 +1511,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       //   Sub:
       //   Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
       unsigned Cost = 0;
-      Cost += ConcreteTTI->getArithmeticInstrCost(Opcode, SumTy, CostKind);
-      Cost += 3 * ConcreteTTI->getCmpSelInstrCost(BinaryOperator::ICmp, SumTy,
-                                                  OverflowTy, CostKind);
-      Cost += 2 * ConcreteTTI->getCmpSelInstrCost(
-                      BinaryOperator::ICmp, OverflowTy, OverflowTy, CostKind);
-      Cost +=
-          ConcreteTTI->getArithmeticInstrCost(BinaryOperator::And, OverflowTy,
+      Cost += thisT()->getArithmeticInstrCost(Opcode, SumTy, CostKind);
+      Cost += 3 * thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, SumTy,
+                                              OverflowTy, CostKind);
+      Cost += 2 * thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, OverflowTy,
+                                              OverflowTy, CostKind);
+      Cost += thisT()->getArithmeticInstrCost(BinaryOperator::And, OverflowTy,
                                               CostKind);
       return Cost;
     }
@@ -1538,9 +1529,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
                             : BinaryOperator::Sub;
 
       unsigned Cost = 0;
-      Cost += ConcreteTTI->getArithmeticInstrCost(Opcode, SumTy, CostKind);
-      Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::ICmp, SumTy,
-                                              OverflowTy, CostKind);
+      Cost += thisT()->getArithmeticInstrCost(Opcode, SumTy, CostKind);
+      Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, SumTy,
+                                          OverflowTy, CostKind);
       return Cost;
     }
     case Intrinsic::smul_with_overflow:
@@ -1554,24 +1545,22 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
           IID == Intrinsic::smul_fix ? Instruction::SExt : Instruction::ZExt;
 
       unsigned Cost = 0;
-      Cost += 2 * ConcreteTTI->getCastInstrCost(ExtOp, ExtTy, MulTy, CostKind);
-      Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::Mul, ExtTy,
-                                                  CostKind);
+      Cost += 2 * thisT()->getCastInstrCost(ExtOp, ExtTy, MulTy, CostKind);
       Cost +=
-          2 * ConcreteTTI->getCastInstrCost(Instruction::Trunc, MulTy, ExtTy,
+          thisT()->getArithmeticInstrCost(Instruction::Mul, ExtTy, CostKind);
+      Cost += 2 * thisT()->getCastInstrCost(Instruction::Trunc, MulTy, ExtTy,
                                             CostKind);
-      Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::LShr, MulTy,
-                                                  CostKind,
-                                                  TTI::OK_AnyValue,
-                                                  TTI::OK_UniformConstantValue);
+      Cost += thisT()->getArithmeticInstrCost(Instruction::LShr, MulTy,
+                                              CostKind, TTI::OK_AnyValue,
+                                              TTI::OK_UniformConstantValue);
 
       if (IID == Intrinsic::smul_with_overflow)
-        Cost += ConcreteTTI->getArithmeticInstrCost(
-            Instruction::AShr, MulTy, CostKind, TTI::OK_AnyValue,
-            TTI::OK_UniformConstantValue);
+        Cost += thisT()->getArithmeticInstrCost(Instruction::AShr, MulTy,
+                                                CostKind, TTI::OK_AnyValue,
+                                                TTI::OK_UniformConstantValue);
 
-      Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::ICmp, MulTy,
-                                              OverflowTy, CostKind);
+      Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, MulTy,
+                                          OverflowTy, CostKind);
       return Cost;
     }
     case Intrinsic::ctpop:
@@ -1628,17 +1617,17 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     // If we can't lower fmuladd into an FMA estimate the cost as a floating
     // point mul followed by an add.
     if (IID == Intrinsic::fmuladd)
-      return ConcreteTTI->getArithmeticInstrCost(BinaryOperator::FMul, RetTy,
-                                                 CostKind) +
-             ConcreteTTI->getArithmeticInstrCost(BinaryOperator::FAdd, RetTy,
-                                                 CostKind);
+      return thisT()->getArithmeticInstrCost(BinaryOperator::FMul, RetTy,
+                                             CostKind) +
+             thisT()->getArithmeticInstrCost(BinaryOperator::FAdd, RetTy,
+                                             CostKind);
     if (IID == Intrinsic::experimental_constrained_fmuladd) {
       IntrinsicCostAttributes FMulAttrs(
         Intrinsic::experimental_constrained_fmul, RetTy, Tys);
       IntrinsicCostAttributes FAddAttrs(
         Intrinsic::experimental_constrained_fadd, RetTy, Tys);
-      return ConcreteTTI->getIntrinsicInstrCost(FMulAttrs, CostKind) +
-             ConcreteTTI->getIntrinsicInstrCost(FAddAttrs, CostKind);
+      return thisT()->getIntrinsicInstrCost(FMulAttrs, CostKind) +
+             thisT()->getIntrinsicInstrCost(FAddAttrs, CostKind);
     }
 
     // Else, assume that we need to scalarize this intrinsic. For math builtins
@@ -1657,7 +1646,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
         ScalarTys.push_back(Ty);
       }
       IntrinsicCostAttributes Attrs(IID, RetTy->getScalarType(), ScalarTys, FMF);
-      unsigned ScalarCost = ConcreteTTI->getIntrinsicInstrCost(Attrs, CostKind);
+      unsigned ScalarCost = thisT()->getIntrinsicInstrCost(Attrs, CostKind);
       for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) {
         if (auto *VTy = dyn_cast<VectorType>(Tys[i])) {
           if (!ICA.skipScalarizationCost())
@@ -1742,9 +1731,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     unsigned NumReduxLevels = Log2_32(NumVecElts);
     unsigned ArithCost = 0;
     unsigned ShuffleCost = 0;
-    auto *ConcreteTTI = static_cast<T *>(this);
     std::pair<unsigned, MVT> LT =
-        ConcreteTTI->getTLI()->getTypeLegalizationCost(DL, Ty);
+        thisT()->getTLI()->getTypeLegalizationCost(DL, Ty);
     unsigned LongVectorCount = 0;
     unsigned MVTLen =
         LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
@@ -1752,10 +1740,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       NumVecElts /= 2;
       VectorType *SubTy = FixedVectorType::get(ScalarTy, NumVecElts);
       // Assume the pairwise shuffles add a cost.
-      ShuffleCost += (IsPairwise + 1) *
-                     ConcreteTTI->getShuffleCost(TTI::SK_ExtractSubvector, Ty,
-                                                 NumVecElts, SubTy);
-      ArithCost += ConcreteTTI->getArithmeticInstrCost(Opcode, SubTy, CostKind);
+      ShuffleCost +=
+          (IsPairwise + 1) * thisT()->getShuffleCost(TTI::SK_ExtractSubvector,
+                                                     Ty, NumVecElts, SubTy);
+      ArithCost += thisT()->getArithmeticInstrCost(Opcode, SubTy, CostKind);
       Ty = SubTy;
       ++LongVectorCount;
     }
@@ -1774,12 +1762,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     if (IsPairwise && NumReduxLevels >= 1)
       NumShuffles += NumReduxLevels - 1;
     ShuffleCost += NumShuffles *
-                   ConcreteTTI->getShuffleCost(TTI::SK_PermuteSingleSrc, Ty,
-                                               0, Ty);
-    ArithCost += NumReduxLevels *
-                 ConcreteTTI->getArithmeticInstrCost(Opcode, Ty);
+                   thisT()->getShuffleCost(TTI::SK_PermuteSingleSrc, Ty, 0, Ty);
+    ArithCost += NumReduxLevels * thisT()->getArithmeticInstrCost(Opcode, Ty);
     return ShuffleCost + ArithCost +
-           ConcreteTTI->getVectorInstrCost(Instruction::ExtractElement, Ty, 0);
+           thisT()->getVectorInstrCost(Instruction::ExtractElement, Ty, 0);
   }
 
   /// Try to calculate op costs for min/max reduction operations.
@@ -1801,9 +1787,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     }
     unsigned MinMaxCost = 0;
     unsigned ShuffleCost = 0;
-    auto *ConcreteTTI = static_cast<T *>(this);
     std::pair<unsigned, MVT> LT =
-        ConcreteTTI->getTLI()->getTypeLegalizationCost(DL, Ty);
+        thisT()->getTLI()->getTypeLegalizationCost(DL, Ty);
     unsigned LongVectorCount = 0;
     unsigned MVTLen =
         LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
@@ -1813,13 +1798,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       CondTy = FixedVectorType::get(ScalarCondTy, NumVecElts);
 
       // Assume the pairwise shuffles add a cost.
-      ShuffleCost += (IsPairwise + 1) *
-                     ConcreteTTI->getShuffleCost(TTI::SK_ExtractSubvector, Ty,
-                                                 NumVecElts, SubTy);
+      ShuffleCost +=
+          (IsPairwise + 1) * thisT()->getShuffleCost(TTI::SK_ExtractSubvector,
+                                                     Ty, NumVecElts, SubTy);
       MinMaxCost +=
-          ConcreteTTI->getCmpSelInstrCost(CmpOpcode, SubTy, CondTy, CostKind) +
-          ConcreteTTI->getCmpSelInstrCost(Instruction::Select, SubTy, CondTy,
-                                          CostKind);
+          thisT()->getCmpSelInstrCost(CmpOpcode, SubTy, CondTy, CostKind) +
+          thisT()->getCmpSelInstrCost(Instruction::Select, SubTy, CondTy,
+                                      CostKind);
       Ty = SubTy;
       ++LongVectorCount;
     }
@@ -1838,17 +1823,16 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     if (IsPairwise && NumReduxLevels >= 1)
       NumShuffles += NumReduxLevels - 1;
     ShuffleCost += NumShuffles *
-                   ConcreteTTI->getShuffleCost(TTI::SK_PermuteSingleSrc, Ty,
-                                               0, Ty);
+                   thisT()->getShuffleCost(TTI::SK_PermuteSingleSrc, Ty, 0, Ty);
     MinMaxCost +=
         NumReduxLevels *
-        (ConcreteTTI->getCmpSelInstrCost(CmpOpcode, Ty, CondTy, CostKind) +
-         ConcreteTTI->getCmpSelInstrCost(Instruction::Select, Ty, CondTy,
-                                         CostKind));
+        (thisT()->getCmpSelInstrCost(CmpOpcode, Ty, CondTy, CostKind) +
+         thisT()->getCmpSelInstrCost(Instruction::Select, Ty, CondTy,
+                                     CostKind));
     // The last min/max should be in vector registers and we counted it above.
     // So just need a single extractelement.
     return ShuffleCost + MinMaxCost +
-           ConcreteTTI->getVectorInstrCost(Instruction::ExtractElement, Ty, 0);
+           thisT()->getVectorInstrCost(Instruction::ExtractElement, Ty, 0);
   }
 
   unsigned getVectorSplitCost() { return 1; }
diff --git a/llvm/include/llvm/CodeGen/CallingConvLower.h b/llvm/include/llvm/CodeGen/CallingConvLower.h
index cc3a75c897d3d..8ebe788ac360a 100644
--- a/llvm/include/llvm/CodeGen/CallingConvLower.h
+++ b/llvm/include/llvm/CodeGen/CallingConvLower.h
@@ -221,9 +221,7 @@ class CCState {
   // ByValRegs[1] describes how "%t" is stored (Begin == r3, End == r4).
   //
   // In case of 8 bytes stack alignment,
-  // ByValRegs may also contain information about wasted registers.
   // In function shown above, r3 would be wasted according to AAPCS rules.
-  // And in that case ByValRegs[1].Waste would be "true".
   // ByValRegs vector size still would be 2,
   // while "%t" goes to the stack: it wouldn't be described in ByValRegs.
   //
@@ -233,19 +231,13 @@ class CCState {
   // 3. Argument analysis (LowerFormatArguments, for example). After
   // some byval argument was analyzed, InRegsParamsProcessed is increased.
   struct ByValInfo {
-    ByValInfo(unsigned B, unsigned E, bool IsWaste = false) :
-      Begin(B), End(E), Waste(IsWaste) {}
+    ByValInfo(unsigned B, unsigned E) : Begin(B), End(E) {}
+
     // First register allocated for current parameter.
     unsigned Begin;
 
     // First after last register allocated for current parameter.
     unsigned End;
-
-    // Means that current range of registers doesn't belong to any
-    // parameters. It was wasted due to stack alignment rules.
-    // For more information see:
-    // AAPCS, 5.5 Parameter Passing, Stage C, C.3.
-    bool Waste;
   };
   SmallVector<ByValInfo, 4 > ByValRegs;
 
diff --git a/llvm/include/llvm/CodeGen/DebugHandlerBase.h b/llvm/include/llvm/CodeGen/DebugHandlerBase.h
index 06a3c85a35f1a..4ff0fdea36ae2 100644
--- a/llvm/include/llvm/CodeGen/DebugHandlerBase.h
+++ b/llvm/include/llvm/CodeGen/DebugHandlerBase.h
@@ -118,6 +118,9 @@ class DebugHandlerBase : public AsmPrinterHandler {
   void beginFunction(const MachineFunction *MF) override;
   void endFunction(const MachineFunction *MF) override;
 
+  void beginBasicBlock(const MachineBasicBlock &MBB) override;
+  void endBasicBlock(const MachineBasicBlock &MBB) override;
+
   /// Return Label preceding the instruction.
   MCSymbol *getLabelBeforeInsn(const MachineInstr *MI);
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
index 88a1837665aab..4d60dffb91dbd 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
@@ -290,6 +290,8 @@ class CallLowering {
     return false;
   }
 
+  virtual bool fallBackToDAGISel(const Function &F) const { return false; }
+
   /// This hook must be implemented to lower the incoming (formal)
   /// arguments, described by \p VRegs, for GlobalISel. Each argument
   /// must end up in the related virtual registers described by \p VRegs.
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index cf3afd8aeabc0..534f988c5e96c 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -58,6 +58,7 @@ enum NodeType {
   /// of the extension
   AssertSext,
   AssertZext,
+  AssertAlign,
 
   /// Various leaf nodes.
   BasicBlock,
@@ -447,11 +448,11 @@ enum NodeType {
   FCANONICALIZE,
 
   /// BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector
-  /// with the specified, possibly variable, elements.  The number of elements
-  /// is required to be a power of two. The types of the operands must all be
-  /// the same and must match the vector element type, except that integer types
-  /// are allowed to be larger than the element type, in which case the operands
-  /// are implicitly truncated.
+  /// with the specified, possibly variable, elements. The types of the
+  /// operands must match the vector element type, except that integer types
+  /// are allowed to be larger than the element type, in which case the
+  /// operands are implicitly truncated. The types of the operands must all
+  /// be the same.
   BUILD_VECTOR,
 
   /// INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element
@@ -869,7 +870,7 @@ enum NodeType {
   /// SDOperands.
   INLINEASM,
 
-  /// INLINEASM_BR - Terminator version of inline asm. Used by asm-goto.
+  /// INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
   INLINEASM_BR,
 
   /// EH_LABEL - Represents a label in mid basic block used to track
diff --git a/llvm/include/llvm/CodeGen/LiveIntervals.h b/llvm/include/llvm/CodeGen/LiveIntervals.h
index 1b921eea782d7..945a40829714b 100644
--- a/llvm/include/llvm/CodeGen/LiveIntervals.h
+++ b/llvm/include/llvm/CodeGen/LiveIntervals.h
@@ -22,7 +22,6 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/IndexedMap.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/LiveInterval.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -40,6 +39,7 @@ namespace llvm {
 
 extern cl::opt<bool> UseSegmentSetForPhysRegs;
 
+class AAResults;
 class BitVector;
 class LiveIntervalCalc;
 class MachineBlockFrequencyInfo;
@@ -56,7 +56,7 @@ class VirtRegMap;
     MachineRegisterInfo* MRI;
     const TargetRegisterInfo* TRI;
     const TargetInstrInfo* TII;
-    AliasAnalysis *AA;
+    AAResults *AA;
     SlotIndexes* Indexes;
     MachineDominatorTree *DomTree = nullptr;
     LiveIntervalCalc *LICalc = nullptr;
@@ -212,7 +212,7 @@ class VirtRegMap;
       return Indexes;
     }
 
-    AliasAnalysis *getAliasAnalysis() const {
+    AAResults *getAliasAnalysis() const {
       return AA;
     }
 
diff --git a/llvm/include/llvm/CodeGen/LiveRangeEdit.h b/llvm/include/llvm/CodeGen/LiveRangeEdit.h
index a0c5ecc3a388b..3c4273130ab2b 100644
--- a/llvm/include/llvm/CodeGen/LiveRangeEdit.h
+++ b/llvm/include/llvm/CodeGen/LiveRangeEdit.h
@@ -22,7 +22,6 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/LiveInterval.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -33,6 +32,7 @@
 
 namespace llvm {
 
+class AAResults;
 class LiveIntervals;
 class MachineBlockFrequencyInfo;
 class MachineInstr;
@@ -68,7 +68,7 @@ class LiveRangeEdit : private MachineRegisterInfo::Delegate {
 
 private:
   LiveInterval *Parent;
-  SmallVectorImpl<unsigned> &NewRegs;
+  SmallVectorImpl<Register> &NewRegs;
   MachineRegisterInfo &MRI;
   LiveIntervals &LIS;
   VirtRegMap *VRM;
@@ -94,7 +94,7 @@ class LiveRangeEdit : private MachineRegisterInfo::Delegate {
   SmallPtrSet<const VNInfo *, 4> Rematted;
 
   /// scanRemattable - Identify the Parent values that may rematerialize.
-  void scanRemattable(AliasAnalysis *aa);
+  void scanRemattable(AAResults *aa);
 
   /// allUsesAvailableAt - Return true if all registers used by OrigMI at
   /// OrigIdx are also available with the same value at UseIdx.
@@ -110,7 +110,7 @@ class LiveRangeEdit : private MachineRegisterInfo::Delegate {
 
   /// Helper for eliminateDeadDefs.
   void eliminateDeadDef(MachineInstr *MI, ToShrinkSet &ToShrink,
-                        AliasAnalysis *AA);
+                        AAResults *AA);
 
   /// MachineRegisterInfo callback to notify when new virtual
   /// registers are created.
@@ -121,7 +121,7 @@ class LiveRangeEdit : private MachineRegisterInfo::Delegate {
   bool useIsKill(const LiveInterval &LI, const MachineOperand &MO) const;
 
   /// Create a new empty interval based on OldReg.
-  LiveInterval &createEmptyIntervalFrom(unsigned OldReg, bool createSubRanges);
+  LiveInterval &createEmptyIntervalFrom(Register OldReg, bool createSubRanges);
 
 public:
   /// Create a LiveRangeEdit for breaking down parent into smaller pieces.
@@ -135,7 +135,7 @@ class LiveRangeEdit : private MachineRegisterInfo::Delegate {
   ///            be done.  This could be the case if called before Regalloc.
   /// @param deadRemats The collection of all the instructions defining an
   ///                   original reg and are dead after remat.
-  LiveRangeEdit(LiveInterval *parent, SmallVectorImpl<unsigned> &newRegs,
+  LiveRangeEdit(LiveInterval *parent, SmallVectorImpl<Register> &newRegs,
                 MachineFunction &MF, LiveIntervals &lis, VirtRegMap *vrm,
                 Delegate *delegate = nullptr,
                 SmallPtrSet<MachineInstr *, 32> *deadRemats = nullptr)
@@ -152,15 +152,15 @@ class LiveRangeEdit : private MachineRegisterInfo::Delegate {
     return *Parent;
   }
 
-  unsigned getReg() const { return getParent().reg; }
+  Register getReg() const { return getParent().reg; }
 
   /// Iterator for accessing the new registers added by this edit.
-  using iterator = SmallVectorImpl<unsigned>::const_iterator;
+  using iterator = SmallVectorImpl<Register>::const_iterator;
   iterator begin() const { return NewRegs.begin() + FirstNew; }
   iterator end() const { return NewRegs.end(); }
   unsigned size() const { return NewRegs.size() - FirstNew; }
   bool empty() const { return size() == 0; }
-  unsigned get(unsigned idx) const { return NewRegs[idx + FirstNew]; }
+  Register get(unsigned idx) const { return NewRegs[idx + FirstNew]; }
 
   /// pop_back - It allows LiveRangeEdit users to drop new registers.
   /// The context is when an original def instruction of a register is
@@ -172,12 +172,12 @@ class LiveRangeEdit : private MachineRegisterInfo::Delegate {
   /// we want to drop it from the NewRegs set.
   void pop_back() { NewRegs.pop_back(); }
 
-  ArrayRef<unsigned> regs() const {
+  ArrayRef<Register> regs() const {
     return makeArrayRef(NewRegs).slice(FirstNew);
   }
 
   /// createFrom - Create a new virtual register based on OldReg.
-  unsigned createFrom(unsigned OldReg);
+  Register createFrom(Register OldReg);
 
   /// create - Create a new register with the same class and original slot as
   /// parent.
@@ -185,17 +185,17 @@ class LiveRangeEdit : private MachineRegisterInfo::Delegate {
     return createEmptyIntervalFrom(getReg(), true);
   }
 
-  unsigned create() { return createFrom(getReg()); }
+  Register create() { return createFrom(getReg()); }
 
   /// anyRematerializable - Return true if any parent values may be
   /// rematerializable.
   /// This function must be called before any rematerialization is attempted.
-  bool anyRematerializable(AliasAnalysis *);
+  bool anyRematerializable(AAResults *);
 
   /// checkRematerializable - Manually add VNI to the list of rematerializable
   /// values if DefMI may be rematerializable.
   bool checkRematerializable(VNInfo *VNI, const MachineInstr *DefMI,
-                             AliasAnalysis *);
+                             AAResults *);
 
   /// Remat - Information needed to rematerialize at a specific location.
   struct Remat {
@@ -234,7 +234,7 @@ class LiveRangeEdit : private MachineRegisterInfo::Delegate {
 
   /// eraseVirtReg - Notify the delegate that Reg is no longer in use, and try
   /// to erase it from LIS.
-  void eraseVirtReg(unsigned Reg);
+  void eraseVirtReg(Register Reg);
 
   /// eliminateDeadDefs - Try to delete machine instructions that are now dead
   /// (allDefsAreDead returns true). This may cause live intervals to be trimmed
@@ -243,8 +243,8 @@ class LiveRangeEdit : private MachineRegisterInfo::Delegate {
   /// allocator.  These registers should not be split into new intervals
   /// as currently those new intervals are not guaranteed to spill.
   void eliminateDeadDefs(SmallVectorImpl<MachineInstr *> &Dead,
-                         ArrayRef<unsigned> RegsBeingSpilled = None,
-                         AliasAnalysis *AA = nullptr);
+                         ArrayRef<Register> RegsBeingSpilled = None,
+                         AAResults *AA = nullptr);
 
   /// calculateRegClassAndHint - Recompute register class and hint for each new
   /// register.
diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h
index c7680c00175ad..b69f6584fe6c2 100644
--- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h
+++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h
@@ -143,10 +143,6 @@ class MachineBasicBlock
   /// branch.
   bool AddressTaken = false;
 
-  /// Indicate that this basic block needs its symbol be emitted regardless of
-  /// whether the flow just falls-through to it.
-  bool LabelMustBeEmitted = false;
-
   /// Indicate that this basic block is the entry block of an EH scope, i.e.,
   /// the block that used to have a catchpad or cleanuppad instruction in the
   /// LLVM IR.
@@ -167,11 +163,8 @@ class MachineBasicBlock
   // Indicate that this basic block ends a section.
   bool IsEndSection = false;
 
-  /// Default target of the callbr of a basic block.
-  bool InlineAsmBrDefaultTarget = false;
-
-  /// List of indirect targets of the callbr of a basic block.
-  SmallPtrSet<const MachineBasicBlock*, 4> InlineAsmBrIndirectTargets;
+  /// Indicate that this basic block is the indirect dest of an INLINEASM_BR.
+  bool IsInlineAsmBrIndirectTarget = false;
 
   /// since getSymbol is a relatively heavy-weight operation, the symbol
   /// is only computed once and is cached.
@@ -209,13 +202,6 @@ class MachineBasicBlock
   /// branch.
   void setHasAddressTaken() { AddressTaken = true; }
 
-  /// Test whether this block must have its label emitted.
-  bool hasLabelMustBeEmitted() const { return LabelMustBeEmitted; }
-
-  /// Set this block to reflect that, regardless how we flow to it, we need
-  /// its label be emitted.
-  void setLabelMustBeEmitted() { LabelMustBeEmitted = true; }
-
   /// Return the MachineFunction containing this basic block.
   const MachineFunction *getParent() const { return xParent; }
   MachineFunction *getParent() { return xParent; }
@@ -468,34 +454,28 @@ class MachineBasicBlock
   /// Returns the section ID of this basic block.
   MBBSectionID getSectionID() const { return SectionID; }
 
+  /// Returns the unique section ID number of this basic block.
+  unsigned getSectionIDNum() const {
+    return ((unsigned)MBBSectionID::SectionType::Cold) -
+           ((unsigned)SectionID.Type) + SectionID.Number;
+  }
+
   /// Sets the section ID for this basic block.
   void setSectionID(MBBSectionID V) { SectionID = V; }
 
+  /// Returns true if this block may have an INLINEASM_BR (overestimate, by
+  /// checking if any of the successors are indirect targets of any inlineasm_br
+  /// in the function).
+  bool mayHaveInlineAsmBr() const;
+
   /// Returns true if this is the indirect dest of an INLINEASM_BR.
-  bool isInlineAsmBrIndirectTarget(const MachineBasicBlock *Tgt) const {
-    return InlineAsmBrIndirectTargets.count(Tgt);
+  bool isInlineAsmBrIndirectTarget() const {
+    return IsInlineAsmBrIndirectTarget;
   }
 
   /// Indicates if this is the indirect dest of an INLINEASM_BR.
-  void addInlineAsmBrIndirectTarget(const MachineBasicBlock *Tgt) {
-    InlineAsmBrIndirectTargets.insert(Tgt);
-  }
-
-  /// Transfers indirect targets to INLINEASM_BR's copy block.
-  void transferInlineAsmBrIndirectTargets(MachineBasicBlock *CopyBB) {
-    for (auto *Target : InlineAsmBrIndirectTargets)
-      CopyBB->addInlineAsmBrIndirectTarget(Target);
-    return InlineAsmBrIndirectTargets.clear();
-  }
-
-  /// Returns true if this is the default dest of an INLINEASM_BR.
-  bool isInlineAsmBrDefaultTarget() const {
-    return InlineAsmBrDefaultTarget;
-  }
-
-  /// Indicates if this is the default deft of an INLINEASM_BR.
-  void setInlineAsmBrDefaultTarget() {
-    InlineAsmBrDefaultTarget = true;
+  void setIsInlineAsmBrIndirectTarget(bool V = true) {
+    IsInlineAsmBrIndirectTarget = V;
   }
 
   /// Returns true if it is legal to hoist instructions into this block.
diff --git a/llvm/include/llvm/CodeGen/MachineDominators.h b/llvm/include/llvm/CodeGen/MachineDominators.h
index 9d31232c9b95b..cf3af4d38223b 100644
--- a/llvm/include/llvm/CodeGen/MachineDominators.h
+++ b/llvm/include/llvm/CodeGen/MachineDominators.h
@@ -93,15 +93,6 @@ class MachineDominatorTree : public MachineFunctionPass {
 
   void getAnalysisUsage(AnalysisUsage &AU) const override;
 
-  /// getRoots -  Return the root blocks of the current CFG.  This may include
-  /// multiple blocks if we are computing post dominators.  For forward
-  /// dominators, this will always be a single block (the entry node).
-  ///
-  const SmallVectorImpl<MachineBasicBlock*> &getRoots() const {
-    applySplitCriticalEdges();
-    return DT->getRoots();
-  }
-
   MachineBasicBlock *getRoot() const {
     applySplitCriticalEdges();
     return DT->getRoot();
@@ -270,7 +261,8 @@ template <class T> struct GraphTraits;
 template <>
 struct GraphTraits<MachineDomTreeNode *>
     : public MachineDomTreeGraphTraitsBase<MachineDomTreeNode,
-                                           MachineDomTreeNode::iterator> {};
+                                           MachineDomTreeNode::const_iterator> {
+};
 
 template <>
 struct GraphTraits<const MachineDomTreeNode *>
diff --git a/llvm/include/llvm/CodeGen/MachineFrameInfo.h b/llvm/include/llvm/CodeGen/MachineFrameInfo.h
index c5e57e6da5492..5cd7f9cde674f 100644
--- a/llvm/include/llvm/CodeGen/MachineFrameInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineFrameInfo.h
@@ -755,11 +755,12 @@ class MachineFrameInfo {
   /// a nonnegative identifier to represent it.
   int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot,
                         const AllocaInst *Alloca = nullptr, uint8_t ID = 0);
-  /// FIXME: Remove this function when transition to Align is over.
-  inline int CreateStackObject(uint64_t Size, unsigned Alignment,
-                               bool isSpillSlot,
-                               const AllocaInst *Alloca = nullptr,
-                               uint8_t ID = 0) {
+  LLVM_ATTRIBUTE_DEPRECATED(
+      inline int CreateStackObject(uint64_t Size, unsigned Alignment,
+                                   bool isSpillSlot,
+                                   const AllocaInst *Alloca = nullptr,
+                                   uint8_t ID = 0),
+      "Use CreateStackObject that takes an Align instead") {
     return CreateStackObject(Size, assumeAligned(Alignment), isSpillSlot,
                              Alloca, ID);
   }
@@ -767,8 +768,9 @@ class MachineFrameInfo {
   /// Create a new statically sized stack object that represents a spill slot,
   /// returning a nonnegative identifier to represent it.
   int CreateSpillStackObject(uint64_t Size, Align Alignment);
-  /// FIXME: Remove this function when transition to Align is over.
-  inline int CreateSpillStackObject(uint64_t Size, unsigned Alignment) {
+  LLVM_ATTRIBUTE_DEPRECATED(
+      inline int CreateSpillStackObject(uint64_t Size, unsigned Alignment),
+      "Use CreateSpillStackObject that takes an Align instead") {
     return CreateSpillStackObject(Size, assumeAligned(Alignment));
   }
 
@@ -783,7 +785,9 @@ class MachineFrameInfo {
   /// created, whether or not the index returned is actually used.
   int CreateVariableSizedObject(Align Alignment, const AllocaInst *Alloca);
   /// FIXME: Remove this function when transition to Align is over.
-  int CreateVariableSizedObject(unsigned Alignment, const AllocaInst *Alloca) {
+  LLVM_ATTRIBUTE_DEPRECATED(int CreateVariableSizedObject(
+                                unsigned Alignment, const AllocaInst *Alloca),
+                            "Use the version that takes an Align instead") {
     return CreateVariableSizedObject(assumeAligned(Alignment), Alloca);
   }
 
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index f0418a8efe398..970d6d7db3345 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -408,10 +408,31 @@ class MachineInstr
   /// Returns the debug location id of this MachineInstr.
   const DebugLoc &getDebugLoc() const { return debugLoc; }
 
+  /// Return the operand containing the offset to be used if this DBG_VALUE
+  /// instruction is indirect; will be an invalid register if this value is
+  /// not indirect, and an immediate with value 0 otherwise.
+  const MachineOperand &getDebugOffset() const {
+    assert(isDebugValue() && "not a DBG_VALUE");
+    return getOperand(1);
+  }
+  MachineOperand &getDebugOffset() {
+    assert(isDebugValue() && "not a DBG_VALUE");
+    return getOperand(1);
+  }
+
+  /// Return the operand for the debug variable referenced by
+  /// this DBG_VALUE instruction.
+  const MachineOperand &getDebugVariableOp() const;
+  MachineOperand &getDebugVariableOp();
+
   /// Return the debug variable referenced by
   /// this DBG_VALUE instruction.
   const DILocalVariable *getDebugVariable() const;
 
+  /// Return the operand for the complex address expression referenced by
+  /// this DBG_VALUE instruction.
+  MachineOperand &getDebugExpressionOp();
+
   /// Return the complex address expression referenced by
   /// this DBG_VALUE instruction.
   const DIExpression *getDebugExpression() const;
@@ -437,6 +458,11 @@ class MachineInstr
   /// Retuns the total number of operands.
   unsigned getNumOperands() const { return NumOperands; }
 
+  /// Returns the total number of operands which are debug locations.
+  unsigned getNumDebugOperands() const {
+    return std::distance(debug_operands().begin(), debug_operands().end());
+  }
+
   const MachineOperand& getOperand(unsigned i) const {
     assert(i < getNumOperands() && "getOperand() out of range!");
     return Operands[i];
@@ -446,6 +472,38 @@ class MachineInstr
     return Operands[i];
   }
 
+  MachineOperand &getDebugOperand(unsigned Index) {
+    assert(Index < getNumDebugOperands() && "getDebugOperand() out of range!");
+    return *(debug_operands().begin() + Index);
+  }
+  const MachineOperand &getDebugOperand(unsigned Index) const {
+    assert(Index < getNumDebugOperands() && "getDebugOperand() out of range!");
+    return *(debug_operands().begin() + Index);
+  }
+
+  /// Returns a pointer to the operand corresponding to a debug use of Reg, or
+  /// nullptr if Reg is not used in any debug operand.
+  const MachineOperand *getDebugOperandForReg(Register Reg) const {
+    const MachineOperand *RegOp =
+        find_if(debug_operands(), [Reg](const MachineOperand &Op) {
+          return Op.isReg() && Op.getReg() == Reg;
+        });
+    return RegOp == adl_end(debug_operands()) ? nullptr : RegOp;
+  }
+  MachineOperand *getDebugOperandForReg(Register Reg) {
+    MachineOperand *RegOp =
+        find_if(debug_operands(), [Reg](const MachineOperand &Op) {
+          return Op.isReg() && Op.getReg() == Reg;
+        });
+    return RegOp == adl_end(debug_operands()) ? nullptr : RegOp;
+  }
+
+  unsigned getDebugOperandIndex(const MachineOperand *Op) const {
+    assert(Op >= adl_begin(debug_operands()) &&
+           Op <= adl_end(debug_operands()) && "Expected a debug operand.");
+    return std::distance(adl_begin(debug_operands()), Op);
+  }
+
   /// Returns the total number of definitions.
   unsigned getNumDefs() const {
     return getNumExplicitDefs() + MCID->getNumImplicitDefs();
@@ -518,6 +576,17 @@ class MachineInstr
   iterator_range<const_mop_iterator> implicit_operands() const {
     return make_range(explicit_operands().end(), operands_end());
   }
+  /// Returns a range over all operands that are used to determine the variable
+  /// location for this DBG_VALUE instruction.
+  iterator_range<mop_iterator> debug_operands() {
+    assert(isDebugValue() && "Must be a debug value instruction.");
+    return make_range(operands_begin(), operands_begin() + 1);
+  }
+  /// \copydoc debug_operands()
+  iterator_range<const_mop_iterator> debug_operands() const {
+    assert(isDebugValue() && "Must be a debug value instruction.");
+    return make_range(operands_begin(), operands_begin() + 1);
+  }
   /// Returns a range over all explicit operands that are register definitions.
   /// Implicit definition are not included!
   iterator_range<mop_iterator> defs() {
@@ -1075,12 +1144,12 @@ class MachineInstr
   bool isDebugLabel() const { return getOpcode() == TargetOpcode::DBG_LABEL; }
   bool isDebugInstr() const { return isDebugValue() || isDebugLabel(); }
 
-  /// A DBG_VALUE is indirect iff the first operand is a register and
-  /// the second operand is an immediate.
+  bool isDebugOffsetImm() const { return getDebugOffset().isImm(); }
+
+  /// A DBG_VALUE is indirect iff the location operand is a register and
+  /// the offset operand is an immediate.
   bool isIndirectDebugValue() const {
-    return isDebugValue()
-      && getOperand(0).isReg()
-      && getOperand(1).isImm();
+    return isDebugValue() && getDebugOperand(0).isReg() && isDebugOffsetImm();
   }
 
   /// A DBG_VALUE is an entry value iff its debug expression contains the
@@ -1090,7 +1159,8 @@ class MachineInstr
   /// Return true if the instruction is a debug value which describes a part of
   /// a variable as unavailable.
   bool isUndefDebugValue() const {
-    return isDebugValue() && getOperand(0).isReg() && !getOperand(0).getReg().isValid();
+    return isDebugValue() && getDebugOperand(0).isReg() &&
+           !getDebugOperand(0).getReg().isValid();
   }
 
   bool isPHI() const {
@@ -1686,6 +1756,16 @@ class MachineInstr
     return getOperand(getNumExplicitDefs()).getIntrinsicID();
   }
 
+  /// Sets all register debug operands in this debug value instruction to be
+  /// undef.
+  void setDebugValueUndef() {
+    assert(isDebugValue() && "Must be a debug value instruction.");
+    for (MachineOperand &MO : debug_operands()) {
+      if (MO.isReg())
+        MO.setReg(0);
+    }
+  }
+
 private:
   /// If this instruction is embedded into a MachineFunction, return the
   /// MachineRegisterInfo object for the current function, otherwise
diff --git a/llvm/include/llvm/CodeGen/MachinePostDominators.h b/llvm/include/llvm/CodeGen/MachinePostDominators.h
index 597bb401a7fa0..cee4294f6317b 100644
--- a/llvm/include/llvm/CodeGen/MachinePostDominators.h
+++ b/llvm/include/llvm/CodeGen/MachinePostDominators.h
@@ -41,10 +41,6 @@ class MachinePostDominatorTree : public MachineFunctionPass {
 
   FunctionPass *createMachinePostDominatorTreePass();
 
-  const SmallVectorImpl<MachineBasicBlock *> &getRoots() const {
-    return PDT->getRoots();
-  }
-
   MachineDomTreeNode *getRootNode() const { return PDT->getRootNode(); }
 
   MachineDomTreeNode *operator[](MachineBasicBlock *BB) const {
diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h
index 87612902b42da..a7edaaa906735 100644
--- a/llvm/include/llvm/CodeGen/MachineScheduler.h
+++ b/llvm/include/llvm/CodeGen/MachineScheduler.h
@@ -80,7 +80,6 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachinePassRegistry.h"
 #include "llvm/CodeGen/RegisterPressure.h"
@@ -102,6 +101,7 @@ extern cl::opt<bool> ForceTopDown;
 extern cl::opt<bool> ForceBottomUp;
 extern cl::opt<bool> VerifyScheduling;
 
+class AAResults;
 class LiveIntervals;
 class MachineDominatorTree;
 class MachineFunction;
@@ -121,7 +121,7 @@ struct MachineSchedContext {
   const MachineLoopInfo *MLI = nullptr;
   const MachineDominatorTree *MDT = nullptr;
   const TargetPassConfig *PassConfig = nullptr;
-  AliasAnalysis *AA = nullptr;
+  AAResults *AA = nullptr;
   LiveIntervals *LIS = nullptr;
 
   RegisterClassInfo *RegClassInfo;
@@ -264,7 +264,7 @@ class MachineSchedStrategy {
 /// PreRA and PostRA MachineScheduler.
 class ScheduleDAGMI : public ScheduleDAGInstrs {
 protected:
-  AliasAnalysis *AA;
+  AAResults *AA;
   LiveIntervals *LIS;
   std::unique_ptr<MachineSchedStrategy> SchedImpl;
 
diff --git a/llvm/include/llvm/CodeGen/ModuloSchedule.h b/llvm/include/llvm/CodeGen/ModuloSchedule.h
index b90e2e993410b..1aa23208cfb96 100644
--- a/llvm/include/llvm/CodeGen/ModuloSchedule.h
+++ b/llvm/include/llvm/CodeGen/ModuloSchedule.h
@@ -277,6 +277,19 @@ class ModuloScheduleExpander {
 /// A reimplementation of ModuloScheduleExpander. It works by generating a
 /// standalone kernel loop and peeling out the prologs and epilogs.
 class PeelingModuloScheduleExpander {
+public:
+  PeelingModuloScheduleExpander(MachineFunction &MF, ModuloSchedule &S,
+                                LiveIntervals *LIS)
+      : Schedule(S), MF(MF), ST(MF.getSubtarget()), MRI(MF.getRegInfo()),
+        TII(ST.getInstrInfo()), LIS(LIS) {}
+
+  void expand();
+
+  /// Runs ModuloScheduleExpander and treats it as a golden input to validate
+  /// aspects of the code generated by PeelingModuloScheduleExpander.
+  void validateAgainstModuloScheduleExpander();
+
+protected:
   ModuloSchedule &Schedule;
   MachineFunction &MF;
   const TargetSubtargetInfo &ST;
@@ -311,24 +324,10 @@ class PeelingModuloScheduleExpander {
   /// Illegal phis that need to be deleted once we re-link stages.
   SmallVector<MachineInstr *, 4> IllegalPhisToDelete;
 
-public:
-  PeelingModuloScheduleExpander(MachineFunction &MF, ModuloSchedule &S,
-                                LiveIntervals *LIS)
-      : Schedule(S), MF(MF), ST(MF.getSubtarget()), MRI(MF.getRegInfo()),
-        TII(ST.getInstrInfo()), LIS(LIS) {}
-
-  void expand();
-
-  /// Runs ModuloScheduleExpander and treats it as a golden input to validate
-  /// aspects of the code generated by PeelingModuloScheduleExpander.
-  void validateAgainstModuloScheduleExpander();
-
-protected:
   /// Converts BB from the original loop body to the rewritten, pipelined
   /// steady-state.
   void rewriteKernel();
 
-private:
   /// Peels one iteration of the rewritten kernel (BB) in the specified
   /// direction.
   MachineBasicBlock *peelKernel(LoopPeelDirection LPD);
@@ -364,7 +363,7 @@ class PeelingModuloScheduleExpander {
   /// coming from a peeled out prologue.
   Register getPhiCanonicalReg(MachineInstr* CanonicalPhi, MachineInstr* Phi);
   /// Target loop info before kernel peeling.
-  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> Info;
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> LoopInfo;
 };
 
 /// Expander that simply annotates each scheduled instruction with a post-instr
diff --git a/llvm/include/llvm/CodeGen/ReachingDefAnalysis.h b/llvm/include/llvm/CodeGen/ReachingDefAnalysis.h
index 1444d4c295aff..a8a436337e075 100644
--- a/llvm/include/llvm/CodeGen/ReachingDefAnalysis.h
+++ b/llvm/include/llvm/CodeGen/ReachingDefAnalysis.h
@@ -189,6 +189,7 @@ class ReachingDefAnalysis : public MachineFunctionPass {
   /// definition is found, recursively search the predecessor blocks for them.
   void getLiveOuts(MachineBasicBlock *MBB, int PhysReg, InstSet &Defs,
                    BlockSet &VisitedBBs) const;
+  void getLiveOuts(MachineBasicBlock *MBB, int PhysReg, InstSet &Defs) const;
 
   /// For the given block, collect the instructions that use the live-in
   /// value of the provided register. Return whether the value is still
diff --git a/llvm/include/llvm/CodeGen/Register.h b/llvm/include/llvm/CodeGen/Register.h
index a933a39185f76..054040cd29a1f 100644
--- a/llvm/include/llvm/CodeGen/Register.h
+++ b/llvm/include/llvm/CodeGen/Register.h
@@ -33,6 +33,8 @@ class Register {
   //
   // Further sentinels can be allocated from the small negative integers.
   // DenseMapInfo<unsigned> uses -1u and -2u.
+  static_assert(std::numeric_limits<decltype(Reg)>::max() >= 0xFFFFFFFF,
+                "Reg isn't large enough to hold full range.");
 
   /// isStackSlot - Sometimes it is useful the be able to store a non-negative
   /// frame index in a variable that normally holds a register. isStackSlot()
@@ -49,13 +51,13 @@ class Register {
   /// Compute the frame index from a register value representing a stack slot.
   static int stackSlot2Index(unsigned Reg) {
     assert(isStackSlot(Reg) && "Not a stack slot");
-    return int(Reg - (1u << 30));
+    return int(Reg - MCRegister::FirstStackSlot);
   }
 
   /// Convert a non-negative frame index to a stack slot register value.
   static unsigned index2StackSlot(int FI) {
     assert(FI >= 0 && "Cannot hold a negative frame index.");
-    return FI + (1u << 30);
+    return FI + MCRegister::FirstStackSlot;
   }
 
   /// Return true if the specified register number is in
@@ -68,20 +70,21 @@ class Register {
   /// the virtual register namespace.
   static bool isVirtualRegister(unsigned Reg) {
     assert(!isStackSlot(Reg) && "Not a register! Check isStackSlot() first.");
-    return int(Reg) < 0;
+    return Reg & MCRegister::VirtualRegFlag;
   }
 
   /// Convert a virtual register number to a 0-based index.
   /// The first virtual register in a function will get the index 0.
   static unsigned virtReg2Index(unsigned Reg) {
     assert(isVirtualRegister(Reg) && "Not a virtual register");
-    return Reg & ~(1u << 31);
+    return Reg & ~MCRegister::VirtualRegFlag;
   }
 
   /// Convert a 0-based index to a virtual register number.
   /// This is the inverse operation of VirtReg2IndexFunctor below.
   static unsigned index2VirtReg(unsigned Index) {
-    return Index | (1u << 31);
+    assert(Index < (1u << 31) && "Index too large for virtual register range.");
+    return Index | MCRegister::VirtualRegFlag;
   }
 
   /// Return true if the specified register number is in the virtual register
@@ -112,9 +115,7 @@ class Register {
     return MCRegister(Reg);
   }
 
-  bool isValid() const {
-    return Reg != 0;
-  }
+  bool isValid() const { return Reg != MCRegister::NoRegister; }
 
   /// Comparisons between register objects
   bool operator==(const Register &Other) const { return Reg == Other.Reg; }
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index f9706ee9f4e4e..f26ab6f287a03 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -931,7 +931,8 @@ class SelectionDAG {
   SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {
     assert(MulImm.getMinSignedBits() <= VT.getSizeInBits() &&
            "Immediate does not fit VT");
-    return getNode(ISD::VSCALE, DL, VT, getConstant(MulImm, DL, VT));
+    return getNode(ISD::VSCALE, DL, VT,
+                   getConstant(MulImm.sextOrTrunc(VT.getSizeInBits()), DL, VT));
   }
 
   /// Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
@@ -1342,6 +1343,9 @@ class SelectionDAG {
   /// Return a freeze using the SDLoc of the value operand.
   SDValue getFreeze(SDValue V);
 
+  /// Return an AssertAlignSDNode.
+  SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A);
+
   /// Return the specified value casted to
   /// the target's desired shift amount type.
   SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index ee2e8ecbd67a9..7c2b49087edd2 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1576,6 +1576,8 @@ class ConstantSDNode : public SDNode {
   uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX) {
     return Value->getLimitedValue(Limit);
   }
+  MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
+  Align getAlignValue() const { return Value->getAlignValue(); }
 
   bool isOne() const { return Value->isOne(); }
   bool isNullValue() const { return Value->isZero(); }
@@ -2526,6 +2528,22 @@ class MachineSDNode : public SDNode {
   }
 };
 
+/// An SDNode that records if a register contains a value that is guaranteed to
+/// be aligned accordingly.
+class AssertAlignSDNode : public SDNode {
+  Align Alignment;
+
+public:
+  AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
+      : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}
+
+  Align getAlign() const { return Alignment; }
+
+  static bool classof(const SDNode *N) {
+    return N->getOpcode() == ISD::AssertAlign;
+  }
+};
+
 class SDNodeIterator : public std::iterator<std::forward_iterator_tag,
                                             SDNode, ptrdiff_t> {
   const SDNode *Node;
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h b/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h
index a90ede759cf34..014523f1af6a4 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h
@@ -51,7 +51,7 @@ class SelectionDAGTargetInfo {
   virtual SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
                                           SDValue Chain, SDValue Op1,
                                           SDValue Op2, SDValue Op3,
-                                          unsigned Align, bool isVolatile,
+                                          Align Alignment, bool isVolatile,
                                           bool AlwaysInline,
                                           MachinePointerInfo DstPtrInfo,
                                           MachinePointerInfo SrcPtrInfo) const {
@@ -66,7 +66,7 @@ class SelectionDAGTargetInfo {
   /// lowering strategy should be used.
   virtual SDValue EmitTargetCodeForMemmove(
       SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1,
-      SDValue Op2, SDValue Op3, unsigned Align, bool isVolatile,
+      SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile,
       MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
     return SDValue();
   }
@@ -80,7 +80,7 @@ class SelectionDAGTargetInfo {
   virtual SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl,
                                           SDValue Chain, SDValue Op1,
                                           SDValue Op2, SDValue Op3,
-                                          unsigned Align, bool isVolatile,
+                                          Align Alignment, bool isVolatile,
                                           MachinePointerInfo DstPtrInfo) const {
     return SDValue();
   }
diff --git a/llvm/include/llvm/CodeGen/TailDuplicator.h b/llvm/include/llvm/CodeGen/TailDuplicator.h
index 0903b93e50ebe..6862bb2c3f44f 100644
--- a/llvm/include/llvm/CodeGen/TailDuplicator.h
+++ b/llvm/include/llvm/CodeGen/TailDuplicator.h
@@ -50,13 +50,13 @@ class TailDuplicator {
   unsigned TailDupSize;
 
   // A list of virtual registers for which to update SSA form.
-  SmallVector<unsigned, 16> SSAUpdateVRs;
+  SmallVector<Register, 16> SSAUpdateVRs;
 
   // For each virtual register in SSAUpdateVals keep a list of source virtual
   // registers.
-  using AvailableValsTy = std::vector<std::pair<MachineBasicBlock *, unsigned>>;
+  using AvailableValsTy = std::vector<std::pair<MachineBasicBlock *, Register>>;
 
-  DenseMap<unsigned, AvailableValsTy> SSAUpdateVals;
+  DenseMap<Register, AvailableValsTy> SSAUpdateVals;
 
 public:
   /// Prepare to run on a specific machine function.
@@ -98,24 +98,24 @@ class TailDuplicator {
 private:
   using RegSubRegPair = TargetInstrInfo::RegSubRegPair;
 
-  void addSSAUpdateEntry(unsigned OrigReg, unsigned NewReg,
+  void addSSAUpdateEntry(Register OrigReg, Register NewReg,
                          MachineBasicBlock *BB);
   void processPHI(MachineInstr *MI, MachineBasicBlock *TailBB,
                   MachineBasicBlock *PredBB,
-                  DenseMap<unsigned, RegSubRegPair> &LocalVRMap,
-                  SmallVectorImpl<std::pair<unsigned, RegSubRegPair>> &Copies,
-                  const DenseSet<unsigned> &UsedByPhi, bool Remove);
+                  DenseMap<Register, RegSubRegPair> &LocalVRMap,
+                  SmallVectorImpl<std::pair<Register, RegSubRegPair>> &Copies,
+                  const DenseSet<Register> &UsedByPhi, bool Remove);
   void duplicateInstruction(MachineInstr *MI, MachineBasicBlock *TailBB,
                             MachineBasicBlock *PredBB,
-                            DenseMap<unsigned, RegSubRegPair> &LocalVRMap,
-                            const DenseSet<unsigned> &UsedByPhi);
+                            DenseMap<Register, RegSubRegPair> &LocalVRMap,
+                            const DenseSet<Register> &UsedByPhi);
   void updateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead,
                             SmallVectorImpl<MachineBasicBlock *> &TDBBs,
                             SmallSetVector<MachineBasicBlock *, 8> &Succs);
   bool canCompletelyDuplicateBB(MachineBasicBlock &BB);
   bool duplicateSimpleBB(MachineBasicBlock *TailBB,
                          SmallVectorImpl<MachineBasicBlock *> &TDBBs,
-                         const DenseSet<unsigned> &RegsUsedByPhi,
+                         const DenseSet<Register> &RegsUsedByPhi,
                          SmallVectorImpl<MachineInstr *> &Copies);
   bool tailDuplicate(bool IsSimple,
                      MachineBasicBlock *TailBB,
@@ -124,7 +124,7 @@ class TailDuplicator {
                      SmallVectorImpl<MachineInstr *> &Copies,
                      SmallVectorImpl<MachineBasicBlock *> *CandidatePtr);
   void appendCopies(MachineBasicBlock *MBB,
-                 SmallVectorImpl<std::pair<unsigned,RegSubRegPair>> &CopyInfos,
+                 SmallVectorImpl<std::pair<Register, RegSubRegPair>> &CopyInfos,
                  SmallVectorImpl<MachineInstr *> &Copies);
 
   void removeDeadBlock(
diff --git a/llvm/include/llvm/CodeGen/TargetCallingConv.h b/llvm/include/llvm/CodeGen/TargetCallingConv.h
index 6ccc1ce11d992..347d7ff404047 100644
--- a/llvm/include/llvm/CodeGen/TargetCallingConv.h
+++ b/llvm/include/llvm/CodeGen/TargetCallingConv.h
@@ -144,13 +144,17 @@ namespace ISD {
       assert(getNonZeroByValAlign() == A && "bitfield overflow");
     }
 
-    unsigned getOrigAlign() const {
+    LLVM_ATTRIBUTE_DEPRECATED(unsigned getOrigAlign() const,
+                              "Use getNonZeroOrigAlign() instead") {
       MaybeAlign A = decodeMaybeAlign(OrigAlign);
       return A ? A->value() : 0;
     }
+    Align getNonZeroOrigAlign() const {
+      return decodeMaybeAlign(OrigAlign).valueOrOne();
+    }
     void setOrigAlign(Align A) {
       OrigAlign = encode(A);
-      assert(getOrigAlign() == A.value() && "bitfield overflow");
+      assert(getNonZeroOrigAlign() == A && "bitfield overflow");
     }
 
     unsigned getByValSize() const { return ByValSize; }
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 6238184e4e853..06f2b3ca38ea7 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1472,7 +1472,7 @@ class TargetLoweringBase {
   /// type has the alignment requirement of another type.
   virtual Align getABIAlignmentForCallingConv(Type *ArgTy,
                                               DataLayout DL) const {
-    return Align(DL.getABITypeAlignment(ArgTy));
+    return DL.getABITypeAlign(ArgTy);
   }
 
   /// If true, then instruction selection should seek to shrink the FP constant
@@ -1594,7 +1594,7 @@ class TargetLoweringBase {
   /// (as defined by the target).
   bool allowsMemoryAccessForAlignment(
       LLVMContext &Context, const DataLayout &DL, EVT VT,
-      unsigned AddrSpace = 0, unsigned Alignment = 1,
+      unsigned AddrSpace = 0, Align Alignment = Align(1),
       MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
       bool *Fast = nullptr) const;
 
@@ -1613,7 +1613,7 @@ class TargetLoweringBase {
   /// target).
   virtual bool
   allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
-                     unsigned AddrSpace = 0, unsigned Alignment = 1,
+                     unsigned AddrSpace = 0, Align Alignment = Align(1),
                      MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
                      bool *Fast = nullptr) const;
 
@@ -2572,7 +2572,7 @@ class TargetLoweringBase {
   /// this information should not be provided because it will generate more
   /// loads.
   virtual bool hasPairedLoad(EVT /*LoadedType*/,
-                             unsigned & /*RequiredAlignment*/) const {
+                             Align & /*RequiredAlignment*/) const {
     return false;
   }
 
@@ -3254,13 +3254,20 @@ class TargetLowering : public TargetLoweringBase {
   /// constant integer.  If so, check to see if there are any bits set in the
   /// constant that are not demanded.  If so, shrink the constant and return
   /// true.
-  bool ShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
+  bool ShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,
+                              const APInt &DemandedElts,
+                              TargetLoweringOpt &TLO) const;
+
+  /// Helper wrapper around ShrinkDemandedConstant, demanding all elements.
+  bool ShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,
                               TargetLoweringOpt &TLO) const;
 
   // Target hook to do target-specific const optimization, which is called by
   // ShrinkDemandedConstant. This function should return true if the target
   // doesn't want ShrinkDemandedConstant to further optimize the constant.
-  virtual bool targetShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
+  virtual bool targetShrinkDemandedConstant(SDValue Op,
+                                            const APInt &DemandedBits,
+                                            const APInt &DemandedElts,
                                             TargetLoweringOpt &TLO) const {
     return false;
   }
@@ -3315,6 +3322,13 @@ class TargetLowering : public TargetLoweringBase {
                                           SelectionDAG &DAG,
                                           unsigned Depth = 0) const;
 
+  /// Helper wrapper around SimplifyMultipleUseDemandedBits, demanding all
+  /// bits from only some vector elements.
+  SDValue SimplifyMultipleUseDemandedVectorElts(SDValue Op,
+                                                const APInt &DemandedElts,
+                                                SelectionDAG &DAG,
+                                                unsigned Depth = 0) const;
+
   /// Look at Vector Op. At this point, we know that only the DemandedElts
   /// elements of the result of Op are ever used downstream.  If we can use
   /// this information to simplify Op, create a new simplified DAG node and
@@ -4414,6 +4428,10 @@ class TargetLowering : public TargetLoweringBase {
   /// only the first Count elements of the vector are used.
   SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const;
 
+  /// Expand an SREM or UREM using SDIV/UDIV or SDIVREM/UDIVREM, if legal.
+  /// Returns true if the expansion was successful.
+  bool expandREM(SDNode *Node, SDValue &Result, SelectionDAG &DAG) const;
+
   //===--------------------------------------------------------------------===//
   // Instruction Emitting Hooks
   //
diff --git a/llvm/include/llvm/CodeGen/TargetPassConfig.h b/llvm/include/llvm/CodeGen/TargetPassConfig.h
index 3a4c09163f325..a18c8b16bf1c0 100644
--- a/llvm/include/llvm/CodeGen/TargetPassConfig.h
+++ b/llvm/include/llvm/CodeGen/TargetPassConfig.h
@@ -167,8 +167,8 @@ class TargetPassConfig : public ImmutablePass {
   /// If hasLimitedCodeGenPipeline is true, this method
   /// returns a string with the name of the options, separated
   /// by \p Separator that caused this pipeline to be limited.
-  std::string
-  getLimitedCodeGenPipelineReason(const char *Separator = "/") const;
+  static std::string
+  getLimitedCodeGenPipelineReason(const char *Separator = "/");
 
   void setDisableVerify(bool Disable) { setOpt(DisableVerify, Disable); }
 
diff --git a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
index 169652e9f592d..e0dfd9c8bbc5a 100644
--- a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
@@ -25,7 +25,6 @@
 #include <memory>
 #include <vector>
 
-
 namespace llvm {
 
 class CallLowering;
@@ -42,9 +41,6 @@ struct MCWriteProcResEntry;
 class RegisterBankInfo;
 class SDep;
 class SelectionDAGTargetInfo;
-struct SubtargetFeatureKV;
-struct SubtargetSubTypeKV;
-struct SubtargetInfoKV;
 class SUnit;
 class TargetFrameLowering;
 class TargetInstrInfo;
diff --git a/llvm/include/llvm/CodeGen/ValueTypes.td b/llvm/include/llvm/CodeGen/ValueTypes.td
index caa3d4daab318..c5eb87cf1d347 100644
--- a/llvm/include/llvm/CodeGen/ValueTypes.td
+++ b/llvm/include/llvm/CodeGen/ValueTypes.td
@@ -96,88 +96,99 @@ def v4f16    : ValueType<64 , 67>;    //    4 x f16 vector value
 def v8f16    : ValueType<128, 68>;    //    8 x f16 vector value
 def v16f16   : ValueType<256, 69>;    //   16 x f16 vector value
 def v32f16   : ValueType<512, 70>;    //   32 x f16 vector value
-def v2bf16   : ValueType<32 , 71>;    //    2 x bf16 vector value
-def v3bf16   : ValueType<48 , 72>;    //    3 x bf16 vector value
-def v4bf16   : ValueType<64 , 73>;    //    4 x bf16 vector value
-def v8bf16   : ValueType<128, 74>;    //    8 x bf16 vector value
-def v16bf16  : ValueType<256, 75>;    //   16 x bf16 vector value
-def v32bf16  : ValueType<512, 76>;    //   32 x bf16 vector value
-def v1f32    : ValueType<32 , 77>;    //    1 x f32 vector value
-def v2f32    : ValueType<64 , 78>;    //    2 x f32 vector value
-def v3f32    : ValueType<96 , 79>;    //    3 x f32 vector value
-def v4f32    : ValueType<128, 80>;    //    4 x f32 vector value
-def v5f32    : ValueType<160, 81>;    //    5 x f32 vector value
-def v8f32    : ValueType<256, 82>;    //    8 x f32 vector value
-def v16f32   : ValueType<512,  83>;   //   16 x f32 vector value
-def v32f32   : ValueType<1024, 84>;   //   32 x f32 vector value
-def v64f32   : ValueType<2048, 85>;   //   64 x f32 vector value
-def v128f32  : ValueType<4096, 86>;   //  128 x f32 vector value
-def v256f32  : ValueType<8182, 87>;   //  256 x f32 vector value
-def v512f32  : ValueType<16384, 88>;  //  512 x f32 vector value
-def v1024f32 : ValueType<32768, 89>;  // 1024 x f32 vector value
-def v2048f32 : ValueType<65536, 90>;  // 2048 x f32 vector value
-def v1f64    : ValueType<64, 91>;     //    1 x f64 vector value
-def v2f64    : ValueType<128, 92>;    //    2 x f64 vector value
-def v4f64    : ValueType<256, 93>;    //    4 x f64 vector value
-def v8f64    : ValueType<512, 94>;    //    8 x f64 vector value
-def v16f64   : ValueType<1024, 95>;   //   16 x f64 vector value
-
-def nxv1i1  : ValueType<1,   96>;  // n x  1 x i1  vector value
-def nxv2i1  : ValueType<2,   97>;  // n x  2 x i1  vector value
-def nxv4i1  : ValueType<4,   98>;  // n x  4 x i1  vector value
-def nxv8i1  : ValueType<8,   99>;  // n x  8 x i1  vector value
-def nxv16i1 : ValueType<16, 100>;  // n x 16 x i1  vector value
-def nxv32i1 : ValueType<32, 101>;  // n x 32 x i1  vector value
-
-def nxv1i8  : ValueType<8,   102>;  // n x  1 x i8  vector value
-def nxv2i8  : ValueType<16,  103>;  // n x  2 x i8  vector value
-def nxv4i8  : ValueType<32,  104>;  // n x  4 x i8  vector value
-def nxv8i8  : ValueType<64,  105>;  // n x  8 x i8  vector value
-def nxv16i8 : ValueType<128, 106>;  // n x 16 x i8  vector value
-def nxv32i8 : ValueType<256, 107>;  // n x 32 x i8  vector value
-
-def nxv1i16 : ValueType<16,  108>; // n x  1 x i16 vector value
-def nxv2i16 : ValueType<32,  109>; // n x  2 x i16 vector value
-def nxv4i16 : ValueType<64,  110>; // n x  4 x i16 vector value
-def nxv8i16 : ValueType<128, 111>; // n x  8 x i16 vector value
-def nxv16i16: ValueType<256, 112>; // n x 16 x i16 vector value
-def nxv32i16: ValueType<512, 113>; // n x 32 x i16 vector value
-
-def nxv1i32 : ValueType<32,  114>; // n x  1 x i32 vector value
-def nxv2i32 : ValueType<64,  115>; // n x  2 x i32 vector value
-def nxv4i32 : ValueType<128, 116>; // n x  4 x i32 vector value
-def nxv8i32 : ValueType<256, 117>; // n x  8 x i32 vector value
-def nxv16i32: ValueType<512, 118>; // n x 16 x i32 vector value
-def nxv32i32: ValueType<1024,119>; // n x 32 x i32 vector value
-
-def nxv1i64 : ValueType<64,  120>; // n x  1 x i64 vector value
-def nxv2i64 : ValueType<128, 121>; // n x  2 x i64 vector value
-def nxv4i64 : ValueType<256, 122>; // n x  4 x i64 vector value
-def nxv8i64 : ValueType<512, 123>; // n x  8 x i64 vector value
-def nxv16i64: ValueType<1024,124>; // n x 16 x i64 vector value
-def nxv32i64: ValueType<2048,125>; // n x 32 x i64 vector value
-
-def nxv2f16  : ValueType<32 , 126>; // n x  2 x f16 vector value
-def nxv4f16  : ValueType<64 , 127>; // n x  4 x f16 vector value
-def nxv8f16  : ValueType<128, 128>; // n x  8 x f16 vector value
-def nxv2bf16 : ValueType<32 , 129>; // n x  2 x bf16 vector value
-def nxv4bf16 : ValueType<64 , 130>; // n x  4 x bf16 vector value
-def nxv8bf16 : ValueType<128, 131>; // n x  8 x bf16 vector value
-def nxv1f32  : ValueType<32 , 132>; // n x  1 x f32 vector value
-def nxv2f32  : ValueType<64 , 133>; // n x  2 x f32 vector value
-def nxv4f32  : ValueType<128, 134>; // n x  4 x f32 vector value
-def nxv8f32  : ValueType<256, 135>; // n x  8 x f32 vector value
-def nxv16f32 : ValueType<512, 136>; // n x 16 x f32 vector value
-def nxv1f64  : ValueType<64,  137>; // n x  1 x f64 vector value
-def nxv2f64  : ValueType<128, 138>; // n x  2 x f64 vector value
-def nxv4f64  : ValueType<256, 139>; // n x  4 x f64 vector value
-def nxv8f64  : ValueType<512, 140>; // n x  8 x f64 vector value
-
-def x86mmx : ValueType<64 , 141>;   // X86 MMX value
-def FlagVT : ValueType<0  , 142>;   // Pre-RA sched glue
-def isVoid : ValueType<0  , 143>;   // Produces no value
-def untyped: ValueType<8  , 144>;   // Produces an untyped value
-def exnref : ValueType<0  , 145>;      // WebAssembly's exnref type
+def v64f16   : ValueType<1024, 71>;   //   64 x f16 vector value
+def v128f16  : ValueType<2048, 72>;   //  128 x f16 vector value
+def v2bf16   : ValueType<32 , 73>;    //    2 x bf16 vector value
+def v3bf16   : ValueType<48 , 74>;    //    3 x bf16 vector value
+def v4bf16   : ValueType<64 , 75>;    //    4 x bf16 vector value
+def v8bf16   : ValueType<128, 76>;    //    8 x bf16 vector value
+def v16bf16  : ValueType<256, 77>;    //   16 x bf16 vector value
+def v32bf16  : ValueType<512, 78>;    //   32 x bf16 vector value
+def v64bf16  : ValueType<1024, 79>;   //   64 x bf16 vector value
+def v128bf16 : ValueType<2048, 80>;   //  128 x bf16 vector value
+def v1f32    : ValueType<32 , 81>;    //    1 x f32 vector value
+def v2f32    : ValueType<64 , 82>;    //    2 x f32 vector value
+def v3f32    : ValueType<96 , 83>;    //    3 x f32 vector value
+def v4f32    : ValueType<128, 84>;    //    4 x f32 vector value
+def v5f32    : ValueType<160, 85>;    //    5 x f32 vector value
+def v8f32    : ValueType<256, 86>;    //    8 x f32 vector value
+def v16f32   : ValueType<512,  87>;   //   16 x f32 vector value
+def v32f32   : ValueType<1024, 88>;   //   32 x f32 vector value
+def v64f32   : ValueType<2048, 89>;   //   64 x f32 vector value
+def v128f32  : ValueType<4096, 90>;   //  128 x f32 vector value
+def v256f32  : ValueType<8182, 91>;   //  256 x f32 vector value
+def v512f32  : ValueType<16384, 92>;  //  512 x f32 vector value
+def v1024f32 : ValueType<32768, 93>;  // 1024 x f32 vector value
+def v2048f32 : ValueType<65536, 94>;  // 2048 x f32 vector value
+def v1f64    : ValueType<64, 95>;     //    1 x f64 vector value
+def v2f64    : ValueType<128, 96>;    //    2 x f64 vector value
+def v4f64    : ValueType<256, 97>;    //    4 x f64 vector value
+def v8f64    : ValueType<512, 98>;    //    8 x f64 vector value
+def v16f64   : ValueType<1024, 99>;   //   16 x f64 vector value
+def v32f64   : ValueType<2048, 100>;  //   32 x f64 vector value
+
+def nxv1i1  : ValueType<1,  101>;  // n x  1 x i1  vector value
+def nxv2i1  : ValueType<2,  102>;  // n x  2 x i1  vector value
+def nxv4i1  : ValueType<4,  103>;  // n x  4 x i1  vector value
+def nxv8i1  : ValueType<8,  104>;  // n x  8 x i1  vector value
+def nxv16i1 : ValueType<16, 105>;  // n x 16 x i1  vector value
+def nxv32i1 : ValueType<32, 106>;  // n x 32 x i1  vector value
+def nxv64i1  : ValueType<64,107>;  // n x  64 x i1  vector value
+
+def nxv1i8  : ValueType<8,   108>;  // n x  1 x i8  vector value
+def nxv2i8  : ValueType<16,  109>;  // n x  2 x i8  vector value
+def nxv4i8  : ValueType<32,  110>;  // n x  4 x i8  vector value
+def nxv8i8  : ValueType<64,  111>;  // n x  8 x i8  vector value
+def nxv16i8 : ValueType<128, 112>;  // n x 16 x i8  vector value
+def nxv32i8 : ValueType<256, 113>;  // n x 32 x i8  vector value
+def nxv64i8  : ValueType<512,  114>;  // n x  64 x i8  vector value
+
+def nxv1i16 : ValueType<16,  115>; // n x  1 x i16 vector value
+def nxv2i16 : ValueType<32,  116>; // n x  2 x i16 vector value
+def nxv4i16 : ValueType<64,  117>; // n x  4 x i16 vector value
+def nxv8i16 : ValueType<128, 118>; // n x  8 x i16 vector value
+def nxv16i16: ValueType<256, 119>; // n x 16 x i16 vector value
+def nxv32i16: ValueType<512, 120>; // n x 32 x i16 vector value
+
+def nxv1i32 : ValueType<32,  121>; // n x  1 x i32 vector value
+def nxv2i32 : ValueType<64,  122>; // n x  2 x i32 vector value
+def nxv4i32 : ValueType<128, 123>; // n x  4 x i32 vector value
+def nxv8i32 : ValueType<256, 124>; // n x  8 x i32 vector value
+def nxv16i32: ValueType<512, 125>; // n x 16 x i32 vector value
+def nxv32i32: ValueType<1024,126>; // n x 32 x i32 vector value
+
+def nxv1i64 : ValueType<64,  127>; // n x  1 x i64 vector value
+def nxv2i64 : ValueType<128, 128>; // n x  2 x i64 vector value
+def nxv4i64 : ValueType<256, 129>; // n x  4 x i64 vector value
+def nxv8i64 : ValueType<512, 130>; // n x  8 x i64 vector value
+def nxv16i64: ValueType<1024,131>; // n x 16 x i64 vector value
+def nxv32i64: ValueType<2048,132>; // n x 32 x i64 vector value
+
+def nxv1f16   : ValueType<32, 133>; // n x   1 x f16 vector value
+def nxv2f16  : ValueType<32 , 134>; // n x  2 x f16 vector value
+def nxv4f16  : ValueType<64 , 135>; // n x  4 x f16 vector value
+def nxv8f16  : ValueType<128, 136>; // n x  8 x f16 vector value
+def nxv16f16  : ValueType<256,137>; // n x  16 x f16 vector value
+def nxv32f16  : ValueType<512,138>; // n x  32 x f16 vector value
+def nxv2bf16 : ValueType<32 , 139>; // n x  2 x bf16 vector value
+def nxv4bf16 : ValueType<64 , 140>; // n x  4 x bf16 vector value
+def nxv8bf16 : ValueType<128, 141>; // n x  8 x bf16 vector value
+def nxv1f32  : ValueType<32 , 142>; // n x  1 x f32 vector value
+def nxv2f32  : ValueType<64 , 143>; // n x  2 x f32 vector value
+def nxv4f32  : ValueType<128, 144>; // n x  4 x f32 vector value
+def nxv8f32  : ValueType<256, 145>; // n x  8 x f32 vector value
+def nxv16f32 : ValueType<512, 146>; // n x 16 x f32 vector value
+def nxv1f64  : ValueType<64,  147>; // n x  1 x f64 vector value
+def nxv2f64  : ValueType<128, 148>; // n x  2 x f64 vector value
+def nxv4f64  : ValueType<256, 149>; // n x  4 x f64 vector value
+def nxv8f64  : ValueType<512, 150>; // n x  8 x f64 vector value
+
+def x86mmx : ValueType<64 , 151>;   // X86 MMX value
+def FlagVT : ValueType<0  , 152>;   // Pre-RA sched glue
+def isVoid : ValueType<0  , 153>;   // Produces no value
+def untyped: ValueType<8  , 154>;   // Produces an untyped value
+def exnref : ValueType<0  , 155>;   // WebAssembly's exnref type
+
 def token  : ValueType<0  , 248>;   // TokenTy
 def MetadataVT: ValueType<0, 249>;  // Metadata
 
diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinkerDeclContext.h b/llvm/include/llvm/DWARFLinker/DWARFLinkerDeclContext.h
index db40254bf6000..e59e15f00a7e5 100644
--- a/llvm/include/llvm/DWARFLinker/DWARFLinkerDeclContext.h
+++ b/llvm/include/llvm/DWARFLinker/DWARFLinkerDeclContext.h
@@ -16,6 +16,7 @@
 #include "llvm/CodeGen/NonRelocatableStringpool.h"
 #include "llvm/DWARFLinker/DWARFLinkerCompileUnit.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 
 namespace llvm {
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h
index 7a728c2508cc8..154f7893aa174 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h
@@ -45,6 +45,24 @@ struct DWARFAddressRange {
     return LowPC < RHS.HighPC && RHS.LowPC < HighPC;
   }
 
+  /// Union two address ranges if they intersect.
+  ///
+  /// This function will union two address ranges if they intersect by
+  /// modifying this range to be the union of both ranges. If the two ranges
+  /// don't intersect this range will be left alone.
+  ///
+  /// \param RHS Another address range to combine with.
+  ///
+  /// \returns false if the ranges don't intersect, true if they do and the
+  /// ranges were combined.
+  bool merge(const DWARFAddressRange &RHS) {
+    if (!intersects(RHS))
+      return false;
+    LowPC = std::min<uint64_t>(LowPC, RHS.LowPC);
+    HighPC = std::max<uint64_t>(HighPC, RHS.HighPC);
+    return true;
+  }
+
   void dump(raw_ostream &OS, uint32_t AddressSize, DIDumpOptions DumpOpts = {},
             const DWARFObject *Obj = nullptr) const;
 };
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h
index b36505760e466..97903a96b3fc4 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h
@@ -362,6 +362,10 @@ class DWARFContext : public DIContext {
     return version == 2 || version == 3 || version == 4 || version == 5;
   }
 
+  static bool isAddressSizeSupported(unsigned AddressSize) {
+    return AddressSize == 2 || AddressSize == 4 || AddressSize == 8;
+  }
+
   std::shared_ptr<DWARFContext> getDWOContext(StringRef AbsolutePath);
 
   const MCRegisterInfo *getRegisterInfo() const { return RegInfo.get(); }
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h
index 60f80bb12aa93..cb347615868bd 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
 #include "llvm/DebugInfo/DWARF/DWARFObject.h"
 #include <cstdint>
 #include <vector>
@@ -67,11 +68,13 @@ class DWARFDebugPubTable {
 
   /// gnu styled tables contains additional information.
   /// This flag determines whether or not section we parse is debug_gnu* table.
-  bool GnuStyle;
+  bool GnuStyle = false;
 
 public:
-  DWARFDebugPubTable(const DWARFObject &Obj, const DWARFSection &Sec,
-                     bool LittleEndian, bool GnuStyle);
+  DWARFDebugPubTable() = default;
+
+  void extract(DWARFDataExtractor Data, bool GnuStyle,
+               function_ref<void(Error)> RecoverableErrorHandler);
 
   void dump(raw_ostream &OS) const;
 
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
index 28929306ca925..1aff2624990f7 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
@@ -142,6 +142,12 @@ class DWARFExpression {
   void print(raw_ostream &OS, const MCRegisterInfo *RegInfo, DWARFUnit *U,
              bool IsEH = false) const;
 
+  /// Print the expression in a format intended to be compact and useful to a
+  /// user, but not perfectly unambiguous, or capable of representing every
+  /// valid DWARF expression. Returns true if the expression was sucessfully
+  /// printed.
+  bool printCompact(raw_ostream &OS, const MCRegisterInfo &RegInfo);
+
   bool verify(DWARFUnit *U);
 
 private:
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h
index a4a3a11d441b8..22b1d722fc89c 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_DEBUGINFO_DWARF_DWARFVERIFIER_H
 #define LLVM_DEBUGINFO_DWARF_DWARFVERIFIER_H
 
+#include "llvm/ADT/Optional.h"
 #include "llvm/DebugInfo/DIContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h"
 #include "llvm/DebugInfo/DWARF/DWARFAddressRange.h"
@@ -56,11 +57,13 @@ class DWARFVerifier {
     typedef std::set<DieRangeInfo>::const_iterator die_range_info_iterator;
 
     /// Inserts the address range. If the range overlaps with an existing
-    /// range, the range is *not* added and an iterator to the overlapping
-    /// range is returned.
+    /// range, the range that it overlaps with will be returned and the two
+    /// address ranges will be unioned together in "Ranges".
     ///
-    /// This is used for finding overlapping ranges within the same DIE.
-    address_range_iterator insert(const DWARFAddressRange &R);
+    /// This is used for finding overlapping ranges in the DW_AT_ranges
+    /// attribute of a DIE. It is also used as a set of address ranges that
+    /// children address ranges must all be contained in.
+    Optional<DWARFAddressRange> insert(const DWARFAddressRange &R);
 
     /// Finds an address range in the sorted vector of ranges.
     address_range_iterator findRange(const DWARFAddressRange &R) const {
diff --git a/llvm/include/llvm/DebugInfo/GSYM/InlineInfo.h b/llvm/include/llvm/DebugInfo/GSYM/InlineInfo.h
index 3b95e3e050bd2..06126da7d007e 100644
--- a/llvm/include/llvm/DebugInfo/GSYM/InlineInfo.h
+++ b/llvm/include/llvm/DebugInfo/GSYM/InlineInfo.h
@@ -46,7 +46,7 @@ class GsymReader;
 /// also makes any encoded addresses easy to relocate as we just need to
 /// relocate the FunctionInfo's start address.
 ///
-/// - The AddressRanges member "Ranges" is encoded using an approriate base
+/// - The AddressRanges member "Ranges" is encoded using an appropriate base
 ///   address as described above.
 /// - UINT8 boolean value that specifies if the InlineInfo object has children.
 /// - UINT32 string table offset that points to the name of the inline
diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h b/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h
index f52c2f6ed3dc5..378d4cdd23e60 100644
--- a/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h
+++ b/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_DEBUGINFO_PDB_RAW_GSISTREAMBUILDER_H
 #define LLVM_DEBUGINFO_PDB_RAW_GSISTREAMBUILDER_H
 
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/DebugInfo/CodeView/SymbolRecord.h"
 #include "llvm/DebugInfo/PDB/Native/GlobalsStream.h"
 #include "llvm/DebugInfo/PDB/Native/RawConstants.h"
@@ -38,6 +39,7 @@ struct MSFLayout;
 namespace pdb {
 struct GSIHashStreamBuilder;
 struct BulkPublic;
+struct SymbolDenseMapInfo;
 
 class GSIStreamBuilder {
 
@@ -57,14 +59,22 @@ class GSIStreamBuilder {
   uint32_t getRecordStreamIndex() const { return RecordStreamIndex; }
 
   // Add public symbols in bulk.
-  void addPublicSymbols(std::vector<BulkPublic> &&Publics);
+  void addPublicSymbols(std::vector<BulkPublic> &&PublicsIn);
 
   void addGlobalSymbol(const codeview::ProcRefSym &Sym);
   void addGlobalSymbol(const codeview::DataSym &Sym);
   void addGlobalSymbol(const codeview::ConstantSym &Sym);
+
+  // Add a pre-serialized global symbol record. The caller must ensure that the
+  // symbol data remains alive until the global stream is committed to disk.
   void addGlobalSymbol(const codeview::CVSymbol &Sym);
 
 private:
+  void finalizePublicBuckets();
+  void finalizeGlobalBuckets(uint32_t RecordZeroOffset);
+
+  template <typename T> void serializeAndAddGlobal(const T &Symbol);
+
   uint32_t calculatePublicsHashStreamSize() const;
   uint32_t calculateGlobalsHashStreamSize() const;
   Error commitSymbolRecordStream(WritableBinaryStreamRef Stream);
@@ -77,13 +87,25 @@ class GSIStreamBuilder {
   msf::MSFBuilder &Msf;
   std::unique_ptr<GSIHashStreamBuilder> PSH;
   std::unique_ptr<GSIHashStreamBuilder> GSH;
-  std::vector<support::ulittle32_t> PubAddrMap;
+
+  // List of all of the public records. These are stored unserialized so that we
+  // can defer copying the names until we are ready to commit the PDB.
+  std::vector<BulkPublic> Publics;
+
+  // List of all of the global records.
+  std::vector<codeview::CVSymbol> Globals;
+
+  // Hash table for deduplicating global typedef and constant records. Only used
+  // for globals.
+  llvm::DenseSet<codeview::CVSymbol, SymbolDenseMapInfo> GlobalsSeen;
 };
 
 /// This struct is equivalent to codeview::PublicSym32, but it has been
 /// optimized for size to speed up bulk serialization and sorting operations
 /// during PDB writing.
 struct BulkPublic {
+  BulkPublic() : Flags(0), BucketIdx(0) {}
+
   const char *Name = nullptr;
   uint32_t NameLen = 0;
 
@@ -93,16 +115,25 @@ struct BulkPublic {
   // Section offset of the symbol in the image.
   uint32_t Offset = 0;
 
-  union {
-    // Section index of the section containing the symbol.
-    uint16_t Segment;
+  // Section index of the section containing the symbol.
+  uint16_t Segment = 0;
+
+  // PublicSymFlags.
+  uint16_t Flags : 4;
 
-    // GSI hash table bucket index.
-    uint16_t BucketIdx;
-  } U{0};
+  // GSI hash table bucket index. The maximum value is IPHR_HASH.
+  uint16_t BucketIdx : 12;
+  static_assert(IPHR_HASH <= 1 << 12, "bitfield too small");
 
-  // PublicSymFlags or hash bucket index
-  uint16_t Flags = 0;
+  void setFlags(codeview::PublicSymFlags F) {
+    Flags = uint32_t(F);
+    assert(Flags == uint32_t(F) && "truncated");
+  }
+
+  void setBucketIdx(uint16_t B) {
+    assert(B < IPHR_HASH);
+    BucketIdx = B;
+  }
 
   StringRef getName() const { return StringRef(Name, NameLen); }
 };
diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeLineNumber.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeLineNumber.h
index f105526adf561..a7ce82c70b08c 100644
--- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeLineNumber.h
+++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeLineNumber.h
@@ -19,7 +19,8 @@ namespace pdb {
 class NativeLineNumber : public IPDBLineNumber {
 public:
   explicit NativeLineNumber(const NativeSession &Session,
-                            const codeview::LineInfo Line, uint32_t Length,
+                            const codeview::LineInfo Line,
+                            uint32_t ColumnNumber, uint32_t Length,
                             uint32_t Section, uint32_t Offset,
                             uint32_t SrcFileId);
 
@@ -39,6 +40,7 @@ class NativeLineNumber : public IPDBLineNumber {
 private:
   const NativeSession &Session;
   const codeview::LineInfo Line;
+  uint32_t ColumnNumber;
   uint32_t Section;
   uint32_t Offset;
   uint32_t Length;
diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h b/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h
index e14ef32099161..90fd19a7a2fba 100644
--- a/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h
+++ b/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h
@@ -75,6 +75,7 @@ class SymbolCache {
   struct LineTableEntry {
     uint64_t Addr;
     codeview::LineInfo Line;
+    uint32_t ColumnNumber;
     uint32_t FileNameIndex;
     bool IsTerminalEntry;
   };
diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBTypes.h b/llvm/include/llvm/DebugInfo/PDB/PDBTypes.h
index 66c842336e901..e7c2ded1bee1d 100644
--- a/llvm/include/llvm/DebugInfo/PDB/PDBTypes.h
+++ b/llvm/include/llvm/DebugInfo/PDB/PDBTypes.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_DEBUGINFO_PDB_PDBTYPES_H
 #define LLVM_DEBUGINFO_PDB_PDBTYPES_H
 
+#include "llvm/ADT/APFloat.h"
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/PDB/IPDBEnumChildren.h"
 #include "llvm/DebugInfo/PDB/IPDBFrameData.h"
@@ -464,6 +465,88 @@ struct Variant {
     char *String;
   } Value;
 
+  bool isIntegralType() const {
+    switch (Type) {
+    case Bool:
+    case Int8:
+    case Int16:
+    case Int32:
+    case Int64:
+    case UInt8:
+    case UInt16:
+    case UInt32:
+    case UInt64:
+      return true;
+    default:
+      return false;
+    }
+  }
+
+#define VARIANT_WIDTH(Enum, NumBits)                                           \
+  case PDB_VariantType::Enum:                                                  \
+    return NumBits;
+
+  unsigned getBitWidth() const {
+    switch (Type) {
+      VARIANT_WIDTH(Bool, 1u)
+      VARIANT_WIDTH(Int8, 8u)
+      VARIANT_WIDTH(Int16, 16u)
+      VARIANT_WIDTH(Int32, 32u)
+      VARIANT_WIDTH(Int64, 64u)
+      VARIANT_WIDTH(Single, 32u)
+      VARIANT_WIDTH(Double, 64u)
+      VARIANT_WIDTH(UInt8, 8u)
+      VARIANT_WIDTH(UInt16, 16u)
+      VARIANT_WIDTH(UInt32, 32u)
+      VARIANT_WIDTH(UInt64, 64u)
+    default:
+      assert(false && "Variant::toAPSInt called on non-numeric type");
+      return 0u;
+    }
+  }
+
+#undef VARIANT_WIDTH
+
+#define VARIANT_APSINT(Enum, NumBits, IsUnsigned)                              \
+  case PDB_VariantType::Enum:                                                  \
+    return APSInt(APInt(NumBits, Value.Enum), IsUnsigned);
+
+  APSInt toAPSInt() const {
+    switch (Type) {
+      VARIANT_APSINT(Bool, 1u, true)
+      VARIANT_APSINT(Int8, 8u, false)
+      VARIANT_APSINT(Int16, 16u, false)
+      VARIANT_APSINT(Int32, 32u, false)
+      VARIANT_APSINT(Int64, 64u, false)
+      VARIANT_APSINT(UInt8, 8u, true)
+      VARIANT_APSINT(UInt16, 16u, true)
+      VARIANT_APSINT(UInt32, 32u, true)
+      VARIANT_APSINT(UInt64, 64u, true)
+    default:
+      assert(false && "Variant::toAPSInt called on non-integral type");
+      return APSInt();
+    }
+  }
+
+#undef VARIANT_APSINT
+
+  APFloat toAPFloat() const {
+    // Float constants may be tagged as integers.
+    switch (Type) {
+    case PDB_VariantType::Single:
+    case PDB_VariantType::UInt32:
+    case PDB_VariantType::Int32:
+      return APFloat(Value.Single);
+    case PDB_VariantType::Double:
+    case PDB_VariantType::UInt64:
+    case PDB_VariantType::Int64:
+      return APFloat(Value.Double);
+    default:
+      assert(false && "Variant::toAPFloat called on non-floating-point type");
+      return APFloat::getZero(APFloat::IEEEsingle());
+    }
+  }
+
 #define VARIANT_EQUAL_CASE(Enum)                                               \
   case PDB_VariantType::Enum:                                                  \
     return Value.Enum == Other.Value.Enum;
diff --git a/llvm/include/llvm/Demangle/ItaniumDemangle.h b/llvm/include/llvm/Demangle/ItaniumDemangle.h
index dcece38992138..6ab873218386b 100644
--- a/llvm/include/llvm/Demangle/ItaniumDemangle.h
+++ b/llvm/include/llvm/Demangle/ItaniumDemangle.h
@@ -5096,6 +5096,22 @@ Node *AbstractManglingParser<Derived, Alloc>::parseSpecialName() {
 //            ::= <special-name>
 template <typename Derived, typename Alloc>
 Node *AbstractManglingParser<Derived, Alloc>::parseEncoding() {
+  // The template parameters of an encoding are unrelated to those of the
+  // enclosing context.
+  class SaveTemplateParams {
+    AbstractManglingParser *Parser;
+    decltype(TemplateParams) OldParams;
+
+  public:
+    SaveTemplateParams(AbstractManglingParser *Parser) : Parser(Parser) {
+      OldParams = std::move(Parser->TemplateParams);
+      Parser->TemplateParams.clear();
+    }
+    ~SaveTemplateParams() {
+      Parser->TemplateParams = std::move(OldParams);
+    }
+  } SaveTemplateParams(this);
+
   if (look() == 'G' || look() == 'T')
     return getDerived().parseSpecialName();
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
index ef251ac302df4..9ecc0464dec1b 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
@@ -18,7 +18,6 @@
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/Twine.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
 #include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
index 880c8ff44c5f0..b3e2bddd716bb 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
@@ -15,7 +15,6 @@
 
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/Twine.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/Support/Error.h"
@@ -26,6 +25,7 @@
 #include <cassert>
 #include <cstdint>
 #include <functional>
+#include <future>
 #include <map>
 #include <memory>
 #include <system_error>
@@ -42,6 +42,7 @@ class GlobalVariable;
 class Module;
 class PointerType;
 class Triple;
+class Twine;
 class Value;
 
 namespace orc {
@@ -53,6 +54,13 @@ namespace orc {
 /// are used by various ORC APIs to support lazy compilation
 class TrampolinePool {
 public:
+  using NotifyLandingResolvedFunction =
+      unique_function<void(JITTargetAddress) const>;
+
+  using ResolveLandingFunction = unique_function<void(
+      JITTargetAddress TrampolineAddr,
+      NotifyLandingResolvedFunction OnLandingResolved) const>;
+
   virtual ~TrampolinePool() {}
 
   /// Get an available trampoline address.
@@ -66,18 +74,15 @@ class TrampolinePool {
 /// A trampoline pool for trampolines within the current process.
 template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
 public:
-  using GetTrampolineLandingFunction =
-      std::function<JITTargetAddress(JITTargetAddress TrampolineAddr)>;
-
   /// Creates a LocalTrampolinePool with the given RunCallback function.
   /// Returns an error if this function is unable to correctly allocate, write
   /// and protect the resolver code block.
   static Expected<std::unique_ptr<LocalTrampolinePool>>
-  Create(GetTrampolineLandingFunction GetTrampolineLanding) {
+  Create(ResolveLandingFunction ResolveLanding) {
     Error Err = Error::success();
 
     auto LTP = std::unique_ptr<LocalTrampolinePool>(
-        new LocalTrampolinePool(std::move(GetTrampolineLanding), Err));
+        new LocalTrampolinePool(std::move(ResolveLanding), Err));
 
     if (Err)
       return std::move(Err);
@@ -108,13 +113,19 @@ template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
   static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) {
     LocalTrampolinePool<ORCABI> *TrampolinePool =
         static_cast<LocalTrampolinePool *>(TrampolinePoolPtr);
-    return TrampolinePool->GetTrampolineLanding(static_cast<JITTargetAddress>(
-        reinterpret_cast<uintptr_t>(TrampolineId)));
+
+    std::promise<JITTargetAddress> LandingAddressP;
+    auto LandingAddressF = LandingAddressP.get_future();
+
+    TrampolinePool->ResolveLanding(pointerToJITTargetAddress(TrampolineId),
+                                   [&](JITTargetAddress LandingAddress) {
+                                     LandingAddressP.set_value(LandingAddress);
+                                   });
+    return LandingAddressF.get();
   }
 
-  LocalTrampolinePool(GetTrampolineLandingFunction GetTrampolineLanding,
-                      Error &Err)
-      : GetTrampolineLanding(std::move(GetTrampolineLanding)) {
+  LocalTrampolinePool(ResolveLandingFunction ResolveLanding, Error &Err)
+      : ResolveLanding(std::move(ResolveLanding)) {
 
     ErrorAsOutParameter _(&Err);
 
@@ -173,7 +184,7 @@ template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
     return Error::success();
   }
 
-  GetTrampolineLandingFunction GetTrampolineLanding;
+  ResolveLandingFunction ResolveLanding;
 
   std::mutex LTPMutex;
   sys::OwningMemoryBlock ResolverBlock;
@@ -241,10 +252,14 @@ class LocalJITCompileCallbackManager : public JITCompileCallbackManager {
                                  JITTargetAddress ErrorHandlerAddress,
                                  Error &Err)
       : JITCompileCallbackManager(nullptr, ES, ErrorHandlerAddress) {
+    using NotifyLandingResolvedFunction =
+        TrampolinePool::NotifyLandingResolvedFunction;
+
     ErrorAsOutParameter _(&Err);
     auto TP = LocalTrampolinePool<ORCABI>::Create(
-        [this](JITTargetAddress TrampolineAddr) {
-          return executeCompileCallback(TrampolineAddr);
+        [this](JITTargetAddress TrampolineAddr,
+               NotifyLandingResolvedFunction NotifyLandingResolved) {
+          NotifyLandingResolved(executeCompileCallback(TrampolineAddr));
         });
 
     if (!TP) {
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h b/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
index 407dc6b7e34f6..01a2b9712e9a4 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
@@ -47,6 +47,9 @@ class LazyCallThroughManager {
                            NotifyResolvedFunction NotifyResolved);
 
 protected:
+  using NotifyLandingResolvedFunction =
+      TrampolinePool::NotifyLandingResolvedFunction;
+
   LazyCallThroughManager(ExecutionSession &ES,
                          JITTargetAddress ErrorHandlerAddr,
                          std::unique_ptr<TrampolinePool> TP);
@@ -56,16 +59,13 @@ class LazyCallThroughManager {
     SymbolStringPtr SymbolName;
   };
 
+  JITTargetAddress reportCallThroughError(Error Err);
   Expected<ReexportsEntry> findReexport(JITTargetAddress TrampolineAddr);
-  Expected<JITTargetAddress> resolveSymbol(const ReexportsEntry &RE);
-
   Error notifyResolved(JITTargetAddress TrampolineAddr,
                        JITTargetAddress ResolvedAddr);
-
-  JITTargetAddress reportCallThroughError(Error Err) {
-    ES.reportError(std::move(Err));
-    return ErrorHandlerAddr;
-  }
+  void resolveTrampolineLandingAddress(
+      JITTargetAddress TrampolineAddr,
+      NotifyLandingResolvedFunction NotifyLandingResolved);
 
   void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) {
     this->TP = std::move(TP);
@@ -87,14 +87,19 @@ class LazyCallThroughManager {
 /// A lazy call-through manager that builds trampolines in the current process.
 class LocalLazyCallThroughManager : public LazyCallThroughManager {
 private:
+  using NotifyTargetResolved = unique_function<void(JITTargetAddress)>;
+
   LocalLazyCallThroughManager(ExecutionSession &ES,
                               JITTargetAddress ErrorHandlerAddr)
       : LazyCallThroughManager(ES, ErrorHandlerAddr, nullptr) {}
 
   template <typename ORCABI> Error init() {
     auto TP = LocalTrampolinePool<ORCABI>::Create(
-        [this](JITTargetAddress TrampolineAddr) {
-          return callThroughToSymbol(TrampolineAddr);
+        [this](JITTargetAddress TrampolineAddr,
+               TrampolinePool::NotifyLandingResolvedFunction
+                   NotifyLandingResolved) {
+          resolveTrampolineLandingAddress(TrampolineAddr,
+                                          std::move(NotifyLandingResolved));
         });
 
     if (!TP)
@@ -104,21 +109,6 @@ class LocalLazyCallThroughManager : public LazyCallThroughManager {
     return Error::success();
   }
 
-  JITTargetAddress callThroughToSymbol(JITTargetAddress TrampolineAddr) {
-    auto Entry = findReexport(TrampolineAddr);
-    if (!Entry)
-      return reportCallThroughError(Entry.takeError());
-
-    auto ResolvedAddr = resolveSymbol(std::move(*Entry));
-    if (!ResolvedAddr)
-      return reportCallThroughError(ResolvedAddr.takeError());
-
-    if (Error Err = notifyResolved(TrampolineAddr, *ResolvedAddr))
-      return reportCallThroughError(std::move(Err));
-
-    return *ResolvedAddr;
-  }
-
 public:
   /// Create a LocalLazyCallThroughManager using the given ABI. See
   /// createLocalLazyCallThroughManager.
diff --git a/llvm/include/llvm/Frontend/Directive/DirectiveBase.td b/llvm/include/llvm/Frontend/Directive/DirectiveBase.td
new file mode 100644
index 0000000000000..3c295a1d7c5f3
--- /dev/null
+++ b/llvm/include/llvm/Frontend/Directive/DirectiveBase.td
@@ -0,0 +1,106 @@
+//===-- DirectiveBase.td - Base directive definition file --*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is the base definition file directives and clauses.
+//
+//===----------------------------------------------------------------------===//
+
+
+// General information about the directive language.
+class DirectiveLanguage {
+  // Name of the directive language such as omp or acc.
+  string name = ?;
+
+  // The C++ namespace that code of this directive language should be placed
+  // into. This namespace is nested in llvm namespace.
+  //
+  // By default, uses the name of the directive language as the only namespace.
+  // To avoid placing in any namespace, use "". To specify nested namespaces,
+  // use "::" as the delimiter, e.g., given "A::B", ops will be placed in
+  // `namespace A { namespace B { <directives-clauses> } }`.
+  string cppNamespace = name;
+
+  // Optional prefix used for the generation of the enumerator in the Directive
+  // enum.
+  string directivePrefix = "";
+
+  // Optional prefix used for the generation of the enumerator in the Clause
+  // enum.
+  string clausePrefix = "";
+
+  // Make the enum values available in the namespace. This allows us to
+  // write something like Enum_X if we have a `using namespace cppNamespace`.
+  bit makeEnumAvailableInNamespace = 0;
+
+  // Generate include and macro to enable LLVM BitmaskEnum.
+  bit enableBitmaskEnumInNamespace = 0;
+
+  // Header file included in the implementation code generated. Ususally the
+  // output file of the declaration code generation. Can be left blank.
+  string includeHeader = "";
+
+  // EnumSet class name used for clauses to generated the allowed clauses map.
+  string clauseEnumSetClass = "";
+}
+
+// Information about a specific clause.
+class Clause<string c> {
+  // Name of the clause.
+  string name = c;
+
+  // Define an alternative name return in get<LanguageName>ClauseName function.
+  string alternativeName = "";
+
+  // Optional class holding value of the clause in clang AST.
+  string clangClass = ?;
+
+  // Is clause implicit? If clause is set as implicit, the default kind will
+  // be return in get<LanguageName>ClauseKind instead of their own kind.
+  bit isImplicit = 0;
+
+  // Set directive used by default when unknown. Function returning the kind
+  // of enumeration will use this clause as the default.
+  bit isDefault = 0;
+}
+
+// Hold information about clause validity by version.
+class VersionedClause<Clause c, int min = 1, int max = 0x7FFFFFFF> {
+  // Actual clause.
+  Clause clause = c;
+
+  // Mininum version number where this clause is valid.
+  int minVersion = min;
+
+  // Maximum version number where this clause is valid.
+  int maxVersion = max;
+}
+
+// Information about a specific directive.
+class Directive<string d> {
+  // Name of the directive. Can be composite directive sepearted by whitespace.
+  string name = d;
+
+  // Define an alternative name return in get<LanguageName>DirectiveName
+  // function.
+  string alternativeName = "";
+
+  // List of allowed clauses for the directive.
+  list<VersionedClause> allowedClauses = [];
+
+  // List of clauses that are allowed to appear only once.
+  list<VersionedClause> allowedOnceClauses = [];
+
+  // List of clauses that are allowed but mutually exclusive.
+  list<VersionedClause> allowedExclusiveClauses = [];
+
+  // List of clauses that are required.
+  list<VersionedClause> requiredClauses = [];
+
+  // Set directive used by default when unknown.
+  bit isDefault = 0;
+}
diff --git a/llvm/include/llvm/Frontend/OpenMP/CMakeLists.txt b/llvm/include/llvm/Frontend/OpenMP/CMakeLists.txt
new file mode 100644
index 0000000000000..3ff89888bfd64
--- /dev/null
+++ b/llvm/include/llvm/Frontend/OpenMP/CMakeLists.txt
@@ -0,0 +1,4 @@
+set(LLVM_TARGET_DEFINITIONS OMP.td)
+tablegen(LLVM OMP.h.inc --gen-directive-decl)
+tablegen(LLVM OMP.cpp.inc --gen-directive-gen)
+add_public_tablegen_target(omp_gen)
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
new file mode 100644
index 0000000000000..a565bdf90b3f6
--- /dev/null
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -0,0 +1,1489 @@
+//===-- OMP.td - OpenMP directive definition file ----------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is the definition file for OpenMP directives and clauses.
+//
+//===----------------------------------------------------------------------===//
+
+include "llvm/Frontend/Directive/DirectiveBase.td"
+
+//===----------------------------------------------------------------------===//
+// Definition of general OpenMP information
+//===----------------------------------------------------------------------===//
+
+def OpenMP : DirectiveLanguage {
+  let name = "OpenMP";
+  let cppNamespace = "omp"; // final namespace will be llvm::omp
+  let directivePrefix = "OMPD_";
+  let clausePrefix = "OMPC_";
+  let makeEnumAvailableInNamespace = 1;
+  let enableBitmaskEnumInNamespace = 1;
+  let includeHeader = "llvm/Frontend/OpenMP/OMP.h.inc";
+  let clauseEnumSetClass = "OmpClauseSet";
+}
+
+//===----------------------------------------------------------------------===//
+// Definition of OpenMP clauses
+//===----------------------------------------------------------------------===//
+
+def OMPC_Allocator : Clause<"allocator"> {
+  let clangClass = "OMPAllocatorClause";
+}
+def OMPC_If : Clause<"if"> { let clangClass = "OMPIfClause"; }
+def OMPC_Final : Clause<"final"> { let clangClass = "OMPFinalClause"; }
+def OMPC_NumThreads : Clause<"num_threads"> {
+  let clangClass = "OMPNumThreadsClause";
+}
+def OMPC_SafeLen : Clause<"safelen"> { let clangClass = "OMPSafelenClause"; }
+def OMPC_SimdLen : Clause<"simdlen"> { let clangClass = "OMPSimdlenClause"; }
+def OMPC_Collapse : Clause<"collapse"> { let clangClass = "OMPCollapseClause"; }
+def OMPC_Default : Clause<"default"> { let clangClass = "OMPDefaultClause"; }
+def OMPC_Private : Clause<"private"> { let clangClass = "OMPPrivateClause"; }
+def OMPC_FirstPrivate : Clause<"firstprivate"> {
+  let clangClass = "OMPFirstprivateClause";
+}
+def OMPC_LastPrivate : Clause<"lastprivate"> {
+  let clangClass = "OMPLastprivateClause";
+}
+def OMPC_Shared : Clause<"shared"> { let clangClass = "OMPSharedClause"; }
+def OMPC_Reduction : Clause<"reduction"> {
+  let clangClass = "OMPReductionClause";
+}
+def OMPC_Linear : Clause<"linear"> { let clangClass = "OMPLinearClause"; }
+def OMPC_Aligned : Clause<"aligned"> { let clangClass = "OMPAlignedClause"; }
+def OMPC_Copyin : Clause<"copyin"> { let clangClass = "OMPCopyinClause"; }
+def OMPC_CopyPrivate : Clause<"copyprivate"> {
+  let clangClass = "OMPCopyprivateClause";
+}
+def OMPC_ProcBind : Clause<"proc_bind"> {
+  let clangClass = "OMPProcBindClause";
+}
+def OMPC_Schedule : Clause<"schedule"> { let clangClass = "OMPScheduleClause"; }
+def OMPC_Ordered : Clause<"ordered"> { let clangClass = "OMPOrderedClause"; }
+def OMPC_NoWait : Clause<"nowait"> { let clangClass = "OMPNowaitClause"; }
+def OMPC_Untied : Clause<"untied"> { let clangClass = "OMPUntiedClause"; }
+def OMPC_Mergeable : Clause<"mergeable"> {
+  let clangClass = "OMPMergeableClause";
+}
+def OMPC_Read : Clause<"read"> { let clangClass = "OMPReadClause"; }
+def OMPC_Write : Clause<"write"> { let clangClass = "OMPWriteClause"; }
+def OMPC_Update : Clause<"update"> { let clangClass = "OMPUpdateClause"; }
+def OMPC_Capture : Clause<"capture"> { let clangClass = "OMPCaptureClause"; }
+def OMPC_SeqCst : Clause<"seq_cst"> { let clangClass = "OMPSeqCstClause"; }
+def OMPC_AcqRel : Clause<"acq_rel"> { let clangClass = "OMPAcqRelClause"; }
+def OMPC_Acquire : Clause<"acquire"> { let clangClass = "OMPAcquireClause"; }
+def OMPC_Release : Clause<"release"> { let clangClass = "OMPReleaseClause"; }
+def OMPC_Relaxed : Clause<"relaxed"> { let clangClass = "OMPRelaxedClause"; }
+def OMPC_Depend : Clause<"depend"> { let clangClass = "OMPDependClause"; }
+def OMPC_Device : Clause<"device"> { let clangClass = "OMPDeviceClause"; }
+def OMPC_Threads : Clause<"threads"> { let clangClass = "OMPThreadsClause"; }
+def OMPC_Simd : Clause<"simd"> { let clangClass = "OMPSIMDClause"; }
+def OMPC_Map : Clause<"map"> { let clangClass = "OMPMapClause"; }
+def OMPC_NumTeams : Clause<"num_teams"> {
+  let clangClass = "OMPNumTeamsClause";
+}
+def OMPC_ThreadLimit : Clause<"thread_limit"> {
+  let clangClass = "OMPThreadLimitClause";
+}
+def OMPC_Priority : Clause<"priority"> {
+  let clangClass = "OMPPriorityClause";
+}
+def OMPC_GrainSize : Clause<"grainsize"> {
+  let clangClass = "OMPGrainsizeClause";
+}
+def OMPC_NoGroup : Clause<"nogroup"> {
+  let clangClass = "OMPNogroupClause";
+}
+def OMPC_NumTasks : Clause<"num_tasks"> {
+  let clangClass = "OMPNumTasksClause";
+}
+def OMPC_Hint : Clause<"hint"> {
+  let clangClass = "OMPHintClause";
+}
+def OMPC_DistSchedule : Clause<"dist_schedule"> {
+  let clangClass = "OMPDistScheduleClause";
+}
+def OMPC_DefaultMap : Clause<"defaultmap"> {
+  let clangClass = "OMPDefaultmapClause";
+}
+def OMPC_To : Clause<"to"> {
+  let clangClass = "OMPToClause";
+}
+def OMPC_From : Clause<"from"> { let clangClass = "OMPFromClause"; }
+def OMPC_UseDevicePtr : Clause<"use_device_ptr"> {
+  let clangClass = "OMPUseDevicePtrClause";
+}
+def OMPC_IsDevicePtr : Clause<"is_device_ptr"> {
+  let clangClass = "OMPIsDevicePtrClause";
+}
+def OMPC_TaskReduction : Clause<"task_reduction"> {
+  let clangClass = "OMPTaskReductionClause";
+}
+def OMPC_InReduction : Clause<"in_reduction"> {
+  let clangClass = "OMPInReductionClause";
+}
+def OMPC_UnifiedAddress : Clause<"unified_address"> {
+  let clangClass = "OMPUnifiedAddressClause";
+}
+def OMPC_UnifiedSharedMemory : Clause<"unified_shared_memory"> {
+  let clangClass = "OMPUnifiedSharedMemoryClause";
+}
+def OMPC_ReverseOffload : Clause<"reverse_offload"> {
+  let clangClass = "OMPReverseOffloadClause";
+}
+def OMPC_DynamicAllocators : Clause<"dynamic_allocators"> {
+  let clangClass = "OMPDynamicAllocatorsClause";
+}
+def OMPC_AtomicDefaultMemOrder : Clause<"atomic_default_mem_order"> {
+  let clangClass = "OMPAtomicDefaultMemOrderClause";
+}
+def OMPC_Allocate : Clause<"allocate"> {
+  let clangClass = "OMPAllocateClause";
+}
+def OMPC_NonTemporal : Clause<"nontemporal"> {
+  let clangClass = "OMPNontemporalClause";
+}
+def OMPC_Order : Clause<"order"> {
+  let clangClass = "OMPOrderClause";
+}
+def OMPC_Destroy : Clause<"destroy"> {
+  let clangClass = "OMPDestroyClause";
+}
+def OMPC_Detach : Clause<"detach"> {
+  let clangClass = "OMPDetachClause";
+}
+def OMPC_Inclusive : Clause<"inclusive"> {
+  let clangClass = "OMPInclusiveClause";
+}
+def OMPC_Exclusive : Clause<"exclusive"> {
+  let clangClass = "OMPExclusiveClause";
+}
+def OMPC_UsesAllocators : Clause<"uses_allocators"> {
+  let clangClass = "OMPUsesAllocatorsClause";
+}
+def OMPC_Affinity : Clause<"affinity"> {
+  let clangClass = "OMPAffinityClause";
+}
+def OMPC_UseDeviceAddr : Clause<"use_device_addr"> {
+  let clangClass = "OMPUseDeviceAddrClause";
+}
+def OMPC_Uniform : Clause<"uniform"> {}
+def OMPC_DeviceType : Clause<"device_type"> {}
+def OMPC_Match : Clause<"match"> {}
+def OMPC_Depobj : Clause<"depobj"> {
+  let clangClass = "OMPDepobjClause";
+  let isImplicit = 1;
+}
+def OMPC_Flush : Clause<"flush"> {
+  let clangClass = "OMPFlushClause";
+  let isImplicit = 1;
+}
+def OMPC_ThreadPrivate : Clause<"threadprivate"> {
+  let alternativeName = "threadprivate or thread local";
+  let isImplicit = 1;
+}
+def OMPC_Unknown : Clause<"unknown"> {
+  let isImplicit = 1;
+  let isDefault = 1;
+}
+def OMPC_Link : Clause<"link"> {}
+def OMPC_Inbranch : Clause<"inbranch"> {}
+def OMPC_Notinbranch : Clause<"notinbranch"> {}
+
+//===----------------------------------------------------------------------===//
+// Definition of OpenMP directives
+//===----------------------------------------------------------------------===//
+
+def OMP_ThreadPrivate : Directive<"threadprivate"> {}
+def OMP_Parallel : Directive<"parallel"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_Allocate>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_ProcBind>,
+  ];
+}
+def OMP_Task : Directive<"task"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Untied>,
+    VersionedClause<OMPC_Mergeable>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_InReduction>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Detach, 50>,
+    VersionedClause<OMPC_Affinity, 50>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Final>,
+    VersionedClause<OMPC_Priority>
+  ];
+}
+def OMP_Simd : Directive<"simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_NonTemporal, 50>,
+    VersionedClause<OMPC_Order, 50>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+    VersionedClause<OMPC_If, 50>,
+  ];
+}
+def OMP_For : Directive<"for"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Ordered>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Order, 50>
+  ];
+}
+def OMP_Do : Directive<"do"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Reduction>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_Ordered>,
+    VersionedClause<OMPC_NoWait>
+  ];
+}
+def OMP_Sections : Directive<"sections"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_Allocate>
+  ];
+}
+def OMP_Section : Directive<"section"> {}
+def OMP_Single : Directive<"single"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_CopyPrivate>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_Allocate>
+  ];
+}
+def OMP_Master : Directive<"master"> {}
+def OMP_Critical : Directive<"critical"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Hint>
+  ];
+}
+def OMP_TaskYield : Directive<"taskyield"> {}
+def OMP_Barrier : Directive<"barrier"> {}
+def OMP_TaskWait : Directive<"taskwait"> {}
+def OMP_TaskGroup : Directive<"taskgroup"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_TaskReduction>,
+    VersionedClause<OMPC_Allocate>
+  ];
+}
+def OMP_Flush : Directive<"flush"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_AcqRel, 50>,
+    VersionedClause<OMPC_Acquire, 50>,
+    VersionedClause<OMPC_Release, 50>,
+    // TODO This should ne `none` instead. Comment carried over from
+    // OMPKinds.def.
+    VersionedClause<OMPC_Flush>
+  ];
+}
+def OMP_Ordered : Directive<"ordered"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Threads>,
+    VersionedClause<OMPC_Simd>,
+    VersionedClause<OMPC_Depend>
+  ];
+}
+def OMP_Atomic : Directive<"atomic"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Read>,
+    VersionedClause<OMPC_Write>,
+    VersionedClause<OMPC_Update>,
+    VersionedClause<OMPC_Capture>,
+    VersionedClause<OMPC_SeqCst>,
+    VersionedClause<OMPC_AcqRel, 50>,
+    VersionedClause<OMPC_Acquire, 50>,
+    VersionedClause<OMPC_Release, 50>,
+    VersionedClause<OMPC_Relaxed, 50>,
+    VersionedClause<OMPC_Hint, 50>
+  ];
+}
+def OMP_Target : Directive<"target"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_UsesAllocators, 50>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_NoWait>
+  ];
+}
+def OMP_Teams : Directive<"teams"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Allocate>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>
+  ];
+}
+def OMP_Cancel : Directive<"cancel"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>
+  ];
+}
+def OMP_Requires : Directive<"requires"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_UnifiedAddress>,
+    VersionedClause<OMPC_UnifiedSharedMemory>,
+    VersionedClause<OMPC_ReverseOffload>,
+    VersionedClause<OMPC_DynamicAllocators>,
+    VersionedClause<OMPC_AtomicDefaultMemOrder>
+  ];
+}
+def OMP_TargetData : Directive<"target data"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_UseDevicePtr>,
+    VersionedClause<OMPC_UseDeviceAddr, 50>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_If>
+  ];
+  let requiredClauses = [
+    VersionedClause<OMPC_Map>
+  ];
+}
+def OMP_TargetEnterData : Directive<"target enter data"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_Map>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_NoWait>
+  ];
+}
+def OMP_TargetExitData : Directive<"target exit data"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_Map>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NoWait>
+  ];
+  let requiredClauses = [
+    VersionedClause<OMPC_Map>
+  ];
+}
+def OMP_TargetParallel : Directive<"target parallel"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_UsesAllocators, 50>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_ProcBind>
+  ];
+}
+def OMP_TargetParallelFor : Directive<"target parallel for"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Ordered>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Order, 50>,
+    VersionedClause<OMPC_UsesAllocators, 50>
+  ];
+}
+def OMP_TargetParallelDo : Directive<"target parallel do"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Allocator>,
+    VersionedClause<OMPC_Order>,
+    VersionedClause<OMPC_UsesAllocators>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Copyin>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_Ordered>,
+    VersionedClause<OMPC_NoWait>
+  ];
+}
+def OMP_TargetUpdate : Directive<"target update"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_To>,
+    VersionedClause<OMPC_From>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_Depend>
+  ];
+}
+def OMP_ParallelFor : Directive<"parallel for"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Ordered>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Order, 50>
+  ];
+}
+def OMP_ParallelDo : Directive<"parallel do"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Linear>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Ordered>,
+    VersionedClause<OMPC_Collapse>
+  ];
+}
+def OMP_ParallelForSimd : Directive<"parallel for simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_Ordered>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_NonTemporal, 50>,
+    VersionedClause<OMPC_Order, 50>
+  ];
+}
+def OMP_ParallelDoSimd : Directive<"parallel do simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_NonTemporal>,
+    VersionedClause<OMPC_Order>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Ordered>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>
+  ];
+}
+def OMP_ParallelMaster : Directive<"parallel master"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Allocate>
+  ];
+}
+def OMP_ParallelSections : Directive<"parallel sections"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Allocate>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_NumThreads>
+  ];
+}
+def OMP_ForSimd : Directive<"for simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_Ordered>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_If, 50>,
+    VersionedClause<OMPC_NonTemporal, 50>,
+    VersionedClause<OMPC_Order, 50>,
+  ];
+}
+def OMP_DoSimd : Directive<"do simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Reduction>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_Ordered>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+    VersionedClause<OMPC_NoWait>
+  ];
+}
+def OMP_CancellationPoint : Directive<"cancellation point"> {}
+def OMP_DeclareReduction : Directive<"declare reduction"> {}
+def OMP_DeclareMapper : Directive<"declare mapper"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Map>
+  ];
+}
+def OMP_DeclareSimd : Directive<"declare simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_Uniform>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_SimdLen>
+  ];
+  let allowedExclusiveClauses = [
+    VersionedClause<OMPC_Inbranch>,
+    VersionedClause<OMPC_Notinbranch>
+  ];
+}
+def OMP_TaskLoop : Directive<"taskloop"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Untied>,
+    VersionedClause<OMPC_Mergeable>,
+    VersionedClause<OMPC_NoGroup>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_InReduction>,
+    VersionedClause<OMPC_Allocate>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_Final>,
+    VersionedClause<OMPC_Priority>,
+  ];
+  let allowedExclusiveClauses = [
+    VersionedClause<OMPC_GrainSize>,
+    VersionedClause<OMPC_NumTasks>
+  ];
+}
+def OMP_TaskLoopSimd : Directive<"taskloop simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_InReduction>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Mergeable>,
+    VersionedClause<OMPC_NoGroup>,
+    VersionedClause<OMPC_NonTemporal, 50>,
+    VersionedClause<OMPC_Order, 50>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Untied>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+    VersionedClause<OMPC_Final>,
+    VersionedClause<OMPC_Priority>
+  ];
+  let allowedExclusiveClauses = [
+    VersionedClause<OMPC_GrainSize>,
+    VersionedClause<OMPC_NumTasks>
+  ];
+}
+def OMP_Distribute : Directive<"distribute"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Allocate>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>
+  ];
+}
+def OMP_DeclareTarget : Directive<"declare target"> {}
+def OMP_EndDeclareTarget : Directive<"end declare target"> {}
+def OMP_DistributeParallelFor : Directive<"distribute parallel for"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Order, 50>
+  ];
+}
+def OMP_DistributeParallelDo : Directive<"distribute parallel do"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Order>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_Linear>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Ordered>
+  ];
+}
+def OMP_DistributeParallelForSimd : Directive<"distribute parallel for simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_NonTemporal, 50>,
+    VersionedClause<OMPC_Order, 50>
+  ];
+}
+def OMP_DistributeParallelDoSimd : Directive<"distribute parallel do simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_NonTemporal>,
+    VersionedClause<OMPC_Order>
+  ];
+}
+def OMP_DistributeSimd : Directive<"distribute simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_NonTemporal, 50>,
+    VersionedClause<OMPC_Order, 50>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_If, 50>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_Ordered>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>
+  ];
+}
+
+def OMP_TargetParallelForSimd : Directive<"target parallel for simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Ordered>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_NonTemporal, 50>,
+    VersionedClause<OMPC_Order, 50>,
+    VersionedClause<OMPC_UsesAllocators, 50>
+  ];
+}
+def OMP_TargetParallelDoSimd : Directive<"target parallel do simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Ordered>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_NonTemporal>,
+    VersionedClause<OMPC_Order>,
+    VersionedClause<OMPC_UsesAllocators>
+  ];
+}
+def OMP_TargetSimd : Directive<"target simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_NonTemporal, 50>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_Order, 50>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_UsesAllocators, 50>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_Schedule>
+  ];
+}
+def OMP_TeamsDistribute : Directive<"teams distribute"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_Allocate>
+  ];
+}
+def OMP_TeamsDistributeSimd : Directive<"teams distribute simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_NonTemporal, 50>,
+    VersionedClause<OMPC_Order, 50>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_If, 50>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+    VersionedClause<OMPC_ThreadLimit>
+  ];
+}
+
+def OMP_TeamsDistributeParallelForSimd :
+    Directive<"teams distribute parallel for simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_NonTemporal, 50>,
+    VersionedClause<OMPC_Order, 50>
+  ];
+}
+def OMP_TeamsDistributeParallelDoSimd :
+    Directive<"teams distribute parallel do simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Order>,
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_NonTemporal>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+    VersionedClause<OMPC_If>,
+  ];
+}
+def OMP_TeamsDistributeParallelFor :
+    Directive<"teams distribute parallel for"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Order, 50>
+  ];
+}
+def OMP_TeamsDistributeParallelDo :
+    Directive<"teams distribute parallel do"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_Linear>
+  ];
+let allowedOnceClauses = [
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_Ordered>,
+    VersionedClause<OMPC_Order>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Schedule>
+  ];
+}
+def OMP_TargetTeams : Directive<"target teams"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_UsesAllocators, 50>,
+    VersionedClause<OMPC_Shared>
+  ];
+
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>
+  ];
+}
+def OMP_TargetTeamsDistribute : Directive<"target teams distribute"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_UsesAllocators, 50>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_LastPrivate>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>
+  ];
+}
+
+def OMP_TargetTeamsDistributeParallelFor :
+    Directive<"target teams distribute parallel for"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Order, 50>,
+    VersionedClause<OMPC_UsesAllocators, 50>
+  ];
+}
+def OMP_TargetTeamsDistributeParallelDo :
+    Directive<"target teams distribute parallel do"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_UsesAllocators>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Ordered>,
+    VersionedClause<OMPC_Order>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Schedule>,
+  ];
+}
+def OMP_TargetTeamsDistributeParallelForSimd :
+    Directive<"target teams distribute parallel for simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_NonTemporal, 50>,
+    VersionedClause<OMPC_Order, 50>,
+    VersionedClause<OMPC_UsesAllocators, 50>
+  ];
+}
+def OMP_TargetTeamsDistributeParallelDoSimd :
+    Directive<"target teams distribute parallel do simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_UsesAllocators>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Ordered>,
+    VersionedClause<OMPC_Order>,
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_NonTemporal>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Schedule>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>
+  ];
+}
+def OMP_TargetTeamsDistributeSimd :
+    Directive<"target teams distribute simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_NonTemporal, 50>,
+    VersionedClause<OMPC_Order, 50>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_UsesAllocators, 50>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+  ];
+}
+def OMP_Allocate : Directive<"allocate"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Allocator>
+  ];
+}
+def OMP_DeclareVariant : Directive<"declare variant"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Match>
+  ];
+}
+def OMP_MasterTaskloop : Directive<"master taskloop"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_Final>,
+    VersionedClause<OMPC_Untied>,
+    VersionedClause<OMPC_Mergeable>,
+    VersionedClause<OMPC_Priority>,
+    VersionedClause<OMPC_GrainSize>,
+    VersionedClause<OMPC_NoGroup>,
+    VersionedClause<OMPC_NumTasks>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_InReduction>,
+    VersionedClause<OMPC_Allocate>
+  ];
+}
+def OMP_ParallelMasterTaskloop :
+    Directive<"parallel master taskloop"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_Final>,
+    VersionedClause<OMPC_Untied>,
+    VersionedClause<OMPC_Mergeable>,
+    VersionedClause<OMPC_Priority>,
+    VersionedClause<OMPC_GrainSize>,
+    VersionedClause<OMPC_NoGroup>,
+    VersionedClause<OMPC_NumTasks>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Copyin>
+  ];
+}
+def OMP_MasterTaskloopSimd : Directive<"master taskloop simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_Final>,
+    VersionedClause<OMPC_Untied>,
+    VersionedClause<OMPC_Mergeable>,
+    VersionedClause<OMPC_Priority>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+    VersionedClause<OMPC_GrainSize>,
+    VersionedClause<OMPC_NoGroup>,
+    VersionedClause<OMPC_NumTasks>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_InReduction>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_NonTemporal, 50>,
+    VersionedClause<OMPC_Order, 50>
+  ];
+}
+def OMP_ParallelMasterTaskloopSimd :
+    Directive<"parallel master taskloop simd"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_LastPrivate>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_Collapse>,
+    VersionedClause<OMPC_Final>,
+    VersionedClause<OMPC_Untied>,
+    VersionedClause<OMPC_Mergeable>,
+    VersionedClause<OMPC_Priority>,
+    VersionedClause<OMPC_GrainSize>,
+    VersionedClause<OMPC_NoGroup>,
+    VersionedClause<OMPC_NumTasks>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_ProcBind>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_Linear>,
+    VersionedClause<OMPC_Aligned>,
+    VersionedClause<OMPC_SafeLen>,
+    VersionedClause<OMPC_SimdLen>,
+    VersionedClause<OMPC_NonTemporal, 50>,
+    VersionedClause<OMPC_Order, 50>
+  ];
+}
+def OMP_Depobj : Directive<"depobj"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Depend, 50>,
+    VersionedClause<OMPC_Destroy, 50>,
+    VersionedClause<OMPC_Update, 50>,
+    // TODO This should ne `none` instead. Comment carried over from
+    // OMPKinds.def.
+    VersionedClause<OMPC_Depobj, 50>
+  ];
+}
+def OMP_Scan : Directive<"scan"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Inclusive, 50>,
+    VersionedClause<OMPC_Exclusive, 50>
+  ];
+}
+def OMP_BeginDeclareVariant : Directive<"begin declare variant"> {}
+def OMP_EndDeclareVariant : Directive<"end declare variant"> {}
+def OMP_ParallelWorkshare : Directive<"parallel workshare"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_NumThreads>,
+    VersionedClause<OMPC_ProcBind>
+  ];
+}
+def OMP_Workshare : Directive<"workshare"> {}
+def OMP_EndDo : Directive<"end do"> {}
+def OMP_EndDoSimd : Directive<"end do simd"> {}
+def OMP_EndSections : Directive<"end sections"> {}
+def OMP_EndSingle : Directive<"end single"> {}
+def OMP_EndWorkshare : Directive<"end workshare"> {}
+def OMP_Unknown : Directive<"unknown"> {
+  let isDefault = 1;
+}
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h
index 1c3cd4c71c9c7..d171d0a2b6c44 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h
@@ -16,6 +16,8 @@
 
 #include "llvm/ADT/BitmaskEnum.h"
 
+#include "llvm/Frontend/OpenMP/OMP.h.inc"
+
 namespace llvm {
 class Type;
 class Module;
@@ -28,24 +30,23 @@ class FunctionType;
 namespace omp {
 LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
 
-/// IDs for all OpenMP directives.
-enum class Directive {
-#define OMP_DIRECTIVE(Enum, ...) Enum,
+/// IDs for all Internal Control Variables (ICVs).
+enum class InternalControlVar {
+#define ICV_DATA_ENV(Enum, ...) Enum,
 #include "llvm/Frontend/OpenMP/OMPKinds.def"
 };
 
-/// IDs for all OpenMP clauses.
-enum class Clause {
-#define OMP_CLAUSE(Enum, ...) Enum,
+#define ICV_DATA_ENV(Enum, ...)                                                \
+  constexpr auto Enum = omp::InternalControlVar::Enum;
+#include "llvm/Frontend/OpenMP/OMPKinds.def"
+
+enum class ICVInitValue {
+#define ICV_DATA_ENV(Enum, Name, EnvVar, Init) Init,
 #include "llvm/Frontend/OpenMP/OMPKinds.def"
 };
 
-/// Make the enum values available in the llvm::omp namespace. This allows us to
-/// write something like OMPD_parallel if we have a `using namespace omp`. At
-/// the same time we do not loose the strong type guarantees of the enum class,
-/// that is we cannot pass an unsigned as Directive without an explicit cast.
-#define OMP_DIRECTIVE(Enum, ...) constexpr auto Enum = omp::Directive::Enum;
-#define OMP_CLAUSE(Enum, ...) constexpr auto Enum = omp::Clause::Enum;
+#define ICV_DATA_ENV(Enum, Name, EnvVar, Init)                                 \
+  constexpr auto Init = omp::ICVInitValue::Init;
 #include "llvm/Frontend/OpenMP/OMPKinds.def"
 
 /// IDs for all omp runtime library (RTL) functions.
@@ -88,50 +89,6 @@ enum class IdentFlag {
 #define OMP_IDENT_FLAG(Enum, ...) constexpr auto Enum = omp::IdentFlag::Enum;
 #include "llvm/Frontend/OpenMP/OMPKinds.def"
 
-/// Parse \p Str and return the directive it matches or OMPD_unknown if none.
-Directive getOpenMPDirectiveKind(StringRef Str);
-
-/// Return a textual representation of the directive \p D.
-StringRef getOpenMPDirectiveName(Directive D);
-
-/// Parse \p Str and return the clause it matches or OMPC_unknown if none.
-Clause getOpenMPClauseKind(StringRef Str);
-
-/// Return a textual representation of the clause \p C.
-StringRef getOpenMPClauseName(Clause C);
-
-/// Return true if \p C is a valid clause for \p D in version \p Version.
-bool isAllowedClauseForDirective(Directive D, Clause C, unsigned Version);
-
-/// Forward declarations for LLVM-IR types (simple, function and structure) are
-/// generated below. Their names are defined and used in OpenMP/OMPKinds.def.
-/// Here we provide the forward declarations, the initializeTypes function will
-/// provide the values.
-///
-///{
-namespace types {
-
-#define OMP_TYPE(VarName, InitValue) extern Type *VarName;
-#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize)                             \
-  extern ArrayType *VarName##Ty;                                               \
-  extern PointerType *VarName##PtrTy;
-#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...)                  \
-  extern FunctionType *VarName;                                                \
-  extern PointerType *VarName##Ptr;
-#define OMP_STRUCT_TYPE(VarName, StrName, ...)                                 \
-  extern StructType *VarName;                                                  \
-  extern PointerType *VarName##Ptr;
-#include "llvm/Frontend/OpenMP/OMPKinds.def"
-
-/// Helper to initialize all types defined in OpenMP/OMPKinds.def.
-void initializeTypes(Module &M);
-
-/// Helper to uninitialize all types defined in OpenMP/OMPKinds.def.
-void uninitializeTypes();
-
-} // namespace types
-///}
-
 } // end namespace omp
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index d684e181a03ea..2a3a64a5f4ac2 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -39,7 +39,7 @@ class OpenMPIRBuilder {
   void finalize();
 
   /// Add attributes known for \p FnID to \p Fn.
-  static void addAttributes(omp::RuntimeFunction FnID, Function &Fn);
+  void addAttributes(omp::RuntimeFunction FnID, Function &Fn);
 
   /// Type used throughout for insertion points.
   using InsertPointTy = IRBuilder<>::InsertPoint;
@@ -199,8 +199,8 @@ class OpenMPIRBuilder {
   }
 
   /// Return the function declaration for the runtime function with \p FnID.
-  static FunctionCallee getOrCreateRuntimeFunction(Module &M,
-                                                   omp::RuntimeFunction FnID);
+  FunctionCallee getOrCreateRuntimeFunction(Module &M,
+                                            omp::RuntimeFunction FnID);
 
   Function *getOrCreateRuntimeFunctionPtr(omp::RuntimeFunction FnID);
 
@@ -315,7 +315,7 @@ class OpenMPIRBuilder {
                              BodyGenCallbackTy BodyGenCB,
                              FinalizeCallbackTy FiniCB);
 
-  /// Generator for '#omp master'
+  /// Generator for '#omp critical'
   ///
   /// \param Loc The insert and source location description.
   /// \param BodyGenCB Callback that will generate the region body code.
@@ -329,7 +329,83 @@ class OpenMPIRBuilder {
                                FinalizeCallbackTy FiniCB,
                                StringRef CriticalName, Value *HintInst);
 
+  /// Generate conditional branch and relevant BasicBlocks through which private
+  /// threads copy the 'copyin' variables from Master copy to threadprivate
+  /// copies.
+  ///
+  /// \param IP insertion block for copyin conditional
+  /// \param MasterVarPtr a pointer to the master variable
+  /// \param PrivateVarPtr a pointer to the threadprivate variable
+  /// \param IntPtrTy Pointer size type
+  /// \param BranchtoEnd Create a branch between the copyin.not.master blocks
+  //				 and copy.in.end block
+  ///
+  /// \returns The insertion point where copying operation to be emitted.
+  InsertPointTy CreateCopyinClauseBlocks(InsertPointTy IP, Value *MasterAddr,
+                                         Value *PrivateAddr,
+                                         llvm::IntegerType *IntPtrTy,
+                                         bool BranchtoEnd = true);
+
+  /// Create a runtime call for kmpc_Alloc
+  ///
+  /// \param Loc The insert and source location description.
+  /// \param Size Size of allocated memory space
+  /// \param Allocator Allocator information instruction
+  /// \param Name Name of call Instruction for OMP_alloc
+  ///
+  /// \returns CallInst to the OMP_Alloc call
+  CallInst *CreateOMPAlloc(const LocationDescription &Loc, Value *Size,
+                           Value *Allocator, std::string Name = "");
+
+  /// Create a runtime call for kmpc_free
+  ///
+  /// \param Loc The insert and source location description.
+  /// \param Addr Address of memory space to be freed
+  /// \param Allocator Allocator information instruction
+  /// \param Name Name of call Instruction for OMP_Free
+  ///
+  /// \returns CallInst to the OMP_Free call
+  CallInst *CreateOMPFree(const LocationDescription &Loc, Value *Addr,
+                          Value *Allocator, std::string Name = "");
+
+  /// Create a runtime call for kmpc_threadprivate_cached
+  ///
+  /// \param Loc The insert and source location description.
+  /// \param Pointer pointer to data to be cached
+  /// \param Size size of data to be cached
+  /// \param Name Name of call Instruction for callinst
+  ///
+  /// \returns CallInst to the thread private cache call.
+  CallInst *CreateCachedThreadPrivate(const LocationDescription &Loc,
+                                      llvm::Value *Pointer,
+                                      llvm::ConstantInt *Size,
+                                      const llvm::Twine &Name = Twine(""));
+
+  /// Declarations for LLVM-IR types (simple, array, function and structure) are
+  /// generated below. Their names are defined and used in OpenMPKinds.def. Here
+  /// we provide the declarations, the initializeTypes function will provide the
+  /// values.
+  ///
+  ///{
+#define OMP_TYPE(VarName, InitValue) Type *VarName = nullptr;
+#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize)                             \
+  ArrayType *VarName##Ty = nullptr;                                            \
+  PointerType *VarName##PtrTy = nullptr;
+#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...)                  \
+  FunctionType *VarName = nullptr;                                             \
+  PointerType *VarName##Ptr = nullptr;
+#define OMP_STRUCT_TYPE(VarName, StrName, ...)                                 \
+  StructType *VarName = nullptr;                                               \
+  PointerType *VarName##Ptr = nullptr;
+#include "llvm/Frontend/OpenMP/OMPKinds.def"
+
+  ///}
+
 private:
+  /// Create all simple and struct types exposed by the runtime and remember
+  /// the llvm::PointerTypes of them for easy access later.
+  void initializeTypes(Module &M);
+
   /// Common interface for generating entry calls for OMP Directives.
   /// if the directive has a region/body, It will set the insertion
   /// point to the body
@@ -387,7 +463,7 @@ class OpenMPIRBuilder {
   /// \param Parts different parts of the final name that needs separation
   /// \param FirstSeparator First separator used between the initial two
   ///        parts of the name.
-  /// \param Separator separator used between all of the rest consecutinve
+  /// \param Separator separator used between all of the rest consecutive
   ///        parts of the name
   static std::string getNameWithSeparators(ArrayRef<StringRef> Parts,
                                            StringRef FirstSeparator,
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index 2fa0c13e14082..93ea63c1c2e60 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -230,6 +230,7 @@ __OMP_IMPLICIT_CLAUSE_NO_CLASS(unknown, "unknown")
 #define __OMP_TYPE(VarName) OMP_TYPE(VarName, Type::get##VarName##Ty(Ctx))
 
 __OMP_TYPE(Void)
+__OMP_TYPE(Int1)
 __OMP_TYPE(Int8)
 __OMP_TYPE(Int32)
 __OMP_TYPE(Int64)
@@ -310,6 +311,51 @@ __OMP_FUNCTION_TYPE(TaskRoutineEntry, false, Int32, Int32,
 
 ///}
 
+/// Internal Control Variables information
+///
+///{
+
+#ifndef ICV_DATA_ENV
+#define ICV_DATA_ENV(Enum, Name, EnvVarName, Init)
+#endif
+
+#define __ICV_DATA_ENV(Name, EnvVarName, Init)                                 \
+  ICV_DATA_ENV(ICV_##Name, #Name, #EnvVarName, Init)
+
+__ICV_DATA_ENV(nthreads, OMP_NUM_THREADS, ICV_IMPLEMENTATION_DEFINED)
+__ICV_DATA_ENV(active_levels, NONE, ICV_ZERO)
+__ICV_DATA_ENV(cancel, OMP_CANCELLATION, ICV_FALSE)
+__ICV_DATA_ENV(__last, last, ICV_LAST)
+
+#undef __ICV_DATA_ENV
+#undef ICV_DATA_ENV
+
+#ifndef ICV_RT_SET
+#define ICV_RT_SET(Name, RTL)
+#endif
+
+#define __ICV_RT_SET(Name, RTL) ICV_RT_SET(ICV_##Name, OMPRTL_##RTL)
+
+__ICV_RT_SET(nthreads, omp_set_num_threads)
+
+#undef __ICV_RT_SET
+#undef ICV_RT_SET
+
+#ifndef ICV_RT_GET
+#define ICV_RT_GET(Name, RTL)
+#endif
+
+#define __ICV_RT_GET(Name, RTL) ICV_RT_GET(ICV_##Name, OMPRTL_##RTL)
+
+__ICV_RT_GET(nthreads, omp_get_max_threads)
+__ICV_RT_GET(active_levels, omp_get_active_level)
+__ICV_RT_GET(cancel, omp_get_cancellation)
+
+#undef __ICV_RT_GET
+#undef ICV_RT_GET
+
+///}
+
 /// Runtime library function (and their attributes)
 ///
 ///{
@@ -538,6 +584,11 @@ __OMP_RTL(__tgt_push_mapper_component, false, Void, VoidPtr, VoidPtr, VoidPtr,
 __OMP_RTL(__kmpc_task_allow_completion_event, false, VoidPtr, IdentPtr,
           /* Int */ Int32, /* kmp_task_t */ VoidPtr)
 
+/// Note that device runtime functions (in the following) do not necessarily
+/// need attributes as we expect to see the definitions.
+__OMP_RTL(__kmpc_kernel_parallel, false, Int1, VoidPtrPtr)
+__OMP_RTL(__kmpc_kernel_prepare_parallel, false, Void, VoidPtr)
+
 __OMP_RTL(__last, false, Void, )
 
 #undef __OMP_RTL
@@ -931,6 +982,7 @@ __OMP_CANCEL_KIND(taskgroup, 4)
 
 __OMP_DEFAULT_KIND(none)
 __OMP_DEFAULT_KIND(shared)
+__OMP_DEFAULT_KIND(firstprivate)
 __OMP_DEFAULT_KIND(unknown)
 
 #undef __OMP_DEFAULT_KIND
@@ -1101,746 +1153,3 @@ OMP_LAST_TRAIT_PROPERTY(
 #undef __OMP_REQUIRES_TRAIT
 #undef OMP_REQUIRES_TRAIT
 ///}
-
-
-/// Clauses allowed per directive
-///
-///{
-
-#ifndef OMP_DIRECTIVE_CLAUSE
-#define OMP_DIRECTIVE_CLAUSE(Directive, MinVersion, MaxVersion, Clause)
-#endif
-
-#define __OMP_DIRECTIVE_CLAUSE(Directive, MinVersion, MaxVersion, Clause)      \
-  OMP_DIRECTIVE_CLAUSE(OMPD_##Directive, unsigned(MinVersion),                 \
-                       unsigned(MaxVersion), OMPC_##Clause)
-
-__OMP_DIRECTIVE_CLAUSE(scan, 50, ~0, inclusive)
-__OMP_DIRECTIVE_CLAUSE(scan, 50, ~0, exclusive)
-
-__OMP_DIRECTIVE_CLAUSE(parallel, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(parallel, 1, ~0, num_threads)
-__OMP_DIRECTIVE_CLAUSE(parallel, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(parallel, 1, ~0, proc_bind)
-__OMP_DIRECTIVE_CLAUSE(parallel, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(parallel, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(parallel, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(parallel, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(parallel, 1, ~0, copyin)
-__OMP_DIRECTIVE_CLAUSE(parallel, 1, ~0, allocate)
-
-__OMP_DIRECTIVE_CLAUSE(simd, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(simd, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(simd, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(simd, 1, ~0, aligned)
-__OMP_DIRECTIVE_CLAUSE(simd, 1, ~0, safelen)
-__OMP_DIRECTIVE_CLAUSE(simd, 1, ~0, simdlen)
-__OMP_DIRECTIVE_CLAUSE(simd, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(simd, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(simd, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(simd, 50, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(simd, 50, ~0, nontemporal)
-__OMP_DIRECTIVE_CLAUSE(simd, 50, ~0, order)
-
-__OMP_DIRECTIVE_CLAUSE(for, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(for, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(for, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(for, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(for, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(for, 1, ~0, schedule)
-__OMP_DIRECTIVE_CLAUSE(for, 1, ~0, ordered)
-__OMP_DIRECTIVE_CLAUSE(for, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(for, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(for, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(for, 50, ~0, order)
-
-__OMP_DIRECTIVE_CLAUSE(for_simd, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(for_simd, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(for_simd, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(for_simd, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(for_simd, 1, ~0, schedule)
-__OMP_DIRECTIVE_CLAUSE(for_simd, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(for_simd, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(for_simd, 1, ~0, safelen)
-__OMP_DIRECTIVE_CLAUSE(for_simd, 1, ~0, simdlen)
-__OMP_DIRECTIVE_CLAUSE(for_simd, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(for_simd, 1, ~0, aligned)
-__OMP_DIRECTIVE_CLAUSE(for_simd, 1, ~1, ordered)
-__OMP_DIRECTIVE_CLAUSE(for_simd, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(for_simd, 50, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(for_simd, 50, ~0, nontemporal)
-__OMP_DIRECTIVE_CLAUSE(for_simd, 50, ~0, order)
-
-__OMP_DIRECTIVE_CLAUSE(sections, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(sections, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(sections, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(sections, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(sections, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(sections, 1, ~0, allocate)
-
-__OMP_DIRECTIVE_CLAUSE(single, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(single, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(single, 1, ~0, copyprivate)
-__OMP_DIRECTIVE_CLAUSE(single, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(single, 1, ~0, allocate)
-
-__OMP_DIRECTIVE_CLAUSE(cancel, 1, ~0, if)
-
-__OMP_DIRECTIVE_CLAUSE(parallel_for, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(parallel_for, 1, ~0, num_threads)
-__OMP_DIRECTIVE_CLAUSE(parallel_for, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(parallel_for, 1, ~0, proc_bind)
-__OMP_DIRECTIVE_CLAUSE(parallel_for, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(parallel_for, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(parallel_for, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(parallel_for, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(parallel_for, 1, ~0, copyin)
-__OMP_DIRECTIVE_CLAUSE(parallel_for, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(parallel_for, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(parallel_for, 1, ~0, schedule)
-__OMP_DIRECTIVE_CLAUSE(parallel_for, 1, ~1, ordered)
-__OMP_DIRECTIVE_CLAUSE(parallel_for, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(parallel_for, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(parallel_for, 50, ~0, order)
-
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, num_threads)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, proc_bind)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, copyin)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, schedule)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, safelen)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, simdlen)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, aligned)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~1, ordered)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 50, ~0, nontemporal)
-__OMP_DIRECTIVE_CLAUSE(parallel_for_simd, 50, ~0, order)
-
-__OMP_DIRECTIVE_CLAUSE(parallel_master, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(parallel_master, 1, ~0, num_threads)
-__OMP_DIRECTIVE_CLAUSE(parallel_master, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(parallel_master, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(parallel_master, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(parallel_master, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(parallel_master, 1, ~0, copyin)
-__OMP_DIRECTIVE_CLAUSE(parallel_master, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(parallel_master, 1, ~0, proc_bind)
-__OMP_DIRECTIVE_CLAUSE(parallel_master, 1, ~0, allocate)
-
-__OMP_DIRECTIVE_CLAUSE(parallel_sections, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(parallel_sections, 1, ~0, num_threads)
-__OMP_DIRECTIVE_CLAUSE(parallel_sections, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(parallel_sections, 1, ~0, proc_bind)
-__OMP_DIRECTIVE_CLAUSE(parallel_sections, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(parallel_sections, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(parallel_sections, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(parallel_sections, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(parallel_sections, 1, ~0, copyin)
-__OMP_DIRECTIVE_CLAUSE(parallel_sections, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(parallel_sections, 1, ~0, allocate)
-
-__OMP_DIRECTIVE_CLAUSE(task, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(task, 1, ~0, final)
-__OMP_DIRECTIVE_CLAUSE(task, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(task, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(task, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(task, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(task, 1, ~0, untied)
-__OMP_DIRECTIVE_CLAUSE(task, 1, ~0, mergeable)
-__OMP_DIRECTIVE_CLAUSE(task, 1, ~0, depend)
-__OMP_DIRECTIVE_CLAUSE(task, 1, ~0, priority)
-__OMP_DIRECTIVE_CLAUSE(task, 1, ~0, in_reduction)
-__OMP_DIRECTIVE_CLAUSE(task, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(task, 50, ~0, detach)
-__OMP_DIRECTIVE_CLAUSE(task, 50, ~0, affinity)
-
-__OMP_DIRECTIVE_CLAUSE(atomic, 1, ~0, read)
-__OMP_DIRECTIVE_CLAUSE(atomic, 1, ~0, write)
-__OMP_DIRECTIVE_CLAUSE(atomic, 1, ~0, update)
-__OMP_DIRECTIVE_CLAUSE(atomic, 1, ~0, capture)
-__OMP_DIRECTIVE_CLAUSE(atomic, 1, ~0, seq_cst)
-__OMP_DIRECTIVE_CLAUSE(atomic, 50, ~0, acq_rel)
-__OMP_DIRECTIVE_CLAUSE(atomic, 50, ~0, acquire)
-__OMP_DIRECTIVE_CLAUSE(atomic, 50, ~0, release)
-__OMP_DIRECTIVE_CLAUSE(atomic, 50, ~0, relaxed)
-__OMP_DIRECTIVE_CLAUSE(atomic, 50, ~0, hint)
-
-__OMP_DIRECTIVE_CLAUSE(target, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(target, 1, ~0, device)
-__OMP_DIRECTIVE_CLAUSE(target, 1, ~0, map)
-__OMP_DIRECTIVE_CLAUSE(target, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(target, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(target, 1, ~0, depend)
-__OMP_DIRECTIVE_CLAUSE(target, 1, ~0, defaultmap)
-__OMP_DIRECTIVE_CLAUSE(target, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(target, 1, ~0, is_device_ptr)
-__OMP_DIRECTIVE_CLAUSE(target, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(target, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(target, 50, ~0, uses_allocators)
-
-__OMP_DIRECTIVE_CLAUSE(requires, 1, ~0, unified_address)
-__OMP_DIRECTIVE_CLAUSE(requires, 1, ~0, unified_shared_memory)
-__OMP_DIRECTIVE_CLAUSE(requires, 1, ~0, reverse_offload)
-__OMP_DIRECTIVE_CLAUSE(requires, 1, ~0, dynamic_allocators)
-__OMP_DIRECTIVE_CLAUSE(requires, 1, ~0, atomic_default_mem_order)
-
-__OMP_DIRECTIVE_CLAUSE(allocate, 1, ~0, allocator)
-
-__OMP_DIRECTIVE_CLAUSE(target_data, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(target_data, 1, ~0, device)
-__OMP_DIRECTIVE_CLAUSE(target_data, 1, ~0, map)
-__OMP_DIRECTIVE_CLAUSE(target_data, 1, ~0, use_device_ptr)
-__OMP_DIRECTIVE_CLAUSE(target_data, 50, ~0, use_device_addr)
-
-__OMP_DIRECTIVE_CLAUSE(target_enter_data, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(target_enter_data, 1, ~0, device)
-__OMP_DIRECTIVE_CLAUSE(target_enter_data, 1, ~0, map)
-__OMP_DIRECTIVE_CLAUSE(target_enter_data, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(target_enter_data, 1, ~0, depend)
-
-__OMP_DIRECTIVE_CLAUSE(target_exit_data, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(target_exit_data, 1, ~0, device)
-__OMP_DIRECTIVE_CLAUSE(target_exit_data, 1, ~0, map)
-__OMP_DIRECTIVE_CLAUSE(target_exit_data, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(target_exit_data, 1, ~0, depend)
-
-__OMP_DIRECTIVE_CLAUSE(target_parallel, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(target_parallel, 1, ~0, device)
-__OMP_DIRECTIVE_CLAUSE(target_parallel, 1, ~0, map)
-__OMP_DIRECTIVE_CLAUSE(target_parallel, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(target_parallel, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(target_parallel, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(target_parallel, 1, ~0, depend)
-__OMP_DIRECTIVE_CLAUSE(target_parallel, 1, ~0, defaultmap)
-__OMP_DIRECTIVE_CLAUSE(target_parallel, 1, ~0, num_threads)
-__OMP_DIRECTIVE_CLAUSE(target_parallel, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(target_parallel, 1, ~0, proc_bind)
-__OMP_DIRECTIVE_CLAUSE(target_parallel, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(target_parallel, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(target_parallel, 1, ~0, is_device_ptr)
-__OMP_DIRECTIVE_CLAUSE(target_parallel, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(target_parallel, 50, ~0, uses_allocators)
-
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, device)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, map)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, depend)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, defaultmap)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, num_threads)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, proc_bind)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, schedule)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~1, ordered)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, is_device_ptr)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 50, ~0, order)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for, 50, ~0, uses_allocators)
-
-__OMP_DIRECTIVE_CLAUSE(target_update, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(target_update, 1, ~0, device)
-__OMP_DIRECTIVE_CLAUSE(target_update, 1, ~0, to)
-__OMP_DIRECTIVE_CLAUSE(target_update, 1, ~0, from)
-__OMP_DIRECTIVE_CLAUSE(target_update, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(target_update, 1, ~0, depend)
-
-__OMP_DIRECTIVE_CLAUSE(teams, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(teams, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(teams, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(teams, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(teams, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(teams, 1, ~0, num_teams)
-__OMP_DIRECTIVE_CLAUSE(teams, 1, ~0, thread_limit)
-__OMP_DIRECTIVE_CLAUSE(teams, 1, ~0, allocate)
-
-__OMP_DIRECTIVE_CLAUSE(ordered, 1, ~0, threads)
-__OMP_DIRECTIVE_CLAUSE(ordered, 1, ~0, simd)
-__OMP_DIRECTIVE_CLAUSE(ordered, 1, ~0, depend)
-
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, final)
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, untied)
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, mergeable)
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, priority)
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, grainsize)
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, nogroup)
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, num_tasks)
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, in_reduction)
-__OMP_DIRECTIVE_CLAUSE(taskloop, 1, ~0, allocate)
-
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, final)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, untied)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, mergeable)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, priority)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, aligned)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, safelen)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, simdlen)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, grainsize)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, nogroup)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, num_tasks)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, in_reduction)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 50, ~0, nontemporal)
-__OMP_DIRECTIVE_CLAUSE(taskloop_simd, 50, ~0, order)
-
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, final)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, untied)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, mergeable)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, priority)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, grainsize)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, nogroup)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, num_tasks)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, in_reduction)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop, 1, ~0, allocate)
-
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, final)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, untied)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, mergeable)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, priority)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, aligned)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, safelen)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, simdlen)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, grainsize)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, nogroup)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, num_tasks)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, in_reduction)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 50, ~0, nontemporal)
-__OMP_DIRECTIVE_CLAUSE(master_taskloop_simd, 50, ~0, order)
-
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, final)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, untied)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, mergeable)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, priority)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, grainsize)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, nogroup)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, num_tasks)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, num_threads)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, proc_bind)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop, 1, ~0, copyin)
-
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, final)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, untied)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, mergeable)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, priority)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, grainsize)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, nogroup)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, num_tasks)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, num_threads)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, proc_bind)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, copyin)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, aligned)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, safelen)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 1, ~0, simdlen)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 50, ~0, nontemporal)
-__OMP_DIRECTIVE_CLAUSE(parallel_master_taskloop_simd, 50, ~0, order)
-
-__OMP_DIRECTIVE_CLAUSE(critical, 1, ~0, hint)
-
-__OMP_DIRECTIVE_CLAUSE(distribute, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(distribute, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(distribute, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(distribute, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(distribute, 1, ~0, dist_schedule)
-__OMP_DIRECTIVE_CLAUSE(distribute, 1, ~0, allocate)
-
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for, 1, ~0, dist_schedule)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for, 1, ~0, num_threads)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for, 1, ~0, proc_bind)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for, 1, ~0, copyin)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for, 1, ~0, schedule)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for, 50, ~0, order)
-
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, dist_schedule)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, num_threads)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, proc_bind)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, copyin)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, schedule)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, aligned)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, safelen)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, simdlen)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 50, ~0, nontemporal)
-__OMP_DIRECTIVE_CLAUSE(distribute_parallel_for_simd, 50, ~0, order)
-
-__OMP_DIRECTIVE_CLAUSE(distribute_simd, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(distribute_simd, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(distribute_simd, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(distribute_simd, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(distribute_simd, 1, ~0, dist_schedule)
-__OMP_DIRECTIVE_CLAUSE(distribute_simd, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(distribute_simd, 1, ~0, aligned)
-__OMP_DIRECTIVE_CLAUSE(distribute_simd, 1, ~0, safelen)
-__OMP_DIRECTIVE_CLAUSE(distribute_simd, 1, ~0, simdlen)
-__OMP_DIRECTIVE_CLAUSE(distribute_simd, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(distribute_simd, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(distribute_simd, 50, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(distribute_simd, 50, ~0, nontemporal)
-__OMP_DIRECTIVE_CLAUSE(distribute_simd, 50, ~0, order)
-
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, device)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, map)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, depend)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, defaultmap)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, num_threads)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, proc_bind)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, schedule)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~1, ordered)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, safelen)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, simdlen)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, aligned)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, is_device_ptr)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 50, ~0, nontemporal)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 50, ~0, order)
-__OMP_DIRECTIVE_CLAUSE(target_parallel_for_simd, 50, ~0, uses_allocators)
-
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, device)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, map)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, depend)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, defaultmap)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, is_device_ptr)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, aligned)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, safelen)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, simdlen)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 50, ~0, nontemporal)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 50, ~0, order)
-__OMP_DIRECTIVE_CLAUSE(target_simd, 50, ~0, uses_allocators)
-
-__OMP_DIRECTIVE_CLAUSE(teams_distribute, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute, 1, ~0, num_teams)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute, 1, ~0, thread_limit)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute, 1, ~0, dist_schedule)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute, 1, ~0, allocate)
-
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 1, ~0, num_teams)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 1, ~0, thread_limit)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 1, ~0, dist_schedule)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 1, ~0, aligned)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 1, ~0, safelen)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 1, ~0, simdlen)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 50, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 50, ~0, nontemporal)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_simd, 50, ~0, order)
-
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, dist_schedule)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, num_threads)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, proc_bind)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, schedule)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, aligned)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, safelen)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, simdlen)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, num_teams)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, thread_limit)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 50, ~0, nontemporal)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for_simd, 50, ~0, order)
-
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 1, ~0, dist_schedule)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 1, ~0, num_threads)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 1, ~0, proc_bind)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 1, ~0, schedule)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 1, ~0, num_teams)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 1, ~0, thread_limit)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 1, ~0, copyin)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(teams_distribute_parallel_for, 50, ~0, order)
-
-__OMP_DIRECTIVE_CLAUSE(target_teams, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(target_teams, 1, ~0, device)
-__OMP_DIRECTIVE_CLAUSE(target_teams, 1, ~0, map)
-__OMP_DIRECTIVE_CLAUSE(target_teams, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(target_teams, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(target_teams, 1, ~0, depend)
-__OMP_DIRECTIVE_CLAUSE(target_teams, 1, ~0, defaultmap)
-__OMP_DIRECTIVE_CLAUSE(target_teams, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(target_teams, 1, ~0, is_device_ptr)
-__OMP_DIRECTIVE_CLAUSE(target_teams, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(target_teams, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(target_teams, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(target_teams, 1, ~0, num_teams)
-__OMP_DIRECTIVE_CLAUSE(target_teams, 1, ~0, thread_limit)
-__OMP_DIRECTIVE_CLAUSE(target_teams, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(target_teams, 50, ~0, uses_allocators)
-
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, device)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, map)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, depend)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, defaultmap)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, is_device_ptr)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, num_teams)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, thread_limit)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, dist_schedule)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute, 50, ~0, uses_allocators)
-
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, device)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, map)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, depend)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, defaultmap)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0,
-                       firstprivate)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0,
-                       is_device_ptr)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, default)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, num_teams)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0,
-                       thread_limit)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0,
-                       dist_schedule)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, num_threads)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, proc_bind)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, schedule)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 50, ~0, order)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for, 50, ~0,
-                       uses_allocators)
-
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0, device)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0, map)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       private)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0, depend)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       defaultmap)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       firstprivate)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       is_device_ptr)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       default)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       reduction)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       num_teams)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       thread_limit)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       lastprivate)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       collapse)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       dist_schedule)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       num_threads)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       proc_bind)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       schedule)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       aligned)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       safelen)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       simdlen)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 1, ~0,
-                       allocate)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 50, ~0,
-                       nontemporal)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 50, ~0, order)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_parallel_for_simd, 50, ~0,
-                       uses_allocators)
-
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, if)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, device)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, map)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, private)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, nowait)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, depend)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, defaultmap)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, firstprivate)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, lastprivate)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, is_device_ptr)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, shared)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, reduction)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, num_teams)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, thread_limit)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, collapse)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, dist_schedule)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, linear)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, aligned)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, safelen)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, simdlen)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 1, ~0, allocate)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 50, ~0, nontemporal)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 50, ~0, order)
-__OMP_DIRECTIVE_CLAUSE(target_teams_distribute_simd, 50, ~0, uses_allocators)
-
-__OMP_DIRECTIVE_CLAUSE(taskgroup, 1, ~0, task_reduction)
-__OMP_DIRECTIVE_CLAUSE(taskgroup, 1, ~0, allocate)
-
-__OMP_DIRECTIVE_CLAUSE(declare_mapper, 1, ~0, map)
-
-__OMP_DIRECTIVE_CLAUSE(declare_variant, 1, ~0, match)
-
-__OMP_DIRECTIVE_CLAUSE(flush, 50, ~0, acq_rel)
-__OMP_DIRECTIVE_CLAUSE(flush, 50, ~0, acquire)
-__OMP_DIRECTIVE_CLAUSE(flush, 50, ~0, release)
-// TODO This should ne `none` instead
-__OMP_DIRECTIVE_CLAUSE(flush, 1, ~0, flush)
-
-__OMP_DIRECTIVE_CLAUSE(depobj, 50, ~0, depend)
-__OMP_DIRECTIVE_CLAUSE(depobj, 50, ~0, destroy)
-__OMP_DIRECTIVE_CLAUSE(depobj, 50, ~0, update)
-// TODO This should ne `none` instead
-__OMP_DIRECTIVE_CLAUSE(depobj, 50, ~0, depobj)
-
-#undef __OMP_DIRECTIVE_CLAUSE
-#undef OMP_DIRECTIVE_CLAUSE
-///}
diff --git a/llvm/include/llvm/IR/AbstractCallSite.h b/llvm/include/llvm/IR/AbstractCallSite.h
index c78afc729d43a..e8cf05001542a 100644
--- a/llvm/include/llvm/IR/AbstractCallSite.h
+++ b/llvm/include/llvm/IR/AbstractCallSite.h
@@ -221,6 +221,27 @@ class AbstractCallSite {
   }
 };
 
+/// Apply function Func to each CB's callback call site.
+template <typename UnaryFunction>
+void forEachCallbackCallSite(const CallBase &CB, UnaryFunction Func) {
+  SmallVector<const Use *, 4u> CallbackUses;
+  AbstractCallSite::getCallbackUses(CB, CallbackUses);
+  for (const Use *U : CallbackUses) {
+    AbstractCallSite ACS(U);
+    assert(ACS && ACS.isCallbackCall() && "must be a callback call");
+    Func(ACS);
+  }
+}
+
+/// Apply function Func to each CB's callback function.
+template <typename UnaryFunction>
+void forEachCallbackFunction(const CallBase &CB, UnaryFunction Func) {
+  forEachCallbackCallSite(CB, [&Func](AbstractCallSite &ACS) {
+    if (Function *Callback = ACS.getCalledFunction())
+      Func(Callback);
+  });
+}
+
 } // end namespace llvm
 
 #endif // LLVM_IR_ABSTRACTCALLSITE_H
diff --git a/llvm/include/llvm/IR/Argument.h b/llvm/include/llvm/IR/Argument.h
index 2bd8e99f12c43..af469e8a5d1aa 100644
--- a/llvm/include/llvm/IR/Argument.h
+++ b/llvm/include/llvm/IR/Argument.h
@@ -75,6 +75,10 @@ class Argument final : public Value {
   /// attribute. These attributes represent arguments being passed by value.
   bool hasPassPointeeByValueAttr() const;
 
+  /// If this argument satisfies has hasPassPointeeByValueAttr, return the
+  /// in-memory ABI size copied to the stack for the call. Otherwise, return 0.
+  uint64_t getPassPointeeByValueCopySize(const DataLayout &DL) const;
+
   /// If this is a byval or inalloca argument, return its alignment.
   /// FIXME: Remove this function once transition to Align is over.
   /// Use getParamAlign() instead.
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index ee48f40ca284f..395f9dbfb1760 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -39,6 +39,9 @@ def Builtin : EnumAttr<"builtin">;
 /// Pass structure by value.
 def ByVal : TypeAttr<"byval">;
 
+/// Parameter or return value may not contain uninitialized or poison bits.
+def NoUndef : EnumAttr<"noundef">;
+
 /// Marks function as being in a cold path.
 def Cold : EnumAttr<"cold">;
 
@@ -233,6 +236,7 @@ def ZExt : EnumAttr<"zeroext">;
 def LessPreciseFPMAD : StrBoolAttr<"less-precise-fpmad">;
 def NoInfsFPMath : StrBoolAttr<"no-infs-fp-math">;
 def NoNansFPMath : StrBoolAttr<"no-nans-fp-math">;
+def NoSignedZerosFPMath : StrBoolAttr<"no-signed-zeros-fp-math">;
 def UnsafeFPMath : StrBoolAttr<"unsafe-fp-math">;
 def NoJumpTables : StrBoolAttr<"no-jump-tables">;
 def NoInlineLineTables : StrBoolAttr<"no-inline-line-tables">;
@@ -270,6 +274,7 @@ class MergeRule<string F> {
 def : MergeRule<"setAND<LessPreciseFPMADAttr>">;
 def : MergeRule<"setAND<NoInfsFPMathAttr>">;
 def : MergeRule<"setAND<NoNansFPMathAttr>">;
+def : MergeRule<"setAND<NoSignedZerosFPMathAttr>">;
 def : MergeRule<"setAND<UnsafeFPMathAttr>">;
 def : MergeRule<"setOR<NoImplicitFloatAttr>">;
 def : MergeRule<"setOR<NoJumpTablesAttr>">;
diff --git a/llvm/include/llvm/IR/Constant.h b/llvm/include/llvm/IR/Constant.h
index 174e7364c5240..9a1d2b80c48e4 100644
--- a/llvm/include/llvm/IR/Constant.h
+++ b/llvm/include/llvm/IR/Constant.h
@@ -43,6 +43,8 @@ class Constant : public User {
   Constant(Type *ty, ValueTy vty, Use *Ops, unsigned NumOps)
     : User(ty, vty, Ops, NumOps) {}
 
+  ~Constant() = default;
+
 public:
   void operator=(const Constant &) = delete;
   Constant(const Constant &) = delete;
diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h
index 25d8f6afc1bdc..3579c9f1ee332 100644
--- a/llvm/include/llvm/IR/Constants.h
+++ b/llvm/include/llvm/IR/Constants.h
@@ -41,11 +41,6 @@
 
 namespace llvm {
 
-class ArrayType;
-class IntegerType;
-class PointerType;
-class StructType;
-class VectorType;
 template <class ConstantClass> struct ConstantAggrKeyType;
 
 /// Base class for constants with no operands.
@@ -156,9 +151,19 @@ class ConstantInt final : public ConstantData {
     return Val.getSExtValue();
   }
 
-  /// Return the constant as an llvm::Align. Note that this method can assert if
-  /// the value does not fit in 64 bits or is not a power of two.
-  inline Align getAlignValue() const { return Align(getZExtValue()); }
+  /// Return the constant as an llvm::MaybeAlign.
+  /// Note that this method can assert if the value does not fit in 64 bits or
+  /// is not a power of two.
+  inline MaybeAlign getMaybeAlignValue() const {
+    return MaybeAlign(getZExtValue());
+  }
+
+  /// Return the constant as an llvm::Align, interpreting `0` as `Align(1)`.
+  /// Note that this method can assert if the value does not fit in 64 bits or
+  /// is not a power of two.
+  inline Align getAlignValue() const {
+    return getMaybeAlignValue().valueOrOne();
+  }
 
   /// A helper method that can be used to determine if the constant contained
   /// within is equal to a constant.  This only works for very small values,
@@ -904,6 +909,8 @@ class ConstantExpr : public Constant {
     setValueSubclassData(Opcode);
   }
 
+  ~ConstantExpr() = default;
+
 public:
   // Static methods to construct a ConstantExpr of different kinds.  Note that
   // these methods may return a object that is not an instance of the
diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h
index ed2336265ab61..17297bb8b3093 100644
--- a/llvm/include/llvm/IR/DataLayout.h
+++ b/llvm/include/llvm/IR/DataLayout.h
@@ -133,7 +133,8 @@ class DataLayout {
     MM_MachO,
     MM_WinCOFF,
     MM_WinCOFFX86,
-    MM_Mips
+    MM_Mips,
+    MM_XCOFF
   };
   ManglingModeT ManglingMode;
 
@@ -309,6 +310,7 @@ class DataLayout {
     case MM_ELF:
     case MM_Mips:
     case MM_WinCOFF:
+    case MM_XCOFF:
       return '\0';
     case MM_MachO:
     case MM_WinCOFFX86:
@@ -329,6 +331,8 @@ class DataLayout {
     case MM_MachO:
     case MM_WinCOFFX86:
       return "L";
+    case MM_XCOFF:
+      return "L..";
     }
     llvm_unreachable("invalid mangling mode");
   }
@@ -574,13 +578,26 @@ class DataLayout {
   /// Returns the preferred alignment of the specified global.
   ///
   /// This includes an explicitly requested alignment (if the global has one).
-  unsigned getPreferredAlignment(const GlobalVariable *GV) const;
+  Align getPreferredAlign(const GlobalVariable *GV) const;
+
+  /// Returns the preferred alignment of the specified global.
+  ///
+  /// This includes an explicitly requested alignment (if the global has one).
+  LLVM_ATTRIBUTE_DEPRECATED(
+      inline unsigned getPreferredAlignment(const GlobalVariable *GV) const,
+      "Use getPreferredAlign instead") {
+    return getPreferredAlign(GV).value();
+  }
 
   /// Returns the preferred alignment of the specified global, returned
   /// in log form.
   ///
   /// This includes an explicitly requested alignment (if the global has one).
-  unsigned getPreferredAlignmentLog(const GlobalVariable *GV) const;
+  LLVM_ATTRIBUTE_DEPRECATED(
+      inline unsigned getPreferredAlignmentLog(const GlobalVariable *GV) const,
+      "Inline where needed") {
+    return Log2(getPreferredAlign(GV));
+  }
 };
 
 inline DataLayout *unwrap(LLVMTargetDataRef P) {
diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h
index 900a4b561cda9..7d7cc4de79374 100644
--- a/llvm/include/llvm/IR/DebugInfoMetadata.h
+++ b/llvm/include/llvm/IR/DebugInfoMetadata.h
@@ -359,7 +359,7 @@ class DIEnumerator : public DINode {
   friend class MDNode;
 
   APInt Value;
-  DIEnumerator(LLVMContext &C, StorageType Storage, APInt Value,
+  DIEnumerator(LLVMContext &C, StorageType Storage, const APInt &Value,
                bool IsUnsigned, ArrayRef<Metadata *> Ops)
       : DINode(C, DIEnumeratorKind, Storage, dwarf::DW_TAG_enumerator, Ops),
         Value(Value) {
@@ -371,13 +371,13 @@ class DIEnumerator : public DINode {
                      Ops) {}
   ~DIEnumerator() = default;
 
-  static DIEnumerator *getImpl(LLVMContext &Context, APInt Value,
+  static DIEnumerator *getImpl(LLVMContext &Context, const APInt &Value,
                                bool IsUnsigned, StringRef Name,
                                StorageType Storage, bool ShouldCreate = true) {
     return getImpl(Context, Value, IsUnsigned,
                    getCanonicalMDString(Context, Name), Storage, ShouldCreate);
   }
-  static DIEnumerator *getImpl(LLVMContext &Context, APInt Value,
+  static DIEnumerator *getImpl(LLVMContext &Context, const APInt &Value,
                                bool IsUnsigned, MDString *Name,
                                StorageType Storage, bool ShouldCreate = true);
 
diff --git a/llvm/include/llvm/IR/DebugLoc.h b/llvm/include/llvm/IR/DebugLoc.h
index 780d17a33661f..4914d733fe0da 100644
--- a/llvm/include/llvm/IR/DebugLoc.h
+++ b/llvm/include/llvm/IR/DebugLoc.h
@@ -85,7 +85,7 @@ namespace llvm {
     /// the chain now is inlined-at the new call site.
     /// \param   InlinedAt    The new outermost inlined-at in the chain.
     /// \param   ReplaceLast  Replace the last location in the inlined-at chain.
-    static DebugLoc appendInlinedAt(DebugLoc DL, DILocation *InlinedAt,
+    static DebugLoc appendInlinedAt(const DebugLoc &DL, DILocation *InlinedAt,
                                     LLVMContext &Ctx,
                                     DenseMap<const MDNode *, MDNode *> &Cache,
                                     bool ReplaceLast = false);
diff --git a/llvm/include/llvm/IR/Dominators.h b/llvm/include/llvm/IR/Dominators.h
index a79be8779b7e3..71595cb15df48 100644
--- a/llvm/include/llvm/IR/Dominators.h
+++ b/llvm/include/llvm/IR/Dominators.h
@@ -208,7 +208,8 @@ template <class Node, class ChildIterator> struct DomTreeGraphTraitsBase {
 
 template <>
 struct GraphTraits<DomTreeNode *>
-    : public DomTreeGraphTraitsBase<DomTreeNode, DomTreeNode::iterator> {};
+    : public DomTreeGraphTraitsBase<DomTreeNode, DomTreeNode::const_iterator> {
+};
 
 template <>
 struct GraphTraits<const DomTreeNode *>
@@ -277,7 +278,7 @@ class DominatorTreeWrapperPass : public FunctionPass {
     AU.setPreservesAll();
   }
 
-  void releaseMemory() override { DT.releaseMemory(); }
+  void releaseMemory() override { DT.reset(); }
 
   void print(raw_ostream &OS, const Module *M = nullptr) const override;
 };
diff --git a/llvm/include/llvm/IR/FPEnv.h b/llvm/include/llvm/IR/FPEnv.h
index eadfef2a82f86..f00cb735932f6 100644
--- a/llvm/include/llvm/IR/FPEnv.h
+++ b/llvm/include/llvm/IR/FPEnv.h
@@ -15,12 +15,11 @@
 #ifndef LLVM_IR_FLOATINGPOINT_H
 #define LLVM_IR_FLOATINGPOINT_H
 
-#include "llvm/ADT/APFloat.h"
+#include "llvm/ADT/FloatingPointMode.h"
 #include "llvm/ADT/Optional.h"
-#include "llvm/ADT/StringRef.h"
-#include <stdint.h>
 
 namespace llvm {
+class StringRef;
 
 namespace fp {
 
diff --git a/llvm/include/llvm/IR/GlobalObject.h b/llvm/include/llvm/IR/GlobalObject.h
index a26c54ce9a3d8..3a7b718845cb6 100644
--- a/llvm/include/llvm/IR/GlobalObject.h
+++ b/llvm/include/llvm/IR/GlobalObject.h
@@ -76,6 +76,10 @@ class GlobalObject : public GlobalValue {
     return Align ? Align->value() : 0;
   }
 
+  /// Returns the alignment of the given variable or function.
+  ///
+  /// Note that for functions this is the alignment of the code, not the
+  /// alignment of a function pointer.
   MaybeAlign getAlign() const {
     unsigned Data = getGlobalValueSubClassData();
     unsigned AlignmentData = Data & AlignmentMask;
@@ -183,6 +187,13 @@ class GlobalObject : public GlobalValue {
   void setVCallVisibilityMetadata(VCallVisibility Visibility);
   VCallVisibility getVCallVisibility() const;
 
+  /// Returns true if the alignment of the value can be unilaterally
+  /// increased.
+  ///
+  /// Note that for functions this is the alignment of the code, not the
+  /// alignment of a function pointer.
+  bool canIncreaseAlignment() const;
+
 protected:
   void copyAttributesFrom(const GlobalObject *Src);
 
diff --git a/llvm/include/llvm/IR/GlobalValue.h b/llvm/include/llvm/IR/GlobalValue.h
index 1c19011c9131c..cf704d1f2374e 100644
--- a/llvm/include/llvm/IR/GlobalValue.h
+++ b/llvm/include/llvm/IR/GlobalValue.h
@@ -185,7 +185,6 @@ class GlobalValue : public Constant {
 
   GlobalValue(const GlobalValue &) = delete;
 
-  unsigned getAlignment() const;
   unsigned getAddressSpace() const;
 
   enum class UnnamedAddr {
@@ -549,10 +548,6 @@ class GlobalValue : public Constant {
     return !(isDeclarationForLinker() || isWeakForLinker());
   }
 
-  // Returns true if the alignment of the value can be unilaterally
-  // increased.
-  bool canIncreaseAlignment() const;
-
   const GlobalObject *getBaseObject() const;
   GlobalObject *getBaseObject() {
     return const_cast<GlobalObject *>(
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index ec042f0740fca..ffec4ff64ca66 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -1733,19 +1733,21 @@ class IRBuilderBase {
     return Insert(new FenceInst(Context, Ordering, SSID), Name);
   }
 
-  AtomicCmpXchgInst *
-  CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New,
-                      AtomicOrdering SuccessOrdering,
-                      AtomicOrdering FailureOrdering,
-                      SyncScope::ID SSID = SyncScope::System) {
-    return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering,
-                                        FailureOrdering, SSID));
+  AtomicCmpXchgInst *CreateAtomicCmpXchg(
+      Value *Ptr, Value *Cmp, Value *New, AtomicOrdering SuccessOrdering,
+      AtomicOrdering FailureOrdering, SyncScope::ID SSID = SyncScope::System) {
+    const DataLayout &DL = BB->getModule()->getDataLayout();
+    Align Alignment(DL.getTypeStoreSize(New->getType()));
+    return Insert(new AtomicCmpXchgInst(
+        Ptr, Cmp, New, Alignment, SuccessOrdering, FailureOrdering, SSID));
   }
 
   AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val,
                                  AtomicOrdering Ordering,
                                  SyncScope::ID SSID = SyncScope::System) {
-    return Insert(new AtomicRMWInst(Op, Ptr, Val, Ordering, SSID));
+    const DataLayout &DL = BB->getModule()->getDataLayout();
+    Align Alignment(DL.getTypeStoreSize(Val->getType()));
+    return Insert(new AtomicRMWInst(Op, Ptr, Val, Alignment, Ordering, SSID));
   }
 
   Value *CreateGEP(Value *Ptr, ArrayRef<Value *> IdxList,
diff --git a/llvm/include/llvm/IR/IRPrintingPasses.h b/llvm/include/llvm/IR/IRPrintingPasses.h
index 230db988f737b..3a1c489ee09f2 100644
--- a/llvm/include/llvm/IR/IRPrintingPasses.h
+++ b/llvm/include/llvm/IR/IRPrintingPasses.h
@@ -19,17 +19,10 @@
 #define LLVM_IR_IRPRINTINGPASSES_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/IR/PassManager.h"
 #include <string>
 
 namespace llvm {
-class Pass;
-class Function;
-class FunctionPass;
-class Module;
-class ModulePass;
-class PreservedAnalyses;
-class raw_ostream;
-template <typename IRUnitT, typename... ExtraArgTs> class AnalysisManager;
 
 /// Create and return a pass that writes the module to the specified
 /// \c raw_ostream.
@@ -71,7 +64,7 @@ extern bool shouldPrintAfterPass(StringRef);
 ///
 /// Note: This pass is for use with the new pass manager. Use the create...Pass
 /// functions above to create passes for use with the legacy pass manager.
-class PrintModulePass {
+class PrintModulePass : public PassInfoMixin<PrintModulePass> {
   raw_ostream &OS;
   std::string Banner;
   bool ShouldPreserveUseListOrder;
@@ -82,15 +75,13 @@ class PrintModulePass {
                   bool ShouldPreserveUseListOrder = false);
 
   PreservedAnalyses run(Module &M, AnalysisManager<Module> &);
-
-  static StringRef name() { return "PrintModulePass"; }
 };
 
 /// Pass for printing a Function as LLVM's text IR assembly.
 ///
 /// Note: This pass is for use with the new pass manager. Use the create...Pass
 /// functions above to create passes for use with the legacy pass manager.
-class PrintFunctionPass {
+class PrintFunctionPass : public PassInfoMixin<PrintFunctionPass> {
   raw_ostream &OS;
   std::string Banner;
 
@@ -99,8 +90,6 @@ class PrintFunctionPass {
   PrintFunctionPass(raw_ostream &OS, const std::string &Banner = "");
 
   PreservedAnalyses run(Function &F, AnalysisManager<Function> &);
-
-  static StringRef name() { return "PrintFunctionPass"; }
 };
 
 } // End llvm namespace
diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index d0af5ccc5240b..07af00ec92402 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -722,41 +722,43 @@ class CmpInst : public Instruction {
   /// Some passes (e.g. InstCombine) depend on the bit-wise characteristics of
   /// FCMP_* values. Changing the bit patterns requires a potential change to
   /// those passes.
-  enum Predicate {
-    // Opcode              U L G E    Intuitive operation
-    FCMP_FALSE =  0,  ///< 0 0 0 0    Always false (always folded)
-    FCMP_OEQ   =  1,  ///< 0 0 0 1    True if ordered and equal
-    FCMP_OGT   =  2,  ///< 0 0 1 0    True if ordered and greater than
-    FCMP_OGE   =  3,  ///< 0 0 1 1    True if ordered and greater than or equal
-    FCMP_OLT   =  4,  ///< 0 1 0 0    True if ordered and less than
-    FCMP_OLE   =  5,  ///< 0 1 0 1    True if ordered and less than or equal
-    FCMP_ONE   =  6,  ///< 0 1 1 0    True if ordered and operands are unequal
-    FCMP_ORD   =  7,  ///< 0 1 1 1    True if ordered (no nans)
-    FCMP_UNO   =  8,  ///< 1 0 0 0    True if unordered: isnan(X) | isnan(Y)
-    FCMP_UEQ   =  9,  ///< 1 0 0 1    True if unordered or equal
-    FCMP_UGT   = 10,  ///< 1 0 1 0    True if unordered or greater than
-    FCMP_UGE   = 11,  ///< 1 0 1 1    True if unordered, greater than, or equal
-    FCMP_ULT   = 12,  ///< 1 1 0 0    True if unordered or less than
-    FCMP_ULE   = 13,  ///< 1 1 0 1    True if unordered, less than, or equal
-    FCMP_UNE   = 14,  ///< 1 1 1 0    True if unordered or not equal
-    FCMP_TRUE  = 15,  ///< 1 1 1 1    Always true (always folded)
+  enum Predicate : unsigned {
+    // Opcode            U L G E    Intuitive operation
+    FCMP_FALSE = 0, ///< 0 0 0 0    Always false (always folded)
+    FCMP_OEQ = 1,   ///< 0 0 0 1    True if ordered and equal
+    FCMP_OGT = 2,   ///< 0 0 1 0    True if ordered and greater than
+    FCMP_OGE = 3,   ///< 0 0 1 1    True if ordered and greater than or equal
+    FCMP_OLT = 4,   ///< 0 1 0 0    True if ordered and less than
+    FCMP_OLE = 5,   ///< 0 1 0 1    True if ordered and less than or equal
+    FCMP_ONE = 6,   ///< 0 1 1 0    True if ordered and operands are unequal
+    FCMP_ORD = 7,   ///< 0 1 1 1    True if ordered (no nans)
+    FCMP_UNO = 8,   ///< 1 0 0 0    True if unordered: isnan(X) | isnan(Y)
+    FCMP_UEQ = 9,   ///< 1 0 0 1    True if unordered or equal
+    FCMP_UGT = 10,  ///< 1 0 1 0    True if unordered or greater than
+    FCMP_UGE = 11,  ///< 1 0 1 1    True if unordered, greater than, or equal
+    FCMP_ULT = 12,  ///< 1 1 0 0    True if unordered or less than
+    FCMP_ULE = 13,  ///< 1 1 0 1    True if unordered, less than, or equal
+    FCMP_UNE = 14,  ///< 1 1 1 0    True if unordered or not equal
+    FCMP_TRUE = 15, ///< 1 1 1 1    Always true (always folded)
     FIRST_FCMP_PREDICATE = FCMP_FALSE,
     LAST_FCMP_PREDICATE = FCMP_TRUE,
     BAD_FCMP_PREDICATE = FCMP_TRUE + 1,
-    ICMP_EQ    = 32,  ///< equal
-    ICMP_NE    = 33,  ///< not equal
-    ICMP_UGT   = 34,  ///< unsigned greater than
-    ICMP_UGE   = 35,  ///< unsigned greater or equal
-    ICMP_ULT   = 36,  ///< unsigned less than
-    ICMP_ULE   = 37,  ///< unsigned less or equal
-    ICMP_SGT   = 38,  ///< signed greater than
-    ICMP_SGE   = 39,  ///< signed greater or equal
-    ICMP_SLT   = 40,  ///< signed less than
-    ICMP_SLE   = 41,  ///< signed less or equal
+    ICMP_EQ = 32,  ///< equal
+    ICMP_NE = 33,  ///< not equal
+    ICMP_UGT = 34, ///< unsigned greater than
+    ICMP_UGE = 35, ///< unsigned greater or equal
+    ICMP_ULT = 36, ///< unsigned less than
+    ICMP_ULE = 37, ///< unsigned less or equal
+    ICMP_SGT = 38, ///< signed greater than
+    ICMP_SGE = 39, ///< signed greater or equal
+    ICMP_SLT = 40, ///< signed less than
+    ICMP_SLE = 41, ///< signed less or equal
     FIRST_ICMP_PREDICATE = ICMP_EQ,
     LAST_ICMP_PREDICATE = ICMP_SLE,
     BAD_ICMP_PREDICATE = ICMP_SLE + 1
   };
+  using PredicateField =
+      Bitfield::Element<Predicate, 0, 6, LAST_ICMP_PREDICATE>;
 
 protected:
   CmpInst(Type *ty, Instruction::OtherOps op, Predicate pred,
@@ -797,15 +799,15 @@ class CmpInst : public Instruction {
   }
 
   /// Return the predicate for this instruction.
-  Predicate getPredicate() const {
-    return Predicate(getSubclassDataFromInstruction());
-  }
+  Predicate getPredicate() const { return getSubclassData<PredicateField>(); }
 
   /// Set the predicate for this instruction to the specified value.
-  void setPredicate(Predicate P) { setInstructionSubclassData(P); }
+  void setPredicate(Predicate P) { setSubclassData<PredicateField>(P); }
 
   static bool isFPPredicate(Predicate P) {
-    return P >= FIRST_FCMP_PREDICATE && P <= LAST_FCMP_PREDICATE;
+    assert(FIRST_FCMP_PREDICATE == 0 &&
+           "FIRST_FCMP_PREDICATE is required to be 0");
+    return P <= LAST_FCMP_PREDICATE;
   }
 
   static bool isIntPredicate(Predicate P) {
@@ -1097,6 +1099,15 @@ using ConstOperandBundleDef = OperandBundleDefT<const Value *>;
 /// as cheap as most other operations on the base class.
 class CallBase : public Instruction {
 protected:
+  // The first two bits are reserved by CallInst for fast retrieval,
+  using CallInstReservedField = Bitfield::Element<unsigned, 0, 2>;
+  using CallingConvField =
+      Bitfield::Element<CallingConv::ID, CallInstReservedField::NextBit, 10,
+                        CallingConv::MaxID>;
+  static_assert(
+      Bitfield::areContiguous<CallInstReservedField, CallingConvField>(),
+      "Bitfields must be contiguous");
+
   /// The last operand is the called operand.
   static constexpr int CalledOperandOpEndIdx = -1;
 
@@ -1130,6 +1141,15 @@ class CallBase : public Instruction {
 public:
   using Instruction::getContext;
 
+  /// Create a clone of \p CB with a different set of operand bundles and
+  /// insert it before \p InsertPt.
+  ///
+  /// The returned call instruction is identical \p CB in every way except that
+  /// the operand bundles for the new instruction are set to the operand bundles
+  /// in \p Bundles.
+  static CallBase *Create(CallBase *CB, ArrayRef<OperandBundleDef> Bundles,
+                          Instruction *InsertPt = nullptr);
+
   static bool classof(const Instruction *I) {
     return I->getOpcode() == Instruction::Call ||
            I->getOpcode() == Instruction::Invoke ||
@@ -1349,14 +1369,11 @@ class CallBase : public Instruction {
   }
 
   CallingConv::ID getCallingConv() const {
-    return static_cast<CallingConv::ID>(getSubclassDataFromInstruction() >> 2);
+    return getSubclassData<CallingConvField>();
   }
 
   void setCallingConv(CallingConv::ID CC) {
-    auto ID = static_cast<unsigned>(CC);
-    assert(!(ID & ~CallingConv::MaxID) && "Unsupported calling convention");
-    setInstructionSubclassData((getSubclassDataFromInstruction() & 3) |
-                               (ID << 2));
+    setSubclassData<CallingConvField>(CC);
   }
 
   /// Check if this call is an inline asm statement.
@@ -1723,6 +1740,9 @@ class CallBase : public Instruction {
 
   /// Determine if the call cannot be tail merged.
   bool cannotMerge() const { return hasFnAttr(Attribute::NoMerge); }
+  void setCannotMerge() {
+    addAttribute(AttributeList::FunctionIndex, Attribute::NoMerge);
+  }
 
   /// Determine if the invoke is convergent
   bool isConvergent() const { return hasFnAttr(Attribute::Convergent); }
diff --git a/llvm/include/llvm/IR/Instruction.h b/llvm/include/llvm/IR/Instruction.h
index e604c6608e865..a03eac0ad40d6 100644
--- a/llvm/include/llvm/IR/Instruction.h
+++ b/llvm/include/llvm/IR/Instruction.h
@@ -15,6 +15,7 @@
 #define LLVM_IR_INSTRUCTION_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/Bitfields.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/ilist_node.h"
@@ -22,6 +23,7 @@
 #include "llvm/IR/SymbolTableListTraits.h"
 #include "llvm/IR/User.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/Casting.h"
 #include <algorithm>
 #include <cassert>
@@ -49,11 +51,33 @@ class Instruction : public User,
   /// O(1) local dominance checks between instructions.
   mutable unsigned Order = 0;
 
-  enum {
-    /// This is a bit stored in the SubClassData field which indicates whether
-    /// this instruction has metadata attached to it or not.
-    HasMetadataBit = 1 << 15
-  };
+protected:
+  // The 15 first bits of `Value::SubclassData` are available for subclasses of
+  // `Instruction` to use.
+  using OpaqueField = Bitfield::Element<uint16_t, 0, 15>;
+
+  // Template alias so that all Instruction storing alignment use the same
+  // definiton.
+  // Valid alignments are powers of two from 2^0 to 2^MaxAlignmentExponent =
+  // 2^29. We store them as Log2(Alignment), so we need 5 bits to encode the 30
+  // possible values.
+  template <unsigned Offset>
+  using AlignmentBitfieldElementT =
+      typename Bitfield::Element<unsigned, Offset, 5,
+                                 Value::MaxAlignmentExponent>;
+
+  template <unsigned Offset>
+  using BoolBitfieldElementT = typename Bitfield::Element<bool, Offset, 1>;
+
+  template <unsigned Offset>
+  using AtomicOrderingBitfieldElementT =
+      typename Bitfield::Element<AtomicOrdering, Offset, 3,
+                                 AtomicOrdering::LAST>;
+
+private:
+  // The last bit is used to store whether the instruction has metadata attached
+  // or not.
+  using HasMetadataField = Bitfield::Element<bool, 15, 1>;
 
 protected:
   ~Instruction(); // Use deleteValue() to delete a generic Instruction.
@@ -471,7 +495,7 @@ class Instruction : public User,
 private:
   /// Return true if we have an entry in the on-the-side metadata hash.
   bool hasMetadataHashEntry() const {
-    return (getSubclassDataFromValue() & HasMetadataBit) != 0;
+    return Bitfield::test<HasMetadataField>(getSubclassDataFromValue());
   }
 
   // These are all implemented in Metadata.cpp.
@@ -763,10 +787,7 @@ class Instruction : public User,
     return Value::getSubclassDataFromValue();
   }
 
-  void setHasMetadataHashEntry(bool V) {
-    setValueSubclassData((getSubclassDataFromValue() & ~HasMetadataBit) |
-                         (V ? HasMetadataBit : 0));
-  }
+  void setHasMetadataHashEntry(bool V) { setSubclassData<HasMetadataField>(V); }
 
   void setParent(BasicBlock *P);
 
@@ -774,14 +795,24 @@ class Instruction : public User,
   // Instruction subclasses can stick up to 15 bits of stuff into the
   // SubclassData field of instruction with these members.
 
-  // Verify that only the low 15 bits are used.
-  void setInstructionSubclassData(unsigned short D) {
-    assert((D & HasMetadataBit) == 0 && "Out of range value put into field");
-    setValueSubclassData((getSubclassDataFromValue() & HasMetadataBit) | D);
-  }
-
-  unsigned getSubclassDataFromInstruction() const {
-    return getSubclassDataFromValue() & ~HasMetadataBit;
+  template <typename BitfieldElement>
+  typename BitfieldElement::Type getSubclassData() const {
+    static_assert(
+        std::is_same<BitfieldElement, HasMetadataField>::value ||
+            !Bitfield::isOverlapping<BitfieldElement, HasMetadataField>(),
+        "Must not overlap with the metadata bit");
+    return Bitfield::get<BitfieldElement>(getSubclassDataFromValue());
+  }
+
+  template <typename BitfieldElement>
+  void setSubclassData(typename BitfieldElement::Type Value) {
+    static_assert(
+        std::is_same<BitfieldElement, HasMetadataField>::value ||
+            !Bitfield::isOverlapping<BitfieldElement, HasMetadataField>(),
+        "Must not overlap with the metadata bit");
+    auto Storage = getSubclassDataFromValue();
+    Bitfield::set<BitfieldElement>(Storage, Value);
+    setValueSubclassData(Storage);
   }
 
   Instruction(Type *Ty, unsigned iType, Use *Ops, unsigned NumOps,
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 7710542efd95d..0afc585dfbe5a 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -16,6 +16,7 @@
 #define LLVM_IR_INSTRUCTIONS_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/Bitfields.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
@@ -59,6 +60,13 @@ class LLVMContext;
 class AllocaInst : public UnaryInstruction {
   Type *AllocatedType;
 
+  using AlignmentField = AlignmentBitfieldElementT<0>;
+  using UsedWithInAllocaField = BoolBitfieldElementT<AlignmentField::NextBit>;
+  using SwiftErrorField = BoolBitfieldElementT<UsedWithInAllocaField::NextBit>;
+  static_assert(Bitfield::areContiguous<AlignmentField, UsedWithInAllocaField,
+                                        SwiftErrorField>(),
+                "Bitfields must be contiguous");
+
 protected:
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
@@ -108,11 +116,15 @@ class AllocaInst : public UnaryInstruction {
   /// Return the alignment of the memory that is being allocated by the
   /// instruction.
   Align getAlign() const {
-    return *decodeMaybeAlign(getSubclassDataFromInstruction() & 31);
+    return Align(1ULL << getSubclassData<AlignmentField>());
+  }
+
+  void setAlignment(Align Align) {
+    setSubclassData<AlignmentField>(Log2(Align));
   }
+
   // FIXME: Remove this one transition to Align is over.
   unsigned getAlignment() const { return getAlign().value(); }
-  void setAlignment(Align Align);
 
   /// Return true if this alloca is in the entry block of the function and is a
   /// constant size. If so, the code generator will fold it into the
@@ -122,25 +134,18 @@ class AllocaInst : public UnaryInstruction {
   /// Return true if this alloca is used as an inalloca argument to a call. Such
   /// allocas are never considered static even if they are in the entry block.
   bool isUsedWithInAlloca() const {
-    return getSubclassDataFromInstruction() & 32;
+    return getSubclassData<UsedWithInAllocaField>();
   }
 
   /// Specify whether this alloca is used to represent the arguments to a call.
   void setUsedWithInAlloca(bool V) {
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~32) |
-                               (V ? 32 : 0));
+    setSubclassData<UsedWithInAllocaField>(V);
   }
 
   /// Return true if this alloca is used as a swifterror argument to a call.
-  bool isSwiftError() const {
-    return getSubclassDataFromInstruction() & 64;
-  }
-
+  bool isSwiftError() const { return getSubclassData<SwiftErrorField>(); }
   /// Specify whether this alloca is used to represent a swifterror.
-  void setSwiftError(bool V) {
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~64) |
-                               (V ? 64 : 0));
-  }
+  void setSwiftError(bool V) { setSubclassData<SwiftErrorField>(V); }
 
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Instruction *I) {
@@ -153,8 +158,9 @@ class AllocaInst : public UnaryInstruction {
 private:
   // Shadow Instruction::setInstructionSubclassData with a private forwarding
   // method so that subclasses cannot accidentally use it.
-  void setInstructionSubclassData(unsigned short D) {
-    Instruction::setInstructionSubclassData(D);
+  template <typename Bitfield>
+  void setSubclassData(typename Bitfield::Type Value) {
+    Instruction::setSubclassData<Bitfield>(Value);
   }
 };
 
@@ -165,6 +171,13 @@ class AllocaInst : public UnaryInstruction {
 /// An instruction for reading from memory. This uses the SubclassData field in
 /// Value to store whether or not the load is volatile.
 class LoadInst : public UnaryInstruction {
+  using VolatileField = BoolBitfieldElementT<0>;
+  using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
+  using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
+  static_assert(
+      Bitfield::areContiguous<VolatileField, AlignmentField, OrderingField>(),
+      "Bitfields must be contiguous");
+
   void AssertOK();
 
 protected:
@@ -194,13 +207,10 @@ class LoadInst : public UnaryInstruction {
            BasicBlock *InsertAtEnd);
 
   /// Return true if this is a load from a volatile memory location.
-  bool isVolatile() const { return getSubclassDataFromInstruction() & 1; }
+  bool isVolatile() const { return getSubclassData<VolatileField>(); }
 
   /// Specify whether this is a volatile load or not.
-  void setVolatile(bool V) {
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
-                               (V ? 1 : 0));
-  }
+  void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
 
   /// Return the alignment of the access that is being performed.
   /// FIXME: Remove this function once transition to Align is over.
@@ -209,21 +219,21 @@ class LoadInst : public UnaryInstruction {
 
   /// Return the alignment of the access that is being performed.
   Align getAlign() const {
-    return *decodeMaybeAlign((getSubclassDataFromInstruction() >> 1) & 31);
+    return Align(1ULL << (getSubclassData<AlignmentField>()));
   }
 
-  void setAlignment(Align Alignment);
+  void setAlignment(Align Align) {
+    setSubclassData<AlignmentField>(Log2(Align));
+  }
 
   /// Returns the ordering constraint of this load instruction.
   AtomicOrdering getOrdering() const {
-    return AtomicOrdering((getSubclassDataFromInstruction() >> 7) & 7);
+    return getSubclassData<OrderingField>();
   }
-
   /// Sets the ordering constraint of this load instruction.  May not be Release
   /// or AcquireRelease.
   void setOrdering(AtomicOrdering Ordering) {
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~(7 << 7)) |
-                               ((unsigned)Ordering << 7));
+    setSubclassData<OrderingField>(Ordering);
   }
 
   /// Returns the synchronization scope ID of this load instruction.
@@ -273,8 +283,9 @@ class LoadInst : public UnaryInstruction {
 private:
   // Shadow Instruction::setInstructionSubclassData with a private forwarding
   // method so that subclasses cannot accidentally use it.
-  void setInstructionSubclassData(unsigned short D) {
-    Instruction::setInstructionSubclassData(D);
+  template <typename Bitfield>
+  void setSubclassData(typename Bitfield::Type Value) {
+    Instruction::setSubclassData<Bitfield>(Value);
   }
 
   /// The synchronization scope ID of this load instruction.  Not quite enough
@@ -289,6 +300,13 @@ class LoadInst : public UnaryInstruction {
 
 /// An instruction for storing to memory.
 class StoreInst : public Instruction {
+  using VolatileField = BoolBitfieldElementT<0>;
+  using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
+  using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
+  static_assert(
+      Bitfield::areContiguous<VolatileField, AlignmentField, OrderingField>(),
+      "Bitfields must be contiguous");
+
   void AssertOK();
 
 protected:
@@ -318,13 +336,10 @@ class StoreInst : public Instruction {
   }
 
   /// Return true if this is a store to a volatile memory location.
-  bool isVolatile() const { return getSubclassDataFromInstruction() & 1; }
+  bool isVolatile() const { return getSubclassData<VolatileField>(); }
 
   /// Specify whether this is a volatile store or not.
-  void setVolatile(bool V) {
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
-                               (V ? 1 : 0));
-  }
+  void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
 
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
@@ -335,21 +350,22 @@ class StoreInst : public Instruction {
   unsigned getAlignment() const { return getAlign().value(); }
 
   Align getAlign() const {
-    return *decodeMaybeAlign((getSubclassDataFromInstruction() >> 1) & 31);
+    return Align(1ULL << (getSubclassData<AlignmentField>()));
   }
 
-  void setAlignment(Align Alignment);
+  void setAlignment(Align Align) {
+    setSubclassData<AlignmentField>(Log2(Align));
+  }
 
   /// Returns the ordering constraint of this store instruction.
   AtomicOrdering getOrdering() const {
-    return AtomicOrdering((getSubclassDataFromInstruction() >> 7) & 7);
+    return getSubclassData<OrderingField>();
   }
 
   /// Sets the ordering constraint of this store instruction.  May not be
   /// Acquire or AcquireRelease.
   void setOrdering(AtomicOrdering Ordering) {
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~(7 << 7)) |
-                               ((unsigned)Ordering << 7));
+    setSubclassData<OrderingField>(Ordering);
   }
 
   /// Returns the synchronization scope ID of this store instruction.
@@ -402,8 +418,9 @@ class StoreInst : public Instruction {
 private:
   // Shadow Instruction::setInstructionSubclassData with a private forwarding
   // method so that subclasses cannot accidentally use it.
-  void setInstructionSubclassData(unsigned short D) {
-    Instruction::setInstructionSubclassData(D);
+  template <typename Bitfield>
+  void setSubclassData(typename Bitfield::Type Value) {
+    Instruction::setSubclassData<Bitfield>(Value);
   }
 
   /// The synchronization scope ID of this store instruction.  Not quite enough
@@ -424,6 +441,8 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(StoreInst, Value)
 
 /// An instruction for ordering other memory operations.
 class FenceInst : public Instruction {
+  using OrderingField = AtomicOrderingBitfieldElementT<0>;
+
   void Init(AtomicOrdering Ordering, SyncScope::ID SSID);
 
 protected:
@@ -448,14 +467,13 @@ class FenceInst : public Instruction {
 
   /// Returns the ordering constraint of this fence instruction.
   AtomicOrdering getOrdering() const {
-    return AtomicOrdering(getSubclassDataFromInstruction() >> 1);
+    return getSubclassData<OrderingField>();
   }
 
   /// Sets the ordering constraint of this fence instruction.  May only be
   /// Acquire, Release, AcquireRelease, or SequentiallyConsistent.
   void setOrdering(AtomicOrdering Ordering) {
-    setInstructionSubclassData((getSubclassDataFromInstruction() & 1) |
-                               ((unsigned)Ordering << 1));
+    setSubclassData<OrderingField>(Ordering);
   }
 
   /// Returns the synchronization scope ID of this fence instruction.
@@ -479,8 +497,9 @@ class FenceInst : public Instruction {
 private:
   // Shadow Instruction::setInstructionSubclassData with a private forwarding
   // method so that subclasses cannot accidentally use it.
-  void setInstructionSubclassData(unsigned short D) {
-    Instruction::setInstructionSubclassData(D);
+  template <typename Bitfield>
+  void setSubclassData(typename Bitfield::Type Value) {
+    Instruction::setSubclassData<Bitfield>(Value);
   }
 
   /// The synchronization scope ID of this fence instruction.  Not quite enough
@@ -500,10 +519,15 @@ class FenceInst : public Instruction {
 /// failure (false) as second element.
 ///
 class AtomicCmpXchgInst : public Instruction {
-  void Init(Value *Ptr, Value *Cmp, Value *NewVal,
+  void Init(Value *Ptr, Value *Cmp, Value *NewVal, Align Align,
             AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
             SyncScope::ID SSID);
 
+  template <unsigned Offset>
+  using AtomicOrderingBitfieldElement =
+      typename Bitfield::Element<AtomicOrdering, Offset, 3,
+                                 AtomicOrdering::LAST>;
+
 protected:
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
@@ -511,71 +535,82 @@ class AtomicCmpXchgInst : public Instruction {
   AtomicCmpXchgInst *cloneImpl() const;
 
 public:
-  AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
+  AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, Align Alignment,
                     AtomicOrdering SuccessOrdering,
-                    AtomicOrdering FailureOrdering,
-                    SyncScope::ID SSID, Instruction *InsertBefore = nullptr);
-  AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
+                    AtomicOrdering FailureOrdering, SyncScope::ID SSID,
+                    Instruction *InsertBefore = nullptr);
+  AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, Align Alignment,
                     AtomicOrdering SuccessOrdering,
-                    AtomicOrdering FailureOrdering,
-                    SyncScope::ID SSID, BasicBlock *InsertAtEnd);
+                    AtomicOrdering FailureOrdering, SyncScope::ID SSID,
+                    BasicBlock *InsertAtEnd);
 
   // allocate space for exactly three operands
   void *operator new(size_t s) {
     return User::operator new(s, 3);
   }
 
+  using VolatileField = BoolBitfieldElementT<0>;
+  using WeakField = BoolBitfieldElementT<VolatileField::NextBit>;
+  using SuccessOrderingField =
+      AtomicOrderingBitfieldElementT<WeakField::NextBit>;
+  using FailureOrderingField =
+      AtomicOrderingBitfieldElementT<SuccessOrderingField::NextBit>;
+  using AlignmentField =
+      AlignmentBitfieldElementT<FailureOrderingField::NextBit>;
+  static_assert(
+      Bitfield::areContiguous<VolatileField, WeakField, SuccessOrderingField,
+                              FailureOrderingField, AlignmentField>(),
+      "Bitfields must be contiguous");
+
+  /// Return the alignment of the memory that is being allocated by the
+  /// instruction.
+  Align getAlign() const {
+    return Align(1ULL << getSubclassData<AlignmentField>());
+  }
+
+  void setAlignment(Align Align) {
+    setSubclassData<AlignmentField>(Log2(Align));
+  }
+
   /// Return true if this is a cmpxchg from a volatile memory
   /// location.
   ///
-  bool isVolatile() const {
-    return getSubclassDataFromInstruction() & 1;
-  }
+  bool isVolatile() const { return getSubclassData<VolatileField>(); }
 
   /// Specify whether this is a volatile cmpxchg.
   ///
-  void setVolatile(bool V) {
-     setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
-                                (unsigned)V);
-  }
+  void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
 
   /// Return true if this cmpxchg may spuriously fail.
-  bool isWeak() const {
-    return getSubclassDataFromInstruction() & 0x100;
-  }
+  bool isWeak() const { return getSubclassData<WeakField>(); }
 
-  void setWeak(bool IsWeak) {
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~0x100) |
-                               (IsWeak << 8));
-  }
+  void setWeak(bool IsWeak) { setSubclassData<WeakField>(IsWeak); }
 
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
 
   /// Returns the success ordering constraint of this cmpxchg instruction.
   AtomicOrdering getSuccessOrdering() const {
-    return AtomicOrdering((getSubclassDataFromInstruction() >> 2) & 7);
+    return getSubclassData<SuccessOrderingField>();
   }
 
   /// Sets the success ordering constraint of this cmpxchg instruction.
   void setSuccessOrdering(AtomicOrdering Ordering) {
     assert(Ordering != AtomicOrdering::NotAtomic &&
            "CmpXchg instructions can only be atomic.");
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~0x1c) |
-                               ((unsigned)Ordering << 2));
+    setSubclassData<SuccessOrderingField>(Ordering);
   }
 
   /// Returns the failure ordering constraint of this cmpxchg instruction.
   AtomicOrdering getFailureOrdering() const {
-    return AtomicOrdering((getSubclassDataFromInstruction() >> 5) & 7);
+    return getSubclassData<FailureOrderingField>();
   }
 
   /// Sets the failure ordering constraint of this cmpxchg instruction.
   void setFailureOrdering(AtomicOrdering Ordering) {
     assert(Ordering != AtomicOrdering::NotAtomic &&
            "CmpXchg instructions can only be atomic.");
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~0xe0) |
-                               ((unsigned)Ordering << 5));
+    setSubclassData<FailureOrderingField>(Ordering);
   }
 
   /// Returns the synchronization scope ID of this cmpxchg instruction.
@@ -637,8 +672,9 @@ class AtomicCmpXchgInst : public Instruction {
 private:
   // Shadow Instruction::setInstructionSubclassData with a private forwarding
   // method so that subclasses cannot accidentally use it.
-  void setInstructionSubclassData(unsigned short D) {
-    Instruction::setInstructionSubclassData(D);
+  template <typename Bitfield>
+  void setSubclassData(typename Bitfield::Type Value) {
+    Instruction::setSubclassData<Bitfield>(Value);
   }
 
   /// The synchronization scope ID of this cmpxchg instruction.  Not quite
@@ -674,7 +710,7 @@ class AtomicRMWInst : public Instruction {
   /// the descriptions, 'p' is the pointer to the instruction's memory location,
   /// 'old' is the initial value of *p, and 'v' is the other value passed to the
   /// instruction.  These instructions always return 'old'.
-  enum BinOp {
+  enum BinOp : unsigned {
     /// *p = v
     Xchg,
     /// *p = old + v
@@ -709,10 +745,21 @@ class AtomicRMWInst : public Instruction {
     BAD_BINOP
   };
 
-  AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
+private:
+  template <unsigned Offset>
+  using AtomicOrderingBitfieldElement =
+      typename Bitfield::Element<AtomicOrdering, Offset, 3,
+                                 AtomicOrdering::LAST>;
+
+  template <unsigned Offset>
+  using BinOpBitfieldElement =
+      typename Bitfield::Element<BinOp, Offset, 4, BinOp::LAST_BINOP>;
+
+public:
+  AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, Align Alignment,
                 AtomicOrdering Ordering, SyncScope::ID SSID,
                 Instruction *InsertBefore = nullptr);
-  AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
+  AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, Align Alignment,
                 AtomicOrdering Ordering, SyncScope::ID SSID,
                 BasicBlock *InsertAtEnd);
 
@@ -721,9 +768,16 @@ class AtomicRMWInst : public Instruction {
     return User::operator new(s, 2);
   }
 
-  BinOp getOperation() const {
-    return static_cast<BinOp>(getSubclassDataFromInstruction() >> 5);
-  }
+  using VolatileField = BoolBitfieldElementT<0>;
+  using AtomicOrderingField =
+      AtomicOrderingBitfieldElementT<VolatileField::NextBit>;
+  using OperationField = BinOpBitfieldElement<AtomicOrderingField::NextBit>;
+  using AlignmentField = AlignmentBitfieldElementT<OperationField::NextBit>;
+  static_assert(Bitfield::areContiguous<VolatileField, AtomicOrderingField,
+                                        OperationField, AlignmentField>(),
+                "Bitfields must be contiguous");
+
+  BinOp getOperation() const { return getSubclassData<OperationField>(); }
 
   static StringRef getOperationName(BinOp Op);
 
@@ -738,38 +792,40 @@ class AtomicRMWInst : public Instruction {
   }
 
   void setOperation(BinOp Operation) {
-    unsigned short SubclassData = getSubclassDataFromInstruction();
-    setInstructionSubclassData((SubclassData & 31) |
-                               (Operation << 5));
+    setSubclassData<OperationField>(Operation);
+  }
+
+  /// Return the alignment of the memory that is being allocated by the
+  /// instruction.
+  Align getAlign() const {
+    return Align(1ULL << getSubclassData<AlignmentField>());
+  }
+
+  void setAlignment(Align Align) {
+    setSubclassData<AlignmentField>(Log2(Align));
   }
 
   /// Return true if this is a RMW on a volatile memory location.
   ///
-  bool isVolatile() const {
-    return getSubclassDataFromInstruction() & 1;
-  }
+  bool isVolatile() const { return getSubclassData<VolatileField>(); }
 
   /// Specify whether this is a volatile RMW or not.
   ///
-  void setVolatile(bool V) {
-     setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
-                                (unsigned)V);
-  }
+  void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
 
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
 
   /// Returns the ordering constraint of this rmw instruction.
   AtomicOrdering getOrdering() const {
-    return AtomicOrdering((getSubclassDataFromInstruction() >> 2) & 7);
+    return getSubclassData<AtomicOrderingField>();
   }
 
   /// Sets the ordering constraint of this rmw instruction.
   void setOrdering(AtomicOrdering Ordering) {
     assert(Ordering != AtomicOrdering::NotAtomic &&
            "atomicrmw instructions can only be atomic.");
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~(7 << 2)) |
-                               ((unsigned)Ordering << 2));
+    setSubclassData<AtomicOrderingField>(Ordering);
   }
 
   /// Returns the synchronization scope ID of this rmw instruction.
@@ -807,13 +863,14 @@ class AtomicRMWInst : public Instruction {
   }
 
 private:
-  void Init(BinOp Operation, Value *Ptr, Value *Val,
+  void Init(BinOp Operation, Value *Ptr, Value *Val, Align Align,
             AtomicOrdering Ordering, SyncScope::ID SSID);
 
   // Shadow Instruction::setInstructionSubclassData with a private forwarding
   // method so that subclasses cannot accidentally use it.
-  void setInstructionSubclassData(unsigned short D) {
-    Instruction::setInstructionSubclassData(D);
+  template <typename Bitfield>
+  void setSubclassData(typename Bitfield::Type Value) {
+    Instruction::setSubclassData<Bitfield>(Value);
   }
 
   /// The synchronization scope ID of this rmw instruction.  Not quite enough
@@ -1542,37 +1599,38 @@ class CallInst : public CallBase {
                                  BasicBlock *InsertAtEnd);
 
   // Note that 'musttail' implies 'tail'.
-  enum TailCallKind {
+  enum TailCallKind : unsigned {
     TCK_None = 0,
     TCK_Tail = 1,
     TCK_MustTail = 2,
-    TCK_NoTail = 3
+    TCK_NoTail = 3,
+    TCK_LAST = TCK_NoTail
   };
+
+  using TailCallKindField = Bitfield::Element<TailCallKind, 0, 2, TCK_LAST>;
+  static_assert(
+      Bitfield::areContiguous<TailCallKindField, CallBase::CallingConvField>(),
+      "Bitfields must be contiguous");
+
   TailCallKind getTailCallKind() const {
-    return TailCallKind(getSubclassDataFromInstruction() & 3);
+    return getSubclassData<TailCallKindField>();
   }
 
   bool isTailCall() const {
-    unsigned Kind = getSubclassDataFromInstruction() & 3;
+    TailCallKind Kind = getTailCallKind();
     return Kind == TCK_Tail || Kind == TCK_MustTail;
   }
 
-  bool isMustTailCall() const {
-    return (getSubclassDataFromInstruction() & 3) == TCK_MustTail;
-  }
+  bool isMustTailCall() const { return getTailCallKind() == TCK_MustTail; }
 
-  bool isNoTailCall() const {
-    return (getSubclassDataFromInstruction() & 3) == TCK_NoTail;
-  }
+  bool isNoTailCall() const { return getTailCallKind() == TCK_NoTail; }
 
-  void setTailCall(bool isTC = true) {
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~3) |
-                               unsigned(isTC ? TCK_Tail : TCK_None));
+  void setTailCallKind(TailCallKind TCK) {
+    setSubclassData<TailCallKindField>(TCK);
   }
 
-  void setTailCallKind(TailCallKind TCK) {
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~3) |
-                               unsigned(TCK));
+  void setTailCall(bool IsTc = true) {
+    setTailCallKind(IsTc ? TCK_Tail : TCK_None);
   }
 
   /// Return true if the call can return twice
@@ -1595,8 +1653,9 @@ class CallInst : public CallBase {
 private:
   // Shadow Instruction::setInstructionSubclassData with a private forwarding
   // method so that subclasses cannot accidentally use it.
-  void setInstructionSubclassData(unsigned short D) {
-    Instruction::setInstructionSubclassData(D);
+  template <typename Bitfield>
+  void setSubclassData(typename Bitfield::Type Value) {
+    Instruction::setSubclassData<Bitfield>(Value);
   }
 };
 
@@ -2715,6 +2774,8 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(PHINode, Value)
 /// cleanup.
 ///
 class LandingPadInst : public Instruction {
+  using CleanupField = BoolBitfieldElementT<0>;
+
   /// The number of operands actually allocated.  NumOperands is
   /// the number actually in use.
   unsigned ReservedSpace;
@@ -2759,13 +2820,10 @@ class LandingPadInst : public Instruction {
   /// Return 'true' if this landingpad instruction is a
   /// cleanup. I.e., it should be run when unwinding even if its landing pad
   /// doesn't catch the exception.
-  bool isCleanup() const { return getSubclassDataFromInstruction() & 1; }
+  bool isCleanup() const { return getSubclassData<CleanupField>(); }
 
   /// Indicate that this landingpad instruction is a cleanup.
-  void setCleanup(bool V) {
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
-                               (V ? 1 : 0));
-  }
+  void setCleanup(bool V) { setSubclassData<CleanupField>(V); }
 
   /// Add a catch or filter clause to the landing pad.
   void addClause(Constant *ClauseVal);
@@ -3752,11 +3810,11 @@ class InvokeInst : public CallBase {
   }
 
 private:
-
   // Shadow Instruction::setInstructionSubclassData with a private forwarding
   // method so that subclasses cannot accidentally use it.
-  void setInstructionSubclassData(unsigned short D) {
-    Instruction::setInstructionSubclassData(D);
+  template <typename Bitfield>
+  void setSubclassData(typename Bitfield::Type Value) {
+    Instruction::setSubclassData<Bitfield>(Value);
   }
 };
 
@@ -3992,11 +4050,11 @@ class CallBrInst : public CallBase {
   }
 
 private:
-
   // Shadow Instruction::setInstructionSubclassData with a private forwarding
   // method so that subclasses cannot accidentally use it.
-  void setInstructionSubclassData(unsigned short D) {
-    Instruction::setInstructionSubclassData(D);
+  template <typename Bitfield>
+  void setSubclassData(typename Bitfield::Type Value) {
+    Instruction::setSubclassData<Bitfield>(Value);
   }
 };
 
@@ -4087,6 +4145,8 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ResumeInst, Value)
 //                         CatchSwitchInst Class
 //===----------------------------------------------------------------------===//
 class CatchSwitchInst : public Instruction {
+  using UnwindDestField = BoolBitfieldElementT<0>;
+
   /// The number of operands actually allocated.  NumOperands is
   /// the number actually in use.
   unsigned ReservedSpace;
@@ -4148,7 +4208,7 @@ class CatchSwitchInst : public Instruction {
   void setParentPad(Value *ParentPad) { setOperand(0, ParentPad); }
 
   // Accessor Methods for CatchSwitch stmt
-  bool hasUnwindDest() const { return getSubclassDataFromInstruction() & 1; }
+  bool hasUnwindDest() const { return getSubclassData<UnwindDestField>(); }
   bool unwindsToCaller() const { return !hasUnwindDest(); }
   BasicBlock *getUnwindDest() const {
     if (hasUnwindDest())
@@ -4434,6 +4494,8 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CatchReturnInst, Value)
 //===----------------------------------------------------------------------===//
 
 class CleanupReturnInst : public Instruction {
+  using UnwindDestField = BoolBitfieldElementT<0>;
+
 private:
   CleanupReturnInst(const CleanupReturnInst &RI);
   CleanupReturnInst(Value *CleanupPad, BasicBlock *UnwindBB, unsigned Values,
@@ -4474,7 +4536,7 @@ class CleanupReturnInst : public Instruction {
   /// Provide fast operand accessors
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
 
-  bool hasUnwindDest() const { return getSubclassDataFromInstruction() & 1; }
+  bool hasUnwindDest() const { return getSubclassData<UnwindDestField>(); }
   bool unwindsToCaller() const { return !hasUnwindDest(); }
 
   /// Convenience accessor.
@@ -4518,8 +4580,9 @@ class CleanupReturnInst : public Instruction {
 
   // Shadow Instruction::setInstructionSubclassData with a private forwarding
   // method so that subclasses cannot accidentally use it.
-  void setInstructionSubclassData(unsigned short D) {
-    Instruction::setInstructionSubclassData(D);
+  template <typename Bitfield>
+  void setSubclassData(typename Bitfield::Type Value) {
+    Instruction::setSubclassData<Bitfield>(Value);
   }
 };
 
diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h
index a44a9a4a70b0f..a9e6525e2f3d5 100644
--- a/llvm/include/llvm/IR/Intrinsics.h
+++ b/llvm/include/llvm/IR/Intrinsics.h
@@ -222,6 +222,13 @@ namespace Intrinsic {
   /// This method returns true on error.
   bool matchIntrinsicVarArg(bool isVarArg, ArrayRef<IITDescriptor> &Infos);
 
+  /// Gets the type arguments of an intrinsic call by matching type contraints
+  /// specified by the .td file. The overloaded types are pushed into the
+  /// AgTys vector.
+  ///
+  /// Returns false if the given function is not a valid intrinsic call.
+  bool getIntrinsicSignature(Function *F, SmallVectorImpl<Type *> &ArgTys);
+
   // Checks if the intrinsic name matches with its signature and if not
   // returns the declaration with the same signature and remangled name.
   llvm::Optional<Function*> remangleIntrinsicFunction(Function *F);
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index a4fb6c6d9cd5f..94741229a2a75 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -117,6 +117,8 @@ class ReadNone<AttrIndex idx> : IntrinsicProperty {
 
 def IntrNoReturn : IntrinsicProperty;
 
+def IntrNoFree : IntrinsicProperty;
+
 def IntrWillReturn : IntrinsicProperty;
 
 // IntrCold - Calls to this intrinsic are cold.
@@ -532,6 +534,7 @@ def int_instrprof_value_profile : Intrinsic<[],
 
 def int_call_preallocated_setup : Intrinsic<[llvm_token_ty], [llvm_i32_ty]>;
 def int_call_preallocated_arg : Intrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_i32_ty]>;
+def int_call_preallocated_teardown : Intrinsic<[], [llvm_token_ty]>;
 
 //===------------------- Standard C Library Intrinsics --------------------===//
 //
@@ -837,6 +840,10 @@ let IntrProperties = [IntrInaccessibleMemOnly, IntrWillReturn] in {
 def int_expect : Intrinsic<[llvm_anyint_ty],
   [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem, IntrWillReturn]>;
 
+def int_expect_with_probability : Intrinsic<[llvm_anyint_ty],
+  [LLVMMatchType<0>, LLVMMatchType<0>, llvm_double_ty],
+  [IntrNoMem, IntrWillReturn]>;
+
 //===-------------------- Bit Manipulation Intrinsics ---------------------===//
 //
 
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 483afe97cc634..3f71f644f9a1d 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -471,6 +471,16 @@ let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {
   def int_aarch64_neon_bfmlalt : AdvSIMD_FML_Intrinsic;
 
 
+  // v8.6-A Bfloat Intrinsics
+  def int_aarch64_neon_bfcvt
+    : Intrinsic<[llvm_bfloat_ty], [llvm_float_ty], [IntrNoMem]>;
+  def int_aarch64_neon_bfcvtn
+    : Intrinsic<[llvm_v8bf16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
+  def int_aarch64_neon_bfcvtn2
+    : Intrinsic<[llvm_v8bf16_ty],
+                [llvm_v8bf16_ty, llvm_v4f32_ty],
+                [IntrNoMem]>;
+
   // v8.2-A FP16 Fused Multiply-Add Long
   def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic;
   def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic;
@@ -1340,6 +1350,16 @@ class SVE_MatMul_Intrinsic
                 [LLVMMatchType<0>, LLVMSubdivide4VectorType<0>, LLVMSubdivide4VectorType<0>],
                 [IntrNoMem]>;
 
+class SVE_4Vec_BF16
+    : Intrinsic<[llvm_nxv4f32_ty],
+                [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty],
+                [IntrNoMem]>;
+
+class SVE_4Vec_BF16_Indexed
+    : Intrinsic<[llvm_nxv4f32_ty],
+                [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i64_ty],
+                [IntrNoMem, ImmArg<ArgIndex<3>>]>;
+
 //
 // Vector tuple creation intrinsics (ACLE)
 //
@@ -1793,6 +1813,9 @@ def int_aarch64_sve_fcvtzs_i32f64   : Builtin_SVCVT<"svcvt_s32_f64_m", llvm_nxv4
 def int_aarch64_sve_fcvtzs_i64f16   : Builtin_SVCVT<"svcvt_s64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
 def int_aarch64_sve_fcvtzs_i64f32   : Builtin_SVCVT<"svcvt_s64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
 
+def int_aarch64_sve_fcvt_bf16f32    : Builtin_SVCVT<"svcvt_bf16_f32_m",   llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
+def int_aarch64_sve_fcvtnt_bf16f32  : Builtin_SVCVT<"svcvtnt_bf16_f32_m", llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
+
 def int_aarch64_sve_fcvtzu_i32f16   : Builtin_SVCVT<"svcvt_u32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
 def int_aarch64_sve_fcvtzu_i32f64   : Builtin_SVCVT<"svcvt_u32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
 def int_aarch64_sve_fcvtzu_i64f16   : Builtin_SVCVT<"svcvt_u64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
@@ -2343,6 +2366,19 @@ def int_aarch64_sve_sudot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
 //
 def int_aarch64_sve_fmmla : AdvSIMD_3VectorArg_Intrinsic;
 
+//
+// SVE ACLE: 7.2. BFloat16 extensions
+//
+
+def int_aarch64_sve_bfdot   : SVE_4Vec_BF16;
+def int_aarch64_sve_bfmlalb : SVE_4Vec_BF16;
+def int_aarch64_sve_bfmlalt : SVE_4Vec_BF16;
+
+def int_aarch64_sve_bfmmla  : SVE_4Vec_BF16;
+
+def int_aarch64_sve_bfdot_lane   : SVE_4Vec_BF16_Indexed;
+def int_aarch64_sve_bfmlalb_lane : SVE_4Vec_BF16_Indexed;
+def int_aarch64_sve_bfmlalt_lane : SVE_4Vec_BF16_Indexed;
 }
 
 //
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 734128d682183..01380afae0068 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -267,6 +267,10 @@ def int_amdgcn_rcp_legacy : GCCBuiltin<"__builtin_amdgcn_rcp_legacy">,
   [IntrNoMem, IntrSpeculatable, IntrWillReturn]
 >;
 
+def int_amdgcn_sqrt :  Intrinsic<
+  [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable, IntrWillReturn]
+>;
+
 def int_amdgcn_rsq :  Intrinsic<
   [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable, IntrWillReturn]
 >;
diff --git a/llvm/include/llvm/IR/IntrinsicsARM.td b/llvm/include/llvm/IR/IntrinsicsARM.td
index adeafbb267b2b..df74e446b965a 100644
--- a/llvm/include/llvm/IR/IntrinsicsARM.td
+++ b/llvm/include/llvm/IR/IntrinsicsARM.td
@@ -785,6 +785,20 @@ def int_arm_neon_usmmla : Neon_MatMul_Intrinsic;
 def int_arm_neon_usdot  : Neon_Dot_Intrinsic;
 
 // v8.6-A Bfloat Intrinsics
+def int_arm_neon_vcvtfp2bf
+    : Intrinsic<[llvm_anyvector_ty], [llvm_v4f32_ty], [IntrNoMem]>;
+def int_arm_neon_vcvtbfp2bf
+    : Intrinsic<[llvm_bfloat_ty], [llvm_float_ty], [IntrNoMem]>;
+
+def int_arm_neon_bfdot : Neon_Dot_Intrinsic;
+def int_arm_neon_bfmmla : Neon_MatMul_Intrinsic;
+
+class Neon_FML_Intrinsic
+  : Intrinsic<[llvm_anyvector_ty],
+              [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
+              [IntrNoMem]>;
+def int_arm_neon_bfmlalb : Neon_FML_Intrinsic;
+def int_arm_neon_bfmlalt : Neon_FML_Intrinsic;
 
 def int_arm_cls: Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
 def int_arm_cls64: Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>;
diff --git a/llvm/include/llvm/IR/IntrinsicsPowerPC.td b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
index 2ca77d90e13c2..12f4a3ce8e28f 100644
--- a/llvm/include/llvm/IR/IntrinsicsPowerPC.td
+++ b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
@@ -28,6 +28,10 @@ let TargetPrefix = "ppc" in {  // All intrinsics start with "llvm.ppc.".
     [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
   def int_ppc_dcbtst: Intrinsic<[], [llvm_ptr_ty],
     [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
+  def int_ppc_dcbt_with_hint: Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty],
+    [IntrArgMemOnly, NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>;
+  def int_ppc_dcbtst_with_hint: Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty],
+    [IntrArgMemOnly, NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>;
   def int_ppc_dcbz  : Intrinsic<[], [llvm_ptr_ty], []>;
   def int_ppc_dcbzl : Intrinsic<[], [llvm_ptr_ty], []>;
 
@@ -36,6 +40,8 @@ let TargetPrefix = "ppc" in {  // All intrinsics start with "llvm.ppc.".
 
   // sync instruction (i.e. sync 0, a.k.a hwsync)
   def int_ppc_sync : Intrinsic<[], [], []>;
+  // isync instruction
+  def int_ppc_isync : Intrinsic<[], [], []>;
   // lwsync is sync 1
   def int_ppc_lwsync : Intrinsic<[], [], []>;
   // eieio instruction
@@ -68,6 +74,19 @@ let TargetPrefix = "ppc" in {  // All intrinsics start with "llvm.ppc.".
       : GCCBuiltin<"__builtin_pextd">,
         Intrinsic <[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
 
+  // Centrifuge Doubleword Builtin.
+  def int_ppc_cfuged
+      : GCCBuiltin<"__builtin_cfuged">,
+        Intrinsic <[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
+
+  // Count Leading / Trailing Zeroes under bit Mask Builtins.
+  def int_ppc_cntlzdm
+      : GCCBuiltin<"__builtin_cntlzdm">,
+        Intrinsic <[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
+  def int_ppc_cnttzdm
+      : GCCBuiltin<"__builtin_cnttzdm">,
+        Intrinsic <[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
+
   def int_ppc_truncf128_round_to_odd
       : GCCBuiltin<"__builtin_truncf128_round_to_odd">,
         Intrinsic <[llvm_double_ty], [llvm_f128_ty], [IntrNoMem]>;
@@ -418,6 +437,16 @@ let TargetPrefix = "ppc" in {  // All intrinsics start with "llvm.ppc.".
               Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
                         [IntrNoMem]>;
 
+  // P10 Vector Centrifuge Builtin.
+  def int_ppc_altivec_vcfuged : GCCBuiltin<"__builtin_altivec_vcfuged">,
+              Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
+                        [IntrNoMem]>;
+
+  // P10 Vector Gather Every Nth Bit Builtin.
+  def int_ppc_altivec_vgnb : GCCBuiltin<"__builtin_altivec_vgnb">,
+              Intrinsic<[llvm_i64_ty], [llvm_v1i128_ty, llvm_i32_ty],
+                        [IntrNoMem, ImmArg<ArgIndex<1>>]>;
+
    // P10 Vector Clear Bytes
    def int_ppc_altivec_vclrlb :  GCCBuiltin<"__builtin_altivec_vclrlb">,
                Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty],
@@ -425,6 +454,83 @@ let TargetPrefix = "ppc" in {  // All intrinsics start with "llvm.ppc.".
    def int_ppc_altivec_vclrrb :  GCCBuiltin<"__builtin_altivec_vclrrb">,
                Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty],
                          [IntrNoMem]>;
+
+  // P10 Vector Shift Double Bit Immediate.
+  def int_ppc_altivec_vsldbi : GCCBuiltin<"__builtin_altivec_vsldbi">,
+              Intrinsic<[llvm_v16i8_ty],
+                        [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty],
+                        [IntrNoMem, ImmArg<ArgIndex<2>>]>;
+  def int_ppc_altivec_vsrdbi : GCCBuiltin<"__builtin_altivec_vsrdbi">,
+              Intrinsic<[llvm_v16i8_ty],
+                        [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty],
+                        [IntrNoMem, ImmArg<ArgIndex<2>>]>;
+
+  // P10 Vector Insert.
+  def int_ppc_altivec_vinsblx : GCCBuiltin<"__builtin_altivec_vinsblx">,
+              Intrinsic<[llvm_v16i8_ty],
+                        [llvm_v16i8_ty, llvm_i64_ty, llvm_i64_ty],
+                        [IntrNoMem]>;
+  def int_ppc_altivec_vinsbrx : GCCBuiltin<"__builtin_altivec_vinsbrx">,
+              Intrinsic<[llvm_v16i8_ty],
+                        [llvm_v16i8_ty, llvm_i64_ty, llvm_i64_ty],
+                        [IntrNoMem]>;
+  def int_ppc_altivec_vinshlx : GCCBuiltin<"__builtin_altivec_vinshlx">,
+              Intrinsic<[llvm_v8i16_ty],
+                        [llvm_v8i16_ty, llvm_i64_ty, llvm_i64_ty],
+                        [IntrNoMem]>;
+  def int_ppc_altivec_vinshrx : GCCBuiltin<"__builtin_altivec_vinshrx">,
+              Intrinsic<[llvm_v8i16_ty],
+                        [llvm_v8i16_ty, llvm_i64_ty, llvm_i64_ty],
+                        [IntrNoMem]>;
+  def int_ppc_altivec_vinswlx : GCCBuiltin<"__builtin_altivec_vinswlx">,
+              Intrinsic<[llvm_v4i32_ty],
+                        [llvm_v4i32_ty, llvm_i64_ty, llvm_i64_ty],
+                        [IntrNoMem]>;
+  def int_ppc_altivec_vinswrx : GCCBuiltin<"__builtin_altivec_vinswrx">,
+              Intrinsic<[llvm_v4i32_ty],
+                        [llvm_v4i32_ty, llvm_i64_ty, llvm_i64_ty],
+                        [IntrNoMem]>;
+  def int_ppc_altivec_vinsdlx : GCCBuiltin<"__builtin_altivec_vinsdlx">,
+              Intrinsic<[llvm_v2i64_ty],
+                        [llvm_v2i64_ty, llvm_i64_ty, llvm_i64_ty],
+                        [IntrNoMem]>;
+  def int_ppc_altivec_vinsdrx : GCCBuiltin<"__builtin_altivec_vinsdrx">,
+              Intrinsic<[llvm_v2i64_ty],
+                        [llvm_v2i64_ty, llvm_i64_ty, llvm_i64_ty],
+                        [IntrNoMem]>;
+  def int_ppc_altivec_vinsbvlx : GCCBuiltin<"__builtin_altivec_vinsbvlx">,
+              Intrinsic<[llvm_v16i8_ty],
+                        [llvm_v16i8_ty, llvm_i64_ty, llvm_v16i8_ty],
+                        [IntrNoMem]>;
+  def int_ppc_altivec_vinsbvrx : GCCBuiltin<"__builtin_altivec_vinsbvrx">,
+              Intrinsic<[llvm_v16i8_ty],
+                        [llvm_v16i8_ty, llvm_i64_ty, llvm_v16i8_ty],
+                        [IntrNoMem]>;
+  def int_ppc_altivec_vinshvlx : GCCBuiltin<"__builtin_altivec_vinshvlx">,
+              Intrinsic<[llvm_v8i16_ty],
+                        [llvm_v8i16_ty, llvm_i64_ty, llvm_v8i16_ty],
+                        [IntrNoMem]>;
+ def int_ppc_altivec_vinshvrx : GCCBuiltin<"__builtin_altivec_vinshvrx">,
+              Intrinsic<[llvm_v8i16_ty],
+                        [llvm_v8i16_ty, llvm_i64_ty, llvm_v8i16_ty],
+                        [IntrNoMem]>;
+  def int_ppc_altivec_vinswvlx : GCCBuiltin<"__builtin_altivec_vinswvlx">,
+              Intrinsic<[llvm_v4i32_ty],
+                        [llvm_v4i32_ty, llvm_i64_ty, llvm_v4i32_ty],
+                        [IntrNoMem]>;
+  def int_ppc_altivec_vinswvrx : GCCBuiltin<"__builtin_altivec_vinswvrx">,
+              Intrinsic<[llvm_v4i32_ty],
+                        [llvm_v4i32_ty, llvm_i64_ty, llvm_v4i32_ty],
+                        [IntrNoMem]>;
+  // P10 Vector Insert with immediate.
+  def int_ppc_altivec_vinsw :
+              Intrinsic<[llvm_v4i32_ty],
+                        [llvm_v4i32_ty, llvm_i64_ty, llvm_i32_ty],
+                        [IntrNoMem, ImmArg<ArgIndex<2>>]>;
+  def int_ppc_altivec_vinsd :
+              Intrinsic<[llvm_v2i64_ty],
+                        [llvm_v2i64_ty, llvm_i64_ty, llvm_i32_ty],
+                        [IntrNoMem, ImmArg<ArgIndex<2>>]>;
 }
 
 // Vector average.
@@ -673,6 +779,14 @@ let TargetPrefix = "ppc" in {  // All PPC intrinsics start with "llvm.ppc.".
               Intrinsic<[llvm_v1i128_ty],
                         [llvm_v1i128_ty, llvm_v1i128_ty, llvm_v1i128_ty],
                         [IntrNoMem]>;
+
+  // P10 Vector Count Leading / Trailing Zeroes under bit Mask Builtins.
+  def int_ppc_altivec_vclzdm : GCCBuiltin<"__builtin_altivec_vclzdm">,
+              Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
+                        [IntrNoMem]>;
+  def int_ppc_altivec_vctzdm : GCCBuiltin<"__builtin_altivec_vctzdm">,
+              Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
+                        [IntrNoMem]>;
 }
 
 def int_ppc_altivec_vsl   : PowerPC_Vec_WWW_Intrinsic<"vsl">;
@@ -953,6 +1067,43 @@ def int_ppc_vsx_xxinsertw :
       PowerPC_VSX_Intrinsic<"xxinsertw",[llvm_v4i32_ty],
                             [llvm_v4i32_ty,llvm_v2i64_ty,llvm_i32_ty],
                             [IntrNoMem]>;
+def int_ppc_vsx_xxeval :
+      PowerPC_VSX_Intrinsic<"xxeval", [llvm_v2i64_ty],
+                           [llvm_v2i64_ty, llvm_v2i64_ty,
+                            llvm_v2i64_ty, llvm_i32_ty],
+                           [IntrNoMem, ImmArg<ArgIndex<3>>]>;
+def int_ppc_vsx_xxgenpcvbm :
+      PowerPC_VSX_Intrinsic<"xxgenpcvbm", [llvm_v16i8_ty],
+                            [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>;
+def int_ppc_vsx_xxgenpcvhm :
+      PowerPC_VSX_Intrinsic<"xxgenpcvhm", [llvm_v8i16_ty],
+                            [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>;
+def int_ppc_vsx_xxgenpcvwm :
+      PowerPC_VSX_Intrinsic<"xxgenpcvwm", [llvm_v4i32_ty],
+                            [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>;
+def int_ppc_vsx_xxgenpcvdm :
+      PowerPC_VSX_Intrinsic<"xxgenpcvdm", [llvm_v2i64_ty],
+                            [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>;
+
+// P10 VSX Vector permute extended.
+def int_ppc_vsx_xxpermx : 
+      GCCBuiltin<"__builtin_vsx_xxpermx">,
+      Intrinsic<[llvm_v16i8_ty],
+                [llvm_v16i8_ty,llvm_v16i8_ty,llvm_v16i8_ty,llvm_i32_ty],
+                [IntrNoMem, ImmArg<ArgIndex<3>>]>;
+// P10 VSX Vector Blend Variable.
+def  int_ppc_vsx_xxblendvb: GCCBuiltin<"__builtin_vsx_xxblendvb">,
+       Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty],
+                 [IntrNoMem]>;
+def  int_ppc_vsx_xxblendvh: GCCBuiltin<"__builtin_vsx_xxblendvh">,
+       Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,llvm_v8i16_ty],
+                 [IntrNoMem]>;
+def  int_ppc_vsx_xxblendvw: GCCBuiltin<"__builtin_vsx_xxblendvw">,
+       Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
+                 [IntrNoMem]>;
+def  int_ppc_vsx_xxblendvd: GCCBuiltin<"__builtin_vsx_xxblendvd">,
+       Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty],
+                 [IntrNoMem]>;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
index 0c08aad22362e..7c9ceb148a472 100644
--- a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
+++ b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
@@ -206,20 +206,6 @@ def int_wasm_nearest :
             [LLVMMatchType<0>],
             [IntrNoMem, IntrSpeculatable]>;
 
-//===----------------------------------------------------------------------===//
-// Bulk memory intrinsics
-//===----------------------------------------------------------------------===//
-
-def int_wasm_memory_init :
-  Intrinsic<[],
-            [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrWriteMem, IntrInaccessibleMemOrArgMemOnly, WriteOnly<ArgIndex<2>>,
-             IntrHasSideEffects, ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>;
-def int_wasm_data_drop :
-  Intrinsic<[],
-            [llvm_i32_ty],
-            [IntrNoDuplicate, IntrHasSideEffects, ImmArg<ArgIndex<0>>]>;
-
 //===----------------------------------------------------------------------===//
 // Thread-local storage intrinsics
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index b3bf187205958..3f86fd075d3a8 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -4948,3 +4948,32 @@ let TargetPrefix = "x86" in {
   def int_x86_xresldtrk : GCCBuiltin<"__builtin_ia32_xresldtrk">,
               Intrinsic<[], [], []>;
 }
+//===----------------------------------------------------------------------===//
+// AMX - Intel AMX extensions
+
+let TargetPrefix = "x86" in {
+  def int_x86_ldtilecfg : GCCBuiltin<"__builtin_ia32_tile_loadconfig">,
+              Intrinsic<[], [llvm_ptr_ty], []>;
+  def int_x86_sttilecfg : GCCBuiltin<"__builtin_ia32_tile_storeconfig">,
+              Intrinsic<[], [llvm_ptr_ty], []>;
+  def int_x86_tilerelease : GCCBuiltin<"__builtin_ia32_tilerelease">,
+              Intrinsic<[], [], []>;
+  def int_x86_tilezero : GCCBuiltin<"__builtin_ia32_tilezero">,
+              Intrinsic<[], [llvm_i8_ty], []>;
+  def int_x86_tileloadd64 : GCCBuiltin<"__builtin_ia32_tileloadd64">,
+              Intrinsic<[], [llvm_i8_ty, llvm_ptr_ty, llvm_i64_ty], []>;
+  def int_x86_tileloaddt164 : GCCBuiltin<"__builtin_ia32_tileloaddt164">,
+              Intrinsic<[], [llvm_i8_ty, llvm_ptr_ty, llvm_i64_ty], []>;
+  def int_x86_tilestored64 : GCCBuiltin<"__builtin_ia32_tilestored64">,
+              Intrinsic<[], [llvm_i8_ty, llvm_ptr_ty, llvm_i64_ty], []>;
+  def int_x86_tdpbssd : GCCBuiltin<"__builtin_ia32_tdpbssd">,
+              Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], []>;
+  def int_x86_tdpbsud : GCCBuiltin<"__builtin_ia32_tdpbsud">,
+              Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], []>;
+  def int_x86_tdpbusd : GCCBuiltin<"__builtin_ia32_tdpbusd">,
+              Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], []>;
+  def int_x86_tdpbuud : GCCBuiltin<"__builtin_ia32_tdpbuud">,
+              Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], []>;
+  def int_x86_tdpbf16ps : GCCBuiltin<"__builtin_ia32_tdpbf16ps">,
+              Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], []>;
+}
diff --git a/llvm/include/llvm/IR/LegacyPassManagers.h b/llvm/include/llvm/IR/LegacyPassManagers.h
index 5044c1f6ed319..6b1ddd4d79f8f 100644
--- a/llvm/include/llvm/IR/LegacyPassManagers.h
+++ b/llvm/include/llvm/IR/LegacyPassManagers.h
@@ -330,7 +330,8 @@ class PMDataManager {
   /// through getAnalysis interface.
   virtual void addLowerLevelRequiredPass(Pass *P, Pass *RequiredPass);
 
-  virtual Pass *getOnTheFlyPass(Pass *P, AnalysisID PI, Function &F);
+  virtual std::tuple<Pass *, bool> getOnTheFlyPass(Pass *P, AnalysisID PI,
+                                                   Function &F);
 
   /// Initialize available analysis information.
   void initializeAnalysisInfo() {
diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h
index 595c7b7d4da0b..4cfd4e916200b 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndex.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h
@@ -553,8 +553,7 @@ class FunctionSummary : public GlobalValueSummary {
     unsigned AlwaysInline : 1;
   };
 
-  /// Describes the uses of a parameter by the range of offsets accessed in the
-  /// function and all of the call targets it is passed to.
+  /// Describes the uses of a parameter by the function.
   struct ParamAccess {
     static constexpr uint32_t RangeWidth = 64;
 
@@ -564,7 +563,7 @@ class FunctionSummary : public GlobalValueSummary {
     struct Call {
       uint64_t ParamNo = 0;
       GlobalValue::GUID Callee = 0;
-      ConstantRange Offsets{RangeWidth, true};
+      ConstantRange Offsets{/*BitWidth=*/RangeWidth, /*isFullSet=*/true};
 
       Call() = default;
       Call(uint64_t ParamNo, GlobalValue::GUID Callee,
@@ -573,7 +572,15 @@ class FunctionSummary : public GlobalValueSummary {
     };
 
     uint64_t ParamNo = 0;
-    ConstantRange Use{RangeWidth, true};
+    /// The range contains byte offsets from the parameter pointer which
+    /// accessed by the function. In the per-module summary, it only includes
+    /// accesses made by the function instructions. In the combined summary, it
+    /// also includes accesses by nested function calls.
+    ConstantRange Use{/*BitWidth=*/RangeWidth, /*isFullSet=*/true};
+    /// In the per-module summary, it summarizes the byte offset applied to each
+    /// pointer parameter before passing to each corresponding callee.
+    /// In the combined summary, it's empty and information is propagated by
+    /// inter-procedural analysis and applied to the Use field.
     std::vector<Call> Calls;
 
     ParamAccess() = default;
@@ -622,7 +629,8 @@ class FunctionSummary : public GlobalValueSummary {
   std::unique_ptr<TypeIdInfo> TIdInfo;
 
   /// Uses for every parameter to this function.
-  std::vector<ParamAccess> ParamAccesses;
+  using ParamAccessesTy = std::vector<ParamAccess>;
+  std::unique_ptr<ParamAccessesTy> ParamAccesses;
 
 public:
   FunctionSummary(GVFlags Flags, unsigned NumInsts, FFlags FunFlags,
@@ -633,19 +641,20 @@ class FunctionSummary : public GlobalValueSummary {
                   std::vector<VFuncId> TypeCheckedLoadVCalls,
                   std::vector<ConstVCall> TypeTestAssumeConstVCalls,
                   std::vector<ConstVCall> TypeCheckedLoadConstVCalls,
-                  std::vector<ParamAccess> ParamAccesses)
+                  std::vector<ParamAccess> Params)
       : GlobalValueSummary(FunctionKind, Flags, std::move(Refs)),
         InstCount(NumInsts), FunFlags(FunFlags), EntryCount(EntryCount),
-        CallGraphEdgeList(std::move(CGEdges)),
-        ParamAccesses(std::move(ParamAccesses)) {
+        CallGraphEdgeList(std::move(CGEdges)) {
     if (!TypeTests.empty() || !TypeTestAssumeVCalls.empty() ||
         !TypeCheckedLoadVCalls.empty() || !TypeTestAssumeConstVCalls.empty() ||
         !TypeCheckedLoadConstVCalls.empty())
-      TIdInfo = std::make_unique<TypeIdInfo>(TypeIdInfo{
-          std::move(TypeTests), std::move(TypeTestAssumeVCalls),
-          std::move(TypeCheckedLoadVCalls),
-          std::move(TypeTestAssumeConstVCalls),
-          std::move(TypeCheckedLoadConstVCalls)});
+      TIdInfo = std::make_unique<TypeIdInfo>(
+          TypeIdInfo{std::move(TypeTests), std::move(TypeTestAssumeVCalls),
+                     std::move(TypeCheckedLoadVCalls),
+                     std::move(TypeTestAssumeConstVCalls),
+                     std::move(TypeCheckedLoadConstVCalls)});
+    if (!Params.empty())
+      ParamAccesses = std::make_unique<ParamAccessesTy>(std::move(Params));
   }
   // Gets the number of readonly and writeonly refs in RefEdgeList
   std::pair<unsigned, unsigned> specialRefCounts() const;
@@ -717,11 +726,20 @@ class FunctionSummary : public GlobalValueSummary {
   }
 
   /// Returns the list of known uses of pointer parameters.
-  ArrayRef<ParamAccess> paramAccesses() const { return ParamAccesses; }
+  ArrayRef<ParamAccess> paramAccesses() const {
+    if (ParamAccesses)
+      return *ParamAccesses;
+    return {};
+  }
 
   /// Sets the list of known uses of pointer parameters.
   void setParamAccesses(std::vector<ParamAccess> NewParams) {
-    ParamAccesses = std::move(NewParams);
+    if (NewParams.empty())
+      ParamAccesses.reset();
+    else if (ParamAccesses)
+      *ParamAccesses = std::move(NewParams);
+    else
+      ParamAccesses = std::make_unique<ParamAccessesTy>(std::move(NewParams));
   }
 
   /// Add a type test to the summary. This is used by WholeProgramDevirt if we
diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h
index e7fb0b5e75433..4d5f292ba9a13 100644
--- a/llvm/include/llvm/IR/PassManager.h
+++ b/llvm/include/llvm/IR/PassManager.h
@@ -503,9 +503,6 @@ class PassManager : public PassInfoMixin<
 
     for (unsigned Idx = 0, Size = Passes.size(); Idx != Size; ++Idx) {
       auto *P = Passes[Idx].get();
-      if (DebugLogging)
-        dbgs() << "Running pass: " << P->name() << " on " << IR.getName()
-               << "\n";
 
       // Check the PassInstrumentation's BeforePass callbacks before running the
       // pass, skip its execution completely if asked to (callback returns
@@ -513,6 +510,10 @@ class PassManager : public PassInfoMixin<
       if (!PI.runBeforePass<IRUnitT>(*P, IR))
         continue;
 
+      if (DebugLogging)
+        dbgs() << "Running pass: " << P->name() << " on " << IR.getName()
+               << "\n";
+
       PreservedAnalyses PassPA;
       {
         TimeTraceScope TimeScope(P->name(), IR.getName());
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 3b8bc9f8b7955..06e8507036acf 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -72,6 +72,7 @@ void initializeAliasSetPrinterPass(PassRegistry&);
 void initializeAlignmentFromAssumptionsPass(PassRegistry&);
 void initializeAlwaysInlinerLegacyPassPass(PassRegistry&);
 void initializeAssumeSimplifyPassLegacyPassPass(PassRegistry &);
+void initializeAssumeBuilderPassLegacyPassPass(PassRegistry &);
 void initializeOpenMPOptLegacyPassPass(PassRegistry &);
 void initializeArgPromotionPass(PassRegistry&);
 void initializeAssumptionCacheTrackerPass(PassRegistry&);
@@ -102,6 +103,7 @@ void initializeCFGViewerLegacyPassPass(PassRegistry&);
 void initializeCFIInstrInserterPass(PassRegistry&);
 void initializeCFLAndersAAWrapperPassPass(PassRegistry&);
 void initializeCFLSteensAAWrapperPassPass(PassRegistry&);
+void initializeCGProfileLegacyPassPass(PassRegistry &);
 void initializeCallGraphDOTPrinterPass(PassRegistry&);
 void initializeCallGraphPrinterLegacyPassPass(PassRegistry&);
 void initializeCallGraphViewerPass(PassRegistry&);
diff --git a/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h b/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
index 5bcfaeedfcf6e..d7ccc0d5a6c53 100644
--- a/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
+++ b/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
@@ -123,7 +123,7 @@ struct LTOCodeGenerator {
   /// name is misleading).  This function should be called before
   /// LTOCodeGenerator::compilexxx(), and
   /// LTOCodeGenerator::writeMergedModules().
-  void setCodeGenDebugOptions(ArrayRef<const char *> Opts);
+  void setCodeGenDebugOptions(ArrayRef<StringRef> Opts);
 
   /// Parse the options set in setCodeGenDebugOptions.
   ///
diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h
index 974155ff3319d..46c5a111c891e 100644
--- a/llvm/include/llvm/MC/MCAsmInfo.h
+++ b/llvm/include/llvm/MC/MCAsmInfo.h
@@ -343,10 +343,6 @@ class MCAsmInfo {
   /// protected visibility.  Defaults to MCSA_Protected
   MCSymbolAttr ProtectedVisibilityAttr = MCSA_Protected;
 
-  // This attribute is used to indicate symbols such as commons on AIX may have
-  // a storage mapping class embedded in the name.
-  bool SymbolsHaveSMC = false;
-
   //===--- Dwarf Emission Directives -----------------------------------===//
 
   /// True if target supports emission of debugging information.  Defaults to
@@ -606,8 +602,6 @@ class MCAsmInfo {
     return ProtectedVisibilityAttr;
   }
 
-  bool getSymbolsHaveSMC() const { return SymbolsHaveSMC; }
-
   bool doesSupportDebugInformation() const { return SupportsDebugInformation; }
 
   bool doesSupportExceptionHandling() const {
diff --git a/llvm/include/llvm/MC/MCAsmLayout.h b/llvm/include/llvm/MC/MCAsmLayout.h
index 45ac96f0b81ec..f95af210a28f3 100644
--- a/llvm/include/llvm/MC/MCAsmLayout.h
+++ b/llvm/include/llvm/MC/MCAsmLayout.h
@@ -49,6 +49,10 @@ class MCAsmLayout {
   /// Get the assembler object this is a layout for.
   MCAssembler &getAssembler() const { return Assembler; }
 
+  /// \returns whether the offset of fragment \p F can be obtained via
+  /// getFragmentOffset.
+  bool canGetFragmentOffset(const MCFragment *F) const;
+
   /// Invalidate the fragments starting with F because it has been
   /// resized. The fragment's size should have already been updated, but
   /// its bundle padding will be recomputed.
diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h
index 3f106c697b01c..45be9bb3d225b 100644
--- a/llvm/include/llvm/MC/MCContext.h
+++ b/llvm/include/llvm/MC/MCContext.h
@@ -57,6 +57,7 @@ namespace llvm {
   class MCSymbol;
   class MCSymbolELF;
   class MCSymbolWasm;
+  class MCSymbolXCOFF;
   class SMLoc;
   class SourceMgr;
 
@@ -308,6 +309,9 @@ namespace llvm {
                                        unsigned UniqueID,
                                        const MCSymbolELF *LinkedToSym);
 
+    MCSymbolXCOFF *createXCOFFSymbolImpl(const StringMapEntry<bool> *Name,
+                                         bool IsTemporary);
+
     /// Map of currently defined macros.
     StringMap<MCAsmMacro> MacroMap;
 
diff --git a/llvm/include/llvm/MC/MCExpr.h b/llvm/include/llvm/MC/MCExpr.h
index 776e116a6e16f..803c0d443beec 100644
--- a/llvm/include/llvm/MC/MCExpr.h
+++ b/llvm/include/llvm/MC/MCExpr.h
@@ -327,6 +327,21 @@ class MCSymbolRefExpr : public MCExpr {
     VK_AMDGPU_ABS32_LO,      // symbol@abs32@lo
     VK_AMDGPU_ABS32_HI,      // symbol@abs32@hi
 
+    VK_VE_HI32,        // symbol@hi
+    VK_VE_LO32,        // symbol@lo
+    VK_VE_PC_HI32,     // symbol@pc_hi
+    VK_VE_PC_LO32,     // symbol@pc_lo
+    VK_VE_GOT_HI32,    // symbol@got_hi
+    VK_VE_GOT_LO32,    // symbol@got_lo
+    VK_VE_GOTOFF_HI32, // symbol@gotoff_hi
+    VK_VE_GOTOFF_LO32, // symbol@gotoff_lo
+    VK_VE_PLT_HI32,    // symbol@plt_hi
+    VK_VE_PLT_LO32,    // symbol@plt_lo
+    VK_VE_TLS_GD_HI32, // symbol@tls_gd_hi
+    VK_VE_TLS_GD_LO32, // symbol@tls_gd_lo
+    VK_VE_TPOFF_HI32,  // symbol@tpoff_hi
+    VK_VE_TPOFF_LO32,  // symbol@tpoff_lo
+
     VK_TPREL,
     VK_DTPREL
   };
diff --git a/llvm/include/llvm/MC/MCFragment.h b/llvm/include/llvm/MC/MCFragment.h
index 4c8a895592ef2..fb7166e82c098 100644
--- a/llvm/include/llvm/MC/MCFragment.h
+++ b/llvm/include/llvm/MC/MCFragment.h
@@ -65,6 +65,9 @@ class MCFragment : public ilist_node_with_parent<MCFragment, MCSection> {
 
   FragmentType Kind;
 
+  /// Whether fragment is being laid out.
+  bool IsBeingLaidOut;
+
 protected:
   bool HasInstructions;
 
diff --git a/llvm/include/llvm/MC/MCParser/MCAsmParser.h b/llvm/include/llvm/MC/MCParser/MCAsmParser.h
index 7d57dd605fcb4..2040089759594 100644
--- a/llvm/include/llvm/MC/MCParser/MCAsmParser.h
+++ b/llvm/include/llvm/MC/MCParser/MCAsmParser.h
@@ -168,6 +168,13 @@ class MCAsmParser {
   virtual void setParsingMSInlineAsm(bool V) = 0;
   virtual bool isParsingMSInlineAsm() = 0;
 
+  virtual bool isParsingMasm() const { return false; }
+
+  virtual bool LookUpFieldOffset(StringRef Base, StringRef Member,
+                                 unsigned &Offset) {
+    return true;
+  }
+
   /// Parse MS-style inline assembly.
   virtual bool parseMSInlineAsm(
       void *AsmLoc, std::string &AsmString, unsigned &NumOutputs,
diff --git a/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h b/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h
index 5d2afe81a54ba..c37889cfc5096 100644
--- a/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h
+++ b/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h
@@ -98,6 +98,8 @@ class MCAsmParserExtension {
     return getParser().parseOptionalToken(T);
   }
 
+  bool ParseDirectiveCGProfile(StringRef, SMLoc);
+
   bool check(bool P, const Twine &Msg) {
     return getParser().check(P, Msg);
   }
diff --git a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h
index ad086eaa539cd..1d10c66b4201f 100644
--- a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h
+++ b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h
@@ -334,7 +334,7 @@ class MCTargetAsmParser : public MCAsmParserExtension {
 
   /// SemaCallback - The Sema callback implementation.  Must be set when parsing
   /// ms-style inline assembly.
-  MCAsmParserSemaCallback *SemaCallback;
+  MCAsmParserSemaCallback *SemaCallback = nullptr;
 
   /// Set of options which affects instrumentation of inline assembly.
   MCTargetOptions MCOptions;
diff --git a/llvm/include/llvm/MC/MCRegister.h b/llvm/include/llvm/MC/MCRegister.h
index 79a8dcdadb417..1f3c4b8494cc1 100644
--- a/llvm/include/llvm/MC/MCRegister.h
+++ b/llvm/include/llvm/MC/MCRegister.h
@@ -35,6 +35,12 @@ class MCRegister {
   //
   // Further sentinels can be allocated from the small negative integers.
   // DenseMapInfo<unsigned> uses -1u and -2u.
+  static_assert(std::numeric_limits<decltype(Reg)>::max() >= 0xFFFFFFFF,
+                "Reg isn't large enough to hold full range.");
+  static constexpr unsigned NoRegister = 0u;
+  static constexpr unsigned FirstPhysicalReg = 1u;
+  static constexpr unsigned FirstStackSlot = 1u << 30;
+  static constexpr unsigned VirtualRegFlag = 1u << 31;
 
   /// This is the portion of the positive number space that is not a physical
   /// register. StackSlot values do not exist in the MC layer, see
@@ -44,14 +50,15 @@ class MCRegister {
   /// slots, so if a variable may contains a stack slot, always check
   /// isStackSlot() first.
   static bool isStackSlot(unsigned Reg) {
-    return int(Reg) >= (1 << 30);
+    return !(Reg & VirtualRegFlag) &&
+           uint32_t(Reg & ~VirtualRegFlag) >= FirstStackSlot;
   }
 
   /// Return true if the specified register number is in
   /// the physical register namespace.
   static bool isPhysicalRegister(unsigned Reg) {
     assert(!isStackSlot(Reg) && "Not a register! Check isStackSlot() first.");
-    return int(Reg) > 0;
+    return Reg >= FirstPhysicalReg && !(Reg & VirtualRegFlag);
   }
 
   /// Return true if the specified register number is in the physical register
@@ -68,9 +75,7 @@ class MCRegister {
     return Reg;
   }
 
-  bool isValid() const {
-    return Reg != 0;
-  }
+  bool isValid() const { return Reg != NoRegister; }
 
   /// Comparisons between register objects
   bool operator==(const MCRegister &Other) const { return Reg == Other.Reg; }
diff --git a/llvm/include/llvm/MC/MCSectionWasm.h b/llvm/include/llvm/MC/MCSectionWasm.h
index 2cf64deaca399..6211afef71dbd 100644
--- a/llvm/include/llvm/MC/MCSectionWasm.h
+++ b/llvm/include/llvm/MC/MCSectionWasm.h
@@ -13,15 +13,14 @@
 #ifndef LLVM_MC_MCSECTIONWASM_H
 #define LLVM_MC_MCSECTIONWASM_H
 
-#include "llvm/ADT/Twine.h"
 #include "llvm/MC/MCSection.h"
-#include "llvm/MC/MCSymbolWasm.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
 
 class MCSymbol;
+class MCSymbolWasm;
+class StringRef;
+class raw_ostream;
 
 /// This represents a section on wasm.
 class MCSectionWasm final : public MCSection {
diff --git a/llvm/include/llvm/MC/MCSectionXCOFF.h b/llvm/include/llvm/MC/MCSectionXCOFF.h
index f6e7b0933d113..eed6b9c2609c2 100644
--- a/llvm/include/llvm/MC/MCSectionXCOFF.h
+++ b/llvm/include/llvm/MC/MCSectionXCOFF.h
@@ -36,13 +36,15 @@ class MCSectionXCOFF final : public MCSection {
   XCOFF::SymbolType Type;
   XCOFF::StorageClass StorageClass;
   MCSymbolXCOFF *const QualName;
+  StringRef SymbolTableName;
   static constexpr unsigned DefaultAlignVal = 4;
 
   MCSectionXCOFF(StringRef Name, XCOFF::StorageMappingClass SMC,
                  XCOFF::SymbolType ST, XCOFF::StorageClass SC, SectionKind K,
-                 MCSymbolXCOFF *QualName, MCSymbol *Begin)
+                 MCSymbolXCOFF *QualName, MCSymbol *Begin,
+                 StringRef SymbolTableName)
       : MCSection(SV_XCOFF, Name, K, Begin), MappingClass(SMC), Type(ST),
-        StorageClass(SC), QualName(QualName) {
+        StorageClass(SC), QualName(QualName), SymbolTableName(SymbolTableName) {
     assert((ST == XCOFF::XTY_SD || ST == XCOFF::XTY_CM || ST == XCOFF::XTY_ER) &&
            "Invalid or unhandled type for csect.");
     assert(QualName != nullptr && "QualName is needed.");
@@ -72,6 +74,7 @@ class MCSectionXCOFF final : public MCSection {
                             const MCExpr *Subsection) const override;
   bool UseCodeAlign() const override;
   bool isVirtualSection() const override;
+  StringRef getSymbolTableName() const { return SymbolTableName; }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index f77ecb5af50ce..d7255a22e9415 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -574,6 +574,16 @@ class MCStreamer {
   virtual void emitXCOFFSymbolLinkageWithVisibility(MCSymbol *Symbol,
                                                     MCSymbolAttr Linkage,
                                                     MCSymbolAttr Visibility);
+
+  /// Emit a XCOFF .rename directive which creates a synonym for an illegal or
+  /// undesirable name.
+  ///
+  /// \param Name - The name used internally in the assembly for references to
+  /// the symbol.
+  /// \param Rename - The value to which the Name parameter is
+  /// changed at the end of assembly.
+  virtual void emitXCOFFRenameDirective(const MCSymbol *Name, StringRef Rename);
+
   /// Emit an ELF .size directive.
   ///
   /// This corresponds to an assembler statement such as:
diff --git a/llvm/include/llvm/MC/MCSymbolXCOFF.h b/llvm/include/llvm/MC/MCSymbolXCOFF.h
index e5bea10fd5d5c..d0379ec08b7d4 100644
--- a/llvm/include/llvm/MC/MCSymbolXCOFF.h
+++ b/llvm/include/llvm/MC/MCSymbolXCOFF.h
@@ -24,6 +24,16 @@ class MCSymbolXCOFF : public MCSymbol {
 
   static bool classof(const MCSymbol *S) { return S->isXCOFF(); }
 
+  static StringRef getUnqualifiedName(StringRef Name) {
+    if (Name.back() == ']') {
+      StringRef Lhs, Rhs;
+      std::tie(Lhs, Rhs) = Name.rsplit('[');
+      assert(!Rhs.empty() && "Invalid SMC format in XCOFF symbol.");
+      return Lhs;
+    }
+    return Name;
+  }
+
   void setStorageClass(XCOFF::StorageClass SC) {
     assert((!StorageClass.hasValue() || StorageClass.getValue() == SC) &&
            "Redefining StorageClass of XCOFF MCSymbol.");
@@ -36,16 +46,7 @@ class MCSymbolXCOFF : public MCSymbol {
     return StorageClass.getValue();
   }
 
-  StringRef getUnqualifiedName() const {
-    const StringRef name = getName();
-    if (name.back() == ']') {
-      StringRef lhs, rhs;
-      std::tie(lhs, rhs) = name.rsplit('[');
-      assert(!rhs.empty() && "Invalid SMC format in XCOFF symbol.");
-      return lhs;
-    }
-    return name;
-  }
+  StringRef getUnqualifiedName() const { return getUnqualifiedName(getName()); }
 
   bool hasRepresentedCsectSet() const { return RepresentedCsect != nullptr; }
 
@@ -57,10 +58,21 @@ class MCSymbolXCOFF : public MCSymbol {
 
   XCOFF::VisibilityType getVisibilityType() const { return VisibilityType; }
 
+  bool hasRename() const { return !SymbolTableName.empty(); }
+
+  void setSymbolTableName(StringRef STN) { SymbolTableName = STN; }
+
+  StringRef getSymbolTableName() const {
+    if (hasRename())
+      return SymbolTableName;
+    return getUnqualifiedName();
+  }
+
 private:
   Optional<XCOFF::StorageClass> StorageClass;
   MCSectionXCOFF *RepresentedCsect = nullptr;
   XCOFF::VisibilityType VisibilityType = XCOFF::SYM_V_UNSPECIFIED;
+  StringRef SymbolTableName;
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCWinCOFFStreamer.h b/llvm/include/llvm/MC/MCWinCOFFStreamer.h
index b5f570ec335c5..1236304b9e5da 100644
--- a/llvm/include/llvm/MC/MCWinCOFFStreamer.h
+++ b/llvm/include/llvm/MC/MCWinCOFFStreamer.h
@@ -64,6 +64,8 @@ class MCWinCOFFStreamer : public MCObjectStreamer {
                       unsigned ByteAlignment) override;
   void emitIdent(StringRef IdentString) override;
   void EmitWinEHHandlerData(SMLoc Loc) override;
+  void emitCGProfileEntry(const MCSymbolRefExpr *From,
+                          const MCSymbolRefExpr *To, uint64_t Count) override;
   void finishImpl() override;
 
   /// \}
@@ -73,6 +75,9 @@ class MCWinCOFFStreamer : public MCObjectStreamer {
 
   void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &STI) override;
 
+  void finalizeCGProfileEntry(const MCSymbolRefExpr *&S);
+  void finalizeCGProfile();
+
 private:
   void Error(const Twine &Msg) const;
 };
diff --git a/llvm/include/llvm/MC/MCXCOFFStreamer.h b/llvm/include/llvm/MC/MCXCOFFStreamer.h
index 416a55f2c8fc6..5fc2efbe52849 100644
--- a/llvm/include/llvm/MC/MCXCOFFStreamer.h
+++ b/llvm/include/llvm/MC/MCXCOFFStreamer.h
@@ -32,6 +32,11 @@ class MCXCOFFStreamer : public MCObjectStreamer {
   void emitXCOFFSymbolLinkageWithVisibility(MCSymbol *Symbol,
                                             MCSymbolAttr Linkage,
                                             MCSymbolAttr Visibility) override;
+  void emitXCOFFRenameDirective(const MCSymbol *Name,
+                                StringRef Rename) override {
+    report_fatal_error("emitXCOFFRenameDirective is not implemented yet on "
+                       "object generation path");
+  }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/Object/ELF.h b/llvm/include/llvm/Object/ELF.h
index 15d473805ecce..c7f2a8e709f0b 100644
--- a/llvm/include/llvm/Object/ELF.h
+++ b/llvm/include/llvm/Object/ELF.h
@@ -516,8 +516,17 @@ Expected<StringRef>
 ELFFile<ELFT>::getSectionStringTable(Elf_Shdr_Range Sections,
                                      WarningHandler WarnHandler) const {
   uint32_t Index = getHeader()->e_shstrndx;
-  if (Index == ELF::SHN_XINDEX)
+  if (Index == ELF::SHN_XINDEX) {
+    // If the section name string table section index is greater than
+    // or equal to SHN_LORESERVE, then the actual index of the section name
+    // string table section is contained in the sh_link field of the section
+    // header at index 0.
+    if (Sections.empty())
+      return createError(
+          "e_shstrndx == SHN_XINDEX, but the section header table is empty");
+
     Index = Sections[0].sh_link;
+  }
 
   if (!Index) // no section string table.
     return "";
diff --git a/llvm/include/llvm/Object/Error.h b/llvm/include/llvm/Object/Error.h
index b7bbf06fc86de..1e109fa131c8a 100644
--- a/llvm/include/llvm/Object/Error.h
+++ b/llvm/include/llvm/Object/Error.h
@@ -13,11 +13,13 @@
 #ifndef LLVM_OBJECT_ERROR_H
 #define LLVM_OBJECT_ERROR_H
 
-#include "llvm/ADT/Twine.h"
 #include "llvm/Support/Error.h"
 #include <system_error>
 
 namespace llvm {
+
+class Twine;
+
 namespace object {
 
 class Binary;
@@ -65,8 +67,8 @@ class BinaryError : public ErrorInfo<BinaryError, ECError> {
 class GenericBinaryError : public ErrorInfo<GenericBinaryError, BinaryError> {
 public:
   static char ID;
-  GenericBinaryError(Twine Msg);
-  GenericBinaryError(Twine Msg, object_error ECOverride);
+  GenericBinaryError(const Twine &Msg);
+  GenericBinaryError(const Twine &Msg, object_error ECOverride);
   const std::string &getMessage() const { return Msg; }
   void log(raw_ostream &OS) const override;
 private:
diff --git a/llvm/include/llvm/ObjectYAML/DWARFEmitter.h b/llvm/include/llvm/ObjectYAML/DWARFEmitter.h
index 716386992c29c..0ec3f90e1686a 100644
--- a/llvm/include/llvm/ObjectYAML/DWARFEmitter.h
+++ b/llvm/include/llvm/ObjectYAML/DWARFEmitter.h
@@ -34,7 +34,7 @@ Error emitDebugStr(raw_ostream &OS, const Data &DI);
 Error emitDebugAranges(raw_ostream &OS, const Data &DI);
 Error emitDebugRanges(raw_ostream &OS, const Data &DI);
 Error emitPubSection(raw_ostream &OS, const PubSection &Sect,
-                     bool IsLittleEndian);
+                     bool IsLittleEndian, bool IsGNUPubSec = false);
 Error emitDebugInfo(raw_ostream &OS, const Data &DI);
 Error emitDebugLine(raw_ostream &OS, const Data &DI);
 Error emitDebugAddr(raw_ostream &OS, const Data &DI);
diff --git a/llvm/include/llvm/ObjectYAML/DWARFYAML.h b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
index f6dc4e07dc1ab..259152ff5d035 100644
--- a/llvm/include/llvm/ObjectYAML/DWARFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
@@ -98,11 +98,7 @@ struct PubSection {
   uint16_t Version;
   uint32_t UnitOffset;
   uint32_t UnitSize;
-  bool IsGNUStyle = false;
   std::vector<PubEntry> Entries;
-
-  PubSection() = default;
-  PubSection(bool IsGNUStyle) : IsGNUStyle(IsGNUStyle) {}
 };
 
 struct FormValue {
@@ -116,11 +112,18 @@ struct Entry {
   std::vector<FormValue> Values;
 };
 
+/// Class that contains helpful context information when mapping YAML into DWARF
+/// data structures.
+struct DWARFContext {
+  bool IsGNUPubSec = false;
+};
+
 struct Unit {
-  InitialLength Length;
+  dwarf::DwarfFormat Format;
+  uint64_t Length;
   uint16_t Version;
   llvm::dwarf::UnitType Type; // Added in DWARF 5
-  uint32_t AbbrOffset;
+  yaml::Hex64 AbbrOffset;
   uint8_t AddrSize;
   std::vector<Entry> Entries;
 };
diff --git a/llvm/include/llvm/ObjectYAML/ELFYAML.h b/llvm/include/llvm/ObjectYAML/ELFYAML.h
index 8c15fccf6b072..e7a2411a3e2aa 100644
--- a/llvm/include/llvm/ObjectYAML/ELFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/ELFYAML.h
@@ -252,6 +252,9 @@ struct RawContentSection : Section {
   static bool classof(const Chunk *S) {
     return S->Kind == ChunkKind::RawContent;
   }
+
+  // Is used when a content is read as an array of bytes.
+  Optional<std::vector<uint8_t>> ContentBuf;
 };
 
 struct NoBitsSection : Section {
diff --git a/llvm/include/llvm/Option/OptParser.td b/llvm/include/llvm/Option/OptParser.td
index a68f17a8b10b3..e32355444d7bc 100644
--- a/llvm/include/llvm/Option/OptParser.td
+++ b/llvm/include/llvm/Option/OptParser.td
@@ -97,6 +97,18 @@ class Option<list<string> prefixes, string name, OptionKind kind> {
   OptionGroup Group = ?;
   Option Alias = ?;
   list<string> AliasArgs = [];
+  string MarshallingKind = ?;
+  code KeyPath = ?;
+  code DefaultValue = ?;
+  bit ShouldAlwaysEmit = 0;
+  // Used by the Flag option kind.
+  bit IsPositive = 1;
+  // Used by the String option kind.
+  code NormalizerRetTy = ?;
+  code NormalizedValuesScope = "";
+  code Normalizer = "";
+  code Denormalizer = "";
+  list<code> NormalizedValues = ?;
 }
 
 // Helpers for defining options.
@@ -130,6 +142,37 @@ class MetaVarName<string name> { string MetaVarName = name; }
 class Values<string value> { string Values = value; }
 class ValuesCode<code valuecode> { code ValuesCode = valuecode; }
 
+// Helpers for defining marshalling information.
+
+class MarshallingInfo<code keypath, code defaultvalue> {
+  code KeyPath = keypath;
+  code DefaultValue = defaultvalue;
+}
+class MarshallingInfoString<code keypath, code defaultvalue, code normalizerretty>
+  : MarshallingInfo<keypath, defaultvalue> {
+  string MarshallingKind = "string";
+  code NormalizerRetTy = normalizerretty;
+}
+
+class MarshallingInfoFlag<code keypath, code defaultvalue>
+  : MarshallingInfo<keypath, defaultvalue> {
+  string MarshallingKind = "flag";
+}
+
+// Mixins for additional marshalling attributes.
+
+class IsNegative { bit IsPositive = 0; }
+class AlwaysEmit { bit ShouldAlwaysEmit = 1; }
+class Normalizer<code normalizer> { code Normalizer = normalizer; }
+class Denormalizer<code denormalizer> { code Denormalizer = denormalizer; }
+class NormalizedValuesScope<code scope> { code NormalizedValuesScope = scope; }
+class NormalizedValues<list<code> definitions> { list<code> NormalizedValues = definitions; } 
+class DenormalizeString { code Denormalizer = "denormalizeString"; }
+class AutoNormalizeEnum {
+  code Normalizer = "normalizeSimpleEnum";
+  code Denormalizer = "denormalizeSimpleEnum";
+}
+
 // Predefined options.
 
 // FIXME: Have generator validate that these appear in correct position (and
diff --git a/llvm/include/llvm/Pass.h b/llvm/include/llvm/Pass.h
index b0af15a7dab22..2fe7aee2e37ee 100644
--- a/llvm/include/llvm/Pass.h
+++ b/llvm/include/llvm/Pass.h
@@ -203,14 +203,17 @@ class Pass {
   template<typename AnalysisType>
   AnalysisType &getAnalysis() const; // Defined in PassAnalysisSupport.h
 
-  template<typename AnalysisType>
-  AnalysisType &getAnalysis(Function &F); // Defined in PassAnalysisSupport.h
+  template <typename AnalysisType>
+  AnalysisType &
+  getAnalysis(Function &F,
+              bool *Changed = nullptr); // Defined in PassAnalysisSupport.h
 
   template<typename AnalysisType>
   AnalysisType &getAnalysisID(AnalysisID PI) const;
 
-  template<typename AnalysisType>
-  AnalysisType &getAnalysisID(AnalysisID PI, Function &F);
+  template <typename AnalysisType>
+  AnalysisType &getAnalysisID(AnalysisID PI, Function &F,
+                              bool *Changed = nullptr);
 };
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/PassAnalysisSupport.h b/llvm/include/llvm/PassAnalysisSupport.h
index 3a6b1d29f3fa3..84df171d38d85 100644
--- a/llvm/include/llvm/PassAnalysisSupport.h
+++ b/llvm/include/llvm/PassAnalysisSupport.h
@@ -24,6 +24,7 @@
 
 #include "llvm/ADT/SmallVector.h"
 #include <cassert>
+#include <tuple>
 #include <utility>
 #include <vector>
 
@@ -167,7 +168,7 @@ class AnalysisResolver {
   }
 
   /// Find pass that is implementing PI. Initialize pass for Function F.
-  Pass *findImplPass(Pass *P, AnalysisID PI, Function &F);
+  std::tuple<Pass *, bool> findImplPass(Pass *P, AnalysisID PI, Function &F);
 
   void addAnalysisImplsPair(AnalysisID PI, Pass *P) {
     if (findImplPass(PI) == P)
@@ -246,23 +247,33 @@ AnalysisType &Pass::getAnalysisID(AnalysisID PI) const {
 
 /// getAnalysis<AnalysisType>() - This function is used by subclasses to get
 /// to the analysis information that they claim to use by overriding the
-/// getAnalysisUsage function.
-template<typename AnalysisType>
-AnalysisType &Pass::getAnalysis(Function &F) {
+/// getAnalysisUsage function. If as part of the dependencies, an IR
+/// transformation is triggered (e.g. because the analysis requires
+/// BreakCriticalEdges), and Changed is non null, *Changed is updated.
+template <typename AnalysisType>
+AnalysisType &Pass::getAnalysis(Function &F, bool *Changed) {
   assert(Resolver &&"Pass has not been inserted into a PassManager object!");
 
-  return getAnalysisID<AnalysisType>(&AnalysisType::ID, F);
+  return getAnalysisID<AnalysisType>(&AnalysisType::ID, F, Changed);
 }
 
-template<typename AnalysisType>
-AnalysisType &Pass::getAnalysisID(AnalysisID PI, Function &F) {
+template <typename AnalysisType>
+AnalysisType &Pass::getAnalysisID(AnalysisID PI, Function &F, bool *Changed) {
   assert(PI && "getAnalysis for unregistered pass!");
   assert(Resolver && "Pass has not been inserted into a PassManager object!");
   // PI *must* appear in AnalysisImpls.  Because the number of passes used
   // should be a small number, we just do a linear search over a (dense)
   // vector.
-  Pass *ResultPass = Resolver->findImplPass(this, PI, F);
+  Pass *ResultPass;
+  bool LocalChanged;
+  std::tie(ResultPass, LocalChanged) = Resolver->findImplPass(this, PI, F);
+
   assert(ResultPass && "Unable to find requested analysis info");
+  if (Changed)
+    *Changed |= LocalChanged;
+  else
+    assert(!LocalChanged &&
+           "A pass trigged a code update but the update status is lost");
 
   // Because the AnalysisType may not be a subclass of pass (for
   // AnalysisGroups), we use getAdjustedAnalysisPointer here to potentially
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index 295a5aacfe7ee..f9b0d939e5f8c 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -515,6 +515,9 @@ class PassBuilder {
   /// returns false.
   Error parseAAPipeline(AAManager &AA, StringRef PipelineText);
 
+  /// Returns true if the pass name is the name of an alias analysis pass.
+  bool isAAPassName(StringRef PassName);
+
   /// Register a callback for a default optimizer pipeline extension
   /// point
   ///
diff --git a/llvm/include/llvm/ProfileData/ProfileCommon.h b/llvm/include/llvm/ProfileData/ProfileCommon.h
index f98a34387fdf4..14c305b3d0c05 100644
--- a/llvm/include/llvm/ProfileData/ProfileCommon.h
+++ b/llvm/include/llvm/ProfileData/ProfileCommon.h
@@ -62,6 +62,10 @@ class ProfileSummaryBuilder {
 public:
   /// A vector of useful cutoff values for detailed summary.
   static const ArrayRef<uint32_t> DefaultCutoffs;
+
+  /// Find the summary entry for a desired percentile of counts.
+  static const ProfileSummaryEntry &
+  getEntryForPercentile(SummaryEntryVector &DS, uint64_t Percentile);
 };
 
 class InstrProfSummaryBuilder final : public ProfileSummaryBuilder {
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index af636a95bbf84..562468333ef47 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -498,6 +498,18 @@ class FunctionSamples {
     return CallsiteSamples;
   }
 
+  /// Return the maximum of sample counts in a function body including functions
+  /// inlined in it.
+  uint64_t getMaxCountInside() const {
+    uint64_t MaxCount = 0;
+    for (const auto &L : getBodySamples())
+      MaxCount = std::max(MaxCount, L.second.getSamples());
+    for (const auto &C : getCallsiteSamples())
+      for (const FunctionSamplesMap::value_type &F : C.second)
+        MaxCount = std::max(MaxCount, F.second.getMaxCountInside());
+    return MaxCount;
+  }
+
   /// Merge the samples in \p Other into this one.
   /// Optionally scale samples by \p Weight.
   sampleprof_error merge(const FunctionSamples &Other, uint64_t Weight = 1) {
diff --git a/llvm/include/llvm/Support/AArch64TargetParser.def b/llvm/include/llvm/Support/AArch64TargetParser.def
index 21f1d36bdb981..13b7cfc4b5cd9 100644
--- a/llvm/include/llvm/Support/AArch64TargetParser.def
+++ b/llvm/include/llvm/Support/AArch64TargetParser.def
@@ -124,6 +124,15 @@ AARCH64_CPU_NAME("cortex-a76", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
 AARCH64_CPU_NAME("cortex-a76ae", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC |
                   AArch64::AEK_SSBS))
+AARCH64_CPU_NAME("cortex-a77", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
+                 (AArch64::AEK_FP16 | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD |
+                  AArch64::AEK_SSBS))
+AARCH64_CPU_NAME("cortex-a78", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
+                 (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC |
+                  AArch64::AEK_SSBS))
+AARCH64_CPU_NAME("cortex-x1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
+                 (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC |
+                  AArch64::AEK_SSBS))
 AARCH64_CPU_NAME("neoverse-e1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | AArch64::AEK_RAS |
                   AArch64::AEK_RCPC | AArch64::AEK_SSBS))
diff --git a/llvm/include/llvm/Support/AArch64TargetParser.h b/llvm/include/llvm/Support/AArch64TargetParser.h
index a1d9543fcc582..b045e31bc92aa 100644
--- a/llvm/include/llvm/Support/AArch64TargetParser.h
+++ b/llvm/include/llvm/Support/AArch64TargetParser.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_SUPPORT_AARCH64TARGETPARSERCOMMON_H
 #define LLVM_SUPPORT_AARCH64TARGETPARSERCOMMON_H
 
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ARMTargetParser.h"
 #include <vector>
diff --git a/llvm/include/llvm/Support/AMDGPUMetadata.h b/llvm/include/llvm/Support/AMDGPUMetadata.h
index eeef4e699c3e0..920c97f7e112c 100644
--- a/llvm/include/llvm/Support/AMDGPUMetadata.h
+++ b/llvm/include/llvm/Support/AMDGPUMetadata.h
@@ -79,7 +79,8 @@ enum class ValueKind : uint8_t {
   Unknown                = 0xff
 };
 
-/// Value types.
+/// Value types. This is deprecated and only remains for compatibility parsing
+/// of old metadata.
 enum class ValueType : uint8_t {
   Struct  = 0,
   I8      = 1,
@@ -164,7 +165,7 @@ constexpr char Offset[] = "Offset";
 constexpr char Align[] = "Align";
 /// Key for Kernel::Arg::Metadata::mValueKind.
 constexpr char ValueKind[] = "ValueKind";
-/// Key for Kernel::Arg::Metadata::mValueType.
+/// Key for Kernel::Arg::Metadata::mValueType. (deprecated)
 constexpr char ValueType[] = "ValueType";
 /// Key for Kernel::Arg::Metadata::mPointeeAlign.
 constexpr char PointeeAlign[] = "PointeeAlign";
@@ -198,8 +199,6 @@ struct Metadata final {
   uint32_t mAlign = 0;
   /// Value kind. Required.
   ValueKind mValueKind = ValueKind::Unknown;
-  /// Value type. Required.
-  ValueType mValueType = ValueType::Unknown;
   /// Pointee alignment in bytes. Optional.
   uint32_t mPointeeAlign = 0;
   /// Address space qualifier. Optional.
diff --git a/llvm/include/llvm/Support/ARMTargetParser.def b/llvm/include/llvm/Support/ARMTargetParser.def
index 5c96225351a94..9f51c841e4296 100644
--- a/llvm/include/llvm/Support/ARMTargetParser.def
+++ b/llvm/include/llvm/Support/ARMTargetParser.def
@@ -292,6 +292,12 @@ ARM_CPU_NAME("cortex-a76", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
             (ARM::AEK_FP16 | ARM::AEK_DOTPROD))
 ARM_CPU_NAME("cortex-a76ae", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
             (ARM::AEK_FP16 | ARM::AEK_DOTPROD))
+ARM_CPU_NAME("cortex-a77", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
+            (ARM::AEK_FP16 | ARM::AEK_DOTPROD))
+ARM_CPU_NAME("cortex-a78",ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
+             (ARM::AEK_FP16 | ARM::AEK_DOTPROD))
+ARM_CPU_NAME("cortex-x1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
+             (ARM::AEK_FP16 | ARM::AEK_DOTPROD))
 ARM_CPU_NAME("neoverse-n1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
              (ARM::AEK_FP16 | ARM::AEK_DOTPROD))
 ARM_CPU_NAME("cyclone", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC)
diff --git a/llvm/include/llvm/Support/ARMTargetParser.h b/llvm/include/llvm/Support/ARMTargetParser.h
index 322e89e3d0636..4e76b3c4b83e0 100644
--- a/llvm/include/llvm/Support/ARMTargetParser.h
+++ b/llvm/include/llvm/Support/ARMTargetParser.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_SUPPORT_ARMTARGETPARSER_H
 #define LLVM_SUPPORT_ARMTARGETPARSER_H
 
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ARMBuildAttributes.h"
 #include <vector>
diff --git a/llvm/include/llvm/Support/AtomicOrdering.h b/llvm/include/llvm/Support/AtomicOrdering.h
index 763bc3ea7b282..a8d89955fa2b5 100644
--- a/llvm/include/llvm/Support/AtomicOrdering.h
+++ b/llvm/include/llvm/Support/AtomicOrdering.h
@@ -53,7 +53,7 @@ template <typename Int> inline bool isValidAtomicOrderingCABI(Int I) {
 ///
 /// not_atomic-->unordered-->relaxed-->release--------------->acq_rel-->seq_cst
 ///                                   \-->consume-->acquire--/
-enum class AtomicOrdering {
+enum class AtomicOrdering : unsigned {
   NotAtomic = 0,
   Unordered = 1,
   Monotonic = 2, // Equivalent to C++'s relaxed.
@@ -61,7 +61,8 @@ enum class AtomicOrdering {
   Acquire = 4,
   Release = 5,
   AcquireRelease = 6,
-  SequentiallyConsistent = 7
+  SequentiallyConsistent = 7,
+  LAST = SequentiallyConsistent
 };
 
 bool operator<(AtomicOrdering, AtomicOrdering) = delete;
diff --git a/llvm/include/llvm/Support/FileCollector.h b/llvm/include/llvm/Support/FileCollector.h
index efe58909c9a0f..2b5e9c669b680 100644
--- a/llvm/include/llvm/Support/FileCollector.h
+++ b/llvm/include/llvm/Support/FileCollector.h
@@ -12,13 +12,13 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringSet.h"
-#include "llvm/ADT/Twine.h"
 #include "llvm/Support/VirtualFileSystem.h"
-
 #include <mutex>
+#include <string>
 
 namespace llvm {
 class FileCollectorFileSystem;
+class Twine;
 
 /// Captures file system interaction and generates data to be later replayed
 /// with the RedirectingFileSystem.
diff --git a/llvm/include/llvm/Support/FormattedStream.h b/llvm/include/llvm/Support/FormattedStream.h
index b49c8d86531db..5f937cfa79840 100644
--- a/llvm/include/llvm/Support/FormattedStream.h
+++ b/llvm/include/llvm/Support/FormattedStream.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_SUPPORT_FORMATTEDSTREAM_H
 #define LLVM_SUPPORT_FORMATTEDSTREAM_H
 
+#include "llvm/ADT/SmallString.h"
 #include "llvm/Support/raw_ostream.h"
 #include <utility>
 
@@ -21,8 +22,11 @@ namespace llvm {
 
 /// formatted_raw_ostream - A raw_ostream that wraps another one and keeps track
 /// of line and column position, allowing padding out to specific column
-/// boundaries and querying the number of lines written to the stream.
-///
+/// boundaries and querying the number of lines written to the stream. This
+/// assumes that the contents of the stream is valid UTF-8 encoded text. This
+/// doesn't attempt to handle everything Unicode can do (combining characters,
+/// right-to-left markers, etc), but should cover the cases likely to appear in
+/// source code or diagnostic messages.
 class formatted_raw_ostream : public raw_ostream {
   /// TheStream - The real stream we output to. We set it to be
   /// unbuffered, since we're already doing our own buffering.
@@ -40,6 +44,14 @@ class formatted_raw_ostream : public raw_ostream {
   ///
   const char *Scanned;
 
+  /// PartialUTF8Char - Either empty or a prefix of a UTF-8 code unit sequence
+  /// for a Unicode scalar value which should be prepended to the buffer for the
+  /// next call to ComputePosition. This is needed when the buffer is flushed
+  /// when it ends part-way through the UTF-8 encoding of a Unicode scalar
+  /// value, so that we can compute the display width of the character once we
+  /// have the rest of it.
+  SmallString<4> PartialUTF8Char;
+
   void write_impl(const char *Ptr, size_t Size) override;
 
   /// current_pos - Return the current position within the stream,
@@ -52,10 +64,16 @@ class formatted_raw_ostream : public raw_ostream {
   }
 
   /// ComputePosition - Examine the given output buffer and figure out the new
-  /// position after output.
-  ///
+  /// position after output. This is safe to call multiple times on the same
+  /// buffer, as it records the most recently scanned character and resumes from
+  /// there when the buffer has not been flushed.
   void ComputePosition(const char *Ptr, size_t size);
 
+  /// UpdatePosition - scan the characters in [Ptr, Ptr+Size), and update the
+  /// line and column numbers. Unlike ComputePosition, this must be called
+  /// exactly once on each region of the buffer.
+  void UpdatePosition(const char *Ptr, size_t Size);
+
   void setStream(raw_ostream &Stream) {
     releaseStream();
 
@@ -105,11 +123,17 @@ class formatted_raw_ostream : public raw_ostream {
   /// \param NewCol - The column to move to.
   formatted_raw_ostream &PadToColumn(unsigned NewCol);
 
-  /// getColumn - Return the column number
-  unsigned getColumn() { return Position.first; }
+  unsigned getColumn() {
+    // Calculate current position, taking buffer contents into account.
+    ComputePosition(getBufferStart(), GetNumBytesInBuffer());
+    return Position.first;
+  }
 
-  /// getLine - Return the line number
-  unsigned getLine() { return Position.second; }
+  unsigned getLine() {
+    // Calculate current position, taking buffer contents into account.
+    ComputePosition(getBufferStart(), GetNumBytesInBuffer());
+    return Position.second;
+  }
 
   raw_ostream &resetColor() override {
     TheStream->resetColor();
diff --git a/llvm/include/llvm/Support/GenericDomTree.h b/llvm/include/llvm/Support/GenericDomTree.h
index f90fe0ba7dae6..10e591a69d369 100644
--- a/llvm/include/llvm/Support/GenericDomTree.h
+++ b/llvm/include/llvm/Support/GenericDomTree.h
@@ -77,18 +77,25 @@ template <class NodeT> class DomTreeNodeBase {
   const_iterator begin() const { return Children.begin(); }
   const_iterator end() const { return Children.end(); }
 
+  DomTreeNodeBase *const &back() const { return Children.back(); }
+  DomTreeNodeBase *&back() { return Children.back(); }
+
+  iterator_range<iterator> children() { return make_range(begin(), end()); }
+  iterator_range<const_iterator> children() const {
+    return make_range(begin(), end());
+  }
+
   NodeT *getBlock() const { return TheBB; }
   DomTreeNodeBase *getIDom() const { return IDom; }
   unsigned getLevel() const { return Level; }
 
-  const std::vector<DomTreeNodeBase *> &getChildren() const { return Children; }
-
   std::unique_ptr<DomTreeNodeBase> addChild(
       std::unique_ptr<DomTreeNodeBase> C) {
     Children.push_back(C.get());
     return C;
   }
 
+  bool isLeaf() const { return Children.empty(); }
   size_t getNumChildren() const { return Children.size(); }
 
   void clearAllChildren() { Children.clear(); }
@@ -276,11 +283,27 @@ class DominatorTreeBase {
   DominatorTreeBase(const DominatorTreeBase &) = delete;
   DominatorTreeBase &operator=(const DominatorTreeBase &) = delete;
 
-  /// getRoots - Return the root blocks of the current CFG.  This may include
-  /// multiple blocks if we are computing post dominators.  For forward
-  /// dominators, this will always be a single block (the entry node).
+  /// Iteration over roots.
   ///
-  const SmallVectorImpl<NodeT *> &getRoots() const { return Roots; }
+  /// This may include multiple blocks if we are computing post dominators.
+  /// For forward dominators, this will always be a single block (the entry
+  /// block).
+  using root_iterator = typename SmallVectorImpl<NodeT *>::iterator;
+  using const_root_iterator = typename SmallVectorImpl<NodeT *>::const_iterator;
+
+  root_iterator root_begin() { return Roots.begin(); }
+  const_root_iterator root_begin() const { return Roots.begin(); }
+  root_iterator root_end() { return Roots.end(); }
+  const_root_iterator root_end() const { return Roots.end(); }
+
+  size_t root_size() const { return Roots.size(); }
+
+  iterator_range<root_iterator> roots() {
+    return make_range(root_begin(), root_end());
+  }
+  iterator_range<const_root_iterator> roots() const {
+    return make_range(root_begin(), root_end());
+  }
 
   /// isPostDominator - Returns true if analysis based of postdoms
   ///
@@ -318,8 +341,6 @@ class DominatorTreeBase {
     return false;
   }
 
-  void releaseMemory() { reset(); }
-
   /// getNode - return the (Post)DominatorTree node for the specified basic
   /// block.  This is the same as using operator[] on this class.  The result
   /// may (but is not required to) be null for a forward (backwards)
@@ -569,8 +590,7 @@ class DominatorTreeBase {
     DomTreeNodeBase<NodeT> *IDomNode = getNode(DomBB);
     assert(IDomNode && "Not immediate dominator specified for block!");
     DFSInfoValid = false;
-    return (DomTreeNodes[BB] = IDomNode->addChild(
-                std::make_unique<DomTreeNodeBase<NodeT>>(BB, IDomNode))).get();
+    return createChild(BB, IDomNode);
   }
 
   /// Add a new node to the forward dominator tree and make it a new root.
@@ -583,8 +603,7 @@ class DominatorTreeBase {
     assert(!this->isPostDominator() &&
            "Cannot change root of post-dominator tree");
     DFSInfoValid = false;
-    DomTreeNodeBase<NodeT> *NewNode = (DomTreeNodes[BB] =
-      std::make_unique<DomTreeNodeBase<NodeT>>(BB, nullptr)).get();
+    DomTreeNodeBase<NodeT> *NewNode = createNode(BB);
     if (Roots.empty()) {
       addRoot(BB);
     } else {
@@ -619,7 +638,7 @@ class DominatorTreeBase {
   void eraseNode(NodeT *BB) {
     DomTreeNodeBase<NodeT> *Node = getNode(BB);
     assert(Node && "Removing node that isn't in dominator tree.");
-    assert(Node->getChildren().empty() && "Node is not a leaf node.");
+    assert(Node->isLeaf() && "Node is not a leaf node.");
 
     DFSInfoValid = false;
 
@@ -753,9 +772,6 @@ class DominatorTreeBase {
     return DomTreeBuilder::Verify(*this, VL);
   }
 
-protected:
-  void addRoot(NodeT *BB) { this->Roots.push_back(BB); }
-
   void reset() {
     DomTreeNodes.clear();
     Roots.clear();
@@ -765,6 +781,21 @@ class DominatorTreeBase {
     SlowQueries = 0;
   }
 
+protected:
+  void addRoot(NodeT *BB) { this->Roots.push_back(BB); }
+
+  DomTreeNodeBase<NodeT> *createChild(NodeT *BB, DomTreeNodeBase<NodeT> *IDom) {
+    return (DomTreeNodes[BB] = IDom->addChild(
+                std::make_unique<DomTreeNodeBase<NodeT>>(BB, IDom)))
+        .get();
+  }
+
+  DomTreeNodeBase<NodeT> *createNode(NodeT *BB) {
+    return (DomTreeNodes[BB] =
+                std::make_unique<DomTreeNodeBase<NodeT>>(BB, nullptr))
+        .get();
+  }
+
   // NewBB is split and now it has one successor. Update dominator tree to
   // reflect this change.
   template <class N>
diff --git a/llvm/include/llvm/Support/GenericDomTreeConstruction.h b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
index 7c94a26705a84..464de4e2b3ba1 100644
--- a/llvm/include/llvm/Support/GenericDomTreeConstruction.h
+++ b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
@@ -187,9 +187,7 @@ struct SemiNCAInfo {
 
     // Add a new tree node for this NodeT, and link it as a child of
     // IDomNode
-    return (DT.DomTreeNodes[BB] = IDomNode->addChild(
-        std::make_unique<DomTreeNodeBase<NodeT>>(BB, IDomNode)))
-        .get();
+    return DT.createChild(BB, IDomNode);
   }
 
   static bool AlwaysDescend(NodePtr, NodePtr) { return true; }
@@ -587,9 +585,7 @@ struct SemiNCAInfo {
     // all real exits (including multiple exit blocks, infinite loops).
     NodePtr Root = IsPostDom ? nullptr : DT.Roots[0];
 
-    DT.RootNode = (DT.DomTreeNodes[Root] =
-                       std::make_unique<DomTreeNodeBase<NodeT>>(Root, nullptr))
-        .get();
+    DT.RootNode = DT.createNode(Root);
     SNCA.attachNewSubtree(DT, DT.RootNode);
   }
 
@@ -610,8 +606,7 @@ struct SemiNCAInfo {
 
       // Add a new tree node for this BasicBlock, and link it as a child of
       // IDomNode.
-      DT.DomTreeNodes[W] = IDomNode->addChild(
-          std::make_unique<DomTreeNodeBase<NodeT>>(W, IDomNode));
+      DT.createChild(W, IDomNode);
     }
   }
 
@@ -661,10 +656,7 @@ struct SemiNCAInfo {
 
       // The unreachable node becomes a new root -- a tree node for it.
       TreeNodePtr VirtualRoot = DT.getNode(nullptr);
-      FromTN =
-          (DT.DomTreeNodes[From] = VirtualRoot->addChild(
-               std::make_unique<DomTreeNodeBase<NodeT>>(From, VirtualRoot)))
-              .get();
+      FromTN = DT.createChild(From, VirtualRoot);
       DT.Roots.push_back(From);
     }
 
@@ -1372,7 +1364,7 @@ struct SemiNCAInfo {
     if (!DT.DFSInfoValid || !DT.Parent)
       return true;
 
-    const NodePtr RootBB = IsPostDom ? nullptr : DT.getRoots()[0];
+    const NodePtr RootBB = IsPostDom ? nullptr : *DT.root_begin();
     const TreeNodePtr Root = DT.getNode(RootBB);
 
     auto PrintNodeAndDFSNums = [](const TreeNodePtr TN) {
@@ -1396,7 +1388,7 @@ struct SemiNCAInfo {
       const TreeNodePtr Node = NodeToTN.second.get();
 
       // Handle tree leaves.
-      if (Node->getChildren().empty()) {
+      if (Node->isLeaf()) {
         if (Node->getDFSNumIn() + 1 != Node->getDFSNumOut()) {
           errs() << "Tree leaf should have DFSOut = DFSIn + 1:\n\t";
           PrintNodeAndDFSNums(Node);
@@ -1508,7 +1500,8 @@ struct SemiNCAInfo {
     for (auto &NodeToTN : DT.DomTreeNodes) {
       const TreeNodePtr TN = NodeToTN.second.get();
       const NodePtr BB = TN->getBlock();
-      if (!BB || TN->getChildren().empty()) continue;
+      if (!BB || TN->isLeaf())
+        continue;
 
       LLVM_DEBUG(dbgs() << "Verifying parent property of node "
                         << BlockNamePrinter(TN) << "\n");
@@ -1517,7 +1510,7 @@ struct SemiNCAInfo {
         return From != BB && To != BB;
       });
 
-      for (TreeNodePtr Child : TN->getChildren())
+      for (TreeNodePtr Child : TN->children())
         if (NodeToInfo.count(Child->getBlock()) != 0) {
           errs() << "Child " << BlockNamePrinter(Child)
                  << " reachable after its parent " << BlockNamePrinter(BB)
@@ -1541,17 +1534,17 @@ struct SemiNCAInfo {
     for (auto &NodeToTN : DT.DomTreeNodes) {
       const TreeNodePtr TN = NodeToTN.second.get();
       const NodePtr BB = TN->getBlock();
-      if (!BB || TN->getChildren().empty()) continue;
+      if (!BB || TN->isLeaf())
+        continue;
 
-      const auto &Siblings = TN->getChildren();
-      for (const TreeNodePtr N : Siblings) {
+      for (const TreeNodePtr N : TN->children()) {
         clear();
         NodePtr BBN = N->getBlock();
         doFullDFSWalk(DT, [BBN](NodePtr From, NodePtr To) {
           return From != BBN && To != BBN;
         });
 
-        for (const TreeNodePtr S : Siblings) {
+        for (const TreeNodePtr S : TN->children()) {
           if (S == N) continue;
 
           if (NodeToInfo.count(S->getBlock()) == 0) {
diff --git a/llvm/include/llvm/Support/LowLevelTypeImpl.h b/llvm/include/llvm/Support/LowLevelTypeImpl.h
index 6959957cfc37f..c1d516f2fe587 100644
--- a/llvm/include/llvm/Support/LowLevelTypeImpl.h
+++ b/llvm/include/llvm/Support/LowLevelTypeImpl.h
@@ -27,6 +27,7 @@
 #define LLVM_SUPPORT_LOWLEVELTYPEIMPL_H
 
 #include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Support/MachineValueType.h"
 #include <cassert>
 
diff --git a/llvm/include/llvm/Support/MachineValueType.h b/llvm/include/llvm/Support/MachineValueType.h
index 9d2d0c85f585e..3bb8220e72e5d 100644
--- a/llvm/include/llvm/Support/MachineValueType.h
+++ b/llvm/include/llvm/Support/MachineValueType.h
@@ -123,93 +123,103 @@ namespace llvm {
       v8f16          =  68,   //    8 x f16
       v16f16         =  69,   //   16 x f16
       v32f16         =  70,   //   32 x f16
-      v2bf16         =  71,   //    2 x bf16
-      v3bf16         =  72,   //    3 x bf16
-      v4bf16         =  73,   //    4 x bf16
-      v8bf16         =  74,   //    8 x bf16
-      v16bf16        =  75,   //   16 x bf16
-      v32bf16        =  76,   //   32 x bf16
-      v1f32          =  77,   //    1 x f32
-      v2f32          =  78,   //    2 x f32
-      v3f32          =  79,   //    3 x f32
-      v4f32          =  80,   //    4 x f32
-      v5f32          =  81,   //    5 x f32
-      v8f32          =  82,   //    8 x f32
-      v16f32         =  83,   //   16 x f32
-      v32f32         =  84,   //   32 x f32
-      v64f32         =  85,   //   64 x f32
-      v128f32        =  86,   //  128 x f32
-      v256f32        =  87,   //  256 x f32
-      v512f32        =  88,   //  512 x f32
-      v1024f32       =  89,   // 1024 x f32
-      v2048f32       =  90,   // 2048 x f32
-      v1f64          =  91,   //    1 x f64
-      v2f64          =  92,   //    2 x f64
-      v4f64          =  93,   //    4 x f64
-      v8f64          =  94,   //    8 x f64
-      v16f64         =  95,   //   16 x f64
+      v64f16         =  71,   //   64 x f16
+      v128f16        =  72,   //  128 x f16
+      v2bf16         =  73,   //    2 x bf16
+      v3bf16         =  74,   //    3 x bf16
+      v4bf16         =  75,   //    4 x bf16
+      v8bf16         =  76,   //    8 x bf16
+      v16bf16        =  77,   //   16 x bf16
+      v32bf16        =  78,   //   32 x bf16
+      v64bf16        =  79,   //   64 x bf16
+      v128bf16       =  80,   //  128 x bf16
+      v1f32          =  81,   //    1 x f32
+      v2f32          =  82,   //    2 x f32
+      v3f32          =  83,   //    3 x f32
+      v4f32          =  84,   //    4 x f32
+      v5f32          =  85,   //    5 x f32
+      v8f32          =  86,   //    8 x f32
+      v16f32         =  87,   //   16 x f32
+      v32f32         =  88,   //   32 x f32
+      v64f32         =  89,   //   64 x f32
+      v128f32        =  90,   //  128 x f32
+      v256f32        =  91,   //  256 x f32
+      v512f32        =  92,   //  512 x f32
+      v1024f32       =  93,   // 1024 x f32
+      v2048f32       =  94,   // 2048 x f32
+      v1f64          =  95,   //    1 x f64
+      v2f64          =  96,   //    2 x f64
+      v4f64          =  97,   //    4 x f64
+      v8f64          =  98,   //    8 x f64
+      v16f64         =  99,   //   16 x f64
+      v32f64         = 100,   //   32 x f64
 
       FIRST_FP_FIXEDLEN_VECTOR_VALUETYPE = v2f16,
-      LAST_FP_FIXEDLEN_VECTOR_VALUETYPE = v16f64,
+      LAST_FP_FIXEDLEN_VECTOR_VALUETYPE = v32f64,
 
       FIRST_FIXEDLEN_VECTOR_VALUETYPE = v1i1,
-      LAST_FIXEDLEN_VECTOR_VALUETYPE = v16f64,
-
-      nxv1i1         =  96,   // n x  1 x i1
-      nxv2i1         =  97,   // n x  2 x i1
-      nxv4i1         =  98,   // n x  4 x i1
-      nxv8i1         =  99,   // n x  8 x i1
-      nxv16i1        = 100,   // n x 16 x i1
-      nxv32i1        = 101,   // n x 32 x i1
-
-      nxv1i8         = 102,   // n x  1 x i8
-      nxv2i8         = 103,   // n x  2 x i8
-      nxv4i8         = 104,   // n x  4 x i8
-      nxv8i8         = 105,   // n x  8 x i8
-      nxv16i8        = 106,   // n x 16 x i8
-      nxv32i8        = 107,   // n x 32 x i8
-
-      nxv1i16        = 108,  // n x  1 x i16
-      nxv2i16        = 109,  // n x  2 x i16
-      nxv4i16        = 110,  // n x  4 x i16
-      nxv8i16        = 111,  // n x  8 x i16
-      nxv16i16       = 112,  // n x 16 x i16
-      nxv32i16       = 113,  // n x 32 x i16
-
-      nxv1i32        = 114,  // n x  1 x i32
-      nxv2i32        = 115,  // n x  2 x i32
-      nxv4i32        = 116,  // n x  4 x i32
-      nxv8i32        = 117,  // n x  8 x i32
-      nxv16i32       = 118,  // n x 16 x i32
-      nxv32i32       = 119,  // n x 32 x i32
-
-      nxv1i64        = 120,  // n x  1 x i64
-      nxv2i64        = 121,  // n x  2 x i64
-      nxv4i64        = 122,  // n x  4 x i64
-      nxv8i64        = 123,  // n x  8 x i64
-      nxv16i64       = 124,  // n x 16 x i64
-      nxv32i64       = 125,  // n x 32 x i64
+      LAST_FIXEDLEN_VECTOR_VALUETYPE = v32f64,
+
+      nxv1i1         = 101,   // n x  1 x i1
+      nxv2i1         = 102,   // n x  2 x i1
+      nxv4i1         = 103,   // n x  4 x i1
+      nxv8i1         = 104,   // n x  8 x i1
+      nxv16i1        = 105,   // n x 16 x i1
+      nxv32i1        = 106,   // n x 32 x i1
+      nxv64i1        = 107,   // n x  64 x i1
+
+      nxv1i8         = 108,   // n x  1 x i8
+      nxv2i8         = 109,   // n x  2 x i8
+      nxv4i8         = 110,   // n x  4 x i8
+      nxv8i8         = 111,   // n x  8 x i8
+      nxv16i8        = 112,   // n x 16 x i8
+      nxv32i8        = 113,   // n x 32 x i8
+      nxv64i8        = 114,   // n x  64 x i8
+
+      nxv1i16        = 115,  // n x  1 x i16
+      nxv2i16        = 116,  // n x  2 x i16
+      nxv4i16        = 117,  // n x  4 x i16
+      nxv8i16        = 118,  // n x  8 x i16
+      nxv16i16       = 119,  // n x 16 x i16
+      nxv32i16       = 120,  // n x 32 x i16
+
+      nxv1i32        = 121,  // n x  1 x i32
+      nxv2i32        = 122,  // n x  2 x i32
+      nxv4i32        = 123,  // n x  4 x i32
+      nxv8i32        = 124,  // n x  8 x i32
+      nxv16i32       = 125,  // n x 16 x i32
+      nxv32i32       = 126,  // n x 32 x i32
+
+      nxv1i64        = 127,  // n x  1 x i64
+      nxv2i64        = 128,  // n x  2 x i64
+      nxv4i64        = 129,  // n x  4 x i64
+      nxv8i64        = 130,  // n x  8 x i64
+      nxv16i64       = 131,  // n x 16 x i64
+      nxv32i64       = 132,  // n x 32 x i64
 
       FIRST_INTEGER_SCALABLE_VECTOR_VALUETYPE = nxv1i1,
       LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE = nxv32i64,
 
-      nxv2f16        = 126,  // n x  2 x f16
-      nxv4f16        = 127,  // n x  4 x f16
-      nxv8f16        = 128,  // n x  8 x f16
-      nxv2bf16       = 129,  // n x  2 x bf16
-      nxv4bf16       = 130,  // n x  4 x bf16
-      nxv8bf16       = 131,  // n x  8 x bf16
-      nxv1f32        = 132,  // n x  1 x f32
-      nxv2f32        = 133,  // n x  2 x f32
-      nxv4f32        = 134,  // n x  4 x f32
-      nxv8f32        = 135,  // n x  8 x f32
-      nxv16f32       = 136,  // n x 16 x f32
-      nxv1f64        = 137,  // n x  1 x f64
-      nxv2f64        = 138,  // n x  2 x f64
-      nxv4f64        = 139,  // n x  4 x f64
-      nxv8f64        = 140,  // n x  8 x f64
-
-      FIRST_FP_SCALABLE_VECTOR_VALUETYPE = nxv2f16,
+      nxv1f16        = 133,   // n x   1 x f16
+      nxv2f16        = 134,  // n x  2 x f16
+      nxv4f16        = 135,  // n x  4 x f16
+      nxv8f16        = 136,  // n x  8 x f16
+      nxv16f16       = 137,   // n x  16 x f16
+      nxv32f16       = 138,   // n x  32 x f16
+      nxv2bf16       = 139,  // n x  2 x bf16
+      nxv4bf16       = 140,  // n x  4 x bf16
+      nxv8bf16       = 141,  // n x  8 x bf16
+      nxv1f32        = 142,  // n x  1 x f32
+      nxv2f32        = 143,  // n x  2 x f32
+      nxv4f32        = 144,  // n x  4 x f32
+      nxv8f32        = 145,  // n x  8 x f32
+      nxv16f32       = 146,  // n x 16 x f32
+      nxv1f64        = 147,  // n x  1 x f64
+      nxv2f64        = 148,  // n x  2 x f64
+      nxv4f64        = 149,  // n x  4 x f64
+      nxv8f64        = 150,  // n x  8 x f64
+
+      FIRST_FP_SCALABLE_VECTOR_VALUETYPE = nxv1f16,
       LAST_FP_SCALABLE_VECTOR_VALUETYPE = nxv8f64,
 
       FIRST_SCALABLE_VECTOR_VALUETYPE = nxv1i1,
@@ -218,20 +228,20 @@ namespace llvm {
       FIRST_VECTOR_VALUETYPE = v1i1,
       LAST_VECTOR_VALUETYPE  = nxv8f64,
 
-      x86mmx         = 141,   // This is an X86 MMX value
+      x86mmx         = 151,   // This is an X86 MMX value
 
-      Glue           = 142,   // This glues nodes together during pre-RA sched
+      Glue           = 152,   // This glues nodes together during pre-RA sched
 
-      isVoid         = 143,   // This has no value
+      isVoid         = 153,   // This has no value
 
-      Untyped        = 144,   // This value takes a register, but has
-                               // unspecified type.  The register class
-                               // will be determined by the opcode.
+      Untyped        = 154,   // This value takes a register, but has
+                              // unspecified type.  The register class
+                              // will be determined by the opcode.
 
-      exnref         = 145,   // WebAssembly's exnref type
+      exnref         = 155,   // WebAssembly's exnref type
 
-      FIRST_VALUETYPE = 1,     // This is always the beginning of the list.
-      LAST_VALUETYPE = 146,   // This always remains at the end of the list.
+      FIRST_VALUETYPE =  1,   // This is always the beginning of the list.
+      LAST_VALUETYPE = 156,   // This always remains at the end of the list.
 
       // This is the current maximum for LAST_VALUETYPE.
       // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors
@@ -389,20 +399,24 @@ namespace llvm {
     bool is1024BitVector() const {
       return (SimpleTy == MVT::v1024i1 || SimpleTy == MVT::v128i8 ||
               SimpleTy == MVT::v64i16  || SimpleTy == MVT::v32i32 ||
-              SimpleTy == MVT::v16i64  || SimpleTy == MVT::v16f64);
+              SimpleTy == MVT::v16i64  || SimpleTy == MVT::v64f16 ||
+              SimpleTy == MVT::v32f32  || SimpleTy == MVT::v16f64 ||
+              SimpleTy == MVT::v64bf16);
     }
 
     /// Return true if this is a 2048-bit vector type.
     bool is2048BitVector() const {
-      return (SimpleTy == MVT::v256i8 || SimpleTy == MVT::v128i16 ||
-              SimpleTy == MVT::v64i32 || SimpleTy == MVT::v32i64);
+      return (SimpleTy == MVT::v256i8  || SimpleTy == MVT::v128i16 ||
+              SimpleTy == MVT::v64i32  || SimpleTy == MVT::v32i64  ||
+              SimpleTy == MVT::v128f16 || SimpleTy == MVT::v64f32  ||
+              SimpleTy == MVT::v32f64  || SimpleTy == MVT::v128bf16);
     }
 
     /// Return true if this is an overloaded type for TableGen.
     bool isOverloaded() const {
-      return (SimpleTy==MVT::Any  ||
-              SimpleTy==MVT::iAny || SimpleTy==MVT::fAny ||
-              SimpleTy==MVT::vAny || SimpleTy==MVT::iPTRAny);
+      return (SimpleTy == MVT::Any || SimpleTy == MVT::iAny ||
+              SimpleTy == MVT::fAny || SimpleTy == MVT::vAny ||
+              SimpleTy == MVT::iPTRAny);
     }
 
     /// Return a VT for a vector type with the same element type but
@@ -456,7 +470,8 @@ namespace llvm {
       case nxv4i1:
       case nxv8i1:
       case nxv16i1:
-      case nxv32i1: return i1;
+      case nxv32i1:
+      case nxv64i1: return i1;
       case v1i8:
       case v2i8:
       case v4i8:
@@ -471,7 +486,8 @@ namespace llvm {
       case nxv4i8:
       case nxv8i8:
       case nxv16i8:
-      case nxv32i8: return i8;
+      case nxv32i8:
+      case nxv64i8: return i8;
       case v1i16:
       case v2i16:
       case v3i16:
@@ -526,15 +542,22 @@ namespace llvm {
       case v8f16:
       case v16f16:
       case v32f16:
+      case v64f16:
+      case v128f16:
+      case nxv1f16:
       case nxv2f16:
       case nxv4f16:
-      case nxv8f16: return f16;
+      case nxv8f16:
+      case nxv16f16:
+      case nxv32f16: return f16;
       case v2bf16:
       case v3bf16:
       case v4bf16:
       case v8bf16:
       case v16bf16:
       case v32bf16:
+      case v64bf16:
+      case v128bf16:
       case nxv2bf16:
       case nxv4bf16:
       case nxv8bf16: return bf16;
@@ -562,6 +585,7 @@ namespace llvm {
       case v4f64:
       case v8f64:
       case v16f64:
+      case v32f64:
       case nxv1f64:
       case nxv2f64:
       case nxv4f64:
@@ -589,12 +613,18 @@ namespace llvm {
       case v128i8:
       case v128i16:
       case v128i32:
+      case v128f16:
+      case v128bf16:
       case v128f32: return 128;
       case v64i1:
       case v64i8:
       case v64i16:
       case v64i32:
-      case v64f32: return 64;
+      case v64f16:
+      case v64bf16:
+      case v64f32:
+      case nxv64i1:
+      case nxv64i8: return 64;
       case v32i1:
       case v32i8:
       case v32i16:
@@ -603,11 +633,13 @@ namespace llvm {
       case v32f16:
       case v32bf16:
       case v32f32:
+      case v32f64:
       case nxv32i1:
       case nxv32i8:
       case nxv32i16:
       case nxv32i32:
-      case nxv32i64: return 32;
+      case nxv32i64:
+      case nxv32f16: return 32;
       case v16i1:
       case v16i8:
       case v16i16:
@@ -622,6 +654,7 @@ namespace llvm {
       case nxv16i16:
       case nxv16i32:
       case nxv16i64:
+      case nxv16f16:
       case nxv16f32: return 16;
       case v8i1:
       case v8i8:
@@ -697,6 +730,7 @@ namespace llvm {
       case nxv1i16:
       case nxv1i32:
       case nxv1i64:
+      case nxv1f16:
       case nxv1f32:
       case nxv1f64: return 1;
       }
@@ -755,7 +789,8 @@ namespace llvm {
       case v1i16: return TypeSize::Fixed(16);
       case nxv16i1:
       case nxv2i8:
-      case nxv1i16: return TypeSize::Scalable(16);
+      case nxv1i16:
+      case nxv1f16: return TypeSize::Scalable(16);
       case f32 :
       case i32 :
       case v32i1:
@@ -787,6 +822,7 @@ namespace llvm {
       case v4bf16:
       case v2f32:
       case v1f64: return TypeSize::Fixed(64);
+      case nxv64i1:
       case nxv8i8:
       case nxv4i16:
       case nxv2i32:
@@ -834,6 +870,7 @@ namespace llvm {
       case nxv16i16:
       case nxv8i32:
       case nxv4i64:
+      case nxv16f16:
       case nxv8f32:
       case nxv4f64: return TypeSize::Scalable(256);
       case v512i1:
@@ -845,9 +882,11 @@ namespace llvm {
       case v32bf16:
       case v16f32:
       case v8f64: return TypeSize::Fixed(512);
+      case nxv64i8:
       case nxv32i16:
       case nxv16i32:
       case nxv8i64:
+      case nxv32f16:
       case nxv16f32:
       case nxv8f64: return TypeSize::Scalable(512);
       case v1024i1:
@@ -855,15 +894,20 @@ namespace llvm {
       case v64i16:
       case v32i32:
       case v16i64:
-      case v16f64:
-      case v32f32: return TypeSize::Fixed(1024);
+      case v64f16:
+      case v64bf16:
+      case v32f32:
+      case v16f64: return TypeSize::Fixed(1024);
       case nxv32i32:
       case nxv16i64: return TypeSize::Scalable(1024);
       case v256i8:
       case v128i16:
       case v64i32:
       case v32i64:
-      case v64f32: return TypeSize::Fixed(2048);
+      case v128f16:
+      case v128bf16:
+      case v64f32:
+      case v32f64: return TypeSize::Fixed(2048);
       case nxv32i64: return TypeSize::Scalable(2048);
       case v128i32:
       case v128f32:  return TypeSize::Fixed(4096);
@@ -1033,20 +1077,24 @@ namespace llvm {
         if (NumElements == 1)  return MVT::v1i128;
         break;
       case MVT::f16:
-        if (NumElements == 2)  return MVT::v2f16;
-        if (NumElements == 3)  return MVT::v3f16;
-        if (NumElements == 4)  return MVT::v4f16;
-        if (NumElements == 8)  return MVT::v8f16;
-        if (NumElements == 16) return MVT::v16f16;
-        if (NumElements == 32) return MVT::v32f16;
+        if (NumElements == 2)   return MVT::v2f16;
+        if (NumElements == 3)   return MVT::v3f16;
+        if (NumElements == 4)   return MVT::v4f16;
+        if (NumElements == 8)   return MVT::v8f16;
+        if (NumElements == 16)  return MVT::v16f16;
+        if (NumElements == 32)  return MVT::v32f16;
+        if (NumElements == 64)  return MVT::v64f16;
+        if (NumElements == 128) return MVT::v128f16;
         break;
       case MVT::bf16:
-        if (NumElements == 2)  return MVT::v2bf16;
-        if (NumElements == 3)  return MVT::v3bf16;
-        if (NumElements == 4)  return MVT::v4bf16;
-        if (NumElements == 8)  return MVT::v8bf16;
-        if (NumElements == 16) return MVT::v16bf16;
-        if (NumElements == 32) return MVT::v32bf16;
+        if (NumElements == 2)   return MVT::v2bf16;
+        if (NumElements == 3)   return MVT::v3bf16;
+        if (NumElements == 4)   return MVT::v4bf16;
+        if (NumElements == 8)   return MVT::v8bf16;
+        if (NumElements == 16)  return MVT::v16bf16;
+        if (NumElements == 32)  return MVT::v32bf16;
+        if (NumElements == 64)  return MVT::v64bf16;
+        if (NumElements == 128) return MVT::v128bf16;
         break;
       case MVT::f32:
         if (NumElements == 1)    return MVT::v1f32;
@@ -1070,6 +1118,7 @@ namespace llvm {
         if (NumElements == 4)  return MVT::v4f64;
         if (NumElements == 8)  return MVT::v8f64;
         if (NumElements == 16) return MVT::v16f64;
+        if (NumElements == 32) return MVT::v32f64;
         break;
       }
       return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE);
@@ -1086,6 +1135,7 @@ namespace llvm {
           if (NumElements == 8)  return MVT::nxv8i1;
           if (NumElements == 16) return MVT::nxv16i1;
           if (NumElements == 32) return MVT::nxv32i1;
+          if (NumElements == 64) return MVT::nxv64i1;
           break;
         case MVT::i8:
           if (NumElements == 1)  return MVT::nxv1i8;
@@ -1094,6 +1144,7 @@ namespace llvm {
           if (NumElements == 8)  return MVT::nxv8i8;
           if (NumElements == 16) return MVT::nxv16i8;
           if (NumElements == 32) return MVT::nxv32i8;
+          if (NumElements == 64) return MVT::nxv64i8;
           break;
         case MVT::i16:
           if (NumElements == 1)  return MVT::nxv1i16;
@@ -1120,9 +1171,12 @@ namespace llvm {
           if (NumElements == 32) return MVT::nxv32i64;
           break;
         case MVT::f16:
+          if (NumElements == 1)  return MVT::nxv1f16;
           if (NumElements == 2)  return MVT::nxv2f16;
           if (NumElements == 4)  return MVT::nxv4f16;
           if (NumElements == 8)  return MVT::nxv8f16;
+          if (NumElements == 16)  return MVT::nxv16f16;
+          if (NumElements == 32)  return MVT::nxv32f16;
           break;
         case MVT::bf16:
           if (NumElements == 2)  return MVT::nxv2bf16;
diff --git a/llvm/include/llvm/Support/Path.h b/llvm/include/llvm/Support/Path.h
index cdfff2aa7a513..83bca5b70bc21 100644
--- a/llvm/include/llvm/Support/Path.h
+++ b/llvm/include/llvm/Support/Path.h
@@ -371,6 +371,13 @@ void system_temp_directory(bool erasedOnReboot, SmallVectorImpl<char> &result);
 /// @result True if a home directory is set, false otherwise.
 bool home_directory(SmallVectorImpl<char> &result);
 
+/// Get the directory where packages should read user-specific configurations.
+/// e.g. $XDG_CONFIG_HOME.
+///
+/// @param result Holds the resulting path name.
+/// @result True if the appropriate path was determined, it need not exist.
+bool user_config_directory(SmallVectorImpl<char> &result);
+
 /// Get the directory where installed packages should put their
 /// machine-local cache, e.g. $XDG_CACHE_HOME.
 ///
diff --git a/llvm/include/llvm/Support/PrettyStackTrace.h b/llvm/include/llvm/Support/PrettyStackTrace.h
index 32975f3c5a353..ac25cffde051c 100644
--- a/llvm/include/llvm/Support/PrettyStackTrace.h
+++ b/llvm/include/llvm/Support/PrettyStackTrace.h
@@ -41,6 +41,9 @@ namespace llvm {
   /// a crash.
   void setBugReportMsg(const char *Msg);
 
+  /// Get the bug report message that will be output upon a crash.
+  const char *getBugReportMsg();
+
   /// PrettyStackTraceEntry - This class is used to represent a frame of the
   /// "pretty" stack trace that is dumped when a program crashes. You can define
   /// subclasses of this and declare them on the program stack: when they are
diff --git a/llvm/include/llvm/Support/Program.h b/llvm/include/llvm/Support/Program.h
index ce0b6db5705ff..dbda064cda058 100644
--- a/llvm/include/llvm/Support/Program.h
+++ b/llvm/include/llvm/Support/Program.h
@@ -57,7 +57,7 @@ namespace sys {
   struct ProcessStatistics {
     std::chrono::microseconds TotalTime;
     std::chrono::microseconds UserTime;
-    uint64_t PeakMemory = 0;
+    uint64_t PeakMemory = 0; ///< Maximum resident set size in KiB.
   };
 
   /// Find the first executable file \p Name in \p Paths.
diff --git a/llvm/include/llvm/Support/Regex.h b/llvm/include/llvm/Support/Regex.h
index b2620ab4cfc93..ae4b9516f194e 100644
--- a/llvm/include/llvm/Support/Regex.h
+++ b/llvm/include/llvm/Support/Regex.h
@@ -16,6 +16,7 @@
 #ifndef LLVM_SUPPORT_REGEX_H
 #define LLVM_SUPPORT_REGEX_H
 
+#include "llvm/ADT/BitmaskEnum.h"
 #include <string>
 
 struct llvm_regex;
@@ -26,20 +27,22 @@ namespace llvm {
 
   class Regex {
   public:
-    enum {
-      NoFlags=0,
+    enum RegexFlags : unsigned {
+      NoFlags = 0,
       /// Compile for matching that ignores upper/lower case distinctions.
-      IgnoreCase=1,
+      IgnoreCase = 1,
       /// Compile for newline-sensitive matching. With this flag '[^' bracket
       /// expressions and '.' never match newline. A ^ anchor matches the
       /// null string after any newline in the string in addition to its normal
       /// function, and the $ anchor matches the null string before any
       /// newline in the string in addition to its normal function.
-      Newline=2,
+      Newline = 2,
       /// By default, the POSIX extended regular expression (ERE) syntax is
       /// assumed. Pass this flag to turn on basic regular expressions (BRE)
       /// instead.
-      BasicRegex=4
+      BasicRegex = 4,
+
+      LLVM_MARK_AS_BITMASK_ENUM(BasicRegex)
     };
 
     Regex();
@@ -47,7 +50,8 @@ namespace llvm {
     ///
     /// \param Regex - referenced string is no longer needed after this
     /// constructor does finish.  Only its compiled form is kept stored.
-    Regex(StringRef Regex, unsigned Flags = NoFlags);
+    Regex(StringRef Regex, RegexFlags Flags = NoFlags);
+    Regex(StringRef Regex, unsigned Flags);
     Regex(const Regex &) = delete;
     Regex &operator=(Regex regex) {
       std::swap(preg, regex.preg);
diff --git a/llvm/include/llvm/Support/SwapByteOrder.h b/llvm/include/llvm/Support/SwapByteOrder.h
index 500df23553079..0e544fc7e71ee 100644
--- a/llvm/include/llvm/Support/SwapByteOrder.h
+++ b/llvm/include/llvm/Support/SwapByteOrder.h
@@ -36,6 +36,10 @@
 #else
 #define BYTE_ORDER LITTLE_ENDIAN
 #endif
+#elif defined(__MVS__)
+#define BIG_ENDIAN 4321
+#define LITTLE_ENDIAN 1234
+#define BYTE_ORDER BIG_ENDIAN
 #else
 #if !defined(BYTE_ORDER) && !defined(_WIN32)
 #include <machine/endian.h>
diff --git a/llvm/include/llvm/Support/SystemUtils.h b/llvm/include/llvm/Support/SystemUtils.h
index 77deddb9ee1c2..786bea3fcfae6 100644
--- a/llvm/include/llvm/Support/SystemUtils.h
+++ b/llvm/include/llvm/Support/SystemUtils.h
@@ -15,17 +15,16 @@
 #define LLVM_SUPPORT_SYSTEMUTILS_H
 
 namespace llvm {
-  class raw_ostream;
+class raw_ostream;
 
 /// Determine if the raw_ostream provided is connected to a terminal. If so,
 /// generate a warning message to errs() advising against display of bitcode
 /// and return true. Otherwise just return false.
 /// Check for output written to a console
 bool CheckBitcodeOutputToConsole(
-  raw_ostream &stream_to_check, ///< The stream to be checked
-  bool print_warning = true     ///< Control whether warnings are printed
+    raw_ostream &stream_to_check ///< The stream to be checked
 );
 
-} // End llvm namespace
+} // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/Support/ThreadPool.h b/llvm/include/llvm/Support/ThreadPool.h
index e9ab815d5717d..528fb32525eb2 100644
--- a/llvm/include/llvm/Support/ThreadPool.h
+++ b/llvm/include/llvm/Support/ThreadPool.h
@@ -13,7 +13,6 @@
 #ifndef LLVM_SUPPORT_THREAD_POOL_H
 #define LLVM_SUPPORT_THREAD_POOL_H
 
-#include "llvm/ADT/BitVector.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/Threading.h"
 #include "llvm/Support/thread.h"
diff --git a/llvm/include/llvm/Support/VersionTuple.h b/llvm/include/llvm/Support/VersionTuple.h
index ad89e40f0f144..6f3711f06f1a0 100644
--- a/llvm/include/llvm/Support/VersionTuple.h
+++ b/llvm/include/llvm/Support/VersionTuple.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_SUPPORT_VERSIONTUPLE_H
 #define LLVM_SUPPORT_VERSIONTUPLE_H
 
+#include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/Optional.h"
 #include <string>
 #include <tuple>
@@ -144,6 +145,10 @@ class VersionTuple {
     return !(X < Y);
   }
 
+  friend llvm::hash_code hash_value(const VersionTuple &VT) {
+    return llvm::hash_combine(VT.Major, VT.Minor, VT.Subminor, VT.Build);
+  }
+
   /// Retrieve a string representation of the version number.
   std::string getAsString() const;
 
diff --git a/llvm/include/llvm/Support/WithColor.h b/llvm/include/llvm/Support/WithColor.h
index ed1f9e5c3de50..eea4a72293392 100644
--- a/llvm/include/llvm/Support/WithColor.h
+++ b/llvm/include/llvm/Support/WithColor.h
@@ -9,14 +9,17 @@
 #ifndef LLVM_SUPPORT_WITHCOLOR_H
 #define LLVM_SUPPORT_WITHCOLOR_H
 
-#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
 
 class Error;
-class raw_ostream;
 class StringRef;
 
+namespace cl {
+class OptionCategory;
+}
+
 extern cl::OptionCategory ColorCategory;
 
 // Symbolic names for various syntax elements.
diff --git a/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h b/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h
index 8e5311054e24a..5697ff9a01dc1 100644
--- a/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h
+++ b/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h
@@ -361,6 +361,7 @@ enum ModRMDecisionType {
   ENUM_ENTRY(ENCODING_RM_CD16,"R/M operand with CDisp scaling of 16")          \
   ENUM_ENTRY(ENCODING_RM_CD32,"R/M operand with CDisp scaling of 32")          \
   ENUM_ENTRY(ENCODING_RM_CD64,"R/M operand with CDisp scaling of 64")          \
+  ENUM_ENTRY(ENCODING_SIB,      "Force SIB operand in ModR/M byte.")           \
   ENUM_ENTRY(ENCODING_VSIB,     "VSIB operand in ModR/M byte.")                \
   ENUM_ENTRY(ENCODING_VSIB_CD2, "VSIB operand with CDisp scaling of 2")        \
   ENUM_ENTRY(ENCODING_VSIB_CD4, "VSIB operand with CDisp scaling of 4")        \
@@ -411,6 +412,7 @@ enum OperandEncoding {
   ENUM_ENTRY(TYPE_IMM,        "immediate operand")                             \
   ENUM_ENTRY(TYPE_UIMM8,      "1-byte unsigned immediate operand")             \
   ENUM_ENTRY(TYPE_M,          "Memory operand")                                \
+  ENUM_ENTRY(TYPE_MSIB,       "Memory operand force sib encoding")             \
   ENUM_ENTRY(TYPE_MVSIBX,     "Memory operand using XMM index")                \
   ENUM_ENTRY(TYPE_MVSIBY,     "Memory operand using YMM index")                \
   ENUM_ENTRY(TYPE_MVSIBZ,     "Memory operand using ZMM index")                \
@@ -424,6 +426,7 @@ enum OperandEncoding {
   ENUM_ENTRY(TYPE_ZMM,        "64-byte")                                       \
   ENUM_ENTRY(TYPE_VK,         "mask register")                                 \
   ENUM_ENTRY(TYPE_VK_PAIR,    "mask register pair")                            \
+  ENUM_ENTRY(TYPE_TMM,        "tile")                                          \
   ENUM_ENTRY(TYPE_SEGMENTREG, "Segment register operand")                      \
   ENUM_ENTRY(TYPE_DEBUGREG,   "Debug register operand")                        \
   ENUM_ENTRY(TYPE_CONTROLREG, "Control register operand")                      \
diff --git a/llvm/include/llvm/Support/X86TargetParser.def b/llvm/include/llvm/Support/X86TargetParser.def
index 3c2614b33bf32..697f8c70f962d 100644
--- a/llvm/include/llvm/Support/X86TargetParser.def
+++ b/llvm/include/llvm/Support/X86TargetParser.def
@@ -20,402 +20,175 @@ X86_VENDOR(VENDOR_AMD,   "amd")
 #undef X86_VENDOR
 
 // This macro is used for cpu types present in compiler-rt/libgcc.
-#ifndef X86_CPU_TYPE_COMPAT
-#define X86_CPU_TYPE_COMPAT(ARCHNAME, ENUM, STR) X86_CPU_TYPE(ARCHNAME, ENUM)
-#endif
-
 #ifndef X86_CPU_TYPE
-#define X86_CPU_TYPE(ARCHNAME, ENUM)
+#define X86_CPU_TYPE(ENUM, STR)
 #endif
 
-#ifndef X86_CPU_TYPE_COMPAT_ALIAS
-#define X86_CPU_TYPE_COMPAT_ALIAS(ENUM, STR)
+#ifndef X86_CPU_TYPE_ALIAS
+#define X86_CPU_TYPE_ALIAS(ENUM, STR)
 #endif
 
-// The first part of this list must match what is implemented in libgcc and
-// compilert-rt. Clang uses this to know how to implement __builtin_cpu_is.
-X86_CPU_TYPE_COMPAT("bonnell",       INTEL_BONNELL,       "bonnell")
-X86_CPU_TYPE_COMPAT("core2",         INTEL_CORE2,         "core2")
-X86_CPU_TYPE_COMPAT("nehalem",       INTEL_COREI7,        "corei7")
-X86_CPU_TYPE_COMPAT("amdfam10",      AMDFAM10H,           "amdfam10h")
-X86_CPU_TYPE_COMPAT("bdver1",        AMDFAM15H,           "amdfam15h")
-X86_CPU_TYPE_COMPAT("silvermont",    INTEL_SILVERMONT,    "silvermont")
-X86_CPU_TYPE_COMPAT("knl",           INTEL_KNL,           "knl")
-X86_CPU_TYPE_COMPAT("btver1",        AMD_BTVER1,          "btver1")
-X86_CPU_TYPE_COMPAT("btver2",        AMD_BTVER2,          "btver2")
-X86_CPU_TYPE_COMPAT("znver1",        AMDFAM17H,           "amdfam17h")
-X86_CPU_TYPE_COMPAT("knm",           INTEL_KNM,           "knm")
-X86_CPU_TYPE_COMPAT("goldmont",      INTEL_GOLDMONT,      "goldmont")
-X86_CPU_TYPE_COMPAT("goldmont-plus", INTEL_GOLDMONT_PLUS, "goldmont-plus")
-X86_CPU_TYPE_COMPAT("tremont",       INTEL_TREMONT,       "tremont")
-// Entries below this are not in libgcc/compiler-rt.
-X86_CPU_TYPE       ("i386",          INTEL_i386)
-X86_CPU_TYPE       ("i486",          INTEL_i486)
-X86_CPU_TYPE       ("pentium",       INTEL_PENTIUM)
-X86_CPU_TYPE       ("pentium-mmx",   INTEL_PENTIUM_MMX)
-X86_CPU_TYPE       ("pentiumpro",    INTEL_PENTIUM_PRO)
-X86_CPU_TYPE       ("pentium2",      INTEL_PENTIUM_II)
-X86_CPU_TYPE       ("pentium3",      INTEL_PENTIUM_III)
-X86_CPU_TYPE       ("pentium4",      INTEL_PENTIUM_IV)
-X86_CPU_TYPE       ("pentium-m",     INTEL_PENTIUM_M)
-X86_CPU_TYPE       ("yonah",         INTEL_CORE_DUO)
-X86_CPU_TYPE       ("nocona",        INTEL_NOCONA)
-X86_CPU_TYPE       ("prescott",      INTEL_PRESCOTT)
-X86_CPU_TYPE       ("i486",          AMD_i486)
-X86_CPU_TYPE       ("pentium",       AMDPENTIUM)
-X86_CPU_TYPE       ("athlon",        AMD_ATHLON)
-X86_CPU_TYPE       ("athlon-xp",     AMD_ATHLON_XP)
-X86_CPU_TYPE       ("k8",            AMD_K8)
-X86_CPU_TYPE       ("k8-sse3",       AMD_K8SSE3)
+// This list must match what is implemented in libgcc and compilert-rt. Clang
+// uses this to know how to implement __builtin_cpu_is.
+X86_CPU_TYPE(INTEL_BONNELL,       "bonnell")
+X86_CPU_TYPE(INTEL_CORE2,         "core2")
+X86_CPU_TYPE(INTEL_COREI7,        "corei7")
+X86_CPU_TYPE(AMDFAM10H,           "amdfam10h")
+X86_CPU_TYPE(AMDFAM15H,           "amdfam15h")
+X86_CPU_TYPE(INTEL_SILVERMONT,    "silvermont")
+X86_CPU_TYPE(INTEL_KNL,           "knl")
+X86_CPU_TYPE(AMD_BTVER1,          "btver1")
+X86_CPU_TYPE(AMD_BTVER2,          "btver2")
+X86_CPU_TYPE(AMDFAM17H,           "amdfam17h")
+X86_CPU_TYPE(INTEL_KNM,           "knm")
+X86_CPU_TYPE(INTEL_GOLDMONT,      "goldmont")
+X86_CPU_TYPE(INTEL_GOLDMONT_PLUS, "goldmont-plus")
+X86_CPU_TYPE(INTEL_TREMONT,       "tremont")
 
 // Alternate names supported by __builtin_cpu_is and target multiversioning.
-X86_CPU_TYPE_COMPAT_ALIAS(INTEL_BONNELL,    "atom")
-X86_CPU_TYPE_COMPAT_ALIAS(AMDFAM10H,        "amdfam10")
-X86_CPU_TYPE_COMPAT_ALIAS(AMDFAM15H,        "amdfam15")
-X86_CPU_TYPE_COMPAT_ALIAS(INTEL_SILVERMONT, "slm")
+X86_CPU_TYPE_ALIAS(INTEL_BONNELL,    "atom")
+X86_CPU_TYPE_ALIAS(AMDFAM10H,        "amdfam10")
+X86_CPU_TYPE_ALIAS(AMDFAM15H,        "amdfam15")
+X86_CPU_TYPE_ALIAS(INTEL_SILVERMONT, "slm")
 
-#undef X86_CPU_TYPE_COMPAT_ALIAS
-#undef X86_CPU_TYPE_COMPAT
+#undef X86_CPU_TYPE_ALIAS
 #undef X86_CPU_TYPE
 
 // This macro is used for cpu subtypes present in compiler-rt/libgcc.
-#ifndef X86_CPU_SUBTYPE_COMPAT
-#define X86_CPU_SUBTYPE_COMPAT(ARCHNAME, ENUM, STR) X86_CPU_SUBTYPE(ARCHNAME, ENUM)
-#endif
-
 #ifndef X86_CPU_SUBTYPE
-#define X86_CPU_SUBTYPE(ARCHNAME, ENUM)
+#define X86_CPU_SUBTYPE(ENUM, STR)
 #endif
 
-// The first part of this list must match what is implemented in libgcc and
-// compilert-rt. Clang uses this to know how to implement __builtin_cpu_is.
-X86_CPU_SUBTYPE_COMPAT("nehalem",        INTEL_COREI7_NEHALEM,        "nehalem")
-X86_CPU_SUBTYPE_COMPAT("westmere",       INTEL_COREI7_WESTMERE,       "westmere")
-X86_CPU_SUBTYPE_COMPAT("sandybridge",    INTEL_COREI7_SANDYBRIDGE,    "sandybridge")
-X86_CPU_SUBTYPE_COMPAT("amdfam10",       AMDFAM10H_BARCELONA,         "barcelona")
-X86_CPU_SUBTYPE_COMPAT("amdfam10",       AMDFAM10H_SHANGHAI,          "shanghai")
-X86_CPU_SUBTYPE_COMPAT("amdfam10",       AMDFAM10H_ISTANBUL,          "istanbul")
-X86_CPU_SUBTYPE_COMPAT("bdver1",         AMDFAM15H_BDVER1,            "bdver1")
-X86_CPU_SUBTYPE_COMPAT("bdver2",         AMDFAM15H_BDVER2,            "bdver2")
-X86_CPU_SUBTYPE_COMPAT("bdver3",         AMDFAM15H_BDVER3,            "bdver3")
-X86_CPU_SUBTYPE_COMPAT("bdver4",         AMDFAM15H_BDVER4,            "bdver4")
-X86_CPU_SUBTYPE_COMPAT("znver1",         AMDFAM17H_ZNVER1,            "znver1")
-X86_CPU_SUBTYPE_COMPAT("ivybridge",      INTEL_COREI7_IVYBRIDGE,      "ivybridge")
-X86_CPU_SUBTYPE_COMPAT("haswell",        INTEL_COREI7_HASWELL,        "haswell")
-X86_CPU_SUBTYPE_COMPAT("broadwell",      INTEL_COREI7_BROADWELL,      "broadwell")
-X86_CPU_SUBTYPE_COMPAT("skylake",        INTEL_COREI7_SKYLAKE,        "skylake")
-X86_CPU_SUBTYPE_COMPAT("skylake-avx512", INTEL_COREI7_SKYLAKE_AVX512, "skylake-avx512")
-X86_CPU_SUBTYPE_COMPAT("cannonlake",     INTEL_COREI7_CANNONLAKE,     "cannonlake")
-X86_CPU_SUBTYPE_COMPAT("icelake-client", INTEL_COREI7_ICELAKE_CLIENT, "icelake-client")
-X86_CPU_SUBTYPE_COMPAT("icelake-server", INTEL_COREI7_ICELAKE_SERVER, "icelake-server")
-X86_CPU_SUBTYPE_COMPAT("znver2",         AMDFAM17H_ZNVER2,            "znver2")
-X86_CPU_SUBTYPE_COMPAT("cascadelake",    INTEL_COREI7_CASCADELAKE,    "cascadelake")
-X86_CPU_SUBTYPE_COMPAT("tigerlake",      INTEL_COREI7_TIGERLAKE,      "tigerlake")
-X86_CPU_SUBTYPE_COMPAT("cooperlake",     INTEL_COREI7_COOPERLAKE,     "cooperlake")
-// Entries below this are not in libgcc/compiler-rt.
-X86_CPU_SUBTYPE       ("core2",          INTEL_CORE2_65)
-X86_CPU_SUBTYPE       ("penryn",         INTEL_CORE2_45)
-X86_CPU_SUBTYPE       ("k6",             AMDPENTIUM_K6)
-X86_CPU_SUBTYPE       ("k6-2",           AMDPENTIUM_K62)
-X86_CPU_SUBTYPE       ("k6-3",           AMDPENTIUM_K63)
-X86_CPU_SUBTYPE       ("geode",          AMDPENTIUM_GEODE)
-#undef X86_CPU_SUBTYPE_COMPAT
+// This list must match what is implemented in libgcc and compilert-rt. Clang
+// uses this to know how to implement __builtin_cpu_is.
+X86_CPU_SUBTYPE(INTEL_COREI7_NEHALEM,        "nehalem")
+X86_CPU_SUBTYPE(INTEL_COREI7_WESTMERE,       "westmere")
+X86_CPU_SUBTYPE(INTEL_COREI7_SANDYBRIDGE,    "sandybridge")
+X86_CPU_SUBTYPE(AMDFAM10H_BARCELONA,         "barcelona")
+X86_CPU_SUBTYPE(AMDFAM10H_SHANGHAI,          "shanghai")
+X86_CPU_SUBTYPE(AMDFAM10H_ISTANBUL,          "istanbul")
+X86_CPU_SUBTYPE(AMDFAM15H_BDVER1,            "bdver1")
+X86_CPU_SUBTYPE(AMDFAM15H_BDVER2,            "bdver2")
+X86_CPU_SUBTYPE(AMDFAM15H_BDVER3,            "bdver3")
+X86_CPU_SUBTYPE(AMDFAM15H_BDVER4,            "bdver4")
+X86_CPU_SUBTYPE(AMDFAM17H_ZNVER1,            "znver1")
+X86_CPU_SUBTYPE(INTEL_COREI7_IVYBRIDGE,      "ivybridge")
+X86_CPU_SUBTYPE(INTEL_COREI7_HASWELL,        "haswell")
+X86_CPU_SUBTYPE(INTEL_COREI7_BROADWELL,      "broadwell")
+X86_CPU_SUBTYPE(INTEL_COREI7_SKYLAKE,        "skylake")
+X86_CPU_SUBTYPE(INTEL_COREI7_SKYLAKE_AVX512, "skylake-avx512")
+X86_CPU_SUBTYPE(INTEL_COREI7_CANNONLAKE,     "cannonlake")
+X86_CPU_SUBTYPE(INTEL_COREI7_ICELAKE_CLIENT, "icelake-client")
+X86_CPU_SUBTYPE(INTEL_COREI7_ICELAKE_SERVER, "icelake-server")
+X86_CPU_SUBTYPE(AMDFAM17H_ZNVER2,            "znver2")
+X86_CPU_SUBTYPE(INTEL_COREI7_CASCADELAKE,    "cascadelake")
+X86_CPU_SUBTYPE(INTEL_COREI7_TIGERLAKE,      "tigerlake")
+X86_CPU_SUBTYPE(INTEL_COREI7_COOPERLAKE,     "cooperlake")
 #undef X86_CPU_SUBTYPE
 
 
 // This macro is used for cpu types present in compiler-rt/libgcc.
 #ifndef X86_FEATURE_COMPAT
-#define X86_FEATURE_COMPAT(VAL, ENUM, STR) X86_FEATURE(VAL, ENUM)
+#define X86_FEATURE_COMPAT(ENUM, STR) X86_FEATURE(ENUM, STR)
 #endif
 
 #ifndef X86_FEATURE
-#define X86_FEATURE(VAL, ENUM)
+#define X86_FEATURE(ENUM, STR)
 #endif
-X86_FEATURE_COMPAT( 0, FEATURE_CMOV,            "cmov")
-X86_FEATURE_COMPAT( 1, FEATURE_MMX,             "mmx")
-X86_FEATURE_COMPAT( 2, FEATURE_POPCNT,          "popcnt")
-X86_FEATURE_COMPAT( 3, FEATURE_SSE,             "sse")
-X86_FEATURE_COMPAT( 4, FEATURE_SSE2,            "sse2")
-X86_FEATURE_COMPAT( 5, FEATURE_SSE3,            "sse3")
-X86_FEATURE_COMPAT( 6, FEATURE_SSSE3,           "ssse3")
-X86_FEATURE_COMPAT( 7, FEATURE_SSE4_1,          "sse4.1")
-X86_FEATURE_COMPAT( 8, FEATURE_SSE4_2,          "sse4.2")
-X86_FEATURE_COMPAT( 9, FEATURE_AVX,             "avx")
-X86_FEATURE_COMPAT(10, FEATURE_AVX2,            "avx2")
-X86_FEATURE_COMPAT(11, FEATURE_SSE4_A,          "sse4a")
-X86_FEATURE_COMPAT(12, FEATURE_FMA4,            "fma4")
-X86_FEATURE_COMPAT(13, FEATURE_XOP,             "xop")
-X86_FEATURE_COMPAT(14, FEATURE_FMA,             "fma")
-X86_FEATURE_COMPAT(15, FEATURE_AVX512F,         "avx512f")
-X86_FEATURE_COMPAT(16, FEATURE_BMI,             "bmi")
-X86_FEATURE_COMPAT(17, FEATURE_BMI2,            "bmi2")
-X86_FEATURE_COMPAT(18, FEATURE_AES,             "aes")
-X86_FEATURE_COMPAT(19, FEATURE_PCLMUL,          "pclmul")
-X86_FEATURE_COMPAT(20, FEATURE_AVX512VL,        "avx512vl")
-X86_FEATURE_COMPAT(21, FEATURE_AVX512BW,        "avx512bw")
-X86_FEATURE_COMPAT(22, FEATURE_AVX512DQ,        "avx512dq")
-X86_FEATURE_COMPAT(23, FEATURE_AVX512CD,        "avx512cd")
-X86_FEATURE_COMPAT(24, FEATURE_AVX512ER,        "avx512er")
-X86_FEATURE_COMPAT(25, FEATURE_AVX512PF,        "avx512pf")
-X86_FEATURE_COMPAT(26, FEATURE_AVX512VBMI,      "avx512vbmi")
-X86_FEATURE_COMPAT(27, FEATURE_AVX512IFMA,      "avx512ifma")
-X86_FEATURE_COMPAT(28, FEATURE_AVX5124VNNIW,    "avx5124vnniw")
-X86_FEATURE_COMPAT(29, FEATURE_AVX5124FMAPS,    "avx5124fmaps")
-X86_FEATURE_COMPAT(30, FEATURE_AVX512VPOPCNTDQ, "avx512vpopcntdq")
-X86_FEATURE_COMPAT(31, FEATURE_AVX512VBMI2,     "avx512vbmi2")
-X86_FEATURE_COMPAT(32, FEATURE_GFNI,            "gfni")
-X86_FEATURE_COMPAT(33, FEATURE_VPCLMULQDQ,      "vpclmulqdq")
-X86_FEATURE_COMPAT(34, FEATURE_AVX512VNNI,      "avx512vnni")
-X86_FEATURE_COMPAT(35, FEATURE_AVX512BITALG,    "avx512bitalg")
-X86_FEATURE_COMPAT(36, FEATURE_AVX512BF16,      "avx512bf16")
-X86_FEATURE_COMPAT(37, FEATURE_AVX512VP2INTERSECT, "avx512vp2intersect")
+
+X86_FEATURE_COMPAT(CMOV,            "cmov")
+X86_FEATURE_COMPAT(MMX,             "mmx")
+X86_FEATURE_COMPAT(POPCNT,          "popcnt")
+X86_FEATURE_COMPAT(SSE,             "sse")
+X86_FEATURE_COMPAT(SSE2,            "sse2")
+X86_FEATURE_COMPAT(SSE3,            "sse3")
+X86_FEATURE_COMPAT(SSSE3,           "ssse3")
+X86_FEATURE_COMPAT(SSE4_1,          "sse4.1")
+X86_FEATURE_COMPAT(SSE4_2,          "sse4.2")
+X86_FEATURE_COMPAT(AVX,             "avx")
+X86_FEATURE_COMPAT(AVX2,            "avx2")
+X86_FEATURE_COMPAT(SSE4_A,          "sse4a")
+X86_FEATURE_COMPAT(FMA4,            "fma4")
+X86_FEATURE_COMPAT(XOP,             "xop")
+X86_FEATURE_COMPAT(FMA,             "fma")
+X86_FEATURE_COMPAT(AVX512F,         "avx512f")
+X86_FEATURE_COMPAT(BMI,             "bmi")
+X86_FEATURE_COMPAT(BMI2,            "bmi2")
+X86_FEATURE_COMPAT(AES,             "aes")
+X86_FEATURE_COMPAT(PCLMUL,          "pclmul")
+X86_FEATURE_COMPAT(AVX512VL,        "avx512vl")
+X86_FEATURE_COMPAT(AVX512BW,        "avx512bw")
+X86_FEATURE_COMPAT(AVX512DQ,        "avx512dq")
+X86_FEATURE_COMPAT(AVX512CD,        "avx512cd")
+X86_FEATURE_COMPAT(AVX512ER,        "avx512er")
+X86_FEATURE_COMPAT(AVX512PF,        "avx512pf")
+X86_FEATURE_COMPAT(AVX512VBMI,      "avx512vbmi")
+X86_FEATURE_COMPAT(AVX512IFMA,      "avx512ifma")
+X86_FEATURE_COMPAT(AVX5124VNNIW,    "avx5124vnniw")
+X86_FEATURE_COMPAT(AVX5124FMAPS,    "avx5124fmaps")
+X86_FEATURE_COMPAT(AVX512VPOPCNTDQ, "avx512vpopcntdq")
+X86_FEATURE_COMPAT(AVX512VBMI2,     "avx512vbmi2")
+X86_FEATURE_COMPAT(GFNI,            "gfni")
+X86_FEATURE_COMPAT(VPCLMULQDQ,      "vpclmulqdq")
+X86_FEATURE_COMPAT(AVX512VNNI,      "avx512vnni")
+X86_FEATURE_COMPAT(AVX512BITALG,    "avx512bitalg")
+X86_FEATURE_COMPAT(AVX512BF16,      "avx512bf16")
+X86_FEATURE_COMPAT(AVX512VP2INTERSECT, "avx512vp2intersect")
 // Features below here are not in libgcc/compiler-rt.
-X86_FEATURE       (64, FEATURE_MOVBE)
-X86_FEATURE       (65, FEATURE_ADX)
-X86_FEATURE       (66, FEATURE_EM64T)
-X86_FEATURE       (67, FEATURE_CLFLUSHOPT)
-X86_FEATURE       (68, FEATURE_SHA)
+X86_FEATURE       (3DNOW,           "3dnow")
+X86_FEATURE       (3DNOWA,          "3dnowa")
+X86_FEATURE       (64BIT,           "64bit")
+X86_FEATURE       (ADX,             "adx")
+X86_FEATURE       (AMX_BF16,        "amx-bf16")
+X86_FEATURE       (AMX_INT8,        "amx-int8")
+X86_FEATURE       (AMX_TILE,        "amx-tile")
+X86_FEATURE       (CLDEMOTE,        "cldemote")
+X86_FEATURE       (CLFLUSHOPT,      "clflushopt")
+X86_FEATURE       (CLWB,            "clwb")
+X86_FEATURE       (CLZERO,          "clzero")
+X86_FEATURE       (CMPXCHG16B,      "cx16")
+X86_FEATURE       (CMPXCHG8B,       "cx8")
+X86_FEATURE       (ENQCMD,          "enqcmd")
+X86_FEATURE       (F16C,            "f16c")
+X86_FEATURE       (FSGSBASE,        "fsgsbase")
+X86_FEATURE       (FXSR,            "fxsr")
+X86_FEATURE       (INVPCID,         "invpcid")
+X86_FEATURE       (LWP,             "lwp")
+X86_FEATURE       (LZCNT,           "lzcnt")
+X86_FEATURE       (MOVBE,           "movbe")
+X86_FEATURE       (MOVDIR64B,       "movdir64b")
+X86_FEATURE       (MOVDIRI,         "movdiri")
+X86_FEATURE       (MWAITX,          "mwaitx")
+X86_FEATURE       (PCONFIG,         "pconfig")
+X86_FEATURE       (PKU,             "pku")
+X86_FEATURE       (PREFETCHWT1,     "prefetchwt1")
+X86_FEATURE       (PRFCHW,          "prfchw")
+X86_FEATURE       (PTWRITE,         "ptwrite")
+X86_FEATURE       (RDPID,           "rdpid")
+X86_FEATURE       (RDRND,           "rdrnd")
+X86_FEATURE       (RDSEED,          "rdseed")
+X86_FEATURE       (RTM,             "rtm")
+X86_FEATURE       (SAHF,            "sahf")
+X86_FEATURE       (SERIALIZE,       "serialize")
+X86_FEATURE       (SGX,             "sgx")
+X86_FEATURE       (SHA,             "sha")
+X86_FEATURE       (SHSTK,           "shstk")
+X86_FEATURE       (TBM,             "tbm")
+X86_FEATURE       (TSXLDTRK,        "tsxldtrk")
+X86_FEATURE       (VAES,            "vaes")
+X86_FEATURE       (VZEROUPPER,      "vzeroupper")
+X86_FEATURE       (WAITPKG,         "waitpkg")
+X86_FEATURE       (WBNOINVD,        "wbnoinvd")
+X86_FEATURE       (X87,             "x87")
+X86_FEATURE       (XSAVE,           "xsave")
+X86_FEATURE       (XSAVEC,          "xsavec")
+X86_FEATURE       (XSAVEOPT,        "xsaveopt")
+X86_FEATURE       (XSAVES,          "xsaves")
+// These features aren't really CPU features, but the frontend can set them.
+X86_FEATURE       (RETPOLINE_EXTERNAL_THUNK,    "retpoline-external-thunk")
+X86_FEATURE       (RETPOLINE_INDIRECT_BRANCHES, "retpoline-indirect-branches")
+X86_FEATURE       (RETPOLINE_INDIRECT_CALLS,    "retpoline-indirect-calls")
+X86_FEATURE       (LVI_CFI,                     "lvi-cfi")
+X86_FEATURE       (LVI_LOAD_HARDENING,          "lvi-load-hardening")
 #undef X86_FEATURE_COMPAT
 #undef X86_FEATURE
-
-
-#ifndef PROC_WITH_FEAT
-#define PROC_WITH_FEAT(ENUM, STRING, IS64BIT, KEYFEATURE)                      \
-  PROC(ENUM, STRING, IS64BIT)
-#endif
-
-#ifndef PROC
-#define PROC(ENUM, STRING, IS64BIT)
-#endif
-
-#ifndef PROC_ALIAS
-#define PROC_ALIAS(ENUM, ALIAS)
-#endif
-
-#define PROC_64_BIT true
-#define PROC_32_BIT false
-
-/// \name i386
-/// i386-generation processors.
-//@{
-PROC(i386, "i386", PROC_32_BIT)
-//@}
-
-/// \name i486
-/// i486-generation processors.
-//@{
-PROC(i486, "i486", PROC_32_BIT)
-PROC(WinChipC6, "winchip-c6", PROC_32_BIT)
-PROC(WinChip2, "winchip2", PROC_32_BIT)
-PROC(C3, "c3", PROC_32_BIT)
-//@}
-
-/// \name i586
-/// i586-generation processors, P5 microarchitecture based.
-//@{
-PROC(i586, "i586", PROC_32_BIT)
-PROC(Pentium, "pentium", PROC_32_BIT)
-PROC(PentiumMMX, "pentium-mmx", PROC_32_BIT)
-//@}
-
-/// \name i686
-/// i686-generation processors, P6 / Pentium M microarchitecture based.
-//@{
-PROC(PentiumPro, "pentiumpro", PROC_32_BIT)
-PROC(i686, "i686", PROC_32_BIT)
-PROC(Pentium2, "pentium2", PROC_32_BIT)
-PROC(Pentium3, "pentium3", PROC_32_BIT)
-PROC_ALIAS(Pentium3, "pentium3m")
-PROC(PentiumM, "pentium-m", PROC_32_BIT)
-PROC(C3_2, "c3-2", PROC_32_BIT)
-
-/// This enumerator is a bit odd, as GCC no longer accepts -march=yonah.
-/// Clang however has some logic to support this.
-// FIXME: Warn, deprecate, and potentially remove this.
-PROC(Yonah, "yonah", PROC_32_BIT)
-//@}
-
-/// \name Netburst
-/// Netburst microarchitecture based processors.
-//@{
-PROC(Pentium4, "pentium4", PROC_32_BIT)
-PROC_ALIAS(Pentium4, "pentium4m")
-
-PROC(Prescott, "prescott", PROC_32_BIT)
-PROC(Nocona, "nocona", PROC_64_BIT)
-//@}
-
-/// \name Core
-/// Core microarchitecture based processors.
-//@{
-PROC_WITH_FEAT(Core2, "core2", PROC_64_BIT, FEATURE_SSSE3)
-
-/// This enumerator, like Yonah, is a bit odd. It is another
-/// codename which GCC no longer accepts as an option to -march, but Clang
-/// has some logic for recognizing it.
-// FIXME: Warn, deprecate, and potentially remove this.
-PROC(Penryn, "penryn", PROC_64_BIT)
-//@}
-
-/// \name Atom
-/// Atom processors
-//@{
-PROC_WITH_FEAT(Bonnell, "bonnell", PROC_64_BIT, FEATURE_SSSE3)
-PROC_ALIAS(Bonnell, "atom")
-
-PROC_WITH_FEAT(Silvermont, "silvermont", PROC_64_BIT, FEATURE_SSE4_2)
-PROC_ALIAS(Silvermont, "slm")
-
-PROC_WITH_FEAT(Goldmont, "goldmont", PROC_64_BIT, FEATURE_SSE4_2)
-PROC_WITH_FEAT(GoldmontPlus, "goldmont-plus", PROC_64_BIT, FEATURE_SSE4_2)
-
-PROC_WITH_FEAT(Tremont, "tremont", PROC_64_BIT, FEATURE_SSE4_2)
-//@}
-
-/// \name Nehalem
-/// Nehalem microarchitecture based processors.
-PROC_WITH_FEAT(Nehalem, "nehalem", PROC_64_BIT, FEATURE_SSE4_2)
-PROC_ALIAS(Nehalem, "corei7")
-
-/// \name Westmere
-/// Westmere microarchitecture based processors.
-PROC_WITH_FEAT(Westmere, "westmere", PROC_64_BIT, FEATURE_PCLMUL)
-
-/// \name Sandy Bridge
-/// Sandy Bridge microarchitecture based processors.
-PROC_WITH_FEAT(SandyBridge, "sandybridge", PROC_64_BIT, FEATURE_AVX)
-PROC_ALIAS(SandyBridge, "corei7-avx")
-
-/// \name Ivy Bridge
-/// Ivy Bridge microarchitecture based processors.
-PROC_WITH_FEAT(IvyBridge, "ivybridge", PROC_64_BIT, FEATURE_AVX)
-PROC_ALIAS(IvyBridge, "core-avx-i")
-
-/// \name Haswell
-/// Haswell microarchitecture based processors.
-PROC_WITH_FEAT(Haswell, "haswell", PROC_64_BIT, FEATURE_AVX2)
-PROC_ALIAS(Haswell, "core-avx2")
-
-/// \name Broadwell
-/// Broadwell microarchitecture based processors.
-PROC_WITH_FEAT(Broadwell, "broadwell", PROC_64_BIT, FEATURE_AVX2)
-
-/// \name Skylake Client
-/// Skylake client microarchitecture based processors.
-PROC_WITH_FEAT(SkylakeClient, "skylake", PROC_64_BIT, FEATURE_AVX2)
-
-/// \name Skylake Server
-/// Skylake server microarchitecture based processors.
-PROC_WITH_FEAT(SkylakeServer, "skylake-avx512", PROC_64_BIT, FEATURE_AVX512F)
-PROC_ALIAS(SkylakeServer, "skx")
-
-/// \name Cascadelake Server
-/// Cascadelake Server microarchitecture based processors.
-PROC_WITH_FEAT(Cascadelake, "cascadelake", PROC_64_BIT, FEATURE_AVX512VNNI)
-
-/// \name Cooperlake Server
-/// Cooperlake Server microarchitecture based processors.
-PROC_WITH_FEAT(Cooperlake, "cooperlake", PROC_64_BIT, FEATURE_AVX512BF16)
-
-/// \name Cannonlake Client
-/// Cannonlake client microarchitecture based processors.
-PROC_WITH_FEAT(Cannonlake, "cannonlake", PROC_64_BIT, FEATURE_AVX512VBMI)
-
-/// \name Icelake Client
-/// Icelake client microarchitecture based processors.
-PROC_WITH_FEAT(IcelakeClient, "icelake-client", PROC_64_BIT, FEATURE_AVX512VBMI2)
-
-/// \name Icelake Server
-/// Icelake server microarchitecture based processors.
-PROC_WITH_FEAT(IcelakeServer, "icelake-server", PROC_64_BIT, FEATURE_AVX512VBMI2)
-
-/// \name Tigerlake
-/// Tigerlake microarchitecture based processors.
-PROC_WITH_FEAT(Tigerlake, "tigerlake", PROC_64_BIT, FEATURE_AVX512VP2INTERSECT)
-
-/// \name Knights Landing
-/// Knights Landing processor.
-PROC_WITH_FEAT(KNL, "knl", PROC_64_BIT, FEATURE_AVX512F)
-
-/// \name Knights Mill
-/// Knights Mill processor.
-PROC_WITH_FEAT(KNM, "knm", PROC_64_BIT, FEATURE_AVX5124FMAPS)
-
-/// \name Lakemont
-/// Lakemont microarchitecture based processors.
-PROC(Lakemont, "lakemont", PROC_32_BIT)
-
-/// \name K6
-/// K6 architecture processors.
-//@{
-PROC(K6, "k6", PROC_32_BIT)
-PROC(K6_2, "k6-2", PROC_32_BIT)
-PROC(K6_3, "k6-3", PROC_32_BIT)
-//@}
-
-/// \name K7
-/// K7 architecture processors.
-//@{
-PROC(Athlon, "athlon", PROC_32_BIT)
-PROC_ALIAS(Athlon, "athlon-tbird")
-
-PROC(AthlonXP, "athlon-xp", PROC_32_BIT)
-PROC_ALIAS(AthlonXP, "athlon-mp")
-PROC_ALIAS(AthlonXP, "athlon-4")
-//@}
-
-/// \name K8
-/// K8 architecture processors.
-//@{
-PROC(K8, "k8", PROC_64_BIT)
-PROC_ALIAS(K8, "athlon64")
-PROC_ALIAS(K8, "athlon-fx")
-PROC_ALIAS(K8, "opteron")
-
-PROC(K8SSE3, "k8-sse3", PROC_64_BIT)
-PROC_ALIAS(K8SSE3, "athlon64-sse3")
-PROC_ALIAS(K8SSE3, "opteron-sse3")
-
-PROC_WITH_FEAT(AMDFAM10, "amdfam10", PROC_64_BIT, FEATURE_SSE4_A)
-PROC_ALIAS(AMDFAM10, "barcelona")
-//@}
-
-/// \name Bobcat
-/// Bobcat architecture processors.
-//@{
-PROC_WITH_FEAT(BTVER1, "btver1", PROC_64_BIT, FEATURE_SSE4_A)
-PROC_WITH_FEAT(BTVER2, "btver2", PROC_64_BIT, FEATURE_BMI)
-//@}
-
-/// \name Bulldozer
-/// Bulldozer architecture processors.
-//@{
-PROC_WITH_FEAT(BDVER1, "bdver1", PROC_64_BIT, FEATURE_XOP)
-PROC_WITH_FEAT(BDVER2, "bdver2", PROC_64_BIT, FEATURE_FMA)
-PROC_WITH_FEAT(BDVER3, "bdver3", PROC_64_BIT, FEATURE_FMA)
-PROC_WITH_FEAT(BDVER4, "bdver4", PROC_64_BIT, FEATURE_AVX2)
-//@}
-
-/// \name zen
-/// Zen architecture processors.
-//@{
-PROC_WITH_FEAT(ZNVER1, "znver1", PROC_64_BIT, FEATURE_AVX2)
-PROC_WITH_FEAT(ZNVER2, "znver2", PROC_64_BIT, FEATURE_AVX2)
-//@}
-
-/// This specification is deprecated and will be removed in the future.
-/// Users should prefer K8.
-// FIXME: Warn on this when the CPU is set to it.
-//@{
-PROC(x86_64, "x86-64", PROC_64_BIT)
-//@}
-
-/// \name Geode
-/// Geode processors.
-//@{
-PROC(Geode, "geode", PROC_32_BIT)
-//@}
-
-#undef PROC
-#undef PROC_ALIAS
-#undef PROC_WITH_FEAT
diff --git a/llvm/include/llvm/Support/X86TargetParser.h b/llvm/include/llvm/Support/X86TargetParser.h
index 7cdf1c5328146..66c474b5c2750 100644
--- a/llvm/include/llvm/Support/X86TargetParser.h
+++ b/llvm/include/llvm/Support/X86TargetParser.h
@@ -34,7 +34,7 @@ enum ProcessorVendors : unsigned {
 // as a proxy for what's in libgcc/compiler-rt.
 enum ProcessorTypes : unsigned {
   CPU_TYPE_DUMMY,
-#define X86_CPU_TYPE(ARCHNAME, ENUM) \
+#define X86_CPU_TYPE(ENUM, STRING) \
   ENUM,
 #include "llvm/Support/X86TargetParser.def"
   CPU_TYPE_MAX
@@ -44,7 +44,7 @@ enum ProcessorTypes : unsigned {
 // as a proxy for what's in libgcc/compiler-rt.
 enum ProcessorSubtypes : unsigned {
   CPU_SUBTYPE_DUMMY,
-#define X86_CPU_SUBTYPE(ARCHNAME, ENUM) \
+#define X86_CPU_SUBTYPE(ENUM, STRING) \
   ENUM,
 #include "llvm/Support/X86TargetParser.def"
   CPU_SUBTYPE_MAX
@@ -53,16 +53,73 @@ enum ProcessorSubtypes : unsigned {
 // This should be kept in sync with libcc/compiler-rt as it should be used
 // by clang as a proxy for what's in libgcc/compiler-rt.
 enum ProcessorFeatures {
-#define X86_FEATURE(VAL, ENUM) \
-  ENUM = VAL,
+#define X86_FEATURE(ENUM, STRING) FEATURE_##ENUM,
 #include "llvm/Support/X86TargetParser.def"
-
+  CPU_FEATURE_MAX
 };
 
 enum CPUKind {
   CK_None,
-#define PROC(ENUM, STRING, IS64BIT) CK_##ENUM,
-#include "llvm/Support/X86TargetParser.def"
+  CK_i386,
+  CK_i486,
+  CK_WinChipC6,
+  CK_WinChip2,
+  CK_C3,
+  CK_i586,
+  CK_Pentium,
+  CK_PentiumMMX,
+  CK_PentiumPro,
+  CK_i686,
+  CK_Pentium2,
+  CK_Pentium3,
+  CK_PentiumM,
+  CK_C3_2,
+  CK_Yonah,
+  CK_Pentium4,
+  CK_Prescott,
+  CK_Nocona,
+  CK_Core2,
+  CK_Penryn,
+  CK_Bonnell,
+  CK_Silvermont,
+  CK_Goldmont,
+  CK_GoldmontPlus,
+  CK_Tremont,
+  CK_Nehalem,
+  CK_Westmere,
+  CK_SandyBridge,
+  CK_IvyBridge,
+  CK_Haswell,
+  CK_Broadwell,
+  CK_SkylakeClient,
+  CK_SkylakeServer,
+  CK_Cascadelake,
+  CK_Cooperlake,
+  CK_Cannonlake,
+  CK_IcelakeClient,
+  CK_IcelakeServer,
+  CK_Tigerlake,
+  CK_KNL,
+  CK_KNM,
+  CK_Lakemont,
+  CK_K6,
+  CK_K6_2,
+  CK_K6_3,
+  CK_Athlon,
+  CK_AthlonXP,
+  CK_K8,
+  CK_K8SSE3,
+  CK_AMDFAM10,
+  CK_BTVER1,
+  CK_BTVER2,
+  CK_BDVER1,
+  CK_BDVER2,
+  CK_BDVER3,
+  CK_BDVER4,
+  CK_ZNVER1,
+  CK_ZNVER2,
+  CK_x86_64,
+  CK_Geode,
 };
 
 /// Parse \p CPU string into a CPUKind. Will only accept 64-bit capable CPUs if
@@ -74,6 +131,17 @@ CPUKind parseArchX86(StringRef CPU, bool Only64Bit = false);
 void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values,
                           bool ArchIs32Bit);
 
+/// Get the key feature prioritizing target multiversioning.
+ProcessorFeatures getKeyFeature(CPUKind Kind);
+
+/// Fill in the features that \p CPU supports into \p Features.
+void getFeaturesForCPU(StringRef CPU, SmallVectorImpl<StringRef> &Features);
+
+/// Fill \p Features with the features that are implied to be enabled/disabled
+/// by the provided \p Feature.
+void getImpliedFeatures(StringRef Feature, bool Enabled,
+                        SmallVectorImpl<StringRef> &Features);
+
 } // namespace X86
 } // namespace llvm
 
diff --git a/llvm/include/llvm/Support/YAMLTraits.h b/llvm/include/llvm/Support/YAMLTraits.h
index f93f36037679a..44e34a4a09b46 100644
--- a/llvm/include/llvm/Support/YAMLTraits.h
+++ b/llvm/include/llvm/Support/YAMLTraits.h
@@ -649,24 +649,25 @@ inline bool isBool(StringRef S) {
 inline QuotingType needsQuotes(StringRef S) {
   if (S.empty())
     return QuotingType::Single;
+
+  QuotingType MaxQuotingNeeded = QuotingType::None;
   if (isSpace(static_cast<unsigned char>(S.front())) ||
       isSpace(static_cast<unsigned char>(S.back())))
-    return QuotingType::Single;
+    MaxQuotingNeeded = QuotingType::Single;
   if (isNull(S))
-    return QuotingType::Single;
+    MaxQuotingNeeded = QuotingType::Single;
   if (isBool(S))
-    return QuotingType::Single;
+    MaxQuotingNeeded = QuotingType::Single;
   if (isNumeric(S))
-    return QuotingType::Single;
+    MaxQuotingNeeded = QuotingType::Single;
 
   // 7.3.3 Plain Style
   // Plain scalars must not begin with most indicators, as this would cause
   // ambiguity with other YAML constructs.
   static constexpr char Indicators[] = R"(-?:\,[]{}#&*!|>'"%@`)";
   if (S.find_first_of(Indicators) == 0)
-    return QuotingType::Single;
+    MaxQuotingNeeded = QuotingType::Single;
 
-  QuotingType MaxQuotingNeeded = QuotingType::None;
   for (unsigned char C : S) {
     // Alphanum is safe.
     if (isAlnum(C))
@@ -684,11 +685,11 @@ inline QuotingType needsQuotes(StringRef S) {
     case 0x9:
       continue;
     // LF(0xA) and CR(0xD) may delimit values and so require at least single
-    // quotes.
+    // quotes. LLVM YAML parser cannot handle single quoted multiline so use
+    // double quoting to produce valid YAML.
     case 0xA:
     case 0xD:
-      MaxQuotingNeeded = QuotingType::Single;
-      continue;
+      return QuotingType::Double;
     // DEL (0x7F) are excluded from the allowed character range.
     case 0x7F:
       return QuotingType::Double;
diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td
index c628fa8b625f8..aab5376db4535 100644
--- a/llvm/include/llvm/Target/Target.td
+++ b/llvm/include/llvm/Target/Target.td
@@ -1017,10 +1017,10 @@ def INLINEASM_BR : StandardPseudoInstruction {
   let OutOperandList = (outs);
   let InOperandList = (ins variable_ops);
   let AsmString = "";
-  let hasSideEffects = 0;  // Note side effect is encoded in an operand.
-  let isTerminator = 1;
-  let isBranch = 1;
-  let isIndirectBranch = 1;
+  // Unlike INLINEASM, this is always treated as having side-effects.
+  let hasSideEffects = 1;
+  // Despite potentially branching, this instruction is intentionally _not_
+  // marked as a terminator or a branch.
 }
 def CFI_INSTRUCTION : StandardPseudoInstruction {
   let OutOperandList = (outs);
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 1f5cfba41a36c..de809bb10d499 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -667,10 +667,11 @@ def intrinsic_w_chain : SDNode<"ISD::INTRINSIC_W_CHAIN",
 def intrinsic_wo_chain : SDNode<"ISD::INTRINSIC_WO_CHAIN",
                                 SDTypeProfile<1, -1, [SDTCisPtrTy<1>]>, []>;
 
-def SDT_assertext : SDTypeProfile<1, 1,
+def SDT_assert : SDTypeProfile<1, 1,
   [SDTCisInt<0>, SDTCisInt<1>, SDTCisSameAs<1, 0>]>;
-def assertsext : SDNode<"ISD::AssertSext", SDT_assertext>;
-def assertzext : SDNode<"ISD::AssertZext", SDT_assertext>;
+def assertsext : SDNode<"ISD::AssertSext", SDT_assert>;
+def assertzext : SDNode<"ISD::AssertZext", SDT_assert>;
+def assertalign : SDNode<"ISD::AssertAlign", SDT_assert>;
 
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index e4fc70253c555..d2666d4b86827 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -891,6 +891,13 @@ struct Attributor {
     // No matching attribute found, create one.
     // Use the static create method.
     auto &AA = AAType::createForPosition(IRP, *this);
+
+    // If we are currenty seeding attributes, enforce seeding rules.
+    if (SeedingPeriod && !shouldSeedAttribute(AA)) {
+      AA.getState().indicatePessimisticFixpoint();
+      return AA;
+    }
+
     registerAA(AA);
 
     // For now we ignore naked and optnone functions.
@@ -918,8 +925,15 @@ struct Attributor {
       return AA;
     }
 
+    // Allow seeded attributes to declare dependencies.
+    // Remember the seeding state.
+    bool OldSeedingPeriod = SeedingPeriod;
+    SeedingPeriod = false;
+
     updateAA(AA);
 
+    SeedingPeriod = OldSeedingPeriod;
+
     if (TrackDependence && AA.getState().isValidState())
       recordDependence(AA, const_cast<AbstractAttribute &>(*QueryingAA),
                        DepClass);
@@ -1036,6 +1050,14 @@ struct Attributor {
     identifyDefaultAbstractAttributes(const_cast<Function &>(F));
   }
 
+  /// Helper function to remove callsite.
+  void removeCallSite(CallInst *CI) {
+    if (!CI)
+      return;
+
+    CGUpdater.removeCallSite(*CI);
+  }
+
   /// Record that \p U is to be replaces with \p NV after information was
   /// manifested. This also triggers deletion of trivially dead istructions.
   bool changeUseAfterManifest(Use &U, Value &NV) {
@@ -1337,6 +1359,10 @@ struct Attributor {
   ChangeStatus
   rewriteFunctionSignatures(SmallPtrSetImpl<Function *> &ModifiedFns);
 
+  /// Check if the Attribute \p AA should be seeded.
+  /// See getOrCreateAAFor.
+  bool shouldSeedAttribute(AbstractAttribute &AA);
+
   /// The set of all abstract attributes.
   ///{
   using AAVector = SmallVector<AbstractAttribute *, 64>;
@@ -1402,6 +1428,10 @@ struct Attributor {
   /// Invoke instructions with at least a single dead successor block.
   SmallVector<WeakVH, 16> InvokeWithDeadSuccessor;
 
+  /// Wheather attributes are being `seeded`, always false after ::run function
+  /// gets called \see getOrCreateAAFor.
+  bool SeedingPeriod = true;
+
   /// Functions, blocks, and instructions we delete after manifest is done.
   ///
   ///{
@@ -2014,6 +2044,9 @@ struct AbstractAttribute : public IRPosition {
 
   /// This function should return the "summarized" assumed state as string.
   virtual const std::string getAsStr() const = 0;
+
+  /// This function should return the name of the AbstractAttribute
+  virtual const std::string getName() const = 0;
   ///}
 
   /// Allow the Attributor access to the protected methods.
@@ -2128,6 +2161,9 @@ struct AAReturnedValues
   static AAReturnedValues &createForPosition(const IRPosition &IRP,
                                              Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AAReturnedValues"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2146,6 +2182,9 @@ struct AANoUnwind
   /// Create an abstract attribute view for the position \p IRP.
   static AANoUnwind &createForPosition(const IRPosition &IRP, Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AANoUnwind"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2164,6 +2203,9 @@ struct AANoSync
   /// Create an abstract attribute view for the position \p IRP.
   static AANoSync &createForPosition(const IRPosition &IRP, Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AANoSync"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2183,6 +2225,9 @@ struct AANonNull
   /// Create an abstract attribute view for the position \p IRP.
   static AANonNull &createForPosition(const IRPosition &IRP, Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AANonNull"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2202,6 +2247,9 @@ struct AANoRecurse
   /// Create an abstract attribute view for the position \p IRP.
   static AANoRecurse &createForPosition(const IRPosition &IRP, Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AANoRecurse"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2221,6 +2269,9 @@ struct AAWillReturn
   /// Create an abstract attribute view for the position \p IRP.
   static AAWillReturn &createForPosition(const IRPosition &IRP, Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AAWillReturn"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2247,6 +2298,9 @@ struct AAUndefinedBehavior
   static AAUndefinedBehavior &createForPosition(const IRPosition &IRP,
                                                 Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AAUndefinedBehavior"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2275,6 +2329,9 @@ struct AAReachability : public StateWrapper<BooleanState, AbstractAttribute> {
   static AAReachability &createForPosition(const IRPosition &IRP,
                                            Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AAReachability"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2294,6 +2351,9 @@ struct AANoAlias
   /// Create an abstract attribute view for the position \p IRP.
   static AANoAlias &createForPosition(const IRPosition &IRP, Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AANoAlias"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2313,6 +2373,9 @@ struct AANoFree
   /// Create an abstract attribute view for the position \p IRP.
   static AANoFree &createForPosition(const IRPosition &IRP, Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AANoFree"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2332,6 +2395,9 @@ struct AANoReturn
   /// Create an abstract attribute view for the position \p IRP.
   static AANoReturn &createForPosition(const IRPosition &IRP, Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AANoReturn"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2386,6 +2452,9 @@ struct AAIsDead : public StateWrapper<BooleanState, AbstractAttribute> {
     return F.hasPersonalityFn() && !canSimplifyInvokeNoUnwind(&F);
   }
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AAIsDead"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 
@@ -2576,6 +2645,9 @@ struct AADereferenceable
   static AADereferenceable &createForPosition(const IRPosition &IRP,
                                               Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AADereferenceable"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2594,6 +2666,9 @@ struct AAAlign : public IRAttribute<
   /// Return known alignment.
   unsigned getKnownAlign() const { return getKnown(); }
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AAAlign"; }
+
   /// Create an abstract attribute view for the position \p IRP.
   static AAAlign &createForPosition(const IRPosition &IRP, Attributor &A);
 
@@ -2648,6 +2723,9 @@ struct AANoCapture
   /// Create an abstract attribute view for the position \p IRP.
   static AANoCapture &createForPosition(const IRPosition &IRP, Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AANoCapture"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2666,6 +2744,9 @@ struct AAValueSimplify : public StateWrapper<BooleanState, AbstractAttribute> {
   static AAValueSimplify &createForPosition(const IRPosition &IRP,
                                             Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AAValueSimplify"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2683,6 +2764,9 @@ struct AAHeapToStack : public StateWrapper<BooleanState, AbstractAttribute> {
   /// Create an abstract attribute view for the position \p IRP.
   static AAHeapToStack &createForPosition(const IRPosition &IRP, Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AAHeapToStack"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2716,6 +2800,9 @@ struct AAPrivatizablePtr
   static AAPrivatizablePtr &createForPosition(const IRPosition &IRP,
                                               Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AAPrivatizablePtr"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2767,6 +2854,9 @@ struct AAMemoryBehavior
   static AAMemoryBehavior &createForPosition(const IRPosition &IRP,
                                              Attributor &A);
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AAMemoryBehavior"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2925,6 +3015,9 @@ struct AAMemoryLocation
     return getMemoryLocationsAsStr(getAssumedNotAccessedLocation());
   }
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AAMemoryLocation"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
@@ -2970,6 +3063,9 @@ struct AAValueConstantRange
     return nullptr;
   }
 
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AAValueConstantRange"; }
+
   /// Unique ID (due to the unique address)
   static const char ID;
 };
diff --git a/llvm/include/llvm/Transforms/IPO/OpenMPOpt.h b/llvm/include/llvm/Transforms/IPO/OpenMPOpt.h
index 0bd81ea8f5431..d96187b73f9bb 100644
--- a/llvm/include/llvm/Transforms/IPO/OpenMPOpt.h
+++ b/llvm/include/llvm/Transforms/IPO/OpenMPOpt.h
@@ -17,6 +17,9 @@ namespace llvm {
 
 namespace omp {
 
+/// Summary of a kernel (=entry point for target offloading).
+using Kernel = Function *;
+
 /// Helper to remember if the module contains OpenMP (runtime calls), to be used
 /// foremost with containsOpenMP.
 struct OpenMPInModule {
@@ -30,8 +33,17 @@ struct OpenMPInModule {
   bool isKnown() { return Value != OpenMP::UNKNOWN; }
   operator bool() { return Value != OpenMP::NOT_FOUND; }
 
+  /// Return the known kernels (=GPU entry points) in the module.
+  SmallPtrSetImpl<Kernel> &getKernels() { return Kernels; }
+
+  /// Identify kernels in the module and populate the Kernels set.
+  void identifyKernels(Module &M);
+
 private:
   enum class OpenMP { FOUND, NOT_FOUND, UNKNOWN } Value = OpenMP::UNKNOWN;
+
+  /// Collection of known kernels (=GPU entry points) in the module.
+  SmallPtrSet<Kernel, 8> Kernels;
 };
 
 /// Helper to determine if \p M contains OpenMP (runtime calls).
diff --git a/llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h b/llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h
index 8b03bcba10e4e..a9928c3f5a40d 100644
--- a/llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h
+++ b/llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h
@@ -156,6 +156,7 @@ class PassManagerBuilder {
 
   bool DisableTailCalls;
   bool DisableUnrollLoops;
+  bool CallGraphProfile;
   bool SLPVectorize;
   bool LoopVectorize;
   bool LoopsInterleaved;
diff --git a/llvm/include/llvm/Transforms/Instrumentation.h b/llvm/include/llvm/Transforms/Instrumentation.h
index 04f211da9819b..d4373d7b39eaa 100644
--- a/llvm/include/llvm/Transforms/Instrumentation.h
+++ b/llvm/include/llvm/Transforms/Instrumentation.h
@@ -88,6 +88,8 @@ ModulePass *createPGOIndirectCallPromotionLegacyPass(bool InLTO = false,
                                                      bool SamplePGO = false);
 FunctionPass *createPGOMemOPSizeOptLegacyPass();
 
+ModulePass *createCGProfileLegacyPass();
+
 // The pgo-specific indirect call promotion function declared below is used by
 // the pgo-driven indirect call promotion and sample profile passes. It's a
 // wrapper around llvm::promoteCall, et al. that additionally computes !prof
diff --git a/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h b/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h
index ef33fa2147d13..7da0bf8c441fa 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h
@@ -25,12 +25,12 @@ class InterestingMemoryOperand {
   bool IsWrite;
   Type *OpType;
   uint64_t TypeSize;
-  unsigned Alignment;
+  MaybeAlign Alignment;
   // The mask Value, if we're looking at a masked load/store.
   Value *MaybeMask;
 
   InterestingMemoryOperand(Instruction *I, unsigned OperandNo, bool IsWrite,
-                           class Type *OpType, unsigned Alignment,
+                           class Type *OpType, MaybeAlign Alignment,
                            Value *MaybeMask = nullptr)
       : IsWrite(IsWrite), OpType(OpType), Alignment(Alignment),
         MaybeMask(MaybeMask) {
diff --git a/llvm/include/llvm/Transforms/Instrumentation/CGProfile.h b/llvm/include/llvm/Transforms/Instrumentation/CGProfile.h
index 28fd3804dec91..4cb45fd42f804 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/CGProfile.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/CGProfile.h
@@ -19,11 +19,6 @@ namespace llvm {
 class CGProfilePass : public PassInfoMixin<CGProfilePass> {
 public:
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
-
-private:
-  void addModuleFlags(
-      Module &M,
-      MapVector<std::pair<Function *, Function *>, uint64_t> &Counts) const;
 };
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h b/llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h
index b2edfe11d58f9..999086a29f87e 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h
@@ -19,6 +19,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/SpecialCaseList.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/Transforms/Instrumentation.h"
 
 namespace llvm {
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 68989f9614bfd..f2818c6b792ec 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -20,7 +20,6 @@
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/InstructionPrecedenceTracking.h"
 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
 #include "llvm/IR/Dominators.h"
@@ -35,6 +34,7 @@
 
 namespace llvm {
 
+class AAResults;
 class AssumptionCache;
 class BasicBlock;
 class BranchInst;
@@ -123,7 +123,7 @@ class GVN : public PassInfoMixin<GVN> {
   }
 
   DominatorTree &getDominatorTree() const { return *DT; }
-  AliasAnalysis *getAliasAnalysis() const { return VN.getAliasAnalysis(); }
+  AAResults *getAliasAnalysis() const { return VN.getAliasAnalysis(); }
   MemoryDependenceResults &getMemDep() const { return *MD; }
 
   bool isPREEnabled() const;
@@ -155,7 +155,7 @@ class GVN : public PassInfoMixin<GVN> {
         DenseMap<std::pair<uint32_t, const BasicBlock *>, uint32_t>;
     PhiTranslateMap PhiTranslateTable;
 
-    AliasAnalysis *AA = nullptr;
+    AAResults *AA = nullptr;
     MemoryDependenceResults *MD = nullptr;
     DominatorTree *DT = nullptr;
 
@@ -191,8 +191,8 @@ class GVN : public PassInfoMixin<GVN> {
     void add(Value *V, uint32_t num);
     void clear();
     void erase(Value *v);
-    void setAliasAnalysis(AliasAnalysis *A) { AA = A; }
-    AliasAnalysis *getAliasAnalysis() const { return AA; }
+    void setAliasAnalysis(AAResults *A) { AA = A; }
+    AAResults *getAliasAnalysis() const { return AA; }
     void setMemDep(MemoryDependenceResults *M) { MD = M; }
     void setDomTree(DominatorTree *D) { DT = D; }
     uint32_t getNextUnusedValueNumber() { return nextValueNumber; }
diff --git a/llvm/include/llvm/Transforms/Utils/Cloning.h b/llvm/include/llvm/Transforms/Utils/Cloning.h
index 4fce03461deb9..dffb7801bc8e1 100644
--- a/llvm/include/llvm/Transforms/Utils/Cloning.h
+++ b/llvm/include/llvm/Transforms/Utils/Cloning.h
@@ -19,7 +19,6 @@
 
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Twine.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/InlineCost.h"
 #include "llvm/IR/ValueHandle.h"
@@ -30,6 +29,7 @@
 
 namespace llvm {
 
+class AAResults;
 class AllocaInst;
 class BasicBlock;
 class BlockFrequencyInfo;
diff --git a/llvm/include/llvm/Transforms/Utils/CodeMoverUtils.h b/llvm/include/llvm/Transforms/Utils/CodeMoverUtils.h
index 3f5c0aed2abfa..630f936471f22 100644
--- a/llvm/include/llvm/Transforms/Utils/CodeMoverUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/CodeMoverUtils.h
@@ -38,14 +38,16 @@ bool isControlFlowEquivalent(const BasicBlock &BB0, const BasicBlock &BB1,
 
 /// Return true if \p I can be safely moved before \p InsertPoint.
 bool isSafeToMoveBefore(Instruction &I, Instruction &InsertPoint,
-                        DominatorTree &DT, const PostDominatorTree &PDT,
-                        DependenceInfo &DI);
+                        DominatorTree &DT,
+                        const PostDominatorTree *PDT = nullptr,
+                        DependenceInfo *DI = nullptr);
 
 /// Return true if all instructions (except the terminator) in \p BB can be
 /// safely moved before \p InsertPoint.
 bool isSafeToMoveBefore(BasicBlock &BB, Instruction &InsertPoint,
-                        DominatorTree &DT, const PostDominatorTree &PDT,
-                        DependenceInfo &DI);
+                        DominatorTree &DT,
+                        const PostDominatorTree *PDT = nullptr,
+                        DependenceInfo *DI = nullptr);
 
 /// Move instructions, in an order-preserving manner, from \p FromBB to the
 /// beginning of \p ToBB when proven safe.
diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h
index 8627421f6654b..f55e336f1f6aa 100644
--- a/llvm/include/llvm/Transforms/Utils/Local.h
+++ b/llvm/include/llvm/Transforms/Utils/Local.h
@@ -19,35 +19,37 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/TinyPtrVector.h"
-#include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/Utils/Local.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Dominators.h"
-#include "llvm/IR/GetElementPtrTypeIterator.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/User.h"
 #include "llvm/IR/Value.h"
+#include "llvm/IR/ValueHandle.h"
 #include "llvm/Support/Casting.h"
 #include <cstdint>
 #include <limits>
 
 namespace llvm {
 
+class AAResults;
 class AllocaInst;
 class AssumptionCache;
 class BasicBlock;
 class BranchInst;
+class CallBase;
 class CallInst;
 class DbgDeclareInst;
 class DbgVariableIntrinsic;
 class DbgValueInst;
 class DIBuilder;
+class DomTreeUpdater;
 class Function;
 class Instruction;
+class InvokeInst;
 class LoadInst;
 class MDNode;
 class MemorySSAUpdater;
@@ -251,7 +253,7 @@ bool simplifyCFG(BasicBlock *BB, const TargetTransformInfo &TTI,
 /// This function is used to flatten a CFG. For example, it uses parallel-and
 /// and parallel-or mode to collapse if-conditions and merge if-regions with
 /// identical statements.
-bool FlattenCFG(BasicBlock *BB, AliasAnalysis *AA = nullptr);
+bool FlattenCFG(BasicBlock *BB, AAResults *AA = nullptr);
 
 /// If this basic block is ONLY a setcc and a branch, and if a predecessor
 /// branches to us and one of our successors, fold the setcc into the
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 12c023f64f588..60446bca53174 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -14,7 +14,6 @@
 #define LLVM_TRANSFORMS_UTILS_LOOPUTILS_H
 
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/IVDescriptors.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
@@ -23,6 +22,7 @@ namespace llvm {
 
 template <typename T> class DomTreeNodeBase;
 using DomTreeNode = DomTreeNodeBase<BasicBlock>;
+class AAResults;
 class AliasSet;
 class AliasSetTracker;
 class BasicBlock;
@@ -117,11 +117,11 @@ struct SinkAndHoistLICMFlags {
 /// dominated by the specified block, and that are in the current loop) in
 /// reverse depth first order w.r.t the DominatorTree. This allows us to visit
 /// uses before definitions, allowing us to sink a loop body in one pass without
-/// iteration. Takes DomTreeNode, AliasAnalysis, LoopInfo, DominatorTree,
+/// iteration. Takes DomTreeNode, AAResults, LoopInfo, DominatorTree,
 /// TargetLibraryInfo, Loop, AliasSet information for all
 /// instructions of the loop and loop safety information as
 /// arguments. Diagnostics is emitted via \p ORE. It returns changed status.
-bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
+bool sinkRegion(DomTreeNode *, AAResults *, LoopInfo *, DominatorTree *,
                 TargetLibraryInfo *, TargetTransformInfo *, Loop *,
                 AliasSetTracker *, MemorySSAUpdater *, ICFLoopSafetyInfo *,
                 SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *);
@@ -130,11 +130,11 @@ bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
 /// dominated by the specified block, and that are in the current loop) in depth
 /// first order w.r.t the DominatorTree.  This allows us to visit definitions
 /// before uses, allowing us to hoist a loop body in one pass without iteration.
-/// Takes DomTreeNode, AliasAnalysis, LoopInfo, DominatorTree,
+/// Takes DomTreeNode, AAResults, LoopInfo, DominatorTree,
 /// TargetLibraryInfo, Loop, AliasSet information for all instructions of the
 /// loop and loop safety information as arguments. Diagnostics is emitted via \p
 /// ORE. It returns changed status.
-bool hoistRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
+bool hoistRegion(DomTreeNode *, AAResults *, LoopInfo *, DominatorTree *,
                  TargetLibraryInfo *, Loop *, AliasSetTracker *,
                  MemorySSAUpdater *, ScalarEvolution *, ICFLoopSafetyInfo *,
                  SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *);
diff --git a/llvm/include/llvm/Transforms/Utils/PredicateInfo.h b/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
index 16f776f7d2923..657b97c67a8b3 100644
--- a/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
+++ b/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
@@ -79,6 +79,10 @@ class PredicateBase : public ilist_node<PredicateBase> {
   // This can be use by passes, when destroying predicateinfo, to know
   // whether they can just drop the intrinsic, or have to merge metadata.
   Value *OriginalOp;
+  // The renamed operand in the condition used for this predicate. For nested
+  // predicates, this is different to OriginalOp which refers to the initial
+  // operand.
+  Value *RenamedOp;
   PredicateBase(const PredicateBase &) = delete;
   PredicateBase &operator=(const PredicateBase &) = delete;
   PredicateBase() = delete;
diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
index 1970cefcefba8..bb3d02b959564 100644
--- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -94,6 +94,7 @@ bool UnrollRuntimeLoopRemainder(
 
 void computePeelCount(Loop *L, unsigned LoopSize,
                       TargetTransformInfo::UnrollingPreferences &UP,
+                      TargetTransformInfo::PeelingPreferences &PP,
                       unsigned &TripCount, ScalarEvolution &SE);
 
 bool canPeel(Loop *L);
@@ -119,6 +120,8 @@ bool computeUnrollCount(Loop *L, const TargetTransformInfo &TTI,
                         unsigned MaxTripCount, bool MaxOrZero,
                         unsigned &TripMultiple, unsigned LoopSize,
                         TargetTransformInfo::UnrollingPreferences &UP,
+                        TargetTransformInfo::PeelingPreferences &PP,
+
                         bool &UseUpperBound);
 
 void simplifyLoopAfterUnroll(Loop *L, bool SimplifyIVs, LoopInfo *LI,
@@ -133,9 +136,13 @@ TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences(
     BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, int OptLevel,
     Optional<unsigned> UserThreshold, Optional<unsigned> UserCount,
     Optional<bool> UserAllowPartial, Optional<bool> UserRuntime,
-    Optional<bool> UserUpperBound, Optional<bool> UserAllowPeeling,
-    Optional<bool> UserAllowProfileBasedPeeling,
-    Optional<unsigned> UserFullUnrollMaxCount);
+    Optional<bool> UserUpperBound, Optional<unsigned> UserFullUnrollMaxCount);
+
+TargetTransformInfo::PeelingPreferences
+gatherPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                         const TargetTransformInfo &TTI,
+                         Optional<bool> UserAllowPeeling,
+                         Optional<bool> UserAllowProfileBasedPeeling);
 
 unsigned ApproximateLoopSize(const Loop *L, unsigned &NumCalls,
                              bool &NotDuplicatable, bool &Convergent,
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index e402d17b20cfc..c6c3450f77608 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -198,7 +198,7 @@ class LoopVectorizationLegality {
 public:
   LoopVectorizationLegality(
       Loop *L, PredicatedScalarEvolution &PSE, DominatorTree *DT,
-      TargetTransformInfo *TTI, TargetLibraryInfo *TLI, AliasAnalysis *AA,
+      TargetTransformInfo *TTI, TargetLibraryInfo *TLI, AAResults *AA,
       Function *F, std::function<const LoopAccessInfo &(Loop &)> *GetLAA,
       LoopInfo *LI, OptimizationRemarkEmitter *ORE,
       LoopVectorizationRequirements *R, LoopVectorizeHints *H, DemandedBits *DB,
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
index 22596b9d0bb70..ecb44a7b1518d 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
@@ -56,13 +56,13 @@
 #ifndef LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZE_H
 #define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZE_H
 
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/CommandLine.h"
 #include <functional>
 
 namespace llvm {
 
+class AAResults;
 class AssumptionCache;
 class BlockFrequencyInfo;
 class DemandedBits;
@@ -146,7 +146,7 @@ struct LoopVectorizePass : public PassInfoMixin<LoopVectorizePass> {
   BlockFrequencyInfo *BFI;
   TargetLibraryInfo *TLI;
   DemandedBits *DB;
-  AliasAnalysis *AA;
+  AAResults *AA;
   AssumptionCache *AC;
   std::function<const LoopAccessInfo &(Loop &)> *GetLAA;
   OptimizationRemarkEmitter *ORE;
@@ -159,7 +159,7 @@ struct LoopVectorizePass : public PassInfoMixin<LoopVectorizePass> {
   runImpl(Function &F, ScalarEvolution &SE_, LoopInfo &LI_,
           TargetTransformInfo &TTI_, DominatorTree &DT_,
           BlockFrequencyInfo &BFI_, TargetLibraryInfo *TLI_, DemandedBits &DB_,
-          AliasAnalysis &AA_, AssumptionCache &AC_,
+          AAResults &AA_, AssumptionCache &AC_,
           std::function<const LoopAccessInfo &(Loop &)> &GetLAA_,
           OptimizationRemarkEmitter &ORE_, ProfileSummaryInfo *PSI_);
 
diff --git a/llvm/include/llvm/module.modulemap b/llvm/include/llvm/module.modulemap
index a36b684916836..b262311a96a07 100644
--- a/llvm/include/llvm/module.modulemap
+++ b/llvm/include/llvm/module.modulemap
@@ -256,6 +256,7 @@ module LLVM_intrinsic_gen {
   module Analysis_PostDominators { header "Analysis/PostDominators.h" export * }
   module Analysis_DomTreeUpdater { header "Analysis/DomTreeUpdater.h" export * }
   module IR_IRBuilder { header "IR/IRBuilder.h" export * }
+  module IR_IRPrintingPasses { header "IR/IRPrintingPasses.h" export * }
   module IR_MatrixBuilder { header "IR/MatrixBuilder.h" export * }
   module IR_PassManager { header "IR/PassManager.h" export * }
   module IR_PassManagerImpl { header "IR/PassManagerImpl.h" export * }
diff --git a/llvm/lib/Analysis/AliasAnalysis.cpp b/llvm/lib/Analysis/AliasAnalysis.cpp
index 9db816ef1fda5..fec2415a0e459 100644
--- a/llvm/lib/Analysis/AliasAnalysis.cpp
+++ b/llvm/lib/Analysis/AliasAnalysis.cpp
@@ -58,7 +58,7 @@ using namespace llvm;
 
 /// Allow disabling BasicAA from the AA results. This is particularly useful
 /// when testing to isolate a single AA implementation.
-static cl::opt<bool> DisableBasicAA("disable-basicaa", cl::Hidden,
+static cl::opt<bool> DisableBasicAA("disable-basic-aa", cl::Hidden,
                                     cl::init(false));
 
 AAResults::AAResults(AAResults &&Arg)
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index 8149154cd3d9e..74664098ce1d4 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -65,16 +65,16 @@
 using namespace llvm;
 
 /// Enable analysis of recursive PHI nodes.
-static cl::opt<bool> EnableRecPhiAnalysis("basicaa-recphi", cl::Hidden,
+static cl::opt<bool> EnableRecPhiAnalysis("basic-aa-recphi", cl::Hidden,
                                           cl::init(false));
 
 /// By default, even on 32-bit architectures we use 64-bit integers for
 /// calculations. This will allow us to more-aggressively decompose indexing
 /// expressions calculated using i64 values (e.g., long long in C) which is
 /// common enough to worry about.
-static cl::opt<bool> ForceAtLeast64Bits("basicaa-force-at-least-64b",
+static cl::opt<bool> ForceAtLeast64Bits("basic-aa-force-at-least-64b",
                                         cl::Hidden, cl::init(true));
-static cl::opt<bool> DoubleCalcBits("basicaa-double-calc-bits",
+static cl::opt<bool> DoubleCalcBits("basic-aa-double-calc-bits",
                                     cl::Hidden, cl::init(false));
 
 /// SearchLimitReached / SearchTimes shows how often the limit of
@@ -433,7 +433,7 @@ static bool isObjectSize(const Value *V, uint64_t Size, const DataLayout &DL,
 /// an issue, for example, in particular for 32b pointers with negative indices
 /// that rely on two's complement wrap-arounds for precise alias information
 /// where the maximum pointer size is 64b.
-static APInt adjustToPointerSize(APInt Offset, unsigned PointerSize) {
+static APInt adjustToPointerSize(const APInt &Offset, unsigned PointerSize) {
   assert(PointerSize <= Offset.getBitWidth() && "Invalid PointerSize!");
   unsigned ShiftBits = Offset.getBitWidth() - PointerSize;
   return (Offset << ShiftBits).ashr(ShiftBits);
@@ -1724,6 +1724,10 @@ AliasResult BasicAAResult::aliasPHI(const PHINode *PN, LocationSize PNSize,
   // Other results are not possible.
   if (Alias == MayAlias)
     return MayAlias;
+  // With recursive phis we cannot guarantee that MustAlias/PartialAlias will
+  // remain valid to all elements and needs to conservatively return MayAlias.
+  if (isRecursive && Alias != NoAlias)
+    return MayAlias;
 
   // If all sources of the PHI node NoAlias or MustAlias V2, then returns
   // NoAlias / MustAlias. Otherwise, returns MayAlias.
@@ -1989,7 +1993,7 @@ void BasicAAResult::GetIndexDifference(
 
 bool BasicAAResult::constantOffsetHeuristic(
     const SmallVectorImpl<VariableGEPIndex> &VarIndices,
-    LocationSize MaybeV1Size, LocationSize MaybeV2Size, APInt BaseOffset,
+    LocationSize MaybeV1Size, LocationSize MaybeV2Size, const APInt &BaseOffset,
     AssumptionCache *AC, DominatorTree *DT) {
   if (VarIndices.size() != 2 || MaybeV1Size == LocationSize::unknown() ||
       MaybeV2Size == LocationSize::unknown())
@@ -2069,13 +2073,13 @@ char BasicAAWrapperPass::ID = 0;
 
 void BasicAAWrapperPass::anchor() {}
 
-INITIALIZE_PASS_BEGIN(BasicAAWrapperPass, "basicaa",
+INITIALIZE_PASS_BEGIN(BasicAAWrapperPass, "basic-aa",
                       "Basic Alias Analysis (stateless AA impl)", true, true)
 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(PhiValuesWrapperPass)
-INITIALIZE_PASS_END(BasicAAWrapperPass, "basicaa",
+INITIALIZE_PASS_END(BasicAAWrapperPass, "basic-aa",
                     "Basic Alias Analysis (stateless AA impl)", true, true)
 
 FunctionPass *llvm::createBasicAAWrapperPass() {
diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
index da711c4acaf6e..a396b5ad21c6a 100644
--- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp
+++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
@@ -1056,10 +1056,10 @@ BranchProbabilityInfo::printEdgeProbability(raw_ostream &OS,
 }
 
 void BranchProbabilityInfo::eraseBlock(const BasicBlock *BB) {
-  for (auto I = Probs.begin(), E = Probs.end(); I != E; ++I) {
-    auto Key = I->first;
-    if (Key.first == BB)
-      Probs.erase(Key);
+  for (const_succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
+    auto MapI = Probs.find(std::make_pair(BB, I.getSuccessorIndex()));
+    if (MapI != Probs.end())
+      Probs.erase(MapI);
   }
 }
 
diff --git a/llvm/lib/Analysis/CGSCCPassManager.cpp b/llvm/lib/Analysis/CGSCCPassManager.cpp
index 64bd6dfe67abd..fd3166f8cd0cd 100644
--- a/llvm/lib/Analysis/CGSCCPassManager.cpp
+++ b/llvm/lib/Analysis/CGSCCPassManager.cpp
@@ -73,14 +73,14 @@ PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &,
       AM.getCachedResult<FunctionAnalysisManagerCGSCCProxy>(*C)->getManager();
 
   for (auto &Pass : Passes) {
-    if (DebugLogging)
-      dbgs() << "Running pass: " << Pass->name() << " on " << *C << "\n";
-
     // Check the PassInstrumentation's BeforePass callbacks before running the
     // pass, skip its execution completely if asked to (callback returns false).
     if (!PI.runBeforePass(*Pass, *C))
       continue;
 
+    if (DebugLogging)
+      dbgs() << "Running pass: " << Pass->name() << " on " << *C << "\n";
+
     PreservedAnalyses PassPA;
     {
       TimeTraceScope TimeScope(Pass->name());
diff --git a/llvm/lib/Analysis/CMakeLists.txt b/llvm/lib/Analysis/CMakeLists.txt
index dc0f8584ae23c..a317579ecc836 100644
--- a/llvm/lib/Analysis/CMakeLists.txt
+++ b/llvm/lib/Analysis/CMakeLists.txt
@@ -1,3 +1,18 @@
+set(CommonMLSources MLInlineAdvisor.cpp)
+set(ReleaseModeMLSources ReleaseModeModelRunner.cpp)
+
+if (DEFINED LLVM_HAVE_TF_AOT)
+  include(TensorFlowCompile)
+  tfcompile(models/inliner serve action InlinerSizeModel llvm::InlinerSizeModel)
+  list(APPEND ReleaseModeMLSources
+    $<TARGET_OBJECTS:tf_xla_runtime_objects>
+    ${GENERATED_OBJS}
+  )
+  set(MLPolicySources ${CommonMLSources} ${ReleaseModeMLSources})
+else()
+  set(LLVM_OPTIONAL_SOURCES ${CommonMLSources} ${ReleaseModeMLSources})
+endif()
+
 add_llvm_component_library(LLVMAnalysis
   AliasAnalysis.cpp
   AliasAnalysisEvaluator.cpp
@@ -75,7 +90,6 @@ add_llvm_component_library(LLVMAnalysis
   ObjCARCAnalysisUtils.cpp
   ObjCARCInstKind.cpp
   OptimizationRemarkEmitter.cpp
-  OrderedInstructions.cpp
   PHITransAddr.cpp
   PhiValues.cpp
   PostDominators.cpp
@@ -86,6 +100,7 @@ add_llvm_component_library(LLVMAnalysis
   RegionPrinter.cpp
   ScalarEvolution.cpp
   ScalarEvolutionAliasAnalysis.cpp
+  ScalarEvolutionDivision.cpp
   ScalarEvolutionNormalization.cpp
   StackLifetime.cpp
   StackSafetyAnalysis.cpp
@@ -102,6 +117,7 @@ add_llvm_component_library(LLVMAnalysis
   ValueTracking.cpp
   VectorUtils.cpp
   VFABIDemangling.cpp
+  ${MLPolicySources}
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Analysis
diff --git a/llvm/lib/Analysis/CallGraph.cpp b/llvm/lib/Analysis/CallGraph.cpp
index 284335863348d..d8abccfdb0958 100644
--- a/llvm/lib/Analysis/CallGraph.cpp
+++ b/llvm/lib/Analysis/CallGraph.cpp
@@ -10,6 +10,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Config/llvm-config.h"
+#include "llvm/IR/AbstractCallSite.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
@@ -31,7 +32,7 @@ using namespace llvm;
 
 CallGraph::CallGraph(Module &M)
     : M(M), ExternalCallingNode(getOrInsertFunction(nullptr)),
-      CallsExternalNode(std::make_unique<CallGraphNode>(nullptr)) {
+      CallsExternalNode(std::make_unique<CallGraphNode>(this, nullptr)) {
   // Add every interesting function to the call graph.
   for (Function &F : M)
     if (!isDbgInfoIntrinsic(F.getIntrinsicID()))
@@ -44,6 +45,11 @@ CallGraph::CallGraph(CallGraph &&Arg)
       CallsExternalNode(std::move(Arg.CallsExternalNode)) {
   Arg.FunctionMap.clear();
   Arg.ExternalCallingNode = nullptr;
+
+  // Update parent CG for all call graph's nodes.
+  CallsExternalNode->CG = this;
+  for (auto &P : FunctionMap)
+    P.second->CG = this;
 }
 
 CallGraph::~CallGraph() {
@@ -99,6 +105,11 @@ void CallGraph::populateCallGraphNode(CallGraphNode *Node) {
           Node->addCalledFunction(Call, CallsExternalNode.get());
         else if (!Callee->isIntrinsic())
           Node->addCalledFunction(Call, getOrInsertFunction(Callee));
+
+        // Add reference to callback functions.
+        forEachCallbackFunction(*Call, [=](Function *CB) {
+          Node->addCalledFunction(nullptr, getOrInsertFunction(CB));
+        });
       }
     }
 }
@@ -178,7 +189,7 @@ CallGraphNode *CallGraph::getOrInsertFunction(const Function *F) {
     return CGN.get();
 
   assert((!F || F->getParent() == &M) && "Function not in current module!");
-  CGN = std::make_unique<CallGraphNode>(const_cast<Function *>(F));
+  CGN = std::make_unique<CallGraphNode>(this, const_cast<Function *>(F));
   return CGN.get();
 }
 
@@ -214,10 +225,15 @@ LLVM_DUMP_METHOD void CallGraphNode::dump() const { print(dbgs()); }
 void CallGraphNode::removeCallEdgeFor(CallBase &Call) {
   for (CalledFunctionsVector::iterator I = CalledFunctions.begin(); ; ++I) {
     assert(I != CalledFunctions.end() && "Cannot find callsite to remove!");
-    if (I->first == &Call) {
+    if (I->first && *I->first == &Call) {
       I->second->DropRef();
       *I = CalledFunctions.back();
       CalledFunctions.pop_back();
+
+      // Remove all references to callback functions if there are any.
+      forEachCallbackFunction(Call, [=](Function *CB) {
+        removeOneAbstractEdgeTo(CG->getOrInsertFunction(CB));
+      });
       return;
     }
   }
@@ -242,7 +258,7 @@ void CallGraphNode::removeOneAbstractEdgeTo(CallGraphNode *Callee) {
   for (CalledFunctionsVector::iterator I = CalledFunctions.begin(); ; ++I) {
     assert(I != CalledFunctions.end() && "Cannot find callee to remove!");
     CallRecord &CR = *I;
-    if (CR.second == Callee && CR.first == nullptr) {
+    if (CR.second == Callee && !CR.first) {
       Callee->DropRef();
       *I = CalledFunctions.back();
       CalledFunctions.pop_back();
@@ -258,11 +274,19 @@ void CallGraphNode::replaceCallEdge(CallBase &Call, CallBase &NewCall,
                                     CallGraphNode *NewNode) {
   for (CalledFunctionsVector::iterator I = CalledFunctions.begin(); ; ++I) {
     assert(I != CalledFunctions.end() && "Cannot find callsite to remove!");
-    if (I->first == &Call) {
+    if (I->first && *I->first == &Call) {
       I->second->DropRef();
       I->first = &NewCall;
       I->second = NewNode;
       NewNode->AddRef();
+
+      // Refresh callback references.
+      forEachCallbackFunction(Call, [=](Function *CB) {
+        removeOneAbstractEdgeTo(CG->getOrInsertFunction(CB));
+      });
+      forEachCallbackFunction(NewCall, [=](Function *CB) {
+        addCalledFunction(nullptr, CG->getOrInsertFunction(CB));
+      });
       return;
     }
   }
diff --git a/llvm/lib/Analysis/CallGraphSCCPass.cpp b/llvm/lib/Analysis/CallGraphSCCPass.cpp
index 79fa372917ca4..fc65936024af6 100644
--- a/llvm/lib/Analysis/CallGraphSCCPass.cpp
+++ b/llvm/lib/Analysis/CallGraphSCCPass.cpp
@@ -227,20 +227,25 @@ bool CGPassManager::RefreshCallGraph(const CallGraphSCC &CurSCC, CallGraph &CG,
 
     // Get the set of call sites currently in the function.
     for (CallGraphNode::iterator I = CGN->begin(), E = CGN->end(); I != E; ) {
+      // Skip "reference" call records that do not have call instruction.
+      if (!I->first) {
+        ++I;
+        continue;
+      }
+
       // If this call site is null, then the function pass deleted the call
       // entirely and the WeakTrackingVH nulled it out.
-      auto *Call = dyn_cast_or_null<CallBase>(I->first);
-      if (!I->first ||
+      auto *Call = dyn_cast_or_null<CallBase>(*I->first);
+      if (!Call ||
           // If we've already seen this call site, then the FunctionPass RAUW'd
           // one call with another, which resulted in two "uses" in the edge
           // list of the same call.
-          Calls.count(I->first) ||
+          Calls.count(Call) ||
 
           // If the call edge is not from a call or invoke, or it is a
           // instrinsic call, then the function pass RAUW'd a call with
           // another value. This can happen when constant folding happens
           // of well known functions etc.
-          !Call ||
           (Call->getCalledFunction() &&
            Call->getCalledFunction()->isIntrinsic() &&
            Intrinsic::isLeaf(Call->getCalledFunction()->getIntrinsicID()))) {
@@ -267,14 +272,13 @@ bool CGPassManager::RefreshCallGraph(const CallGraphSCC &CurSCC, CallGraph &CG,
         continue;
       }
 
-      assert(!Calls.count(I->first) &&
-             "Call site occurs in node multiple times");
+      assert(!Calls.count(Call) && "Call site occurs in node multiple times");
 
       if (Call) {
         Function *Callee = Call->getCalledFunction();
         // Ignore intrinsics because they're not really function calls.
         if (!Callee || !(Callee->isIntrinsic()))
-          Calls.insert(std::make_pair(I->first, I->second));
+          Calls.insert(std::make_pair(Call, I->second));
       }
       ++I;
     }
diff --git a/llvm/lib/Analysis/CaptureTracking.cpp b/llvm/lib/Analysis/CaptureTracking.cpp
index d7ddb7d9e0a59..8b101e3b2cc47 100644
--- a/llvm/lib/Analysis/CaptureTracking.cpp
+++ b/llvm/lib/Analysis/CaptureTracking.cpp
@@ -25,6 +25,7 @@
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
+#include "llvm/Support/CommandLine.h"
 
 using namespace llvm;
 
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 76e7b2906c913..a414336fb21be 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -2607,8 +2607,8 @@ static Constant *ConstantFoldScalarCall3(StringRef Name,
           // how rounding should be done, and provide their own folding to be
           // consistent with rounding. This is the same approach as used by
           // DAGTypeLegalizer::ExpandIntRes_MULFIX.
-          APInt Lhs = Op1->getValue();
-          APInt Rhs = Op2->getValue();
+          const APInt &Lhs = Op1->getValue();
+          const APInt &Rhs = Op2->getValue();
           unsigned Scale = Op3->getValue().getZExtValue();
           unsigned Width = Lhs.getBitWidth();
           assert(Scale < Width && "Illegal scale.");
diff --git a/llvm/lib/Analysis/DomTreeUpdater.cpp b/llvm/lib/Analysis/DomTreeUpdater.cpp
index e0e17de20121b..9594da0a4f91d 100644
--- a/llvm/lib/Analysis/DomTreeUpdater.cpp
+++ b/llvm/lib/Analysis/DomTreeUpdater.cpp
@@ -507,7 +507,7 @@ LLVM_DUMP_METHOD void DomTreeUpdater::dump() const {
 
   OS << "Pending DeletedBBs:\n";
   Index = 0;
-  for (auto BB : DeletedBBs) {
+  for (const auto *BB : DeletedBBs) {
     OS << "  " << Index << " : ";
     ++Index;
     if (BB->hasName())
@@ -519,7 +519,7 @@ LLVM_DUMP_METHOD void DomTreeUpdater::dump() const {
 
   OS << "Pending Callbacks:\n";
   Index = 0;
-  for (auto BB : Callbacks) {
+  for (const auto &BB : Callbacks) {
     OS << "  " << Index << " : ";
     ++Index;
     if (BB->hasName())
diff --git a/llvm/lib/Analysis/GlobalsModRef.cpp b/llvm/lib/Analysis/GlobalsModRef.cpp
index a281626505a32..8c8ccf04ebba2 100644
--- a/llvm/lib/Analysis/GlobalsModRef.cpp
+++ b/llvm/lib/Analysis/GlobalsModRef.cpp
@@ -17,6 +17,7 @@
 #include "llvm/ADT/SCCIterator.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/CallGraph.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -28,6 +29,7 @@
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/CommandLine.h"
+
 using namespace llvm;
 
 #define DEBUG_TYPE "globalsmodref-aa"
diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp
index 2d7b4cabebb16..9a3e5fa0df722 100644
--- a/llvm/lib/Analysis/InlineAdvisor.cpp
+++ b/llvm/lib/Analysis/InlineAdvisor.cpp
@@ -20,6 +20,7 @@
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/raw_ostream.h"
 
 #include <sstream>
@@ -154,7 +155,9 @@ bool InlineAdvisorAnalysis::Result::tryCreate(InlineParams Params,
     // To be added subsequently under conditional compilation.
     break;
   case InliningAdvisorMode::Release:
-    // To be added subsequently under conditional compilation.
+#ifdef LLVM_HAVE_TF_AOT
+    Advisor = llvm::getReleaseModeAdvisor(M, MAM);
+#endif
     break;
   }
   return !!Advisor;
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index 66e8184e7ec2d..33d714406d7f2 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -110,6 +110,10 @@ static cl::opt<bool> InlineCallerSupersetNoBuiltin(
     cl::desc("Allow inlining when caller has a superset of callee's nobuiltin "
              "attributes."));
 
+static cl::opt<bool> DisableGEPConstOperand(
+    "disable-gep-const-evaluation", cl::Hidden, cl::init(false),
+    cl::desc("Disables evaluation of GetElementPtr with constant operands"));
+
 namespace {
 class InlineCostCallAnalyzer;
 
@@ -402,6 +406,12 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
 
   InlineResult analyze();
 
+  Optional<Constant*> getSimplifiedValue(Instruction *I) {
+    if (SimplifiedValues.find(I) != SimplifiedValues.end())
+      return SimplifiedValues[I];
+    return None;
+  }
+
   // Keep a bunch of stats about the cost savings found so we can print them
   // out when debugging.
   unsigned NumConstantArgs = 0;
@@ -726,6 +736,10 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
 
   void dump();
 
+  // Prints the same analysis as dump(), but its definition is not dependent
+  // on the build.
+  void print();
+
   Optional<InstructionCostDetail> getCostDetails(const Instruction *I) {
     if (InstructionCostDetailMap.find(I) != InstructionCostDetailMap.end())
       return InstructionCostDetailMap[I];
@@ -766,6 +780,11 @@ void InlineCostAnnotationWriter::emitInstructionAnnot(const Instruction *I,
     if (Record->hasThresholdChanged())
       OS << ", threshold delta = " << Record->getThresholdDelta();
   }
+  auto C = ICCA->getSimplifiedValue(const_cast<Instruction *>(I));
+  if (C) {
+    OS << ", simplified to ";
+    C.getValue()->print(OS, true);
+  }
   OS << "\n";
 }
 
@@ -838,10 +857,22 @@ bool CallAnalyzer::visitAlloca(AllocaInst &I) {
   if (I.isArrayAllocation()) {
     Constant *Size = SimplifiedValues.lookup(I.getArraySize());
     if (auto *AllocSize = dyn_cast_or_null<ConstantInt>(Size)) {
+      // Sometimes a dynamic alloca could be converted into a static alloca
+      // after this constant prop, and become a huge static alloca on an
+      // unconditional CFG path. Avoid inlining if this is going to happen above
+      // a threshold.
+      // FIXME: If the threshold is removed or lowered too much, we could end up
+      // being too pessimistic and prevent inlining non-problematic code. This
+      // could result in unintended perf regressions. A better overall strategy
+      // is needed to track stack usage during inlining.
       Type *Ty = I.getAllocatedType();
       AllocatedSize = SaturatingMultiplyAdd(
           AllocSize->getLimitedValue(), DL.getTypeAllocSize(Ty).getFixedSize(),
           AllocatedSize);
+      if (AllocatedSize > InlineConstants::MaxSimplifiedDynamicAllocaToInline) {
+        HasDynamicAlloca = true;
+        return false;
+      }
       return Base::visitAlloca(I);
     }
   }
@@ -992,6 +1023,16 @@ bool CallAnalyzer::visitGetElementPtr(GetElementPtrInst &I) {
     return true;
   };
 
+  if (!DisableGEPConstOperand)
+    if (simplifyInstruction(I, [&](SmallVectorImpl<Constant *> &COps) {
+        SmallVector<Constant *, 2> Indices;
+        for (unsigned int Index = 1 ; Index < COps.size() ; ++Index)
+            Indices.push_back(COps[Index]);
+        return ConstantExpr::getGetElementPtr(I.getSourceElementType(), COps[0],
+                                              Indices, I.isInBounds());
+        }))
+      return true;
+
   if ((I.isInBounds() && canFoldInboundsGEP(I)) || IsGEPOffsetConstant(I)) {
     if (SROAArg)
       SROAArgValues[&I] = SROAArg;
@@ -2168,9 +2209,7 @@ InlineResult CallAnalyzer::analyze() {
   return finalizeAnalysis();
 }
 
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-/// Dump stats about this call's analysis.
-LLVM_DUMP_METHOD void InlineCostCallAnalyzer::dump() {
+void InlineCostCallAnalyzer::print() {
 #define DEBUG_PRINT_STAT(x) dbgs() << "      " #x ": " << x << "\n"
   if (PrintInstructionComments)
     F.print(dbgs(), &Writer);
@@ -2189,6 +2228,12 @@ LLVM_DUMP_METHOD void InlineCostCallAnalyzer::dump() {
   DEBUG_PRINT_STAT(Threshold);
 #undef DEBUG_PRINT_STAT
 }
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+/// Dump stats about this call's analysis.
+LLVM_DUMP_METHOD void InlineCostCallAnalyzer::dump() {
+  print();
+}
 #endif
 
 /// Test that there are no attribute conflicts between Caller and Callee
@@ -2510,3 +2555,40 @@ InlineParams llvm::getInlineParams(unsigned OptLevel, unsigned SizeOptLevel) {
     Params.LocallyHotCallSiteThreshold = LocallyHotCallSiteThreshold;
   return Params;
 }
+
+PreservedAnalyses
+InlineCostAnnotationPrinterPass::run(Function &F,
+                                     FunctionAnalysisManager &FAM) {
+  PrintInstructionComments = true;
+  std::function<AssumptionCache &(Function &)> GetAssumptionCache = [&](
+      Function &F) -> AssumptionCache & {
+    return FAM.getResult<AssumptionAnalysis>(F);
+  };
+  Module *M = F.getParent();
+  ProfileSummaryInfo PSI(*M);
+  DataLayout DL(M);
+  TargetTransformInfo TTI(DL);
+  // FIXME: Redesign the usage of InlineParams to expand the scope of this pass.
+  // In the current implementation, the type of InlineParams doesn't matter as
+  // the pass serves only for verification of inliner's decisions.
+  // We can add a flag which determines InlineParams for this run. Right now,
+  // the default InlineParams are used.
+  const InlineParams Params = llvm::getInlineParams();
+    for (BasicBlock &BB : F) {
+    for (Instruction &I : BB) {
+      if (CallInst *CI = dyn_cast<CallInst>(&I)) {
+        Function *CalledFunction = CI->getCalledFunction();
+        if (!CalledFunction || CalledFunction->isDeclaration())
+          continue;
+        OptimizationRemarkEmitter ORE(CalledFunction);
+        InlineCostCallAnalyzer ICCA(*CalledFunction, *CI, Params, TTI,
+                                    GetAssumptionCache, nullptr, &PSI, &ORE);
+        ICCA.analyze();
+        OS << "      Analyzing call of " << CalledFunction->getName()
+           << "... (caller:" << CI->getCaller()->getName() << ")\n";
+        ICCA.print();
+      }
+    }
+  }
+  return PreservedAnalyses::all();
+}
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 02d945beabab2..0975a65d183e4 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -3281,6 +3281,30 @@ static Value *simplifyICmpWithMinMax(CmpInst::Predicate Pred, Value *LHS,
   return nullptr;
 }
 
+static Value *simplifyICmpWithDominatingAssume(CmpInst::Predicate Predicate,
+                                               Value *LHS, Value *RHS,
+                                               const SimplifyQuery &Q) {
+  // Gracefully handle instructions that have not been inserted yet.
+  if (!Q.AC || !Q.CxtI || !Q.CxtI->getParent())
+    return nullptr;
+
+  for (Value *AssumeBaseOp : {LHS, RHS}) {
+    for (auto &AssumeVH : Q.AC->assumptionsFor(AssumeBaseOp)) {
+      if (!AssumeVH)
+        continue;
+
+      CallInst *Assume = cast<CallInst>(AssumeVH);
+      if (Optional<bool> Imp =
+              isImpliedCondition(Assume->getArgOperand(0), Predicate, LHS, RHS,
+                                 Q.DL))
+        if (isValidAssumeForContext(Assume, Q.CxtI, Q.DT))
+          return ConstantInt::get(GetCompareTy(LHS), *Imp);
+    }
+  }
+
+  return nullptr;
+}
+
 /// Given operands for an ICmpInst, see if we can fold the result.
 /// If not, this returns null.
 static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
@@ -3381,6 +3405,15 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
                                           MaxRecurse-1))
             return V;
       }
+      // Fold (zext X) ule (sext X), (zext X) sge (sext X) to true.
+      else if (SExtInst *RI = dyn_cast<SExtInst>(RHS)) {
+        if (SrcOp == RI->getOperand(0)) {
+          if (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_SGE)
+            return ConstantInt::getTrue(ITy);
+          if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_SLT)
+            return ConstantInt::getFalse(ITy);
+        }
+      }
       // Turn icmp (zext X), Cst into a compare of X and Cst if Cst is extended
       // too.  If not, then try to deduce the result of the comparison.
       else if (ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
@@ -3440,6 +3473,15 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
                                           Q, MaxRecurse-1))
             return V;
       }
+      // Fold (sext X) uge (zext X), (sext X) sle (zext X) to true.
+      else if (ZExtInst *RI = dyn_cast<ZExtInst>(RHS)) {
+        if (SrcOp == RI->getOperand(0)) {
+          if (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_SLE)
+            return ConstantInt::getTrue(ITy);
+          if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_SGT)
+            return ConstantInt::getFalse(ITy);
+        }
+      }
       // Turn icmp (sext X), Cst into a compare of X and Cst if Cst is extended
       // too.  If not, then try to deduce the result of the comparison.
       else if (ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
@@ -3515,6 +3557,9 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
   if (Value *V = simplifyICmpWithMinMax(Pred, LHS, RHS, Q, MaxRecurse))
     return V;
 
+  if (Value *V = simplifyICmpWithDominatingAssume(Pred, LHS, RHS, Q))
+    return V;
+
   // Simplify comparisons of related pointers using a powerful, recursive
   // GEP-walk when we have target data available..
   if (LHS->getType()->isPointerTy())
@@ -4073,9 +4118,15 @@ static Value *SimplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal,
   if (TrueVal == FalseVal)
     return TrueVal;
 
-  if (isa<UndefValue>(TrueVal))   // select ?, undef, X -> X
+  // If the true or false value is undef, we can fold to the other value as
+  // long as the other value isn't poison.
+  // select ?, undef, X -> X
+  if (isa<UndefValue>(TrueVal) &&
+      isGuaranteedNotToBeUndefOrPoison(FalseVal, Q.CxtI, Q.DT))
     return FalseVal;
-  if (isa<UndefValue>(FalseVal))   // select ?, X, undef -> X
+  // select ?, X, undef -> X
+  if (isa<UndefValue>(FalseVal) &&
+      isGuaranteedNotToBeUndefOrPoison(TrueVal, Q.CxtI, Q.DT))
     return TrueVal;
 
   // Deal with partial undef vector constants: select ?, VecC, VecC' --> VecC''
@@ -4095,9 +4146,11 @@ static Value *SimplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal,
       // one element is undef, choose the defined element as the safe result.
       if (TEltC == FEltC)
         NewC.push_back(TEltC);
-      else if (isa<UndefValue>(TEltC))
+      else if (isa<UndefValue>(TEltC) &&
+               isGuaranteedNotToBeUndefOrPoison(FEltC))
         NewC.push_back(FEltC);
-      else if (isa<UndefValue>(FEltC))
+      else if (isa<UndefValue>(FEltC) &&
+               isGuaranteedNotToBeUndefOrPoison(TEltC))
         NewC.push_back(TEltC);
       else
         break;
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 12f7e6869a639..fb14008a2b471 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -1093,17 +1093,41 @@ Optional<ValueLatticeElement> LazyValueInfoImpl::solveBlockValueExtractValue(
   return ValueLatticeElement::getOverdefined();
 }
 
+static bool matchICmpOperand(const APInt *&Offset, Value *LHS, Value *Val,
+                             ICmpInst::Predicate Pred) {
+  if (LHS == Val)
+    return true;
+
+  // Handle range checking idiom produced by InstCombine. We will subtract the
+  // offset from the allowed range for RHS in this case.
+  if (match(LHS, m_Add(m_Specific(Val), m_APInt(Offset))))
+    return true;
+
+  // If (x | y) < C, then (x < C) && (y < C).
+  if (match(LHS, m_c_Or(m_Specific(Val), m_Value())) &&
+      (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE))
+    return true;
+
+  // If (x & y) > C, then (x > C) && (y > C).
+  if (match(LHS, m_c_And(m_Specific(Val), m_Value())) &&
+      (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE))
+    return true;
+
+  return false;
+}
+
 static ValueLatticeElement getValueFromICmpCondition(Value *Val, ICmpInst *ICI,
                                                      bool isTrueDest) {
   Value *LHS = ICI->getOperand(0);
   Value *RHS = ICI->getOperand(1);
-  CmpInst::Predicate Predicate = ICI->getPredicate();
+
+  // Get the predicate that must hold along the considered edge.
+  CmpInst::Predicate EdgePred =
+      isTrueDest ? ICI->getPredicate() : ICI->getInversePredicate();
 
   if (isa<Constant>(RHS)) {
     if (ICI->isEquality() && LHS == Val) {
-      // We know that V has the RHS constant if this is a true SETEQ or
-      // false SETNE.
-      if (isTrueDest == (Predicate == ICmpInst::ICMP_EQ))
+      if (EdgePred == ICmpInst::ICMP_EQ)
         return ValueLatticeElement::get(cast<Constant>(RHS));
       else if (!isa<UndefValue>(RHS))
         return ValueLatticeElement::getNot(cast<Constant>(RHS));
@@ -1113,47 +1137,31 @@ static ValueLatticeElement getValueFromICmpCondition(Value *Val, ICmpInst *ICI,
   if (!Val->getType()->isIntegerTy())
     return ValueLatticeElement::getOverdefined();
 
-  // Use ConstantRange::makeAllowedICmpRegion in order to determine the possible
-  // range of Val guaranteed by the condition. Recognize comparisons in the from
-  // of:
-  //  icmp <pred> Val, ...
-  //  icmp <pred> (add Val, Offset), ...
-  // The latter is the range checking idiom that InstCombine produces. Subtract
-  // the offset from the allowed range for RHS in this case.
-
-  // Val or (add Val, Offset) can be on either hand of the comparison
-  if (LHS != Val && !match(LHS, m_Add(m_Specific(Val), m_ConstantInt()))) {
+  const APInt *Offset = nullptr;
+  if (!matchICmpOperand(Offset, LHS, Val, EdgePred)) {
     std::swap(LHS, RHS);
-    Predicate = CmpInst::getSwappedPredicate(Predicate);
+    EdgePred = CmpInst::getSwappedPredicate(EdgePred);
+    if (!matchICmpOperand(Offset, LHS, Val, EdgePred))
+      return ValueLatticeElement::getOverdefined();
   }
 
-  ConstantInt *Offset = nullptr;
-  if (LHS != Val)
-    match(LHS, m_Add(m_Specific(Val), m_ConstantInt(Offset)));
-
-  if (LHS == Val || Offset) {
-    // Calculate the range of values that are allowed by the comparison
-    ConstantRange RHSRange(RHS->getType()->getIntegerBitWidth(),
-                           /*isFullSet=*/true);
-    if (ConstantInt *CI = dyn_cast<ConstantInt>(RHS))
-      RHSRange = ConstantRange(CI->getValue());
-    else if (Instruction *I = dyn_cast<Instruction>(RHS))
-      if (auto *Ranges = I->getMetadata(LLVMContext::MD_range))
-        RHSRange = getConstantRangeFromMetadata(*Ranges);
-
-    // If we're interested in the false dest, invert the condition
-    CmpInst::Predicate Pred =
-            isTrueDest ? Predicate : CmpInst::getInversePredicate(Predicate);
-    ConstantRange TrueValues =
-            ConstantRange::makeAllowedICmpRegion(Pred, RHSRange);
-
-    if (Offset) // Apply the offset from above.
-      TrueValues = TrueValues.subtract(Offset->getValue());
-
-    return ValueLatticeElement::getRange(std::move(TrueValues));
-  }
+  // Calculate the range of values that are allowed by the comparison.
+  ConstantRange RHSRange(RHS->getType()->getIntegerBitWidth(),
+                         /*isFullSet=*/true);
+  if (ConstantInt *CI = dyn_cast<ConstantInt>(RHS))
+    RHSRange = ConstantRange(CI->getValue());
+  else if (Instruction *I = dyn_cast<Instruction>(RHS))
+    if (auto *Ranges = I->getMetadata(LLVMContext::MD_range))
+      RHSRange = getConstantRangeFromMetadata(*Ranges);
 
-  return ValueLatticeElement::getOverdefined();
+  // If we're interested in the false dest, invert the condition
+  ConstantRange TrueValues =
+      ConstantRange::makeAllowedICmpRegion(EdgePred, RHSRange);
+
+  if (Offset) // Apply the offset from above.
+    TrueValues = TrueValues.subtract(*Offset);
+
+  return ValueLatticeElement::getRange(std::move(TrueValues));
 }
 
 // Handle conditions of the form
diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp
index 1c34fc43dadeb..564c00dbad983 100644
--- a/llvm/lib/Analysis/Lint.cpp
+++ b/llvm/lib/Analysis/Lint.cpp
@@ -93,9 +93,8 @@ namespace {
     void visitFunction(Function &F);
 
     void visitCallBase(CallBase &CB);
-    void visitMemoryReference(Instruction &I, Value *Ptr,
-                              uint64_t Size, unsigned Align,
-                              Type *Ty, unsigned Flags);
+    void visitMemoryReference(Instruction &I, Value *Ptr, uint64_t Size,
+                              MaybeAlign Alignment, Type *Ty, unsigned Flags);
     void visitEHBeginCatch(IntrinsicInst *II);
     void visitEHEndCatch(IntrinsicInst *II);
 
@@ -222,7 +221,7 @@ void Lint::visitFunction(Function &F) {
 void Lint::visitCallBase(CallBase &I) {
   Value *Callee = I.getCalledOperand();
 
-  visitMemoryReference(I, Callee, MemoryLocation::UnknownSize, 0, nullptr,
+  visitMemoryReference(I, Callee, MemoryLocation::UnknownSize, None, nullptr,
                        MemRef::Callee);
 
   if (Function *F = dyn_cast<Function>(findValue(Callee,
@@ -285,7 +284,7 @@ void Lint::visitCallBase(CallBase &I) {
           Type *Ty =
             cast<PointerType>(Formal->getType())->getElementType();
           visitMemoryReference(I, Actual, DL->getTypeStoreSize(Ty),
-                               DL->getABITypeAlignment(Ty), Ty,
+                               DL->getABITypeAlign(Ty), Ty,
                                MemRef::Read | MemRef::Write);
         }
       }
@@ -321,9 +320,9 @@ void Lint::visitCallBase(CallBase &I) {
       MemCpyInst *MCI = cast<MemCpyInst>(&I);
       // TODO: If the size is known, use it.
       visitMemoryReference(I, MCI->getDest(), MemoryLocation::UnknownSize,
-                           MCI->getDestAlignment(), nullptr, MemRef::Write);
+                           MCI->getDestAlign(), nullptr, MemRef::Write);
       visitMemoryReference(I, MCI->getSource(), MemoryLocation::UnknownSize,
-                           MCI->getSourceAlignment(), nullptr, MemRef::Read);
+                           MCI->getSourceAlign(), nullptr, MemRef::Read);
 
       // Check that the memcpy arguments don't overlap. The AliasAnalysis API
       // isn't expressive enough for what we really want to do. Known partial
@@ -342,10 +341,10 @@ void Lint::visitCallBase(CallBase &I) {
     case Intrinsic::memcpy_inline: {
       MemCpyInlineInst *MCII = cast<MemCpyInlineInst>(&I);
       const uint64_t Size = MCII->getLength()->getValue().getLimitedValue();
-      visitMemoryReference(I, MCII->getDest(), Size, MCII->getDestAlignment(),
+      visitMemoryReference(I, MCII->getDest(), Size, MCII->getDestAlign(),
                            nullptr, MemRef::Write);
-      visitMemoryReference(I, MCII->getSource(), Size,
-                           MCII->getSourceAlignment(), nullptr, MemRef::Read);
+      visitMemoryReference(I, MCII->getSource(), Size, MCII->getSourceAlign(),
+                           nullptr, MemRef::Read);
 
       // Check that the memcpy arguments don't overlap. The AliasAnalysis API
       // isn't expressive enough for what we really want to do. Known partial
@@ -359,16 +358,16 @@ void Lint::visitCallBase(CallBase &I) {
       MemMoveInst *MMI = cast<MemMoveInst>(&I);
       // TODO: If the size is known, use it.
       visitMemoryReference(I, MMI->getDest(), MemoryLocation::UnknownSize,
-                           MMI->getDestAlignment(), nullptr, MemRef::Write);
+                           MMI->getDestAlign(), nullptr, MemRef::Write);
       visitMemoryReference(I, MMI->getSource(), MemoryLocation::UnknownSize,
-                           MMI->getSourceAlignment(), nullptr, MemRef::Read);
+                           MMI->getSourceAlign(), nullptr, MemRef::Read);
       break;
     }
     case Intrinsic::memset: {
       MemSetInst *MSI = cast<MemSetInst>(&I);
       // TODO: If the size is known, use it.
       visitMemoryReference(I, MSI->getDest(), MemoryLocation::UnknownSize,
-                           MSI->getDestAlignment(), nullptr, MemRef::Write);
+                           MSI->getDestAlign(), nullptr, MemRef::Write);
       break;
     }
 
@@ -378,17 +377,17 @@ void Lint::visitCallBase(CallBase &I) {
              &I);
 
       visitMemoryReference(I, I.getArgOperand(0), MemoryLocation::UnknownSize,
-                           0, nullptr, MemRef::Read | MemRef::Write);
+                           None, nullptr, MemRef::Read | MemRef::Write);
       break;
     case Intrinsic::vacopy:
       visitMemoryReference(I, I.getArgOperand(0), MemoryLocation::UnknownSize,
-                           0, nullptr, MemRef::Write);
+                           None, nullptr, MemRef::Write);
       visitMemoryReference(I, I.getArgOperand(1), MemoryLocation::UnknownSize,
-                           0, nullptr, MemRef::Read);
+                           None, nullptr, MemRef::Read);
       break;
     case Intrinsic::vaend:
       visitMemoryReference(I, I.getArgOperand(0), MemoryLocation::UnknownSize,
-                           0, nullptr, MemRef::Read | MemRef::Write);
+                           None, nullptr, MemRef::Read | MemRef::Write);
       break;
 
     case Intrinsic::stackrestore:
@@ -396,7 +395,7 @@ void Lint::visitCallBase(CallBase &I) {
       // stack pointer, which the compiler may read from or write to
       // at any time, so check it for both readability and writeability.
       visitMemoryReference(I, I.getArgOperand(0), MemoryLocation::UnknownSize,
-                           0, nullptr, MemRef::Read | MemRef::Write);
+                           None, nullptr, MemRef::Read | MemRef::Write);
       break;
     }
 }
@@ -414,9 +413,8 @@ void Lint::visitReturnInst(ReturnInst &I) {
 
 // TODO: Check that the reference is in bounds.
 // TODO: Check readnone/readonly function attributes.
-void Lint::visitMemoryReference(Instruction &I,
-                                Value *Ptr, uint64_t Size, unsigned Align,
-                                Type *Ty, unsigned Flags) {
+void Lint::visitMemoryReference(Instruction &I, Value *Ptr, uint64_t Size,
+                                MaybeAlign Align, Type *Ty, unsigned Flags) {
   // If no memory is being referenced, it doesn't matter if the pointer
   // is valid.
   if (Size == 0)
@@ -467,15 +465,13 @@ void Lint::visitMemoryReference(Instruction &I,
     // something we can handle and if so extract the size of this base object
     // along with its alignment.
     uint64_t BaseSize = MemoryLocation::UnknownSize;
-    unsigned BaseAlign = 0;
+    MaybeAlign BaseAlign;
 
     if (AllocaInst *AI = dyn_cast<AllocaInst>(Base)) {
       Type *ATy = AI->getAllocatedType();
       if (!AI->isArrayAllocation() && ATy->isSized())
         BaseSize = DL->getTypeAllocSize(ATy);
-      BaseAlign = AI->getAlignment();
-      if (BaseAlign == 0 && ATy->isSized())
-        BaseAlign = DL->getABITypeAlignment(ATy);
+      BaseAlign = AI->getAlign();
     } else if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Base)) {
       // If the global may be defined differently in another compilation unit
       // then don't warn about funky memory accesses.
@@ -483,9 +479,9 @@ void Lint::visitMemoryReference(Instruction &I,
         Type *GTy = GV->getValueType();
         if (GTy->isSized())
           BaseSize = DL->getTypeAllocSize(GTy);
-        BaseAlign = GV->getAlignment();
-        if (BaseAlign == 0 && GTy->isSized())
-          BaseAlign = DL->getABITypeAlignment(GTy);
+        BaseAlign = GV->getAlign();
+        if (!BaseAlign && GTy->isSized())
+          BaseAlign = DL->getABITypeAlign(GTy);
       }
     }
 
@@ -498,24 +494,24 @@ void Lint::visitMemoryReference(Instruction &I,
 
     // Accesses that say that the memory is more aligned than it is are not
     // defined.
-    if (Align == 0 && Ty && Ty->isSized())
-      Align = DL->getABITypeAlignment(Ty);
-    Assert(!BaseAlign || Align <= MinAlign(BaseAlign, Offset),
-           "Undefined behavior: Memory reference address is misaligned", &I);
+    if (!Align && Ty && Ty->isSized())
+      Align = DL->getABITypeAlign(Ty);
+    if (BaseAlign && Align)
+      Assert(*Align <= commonAlignment(*BaseAlign, Offset),
+             "Undefined behavior: Memory reference address is misaligned", &I);
   }
 }
 
 void Lint::visitLoadInst(LoadInst &I) {
   visitMemoryReference(I, I.getPointerOperand(),
-                       DL->getTypeStoreSize(I.getType()), I.getAlignment(),
+                       DL->getTypeStoreSize(I.getType()), I.getAlign(),
                        I.getType(), MemRef::Read);
 }
 
 void Lint::visitStoreInst(StoreInst &I) {
   visitMemoryReference(I, I.getPointerOperand(),
                        DL->getTypeStoreSize(I.getOperand(0)->getType()),
-                       I.getAlignment(),
-                       I.getOperand(0)->getType(), MemRef::Write);
+                       I.getAlign(), I.getOperand(0)->getType(), MemRef::Write);
 }
 
 void Lint::visitXor(BinaryOperator &I) {
@@ -614,12 +610,12 @@ void Lint::visitAllocaInst(AllocaInst &I) {
 }
 
 void Lint::visitVAArgInst(VAArgInst &I) {
-  visitMemoryReference(I, I.getOperand(0), MemoryLocation::UnknownSize, 0,
+  visitMemoryReference(I, I.getOperand(0), MemoryLocation::UnknownSize, None,
                        nullptr, MemRef::Read | MemRef::Write);
 }
 
 void Lint::visitIndirectBrInst(IndirectBrInst &I) {
-  visitMemoryReference(I, I.getAddress(), MemoryLocation::UnknownSize, 0,
+  visitMemoryReference(I, I.getAddress(), MemoryLocation::UnknownSize, None,
                        nullptr, MemRef::Branchee);
 
   Assert(I.getNumDestinations() != 0,
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index 470a45368087c..e5245225d905d 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -41,6 +41,7 @@ static bool isDereferenceableAndAlignedPointer(
     const Value *V, Align Alignment, const APInt &Size, const DataLayout &DL,
     const Instruction *CtxI, const DominatorTree *DT,
     SmallPtrSetImpl<const Value *> &Visited, unsigned MaxDepth) {
+  assert(V->getType()->isPointerTy() && "Base must be pointer");
 
   // Recursion limit.
   if (MaxDepth-- == 0)
@@ -54,10 +55,11 @@ static bool isDereferenceableAndAlignedPointer(
   // malloc may return null.
 
   // bitcast instructions are no-ops as far as dereferenceability is concerned.
-  if (const BitCastOperator *BC = dyn_cast<BitCastOperator>(V))
-    return isDereferenceableAndAlignedPointer(BC->getOperand(0), Alignment,
-                                              Size, DL, CtxI, DT, Visited,
-                                              MaxDepth);
+  if (const BitCastOperator *BC = dyn_cast<BitCastOperator>(V)) {
+    if (BC->getSrcTy()->isPointerTy())
+      return isDereferenceableAndAlignedPointer(
+          BC->getOperand(0), Alignment, Size, DL, CtxI, DT, Visited, MaxDepth);
+  }
 
   bool CheckForNonNull = false;
   APInt KnownDerefBytes(Size.getBitWidth(),
@@ -352,7 +354,7 @@ Value *llvm::FindAvailableLoadedValue(LoadInst *Load,
                                       BasicBlock *ScanBB,
                                       BasicBlock::iterator &ScanFrom,
                                       unsigned MaxInstsToScan,
-                                      AliasAnalysis *AA, bool *IsLoad,
+                                      AAResults *AA, bool *IsLoad,
                                       unsigned *NumScanedInst) {
   // Don't CSE load that is volatile or anything stronger than unordered.
   if (!Load->isUnordered())
@@ -389,7 +391,7 @@ Value *llvm::FindAvailablePtrLoadStore(Value *Ptr, Type *AccessTy,
                                        bool AtLeastAtomic, BasicBlock *ScanBB,
                                        BasicBlock::iterator &ScanFrom,
                                        unsigned MaxInstsToScan,
-                                       AliasAnalysis *AA, bool *IsLoadCSE,
+                                       AAResults *AA, bool *IsLoadCSE,
                                        unsigned *NumScanedInst) {
   if (MaxInstsToScan == 0)
     MaxInstsToScan = ~0U;
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 3789be757acaa..ae282a7a10952 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -505,7 +505,7 @@ class AccessAnalysis {
   typedef PointerIntPair<Value *, 1, bool> MemAccessInfo;
   typedef SmallVector<MemAccessInfo, 8> MemAccessInfoList;
 
-  AccessAnalysis(const DataLayout &Dl, Loop *TheLoop, AliasAnalysis *AA,
+  AccessAnalysis(const DataLayout &Dl, Loop *TheLoop, AAResults *AA,
                  LoopInfo *LI, MemoryDepChecker::DepCandidates &DA,
                  PredicatedScalarEvolution &PSE)
       : DL(Dl), TheLoop(TheLoop), AST(*AA), LI(LI), DepCands(DA),
@@ -1812,7 +1812,7 @@ bool LoopAccessInfo::canAnalyzeLoop() {
   return true;
 }
 
-void LoopAccessInfo::analyzeLoop(AliasAnalysis *AA, LoopInfo *LI,
+void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI,
                                  const TargetLibraryInfo *TLI,
                                  DominatorTree *DT) {
   typedef SmallPtrSet<Value*, 16> ValueSet;
@@ -1835,6 +1835,10 @@ void LoopAccessInfo::analyzeLoop(AliasAnalysis *AA, LoopInfo *LI,
 
   const bool IsAnnotatedParallel = TheLoop->isAnnotatedParallel();
 
+  const bool EnableMemAccessVersioningOfLoop =
+      EnableMemAccessVersioning &&
+      !TheLoop->getHeader()->getParent()->hasOptSize();
+
   // For each block.
   for (BasicBlock *BB : TheLoop->blocks()) {
     // Scan the BB and collect legal loads and stores. Also detect any
@@ -1890,7 +1894,7 @@ void LoopAccessInfo::analyzeLoop(AliasAnalysis *AA, LoopInfo *LI,
         NumLoads++;
         Loads.push_back(Ld);
         DepChecker->addAccess(Ld);
-        if (EnableMemAccessVersioning)
+        if (EnableMemAccessVersioningOfLoop)
           collectStridedAccess(Ld);
         continue;
       }
@@ -1914,7 +1918,7 @@ void LoopAccessInfo::analyzeLoop(AliasAnalysis *AA, LoopInfo *LI,
         NumStores++;
         Stores.push_back(St);
         DepChecker->addAccess(St);
-        if (EnableMemAccessVersioning)
+        if (EnableMemAccessVersioningOfLoop)
           collectStridedAccess(St);
       }
     } // Next instr.
@@ -2205,7 +2209,7 @@ void LoopAccessInfo::collectStridedAccess(Value *MemAccess) {
 }
 
 LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE,
-                               const TargetLibraryInfo *TLI, AliasAnalysis *AA,
+                               const TargetLibraryInfo *TLI, AAResults *AA,
                                DominatorTree *DT, LoopInfo *LI)
     : PSE(std::make_unique<PredicatedScalarEvolution>(*SE, *L)),
       PtrRtChecking(std::make_unique<RuntimePointerChecking>(SE)),
diff --git a/llvm/lib/Analysis/MLInlineAdvisor.cpp b/llvm/lib/Analysis/MLInlineAdvisor.cpp
new file mode 100644
index 0000000000000..45873f260f23d
--- /dev/null
+++ b/llvm/lib/Analysis/MLInlineAdvisor.cpp
@@ -0,0 +1,301 @@
+//===- MLInlineAdvisor.cpp - machine learned InlineAdvisor ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the interface between the inliner and a learned model.
+// It delegates model evaluation to either the AOT compiled model (the
+// 'release' mode) or a runtime-loaded model (the 'development' case).
+//
+//===----------------------------------------------------------------------===//
+#include <limits>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "llvm/ADT/SCCIterator.h"
+#include "llvm/Analysis/CallGraph.h"
+#include "llvm/Analysis/InlineCost.h"
+#include "llvm/Analysis/InlineFeaturesAnalysis.h"
+#include "llvm/Analysis/MLInlineAdvisor.h"
+#include "llvm/Analysis/MLModelRunner.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Path.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "inline-ml"
+
+static cl::opt<float> SizeIncreaseThreshold(
+    "ml-advisor-size-increase-threshold", cl::Hidden,
+    cl::desc("Maximum factor by which expected native size may increase before "
+             "blocking any further inlining."),
+    cl::init(2.0));
+
+const std::array<std::string, NumberOfFeatures> llvm::FeatureNameMap{
+#define POPULATE_NAMES(INDEX_NAME, NAME, COMMENT) NAME,
+    INLINE_FEATURE_ITERATOR(POPULATE_NAMES)
+#undef POPULATE_NAMES
+};
+
+const char *const llvm::DecisionName = "inlining_decision";
+const char *const llvm::DefaultDecisionName = "inlining_default";
+const char *const llvm::RewardName = "delta_size";
+
+CallBase *getInlinableCS(Instruction &I) {
+  if (auto *CS = dyn_cast<CallBase>(&I))
+    if (Function *Callee = CS->getCalledFunction()) {
+      if (!Callee->isDeclaration()) {
+        return CS;
+      }
+    }
+  return nullptr;
+}
+
+MLInlineAdvisor::MLInlineAdvisor(Module &M, ModuleAnalysisManager &MAM,
+                                 std::unique_ptr<MLModelRunner> Runner)
+    : InlineAdvisor(
+          MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager()),
+      M(M), ModelRunner(std::move(Runner)), CG(new CallGraph(M)),
+      InitialIRSize(getModuleIRSize()), CurrentIRSize(InitialIRSize) {
+  assert(ModelRunner);
+
+  // Extract the 'call site height' feature - the position of a call site
+  // relative to the farthest statically reachable SCC node. We don't mutate
+  // this value while inlining happens. Empirically, this feature proved
+  // critical in behavioral cloning - i.e. training a model to mimic the manual
+  // heuristic's decisions - and, thus, equally important for training for
+  // improvement.
+  for (auto I = scc_begin(CG.get()); !I.isAtEnd(); ++I) {
+    const std::vector<CallGraphNode *> &CGNodes = *I;
+    unsigned Level = 0;
+    for (auto *CGNode : CGNodes) {
+      Function *F = CGNode->getFunction();
+      if (!F || F->isDeclaration())
+        continue;
+      for (auto &I : instructions(F)) {
+        if (auto *CS = getInlinableCS(I)) {
+          auto *Called = CS->getCalledFunction();
+          auto Pos = FunctionLevels.find(Called);
+          // In bottom up traversal, an inlinable callee is either in the
+          // same SCC, or to a function in a visited SCC. So not finding its
+          // level means we haven't visited it yet, meaning it's in this SCC.
+          if (Pos == FunctionLevels.end())
+            continue;
+          Level = std::max(Level, Pos->second + 1);
+        }
+      }
+    }
+    for (auto *CGNode : CGNodes) {
+      Function *F = CGNode->getFunction();
+      if (F && !F->isDeclaration())
+        FunctionLevels[F] = Level;
+    }
+  }
+}
+
+void MLInlineAdvisor::onPassEntry() {
+  // Function passes executed between InlinerPass runs may have changed the
+  // module-wide features.
+  NodeCount = 0;
+  EdgeCount = 0;
+  for (auto &F : M)
+    if (!F.isDeclaration()) {
+      ++NodeCount;
+      EdgeCount += getLocalCalls(F);
+    }
+}
+
+int64_t MLInlineAdvisor::getLocalCalls(Function &F) {
+  return FAM.getResult<InlineFeaturesAnalysis>(F).DirectCallsToDefinedFunctions;
+}
+
+// Update the internal state of the advisor, and force invalidate feature
+// analysis. Currently, we maintain minimal (and very simple) global state - the
+// number of functions and the number of static calls. We also keep track of the
+// total IR size in this module, to stop misbehaving policies at a certain bloat
+// factor (SizeIncreaseThreshold)
+void MLInlineAdvisor::onSuccessfulInlining(const MLInlineAdvice &Advice,
+                                           bool CalleeWasDeleted) {
+  assert(!ForceStop);
+  Function *Caller = Advice.getCaller();
+  Function *Callee = Advice.getCallee();
+
+  // The caller features aren't valid anymore.
+  FAM.invalidate<InlineFeaturesAnalysis>(*Caller);
+  int64_t IRSizeAfter =
+      getIRSize(*Caller) + (CalleeWasDeleted ? 0 : Advice.CalleeIRSize);
+  CurrentIRSize += IRSizeAfter - (Advice.CallerIRSize + Advice.CalleeIRSize);
+  if (CurrentIRSize > SizeIncreaseThreshold * InitialIRSize)
+    ForceStop = true;
+
+  // We can delta-update module-wide features. We know the inlining only changed
+  // the caller, and maybe the callee (by deleting the latter).
+  // Nodes are simple to update.
+  // For edges, we 'forget' the edges that the caller and callee used to have
+  // before inlining, and add back what they currently have together.
+  int64_t NewCallerAndCalleeEdges =
+      FAM.getResult<InlineFeaturesAnalysis>(*Caller)
+          .DirectCallsToDefinedFunctions;
+
+  if (CalleeWasDeleted)
+    --NodeCount;
+  else
+    NewCallerAndCalleeEdges += FAM.getResult<InlineFeaturesAnalysis>(*Callee)
+                                   .DirectCallsToDefinedFunctions;
+  EdgeCount += (NewCallerAndCalleeEdges - Advice.CallerAndCalleeEdges);
+  assert(CurrentIRSize >= 0 && EdgeCount >= 0 && NodeCount >= 0);
+}
+
+int64_t MLInlineAdvisor::getModuleIRSize() const {
+  int64_t Ret = 0;
+  for (auto &F : CG->getModule())
+    if (!F.isDeclaration())
+      Ret += getIRSize(F);
+  return Ret;
+}
+
+std::unique_ptr<InlineAdvice> MLInlineAdvisor::getAdvice(CallBase &CB) {
+  auto &Caller = *CB.getCaller();
+  auto &Callee = *CB.getCalledFunction();
+
+  auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
+    return FAM.getResult<AssumptionAnalysis>(F);
+  };
+  auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
+    return FAM.getResult<TargetLibraryAnalysis>(F);
+  };
+
+  auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee);
+  auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(Caller);
+
+  auto TrivialDecision =
+      llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI);
+
+  // If this is a "never inline" case, there won't be any changes to internal
+  // state we need to track, so we can just return the base InlineAdvice, which
+  // will do nothing interesting.
+  // Same thing if this is a recursive case.
+  if ((TrivialDecision.hasValue() && !TrivialDecision->isSuccess()) ||
+      &Caller == &Callee)
+    return std::make_unique<InlineAdvice>(this, CB, ORE, false);
+
+  bool Mandatory = TrivialDecision.hasValue() && TrivialDecision->isSuccess();
+
+  // If we need to stop, we won't want to track anymore any state changes, so
+  // we just return the base InlineAdvice, which acts as a noop.
+  if (ForceStop) {
+    ORE.emit([&] {
+      return OptimizationRemarkMissed(DEBUG_TYPE, "ForceStop", &CB)
+             << "Won't attempt inlining because module size grew too much.";
+    });
+    return std::make_unique<InlineAdvice>(this, CB, ORE, Mandatory);
+  }
+
+  int CostEstimate = 0;
+  if (!Mandatory) {
+    auto IsCallSiteInlinable =
+        llvm::getInliningCostEstimate(CB, TIR, GetAssumptionCache);
+    if (!IsCallSiteInlinable) {
+      // We can't inline this for correctness reasons, so return the base
+      // InlineAdvice, as we don't care about tracking any state changes (which
+      // won't happen).
+      return std::make_unique<InlineAdvice>(this, CB, ORE, false);
+    }
+    CostEstimate = *IsCallSiteInlinable;
+  }
+
+  if (Mandatory)
+    return getMandatoryAdvice(CB, ORE);
+
+  auto NrCtantParams = 0;
+  for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
+    NrCtantParams += (isa<Constant>(*I));
+  }
+
+  auto &CallerBefore = FAM.getResult<InlineFeaturesAnalysis>(Caller);
+  auto &CalleeBefore = FAM.getResult<InlineFeaturesAnalysis>(Callee);
+
+  ModelRunner->setFeature(FeatureIndex::CalleeBasicBlockCount,
+                          CalleeBefore.BasicBlockCount);
+  ModelRunner->setFeature(FeatureIndex::CallSiteHeight,
+                          FunctionLevels[&Caller]);
+  ModelRunner->setFeature(FeatureIndex::NodeCount, NodeCount);
+  ModelRunner->setFeature(FeatureIndex::NrCtantParams, NrCtantParams);
+  ModelRunner->setFeature(FeatureIndex::CostEstimate, CostEstimate);
+  ModelRunner->setFeature(FeatureIndex::EdgeCount, EdgeCount);
+  ModelRunner->setFeature(FeatureIndex::CallerUsers, CallerBefore.Uses);
+  ModelRunner->setFeature(FeatureIndex::CallerConditionallyExecutedBlocks,
+                          CallerBefore.BlocksReachedFromConditionalInstruction);
+  ModelRunner->setFeature(FeatureIndex::CallerBasicBlockCount,
+                          CallerBefore.BasicBlockCount);
+  ModelRunner->setFeature(FeatureIndex::CalleeConditionallyExecutedBlocks,
+                          CalleeBefore.BlocksReachedFromConditionalInstruction);
+  ModelRunner->setFeature(FeatureIndex::CalleeUsers, CalleeBefore.Uses);
+  return getAdviceFromModel(CB, ORE);
+}
+
+std::unique_ptr<MLInlineAdvice>
+MLInlineAdvisor::getAdviceFromModel(CallBase &CB,
+                                    OptimizationRemarkEmitter &ORE) {
+  return std::make_unique<MLInlineAdvice>(this, CB, ORE, ModelRunner->run());
+}
+
+std::unique_ptr<MLInlineAdvice>
+MLInlineAdvisor::getMandatoryAdvice(CallBase &CB,
+                                    OptimizationRemarkEmitter &ORE) {
+  return std::make_unique<MLInlineAdvice>(this, CB, ORE, true);
+}
+
+void MLInlineAdvice::reportContextForRemark(
+    DiagnosticInfoOptimizationBase &OR) {
+  using namespace ore;
+  OR << NV("Callee", Callee->getName());
+  for (size_t I = 0; I < NumberOfFeatures; ++I)
+    OR << NV(FeatureNameMap[I], getAdvisor()->getModelRunner().getFeature(I));
+  OR << NV("ShouldInline", isInliningRecommended());
+}
+
+void MLInlineAdvice::recordInliningImpl() {
+  ORE.emit([&]() {
+    OptimizationRemark R(DEBUG_TYPE, "InliningSuccess", DLoc, Block);
+    reportContextForRemark(R);
+    return R;
+  });
+  getAdvisor()->onSuccessfulInlining(*this, /*CalleeWasDeleted*/ false);
+}
+
+void MLInlineAdvice::recordInliningWithCalleeDeletedImpl() {
+  ORE.emit([&]() {
+    OptimizationRemark R(DEBUG_TYPE, "InliningSuccessWithCalleeDeleted", DLoc,
+                         Block);
+    reportContextForRemark(R);
+    return R;
+  });
+  getAdvisor()->onSuccessfulInlining(*this, /*CalleeWasDeleted*/ true);
+}
+
+void MLInlineAdvice::recordUnsuccessfulInliningImpl(
+    const InlineResult &Result) {
+  ORE.emit([&]() {
+    OptimizationRemarkMissed R(DEBUG_TYPE, "InliningAttemptedAndUnsuccessful",
+                               DLoc, Block);
+    reportContextForRemark(R);
+    return R;
+  });
+}
+void MLInlineAdvice::recordUnattemptedInliningImpl() {
+  ORE.emit([&]() {
+    OptimizationRemarkMissed R(DEBUG_TYPE, "IniningNotAttempted", DLoc, Block);
+    reportContextForRemark(R);
+    return R;
+  });
+}
\ No newline at end of file
diff --git a/llvm/lib/Analysis/MemDepPrinter.cpp b/llvm/lib/Analysis/MemDepPrinter.cpp
index 2c57e63251c66..9524ec96bb61a 100644
--- a/llvm/lib/Analysis/MemDepPrinter.cpp
+++ b/llvm/lib/Analysis/MemDepPrinter.cpp
@@ -10,6 +10,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/SetVector.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
 #include "llvm/Analysis/Passes.h"
 #include "llvm/IR/InstIterator.h"
@@ -17,6 +18,7 @@
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
+
 using namespace llvm;
 
 namespace {
diff --git a/llvm/lib/Analysis/MemDerefPrinter.cpp b/llvm/lib/Analysis/MemDerefPrinter.cpp
index f8f4f9423f62a..564410b8af083 100644
--- a/llvm/lib/Analysis/MemDerefPrinter.cpp
+++ b/llvm/lib/Analysis/MemDerefPrinter.cpp
@@ -10,9 +10,11 @@
 #include "llvm/Analysis/Passes.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Instructions.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
diff --git a/llvm/lib/Analysis/OrderedInstructions.cpp b/llvm/lib/Analysis/OrderedInstructions.cpp
deleted file mode 100644
index 58d9a618184ab..0000000000000
--- a/llvm/lib/Analysis/OrderedInstructions.cpp
+++ /dev/null
@@ -1,59 +0,0 @@
-//===-- OrderedInstructions.cpp - Instruction dominance function ---------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines utility to check dominance relation of 2 instructions.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Analysis/OrderedInstructions.h"
-#include "llvm/IR/Dominators.h"
-
-using namespace llvm;
-
-bool OrderedInstructions::localDominates(const Instruction *InstA,
-                                         const Instruction *InstB) const {
-  assert(InstA->getParent() == InstB->getParent() &&
-         "Instructions must be in the same basic block");
-
-  return InstA->comesBefore(InstB);
-}
-
-/// Given 2 instructions, check for dominance relation if the instructions are
-/// in the same basic block. Otherwise, use dominator tree.
-bool OrderedInstructions::dominates(const Instruction *InstA,
-                                    const Instruction *InstB) const {
-  // Use ordered basic block to do dominance check in case the 2 instructions
-  // are in the same basic block.
-  if (InstA->getParent() == InstB->getParent())
-    return localDominates(InstA, InstB);
-  return DT->dominates(InstA->getParent(), InstB->getParent());
-}
-
-bool OrderedInstructions::dfsBefore(const Instruction *InstA,
-                                    const Instruction *InstB) const {
-  // Use ordered basic block in case the 2 instructions are in the same basic
-  // block.
-  if (InstA->getParent() == InstB->getParent())
-    return localDominates(InstA, InstB);
-
-  DomTreeNode *DA = DT->getNode(InstA->getParent());
-  DomTreeNode *DB = DT->getNode(InstB->getParent());
-  return DA->getDFSNumIn() < DB->getDFSNumIn();
-}
-
-bool OrderedInstructions::domTreeLevelBefore(const Instruction *InstA,
-                                             const Instruction *InstB) const {
-  // Use ordered basic block in case the 2 instructions are in the same basic
-  // block.
-  if (InstA->getParent() == InstB->getParent())
-    return localDominates(InstA, InstB);
-
-  DomTreeNode *DA = DT->getNode(InstA->getParent());
-  DomTreeNode *DB = DT->getNode(InstB->getParent());
-  return DA->getLevel() < DB->getLevel();
-}
diff --git a/llvm/lib/Analysis/ProfileSummaryInfo.cpp b/llvm/lib/Analysis/ProfileSummaryInfo.cpp
index 655fc244cb39c..c9671d4f5c2e1 100644
--- a/llvm/lib/Analysis/ProfileSummaryInfo.cpp
+++ b/llvm/lib/Analysis/ProfileSummaryInfo.cpp
@@ -19,6 +19,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/ProfileSummary.h"
 #include "llvm/InitializePasses.h"
+#include "llvm/ProfileData/ProfileCommon.h"
 #include "llvm/Support/CommandLine.h"
 using namespace llvm;
 
@@ -86,19 +87,6 @@ static cl::opt<double> PartialSampleProfileWorkingSetSizeScaleFactor(
              "and the factor to scale the working set size to use the same "
              "shared thresholds as PGO."));
 
-// Find the summary entry for a desired percentile of counts.
-static const ProfileSummaryEntry &getEntryForPercentile(SummaryEntryVector &DS,
-                                                        uint64_t Percentile) {
-  auto It = partition_point(DS, [=](const ProfileSummaryEntry &Entry) {
-    return Entry.Cutoff < Percentile;
-  });
-  // The required percentile has to be <= one of the percentiles in the
-  // detailed summary.
-  if (It == DS.end())
-    report_fatal_error("Desired percentile exceeds the maximum cutoff");
-  return *It;
-}
-
 // The profile summary metadata may be attached either by the frontend or by
 // any backend passes (IR level instrumentation, for example). This method
 // checks if the Summary is null and if so checks if the summary metadata is now
@@ -284,13 +272,13 @@ bool ProfileSummaryInfo::isFunctionEntryCold(const Function *F) const {
 /// Compute the hot and cold thresholds.
 void ProfileSummaryInfo::computeThresholds() {
   auto &DetailedSummary = Summary->getDetailedSummary();
-  auto &HotEntry =
-      getEntryForPercentile(DetailedSummary, ProfileSummaryCutoffHot);
+  auto &HotEntry = ProfileSummaryBuilder::getEntryForPercentile(
+      DetailedSummary, ProfileSummaryCutoffHot);
   HotCountThreshold = HotEntry.MinCount;
   if (ProfileSummaryHotCount.getNumOccurrences() > 0)
     HotCountThreshold = ProfileSummaryHotCount;
-  auto &ColdEntry =
-      getEntryForPercentile(DetailedSummary, ProfileSummaryCutoffCold);
+  auto &ColdEntry = ProfileSummaryBuilder::getEntryForPercentile(
+      DetailedSummary, ProfileSummaryCutoffCold);
   ColdCountThreshold = ColdEntry.MinCount;
   if (ProfileSummaryColdCount.getNumOccurrences() > 0)
     ColdCountThreshold = ProfileSummaryColdCount;
@@ -324,8 +312,8 @@ ProfileSummaryInfo::computeThreshold(int PercentileCutoff) const {
     return iter->second;
   }
   auto &DetailedSummary = Summary->getDetailedSummary();
-  auto &Entry =
-      getEntryForPercentile(DetailedSummary, PercentileCutoff);
+  auto &Entry = ProfileSummaryBuilder::getEntryForPercentile(DetailedSummary,
+                                                             PercentileCutoff);
   uint64_t CountThreshold = Entry.MinCount;
   ThresholdCache[PercentileCutoff] = CountThreshold;
   return CountThreshold;
diff --git a/llvm/lib/Analysis/ReleaseModeModelRunner.cpp b/llvm/lib/Analysis/ReleaseModeModelRunner.cpp
new file mode 100644
index 0000000000000..4c0ffbc17ff7a
--- /dev/null
+++ b/llvm/lib/Analysis/ReleaseModeModelRunner.cpp
@@ -0,0 +1,87 @@
+//===- ReleaseModeModelRunner.cpp - Fast, precompiled model runner  -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a model runner wrapping an AOT compiled ML model.
+// Only inference is supported.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/InlineModelFeatureMaps.h"
+#include "llvm/Analysis/MLInlineAdvisor.h"
+
+// codegen-ed file
+#include "InlinerSizeModel.h" // NOLINT
+
+#include <memory>
+#include <vector>
+
+using namespace llvm;
+namespace {
+
+static const char *const FeedPrefix = "feed_";
+static const char *const FetchPrefix = "fetch_";
+
+/// MLModelRunner - production mode implementation. It uses a AOT-compiled
+/// SavedModel for efficient execution.
+class ReleaseModeModelRunner final : public MLModelRunner {
+public:
+  ReleaseModeModelRunner(LLVMContext &Ctx);
+  virtual ~ReleaseModeModelRunner() = default;
+
+  bool run() override;
+
+  void setFeature(FeatureIndex Index, int64_t Value) override;
+  int64_t getFeature(int Index) const override;
+
+private:
+  std::vector<int32_t> FeatureIndices;
+  int32_t ResultIndex = -1;
+  std::unique_ptr<llvm::InlinerSizeModel> CompiledModel;
+};
+} // namespace
+
+ReleaseModeModelRunner::ReleaseModeModelRunner(LLVMContext &Ctx)
+    : MLModelRunner(Ctx),
+      CompiledModel(std::make_unique<llvm::InlinerSizeModel>()) {
+  assert(CompiledModel && "The CompiledModel should be valid");
+
+  FeatureIndices.reserve(NumberOfFeatures);
+
+  for (size_t I = 0; I < NumberOfFeatures; ++I) {
+    const int Index =
+        CompiledModel->LookupArgIndex(FeedPrefix + FeatureNameMap[I]);
+    assert(Index >= 0 && "Cannot find Feature in inlining model");
+    FeatureIndices[I] = Index;
+  }
+
+  ResultIndex =
+      CompiledModel->LookupResultIndex(std::string(FetchPrefix) + DecisionName);
+  assert(ResultIndex >= 0 && "Cannot find DecisionName in inlining model");
+}
+
+int64_t ReleaseModeModelRunner::getFeature(int Index) const {
+  return *static_cast<int64_t *>(
+      CompiledModel->arg_data(FeatureIndices[Index]));
+}
+
+void ReleaseModeModelRunner::setFeature(FeatureIndex Index, int64_t Value) {
+  *static_cast<int64_t *>(CompiledModel->arg_data(
+      FeatureIndices[static_cast<size_t>(Index)])) = Value;
+}
+
+bool ReleaseModeModelRunner::run() {
+  CompiledModel->Run();
+  return static_cast<bool>(
+      *static_cast<int64_t *>(CompiledModel->result_data(ResultIndex)));
+}
+
+std::unique_ptr<InlineAdvisor>
+llvm::getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM) {
+  auto AOTRunner = std::make_unique<ReleaseModeModelRunner>(M.getContext());
+  return std::make_unique<MLInlineAdvisor>(M, MAM, std::move(AOTRunner));
+}
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index eb7fd1dc36f6e..48c686b732608 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -79,6 +79,7 @@
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/ScalarEvolutionDivision.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -847,30 +848,6 @@ static void GroupByComplexity(SmallVectorImpl<const SCEV *> &Ops,
   }
 }
 
-// Returns the size of the SCEV S.
-static inline int sizeOfSCEV(const SCEV *S) {
-  struct FindSCEVSize {
-    int Size = 0;
-
-    FindSCEVSize() = default;
-
-    bool follow(const SCEV *S) {
-      ++Size;
-      // Keep looking at all operands of S.
-      return true;
-    }
-
-    bool isDone() const {
-      return false;
-    }
-  };
-
-  FindSCEVSize F;
-  SCEVTraversal<FindSCEVSize> ST(F);
-  ST.visitAll(S);
-  return F.Size;
-}
-
 /// Returns true if \p Ops contains a huge SCEV (the subtree of S contains at
 /// least HugeExprThreshold nodes).
 static bool hasHugeExpression(ArrayRef<const SCEV *> Ops) {
@@ -879,238 +856,6 @@ static bool hasHugeExpression(ArrayRef<const SCEV *> Ops) {
   });
 }
 
-namespace {
-
-struct SCEVDivision : public SCEVVisitor<SCEVDivision, void> {
-public:
-  // Computes the Quotient and Remainder of the division of Numerator by
-  // Denominator.
-  static void divide(ScalarEvolution &SE, const SCEV *Numerator,
-                     const SCEV *Denominator, const SCEV **Quotient,
-                     const SCEV **Remainder) {
-    assert(Numerator && Denominator && "Uninitialized SCEV");
-
-    SCEVDivision D(SE, Numerator, Denominator);
-
-    // Check for the trivial case here to avoid having to check for it in the
-    // rest of the code.
-    if (Numerator == Denominator) {
-      *Quotient = D.One;
-      *Remainder = D.Zero;
-      return;
-    }
-
-    if (Numerator->isZero()) {
-      *Quotient = D.Zero;
-      *Remainder = D.Zero;
-      return;
-    }
-
-    // A simple case when N/1. The quotient is N.
-    if (Denominator->isOne()) {
-      *Quotient = Numerator;
-      *Remainder = D.Zero;
-      return;
-    }
-
-    // Split the Denominator when it is a product.
-    if (const SCEVMulExpr *T = dyn_cast<SCEVMulExpr>(Denominator)) {
-      const SCEV *Q, *R;
-      *Quotient = Numerator;
-      for (const SCEV *Op : T->operands()) {
-        divide(SE, *Quotient, Op, &Q, &R);
-        *Quotient = Q;
-
-        // Bail out when the Numerator is not divisible by one of the terms of
-        // the Denominator.
-        if (!R->isZero()) {
-          *Quotient = D.Zero;
-          *Remainder = Numerator;
-          return;
-        }
-      }
-      *Remainder = D.Zero;
-      return;
-    }
-
-    D.visit(Numerator);
-    *Quotient = D.Quotient;
-    *Remainder = D.Remainder;
-  }
-
-  // Except in the trivial case described above, we do not know how to divide
-  // Expr by Denominator for the following functions with empty implementation.
-  void visitTruncateExpr(const SCEVTruncateExpr *Numerator) {}
-  void visitZeroExtendExpr(const SCEVZeroExtendExpr *Numerator) {}
-  void visitSignExtendExpr(const SCEVSignExtendExpr *Numerator) {}
-  void visitUDivExpr(const SCEVUDivExpr *Numerator) {}
-  void visitSMaxExpr(const SCEVSMaxExpr *Numerator) {}
-  void visitUMaxExpr(const SCEVUMaxExpr *Numerator) {}
-  void visitSMinExpr(const SCEVSMinExpr *Numerator) {}
-  void visitUMinExpr(const SCEVUMinExpr *Numerator) {}
-  void visitUnknown(const SCEVUnknown *Numerator) {}
-  void visitCouldNotCompute(const SCEVCouldNotCompute *Numerator) {}
-
-  void visitConstant(const SCEVConstant *Numerator) {
-    if (const SCEVConstant *D = dyn_cast<SCEVConstant>(Denominator)) {
-      APInt NumeratorVal = Numerator->getAPInt();
-      APInt DenominatorVal = D->getAPInt();
-      uint32_t NumeratorBW = NumeratorVal.getBitWidth();
-      uint32_t DenominatorBW = DenominatorVal.getBitWidth();
-
-      if (NumeratorBW > DenominatorBW)
-        DenominatorVal = DenominatorVal.sext(NumeratorBW);
-      else if (NumeratorBW < DenominatorBW)
-        NumeratorVal = NumeratorVal.sext(DenominatorBW);
-
-      APInt QuotientVal(NumeratorVal.getBitWidth(), 0);
-      APInt RemainderVal(NumeratorVal.getBitWidth(), 0);
-      APInt::sdivrem(NumeratorVal, DenominatorVal, QuotientVal, RemainderVal);
-      Quotient = SE.getConstant(QuotientVal);
-      Remainder = SE.getConstant(RemainderVal);
-      return;
-    }
-  }
-
-  void visitAddRecExpr(const SCEVAddRecExpr *Numerator) {
-    const SCEV *StartQ, *StartR, *StepQ, *StepR;
-    if (!Numerator->isAffine())
-      return cannotDivide(Numerator);
-    divide(SE, Numerator->getStart(), Denominator, &StartQ, &StartR);
-    divide(SE, Numerator->getStepRecurrence(SE), Denominator, &StepQ, &StepR);
-    // Bail out if the types do not match.
-    Type *Ty = Denominator->getType();
-    if (Ty != StartQ->getType() || Ty != StartR->getType() ||
-        Ty != StepQ->getType() || Ty != StepR->getType())
-      return cannotDivide(Numerator);
-    Quotient = SE.getAddRecExpr(StartQ, StepQ, Numerator->getLoop(),
-                                Numerator->getNoWrapFlags());
-    Remainder = SE.getAddRecExpr(StartR, StepR, Numerator->getLoop(),
-                                 Numerator->getNoWrapFlags());
-  }
-
-  void visitAddExpr(const SCEVAddExpr *Numerator) {
-    SmallVector<const SCEV *, 2> Qs, Rs;
-    Type *Ty = Denominator->getType();
-
-    for (const SCEV *Op : Numerator->operands()) {
-      const SCEV *Q, *R;
-      divide(SE, Op, Denominator, &Q, &R);
-
-      // Bail out if types do not match.
-      if (Ty != Q->getType() || Ty != R->getType())
-        return cannotDivide(Numerator);
-
-      Qs.push_back(Q);
-      Rs.push_back(R);
-    }
-
-    if (Qs.size() == 1) {
-      Quotient = Qs[0];
-      Remainder = Rs[0];
-      return;
-    }
-
-    Quotient = SE.getAddExpr(Qs);
-    Remainder = SE.getAddExpr(Rs);
-  }
-
-  void visitMulExpr(const SCEVMulExpr *Numerator) {
-    SmallVector<const SCEV *, 2> Qs;
-    Type *Ty = Denominator->getType();
-
-    bool FoundDenominatorTerm = false;
-    for (const SCEV *Op : Numerator->operands()) {
-      // Bail out if types do not match.
-      if (Ty != Op->getType())
-        return cannotDivide(Numerator);
-
-      if (FoundDenominatorTerm) {
-        Qs.push_back(Op);
-        continue;
-      }
-
-      // Check whether Denominator divides one of the product operands.
-      const SCEV *Q, *R;
-      divide(SE, Op, Denominator, &Q, &R);
-      if (!R->isZero()) {
-        Qs.push_back(Op);
-        continue;
-      }
-
-      // Bail out if types do not match.
-      if (Ty != Q->getType())
-        return cannotDivide(Numerator);
-
-      FoundDenominatorTerm = true;
-      Qs.push_back(Q);
-    }
-
-    if (FoundDenominatorTerm) {
-      Remainder = Zero;
-      if (Qs.size() == 1)
-        Quotient = Qs[0];
-      else
-        Quotient = SE.getMulExpr(Qs);
-      return;
-    }
-
-    if (!isa<SCEVUnknown>(Denominator))
-      return cannotDivide(Numerator);
-
-    // The Remainder is obtained by replacing Denominator by 0 in Numerator.
-    ValueToValueMap RewriteMap;
-    RewriteMap[cast<SCEVUnknown>(Denominator)->getValue()] =
-        cast<SCEVConstant>(Zero)->getValue();
-    Remainder = SCEVParameterRewriter::rewrite(Numerator, SE, RewriteMap, true);
-
-    if (Remainder->isZero()) {
-      // The Quotient is obtained by replacing Denominator by 1 in Numerator.
-      RewriteMap[cast<SCEVUnknown>(Denominator)->getValue()] =
-          cast<SCEVConstant>(One)->getValue();
-      Quotient =
-          SCEVParameterRewriter::rewrite(Numerator, SE, RewriteMap, true);
-      return;
-    }
-
-    // Quotient is (Numerator - Remainder) divided by Denominator.
-    const SCEV *Q, *R;
-    const SCEV *Diff = SE.getMinusSCEV(Numerator, Remainder);
-    // This SCEV does not seem to simplify: fail the division here.
-    if (sizeOfSCEV(Diff) > sizeOfSCEV(Numerator))
-      return cannotDivide(Numerator);
-    divide(SE, Diff, Denominator, &Q, &R);
-    if (R != Zero)
-      return cannotDivide(Numerator);
-    Quotient = Q;
-  }
-
-private:
-  SCEVDivision(ScalarEvolution &S, const SCEV *Numerator,
-               const SCEV *Denominator)
-      : SE(S), Denominator(Denominator) {
-    Zero = SE.getZero(Denominator->getType());
-    One = SE.getOne(Denominator->getType());
-
-    // We generally do not know how to divide Expr by Denominator. We
-    // initialize the division to a "cannot divide" state to simplify the rest
-    // of the code.
-    cannotDivide(Numerator);
-  }
-
-  // Convenience function for giving up on the division. We set the quotient to
-  // be equal to zero and the remainder to be equal to the numerator.
-  void cannotDivide(const SCEV *Numerator) {
-    Quotient = Zero;
-    Remainder = Numerator;
-  }
-
-  ScalarEvolution &SE;
-  const SCEV *Denominator, *Quotient, *Remainder, *Zero, *One;
-};
-
-} // end anonymous namespace
-
 //===----------------------------------------------------------------------===//
 //                      Simple SCEV method implementations
 //===----------------------------------------------------------------------===//
@@ -1608,7 +1353,7 @@ bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start,
 static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
                                             const SCEVConstant *ConstantTerm,
                                             const SCEVAddExpr *WholeAddExpr) {
-  const APInt C = ConstantTerm->getAPInt();
+  const APInt &C = ConstantTerm->getAPInt();
   const unsigned BitWidth = C.getBitWidth();
   // Find number of trailing zeros of (x + y + ...) w/o the C first:
   uint32_t TZ = BitWidth;
@@ -3572,7 +3317,10 @@ ScalarEvolution::getGEPExpr(GEPOperator *GEP,
   }
 
   // Add the total offset from all the GEP indices to the base.
-  return getAddExpr(BaseExpr, TotalOffset, Wrap);
+  auto *GEPExpr = getAddExpr(BaseExpr, TotalOffset, Wrap);
+  assert(BaseExpr->getType() == GEPExpr->getType() &&
+         "GEP should not change type mid-flight.");
+  return GEPExpr;
 }
 
 std::tuple<SCEV *, FoldingSetNodeID, void *>
@@ -4208,23 +3956,25 @@ const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) {
   if (!V->getType()->isPointerTy())
     return V;
 
-  if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(V)) {
-    return getPointerBase(Cast->getOperand());
-  } else if (const SCEVNAryExpr *NAry = dyn_cast<SCEVNAryExpr>(V)) {
-    const SCEV *PtrOp = nullptr;
-    for (const SCEV *NAryOp : NAry->operands()) {
-      if (NAryOp->getType()->isPointerTy()) {
-        // Cannot find the base of an expression with multiple pointer operands.
-        if (PtrOp)
-          return V;
-        PtrOp = NAryOp;
+  while (true) {
+    if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(V)) {
+      V = Cast->getOperand();
+    } else if (const SCEVNAryExpr *NAry = dyn_cast<SCEVNAryExpr>(V)) {
+      const SCEV *PtrOp = nullptr;
+      for (const SCEV *NAryOp : NAry->operands()) {
+        if (NAryOp->getType()->isPointerTy()) {
+          // Cannot find the base of an expression with multiple pointer ops.
+          if (PtrOp)
+            return V;
+          PtrOp = NAryOp;
+        }
       }
-    }
-    if (!PtrOp)
+      if (!PtrOp) // All operands were non-pointer.
+        return V;
+      V = PtrOp;
+    } else // Not something we can look further into.
       return V;
-    return getPointerBase(PtrOp);
   }
-  return V;
 }
 
 /// Push users of the given Instruction onto the given Worklist.
@@ -6553,6 +6303,20 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) {
       return getSCEV(U->getOperand(0));
     break;
 
+  case Instruction::SDiv:
+    // If both operands are non-negative, this is just an udiv.
+    if (isKnownNonNegative(getSCEV(U->getOperand(0))) &&
+        isKnownNonNegative(getSCEV(U->getOperand(1))))
+      return getUDivExpr(getSCEV(U->getOperand(0)), getSCEV(U->getOperand(1)));
+    break;
+
+  case Instruction::SRem:
+    // If both operands are non-negative, this is just an urem.
+    if (isKnownNonNegative(getSCEV(U->getOperand(0))) &&
+        isKnownNonNegative(getSCEV(U->getOperand(1))))
+      return getURemExpr(getSCEV(U->getOperand(0)), getSCEV(U->getOperand(1)));
+    break;
+
   // It's tempting to handle inttoptr and ptrtoint as no-ops, however this can
   // lead to pointer expressions which cannot safely be expanded to GEPs,
   // because ScalarEvolution doesn't respect the GEP aliasing rules when
diff --git a/llvm/lib/Analysis/ScalarEvolutionDivision.cpp b/llvm/lib/Analysis/ScalarEvolutionDivision.cpp
new file mode 100644
index 0000000000000..19bf5766f4480
--- /dev/null
+++ b/llvm/lib/Analysis/ScalarEvolutionDivision.cpp
@@ -0,0 +1,259 @@
+//===- ScalarEvolutionDivision.h - See below --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the class that knows how to divide SCEV's.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/ScalarEvolutionDivision.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/ErrorHandling.h"
+#include <cassert>
+#include <cstdint>
+
+namespace llvm {
+class Type;
+}
+
+using namespace llvm;
+
+namespace {
+
+static inline int sizeOfSCEV(const SCEV *S) {
+  struct FindSCEVSize {
+    int Size = 0;
+
+    FindSCEVSize() = default;
+
+    bool follow(const SCEV *S) {
+      ++Size;
+      // Keep looking at all operands of S.
+      return true;
+    }
+
+    bool isDone() const { return false; }
+  };
+
+  FindSCEVSize F;
+  SCEVTraversal<FindSCEVSize> ST(F);
+  ST.visitAll(S);
+  return F.Size;
+}
+
+} // namespace
+
+// Computes the Quotient and Remainder of the division of Numerator by
+// Denominator.
+void SCEVDivision::divide(ScalarEvolution &SE, const SCEV *Numerator,
+                          const SCEV *Denominator, const SCEV **Quotient,
+                          const SCEV **Remainder) {
+  assert(Numerator && Denominator && "Uninitialized SCEV");
+
+  SCEVDivision D(SE, Numerator, Denominator);
+
+  // Check for the trivial case here to avoid having to check for it in the
+  // rest of the code.
+  if (Numerator == Denominator) {
+    *Quotient = D.One;
+    *Remainder = D.Zero;
+    return;
+  }
+
+  if (Numerator->isZero()) {
+    *Quotient = D.Zero;
+    *Remainder = D.Zero;
+    return;
+  }
+
+  // A simple case when N/1. The quotient is N.
+  if (Denominator->isOne()) {
+    *Quotient = Numerator;
+    *Remainder = D.Zero;
+    return;
+  }
+
+  // Split the Denominator when it is a product.
+  if (const SCEVMulExpr *T = dyn_cast<SCEVMulExpr>(Denominator)) {
+    const SCEV *Q, *R;
+    *Quotient = Numerator;
+    for (const SCEV *Op : T->operands()) {
+      divide(SE, *Quotient, Op, &Q, &R);
+      *Quotient = Q;
+
+      // Bail out when the Numerator is not divisible by one of the terms of
+      // the Denominator.
+      if (!R->isZero()) {
+        *Quotient = D.Zero;
+        *Remainder = Numerator;
+        return;
+      }
+    }
+    *Remainder = D.Zero;
+    return;
+  }
+
+  D.visit(Numerator);
+  *Quotient = D.Quotient;
+  *Remainder = D.Remainder;
+}
+
+void SCEVDivision::visitConstant(const SCEVConstant *Numerator) {
+  if (const SCEVConstant *D = dyn_cast<SCEVConstant>(Denominator)) {
+    APInt NumeratorVal = Numerator->getAPInt();
+    APInt DenominatorVal = D->getAPInt();
+    uint32_t NumeratorBW = NumeratorVal.getBitWidth();
+    uint32_t DenominatorBW = DenominatorVal.getBitWidth();
+
+    if (NumeratorBW > DenominatorBW)
+      DenominatorVal = DenominatorVal.sext(NumeratorBW);
+    else if (NumeratorBW < DenominatorBW)
+      NumeratorVal = NumeratorVal.sext(DenominatorBW);
+
+    APInt QuotientVal(NumeratorVal.getBitWidth(), 0);
+    APInt RemainderVal(NumeratorVal.getBitWidth(), 0);
+    APInt::sdivrem(NumeratorVal, DenominatorVal, QuotientVal, RemainderVal);
+    Quotient = SE.getConstant(QuotientVal);
+    Remainder = SE.getConstant(RemainderVal);
+    return;
+  }
+}
+
+void SCEVDivision::visitAddRecExpr(const SCEVAddRecExpr *Numerator) {
+  const SCEV *StartQ, *StartR, *StepQ, *StepR;
+  if (!Numerator->isAffine())
+    return cannotDivide(Numerator);
+  divide(SE, Numerator->getStart(), Denominator, &StartQ, &StartR);
+  divide(SE, Numerator->getStepRecurrence(SE), Denominator, &StepQ, &StepR);
+  // Bail out if the types do not match.
+  Type *Ty = Denominator->getType();
+  if (Ty != StartQ->getType() || Ty != StartR->getType() ||
+      Ty != StepQ->getType() || Ty != StepR->getType())
+    return cannotDivide(Numerator);
+  Quotient = SE.getAddRecExpr(StartQ, StepQ, Numerator->getLoop(),
+                              Numerator->getNoWrapFlags());
+  Remainder = SE.getAddRecExpr(StartR, StepR, Numerator->getLoop(),
+                               Numerator->getNoWrapFlags());
+}
+
+void SCEVDivision::visitAddExpr(const SCEVAddExpr *Numerator) {
+  SmallVector<const SCEV *, 2> Qs, Rs;
+  Type *Ty = Denominator->getType();
+
+  for (const SCEV *Op : Numerator->operands()) {
+    const SCEV *Q, *R;
+    divide(SE, Op, Denominator, &Q, &R);
+
+    // Bail out if types do not match.
+    if (Ty != Q->getType() || Ty != R->getType())
+      return cannotDivide(Numerator);
+
+    Qs.push_back(Q);
+    Rs.push_back(R);
+  }
+
+  if (Qs.size() == 1) {
+    Quotient = Qs[0];
+    Remainder = Rs[0];
+    return;
+  }
+
+  Quotient = SE.getAddExpr(Qs);
+  Remainder = SE.getAddExpr(Rs);
+}
+
+void SCEVDivision::visitMulExpr(const SCEVMulExpr *Numerator) {
+  SmallVector<const SCEV *, 2> Qs;
+  Type *Ty = Denominator->getType();
+
+  bool FoundDenominatorTerm = false;
+  for (const SCEV *Op : Numerator->operands()) {
+    // Bail out if types do not match.
+    if (Ty != Op->getType())
+      return cannotDivide(Numerator);
+
+    if (FoundDenominatorTerm) {
+      Qs.push_back(Op);
+      continue;
+    }
+
+    // Check whether Denominator divides one of the product operands.
+    const SCEV *Q, *R;
+    divide(SE, Op, Denominator, &Q, &R);
+    if (!R->isZero()) {
+      Qs.push_back(Op);
+      continue;
+    }
+
+    // Bail out if types do not match.
+    if (Ty != Q->getType())
+      return cannotDivide(Numerator);
+
+    FoundDenominatorTerm = true;
+    Qs.push_back(Q);
+  }
+
+  if (FoundDenominatorTerm) {
+    Remainder = Zero;
+    if (Qs.size() == 1)
+      Quotient = Qs[0];
+    else
+      Quotient = SE.getMulExpr(Qs);
+    return;
+  }
+
+  if (!isa<SCEVUnknown>(Denominator))
+    return cannotDivide(Numerator);
+
+  // The Remainder is obtained by replacing Denominator by 0 in Numerator.
+  ValueToValueMap RewriteMap;
+  RewriteMap[cast<SCEVUnknown>(Denominator)->getValue()] =
+      cast<SCEVConstant>(Zero)->getValue();
+  Remainder = SCEVParameterRewriter::rewrite(Numerator, SE, RewriteMap, true);
+
+  if (Remainder->isZero()) {
+    // The Quotient is obtained by replacing Denominator by 1 in Numerator.
+    RewriteMap[cast<SCEVUnknown>(Denominator)->getValue()] =
+        cast<SCEVConstant>(One)->getValue();
+    Quotient = SCEVParameterRewriter::rewrite(Numerator, SE, RewriteMap, true);
+    return;
+  }
+
+  // Quotient is (Numerator - Remainder) divided by Denominator.
+  const SCEV *Q, *R;
+  const SCEV *Diff = SE.getMinusSCEV(Numerator, Remainder);
+  // This SCEV does not seem to simplify: fail the division here.
+  if (sizeOfSCEV(Diff) > sizeOfSCEV(Numerator))
+    return cannotDivide(Numerator);
+  divide(SE, Diff, Denominator, &Q, &R);
+  if (R != Zero)
+    return cannotDivide(Numerator);
+  Quotient = Q;
+}
+
+SCEVDivision::SCEVDivision(ScalarEvolution &S, const SCEV *Numerator,
+                           const SCEV *Denominator)
+    : SE(S), Denominator(Denominator) {
+  Zero = SE.getZero(Denominator->getType());
+  One = SE.getOne(Denominator->getType());
+
+  // We generally do not know how to divide Expr by Denominator. We initialize
+  // the division to a "cannot divide" state to simplify the rest of the code.
+  cannotDivide(Numerator);
+}
+
+// Convenience function for giving up on the division. We set the quotient to
+// be equal to zero and the remainder to be equal to the numerator.
+void SCEVDivision::cannotDivide(const SCEV *Numerator) {
+  Quotient = Zero;
+  Remainder = Numerator;
+}
diff --git a/llvm/lib/Analysis/StackSafetyAnalysis.cpp b/llvm/lib/Analysis/StackSafetyAnalysis.cpp
index 28d39b3ea3ad8..bbfc303aefac4 100644
--- a/llvm/lib/Analysis/StackSafetyAnalysis.cpp
+++ b/llvm/lib/Analysis/StackSafetyAnalysis.cpp
@@ -11,9 +11,11 @@
 #include "llvm/Analysis/StackSafetyAnalysis.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Analysis/StackLifetime.h"
 #include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/GlobalValue.h"
@@ -57,7 +59,7 @@ template <typename CalleeTy> struct CallInfo {
   // Range should never set to empty-set, that is an invalid access range
   // that can cause empty-set to be propagated with ConstantRange::add
   ConstantRange Offset;
-  CallInfo(const CalleeTy *Callee, size_t ParamNo, ConstantRange Offset)
+  CallInfo(const CalleeTy *Callee, size_t ParamNo, const ConstantRange &Offset)
       : Callee(Callee), ParamNo(ParamNo), Offset(Offset) {}
 };
 
@@ -200,12 +202,13 @@ class StackSafetyLocalAnalysis {
 
   ConstantRange offsetFrom(Value *Addr, Value *Base);
   ConstantRange getAccessRange(Value *Addr, Value *Base,
-                               ConstantRange SizeRange);
+                               const ConstantRange &SizeRange);
   ConstantRange getAccessRange(Value *Addr, Value *Base, TypeSize Size);
   ConstantRange getMemIntrinsicAccessRange(const MemIntrinsic *MI, const Use &U,
                                            Value *Base);
 
-  bool analyzeAllUses(Value *Ptr, UseInfo<GlobalValue> &AS);
+  bool analyzeAllUses(Value *Ptr, UseInfo<GlobalValue> &AS,
+                      const StackLifetime &SL);
 
 public:
   StackSafetyLocalAnalysis(Function &F, ScalarEvolution &SE)
@@ -234,7 +237,7 @@ ConstantRange StackSafetyLocalAnalysis::offsetFrom(Value *Addr, Value *Base) {
 
 ConstantRange
 StackSafetyLocalAnalysis::getAccessRange(Value *Addr, Value *Base,
-                                         ConstantRange SizeRange) {
+                                         const ConstantRange &SizeRange) {
   // Zero-size loads and stores do not access memory.
   if (SizeRange.isEmptySet())
     return ConstantRange::getEmpty(PointerSize);
@@ -289,20 +292,29 @@ ConstantRange StackSafetyLocalAnalysis::getMemIntrinsicAccessRange(
 /// The function analyzes all local uses of Ptr (alloca or argument) and
 /// calculates local access range and all function calls where it was used.
 bool StackSafetyLocalAnalysis::analyzeAllUses(Value *Ptr,
-                                              UseInfo<GlobalValue> &US) {
+                                              UseInfo<GlobalValue> &US,
+                                              const StackLifetime &SL) {
   SmallPtrSet<const Value *, 16> Visited;
   SmallVector<const Value *, 8> WorkList;
   WorkList.push_back(Ptr);
+  const AllocaInst *AI = dyn_cast<AllocaInst>(Ptr);
 
   // A DFS search through all uses of the alloca in bitcasts/PHI/GEPs/etc.
   while (!WorkList.empty()) {
     const Value *V = WorkList.pop_back_val();
     for (const Use &UI : V->uses()) {
-      const auto *I = cast<const Instruction>(UI.getUser());
+      const auto *I = cast<Instruction>(UI.getUser());
+      if (!SL.isReachable(I))
+        continue;
+
       assert(V == UI.get());
 
       switch (I->getOpcode()) {
       case Instruction::Load: {
+        if (AI && !SL.isAliveAfter(AI, I)) {
+          US.updateRange(UnknownRange);
+          return false;
+        }
         US.updateRange(
             getAccessRange(UI, Ptr, DL.getTypeStoreSize(I->getType())));
         break;
@@ -317,6 +329,10 @@ bool StackSafetyLocalAnalysis::analyzeAllUses(Value *Ptr,
           US.updateRange(UnknownRange);
           return false;
         }
+        if (AI && !SL.isAliveAfter(AI, I)) {
+          US.updateRange(UnknownRange);
+          return false;
+        }
         US.updateRange(getAccessRange(
             UI, Ptr, DL.getTypeStoreSize(I->getOperand(0)->getType())));
         break;
@@ -334,6 +350,11 @@ bool StackSafetyLocalAnalysis::analyzeAllUses(Value *Ptr,
         if (I->isLifetimeStartOrEnd())
           break;
 
+        if (AI && !SL.isAliveAfter(AI, I)) {
+          US.updateRange(UnknownRange);
+          return false;
+        }
+
         if (const MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I)) {
           US.updateRange(getMemIntrinsicAccessRange(MI, UI, Ptr));
           break;
@@ -384,11 +405,16 @@ FunctionInfo<GlobalValue> StackSafetyLocalAnalysis::run() {
 
   LLVM_DEBUG(dbgs() << "[StackSafety] " << F.getName() << "\n");
 
-  for (auto &I : instructions(F)) {
-    if (auto *AI = dyn_cast<AllocaInst>(&I)) {
-      auto &UI = Info.Allocas.emplace(AI, PointerSize).first->second;
-      analyzeAllUses(AI, UI);
-    }
+  SmallVector<AllocaInst *, 64> Allocas;
+  for (auto &I : instructions(F))
+    if (auto *AI = dyn_cast<AllocaInst>(&I))
+      Allocas.push_back(AI);
+  StackLifetime SL(F, Allocas, StackLifetime::LivenessType::Must);
+  SL.run();
+
+  for (auto *AI : Allocas) {
+    auto &UI = Info.Allocas.emplace(AI, PointerSize).first->second;
+    analyzeAllUses(AI, UI, SL);
   }
 
   for (Argument &A : make_range(F.arg_begin(), F.arg_end())) {
@@ -396,7 +422,7 @@ FunctionInfo<GlobalValue> StackSafetyLocalAnalysis::run() {
     // processing.
     if (A.getType()->isPointerTy() && !A.hasByValAttr()) {
       auto &UI = Info.Params.emplace(A.getArgNo(), PointerSize).first->second;
-      analyzeAllUses(&A, UI);
+      analyzeAllUses(&A, UI, SL);
     }
   }
 
@@ -722,15 +748,16 @@ const StackSafetyGlobalInfo::InfoTy &StackSafetyGlobalInfo::getInfo() const {
   return *Info;
 }
 
-// Converts a StackSafetyFunctionInfo to the relevant FunctionSummary
-// constructor fields
 std::vector<FunctionSummary::ParamAccess>
 StackSafetyInfo::getParamAccesses() const {
-  assert(needsParamAccessSummary(*F->getParent()));
-
+  // Implementation transforms internal representation of parameter information
+  // into FunctionSummary format.
   std::vector<FunctionSummary::ParamAccess> ParamAccesses;
   for (const auto &KV : getInfo().Info.Params) {
     auto &PS = KV.second;
+    // Parameter accessed by any or unknown offset, represented as FullSet by
+    // StackSafety, is handled as the parameter for which we have no
+    // StackSafety info at all. So drop it to reduce summary size.
     if (PS.Range.isFullSet())
       continue;
 
@@ -739,6 +766,10 @@ StackSafetyInfo::getParamAccesses() const {
 
     Param.Calls.reserve(PS.Calls.size());
     for (auto &C : PS.Calls) {
+      // Parameter forwarded into another function by any or unknown offset
+      // will make ParamAccess::Range as FullSet anyway. So we can drop the
+      // entire parameter like we did above.
+      // TODO(vitalybuka): Return already filtered parameters from getInfo().
       if (C.Offset.isFullSet()) {
         ParamAccesses.pop_back();
         break;
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 693b5b7a493c4..2f051e53790b1 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -68,7 +68,7 @@ IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id,
                                                  const CallBase &CI) :
   II(dyn_cast<IntrinsicInst>(&CI)),  RetTy(CI.getType()), IID(Id) {
 
-  if (auto *FPMO = dyn_cast<FPMathOperator>(&CI))
+  if (const auto *FPMO = dyn_cast<FPMathOperator>(&CI))
     FMF = FPMO->getFastMathFlags();
 
   FunctionType *FTy =
@@ -96,7 +96,7 @@ IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id,
                                                  unsigned ScalarCost) :
     RetTy(CI.getType()), IID(Id), VF(Factor), ScalarizationCost(ScalarCost) {
 
-  if (auto *FPMO = dyn_cast<FPMathOperator>(&CI))
+  if (const auto *FPMO = dyn_cast<FPMathOperator>(&CI))
     FMF = FPMO->getFastMathFlags();
 
   Arguments.insert(Arguments.begin(), CI.arg_begin(), CI.arg_end());
@@ -136,8 +136,8 @@ IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
 }
 
 IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *Ty,
-                                                 ArrayRef<Value *> Args) :
-    RetTy(Ty), IID(Id) {
+                                                 ArrayRef<const Value *> Args)
+    : RetTy(Ty), IID(Id) {
 
   Arguments.insert(Arguments.begin(), Args.begin(), Args.end());
   ParamTys.reserve(Arguments.size());
@@ -327,6 +327,11 @@ void TargetTransformInfo::getUnrollingPreferences(
   return TTIImpl->getUnrollingPreferences(L, SE, UP);
 }
 
+void TargetTransformInfo::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                                                PeelingPreferences &PP) const {
+  return TTIImpl->getPeelingPreferences(L, SE, PP);
+}
+
 bool TargetTransformInfo::isLegalAddImmediate(int64_t Imm) const {
   return TTIImpl->isLegalAddImmediate(Imm);
 }
@@ -633,11 +638,12 @@ unsigned TargetTransformInfo::getMaxInterleaveFactor(unsigned VF) const {
 }
 
 TargetTransformInfo::OperandValueKind
-TargetTransformInfo::getOperandInfo(Value *V, OperandValueProperties &OpProps) {
+TargetTransformInfo::getOperandInfo(const Value *V,
+                                    OperandValueProperties &OpProps) {
   OperandValueKind OpInfo = OK_AnyValue;
   OpProps = OP_None;
 
-  if (auto *CI = dyn_cast<ConstantInt>(V)) {
+  if (const auto *CI = dyn_cast<ConstantInt>(V)) {
     if (CI->getValue().isPowerOf2())
       OpProps = OP_PowerOf2;
     return OK_UniformConstantValue;
@@ -646,7 +652,7 @@ TargetTransformInfo::getOperandInfo(Value *V, OperandValueProperties &OpProps) {
   // A broadcast shuffle creates a uniform value.
   // TODO: Add support for non-zero index broadcasts.
   // TODO: Add support for different source vector width.
-  if (auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(V))
+  if (const auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(V))
     if (ShuffleInst->isZeroEltSplat())
       OpInfo = OK_UniformValue;
 
@@ -661,7 +667,7 @@ TargetTransformInfo::getOperandInfo(Value *V, OperandValueProperties &OpProps) {
       if (auto *CI = dyn_cast<ConstantInt>(Splat))
         if (CI->getValue().isPowerOf2())
           OpProps = OP_PowerOf2;
-    } else if (auto *CDS = dyn_cast<ConstantDataSequential>(V)) {
+    } else if (const auto *CDS = dyn_cast<ConstantDataSequential>(V)) {
       OpProps = OP_PowerOf2;
       for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) {
         if (auto *CI = dyn_cast<ConstantInt>(CDS->getElementAsConstant(I)))
@@ -756,10 +762,9 @@ int TargetTransformInfo::getMemoryOpCost(unsigned Opcode, Type *Src,
   return Cost;
 }
 
-int TargetTransformInfo::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
-                                               unsigned Alignment,
-                                               unsigned AddressSpace,
-                                               TTI::TargetCostKind CostKind) const {
+int TargetTransformInfo::getMaskedMemoryOpCost(
+    unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
+    TTI::TargetCostKind CostKind) const {
   int Cost =
       TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
                                      CostKind);
@@ -768,9 +773,8 @@ int TargetTransformInfo::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
 }
 
 int TargetTransformInfo::getGatherScatterOpCost(
-    unsigned Opcode, Type *DataTy, Value *Ptr, bool VariableMask,
-    unsigned Alignment, TTI::TargetCostKind CostKind,
-    const Instruction *I) const {
+    unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
+    Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
   int Cost = TTIImpl->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
                                              Alignment, CostKind, I);
   assert(Cost >= 0 && "TTI should not produce negative costs!");
@@ -779,8 +783,7 @@ int TargetTransformInfo::getGatherScatterOpCost(
 
 int TargetTransformInfo::getInterleavedMemoryOpCost(
     unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
-    unsigned Alignment, unsigned AddressSpace,
-    TTI::TargetCostKind CostKind,
+    Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
     bool UseMaskForCond, bool UseMaskForGaps) const {
   int Cost = TTIImpl->getInterleavedMemoryOpCost(
       Opcode, VecTy, Factor, Indices, Alignment, AddressSpace, CostKind,
@@ -912,13 +915,13 @@ bool TargetTransformInfo::isLegalToVectorizeStore(StoreInst *SI) const {
 }
 
 bool TargetTransformInfo::isLegalToVectorizeLoadChain(
-    unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const {
+    unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const {
   return TTIImpl->isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment,
                                               AddrSpace);
 }
 
 bool TargetTransformInfo::isLegalToVectorizeStoreChain(
-    unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const {
+    unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const {
   return TTIImpl->isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment,
                                                AddrSpace);
 }
@@ -1055,6 +1058,8 @@ static TTI::ReductionKind matchPairwiseReductionAtLevel(Instruction *I,
   // Check that the next levels binary operation exists and matches with the
   // current one.
   if (Level + 1 != NumLevels) {
+    if (!isa<Instruction>(NextLevelOp))
+      return TTI::RK_None;
     Optional<TTI::ReductionData> NextLevelRD =
         getReductionData(cast<Instruction>(NextLevelOp));
     if (!NextLevelRD || !RD->hasSameData(*NextLevelRD))
@@ -1076,7 +1081,7 @@ static TTI::ReductionKind matchPairwiseReductionAtLevel(Instruction *I,
     return RD->Kind;
 
   // Match next level.
-  return matchPairwiseReductionAtLevel(cast<Instruction>(NextLevelOp), Level,
+  return matchPairwiseReductionAtLevel(dyn_cast<Instruction>(NextLevelOp), Level,
                                        NumLevels);
 }
 
diff --git a/llvm/lib/Analysis/ValueLatticeUtils.cpp b/llvm/lib/Analysis/ValueLatticeUtils.cpp
index 3f9287e26ce7e..53638c351f722 100644
--- a/llvm/lib/Analysis/ValueLatticeUtils.cpp
+++ b/llvm/lib/Analysis/ValueLatticeUtils.cpp
@@ -28,16 +28,14 @@ bool llvm::canTrackGlobalVariableInterprocedurally(GlobalVariable *GV) {
   if (GV->isConstant() || !GV->hasLocalLinkage() ||
       !GV->hasDefinitiveInitializer())
     return false;
-  return !any_of(GV->users(), [&](User *U) {
-    if (auto *Store = dyn_cast<StoreInst>(U)) {
-      if (Store->getValueOperand() == GV || Store->isVolatile())
-        return true;
-    } else if (auto *Load = dyn_cast<LoadInst>(U)) {
-      if (Load->isVolatile())
-        return true;
-    } else {
-      return true;
-    }
+  return all_of(GV->users(), [&](User *U) {
+    // Currently all users of a global variable have to be none-volatile loads
+    // or stores and the global cannot be stored itself.
+    if (auto *Store = dyn_cast<StoreInst>(U))
+      return Store->getValueOperand() != GV && !Store->isVolatile();
+    if (auto *Load = dyn_cast<LoadInst>(U))
+      return !Load->isVolatile();
+
     return false;
   });
 }
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 927add44dda33..614c8bb2f1e64 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -1444,17 +1444,9 @@ static void computeKnownBitsFromOperator(const Operator *I,
     Known.Zero.setHighBits(Leaders);
     break;
   }
-
-  case Instruction::Alloca: {
-    const AllocaInst *AI = cast<AllocaInst>(I);
-    unsigned Align = AI->getAlignment();
-    if (Align == 0)
-      Align = Q.DL.getABITypeAlignment(AI->getAllocatedType());
-
-    if (Align > 0)
-      Known.Zero.setLowBits(countTrailingZeros(Align));
+  case Instruction::Alloca:
+    Known.Zero.setLowBits(Log2(cast<AllocaInst>(I)->getAlign()));
     break;
-  }
   case Instruction::GetElementPtr: {
     // Analyze all of the subscripts of this getelementptr instruction
     // to determine if we can prove known low zero bits.
@@ -4102,12 +4094,17 @@ llvm::getArgumentAliasingToReturnedPointer(const CallBase *Call,
 
 bool llvm::isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
     const CallBase *Call, bool MustPreserveNullness) {
-  return Call->getIntrinsicID() == Intrinsic::launder_invariant_group ||
-         Call->getIntrinsicID() == Intrinsic::strip_invariant_group ||
-         Call->getIntrinsicID() == Intrinsic::aarch64_irg ||
-         Call->getIntrinsicID() == Intrinsic::aarch64_tagp ||
-         (!MustPreserveNullness &&
-          Call->getIntrinsicID() == Intrinsic::ptrmask);
+  switch (Call->getIntrinsicID()) {
+  case Intrinsic::launder_invariant_group:
+  case Intrinsic::strip_invariant_group:
+  case Intrinsic::aarch64_irg:
+  case Intrinsic::aarch64_tagp:
+    return true;
+  case Intrinsic::ptrmask:
+    return !MustPreserveNullness;
+  default:
+    return false;
+  }
 }
 
 /// \p PN defines a loop-variant pointer to an object.  Check if the
@@ -4147,6 +4144,8 @@ Value *llvm::GetUnderlyingObject(Value *V, const DataLayout &DL,
     } else if (Operator::getOpcode(V) == Instruction::BitCast ||
                Operator::getOpcode(V) == Instruction::AddrSpaceCast) {
       V = cast<Operator>(V)->getOperand(0);
+      if (!V->getType()->isPointerTy())
+        return V;
     } else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) {
       if (GA->isInterposable())
         return V;
diff --git a/llvm/lib/Analysis/models/inliner/saved_model.pb b/llvm/lib/Analysis/models/inliner/saved_model.pb
new file mode 100644
index 0000000000000..5488989454f72
Binary files /dev/null and b/llvm/lib/Analysis/models/inliner/saved_model.pb differ
diff --git a/llvm/lib/Analysis/models/inliner/variables/variables.data-00000-of-00002 b/llvm/lib/Analysis/models/inliner/variables/variables.data-00000-of-00002
new file mode 100644
index 0000000000000..58ebd0fc98711
Binary files /dev/null and b/llvm/lib/Analysis/models/inliner/variables/variables.data-00000-of-00002 differ
diff --git a/llvm/lib/Analysis/models/inliner/variables/variables.data-00001-of-00002 b/llvm/lib/Analysis/models/inliner/variables/variables.data-00001-of-00002
new file mode 100644
index 0000000000000..1f1f1b151a71f
Binary files /dev/null and b/llvm/lib/Analysis/models/inliner/variables/variables.data-00001-of-00002 differ
diff --git a/llvm/lib/Analysis/models/inliner/variables/variables.index b/llvm/lib/Analysis/models/inliner/variables/variables.index
new file mode 100644
index 0000000000000..318d5a2443c2b
Binary files /dev/null and b/llvm/lib/Analysis/models/inliner/variables/variables.index differ
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index 2a39e19a68b75..777ce3abdddd5 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -664,6 +664,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(noreturn);
   KEYWORD(nosync);
   KEYWORD(nocf_check);
+  KEYWORD(noundef);
   KEYWORD(nounwind);
   KEYWORD(null_pointer_is_valid);
   KEYWORD(optforfuzzing);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 4e75ca4ae5357..e3a52c7882a2f 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -1374,6 +1374,7 @@ bool LLParser::ParseFnAttributeValuePairs(AttrBuilder &B,
     case lltok::kw_inalloca:
     case lltok::kw_nest:
     case lltok::kw_noalias:
+    case lltok::kw_noundef:
     case lltok::kw_nocapture:
     case lltok::kw_nonnull:
     case lltok::kw_returned:
@@ -1641,7 +1642,7 @@ bool LLParser::ParseOptionalParamAttrs(AttrBuilder &B) {
     }
     case lltok::kw_align: {
       MaybeAlign Alignment;
-      if (ParseOptionalAlignment(Alignment))
+      if (ParseOptionalAlignment(Alignment, true))
         return true;
       B.addAlignmentAttr(Alignment);
       continue;
@@ -1677,6 +1678,9 @@ bool LLParser::ParseOptionalParamAttrs(AttrBuilder &B) {
     case lltok::kw_inalloca:        B.addAttribute(Attribute::InAlloca); break;
     case lltok::kw_inreg:           B.addAttribute(Attribute::InReg); break;
     case lltok::kw_nest:            B.addAttribute(Attribute::Nest); break;
+    case lltok::kw_noundef:
+      B.addAttribute(Attribute::NoUndef);
+      break;
     case lltok::kw_noalias:         B.addAttribute(Attribute::NoAlias); break;
     case lltok::kw_nocapture:       B.addAttribute(Attribute::NoCapture); break;
     case lltok::kw_nofree:          B.addAttribute(Attribute::NoFree); break;
@@ -1774,6 +1778,9 @@ bool LLParser::ParseOptionalReturnAttrs(AttrBuilder &B) {
     }
     case lltok::kw_inreg:           B.addAttribute(Attribute::InReg); break;
     case lltok::kw_noalias:         B.addAttribute(Attribute::NoAlias); break;
+    case lltok::kw_noundef:
+      B.addAttribute(Attribute::NoUndef);
+      break;
     case lltok::kw_nonnull:         B.addAttribute(Attribute::NonNull); break;
     case lltok::kw_signext:         B.addAttribute(Attribute::SExt); break;
     case lltok::kw_zeroext:         B.addAttribute(Attribute::ZExt); break;
@@ -2127,14 +2134,26 @@ bool LLParser::ParseOptionalFunctionMetadata(Function &F) {
 /// ParseOptionalAlignment
 ///   ::= /* empty */
 ///   ::= 'align' 4
-bool LLParser::ParseOptionalAlignment(MaybeAlign &Alignment) {
+bool LLParser::ParseOptionalAlignment(MaybeAlign &Alignment, bool AllowParens) {
   Alignment = None;
   if (!EatIfPresent(lltok::kw_align))
     return false;
   LocTy AlignLoc = Lex.getLoc();
   uint32_t Value = 0;
+
+  LocTy ParenLoc = Lex.getLoc();
+  bool HaveParens = false;
+  if (AllowParens) {
+    if (EatIfPresent(lltok::lparen))
+      HaveParens = true;
+  }
+
   if (ParseUInt32(Value))
     return true;
+
+  if (HaveParens && !EatIfPresent(lltok::rparen))
+    return Error(ParenLoc, "expected ')'");
+
   if (!isPowerOf2_32(Value))
     return Error(AlignLoc, "alignment is not a power of two");
   if (Value > Value::MaximumAlignment)
@@ -3650,9 +3669,10 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) {
             ExplicitTypeLoc,
             "explicit pointee type doesn't match operand's pointee type");
 
-      unsigned GEPWidth = BaseType->isVectorTy()
-                              ? cast<VectorType>(BaseType)->getNumElements()
-                              : 0;
+      unsigned GEPWidth =
+          BaseType->isVectorTy()
+              ? cast<FixedVectorType>(BaseType)->getNumElements()
+              : 0;
 
       ArrayRef<Constant *> Indices(Elts.begin() + 1, Elts.end());
       for (Constant *Val : Indices) {
@@ -3660,7 +3680,7 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) {
         if (!ValTy->isIntOrIntVectorTy())
           return Error(ID.Loc, "getelementptr index must be an integer");
         if (auto *ValVTy = dyn_cast<VectorType>(ValTy)) {
-          unsigned ValNumEl = ValVTy->getNumElements();
+          unsigned ValNumEl = cast<FixedVectorType>(ValVTy)->getNumElements();
           if (GEPWidth && (ValNumEl != GEPWidth))
             return Error(
                 ID.Loc,
@@ -7196,8 +7216,13 @@ int LLParser::ParseCmpXchg(Instruction *&Inst, PerFunctionState &PFS) {
     return Error(NewLoc, "new value and pointer type do not match");
   if (!New->getType()->isFirstClassType())
     return Error(NewLoc, "cmpxchg operand must be a first class value");
+
+  Align Alignment(
+      PFS.getFunction().getParent()->getDataLayout().getTypeStoreSize(
+          Cmp->getType()));
+
   AtomicCmpXchgInst *CXI = new AtomicCmpXchgInst(
-      Ptr, Cmp, New, SuccessOrdering, FailureOrdering, SSID);
+      Ptr, Cmp, New, Alignment, SuccessOrdering, FailureOrdering, SSID);
   CXI->setVolatile(isVolatile);
   CXI->setWeak(isWeak);
   Inst = CXI;
@@ -7281,9 +7306,11 @@ int LLParser::ParseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
   if (Size < 8 || (Size & (Size - 1)))
     return Error(ValLoc, "atomicrmw operand must be power-of-two byte-sized"
                          " integer");
-
+  Align Alignment(
+      PFS.getFunction().getParent()->getDataLayout().getTypeStoreSize(
+          Val->getType()));
   AtomicRMWInst *RMWI =
-    new AtomicRMWInst(Operation, Ptr, Val, Ordering, SSID);
+      new AtomicRMWInst(Operation, Ptr, Val, Alignment, Ordering, SSID);
   RMWI->setVolatile(isVolatile);
   Inst = RMWI;
   return AteExtraComma ? InstExtraComma : InstNormal;
diff --git a/llvm/lib/AsmParser/LLParser.h b/llvm/lib/AsmParser/LLParser.h
index cc3aae6be5933..ebd8655dc35e9 100644
--- a/llvm/lib/AsmParser/LLParser.h
+++ b/llvm/lib/AsmParser/LLParser.h
@@ -272,7 +272,8 @@ namespace llvm {
     void ParseOptionalVisibility(unsigned &Res);
     void ParseOptionalDLLStorageClass(unsigned &Res);
     bool ParseOptionalCallingConv(unsigned &CC);
-    bool ParseOptionalAlignment(MaybeAlign &Alignment);
+    bool ParseOptionalAlignment(MaybeAlign &Alignment,
+                                bool AllowParens = false);
     bool ParseOptionalDerefAttrBytes(lltok::Kind AttrKind, uint64_t &Bytes);
     bool ParseScopeAndOrdering(bool isAtomic, SyncScope::ID &SSID,
                                AtomicOrdering &Ordering);
diff --git a/llvm/lib/AsmParser/LLToken.h b/llvm/lib/AsmParser/LLToken.h
index 0c190bfb63b3e..0fb3bae77dd3f 100644
--- a/llvm/lib/AsmParser/LLToken.h
+++ b/llvm/lib/AsmParser/LLToken.h
@@ -196,6 +196,7 @@ enum Kind {
   kw_naked,
   kw_nest,
   kw_noalias,
+  kw_noundef,
   kw_nobuiltin,
   kw_nocapture,
   kw_noduplicate,
diff --git a/llvm/lib/BinaryFormat/AMDGPUMetadataVerifier.cpp b/llvm/lib/BinaryFormat/AMDGPUMetadataVerifier.cpp
index e8b9e12ce4c8d..cd1d872cc2194 100644
--- a/llvm/lib/BinaryFormat/AMDGPUMetadataVerifier.cpp
+++ b/llvm/lib/BinaryFormat/AMDGPUMetadataVerifier.cpp
@@ -127,25 +127,6 @@ bool MetadataVerifier::verifyKernelArgs(msgpack::DocNode &Node) {
                                .Default(false);
                          }))
     return false;
-  if (!verifyScalarEntry(ArgsMap, ".value_type", true,
-                         msgpack::Type::String,
-                         [](msgpack::DocNode &SNode) {
-                           return StringSwitch<bool>(SNode.getString())
-                               .Case("struct", true)
-                               .Case("i8", true)
-                               .Case("u8", true)
-                               .Case("i16", true)
-                               .Case("u16", true)
-                               .Case("f16", true)
-                               .Case("i32", true)
-                               .Case("u32", true)
-                               .Case("f32", true)
-                               .Case("i64", true)
-                               .Case("u64", true)
-                               .Case("f64", true)
-                               .Default(false);
-                         }))
-    return false;
   if (!verifyIntegerEntry(ArgsMap, ".pointee_align", false))
     return false;
   if (!verifyScalarEntry(ArgsMap, ".address_space", false,
diff --git a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp
index 64530f3531625..2ce064c7685a7 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp
@@ -130,7 +130,7 @@ static Optional<const char *> GetCodeName(unsigned CodeID, unsigned BlockID,
       STRINGIFY_CODE(MODULE_CODE, DATALAYOUT)
       STRINGIFY_CODE(MODULE_CODE, ASM)
       STRINGIFY_CODE(MODULE_CODE, SECTIONNAME)
-      STRINGIFY_CODE(MODULE_CODE, DEPLIB) // FIXME: Remove in 4.0
+      STRINGIFY_CODE(MODULE_CODE, DEPLIB) // Deprecated, present in old bitcode
       STRINGIFY_CODE(MODULE_CODE, GLOBALVAR)
       STRINGIFY_CODE(MODULE_CODE, FUNCTION)
       STRINGIFY_CODE(MODULE_CODE, ALIAS)
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index a428416244f92..659e26c2bd250 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -1317,8 +1317,6 @@ static void addRawAttributeValue(AttrBuilder &B, uint64_t Val) {
 /// 'encodeLLVMAttributesForBitcode'.
 static void decodeLLVMAttributesForBitcode(AttrBuilder &B,
                                            uint64_t EncodedAttrs) {
-  // FIXME: Remove in 4.0.
-
   // The alignment is stored as a 16-bit raw value from bits 31--16.  We shift
   // the bits above 31 down by 11 bits.
   unsigned Alignment = (EncodedAttrs & (0xffffULL << 16)) >> 16;
@@ -1369,7 +1367,7 @@ Error BitcodeReader::parseAttributeBlock() {
     default:  // Default behavior: ignore.
       break;
     case bitc::PARAMATTR_CODE_ENTRY_OLD: // ENTRY: [paramidx0, attr0, ...]
-      // FIXME: Remove in 4.0.
+      // Deprecated, but still needed to read old bitcode files.
       if (Record.size() & 1)
         return error("Invalid record");
 
@@ -1532,6 +1530,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
     return Attribute::SanitizeMemTag;
   case bitc::ATTR_KIND_PREALLOCATED:
     return Attribute::Preallocated;
+  case bitc::ATTR_KIND_NOUNDEF:
+    return Attribute::NoUndef;
   }
 }
 
@@ -1777,7 +1777,7 @@ Error BitcodeReader::parseTypeTableBody() {
       break;
     }
     case bitc::TYPE_CODE_FUNCTION_OLD: {
-      // FIXME: attrid is dead, remove it in LLVM 4.0
+      // Deprecated, but still needed to read old bitcode files.
       // FUNCTION: [vararg, attrid, retty, paramty x N]
       if (Record.size() < 3)
         return error("Invalid record");
@@ -2337,6 +2337,7 @@ Error BitcodeReader::parseConstants() {
     uint64_t Op0Idx;
     uint64_t Op1Idx;
     uint64_t Op2Idx;
+    unsigned CstNo;
   };
   std::vector<DelayedShufTy> DelayedShuffles;
   while (true) {
@@ -2350,9 +2351,6 @@ Error BitcodeReader::parseConstants() {
     case BitstreamEntry::Error:
       return error("Malformed block");
     case BitstreamEntry::EndBlock:
-      if (NextCstNo != ValueList.size())
-        return error("Invalid constant reference");
-
       // Once all the constants have been read, go through and resolve forward
       // references.
       //
@@ -2365,6 +2363,7 @@ Error BitcodeReader::parseConstants() {
         uint64_t Op0Idx = DelayedShuffle.Op0Idx;
         uint64_t Op1Idx = DelayedShuffle.Op1Idx;
         uint64_t Op2Idx = DelayedShuffle.Op2Idx;
+        uint64_t CstNo = DelayedShuffle.CstNo;
         Constant *Op0 = ValueList.getConstantFwdRef(Op0Idx, OpTy);
         Constant *Op1 = ValueList.getConstantFwdRef(Op1Idx, OpTy);
         Type *ShufTy =
@@ -2375,9 +2374,12 @@ Error BitcodeReader::parseConstants() {
         SmallVector<int, 16> Mask;
         ShuffleVectorInst::getShuffleMask(Op2, Mask);
         Value *V = ConstantExpr::getShuffleVector(Op0, Op1, Mask);
-        ValueList.assignValue(V, NextCstNo, DelayedShuffle.CurFullTy);
-        ++NextCstNo;
+        ValueList.assignValue(V, CstNo, DelayedShuffle.CurFullTy);
       }
+
+      if (NextCstNo != ValueList.size())
+        return error("Invalid constant reference");
+
       ValueList.resolveConstantForwardRefs();
       return Error::success();
     case BitstreamEntry::Record:
@@ -2700,8 +2702,10 @@ Error BitcodeReader::parseConstants() {
         if (!IdxTy)
           return error("Invalid record");
         Op1 = ValueList.getConstantFwdRef(Record[3], IdxTy);
-      } else // TODO: Remove with llvm 4.0
+      } else {
+        // Deprecated, but still needed to read old bitcode files.
         Op1 = ValueList.getConstantFwdRef(Record[2], Type::getInt32Ty(Context));
+      }
       if (!Op1)
         return error("Invalid record");
       V = ConstantExpr::getExtractElement(Op0, Op1);
@@ -2721,8 +2725,10 @@ Error BitcodeReader::parseConstants() {
         if (!IdxTy)
           return error("Invalid record");
         Op2 = ValueList.getConstantFwdRef(Record[3], IdxTy);
-      } else // TODO: Remove with llvm 4.0
+      } else {
+        // Deprecated, but still needed to read old bitcode files.
         Op2 = ValueList.getConstantFwdRef(Record[2], Type::getInt32Ty(Context));
+      }
       if (!Op2)
         return error("Invalid record");
       V = ConstantExpr::getInsertElement(Op0, Op1, Op2);
@@ -2733,7 +2739,8 @@ Error BitcodeReader::parseConstants() {
       if (Record.size() < 3 || !OpTy)
         return error("Invalid record");
       DelayedShuffles.push_back(
-          {OpTy, OpTy, CurFullTy, Record[0], Record[1], Record[2]});
+          {OpTy, OpTy, CurFullTy, Record[0], Record[1], Record[2], NextCstNo});
+      ++NextCstNo;
       continue;
     }
     case bitc::CST_CODE_CE_SHUFVEC_EX: { // [opty, opval, opval, opval]
@@ -2743,7 +2750,8 @@ Error BitcodeReader::parseConstants() {
       if (Record.size() < 4 || !RTy || !OpTy)
         return error("Invalid record");
       DelayedShuffles.push_back(
-          {OpTy, RTy, CurFullTy, Record[1], Record[2], Record[3]});
+          {OpTy, RTy, CurFullTy, Record[1], Record[2], Record[3], NextCstNo});
+      ++NextCstNo;
       continue;
     }
     case bitc::CST_CODE_CE_CMP: {     // CE_CMP: [opty, opval, opval, pred]
@@ -2762,7 +2770,7 @@ Error BitcodeReader::parseConstants() {
       break;
     }
     // This maintains backward compatibility, pre-asm dialect keywords.
-    // FIXME: Remove with the 4.0 release.
+    // Deprecated, but still needed to read old bitcode files.
     case bitc::CST_CODE_INLINEASM_OLD: {
       if (Record.size() < 2)
         return error("Invalid record");
@@ -3163,8 +3171,8 @@ Error BitcodeReader::parseGlobalVarRecord(ArrayRef<uint64_t> Record) {
   }
   GlobalValue::VisibilityTypes Visibility = GlobalValue::DefaultVisibility;
   // Local linkage must have default visibility.
+  // auto-upgrade `hidden` and `protected` for old bitcode.
   if (Record.size() > 6 && !GlobalValue::isLocalLinkage(Linkage))
-    // FIXME: Change to an error if non-default in 4.0.
     Visibility = getDecodedVisibility(Record[6]);
 
   GlobalVariable::ThreadLocalMode TLM = GlobalVariable::NotThreadLocal;
@@ -3293,8 +3301,8 @@ Error BitcodeReader::parseFunctionRecord(ArrayRef<uint64_t> Record) {
     Func->setSection(SectionTable[Record[6] - 1]);
   }
   // Local linkage must have default visibility.
+  // auto-upgrade `hidden` and `protected` for old bitcode.
   if (!Func->hasLocalLinkage())
-    // FIXME: Change to an error if non-default in 4.0.
     Func->setVisibility(getDecodedVisibility(Record[7]));
   if (Record.size() > 8 && Record[8]) {
     if (Record[8] - 1 >= GCTable.size())
@@ -3401,12 +3409,11 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord(
 
   assert(NewGA->getValueType() == flattenPointerTypes(FullTy) &&
          "Incorrect fully structured type provided for GlobalIndirectSymbol");
-  // Old bitcode files didn't have visibility field.
   // Local linkage must have default visibility.
+  // auto-upgrade `hidden` and `protected` for old bitcode.
   if (OpNum != Record.size()) {
     auto VisInd = OpNum++;
     if (!NewGA->hasLocalLinkage())
-      // FIXME: Change to an error if non-default in 4.0.
       NewGA->setVisibility(getDecodedVisibility(Record[VisInd]));
   }
   if (BitCode == bitc::MODULE_CODE_ALIAS ||
@@ -3659,7 +3666,7 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit,
       break;
     }
     case bitc::MODULE_CODE_DEPLIB: {  // DEPLIB: [strchr x N]
-      // FIXME: Remove in 4.0.
+      // Deprecated, but still needed to read old bitcode files.
       std::string S;
       if (convertToString(Record, 0, S))
         return error("Invalid record");
@@ -5015,8 +5022,10 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
       else
         FailureOrdering = getDecodedOrdering(Record[OpNum + 3]);
 
-      I = new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering, FailureOrdering,
-                                SSID);
+      Align Alignment(
+          TheModule->getDataLayout().getTypeStoreSize(Cmp->getType()));
+      I = new AtomicCmpXchgInst(Ptr, Cmp, New, Alignment, SuccessOrdering,
+                                FailureOrdering, SSID);
       FullTy = StructType::get(Context, {FullTy, Type::getInt1Ty(Context)});
       cast<AtomicCmpXchgInst>(I)->setVolatile(Record[OpNum]);
 
@@ -5053,7 +5062,9 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
           Ordering == AtomicOrdering::Unordered)
         return error("Invalid record");
       SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 3]);
-      I = new AtomicRMWInst(Operation, Ptr, Val, Ordering, SSID);
+      Align Alignment(
+          TheModule->getDataLayout().getTypeStoreSize(Val->getType()));
+      I = new AtomicRMWInst(Operation, Ptr, Val, Alignment, Ordering, SSID);
       FullTy = getPointerElementFlatType(FullTy);
       cast<AtomicRMWInst>(I)->setVolatile(Record[OpNum+1]);
       InstructionList.push_back(I);
diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
index 34c93beebb013..a8bf579bd1801 100644
--- a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
+++ b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
@@ -1132,7 +1132,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     break;
   }
   case bitc::METADATA_OLD_FN_NODE: {
-    // FIXME: Remove in 4.0.
+    // Deprecated, but still needed to read old bitcode files.
     // This is a LocalAsMetadata record, the only type of function-local
     // metadata.
     if (Record.size() % 2 == 1)
@@ -1162,7 +1162,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     break;
   }
   case bitc::METADATA_OLD_NODE: {
-    // FIXME: Remove in 4.0.
+    // Deprecated, but still needed to read old bitcode files.
     if (Record.size() % 2 == 1)
       return error("Invalid record");
 
diff --git a/llvm/lib/Bitcode/Reader/ValueList.cpp b/llvm/lib/Bitcode/Reader/ValueList.cpp
index 431995fd40ac7..63a206eeb022c 100644
--- a/llvm/lib/Bitcode/Reader/ValueList.cpp
+++ b/llvm/lib/Bitcode/Reader/ValueList.cpp
@@ -220,6 +220,6 @@ void BitcodeReaderValueList::resolveConstantForwardRefs() {
 
     // Update all ValueHandles, they should be the only users at this point.
     Placeholder->replaceAllUsesWith(RealVal);
-    Placeholder->deleteValue();
+    delete cast<ConstantPlaceHolder>(Placeholder);
   }
 }
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 653d7f96e3ef2..9c15a5f9f193a 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -731,6 +731,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_SANITIZE_MEMTAG;
   case Attribute::Preallocated:
     return bitc::ATTR_KIND_PREALLOCATED;
+  case Attribute::NoUndef:
+    return bitc::ATTR_KIND_NOUNDEF;
   case Attribute::EndAttrKinds:
     llvm_unreachable("Can not encode end-attribute kinds marker.");
   case Attribute::None:
@@ -961,7 +963,7 @@ void ModuleBitcodeWriter::writeTypeTable() {
       // VECTOR [numelts, eltty] or
       //        [numelts, eltty, scalable]
       Code = bitc::TYPE_CODE_VECTOR;
-      TypeVals.push_back(VT->getNumElements());
+      TypeVals.push_back(VT->getElementCount().Min);
       TypeVals.push_back(VE.getTypeID(VT->getElementType()));
       if (isa<ScalableVectorType>(VT))
         TypeVals.push_back(true);
@@ -1179,7 +1181,7 @@ void ModuleBitcodeWriter::writeModuleInfo() {
   std::map<std::string, unsigned> GCMap;
   unsigned MaxAlignment = 0;
   unsigned MaxGlobalType = 0;
-  for (const GlobalValue &GV : M.globals()) {
+  for (const GlobalVariable &GV : M.globals()) {
     MaxAlignment = std::max(MaxAlignment, GV.getAlignment());
     MaxGlobalType = std::max(MaxGlobalType, VE.getTypeID(GV.getValueType()));
     if (GV.hasSection()) {
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index a81a42a6f2b41..4d7c36041398f 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -157,11 +157,11 @@ static gcp_map_type &getGCMap(void *&P) {
 
 /// getGVAlignment - Return the alignment to use for the specified global
 /// value.  This rounds up to the preferred alignment if possible and legal.
-Align AsmPrinter::getGVAlignment(const GlobalValue *GV, const DataLayout &DL,
+Align AsmPrinter::getGVAlignment(const GlobalObject *GV, const DataLayout &DL,
                                  Align InAlign) {
   Align Alignment;
   if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV))
-    Alignment = Align(DL.getPreferredAlignment(GVar));
+    Alignment = DL.getPreferredAlign(GVar);
 
   // If InAlign is specified, round it to it.
   if (InAlign > Alignment)
@@ -244,7 +244,6 @@ const MCSection *AsmPrinter::getCurrentSection() const {
 void AsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesAll();
   MachineFunctionPass::getAnalysisUsage(AU);
-  AU.addRequired<MachineModuleInfoWrapperPass>();
   AU.addRequired<MachineOptimizationRemarkEmitterPass>();
   AU.addRequired<GCModuleInfo>();
 }
@@ -306,14 +305,14 @@ bool AsmPrinter::doInitialization(Module &M) {
   }
 
   if (MAI->doesSupportDebugInformation()) {
-    bool EmitCodeView = MMI->getModule()->getCodeViewFlag();
+    bool EmitCodeView = M.getCodeViewFlag();
     if (EmitCodeView && TM.getTargetTriple().isOSWindows()) {
       Handlers.emplace_back(std::make_unique<CodeViewDebug>(this),
                             DbgTimerName, DbgTimerDescription,
                             CodeViewLineTablesGroupName,
                             CodeViewLineTablesGroupDescription);
     }
-    if (!EmitCodeView || MMI->getModule()->getDwarfVersion()) {
+    if (!EmitCodeView || M.getDwarfVersion()) {
       DD = new DwarfDebug(this, &M);
       DD->beginModule();
       Handlers.emplace_back(std::unique_ptr<DwarfDebug>(DD), DbgTimerName,
@@ -376,8 +375,7 @@ bool AsmPrinter::doInitialization(Module &M) {
                           DWARFGroupDescription);
 
   // Emit tables for any value of cfguard flag (i.e. cfguard=1 or cfguard=2).
-  if (mdconst::extract_or_null<ConstantInt>(
-          MMI->getModule()->getModuleFlag("cfguard")))
+  if (mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("cfguard")))
     Handlers.emplace_back(std::make_unique<WinCFGuard>(this), CFGuardName,
                           CFGuardDescription, DWARFGroupName,
                           DWARFGroupDescription);
@@ -684,8 +682,7 @@ void AsmPrinter::emitFunctionHeader() {
   if (!MAI->hasVisibilityOnlyWithLinkage())
     emitVisibility(CurrentFnSym, F.getVisibility());
 
-  if (MAI->needsFunctionDescriptors() &&
-      F.getLinkage() != GlobalValue::InternalLinkage)
+  if (MAI->needsFunctionDescriptors())
     emitLinkage(&F, CurrentFnDescSym);
 
   emitLinkage(&F, CurrentFnSym);
@@ -879,7 +876,7 @@ static bool emitDebugValueComment(const MachineInstr *MI, AsmPrinter &AP) {
   OS << " <- ";
 
   // The second operand is only an offset if it's an immediate.
-  bool MemLoc = MI->getOperand(0).isReg() && MI->getOperand(1).isImm();
+  bool MemLoc = MI->isIndirectDebugValue();
   int64_t Offset = MemLoc ? MI->getOperand(1).getImm() : 0;
   const DIExpression *Expr = MI->getDebugExpression();
   if (Expr->getNumElements()) {
@@ -898,11 +895,11 @@ static bool emitDebugValueComment(const MachineInstr *MI, AsmPrinter &AP) {
   }
 
   // Register or immediate value. Register 0 means undef.
-  if (MI->getOperand(0).isFPImm()) {
-    APFloat APF = APFloat(MI->getOperand(0).getFPImm()->getValueAPF());
-    if (MI->getOperand(0).getFPImm()->getType()->isFloatTy()) {
+  if (MI->getDebugOperand(0).isFPImm()) {
+    APFloat APF = APFloat(MI->getDebugOperand(0).getFPImm()->getValueAPF());
+    if (MI->getDebugOperand(0).getFPImm()->getType()->isFloatTy()) {
       OS << (double)APF.convertToFloat();
-    } else if (MI->getOperand(0).getFPImm()->getType()->isDoubleTy()) {
+    } else if (MI->getDebugOperand(0).getFPImm()->getType()->isDoubleTy()) {
       OS << APF.convertToDouble();
     } else {
       // There is no good way to print long double.  Convert a copy to
@@ -912,23 +909,23 @@ static bool emitDebugValueComment(const MachineInstr *MI, AsmPrinter &AP) {
                   &ignored);
       OS << "(long double) " << APF.convertToDouble();
     }
-  } else if (MI->getOperand(0).isImm()) {
-    OS << MI->getOperand(0).getImm();
-  } else if (MI->getOperand(0).isCImm()) {
-    MI->getOperand(0).getCImm()->getValue().print(OS, false /*isSigned*/);
-  } else if (MI->getOperand(0).isTargetIndex()) {
-    auto Op = MI->getOperand(0);
+  } else if (MI->getDebugOperand(0).isImm()) {
+    OS << MI->getDebugOperand(0).getImm();
+  } else if (MI->getDebugOperand(0).isCImm()) {
+    MI->getDebugOperand(0).getCImm()->getValue().print(OS, false /*isSigned*/);
+  } else if (MI->getDebugOperand(0).isTargetIndex()) {
+    auto Op = MI->getDebugOperand(0);
     OS << "!target-index(" << Op.getIndex() << "," << Op.getOffset() << ")";
     return true;
   } else {
     Register Reg;
-    if (MI->getOperand(0).isReg()) {
-      Reg = MI->getOperand(0).getReg();
+    if (MI->getDebugOperand(0).isReg()) {
+      Reg = MI->getDebugOperand(0).getReg();
     } else {
-      assert(MI->getOperand(0).isFI() && "Unknown operand type");
+      assert(MI->getDebugOperand(0).isFI() && "Unknown operand type");
       const TargetFrameLowering *TFI = AP.MF->getSubtarget().getFrameLowering();
-      Offset += TFI->getFrameIndexReference(*AP.MF,
-                                            MI->getOperand(0).getIndex(), Reg);
+      Offset += TFI->getFrameIndexReference(
+          *AP.MF, MI->getDebugOperand(0).getIndex(), Reg);
       MemLoc = true;
     }
     if (Reg == 0) {
@@ -1051,9 +1048,9 @@ void AsmPrinter::emitStackSizeSection(const MachineFunction &MF) {
   OutStreamer->PopSection();
 }
 
-static bool needFuncLabelsForEHOrDebugInfo(const MachineFunction &MF,
-                                           MachineModuleInfo *MMI) {
-  if (!MF.getLandingPads().empty() || MF.hasEHFunclets() || MMI->hasDebugInfo())
+static bool needFuncLabelsForEHOrDebugInfo(const MachineFunction &MF) {
+  MachineModuleInfo &MMI = MF.getMMI();
+  if (!MF.getLandingPads().empty() || MF.hasEHFunclets() || MMI.hasDebugInfo())
     return true;
 
   // We might emit an EH table that uses function begin and end labels even if
@@ -1209,6 +1206,8 @@ void AsmPrinter::emitFunctionBody() {
       if (!MBB.sameSection(&MF->front())) {
         if (MAI->hasDotTypeDotSizeDirective())
           emitELFSizeDirective(CurrentSectionBeginSym);
+        MBBSectionRanges[MBB.getSectionIDNum()] =
+            MBBSectionRange{CurrentSectionBeginSym, CurrentBBEnd};
       }
     }
     emitBasicBlockEnd(MBB);
@@ -1261,7 +1260,7 @@ void AsmPrinter::emitFunctionBody() {
   // Emit target-specific gunk after the function body.
   emitFunctionBodyEnd();
 
-  if (needFuncLabelsForEHOrDebugInfo(*MF, MMI) ||
+  if (needFuncLabelsForEHOrDebugInfo(*MF) ||
       MAI->hasDotTypeDotSizeDirective()) {
     // Create a symbol for the end of function.
     CurrentFnEnd = createTempSymbol("func_end");
@@ -1285,6 +1284,8 @@ void AsmPrinter::emitFunctionBody() {
     HI.Handler->markFunctionEnd();
   }
 
+  MBBSectionRanges[MF->front().getSectionIDNum()] =
+      MBBSectionRange{CurrentFnBegin, CurrentFnEnd};
 
   // Print out jump tables referenced by the function.
   emitJumpTableInfo();
@@ -1784,12 +1785,13 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) {
   CurrentFnSymForSize = CurrentFnSym;
   CurrentFnBegin = nullptr;
   CurrentSectionBeginSym = nullptr;
+  MBBSectionRanges.clear();
   CurExceptionSym = nullptr;
   bool NeedsLocalForSize = MAI->needsLocalForSize();
   if (F.hasFnAttribute("patchable-function-entry") ||
       F.hasFnAttribute("function-instrument") ||
       F.hasFnAttribute("xray-instruction-threshold") ||
-      needFuncLabelsForEHOrDebugInfo(MF, MMI) || NeedsLocalForSize ||
+      needFuncLabelsForEHOrDebugInfo(MF) || NeedsLocalForSize ||
       MF.getTarget().Options.EmitStackSizeSection) {
     CurrentFnBegin = createTempSymbol("func_begin");
     if (NeedsLocalForSize)
@@ -3055,16 +3057,13 @@ void AsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) {
 
   if (MBB.pred_empty() ||
       (!MF->hasBBLabels() && isBlockOnlyReachableByFallthrough(&MBB) &&
-       !MBB.isEHFuncletEntry() && !MBB.hasLabelMustBeEmitted())) {
+       !MBB.isEHFuncletEntry())) {
     if (isVerbose()) {
       // NOTE: Want this comment at start of line, don't emit with AddComment.
       OutStreamer->emitRawComment(" %bb." + Twine(MBB.getNumber()) + ":",
                                   false);
     }
   } else {
-    if (isVerbose() && MBB.hasLabelMustBeEmitted()) {
-      OutStreamer->AddComment("Label of block must be emitted");
-    }
     // Switch to a new section if this basic block must begin a section.
     if (MBB.isBeginSection()) {
       OutStreamer->SwitchSection(
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
index 2a8ab091bf9a7..d81a9be26d39b 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
@@ -181,7 +181,8 @@ void AsmPrinter::emitDwarfStringOffset(DwarfStringPoolEntry S) const {
 }
 
 void AsmPrinter::emitDwarfOffset(const MCSymbol *Label, uint64_t Offset) const {
-  emitLabelPlusOffset(Label, Offset, MAI->getCodePointerSize());
+  // TODO: Support DWARF64
+  emitLabelPlusOffset(Label, Offset, 4);
 }
 
 void AsmPrinter::emitCallSiteOffset(const MCSymbol *Hi, const MCSymbol *Lo,
diff --git a/llvm/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp b/llvm/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp
index 170fc8b6d49f5..584b7614915d8 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp
@@ -47,7 +47,8 @@ static Register isDescribedByReg(const MachineInstr &MI) {
     return 0;
   // If location of variable is described using a register (directly or
   // indirectly), this register is always a first operand.
-  return MI.getOperand(0).isReg() ? MI.getOperand(0).getReg() : Register();
+  return MI.getDebugOperand(0).isReg() ? MI.getDebugOperand(0).getReg()
+                                       : Register();
 }
 
 bool DbgValueHistoryMap::startDbgValue(InlinedEntity Var,
diff --git a/llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp b/llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp
index 86528acad1fd7..880791a06d931 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp
@@ -32,9 +32,9 @@ DbgVariableLocation::extractFromMachineInstruction(
   DbgVariableLocation Location;
   if (!Instruction.isDebugValue())
     return None;
-  if (!Instruction.getOperand(0).isReg())
+  if (!Instruction.getDebugOperand(0).isReg())
     return None;
-  Location.Register = Instruction.getOperand(0).getReg();
+  Location.Register = Instruction.getDebugOperand(0).getReg();
   Location.FragmentInfo.reset();
   // We only handle expressions generated by DIExpression::appendOffset,
   // which doesn't require a full stack machine.
@@ -200,7 +200,7 @@ void DebugHandlerBase::beginFunction(const MachineFunction *MF) {
       continue;
 
     auto IsDescribedByReg = [](const MachineInstr *MI) {
-      return MI->getOperand(0).isReg() && MI->getOperand(0).getReg();
+      return MI->getDebugOperand(0).isReg() && MI->getDebugOperand(0).getReg();
     };
 
     // The first mention of a function argument gets the CurrentFnBegin label,
@@ -327,3 +327,17 @@ void DebugHandlerBase::endFunction(const MachineFunction *MF) {
   LabelsBeforeInsn.clear();
   LabelsAfterInsn.clear();
 }
+
+void DebugHandlerBase::beginBasicBlock(const MachineBasicBlock &MBB) {
+  if (!MBB.isBeginSection())
+    return;
+
+  PrevLabel = MBB.getSymbol();
+}
+
+void DebugHandlerBase::endBasicBlock(const MachineBasicBlock &MBB) {
+  if (!MBB.isEndSection())
+    return;
+
+  PrevLabel = nullptr;
+}
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index 8fdc0201327af..296c380ae5508 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -396,7 +396,14 @@ void DwarfCompileUnit::attachLowHighPC(DIE &D, const MCSymbol *Begin,
 DIE &DwarfCompileUnit::updateSubprogramScopeDIE(const DISubprogram *SP) {
   DIE *SPDie = getOrCreateSubprogramDIE(SP, includeMinimalInlineScopes());
 
-  attachLowHighPC(*SPDie, Asm->getFunctionBegin(), Asm->getFunctionEnd());
+  SmallVector<RangeSpan, 2> BB_List;
+  // If basic block sections are on, ranges for each basic block section has
+  // to be emitted separately.
+  for (const auto &R : Asm->MBBSectionRanges)
+    BB_List.push_back({R.second.BeginLabel, R.second.EndLabel});
+
+  attachRangesOrLowHighPC(*SPDie, BB_List);
+
   if (DD->useAppleExtensionAttributes() &&
       !DD->getCurrentFunction()->getTarget().Options.DisableFramePointerElim(
           *DD->getCurrentFunction()))
@@ -434,9 +441,12 @@ DIE &DwarfCompileUnit::updateSubprogramScopeDIE(const DISubprogram *SP) {
         // GetExternalSymbolSymbol does, since if there's no code that
         // refers to this symbol, we have to set it here.
         SPSym->setType(wasm::WASM_SYMBOL_TYPE_GLOBAL);
-        // FIXME: need to check subtarget to see if its wasm64, but we
-        // can't cast to WebAssemblySubtarget here.
-        SPSym->setGlobalType(wasm::WasmGlobalType{wasm::WASM_TYPE_I32, true});
+        SPSym->setGlobalType(wasm::WasmGlobalType{
+            uint8_t(Asm->getSubtargetInfo().getTargetTriple().getArch() ==
+                            Triple::wasm64
+                        ? wasm::WASM_TYPE_I64
+                        : wasm::WASM_TYPE_I32),
+            true});
         DIELoc *Loc = new (DIEValueAllocator) DIELoc;
         addUInt(*Loc, dwarf::DW_FORM_data1, dwarf::DW_OP_WASM_location);
         addSInt(*Loc, dwarf::DW_FORM_sdata, FrameBase.Location.WasmLoc.Kind);
@@ -567,9 +577,33 @@ void DwarfCompileUnit::attachRangesOrLowHighPC(
     DIE &Die, const SmallVectorImpl<InsnRange> &Ranges) {
   SmallVector<RangeSpan, 2> List;
   List.reserve(Ranges.size());
-  for (const InsnRange &R : Ranges)
-    List.push_back(
-        {DD->getLabelBeforeInsn(R.first), DD->getLabelAfterInsn(R.second)});
+  for (const InsnRange &R : Ranges) {
+    auto *BeginLabel = DD->getLabelBeforeInsn(R.first);
+    auto *EndLabel = DD->getLabelAfterInsn(R.second);
+
+    const auto *BeginMBB = R.first->getParent();
+    const auto *EndMBB = R.second->getParent();
+
+    const auto *MBB = BeginMBB;
+    // Basic block sections allows basic block subsets to be placed in unique
+    // sections. For each section, the begin and end label must be added to the
+    // list. If there is more than one range, debug ranges must be used.
+    // Otherwise, low/high PC can be used.
+    // FIXME: Debug Info Emission depends on block order and this assumes that
+    // the order of blocks will be frozen beyond this point.
+    do {
+      if (MBB->sameSection(EndMBB) || MBB->isEndSection()) {
+        auto MBBSectionRange = Asm->MBBSectionRanges[MBB->getSectionIDNum()];
+        List.push_back(
+            {MBB->sameSection(BeginMBB) ? BeginLabel
+                                        : MBBSectionRange.BeginLabel,
+             MBB->sameSection(EndMBB) ? EndLabel : MBBSectionRange.EndLabel});
+      }
+      if (MBB->sameSection(EndMBB))
+        break;
+      MBB = MBB->getNextNode();
+    } while (true);
+  }
   attachRangesOrLowHighPC(Die, std::move(List));
 }
 
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 061330677680c..45ed5256deb99 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -167,6 +167,11 @@ static cl::opt<LinkageNameOption>
                                             "Abstract subprograms")),
                       cl::init(DefaultLinkageNames));
 
+static cl::opt<unsigned> LocationAnalysisSizeLimit(
+    "singlevarlocation-input-bb-limit",
+    cl::desc("Maximum block size to analyze for single-location variables"),
+    cl::init(30000), cl::Hidden);
+
 static const char *const DWARFGroupName = "dwarf";
 static const char *const DWARFGroupDescription = "DWARF Emission";
 static const char *const DbgTimerName = "writer";
@@ -236,26 +241,26 @@ const DIType *DbgVariable::getType() const {
 static DbgValueLoc getDebugLocValue(const MachineInstr *MI) {
   const DIExpression *Expr = MI->getDebugExpression();
   assert(MI->getNumOperands() == 4);
-  if (MI->getOperand(0).isReg()) {
-    auto RegOp = MI->getOperand(0);
-    auto Op1 = MI->getOperand(1);
+  if (MI->getDebugOperand(0).isReg()) {
+    auto RegOp = MI->getDebugOperand(0);
+    auto Op1 = MI->getDebugOffset();
     // If the second operand is an immediate, this is a
     // register-indirect address.
     assert((!Op1.isImm() || (Op1.getImm() == 0)) && "unexpected offset");
     MachineLocation MLoc(RegOp.getReg(), Op1.isImm());
     return DbgValueLoc(Expr, MLoc);
   }
-  if (MI->getOperand(0).isTargetIndex()) {
-    auto Op = MI->getOperand(0);
+  if (MI->getDebugOperand(0).isTargetIndex()) {
+    auto Op = MI->getDebugOperand(0);
     return DbgValueLoc(Expr,
                        TargetIndexLocation(Op.getIndex(), Op.getOffset()));
   }
-  if (MI->getOperand(0).isImm())
-    return DbgValueLoc(Expr, MI->getOperand(0).getImm());
-  if (MI->getOperand(0).isFPImm())
-    return DbgValueLoc(Expr, MI->getOperand(0).getFPImm());
-  if (MI->getOperand(0).isCImm())
-    return DbgValueLoc(Expr, MI->getOperand(0).getCImm());
+  if (MI->getDebugOperand(0).isImm())
+    return DbgValueLoc(Expr, MI->getDebugOperand(0).getImm());
+  if (MI->getDebugOperand(0).isFPImm())
+    return DbgValueLoc(Expr, MI->getDebugOperand(0).getFPImm());
+  if (MI->getDebugOperand(0).isCImm())
+    return DbgValueLoc(Expr, MI->getDebugOperand(0).getCImm());
 
   llvm_unreachable("Unexpected 4-operand DBG_VALUE instruction!");
 }
@@ -1577,7 +1582,7 @@ static bool validThroughout(LexicalScopes &LScopes,
   // throughout the function. This is a hack, presumably for DWARF v2 and not
   // necessarily correct. It would be much better to use a dbg.declare instead
   // if we know the constant is live throughout the scope.
-  if (DbgValue->getOperand(0).isImm() && MBB->pred_empty())
+  if (DbgValue->getDebugOperand(0).isImm() && MBB->pred_empty())
     return true;
 
   // Now check for situations where an "open-ended" DBG_VALUE isn't enough to
@@ -1637,8 +1642,10 @@ static bool validThroughout(LexicalScopes &LScopes,
 // [1-3)    [(reg0, fragment 0, 32), (reg1, fragment 32, 32)]
 // [3-4)    [(reg1, fragment 32, 32), (123, fragment 64, 32)]
 // [4-)     [(@g, fragment 0, 96)]
-bool DwarfDebug::buildLocationList(SmallVectorImpl<DebugLocEntry> &DebugLoc,
-                                   const DbgValueHistoryMap::Entries &Entries) {
+bool DwarfDebug::buildLocationList(
+    SmallVectorImpl<DebugLocEntry> &DebugLoc,
+    const DbgValueHistoryMap::Entries &Entries,
+    DenseSet<const MachineBasicBlock *> &VeryLargeBlocks) {
   using OpenRange =
       std::pair<DbgValueHistoryMap::EntryIndex, DbgValueLoc>;
   SmallVector<OpenRange, 4> OpenRanges;
@@ -1664,7 +1671,8 @@ bool DwarfDebug::buildLocationList(SmallVectorImpl<DebugLocEntry> &DebugLoc,
 
     const MCSymbol *EndLabel;
     if (std::next(EI) == Entries.end()) {
-      EndLabel = Asm->getFunctionEnd();
+      const MachineBasicBlock &EndMBB = Asm->MF->back();
+      EndLabel = Asm->MBBSectionRanges[EndMBB.getSectionIDNum()].EndLabel;
       if (EI->isClobber())
         EndMI = EI->getInstr();
     }
@@ -1733,8 +1741,14 @@ bool DwarfDebug::buildLocationList(SmallVectorImpl<DebugLocEntry> &DebugLoc,
       DebugLoc.pop_back();
   }
 
-  return DebugLoc.size() == 1 && isSafeForSingleLocation &&
-         validThroughout(LScopes, StartDebugMI, EndMI);
+  // If there's a single entry, safe for a single location, and not part of
+  // an over-sized basic block, then ask validThroughout whether this
+  // location can be represented as a single variable location.
+  if (DebugLoc.size() != 1 || !isSafeForSingleLocation)
+    return false;
+  if (VeryLargeBlocks.count(StartDebugMI->getParent()))
+    return false;
+  return validThroughout(LScopes, StartDebugMI, EndMI);
 }
 
 DbgEntity *DwarfDebug::createConcreteEntity(DwarfCompileUnit &TheCU,
@@ -1766,6 +1780,13 @@ void DwarfDebug::collectEntityInfo(DwarfCompileUnit &TheCU,
   // Grab the variable info that was squirreled away in the MMI side-table.
   collectVariableInfoFromMFTable(TheCU, Processed);
 
+  // Identify blocks that are unreasonably sized, so that we can later
+  // skip lexical scope analysis over them.
+  DenseSet<const MachineBasicBlock *> VeryLargeBlocks;
+  for (const auto &MBB : *CurFn)
+    if (MBB.size() > LocationAnalysisSizeLimit)
+      VeryLargeBlocks.insert(&MBB);
+
   for (const auto &I : DbgValues) {
     InlinedEntity IV = I.first;
     if (Processed.count(IV))
@@ -1802,7 +1823,8 @@ void DwarfDebug::collectEntityInfo(DwarfCompileUnit &TheCU,
     if (HistSize == 1 || SingleValueWithClobber) {
       const auto *End =
           SingleValueWithClobber ? HistoryMapEntries[1].getInstr() : nullptr;
-      if (validThroughout(LScopes, MInsn, End)) {
+      if (VeryLargeBlocks.count(MInsn->getParent()) == 0 &&
+          validThroughout(LScopes, MInsn, End)) {
         RegVar->initializeDbgValue(MInsn);
         continue;
       }
@@ -1817,7 +1839,8 @@ void DwarfDebug::collectEntityInfo(DwarfCompileUnit &TheCU,
 
     // Build the location list for this variable.
     SmallVector<DebugLocEntry, 8> Entries;
-    bool isValidSingleLocation = buildLocationList(Entries, HistoryMapEntries);
+    bool isValidSingleLocation =
+        buildLocationList(Entries, HistoryMapEntries, VeryLargeBlocks);
 
     // Check whether buildLocationList managed to merge all locations to one
     // that is valid throughout the variable's scope. If so, produce single
@@ -2115,7 +2138,9 @@ void DwarfDebug::endFunctionImpl(const MachineFunction *MF) {
   collectEntityInfo(TheCU, SP, Processed);
 
   // Add the range of this function to the list of ranges for the CU.
-  TheCU.addRange({Asm->getFunctionBegin(), Asm->getFunctionEnd()});
+  // With basic block sections, add ranges for all basic block sections.
+  for (const auto &R : Asm->MBBSectionRanges)
+    TheCU.addRange({R.second.BeginLabel, R.second.EndLabel});
 
   // Under -gmlt, skip building the subprogram if there are no inlined
   // subroutines inside it. But with -fdebug-info-for-profiling, the subprogram
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
index bb0f550b96541..d7a4b2abf52b3 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -592,8 +592,10 @@ class DwarfDebug : public DebugHandlerBase {
   /// function that describe the same variable. If the resulting 
   /// list has only one entry that is valid for entire variable's
   /// scope return true.
-  bool buildLocationList(SmallVectorImpl<DebugLocEntry> &DebugLoc,
-                         const DbgValueHistoryMap::Entries &Entries);
+  bool buildLocationList(
+      SmallVectorImpl<DebugLocEntry> &DebugLoc,
+      const DbgValueHistoryMap::Entries &Entries,
+      DenseSet<const MachineBasicBlock *> &VeryLargeBlocks);
 
   /// Collect variable information from the side table maintained by MF.
   void collectVariableInfoFromMFTable(DwarfCompileUnit &TheCU,
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
index 7b64c2238bd6a..d4762121d1050 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
@@ -237,8 +237,17 @@ bool DwarfExpression::addMachineRegExpression(const TargetRegisterInfo &TRI,
   // If the register can only be described by a complex expression (i.e.,
   // multiple subregisters) it doesn't safely compose with another complex
   // expression. For example, it is not possible to apply a DW_OP_deref
-  // operation to multiple DW_OP_pieces.
-  if (HasComplexExpression && DwarfRegs.size() > 1) {
+  // operation to multiple DW_OP_pieces, since composite location descriptions
+  // do not push anything on the DWARF stack.
+  //
+  // DW_OP_entry_value operations can only hold a DWARF expression or a
+  // register location description, so we can't emit a single entry value
+  // covering a composite location description. In the future we may want to
+  // emit entry value operations for each register location in the composite
+  // location, but until that is supported do not emit anything.
+  if ((HasComplexExpression || IsEmittingEntryValue) && DwarfRegs.size() > 1) {
+    if (IsEmittingEntryValue)
+      cancelEntryValue();
     DwarfRegs.clear();
     LocationKind = Unknown;
     return false;
@@ -349,7 +358,6 @@ void DwarfExpression::beginEntryValueExpression(
   assert(Op->getArg(0) == 1 &&
          "Can currently only emit entry values covering a single operation");
 
-  emitOp(CU.getDwarf5OrGNULocationAtom(dwarf::DW_OP_entry_value));
   IsEmittingEntryValue = true;
   enableTemporaryBuffer();
 }
@@ -358,6 +366,8 @@ void DwarfExpression::finalizeEntryValue() {
   assert(IsEmittingEntryValue && "Entry value not open?");
   disableTemporaryBuffer();
 
+  emitOp(CU.getDwarf5OrGNULocationAtom(dwarf::DW_OP_entry_value));
+
   // Emit the entry value's size operand.
   unsigned Size = getTemporaryBufferSize();
   emitUnsigned(Size);
@@ -368,6 +378,18 @@ void DwarfExpression::finalizeEntryValue() {
   IsEmittingEntryValue = false;
 }
 
+void DwarfExpression::cancelEntryValue() {
+  assert(IsEmittingEntryValue && "Entry value not open?");
+  disableTemporaryBuffer();
+
+  // The temporary buffer can't be emptied, so for now just assert that nothing
+  // has been emitted to it.
+  assert(getTemporaryBufferSize() == 0 &&
+         "Began emitting entry value block before cancelling entry value");
+
+  IsEmittingEntryValue = false;
+}
+
 unsigned DwarfExpression::getOrCreateBaseType(unsigned BitSize,
                                               dwarf::TypeKind Encoding) {
   // Reuse the base_type if we already have one in this CU otherwise we
@@ -401,6 +423,10 @@ static bool isMemoryLocation(DIExpressionCursor ExprCursor) {
 
 void DwarfExpression::addExpression(DIExpressionCursor &&ExprCursor,
                                     unsigned FragmentOffsetInBits) {
+  // Entry values can currently only cover the initial register location,
+  // and not any other parts of the following DWARF expression.
+  assert(!IsEmittingEntryValue && "Can't emit entry value around expression");
+
   // If we need to mask out a subregister, do it now, unless the next
   // operation would emit an OpPiece anyway.
   auto N = ExprCursor.peek();
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
index 42be827cd5a09..757b175114535 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
@@ -276,6 +276,9 @@ class DwarfExpression {
   /// DWARF block which has been emitted to the temporary buffer.
   void finalizeEntryValue();
 
+  /// Cancel the emission of an entry value.
+  void cancelEntryValue();
+
   ~DwarfExpression() = default;
 
 public:
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 9bec110604d9b..a5030305435c1 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -89,7 +89,7 @@ namespace {
         AtomicRMWInst *I,
         TargetLoweringBase::AtomicExpansionKind ExpansionKind);
     AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI);
-    void expandPartwordCmpXchg(AtomicCmpXchgInst *I);
+    bool expandPartwordCmpXchg(AtomicCmpXchgInst *I);
     void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI);
     void expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI);
 
@@ -105,7 +105,7 @@ namespace {
     bool isIdempotentRMW(AtomicRMWInst *RMWI);
     bool simplifyIdempotentRMW(AtomicRMWInst *RMWI);
 
-    bool expandAtomicOpToLibcall(Instruction *I, unsigned Size, unsigned Align,
+    bool expandAtomicOpToLibcall(Instruction *I, unsigned Size, Align Alignment,
                                  Value *PointerOperand, Value *ValueOperand,
                                  Value *CASExpected, AtomicOrdering Ordering,
                                  AtomicOrdering Ordering2,
@@ -152,47 +152,15 @@ static unsigned getAtomicOpSize(AtomicCmpXchgInst *CASI) {
   return DL.getTypeStoreSize(CASI->getCompareOperand()->getType());
 }
 
-// Helper functions to retrieve the alignment of atomic instructions.
-static unsigned getAtomicOpAlign(LoadInst *LI) {
-  unsigned Align = LI->getAlignment();
-  // In the future, if this IR restriction is relaxed, we should
-  // return DataLayout::getABITypeAlignment when there's no align
-  // value.
-  assert(Align != 0 && "An atomic LoadInst always has an explicit alignment");
-  return Align;
-}
-
-static unsigned getAtomicOpAlign(StoreInst *SI) {
-  unsigned Align = SI->getAlignment();
-  // In the future, if this IR restriction is relaxed, we should
-  // return DataLayout::getABITypeAlignment when there's no align
-  // value.
-  assert(Align != 0 && "An atomic StoreInst always has an explicit alignment");
-  return Align;
-}
-
-static unsigned getAtomicOpAlign(AtomicRMWInst *RMWI) {
-  // TODO(PR27168): This instruction has no alignment attribute, but unlike the
-  // default alignment for load/store, the default here is to assume
-  // it has NATURAL alignment, not DataLayout-specified alignment.
-  const DataLayout &DL = RMWI->getModule()->getDataLayout();
-  return DL.getTypeStoreSize(RMWI->getValOperand()->getType());
-}
-
-static unsigned getAtomicOpAlign(AtomicCmpXchgInst *CASI) {
-  // TODO(PR27168): same comment as above.
-  const DataLayout &DL = CASI->getModule()->getDataLayout();
-  return DL.getTypeStoreSize(CASI->getCompareOperand()->getType());
-}
-
 // Determine if a particular atomic operation has a supported size,
 // and is of appropriate alignment, to be passed through for target
 // lowering. (Versus turning into a __atomic libcall)
 template <typename Inst>
 static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
   unsigned Size = getAtomicOpSize(I);
-  unsigned Align = getAtomicOpAlign(I);
-  return Align >= Size && Size <= TLI->getMaxAtomicSizeInBitsSupported() / 8;
+  Align Alignment = I->getAlign();
+  return Alignment >= Size &&
+         Size <= TLI->getMaxAtomicSizeInBitsSupported() / 8;
 }
 
 bool AtomicExpand::runOnFunction(Function &F) {
@@ -858,7 +826,7 @@ AtomicRMWInst *AtomicExpand::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
   return NewAI;
 }
 
-void AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
+bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
   // The basic idea here is that we're expanding a cmpxchg of a
   // smaller memory size up to a word-sized cmpxchg. To do this, we
   // need to add a retry-loop for strong cmpxchg, so that
@@ -981,6 +949,7 @@ void AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
 
   CI->replaceAllUsesWith(Res);
   CI->eraseFromParent();
+  return true;
 }
 
 void AtomicExpand::expandAtomicOpToLLSC(
@@ -1480,7 +1449,7 @@ bool AtomicExpand::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
     llvm_unreachable("Unhandled case in tryExpandAtomicCmpXchg");
   case TargetLoweringBase::AtomicExpansionKind::None:
     if (ValueSize < MinCASSize)
-      expandPartwordCmpXchg(CI);
+      return expandPartwordCmpXchg(CI);
     return false;
   case TargetLoweringBase::AtomicExpansionKind::LLSC: {
     return expandAtomicCmpXchg(CI);
@@ -1513,7 +1482,7 @@ bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
 // must be one of the potentially-specialized sizes, and the value
 // type must actually exist in C on the target (otherwise, the
 // function wouldn't actually be defined.)
-static bool canUseSizedAtomicCall(unsigned Size, unsigned Align,
+static bool canUseSizedAtomicCall(unsigned Size, Align Alignment,
                                   const DataLayout &DL) {
   // TODO: "LargestSize" is an approximation for "largest type that
   // you can express in C". It seems to be the case that int128 is
@@ -1523,7 +1492,7 @@ static bool canUseSizedAtomicCall(unsigned Size, unsigned Align,
   // really be some more reliable way in LLVM of determining integer
   // sizes which are valid in the target's C ABI...
   unsigned LargestSize = DL.getLargestLegalIntTypeSizeInBits() >= 64 ? 16 : 8;
-  return Align >= Size &&
+  return Alignment >= Size &&
          (Size == 1 || Size == 2 || Size == 4 || Size == 8 || Size == 16) &&
          Size <= LargestSize;
 }
@@ -1533,10 +1502,9 @@ void AtomicExpand::expandAtomicLoadToLibcall(LoadInst *I) {
       RTLIB::ATOMIC_LOAD,   RTLIB::ATOMIC_LOAD_1, RTLIB::ATOMIC_LOAD_2,
       RTLIB::ATOMIC_LOAD_4, RTLIB::ATOMIC_LOAD_8, RTLIB::ATOMIC_LOAD_16};
   unsigned Size = getAtomicOpSize(I);
-  unsigned Align = getAtomicOpAlign(I);
 
   bool expanded = expandAtomicOpToLibcall(
-      I, Size, Align, I->getPointerOperand(), nullptr, nullptr,
+      I, Size, I->getAlign(), I->getPointerOperand(), nullptr, nullptr,
       I->getOrdering(), AtomicOrdering::NotAtomic, Libcalls);
   (void)expanded;
   assert(expanded && "expandAtomicOpToLibcall shouldn't fail tor Load");
@@ -1547,11 +1515,10 @@ void AtomicExpand::expandAtomicStoreToLibcall(StoreInst *I) {
       RTLIB::ATOMIC_STORE,   RTLIB::ATOMIC_STORE_1, RTLIB::ATOMIC_STORE_2,
       RTLIB::ATOMIC_STORE_4, RTLIB::ATOMIC_STORE_8, RTLIB::ATOMIC_STORE_16};
   unsigned Size = getAtomicOpSize(I);
-  unsigned Align = getAtomicOpAlign(I);
 
   bool expanded = expandAtomicOpToLibcall(
-      I, Size, Align, I->getPointerOperand(), I->getValueOperand(), nullptr,
-      I->getOrdering(), AtomicOrdering::NotAtomic, Libcalls);
+      I, Size, I->getAlign(), I->getPointerOperand(), I->getValueOperand(),
+      nullptr, I->getOrdering(), AtomicOrdering::NotAtomic, Libcalls);
   (void)expanded;
   assert(expanded && "expandAtomicOpToLibcall shouldn't fail tor Store");
 }
@@ -1562,10 +1529,9 @@ void AtomicExpand::expandAtomicCASToLibcall(AtomicCmpXchgInst *I) {
       RTLIB::ATOMIC_COMPARE_EXCHANGE_2, RTLIB::ATOMIC_COMPARE_EXCHANGE_4,
       RTLIB::ATOMIC_COMPARE_EXCHANGE_8, RTLIB::ATOMIC_COMPARE_EXCHANGE_16};
   unsigned Size = getAtomicOpSize(I);
-  unsigned Align = getAtomicOpAlign(I);
 
   bool expanded = expandAtomicOpToLibcall(
-      I, Size, Align, I->getPointerOperand(), I->getNewValOperand(),
+      I, Size, I->getAlign(), I->getPointerOperand(), I->getNewValOperand(),
       I->getCompareOperand(), I->getSuccessOrdering(), I->getFailureOrdering(),
       Libcalls);
   (void)expanded;
@@ -1635,13 +1601,12 @@ void AtomicExpand::expandAtomicRMWToLibcall(AtomicRMWInst *I) {
   ArrayRef<RTLIB::Libcall> Libcalls = GetRMWLibcall(I->getOperation());
 
   unsigned Size = getAtomicOpSize(I);
-  unsigned Align = getAtomicOpAlign(I);
 
   bool Success = false;
   if (!Libcalls.empty())
     Success = expandAtomicOpToLibcall(
-        I, Size, Align, I->getPointerOperand(), I->getValOperand(), nullptr,
-        I->getOrdering(), AtomicOrdering::NotAtomic, Libcalls);
+        I, Size, I->getAlign(), I->getPointerOperand(), I->getValOperand(),
+        nullptr, I->getOrdering(), AtomicOrdering::NotAtomic, Libcalls);
 
   // The expansion failed: either there were no libcalls at all for
   // the operation (min/max), or there were only size-specialized
@@ -1672,7 +1637,7 @@ void AtomicExpand::expandAtomicRMWToLibcall(AtomicRMWInst *I) {
 // 'I' are extracted from the Instruction subclass by the
 // caller. Depending on the particular call, some will be null.
 bool AtomicExpand::expandAtomicOpToLibcall(
-    Instruction *I, unsigned Size, unsigned Align, Value *PointerOperand,
+    Instruction *I, unsigned Size, Align Alignment, Value *PointerOperand,
     Value *ValueOperand, Value *CASExpected, AtomicOrdering Ordering,
     AtomicOrdering Ordering2, ArrayRef<RTLIB::Libcall> Libcalls) {
   assert(Libcalls.size() == 6);
@@ -1683,10 +1648,10 @@ bool AtomicExpand::expandAtomicOpToLibcall(
   IRBuilder<> Builder(I);
   IRBuilder<> AllocaBuilder(&I->getFunction()->getEntryBlock().front());
 
-  bool UseSizedLibcall = canUseSizedAtomicCall(Size, Align, DL);
+  bool UseSizedLibcall = canUseSizedAtomicCall(Size, Alignment, DL);
   Type *SizedIntTy = Type::getIntNTy(Ctx, Size * 8);
 
-  const llvm::Align AllocaAlignment(DL.getPrefTypeAlignment(SizedIntTy));
+  const Align AllocaAlignment = DL.getPrefTypeAlign(SizedIntTy);
 
   // TODO: the "order" argument type is "int", not int32. So
   // getInt32Ty may be wrong if the arch uses e.g. 16-bit ints.
diff --git a/llvm/lib/CodeGen/BBSectionsPrepare.cpp b/llvm/lib/CodeGen/BBSectionsPrepare.cpp
index 5f92689c5e421..a35c4d813acca 100644
--- a/llvm/lib/CodeGen/BBSectionsPrepare.cpp
+++ b/llvm/lib/CodeGen/BBSectionsPrepare.cpp
@@ -448,7 +448,7 @@ bool BBSectionsPrepare::doInitialization(Module &M) {
 
 void BBSectionsPrepare::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesAll();
-  AU.addRequired<MachineModuleInfoWrapperPass>();
+  MachineFunctionPass::getAnalysisUsage(AU);
 }
 
 MachineFunctionPass *
diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp
index df79019a6402d..c6d5aa37834f6 100644
--- a/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/llvm/lib/CodeGen/BranchFolding.cpp
@@ -135,10 +135,8 @@ bool BranchFolderPass::runOnMachineFunction(MachineFunction &MF) {
   BranchFolder Folder(EnableTailMerge, /*CommonHoist=*/true, MBBFreqInfo,
                       getAnalysis<MachineBranchProbabilityInfo>(),
                       &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI());
-  auto *MMIWP = getAnalysisIfAvailable<MachineModuleInfoWrapperPass>();
-  return Folder.OptimizeFunction(
-      MF, MF.getSubtarget().getInstrInfo(), MF.getSubtarget().getRegisterInfo(),
-      MMIWP ? &MMIWP->getMMI() : nullptr);
+  return Folder.OptimizeFunction(MF, MF.getSubtarget().getInstrInfo(),
+                                 MF.getSubtarget().getRegisterInfo());
 }
 
 BranchFolder::BranchFolder(bool defaultEnableTailMerge, bool CommonHoist,
@@ -184,7 +182,6 @@ void BranchFolder::RemoveDeadBlock(MachineBasicBlock *MBB) {
 bool BranchFolder::OptimizeFunction(MachineFunction &MF,
                                     const TargetInstrInfo *tii,
                                     const TargetRegisterInfo *tri,
-                                    MachineModuleInfo *mmi,
                                     MachineLoopInfo *mli, bool AfterPlacement) {
   if (!tii) return false;
 
@@ -194,7 +191,6 @@ bool BranchFolder::OptimizeFunction(MachineFunction &MF,
   AfterBlockPlacement = AfterPlacement;
   TII = tii;
   TRI = tri;
-  MMI = mmi;
   MLI = mli;
   this->MRI = &MRI;
 
@@ -861,7 +857,7 @@ void BranchFolder::mergeCommonTails(unsigned commonTailIndex) {
       LiveRegs.clear();
       LiveRegs.addLiveOuts(*Pred);
       MachineBasicBlock::iterator InsertBefore = Pred->getFirstTerminator();
-      for (unsigned Reg : NewLiveIns) {
+      for (Register Reg : NewLiveIns) {
         if (!LiveRegs.available(*MRI, Reg))
           continue;
         DebugLoc DL;
@@ -1087,8 +1083,9 @@ bool BranchFolder::TailMergeBlocks(MachineFunction &MF) {
       if (!UniquePreds.insert(PBB).second)
         continue;
 
-      // Skip blocks which may jump to a landing pad. Can't tail merge these.
-      if (PBB->hasEHPadSuccessor())
+      // Skip blocks which may jump to a landing pad or jump from an asm blob.
+      // Can't tail merge these.
+      if (PBB->hasEHPadSuccessor() || PBB->mayHaveInlineAsmBr())
         continue;
 
       // After block placement, only consider predecessors that belong to the
@@ -1669,13 +1666,15 @@ bool BranchFolder::OptimizeBlock(MachineBasicBlock *MBB) {
 
     if (!MBB->isEHPad()) {
       // Check all the predecessors of this block.  If one of them has no fall
-      // throughs, move this block right after it.
+      // throughs, and analyzeBranch thinks it _could_ fallthrough to this
+      // block, move this block right after it.
       for (MachineBasicBlock *PredBB : MBB->predecessors()) {
         // Analyze the branch at the end of the pred.
         MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr;
         SmallVector<MachineOperand, 4> PredCond;
         if (PredBB != MBB && !PredBB->canFallThrough() &&
             !TII->analyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true) &&
+            (PredTBB == MBB || PredFBB == MBB) &&
             (!CurFallsThru || !CurTBB || !CurFBB) &&
             (!CurFallsThru || MBB->getNumber() >= PredBB->getNumber())) {
           // If the current block doesn't fall through, just move it.
@@ -1701,21 +1700,24 @@ bool BranchFolder::OptimizeBlock(MachineBasicBlock *MBB) {
     }
 
     if (!CurFallsThru) {
-      // Check all successors to see if we can move this block before it.
-      for (MachineBasicBlock *SuccBB : MBB->successors()) {
-        // Analyze the branch at the end of the block before the succ.
-        MachineFunction::iterator SuccPrev = --SuccBB->getIterator();
-
-        // If this block doesn't already fall-through to that successor, and if
-        // the succ doesn't already have a block that can fall through into it,
-        // and if the successor isn't an EH destination, we can arrange for the
-        // fallthrough to happen.
-        if (SuccBB != MBB && &*SuccPrev != MBB &&
-            !SuccPrev->canFallThrough() && !CurUnAnalyzable &&
-            !SuccBB->isEHPad()) {
-          MBB->moveBefore(SuccBB);
-          MadeChange = true;
-          goto ReoptimizeBlock;
+      // Check analyzable branch-successors to see if we can move this block
+      // before one.
+      if (!CurUnAnalyzable) {
+        for (MachineBasicBlock *SuccBB : {CurFBB, CurTBB}) {
+          if (!SuccBB)
+            continue;
+          // Analyze the branch at the end of the block before the succ.
+          MachineFunction::iterator SuccPrev = --SuccBB->getIterator();
+
+          // If this block doesn't already fall-through to that successor, and
+          // if the succ doesn't already have a block that can fall through into
+          // it, we can arrange for the fallthrough to happen.
+          if (SuccBB != MBB && &*SuccPrev != MBB &&
+              !SuccPrev->canFallThrough()) {
+            MBB->moveBefore(SuccBB);
+            MadeChange = true;
+            goto ReoptimizeBlock;
+          }
         }
       }
 
@@ -1774,9 +1776,9 @@ static MachineBasicBlock *findFalseBlock(MachineBasicBlock *BB,
 }
 
 template <class Container>
-static void addRegAndItsAliases(unsigned Reg, const TargetRegisterInfo *TRI,
+static void addRegAndItsAliases(Register Reg, const TargetRegisterInfo *TRI,
                                 Container &Set) {
-  if (Register::isPhysicalRegister(Reg)) {
+  if (Reg.isPhysical()) {
     for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
       Set.insert(*AI);
   } else {
@@ -1795,8 +1797,8 @@ static
 MachineBasicBlock::iterator findHoistingInsertPosAndDeps(MachineBasicBlock *MBB,
                                                   const TargetInstrInfo *TII,
                                                   const TargetRegisterInfo *TRI,
-                                                  SmallSet<unsigned,4> &Uses,
-                                                  SmallSet<unsigned,4> &Defs) {
+                                                  SmallSet<Register, 4> &Uses,
+                                                  SmallSet<Register, 4> &Defs) {
   MachineBasicBlock::iterator Loc = MBB->getFirstTerminator();
   if (!TII->isUnpredicatedTerminator(*Loc))
     return MBB->end();
@@ -1907,14 +1909,14 @@ bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) {
   // Find a suitable position to hoist the common instructions to. Also figure
   // out which registers are used or defined by instructions from the insertion
   // point to the end of the block.
-  SmallSet<unsigned, 4> Uses, Defs;
+  SmallSet<Register, 4> Uses, Defs;
   MachineBasicBlock::iterator Loc =
     findHoistingInsertPosAndDeps(MBB, TII, TRI, Uses, Defs);
   if (Loc == MBB->end())
     return false;
 
   bool HasDups = false;
-  SmallSet<unsigned, 4> ActiveDefsSet, AllDefsSet;
+  SmallSet<Register, 4> ActiveDefsSet, AllDefsSet;
   MachineBasicBlock::iterator TIB = TBB->begin();
   MachineBasicBlock::iterator FIB = FBB->begin();
   MachineBasicBlock::iterator TIE = TBB->end();
@@ -1998,7 +2000,7 @@ bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) {
       if (!AllDefsSet.count(Reg)) {
         continue;
       }
-      if (Register::isPhysicalRegister(Reg)) {
+      if (Reg.isPhysical()) {
         for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
           ActiveDefsSet.erase(*AI);
       } else {
@@ -2011,7 +2013,7 @@ bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) {
       if (!MO.isReg() || !MO.isDef() || MO.isDead())
         continue;
       Register Reg = MO.getReg();
-      if (!Reg || Register::isVirtualRegister(Reg))
+      if (!Reg || Reg.isVirtual())
         continue;
       addRegAndItsAliases(Reg, TRI, ActiveDefsSet);
       addRegAndItsAliases(Reg, TRI, AllDefsSet);
diff --git a/llvm/lib/CodeGen/BranchFolding.h b/llvm/lib/CodeGen/BranchFolding.h
index 56c14418ec341..49c6bcae2db43 100644
--- a/llvm/lib/CodeGen/BranchFolding.h
+++ b/llvm/lib/CodeGen/BranchFolding.h
@@ -45,7 +45,7 @@ class TargetRegisterInfo;
     /// given function.  Block placement changes the layout and may create new
     /// tail merging opportunities.
     bool OptimizeFunction(MachineFunction &MF, const TargetInstrInfo *tii,
-                          const TargetRegisterInfo *tri, MachineModuleInfo *mmi,
+                          const TargetRegisterInfo *tri,
                           MachineLoopInfo *mli = nullptr,
                           bool AfterPlacement = false);
 
@@ -124,7 +124,6 @@ class TargetRegisterInfo;
     const TargetInstrInfo *TII;
     const MachineRegisterInfo *MRI;
     const TargetRegisterInfo *TRI;
-    MachineModuleInfo *MMI;
     MachineLoopInfo *MLI;
     LivePhysRegs LiveRegs;
 
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 8b45cb841aa37..e8b8e6c93cf0d 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -60,7 +60,6 @@
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsAArch64.h"
-#include "llvm/IR/IntrinsicsX86.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Module.h"
@@ -246,7 +245,7 @@ static cl::opt<bool>
                               "CodeGenPrepare."));
 
 static cl::opt<bool> OptimizePhiTypes(
-    "cgp-optimize-phi-types", cl::Hidden, cl::init(true),
+    "cgp-optimize-phi-types", cl::Hidden, cl::init(false),
     cl::desc("Enable converting phi types in CodeGenPrepare"));
 
 namespace {
@@ -2823,8 +2822,9 @@ class TypePromotionTransaction {
   TypePromotionTransaction(SetOfInstrs &RemovedInsts)
       : RemovedInsts(RemovedInsts) {}
 
-  /// Advocate every changes made in that transaction.
-  void commit();
+  /// Advocate every changes made in that transaction. Return true if any change
+  /// happen.
+  bool commit();
 
   /// Undo all the changes made after the given point.
   void rollback(ConstRestorationPt Point);
@@ -2930,11 +2930,13 @@ TypePromotionTransaction::getRestorationPoint() const {
   return !Actions.empty() ? Actions.back().get() : nullptr;
 }
 
-void TypePromotionTransaction::commit() {
+bool TypePromotionTransaction::commit() {
   for (CommitPt It = Actions.begin(), EndIt = Actions.end(); It != EndIt;
        ++It)
     (*It)->commit();
+  bool Modified = !Actions.empty();
   Actions.clear();
+  return Modified;
 }
 
 void TypePromotionTransaction::rollback(
@@ -4354,15 +4356,20 @@ bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode,
           cast<ConstantInt>(AddrInst->getOperand(i))->getZExtValue();
         ConstantOffset += SL->getElementOffset(Idx);
       } else {
-        uint64_t TypeSize = DL.getTypeAllocSize(GTI.getIndexedType());
-        if (ConstantInt *CI = dyn_cast<ConstantInt>(AddrInst->getOperand(i))) {
-          const APInt &CVal = CI->getValue();
-          if (CVal.getMinSignedBits() <= 64) {
-            ConstantOffset += CVal.getSExtValue() * TypeSize;
-            continue;
+        TypeSize TS = DL.getTypeAllocSize(GTI.getIndexedType());
+        if (TS.isNonZero()) {
+          // The optimisations below currently only work for fixed offsets.
+          if (TS.isScalable())
+            return false;
+          int64_t TypeSize = TS.getFixedSize();
+          if (ConstantInt *CI =
+                  dyn_cast<ConstantInt>(AddrInst->getOperand(i))) {
+            const APInt &CVal = CI->getValue();
+            if (CVal.getMinSignedBits() <= 64) {
+              ConstantOffset += CVal.getSExtValue() * TypeSize;
+              continue;
+            }
           }
-        }
-        if (TypeSize) {  // Scales of zero don't do anything.
           // We only allow one variable index at the moment.
           if (VariableOperand != -1)
             return false;
@@ -4960,7 +4967,7 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
     TPT.rollback(LastKnownGood);
     return false;
   }
-  TPT.commit();
+  bool Modified = TPT.commit();
 
   // Get the combined AddrMode (or the only AddrMode, if we only had one).
   ExtAddrMode AddrMode = AddrModes.getAddrMode();
@@ -4974,7 +4981,7 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
                   })) {
     LLVM_DEBUG(dbgs() << "CGP: Found      local addrmode: " << AddrMode
                       << "\n");
-    return false;
+    return Modified;
   }
 
   // Insert this computation right after this user.  Since our caller is
@@ -5015,7 +5022,7 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
       // We can't add more than one pointer together, nor can we scale a
       // pointer (both of which seem meaningless).
       if (ResultPtr || AddrMode.Scale != 1)
-        return false;
+        return Modified;
 
       ResultPtr = AddrMode.ScaledReg;
       AddrMode.Scale = 0;
@@ -5032,12 +5039,12 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
       Type *ScaledRegTy = AddrMode.ScaledReg->getType();
       if (cast<IntegerType>(IntPtrTy)->getBitWidth() >
           cast<IntegerType>(ScaledRegTy)->getBitWidth())
-        return false;
+        return Modified;
     }
 
     if (AddrMode.BaseGV) {
       if (ResultPtr)
-        return false;
+        return Modified;
 
       ResultPtr = AddrMode.BaseGV;
     }
@@ -5061,7 +5068,7 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
         !AddrMode.BaseReg && !AddrMode.Scale && !AddrMode.BaseOffs) {
       SunkAddr = Constant::getNullValue(Addr->getType());
     } else if (!ResultPtr) {
-      return false;
+      return Modified;
     } else {
       Type *I8PtrTy =
           Builder.getInt8PtrTy(Addr->getType()->getPointerAddressSpace());
@@ -5146,7 +5153,7 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
         (ScalePtrTy && DL->isNonIntegralPointerType(ScalePtrTy)) ||
         (AddrMode.BaseGV &&
          DL->isNonIntegralPointerType(AddrMode.BaseGV->getType())))
-      return false;
+      return Modified;
 
     LLVM_DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode
                       << " for " << *MemoryInst << "\n");
@@ -5186,7 +5193,7 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
         Instruction *I = dyn_cast_or_null<Instruction>(Result);
         if (I && (Result != AddrMode.BaseReg))
           I->eraseFromParent();
-        return false;
+        return Modified;
       }
       if (AddrMode.Scale != 1)
         V = Builder.CreateMul(V, ConstantInt::get(IntPtrTy, AddrMode.Scale),
@@ -5322,7 +5329,7 @@ bool CodeGenPrepare::optimizeGatherScatterInst(Instruction *MemoryInst,
   if (!RewriteGEP && Ops.size() == 2)
     return false;
 
-  unsigned NumElts = cast<VectorType>(Ptr->getType())->getNumElements();
+  unsigned NumElts = cast<FixedVectorType>(Ptr->getType())->getNumElements();
 
   IRBuilder<> Builder(MemoryInst);
 
@@ -5577,9 +5584,9 @@ bool CodeGenPrepare::mergeSExts(Function &F) {
   return Changed;
 }
 
-// Spliting large data structures so that the GEPs accessing them can have
+// Splitting large data structures so that the GEPs accessing them can have
 // smaller offsets so that they can be sunk to the same blocks as their users.
-// For example, a large struct starting from %base is splitted into two parts
+// For example, a large struct starting from %base is split into two parts
 // where the second part starts from %new_base.
 //
 // Before:
@@ -5654,7 +5661,7 @@ bool CodeGenPrepare::splitLargeGEPOffsets() {
                                         GEP->getAddressSpace())) {
           // We need to create a new base if the offset to the current base is
           // too large to fit into the addressing mode. So, a very large struct
-          // may be splitted into several parts.
+          // may be split into several parts.
           BaseGEP = GEP;
           BaseOffset = Offset;
           NewBaseGEP = nullptr;
@@ -6621,7 +6628,7 @@ bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst *SVI) {
   if (!NewType)
     return false;
 
-  VectorType *SVIVecType = cast<VectorType>(SVI->getType());
+  auto *SVIVecType = cast<FixedVectorType>(SVI->getType());
   assert(!NewType->isVectorTy() && "Expected a scalar type!");
   assert(NewType->getScalarSizeInBits() == SVIVecType->getScalarSizeInBits() &&
          "Expected a type of the same size!");
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index a67072ce340eb..96d4efb856c1e 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -759,7 +759,7 @@ void updateDomTree(MachineDominatorTree *DomTree, const SSAIfConv &IfConv,
     assert(Node != HeadNode && "Cannot erase the head node");
     while (Node->getNumChildren()) {
       assert(Node->getBlock() == IfConv.Tail && "Unexpected children");
-      DomTree->changeImmediateDominator(Node->getChildren().back(), HeadNode);
+      DomTree->changeImmediateDominator(Node->back(), HeadNode);
     }
     DomTree->eraseNode(B);
   }
diff --git a/llvm/lib/CodeGen/ExpandReductions.cpp b/llvm/lib/CodeGen/ExpandReductions.cpp
index 294edc49fd40e..45f21c1085dda 100644
--- a/llvm/lib/CodeGen/ExpandReductions.cpp
+++ b/llvm/lib/CodeGen/ExpandReductions.cpp
@@ -125,7 +125,8 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
       if (!FMF.allowReassoc())
         Rdx = getOrderedReduction(Builder, Acc, Vec, getOpcode(ID), MRK);
       else {
-        if (!isPowerOf2_32(cast<VectorType>(Vec->getType())->getNumElements()))
+        if (!isPowerOf2_32(
+                cast<FixedVectorType>(Vec->getType())->getNumElements()))
           continue;
 
         Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), MRK);
@@ -146,7 +147,8 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
     case Intrinsic::experimental_vector_reduce_fmax:
     case Intrinsic::experimental_vector_reduce_fmin: {
       Value *Vec = II->getArgOperand(0);
-      if (!isPowerOf2_32(cast<VectorType>(Vec->getType())->getNumElements()))
+      if (!isPowerOf2_32(
+              cast<FixedVectorType>(Vec->getType())->getNumElements()))
         continue;
 
       Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), MRK);
diff --git a/llvm/lib/CodeGen/FixupStatepointCallerSaved.cpp b/llvm/lib/CodeGen/FixupStatepointCallerSaved.cpp
index 0c8d5218bab54..27319804049de 100644
--- a/llvm/lib/CodeGen/FixupStatepointCallerSaved.cpp
+++ b/llvm/lib/CodeGen/FixupStatepointCallerSaved.cpp
@@ -132,7 +132,7 @@ class FrameIndexesCache {
       }
       return FI;
     }
-    int FI = MFI.CreateSpillStackObject(Size, Size);
+    int FI = MFI.CreateSpillStackObject(Size, Align(Size));
     NumSpillSlotsAllocated++;
     Line.Slots.push_back(FI);
     ++Line.Index;
diff --git a/llvm/lib/CodeGen/GCRootLowering.cpp b/llvm/lib/CodeGen/GCRootLowering.cpp
index 5aff2d79435f5..c6730aa6b00d8 100644
--- a/llvm/lib/CodeGen/GCRootLowering.cpp
+++ b/llvm/lib/CodeGen/GCRootLowering.cpp
@@ -57,7 +57,6 @@ class LowerIntrinsics : public FunctionPass {
 /// GCMetadata record for each function.
 class GCMachineCodeAnalysis : public MachineFunctionPass {
   GCFunctionInfo *FI;
-  MachineModuleInfo *MMI;
   const TargetInstrInfo *TII;
 
   void FindSafePoints(MachineFunction &MF);
@@ -249,7 +248,6 @@ GCMachineCodeAnalysis::GCMachineCodeAnalysis() : MachineFunctionPass(ID) {}
 void GCMachineCodeAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
   MachineFunctionPass::getAnalysisUsage(AU);
   AU.setPreservesAll();
-  AU.addRequired<MachineModuleInfoWrapperPass>();
   AU.addRequired<GCModuleInfo>();
 }
 
@@ -310,7 +308,6 @@ bool GCMachineCodeAnalysis::runOnMachineFunction(MachineFunction &MF) {
     return false;
 
   FI = &getAnalysis<GCModuleInfo>().getFunctionInfo(MF.getFunction());
-  MMI = &getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
   TII = MF.getSubtarget().getInstrInfo();
 
   // Find the size of the stack frame.  There may be no correct static frame
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index 5bcc242d3528e..a7146515c4c96 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -119,7 +119,7 @@ void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
   }
   if (Attrs.hasAttribute(OpIdx, Attribute::Nest))
     Flags.setNest();
-  Flags.setOrigAlign(Align(DL.getABITypeAlignment(Arg.Ty)));
+  Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty));
 }
 
 template void
@@ -194,11 +194,11 @@ bool CallLowering::handleAssignments(CCState &CCInfo,
 
   unsigned NumArgs = Args.size();
   for (unsigned i = 0; i != NumArgs; ++i) {
-    MVT CurVT = MVT::getVT(Args[i].Ty);
-    if (Handler.assignArg(i, CurVT, CurVT, CCValAssign::Full, Args[i],
-                          Args[i].Flags[0], CCInfo)) {
-      if (!CurVT.isValid())
-        return false;
+    EVT CurVT = EVT::getEVT(Args[i].Ty);
+    if (!CurVT.isSimple() ||
+        Handler.assignArg(i, CurVT.getSimpleVT(), CurVT.getSimpleVT(),
+                          CCValAssign::Full, Args[i], Args[i].Flags[0],
+                          CCInfo)) {
       MVT NewVT = TLI->getRegisterTypeForCallingConv(
           F.getContext(), F.getCallingConv(), EVT(CurVT));
 
@@ -309,8 +309,10 @@ bool CallLowering::handleAssignments(CCState &CCInfo,
     // FIXME: Pack registers if we have more than one.
     Register ArgReg = Args[i].Regs[0];
 
-    MVT OrigVT = MVT::getVT(Args[i].Ty);
-    MVT VAVT = VA.getValVT();
+    EVT OrigVT = EVT::getEVT(Args[i].Ty);
+    EVT VAVT = VA.getValVT();
+    const LLT OrigTy = getLLTForType(*Args[i].Ty, DL);
+
     if (VA.isRegLoc()) {
       if (Handler.isIncomingArgumentHandler() && VAVT != OrigVT) {
         if (VAVT.getSizeInBits() < OrigVT.getSizeInBits()) {
@@ -332,7 +334,7 @@ bool CallLowering::handleAssignments(CCState &CCInfo,
           MIRBuilder.buildMerge(Args[i].OrigRegs[0], Args[i].Regs);
           continue;
         }
-        const LLT VATy(VAVT);
+        const LLT VATy(VAVT.getSimpleVT());
         Register NewReg =
             MIRBuilder.getMRI()->createGenericVirtualRegister(VATy);
         Handler.assignValueToReg(NewReg, VA.getLocReg(), VA);
@@ -340,7 +342,6 @@ bool CallLowering::handleAssignments(CCState &CCInfo,
         // or do an unmerge to get the lower block of elements.
         if (VATy.isVector() &&
             VATy.getNumElements() > OrigVT.getVectorNumElements()) {
-          const LLT OrigTy(OrigVT);
           // Just handle the case where the VA type is 2 * original type.
           if (VATy.getNumElements() != OrigVT.getVectorNumElements() * 2) {
             LLVM_DEBUG(dbgs()
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index fae73873ad30d..bbdefe3e5ca4d 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -232,12 +232,8 @@ int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
   // Always allocate at least one byte.
   Size = std::max<uint64_t>(Size, 1u);
 
-  unsigned Alignment = AI.getAlignment();
-  if (!Alignment)
-    Alignment = DL->getABITypeAlignment(AI.getAllocatedType());
-
   int &FI = FrameIndices[&AI];
-  FI = MF->getFrameInfo().CreateStackObject(Size, Alignment, false, &AI);
+  FI = MF->getFrameInfo().CreateStackObject(Size, AI.getAlign(), false, &AI);
   return FI;
 }
 
@@ -1063,7 +1059,7 @@ bool IRTranslator::translateGetElementPtr(const User &U,
   // splat vector.
   unsigned VectorWidth = 0;
   if (auto *VT = dyn_cast<VectorType>(U.getType()))
-    VectorWidth = VT->getNumElements();
+    VectorWidth = cast<FixedVectorType>(VT)->getNumElements();
 
   // We might need to splat the base pointer into a vector if the offsets
   // are vectors.
@@ -1942,7 +1938,7 @@ bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
   // anyway but that's not guaranteed.
   MIRBuilder.buildInstr(TargetOpcode::G_VAARG, {getOrCreateVReg(U)},
                         {getOrCreateVReg(*U.getOperand(0)),
-                         uint64_t(DL->getABITypeAlignment(U.getType()))});
+                         DL->getABITypeAlign(U.getType()).value()});
   return true;
 }
 
@@ -1950,7 +1946,7 @@ bool IRTranslator::translateInsertElement(const User &U,
                                           MachineIRBuilder &MIRBuilder) {
   // If it is a <1 x Ty> vector, use the scalar as it is
   // not a legal vector type in LLT.
-  if (cast<VectorType>(U.getType())->getNumElements() == 1)
+  if (cast<FixedVectorType>(U.getType())->getNumElements() == 1)
     return translateCopy(U, *U.getOperand(1), MIRBuilder);
 
   Register Res = getOrCreateVReg(U);
@@ -1965,7 +1961,7 @@ bool IRTranslator::translateExtractElement(const User &U,
                                            MachineIRBuilder &MIRBuilder) {
   // If it is a <1 x Ty> vector, use the scalar as it is
   // not a legal vector type in LLT.
-  if (cast<VectorType>(U.getOperand(0)->getType())->getNumElements() == 1)
+  if (cast<FixedVectorType>(U.getOperand(0)->getType())->getNumElements() == 1)
     return translateCopy(U, *U.getOperand(0), MIRBuilder);
 
   Register Res = getOrCreateVReg(U);
@@ -2023,9 +2019,6 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U,
                                           MachineIRBuilder &MIRBuilder) {
   const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
 
-  if (I.isWeak())
-    return false;
-
   auto &TLI = *MF->getSubtarget().getTargetLowering();
   auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
 
@@ -2391,10 +2384,18 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
   // Make our arguments/constants entry block fallthrough to the IR entry block.
   EntryBB->addSuccessor(&getMBB(F.front()));
 
+  if (CLI->fallBackToDAGISel(F)) {
+    OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
+                               F.getSubprogram(), &F.getEntryBlock());
+    R << "unable to lower function: " << ore::NV("Prototype", F.getType());
+    reportTranslationError(*MF, *TPC, *ORE, R);
+    return false;
+  }
+
   // Lower the actual args into this basic block.
   SmallVector<ArrayRef<Register>, 8> VRegArgs;
   for (const Argument &Arg: F.args()) {
-    if (DL->getTypeStoreSize(Arg.getType()) == 0)
+    if (DL->getTypeStoreSize(Arg.getType()).isZero())
       continue; // Don't handle zero sized types.
     ArrayRef<Register> VRegs = getOrCreateVRegs(Arg);
     VRegArgs.push_back(VRegs);
diff --git a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
index 3ac52b8e3e736..241d5bace248b 100644
--- a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
@@ -232,6 +232,11 @@ static void computeConstraintToUse(const TargetLowering *TLI,
   }
 }
 
+static unsigned getNumOpRegs(const MachineInstr &I, unsigned OpIdx) {
+  unsigned Flag = I.getOperand(OpIdx).getImm();
+  return InlineAsm::getNumOperandRegisters(Flag);
+}
+
 bool InlineAsmLowering::lowerInlineAsm(
     MachineIRBuilder &MIRBuilder, const CallBase &Call,
     std::function<ArrayRef<Register>(const Value &Val)> GetOrCreateVRegs)
@@ -317,6 +322,10 @@ bool InlineAsmLowering::lowerInlineAsm(
                   .addExternalSymbol(IA->getAsmString().c_str())
                   .addImm(ExtraInfo.get());
 
+  // Starting from this operand: flag followed by register(s) will be added as
+  // operands to Inst for each constraint. Used for matching input constraints.
+  unsigned StartIdx = Inst->getNumOperands();
+
   // Collects the output operands for later processing
   GISelAsmOperandInfoVector OutputOperands;
 
@@ -390,8 +399,43 @@ bool InlineAsmLowering::lowerInlineAsm(
       break;
     case InlineAsm::isInput: {
       if (OpInfo.isMatchingInputConstraint()) {
-        LLVM_DEBUG(dbgs() << "Tied input operands not supported yet\n");
-        return false;
+        unsigned DefIdx = OpInfo.getMatchedOperand();
+        // Find operand with register def that corresponds to DefIdx.
+        unsigned InstFlagIdx = StartIdx;
+        for (unsigned i = 0; i < DefIdx; ++i)
+          InstFlagIdx += getNumOpRegs(*Inst, InstFlagIdx) + 1;
+        assert(getNumOpRegs(*Inst, InstFlagIdx) == 1 && "Wrong flag");
+
+        unsigned MatchedOperandFlag = Inst->getOperand(InstFlagIdx).getImm();
+        if (InlineAsm::isMemKind(MatchedOperandFlag)) {
+          LLVM_DEBUG(dbgs() << "Matching input constraint to mem operand not "
+                               "supported. This should be target specific.\n");
+          return false;
+        }
+        if (!InlineAsm::isRegDefKind(MatchedOperandFlag) &&
+            !InlineAsm::isRegDefEarlyClobberKind(MatchedOperandFlag)) {
+          LLVM_DEBUG(dbgs() << "Unknown matching constraint\n");
+          return false;
+        }
+
+        // We want to tie input to register in next operand.
+        unsigned DefRegIdx = InstFlagIdx + 1;
+        Register Def = Inst->getOperand(DefRegIdx).getReg();
+
+        // Copy input to new vreg with same reg class as Def
+        const TargetRegisterClass *RC = MRI->getRegClass(Def);
+        ArrayRef<Register> SrcRegs = GetOrCreateVRegs(*OpInfo.CallOperandVal);
+        assert(SrcRegs.size() == 1 && "Single register is expected here");
+        Register Tmp = MRI->createVirtualRegister(RC);
+        MIRBuilder.buildCopy(Tmp, SrcRegs[0]);
+
+        // Add Flag and input register operand (Tmp) to Inst. Tie Tmp to Def.
+        unsigned UseFlag = InlineAsm::getFlagWord(InlineAsm::Kind_RegUse, 1);
+        unsigned Flag = InlineAsm::getFlagWordForMatchingOp(UseFlag, DefIdx);
+        Inst.addImm(Flag);
+        Inst.addReg(Tmp);
+        Inst->tieOperands(DefRegIdx, Inst->getNumOperands() - 1);
+        break;
       }
 
       if (OpInfo.ConstraintType == TargetLowering::C_Other &&
diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
index fc114c6edc0d0..f32278d070527 100644
--- a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
@@ -223,9 +223,6 @@ bool InstructionSelect::runOnMachineFunction(MachineFunction &MF) {
     return false;
   }
 #endif
-  auto &TLI = *MF.getSubtarget().getTargetLowering();
-  TLI.finalizeLowering(MF);
-
   // Determine if there are any calls in this machine function. Ported from
   // SelectionDAG.
   MachineFrameInfo &MFI = MF.getFrameInfo();
@@ -241,6 +238,9 @@ bool InstructionSelect::runOnMachineFunction(MachineFunction &MF) {
     }
   }
 
+  // FIXME: FinalizeISel pass calls finalizeLowering, so it's called twice.
+  auto &TLI = *MF.getSubtarget().getTargetLowering();
+  TLI.finalizeLowering(MF);
 
   LLVM_DEBUG({
     dbgs() << "Rules covered by selecting function: " << MF.getName() << ":";
@@ -249,11 +249,7 @@ bool InstructionSelect::runOnMachineFunction(MachineFunction &MF) {
     dbgs() << "\n\n";
   });
   CoverageInfo.emit(CoveragePrefix,
-                    MF.getSubtarget()
-                        .getTargetLowering()
-                        ->getTargetMachine()
-                        .getTarget()
-                        .getBackendName());
+                    TLI.getTargetMachine().getTarget().getBackendName());
 
   // If we successfully selected the function nothing is going to use the vreg
   // types after us (otherwise MIRPrinter would need them). Make sure the types
diff --git a/llvm/lib/CodeGen/GlobalMerge.cpp b/llvm/lib/CodeGen/GlobalMerge.cpp
index 6d736aa981a3c..1e20c02ba160e 100644
--- a/llvm/lib/CodeGen/GlobalMerge.cpp
+++ b/llvm/lib/CodeGen/GlobalMerge.cpp
@@ -464,7 +464,7 @@ bool GlobalMerge::doMerge(const SmallVectorImpl<GlobalVariable *> &Globals,
       Type *Ty = Globals[j]->getValueType();
 
       // Make sure we use the same alignment AsmPrinter would use.
-      Align Alignment(DL.getPreferredAlignment(Globals[j]));
+      Align Alignment = DL.getPreferredAlign(Globals[j]);
       unsigned Padding = alignTo(MergedSize, Alignment) - MergedSize;
       MergedSize += Padding;
       MergedSize += DL.getTypeAllocSize(Ty);
diff --git a/llvm/lib/CodeGen/HardwareLoops.cpp b/llvm/lib/CodeGen/HardwareLoops.cpp
index 0359cabc3589c..0ba7e920e5075 100644
--- a/llvm/lib/CodeGen/HardwareLoops.cpp
+++ b/llvm/lib/CodeGen/HardwareLoops.cpp
@@ -20,6 +20,7 @@
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
@@ -244,14 +245,17 @@ bool HardwareLoops::runOnFunction(Function &F) {
 // converted and the parent loop doesn't support containing a hardware loop.
 bool HardwareLoops::TryConvertLoop(Loop *L) {
   // Process nested loops first.
-  for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) {
-    if (TryConvertLoop(*I)) {
-      reportHWLoopFailure("nested hardware-loops not supported", "HWLoopNested",
-                          ORE, L);
-      return true; // Stop search.
-    }
+  bool AnyChanged = false;
+  for (Loop *SL : *L)
+    AnyChanged |= TryConvertLoop(SL);
+  if (AnyChanged) {
+    reportHWLoopFailure("nested hardware-loops not supported", "HWLoopNested",
+                        ORE, L);
+    return true; // Stop search.
   }
 
+  LLVM_DEBUG(dbgs() << "HWLoops: Loop " << L->getHeader()->getName() << "\n");
+
   HardwareLoopInfo HWLoopInfo(L);
   if (!HWLoopInfo.canAnalyze(*LI)) {
     reportHWLoopFailure("cannot analyze loop, irreducible control flow",
diff --git a/llvm/lib/CodeGen/IfConversion.cpp b/llvm/lib/CodeGen/IfConversion.cpp
index a3cace837eb9f..1a5c5d6850172 100644
--- a/llvm/lib/CodeGen/IfConversion.cpp
+++ b/llvm/lib/CodeGen/IfConversion.cpp
@@ -463,10 +463,7 @@ bool IfConverter::runOnMachineFunction(MachineFunction &MF) {
   if (!PreRegAlloc) {
     // Tail merge tend to expose more if-conversion opportunities.
     BranchFolder BF(true, false, MBFI, *MBPI, PSI);
-    auto *MMIWP = getAnalysisIfAvailable<MachineModuleInfoWrapperPass>();
-    BFChange = BF.OptimizeFunction(
-        MF, TII, ST.getRegisterInfo(),
-        MMIWP ? &MMIWP->getMMI() : nullptr);
+    BFChange = BF.OptimizeFunction(MF, TII, ST.getRegisterInfo());
   }
 
   LLVM_DEBUG(dbgs() << "\nIfcvt: function (" << ++FnNum << ") \'"
@@ -605,10 +602,7 @@ bool IfConverter::runOnMachineFunction(MachineFunction &MF) {
 
   if (MadeChange && IfCvtBranchFold) {
     BranchFolder BF(false, false, MBFI, *MBPI, PSI);
-    auto *MMIWP = getAnalysisIfAvailable<MachineModuleInfoWrapperPass>();
-    BF.OptimizeFunction(
-        MF, TII, MF.getSubtarget().getRegisterInfo(),
-        MMIWP ? &MMIWP->getMMI() : nullptr);
+    BF.OptimizeFunction(MF, TII, MF.getSubtarget().getRegisterInfo());
   }
 
   MadeChange |= BFChange;
diff --git a/llvm/lib/CodeGen/InlineSpiller.cpp b/llvm/lib/CodeGen/InlineSpiller.cpp
index 90dfaf67b062a..41eef2fed840e 100644
--- a/llvm/lib/CodeGen/InlineSpiller.cpp
+++ b/llvm/lib/CodeGen/InlineSpiller.cpp
@@ -114,10 +114,10 @@ class HoistSpillHelper : private LiveRangeEdit::Delegate {
   /// This is the map from original register to a set containing all its
   /// siblings. To hoist a spill to another BB, we need to find out a live
   /// sibling there and use it as the source of the new spill.
-  DenseMap<unsigned, SmallSetVector<unsigned, 16>> Virt2SiblingsMap;
+  DenseMap<Register, SmallSetVector<Register, 16>> Virt2SiblingsMap;
 
   bool isSpillCandBB(LiveInterval &OrigLI, VNInfo &OrigVNI,
-                     MachineBasicBlock &BB, unsigned &LiveReg);
+                     MachineBasicBlock &BB, Register &LiveReg);
 
   void rmRedundantSpills(
       SmallPtrSet<MachineInstr *, 16> &Spills,
@@ -176,7 +176,7 @@ class InlineSpiller : public Spiller {
   unsigned Original;
 
   // All registers to spill to StackSlot, including the main register.
-  SmallVector<unsigned, 8> RegsToSpill;
+  SmallVector<Register, 8> RegsToSpill;
 
   // All COPY instructions to/from snippets.
   // They are ignored since both operands refer to the same stack slot.
@@ -212,24 +212,24 @@ class InlineSpiller : public Spiller {
   bool isSnippet(const LiveInterval &SnipLI);
   void collectRegsToSpill();
 
-  bool isRegToSpill(unsigned Reg) { return is_contained(RegsToSpill, Reg); }
+  bool isRegToSpill(Register Reg) { return is_contained(RegsToSpill, Reg); }
 
-  bool isSibling(unsigned Reg);
+  bool isSibling(Register Reg);
   bool hoistSpillInsideBB(LiveInterval &SpillLI, MachineInstr &CopyMI);
   void eliminateRedundantSpills(LiveInterval &LI, VNInfo *VNI);
 
   void markValueUsed(LiveInterval*, VNInfo*);
-  bool canGuaranteeAssignmentAfterRemat(unsigned VReg, MachineInstr &MI);
+  bool canGuaranteeAssignmentAfterRemat(Register VReg, MachineInstr &MI);
   bool reMaterializeFor(LiveInterval &, MachineInstr &MI);
   void reMaterializeAll();
 
-  bool coalesceStackAccess(MachineInstr *MI, unsigned Reg);
+  bool coalesceStackAccess(MachineInstr *MI, Register Reg);
   bool foldMemoryOperand(ArrayRef<std::pair<MachineInstr *, unsigned>>,
                          MachineInstr *LoadMI = nullptr);
-  void insertReload(unsigned VReg, SlotIndex, MachineBasicBlock::iterator MI);
-  void insertSpill(unsigned VReg, bool isKill, MachineBasicBlock::iterator MI);
+  void insertReload(Register VReg, SlotIndex, MachineBasicBlock::iterator MI);
+  void insertSpill(Register VReg, bool isKill, MachineBasicBlock::iterator MI);
 
-  void spillAroundUses(unsigned Reg);
+  void spillAroundUses(Register Reg);
   void spillAll();
 };
 
@@ -259,21 +259,21 @@ Spiller *llvm::createInlineSpiller(MachineFunctionPass &pass,
 
 /// isFullCopyOf - If MI is a COPY to or from Reg, return the other register,
 /// otherwise return 0.
-static unsigned isFullCopyOf(const MachineInstr &MI, unsigned Reg) {
+static Register isFullCopyOf(const MachineInstr &MI, Register Reg) {
   if (!MI.isFullCopy())
-    return 0;
+    return Register();
   if (MI.getOperand(0).getReg() == Reg)
     return MI.getOperand(1).getReg();
   if (MI.getOperand(1).getReg() == Reg)
     return MI.getOperand(0).getReg();
-  return 0;
+  return Register();
 }
 
 /// isSnippet - Identify if a live interval is a snippet that should be spilled.
 /// It is assumed that SnipLI is a virtual register with the same original as
 /// Edit->getReg().
 bool InlineSpiller::isSnippet(const LiveInterval &SnipLI) {
-  unsigned Reg = Edit->getReg();
+  Register Reg = Edit->getReg();
 
   // A snippet is a tiny live range with only a single instruction using it
   // besides copies to/from Reg or spills/fills. We accept:
@@ -317,7 +317,7 @@ bool InlineSpiller::isSnippet(const LiveInterval &SnipLI) {
 /// collectRegsToSpill - Collect live range snippets that only have a single
 /// real use.
 void InlineSpiller::collectRegsToSpill() {
-  unsigned Reg = Edit->getReg();
+  Register Reg = Edit->getReg();
 
   // Main register always spills.
   RegsToSpill.assign(1, Reg);
@@ -331,7 +331,7 @@ void InlineSpiller::collectRegsToSpill() {
   for (MachineRegisterInfo::reg_instr_iterator
        RI = MRI.reg_instr_begin(Reg), E = MRI.reg_instr_end(); RI != E; ) {
     MachineInstr &MI = *RI++;
-    unsigned SnipReg = isFullCopyOf(MI, Reg);
+    Register SnipReg = isFullCopyOf(MI, Reg);
     if (!isSibling(SnipReg))
       continue;
     LiveInterval &SnipLI = LIS.getInterval(SnipReg);
@@ -346,8 +346,8 @@ void InlineSpiller::collectRegsToSpill() {
   }
 }
 
-bool InlineSpiller::isSibling(unsigned Reg) {
-  return Register::isVirtualRegister(Reg) && VRM.getOriginal(Reg) == Original;
+bool InlineSpiller::isSibling(Register Reg) {
+  return Reg.isVirtual() && VRM.getOriginal(Reg) == Original;
 }
 
 /// It is beneficial to spill to earlier place in the same BB in case
@@ -432,7 +432,7 @@ void InlineSpiller::eliminateRedundantSpills(LiveInterval &SLI, VNInfo *VNI) {
   do {
     LiveInterval *LI;
     std::tie(LI, VNI) = WorkList.pop_back_val();
-    unsigned Reg = LI->reg;
+    Register Reg = LI->reg;
     LLVM_DEBUG(dbgs() << "Checking redundant spills for " << VNI->id << '@'
                       << VNI->def << " in " << *LI << '\n');
 
@@ -456,7 +456,7 @@ void InlineSpiller::eliminateRedundantSpills(LiveInterval &SLI, VNInfo *VNI) {
         continue;
 
       // Follow sibling copies down the dominator tree.
-      if (unsigned DstReg = isFullCopyOf(MI, Reg)) {
+      if (Register DstReg = isFullCopyOf(MI, Reg)) {
         if (isSibling(DstReg)) {
            LiveInterval &DstLI = LIS.getInterval(DstReg);
            VNInfo *DstVNI = DstLI.getVNInfoAt(Idx.getRegSlot());
@@ -518,7 +518,7 @@ void InlineSpiller::markValueUsed(LiveInterval *LI, VNInfo *VNI) {
   } while (!WorkList.empty());
 }
 
-bool InlineSpiller::canGuaranteeAssignmentAfterRemat(unsigned VReg,
+bool InlineSpiller::canGuaranteeAssignmentAfterRemat(Register VReg,
                                                      MachineInstr &MI) {
   if (!RestrictStatepointRemat)
     return true;
@@ -615,7 +615,7 @@ bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, MachineInstr &MI) {
   }
 
   // Allocate a new register for the remat.
-  unsigned NewVReg = Edit->createFrom(Original);
+  Register NewVReg = Edit->createFrom(Original);
 
   // Finally we can rematerialize OrigMI before MI.
   SlotIndex DefIdx =
@@ -654,7 +654,7 @@ void InlineSpiller::reMaterializeAll() {
 
   // Try to remat before all uses of snippets.
   bool anyRemat = false;
-  for (unsigned Reg : RegsToSpill) {
+  for (Register Reg : RegsToSpill) {
     LiveInterval &LI = LIS.getInterval(Reg);
     for (MachineRegisterInfo::reg_bundle_iterator
            RegI = MRI.reg_bundle_begin(Reg), E = MRI.reg_bundle_end();
@@ -675,7 +675,7 @@ void InlineSpiller::reMaterializeAll() {
     return;
 
   // Remove any values that were completely rematted.
-  for (unsigned Reg : RegsToSpill) {
+  for (Register Reg : RegsToSpill) {
     LiveInterval &LI = LIS.getInterval(Reg);
     for (LiveInterval::vni_iterator I = LI.vni_begin(), E = LI.vni_end();
          I != E; ++I) {
@@ -705,7 +705,7 @@ void InlineSpiller::reMaterializeAll() {
   // So to get rid of unused reg, we need to check whether it has non-dbg
   // reference instead of whether it has non-empty interval.
   unsigned ResultPos = 0;
-  for (unsigned Reg : RegsToSpill) {
+  for (Register Reg : RegsToSpill) {
     if (MRI.reg_nodbg_empty(Reg)) {
       Edit->eraseVirtReg(Reg);
       continue;
@@ -727,9 +727,9 @@ void InlineSpiller::reMaterializeAll() {
 //===----------------------------------------------------------------------===//
 
 /// If MI is a load or store of StackSlot, it can be removed.
-bool InlineSpiller::coalesceStackAccess(MachineInstr *MI, unsigned Reg) {
+bool InlineSpiller::coalesceStackAccess(MachineInstr *MI, Register Reg) {
   int FI = 0;
-  unsigned InstrReg = TII.isLoadFromStackSlot(*MI, FI);
+  Register InstrReg = TII.isLoadFromStackSlot(*MI, FI);
   bool IsLoad = InstrReg;
   if (!IsLoad)
     InstrReg = TII.isStoreToStackSlot(*MI, FI);
@@ -763,7 +763,7 @@ static void dumpMachineInstrRangeWithSlotIndex(MachineBasicBlock::iterator B,
                                                MachineBasicBlock::iterator E,
                                                LiveIntervals const &LIS,
                                                const char *const header,
-                                               unsigned VReg =0) {
+                                               Register VReg = Register()) {
   char NextLine = '\n';
   char SlotIndent = '\t';
 
@@ -808,7 +808,7 @@ foldMemoryOperand(ArrayRef<std::pair<MachineInstr *, unsigned>> Ops,
     return false;
 
   bool WasCopy = MI->isCopy();
-  unsigned ImpReg = 0;
+  Register ImpReg;
 
   // Spill subregs if the target allows it.
   // We always want to spill subregs for stackmap/patchpoint pseudos.
@@ -912,7 +912,7 @@ foldMemoryOperand(ArrayRef<std::pair<MachineInstr *, unsigned>> Ops,
   return true;
 }
 
-void InlineSpiller::insertReload(unsigned NewVReg,
+void InlineSpiller::insertReload(Register NewVReg,
                                  SlotIndex Idx,
                                  MachineBasicBlock::iterator MI) {
   MachineBasicBlock &MBB = *MI->getParent();
@@ -943,7 +943,7 @@ static bool isRealSpill(const MachineInstr &Def) {
 }
 
 /// insertSpill - Insert a spill of NewVReg after MI.
-void InlineSpiller::insertSpill(unsigned NewVReg, bool isKill,
+void InlineSpiller::insertSpill(Register NewVReg, bool isKill,
                                  MachineBasicBlock::iterator MI) {
   // Spill are not terminators, so inserting spills after terminators will
   // violate invariants in MachineVerifier.
@@ -975,7 +975,7 @@ void InlineSpiller::insertSpill(unsigned NewVReg, bool isKill,
 }
 
 /// spillAroundUses - insert spill code around each use of Reg.
-void InlineSpiller::spillAroundUses(unsigned Reg) {
+void InlineSpiller::spillAroundUses(Register Reg) {
   LLVM_DEBUG(dbgs() << "spillAroundUses " << printReg(Reg) << '\n');
   LiveInterval &OldLI = LIS.getInterval(Reg);
 
@@ -1018,7 +1018,7 @@ void InlineSpiller::spillAroundUses(unsigned Reg) {
         Idx = VNI->def;
 
     // Check for a sibling copy.
-    unsigned SibReg = isFullCopyOf(*MI, Reg);
+    Register SibReg = isFullCopyOf(*MI, Reg);
     if (SibReg && isSibling(SibReg)) {
       // This may actually be a copy between snippets.
       if (isRegToSpill(SibReg)) {
@@ -1047,7 +1047,7 @@ void InlineSpiller::spillAroundUses(unsigned Reg) {
 
     // Create a new virtual register for spill/fill.
     // FIXME: Infer regclass from instruction alone.
-    unsigned NewVReg = Edit->createFrom(Reg);
+    Register NewVReg = Edit->createFrom(Reg);
 
     if (RI.Reads)
       insertReload(NewVReg, Idx, MI);
@@ -1088,13 +1088,13 @@ void InlineSpiller::spillAll() {
     VRM.assignVirt2StackSlot(Edit->getReg(), StackSlot);
 
   assert(StackInt->getNumValNums() == 1 && "Bad stack interval values");
-  for (unsigned Reg : RegsToSpill)
+  for (Register Reg : RegsToSpill)
     StackInt->MergeSegmentsInAsValue(LIS.getInterval(Reg),
                                      StackInt->getValNumInfo(0));
   LLVM_DEBUG(dbgs() << "Merged spilled regs: " << *StackInt << '\n');
 
   // Spill around uses of all RegsToSpill.
-  for (unsigned Reg : RegsToSpill)
+  for (Register Reg : RegsToSpill)
     spillAroundUses(Reg);
 
   // Hoisted spills may cause dead code.
@@ -1104,7 +1104,7 @@ void InlineSpiller::spillAll() {
   }
 
   // Finally delete the SnippetCopies.
-  for (unsigned Reg : RegsToSpill) {
+  for (Register Reg : RegsToSpill) {
     for (MachineRegisterInfo::reg_instr_iterator
          RI = MRI.reg_instr_begin(Reg), E = MRI.reg_instr_end();
          RI != E; ) {
@@ -1117,7 +1117,7 @@ void InlineSpiller::spillAll() {
   }
 
   // Delete all spilled registers.
-  for (unsigned Reg : RegsToSpill)
+  for (Register Reg : RegsToSpill)
     Edit->eraseVirtReg(Reg);
 }
 
@@ -1186,18 +1186,18 @@ bool HoistSpillHelper::rmFromMergeableSpills(MachineInstr &Spill,
 /// Check BB to see if it is a possible target BB to place a hoisted spill,
 /// i.e., there should be a living sibling of OrigReg at the insert point.
 bool HoistSpillHelper::isSpillCandBB(LiveInterval &OrigLI, VNInfo &OrigVNI,
-                                     MachineBasicBlock &BB, unsigned &LiveReg) {
+                                     MachineBasicBlock &BB, Register &LiveReg) {
   SlotIndex Idx;
-  unsigned OrigReg = OrigLI.reg;
+  Register OrigReg = OrigLI.reg;
   MachineBasicBlock::iterator MI = IPA.getLastInsertPointIter(OrigLI, BB);
   if (MI != BB.end())
     Idx = LIS.getInstructionIndex(*MI);
   else
     Idx = LIS.getMBBEndIdx(&BB).getPrevSlot();
-  SmallSetVector<unsigned, 16> &Siblings = Virt2SiblingsMap[OrigReg];
+  SmallSetVector<Register, 16> &Siblings = Virt2SiblingsMap[OrigReg];
   assert(OrigLI.getVNInfoAt(Idx) == &OrigVNI && "Unexpected VNI");
 
-  for (auto const SibReg : Siblings) {
+  for (const Register &SibReg : Siblings) {
     LiveInterval &LI = LIS.getInterval(SibReg);
     VNInfo *VNI = LI.getVNInfoAt(Idx);
     if (VNI) {
@@ -1306,10 +1306,7 @@ void HoistSpillHelper::getVisitOrders(
   Orders.push_back(MDT.getBase().getNode(Root));
   do {
     MachineDomTreeNode *Node = Orders[idx++];
-    const std::vector<MachineDomTreeNode *> &Children = Node->getChildren();
-    unsigned NumChildren = Children.size();
-    for (unsigned i = 0; i != NumChildren; ++i) {
-      MachineDomTreeNode *Child = Children[i];
+    for (MachineDomTreeNode *Child : Node->children()) {
       if (WorkSet.count(Child))
         Orders.push_back(Child);
     }
@@ -1377,10 +1374,7 @@ void HoistSpillHelper::runHoistSpills(
 
     // Collect spills in subtree of current node (*RIt) to
     // SpillsInSubTreeMap[*RIt].first.
-    const std::vector<MachineDomTreeNode *> &Children = (*RIt)->getChildren();
-    unsigned NumChildren = Children.size();
-    for (unsigned i = 0; i != NumChildren; ++i) {
-      MachineDomTreeNode *Child = Children[i];
+    for (MachineDomTreeNode *Child : (*RIt)->children()) {
       if (SpillsInSubTreeMap.find(Child) == SpillsInSubTreeMap.end())
         continue;
       // The stmt "SpillsInSubTree = SpillsInSubTreeMap[*RIt].first" below
@@ -1406,7 +1400,7 @@ void HoistSpillHelper::runHoistSpills(
       continue;
 
     // Check whether Block is a possible candidate to insert spill.
-    unsigned LiveReg = 0;
+    Register LiveReg;
     if (!isSpillCandBB(OrigLI, OrigVNI, *Block, LiveReg))
       continue;
 
@@ -1468,12 +1462,12 @@ void HoistSpillHelper::runHoistSpills(
 /// inside its subtree to that node. In this way, we can get benefit locally
 /// even if hoisting all the equal spills to one cold place is impossible.
 void HoistSpillHelper::hoistAllSpills() {
-  SmallVector<unsigned, 4> NewVRegs;
+  SmallVector<Register, 4> NewVRegs;
   LiveRangeEdit Edit(nullptr, NewVRegs, MF, LIS, &VRM, this);
 
   for (unsigned i = 0, e = MRI.getNumVirtRegs(); i != e; ++i) {
-    unsigned Reg = Register::index2VirtReg(i);
-    unsigned Original = VRM.getPreSplitReg(Reg);
+    Register Reg = Register::index2VirtReg(i);
+    Register Original = VRM.getPreSplitReg(Reg);
     if (!MRI.def_empty(Reg))
       Virt2SiblingsMap[Original].insert(Reg);
   }
@@ -1521,7 +1515,7 @@ void HoistSpillHelper::hoistAllSpills() {
     // Insert hoisted spills.
     for (auto const &Insert : SpillsToIns) {
       MachineBasicBlock *BB = Insert.first;
-      unsigned LiveReg = Insert.second;
+      Register LiveReg = Insert.second;
       MachineBasicBlock::iterator MI = IPA.getLastInsertPointIter(OrigLI, *BB);
       TII.storeRegToStackSlot(*BB, MI, LiveReg, false, Slot,
                               MRI.getRegClass(LiveReg), &TRI);
diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
index d9ea1cac9574d..c4d83547a06c6 100644
--- a/llvm/lib/CodeGen/InterleavedAccessPass.cpp
+++ b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
@@ -280,7 +280,7 @@ static bool isReInterleaveMask(ArrayRef<int> Mask, unsigned &Factor,
 
 bool InterleavedAccess::lowerInterleavedLoad(
     LoadInst *LI, SmallVector<Instruction *, 32> &DeadInsts) {
-  if (!LI->isSimple())
+  if (!LI->isSimple() || isa<ScalableVectorType>(LI->getType()))
     return false;
 
   SmallVector<ShuffleVectorInst *, 4> Shuffles;
@@ -308,7 +308,8 @@ bool InterleavedAccess::lowerInterleavedLoad(
 
   unsigned Factor, Index;
 
-  unsigned NumLoadElements = cast<VectorType>(LI->getType())->getNumElements();
+  unsigned NumLoadElements =
+      cast<FixedVectorType>(LI->getType())->getNumElements();
   // Check if the first shufflevector is DE-interleave shuffle.
   if (!isDeInterleaveMask(Shuffles[0]->getShuffleMask(), Factor, Index,
                           MaxFactor, NumLoadElements))
@@ -421,13 +422,13 @@ bool InterleavedAccess::lowerInterleavedStore(
     return false;
 
   ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(SI->getValueOperand());
-  if (!SVI || !SVI->hasOneUse())
+  if (!SVI || !SVI->hasOneUse() || isa<ScalableVectorType>(SVI->getType()))
     return false;
 
   // Check if the shufflevector is RE-interleave shuffle.
   unsigned Factor;
   unsigned OpNumElts =
-      cast<VectorType>(SVI->getOperand(0)->getType())->getNumElements();
+      cast<FixedVectorType>(SVI->getOperand(0)->getType())->getNumElements();
   if (!isReInterleaveMask(SVI->getShuffleMask(), Factor, MaxFactor, OpNumElts))
     return false;
 
diff --git a/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp b/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
index a0ed5eea06515..f7131926ee658 100644
--- a/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
+++ b/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
@@ -1200,14 +1200,15 @@ bool InterleavedLoadCombineImpl::combine(std::list<VectorInfo> &InterleavedLoad,
   IRBuilder<> Builder(InsertionPoint);
   Type *ETy = InterleavedLoad.front().SVI->getType()->getElementType();
   unsigned ElementsPerSVI =
-      InterleavedLoad.front().SVI->getType()->getNumElements();
+      cast<FixedVectorType>(InterleavedLoad.front().SVI->getType())
+          ->getNumElements();
   FixedVectorType *ILTy = FixedVectorType::get(ETy, Factor * ElementsPerSVI);
 
   SmallVector<unsigned, 4> Indices;
   for (unsigned i = 0; i < Factor; i++)
     Indices.push_back(i);
   InterleavedCost = TTI.getInterleavedMemoryOpCost(
-      Instruction::Load, ILTy, Factor, Indices, InsertionPoint->getAlignment(),
+      Instruction::Load, ILTy, Factor, Indices, InsertionPoint->getAlign(),
       InsertionPoint->getPointerAddressSpace());
 
   if (InterleavedCost >= InstructionCost) {
diff --git a/llvm/lib/CodeGen/LiveDebugValues.cpp b/llvm/lib/CodeGen/LiveDebugValues.cpp
index c4b551e1ed5c5..07a275b546f62 100644
--- a/llvm/lib/CodeGen/LiveDebugValues.cpp
+++ b/llvm/lib/CodeGen/LiveDebugValues.cpp
@@ -174,7 +174,8 @@ static Register isDbgValueDescribedByReg(const MachineInstr &MI) {
   assert(MI.getNumOperands() == 4 && "malformed DBG_VALUE");
   // If location of variable is described using a register (directly
   // or indirectly), this register is always a first operand.
-  return MI.getOperand(0).isReg() ? MI.getOperand(0).getReg() : Register();
+  return MI.getDebugOperand(0).isReg() ? MI.getDebugOperand(0).getReg()
+                                       : Register();
 }
 
 /// If \p Op is a stack or frame register return true, otherwise return false.
@@ -188,7 +189,7 @@ static bool isRegOtherThanSPAndFP(const MachineOperand &Op,
 
   const MachineFunction *MF = MI.getParent()->getParent();
   const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
-  unsigned SP = TLI->getStackPointerRegisterToSaveRestore();
+  Register SP = TLI->getStackPointerRegisterToSaveRestore();
   Register FP = TRI->getFrameRegister(*MF);
   Register Reg = Op.getReg();
 
@@ -334,15 +335,15 @@ class LiveDebugValues : public MachineFunctionPass {
       if (int RegNo = isDbgValueDescribedByReg(MI)) {
         Kind = RegisterKind;
         Loc.RegNo = RegNo;
-      } else if (MI.getOperand(0).isImm()) {
+      } else if (MI.getDebugOperand(0).isImm()) {
         Kind = ImmediateKind;
-        Loc.Immediate = MI.getOperand(0).getImm();
-      } else if (MI.getOperand(0).isFPImm()) {
+        Loc.Immediate = MI.getDebugOperand(0).getImm();
+      } else if (MI.getDebugOperand(0).isFPImm()) {
         Kind = ImmediateKind;
-        Loc.FPImm = MI.getOperand(0).getFPImm();
-      } else if (MI.getOperand(0).isCImm()) {
+        Loc.FPImm = MI.getDebugOperand(0).getFPImm();
+      } else if (MI.getDebugOperand(0).isCImm()) {
         Kind = ImmediateKind;
-        Loc.CImm = MI.getOperand(0).getCImm();
+        Loc.CImm = MI.getDebugOperand(0).getCImm();
       }
 
       // We create the debug entry values from the factory functions rather than
@@ -353,7 +354,7 @@ class LiveDebugValues : public MachineFunctionPass {
     /// Take the variable and machine-location in DBG_VALUE MI, and build an
     /// entry location using the given expression.
     static VarLoc CreateEntryLoc(const MachineInstr &MI, LexicalScopes &LS,
-                                 const DIExpression *EntryExpr, unsigned Reg) {
+                                 const DIExpression *EntryExpr, Register Reg) {
       VarLoc VL(MI, LS);
       assert(VL.Kind == RegisterKind);
       VL.Kind = EntryValueKind;
@@ -382,7 +383,7 @@ class LiveDebugValues : public MachineFunctionPass {
     static VarLoc CreateEntryCopyBackupLoc(const MachineInstr &MI,
                                            LexicalScopes &LS,
                                            const DIExpression *EntryExpr,
-                                           unsigned NewReg) {
+                                           Register NewReg) {
       VarLoc VL(MI, LS);
       assert(VL.Kind == RegisterKind);
       VL.Kind = EntryValueCopyBackupKind;
@@ -394,7 +395,7 @@ class LiveDebugValues : public MachineFunctionPass {
     /// Copy the register location in DBG_VALUE MI, updating the register to
     /// be NewReg.
     static VarLoc CreateCopyLoc(const MachineInstr &MI, LexicalScopes &LS,
-                                unsigned NewReg) {
+                                Register NewReg) {
       VarLoc VL(MI, LS);
       assert(VL.Kind == RegisterKind);
       VL.Loc.RegNo = NewReg;
@@ -430,8 +431,8 @@ class LiveDebugValues : public MachineFunctionPass {
         // expression. The register location of such DBG_VALUE is always the one
         // from the entry DBG_VALUE, it does not matter if the entry value was
         // copied in to another register due to some optimizations.
-        return BuildMI(MF, DbgLoc, IID, Indirect, MI.getOperand(0).getReg(),
-                       Var, Expr);
+        return BuildMI(MF, DbgLoc, IID, Indirect,
+                       MI.getDebugOperand(0).getReg(), Var, Expr);
       case RegisterKind:
         // Register locations are like the source DBG_VALUE, but with the
         // register number from this VarLoc.
@@ -447,7 +448,7 @@ class LiveDebugValues : public MachineFunctionPass {
         return BuildMI(MF, DbgLoc, IID, true, Base, Var, SpillExpr);
       }
       case ImmediateKind: {
-        MachineOperand MO = MI.getOperand(0);
+        MachineOperand MO = MI.getDebugOperand(0);
         return BuildMI(MF, DbgLoc, IID, Indirect, MO, Var, DIExpr);
       }
       case EntryValueBackupKind:
@@ -735,7 +736,7 @@ class LiveDebugValues : public MachineFunctionPass {
   /// TODO: Store optimization can fold spills into other stores (including
   /// other spills). We do not handle this yet (more than one memory operand).
   bool isLocationSpill(const MachineInstr &MI, MachineFunction *MF,
-                       unsigned &Reg);
+                       Register &Reg);
 
   /// Returns true if the given machine instruction is a debug value which we
   /// can emit entry values for.
@@ -749,14 +750,14 @@ class LiveDebugValues : public MachineFunctionPass {
   /// and set \p Reg to the spilled register.
   Optional<VarLoc::SpillLoc> isRestoreInstruction(const MachineInstr &MI,
                                                   MachineFunction *MF,
-                                                  unsigned &Reg);
+                                                  Register &Reg);
   /// Given a spill instruction, extract the register and offset used to
   /// address the spill location in a target independent way.
   VarLoc::SpillLoc extractSpillBaseRegAndOffset(const MachineInstr &MI);
   void insertTransferDebugPair(MachineInstr &MI, OpenRangesSet &OpenRanges,
                                TransferMap &Transfers, VarLocMap &VarLocIDs,
                                LocIndex OldVarID, TransferKind Kind,
-                               unsigned NewReg = 0);
+                               Register NewReg = Register());
 
   void transferDebugValue(const MachineInstr &MI, OpenRangesSet &OpenRanges,
                           VarLocMap &VarLocIDs);
@@ -1020,7 +1021,7 @@ bool LiveDebugValues::removeEntryValue(const MachineInstr &MI,
   // the entry value any more. In addition, if the debug expression from the
   // DBG_VALUE is not empty, we can assume the parameter's value has changed
   // indicating that we should stop tracking its entry value as well.
-  if (!MI.getOperand(0).isReg() ||
+  if (!MI.getDebugOperand(0).isReg() ||
       MI.getDebugExpression()->getNumElements() != 0)
     return true;
 
@@ -1028,7 +1029,7 @@ bool LiveDebugValues::removeEntryValue(const MachineInstr &MI,
   // it means the parameter's value has not changed and we should be able to use
   // its entry value.
   bool TrySalvageEntryValue = false;
-  Register Reg = MI.getOperand(0).getReg();
+  Register Reg = MI.getDebugOperand(0).getReg();
   auto I = std::next(MI.getReverseIterator());
   const MachineOperand *SrcRegOp, *DestRegOp;
   if (I != MI.getParent()->rend()) {
@@ -1050,7 +1051,7 @@ bool LiveDebugValues::removeEntryValue(const MachineInstr &MI,
     for (uint64_t ID : OpenRanges.getEntryValueBackupVarLocs()) {
       const VarLoc &VL = VarLocIDs[LocIndex::fromRawInteger(ID)];
       if (VL.getEntryValueCopyBackupReg() == Reg &&
-          VL.MI.getOperand(0).getReg() == SrcRegOp->getReg())
+          VL.MI.getDebugOperand(0).getReg() == SrcRegOp->getReg())
         return false;
     }
   }
@@ -1088,8 +1089,8 @@ void LiveDebugValues::transferDebugValue(const MachineInstr &MI,
     }
   }
 
-  if (isDbgValueDescribedByReg(MI) || MI.getOperand(0).isImm() ||
-      MI.getOperand(0).isFPImm() || MI.getOperand(0).isCImm()) {
+  if (isDbgValueDescribedByReg(MI) || MI.getDebugOperand(0).isImm() ||
+      MI.getDebugOperand(0).isFPImm() || MI.getDebugOperand(0).isCImm()) {
     // Use normal VarLoc constructor for registers and immediates.
     VarLoc VL(MI, LS);
     // End all previous ranges of VL.Var.
@@ -1102,7 +1103,8 @@ void LiveDebugValues::transferDebugValue(const MachineInstr &MI,
     llvm_unreachable("DBG_VALUE with mem operand encountered after regalloc?");
   } else {
     // This must be an undefined location. If it has an open range, erase it.
-    assert(MI.getOperand(0).isReg() && MI.getOperand(0).getReg() == 0 &&
+    assert(MI.getDebugOperand(0).isReg() &&
+           MI.getDebugOperand(0).getReg() == 0 &&
            "Unexpected non-undef DBG_VALUE encountered");
     VarLoc VL(MI, LS);
     OpenRanges.erase(VL);
@@ -1151,7 +1153,7 @@ void LiveDebugValues::emitEntryValues(MachineInstr &MI,
 void LiveDebugValues::insertTransferDebugPair(
     MachineInstr &MI, OpenRangesSet &OpenRanges, TransferMap &Transfers,
     VarLocMap &VarLocIDs, LocIndex OldVarID, TransferKind Kind,
-    unsigned NewReg) {
+    Register NewReg) {
   const MachineInstr *DebugInstr = &VarLocIDs[OldVarID].MI;
 
   auto ProcessVarLoc = [&MI, &OpenRanges, &Transfers, &VarLocIDs](VarLoc &VL) {
@@ -1226,7 +1228,7 @@ void LiveDebugValues::transferRegisterDef(
 
   MachineFunction *MF = MI.getMF();
   const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
-  unsigned SP = TLI->getStackPointerRegisterToSaveRestore();
+  Register SP = TLI->getStackPointerRegisterToSaveRestore();
 
   // Find the regs killed by MI, and find regmasks of preserved regs.
   DefinedRegsSet DeadRegs;
@@ -1297,11 +1299,11 @@ bool LiveDebugValues::isSpillInstruction(const MachineInstr &MI,
 }
 
 bool LiveDebugValues::isLocationSpill(const MachineInstr &MI,
-                                      MachineFunction *MF, unsigned &Reg) {
+                                      MachineFunction *MF, Register &Reg) {
   if (!isSpillInstruction(MI, MF))
     return false;
 
-  auto isKilledReg = [&](const MachineOperand MO, unsigned &Reg) {
+  auto isKilledReg = [&](const MachineOperand MO, Register &Reg) {
     if (!MO.isReg() || !MO.isUse()) {
       Reg = 0;
       return false;
@@ -1323,7 +1325,7 @@ bool LiveDebugValues::isLocationSpill(const MachineInstr &MI,
       // Skip next instruction that points to basic block end iterator.
       if (MI.getParent()->end() == NextI)
         continue;
-      unsigned RegNext;
+      Register RegNext;
       for (const MachineOperand &MONext : NextI->operands()) {
         // Return true if we came across the register from the
         // previous spill instruction that is killed in NextI.
@@ -1338,7 +1340,7 @@ bool LiveDebugValues::isLocationSpill(const MachineInstr &MI,
 
 Optional<LiveDebugValues::VarLoc::SpillLoc>
 LiveDebugValues::isRestoreInstruction(const MachineInstr &MI,
-                                      MachineFunction *MF, unsigned &Reg) {
+                                      MachineFunction *MF, Register &Reg) {
   if (!MI.hasOneMemOperand())
     return None;
 
@@ -1364,7 +1366,7 @@ void LiveDebugValues::transferSpillOrRestoreInst(MachineInstr &MI,
                                                  TransferMap &Transfers) {
   MachineFunction *MF = MI.getMF();
   TransferKind TKind;
-  unsigned Reg;
+  Register Reg;
   Optional<VarLoc::SpillLoc> Loc;
 
   LLVM_DEBUG(dbgs() << "Examining instruction: "; MI.dump(););
@@ -1461,7 +1463,7 @@ void LiveDebugValues::transferRegisterCopy(MachineInstr &MI,
   if (!DestRegOp->isDef())
     return;
 
-  auto isCalleeSavedReg = [&](unsigned Reg) {
+  auto isCalleeSavedReg = [&](Register Reg) {
     for (MCRegAliasIterator RAI(Reg, TRI, true); RAI.isValid(); ++RAI)
       if (CalleeSavedRegs.test(*RAI))
         return true;
@@ -1738,14 +1740,14 @@ bool LiveDebugValues::isEntryValueCandidate(
   // Only consider parameters that are described using registers. Parameters
   // that are passed on the stack are not yet supported, so ignore debug
   // values that are described by the frame or stack pointer.
-  if (!isRegOtherThanSPAndFP(MI.getOperand(0), MI, TRI))
+  if (!isRegOtherThanSPAndFP(MI.getDebugOperand(0), MI, TRI))
     return false;
 
   // If a parameter's value has been propagated from the caller, then the
   // parameter's DBG_VALUE may be described using a register defined by some
   // instruction in the entry block, in which case we shouldn't create an
   // entry value.
-  if (DefinedRegs.count(MI.getOperand(0).getReg()))
+  if (DefinedRegs.count(MI.getDebugOperand(0).getReg()))
     return false;
 
   // TODO: Add support for parameters that have a pre-existing debug expressions
diff --git a/llvm/lib/CodeGen/LiveDebugVariables.cpp b/llvm/lib/CodeGen/LiveDebugVariables.cpp
index a686d12c45a9b..158e873370b19 100644
--- a/llvm/lib/CodeGen/LiveDebugVariables.cpp
+++ b/llvm/lib/CodeGen/LiveDebugVariables.cpp
@@ -184,7 +184,7 @@ class UserValue {
 
   /// Replace OldLocNo ranges with NewRegs ranges where NewRegs
   /// is live. Returns true if any changes were made.
-  bool splitLocation(unsigned OldLocNo, ArrayRef<unsigned> NewRegs,
+  bool splitLocation(unsigned OldLocNo, ArrayRef<Register> NewRegs,
                      LiveIntervals &LIS);
 
 public:
@@ -332,7 +332,7 @@ class UserValue {
 
   /// Replace OldReg ranges with NewRegs ranges where NewRegs is
   /// live. Returns true if any changes were made.
-  bool splitRegister(unsigned OldReg, ArrayRef<unsigned> NewRegs,
+  bool splitRegister(Register OldReg, ArrayRef<Register> NewRegs,
                      LiveIntervals &LIS);
 
   /// Rewrite virtual register locations according to the provided virtual
@@ -420,7 +420,7 @@ class LDVImpl {
                           const DebugLoc &DL);
 
   /// Find the EC leader for VirtReg or null.
-  UserValue *lookupVirtReg(unsigned VirtReg);
+  UserValue *lookupVirtReg(Register VirtReg);
 
   /// Add DBG_VALUE instruction to our maps.
   ///
@@ -470,10 +470,10 @@ class LDVImpl {
   }
 
   /// Map virtual register to an equivalence class.
-  void mapVirtReg(unsigned VirtReg, UserValue *EC);
+  void mapVirtReg(Register VirtReg, UserValue *EC);
 
   /// Replace all references to OldReg with NewRegs.
-  void splitRegister(unsigned OldReg, ArrayRef<unsigned> NewRegs);
+  void splitRegister(Register OldReg, ArrayRef<Register> NewRegs);
 
   /// Recreate DBG_VALUE instruction from data structures.
   void emitDebugValues(VirtRegMap *VRM);
@@ -593,13 +593,13 @@ UserValue *LDVImpl::getUserValue(const DILocalVariable *Var,
   return UV;
 }
 
-void LDVImpl::mapVirtReg(unsigned VirtReg, UserValue *EC) {
+void LDVImpl::mapVirtReg(Register VirtReg, UserValue *EC) {
   assert(Register::isVirtualRegister(VirtReg) && "Only map VirtRegs");
   UserValue *&Leader = virtRegToEqClass[VirtReg];
   Leader = UserValue::merge(Leader, EC);
 }
 
-UserValue *LDVImpl::lookupVirtReg(unsigned VirtReg) {
+UserValue *LDVImpl::lookupVirtReg(Register VirtReg) {
   if (UserValue *UV = virtRegToEqClass.lookup(VirtReg))
     return UV->getLeader();
   return nullptr;
@@ -608,8 +608,8 @@ UserValue *LDVImpl::lookupVirtReg(unsigned VirtReg) {
 bool LDVImpl::handleDebugValue(MachineInstr &MI, SlotIndex Idx) {
   // DBG_VALUE loc, offset, variable
   if (MI.getNumOperands() != 4 ||
-      !(MI.getOperand(1).isReg() || MI.getOperand(1).isImm()) ||
-      !MI.getOperand(2).isMetadata()) {
+      !(MI.getDebugOffset().isReg() || MI.getDebugOffset().isImm()) ||
+      !MI.getDebugVariableOp().isMetadata()) {
     LLVM_DEBUG(dbgs() << "Can't handle " << MI);
     return false;
   }
@@ -622,9 +622,9 @@ bool LDVImpl::handleDebugValue(MachineInstr &MI, SlotIndex Idx) {
   // (and if the machine verifier is improved to catch this), then these checks
   // could be removed or replaced by asserts.
   bool Discard = false;
-  if (MI.getOperand(0).isReg() &&
-      Register::isVirtualRegister(MI.getOperand(0).getReg())) {
-    const Register Reg = MI.getOperand(0).getReg();
+  if (MI.getDebugOperand(0).isReg() &&
+      Register::isVirtualRegister(MI.getDebugOperand(0).getReg())) {
+    const Register Reg = MI.getDebugOperand(0).getReg();
     if (!LIS->hasInterval(Reg)) {
       // The DBG_VALUE is described by a virtual register that does not have a
       // live interval. Discard the DBG_VALUE.
@@ -648,14 +648,15 @@ bool LDVImpl::handleDebugValue(MachineInstr &MI, SlotIndex Idx) {
   }
 
   // Get or create the UserValue for (variable,offset) here.
-  bool IsIndirect = MI.getOperand(1).isImm();
+  bool IsIndirect = MI.isDebugOffsetImm();
   if (IsIndirect)
-    assert(MI.getOperand(1).getImm() == 0 && "DBG_VALUE with nonzero offset");
+    assert(MI.getDebugOffset().getImm() == 0 &&
+           "DBG_VALUE with nonzero offset");
   const DILocalVariable *Var = MI.getDebugVariable();
   const DIExpression *Expr = MI.getDebugExpression();
   UserValue *UV = getUserValue(Var, Expr->getFragmentInfo(), MI.getDebugLoc());
   if (!Discard)
-    UV->addDef(Idx, MI.getOperand(0), IsIndirect, *Expr);
+    UV->addDef(Idx, MI.getDebugOperand(0), IsIndirect, *Expr);
   else {
     MachineOperand MO = MachineOperand::CreateReg(0U, false);
     MO.setIsDebug();
@@ -1029,7 +1030,7 @@ LiveDebugVariables::~LiveDebugVariables() {
 //===----------------------------------------------------------------------===//
 
 bool
-UserValue::splitLocation(unsigned OldLocNo, ArrayRef<unsigned> NewRegs,
+UserValue::splitLocation(unsigned OldLocNo, ArrayRef<Register> NewRegs,
                          LiveIntervals& LIS) {
   LLVM_DEBUG({
     dbgs() << "Splitting Loc" << OldLocNo << '\t';
@@ -1129,7 +1130,7 @@ UserValue::splitLocation(unsigned OldLocNo, ArrayRef<unsigned> NewRegs,
 }
 
 bool
-UserValue::splitRegister(unsigned OldReg, ArrayRef<unsigned> NewRegs,
+UserValue::splitRegister(Register OldReg, ArrayRef<Register> NewRegs,
                          LiveIntervals &LIS) {
   bool DidChange = false;
   // Split locations referring to OldReg. Iterate backwards so splitLocation can
@@ -1144,7 +1145,7 @@ UserValue::splitRegister(unsigned OldReg, ArrayRef<unsigned> NewRegs,
   return DidChange;
 }
 
-void LDVImpl::splitRegister(unsigned OldReg, ArrayRef<unsigned> NewRegs) {
+void LDVImpl::splitRegister(Register OldReg, ArrayRef<Register> NewRegs) {
   bool DidChange = false;
   for (UserValue *UV = lookupVirtReg(OldReg); UV; UV = UV->getNext())
     DidChange |= UV->splitRegister(OldReg, NewRegs, *LIS);
@@ -1159,7 +1160,7 @@ void LDVImpl::splitRegister(unsigned OldReg, ArrayRef<unsigned> NewRegs) {
 }
 
 void LiveDebugVariables::
-splitRegister(unsigned OldReg, ArrayRef<unsigned> NewRegs, LiveIntervals &LIS) {
+splitRegister(Register OldReg, ArrayRef<Register> NewRegs, LiveIntervals &LIS) {
   if (pImpl)
     static_cast<LDVImpl*>(pImpl)->splitRegister(OldReg, NewRegs);
 }
diff --git a/llvm/lib/CodeGen/LiveDebugVariables.h b/llvm/lib/CodeGen/LiveDebugVariables.h
index 0cbe10c6a4227..74e738ec3e568 100644
--- a/llvm/lib/CodeGen/LiveDebugVariables.h
+++ b/llvm/lib/CodeGen/LiveDebugVariables.h
@@ -41,7 +41,7 @@ class LLVM_LIBRARY_VISIBILITY LiveDebugVariables : public MachineFunctionPass {
   /// splitRegister - Move any user variables in OldReg to the live ranges in
   /// NewRegs where they are live. Mark the values as unavailable where no new
   /// register is live.
-  void splitRegister(unsigned OldReg, ArrayRef<unsigned> NewRegs,
+  void splitRegister(Register OldReg, ArrayRef<Register> NewRegs,
                      LiveIntervals &LIS);
 
   /// emitDebugValues - Emit new DBG_VALUE instructions reflecting the changes
diff --git a/llvm/lib/CodeGen/LiveIntervals.cpp b/llvm/lib/CodeGen/LiveIntervals.cpp
index 2bbe036e8425a..e8ee0599e1a2c 100644
--- a/llvm/lib/CodeGen/LiveIntervals.cpp
+++ b/llvm/lib/CodeGen/LiveIntervals.cpp
@@ -1011,6 +1011,20 @@ class LiveIntervals::HMEditor {
           }
         }
         updateRange(LI, Reg, LaneBitmask::getNone());
+        // If main range has a hole and we are moving a subrange use across
+        // the hole updateRange() cannot properly handle it since it only
+        // gets the LiveRange and not the whole LiveInterval. As a result
+        // we may end up with a main range not covering all subranges.
+        // This is extremely rare case, so let's check and reconstruct the
+        // main range.
+        for (LiveInterval::SubRange &S : LI.subranges()) {
+          if (LI.covers(S))
+            continue;
+          LI.clear();
+          LIS.constructMainRangeFromSubranges(LI);
+          break;
+        }
+
         continue;
       }
 
diff --git a/llvm/lib/CodeGen/LiveRangeEdit.cpp b/llvm/lib/CodeGen/LiveRangeEdit.cpp
index fa2285a1348a0..9de77c19a23a2 100644
--- a/llvm/lib/CodeGen/LiveRangeEdit.cpp
+++ b/llvm/lib/CodeGen/LiveRangeEdit.cpp
@@ -12,6 +12,7 @@
 
 #include "llvm/CodeGen/LiveRangeEdit.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/CalcSpillWeights.h"
 #include "llvm/CodeGen/LiveIntervals.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -30,7 +31,7 @@ STATISTIC(NumFracRanges,     "Number of live ranges fractured by DCE");
 
 void LiveRangeEdit::Delegate::anchor() { }
 
-LiveInterval &LiveRangeEdit::createEmptyIntervalFrom(unsigned OldReg,
+LiveInterval &LiveRangeEdit::createEmptyIntervalFrom(Register OldReg,
                                                      bool createSubRanges) {
   Register VReg = MRI.createVirtualRegister(MRI.getRegClass(OldReg));
   if (VRM)
@@ -51,7 +52,7 @@ LiveInterval &LiveRangeEdit::createEmptyIntervalFrom(unsigned OldReg,
   return LI;
 }
 
-unsigned LiveRangeEdit::createFrom(unsigned OldReg) {
+Register LiveRangeEdit::createFrom(Register OldReg) {
   Register VReg = MRI.createVirtualRegister(MRI.getRegClass(OldReg));
   if (VRM) {
     VRM->setIsSplitFromReg(VReg, VRM->getOriginal(OldReg));
@@ -69,7 +70,7 @@ unsigned LiveRangeEdit::createFrom(unsigned OldReg) {
 
 bool LiveRangeEdit::checkRematerializable(VNInfo *VNI,
                                           const MachineInstr *DefMI,
-                                          AliasAnalysis *aa) {
+                                          AAResults *aa) {
   assert(DefMI && "Missing instruction");
   ScannedRemattable = true;
   if (!TII.isTriviallyReMaterializable(*DefMI, aa))
@@ -78,7 +79,7 @@ bool LiveRangeEdit::checkRematerializable(VNInfo *VNI,
   return true;
 }
 
-void LiveRangeEdit::scanRemattable(AliasAnalysis *aa) {
+void LiveRangeEdit::scanRemattable(AAResults *aa) {
   for (VNInfo *VNI : getParent().valnos) {
     if (VNI->isUnused())
       continue;
@@ -95,7 +96,7 @@ void LiveRangeEdit::scanRemattable(AliasAnalysis *aa) {
   ScannedRemattable = true;
 }
 
-bool LiveRangeEdit::anyRematerializable(AliasAnalysis *aa) {
+bool LiveRangeEdit::anyRematerializable(AAResults *aa) {
   if (!ScannedRemattable)
     scanRemattable(aa);
   return !Remattable.empty();
@@ -177,7 +178,7 @@ SlotIndex LiveRangeEdit::rematerializeAt(MachineBasicBlock &MBB,
   return LIS.getSlotIndexes()->insertMachineInstrInMaps(*MI, Late).getRegSlot();
 }
 
-void LiveRangeEdit::eraseVirtReg(unsigned Reg) {
+void LiveRangeEdit::eraseVirtReg(Register Reg) {
   if (TheDelegate && TheDelegate->LRE_CanEraseVirtReg(Reg))
     LIS.removeInterval(Reg);
 }
@@ -259,7 +260,7 @@ bool LiveRangeEdit::useIsKill(const LiveInterval &LI,
 
 /// Find all live intervals that need to shrink, then remove the instruction.
 void LiveRangeEdit::eliminateDeadDef(MachineInstr *MI, ToShrinkSet &ToShrink,
-                                     AliasAnalysis *AA) {
+                                     AAResults *AA) {
   assert(MI->allDefsAreDead() && "Def isn't really dead");
   SlotIndex Idx = LIS.getInstructionIndex(*MI).getRegSlot();
 
@@ -382,7 +383,7 @@ void LiveRangeEdit::eliminateDeadDef(MachineInstr *MI, ToShrinkSet &ToShrink,
   // Erase any virtregs that are now empty and unused. There may be <undef>
   // uses around. Keep the empty live range in that case.
   for (unsigned i = 0, e = RegsToErase.size(); i != e; ++i) {
-    unsigned Reg = RegsToErase[i];
+    Register Reg = RegsToErase[i];
     if (LIS.hasInterval(Reg) && MRI.reg_nodbg_empty(Reg)) {
       ToShrink.remove(&LIS.getInterval(Reg));
       eraseVirtReg(Reg);
@@ -391,8 +392,8 @@ void LiveRangeEdit::eliminateDeadDef(MachineInstr *MI, ToShrinkSet &ToShrink,
 }
 
 void LiveRangeEdit::eliminateDeadDefs(SmallVectorImpl<MachineInstr *> &Dead,
-                                      ArrayRef<unsigned> RegsBeingSpilled,
-                                      AliasAnalysis *AA) {
+                                      ArrayRef<Register> RegsBeingSpilled,
+                                      AAResults *AA) {
   ToShrinkSet ToShrink;
 
   for (;;) {
diff --git a/llvm/lib/CodeGen/LiveRangeShrink.cpp b/llvm/lib/CodeGen/LiveRangeShrink.cpp
index 2ebc8d7576d15..26439a6569179 100644
--- a/llvm/lib/CodeGen/LiveRangeShrink.cpp
+++ b/llvm/lib/CodeGen/LiveRangeShrink.cpp
@@ -234,8 +234,7 @@ bool LiveRangeShrink::runOnMachineFunction(MachineFunction &MF) {
         MachineBasicBlock::iterator EndIter = std::next(MI.getIterator());
         if (MI.getOperand(0).isReg())
           for (; EndIter != MBB.end() && EndIter->isDebugValue() &&
-                 EndIter->getOperand(0).isReg() &&
-                 EndIter->getOperand(0).getReg() == MI.getOperand(0).getReg();
+                 EndIter->getDebugOperandForReg(MI.getOperand(0).getReg());
                ++EndIter, ++Next)
             IOM[&*EndIter] = NewOrder;
         MBB.splice(I, &MBB, MI.getIterator(), EndIter);
diff --git a/llvm/lib/CodeGen/LowLevelType.cpp b/llvm/lib/CodeGen/LowLevelType.cpp
index 40dfa696a2b9b..33752a1f9230f 100644
--- a/llvm/lib/CodeGen/LowLevelType.cpp
+++ b/llvm/lib/CodeGen/LowLevelType.cpp
@@ -19,7 +19,7 @@ using namespace llvm;
 
 LLT llvm::getLLTForType(Type &Ty, const DataLayout &DL) {
   if (auto VTy = dyn_cast<VectorType>(&Ty)) {
-    auto NumElements = VTy->getNumElements();
+    auto NumElements = cast<FixedVectorType>(VTy)->getNumElements();
     LLT ScalarTy = getLLTForType(*VTy->getElementType(), DL);
     if (NumElements == 1)
       return ScalarTy;
diff --git a/llvm/lib/CodeGen/LowerEmuTLS.cpp b/llvm/lib/CodeGen/LowerEmuTLS.cpp
index 1b41f773583a6..36b863178b474 100644
--- a/llvm/lib/CodeGen/LowerEmuTLS.cpp
+++ b/llvm/lib/CodeGen/LowerEmuTLS.cpp
@@ -128,12 +128,7 @@ bool LowerEmuTLS::addEmuTlsVar(Module &M, const GlobalVariable *GV) {
     return true;
 
   Type *GVType = GV->getValueType();
-  unsigned GVAlignment = GV->getAlignment();
-  if (!GVAlignment) {
-    // When LLVM IL declares a variable without alignment, use
-    // the ABI default alignment for the type.
-    GVAlignment = DL.getABITypeAlignment(GVType);
-  }
+  Align GVAlignment = DL.getValueOrABITypeAlignment(GV->getAlign(), GVType);
 
   // Define "__emutls_t.*" if there is InitValue
   GlobalVariable *EmuTlsTmplVar = nullptr;
@@ -144,21 +139,20 @@ bool LowerEmuTLS::addEmuTlsVar(Module &M, const GlobalVariable *GV) {
     assert(EmuTlsTmplVar && "Failed to create emualted TLS initializer");
     EmuTlsTmplVar->setConstant(true);
     EmuTlsTmplVar->setInitializer(const_cast<Constant*>(InitValue));
-    EmuTlsTmplVar->setAlignment(Align(GVAlignment));
+    EmuTlsTmplVar->setAlignment(GVAlignment);
     copyLinkageVisibility(M, GV, EmuTlsTmplVar);
   }
 
   // Define "__emutls_v.*" with initializer and alignment.
   Constant *ElementValues[4] = {
       ConstantInt::get(WordType, DL.getTypeStoreSize(GVType)),
-      ConstantInt::get(WordType, GVAlignment),
-      NullPtr, EmuTlsTmplVar ? EmuTlsTmplVar : NullPtr
-  };
+      ConstantInt::get(WordType, GVAlignment.value()), NullPtr,
+      EmuTlsTmplVar ? EmuTlsTmplVar : NullPtr};
   ArrayRef<Constant*> ElementValueArray(ElementValues, 4);
   EmuTlsVar->setInitializer(
       ConstantStruct::get(EmuTlsVarType, ElementValueArray));
-  Align MaxAlignment(std::max(DL.getABITypeAlignment(WordType),
-                              DL.getABITypeAlignment(VoidPtrType)));
+  Align MaxAlignment =
+      std::max(DL.getABITypeAlign(WordType), DL.getABITypeAlign(VoidPtrType));
   EmuTlsVar->setAlignment(MaxAlignment);
   return true;
 }
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index d6bf273433a2c..ded31cd08fb5c 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -563,7 +563,7 @@ MIParser::MIParser(PerFunctionMIParsingState &PFS, SMDiagnostic &Error,
 
 void MIParser::lex(unsigned SkipChar) {
   CurrentSource = lexMIToken(
-      CurrentSource.data() + SkipChar, Token,
+      CurrentSource.slice(SkipChar, StringRef::npos), Token,
       [this](StringRef::iterator Loc, const Twine &Msg) { error(Loc, Msg); });
 }
 
diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp
index e8487412264f6..2d4b60435d962 100644
--- a/llvm/lib/CodeGen/MachineBasicBlock.cpp
+++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp
@@ -277,8 +277,16 @@ LLVM_DUMP_METHOD void MachineBasicBlock::dump() const {
 }
 #endif
 
+bool MachineBasicBlock::mayHaveInlineAsmBr() const {
+  for (const MachineBasicBlock *Succ : successors()) {
+    if (Succ->isInlineAsmBrIndirectTarget())
+      return true;
+  }
+  return false;
+}
+
 bool MachineBasicBlock::isLegalToHoistInto() const {
-  if (isReturnBlock() || hasEHPadSuccessor())
+  if (isReturnBlock() || hasEHPadSuccessor() || mayHaveInlineAsmBr())
     return false;
   return true;
 }
@@ -1132,7 +1140,7 @@ bool MachineBasicBlock::canSplitCriticalEdge(
 
   // Splitting the critical edge to a callbr's indirect block isn't advised.
   // Don't do it in this generic function.
-  if (isInlineAsmBrIndirectTarget(Succ))
+  if (Succ->isInlineAsmBrIndirectTarget())
     return false;
 
   const MachineFunction *MF = getParent();
diff --git a/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
index 229e23de1e534..783d22fafee9d 100644
--- a/llvm/lib/CodeGen/MachineBlockPlacement.cpp
+++ b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
@@ -3329,9 +3329,7 @@ bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &MF) {
     BranchFolder BF(/*EnableTailMerge=*/true, /*CommonHoist=*/false, *MBFI,
                     *MBPI, PSI, TailMergeSize);
 
-    auto *MMIWP = getAnalysisIfAvailable<MachineModuleInfoWrapperPass>();
-    if (BF.OptimizeFunction(MF, TII, MF.getSubtarget().getRegisterInfo(),
-                            MMIWP ? &MMIWP->getMMI() : nullptr, MLI,
+    if (BF.OptimizeFunction(MF, TII, MF.getSubtarget().getRegisterInfo(), MLI,
                             /*AfterPlacement=*/true)) {
       // Redo the layout if tail merging creates/removes/moves blocks.
       BlockToChain.clear();
diff --git a/llvm/lib/CodeGen/MachineCSE.cpp b/llvm/lib/CodeGen/MachineCSE.cpp
index 8c195adb444d5..09531276bc10d 100644
--- a/llvm/lib/CodeGen/MachineCSE.cpp
+++ b/llvm/lib/CodeGen/MachineCSE.cpp
@@ -747,9 +747,8 @@ bool MachineCSE::PerformCSE(MachineDomTreeNode *Node) {
   do {
     Node = WorkList.pop_back_val();
     Scopes.push_back(Node);
-    const std::vector<MachineDomTreeNode*> &Children = Node->getChildren();
-    OpenChildren[Node] = Children.size();
-    for (MachineDomTreeNode *Child : Children)
+    OpenChildren[Node] = Node->getNumChildren();
+    for (MachineDomTreeNode *Child : Node->children())
       WorkList.push_back(Child);
   } while (!WorkList.empty());
 
@@ -862,8 +861,7 @@ bool MachineCSE::PerformSimplePRE(MachineDominatorTree *DT) {
   BBs.push_back(DT->getRootNode());
   do {
     auto Node = BBs.pop_back_val();
-    const std::vector<MachineDomTreeNode *> &Children = Node->getChildren();
-    for (MachineDomTreeNode *Child : Children)
+    for (MachineDomTreeNode *Child : Node->children())
       BBs.push_back(Child);
 
     MachineBasicBlock *MBB = Node->getBlock();
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 987de0cb3b0e5..d4181591deabc 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -834,11 +834,26 @@ const DILabel *MachineInstr::getDebugLabel() const {
   return cast<DILabel>(getOperand(0).getMetadata());
 }
 
+const MachineOperand &MachineInstr::getDebugVariableOp() const {
+  assert(isDebugValue() && "not a DBG_VALUE");
+  return getOperand(2);
+}
+
+MachineOperand &MachineInstr::getDebugVariableOp() {
+  assert(isDebugValue() && "not a DBG_VALUE");
+  return getOperand(2);
+}
+
 const DILocalVariable *MachineInstr::getDebugVariable() const {
   assert(isDebugValue() && "not a DBG_VALUE");
   return cast<DILocalVariable>(getOperand(2).getMetadata());
 }
 
+MachineOperand &MachineInstr::getDebugExpressionOp() {
+  assert(isDebugValue() && "not a DBG_VALUE");
+  return getOperand(3);
+}
+
 const DIExpression *MachineInstr::getDebugExpression() const {
   assert(isDebugValue() && "not a DBG_VALUE");
   return cast<DIExpression>(getOperand(3).getMetadata());
@@ -1786,12 +1801,12 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST,
   }
 
   // Print extra comments for DEBUG_VALUE.
-  if (isDebugValue() && getOperand(e - 2).isMetadata()) {
+  if (isDebugValue() && getDebugVariableOp().isMetadata()) {
     if (!HaveSemi) {
       OS << ";";
       HaveSemi = true;
     }
-    auto *DV = cast<DILocalVariable>(getOperand(e - 2).getMetadata());
+    auto *DV = getDebugVariable();
     OS << " line no:" <<  DV->getLine();
     if (isIndirectDebugValue())
       OS << " indirect";
@@ -2097,7 +2112,8 @@ static const DIExpression *computeExprForSpill(const MachineInstr &MI) {
 
   const DIExpression *Expr = MI.getDebugExpression();
   if (MI.isIndirectDebugValue()) {
-    assert(MI.getOperand(1).getImm() == 0 && "DBG_VALUE with nonzero offset");
+    assert(MI.getDebugOffset().getImm() == 0 &&
+           "DBG_VALUE with nonzero offset");
     Expr = DIExpression::prepend(Expr, DIExpression::DerefBefore);
   }
   return Expr;
@@ -2117,9 +2133,9 @@ MachineInstr *llvm::buildDbgValueForSpill(MachineBasicBlock &BB,
 
 void llvm::updateDbgValueForSpill(MachineInstr &Orig, int FrameIndex) {
   const DIExpression *Expr = computeExprForSpill(Orig);
-  Orig.getOperand(0).ChangeToFrameIndex(FrameIndex);
-  Orig.getOperand(1).ChangeToImmediate(0U);
-  Orig.getOperand(3).setMetadata(Expr);
+  Orig.getDebugOperand(0).ChangeToFrameIndex(FrameIndex);
+  Orig.getDebugOffset().ChangeToImmediate(0U);
+  Orig.getDebugExpressionOp().setMetadata(Expr);
 }
 
 void MachineInstr::collectDebugValues(
@@ -2133,8 +2149,7 @@ void MachineInstr::collectDebugValues(
        DI != DE; ++DI) {
     if (!DI->isDebugValue())
       return;
-    if (DI->getOperand(0).isReg() &&
-        DI->getOperand(0).getReg() == MI.getOperand(0).getReg())
+    if (DI->getDebugOperandForReg(MI.getOperand(0).getReg()))
       DbgValues.push_back(&*DI);
   }
 }
@@ -2146,21 +2161,20 @@ void MachineInstr::changeDebugValuesDefReg(Register Reg) {
   if (!getOperand(0).isReg())
     return;
 
-  unsigned DefReg = getOperand(0).getReg();
+  Register DefReg = getOperand(0).getReg();
   auto *MRI = getRegInfo();
   for (auto &MO : MRI->use_operands(DefReg)) {
     auto *DI = MO.getParent();
     if (!DI->isDebugValue())
       continue;
-    if (DI->getOperand(0).isReg() &&
-        DI->getOperand(0).getReg() == DefReg){
+    if (DI->getDebugOperandForReg(DefReg)) {
       DbgValues.push_back(DI);
     }
   }
 
   // Propagate Reg to debug value instructions.
   for (auto *DBI : DbgValues)
-    DBI->getOperand(0).setReg(Reg);
+    DBI->getDebugOperandForReg(DefReg)->setReg(Reg);
 }
 
 using MMOList = SmallVector<const MachineMemOperand *, 2>;
diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index 98638b9fa7377..5e8a916b3b3b1 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -737,8 +737,7 @@ void MachineLICMBase::HoistOutOfLoop(MachineDomTreeNode *HeaderN) {
       continue;
 
     Scopes.push_back(Node);
-    const std::vector<MachineDomTreeNode*> &Children = Node->getChildren();
-    unsigned NumChildren = Children.size();
+    unsigned NumChildren = Node->getNumChildren();
 
     // Don't hoist things out of a large switch statement.  This often causes
     // code to be hoisted that wasn't going to be executed, and increases
@@ -747,13 +746,14 @@ void MachineLICMBase::HoistOutOfLoop(MachineDomTreeNode *HeaderN) {
       NumChildren = 0;
 
     OpenChildren[Node] = NumChildren;
-    // Add children in reverse order as then the next popped worklist node is
-    // the first child of this node.  This means we ultimately traverse the
-    // DOM tree in exactly the same order as if we'd recursed.
-    for (int i = (int)NumChildren-1; i >= 0; --i) {
-      MachineDomTreeNode *Child = Children[i];
-      ParentMap[Child] = Node;
-      WorkList.push_back(Child);
+    if (NumChildren) {
+      // Add children in reverse order as then the next popped worklist node is
+      // the first child of this node.  This means we ultimately traverse the
+      // DOM tree in exactly the same order as if we'd recursed.
+      for (MachineDomTreeNode *Child : reverse(Node->children())) {
+        ParentMap[Child] = Node;
+        WorkList.push_back(Child);
+      }
     }
   }
 
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index 5e4d5edb9ce68..2b4fd654e46c0 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -25,6 +25,7 @@
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/IRPrintingPasses.h"
+#include "llvm/IR/Instructions.h"
 #include "llvm/IR/ModuleSlotTracker.h"
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/Target/TargetIntrinsicInfo.h"
@@ -669,7 +670,7 @@ static void printCFI(raw_ostream &OS, const MCCFIInstruction &CFI,
       size_t e = CFI.getValues().size() - 1;
       for (size_t i = 0; i < e; ++i)
         OS << format("0x%02x", uint8_t(CFI.getValues()[i])) << ", ";
-      OS << format("0x%02x", uint8_t(CFI.getValues()[e])) << ", ";
+      OS << format("0x%02x", uint8_t(CFI.getValues()[e]));
     }
     break;
   }
diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp
index d70638bd38400..f9d099e029956 100644
--- a/llvm/lib/CodeGen/MachineOutliner.cpp
+++ b/llvm/lib/CodeGen/MachineOutliner.cpp
@@ -618,9 +618,6 @@ MachineFunction *MachineOutliner::createOutlinedFunction(
   F->setLinkage(GlobalValue::InternalLinkage);
   F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
 
-  // FIXME: Set nounwind, so we don't generate eh_frame? Haven't verified it's
-  // necessary.
-
   // Set optsize/minsize, so we don't insert padding between outlined
   // functions.
   F->addFnAttr(Attribute::OptimizeForSize);
@@ -635,6 +632,12 @@ MachineFunction *MachineOutliner::createOutlinedFunction(
   if (ParentFn.hasFnAttribute("target-features"))
     F->addFnAttr(ParentFn.getFnAttribute("target-features"));
 
+  // Set nounwind, so we don't generate eh_frame.
+  if (llvm::all_of(OF.Candidates, [](const outliner::Candidate &C) {
+        return C.getMF()->getFunction().hasFnAttribute(Attribute::NoUnwind);
+      }))
+    F->addFnAttr(Attribute::NoUnwind);
+
   BasicBlock *EntryBB = BasicBlock::Create(C, "entry", F);
   IRBuilder<> Builder(EntryBB);
   Builder.CreateRetVoid();
diff --git a/llvm/lib/CodeGen/MachineRegisterInfo.cpp b/llvm/lib/CodeGen/MachineRegisterInfo.cpp
index f627f58a73628..4c733738840a1 100644
--- a/llvm/lib/CodeGen/MachineRegisterInfo.cpp
+++ b/llvm/lib/CodeGen/MachineRegisterInfo.cpp
@@ -550,7 +550,7 @@ void MachineRegisterInfo::markUsesInDebugValueAsUndef(Register Reg) const {
     nextI = std::next(I);  // I is invalidated by the setReg
     MachineInstr *UseMI = &*I;
     if (UseMI->isDebugValue())
-      UseMI->getOperand(0).setReg(0U);
+      UseMI->getDebugOperandForReg(Reg)->setReg(0U);
   }
 }
 
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 0f21c97a30f68..cf75d531deb27 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -1573,8 +1573,13 @@ void BaseMemOpClusterMutation::clusterNeighboringMemOps(
     bool OffsetIsScalable;
     unsigned Width;
     if (TII->getMemOperandsWithOffsetWidth(MI, BaseOps, Offset,
-                                           OffsetIsScalable, Width, TRI))
+                                           OffsetIsScalable, Width, TRI)) {
       MemOpRecords.push_back(MemOpInfo(SU, BaseOps, Offset, Width));
+
+      LLVM_DEBUG(dbgs() << "Num BaseOps: " << BaseOps.size() << ", Offset: "
+                        << Offset << ", OffsetIsScalable: " << OffsetIsScalable
+                        << ", Width: " << Width << "\n");
+    }
 #ifndef NDEBUG
     for (auto *Op : BaseOps)
       assert(Op);
@@ -1630,6 +1635,10 @@ void BaseMemOpClusterMutation::clusterNeighboringMemOps(
                         << ")\n");
       DAG->addEdge(Succ.getSUnit(), SDep(SUb, SDep::Artificial));
     }
+
+    LLVM_DEBUG(dbgs() << "  Curr cluster length: " << ClusterLength
+                      << ", Curr cluster bytes: " << CurrentClusterBytes
+                      << "\n");
   }
 }
 
diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp
index 74b71eb7ebdf8..5f958bbc31b7a 100644
--- a/llvm/lib/CodeGen/MachineSink.cpp
+++ b/llvm/lib/CodeGen/MachineSink.cpp
@@ -432,7 +432,7 @@ void MachineSinking::ProcessDbgInst(MachineInstr &MI) {
                     MI.getDebugLoc()->getInlinedAt());
   bool SeenBefore = SeenDbgVars.count(Var) != 0;
 
-  MachineOperand &MO = MI.getOperand(0);
+  MachineOperand &MO = MI.getDebugOperand(0);
   if (MO.isReg() && MO.getReg().isVirtual())
     SeenDbgUsers[MO.getReg()].push_back(SeenDbgUser(&MI, SeenBefore));
 
@@ -623,14 +623,13 @@ MachineSinking::GetAllSortedSuccessors(MachineInstr &MI, MachineBasicBlock *MBB,
   //   if () {} else {}
   //   use x
   //
-  const std::vector<MachineDomTreeNode *> &Children =
-    DT->getNode(MBB)->getChildren();
-  for (const auto &DTChild : Children)
+  for (MachineDomTreeNode *DTChild : DT->getNode(MBB)->children()) {
     // DomTree children of MBB that have MBB as immediate dominator are added.
     if (DTChild->getIDom()->getBlock() == MI.getParent() &&
         // Skip MBBs already added to the AllSuccs vector above.
         !MBB->isSuccessor(DTChild->getBlock()))
       AllSuccs.push_back(DTChild->getBlock());
+  }
 
   // Sort Successors according to their loop depth or block frequency info.
   llvm::stable_sort(
@@ -734,6 +733,13 @@ MachineSinking::FindSuccToSinkTo(MachineInstr &MI, MachineBasicBlock *MBB,
   if (SuccToSinkTo && SuccToSinkTo->isEHPad())
     return nullptr;
 
+  // It ought to be okay to sink instructions into an INLINEASM_BR target, but
+  // only if we make sure that MI occurs _before_ an INLINEASM_BR instruction in
+  // the source block (which this code does not yet do). So for now, forbid
+  // doing so.
+  if (SuccToSinkTo && SuccToSinkTo->isInlineAsmBrIndirectTarget())
+    return nullptr;
+
   return SuccToSinkTo;
 }
 
@@ -796,7 +802,7 @@ static bool attemptDebugCopyProp(MachineInstr &SinkInst, MachineInstr &DbgMI) {
   // Copy DBG_VALUE operand and set the original to undef. We then check to
   // see whether this is something that can be copy-forwarded. If it isn't,
   // continue around the loop.
-  MachineOperand DbgMO = DbgMI.getOperand(0);
+  MachineOperand &DbgMO = DbgMI.getDebugOperand(0);
 
   const MachineOperand *SrcMO = nullptr, *DstMO = nullptr;
   auto CopyOperands = TII.isCopyInstr(SinkInst);
@@ -830,8 +836,8 @@ static bool attemptDebugCopyProp(MachineInstr &SinkInst, MachineInstr &DbgMI) {
   if (PostRA && DbgMO.getReg() != DstMO->getReg())
     return false;
 
-  DbgMI.getOperand(0).setReg(SrcMO->getReg());
-  DbgMI.getOperand(0).setSubReg(SrcMO->getSubReg());
+  DbgMO.setReg(SrcMO->getReg());
+  DbgMO.setSubReg(SrcMO->getSubReg());
   return true;
 }
 
@@ -866,7 +872,7 @@ static void performSink(MachineInstr &MI, MachineBasicBlock &SuccToSinkTo,
     SuccToSinkTo.insert(InsertPos, NewDbgMI);
 
     if (!attemptDebugCopyProp(MI, *DbgMI))
-      DbgMI->getOperand(0).setReg(0);
+      DbgMI->setDebugValueUndef();
   }
 }
 
@@ -1000,7 +1006,7 @@ bool MachineSinking::SinkInstruction(MachineInstr &MI, bool &SawStore,
         // This DBG_VALUE would re-order assignments. If we can't copy-propagate
         // it, it can't be recovered. Set it undef.
         if (!attemptDebugCopyProp(MI, *DbgMI))
-          DbgMI->getOperand(0).setReg(0);
+          DbgMI->setDebugValueUndef();
       } else {
         DbgUsersToSink.push_back(DbgMI);
       }
@@ -1049,7 +1055,7 @@ void MachineSinking::SalvageUnsunkDebugUsersOfCopy(
       if (User.getParent() == MI.getParent())
         continue;
 
-      assert(User.getOperand(0).isReg() &&
+      assert(User.getDebugOperand(0).isReg() &&
              "DBG_VALUE user of vreg, but non reg operand?");
       DbgDefUsers.push_back(&User);
     }
@@ -1058,8 +1064,8 @@ void MachineSinking::SalvageUnsunkDebugUsersOfCopy(
   // Point the users of this copy that are no longer dominated, at the source
   // of the copy.
   for (auto *User : DbgDefUsers) {
-    User->getOperand(0).setReg(MI.getOperand(1).getReg());
-    User->getOperand(0).setSubReg(MI.getOperand(1).getSubReg());
+    User->getDebugOperand(0).setReg(MI.getOperand(1).getReg());
+    User->getDebugOperand(0).setSubReg(MI.getOperand(1).getSubReg());
   }
 }
 
@@ -1305,7 +1311,7 @@ bool PostRAMachineSinking::tryToSinkCopy(MachineBasicBlock &CurBB,
     // We must sink this DBG_VALUE if its operand is sunk. To avoid searching
     // for DBG_VALUEs later, record them when they're encountered.
     if (MI->isDebugValue()) {
-      auto &MO = MI->getOperand(0);
+      auto &MO = MI->getDebugOperand(0);
       if (MO.isReg() && Register::isPhysicalRegister(MO.getReg())) {
         // Bail if we can already tell the sink would be rejected, rather
         // than needlessly accumulating lots of DBG_VALUEs.
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index c032b6989c86f..c1a2c4e0bc6e6 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -584,7 +584,6 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
     // it is an entry block or landing pad.
     for (const auto &LI : MBB->liveins()) {
       if (isAllocatable(LI.PhysReg) && !MBB->isEHPad() &&
-          !MBB->isInlineAsmBrDefaultTarget() &&
           MBB->getIterator() != MBB->getParent()->begin()) {
         report("MBB has allocatable live-in, but isn't entry or landing-pad.", MBB);
         report_context(LI.PhysReg);
@@ -730,7 +729,7 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
         continue;
       // Also accept successors which are for exception-handling or might be
       // inlineasm_br targets.
-      if (SuccMBB->isEHPad() || MBB->isInlineAsmBrIndirectTarget(SuccMBB))
+      if (SuccMBB->isEHPad() || SuccMBB->isInlineAsmBrIndirectTarget())
         continue;
       report("MBB has unexpected successors which are not branch targets, "
              "fallthrough, EHPads, or inlineasm_br targets.",
@@ -1019,6 +1018,10 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
 
     if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
       report("bitcast sizes must match", MI);
+
+    if (SrcTy == DstTy)
+      report("bitcast must change the type", MI);
+
     break;
   }
   case TargetOpcode::G_INTTOPTR:
@@ -1695,6 +1698,15 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
       if (!RC) {
         // This is a generic virtual register.
 
+        // Do not allow undef uses for generic virtual registers. This ensures
+        // getVRegDef can never fail and return null on a generic register.
+        //
+        // FIXME: This restriction should probably be broadened to all SSA
+        // MIR. However, DetectDeadLanes/ProcessImplicitDefs technically still
+        // run on the SSA function just before phi elimination.
+        if (MO->isUndef())
+          report("Generic virtual register use cannot be undef", MO, MONum);
+
         // If we're post-Select, we can't have gvregs anymore.
         if (isFunctionSelected) {
           report("Generic virtual register invalid in a Selected function",
diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp b/llvm/lib/CodeGen/ModuloSchedule.cpp
index a8088dc77cf47..d85b1b7988cec 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/CodeGen/ModuloSchedule.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/CodeGen/LiveIntervals.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineLoopUtils.h"
@@ -1946,7 +1947,7 @@ void PeelingModuloScheduleExpander::fixupBranches() {
     SmallVector<MachineOperand, 4> Cond;
     TII->removeBranch(*Prolog);
     Optional<bool> StaticallyGreater =
-        Info->createTripCountGreaterCondition(TC, *Prolog, Cond);
+        LoopInfo->createTripCountGreaterCondition(TC, *Prolog, Cond);
     if (!StaticallyGreater.hasValue()) {
       LLVM_DEBUG(dbgs() << "Dynamic: TC > " << TC << "\n");
       // Dynamically branch based on Cond.
@@ -1974,10 +1975,10 @@ void PeelingModuloScheduleExpander::fixupBranches() {
   }
 
   if (!KernelDisposed) {
-    Info->adjustTripCount(-(Schedule.getNumStages() - 1));
-    Info->setPreheader(Prologs.back());
+    LoopInfo->adjustTripCount(-(Schedule.getNumStages() - 1));
+    LoopInfo->setPreheader(Prologs.back());
   } else {
-    Info->disposed();
+    LoopInfo->disposed();
   }
 }
 
@@ -1990,8 +1991,8 @@ void PeelingModuloScheduleExpander::expand() {
   BB = Schedule.getLoop()->getTopBlock();
   Preheader = Schedule.getLoop()->getLoopPreheader();
   LLVM_DEBUG(Schedule.dump());
-  Info = TII->analyzeLoopForPipelining(BB);
-  assert(Info);
+  LoopInfo = TII->analyzeLoopForPipelining(BB);
+  assert(LoopInfo);
 
   rewriteKernel();
   peelPrologAndEpilogs();
diff --git a/llvm/lib/CodeGen/PHIEliminationUtils.cpp b/llvm/lib/CodeGen/PHIEliminationUtils.cpp
index 3a2cdaf3bd3cb..bae96eb84521a 100644
--- a/llvm/lib/CodeGen/PHIEliminationUtils.cpp
+++ b/llvm/lib/CodeGen/PHIEliminationUtils.cpp
@@ -26,8 +26,9 @@ llvm::findPHICopyInsertPoint(MachineBasicBlock* MBB, MachineBasicBlock* SuccMBB,
 
   // Usually, we just want to insert the copy before the first terminator
   // instruction. However, for the edge going to a landing pad, we must insert
-  // the copy before the call/invoke instruction.
-  if (!SuccMBB->isEHPad())
+  // the copy before the call/invoke instruction. Similarly for an INLINEASM_BR
+  // going to an indirect target.
+  if (!SuccMBB->isEHPad() && !SuccMBB->isInlineAsmBrIndirectTarget())
     return MBB->getFirstTerminator();
 
   // Discover any defs/uses in this basic block.
diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
index 70dd1d1bd70b2..a489f493d5ee8 100644
--- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
@@ -185,7 +185,7 @@ static void stashEntryDbgValues(MachineBasicBlock &MBB,
       break;
     if (!MI.isDebugValue() || !MI.getDebugVariable()->isParameter())
       continue;
-    if (MI.getOperand(0).isFI()) {
+    if (MI.getDebugOperand(0).isFI()) {
       // We can only emit valid locations for frame indices after the frame
       // setup, so do not stash away them.
       FrameIndexValues.push_back(&MI);
@@ -1234,10 +1234,10 @@ void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &MF,
           bool WithStackValue = true;
           DIExpr = DIExpression::prependOpcodes(DIExpr, Ops, WithStackValue);
           // Make the DBG_VALUE direct.
-          MI.getOperand(1).ChangeToRegister(0, false);
+          MI.getDebugOffset().ChangeToRegister(0, false);
         }
         DIExpr = DIExpression::prepend(DIExpr, PrependFlags, Offset);
-        MI.getOperand(3).setMetadata(DIExpr);
+        MI.getDebugExpressionOp().setMetadata(DIExpr);
         continue;
       }
 
diff --git a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
index ea667f1704f6b..5bd8b4b8e27f6 100644
--- a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
+++ b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
@@ -389,6 +389,12 @@ ReachingDefAnalysis::getGlobalUses(MachineInstr *MI, int PhysReg,
   }
 }
 
+void ReachingDefAnalysis::getLiveOuts(MachineBasicBlock *MBB, int PhysReg,
+                                      InstSet &Defs) const {
+  SmallPtrSet<MachineBasicBlock*, 2> VisitedBBs;
+  getLiveOuts(MBB, PhysReg, Defs, VisitedBBs);
+}
+
 void
 ReachingDefAnalysis::getLiveOuts(MachineBasicBlock *MBB, int PhysReg,
                                  InstSet &Defs, BlockSet &VisitedBBs) const {
diff --git a/llvm/lib/CodeGen/RegAllocBase.cpp b/llvm/lib/CodeGen/RegAllocBase.cpp
index 209edd8d50073..d228268536724 100644
--- a/llvm/lib/CodeGen/RegAllocBase.cpp
+++ b/llvm/lib/CodeGen/RegAllocBase.cpp
@@ -107,7 +107,7 @@ void RegAllocBase::allocatePhysRegs() {
                       << TRI->getRegClassName(MRI->getRegClass(VirtReg->reg))
                       << ':' << *VirtReg << " w=" << VirtReg->weight << '\n');
 
-    using VirtRegVec = SmallVector<unsigned, 4>;
+    using VirtRegVec = SmallVector<Register, 4>;
 
     VirtRegVec SplitVRegs;
     unsigned AvailablePhysReg = selectOrSplit(*VirtReg, SplitVRegs);
diff --git a/llvm/lib/CodeGen/RegAllocBase.h b/llvm/lib/CodeGen/RegAllocBase.h
index 6a7cc5ba43084..8e931eaae99a0 100644
--- a/llvm/lib/CodeGen/RegAllocBase.h
+++ b/llvm/lib/CodeGen/RegAllocBase.h
@@ -101,8 +101,8 @@ class RegAllocBase {
   // Each call must guarantee forward progess by returning an available PhysReg
   // or new set of split live virtual registers. It is up to the splitter to
   // converge quickly toward fully spilled live ranges.
-  virtual unsigned selectOrSplit(LiveInterval &VirtReg,
-                                 SmallVectorImpl<unsigned> &splitLVRs) = 0;
+  virtual Register selectOrSplit(LiveInterval &VirtReg,
+                                 SmallVectorImpl<Register> &splitLVRs) = 0;
 
   // Use this group name for NamedRegionTimer.
   static const char TimerGroupName[];
diff --git a/llvm/lib/CodeGen/RegAllocBasic.cpp b/llvm/lib/CodeGen/RegAllocBasic.cpp
index ba0107953f0f1..5009bcc0a3973 100644
--- a/llvm/lib/CodeGen/RegAllocBasic.cpp
+++ b/llvm/lib/CodeGen/RegAllocBasic.cpp
@@ -100,8 +100,8 @@ class RABasic : public MachineFunctionPass,
     return LI;
   }
 
-  unsigned selectOrSplit(LiveInterval &VirtReg,
-                         SmallVectorImpl<unsigned> &SplitVRegs) override;
+  Register selectOrSplit(LiveInterval &VirtReg,
+                         SmallVectorImpl<Register> &SplitVRegs) override;
 
   /// Perform register allocation.
   bool runOnMachineFunction(MachineFunction &mf) override;
@@ -114,8 +114,8 @@ class RABasic : public MachineFunctionPass,
   // Helper for spilling all live virtual registers currently unified under preg
   // that interfere with the most recently queried lvr.  Return true if spilling
   // was successful, and append any new spilled/split intervals to splitLVRs.
-  bool spillInterferences(LiveInterval &VirtReg, unsigned PhysReg,
-                          SmallVectorImpl<unsigned> &SplitVRegs);
+  bool spillInterferences(LiveInterval &VirtReg, Register PhysReg,
+                          SmallVectorImpl<Register> &SplitVRegs);
 
   static char ID;
 };
@@ -201,8 +201,8 @@ void RABasic::releaseMemory() {
 // Spill or split all live virtual registers currently unified under PhysReg
 // that interfere with VirtReg. The newly spilled or split live intervals are
 // returned by appending them to SplitVRegs.
-bool RABasic::spillInterferences(LiveInterval &VirtReg, unsigned PhysReg,
-                                 SmallVectorImpl<unsigned> &SplitVRegs) {
+bool RABasic::spillInterferences(LiveInterval &VirtReg, Register PhysReg,
+                                 SmallVectorImpl<Register> &SplitVRegs) {
   // Record each interference and determine if all are spillable before mutating
   // either the union or live intervals.
   SmallVector<LiveInterval*, 8> Intfs;
@@ -253,14 +253,14 @@ bool RABasic::spillInterferences(LiveInterval &VirtReg, unsigned PhysReg,
 // |vregs| * |machineregs|. And since the number of interference tests is
 // minimal, there is no value in caching them outside the scope of
 // selectOrSplit().
-unsigned RABasic::selectOrSplit(LiveInterval &VirtReg,
-                                SmallVectorImpl<unsigned> &SplitVRegs) {
+Register RABasic::selectOrSplit(LiveInterval &VirtReg,
+                                SmallVectorImpl<Register> &SplitVRegs) {
   // Populate a list of physical register spill candidates.
-  SmallVector<unsigned, 8> PhysRegSpillCands;
+  SmallVector<Register, 8> PhysRegSpillCands;
 
   // Check for an available register in this class.
   AllocationOrder Order(VirtReg.reg, *VRM, RegClassInfo, Matrix);
-  while (unsigned PhysReg = Order.next()) {
+  while (Register PhysReg = Order.next()) {
     // Check for interference in PhysReg
     switch (Matrix->checkInterference(VirtReg, PhysReg)) {
     case LiveRegMatrix::IK_Free:
@@ -279,7 +279,7 @@ unsigned RABasic::selectOrSplit(LiveInterval &VirtReg,
   }
 
   // Try to spill another interfering reg with less spill weight.
-  for (SmallVectorImpl<unsigned>::iterator PhysRegI = PhysRegSpillCands.begin(),
+  for (SmallVectorImpl<Register>::iterator PhysRegI = PhysRegSpillCands.begin(),
        PhysRegE = PhysRegSpillCands.end(); PhysRegI != PhysRegE; ++PhysRegI) {
     if (!spillInterferences(VirtReg, *PhysRegI, SplitVRegs))
       continue;
diff --git a/llvm/lib/CodeGen/RegAllocFast.cpp b/llvm/lib/CodeGen/RegAllocFast.cpp
index df7981a2928c6..5396f9f3a1432 100644
--- a/llvm/lib/CodeGen/RegAllocFast.cpp
+++ b/llvm/lib/CodeGen/RegAllocFast.cpp
@@ -1156,7 +1156,7 @@ void RegAllocFast::allocateInstruction(MachineInstr &MI) {
 }
 
 void RegAllocFast::handleDebugValue(MachineInstr &MI) {
-  MachineOperand &MO = MI.getOperand(0);
+  MachineOperand &MO = MI.getDebugOperand(0);
 
   // Ignore DBG_VALUEs that aren't based on virtual registers. These are
   // mostly constants and frame indices.
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index a5871a5fbb07e..41cf002612654 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -417,7 +417,7 @@ class RAGreedy : public MachineFunctionPass,
   Spiller &spiller() override { return *SpillerInstance; }
   void enqueue(LiveInterval *LI) override;
   LiveInterval *dequeue() override;
-  unsigned selectOrSplit(LiveInterval&, SmallVectorImpl<unsigned>&) override;
+  Register selectOrSplit(LiveInterval&, SmallVectorImpl<Register>&) override;
   void aboutToRemoveInterval(LiveInterval &) override;
 
   /// Perform register allocation.
@@ -431,7 +431,7 @@ class RAGreedy : public MachineFunctionPass,
   static char ID;
 
 private:
-  unsigned selectOrSplitImpl(LiveInterval &, SmallVectorImpl<unsigned> &,
+  Register selectOrSplitImpl(LiveInterval &, SmallVectorImpl<Register> &,
                              SmallVirtRegSet &, unsigned = 0);
 
   bool LRE_CanEraseVirtReg(unsigned) override;
@@ -456,30 +456,30 @@ class RAGreedy : public MachineFunctionPass,
   bool calcCompactRegion(GlobalSplitCandidate&);
   void splitAroundRegion(LiveRangeEdit&, ArrayRef<unsigned>);
   void calcGapWeights(unsigned, SmallVectorImpl<float>&);
-  unsigned canReassign(LiveInterval &VirtReg, unsigned PrevReg);
+  Register canReassign(LiveInterval &VirtReg, Register PrevReg);
   bool shouldEvict(LiveInterval &A, bool, LiveInterval &B, bool);
-  bool canEvictInterference(LiveInterval&, unsigned, bool, EvictionCost&,
+  bool canEvictInterference(LiveInterval&, Register, bool, EvictionCost&,
                             const SmallVirtRegSet&);
-  bool canEvictInterferenceInRange(LiveInterval &VirtReg, unsigned PhysReg,
+  bool canEvictInterferenceInRange(LiveInterval &VirtReg, Register oPhysReg,
                                    SlotIndex Start, SlotIndex End,
                                    EvictionCost &MaxCost);
   unsigned getCheapestEvicteeWeight(const AllocationOrder &Order,
                                     LiveInterval &VirtReg, SlotIndex Start,
                                     SlotIndex End, float *BestEvictWeight);
-  void evictInterference(LiveInterval&, unsigned,
-                         SmallVectorImpl<unsigned>&);
+  void evictInterference(LiveInterval&, Register,
+                         SmallVectorImpl<Register>&);
   bool mayRecolorAllInterferences(unsigned PhysReg, LiveInterval &VirtReg,
                                   SmallLISet &RecoloringCandidates,
                                   const SmallVirtRegSet &FixedRegisters);
 
-  unsigned tryAssign(LiveInterval&, AllocationOrder&,
-                     SmallVectorImpl<unsigned>&,
+  Register tryAssign(LiveInterval&, AllocationOrder&,
+                     SmallVectorImpl<Register>&,
                      const SmallVirtRegSet&);
   unsigned tryEvict(LiveInterval&, AllocationOrder&,
-                    SmallVectorImpl<unsigned>&, unsigned,
+                    SmallVectorImpl<Register>&, unsigned,
                     const SmallVirtRegSet&);
   unsigned tryRegionSplit(LiveInterval&, AllocationOrder&,
-                          SmallVectorImpl<unsigned>&);
+                          SmallVectorImpl<Register>&);
   /// Calculate cost of region splitting.
   unsigned calculateRegionSplitCost(LiveInterval &VirtReg,
                                     AllocationOrder &Order,
@@ -489,26 +489,26 @@ class RAGreedy : public MachineFunctionPass,
   /// Perform region splitting.
   unsigned doRegionSplit(LiveInterval &VirtReg, unsigned BestCand,
                          bool HasCompact,
-                         SmallVectorImpl<unsigned> &NewVRegs);
+                         SmallVectorImpl<Register> &NewVRegs);
   /// Check other options before using a callee-saved register for the first
   /// time.
   unsigned tryAssignCSRFirstTime(LiveInterval &VirtReg, AllocationOrder &Order,
-                                 unsigned PhysReg, unsigned &CostPerUseLimit,
-                                 SmallVectorImpl<unsigned> &NewVRegs);
+                                 Register PhysReg, unsigned &CostPerUseLimit,
+                                 SmallVectorImpl<Register> &NewVRegs);
   void initializeCSRCost();
   unsigned tryBlockSplit(LiveInterval&, AllocationOrder&,
-                         SmallVectorImpl<unsigned>&);
+                         SmallVectorImpl<Register>&);
   unsigned tryInstructionSplit(LiveInterval&, AllocationOrder&,
-                               SmallVectorImpl<unsigned>&);
+                               SmallVectorImpl<Register>&);
   unsigned tryLocalSplit(LiveInterval&, AllocationOrder&,
-    SmallVectorImpl<unsigned>&);
+    SmallVectorImpl<Register>&);
   unsigned trySplit(LiveInterval&, AllocationOrder&,
-                    SmallVectorImpl<unsigned>&,
+                    SmallVectorImpl<Register>&,
                     const SmallVirtRegSet&);
   unsigned tryLastChanceRecoloring(LiveInterval &, AllocationOrder &,
-                                   SmallVectorImpl<unsigned> &,
+                                   SmallVectorImpl<Register> &,
                                    SmallVirtRegSet &, unsigned);
-  bool tryRecoloringCandidates(PQueue &, SmallVectorImpl<unsigned> &,
+  bool tryRecoloringCandidates(PQueue &, SmallVectorImpl<Register> &,
                                SmallVirtRegSet &, unsigned);
   void tryHintRecoloring(LiveInterval &);
   void tryHintsRecoloring();
@@ -518,12 +518,12 @@ class RAGreedy : public MachineFunctionPass,
     /// The frequency of the copy.
     BlockFrequency Freq;
     /// The virtual register or physical register.
-    unsigned Reg;
+    Register Reg;
     /// Its currently assigned register.
     /// In case of a physical register Reg == PhysReg.
-    unsigned PhysReg;
+    MCRegister PhysReg;
 
-    HintInfo(BlockFrequency Freq, unsigned Reg, unsigned PhysReg)
+    HintInfo(BlockFrequency Freq, Register Reg, MCRegister PhysReg)
         : Freq(Freq), Reg(Reg), PhysReg(PhysReg) {}
   };
   using HintsInfo = SmallVector<HintInfo, 4>;
@@ -531,7 +531,7 @@ class RAGreedy : public MachineFunctionPass,
   BlockFrequency getBrokenHintFreq(const HintsInfo &, unsigned);
   void collectHintInfo(unsigned, HintsInfo &);
 
-  bool isUnusedCalleeSavedReg(unsigned PhysReg) const;
+  bool isUnusedCalleeSavedReg(MCRegister PhysReg) const;
 
   /// Compute and report the number of spills and reloads for a loop.
   void reportNumberOfSplillsReloads(MachineLoop *L, unsigned &Reloads,
@@ -752,12 +752,12 @@ LiveInterval *RAGreedy::dequeue(PQueue &CurQueue) {
 //===----------------------------------------------------------------------===//
 
 /// tryAssign - Try to assign VirtReg to an available register.
-unsigned RAGreedy::tryAssign(LiveInterval &VirtReg,
+Register RAGreedy::tryAssign(LiveInterval &VirtReg,
                              AllocationOrder &Order,
-                             SmallVectorImpl<unsigned> &NewVRegs,
+                             SmallVectorImpl<Register> &NewVRegs,
                              const SmallVirtRegSet &FixedRegisters) {
   Order.rewind();
-  unsigned PhysReg;
+  Register PhysReg;
   while ((PhysReg = Order.next()))
     if (!Matrix->checkInterference(VirtReg, PhysReg))
       break;
@@ -768,7 +768,7 @@ unsigned RAGreedy::tryAssign(LiveInterval &VirtReg,
 
   // If we missed a simple hint, try to cheaply evict interference from the
   // preferred register.
-  if (unsigned Hint = MRI->getSimpleHint(VirtReg.reg))
+  if (Register Hint = MRI->getSimpleHint(VirtReg.reg))
     if (Order.isHint(Hint)) {
       LLVM_DEBUG(dbgs() << "missed hint " << printReg(Hint, TRI) << '\n');
       EvictionCost MaxCost;
@@ -791,7 +791,7 @@ unsigned RAGreedy::tryAssign(LiveInterval &VirtReg,
 
   LLVM_DEBUG(dbgs() << printReg(PhysReg, TRI) << " is available at cost "
                     << Cost << '\n');
-  unsigned CheapReg = tryEvict(VirtReg, Order, NewVRegs, Cost, FixedRegisters);
+  Register CheapReg = tryEvict(VirtReg, Order, NewVRegs, Cost, FixedRegisters);
   return CheapReg ? CheapReg : PhysReg;
 }
 
@@ -799,9 +799,9 @@ unsigned RAGreedy::tryAssign(LiveInterval &VirtReg,
 //                         Interference eviction
 //===----------------------------------------------------------------------===//
 
-unsigned RAGreedy::canReassign(LiveInterval &VirtReg, unsigned PrevReg) {
+Register RAGreedy::canReassign(LiveInterval &VirtReg, Register PrevReg) {
   AllocationOrder Order(VirtReg.reg, *VRM, RegClassInfo, Matrix);
-  unsigned PhysReg;
+  Register PhysReg;
   while ((PhysReg = Order.next())) {
     if (PhysReg == PrevReg)
       continue;
@@ -862,7 +862,7 @@ bool RAGreedy::shouldEvict(LiveInterval &A, bool IsHint,
 /// @param MaxCost Only look for cheaper candidates and update with new cost
 ///                when returning true.
 /// @returns True when interference can be evicted cheaper than MaxCost.
-bool RAGreedy::canEvictInterference(LiveInterval &VirtReg, unsigned PhysReg,
+bool RAGreedy::canEvictInterference(LiveInterval &VirtReg, Register PhysReg,
                                     bool IsHint, EvictionCost &MaxCost,
                                     const SmallVirtRegSet &FixedRegisters) {
   // It is only possible to evict virtual register interference.
@@ -960,7 +960,7 @@ bool RAGreedy::canEvictInterference(LiveInterval &VirtReg, unsigned PhysReg,
 ///                when returning true.
 /// \return True when interference can be evicted cheaper than MaxCost.
 bool RAGreedy::canEvictInterferenceInRange(LiveInterval &VirtReg,
-                                           unsigned PhysReg, SlotIndex Start,
+                                           Register PhysReg, SlotIndex Start,
                                            SlotIndex End,
                                            EvictionCost &MaxCost) {
   EvictionCost Cost;
@@ -1038,8 +1038,8 @@ unsigned RAGreedy::getCheapestEvicteeWeight(const AllocationOrder &Order,
 /// evictInterference - Evict any interferring registers that prevent VirtReg
 /// from being assigned to Physreg. This assumes that canEvictInterference
 /// returned true.
-void RAGreedy::evictInterference(LiveInterval &VirtReg, unsigned PhysReg,
-                                 SmallVectorImpl<unsigned> &NewVRegs) {
+void RAGreedy::evictInterference(LiveInterval &VirtReg, Register PhysReg,
+                                 SmallVectorImpl<Register> &NewVRegs) {
   // Make sure that VirtReg has a cascade number, and assign that cascade
   // number to every evicted register. These live ranges than then only be
   // evicted by a newer cascade, preventing infinite loops.
@@ -1084,9 +1084,9 @@ void RAGreedy::evictInterference(LiveInterval &VirtReg, unsigned PhysReg,
 
 /// Returns true if the given \p PhysReg is a callee saved register and has not
 /// been used for allocation yet.
-bool RAGreedy::isUnusedCalleeSavedReg(unsigned PhysReg) const {
-  unsigned CSR = RegClassInfo.getLastCalleeSavedAlias(PhysReg);
-  if (CSR == 0)
+bool RAGreedy::isUnusedCalleeSavedReg(MCRegister PhysReg) const {
+  MCRegister CSR = RegClassInfo.getLastCalleeSavedAlias(PhysReg);
+  if (!CSR)
     return false;
 
   return !Matrix->isPhysRegUsed(PhysReg);
@@ -1098,7 +1098,7 @@ bool RAGreedy::isUnusedCalleeSavedReg(unsigned PhysReg) const {
 /// @return         Physreg to assign VirtReg, or 0.
 unsigned RAGreedy::tryEvict(LiveInterval &VirtReg,
                             AllocationOrder &Order,
-                            SmallVectorImpl<unsigned> &NewVRegs,
+                            SmallVectorImpl<Register> &NewVRegs,
                             unsigned CostPerUseLimit,
                             const SmallVirtRegSet &FixedRegisters) {
   NamedRegionTimer T("evict", "Evict", TimerGroupName, TimerGroupDescription,
@@ -1135,7 +1135,7 @@ unsigned RAGreedy::tryEvict(LiveInterval &VirtReg,
   }
 
   Order.rewind();
-  while (unsigned PhysReg = Order.next(OrderLimit)) {
+  while (MCRegister PhysReg = Order.next(OrderLimit)) {
     if (TRI->getCostPerUse(PhysReg) >= CostPerUseLimit)
       continue;
     // The first use of a callee-saved register in a function has cost 1.
@@ -1809,7 +1809,7 @@ void RAGreedy::splitAroundRegion(LiveRangeEdit &LREdit,
 }
 
 unsigned RAGreedy::tryRegionSplit(LiveInterval &VirtReg, AllocationOrder &Order,
-                                  SmallVectorImpl<unsigned> &NewVRegs) {
+                                  SmallVectorImpl<Register> &NewVRegs) {
   if (!TRI->shouldRegionSplitForVirtReg(*MF, VirtReg))
     return 0;
   unsigned NumCands = 0;
@@ -1953,7 +1953,7 @@ unsigned RAGreedy::calculateRegionSplitCost(LiveInterval &VirtReg,
 
 unsigned RAGreedy::doRegionSplit(LiveInterval &VirtReg, unsigned BestCand,
                                  bool HasCompact,
-                                 SmallVectorImpl<unsigned> &NewVRegs) {
+                                 SmallVectorImpl<Register> &NewVRegs) {
   SmallVector<unsigned, 8> UsedCands;
   // Prepare split editor.
   LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats);
@@ -1999,9 +1999,9 @@ unsigned RAGreedy::doRegionSplit(LiveInterval &VirtReg, unsigned BestCand,
 /// creates a lot of local live ranges, that will be split by tryLocalSplit if
 /// they don't allocate.
 unsigned RAGreedy::tryBlockSplit(LiveInterval &VirtReg, AllocationOrder &Order,
-                                 SmallVectorImpl<unsigned> &NewVRegs) {
+                                 SmallVectorImpl<Register> &NewVRegs) {
   assert(&SA->getParent() == &VirtReg && "Live range wasn't analyzed");
-  unsigned Reg = VirtReg.reg;
+  Register Reg = VirtReg.reg;
   bool SingleInstrs = RegClassInfo.isProperSubClass(MRI->getRegClass(Reg));
   LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats);
   SE->reset(LREdit, SplitSpillMode);
@@ -2066,7 +2066,7 @@ static unsigned getNumAllocatableRegsForConstraints(
 /// This is similar to spilling to a larger register class.
 unsigned
 RAGreedy::tryInstructionSplit(LiveInterval &VirtReg, AllocationOrder &Order,
-                              SmallVectorImpl<unsigned> &NewVRegs) {
+                              SmallVectorImpl<Register> &NewVRegs) {
   const TargetRegisterClass *CurRC = MRI->getRegClass(VirtReg.reg);
   // There is no point to this if there are no larger sub-classes.
   if (!RegClassInfo.isProperSubClass(CurRC))
@@ -2209,7 +2209,7 @@ void RAGreedy::calcGapWeights(unsigned PhysReg,
 /// basic block.
 ///
 unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order,
-                                 SmallVectorImpl<unsigned> &NewVRegs) {
+                                 SmallVectorImpl<Register> &NewVRegs) {
   // TODO: the function currently only handles a single UseBlock; it should be
   // possible to generalize.
   if (SA->getUseBlocks().size() != 1)
@@ -2440,7 +2440,7 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order,
 /// assignable.
 /// @return Physreg when VirtReg may be assigned and/or new NewVRegs.
 unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order,
-                            SmallVectorImpl<unsigned>&NewVRegs,
+                            SmallVectorImpl<Register> &NewVRegs,
                             const SmallVirtRegSet &FixedRegisters) {
   // Ranges must be Split2 or less.
   if (getStage(VirtReg) >= RS_Spill)
@@ -2451,7 +2451,7 @@ unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order,
     NamedRegionTimer T("local_split", "Local Splitting", TimerGroupName,
                        TimerGroupDescription, TimePassesIsEnabled);
     SA->analyze(&VirtReg);
-    unsigned PhysReg = tryLocalSplit(VirtReg, Order, NewVRegs);
+    Register PhysReg = tryLocalSplit(VirtReg, Order, NewVRegs);
     if (PhysReg || !NewVRegs.empty())
       return PhysReg;
     return tryInstructionSplit(VirtReg, Order, NewVRegs);
@@ -2469,7 +2469,7 @@ unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order,
   if (SA->didRepairRange()) {
     // VirtReg has changed, so all cached queries are invalid.
     Matrix->invalidateVirtRegs();
-    if (unsigned PhysReg = tryAssign(VirtReg, Order, NewVRegs, FixedRegisters))
+    if (Register PhysReg = tryAssign(VirtReg, Order, NewVRegs, FixedRegisters))
       return PhysReg;
   }
 
@@ -2584,7 +2584,7 @@ RAGreedy::mayRecolorAllInterferences(unsigned PhysReg, LiveInterval &VirtReg,
 /// exists.
 unsigned RAGreedy::tryLastChanceRecoloring(LiveInterval &VirtReg,
                                            AllocationOrder &Order,
-                                           SmallVectorImpl<unsigned> &NewVRegs,
+                                           SmallVectorImpl<Register> &NewVRegs,
                                            SmallVirtRegSet &FixedRegisters,
                                            unsigned Depth) {
   LLVM_DEBUG(dbgs() << "Try last chance recoloring for " << VirtReg << '\n');
@@ -2605,15 +2605,15 @@ unsigned RAGreedy::tryLastChanceRecoloring(LiveInterval &VirtReg,
   SmallLISet RecoloringCandidates;
   // Record the original mapping virtual register to physical register in case
   // the recoloring fails.
-  DenseMap<unsigned, unsigned> VirtRegToPhysReg;
+  DenseMap<Register, Register> VirtRegToPhysReg;
   // Mark VirtReg as fixed, i.e., it will not be recolored pass this point in
   // this recoloring "session".
   assert(!FixedRegisters.count(VirtReg.reg));
   FixedRegisters.insert(VirtReg.reg);
-  SmallVector<unsigned, 4> CurrentNewVRegs;
+  SmallVector<Register, 4> CurrentNewVRegs;
 
   Order.rewind();
-  while (unsigned PhysReg = Order.next()) {
+  while (Register PhysReg = Order.next()) {
     LLVM_DEBUG(dbgs() << "Try to assign: " << VirtReg << " to "
                       << printReg(PhysReg, TRI) << '\n');
     RecoloringCandidates.clear();
@@ -2644,7 +2644,7 @@ unsigned RAGreedy::tryLastChanceRecoloring(LiveInterval &VirtReg,
     for (SmallLISet::iterator It = RecoloringCandidates.begin(),
                               EndIt = RecoloringCandidates.end();
          It != EndIt; ++It) {
-      unsigned ItVirtReg = (*It)->reg;
+      Register ItVirtReg = (*It)->reg;
       enqueue(RecoloringQueue, *It);
       assert(VRM->hasPhys(ItVirtReg) &&
              "Interferences are supposed to be with allocated variables");
@@ -2667,7 +2667,7 @@ unsigned RAGreedy::tryLastChanceRecoloring(LiveInterval &VirtReg,
     if (tryRecoloringCandidates(RecoloringQueue, CurrentNewVRegs,
                                 FixedRegisters, Depth)) {
       // Push the queued vregs into the main queue.
-      for (unsigned NewVReg : CurrentNewVRegs)
+      for (Register NewVReg : CurrentNewVRegs)
         NewVRegs.push_back(NewVReg);
       // Do not mess up with the global assignment process.
       // I.e., VirtReg must be unassigned.
@@ -2686,7 +2686,7 @@ unsigned RAGreedy::tryLastChanceRecoloring(LiveInterval &VirtReg,
     // don't add it to NewVRegs because its physical register will be restored
     // below. Other vregs in CurrentNewVRegs are created by calling
     // selectOrSplit and should be added into NewVRegs.
-    for (SmallVectorImpl<unsigned>::iterator Next = CurrentNewVRegs.begin(),
+    for (SmallVectorImpl<Register>::iterator Next = CurrentNewVRegs.begin(),
                                              End = CurrentNewVRegs.end();
          Next != End; ++Next) {
       if (RecoloringCandidates.count(&LIS->getInterval(*Next)))
@@ -2697,10 +2697,10 @@ unsigned RAGreedy::tryLastChanceRecoloring(LiveInterval &VirtReg,
     for (SmallLISet::iterator It = RecoloringCandidates.begin(),
                               EndIt = RecoloringCandidates.end();
          It != EndIt; ++It) {
-      unsigned ItVirtReg = (*It)->reg;
+      Register ItVirtReg = (*It)->reg;
       if (VRM->hasPhys(ItVirtReg))
         Matrix->unassign(**It);
-      unsigned ItPhysReg = VirtRegToPhysReg[ItVirtReg];
+      Register ItPhysReg = VirtRegToPhysReg[ItVirtReg];
       Matrix->assign(**It, ItPhysReg);
     }
   }
@@ -2718,14 +2718,14 @@ unsigned RAGreedy::tryLastChanceRecoloring(LiveInterval &VirtReg,
 /// \return true if all virtual registers in RecoloringQueue were successfully
 /// recolored, false otherwise.
 bool RAGreedy::tryRecoloringCandidates(PQueue &RecoloringQueue,
-                                       SmallVectorImpl<unsigned> &NewVRegs,
+                                       SmallVectorImpl<Register> &NewVRegs,
                                        SmallVirtRegSet &FixedRegisters,
                                        unsigned Depth) {
   while (!RecoloringQueue.empty()) {
     LiveInterval *LI = dequeue(RecoloringQueue);
     LLVM_DEBUG(dbgs() << "Try to recolor: " << *LI << '\n');
-    unsigned PhysReg;
-    PhysReg = selectOrSplitImpl(*LI, NewVRegs, FixedRegisters, Depth + 1);
+    Register PhysReg = selectOrSplitImpl(*LI, NewVRegs, FixedRegisters,
+                                         Depth + 1);
     // When splitting happens, the live-range may actually be empty.
     // In that case, this is okay to continue the recoloring even
     // if we did not find an alternative color for it. Indeed,
@@ -2752,12 +2752,12 @@ bool RAGreedy::tryRecoloringCandidates(PQueue &RecoloringQueue,
 //                            Main Entry Point
 //===----------------------------------------------------------------------===//
 
-unsigned RAGreedy::selectOrSplit(LiveInterval &VirtReg,
-                                 SmallVectorImpl<unsigned> &NewVRegs) {
+Register RAGreedy::selectOrSplit(LiveInterval &VirtReg,
+                                 SmallVectorImpl<Register> &NewVRegs) {
   CutOffInfo = CO_None;
   LLVMContext &Ctx = MF->getFunction().getContext();
   SmallVirtRegSet FixedRegisters;
-  unsigned Reg = selectOrSplitImpl(VirtReg, NewVRegs, FixedRegisters);
+  Register Reg = selectOrSplitImpl(VirtReg, NewVRegs, FixedRegisters);
   if (Reg == ~0U && (CutOffInfo != CO_None)) {
     uint8_t CutOffEncountered = CutOffInfo & (CO_Depth | CO_Interf);
     if (CutOffEncountered == CO_Depth)
@@ -2784,9 +2784,9 @@ unsigned RAGreedy::selectOrSplit(LiveInterval &VirtReg,
 /// to use the CSR; otherwise return 0.
 unsigned RAGreedy::tryAssignCSRFirstTime(LiveInterval &VirtReg,
                                          AllocationOrder &Order,
-                                         unsigned PhysReg,
+                                         Register PhysReg,
                                          unsigned &CostPerUseLimit,
-                                         SmallVectorImpl<unsigned> &NewVRegs) {
+                                         SmallVectorImpl<Register> &NewVRegs) {
   if (getStage(VirtReg) == RS_Spill && VirtReg.isSpillable()) {
     // We choose spill over using the CSR for the first time if the spill cost
     // is lower than CSRCost.
@@ -3013,8 +3013,8 @@ void RAGreedy::tryHintsRecoloring() {
   }
 }
 
-unsigned RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg,
-                                     SmallVectorImpl<unsigned> &NewVRegs,
+Register RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg,
+                                     SmallVectorImpl<Register> &NewVRegs,
                                      SmallVirtRegSet &FixedRegisters,
                                      unsigned Depth) {
   unsigned CostPerUseLimit = ~0u;
@@ -3028,7 +3028,7 @@ unsigned RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg,
     // register.
     if (CSRCost.getFrequency() && isUnusedCalleeSavedReg(PhysReg) &&
         NewVRegs.empty()) {
-      unsigned CSRReg = tryAssignCSRFirstTime(VirtReg, Order, PhysReg,
+      Register CSRReg = tryAssignCSRFirstTime(VirtReg, Order, PhysReg,
                                               CostPerUseLimit, NewVRegs);
       if (CSRReg || !NewVRegs.empty())
         // Return now if we decide to use a CSR or create new vregs due to
@@ -3046,10 +3046,10 @@ unsigned RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg,
   // queue. The RS_Split ranges already failed to do this, and they should not
   // get a second chance until they have been split.
   if (Stage != RS_Split)
-    if (unsigned PhysReg =
+    if (Register PhysReg =
             tryEvict(VirtReg, Order, NewVRegs, CostPerUseLimit,
                      FixedRegisters)) {
-      unsigned Hint = MRI->getSimpleHint(VirtReg.reg);
+      Register Hint = MRI->getSimpleHint(VirtReg.reg);
       // If VirtReg has a hint and that hint is broken record this
       // virtual register as a recoloring candidate for broken hint.
       // Indeed, since we evicted a variable in its neighborhood it is
@@ -3078,7 +3078,7 @@ unsigned RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg,
   if (Stage < RS_Spill) {
     // Try splitting VirtReg or interferences.
     unsigned NewVRegSizeBefore = NewVRegs.size();
-    unsigned PhysReg = trySplit(VirtReg, Order, NewVRegs, FixedRegisters);
+    Register PhysReg = trySplit(VirtReg, Order, NewVRegs, FixedRegisters);
     if (PhysReg || (NewVRegs.size() - NewVRegSizeBefore)) {
       // If VirtReg got split, the eviction info is no longer relevant.
       LastEvicted.clearEvicteeInfo(VirtReg.reg);
@@ -3147,7 +3147,6 @@ void RAGreedy::reportNumberOfSplillsReloads(MachineLoop *L, unsigned &Reloads,
   }
 
   const MachineFrameInfo &MFI = MF->getFrameInfo();
-  const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
   int FI;
 
   for (MachineBasicBlock *MBB : L->getBlocks())
diff --git a/llvm/lib/CodeGen/RegAllocPBQP.cpp b/llvm/lib/CodeGen/RegAllocPBQP.cpp
index 765bce8fc7215..7590dbf1b9774 100644
--- a/llvm/lib/CodeGen/RegAllocPBQP.cpp
+++ b/llvm/lib/CodeGen/RegAllocPBQP.cpp
@@ -166,7 +166,7 @@ class RegAllocPBQP : public MachineFunctionPass {
   void initializeGraph(PBQPRAGraph &G, VirtRegMap &VRM, Spiller &VRegSpiller);
 
   /// Spill the given VReg.
-  void spillVReg(unsigned VReg, SmallVectorImpl<unsigned> &NewIntervals,
+  void spillVReg(Register VReg, SmallVectorImpl<Register> &NewIntervals,
                  MachineFunction &MF, LiveIntervals &LIS, VirtRegMap &VRM,
                  Spiller &VRegSpiller);
 
@@ -637,7 +637,7 @@ void RegAllocPBQP::initializeGraph(PBQPRAGraph &G, VirtRegMap &VRM,
     // Check for vregs that have no allowed registers. These should be
     // pre-spilled and the new vregs added to the worklist.
     if (VRegAllowed.empty()) {
-      SmallVector<unsigned, 8> NewVRegs;
+      SmallVector<Register, 8> NewVRegs;
       spillVReg(VReg, NewVRegs, MF, LIS, VRM, VRegSpiller);
       Worklist.insert(Worklist.end(), NewVRegs.begin(), NewVRegs.end());
       continue;
@@ -673,8 +673,8 @@ void RegAllocPBQP::initializeGraph(PBQPRAGraph &G, VirtRegMap &VRM,
   }
 }
 
-void RegAllocPBQP::spillVReg(unsigned VReg,
-                             SmallVectorImpl<unsigned> &NewIntervals,
+void RegAllocPBQP::spillVReg(Register VReg,
+                             SmallVectorImpl<Register> &NewIntervals,
                              MachineFunction &MF, LiveIntervals &LIS,
                              VirtRegMap &VRM, Spiller &VRegSpiller) {
   VRegsToAlloc.erase(VReg);
@@ -730,7 +730,7 @@ bool RegAllocPBQP::mapPBQPToRegAlloc(const PBQPRAGraph &G,
     } else {
       // Spill VReg. If this introduces new intervals we'll need another round
       // of allocation.
-      SmallVector<unsigned, 8> NewVRegs;
+      SmallVector<Register, 8> NewVRegs;
       spillVReg(VReg, NewVRegs, MF, LIS, VRM, VRegSpiller);
       AnotherRoundNeeded |= !NewVRegs.empty();
     }
diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp
index e24de551b7117..17160a9f42cd5 100644
--- a/llvm/lib/CodeGen/RegisterCoalescer.cpp
+++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp
@@ -571,7 +571,7 @@ void RegisterCoalescer::getAnalysisUsage(AnalysisUsage &AU) const {
 }
 
 void RegisterCoalescer::eliminateDeadDefs() {
-  SmallVector<unsigned, 8> NewRegs;
+  SmallVector<Register, 8> NewRegs;
   LiveRangeEdit(nullptr, NewRegs, *MF, *LIS,
                 nullptr, this).eliminateDeadDefs(DeadDefs);
 }
@@ -1064,7 +1064,9 @@ bool RegisterCoalescer::removePartialRedundancy(const CoalescerPair &CP,
     return false;
 
   MachineBasicBlock &MBB = *CopyMI.getParent();
-  if (MBB.isEHPad())
+  // If this block is the target of an invoke/inlineasm_br, moving the copy into
+  // the predecessor is tricker, and we don't handle it.
+  if (MBB.isEHPad() || MBB.isInlineAsmBrIndirectTarget())
     return false;
 
   if (MBB.pred_size() != 2)
@@ -2429,7 +2431,7 @@ class JoinVals {
   /// Add foreign virtual registers to ShrinkRegs if their live range ended at
   /// the erased instrs.
   void eraseInstrs(SmallPtrSetImpl<MachineInstr*> &ErasedInstrs,
-                   SmallVectorImpl<unsigned> &ShrinkRegs,
+                   SmallVectorImpl<Register> &ShrinkRegs,
                    LiveInterval *LI = nullptr);
 
   /// Remove liverange defs at places where implicit defs will be removed.
@@ -3171,7 +3173,7 @@ void JoinVals::removeImplicitDefs() {
 }
 
 void JoinVals::eraseInstrs(SmallPtrSetImpl<MachineInstr*> &ErasedInstrs,
-                           SmallVectorImpl<unsigned> &ShrinkRegs,
+                           SmallVectorImpl<Register> &ShrinkRegs,
                            LiveInterval *LI) {
   for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
     // Get the def location before markUnused() below invalidates it.
@@ -3439,7 +3441,7 @@ bool RegisterCoalescer::joinVirtRegs(CoalescerPair &CP) {
 
   // Erase COPY and IMPLICIT_DEF instructions. This may cause some external
   // registers to require trimming.
-  SmallVector<unsigned, 8> ShrinkRegs;
+  SmallVector<Register, 8> ShrinkRegs;
   LHSVals.eraseInstrs(ErasedInstrs, ShrinkRegs, &LHS);
   RHSVals.eraseInstrs(ErasedInstrs, ShrinkRegs);
   while (!ShrinkRegs.empty())
@@ -3488,7 +3490,7 @@ void RegisterCoalescer::buildVRegToDbgValueMap(MachineFunction &MF)
   // vreg => DbgValueLoc map.
   auto CloseNewDVRange = [this, &ToInsert](SlotIndex Slot) {
     for (auto *X : ToInsert)
-      DbgVRegToValues[X->getOperand(0).getReg()].push_back({Slot, X});
+      DbgVRegToValues[X->getDebugOperand(0).getReg()].push_back({Slot, X});
 
     ToInsert.clear();
   };
@@ -3500,8 +3502,8 @@ void RegisterCoalescer::buildVRegToDbgValueMap(MachineFunction &MF)
     SlotIndex CurrentSlot = Slots.getMBBStartIdx(&MBB);
 
     for (auto &MI : MBB) {
-      if (MI.isDebugValue() && MI.getOperand(0).isReg() &&
-          MI.getOperand(0).getReg().isVirtual()) {
+      if (MI.isDebugValue() && MI.getDebugOperand(0).isReg() &&
+          MI.getDebugOperand(0).getReg().isVirtual()) {
         ToInsert.push_back(&MI);
       } else if (!MI.isDebugInstr()) {
         CurrentSlot = Slots.getInstructionIndex(MI);
@@ -3600,10 +3602,10 @@ void RegisterCoalescer::checkMergingChangesDbgValuesImpl(unsigned Reg,
       // "Other" is live and there is a DBG_VALUE of Reg: test if we should
       // set it undef.
       if (DbgValueSetIt->first >= SegmentIt->start &&
-          DbgValueSetIt->second->getOperand(0).getReg() != 0 &&
+          DbgValueSetIt->second->getDebugOperand(0).getReg() != 0 &&
           ShouldUndef(DbgValueSetIt->first)) {
         // Mark undef, erase record of this DBG_VALUE to avoid revisiting.
-        DbgValueSetIt->second->getOperand(0).setReg(0);
+        DbgValueSetIt->second->setDebugValueUndef();
         continue;
       }
       ++DbgValueSetIt;
diff --git a/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
index d091f996df178..c93b296174384 100644
--- a/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
@@ -83,7 +83,7 @@ static bool isConstantIntVector(Value *Mask) {
   if (!C)
     return false;
 
-  unsigned NumElts = cast<VectorType>(Mask->getType())->getNumElements();
+  unsigned NumElts = cast<FixedVectorType>(Mask->getType())->getNumElements();
   for (unsigned i = 0; i != NumElts; ++i) {
     Constant *CElt = C->getAggregateElement(i);
     if (!CElt || !isa<ConstantInt>(CElt))
@@ -132,7 +132,7 @@ static void scalarizeMaskedLoad(CallInst *CI, bool &ModifiedDT) {
   Value *Src0 = CI->getArgOperand(3);
 
   const Align AlignVal = cast<ConstantInt>(Alignment)->getAlignValue();
-  VectorType *VecType = cast<VectorType>(CI->getType());
+  VectorType *VecType = cast<FixedVectorType>(CI->getType());
 
   Type *EltTy = VecType->getElementType();
 
@@ -158,7 +158,7 @@ static void scalarizeMaskedLoad(CallInst *CI, bool &ModifiedDT) {
   Type *NewPtrType =
       EltTy->getPointerTo(Ptr->getType()->getPointerAddressSpace());
   Value *FirstEltPtr = Builder.CreateBitCast(Ptr, NewPtrType);
-  unsigned VectorWidth = VecType->getNumElements();
+  unsigned VectorWidth = cast<FixedVectorType>(VecType)->getNumElements();
 
   // The result vector
   Value *VResult = Src0;
@@ -271,7 +271,7 @@ static void scalarizeMaskedStore(CallInst *CI, bool &ModifiedDT) {
   Value *Mask = CI->getArgOperand(3);
 
   const Align AlignVal = cast<ConstantInt>(Alignment)->getAlignValue();
-  VectorType *VecType = cast<VectorType>(Src->getType());
+  auto *VecType = cast<VectorType>(Src->getType());
 
   Type *EltTy = VecType->getElementType();
 
@@ -295,7 +295,7 @@ static void scalarizeMaskedStore(CallInst *CI, bool &ModifiedDT) {
   Type *NewPtrType =
       EltTy->getPointerTo(Ptr->getType()->getPointerAddressSpace());
   Value *FirstEltPtr = Builder.CreateBitCast(Ptr, NewPtrType);
-  unsigned VectorWidth = VecType->getNumElements();
+  unsigned VectorWidth = cast<FixedVectorType>(VecType)->getNumElements();
 
   if (isConstantIntVector(Mask)) {
     for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
@@ -396,14 +396,14 @@ static void scalarizeMaskedGather(CallInst *CI, bool &ModifiedDT) {
   Value *Mask = CI->getArgOperand(2);
   Value *Src0 = CI->getArgOperand(3);
 
-  VectorType *VecType = cast<VectorType>(CI->getType());
+  auto *VecType = cast<FixedVectorType>(CI->getType());
   Type *EltTy = VecType->getElementType();
 
   IRBuilder<> Builder(CI->getContext());
   Instruction *InsertPt = CI;
   BasicBlock *IfBlock = CI->getParent();
   Builder.SetInsertPoint(InsertPt);
-  unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue();
+  MaybeAlign AlignVal = cast<ConstantInt>(Alignment)->getMaybeAlignValue();
 
   Builder.SetCurrentDebugLocation(CI->getDebugLoc());
 
@@ -417,8 +417,8 @@ static void scalarizeMaskedGather(CallInst *CI, bool &ModifiedDT) {
       if (cast<Constant>(Mask)->getAggregateElement(Idx)->isNullValue())
         continue;
       Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
-      LoadInst *Load = Builder.CreateAlignedLoad(
-          EltTy, Ptr, MaybeAlign(AlignVal), "Load" + Twine(Idx));
+      LoadInst *Load =
+          Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
       VResult =
           Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
     }
@@ -462,8 +462,8 @@ static void scalarizeMaskedGather(CallInst *CI, bool &ModifiedDT) {
     Builder.SetInsertPoint(InsertPt);
 
     Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
-    LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Ptr, MaybeAlign(AlignVal),
-                                               "Load" + Twine(Idx));
+    LoadInst *Load =
+        Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
     Value *NewVResult =
         Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
 
@@ -520,8 +520,8 @@ static void scalarizeMaskedScatter(CallInst *CI, bool &ModifiedDT) {
   Value *Alignment = CI->getArgOperand(2);
   Value *Mask = CI->getArgOperand(3);
 
-  assert(isa<VectorType>(Src->getType()) &&
-         "Unexpected data type in masked scatter intrinsic");
+  auto *SrcFVTy = cast<FixedVectorType>(Src->getType());
+
   assert(
       isa<VectorType>(Ptrs->getType()) &&
       isa<PointerType>(cast<VectorType>(Ptrs->getType())->getElementType()) &&
@@ -533,8 +533,8 @@ static void scalarizeMaskedScatter(CallInst *CI, bool &ModifiedDT) {
   Builder.SetInsertPoint(InsertPt);
   Builder.SetCurrentDebugLocation(CI->getDebugLoc());
 
-  MaybeAlign AlignVal(cast<ConstantInt>(Alignment)->getZExtValue());
-  unsigned VectorWidth = cast<VectorType>(Src->getType())->getNumElements();
+  MaybeAlign AlignVal = cast<ConstantInt>(Alignment)->getMaybeAlignValue();
+  unsigned VectorWidth = SrcFVTy->getNumElements();
 
   // Shorten the way if the mask is a vector of constants.
   if (isConstantIntVector(Mask)) {
@@ -605,7 +605,7 @@ static void scalarizeMaskedExpandLoad(CallInst *CI, bool &ModifiedDT) {
   Value *Mask = CI->getArgOperand(1);
   Value *PassThru = CI->getArgOperand(2);
 
-  VectorType *VecType = cast<VectorType>(CI->getType());
+  auto *VecType = cast<FixedVectorType>(CI->getType());
 
   Type *EltTy = VecType->getElementType();
 
@@ -718,7 +718,7 @@ static void scalarizeMaskedCompressStore(CallInst *CI, bool &ModifiedDT) {
   Value *Ptr = CI->getArgOperand(1);
   Value *Mask = CI->getArgOperand(2);
 
-  VectorType *VecType = cast<VectorType>(Src->getType());
+  auto *VecType = cast<FixedVectorType>(Src->getType());
 
   IRBuilder<> Builder(CI->getContext());
   Instruction *InsertPt = CI;
diff --git a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
index edc5c4bdda3dc..10da2d4217977 100644
--- a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
+++ b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
@@ -18,6 +18,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/SparseSet.h"
 #include "llvm/ADT/iterator_range.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/CodeGen/LiveIntervals.h"
 #include "llvm/CodeGen/LivePhysRegs.h"
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 01a771ebe8a7e..42e6e12f3f027 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -455,6 +455,7 @@ namespace {
     SDValue visitZERO_EXTEND(SDNode *N);
     SDValue visitANY_EXTEND(SDNode *N);
     SDValue visitAssertExt(SDNode *N);
+    SDValue visitAssertAlign(SDNode *N);
     SDValue visitSIGN_EXTEND_INREG(SDNode *N);
     SDValue visitSIGN_EXTEND_VECTOR_INREG(SDNode *N);
     SDValue visitZERO_EXTEND_VECTOR_INREG(SDNode *N);
@@ -637,6 +638,19 @@ namespace {
           : MemNode(N), OffsetFromBase(Offset) {}
     };
 
+    // Classify the origin of a stored value.
+    enum class StoreSource { Unknown, Constant, Extract, Load };
+    StoreSource getStoreSource(SDValue StoreVal) {
+      if (isa<ConstantSDNode>(StoreVal) || isa<ConstantFPSDNode>(StoreVal))
+        return StoreSource::Constant;
+      if (StoreVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
+          StoreVal.getOpcode() == ISD::EXTRACT_SUBVECTOR)
+        return StoreSource::Extract;
+      if (isa<LoadSDNode>(StoreVal))
+        return StoreSource::Load;
+      return StoreSource::Unknown;
+    }
+
     /// This is a helper function for visitMUL to check the profitability
     /// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
     /// MulNode is the original multiply, AddNode is (add x, c1),
@@ -664,43 +678,66 @@ namespace {
     /// can be combined into narrow loads.
     bool BackwardsPropagateMask(SDNode *N);
 
-    /// Helper function for MergeConsecutiveStores which merges the
-    /// component store chains.
+    /// Helper function for mergeConsecutiveStores which merges the component
+    /// store chains.
     SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
                                 unsigned NumStores);
 
-    /// This is a helper function for MergeConsecutiveStores. When the
-    /// source elements of the consecutive stores are all constants or
-    /// all extracted vector elements, try to merge them into one
-    /// larger store introducing bitcasts if necessary.  \return True
-    /// if a merged store was created.
-    bool MergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
+    /// This is a helper function for mergeConsecutiveStores. When the source
+    /// elements of the consecutive stores are all constants or all extracted
+    /// vector elements, try to merge them into one larger store introducing
+    /// bitcasts if necessary.  \return True if a merged store was created.
+    bool mergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
                                          EVT MemVT, unsigned NumStores,
                                          bool IsConstantSrc, bool UseVector,
                                          bool UseTrunc);
 
-    /// This is a helper function for MergeConsecutiveStores. Stores
-    /// that potentially may be merged with St are placed in
-    /// StoreNodes. RootNode is a chain predecessor to all store
-    /// candidates.
+    /// This is a helper function for mergeConsecutiveStores. Stores that
+    /// potentially may be merged with St are placed in StoreNodes. RootNode is
+    /// a chain predecessor to all store candidates.
     void getStoreMergeCandidates(StoreSDNode *St,
                                  SmallVectorImpl<MemOpLink> &StoreNodes,
                                  SDNode *&Root);
 
-    /// Helper function for MergeConsecutiveStores. Checks if
-    /// candidate stores have indirect dependency through their
-    /// operands. RootNode is the predecessor to all stores calculated
-    /// by getStoreMergeCandidates and is used to prune the dependency check.
-    /// \return True if safe to merge.
+    /// Helper function for mergeConsecutiveStores. Checks if candidate stores
+    /// have indirect dependency through their operands. RootNode is the
+    /// predecessor to all stores calculated by getStoreMergeCandidates and is
+    /// used to prune the dependency check. \return True if safe to merge.
     bool checkMergeStoreCandidatesForDependencies(
         SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores,
         SDNode *RootNode);
 
+    /// This is a helper function for mergeConsecutiveStores. Given a list of
+    /// store candidates, find the first N that are consecutive in memory.
+    /// Returns 0 if there are not at least 2 consecutive stores to try merging.
+    unsigned getConsecutiveStores(SmallVectorImpl<MemOpLink> &StoreNodes,
+                                  int64_t ElementSizeBytes) const;
+
+    /// This is a helper function for mergeConsecutiveStores. It is used for
+    /// store chains that are composed entirely of constant values.
+    bool tryStoreMergeOfConstants(SmallVectorImpl<MemOpLink> &StoreNodes,
+                                  unsigned NumConsecutiveStores,
+                                  EVT MemVT, SDNode *Root, bool AllowVectors);
+
+    /// This is a helper function for mergeConsecutiveStores. It is used for
+    /// store chains that are composed entirely of extracted vector elements.
+    /// When extracting multiple vector elements, try to store them in one
+    /// vector store rather than a sequence of scalar stores.
+    bool tryStoreMergeOfExtracts(SmallVectorImpl<MemOpLink> &StoreNodes,
+                                 unsigned NumConsecutiveStores, EVT MemVT,
+                                 SDNode *Root);
+
+    /// This is a helper function for mergeConsecutiveStores. It is used for
+    /// store chains that are composed entirely of loaded values.
+    bool tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
+                              unsigned NumConsecutiveStores, EVT MemVT,
+                              SDNode *Root, bool AllowVectors,
+                              bool IsNonTemporalStore, bool IsNonTemporalLoad);
+
     /// Merge consecutive store operations into a wide store.
     /// This optimization uses wide integers or vectors when possible.
-    /// \return number of stores that were merged into a merged store (the
-    /// affected nodes are stored as a prefix in \p StoreNodes).
-    bool MergeConsecutiveStores(StoreSDNode *St);
+    /// \return true if stores were merged.
+    bool mergeConsecutiveStores(StoreSDNode *St);
 
     /// Try to transform a truncation where C is a constant:
     ///     (trunc (and X, C)) -> (and (trunc X), (trunc C))
@@ -1601,6 +1638,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
   case ISD::ANY_EXTEND:         return visitANY_EXTEND(N);
   case ISD::AssertSext:
   case ISD::AssertZext:         return visitAssertExt(N);
+  case ISD::AssertAlign:        return visitAssertAlign(N);
   case ISD::SIGN_EXTEND_INREG:  return visitSIGN_EXTEND_INREG(N);
   case ISD::SIGN_EXTEND_VECTOR_INREG: return visitSIGN_EXTEND_VECTOR_INREG(N);
   case ISD::ZERO_EXTEND_VECTOR_INREG: return visitZERO_EXTEND_VECTOR_INREG(N);
@@ -2369,6 +2407,16 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
     return DAG.getVScale(DL, VT, C0 + C1);
   }
 
+  // fold a+vscale(c1)+vscale(c2) -> a+vscale(c1+c2)
+  if ((N0.getOpcode() == ISD::ADD) &&
+      (N0.getOperand(1).getOpcode() == ISD::VSCALE) &&
+      (N1.getOpcode() == ISD::VSCALE)) {
+    auto VS0 = N0.getOperand(1)->getConstantOperandAPInt(0);
+    auto VS1 = N1->getConstantOperandAPInt(0);
+    auto VS = DAG.getVScale(DL, VT, VS0 + VS1);
+    return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), VS);
+  }
+
   return SDValue();
 }
 
@@ -4873,11 +4921,16 @@ bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST,
     return false;
 
   // Ensure that this isn't going to produce an unsupported memory access.
-  if (ShAmt &&
-      !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
-                              LDST->getAddressSpace(), ShAmt / 8,
-                              LDST->getMemOperand()->getFlags()))
-    return false;
+  if (ShAmt) {
+    assert(ShAmt % 8 == 0 && "ShAmt is byte offset");
+    const unsigned ByteShAmt = ShAmt / 8;
+    const Align LDSTAlign = LDST->getAlign();
+    const Align NarrowAlign = commonAlignment(LDSTAlign, ByteShAmt);
+    if (!TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
+                                LDST->getAddressSpace(), NarrowAlign,
+                                LDST->getMemOperand()->getFlags()))
+      return false;
+  }
 
   // It's not possible to generate a constant of extended or untyped type.
   EVT PtrType = LDST->getBasePtr().getValueType();
@@ -8491,7 +8544,7 @@ SDValue DAGCombiner::visitFunnelShift(SDNode *N) {
           SDLoc DL(RHS);
           uint64_t PtrOff =
               IsFSHL ? (((BitWidth - ShAmt) % BitWidth) / 8) : (ShAmt / 8);
-          unsigned NewAlign = MinAlign(RHS->getAlignment(), PtrOff);
+          Align NewAlign = commonAlignment(RHS->getAlign(), PtrOff);
           bool Fast = false;
           if (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
                                      RHS->getAddressSpace(), NewAlign,
@@ -10696,6 +10749,45 @@ SDValue DAGCombiner::visitAssertExt(SDNode *N) {
   return SDValue();
 }
 
+SDValue DAGCombiner::visitAssertAlign(SDNode *N) {
+  SDLoc DL(N);
+
+  Align AL = cast<AssertAlignSDNode>(N)->getAlign();
+  SDValue N0 = N->getOperand(0);
+
+  // Fold (assertalign (assertalign x, AL0), AL1) ->
+  // (assertalign x, max(AL0, AL1))
+  if (auto *AAN = dyn_cast<AssertAlignSDNode>(N0))
+    return DAG.getAssertAlign(DL, N0.getOperand(0),
+                              std::max(AL, AAN->getAlign()));
+
+  // In rare cases, there are trivial arithmetic ops in source operands. Sink
+  // this assert down to source operands so that those arithmetic ops could be
+  // exposed to the DAG combining.
+  switch (N0.getOpcode()) {
+  default:
+    break;
+  case ISD::ADD:
+  case ISD::SUB: {
+    unsigned AlignShift = Log2(AL);
+    SDValue LHS = N0.getOperand(0);
+    SDValue RHS = N0.getOperand(1);
+    unsigned LHSAlignShift = DAG.computeKnownBits(LHS).countMinTrailingZeros();
+    unsigned RHSAlignShift = DAG.computeKnownBits(RHS).countMinTrailingZeros();
+    if (LHSAlignShift >= AlignShift || RHSAlignShift >= AlignShift) {
+      if (LHSAlignShift < AlignShift)
+        LHS = DAG.getAssertAlign(DL, LHS, AL);
+      if (RHSAlignShift < AlignShift)
+        RHS = DAG.getAssertAlign(DL, RHS, AL);
+      return DAG.getNode(N0.getOpcode(), DL, N0.getValueType(), LHS, RHS);
+    }
+    break;
+  }
+  }
+
+  return SDValue();
+}
+
 /// If the result of a wider load is shifted to right of N  bits and then
 /// truncated to a narrower type and where N is a multiple of number of bits of
 /// the narrower type, transform it to a narrower load from address + N / num of
@@ -10907,26 +10999,27 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   EVT VT = N->getValueType(0);
-  EVT EVT = cast<VTSDNode>(N1)->getVT();
+  EVT ExtVT = cast<VTSDNode>(N1)->getVT();
   unsigned VTBits = VT.getScalarSizeInBits();
-  unsigned EVTBits = EVT.getScalarSizeInBits();
+  unsigned ExtVTBits = ExtVT.getScalarSizeInBits();
 
+  // sext_vector_inreg(undef) = 0 because the top bit will all be the same.
   if (N0.isUndef())
-    return DAG.getUNDEF(VT);
+    return DAG.getConstant(0, SDLoc(N), VT);
 
   // fold (sext_in_reg c1) -> c1
   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
     return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1);
 
   // If the input is already sign extended, just drop the extension.
-  if (DAG.ComputeNumSignBits(N0) >= VTBits-EVTBits+1)
+  if (DAG.ComputeNumSignBits(N0) >= (VTBits - ExtVTBits + 1))
     return N0;
 
   // fold (sext_in_reg (sext_in_reg x, VT2), VT1) -> (sext_in_reg x, minVT) pt2
   if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
-      EVT.bitsLT(cast<VTSDNode>(N0.getOperand(1))->getVT()))
-    return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
-                       N0.getOperand(0), N1);
+      ExtVT.bitsLT(cast<VTSDNode>(N0.getOperand(1))->getVT()))
+    return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0.getOperand(0),
+                       N1);
 
   // fold (sext_in_reg (sext x)) -> (sext x)
   // fold (sext_in_reg (aext x)) -> (sext x)
@@ -10935,8 +11028,8 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
   if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) {
     SDValue N00 = N0.getOperand(0);
     unsigned N00Bits = N00.getScalarValueSizeInBits();
-    if ((N00Bits <= EVTBits ||
-         (N00Bits - DAG.ComputeNumSignBits(N00)) < EVTBits) &&
+    if ((N00Bits <= ExtVTBits ||
+         (N00Bits - DAG.ComputeNumSignBits(N00)) < ExtVTBits) &&
         (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
       return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00);
   }
@@ -10945,7 +11038,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
   if ((N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG ||
        N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG ||
        N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) &&
-      N0.getOperand(0).getScalarValueSizeInBits() == EVTBits) {
+      N0.getOperand(0).getScalarValueSizeInBits() == ExtVTBits) {
     if (!LegalOperations ||
         TLI.isOperationLegal(ISD::SIGN_EXTEND_VECTOR_INREG, VT))
       return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT,
@@ -10956,14 +11049,14 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
   // iff we are extending the source sign bit.
   if (N0.getOpcode() == ISD::ZERO_EXTEND) {
     SDValue N00 = N0.getOperand(0);
-    if (N00.getScalarValueSizeInBits() == EVTBits &&
+    if (N00.getScalarValueSizeInBits() == ExtVTBits &&
         (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
       return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1);
   }
 
   // fold (sext_in_reg x) -> (zext_in_reg x) if the sign bit is known zero.
-  if (DAG.MaskedValueIsZero(N0, APInt::getOneBitSet(VTBits, EVTBits - 1)))
-    return DAG.getZeroExtendInReg(N0, SDLoc(N), EVT);
+  if (DAG.MaskedValueIsZero(N0, APInt::getOneBitSet(VTBits, ExtVTBits - 1)))
+    return DAG.getZeroExtendInReg(N0, SDLoc(N), ExtVT);
 
   // fold operands of sext_in_reg based on knowledge that the top bits are not
   // demanded.
@@ -10980,11 +11073,11 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
   // We already fold "(sext_in_reg (srl X, 25), i8) -> srl X, 25" above.
   if (N0.getOpcode() == ISD::SRL) {
     if (auto *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1)))
-      if (ShAmt->getAPIntValue().ule(VTBits - EVTBits)) {
+      if (ShAmt->getAPIntValue().ule(VTBits - ExtVTBits)) {
         // We can turn this into an SRA iff the input to the SRL is already sign
         // extended enough.
         unsigned InSignBits = DAG.ComputeNumSignBits(N0.getOperand(0));
-        if (((VTBits - EVTBits) - ShAmt->getZExtValue()) < InSignBits)
+        if (((VTBits - ExtVTBits) - ShAmt->getZExtValue()) < InSignBits)
           return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0.getOperand(0),
                              N0.getOperand(1));
       }
@@ -10996,14 +11089,14 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
   // extends that the target does support.
   if (ISD::isEXTLoad(N0.getNode()) &&
       ISD::isUNINDEXEDLoad(N0.getNode()) &&
-      EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
+      ExtVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
       ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple() &&
         N0.hasOneUse()) ||
-       TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, EVT))) {
+       TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) {
     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
     SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
                                      LN0->getChain(),
-                                     LN0->getBasePtr(), EVT,
+                                     LN0->getBasePtr(), ExtVT,
                                      LN0->getMemOperand());
     CombineTo(N, ExtLoad);
     CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
@@ -11013,13 +11106,13 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
   // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use
   if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
       N0.hasOneUse() &&
-      EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
+      ExtVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
       ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple()) &&
-       TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, EVT))) {
+       TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) {
     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
     SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
                                      LN0->getChain(),
-                                     LN0->getBasePtr(), EVT,
+                                     LN0->getBasePtr(), ExtVT,
                                      LN0->getMemOperand());
     CombineTo(N, ExtLoad);
     CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
@@ -11027,11 +11120,10 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
   }
 
   // Form (sext_inreg (bswap >> 16)) or (sext_inreg (rotl (bswap) 16))
-  if (EVTBits <= 16 && N0.getOpcode() == ISD::OR) {
+  if (ExtVTBits <= 16 && N0.getOpcode() == ISD::OR) {
     if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
                                            N0.getOperand(1), false))
-      return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
-                         BSwap, N1);
+      return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, BSwap, N1);
   }
 
   return SDValue();
@@ -11041,8 +11133,9 @@ SDValue DAGCombiner::visitSIGN_EXTEND_VECTOR_INREG(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
 
+  // sext_vector_inreg(undef) = 0 because the top bit will all be the same.
   if (N0.isUndef())
-    return DAG.getUNDEF(VT);
+    return DAG.getConstant(0, SDLoc(N), VT);
 
   if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
     return Res;
@@ -11057,8 +11150,9 @@ SDValue DAGCombiner::visitZERO_EXTEND_VECTOR_INREG(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
 
+  // zext_vector_inreg(undef) = 0 because the top bits will be zero.
   if (N0.isUndef())
-    return DAG.getUNDEF(VT);
+    return DAG.getConstant(0, SDLoc(N), VT);
 
   if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
     return Res;
@@ -11410,14 +11504,14 @@ SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
   unsigned LD1Bytes = LD1VT.getStoreSize();
   if (ISD::isNON_EXTLoad(LD2) && LD2->hasOneUse() &&
       DAG.areNonVolatileConsecutiveLoads(LD2, LD1, LD1Bytes, 1)) {
-    unsigned Align = LD1->getAlignment();
-    unsigned NewAlign = DAG.getDataLayout().getABITypeAlignment(
+    Align Alignment = LD1->getAlign();
+    Align NewAlign = DAG.getDataLayout().getABITypeAlign(
         VT.getTypeForEVT(*DAG.getContext()));
 
-    if (NewAlign <= Align &&
+    if (NewAlign <= Alignment &&
         (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)))
       return DAG.getLoad(VT, SDLoc(N), LD1->getChain(), LD1->getBasePtr(),
-                         LD1->getPointerInfo(), Align);
+                         LD1->getPointerInfo(), Alignment);
   }
 
   return SDValue();
@@ -11892,6 +11986,8 @@ SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
 
   SDNodeFlags Flags = N->getFlags();
   bool CanFuse = Options.UnsafeFPMath || isContractable(N);
+  bool CanReassociate =
+      Options.UnsafeFPMath || N->getFlags().hasAllowReassociation();
   bool AllowFusionGlobally = (Options.AllowFPOpFusion == FPOpFusion::Fast ||
                               CanFuse || HasFMAD);
   // If the addition is not contractable, do not combine.
@@ -11934,13 +12030,14 @@ SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
 
   // fadd (fma A, B, (fmul C, D)), E --> fma A, B, (fma C, D, E)
   // fadd E, (fma A, B, (fmul C, D)) --> fma A, B, (fma C, D, E)
+  // This requires reassociation because it changes the order of operations.
   SDValue FMA, E;
-  if (CanFuse && N0.getOpcode() == PreferredFusedOpcode &&
+  if (CanReassociate && N0.getOpcode() == PreferredFusedOpcode &&
       N0.getOperand(2).getOpcode() == ISD::FMUL && N0.hasOneUse() &&
       N0.getOperand(2).hasOneUse()) {
     FMA = N0;
     E = N1;
-  } else if (CanFuse && N1.getOpcode() == PreferredFusedOpcode &&
+  } else if (CanReassociate && N1.getOpcode() == PreferredFusedOpcode &&
              N1.getOperand(2).getOpcode() == ISD::FMUL && N1.hasOneUse() &&
              N1.getOperand(2).hasOneUse()) {
     FMA = N1;
@@ -13175,22 +13272,39 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
     } else if (N1.getOpcode() == ISD::FMUL) {
       // Look through an FMUL. Even though this won't remove the FDIV directly,
       // it's still worthwhile to get rid of the FSQRT if possible.
-      SDValue SqrtOp;
-      SDValue OtherOp;
+      SDValue Sqrt, Y;
       if (N1.getOperand(0).getOpcode() == ISD::FSQRT) {
-        SqrtOp = N1.getOperand(0);
-        OtherOp = N1.getOperand(1);
+        Sqrt = N1.getOperand(0);
+        Y = N1.getOperand(1);
       } else if (N1.getOperand(1).getOpcode() == ISD::FSQRT) {
-        SqrtOp = N1.getOperand(1);
-        OtherOp = N1.getOperand(0);
+        Sqrt = N1.getOperand(1);
+        Y = N1.getOperand(0);
       }
-      if (SqrtOp.getNode()) {
+      if (Sqrt.getNode()) {
+        // If the other multiply operand is known positive, pull it into the
+        // sqrt. That will eliminate the division if we convert to an estimate:
+        // X / (fabs(A) * sqrt(Z)) --> X / sqrt(A*A*Z) --> X * rsqrt(A*A*Z)
+        // TODO: Also fold the case where A == Z (fabs is missing).
+        if (Flags.hasAllowReassociation() && N1.hasOneUse() &&
+            N1->getFlags().hasAllowReassociation() && Sqrt.hasOneUse() &&
+            Y.getOpcode() == ISD::FABS && Y.hasOneUse()) {
+          SDValue AA = DAG.getNode(ISD::FMUL, DL, VT, Y.getOperand(0),
+                                   Y.getOperand(0), Flags);
+          SDValue AAZ =
+              DAG.getNode(ISD::FMUL, DL, VT, AA, Sqrt.getOperand(0), Flags);
+          if (SDValue Rsqrt = buildRsqrtEstimate(AAZ, Flags))
+            return DAG.getNode(ISD::FMUL, DL, VT, N0, Rsqrt, Flags);
+
+          // Estimate creation failed. Clean up speculatively created nodes.
+          recursivelyDeleteUnusedNodes(AAZ.getNode());
+        }
+
         // We found a FSQRT, so try to make this fold:
-        // x / (y * sqrt(z)) -> x * (rsqrt(z) / y)
-        if (SDValue RV = buildRsqrtEstimate(SqrtOp.getOperand(0), Flags)) {
-          RV = DAG.getNode(ISD::FDIV, SDLoc(N1), VT, RV, OtherOp, Flags);
-          AddToWorklist(RV.getNode());
-          return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags);
+        // X / (Y * sqrt(Z)) -> X * (rsqrt(Z) / Y)
+        if (SDValue Rsqrt = buildRsqrtEstimate(Sqrt.getOperand(0), Flags)) {
+          SDValue Div = DAG.getNode(ISD::FDIV, SDLoc(N1), VT, Rsqrt, Y, Flags);
+          AddToWorklist(Div.getNode());
+          return DAG.getNode(ISD::FMUL, DL, VT, N0, Div, Flags);
         }
       }
     }
@@ -14967,11 +15081,11 @@ struct LoadedSlice {
   }
 
   /// Get the alignment of the load used for this slice.
-  unsigned getAlignment() const {
-    unsigned Alignment = Origin->getAlignment();
+  Align getAlign() const {
+    Align Alignment = Origin->getAlign();
     uint64_t Offset = getOffsetFromBase();
     if (Offset != 0)
-      Alignment = MinAlign(Alignment, Alignment + Offset);
+      Alignment = commonAlignment(Alignment, Alignment.value() + Offset);
     return Alignment;
   }
 
@@ -15067,8 +15181,8 @@ struct LoadedSlice {
     // Create the load for the slice.
     SDValue LastInst =
         DAG->getLoad(SliceType, SDLoc(Origin), Origin->getChain(), BaseAddr,
-                     Origin->getPointerInfo().getWithOffset(Offset),
-                     getAlignment(), Origin->getMemOperand()->getFlags());
+                     Origin->getPointerInfo().getWithOffset(Offset), getAlign(),
+                     Origin->getMemOperand()->getFlags());
     // If the final type is not the same as the loaded type, this means that
     // we have to pad with zero. Create a zero extend for that.
     EVT FinalType = Inst->getValueType(0);
@@ -15109,10 +15223,10 @@ struct LoadedSlice {
 
     // Check if it will be merged with the load.
     // 1. Check the alignment constraint.
-    unsigned RequiredAlignment = DAG->getDataLayout().getABITypeAlignment(
+    Align RequiredAlignment = DAG->getDataLayout().getABITypeAlign(
         ResVT.getTypeForEVT(*DAG->getContext()));
 
-    if (RequiredAlignment > getAlignment())
+    if (RequiredAlignment > getAlign())
       return false;
 
     // 2. Check that the load is a legal operation for that type.
@@ -15198,14 +15312,14 @@ static void adjustCostForPairing(SmallVectorImpl<LoadedSlice> &LoadedSlices,
       continue;
 
     // Check if the target supplies paired loads for this type.
-    unsigned RequiredAlignment = 0;
+    Align RequiredAlignment;
     if (!TLI.hasPairedLoad(LoadedType, RequiredAlignment)) {
       // move to the next pair, this type is hopeless.
       Second = nullptr;
       continue;
     }
     // Check if we meet the alignment requirement.
-    if (RequiredAlignment > First->getAlignment())
+    if (First->getAlign() < RequiredAlignment)
       continue;
 
     // Check that both loads are next to each other in memory.
@@ -15600,9 +15714,9 @@ SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) {
       if (DAG.getDataLayout().isBigEndian())
         PtrOff = (BitWidth + 7 - NewBW) / 8 - PtrOff;
 
-      unsigned NewAlign = MinAlign(LD->getAlignment(), PtrOff);
+      Align NewAlign = commonAlignment(LD->getAlign(), PtrOff);
       Type *NewVTTy = NewVT.getTypeForEVT(*DAG.getContext());
-      if (NewAlign < DAG.getDataLayout().getABITypeAlignment(NewVTTy))
+      if (NewAlign < DAG.getDataLayout().getABITypeAlign(NewVTTy))
         return SDValue();
 
       SDValue NewPtr = DAG.getMemBasePlusOffset(Ptr, PtrOff, SDLoc(LD));
@@ -15655,10 +15769,10 @@ SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) {
         !TLI.isDesirableToTransformToIntegerOp(ISD::STORE, VT))
       return SDValue();
 
-    unsigned LDAlign = LD->getAlignment();
-    unsigned STAlign = ST->getAlignment();
+    Align LDAlign = LD->getAlign();
+    Align STAlign = ST->getAlign();
     Type *IntVTTy = IntVT.getTypeForEVT(*DAG.getContext());
-    unsigned ABIAlign = DAG.getDataLayout().getABITypeAlignment(IntVTTy);
+    Align ABIAlign = DAG.getDataLayout().getABITypeAlign(IntVTTy);
     if (LDAlign < ABIAlign || STAlign < ABIAlign)
       return SDValue();
 
@@ -15775,7 +15889,7 @@ SDValue DAGCombiner::getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
   return DAG.getTokenFactor(StoreDL, Chains);
 }
 
-bool DAGCombiner::MergeStoresOfConstantsOrVecElts(
+bool DAGCombiner::mergeStoresOfConstantsOrVecElts(
     SmallVectorImpl<MemOpLink> &StoreNodes, EVT MemVT, unsigned NumStores,
     bool IsConstantSrc, bool UseVector, bool UseTrunc) {
   // Make sure we have something to merge.
@@ -15949,14 +16063,12 @@ void DAGCombiner::getStoreMergeCandidates(
   if (BasePtr.getBase().isUndef())
     return;
 
-  bool IsConstantSrc = isa<ConstantSDNode>(Val) || isa<ConstantFPSDNode>(Val);
-  bool IsExtractVecSrc = (Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
-                          Val.getOpcode() == ISD::EXTRACT_SUBVECTOR);
-  bool IsLoadSrc = isa<LoadSDNode>(Val);
+  StoreSource StoreSrc = getStoreSource(Val);
+  assert(StoreSrc != StoreSource::Unknown && "Expected known source for store");
   BaseIndexOffset LBasePtr;
   // Match on loadbaseptr if relevant.
   EVT LoadVT;
-  if (IsLoadSrc) {
+  if (StoreSrc == StoreSource::Load) {
     auto *Ld = cast<LoadSDNode>(Val);
     LBasePtr = BaseIndexOffset::match(Ld, DAG);
     LoadVT = Ld->getMemoryVT();
@@ -15984,7 +16096,7 @@ void DAGCombiner::getStoreMergeCandidates(
     // Allow merging constants of different types as integers.
     bool NoTypeMatch = (MemVT.isInteger()) ? !MemVT.bitsEq(Other->getMemoryVT())
                                            : Other->getMemoryVT() != MemVT;
-    if (IsLoadSrc) {
+    if (StoreSrc == StoreSource::Load) {
       if (NoTypeMatch)
         return false;
       // The Load's Base Ptr must also match
@@ -16008,13 +16120,13 @@ void DAGCombiner::getStoreMergeCandidates(
       } else
         return false;
     }
-    if (IsConstantSrc) {
+    if (StoreSrc == StoreSource::Constant) {
       if (NoTypeMatch)
         return false;
       if (!(isa<ConstantSDNode>(OtherBC) || isa<ConstantFPSDNode>(OtherBC)))
         return false;
     }
-    if (IsExtractVecSrc) {
+    if (StoreSrc == StoreSource::Extract) {
       // Do not merge truncated stores here.
       if (Other->isTruncatingStore())
         return false;
@@ -16155,84 +16267,22 @@ bool DAGCombiner::checkMergeStoreCandidatesForDependencies(
   return true;
 }
 
-bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
-  if (OptLevel == CodeGenOpt::None || !EnableStoreMerging)
-    return false;
-
-  // TODO: Extend this function to merge stores of scalable vectors.
-  // (i.e. two <vscale x 8 x i8> stores can be merged to one <vscale x 16 x i8>
-  // store since we know <vscale x 16 x i8> is exactly twice as large as
-  // <vscale x 8 x i8>). Until then, bail out for scalable vectors.
-  EVT MemVT = St->getMemoryVT();
-  if (MemVT.isScalableVector())
-    return false;
-
-  int64_t ElementSizeBytes = MemVT.getStoreSize();
-  unsigned NumMemElts = MemVT.isVector() ? MemVT.getVectorNumElements() : 1;
-
-  if (MemVT.getSizeInBits() * 2 > MaximumLegalStoreInBits)
-    return false;
-
-  bool NoVectors = DAG.getMachineFunction().getFunction().hasFnAttribute(
-      Attribute::NoImplicitFloat);
-
-  // This function cannot currently deal with non-byte-sized memory sizes.
-  if (ElementSizeBytes * 8 != (int64_t)MemVT.getSizeInBits())
-    return false;
-
-  if (!MemVT.isSimple())
-    return false;
-
-  // Perform an early exit check. Do not bother looking at stored values that
-  // are not constants, loads, or extracted vector elements.
-  SDValue StoredVal = peekThroughBitcasts(St->getValue());
-  bool IsLoadSrc = isa<LoadSDNode>(StoredVal);
-  bool IsConstantSrc = isa<ConstantSDNode>(StoredVal) ||
-                       isa<ConstantFPSDNode>(StoredVal);
-  bool IsExtractVecSrc = (StoredVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
-                          StoredVal.getOpcode() == ISD::EXTRACT_SUBVECTOR);
-  bool IsNonTemporalStore = St->isNonTemporal();
-  bool IsNonTemporalLoad =
-      IsLoadSrc && cast<LoadSDNode>(StoredVal)->isNonTemporal();
-
-  if (!IsConstantSrc && !IsLoadSrc && !IsExtractVecSrc)
-    return false;
-
-  SmallVector<MemOpLink, 8> StoreNodes;
-  SDNode *RootNode;
-  // Find potential store merge candidates by searching through chain sub-DAG
-  getStoreMergeCandidates(St, StoreNodes, RootNode);
-
-  // Check if there is anything to merge.
-  if (StoreNodes.size() < 2)
-    return false;
-
-  // Sort the memory operands according to their distance from the
-  // base pointer.
-  llvm::sort(StoreNodes, [](MemOpLink LHS, MemOpLink RHS) {
-    return LHS.OffsetFromBase < RHS.OffsetFromBase;
-  });
-
-  // Store Merge attempts to merge the lowest stores. This generally
-  // works out as if successful, as the remaining stores are checked
-  // after the first collection of stores is merged. However, in the
-  // case that a non-mergeable store is found first, e.g., {p[-2],
-  // p[0], p[1], p[2], p[3]}, we would fail and miss the subsequent
-  // mergeable cases. To prevent this, we prune such stores from the
-  // front of StoreNodes here.
-
-  bool RV = false;
-  while (StoreNodes.size() > 1) {
+unsigned
+DAGCombiner::getConsecutiveStores(SmallVectorImpl<MemOpLink> &StoreNodes,
+                                  int64_t ElementSizeBytes) const {
+  while (true) {
+    // Find a store past the width of the first store.
     size_t StartIdx = 0;
     while ((StartIdx + 1 < StoreNodes.size()) &&
            StoreNodes[StartIdx].OffsetFromBase + ElementSizeBytes !=
-               StoreNodes[StartIdx + 1].OffsetFromBase)
+              StoreNodes[StartIdx + 1].OffsetFromBase)
       ++StartIdx;
 
     // Bail if we don't have enough candidates to merge.
     if (StartIdx + 1 >= StoreNodes.size())
-      return RV;
+      return 0;
 
+    // Trim stores that overlapped with the first store.
     if (StartIdx)
       StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + StartIdx);
 
@@ -16248,303 +16298,332 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
         break;
       NumConsecutiveStores = i + 1;
     }
+    if (NumConsecutiveStores > 1)
+      return NumConsecutiveStores;
 
-    if (NumConsecutiveStores < 2) {
-      StoreNodes.erase(StoreNodes.begin(),
-                       StoreNodes.begin() + NumConsecutiveStores);
-      continue;
-    }
-
-    // The node with the lowest store address.
-    LLVMContext &Context = *DAG.getContext();
-    const DataLayout &DL = DAG.getDataLayout();
-
-    // Store the constants into memory as one consecutive store.
-    if (IsConstantSrc) {
-      while (NumConsecutiveStores >= 2) {
-        LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
-        unsigned FirstStoreAS = FirstInChain->getAddressSpace();
-        unsigned FirstStoreAlign = FirstInChain->getAlignment();
-        unsigned LastLegalType = 1;
-        unsigned LastLegalVectorType = 1;
-        bool LastIntegerTrunc = false;
-        bool NonZero = false;
-        unsigned FirstZeroAfterNonZero = NumConsecutiveStores;
-        for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
-          StoreSDNode *ST = cast<StoreSDNode>(StoreNodes[i].MemNode);
-          SDValue StoredVal = ST->getValue();
-          bool IsElementZero = false;
-          if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(StoredVal))
-            IsElementZero = C->isNullValue();
-          else if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(StoredVal))
-            IsElementZero = C->getConstantFPValue()->isNullValue();
-          if (IsElementZero) {
-            if (NonZero && FirstZeroAfterNonZero == NumConsecutiveStores)
-              FirstZeroAfterNonZero = i;
-          }
-          NonZero |= !IsElementZero;
-
-          // Find a legal type for the constant store.
-          unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
-          EVT StoreTy = EVT::getIntegerVT(Context, SizeInBits);
-          bool IsFast = false;
+    // There are no consecutive stores at the start of the list.
+    // Remove the first store and try again.
+    StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + 1);
+  }
+}
 
-          // Break early when size is too large to be legal.
-          if (StoreTy.getSizeInBits() > MaximumLegalStoreInBits)
-            break;
+bool DAGCombiner::tryStoreMergeOfConstants(
+    SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumConsecutiveStores,
+    EVT MemVT, SDNode *RootNode, bool AllowVectors) {
+  LLVMContext &Context = *DAG.getContext();
+  const DataLayout &DL = DAG.getDataLayout();
+  int64_t ElementSizeBytes = MemVT.getStoreSize();
+  unsigned NumMemElts = MemVT.isVector() ? MemVT.getVectorNumElements() : 1;
+  bool MadeChange = false;
+
+  // Store the constants into memory as one consecutive store.
+  while (NumConsecutiveStores >= 2) {
+    LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
+    unsigned FirstStoreAS = FirstInChain->getAddressSpace();
+    unsigned FirstStoreAlign = FirstInChain->getAlignment();
+    unsigned LastLegalType = 1;
+    unsigned LastLegalVectorType = 1;
+    bool LastIntegerTrunc = false;
+    bool NonZero = false;
+    unsigned FirstZeroAfterNonZero = NumConsecutiveStores;
+    for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
+      StoreSDNode *ST = cast<StoreSDNode>(StoreNodes[i].MemNode);
+      SDValue StoredVal = ST->getValue();
+      bool IsElementZero = false;
+      if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(StoredVal))
+        IsElementZero = C->isNullValue();
+      else if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(StoredVal))
+        IsElementZero = C->getConstantFPValue()->isNullValue();
+      if (IsElementZero) {
+        if (NonZero && FirstZeroAfterNonZero == NumConsecutiveStores)
+          FirstZeroAfterNonZero = i;
+      }
+      NonZero |= !IsElementZero;
 
-          if (TLI.isTypeLegal(StoreTy) &&
-              TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) &&
-              TLI.allowsMemoryAccess(Context, DL, StoreTy,
-                                     *FirstInChain->getMemOperand(), &IsFast) &&
-              IsFast) {
-            LastIntegerTrunc = false;
-            LastLegalType = i + 1;
-            // Or check whether a truncstore is legal.
-          } else if (TLI.getTypeAction(Context, StoreTy) ==
-                     TargetLowering::TypePromoteInteger) {
-            EVT LegalizedStoredValTy =
-                TLI.getTypeToTransformTo(Context, StoredVal.getValueType());
-            if (TLI.isTruncStoreLegal(LegalizedStoredValTy, StoreTy) &&
-                TLI.canMergeStoresTo(FirstStoreAS, LegalizedStoredValTy, DAG) &&
-                TLI.allowsMemoryAccess(Context, DL, StoreTy,
-                                       *FirstInChain->getMemOperand(),
-                                       &IsFast) &&
-                IsFast) {
-              LastIntegerTrunc = true;
-              LastLegalType = i + 1;
-            }
-          }
+      // Find a legal type for the constant store.
+      unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
+      EVT StoreTy = EVT::getIntegerVT(Context, SizeInBits);
+      bool IsFast = false;
 
-          // We only use vectors if the constant is known to be zero or the
-          // target allows it and the function is not marked with the
-          // noimplicitfloat attribute.
-          if ((!NonZero ||
-               TLI.storeOfVectorConstantIsCheap(MemVT, i + 1, FirstStoreAS)) &&
-              !NoVectors) {
-            // Find a legal type for the vector store.
-            unsigned Elts = (i + 1) * NumMemElts;
-            EVT Ty = EVT::getVectorVT(Context, MemVT.getScalarType(), Elts);
-            if (TLI.isTypeLegal(Ty) && TLI.isTypeLegal(MemVT) &&
-                TLI.canMergeStoresTo(FirstStoreAS, Ty, DAG) &&
-                TLI.allowsMemoryAccess(
-                    Context, DL, Ty, *FirstInChain->getMemOperand(), &IsFast) &&
-                IsFast)
-              LastLegalVectorType = i + 1;
-          }
-        }
+      // Break early when size is too large to be legal.
+      if (StoreTy.getSizeInBits() > MaximumLegalStoreInBits)
+        break;
 
-        bool UseVector = (LastLegalVectorType > LastLegalType) && !NoVectors;
-        unsigned NumElem = (UseVector) ? LastLegalVectorType : LastLegalType;
-
-        // Check if we found a legal integer type that creates a meaningful
-        // merge.
-        if (NumElem < 2) {
-          // We know that candidate stores are in order and of correct
-          // shape. While there is no mergeable sequence from the
-          // beginning one may start later in the sequence. The only
-          // reason a merge of size N could have failed where another of
-          // the same size would not have, is if the alignment has
-          // improved or we've dropped a non-zero value. Drop as many
-          // candidates as we can here.
-          unsigned NumSkip = 1;
-          while (
-              (NumSkip < NumConsecutiveStores) &&
-              (NumSkip < FirstZeroAfterNonZero) &&
-              (StoreNodes[NumSkip].MemNode->getAlignment() <= FirstStoreAlign))
-            NumSkip++;
-
-          StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumSkip);
-          NumConsecutiveStores -= NumSkip;
-          continue;
+      if (TLI.isTypeLegal(StoreTy) &&
+          TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) &&
+          TLI.allowsMemoryAccess(Context, DL, StoreTy,
+                                 *FirstInChain->getMemOperand(), &IsFast) &&
+          IsFast) {
+        LastIntegerTrunc = false;
+        LastLegalType = i + 1;
+        // Or check whether a truncstore is legal.
+      } else if (TLI.getTypeAction(Context, StoreTy) ==
+                 TargetLowering::TypePromoteInteger) {
+        EVT LegalizedStoredValTy =
+            TLI.getTypeToTransformTo(Context, StoredVal.getValueType());
+        if (TLI.isTruncStoreLegal(LegalizedStoredValTy, StoreTy) &&
+            TLI.canMergeStoresTo(FirstStoreAS, LegalizedStoredValTy, DAG) &&
+            TLI.allowsMemoryAccess(Context, DL, StoreTy,
+                                   *FirstInChain->getMemOperand(), &IsFast) &&
+            IsFast) {
+          LastIntegerTrunc = true;
+          LastLegalType = i + 1;
         }
+      }
 
-        // Check that we can merge these candidates without causing a cycle.
-        if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumElem,
-                                                      RootNode)) {
-          StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
-          NumConsecutiveStores -= NumElem;
-          continue;
-        }
+      // We only use vectors if the constant is known to be zero or the
+      // target allows it and the function is not marked with the
+      // noimplicitfloat attribute.
+      if ((!NonZero ||
+           TLI.storeOfVectorConstantIsCheap(MemVT, i + 1, FirstStoreAS)) &&
+          AllowVectors) {
+        // Find a legal type for the vector store.
+        unsigned Elts = (i + 1) * NumMemElts;
+        EVT Ty = EVT::getVectorVT(Context, MemVT.getScalarType(), Elts);
+        if (TLI.isTypeLegal(Ty) && TLI.isTypeLegal(MemVT) &&
+            TLI.canMergeStoresTo(FirstStoreAS, Ty, DAG) &&
+            TLI.allowsMemoryAccess(Context, DL, Ty,
+                                   *FirstInChain->getMemOperand(), &IsFast) &&
+            IsFast)
+          LastLegalVectorType = i + 1;
+      }
+    }
 
-        RV |= MergeStoresOfConstantsOrVecElts(StoreNodes, MemVT, NumElem, true,
-                                              UseVector, LastIntegerTrunc);
+    bool UseVector = (LastLegalVectorType > LastLegalType) && AllowVectors;
+    unsigned NumElem = (UseVector) ? LastLegalVectorType : LastLegalType;
+
+    // Check if we found a legal integer type that creates a meaningful
+    // merge.
+    if (NumElem < 2) {
+      // We know that candidate stores are in order and of correct
+      // shape. While there is no mergeable sequence from the
+      // beginning one may start later in the sequence. The only
+      // reason a merge of size N could have failed where another of
+      // the same size would not have, is if the alignment has
+      // improved or we've dropped a non-zero value. Drop as many
+      // candidates as we can here.
+      unsigned NumSkip = 1;
+      while ((NumSkip < NumConsecutiveStores) &&
+             (NumSkip < FirstZeroAfterNonZero) &&
+             (StoreNodes[NumSkip].MemNode->getAlignment() <= FirstStoreAlign))
+        NumSkip++;
+
+      StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumSkip);
+      NumConsecutiveStores -= NumSkip;
+      continue;
+    }
 
-        // Remove merged stores for next iteration.
-        StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
-        NumConsecutiveStores -= NumElem;
-      }
+    // Check that we can merge these candidates without causing a cycle.
+    if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumElem,
+                                                  RootNode)) {
+      StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
+      NumConsecutiveStores -= NumElem;
       continue;
     }
 
-    // When extracting multiple vector elements, try to store them
-    // in one vector store rather than a sequence of scalar stores.
-    if (IsExtractVecSrc) {
-      // Loop on Consecutive Stores on success.
-      while (NumConsecutiveStores >= 2) {
-        LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
-        unsigned FirstStoreAS = FirstInChain->getAddressSpace();
-        unsigned FirstStoreAlign = FirstInChain->getAlignment();
-        unsigned NumStoresToMerge = 1;
-        for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
-          // Find a legal type for the vector store.
-          unsigned Elts = (i + 1) * NumMemElts;
-          EVT Ty =
-              EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(), Elts);
-          bool IsFast = false;
-
-          // Break early when size is too large to be legal.
-          if (Ty.getSizeInBits() > MaximumLegalStoreInBits)
-            break;
+    MadeChange |= mergeStoresOfConstantsOrVecElts(
+        StoreNodes, MemVT, NumElem, true, UseVector, LastIntegerTrunc);
 
-          if (TLI.isTypeLegal(Ty) &&
-              TLI.canMergeStoresTo(FirstStoreAS, Ty, DAG) &&
-              TLI.allowsMemoryAccess(Context, DL, Ty,
-                                     *FirstInChain->getMemOperand(), &IsFast) &&
-              IsFast)
-            NumStoresToMerge = i + 1;
-        }
+    // Remove merged stores for next iteration.
+    StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
+    NumConsecutiveStores -= NumElem;
+  }
+  return MadeChange;
+}
 
-        // Check if we found a legal integer type creating a meaningful
-        // merge.
-        if (NumStoresToMerge < 2) {
-          // We know that candidate stores are in order and of correct
-          // shape. While there is no mergeable sequence from the
-          // beginning one may start later in the sequence. The only
-          // reason a merge of size N could have failed where another of
-          // the same size would not have, is if the alignment has
-          // improved. Drop as many candidates as we can here.
-          unsigned NumSkip = 1;
-          while (
-              (NumSkip < NumConsecutiveStores) &&
-              (StoreNodes[NumSkip].MemNode->getAlignment() <= FirstStoreAlign))
-            NumSkip++;
-
-          StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumSkip);
-          NumConsecutiveStores -= NumSkip;
-          continue;
-        }
+bool DAGCombiner::tryStoreMergeOfExtracts(
+    SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumConsecutiveStores,
+    EVT MemVT, SDNode *RootNode) {
+  LLVMContext &Context = *DAG.getContext();
+  const DataLayout &DL = DAG.getDataLayout();
+  unsigned NumMemElts = MemVT.isVector() ? MemVT.getVectorNumElements() : 1;
+  bool MadeChange = false;
+
+  // Loop on Consecutive Stores on success.
+  while (NumConsecutiveStores >= 2) {
+    LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
+    unsigned FirstStoreAS = FirstInChain->getAddressSpace();
+    unsigned FirstStoreAlign = FirstInChain->getAlignment();
+    unsigned NumStoresToMerge = 1;
+    for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
+      // Find a legal type for the vector store.
+      unsigned Elts = (i + 1) * NumMemElts;
+      EVT Ty = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(), Elts);
+      bool IsFast = false;
 
-        // Check that we can merge these candidates without causing a cycle.
-        if (!checkMergeStoreCandidatesForDependencies(
-                StoreNodes, NumStoresToMerge, RootNode)) {
-          StoreNodes.erase(StoreNodes.begin(),
-                           StoreNodes.begin() + NumStoresToMerge);
-          NumConsecutiveStores -= NumStoresToMerge;
-          continue;
-        }
+      // Break early when size is too large to be legal.
+      if (Ty.getSizeInBits() > MaximumLegalStoreInBits)
+        break;
 
-        RV |= MergeStoresOfConstantsOrVecElts(
-            StoreNodes, MemVT, NumStoresToMerge, false, true, false);
+      if (TLI.isTypeLegal(Ty) && TLI.canMergeStoresTo(FirstStoreAS, Ty, DAG) &&
+          TLI.allowsMemoryAccess(Context, DL, Ty,
+                                 *FirstInChain->getMemOperand(), &IsFast) &&
+          IsFast)
+        NumStoresToMerge = i + 1;
+    }
+
+    // Check if we found a legal integer type creating a meaningful
+    // merge.
+    if (NumStoresToMerge < 2) {
+      // We know that candidate stores are in order and of correct
+      // shape. While there is no mergeable sequence from the
+      // beginning one may start later in the sequence. The only
+      // reason a merge of size N could have failed where another of
+      // the same size would not have, is if the alignment has
+      // improved. Drop as many candidates as we can here.
+      unsigned NumSkip = 1;
+      while ((NumSkip < NumConsecutiveStores) &&
+             (StoreNodes[NumSkip].MemNode->getAlignment() <= FirstStoreAlign))
+        NumSkip++;
+
+      StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumSkip);
+      NumConsecutiveStores -= NumSkip;
+      continue;
+    }
 
-        StoreNodes.erase(StoreNodes.begin(),
-                         StoreNodes.begin() + NumStoresToMerge);
-        NumConsecutiveStores -= NumStoresToMerge;
-      }
+    // Check that we can merge these candidates without causing a cycle.
+    if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumStoresToMerge,
+                                                  RootNode)) {
+      StoreNodes.erase(StoreNodes.begin(),
+                       StoreNodes.begin() + NumStoresToMerge);
+      NumConsecutiveStores -= NumStoresToMerge;
       continue;
     }
 
-    // Below we handle the case of multiple consecutive stores that
-    // come from multiple consecutive loads. We merge them into a single
-    // wide load and a single wide store.
+    MadeChange |= mergeStoresOfConstantsOrVecElts(
+        StoreNodes, MemVT, NumStoresToMerge, false, true, false);
+
+    StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumStoresToMerge);
+    NumConsecutiveStores -= NumStoresToMerge;
+  }
+  return MadeChange;
+}
+
+bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
+                                       unsigned NumConsecutiveStores, EVT MemVT,
+                                       SDNode *RootNode, bool AllowVectors,
+                                       bool IsNonTemporalStore,
+                                       bool IsNonTemporalLoad) {
+  LLVMContext &Context = *DAG.getContext();
+  const DataLayout &DL = DAG.getDataLayout();
+  int64_t ElementSizeBytes = MemVT.getStoreSize();
+  unsigned NumMemElts = MemVT.isVector() ? MemVT.getVectorNumElements() : 1;
+  bool MadeChange = false;
 
-    // Look for load nodes which are used by the stored values.
-    SmallVector<MemOpLink, 8> LoadNodes;
+  int64_t StartAddress = StoreNodes[0].OffsetFromBase;
 
-    // Find acceptable loads. Loads need to have the same chain (token factor),
-    // must not be zext, volatile, indexed, and they must be consecutive.
-    BaseIndexOffset LdBasePtr;
+  // Look for load nodes which are used by the stored values.
+  SmallVector<MemOpLink, 8> LoadNodes;
 
-    for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
-      StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
-      SDValue Val = peekThroughBitcasts(St->getValue());
-      LoadSDNode *Ld = cast<LoadSDNode>(Val);
-
-      BaseIndexOffset LdPtr = BaseIndexOffset::match(Ld, DAG);
-      // If this is not the first ptr that we check.
-      int64_t LdOffset = 0;
-      if (LdBasePtr.getBase().getNode()) {
-        // The base ptr must be the same.
-        if (!LdBasePtr.equalBaseIndex(LdPtr, DAG, LdOffset))
-          break;
-      } else {
-        // Check that all other base pointers are the same as this one.
-        LdBasePtr = LdPtr;
-      }
+  // Find acceptable loads. Loads need to have the same chain (token factor),
+  // must not be zext, volatile, indexed, and they must be consecutive.
+  BaseIndexOffset LdBasePtr;
+
+  for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
+    StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
+    SDValue Val = peekThroughBitcasts(St->getValue());
+    LoadSDNode *Ld = cast<LoadSDNode>(Val);
 
-      // We found a potential memory operand to merge.
-      LoadNodes.push_back(MemOpLink(Ld, LdOffset));
+    BaseIndexOffset LdPtr = BaseIndexOffset::match(Ld, DAG);
+    // If this is not the first ptr that we check.
+    int64_t LdOffset = 0;
+    if (LdBasePtr.getBase().getNode()) {
+      // The base ptr must be the same.
+      if (!LdBasePtr.equalBaseIndex(LdPtr, DAG, LdOffset))
+        break;
+    } else {
+      // Check that all other base pointers are the same as this one.
+      LdBasePtr = LdPtr;
+    }
+
+    // We found a potential memory operand to merge.
+    LoadNodes.push_back(MemOpLink(Ld, LdOffset));
+  }
+
+  while (NumConsecutiveStores >= 2 && LoadNodes.size() >= 2) {
+    // If we have load/store pair instructions and we only have two values,
+    // don't bother merging.
+    Align RequiredAlignment;
+    if (LoadNodes.size() == 2 && TLI.hasPairedLoad(MemVT, RequiredAlignment) &&
+        StoreNodes[0].MemNode->getAlign() >= RequiredAlignment) {
+      StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + 2);
+      LoadNodes.erase(LoadNodes.begin(), LoadNodes.begin() + 2);
+      break;
     }
+    LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
+    unsigned FirstStoreAS = FirstInChain->getAddressSpace();
+    unsigned FirstStoreAlign = FirstInChain->getAlignment();
+    LoadSDNode *FirstLoad = cast<LoadSDNode>(LoadNodes[0].MemNode);
 
-    while (NumConsecutiveStores >= 2 && LoadNodes.size() >= 2) {
-      // If we have load/store pair instructions and we only have two values,
-      // don't bother merging.
-      unsigned RequiredAlignment;
-      if (LoadNodes.size() == 2 &&
-          TLI.hasPairedLoad(MemVT, RequiredAlignment) &&
-          StoreNodes[0].MemNode->getAlignment() >= RequiredAlignment) {
-        StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + 2);
-        LoadNodes.erase(LoadNodes.begin(), LoadNodes.begin() + 2);
+    // Scan the memory operations on the chain and find the first
+    // non-consecutive load memory address. These variables hold the index in
+    // the store node array.
+
+    unsigned LastConsecutiveLoad = 1;
+
+    // This variable refers to the size and not index in the array.
+    unsigned LastLegalVectorType = 1;
+    unsigned LastLegalIntegerType = 1;
+    bool isDereferenceable = true;
+    bool DoIntegerTruncate = false;
+    StartAddress = LoadNodes[0].OffsetFromBase;
+    SDValue LoadChain = FirstLoad->getChain();
+    for (unsigned i = 1; i < LoadNodes.size(); ++i) {
+      // All loads must share the same chain.
+      if (LoadNodes[i].MemNode->getChain() != LoadChain)
         break;
-      }
-      LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
-      unsigned FirstStoreAS = FirstInChain->getAddressSpace();
-      unsigned FirstStoreAlign = FirstInChain->getAlignment();
-      LoadSDNode *FirstLoad = cast<LoadSDNode>(LoadNodes[0].MemNode);
-      unsigned FirstLoadAlign = FirstLoad->getAlignment();
-
-      // Scan the memory operations on the chain and find the first
-      // non-consecutive load memory address. These variables hold the index in
-      // the store node array.
-
-      unsigned LastConsecutiveLoad = 1;
-
-      // This variable refers to the size and not index in the array.
-      unsigned LastLegalVectorType = 1;
-      unsigned LastLegalIntegerType = 1;
-      bool isDereferenceable = true;
-      bool DoIntegerTruncate = false;
-      StartAddress = LoadNodes[0].OffsetFromBase;
-      SDValue FirstChain = FirstLoad->getChain();
-      for (unsigned i = 1; i < LoadNodes.size(); ++i) {
-        // All loads must share the same chain.
-        if (LoadNodes[i].MemNode->getChain() != FirstChain)
-          break;
 
-        int64_t CurrAddress = LoadNodes[i].OffsetFromBase;
-        if (CurrAddress - StartAddress != (ElementSizeBytes * i))
-          break;
-        LastConsecutiveLoad = i;
+      int64_t CurrAddress = LoadNodes[i].OffsetFromBase;
+      if (CurrAddress - StartAddress != (ElementSizeBytes * i))
+        break;
+      LastConsecutiveLoad = i;
 
-        if (isDereferenceable && !LoadNodes[i].MemNode->isDereferenceable())
-          isDereferenceable = false;
+      if (isDereferenceable && !LoadNodes[i].MemNode->isDereferenceable())
+        isDereferenceable = false;
 
-        // Find a legal type for the vector store.
-        unsigned Elts = (i + 1) * NumMemElts;
-        EVT StoreTy = EVT::getVectorVT(Context, MemVT.getScalarType(), Elts);
+      // Find a legal type for the vector store.
+      unsigned Elts = (i + 1) * NumMemElts;
+      EVT StoreTy = EVT::getVectorVT(Context, MemVT.getScalarType(), Elts);
 
-        // Break early when size is too large to be legal.
-        if (StoreTy.getSizeInBits() > MaximumLegalStoreInBits)
-          break;
+      // Break early when size is too large to be legal.
+      if (StoreTy.getSizeInBits() > MaximumLegalStoreInBits)
+        break;
 
-        bool IsFastSt = false;
-        bool IsFastLd = false;
-        if (TLI.isTypeLegal(StoreTy) &&
-            TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) &&
-            TLI.allowsMemoryAccess(Context, DL, StoreTy,
-                                   *FirstInChain->getMemOperand(), &IsFastSt) &&
-            IsFastSt &&
-            TLI.allowsMemoryAccess(Context, DL, StoreTy,
-                                   *FirstLoad->getMemOperand(), &IsFastLd) &&
-            IsFastLd) {
-          LastLegalVectorType = i + 1;
-        }
+      bool IsFastSt = false;
+      bool IsFastLd = false;
+      if (TLI.isTypeLegal(StoreTy) &&
+          TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) &&
+          TLI.allowsMemoryAccess(Context, DL, StoreTy,
+                                 *FirstInChain->getMemOperand(), &IsFastSt) &&
+          IsFastSt &&
+          TLI.allowsMemoryAccess(Context, DL, StoreTy,
+                                 *FirstLoad->getMemOperand(), &IsFastLd) &&
+          IsFastLd) {
+        LastLegalVectorType = i + 1;
+      }
 
-        // Find a legal type for the integer store.
-        unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
-        StoreTy = EVT::getIntegerVT(Context, SizeInBits);
-        if (TLI.isTypeLegal(StoreTy) &&
-            TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) &&
+      // Find a legal type for the integer store.
+      unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
+      StoreTy = EVT::getIntegerVT(Context, SizeInBits);
+      if (TLI.isTypeLegal(StoreTy) &&
+          TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) &&
+          TLI.allowsMemoryAccess(Context, DL, StoreTy,
+                                 *FirstInChain->getMemOperand(), &IsFastSt) &&
+          IsFastSt &&
+          TLI.allowsMemoryAccess(Context, DL, StoreTy,
+                                 *FirstLoad->getMemOperand(), &IsFastLd) &&
+          IsFastLd) {
+        LastLegalIntegerType = i + 1;
+        DoIntegerTruncate = false;
+        // Or check whether a truncstore and extload is legal.
+      } else if (TLI.getTypeAction(Context, StoreTy) ==
+                 TargetLowering::TypePromoteInteger) {
+        EVT LegalizedStoredValTy = TLI.getTypeToTransformTo(Context, StoreTy);
+        if (TLI.isTruncStoreLegal(LegalizedStoredValTy, StoreTy) &&
+            TLI.canMergeStoresTo(FirstStoreAS, LegalizedStoredValTy, DAG) &&
+            TLI.isLoadExtLegal(ISD::ZEXTLOAD, LegalizedStoredValTy, StoreTy) &&
+            TLI.isLoadExtLegal(ISD::SEXTLOAD, LegalizedStoredValTy, StoreTy) &&
+            TLI.isLoadExtLegal(ISD::EXTLOAD, LegalizedStoredValTy, StoreTy) &&
             TLI.allowsMemoryAccess(Context, DL, StoreTy,
                                    *FirstInChain->getMemOperand(), &IsFastSt) &&
             IsFastSt &&
@@ -16552,149 +16631,215 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
                                    *FirstLoad->getMemOperand(), &IsFastLd) &&
             IsFastLd) {
           LastLegalIntegerType = i + 1;
-          DoIntegerTruncate = false;
-          // Or check whether a truncstore and extload is legal.
-        } else if (TLI.getTypeAction(Context, StoreTy) ==
-                   TargetLowering::TypePromoteInteger) {
-          EVT LegalizedStoredValTy = TLI.getTypeToTransformTo(Context, StoreTy);
-          if (TLI.isTruncStoreLegal(LegalizedStoredValTy, StoreTy) &&
-              TLI.canMergeStoresTo(FirstStoreAS, LegalizedStoredValTy, DAG) &&
-              TLI.isLoadExtLegal(ISD::ZEXTLOAD, LegalizedStoredValTy,
-                                 StoreTy) &&
-              TLI.isLoadExtLegal(ISD::SEXTLOAD, LegalizedStoredValTy,
-                                 StoreTy) &&
-              TLI.isLoadExtLegal(ISD::EXTLOAD, LegalizedStoredValTy, StoreTy) &&
-              TLI.allowsMemoryAccess(Context, DL, StoreTy,
-                                     *FirstInChain->getMemOperand(),
-                                     &IsFastSt) &&
-              IsFastSt &&
-              TLI.allowsMemoryAccess(Context, DL, StoreTy,
-                                     *FirstLoad->getMemOperand(), &IsFastLd) &&
-              IsFastLd) {
-            LastLegalIntegerType = i + 1;
-            DoIntegerTruncate = true;
-          }
+          DoIntegerTruncate = true;
         }
       }
+    }
 
-      // Only use vector types if the vector type is larger than the integer
-      // type. If they are the same, use integers.
-      bool UseVectorTy =
-          LastLegalVectorType > LastLegalIntegerType && !NoVectors;
-      unsigned LastLegalType =
-          std::max(LastLegalVectorType, LastLegalIntegerType);
-
-      // We add +1 here because the LastXXX variables refer to location while
-      // the NumElem refers to array/index size.
-      unsigned NumElem =
-          std::min(NumConsecutiveStores, LastConsecutiveLoad + 1);
-      NumElem = std::min(LastLegalType, NumElem);
-
-      if (NumElem < 2) {
-        // We know that candidate stores are in order and of correct
-        // shape. While there is no mergeable sequence from the
-        // beginning one may start later in the sequence. The only
-        // reason a merge of size N could have failed where another of
-        // the same size would not have is if the alignment or either
-        // the load or store has improved. Drop as many candidates as we
-        // can here.
-        unsigned NumSkip = 1;
-        while ((NumSkip < LoadNodes.size()) &&
-               (LoadNodes[NumSkip].MemNode->getAlignment() <= FirstLoadAlign) &&
-               (StoreNodes[NumSkip].MemNode->getAlignment() <= FirstStoreAlign))
-          NumSkip++;
-        StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumSkip);
-        LoadNodes.erase(LoadNodes.begin(), LoadNodes.begin() + NumSkip);
-        NumConsecutiveStores -= NumSkip;
-        continue;
-      }
+    // Only use vector types if the vector type is larger than the integer
+    // type. If they are the same, use integers.
+    bool UseVectorTy =
+        LastLegalVectorType > LastLegalIntegerType && AllowVectors;
+    unsigned LastLegalType =
+        std::max(LastLegalVectorType, LastLegalIntegerType);
+
+    // We add +1 here because the LastXXX variables refer to location while
+    // the NumElem refers to array/index size.
+    unsigned NumElem = std::min(NumConsecutiveStores, LastConsecutiveLoad + 1);
+    NumElem = std::min(LastLegalType, NumElem);
+    unsigned FirstLoadAlign = FirstLoad->getAlignment();
+
+    if (NumElem < 2) {
+      // We know that candidate stores are in order and of correct
+      // shape. While there is no mergeable sequence from the
+      // beginning one may start later in the sequence. The only
+      // reason a merge of size N could have failed where another of
+      // the same size would not have is if the alignment or either
+      // the load or store has improved. Drop as many candidates as we
+      // can here.
+      unsigned NumSkip = 1;
+      while ((NumSkip < LoadNodes.size()) &&
+             (LoadNodes[NumSkip].MemNode->getAlignment() <= FirstLoadAlign) &&
+             (StoreNodes[NumSkip].MemNode->getAlignment() <= FirstStoreAlign))
+        NumSkip++;
+      StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumSkip);
+      LoadNodes.erase(LoadNodes.begin(), LoadNodes.begin() + NumSkip);
+      NumConsecutiveStores -= NumSkip;
+      continue;
+    }
 
-      // Check that we can merge these candidates without causing a cycle.
-      if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumElem,
-                                                    RootNode)) {
-        StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
-        LoadNodes.erase(LoadNodes.begin(), LoadNodes.begin() + NumElem);
-        NumConsecutiveStores -= NumElem;
-        continue;
-      }
+    // Check that we can merge these candidates without causing a cycle.
+    if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumElem,
+                                                  RootNode)) {
+      StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
+      LoadNodes.erase(LoadNodes.begin(), LoadNodes.begin() + NumElem);
+      NumConsecutiveStores -= NumElem;
+      continue;
+    }
 
-      // Find if it is better to use vectors or integers to load and store
-      // to memory.
-      EVT JointMemOpVT;
-      if (UseVectorTy) {
-        // Find a legal type for the vector store.
-        unsigned Elts = NumElem * NumMemElts;
-        JointMemOpVT = EVT::getVectorVT(Context, MemVT.getScalarType(), Elts);
-      } else {
-        unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
-        JointMemOpVT = EVT::getIntegerVT(Context, SizeInBits);
-      }
+    // Find if it is better to use vectors or integers to load and store
+    // to memory.
+    EVT JointMemOpVT;
+    if (UseVectorTy) {
+      // Find a legal type for the vector store.
+      unsigned Elts = NumElem * NumMemElts;
+      JointMemOpVT = EVT::getVectorVT(Context, MemVT.getScalarType(), Elts);
+    } else {
+      unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
+      JointMemOpVT = EVT::getIntegerVT(Context, SizeInBits);
+    }
+
+    SDLoc LoadDL(LoadNodes[0].MemNode);
+    SDLoc StoreDL(StoreNodes[0].MemNode);
+
+    // The merged loads are required to have the same incoming chain, so
+    // using the first's chain is acceptable.
+
+    SDValue NewStoreChain = getMergeStoreChains(StoreNodes, NumElem);
+    AddToWorklist(NewStoreChain.getNode());
+
+    MachineMemOperand::Flags LdMMOFlags =
+        isDereferenceable ? MachineMemOperand::MODereferenceable
+                          : MachineMemOperand::MONone;
+    if (IsNonTemporalLoad)
+      LdMMOFlags |= MachineMemOperand::MONonTemporal;
+
+    MachineMemOperand::Flags StMMOFlags = IsNonTemporalStore
+                                              ? MachineMemOperand::MONonTemporal
+                                              : MachineMemOperand::MONone;
+
+    SDValue NewLoad, NewStore;
+    if (UseVectorTy || !DoIntegerTruncate) {
+      NewLoad = DAG.getLoad(
+          JointMemOpVT, LoadDL, FirstLoad->getChain(), FirstLoad->getBasePtr(),
+          FirstLoad->getPointerInfo(), FirstLoadAlign, LdMMOFlags);
+      NewStore = DAG.getStore(
+          NewStoreChain, StoreDL, NewLoad, FirstInChain->getBasePtr(),
+          FirstInChain->getPointerInfo(), FirstStoreAlign, StMMOFlags);
+    } else { // This must be the truncstore/extload case
+      EVT ExtendedTy =
+          TLI.getTypeToTransformTo(*DAG.getContext(), JointMemOpVT);
+      NewLoad = DAG.getExtLoad(ISD::EXTLOAD, LoadDL, ExtendedTy,
+                               FirstLoad->getChain(), FirstLoad->getBasePtr(),
+                               FirstLoad->getPointerInfo(), JointMemOpVT,
+                               FirstLoadAlign, LdMMOFlags);
+      NewStore = DAG.getTruncStore(NewStoreChain, StoreDL, NewLoad,
+                                   FirstInChain->getBasePtr(),
+                                   FirstInChain->getPointerInfo(), JointMemOpVT,
+                                   FirstInChain->getAlignment(),
+                                   FirstInChain->getMemOperand()->getFlags());
+    }
+
+    // Transfer chain users from old loads to the new load.
+    for (unsigned i = 0; i < NumElem; ++i) {
+      LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode);
+      DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1),
+                                    SDValue(NewLoad.getNode(), 1));
+    }
+
+    // Replace all stores with the new store. Recursively remove corresponding
+    // values if they are no longer used.
+    for (unsigned i = 0; i < NumElem; ++i) {
+      SDValue Val = StoreNodes[i].MemNode->getOperand(1);
+      CombineTo(StoreNodes[i].MemNode, NewStore);
+      if (Val.getNode()->use_empty())
+        recursivelyDeleteUnusedNodes(Val.getNode());
+    }
+
+    MadeChange = true;
+    StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
+    LoadNodes.erase(LoadNodes.begin(), LoadNodes.begin() + NumElem);
+    NumConsecutiveStores -= NumElem;
+  }
+  return MadeChange;
+}
+
+bool DAGCombiner::mergeConsecutiveStores(StoreSDNode *St) {
+  if (OptLevel == CodeGenOpt::None || !EnableStoreMerging)
+    return false;
 
-      SDLoc LoadDL(LoadNodes[0].MemNode);
-      SDLoc StoreDL(StoreNodes[0].MemNode);
-
-      // The merged loads are required to have the same incoming chain, so
-      // using the first's chain is acceptable.
-
-      SDValue NewStoreChain = getMergeStoreChains(StoreNodes, NumElem);
-      AddToWorklist(NewStoreChain.getNode());
-
-      MachineMemOperand::Flags LdMMOFlags =
-          isDereferenceable ? MachineMemOperand::MODereferenceable
-                            : MachineMemOperand::MONone;
-      if (IsNonTemporalLoad)
-        LdMMOFlags |= MachineMemOperand::MONonTemporal;
-
-      MachineMemOperand::Flags StMMOFlags =
-          IsNonTemporalStore ? MachineMemOperand::MONonTemporal
-                             : MachineMemOperand::MONone;
-
-      SDValue NewLoad, NewStore;
-      if (UseVectorTy || !DoIntegerTruncate) {
-        NewLoad =
-            DAG.getLoad(JointMemOpVT, LoadDL, FirstLoad->getChain(),
-                        FirstLoad->getBasePtr(), FirstLoad->getPointerInfo(),
-                        FirstLoadAlign, LdMMOFlags);
-        NewStore = DAG.getStore(
-            NewStoreChain, StoreDL, NewLoad, FirstInChain->getBasePtr(),
-            FirstInChain->getPointerInfo(), FirstStoreAlign, StMMOFlags);
-      } else { // This must be the truncstore/extload case
-        EVT ExtendedTy =
-            TLI.getTypeToTransformTo(*DAG.getContext(), JointMemOpVT);
-        NewLoad = DAG.getExtLoad(ISD::EXTLOAD, LoadDL, ExtendedTy,
-                                 FirstLoad->getChain(), FirstLoad->getBasePtr(),
-                                 FirstLoad->getPointerInfo(), JointMemOpVT,
-                                 FirstLoadAlign, LdMMOFlags);
-        NewStore = DAG.getTruncStore(NewStoreChain, StoreDL, NewLoad,
-                                     FirstInChain->getBasePtr(),
-                                     FirstInChain->getPointerInfo(),
-                                     JointMemOpVT, FirstInChain->getAlignment(),
-                                     FirstInChain->getMemOperand()->getFlags());
-      }
+  // TODO: Extend this function to merge stores of scalable vectors.
+  // (i.e. two <vscale x 8 x i8> stores can be merged to one <vscale x 16 x i8>
+  // store since we know <vscale x 16 x i8> is exactly twice as large as
+  // <vscale x 8 x i8>). Until then, bail out for scalable vectors.
+  EVT MemVT = St->getMemoryVT();
+  if (MemVT.isScalableVector())
+    return false;
+  if (!MemVT.isSimple() || MemVT.getSizeInBits() * 2 > MaximumLegalStoreInBits)
+    return false;
 
-      // Transfer chain users from old loads to the new load.
-      for (unsigned i = 0; i < NumElem; ++i) {
-        LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode);
-        DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1),
-                                      SDValue(NewLoad.getNode(), 1));
-      }
+  // This function cannot currently deal with non-byte-sized memory sizes.
+  int64_t ElementSizeBytes = MemVT.getStoreSize();
+  if (ElementSizeBytes * 8 != (int64_t)MemVT.getSizeInBits())
+    return false;
 
-      // Replace the all stores with the new store. Recursively remove
-      // corresponding value if its no longer used.
-      for (unsigned i = 0; i < NumElem; ++i) {
-        SDValue Val = StoreNodes[i].MemNode->getOperand(1);
-        CombineTo(StoreNodes[i].MemNode, NewStore);
-        if (Val.getNode()->use_empty())
-          recursivelyDeleteUnusedNodes(Val.getNode());
-      }
+  // Do not bother looking at stored values that are not constants, loads, or
+  // extracted vector elements.
+  SDValue StoredVal = peekThroughBitcasts(St->getValue());
+  const StoreSource StoreSrc = getStoreSource(StoredVal);
+  if (StoreSrc == StoreSource::Unknown)
+    return false;
 
-      RV = true;
-      StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
-      LoadNodes.erase(LoadNodes.begin(), LoadNodes.begin() + NumElem);
-      NumConsecutiveStores -= NumElem;
+  SmallVector<MemOpLink, 8> StoreNodes;
+  SDNode *RootNode;
+  // Find potential store merge candidates by searching through chain sub-DAG
+  getStoreMergeCandidates(St, StoreNodes, RootNode);
+
+  // Check if there is anything to merge.
+  if (StoreNodes.size() < 2)
+    return false;
+
+  // Sort the memory operands according to their distance from the
+  // base pointer.
+  llvm::sort(StoreNodes, [](MemOpLink LHS, MemOpLink RHS) {
+    return LHS.OffsetFromBase < RHS.OffsetFromBase;
+  });
+
+  bool AllowVectors = !DAG.getMachineFunction().getFunction().hasFnAttribute(
+      Attribute::NoImplicitFloat);
+  bool IsNonTemporalStore = St->isNonTemporal();
+  bool IsNonTemporalLoad = StoreSrc == StoreSource::Load &&
+                           cast<LoadSDNode>(StoredVal)->isNonTemporal();
+
+  // Store Merge attempts to merge the lowest stores. This generally
+  // works out as if successful, as the remaining stores are checked
+  // after the first collection of stores is merged. However, in the
+  // case that a non-mergeable store is found first, e.g., {p[-2],
+  // p[0], p[1], p[2], p[3]}, we would fail and miss the subsequent
+  // mergeable cases. To prevent this, we prune such stores from the
+  // front of StoreNodes here.
+  bool MadeChange = false;
+  while (StoreNodes.size() > 1) {
+    unsigned NumConsecutiveStores =
+        getConsecutiveStores(StoreNodes, ElementSizeBytes);
+    // There are no more stores in the list to examine.
+    if (NumConsecutiveStores == 0)
+      return MadeChange;
+
+    // We have at least 2 consecutive stores. Try to merge them.
+    assert(NumConsecutiveStores >= 2 && "Expected at least 2 stores");
+    switch (StoreSrc) {
+    case StoreSource::Constant:
+      MadeChange |= tryStoreMergeOfConstants(StoreNodes, NumConsecutiveStores,
+                                             MemVT, RootNode, AllowVectors);
+      break;
+
+    case StoreSource::Extract:
+      MadeChange |= tryStoreMergeOfExtracts(StoreNodes, NumConsecutiveStores,
+                                            MemVT, RootNode);
+      break;
+
+    case StoreSource::Load:
+      MadeChange |= tryStoreMergeOfLoads(StoreNodes, NumConsecutiveStores,
+                                         MemVT, RootNode, AllowVectors,
+                                         IsNonTemporalStore, IsNonTemporalLoad);
+      break;
+
+    default:
+      llvm_unreachable("Unhandled store source type");
     }
   }
-  return RV;
+  return MadeChange;
 }
 
 SDValue DAGCombiner::replaceStoreChain(StoreSDNode *ST, SDValue BetterChain) {
@@ -16963,7 +17108,7 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) {
       // There can be multiple store sequences on the same chain.
       // Keep trying to merge store sequences until we are unable to do so
       // or until we merge the last store on the chain.
-      bool Changed = MergeConsecutiveStores(ST);
+      bool Changed = mergeConsecutiveStores(ST);
       if (!Changed) break;
       // Return N as merge only uses CombineTo and no worklist clean
       // up is necessary.
@@ -17384,11 +17529,12 @@ SDValue DAGCombiner::scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
 
   EVT ResultVT = EVE->getValueType(0);
   EVT VecEltVT = InVecVT.getVectorElementType();
-  unsigned Align = OriginalLoad->getAlignment();
-  unsigned NewAlign = DAG.getDataLayout().getABITypeAlignment(
+  Align Alignment = OriginalLoad->getAlign();
+  Align NewAlign = DAG.getDataLayout().getABITypeAlign(
       VecEltVT.getTypeForEVT(*DAG.getContext()));
 
-  if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, VecEltVT))
+  if (NewAlign > Alignment ||
+      !TLI.isOperationLegalOrCustom(ISD::LOAD, VecEltVT))
     return SDValue();
 
   ISD::LoadExtType ExtTy = ResultVT.bitsGT(VecEltVT) ?
@@ -17396,7 +17542,7 @@ SDValue DAGCombiner::scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
   if (!TLI.shouldReduceLoadWidth(OriginalLoad, ExtTy, VecEltVT))
     return SDValue();
 
-  Align = NewAlign;
+  Alignment = NewAlign;
 
   SDValue NewPtr = OriginalLoad->getBasePtr();
   SDValue Offset;
@@ -17436,13 +17582,13 @@ SDValue DAGCombiner::scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
                                    : ISD::EXTLOAD;
     Load = DAG.getExtLoad(ExtType, SDLoc(EVE), ResultVT,
                           OriginalLoad->getChain(), NewPtr, MPI, VecEltVT,
-                          Align, OriginalLoad->getMemOperand()->getFlags(),
+                          Alignment, OriginalLoad->getMemOperand()->getFlags(),
                           OriginalLoad->getAAInfo());
     Chain = Load.getValue(1);
   } else {
-    Load = DAG.getLoad(VecEltVT, SDLoc(EVE), OriginalLoad->getChain(), NewPtr,
-                       MPI, Align, OriginalLoad->getMemOperand()->getFlags(),
-                       OriginalLoad->getAAInfo());
+    Load = DAG.getLoad(
+        VecEltVT, SDLoc(EVE), OriginalLoad->getChain(), NewPtr, MPI, Alignment,
+        OriginalLoad->getMemOperand()->getFlags(), OriginalLoad->getAAInfo());
     Chain = Load.getValue(1);
     if (ResultVT.bitsLT(VecEltVT))
       Load = DAG.getNode(ISD::TRUNCATE, SDLoc(EVE), ResultVT, Load);
@@ -17767,8 +17913,11 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
       Elt = (Idx < (int)NumElts) ? Idx : Idx - (int)NumElts;
       Index = DAG.getConstant(Elt, DL, Index.getValueType());
     }
-  } else if (VecOp.getOpcode() == ISD::CONCAT_VECTORS &&
-             !BCNumEltsChanged && VecVT.getVectorElementType() == ScalarVT) {
+  } else if (VecOp.getOpcode() == ISD::CONCAT_VECTORS && !BCNumEltsChanged &&
+             VecVT.getVectorElementType() == ScalarVT &&
+             (!LegalTypes ||
+              TLI.isTypeLegal(
+                  VecOp.getOperand(0).getValueType().getVectorElementType()))) {
     // extract_vector_elt (concat_vectors v2i16:a, v2i16:b), 0
     //      -> extract_vector_elt a, 0
     // extract_vector_elt (concat_vectors v2i16:a, v2i16:b), 1
@@ -17869,6 +18018,11 @@ SDValue DAGCombiner::reduceBuildVecExtToExtBuildVec(SDNode *N) {
   if (!ValidTypes)
     return SDValue();
 
+  // If we already have a splat buildvector, then don't fold it if it means
+  // introducing zeros.
+  if (!AllAnyExt && DAG.isSplatValue(SDValue(N, 0), /*AllowUndefs*/ true))
+    return SDValue();
+
   bool isLE = DAG.getDataLayout().isLittleEndian();
   unsigned ElemRatio = OutScalarTy.getSizeInBits()/SourceType.getSizeInBits();
   assert(ElemRatio > 1 && "Invalid element size ratio");
@@ -18238,6 +18392,9 @@ SDValue DAGCombiner::reduceBuildVecToShuffle(SDNode *N) {
       return SDValue();
     SDValue ExtractedFromVec = Op.getOperand(0);
 
+    if (ExtractedFromVec.getValueType().isScalableVector())
+      return SDValue();
+
     const APInt &ExtractIdx = Op.getConstantOperandAPInt(1);
     if (ExtractIdx.uge(ExtractedFromVec.getValueType().getVectorNumElements()))
       return SDValue();
@@ -19165,13 +19322,13 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
       V.getOperand(0).getValueType().isVector()) {
     SDValue SrcOp = V.getOperand(0);
     EVT SrcVT = SrcOp.getValueType();
-    unsigned SrcNumElts = SrcVT.getVectorNumElements();
-    unsigned DestNumElts = V.getValueType().getVectorNumElements();
+    unsigned SrcNumElts = SrcVT.getVectorMinNumElements();
+    unsigned DestNumElts = V.getValueType().getVectorMinNumElements();
     if ((SrcNumElts % DestNumElts) == 0) {
       unsigned SrcDestRatio = SrcNumElts / DestNumElts;
-      unsigned NewExtNumElts = NVT.getVectorNumElements() * SrcDestRatio;
+      ElementCount NewExtEC = NVT.getVectorElementCount() * SrcDestRatio;
       EVT NewExtVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
-                                      NewExtNumElts);
+                                      NewExtEC);
       if (TLI.isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, NewExtVT)) {
         SDLoc DL(N);
         SDValue NewIndex = DAG.getVectorIdxConstant(ExtIdx * SrcDestRatio, DL);
@@ -19182,14 +19339,14 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
     }
     if ((DestNumElts % SrcNumElts) == 0) {
       unsigned DestSrcRatio = DestNumElts / SrcNumElts;
-      if ((NVT.getVectorNumElements() % DestSrcRatio) == 0) {
-        unsigned NewExtNumElts = NVT.getVectorNumElements() / DestSrcRatio;
+      if ((NVT.getVectorMinNumElements() % DestSrcRatio) == 0) {
+        ElementCount NewExtEC = NVT.getVectorElementCount() / DestSrcRatio;
         EVT ScalarVT = SrcVT.getScalarType();
         if ((ExtIdx % DestSrcRatio) == 0) {
           SDLoc DL(N);
           unsigned IndexValScaled = ExtIdx / DestSrcRatio;
           EVT NewExtVT =
-              EVT::getVectorVT(*DAG.getContext(), ScalarVT, NewExtNumElts);
+              EVT::getVectorVT(*DAG.getContext(), ScalarVT, NewExtEC);
           if (TLI.isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, NewExtVT)) {
             SDValue NewIndex = DAG.getVectorIdxConstant(IndexValScaled, DL);
             SDValue NewExtract =
@@ -19197,7 +19354,7 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
                             V.getOperand(0), NewIndex);
             return DAG.getBitcast(NVT, NewExtract);
           }
-          if (NewExtNumElts == 1 &&
+          if (NewExtEC == 1 &&
               TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, ScalarVT)) {
             SDValue NewIndex = DAG.getVectorIdxConstant(IndexValScaled, DL);
             SDValue NewExtract =
@@ -19211,14 +19368,14 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
   }
 
   if (V.getOpcode() == ISD::CONCAT_VECTORS) {
-    unsigned ExtNumElts = NVT.getVectorNumElements();
+    unsigned ExtNumElts = NVT.getVectorMinNumElements();
     EVT ConcatSrcVT = V.getOperand(0).getValueType();
     assert(ConcatSrcVT.getVectorElementType() == NVT.getVectorElementType() &&
            "Concat and extract subvector do not change element type");
     assert((ExtIdx % ExtNumElts) == 0 &&
            "Extract index is not a multiple of the input vector length.");
 
-    unsigned ConcatSrcNumElts = ConcatSrcVT.getVectorNumElements();
+    unsigned ConcatSrcNumElts = ConcatSrcVT.getVectorMinNumElements();
     unsigned ConcatOpIdx = ExtIdx / ConcatSrcNumElts;
 
     // If the concatenated source types match this extract, it's a direct
@@ -19232,7 +19389,7 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
     // concat operand. Example:
     //   v2i8 extract_subvec (v16i8 concat (v8i8 X), (v8i8 Y), 14 -->
     //   v2i8 extract_subvec v8i8 Y, 6
-    if (ConcatSrcNumElts % ExtNumElts == 0) {
+    if (NVT.isFixedLengthVector() && ConcatSrcNumElts % ExtNumElts == 0) {
       SDLoc DL(N);
       unsigned NewExtIdx = ExtIdx - ConcatOpIdx * ConcatSrcNumElts;
       assert(NewExtIdx + ExtNumElts <= ConcatSrcNumElts &&
@@ -20214,7 +20371,9 @@ SDValue DAGCombiner::visitSCALAR_TO_VECTOR(SDNode *N) {
 
   // Replace a SCALAR_TO_VECTOR(EXTRACT_VECTOR_ELT(V,C0)) pattern
   // with a VECTOR_SHUFFLE and possible truncate.
-  if (InVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
+  if (InVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+      VT.isFixedLengthVector() &&
+      InVal->getOperand(0).getValueType().isFixedLengthVector()) {
     SDValue InVec = InVal->getOperand(0);
     SDValue EltNo = InVal->getOperand(1);
     auto InVecT = InVec.getValueType();
@@ -21793,10 +21952,10 @@ bool operator!=(const UnitT &, const UnitT &) { return false; }
 // redundant, as this function gets called when visiting every store
 // node, so why not let the work be done on each store as it's visited?
 //
-// I believe this is mainly important because MergeConsecutiveStores
+// I believe this is mainly important because mergeConsecutiveStores
 // is unable to deal with merging stores of different sizes, so unless
 // we improve the chains of all the potential candidates up-front
-// before running MergeConsecutiveStores, it might only see some of
+// before running mergeConsecutiveStores, it might only see some of
 // the nodes that will eventually be candidates, and then not be able
 // to go from a partially-merged state to the desired final
 // fully-merged state.
diff --git a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
index 7a5fd7d24c681..5cf83cff3a903 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
@@ -190,7 +190,7 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf,
           // stack allocation for each one.
           // Inform the Frame Information that we have variable-sized objects.
           MF->getFrameInfo().CreateVariableSizedObject(
-              Alignment <= StackAlign ? 0 : Alignment.value(), AI);
+              Alignment <= StackAlign ? Align(1) : Alignment, AI);
         }
       }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 9969786d8d43a..6a6004c158bb8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1390,12 +1390,17 @@ SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
 }
 
 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
+  assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
+          Node->getOpcode() == ISD::CONCAT_VECTORS) &&
+         "Unexpected opcode!");
+
   // We can't handle this case efficiently.  Allocate a sufficiently
-  // aligned object on the stack, store each element into it, then load
+  // aligned object on the stack, store each operand into it, then load
   // the result as a vector.
   // Create the stack frame object.
   EVT VT = Node->getValueType(0);
-  EVT EltVT = VT.getVectorElementType();
+  EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType()
+                                           : Node->getOperand(0).getValueType();
   SDLoc dl(Node);
   SDValue FIPtr = DAG.CreateStackTemporary(VT);
   int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
@@ -1404,7 +1409,7 @@ SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
 
   // Emit a store of each element to the stack slot.
   SmallVector<SDValue, 8> Stores;
-  unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
+  unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
   assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
   // Store (in the right endianness) the elements to memory.
   for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
@@ -1417,11 +1422,11 @@ SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
 
     // If the destination vector element type is narrower than the source
     // element type, only store the bits necessary.
-    if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
+    if (MemVT.bitsLT(Node->getOperand(i).getValueType()))
       Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
                                          Node->getOperand(i), Idx,
-                                         PtrInfo.getWithOffset(Offset), EltVT));
-    } else
+                                         PtrInfo.getWithOffset(Offset), MemVT));
+    else
       Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
                                     Idx, PtrInfo.getWithOffset(Offset)));
   }
@@ -1599,17 +1604,17 @@ void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
   SDValue Size  = Tmp2.getOperand(1);
   SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
   Chain = SP.getValue(1);
-  unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
+  Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
   const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
   unsigned Opc =
     TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
     ISD::ADD : ISD::SUB;
 
-  unsigned StackAlign = TFL->getStackAlignment();
+  Align StackAlign = TFL->getStackAlign();
   Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size);       // Value
-  if (Align > StackAlign)
+  if (Alignment > StackAlign)
     Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
-                       DAG.getConstant(-(uint64_t)Align, dl, VT));
+                       DAG.getConstant(-Alignment.value(), dl, VT));
   Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);     // Output chain
 
   Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
@@ -3343,26 +3348,10 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     break;
   }
   case ISD::UREM:
-  case ISD::SREM: {
-    EVT VT = Node->getValueType(0);
-    bool isSigned = Node->getOpcode() == ISD::SREM;
-    unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
-    unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
-    Tmp2 = Node->getOperand(0);
-    Tmp3 = Node->getOperand(1);
-    if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
-      SDVTList VTs = DAG.getVTList(VT, VT);
-      Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
-      Results.push_back(Tmp1);
-    } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
-      // X % Y -> X-X/Y*Y
-      Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
-      Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
-      Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
+  case ISD::SREM:
+    if (TLI.expandREM(Node, Tmp1, DAG))
       Results.push_back(Tmp1);
-    }
     break;
-  }
   case ISD::UDIV:
   case ISD::SDIV: {
     bool isSigned = Node->getOpcode() == ISD::SDIV;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 873812d074d9c..74071f763dbf0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -1723,7 +1723,14 @@ SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
   SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
   SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
   NewOps[OpNo] = Mask;
-  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
+  SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
+  if (Res == N)
+    return SDValue(Res, 0);
+
+  // Update triggered CSE, do our own replacement since caller can't.
+  ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
+  ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
+  return SDValue();
 }
 
 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
@@ -1744,7 +1751,14 @@ SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
   } else
     NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
 
-  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
+  SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
+  if (Res == N)
+    return SDValue(Res, 0);
+
+  // Update triggered CSE, do our own replacement since caller can't.
+  ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
+  ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
+  return SDValue();
 }
 
 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
@@ -4320,7 +4334,6 @@ SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
   EVT OutVT = N->getValueType(0);
   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
   assert(NOutVT.isVector() && "This type must be promoted to a vector type");
-  unsigned OutNumElems = OutVT.getVectorNumElements();
   EVT NOutVTElem = NOutVT.getVectorElementType();
 
   SDLoc dl(N);
@@ -4357,6 +4370,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
 
   EVT InVT = InOp0.getValueType();
 
+  unsigned OutNumElems = OutVT.getVectorNumElements();
   SmallVector<SDValue, 8> Ops;
   Ops.reserve(OutNumElems);
   for (unsigned i = 0; i != OutNumElems; ++i) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
index 2e1377c2c1735..ae087d3bbd8cb 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
@@ -835,9 +835,9 @@ void DAGTypeLegalizer::GetSplitVector(SDValue Op, SDValue &Lo,
 void DAGTypeLegalizer::SetSplitVector(SDValue Op, SDValue Lo,
                                       SDValue Hi) {
   assert(Lo.getValueType().getVectorElementType() ==
-         Op.getValueType().getVectorElementType() &&
-         2*Lo.getValueType().getVectorNumElements() ==
-         Op.getValueType().getVectorNumElements() &&
+             Op.getValueType().getVectorElementType() &&
+         Lo.getValueType().getVectorElementCount() * 2 ==
+             Op.getValueType().getVectorElementCount() &&
          Hi.getValueType() == Lo.getValueType() &&
          "Invalid type for split vector");
   // Lo/Hi may have been newly allocated, if so, add nodeid's as relevant.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 93ce338ff2327..6409f924920dd 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -145,6 +145,7 @@ class VectorLegalizer {
   void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
   SDValue ExpandStrictFPOp(SDNode *Node);
   void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
+  void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
 
   void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
 
@@ -867,6 +868,10 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
   case ISD::VECREDUCE_FMIN:
     Results.push_back(TLI.expandVecReduce(Node, DAG));
     return;
+  case ISD::SREM:
+  case ISD::UREM:
+    ExpandREM(Node, Results);
+    return;
   }
 
   Results.push_back(DAG.UnrollVectorOp(Node));
@@ -1353,6 +1358,17 @@ void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
   UnrollStrictFPOp(Node, Results);
 }
 
+void VectorLegalizer::ExpandREM(SDNode *Node,
+                                SmallVectorImpl<SDValue> &Results) {
+  assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) &&
+         "Expected REM node");
+
+  SDValue Result;
+  if (!TLI.expandREM(Node, Result, DAG))
+    Result = DAG.UnrollVectorOp(Node);
+  Results.push_back(Result);
+}
+
 void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
                                        SmallVectorImpl<SDValue> &Results) {
   EVT VT = Node->getValueType(0);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 297b8aa3e848f..550174f0df72a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -1537,11 +1537,22 @@ void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
                    LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(),
                    MMOFlags, AAInfo);
 
-  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
-  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
-  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
-                   LD->getPointerInfo().getWithOffset(IncrementSize), HiMemVT,
-                   LD->getOriginalAlign(), MMOFlags, AAInfo);
+  unsigned IncrementSize = LoMemVT.getSizeInBits().getKnownMinSize() / 8;
+
+  MachinePointerInfo MPI;
+  if (LoVT.isScalableVector()) {
+    SDValue BytesIncrement = DAG.getVScale(
+        dl, Ptr.getValueType(),
+        APInt(Ptr.getValueSizeInBits().getFixedSize(), IncrementSize));
+    MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
+    Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, BytesIncrement);
+  } else {
+    MPI = LD->getPointerInfo().getWithOffset(IncrementSize);
+    Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
+  }
+
+  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI,
+                   HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
 
   // Build a factor node to remember that this load is independent of the
   // other one.
@@ -1767,8 +1778,7 @@ void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
   // more effectively move in the right direction and prevent falling down
   // to scalarization in many cases due to the input vector being split too
   // far.
-  unsigned NumElements = SrcVT.getVectorNumElements();
-  if ((NumElements & 1) == 0 &&
+  if ((SrcVT.getVectorMinNumElements() & 1) == 0 &&
       SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) {
     LLVMContext &Ctx = *DAG.getContext();
     EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
@@ -1912,11 +1922,11 @@ void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
   SDValue SV = N->getOperand(2);
   SDLoc dl(N);
 
-  const unsigned Alignment = DAG.getDataLayout().getABITypeAlignment(
-      NVT.getTypeForEVT(*DAG.getContext()));
+  const Align Alignment =
+      DAG.getDataLayout().getABITypeAlign(NVT.getTypeForEVT(*DAG.getContext()));
 
-  Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment);
-  Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment);
+  Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value());
+  Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value());
   Chain = Hi.getValue(1);
 
   // Modified the chain - switch anything that used the old chain to use
@@ -2479,7 +2489,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
   if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
     return TLI.scalarizeVectorStore(N, DAG);
 
-  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
+  unsigned IncrementSize = LoMemVT.getSizeInBits().getKnownMinSize() / 8;
 
   if (isTruncating)
     Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
@@ -2488,17 +2498,24 @@ SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
     Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
                       AAInfo);
 
-  // Increment the pointer to the other half.
-  Ptr = DAG.getObjectPtrOffset(DL, Ptr, IncrementSize);
+  MachinePointerInfo MPI;
+  if (LoMemVT.isScalableVector()) {
+    SDValue BytesIncrement = DAG.getVScale(
+        DL, Ptr.getValueType(),
+        APInt(Ptr.getValueSizeInBits().getFixedSize(), IncrementSize));
+    MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
+    Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement);
+  } else {
+    MPI = N->getPointerInfo().getWithOffset(IncrementSize);
+    // Increment the pointer to the other half.
+    Ptr = DAG.getObjectPtrOffset(DL, Ptr, IncrementSize);
+  }
 
   if (isTruncating)
-    Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
-                           N->getPointerInfo().getWithOffset(IncrementSize),
+    Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI,
                            HiMemVT, Alignment, MMOFlags, AAInfo);
   else
-    Hi = DAG.getStore(Ch, DL, Hi, Ptr,
-                      N->getPointerInfo().getWithOffset(IncrementSize),
-                      Alignment, MMOFlags, AAInfo);
+    Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo);
 
   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
 }
@@ -2593,9 +2610,9 @@ SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
   SDValue Chain;
   if (N->isStrictFPOpcode()) {
     HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
-                         {N->getOperand(0), HalfLo});
+                         {N->getOperand(0), InLoVec});
     HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
-                         {N->getOperand(0), HalfHi});
+                         {N->getOperand(0), InHiVec});
     // Legalize the chain result - switch anything that used the old chain to
     // use the new one.
     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
@@ -2639,9 +2656,11 @@ SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
   SDLoc DL(N);
   GetSplitVector(N->getOperand(0), Lo0, Hi0);
   GetSplitVector(N->getOperand(1), Lo1, Hi1);
-  unsigned PartElements = Lo0.getValueType().getVectorNumElements();
-  EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
-  EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
+  auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
+
+  LLVMContext &Context = *DAG.getContext();
+  EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
+  EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
 
   LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
   HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index 731cd2396b626..ce20d506586f0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -1033,57 +1033,6 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
     }
   }
 
-  // Split after an INLINEASM_BR block with outputs. This allows us to keep the
-  // copy to/from register instructions from being between two terminator
-  // instructions, which causes the machine instruction verifier agita.
-  auto TI = llvm::find_if(*BB, [](const MachineInstr &MI){
-    return MI.getOpcode() == TargetOpcode::INLINEASM_BR;
-  });
-  auto SplicePt = TI != BB->end() ? std::next(TI) : BB->end();
-  if (TI != BB->end() && SplicePt != BB->end() &&
-      TI->getOpcode() == TargetOpcode::INLINEASM_BR &&
-      SplicePt->getOpcode() == TargetOpcode::COPY) {
-    MachineBasicBlock *FallThrough = BB->getFallThrough();
-    if (!FallThrough)
-      for (const MachineOperand &MO : BB->back().operands())
-        if (MO.isMBB()) {
-          FallThrough = MO.getMBB();
-          break;
-        }
-    assert(FallThrough && "Cannot find default dest block for callbr!");
-
-    MachineBasicBlock *CopyBB = MF.CreateMachineBasicBlock(BB->getBasicBlock());
-    MachineFunction::iterator BBI(*BB);
-    MF.insert(++BBI, CopyBB);
-
-    CopyBB->splice(CopyBB->begin(), BB, SplicePt, BB->end());
-    CopyBB->setInlineAsmBrDefaultTarget();
-
-    CopyBB->addSuccessor(FallThrough, BranchProbability::getOne());
-    BB->removeSuccessor(FallThrough);
-    BB->addSuccessor(CopyBB, BranchProbability::getOne());
-
-    // Mark all physical registers defined in the original block as being live
-    // on entry to the copy block.
-    for (const auto &MI : *CopyBB)
-      for (const MachineOperand &MO : MI.operands())
-        if (MO.isReg()) {
-          Register reg = MO.getReg();
-          if (Register::isPhysicalRegister(reg)) {
-            CopyBB->addLiveIn(reg);
-            break;
-          }
-        }
-
-    CopyBB->normalizeSuccProbs();
-    BB->normalizeSuccProbs();
-
-    BB->transferInlineAsmBrIndirectTargets(CopyBB);
-
-    InsertPos = CopyBB->end();
-    return CopyBB;
-  }
-
   InsertPos = Emitter.getInsertPos();
   return Emitter.getBlock();
 }
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 9ea86de65bc1f..8065091208694 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -3166,6 +3166,15 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
     Known.One  &= (~Known.Zero);
     break;
   }
+  case ISD::AssertAlign: {
+    unsigned LogOfAlign = Log2(cast<AssertAlignSDNode>(Op)->getAlign());
+    assert(LogOfAlign != 0);
+    // If a node is guaranteed to be aligned, set low zero bits accordingly as
+    // well as clearing one bits.
+    Known.Zero.setLowBits(LogOfAlign);
+    Known.One.clearLowBits(LogOfAlign);
+    break;
+  }
   case ISD::FGETSIGN:
     // All bits are zero except the low bit.
     Known.Zero.setBitsFrom(1);
@@ -4545,7 +4554,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
       // FIXME need to be more flexible about rounding mode.
       (void)V.convert(APFloat::IEEEhalf(),
                       APFloat::rmNearestTiesToEven, &Ignored);
-      return getConstant(V.bitcastToAPInt(), DL, VT);
+      return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
     }
     }
   }
@@ -4793,6 +4802,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     if (OpOpcode == ISD::FNEG)  // abs(-X) -> abs(X)
       return getNode(ISD::FABS, DL, VT, Operand.getOperand(0));
     break;
+  case ISD::VSCALE:
+    assert(VT == Operand.getValueType() && "Unexpected VT!");
+    break;
   }
 
   SDNode *N;
@@ -5186,6 +5198,34 @@ SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
   return SDValue();
 }
 
+SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
+  assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
+
+  // There's no need to assert on a byte-aligned pointer. All pointers are at
+  // least byte aligned.
+  if (A == Align(1))
+    return Val;
+
+  FoldingSetNodeID ID;
+  AddNodeIDNode(ID, ISD::AssertAlign, getVTList(Val.getValueType()), {Val});
+  ID.AddInteger(A.value());
+
+  void *IP = nullptr;
+  if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
+    return SDValue(E, 0);
+
+  auto *N = newSDNode<AssertAlignSDNode>(DL.getIROrder(), DL.getDebugLoc(),
+                                         Val.getValueType(), A);
+  createOperands(N, {Val});
+
+  CSEMap.InsertNode(N, IP);
+  InsertNode(N);
+
+  SDValue V(N, 0);
+  NewSDValueDbgMsg(V, "Creating new node: ", this);
+  return V;
+}
+
 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                               SDValue N1, SDValue N2, const SDNodeFlags Flags) {
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
@@ -5529,7 +5569,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     // the concat have the same type as the extract.
     if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
         N1.getNumOperands() > 0 && VT == N1.getOperand(0).getValueType()) {
-      unsigned Factor = VT.getVectorNumElements();
+      unsigned Factor = VT.getVectorMinNumElements();
       return N1.getOperand(N2C->getZExtValue() / Factor);
     }
 
@@ -6374,8 +6414,8 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
   // code. If the target chooses to do this, this is the next best.
   if (TSI) {
     SDValue Result = TSI->EmitTargetCodeForMemcpy(
-        *this, dl, Chain, Dst, Src, Size, Alignment.value(), isVol,
-        AlwaysInline, DstPtrInfo, SrcPtrInfo);
+        *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline,
+        DstPtrInfo, SrcPtrInfo);
     if (Result.getNode())
       return Result;
   }
@@ -6487,9 +6527,9 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
   // Then check to see if we should lower the memmove with target-specific
   // code. If the target chooses to do this, this is the next best.
   if (TSI) {
-    SDValue Result = TSI->EmitTargetCodeForMemmove(
-        *this, dl, Chain, Dst, Src, Size, Alignment.value(), isVol, DstPtrInfo,
-        SrcPtrInfo);
+    SDValue Result =
+        TSI->EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size,
+                                      Alignment, isVol, DstPtrInfo, SrcPtrInfo);
     if (Result.getNode())
       return Result;
   }
@@ -6590,7 +6630,7 @@ SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
   // code. If the target chooses to do this, this is the next best.
   if (TSI) {
     SDValue Result = TSI->EmitTargetCodeForMemset(
-        *this, dl, Chain, Dst, Src, Size, Alignment.value(), isVol, DstPtrInfo);
+        *this, dl, Chain, Dst, Src, Size, Alignment, isVol, DstPtrInfo);
     if (Result.getNode())
       return Result;
   }
@@ -9344,6 +9384,28 @@ SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
     PrevOp = Op;
   }
 
+  // Handle subvector reductions, which tend to appear after the shuffle
+  // reduction stages.
+  while (Op.getOpcode() == CandidateBinOp) {
+    unsigned NumElts = Op.getValueType().getVectorNumElements();
+    SDValue Op0 = Op.getOperand(0);
+    SDValue Op1 = Op.getOperand(1);
+    if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
+        Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
+        Op0.getOperand(0) != Op1.getOperand(0))
+      break;
+    SDValue Src = Op0.getOperand(0);
+    unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
+    if (NumSrcElts != (2 * NumElts))
+      break;
+    if (!(Op0.getConstantOperandAPInt(1) == 0 &&
+          Op1.getConstantOperandAPInt(1) == NumElts) &&
+        !(Op1.getConstantOperandAPInt(1) == 0 &&
+          Op0.getConstantOperandAPInt(1) == NumElts))
+      break;
+    Op = Src;
+  }
+
   BinOp = (ISD::NodeType)CandidateBinOp;
   return Op;
 }
@@ -9577,14 +9639,22 @@ SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
 std::pair<SDValue, SDValue>
 SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
                           const EVT &HiVT) {
-  assert(LoVT.getVectorNumElements() + HiVT.getVectorNumElements() <=
-         N.getValueType().getVectorNumElements() &&
+  assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
+         LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
+         "Splitting vector with an invalid mixture of fixed and scalable "
+         "vector types");
+  assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
+             N.getValueType().getVectorMinNumElements() &&
          "More vector elements requested than available!");
   SDValue Lo, Hi;
   Lo =
       getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, getVectorIdxConstant(0, DL));
+  // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
+  // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
+  // IDX with the runtime scaling factor of the result vector type. For
+  // fixed-width result vectors, that runtime scaling factor is 1.
   Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
-               getVectorIdxConstant(LoVT.getVectorNumElements(), DL));
+               getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
   return std::make_pair(Lo, Hi);
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 1c6843489b7d5..c8b72abb9b7d6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -135,6 +135,11 @@ using namespace SwitchCG;
 /// some float libcalls (6, 8 or 12 bits).
 static unsigned LimitFloatPrecision;
 
+static cl::opt<bool>
+    InsertAssertAlign("insert-assert-align", cl::init(true),
+                      cl::desc("Insert the experimental `assertalign` node."),
+                      cl::ReallyHidden);
+
 static cl::opt<unsigned, true>
     LimitFPPrecision("limit-float-precision",
                      cl::desc("Generate low-precision inline sequences "
@@ -651,7 +656,7 @@ static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
 
 static SDValue widenVectorToPartType(SelectionDAG &DAG,
                                      SDValue Val, const SDLoc &DL, EVT PartVT) {
-  if (!PartVT.isVector())
+  if (!PartVT.isFixedLengthVector())
     return SDValue();
 
   EVT ValueVT = Val.getValueType();
@@ -697,8 +702,9 @@ static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
       Val = Widened;
     } else if (PartVT.isVector() &&
                PartEVT.getVectorElementType().bitsGE(
-                 ValueVT.getVectorElementType()) &&
-               PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
+                   ValueVT.getVectorElementType()) &&
+               PartEVT.getVectorElementCount() ==
+                   ValueVT.getVectorElementCount()) {
 
       // Promoted vector extract
       Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
@@ -740,12 +746,16 @@ static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
   NumParts = NumRegs; // Silence a compiler warning.
   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
 
-  unsigned IntermediateNumElts = IntermediateVT.isVector() ?
-    IntermediateVT.getVectorNumElements() : 1;
+  assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
+         "Mixing scalable and fixed vectors when copying in parts");
+
+  ElementCount DestEltCnt;
+
+  if (IntermediateVT.isVector())
+    DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
+  else
+    DestEltCnt = ElementCount(NumIntermediates, false);
 
-  // Convert the vector to the appropriate type if necessary.
-  auto DestEltCnt = ElementCount(NumIntermediates * IntermediateNumElts,
-                                 ValueVT.isScalableVector());
   EVT BuiltVectorTy = EVT::getVectorVT(
       *DAG.getContext(), IntermediateVT.getScalarType(), DestEltCnt);
   if (ValueVT != BuiltVectorTy) {
@@ -759,6 +769,9 @@ static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
   SmallVector<SDValue, 8> Ops(NumIntermediates);
   for (unsigned i = 0; i != NumIntermediates; ++i) {
     if (IntermediateVT.isVector()) {
+      // This does something sensible for scalable vectors - see the
+      // definition of EXTRACT_SUBVECTOR for further details.
+      unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
       Ops[i] =
           DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
                       DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
@@ -2291,7 +2304,9 @@ void SelectionDAGBuilder::visitBr(const BranchInst &I) {
 
   // If this is a series of conditions that are or'd or and'd together, emit
   // this as a sequence of branches instead of setcc's with and/or operations.
-  // As long as jumps are not expensive, this should improve performance.
+  // As long as jumps are not expensive (exceptions for multi-use logic ops,
+  // unpredictable branches, and vector extracts because those jumps are likely
+  // expensive for any target), this should improve performance.
   // For example, instead of something like:
   //     cmp A, B
   //     C = seteq
@@ -2306,9 +2321,12 @@ void SelectionDAGBuilder::visitBr(const BranchInst &I) {
   //     jle foo
   if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
     Instruction::BinaryOps Opcode = BOp->getOpcode();
+    Value *Vec, *BOp0 = BOp->getOperand(0), *BOp1 = BOp->getOperand(1);
     if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
         !I.hasMetadata(LLVMContext::MD_unpredictable) &&
-        (Opcode == Instruction::And || Opcode == Instruction::Or)) {
+        (Opcode == Instruction::And || Opcode == Instruction::Or) &&
+        !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
+          match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
                            Opcode,
                            getEdgeProbability(BrMBB, Succ0MBB),
@@ -2650,6 +2668,11 @@ SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
   // Passing 'true' for doesNotReturn above won't generate the trap for us.
   if (TM.getTargetTriple().isPS4CPU())
     Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
+  // WebAssembly needs an unreachable instruction after a non-returning call,
+  // because the function return type can be different from __stack_chk_fail's
+  // return type (void).
+  if (TM.getTargetTriple().isWasm())
+    Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
 
   DAG.setRoot(Chain);
 }
@@ -2880,14 +2903,13 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
 
   // Retrieve successors.
   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
-  Return->setInlineAsmBrDefaultTarget();
 
   // Update successor info.
   addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
   for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
     MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)];
     addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
-    CallBrMBB->addInlineAsmBrIndirectTarget(Target);
+    Target->setIsInlineAsmBrIndirectTarget();
   }
   CallBrMBB->normalizeSuccProbs();
 
@@ -2960,16 +2982,6 @@ void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
   for (unsigned i = 0, e = SL->BitTestCases.size(); i != e; ++i)
     if (SL->BitTestCases[i].Parent == First)
       SL->BitTestCases[i].Parent = Last;
-
-  // SelectionDAGISel::FinishBasicBlock will add PHI operands for the
-  // successors of the fallthrough block. Here, we add PHI operands for the
-  // successors of the INLINEASM_BR block itself.
-  if (First->getFirstTerminator()->getOpcode() == TargetOpcode::INLINEASM_BR)
-    for (std::pair<MachineInstr *, unsigned> &pair : FuncInfo.PHINodesToUpdate)
-      if (First->isSuccessor(pair.first->getParent()))
-        MachineInstrBuilder(*First->getParent(), pair.first)
-            .addReg(pair.second)
-            .addMBB(First);
 }
 
 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
@@ -4208,8 +4220,7 @@ void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
     // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
     Src0 = I.getArgOperand(0);
     Ptr = I.getArgOperand(1);
-    Alignment =
-        MaybeAlign(cast<ConstantInt>(I.getArgOperand(2))->getZExtValue());
+    Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue();
     Mask = I.getArgOperand(3);
   };
   auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
@@ -4284,7 +4295,7 @@ static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
 
     Base = SDB->getValue(C);
 
-    unsigned NumElts = cast<VectorType>(Ptr->getType())->getNumElements();
+    unsigned NumElts = cast<FixedVectorType>(Ptr->getType())->getNumElements();
     EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
     Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
     IndexType = ISD::SIGNED_SCALED;
@@ -4323,9 +4334,9 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
   SDValue Src0 = getValue(I.getArgOperand(0));
   SDValue Mask = getValue(I.getArgOperand(3));
   EVT VT = Src0.getValueType();
-  MaybeAlign Alignment(cast<ConstantInt>(I.getArgOperand(2))->getZExtValue());
-  if (!Alignment)
-    Alignment = DAG.getEVTAlign(VT);
+  Align Alignment = cast<ConstantInt>(I.getArgOperand(2))
+                        ->getMaybeAlignValue()
+                        .getValueOr(DAG.getEVTAlign(VT));
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   AAMDNodes AAInfo;
@@ -4343,7 +4354,7 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
       MachinePointerInfo(AS), MachineMemOperand::MOStore,
       // TODO: Make MachineMemOperands aware of scalable
       // vectors.
-      MemoryLocation::UnknownSize, *Alignment, AAInfo);
+      MemoryLocation::UnknownSize, Alignment, AAInfo);
   if (!UniformBase) {
     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
     Index = getValue(Ptr);
@@ -4364,8 +4375,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
                               MaybeAlign &Alignment) {
     // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
     Ptr = I.getArgOperand(0);
-    Alignment =
-        MaybeAlign(cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
+    Alignment = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();
     Mask = I.getArgOperand(2);
     Src0 = I.getArgOperand(3);
   };
@@ -4434,9 +4444,9 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
-  MaybeAlign Alignment(cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
-  if (!Alignment)
-    Alignment = DAG.getEVTAlign(VT);
+  Align Alignment = cast<ConstantInt>(I.getArgOperand(1))
+                        ->getMaybeAlignValue()
+                        .getValueOr(DAG.getEVTAlign(VT));
 
   AAMDNodes AAInfo;
   I.getAAMetadata(AAInfo);
@@ -4454,7 +4464,7 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
       MachinePointerInfo(AS), MachineMemOperand::MOLoad,
       // TODO: Make MachineMemOperands aware of scalable
       // vectors.
-      MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
+      MemoryLocation::UnknownSize, Alignment, AAInfo, Ranges);
 
   if (!UniformBase) {
     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
@@ -4747,6 +4757,15 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
     } else
       Result = lowerRangeToAssertZExt(DAG, I, Result);
 
+    MaybeAlign Alignment = I.getRetAlign();
+    if (!Alignment)
+      Alignment = F->getAttributes().getRetAlignment();
+    // Insert `assertalign` node if there's an alignment.
+    if (InsertAssertAlign && Alignment) {
+      Result =
+          DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
+    }
+
     setValue(&I, Result);
   }
 }
@@ -6900,6 +6919,39 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
                              DAG.getZExtOrTrunc(Const, getCurSDLoc(), PtrVT)));
     return;
   }
+  case Intrinsic::get_active_lane_mask: {
+    auto DL = getCurSDLoc();
+    SDValue Index = getValue(I.getOperand(0));
+    SDValue BTC = getValue(I.getOperand(1));
+    Type *ElementTy = I.getOperand(0)->getType();
+    EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
+    unsigned VecWidth = VT.getVectorNumElements();
+
+    SmallVector<SDValue, 16> OpsBTC;
+    SmallVector<SDValue, 16> OpsIndex;
+    SmallVector<SDValue, 16> OpsStepConstants;
+    for (unsigned i = 0; i < VecWidth; i++) {
+      OpsBTC.push_back(BTC);
+      OpsIndex.push_back(Index);
+      OpsStepConstants.push_back(DAG.getConstant(i, DL, MVT::getVT(ElementTy)));
+    }
+
+    EVT CCVT = MVT::i1;
+    CCVT = EVT::getVectorVT(I.getContext(), CCVT, VecWidth);
+
+    auto VecTy = MVT::getVT(FixedVectorType::get(ElementTy, VecWidth));
+    SDValue VectorIndex = DAG.getBuildVector(VecTy, DL, OpsIndex);
+    SDValue VectorStep = DAG.getBuildVector(VecTy, DL, OpsStepConstants);
+    SDValue VectorInduction = DAG.getNode(
+       ISD::UADDO, DL, DAG.getVTList(VecTy, CCVT), VectorIndex, VectorStep);
+    SDValue VectorBTC = DAG.getBuildVector(VecTy, DL, OpsBTC);
+    SDValue SetCC = DAG.getSetCC(DL, CCVT, VectorInduction.getValue(0),
+                                 VectorBTC, ISD::CondCode::SETULE);
+    setValue(&I, DAG.getNode(ISD::AND, DL, CCVT,
+                             DAG.getNOT(DL, VectorInduction.getValue(1), CCVT),
+                             SetCC));
+    return;
+  }
   }
 }
 
@@ -7798,7 +7850,6 @@ class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
   }
 };
 
-using SDISelAsmOperandInfoVector = SmallVector<SDISelAsmOperandInfo, 16>;
 
 } // end anonymous namespace
 
@@ -7860,9 +7911,9 @@ static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
   Type *Ty = OpVal->getType();
   auto &DL = DAG.getDataLayout();
   uint64_t TySize = DL.getTypeAllocSize(Ty);
-  unsigned Align = DL.getPrefTypeAlignment(Ty);
   MachineFunction &MF = DAG.getMachineFunction();
-  int SSFI = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
+  int SSFI = MF.getFrameInfo().CreateStackObject(
+      TySize, DL.getPrefTypeAlign(Ty), false);
   SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
   Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
                             MachinePointerInfo::getFixedStack(MF, SSFI),
@@ -8044,7 +8095,7 @@ void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call) {
   const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
 
   /// ConstraintOperands - Information about all of the constraints.
-  SDISelAsmOperandInfoVector ConstraintOperands;
+  SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
@@ -8570,7 +8621,7 @@ void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
   SDValue V = DAG.getVAArg(
       TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),
       getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),
-      DL.getABITypeAlignment(I.getType()));
+      DL.getABITypeAlign(I.getType()).value());
   DAG.setRoot(V.getValue(1));
 
   if (I.getType()->isPointerTy())
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 7f9b8b7b28a38..42e3016e65b83 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -106,6 +106,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::TokenFactor:                return "TokenFactor";
   case ISD::AssertSext:                 return "AssertSext";
   case ISD::AssertZext:                 return "AssertZext";
+  case ISD::AssertAlign:                return "AssertAlign";
 
   case ISD::BasicBlock:                 return "BasicBlock";
   case ISD::VALUETYPE:                  return "ValueType";
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 733762339d612..1f0432196a2d7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -661,36 +661,6 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
   // Determine if floating point is used for msvc
   computeUsesMSVCFloatingPoint(TM.getTargetTriple(), Fn, MF->getMMI());
 
-  // Replace forward-declared registers with the registers containing
-  // the desired value.
-  for (DenseMap<Register, Register>::iterator
-       I = FuncInfo->RegFixups.begin(), E = FuncInfo->RegFixups.end();
-       I != E; ++I) {
-    Register From = I->first;
-    Register To = I->second;
-    // If To is also scheduled to be replaced, find what its ultimate
-    // replacement is.
-    while (true) {
-      DenseMap<Register, Register>::iterator J = FuncInfo->RegFixups.find(To);
-      if (J == E) break;
-      To = J->second;
-    }
-    // Make sure the new register has a sufficiently constrained register class.
-    if (Register::isVirtualRegister(From) && Register::isVirtualRegister(To))
-      MRI.constrainRegClass(To, MRI.getRegClass(From));
-    // Replace it.
-
-
-    // Replacing one register with another won't touch the kill flags.
-    // We need to conservatively clear the kill flags as a kill on the old
-    // register might dominate existing uses of the new register.
-    if (!MRI.use_empty(To))
-      MRI.clearKillFlags(From);
-    MRI.replaceRegWith(From, To);
-  }
-
-  TLI->finalizeLowering(*MF);
-
   // Release function-specific state. SDB and CurDAG are already cleared
   // at this point.
   FuncInfo->clear();
@@ -2820,6 +2790,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
     return;
   case ISD::AssertSext:
   case ISD::AssertZext:
+  case ISD::AssertAlign:
     ReplaceUses(SDValue(NodeToMatch, 0), NodeToMatch->getOperand(0));
     CurDAG->RemoveDeadNode(NodeToMatch);
     return;
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index a55c7c8189a04..2cb57c1d1ccc8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -42,6 +42,7 @@
 #include "llvm/IR/Statepoint.h"
 #include "llvm/IR/Type.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/MachineValueType.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
@@ -220,6 +221,28 @@ static Optional<int> findPreviousSpillSlot(const Value *Val,
   return None;
 }
 
+
+/// Return true if-and-only-if the given SDValue can be lowered as either a
+/// constant argument or a stack reference.  The key point is that the value
+/// doesn't need to be spilled or tracked as a vreg use.
+static bool willLowerDirectly(SDValue Incoming) {
+  // We are making an unchecked assumption that the frame size <= 2^16 as that
+  // is the largest offset which can be encoded in the stackmap format.
+  if (isa<FrameIndexSDNode>(Incoming))
+    return true;
+
+  // The largest constant describeable in the StackMap format is 64 bits.
+  // Potential Optimization:  Constants values are sign extended by consumer,
+  // and thus there are many constants of static type > 64 bits whose value
+  // happens to be sext(Con64) and could thus be lowered directly.
+  if (Incoming.getValueType().getSizeInBits() > 64)
+    return false;
+
+  return (isa<ConstantSDNode>(Incoming) || isa<ConstantFPSDNode>(Incoming) ||
+          Incoming.isUndef());
+}
+
+
 /// Try to find existing copies of the incoming values in stack slots used for
 /// statepoint spilling.  If we can find a spill slot for the incoming value,
 /// mark that slot as allocated, and reuse the same slot for this safepoint.
@@ -229,12 +252,10 @@ static void reservePreviousStackSlotForValue(const Value *IncomingValue,
                                              SelectionDAGBuilder &Builder) {
   SDValue Incoming = Builder.getValue(IncomingValue);
 
-  if (isa<ConstantSDNode>(Incoming) || isa<ConstantFPSDNode>(Incoming) ||
-      isa<FrameIndexSDNode>(Incoming) || Incoming.isUndef()) {
-    // We won't need to spill this, so no need to check for previously
-    // allocated stack slots
+  // If we won't spill this, we don't need to check for previously allocated
+  // stack slots.
+  if (willLowerDirectly(Incoming))
     return;
-  }
 
   SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
   if (OldLocation.getNode())
@@ -383,45 +404,53 @@ lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot,
                              SmallVectorImpl<SDValue> &Ops,
                              SmallVectorImpl<MachineMemOperand *> &MemRefs,
                              SelectionDAGBuilder &Builder) {
-  // Note: We know all of these spills are independent, but don't bother to
-  // exploit that chain wise.  DAGCombine will happily do so as needed, so
-  // doing it here would be a small compile time win at most.
-  SDValue Chain = Builder.getRoot();
-
-  if (Incoming.isUndef() && Incoming.getValueType().getSizeInBits() <= 64) {
-    // Put an easily recognized constant that's unlikely to be a valid
-    // value so that uses of undef by the consumer of the stackmap is
-    // easily recognized. This is legal since the compiler is always
-    // allowed to chose an arbitrary value for undef.
-    pushStackMapConstant(Ops, Builder, 0xFEFEFEFE);
-    return;
+  
+  if (willLowerDirectly(Incoming)) {
+    if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
+      // This handles allocas as arguments to the statepoint (this is only
+      // really meaningful for a deopt value.  For GC, we'd be trying to
+      // relocate the address of the alloca itself?)
+      assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
+             "Incoming value is a frame index!");
+      Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
+                                                    Builder.getFrameIndexTy()));
+
+      auto &MF = Builder.DAG.getMachineFunction();
+      auto *MMO = getMachineMemOperand(MF, *FI);
+      MemRefs.push_back(MMO);
+      return;
+    }
+
+    assert(Incoming.getValueType().getSizeInBits() <= 64);
+    
+    if (Incoming.isUndef()) {
+      // Put an easily recognized constant that's unlikely to be a valid
+      // value so that uses of undef by the consumer of the stackmap is
+      // easily recognized. This is legal since the compiler is always
+      // allowed to chose an arbitrary value for undef.
+      pushStackMapConstant(Ops, Builder, 0xFEFEFEFE);
+      return;
+    }
+
+    // If the original value was a constant, make sure it gets recorded as
+    // such in the stackmap.  This is required so that the consumer can
+    // parse any internal format to the deopt state.  It also handles null
+    // pointers and other constant pointers in GC states.
+    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
+      pushStackMapConstant(Ops, Builder, C->getSExtValue());
+      return;
+    } else if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Incoming)) {
+      pushStackMapConstant(Ops, Builder,
+                           C->getValueAPF().bitcastToAPInt().getZExtValue());
+      return;
+    }
+
+    llvm_unreachable("unhandled direct lowering case");
   }
 
-  // If the original value was a constant, make sure it gets recorded as
-  // such in the stackmap.  This is required so that the consumer can
-  // parse any internal format to the deopt state.  It also handles null
-  // pointers and other constant pointers in GC states.  Note the constant
-  // vectors do not appear to actually hit this path and that anything larger
-  // than an i64 value (not type!) will fail asserts here.
-  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Incoming)) {
-    pushStackMapConstant(Ops, Builder,
-                         C->getValueAPF().bitcastToAPInt().getZExtValue());
-  } else if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
-    pushStackMapConstant(Ops, Builder, C->getSExtValue());
-  } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
-    // This handles allocas as arguments to the statepoint (this is only
-    // really meaningful for a deopt value.  For GC, we'd be trying to
-    // relocate the address of the alloca itself?)
-    assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
-           "Incoming value is a frame index!");
-    Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
-                                                  Builder.getFrameIndexTy()));
 
-    auto &MF = Builder.DAG.getMachineFunction();
-    auto *MMO = getMachineMemOperand(MF, *FI);
-    MemRefs.push_back(MMO);
 
-  } else if (!RequireSpillSlot) {
+  if (!RequireSpillSlot) {
     // If this value is live in (not live-on-return, or live-through), we can
     // treat it the same way patchpoint treats it's "live in" values.  We'll
     // end up folding some of these into stack references, but they'll be
@@ -431,20 +460,20 @@ lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot,
     // fix-up pass should be executed to force spilling of such registers.
     Ops.push_back(Incoming);
   } else {
-    // Otherwise, locate a spill slot and explicitly spill it so it
-    // can be found by the runtime later.  We currently do not support
-    // tracking values through callee saved registers to their eventual
-    // spill location.  This would be a useful optimization, but would
-    // need to be optional since it requires a lot of complexity on the
-    // runtime side which not all would support.
+    // Otherwise, locate a spill slot and explicitly spill it so it can be
+    // found by the runtime later.  Note: We know all of these spills are
+    // independent, but don't bother to exploit that chain wise.  DAGCombine
+    // will happily do so as needed, so doing it here would be a small compile
+    // time win at most. 
+    SDValue Chain = Builder.getRoot();
     auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder);
     Ops.push_back(std::get<0>(Res));
     if (auto *MMO = std::get<2>(Res))
       MemRefs.push_back(MMO);
     Chain = std::get<1>(Res);;
+    Builder.DAG.setRoot(Chain);
   }
 
-  Builder.DAG.setRoot(Chain);
 }
 
 /// Lower deopt state and gc pointer arguments of the statepoint.  The actual
@@ -884,36 +913,38 @@ SelectionDAGBuilder::LowerStatepoint(const GCStatepointInst &I,
   // Export the result value if needed
   const GCResultInst *GCResult = I.getGCResult();
   Type *RetTy = I.getActualReturnType();
-  if (!RetTy->isVoidTy() && GCResult) {
-    if (GCResult->getParent() != I.getParent()) {
-      // Result value will be used in a different basic block so we need to
-      // export it now.  Default exporting mechanism will not work here because
-      // statepoint call has a different type than the actual call. It means
-      // that by default llvm will create export register of the wrong type
-      // (always i32 in our case). So instead we need to create export register
-      // with correct type manually.
-      // TODO: To eliminate this problem we can remove gc.result intrinsics
-      //       completely and make statepoint call to return a tuple.
-      unsigned Reg = FuncInfo.CreateRegs(RetTy);
-      RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
-                       DAG.getDataLayout(), Reg, RetTy,
-                       I.getCallingConv());
-      SDValue Chain = DAG.getEntryNode();
-
-      RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
-      PendingExports.push_back(Chain);
-      FuncInfo.ValueMap[&I] = Reg;
-    } else {
-      // Result value will be used in a same basic block. Don't export it or
-      // perform any explicit register copies.
-      // We'll replace the actuall call node shortly. gc_result will grab
-      // this value.
-      setValue(&I, ReturnValue);
-    }
-  } else {
-    // The token value is never used from here on, just generate a poison value
+
+  if (RetTy->isVoidTy() || !GCResult) {
+    // The return value is not needed, just generate a poison value. 
     setValue(&I, DAG.getIntPtrConstant(-1, getCurSDLoc()));
+    return;
+  }
+
+  if (GCResult->getParent() == I.getParent()) {
+    // Result value will be used in a same basic block. Don't export it or
+    // perform any explicit register copies. The gc_result will simply grab
+    // this value. 
+    setValue(&I, ReturnValue);
+    return;
   }
+  
+  // Result value will be used in a different basic block so we need to export
+  // it now.  Default exporting mechanism will not work here because statepoint
+  // call has a different type than the actual call. It means that by default
+  // llvm will create export register of the wrong type (always i32 in our
+  // case). So instead we need to create export register with correct type
+  // manually.
+  // TODO: To eliminate this problem we can remove gc.result intrinsics
+  //       completely and make statepoint call to return a tuple.
+  unsigned Reg = FuncInfo.CreateRegs(RetTy);
+  RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
+                   DAG.getDataLayout(), Reg, RetTy,
+                   I.getCallingConv());
+  SDValue Chain = DAG.getEntryNode();
+  
+  RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
+  PendingExports.push_back(Chain);
+  FuncInfo.ValueMap[&I] = Reg;
 }
 
 void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl(
@@ -959,23 +990,23 @@ void SelectionDAGBuilder::LowerCallSiteWithDeoptBundle(
 void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
   // The result value of the gc_result is simply the result of the actual
   // call.  We've already emitted this, so just grab the value.
-  const GCStatepointInst *I = CI.getStatepoint();
-
-  if (I->getParent() != CI.getParent()) {
-    // Statepoint is in different basic block so we should have stored call
-    // result in a virtual register.
-    // We can not use default getValue() functionality to copy value from this
-    // register because statepoint and actual call return types can be
-    // different, and getValue() will use CopyFromReg of the wrong type,
-    // which is always i32 in our case.
-    Type *RetTy = I->getActualReturnType();
-    SDValue CopyFromReg = getCopyFromRegs(I, RetTy);
-
-    assert(CopyFromReg.getNode());
-    setValue(&CI, CopyFromReg);
-  } else {
-    setValue(&CI, getValue(I));
+  const GCStatepointInst *SI = CI.getStatepoint();
+
+  if (SI->getParent() == CI.getParent()) {
+    setValue(&CI, getValue(SI));
+    return;
   }
+  // Statepoint is in different basic block so we should have stored call
+  // result in a virtual register.
+  // We can not use default getValue() functionality to copy value from this
+  // register because statepoint and actual call return types can be
+  // different, and getValue() will use CopyFromReg of the wrong type,
+  // which is always i32 in our case.
+  Type *RetTy = SI->getActualReturnType();
+  SDValue CopyFromReg = getCopyFromRegs(SI, RetTy);
+  
+  assert(CopyFromReg.getNode());
+  setValue(&CI, CopyFromReg);
 }
 
 void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 407e1756569ef..96df20039b15d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -483,13 +483,15 @@ TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
 /// If the specified instruction has a constant integer operand and there are
 /// bits set in that constant that are not demanded, then clear those bits and
 /// return true.
-bool TargetLowering::ShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
+bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
+                                            const APInt &DemandedBits,
+                                            const APInt &DemandedElts,
                                             TargetLoweringOpt &TLO) const {
   SDLoc DL(Op);
   unsigned Opcode = Op.getOpcode();
 
   // Do target-specific constant optimization.
-  if (targetShrinkDemandedConstant(Op, Demanded, TLO))
+  if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
     return TLO.New.getNode();
 
   // FIXME: ISD::SELECT, ISD::SELECT_CC
@@ -505,12 +507,12 @@ bool TargetLowering::ShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
 
     // If this is a 'not' op, don't touch it because that's a canonical form.
     const APInt &C = Op1C->getAPIntValue();
-    if (Opcode == ISD::XOR && Demanded.isSubsetOf(C))
+    if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))
       return false;
 
-    if (!C.isSubsetOf(Demanded)) {
+    if (!C.isSubsetOf(DemandedBits)) {
       EVT VT = Op.getValueType();
-      SDValue NewC = TLO.DAG.getConstant(Demanded & C, DL, VT);
+      SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
       SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
       return TLO.CombineTo(Op, NewOp);
     }
@@ -522,6 +524,16 @@ bool TargetLowering::ShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
   return false;
 }
 
+bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
+                                            const APInt &DemandedBits,
+                                            TargetLoweringOpt &TLO) const {
+  EVT VT = Op.getValueType();
+  APInt DemandedElts = VT.isVector()
+                           ? APInt::getAllOnesValue(VT.getVectorNumElements())
+                           : APInt(1, 1);
+  return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
+}
+
 /// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
 /// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
 /// generalized for targets with other types of implicit widening casts.
@@ -773,6 +785,21 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
       return Op0;
     break;
   }
+  case ISD::ANY_EXTEND_VECTOR_INREG:
+  case ISD::SIGN_EXTEND_VECTOR_INREG:
+  case ISD::ZERO_EXTEND_VECTOR_INREG: {
+    // If we only want the lowest element and none of extended bits, then we can
+    // return the bitcasted source vector.
+    SDValue Src = Op.getOperand(0);
+    EVT SrcVT = Src.getValueType();
+    EVT DstVT = Op.getValueType();
+    if (DemandedElts == 1 && DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
+        DAG.getDataLayout().isLittleEndian() &&
+        DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
+      return DAG.getBitcast(DstVT, Src);
+    }
+    break;
+  }
   case ISD::INSERT_VECTOR_ELT: {
     // If we don't demand the inserted element, return the base vector.
     SDValue Vec = Op.getOperand(0);
@@ -837,6 +864,14 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
                                          Depth);
 }
 
+SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
+    SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
+    unsigned Depth) const {
+  APInt DemandedBits = APInt::getAllOnesValue(Op.getScalarValueSizeInBits());
+  return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
+                                         Depth);
+}
+
 /// Look at Op. At this point, we know that only the OriginalDemandedBits of the
 /// result of Op are ever used downstream. If we can use this information to
 /// simplify Op, create a new simplified DAG node and return true, returning the
@@ -1031,6 +1066,8 @@ bool TargetLowering::SimplifyDemandedBits(
   case ISD::EXTRACT_SUBVECTOR: {
     // Offset the demanded elts by the subvector index.
     SDValue Src = Op.getOperand(0);
+    if (Src.getValueType().isScalableVector())
+      break;
     uint64_t Idx = Op.getConstantOperandVal(1);
     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
     APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
@@ -1148,7 +1185,8 @@ bool TargetLowering::SimplifyDemandedBits(
 
       // If any of the set bits in the RHS are known zero on the LHS, shrink
       // the constant.
-      if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits, TLO))
+      if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,
+                                 DemandedElts, TLO))
         return true;
 
       // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
@@ -1196,7 +1234,8 @@ bool TargetLowering::SimplifyDemandedBits(
     if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
       return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
     // If the RHS is a constant, see if we can simplify it.
-    if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, TLO))
+    if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,
+                               TLO))
       return true;
     // If the operation can be done in a smaller type, do so.
     if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
@@ -1239,7 +1278,7 @@ bool TargetLowering::SimplifyDemandedBits(
     if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
       return TLO.CombineTo(Op, Op1);
     // If the RHS is a constant, see if we can simplify it.
-    if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
+    if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
       return true;
     // If the operation can be done in a smaller type, do so.
     if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
@@ -1291,7 +1330,8 @@ bool TargetLowering::SimplifyDemandedBits(
     if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
       return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
 
-    if (ConstantSDNode *C = isConstOrConstSplat(Op1)) {
+    ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);
+    if (C) {
       // If one side is a constant, and all of the known set bits on the other
       // side are also set in the constant, turn this into an AND, as we know
       // the bits will be cleared.
@@ -1306,18 +1346,19 @@ bool TargetLowering::SimplifyDemandedBits(
       // If the RHS is a constant, see if we can change it. Don't alter a -1
       // constant because that's a 'not' op, and that is better for combining
       // and codegen.
-      if (!C->isAllOnesValue()) {
-        if (DemandedBits.isSubsetOf(C->getAPIntValue())) {
-          // We're flipping all demanded bits. Flip the undemanded bits too.
-          SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
-          return TLO.CombineTo(Op, New);
-        }
-        // If we can't turn this into a 'not', try to shrink the constant.
-        if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
-          return true;
+      if (!C->isAllOnesValue() &&
+          DemandedBits.isSubsetOf(C->getAPIntValue())) {
+        // We're flipping all demanded bits. Flip the undemanded bits too.
+        SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
+        return TLO.CombineTo(Op, New);
       }
     }
 
+    // If we can't turn this into a 'not', try to shrink the constant.
+    if (!C || !C->isAllOnesValue())
+      if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
+        return true;
+
     Known ^= Known2;
     break;
   }
@@ -1332,7 +1373,7 @@ bool TargetLowering::SimplifyDemandedBits(
     assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
 
     // If the operands are constants, see if we can simplify them.
-    if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
+    if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
       return true;
 
     // Only known if known in both the LHS and RHS.
@@ -1350,7 +1391,7 @@ bool TargetLowering::SimplifyDemandedBits(
     assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
 
     // If the operands are constants, see if we can simplify them.
-    if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
+    if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
       return true;
 
     // Only known if known in both the LHS and RHS.
@@ -2308,14 +2349,10 @@ bool TargetLowering::SimplifyDemandedVectorElts(
   // Helper for demanding the specified elements and all the bits of both binary
   // operands.
   auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
-    unsigned NumBits0 = Op0.getScalarValueSizeInBits();
-    unsigned NumBits1 = Op1.getScalarValueSizeInBits();
-    APInt DemandedBits0 = APInt::getAllOnesValue(NumBits0);
-    APInt DemandedBits1 = APInt::getAllOnesValue(NumBits1);
-    SDValue NewOp0 = SimplifyMultipleUseDemandedBits(
-        Op0, DemandedBits0, DemandedElts, TLO.DAG, Depth + 1);
-    SDValue NewOp1 = SimplifyMultipleUseDemandedBits(
-        Op1, DemandedBits1, DemandedElts, TLO.DAG, Depth + 1);
+    SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
+                                                           TLO.DAG, Depth + 1);
+    SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
+                                                           TLO.DAG, Depth + 1);
     if (NewOp0 || NewOp1) {
       SDValue NewOp = TLO.DAG.getNode(
           Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);
@@ -2496,11 +2533,10 @@ bool TargetLowering::SimplifyDemandedVectorElts(
     // Attempt to avoid multi-use ops if we don't need anything from them.
     if (!DemandedSrcElts.isAllOnesValue() ||
         !DemandedSubElts.isAllOnesValue()) {
-      APInt DemandedBits = APInt::getAllOnesValue(VT.getScalarSizeInBits());
-      SDValue NewSrc = SimplifyMultipleUseDemandedBits(
-          Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
-      SDValue NewSub = SimplifyMultipleUseDemandedBits(
-          Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);
+      SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
+          Src, DemandedSrcElts, TLO.DAG, Depth + 1);
+      SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
+          Sub, DemandedSubElts, TLO.DAG, Depth + 1);
       if (NewSrc || NewSub) {
         NewSrc = NewSrc ? NewSrc : Src;
         NewSub = NewSub ? NewSub : Sub;
@@ -2514,6 +2550,8 @@ bool TargetLowering::SimplifyDemandedVectorElts(
   case ISD::EXTRACT_SUBVECTOR: {
     // Offset the demanded elts by the subvector index.
     SDValue Src = Op.getOperand(0);
+    if (Src.getValueType().isScalableVector())
+      break;
     uint64_t Idx = Op.getConstantOperandVal(1);
     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
     APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
@@ -2527,9 +2565,8 @@ bool TargetLowering::SimplifyDemandedVectorElts(
 
     // Attempt to avoid multi-use ops if we don't need anything from them.
     if (!DemandedElts.isAllOnesValue()) {
-      APInt DemandedBits = APInt::getAllOnesValue(VT.getScalarSizeInBits());
-      SDValue NewSrc = SimplifyMultipleUseDemandedBits(
-          Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
+      SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
+          Src, DemandedSrcElts, TLO.DAG, Depth + 1);
       if (NewSrc) {
         SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
                                         Op.getOperand(1));
@@ -6080,6 +6117,14 @@ bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
   return Ok;
 }
 
+// Check that (every element of) Z is undef or not an exact multiple of BW.
+static bool isNonZeroModBitWidth(SDValue Z, unsigned BW) {
+  return ISD::matchUnaryPredicate(
+      Z,
+      [=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(BW) != 0; },
+      true);
+}
+
 bool TargetLowering::expandFunnelShift(SDNode *Node, SDValue &Result,
                                        SelectionDAG &DAG) const {
   EVT VT = Node->getValueType(0);
@@ -6090,40 +6135,52 @@ bool TargetLowering::expandFunnelShift(SDNode *Node, SDValue &Result,
                         !isOperationLegalOrCustomOrPromote(ISD::OR, VT)))
     return false;
 
-  // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
-  // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
   SDValue X = Node->getOperand(0);
   SDValue Y = Node->getOperand(1);
   SDValue Z = Node->getOperand(2);
 
-  unsigned EltSizeInBits = VT.getScalarSizeInBits();
+  unsigned BW = VT.getScalarSizeInBits();
   bool IsFSHL = Node->getOpcode() == ISD::FSHL;
   SDLoc DL(SDValue(Node, 0));
 
   EVT ShVT = Z.getValueType();
-  SDValue Mask = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
-  SDValue ShAmt, InvShAmt;
-  if (isPowerOf2_32(EltSizeInBits)) {
-    // Z % BW -> Z & (BW - 1)
-    ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask);
-    // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
-    InvShAmt = DAG.getNode(ISD::AND, DL, ShVT, DAG.getNOT(DL, Z, ShVT), Mask);
-  } else {
-    SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);
-    ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
-    InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, Mask, ShAmt);
-  }
 
-  SDValue One = DAG.getConstant(1, DL, ShVT);
   SDValue ShX, ShY;
-  if (IsFSHL) {
-    ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt);
-    SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One);
-    ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt);
+  SDValue ShAmt, InvShAmt;
+  if (isNonZeroModBitWidth(Z, BW)) {
+    // fshl: X << C | Y >> (BW - C)
+    // fshr: X << (BW - C) | Y >> C
+    // where C = Z % BW is not zero
+    SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
+    ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
+    InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt);
+    ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt);
+    ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt);
   } else {
-    SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One);
-    ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt);
-    ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt);
+    // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
+    // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
+    SDValue Mask = DAG.getConstant(BW - 1, DL, ShVT);
+    if (isPowerOf2_32(BW)) {
+      // Z % BW -> Z & (BW - 1)
+      ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask);
+      // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
+      InvShAmt = DAG.getNode(ISD::AND, DL, ShVT, DAG.getNOT(DL, Z, ShVT), Mask);
+    } else {
+      SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
+      ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
+      InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, Mask, ShAmt);
+    }
+
+    SDValue One = DAG.getConstant(1, DL, ShVT);
+    if (IsFSHL) {
+      ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt);
+      SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One);
+      ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt);
+    } else {
+      SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One);
+      ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt);
+      ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt);
+    }
   }
   Result = DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
   return true;
@@ -6140,12 +6197,15 @@ bool TargetLowering::expandROT(SDNode *Node, SDValue &Result,
   SDLoc DL(SDValue(Node, 0));
 
   EVT ShVT = Op1.getValueType();
-  SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);
+  SDValue Zero = DAG.getConstant(0, DL, ShVT);
+
+  assert(isPowerOf2_32(EltSizeInBits) && EltSizeInBits > 1 &&
+         "Expecting the type bitwidth to be a power of 2");
 
-  // If a rotate in the other direction is legal, use it.
+  // If a rotate in the other direction is supported, use it.
   unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
-  if (isOperationLegal(RevRot, VT)) {
-    SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, Op1);
+  if (isOperationLegalOrCustom(RevRot, VT)) {
+    SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
     Result = DAG.getNode(RevRot, DL, VT, Op0, Sub);
     return true;
   }
@@ -6158,15 +6218,13 @@ bool TargetLowering::expandROT(SDNode *Node, SDValue &Result,
     return false;
 
   // Otherwise,
-  //   (rotl x, c) -> (or (shl x, (and c, w-1)), (srl x, (and w-c, w-1)))
-  //   (rotr x, c) -> (or (srl x, (and c, w-1)), (shl x, (and w-c, w-1)))
+  //   (rotl x, c) -> (or (shl x, (and c, w-1)), (srl x, (and -c, w-1)))
+  //   (rotr x, c) -> (or (srl x, (and c, w-1)), (shl x, (and -c, w-1)))
   //
-  assert(isPowerOf2_32(EltSizeInBits) && EltSizeInBits > 1 &&
-         "Expecting the type bitwidth to be a power of 2");
   unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
   unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
   SDValue BitWidthMinusOneC = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
-  SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, Op1);
+  SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
   SDValue And0 = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC);
   SDValue And1 = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC);
   Result = DAG.getNode(ISD::OR, DL, VT, DAG.getNode(ShOpc, DL, VT, Op0, And0),
@@ -7802,3 +7860,26 @@ SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
     Res = DAG.getNode(ISD::ANY_EXTEND, dl, Node->getValueType(0), Res);
   return Res;
 }
+
+bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,
+                               SelectionDAG &DAG) const {
+  EVT VT = Node->getValueType(0);
+  SDLoc dl(Node);
+  bool isSigned = Node->getOpcode() == ISD::SREM;
+  unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
+  unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
+  SDValue Dividend = Node->getOperand(0);
+  SDValue Divisor = Node->getOperand(1);
+  if (isOperationLegalOrCustom(DivRemOpc, VT)) {
+    SDVTList VTs = DAG.getVTList(VT, VT);
+    Result = DAG.getNode(DivRemOpc, dl, VTs, Dividend, Divisor).getValue(1);
+    return true;
+  } else if (isOperationLegalOrCustom(DivOpc, VT)) {
+    // X % Y -> X-X/Y*Y
+    SDValue Divide = DAG.getNode(DivOpc, dl, VT, Dividend, Divisor);
+    SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor);
+    Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);
+    return true;
+  }
+  return false;
+}
diff --git a/llvm/lib/CodeGen/ShrinkWrap.cpp b/llvm/lib/CodeGen/ShrinkWrap.cpp
index 85dd4f59fa13e..ce43fb1fbd4b0 100644
--- a/llvm/lib/CodeGen/ShrinkWrap.cpp
+++ b/llvm/lib/CodeGen/ShrinkWrap.cpp
@@ -494,17 +494,15 @@ bool ShrinkWrap::runOnMachineFunction(MachineFunction &MF) {
                                "EH Funclets are not supported yet.",
                                MBB.front().getDebugLoc(), &MBB);
 
-    if (MBB.isEHPad()) {
-      // Push the prologue and epilogue outside of
-      // the region that may throw by making sure
-      // that all the landing pads are at least at the
-      // boundary of the save and restore points.
-      // The problem with exceptions is that the throw
-      // is not properly modeled and in particular, a
-      // basic block can jump out from the middle.
+    if (MBB.isEHPad() || MBB.isInlineAsmBrIndirectTarget()) {
+      // Push the prologue and epilogue outside of the region that may throw (or
+      // jump out via inlineasm_br), by making sure that all the landing pads
+      // are at least at the boundary of the save and restore points.  The
+      // problem is that a basic block can jump out from the middle in these
+      // cases, which we do not handle.
       updateSaveRestorePoints(MBB, RS.get());
       if (!ArePointsInteresting()) {
-        LLVM_DEBUG(dbgs() << "EHPad prevents shrink-wrapping\n");
+        LLVM_DEBUG(dbgs() << "EHPad/inlineasm_br prevents shrink-wrapping\n");
         return false;
       }
       continue;
diff --git a/llvm/lib/CodeGen/SplitKit.cpp b/llvm/lib/CodeGen/SplitKit.cpp
index a5d4e1206f16e..8dec620536a7d 100644
--- a/llvm/lib/CodeGen/SplitKit.cpp
+++ b/llvm/lib/CodeGen/SplitKit.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/LiveInterval.h"
 #include "llvm/CodeGen/LiveIntervalCalc.h"
 #include "llvm/CodeGen/LiveIntervals.h"
@@ -79,10 +80,15 @@ InsertPointAnalysis::computeLastInsertPoint(const LiveInterval &CurLI,
   std::pair<SlotIndex, SlotIndex> &LIP = LastInsertPoint[Num];
   SlotIndex MBBEnd = LIS.getMBBEndIdx(&MBB);
 
-  SmallVector<const MachineBasicBlock *, 1> EHPadSuccessors;
-  for (const MachineBasicBlock *SMBB : MBB.successors())
-    if (SMBB->isEHPad())
-      EHPadSuccessors.push_back(SMBB);
+  SmallVector<const MachineBasicBlock *, 1> ExceptionalSuccessors;
+  bool EHPadSuccessor = false;
+  for (const MachineBasicBlock *SMBB : MBB.successors()) {
+    if (SMBB->isEHPad()) {
+      ExceptionalSuccessors.push_back(SMBB);
+      EHPadSuccessor = true;
+    } else if (SMBB->isInlineAsmBrIndirectTarget())
+      ExceptionalSuccessors.push_back(SMBB);
+  }
 
   // Compute insert points on the first call. The pair is independent of the
   // current live interval.
@@ -93,15 +99,17 @@ InsertPointAnalysis::computeLastInsertPoint(const LiveInterval &CurLI,
     else
       LIP.first = LIS.getInstructionIndex(*FirstTerm);
 
-    // If there is a landing pad successor, also find the call instruction.
-    if (EHPadSuccessors.empty())
+    // If there is a landing pad or inlineasm_br successor, also find the
+    // instruction. If there is no such instruction, we don't need to do
+    // anything special.  We assume there cannot be multiple instructions that
+    // are Calls with EHPad successors or INLINEASM_BR in a block. Further, we
+    // assume that if there are any, they will be after any other call
+    // instructions in the block.
+    if (ExceptionalSuccessors.empty())
       return LIP.first;
-    // There may not be a call instruction (?) in which case we ignore LPad.
-    LIP.second = LIP.first;
-    for (MachineBasicBlock::const_iterator I = MBB.end(), E = MBB.begin();
-         I != E;) {
-      --I;
-      if (I->isCall()) {
+    for (auto I = MBB.rbegin(), E = MBB.rend(); I != E; ++I) {
+      if ((EHPadSuccessor && I->isCall()) ||
+          I->getOpcode() == TargetOpcode::INLINEASM_BR) {
         LIP.second = LIS.getInstructionIndex(*I);
         break;
       }
@@ -113,7 +121,7 @@ InsertPointAnalysis::computeLastInsertPoint(const LiveInterval &CurLI,
   if (!LIP.second)
     return LIP.first;
 
-  if (none_of(EHPadSuccessors, [&](const MachineBasicBlock *EHPad) {
+  if (none_of(ExceptionalSuccessors, [&](const MachineBasicBlock *EHPad) {
         return LIS.isLiveInToMBB(CurLI, EHPad);
       }))
     return LIP.first;
diff --git a/llvm/lib/CodeGen/SplitKit.h b/llvm/lib/CodeGen/SplitKit.h
index 1d1835cdd510f..3ab5f2585f346 100644
--- a/llvm/lib/CodeGen/SplitKit.h
+++ b/llvm/lib/CodeGen/SplitKit.h
@@ -34,6 +34,7 @@
 
 namespace llvm {
 
+class AAResults;
 class LiveIntervals;
 class LiveRangeEdit;
 class MachineBlockFrequencyInfo;
@@ -53,7 +54,7 @@ class LLVM_LIBRARY_VISIBILITY InsertPointAnalysis {
   /// Last legal insert point in each basic block in the current function.
   /// The first entry is the first terminator, the second entry is the
   /// last valid point to insert a split or spill for a variable that is
-  /// live into a landing pad successor.
+  /// live into a landing pad or inlineasm_br successor.
   SmallVector<std::pair<SlotIndex, SlotIndex>, 8> LastInsertPoint;
 
   SlotIndex computeLastInsertPoint(const LiveInterval &CurLI,
@@ -256,7 +257,7 @@ class LLVM_LIBRARY_VISIBILITY SplitAnalysis {
 ///
 class LLVM_LIBRARY_VISIBILITY SplitEditor {
   SplitAnalysis &SA;
-  AliasAnalysis &AA;
+  AAResults &AA;
   LiveIntervals &LIS;
   VirtRegMap &VRM;
   MachineRegisterInfo &MRI;
@@ -442,7 +443,7 @@ class LLVM_LIBRARY_VISIBILITY SplitEditor {
 public:
   /// Create a new SplitEditor for editing the LiveInterval analyzed by SA.
   /// Newly created intervals will be appended to newIntervals.
-  SplitEditor(SplitAnalysis &sa, AliasAnalysis &aa, LiveIntervals &lis,
+  SplitEditor(SplitAnalysis &sa, AAResults &aa, LiveIntervals &lis,
               VirtRegMap &vrm, MachineDominatorTree &mdt,
               MachineBlockFrequencyInfo &mbfi);
 
diff --git a/llvm/lib/CodeGen/TailDuplicator.cpp b/llvm/lib/CodeGen/TailDuplicator.cpp
index e3ecf52f875d1..bd554189f12b5 100644
--- a/llvm/lib/CodeGen/TailDuplicator.cpp
+++ b/llvm/lib/CodeGen/TailDuplicator.cpp
@@ -206,11 +206,11 @@ bool TailDuplicator::tailDuplicateAndUpdate(
       }
 
       // Add the new vregs as available values.
-      DenseMap<unsigned, AvailableValsTy>::iterator LI =
+      DenseMap<Register, AvailableValsTy>::iterator LI =
           SSAUpdateVals.find(VReg);
       for (unsigned j = 0, ee = LI->second.size(); j != ee; ++j) {
         MachineBasicBlock *SrcBB = LI->second[j].first;
-        unsigned SrcReg = LI->second[j].second;
+        Register SrcReg = LI->second[j].second;
         SSAUpdate.AddAvailableValue(SrcBB, SrcReg);
       }
 
@@ -294,7 +294,7 @@ bool TailDuplicator::tailDuplicateBlocks() {
   return MadeChange;
 }
 
-static bool isDefLiveOut(unsigned Reg, MachineBasicBlock *BB,
+static bool isDefLiveOut(Register Reg, MachineBasicBlock *BB,
                          const MachineRegisterInfo *MRI) {
   for (MachineInstr &UseMI : MRI->use_instructions(Reg)) {
     if (UseMI.isDebugValue())
@@ -316,7 +316,7 @@ static unsigned getPHISrcRegOpIdx(MachineInstr *MI, MachineBasicBlock *SrcBB) {
 // used to determine which registers are liveout while modifying the
 // block (which is why we need to copy the information).
 static void getRegsUsedByPHIs(const MachineBasicBlock &BB,
-                              DenseSet<unsigned> *UsedByPhi) {
+                              DenseSet<Register> *UsedByPhi) {
   for (const auto &MI : BB) {
     if (!MI.isPHI())
       break;
@@ -328,9 +328,9 @@ static void getRegsUsedByPHIs(const MachineBasicBlock &BB,
 }
 
 /// Add a definition and source virtual registers pair for SSA update.
-void TailDuplicator::addSSAUpdateEntry(unsigned OrigReg, unsigned NewReg,
+void TailDuplicator::addSSAUpdateEntry(Register OrigReg, Register NewReg,
                                        MachineBasicBlock *BB) {
-  DenseMap<unsigned, AvailableValsTy>::iterator LI =
+  DenseMap<Register, AvailableValsTy>::iterator LI =
       SSAUpdateVals.find(OrigReg);
   if (LI != SSAUpdateVals.end())
     LI->second.push_back(std::make_pair(BB, NewReg));
@@ -346,9 +346,9 @@ void TailDuplicator::addSSAUpdateEntry(unsigned OrigReg, unsigned NewReg,
 /// source register that's contributed by PredBB and update SSA update map.
 void TailDuplicator::processPHI(
     MachineInstr *MI, MachineBasicBlock *TailBB, MachineBasicBlock *PredBB,
-    DenseMap<unsigned, RegSubRegPair> &LocalVRMap,
-    SmallVectorImpl<std::pair<unsigned, RegSubRegPair>> &Copies,
-    const DenseSet<unsigned> &RegsUsedByPhi, bool Remove) {
+    DenseMap<Register, RegSubRegPair> &LocalVRMap,
+    SmallVectorImpl<std::pair<Register, RegSubRegPair>> &Copies,
+    const DenseSet<Register> &RegsUsedByPhi, bool Remove) {
   Register DefReg = MI->getOperand(0).getReg();
   unsigned SrcOpIdx = getPHISrcRegOpIdx(MI, PredBB);
   assert(SrcOpIdx && "Unable to find matching PHI source?");
@@ -378,8 +378,8 @@ void TailDuplicator::processPHI(
 /// the source operands due to earlier PHI translation.
 void TailDuplicator::duplicateInstruction(
     MachineInstr *MI, MachineBasicBlock *TailBB, MachineBasicBlock *PredBB,
-    DenseMap<unsigned, RegSubRegPair> &LocalVRMap,
-    const DenseSet<unsigned> &UsedByPhi) {
+    DenseMap<Register, RegSubRegPair> &LocalVRMap,
+    const DenseSet<Register> &UsedByPhi) {
   // Allow duplication of CFI instructions.
   if (MI->isCFIInstruction()) {
     BuildMI(*PredBB, PredBB->end(), PredBB->findDebugLoc(PredBB->begin()),
@@ -504,7 +504,7 @@ void TailDuplicator::updateSuccessorsPHIs(
       // If Idx is set, the operands at Idx and Idx+1 must be removed.
       // We reuse the location to avoid expensive RemoveOperand calls.
 
-      DenseMap<unsigned, AvailableValsTy>::iterator LI =
+      DenseMap<Register, AvailableValsTy>::iterator LI =
           SSAUpdateVals.find(Reg);
       if (LI != SSAUpdateVals.end()) {
         // This register is defined in the tail block.
@@ -517,7 +517,7 @@ void TailDuplicator::updateSuccessorsPHIs(
           if (!SrcBB->isSuccessor(SuccBB))
             continue;
 
-          unsigned SrcReg = LI->second[j].second;
+          Register SrcReg = LI->second[j].second;
           if (Idx != 0) {
             MI.getOperand(Idx).setReg(SrcReg);
             MI.getOperand(Idx + 1).setMBB(SrcBB);
@@ -708,7 +708,7 @@ bool TailDuplicator::canCompletelyDuplicateBB(MachineBasicBlock &BB) {
 
 bool TailDuplicator::duplicateSimpleBB(
     MachineBasicBlock *TailBB, SmallVectorImpl<MachineBasicBlock *> &TDBBs,
-    const DenseSet<unsigned> &UsedByPhi,
+    const DenseSet<Register> &UsedByPhi,
     SmallVectorImpl<MachineInstr *> &Copies) {
   SmallPtrSet<MachineBasicBlock *, 8> Succs(TailBB->succ_begin(),
                                             TailBB->succ_end());
@@ -716,7 +716,7 @@ bool TailDuplicator::duplicateSimpleBB(
                                             TailBB->pred_end());
   bool Changed = false;
   for (MachineBasicBlock *PredBB : Preds) {
-    if (PredBB->hasEHPadSuccessor())
+    if (PredBB->hasEHPadSuccessor() || PredBB->mayHaveInlineAsmBr())
       continue;
 
     if (bothUsedInPHI(*PredBB, Succs))
@@ -815,7 +815,7 @@ bool TailDuplicator::tailDuplicate(bool IsSimple, MachineBasicBlock *TailBB,
 
   bool ShouldUpdateTerminators = TailBB->canFallThrough();
 
-  DenseSet<unsigned> UsedByPhi;
+  DenseSet<Register> UsedByPhi;
   getRegsUsedByPHIs(*TailBB, &UsedByPhi);
 
   if (IsSimple)
@@ -860,8 +860,8 @@ bool TailDuplicator::tailDuplicate(bool IsSimple, MachineBasicBlock *TailBB,
     TII->removeBranch(*PredBB);
 
     // Clone the contents of TailBB into PredBB.
-    DenseMap<unsigned, RegSubRegPair> LocalVRMap;
-    SmallVector<std::pair<unsigned, RegSubRegPair>, 4> CopyInfos;
+    DenseMap<Register, RegSubRegPair> LocalVRMap;
+    SmallVector<std::pair<Register, RegSubRegPair>, 4> CopyInfos;
     for (MachineBasicBlock::iterator I = TailBB->begin(), E = TailBB->end();
          I != E; /* empty */) {
       MachineInstr *MI = &*I;
@@ -920,8 +920,8 @@ bool TailDuplicator::tailDuplicate(bool IsSimple, MachineBasicBlock *TailBB,
     // duplicating the instructions in all cases.
     TII->removeBranch(*PrevBB);
     if (PreRegAlloc) {
-      DenseMap<unsigned, RegSubRegPair> LocalVRMap;
-      SmallVector<std::pair<unsigned, RegSubRegPair>, 4> CopyInfos;
+      DenseMap<Register, RegSubRegPair> LocalVRMap;
+      SmallVector<std::pair<Register, RegSubRegPair>, 4> CopyInfos;
       MachineBasicBlock::iterator I = TailBB->begin();
       // Process PHI instructions first.
       while (I != TailBB->end() && I->isPHI()) {
@@ -988,8 +988,8 @@ bool TailDuplicator::tailDuplicate(bool IsSimple, MachineBasicBlock *TailBB,
     if (PredBB->succ_size() != 1)
       continue;
 
-    DenseMap<unsigned, RegSubRegPair> LocalVRMap;
-    SmallVector<std::pair<unsigned, RegSubRegPair>, 4> CopyInfos;
+    DenseMap<Register, RegSubRegPair> LocalVRMap;
+    SmallVector<std::pair<Register, RegSubRegPair>, 4> CopyInfos;
     MachineBasicBlock::iterator I = TailBB->begin();
     // Process PHI instructions first.
     while (I != TailBB->end() && I->isPHI()) {
@@ -1007,7 +1007,7 @@ bool TailDuplicator::tailDuplicate(bool IsSimple, MachineBasicBlock *TailBB,
 /// At the end of the block \p MBB generate COPY instructions between registers
 /// described by \p CopyInfos. Append resulting instructions to \p Copies.
 void TailDuplicator::appendCopies(MachineBasicBlock *MBB,
-      SmallVectorImpl<std::pair<unsigned,RegSubRegPair>> &CopyInfos,
+      SmallVectorImpl<std::pair<Register, RegSubRegPair>> &CopyInfos,
       SmallVectorImpl<MachineInstr*> &Copies) {
   MachineBasicBlock::iterator Loc = MBB->getFirstTerminator();
   const MCInstrDesc &CopyD = TII->get(TargetOpcode::COPY);
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 9dbd612c7f5f5..24f3f96d0b1da 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -999,6 +999,10 @@ bool TargetInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
   if (MI.isTerminator() || MI.isPosition())
     return true;
 
+  // INLINEASM_BR can jump to another block
+  if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
+    return true;
+
   // Don't attempt to schedule around any instruction that defines
   // a stack-oriented pointer, as it's unlikely to be profitable. This
   // saves compile time, because it doesn't require every single
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index c8220928af935..2c94c2c62e5f0 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1573,14 +1573,14 @@ unsigned TargetLoweringBase::getByValTypeAlignment(Type *Ty,
 
 bool TargetLoweringBase::allowsMemoryAccessForAlignment(
     LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
-    unsigned Alignment, MachineMemOperand::Flags Flags, bool *Fast) const {
+    Align Alignment, MachineMemOperand::Flags Flags, bool *Fast) const {
   // Check if the specified alignment is sufficient based on the data layout.
   // TODO: While using the data layout works in practice, a better solution
   // would be to implement this check directly (make this a virtual function).
   // For example, the ABI alignment may change based on software platform while
   // this function should only be affected by hardware implementation.
   Type *Ty = VT.getTypeForEVT(Context);
-  if (Alignment >= DL.getABITypeAlign(Ty).value()) {
+  if (Alignment >= DL.getABITypeAlign(Ty)) {
     // Assume that an access that meets the ABI-specified alignment is fast.
     if (Fast != nullptr)
       *Fast = true;
@@ -1588,20 +1588,22 @@ bool TargetLoweringBase::allowsMemoryAccessForAlignment(
   }
 
   // This is a misaligned access.
-  return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Flags, Fast);
+  return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment.value(), Flags,
+                                        Fast);
 }
 
 bool TargetLoweringBase::allowsMemoryAccessForAlignment(
     LLVMContext &Context, const DataLayout &DL, EVT VT,
     const MachineMemOperand &MMO, bool *Fast) const {
   return allowsMemoryAccessForAlignment(Context, DL, VT, MMO.getAddrSpace(),
-                                        MMO.getAlign().value(), MMO.getFlags(),
-                                        Fast);
+                                        MMO.getAlign(), MMO.getFlags(), Fast);
 }
 
-bool TargetLoweringBase::allowsMemoryAccess(
-    LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
-    unsigned Alignment, MachineMemOperand::Flags Flags, bool *Fast) const {
+bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
+                                            const DataLayout &DL, EVT VT,
+                                            unsigned AddrSpace, Align Alignment,
+                                            MachineMemOperand::Flags Flags,
+                                            bool *Fast) const {
   return allowsMemoryAccessForAlignment(Context, DL, VT, AddrSpace, Alignment,
                                         Flags, Fast);
 }
@@ -1610,8 +1612,8 @@ bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
                                             const DataLayout &DL, EVT VT,
                                             const MachineMemOperand &MMO,
                                             bool *Fast) const {
-  return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(),
-                            MMO.getAlign().value(), MMO.getFlags(), Fast);
+  return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), MMO.getAlign(),
+                            MMO.getFlags(), Fast);
 }
 
 BranchProbability TargetLoweringBase::getPredictableBranchThreshold() const {
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index 03254ed77d71c..27bebe503ce66 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -599,11 +599,11 @@ getELFSectionNameForGlobal(const GlobalObject *GO, SectionKind Kind,
     // We also need alignment here.
     // FIXME: this is getting the alignment of the character, not the
     // alignment of the global!
-    unsigned Align = GO->getParent()->getDataLayout().getPreferredAlignment(
+    Align Alignment = GO->getParent()->getDataLayout().getPreferredAlign(
         cast<GlobalVariable>(GO));
 
     std::string SizeSpec = ".rodata.str" + utostr(EntrySize) + ".";
-    Name = SizeSpec + utostr(Align);
+    Name = SizeSpec + utostr(Alignment.value());
   } else if (Kind.isMergeableConst()) {
     Name = ".rodata.cst";
     Name += utostr(EntrySize);
@@ -1145,16 +1145,16 @@ MCSection *TargetLoweringObjectFileMachO::SelectSectionForGlobal(
 
   // FIXME: Alignment check should be handled by section classifier.
   if (Kind.isMergeable1ByteCString() &&
-      GO->getParent()->getDataLayout().getPreferredAlignment(
-          cast<GlobalVariable>(GO)) < 32)
+      GO->getParent()->getDataLayout().getPreferredAlign(
+          cast<GlobalVariable>(GO)) < Align(32))
     return CStringSection;
 
   // Do not put 16-bit arrays in the UString section if they have an
   // externally visible label, this runs into issues with certain linker
   // versions.
   if (Kind.isMergeable2ByteCString() && !GO->hasExternalLinkage() &&
-      GO->getParent()->getDataLayout().getPreferredAlignment(
-          cast<GlobalVariable>(GO)) < 32)
+      GO->getParent()->getDataLayout().getPreferredAlign(
+          cast<GlobalVariable>(GO)) < Align(32))
     return UStringSection;
 
   // With MachO only variables whose corresponding symbol starts with 'l' or
@@ -1765,7 +1765,7 @@ static std::string scalarConstantToHexString(const Constant *C) {
   } else {
     unsigned NumElements;
     if (auto *VTy = dyn_cast<VectorType>(Ty))
-      NumElements = VTy->getNumElements();
+      NumElements = cast<FixedVectorType>(VTy)->getNumElements();
     else
       NumElements = Ty->getArrayNumElements();
     std::string HexString;
@@ -2043,13 +2043,13 @@ MCSection *TargetLoweringObjectFileXCOFF::SelectSectionForGlobal(
   }
 
   if (Kind.isMergeableCString()) {
-    unsigned Align = GO->getParent()->getDataLayout().getPreferredAlignment(
+    Align Alignment = GO->getParent()->getDataLayout().getPreferredAlign(
         cast<GlobalVariable>(GO));
 
     unsigned EntrySize = getEntrySizeForKind(Kind);
     std::string SizeSpec = ".rodata.str" + utostr(EntrySize) + ".";
     SmallString<128> Name;
-    Name = SizeSpec + utostr(Align);
+    Name = SizeSpec + utostr(Alignment.value());
 
     return getContext().getXCOFFSection(
         Name, XCOFF::XMC_RO, XCOFF::XTY_SD,
@@ -2168,6 +2168,6 @@ MCSection *TargetLoweringObjectFileXCOFF::getSectionForFunctionDescriptor(
 MCSection *TargetLoweringObjectFileXCOFF::getSectionForTOCEntry(
     const MCSymbol *Sym) const {
   return getContext().getXCOFFSection(
-      cast<MCSymbolXCOFF>(Sym)->getUnqualifiedName(), XCOFF::XMC_TC,
+      cast<MCSymbolXCOFF>(Sym)->getSymbolTableName(), XCOFF::XMC_TC,
       XCOFF::XTY_SD, XCOFF::C_HIDEXT, SectionKind::getData());
 }
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 241357be53941..e0fdb0cefcb8b 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -472,7 +472,7 @@ bool TargetPassConfig::hasLimitedCodeGenPipeline() {
 }
 
 std::string
-TargetPassConfig::getLimitedCodeGenPipelineReason(const char *Separator) const {
+TargetPassConfig::getLimitedCodeGenPipelineReason(const char *Separator) {
   if (!hasLimitedCodeGenPipeline())
     return std::string();
   std::string Res;
diff --git a/llvm/lib/CodeGen/UnreachableBlockElim.cpp b/llvm/lib/CodeGen/UnreachableBlockElim.cpp
index dc40cb081f6c4..f5dc589a98cbf 100644
--- a/llvm/lib/CodeGen/UnreachableBlockElim.cpp
+++ b/llvm/lib/CodeGen/UnreachableBlockElim.cpp
@@ -81,7 +81,7 @@ namespace {
   class UnreachableMachineBlockElim : public MachineFunctionPass {
     bool runOnMachineFunction(MachineFunction &F) override;
     void getAnalysisUsage(AnalysisUsage &AU) const override;
-    MachineModuleInfo *MMI;
+
   public:
     static char ID; // Pass identification, replacement for typeid
     UnreachableMachineBlockElim() : MachineFunctionPass(ID) {}
@@ -104,8 +104,6 @@ bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
   df_iterator_default_set<MachineBasicBlock*> Reachable;
   bool ModifiedPHI = false;
 
-  auto *MMIWP = getAnalysisIfAvailable<MachineModuleInfoWrapperPass>();
-  MMI = MMIWP ? &MMIWP->getMMI() : nullptr;
   MachineDominatorTree *MDT = getAnalysisIfAvailable<MachineDominatorTree>();
   MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>();
 
diff --git a/llvm/lib/CodeGen/ValueTypes.cpp b/llvm/lib/CodeGen/ValueTypes.cpp
index 0fe2f73c9def9..66bcdd9b2c4a4 100644
--- a/llvm/lib/CodeGen/ValueTypes.cpp
+++ b/llvm/lib/CodeGen/ValueTypes.cpp
@@ -122,7 +122,14 @@ EVT EVT::getExtendedVectorElementType() const {
 
 unsigned EVT::getExtendedVectorNumElements() const {
   assert(isExtended() && "Type is not extended!");
-  return cast<VectorType>(LLVMTy)->getNumElements();
+  ElementCount EC = cast<VectorType>(LLVMTy)->getElementCount();
+  if (EC.Scalable) {
+    WithColor::warning()
+        << "The code that requested the fixed number of elements has made the "
+           "assumption that this vector is not scalable. This assumption was "
+           "not correct, and this may lead to broken code\n";
+  }
+  return EC.Min;
 }
 
 ElementCount EVT::getExtendedVectorElementCount() const {
@@ -299,6 +306,10 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const {
     return FixedVectorType::get(Type::getHalfTy(Context), 16);
   case MVT::v32f16:
     return FixedVectorType::get(Type::getHalfTy(Context), 32);
+  case MVT::v64f16:
+    return FixedVectorType::get(Type::getBFloatTy(Context), 64);
+  case MVT::v128f16:
+    return FixedVectorType::get(Type::getBFloatTy(Context), 128);
   case MVT::v2bf16:
     return FixedVectorType::get(Type::getBFloatTy(Context), 2);
   case MVT::v3bf16:
@@ -311,6 +322,10 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const {
     return FixedVectorType::get(Type::getBFloatTy(Context), 16);
   case MVT::v32bf16:
     return FixedVectorType::get(Type::getBFloatTy(Context), 32);
+  case MVT::v64bf16:
+    return FixedVectorType::get(Type::getBFloatTy(Context), 64);
+  case MVT::v128bf16:
+    return FixedVectorType::get(Type::getBFloatTy(Context), 128);
   case MVT::v1f32:
     return FixedVectorType::get(Type::getFloatTy(Context), 1);
   case MVT::v2f32:
@@ -349,6 +364,8 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const {
     return FixedVectorType::get(Type::getDoubleTy(Context), 8);
   case MVT::v16f64:
     return FixedVectorType::get(Type::getDoubleTy(Context), 16);
+  case MVT::v32f64:
+    return FixedVectorType::get(Type::getDoubleTy(Context), 32);
   case MVT::nxv1i1:
     return ScalableVectorType::get(Type::getInt1Ty(Context), 1);
   case MVT::nxv2i1:
@@ -361,6 +378,8 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const {
     return ScalableVectorType::get(Type::getInt1Ty(Context), 16);
   case MVT::nxv32i1:
     return ScalableVectorType::get(Type::getInt1Ty(Context), 32);
+  case MVT::nxv64i1:
+    return ScalableVectorType::get(Type::getInt1Ty(Context), 64);
   case MVT::nxv1i8:
     return ScalableVectorType::get(Type::getInt8Ty(Context), 1);
   case MVT::nxv2i8:
@@ -373,6 +392,8 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const {
     return ScalableVectorType::get(Type::getInt8Ty(Context), 16);
   case MVT::nxv32i8:
     return ScalableVectorType::get(Type::getInt8Ty(Context), 32);
+  case MVT::nxv64i8:
+    return ScalableVectorType::get(Type::getInt8Ty(Context), 64);
   case MVT::nxv1i16:
     return ScalableVectorType::get(Type::getInt16Ty(Context), 1);
   case MVT::nxv2i16:
@@ -409,12 +430,18 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const {
     return ScalableVectorType::get(Type::getInt64Ty(Context), 16);
   case MVT::nxv32i64:
     return ScalableVectorType::get(Type::getInt64Ty(Context), 32);
+  case MVT::nxv1f16:
+    return ScalableVectorType::get(Type::getHalfTy(Context), 1);
   case MVT::nxv2f16:
     return ScalableVectorType::get(Type::getHalfTy(Context), 2);
   case MVT::nxv4f16:
     return ScalableVectorType::get(Type::getHalfTy(Context), 4);
   case MVT::nxv8f16:
     return ScalableVectorType::get(Type::getHalfTy(Context), 8);
+  case MVT::nxv16f16:
+    return ScalableVectorType::get(Type::getHalfTy(Context), 16);
+  case MVT::nxv32f16:
+    return ScalableVectorType::get(Type::getHalfTy(Context), 32);
   case MVT::nxv2bf16:
     return ScalableVectorType::get(Type::getBFloatTy(Context), 2);
   case MVT::nxv4bf16:
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp b/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
index ebe2600e06892..bf62194977706 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
@@ -336,6 +336,13 @@ static void dumpLoclistsSection(raw_ostream &OS, DIDumpOptions DumpOpts,
   }
 }
 
+static void dumpPubTableSection(raw_ostream &OS, DIDumpOptions DumpOpts,
+                                DWARFDataExtractor Data, bool GnuStyle) {
+  DWARFDebugPubTable Table;
+  Table.extract(Data, GnuStyle, DumpOpts.RecoverableErrorHandler);
+  Table.dump(OS);
+}
+
 void DWARFContext::dump(
     raw_ostream &OS, DIDumpOptions DumpOpts,
     std::array<Optional<uint64_t>, DIDT_ID_Count> DumpOffsets) {
@@ -626,26 +633,32 @@ void DWARFContext::dump(
   }
 
   if (shouldDump(Explicit, ".debug_pubnames", DIDT_ID_DebugPubnames,
-                 DObj->getPubnamesSection().Data))
-    DWARFDebugPubTable(*DObj, DObj->getPubnamesSection(), isLittleEndian(), false)
-        .dump(OS);
+                 DObj->getPubnamesSection().Data)) {
+    DWARFDataExtractor PubTableData(*DObj, DObj->getPubnamesSection(),
+                                    isLittleEndian(), 0);
+    dumpPubTableSection(OS, DumpOpts, PubTableData, /*GnuStyle=*/false);
+  }
 
   if (shouldDump(Explicit, ".debug_pubtypes", DIDT_ID_DebugPubtypes,
-                 DObj->getPubtypesSection().Data))
-    DWARFDebugPubTable(*DObj, DObj->getPubtypesSection(), isLittleEndian(), false)
-        .dump(OS);
+                 DObj->getPubtypesSection().Data)) {
+    DWARFDataExtractor PubTableData(*DObj, DObj->getPubtypesSection(),
+                                    isLittleEndian(), 0);
+    dumpPubTableSection(OS, DumpOpts, PubTableData, /*GnuStyle=*/false);
+  }
 
   if (shouldDump(Explicit, ".debug_gnu_pubnames", DIDT_ID_DebugGnuPubnames,
-                 DObj->getGnuPubnamesSection().Data))
-    DWARFDebugPubTable(*DObj, DObj->getGnuPubnamesSection(), isLittleEndian(),
-                       true /* GnuStyle */)
-        .dump(OS);
+                 DObj->getGnuPubnamesSection().Data)) {
+    DWARFDataExtractor PubTableData(*DObj, DObj->getGnuPubnamesSection(),
+                                    isLittleEndian(), 0);
+    dumpPubTableSection(OS, DumpOpts, PubTableData, /*GnuStyle=*/true);
+  }
 
   if (shouldDump(Explicit, ".debug_gnu_pubtypes", DIDT_ID_DebugGnuPubtypes,
-                 DObj->getGnuPubtypesSection().Data))
-    DWARFDebugPubTable(*DObj, DObj->getGnuPubtypesSection(), isLittleEndian(),
-                       true /* GnuStyle */)
-        .dump(OS);
+                 DObj->getGnuPubtypesSection().Data)) {
+    DWARFDataExtractor PubTableData(*DObj, DObj->getGnuPubtypesSection(),
+                                    isLittleEndian(), 0);
+    dumpPubTableSection(OS, DumpOpts, PubTableData, /*GnuStyle=*/true);
+  }
 
   if (shouldDump(Explicit, ".debug_str_offsets", DIDT_ID_DebugStrOffsets,
                  DObj->getStrOffsetsSection().Data))
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
index 746ba379c0ced..0a1b75592290c 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
@@ -36,129 +36,130 @@ const uint8_t DWARF_CFI_PRIMARY_OPERAND_MASK = 0x3f;
 
 Error CFIProgram::parse(DWARFDataExtractor Data, uint64_t *Offset,
                         uint64_t EndOffset) {
-  while (*Offset < EndOffset) {
-    uint8_t Opcode = Data.getRelocatedValue(1, Offset);
-    // Some instructions have a primary opcode encoded in the top bits.
-    uint8_t Primary = Opcode & DWARF_CFI_PRIMARY_OPCODE_MASK;
+  DataExtractor::Cursor C(*Offset);
+  while (C && C.tell() < EndOffset) {
+    uint8_t Opcode = Data.getRelocatedValue(C, 1);
+    if (!C)
+      break;
 
-    if (Primary) {
+    // Some instructions have a primary opcode encoded in the top bits.
+    if (uint8_t Primary = Opcode & DWARF_CFI_PRIMARY_OPCODE_MASK) {
       // If it's a primary opcode, the first operand is encoded in the bottom
       // bits of the opcode itself.
       uint64_t Op1 = Opcode & DWARF_CFI_PRIMARY_OPERAND_MASK;
       switch (Primary) {
-      default:
-        return createStringError(errc::illegal_byte_sequence,
-                                 "Invalid primary CFI opcode 0x%" PRIx8,
-                                 Primary);
       case DW_CFA_advance_loc:
       case DW_CFA_restore:
         addInstruction(Primary, Op1);
         break;
       case DW_CFA_offset:
-        addInstruction(Primary, Op1, Data.getULEB128(Offset));
+        addInstruction(Primary, Op1, Data.getULEB128(C));
         break;
-      }
-    } else {
-      // Extended opcode - its value is Opcode itself.
-      switch (Opcode) {
       default:
-        return createStringError(errc::illegal_byte_sequence,
-                                 "Invalid extended CFI opcode 0x%" PRIx8,
-                                 Opcode);
-      case DW_CFA_nop:
-      case DW_CFA_remember_state:
-      case DW_CFA_restore_state:
-      case DW_CFA_GNU_window_save:
-        // No operands
-        addInstruction(Opcode);
-        break;
-      case DW_CFA_set_loc:
-        // Operands: Address
-        addInstruction(Opcode, Data.getRelocatedAddress(Offset));
-        break;
-      case DW_CFA_advance_loc1:
-        // Operands: 1-byte delta
-        addInstruction(Opcode, Data.getRelocatedValue(1, Offset));
-        break;
-      case DW_CFA_advance_loc2:
-        // Operands: 2-byte delta
-        addInstruction(Opcode, Data.getRelocatedValue(2, Offset));
-        break;
-      case DW_CFA_advance_loc4:
-        // Operands: 4-byte delta
-        addInstruction(Opcode, Data.getRelocatedValue(4, Offset));
-        break;
-      case DW_CFA_restore_extended:
-      case DW_CFA_undefined:
-      case DW_CFA_same_value:
-      case DW_CFA_def_cfa_register:
-      case DW_CFA_def_cfa_offset:
-      case DW_CFA_GNU_args_size:
-        // Operands: ULEB128
-        addInstruction(Opcode, Data.getULEB128(Offset));
-        break;
-      case DW_CFA_def_cfa_offset_sf:
-        // Operands: SLEB128
-        addInstruction(Opcode, Data.getSLEB128(Offset));
-        break;
-      case DW_CFA_offset_extended:
-      case DW_CFA_register:
-      case DW_CFA_def_cfa:
-      case DW_CFA_val_offset: {
-        // Operands: ULEB128, ULEB128
-        // Note: We can not embed getULEB128 directly into function
-        // argument list. getULEB128 changes Offset and order of evaluation
-        // for arguments is unspecified.
-        auto op1 = Data.getULEB128(Offset);
-        auto op2 = Data.getULEB128(Offset);
-        addInstruction(Opcode, op1, op2);
-        break;
-        }
-        case DW_CFA_offset_extended_sf:
-        case DW_CFA_def_cfa_sf:
-        case DW_CFA_val_offset_sf: {
-          // Operands: ULEB128, SLEB128
-          // Note: see comment for the previous case
-          auto op1 = Data.getULEB128(Offset);
-          auto op2 = (uint64_t)Data.getSLEB128(Offset);
-          addInstruction(Opcode, op1, op2);
-          break;
-        }
-        case DW_CFA_def_cfa_expression: {
-          uint32_t ExprLength = Data.getULEB128(Offset);
-          addInstruction(Opcode, 0);
-          DataExtractor Extractor(
-              Data.getData().slice(*Offset, *Offset + ExprLength),
-              Data.isLittleEndian(), Data.getAddressSize());
-          // Note. We do not pass the DWARF format to DWARFExpression, because
-          // DW_OP_call_ref, the only operation which depends on the format, is
-          // prohibited in call frame instructions, see sec. 6.4.2 in DWARFv5.
-          Instructions.back().Expression =
-              DWARFExpression(Extractor, Data.getAddressSize());
-          *Offset += ExprLength;
-          break;
-        }
-        case DW_CFA_expression:
-        case DW_CFA_val_expression: {
-          auto RegNum = Data.getULEB128(Offset);
-          auto BlockLength = Data.getULEB128(Offset);
-          addInstruction(Opcode, RegNum, 0);
-          DataExtractor Extractor(
-              Data.getData().slice(*Offset, *Offset + BlockLength),
-              Data.isLittleEndian(), Data.getAddressSize());
-          // Note. We do not pass the DWARF format to DWARFExpression, because
-          // DW_OP_call_ref, the only operation which depends on the format, is
-          // prohibited in call frame instructions, see sec. 6.4.2 in DWARFv5.
-          Instructions.back().Expression =
-              DWARFExpression(Extractor, Data.getAddressSize());
-          *Offset += BlockLength;
-          break;
-        }
+        llvm_unreachable("invalid primary CFI opcode");
       }
+      continue;
+    }
+
+    // Extended opcode - its value is Opcode itself.
+    switch (Opcode) {
+    default:
+      return createStringError(errc::illegal_byte_sequence,
+                               "invalid extended CFI opcode 0x%" PRIx8, Opcode);
+    case DW_CFA_nop:
+    case DW_CFA_remember_state:
+    case DW_CFA_restore_state:
+    case DW_CFA_GNU_window_save:
+      // No operands
+      addInstruction(Opcode);
+      break;
+    case DW_CFA_set_loc:
+      // Operands: Address
+      addInstruction(Opcode, Data.getRelocatedAddress(C));
+      break;
+    case DW_CFA_advance_loc1:
+      // Operands: 1-byte delta
+      addInstruction(Opcode, Data.getRelocatedValue(C, 1));
+      break;
+    case DW_CFA_advance_loc2:
+      // Operands: 2-byte delta
+      addInstruction(Opcode, Data.getRelocatedValue(C, 2));
+      break;
+    case DW_CFA_advance_loc4:
+      // Operands: 4-byte delta
+      addInstruction(Opcode, Data.getRelocatedValue(C, 4));
+      break;
+    case DW_CFA_restore_extended:
+    case DW_CFA_undefined:
+    case DW_CFA_same_value:
+    case DW_CFA_def_cfa_register:
+    case DW_CFA_def_cfa_offset:
+    case DW_CFA_GNU_args_size:
+      // Operands: ULEB128
+      addInstruction(Opcode, Data.getULEB128(C));
+      break;
+    case DW_CFA_def_cfa_offset_sf:
+      // Operands: SLEB128
+      addInstruction(Opcode, Data.getSLEB128(C));
+      break;
+    case DW_CFA_offset_extended:
+    case DW_CFA_register:
+    case DW_CFA_def_cfa:
+    case DW_CFA_val_offset: {
+      // Operands: ULEB128, ULEB128
+      // Note: We can not embed getULEB128 directly into function
+      // argument list. getULEB128 changes Offset and order of evaluation
+      // for arguments is unspecified.
+      uint64_t op1 = Data.getULEB128(C);
+      uint64_t op2 = Data.getULEB128(C);
+      addInstruction(Opcode, op1, op2);
+      break;
+    }
+    case DW_CFA_offset_extended_sf:
+    case DW_CFA_def_cfa_sf:
+    case DW_CFA_val_offset_sf: {
+      // Operands: ULEB128, SLEB128
+      // Note: see comment for the previous case
+      uint64_t op1 = Data.getULEB128(C);
+      uint64_t op2 = (uint64_t)Data.getSLEB128(C);
+      addInstruction(Opcode, op1, op2);
+      break;
+    }
+    case DW_CFA_def_cfa_expression: {
+      uint64_t ExprLength = Data.getULEB128(C);
+      addInstruction(Opcode, 0);
+      StringRef Expression = Data.getBytes(C, ExprLength);
+
+      DataExtractor Extractor(Expression, Data.isLittleEndian(),
+                              Data.getAddressSize());
+      // Note. We do not pass the DWARF format to DWARFExpression, because
+      // DW_OP_call_ref, the only operation which depends on the format, is
+      // prohibited in call frame instructions, see sec. 6.4.2 in DWARFv5.
+      Instructions.back().Expression =
+          DWARFExpression(Extractor, Data.getAddressSize());
+      break;
+    }
+    case DW_CFA_expression:
+    case DW_CFA_val_expression: {
+      uint64_t RegNum = Data.getULEB128(C);
+      addInstruction(Opcode, RegNum, 0);
+
+      uint64_t BlockLength = Data.getULEB128(C);
+      StringRef Expression = Data.getBytes(C, BlockLength);
+      DataExtractor Extractor(Expression, Data.isLittleEndian(),
+                              Data.getAddressSize());
+      // Note. We do not pass the DWARF format to DWARFExpression, because
+      // DW_OP_call_ref, the only operation which depends on the format, is
+      // prohibited in call frame instructions, see sec. 6.4.2 in DWARFv5.
+      Instructions.back().Expression =
+          DWARFExpression(Extractor, Data.getAddressSize());
+      break;
+    }
     }
   }
 
-  return Error::success();
+  *Offset = C.tell();
+  return C.takeError();
 }
 
 namespace {
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
index f45ab85b0ed8c..3ca21e97888cf 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
@@ -351,48 +351,46 @@ Error DWARFDebugLine::Prologue::parse(
   const uint64_t PrologueOffset = *OffsetPtr;
 
   clear();
-  Error Err = Error::success();
+  DataExtractor::Cursor Cursor(*OffsetPtr);
   std::tie(TotalLength, FormParams.Format) =
-      DebugLineData.getInitialLength(OffsetPtr, &Err);
-  if (Err)
-    return createStringError(
-        errc::invalid_argument,
-        "parsing line table prologue at offset 0x%8.8" PRIx64 ": %s",
-        PrologueOffset, toString(std::move(Err)).c_str());
+      DebugLineData.getInitialLength(Cursor);
 
-  DebugLineData = DWARFDataExtractor(DebugLineData, *OffsetPtr + TotalLength);
-  FormParams.Version = DebugLineData.getU16(OffsetPtr);
-  if (!versionIsSupported(getVersion()))
+  DebugLineData =
+      DWARFDataExtractor(DebugLineData, Cursor.tell() + TotalLength);
+  FormParams.Version = DebugLineData.getU16(Cursor);
+  if (Cursor && !versionIsSupported(getVersion())) {
     // Treat this error as unrecoverable - we cannot be sure what any of
     // the data represents including the length field, so cannot skip it or make
     // any reasonable assumptions.
+    *OffsetPtr = Cursor.tell();
     return createStringError(
         errc::not_supported,
         "parsing line table prologue at offset 0x%8.8" PRIx64
         ": unsupported version %" PRIu16,
         PrologueOffset, getVersion());
+  }
 
   if (getVersion() >= 5) {
-    FormParams.AddrSize = DebugLineData.getU8(OffsetPtr);
-    assert((DebugLineData.getAddressSize() == 0 ||
+    FormParams.AddrSize = DebugLineData.getU8(Cursor);
+    assert((!Cursor || DebugLineData.getAddressSize() == 0 ||
             DebugLineData.getAddressSize() == getAddressSize()) &&
            "Line table header and data extractor disagree");
-    SegSelectorSize = DebugLineData.getU8(OffsetPtr);
+    SegSelectorSize = DebugLineData.getU8(Cursor);
   }
 
   PrologueLength =
-      DebugLineData.getRelocatedValue(sizeofPrologueLength(), OffsetPtr);
-  const uint64_t EndPrologueOffset = PrologueLength + *OffsetPtr;
+      DebugLineData.getRelocatedValue(Cursor, sizeofPrologueLength());
+  const uint64_t EndPrologueOffset = PrologueLength + Cursor.tell();
   DebugLineData = DWARFDataExtractor(DebugLineData, EndPrologueOffset);
-  MinInstLength = DebugLineData.getU8(OffsetPtr);
+  MinInstLength = DebugLineData.getU8(Cursor);
   if (getVersion() >= 4)
-    MaxOpsPerInst = DebugLineData.getU8(OffsetPtr);
-  DefaultIsStmt = DebugLineData.getU8(OffsetPtr);
-  LineBase = DebugLineData.getU8(OffsetPtr);
-  LineRange = DebugLineData.getU8(OffsetPtr);
-  OpcodeBase = DebugLineData.getU8(OffsetPtr);
+    MaxOpsPerInst = DebugLineData.getU8(Cursor);
+  DefaultIsStmt = DebugLineData.getU8(Cursor);
+  LineBase = DebugLineData.getU8(Cursor);
+  LineRange = DebugLineData.getU8(Cursor);
+  OpcodeBase = DebugLineData.getU8(Cursor);
 
-  if (OpcodeBase == 0) {
+  if (Cursor && OpcodeBase == 0) {
     // If the opcode base is 0, we cannot read the standard opcode lengths (of
     // which there are supposed to be one fewer than the opcode base). Assume
     // there are no standard opcodes and continue parsing.
@@ -401,14 +399,24 @@ Error DWARFDebugLine::Prologue::parse(
         "parsing line table prologue at offset 0x%8.8" PRIx64
         " found opcode base of 0. Assuming no standard opcodes",
         PrologueOffset));
-  } else {
+  } else if (Cursor) {
     StandardOpcodeLengths.reserve(OpcodeBase - 1);
     for (uint32_t I = 1; I < OpcodeBase; ++I) {
-      uint8_t OpLen = DebugLineData.getU8(OffsetPtr);
+      uint8_t OpLen = DebugLineData.getU8(Cursor);
       StandardOpcodeLengths.push_back(OpLen);
     }
   }
 
+  *OffsetPtr = Cursor.tell();
+  // A corrupt file name or directory table does not prevent interpretation of
+  // the main line program, so check the cursor state now so that its errors can
+  // be handled separately.
+  if (!Cursor)
+    return createStringError(
+        errc::invalid_argument,
+        "parsing line table prologue at offset 0x%8.8" PRIx64 ": %s",
+        PrologueOffset, toString(Cursor.takeError()).c_str());
+
   Error E =
       getVersion() >= 5
           ? parseV5DirFileTables(DebugLineData, OffsetPtr, FormParams, Ctx, U,
@@ -784,15 +792,21 @@ Error DWARFDebugLine::LineTable::parse(
 
       // Tolerate zero-length; assume length is correct and soldier on.
       if (Len == 0) {
-        if (Verbose)
+        if (Cursor && Verbose)
           *OS << "Badly formed extended line op (length 0)\n";
-        if (!Cursor)
+        if (!Cursor) {
+          if (Verbose)
+            *OS << "\n";
           RecoverableErrorHandler(Cursor.takeError());
+        }
         *OffsetPtr = Cursor.tell();
         continue;
       }
 
       uint8_t SubOpcode = TableData.getU8(Cursor);
+      // OperandOffset will be the same as ExtOffset, if it was not possible to
+      // read the SubOpcode.
+      uint64_t OperandOffset = Cursor.tell();
       if (Verbose)
         *OS << LNExtendedString(SubOpcode);
       switch (SubOpcode) {
@@ -805,6 +819,9 @@ Error DWARFDebugLine::LineTable::parse(
         // address is that of the byte after the last target machine instruction
         // of the sequence.
         State.Row.EndSequence = true;
+        // No need to test the Cursor is valid here, since it must be to get
+        // into this code path - if it were invalid, the default case would be
+        // followed.
         if (Verbose) {
           *OS << "\n";
           OS->indent(12);
@@ -858,7 +875,7 @@ Error DWARFDebugLine::LineTable::parse(
               TableData.setAddressSize(ExtractorAddressSize);
           }
 
-          if (Verbose)
+          if (Cursor && Verbose)
             *OS << format(" (0x%16.16" PRIx64 ")", State.Row.Address.Address);
         }
         break;
@@ -893,7 +910,7 @@ Error DWARFDebugLine::LineTable::parse(
           FileEntry.ModTime = TableData.getULEB128(Cursor);
           FileEntry.Length = TableData.getULEB128(Cursor);
           Prologue.FileNames.push_back(FileEntry);
-          if (Verbose)
+          if (Cursor && Verbose)
             *OS << " (" << Name << ", dir=" << FileEntry.DirIdx << ", mod_time="
                 << format("(0x%16.16" PRIx64 ")", FileEntry.ModTime)
                 << ", length=" << FileEntry.Length << ")";
@@ -902,12 +919,12 @@ Error DWARFDebugLine::LineTable::parse(
 
       case DW_LNE_set_discriminator:
         State.Row.Discriminator = TableData.getULEB128(Cursor);
-        if (Verbose)
+        if (Cursor && Verbose)
           *OS << " (" << State.Row.Discriminator << ")";
         break;
 
       default:
-        if (Verbose)
+        if (Cursor && Verbose)
           *OS << format("Unrecognized extended op 0x%02.02" PRIx8, SubOpcode)
               << format(" length %" PRIx64, Len);
         // Len doesn't include the zero opcode byte or the length itself, but
@@ -926,6 +943,23 @@ Error DWARFDebugLine::LineTable::parse(
             "unexpected line op length at offset 0x%8.8" PRIx64
             " expected 0x%2.2" PRIx64 " found 0x%2.2" PRIx64,
             ExtOffset, Len, Cursor.tell() - ExtOffset));
+      if (!Cursor && Verbose) {
+        DWARFDataExtractor::Cursor ByteCursor(OperandOffset);
+        uint8_t Byte = TableData.getU8(ByteCursor);
+        if (ByteCursor) {
+          *OS << " (<parsing error>";
+          do {
+            *OS << format(" %2.2" PRIx8, Byte);
+            Byte = TableData.getU8(ByteCursor);
+          } while (ByteCursor);
+          *OS << ")";
+        }
+
+        // The only parse failure in this case should be if the end was reached.
+        // In that case, throw away the error, as the main Cursor's error will
+        // be sufficient.
+        consumeError(ByteCursor.takeError());
+      }
       *OffsetPtr = End;
     } else if (Opcode < Prologue.OpcodeBase) {
       if (Verbose)
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugPubTable.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugPubTable.cpp
index eecca87d52704..5031acdb54efc 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDebugPubTable.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugPubTable.cpp
@@ -11,6 +11,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Support/DataExtractor.h"
+#include "llvm/Support/Errc.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdint>
@@ -18,33 +19,74 @@
 using namespace llvm;
 using namespace dwarf;
 
-DWARFDebugPubTable::DWARFDebugPubTable(const DWARFObject &Obj,
-                                       const DWARFSection &Sec,
-                                       bool LittleEndian, bool GnuStyle)
-    : GnuStyle(GnuStyle) {
-  DWARFDataExtractor PubNames(Obj, Sec, LittleEndian, 0);
+void DWARFDebugPubTable::extract(
+    DWARFDataExtractor Data, bool GnuStyle,
+    function_ref<void(Error)> RecoverableErrorHandler) {
+  this->GnuStyle = GnuStyle;
+  Sets.clear();
   uint64_t Offset = 0;
-  while (PubNames.isValidOffset(Offset)) {
+  while (Data.isValidOffset(Offset)) {
+    uint64_t SetOffset = Offset;
     Sets.push_back({});
-    Set &SetData = Sets.back();
+    Set &NewSet = Sets.back();
 
-    std::tie(SetData.Length, SetData.Format) =
-        PubNames.getInitialLength(&Offset);
-    const unsigned OffsetSize = dwarf::getDwarfOffsetByteSize(SetData.Format);
+    DataExtractor::Cursor C(Offset);
+    std::tie(NewSet.Length, NewSet.Format) = Data.getInitialLength(C);
+    if (!C) {
+      // Drop the newly added set because it does not contain anything useful
+      // to dump.
+      Sets.pop_back();
+      RecoverableErrorHandler(createStringError(
+          errc::invalid_argument,
+          "name lookup table at offset 0x%" PRIx64 " parsing failed: %s",
+          SetOffset, toString(C.takeError()).c_str()));
+      return;
+    }
+
+    Offset = C.tell() + NewSet.Length;
+    DWARFDataExtractor SetData(Data, Offset);
+    const unsigned OffsetSize = dwarf::getDwarfOffsetByteSize(NewSet.Format);
+
+    NewSet.Version = SetData.getU16(C);
+    NewSet.Offset = SetData.getRelocatedValue(C, OffsetSize);
+    NewSet.Size = SetData.getUnsigned(C, OffsetSize);
 
-    SetData.Version = PubNames.getU16(&Offset);
-    SetData.Offset = PubNames.getRelocatedValue(OffsetSize, &Offset);
-    SetData.Size = PubNames.getUnsigned(&Offset, OffsetSize);
+    if (!C) {
+      // Preserve the newly added set because at least some fields of the header
+      // are read and can be dumped.
+      RecoverableErrorHandler(
+          createStringError(errc::invalid_argument,
+                            "name lookup table at offset 0x%" PRIx64
+                            " does not have a complete header: %s",
+                            SetOffset, toString(C.takeError()).c_str()));
+      continue;
+    }
 
-    while (Offset < Sec.Data.size()) {
-      uint64_t DieRef = PubNames.getUnsigned(&Offset, OffsetSize);
+    while (C) {
+      uint64_t DieRef = SetData.getUnsigned(C, OffsetSize);
       if (DieRef == 0)
         break;
-      uint8_t IndexEntryValue = GnuStyle ? PubNames.getU8(&Offset) : 0;
-      StringRef Name = PubNames.getCStrRef(&Offset);
-      SetData.Entries.push_back(
-          {DieRef, PubIndexEntryDescriptor(IndexEntryValue), Name});
+      uint8_t IndexEntryValue = GnuStyle ? SetData.getU8(C) : 0;
+      StringRef Name = SetData.getCStrRef(C);
+      if (C)
+        NewSet.Entries.push_back(
+            {DieRef, PubIndexEntryDescriptor(IndexEntryValue), Name});
+    }
+
+    if (!C) {
+      RecoverableErrorHandler(createStringError(
+          errc::invalid_argument,
+          "name lookup table at offset 0x%" PRIx64 " parsing failed: %s",
+          SetOffset, toString(C.takeError()).c_str()));
+      continue;
     }
+    if (C.tell() != Offset)
+      RecoverableErrorHandler(createStringError(
+          errc::invalid_argument,
+          "name lookup table at offset 0x%" PRIx64
+          " has a terminator at offset 0x%" PRIx64
+          " before the expected end at 0x%" PRIx64,
+          SetOffset, C.tell() - OffsetSize, Offset - OffsetSize));
   }
 }
 
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp b/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
index 28b24b3baaab3..d3c1cd5bb88f4 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
@@ -374,4 +374,74 @@ bool DWARFExpression::verify(DWARFUnit *U) {
   return true;
 }
 
+/// A user-facing string representation of a DWARF expression. This might be an
+/// Address expression, in which case it will be implicitly dereferenced, or a
+/// Value expression.
+struct PrintedExpr {
+  enum ExprKind {
+    Address,
+    Value,
+  };
+  ExprKind Kind;
+  SmallString<16> String;
+
+  PrintedExpr(ExprKind K = Address) : Kind(K) {}
+};
+
+static bool printCompactDWARFExpr(raw_ostream &OS, DWARFExpression::iterator I,
+                                  const DWARFExpression::iterator E,
+                                  const MCRegisterInfo &MRI) {
+  SmallVector<PrintedExpr, 4> Stack;
+
+  while (I != E) {
+    DWARFExpression::Operation &Op = *I;
+    uint8_t Opcode = Op.getCode();
+    switch (Opcode) {
+    case dwarf::DW_OP_regx: {
+      // DW_OP_regx: A register, with the register num given as an operand.
+      // Printed as the plain register name.
+      uint64_t DwarfRegNum = Op.getRawOperand(0);
+      Optional<unsigned> LLVMRegNum = MRI.getLLVMRegNum(DwarfRegNum, false);
+      if (!LLVMRegNum) {
+        OS << "<unknown register " << DwarfRegNum << ">";
+        return false;
+      }
+      raw_svector_ostream S(Stack.emplace_back(PrintedExpr::Value).String);
+      S << MRI.getName(*LLVMRegNum);
+      break;
+    }
+    default:
+      if (Opcode >= dwarf::DW_OP_reg0 && Opcode <= dwarf::DW_OP_reg31) {
+        // DW_OP_reg<N>: A register, with the register num implied by the
+        // opcode. Printed as the plain register name.
+        uint64_t DwarfRegNum = Opcode - dwarf::DW_OP_reg0;
+        Optional<unsigned> LLVMRegNum = MRI.getLLVMRegNum(DwarfRegNum, false);
+        if (!LLVMRegNum) {
+          OS << "<unknown register " << DwarfRegNum << ">";
+          return false;
+        }
+        raw_svector_ostream S(Stack.emplace_back(PrintedExpr::Value).String);
+        S << MRI.getName(*LLVMRegNum);
+      } else {
+        // If we hit an unknown operand, we don't know its effect on the stack,
+        // so bail out on the whole expression.
+        OS << "<unknown op " << dwarf::OperationEncodingString(Opcode) << " ("
+           << (int)Opcode << ")>";
+        return false;
+      }
+      break;
+    }
+    ++I;
+  }
+
+  assert(Stack.size() == 1 && "expected one value on stack");
+  OS << Stack.front().String;
+
+  return true;
+}
+
+bool DWARFExpression::printCompact(raw_ostream &OS, const MCRegisterInfo &MRI) {
+  return printCompactDWARFExpr(OS, begin(), end(), MRI);
+}
+
 } // namespace llvm
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp b/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
index f902041670507..a6d44f04e4680 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
@@ -300,7 +300,7 @@ bool DWARFUnitHeader::extract(DWARFContext &Context,
                 TypeOffset < getLength() + getUnitLengthFieldByteSize();
   bool LengthOK = debug_info.isValidOffset(getNextUnitOffset() - 1);
   bool VersionOK = DWARFContext::isSupportedVersion(getVersion());
-  bool AddrSizeOK = getAddressByteSize() == 4 || getAddressByteSize() == 8;
+  bool AddrSizeOK = DWARFContext::isAddressSizeSupported(getAddressByteSize());
 
   if (!LengthOK || !VersionOK || !AddrSizeOK || !TypeOffsetOK)
     return false;
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
index d5b016728c71d..3a83317a73a3b 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
@@ -26,24 +26,26 @@ using namespace llvm;
 using namespace dwarf;
 using namespace object;
 
-DWARFVerifier::DieRangeInfo::address_range_iterator
+Optional<DWARFAddressRange>
 DWARFVerifier::DieRangeInfo::insert(const DWARFAddressRange &R) {
   auto Begin = Ranges.begin();
   auto End = Ranges.end();
   auto Pos = std::lower_bound(Begin, End, R);
 
   if (Pos != End) {
-    if (Pos->intersects(R))
-      return std::move(Pos);
-    if (Pos != Begin) {
-      auto Iter = Pos - 1;
-      if (Iter->intersects(R))
-        return std::move(Iter);
-    }
+    DWARFAddressRange Range(*Pos);
+    if (Pos->merge(R))
+      return Range;
+  }
+  if (Pos != Begin) {
+    auto Iter = Pos - 1;
+    DWARFAddressRange Range(*Iter);
+    if (Iter->merge(R))
+      return Range;
   }
 
   Ranges.insert(Pos, R);
-  return Ranges.end();
+  return None;
 }
 
 DWARFVerifier::DieRangeInfo::die_range_info_iterator
@@ -133,7 +135,7 @@ bool DWARFVerifier::verifyUnitHeader(const DWARFDataExtractor DebugInfoData,
 
   ValidLength = DebugInfoData.isValidOffset(OffsetStart + Length + 3);
   ValidVersion = DWARFContext::isSupportedVersion(Version);
-  ValidAddrSize = AddrSize == 4 || AddrSize == 8;
+  ValidAddrSize = DWARFContext::isAddressSizeSupported(AddrSize);
   if (!ValidLength || !ValidVersion || !ValidAddrSize || !ValidAbbrevOffset ||
       !ValidType) {
     Success = false;
@@ -397,22 +399,30 @@ unsigned DWARFVerifier::verifyDieRanges(const DWARFDie &Die,
   // processing an object file.
 
   if (!IsObjectFile || IsMachOObject || Die.getTag() != DW_TAG_compile_unit) {
+    bool DumpDieAfterError = false;
     for (auto Range : Ranges) {
       if (!Range.valid()) {
         ++NumErrors;
         error() << "Invalid address range " << Range << "\n";
+        DumpDieAfterError = true;
         continue;
       }
 
-      // Verify that ranges don't intersect.
-      const auto IntersectingRange = RI.insert(Range);
-      if (IntersectingRange != RI.Ranges.end()) {
+      // Verify that ranges don't intersect and also build up the DieRangeInfo
+      // address ranges. Don't break out of the loop below early, or we will
+      // think this DIE doesn't have all of the address ranges it is supposed
+      // to have. Compile units often have DW_AT_ranges that can contain one or
+      // more dead stripped address ranges which tend to all be at the same
+      // address: 0 or -1.
+      if (auto PrevRange = RI.insert(Range)) {
         ++NumErrors;
-        error() << "DIE has overlapping address ranges: " << Range << " and "
-                << *IntersectingRange << "\n";
-        break;
+        error() << "DIE has overlapping ranges in DW_AT_ranges attribute: "
+                << *PrevRange << " and " << Range << '\n';
+        DumpDieAfterError = true;
       }
     }
+    if (DumpDieAfterError)
+      dump(Die, 2) << '\n';
   }
 
   // Verify that children don't intersect.
diff --git a/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp b/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
index 998ba1518b4f9..4e58489f14014 100644
--- a/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
@@ -13,8 +13,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h"
-
-#include "llvm/ADT/DenseSet.h"
 #include "llvm/DebugInfo/CodeView/RecordName.h"
 #include "llvm/DebugInfo/CodeView/SymbolDeserializer.h"
 #include "llvm/DebugInfo/CodeView/SymbolRecord.h"
@@ -36,58 +34,53 @@ using namespace llvm::msf;
 using namespace llvm::pdb;
 using namespace llvm::codeview;
 
+// Helper class for building the public and global PDB hash table buckets.
 struct llvm::pdb::GSIHashStreamBuilder {
-  struct SymbolDenseMapInfo {
-    static inline CVSymbol getEmptyKey() {
-      static CVSymbol Empty;
-      return Empty;
-    }
-    static inline CVSymbol getTombstoneKey() {
-      static CVSymbol Tombstone(
-          DenseMapInfo<ArrayRef<uint8_t>>::getTombstoneKey());
-      return Tombstone;
-    }
-    static unsigned getHashValue(const CVSymbol &Val) {
-      return xxHash64(Val.RecordData);
-    }
-    static bool isEqual(const CVSymbol &LHS, const CVSymbol &RHS) {
-      return LHS.RecordData == RHS.RecordData;
-    }
-  };
+  // Sum of the size of all public or global records.
+  uint32_t RecordByteSize = 0;
 
-  std::vector<CVSymbol> Records;
-  llvm::DenseSet<CVSymbol, SymbolDenseMapInfo> SymbolHashes;
   std::vector<PSHashRecord> HashRecords;
+
+  // The hash bitmap has `ceil((IPHR_HASH + 1) / 32)` words in it. The
+  // reference implementation builds a hash table with IPHR_HASH buckets in it.
+  // The last bucket is used to link together free hash table cells in a linked
+  // list, but it is always empty in the compressed, on-disk format. However,
+  // the bitmap must have a bit for it.
   std::array<support::ulittle32_t, (IPHR_HASH + 32) / 32> HashBitmap;
+
   std::vector<support::ulittle32_t> HashBuckets;
 
   uint32_t calculateSerializedLength() const;
-  uint32_t calculateRecordByteSize() const;
   Error commit(BinaryStreamWriter &Writer);
 
-  void finalizeBuckets(uint32_t RecordZeroOffset);
+  void finalizePublicBuckets();
+  void finalizeGlobalBuckets(uint32_t RecordZeroOffset);
 
-  // Finalize public symbol buckets.
+  // Assign public and global symbol records into hash table buckets.
+  // Modifies the list of records to store the bucket index, but does not
+  // change the order.
   void finalizeBuckets(uint32_t RecordZeroOffset,
-                       std::vector<BulkPublic> &&Publics);
-
-  template <typename T> void addSymbol(const T &Symbol, MSFBuilder &Msf) {
-    T Copy(Symbol);
-    addSymbol(SymbolSerializer::writeOneSymbol(Copy, Msf.getAllocator(),
-                                               CodeViewContainer::Pdb));
-  }
-  void addSymbol(const CVSymbol &Symbol);
+                       MutableArrayRef<BulkPublic> Globals);
 };
 
-void GSIHashStreamBuilder::addSymbol(const codeview::CVSymbol &Symbol) {
-  // Ignore duplicate typedefs and constants.
-  if (Symbol.kind() == S_UDT || Symbol.kind() == S_CONSTANT) {
-    auto Iter = SymbolHashes.insert(Symbol);
-    if (!Iter.second)
-      return;
+// DenseMapInfo implementation for deduplicating symbol records.
+struct llvm::pdb::SymbolDenseMapInfo {
+  static inline CVSymbol getEmptyKey() {
+    static CVSymbol Empty;
+    return Empty;
   }
-  Records.push_back(Symbol);
-}
+  static inline CVSymbol getTombstoneKey() {
+    static CVSymbol Tombstone(
+        DenseMapInfo<ArrayRef<uint8_t>>::getTombstoneKey());
+    return Tombstone;
+  }
+  static unsigned getHashValue(const CVSymbol &Val) {
+    return xxHash64(Val.RecordData);
+  }
+  static bool isEqual(const CVSymbol &LHS, const CVSymbol &RHS) {
+    return LHS.RecordData == RHS.RecordData;
+  }
+};
 
 namespace {
 LLVM_PACKED_START
@@ -118,7 +111,7 @@ static CVSymbol serializePublic(uint8_t *Mem, const BulkPublic &Pub) {
   FixedMem->Prefix.RecordLen = static_cast<uint16_t>(Size - 2);
   FixedMem->Pub.Flags = Pub.Flags;
   FixedMem->Pub.Offset = Pub.Offset;
-  FixedMem->Pub.Segment = Pub.U.Segment;
+  FixedMem->Pub.Segment = Pub.Segment;
   char *NameMem = reinterpret_cast<char *>(FixedMem + 1);
   memcpy(NameMem, Pub.Name, NameLen);
   // Zero the null terminator and remaining bytes.
@@ -134,13 +127,6 @@ uint32_t GSIHashStreamBuilder::calculateSerializedLength() const {
   return Size;
 }
 
-uint32_t GSIHashStreamBuilder::calculateRecordByteSize() const {
-  uint32_t Size = 0;
-  for (const auto &Sym : Records)
-    Size += Sym.length();
-  return Size;
-}
-
 Error GSIHashStreamBuilder::commit(BinaryStreamWriter &Writer) {
   GSIHashHeader Header;
   Header.VerSignature = GSIHashHeader::HdrSignature;
@@ -180,82 +166,119 @@ static int gsiRecordCmp(StringRef S1, StringRef S2) {
   return S1.compare_lower(S2.data());
 }
 
-void GSIHashStreamBuilder::finalizeBuckets(uint32_t RecordZeroOffset) {
-  // Build up a list of globals to be bucketed. This repurposes the BulkPublic
-  // struct with different meanings for the fields to avoid reallocating a new
-  // vector during public symbol table hash construction.
-  std::vector<BulkPublic> Globals;
-  Globals.resize(Records.size());
+void GSIStreamBuilder::finalizePublicBuckets() {
+  PSH->finalizeBuckets(0, Publics);
+}
+
+void GSIStreamBuilder::finalizeGlobalBuckets(uint32_t RecordZeroOffset) {
+  // Build up a list of globals to be bucketed. Use the BulkPublic data
+  // structure for this purpose, even though these are global records, not
+  // public records. Most of the same fields are required:
+  // - Name
+  // - NameLen
+  // - SymOffset
+  // - BucketIdx
+  // The dead fields are Offset, Segment, and Flags.
+  std::vector<BulkPublic> Records;
+  Records.resize(Globals.size());
   uint32_t SymOffset = RecordZeroOffset;
-  for (size_t I = 0, E = Records.size(); I < E; ++I) {
-    StringRef Name = getSymbolName(Records[I]);
-    Globals[I].Name = Name.data();
-    Globals[I].NameLen = Name.size();
-    Globals[I].SymOffset = SymOffset;
-    SymOffset += Records[I].length();
+  for (size_t I = 0, E = Globals.size(); I < E; ++I) {
+    StringRef Name = getSymbolName(Globals[I]);
+    Records[I].Name = Name.data();
+    Records[I].NameLen = Name.size();
+    Records[I].SymOffset = SymOffset;
+    SymOffset += Globals[I].length();
   }
 
-  finalizeBuckets(RecordZeroOffset, std::move(Globals));
+  GSH->finalizeBuckets(RecordZeroOffset, Records);
 }
 
-void GSIHashStreamBuilder::finalizeBuckets(uint32_t RecordZeroOffset,
-                                           std::vector<BulkPublic> &&Globals) {
-  // Hash every name in parallel. The Segment field is no longer needed, so
-  // store the BucketIdx in a union.
-  parallelForEachN(0, Globals.size(), [&](size_t I) {
-    Globals[I].U.BucketIdx = hashStringV1(Globals[I].Name) % IPHR_HASH;
+void GSIHashStreamBuilder::finalizeBuckets(
+    uint32_t RecordZeroOffset, MutableArrayRef<BulkPublic> Records) {
+  // Hash every name in parallel.
+  parallelForEachN(0, Records.size(), [&](size_t I) {
+    Records[I].setBucketIdx(hashStringV1(Records[I].Name) % IPHR_HASH);
   });
 
-  // Parallel sort by bucket index, then name within the buckets. Within the
-  // buckets, sort each bucket by memcmp of the symbol's name.  It's important
-  // that we use the same sorting algorithm as is used by the reference
-  // implementation to ensure that the search for a record within a bucket can
-  // properly early-out when it detects the record won't be found.  The
-  // algorithm used here corredsponds to the function
+  // Count up the size of each bucket. Then, use an exclusive prefix sum to
+  // calculate the bucket start offsets. This is C++17 std::exclusive_scan, but
+  // we can't use it yet.
+  uint32_t BucketStarts[IPHR_HASH] = {0};
+  for (const BulkPublic &P : Records)
+    ++BucketStarts[P.BucketIdx];
+  uint32_t Sum = 0;
+  for (uint32_t &B : BucketStarts) {
+    uint32_t Size = B;
+    B = Sum;
+    Sum += Size;
+  }
+
+  // Place globals into the hash table in bucket order. When placing a global,
+  // update the bucket start. Every hash table slot should be filled. Always use
+  // a refcount of one for now.
+  HashRecords.resize(Records.size());
+  uint32_t BucketCursors[IPHR_HASH];
+  memcpy(BucketCursors, BucketStarts, sizeof(BucketCursors));
+  for (int I = 0, E = Records.size(); I < E; ++I) {
+    uint32_t HashIdx = BucketCursors[Records[I].BucketIdx]++;
+    HashRecords[HashIdx].Off = I;
+    HashRecords[HashIdx].CRef = 1;
+  }
+
+  // Within the buckets, sort each bucket by memcmp of the symbol's name.  It's
+  // important that we use the same sorting algorithm as is used by the
+  // reference implementation to ensure that the search for a record within a
+  // bucket can properly early-out when it detects the record won't be found.
+  // The algorithm used here corresponds to the function
   // caseInsensitiveComparePchPchCchCch in the reference implementation.
-  auto BucketCmp = [](const BulkPublic &L, const BulkPublic &R) {
-    if (L.U.BucketIdx != R.U.BucketIdx)
-      return L.U.BucketIdx < R.U.BucketIdx;
-    int Cmp = gsiRecordCmp(L.getName(), R.getName());
-    if (Cmp != 0)
-      return Cmp < 0;
-    // This comparison is necessary to make the sorting stable in the presence
-    // of two static globals with the same name. The easiest way to observe
-    // this is with S_LDATA32 records.
-    return L.SymOffset < R.SymOffset;
-  };
-  parallelSort(Globals, BucketCmp);
-
-  // Zero out the bucket index bitmap.
-  for (ulittle32_t &Word : HashBitmap)
-    Word = 0;
-
-  // Compute the three tables: the hash records in bucket and chain order, the
-  // bucket presence bitmap, and the bucket chain start offsets.
-  HashRecords.reserve(Globals.size());
-  uint32_t LastBucketIdx = ~0U;
-  for (const BulkPublic &Global : Globals) {
-    // If this is a new bucket, add it to the bitmap and the start offset map.
-    uint32_t BucketIdx = Global.U.BucketIdx;
-    if (LastBucketIdx != BucketIdx) {
-      HashBitmap[BucketIdx / 32] |= 1U << (BucketIdx % 32);
+  parallelForEachN(0, IPHR_HASH, [&](size_t I) {
+    auto B = HashRecords.begin() + BucketStarts[I];
+    auto E = HashRecords.begin() + BucketCursors[I];
+    if (B == E)
+      return;
+    auto BucketCmp = [Records](const PSHashRecord &LHash,
+                               const PSHashRecord &RHash) {
+      const BulkPublic &L = Records[uint32_t(LHash.Off)];
+      const BulkPublic &R = Records[uint32_t(RHash.Off)];
+      assert(L.BucketIdx == R.BucketIdx);
+      int Cmp = gsiRecordCmp(L.getName(), R.getName());
+      if (Cmp != 0)
+        return Cmp < 0;
+      // This comparison is necessary to make the sorting stable in the presence
+      // of two static globals with the same name. The easiest way to observe
+      // this is with S_LDATA32 records.
+      return L.SymOffset < R.SymOffset;
+    };
+    llvm::sort(B, E, BucketCmp);
+
+    // After we are done sorting, replace the global indices with the stream
+    // offsets of each global. Add one when writing symbol offsets to disk.
+    // See GSI1::fixSymRecs.
+    for (PSHashRecord &HRec : make_range(B, E))
+      HRec.Off = Records[uint32_t(HRec.Off)].SymOffset + 1;
+  });
+
+  // For each non-empty bucket, push the bucket start offset into HashBuckets
+  // and set a bit in the hash bitmap.
+  for (uint32_t I = 0; I < HashBitmap.size(); ++I) {
+    uint32_t Word = 0;
+    for (uint32_t J = 0; J < 32; ++J) {
+      // Skip empty buckets.
+      uint32_t BucketIdx = I * 32 + J;
+      if (BucketIdx >= IPHR_HASH ||
+          BucketStarts[BucketIdx] == BucketCursors[BucketIdx])
+        continue;
+      Word |= (1U << J);
 
       // Calculate what the offset of the first hash record in the chain would
       // be if it were inflated to contain 32-bit pointers. On a 32-bit system,
       // each record would be 12 bytes. See HROffsetCalc in gsi.h.
       const int SizeOfHROffsetCalc = 12;
       ulittle32_t ChainStartOff =
-          ulittle32_t(HashRecords.size() * SizeOfHROffsetCalc);
+          ulittle32_t(BucketStarts[BucketIdx] * SizeOfHROffsetCalc);
       HashBuckets.push_back(ChainStartOff);
-      LastBucketIdx = BucketIdx;
     }
-
-    // Create the hash record. Add one when writing symbol offsets to disk.
-    // See GSI1::fixSymRecs. Always use a refcount of 1 for now.
-    PSHashRecord HRec;
-    HRec.Off = Global.SymOffset + 1;
-    HRec.CRef = 1;
-    HashRecords.push_back(HRec);
+    HashBitmap[I] = Word;
   }
 }
 
@@ -269,7 +292,7 @@ uint32_t GSIStreamBuilder::calculatePublicsHashStreamSize() const {
   uint32_t Size = 0;
   Size += sizeof(PublicsStreamHeader);
   Size += PSH->calculateSerializedLength();
-  Size += PubAddrMap.size() * sizeof(uint32_t); // AddrMap
+  Size += Publics.size() * sizeof(uint32_t); // AddrMap
   // FIXME: Add thunk map and section offsets for incremental linking.
 
   return Size;
@@ -281,9 +304,8 @@ uint32_t GSIStreamBuilder::calculateGlobalsHashStreamSize() const {
 
 Error GSIStreamBuilder::finalizeMsfLayout() {
   // First we write public symbol records, then we write global symbol records.
-  uint32_t PublicsSize = PSH->calculateRecordByteSize();
-  uint32_t GlobalsSize = GSH->calculateRecordByteSize();
-  GSH->finalizeBuckets(PublicsSize);
+  finalizePublicBuckets();
+  finalizeGlobalBuckets(PSH->RecordByteSize);
 
   Expected<uint32_t> Idx = Msf.addStream(calculateGlobalsHashStreamSize());
   if (!Idx)
@@ -295,7 +317,7 @@ Error GSIStreamBuilder::finalizeMsfLayout() {
     return Idx.takeError();
   PublicsStreamIndex = *Idx;
 
-  uint32_t RecordBytes = PublicsSize + GlobalsSize;
+  uint32_t RecordBytes = PSH->RecordByteSize + GSH->RecordByteSize;
 
   Idx = Msf.addStream(RecordBytes);
   if (!Idx)
@@ -304,72 +326,68 @@ Error GSIStreamBuilder::finalizeMsfLayout() {
   return Error::success();
 }
 
-void GSIStreamBuilder::addPublicSymbols(std::vector<BulkPublic> &&Publics) {
+void GSIStreamBuilder::addPublicSymbols(std::vector<BulkPublic> &&PublicsIn) {
+  assert(Publics.empty() && PSH->RecordByteSize == 0 &&
+         "publics can only be added once");
+  Publics = std::move(PublicsIn);
+
   // Sort the symbols by name. PDBs contain lots of symbols, so use parallelism.
   parallelSort(Publics, [](const BulkPublic &L, const BulkPublic &R) {
     return L.getName() < R.getName();
   });
 
-  // Assign offsets and allocate one contiguous block of memory for all public
-  // symbols.
+  // Assign offsets and calculate the length of the public symbol records.
   uint32_t SymOffset = 0;
   for (BulkPublic &Pub : Publics) {
     Pub.SymOffset = SymOffset;
     SymOffset += sizeOfPublic(Pub);
   }
-  uint8_t *Mem =
-      reinterpret_cast<uint8_t *>(Msf.getAllocator().Allocate(SymOffset, 4));
-
-  // Instead of storing individual CVSymbol records, store them as one giant
-  // buffer.
-  // FIXME: This is kind of a hack. This makes Records.size() wrong, and we have
-  // to account for that elsewhere.
-  PSH->Records.push_back(CVSymbol(makeArrayRef(Mem, SymOffset)));
-
-  // Serialize them in parallel.
-  parallelForEachN(0, Publics.size(), [&](size_t I) {
-    const BulkPublic &Pub = Publics[I];
-    serializePublic(Mem + Pub.SymOffset, Pub);
-  });
-
-  // Re-sort the publics by address so we can build the address map. We no
-  // longer need the original ordering.
-  auto AddrCmp = [](const BulkPublic &L, const BulkPublic &R) {
-    if (L.U.Segment != R.U.Segment)
-      return L.U.Segment < R.U.Segment;
-    if (L.Offset != R.Offset)
-      return L.Offset < R.Offset;
-    // parallelSort is unstable, so we have to do name comparison to ensure
-    // that two names for the same location come out in a determinstic order.
-    return L.getName() < R.getName();
-  };
-  parallelSort(Publics, AddrCmp);
-
-  // Fill in the symbol offsets in the appropriate order.
-  PubAddrMap.reserve(Publics.size());
-  for (const BulkPublic &Pub : Publics)
-    PubAddrMap.push_back(ulittle32_t(Pub.SymOffset));
 
-  // Finalize public symbol buckets immediately after they have been added.
-  // They should all be warm in the cache at this point, so go ahead and do it
-  // now.
-  PSH->finalizeBuckets(0, std::move(Publics));
+  // Remember the length of the public stream records.
+  PSH->RecordByteSize = SymOffset;
 }
 
 void GSIStreamBuilder::addGlobalSymbol(const ProcRefSym &Sym) {
-  GSH->addSymbol(Sym, Msf);
+  serializeAndAddGlobal(Sym);
 }
 
 void GSIStreamBuilder::addGlobalSymbol(const DataSym &Sym) {
-  GSH->addSymbol(Sym, Msf);
+  serializeAndAddGlobal(Sym);
 }
 
 void GSIStreamBuilder::addGlobalSymbol(const ConstantSym &Sym) {
-  GSH->addSymbol(Sym, Msf);
+  serializeAndAddGlobal(Sym);
+}
+
+template <typename T>
+void GSIStreamBuilder::serializeAndAddGlobal(const T &Symbol) {
+  T Copy(Symbol);
+  addGlobalSymbol(SymbolSerializer::writeOneSymbol(Copy, Msf.getAllocator(),
+                                                   CodeViewContainer::Pdb));
+}
+
+void GSIStreamBuilder::addGlobalSymbol(const codeview::CVSymbol &Symbol) {
+  // Ignore duplicate typedefs and constants.
+  if (Symbol.kind() == S_UDT || Symbol.kind() == S_CONSTANT) {
+    auto Iter = GlobalsSeen.insert(Symbol);
+    if (!Iter.second)
+      return;
+  }
+  GSH->RecordByteSize += Symbol.length();
+  Globals.push_back(Symbol);
 }
 
-void GSIStreamBuilder::addGlobalSymbol(const codeview::CVSymbol &Sym) {
-  GSH->addSymbol(Sym);
+// Serialize each public and write it.
+static Error writePublics(BinaryStreamWriter &Writer,
+                          ArrayRef<BulkPublic> Publics) {
+  std::vector<uint8_t> Storage;
+  for (const BulkPublic &Pub : Publics) {
+    Storage.resize(sizeOfPublic(Pub));
+    serializePublic(Storage.data(), Pub);
+    if (Error E = Writer.writeBytes(Storage))
+      return E;
+  }
+  return Error::success();
 }
 
 static Error writeRecords(BinaryStreamWriter &Writer,
@@ -387,14 +405,42 @@ Error GSIStreamBuilder::commitSymbolRecordStream(
   // Write public symbol records first, followed by global symbol records.  This
   // must match the order that we assume in finalizeMsfLayout when computing
   // PSHZero and GSHZero.
-  if (auto EC = writeRecords(Writer, PSH->Records))
+  if (auto EC = writePublics(Writer, Publics))
     return EC;
-  if (auto EC = writeRecords(Writer, GSH->Records))
+  if (auto EC = writeRecords(Writer, Globals))
     return EC;
 
   return Error::success();
 }
 
+static std::vector<support::ulittle32_t>
+computeAddrMap(ArrayRef<BulkPublic> Publics) {
+  // Build a parallel vector of indices into the Publics vector, and sort it by
+  // address.
+  std::vector<ulittle32_t> PubAddrMap;
+  PubAddrMap.reserve(Publics.size());
+  for (int I = 0, E = Publics.size(); I < E; ++I)
+    PubAddrMap.push_back(ulittle32_t(I));
+
+  auto AddrCmp = [Publics](const ulittle32_t &LIdx, const ulittle32_t &RIdx) {
+    const BulkPublic &L = Publics[LIdx];
+    const BulkPublic &R = Publics[RIdx];
+    if (L.Segment != R.Segment)
+      return L.Segment < R.Segment;
+    if (L.Offset != R.Offset)
+      return L.Offset < R.Offset;
+    // parallelSort is unstable, so we have to do name comparison to ensure
+    // that two names for the same location come out in a deterministic order.
+    return L.getName() < R.getName();
+  };
+  parallelSort(PubAddrMap, AddrCmp);
+
+  // Rewrite the public symbol indices into symbol offsets.
+  for (ulittle32_t &Entry : PubAddrMap)
+    Entry = Publics[Entry].SymOffset;
+  return PubAddrMap;
+}
+
 Error GSIStreamBuilder::commitPublicsHashStream(
     WritableBinaryStreamRef Stream) {
   BinaryStreamWriter Writer(Stream);
@@ -402,7 +448,7 @@ Error GSIStreamBuilder::commitPublicsHashStream(
 
   // FIXME: Fill these in. They are for incremental linking.
   Header.SymHash = PSH->calculateSerializedLength();
-  Header.AddrMap = PubAddrMap.size() * 4;
+  Header.AddrMap = Publics.size() * 4;
   Header.NumThunks = 0;
   Header.SizeOfThunk = 0;
   Header.ISectThunkTable = 0;
@@ -415,6 +461,8 @@ Error GSIStreamBuilder::commitPublicsHashStream(
   if (auto EC = PSH->commit(Writer))
     return EC;
 
+  std::vector<support::ulittle32_t> PubAddrMap = computeAddrMap(Publics);
+  assert(PubAddrMap.size() == Publics.size());
   if (auto EC = Writer.writeArray(makeArrayRef(PubAddrMap)))
     return EC;
 
diff --git a/llvm/lib/DebugInfo/PDB/Native/NativeLineNumber.cpp b/llvm/lib/DebugInfo/PDB/Native/NativeLineNumber.cpp
index f493c18079424..2535e09baf625 100644
--- a/llvm/lib/DebugInfo/PDB/Native/NativeLineNumber.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/NativeLineNumber.cpp
@@ -13,10 +13,11 @@ using namespace llvm::pdb;
 
 NativeLineNumber::NativeLineNumber(const NativeSession &Session,
                                    const codeview::LineInfo Line,
-                                   uint32_t Section, uint32_t Offset,
-                                   uint32_t Length, uint32_t SrcFileId)
-    : Session(Session), Line(Line), Section(Section), Offset(Offset),
-      Length(Length), SrcFileId(SrcFileId) {}
+                                   uint32_t ColumnNumber, uint32_t Section,
+                                   uint32_t Offset, uint32_t Length,
+                                   uint32_t SrcFileId)
+    : Session(Session), Line(Line), ColumnNumber(ColumnNumber),
+      Section(Section), Offset(Offset), Length(Length), SrcFileId(SrcFileId) {}
 
 uint32_t NativeLineNumber::getLineNumber() const { return Line.getStartLine(); }
 
@@ -24,7 +25,7 @@ uint32_t NativeLineNumber::getLineNumberEnd() const {
   return Line.getEndLine();
 }
 
-uint32_t NativeLineNumber::getColumnNumber() const { return 0; }
+uint32_t NativeLineNumber::getColumnNumber() const { return ColumnNumber; }
 
 uint32_t NativeLineNumber::getColumnNumberEnd() const { return 0; }
 
diff --git a/llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp b/llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp
index 83cf77aae862a..9f15907b519e8 100644
--- a/llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp
@@ -460,18 +460,33 @@ SymbolCache::findLineTable(uint16_t Modi) const {
         continue;
 
       std::vector<LineTableEntry> Entries;
+
+      // If there are column numbers, then they should be in a parallel stream
+      // to the line numbers.
+      auto ColIt = Group.Columns.begin();
+      auto ColsEnd = Group.Columns.end();
+
       for (const LineNumberEntry &LN : Group.LineNumbers) {
-        LineInfo Line(LN.Flags);
         uint64_t VA =
             Session.getVAFromSectOffset(RelocSegment, RelocOffset + LN.Offset);
-        Entries.push_back({VA, Line, Group.NameIndex, false});
+        LineInfo Line(LN.Flags);
+        uint32_t ColNum = 0;
+
+        if (Lines.hasColumnInfo() && ColIt != ColsEnd) {
+          ColNum = ColIt->StartColumn;
+          ++ColIt;
+        }
+        Entries.push_back({VA, Line, ColNum, Group.NameIndex, false});
       }
 
       // Add a terminal entry line to mark the end of this subsection.
-      LineInfo LastLine(Group.LineNumbers.back().Flags);
       uint64_t VA = Session.getVAFromSectOffset(
           RelocSegment, RelocOffset + Lines.header()->CodeSize);
-      Entries.push_back({VA, LastLine, Group.NameIndex, true});
+      LineInfo LastLine(Group.LineNumbers.back().Flags);
+      uint32_t ColNum =
+          (Lines.hasColumnInfo()) ? Group.Columns.back().StartColumn : 0;
+      Entries.push_back({VA, LastLine, ColNum, Group.NameIndex, true});
+
       EntryList.push_back(Entries);
     }
   }
@@ -571,8 +586,8 @@ SymbolCache::findLineNumbersByVA(uint64_t VA, uint32_t Length) const {
     auto ChecksumIter =
         ExpectedChecksums->getArray().at(LineIter->FileNameIndex);
     uint32_t SrcFileId = getOrCreateSourceFile(*ChecksumIter);
-    NativeLineNumber LineNum(Session, LineIter->Line, LineSect, LineOff,
-                             LineLength, SrcFileId);
+    NativeLineNumber LineNum(Session, LineIter->Line, LineIter->ColumnNumber,
+                             LineSect, LineOff, LineLength, SrcFileId);
     LineNumbers.push_back(LineNum);
     ++LineIter;
   }
diff --git a/llvm/lib/ExecutionEngine/ExecutionEngine.cpp b/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
index 6e8f88e21c8bb..d8bd671c66619 100644
--- a/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
+++ b/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
@@ -9,6 +9,8 @@
 // This file defines the common interface used by the various execution engine
 // subclasses.
 //
+// FIXME: This file needs to be updated to support scalable vectors
+//
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ExecutionEngine/ExecutionEngine.h"
@@ -108,7 +110,7 @@ class GVMemoryBlock final : public CallbackVH {
     Type *ElTy = GV->getValueType();
     size_t GVSize = (size_t)TD.getTypeAllocSize(ElTy);
     void *RawMemory = ::operator new(
-        alignTo(sizeof(GVMemoryBlock), TD.getPreferredAlignment(GV)) + GVSize);
+        alignTo(sizeof(GVMemoryBlock), TD.getPreferredAlign(GV)) + GVSize);
     new(RawMemory) GVMemoryBlock(GV);
     return static_cast<char*>(RawMemory) + sizeof(GVMemoryBlock);
   }
@@ -624,10 +626,12 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
         }
       }
       break;
-      case Type::FixedVectorTyID:
       case Type::ScalableVectorTyID:
+        report_fatal_error(
+            "Scalable vector support not yet implemented in ExecutionEngine");
+      case Type::FixedVectorTyID:
         // if the whole vector is 'undef' just reserve memory for the value.
-        auto *VTy = cast<VectorType>(C->getType());
+        auto *VTy = cast<FixedVectorType>(C->getType());
         Type *ElemTy = VTy->getElementType();
         unsigned int elemNum = VTy->getNumElements();
         Result.AggregateVal.resize(elemNum);
@@ -915,8 +919,10 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
     else
       llvm_unreachable("Unknown constant pointer type!");
     break;
-  case Type::FixedVectorTyID:
-  case Type::ScalableVectorTyID: {
+  case Type::ScalableVectorTyID:
+    report_fatal_error(
+        "Scalable vector support not yet implemented in ExecutionEngine");
+  case Type::FixedVectorTyID: {
     unsigned elemNum;
     Type* ElemTy;
     const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(C);
@@ -927,9 +933,9 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
         elemNum = CDV->getNumElements();
         ElemTy = CDV->getElementType();
     } else if (CV || CAZ) {
-        auto* VTy = cast<VectorType>(C->getType());
-        elemNum = VTy->getNumElements();
-        ElemTy = VTy->getElementType();
+      auto *VTy = cast<FixedVectorType>(C->getType());
+      elemNum = VTy->getNumElements();
+      ElemTy = VTy->getElementType();
     } else {
         llvm_unreachable("Unknown constant vector type!");
     }
@@ -1098,9 +1104,11 @@ void ExecutionEngine::LoadValueFromMemory(GenericValue &Result,
     Result.IntVal = APInt(80, y);
     break;
   }
-  case Type::FixedVectorTyID:
-  case Type::ScalableVectorTyID: {
-    auto *VT = cast<VectorType>(Ty);
+  case Type::ScalableVectorTyID:
+    report_fatal_error(
+        "Scalable vector support not yet implemented in ExecutionEngine");
+  case Type::FixedVectorTyID: {
+    auto *VT = cast<FixedVectorType>(Ty);
     Type *ElemT = VT->getElementType();
     const unsigned numElems = VT->getNumElements();
     if (ElemT->isFloatTy()) {
diff --git a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
index 2812159b0076c..ff66955082d85 100644
--- a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
@@ -35,6 +35,11 @@ Expected<JITTargetAddress> LazyCallThroughManager::getCallThroughTrampoline(
   return *Trampoline;
 }
 
+JITTargetAddress LazyCallThroughManager::reportCallThroughError(Error Err) {
+  ES.reportError(std::move(Err));
+  return ErrorHandlerAddr;
+}
+
 Expected<LazyCallThroughManager::ReexportsEntry>
 LazyCallThroughManager::findReexport(JITTargetAddress TrampolineAddr) {
   std::lock_guard<std::mutex> Lock(LCTMMutex);
@@ -46,19 +51,6 @@ LazyCallThroughManager::findReexport(JITTargetAddress TrampolineAddr) {
   return I->second;
 }
 
-Expected<JITTargetAddress>
-LazyCallThroughManager::resolveSymbol(const ReexportsEntry &RE) {
-  auto LookupResult =
-      ES.lookup(makeJITDylibSearchOrder(RE.SourceJD,
-                                        JITDylibLookupFlags::MatchAllSymbols),
-                RE.SymbolName, SymbolState::Ready);
-
-  if (!LookupResult)
-    return LookupResult.takeError();
-
-  return LookupResult->getAddress();
-}
-
 Error LazyCallThroughManager::notifyResolved(JITTargetAddress TrampolineAddr,
                                              JITTargetAddress ResolvedAddr) {
   NotifyResolvedFunction NotifyResolved;
@@ -74,6 +66,37 @@ Error LazyCallThroughManager::notifyResolved(JITTargetAddress TrampolineAddr,
   return NotifyResolved ? NotifyResolved(ResolvedAddr) : Error::success();
 }
 
+void LazyCallThroughManager::resolveTrampolineLandingAddress(
+    JITTargetAddress TrampolineAddr,
+    NotifyLandingResolvedFunction NotifyLandingResolved) {
+
+  auto Entry = findReexport(TrampolineAddr);
+  if (!Entry)
+    return NotifyLandingResolved(reportCallThroughError(Entry.takeError()));
+
+  ES.lookup(
+      LookupKind::Static,
+      makeJITDylibSearchOrder(Entry->SourceJD,
+                              JITDylibLookupFlags::MatchAllSymbols),
+      SymbolLookupSet({Entry->SymbolName}), SymbolState::Ready,
+      [this, TrampolineAddr, SymbolName = Entry->SymbolName,
+       NotifyLandingResolved = std::move(NotifyLandingResolved)](
+          Expected<SymbolMap> Result) mutable {
+        if (Result) {
+          assert(Result->size() == 1 && "Unexpected result size");
+          assert(Result->count(SymbolName) && "Unexpected result value");
+          JITTargetAddress LandingAddr = (*Result)[SymbolName].getAddress();
+
+          if (auto Err = notifyResolved(TrampolineAddr, LandingAddr))
+            NotifyLandingResolved(reportCallThroughError(std::move(Err)));
+          else
+            NotifyLandingResolved(LandingAddr);
+        } else
+          NotifyLandingResolved(reportCallThroughError(Result.takeError()));
+      },
+      NoDependenciesToRegister);
+}
+
 Expected<std::unique_ptr<LazyCallThroughManager>>
 createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES,
                                   JITTargetAddress ErrorHandlerAddr) {
diff --git a/llvm/lib/Frontend/OpenMP/CMakeLists.txt b/llvm/lib/Frontend/OpenMP/CMakeLists.txt
index 7c332c106c190..068283fd82e07 100644
--- a/llvm/lib/Frontend/OpenMP/CMakeLists.txt
+++ b/llvm/lib/Frontend/OpenMP/CMakeLists.txt
@@ -1,5 +1,9 @@
+set(LLVM_TARGET_DEFINITIONS ${LLVM_MAIN_INCLUDE_DIR}/llvm/Frontend/OpenMP/OMP.td)
+tablegen(LLVM OMP.cpp --gen-directive-impl)
+add_public_tablegen_target(omp_cpp)
+
 add_llvm_component_library(LLVMFrontendOpenMP
-  OMPConstants.cpp
+  OMP.cpp # Generated by tablegen above
   OMPContext.cpp
   OMPIRBuilder.cpp
 
@@ -9,4 +13,6 @@ add_llvm_component_library(LLVMFrontendOpenMP
 
   DEPENDS
   intrinsics_gen
+  omp_gen
+  omp_cpp
   )
diff --git a/llvm/lib/Frontend/OpenMP/OMPConstants.cpp b/llvm/lib/Frontend/OpenMP/OMPConstants.cpp
deleted file mode 100644
index ed19ae38c6983..0000000000000
--- a/llvm/lib/Frontend/OpenMP/OMPConstants.cpp
+++ /dev/null
@@ -1,121 +0,0 @@
-//===- OMPConstants.cpp - Helpers related to OpenMP code generation ---===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Frontend/OpenMP/OMPConstants.h"
-
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/StringSwitch.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Type.h"
-
-using namespace llvm;
-using namespace omp;
-using namespace types;
-
-Directive llvm::omp::getOpenMPDirectiveKind(StringRef Str) {
-  return llvm::StringSwitch<Directive>(Str)
-#define OMP_DIRECTIVE(Enum, Str) .Case(Str, Enum)
-#include "llvm/Frontend/OpenMP/OMPKinds.def"
-      .Default(OMPD_unknown);
-}
-
-StringRef llvm::omp::getOpenMPDirectiveName(Directive Kind) {
-  switch (Kind) {
-#define OMP_DIRECTIVE(Enum, Str)                                               \
-  case Enum:                                                                   \
-    return Str;
-#include "llvm/Frontend/OpenMP/OMPKinds.def"
-  }
-  llvm_unreachable("Invalid OpenMP directive kind");
-}
-
-Clause llvm::omp::getOpenMPClauseKind(StringRef Str) {
-  return llvm::StringSwitch<Clause>(Str)
-#define OMP_CLAUSE(Enum, Str, Implicit)                                        \
-  .Case(Str, Implicit ? OMPC_unknown : Enum)
-#include "llvm/Frontend/OpenMP/OMPKinds.def"
-      .Default(OMPC_unknown);
-}
-
-StringRef llvm::omp::getOpenMPClauseName(Clause C) {
-  switch (C) {
-#define OMP_CLAUSE(Enum, Str, ...)                                             \
-  case Enum:                                                                   \
-    return Str;
-#include "llvm/Frontend/OpenMP/OMPKinds.def"
-  }
-  llvm_unreachable("Invalid OpenMP clause kind");
-}
-
-bool llvm::omp::isAllowedClauseForDirective(Directive D, Clause C,
-                                            unsigned Version) {
-  assert(unsigned(D) <= unsigned(OMPD_unknown));
-  assert(unsigned(C) <= unsigned(OMPC_unknown));
-#define OMP_DIRECTIVE_CLAUSE(Dir, MinVersion, MaxVersion, Cl)                  \
-  if (D == Dir && C == Cl && MinVersion <= Version && MaxVersion >= Version)   \
-    return true;
-#include "llvm/Frontend/OpenMP/OMPKinds.def"
-  return false;
-}
-
-/// Declarations for LLVM-IR types (simple, array, function and structure) are
-/// generated below. Their names are defined and used in OpenMPKinds.def. Here
-/// we provide the declarations, the initializeTypes function will provide the
-/// values.
-///
-///{
-#define OMP_TYPE(VarName, InitValue) Type *llvm::omp::types::VarName = nullptr;
-#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize)                             \
-  ArrayType *llvm::omp::types::VarName##Ty = nullptr;                          \
-  PointerType *llvm::omp::types::VarName##PtrTy = nullptr;
-#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...)                  \
-  FunctionType *llvm::omp::types::VarName = nullptr;                           \
-  PointerType *llvm::omp::types::VarName##Ptr = nullptr;
-#define OMP_STRUCT_TYPE(VarName, StrName, ...)                                 \
-  StructType *llvm::omp::types::VarName = nullptr;                             \
-  PointerType *llvm::omp::types::VarName##Ptr = nullptr;
-#include "llvm/Frontend/OpenMP/OMPKinds.def"
-
-///}
-
-void llvm::omp::types::initializeTypes(Module &M) {
-  if (Void)
-    return;
-
-  LLVMContext &Ctx = M.getContext();
-  // Create all simple and struct types exposed by the runtime and remember
-  // the llvm::PointerTypes of them for easy access later.
-  StructType *T;
-#define OMP_TYPE(VarName, InitValue) VarName = InitValue;
-#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize)                             \
-  VarName##Ty = ArrayType::get(ElemTy, ArraySize);                             \
-  VarName##PtrTy = PointerType::getUnqual(VarName##Ty);
-#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...)                  \
-  VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg);            \
-  VarName##Ptr = PointerType::getUnqual(VarName);
-#define OMP_STRUCT_TYPE(VarName, StructName, ...)                              \
-  T = M.getTypeByName(StructName);                                             \
-  if (!T)                                                                      \
-    T = StructType::create(Ctx, {__VA_ARGS__}, StructName);                    \
-  VarName = T;                                                                 \
-  VarName##Ptr = PointerType::getUnqual(T);
-#include "llvm/Frontend/OpenMP/OMPKinds.def"
-}
-
-void llvm::omp::types::uninitializeTypes() {
-#define OMP_TYPE(VarName, InitValue) VarName = nullptr;
-#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...)                  \
-  VarName = nullptr;                                                           \
-  VarName##Ptr = nullptr;
-#define OMP_STRUCT_TYPE(VarName, StrName, ...)                                 \
-  VarName = nullptr;                                                           \
-  VarName##Ptr = nullptr;
-#include "llvm/Frontend/OpenMP/OMPKinds.def"
-}
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index c9011d3422d7c..b7212edab6ab2 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -31,7 +31,6 @@
 
 using namespace llvm;
 using namespace omp;
-using namespace types;
 
 static cl::opt<bool>
     OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
@@ -611,9 +610,6 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::CreateParallel(
          "Unexpected finalization stack state!");
 
   Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator();
-  assert(PRegPreFiniTI->getNumSuccessors() == 1 &&
-         PRegPreFiniTI->getSuccessor(0) == PRegExitBB &&
-         "Unexpected CFG structure!");
 
   InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator());
   FiniCB(PreFiniIP);
@@ -948,6 +944,105 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveExit(
                                   ExitCall->getIterator());
 }
 
+OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::CreateCopyinClauseBlocks(
+    InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr,
+    llvm::IntegerType *IntPtrTy, bool BranchtoEnd) {
+  if (!IP.isSet())
+    return IP;
+
+  IRBuilder<>::InsertPointGuard IPG(Builder);
+
+  // creates the following CFG structure
+  //	   OMP_Entry : (MasterAddr != PrivateAddr)?
+  //       F     T
+  //       |      \
+  //       |     copin.not.master
+  //       |      /
+  //       v     /
+  //   copyin.not.master.end
+  //		     |
+  //         v
+  //   OMP.Entry.Next
+
+  BasicBlock *OMP_Entry = IP.getBlock();
+  Function *CurFn = OMP_Entry->getParent();
+  BasicBlock *CopyBegin =
+      BasicBlock::Create(M.getContext(), "copyin.not.master", CurFn);
+  BasicBlock *CopyEnd = nullptr;
+
+  // If entry block is terminated, split to preserve the branch to following
+  // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is.
+  if (isa_and_nonnull<BranchInst>(OMP_Entry->getTerminator())) {
+    CopyEnd = OMP_Entry->splitBasicBlock(OMP_Entry->getTerminator(),
+                                         "copyin.not.master.end");
+    OMP_Entry->getTerminator()->eraseFromParent();
+  } else {
+    CopyEnd =
+        BasicBlock::Create(M.getContext(), "copyin.not.master.end", CurFn);
+  }
+
+  Builder.SetInsertPoint(OMP_Entry);
+  Value *MasterPtr = Builder.CreatePtrToInt(MasterAddr, IntPtrTy);
+  Value *PrivatePtr = Builder.CreatePtrToInt(PrivateAddr, IntPtrTy);
+  Value *cmp = Builder.CreateICmpNE(MasterPtr, PrivatePtr);
+  Builder.CreateCondBr(cmp, CopyBegin, CopyEnd);
+
+  Builder.SetInsertPoint(CopyBegin);
+  if (BranchtoEnd)
+    Builder.SetInsertPoint(Builder.CreateBr(CopyEnd));
+
+  return Builder.saveIP();
+}
+
+CallInst *OpenMPIRBuilder::CreateOMPAlloc(const LocationDescription &Loc,
+                                          Value *Size, Value *Allocator,
+                                          std::string Name) {
+  IRBuilder<>::InsertPointGuard IPG(Builder);
+  Builder.restoreIP(Loc.IP);
+
+  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
+  Value *Ident = getOrCreateIdent(SrcLocStr);
+  Value *ThreadId = getOrCreateThreadID(Ident);
+  Value *Args[] = {ThreadId, Size, Allocator};
+
+  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_alloc);
+
+  return Builder.CreateCall(Fn, Args, Name);
+}
+
+CallInst *OpenMPIRBuilder::CreateOMPFree(const LocationDescription &Loc,
+                                         Value *Addr, Value *Allocator,
+                                         std::string Name) {
+  IRBuilder<>::InsertPointGuard IPG(Builder);
+  Builder.restoreIP(Loc.IP);
+
+  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
+  Value *Ident = getOrCreateIdent(SrcLocStr);
+  Value *ThreadId = getOrCreateThreadID(Ident);
+  Value *Args[] = {ThreadId, Addr, Allocator};
+  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_free);
+  return Builder.CreateCall(Fn, Args, Name);
+}
+
+CallInst *OpenMPIRBuilder::CreateCachedThreadPrivate(
+    const LocationDescription &Loc, llvm::Value *Pointer,
+    llvm::ConstantInt *Size, const llvm::Twine &Name) {
+  IRBuilder<>::InsertPointGuard IPG(Builder);
+  Builder.restoreIP(Loc.IP);
+
+  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
+  Value *Ident = getOrCreateIdent(SrcLocStr);
+  Value *ThreadId = getOrCreateThreadID(Ident);
+  Constant *ThreadPrivateCache =
+      getOrCreateOMPInternalVariable(Int8PtrPtr, Name);
+  llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache};
+
+  Function *Fn =
+  		getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_threadprivate_cached);
+
+  return Builder.CreateCall(Fn, Args);
+}
+
 std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts,
                                                    StringRef FirstSeparator,
                                                    StringRef Separator) {
@@ -996,3 +1091,24 @@ Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) {
   std::string Name = getNameWithSeparators({Prefix, "var"}, ".", ".");
   return getOrCreateOMPInternalVariable(KmpCriticalNameTy, Name);
 }
+
+// Create all simple and struct types exposed by the runtime and remember
+// the llvm::PointerTypes of them for easy access later.
+void OpenMPIRBuilder::initializeTypes(Module &M) {
+  LLVMContext &Ctx = M.getContext();
+  StructType *T;
+#define OMP_TYPE(VarName, InitValue) VarName = InitValue;
+#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize)                             \
+  VarName##Ty = ArrayType::get(ElemTy, ArraySize);                             \
+  VarName##PtrTy = PointerType::getUnqual(VarName##Ty);
+#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...)                  \
+  VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg);            \
+  VarName##Ptr = PointerType::getUnqual(VarName);
+#define OMP_STRUCT_TYPE(VarName, StructName, ...)                              \
+  T = M.getTypeByName(StructName);                                             \
+  if (!T)                                                                      \
+    T = StructType::create(Ctx, {__VA_ARGS__}, StructName);                    \
+  VarName = T;                                                                 \
+  VarName##Ptr = PointerType::getUnqual(T);
+#include "llvm/Frontend/OpenMP/OMPKinds.def"
+}
diff --git a/llvm/lib/FuzzMutate/FuzzerCLI.cpp b/llvm/lib/FuzzMutate/FuzzerCLI.cpp
index 32eba275da7b9..be0d5bfcab46d 100644
--- a/llvm/lib/FuzzMutate/FuzzerCLI.cpp
+++ b/llvm/lib/FuzzMutate/FuzzerCLI.cpp
@@ -111,7 +111,7 @@ void llvm::handleExecNameEncodedOptimizerOpts(StringRef ExecName) {
     } else if (Opt == "indvars") {
       Args.push_back("-passes=indvars");
     } else if (Opt == "strength_reduce") {
-      Args.push_back("-passes=strength-reduce");
+      Args.push_back("-passes=loop-reduce");
     } else if (Opt == "irce") {
       Args.push_back("-passes=irce");
 
diff --git a/llvm/lib/FuzzMutate/Operations.cpp b/llvm/lib/FuzzMutate/Operations.cpp
index d59b3ccbe50a2..a37fd5454dd42 100644
--- a/llvm/lib/FuzzMutate/Operations.cpp
+++ b/llvm/lib/FuzzMutate/Operations.cpp
@@ -302,7 +302,7 @@ static SourcePred validShuffleVectorIndex() {
     return ShuffleVectorInst::isValidOperands(Cur[0], Cur[1], V);
   };
   auto Make = [](ArrayRef<Value *> Cur, ArrayRef<Type *> Ts) {
-    auto *FirstTy = cast<VectorType>(Cur[0]->getType());
+    auto *FirstTy = cast<FixedVectorType>(Cur[0]->getType());
     auto *Int32Ty = Type::getInt32Ty(Cur[0]->getContext());
     // TODO: It's straighforward to make up reasonable values, but listing them
     // exhaustively would be insane. Come up with a couple of sensible ones.
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 53bfae23441b5..d408d7a4705b4 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -1654,7 +1654,7 @@ struct MDFieldPrinter {
                      bool ShouldSkipNull = true);
   template <class IntTy>
   void printInt(StringRef Name, IntTy Int, bool ShouldSkipZero = true);
-  void printAPInt(StringRef Name, APInt Int, bool IsUnsigned,
+  void printAPInt(StringRef Name, const APInt &Int, bool IsUnsigned,
                   bool ShouldSkipZero);
   void printBool(StringRef Name, bool Value, Optional<bool> Default = None);
   void printDIFlags(StringRef Name, DINode::DIFlags Flags);
@@ -1731,8 +1731,8 @@ void MDFieldPrinter::printInt(StringRef Name, IntTy Int, bool ShouldSkipZero) {
   Out << FS << Name << ": " << Int;
 }
 
-void MDFieldPrinter::printAPInt(StringRef Name, APInt Int, bool IsUnsigned,
-                                bool ShouldSkipZero) {
+void MDFieldPrinter::printAPInt(StringRef Name, const APInt &Int,
+                                bool IsUnsigned, bool ShouldSkipZero) {
   if (ShouldSkipZero && Int.isNullValue())
     return;
 
@@ -2510,10 +2510,10 @@ class AssemblyWriter {
   void printTypeIdInfo(const FunctionSummary::TypeIdInfo &TIDInfo);
   void printVFuncId(const FunctionSummary::VFuncId VFId);
   void
-  printNonConstVCalls(const std::vector<FunctionSummary::VFuncId> VCallList,
+  printNonConstVCalls(const std::vector<FunctionSummary::VFuncId> &VCallList,
                       const char *Tag);
   void
-  printConstVCalls(const std::vector<FunctionSummary::ConstVCall> VCallList,
+  printConstVCalls(const std::vector<FunctionSummary::ConstVCall> &VCallList,
                    const char *Tag);
 
 private:
@@ -3184,7 +3184,7 @@ void AssemblyWriter::printVFuncId(const FunctionSummary::VFuncId VFId) {
 }
 
 void AssemblyWriter::printNonConstVCalls(
-    const std::vector<FunctionSummary::VFuncId> VCallList, const char *Tag) {
+    const std::vector<FunctionSummary::VFuncId> &VCallList, const char *Tag) {
   Out << Tag << ": (";
   FieldSeparator FS;
   for (auto &VFuncId : VCallList) {
@@ -3195,7 +3195,8 @@ void AssemblyWriter::printNonConstVCalls(
 }
 
 void AssemblyWriter::printConstVCalls(
-    const std::vector<FunctionSummary::ConstVCall> VCallList, const char *Tag) {
+    const std::vector<FunctionSummary::ConstVCall> &VCallList,
+    const char *Tag) {
   Out << Tag << ": (";
   FieldSeparator FS;
   for (auto &ConstVCall : VCallList) {
diff --git a/llvm/lib/IR/AttributeImpl.h b/llvm/lib/IR/AttributeImpl.h
index 6aca780e3e5d1..5c334348cde37 100644
--- a/llvm/lib/IR/AttributeImpl.h
+++ b/llvm/lib/IR/AttributeImpl.h
@@ -193,11 +193,11 @@ class AttributeBitSet {
 
 public:
   bool hasAttribute(Attribute::AttrKind Kind) const {
-    return AvailableAttrs[Kind / 8] & ((uint64_t)1) << (Kind % 8);
+    return AvailableAttrs[Kind / 8] & (1 << (Kind % 8));
   }
 
   void addAttribute(Attribute::AttrKind Kind) {
-    AvailableAttrs[Kind / 8] |= 1ULL << (Kind % 8);
+    AvailableAttrs[Kind / 8] |= 1 << (Kind % 8);
   }
 };
 
@@ -282,6 +282,8 @@ class AttributeListImpl final
   unsigned NumAttrSets; ///< Number of entries in this set.
   /// Available enum function attributes.
   AttributeBitSet AvailableFunctionAttrs;
+  /// Union of enum attributes available at any index.
+  AttributeBitSet AvailableSomewhereAttrs;
 
   // Helper fn for TrailingObjects class.
   size_t numTrailingObjects(OverloadToken<AttributeSet>) { return NumAttrSets; }
@@ -299,6 +301,12 @@ class AttributeListImpl final
     return AvailableFunctionAttrs.hasAttribute(Kind);
   }
 
+  /// Return true if the specified attribute is set for at least one
+  /// parameter or for the return value. If Index is not nullptr, the index
+  /// of a parameter with the specified attribute is provided.
+  bool hasAttrSomewhere(Attribute::AttrKind Kind,
+                        unsigned *Index = nullptr) const;
+
   using iterator = const AttributeSet *;
 
   iterator begin() const { return getTrailingObjects<AttributeSet>(); }
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index d8798eb12ae78..f67d96a854f4d 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -22,8 +22,8 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/Twine.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/LLVMContext.h"
@@ -443,6 +443,8 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
     return "cold";
   if (hasAttribute(Attribute::ImmArg))
     return "immarg";
+  if (hasAttribute(Attribute::NoUndef))
+    return "noundef";
 
   if (hasAttribute(Attribute::ByVal)) {
     std::string Result;
@@ -597,6 +599,8 @@ Type *AttributeImpl::getValueAsType() const {
 }
 
 bool AttributeImpl::operator<(const AttributeImpl &AI) const {
+  if (this == &AI)
+    return false;
   // This sorts the attributes with Attribute::AttrKinds coming first (sorted
   // relative to their enum value) and then strings.
   if (isEnumAttribute()) {
@@ -965,11 +969,9 @@ std::string AttributeSetNode::getAsString(bool InAttrGrp) const {
 //===----------------------------------------------------------------------===//
 
 /// Map from AttributeList index to the internal array index. Adding one happens
-/// to work, but it relies on unsigned integer wrapping. MSVC warns about
-/// unsigned wrapping in constexpr functions, so write out the conditional. LLVM
-/// folds it to add anyway.
+/// to work, because -1 wraps around to 0.
 static constexpr unsigned attrIdxToArrayIdx(unsigned Index) {
-  return Index == AttributeList::FunctionIndex ? 0 : Index + 1;
+  return Index + 1;
 }
 
 AttributeListImpl::AttributeListImpl(ArrayRef<AttributeSet> Sets)
@@ -979,13 +981,18 @@ AttributeListImpl::AttributeListImpl(ArrayRef<AttributeSet> Sets)
   // There's memory after the node where we can store the entries in.
   llvm::copy(Sets, getTrailingObjects<AttributeSet>());
 
-  // Initialize AvailableFunctionAttrs summary bitset.
+  // Initialize AvailableFunctionAttrs and AvailableSomewhereAttrs
+  // summary bitsets.
   static_assert(attrIdxToArrayIdx(AttributeList::FunctionIndex) == 0U,
                 "function should be stored in slot 0");
-  for (const auto &I : Sets[0]) {
+  for (const auto &I : Sets[0])
     if (!I.isStringAttribute())
       AvailableFunctionAttrs.addAttribute(I.getKindAsEnum());
-  }
+
+  for (const auto &Set : Sets)
+    for (const auto &I : Set)
+      if (!I.isStringAttribute())
+        AvailableSomewhereAttrs.addAttribute(I.getKindAsEnum());
 }
 
 void AttributeListImpl::Profile(FoldingSetNodeID &ID) const {
@@ -998,6 +1005,24 @@ void AttributeListImpl::Profile(FoldingSetNodeID &ID,
     ID.AddPointer(Set.SetNode);
 }
 
+bool AttributeListImpl::hasAttrSomewhere(Attribute::AttrKind Kind,
+                                        unsigned *Index) const {
+  if (!AvailableSomewhereAttrs.hasAttribute(Kind))
+    return false;
+
+  if (Index) {
+    for (unsigned I = 0, E = NumAttrSets; I != E; ++I) {
+      if (begin()[I].hasAttribute(Kind)) {
+        *Index = I - 1;
+        break;
+      }
+    }
+  }
+
+  return true;
+}
+
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 LLVM_DUMP_METHOD void AttributeListImpl::dump() const {
   AttributeList(const_cast<AttributeListImpl *>(this)).dump();
@@ -1403,17 +1428,7 @@ bool AttributeList::hasParamAttribute(unsigned ArgNo,
 
 bool AttributeList::hasAttrSomewhere(Attribute::AttrKind Attr,
                                      unsigned *Index) const {
-  if (!pImpl) return false;
-
-  for (unsigned I = index_begin(), E = index_end(); I != E; ++I) {
-    if (hasAttribute(I, Attr)) {
-      if (Index)
-        *Index = I;
-      return true;
-    }
-  }
-
-  return false;
+  return pImpl && pImpl->hasAttrSomewhere(Attr, Index);
 }
 
 Attribute AttributeList::getAttribute(unsigned Index,
@@ -1695,7 +1710,7 @@ AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) {
 
   Attrs |= B.Attrs;
 
-  for (auto I : B.td_attrs())
+  for (const auto &I : B.td_attrs())
     TargetDepAttrs[I.first] = I.second;
 
   return *this;
@@ -1726,7 +1741,7 @@ AttrBuilder &AttrBuilder::remove(const AttrBuilder &B) {
 
   Attrs &= ~B.Attrs;
 
-  for (auto I : B.td_attrs())
+  for (const auto &I : B.td_attrs())
     TargetDepAttrs.erase(I.first);
 
   return *this;
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 3e0c3b91a04cf..1e8fdb5066195 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -3757,10 +3757,10 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
     auto *MemCI = cast<MemIntrinsic>(NewCall);
     // All mem intrinsics support dest alignment.
     const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
-    MemCI->setDestAlignment(Align->getZExtValue());
+    MemCI->setDestAlignment(Align->getMaybeAlignValue());
     // Memcpy/Memmove also support source alignment.
     if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
-      MTI->setSourceAlignment(Align->getZExtValue());
+      MTI->setSourceAlignment(Align->getMaybeAlignValue());
     break;
   }
   }
@@ -4167,7 +4167,7 @@ void llvm::UpgradeSectionAttributes(Module &M) {
   }
 }
 
-
+namespace {
 // Prior to LLVM 10.0, the strictfp attribute could be used on individual
 // callsites within a function that did not also have the strictfp attribute.
 // Since 10.0, if strict FP semantics are needed within a function, the
@@ -4185,7 +4185,7 @@ struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
   void visitCallBase(CallBase &Call) {
     if (!Call.isStrictFP())
       return;
-    if (dyn_cast<ConstrainedFPIntrinsic>(&Call))
+    if (isa<ConstrainedFPIntrinsic>(&Call))
       return;
     // If we get here, the caller doesn't have the strictfp attribute
     // but this callsite does. Replace the strictfp attribute with nobuiltin.
@@ -4193,6 +4193,7 @@ struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
     Call.addAttribute(AttributeList::FunctionIndex, Attribute::NoBuiltin);
   }
 };
+} // namespace
 
 void llvm::UpgradeFunctionAttributes(Function &F) {
   // If a function definition doesn't have the strictfp attribute,
diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp
index ef584afc68bcf..f02246cda7fc6 100644
--- a/llvm/lib/IR/ConstantFold.cpp
+++ b/llvm/lib/IR/ConstantFold.cpp
@@ -779,10 +779,30 @@ Constant *llvm::ConstantFoldSelectInstruction(Constant *Cond,
     if (isa<UndefValue>(V1)) return V1;
     return V2;
   }
-  if (isa<UndefValue>(V1)) return V2;
-  if (isa<UndefValue>(V2)) return V1;
+
   if (V1 == V2) return V1;
 
+  // If the true or false value is undef, we can fold to the other value as
+  // long as the other value isn't poison.
+  auto NotPoison = [](Constant *C) {
+    // TODO: We can analyze ConstExpr by opcode to determine if there is any
+    //       possibility of poison.
+    if (isa<ConstantExpr>(C))
+      return false;
+
+    if (isa<ConstantInt>(C) || isa<GlobalVariable>(C) || isa<ConstantFP>(C) ||
+        isa<ConstantPointerNull>(C) || isa<Function>(C))
+      return true;
+
+    if (C->getType()->isVectorTy())
+      return !C->containsUndefElement() && !C->containsConstantExpression();
+
+    // TODO: Recursively analyze aggregates or other constants.
+    return false;
+  };
+  if (isa<UndefValue>(V1) && NotPoison(V2)) return V2;
+  if (isa<UndefValue>(V2) && NotPoison(V1)) return V1;
+
   if (ConstantExpr *TrueVal = dyn_cast<ConstantExpr>(V1)) {
     if (TrueVal->getOpcode() == Instruction::Select)
       if (TrueVal->getOperand(0) == Cond)
@@ -1239,8 +1259,8 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1,
             // Without a datalayout we have to assume the worst case: that the
             // function pointer isn't aligned at all.
             GVAlign = llvm::None;
-          } else {
-            GVAlign = MaybeAlign(GV->getAlignment());
+          } else if (isa<GlobalVariable>(GV)) {
+            GVAlign = cast<GlobalVariable>(GV)->getAlign();
           }
 
           if (GVAlign && *GVAlign > 1) {
@@ -2298,18 +2318,17 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C,
       assert(Ty && "Invalid indices for GEP!");
       Type *OrigGEPTy = PointerType::get(Ty, PtrTy->getAddressSpace());
       Type *GEPTy = PointerType::get(Ty, PtrTy->getAddressSpace());
-      if (VectorType *VT = dyn_cast<VectorType>(C->getType())) {
-        // FIXME: handle scalable vectors (use getElementCount())
-        GEPTy = FixedVectorType::get(
-            OrigGEPTy, cast<FixedVectorType>(VT)->getNumElements());
-      }
+      if (VectorType *VT = dyn_cast<VectorType>(C->getType()))
+        GEPTy = VectorType::get(OrigGEPTy, VT->getElementCount());
+
       // The GEP returns a vector of pointers when one of more of
       // its arguments is a vector.
       for (unsigned i = 0, e = Idxs.size(); i != e; ++i) {
         if (auto *VT = dyn_cast<VectorType>(Idxs[i]->getType())) {
-          // FIXME: handle scalable vectors
-          GEPTy = FixedVectorType::get(
-              OrigGEPTy, cast<FixedVectorType>(VT)->getNumElements());
+          assert((!isa<VectorType>(GEPTy) || isa<ScalableVectorType>(GEPTy) ==
+                                                 isa<ScalableVectorType>(VT)) &&
+                 "Mismatched GEPTy vector types");
+          GEPTy = VectorType::get(OrigGEPTy, VT->getElementCount());
           break;
         }
       }
diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp
index daa15bb2a9b05..cbbcca20ea518 100644
--- a/llvm/lib/IR/Constants.cpp
+++ b/llvm/lib/IR/Constants.cpp
@@ -463,7 +463,74 @@ void Constant::destroyConstant() {
   }
 
   // Value has no outstanding references it is safe to delete it now...
-  delete this;
+  deleteConstant(this);
+}
+
+void llvm::deleteConstant(Constant *C) {
+  switch (C->getValueID()) {
+  case Constant::ConstantIntVal:
+    delete static_cast<ConstantInt *>(C);
+    break;
+  case Constant::ConstantFPVal:
+    delete static_cast<ConstantFP *>(C);
+    break;
+  case Constant::ConstantAggregateZeroVal:
+    delete static_cast<ConstantAggregateZero *>(C);
+    break;
+  case Constant::ConstantArrayVal:
+    delete static_cast<ConstantArray *>(C);
+    break;
+  case Constant::ConstantStructVal:
+    delete static_cast<ConstantStruct *>(C);
+    break;
+  case Constant::ConstantVectorVal:
+    delete static_cast<ConstantVector *>(C);
+    break;
+  case Constant::ConstantPointerNullVal:
+    delete static_cast<ConstantPointerNull *>(C);
+    break;
+  case Constant::ConstantDataArrayVal:
+    delete static_cast<ConstantDataArray *>(C);
+    break;
+  case Constant::ConstantDataVectorVal:
+    delete static_cast<ConstantDataVector *>(C);
+    break;
+  case Constant::ConstantTokenNoneVal:
+    delete static_cast<ConstantTokenNone *>(C);
+    break;
+  case Constant::BlockAddressVal:
+    delete static_cast<BlockAddress *>(C);
+    break;
+  case Constant::UndefValueVal:
+    delete static_cast<UndefValue *>(C);
+    break;
+  case Constant::ConstantExprVal:
+    if (isa<UnaryConstantExpr>(C))
+      delete static_cast<UnaryConstantExpr *>(C);
+    else if (isa<BinaryConstantExpr>(C))
+      delete static_cast<BinaryConstantExpr *>(C);
+    else if (isa<SelectConstantExpr>(C))
+      delete static_cast<SelectConstantExpr *>(C);
+    else if (isa<ExtractElementConstantExpr>(C))
+      delete static_cast<ExtractElementConstantExpr *>(C);
+    else if (isa<InsertElementConstantExpr>(C))
+      delete static_cast<InsertElementConstantExpr *>(C);
+    else if (isa<ShuffleVectorConstantExpr>(C))
+      delete static_cast<ShuffleVectorConstantExpr *>(C);
+    else if (isa<ExtractValueConstantExpr>(C))
+      delete static_cast<ExtractValueConstantExpr *>(C);
+    else if (isa<InsertValueConstantExpr>(C))
+      delete static_cast<InsertValueConstantExpr *>(C);
+    else if (isa<GetElementPtrConstantExpr>(C))
+      delete static_cast<GetElementPtrConstantExpr *>(C);
+    else if (isa<CompareConstantExpr>(C))
+      delete static_cast<CompareConstantExpr *>(C);
+    else
+      llvm_unreachable("Unexpected constant expr");
+    break;
+  default:
+    llvm_unreachable("Unexpected constant");
+  }
 }
 
 static bool canTrapImpl(const Constant *C,
@@ -1518,6 +1585,27 @@ Constant *Constant::getSplatValue(bool AllowUndefs) const {
     return CV->getSplatValue();
   if (const ConstantVector *CV = dyn_cast<ConstantVector>(this))
     return CV->getSplatValue(AllowUndefs);
+
+  // Check if this is a constant expression splat of the form returned by
+  // ConstantVector::getSplat()
+  const auto *Shuf = dyn_cast<ConstantExpr>(this);
+  if (Shuf && Shuf->getOpcode() == Instruction::ShuffleVector &&
+      isa<UndefValue>(Shuf->getOperand(1))) {
+
+    const auto *IElt = dyn_cast<ConstantExpr>(Shuf->getOperand(0));
+    if (IElt && IElt->getOpcode() == Instruction::InsertElement &&
+        isa<UndefValue>(IElt->getOperand(0))) {
+
+      ArrayRef<int> Mask = Shuf->getShuffleMask();
+      Constant *SplatVal = IElt->getOperand(1);
+      ConstantInt *Index = dyn_cast<ConstantInt>(IElt->getOperand(2));
+
+      if (Index && Index->getValue() == 0 &&
+          std::all_of(Mask.begin(), Mask.end(), [](int I) { return I == 0; }))
+        return SplatVal;
+    }
+  }
+
   return nullptr;
 }
 
@@ -2175,15 +2263,20 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C,
   std::vector<Constant*> ArgVec;
   ArgVec.reserve(1 + Idxs.size());
   ArgVec.push_back(C);
-  for (unsigned i = 0, e = Idxs.size(); i != e; ++i) {
+  auto GTI = gep_type_begin(Ty, Idxs), GTE = gep_type_end(Ty, Idxs);
+  for (; GTI != GTE; ++GTI) {
+    auto *Idx = cast<Constant>(GTI.getOperand());
     assert(
-        (!isa<VectorType>(Idxs[i]->getType()) ||
-         cast<VectorType>(Idxs[i]->getType())->getElementCount() == EltCount) &&
+        (!isa<VectorType>(Idx->getType()) ||
+         cast<VectorType>(Idx->getType())->getElementCount() == EltCount) &&
         "getelementptr index type missmatch");
 
-    Constant *Idx = cast<Constant>(Idxs[i]);
-    if (EltCount.Min != 0 && !Idxs[i]->getType()->isVectorTy())
+    if (GTI.isStruct() && Idx->getType()->isVectorTy()) {
+      Idx = Idx->getSplatValue();
+    } else if (GTI.isSequential() && EltCount.Min != 0 &&
+               !Idx->getType()->isVectorTy()) {
       Idx = ConstantVector::getSplat(EltCount, Idx);
+    }
     ArgVec.push_back(Idx);
   }
 
diff --git a/llvm/lib/IR/ConstantsContext.h b/llvm/lib/IR/ConstantsContext.h
index 4cd2621f625df..95c5ab6d0ee4d 100644
--- a/llvm/lib/IR/ConstantsContext.h
+++ b/llvm/lib/IR/ConstantsContext.h
@@ -43,7 +43,7 @@ namespace llvm {
 
 /// UnaryConstantExpr - This class is private to Constants.cpp, and is used
 /// behind the scenes to implement unary constant exprs.
-class UnaryConstantExpr : public ConstantExpr {
+class UnaryConstantExpr final : public ConstantExpr {
 public:
   UnaryConstantExpr(unsigned Opcode, Constant *C, Type *Ty)
     : ConstantExpr(Ty, Opcode, &Op<0>(), 1) {
@@ -56,11 +56,19 @@ class UnaryConstantExpr : public ConstantExpr {
   }
 
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  static bool classof(const ConstantExpr *CE) {
+    return Instruction::isCast(CE->getOpcode()) ||
+           Instruction::isUnaryOp(CE->getOpcode());
+  }
+  static bool classof(const Value *V) {
+    return isa<ConstantExpr>(V) && classof(cast<ConstantExpr>(V));
+  }
 };
 
 /// BinaryConstantExpr - This class is private to Constants.cpp, and is used
 /// behind the scenes to implement binary constant exprs.
-class BinaryConstantExpr : public ConstantExpr {
+class BinaryConstantExpr final : public ConstantExpr {
 public:
   BinaryConstantExpr(unsigned Opcode, Constant *C1, Constant *C2,
                      unsigned Flags)
@@ -77,11 +85,18 @@ class BinaryConstantExpr : public ConstantExpr {
 
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  static bool classof(const ConstantExpr *CE) {
+    return Instruction::isBinaryOp(CE->getOpcode());
+  }
+  static bool classof(const Value *V) {
+    return isa<ConstantExpr>(V) && classof(cast<ConstantExpr>(V));
+  }
 };
 
 /// SelectConstantExpr - This class is private to Constants.cpp, and is used
 /// behind the scenes to implement select constant exprs.
-class SelectConstantExpr : public ConstantExpr {
+class SelectConstantExpr final : public ConstantExpr {
 public:
   SelectConstantExpr(Constant *C1, Constant *C2, Constant *C3)
     : ConstantExpr(C2->getType(), Instruction::Select, &Op<0>(), 3) {
@@ -97,12 +112,19 @@ class SelectConstantExpr : public ConstantExpr {
 
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  static bool classof(const ConstantExpr *CE) {
+    return CE->getOpcode() == Instruction::Select;
+  }
+  static bool classof(const Value *V) {
+    return isa<ConstantExpr>(V) && classof(cast<ConstantExpr>(V));
+  }
 };
 
 /// ExtractElementConstantExpr - This class is private to
 /// Constants.cpp, and is used behind the scenes to implement
 /// extractelement constant exprs.
-class ExtractElementConstantExpr : public ConstantExpr {
+class ExtractElementConstantExpr final : public ConstantExpr {
 public:
   ExtractElementConstantExpr(Constant *C1, Constant *C2)
     : ConstantExpr(cast<VectorType>(C1->getType())->getElementType(),
@@ -118,12 +140,19 @@ class ExtractElementConstantExpr : public ConstantExpr {
 
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  static bool classof(const ConstantExpr *CE) {
+    return CE->getOpcode() == Instruction::ExtractElement;
+  }
+  static bool classof(const Value *V) {
+    return isa<ConstantExpr>(V) && classof(cast<ConstantExpr>(V));
+  }
 };
 
 /// InsertElementConstantExpr - This class is private to
 /// Constants.cpp, and is used behind the scenes to implement
 /// insertelement constant exprs.
-class InsertElementConstantExpr : public ConstantExpr {
+class InsertElementConstantExpr final : public ConstantExpr {
 public:
   InsertElementConstantExpr(Constant *C1, Constant *C2, Constant *C3)
     : ConstantExpr(C1->getType(), Instruction::InsertElement,
@@ -140,12 +169,19 @@ class InsertElementConstantExpr : public ConstantExpr {
 
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  static bool classof(const ConstantExpr *CE) {
+    return CE->getOpcode() == Instruction::InsertElement;
+  }
+  static bool classof(const Value *V) {
+    return isa<ConstantExpr>(V) && classof(cast<ConstantExpr>(V));
+  }
 };
 
 /// ShuffleVectorConstantExpr - This class is private to
 /// Constants.cpp, and is used behind the scenes to implement
 /// shufflevector constant exprs.
-class ShuffleVectorConstantExpr : public ConstantExpr {
+class ShuffleVectorConstantExpr final : public ConstantExpr {
 public:
   ShuffleVectorConstantExpr(Constant *C1, Constant *C2, ArrayRef<int> Mask)
       : ConstantExpr(VectorType::get(
@@ -168,12 +204,19 @@ class ShuffleVectorConstantExpr : public ConstantExpr {
 
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  static bool classof(const ConstantExpr *CE) {
+    return CE->getOpcode() == Instruction::ShuffleVector;
+  }
+  static bool classof(const Value *V) {
+    return isa<ConstantExpr>(V) && classof(cast<ConstantExpr>(V));
+  }
 };
 
 /// ExtractValueConstantExpr - This class is private to
 /// Constants.cpp, and is used behind the scenes to implement
 /// extractvalue constant exprs.
-class ExtractValueConstantExpr : public ConstantExpr {
+class ExtractValueConstantExpr final : public ConstantExpr {
 public:
   ExtractValueConstantExpr(Constant *Agg, ArrayRef<unsigned> IdxList,
                            Type *DestTy)
@@ -204,7 +247,7 @@ class ExtractValueConstantExpr : public ConstantExpr {
 /// InsertValueConstantExpr - This class is private to
 /// Constants.cpp, and is used behind the scenes to implement
 /// insertvalue constant exprs.
-class InsertValueConstantExpr : public ConstantExpr {
+class InsertValueConstantExpr final : public ConstantExpr {
 public:
   InsertValueConstantExpr(Constant *Agg, Constant *Val,
                           ArrayRef<unsigned> IdxList, Type *DestTy)
@@ -235,7 +278,7 @@ class InsertValueConstantExpr : public ConstantExpr {
 
 /// GetElementPtrConstantExpr - This class is private to Constants.cpp, and is
 /// used behind the scenes to implement getelementpr constant exprs.
-class GetElementPtrConstantExpr : public ConstantExpr {
+class GetElementPtrConstantExpr final : public ConstantExpr {
   Type *SrcElementTy;
   Type *ResElementTy;
 
@@ -269,7 +312,7 @@ class GetElementPtrConstantExpr : public ConstantExpr {
 // CompareConstantExpr - This class is private to Constants.cpp, and is used
 // behind the scenes to implement ICmp and FCmp constant expressions. This is
 // needed in order to store the predicate value for these instructions.
-class CompareConstantExpr : public ConstantExpr {
+class CompareConstantExpr final : public ConstantExpr {
 public:
   unsigned short predicate;
   CompareConstantExpr(Type *ty, Instruction::OtherOps opc,
@@ -597,6 +640,10 @@ struct ConstantExprKeyType {
   }
 };
 
+// Free memory for a given constant.  Assumes the constant has already been
+// removed from all relevant maps.
+void deleteConstant(Constant *C);
+
 template <class ConstantClass> class ConstantUniqueMap {
 public:
   using ValType = typename ConstantInfo<ConstantClass>::ValType;
@@ -660,7 +707,7 @@ template <class ConstantClass> class ConstantUniqueMap {
 
   void freeConstants() {
     for (auto &I : Map)
-      delete I; // Asserts that use_empty().
+      deleteConstant(I);
   }
 
 private:
@@ -733,6 +780,11 @@ template <class ConstantClass> class ConstantUniqueMap {
   }
 };
 
+template <> inline void ConstantUniqueMap<InlineAsm>::freeConstants() {
+  for (auto &I : Map)
+    delete I;
+}
+
 } // end namespace llvm
 
 #endif // LLVM_LIB_IR_CONSTANTSCONTEXT_H
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index 52755f820b662..6f3bbc80d4fd5 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -2003,7 +2003,7 @@ LLVMTypeRef LLVMGlobalGetValueType(LLVMValueRef Global) {
 
 unsigned LLVMGetAlignment(LLVMValueRef V) {
   Value *P = unwrap<Value>(V);
-  if (GlobalValue *GV = dyn_cast<GlobalValue>(P))
+  if (GlobalObject *GV = dyn_cast<GlobalObject>(P))
     return GV->getAlignment();
   if (AllocaInst *AI = dyn_cast<AllocaInst>(P))
     return AI->getAlignment();
@@ -2013,7 +2013,7 @@ unsigned LLVMGetAlignment(LLVMValueRef V) {
     return SI->getAlignment();
 
   llvm_unreachable(
-      "only GlobalValue, AllocaInst, LoadInst and StoreInst have alignment");
+      "only GlobalObject, AllocaInst, LoadInst and StoreInst have alignment");
 }
 
 void LLVMSetAlignment(LLVMValueRef V, unsigned Bytes) {
diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp
index 0fe3199b9b49c..c44737c5bfc24 100644
--- a/llvm/lib/IR/DataLayout.cpp
+++ b/llvm/lib/IR/DataLayout.cpp
@@ -52,7 +52,7 @@ StructLayout::StructLayout(StructType *ST, const DataLayout &DL) {
   // Loop over each of the elements, placing them in memory.
   for (unsigned i = 0, e = NumElements; i != e; ++i) {
     Type *Ty = ST->getElementType(i);
-    const Align TyAlign(ST->isPacked() ? 1 : DL.getABITypeAlignment(Ty));
+    const Align TyAlign = ST->isPacked() ? Align(1) : DL.getABITypeAlign(Ty);
 
     // Add padding if necessary to align the data element properly.
     if (!isAligned(TyAlign, StructSize)) {
@@ -153,6 +153,8 @@ const char *DataLayout::getManglingComponent(const Triple &T) {
     return "-m:o";
   if (T.isOSWindows() && T.isOSBinFormatCOFF())
     return T.getArch() == Triple::x86 ? "-m:x" : "-m:w";
+  if (T.isOSBinFormatXCOFF())
+    return "-m:a";
   return "-m:e";
 }
 
@@ -260,8 +262,8 @@ void DataLayout::parseSpecifier(StringRef Desc) {
 
     switch (Specifier) {
     case 's':
-      // Ignored for backward compatibility.
-      // FIXME: remove this on LLVM 4.0.
+      // Deprecated, but ignoring here to preserve loading older textual llvm
+      // ASM file
       break;
     case 'E':
       BigEndian = true;
@@ -444,6 +446,9 @@ void DataLayout::parseSpecifier(StringRef Desc) {
       case 'x':
         ManglingMode = MM_WinCOFFX86;
         break;
+      case 'a':
+        ManglingMode = MM_XCOFF;
+        break;
       }
       break;
     default:
@@ -792,7 +797,7 @@ Type *DataLayout::getIntPtrType(Type *Ty) const {
   unsigned NumBits = getPointerTypeSizeInBits(Ty);
   IntegerType *IntTy = IntegerType::get(Ty->getContext(), NumBits);
   if (VectorType *VecTy = dyn_cast<VectorType>(Ty))
-    return FixedVectorType::get(IntTy, VecTy->getNumElements());
+    return VectorType::get(IntTy, VecTy);
   return IntTy;
 }
 
@@ -814,7 +819,7 @@ Type *DataLayout::getIndexType(Type *Ty) const {
   unsigned NumBits = getIndexTypeSizeInBits(Ty);
   IntegerType *IntTy = IntegerType::get(Ty->getContext(), NumBits);
   if (VectorType *VecTy = dyn_cast<VectorType>(Ty))
-    return FixedVectorType::get(IntTy, VecTy->getNumElements());
+    return VectorType::get(IntTy, VecTy);
   return IntTy;
 }
 
@@ -846,15 +851,14 @@ int64_t DataLayout::getIndexedOffsetInType(Type *ElemTy,
   return Result;
 }
 
-/// getPreferredAlignment - Return the preferred alignment of the specified
-/// global.  This includes an explicitly requested alignment (if the global
-/// has one).
-unsigned DataLayout::getPreferredAlignment(const GlobalVariable *GV) const {
-  unsigned GVAlignment = GV->getAlignment();
+/// getPreferredAlign - Return the preferred alignment of the specified global.
+/// This includes an explicitly requested alignment (if the global has one).
+Align DataLayout::getPreferredAlign(const GlobalVariable *GV) const {
+  MaybeAlign GVAlignment = GV->getAlign();
   // If a section is specified, always precisely honor explicit alignment,
   // so we don't insert padding into a section we don't control.
   if (GVAlignment && GV->hasSection())
-    return GVAlignment;
+    return *GVAlignment;
 
   // If no explicit alignment is specified, compute the alignment based on
   // the IR type. If an alignment is specified, increase it to match the ABI
@@ -863,30 +867,24 @@ unsigned DataLayout::getPreferredAlignment(const GlobalVariable *GV) const {
   // FIXME: Not sure it makes sense to use the alignment of the type if
   // there's already an explicit alignment specification.
   Type *ElemType = GV->getValueType();
-  unsigned Alignment = getPrefTypeAlignment(ElemType);
-  if (GVAlignment >= Alignment) {
-    Alignment = GVAlignment;
-  } else if (GVAlignment != 0) {
-    Alignment = std::max(GVAlignment, getABITypeAlignment(ElemType));
+  Align Alignment = getPrefTypeAlign(ElemType);
+  if (GVAlignment) {
+    if (*GVAlignment >= Alignment)
+      Alignment = *GVAlignment;
+    else
+      Alignment = std::max(*GVAlignment, getABITypeAlign(ElemType));
   }
 
   // If no explicit alignment is specified, and the global is large, increase
   // the alignment to 16.
   // FIXME: Why 16, specifically?
-  if (GV->hasInitializer() && GVAlignment == 0) {
-    if (Alignment < 16) {
+  if (GV->hasInitializer() && !GVAlignment) {
+    if (Alignment < Align(16)) {
       // If the global is not external, see if it is large.  If so, give it a
       // larger alignment.
       if (getTypeSizeInBits(ElemType) > 128)
-        Alignment = 16;    // 16-byte alignment.
+        Alignment = Align(16); // 16-byte alignment.
     }
   }
   return Alignment;
 }
-
-/// getPreferredAlignmentLog - Return the preferred alignment of the
-/// specified global, returned in log form.  This includes an explicitly
-/// requested alignment (if the global has one).
-unsigned DataLayout::getPreferredAlignmentLog(const GlobalVariable *GV) const {
-  return Log2_32(getPreferredAlignment(GV));
-}
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index 254b6fe563e37..190b220dc9aa7 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -647,7 +647,7 @@ bool llvm::stripNonLineTableDebugInfo(Module &M) {
     }
     for (auto &BB : F) {
       for (auto &I : BB) {
-        auto remapDebugLoc = [&](DebugLoc DL) -> DebugLoc {
+        auto remapDebugLoc = [&](const DebugLoc &DL) -> DebugLoc {
           auto *Scope = DL.getScope();
           MDNode *InlinedAt = DL.getInlinedAt();
           Scope = remap(Scope);
diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index ea90d6842c317..110d94116f102 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -435,7 +435,7 @@ DISubrange::BoundType DISubrange::getStride() const {
   return BoundType();
 }
 
-DIEnumerator *DIEnumerator::getImpl(LLVMContext &Context, APInt Value,
+DIEnumerator *DIEnumerator::getImpl(LLVMContext &Context, const APInt &Value,
                                     bool IsUnsigned, MDString *Name,
                                     StorageType Storage, bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
diff --git a/llvm/lib/IR/DebugLoc.cpp b/llvm/lib/IR/DebugLoc.cpp
index 14d1396f15433..e945cbcba7825 100644
--- a/llvm/lib/IR/DebugLoc.cpp
+++ b/llvm/lib/IR/DebugLoc.cpp
@@ -79,7 +79,7 @@ DebugLoc DebugLoc::get(unsigned Line, unsigned Col, const MDNode *Scope,
                          const_cast<MDNode *>(InlinedAt), ImplicitCode);
 }
 
-DebugLoc DebugLoc::appendInlinedAt(DebugLoc DL, DILocation *InlinedAt,
+DebugLoc DebugLoc::appendInlinedAt(const DebugLoc &DL, DILocation *InlinedAt,
                                    LLVMContext &Ctx,
                                    DenseMap<const MDNode *, MDNode *> &Cache,
                                    bool ReplaceLast) {
diff --git a/llvm/lib/IR/FPEnv.cpp b/llvm/lib/IR/FPEnv.cpp
index 40f999e05b98e..516c702acec70 100644
--- a/llvm/lib/IR/FPEnv.cpp
+++ b/llvm/lib/IR/FPEnv.cpp
@@ -12,8 +12,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/ADT/StringSwitch.h"
 #include "llvm/IR/FPEnv.h"
+#include "llvm/ADT/StringSwitch.h"
 
 namespace llvm {
 
@@ -80,4 +80,4 @@ Optional<StringRef> ExceptionBehaviorToStr(fp::ExceptionBehavior UseExcept) {
   }
   return ExceptStr;
 }
-}
+} // namespace llvm
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index 972beec440341..0ec0cce83a8c5 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -128,6 +128,27 @@ bool Argument::hasPassPointeeByValueAttr() const {
          Attrs.hasParamAttribute(getArgNo(), Attribute::Preallocated);
 }
 
+uint64_t Argument::getPassPointeeByValueCopySize(const DataLayout &DL) const {
+  AttributeSet ParamAttrs
+    = getParent()->getAttributes().getParamAttributes(getArgNo());
+
+  // FIXME: All the type carrying attributes are mutually exclusive, so there
+  // should be a single query to get the stored type that handles any of them.
+  if (Type *ByValTy = ParamAttrs.getByValType())
+    return DL.getTypeAllocSize(ByValTy);
+  if (Type *PreAllocTy = ParamAttrs.getPreallocatedType())
+    return DL.getTypeAllocSize(PreAllocTy);
+
+  // FIXME: inalloca always depends on pointee element type. It's also possible
+  // for byval to miss it.
+  if (ParamAttrs.hasAttribute(Attribute::InAlloca) ||
+      ParamAttrs.hasAttribute(Attribute::ByVal) ||
+      ParamAttrs.hasAttribute(Attribute::Preallocated))
+    return DL.getTypeAllocSize(cast<PointerType>(getType())->getElementType());
+
+  return 0;
+}
+
 unsigned Argument::getParamAlignment() const {
   assert(getType()->isPointerTy() && "Only pointers have alignments");
   return getParent()->getParamAlignment(getArgNo());
@@ -1424,32 +1445,41 @@ Intrinsic::matchIntrinsicVarArg(bool isVarArg,
   return true;
 }
 
-Optional<Function*> Intrinsic::remangleIntrinsicFunction(Function *F) {
+bool Intrinsic::getIntrinsicSignature(Function *F,
+                                      SmallVectorImpl<Type *> &ArgTys) {
   Intrinsic::ID ID = F->getIntrinsicID();
   if (!ID)
-    return None;
+    return false;
 
-  FunctionType *FTy = F->getFunctionType();
-  // Accumulate an array of overloaded types for the given intrinsic
-  SmallVector<Type *, 4> ArgTys;
-  {
-    SmallVector<Intrinsic::IITDescriptor, 8> Table;
-    getIntrinsicInfoTableEntries(ID, Table);
-    ArrayRef<Intrinsic::IITDescriptor> TableRef = Table;
-
-    if (Intrinsic::matchIntrinsicSignature(FTy, TableRef, ArgTys))
-      return None;
-    if (Intrinsic::matchIntrinsicVarArg(FTy->isVarArg(), TableRef))
-      return None;
+  SmallVector<Intrinsic::IITDescriptor, 8> Table;
+  getIntrinsicInfoTableEntries(ID, Table);
+  ArrayRef<Intrinsic::IITDescriptor> TableRef = Table;
+
+  if (Intrinsic::matchIntrinsicSignature(F->getFunctionType(), TableRef,
+                                         ArgTys) !=
+      Intrinsic::MatchIntrinsicTypesResult::MatchIntrinsicTypes_Match) {
+    return false;
   }
+  if (Intrinsic::matchIntrinsicVarArg(F->getFunctionType()->isVarArg(),
+                                      TableRef))
+    return false;
+  return true;
+}
 
+Optional<Function *> Intrinsic::remangleIntrinsicFunction(Function *F) {
+  SmallVector<Type *, 4> ArgTys;
+  if (!getIntrinsicSignature(F, ArgTys))
+    return None;
+
+  Intrinsic::ID ID = F->getIntrinsicID();
   StringRef Name = F->getName();
   if (Name == Intrinsic::getName(ID, ArgTys))
     return None;
 
   auto NewDecl = Intrinsic::getDeclaration(F->getParent(), ID, ArgTys);
   NewDecl->setCallingConv(F->getCallingConv());
-  assert(NewDecl->getFunctionType() == FTy && "Shouldn't change the signature");
+  assert(NewDecl->getFunctionType() == F->getFunctionType() &&
+         "Shouldn't change the signature");
   return NewDecl;
 }
 
diff --git a/llvm/lib/IR/Globals.cpp b/llvm/lib/IR/Globals.cpp
index 34c767ff36720..dd8e62164de1e 100644
--- a/llvm/lib/IR/Globals.cpp
+++ b/llvm/lib/IR/Globals.cpp
@@ -108,20 +108,6 @@ bool GlobalValue::canBenefitFromLocalAlias() const {
          !isa<GlobalIFunc>(this) && !hasComdat();
 }
 
-unsigned GlobalValue::getAlignment() const {
-  if (auto *GA = dyn_cast<GlobalAlias>(this)) {
-    // In general we cannot compute this at the IR level, but we try.
-    if (const GlobalObject *GO = GA->getBaseObject())
-      return GO->getAlignment();
-
-    // FIXME: we should also be able to handle:
-    // Alias = Global + Offset
-    // Alias = Absolute
-    return 0;
-  }
-  return cast<GlobalObject>(this)->getAlignment();
-}
-
 unsigned GlobalValue::getAddressSpace() const {
   PointerType *PtrTy = getType();
   return PtrTy->getAddressSpace();
@@ -252,7 +238,7 @@ bool GlobalValue::isDeclaration() const {
   return false;
 }
 
-bool GlobalValue::canIncreaseAlignment() const {
+bool GlobalObject::canIncreaseAlignment() const {
   // Firstly, can only increase the alignment of a global if it
   // is a strong definition.
   if (!isStrongDefinitionForLinker())
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index bd4978e64a81f..1fffce015f707 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -23,8 +23,8 @@
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Operator.h"
 #include "llvm/IR/NoFolder.h"
+#include "llvm/IR/Operator.h"
 #include "llvm/IR/Statepoint.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 2087188e3edd2..f650ad9130ac5 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -53,7 +53,7 @@ Optional<uint64_t>
 AllocaInst::getAllocationSizeInBits(const DataLayout &DL) const {
   uint64_t Size = DL.getTypeAllocSizeInBits(getAllocatedType());
   if (isArrayAllocation()) {
-    auto C = dyn_cast<ConstantInt>(getArraySize());
+    auto *C = dyn_cast<ConstantInt>(getArraySize());
     if (!C)
       return None;
     Size *= C->getZExtValue();
@@ -247,6 +247,20 @@ void LandingPadInst::addClause(Constant *Val) {
 //                        CallBase Implementation
 //===----------------------------------------------------------------------===//
 
+CallBase *CallBase::Create(CallBase *CB, ArrayRef<OperandBundleDef> Bundles,
+                           Instruction *InsertPt) {
+  switch (CB->getOpcode()) {
+  case Instruction::Call:
+    return CallInst::Create(cast<CallInst>(CB), Bundles, InsertPt);
+  case Instruction::Invoke:
+    return InvokeInst::Create(cast<InvokeInst>(CB), Bundles, InsertPt);
+  case Instruction::CallBr:
+    return CallBrInst::Create(cast<CallBrInst>(CB), Bundles, InsertPt);
+  default:
+    llvm_unreachable("Unknown CallBase sub-class!");
+  }
+}
+
 Function *CallBase::getCaller() { return getParent()->getParent(); }
 
 unsigned CallBase::getNumSubclassExtraOperandsDynamic() const {
@@ -960,7 +974,8 @@ CleanupReturnInst::CleanupReturnInst(const CleanupReturnInst &CRI)
                   OperandTraits<CleanupReturnInst>::op_end(this) -
                       CRI.getNumOperands(),
                   CRI.getNumOperands()) {
-  setInstructionSubclassData(CRI.getSubclassDataFromInstruction());
+  setSubclassData<Instruction::OpaqueField>(
+      CRI.getSubclassData<Instruction::OpaqueField>());
   Op<0>() = CRI.Op<0>();
   if (CRI.hasUnwindDest())
     Op<1>() = CRI.Op<1>();
@@ -968,7 +983,7 @@ CleanupReturnInst::CleanupReturnInst(const CleanupReturnInst &CRI)
 
 void CleanupReturnInst::init(Value *CleanupPad, BasicBlock *UnwindBB) {
   if (UnwindBB)
-    setInstructionSubclassData(getSubclassDataFromInstruction() | 1);
+    setSubclassData<UnwindDestField>(true);
 
   Op<0>() = CleanupPad;
   if (UnwindBB)
@@ -1072,7 +1087,7 @@ void CatchSwitchInst::init(Value *ParentPad, BasicBlock *UnwindDest,
 
   Op<0>() = ParentPad;
   if (UnwindDest) {
-    setInstructionSubclassData(getSubclassDataFromInstruction() | 1);
+    setSubclassData<UnwindDestField>(true);
     setUnwindDest(UnwindDest);
   }
 }
@@ -1296,13 +1311,6 @@ AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
   setName(Name);
 }
 
-void AllocaInst::setAlignment(Align Align) {
-  assert(Align <= MaximumAlignment &&
-         "Alignment is greater than MaximumAlignment!");
-  setInstructionSubclassData((getSubclassDataFromInstruction() & ~31) |
-                             encode(Align));
-  assert(getAlignment() == Align.value() && "Alignment representation error!");
-}
 
 bool AllocaInst::isArrayAllocation() const {
   if (ConstantInt *CI = dyn_cast<ConstantInt>(getOperand(0)))
@@ -1394,14 +1402,6 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
   setName(Name);
 }
 
-void LoadInst::setAlignment(Align Align) {
-  assert(Align <= MaximumAlignment &&
-         "Alignment is greater than MaximumAlignment!");
-  setInstructionSubclassData((getSubclassDataFromInstruction() & ~(31 << 1)) |
-                             (encode(Align) << 1));
-  assert(getAlign() == Align && "Alignment representation error!");
-}
-
 //===----------------------------------------------------------------------===//
 //                           StoreInst Implementation
 //===----------------------------------------------------------------------===//
@@ -1473,20 +1473,13 @@ StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, Align Align,
   AssertOK();
 }
 
-void StoreInst::setAlignment(Align Alignment) {
-  assert(Alignment <= MaximumAlignment &&
-         "Alignment is greater than MaximumAlignment!");
-  setInstructionSubclassData((getSubclassDataFromInstruction() & ~(31 << 1)) |
-                             (encode(Alignment) << 1));
-  assert(getAlign() == Alignment && "Alignment representation error!");
-}
 
 //===----------------------------------------------------------------------===//
 //                       AtomicCmpXchgInst Implementation
 //===----------------------------------------------------------------------===//
 
 void AtomicCmpXchgInst::Init(Value *Ptr, Value *Cmp, Value *NewVal,
-                             AtomicOrdering SuccessOrdering,
+                             Align Alignment, AtomicOrdering SuccessOrdering,
                              AtomicOrdering FailureOrdering,
                              SyncScope::ID SSID) {
   Op<0>() = Ptr;
@@ -1495,6 +1488,7 @@ void AtomicCmpXchgInst::Init(Value *Ptr, Value *Cmp, Value *NewVal,
   setSuccessOrdering(SuccessOrdering);
   setFailureOrdering(FailureOrdering);
   setSyncScopeID(SSID);
+  setAlignment(Alignment);
 
   assert(getOperand(0) && getOperand(1) && getOperand(2) &&
          "All operands must be non-null!");
@@ -1519,6 +1513,7 @@ void AtomicCmpXchgInst::Init(Value *Ptr, Value *Cmp, Value *NewVal,
 }
 
 AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
+                                     Align Alignment,
                                      AtomicOrdering SuccessOrdering,
                                      AtomicOrdering FailureOrdering,
                                      SyncScope::ID SSID,
@@ -1527,10 +1522,11 @@ AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
           StructType::get(Cmp->getType(), Type::getInt1Ty(Cmp->getContext())),
           AtomicCmpXchg, OperandTraits<AtomicCmpXchgInst>::op_begin(this),
           OperandTraits<AtomicCmpXchgInst>::operands(this), InsertBefore) {
-  Init(Ptr, Cmp, NewVal, SuccessOrdering, FailureOrdering, SSID);
+  Init(Ptr, Cmp, NewVal, Alignment, SuccessOrdering, FailureOrdering, SSID);
 }
 
 AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
+                                     Align Alignment,
                                      AtomicOrdering SuccessOrdering,
                                      AtomicOrdering FailureOrdering,
                                      SyncScope::ID SSID,
@@ -1539,7 +1535,7 @@ AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
           StructType::get(Cmp->getType(), Type::getInt1Ty(Cmp->getContext())),
           AtomicCmpXchg, OperandTraits<AtomicCmpXchgInst>::op_begin(this),
           OperandTraits<AtomicCmpXchgInst>::operands(this), InsertAtEnd) {
-  Init(Ptr, Cmp, NewVal, SuccessOrdering, FailureOrdering, SSID);
+  Init(Ptr, Cmp, NewVal, Alignment, SuccessOrdering, FailureOrdering, SSID);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1547,13 +1543,14 @@ AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
 //===----------------------------------------------------------------------===//
 
 void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val,
-                         AtomicOrdering Ordering,
+                         Align Alignment, AtomicOrdering Ordering,
                          SyncScope::ID SSID) {
   Op<0>() = Ptr;
   Op<1>() = Val;
   setOperation(Operation);
   setOrdering(Ordering);
   setSyncScopeID(SSID);
+  setAlignment(Alignment);
 
   assert(getOperand(0) && getOperand(1) &&
          "All operands must be non-null!");
@@ -1567,25 +1564,21 @@ void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val,
 }
 
 AtomicRMWInst::AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
-                             AtomicOrdering Ordering,
-                             SyncScope::ID SSID,
-                             Instruction *InsertBefore)
-  : Instruction(Val->getType(), AtomicRMW,
-                OperandTraits<AtomicRMWInst>::op_begin(this),
-                OperandTraits<AtomicRMWInst>::operands(this),
-                InsertBefore) {
-  Init(Operation, Ptr, Val, Ordering, SSID);
+                             Align Alignment, AtomicOrdering Ordering,
+                             SyncScope::ID SSID, Instruction *InsertBefore)
+    : Instruction(Val->getType(), AtomicRMW,
+                  OperandTraits<AtomicRMWInst>::op_begin(this),
+                  OperandTraits<AtomicRMWInst>::operands(this), InsertBefore) {
+  Init(Operation, Ptr, Val, Alignment, Ordering, SSID);
 }
 
 AtomicRMWInst::AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
-                             AtomicOrdering Ordering,
-                             SyncScope::ID SSID,
-                             BasicBlock *InsertAtEnd)
-  : Instruction(Val->getType(), AtomicRMW,
-                OperandTraits<AtomicRMWInst>::op_begin(this),
-                OperandTraits<AtomicRMWInst>::operands(this),
-                InsertAtEnd) {
-  Init(Operation, Ptr, Val, Ordering, SSID);
+                             Align Alignment, AtomicOrdering Ordering,
+                             SyncScope::ID SSID, BasicBlock *InsertAtEnd)
+    : Instruction(Val->getType(), AtomicRMW,
+                  OperandTraits<AtomicRMWInst>::op_begin(this),
+                  OperandTraits<AtomicRMWInst>::operands(this), InsertAtEnd) {
+  Init(Operation, Ptr, Val, Alignment, Ordering, SSID);
 }
 
 StringRef AtomicRMWInst::getOperationName(BinOp Op) {
@@ -1668,29 +1661,29 @@ GetElementPtrInst::GetElementPtrInst(const GetElementPtrInst &GEPI)
 }
 
 Type *GetElementPtrInst::getTypeAtIndex(Type *Ty, Value *Idx) {
-  if (auto Struct = dyn_cast<StructType>(Ty)) {
+  if (auto *Struct = dyn_cast<StructType>(Ty)) {
     if (!Struct->indexValid(Idx))
       return nullptr;
     return Struct->getTypeAtIndex(Idx);
   }
   if (!Idx->getType()->isIntOrIntVectorTy())
     return nullptr;
-  if (auto Array = dyn_cast<ArrayType>(Ty))
+  if (auto *Array = dyn_cast<ArrayType>(Ty))
     return Array->getElementType();
-  if (auto Vector = dyn_cast<VectorType>(Ty))
+  if (auto *Vector = dyn_cast<VectorType>(Ty))
     return Vector->getElementType();
   return nullptr;
 }
 
 Type *GetElementPtrInst::getTypeAtIndex(Type *Ty, uint64_t Idx) {
-  if (auto Struct = dyn_cast<StructType>(Ty)) {
+  if (auto *Struct = dyn_cast<StructType>(Ty)) {
     if (Idx >= Struct->getNumElements())
       return nullptr;
     return Struct->getElementType(Idx);
   }
-  if (auto Array = dyn_cast<ArrayType>(Ty))
+  if (auto *Array = dyn_cast<ArrayType>(Ty))
     return Array->getElementType();
-  if (auto Vector = dyn_cast<VectorType>(Ty))
+  if (auto *Vector = dyn_cast<VectorType>(Ty))
     return Vector->getElementType();
   return nullptr;
 }
@@ -4275,10 +4268,9 @@ StoreInst *StoreInst::cloneImpl() const {
 }
 
 AtomicCmpXchgInst *AtomicCmpXchgInst::cloneImpl() const {
-  AtomicCmpXchgInst *Result =
-    new AtomicCmpXchgInst(getOperand(0), getOperand(1), getOperand(2),
-                          getSuccessOrdering(), getFailureOrdering(),
-                          getSyncScopeID());
+  AtomicCmpXchgInst *Result = new AtomicCmpXchgInst(
+      getOperand(0), getOperand(1), getOperand(2), getAlign(),
+      getSuccessOrdering(), getFailureOrdering(), getSyncScopeID());
   Result->setVolatile(isVolatile());
   Result->setWeak(isWeak());
   return Result;
@@ -4286,8 +4278,8 @@ AtomicCmpXchgInst *AtomicCmpXchgInst::cloneImpl() const {
 
 AtomicRMWInst *AtomicRMWInst::cloneImpl() const {
   AtomicRMWInst *Result =
-    new AtomicRMWInst(getOperation(), getOperand(0), getOperand(1),
-                      getOrdering(), getSyncScopeID());
+      new AtomicRMWInst(getOperation(), getOperand(0), getOperand(1),
+                        getAlign(), getOrdering(), getSyncScopeID());
   Result->setVolatile(isVolatile());
   return Result;
 }
diff --git a/llvm/lib/IR/LegacyPassManager.cpp b/llvm/lib/IR/LegacyPassManager.cpp
index d5c7e2874825d..4189aea46294c 100644
--- a/llvm/lib/IR/LegacyPassManager.cpp
+++ b/llvm/lib/IR/LegacyPassManager.cpp
@@ -33,7 +33,6 @@
 #include <algorithm>
 #include <unordered_set>
 using namespace llvm;
-using namespace llvm::legacy;
 
 // See PassManagers.h for Pass Manager infrastructure overview.
 
@@ -387,8 +386,68 @@ class FunctionPassManagerImpl : public Pass,
 void FunctionPassManagerImpl::anchor() {}
 
 char FunctionPassManagerImpl::ID = 0;
-} // End of legacy namespace
-} // End of llvm namespace
+
+//===----------------------------------------------------------------------===//
+// FunctionPassManagerImpl implementation
+//
+bool FunctionPassManagerImpl::doInitialization(Module &M) {
+  bool Changed = false;
+
+  dumpArguments();
+  dumpPasses();
+
+  for (ImmutablePass *ImPass : getImmutablePasses())
+    Changed |= ImPass->doInitialization(M);
+
+  for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index)
+    Changed |= getContainedManager(Index)->doInitialization(M);
+
+  return Changed;
+}
+
+bool FunctionPassManagerImpl::doFinalization(Module &M) {
+  bool Changed = false;
+
+  for (int Index = getNumContainedManagers() - 1; Index >= 0; --Index)
+    Changed |= getContainedManager(Index)->doFinalization(M);
+
+  for (ImmutablePass *ImPass : getImmutablePasses())
+    Changed |= ImPass->doFinalization(M);
+
+  return Changed;
+}
+
+void FunctionPassManagerImpl::releaseMemoryOnTheFly() {
+  if (!wasRun)
+    return;
+  for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) {
+    FPPassManager *FPPM = getContainedManager(Index);
+    for (unsigned Index = 0; Index < FPPM->getNumContainedPasses(); ++Index) {
+      FPPM->getContainedPass(Index)->releaseMemory();
+    }
+  }
+  wasRun = false;
+}
+
+// Execute all the passes managed by this top level manager.
+// Return true if any function is modified by a pass.
+bool FunctionPassManagerImpl::run(Function &F) {
+  bool Changed = false;
+
+  initializeAllAnalysisInfo();
+  for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) {
+    Changed |= getContainedManager(Index)->runOnFunction(F);
+    F.getContext().yield();
+  }
+
+  for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index)
+    getContainedManager(Index)->cleanup();
+
+  wasRun = true;
+  return Changed;
+}
+} // namespace legacy
+} // namespace llvm
 
 namespace {
 //===----------------------------------------------------------------------===//
@@ -406,7 +465,7 @@ class MPPassManager : public Pass, public PMDataManager {
   // Delete on the fly managers.
   ~MPPassManager() override {
     for (auto &OnTheFlyManager : OnTheFlyManagers) {
-      FunctionPassManagerImpl *FPP = OnTheFlyManager.second;
+      legacy::FunctionPassManagerImpl *FPP = OnTheFlyManager.second;
       delete FPP;
     }
   }
@@ -437,7 +496,8 @@ class MPPassManager : public Pass, public PMDataManager {
   /// Return function pass corresponding to PassInfo PI, that is
   /// required by module pass MP. Instantiate analysis pass, by using
   /// its runOnFunction() for function F.
-  Pass* getOnTheFlyPass(Pass *MP, AnalysisID PI, Function &F) override;
+  std::tuple<Pass *, bool> getOnTheFlyPass(Pass *MP, AnalysisID PI,
+                                           Function &F) override;
 
   StringRef getPassName() const override { return "Module Pass Manager"; }
 
@@ -450,7 +510,7 @@ class MPPassManager : public Pass, public PMDataManager {
     for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) {
       ModulePass *MP = getContainedPass(Index);
       MP->dumpPassStructure(Offset + 1);
-      MapVector<Pass *, FunctionPassManagerImpl *>::const_iterator I =
+      MapVector<Pass *, legacy::FunctionPassManagerImpl *>::const_iterator I =
           OnTheFlyManagers.find(MP);
       if (I != OnTheFlyManagers.end())
         I->second->dumpPassStructure(Offset + 2);
@@ -470,7 +530,7 @@ class MPPassManager : public Pass, public PMDataManager {
  private:
   /// Collection of on the fly FPPassManagers. These managers manage
   /// function passes that are required by module passes.
-   MapVector<Pass *, FunctionPassManagerImpl *> OnTheFlyManagers;
+   MapVector<Pass *, legacy::FunctionPassManagerImpl *> OnTheFlyManagers;
 };
 
 char MPPassManager::ID = 0;
@@ -533,8 +593,35 @@ class PassManagerImpl : public Pass,
 void PassManagerImpl::anchor() {}
 
 char PassManagerImpl::ID = 0;
-} // End of legacy namespace
-} // End of llvm namespace
+
+//===----------------------------------------------------------------------===//
+// PassManagerImpl implementation
+
+//
+/// run - Execute all of the passes scheduled for execution.  Keep track of
+/// whether any of the passes modifies the module, and if so, return true.
+bool PassManagerImpl::run(Module &M) {
+  bool Changed = false;
+
+  dumpArguments();
+  dumpPasses();
+
+  for (ImmutablePass *ImPass : getImmutablePasses())
+    Changed |= ImPass->doInitialization(M);
+
+  initializeAllAnalysisInfo();
+  for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) {
+    Changed |= getContainedManager(Index)->runOnModule(M);
+    M.getContext().yield();
+  }
+
+  for (ImmutablePass *ImPass : getImmutablePasses())
+    Changed |= ImPass->doFinalization(M);
+
+  return Changed;
+}
+} // namespace legacy
+} // namespace llvm
 
 //===----------------------------------------------------------------------===//
 // PMTopLevelManager implementation
@@ -1290,7 +1377,8 @@ void PMDataManager::addLowerLevelRequiredPass(Pass *P, Pass *RequiredPass) {
   llvm_unreachable("Unable to schedule pass");
 }
 
-Pass *PMDataManager::getOnTheFlyPass(Pass *P, AnalysisID PI, Function &F) {
+std::tuple<Pass *, bool> PMDataManager::getOnTheFlyPass(Pass *P, AnalysisID PI,
+                                                        Function &F) {
   llvm_unreachable("Unable to find on the fly pass");
 }
 
@@ -1307,17 +1395,20 @@ Pass *AnalysisResolver::getAnalysisIfAvailable(AnalysisID ID, bool dir) const {
   return PM.findAnalysisPass(ID, dir);
 }
 
-Pass *AnalysisResolver::findImplPass(Pass *P, AnalysisID AnalysisPI,
-                                     Function &F) {
+std::tuple<Pass *, bool>
+AnalysisResolver::findImplPass(Pass *P, AnalysisID AnalysisPI, Function &F) {
   return PM.getOnTheFlyPass(P, AnalysisPI, F);
 }
 
+namespace llvm {
+namespace legacy {
+
 //===----------------------------------------------------------------------===//
 // FunctionPassManager implementation
 
 /// Create new Function pass manager
 FunctionPassManager::FunctionPassManager(Module *m) : M(m) {
-  FPM = new FunctionPassManagerImpl();
+  FPM = new legacy::FunctionPassManagerImpl();
   // FPM is the top level manager.
   FPM->setTopLevelManager(FPM);
 
@@ -1356,36 +1447,8 @@ bool FunctionPassManager::doInitialization() {
 bool FunctionPassManager::doFinalization() {
   return FPM->doFinalization(*M);
 }
-
-//===----------------------------------------------------------------------===//
-// FunctionPassManagerImpl implementation
-//
-bool FunctionPassManagerImpl::doInitialization(Module &M) {
-  bool Changed = false;
-
-  dumpArguments();
-  dumpPasses();
-
-  for (ImmutablePass *ImPass : getImmutablePasses())
-    Changed |= ImPass->doInitialization(M);
-
-  for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index)
-    Changed |= getContainedManager(Index)->doInitialization(M);
-
-  return Changed;
-}
-
-bool FunctionPassManagerImpl::doFinalization(Module &M) {
-  bool Changed = false;
-
-  for (int Index = getNumContainedManagers() - 1; Index >= 0; --Index)
-    Changed |= getContainedManager(Index)->doFinalization(M);
-
-  for (ImmutablePass *ImPass : getImmutablePasses())
-    Changed |= ImPass->doFinalization(M);
-
-  return Changed;
-}
+} // namespace legacy
+} // namespace llvm
 
 /// cleanup - After running all passes, clean up pass manager cache.
 void FPPassManager::cleanup() {
@@ -1397,35 +1460,6 @@ void FPPassManager::cleanup() {
  }
 }
 
-void FunctionPassManagerImpl::releaseMemoryOnTheFly() {
-  if (!wasRun)
-    return;
-  for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) {
-    FPPassManager *FPPM = getContainedManager(Index);
-    for (unsigned Index = 0; Index < FPPM->getNumContainedPasses(); ++Index) {
-      FPPM->getContainedPass(Index)->releaseMemory();
-    }
-  }
-  wasRun = false;
-}
-
-// Execute all the passes managed by this top level manager.
-// Return true if any function is modified by a pass.
-bool FunctionPassManagerImpl::run(Function &F) {
-  bool Changed = false;
-
-  initializeAllAnalysisInfo();
-  for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) {
-    Changed |= getContainedManager(Index)->runOnFunction(F);
-    F.getContext().yield();
-  }
-
-  for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index)
-    getContainedManager(Index)->cleanup();
-
-  wasRun = true;
-  return Changed;
-}
 
 //===----------------------------------------------------------------------===//
 // FPPassManager implementation
@@ -1552,7 +1586,7 @@ MPPassManager::runOnModule(Module &M) {
 
   // Initialize on-the-fly passes
   for (auto &OnTheFlyManager : OnTheFlyManagers) {
-    FunctionPassManagerImpl *FPP = OnTheFlyManager.second;
+    legacy::FunctionPassManagerImpl *FPP = OnTheFlyManager.second;
     Changed |= FPP->doInitialization(M);
   }
 
@@ -1613,7 +1647,7 @@ MPPassManager::runOnModule(Module &M) {
 
   // Finalize on-the-fly passes
   for (auto &OnTheFlyManager : OnTheFlyManagers) {
-    FunctionPassManagerImpl *FPP = OnTheFlyManager.second;
+    legacy::FunctionPassManagerImpl *FPP = OnTheFlyManager.second;
     // We don't know when is the last time an on-the-fly pass is run,
     // so we need to releaseMemory / finalize here
     FPP->releaseMemoryOnTheFly();
@@ -1634,9 +1668,9 @@ void MPPassManager::addLowerLevelRequiredPass(Pass *P, Pass *RequiredPass) {
           RequiredPass->getPotentialPassManagerType()) &&
          "Unable to handle Pass that requires lower level Analysis pass");
 
-  FunctionPassManagerImpl *FPP = OnTheFlyManagers[P];
+  legacy::FunctionPassManagerImpl *FPP = OnTheFlyManagers[P];
   if (!FPP) {
-    FPP = new FunctionPassManagerImpl();
+    FPP = new legacy::FunctionPassManagerImpl();
     // FPP is the top level manager.
     FPP->setTopLevelManager(FPP);
 
@@ -1665,42 +1699,19 @@ void MPPassManager::addLowerLevelRequiredPass(Pass *P, Pass *RequiredPass) {
 /// Return function pass corresponding to PassInfo PI, that is
 /// required by module pass MP. Instantiate analysis pass, by using
 /// its runOnFunction() for function F.
-Pass* MPPassManager::getOnTheFlyPass(Pass *MP, AnalysisID PI, Function &F){
-  FunctionPassManagerImpl *FPP = OnTheFlyManagers[MP];
+std::tuple<Pass *, bool> MPPassManager::getOnTheFlyPass(Pass *MP, AnalysisID PI,
+                                                        Function &F) {
+  legacy::FunctionPassManagerImpl *FPP = OnTheFlyManagers[MP];
   assert(FPP && "Unable to find on the fly pass");
 
   FPP->releaseMemoryOnTheFly();
-  FPP->run(F);
-  return ((PMTopLevelManager*)FPP)->findAnalysisPass(PI);
+  bool Changed = FPP->run(F);
+  return std::make_tuple(((PMTopLevelManager *)FPP)->findAnalysisPass(PI),
+                         Changed);
 }
 
-
-//===----------------------------------------------------------------------===//
-// PassManagerImpl implementation
-
-//
-/// run - Execute all of the passes scheduled for execution.  Keep track of
-/// whether any of the passes modifies the module, and if so, return true.
-bool PassManagerImpl::run(Module &M) {
-  bool Changed = false;
-
-  dumpArguments();
-  dumpPasses();
-
-  for (ImmutablePass *ImPass : getImmutablePasses())
-    Changed |= ImPass->doInitialization(M);
-
-  initializeAllAnalysisInfo();
-  for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) {
-    Changed |= getContainedManager(Index)->runOnModule(M);
-    M.getContext().yield();
-  }
-
-  for (ImmutablePass *ImPass : getImmutablePasses())
-    Changed |= ImPass->doFinalization(M);
-
-  return Changed;
-}
+namespace llvm {
+namespace legacy {
 
 //===----------------------------------------------------------------------===//
 // PassManager implementation
@@ -1725,6 +1736,8 @@ void PassManager::add(Pass *P) {
 bool PassManager::run(Module &M) {
   return PM->run(M);
 }
+} // namespace legacy
+} // namespace llvm
 
 //===----------------------------------------------------------------------===//
 // PMStack implementation
@@ -1815,4 +1828,4 @@ void FunctionPass::assignPassManager(PMStack &PMS,
   PM->add(this);
 }
 
-PassManagerBase::~PassManagerBase() {}
+legacy::PassManagerBase::~PassManagerBase() {}
diff --git a/llvm/lib/IR/Mangler.cpp b/llvm/lib/IR/Mangler.cpp
index ba6ca7abae585..0d66e321c396b 100644
--- a/llvm/lib/IR/Mangler.cpp
+++ b/llvm/lib/IR/Mangler.cpp
@@ -94,15 +94,18 @@ static void addByteCountSuffix(raw_ostream &OS, const Function *F,
                                const DataLayout &DL) {
   // Calculate arguments size total.
   unsigned ArgWords = 0;
+
+  const unsigned PtrSize = DL.getPointerSize();
+
   for (Function::const_arg_iterator AI = F->arg_begin(), AE = F->arg_end();
        AI != AE; ++AI) {
-    Type *Ty = AI->getType();
     // 'Dereference' type in case of byval or inalloca parameter attribute.
-    if (AI->hasPassPointeeByValueAttr())
-      Ty = cast<PointerType>(Ty)->getElementType();
+    uint64_t AllocSize = AI->hasPassPointeeByValueAttr() ?
+      AI->getPassPointeeByValueCopySize(DL) :
+      DL.getTypeAllocSize(AI->getType());
+
     // Size should be aligned to pointer size.
-    unsigned PtrSize = DL.getPointerSize();
-    ArgWords += alignTo(DL.getTypeAllocSize(Ty), PtrSize);
+    ArgWords += alignTo(AllocSize, PtrSize);
   }
 
   OS << '@' << ArgWords;
diff --git a/llvm/lib/IR/Operator.cpp b/llvm/lib/IR/Operator.cpp
index df6f08da62ee0..0f70fc37dee28 100644
--- a/llvm/lib/IR/Operator.cpp
+++ b/llvm/lib/IR/Operator.cpp
@@ -134,4 +134,4 @@ bool GEPOperator::accumulateConstantOffset(
   }
   return true;
 }
-}
+} // namespace llvm
diff --git a/llvm/lib/IR/PassManager.cpp b/llvm/lib/IR/PassManager.cpp
index a8aa1e138d42e..624827ff8cd94 100644
--- a/llvm/lib/IR/PassManager.cpp
+++ b/llvm/lib/IR/PassManager.cpp
@@ -89,7 +89,7 @@ bool FunctionAnalysisManagerModuleProxy::Result::invalidate(
   // Return false to indicate that this result is still a valid proxy.
   return false;
 }
-}
+} // namespace llvm
 
 AnalysisSetKey CFGAnalyses::SetKey;
 
diff --git a/llvm/lib/IR/ProfileSummary.cpp b/llvm/lib/IR/ProfileSummary.cpp
index fbd2fa5ed8a79..ac6bcd9fe3afd 100644
--- a/llvm/lib/IR/ProfileSummary.cpp
+++ b/llvm/lib/IR/ProfileSummary.cpp
@@ -18,6 +18,7 @@
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Type.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Format.h"
 
 using namespace llvm;
 
diff --git a/llvm/lib/IR/SafepointIRVerifier.cpp b/llvm/lib/IR/SafepointIRVerifier.cpp
index 01c0132261595..6bf7caa50a121 100644
--- a/llvm/lib/IR/SafepointIRVerifier.cpp
+++ b/llvm/lib/IR/SafepointIRVerifier.cpp
@@ -208,7 +208,7 @@ PreservedAnalyses SafepointIRVerifierPass::run(Function &F,
   Verify(F, DT, CD);
   return PreservedAnalyses::all();
 }
-}
+} // namespace llvm
 
 namespace {
 
diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp
index bb077c1fd0199..d869a6e07ccad 100644
--- a/llvm/lib/IR/Type.cpp
+++ b/llvm/lib/IR/Type.cpp
@@ -548,6 +548,8 @@ bool StructType::indexValid(const Value *V) const {
   // vector case all of the indices must be equal.
   if (!V->getType()->isIntOrIntVectorTy(32))
     return false;
+  if (isa<ScalableVectorType>(V->getType()))
+    return false;
   const Constant *C = dyn_cast<Constant>(V);
   if (C && V->getType()->isVectorTy())
     C = C->getSplatValue();
diff --git a/llvm/lib/IR/Use.cpp b/llvm/lib/IR/Use.cpp
index 67ce3d26f2b82..dc0716b85372a 100644
--- a/llvm/lib/IR/Use.cpp
+++ b/llvm/lib/IR/Use.cpp
@@ -48,4 +48,4 @@ void Use::zap(Use *Start, const Use *Stop, bool del) {
     ::operator delete(Start);
 }
 
-} // End llvm namespace
+} // namespace llvm
diff --git a/llvm/lib/IR/User.cpp b/llvm/lib/IR/User.cpp
index ab7208c318aa8..7da592f401273 100644
--- a/llvm/lib/IR/User.cpp
+++ b/llvm/lib/IR/User.cpp
@@ -195,4 +195,4 @@ LLVM_NO_SANITIZE_MEMORY_ATTRIBUTE void User::operator delete(void *Usr) {
   }
 }
 
-} // End llvm namespace
+} // namespace llvm
diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp
index 0fd147c3a44dc..efb8d53e8964b 100644
--- a/llvm/lib/IR/Value.cpp
+++ b/llvm/lib/IR/Value.cpp
@@ -111,6 +111,10 @@ void Value::deleteValue() {
     static_cast<DerivedUser *>(this)->DeleteValue(                             \
         static_cast<DerivedUser *>(this));                                     \
     break;
+#define HANDLE_CONSTANT(Name)                                                  \
+  case Value::Name##Val:                                                       \
+    llvm_unreachable("constants should be destroyed with destroyConstant");    \
+    break;
 #define HANDLE_INSTRUCTION(Name)  /* nothing */
 #include "llvm/IR/Value.def"
 
@@ -552,6 +556,8 @@ static const Value *stripPointerCastsAndOffsets(
       V = GEP->getPointerOperand();
     } else if (Operator::getOpcode(V) == Instruction::BitCast) {
       V = cast<Operator>(V)->getOperand(0);
+      if (!V->getType()->isPointerTy())
+        return V;
     } else if (StripKind != PSK_ZeroIndicesSameRepresentation &&
                Operator::getOpcode(V) == Instruction::AddrSpaceCast) {
       // TODO: If we know an address space cast will not change the
@@ -766,7 +772,7 @@ Align Value::getPointerAlignment(const DataLayout &DL) const {
           // it the preferred alignment. Otherwise, we have to assume that it
           // may only have the minimum ABI alignment.
           if (GVar->isStrongDefinitionForLinker())
-            return Align(DL.getPreferredAlignment(GVar));
+            return DL.getPreferredAlign(GVar);
           else
             return DL.getABITypeAlign(ObjectType);
         }
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index ee4bb72cee173..8fa87b7489013 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -570,8 +570,9 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) {
   Assert(!GV.isDeclaration() || GV.hasValidDeclarationLinkage(),
          "Global is external, but doesn't have external or weak linkage!", &GV);
 
-  Assert(GV.getAlignment() <= Value::MaximumAlignment,
-         "huge alignment values are unsupported", &GV);
+  if (const GlobalObject *GO = dyn_cast<GlobalObject>(&GV))
+    Assert(GO->getAlignment() <= Value::MaximumAlignment,
+           "huge alignment values are unsupported", GO);
   Assert(!GV.hasAppendingLinkage() || isa<GlobalVariable>(GV),
          "Only global variables can have appending linkage!", &GV);
 
@@ -4565,6 +4566,9 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
                "llvm.call.preallocated.alloc arg index must be between 0 and "
                "corresponding "
                "llvm.call.preallocated.setup's argument count");
+      } else if (Fn && Fn->getIntrinsicID() ==
+                           Intrinsic::call_preallocated_teardown) {
+        // nothing to do
       } else {
         Assert(!FoundCall, "Can have at most one call corresponding to a "
                            "llvm.call.preallocated.setup");
@@ -4613,6 +4617,14 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
            "call site attribute");
     break;
   }
+  case Intrinsic::call_preallocated_teardown: {
+    auto *Token = dyn_cast<CallBase>(Call.getArgOperand(0));
+    Assert(Token && Token->getCalledFunction()->getIntrinsicID() ==
+                        Intrinsic::call_preallocated_setup,
+           "llvm.call.preallocated.teardown token argument must be a "
+           "llvm.call.preallocated.setup");
+    break;
+  }
   case Intrinsic::gcroot:
   case Intrinsic::gcwrite:
   case Intrinsic::gcread:
diff --git a/llvm/lib/LTO/LTOCodeGenerator.cpp b/llvm/lib/LTO/LTOCodeGenerator.cpp
index a8a7877f66dad..25ab1404b4e12 100644
--- a/llvm/lib/LTO/LTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/LTOCodeGenerator.cpp
@@ -632,9 +632,9 @@ bool LTOCodeGenerator::compileOptimized(ArrayRef<raw_pwrite_stream *> Out) {
   return true;
 }
 
-void LTOCodeGenerator::setCodeGenDebugOptions(ArrayRef<const char *> Options) {
+void LTOCodeGenerator::setCodeGenDebugOptions(ArrayRef<StringRef> Options) {
   for (StringRef Option : Options)
-    CodegenOptions.push_back(std::string(Option));
+    CodegenOptions.push_back(Option.str());
 }
 
 void LTOCodeGenerator::parseCodeGenDebugOptions() {
diff --git a/llvm/lib/LTO/LTOModule.cpp b/llvm/lib/LTO/LTOModule.cpp
index 0d37e919ff5a8..ebe779aea62ea 100644
--- a/llvm/lib/LTO/LTOModule.cpp
+++ b/llvm/lib/LTO/LTOModule.cpp
@@ -414,9 +414,8 @@ void LTOModule::addDefinedFunctionSymbol(StringRef Name, const Function *F) {
 
 void LTOModule::addDefinedSymbol(StringRef Name, const GlobalValue *def,
                                  bool isFunction) {
-  // set alignment part log2() can have rounding errors
-  uint32_t align = def->getAlignment();
-  uint32_t attr = align ? countTrailingZeros(align) : 0;
+  const GlobalObject *go = dyn_cast<GlobalObject>(def);
+  uint32_t attr = go ? Log2(go->getAlign().valueOrOne()) : 0;
 
   // set permissions part
   if (isFunction) {
diff --git a/llvm/lib/MC/MCAsmInfoXCOFF.cpp b/llvm/lib/MC/MCAsmInfoXCOFF.cpp
index 1531f61da95e7..b5c5bb3ace8ed 100644
--- a/llvm/lib/MC/MCAsmInfoXCOFF.cpp
+++ b/llvm/lib/MC/MCAsmInfoXCOFF.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/MC/MCAsmInfoXCOFF.h"
+#include "llvm/ADT/StringExtras.h"
 
 using namespace llvm;
 
@@ -32,7 +33,6 @@ MCAsmInfoXCOFF::MCAsmInfoXCOFF() {
   COMMDirectiveAlignmentIsInBytes = false;
   LCOMMDirectiveAlignmentType = LCOMM::Log2Alignment;
   HasDotTypeDotSizeDirective = false;
-  SymbolsHaveSMC = true;
   UseIntegratedAssembler = false;
   NeedsFunctionDescriptors = true;
 }
@@ -43,5 +43,8 @@ bool MCAsmInfoXCOFF::isAcceptableChar(char C) const {
   if (C == '[' || C == ']')
     return true;
 
-  return MCAsmInfo::isAcceptableChar(C);
+  // For AIX assembler, symbols may consist of numeric digits,
+  // underscores, periods, uppercase or lowercase letters, or
+  // any combination of these.
+  return isAlnum(C) || C == '_' || C == '.';
 }
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index 9a86895a2fe13..0747ab2372abe 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -28,6 +28,7 @@
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSymbolXCOFF.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/FormattedStream.h"
@@ -174,6 +175,8 @@ class MCAsmStreamer final : public MCStreamer {
   void emitXCOFFSymbolLinkageWithVisibility(MCSymbol *Symbol,
                                             MCSymbolAttr Linakge,
                                             MCSymbolAttr Visibility) override;
+  void emitXCOFFRenameDirective(const MCSymbol *Name,
+                                StringRef Rename) override;
 
   void emitELFSize(MCSymbol *Symbol, const MCExpr *Value) override;
   void emitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
@@ -797,6 +800,11 @@ void MCAsmStreamer::emitXCOFFLocalCommonSymbol(MCSymbol *LabelSym,
 
 void MCAsmStreamer::emitXCOFFSymbolLinkageWithVisibility(
     MCSymbol *Symbol, MCSymbolAttr Linkage, MCSymbolAttr Visibility) {
+  // Print symbol's rename (original name contains invalid character(s)) if
+  // there is one.
+  if (cast<MCSymbolXCOFF>(Symbol)->hasRename())
+    emitXCOFFRenameDirective(Symbol,
+                             cast<MCSymbolXCOFF>(Symbol)->getSymbolTableName());
 
   switch (Linkage) {
   case MCSA_Global:
@@ -808,6 +816,9 @@ void MCAsmStreamer::emitXCOFFSymbolLinkageWithVisibility(
   case MCSA_Extern:
     OS << "\t.extern\t";
     break;
+  case MCSA_LGlobal:
+    OS << "\t.lglobl\t";
+    break;
   default:
     report_fatal_error("unhandled linkage type");
   }
@@ -830,6 +841,22 @@ void MCAsmStreamer::emitXCOFFSymbolLinkageWithVisibility(
   EmitEOL();
 }
 
+void MCAsmStreamer::emitXCOFFRenameDirective(const MCSymbol *Name,
+                                             StringRef Rename) {
+  OS << "\t.rename\t";
+  Name->print(OS, MAI);
+  const char DQ = '"';
+  OS << ',' << DQ;
+  for (char C : Rename) {
+    // To escape a double quote character, the character should be doubled.
+    if (C == DQ)
+      OS << DQ;
+    OS << C;
+  }
+  OS << DQ;
+  EmitEOL();
+}
+
 void MCAsmStreamer::emitELFSize(MCSymbol *Symbol, const MCExpr *Value) {
   assert(MAI->hasDotTypeDotSizeDirective());
   OS << "\t.size\t";
@@ -841,6 +868,12 @@ void MCAsmStreamer::emitELFSize(MCSymbol *Symbol, const MCExpr *Value) {
 
 void MCAsmStreamer::emitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                                      unsigned ByteAlignment) {
+  // Print symbol's rename (original name contains invalid character(s)) if
+  // there is one.
+  MCSymbolXCOFF *XSym = dyn_cast<MCSymbolXCOFF>(Symbol);
+  if (XSym && XSym->hasRename())
+    emitXCOFFRenameDirective(XSym, XSym->getSymbolTableName());
+
   OS << "\t.comm\t";
   Symbol->print(OS, MAI);
   OS << ',' << Size;
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 47b25c5a299b2..3ca8714b7817c 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -397,6 +397,9 @@ void MCAsmLayout::layoutFragment(MCFragment *F) {
   assert((!Prev || isFragmentValid(Prev)) &&
          "Attempt to compute fragment before its predecessor!");
 
+  assert(!F->IsBeingLaidOut && "Already being laid out!");
+  F->IsBeingLaidOut = true;
+
   ++stats::FragmentLayouts;
 
   // Compute fragment offset and size.
@@ -404,6 +407,7 @@ void MCAsmLayout::layoutFragment(MCFragment *F) {
     F->Offset = Prev->Offset + getAssembler().computeFragmentSize(*this, *Prev);
   else
     F->Offset = 0;
+  F->IsBeingLaidOut = false;
   LastValidFragment[F->getParent()] = F;
 
   // If bundling is enabled and this fragment has instructions in it, it has to
@@ -816,48 +820,57 @@ void MCAssembler::layout(MCAsmLayout &Layout) {
   // Evaluate and apply the fixups, generating relocation entries as necessary.
   for (MCSection &Sec : *this) {
     for (MCFragment &Frag : Sec) {
-      // Data and relaxable fragments both have fixups.  So only process
-      // those here.
-      // FIXME: Is there a better way to do this?  MCEncodedFragmentWithFixups
-      // being templated makes this tricky.
-      if (isa<MCEncodedFragment>(&Frag) &&
-          isa<MCCompactEncodedInstFragment>(&Frag))
-        continue;
-      if (!isa<MCEncodedFragment>(&Frag) && !isa<MCCVDefRangeFragment>(&Frag) &&
-          !isa<MCAlignFragment>(&Frag))
-        continue;
       ArrayRef<MCFixup> Fixups;
       MutableArrayRef<char> Contents;
       const MCSubtargetInfo *STI = nullptr;
-      if (auto *FragWithFixups = dyn_cast<MCDataFragment>(&Frag)) {
-        Fixups = FragWithFixups->getFixups();
-        Contents = FragWithFixups->getContents();
-        STI = FragWithFixups->getSubtargetInfo();
-        assert(!FragWithFixups->hasInstructions() || STI != nullptr);
-      } else if (auto *FragWithFixups = dyn_cast<MCRelaxableFragment>(&Frag)) {
-        Fixups = FragWithFixups->getFixups();
-        Contents = FragWithFixups->getContents();
-        STI = FragWithFixups->getSubtargetInfo();
-        assert(!FragWithFixups->hasInstructions() || STI != nullptr);
-      } else if (auto *FragWithFixups = dyn_cast<MCCVDefRangeFragment>(&Frag)) {
-        Fixups = FragWithFixups->getFixups();
-        Contents = FragWithFixups->getContents();
-      } else if (auto *FragWithFixups = dyn_cast<MCDwarfLineAddrFragment>(&Frag)) {
-        Fixups = FragWithFixups->getFixups();
-        Contents = FragWithFixups->getContents();
-      } else if (auto *AF = dyn_cast<MCAlignFragment>(&Frag)) {
+
+      // Process MCAlignFragment and MCEncodedFragmentWithFixups here.
+      switch (Frag.getKind()) {
+      default:
+        continue;
+      case MCFragment::FT_Align: {
+        MCAlignFragment &AF = cast<MCAlignFragment>(Frag);
         // Insert fixup type for code alignment if the target define
         // shouldInsertFixupForCodeAlign target hook.
-        if (Sec.UseCodeAlign() && AF->hasEmitNops()) {
-          getBackend().shouldInsertFixupForCodeAlign(*this, Layout, *AF);
-        }
+        if (Sec.UseCodeAlign() && AF.hasEmitNops())
+          getBackend().shouldInsertFixupForCodeAlign(*this, Layout, AF);
         continue;
-      } else if (auto *FragWithFixups =
-                     dyn_cast<MCDwarfCallFrameFragment>(&Frag)) {
-        Fixups = FragWithFixups->getFixups();
-        Contents = FragWithFixups->getContents();
-      } else
-        llvm_unreachable("Unknown fragment with fixups!");
+      }
+      case MCFragment::FT_Data: {
+        MCDataFragment &DF = cast<MCDataFragment>(Frag);
+        Fixups = DF.getFixups();
+        Contents = DF.getContents();
+        STI = DF.getSubtargetInfo();
+        assert(!DF.hasInstructions() || STI != nullptr);
+        break;
+      }
+      case MCFragment::FT_Relaxable: {
+        MCRelaxableFragment &RF = cast<MCRelaxableFragment>(Frag);
+        Fixups = RF.getFixups();
+        Contents = RF.getContents();
+        STI = RF.getSubtargetInfo();
+        assert(!RF.hasInstructions() || STI != nullptr);
+        break;
+      }
+      case MCFragment::FT_CVDefRange: {
+        MCCVDefRangeFragment &CF = cast<MCCVDefRangeFragment>(Frag);
+        Fixups = CF.getFixups();
+        Contents = CF.getContents();
+        break;
+      }
+      case MCFragment::FT_Dwarf: {
+        MCDwarfLineAddrFragment &DF = cast<MCDwarfLineAddrFragment>(Frag);
+        Fixups = DF.getFixups();
+        Contents = DF.getContents();
+        break;
+      }
+      case MCFragment::FT_DwarfFrame: {
+        MCDwarfCallFrameFragment &DF = cast<MCDwarfCallFrameFragment>(Frag);
+        Fixups = DF.getFixups();
+        Contents = DF.getContents();
+        break;
+      }
+      }
       for (const MCFixup &Fixup : Fixups) {
         uint64_t FixedValue;
         bool IsResolved;
diff --git a/llvm/lib/MC/MCContext.cpp b/llvm/lib/MC/MCContext.cpp
index 92d99ec577b5a..a0f9212f3b142 100644
--- a/llvm/lib/MC/MCContext.cpp
+++ b/llvm/lib/MC/MCContext.cpp
@@ -182,7 +182,7 @@ MCSymbol *MCContext::createSymbolImpl(const StringMapEntry<bool> *Name,
     case MCObjectFileInfo::IsWasm:
       return new (Name, *this) MCSymbolWasm(Name, IsTemporary);
     case MCObjectFileInfo::IsXCOFF:
-      return new (Name, *this) MCSymbolXCOFF(Name, IsTemporary);
+      return createXCOFFSymbolImpl(Name, IsTemporary);
     }
   }
   return new (Name, *this) MCSymbol(MCSymbol::SymbolKindUnset, Name,
@@ -292,6 +292,61 @@ void MCContext::registerInlineAsmLabel(MCSymbol *Sym) {
   InlineAsmUsedLabelNames[Sym->getName()] = Sym;
 }
 
+MCSymbolXCOFF *
+MCContext::createXCOFFSymbolImpl(const StringMapEntry<bool> *Name,
+                                 bool IsTemporary) {
+  if (!Name)
+    return new (nullptr, *this) MCSymbolXCOFF(nullptr, IsTemporary);
+
+  StringRef OriginalName = Name->first();
+  if (OriginalName.startswith("._Renamed..") ||
+      OriginalName.startswith("_Renamed.."))
+    reportError(SMLoc(), "invalid symbol name from source");
+
+  if (MAI->isValidUnquotedName(OriginalName))
+    return new (Name, *this) MCSymbolXCOFF(Name, IsTemporary);
+
+  // Now we have a name that contains invalid character(s) for XCOFF symbol.
+  // Let's replace with something valid, but save the original name so that
+  // we could still use the original name in the symbol table.
+  SmallString<128> InvalidName(OriginalName);
+
+  // If it's an entry point symbol, we will keep the '.'
+  // in front for the convention purpose. Otherwise, add "_Renamed.."
+  // as prefix to signal this is an renamed symbol.
+  const bool IsEntryPoint = !InvalidName.empty() && InvalidName[0] == '.';
+  SmallString<128> ValidName =
+      StringRef(IsEntryPoint ? "._Renamed.." : "_Renamed..");
+
+  // Append the hex values of '_' and invalid characters with "_Renamed..";
+  // at the same time replace invalid characters with '_'.
+  for (size_t I = 0; I < InvalidName.size(); ++I) {
+    if (!MAI->isAcceptableChar(InvalidName[I]) || InvalidName[I] == '_') {
+      raw_svector_ostream(ValidName).write_hex(InvalidName[I]);
+      InvalidName[I] = '_';
+    }
+  }
+
+  // Skip entry point symbol's '.' as we already have a '.' in front of
+  // "_Renamed".
+  if (IsEntryPoint)
+    ValidName.append(InvalidName.substr(1, InvalidName.size() - 1));
+  else
+    ValidName.append(InvalidName);
+
+  auto NameEntry = UsedNames.insert(std::make_pair(ValidName, true));
+  assert((NameEntry.second || !NameEntry.first->second) &&
+         "This name is used somewhere else.");
+  // Mark the name as used for a non-section symbol.
+  NameEntry.first->second = true;
+  // Have the MCSymbol object itself refer to the copy of the string
+  // that is embedded in the UsedNames entry.
+  MCSymbolXCOFF *XSym = new (&*NameEntry.first, *this)
+      MCSymbolXCOFF(&*NameEntry.first, IsTemporary);
+  XSym->setSymbolTableName(MCSymbolXCOFF::getUnqualifiedName(OriginalName));
+  return XSym;
+}
+
 //===----------------------------------------------------------------------===//
 // Section Management
 //===----------------------------------------------------------------------===//
@@ -610,15 +665,18 @@ MCSectionXCOFF *MCContext::getXCOFFSection(StringRef Section,
 
   // Otherwise, return a new section.
   StringRef CachedName = Entry.first.SectionName;
-  MCSymbol *QualName = getOrCreateSymbol(
-      CachedName + "[" + XCOFF::getMappingClassString(SMC) + "]");
+  MCSymbolXCOFF *QualName = cast<MCSymbolXCOFF>(getOrCreateSymbol(
+      CachedName + "[" + XCOFF::getMappingClassString(SMC) + "]"));
 
   MCSymbol *Begin = nullptr;
   if (BeginSymName)
     Begin = createTempSymbol(BeginSymName, false);
 
-  MCSectionXCOFF *Result = new (XCOFFAllocator.Allocate()) MCSectionXCOFF(
-      CachedName, SMC, Type, SC, Kind, cast<MCSymbolXCOFF>(QualName), Begin);
+  // QualName->getUnqualifiedName() and CachedName are the same except when
+  // CachedName contains invalid character(s) such as '$' for an XCOFF symbol.
+  MCSectionXCOFF *Result = new (XCOFFAllocator.Allocate())
+      MCSectionXCOFF(QualName->getUnqualifiedName(), SMC, Type, SC, Kind,
+                     QualName, Begin, CachedName);
   Entry.second = Result;
 
   auto *F = new MCDataFragment();
diff --git a/llvm/lib/MC/MCDisassembler/MCDisassembler.cpp b/llvm/lib/MC/MCDisassembler/MCDisassembler.cpp
index 9cdacb64c4f47..a58e8f6d9bcc3 100644
--- a/llvm/lib/MC/MCDisassembler/MCDisassembler.cpp
+++ b/llvm/lib/MC/MCDisassembler/MCDisassembler.cpp
@@ -47,7 +47,7 @@ void MCDisassembler::setSymbolizer(std::unique_ptr<MCSymbolizer> Symzer) {
   case XCOFF::XMC_##A:                                                         \
     return P;
 
-uint8_t getSMCPriority(XCOFF::StorageMappingClass SMC) {
+static uint8_t getSMCPriority(XCOFF::StorageMappingClass SMC) {
   switch (SMC) {
     SMC_PCASE(PR, 1)
     SMC_PCASE(RO, 1)
diff --git a/llvm/lib/MC/MCExpr.cpp b/llvm/lib/MC/MCExpr.cpp
index 429531165585e..ecf63b10f73fc 100644
--- a/llvm/lib/MC/MCExpr.cpp
+++ b/llvm/lib/MC/MCExpr.cpp
@@ -342,6 +342,20 @@ StringRef MCSymbolRefExpr::getVariantKindName(VariantKind Kind) {
   case VK_AMDGPU_REL64: return "rel64";
   case VK_AMDGPU_ABS32_LO: return "abs32@lo";
   case VK_AMDGPU_ABS32_HI: return "abs32@hi";
+  case VK_VE_HI32: return "hi";
+  case VK_VE_LO32: return "lo";
+  case VK_VE_PC_HI32: return "pc_hi";
+  case VK_VE_PC_LO32: return "pc_lo";
+  case VK_VE_GOT_HI32: return "got_hi";
+  case VK_VE_GOT_LO32: return "got_lo";
+  case VK_VE_GOTOFF_HI32: return "gotoff_hi";
+  case VK_VE_GOTOFF_LO32: return "gotoff_lo";
+  case VK_VE_PLT_HI32: return "plt_hi";
+  case VK_VE_PLT_LO32: return "plt_lo";
+  case VK_VE_TLS_GD_HI32: return "tls_gd_hi";
+  case VK_VE_TLS_GD_LO32: return "tls_gd_lo";
+  case VK_VE_TPOFF_HI32: return "tpoff_hi";
+  case VK_VE_TPOFF_LO32: return "tpoff_lo";
   }
   llvm_unreachable("Invalid variant kind");
 }
@@ -463,6 +477,20 @@ MCSymbolRefExpr::getVariantKindForName(StringRef Name) {
     .Case("rel64", VK_AMDGPU_REL64)
     .Case("abs32@lo", VK_AMDGPU_ABS32_LO)
     .Case("abs32@hi", VK_AMDGPU_ABS32_HI)
+    .Case("hi", VK_VE_HI32)
+    .Case("lo", VK_VE_LO32)
+    .Case("pc_hi", VK_VE_PC_HI32)
+    .Case("pc_lo", VK_VE_PC_LO32)
+    .Case("got_hi", VK_VE_GOT_HI32)
+    .Case("got_lo", VK_VE_GOT_LO32)
+    .Case("gotoff_hi", VK_VE_GOTOFF_HI32)
+    .Case("gotoff_lo", VK_VE_GOTOFF_LO32)
+    .Case("plt_hi", VK_VE_PLT_HI32)
+    .Case("plt_lo", VK_VE_PLT_LO32)
+    .Case("tls_gd_hi", VK_VE_TLS_GD_HI32)
+    .Case("tls_gd_lo", VK_VE_TLS_GD_LO32)
+    .Case("tpoff_hi", VK_VE_TPOFF_HI32)
+    .Case("tpoff_lo", VK_VE_TPOFF_LO32)
     .Default(VK_Invalid);
 }
 
@@ -547,8 +575,10 @@ static void AttemptToFoldSymbolOffsetDifference(
   if (!Asm->getWriter().isSymbolRefDifferenceFullyResolved(*Asm, A, B, InSet))
     return;
 
-  if (SA.getFragment() == SB.getFragment() && !SA.isVariable() &&
-      !SA.isUnset() && !SB.isVariable() && !SB.isUnset()) {
+  MCFragment *FA = SA.getFragment();
+  MCFragment *FB = SB.getFragment();
+  if (FA == FB && !SA.isVariable() && !SA.isUnset() && !SB.isVariable() &&
+      !SB.isUnset()) {
     Addend += (SA.getOffset() - SB.getOffset());
 
     // Pointers to Thumb symbols need to have their low-bit set to allow
@@ -570,12 +600,17 @@ static void AttemptToFoldSymbolOffsetDifference(
   if (!Layout)
     return;
 
-  const MCSection &SecA = *SA.getFragment()->getParent();
-  const MCSection &SecB = *SB.getFragment()->getParent();
+  const MCSection &SecA = *FA->getParent();
+  const MCSection &SecB = *FB->getParent();
 
   if ((&SecA != &SecB) && !Addrs)
     return;
 
+  // One of the symbol involved is part of a fragment being laid out. Quit now
+  // to avoid a self loop.
+  if (!Layout->canGetFragmentOffset(FA) || !Layout->canGetFragmentOffset(FB))
+    return;
+
   // Eagerly evaluate.
   Addend += Layout->getSymbolOffset(A->getSymbol()) -
             Layout->getSymbolOffset(B->getSymbol());
diff --git a/llvm/lib/MC/MCFragment.cpp b/llvm/lib/MC/MCFragment.cpp
index 5252c3f146720..8e90e07a4dbfd 100644
--- a/llvm/lib/MC/MCFragment.cpp
+++ b/llvm/lib/MC/MCFragment.cpp
@@ -48,6 +48,25 @@ bool MCAsmLayout::isFragmentValid(const MCFragment *F) const {
   return F->getLayoutOrder() <= LastValid->getLayoutOrder();
 }
 
+bool MCAsmLayout::canGetFragmentOffset(const MCFragment *F) const {
+  MCSection *Sec = F->getParent();
+  MCSection::iterator I;
+  if (MCFragment *LastValid = LastValidFragment[Sec]) {
+    // Fragment already valid, offset is available.
+    if (F->getLayoutOrder() <= LastValid->getLayoutOrder())
+      return true;
+    I = ++MCSection::iterator(LastValid);
+  } else
+    I = Sec->begin();
+
+  // A fragment ordered before F is currently being laid out.
+  const MCFragment *FirstInvalidFragment = &*I;
+  if (FirstInvalidFragment->IsBeingLaidOut)
+    return false;
+
+  return true;
+}
+
 void MCAsmLayout::invalidateFragmentsFrom(MCFragment *F) {
   // If this fragment wasn't already valid, we don't need to do anything.
   if (!isFragmentValid(F))
@@ -235,7 +254,7 @@ void ilist_alloc_traits<MCFragment>::deleteNode(MCFragment *V) { V->destroy(); }
 MCFragment::MCFragment(FragmentType Kind, bool HasInstructions,
                        MCSection *Parent)
     : Parent(Parent), Atom(nullptr), Offset(~UINT64_C(0)), LayoutOrder(0),
-      Kind(Kind), HasInstructions(HasInstructions) {
+      Kind(Kind), IsBeingLaidOut(false), HasInstructions(HasInstructions) {
   if (Parent && !isa<MCDummyFragment>(*this))
     Parent->getFragmentList().push_back(this);
 }
diff --git a/llvm/lib/MC/MCParser/COFFAsmParser.cpp b/llvm/lib/MC/MCParser/COFFAsmParser.cpp
index 649023b59bbbe..2104fb83b3092 100644
--- a/llvm/lib/MC/MCParser/COFFAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/COFFAsmParser.cpp
@@ -70,6 +70,7 @@ class COFFAsmParser : public MCAsmParserExtension {
     addDirectiveHandler<&COFFAsmParser::ParseDirectiveLinkOnce>(".linkonce");
     addDirectiveHandler<&COFFAsmParser::ParseDirectiveRVA>(".rva");
     addDirectiveHandler<&COFFAsmParser::ParseDirectiveSymbolAttribute>(".weak");
+    addDirectiveHandler<&COFFAsmParser::ParseDirectiveCGProfile>(".cg_profile");
 
     // Win64 EH directives.
     addDirectiveHandler<&COFFAsmParser::ParseSEHDirectiveStartProc>(
@@ -125,6 +126,7 @@ class COFFAsmParser : public MCAsmParserExtension {
   bool parseCOMDATType(COFF::COMDATType &Type);
   bool ParseDirectiveLinkOnce(StringRef, SMLoc);
   bool ParseDirectiveRVA(StringRef, SMLoc);
+  bool ParseDirectiveCGProfile(StringRef, SMLoc);
 
   // Win64 EH directives.
   bool ParseSEHDirectiveStartProc(StringRef, SMLoc);
@@ -299,6 +301,10 @@ bool COFFAsmParser::ParseDirectiveSymbolAttribute(StringRef Directive, SMLoc) {
   return false;
 }
 
+bool COFFAsmParser::ParseDirectiveCGProfile(StringRef S, SMLoc Loc) {
+  return MCAsmParserExtension::ParseDirectiveCGProfile(S, Loc);
+}
+
 bool COFFAsmParser::ParseSectionSwitch(StringRef Section,
                                        unsigned Characteristics,
                                        SectionKind Kind) {
@@ -321,7 +327,7 @@ bool COFFAsmParser::ParseSectionSwitch(StringRef Section,
 }
 
 bool COFFAsmParser::ParseSectionName(StringRef &SectionName) {
-  if (!getLexer().is(AsmToken::Identifier))
+  if (!getLexer().is(AsmToken::Identifier) && !getLexer().is(AsmToken::String))
     return true;
 
   SectionName = getTok().getIdentifier();
diff --git a/llvm/lib/MC/MCParser/ELFAsmParser.cpp b/llvm/lib/MC/MCParser/ELFAsmParser.cpp
index a80e8a5832ef6..e5ab13bc719d4 100644
--- a/llvm/lib/MC/MCParser/ELFAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/ELFAsmParser.cpp
@@ -862,45 +862,8 @@ bool ELFAsmParser::ParseDirectiveSubsection(StringRef, SMLoc) {
   return false;
 }
 
-/// ParseDirectiveCGProfile
-///  ::= .cg_profile identifier, identifier, <number>
-bool ELFAsmParser::ParseDirectiveCGProfile(StringRef, SMLoc) {
-  StringRef From;
-  SMLoc FromLoc = getLexer().getLoc();
-  if (getParser().parseIdentifier(From))
-    return TokError("expected identifier in directive");
-
-  if (getLexer().isNot(AsmToken::Comma))
-    return TokError("expected a comma");
-  Lex();
-
-  StringRef To;
-  SMLoc ToLoc = getLexer().getLoc();
-  if (getParser().parseIdentifier(To))
-    return TokError("expected identifier in directive");
-
-  if (getLexer().isNot(AsmToken::Comma))
-    return TokError("expected a comma");
-  Lex();
-
-  int64_t Count;
-  if (getParser().parseIntToken(
-          Count, "expected integer count in '.cg_profile' directive"))
-    return true;
-
-  if (getLexer().isNot(AsmToken::EndOfStatement))
-    return TokError("unexpected token in directive");
-
-  MCSymbol *FromSym = getContext().getOrCreateSymbol(From);
-  MCSymbol *ToSym = getContext().getOrCreateSymbol(To);
-
-  getStreamer().emitCGProfileEntry(
-      MCSymbolRefExpr::create(FromSym, MCSymbolRefExpr::VK_None, getContext(),
-                              FromLoc),
-      MCSymbolRefExpr::create(ToSym, MCSymbolRefExpr::VK_None, getContext(),
-                              ToLoc),
-      Count);
-  return false;
+bool ELFAsmParser::ParseDirectiveCGProfile(StringRef S, SMLoc Loc) {
+  return MCAsmParserExtension::ParseDirectiveCGProfile(S, Loc);
 }
 
 namespace llvm {
diff --git a/llvm/lib/MC/MCParser/MCAsmParserExtension.cpp b/llvm/lib/MC/MCParser/MCAsmParserExtension.cpp
index 18d18f0cf6ed5..0b5046cd8fad5 100644
--- a/llvm/lib/MC/MCParser/MCAsmParserExtension.cpp
+++ b/llvm/lib/MC/MCParser/MCAsmParserExtension.cpp
@@ -7,6 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCStreamer.h"
 
 using namespace llvm;
 
@@ -17,3 +19,44 @@ MCAsmParserExtension::~MCAsmParserExtension() = default;
 void MCAsmParserExtension::Initialize(MCAsmParser &Parser) {
   this->Parser = &Parser;
 }
+
+/// ParseDirectiveCGProfile
+///  ::= .cg_profile identifier, identifier, <number>
+bool MCAsmParserExtension::ParseDirectiveCGProfile(StringRef, SMLoc) {
+  StringRef From;
+  SMLoc FromLoc = getLexer().getLoc();
+  if (getParser().parseIdentifier(From))
+    return TokError("expected identifier in directive");
+
+  if (getLexer().isNot(AsmToken::Comma))
+    return TokError("expected a comma");
+  Lex();
+
+  StringRef To;
+  SMLoc ToLoc = getLexer().getLoc();
+  if (getParser().parseIdentifier(To))
+    return TokError("expected identifier in directive");
+
+  if (getLexer().isNot(AsmToken::Comma))
+    return TokError("expected a comma");
+  Lex();
+
+  int64_t Count;
+  if (getParser().parseIntToken(
+          Count, "expected integer count in '.cg_profile' directive"))
+    return true;
+
+  if (getLexer().isNot(AsmToken::EndOfStatement))
+    return TokError("unexpected token in directive");
+
+  MCSymbol *FromSym = getContext().getOrCreateSymbol(From);
+  MCSymbol *ToSym = getContext().getOrCreateSymbol(To);
+
+  getStreamer().emitCGProfileEntry(
+      MCSymbolRefExpr::create(FromSym, MCSymbolRefExpr::VK_None, getContext(),
+                              FromLoc),
+      MCSymbolRefExpr::create(ToSym, MCSymbolRefExpr::VK_None, getContext(),
+                              ToLoc),
+      Count);
+  return false;
+}
diff --git a/llvm/lib/MC/MCParser/MasmParser.cpp b/llvm/lib/MC/MCParser/MasmParser.cpp
index ad44fd51712bc..3dbd00aae47a2 100644
--- a/llvm/lib/MC/MCParser/MasmParser.cpp
+++ b/llvm/lib/MC/MCParser/MasmParser.cpp
@@ -80,8 +80,7 @@ namespace {
 typedef std::vector<AsmToken> MCAsmMacroArgument;
 typedef std::vector<MCAsmMacroArgument> MCAsmMacroArguments;
 
-/// Helper class for storing information about an active macro
-/// instantiation.
+/// Helper class for storing information about an active macro instantiation.
 struct MacroInstantiation {
   /// The location of the instantiation.
   SMLoc InstantiationLoc;
@@ -113,6 +112,238 @@ struct ParseStatementInfo {
       : AsmRewrites(rewrites) {}
 };
 
+enum FieldType {
+  FT_INTEGRAL, // Initializer: integer expression, stored as an MCExpr.
+  FT_REAL,     // Initializer: real number, stored as an APInt.
+  FT_STRUCT    // Initializer: struct initializer, stored recursively.
+};
+
+struct FieldInfo;
+struct StructInfo {
+  StringRef Name;
+  bool IsUnion = false;
+  size_t Alignment = 0;
+  size_t Size = 0;
+  std::vector<FieldInfo> Fields;
+  StringMap<size_t> FieldsByName;
+
+  FieldInfo &addField(StringRef FieldName, FieldType FT);
+
+  StructInfo() = default;
+
+  StructInfo(StringRef StructName, bool Union, unsigned AlignmentValue)
+      : Name(StructName), IsUnion(Union), Alignment(AlignmentValue) {}
+};
+
+// FIXME: This should probably use a class hierarchy, raw pointers between the
+// objects, and dynamic type resolution instead of a union. On the other hand,
+// ownership then becomes much more complicated; the obvious thing would be to
+// use BumpPtrAllocator, but the lack of a destructor makes that messy.
+
+struct StructInitializer;
+struct IntFieldInfo {
+  SmallVector<const MCExpr *, 1> Values;
+
+  IntFieldInfo() = default;
+  IntFieldInfo(const SmallVector<const MCExpr *, 1> &V) { Values = V; }
+  IntFieldInfo(SmallVector<const MCExpr *, 1> &&V) { Values = V; }
+};
+struct RealFieldInfo {
+  SmallVector<APInt, 1> AsIntValues;
+
+  RealFieldInfo() = default;
+  RealFieldInfo(const SmallVector<APInt, 1> &V) { AsIntValues = V; }
+  RealFieldInfo(SmallVector<APInt, 1> &&V) { AsIntValues = V; }
+};
+struct StructFieldInfo {
+  std::vector<StructInitializer> Initializers;
+  StructInfo Structure;
+
+  StructFieldInfo() = default;
+  StructFieldInfo(const std::vector<StructInitializer> &V, StructInfo S) {
+    Initializers = V;
+    Structure = S;
+  }
+  StructFieldInfo(std::vector<StructInitializer> &&V, StructInfo S) {
+    Initializers = V;
+    Structure = S;
+  }
+};
+
+class FieldInitializer {
+public:
+  FieldType FT;
+  union {
+    IntFieldInfo IntInfo;
+    RealFieldInfo RealInfo;
+    StructFieldInfo StructInfo;
+  };
+
+  ~FieldInitializer() {
+    switch (FT) {
+    case FT_INTEGRAL:
+      IntInfo.~IntFieldInfo();
+      break;
+    case FT_REAL:
+      RealInfo.~RealFieldInfo();
+      break;
+    case FT_STRUCT:
+      StructInfo.~StructFieldInfo();
+      break;
+    }
+  }
+
+  FieldInitializer(FieldType FT) : FT(FT) {
+    switch (FT) {
+    case FT_INTEGRAL:
+      new (&IntInfo) IntFieldInfo();
+      break;
+    case FT_REAL:
+      new (&RealInfo) RealFieldInfo();
+      break;
+    case FT_STRUCT:
+      new (&StructInfo) StructFieldInfo();
+      break;
+    }
+  }
+
+  FieldInitializer(SmallVector<const MCExpr *, 1> &&Values) : FT(FT_INTEGRAL) {
+    new (&IntInfo) IntFieldInfo(Values);
+  }
+
+  FieldInitializer(SmallVector<APInt, 1> &&AsIntValues) : FT(FT_REAL) {
+    new (&RealInfo) RealFieldInfo(AsIntValues);
+  }
+
+  FieldInitializer(std::vector<StructInitializer> &&Initializers,
+                   struct StructInfo Structure)
+      : FT(FT_STRUCT) {
+    new (&StructInfo) StructFieldInfo(Initializers, Structure);
+  }
+
+  FieldInitializer(const FieldInitializer &Initializer) : FT(Initializer.FT) {
+    switch (FT) {
+    case FT_INTEGRAL:
+      new (&IntInfo) IntFieldInfo(Initializer.IntInfo);
+      break;
+    case FT_REAL:
+      new (&RealInfo) RealFieldInfo(Initializer.RealInfo);
+      break;
+    case FT_STRUCT:
+      new (&StructInfo) StructFieldInfo(Initializer.StructInfo);
+      break;
+    }
+  }
+
+  FieldInitializer(FieldInitializer &&Initializer) : FT(Initializer.FT) {
+    switch (FT) {
+    case FT_INTEGRAL:
+      new (&IntInfo) IntFieldInfo(Initializer.IntInfo);
+      break;
+    case FT_REAL:
+      new (&RealInfo) RealFieldInfo(Initializer.RealInfo);
+      break;
+    case FT_STRUCT:
+      new (&StructInfo) StructFieldInfo(Initializer.StructInfo);
+      break;
+    }
+  }
+
+  FieldInitializer &operator=(const FieldInitializer &Initializer) {
+    if (FT != Initializer.FT) {
+      switch (FT) {
+      case FT_INTEGRAL:
+        IntInfo.~IntFieldInfo();
+        break;
+      case FT_REAL:
+        RealInfo.~RealFieldInfo();
+        break;
+      case FT_STRUCT:
+        StructInfo.~StructFieldInfo();
+        break;
+      }
+    }
+    FT = Initializer.FT;
+    switch (FT) {
+    case FT_INTEGRAL:
+      IntInfo = Initializer.IntInfo;
+      break;
+    case FT_REAL:
+      RealInfo = Initializer.RealInfo;
+      break;
+    case FT_STRUCT:
+      StructInfo = Initializer.StructInfo;
+      break;
+    }
+    return *this;
+  }
+
+  FieldInitializer &operator=(FieldInitializer &&Initializer) {
+    if (FT != Initializer.FT) {
+      switch (FT) {
+      case FT_INTEGRAL:
+        IntInfo.~IntFieldInfo();
+        break;
+      case FT_REAL:
+        RealInfo.~RealFieldInfo();
+        break;
+      case FT_STRUCT:
+        StructInfo.~StructFieldInfo();
+        break;
+      }
+    }
+    FT = Initializer.FT;
+    switch (FT) {
+    case FT_INTEGRAL:
+      IntInfo = Initializer.IntInfo;
+      break;
+    case FT_REAL:
+      RealInfo = Initializer.RealInfo;
+      break;
+    case FT_STRUCT:
+      StructInfo = Initializer.StructInfo;
+      break;
+    }
+    return *this;
+  }
+};
+
+struct StructInitializer {
+  std::vector<FieldInitializer> FieldInitializers;
+};
+
+struct FieldInfo {
+  // Offset of the field within the containing STRUCT.
+  size_t Offset = 0;
+
+  // Total size of the field (= LengthOf * Type).
+  size_t SizeOf = 0;
+
+  // Number of elements in the field (1 if scalar, >1 if an array).
+  size_t LengthOf = 0;
+
+  // Size of a single entry in this field, in bytes ("type" in MASM standards).
+  size_t Type = 0;
+
+  FieldInitializer Contents;
+
+  FieldInfo(FieldType FT) : Contents(FT) {}
+};
+
+FieldInfo &StructInfo::addField(StringRef FieldName, FieldType FT) {
+  if (!FieldName.empty())
+    FieldsByName[FieldName] = Fields.size();
+  Fields.emplace_back(FT);
+  FieldInfo &Field = Fields.back();
+  if (IsUnion) {
+    Field.Offset = 0;
+  } else {
+    Size = llvm::alignTo(Size, Alignment);
+    Field.Offset = Size;
+  }
+  return Field;
+}
+
 /// The concrete assembly parser instance.
 // Note that this is a full MCAsmParser, not an MCAsmParserExtension!
 // It's a peer of AsmParser, not of COFFAsmParser, WasmAsmParser, etc.
@@ -139,7 +370,7 @@ class MasmParser : public MCAsmParser {
   /// addDirectiveHandler.
   StringMap<ExtensionDirectiveHandler> ExtensionDirectiveMap;
 
-  /// maps assembly-time variable names to variables
+  /// maps assembly-time variable names to variables.
   struct Variable {
     StringRef Name;
     bool Redefinable = true;
@@ -149,6 +380,15 @@ class MasmParser : public MCAsmParser {
   };
   StringMap<Variable> Variables;
 
+  /// Stack of active struct definitions.
+  SmallVector<StructInfo, 1> StructInProgress;
+
+  /// Maps struct tags to struct definitions.
+  StringMap<StructInfo> Structs;
+
+  /// Maps data location names to user-defined types.
+  StringMap<const StructInfo *> KnownType;
+
   /// Stack of active macro instantiations.
   std::vector<MacroInstantiation*> ActiveMacros;
 
@@ -190,6 +430,9 @@ class MasmParser : public MCAsmParser {
   // Is alt macro mode enabled.
   bool AltMacroMode = false;
 
+  // Current <...> expression depth.
+  unsigned AngleBracketDepth = 0U;
+
 public:
   MasmParser(SourceMgr &SM, MCContext &Ctx, MCStreamer &Out,
              const MCAsmInfo &MAI, unsigned CB);
@@ -245,6 +488,11 @@ class MasmParser : public MCAsmParser {
   }
   bool isParsingMSInlineAsm() override { return ParsingMSInlineAsm; }
 
+  bool isParsingMasm() const override { return true; }
+
+  bool LookUpFieldOffset(StringRef Base, StringRef Member,
+                         unsigned &Offset) override;
+
   bool parseMSInlineAsm(void *AsmLoc, std::string &AsmString,
                         unsigned &NumOutputs, unsigned &NumInputs,
                         SmallVectorImpl<std::pair<void *,bool>> &OpDecls,
@@ -313,6 +561,9 @@ class MasmParser : public MCAsmParser {
   }
   static void DiagHandler(const SMDiagnostic &Diag, void *Context);
 
+  bool LookUpFieldOffset(const StructInfo &Structure, StringRef Member,
+                         unsigned &Offset);
+
   /// Should we emit DWARF describing this assembler source?  (Returns false if
   /// the source has .file directives, which means we don't want to generate
   /// info describing the assembler source itself.)
@@ -462,11 +713,14 @@ class MasmParser : public MCAsmParser {
     DK_ERRE,
     DK_ERRNZ,
     DK_ECHO,
+    DK_STRUCT,
+    DK_UNION,
+    DK_ENDS,
     DK_END
   };
 
-  /// Maps directive name --> DirectiveKind enum, for
-  /// directives parsed by this class.
+  /// Maps directive name --> DirectiveKind enum, for directives parsed by this
+  /// class.
   StringMap<DirectiveKind> DirectiveKindMap;
 
   // Codeview def_range type parsing.
@@ -478,8 +732,8 @@ class MasmParser : public MCAsmParser {
     CVDR_DEFRANGE_REGISTER_REL
   };
 
-  /// Maps Codeview def_range types --> CVDefRangeType enum, for
-  /// Codeview def_range types parsed by this class.
+  /// Maps Codeview def_range types --> CVDefRangeType enum, for Codeview
+  /// def_range types parsed by this class.
   StringMap<CVDefRangeType> CVDefRangeTypeMap;
 
   bool parseInitValue(unsigned Size);
@@ -488,20 +742,85 @@ class MasmParser : public MCAsmParser {
   bool parseDirectiveAscii(StringRef IDVal, bool ZeroTerminated);
 
   // "byte", "word", ...
-  bool parseScalarInstList(unsigned Size,
-                           SmallVectorImpl<const MCExpr *> &Values);
+  bool emitIntValue(const MCExpr *Value, unsigned Size);
+  bool parseScalarInitializer(unsigned Size,
+                              SmallVectorImpl<const MCExpr *> &Values,
+                              unsigned StringPadLength = 0);
+  bool parseScalarInstList(
+      unsigned Size, SmallVectorImpl<const MCExpr *> &Values,
+      const AsmToken::TokenKind EndToken = AsmToken::EndOfStatement);
+  bool emitIntegralValues(unsigned Size);
+  bool addIntegralField(StringRef Name, unsigned Size);
   bool parseDirectiveValue(StringRef IDVal, unsigned Size);
   bool parseDirectiveNamedValue(StringRef IDVal, unsigned Size, StringRef Name,
                                 SMLoc NameLoc);
 
   // "real4", "real8"
+  bool emitRealValues(const fltSemantics &Semantics);
+  bool addRealField(StringRef Name, const fltSemantics &Semantics);
   bool parseDirectiveRealValue(StringRef IDVal, const fltSemantics &Semantics);
-  bool parseRealInstList(const fltSemantics &Semantics,
-                         SmallVectorImpl<APInt> &Values);
+  bool parseRealInstList(
+      const fltSemantics &Semantics, SmallVectorImpl<APInt> &Values,
+      const AsmToken::TokenKind EndToken = AsmToken::EndOfStatement);
   bool parseDirectiveNamedRealValue(StringRef IDVal,
                                     const fltSemantics &Semantics,
                                     StringRef Name, SMLoc NameLoc);
 
+  bool parseOptionalAngleBracketOpen();
+  bool parseAngleBracketClose(const Twine &Msg = "expected '>'");
+
+  bool parseFieldInitializer(const FieldInfo &Field,
+                             FieldInitializer &Initializer);
+  bool parseFieldInitializer(const FieldInfo &Field,
+                             const IntFieldInfo &Contents,
+                             FieldInitializer &Initializer);
+  bool parseFieldInitializer(const FieldInfo &Field,
+                             const RealFieldInfo &Contents,
+                             FieldInitializer &Initializer);
+  bool parseFieldInitializer(const FieldInfo &Field,
+                             const StructFieldInfo &Contents,
+                             FieldInitializer &Initializer);
+
+  bool parseStructInitializer(const StructInfo &Structure,
+                              StructInitializer &Initializer);
+  bool parseStructInstList(
+      const StructInfo &Structure, std::vector<StructInitializer> &Initializers,
+      const AsmToken::TokenKind EndToken = AsmToken::EndOfStatement);
+
+  bool emitFieldValue(const FieldInfo &Field);
+  bool emitFieldValue(const FieldInfo &Field, const IntFieldInfo &Contents);
+  bool emitFieldValue(const FieldInfo &Field, const RealFieldInfo &Contents);
+  bool emitFieldValue(const FieldInfo &Field, const StructFieldInfo &Contents);
+
+  bool emitStructValue(const StructInfo &Structure);
+
+  bool emitFieldInitializer(const FieldInfo &Field,
+                            const FieldInitializer &Initializer);
+  bool emitFieldInitializer(const FieldInfo &Field,
+                            const IntFieldInfo &Contents,
+                            const IntFieldInfo &Initializer);
+  bool emitFieldInitializer(const FieldInfo &Field,
+                            const RealFieldInfo &Contents,
+                            const RealFieldInfo &Initializer);
+  bool emitFieldInitializer(const FieldInfo &Field,
+                            const StructFieldInfo &Contents,
+                            const StructFieldInfo &Initializer);
+
+  bool emitStructInitializer(const StructInfo &Structure,
+                             const StructInitializer &Initializer);
+
+  // User-defined types (structs, unions):
+  bool emitStructValue(const StructInfo &Structure,
+                       const StructInitializer &Initializer,
+                       size_t InitialOffset = 0, size_t InitialField = 0);
+  bool emitStructValues(const StructInfo &Structure);
+  bool addStructField(StringRef Name, const StructInfo &Structure);
+  bool parseDirectiveStructValue(const StructInfo &Structure,
+                                 StringRef Directive, SMLoc DirLoc);
+  bool parseDirectiveNamedStructValue(const StructInfo &Structure,
+                                      StringRef Directive, SMLoc DirLoc,
+                                      StringRef Name);
+
   // "=", "equ", "textequ"
   bool parseDirectiveEquate(StringRef IDVal, StringRef Name,
                             DirectiveKind DirKind);
@@ -560,8 +879,14 @@ class MasmParser : public MCAsmParser {
   // alternate macro mode directives
   bool parseDirectiveAltmacro(StringRef Directive);
 
-  /// Parse a directive like ".globl" which
-  /// accepts a single symbol (which should be a label or an external).
+  bool parseDirectiveStruct(StringRef Directive, DirectiveKind DirKind,
+                            StringRef Name, SMLoc NameLoc);
+  bool parseDirectiveNestedStruct(StringRef Directive, DirectiveKind DirKind);
+  bool parseDirectiveEnds(StringRef Name, SMLoc NameLoc);
+  bool parseDirectiveNestedEnds();
+
+  /// Parse a directive like ".globl" which accepts a single symbol (which
+  /// should be a label or an external).
   bool parseDirectiveSymbolAttribute(MCSymbolAttr Attr);
 
   bool parseDirectiveComm(bool IsLocal); // ".comm" and ".lcomm"
@@ -1022,7 +1347,7 @@ bool MasmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
         return Error(FirstTokenLoc, "invalid token in expression");
       }
     }
-    // Parse symbol variant
+    // Parse symbol variant.
     std::pair<StringRef, StringRef> Split;
     if (!MAI.useParensForSymbolVariant()) {
       if (FirstTokenKind == AsmToken::String) {
@@ -1058,7 +1383,7 @@ bool MasmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
 
     MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
 
-    // Lookup the symbol variant if used.
+    // Look up the symbol variant if used.
     if (!Split.second.empty()) {
       Variant = MCSymbolRefExpr::getVariantKindForName(Split.second);
       if (Variant != MCSymbolRefExpr::VK_Invalid) {
@@ -1071,6 +1396,27 @@ bool MasmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
       }
     }
 
+    // Find the field offset if used.
+    unsigned Offset = 0;
+    Split = SymbolName.split('.');
+    if (!Split.second.empty()) {
+      SymbolName = Split.first;
+      if (Structs.count(SymbolName.lower()) &&
+          !LookUpFieldOffset(SymbolName, Split.second, Offset)) {
+        // This is actually a reference to a field offset.
+        Res = MCConstantExpr::create(Offset, getContext());
+        return false;
+      }
+
+      auto TypeIt = KnownType.find(SymbolName);
+      if (TypeIt == KnownType.end() ||
+          LookUpFieldOffset(*TypeIt->second, Split.second, Offset)) {
+        std::pair<StringRef, StringRef> BaseMember = Split.second.split('.');
+        StringRef Base = BaseMember.first, Member = BaseMember.second;
+        LookUpFieldOffset(Base, Member, Offset);
+      }
+    }
+
     MCSymbol *Sym = getContext().getInlineAsmLabel(SymbolName);
     if (!Sym)
       Sym = getContext().getOrCreateSymbol(SymbolName);
@@ -1091,7 +1437,15 @@ bool MasmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
     }
 
     // Otherwise create a symbol ref.
-    Res = MCSymbolRefExpr::create(Sym, Variant, getContext(), FirstTokenLoc);
+    const MCExpr *SymRef =
+        MCSymbolRefExpr::create(Sym, Variant, getContext(), FirstTokenLoc);
+    if (Offset) {
+      Res = MCBinaryExpr::create(MCBinaryExpr::Add, SymRef,
+                                 MCConstantExpr::create(Offset, getContext()),
+                                 getContext());
+    } else {
+      Res = SymRef;
+    }
     return false;
   }
   case AsmToken::BigNum:
@@ -1102,10 +1456,10 @@ bool MasmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
     Res = MCConstantExpr::create(IntVal, getContext());
     EndLoc = Lexer.getTok().getEndLoc();
     Lex(); // Eat token.
-    // Look for 'b' or 'f' following an Integer as a directional label
+    // Look for 'b' or 'f' following an Integer as a directional label.
     if (Lexer.getKind() == AsmToken::Identifier) {
       StringRef IDVal = getTok().getString();
-      // Lookup the symbol variant if used.
+      // Look up the symbol variant if used.
       std::pair<StringRef, StringRef> Split = IDVal.split('@');
       MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
       if (Split.first.size() != IDVal.size()) {
@@ -1322,7 +1676,8 @@ bool MasmParser::parseAbsoluteExpression(int64_t &Res) {
 
 static unsigned getGNUBinOpPrecedence(AsmToken::TokenKind K,
                                       MCBinaryExpr::Opcode &Kind,
-                                      bool ShouldUseLogicalShr) {
+                                      bool ShouldUseLogicalShr,
+                                      bool EndExpressionAtGreater) {
   switch (K) {
   default:
     return 0; // not a binop.
@@ -1350,6 +1705,8 @@ static unsigned getGNUBinOpPrecedence(AsmToken::TokenKind K,
     Kind = MCBinaryExpr::LTE;
     return 3;
   case AsmToken::Greater:
+    if (EndExpressionAtGreater)
+      return 0;
     Kind = MCBinaryExpr::GT;
     return 3;
   case AsmToken::GreaterEqual:
@@ -1391,6 +1748,8 @@ static unsigned getGNUBinOpPrecedence(AsmToken::TokenKind K,
     Kind = MCBinaryExpr::Shl;
     return 6;
   case AsmToken::GreaterGreater:
+    if (EndExpressionAtGreater)
+      return 0;
     Kind = ShouldUseLogicalShr ? MCBinaryExpr::LShr : MCBinaryExpr::AShr;
     return 6;
   }
@@ -1399,7 +1758,8 @@ static unsigned getGNUBinOpPrecedence(AsmToken::TokenKind K,
 unsigned MasmParser::getBinOpPrecedence(AsmToken::TokenKind K,
                                         MCBinaryExpr::Opcode &Kind) {
   bool ShouldUseLogicalShr = MAI.shouldUseLogicalShr();
-  return getGNUBinOpPrecedence(K, Kind, ShouldUseLogicalShr);
+  return getGNUBinOpPrecedence(K, Kind, ShouldUseLogicalShr,
+                               AngleBracketDepth > 0);
 }
 
 /// Parse all binary operators with precedence >= 'Precedence'.
@@ -1442,11 +1802,11 @@ bool MasmParser::parseBinOpRHS(unsigned Precedence, const MCExpr *&Res,
 bool MasmParser::parseStatement(ParseStatementInfo &Info,
                                 MCAsmParserSemaCallback *SI) {
   assert(!hasPendingError() && "parseStatement started with pending error");
-  // Eat initial spaces and comments
+  // Eat initial spaces and comments.
   while (Lexer.is(AsmToken::Space))
     Lex();
   if (Lexer.is(AsmToken::EndOfStatement)) {
-    // if this is a line comment we can drop it safely
+    // If this is a line comment we can drop it safely.
     if (getTok().getString().empty() || getTok().getString().front() == '\r' ||
         getTok().getString().front() == '\n')
       Out.AddBlankLine();
@@ -1676,6 +2036,13 @@ bool MasmParser::parseStatement(ParseStatementInfo &Info,
 
     getTargetParser().flushPendingInstructions(getStreamer());
 
+    // Special-case handling of structure-end directives at higher priority,
+    // since ENDS is overloaded as a segment-end directive.
+    if (IDVal.equals_lower("ends") && StructInProgress.size() > 1 &&
+        getTok().is(AsmToken::EndOfStatement)) {
+      return parseDirectiveNestedEnds();
+    }
+
     // First, check the extension directive map to see if any extension has
     // registered itself to parse this directive.
     std::pair<MCAsmParserExtension *, DirectiveHandler> Handler =
@@ -1733,6 +2100,11 @@ bool MasmParser::parseStatement(ParseStatementInfo &Info,
       return parseDirectiveRealValue(IDVal, APFloat::IEEEsingle());
     case DK_REAL8:
       return parseDirectiveRealValue(IDVal, APFloat::IEEEdouble());
+    case DK_STRUCT:
+    case DK_UNION:
+      return parseDirectiveNestedStruct(IDVal, DirKind);
+    case DK_ENDS:
+      return parseDirectiveNestedEnds();
     case DK_ALIGN:
       return parseDirectiveAlign();
     case DK_ORG:
@@ -1876,6 +2248,12 @@ bool MasmParser::parseStatement(ParseStatementInfo &Info,
     return Error(IDLoc, "unknown directive");
   }
 
+  // We also check if this is allocating memory with user-defined type.
+  auto IDIt = Structs.find(IDVal.lower());
+  if (IDIt != Structs.end())
+    return parseDirectiveStructValue(/*Structure=*/IDIt->getValue(), IDVal,
+                                     IDLoc);
+
   // Non-conditional Microsoft directives sometimes follow their first argument.
   const AsmToken nextTok = getTok();
   const StringRef nextVal = nextTok.getString();
@@ -1892,6 +2270,13 @@ bool MasmParser::parseStatement(ParseStatementInfo &Info,
 
   getTargetParser().flushPendingInstructions(getStreamer());
 
+  // Special-case handling of structure-end directives at higher priority, since
+  // ENDS is overloaded as a segment-end directive.
+  if (nextVal.equals_lower("ends") && StructInProgress.size() == 1) {
+    Lex();
+    return parseDirectiveEnds(IDVal, IDLoc);
+  }
+
   // First, check the extension directive map to see if any extension has
   // registered itself to parse this directive.
   std::pair<MCAsmParserExtension *, DirectiveHandler> Handler =
@@ -1902,7 +2287,7 @@ bool MasmParser::parseStatement(ParseStatementInfo &Info,
     return (*Handler.second)(Handler.first, nextVal, nextLoc);
   }
 
-  // Finally, if no one else is interested in this directive, it must be
+  // If no one else is interested in this directive, it must be
   // generic and familiar to this class.
   DirKindIt = DirectiveKindMap.find(nextVal.lower());
   DirKind = (DirKindIt == DirectiveKindMap.end())
@@ -1943,6 +2328,21 @@ bool MasmParser::parseStatement(ParseStatementInfo &Info,
     Lex();
     return parseDirectiveNamedRealValue(nextVal, APFloat::IEEEdouble(), IDVal,
                                         IDLoc);
+  case DK_STRUCT:
+  case DK_UNION:
+    Lex();
+    return parseDirectiveStruct(nextVal, DirKind, IDVal, IDLoc);
+  case DK_ENDS:
+    Lex();
+    return parseDirectiveEnds(IDVal, IDLoc);
+  }
+
+  // Finally, we check if this is allocating a variable with user-defined type.
+  auto NextIt = Structs.find(nextVal.lower());
+  if (NextIt != Structs.end()) {
+    Lex();
+    return parseDirectiveNamedStructValue(/*Structure=*/NextIt->getValue(),
+                                          nextVal, nextLoc, IDVal);
   }
 
   // __asm _emit or __asm __emit
@@ -2027,19 +2427,19 @@ bool MasmParser::parseStatement(ParseStatementInfo &Info,
   return false;
 }
 
-// Parse and erase curly braces marking block start/end
+// Parse and erase curly braces marking block start/end.
 bool MasmParser::parseCurlyBlockScope(
     SmallVectorImpl<AsmRewrite> &AsmStrRewrites) {
-  // Identify curly brace marking block start/end
+  // Identify curly brace marking block start/end.
   if (Lexer.isNot(AsmToken::LCurly) && Lexer.isNot(AsmToken::RCurly))
     return false;
 
   SMLoc StartLoc = Lexer.getLoc();
-  Lex(); // Eat the brace
+  Lex(); // Eat the brace.
   if (Lexer.is(AsmToken::EndOfStatement))
-    Lex(); // Eat EndOfStatement following the brace
+    Lex(); // Eat EndOfStatement following the brace.
 
-  // Erase the block start/end brace from the output asm string
+  // Erase the block start/end brace from the output asm string.
   AsmStrRewrites.emplace_back(AOK_Skip, StartLoc, Lexer.getLoc().getPointer() -
                                                   StartLoc.getPointer());
   return true;
@@ -2346,7 +2746,7 @@ bool MasmParser::parseMacroArgument(MCAsmMacroArgument &MA, bool Vararg) {
 
       if (Lexer.is(AsmToken::Space)) {
         SpaceEaten = true;
-        Lexer.Lex(); // Eat spaces
+        Lexer.Lex(); // Eat spaces.
       }
 
       // Spaces can delimit parameters, but could also be part an expression.
@@ -2429,7 +2829,7 @@ bool MasmParser::parseMacroArguments(const MCAsmMacro *M,
     if (AltMacroMode && Lexer.is(AsmToken::Percent)) {
       const MCExpr *AbsoluteExp;
       int64_t Value;
-      /// Eat '%'
+      /// Eat '%'.
       Lex();
       if (parseExpression(AbsoluteExp, EndLoc))
         return false;
@@ -2446,7 +2846,7 @@ bool MasmParser::parseMacroArguments(const MCAsmMacro *M,
       const char *StrChar = StrLoc.getPointer();
       const char *EndChar = EndLoc.getPointer();
       jumpToLoc(EndLoc, CurBuffer);
-      /// Eat from '<' to '>'
+      /// Eat from '<' to '>'.
       Lex();
       AsmToken newToken(AsmToken::String,
                         StringRef(StrChar, EndChar - StrChar));
@@ -2632,7 +3032,7 @@ bool MasmParser::parseDirectiveEquate(StringRef IDVal, StringRef Name,
       Var.IsText = true;
       Var.TextValue = Value;
 
-      // Accept a text-list, not just one text-item
+      // Accept a text-list, not just one text-item.
       auto parseItem = [&]() -> bool {
         if (parseTextItem(Value))
           return true;
@@ -2648,7 +3048,7 @@ bool MasmParser::parseDirectiveEquate(StringRef IDVal, StringRef Name,
   if (DirKind == DK_TEXTEQU)
     return TokError("expected <text> in '" + Twine(IDVal) + "' directive");
 
-  // Parse as expression assignment
+  // Parse as expression assignment.
   const MCExpr *Expr;
   SMLoc EndLoc, StartLoc = Lexer.getLoc();
   if (parseExpression(Expr, EndLoc))
@@ -2746,7 +3146,7 @@ bool MasmParser::parseAngleBracketString(std::string &Data) {
     const char *StartChar = StartLoc.getPointer() + 1;
     const char *EndChar = EndLoc.getPointer() - 1;
     jumpToLoc(EndLoc, CurBuffer);
-    /// Eat from '<' to '>'
+    // Eat from '<' to '>'.
     Lex();
 
     Data = angleBracketString(StringRef(StartChar, EndChar - StartChar));
@@ -2779,89 +3179,140 @@ bool MasmParser::parseDirectiveAscii(StringRef IDVal, bool ZeroTerminated) {
   return false;
 }
 
-bool MasmParser::parseScalarInstList(unsigned Size,
-                                     SmallVectorImpl<const MCExpr *> &Values) {
-  do {
-    if (getTok().is(AsmToken::String)) {
-      StringRef Value = getTok().getStringContents();
-      if (Size == 1) {
-        // Treat each character as an initializer.
-        for (const char CharVal : Value)
-          Values.push_back(MCConstantExpr::create(CharVal, getContext()));
-      } else {
-        // Treat the string as an initial value in big-endian representation.
-        if (Value.size() > Size)
-          return Error(getTok().getLoc(), "out of range literal value");
-
-        uint64_t IntValue = 0;
-        for (const unsigned char CharVal : Value.bytes())
-          IntValue = (IntValue << 8) | CharVal;
-        Values.push_back(MCConstantExpr::create(IntValue, getContext()));
-      }
-      Lex();
+bool MasmParser::emitIntValue(const MCExpr *Value, unsigned Size) {
+  // Special case constant expressions to match code generator.
+  if (const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(Value)) {
+    assert(Size <= 8 && "Invalid size");
+    int64_t IntValue = MCE->getValue();
+    if (!isUIntN(8 * Size, IntValue) && !isIntN(8 * Size, IntValue))
+      return Error(MCE->getLoc(), "out of range literal value");
+    getStreamer().emitIntValue(IntValue, Size);
+  } else {
+    const MCSymbolRefExpr *MSE = dyn_cast<MCSymbolRefExpr>(Value);
+    if (MSE && MSE->getSymbol().getName() == "?") {
+      // ? initializer; treat as 0.
+      getStreamer().emitIntValue(0, Size);
     } else {
-      const MCExpr *Value;
-      if (checkForValidSection() || parseExpression(Value))
+      getStreamer().emitValue(Value, Size, Value->getLoc());
+    }
+  }
+  return false;
+}
+
+bool MasmParser::parseScalarInitializer(unsigned Size,
+                                        SmallVectorImpl<const MCExpr *> &Values,
+                                        unsigned StringPadLength) {
+  if (getTok().is(AsmToken::String)) {
+    StringRef Value = getTok().getStringContents();
+    if (Size == 1) {
+      // Treat each character as an initializer.
+      for (const char CharVal : Value)
+        Values.push_back(MCConstantExpr::create(CharVal, getContext()));
+
+      // Pad the string with spaces to the specified length.
+      for (size_t i = Value.size(); i < StringPadLength; ++i)
+        Values.push_back(MCConstantExpr::create(' ', getContext()));
+    } else {
+      // Treat the string as an initial value in big-endian representation.
+      if (Value.size() > Size)
+        return Error(getTok().getLoc(), "out of range literal value");
+
+      uint64_t IntValue = 0;
+      for (const unsigned char CharVal : Value.bytes())
+        IntValue = (IntValue << 8) | CharVal;
+      Values.push_back(MCConstantExpr::create(IntValue, getContext()));
+    }
+    Lex();
+  } else {
+    const MCExpr *Value;
+    if (checkForValidSection() || parseExpression(Value))
+      return true;
+    if (getTok().is(AsmToken::Identifier) &&
+        getTok().getString().equals_lower("dup")) {
+      Lex(); // Eat 'dup'.
+      const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(Value);
+      if (!MCE)
+        return Error(Value->getLoc(),
+                     "cannot repeat value a non-constant number of times");
+      const int64_t Repetitions = MCE->getValue();
+      if (Repetitions < 0)
+        return Error(Value->getLoc(),
+                     "cannot repeat value a negative number of times");
+
+      SmallVector<const MCExpr *, 1> DuplicatedValues;
+      if (parseToken(AsmToken::LParen,
+                     "parentheses required for 'dup' contents") ||
+          parseScalarInstList(Size, DuplicatedValues) ||
+          parseToken(AsmToken::RParen, "unmatched parentheses"))
         return true;
-      if (getTok().is(AsmToken::Identifier) &&
-          getTok().getString().equals_lower("dup")) {
-        Lex();  // eat 'dup'
-        const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(Value);
-        if (!MCE)
-          return Error(Value->getLoc(),
-                       "cannot repeat value a non-constant number of times");
-        const int64_t Repetitions = MCE->getValue();
-        if (Repetitions < 0)
-          return Error(Value->getLoc(),
-                       "cannot repeat value a negative number of times");
-
-        SmallVector<const MCExpr *, 1> DuplicatedValues;
-        if (parseToken(AsmToken::LParen,
-                       "parentheses required for 'dup' contents") ||
-            parseScalarInstList(Size, DuplicatedValues) ||
-            parseToken(AsmToken::RParen, "unmatched parentheses"))
-          return true;
 
-        for (int i = 0; i < Repetitions; ++i)
-          Values.append(DuplicatedValues.begin(), DuplicatedValues.end());
-      } else {
-        Values.push_back(Value);
-      }
+      for (int i = 0; i < Repetitions; ++i)
+        Values.append(DuplicatedValues.begin(), DuplicatedValues.end());
+    } else {
+      Values.push_back(Value);
     }
+  }
+  return false;
+}
 
-    // Continue if we see a comma. (Also, allow line continuation.)
-  } while (parseOptionalToken(AsmToken::Comma) &&
-           (getTok().isNot(AsmToken::EndOfStatement) ||
-            !parseToken(AsmToken::EndOfStatement)));
+bool MasmParser::parseScalarInstList(unsigned Size,
+                                     SmallVectorImpl<const MCExpr *> &Values,
+                                     const AsmToken::TokenKind EndToken) {
+  while (getTok().isNot(EndToken) &&
+         (EndToken != AsmToken::Greater ||
+          getTok().isNot(AsmToken::GreaterGreater))) {
+    parseScalarInitializer(Size, Values);
+
+    // If we see a comma, continue, and allow line continuation.
+    if (!parseOptionalToken(AsmToken::Comma))
+      break;
+    parseOptionalToken(AsmToken::EndOfStatement);
+  }
+  return false;
+}
+
+bool MasmParser::emitIntegralValues(unsigned Size) {
+  SmallVector<const MCExpr *, 1> Values;
+  if (checkForValidSection() || parseScalarInstList(Size, Values))
+    return true;
+
+  for (auto Value : Values) {
+    emitIntValue(Value, Size);
+  }
+  return false;
+}
+
+// Add a field to the current structure.
+bool MasmParser::addIntegralField(StringRef Name, unsigned Size) {
+  StructInfo &Struct = StructInProgress.back();
+  FieldInfo &Field = Struct.addField(Name, FT_INTEGRAL);
+  IntFieldInfo &IntInfo = Field.Contents.IntInfo;
 
+  Field.Type = Size;
+
+  if (parseScalarInstList(Size, IntInfo.Values))
+    return true;
+
+  Field.SizeOf = Field.Type * IntInfo.Values.size();
+  Field.LengthOf = IntInfo.Values.size();
+  if (Struct.IsUnion)
+    Struct.Size = std::max(Struct.Size, Field.SizeOf);
+  else
+    Struct.Size += Field.SizeOf;
   return false;
 }
 
 /// parseDirectiveValue
 ///  ::= (byte | word | ... ) [ expression (, expression)* ]
 bool MasmParser::parseDirectiveValue(StringRef IDVal, unsigned Size) {
-  SmallVector<const MCExpr *, 1> Values;
-  if (parseScalarInstList(Size, Values))
+  if (StructInProgress.empty()) {
+    // Initialize data value.
+    if (emitIntegralValues(Size))
+      return addErrorSuffix(" in '" + Twine(IDVal) + "' directive");
+  } else if (addIntegralField("", Size)) {
     return addErrorSuffix(" in '" + Twine(IDVal) + "' directive");
-
-  for (const MCExpr *Value : Values) {
-    // Special case constant expressions to match code generator.
-    if (const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(Value)) {
-      assert(Size <= 8 && "Invalid size");
-      int64_t IntValue = MCE->getValue();
-      if (!isUIntN(8 * Size, IntValue) && !isIntN(8 * Size, IntValue))
-        return Error(MCE->getLoc(), "out of range literal value");
-      getStreamer().emitIntValue(IntValue, Size);
-    } else {
-      const MCSymbolRefExpr *MSE = dyn_cast<MCSymbolRefExpr>(Value);
-      if (MSE && MSE->getSymbol().getName() == "?") {
-        // ? initializer; treat as 0.
-        getStreamer().emitIntValue(0, Size);
-      } else {
-        getStreamer().emitValue(Value, Size, Value->getLoc());
-      }
-    }
   }
+
   return false;
 }
 
@@ -2869,9 +3320,17 @@ bool MasmParser::parseDirectiveValue(StringRef IDVal, unsigned Size) {
 ///  ::= name (byte | word | ... ) [ expression (, expression)* ]
 bool MasmParser::parseDirectiveNamedValue(StringRef IDVal, unsigned Size,
                                           StringRef Name, SMLoc NameLoc) {
-  MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
-  getStreamer().emitLabel(Sym);
-  return parseDirectiveValue(IDVal, Size);
+  if (StructInProgress.empty()) {
+    // Initialize named data value.
+    MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
+    getStreamer().emitLabel(Sym);
+    if (emitIntegralValues(Size))
+      return addErrorSuffix(" in '" + Twine(IDVal) + "' directive");
+  } else if (addIntegralField(Name, Size)) {
+    return addErrorSuffix(" in '" + Twine(IDVal) + "' directive");
+  }
+
+  return false;
 }
 
 static bool parseHexOcta(MasmParser &Asm, uint64_t &hi, uint64_t &lo) {
@@ -2900,8 +3359,9 @@ bool MasmParser::parseRealValue(const fltSemantics &Semantics, APInt &Res) {
   if (getLexer().is(AsmToken::Minus)) {
     Lexer.Lex();
     IsNeg = true;
-  } else if (getLexer().is(AsmToken::Plus))
+  } else if (getLexer().is(AsmToken::Plus)) {
     Lexer.Lex();
+  }
 
   if (Lexer.is(AsmToken::Error))
     return TokError(Lexer.getErr());
@@ -2913,16 +3373,19 @@ bool MasmParser::parseRealValue(const fltSemantics &Semantics, APInt &Res) {
   APFloat Value(Semantics);
   StringRef IDVal = getTok().getString();
   if (getLexer().is(AsmToken::Identifier)) {
-    if (!IDVal.compare_lower("infinity") || !IDVal.compare_lower("inf"))
+    if (IDVal.equals_lower("infinity") || IDVal.equals_lower("inf"))
       Value = APFloat::getInf(Semantics);
-    else if (!IDVal.compare_lower("nan"))
+    else if (IDVal.equals_lower("nan"))
       Value = APFloat::getNaN(Semantics, false, ~0);
+    else if (IDVal.equals_lower("?"))
+      Value = APFloat::getZero(Semantics);
     else
       return TokError("invalid floating point literal");
   } else if (errorToBool(
                  Value.convertFromString(IDVal, APFloat::rmNearestTiesToEven)
-                     .takeError()))
+                     .takeError())) {
     return TokError("invalid floating point literal");
+  }
   if (IsNeg)
     Value.changeSign();
 
@@ -2935,8 +3398,11 @@ bool MasmParser::parseRealValue(const fltSemantics &Semantics, APInt &Res) {
 }
 
 bool MasmParser::parseRealInstList(const fltSemantics &Semantics,
-                                   SmallVectorImpl<APInt> &ValuesAsInt) {
-  do {
+                                   SmallVectorImpl<APInt> &ValuesAsInt,
+                                   const AsmToken::TokenKind EndToken) {
+  while (getTok().isNot(EndToken) ||
+         (EndToken == AsmToken::Greater &&
+          getTok().isNot(AsmToken::GreaterGreater))) {
     const AsmToken NextTok = Lexer.peekTok();
     if (NextTok.is(AsmToken::Identifier) &&
         NextTok.getString().equals_lower("dup")) {
@@ -2967,11 +3433,48 @@ bool MasmParser::parseRealInstList(const fltSemantics &Semantics,
         return true;
       ValuesAsInt.push_back(AsInt);
     }
+
     // Continue if we see a comma. (Also, allow line continuation.)
-  } while (parseOptionalToken(AsmToken::Comma) &&
-           (getTok().isNot(AsmToken::EndOfStatement) ||
-            !parseToken(AsmToken::EndOfStatement)));
+    if (!parseOptionalToken(AsmToken::Comma))
+      break;
+    parseOptionalToken(AsmToken::EndOfStatement);
+  }
+
+  return false;
+}
 
+// Initialize real data values.
+bool MasmParser::emitRealValues(const fltSemantics &Semantics) {
+  SmallVector<APInt, 1> ValuesAsInt;
+  if (parseRealInstList(Semantics, ValuesAsInt))
+    return true;
+
+  for (const APInt &AsInt : ValuesAsInt) {
+    getStreamer().emitIntValue(AsInt.getLimitedValue(),
+                               AsInt.getBitWidth() / 8);
+  }
+  return false;
+}
+
+// Add a real field to the current struct.
+bool MasmParser::addRealField(StringRef Name, const fltSemantics &Semantics) {
+  StructInfo &Struct = StructInProgress.back();
+  FieldInfo &Field = Struct.addField(Name, FT_REAL);
+  RealFieldInfo &RealInfo = Field.Contents.RealInfo;
+
+  Field.SizeOf = 0;
+
+  if (checkForValidSection() ||
+      parseRealInstList(Semantics, RealInfo.AsIntValues))
+    return true;
+
+  Field.Type = RealInfo.AsIntValues.back().getBitWidth() / 8;
+  Field.LengthOf = RealInfo.AsIntValues.size();
+  Field.SizeOf = Field.Type * Field.LengthOf;
+  if (Struct.IsUnion)
+    Struct.Size = std::max(Struct.Size, Field.SizeOf);
+  else
+    Struct.Size += Field.SizeOf;
   return false;
 }
 
@@ -2982,13 +3485,12 @@ bool MasmParser::parseDirectiveRealValue(StringRef IDVal,
   if (checkForValidSection())
     return true;
 
-  SmallVector<APInt, 1> ValuesAsInt;
-  if (parseRealInstList(Semantics, ValuesAsInt))
+  if (StructInProgress.empty()) {
+    // Initialize data value.
+    if (emitRealValues(Semantics))
+      return addErrorSuffix(" in '" + Twine(IDVal) + "' directive");
+  } else if (addRealField("", Semantics)) {
     return addErrorSuffix(" in '" + Twine(IDVal) + "' directive");
-
-  for (const APInt &AsInt : ValuesAsInt) {
-    getStreamer().emitIntValue(AsInt.getLimitedValue(),
-                               AsInt.getBitWidth() / 8);
   }
   return false;
 }
@@ -2998,9 +3500,636 @@ bool MasmParser::parseDirectiveRealValue(StringRef IDVal,
 bool MasmParser::parseDirectiveNamedRealValue(StringRef IDVal,
                                               const fltSemantics &Semantics,
                                               StringRef Name, SMLoc NameLoc) {
-  MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
-  getStreamer().emitLabel(Sym);
-  return parseDirectiveRealValue(IDVal, Semantics);
+  if (checkForValidSection())
+    return true;
+
+  if (StructInProgress.empty()) {
+    // Initialize named data value.
+    MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
+    getStreamer().emitLabel(Sym);
+    if (emitRealValues(Semantics))
+      return addErrorSuffix(" in '" + Twine(IDVal) + "' directive");
+  } else if (addRealField(Name, Semantics)) {
+    return addErrorSuffix(" in '" + Twine(IDVal) + "' directive");
+  }
+  return false;
+}
+
+bool MasmParser::parseOptionalAngleBracketOpen() {
+  const AsmToken Tok = getTok();
+  if (parseOptionalToken(AsmToken::LessLess)) {
+    AngleBracketDepth++;
+    Lexer.UnLex(AsmToken(AsmToken::Less, Tok.getString().substr(1)));
+    return true;
+  } else if (parseOptionalToken(AsmToken::LessGreater)) {
+    AngleBracketDepth++;
+    Lexer.UnLex(AsmToken(AsmToken::Greater, Tok.getString().substr(1)));
+    return true;
+  } else if (parseOptionalToken(AsmToken::Less)) {
+    AngleBracketDepth++;
+    return true;
+  }
+
+  return false;
+}
+
+bool MasmParser::parseAngleBracketClose(const Twine &Msg) {
+  const AsmToken Tok = getTok();
+  if (parseOptionalToken(AsmToken::GreaterGreater)) {
+    Lexer.UnLex(AsmToken(AsmToken::Greater, Tok.getString().substr(1)));
+  } else if (parseToken(AsmToken::Greater, Msg)) {
+    return true;
+  }
+  AngleBracketDepth--;
+  return false;
+}
+
+bool MasmParser::parseFieldInitializer(const FieldInfo &Field,
+                                       const IntFieldInfo &Contents,
+                                       FieldInitializer &Initializer) {
+  SMLoc Loc = getTok().getLoc();
+
+  SmallVector<const MCExpr *, 1> Values;
+  if (parseOptionalToken(AsmToken::LCurly)) {
+    if (Field.LengthOf == 1 && Field.Type > 1)
+      return Error(Loc, "Cannot initialize scalar field with array value");
+    if (parseScalarInstList(Field.Type, Values, AsmToken::RCurly) ||
+        parseToken(AsmToken::RCurly))
+      return true;
+  } else if (parseOptionalAngleBracketOpen()) {
+    if (Field.LengthOf == 1 && Field.Type > 1)
+      return Error(Loc, "Cannot initialize scalar field with array value");
+    if (parseScalarInstList(Field.Type, Values, AsmToken::Greater) ||
+        parseAngleBracketClose())
+      return true;
+  } else if (Field.LengthOf > 1 && Field.Type > 1) {
+    return Error(Loc, "Cannot initialize array field with scalar value");
+  } else if (parseScalarInitializer(Field.Type, Values,
+                                    /*StringPadLength=*/Field.LengthOf)) {
+    return true;
+  }
+
+  if (Values.size() > Field.LengthOf) {
+    return Error(Loc, "Initializer too long for field; expected at most " +
+                          std::to_string(Field.LengthOf) + " elements, got " +
+                          std::to_string(Values.size()));
+  }
+  // Default-initialize all remaining values.
+  Values.append(Contents.Values.begin() + Values.size(), Contents.Values.end());
+
+  Initializer = FieldInitializer(std::move(Values));
+  return false;
+}
+
+bool MasmParser::parseFieldInitializer(const FieldInfo &Field,
+                                       const RealFieldInfo &Contents,
+                                       FieldInitializer &Initializer) {
+  const fltSemantics &Semantics =
+      (Field.Type == 4) ? APFloat::IEEEsingle() : APFloat::IEEEdouble();
+
+  SMLoc Loc = getTok().getLoc();
+
+  SmallVector<APInt, 1> AsIntValues;
+  if (parseOptionalToken(AsmToken::LCurly)) {
+    if (Field.LengthOf == 1)
+      return Error(Loc, "Cannot initialize scalar field with array value");
+    if (parseRealInstList(Semantics, AsIntValues, AsmToken::RCurly) ||
+        parseToken(AsmToken::RCurly))
+      return true;
+  } else if (parseOptionalAngleBracketOpen()) {
+    if (Field.LengthOf == 1)
+      return Error(Loc, "Cannot initialize scalar field with array value");
+    if (parseRealInstList(Semantics, AsIntValues, AsmToken::Greater) ||
+        parseAngleBracketClose())
+      return true;
+  } else if (Field.LengthOf > 1) {
+    return Error(Loc, "Cannot initialize array field with scalar value");
+  } else {
+    AsIntValues.emplace_back();
+    if (parseRealValue(Semantics, AsIntValues.back()))
+      return true;
+  }
+
+  if (AsIntValues.size() > Field.LengthOf) {
+    return Error(Loc, "Initializer too long for field; expected at most " +
+                          std::to_string(Field.LengthOf) + " elements, got " +
+                          std::to_string(AsIntValues.size()));
+  }
+  // Default-initialize all remaining values.
+  AsIntValues.append(Contents.AsIntValues.begin() + AsIntValues.size(),
+                     Contents.AsIntValues.end());
+
+  Initializer = FieldInitializer(std::move(AsIntValues));
+  return false;
+}
+
+bool MasmParser::parseFieldInitializer(const FieldInfo &Field,
+                                       const StructFieldInfo &Contents,
+                                       FieldInitializer &Initializer) {
+  SMLoc Loc = getTok().getLoc();
+
+  std::vector<StructInitializer> Initializers;
+  if (Field.LengthOf > 1) {
+    if (parseOptionalToken(AsmToken::LCurly)) {
+      if (parseStructInstList(Contents.Structure, Initializers,
+                              AsmToken::RCurly) ||
+          parseToken(AsmToken::RCurly))
+        return true;
+    } else if (parseOptionalAngleBracketOpen()) {
+      if (parseStructInstList(Contents.Structure, Initializers,
+                              AsmToken::Greater) ||
+          parseAngleBracketClose())
+        return true;
+    } else {
+      return Error(Loc, "Cannot initialize array field with scalar value");
+    }
+  } else {
+    Initializers.emplace_back();
+    if (parseStructInitializer(Contents.Structure, Initializers.back()))
+      return true;
+  }
+
+  if (Initializers.size() > Field.LengthOf) {
+    return Error(Loc, "Initializer too long for field; expected at most " +
+                          std::to_string(Field.LengthOf) + " elements, got " +
+                          std::to_string(Initializers.size()));
+  }
+  // Default-initialize all remaining values.
+  Initializers.insert(Initializers.end(),
+                      Contents.Initializers.begin() + Initializers.size(),
+                      Contents.Initializers.end());
+
+  Initializer = FieldInitializer(std::move(Initializers), Contents.Structure);
+  return false;
+}
+
+bool MasmParser::parseFieldInitializer(const FieldInfo &Field,
+                                       FieldInitializer &Initializer) {
+  switch (Field.Contents.FT) {
+  case FT_INTEGRAL:
+    return parseFieldInitializer(Field, Field.Contents.IntInfo, Initializer);
+  case FT_REAL:
+    return parseFieldInitializer(Field, Field.Contents.RealInfo, Initializer);
+  case FT_STRUCT:
+    return parseFieldInitializer(Field, Field.Contents.StructInfo, Initializer);
+  }
+  llvm_unreachable("Unhandled FieldType enum");
+}
+
+bool MasmParser::parseStructInitializer(const StructInfo &Structure,
+                                        StructInitializer &Initializer) {
+  const AsmToken FirstToken = getTok();
+
+  Optional<AsmToken::TokenKind> EndToken;
+  if (parseOptionalToken(AsmToken::LCurly)) {
+    EndToken = AsmToken::RCurly;
+  } else if (parseOptionalAngleBracketOpen()) {
+    EndToken = AsmToken::Greater;
+    AngleBracketDepth++;
+  } else if (FirstToken.is(AsmToken::Identifier) &&
+             FirstToken.getString() == "?") {
+    // ? initializer; leave EndToken uninitialized to treat as empty.
+    if (parseToken(AsmToken::Identifier))
+      return true;
+  } else {
+    return Error(FirstToken.getLoc(), "Expected struct initializer");
+  }
+
+  auto &FieldInitializers = Initializer.FieldInitializers;
+  size_t FieldIndex = 0;
+  if (EndToken.hasValue()) {
+    // Initialize all fields with given initializers.
+    while (getTok().isNot(EndToken.getValue()) &&
+           FieldIndex < Structure.Fields.size()) {
+      const FieldInfo &Field = Structure.Fields[FieldIndex++];
+      if (parseOptionalToken(AsmToken::Comma)) {
+        // Empty initializer; use the default and continue. (Also, allow line
+        // continuation.)
+        FieldInitializers.push_back(Field.Contents);
+        parseOptionalToken(AsmToken::EndOfStatement);
+        continue;
+      }
+      FieldInitializers.emplace_back(Field.Contents.FT);
+      if (parseFieldInitializer(Field, FieldInitializers.back()))
+        return true;
+
+      // Continue if we see a comma. (Also, allow line continuation.)
+      SMLoc CommaLoc = getTok().getLoc();
+      if (!parseOptionalToken(AsmToken::Comma))
+        break;
+      if (FieldIndex == Structure.Fields.size())
+        return Error(CommaLoc, "'" + Structure.Name +
+                                   "' initializer initializes too many fields");
+      parseOptionalToken(AsmToken::EndOfStatement);
+    }
+  }
+  // Default-initialize all remaining fields.
+  for (auto It = Structure.Fields.begin() + FieldIndex;
+       It != Structure.Fields.end(); ++It) {
+    const FieldInfo &Field = *It;
+    FieldInitializers.push_back(Field.Contents);
+  }
+
+  if (EndToken.hasValue()) {
+    if (EndToken.getValue() == AsmToken::Greater)
+      return parseAngleBracketClose();
+
+    return parseToken(EndToken.getValue());
+  }
+
+  return false;
+}
+
+bool MasmParser::parseStructInstList(
+    const StructInfo &Structure, std::vector<StructInitializer> &Initializers,
+    const AsmToken::TokenKind EndToken) {
+  while (getTok().isNot(EndToken) ||
+         (EndToken == AsmToken::Greater &&
+          getTok().isNot(AsmToken::GreaterGreater))) {
+    const AsmToken NextTok = Lexer.peekTok();
+    if (NextTok.is(AsmToken::Identifier) &&
+        NextTok.getString().equals_lower("dup")) {
+      const MCExpr *Value;
+      if (parseExpression(Value) || parseToken(AsmToken::Identifier))
+        return true;
+      const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(Value);
+      if (!MCE)
+        return Error(Value->getLoc(),
+                     "cannot repeat value a non-constant number of times");
+      const int64_t Repetitions = MCE->getValue();
+      if (Repetitions < 0)
+        return Error(Value->getLoc(),
+                     "cannot repeat value a negative number of times");
+
+      std::vector<StructInitializer> DuplicatedValues;
+      if (parseToken(AsmToken::LParen,
+                     "parentheses required for 'dup' contents") ||
+          parseStructInstList(Structure, DuplicatedValues) ||
+          parseToken(AsmToken::RParen, "unmatched parentheses"))
+        return true;
+
+      for (int i = 0; i < Repetitions; ++i)
+        Initializers.insert(Initializers.end(), DuplicatedValues.begin(),
+                            DuplicatedValues.end());
+    } else {
+      Initializers.emplace_back();
+      if (parseStructInitializer(Structure, Initializers.back()))
+        return true;
+    }
+
+    // Continue if we see a comma. (Also, allow line continuation.)
+    if (!parseOptionalToken(AsmToken::Comma))
+      break;
+    parseOptionalToken(AsmToken::EndOfStatement);
+  }
+
+  return false;
+}
+
+bool MasmParser::emitFieldValue(const FieldInfo &Field,
+                                const IntFieldInfo &Contents) {
+  // Default-initialize all values.
+  for (const MCExpr *Value : Contents.Values) {
+    if (emitIntValue(Value, Field.Type))
+      return true;
+  }
+  return false;
+}
+
+bool MasmParser::emitFieldValue(const FieldInfo &Field,
+                                const RealFieldInfo &Contents) {
+  for (const APInt &AsInt : Contents.AsIntValues) {
+    getStreamer().emitIntValue(AsInt.getLimitedValue(),
+                               AsInt.getBitWidth() / 8);
+  }
+  return false;
+}
+
+bool MasmParser::emitFieldValue(const FieldInfo &Field,
+                                const StructFieldInfo &Contents) {
+  for (const auto &Initializer : Contents.Initializers) {
+    size_t Index = 0, Offset = 0;
+    for (const auto &SubField : Contents.Structure.Fields) {
+      getStreamer().emitZeros(SubField.Offset - Offset);
+      Offset = SubField.Offset + SubField.SizeOf;
+      emitFieldInitializer(SubField, Initializer.FieldInitializers[Index++]);
+    }
+  }
+  return false;
+}
+
+bool MasmParser::emitFieldValue(const FieldInfo &Field) {
+  switch (Field.Contents.FT) {
+  case FT_INTEGRAL:
+    return emitFieldValue(Field, Field.Contents.IntInfo);
+  case FT_REAL:
+    return emitFieldValue(Field, Field.Contents.RealInfo);
+  case FT_STRUCT:
+    return emitFieldValue(Field, Field.Contents.StructInfo);
+  }
+  llvm_unreachable("Unhandled FieldType enum");
+}
+
+bool MasmParser::emitStructValue(const StructInfo &Structure) {
+  size_t Offset = 0;
+  for (const auto &Field : Structure.Fields) {
+    getStreamer().emitZeros(Field.Offset - Offset);
+    if (emitFieldValue(Field))
+      return true;
+    Offset = Field.Offset + Field.SizeOf;
+  }
+  // Add final padding.
+  if (Offset != Structure.Size)
+    getStreamer().emitZeros(Structure.Size - Offset);
+  return false;
+}
+
+bool MasmParser::emitFieldInitializer(const FieldInfo &Field,
+                                      const IntFieldInfo &Contents,
+                                      const IntFieldInfo &Initializer) {
+  for (const auto &Value : Initializer.Values) {
+    if (emitIntValue(Value, Field.Type))
+      return true;
+  }
+  // Default-initialize all remaining values.
+  for (auto it = Contents.Values.begin() + Initializer.Values.size();
+       it != Contents.Values.end(); ++it) {
+    const auto &Value = *it;
+    if (emitIntValue(Value, Field.Type))
+      return true;
+  }
+  return false;
+}
+
+bool MasmParser::emitFieldInitializer(const FieldInfo &Field,
+                                      const RealFieldInfo &Contents,
+                                      const RealFieldInfo &Initializer) {
+  for (const auto &AsInt : Initializer.AsIntValues) {
+    getStreamer().emitIntValue(AsInt.getLimitedValue(),
+                               AsInt.getBitWidth() / 8);
+  }
+  // Default-initialize all remaining values.
+  for (auto It = Contents.AsIntValues.begin() + Initializer.AsIntValues.size();
+       It != Contents.AsIntValues.end(); ++It) {
+    const auto &AsInt = *It;
+    getStreamer().emitIntValue(AsInt.getLimitedValue(),
+                               AsInt.getBitWidth() / 8);
+  }
+  return false;
+}
+
+bool MasmParser::emitFieldInitializer(const FieldInfo &Field,
+                                      const StructFieldInfo &Contents,
+                                      const StructFieldInfo &Initializer) {
+  for (const auto &Init : Initializer.Initializers) {
+    emitStructInitializer(Contents.Structure, Init);
+  }
+  // Default-initialize all remaining values.
+  for (auto It =
+           Contents.Initializers.begin() + Initializer.Initializers.size();
+       It != Contents.Initializers.end(); ++It) {
+    const auto &Init = *It;
+    emitStructInitializer(Contents.Structure, Init);
+  }
+  return false;
+}
+
+bool MasmParser::emitFieldInitializer(const FieldInfo &Field,
+                                      const FieldInitializer &Initializer) {
+  switch (Field.Contents.FT) {
+  case FT_INTEGRAL:
+    return emitFieldInitializer(Field, Field.Contents.IntInfo,
+                                Initializer.IntInfo);
+  case FT_REAL:
+    return emitFieldInitializer(Field, Field.Contents.RealInfo,
+                                Initializer.RealInfo);
+  case FT_STRUCT:
+    return emitFieldInitializer(Field, Field.Contents.StructInfo,
+                                Initializer.StructInfo);
+  }
+  llvm_unreachable("Unhandled FieldType enum");
+}
+
+bool MasmParser::emitStructInitializer(const StructInfo &Structure,
+                                       const StructInitializer &Initializer) {
+  size_t Index = 0, Offset = 0;
+  for (const auto &Init : Initializer.FieldInitializers) {
+    const auto &Field = Structure.Fields[Index++];
+    getStreamer().emitZeros(Field.Offset - Offset);
+    Offset = Field.Offset + Field.SizeOf;
+    if (emitFieldInitializer(Field, Init))
+      return true;
+  }
+  // Default-initialize all remaining fields.
+  for (auto It =
+           Structure.Fields.begin() + Initializer.FieldInitializers.size();
+       It != Structure.Fields.end(); ++It) {
+    const auto &Field = *It;
+    getStreamer().emitZeros(Field.Offset - Offset);
+    Offset = Field.Offset + Field.SizeOf;
+    if (emitFieldValue(Field))
+      return true;
+  }
+  // Add final padding.
+  if (Offset != Structure.Size)
+    getStreamer().emitZeros(Structure.Size - Offset);
+  return false;
+}
+
+// Set data values from initializers.
+bool MasmParser::emitStructValues(const StructInfo &Structure) {
+  std::vector<StructInitializer> Initializers;
+  if (parseStructInstList(Structure, Initializers))
+    return true;
+
+  for (const auto &Initializer : Initializers) {
+    if (emitStructInitializer(Structure, Initializer))
+      return true;
+  }
+
+  return false;
+}
+
+// Declare a field in the current struct.
+bool MasmParser::addStructField(StringRef Name, const StructInfo &Structure) {
+  StructInfo &OwningStruct = StructInProgress.back();
+  FieldInfo &Field = OwningStruct.addField(Name, FT_STRUCT);
+  StructFieldInfo &StructInfo = Field.Contents.StructInfo;
+
+  StructInfo.Structure = Structure;
+  Field.Type = Structure.Size;
+
+  if (parseStructInstList(Structure, StructInfo.Initializers))
+    return true;
+
+  Field.LengthOf = StructInfo.Initializers.size();
+  Field.SizeOf = Field.Type * Field.LengthOf;
+  if (OwningStruct.IsUnion)
+    OwningStruct.Size = std::max(OwningStruct.Size, Field.SizeOf);
+  else
+    OwningStruct.Size += Field.SizeOf;
+
+  return false;
+}
+
+/// parseDirectiveStructValue
+///  ::= struct-id (<struct-initializer> | {struct-initializer})
+///                [, (<struct-initializer> | {struct-initializer})]*
+bool MasmParser::parseDirectiveStructValue(const StructInfo &Structure,
+                                           StringRef Directive, SMLoc DirLoc) {
+  if (StructInProgress.empty()) {
+    if (emitStructValues(Structure))
+      return true;
+  } else if (addStructField("", Structure)) {
+    return addErrorSuffix(" in '" + Twine(Directive) + "' directive");
+  }
+
+  return false;
+}
+
+/// parseDirectiveNamedValue
+///  ::= name (byte | word | ... ) [ expression (, expression)* ]
+bool MasmParser::parseDirectiveNamedStructValue(const StructInfo &Structure,
+                                                StringRef Directive,
+                                                SMLoc DirLoc, StringRef Name) {
+  if (StructInProgress.empty()) {
+    // Initialize named data value.
+    MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
+    getStreamer().emitLabel(Sym);
+    KnownType[Name] = &Structure;
+    if (emitStructValues(Structure))
+      return true;
+  } else if (addStructField(Name, Structure)) {
+    return addErrorSuffix(" in '" + Twine(Directive) + "' directive");
+  }
+
+  return false;
+}
+
+/// parseDirectiveStruct
+///  ::= <name> (STRUC | STRUCT | UNION) [fieldAlign] [, NONUNIQUE]
+///      (dataDir | generalDir | offsetDir | nestedStruct)+
+///      <name> ENDS
+////// dataDir = data declaration
+////// offsetDir = EVEN, ORG, ALIGN
+bool MasmParser::parseDirectiveStruct(StringRef Directive,
+                                      DirectiveKind DirKind, StringRef Name,
+                                      SMLoc NameLoc) {
+  // We ignore NONUNIQUE; we do not support OPTION M510 or OPTION OLDSTRUCTS
+  // anyway, so all field accesses must be qualified.
+  AsmToken NextTok = getTok();
+  int64_t AlignmentValue = 1;
+  if (NextTok.isNot(AsmToken::Comma) &&
+      NextTok.isNot(AsmToken::EndOfStatement) &&
+      parseAbsoluteExpression(AlignmentValue)) {
+    return addErrorSuffix(" in alignment value for '" + Twine(Directive) +
+                          "' directive");
+  }
+  if (!isPowerOf2_64(AlignmentValue)) {
+    return Error(NextTok.getLoc(), "alignment must be a power of two; was " +
+                                       std::to_string(AlignmentValue));
+  }
+
+  StringRef Qualifier;
+  SMLoc QualifierLoc;
+  if (parseOptionalToken(AsmToken::Comma)) {
+    QualifierLoc = getTok().getLoc();
+    if (parseIdentifier(Qualifier))
+      return addErrorSuffix(" in '" + Twine(Directive) + "' directive");
+    if (!Qualifier.equals_lower("nonunique"))
+      return Error(QualifierLoc, "Unrecognized qualifier for '" +
+                                     Twine(Directive) +
+                                     "' directive; expected none or NONUNIQUE");
+  }
+
+  if (parseToken(AsmToken::EndOfStatement))
+    return addErrorSuffix(" in '" + Twine(Directive) + "' directive");
+
+  StructInProgress.emplace_back(Name, DirKind == DK_UNION, AlignmentValue);
+  return false;
+}
+
+/// parseDirectiveNestedStruct
+///  ::= (STRUC | STRUCT | UNION) [name]
+///      (dataDir | generalDir | offsetDir | nestedStruct)+
+///      ENDS
+bool MasmParser::parseDirectiveNestedStruct(StringRef Directive,
+                                            DirectiveKind DirKind) {
+  if (StructInProgress.empty())
+    return TokError("missing name in top-level '" + Twine(Directive) +
+                    "' directive");
+
+  StringRef Name;
+  if (getTok().is(AsmToken::Identifier)) {
+    Name = getTok().getIdentifier();
+    parseToken(AsmToken::Identifier);
+  }
+  if (parseToken(AsmToken::EndOfStatement))
+    return addErrorSuffix(" in '" + Twine(Directive) + "' directive");
+
+  StructInProgress.emplace_back(Name, DirKind == DK_UNION,
+                                StructInProgress.back().Alignment);
+  return false;
+}
+
+bool MasmParser::parseDirectiveEnds(StringRef Name, SMLoc NameLoc) {
+  if (StructInProgress.empty())
+    return Error(NameLoc, "ENDS directive without matching STRUC/STRUCT/UNION");
+  if (StructInProgress.size() > 1)
+    return Error(NameLoc, "unexpected name in nested ENDS directive");
+  if (StructInProgress.back().Name.compare_lower(Name))
+    return Error(NameLoc, "mismatched name in ENDS directive; expected '" +
+                              StructInProgress.back().Name + "'");
+  StructInfo Structure = StructInProgress.pop_back_val();
+  if (Structure.Size % Structure.Alignment != 0) {
+    // Pad to make the structure's size divisible by its alignment.
+    Structure.Size +=
+        Structure.Alignment - (Structure.Size % Structure.Alignment);
+  }
+  Structs[Name.lower()] = Structure;
+
+  if (parseToken(AsmToken::EndOfStatement))
+    return addErrorSuffix(" in ENDS directive");
+
+  return false;
+}
+
+bool MasmParser::parseDirectiveNestedEnds() {
+  if (StructInProgress.empty())
+    return TokError("ENDS directive without matching STRUC/STRUCT/UNION");
+  if (StructInProgress.size() == 1)
+    return TokError("missing name in top-level ENDS directive");
+
+  if (parseToken(AsmToken::EndOfStatement))
+    return addErrorSuffix(" in nested ENDS directive");
+
+  StructInfo Structure = StructInProgress.pop_back_val();
+  if (Structure.Size % Structure.Alignment != 0) {
+    // Pad to make the structure's size divisible by its alignment.
+    Structure.Size +=
+        Structure.Alignment - (Structure.Size % Structure.Alignment);
+  }
+  StructInfo &ParentStruct = StructInProgress.back();
+
+  FieldInfo &Field = ParentStruct.addField(Structure.Name, FT_STRUCT);
+  StructFieldInfo &StructInfo = Field.Contents.StructInfo;
+  Field.Type = Structure.Size;
+  Field.LengthOf = 1;
+  Field.SizeOf = Structure.Size;
+
+  if (ParentStruct.IsUnion)
+    ParentStruct.Size = std::max(ParentStruct.Size, Field.SizeOf);
+  else
+    ParentStruct.Size += Field.SizeOf;
+
+  StructInfo.Structure = Structure;
+  StructInfo.Initializers.emplace_back();
+  auto &FieldInitializers = StructInfo.Initializers.back().FieldInitializers;
+  for (const auto &SubField : Structure.Fields) {
+    FieldInitializers.push_back(SubField.Contents);
+  }
+
+  return false;
 }
 
 /// parseDirectiveOrg
@@ -3031,7 +4160,7 @@ bool MasmParser::parseDirectiveAlign() {
 
   if (checkForValidSection())
     return addErrorSuffix(" in align directive");
-  // Ignore empty 'align' directives
+  // Ignore empty 'align' directives.
   if (getTok().is(AsmToken::EndOfStatement)) {
     Warning(AlignmentLoc, "align directive with no operand is ignored");
     return parseToken(AsmToken::EndOfStatement);
@@ -3043,9 +4172,8 @@ bool MasmParser::parseDirectiveAlign() {
   // Always emit an alignment here even if we thrown an error.
   bool ReturnVal = false;
 
-  // Reject alignments that aren't either a power of two or zero,
-  // for gas compatibility. Alignment of zero is silently rounded
-  // up to one.
+  // Reject alignments that aren't either a power of two or zero, for gas
+  // compatibility. Alignment of zero is silently rounded up to one.
   if (Alignment == 0)
     Alignment = 1;
   if (!isPowerOf2_64(Alignment))
@@ -4086,7 +5214,7 @@ bool MasmParser::parseDirectiveMacro(SMLoc DirectiveLoc) {
     if (parseIdentifier(Parameter.Name))
       return TokError("expected identifier in '.macro' directive");
 
-    // Emit an error if two (or more) named parameters share the same name
+    // Emit an error if two (or more) named parameters share the same name.
     for (const MCAsmMacroParameter& CurrParam : Parameters)
       if (CurrParam.Name.equals(Parameter.Name))
         return TokError("macro '" + Name + "' has multiple parameters"
@@ -4135,7 +5263,7 @@ bool MasmParser::parseDirectiveMacro(SMLoc DirectiveLoc) {
   // Eat just the end of statement.
   Lexer.Lex();
 
-  // Consuming deferred text, so use Lexer.Lex to ignore Lexing Errors
+  // Consuming deferred text, so use Lexer.Lex to ignore Lexing Errors.
   AsmToken EndToken, StartToken = getTok();
   unsigned MacroDepth = 0;
   // Lex the macro definition.
@@ -4443,7 +5571,7 @@ bool MasmParser::parseDirectiveComm(bool IsLocal) {
   if (!Sym->isUndefined())
     return Error(IDLoc, "invalid symbol redefinition");
 
-  // Create the Symbol as a common or local common with Size and Pow2Alignment
+  // Create the Symbol as a common or local common with Size and Pow2Alignment.
   if (IsLocal) {
     getStreamer().emitLocalCommonSymbol(Sym, Size, 1 << Pow2Alignment);
     return false;
@@ -5128,6 +6256,10 @@ void MasmParser::initializeDirectiveKindMap() {
   DirectiveKindMap["dq"] = DK_DQ;
   DirectiveKindMap["dw"] = DK_DW;
   DirectiveKindMap["echo"] = DK_ECHO;
+  DirectiveKindMap["struc"] = DK_STRUCT;
+  DirectiveKindMap["struct"] = DK_STRUCT;
+  DirectiveKindMap["union"] = DK_UNION;
+  DirectiveKindMap["ends"] = DK_ENDS;
 }
 
 MCAsmMacro *MasmParser::parseMacroLikeBody(SMLoc DirectiveLoc) {
@@ -5387,6 +6519,49 @@ static int rewritesSort(const AsmRewrite *AsmRewriteA,
   llvm_unreachable("Unstable rewrite sort.");
 }
 
+bool MasmParser::LookUpFieldOffset(StringRef Base, StringRef Member,
+                                   unsigned &Offset) {
+  if (Base.empty())
+    return true;
+
+  auto TypeIt = KnownType.find(Base);
+  if (TypeIt != KnownType.end())
+    return LookUpFieldOffset(*TypeIt->second, Member, Offset);
+
+  auto StructIt = Structs.find(Base.lower());
+  if (StructIt != Structs.end())
+    return LookUpFieldOffset(StructIt->second, Member, Offset);
+
+  return true;
+}
+
+bool MasmParser::LookUpFieldOffset(const StructInfo &Structure,
+                                   StringRef Member, unsigned &Offset) {
+  std::pair<StringRef, StringRef> Split = Member.split('.');
+  const StringRef FieldName = Split.first, FieldMember = Split.second;
+
+  auto FieldIt = Structure.FieldsByName.find(FieldName.lower());
+  if (FieldIt == Structure.FieldsByName.end())
+    return true;
+
+  const FieldInfo &Field = Structure.Fields[FieldIt->second];
+  if (FieldMember.empty()) {
+    Offset = Field.Offset;
+    return false;
+  }
+
+  if (Field.Contents.FT != FT_STRUCT)
+    return true;
+  const StructFieldInfo &StructInfo = Field.Contents.StructInfo;
+
+  bool Result = LookUpFieldOffset(StructInfo.Structure, FieldMember, Offset);
+  if (Result)
+    return true;
+
+  Offset += Field.Offset;
+  return false;
+}
+
 bool MasmParser::parseMSInlineAsm(
     void *AsmLoc, std::string &AsmString, unsigned &NumOutputs,
     unsigned &NumInputs, SmallVectorImpl<std::pair<void *, bool>> &OpDecls,
@@ -5410,7 +6585,7 @@ bool MasmParser::parseMSInlineAsm(
   unsigned InputIdx = 0;
   unsigned OutputIdx = 0;
   while (getLexer().isNot(AsmToken::Eof)) {
-    // Parse curly braces marking block start/end
+    // Parse curly braces marking block start/end.
     if (parseCurlyBlockScope(AsmStrRewrites))
       continue;
 
@@ -5456,7 +6631,7 @@ bool MasmParser::parseMSInlineAsm(
 
       StringRef Constraint = Operand.getConstraint();
       if (Operand.isImm()) {
-        // Offset as immediate
+        // Offset as immediate.
         if (Operand.isOffsetOfLocal())
           Constraint = "r";
         else
diff --git a/llvm/lib/MC/MCSectionWasm.cpp b/llvm/lib/MC/MCSectionWasm.cpp
index 995b811b758e2..27ed51802a2ec 100644
--- a/llvm/lib/MC/MCSectionWasm.cpp
+++ b/llvm/lib/MC/MCSectionWasm.cpp
@@ -10,6 +10,7 @@
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolWasm.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index 0b650ef936bff..6d3a933c96a37 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -1071,6 +1071,12 @@ void MCStreamer::emitXCOFFSymbolLinkageWithVisibility(MCSymbol *Symbol,
                    "XCOFF targets");
 }
 
+void MCStreamer::emitXCOFFRenameDirective(const MCSymbol *Name,
+                                          StringRef Rename) {
+  llvm_unreachable("emitXCOFFRenameDirective is only supported on "
+                   "XCOFF targets");
+}
+
 void MCStreamer::emitELFSize(MCSymbol *Symbol, const MCExpr *Value) {}
 void MCStreamer::emitELFSymverDirective(StringRef AliasName,
                                         const MCSymbol *Aliasee) {}
@@ -1128,6 +1134,79 @@ MCSymbol *MCStreamer::endSection(MCSection *Section) {
   return Sym;
 }
 
+static VersionTuple
+targetVersionOrMinimumSupportedOSVersion(const Triple &Target,
+                                         VersionTuple TargetVersion) {
+  VersionTuple Min = Target.getMinimumSupportedOSVersion();
+  return !Min.empty() && Min > TargetVersion ? Min : TargetVersion;
+}
+
+static MCVersionMinType
+getMachoVersionMinLoadCommandType(const Triple &Target) {
+  assert(Target.isOSDarwin() && "expected a darwin OS");
+  switch (Target.getOS()) {
+  case Triple::MacOSX:
+  case Triple::Darwin:
+    return MCVM_OSXVersionMin;
+  case Triple::IOS:
+    assert(!Target.isMacCatalystEnvironment() &&
+           "mac Catalyst should use LC_BUILD_VERSION");
+    return MCVM_IOSVersionMin;
+  case Triple::TvOS:
+    return MCVM_TvOSVersionMin;
+  case Triple::WatchOS:
+    return MCVM_WatchOSVersionMin;
+  default:
+    break;
+  }
+  llvm_unreachable("unexpected OS type");
+}
+
+static VersionTuple getMachoBuildVersionSupportedOS(const Triple &Target) {
+  assert(Target.isOSDarwin() && "expected a darwin OS");
+  switch (Target.getOS()) {
+  case Triple::MacOSX:
+  case Triple::Darwin:
+    return VersionTuple(10, 14);
+  case Triple::IOS:
+    // Mac Catalyst always uses the build version load command.
+    if (Target.isMacCatalystEnvironment())
+      return VersionTuple();
+    LLVM_FALLTHROUGH;
+  case Triple::TvOS:
+    return VersionTuple(12);
+  case Triple::WatchOS:
+    return VersionTuple(5);
+  default:
+    break;
+  }
+  llvm_unreachable("unexpected OS type");
+}
+
+static MachO::PlatformType
+getMachoBuildVersionPlatformType(const Triple &Target) {
+  assert(Target.isOSDarwin() && "expected a darwin OS");
+  switch (Target.getOS()) {
+  case Triple::MacOSX:
+  case Triple::Darwin:
+    return MachO::PLATFORM_MACOS;
+  case Triple::IOS:
+    if (Target.isMacCatalystEnvironment())
+      return MachO::PLATFORM_MACCATALYST;
+    return Target.isSimulatorEnvironment() ? MachO::PLATFORM_IOSSIMULATOR
+                                           : MachO::PLATFORM_IOS;
+  case Triple::TvOS:
+    return Target.isSimulatorEnvironment() ? MachO::PLATFORM_TVOSSIMULATOR
+                                           : MachO::PLATFORM_TVOS;
+  case Triple::WatchOS:
+    return Target.isSimulatorEnvironment() ? MachO::PLATFORM_WATCHOSSIMULATOR
+                                           : MachO::PLATFORM_WATCHOS;
+  default:
+    break;
+  }
+  llvm_unreachable("unexpected OS type");
+}
+
 void MCStreamer::emitVersionForTarget(const Triple &Target,
                                       const VersionTuple &SDKVersion) {
   if (!Target.isOSBinFormatMachO() || !Target.isOSDarwin())
@@ -1136,33 +1215,37 @@ void MCStreamer::emitVersionForTarget(const Triple &Target,
   if (Target.getOSMajorVersion() == 0)
     return;
 
-  unsigned Major;
-  unsigned Minor;
-  unsigned Update;
-  if (Target.isMacCatalystEnvironment()) {
-    // Mac Catalyst always uses the build version load command.
+  unsigned Major = 0;
+  unsigned Minor = 0;
+  unsigned Update = 0;
+  switch (Target.getOS()) {
+  case Triple::MacOSX:
+  case Triple::Darwin:
+    Target.getMacOSXVersion(Major, Minor, Update);
+    break;
+  case Triple::IOS:
+  case Triple::TvOS:
     Target.getiOSVersion(Major, Minor, Update);
-    assert(Major && "A non-zero major version is expected");
-    emitBuildVersion(MachO::PLATFORM_MACCATALYST, Major, Minor, Update,
-                     SDKVersion);
-    return;
-  }
-
-  MCVersionMinType VersionType;
-  if (Target.isWatchOS()) {
-    VersionType = MCVM_WatchOSVersionMin;
+    break;
+  case Triple::WatchOS:
     Target.getWatchOSVersion(Major, Minor, Update);
-  } else if (Target.isTvOS()) {
-    VersionType = MCVM_TvOSVersionMin;
-    Target.getiOSVersion(Major, Minor, Update);
-  } else if (Target.isMacOSX()) {
-    VersionType = MCVM_OSXVersionMin;
-    if (!Target.getMacOSXVersion(Major, Minor, Update))
-      Major = 0;
-  } else {
-    VersionType = MCVM_IOSVersionMin;
-    Target.getiOSVersion(Major, Minor, Update);
+    break;
+  default:
+    llvm_unreachable("unexpected OS type");
   }
-  if (Major != 0)
-    emitVersionMin(VersionType, Major, Minor, Update, SDKVersion);
+  assert(Major != 0 && "A non-zero major version is expected");
+  auto LinkedTargetVersion = targetVersionOrMinimumSupportedOSVersion(
+      Target, VersionTuple(Major, Minor, Update));
+  auto BuildVersionOSVersion = getMachoBuildVersionSupportedOS(Target);
+  if (BuildVersionOSVersion.empty() ||
+      LinkedTargetVersion >= BuildVersionOSVersion)
+    return emitBuildVersion(getMachoBuildVersionPlatformType(Target),
+                            LinkedTargetVersion.getMajor(),
+                            *LinkedTargetVersion.getMinor(),
+                            *LinkedTargetVersion.getSubminor(), SDKVersion);
+
+  emitVersionMin(getMachoVersionMinLoadCommandType(Target),
+                 LinkedTargetVersion.getMajor(),
+                 *LinkedTargetVersion.getMinor(),
+                 *LinkedTargetVersion.getSubminor(), SDKVersion);
 }
diff --git a/llvm/lib/MC/MCSymbolXCOFF.cpp b/llvm/lib/MC/MCSymbolXCOFF.cpp
index 53aec88c89780..536153e5518b5 100644
--- a/llvm/lib/MC/MCSymbolXCOFF.cpp
+++ b/llvm/lib/MC/MCSymbolXCOFF.cpp
@@ -17,6 +17,9 @@ MCSectionXCOFF *MCSymbolXCOFF::getRepresentedCsect() const {
           RepresentedCsect->getCSectType() == XCOFF::XTY_ER) &&
          "Symbol does not represent a csect; MCSectionXCOFF that represents "
          "the symbol should not be (but is) set.");
+  assert(getSymbolTableName().equals(RepresentedCsect->getSymbolTableName()) &&
+         "SymbolTableNames need to be the same for this symbol and its csect "
+         "representation.");
   return RepresentedCsect;
 }
 
@@ -29,5 +32,8 @@ void MCSymbolXCOFF::setRepresentedCsect(MCSectionXCOFF *C) {
           C->getCSectType() == XCOFF::XTY_ER) &&
          "Symbol does not represent a csect; can only set a MCSectionXCOFF "
          "representation for a csect.");
+  assert(getSymbolTableName().equals(C->getSymbolTableName()) &&
+         "SymbolTableNames need to be the same for this symbol and its csect "
+         "representation.");
   RepresentedCsect = C;
 }
diff --git a/llvm/lib/MC/MCWinCOFFStreamer.cpp b/llvm/lib/MC/MCWinCOFFStreamer.cpp
index 7f0f7fccd5426..d8fde4004d44b 100644
--- a/llvm/lib/MC/MCWinCOFFStreamer.cpp
+++ b/llvm/lib/MC/MCWinCOFFStreamer.cpp
@@ -328,7 +328,34 @@ void MCWinCOFFStreamer::EmitWinEHHandlerData(SMLoc Loc) {
   llvm_unreachable("not implemented");
 }
 
+void MCWinCOFFStreamer::emitCGProfileEntry(const MCSymbolRefExpr *From,
+                                           const MCSymbolRefExpr *To,
+                                           uint64_t Count) {
+  // Ignore temporary symbols for now.
+  if (!From->getSymbol().isTemporary() && !To->getSymbol().isTemporary())
+    getAssembler().CGProfile.push_back({From, To, Count});
+}
+
+void MCWinCOFFStreamer::finalizeCGProfileEntry(const MCSymbolRefExpr *&SRE) {
+  const MCSymbol *S = &SRE->getSymbol();
+  bool Created;
+  getAssembler().registerSymbol(*S, &Created);
+  if (Created) {
+    cast<MCSymbolCOFF>(S)->setIsWeakExternal();
+    cast<MCSymbolCOFF>(S)->setExternal(true);
+  }
+}
+
+void MCWinCOFFStreamer::finalizeCGProfile() {
+  for (MCAssembler::CGProfileEntry &E : getAssembler().CGProfile) {
+    finalizeCGProfileEntry(E.From);
+    finalizeCGProfileEntry(E.To);
+  }
+}
+
 void MCWinCOFFStreamer::finishImpl() {
+  finalizeCGProfile();
+
   MCObjectStreamer::finishImpl();
 }
 
diff --git a/llvm/lib/MC/WasmObjectWriter.cpp b/llvm/lib/MC/WasmObjectWriter.cpp
index 0916ce5145191..f51d908c53e13 100644
--- a/llvm/lib/MC/WasmObjectWriter.cpp
+++ b/llvm/lib/MC/WasmObjectWriter.cpp
@@ -108,7 +108,7 @@ struct WasmDataSegment {
   MCSectionWasm *Section;
   StringRef Name;
   uint32_t InitFlags;
-  uint32_t Offset;
+  uint64_t Offset;
   uint32_t Alignment;
   uint32_t LinkerFlags;
   SmallVector<char, 4> Data;
@@ -224,11 +224,8 @@ class WasmObjectWriter : public MCObjectWriter {
 
   // Relocations for fixing up references in the code section.
   std::vector<WasmRelocationEntry> CodeRelocations;
-  uint32_t CodeSectionIndex;
-
   // Relocations for fixing up references in the data section.
   std::vector<WasmRelocationEntry> DataRelocations;
-  uint32_t DataSectionIndex;
 
   // Index values to use for fixing up call_indirect type indices.
   // Maps function symbols to the index of the type of the function
@@ -329,15 +326,15 @@ class WasmObjectWriter : public MCObjectWriter {
   void writeValueType(wasm::ValType Ty) { W.OS << static_cast<char>(Ty); }
 
   void writeTypeSection(ArrayRef<WasmSignature> Signatures);
-  void writeImportSection(ArrayRef<wasm::WasmImport> Imports, uint32_t DataSize,
+  void writeImportSection(ArrayRef<wasm::WasmImport> Imports, uint64_t DataSize,
                           uint32_t NumElements);
   void writeFunctionSection(ArrayRef<WasmFunction> Functions);
   void writeExportSection(ArrayRef<wasm::WasmExport> Exports);
   void writeElemSection(ArrayRef<uint32_t> TableElems);
   void writeDataCountSection();
-  void writeCodeSection(const MCAssembler &Asm, const MCAsmLayout &Layout,
-                        ArrayRef<WasmFunction> Functions);
-  void writeDataSection(const MCAsmLayout &Layout);
+  uint32_t writeCodeSection(const MCAssembler &Asm, const MCAsmLayout &Layout,
+                            ArrayRef<WasmFunction> Functions);
+  uint32_t writeDataSection(const MCAsmLayout &Layout);
   void writeEventSection(ArrayRef<wasm::WasmEventType> Events);
   void writeGlobalSection(ArrayRef<wasm::WasmGlobal> Globals);
   void writeRelocSection(uint32_t SectionIndex, StringRef Name,
@@ -733,12 +730,12 @@ void WasmObjectWriter::writeTypeSection(ArrayRef<WasmSignature> Signatures) {
 }
 
 void WasmObjectWriter::writeImportSection(ArrayRef<wasm::WasmImport> Imports,
-                                          uint32_t DataSize,
+                                          uint64_t DataSize,
                                           uint32_t NumElements) {
   if (Imports.empty())
     return;
 
-  uint32_t NumPages = (DataSize + wasm::WasmPageSize - 1) / wasm::WasmPageSize;
+  uint64_t NumPages = (DataSize + wasm::WasmPageSize - 1) / wasm::WasmPageSize;
 
   SectionBookkeeping Section;
   startSection(Section, wasm::WASM_SEC_IMPORT);
@@ -758,8 +755,8 @@ void WasmObjectWriter::writeImportSection(ArrayRef<wasm::WasmImport> Imports,
       W.OS << char(Import.Global.Mutable ? 1 : 0);
       break;
     case wasm::WASM_EXTERNAL_MEMORY:
-      encodeULEB128(0, W.OS);        // flags
-      encodeULEB128(NumPages, W.OS); // initial
+      encodeULEB128(Import.Memory.Flags, W.OS);
+      encodeULEB128(NumPages, W.OS);  // initial
       break;
     case wasm::WASM_EXTERNAL_TABLE:
       W.OS << char(Import.Table.ElemType);
@@ -833,6 +830,9 @@ void WasmObjectWriter::writeGlobalSection(ArrayRef<wasm::WasmGlobal> Globals) {
     case wasm::WASM_TYPE_F64:
       writeI64(0);
       break;
+    case wasm::WASM_TYPE_EXTERNREF:
+      writeValueType(wasm::ValType::EXTERNREF);
+      break;
     default:
       llvm_unreachable("unexpected type");
     }
@@ -891,15 +891,14 @@ void WasmObjectWriter::writeDataCountSection() {
   endSection(Section);
 }
 
-void WasmObjectWriter::writeCodeSection(const MCAssembler &Asm,
-                                        const MCAsmLayout &Layout,
-                                        ArrayRef<WasmFunction> Functions) {
+uint32_t WasmObjectWriter::writeCodeSection(const MCAssembler &Asm,
+                                            const MCAsmLayout &Layout,
+                                            ArrayRef<WasmFunction> Functions) {
   if (Functions.empty())
-    return;
+    return 0;
 
   SectionBookkeeping Section;
   startSection(Section, wasm::WASM_SEC_CODE);
-  CodeSectionIndex = Section.Index;
 
   encodeULEB128(Functions.size(), W.OS);
 
@@ -919,15 +918,15 @@ void WasmObjectWriter::writeCodeSection(const MCAssembler &Asm,
   applyRelocations(CodeRelocations, Section.ContentsOffset, Layout);
 
   endSection(Section);
+  return Section.Index;
 }
 
-void WasmObjectWriter::writeDataSection(const MCAsmLayout &Layout) {
+uint32_t WasmObjectWriter::writeDataSection(const MCAsmLayout &Layout) {
   if (DataSegments.empty())
-    return;
+    return 0;
 
   SectionBookkeeping Section;
   startSection(Section, wasm::WASM_SEC_DATA);
-  DataSectionIndex = Section.Index;
 
   encodeULEB128(DataSegments.size(), W.OS); // count
 
@@ -936,7 +935,9 @@ void WasmObjectWriter::writeDataSection(const MCAsmLayout &Layout) {
     if (Segment.InitFlags & wasm::WASM_SEGMENT_HAS_MEMINDEX)
       encodeULEB128(0, W.OS); // memory index
     if ((Segment.InitFlags & wasm::WASM_SEGMENT_IS_PASSIVE) == 0) {
-      W.OS << char(wasm::WASM_OPCODE_I32_CONST);
+      W.OS << char(Segment.Offset > std::numeric_limits<int32_t>().max()
+                     ? wasm::WASM_OPCODE_I64_CONST
+                     : wasm::WASM_OPCODE_I32_CONST);
       encodeSLEB128(Segment.Offset, W.OS); // offset
       W.OS << char(wasm::WASM_OPCODE_END);
     }
@@ -949,6 +950,7 @@ void WasmObjectWriter::writeDataSection(const MCAsmLayout &Layout) {
   applyRelocations(DataRelocations, Section.ContentsOffset, Layout);
 
   endSection(Section);
+  return Section.Index;
 }
 
 void WasmObjectWriter::writeRelocSection(
@@ -1187,7 +1189,7 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
   SmallVector<wasm::WasmSymbolInfo, 4> SymbolInfos;
   SmallVector<std::pair<uint16_t, uint32_t>, 2> InitFuncs;
   std::map<StringRef, std::vector<WasmComdatEntry>> Comdats;
-  uint32_t DataSize = 0;
+  uint64_t DataSize = 0;
 
   // For now, always emit the memory import, since loads and stores are not
   // valid without it. In the future, we could perhaps be more clever and omit
@@ -1196,6 +1198,8 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
   MemImport.Module = "env";
   MemImport.Field = "__linear_memory";
   MemImport.Kind = wasm::WASM_EXTERNAL_MEMORY;
+  MemImport.Memory.Flags = is64Bit() ? wasm::WASM_LIMITS_FLAG_IS_64
+                                     : wasm::WASM_LIMITS_FLAG_NONE;
   Imports.push_back(MemImport);
 
   // For now, always emit the table section, since indirect calls are not
@@ -1462,6 +1466,9 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
         case wasm::WASM_TYPE_F64:
           Global.InitExpr.Opcode = wasm::WASM_OPCODE_F64_CONST;
           break;
+        case wasm::WASM_TYPE_EXTERNREF:
+          Global.InitExpr.Opcode = wasm::WASM_OPCODE_REF_NULL;
+          break;
         default:
           llvm_unreachable("unexpected type");
         }
@@ -1692,8 +1699,8 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
   writeExportSection(Exports);
   writeElemSection(TableElems);
   writeDataCountSection();
-  writeCodeSection(Asm, Layout, Functions);
-  writeDataSection(Layout);
+  uint32_t CodeSectionIndex = writeCodeSection(Asm, Layout, Functions);
+  uint32_t DataSectionIndex = writeDataSection(Layout);
   for (auto &CustomSection : CustomSections)
     writeCustomSection(CustomSection, Asm, Layout);
   writeLinkingMetaDataSection(SymbolInfos, InitFuncs, Comdats);
diff --git a/llvm/lib/MC/WinCOFFObjectWriter.cpp b/llvm/lib/MC/WinCOFFObjectWriter.cpp
index c6829f5e107aa..4796ef531054b 100644
--- a/llvm/lib/MC/WinCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/WinCOFFObjectWriter.cpp
@@ -154,6 +154,8 @@ class WinCOFFObjectWriter : public MCObjectWriter {
   MCSectionCOFF *AddrsigSection;
   std::vector<const MCSymbol *> AddrsigSyms;
 
+  MCSectionCOFF *CGProfileSection = nullptr;
+
   WinCOFFObjectWriter(std::unique_ptr<MCWinCOFFObjectTargetWriter> MOTW,
                       raw_pwrite_stream &OS);
 
@@ -674,6 +676,13 @@ void WinCOFFObjectWriter::executePostLayoutBinding(MCAssembler &Asm,
     Asm.registerSection(*AddrsigSection);
   }
 
+  if (!Asm.CGProfile.empty()) {
+    CGProfileSection = Asm.getContext().getCOFFSection(
+        ".llvm.call-graph-profile", COFF::IMAGE_SCN_LNK_REMOVE,
+        SectionKind::getMetadata());
+    Asm.registerSection(*CGProfileSection);
+  }
+
   // "Define" each section & symbol. This creates section & symbol
   // entries in the staging area.
   for (const auto &Section : Asm)
@@ -1099,6 +1108,20 @@ uint64_t WinCOFFObjectWriter::writeObject(MCAssembler &Asm,
     }
   }
 
+  // Create the contents of the .llvm.call-graph-profile section.
+  if (CGProfileSection) {
+    auto *Frag = new MCDataFragment(CGProfileSection);
+    Frag->setLayoutOrder(0);
+    raw_svector_ostream OS(Frag->getContents());
+    for (const MCAssembler::CGProfileEntry &CGPE : Asm.CGProfile) {
+      uint32_t FromIndex = CGPE.From->getSymbol().getIndex();
+      uint32_t ToIndex = CGPE.To->getSymbol().getIndex();
+      support::endian::write(OS, FromIndex, W.Endian);
+      support::endian::write(OS, ToIndex, W.Endian);
+      support::endian::write(OS, CGPE.Count, W.Endian);
+    }
+  }
+
   assignFileOffsets(Asm, Layout);
 
   // MS LINK expects to be able to use this timestamp to implement their
diff --git a/llvm/lib/MC/XCOFFObjectWriter.cpp b/llvm/lib/MC/XCOFFObjectWriter.cpp
index 70146315e1209..0dabdc9777d67 100644
--- a/llvm/lib/MC/XCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/XCOFFObjectWriter.cpp
@@ -68,7 +68,7 @@ struct Symbol {
   XCOFF::StorageClass getStorageClass() const {
     return MCSym->getStorageClass();
   }
-  StringRef getName() const { return MCSym->getName(); }
+  StringRef getSymbolTableName() const { return MCSym->getSymbolTableName(); }
   Symbol(const MCSymbolXCOFF *MCSym) : MCSym(MCSym), SymbolTableIndex(-1) {}
 };
 
@@ -81,7 +81,7 @@ struct ControlSection {
 
   SmallVector<Symbol, 1> Syms;
   SmallVector<XCOFFRelocation, 1> Relocations;
-  StringRef getName() const { return MCCsect->getName(); }
+  StringRef getSymbolTableName() const { return MCCsect->getSymbolTableName(); }
   ControlSection(const MCSectionXCOFF *MCSec)
       : MCCsect(MCSec), SymbolTableIndex(-1), Address(-1), Size(0) {}
 };
@@ -334,8 +334,8 @@ void XCOFFObjectWriter::executePostLayoutBinding(MCAssembler &Asm,
 
     // If the name does not fit in the storage provided in the symbol table
     // entry, add it to the string table.
-    if (nameShouldBeInStringTable(MCSec->getName()))
-      Strings.add(MCSec->getName());
+    if (nameShouldBeInStringTable(MCSec->getSymbolTableName()))
+      Strings.add(MCSec->getSymbolTableName());
 
     CsectGroup &Group = getCsectGroup(MCSec);
     Group.emplace_back(MCSec);
@@ -354,8 +354,8 @@ void XCOFFObjectWriter::executePostLayoutBinding(MCAssembler &Asm,
       // Handle undefined symbol.
       UndefinedCsects.emplace_back(ContainingCsect);
       SectionMap[ContainingCsect] = &UndefinedCsects.back();
-      if (nameShouldBeInStringTable(ContainingCsect->getName()))
-        Strings.add(ContainingCsect->getName());
+      if (nameShouldBeInStringTable(ContainingCsect->getSymbolTableName()))
+        Strings.add(ContainingCsect->getSymbolTableName());
       continue;
     }
 
@@ -375,8 +375,8 @@ void XCOFFObjectWriter::executePostLayoutBinding(MCAssembler &Asm,
 
     // If the name does not fit in the storage provided in the symbol table
     // entry, add it to the string table.
-    if (nameShouldBeInStringTable(XSym->getName()))
-      Strings.add(XSym->getName());
+    if (nameShouldBeInStringTable(XSym->getSymbolTableName()))
+      Strings.add(XSym->getSymbolTableName());
   }
 
   Strings.finalize();
@@ -555,7 +555,7 @@ void XCOFFObjectWriter::writeSymbolTableEntryForCsectMemberLabel(
     const Symbol &SymbolRef, const ControlSection &CSectionRef,
     int16_t SectionIndex, uint64_t SymbolOffset) {
   // Name or Zeros and string table offset
-  writeSymbolName(SymbolRef.getName());
+  writeSymbolName(SymbolRef.getSymbolTableName());
   assert(SymbolOffset <= UINT32_MAX - CSectionRef.Address &&
          "Symbol address overflows.");
   W.write<uint32_t>(CSectionRef.Address + SymbolOffset);
@@ -592,7 +592,7 @@ void XCOFFObjectWriter::writeSymbolTableEntryForControlSection(
     const ControlSection &CSectionRef, int16_t SectionIndex,
     XCOFF::StorageClass StorageClass) {
   // n_name, n_zeros, n_offset
-  writeSymbolName(CSectionRef.getName());
+  writeSymbolName(CSectionRef.getSymbolTableName());
   // n_value
   W.write<uint32_t>(CSectionRef.Address);
   // n_scnum
diff --git a/llvm/lib/Object/Error.cpp b/llvm/lib/Object/Error.cpp
index 010c5b42dac25..bc75bc6c0445c 100644
--- a/llvm/lib/Object/Error.cpp
+++ b/llvm/lib/Object/Error.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Object/Error.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ManagedStatic.h"
 
@@ -60,9 +61,10 @@ void BinaryError::anchor() {}
 char BinaryError::ID = 0;
 char GenericBinaryError::ID = 0;
 
-GenericBinaryError::GenericBinaryError(Twine Msg) : Msg(Msg.str()) {}
+GenericBinaryError::GenericBinaryError(const Twine &Msg) : Msg(Msg.str()) {}
 
-GenericBinaryError::GenericBinaryError(Twine Msg, object_error ECOverride)
+GenericBinaryError::GenericBinaryError(const Twine &Msg,
+                                       object_error ECOverride)
     : Msg(Msg.str()) {
   setErrorCode(make_error_code(ECOverride));
 }
diff --git a/llvm/lib/Object/IRSymtab.cpp b/llvm/lib/Object/IRSymtab.cpp
index a92cb02ed290f..e39cb732add10 100644
--- a/llvm/lib/Object/IRSymtab.cpp
+++ b/llvm/lib/Object/IRSymtab.cpp
@@ -264,9 +264,13 @@ Error Builder::addSymbol(const ModuleSymbolTable &Msymtab,
   Sym.Flags |= unsigned(GV->getVisibility()) << storage::Symbol::FB_visibility;
 
   if (Flags & object::BasicSymbolRef::SF_Common) {
+    auto *GVar = dyn_cast<GlobalVariable>(GV);
+    if (!GVar)
+      return make_error<StringError>("Only variables can have common linkage!",
+                                     inconvertibleErrorCode());
     Uncommon().CommonSize = GV->getParent()->getDataLayout().getTypeAllocSize(
         GV->getType()->getElementType());
-    Uncommon().CommonAlign = GV->getAlignment();
+    Uncommon().CommonAlign = GVar->getAlignment();
   }
 
   const GlobalObject *Base = GV->getBaseObject();
diff --git a/llvm/lib/Object/ModuleSymbolTable.cpp b/llvm/lib/Object/ModuleSymbolTable.cpp
index 45bcf7481890a..7f3055b5dcfab 100644
--- a/llvm/lib/Object/ModuleSymbolTable.cpp
+++ b/llvm/lib/Object/ModuleSymbolTable.cpp
@@ -23,6 +23,7 @@
 #include "llvm/IR/GlobalAlias.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Module.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
@@ -116,6 +117,10 @@ initializeRecordStreamer(const Module &M,
   if (!TAP)
     return;
 
+  // Module-level inline asm is assumed to use At&t syntax (see
+  // AsmPrinter::doInitialization()).
+  Parser->setAssemblerDialect(InlineAsm::AD_ATT);
+
   Parser->setTargetParser(*TAP);
   if (Parser->Run(false))
     return;
diff --git a/llvm/lib/Object/RelocationResolver.cpp b/llvm/lib/Object/RelocationResolver.cpp
index d45da76280ddc..3f3f79b0f4ff7 100644
--- a/llvm/lib/Object/RelocationResolver.cpp
+++ b/llvm/lib/Object/RelocationResolver.cpp
@@ -127,6 +127,27 @@ static uint64_t resolveMips64(RelocationRef R, uint64_t S, uint64_t A) {
   }
 }
 
+static bool supportsMSP430(uint64_t Type) {
+  switch (Type) {
+  case ELF::R_MSP430_32:
+  case ELF::R_MSP430_16_BYTE:
+    return true;
+  default:
+    return false;
+  }
+}
+
+static uint64_t resolveMSP430(RelocationRef R, uint64_t S, uint64_t A) {
+  switch (R.getType()) {
+  case ELF::R_MSP430_32:
+    return (S + getELFAddend(R)) & 0xFFFFFFFF;
+  case ELF::R_MSP430_16_BYTE:
+    return (S + getELFAddend(R)) & 0xFFFF;
+  default:
+    llvm_unreachable("Invalid relocation type");
+  }
+}
+
 static bool supportsPPC64(uint64_t Type) {
   switch (Type) {
   case ELF::R_PPC64_ADDR32:
@@ -614,6 +635,8 @@ getRelocationResolver(const ObjectFile &Obj) {
     case Triple::mipsel:
     case Triple::mips:
       return {supportsMips32, resolveMips32};
+    case Triple::msp430:
+      return {supportsMSP430, resolveMSP430};
     case Triple::sparc:
       return {supportsSparc32, resolveSparc32};
     case Triple::hexagon:
diff --git a/llvm/lib/Object/WasmObjectFile.cpp b/llvm/lib/Object/WasmObjectFile.cpp
index 023948e2046e0..bb2e81d64047f 100644
--- a/llvm/lib/Object/WasmObjectFile.cpp
+++ b/llvm/lib/Object/WasmObjectFile.cpp
@@ -184,6 +184,14 @@ static Error readInitExpr(wasm::WasmInitExpr &Expr,
   case wasm::WASM_OPCODE_GLOBAL_GET:
     Expr.Value.Global = readULEB128(Ctx);
     break;
+  case wasm::WASM_OPCODE_REF_NULL: {
+    wasm::ValType Ty = static_cast<wasm::ValType>(readULEB128(Ctx));
+    if (Ty != wasm::ValType::EXTERNREF) {
+      return make_error<GenericBinaryError>("Invalid type for ref.null",
+                                            object_error::parse_failed);
+    }
+    break;
+  }
   default:
     return make_error<GenericBinaryError>("Invalid opcode in init_expr",
                                           object_error::parse_failed);
@@ -200,9 +208,9 @@ static Error readInitExpr(wasm::WasmInitExpr &Expr,
 static wasm::WasmLimits readLimits(WasmObjectFile::ReadContext &Ctx) {
   wasm::WasmLimits Result;
   Result.Flags = readVaruint32(Ctx);
-  Result.Initial = readVaruint32(Ctx);
+  Result.Initial = readVaruint64(Ctx);
   if (Result.Flags & wasm::WASM_LIMITS_FLAG_HAS_MAX)
-    Result.Maximum = readVaruint32(Ctx);
+    Result.Maximum = readVaruint64(Ctx);
   return Result;
 }
 
diff --git a/llvm/lib/ObjectYAML/DWARFEmitter.cpp b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
index 6c60f0423c00d..a8b467af7b2d5 100644
--- a/llvm/lib/ObjectYAML/DWARFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
@@ -23,6 +23,7 @@
 #include "llvm/Support/LEB128.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/SwapByteOrder.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
@@ -110,6 +111,10 @@ Error DWARFYAML::emitDebugAbbrev(raw_ostream &OS, const DWARFYAML::Data &DI) {
     encodeULEB128(0, OS);
   }
 
+  // The abbreviations for a given compilation unit end with an entry consisting
+  // of a 0 byte for the abbreviation code.
+  OS.write_zeros(1);
+
   return Error::success();
 }
 
@@ -183,15 +188,15 @@ Error DWARFYAML::emitDebugRanges(raw_ostream &OS, const DWARFYAML::Data &DI) {
 
 Error DWARFYAML::emitPubSection(raw_ostream &OS,
                                 const DWARFYAML::PubSection &Sect,
-                                bool IsLittleEndian) {
+                                bool IsLittleEndian, bool IsGNUPubSec) {
   writeInitialLength(Sect.Length, OS, IsLittleEndian);
   writeInteger((uint16_t)Sect.Version, OS, IsLittleEndian);
   writeInteger((uint32_t)Sect.UnitOffset, OS, IsLittleEndian);
   writeInteger((uint32_t)Sect.UnitSize, OS, IsLittleEndian);
   for (auto Entry : Sect.Entries) {
     writeInteger((uint32_t)Entry.DieOffset, OS, IsLittleEndian);
-    if (Sect.IsGNUStyle)
-      writeInteger((uint32_t)Entry.Descriptor, OS, IsLittleEndian);
+    if (IsGNUPubSec)
+      writeInteger((uint8_t)Entry.Descriptor, OS, IsLittleEndian);
     OS.write(Entry.Name.data(), Entry.Name.size());
     OS.write('\0');
   }
@@ -207,14 +212,18 @@ class DumpVisitor : public DWARFYAML::ConstVisitor {
 
 protected:
   void onStartCompileUnit(const DWARFYAML::Unit &CU) override {
-    writeInitialLength(CU.Length, OS, DebugInfo.IsLittleEndian);
+    writeInitialLength(CU.Format, CU.Length, OS, DebugInfo.IsLittleEndian);
     writeInteger((uint16_t)CU.Version, OS, DebugInfo.IsLittleEndian);
-    if(CU.Version >= 5) {
+    if (CU.Version >= 5) {
       writeInteger((uint8_t)CU.Type, OS, DebugInfo.IsLittleEndian);
       writeInteger((uint8_t)CU.AddrSize, OS, DebugInfo.IsLittleEndian);
-      writeInteger((uint32_t)CU.AbbrOffset, OS, DebugInfo.IsLittleEndian);
-    }else {
-      writeInteger((uint32_t)CU.AbbrOffset, OS, DebugInfo.IsLittleEndian);
+      cantFail(writeVariableSizedInteger(CU.AbbrOffset,
+                                         CU.Format == dwarf::DWARF64 ? 8 : 4,
+                                         OS, DebugInfo.IsLittleEndian));
+    } else {
+      cantFail(writeVariableSizedInteger(CU.AbbrOffset,
+                                         CU.Format == dwarf::DWARF64 ? 8 : 4,
+                                         OS, DebugInfo.IsLittleEndian));
       writeInteger((uint8_t)CU.AddrSize, OS, DebugInfo.IsLittleEndian);
     }
   }
@@ -267,9 +276,7 @@ class DumpVisitor : public DWARFYAML::ConstVisitor {
 
 Error DWARFYAML::emitDebugInfo(raw_ostream &OS, const DWARFYAML::Data &DI) {
   DumpVisitor Visitor(DI, OS);
-  Visitor.traverseDebugInfo();
-
-  return Error::success();
+  return Visitor.traverseDebugInfo();
 }
 
 static void emitFileEntry(raw_ostream &OS, const DWARFYAML::File &File) {
@@ -433,38 +440,36 @@ class DIEFixupVisitor : public DWARFYAML::Visitor {
 public:
   DIEFixupVisitor(DWARFYAML::Data &DI) : DWARFYAML::Visitor(DI){};
 
-private:
-  virtual void onStartCompileUnit(DWARFYAML::Unit &CU) {
+protected:
+  void onStartCompileUnit(DWARFYAML::Unit &CU) override {
     // Size of the unit header, excluding the length field itself.
     Length = CU.Version >= 5 ? 8 : 7;
   }
 
-  virtual void onEndCompileUnit(DWARFYAML::Unit &CU) {
-    CU.Length.setLength(Length);
-  }
+  void onEndCompileUnit(DWARFYAML::Unit &CU) override { CU.Length = Length; }
 
-  virtual void onStartDIE(DWARFYAML::Unit &CU, DWARFYAML::Entry &DIE) {
+  void onStartDIE(DWARFYAML::Unit &CU, DWARFYAML::Entry &DIE) override {
     Length += getULEB128Size(DIE.AbbrCode);
   }
 
-  virtual void onValue(const uint8_t U) { Length += 1; }
-  virtual void onValue(const uint16_t U) { Length += 2; }
-  virtual void onValue(const uint32_t U) { Length += 4; }
-  virtual void onValue(const uint64_t U, const bool LEB = false) {
+  void onValue(const uint8_t U) override { Length += 1; }
+  void onValue(const uint16_t U) override { Length += 2; }
+  void onValue(const uint32_t U) override { Length += 4; }
+  void onValue(const uint64_t U, const bool LEB = false) override {
     if (LEB)
       Length += getULEB128Size(U);
     else
       Length += 8;
   }
-  virtual void onValue(const int64_t S, const bool LEB = false) {
+  void onValue(const int64_t S, const bool LEB = false) override {
     if (LEB)
       Length += getSLEB128Size(S);
     else
       Length += 8;
   }
-  virtual void onValue(const StringRef String) { Length += String.size() + 1; }
+  void onValue(const StringRef String) override { Length += String.size() + 1; }
 
-  virtual void onValue(const MemoryBufferRef MBR) {
+  void onValue(const MemoryBufferRef MBR) override {
     Length += MBR.getBufferSize();
   }
 };
@@ -473,17 +478,24 @@ class DIEFixupVisitor : public DWARFYAML::Visitor {
 Expected<StringMap<std::unique_ptr<MemoryBuffer>>>
 DWARFYAML::emitDebugSections(StringRef YAMLString, bool ApplyFixups,
                              bool IsLittleEndian) {
-  yaml::Input YIn(YAMLString);
+  auto CollectDiagnostic = [](const SMDiagnostic &Diag, void *DiagContext) {
+    *static_cast<SMDiagnostic *>(DiagContext) = Diag;
+  };
+
+  SMDiagnostic GeneratedDiag;
+  yaml::Input YIn(YAMLString, /*Ctxt=*/nullptr, CollectDiagnostic,
+                  &GeneratedDiag);
 
   DWARFYAML::Data DI;
   DI.IsLittleEndian = IsLittleEndian;
   YIn >> DI;
   if (YIn.error())
-    return errorCodeToError(YIn.error());
+    return createStringError(YIn.error(), GeneratedDiag.getMessage());
 
   if (ApplyFixups) {
     DIEFixupVisitor DIFixer(DI);
-    DIFixer.traverseDebugInfo();
+    if (Error Err = DIFixer.traverseDebugInfo())
+      return std::move(Err);
   }
 
   StringMap<std::unique_ptr<MemoryBuffer>> DebugSections;
diff --git a/llvm/lib/ObjectYAML/DWARFVisitor.cpp b/llvm/lib/ObjectYAML/DWARFVisitor.cpp
index fa443cf9e00db..f478a1b84397f 100644
--- a/llvm/lib/ObjectYAML/DWARFVisitor.cpp
+++ b/llvm/lib/ObjectYAML/DWARFVisitor.cpp
@@ -9,7 +9,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "DWARFVisitor.h"
+#include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/ObjectYAML/DWARFYAML.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
 
 using namespace llvm;
 
@@ -34,7 +37,7 @@ void DWARFYAML::VisitorImpl<T>::onVariableSizeValue(uint64_t U, unsigned Size) {
 }
 
 static unsigned getOffsetSize(const DWARFYAML::Unit &Unit) {
-  return Unit.Length.isDWARF64() ? 8 : 4;
+  return Unit.Format == dwarf::DWARF64 ? 8 : 4;
 }
 
 static unsigned getRefSize(const DWARFYAML::Unit &Unit) {
@@ -43,18 +46,24 @@ static unsigned getRefSize(const DWARFYAML::Unit &Unit) {
   return getOffsetSize(Unit);
 }
 
-template <typename T> void DWARFYAML::VisitorImpl<T>::traverseDebugInfo() {
+template <typename T> Error DWARFYAML::VisitorImpl<T>::traverseDebugInfo() {
   for (auto &Unit : DebugInfo.CompileUnits) {
     onStartCompileUnit(Unit);
     if (Unit.Entries.empty())
       continue;
-    auto FirstAbbrevCode = Unit.Entries[0].AbbrCode;
 
     for (auto &Entry : Unit.Entries) {
       onStartDIE(Unit, Entry);
-      if (Entry.AbbrCode == 0u)
+      uint32_t AbbrCode = Entry.AbbrCode;
+      if (AbbrCode == 0 || Entry.Values.empty())
         continue;
-      auto &Abbrev = DebugInfo.AbbrevDecls[Entry.AbbrCode - FirstAbbrevCode];
+
+      if (AbbrCode > DebugInfo.AbbrevDecls.size())
+        return createStringError(
+            errc::invalid_argument,
+            "abbrev code must be less than or equal to the number of "
+            "entries in abbreviation table");
+      const DWARFYAML::Abbrev &Abbrev = DebugInfo.AbbrevDecls[AbbrCode - 1];
       auto FormVal = Entry.Values.begin();
       auto AbbrForm = Abbrev.Attributes.begin();
       for (;
@@ -172,6 +181,8 @@ template <typename T> void DWARFYAML::VisitorImpl<T>::traverseDebugInfo() {
     }
     onEndCompileUnit(Unit);
   }
+
+  return Error::success();
 }
 
 // Explicitly instantiate the two template expansions.
diff --git a/llvm/lib/ObjectYAML/DWARFVisitor.h b/llvm/lib/ObjectYAML/DWARFVisitor.h
index 50e88aa7a26bf..3b2c4303c7f7f 100644
--- a/llvm/lib/ObjectYAML/DWARFVisitor.h
+++ b/llvm/lib/ObjectYAML/DWARFVisitor.h
@@ -16,6 +16,7 @@
 #include "llvm/Support/MemoryBuffer.h"
 
 namespace llvm {
+class Error;
 
 namespace DWARFYAML {
 
@@ -68,7 +69,7 @@ template <typename T> class VisitorImpl {
 
   virtual ~VisitorImpl() {}
 
-  void traverseDebugInfo();
+  Error traverseDebugInfo();
 
 private:
   void onVariableSizeValue(uint64_t U, unsigned Size);
diff --git a/llvm/lib/ObjectYAML/DWARFYAML.cpp b/llvm/lib/ObjectYAML/DWARFYAML.cpp
index 1f62fb7e2e561..bedf31dc8179f 100644
--- a/llvm/lib/ObjectYAML/DWARFYAML.cpp
+++ b/llvm/lib/ObjectYAML/DWARFYAML.cpp
@@ -38,14 +38,23 @@ SetVector<StringRef> DWARFYAML::Data::getUsedSectionNames() const {
     SecNames.insert("debug_abbrev");
   if (!CompileUnits.empty())
     SecNames.insert("debug_info");
+  if (PubNames)
+    SecNames.insert("debug_pubnames");
+  if (PubTypes)
+    SecNames.insert("debug_pubtypes");
+  if (GNUPubNames)
+    SecNames.insert("debug_gnu_pubnames");
+  if (GNUPubTypes)
+    SecNames.insert("debug_gnu_pubtypes");
   return SecNames;
 }
 
 namespace yaml {
 
 void MappingTraits<DWARFYAML::Data>::mapping(IO &IO, DWARFYAML::Data &DWARF) {
-  auto oldContext = IO.getContext();
-  IO.setContext(&DWARF);
+  void *OldContext = IO.getContext();
+  DWARFYAML::DWARFContext DWARFCtx;
+  IO.setContext(&DWARFCtx);
   IO.mapOptional("debug_str", DWARF.DebugStrings);
   IO.mapOptional("debug_abbrev", DWARF.AbbrevDecls);
   if (!DWARF.ARanges.empty() || !IO.outputting())
@@ -54,12 +63,13 @@ void MappingTraits<DWARFYAML::Data>::mapping(IO &IO, DWARFYAML::Data &DWARF) {
     IO.mapOptional("debug_ranges", DWARF.DebugRanges);
   IO.mapOptional("debug_pubnames", DWARF.PubNames);
   IO.mapOptional("debug_pubtypes", DWARF.PubTypes);
+  DWARFCtx.IsGNUPubSec = true;
   IO.mapOptional("debug_gnu_pubnames", DWARF.GNUPubNames);
   IO.mapOptional("debug_gnu_pubtypes", DWARF.GNUPubTypes);
   IO.mapOptional("debug_info", DWARF.CompileUnits);
   IO.mapOptional("debug_line", DWARF.DebugLines);
   IO.mapOptional("debug_addr", DWARF.DebugAddr);
-  IO.setContext(&oldContext);
+  IO.setContext(OldContext);
 }
 
 void MappingTraits<DWARFYAML::Abbrev>::mapping(IO &IO,
@@ -111,26 +121,22 @@ void MappingTraits<DWARFYAML::Ranges>::mapping(IO &IO,
 void MappingTraits<DWARFYAML::PubEntry>::mapping(IO &IO,
                                                  DWARFYAML::PubEntry &Entry) {
   IO.mapRequired("DieOffset", Entry.DieOffset);
-  if (reinterpret_cast<DWARFYAML::PubSection *>(IO.getContext())->IsGNUStyle)
+  if (static_cast<DWARFYAML::DWARFContext *>(IO.getContext())->IsGNUPubSec)
     IO.mapRequired("Descriptor", Entry.Descriptor);
   IO.mapRequired("Name", Entry.Name);
 }
 
 void MappingTraits<DWARFYAML::PubSection>::mapping(
     IO &IO, DWARFYAML::PubSection &Section) {
-  auto OldContext = IO.getContext();
-  IO.setContext(&Section);
-
   IO.mapRequired("Length", Section.Length);
   IO.mapRequired("Version", Section.Version);
   IO.mapRequired("UnitOffset", Section.UnitOffset);
   IO.mapRequired("UnitSize", Section.UnitSize);
   IO.mapRequired("Entries", Section.Entries);
-
-  IO.setContext(OldContext);
 }
 
 void MappingTraits<DWARFYAML::Unit>::mapping(IO &IO, DWARFYAML::Unit &Unit) {
+  IO.mapOptional("Format", Unit.Format, dwarf::DWARF32);
   IO.mapRequired("Length", Unit.Length);
   IO.mapRequired("Version", Unit.Version);
   if (Unit.Version >= 5)
diff --git a/llvm/lib/ObjectYAML/ELFEmitter.cpp b/llvm/lib/ObjectYAML/ELFEmitter.cpp
index 7b5753be99207..218e7df8e39a5 100644
--- a/llvm/lib/ObjectYAML/ELFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/ELFEmitter.cpp
@@ -943,6 +943,16 @@ Expected<uint64_t> emitDWARF(typename ELFT::Shdr &SHeader, StringRef Name,
     Err = DWARFYAML::emitDebugAbbrev(*OS, DWARF);
   else if (Name == ".debug_info")
     Err = DWARFYAML::emitDebugInfo(*OS, DWARF);
+  else if (Name == ".debug_pubnames")
+    Err = DWARFYAML::emitPubSection(*OS, *DWARF.PubNames, DWARF.IsLittleEndian);
+  else if (Name == ".debug_pubtypes")
+    Err = DWARFYAML::emitPubSection(*OS, *DWARF.PubTypes, DWARF.IsLittleEndian);
+  else if (Name == ".debug_gnu_pubnames")
+    Err = DWARFYAML::emitPubSection(*OS, *DWARF.GNUPubNames,
+                                    DWARF.IsLittleEndian, /*IsGNUStyle=*/true);
+  else if (Name == ".debug_gnu_pubtypes")
+    Err = DWARFYAML::emitPubSection(*OS, *DWARF.GNUPubTypes,
+                                    DWARF.IsLittleEndian, /*IsGNUStyle=*/true);
   else
     llvm_unreachable("unexpected emitDWARF() call");
 
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index 13f4e65af5d35..51ca5dd726216 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -1102,6 +1102,17 @@ static void sectionMapping(IO &IO, ELFYAML::DynamicSection &Section) {
 static void sectionMapping(IO &IO, ELFYAML::RawContentSection &Section) {
   commonSectionMapping(IO, Section);
   IO.mapOptional("Content", Section.Content);
+
+  // We also support reading a content as array of bytes using the ContentArray
+  // key. obj2yaml never prints this field.
+  assert(!IO.outputting() || !Section.ContentBuf.hasValue());
+  IO.mapOptional("ContentArray", Section.ContentBuf);
+  if (Section.ContentBuf) {
+    if (Section.Content)
+      IO.setError("Content and ContentArray can't be used together");
+    Section.Content = yaml::BinaryRef(*Section.ContentBuf);
+  }
+
   IO.mapOptional("Size", Section.Size);
   IO.mapOptional("Info", Section.Info);
 }
diff --git a/llvm/lib/ObjectYAML/WasmYAML.cpp b/llvm/lib/ObjectYAML/WasmYAML.cpp
index 3e9ba9122428f..d1aa1181a3445 100644
--- a/llvm/lib/ObjectYAML/WasmYAML.cpp
+++ b/llvm/lib/ObjectYAML/WasmYAML.cpp
@@ -433,6 +433,11 @@ void MappingTraits<wasm::WasmInitExpr>::mapping(IO &IO,
   case wasm::WASM_OPCODE_GLOBAL_GET:
     IO.mapRequired("Index", Expr.Value.Global);
     break;
+  case wasm::WASM_OPCODE_REF_NULL: {
+    WasmYAML::ValueType Ty = wasm::WASM_TYPE_EXTERNREF;
+    IO.mapRequired("Type", Ty);
+    break;
+  }
   }
 }
 
@@ -517,6 +522,7 @@ void ScalarBitSetTraits<WasmYAML::LimitFlags>::bitset(
 #define BCase(X) IO.bitSetCase(Value, #X, wasm::WASM_LIMITS_FLAG_##X)
   BCase(HAS_MAX);
   BCase(IS_SHARED);
+  BCase(IS_64);
 #undef BCase
 }
 
@@ -559,6 +565,8 @@ void ScalarEnumerationTraits<WasmYAML::ValueType>::enumeration(
   ECase(F64);
   ECase(V128);
   ECase(FUNCREF);
+  ECase(EXNREF);
+  ECase(EXTERNREF);
   ECase(FUNC);
 #undef ECase
 }
@@ -583,6 +591,7 @@ void ScalarEnumerationTraits<WasmYAML::Opcode>::enumeration(
   ECase(F64_CONST);
   ECase(F32_CONST);
   ECase(GLOBAL_GET);
+  ECase(REF_NULL);
 #undef ECase
 }
 
diff --git a/llvm/lib/Passes/LLVMBuild.txt b/llvm/lib/Passes/LLVMBuild.txt
index 438fc5c7c2d46..4a269467a934a 100644
--- a/llvm/lib/Passes/LLVMBuild.txt
+++ b/llvm/lib/Passes/LLVMBuild.txt
@@ -18,4 +18,4 @@
 type = Library
 name = Passes
 parent = Libraries
-required_libraries = AggressiveInstCombine Analysis CodeGen Core Coroutines IPO InstCombine Scalar Support Target TransformUtils Vectorize Instrumentation
+required_libraries = AggressiveInstCombine Analysis Core Coroutines IPO InstCombine Scalar Support Target TransformUtils Vectorize Instrumentation
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index d7f2a64e94d8d..771cdfd17aa54 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -57,9 +57,6 @@
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
-#include "llvm/CodeGen/MachineModuleInfo.h"
-#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
-#include "llvm/CodeGen/UnreachableBlockElim.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/IRPrintingPasses.h"
 #include "llvm/IR/PassManager.h"
@@ -251,10 +248,6 @@ static cl::opt<bool>
     EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden,
               cl::desc("Enable control height reduction optimization (CHR)"));
 
-static cl::opt<bool> EnableCallGraphProfile(
-    "enable-npm-call-graph-profile", cl::init(true), cl::Hidden,
-    cl::desc("Enable call graph profile pass for the new PM (default = on)"));
-
 /// Flag to enable inline deferral during PGO.
 static cl::opt<bool>
     EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true),
@@ -270,7 +263,7 @@ PipelineTuningOptions::PipelineTuningOptions() {
   Coroutines = false;
   LicmMssaOptCap = SetLicmMssaOptCap;
   LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
-  CallGraphProfile = EnableCallGraphProfile;
+  CallGraphProfile = true;
 }
 
 extern cl::opt<bool> EnableHotColdSplit;
@@ -302,11 +295,16 @@ const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Oz = {
 
 namespace {
 
+// The following passes/analyses have custom names, otherwise their name will
+// include `(anonymous namespace)`. These are special since they are only for
+// testing purposes and don't live in a header file.
+
 /// No-op module pass which does nothing.
-struct NoOpModulePass {
+struct NoOpModulePass : PassInfoMixin<NoOpModulePass> {
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &) {
     return PreservedAnalyses::all();
   }
+
   static StringRef name() { return "NoOpModulePass"; }
 };
 
@@ -322,7 +320,7 @@ class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
 };
 
 /// No-op CGSCC pass which does nothing.
-struct NoOpCGSCCPass {
+struct NoOpCGSCCPass : PassInfoMixin<NoOpCGSCCPass> {
   PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &,
                         LazyCallGraph &, CGSCCUpdateResult &UR) {
     return PreservedAnalyses::all();
@@ -344,7 +342,7 @@ class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
 };
 
 /// No-op function pass which does nothing.
-struct NoOpFunctionPass {
+struct NoOpFunctionPass : PassInfoMixin<NoOpFunctionPass> {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &) {
     return PreservedAnalyses::all();
   }
@@ -363,7 +361,7 @@ class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
 };
 
 /// No-op loop pass which does nothing.
-struct NoOpLoopPass {
+struct NoOpLoopPass : PassInfoMixin<NoOpLoopPass> {
   PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
                         LoopStandardAnalysisResults &, LPMUpdater &) {
     return PreservedAnalyses::all();
@@ -389,7 +387,7 @@ AnalysisKey NoOpCGSCCAnalysis::Key;
 AnalysisKey NoOpFunctionAnalysis::Key;
 AnalysisKey NoOpLoopAnalysis::Key;
 
-} // End anonymous namespace.
+} // namespace
 
 void PassBuilder::invokePeepholeEPCallbacks(
     FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
@@ -511,8 +509,7 @@ FunctionPassManager PassBuilder::buildO1FunctionSimplificationPipeline(
       std::move(LPM1), EnableMSSALoopDependency, DebugLogging));
   FPM.addPass(SimplifyCFGPass());
   FPM.addPass(InstCombinePass());
-  // The loop passes in LPM2 (IndVarSimplifyPass, LoopIdiomRecognizePass,
-  // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
+  // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
   // *All* loop passes must preserve it, in order to be able to use it.
   FPM.addPass(createFunctionToLoopPassAdaptor(
       std::move(LPM2), /*UseMemorySSA=*/false, DebugLogging));
@@ -588,7 +585,7 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
   }
 
   // Speculative execution if the target has divergent branches; otherwise nop.
-  FPM.addPass(SpeculativeExecutionPass());
+  FPM.addPass(SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
 
   // Optimize based on known information about branches, and cleanup afterward.
   FPM.addPass(JumpThreadingPass());
@@ -2660,3 +2657,11 @@ Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
 
   return Error::success();
 }
+
+bool PassBuilder::isAAPassName(StringRef PassName) {
+#define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS)                             \
+  if (PassName == NAME)                                                        \
+    return true;
+#include "PassRegistry.def"
+  return false;
+}
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 1edb0fe5c2306..eb2b740db5612 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -73,7 +73,6 @@ MODULE_PASS("partial-inliner", PartialInlinerPass())
 MODULE_PASS("pgo-icall-prom", PGOIndirectCallPromotion())
 MODULE_PASS("pgo-instr-gen", PGOInstrumentationGen())
 MODULE_PASS("pgo-instr-use", PGOInstrumentationUse())
-MODULE_PASS("pre-isel-intrinsic-lowering", PreISelIntrinsicLoweringPass())
 MODULE_PASS("print-profile-summary", ProfileSummaryPrinterPass(dbgs()))
 MODULE_PASS("print-callgraph", CallGraphPrinterPass(dbgs()))
 MODULE_PASS("print", PrintModulePass(dbgs()))
@@ -210,6 +209,7 @@ FUNCTION_PASS("guard-widening", GuardWideningPass())
 FUNCTION_PASS("load-store-vectorizer", LoadStoreVectorizerPass())
 FUNCTION_PASS("loop-simplify", LoopSimplifyPass())
 FUNCTION_PASS("loop-sink", LoopSinkPass())
+FUNCTION_PASS("loop-unroll-and-jam", LoopUnrollAndJamPass())
 FUNCTION_PASS("lowerinvoke", LowerInvokePass())
 FUNCTION_PASS("mem2reg", PromotePass())
 FUNCTION_PASS("memcpyopt", MemCpyOptPass())
@@ -221,7 +221,7 @@ FUNCTION_PASS("partially-inline-libcalls", PartiallyInlineLibCallsPass())
 FUNCTION_PASS("lcssa", LCSSAPass())
 FUNCTION_PASS("loop-data-prefetch", LoopDataPrefetchPass())
 FUNCTION_PASS("loop-load-elim", LoopLoadEliminationPass())
-FUNCTION_PASS("loop-fuse", LoopFusePass())
+FUNCTION_PASS("loop-fusion", LoopFusePass())
 FUNCTION_PASS("loop-distribute", LoopDistributePass())
 FUNCTION_PASS("pgo-memop-opt", PGOMemOPSizeOpt())
 FUNCTION_PASS("print", PrintFunctionPass(dbgs()))
@@ -233,23 +233,24 @@ FUNCTION_PASS("print<domtree>", DominatorTreePrinterPass(dbgs()))
 FUNCTION_PASS("print<postdomtree>", PostDominatorTreePrinterPass(dbgs()))
 FUNCTION_PASS("print<demanded-bits>", DemandedBitsPrinterPass(dbgs()))
 FUNCTION_PASS("print<domfrontier>", DominanceFrontierPrinterPass(dbgs()))
+FUNCTION_PASS("print<inline-cost>", InlineCostAnnotationPrinterPass(dbgs()))
 FUNCTION_PASS("print<loops>", LoopPrinterPass(dbgs()))
 FUNCTION_PASS("print<memoryssa>", MemorySSAPrinterPass(dbgs()))
 FUNCTION_PASS("print<phi-values>", PhiValuesPrinterPass(dbgs()))
 FUNCTION_PASS("print<regions>", RegionInfoPrinterPass(dbgs()))
 FUNCTION_PASS("print<scalar-evolution>", ScalarEvolutionPrinterPass(dbgs()))
 FUNCTION_PASS("print<stack-safety-local>", StackSafetyPrinterPass(dbgs()))
+FUNCTION_PASS("print-predicateinfo", PredicateInfoPrinterPass(dbgs()))
 FUNCTION_PASS("reassociate", ReassociatePass())
 FUNCTION_PASS("scalarizer", ScalarizerPass())
 FUNCTION_PASS("sccp", SCCPPass())
+FUNCTION_PASS("simplifycfg", SimplifyCFGPass())
 FUNCTION_PASS("sink", SinkingPass())
 FUNCTION_PASS("slp-vectorizer", SLPVectorizerPass())
 FUNCTION_PASS("speculative-execution", SpeculativeExecutionPass())
 FUNCTION_PASS("spec-phis", SpeculateAroundPHIsPass())
 FUNCTION_PASS("sroa", SROA())
 FUNCTION_PASS("tailcallelim", TailCallElimPass())
-FUNCTION_PASS("unreachableblockelim", UnreachableBlockElimPass())
-FUNCTION_PASS("unroll-and-jam", LoopUnrollAndJamPass())
 FUNCTION_PASS("vector-combine", VectorCombinePass())
 FUNCTION_PASS("verify", VerifierPass())
 FUNCTION_PASS("verify<domtree>", DominatorTreeVerifierPass())
@@ -271,7 +272,7 @@ FUNCTION_PASS("tsan", ThreadSanitizerPass())
 #ifndef FUNCTION_PASS_WITH_PARAMS
 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)
 #endif
-FUNCTION_PASS_WITH_PARAMS("unroll",
+FUNCTION_PASS_WITH_PARAMS("loop-unroll",
                            [](LoopUnrollOptions Opts) {
                              return LoopUnrollPass(Opts);
                            },
@@ -331,9 +332,9 @@ LOOP_PASS("no-op-loop", NoOpLoopPass())
 LOOP_PASS("print", PrintLoopPass(dbgs()))
 LOOP_PASS("loop-deletion", LoopDeletionPass())
 LOOP_PASS("simplify-cfg", LoopSimplifyCFGPass())
-LOOP_PASS("strength-reduce", LoopStrengthReducePass())
+LOOP_PASS("loop-reduce", LoopStrengthReducePass())
 LOOP_PASS("indvars", IndVarSimplifyPass())
-LOOP_PASS("unroll-full", LoopFullUnrollPass())
+LOOP_PASS("loop-unroll-full", LoopFullUnrollPass())
 LOOP_PASS("print-access-info", LoopAccessInfoPrinterPass(dbgs()))
 LOOP_PASS("print<ddg>", DDGAnalysisPrinterPass(dbgs()))
 LOOP_PASS("print<ivusers>", IVUsersPrinterPass(dbgs()))
@@ -341,6 +342,7 @@ LOOP_PASS("print<loopnest>", LoopNestPrinterPass(dbgs()))
 LOOP_PASS("print<loop-cache-cost>", LoopCachePrinterPass(dbgs()))
 LOOP_PASS("loop-predication", LoopPredicationPass())
 LOOP_PASS("guard-widening", GuardWideningPass())
+LOOP_PASS("simple-loop-unswitch", SimpleLoopUnswitchPass())
 #undef LOOP_PASS
 
 #ifndef LOOP_PASS_WITH_PARAMS
diff --git a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp
index 3299b5f920694..5d3a076409427 100644
--- a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp
+++ b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp
@@ -31,6 +31,19 @@ static const uint32_t DefaultCutoffsData[] = {
 const ArrayRef<uint32_t> ProfileSummaryBuilder::DefaultCutoffs =
     DefaultCutoffsData;
 
+const ProfileSummaryEntry &
+ProfileSummaryBuilder::getEntryForPercentile(SummaryEntryVector &DS,
+                                             uint64_t Percentile) {
+  auto It = partition_point(DS, [=](const ProfileSummaryEntry &Entry) {
+    return Entry.Cutoff < Percentile;
+  });
+  // The required percentile has to be <= one of the percentiles in the
+  // detailed summary.
+  if (It == DS.end())
+    report_fatal_error("Desired percentile exceeds the maximum cutoff");
+  return *It;
+}
+
 void InstrProfSummaryBuilder::addRecord(const InstrProfRecord &R) {
   // The first counter is not necessarily an entry count for IR
   // instrumentation profiles.
diff --git a/llvm/lib/Support/AMDGPUMetadata.cpp b/llvm/lib/Support/AMDGPUMetadata.cpp
index 4ea197a973894..bfa1fe86cd3ed 100644
--- a/llvm/lib/Support/AMDGPUMetadata.cpp
+++ b/llvm/lib/Support/AMDGPUMetadata.cpp
@@ -111,7 +111,11 @@ struct MappingTraits<Kernel::Arg::Metadata> {
     YIO.mapRequired(Kernel::Arg::Key::Size, MD.mSize);
     YIO.mapRequired(Kernel::Arg::Key::Align, MD.mAlign);
     YIO.mapRequired(Kernel::Arg::Key::ValueKind, MD.mValueKind);
-    YIO.mapRequired(Kernel::Arg::Key::ValueType, MD.mValueType);
+
+    // Removed. Accepted for parsing compatibility, but not emitted.
+    Optional<ValueType> Unused;
+    YIO.mapOptional(Kernel::Arg::Key::ValueType, Unused);
+
     YIO.mapOptional(Kernel::Arg::Key::PointeeAlign, MD.mPointeeAlign,
                     uint32_t(0));
     YIO.mapOptional(Kernel::Arg::Key::AddrSpaceQual, MD.mAddrSpaceQual,
diff --git a/llvm/lib/Support/CommandLine.cpp b/llvm/lib/Support/CommandLine.cpp
index cee96083f7000..12ef0d511b147 100644
--- a/llvm/lib/Support/CommandLine.cpp
+++ b/llvm/lib/Support/CommandLine.cpp
@@ -2537,7 +2537,7 @@ class VersionPrinter {
     // information.
     if (ExtraVersionPrinters != nullptr) {
       outs() << '\n';
-      for (auto I : *ExtraVersionPrinters)
+      for (const auto &I : *ExtraVersionPrinters)
         I(outs());
     }
 
diff --git a/llvm/lib/Support/DebugCounter.cpp b/llvm/lib/Support/DebugCounter.cpp
index 713b419698188..8c579f395282e 100644
--- a/llvm/lib/Support/DebugCounter.cpp
+++ b/llvm/lib/Support/DebugCounter.cpp
@@ -31,7 +31,7 @@ class DebugCounterList : public cl::list<std::string, DebugCounter> {
     // width, so we do the same.
     Option::printHelpStr(HelpStr, GlobalWidth, ArgStr.size() + 6);
     const auto &CounterInstance = DebugCounter::instance();
-    for (auto Name : CounterInstance) {
+    for (const auto &Name : CounterInstance) {
       const auto Info =
           CounterInstance.getCounterInfo(CounterInstance.getCounterId(Name));
       size_t NumSpaces = GlobalWidth - Info.first.size() - 8;
diff --git a/llvm/lib/Support/FileCollector.cpp b/llvm/lib/Support/FileCollector.cpp
index cb83d845695e8..59755556a5a3a 100644
--- a/llvm/lib/Support/FileCollector.cpp
+++ b/llvm/lib/Support/FileCollector.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Support/FileCollector.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
diff --git a/llvm/lib/Support/FormattedStream.cpp b/llvm/lib/Support/FormattedStream.cpp
index 4eb747038bb9e..5716afc187e48 100644
--- a/llvm/lib/Support/FormattedStream.cpp
+++ b/llvm/lib/Support/FormattedStream.cpp
@@ -11,7 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/FormattedStream.h"
+#include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/Unicode.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 
@@ -19,16 +21,22 @@ using namespace llvm;
 
 /// UpdatePosition - Examine the given char sequence and figure out which
 /// column we end up in after output, and how many line breaks are contained.
-///
-static void UpdatePosition(std::pair<unsigned, unsigned> &Position, const char *Ptr, size_t Size) {
+/// This assumes that the input string is well-formed UTF-8, and takes into
+/// account Unicode characters which render as multiple columns wide.
+void formatted_raw_ostream::UpdatePosition(const char *Ptr, size_t Size) {
   unsigned &Column = Position.first;
   unsigned &Line = Position.second;
 
-  // Keep track of the current column and line by scanning the string for
-  // special characters
-  for (const char *End = Ptr + Size; Ptr != End; ++Ptr) {
-    ++Column;
-    switch (*Ptr) {
+  auto ProcessUTF8CodePoint = [&Line, &Column](StringRef CP) {
+    int Width = sys::unicode::columnWidthUTF8(CP);
+    if (Width != sys::unicode::ErrorNonPrintableCharacter)
+      Column += Width;
+
+    // The only special whitespace characters we care about are single-byte.
+    if (CP.size() > 1)
+      return;
+
+    switch (CP[0]) {
     case '\n':
       Line += 1;
       LLVM_FALLTHROUGH;
@@ -40,6 +48,46 @@ static void UpdatePosition(std::pair<unsigned, unsigned> &Position, const char *
       Column += (8 - (Column & 0x7)) & 0x7;
       break;
     }
+  };
+
+  // If we have a partial UTF-8 sequence from the previous buffer, check that
+  // first.
+  if (PartialUTF8Char.size()) {
+    size_t BytesFromBuffer =
+        getNumBytesForUTF8(PartialUTF8Char[0]) - PartialUTF8Char.size();
+    if (Size < BytesFromBuffer) {
+      // If we still don't have enough bytes for a complete code point, just
+      // append what we have.
+      PartialUTF8Char.append(StringRef(Ptr, Size));
+      return;
+    } else {
+      // The first few bytes from the buffer will complete the code point.
+      // Concatenate them and process their effect on the line and column
+      // numbers.
+      PartialUTF8Char.append(StringRef(Ptr, BytesFromBuffer));
+      ProcessUTF8CodePoint(PartialUTF8Char);
+      PartialUTF8Char.clear();
+      Ptr += BytesFromBuffer;
+      Size -= BytesFromBuffer;
+    }
+  }
+
+  // Now scan the rest of the buffer.
+  unsigned NumBytes;
+  for (const char *End = Ptr + Size; Ptr < End; Ptr += NumBytes) {
+    NumBytes = getNumBytesForUTF8(*Ptr);
+
+    // The buffer might end part way through a UTF-8 code unit sequence for a
+    // Unicode scalar value if it got flushed. If this happens, we can't know
+    // the display width until we see the rest of the code point. Stash the
+    // bytes we do have, so that we can reconstruct the whole code point later,
+    // even if the buffer is being flushed.
+    if ((unsigned)(End - Ptr) < NumBytes) {
+      PartialUTF8Char = StringRef(Ptr, End - Ptr);
+      return;
+    }
+
+    ProcessUTF8CodePoint(StringRef(Ptr, NumBytes));
   }
 }
 
@@ -52,9 +100,9 @@ void formatted_raw_ostream::ComputePosition(const char *Ptr, size_t Size) {
   if (Ptr <= Scanned && Scanned <= Ptr + Size)
     // Scan all characters added since our last scan to determine the new
     // column.
-    UpdatePosition(Position, Scanned, Size - (Scanned - Ptr));
+    UpdatePosition(Scanned, Size - (Scanned - Ptr));
   else
-    UpdatePosition(Position, Ptr, Size);
+    UpdatePosition(Ptr, Size);
 
   // Update the scanning pointer.
   Scanned = Ptr + Size;
diff --git a/llvm/lib/Support/Host.cpp b/llvm/lib/Support/Host.cpp
index c6b12e6516a45..658c1ee74cfec 100644
--- a/llvm/lib/Support/Host.cpp
+++ b/llvm/lib/Support/Host.cpp
@@ -204,6 +204,9 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) {
             .Case("0xd09", "cortex-a73")
             .Case("0xd0a", "cortex-a75")
             .Case("0xd0b", "cortex-a76")
+            .Case("0xd0d", "cortex-a77")
+            .Case("0xd41", "cortex-a78")
+            .Case("0xd44", "cortex-x1")
             .Case("0xd0c", "neoverse-n1")
             .Default("generic");
   }
@@ -580,59 +583,32 @@ static void detectX86FamilyModel(unsigned EAX, unsigned *Family,
   }
 }
 
-static void
+static StringRef
 getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
-                                const unsigned (&Features)[3],
+                                const unsigned *Features,
                                 unsigned *Type, unsigned *Subtype) {
   auto testFeature = [&](unsigned F) {
     return (Features[F / 32] & (1U << (F % 32))) != 0;
   };
 
+  StringRef CPU;
+
   switch (Family) {
   case 3:
-    *Type = X86::INTEL_i386;
+    CPU = "i386";
     break;
   case 4:
-    *Type = X86::INTEL_i486;
+    CPU = "i486";
     break;
   case 5:
     if (testFeature(X86::FEATURE_MMX)) {
-      *Type = X86::INTEL_PENTIUM_MMX;
+      CPU = "pentium-mmx";
       break;
     }
-    *Type = X86::INTEL_PENTIUM;
+    CPU = "pentium";
     break;
   case 6:
     switch (Model) {
-    case 0x01: // Pentium Pro processor
-      *Type = X86::INTEL_PENTIUM_PRO;
-      break;
-    case 0x03: // Intel Pentium II OverDrive processor, Pentium II processor,
-               // model 03
-    case 0x05: // Pentium II processor, model 05, Pentium II Xeon processor,
-               // model 05, and Intel Celeron processor, model 05
-    case 0x06: // Celeron processor, model 06
-      *Type = X86::INTEL_PENTIUM_II;
-      break;
-    case 0x07: // Pentium III processor, model 07, and Pentium III Xeon
-               // processor, model 07
-    case 0x08: // Pentium III processor, model 08, Pentium III Xeon processor,
-               // model 08, and Celeron processor, model 08
-    case 0x0a: // Pentium III Xeon processor, model 0Ah
-    case 0x0b: // Pentium III processor, model 0Bh
-      *Type = X86::INTEL_PENTIUM_III;
-      break;
-    case 0x09: // Intel Pentium M processor, Intel Celeron M processor model 09.
-    case 0x0d: // Intel Pentium M processor, Intel Celeron M processor, model
-               // 0Dh. All processors are manufactured using the 90 nm process.
-    case 0x15: // Intel EP80579 Integrated Processor and Intel EP80579
-               // Integrated Processor with Intel QuickAssist Technology
-      *Type = X86::INTEL_PENTIUM_M;
-      break;
-    case 0x0e: // Intel Core Duo processor, Intel Core Solo processor, model
-               // 0Eh. All processors are manufactured using the 65 nm process.
-      *Type = X86::INTEL_CORE_DUO;
-      break;   // yonah
     case 0x0f: // Intel Core 2 Duo processor, Intel Core 2 Duo mobile
                // processor, Intel Core 2 Quad processor, Intel Core 2 Quad
                // mobile processor, Intel Core 2 Extreme processor, Intel
@@ -640,8 +616,8 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
                // 0Fh. All processors are manufactured using the 65 nm process.
     case 0x16: // Intel Celeron processor model 16h. All processors are
                // manufactured using the 65 nm process
-      *Type = X86::INTEL_CORE2; // "core2"
-      *Subtype = X86::INTEL_CORE2_65;
+      CPU = "core2";
+      *Type = X86::INTEL_CORE2;
       break;
     case 0x17: // Intel Core 2 Extreme processor, Intel Xeon processor, model
                // 17h. All processors are manufactured using the 45 nm process.
@@ -649,34 +625,38 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
                // 45nm: Penryn , Wolfdale, Yorkfield (XE)
     case 0x1d: // Intel Xeon processor MP. All processors are manufactured using
                // the 45 nm process.
-      *Type = X86::INTEL_CORE2; // "penryn"
-      *Subtype = X86::INTEL_CORE2_45;
+      CPU = "penryn";
+      *Type = X86::INTEL_CORE2;
       break;
     case 0x1a: // Intel Core i7 processor and Intel Xeon processor. All
                // processors are manufactured using the 45 nm process.
     case 0x1e: // Intel(R) Core(TM) i7 CPU         870  @ 2.93GHz.
                // As found in a Summer 2010 model iMac.
     case 0x1f:
-    case 0x2e:             // Nehalem EX
-      *Type = X86::INTEL_COREI7; // "nehalem"
+    case 0x2e:              // Nehalem EX
+      CPU = "nehalem";
+      *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_NEHALEM;
       break;
     case 0x25: // Intel Core i7, laptop version.
     case 0x2c: // Intel Core i7 processor and Intel Xeon processor. All
                // processors are manufactured using the 32 nm process.
     case 0x2f: // Westmere EX
-      *Type = X86::INTEL_COREI7; // "westmere"
+      CPU = "westmere";
+      *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_WESTMERE;
       break;
     case 0x2a: // Intel Core i7 processor. All processors are manufactured
                // using the 32 nm process.
     case 0x2d:
-      *Type = X86::INTEL_COREI7; //"sandybridge"
+      CPU = "sandybridge";
+      *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_SANDYBRIDGE;
       break;
     case 0x3a:
-    case 0x3e:             // Ivy Bridge EP
-      *Type = X86::INTEL_COREI7; // "ivybridge"
+    case 0x3e:              // Ivy Bridge EP
+      CPU = "ivybridge";
+      *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_IVYBRIDGE;
       break;
 
@@ -685,7 +665,8 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
     case 0x3f:
     case 0x45:
     case 0x46:
-      *Type = X86::INTEL_COREI7; // "haswell"
+      CPU = "haswell";
+      *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_HASWELL;
       break;
 
@@ -694,7 +675,8 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
     case 0x47:
     case 0x4f:
     case 0x56:
-      *Type = X86::INTEL_COREI7; // "broadwell"
+      CPU = "broadwell";
+      *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_BROADWELL;
       break;
 
@@ -705,39 +687,47 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
     case 0x9e:              // Kaby Lake desktop
     case 0xa5:              // Comet Lake-H/S
     case 0xa6:              // Comet Lake-U
-      *Type = X86::INTEL_COREI7; // "skylake"
+      CPU = "skylake";
+      *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_SKYLAKE;
       break;
 
     // Skylake Xeon:
     case 0x55:
       *Type = X86::INTEL_COREI7;
-      if (testFeature(X86::FEATURE_AVX512BF16))
-        *Subtype = X86::INTEL_COREI7_COOPERLAKE; // "cooperlake"
-      else if (testFeature(X86::FEATURE_AVX512VNNI))
-        *Subtype = X86::INTEL_COREI7_CASCADELAKE; // "cascadelake"
-      else
-        *Subtype = X86::INTEL_COREI7_SKYLAKE_AVX512; // "skylake-avx512"
+      if (testFeature(X86::FEATURE_AVX512BF16)) {
+        CPU = "cooperlake";
+        *Subtype = X86::INTEL_COREI7_COOPERLAKE;
+      } else if (testFeature(X86::FEATURE_AVX512VNNI)) {
+        CPU = "cascadelake";
+        *Subtype = X86::INTEL_COREI7_CASCADELAKE;
+      } else {
+        CPU = "skylake-avx512";
+        *Subtype = X86::INTEL_COREI7_SKYLAKE_AVX512;
+      }
       break;
 
     // Cannonlake:
     case 0x66:
+      CPU = "cannonlake";
       *Type = X86::INTEL_COREI7;
-      *Subtype = X86::INTEL_COREI7_CANNONLAKE; // "cannonlake"
+      *Subtype = X86::INTEL_COREI7_CANNONLAKE;
       break;
 
     // Icelake:
     case 0x7d:
     case 0x7e:
+      CPU = "icelake-client";
       *Type = X86::INTEL_COREI7;
-      *Subtype = X86::INTEL_COREI7_ICELAKE_CLIENT; // "icelake-client"
+      *Subtype = X86::INTEL_COREI7_ICELAKE_CLIENT;
       break;
 
     // Icelake Xeon:
     case 0x6a:
     case 0x6c:
+      CPU = "icelake-server";
       *Type = X86::INTEL_COREI7;
-      *Subtype = X86::INTEL_COREI7_ICELAKE_SERVER; // "icelake-server"
+      *Subtype = X86::INTEL_COREI7_ICELAKE_SERVER;
       break;
 
     case 0x1c: // Most 45 nm Intel Atom processors
@@ -745,8 +735,9 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
     case 0x27: // 32 nm Atom Medfield
     case 0x35: // 32 nm Atom Midview
     case 0x36: // 32 nm Atom Midview
+      CPU = "bonnell";
       *Type = X86::INTEL_BONNELL;
-      break; // "bonnell"
+      break;
 
     // Atom Silvermont codes from the Intel software optimization guide.
     case 0x37:
@@ -755,14 +746,17 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
     case 0x5a:
     case 0x5d:
     case 0x4c: // really airmont
+      CPU = "silvermont";
       *Type = X86::INTEL_SILVERMONT;
-      break; // "silvermont"
+      break;
     // Goldmont:
     case 0x5c: // Apollo Lake
     case 0x5f: // Denverton
+      CPU = "goldmont";
       *Type = X86::INTEL_GOLDMONT;
-      break; // "goldmont"
+      break;
     case 0x7a:
+      CPU = "goldmont-plus";
       *Type = X86::INTEL_GOLDMONT_PLUS;
       break;
     case 0x86:
@@ -770,193 +764,140 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
       break;
 
     case 0x57:
-      *Type = X86::INTEL_KNL; // knl
+      CPU = "tremont";
+      *Type = X86::INTEL_KNL;
       break;
 
     case 0x85:
-      *Type = X86::INTEL_KNM; // knm
+      CPU = "knm";
+      *Type = X86::INTEL_KNM;
       break;
 
     default: // Unknown family 6 CPU, try to guess.
+      // Don't both with Type/Subtype here, they aren't used by the caller.
+      // They're used above to keep the code in sync with compiler-rt.
       // TODO detect tigerlake host from model
       if (testFeature(X86::FEATURE_AVX512VP2INTERSECT)) {
-        *Type = X86::INTEL_COREI7;
-        *Subtype = X86::INTEL_COREI7_TIGERLAKE;
-        break;
-      }
-
-      if (testFeature(X86::FEATURE_AVX512VBMI2)) {
-        *Type = X86::INTEL_COREI7;
-        *Subtype = X86::INTEL_COREI7_ICELAKE_CLIENT;
-        break;
-      }
-
-      if (testFeature(X86::FEATURE_AVX512VBMI)) {
-        *Type = X86::INTEL_COREI7;
-        *Subtype = X86::INTEL_COREI7_CANNONLAKE;
-        break;
-      }
-
-      if (testFeature(X86::FEATURE_AVX512BF16)) {
-        *Type = X86::INTEL_COREI7;
-        *Subtype = X86::INTEL_COREI7_COOPERLAKE;
-        break;
-      }
-
-      if (testFeature(X86::FEATURE_AVX512VNNI)) {
-        *Type = X86::INTEL_COREI7;
-        *Subtype = X86::INTEL_COREI7_CASCADELAKE;
-        break;
-      }
-
-      if (testFeature(X86::FEATURE_AVX512VL)) {
-        *Type = X86::INTEL_COREI7;
-        *Subtype = X86::INTEL_COREI7_SKYLAKE_AVX512;
-        break;
-      }
-
-      if (testFeature(X86::FEATURE_AVX512ER)) {
-        *Type = X86::INTEL_KNL; // knl
-        break;
+        CPU = "tigerlake";
+      } else if (testFeature(X86::FEATURE_AVX512VBMI2)) {
+        CPU = "icelake-client";
+      } else if (testFeature(X86::FEATURE_AVX512VBMI)) {
+        CPU = "cannonlake";
+      } else if (testFeature(X86::FEATURE_AVX512BF16)) {
+        CPU = "cooperlake";
+      } else if (testFeature(X86::FEATURE_AVX512VNNI)) {
+        CPU = "cascadelake";
+      } else if (testFeature(X86::FEATURE_AVX512VL)) {
+        CPU = "skylake-avx512";
+      } else if (testFeature(X86::FEATURE_AVX512ER)) {
+        CPU = "knl";
+      } else if (testFeature(X86::FEATURE_CLFLUSHOPT)) {
+        if (testFeature(X86::FEATURE_SHA))
+          CPU = "goldmont";
+        else
+          CPU = "skylake";
+      } else if (testFeature(X86::FEATURE_ADX)) {
+        CPU = "broadwell";
+      } else if (testFeature(X86::FEATURE_AVX2)) {
+        CPU = "haswell";
+      } else if (testFeature(X86::FEATURE_AVX)) {
+        CPU = "sandybridge";
+      } else if (testFeature(X86::FEATURE_SSE4_2)) {
+        if (testFeature(X86::FEATURE_MOVBE))
+          CPU = "silvermont";
+        else
+          CPU = "nehalem";
+      } else if (testFeature(X86::FEATURE_SSE4_1)) {
+        CPU = "penryn";
+      } else if (testFeature(X86::FEATURE_SSSE3)) {
+        if (testFeature(X86::FEATURE_MOVBE))
+          CPU = "bonnell";
+        else
+          CPU = "core2";
+      } else if (testFeature(X86::FEATURE_64BIT)) {
+        CPU = "core2";
+      } else if (testFeature(X86::FEATURE_SSE3)) {
+        CPU = "yonah";
+      } else if (testFeature(X86::FEATURE_SSE2)) {
+        CPU = "pentium-m";
+      } else if (testFeature(X86::FEATURE_SSE)) {
+        CPU = "pentium3";
+      } else if (testFeature(X86::FEATURE_MMX)) {
+        CPU = "pentium2";
+      } else {
+        CPU = "pentiumpro";
       }
-
-      if (testFeature(X86::FEATURE_CLFLUSHOPT)) {
-        if (testFeature(X86::FEATURE_SHA)) {
-          *Type = X86::INTEL_GOLDMONT;
-        } else {
-          *Type = X86::INTEL_COREI7;
-          *Subtype = X86::INTEL_COREI7_SKYLAKE;
-        }
-        break;
-      }
-      if (testFeature(X86::FEATURE_ADX)) {
-        *Type = X86::INTEL_COREI7;
-        *Subtype = X86::INTEL_COREI7_BROADWELL;
-        break;
-      }
-      if (testFeature(X86::FEATURE_AVX2)) {
-        *Type = X86::INTEL_COREI7;
-        *Subtype = X86::INTEL_COREI7_HASWELL;
-        break;
-      }
-      if (testFeature(X86::FEATURE_AVX)) {
-        *Type = X86::INTEL_COREI7;
-        *Subtype = X86::INTEL_COREI7_SANDYBRIDGE;
-        break;
-      }
-      if (testFeature(X86::FEATURE_SSE4_2)) {
-        if (testFeature(X86::FEATURE_MOVBE)) {
-          *Type = X86::INTEL_SILVERMONT;
-        } else {
-          *Type = X86::INTEL_COREI7;
-          *Subtype = X86::INTEL_COREI7_NEHALEM;
-        }
-        break;
-      }
-      if (testFeature(X86::FEATURE_SSE4_1)) {
-        *Type = X86::INTEL_CORE2; // "penryn"
-        *Subtype = X86::INTEL_CORE2_45;
-        break;
-      }
-      if (testFeature(X86::FEATURE_SSSE3)) {
-        if (testFeature(X86::FEATURE_MOVBE)) {
-          *Type = X86::INTEL_BONNELL; // "bonnell"
-        } else {
-          *Type = X86::INTEL_CORE2; // "core2"
-          *Subtype = X86::INTEL_CORE2_65;
-        }
-        break;
-      }
-      if (testFeature(X86::FEATURE_EM64T)) {
-        *Type = X86::INTEL_CORE2; // "core2"
-        *Subtype = X86::INTEL_CORE2_65;
-        break;
-      }
-      if (testFeature(X86::FEATURE_SSE3)) {
-        *Type = X86::INTEL_CORE_DUO;
-        break;
-      }
-      if (testFeature(X86::FEATURE_SSE2)) {
-        *Type = X86::INTEL_PENTIUM_M;
-        break;
-      }
-      if (testFeature(X86::FEATURE_SSE)) {
-        *Type = X86::INTEL_PENTIUM_III;
-        break;
-      }
-      if (testFeature(X86::FEATURE_MMX)) {
-        *Type = X86::INTEL_PENTIUM_II;
-        break;
-      }
-      *Type = X86::INTEL_PENTIUM_PRO;
       break;
     }
     break;
   case 15: {
-    if (testFeature(X86::FEATURE_EM64T)) {
-      *Type = X86::INTEL_NOCONA;
+    if (testFeature(X86::FEATURE_64BIT)) {
+      CPU = "nocona";
       break;
     }
     if (testFeature(X86::FEATURE_SSE3)) {
-      *Type = X86::INTEL_PRESCOTT;
+      CPU = "prescott";
       break;
     }
-    *Type = X86::INTEL_PENTIUM_IV;
+    CPU = "pentium4";
     break;
   }
   default:
-    break; /*"generic"*/
+    break; // Unknown.
   }
+
+  return CPU;
 }
 
-static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
-                                          const unsigned (&Features)[3],
-                                          unsigned *Type, unsigned *Subtype) {
+static StringRef
+getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
+                              const unsigned *Features,
+                              unsigned *Type, unsigned *Subtype) {
   auto testFeature = [&](unsigned F) {
     return (Features[F / 32] & (1U << (F % 32))) != 0;
   };
 
-  // FIXME: this poorly matches the generated SubtargetFeatureKV table.  There
-  // appears to be no way to generate the wide variety of AMD-specific targets
-  // from the information returned from CPUID.
+  StringRef CPU;
+
   switch (Family) {
   case 4:
-    *Type = X86::AMD_i486;
+    CPU = "i486";
     break;
   case 5:
-    *Type = X86::AMDPENTIUM;
+    CPU = "pentium";
     switch (Model) {
     case 6:
     case 7:
-      *Subtype = X86::AMDPENTIUM_K6;
-      break; // "k6"
+      CPU = "k6";
+      break;
     case 8:
-      *Subtype = X86::AMDPENTIUM_K62;
-      break; // "k6-2"
+      CPU = "k6-2";
+      break;
     case 9:
     case 13:
-      *Subtype = X86::AMDPENTIUM_K63;
-      break; // "k6-3"
+      CPU = "k6-3";
+      break;
     case 10:
-      *Subtype = X86::AMDPENTIUM_GEODE;
-      break; // "geode"
+      CPU = "geode";
+      break;
     }
     break;
   case 6:
     if (testFeature(X86::FEATURE_SSE)) {
-      *Type = X86::AMD_ATHLON_XP;
-      break; // "athlon-xp"
+      CPU = "athlon-xp";
+      break;
     }
-    *Type = X86::AMD_ATHLON;
-    break; // "athlon"
+    CPU = "athlon";
+    break;
   case 15:
     if (testFeature(X86::FEATURE_SSE3)) {
-      *Type = X86::AMD_K8SSE3;
-      break; // "k8-sse3"
+      CPU = "k8-sse3";
+      break;
     }
-    *Type = X86::AMD_K8;
-    break; // "k8"
+    CPU = "k8";
+    break;
   case 16:
+    CPU = "amdfam10";
     *Type = X86::AMDFAM10H; // "amdfam10"
     switch (Model) {
     case 2:
@@ -971,51 +912,58 @@ static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
     }
     break;
   case 20:
+    CPU = "btver1";
     *Type = X86::AMD_BTVER1;
-    break; // "btver1";
+    break;
   case 21:
+    CPU = "bdver1";
     *Type = X86::AMDFAM15H;
     if (Model >= 0x60 && Model <= 0x7f) {
+      CPU = "bdver4";
       *Subtype = X86::AMDFAM15H_BDVER4;
-      break; // "bdver4"; 60h-7Fh: Excavator
+      break; // 60h-7Fh: Excavator
     }
     if (Model >= 0x30 && Model <= 0x3f) {
+      CPU = "bdver3";
       *Subtype = X86::AMDFAM15H_BDVER3;
-      break; // "bdver3"; 30h-3Fh: Steamroller
+      break; // 30h-3Fh: Steamroller
     }
     if ((Model >= 0x10 && Model <= 0x1f) || Model == 0x02) {
+      CPU = "bdver2";
       *Subtype = X86::AMDFAM15H_BDVER2;
-      break; // "bdver2"; 02h, 10h-1Fh: Piledriver
+      break; // 02h, 10h-1Fh: Piledriver
     }
     if (Model <= 0x0f) {
       *Subtype = X86::AMDFAM15H_BDVER1;
-      break; // "bdver1"; 00h-0Fh: Bulldozer
+      break; // 00h-0Fh: Bulldozer
     }
     break;
   case 22:
+    CPU = "btver2";
     *Type = X86::AMD_BTVER2;
-    break; // "btver2"
+    break;
   case 23:
+    CPU = "znver1";
     *Type = X86::AMDFAM17H;
     if ((Model >= 0x30 && Model <= 0x3f) || Model == 0x71) {
+      CPU = "znver2";
       *Subtype = X86::AMDFAM17H_ZNVER2;
-      break; // "znver2"; 30h-3fh, 71h: Zen2
+      break; // 30h-3fh, 71h: Zen2
     }
     if (Model <= 0x0f) {
       *Subtype = X86::AMDFAM17H_ZNVER1;
-      break; // "znver1"; 00h-0Fh: Zen1
+      break; // 00h-0Fh: Zen1
     }
     break;
   default:
-    break; // "generic"
+    break; // Unknown AMD CPU.
   }
+
+  return CPU;
 }
 
 static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
-                                 unsigned (&Features)[3]) {
-  Features[0] = 0;
-  Features[1] = 0;
-  Features[2] = 0;
+                                 unsigned *Features) {
   unsigned EAX, EBX;
 
   auto setFeature = [&](unsigned F) {
@@ -1142,7 +1090,7 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
     setFeature(X86::FEATURE_FMA4);
 
   if (HasExtLeaf1 && ((EDX >> 29) & 1))
-    setFeature(X86::FEATURE_EM64T);
+    setFeature(X86::FEATURE_64BIT);
 }
 
 StringRef sys::getHostCPUName() {
@@ -1157,30 +1105,27 @@ StringRef sys::getHostCPUName() {
   getX86CpuIDAndInfo(0x1, &EAX, &EBX, &ECX, &EDX);
 
   unsigned Family = 0, Model = 0;
-  unsigned Features[3] = {0, 0, 0};
+  unsigned Features[(X86::CPU_FEATURE_MAX + 31) / 32] = {0};
   detectX86FamilyModel(EAX, &Family, &Model);
   getAvailableFeatures(ECX, EDX, MaxLeaf, Features);
 
+  // These aren't consumed in this file, but we try to keep some source code the
+  // same or similar to compiler-rt.
   unsigned Type = 0;
   unsigned Subtype = 0;
 
+  StringRef CPU;
+
   if (Vendor == SIG_INTEL) {
-    getIntelProcessorTypeAndSubtype(Family, Model, Features, &Type, &Subtype);
+    CPU = getIntelProcessorTypeAndSubtype(Family, Model, Features, &Type,
+                                          &Subtype);
   } else if (Vendor == SIG_AMD) {
-    getAMDProcessorTypeAndSubtype(Family, Model, Features, &Type, &Subtype);
+    CPU = getAMDProcessorTypeAndSubtype(Family, Model, Features, &Type,
+                                        &Subtype);
   }
 
-  // Check subtypes first since those are more specific.
-#define X86_CPU_SUBTYPE(ARCHNAME, ENUM) \
-  if (Subtype == X86::ENUM) \
-    return ARCHNAME;
-#include "llvm/Support/X86TargetParser.def"
-
-  // Now check types.
-#define X86_CPU_TYPE(ARCHNAME, ENUM) \
-  if (Type == X86::ENUM) \
-    return ARCHNAME;
-#include "llvm/Support/X86TargetParser.def"
+  if (!CPU.empty())
+    return CPU;
 
   return "generic";
 }
@@ -1363,13 +1308,8 @@ int sys::getHostNumPhysicalCores() {
 bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
   unsigned EAX = 0, EBX = 0, ECX = 0, EDX = 0;
   unsigned MaxLevel;
-  union {
-    unsigned u[3];
-    char c[12];
-  } text;
 
-  if (getX86CpuIDAndInfo(0, &MaxLevel, text.u + 0, text.u + 2, text.u + 1) ||
-      MaxLevel < 1)
+  if (getX86CpuIDAndInfo(0, &MaxLevel, &EBX, &ECX, &EDX) || MaxLevel < 1)
     return false;
 
   getX86CpuIDAndInfo(1, &EAX, &EBX, &ECX, &EDX);
@@ -1395,8 +1335,8 @@ bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
   // If CPUID indicates support for XSAVE, XRESTORE and AVX, and XGETBV
   // indicates that the AVX registers will be saved and restored on context
   // switch, then we have full AVX support.
-  bool HasAVXSave = ((ECX >> 27) & 1) && ((ECX >> 28) & 1) &&
-                    !getX86XCR0(&EAX, &EDX) && ((EAX & 0x6) == 0x6);
+  bool HasXSave = ((ECX >> 27) & 1) && !getX86XCR0(&EAX, &EDX);
+  bool HasAVXSave = HasXSave && ((ECX >> 28) & 1) && ((EAX & 0x6) == 0x6);
 #if defined(__APPLE__)
   // Darwin lazily saves the AVX512 context on first use: trust that the OS will
   // save the AVX512 context if we use AVX512 instructions, even the bit is not
@@ -1406,6 +1346,9 @@ bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
   // AVX512 requires additional context to be saved by the OS.
   bool HasAVX512Save = HasAVXSave && ((EAX & 0xe0) == 0xe0);
 #endif
+  // AMX requires additional context to be saved by the OS.
+  const unsigned AMXBits = (1 << 17) | (1 << 18);
+  bool HasAMXSave = HasXSave && ((EAX & AMXBits) == AMXBits);
 
   Features["avx"]   = HasAVXSave;
   Features["fma"]   = ((ECX >> 12) & 1) && HasAVXSave;
@@ -1496,6 +1439,9 @@ bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
   // detecting features using the "-march=native" flag.
   // For more info, see X86 ISA docs.
   Features["pconfig"] = HasLeaf7 && ((EDX >> 18) & 1);
+  Features["amx-bf16"]   = HasLeaf7 && ((EDX >> 22) & 1) && HasAMXSave;
+  Features["amx-tile"]   = HasLeaf7 && ((EDX >> 24) & 1) && HasAMXSave;
+  Features["amx-int8"]   = HasLeaf7 && ((EDX >> 25) & 1) && HasAMXSave;
   bool HasLeaf7Subleaf1 =
       MaxLevel >= 7 && !getX86CpuIDAndInfoEx(0x7, 0x1, &EAX, &EBX, &ECX, &EDX);
   Features["avx512bf16"] = HasLeaf7Subleaf1 && ((EAX >> 5) & 1) && HasAVX512Save;
diff --git a/llvm/lib/Support/Parallel.cpp b/llvm/lib/Support/Parallel.cpp
index 7f6ce82763d6e..9a2e1003da5a2 100644
--- a/llvm/lib/Support/Parallel.cpp
+++ b/llvm/lib/Support/Parallel.cpp
@@ -9,9 +9,6 @@
 #include "llvm/Support/Parallel.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/ManagedStatic.h"
-
-#if LLVM_ENABLE_THREADS
-
 #include "llvm/Support/Threading.h"
 
 #include <atomic>
@@ -22,6 +19,8 @@
 
 llvm::ThreadPoolStrategy llvm::parallel::strategy;
 
+#if LLVM_ENABLE_THREADS
+
 namespace llvm {
 namespace parallel {
 namespace detail {
diff --git a/llvm/lib/Support/PrettyStackTrace.cpp b/llvm/lib/Support/PrettyStackTrace.cpp
index 3d310ed8916ea..9072f9d2d2eef 100644
--- a/llvm/lib/Support/PrettyStackTrace.cpp
+++ b/llvm/lib/Support/PrettyStackTrace.cpp
@@ -33,6 +33,10 @@
 
 using namespace llvm;
 
+static const char *BugReportMsg =
+    "PLEASE submit a bug report to " BUG_REPORT_URL
+    " and include the crash backtrace.\n";
+
 // If backtrace support is not enabled, compile out support for pretty stack
 // traces.  This has the secondary effect of not requiring thread local storage
 // when backtrace support is disabled.
@@ -142,10 +146,6 @@ using CrashHandlerStringStorage =
 static CrashHandlerStringStorage crashHandlerStringStorage;
 #endif
 
-static const char *BugReportMsg =
-    "PLEASE submit a bug report to " BUG_REPORT_URL
-    " and include the crash backtrace.\n";
-
 /// This callback is run if a fatal signal is delivered to the process, it
 /// prints the pretty stack trace.
 static void CrashHandler(void *) {
@@ -203,9 +203,11 @@ static void printForSigInfoIfNeeded() {
 #endif // ENABLE_BACKTRACES
 
 void llvm::setBugReportMsg(const char *Msg) {
-#if ENABLE_BACKTRACES
   BugReportMsg = Msg;
-#endif
+}
+
+const char *llvm::getBugReportMsg() {
+  return BugReportMsg;
 }
 
 PrettyStackTraceEntry::PrettyStackTraceEntry() {
diff --git a/llvm/lib/Support/Regex.cpp b/llvm/lib/Support/Regex.cpp
index f065adadc62bb..0d5cc1c00db11 100644
--- a/llvm/lib/Support/Regex.cpp
+++ b/llvm/lib/Support/Regex.cpp
@@ -26,7 +26,7 @@ using namespace llvm;
 
 Regex::Regex() : preg(nullptr), error(REG_BADPAT) {}
 
-Regex::Regex(StringRef regex, unsigned Flags) {
+Regex::Regex(StringRef regex, RegexFlags Flags) {
   unsigned flags = 0;
   preg = new llvm_regex();
   preg->re_endp = regex.end();
@@ -39,6 +39,9 @@ Regex::Regex(StringRef regex, unsigned Flags) {
   error = llvm_regcomp(preg, regex.data(), flags|REG_PEND);
 }
 
+Regex::Regex(StringRef regex, unsigned Flags)
+    : Regex(regex, static_cast<RegexFlags>(Flags)) {}
+
 Regex::Regex(Regex &&regex) {
   preg = regex.preg;
   error = regex.error;
diff --git a/llvm/lib/Support/SystemUtils.cpp b/llvm/lib/Support/SystemUtils.cpp
index 47e0c72ec7c13..f1149e48dce5d 100644
--- a/llvm/lib/Support/SystemUtils.cpp
+++ b/llvm/lib/Support/SystemUtils.cpp
@@ -15,15 +15,12 @@
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
-bool llvm::CheckBitcodeOutputToConsole(raw_ostream &stream_to_check,
-                                       bool print_warning) {
+bool llvm::CheckBitcodeOutputToConsole(raw_ostream &stream_to_check) {
   if (stream_to_check.is_displayed()) {
-    if (print_warning) {
-      errs() << "WARNING: You're attempting to print out a bitcode file.\n"
-                "This is inadvisable as it may cause display problems. If\n"
-                "you REALLY want to taste LLVM bitcode first-hand, you\n"
-                "can force output with the `-f' option.\n\n";
-    }
+    errs() << "WARNING: You're attempting to print out a bitcode file.\n"
+              "This is inadvisable as it may cause display problems. If\n"
+              "you REALLY want to taste LLVM bitcode first-hand, you\n"
+              "can force output with the `-f' option.\n\n";
     return true;
   }
   return false;
diff --git a/llvm/lib/Support/Triple.cpp b/llvm/lib/Support/Triple.cpp
index da6b877a8504a..fec1985ccacae 100644
--- a/llvm/lib/Support/Triple.cpp
+++ b/llvm/lib/Support/Triple.cpp
@@ -15,6 +15,7 @@
 #include "llvm/Support/Host.h"
 #include "llvm/Support/SwapByteOrder.h"
 #include "llvm/Support/TargetParser.h"
+#include "llvm/Support/VersionTuple.h"
 #include <cassert>
 #include <cstring>
 using namespace llvm;
@@ -1086,17 +1087,23 @@ bool Triple::getMacOSXVersion(unsigned &Major, unsigned &Minor,
     // Darwin version numbers are skewed from OS X versions.
     if (Major < 4)
       return false;
-    Micro = 0;
-    Minor = Major - 4;
-    Major = 10;
+    if (Major <= 19) {
+      Micro = 0;
+      Minor = Major - 4;
+      Major = 10;
+    } else {
+      Micro = 0;
+      Minor = 0;
+      // darwin20+ corresponds to macOS 11+.
+      Major = 11 + Major - 20;
+    }
     break;
   case MacOSX:
     // Default to 10.4.
     if (Major == 0) {
       Major = 10;
       Minor = 4;
-    }
-    if (Major != 10)
+    } else if (Major < 10)
       return false;
     break;
   case IOS:
@@ -1600,6 +1607,52 @@ std::string Triple::merge(const Triple &Other) const {
   return Other.str();
 }
 
+bool Triple::isMacOSXVersionLT(unsigned Major, unsigned Minor,
+                               unsigned Micro) const {
+  assert(isMacOSX() && "Not an OS X triple!");
+
+  // If this is OS X, expect a sane version number.
+  if (getOS() == Triple::MacOSX)
+    return isOSVersionLT(Major, Minor, Micro);
+
+  // Otherwise, compare to the "Darwin" number.
+  if (Major == 10) {
+    return isOSVersionLT(Minor + 4, Micro, 0);
+  } else {
+    assert(Major >= 11 && "Unexpected major version");
+    return isOSVersionLT(Major - 11 + 20, Minor, Micro);
+  }
+}
+
+VersionTuple Triple::getMinimumSupportedOSVersion() const {
+  if (getVendor() != Triple::Apple || getArch() != Triple::aarch64)
+    return VersionTuple();
+  switch (getOS()) {
+  case Triple::MacOSX:
+    // ARM64 slice is supported starting from macOS 11.0+.
+    return VersionTuple(11, 0, 0);
+  case Triple::IOS:
+    // ARM64 slice is supported starting from Mac Catalyst 14 (macOS 11).
+    // ARM64 simulators are supported for iOS 14+.
+    if (isMacCatalystEnvironment() || isSimulatorEnvironment())
+      return VersionTuple(14, 0, 0);
+    break;
+  case Triple::TvOS:
+    // ARM64 simulators are supported for tvOS 14+.
+    if (isSimulatorEnvironment())
+      return VersionTuple(14, 0, 0);
+    break;
+  case Triple::WatchOS:
+    // ARM64 simulators are supported for watchOS 7+.
+    if (isSimulatorEnvironment())
+      return VersionTuple(7, 0, 0);
+    break;
+  default:
+    break;
+  }
+  return VersionTuple();
+}
+
 StringRef Triple::getARMCPUForArch(StringRef MArch) const {
   if (MArch.empty())
     MArch = getArchName();
@@ -1662,3 +1715,16 @@ StringRef Triple::getARMCPUForArch(StringRef MArch) const {
 
   llvm_unreachable("invalid arch name");
 }
+
+VersionTuple Triple::getCanonicalVersionForOS(OSType OSKind,
+                                              const VersionTuple &Version) {
+  switch (OSKind) {
+  case MacOSX:
+    // macOS 10.16 is canonicalized to macOS 11.
+    if (Version == VersionTuple(10, 16))
+      return VersionTuple(11, 0);
+    LLVM_FALLTHROUGH;
+  default:
+    return Version;
+  }
+}
diff --git a/llvm/lib/Support/Unix/Path.inc b/llvm/lib/Support/Unix/Path.inc
index bf720b318dedf..d91b269cc6d33 100644
--- a/llvm/lib/Support/Unix/Path.inc
+++ b/llvm/lib/Support/Unix/Path.inc
@@ -48,6 +48,8 @@ extern char **environ;
 #endif
 #elif defined(__DragonFly__)
 #include <sys/mount.h>
+#elif defined(__MVS__)
+#include <sys/ps.h>
 #endif
 
 // Both stdio.h and cstdio are included via different paths and
@@ -56,9 +58,13 @@ extern char **environ;
 #undef ferror
 #undef feof
 
+#if !defined(PATH_MAX)
 // For GNU Hurd
-#if defined(__GNU__) && !defined(PATH_MAX)
-# define PATH_MAX 4096
+#if defined(__GNU__)
+#define PATH_MAX 4096
+#elif defined(__MVS__)
+#define PATH_MAX _XOPEN_PATH_MAX
+#endif
 #endif
 
 #include <sys/types.h>
@@ -100,7 +106,8 @@ typedef uint_t uint;
 #define STATVFS_F_FRSIZE(vfs) static_cast<uint64_t>(vfs.f_bsize)
 #endif
 
-#if defined(__NetBSD__) || defined(__DragonFly__) || defined(__GNU__)
+#if defined(__NetBSD__) || defined(__DragonFly__) || defined(__GNU__) || \
+    defined(__MVS__)
 #define STATVFS_F_FLAG(vfs) (vfs).f_flag
 #else
 #define STATVFS_F_FLAG(vfs) (vfs).f_flags
@@ -265,6 +272,26 @@ std::string getMainExecutable(const char *argv0, void *MainAddr) {
   // Fall back to the classical detection.
   if (getprogpath(exe_path, argv0))
     return exe_path;
+#elif defined(__MVS__)
+  int token = 0;
+  W_PSPROC buf;
+  char exe_path[PS_PATHBLEN];
+  pid_t pid = getpid();
+
+  memset(&buf, 0, sizeof(buf));
+  buf.ps_pathptr = exe_path;
+  buf.ps_pathlen = sizeof(exe_path);
+
+  while (true) {
+    if ((token = w_getpsent(token, &buf, sizeof(buf))) <= 0)
+      break;
+    if (buf.ps_pid != pid)
+      continue;
+    char real_path[PATH_MAX];
+    if (realpath(exe_path, real_path))
+      return std::string(real_path);
+    break;  // Found entry, but realpath failed.
+  }
 #elif defined(HAVE_DLFCN_H) && defined(HAVE_DLADDR)
   // Use dladdr to get executable path if available.
   Dl_info DLInfo;
@@ -493,6 +520,10 @@ static bool is_local_impl(struct STATVFS &Vfs) {
 
   // vmount entry not found; "remote" is the conservative answer.
   return false;
+#elif defined(__MVS__)
+  // The file system can have an arbitrary structure on z/OS; must go with the
+  // conservative answer.
+  return false;
 #else
   return !!(STATVFS_F_FLAG(Vfs) & MNT_LOCAL);
 #endif
@@ -1133,19 +1164,6 @@ bool home_directory(SmallVectorImpl<char> &result) {
   return true;
 }
 
-bool cache_directory(SmallVectorImpl<char> &result) {
-  if (const char *RequestedDir = getenv("XDG_CACHE_HOME")) {
-    result.clear();
-    result.append(RequestedDir, RequestedDir + strlen(RequestedDir));
-    return true;
-  }
-  if (!home_directory(result)) {
-    return false;
-  }
-  append(result, ".cache");
-  return true;
-}
-
 static bool getDarwinConfDir(bool TempDir, SmallVectorImpl<char> &Result) {
   #if defined(_CS_DARWIN_USER_TEMP_DIR) && defined(_CS_DARWIN_USER_CACHE_DIR)
   // On Darwin, use DARWIN_USER_TEMP_DIR or DARWIN_USER_CACHE_DIR.
@@ -1171,6 +1189,51 @@ static bool getDarwinConfDir(bool TempDir, SmallVectorImpl<char> &Result) {
   return false;
 }
 
+bool user_config_directory(SmallVectorImpl<char> &result) {
+#ifdef __APPLE__
+  // Mac: ~/Library/Preferences/
+  if (home_directory(result)) {
+    append(result, "Library", "Preferences");
+    return true;
+  }
+#else
+  // XDG_CONFIG_HOME as defined in the XDG Base Directory Specification:
+  // http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
+  if (const char *RequestedDir = getenv("XDG_CONFIG_HOME")) {
+    result.clear();
+    result.append(RequestedDir, RequestedDir + strlen(RequestedDir));
+    return true;
+  }
+#endif
+  // Fallback: ~/.config
+  if (!home_directory(result)) {
+    return false;
+  }
+  append(result, ".config");
+  return true;
+}
+
+bool cache_directory(SmallVectorImpl<char> &result) {
+#ifdef __APPLE__
+  if (getDarwinConfDir(false/*tempDir*/, result)) {
+    return true;
+  }
+#else
+  // XDG_CACHE_HOME as defined in the XDG Base Directory Specification:
+  // http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
+  if (const char *RequestedDir = getenv("XDG_CACHE_HOME")) {
+    result.clear();
+    result.append(RequestedDir, RequestedDir + strlen(RequestedDir));
+    return true;
+  }
+#endif
+  if (!home_directory(result)) {
+    return false;
+  }
+  append(result, ".cache");
+  return true;
+}
+
 static const char *getEnvTempDir() {
   // Check whether the temporary directory is specified by an environment
   // variable.
diff --git a/llvm/lib/Support/Unix/Program.inc b/llvm/lib/Support/Unix/Program.inc
index f834e95a93d22..8f41fc0151635 100644
--- a/llvm/lib/Support/Unix/Program.inc
+++ b/llvm/lib/Support/Unix/Program.inc
@@ -15,6 +15,8 @@
 //===          is guaranteed to work on *all* UNIX variants.
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Support/Program.h"
+
 #include "Unix.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Config/config.h"
@@ -59,8 +61,7 @@
 #endif
 #endif
 
-namespace llvm {
-
+using namespace llvm;
 using namespace sys;
 
 ProcessInfo::ProcessInfo() : Pid(0), ReturnCode(0) {}
@@ -161,8 +162,6 @@ static void SetMemoryLimits(unsigned size) {
 #endif
 }
 
-}
-
 static std::vector<const char *>
 toNullTerminatedCStringArray(ArrayRef<StringRef> Strings, StringSaver &Saver) {
   std::vector<const char *> Result;
@@ -330,10 +329,54 @@ static bool Execute(ProcessInfo &PI, StringRef Program,
 }
 
 namespace llvm {
+namespace sys {
+
+#ifndef _AIX
+using ::wait4;
+#else
+static pid_t (wait4)(pid_t pid, int *status, int options, struct rusage *usage);
+#endif
+
+} // namespace sys
+} // namespace llvm
 
-ProcessInfo sys::Wait(const ProcessInfo &PI, unsigned SecondsToWait,
-                      bool WaitUntilTerminates, std::string *ErrMsg,
-                      Optional<ProcessStatistics> *ProcStat) {
+#ifdef _AIX
+#ifndef _ALL_SOURCE
+extern "C" pid_t (wait4)(pid_t pid, int *status, int options,
+                         struct rusage *usage);
+#endif
+pid_t (llvm::sys::wait4)(pid_t pid, int *status, int options,
+                         struct rusage *usage) {
+  assert(pid > 0 && "Only expecting to handle actual PID values!");
+  assert((options & ~WNOHANG) == 0 && "Expecting WNOHANG at most!");
+  assert(usage && "Expecting usage collection!");
+
+  // AIX wait4 does not work well with WNOHANG.
+  if (!(options & WNOHANG))
+    return ::wait4(pid, status, options, usage);
+
+  // For WNOHANG, we use waitid (which supports WNOWAIT) until the child process
+  // has terminated.
+  siginfo_t WaitIdInfo;
+  WaitIdInfo.si_pid = 0;
+  int WaitIdRetVal =
+      waitid(P_PID, pid, &WaitIdInfo, WNOWAIT | WEXITED | options);
+
+  if (WaitIdRetVal == -1 || WaitIdInfo.si_pid == 0)
+    return WaitIdRetVal;
+
+  assert(WaitIdInfo.si_pid == pid);
+
+  // The child has already terminated, so a blocking wait on it is okay in the
+  // absence of indiscriminate `wait` calls from the current process (which
+  // would cause the call here to fail with ECHILD).
+  return ::wait4(pid, status, options & ~WNOHANG, usage);
+}
+#endif
+
+ProcessInfo llvm::sys::Wait(const ProcessInfo &PI, unsigned SecondsToWait,
+                            bool WaitUntilTerminates, std::string *ErrMsg,
+                            Optional<ProcessStatistics> *ProcStat) {
   struct sigaction Act, Old;
   assert(PI.Pid && "invalid pid to wait on, process not started?");
 
@@ -349,6 +392,7 @@ ProcessInfo sys::Wait(const ProcessInfo &PI, unsigned SecondsToWait,
     Act.sa_handler = TimeOutHandler;
     sigemptyset(&Act.sa_mask);
     sigaction(SIGALRM, &Act, &Old);
+    // FIXME The alarm signal may be delivered to another thread.
     alarm(SecondsToWait);
   } else if (SecondsToWait == 0)
     WaitPidOptions = WNOHANG;
@@ -361,7 +405,7 @@ ProcessInfo sys::Wait(const ProcessInfo &PI, unsigned SecondsToWait,
     ProcStat->reset();
 
   do {
-    WaitResult.Pid = wait4(ChildPid, &status, WaitPidOptions, &Info);
+    WaitResult.Pid = sys::wait4(ChildPid, &status, WaitPidOptions, &Info);
   } while (WaitUntilTerminates && WaitResult.Pid == -1 && errno == EINTR);
 
   if (WaitResult.Pid != PI.Pid) {
@@ -378,6 +422,8 @@ ProcessInfo sys::Wait(const ProcessInfo &PI, unsigned SecondsToWait,
         sigaction(SIGALRM, &Old, nullptr);
 
         // Wait for child to die
+        // FIXME This could grab some other child process out from another
+        // waiting thread and then leave a zombie anyway.
         if (wait(&status) != ChildPid)
           MakeErrMsg(ErrMsg, "Child timed out but wouldn't die");
         else
@@ -440,12 +486,12 @@ ProcessInfo sys::Wait(const ProcessInfo &PI, unsigned SecondsToWait,
   return WaitResult;
 }
 
-std::error_code sys::ChangeStdinToBinary() {
+std::error_code llvm::sys::ChangeStdinToBinary() {
   // Do nothing, as Unix doesn't differentiate between text and binary.
   return std::error_code();
 }
 
-std::error_code sys::ChangeStdoutToBinary() {
+std::error_code llvm::sys::ChangeStdoutToBinary() {
   // Do nothing, as Unix doesn't differentiate between text and binary.
   return std::error_code();
 }
@@ -507,4 +553,3 @@ bool llvm::sys::commandLineFitsWithinSystemLimits(StringRef Program,
 
   return true;
 }
-}
diff --git a/llvm/lib/Support/Unix/Threading.inc b/llvm/lib/Support/Unix/Threading.inc
index 5c1bfc5620c64..2d0aacabf0920 100644
--- a/llvm/lib/Support/Unix/Threading.inc
+++ b/llvm/lib/Support/Unix/Threading.inc
@@ -37,6 +37,10 @@
 #include <lwp.h> // For _lwp_self()
 #endif
 
+#if defined(__OpenBSD__)
+#include <unistd.h> // For getthrid()
+#endif
+
 #if defined(__linux__)
 #include <sched.h>       // For sched_getaffinity
 #include <sys/syscall.h> // For syscall codes
@@ -90,6 +94,10 @@ llvm_execute_on_thread_impl(void *(*ThreadFunc)(void *), void *Arg,
     if ((errnum = ::pthread_join(Thread, nullptr)) != 0) {
       ReportErrnumFatal("pthread_join failed", errnum);
     }
+  } else if (JP == JoiningPolicy::Detach) {
+    if ((errnum = ::pthread_detach(Thread)) != 0) {
+      ReportErrnumFatal("pthread_detach failed", errnum);
+    }
   }
 }
 
@@ -105,6 +113,8 @@ uint64_t llvm::get_threadid() {
   return uint64_t(pthread_getthreadid_np());
 #elif defined(__NetBSD__)
   return uint64_t(_lwp_self());
+#elif defined(__OpenBSD__)
+  return uint64_t(getthrid());
 #elif defined(__ANDROID__)
   return uint64_t(gettid());
 #elif defined(__linux__)
diff --git a/llvm/lib/Support/Windows/Path.inc b/llvm/lib/Support/Windows/Path.inc
index ec62e656ddf0e..399a0cc7a25c5 100644
--- a/llvm/lib/Support/Windows/Path.inc
+++ b/llvm/lib/Support/Windows/Path.inc
@@ -555,6 +555,11 @@ std::error_code rename(const Twine &From, const Twine &To) {
                       NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
     if (FromHandle)
       break;
+
+    // We don't want to loop if the file doesn't exist.
+    auto EC = mapWindowsError(GetLastError());
+    if (EC == errc::no_such_file_or_directory)
+      return EC;
   }
   if (!FromHandle)
     return mapWindowsError(GetLastError());
@@ -1367,6 +1372,12 @@ bool home_directory(SmallVectorImpl<char> &result) {
   return getKnownFolderPath(FOLDERID_Profile, result);
 }
 
+bool user_config_directory(SmallVectorImpl<char> &result) {
+  // Either local or roaming appdata may be suitable in some cases, depending
+  // on the data. Local is more conservative, Roaming may not always be correct.
+  return getKnownFolderPath(FOLDERID_LocalAppData, result);
+}
+
 bool cache_directory(SmallVectorImpl<char> &result) {
   return getKnownFolderPath(FOLDERID_LocalAppData, result);
 }
diff --git a/llvm/lib/Support/WithColor.cpp b/llvm/lib/Support/WithColor.cpp
index 7e5fa17bb29ca..cb5f413d44b78 100644
--- a/llvm/lib/Support/WithColor.cpp
+++ b/llvm/lib/Support/WithColor.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/WithColor.h"
-#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/CommandLine.h"
 
 using namespace llvm;
 
diff --git a/llvm/lib/Support/X86TargetParser.cpp b/llvm/lib/Support/X86TargetParser.cpp
index ba85ed7ee6260..572d1203aaf21 100644
--- a/llvm/lib/Support/X86TargetParser.cpp
+++ b/llvm/lib/Support/X86TargetParser.cpp
@@ -15,44 +15,581 @@
 #include "llvm/ADT/Triple.h"
 
 using namespace llvm;
+using namespace llvm::X86;
 
-bool checkCPUKind(llvm::X86::CPUKind Kind, bool Only64Bit) {
-  using namespace X86;
-  // Perform any per-CPU checks necessary to determine if this CPU is
-  // acceptable.
-  switch (Kind) {
-  case CK_None:
-    // No processor selected!
-    return false;
-#define PROC(ENUM, STRING, IS64BIT)                                            \
-  case CK_##ENUM:                                                              \
-    return IS64BIT || !Only64Bit;
-#include "llvm/Support/X86TargetParser.def"
+namespace {
+
+/// Container class for CPU features.
+/// This is a constexpr reimplementation of a subset of std::bitset. It would be
+/// nice to use std::bitset directly, but it doesn't support constant
+/// initialization.
+class FeatureBitset {
+  static constexpr unsigned NUM_FEATURE_WORDS =
+      (X86::CPU_FEATURE_MAX + 31) / 32;
+
+  // This cannot be a std::array, operator[] is not constexpr until C++17.
+  uint32_t Bits[NUM_FEATURE_WORDS] = {};
+
+public:
+  constexpr FeatureBitset() = default;
+  constexpr FeatureBitset(std::initializer_list<unsigned> Init) {
+    for (auto I : Init)
+      set(I);
   }
-  llvm_unreachable("Unhandled CPU kind");
-}
 
-X86::CPUKind llvm::X86::parseArchX86(StringRef CPU, bool Only64Bit) {
-  X86::CPUKind Kind = llvm::StringSwitch<CPUKind>(CPU)
-#define PROC(ENUM, STRING, IS64BIT) .Case(STRING, CK_##ENUM)
-#define PROC_ALIAS(ENUM, ALIAS) .Case(ALIAS, CK_##ENUM)
+  constexpr FeatureBitset &set(unsigned I) {
+    // GCC <6.2 crashes if this is written in a single statement.
+    uint32_t NewBits = Bits[I / 32] | (uint32_t(1) << (I % 32));
+    Bits[I / 32] = NewBits;
+    return *this;
+  }
+
+  constexpr bool operator[](unsigned I) const {
+    uint32_t Mask = uint32_t(1) << (I % 32);
+    return (Bits[I / 32] & Mask) != 0;
+  }
+
+  constexpr FeatureBitset &operator&=(const FeatureBitset &RHS) {
+    for (unsigned I = 0, E = array_lengthof(Bits); I != E; ++I) {
+      // GCC <6.2 crashes if this is written in a single statement.
+      uint32_t NewBits = Bits[I] & RHS.Bits[I];
+      Bits[I] = NewBits;
+    }
+    return *this;
+  }
+
+  constexpr FeatureBitset &operator|=(const FeatureBitset &RHS) {
+    for (unsigned I = 0, E = array_lengthof(Bits); I != E; ++I) {
+      // GCC <6.2 crashes if this is written in a single statement.
+      uint32_t NewBits = Bits[I] | RHS.Bits[I];
+      Bits[I] = NewBits;
+    }
+    return *this;
+  }
+
+  // gcc 5.3 miscompiles this if we try to write this using operator&=.
+  constexpr FeatureBitset operator&(const FeatureBitset &RHS) const {
+    FeatureBitset Result;
+    for (unsigned I = 0, E = array_lengthof(Bits); I != E; ++I)
+      Result.Bits[I] = Bits[I] & RHS.Bits[I];
+    return Result;
+  }
+
+  // gcc 5.3 miscompiles this if we try to write this using operator&=.
+  constexpr FeatureBitset operator|(const FeatureBitset &RHS) const {
+    FeatureBitset Result;
+    for (unsigned I = 0, E = array_lengthof(Bits); I != E; ++I)
+      Result.Bits[I] = Bits[I] | RHS.Bits[I];
+    return Result;
+  }
+
+  constexpr FeatureBitset operator~() const {
+    FeatureBitset Result;
+    for (unsigned I = 0, E = array_lengthof(Bits); I != E; ++I)
+      Result.Bits[I] = ~Bits[I];
+    return Result;
+  }
+};
+
+struct ProcInfo {
+  StringLiteral Name;
+  X86::CPUKind Kind;
+  unsigned KeyFeature;
+  FeatureBitset Features;
+};
+
+struct FeatureInfo {
+  StringLiteral Name;
+  FeatureBitset ImpliedFeatures;
+};
+
+} // end anonymous namespace
+
+#define X86_FEATURE(ENUM, STRING)                                              \
+  static constexpr FeatureBitset Feature##ENUM = {X86::FEATURE_##ENUM};
 #include "llvm/Support/X86TargetParser.def"
-      .Default(CK_None);
 
-  if (!checkCPUKind(Kind, Only64Bit))
-    Kind = CK_None;
+// Pentium with MMX.
+static constexpr FeatureBitset FeaturesPentiumMMX =
+    FeatureX87 | FeatureCMPXCHG8B | FeatureMMX;
+
+// Pentium 2 and 3.
+static constexpr FeatureBitset FeaturesPentium2 =
+    FeatureX87 | FeatureCMPXCHG8B | FeatureMMX | FeatureFXSR;
+static constexpr FeatureBitset FeaturesPentium3 = FeaturesPentium2 | FeatureSSE;
+
+// Pentium 4 CPUs
+static constexpr FeatureBitset FeaturesPentium4 =
+    FeaturesPentium3 | FeatureSSE2;
+static constexpr FeatureBitset FeaturesPrescott =
+    FeaturesPentium4 | FeatureSSE3;
+static constexpr FeatureBitset FeaturesNocona =
+    FeaturesPrescott | Feature64BIT | FeatureCMPXCHG16B;
+
+// Basic 64-bit capable CPU.
+static constexpr FeatureBitset FeaturesX86_64 = FeaturesPentium4 | Feature64BIT;
+
+// Intel Core CPUs
+static constexpr FeatureBitset FeaturesCore2 =
+    FeaturesNocona | FeatureSAHF | FeatureSSSE3;
+static constexpr FeatureBitset FeaturesPenryn = FeaturesCore2 | FeatureSSE4_1;
+static constexpr FeatureBitset FeaturesNehalem =
+    FeaturesPenryn | FeaturePOPCNT | FeatureSSE4_2;
+static constexpr FeatureBitset FeaturesWestmere =
+    FeaturesNehalem | FeaturePCLMUL;
+static constexpr FeatureBitset FeaturesSandyBridge =
+    FeaturesWestmere | FeatureAVX | FeatureXSAVE | FeatureXSAVEOPT;
+static constexpr FeatureBitset FeaturesIvyBridge =
+    FeaturesSandyBridge | FeatureF16C | FeatureFSGSBASE | FeatureRDRND;
+static constexpr FeatureBitset FeaturesHaswell =
+    FeaturesIvyBridge | FeatureAVX2 | FeatureBMI | FeatureBMI2 | FeatureFMA |
+    FeatureINVPCID | FeatureLZCNT | FeatureMOVBE;
+static constexpr FeatureBitset FeaturesBroadwell =
+    FeaturesHaswell | FeatureADX | FeaturePRFCHW | FeatureRDSEED;
+
+// Intel Knights Landing and Knights Mill
+// Knights Landing has feature parity with Broadwell.
+static constexpr FeatureBitset FeaturesKNL =
+    FeaturesBroadwell | FeatureAES | FeatureAVX512F | FeatureAVX512CD |
+    FeatureAVX512ER | FeatureAVX512PF | FeaturePREFETCHWT1;
+static constexpr FeatureBitset FeaturesKNM =
+    FeaturesKNL | FeatureAVX512VPOPCNTDQ;
+
+// Intel Skylake processors.
+static constexpr FeatureBitset FeaturesSkylakeClient =
+    FeaturesBroadwell | FeatureAES | FeatureCLFLUSHOPT | FeatureXSAVEC |
+    FeatureXSAVES | FeatureSGX;
+// SkylakeServer inherits all SkylakeClient features except SGX.
+// FIXME: That doesn't match gcc.
+static constexpr FeatureBitset FeaturesSkylakeServer =
+    (FeaturesSkylakeClient & ~FeatureSGX) | FeatureAVX512F | FeatureAVX512CD |
+    FeatureAVX512DQ | FeatureAVX512BW | FeatureAVX512VL | FeatureCLWB |
+    FeaturePKU;
+static constexpr FeatureBitset FeaturesCascadeLake =
+    FeaturesSkylakeServer | FeatureAVX512VNNI;
+static constexpr FeatureBitset FeaturesCooperLake =
+    FeaturesCascadeLake | FeatureAVX512BF16;
+
+// Intel 10nm processors.
+static constexpr FeatureBitset FeaturesCannonlake =
+    FeaturesSkylakeClient | FeatureAVX512F | FeatureAVX512CD | FeatureAVX512DQ |
+    FeatureAVX512BW | FeatureAVX512VL | FeatureAVX512IFMA | FeatureAVX512VBMI |
+    FeaturePKU | FeatureSHA;
+static constexpr FeatureBitset FeaturesICLClient =
+    FeaturesCannonlake | FeatureAVX512BITALG | FeatureAVX512VBMI2 |
+    FeatureAVX512VNNI | FeatureAVX512VPOPCNTDQ | FeatureCLWB | FeatureGFNI |
+    FeatureRDPID | FeatureVAES | FeatureVPCLMULQDQ;
+static constexpr FeatureBitset FeaturesICLServer =
+    FeaturesICLClient | FeaturePCONFIG | FeatureWBNOINVD;
+static constexpr FeatureBitset FeaturesTigerlake =
+    FeaturesICLClient | FeatureAVX512VP2INTERSECT | FeatureMOVDIR64B |
+    FeatureMOVDIRI | FeatureSHSTK;
+
+// Intel Atom processors.
+// Bonnell has feature parity with Core2 and adds MOVBE.
+static constexpr FeatureBitset FeaturesBonnell = FeaturesCore2 | FeatureMOVBE;
+// Silvermont has parity with Westmere and Bonnell plus PRFCHW and RDRND.
+static constexpr FeatureBitset FeaturesSilvermont =
+    FeaturesBonnell | FeaturesWestmere | FeaturePRFCHW | FeatureRDRND;
+static constexpr FeatureBitset FeaturesGoldmont =
+    FeaturesSilvermont | FeatureAES | FeatureCLFLUSHOPT | FeatureFSGSBASE |
+    FeatureRDSEED | FeatureSHA | FeatureXSAVE | FeatureXSAVEC |
+    FeatureXSAVEOPT | FeatureXSAVES;
+static constexpr FeatureBitset FeaturesGoldmontPlus =
+    FeaturesGoldmont | FeaturePTWRITE | FeatureRDPID | FeatureSGX;
+static constexpr FeatureBitset FeaturesTremont =
+    FeaturesGoldmontPlus | FeatureCLWB | FeatureGFNI;
+
+// Geode Processor.
+static constexpr FeatureBitset FeaturesGeode =
+    FeatureX87 | FeatureCMPXCHG8B | FeatureMMX | Feature3DNOW | Feature3DNOWA;
+
+// K6 processor.
+static constexpr FeatureBitset FeaturesK6 =
+    FeatureX87 | FeatureCMPXCHG8B | FeatureMMX;
+
+// K7 and K8 architecture processors.
+static constexpr FeatureBitset FeaturesAthlon =
+    FeatureX87 | FeatureCMPXCHG8B | FeatureMMX | Feature3DNOW | Feature3DNOWA;
+static constexpr FeatureBitset FeaturesAthlonXP =
+    FeaturesAthlon | FeatureFXSR | FeatureSSE;
+static constexpr FeatureBitset FeaturesK8 =
+    FeaturesAthlonXP | FeatureSSE2 | Feature64BIT;
+static constexpr FeatureBitset FeaturesK8SSE3 = FeaturesK8 | FeatureSSE3;
+static constexpr FeatureBitset FeaturesAMDFAM10 =
+    FeaturesK8SSE3 | FeatureCMPXCHG16B | FeatureLZCNT | FeaturePOPCNT |
+    FeaturePRFCHW | FeatureSAHF | FeatureSSE4_A;
+
+// Bobcat architecture processors.
+static constexpr FeatureBitset FeaturesBTVER1 =
+    FeatureX87 | FeatureCMPXCHG8B | FeatureCMPXCHG16B | Feature64BIT |
+    FeatureFXSR | FeatureLZCNT | FeatureMMX | FeaturePOPCNT | FeaturePRFCHW |
+    FeatureSSE | FeatureSSE2 | FeatureSSE3 | FeatureSSSE3 | FeatureSSE4_A |
+    FeatureSAHF;
+static constexpr FeatureBitset FeaturesBTVER2 =
+    FeaturesBTVER1 | FeatureAES | FeatureAVX | FeatureBMI | FeatureF16C |
+    FeatureMOVBE | FeaturePCLMUL | FeatureXSAVE | FeatureXSAVEOPT;
+
+// AMD Bulldozer architecture processors.
+static constexpr FeatureBitset FeaturesBDVER1 =
+    FeatureX87 | FeatureAES | FeatureAVX | FeatureCMPXCHG8B |
+    FeatureCMPXCHG16B | Feature64BIT | FeatureFMA4 | FeatureFXSR | FeatureLWP |
+    FeatureLZCNT | FeatureMMX | FeaturePCLMUL | FeaturePOPCNT | FeaturePRFCHW |
+    FeatureSAHF | FeatureSSE | FeatureSSE2 | FeatureSSE3 | FeatureSSSE3 |
+    FeatureSSE4_1 | FeatureSSE4_2 | FeatureSSE4_A | FeatureXOP | FeatureXSAVE;
+static constexpr FeatureBitset FeaturesBDVER2 =
+    FeaturesBDVER1 | FeatureBMI | FeatureFMA | FeatureF16C | FeatureTBM;
+static constexpr FeatureBitset FeaturesBDVER3 =
+    FeaturesBDVER2 | FeatureFSGSBASE | FeatureXSAVEOPT;
+static constexpr FeatureBitset FeaturesBDVER4 =
+    FeaturesBDVER3 | FeatureAVX2 | FeatureBMI2 | FeatureMOVBE | FeatureMWAITX |
+    FeatureRDRND;
+
+// AMD Zen architecture processors.
+static constexpr FeatureBitset FeaturesZNVER1 =
+    FeatureX87 | FeatureADX | FeatureAES | FeatureAVX | FeatureAVX2 |
+    FeatureBMI | FeatureBMI2 | FeatureCLFLUSHOPT | FeatureCLZERO |
+    FeatureCMPXCHG8B | FeatureCMPXCHG16B | Feature64BIT | FeatureF16C |
+    FeatureFMA | FeatureFSGSBASE | FeatureFXSR | FeatureLZCNT | FeatureMMX |
+    FeatureMOVBE | FeatureMWAITX | FeaturePCLMUL | FeaturePOPCNT |
+    FeaturePRFCHW | FeatureRDRND | FeatureRDSEED | FeatureSAHF | FeatureSHA |
+    FeatureSSE | FeatureSSE2 | FeatureSSE3 | FeatureSSSE3 | FeatureSSE4_1 |
+    FeatureSSE4_2 | FeatureSSE4_A | FeatureXSAVE | FeatureXSAVEC |
+    FeatureXSAVEOPT | FeatureXSAVES;
+static constexpr FeatureBitset FeaturesZNVER2 =
+    FeaturesZNVER1 | FeatureCLWB | FeatureRDPID | FeatureWBNOINVD;
 
-  return Kind;
+static constexpr ProcInfo Processors[] = {
+  // Empty processor. Include X87 and CMPXCHG8 for backwards compatibility.
+  { {""}, CK_None, ~0U, FeatureX87 | FeatureCMPXCHG8B },
+  // i386-generation processors.
+  { {"i386"}, CK_i386, ~0U, FeatureX87 },
+  // i486-generation processors.
+  { {"i486"}, CK_i486, ~0U, FeatureX87 },
+  { {"winchip-c6"}, CK_WinChipC6, ~0U, FeaturesPentiumMMX },
+  { {"winchip2"}, CK_WinChip2, ~0U, FeaturesPentiumMMX | Feature3DNOW },
+  { {"c3"}, CK_C3, ~0U, FeaturesPentiumMMX | Feature3DNOW },
+  // i586-generation processors, P5 microarchitecture based.
+  { {"i586"}, CK_i586, ~0U, FeatureX87 | FeatureCMPXCHG8B },
+  { {"pentium"}, CK_Pentium, ~0U, FeatureX87 | FeatureCMPXCHG8B },
+  { {"pentium-mmx"}, CK_PentiumMMX, ~0U, FeaturesPentiumMMX },
+  // i686-generation processors, P6 / Pentium M microarchitecture based.
+  { {"pentiumpro"}, CK_PentiumPro, ~0U, FeatureX87 | FeatureCMPXCHG8B },
+  { {"i686"}, CK_i686, ~0U, FeatureX87 | FeatureCMPXCHG8B },
+  { {"pentium2"}, CK_Pentium2, ~0U, FeaturesPentium2 },
+  { {"pentium3"}, CK_Pentium3, ~0U, FeaturesPentium3 },
+  { {"pentium3m"}, CK_Pentium3, ~0U, FeaturesPentium3 },
+  { {"pentium-m"}, CK_PentiumM, ~0U, FeaturesPentium4 },
+  { {"c3-2"}, CK_C3_2, ~0U, FeaturesPentium3 },
+  { {"yonah"}, CK_Yonah, ~0U, FeaturesPrescott },
+  // Netburst microarchitecture based processors.
+  { {"pentium4"}, CK_Pentium4, ~0U, FeaturesPentium4 },
+  { {"pentium4m"}, CK_Pentium4, ~0U, FeaturesPentium4 },
+  { {"prescott"}, CK_Prescott, ~0U, FeaturesPrescott },
+  { {"nocona"}, CK_Nocona, ~0U, FeaturesNocona },
+  // Core microarchitecture based processors.
+  { {"core2"}, CK_Core2, ~0U, FeaturesCore2 },
+  { {"penryn"}, CK_Penryn, ~0U, FeaturesPenryn },
+  // Atom processors
+  { {"bonnell"}, CK_Bonnell, FEATURE_SSSE3, FeaturesBonnell },
+  { {"atom"}, CK_Bonnell, FEATURE_SSSE3, FeaturesBonnell },
+  { {"silvermont"}, CK_Silvermont, FEATURE_SSE4_2, FeaturesSilvermont },
+  { {"slm"}, CK_Silvermont, FEATURE_SSE4_2, FeaturesSilvermont },
+  { {"goldmont"}, CK_Goldmont, FEATURE_SSE4_2, FeaturesGoldmont },
+  { {"goldmont-plus"}, CK_GoldmontPlus, FEATURE_SSE4_2, FeaturesGoldmontPlus },
+  { {"tremont"}, CK_Tremont, FEATURE_SSE4_2, FeaturesTremont },
+  // Nehalem microarchitecture based processors.
+  { {"nehalem"}, CK_Nehalem, FEATURE_SSE4_2, FeaturesNehalem },
+  { {"corei7"}, CK_Nehalem, FEATURE_SSE4_2, FeaturesNehalem },
+  // Westmere microarchitecture based processors.
+  { {"westmere"}, CK_Westmere, FEATURE_PCLMUL, FeaturesWestmere },
+  // Sandy Bridge microarchitecture based processors.
+  { {"sandybridge"}, CK_SandyBridge, FEATURE_AVX, FeaturesSandyBridge },
+  { {"corei7-avx"}, CK_SandyBridge, FEATURE_AVX, FeaturesSandyBridge },
+  // Ivy Bridge microarchitecture based processors.
+  { {"ivybridge"}, CK_IvyBridge, FEATURE_AVX, FeaturesIvyBridge },
+  { {"core-avx-i"}, CK_IvyBridge, FEATURE_AVX, FeaturesIvyBridge },
+  // Haswell microarchitecture based processors.
+  { {"haswell"}, CK_Haswell, FEATURE_AVX2, FeaturesHaswell },
+  { {"core-avx2"}, CK_Haswell, FEATURE_AVX2, FeaturesHaswell },
+  // Broadwell microarchitecture based processors.
+  { {"broadwell"}, CK_Broadwell, FEATURE_AVX2, FeaturesBroadwell },
+  // Skylake client microarchitecture based processors.
+  { {"skylake"}, CK_SkylakeClient, FEATURE_AVX2, FeaturesSkylakeClient },
+  // Skylake server microarchitecture based processors.
+  { {"skylake-avx512"}, CK_SkylakeServer, FEATURE_AVX512F, FeaturesSkylakeServer },
+  { {"skx"}, CK_SkylakeServer, FEATURE_AVX512F, FeaturesSkylakeServer },
+  // Cascadelake Server microarchitecture based processors.
+  { {"cascadelake"}, CK_Cascadelake, FEATURE_AVX512VNNI, FeaturesCascadeLake },
+  // Cooperlake Server microarchitecture based processors.
+  { {"cooperlake"}, CK_Cooperlake, FEATURE_AVX512BF16, FeaturesCooperLake },
+  // Cannonlake client microarchitecture based processors.
+  { {"cannonlake"}, CK_Cannonlake, FEATURE_AVX512VBMI, FeaturesCannonlake },
+  // Icelake client microarchitecture based processors.
+  { {"icelake-client"}, CK_IcelakeClient, FEATURE_AVX512VBMI2, FeaturesICLClient },
+  // Icelake server microarchitecture based processors.
+  { {"icelake-server"}, CK_IcelakeServer, FEATURE_AVX512VBMI2, FeaturesICLServer },
+  // Tigerlake microarchitecture based processors.
+  { {"tigerlake"}, CK_Tigerlake, FEATURE_AVX512VP2INTERSECT, FeaturesTigerlake },
+  // Knights Landing processor.
+  { {"knl"}, CK_KNL, FEATURE_AVX512F, FeaturesKNL },
+  // Knights Mill processor.
+  { {"knm"}, CK_KNM, FEATURE_AVX5124FMAPS, FeaturesKNM },
+  // Lakemont microarchitecture based processors.
+  { {"lakemont"}, CK_Lakemont, ~0U, FeatureCMPXCHG8B },
+  // K6 architecture processors.
+  { {"k6"}, CK_K6, ~0U, FeaturesK6 },
+  { {"k6-2"}, CK_K6_2, ~0U, FeaturesK6 | Feature3DNOW },
+  { {"k6-3"}, CK_K6_3, ~0U, FeaturesK6 | Feature3DNOW },
+  // K7 architecture processors.
+  { {"athlon"}, CK_Athlon, ~0U, FeaturesAthlon },
+  { {"athlon-tbird"}, CK_Athlon, ~0U, FeaturesAthlon },
+  { {"athlon-xp"}, CK_AthlonXP, ~0U, FeaturesAthlonXP },
+  { {"athlon-mp"}, CK_AthlonXP, ~0U, FeaturesAthlonXP },
+  { {"athlon-4"}, CK_AthlonXP, ~0U, FeaturesAthlonXP },
+  // K8 architecture processors.
+  { {"k8"}, CK_K8, ~0U, FeaturesK8 },
+  { {"athlon64"}, CK_K8, ~0U, FeaturesK8 },
+  { {"athlon-fx"}, CK_K8, ~0U, FeaturesK8 },
+  { {"opteron"}, CK_K8, ~0U, FeaturesK8 },
+  { {"k8-sse3"}, CK_K8SSE3, ~0U, FeaturesK8SSE3 },
+  { {"athlon64-sse3"}, CK_K8SSE3, ~0U, FeaturesK8SSE3 },
+  { {"opteron-sse3"}, CK_K8SSE3, ~0U, FeaturesK8SSE3 },
+  { {"amdfam10"}, CK_AMDFAM10, FEATURE_SSE4_A, FeaturesAMDFAM10 },
+  { {"barcelona"}, CK_AMDFAM10, FEATURE_SSE4_A, FeaturesAMDFAM10 },
+  // Bobcat architecture processors.
+  { {"btver1"}, CK_BTVER1, FEATURE_SSE4_A, FeaturesBTVER1 },
+  { {"btver2"}, CK_BTVER2, FEATURE_BMI, FeaturesBTVER2 },
+  // Bulldozer architecture processors.
+  { {"bdver1"}, CK_BDVER1, FEATURE_XOP, FeaturesBDVER1 },
+  { {"bdver2"}, CK_BDVER2, FEATURE_FMA, FeaturesBDVER2 },
+  { {"bdver3"}, CK_BDVER3, FEATURE_FMA, FeaturesBDVER3 },
+  { {"bdver4"}, CK_BDVER4, FEATURE_AVX2, FeaturesBDVER4 },
+  // Zen architecture processors.
+  { {"znver1"}, CK_ZNVER1, FEATURE_AVX2, FeaturesZNVER1 },
+  { {"znver2"}, CK_ZNVER2, FEATURE_AVX2, FeaturesZNVER2 },
+  // Generic 64-bit processor.
+  { {"x86-64"}, CK_x86_64, ~0U, FeaturesX86_64 },
+  // Geode processors.
+  { {"geode"}, CK_Geode, ~0U, FeaturesGeode },
+};
+
+X86::CPUKind llvm::X86::parseArchX86(StringRef CPU, bool Only64Bit) {
+  for (const auto &P : Processors)
+    if (P.Name == CPU && (P.Features[FEATURE_64BIT] || !Only64Bit))
+      return P.Kind;
+
+  return CK_None;
 }
 
 void llvm::X86::fillValidCPUArchList(SmallVectorImpl<StringRef> &Values,
                                      bool Only64Bit) {
-#define PROC(ENUM, STRING, IS64BIT)                                            \
-  if (IS64BIT || !Only64Bit)                                                   \
-    Values.emplace_back(STRING);
-  // For aliases we need to lookup the CPUKind to get the 64-bit ness.
-#define PROC_ALIAS(ENUM, ALIAS)                                                \
-  if (checkCPUKind(CK_##ENUM, Only64Bit))                                      \
-    Values.emplace_back(ALIAS);
+  for (const auto &P : Processors)
+    if (!P.Name.empty() && (P.Features[FEATURE_64BIT] || !Only64Bit))
+      Values.emplace_back(P.Name);
+}
+
+ProcessorFeatures llvm::X86::getKeyFeature(X86::CPUKind Kind) {
+  // FIXME: Can we avoid a linear search here? The table might be sorted by
+  // CPUKind so we could binary search?
+  for (const auto &P : Processors) {
+    if (P.Kind == Kind) {
+      assert(P.KeyFeature != ~0U && "Processor does not have a key feature.");
+      return static_cast<ProcessorFeatures>(P.KeyFeature);
+    }
+  }
+
+  llvm_unreachable("Unable to find CPU kind!");
+}
+
+// Features with no dependencies.
+static constexpr FeatureBitset ImpliedFeatures64BIT = {};
+static constexpr FeatureBitset ImpliedFeaturesADX = {};
+static constexpr FeatureBitset ImpliedFeaturesBMI = {};
+static constexpr FeatureBitset ImpliedFeaturesBMI2 = {};
+static constexpr FeatureBitset ImpliedFeaturesCLDEMOTE = {};
+static constexpr FeatureBitset ImpliedFeaturesCLFLUSHOPT = {};
+static constexpr FeatureBitset ImpliedFeaturesCLWB = {};
+static constexpr FeatureBitset ImpliedFeaturesCLZERO = {};
+static constexpr FeatureBitset ImpliedFeaturesCMOV = {};
+static constexpr FeatureBitset ImpliedFeaturesCMPXCHG16B = {};
+static constexpr FeatureBitset ImpliedFeaturesCMPXCHG8B = {};
+static constexpr FeatureBitset ImpliedFeaturesENQCMD = {};
+static constexpr FeatureBitset ImpliedFeaturesFSGSBASE = {};
+static constexpr FeatureBitset ImpliedFeaturesFXSR = {};
+static constexpr FeatureBitset ImpliedFeaturesINVPCID = {};
+static constexpr FeatureBitset ImpliedFeaturesLWP = {};
+static constexpr FeatureBitset ImpliedFeaturesLZCNT = {};
+static constexpr FeatureBitset ImpliedFeaturesMWAITX = {};
+static constexpr FeatureBitset ImpliedFeaturesMOVBE = {};
+static constexpr FeatureBitset ImpliedFeaturesMOVDIR64B = {};
+static constexpr FeatureBitset ImpliedFeaturesMOVDIRI = {};
+static constexpr FeatureBitset ImpliedFeaturesPCONFIG = {};
+static constexpr FeatureBitset ImpliedFeaturesPOPCNT = {};
+static constexpr FeatureBitset ImpliedFeaturesPKU = {};
+static constexpr FeatureBitset ImpliedFeaturesPREFETCHWT1 = {};
+static constexpr FeatureBitset ImpliedFeaturesPRFCHW = {};
+static constexpr FeatureBitset ImpliedFeaturesPTWRITE = {};
+static constexpr FeatureBitset ImpliedFeaturesRDPID = {};
+static constexpr FeatureBitset ImpliedFeaturesRDRND = {};
+static constexpr FeatureBitset ImpliedFeaturesRDSEED = {};
+static constexpr FeatureBitset ImpliedFeaturesRTM = {};
+static constexpr FeatureBitset ImpliedFeaturesSAHF = {};
+static constexpr FeatureBitset ImpliedFeaturesSERIALIZE = {};
+static constexpr FeatureBitset ImpliedFeaturesSGX = {};
+static constexpr FeatureBitset ImpliedFeaturesSHSTK = {};
+static constexpr FeatureBitset ImpliedFeaturesTBM = {};
+static constexpr FeatureBitset ImpliedFeaturesTSXLDTRK = {};
+static constexpr FeatureBitset ImpliedFeaturesWAITPKG = {};
+static constexpr FeatureBitset ImpliedFeaturesWBNOINVD = {};
+static constexpr FeatureBitset ImpliedFeaturesVZEROUPPER = {};
+static constexpr FeatureBitset ImpliedFeaturesX87 = {};
+static constexpr FeatureBitset ImpliedFeaturesXSAVE = {};
+
+// Not really CPU features, but need to be in the table because clang uses
+// target features to communicate them to the backend.
+static constexpr FeatureBitset ImpliedFeaturesRETPOLINE_EXTERNAL_THUNK = {};
+static constexpr FeatureBitset ImpliedFeaturesRETPOLINE_INDIRECT_BRANCHES = {};
+static constexpr FeatureBitset ImpliedFeaturesRETPOLINE_INDIRECT_CALLS = {};
+static constexpr FeatureBitset ImpliedFeaturesLVI_CFI = {};
+static constexpr FeatureBitset ImpliedFeaturesLVI_LOAD_HARDENING = {};
+
+// XSAVE features are dependent on basic XSAVE.
+static constexpr FeatureBitset ImpliedFeaturesXSAVEC = FeatureXSAVE;
+static constexpr FeatureBitset ImpliedFeaturesXSAVEOPT = FeatureXSAVE;
+static constexpr FeatureBitset ImpliedFeaturesXSAVES = FeatureXSAVE;
+
+// MMX->3DNOW->3DNOWA chain.
+static constexpr FeatureBitset ImpliedFeaturesMMX = {};
+static constexpr FeatureBitset ImpliedFeatures3DNOW = FeatureMMX;
+static constexpr FeatureBitset ImpliedFeatures3DNOWA = Feature3DNOW;
+
+// SSE/AVX/AVX512F chain.
+static constexpr FeatureBitset ImpliedFeaturesSSE = {};
+static constexpr FeatureBitset ImpliedFeaturesSSE2 = FeatureSSE;
+static constexpr FeatureBitset ImpliedFeaturesSSE3 = FeatureSSE2;
+static constexpr FeatureBitset ImpliedFeaturesSSSE3 = FeatureSSE3;
+static constexpr FeatureBitset ImpliedFeaturesSSE4_1 = FeatureSSSE3;
+static constexpr FeatureBitset ImpliedFeaturesSSE4_2 = FeatureSSE4_1;
+static constexpr FeatureBitset ImpliedFeaturesAVX = FeatureSSE4_2;
+static constexpr FeatureBitset ImpliedFeaturesAVX2 = FeatureAVX;
+static constexpr FeatureBitset ImpliedFeaturesAVX512F =
+    FeatureAVX2 | FeatureF16C | FeatureFMA;
+
+// Vector extensions that build on SSE or AVX.
+static constexpr FeatureBitset ImpliedFeaturesAES = FeatureSSE2;
+static constexpr FeatureBitset ImpliedFeaturesF16C = FeatureAVX;
+static constexpr FeatureBitset ImpliedFeaturesFMA = FeatureAVX;
+static constexpr FeatureBitset ImpliedFeaturesGFNI = FeatureSSE2;
+static constexpr FeatureBitset ImpliedFeaturesPCLMUL = FeatureSSE2;
+static constexpr FeatureBitset ImpliedFeaturesSHA = FeatureSSE2;
+static constexpr FeatureBitset ImpliedFeaturesVAES = FeatureAES | FeatureAVX;
+static constexpr FeatureBitset ImpliedFeaturesVPCLMULQDQ =
+    FeatureAVX | FeaturePCLMUL;
+
+// AVX512 features.
+static constexpr FeatureBitset ImpliedFeaturesAVX512CD = FeatureAVX512F;
+static constexpr FeatureBitset ImpliedFeaturesAVX512BW = FeatureAVX512F;
+static constexpr FeatureBitset ImpliedFeaturesAVX512DQ = FeatureAVX512F;
+static constexpr FeatureBitset ImpliedFeaturesAVX512ER = FeatureAVX512F;
+static constexpr FeatureBitset ImpliedFeaturesAVX512PF = FeatureAVX512F;
+static constexpr FeatureBitset ImpliedFeaturesAVX512VL = FeatureAVX512F;
+
+static constexpr FeatureBitset ImpliedFeaturesAVX512BF16 = FeatureAVX512BW;
+static constexpr FeatureBitset ImpliedFeaturesAVX512BITALG = FeatureAVX512BW;
+static constexpr FeatureBitset ImpliedFeaturesAVX512IFMA = FeatureAVX512F;
+static constexpr FeatureBitset ImpliedFeaturesAVX512VNNI = FeatureAVX512F;
+static constexpr FeatureBitset ImpliedFeaturesAVX512VPOPCNTDQ = FeatureAVX512F;
+static constexpr FeatureBitset ImpliedFeaturesAVX512VBMI = FeatureAVX512BW;
+static constexpr FeatureBitset ImpliedFeaturesAVX512VBMI2 = FeatureAVX512BW;
+static constexpr FeatureBitset ImpliedFeaturesAVX512VP2INTERSECT =
+    FeatureAVX512F;
+
+// FIXME: These two aren't really implemented and just exist in the feature
+// list for __builtin_cpu_supports. So omit their dependencies.
+static constexpr FeatureBitset ImpliedFeaturesAVX5124FMAPS = {};
+static constexpr FeatureBitset ImpliedFeaturesAVX5124VNNIW = {};
+
+// SSE4_A->FMA4->XOP chain.
+static constexpr FeatureBitset ImpliedFeaturesSSE4_A = FeatureSSSE3;
+static constexpr FeatureBitset ImpliedFeaturesFMA4 = FeatureAVX | FeatureSSE4_A;
+static constexpr FeatureBitset ImpliedFeaturesXOP = FeatureFMA4;
+
+// AMX Features
+static constexpr FeatureBitset ImpliedFeaturesAMX_TILE = {};
+static constexpr FeatureBitset ImpliedFeaturesAMX_BF16 = FeatureAMX_TILE;
+static constexpr FeatureBitset ImpliedFeaturesAMX_INT8 = FeatureAMX_TILE;
+
+static constexpr FeatureInfo FeatureInfos[X86::CPU_FEATURE_MAX] = {
+#define X86_FEATURE(ENUM, STR) {{STR}, ImpliedFeatures##ENUM},
 #include "llvm/Support/X86TargetParser.def"
+};
+
+// Convert the set bits in FeatureBitset to a list of strings.
+static void getFeatureBitsAsStrings(const FeatureBitset &Bits,
+                                    SmallVectorImpl<StringRef> &Features) {
+  for (unsigned i = 0; i != CPU_FEATURE_MAX; ++i)
+    if (Bits[i] && !FeatureInfos[i].Name.empty())
+      Features.push_back(FeatureInfos[i].Name);
+}
+
+void llvm::X86::getFeaturesForCPU(StringRef CPU,
+                                  SmallVectorImpl<StringRef> &EnabledFeatures) {
+  auto I = llvm::find_if(Processors,
+                         [&](const ProcInfo &P) { return P.Name == CPU; });
+  assert(I != std::end(Processors) && "Processor not found!");
+
+  FeatureBitset Bits = I->Features;
+
+  // Remove the 64-bit feature which we only use to validate if a CPU can
+  // be used with 64-bit mode.
+  Bits &= ~Feature64BIT;
+
+  // Add the string version of all set bits.
+  getFeatureBitsAsStrings(Bits, EnabledFeatures);
+}
+
+// For each feature that is (transitively) implied by this feature, set it.
+static void getImpliedEnabledFeatures(FeatureBitset &Bits,
+                                      const FeatureBitset &Implies) {
+  Bits |= Implies;
+  for (unsigned i = 0; i != CPU_FEATURE_MAX; ++i) {
+    if (Implies[i])
+      getImpliedEnabledFeatures(Bits, FeatureInfos[i].ImpliedFeatures);
+  }
+}
+
+/// Create bit vector of features that are implied disabled if the feature
+/// passed in Value is disabled.
+static void getImpliedDisabledFeatures(FeatureBitset &Bits, unsigned Value) {
+  // Check all features looking for any dependent on this feature. If we find
+  // one, mark it and recursively find any feature that depend on it.
+  for (unsigned i = 0; i != CPU_FEATURE_MAX; ++i) {
+    if (FeatureInfos[i].ImpliedFeatures[Value]) {
+      Bits.set(i);
+      getImpliedDisabledFeatures(Bits, i);
+    }
+  }
+}
+
+void llvm::X86::getImpliedFeatures(
+    StringRef Feature, bool Enabled,
+    SmallVectorImpl<StringRef> &ImpliedFeatures) {
+  auto I = llvm::find_if(
+      FeatureInfos, [&](const FeatureInfo &FI) { return FI.Name == Feature; });
+  if (I == std::end(FeatureInfos)) {
+    // FIXME: This shouldn't happen, but may not have all features in the table
+    // yet.
+    return;
+  }
+
+  FeatureBitset ImpliedBits;
+  if (Enabled)
+    getImpliedEnabledFeatures(ImpliedBits, I->ImpliedFeatures);
+  else
+    getImpliedDisabledFeatures(ImpliedBits,
+                               std::distance(std::begin(FeatureInfos), I));
+
+  // Convert all the found bits into strings.
+  getFeatureBitsAsStrings(ImpliedBits, ImpliedFeatures);
 }
diff --git a/llvm/lib/Support/YAMLTraits.cpp b/llvm/lib/Support/YAMLTraits.cpp
index 752fab2be9b38..9ac7c65e19f73 100644
--- a/llvm/lib/Support/YAMLTraits.cpp
+++ b/llvm/lib/Support/YAMLTraits.cpp
@@ -878,12 +878,12 @@ StringRef ScalarTraits<StringRef>::input(StringRef Scalar, void *,
 }
 
 void ScalarTraits<std::string>::output(const std::string &Val, void *,
-                                     raw_ostream &Out) {
+                                       raw_ostream &Out) {
   Out << Val;
 }
 
 StringRef ScalarTraits<std::string>::input(StringRef Scalar, void *,
-                                         std::string &Val) {
+                                           std::string &Val) {
   Val = Scalar.str();
   return StringRef();
 }
diff --git a/llvm/lib/Support/Z3Solver.cpp b/llvm/lib/Support/Z3Solver.cpp
index 3e0cb493559a1..9485536d13120 100644
--- a/llvm/lib/Support/Z3Solver.cpp
+++ b/llvm/lib/Support/Z3Solver.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Config/config.h"
 #include "llvm/Support/SMTAPI.h"
@@ -723,10 +724,25 @@ class Z3Solver : public SMTSolver {
   }
 
   SMTExprRef mkBitvector(const llvm::APSInt Int, unsigned BitWidth) override {
-    const SMTSortRef Sort = getBitvectorSort(BitWidth);
-    return newExprRef(
-        Z3Expr(Context, Z3_mk_numeral(Context.Context, Int.toString(10).c_str(),
-                                      toZ3Sort(*Sort).Sort)));
+    const Z3_sort Z3Sort = toZ3Sort(*getBitvectorSort(BitWidth)).Sort;
+
+    // Slow path, when 64 bits are not enough.
+    if (LLVM_UNLIKELY(Int.getBitWidth() > 64u)) {
+      SmallString<40> Buffer;
+      Int.toString(Buffer, 10);
+      return newExprRef(Z3Expr(
+          Context, Z3_mk_numeral(Context.Context, Buffer.c_str(), Z3Sort)));
+    }
+
+    const int64_t BitReprAsSigned = Int.getExtValue();
+    const uint64_t BitReprAsUnsigned =
+        reinterpret_cast<const uint64_t &>(BitReprAsSigned);
+
+    Z3_ast Literal =
+        Int.isSigned()
+            ? Z3_mk_int64(Context.Context, BitReprAsSigned, Z3Sort)
+            : Z3_mk_unsigned_int64(Context.Context, BitReprAsUnsigned, Z3Sort);
+    return newExprRef(Z3Expr(Context, Literal));
   }
 
   SMTExprRef mkFloat(const llvm::APFloat Float) override {
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index ce559c9d6c2fc..534af9686af06 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -103,6 +103,24 @@ def FeatureCCPP : SubtargetFeature<"ccpp", "HasCCPP",
 def FeatureSVE : SubtargetFeature<"sve", "HasSVE", "true",
   "Enable Scalable Vector Extension (SVE) instructions", [FeatureFullFP16]>;
 
+// This flag is currently still labeled as Experimental, but when fully
+// implemented this should tell the compiler to use the zeroing pseudos to
+// benefit from the reverse instructions (e.g. SUB vs SUBR) if the inactive
+// lanes are known to be zero. The pseudos will then be expanded using the
+// MOVPRFX instruction to zero the inactive lanes. This feature should only be
+// enabled if MOVPRFX instructions are known to merge with the destructive
+// operations they prefix.
+//
+// This feature could similarly be extended to support cheap merging of _any_
+// value into the inactive lanes using the MOVPRFX instruction that uses
+// merging-predication.
+def FeatureExperimentalZeroingPseudos
+    : SubtargetFeature<"use-experimental-zeroing-pseudos",
+                       "UseExperimentalZeroingPseudos", "true",
+                       "Hint to the compiler that the MOVPRFX instruction is "
+                       "merged with destructive operations",
+                       []>;
+
 def FeatureSVE2 : SubtargetFeature<"sve2", "HasSVE2", "true",
   "Enable Scalable Vector Extension 2 (SVE2) instructions", [FeatureSVE]>;
 
@@ -608,6 +626,46 @@ def ProcA76     : SubtargetFeature<"a76", "ARMProcFamily", "CortexA76",
                                     FeatureSSBS
                                     ]>;
 
+def ProcA77     : SubtargetFeature<"a77", "ARMProcFamily", "CortexA77",
+                                   "Cortex-A77 ARM processors", [
+                                    HasV8_2aOps,
+                                    FeatureFPARMv8,
+                                    FeatureNEON, FeatureRCPC,
+                                    FeatureCrypto,
+                                    FeatureFullFP16,
+                                    FeatureDotProd
+                                    ]>;
+
+def ProcA78 : SubtargetFeature<"cortex-a78", "ARMProcFamily",
+                               "CortexA78",
+                               "Cortex-A78 ARM processors", [
+                               HasV8_2aOps,
+                               FeatureCrypto,
+                               FeatureFPARMv8,
+                               FeatureFuseAES,
+                               FeatureNEON,
+                               FeatureRCPC,
+                               FeaturePerfMon,
+                               FeaturePostRAScheduler,
+                               FeatureSPE,
+                               FeatureFullFP16,
+                               FeatureSSBS,
+                               FeatureDotProd]>;
+
+def ProcX1 : SubtargetFeature<"cortex-x1", "ARMProcFamily", "CortexX1",
+                                  "Cortex-X1 ARM processors", [
+                                  HasV8_2aOps,
+                                  FeatureCrypto,
+                                  FeatureFPARMv8,
+                                  FeatureFuseAES,
+                                  FeatureNEON,
+                                  FeatureRCPC,
+                                  FeaturePerfMon,
+                                  FeaturePostRAScheduler,
+                                  FeatureSPE,
+                                  FeatureFullFP16,
+                                  FeatureDotProd]>;
+
 def ProcA64FX : SubtargetFeature<"a64fx", "ARMProcFamily", "A64FX",
                                  "Fujitsu A64FX processors", [
                                   HasV8_2aOps,
@@ -949,6 +1007,9 @@ def : ProcessorModel<"cortex-a73", CortexA57Model, [ProcA73]>;
 def : ProcessorModel<"cortex-a75", CortexA57Model, [ProcA75]>;
 def : ProcessorModel<"cortex-a76", CortexA57Model, [ProcA76]>;
 def : ProcessorModel<"cortex-a76ae", CortexA57Model, [ProcA76]>;
+def : ProcessorModel<"cortex-a77", CortexA57Model, [ProcA77]>;
+def : ProcessorModel<"cortex-a78", CortexA57Model, [ProcA78]>;
+def : ProcessorModel<"cortex-x1", CortexA57Model, [ProcX1]>;
 def : ProcessorModel<"neoverse-e1", CortexA53Model, [ProcNeoverseE1]>;
 def : ProcessorModel<"neoverse-n1", CortexA57Model, [ProcNeoverseN1]>;
 def : ProcessorModel<"exynos-m3", ExynosM3Model, [ProcExynosM3]>;
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index f4ccf6592f059..3a94820dac8d3 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -749,11 +749,10 @@ void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
   assert(NOps == 4);
   OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
   // cast away const; DIetc do not take const operands for some reason.
-  OS << cast<DILocalVariable>(MI->getOperand(NOps - 2).getMetadata())
-            ->getName();
+  OS << MI->getDebugVariable()->getName();
   OS << " <- ";
   // Frame address.  Currently handles register +- offset only.
-  assert(MI->getOperand(0).isReg() && MI->getOperand(1).isImm());
+  assert(MI->getDebugOperand(0).isReg() && MI->isDebugOffsetImm());
   OS << '[';
   printOperand(MI, 0, OS);
   OS << '+';
diff --git a/llvm/lib/Target/AArch64/AArch64CallingConvention.cpp b/llvm/lib/Target/AArch64/AArch64CallingConvention.cpp
index 2d0d7a50c206f..84ec5afcc9c19 100644
--- a/llvm/lib/Target/AArch64/AArch64CallingConvention.cpp
+++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.cpp
@@ -42,7 +42,7 @@ static bool finishStackBlock(SmallVectorImpl<CCValAssign> &PendingMembers,
   unsigned Size = LocVT.getSizeInBits() / 8;
   const Align StackAlign =
       State.getMachineFunction().getDataLayout().getStackAlignment();
-  const Align OrigAlign(ArgFlags.getOrigAlign());
+  const Align OrigAlign = ArgFlags.getNonZeroOrigAlign();
   const Align Alignment = std::min(OrigAlign, StackAlign);
 
   for (auto &It : PendingMembers) {
diff --git a/llvm/lib/Target/AArch64/AArch64CallingConvention.td b/llvm/lib/Target/AArch64/AArch64CallingConvention.td
index a2219a240b9bc..fdcc890bf5892 100644
--- a/llvm/lib/Target/AArch64/AArch64CallingConvention.td
+++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.td
@@ -10,9 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-/// CCIfAlign - Match of the original alignment of the arg
-class CCIfAlign<string Align, CCAction A> :
-  CCIf<!strconcat("ArgFlags.getOrigAlign() == ", Align), A>;
 /// CCIfBigEndian - Match only if we're in big endian mode.
 class CCIfBigEndian<CCAction A> :
   CCIf<"State.getMachineFunction().getDataLayout().isBigEndian()", A>;
diff --git a/llvm/lib/Target/AArch64/AArch64ConditionalCompares.cpp b/llvm/lib/Target/AArch64/AArch64ConditionalCompares.cpp
index 4613002de5ff9..82e8df3b73f90 100644
--- a/llvm/lib/Target/AArch64/AArch64ConditionalCompares.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ConditionalCompares.cpp
@@ -828,7 +828,7 @@ void AArch64ConditionalCompares::updateDomTree(
     assert(Node != HeadNode && "Cannot erase the head node");
     assert(Node->getIDom() == HeadNode && "CmpBB should be dominated by Head");
     while (Node->getNumChildren())
-      DomTree->changeImmediateDominator(Node->getChildren().back(), HeadNode);
+      DomTree->changeImmediateDominator(Node->back(), HeadNode);
     DomTree->eraseNode(RemovedMBB);
   }
 }
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index b9034862c2707..9e65ad2e18f95 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -438,16 +438,17 @@ bool AArch64ExpandPseudo::expand_DestructiveOp(
     DOPRegIsUnique = true;
     break;
   }
-
-  assert (DOPRegIsUnique && "The destructive operand should be unique");
 #endif
 
   // Resolve the reverse opcode
   if (UseRev) {
-    if (AArch64::getSVERevInstr(Opcode) != -1)
-      Opcode = AArch64::getSVERevInstr(Opcode);
-    else if (AArch64::getSVEOrigInstr(Opcode) != -1)
-      Opcode = AArch64::getSVEOrigInstr(Opcode);
+    int NewOpcode;
+    // e.g. DIV -> DIVR
+    if ((NewOpcode = AArch64::getSVERevInstr(Opcode)) != -1)
+      Opcode = NewOpcode;
+    // e.g. DIVR -> DIV
+    else if ((NewOpcode = AArch64::getSVENonRevInstr(Opcode)) != -1)
+      Opcode = NewOpcode;
   }
 
   // Get the right MOVPRFX
@@ -480,6 +481,9 @@ bool AArch64ExpandPseudo::expand_DestructiveOp(
   //
   MachineInstrBuilder PRFX, DOP;
   if (FalseZero) {
+#ifndef NDEBUG
+    assert(DOPRegIsUnique && "The destructive operand should be unique");
+#endif
     assert(ElementSize != AArch64::ElementSizeNone &&
            "This instruction is unpredicated");
 
@@ -492,6 +496,9 @@ bool AArch64ExpandPseudo::expand_DestructiveOp(
     // After the movprfx, the destructive operand is same as Dst
     DOPIdx = 0;
   } else if (DstReg != MI.getOperand(DOPIdx).getReg()) {
+#ifndef NDEBUG
+    assert(DOPRegIsUnique && "The destructive operand should be unique");
+#endif
     PRFX = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(MovPrfx))
                .addReg(DstReg, RegState::Define)
                .addReg(MI.getOperand(DOPIdx).getReg());
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 29aa3d66989a2..bd76855f7c644 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -177,6 +177,38 @@ static cl::opt<bool> StackTaggingMergeSetTag(
 
 STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
 
+/// Returns the argument pop size.
+static uint64_t getArgumentPopSize(MachineFunction &MF,
+                                   MachineBasicBlock &MBB) {
+  MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
+  bool IsTailCallReturn = false;
+  if (MBB.end() != MBBI) {
+    unsigned RetOpcode = MBBI->getOpcode();
+    IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
+                       RetOpcode == AArch64::TCRETURNri ||
+                       RetOpcode == AArch64::TCRETURNriBTI;
+  }
+  AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
+
+  uint64_t ArgumentPopSize = 0;
+  if (IsTailCallReturn) {
+    MachineOperand &StackAdjust = MBBI->getOperand(1);
+
+    // For a tail-call in a callee-pops-arguments environment, some or all of
+    // the stack may actually be in use for the call's arguments, this is
+    // calculated during LowerCall and consumed here...
+    ArgumentPopSize = StackAdjust.getImm();
+  } else {
+    // ... otherwise the amount to pop is *all* of the argument space,
+    // conveniently stored in the MachineFunctionInfo by
+    // LowerFormalArguments. This will, of course, be zero for the C calling
+    // convention.
+    ArgumentPopSize = AFI->getArgumentStackToRestore();
+  }
+
+  return ArgumentPopSize;
+}
+
 /// This is the biggest offset to the stack pointer we can encode in aarch64
 /// instructions (without using a separate calculation and a temp register).
 /// Note that the exception here are vector stores/loads which cannot encode any
@@ -1416,7 +1448,6 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
   DebugLoc DL;
-  bool IsTailCallReturn = false;
   bool NeedsWinCFI = needsWinCFI(MF);
   bool HasWinCFI = false;
   bool IsFunclet = false;
@@ -1427,10 +1458,6 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
 
   if (MBB.end() != MBBI) {
     DL = MBBI->getDebugLoc();
-    unsigned RetOpcode = MBBI->getOpcode();
-    IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
-                       RetOpcode == AArch64::TCRETURNri ||
-                       RetOpcode == AArch64::TCRETURNriBTI;
     IsFunclet = isFuncletReturnInstr(*MBBI);
   }
 
@@ -1445,21 +1472,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
 
   // Initial and residual are named for consistency with the prologue. Note that
   // in the epilogue, the residual adjustment is executed first.
-  uint64_t ArgumentPopSize = 0;
-  if (IsTailCallReturn) {
-    MachineOperand &StackAdjust = MBBI->getOperand(1);
-
-    // For a tail-call in a callee-pops-arguments environment, some or all of
-    // the stack may actually be in use for the call's arguments, this is
-    // calculated during LowerCall and consumed here...
-    ArgumentPopSize = StackAdjust.getImm();
-  } else {
-    // ... otherwise the amount to pop is *all* of the argument space,
-    // conveniently stored in the MachineFunctionInfo by
-    // LowerFormalArguments. This will, of course, be zero for the C calling
-    // convention.
-    ArgumentPopSize = AFI->getArgumentStackToRestore();
-  }
+  uint64_t ArgumentPopSize = getArgumentPopSize(MF, MBB);
 
   // The stack frame should be like below,
   //
@@ -2519,8 +2532,8 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
       const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
       const TargetRegisterClass &RC = AArch64::GPR64RegClass;
       unsigned Size = TRI->getSpillSize(RC);
-      unsigned Align = TRI->getSpillAlignment(RC);
-      int FI = MFI.CreateStackObject(Size, Align, false);
+      Align Alignment = TRI->getSpillAlign(RC);
+      int FI = MFI.CreateStackObject(Size, Alignment, false);
       RS->addScavengingFrameIndex(FI);
       LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
                         << " as the emergency spill slot.\n");
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 9b26f0034dcfc..10c4778533533 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -904,16 +904,9 @@ bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
     if (!GAN)
       return true;
 
-    if (GAN->getOffset() % Size == 0) {
-      const GlobalValue *GV = GAN->getGlobal();
-      unsigned Alignment = GV->getAlignment();
-      Type *Ty = GV->getValueType();
-      if (Alignment == 0 && Ty->isSized())
-        Alignment = DL.getABITypeAlignment(Ty);
-
-      if (Alignment >= Size)
-        return true;
-    }
+    if (GAN->getOffset() % Size == 0 &&
+        GAN->getGlobal()->getPointerAlignment(DL) >= Size)
+      return true;
   }
 
   if (CurDAG->isBaseWithConstantOffset(N)) {
@@ -3247,6 +3240,63 @@ void AArch64DAGToDAGISel::SelectTagP(SDNode *N) {
   ReplaceNode(N, N3);
 }
 
+// NOTE: We cannot use EXTRACT_SUBREG in all cases because the fixed length
+// vector types larger than NEON don't have a matching SubRegIndex.
+static SDNode *extractSubReg(SelectionDAG *DAG, EVT VT, SDValue V) {
+  assert(V.getValueType().isScalableVector() &&
+         V.getValueType().getSizeInBits().getKnownMinSize() ==
+             AArch64::SVEBitsPerBlock &&
+         "Expected to extract from a packed scalable vector!");
+  assert(VT.isFixedLengthVector() &&
+         "Expected to extract a fixed length vector!");
+
+  SDLoc DL(V);
+  switch (VT.getSizeInBits()) {
+  case 64: {
+    auto SubReg = DAG->getTargetConstant(AArch64::dsub, DL, MVT::i32);
+    return DAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, VT, V, SubReg);
+  }
+  case 128: {
+    auto SubReg = DAG->getTargetConstant(AArch64::zsub, DL, MVT::i32);
+    return DAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, VT, V, SubReg);
+  }
+  default: {
+    auto RC = DAG->getTargetConstant(AArch64::ZPRRegClassID, DL, MVT::i64);
+    return DAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC);
+  }
+  }
+}
+
+// NOTE: We cannot use INSERT_SUBREG in all cases because the fixed length
+// vector types larger than NEON don't have a matching SubRegIndex.
+static SDNode *insertSubReg(SelectionDAG *DAG, EVT VT, SDValue V) {
+  assert(VT.isScalableVector() &&
+         VT.getSizeInBits().getKnownMinSize() == AArch64::SVEBitsPerBlock &&
+         "Expected to insert into a packed scalable vector!");
+  assert(V.getValueType().isFixedLengthVector() &&
+         "Expected to insert a fixed length vector!");
+
+  SDLoc DL(V);
+  switch (V.getValueType().getSizeInBits()) {
+  case 64: {
+    auto SubReg = DAG->getTargetConstant(AArch64::dsub, DL, MVT::i32);
+    auto Container = DAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
+    return DAG->getMachineNode(TargetOpcode::INSERT_SUBREG, DL, VT,
+                               SDValue(Container, 0), V, SubReg);
+  }
+  case 128: {
+    auto SubReg = DAG->getTargetConstant(AArch64::zsub, DL, MVT::i32);
+    auto Container = DAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
+    return DAG->getMachineNode(TargetOpcode::INSERT_SUBREG, DL, VT,
+                               SDValue(Container, 0), V, SubReg);
+  }
+  default: {
+    auto RC = DAG->getTargetConstant(AArch64::ZPRRegClassID, DL, MVT::i64);
+    return DAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC);
+  }
+  }
+}
+
 void AArch64DAGToDAGISel::Select(SDNode *Node) {
   // If we have a custom node, we already have selected!
   if (Node->isMachineOpcode()) {
@@ -3320,6 +3370,52 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
       return;
     break;
 
+  case ISD::EXTRACT_SUBVECTOR: {
+    // Bail when not a "cast" like extract_subvector.
+    if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue() != 0)
+      break;
+
+    // Bail when normal isel can do the job.
+    EVT InVT = Node->getOperand(0).getValueType();
+    if (VT.isScalableVector() || InVT.isFixedLengthVector())
+      break;
+
+    // NOTE: We can only get here when doing fixed length SVE code generation.
+    // We do manual selection because the types involved are not linked to real
+    // registers (despite being legal) and must be coerced into SVE registers.
+    //
+    // NOTE: If the above changes, be aware that selection will still not work
+    // because the td definition of extract_vector does not support extracting
+    // a fixed length vector from a scalable vector.
+
+    ReplaceNode(Node, extractSubReg(CurDAG, VT, Node->getOperand(0)));
+    return;
+  }
+
+  case ISD::INSERT_SUBVECTOR: {
+    // Bail when not a "cast" like insert_subvector.
+    if (cast<ConstantSDNode>(Node->getOperand(2))->getZExtValue() != 0)
+      break;
+    if (!Node->getOperand(0).isUndef())
+      break;
+
+    // Bail when normal isel should do the job.
+    EVT InVT = Node->getOperand(1).getValueType();
+    if (VT.isFixedLengthVector() || InVT.isScalableVector())
+      break;
+
+    // NOTE: We can only get here when doing fixed length SVE code generation.
+    // We do manual selection because the types involved are not linked to real
+    // registers (despite being legal) and must be coerced into SVE registers.
+    //
+    // NOTE: If the above changes, be aware that selection will still not work
+    // because the td definition of insert_vector does not support inserting a
+    // fixed length vector into a scalable vector.
+
+    ReplaceNode(Node, insertSubReg(CurDAG, VT, Node->getOperand(1)));
+    return;
+  }
+
   case ISD::Constant: {
     // Materialize zero constants as copies from WZR/XZR.  This allows
     // the coalescer to propagate these into other instructions.
@@ -3992,7 +4088,8 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
         SelectPredicatedStore</*Scale=*/0>(Node, 2, AArch64::ST2B,
                                            AArch64::ST2B_IMM);
         return;
-      } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16) {
+      } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
+                 (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) {
         SelectPredicatedStore</*Scale=*/1>(Node, 2, AArch64::ST2H,
                                            AArch64::ST2H_IMM);
         return;
@@ -4012,7 +4109,8 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
         SelectPredicatedStore</*Scale=*/0>(Node, 3, AArch64::ST3B,
                                            AArch64::ST3B_IMM);
         return;
-      } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16) {
+      } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
+                 (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) {
         SelectPredicatedStore</*Scale=*/1>(Node, 3, AArch64::ST3H,
                                            AArch64::ST3H_IMM);
         return;
@@ -4032,7 +4130,8 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
         SelectPredicatedStore</*Scale=*/0>(Node, 4, AArch64::ST4B,
                                            AArch64::ST4B_IMM);
         return;
-      } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16) {
+      } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
+                 (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) {
         SelectPredicatedStore</*Scale=*/1>(Node, 4, AArch64::ST4H,
                                            AArch64::ST4H_IMM);
         return;
@@ -4640,11 +4739,12 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
     }
     break;
   }
-  case AArch64ISD::SVE_LD2: {
+  case AArch64ISD::SVE_LD2_MERGE_ZERO: {
     if (VT == MVT::nxv16i8) {
       SelectPredicatedLoad(Node, 2, AArch64::LD2B_IMM);
       return;
-    } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16) {
+    } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
+               (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) {
       SelectPredicatedLoad(Node, 2, AArch64::LD2H_IMM);
       return;
     } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
@@ -4656,11 +4756,12 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
     }
     break;
   }
-  case AArch64ISD::SVE_LD3: {
+  case AArch64ISD::SVE_LD3_MERGE_ZERO: {
     if (VT == MVT::nxv16i8) {
       SelectPredicatedLoad(Node, 3, AArch64::LD3B_IMM);
       return;
-    } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16) {
+    } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
+               (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) {
       SelectPredicatedLoad(Node, 3, AArch64::LD3H_IMM);
       return;
     } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
@@ -4672,11 +4773,12 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
     }
     break;
   }
-  case AArch64ISD::SVE_LD4: {
+  case AArch64ISD::SVE_LD4_MERGE_ZERO: {
     if (VT == MVT::nxv16i8) {
       SelectPredicatedLoad(Node, 4, AArch64::LD4B_IMM);
       return;
-    } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16) {
+    } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
+               (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) {
       SelectPredicatedLoad(Node, 4, AArch64::LD4H_IMM);
       return;
     } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
@@ -4733,12 +4835,12 @@ static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
   // For custom ISD nodes, we have to look at them individually to extract the
   // type of the data moved to/from memory.
   switch (Opcode) {
-  case AArch64ISD::LD1:
-  case AArch64ISD::LD1S:
-  case AArch64ISD::LDNF1:
-  case AArch64ISD::LDNF1S:
+  case AArch64ISD::LD1_MERGE_ZERO:
+  case AArch64ISD::LD1S_MERGE_ZERO:
+  case AArch64ISD::LDNF1_MERGE_ZERO:
+  case AArch64ISD::LDNF1S_MERGE_ZERO:
     return cast<VTSDNode>(Root->getOperand(3))->getVT();
-  case AArch64ISD::ST1:
+  case AArch64ISD::ST1_PRED:
     return cast<VTSDNode>(Root->getOperand(4))->getVT();
   default:
     break;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 8eefbe62309ee..65ccc18ed6013 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -116,6 +116,18 @@ EnableOptimizeLogicalImm("aarch64-enable-logical-imm", cl::Hidden,
 /// Value type used for condition codes.
 static const MVT MVT_CC = MVT::i32;
 
+/// Returns true if VT's elements occupy the lowest bit positions of its
+/// associated register class without any intervening space.
+///
+/// For example, nxv2f16, nxv4f16 and nxv8f16 are legal types that belong to the
+/// same register class, but only nxv8f16 can be treated as a packed vector.
+static inline bool isPackedVectorType(EVT VT, SelectionDAG &DAG) {
+  assert(VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
+         "Expected legal vector type!");
+  return VT.isFixedLengthVector() ||
+         VT.getSizeInBits().getKnownMinSize() == AArch64::SVEBitsPerBlock;
+}
+
 AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
                                              const AArch64Subtarget &STI)
     : TargetLowering(TM), Subtarget(&STI) {
@@ -177,13 +189,16 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     addRegisterClass(MVT::nxv2f16, &AArch64::ZPRRegClass);
     addRegisterClass(MVT::nxv4f16, &AArch64::ZPRRegClass);
     addRegisterClass(MVT::nxv8f16, &AArch64::ZPRRegClass);
-    addRegisterClass(MVT::nxv2bf16, &AArch64::ZPRRegClass);
-    addRegisterClass(MVT::nxv4bf16, &AArch64::ZPRRegClass);
-    addRegisterClass(MVT::nxv8bf16, &AArch64::ZPRRegClass);
     addRegisterClass(MVT::nxv2f32, &AArch64::ZPRRegClass);
     addRegisterClass(MVT::nxv4f32, &AArch64::ZPRRegClass);
     addRegisterClass(MVT::nxv2f64, &AArch64::ZPRRegClass);
 
+    if (Subtarget->hasBF16()) {
+      addRegisterClass(MVT::nxv2bf16, &AArch64::ZPRRegClass);
+      addRegisterClass(MVT::nxv4bf16, &AArch64::ZPRRegClass);
+      addRegisterClass(MVT::nxv8bf16, &AArch64::ZPRRegClass);
+    }
+
     if (useSVEForFixedLengthVectors()) {
       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
         if (useSVEForFixedLengthVectorVT(VT))
@@ -199,6 +214,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::UADDSAT, VT, Legal);
       setOperationAction(ISD::SSUBSAT, VT, Legal);
       setOperationAction(ISD::USUBSAT, VT, Legal);
+      setOperationAction(ISD::UREM, VT, Expand);
+      setOperationAction(ISD::SREM, VT, Expand);
+      setOperationAction(ISD::SDIVREM, VT, Expand);
+      setOperationAction(ISD::UDIVREM, VT, Expand);
     }
 
     for (auto VT :
@@ -831,6 +850,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::UADDSAT, VT, Legal);
       setOperationAction(ISD::SSUBSAT, VT, Legal);
       setOperationAction(ISD::USUBSAT, VT, Legal);
+
+      setOperationAction(ISD::TRUNCATE, VT, Custom);
     }
     for (MVT VT : { MVT::v4f16, MVT::v2f32,
                     MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
@@ -899,6 +920,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     // D68877 for more details.
     for (MVT VT : MVT::integer_scalable_vector_valuetypes()) {
       if (isTypeLegal(VT)) {
+        setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
         setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
         setOperationAction(ISD::SELECT, VT, Custom);
         setOperationAction(ISD::SDIV, VT, Custom);
@@ -912,20 +934,42 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
         setOperationAction(ISD::SRA, VT, Custom);
         if (VT.getScalarType() == MVT::i1)
           setOperationAction(ISD::SETCC, VT, Custom);
-      } else {
-        for (auto VT : { MVT::nxv8i8, MVT::nxv4i16, MVT::nxv2i32 })
-          setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
       }
     }
+
+    for (auto VT : {MVT::nxv8i8, MVT::nxv4i16, MVT::nxv2i32})
+      setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
+
     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
 
     for (MVT VT : MVT::fp_scalable_vector_valuetypes()) {
       if (isTypeLegal(VT)) {
+        setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
         setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
         setOperationAction(ISD::SELECT, VT, Custom);
+        setOperationAction(ISD::FMA, VT, Custom);
       }
     }
+
+    // NOTE: Currently this has to happen after computeRegisterProperties rather
+    // than the preferred option of combining it with the addRegisterClass call.
+    if (useSVEForFixedLengthVectors()) {
+      for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
+        if (useSVEForFixedLengthVectorVT(VT))
+          addTypeForFixedLengthSVE(VT);
+      for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
+        if (useSVEForFixedLengthVectorVT(VT))
+          addTypeForFixedLengthSVE(VT);
+
+      // 64bit results can mean a bigger than NEON input.
+      for (auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32})
+        setOperationAction(ISD::TRUNCATE, VT, Custom);
+
+      // 128bit results imply a bigger than NEON input.
+      for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
+        setOperationAction(ISD::TRUNCATE, VT, Custom);
+    }
   }
 
   PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive();
@@ -1010,6 +1054,24 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) {
   }
 }
 
+void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
+  assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
+
+  // By default everything must be expanded.
+  for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
+    setOperationAction(Op, VT, Expand);
+
+  // We use EXTRACT_SUBVECTOR to "cast" a scalable vector to a fixed length one.
+  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
+
+  // Lower fixed length vector operations to scalable equivalents.
+  setOperationAction(ISD::ADD, VT, Custom);
+  setOperationAction(ISD::FADD, VT, Custom);
+  setOperationAction(ISD::LOAD, VT, Custom);
+  setOperationAction(ISD::STORE, VT, Custom);
+  setOperationAction(ISD::TRUNCATE, VT, Custom);
+}
+
 void AArch64TargetLowering::addDRTypeForNEON(MVT VT) {
   addRegisterClass(VT, &AArch64::FPR64RegClass);
   addTypeForNEON(VT, MVT::v2i32);
@@ -1125,7 +1187,8 @@ static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm,
 }
 
 bool AArch64TargetLowering::targetShrinkDemandedConstant(
-    SDValue Op, const APInt &Demanded, TargetLoweringOpt &TLO) const {
+    SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
+    TargetLoweringOpt &TLO) const {
   // Delay this optimization to as late as possible.
   if (!TLO.LegalOps)
     return false;
@@ -1142,7 +1205,7 @@ bool AArch64TargetLowering::targetShrinkDemandedConstant(
          "i32 or i64 is expected after legalization.");
 
   // Exit early if we demand all bits.
-  if (Demanded.countPopulation() == Size)
+  if (DemandedBits.countPopulation() == Size)
     return false;
 
   unsigned NewOpc;
@@ -1163,7 +1226,7 @@ bool AArch64TargetLowering::targetShrinkDemandedConstant(
   if (!C)
     return false;
   uint64_t Imm = C->getZExtValue();
-  return optimizeLogicalImm(Op, Size, Imm, Demanded, TLO, NewOpc);
+  return optimizeLogicalImm(Op, Size, Imm, DemandedBits, TLO, NewOpc);
 }
 
 /// computeKnownBitsForTargetNode - Determine which of the bits specified in
@@ -1298,243 +1361,246 @@ AArch64TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
 }
 
 const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
+#define MAKE_CASE(V)                                                           \
+  case V:                                                                      \
+    return #V;
   switch ((AArch64ISD::NodeType)Opcode) {
-  case AArch64ISD::FIRST_NUMBER:      break;
-  case AArch64ISD::CALL:              return "AArch64ISD::CALL";
-  case AArch64ISD::ADRP:              return "AArch64ISD::ADRP";
-  case AArch64ISD::ADR:               return "AArch64ISD::ADR";
-  case AArch64ISD::ADDlow:            return "AArch64ISD::ADDlow";
-  case AArch64ISD::LOADgot:           return "AArch64ISD::LOADgot";
-  case AArch64ISD::RET_FLAG:          return "AArch64ISD::RET_FLAG";
-  case AArch64ISD::BRCOND:            return "AArch64ISD::BRCOND";
-  case AArch64ISD::CSEL:              return "AArch64ISD::CSEL";
-  case AArch64ISD::FCSEL:             return "AArch64ISD::FCSEL";
-  case AArch64ISD::CSINV:             return "AArch64ISD::CSINV";
-  case AArch64ISD::CSNEG:             return "AArch64ISD::CSNEG";
-  case AArch64ISD::CSINC:             return "AArch64ISD::CSINC";
-  case AArch64ISD::THREAD_POINTER:    return "AArch64ISD::THREAD_POINTER";
-  case AArch64ISD::TLSDESC_CALLSEQ:   return "AArch64ISD::TLSDESC_CALLSEQ";
-  case AArch64ISD::SDIV_PRED:         return "AArch64ISD::SDIV_PRED";
-  case AArch64ISD::UDIV_PRED:         return "AArch64ISD::UDIV_PRED";
-  case AArch64ISD::SMIN_PRED:         return "AArch64ISD::SMIN_PRED";
-  case AArch64ISD::UMIN_PRED:         return "AArch64ISD::UMIN_PRED";
-  case AArch64ISD::SMAX_PRED:         return "AArch64ISD::SMAX_PRED";
-  case AArch64ISD::UMAX_PRED:         return "AArch64ISD::UMAX_PRED";
-  case AArch64ISD::SHL_PRED:          return "AArch64ISD::SHL_PRED";
-  case AArch64ISD::SRL_PRED:          return "AArch64ISD::SRL_PRED";
-  case AArch64ISD::SRA_PRED:          return "AArch64ISD::SRA_PRED";
-  case AArch64ISD::SETCC_PRED:        return "AArch64ISD::SETCC_PRED";
-  case AArch64ISD::ADC:               return "AArch64ISD::ADC";
-  case AArch64ISD::SBC:               return "AArch64ISD::SBC";
-  case AArch64ISD::ADDS:              return "AArch64ISD::ADDS";
-  case AArch64ISD::SUBS:              return "AArch64ISD::SUBS";
-  case AArch64ISD::ADCS:              return "AArch64ISD::ADCS";
-  case AArch64ISD::SBCS:              return "AArch64ISD::SBCS";
-  case AArch64ISD::ANDS:              return "AArch64ISD::ANDS";
-  case AArch64ISD::CCMP:              return "AArch64ISD::CCMP";
-  case AArch64ISD::CCMN:              return "AArch64ISD::CCMN";
-  case AArch64ISD::FCCMP:             return "AArch64ISD::FCCMP";
-  case AArch64ISD::FCMP:              return "AArch64ISD::FCMP";
-  case AArch64ISD::STRICT_FCMP:       return "AArch64ISD::STRICT_FCMP";
-  case AArch64ISD::STRICT_FCMPE:      return "AArch64ISD::STRICT_FCMPE";
-  case AArch64ISD::DUP:               return "AArch64ISD::DUP";
-  case AArch64ISD::DUPLANE8:          return "AArch64ISD::DUPLANE8";
-  case AArch64ISD::DUPLANE16:         return "AArch64ISD::DUPLANE16";
-  case AArch64ISD::DUPLANE32:         return "AArch64ISD::DUPLANE32";
-  case AArch64ISD::DUPLANE64:         return "AArch64ISD::DUPLANE64";
-  case AArch64ISD::MOVI:              return "AArch64ISD::MOVI";
-  case AArch64ISD::MOVIshift:         return "AArch64ISD::MOVIshift";
-  case AArch64ISD::MOVIedit:          return "AArch64ISD::MOVIedit";
-  case AArch64ISD::MOVImsl:           return "AArch64ISD::MOVImsl";
-  case AArch64ISD::FMOV:              return "AArch64ISD::FMOV";
-  case AArch64ISD::MVNIshift:         return "AArch64ISD::MVNIshift";
-  case AArch64ISD::MVNImsl:           return "AArch64ISD::MVNImsl";
-  case AArch64ISD::BICi:              return "AArch64ISD::BICi";
-  case AArch64ISD::ORRi:              return "AArch64ISD::ORRi";
-  case AArch64ISD::BSP:               return "AArch64ISD::BSP";
-  case AArch64ISD::NEG:               return "AArch64ISD::NEG";
-  case AArch64ISD::EXTR:              return "AArch64ISD::EXTR";
-  case AArch64ISD::ZIP1:              return "AArch64ISD::ZIP1";
-  case AArch64ISD::ZIP2:              return "AArch64ISD::ZIP2";
-  case AArch64ISD::UZP1:              return "AArch64ISD::UZP1";
-  case AArch64ISD::UZP2:              return "AArch64ISD::UZP2";
-  case AArch64ISD::TRN1:              return "AArch64ISD::TRN1";
-  case AArch64ISD::TRN2:              return "AArch64ISD::TRN2";
-  case AArch64ISD::REV16:             return "AArch64ISD::REV16";
-  case AArch64ISD::REV32:             return "AArch64ISD::REV32";
-  case AArch64ISD::REV64:             return "AArch64ISD::REV64";
-  case AArch64ISD::EXT:               return "AArch64ISD::EXT";
-  case AArch64ISD::VSHL:              return "AArch64ISD::VSHL";
-  case AArch64ISD::VLSHR:             return "AArch64ISD::VLSHR";
-  case AArch64ISD::VASHR:             return "AArch64ISD::VASHR";
-  case AArch64ISD::VSLI:              return "AArch64ISD::VSLI";
-  case AArch64ISD::VSRI:              return "AArch64ISD::VSRI";
-  case AArch64ISD::CMEQ:              return "AArch64ISD::CMEQ";
-  case AArch64ISD::CMGE:              return "AArch64ISD::CMGE";
-  case AArch64ISD::CMGT:              return "AArch64ISD::CMGT";
-  case AArch64ISD::CMHI:              return "AArch64ISD::CMHI";
-  case AArch64ISD::CMHS:              return "AArch64ISD::CMHS";
-  case AArch64ISD::FCMEQ:             return "AArch64ISD::FCMEQ";
-  case AArch64ISD::FCMGE:             return "AArch64ISD::FCMGE";
-  case AArch64ISD::FCMGT:             return "AArch64ISD::FCMGT";
-  case AArch64ISD::CMEQz:             return "AArch64ISD::CMEQz";
-  case AArch64ISD::CMGEz:             return "AArch64ISD::CMGEz";
-  case AArch64ISD::CMGTz:             return "AArch64ISD::CMGTz";
-  case AArch64ISD::CMLEz:             return "AArch64ISD::CMLEz";
-  case AArch64ISD::CMLTz:             return "AArch64ISD::CMLTz";
-  case AArch64ISD::FCMEQz:            return "AArch64ISD::FCMEQz";
-  case AArch64ISD::FCMGEz:            return "AArch64ISD::FCMGEz";
-  case AArch64ISD::FCMGTz:            return "AArch64ISD::FCMGTz";
-  case AArch64ISD::FCMLEz:            return "AArch64ISD::FCMLEz";
-  case AArch64ISD::FCMLTz:            return "AArch64ISD::FCMLTz";
-  case AArch64ISD::SADDV:             return "AArch64ISD::SADDV";
-  case AArch64ISD::UADDV:             return "AArch64ISD::UADDV";
-  case AArch64ISD::SMINV:             return "AArch64ISD::SMINV";
-  case AArch64ISD::UMINV:             return "AArch64ISD::UMINV";
-  case AArch64ISD::SMAXV:             return "AArch64ISD::SMAXV";
-  case AArch64ISD::UMAXV:             return "AArch64ISD::UMAXV";
-  case AArch64ISD::SMAXV_PRED:        return "AArch64ISD::SMAXV_PRED";
-  case AArch64ISD::UMAXV_PRED:        return "AArch64ISD::UMAXV_PRED";
-  case AArch64ISD::SMINV_PRED:        return "AArch64ISD::SMINV_PRED";
-  case AArch64ISD::UMINV_PRED:        return "AArch64ISD::UMINV_PRED";
-  case AArch64ISD::ORV_PRED:          return "AArch64ISD::ORV_PRED";
-  case AArch64ISD::EORV_PRED:         return "AArch64ISD::EORV_PRED";
-  case AArch64ISD::ANDV_PRED:         return "AArch64ISD::ANDV_PRED";
-  case AArch64ISD::CLASTA_N:          return "AArch64ISD::CLASTA_N";
-  case AArch64ISD::CLASTB_N:          return "AArch64ISD::CLASTB_N";
-  case AArch64ISD::LASTA:             return "AArch64ISD::LASTA";
-  case AArch64ISD::LASTB:             return "AArch64ISD::LASTB";
-  case AArch64ISD::REV:               return "AArch64ISD::REV";
-  case AArch64ISD::REINTERPRET_CAST:  return "AArch64ISD::REINTERPRET_CAST";
-  case AArch64ISD::TBL:               return "AArch64ISD::TBL";
-  case AArch64ISD::FADDA_PRED:        return "AArch64ISD::FADDA_PRED";
-  case AArch64ISD::FADDV_PRED:        return "AArch64ISD::FADDV_PRED";
-  case AArch64ISD::FMAXV_PRED:        return "AArch64ISD::FMAXV_PRED";
-  case AArch64ISD::FMAXNMV_PRED:      return "AArch64ISD::FMAXNMV_PRED";
-  case AArch64ISD::FMINV_PRED:        return "AArch64ISD::FMINV_PRED";
-  case AArch64ISD::FMINNMV_PRED:      return "AArch64ISD::FMINNMV_PRED";
-  case AArch64ISD::NOT:               return "AArch64ISD::NOT";
-  case AArch64ISD::BIT:               return "AArch64ISD::BIT";
-  case AArch64ISD::CBZ:               return "AArch64ISD::CBZ";
-  case AArch64ISD::CBNZ:              return "AArch64ISD::CBNZ";
-  case AArch64ISD::TBZ:               return "AArch64ISD::TBZ";
-  case AArch64ISD::TBNZ:              return "AArch64ISD::TBNZ";
-  case AArch64ISD::TC_RETURN:         return "AArch64ISD::TC_RETURN";
-  case AArch64ISD::PREFETCH:          return "AArch64ISD::PREFETCH";
-  case AArch64ISD::SITOF:             return "AArch64ISD::SITOF";
-  case AArch64ISD::UITOF:             return "AArch64ISD::UITOF";
-  case AArch64ISD::NVCAST:            return "AArch64ISD::NVCAST";
-  case AArch64ISD::SQSHL_I:           return "AArch64ISD::SQSHL_I";
-  case AArch64ISD::UQSHL_I:           return "AArch64ISD::UQSHL_I";
-  case AArch64ISD::SRSHR_I:           return "AArch64ISD::SRSHR_I";
-  case AArch64ISD::URSHR_I:           return "AArch64ISD::URSHR_I";
-  case AArch64ISD::SQSHLU_I:          return "AArch64ISD::SQSHLU_I";
-  case AArch64ISD::WrapperLarge:      return "AArch64ISD::WrapperLarge";
-  case AArch64ISD::LD2post:           return "AArch64ISD::LD2post";
-  case AArch64ISD::LD3post:           return "AArch64ISD::LD3post";
-  case AArch64ISD::LD4post:           return "AArch64ISD::LD4post";
-  case AArch64ISD::ST2post:           return "AArch64ISD::ST2post";
-  case AArch64ISD::ST3post:           return "AArch64ISD::ST3post";
-  case AArch64ISD::ST4post:           return "AArch64ISD::ST4post";
-  case AArch64ISD::LD1x2post:         return "AArch64ISD::LD1x2post";
-  case AArch64ISD::LD1x3post:         return "AArch64ISD::LD1x3post";
-  case AArch64ISD::LD1x4post:         return "AArch64ISD::LD1x4post";
-  case AArch64ISD::ST1x2post:         return "AArch64ISD::ST1x2post";
-  case AArch64ISD::ST1x3post:         return "AArch64ISD::ST1x3post";
-  case AArch64ISD::ST1x4post:         return "AArch64ISD::ST1x4post";
-  case AArch64ISD::LD1DUPpost:        return "AArch64ISD::LD1DUPpost";
-  case AArch64ISD::LD2DUPpost:        return "AArch64ISD::LD2DUPpost";
-  case AArch64ISD::LD3DUPpost:        return "AArch64ISD::LD3DUPpost";
-  case AArch64ISD::LD4DUPpost:        return "AArch64ISD::LD4DUPpost";
-  case AArch64ISD::LD1LANEpost:       return "AArch64ISD::LD1LANEpost";
-  case AArch64ISD::LD2LANEpost:       return "AArch64ISD::LD2LANEpost";
-  case AArch64ISD::LD3LANEpost:       return "AArch64ISD::LD3LANEpost";
-  case AArch64ISD::LD4LANEpost:       return "AArch64ISD::LD4LANEpost";
-  case AArch64ISD::ST2LANEpost:       return "AArch64ISD::ST2LANEpost";
-  case AArch64ISD::ST3LANEpost:       return "AArch64ISD::ST3LANEpost";
-  case AArch64ISD::ST4LANEpost:       return "AArch64ISD::ST4LANEpost";
-  case AArch64ISD::SMULL:             return "AArch64ISD::SMULL";
-  case AArch64ISD::UMULL:             return "AArch64ISD::UMULL";
-  case AArch64ISD::FRECPE:            return "AArch64ISD::FRECPE";
-  case AArch64ISD::FRECPS:            return "AArch64ISD::FRECPS";
-  case AArch64ISD::FRSQRTE:           return "AArch64ISD::FRSQRTE";
-  case AArch64ISD::FRSQRTS:           return "AArch64ISD::FRSQRTS";
-  case AArch64ISD::STG:               return "AArch64ISD::STG";
-  case AArch64ISD::STZG:              return "AArch64ISD::STZG";
-  case AArch64ISD::ST2G:              return "AArch64ISD::ST2G";
-  case AArch64ISD::STZ2G:             return "AArch64ISD::STZ2G";
-  case AArch64ISD::SUNPKHI:           return "AArch64ISD::SUNPKHI";
-  case AArch64ISD::SUNPKLO:           return "AArch64ISD::SUNPKLO";
-  case AArch64ISD::UUNPKHI:           return "AArch64ISD::UUNPKHI";
-  case AArch64ISD::UUNPKLO:           return "AArch64ISD::UUNPKLO";
-  case AArch64ISD::INSR:              return "AArch64ISD::INSR";
-  case AArch64ISD::PTEST:             return "AArch64ISD::PTEST";
-  case AArch64ISD::PTRUE:             return "AArch64ISD::PTRUE";
-  case AArch64ISD::LD1:               return "AArch64ISD::LD1";
-  case AArch64ISD::LD1S:              return "AArch64ISD::LD1S";
-  case AArch64ISD::LDNF1:             return "AArch64ISD::LDNF1";
-  case AArch64ISD::LDNF1S:            return "AArch64ISD::LDNF1S";
-  case AArch64ISD::LDFF1:             return "AArch64ISD::LDFF1";
-  case AArch64ISD::LDFF1S:            return "AArch64ISD::LDFF1S";
-  case AArch64ISD::LD1RQ:             return "AArch64ISD::LD1RQ";
-  case AArch64ISD::LD1RO:             return "AArch64ISD::LD1RO";
-  case AArch64ISD::SVE_LD2:           return "AArch64ISD::SVE_LD2";
-  case AArch64ISD::SVE_LD3:           return "AArch64ISD::SVE_LD3";
-  case AArch64ISD::SVE_LD4:           return "AArch64ISD::SVE_LD4";
-  case AArch64ISD::GLD1:              return "AArch64ISD::GLD1";
-  case AArch64ISD::GLD1_SCALED:       return "AArch64ISD::GLD1_SCALED";
-  case AArch64ISD::GLD1_SXTW:         return "AArch64ISD::GLD1_SXTW";
-  case AArch64ISD::GLD1_UXTW:         return "AArch64ISD::GLD1_UXTW";
-  case AArch64ISD::GLD1_SXTW_SCALED:  return "AArch64ISD::GLD1_SXTW_SCALED";
-  case AArch64ISD::GLD1_UXTW_SCALED:  return "AArch64ISD::GLD1_UXTW_SCALED";
-  case AArch64ISD::GLD1_IMM:          return "AArch64ISD::GLD1_IMM";
-  case AArch64ISD::GLD1S:             return "AArch64ISD::GLD1S";
-  case AArch64ISD::GLD1S_SCALED:      return "AArch64ISD::GLD1S_SCALED";
-  case AArch64ISD::GLD1S_SXTW:        return "AArch64ISD::GLD1S_SXTW";
-  case AArch64ISD::GLD1S_UXTW:        return "AArch64ISD::GLD1S_UXTW";
-  case AArch64ISD::GLD1S_SXTW_SCALED: return "AArch64ISD::GLD1S_SXTW_SCALED";
-  case AArch64ISD::GLD1S_UXTW_SCALED: return "AArch64ISD::GLD1S_UXTW_SCALED";
-  case AArch64ISD::GLD1S_IMM:         return "AArch64ISD::GLD1S_IMM";
-  case AArch64ISD::GLDFF1:            return "AArch64ISD::GLDFF1";
-  case AArch64ISD::GLDFF1_SCALED:     return "AArch64ISD::GLDFF1_SCALED";
-  case AArch64ISD::GLDFF1_SXTW:       return "AArch64ISD::GLDFF1_SXTW";
-  case AArch64ISD::GLDFF1_UXTW:       return "AArch64ISD::GLDFF1_UXTW";
-  case AArch64ISD::GLDFF1_SXTW_SCALED:return "AArch64ISD::GLDFF1_SXTW_SCALED";
-  case AArch64ISD::GLDFF1_UXTW_SCALED:return "AArch64ISD::GLDFF1_UXTW_SCALED";
-  case AArch64ISD::GLDFF1_IMM:        return "AArch64ISD::GLDFF1_IMM";
-  case AArch64ISD::GLDFF1S:           return "AArch64ISD::GLDFF1S";
-  case AArch64ISD::GLDFF1S_SCALED:    return "AArch64ISD::GLDFF1S_SCALED";
-  case AArch64ISD::GLDFF1S_SXTW:      return "AArch64ISD::GLDFF1S_SXTW";
-  case AArch64ISD::GLDFF1S_UXTW:      return "AArch64ISD::GLDFF1S_UXTW";
-  case AArch64ISD::GLDFF1S_SXTW_SCALED:
-    return "AArch64ISD::GLDFF1S_SXTW_SCALED";
-  case AArch64ISD::GLDFF1S_UXTW_SCALED:
-    return "AArch64ISD::GLDFF1S_UXTW_SCALED";
-  case AArch64ISD::GLDFF1S_IMM:       return "AArch64ISD::GLDFF1S_IMM";
-
-  case AArch64ISD::GLDNT1:            return "AArch64ISD::GLDNT1";
-  case AArch64ISD::GLDNT1_INDEX:      return "AArch64ISD::GLDNT1_INDEX";
-  case AArch64ISD::GLDNT1S:           return "AArch64ISD::GLDNT1S";
-
-  case AArch64ISD::ST1:               return "AArch64ISD::ST1";
-
-  case AArch64ISD::SST1:              return "AArch64ISD::SST1";
-  case AArch64ISD::SST1_SCALED:       return "AArch64ISD::SST1_SCALED";
-  case AArch64ISD::SST1_SXTW:         return "AArch64ISD::SST1_SXTW";
-  case AArch64ISD::SST1_UXTW:         return "AArch64ISD::SST1_UXTW";
-  case AArch64ISD::SST1_SXTW_SCALED:  return "AArch64ISD::SST1_SXTW_SCALED";
-  case AArch64ISD::SST1_UXTW_SCALED:  return "AArch64ISD::SST1_UXTW_SCALED";
-  case AArch64ISD::SST1_IMM:          return "AArch64ISD::SST1_IMM";
-
-  case AArch64ISD::SSTNT1:            return "AArch64ISD::SSTNT1";
-  case AArch64ISD::SSTNT1_INDEX:      return "AArch64ISD::SSTNT1_INDEX";
-
-  case AArch64ISD::LDP:               return "AArch64ISD::LDP";
-  case AArch64ISD::STP:               return "AArch64ISD::STP";
-  case AArch64ISD::STNP:              return "AArch64ISD::STNP";
-  case AArch64ISD::DUP_PRED:          return "AArch64ISD::DUP_PRED";
-  case AArch64ISD::INDEX_VECTOR:      return "AArch64ISD::INDEX_VECTOR";
-  }
+  case AArch64ISD::FIRST_NUMBER:
+    break;
+    MAKE_CASE(AArch64ISD::CALL)
+    MAKE_CASE(AArch64ISD::ADRP)
+    MAKE_CASE(AArch64ISD::ADR)
+    MAKE_CASE(AArch64ISD::ADDlow)
+    MAKE_CASE(AArch64ISD::LOADgot)
+    MAKE_CASE(AArch64ISD::RET_FLAG)
+    MAKE_CASE(AArch64ISD::BRCOND)
+    MAKE_CASE(AArch64ISD::CSEL)
+    MAKE_CASE(AArch64ISD::FCSEL)
+    MAKE_CASE(AArch64ISD::CSINV)
+    MAKE_CASE(AArch64ISD::CSNEG)
+    MAKE_CASE(AArch64ISD::CSINC)
+    MAKE_CASE(AArch64ISD::THREAD_POINTER)
+    MAKE_CASE(AArch64ISD::TLSDESC_CALLSEQ)
+    MAKE_CASE(AArch64ISD::ADD_PRED)
+    MAKE_CASE(AArch64ISD::SDIV_PRED)
+    MAKE_CASE(AArch64ISD::UDIV_PRED)
+    MAKE_CASE(AArch64ISD::SMIN_MERGE_OP1)
+    MAKE_CASE(AArch64ISD::UMIN_MERGE_OP1)
+    MAKE_CASE(AArch64ISD::SMAX_MERGE_OP1)
+    MAKE_CASE(AArch64ISD::UMAX_MERGE_OP1)
+    MAKE_CASE(AArch64ISD::SHL_MERGE_OP1)
+    MAKE_CASE(AArch64ISD::SRL_MERGE_OP1)
+    MAKE_CASE(AArch64ISD::SRA_MERGE_OP1)
+    MAKE_CASE(AArch64ISD::SETCC_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::ADC)
+    MAKE_CASE(AArch64ISD::SBC)
+    MAKE_CASE(AArch64ISD::ADDS)
+    MAKE_CASE(AArch64ISD::SUBS)
+    MAKE_CASE(AArch64ISD::ADCS)
+    MAKE_CASE(AArch64ISD::SBCS)
+    MAKE_CASE(AArch64ISD::ANDS)
+    MAKE_CASE(AArch64ISD::CCMP)
+    MAKE_CASE(AArch64ISD::CCMN)
+    MAKE_CASE(AArch64ISD::FCCMP)
+    MAKE_CASE(AArch64ISD::FCMP)
+    MAKE_CASE(AArch64ISD::STRICT_FCMP)
+    MAKE_CASE(AArch64ISD::STRICT_FCMPE)
+    MAKE_CASE(AArch64ISD::DUP)
+    MAKE_CASE(AArch64ISD::DUPLANE8)
+    MAKE_CASE(AArch64ISD::DUPLANE16)
+    MAKE_CASE(AArch64ISD::DUPLANE32)
+    MAKE_CASE(AArch64ISD::DUPLANE64)
+    MAKE_CASE(AArch64ISD::MOVI)
+    MAKE_CASE(AArch64ISD::MOVIshift)
+    MAKE_CASE(AArch64ISD::MOVIedit)
+    MAKE_CASE(AArch64ISD::MOVImsl)
+    MAKE_CASE(AArch64ISD::FMOV)
+    MAKE_CASE(AArch64ISD::MVNIshift)
+    MAKE_CASE(AArch64ISD::MVNImsl)
+    MAKE_CASE(AArch64ISD::BICi)
+    MAKE_CASE(AArch64ISD::ORRi)
+    MAKE_CASE(AArch64ISD::BSP)
+    MAKE_CASE(AArch64ISD::NEG)
+    MAKE_CASE(AArch64ISD::EXTR)
+    MAKE_CASE(AArch64ISD::ZIP1)
+    MAKE_CASE(AArch64ISD::ZIP2)
+    MAKE_CASE(AArch64ISD::UZP1)
+    MAKE_CASE(AArch64ISD::UZP2)
+    MAKE_CASE(AArch64ISD::TRN1)
+    MAKE_CASE(AArch64ISD::TRN2)
+    MAKE_CASE(AArch64ISD::REV16)
+    MAKE_CASE(AArch64ISD::REV32)
+    MAKE_CASE(AArch64ISD::REV64)
+    MAKE_CASE(AArch64ISD::EXT)
+    MAKE_CASE(AArch64ISD::VSHL)
+    MAKE_CASE(AArch64ISD::VLSHR)
+    MAKE_CASE(AArch64ISD::VASHR)
+    MAKE_CASE(AArch64ISD::VSLI)
+    MAKE_CASE(AArch64ISD::VSRI)
+    MAKE_CASE(AArch64ISD::CMEQ)
+    MAKE_CASE(AArch64ISD::CMGE)
+    MAKE_CASE(AArch64ISD::CMGT)
+    MAKE_CASE(AArch64ISD::CMHI)
+    MAKE_CASE(AArch64ISD::CMHS)
+    MAKE_CASE(AArch64ISD::FCMEQ)
+    MAKE_CASE(AArch64ISD::FCMGE)
+    MAKE_CASE(AArch64ISD::FCMGT)
+    MAKE_CASE(AArch64ISD::CMEQz)
+    MAKE_CASE(AArch64ISD::CMGEz)
+    MAKE_CASE(AArch64ISD::CMGTz)
+    MAKE_CASE(AArch64ISD::CMLEz)
+    MAKE_CASE(AArch64ISD::CMLTz)
+    MAKE_CASE(AArch64ISD::FCMEQz)
+    MAKE_CASE(AArch64ISD::FCMGEz)
+    MAKE_CASE(AArch64ISD::FCMGTz)
+    MAKE_CASE(AArch64ISD::FCMLEz)
+    MAKE_CASE(AArch64ISD::FCMLTz)
+    MAKE_CASE(AArch64ISD::SADDV)
+    MAKE_CASE(AArch64ISD::UADDV)
+    MAKE_CASE(AArch64ISD::SRHADD)
+    MAKE_CASE(AArch64ISD::URHADD)
+    MAKE_CASE(AArch64ISD::SMINV)
+    MAKE_CASE(AArch64ISD::UMINV)
+    MAKE_CASE(AArch64ISD::SMAXV)
+    MAKE_CASE(AArch64ISD::UMAXV)
+    MAKE_CASE(AArch64ISD::SMAXV_PRED)
+    MAKE_CASE(AArch64ISD::UMAXV_PRED)
+    MAKE_CASE(AArch64ISD::SMINV_PRED)
+    MAKE_CASE(AArch64ISD::UMINV_PRED)
+    MAKE_CASE(AArch64ISD::ORV_PRED)
+    MAKE_CASE(AArch64ISD::EORV_PRED)
+    MAKE_CASE(AArch64ISD::ANDV_PRED)
+    MAKE_CASE(AArch64ISD::CLASTA_N)
+    MAKE_CASE(AArch64ISD::CLASTB_N)
+    MAKE_CASE(AArch64ISD::LASTA)
+    MAKE_CASE(AArch64ISD::LASTB)
+    MAKE_CASE(AArch64ISD::REV)
+    MAKE_CASE(AArch64ISD::REINTERPRET_CAST)
+    MAKE_CASE(AArch64ISD::TBL)
+    MAKE_CASE(AArch64ISD::FADD_PRED)
+    MAKE_CASE(AArch64ISD::FADDA_PRED)
+    MAKE_CASE(AArch64ISD::FADDV_PRED)
+    MAKE_CASE(AArch64ISD::FMA_PRED)
+    MAKE_CASE(AArch64ISD::FMAXV_PRED)
+    MAKE_CASE(AArch64ISD::FMAXNMV_PRED)
+    MAKE_CASE(AArch64ISD::FMINV_PRED)
+    MAKE_CASE(AArch64ISD::FMINNMV_PRED)
+    MAKE_CASE(AArch64ISD::NOT)
+    MAKE_CASE(AArch64ISD::BIT)
+    MAKE_CASE(AArch64ISD::CBZ)
+    MAKE_CASE(AArch64ISD::CBNZ)
+    MAKE_CASE(AArch64ISD::TBZ)
+    MAKE_CASE(AArch64ISD::TBNZ)
+    MAKE_CASE(AArch64ISD::TC_RETURN)
+    MAKE_CASE(AArch64ISD::PREFETCH)
+    MAKE_CASE(AArch64ISD::SITOF)
+    MAKE_CASE(AArch64ISD::UITOF)
+    MAKE_CASE(AArch64ISD::NVCAST)
+    MAKE_CASE(AArch64ISD::SQSHL_I)
+    MAKE_CASE(AArch64ISD::UQSHL_I)
+    MAKE_CASE(AArch64ISD::SRSHR_I)
+    MAKE_CASE(AArch64ISD::URSHR_I)
+    MAKE_CASE(AArch64ISD::SQSHLU_I)
+    MAKE_CASE(AArch64ISD::WrapperLarge)
+    MAKE_CASE(AArch64ISD::LD2post)
+    MAKE_CASE(AArch64ISD::LD3post)
+    MAKE_CASE(AArch64ISD::LD4post)
+    MAKE_CASE(AArch64ISD::ST2post)
+    MAKE_CASE(AArch64ISD::ST3post)
+    MAKE_CASE(AArch64ISD::ST4post)
+    MAKE_CASE(AArch64ISD::LD1x2post)
+    MAKE_CASE(AArch64ISD::LD1x3post)
+    MAKE_CASE(AArch64ISD::LD1x4post)
+    MAKE_CASE(AArch64ISD::ST1x2post)
+    MAKE_CASE(AArch64ISD::ST1x3post)
+    MAKE_CASE(AArch64ISD::ST1x4post)
+    MAKE_CASE(AArch64ISD::LD1DUPpost)
+    MAKE_CASE(AArch64ISD::LD2DUPpost)
+    MAKE_CASE(AArch64ISD::LD3DUPpost)
+    MAKE_CASE(AArch64ISD::LD4DUPpost)
+    MAKE_CASE(AArch64ISD::LD1LANEpost)
+    MAKE_CASE(AArch64ISD::LD2LANEpost)
+    MAKE_CASE(AArch64ISD::LD3LANEpost)
+    MAKE_CASE(AArch64ISD::LD4LANEpost)
+    MAKE_CASE(AArch64ISD::ST2LANEpost)
+    MAKE_CASE(AArch64ISD::ST3LANEpost)
+    MAKE_CASE(AArch64ISD::ST4LANEpost)
+    MAKE_CASE(AArch64ISD::SMULL)
+    MAKE_CASE(AArch64ISD::UMULL)
+    MAKE_CASE(AArch64ISD::FRECPE)
+    MAKE_CASE(AArch64ISD::FRECPS)
+    MAKE_CASE(AArch64ISD::FRSQRTE)
+    MAKE_CASE(AArch64ISD::FRSQRTS)
+    MAKE_CASE(AArch64ISD::STG)
+    MAKE_CASE(AArch64ISD::STZG)
+    MAKE_CASE(AArch64ISD::ST2G)
+    MAKE_CASE(AArch64ISD::STZ2G)
+    MAKE_CASE(AArch64ISD::SUNPKHI)
+    MAKE_CASE(AArch64ISD::SUNPKLO)
+    MAKE_CASE(AArch64ISD::UUNPKHI)
+    MAKE_CASE(AArch64ISD::UUNPKLO)
+    MAKE_CASE(AArch64ISD::INSR)
+    MAKE_CASE(AArch64ISD::PTEST)
+    MAKE_CASE(AArch64ISD::PTRUE)
+    MAKE_CASE(AArch64ISD::LD1_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::LD1S_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::LDNF1_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::LDNF1S_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::LDFF1_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::LDFF1S_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::LD1RQ_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::LD1RO_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::SVE_LD2_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::SVE_LD3_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::SVE_LD4_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLD1_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLD1_SCALED_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLD1_SXTW_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLD1_UXTW_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLD1_IMM_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLD1S_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLD1S_SCALED_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLD1S_SXTW_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLD1S_UXTW_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLD1S_IMM_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDFF1_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDFF1_SCALED_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDFF1_SXTW_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDFF1_UXTW_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDFF1_IMM_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDFF1S_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDFF1S_SCALED_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDFF1S_SXTW_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDFF1S_UXTW_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDFF1S_SXTW_SCALED_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDFF1S_UXTW_SCALED_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDFF1S_IMM_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDNT1_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDNT1_INDEX_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::GLDNT1S_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::ST1_PRED)
+    MAKE_CASE(AArch64ISD::SST1_PRED)
+    MAKE_CASE(AArch64ISD::SST1_SCALED_PRED)
+    MAKE_CASE(AArch64ISD::SST1_SXTW_PRED)
+    MAKE_CASE(AArch64ISD::SST1_UXTW_PRED)
+    MAKE_CASE(AArch64ISD::SST1_SXTW_SCALED_PRED)
+    MAKE_CASE(AArch64ISD::SST1_UXTW_SCALED_PRED)
+    MAKE_CASE(AArch64ISD::SST1_IMM_PRED)
+    MAKE_CASE(AArch64ISD::SSTNT1_PRED)
+    MAKE_CASE(AArch64ISD::SSTNT1_INDEX_PRED)
+    MAKE_CASE(AArch64ISD::LDP)
+    MAKE_CASE(AArch64ISD::STP)
+    MAKE_CASE(AArch64ISD::STNP)
+    MAKE_CASE(AArch64ISD::DUP_MERGE_PASSTHRU)
+    MAKE_CASE(AArch64ISD::INDEX_VECTOR)
+  }
+#undef MAKE_CASE
   return nullptr;
 }
 
@@ -3222,6 +3288,14 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     return DAG.getNode(Opcode, dl, Ty, Op.getOperand(1), Op.getOperand(2),
                        Op.getOperand(3));
   }
+
+  case Intrinsic::aarch64_neon_srhadd:
+  case Intrinsic::aarch64_neon_urhadd: {
+    bool IsSignedAdd = IntNo == Intrinsic::aarch64_neon_srhadd;
+    unsigned Opcode = IsSignedAdd ? AArch64ISD::SRHADD : AArch64ISD::URHADD;
+    return DAG.getNode(Opcode, dl, Op.getValueType(), Op.getOperand(1),
+                       Op.getOperand(2));
+  }
   }
 }
 
@@ -3276,10 +3350,13 @@ SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
   EVT MemVT = StoreNode->getMemoryVT();
 
   if (VT.isVector()) {
+    if (useSVEForFixedLengthVectorVT(VT))
+      return LowerFixedLengthVectorStoreToSVE(Op, DAG);
+
     unsigned AS = StoreNode->getAddressSpace();
-    unsigned Align = StoreNode->getAlignment();
-    if (Align < MemVT.getStoreSize() &&
-        !allowsMisalignedMemoryAccesses(MemVT, AS, Align,
+    Align Alignment = StoreNode->getAlign();
+    if (Alignment < MemVT.getStoreSize() &&
+        !allowsMisalignedMemoryAccesses(MemVT, AS, Alignment.value(),
                                         StoreNode->getMemOperand()->getFlags(),
                                         nullptr)) {
       return scalarizeVectorStore(StoreNode, DAG);
@@ -3382,11 +3459,15 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
   case ISD::UMULO:
     return LowerXALUO(Op, DAG);
   case ISD::FADD:
+    if (useSVEForFixedLengthVectorVT(Op.getValueType()))
+      return LowerToPredicatedOp(Op, DAG, AArch64ISD::FADD_PRED);
     return LowerF128Call(Op, DAG, RTLIB::ADD_F128);
   case ISD::FSUB:
     return LowerF128Call(Op, DAG, RTLIB::SUB_F128);
   case ISD::FMUL:
     return LowerF128Call(Op, DAG, RTLIB::MUL_F128);
+  case ISD::FMA:
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMA_PRED);
   case ISD::FDIV:
     return LowerF128Call(Op, DAG, RTLIB::DIV_F128);
   case ISD::FP_ROUND:
@@ -3414,18 +3495,20 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
     return LowerSPLAT_VECTOR(Op, DAG);
   case ISD::EXTRACT_SUBVECTOR:
     return LowerEXTRACT_SUBVECTOR(Op, DAG);
+  case ISD::INSERT_SUBVECTOR:
+    return LowerINSERT_SUBVECTOR(Op, DAG);
   case ISD::SDIV:
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::SDIV_PRED);
   case ISD::UDIV:
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::UDIV_PRED);
   case ISD::SMIN:
-    return LowerToPredicatedOp(Op, DAG, AArch64ISD::SMIN_PRED);
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::SMIN_MERGE_OP1);
   case ISD::UMIN:
-    return LowerToPredicatedOp(Op, DAG, AArch64ISD::UMIN_PRED);
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::UMIN_MERGE_OP1);
   case ISD::SMAX:
-    return LowerToPredicatedOp(Op, DAG, AArch64ISD::SMAX_PRED);
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::SMAX_MERGE_OP1);
   case ISD::UMAX:
-    return LowerToPredicatedOp(Op, DAG, AArch64ISD::UMAX_PRED);
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::UMAX_MERGE_OP1);
   case ISD::SRA:
   case ISD::SRL:
   case ISD::SHL:
@@ -3481,6 +3564,16 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
     return LowerDYNAMIC_STACKALLOC(Op, DAG);
   case ISD::VSCALE:
     return LowerVSCALE(Op, DAG);
+  case ISD::TRUNCATE:
+    return LowerTRUNCATE(Op, DAG);
+  case ISD::LOAD:
+    if (useSVEForFixedLengthVectorVT(Op.getValueType()))
+      return LowerFixedLengthVectorLoadToSVE(Op, DAG);
+    llvm_unreachable("Unexpected request to lower ISD::LOAD");
+  case ISD::ADD:
+    if (useSVEForFixedLengthVectorVT(Op.getValueType()))
+      return LowerToPredicatedOp(Op, DAG, AArch64ISD::ADD_PRED);
+    llvm_unreachable("Unexpected request to lower ISD::ADD");
   }
 }
 
@@ -3489,18 +3582,20 @@ bool AArch64TargetLowering::useSVEForFixedLengthVectors() const {
   return Subtarget->hasSVE() && Subtarget->getMinSVEVectorSizeInBits() >= 256;
 }
 
-bool AArch64TargetLowering::useSVEForFixedLengthVectorVT(MVT VT) const {
-  assert(VT.isFixedLengthVector());
+bool AArch64TargetLowering::useSVEForFixedLengthVectorVT(EVT VT) const {
   if (!useSVEForFixedLengthVectors())
     return false;
 
+  if (!VT.isFixedLengthVector())
+    return false;
+
   // Fixed length predicates should be promoted to i8.
   // NOTE: This is consistent with how NEON (and thus 64/128bit vectors) work.
   if (VT.getVectorElementType() == MVT::i1)
     return false;
 
   // Don't use SVE for vectors we cannot scalarize if required.
-  switch (VT.getVectorElementType().SimpleTy) {
+  switch (VT.getVectorElementType().getSimpleVT().SimpleTy) {
   default:
     return false;
   case MVT::i8:
@@ -3868,7 +3963,7 @@ void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo,
         // The extra size here, if triggered, will always be 8.
         MFI.CreateFixedObject(16 - (GPRSaveSize & 15), -(int)alignTo(GPRSaveSize, 16), false);
     } else
-      GPRIdx = MFI.CreateStackObject(GPRSaveSize, 8, false);
+      GPRIdx = MFI.CreateStackObject(GPRSaveSize, Align(8), false);
 
     SDValue FIN = DAG.getFrameIndex(GPRIdx, PtrVT);
 
@@ -3900,7 +3995,7 @@ void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo,
     unsigned FPRSaveSize = 16 * (NumFPRArgRegs - FirstVariadicFPR);
     int FPRIdx = 0;
     if (FPRSaveSize != 0) {
-      FPRIdx = MFI.CreateStackObject(FPRSaveSize, 16, false);
+      FPRIdx = MFI.CreateStackObject(FPRSaveSize, Align(16), false);
 
       SDValue FIN = DAG.getFrameIndex(FPRIdx, PtrVT);
 
@@ -4374,9 +4469,9 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
              "Only scalable vectors can be passed indirectly");
       MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
       Type *Ty = EVT(VA.getValVT()).getTypeForEVT(*DAG.getContext());
-      unsigned Align = DAG.getDataLayout().getPrefTypeAlignment(Ty);
+      Align Alignment = DAG.getDataLayout().getPrefTypeAlign(Ty);
       int FI = MFI.CreateStackObject(
-          VA.getValVT().getStoreSize().getKnownMinSize(), Align, false);
+          VA.getValVT().getStoreSize().getKnownMinSize(), Alignment, false);
       MFI.setStackID(FI, TargetStackID::SVEVector);
 
       SDValue SpillSlot = DAG.getFrameIndex(
@@ -6042,7 +6137,7 @@ SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
   SDLoc DL(Op);
   SDValue Chain = Op.getOperand(0);
   SDValue Addr = Op.getOperand(1);
-  unsigned Align = Op.getConstantOperandVal(3);
+  MaybeAlign Align(Op.getConstantOperandVal(3));
   unsigned MinSlotSize = Subtarget->isTargetILP32() ? 4 : 8;
   auto PtrVT = getPointerTy(DAG.getDataLayout());
   auto PtrMemVT = getPointerMemTy(DAG.getDataLayout());
@@ -6051,12 +6146,11 @@ SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
   Chain = VAList.getValue(1);
   VAList = DAG.getZExtOrTrunc(VAList, DL, PtrVT);
 
-  if (Align > MinSlotSize) {
-    assert(((Align & (Align - 1)) == 0) && "Expected Align to be a power of 2");
+  if (Align && *Align > MinSlotSize) {
     VAList = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
-                         DAG.getConstant(Align - 1, DL, PtrVT));
+                         DAG.getConstant(Align->value() - 1, DL, PtrVT));
     VAList = DAG.getNode(ISD::AND, DL, PtrVT, VAList,
-                         DAG.getConstant(-(int64_t)Align, DL, PtrVT));
+                         DAG.getConstant(-(int64_t)Align->value(), DL, PtrVT));
   }
 
   Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
@@ -7804,24 +7898,6 @@ SDValue AArch64TargetLowering::LowerDUPQLane(SDValue Op,
   return DAG.getNode(ISD::BITCAST, DL, VT, TBL);
 }
 
-SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
-                                        SelectionDAG &DAG,
-                                        unsigned NewOp) const {
-  EVT VT = Op.getValueType();
-  SDLoc DL(Op);
-
-  assert(Op.getOperand(0).getValueType().isScalableVector() &&
-         Op.getOperand(1).getValueType().isScalableVector() &&
-         "Only scalable vectors are supported");
-
-  auto PredTy =
-      VT.getVectorVT(*DAG.getContext(), MVT::i1, VT.getVectorElementCount());
-  SDValue Mask = getPTrue(DAG, DL, PredTy, AArch64SVEPredPattern::all);
-
-  SmallVector<SDValue, 4> Operands = {Mask};
-  Operands.append(Op->op_begin(), Op->op_end());
-  return DAG.getNode(NewOp, DL, VT, Operands);
-}
 
 static bool resolveBuildVector(BuildVectorSDNode *BVN, APInt &CnstBits,
                                APInt &UndefBits) {
@@ -8631,35 +8707,57 @@ AArch64TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
 
 SDValue AArch64TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
                                                       SelectionDAG &DAG) const {
-  assert(!Op.getValueType().isScalableVector() &&
-         "Unexpected scalable type for custom lowering EXTRACT_SUBVECTOR");
+  assert(Op.getValueType().isFixedLengthVector() &&
+         "Only cases that extract a fixed length vector are supported!");
 
-  EVT VT = Op.getOperand(0).getValueType();
-  SDLoc dl(Op);
-  // Just in case...
-  if (!VT.isVector())
-    return SDValue();
+  EVT InVT = Op.getOperand(0).getValueType();
+  unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
+  unsigned Size = Op.getValueSizeInBits();
 
-  ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(1));
-  if (!Cst)
-    return SDValue();
-  unsigned Val = Cst->getZExtValue();
+  if (InVT.isScalableVector()) {
+    // This will be matched by custom code during ISelDAGToDAG.
+    if (Idx == 0 && isPackedVectorType(InVT, DAG))
+      return Op;
 
-  unsigned Size = Op.getValueSizeInBits();
+    return SDValue();
+  }
 
   // This will get lowered to an appropriate EXTRACT_SUBREG in ISel.
-  if (Val == 0)
+  if (Idx == 0 && InVT.getSizeInBits() <= 128)
     return Op;
 
   // If this is extracting the upper 64-bits of a 128-bit vector, we match
   // that directly.
-  if (Size == 64 && Val * VT.getScalarSizeInBits() == 64)
+  if (Size == 64 && Idx * InVT.getScalarSizeInBits() == 64)
+    return Op;
+
+  return SDValue();
+}
+
+SDValue AArch64TargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
+                                                     SelectionDAG &DAG) const {
+  assert(Op.getValueType().isScalableVector() &&
+         "Only expect to lower inserts into scalable vectors!");
+
+  EVT InVT = Op.getOperand(1).getValueType();
+  unsigned Idx = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
+
+  // We don't have any patterns for scalable vector yet.
+  if (InVT.isScalableVector() || !useSVEForFixedLengthVectorVT(InVT))
+    return SDValue();
+
+  // This will be matched by custom code during ISelDAGToDAG.
+  if (Idx == 0 && isPackedVectorType(InVT, DAG) && Op.getOperand(0).isUndef())
     return Op;
 
   return SDValue();
 }
 
 bool AArch64TargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
+  // Currently no fixed length shuffles that require SVE are legal.
+  if (useSVEForFixedLengthVectorVT(VT))
+    return false;
+
   if (VT.getVectorNumElements() == 4 &&
       (VT.is128BitVector() || VT.is64BitVector())) {
     unsigned PFIndexes[4];
@@ -8739,6 +8837,81 @@ static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) {
   return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
 }
 
+// Attempt to form urhadd(OpA, OpB) from
+// truncate(vlshr(sub(zext(OpB), xor(zext(OpA), Ones(ElemSizeInBits))), 1)).
+// The original form of this expression is
+// truncate(srl(add(zext(OpB), add(zext(OpA), 1)), 1)) and before this function
+// is called the srl will have been lowered to AArch64ISD::VLSHR and the
+// ((OpA + OpB + 1) >> 1) expression will have been changed to (OpB - (~OpA)).
+// This pass can also recognize a variant of this pattern that uses sign
+// extension instead of zero extension and form a srhadd(OpA, OpB) from it.
+SDValue AArch64TargetLowering::LowerTRUNCATE(SDValue Op,
+                                             SelectionDAG &DAG) const {
+  EVT VT = Op.getValueType();
+
+  if (!VT.isVector() || VT.isScalableVector())
+    return Op;
+
+  if (useSVEForFixedLengthVectorVT(Op.getOperand(0).getValueType()))
+    return LowerFixedLengthVectorTruncateToSVE(Op, DAG);
+
+  // Since we are looking for a right shift by a constant value of 1 and we are
+  // operating on types at least 16 bits in length (sign/zero extended OpA and
+  // OpB, which are at least 8 bits), it follows that the truncate will always
+  // discard the shifted-in bit and therefore the right shift will be logical
+  // regardless of the signedness of OpA and OpB.
+  SDValue Shift = Op.getOperand(0);
+  if (Shift.getOpcode() != AArch64ISD::VLSHR)
+    return Op;
+
+  // Is the right shift using an immediate value of 1?
+  uint64_t ShiftAmount = Shift.getConstantOperandVal(1);
+  if (ShiftAmount != 1)
+    return Op;
+
+  SDValue Sub = Shift->getOperand(0);
+  if (Sub.getOpcode() != ISD::SUB)
+    return Op;
+
+  SDValue Xor = Sub.getOperand(1);
+  if (Xor.getOpcode() != ISD::XOR)
+    return Op;
+
+  SDValue ExtendOpA = Xor.getOperand(0);
+  SDValue ExtendOpB = Sub.getOperand(0);
+  unsigned ExtendOpAOpc = ExtendOpA.getOpcode();
+  unsigned ExtendOpBOpc = ExtendOpB.getOpcode();
+  if (!(ExtendOpAOpc == ExtendOpBOpc &&
+        (ExtendOpAOpc == ISD::ZERO_EXTEND || ExtendOpAOpc == ISD::SIGN_EXTEND)))
+    return Op;
+
+  // Is the result of the right shift being truncated to the same value type as
+  // the original operands, OpA and OpB?
+  SDValue OpA = ExtendOpA.getOperand(0);
+  SDValue OpB = ExtendOpB.getOperand(0);
+  EVT OpAVT = OpA.getValueType();
+  assert(ExtendOpA.getValueType() == ExtendOpB.getValueType());
+  if (!(VT == OpAVT && OpAVT == OpB.getValueType()))
+    return Op;
+
+  // Is the XOR using a constant amount of all ones in the right hand side?
+  uint64_t C;
+  if (!isAllConstantBuildVector(Xor.getOperand(1), C))
+    return Op;
+
+  unsigned ElemSizeInBits = VT.getScalarSizeInBits();
+  APInt CAsAPInt(ElemSizeInBits, C);
+  if (CAsAPInt != APInt::getAllOnesValue(ElemSizeInBits))
+    return Op;
+
+  SDLoc DL(Op);
+  bool IsSignExtend = ExtendOpAOpc == ISD::SIGN_EXTEND;
+  unsigned RHADDOpc = IsSignExtend ? AArch64ISD::SRHADD : AArch64ISD::URHADD;
+  SDValue ResultURHADD = DAG.getNode(RHADDOpc, DL, VT, OpA, OpB);
+
+  return ResultURHADD;
+}
+
 SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
                                                       SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
@@ -8755,7 +8928,7 @@ SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
 
   case ISD::SHL:
     if (VT.isScalableVector())
-      return LowerToPredicatedOp(Op, DAG, AArch64ISD::SHL_PRED);
+      return LowerToPredicatedOp(Op, DAG, AArch64ISD::SHL_MERGE_OP1);
 
     if (isVShiftLImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize)
       return DAG.getNode(AArch64ISD::VSHL, DL, VT, Op.getOperand(0),
@@ -8767,8 +8940,8 @@ SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
   case ISD::SRA:
   case ISD::SRL:
     if (VT.isScalableVector()) {
-      unsigned Opc = Op.getOpcode() == ISD::SRA ? AArch64ISD::SRA_PRED
-                                                : AArch64ISD::SRL_PRED;
+      unsigned Opc = Op.getOpcode() == ISD::SRA ? AArch64ISD::SRA_MERGE_OP1
+                                                : AArch64ISD::SRL_MERGE_OP1;
       return LowerToPredicatedOp(Op, DAG, Opc);
     }
 
@@ -8897,7 +9070,7 @@ SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
   if (Op.getValueType().isScalableVector()) {
     if (Op.getOperand(0).getValueType().isFloatingPoint())
       return Op;
-    return LowerToPredicatedOp(Op, DAG, AArch64ISD::SETCC_PRED);
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::SETCC_MERGE_ZERO);
   }
 
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
@@ -9075,7 +9248,8 @@ AArch64TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
   SDNode *Node = Op.getNode();
   SDValue Chain = Op.getOperand(0);
   SDValue Size = Op.getOperand(1);
-  unsigned Align = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
+  MaybeAlign Align =
+      cast<ConstantSDNode>(Op.getOperand(2))->getMaybeAlignValue();
   EVT VT = Node->getValueType(0);
 
   if (DAG.getMachineFunction().getFunction().hasFnAttribute(
@@ -9085,7 +9259,7 @@ AArch64TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
     SP = DAG.getNode(ISD::SUB, dl, MVT::i64, SP, Size);
     if (Align)
       SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
-                       DAG.getConstant(-(uint64_t)Align, dl, VT));
+                       DAG.getConstant(-(uint64_t)Align->value(), dl, VT));
     Chain = DAG.getCopyToReg(Chain, dl, AArch64::SP, SP);
     SDValue Ops[2] = {SP, Chain};
     return DAG.getMergeValues(Ops, dl);
@@ -9100,7 +9274,7 @@ AArch64TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
   SP = DAG.getNode(ISD::SUB, dl, MVT::i64, SP, Size);
   if (Align)
     SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
-                     DAG.getConstant(-(uint64_t)Align, dl, VT));
+                     DAG.getConstant(-(uint64_t)Align->value(), dl, VT));
   Chain = DAG.getCopyToReg(Chain, dl, AArch64::SP, SP);
 
   Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
@@ -9216,7 +9390,7 @@ bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.memVT = MVT::getVT(PtrTy->getElementType());
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = 0;
-    Info.align = MaybeAlign(DL.getABITypeAlignment(PtrTy->getElementType()));
+    Info.align = DL.getABITypeAlign(PtrTy->getElementType());
     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
     return true;
   }
@@ -9227,7 +9401,7 @@ bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.memVT = MVT::getVT(PtrTy->getElementType());
     Info.ptrVal = I.getArgOperand(1);
     Info.offset = 0;
-    Info.align = MaybeAlign(DL.getABITypeAlignment(PtrTy->getElementType()));
+    Info.align = DL.getABITypeAlign(PtrTy->getElementType());
     Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
     return true;
   }
@@ -9255,7 +9429,7 @@ bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.memVT = MVT::getVT(I.getType());
     Info.ptrVal = I.getArgOperand(1);
     Info.offset = 0;
-    Info.align = MaybeAlign(DL.getABITypeAlignment(PtrTy->getElementType()));
+    Info.align = DL.getABITypeAlign(PtrTy->getElementType());
     Info.flags = MachineMemOperand::MOLoad;
     if (Intrinsic == Intrinsic::aarch64_sve_ldnt1)
       Info.flags |= MachineMemOperand::MONonTemporal;
@@ -9267,7 +9441,7 @@ bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.memVT = MVT::getVT(I.getOperand(0)->getType());
     Info.ptrVal = I.getArgOperand(2);
     Info.offset = 0;
-    Info.align = MaybeAlign(DL.getABITypeAlignment(PtrTy->getElementType()));
+    Info.align = DL.getABITypeAlign(PtrTy->getElementType());
     Info.flags = MachineMemOperand::MOStore;
     if (Intrinsic == Intrinsic::aarch64_sve_stnt1)
       Info.flags |= MachineMemOperand::MONonTemporal;
@@ -9452,8 +9626,8 @@ static bool areExtractShuffleVectors(Value *Op1, Value *Op2) {
   };
 
   auto extractHalf = [](Value *FullV, Value *HalfV) {
-    auto *FullVT = cast<VectorType>(FullV->getType());
-    auto *HalfVT = cast<VectorType>(HalfV->getType());
+    auto *FullVT = cast<FixedVectorType>(FullV->getType());
+    auto *HalfVT = cast<FixedVectorType>(HalfV->getType());
     return FullVT->getNumElements() == 2 * HalfVT->getNumElements();
   };
 
@@ -9473,7 +9647,7 @@ static bool areExtractShuffleVectors(Value *Op1, Value *Op2) {
   // elements.
   int M1Start = -1;
   int M2Start = -1;
-  int NumElements = cast<VectorType>(Op1->getType())->getNumElements() * 2;
+  int NumElements = cast<FixedVectorType>(Op1->getType())->getNumElements() * 2;
   if (!ShuffleVectorInst::isExtractSubvectorMask(M1, NumElements, M1Start) ||
       !ShuffleVectorInst::isExtractSubvectorMask(M2, NumElements, M2Start) ||
       M1Start != M2Start || (M1Start != 0 && M2Start != (NumElements / 2)))
@@ -9572,12 +9746,12 @@ bool AArch64TargetLowering::shouldSinkOperands(
 }
 
 bool AArch64TargetLowering::hasPairedLoad(EVT LoadedType,
-                                          unsigned &RequiredAligment) const {
+                                          Align &RequiredAligment) const {
   if (!LoadedType.isSimple() ||
       (!LoadedType.isInteger() && !LoadedType.isFloatingPoint()))
     return false;
   // Cyclone supports unaligned accesses.
-  RequiredAligment = 0;
+  RequiredAligment = Align(1);
   unsigned NumBits = LoadedType.getSizeInBits();
   return NumBits == 32 || NumBits == 64;
 }
@@ -9605,7 +9779,7 @@ bool AArch64TargetLowering::isLegalInterleavedAccessType(
   unsigned ElSize = DL.getTypeSizeInBits(VecTy->getElementType());
 
   // Ensure the number of vector elements is greater than 1.
-  if (VecTy->getNumElements() < 2)
+  if (cast<FixedVectorType>(VecTy)->getNumElements() < 2)
     return false;
 
   // Ensure the element type is legal.
@@ -9639,22 +9813,24 @@ bool AArch64TargetLowering::lowerInterleavedLoad(
 
   const DataLayout &DL = LI->getModule()->getDataLayout();
 
-  VectorType *VecTy = Shuffles[0]->getType();
+  VectorType *VTy = Shuffles[0]->getType();
 
   // Skip if we do not have NEON and skip illegal vector types. We can
   // "legalize" wide vector types into multiple interleaved accesses as long as
   // the vector types are divisible by 128.
-  if (!Subtarget->hasNEON() || !isLegalInterleavedAccessType(VecTy, DL))
+  if (!Subtarget->hasNEON() || !isLegalInterleavedAccessType(VTy, DL))
     return false;
 
-  unsigned NumLoads = getNumInterleavedAccesses(VecTy, DL);
+  unsigned NumLoads = getNumInterleavedAccesses(VTy, DL);
+
+  auto *FVTy = cast<FixedVectorType>(VTy);
 
   // A pointer vector can not be the return type of the ldN intrinsics. Need to
   // load integer vectors first and then convert to pointer vectors.
-  Type *EltTy = VecTy->getElementType();
+  Type *EltTy = FVTy->getElementType();
   if (EltTy->isPointerTy())
-    VecTy =
-        FixedVectorType::get(DL.getIntPtrType(EltTy), VecTy->getNumElements());
+    FVTy =
+        FixedVectorType::get(DL.getIntPtrType(EltTy), FVTy->getNumElements());
 
   IRBuilder<> Builder(LI);
 
@@ -9664,19 +9840,19 @@ bool AArch64TargetLowering::lowerInterleavedLoad(
   if (NumLoads > 1) {
     // If we're going to generate more than one load, reset the sub-vector type
     // to something legal.
-    VecTy = FixedVectorType::get(VecTy->getElementType(),
-                                 VecTy->getNumElements() / NumLoads);
+    FVTy = FixedVectorType::get(FVTy->getElementType(),
+                                FVTy->getNumElements() / NumLoads);
 
     // We will compute the pointer operand of each load from the original base
     // address using GEPs. Cast the base address to a pointer to the scalar
     // element type.
     BaseAddr = Builder.CreateBitCast(
         BaseAddr,
-        VecTy->getElementType()->getPointerTo(LI->getPointerAddressSpace()));
+        FVTy->getElementType()->getPointerTo(LI->getPointerAddressSpace()));
   }
 
-  Type *PtrTy = VecTy->getPointerTo(LI->getPointerAddressSpace());
-  Type *Tys[2] = {VecTy, PtrTy};
+  Type *PtrTy = FVTy->getPointerTo(LI->getPointerAddressSpace());
+  Type *Tys[2] = {FVTy, PtrTy};
   static const Intrinsic::ID LoadInts[3] = {Intrinsic::aarch64_neon_ld2,
                                             Intrinsic::aarch64_neon_ld3,
                                             Intrinsic::aarch64_neon_ld4};
@@ -9693,8 +9869,8 @@ bool AArch64TargetLowering::lowerInterleavedLoad(
     // If we're generating more than one load, compute the base address of
     // subsequent loads as an offset from the previous.
     if (LoadCount > 0)
-      BaseAddr = Builder.CreateConstGEP1_32(VecTy->getElementType(), BaseAddr,
-                                            VecTy->getNumElements() * Factor);
+      BaseAddr = Builder.CreateConstGEP1_32(FVTy->getElementType(), BaseAddr,
+                                            FVTy->getNumElements() * Factor);
 
     CallInst *LdN = Builder.CreateCall(
         LdNFunc, Builder.CreateBitCast(BaseAddr, PtrTy), "ldN");
@@ -9710,7 +9886,7 @@ bool AArch64TargetLowering::lowerInterleavedLoad(
       if (EltTy->isPointerTy())
         SubVec = Builder.CreateIntToPtr(
             SubVec, FixedVectorType::get(SVI->getType()->getElementType(),
-                                         VecTy->getNumElements()));
+                                         FVTy->getNumElements()));
       SubVecs[SVI].push_back(SubVec);
     }
   }
@@ -9761,7 +9937,7 @@ bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI,
   assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
          "Invalid interleave factor");
 
-  VectorType *VecTy = SVI->getType();
+  auto *VecTy = cast<FixedVectorType>(SVI->getType());
   assert(VecTy->getNumElements() % Factor == 0 && "Invalid interleaved store");
 
   unsigned LaneLen = VecTy->getNumElements() / Factor;
@@ -9786,7 +9962,8 @@ bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI,
   // vectors to integer vectors.
   if (EltTy->isPointerTy()) {
     Type *IntTy = DL.getIntPtrType(EltTy);
-    unsigned NumOpElts = cast<VectorType>(Op0->getType())->getNumElements();
+    unsigned NumOpElts =
+        cast<FixedVectorType>(Op0->getType())->getNumElements();
 
     // Convert to the corresponding integer vector.
     auto *IntVecTy = FixedVectorType::get(IntTy, NumOpElts);
@@ -9892,9 +10069,9 @@ SDValue AArch64TargetLowering::LowerSVEStructLoad(unsigned Intrinsic,
 
   unsigned N, Opcode;
   static std::map<unsigned, std::pair<unsigned, unsigned>> IntrinsicMap = {
-      {Intrinsic::aarch64_sve_ld2, {2, AArch64ISD::SVE_LD2}},
-      {Intrinsic::aarch64_sve_ld3, {3, AArch64ISD::SVE_LD3}},
-      {Intrinsic::aarch64_sve_ld4, {4, AArch64ISD::SVE_LD4}}};
+      {Intrinsic::aarch64_sve_ld2, {2, AArch64ISD::SVE_LD2_MERGE_ZERO}},
+      {Intrinsic::aarch64_sve_ld3, {3, AArch64ISD::SVE_LD3_MERGE_ZERO}},
+      {Intrinsic::aarch64_sve_ld4, {4, AArch64ISD::SVE_LD4_MERGE_ZERO}}};
 
   std::tie(N, Opcode) = IntrinsicMap[Intrinsic];
   assert(VT.getVectorElementCount().Min % N == 0 &&
@@ -10835,26 +11012,26 @@ static SDValue performSVEAndCombine(SDNode *N,
   // SVE load instructions perform an implicit zero-extend, which makes them
   // perfect candidates for combining.
   switch (Opc) {
-  case AArch64ISD::LD1:
-  case AArch64ISD::LDNF1:
-  case AArch64ISD::LDFF1:
+  case AArch64ISD::LD1_MERGE_ZERO:
+  case AArch64ISD::LDNF1_MERGE_ZERO:
+  case AArch64ISD::LDFF1_MERGE_ZERO:
     MemVT = cast<VTSDNode>(Src->getOperand(3))->getVT();
     break;
-  case AArch64ISD::GLD1:
-  case AArch64ISD::GLD1_SCALED:
-  case AArch64ISD::GLD1_SXTW:
-  case AArch64ISD::GLD1_SXTW_SCALED:
-  case AArch64ISD::GLD1_UXTW:
-  case AArch64ISD::GLD1_UXTW_SCALED:
-  case AArch64ISD::GLD1_IMM:
-  case AArch64ISD::GLDFF1:
-  case AArch64ISD::GLDFF1_SCALED:
-  case AArch64ISD::GLDFF1_SXTW:
-  case AArch64ISD::GLDFF1_SXTW_SCALED:
-  case AArch64ISD::GLDFF1_UXTW:
-  case AArch64ISD::GLDFF1_UXTW_SCALED:
-  case AArch64ISD::GLDFF1_IMM:
-  case AArch64ISD::GLDNT1:
+  case AArch64ISD::GLD1_MERGE_ZERO:
+  case AArch64ISD::GLD1_SCALED_MERGE_ZERO:
+  case AArch64ISD::GLD1_SXTW_MERGE_ZERO:
+  case AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO:
+  case AArch64ISD::GLD1_UXTW_MERGE_ZERO:
+  case AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO:
+  case AArch64ISD::GLD1_IMM_MERGE_ZERO:
+  case AArch64ISD::GLDFF1_MERGE_ZERO:
+  case AArch64ISD::GLDFF1_SCALED_MERGE_ZERO:
+  case AArch64ISD::GLDFF1_SXTW_MERGE_ZERO:
+  case AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO:
+  case AArch64ISD::GLDFF1_UXTW_MERGE_ZERO:
+  case AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO:
+  case AArch64ISD::GLDFF1_IMM_MERGE_ZERO:
+  case AArch64ISD::GLDNT1_MERGE_ZERO:
     MemVT = cast<VTSDNode>(Src->getOperand(4))->getVT();
     break;
   default:
@@ -10944,6 +11121,7 @@ static SDValue performConcatVectorsCombine(SDNode *N,
   SDLoc dl(N);
   EVT VT = N->getValueType(0);
   SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
+  unsigned N0Opc = N0->getOpcode(), N1Opc = N1->getOpcode();
 
   // Optimize concat_vectors of truncated vectors, where the intermediate
   // type is illegal, to avoid said illegality,  e.g.,
@@ -10956,9 +11134,8 @@ static SDValue performConcatVectorsCombine(SDNode *N,
   // This isn't really target-specific, but ISD::TRUNCATE legality isn't keyed
   // on both input and result type, so we might generate worse code.
   // On AArch64 we know it's fine for v2i64->v4i16 and v4i32->v8i8.
-  if (N->getNumOperands() == 2 &&
-      N0->getOpcode() == ISD::TRUNCATE &&
-      N1->getOpcode() == ISD::TRUNCATE) {
+  if (N->getNumOperands() == 2 && N0Opc == ISD::TRUNCATE &&
+      N1Opc == ISD::TRUNCATE) {
     SDValue N00 = N0->getOperand(0);
     SDValue N10 = N1->getOperand(0);
     EVT N00VT = N00.getValueType();
@@ -10983,6 +11160,52 @@ static SDValue performConcatVectorsCombine(SDNode *N,
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
+  // Optimise concat_vectors of two [us]rhadds that use extracted subvectors
+  // from the same original vectors. Combine these into a single [us]rhadd that
+  // operates on the two original vectors. Example:
+  //  (v16i8 (concat_vectors (v8i8 (urhadd (extract_subvector (v16i8 OpA, <0>),
+  //                                        extract_subvector (v16i8 OpB,
+  //                                        <0>))),
+  //                         (v8i8 (urhadd (extract_subvector (v16i8 OpA, <8>),
+  //                                        extract_subvector (v16i8 OpB,
+  //                                        <8>)))))
+  // ->
+  //  (v16i8(urhadd(v16i8 OpA, v16i8 OpB)))
+  if (N->getNumOperands() == 2 && N0Opc == N1Opc &&
+      (N0Opc == AArch64ISD::URHADD || N0Opc == AArch64ISD::SRHADD)) {
+    SDValue N00 = N0->getOperand(0);
+    SDValue N01 = N0->getOperand(1);
+    SDValue N10 = N1->getOperand(0);
+    SDValue N11 = N1->getOperand(1);
+
+    EVT N00VT = N00.getValueType();
+    EVT N10VT = N10.getValueType();
+
+    if (N00->getOpcode() == ISD::EXTRACT_SUBVECTOR &&
+        N01->getOpcode() == ISD::EXTRACT_SUBVECTOR &&
+        N10->getOpcode() == ISD::EXTRACT_SUBVECTOR &&
+        N11->getOpcode() == ISD::EXTRACT_SUBVECTOR && N00VT == N10VT) {
+      SDValue N00Source = N00->getOperand(0);
+      SDValue N01Source = N01->getOperand(0);
+      SDValue N10Source = N10->getOperand(0);
+      SDValue N11Source = N11->getOperand(0);
+
+      if (N00Source == N10Source && N01Source == N11Source &&
+          N00Source.getValueType() == VT && N01Source.getValueType() == VT) {
+        assert(N0.getValueType() == N1.getValueType());
+
+        uint64_t N00Index = N00.getConstantOperandVal(1);
+        uint64_t N01Index = N01.getConstantOperandVal(1);
+        uint64_t N10Index = N10.getConstantOperandVal(1);
+        uint64_t N11Index = N11.getConstantOperandVal(1);
+
+        if (N00Index == N01Index && N10Index == N11Index && N00Index == 0 &&
+            N10Index == N00VT.getVectorNumElements())
+          return DAG.getNode(N0Opc, dl, VT, N00Source, N01Source);
+      }
+    }
+  }
+
   // If we see a (concat_vectors (v1x64 A), (v1x64 A)) it's really a vector
   // splat. The indexed instructions are going to be expecting a DUPLANE64, so
   // canonicalise to that.
@@ -11001,7 +11224,7 @@ static SDValue performConcatVectorsCombine(SDNode *N,
   // becomes
   //    (bitconvert (concat_vectors (v4i16 (bitconvert LHS)), RHS))
 
-  if (N1->getOpcode() != ISD::BITCAST)
+  if (N1Opc != ISD::BITCAST)
     return SDValue();
   SDValue RHS = N1->getOperand(0);
   MVT RHSTy = RHS.getValueType().getSimpleVT();
@@ -11488,8 +11711,10 @@ static SDValue LowerSVEIntrinsicDUP(SDNode *N, SelectionDAG &DAG) {
   if ((ScalarTy == MVT::i8) || (ScalarTy == MVT::i16))
     Scalar = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Scalar);
 
-  return DAG.getNode(AArch64ISD::DUP_PRED, dl, N->getValueType(0),
-                     N->getOperand(1), N->getOperand(2), Scalar);
+  SDValue Passthru = N->getOperand(1);
+  SDValue Pred = N->getOperand(2);
+  return DAG.getNode(AArch64ISD::DUP_MERGE_PASSTHRU, dl, N->getValueType(0),
+                     Pred, Scalar, Passthru);
 }
 
 static SDValue LowerSVEIntrinsicEXT(SDNode *N, SelectionDAG &DAG) {
@@ -11574,8 +11799,8 @@ static SDValue tryConvertSVEWideCompare(SDNode *N, ISD::CondCode CC,
       return SDValue();
 
     SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, CmpVT, Imm);
-    return DAG.getNode(AArch64ISD::SETCC_PRED, DL, VT, Pred, N->getOperand(2),
-                       Splat, DAG.getCondCode(CC));
+    return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, DL, VT, Pred,
+                       N->getOperand(2), Splat, DAG.getCondCode(CC));
   }
 
   return SDValue();
@@ -11722,68 +11947,62 @@ static SDValue performIntrinsicCombine(SDNode *N,
                        N->getOperand(1));
   case Intrinsic::aarch64_sve_ext:
     return LowerSVEIntrinsicEXT(N, DAG);
-  case Intrinsic::aarch64_sve_sdiv:
-    return DAG.getNode(AArch64ISD::SDIV_PRED, SDLoc(N), N->getValueType(0),
-                       N->getOperand(1), N->getOperand(2), N->getOperand(3));
-  case Intrinsic::aarch64_sve_udiv:
-    return DAG.getNode(AArch64ISD::UDIV_PRED, SDLoc(N), N->getValueType(0),
-                       N->getOperand(1), N->getOperand(2), N->getOperand(3));
   case Intrinsic::aarch64_sve_smin:
-    return DAG.getNode(AArch64ISD::SMIN_PRED, SDLoc(N), N->getValueType(0),
+    return DAG.getNode(AArch64ISD::SMIN_MERGE_OP1, SDLoc(N), N->getValueType(0),
                        N->getOperand(1), N->getOperand(2), N->getOperand(3));
   case Intrinsic::aarch64_sve_umin:
-    return DAG.getNode(AArch64ISD::UMIN_PRED, SDLoc(N), N->getValueType(0),
+    return DAG.getNode(AArch64ISD::UMIN_MERGE_OP1, SDLoc(N), N->getValueType(0),
                        N->getOperand(1), N->getOperand(2), N->getOperand(3));
   case Intrinsic::aarch64_sve_smax:
-    return DAG.getNode(AArch64ISD::SMAX_PRED, SDLoc(N), N->getValueType(0),
+    return DAG.getNode(AArch64ISD::SMAX_MERGE_OP1, SDLoc(N), N->getValueType(0),
                        N->getOperand(1), N->getOperand(2), N->getOperand(3));
   case Intrinsic::aarch64_sve_umax:
-    return DAG.getNode(AArch64ISD::UMAX_PRED, SDLoc(N), N->getValueType(0),
+    return DAG.getNode(AArch64ISD::UMAX_MERGE_OP1, SDLoc(N), N->getValueType(0),
                        N->getOperand(1), N->getOperand(2), N->getOperand(3));
   case Intrinsic::aarch64_sve_lsl:
-    return DAG.getNode(AArch64ISD::SHL_PRED, SDLoc(N), N->getValueType(0),
+    return DAG.getNode(AArch64ISD::SHL_MERGE_OP1, SDLoc(N), N->getValueType(0),
                        N->getOperand(1), N->getOperand(2), N->getOperand(3));
   case Intrinsic::aarch64_sve_lsr:
-    return DAG.getNode(AArch64ISD::SRL_PRED, SDLoc(N), N->getValueType(0),
+    return DAG.getNode(AArch64ISD::SRL_MERGE_OP1, SDLoc(N), N->getValueType(0),
                        N->getOperand(1), N->getOperand(2), N->getOperand(3));
   case Intrinsic::aarch64_sve_asr:
-    return DAG.getNode(AArch64ISD::SRA_PRED, SDLoc(N), N->getValueType(0),
+    return DAG.getNode(AArch64ISD::SRA_MERGE_OP1, SDLoc(N), N->getValueType(0),
                        N->getOperand(1), N->getOperand(2), N->getOperand(3));
   case Intrinsic::aarch64_sve_cmphs:
     if (!N->getOperand(2).getValueType().isFloatingPoint())
-      return DAG.getNode(AArch64ISD::SETCC_PRED, SDLoc(N), N->getValueType(0),
-                         N->getOperand(1), N->getOperand(2), N->getOperand(3),
-                         DAG.getCondCode(ISD::SETUGE));
+      return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
+                         N->getValueType(0), N->getOperand(1), N->getOperand(2),
+                         N->getOperand(3), DAG.getCondCode(ISD::SETUGE));
     break;
   case Intrinsic::aarch64_sve_cmphi:
     if (!N->getOperand(2).getValueType().isFloatingPoint())
-      return DAG.getNode(AArch64ISD::SETCC_PRED, SDLoc(N), N->getValueType(0),
-                         N->getOperand(1), N->getOperand(2), N->getOperand(3),
-                         DAG.getCondCode(ISD::SETUGT));
+      return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
+                         N->getValueType(0), N->getOperand(1), N->getOperand(2),
+                         N->getOperand(3), DAG.getCondCode(ISD::SETUGT));
     break;
   case Intrinsic::aarch64_sve_cmpge:
     if (!N->getOperand(2).getValueType().isFloatingPoint())
-      return DAG.getNode(AArch64ISD::SETCC_PRED, SDLoc(N), N->getValueType(0),
-                         N->getOperand(1), N->getOperand(2), N->getOperand(3),
-                         DAG.getCondCode(ISD::SETGE));
+      return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
+                         N->getValueType(0), N->getOperand(1), N->getOperand(2),
+                         N->getOperand(3), DAG.getCondCode(ISD::SETGE));
     break;
   case Intrinsic::aarch64_sve_cmpgt:
     if (!N->getOperand(2).getValueType().isFloatingPoint())
-      return DAG.getNode(AArch64ISD::SETCC_PRED, SDLoc(N), N->getValueType(0),
-                         N->getOperand(1), N->getOperand(2), N->getOperand(3),
-                         DAG.getCondCode(ISD::SETGT));
+      return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
+                         N->getValueType(0), N->getOperand(1), N->getOperand(2),
+                         N->getOperand(3), DAG.getCondCode(ISD::SETGT));
     break;
   case Intrinsic::aarch64_sve_cmpeq:
     if (!N->getOperand(2).getValueType().isFloatingPoint())
-      return DAG.getNode(AArch64ISD::SETCC_PRED, SDLoc(N), N->getValueType(0),
-                         N->getOperand(1), N->getOperand(2), N->getOperand(3),
-                         DAG.getCondCode(ISD::SETEQ));
+      return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
+                         N->getValueType(0), N->getOperand(1), N->getOperand(2),
+                         N->getOperand(3), DAG.getCondCode(ISD::SETEQ));
     break;
   case Intrinsic::aarch64_sve_cmpne:
     if (!N->getOperand(2).getValueType().isFloatingPoint())
-      return DAG.getNode(AArch64ISD::SETCC_PRED, SDLoc(N), N->getValueType(0),
-                         N->getOperand(1), N->getOperand(2), N->getOperand(3),
-                         DAG.getCondCode(ISD::SETNE));
+      return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
+                         N->getValueType(0), N->getOperand(1), N->getOperand(2),
+                         N->getOperand(3), DAG.getCondCode(ISD::SETNE));
     break;
   case Intrinsic::aarch64_sve_fadda:
     return combineSVEReductionOrderedFP(N, AArch64ISD::FADDA_PRED, DAG);
@@ -11991,6 +12210,7 @@ static MVT getSVEContainerType(EVT ContentTy) {
   case MVT::nxv8i8:
   case MVT::nxv8i16:
   case MVT::nxv8f16:
+  case MVT::nxv8bf16:
     return MVT::nxv8i16;
   case MVT::nxv16i8:
     return MVT::nxv16i8;
@@ -12028,6 +12248,10 @@ static SDValue performLDNT1Combine(SDNode *N, SelectionDAG &DAG) {
   EVT VT = N->getValueType(0);
   EVT PtrTy = N->getOperand(3).getValueType();
 
+  if (VT == MVT::nxv8bf16 &&
+      !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
+    return SDValue();
+
   EVT LoadVT = VT;
   if (VT.isFloatingPoint())
     LoadVT = VT.changeTypeToInteger();
@@ -12050,7 +12274,8 @@ static SDValue performLDNT1Combine(SDNode *N, SelectionDAG &DAG) {
 
 template <unsigned Opcode>
 static SDValue performLD1ReplicateCombine(SDNode *N, SelectionDAG &DAG) {
-  static_assert(Opcode == AArch64ISD::LD1RQ || Opcode == AArch64ISD::LD1RO,
+  static_assert(Opcode == AArch64ISD::LD1RQ_MERGE_ZERO ||
+                    Opcode == AArch64ISD::LD1RO_MERGE_ZERO,
                 "Unsupported opcode.");
   SDLoc DL(N);
   EVT VT = N->getValueType(0);
@@ -12076,6 +12301,10 @@ static SDValue performST1Combine(SDNode *N, SelectionDAG &DAG) {
   EVT HwSrcVt = getSVEContainerType(DataVT);
   SDValue InputVT = DAG.getValueType(DataVT);
 
+  if (DataVT == MVT::nxv8bf16 &&
+      !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
+    return SDValue();
+
   if (DataVT.isFloatingPoint())
     InputVT = DAG.getValueType(HwSrcVt);
 
@@ -12092,7 +12321,7 @@ static SDValue performST1Combine(SDNode *N, SelectionDAG &DAG) {
                     InputVT
                   };
 
-  return DAG.getNode(AArch64ISD::ST1, DL, N->getValueType(0), Ops);
+  return DAG.getNode(AArch64ISD::ST1_PRED, DL, N->getValueType(0), Ops);
 }
 
 static SDValue performSTNT1Combine(SDNode *N, SelectionDAG &DAG) {
@@ -12102,6 +12331,10 @@ static SDValue performSTNT1Combine(SDNode *N, SelectionDAG &DAG) {
   EVT DataVT = Data.getValueType();
   EVT PtrTy = N->getOperand(4).getValueType();
 
+  if (DataVT == MVT::nxv8bf16 &&
+      !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
+    return SDValue();
+
   if (DataVT.isFloatingPoint())
     Data = DAG.getNode(ISD::BITCAST, DL, DataVT.changeTypeToInteger(), Data);
 
@@ -12329,6 +12562,9 @@ static SDValue performPostLD1Combine(SDNode *N,
   SelectionDAG &DAG = DCI.DAG;
   EVT VT = N->getValueType(0);
 
+  if (VT.isScalableVector())
+    return SDValue();
+
   unsigned LoadIdx = IsLaneOp ? 1 : 0;
   SDNode *LD = N->getOperand(LoadIdx).getNode();
   // If it is not LOAD, can not do such combine.
@@ -13246,10 +13482,10 @@ static SDValue performScatterStoreCombine(SDNode *N, SelectionDAG &DAG,
   // For "scalar + vector of indices", just scale the indices. This only
   // applies to non-temporal scatters because there's no instruction that takes
   // indicies.
-  if (Opcode == AArch64ISD::SSTNT1_INDEX) {
+  if (Opcode == AArch64ISD::SSTNT1_INDEX_PRED) {
     Offset =
         getScaledOffsetForBitWidth(DAG, Offset, DL, SrcElVT.getSizeInBits());
-    Opcode = AArch64ISD::SSTNT1;
+    Opcode = AArch64ISD::SSTNT1_PRED;
   }
 
   // In the case of non-temporal gather loads there's only one SVE instruction
@@ -13257,8 +13493,8 @@ static SDValue performScatterStoreCombine(SDNode *N, SelectionDAG &DAG,
   //    * stnt1{b|h|w|d} { z0.s }, p0/z, [z0.s, x0]
   // Since we do have intrinsics that allow the arguments to be in a different
   // order, we may need to swap them to match the spec.
-  if (Opcode == AArch64ISD::SSTNT1 && Offset.getValueType().isVector())
-      std::swap(Base, Offset);
+  if (Opcode == AArch64ISD::SSTNT1_PRED && Offset.getValueType().isVector())
+    std::swap(Base, Offset);
 
   // SST1_IMM requires that the offset is an immediate that is:
   //    * a multiple of #SizeInBytes,
@@ -13266,13 +13502,13 @@ static SDValue performScatterStoreCombine(SDNode *N, SelectionDAG &DAG,
   // where #SizeInBytes is the size in bytes of the stored items. For
   // immediates outside that range and non-immediate scalar offsets use SST1 or
   // SST1_UXTW instead.
-  if (Opcode == AArch64ISD::SST1_IMM) {
+  if (Opcode == AArch64ISD::SST1_IMM_PRED) {
     if (!isValidImmForSVEVecImmAddrMode(Offset,
                                         SrcVT.getScalarSizeInBits() / 8)) {
       if (MVT::nxv4i32 == Base.getValueType().getSimpleVT().SimpleTy)
-        Opcode = AArch64ISD::SST1_UXTW;
+        Opcode = AArch64ISD::SST1_UXTW_PRED;
       else
-        Opcode = AArch64ISD::SST1;
+        Opcode = AArch64ISD::SST1_PRED;
 
       std::swap(Base, Offset);
     }
@@ -13343,10 +13579,10 @@ static SDValue performGatherLoadCombine(SDNode *N, SelectionDAG &DAG,
   // For "scalar + vector of indices", just scale the indices. This only
   // applies to non-temporal gathers because there's no instruction that takes
   // indicies.
-  if (Opcode == AArch64ISD::GLDNT1_INDEX) {
+  if (Opcode == AArch64ISD::GLDNT1_INDEX_MERGE_ZERO) {
     Offset = getScaledOffsetForBitWidth(DAG, Offset, DL,
                                         RetVT.getScalarSizeInBits());
-    Opcode = AArch64ISD::GLDNT1;
+    Opcode = AArch64ISD::GLDNT1_MERGE_ZERO;
   }
 
   // In the case of non-temporal gather loads there's only one SVE instruction
@@ -13354,24 +13590,28 @@ static SDValue performGatherLoadCombine(SDNode *N, SelectionDAG &DAG,
   //    * ldnt1{b|h|w|d} { z0.s }, p0/z, [z0.s, x0]
   // Since we do have intrinsics that allow the arguments to be in a different
   // order, we may need to swap them to match the spec.
-  if (Opcode == AArch64ISD::GLDNT1 && Offset.getValueType().isVector())
-      std::swap(Base, Offset);
+  if (Opcode == AArch64ISD::GLDNT1_MERGE_ZERO &&
+      Offset.getValueType().isVector())
+    std::swap(Base, Offset);
 
   // GLD{FF}1_IMM requires that the offset is an immediate that is:
   //    * a multiple of #SizeInBytes,
   //    * in the range [0, 31 x #SizeInBytes],
   // where #SizeInBytes is the size in bytes of the loaded items. For
-  // immediates outside that range and non-immediate scalar offsets use GLD1 or
-  // GLD1_UXTW instead.
-  if (Opcode == AArch64ISD::GLD1_IMM || Opcode == AArch64ISD::GLDFF1_IMM) {
+  // immediates outside that range and non-immediate scalar offsets use
+  // GLD1_MERGE_ZERO or GLD1_UXTW_MERGE_ZERO instead.
+  if (Opcode == AArch64ISD::GLD1_IMM_MERGE_ZERO ||
+      Opcode == AArch64ISD::GLDFF1_IMM_MERGE_ZERO) {
     if (!isValidImmForSVEVecImmAddrMode(Offset,
                                         RetVT.getScalarSizeInBits() / 8)) {
       if (MVT::nxv4i32 == Base.getValueType().getSimpleVT().SimpleTy)
-        Opcode = (Opcode == AArch64ISD::GLD1_IMM) ? AArch64ISD::GLD1_UXTW
-                                                  : AArch64ISD::GLDFF1_UXTW;
+        Opcode = (Opcode == AArch64ISD::GLD1_IMM_MERGE_ZERO)
+                     ? AArch64ISD::GLD1_UXTW_MERGE_ZERO
+                     : AArch64ISD::GLDFF1_UXTW_MERGE_ZERO;
       else
-        Opcode = (Opcode == AArch64ISD::GLD1_IMM) ? AArch64ISD::GLD1
-                                                  : AArch64ISD::GLDFF1;
+        Opcode = (Opcode == AArch64ISD::GLD1_IMM_MERGE_ZERO)
+                     ? AArch64ISD::GLD1_MERGE_ZERO
+                     : AArch64ISD::GLDFF1_MERGE_ZERO;
 
       std::swap(Base, Offset);
     }
@@ -13464,62 +13704,62 @@ performSignExtendInRegCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
   unsigned NewOpc;
   unsigned MemVTOpNum = 4;
   switch (Opc) {
-  case AArch64ISD::LD1:
-    NewOpc = AArch64ISD::LD1S;
+  case AArch64ISD::LD1_MERGE_ZERO:
+    NewOpc = AArch64ISD::LD1S_MERGE_ZERO;
     MemVTOpNum = 3;
     break;
-  case AArch64ISD::LDNF1:
-    NewOpc = AArch64ISD::LDNF1S;
+  case AArch64ISD::LDNF1_MERGE_ZERO:
+    NewOpc = AArch64ISD::LDNF1S_MERGE_ZERO;
     MemVTOpNum = 3;
     break;
-  case AArch64ISD::LDFF1:
-    NewOpc = AArch64ISD::LDFF1S;
+  case AArch64ISD::LDFF1_MERGE_ZERO:
+    NewOpc = AArch64ISD::LDFF1S_MERGE_ZERO;
     MemVTOpNum = 3;
     break;
-  case AArch64ISD::GLD1:
-    NewOpc = AArch64ISD::GLD1S;
+  case AArch64ISD::GLD1_MERGE_ZERO:
+    NewOpc = AArch64ISD::GLD1S_MERGE_ZERO;
     break;
-  case AArch64ISD::GLD1_SCALED:
-    NewOpc = AArch64ISD::GLD1S_SCALED;
+  case AArch64ISD::GLD1_SCALED_MERGE_ZERO:
+    NewOpc = AArch64ISD::GLD1S_SCALED_MERGE_ZERO;
     break;
-  case AArch64ISD::GLD1_SXTW:
-    NewOpc = AArch64ISD::GLD1S_SXTW;
+  case AArch64ISD::GLD1_SXTW_MERGE_ZERO:
+    NewOpc = AArch64ISD::GLD1S_SXTW_MERGE_ZERO;
     break;
-  case AArch64ISD::GLD1_SXTW_SCALED:
-    NewOpc = AArch64ISD::GLD1S_SXTW_SCALED;
+  case AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO:
+    NewOpc = AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO;
     break;
-  case AArch64ISD::GLD1_UXTW:
-    NewOpc = AArch64ISD::GLD1S_UXTW;
+  case AArch64ISD::GLD1_UXTW_MERGE_ZERO:
+    NewOpc = AArch64ISD::GLD1S_UXTW_MERGE_ZERO;
     break;
-  case AArch64ISD::GLD1_UXTW_SCALED:
-    NewOpc = AArch64ISD::GLD1S_UXTW_SCALED;
+  case AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO:
+    NewOpc = AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO;
     break;
-  case AArch64ISD::GLD1_IMM:
-    NewOpc = AArch64ISD::GLD1S_IMM;
+  case AArch64ISD::GLD1_IMM_MERGE_ZERO:
+    NewOpc = AArch64ISD::GLD1S_IMM_MERGE_ZERO;
     break;
-  case AArch64ISD::GLDFF1:
-    NewOpc = AArch64ISD::GLDFF1S;
+  case AArch64ISD::GLDFF1_MERGE_ZERO:
+    NewOpc = AArch64ISD::GLDFF1S_MERGE_ZERO;
     break;
-  case AArch64ISD::GLDFF1_SCALED:
-    NewOpc = AArch64ISD::GLDFF1S_SCALED;
+  case AArch64ISD::GLDFF1_SCALED_MERGE_ZERO:
+    NewOpc = AArch64ISD::GLDFF1S_SCALED_MERGE_ZERO;
     break;
-  case AArch64ISD::GLDFF1_SXTW:
-    NewOpc = AArch64ISD::GLDFF1S_SXTW;
+  case AArch64ISD::GLDFF1_SXTW_MERGE_ZERO:
+    NewOpc = AArch64ISD::GLDFF1S_SXTW_MERGE_ZERO;
     break;
-  case AArch64ISD::GLDFF1_SXTW_SCALED:
-    NewOpc = AArch64ISD::GLDFF1S_SXTW_SCALED;
+  case AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO:
+    NewOpc = AArch64ISD::GLDFF1S_SXTW_SCALED_MERGE_ZERO;
     break;
-  case AArch64ISD::GLDFF1_UXTW:
-    NewOpc = AArch64ISD::GLDFF1S_UXTW;
+  case AArch64ISD::GLDFF1_UXTW_MERGE_ZERO:
+    NewOpc = AArch64ISD::GLDFF1S_UXTW_MERGE_ZERO;
     break;
-  case AArch64ISD::GLDFF1_UXTW_SCALED:
-    NewOpc = AArch64ISD::GLDFF1S_UXTW_SCALED;
+  case AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO:
+    NewOpc = AArch64ISD::GLDFF1S_UXTW_SCALED_MERGE_ZERO;
     break;
-  case AArch64ISD::GLDFF1_IMM:
-    NewOpc = AArch64ISD::GLDFF1S_IMM;
+  case AArch64ISD::GLDFF1_IMM_MERGE_ZERO:
+    NewOpc = AArch64ISD::GLDFF1S_IMM_MERGE_ZERO;
     break;
-  case AArch64ISD::GLDNT1:
-    NewOpc = AArch64ISD::GLDNT1S;
+  case AArch64ISD::GLDNT1_MERGE_ZERO:
+    NewOpc = AArch64ISD::GLDNT1S_MERGE_ZERO;
     break;
   default:
     return SDValue();
@@ -13697,89 +13937,101 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
     case Intrinsic::aarch64_sve_ldnt1:
       return performLDNT1Combine(N, DAG);
     case Intrinsic::aarch64_sve_ld1rq:
-      return performLD1ReplicateCombine<AArch64ISD::LD1RQ>(N, DAG);
+      return performLD1ReplicateCombine<AArch64ISD::LD1RQ_MERGE_ZERO>(N, DAG);
     case Intrinsic::aarch64_sve_ld1ro:
-      return performLD1ReplicateCombine<AArch64ISD::LD1RO>(N, DAG);
+      return performLD1ReplicateCombine<AArch64ISD::LD1RO_MERGE_ZERO>(N, DAG);
     case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1);
+      return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1_MERGE_ZERO);
     case Intrinsic::aarch64_sve_ldnt1_gather:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1);
+      return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1_MERGE_ZERO);
     case Intrinsic::aarch64_sve_ldnt1_gather_index:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1_INDEX);
+      return performGatherLoadCombine(N, DAG,
+                                      AArch64ISD::GLDNT1_INDEX_MERGE_ZERO);
     case Intrinsic::aarch64_sve_ldnt1_gather_uxtw:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1);
+      return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1_MERGE_ZERO);
     case Intrinsic::aarch64_sve_ld1:
-      return performLD1Combine(N, DAG, AArch64ISD::LD1);
+      return performLD1Combine(N, DAG, AArch64ISD::LD1_MERGE_ZERO);
     case Intrinsic::aarch64_sve_ldnf1:
-      return performLD1Combine(N, DAG, AArch64ISD::LDNF1);
+      return performLD1Combine(N, DAG, AArch64ISD::LDNF1_MERGE_ZERO);
     case Intrinsic::aarch64_sve_ldff1:
-      return performLD1Combine(N, DAG, AArch64ISD::LDFF1);
+      return performLD1Combine(N, DAG, AArch64ISD::LDFF1_MERGE_ZERO);
     case Intrinsic::aarch64_sve_st1:
       return performST1Combine(N, DAG);
     case Intrinsic::aarch64_sve_stnt1:
       return performSTNT1Combine(N, DAG);
     case Intrinsic::aarch64_sve_stnt1_scatter_scalar_offset:
-      return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1);
+      return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1_PRED);
     case Intrinsic::aarch64_sve_stnt1_scatter_uxtw:
-      return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1);
+      return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1_PRED);
     case Intrinsic::aarch64_sve_stnt1_scatter:
-      return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1);
+      return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1_PRED);
     case Intrinsic::aarch64_sve_stnt1_scatter_index:
-      return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1_INDEX);
+      return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1_INDEX_PRED);
     case Intrinsic::aarch64_sve_ld1_gather:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1);
+      return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_MERGE_ZERO);
     case Intrinsic::aarch64_sve_ld1_gather_index:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_SCALED);
+      return performGatherLoadCombine(N, DAG,
+                                      AArch64ISD::GLD1_SCALED_MERGE_ZERO);
     case Intrinsic::aarch64_sve_ld1_gather_sxtw:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_SXTW,
+      return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_SXTW_MERGE_ZERO,
                                       /*OnlyPackedOffsets=*/false);
     case Intrinsic::aarch64_sve_ld1_gather_uxtw:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_UXTW,
+      return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_UXTW_MERGE_ZERO,
                                       /*OnlyPackedOffsets=*/false);
     case Intrinsic::aarch64_sve_ld1_gather_sxtw_index:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_SXTW_SCALED,
+      return performGatherLoadCombine(N, DAG,
+                                      AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO,
                                       /*OnlyPackedOffsets=*/false);
     case Intrinsic::aarch64_sve_ld1_gather_uxtw_index:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_UXTW_SCALED,
+      return performGatherLoadCombine(N, DAG,
+                                      AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO,
                                       /*OnlyPackedOffsets=*/false);
     case Intrinsic::aarch64_sve_ld1_gather_scalar_offset:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_IMM);
+      return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_IMM_MERGE_ZERO);
     case Intrinsic::aarch64_sve_ldff1_gather:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLDFF1);
+      return performGatherLoadCombine(N, DAG, AArch64ISD::GLDFF1_MERGE_ZERO);
     case Intrinsic::aarch64_sve_ldff1_gather_index:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLDFF1_SCALED);
+      return performGatherLoadCombine(N, DAG,
+                                      AArch64ISD::GLDFF1_SCALED_MERGE_ZERO);
     case Intrinsic::aarch64_sve_ldff1_gather_sxtw:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLDFF1_SXTW,
+      return performGatherLoadCombine(N, DAG,
+                                      AArch64ISD::GLDFF1_SXTW_MERGE_ZERO,
                                       /*OnlyPackedOffsets=*/false);
     case Intrinsic::aarch64_sve_ldff1_gather_uxtw:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLDFF1_UXTW,
+      return performGatherLoadCombine(N, DAG,
+                                      AArch64ISD::GLDFF1_UXTW_MERGE_ZERO,
                                       /*OnlyPackedOffsets=*/false);
     case Intrinsic::aarch64_sve_ldff1_gather_sxtw_index:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLDFF1_SXTW_SCALED,
+      return performGatherLoadCombine(N, DAG,
+                                      AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO,
                                       /*OnlyPackedOffsets=*/false);
     case Intrinsic::aarch64_sve_ldff1_gather_uxtw_index:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLDFF1_UXTW_SCALED,
+      return performGatherLoadCombine(N, DAG,
+                                      AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO,
                                       /*OnlyPackedOffsets=*/false);
     case Intrinsic::aarch64_sve_ldff1_gather_scalar_offset:
-      return performGatherLoadCombine(N, DAG, AArch64ISD::GLDFF1_IMM);
+      return performGatherLoadCombine(N, DAG,
+                                      AArch64ISD::GLDFF1_IMM_MERGE_ZERO);
     case Intrinsic::aarch64_sve_st1_scatter:
-      return performScatterStoreCombine(N, DAG, AArch64ISD::SST1);
+      return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_PRED);
     case Intrinsic::aarch64_sve_st1_scatter_index:
-      return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_SCALED);
+      return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_SCALED_PRED);
     case Intrinsic::aarch64_sve_st1_scatter_sxtw:
-      return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_SXTW,
+      return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_SXTW_PRED,
                                         /*OnlyPackedOffsets=*/false);
     case Intrinsic::aarch64_sve_st1_scatter_uxtw:
-      return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_UXTW,
+      return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_UXTW_PRED,
                                         /*OnlyPackedOffsets=*/false);
     case Intrinsic::aarch64_sve_st1_scatter_sxtw_index:
-      return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_SXTW_SCALED,
+      return performScatterStoreCombine(N, DAG,
+                                        AArch64ISD::SST1_SXTW_SCALED_PRED,
                                         /*OnlyPackedOffsets=*/false);
     case Intrinsic::aarch64_sve_st1_scatter_uxtw_index:
-      return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_UXTW_SCALED,
+      return performScatterStoreCombine(N, DAG,
+                                        AArch64ISD::SST1_UXTW_SCALED_PRED,
                                         /*OnlyPackedOffsets=*/false);
     case Intrinsic::aarch64_sve_st1_scatter_scalar_offset:
-      return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_IMM);
+      return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_IMM_PRED);
     case Intrinsic::aarch64_sve_tuple_get: {
       SDLoc DL(N);
       SDValue Chain = N->getOperand(0);
@@ -13787,9 +14039,6 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
       SDValue Idx = N->getOperand(3);
 
       uint64_t IdxConst = cast<ConstantSDNode>(Idx)->getZExtValue();
-      if (IdxConst > Src1->getNumOperands() - 1)
-        report_fatal_error("index larger than expected");
-
       EVT ResVT = N->getValueType(0);
       uint64_t NumLanes = ResVT.getVectorElementCount().Min;
       SDValue Val =
@@ -14653,3 +14902,246 @@ bool AArch64TargetLowering::fallBackToDAGISel(const Instruction &Inst) const {
 
   return false;
 }
+
+// Return the largest legal scalable vector type that matches VT's element type.
+static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT) {
+  assert(VT.isFixedLengthVector() &&
+         DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
+         "Expected legal fixed length vector!");
+  switch (VT.getVectorElementType().getSimpleVT().SimpleTy) {
+  default:
+    llvm_unreachable("unexpected element type for SVE container");
+  case MVT::i8:
+    return EVT(MVT::nxv16i8);
+  case MVT::i16:
+    return EVT(MVT::nxv8i16);
+  case MVT::i32:
+    return EVT(MVT::nxv4i32);
+  case MVT::i64:
+    return EVT(MVT::nxv2i64);
+  case MVT::f16:
+    return EVT(MVT::nxv8f16);
+  case MVT::f32:
+    return EVT(MVT::nxv4f32);
+  case MVT::f64:
+    return EVT(MVT::nxv2f64);
+  }
+}
+
+// Return a PTRUE with active lanes corresponding to the extent of VT.
+static SDValue getPredicateForFixedLengthVector(SelectionDAG &DAG, SDLoc &DL,
+                                                EVT VT) {
+  assert(VT.isFixedLengthVector() &&
+         DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
+         "Expected legal fixed length vector!");
+
+  int PgPattern;
+  switch (VT.getVectorNumElements()) {
+  default:
+    llvm_unreachable("unexpected element count for SVE predicate");
+  case 1:
+    PgPattern = AArch64SVEPredPattern::vl1;
+    break;
+  case 2:
+    PgPattern = AArch64SVEPredPattern::vl2;
+    break;
+  case 4:
+    PgPattern = AArch64SVEPredPattern::vl4;
+    break;
+  case 8:
+    PgPattern = AArch64SVEPredPattern::vl8;
+    break;
+  case 16:
+    PgPattern = AArch64SVEPredPattern::vl16;
+    break;
+  case 32:
+    PgPattern = AArch64SVEPredPattern::vl32;
+    break;
+  case 64:
+    PgPattern = AArch64SVEPredPattern::vl64;
+    break;
+  case 128:
+    PgPattern = AArch64SVEPredPattern::vl128;
+    break;
+  case 256:
+    PgPattern = AArch64SVEPredPattern::vl256;
+    break;
+  }
+
+  // TODO: For vectors that are exactly getMaxSVEVectorSizeInBits big, we can
+  // use AArch64SVEPredPattern::all, which can enable the use of unpredicated
+  // variants of instructions when available.
+
+  MVT MaskVT;
+  switch (VT.getVectorElementType().getSimpleVT().SimpleTy) {
+  default:
+    llvm_unreachable("unexpected element type for SVE predicate");
+  case MVT::i8:
+    MaskVT = MVT::nxv16i1;
+    break;
+  case MVT::i16:
+  case MVT::f16:
+    MaskVT = MVT::nxv8i1;
+    break;
+  case MVT::i32:
+  case MVT::f32:
+    MaskVT = MVT::nxv4i1;
+    break;
+  case MVT::i64:
+  case MVT::f64:
+    MaskVT = MVT::nxv2i1;
+    break;
+  }
+
+  return DAG.getNode(AArch64ISD::PTRUE, DL, MaskVT,
+                     DAG.getTargetConstant(PgPattern, DL, MVT::i64));
+}
+
+static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
+                                             EVT VT) {
+  assert(VT.isScalableVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
+         "Expected legal scalable vector!");
+  auto PredTy = VT.changeVectorElementType(MVT::i1);
+  return getPTrue(DAG, DL, PredTy, AArch64SVEPredPattern::all);
+}
+
+static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT) {
+  if (VT.isFixedLengthVector())
+    return getPredicateForFixedLengthVector(DAG, DL, VT);
+
+  return getPredicateForScalableVector(DAG, DL, VT);
+}
+
+// Grow V to consume an entire SVE register.
+static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V) {
+  assert(VT.isScalableVector() &&
+         "Expected to convert into a scalable vector!");
+  assert(V.getValueType().isFixedLengthVector() &&
+         "Expected a fixed length vector operand!");
+  SDLoc DL(V);
+  SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
+  return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
+}
+
+// Shrink V so it's just big enough to maintain a VT's worth of data.
+static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V) {
+  assert(VT.isFixedLengthVector() &&
+         "Expected to convert into a fixed length vector!");
+  assert(V.getValueType().isScalableVector() &&
+         "Expected a scalable vector operand!");
+  SDLoc DL(V);
+  SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
+  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
+}
+
+// Convert all fixed length vector loads larger than NEON to masked_loads.
+SDValue AArch64TargetLowering::LowerFixedLengthVectorLoadToSVE(
+    SDValue Op, SelectionDAG &DAG) const {
+  auto Load = cast<LoadSDNode>(Op);
+
+  SDLoc DL(Op);
+  EVT VT = Op.getValueType();
+  EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
+
+  auto NewLoad = DAG.getMaskedLoad(
+      ContainerVT, DL, Load->getChain(), Load->getBasePtr(), Load->getOffset(),
+      getPredicateForFixedLengthVector(DAG, DL, VT), DAG.getUNDEF(ContainerVT),
+      Load->getMemoryVT(), Load->getMemOperand(), Load->getAddressingMode(),
+      Load->getExtensionType());
+
+  auto Result = convertFromScalableVector(DAG, VT, NewLoad);
+  SDValue MergedValues[2] = {Result, Load->getChain()};
+  return DAG.getMergeValues(MergedValues, DL);
+}
+
+// Convert all fixed length vector stores larger than NEON to masked_stores.
+SDValue AArch64TargetLowering::LowerFixedLengthVectorStoreToSVE(
+    SDValue Op, SelectionDAG &DAG) const {
+  auto Store = cast<StoreSDNode>(Op);
+
+  SDLoc DL(Op);
+  EVT VT = Store->getValue().getValueType();
+  EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
+
+  auto NewValue = convertToScalableVector(DAG, ContainerVT, Store->getValue());
+  return DAG.getMaskedStore(
+      Store->getChain(), DL, NewValue, Store->getBasePtr(), Store->getOffset(),
+      getPredicateForFixedLengthVector(DAG, DL, VT), Store->getMemoryVT(),
+      Store->getMemOperand(), Store->getAddressingMode(),
+      Store->isTruncatingStore());
+}
+
+SDValue AArch64TargetLowering::LowerFixedLengthVectorTruncateToSVE(
+    SDValue Op, SelectionDAG &DAG) const {
+  EVT VT = Op.getValueType();
+  assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
+
+  SDLoc DL(Op);
+  SDValue Val = Op.getOperand(0);
+  EVT ContainerVT = getContainerForFixedLengthVector(DAG, Val.getValueType());
+  Val = convertToScalableVector(DAG, ContainerVT, Val);
+
+  // Repeatedly truncate Val until the result is of the desired element type.
+  switch (ContainerVT.getSimpleVT().SimpleTy) {
+  default:
+    llvm_unreachable("unimplemented container type");
+  case MVT::nxv2i64:
+    Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv4i32, Val);
+    Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv4i32, Val, Val);
+    if (VT.getVectorElementType() == MVT::i32)
+      break;
+    LLVM_FALLTHROUGH;
+  case MVT::nxv4i32:
+    Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv8i16, Val);
+    Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv8i16, Val, Val);
+    if (VT.getVectorElementType() == MVT::i16)
+      break;
+    LLVM_FALLTHROUGH;
+  case MVT::nxv8i16:
+    Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv16i8, Val);
+    Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv16i8, Val, Val);
+    assert(VT.getVectorElementType() == MVT::i8 && "Unexpected element type!");
+    break;
+  }
+
+  return convertFromScalableVector(DAG, VT, Val);
+}
+
+SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
+                                                   SelectionDAG &DAG,
+                                                   unsigned NewOp) const {
+  EVT VT = Op.getValueType();
+  SDLoc DL(Op);
+  auto Pg = getPredicateForVector(DAG, DL, VT);
+
+  if (useSVEForFixedLengthVectorVT(VT)) {
+    EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
+
+    // Create list of operands by convereting existing ones to scalable types.
+    SmallVector<SDValue, 4> Operands = {Pg};
+    for (const SDValue &V : Op->op_values()) {
+      if (isa<CondCodeSDNode>(V)) {
+        Operands.push_back(V);
+        continue;
+      }
+
+      assert(useSVEForFixedLengthVectorVT(V.getValueType()) &&
+             "Only fixed length vectors are supported!");
+      Operands.push_back(convertToScalableVector(DAG, ContainerVT, V));
+    }
+
+    auto ScalableRes = DAG.getNode(NewOp, DL, ContainerVT, Operands);
+    return convertFromScalableVector(DAG, VT, ScalableRes);
+  }
+
+  assert(VT.isScalableVector() && "Only expect to lower scalable vector op!");
+
+  SmallVector<SDValue, 4> Operands = {Pg};
+  for (const SDValue &V : Op->op_values()) {
+    assert((isa<CondCodeSDNode>(V) || V.getValueType().isScalableVector()) &&
+           "Only scalable vectors are supported!");
+    Operands.push_back(V);
+  }
+
+  return DAG.getNode(NewOp, DL, VT, Operands);
+}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 3baae53312fa4..4fe77481706b3 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -25,6 +25,26 @@ namespace llvm {
 
 namespace AArch64ISD {
 
+// For predicated nodes where the result is a vector, the operation is
+// controlled by a governing predicate and the inactive lanes are explicitly
+// defined with a value, please stick the following naming convention:
+//
+//    _MERGE_OP<n>        The result value is a vector with inactive lanes equal
+//                        to source operand OP<n>.
+//
+//    _MERGE_ZERO         The result value is a vector with inactive lanes
+//                        actively zeroed.
+//
+//    _MERGE_PASSTHRU     The result value is a vector with inactive lanes equal
+//                        to the last source operand which only purpose is being
+//                        a passthru value.
+//
+// For other cases where no explicit action is needed to set the inactive lanes,
+// or when the result is not a vector and it is needed or helpful to
+// distinguish a node from similar unpredicated nodes, use:
+//
+//    _PRED
+//
 enum NodeType : unsigned {
   FIRST_NUMBER = ISD::BUILTIN_OP_END,
   WrapperLarge, // 4-instruction MOVZ/MOVK sequence for 64-bit addresses.
@@ -53,16 +73,20 @@ enum NodeType : unsigned {
   SBC, // adc, sbc instructions
 
   // Arithmetic instructions
+  ADD_PRED,
+  FADD_PRED,
   SDIV_PRED,
   UDIV_PRED,
-  SMIN_PRED,
-  UMIN_PRED,
-  SMAX_PRED,
-  UMAX_PRED,
-  SHL_PRED,
-  SRL_PRED,
-  SRA_PRED,
-  SETCC_PRED,
+  FMA_PRED,
+  SMIN_MERGE_OP1,
+  UMIN_MERGE_OP1,
+  SMAX_MERGE_OP1,
+  UMAX_MERGE_OP1,
+  SHL_MERGE_OP1,
+  SRL_MERGE_OP1,
+  SRA_MERGE_OP1,
+
+  SETCC_MERGE_ZERO,
 
   // Arithmetic instructions which write flags.
   ADDS,
@@ -164,6 +188,10 @@ enum NodeType : unsigned {
   SADDV,
   UADDV,
 
+  // Vector rounding halving addition
+  SRHADD,
+  URHADD,
+
   // Vector across-lanes min/max
   // Only the lower result lane is defined.
   SMINV,
@@ -241,80 +269,81 @@ enum NodeType : unsigned {
   PTEST,
   PTRUE,
 
-  DUP_PRED,
+  DUP_MERGE_PASSTHRU,
   INDEX_VECTOR,
 
   REINTERPRET_CAST,
 
-  LD1,
-  LD1S,
-  LDNF1,
-  LDNF1S,
-  LDFF1,
-  LDFF1S,
-  LD1RQ,
-  LD1RO,
+  LD1_MERGE_ZERO,
+  LD1S_MERGE_ZERO,
+  LDNF1_MERGE_ZERO,
+  LDNF1S_MERGE_ZERO,
+  LDFF1_MERGE_ZERO,
+  LDFF1S_MERGE_ZERO,
+  LD1RQ_MERGE_ZERO,
+  LD1RO_MERGE_ZERO,
 
   // Structured loads.
-  SVE_LD2,
-  SVE_LD3,
-  SVE_LD4,
+  SVE_LD2_MERGE_ZERO,
+  SVE_LD3_MERGE_ZERO,
+  SVE_LD4_MERGE_ZERO,
 
   // Unsigned gather loads.
-  GLD1,
-  GLD1_SCALED,
-  GLD1_UXTW,
-  GLD1_SXTW,
-  GLD1_UXTW_SCALED,
-  GLD1_SXTW_SCALED,
-  GLD1_IMM,
+  GLD1_MERGE_ZERO,
+  GLD1_SCALED_MERGE_ZERO,
+  GLD1_UXTW_MERGE_ZERO,
+  GLD1_SXTW_MERGE_ZERO,
+  GLD1_UXTW_SCALED_MERGE_ZERO,
+  GLD1_SXTW_SCALED_MERGE_ZERO,
+  GLD1_IMM_MERGE_ZERO,
 
   // Signed gather loads
-  GLD1S,
-  GLD1S_SCALED,
-  GLD1S_UXTW,
-  GLD1S_SXTW,
-  GLD1S_UXTW_SCALED,
-  GLD1S_SXTW_SCALED,
-  GLD1S_IMM,
+  GLD1S_MERGE_ZERO,
+  GLD1S_SCALED_MERGE_ZERO,
+  GLD1S_UXTW_MERGE_ZERO,
+  GLD1S_SXTW_MERGE_ZERO,
+  GLD1S_UXTW_SCALED_MERGE_ZERO,
+  GLD1S_SXTW_SCALED_MERGE_ZERO,
+  GLD1S_IMM_MERGE_ZERO,
 
   // Unsigned gather loads.
-  GLDFF1,
-  GLDFF1_SCALED,
-  GLDFF1_UXTW,
-  GLDFF1_SXTW,
-  GLDFF1_UXTW_SCALED,
-  GLDFF1_SXTW_SCALED,
-  GLDFF1_IMM,
+  GLDFF1_MERGE_ZERO,
+  GLDFF1_SCALED_MERGE_ZERO,
+  GLDFF1_UXTW_MERGE_ZERO,
+  GLDFF1_SXTW_MERGE_ZERO,
+  GLDFF1_UXTW_SCALED_MERGE_ZERO,
+  GLDFF1_SXTW_SCALED_MERGE_ZERO,
+  GLDFF1_IMM_MERGE_ZERO,
 
   // Signed gather loads.
-  GLDFF1S,
-  GLDFF1S_SCALED,
-  GLDFF1S_UXTW,
-  GLDFF1S_SXTW,
-  GLDFF1S_UXTW_SCALED,
-  GLDFF1S_SXTW_SCALED,
-  GLDFF1S_IMM,
+  GLDFF1S_MERGE_ZERO,
+  GLDFF1S_SCALED_MERGE_ZERO,
+  GLDFF1S_UXTW_MERGE_ZERO,
+  GLDFF1S_SXTW_MERGE_ZERO,
+  GLDFF1S_UXTW_SCALED_MERGE_ZERO,
+  GLDFF1S_SXTW_SCALED_MERGE_ZERO,
+  GLDFF1S_IMM_MERGE_ZERO,
 
   // Non-temporal gather loads
-  GLDNT1,
-  GLDNT1_INDEX,
-  GLDNT1S,
+  GLDNT1_MERGE_ZERO,
+  GLDNT1_INDEX_MERGE_ZERO,
+  GLDNT1S_MERGE_ZERO,
 
-  ST1,
+  // Contiguous masked store.
+  ST1_PRED,
 
   // Scatter store
-  SST1,
-  SST1_SCALED,
-  SST1_UXTW,
-  SST1_SXTW,
-  SST1_UXTW_SCALED,
-  SST1_SXTW_SCALED,
-  SST1_IMM,
+  SST1_PRED,
+  SST1_SCALED_PRED,
+  SST1_UXTW_PRED,
+  SST1_SXTW_PRED,
+  SST1_UXTW_SCALED_PRED,
+  SST1_SXTW_SCALED_PRED,
+  SST1_IMM_PRED,
 
   // Non-temporal scatter store
-  SSTNT1,
-  SSTNT1_INDEX,
+  SSTNT1_PRED,
+  SSTNT1_INDEX_PRED,
 
   // Strict (exception-raising) floating point comparison
   STRICT_FCMP = ISD::FIRST_TARGET_STRICTFP_OPCODE,
@@ -401,7 +430,8 @@ class AArch64TargetLowering : public TargetLowering {
     return MVT::getIntegerVT(64);
   }
 
-  bool targetShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
+  bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,
+                                    const APInt &DemandedElts,
                                     TargetLoweringOpt &TLO) const override;
 
   MVT getScalarShiftAmountTy(const DataLayout &DL, EVT) const override;
@@ -480,7 +510,7 @@ class AArch64TargetLowering : public TargetLowering {
   bool shouldSinkOperands(Instruction *I,
                           SmallVectorImpl<Use *> &Ops) const override;
 
-  bool hasPairedLoad(EVT LoadedType, unsigned &RequiredAligment) const override;
+  bool hasPairedLoad(EVT LoadedType, Align &RequiredAligment) const override;
 
   unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
 
@@ -705,6 +735,16 @@ class AArch64TargetLowering : public TargetLowering {
 
   bool fallBackToDAGISel(const Instruction &Inst) const override;
 
+  /// SVE code generation for fixed length vectors does not custom lower
+  /// BUILD_VECTOR. This makes BUILD_VECTOR legalisation a source of stores to
+  /// merge. However, merging them creates a BUILD_VECTOR that is just as
+  /// illegal as the original, thus leading to an infinite legalisation loop.
+  /// NOTE: Once BUILD_VECTOR is legal or can be custom lowered for all legal
+  /// vector types this override can be removed.
+  bool mergeStoresAfterLegalization(EVT VT) const override {
+    return !useSVEForFixedLengthVectors();
+  }
+
 private:
   /// Keep a pointer to the AArch64Subtarget around so that we can
   /// make the right decision when generating code for different targets.
@@ -713,6 +753,7 @@ class AArch64TargetLowering : public TargetLowering {
   bool isExtFreeImpl(const Instruction *Ext) const override;
 
   void addTypeForNEON(MVT VT, MVT PromotedBitwiseVT);
+  void addTypeForFixedLengthSVE(MVT VT);
   void addDRTypeForNEON(MVT VT);
   void addQRTypeForNEON(MVT VT);
 
@@ -820,6 +861,7 @@ class AArch64TargetLowering : public TargetLowering {
   SDValue LowerToPredicatedOp(SDValue Op, SelectionDAG &DAG,
                               unsigned NewOp) const;
   SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVectorSRA_SRL_SHL(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerShiftRightParts(SDValue Op, SelectionDAG &DAG) const;
@@ -837,6 +879,7 @@ class AArch64TargetLowering : public TargetLowering {
   SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVSCALE(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerATOMIC_LOAD_AND(SDValue Op, SelectionDAG &DAG) const;
@@ -847,6 +890,11 @@ class AArch64TargetLowering : public TargetLowering {
   SDValue LowerSVEStructLoad(unsigned Intrinsic, ArrayRef<SDValue> LoadOps,
                              EVT VT, SelectionDAG &DAG, const SDLoc &DL) const;
 
+  SDValue LowerFixedLengthVectorLoadToSVE(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerFixedLengthVectorStoreToSVE(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerFixedLengthVectorTruncateToSVE(SDValue Op,
+                                              SelectionDAG &DAG) const;
+
   SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
                         SmallVectorImpl<SDNode *> &Created) const override;
   SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
@@ -912,7 +960,7 @@ class AArch64TargetLowering : public TargetLowering {
                       const TargetTransformInfo *TTI) const override;
 
   bool useSVEForFixedLengthVectors() const;
-  bool useSVEForFixedLengthVectorVT(MVT VT) const;
+  bool useSVEForFixedLengthVectorVT(EVT VT) const;
 };
 
 namespace AArch64 {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 8f5202af96e4d..6df7970f4d82b 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -1647,6 +1647,8 @@ class BaseAuthLoad<bit M, bit W, dag oops, dag iops, string asm,
   let Inst{10} = 1;
   let Inst{9-5} = Rn;
   let Inst{4-0} = Rt;
+
+  let DecoderMethod = "DecodeAuthLoadInstruction";
 }
 
 multiclass AuthLoad<bit M, string asm, Operand opr> {
@@ -7911,15 +7913,18 @@ class SIMDThreeSameVectorBF16MatrixMul<string asm>
 class SIMD_BFCVTN
   : BaseSIMDMixedTwoVector<0, 0, 0b10, 0b10110, V128, V128,
                            "bfcvtn", ".4h", ".4s",
-    []>;
+    [(set (v8bf16 V128:$Rd),
+          (int_aarch64_neon_bfcvtn (v4f32 V128:$Rn)))]>;
 
 class SIMD_BFCVTN2
   : BaseSIMDMixedTwoVectorTied<1, 0, 0b10, 0b10110, V128, V128,
                            "bfcvtn2", ".8h", ".4s",
-    []>;
+    [(set (v8bf16 V128:$dst),
+          (int_aarch64_neon_bfcvtn2 (v8bf16 V128:$Rd), (v4f32 V128:$Rn)))]>;
 
 class BF16ToSinglePrecision<string asm>
-  : I<(outs FPR16:$Rd), (ins FPR32:$Rn), asm, "\t$Rd, $Rn", "", []>,
+  : I<(outs FPR16:$Rd), (ins FPR32:$Rn), asm, "\t$Rd, $Rn", "",
+    [(set (bf16 FPR16:$Rd), (int_aarch64_neon_bfcvt (f32 FPR32:$Rn)))]>,
     Sched<[WriteFCvt]> {
   bits<5> Rd;
   bits<5> Rn;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index c71e345266eb6..298c04d81708d 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -442,7 +442,7 @@ enum FalseLaneType {
 
 int getSVEPseudoMap(uint16_t Opcode);
 int getSVERevInstr(uint16_t Opcode);
-int getSVEOrigInstr(uint16_t Opcode);
+int getSVENonRevInstr(uint16_t Opcode);
 }
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index b56c5d9ff851f..f4a5f639e4973 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -155,6 +155,8 @@ def HasMatMulFP64    : Predicate<"Subtarget->hasMatMulFP64()">,
 def IsLE             : Predicate<"Subtarget->isLittleEndian()">;
 def IsBE             : Predicate<"!Subtarget->isLittleEndian()">;
 def IsWindows        : Predicate<"Subtarget->isTargetWindows()">;
+def UseExperimentalZeroingPseudos
+    : Predicate<"Subtarget->useExperimentalZeroingPseudos()">;
 def UseAlternateSExtLoadCVTF32
     : Predicate<"Subtarget->useAlternateSExtLoadCVTF32Pattern()">;
 
@@ -552,6 +554,9 @@ def AArch64uminv    : SDNode<"AArch64ISD::UMINV", SDT_AArch64UnaryVec>;
 def AArch64smaxv    : SDNode<"AArch64ISD::SMAXV", SDT_AArch64UnaryVec>;
 def AArch64umaxv    : SDNode<"AArch64ISD::UMAXV", SDT_AArch64UnaryVec>;
 
+def AArch64srhadd   : SDNode<"AArch64ISD::SRHADD", SDT_AArch64binvec>;
+def AArch64urhadd   : SDNode<"AArch64ISD::URHADD", SDT_AArch64binvec>;
+
 def SDT_AArch64SETTAG : SDTypeProfile<0, 2, [SDTCisPtrTy<0>, SDTCisPtrTy<1>]>;
 def AArch64stg : SDNode<"AArch64ISD::STG", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 def AArch64stzg : SDNode<"AArch64ISD::STZG", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
@@ -3955,12 +3960,16 @@ defm URSQRTE: SIMDTwoVectorS<1, 1, 0b11100, "ursqrte", int_aarch64_neon_ursqrte>
 defm USQADD : SIMDTwoVectorBHSDTied<1, 0b00011, "usqadd",int_aarch64_neon_usqadd>;
 defm XTN    : SIMDMixedTwoVector<0, 0b10010, "xtn", trunc>;
 
-def : Pat<(v4f16 (AArch64rev32 V64:$Rn)), (REV32v4i16 V64:$Rn)>;
-def : Pat<(v4f16 (AArch64rev64 V64:$Rn)), (REV64v4i16 V64:$Rn)>;
-def : Pat<(v8f16 (AArch64rev32 V128:$Rn)), (REV32v8i16 V128:$Rn)>;
-def : Pat<(v8f16 (AArch64rev64 V128:$Rn)), (REV64v8i16 V128:$Rn)>;
-def : Pat<(v2f32 (AArch64rev64 V64:$Rn)), (REV64v2i32 V64:$Rn)>;
-def : Pat<(v4f32 (AArch64rev64 V128:$Rn)), (REV64v4i32 V128:$Rn)>;
+def : Pat<(v4f16  (AArch64rev32 V64:$Rn)),  (REV32v4i16 V64:$Rn)>;
+def : Pat<(v4f16  (AArch64rev64 V64:$Rn)),  (REV64v4i16 V64:$Rn)>;
+def : Pat<(v4bf16 (AArch64rev32 V64:$Rn)),  (REV32v4i16 V64:$Rn)>;
+def : Pat<(v4bf16 (AArch64rev64 V64:$Rn)),  (REV64v4i16 V64:$Rn)>;
+def : Pat<(v8f16  (AArch64rev32 V128:$Rn)), (REV32v8i16 V128:$Rn)>;
+def : Pat<(v8f16  (AArch64rev64 V128:$Rn)), (REV64v8i16 V128:$Rn)>;
+def : Pat<(v8bf16 (AArch64rev32 V128:$Rn)), (REV32v8i16 V128:$Rn)>;
+def : Pat<(v8bf16 (AArch64rev64 V128:$Rn)), (REV64v8i16 V128:$Rn)>;
+def : Pat<(v2f32  (AArch64rev64 V64:$Rn)),  (REV64v2i32 V64:$Rn)>;
+def : Pat<(v4f32  (AArch64rev64 V128:$Rn)), (REV64v4i32 V128:$Rn)>;
 
 // Patterns for vector long shift (by element width). These need to match all
 // three of zext, sext and anyext so it's easier to pull the patterns out of the
@@ -4067,7 +4076,7 @@ defm SQRDMULH : SIMDThreeSameVectorHS<1,0b10110,"sqrdmulh",int_aarch64_neon_sqrd
 defm SQRSHL   : SIMDThreeSameVector<0,0b01011,"sqrshl", int_aarch64_neon_sqrshl>;
 defm SQSHL    : SIMDThreeSameVector<0,0b01001,"sqshl", int_aarch64_neon_sqshl>;
 defm SQSUB    : SIMDThreeSameVector<0,0b00101,"sqsub", int_aarch64_neon_sqsub>;
-defm SRHADD   : SIMDThreeSameVectorBHS<0,0b00010,"srhadd",int_aarch64_neon_srhadd>;
+defm SRHADD   : SIMDThreeSameVectorBHS<0,0b00010,"srhadd", AArch64srhadd>;
 defm SRSHL    : SIMDThreeSameVector<0,0b01010,"srshl", int_aarch64_neon_srshl>;
 defm SSHL     : SIMDThreeSameVector<0,0b01000,"sshl", int_aarch64_neon_sshl>;
 defm SUB      : SIMDThreeSameVector<1,0b10000,"sub", sub>;
@@ -4084,7 +4093,7 @@ defm UQADD    : SIMDThreeSameVector<1,0b00001,"uqadd", int_aarch64_neon_uqadd>;
 defm UQRSHL   : SIMDThreeSameVector<1,0b01011,"uqrshl", int_aarch64_neon_uqrshl>;
 defm UQSHL    : SIMDThreeSameVector<1,0b01001,"uqshl", int_aarch64_neon_uqshl>;
 defm UQSUB    : SIMDThreeSameVector<1,0b00101,"uqsub", int_aarch64_neon_uqsub>;
-defm URHADD   : SIMDThreeSameVectorBHS<1,0b00010,"urhadd", int_aarch64_neon_urhadd>;
+defm URHADD   : SIMDThreeSameVectorBHS<1,0b00010,"urhadd", AArch64urhadd>;
 defm URSHL    : SIMDThreeSameVector<1,0b01010,"urshl", int_aarch64_neon_urshl>;
 defm USHL     : SIMDThreeSameVector<1,0b01000,"ushl", int_aarch64_neon_ushl>;
 defm SQRDMLAH : SIMDThreeSameVectorSQRDMLxHTiedHS<1,0b10000,"sqrdmlah",
diff --git a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
index 0baf22a51c92b..cb4dc8462f68d 100644
--- a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
@@ -403,12 +403,6 @@ class AArch64IndirectThunks : public MachineFunctionPass {
   bool doInitialization(Module &M) override;
   bool runOnMachineFunction(MachineFunction &MF) override;
 
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    MachineFunctionPass::getAnalysisUsage(AU);
-    AU.addRequired<MachineModuleInfoWrapperPass>();
-    AU.addPreserved<MachineModuleInfoWrapperPass>();
-  }
-
 private:
   std::tuple<SLSBLRThunkInserter> TIs;
 
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 9bab0e4d33e24..28a54e6f7d79f 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -10,6 +10,21 @@
 //
 //===----------------------------------------------------------------------===//
 
+// For predicated nodes where the entire operation is controlled by a governing
+// predicate, please stick to a similar naming convention as used for the
+// ISD nodes:
+//
+//    SDNode      <=>     AArch64ISD
+//    -------------------------------
+//    _m<n>       <=>     _MERGE_OP<n>
+//    _mt         <=>     _MERGE_PASSTHRU
+//    _z          <=>     _MERGE_ZERO
+//    _p          <=>     _PRED
+//
+//  Given the context of this file, it is not strictly necessary to use _p to
+//  distinguish predicated from unpredicated nodes given that most SVE
+//  instructions are predicated.
+
 // Contiguous loads - node definitions
 //
 def SDT_AArch64_LD1 : SDTypeProfile<1, 3, [
@@ -17,16 +32,16 @@ def SDT_AArch64_LD1 : SDTypeProfile<1, 3, [
   SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>
 ]>;
 
-def AArch64ld1  : SDNode<"AArch64ISD::LD1",    SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;
-def AArch64ld1s : SDNode<"AArch64ISD::LD1S",   SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;
+def AArch64ld1_z  : SDNode<"AArch64ISD::LD1_MERGE_ZERO",    SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;
+def AArch64ld1s_z : SDNode<"AArch64ISD::LD1S_MERGE_ZERO",   SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;
 
 // Non-faulting & first-faulting loads - node definitions
 //
-def AArch64ldnf1 : SDNode<"AArch64ISD::LDNF1", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1 : SDNode<"AArch64ISD::LDFF1", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldnf1_z : SDNode<"AArch64ISD::LDNF1_MERGE_ZERO", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1_z : SDNode<"AArch64ISD::LDFF1_MERGE_ZERO", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
 
-def AArch64ldnf1s : SDNode<"AArch64ISD::LDNF1S", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1s : SDNode<"AArch64ISD::LDFF1S", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldnf1s_z : SDNode<"AArch64ISD::LDNF1S_MERGE_ZERO", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1s_z : SDNode<"AArch64ISD::LDFF1S_MERGE_ZERO", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
 
 // Contiguous load and replicate - node definitions
 //
@@ -36,8 +51,8 @@ def SDT_AArch64_LD1Replicate : SDTypeProfile<1, 2, [
   SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>
 ]>;
 
-def AArch64ld1rq : SDNode<"AArch64ISD::LD1RQ",  SDT_AArch64_LD1Replicate, [SDNPHasChain, SDNPMayLoad]>;
-def AArch64ld1ro : SDNode<"AArch64ISD::LD1RO",  SDT_AArch64_LD1Replicate, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1rq_z : SDNode<"AArch64ISD::LD1RQ_MERGE_ZERO",  SDT_AArch64_LD1Replicate, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1ro_z : SDNode<"AArch64ISD::LD1RO_MERGE_ZERO",  SDT_AArch64_LD1Replicate, [SDNPHasChain, SDNPMayLoad]>;
 
 // Gather loads - node definitions
 //
@@ -51,40 +66,40 @@ def SDT_AArch64_GATHER_VS : SDTypeProfile<1, 4, [
   SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>
 ]>;
 
-def AArch64ld1_gather             : SDNode<"AArch64ISD::GLD1",             SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
-def AArch64ld1_gather_scaled      : SDNode<"AArch64ISD::GLD1_SCALED",      SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
-def AArch64ld1_gather_uxtw        : SDNode<"AArch64ISD::GLD1_UXTW",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
-def AArch64ld1_gather_sxtw        : SDNode<"AArch64ISD::GLD1_SXTW",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
-def AArch64ld1_gather_uxtw_scaled : SDNode<"AArch64ISD::GLD1_UXTW_SCALED", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
-def AArch64ld1_gather_sxtw_scaled : SDNode<"AArch64ISD::GLD1_SXTW_SCALED", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
-def AArch64ld1_gather_imm         : SDNode<"AArch64ISD::GLD1_IMM",         SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad]>;
-
-def AArch64ld1s_gather             : SDNode<"AArch64ISD::GLD1S",             SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
-def AArch64ld1s_gather_scaled      : SDNode<"AArch64ISD::GLD1S_SCALED",      SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
-def AArch64ld1s_gather_uxtw        : SDNode<"AArch64ISD::GLD1S_UXTW",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
-def AArch64ld1s_gather_sxtw        : SDNode<"AArch64ISD::GLD1S_SXTW",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
-def AArch64ld1s_gather_uxtw_scaled : SDNode<"AArch64ISD::GLD1S_UXTW_SCALED", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
-def AArch64ld1s_gather_sxtw_scaled : SDNode<"AArch64ISD::GLD1S_SXTW_SCALED", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
-def AArch64ld1s_gather_imm         : SDNode<"AArch64ISD::GLD1S_IMM",         SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad]>;
-
-def AArch64ldff1_gather             : SDNode<"AArch64ISD::GLDFF1",             SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1_gather_scaled      : SDNode<"AArch64ISD::GLDFF1_SCALED",      SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1_gather_uxtw        : SDNode<"AArch64ISD::GLDFF1_UXTW",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1_gather_sxtw        : SDNode<"AArch64ISD::GLDFF1_SXTW",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1_gather_uxtw_scaled : SDNode<"AArch64ISD::GLDFF1_UXTW_SCALED", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1_gather_sxtw_scaled : SDNode<"AArch64ISD::GLDFF1_SXTW_SCALED", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1_gather_imm         : SDNode<"AArch64ISD::GLDFF1_IMM",         SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-
-def AArch64ldff1s_gather             : SDNode<"AArch64ISD::GLDFF1S",             SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1s_gather_scaled      : SDNode<"AArch64ISD::GLDFF1S_SCALED",      SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1s_gather_uxtw        : SDNode<"AArch64ISD::GLDFF1S_UXTW",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1s_gather_sxtw        : SDNode<"AArch64ISD::GLDFF1S_SXTW",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1s_gather_uxtw_scaled : SDNode<"AArch64ISD::GLDFF1S_UXTW_SCALED", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1s_gather_sxtw_scaled : SDNode<"AArch64ISD::GLDFF1S_SXTW_SCALED", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1s_gather_imm         : SDNode<"AArch64ISD::GLDFF1S_IMM",         SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-
-def AArch64ldnt1_gather  : SDNode<"AArch64ISD::GLDNT1",  SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad]>;
-def AArch64ldnt1s_gather : SDNode<"AArch64ISD::GLDNT1S", SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1_gather_z             : SDNode<"AArch64ISD::GLD1_MERGE_ZERO",             SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1_gather_scaled_z      : SDNode<"AArch64ISD::GLD1_SCALED_MERGE_ZERO",      SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1_gather_uxtw_z        : SDNode<"AArch64ISD::GLD1_UXTW_MERGE_ZERO",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1_gather_sxtw_z        : SDNode<"AArch64ISD::GLD1_SXTW_MERGE_ZERO",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1_gather_uxtw_scaled_z : SDNode<"AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1_gather_sxtw_scaled_z : SDNode<"AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1_gather_imm_z         : SDNode<"AArch64ISD::GLD1_IMM_MERGE_ZERO",         SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad]>;
+
+def AArch64ld1s_gather_z             : SDNode<"AArch64ISD::GLD1S_MERGE_ZERO",             SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1s_gather_scaled_z      : SDNode<"AArch64ISD::GLD1S_SCALED_MERGE_ZERO",      SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1s_gather_uxtw_z        : SDNode<"AArch64ISD::GLD1S_UXTW_MERGE_ZERO",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1s_gather_sxtw_z        : SDNode<"AArch64ISD::GLD1S_SXTW_MERGE_ZERO",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1s_gather_uxtw_scaled_z : SDNode<"AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1s_gather_sxtw_scaled_z : SDNode<"AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1s_gather_imm_z         : SDNode<"AArch64ISD::GLD1S_IMM_MERGE_ZERO",         SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad]>;
+
+def AArch64ldff1_gather_z             : SDNode<"AArch64ISD::GLDFF1_MERGE_ZERO",             SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1_gather_scaled_z      : SDNode<"AArch64ISD::GLDFF1_SCALED_MERGE_ZERO",      SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1_gather_uxtw_z        : SDNode<"AArch64ISD::GLDFF1_UXTW_MERGE_ZERO",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1_gather_sxtw_z        : SDNode<"AArch64ISD::GLDFF1_SXTW_MERGE_ZERO",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1_gather_uxtw_scaled_z : SDNode<"AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1_gather_sxtw_scaled_z : SDNode<"AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1_gather_imm_z         : SDNode<"AArch64ISD::GLDFF1_IMM_MERGE_ZERO",         SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+
+def AArch64ldff1s_gather_z             : SDNode<"AArch64ISD::GLDFF1S_MERGE_ZERO",             SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1s_gather_scaled_z      : SDNode<"AArch64ISD::GLDFF1S_SCALED_MERGE_ZERO",      SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1s_gather_uxtw_z        : SDNode<"AArch64ISD::GLDFF1S_UXTW_MERGE_ZERO",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1s_gather_sxtw_z        : SDNode<"AArch64ISD::GLDFF1S_SXTW_MERGE_ZERO",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1s_gather_uxtw_scaled_z : SDNode<"AArch64ISD::GLDFF1S_UXTW_SCALED_MERGE_ZERO", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1s_gather_sxtw_scaled_z : SDNode<"AArch64ISD::GLDFF1S_SXTW_SCALED_MERGE_ZERO", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1s_gather_imm_z         : SDNode<"AArch64ISD::GLDFF1S_IMM_MERGE_ZERO",         SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+
+def AArch64ldnt1_gather_z  : SDNode<"AArch64ISD::GLDNT1_MERGE_ZERO",  SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ldnt1s_gather_z : SDNode<"AArch64ISD::GLDNT1S_MERGE_ZERO", SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad]>;
 
 // Contiguous stores - node definitions
 //
@@ -93,7 +108,7 @@ def SDT_AArch64_ST1 : SDTypeProfile<0, 4, [
   SDTCVecEltisVT<2,i1>, SDTCisSameNumEltsAs<0,2>
 ]>;
 
-def AArch64st1 : SDNode<"AArch64ISD::ST1", SDT_AArch64_ST1, [SDNPHasChain, SDNPMayStore]>;
+def AArch64st1 : SDNode<"AArch64ISD::ST1_PRED", SDT_AArch64_ST1, [SDNPHasChain, SDNPMayStore]>;
 
 // Scatter stores - node definitions
 //
@@ -107,15 +122,15 @@ def SDT_AArch64_SCATTER_VS : SDTypeProfile<0, 5, [
   SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>
 ]>;
 
-def AArch64st1_scatter             : SDNode<"AArch64ISD::SST1",             SDT_AArch64_SCATTER_SV, [SDNPHasChain, SDNPMayStore]>;
-def AArch64st1_scatter_scaled      : SDNode<"AArch64ISD::SST1_SCALED",      SDT_AArch64_SCATTER_SV, [SDNPHasChain, SDNPMayStore]>;
-def AArch64st1_scatter_uxtw        : SDNode<"AArch64ISD::SST1_UXTW",        SDT_AArch64_SCATTER_SV, [SDNPHasChain, SDNPMayStore]>;
-def AArch64st1_scatter_sxtw        : SDNode<"AArch64ISD::SST1_SXTW",        SDT_AArch64_SCATTER_SV, [SDNPHasChain, SDNPMayStore]>;
-def AArch64st1_scatter_uxtw_scaled : SDNode<"AArch64ISD::SST1_UXTW_SCALED", SDT_AArch64_SCATTER_SV, [SDNPHasChain, SDNPMayStore]>;
-def AArch64st1_scatter_sxtw_scaled : SDNode<"AArch64ISD::SST1_SXTW_SCALED", SDT_AArch64_SCATTER_SV, [SDNPHasChain, SDNPMayStore]>;
-def AArch64st1_scatter_imm         : SDNode<"AArch64ISD::SST1_IMM",         SDT_AArch64_SCATTER_VS, [SDNPHasChain, SDNPMayStore]>;
+def AArch64st1_scatter             : SDNode<"AArch64ISD::SST1_PRED",             SDT_AArch64_SCATTER_SV, [SDNPHasChain, SDNPMayStore]>;
+def AArch64st1_scatter_scaled      : SDNode<"AArch64ISD::SST1_SCALED_PRED",      SDT_AArch64_SCATTER_SV, [SDNPHasChain, SDNPMayStore]>;
+def AArch64st1_scatter_uxtw        : SDNode<"AArch64ISD::SST1_UXTW_PRED",        SDT_AArch64_SCATTER_SV, [SDNPHasChain, SDNPMayStore]>;
+def AArch64st1_scatter_sxtw        : SDNode<"AArch64ISD::SST1_SXTW_PRED",        SDT_AArch64_SCATTER_SV, [SDNPHasChain, SDNPMayStore]>;
+def AArch64st1_scatter_uxtw_scaled : SDNode<"AArch64ISD::SST1_UXTW_SCALED_PRED", SDT_AArch64_SCATTER_SV, [SDNPHasChain, SDNPMayStore]>;
+def AArch64st1_scatter_sxtw_scaled : SDNode<"AArch64ISD::SST1_SXTW_SCALED_PRED", SDT_AArch64_SCATTER_SV, [SDNPHasChain, SDNPMayStore]>;
+def AArch64st1_scatter_imm         : SDNode<"AArch64ISD::SST1_IMM_PRED",         SDT_AArch64_SCATTER_VS, [SDNPHasChain, SDNPMayStore]>;
 
-def AArch64stnt1_scatter : SDNode<"AArch64ISD::SSTNT1", SDT_AArch64_SCATTER_VS, [SDNPHasChain, SDNPMayStore]>;
+def AArch64stnt1_scatter : SDNode<"AArch64ISD::SSTNT1_PRED", SDT_AArch64_SCATTER_VS, [SDNPHasChain, SDNPMayStore]>;
 
 // AArch64 SVE/SVE2 - the remaining node definitions
 //
@@ -132,40 +147,51 @@ def sve_cntw_imm_neg : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, -4>">;
 def sve_cntd_imm_neg : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, -2>">;
 
 def SDT_AArch64Reduce : SDTypeProfile<1, 2, [SDTCisVec<1>, SDTCisVec<2>]>;
-def AArch64faddv_pred   : SDNode<"AArch64ISD::FADDV_PRED",   SDT_AArch64Reduce>;
-def AArch64fmaxv_pred   : SDNode<"AArch64ISD::FMAXV_PRED",   SDT_AArch64Reduce>;
-def AArch64fmaxnmv_pred : SDNode<"AArch64ISD::FMAXNMV_PRED", SDT_AArch64Reduce>;
-def AArch64fminv_pred   : SDNode<"AArch64ISD::FMINV_PRED",   SDT_AArch64Reduce>;
-def AArch64fminnmv_pred : SDNode<"AArch64ISD::FMINNMV_PRED", SDT_AArch64Reduce>;
-def AArch64smaxv_pred   : SDNode<"AArch64ISD::SMAXV_PRED",   SDT_AArch64Reduce>;
-def AArch64umaxv_pred   : SDNode<"AArch64ISD::UMAXV_PRED",   SDT_AArch64Reduce>;
-def AArch64sminv_pred   : SDNode<"AArch64ISD::SMINV_PRED",   SDT_AArch64Reduce>;
-def AArch64uminv_pred   : SDNode<"AArch64ISD::UMINV_PRED",   SDT_AArch64Reduce>;
-def AArch64orv_pred     : SDNode<"AArch64ISD::ORV_PRED",     SDT_AArch64Reduce>;
-def AArch64eorv_pred    : SDNode<"AArch64ISD::EORV_PRED",    SDT_AArch64Reduce>;
-def AArch64andv_pred    : SDNode<"AArch64ISD::ANDV_PRED",    SDT_AArch64Reduce>;
-def AArch64lasta        : SDNode<"AArch64ISD::LASTA",        SDT_AArch64Reduce>;
-def AArch64lastb        : SDNode<"AArch64ISD::LASTB",        SDT_AArch64Reduce>;
+def AArch64faddv_p   : SDNode<"AArch64ISD::FADDV_PRED",   SDT_AArch64Reduce>;
+def AArch64fmaxv_p   : SDNode<"AArch64ISD::FMAXV_PRED",   SDT_AArch64Reduce>;
+def AArch64fmaxnmv_p : SDNode<"AArch64ISD::FMAXNMV_PRED", SDT_AArch64Reduce>;
+def AArch64fminv_p   : SDNode<"AArch64ISD::FMINV_PRED",   SDT_AArch64Reduce>;
+def AArch64fminnmv_p : SDNode<"AArch64ISD::FMINNMV_PRED", SDT_AArch64Reduce>;
+def AArch64smaxv_p   : SDNode<"AArch64ISD::SMAXV_PRED",   SDT_AArch64Reduce>;
+def AArch64umaxv_p   : SDNode<"AArch64ISD::UMAXV_PRED",   SDT_AArch64Reduce>;
+def AArch64sminv_p   : SDNode<"AArch64ISD::SMINV_PRED",   SDT_AArch64Reduce>;
+def AArch64uminv_p   : SDNode<"AArch64ISD::UMINV_PRED",   SDT_AArch64Reduce>;
+def AArch64orv_p     : SDNode<"AArch64ISD::ORV_PRED",     SDT_AArch64Reduce>;
+def AArch64eorv_p    : SDNode<"AArch64ISD::EORV_PRED",    SDT_AArch64Reduce>;
+def AArch64andv_p    : SDNode<"AArch64ISD::ANDV_PRED",    SDT_AArch64Reduce>;
+def AArch64lasta     : SDNode<"AArch64ISD::LASTA",        SDT_AArch64Reduce>;
+def AArch64lastb     : SDNode<"AArch64ISD::LASTB",        SDT_AArch64Reduce>;
 
 def SDT_AArch64Arith : SDTypeProfile<1, 3, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>,
   SDTCVecEltisVT<1,i1>, SDTCisSameAs<2,3>
 ]>;
 
-def AArch64sdiv_pred :  SDNode<"AArch64ISD::SDIV_PRED", SDT_AArch64Arith>;
-def AArch64udiv_pred :  SDNode<"AArch64ISD::UDIV_PRED", SDT_AArch64Arith>;
-def AArch64smin_pred :  SDNode<"AArch64ISD::SMIN_PRED", SDT_AArch64Arith>;
-def AArch64umin_pred :  SDNode<"AArch64ISD::UMIN_PRED", SDT_AArch64Arith>;
-def AArch64smax_pred :  SDNode<"AArch64ISD::SMAX_PRED", SDT_AArch64Arith>;
-def AArch64umax_pred :  SDNode<"AArch64ISD::UMAX_PRED", SDT_AArch64Arith>;
-def AArch64lsl_pred  :  SDNode<"AArch64ISD::SHL_PRED",  SDT_AArch64Arith>;
-def AArch64lsr_pred  :  SDNode<"AArch64ISD::SRL_PRED",  SDT_AArch64Arith>;
-def AArch64asr_pred  :  SDNode<"AArch64ISD::SRA_PRED",  SDT_AArch64Arith>;
+def SDT_AArch64FMA : SDTypeProfile<1, 4, [
+  SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>, SDTCisVec<4>,
+  SDTCVecEltisVT<1,i1>, SDTCisSameAs<2,3>, SDTCisSameAs<3,4>
+]>;
+
+// Predicated operations with the result of inactive lanes being unspecified.
+def AArch64add_p  : SDNode<"AArch64ISD::ADD_PRED",  SDT_AArch64Arith>;
+def AArch64fadd_p : SDNode<"AArch64ISD::FADD_PRED", SDT_AArch64Arith>;
+def AArch64fma_p  : SDNode<"AArch64ISD::FMA_PRED",  SDT_AArch64FMA>;
+def AArch64sdiv_p : SDNode<"AArch64ISD::SDIV_PRED", SDT_AArch64Arith>;
+def AArch64udiv_p : SDNode<"AArch64ISD::UDIV_PRED", SDT_AArch64Arith>;
+
+// Merging op1 into the inactive lanes.
+def AArch64smin_m1 :  SDNode<"AArch64ISD::SMIN_MERGE_OP1", SDT_AArch64Arith>;
+def AArch64umin_m1 :  SDNode<"AArch64ISD::UMIN_MERGE_OP1", SDT_AArch64Arith>;
+def AArch64smax_m1 :  SDNode<"AArch64ISD::SMAX_MERGE_OP1", SDT_AArch64Arith>;
+def AArch64umax_m1 :  SDNode<"AArch64ISD::UMAX_MERGE_OP1", SDT_AArch64Arith>;
+def AArch64lsl_m1  :  SDNode<"AArch64ISD::SHL_MERGE_OP1",  SDT_AArch64Arith>;
+def AArch64lsr_m1  :  SDNode<"AArch64ISD::SRL_MERGE_OP1",  SDT_AArch64Arith>;
+def AArch64asr_m1  :  SDNode<"AArch64ISD::SRA_MERGE_OP1",  SDT_AArch64Arith>;
 
 def SDT_AArch64ReduceWithInit : SDTypeProfile<1, 3, [SDTCisVec<1>, SDTCisVec<3>]>;
 def AArch64clasta_n   : SDNode<"AArch64ISD::CLASTA_N",   SDT_AArch64ReduceWithInit>;
 def AArch64clastb_n   : SDNode<"AArch64ISD::CLASTB_N",   SDT_AArch64ReduceWithInit>;
-def AArch64fadda_pred : SDNode<"AArch64ISD::FADDA_PRED", SDT_AArch64ReduceWithInit>;
+def AArch64fadda_p    : SDNode<"AArch64ISD::FADDA_PRED", SDT_AArch64ReduceWithInit>;
 
 def SDT_AArch64Rev   : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisSameAs<0,1>]>;
 def AArch64rev       : SDNode<"AArch64ISD::REV", SDT_AArch64Rev>;
@@ -173,8 +199,8 @@ def AArch64rev       : SDNode<"AArch64ISD::REV", SDT_AArch64Rev>;
 def SDT_AArch64PTest : SDTypeProfile<0, 2, [SDTCisVec<0>, SDTCisSameAs<0,1>]>;
 def AArch64ptest     : SDNode<"AArch64ISD::PTEST", SDT_AArch64PTest>;
 
-def SDT_AArch64DUP_PRED  : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0,1>, SDTCisVec<2>, SDTCVecEltisVT<2,i1>]>;
-def AArch64dup_pred : SDNode<"AArch64ISD::DUP_PRED", SDT_AArch64DUP_PRED>;
+def SDT_AArch64DUP_PRED  : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0, 3>, SDTCisVec<1>, SDTCVecEltisVT<1,i1>]>;
+def AArch64dup_mt : SDNode<"AArch64ISD::DUP_MERGE_PASSTHRU", SDT_AArch64DUP_PRED>;
 
 def SDT_IndexVector : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<1, 2>, SDTCisInt<2>]>;
 def index_vector : SDNode<"AArch64ISD::INDEX_VECTOR", SDT_IndexVector, []>;
@@ -201,12 +227,16 @@ let Predicates = [HasSVE] in {
   defm BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic", null_frag>;
 
   defm ADD_ZPmZ  : sve_int_bin_pred_arit_0<0b000, "add",  "ADD_ZPZZ", int_aarch64_sve_add, DestructiveBinaryComm>;
-  defm SUB_ZPmZ  : sve_int_bin_pred_arit_0<0b001, "sub",  "SUB_ZPZZ", int_aarch64_sve_sub, DestructiveBinaryCommWithRev, "SUBR_ZPmZ", 1>;
-  defm SUBR_ZPmZ : sve_int_bin_pred_arit_0<0b011, "subr", "SUBR_ZPZZ", int_aarch64_sve_subr, DestructiveBinaryCommWithRev, "SUB_ZPmZ", 0>;
+  defm SUB_ZPmZ  : sve_int_bin_pred_arit_0<0b001, "sub",  "SUB_ZPZZ", int_aarch64_sve_sub, DestructiveBinaryCommWithRev, "SUBR_ZPmZ">;
+  defm SUBR_ZPmZ : sve_int_bin_pred_arit_0<0b011, "subr", "SUBR_ZPZZ", int_aarch64_sve_subr, DestructiveBinaryCommWithRev, "SUB_ZPmZ", /*isReverseInstr*/ 1>;
 
-  defm ADD_ZPZZ  : sve_int_bin_pred_zx<int_aarch64_sve_add>;
-  defm SUB_ZPZZ  : sve_int_bin_pred_zx<int_aarch64_sve_sub>;
-  defm SUBR_ZPZZ : sve_int_bin_pred_zx<int_aarch64_sve_subr>;
+  defm ADD_ZPZZ  : sve_int_bin_pred_bhsd<AArch64add_p>;
+
+  let Predicates = [HasSVE, UseExperimentalZeroingPseudos] in {
+    defm ADD_ZPZZ  : sve_int_bin_pred_zeroing_bhsd<int_aarch64_sve_add>;
+    defm SUB_ZPZZ  : sve_int_bin_pred_zeroing_bhsd<int_aarch64_sve_sub>;
+    defm SUBR_ZPZZ : sve_int_bin_pred_zeroing_bhsd<int_aarch64_sve_subr>;
+  }
 
   defm ORR_ZPmZ : sve_int_bin_pred_log<0b000, "orr", int_aarch64_sve_orr>;
   defm EOR_ZPmZ : sve_int_bin_pred_log<0b001, "eor", int_aarch64_sve_eor>;
@@ -229,22 +259,22 @@ let Predicates = [HasSVE] in {
   // SVE predicated integer reductions.
   defm SADDV_VPZ : sve_int_reduce_0_saddv<0b000, "saddv", int_aarch64_sve_saddv>;
   defm UADDV_VPZ : sve_int_reduce_0_uaddv<0b001, "uaddv", int_aarch64_sve_uaddv, int_aarch64_sve_saddv>;
-  defm SMAXV_VPZ : sve_int_reduce_1<0b000, "smaxv", AArch64smaxv_pred>;
-  defm UMAXV_VPZ : sve_int_reduce_1<0b001, "umaxv", AArch64umaxv_pred>;
-  defm SMINV_VPZ : sve_int_reduce_1<0b010, "sminv", AArch64sminv_pred>;
-  defm UMINV_VPZ : sve_int_reduce_1<0b011, "uminv", AArch64uminv_pred>;
-  defm ORV_VPZ   : sve_int_reduce_2<0b000, "orv", AArch64orv_pred>;
-  defm EORV_VPZ  : sve_int_reduce_2<0b001, "eorv", AArch64eorv_pred>;
-  defm ANDV_VPZ  : sve_int_reduce_2<0b010, "andv", AArch64andv_pred>;
+  defm SMAXV_VPZ : sve_int_reduce_1<0b000, "smaxv", AArch64smaxv_p>;
+  defm UMAXV_VPZ : sve_int_reduce_1<0b001, "umaxv", AArch64umaxv_p>;
+  defm SMINV_VPZ : sve_int_reduce_1<0b010, "sminv", AArch64sminv_p>;
+  defm UMINV_VPZ : sve_int_reduce_1<0b011, "uminv", AArch64uminv_p>;
+  defm ORV_VPZ   : sve_int_reduce_2<0b000, "orv", AArch64orv_p>;
+  defm EORV_VPZ  : sve_int_reduce_2<0b001, "eorv", AArch64eorv_p>;
+  defm ANDV_VPZ  : sve_int_reduce_2<0b010, "andv", AArch64andv_p>;
 
   defm ORR_ZI : sve_int_log_imm<0b00, "orr", "orn", or>;
   defm EOR_ZI : sve_int_log_imm<0b01, "eor", "eon", xor>;
   defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>;
 
-  defm SMAX_ZI   : sve_int_arith_imm1<0b00, "smax", AArch64smax_pred>;
-  defm SMIN_ZI   : sve_int_arith_imm1<0b10, "smin", AArch64smin_pred>;
-  defm UMAX_ZI   : sve_int_arith_imm1_unsigned<0b01, "umax", AArch64umax_pred>;
-  defm UMIN_ZI   : sve_int_arith_imm1_unsigned<0b11, "umin", AArch64umin_pred>;
+  defm SMAX_ZI   : sve_int_arith_imm1<0b00, "smax", AArch64smax_m1>;
+  defm SMIN_ZI   : sve_int_arith_imm1<0b10, "smin", AArch64smin_m1>;
+  defm UMAX_ZI   : sve_int_arith_imm1_unsigned<0b01, "umax", AArch64umax_m1>;
+  defm UMIN_ZI   : sve_int_arith_imm1_unsigned<0b11, "umin", AArch64umin_m1>;
 
   defm MUL_ZI     : sve_int_arith_imm2<"mul", mul>;
   defm MUL_ZPmZ   : sve_int_bin_pred_arit_2<0b000, "mul",   int_aarch64_sve_mul>;
@@ -261,10 +291,13 @@ let Predicates = [HasSVE] in {
   def : Pat<(mul nxv2i64:$Op1, nxv2i64:$Op2),
             (MUL_ZPmZ_D (PTRUE_D 31), $Op1, $Op2)>;
 
-  defm SDIV_ZPmZ  : sve_int_bin_pred_arit_2_div<0b100, "sdiv",  AArch64sdiv_pred>;
-  defm UDIV_ZPmZ  : sve_int_bin_pred_arit_2_div<0b101, "udiv",  AArch64udiv_pred>;
-  defm SDIVR_ZPmZ : sve_int_bin_pred_arit_2_div<0b110, "sdivr", int_aarch64_sve_sdivr>;
-  defm UDIVR_ZPmZ : sve_int_bin_pred_arit_2_div<0b111, "udivr", int_aarch64_sve_udivr>;
+  defm SDIV_ZPmZ  : sve_int_bin_pred_arit_2_div<0b100, "sdiv",  "SDIV_ZPZZ", int_aarch64_sve_sdiv, DestructiveBinaryCommWithRev, "SDIVR_ZPmZ">;
+  defm UDIV_ZPmZ  : sve_int_bin_pred_arit_2_div<0b101, "udiv",  "UDIV_ZPZZ", int_aarch64_sve_udiv, DestructiveBinaryCommWithRev, "UDIVR_ZPmZ">;
+  defm SDIVR_ZPmZ : sve_int_bin_pred_arit_2_div<0b110, "sdivr", "SDIVR_ZPZZ", int_aarch64_sve_sdivr, DestructiveBinaryCommWithRev, "SDIV_ZPmZ", /*isReverseInstr*/ 1>;
+  defm UDIVR_ZPmZ : sve_int_bin_pred_arit_2_div<0b111, "udivr", "UDIVR_ZPZZ", int_aarch64_sve_udivr, DestructiveBinaryCommWithRev, "UDIV_ZPmZ", /*isReverseInstr*/ 1>;
+
+  defm SDIV_ZPZZ  : sve_int_bin_pred_sd<AArch64sdiv_p>;
+  defm UDIV_ZPZZ  : sve_int_bin_pred_sd<AArch64udiv_p>;
 
   defm SDOT_ZZZ : sve_intx_dot<0b0, "sdot", int_aarch64_sve_sdot>;
   defm UDOT_ZZZ : sve_intx_dot<0b1, "udot", int_aarch64_sve_udot>;
@@ -284,15 +317,20 @@ let Predicates = [HasSVE] in {
   defm CLS_ZPmZ  : sve_int_un_pred_arit_1<   0b000, "cls",  int_aarch64_sve_cls>;
   defm CLZ_ZPmZ  : sve_int_un_pred_arit_1<   0b001, "clz",  int_aarch64_sve_clz>;
   defm CNT_ZPmZ  : sve_int_un_pred_arit_1<   0b010, "cnt",  int_aarch64_sve_cnt>;
+
+ let Predicates = [HasSVE, HasBF16] in {
+  def : SVE_3_Op_Pat<nxv8i16, int_aarch64_sve_cnt, nxv8i16, nxv8i1, nxv8bf16, !cast<Instruction>(CNT_ZPmZ_H)>;
+ }
+
   defm CNOT_ZPmZ : sve_int_un_pred_arit_1<   0b011, "cnot", int_aarch64_sve_cnot>;
   defm NOT_ZPmZ  : sve_int_un_pred_arit_1<   0b110, "not",  int_aarch64_sve_not>;
   defm FABS_ZPmZ : sve_int_un_pred_arit_1_fp<0b100, "fabs", int_aarch64_sve_fabs>;
   defm FNEG_ZPmZ : sve_int_un_pred_arit_1_fp<0b101, "fneg", int_aarch64_sve_fneg>;
 
-  defm SMAX_ZPmZ : sve_int_bin_pred_arit_1<0b000, "smax", AArch64smax_pred>;
-  defm UMAX_ZPmZ : sve_int_bin_pred_arit_1<0b001, "umax", AArch64umax_pred>;
-  defm SMIN_ZPmZ : sve_int_bin_pred_arit_1<0b010, "smin", AArch64smin_pred>;
-  defm UMIN_ZPmZ : sve_int_bin_pred_arit_1<0b011, "umin", AArch64umin_pred>;
+  defm SMAX_ZPmZ : sve_int_bin_pred_arit_1<0b000, "smax", AArch64smax_m1>;
+  defm UMAX_ZPmZ : sve_int_bin_pred_arit_1<0b001, "umax", AArch64umax_m1>;
+  defm SMIN_ZPmZ : sve_int_bin_pred_arit_1<0b010, "smin", AArch64smin_m1>;
+  defm UMIN_ZPmZ : sve_int_bin_pred_arit_1<0b011, "umin", AArch64umin_m1>;
   defm SABD_ZPmZ : sve_int_bin_pred_arit_1<0b100, "sabd", int_aarch64_sve_sabd>;
   defm UABD_ZPmZ : sve_int_bin_pred_arit_1<0b101, "uabd", int_aarch64_sve_uabd>;
 
@@ -309,9 +347,9 @@ let Predicates = [HasSVE] in {
   defm FMIN_ZPmI    : sve_fp_2op_i_p_zds<0b111, "fmin", sve_fpimm_zero_one>;
 
   defm FADD_ZPmZ   : sve_fp_2op_p_zds<0b0000, "fadd", "FADD_ZPZZ", int_aarch64_sve_fadd, DestructiveBinaryComm>;
-  defm FSUB_ZPmZ   : sve_fp_2op_p_zds<0b0001, "fsub", "FSUB_ZPZZ", int_aarch64_sve_fsub, DestructiveBinaryCommWithRev, "FSUBR_ZPmZ", 1>;
+  defm FSUB_ZPmZ   : sve_fp_2op_p_zds<0b0001, "fsub", "FSUB_ZPZZ", int_aarch64_sve_fsub, DestructiveBinaryCommWithRev, "FSUBR_ZPmZ">;
   defm FMUL_ZPmZ   : sve_fp_2op_p_zds<0b0010, "fmul", "FMUL_ZPZZ", int_aarch64_sve_fmul, DestructiveBinaryComm>;
-  defm FSUBR_ZPmZ  : sve_fp_2op_p_zds<0b0011, "fsubr", "FSUBR_ZPZZ", int_aarch64_sve_fsubr, DestructiveBinaryCommWithRev, "FSUB_ZPmZ", 0>;
+  defm FSUBR_ZPmZ  : sve_fp_2op_p_zds<0b0011, "fsubr", "FSUBR_ZPZZ", int_aarch64_sve_fsubr, DestructiveBinaryCommWithRev, "FSUB_ZPmZ", /*isReverseInstr*/ 1>;
   defm FMAXNM_ZPmZ : sve_fp_2op_p_zds<0b0100, "fmaxnm", "FMAXNM_ZPZZ", int_aarch64_sve_fmaxnm, DestructiveBinaryComm>;
   defm FMINNM_ZPmZ : sve_fp_2op_p_zds<0b0101, "fminnm", "FMINNM_ZPZZ", int_aarch64_sve_fminnm, DestructiveBinaryComm>;
   defm FMAX_ZPmZ   : sve_fp_2op_p_zds<0b0110, "fmax", "FMAX_ZPZZ", int_aarch64_sve_fmax, DestructiveBinaryComm>;
@@ -319,21 +357,25 @@ let Predicates = [HasSVE] in {
   defm FABD_ZPmZ   : sve_fp_2op_p_zds<0b1000, "fabd", "FABD_ZPZZ", int_aarch64_sve_fabd, DestructiveBinaryComm>;
   defm FSCALE_ZPmZ : sve_fp_2op_p_zds_fscale<0b1001, "fscale", int_aarch64_sve_fscale>;
   defm FMULX_ZPmZ  : sve_fp_2op_p_zds<0b1010, "fmulx", "FMULX_ZPZZ", int_aarch64_sve_fmulx, DestructiveBinaryComm>;
-  defm FDIVR_ZPmZ  : sve_fp_2op_p_zds<0b1100, "fdivr", "FDIVR_ZPZZ", int_aarch64_sve_fdivr, DestructiveBinaryCommWithRev, "FDIV_ZPmZ", 0>;
-  defm FDIV_ZPmZ   : sve_fp_2op_p_zds<0b1101, "fdiv", "FDIV_ZPZZ", int_aarch64_sve_fdiv, DestructiveBinaryCommWithRev, "FDIVR_ZPmZ", 1>;
-
-  defm FADD_ZPZZ   : sve_fp_2op_p_zds_zx<int_aarch64_sve_fadd>;
-  defm FSUB_ZPZZ   : sve_fp_2op_p_zds_zx<int_aarch64_sve_fsub>;
-  defm FMUL_ZPZZ   : sve_fp_2op_p_zds_zx<int_aarch64_sve_fmul>;
-  defm FSUBR_ZPZZ  : sve_fp_2op_p_zds_zx<int_aarch64_sve_fsubr>;
-  defm FMAXNM_ZPZZ : sve_fp_2op_p_zds_zx<int_aarch64_sve_fmaxnm>;
-  defm FMINNM_ZPZZ : sve_fp_2op_p_zds_zx<int_aarch64_sve_fminnm>;
-  defm FMAX_ZPZZ   : sve_fp_2op_p_zds_zx<int_aarch64_sve_fmax>;
-  defm FMIN_ZPZZ   : sve_fp_2op_p_zds_zx<int_aarch64_sve_fmin>;
-  defm FABD_ZPZZ   : sve_fp_2op_p_zds_zx<int_aarch64_sve_fabd>;
-  defm FMULX_ZPZZ  : sve_fp_2op_p_zds_zx<int_aarch64_sve_fmulx>;
-  defm FDIVR_ZPZZ  : sve_fp_2op_p_zds_zx<int_aarch64_sve_fdivr>;
-  defm FDIV_ZPZZ   : sve_fp_2op_p_zds_zx<int_aarch64_sve_fdiv>;
+  defm FDIVR_ZPmZ  : sve_fp_2op_p_zds<0b1100, "fdivr", "FDIVR_ZPZZ", int_aarch64_sve_fdivr, DestructiveBinaryCommWithRev, "FDIV_ZPmZ", /*isReverseInstr*/ 1>;
+  defm FDIV_ZPmZ   : sve_fp_2op_p_zds<0b1101, "fdiv", "FDIV_ZPZZ", int_aarch64_sve_fdiv, DestructiveBinaryCommWithRev, "FDIVR_ZPmZ">;
+
+  defm FADD_ZPZZ   : sve_fp_bin_pred_hfd<AArch64fadd_p>;
+
+  let Predicates = [HasSVE, UseExperimentalZeroingPseudos] in {
+    defm FADD_ZPZZ   : sve_fp_2op_p_zds_zeroing_hsd<int_aarch64_sve_fadd>;
+    defm FSUB_ZPZZ   : sve_fp_2op_p_zds_zeroing_hsd<int_aarch64_sve_fsub>;
+    defm FMUL_ZPZZ   : sve_fp_2op_p_zds_zeroing_hsd<int_aarch64_sve_fmul>;
+    defm FSUBR_ZPZZ  : sve_fp_2op_p_zds_zeroing_hsd<int_aarch64_sve_fsubr>;
+    defm FMAXNM_ZPZZ : sve_fp_2op_p_zds_zeroing_hsd<int_aarch64_sve_fmaxnm>;
+    defm FMINNM_ZPZZ : sve_fp_2op_p_zds_zeroing_hsd<int_aarch64_sve_fminnm>;
+    defm FMAX_ZPZZ   : sve_fp_2op_p_zds_zeroing_hsd<int_aarch64_sve_fmax>;
+    defm FMIN_ZPZZ   : sve_fp_2op_p_zds_zeroing_hsd<int_aarch64_sve_fmin>;
+    defm FABD_ZPZZ   : sve_fp_2op_p_zds_zeroing_hsd<int_aarch64_sve_fabd>;
+    defm FMULX_ZPZZ  : sve_fp_2op_p_zds_zeroing_hsd<int_aarch64_sve_fmulx>;
+    defm FDIVR_ZPZZ  : sve_fp_2op_p_zds_zeroing_hsd<int_aarch64_sve_fdivr>;
+    defm FDIV_ZPZZ   : sve_fp_2op_p_zds_zeroing_hsd<int_aarch64_sve_fdiv>;
+  }
 
   defm FADD_ZZZ    : sve_fp_3op_u_zd<0b000, "fadd",    fadd>;
   defm FSUB_ZZZ    : sve_fp_3op_u_zd<0b001, "fsub",    fsub>;
@@ -357,6 +399,16 @@ let Predicates = [HasSVE] in {
   defm FNMAD_ZPmZZ : sve_fp_3op_p_zds_b<0b10, "fnmad", int_aarch64_sve_fnmad>;
   defm FNMSB_ZPmZZ : sve_fp_3op_p_zds_b<0b11, "fnmsb", int_aarch64_sve_fnmsb>;
 
+  // Add patterns for FMA where disabled lanes are undef.
+  // FIXME: Implement a pseudo so we can choose a better instruction after
+  // regalloc.
+  def : Pat<(nxv8f16 (AArch64fma_p nxv8i1:$P, nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3)),
+            (FMLA_ZPmZZ_H $P, $Op3, $Op1, $Op2)>;
+  def : Pat<(nxv4f32 (AArch64fma_p nxv4i1:$P, nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3)),
+            (FMLA_ZPmZZ_S $P, $Op3, $Op1, $Op2)>;
+  def : Pat<(nxv2f64 (AArch64fma_p nxv2i1:$P, nxv2f64:$Op1, nxv2f64:$Op2, nxv2f64:$Op3)),
+            (FMLA_ZPmZZ_D $P, $Op3, $Op1, $Op2)>;
+
   defm FTMAD_ZZI : sve_fp_ftmad<"ftmad", int_aarch64_sve_ftmad_x>;
 
   defm FMLA_ZZZI : sve_fp_fma_by_indexed_elem<0b0, "fmla", int_aarch64_sve_fmla_lane>;
@@ -366,12 +418,12 @@ let Predicates = [HasSVE] in {
   defm FMUL_ZZZI   : sve_fp_fmul_by_indexed_elem<"fmul", int_aarch64_sve_fmul_lane>;
 
   // SVE floating point reductions.
-  defm FADDA_VPZ   : sve_fp_2op_p_vd<0b000, "fadda",   AArch64fadda_pred>;
-  defm FADDV_VPZ   : sve_fp_fast_red<0b000, "faddv",   AArch64faddv_pred>;
-  defm FMAXNMV_VPZ : sve_fp_fast_red<0b100, "fmaxnmv", AArch64fmaxnmv_pred>;
-  defm FMINNMV_VPZ : sve_fp_fast_red<0b101, "fminnmv", AArch64fminnmv_pred>;
-  defm FMAXV_VPZ   : sve_fp_fast_red<0b110, "fmaxv",   AArch64fmaxv_pred>;
-  defm FMINV_VPZ   : sve_fp_fast_red<0b111, "fminv",   AArch64fminv_pred>;
+  defm FADDA_VPZ   : sve_fp_2op_p_vd<0b000, "fadda",   AArch64fadda_p>;
+  defm FADDV_VPZ   : sve_fp_fast_red<0b000, "faddv",   AArch64faddv_p>;
+  defm FMAXNMV_VPZ : sve_fp_fast_red<0b100, "fmaxnmv", AArch64fmaxnmv_p>;
+  defm FMINNMV_VPZ : sve_fp_fast_red<0b101, "fminnmv", AArch64fminnmv_p>;
+  defm FMAXV_VPZ   : sve_fp_fast_red<0b110, "fmaxv",   AArch64fmaxv_p>;
+  defm FMINV_VPZ   : sve_fp_fast_red<0b111, "fminv",   AArch64fminv_p>;
 
   // Use more efficient NEON instructions to extract elements within the NEON
   // part (first 128bits) of an SVE register.
@@ -397,8 +449,13 @@ let Predicates = [HasSVE] in {
   defm DUP_ZZI : sve_int_perm_dup_i<"dup">;
 
   // Splat scalar register (predicated)
-  defm CPY_ZPmR : sve_int_perm_cpy_r<"cpy", AArch64dup_pred>;
-  defm CPY_ZPmV : sve_int_perm_cpy_v<"cpy", AArch64dup_pred>;
+  defm CPY_ZPmR : sve_int_perm_cpy_r<"cpy", AArch64dup_mt>;
+  defm CPY_ZPmV : sve_int_perm_cpy_v<"cpy", AArch64dup_mt>;
+
+  let Predicates = [HasSVE, HasBF16] in {
+    def : Pat<(nxv8bf16 (AArch64dup_mt nxv8i1:$pg, bf16:$splat, nxv8bf16:$passthru)),
+              (CPY_ZPmV_H $passthru, $pg, $splat)>;
+  }
 
   // Duplicate FP scalar into all vector elements
   def : Pat<(nxv8f16 (AArch64dup (f16 FPR16:$src))),
@@ -413,6 +470,10 @@ let Predicates = [HasSVE] in {
             (DUP_ZZI_S (INSERT_SUBREG (IMPLICIT_DEF), FPR32:$src, ssub), 0)>;
   def : Pat<(nxv2f64 (AArch64dup (f64 FPR64:$src))),
             (DUP_ZZI_D (INSERT_SUBREG (IMPLICIT_DEF), FPR64:$src, dsub), 0)>;
+  let Predicates = [HasSVE, HasBF16] in {
+    def : Pat<(nxv8bf16 (AArch64dup (bf16 FPR16:$src))),
+              (DUP_ZZI_H (INSERT_SUBREG (IMPLICIT_DEF), FPR16:$src, hsub), 0)>;
+  }
 
   // Duplicate +0.0 into all vector elements
   def : Pat<(nxv8f16 (AArch64dup (f16 fpimm0))), (DUP_ZI_H 0, 0)>;
@@ -421,6 +482,9 @@ let Predicates = [HasSVE] in {
   def : Pat<(nxv4f32 (AArch64dup (f32 fpimm0))), (DUP_ZI_S 0, 0)>;
   def : Pat<(nxv2f32 (AArch64dup (f32 fpimm0))), (DUP_ZI_S 0, 0)>;
   def : Pat<(nxv2f64 (AArch64dup (f64 fpimm0))), (DUP_ZI_D 0, 0)>;
+  let Predicates = [HasSVE, HasBF16] in {
+    def : Pat<(nxv8bf16 (AArch64dup (bf16 fpimm0))), (DUP_ZI_H 0, 0)>;
+  }
 
   // Duplicate Int immediate into all vector elements
   def : Pat<(nxv16i8 (AArch64dup (i32 (SVE8BitLslImm i32:$a, i32:$b)))),
@@ -452,11 +516,21 @@ let Predicates = [HasSVE] in {
   defm SEL_ZPZZ    : sve_int_sel_vvv<"sel", vselect>;
 
   defm SPLICE_ZPZ : sve_int_perm_splice<"splice", int_aarch64_sve_splice>;
+
+  let Predicates = [HasSVE, HasBF16] in {
+    def : SVE_3_Op_Pat<nxv8bf16, vselect, nxv8i1, nxv8bf16, nxv8bf16, SEL_ZPZZ_H>;
+    def : SVE_3_Op_Pat<nxv8bf16, int_aarch64_sve_splice, nxv8i1, nxv8bf16, nxv8bf16, SPLICE_ZPZ_H>;
+  }
+
   defm COMPACT_ZPZ : sve_int_perm_compact<"compact", int_aarch64_sve_compact>;
   defm INSR_ZR : sve_int_perm_insrs<"insr", AArch64insr>;
   defm INSR_ZV : sve_int_perm_insrv<"insr", AArch64insr>;
   defm EXT_ZZI : sve_int_perm_extract_i<"ext", AArch64ext>;
 
+  let Predicates = [HasSVE, HasBF16] in {
+    def : SVE_2_Op_Pat<nxv8bf16, AArch64insr, nxv8bf16, bf16, INSR_ZV_H>;
+  }
+
   defm RBIT_ZPmZ : sve_int_perm_rev_rbit<"rbit", int_aarch64_sve_rbit>;
   defm REVB_ZPmZ : sve_int_perm_rev_revb<"revb", int_aarch64_sve_revb, bswap>;
   defm REVH_ZPmZ : sve_int_perm_rev_revh<"revh", int_aarch64_sve_revh>;
@@ -465,6 +539,10 @@ let Predicates = [HasSVE] in {
   defm REV_PP : sve_int_perm_reverse_p<"rev", AArch64rev>;
   defm REV_ZZ : sve_int_perm_reverse_z<"rev", AArch64rev>;
 
+  let Predicates = [HasSVE, HasBF16] in {
+    def : SVE_1_Op_Pat<nxv8bf16, AArch64rev, nxv8bf16, REV_ZZ_H>;
+  }
+
   defm SUNPKLO_ZZ : sve_int_perm_unpk<0b00, "sunpklo", AArch64sunpklo>;
   defm SUNPKHI_ZZ : sve_int_perm_unpk<0b01, "sunpkhi", AArch64sunpkhi>;
   defm UUNPKLO_ZZ : sve_int_perm_unpk<0b10, "uunpklo", AArch64uunpklo>;
@@ -521,11 +599,23 @@ let Predicates = [HasSVE] in {
   defm CLASTA_ZPZ : sve_int_perm_clast_zz<0, "clasta", int_aarch64_sve_clasta>;
   defm CLASTB_ZPZ : sve_int_perm_clast_zz<1, "clastb", int_aarch64_sve_clastb>;
 
+  let Predicates = [HasSVE, HasBF16] in {
+    def : SVE_3_Op_Pat<bf16,     AArch64clasta_n,        nxv8i1, bf16,     nxv8bf16, CLASTA_VPZ_H>;
+    def : SVE_3_Op_Pat<bf16,     AArch64clastb_n,        nxv8i1, bf16,     nxv8bf16, CLASTB_VPZ_H>;
+    def : SVE_3_Op_Pat<nxv8bf16, int_aarch64_sve_clasta, nxv8i1, nxv8bf16, nxv8bf16, CLASTA_ZPZ_H>;
+    def : SVE_3_Op_Pat<nxv8bf16, int_aarch64_sve_clastb, nxv8i1, nxv8bf16, nxv8bf16, CLASTB_ZPZ_H>;
+  }
+
   defm LASTA_RPZ : sve_int_perm_last_r<0, "lasta", AArch64lasta>;
   defm LASTB_RPZ : sve_int_perm_last_r<1, "lastb", AArch64lastb>;
   defm LASTA_VPZ : sve_int_perm_last_v<0, "lasta", AArch64lasta>;
   defm LASTB_VPZ : sve_int_perm_last_v<1, "lastb", AArch64lastb>;
 
+  let Predicates = [HasSVE, HasBF16] in {
+    def : SVE_2_Op_Pat<bf16, AArch64lasta, nxv8i1, nxv8bf16, LASTA_VPZ_H>;
+    def : SVE_2_Op_Pat<bf16, AArch64lastb, nxv8i1, nxv8bf16, LASTB_VPZ_H>;
+  }
+
   // continuous load with reg+immediate
   defm LD1B_IMM    : sve_mem_cld_si<0b0000, "ld1b",  Z_b, ZPR8>;
   defm LD1B_H_IMM  : sve_mem_cld_si<0b0001, "ld1b",  Z_h, ZPR16>;
@@ -656,115 +746,115 @@ let Predicates = [HasSVE] in {
 
   // Gathers using unscaled 32-bit offsets, e.g.
   //    ld1h z0.s, p0/z, [x0, z0.s, uxtw]
-  defm GLD1SB_S   : sve_mem_32b_gld_vs_32_unscaled<0b0000, "ld1sb",   AArch64ld1s_gather_sxtw,   AArch64ld1s_gather_uxtw,   ZPR32ExtSXTW8Only, ZPR32ExtUXTW8Only, nxv4i8>;
-  defm GLDFF1SB_S : sve_mem_32b_gld_vs_32_unscaled<0b0001, "ldff1sb", AArch64ldff1s_gather_sxtw, AArch64ldff1s_gather_uxtw, ZPR32ExtSXTW8Only, ZPR32ExtUXTW8Only, nxv4i8>;
-  defm GLD1B_S    : sve_mem_32b_gld_vs_32_unscaled<0b0010, "ld1b",    AArch64ld1_gather_sxtw,    AArch64ld1_gather_uxtw,    ZPR32ExtSXTW8Only, ZPR32ExtUXTW8Only, nxv4i8>;
-  defm GLDFF1B_S  : sve_mem_32b_gld_vs_32_unscaled<0b0011, "ldff1b",  AArch64ldff1_gather_sxtw,  AArch64ldff1_gather_uxtw,  ZPR32ExtSXTW8Only, ZPR32ExtUXTW8Only, nxv4i8>;
-  defm GLD1SH_S   : sve_mem_32b_gld_vs_32_unscaled<0b0100, "ld1sh",   AArch64ld1s_gather_sxtw,   AArch64ld1s_gather_uxtw,   ZPR32ExtSXTW8,     ZPR32ExtUXTW8,     nxv4i16>;
-  defm GLDFF1SH_S : sve_mem_32b_gld_vs_32_unscaled<0b0101, "ldff1sh", AArch64ldff1s_gather_sxtw, AArch64ldff1s_gather_uxtw, ZPR32ExtSXTW8,     ZPR32ExtUXTW8,     nxv4i16>;
-  defm GLD1H_S    : sve_mem_32b_gld_vs_32_unscaled<0b0110, "ld1h",    AArch64ld1_gather_sxtw,    AArch64ld1_gather_uxtw,    ZPR32ExtSXTW8,     ZPR32ExtUXTW8,     nxv4i16>;
-  defm GLDFF1H_S  : sve_mem_32b_gld_vs_32_unscaled<0b0111, "ldff1h",  AArch64ldff1_gather_sxtw,  AArch64ldff1_gather_uxtw,  ZPR32ExtSXTW8,     ZPR32ExtUXTW8,     nxv4i16>;
-  defm GLD1W      : sve_mem_32b_gld_vs_32_unscaled<0b1010, "ld1w",    AArch64ld1_gather_sxtw,    AArch64ld1_gather_uxtw,    ZPR32ExtSXTW8,     ZPR32ExtUXTW8,     nxv4i32>;
-  defm GLDFF1W    : sve_mem_32b_gld_vs_32_unscaled<0b1011, "ldff1w",  AArch64ldff1_gather_sxtw,  AArch64ldff1_gather_uxtw,  ZPR32ExtSXTW8,     ZPR32ExtUXTW8,     nxv4i32>;
+  defm GLD1SB_S   : sve_mem_32b_gld_vs_32_unscaled<0b0000, "ld1sb",   AArch64ld1s_gather_sxtw_z,   AArch64ld1s_gather_uxtw_z,   ZPR32ExtSXTW8Only, ZPR32ExtUXTW8Only, nxv4i8>;
+  defm GLDFF1SB_S : sve_mem_32b_gld_vs_32_unscaled<0b0001, "ldff1sb", AArch64ldff1s_gather_sxtw_z, AArch64ldff1s_gather_uxtw_z, ZPR32ExtSXTW8Only, ZPR32ExtUXTW8Only, nxv4i8>;
+  defm GLD1B_S    : sve_mem_32b_gld_vs_32_unscaled<0b0010, "ld1b",    AArch64ld1_gather_sxtw_z,    AArch64ld1_gather_uxtw_z,    ZPR32ExtSXTW8Only, ZPR32ExtUXTW8Only, nxv4i8>;
+  defm GLDFF1B_S  : sve_mem_32b_gld_vs_32_unscaled<0b0011, "ldff1b",  AArch64ldff1_gather_sxtw_z,  AArch64ldff1_gather_uxtw_z,  ZPR32ExtSXTW8Only, ZPR32ExtUXTW8Only, nxv4i8>;
+  defm GLD1SH_S   : sve_mem_32b_gld_vs_32_unscaled<0b0100, "ld1sh",   AArch64ld1s_gather_sxtw_z,   AArch64ld1s_gather_uxtw_z,   ZPR32ExtSXTW8,     ZPR32ExtUXTW8,     nxv4i16>;
+  defm GLDFF1SH_S : sve_mem_32b_gld_vs_32_unscaled<0b0101, "ldff1sh", AArch64ldff1s_gather_sxtw_z, AArch64ldff1s_gather_uxtw_z, ZPR32ExtSXTW8,     ZPR32ExtUXTW8,     nxv4i16>;
+  defm GLD1H_S    : sve_mem_32b_gld_vs_32_unscaled<0b0110, "ld1h",    AArch64ld1_gather_sxtw_z,    AArch64ld1_gather_uxtw_z,    ZPR32ExtSXTW8,     ZPR32ExtUXTW8,     nxv4i16>;
+  defm GLDFF1H_S  : sve_mem_32b_gld_vs_32_unscaled<0b0111, "ldff1h",  AArch64ldff1_gather_sxtw_z,  AArch64ldff1_gather_uxtw_z,  ZPR32ExtSXTW8,     ZPR32ExtUXTW8,     nxv4i16>;
+  defm GLD1W      : sve_mem_32b_gld_vs_32_unscaled<0b1010, "ld1w",    AArch64ld1_gather_sxtw_z,    AArch64ld1_gather_uxtw_z,    ZPR32ExtSXTW8,     ZPR32ExtUXTW8,     nxv4i32>;
+  defm GLDFF1W    : sve_mem_32b_gld_vs_32_unscaled<0b1011, "ldff1w",  AArch64ldff1_gather_sxtw_z,  AArch64ldff1_gather_uxtw_z,  ZPR32ExtSXTW8,     ZPR32ExtUXTW8,     nxv4i32>;
 
   // Gathers using scaled 32-bit offsets, e.g.
   //    ld1h z0.s, p0/z, [x0, z0.s, uxtw #1]
-  defm GLD1SH_S   : sve_mem_32b_gld_sv_32_scaled<0b0100, "ld1sh",   AArch64ld1s_gather_sxtw_scaled,   AArch64ld1s_gather_uxtw_scaled,   ZPR32ExtSXTW16, ZPR32ExtUXTW16, nxv4i16>;
-  defm GLDFF1SH_S : sve_mem_32b_gld_sv_32_scaled<0b0101, "ldff1sh", AArch64ldff1s_gather_sxtw_scaled, AArch64ldff1s_gather_uxtw_scaled, ZPR32ExtSXTW16, ZPR32ExtUXTW16, nxv4i16>;
-  defm GLD1H_S    : sve_mem_32b_gld_sv_32_scaled<0b0110, "ld1h",    AArch64ld1_gather_sxtw_scaled,    AArch64ld1_gather_uxtw_scaled,    ZPR32ExtSXTW16, ZPR32ExtUXTW16, nxv4i16>;
-  defm GLDFF1H_S  : sve_mem_32b_gld_sv_32_scaled<0b0111, "ldff1h",  AArch64ldff1_gather_sxtw_scaled,  AArch64ldff1_gather_uxtw_scaled,  ZPR32ExtSXTW16, ZPR32ExtUXTW16, nxv4i16>;
-  defm GLD1W      : sve_mem_32b_gld_sv_32_scaled<0b1010, "ld1w",    AArch64ld1_gather_sxtw_scaled,    AArch64ld1_gather_uxtw_scaled,    ZPR32ExtSXTW32, ZPR32ExtUXTW32, nxv4i32>;
-  defm GLDFF1W    : sve_mem_32b_gld_sv_32_scaled<0b1011, "ldff1w",  AArch64ldff1_gather_sxtw_scaled,  AArch64ldff1_gather_uxtw_scaled,  ZPR32ExtSXTW32, ZPR32ExtUXTW32, nxv4i32>;
+  defm GLD1SH_S   : sve_mem_32b_gld_sv_32_scaled<0b0100, "ld1sh",   AArch64ld1s_gather_sxtw_scaled_z,   AArch64ld1s_gather_uxtw_scaled_z,   ZPR32ExtSXTW16, ZPR32ExtUXTW16, nxv4i16>;
+  defm GLDFF1SH_S : sve_mem_32b_gld_sv_32_scaled<0b0101, "ldff1sh", AArch64ldff1s_gather_sxtw_scaled_z, AArch64ldff1s_gather_uxtw_scaled_z, ZPR32ExtSXTW16, ZPR32ExtUXTW16, nxv4i16>;
+  defm GLD1H_S    : sve_mem_32b_gld_sv_32_scaled<0b0110, "ld1h",    AArch64ld1_gather_sxtw_scaled_z,    AArch64ld1_gather_uxtw_scaled_z,    ZPR32ExtSXTW16, ZPR32ExtUXTW16, nxv4i16>;
+  defm GLDFF1H_S  : sve_mem_32b_gld_sv_32_scaled<0b0111, "ldff1h",  AArch64ldff1_gather_sxtw_scaled_z,  AArch64ldff1_gather_uxtw_scaled_z,  ZPR32ExtSXTW16, ZPR32ExtUXTW16, nxv4i16>;
+  defm GLD1W      : sve_mem_32b_gld_sv_32_scaled<0b1010, "ld1w",    AArch64ld1_gather_sxtw_scaled_z,    AArch64ld1_gather_uxtw_scaled_z,    ZPR32ExtSXTW32, ZPR32ExtUXTW32, nxv4i32>;
+  defm GLDFF1W    : sve_mem_32b_gld_sv_32_scaled<0b1011, "ldff1w",  AArch64ldff1_gather_sxtw_scaled_z,  AArch64ldff1_gather_uxtw_scaled_z,  ZPR32ExtSXTW32, ZPR32ExtUXTW32, nxv4i32>;
 
   // Gathers using 32-bit pointers with scaled offset, e.g.
   //    ld1h z0.s, p0/z, [z0.s, #16]
-  defm GLD1SB_S   : sve_mem_32b_gld_vi_32_ptrs<0b0000, "ld1sb",   imm0_31, AArch64ld1s_gather_imm,   nxv4i8>;
-  defm GLDFF1SB_S : sve_mem_32b_gld_vi_32_ptrs<0b0001, "ldff1sb", imm0_31, AArch64ldff1s_gather_imm, nxv4i8>;
-  defm GLD1B_S    : sve_mem_32b_gld_vi_32_ptrs<0b0010, "ld1b",    imm0_31, AArch64ld1_gather_imm,    nxv4i8>;
-  defm GLDFF1B_S  : sve_mem_32b_gld_vi_32_ptrs<0b0011, "ldff1b",  imm0_31, AArch64ldff1_gather_imm,  nxv4i8>;
-  defm GLD1SH_S   : sve_mem_32b_gld_vi_32_ptrs<0b0100, "ld1sh",   uimm5s2, AArch64ld1s_gather_imm,   nxv4i16>;
-  defm GLDFF1SH_S : sve_mem_32b_gld_vi_32_ptrs<0b0101, "ldff1sh", uimm5s2, AArch64ldff1s_gather_imm, nxv4i16>;
-  defm GLD1H_S    : sve_mem_32b_gld_vi_32_ptrs<0b0110, "ld1h",    uimm5s2, AArch64ld1_gather_imm,    nxv4i16>;
-  defm GLDFF1H_S  : sve_mem_32b_gld_vi_32_ptrs<0b0111, "ldff1h",  uimm5s2, AArch64ldff1_gather_imm,  nxv4i16>;
-  defm GLD1W      : sve_mem_32b_gld_vi_32_ptrs<0b1010, "ld1w",    uimm5s4, AArch64ld1_gather_imm,    nxv4i32>;
-  defm GLDFF1W    : sve_mem_32b_gld_vi_32_ptrs<0b1011, "ldff1w",  uimm5s4, AArch64ldff1_gather_imm,  nxv4i32>;
+  defm GLD1SB_S   : sve_mem_32b_gld_vi_32_ptrs<0b0000, "ld1sb",   imm0_31, AArch64ld1s_gather_imm_z,   nxv4i8>;
+  defm GLDFF1SB_S : sve_mem_32b_gld_vi_32_ptrs<0b0001, "ldff1sb", imm0_31, AArch64ldff1s_gather_imm_z, nxv4i8>;
+  defm GLD1B_S    : sve_mem_32b_gld_vi_32_ptrs<0b0010, "ld1b",    imm0_31, AArch64ld1_gather_imm_z,    nxv4i8>;
+  defm GLDFF1B_S  : sve_mem_32b_gld_vi_32_ptrs<0b0011, "ldff1b",  imm0_31, AArch64ldff1_gather_imm_z,  nxv4i8>;
+  defm GLD1SH_S   : sve_mem_32b_gld_vi_32_ptrs<0b0100, "ld1sh",   uimm5s2, AArch64ld1s_gather_imm_z,   nxv4i16>;
+  defm GLDFF1SH_S : sve_mem_32b_gld_vi_32_ptrs<0b0101, "ldff1sh", uimm5s2, AArch64ldff1s_gather_imm_z, nxv4i16>;
+  defm GLD1H_S    : sve_mem_32b_gld_vi_32_ptrs<0b0110, "ld1h",    uimm5s2, AArch64ld1_gather_imm_z,    nxv4i16>;
+  defm GLDFF1H_S  : sve_mem_32b_gld_vi_32_ptrs<0b0111, "ldff1h",  uimm5s2, AArch64ldff1_gather_imm_z,  nxv4i16>;
+  defm GLD1W      : sve_mem_32b_gld_vi_32_ptrs<0b1010, "ld1w",    uimm5s4, AArch64ld1_gather_imm_z,    nxv4i32>;
+  defm GLDFF1W    : sve_mem_32b_gld_vi_32_ptrs<0b1011, "ldff1w",  uimm5s4, AArch64ldff1_gather_imm_z,  nxv4i32>;
 
   // Gathers using 64-bit pointers with scaled offset, e.g.
   //    ld1h z0.d, p0/z, [z0.d, #16]
-  defm GLD1SB_D   : sve_mem_64b_gld_vi_64_ptrs<0b0000, "ld1sb",   imm0_31, AArch64ld1s_gather_imm,   nxv2i8>;
-  defm GLDFF1SB_D : sve_mem_64b_gld_vi_64_ptrs<0b0001, "ldff1sb", imm0_31, AArch64ldff1s_gather_imm, nxv2i8>;
-  defm GLD1B_D    : sve_mem_64b_gld_vi_64_ptrs<0b0010, "ld1b",    imm0_31, AArch64ld1_gather_imm,    nxv2i8>;
-  defm GLDFF1B_D  : sve_mem_64b_gld_vi_64_ptrs<0b0011, "ldff1b",  imm0_31, AArch64ldff1_gather_imm,  nxv2i8>;
-  defm GLD1SH_D   : sve_mem_64b_gld_vi_64_ptrs<0b0100, "ld1sh",   uimm5s2, AArch64ld1s_gather_imm,   nxv2i16>;
-  defm GLDFF1SH_D : sve_mem_64b_gld_vi_64_ptrs<0b0101, "ldff1sh", uimm5s2, AArch64ldff1s_gather_imm, nxv2i16>;
-  defm GLD1H_D    : sve_mem_64b_gld_vi_64_ptrs<0b0110, "ld1h",    uimm5s2, AArch64ld1_gather_imm,    nxv2i16>;
-  defm GLDFF1H_D  : sve_mem_64b_gld_vi_64_ptrs<0b0111, "ldff1h",  uimm5s2, AArch64ldff1_gather_imm,  nxv2i16>;
-  defm GLD1SW_D   : sve_mem_64b_gld_vi_64_ptrs<0b1000, "ld1sw",   uimm5s4, AArch64ld1s_gather_imm,   nxv2i32>;
-  defm GLDFF1SW_D : sve_mem_64b_gld_vi_64_ptrs<0b1001, "ldff1sw", uimm5s4, AArch64ldff1s_gather_imm, nxv2i32>;
-  defm GLD1W_D    : sve_mem_64b_gld_vi_64_ptrs<0b1010, "ld1w",    uimm5s4, AArch64ld1_gather_imm,    nxv2i32>;
-  defm GLDFF1W_D  : sve_mem_64b_gld_vi_64_ptrs<0b1011, "ldff1w",  uimm5s4, AArch64ldff1_gather_imm,  nxv2i32>;
-  defm GLD1D      : sve_mem_64b_gld_vi_64_ptrs<0b1110, "ld1d",    uimm5s8, AArch64ld1_gather_imm,    nxv2i64>;
-  defm GLDFF1D    : sve_mem_64b_gld_vi_64_ptrs<0b1111, "ldff1d",  uimm5s8, AArch64ldff1_gather_imm,  nxv2i64>;
+  defm GLD1SB_D   : sve_mem_64b_gld_vi_64_ptrs<0b0000, "ld1sb",   imm0_31, AArch64ld1s_gather_imm_z,   nxv2i8>;
+  defm GLDFF1SB_D : sve_mem_64b_gld_vi_64_ptrs<0b0001, "ldff1sb", imm0_31, AArch64ldff1s_gather_imm_z, nxv2i8>;
+  defm GLD1B_D    : sve_mem_64b_gld_vi_64_ptrs<0b0010, "ld1b",    imm0_31, AArch64ld1_gather_imm_z,    nxv2i8>;
+  defm GLDFF1B_D  : sve_mem_64b_gld_vi_64_ptrs<0b0011, "ldff1b",  imm0_31, AArch64ldff1_gather_imm_z,  nxv2i8>;
+  defm GLD1SH_D   : sve_mem_64b_gld_vi_64_ptrs<0b0100, "ld1sh",   uimm5s2, AArch64ld1s_gather_imm_z,   nxv2i16>;
+  defm GLDFF1SH_D : sve_mem_64b_gld_vi_64_ptrs<0b0101, "ldff1sh", uimm5s2, AArch64ldff1s_gather_imm_z, nxv2i16>;
+  defm GLD1H_D    : sve_mem_64b_gld_vi_64_ptrs<0b0110, "ld1h",    uimm5s2, AArch64ld1_gather_imm_z,    nxv2i16>;
+  defm GLDFF1H_D  : sve_mem_64b_gld_vi_64_ptrs<0b0111, "ldff1h",  uimm5s2, AArch64ldff1_gather_imm_z,  nxv2i16>;
+  defm GLD1SW_D   : sve_mem_64b_gld_vi_64_ptrs<0b1000, "ld1sw",   uimm5s4, AArch64ld1s_gather_imm_z,   nxv2i32>;
+  defm GLDFF1SW_D : sve_mem_64b_gld_vi_64_ptrs<0b1001, "ldff1sw", uimm5s4, AArch64ldff1s_gather_imm_z, nxv2i32>;
+  defm GLD1W_D    : sve_mem_64b_gld_vi_64_ptrs<0b1010, "ld1w",    uimm5s4, AArch64ld1_gather_imm_z,    nxv2i32>;
+  defm GLDFF1W_D  : sve_mem_64b_gld_vi_64_ptrs<0b1011, "ldff1w",  uimm5s4, AArch64ldff1_gather_imm_z,  nxv2i32>;
+  defm GLD1D      : sve_mem_64b_gld_vi_64_ptrs<0b1110, "ld1d",    uimm5s8, AArch64ld1_gather_imm_z,    nxv2i64>;
+  defm GLDFF1D    : sve_mem_64b_gld_vi_64_ptrs<0b1111, "ldff1d",  uimm5s8, AArch64ldff1_gather_imm_z,  nxv2i64>;
 
   // Gathers using unscaled 64-bit offsets, e.g.
   //    ld1h z0.d, p0/z, [x0, z0.d]
-  defm GLD1SB_D   : sve_mem_64b_gld_vs2_64_unscaled<0b0000, "ld1sb",   AArch64ld1s_gather,   nxv2i8>;
-  defm GLDFF1SB_D : sve_mem_64b_gld_vs2_64_unscaled<0b0001, "ldff1sb", AArch64ldff1s_gather, nxv2i8>;
-  defm GLD1B_D    : sve_mem_64b_gld_vs2_64_unscaled<0b0010, "ld1b",    AArch64ld1_gather,    nxv2i8>;
-  defm GLDFF1B_D  : sve_mem_64b_gld_vs2_64_unscaled<0b0011, "ldff1b",  AArch64ldff1_gather,  nxv2i8>;
-  defm GLD1SH_D   : sve_mem_64b_gld_vs2_64_unscaled<0b0100, "ld1sh",   AArch64ld1s_gather,   nxv2i16>;
-  defm GLDFF1SH_D : sve_mem_64b_gld_vs2_64_unscaled<0b0101, "ldff1sh", AArch64ldff1s_gather, nxv2i16>;
-  defm GLD1H_D    : sve_mem_64b_gld_vs2_64_unscaled<0b0110, "ld1h",    AArch64ld1_gather,    nxv2i16>;
-  defm GLDFF1H_D  : sve_mem_64b_gld_vs2_64_unscaled<0b0111, "ldff1h",  AArch64ldff1_gather,  nxv2i16>;
-  defm GLD1SW_D   : sve_mem_64b_gld_vs2_64_unscaled<0b1000, "ld1sw",   AArch64ld1s_gather,   nxv2i32>;
-  defm GLDFF1SW_D : sve_mem_64b_gld_vs2_64_unscaled<0b1001, "ldff1sw", AArch64ldff1s_gather, nxv2i32>;
-  defm GLD1W_D    : sve_mem_64b_gld_vs2_64_unscaled<0b1010, "ld1w",    AArch64ld1_gather,    nxv2i32>;
-  defm GLDFF1W_D  : sve_mem_64b_gld_vs2_64_unscaled<0b1011, "ldff1w",  AArch64ldff1_gather,  nxv2i32>;
-  defm GLD1D      : sve_mem_64b_gld_vs2_64_unscaled<0b1110, "ld1d",    AArch64ld1_gather,    nxv2i64>;
-  defm GLDFF1D    : sve_mem_64b_gld_vs2_64_unscaled<0b1111, "ldff1d",  AArch64ldff1_gather,  nxv2i64>;
+  defm GLD1SB_D   : sve_mem_64b_gld_vs2_64_unscaled<0b0000, "ld1sb",   AArch64ld1s_gather_z,   nxv2i8>;
+  defm GLDFF1SB_D : sve_mem_64b_gld_vs2_64_unscaled<0b0001, "ldff1sb", AArch64ldff1s_gather_z, nxv2i8>;
+  defm GLD1B_D    : sve_mem_64b_gld_vs2_64_unscaled<0b0010, "ld1b",    AArch64ld1_gather_z,    nxv2i8>;
+  defm GLDFF1B_D  : sve_mem_64b_gld_vs2_64_unscaled<0b0011, "ldff1b",  AArch64ldff1_gather_z,  nxv2i8>;
+  defm GLD1SH_D   : sve_mem_64b_gld_vs2_64_unscaled<0b0100, "ld1sh",   AArch64ld1s_gather_z,   nxv2i16>;
+  defm GLDFF1SH_D : sve_mem_64b_gld_vs2_64_unscaled<0b0101, "ldff1sh", AArch64ldff1s_gather_z, nxv2i16>;
+  defm GLD1H_D    : sve_mem_64b_gld_vs2_64_unscaled<0b0110, "ld1h",    AArch64ld1_gather_z,    nxv2i16>;
+  defm GLDFF1H_D  : sve_mem_64b_gld_vs2_64_unscaled<0b0111, "ldff1h",  AArch64ldff1_gather_z,  nxv2i16>;
+  defm GLD1SW_D   : sve_mem_64b_gld_vs2_64_unscaled<0b1000, "ld1sw",   AArch64ld1s_gather_z,   nxv2i32>;
+  defm GLDFF1SW_D : sve_mem_64b_gld_vs2_64_unscaled<0b1001, "ldff1sw", AArch64ldff1s_gather_z, nxv2i32>;
+  defm GLD1W_D    : sve_mem_64b_gld_vs2_64_unscaled<0b1010, "ld1w",    AArch64ld1_gather_z,    nxv2i32>;
+  defm GLDFF1W_D  : sve_mem_64b_gld_vs2_64_unscaled<0b1011, "ldff1w",  AArch64ldff1_gather_z,  nxv2i32>;
+  defm GLD1D      : sve_mem_64b_gld_vs2_64_unscaled<0b1110, "ld1d",    AArch64ld1_gather_z,    nxv2i64>;
+  defm GLDFF1D    : sve_mem_64b_gld_vs2_64_unscaled<0b1111, "ldff1d",  AArch64ldff1_gather_z,  nxv2i64>;
 
   // Gathers using scaled 64-bit offsets, e.g.
   //    ld1h z0.d, p0/z, [x0, z0.d, lsl #1]
-  defm GLD1SH_D   : sve_mem_64b_gld_sv2_64_scaled<0b0100, "ld1sh",    AArch64ld1s_gather_scaled,   ZPR64ExtLSL16, nxv2i16>;
-  defm GLDFF1SH_D : sve_mem_64b_gld_sv2_64_scaled<0b0101, "ldff1sh",  AArch64ldff1s_gather_scaled, ZPR64ExtLSL16, nxv2i16>;
-  defm GLD1H_D    : sve_mem_64b_gld_sv2_64_scaled<0b0110, "ld1h",     AArch64ld1_gather_scaled,    ZPR64ExtLSL16, nxv2i16>;
-  defm GLDFF1H_D  : sve_mem_64b_gld_sv2_64_scaled<0b0111, "ldff1h",   AArch64ldff1_gather_scaled,  ZPR64ExtLSL16, nxv2i16>;
-  defm GLD1SW_D   : sve_mem_64b_gld_sv2_64_scaled<0b1000, "ld1sw",    AArch64ld1s_gather_scaled,   ZPR64ExtLSL32, nxv2i32>;
-  defm GLDFF1SW_D : sve_mem_64b_gld_sv2_64_scaled<0b1001, "ldff1sw",  AArch64ldff1s_gather_scaled, ZPR64ExtLSL32, nxv2i32>;
-  defm GLD1W_D    : sve_mem_64b_gld_sv2_64_scaled<0b1010, "ld1w",     AArch64ld1_gather_scaled,    ZPR64ExtLSL32, nxv2i32>;
-  defm GLDFF1W_D  : sve_mem_64b_gld_sv2_64_scaled<0b1011, "ldff1w",   AArch64ldff1_gather_scaled,  ZPR64ExtLSL32, nxv2i32>;
-  defm GLD1D      : sve_mem_64b_gld_sv2_64_scaled<0b1110, "ld1d",     AArch64ld1_gather_scaled,    ZPR64ExtLSL64, nxv2i64>;
-  defm GLDFF1D    : sve_mem_64b_gld_sv2_64_scaled<0b1111, "ldff1d",   AArch64ldff1_gather_scaled,  ZPR64ExtLSL64, nxv2i64>;
+  defm GLD1SH_D   : sve_mem_64b_gld_sv2_64_scaled<0b0100, "ld1sh",    AArch64ld1s_gather_scaled_z,   ZPR64ExtLSL16, nxv2i16>;
+  defm GLDFF1SH_D : sve_mem_64b_gld_sv2_64_scaled<0b0101, "ldff1sh",  AArch64ldff1s_gather_scaled_z, ZPR64ExtLSL16, nxv2i16>;
+  defm GLD1H_D    : sve_mem_64b_gld_sv2_64_scaled<0b0110, "ld1h",     AArch64ld1_gather_scaled_z,    ZPR64ExtLSL16, nxv2i16>;
+  defm GLDFF1H_D  : sve_mem_64b_gld_sv2_64_scaled<0b0111, "ldff1h",   AArch64ldff1_gather_scaled_z,  ZPR64ExtLSL16, nxv2i16>;
+  defm GLD1SW_D   : sve_mem_64b_gld_sv2_64_scaled<0b1000, "ld1sw",    AArch64ld1s_gather_scaled_z,   ZPR64ExtLSL32, nxv2i32>;
+  defm GLDFF1SW_D : sve_mem_64b_gld_sv2_64_scaled<0b1001, "ldff1sw",  AArch64ldff1s_gather_scaled_z, ZPR64ExtLSL32, nxv2i32>;
+  defm GLD1W_D    : sve_mem_64b_gld_sv2_64_scaled<0b1010, "ld1w",     AArch64ld1_gather_scaled_z,    ZPR64ExtLSL32, nxv2i32>;
+  defm GLDFF1W_D  : sve_mem_64b_gld_sv2_64_scaled<0b1011, "ldff1w",   AArch64ldff1_gather_scaled_z,  ZPR64ExtLSL32, nxv2i32>;
+  defm GLD1D      : sve_mem_64b_gld_sv2_64_scaled<0b1110, "ld1d",     AArch64ld1_gather_scaled_z,    ZPR64ExtLSL64, nxv2i64>;
+  defm GLDFF1D    : sve_mem_64b_gld_sv2_64_scaled<0b1111, "ldff1d",   AArch64ldff1_gather_scaled_z,  ZPR64ExtLSL64, nxv2i64>;
 
   // Gathers using unscaled 32-bit offsets unpacked in 64-bits elements, e.g.
   //    ld1h z0.d, p0/z, [x0, z0.d, uxtw]
-  defm GLD1SB_D   : sve_mem_64b_gld_vs_32_unscaled<0b0000, "ld1sb",   AArch64ld1s_gather_sxtw,   AArch64ld1s_gather_uxtw,   ZPR64ExtSXTW8Only, ZPR64ExtUXTW8Only, nxv2i8>;
-  defm GLDFF1SB_D : sve_mem_64b_gld_vs_32_unscaled<0b0001, "ldff1sb", AArch64ldff1s_gather_sxtw, AArch64ldff1s_gather_uxtw, ZPR64ExtSXTW8Only, ZPR64ExtUXTW8Only, nxv2i8>;
-  defm GLD1B_D    : sve_mem_64b_gld_vs_32_unscaled<0b0010, "ld1b",    AArch64ld1_gather_sxtw,    AArch64ld1_gather_uxtw,    ZPR64ExtSXTW8Only, ZPR64ExtUXTW8Only, nxv2i8>;
-  defm GLDFF1B_D  : sve_mem_64b_gld_vs_32_unscaled<0b0011, "ldff1b",  AArch64ldff1_gather_sxtw,  AArch64ldff1_gather_uxtw,  ZPR64ExtSXTW8Only, ZPR64ExtUXTW8Only, nxv2i8>;
-  defm GLD1SH_D   : sve_mem_64b_gld_vs_32_unscaled<0b0100, "ld1sh",   AArch64ld1s_gather_sxtw,   AArch64ld1s_gather_uxtw,   ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i16>;
-  defm GLDFF1SH_D : sve_mem_64b_gld_vs_32_unscaled<0b0101, "ldff1sh", AArch64ldff1s_gather_sxtw, AArch64ldff1s_gather_uxtw, ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i16>;
-  defm GLD1H_D    : sve_mem_64b_gld_vs_32_unscaled<0b0110, "ld1h",    AArch64ld1_gather_sxtw,    AArch64ld1_gather_uxtw,    ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i16>;
-  defm GLDFF1H_D  : sve_mem_64b_gld_vs_32_unscaled<0b0111, "ldff1h",  AArch64ldff1_gather_sxtw,  AArch64ldff1_gather_uxtw,  ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i16>;
-  defm GLD1SW_D   : sve_mem_64b_gld_vs_32_unscaled<0b1000, "ld1sw",   AArch64ld1s_gather_sxtw,   AArch64ld1s_gather_uxtw,   ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i32>;
-  defm GLDFF1SW_D : sve_mem_64b_gld_vs_32_unscaled<0b1001, "ldff1sw", AArch64ldff1s_gather_sxtw, AArch64ldff1s_gather_uxtw, ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i32>;
-  defm GLD1W_D    : sve_mem_64b_gld_vs_32_unscaled<0b1010, "ld1w",    AArch64ld1_gather_sxtw,    AArch64ld1_gather_uxtw,    ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i32>;
-  defm GLDFF1W_D  : sve_mem_64b_gld_vs_32_unscaled<0b1011, "ldff1w",  AArch64ldff1_gather_sxtw,  AArch64ldff1_gather_uxtw,  ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i32>;
-  defm GLD1D      : sve_mem_64b_gld_vs_32_unscaled<0b1110, "ld1d",    AArch64ld1_gather_sxtw,    AArch64ld1_gather_uxtw,    ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i64>;
-  defm GLDFF1D    : sve_mem_64b_gld_vs_32_unscaled<0b1111, "ldff1d",  AArch64ldff1_gather_sxtw,  AArch64ldff1_gather_uxtw,  ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i64>;
+  defm GLD1SB_D   : sve_mem_64b_gld_vs_32_unscaled<0b0000, "ld1sb",   AArch64ld1s_gather_sxtw_z,   AArch64ld1s_gather_uxtw_z,   ZPR64ExtSXTW8Only, ZPR64ExtUXTW8Only, nxv2i8>;
+  defm GLDFF1SB_D : sve_mem_64b_gld_vs_32_unscaled<0b0001, "ldff1sb", AArch64ldff1s_gather_sxtw_z, AArch64ldff1s_gather_uxtw_z, ZPR64ExtSXTW8Only, ZPR64ExtUXTW8Only, nxv2i8>;
+  defm GLD1B_D    : sve_mem_64b_gld_vs_32_unscaled<0b0010, "ld1b",    AArch64ld1_gather_sxtw_z,    AArch64ld1_gather_uxtw_z,    ZPR64ExtSXTW8Only, ZPR64ExtUXTW8Only, nxv2i8>;
+  defm GLDFF1B_D  : sve_mem_64b_gld_vs_32_unscaled<0b0011, "ldff1b",  AArch64ldff1_gather_sxtw_z,  AArch64ldff1_gather_uxtw_z,  ZPR64ExtSXTW8Only, ZPR64ExtUXTW8Only, nxv2i8>;
+  defm GLD1SH_D   : sve_mem_64b_gld_vs_32_unscaled<0b0100, "ld1sh",   AArch64ld1s_gather_sxtw_z,   AArch64ld1s_gather_uxtw_z,   ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i16>;
+  defm GLDFF1SH_D : sve_mem_64b_gld_vs_32_unscaled<0b0101, "ldff1sh", AArch64ldff1s_gather_sxtw_z, AArch64ldff1s_gather_uxtw_z, ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i16>;
+  defm GLD1H_D    : sve_mem_64b_gld_vs_32_unscaled<0b0110, "ld1h",    AArch64ld1_gather_sxtw_z,    AArch64ld1_gather_uxtw_z,    ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i16>;
+  defm GLDFF1H_D  : sve_mem_64b_gld_vs_32_unscaled<0b0111, "ldff1h",  AArch64ldff1_gather_sxtw_z,  AArch64ldff1_gather_uxtw_z,  ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i16>;
+  defm GLD1SW_D   : sve_mem_64b_gld_vs_32_unscaled<0b1000, "ld1sw",   AArch64ld1s_gather_sxtw_z,   AArch64ld1s_gather_uxtw_z,   ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i32>;
+  defm GLDFF1SW_D : sve_mem_64b_gld_vs_32_unscaled<0b1001, "ldff1sw", AArch64ldff1s_gather_sxtw_z, AArch64ldff1s_gather_uxtw_z, ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i32>;
+  defm GLD1W_D    : sve_mem_64b_gld_vs_32_unscaled<0b1010, "ld1w",    AArch64ld1_gather_sxtw_z,    AArch64ld1_gather_uxtw_z,    ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i32>;
+  defm GLDFF1W_D  : sve_mem_64b_gld_vs_32_unscaled<0b1011, "ldff1w",  AArch64ldff1_gather_sxtw_z,  AArch64ldff1_gather_uxtw_z,  ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i32>;
+  defm GLD1D      : sve_mem_64b_gld_vs_32_unscaled<0b1110, "ld1d",    AArch64ld1_gather_sxtw_z,    AArch64ld1_gather_uxtw_z,    ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i64>;
+  defm GLDFF1D    : sve_mem_64b_gld_vs_32_unscaled<0b1111, "ldff1d",  AArch64ldff1_gather_sxtw_z,  AArch64ldff1_gather_uxtw_z,  ZPR64ExtSXTW8, ZPR64ExtUXTW8, nxv2i64>;
 
   // Gathers using scaled 32-bit offsets unpacked in 64-bits elements, e.g.
   //    ld1h z0.d, p0/z, [x0, z0.d, uxtw #1]
-  defm GLD1SH_D   : sve_mem_64b_gld_sv_32_scaled<0b0100, "ld1sh",   AArch64ld1s_gather_sxtw_scaled,   AArch64ld1s_gather_uxtw_scaled,   ZPR64ExtSXTW16, ZPR64ExtUXTW16, nxv2i16>;
-  defm GLDFF1SH_D : sve_mem_64b_gld_sv_32_scaled<0b0101, "ldff1sh", AArch64ldff1s_gather_sxtw_scaled, AArch64ldff1s_gather_uxtw_scaled, ZPR64ExtSXTW16, ZPR64ExtUXTW16, nxv2i16>;
-  defm GLD1H_D    : sve_mem_64b_gld_sv_32_scaled<0b0110, "ld1h",    AArch64ld1_gather_sxtw_scaled,    AArch64ld1_gather_uxtw_scaled,    ZPR64ExtSXTW16, ZPR64ExtUXTW16, nxv2i16>;
-  defm GLDFF1H_D  : sve_mem_64b_gld_sv_32_scaled<0b0111, "ldff1h",  AArch64ldff1_gather_sxtw_scaled,  AArch64ldff1_gather_uxtw_scaled,  ZPR64ExtSXTW16, ZPR64ExtUXTW16, nxv2i16>;
-  defm GLD1SW_D   : sve_mem_64b_gld_sv_32_scaled<0b1000, "ld1sw",   AArch64ld1s_gather_sxtw_scaled,   AArch64ld1s_gather_uxtw_scaled,   ZPR64ExtSXTW32, ZPR64ExtUXTW32, nxv2i32>;
-  defm GLDFF1SW_D : sve_mem_64b_gld_sv_32_scaled<0b1001, "ldff1sw", AArch64ldff1s_gather_sxtw_scaled, AArch64ldff1s_gather_uxtw_scaled, ZPR64ExtSXTW32, ZPR64ExtUXTW32, nxv2i32>;
-  defm GLD1W_D    : sve_mem_64b_gld_sv_32_scaled<0b1010, "ld1w",    AArch64ld1_gather_sxtw_scaled,    AArch64ld1_gather_uxtw_scaled,    ZPR64ExtSXTW32, ZPR64ExtUXTW32, nxv2i32>;
-  defm GLDFF1W_D  : sve_mem_64b_gld_sv_32_scaled<0b1011, "ldff1w",  AArch64ldff1_gather_sxtw_scaled,  AArch64ldff1_gather_uxtw_scaled,  ZPR64ExtSXTW32, ZPR64ExtUXTW32, nxv2i32>;
-  defm GLD1D      : sve_mem_64b_gld_sv_32_scaled<0b1110, "ld1d",    AArch64ld1_gather_sxtw_scaled,    AArch64ld1_gather_uxtw_scaled,    ZPR64ExtSXTW64, ZPR64ExtUXTW64, nxv2i64>;
-  defm GLDFF1D    : sve_mem_64b_gld_sv_32_scaled<0b1111, "ldff1d",  AArch64ldff1_gather_sxtw_scaled,  AArch64ldff1_gather_uxtw_scaled,  ZPR64ExtSXTW64, ZPR64ExtUXTW64, nxv2i64>;
+  defm GLD1SH_D   : sve_mem_64b_gld_sv_32_scaled<0b0100, "ld1sh",   AArch64ld1s_gather_sxtw_scaled_z,   AArch64ld1s_gather_uxtw_scaled_z,   ZPR64ExtSXTW16, ZPR64ExtUXTW16, nxv2i16>;
+  defm GLDFF1SH_D : sve_mem_64b_gld_sv_32_scaled<0b0101, "ldff1sh", AArch64ldff1s_gather_sxtw_scaled_z, AArch64ldff1s_gather_uxtw_scaled_z, ZPR64ExtSXTW16, ZPR64ExtUXTW16, nxv2i16>;
+  defm GLD1H_D    : sve_mem_64b_gld_sv_32_scaled<0b0110, "ld1h",    AArch64ld1_gather_sxtw_scaled_z,    AArch64ld1_gather_uxtw_scaled_z,    ZPR64ExtSXTW16, ZPR64ExtUXTW16, nxv2i16>;
+  defm GLDFF1H_D  : sve_mem_64b_gld_sv_32_scaled<0b0111, "ldff1h",  AArch64ldff1_gather_sxtw_scaled_z,  AArch64ldff1_gather_uxtw_scaled_z,  ZPR64ExtSXTW16, ZPR64ExtUXTW16, nxv2i16>;
+  defm GLD1SW_D   : sve_mem_64b_gld_sv_32_scaled<0b1000, "ld1sw",   AArch64ld1s_gather_sxtw_scaled_z,   AArch64ld1s_gather_uxtw_scaled_z,   ZPR64ExtSXTW32, ZPR64ExtUXTW32, nxv2i32>;
+  defm GLDFF1SW_D : sve_mem_64b_gld_sv_32_scaled<0b1001, "ldff1sw", AArch64ldff1s_gather_sxtw_scaled_z, AArch64ldff1s_gather_uxtw_scaled_z, ZPR64ExtSXTW32, ZPR64ExtUXTW32, nxv2i32>;
+  defm GLD1W_D    : sve_mem_64b_gld_sv_32_scaled<0b1010, "ld1w",    AArch64ld1_gather_sxtw_scaled_z,    AArch64ld1_gather_uxtw_scaled_z,    ZPR64ExtSXTW32, ZPR64ExtUXTW32, nxv2i32>;
+  defm GLDFF1W_D  : sve_mem_64b_gld_sv_32_scaled<0b1011, "ldff1w",  AArch64ldff1_gather_sxtw_scaled_z,  AArch64ldff1_gather_uxtw_scaled_z,  ZPR64ExtSXTW32, ZPR64ExtUXTW32, nxv2i32>;
+  defm GLD1D      : sve_mem_64b_gld_sv_32_scaled<0b1110, "ld1d",    AArch64ld1_gather_sxtw_scaled_z,    AArch64ld1_gather_uxtw_scaled_z,    ZPR64ExtSXTW64, ZPR64ExtUXTW64, nxv2i64>;
+  defm GLDFF1D    : sve_mem_64b_gld_sv_32_scaled<0b1111, "ldff1d",  AArch64ldff1_gather_sxtw_scaled_z,  AArch64ldff1_gather_uxtw_scaled_z,  ZPR64ExtSXTW64, ZPR64ExtUXTW64, nxv2i64>;
 
   // Non-temporal contiguous loads (register + immediate)
   defm LDNT1B_ZRI : sve_mem_cldnt_si<0b00, "ldnt1b", Z_b, ZPR8>;
@@ -992,6 +1082,10 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
 
   defm TBL_ZZZ  : sve_int_perm_tbl<"tbl", AArch64tbl>;
 
+  let Predicates = [HasSVE, HasBF16] in {
+    def : SVE_2_Op_Pat<nxv8bf16, AArch64tbl, nxv8bf16, nxv8i16, TBL_ZZZ_H>;
+  }
+
   defm ZIP1_ZZZ : sve_int_perm_bin_perm_zz<0b000, "zip1", AArch64zip1>;
   defm ZIP2_ZZZ : sve_int_perm_bin_perm_zz<0b001, "zip2", AArch64zip2>;
   defm UZP1_ZZZ : sve_int_perm_bin_perm_zz<0b010, "uzp1", AArch64uzp1>;
@@ -999,6 +1093,15 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   defm TRN1_ZZZ : sve_int_perm_bin_perm_zz<0b100, "trn1", AArch64trn1>;
   defm TRN2_ZZZ : sve_int_perm_bin_perm_zz<0b101, "trn2", AArch64trn2>;
 
+  let Predicates = [HasSVE, HasBF16] in {
+    def : SVE_2_Op_Pat<nxv8bf16, AArch64zip1, nxv8bf16, nxv8bf16, ZIP1_ZZZ_H>;
+    def : SVE_2_Op_Pat<nxv8bf16, AArch64zip2, nxv8bf16, nxv8bf16, ZIP2_ZZZ_H>;
+    def : SVE_2_Op_Pat<nxv8bf16, AArch64uzp1, nxv8bf16, nxv8bf16, UZP1_ZZZ_H>;
+    def : SVE_2_Op_Pat<nxv8bf16, AArch64uzp2, nxv8bf16, nxv8bf16, UZP2_ZZZ_H>;
+    def : SVE_2_Op_Pat<nxv8bf16, AArch64trn1, nxv8bf16, nxv8bf16, TRN1_ZZZ_H>;
+    def : SVE_2_Op_Pat<nxv8bf16, AArch64trn2, nxv8bf16, nxv8bf16, TRN2_ZZZ_H>;
+  }
+
   defm ZIP1_PPP : sve_int_perm_bin_perm_pp<0b000, "zip1", AArch64zip1>;
   defm ZIP2_PPP : sve_int_perm_bin_perm_pp<0b001, "zip2", AArch64zip2>;
   defm UZP1_PPP : sve_int_perm_bin_perm_pp<0b010, "uzp1", AArch64uzp1>;
@@ -1163,9 +1266,9 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   defm INDEX_II : sve_int_index_ii<"index", index_vector>;
 
   // Unpredicated shifts
-  defm ASR_ZZI : sve_int_bin_cons_shift_imm_right<0b00, "asr", AArch64asr_pred>;
-  defm LSR_ZZI : sve_int_bin_cons_shift_imm_right<0b01, "lsr", AArch64lsr_pred>;
-  defm LSL_ZZI : sve_int_bin_cons_shift_imm_left< 0b11, "lsl", AArch64lsl_pred>;
+  defm ASR_ZZI : sve_int_bin_cons_shift_imm_right<0b00, "asr", AArch64asr_m1>;
+  defm LSR_ZZI : sve_int_bin_cons_shift_imm_right<0b01, "lsr", AArch64lsr_m1>;
+  defm LSL_ZZI : sve_int_bin_cons_shift_imm_left< 0b11, "lsl", AArch64lsl_m1>;
 
   defm ASR_WIDE_ZZZ : sve_int_bin_cons_shift_wide<0b00, "asr">;
   defm LSR_WIDE_ZZZ : sve_int_bin_cons_shift_wide<0b01, "lsr">;
@@ -1177,17 +1280,19 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   defm LSL_ZPmI  : sve_int_bin_pred_shift_imm_left< 0b0011, "lsl">;
   defm ASRD_ZPmI : sve_int_bin_pred_shift_imm_right<0b0100, "asrd", "ASRD_ZPZI", int_aarch64_sve_asrd>;
 
-  defm ASR_ZPZZ  : sve_int_bin_pred_zx<AArch64asr_pred>;
-  defm LSR_ZPZZ  : sve_int_bin_pred_zx<AArch64lsr_pred>;
-  defm LSL_ZPZZ  : sve_int_bin_pred_zx<AArch64lsl_pred>;
-  defm ASRD_ZPZI : sve_int_bin_pred_shift_0_right_zx<int_aarch64_sve_asrd>;
+  let Predicates = [HasSVE, UseExperimentalZeroingPseudos] in {
+    defm ASR_ZPZZ    : sve_int_bin_pred_zeroing_bhsd<AArch64asr_m1>;
+    defm LSR_ZPZZ    : sve_int_bin_pred_zeroing_bhsd<AArch64lsr_m1>;
+    defm LSL_ZPZZ    : sve_int_bin_pred_zeroing_bhsd<AArch64lsl_m1>;
+    defm ASRD_ZPZI   : sve_int_bin_pred_shift_imm_right_zeroing_bhsd<int_aarch64_sve_asrd>;
+  }
 
-  defm ASR_ZPmZ  : sve_int_bin_pred_shift<0b000, "asr", "ASR_ZPZZ", AArch64asr_pred, "ASRR_ZPmZ", 1>;
-  defm LSR_ZPmZ  : sve_int_bin_pred_shift<0b001, "lsr", "LSR_ZPZZ", AArch64lsr_pred, "LSRR_ZPmZ", 1>;
-  defm LSL_ZPmZ  : sve_int_bin_pred_shift<0b011, "lsl", "LSL_ZPZZ", AArch64lsl_pred, "LSLR_ZPmZ", 1>;
-  defm ASRR_ZPmZ : sve_int_bin_pred_shift<0b100, "asrr", "ASRR_ZPZZ", null_frag, "ASR_ZPmZ", 0>;
-  defm LSRR_ZPmZ : sve_int_bin_pred_shift<0b101, "lsrr", "LSRR_ZPZZ", null_frag, "LSR_ZPmZ", 0>;
-  defm LSLR_ZPmZ : sve_int_bin_pred_shift<0b111, "lslr", "LSLR_ZPZZ", null_frag, "LSL_ZPmZ", 0>;
+  defm ASR_ZPmZ  : sve_int_bin_pred_shift<0b000, "asr", "ASR_ZPZZ", AArch64asr_m1, "ASRR_ZPmZ">;
+  defm LSR_ZPmZ  : sve_int_bin_pred_shift<0b001, "lsr", "LSR_ZPZZ", AArch64lsr_m1, "LSRR_ZPmZ">;
+  defm LSL_ZPmZ  : sve_int_bin_pred_shift<0b011, "lsl", "LSL_ZPZZ", AArch64lsl_m1, "LSLR_ZPmZ">;
+  defm ASRR_ZPmZ : sve_int_bin_pred_shift<0b100, "asrr", "ASRR_ZPZZ", null_frag, "ASR_ZPmZ", /*isReverseInstr*/ 1>;
+  defm LSRR_ZPmZ : sve_int_bin_pred_shift<0b101, "lsrr", "LSRR_ZPZZ", null_frag, "LSR_ZPmZ", /*isReverseInstr*/ 1>;
+  defm LSLR_ZPmZ : sve_int_bin_pred_shift<0b111, "lslr", "LSLR_ZPZZ", null_frag, "LSL_ZPmZ", /*isReverseInstr*/ 1>;
 
   defm ASR_WIDE_ZPmZ : sve_int_bin_pred_shift_wide<0b000, "asr", int_aarch64_sve_asr_wide>;
   defm LSR_WIDE_ZPmZ : sve_int_bin_pred_shift_wide<0b001, "lsr", int_aarch64_sve_lsr_wide>;
@@ -1239,15 +1344,15 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   defm FSQRT_ZPmZ  : sve_fp_2op_p_zd_HSD<0b01101, "fsqrt",  int_aarch64_sve_fsqrt>;
 
   let Predicates = [HasBF16, HasSVE] in {
-    def BFDOT_ZZZ    : sve_bfloat_dot<"bfdot">;
-    def BFDOT_ZZI    : sve_bfloat_dot_indexed<"bfdot">;
-    def BFMMLA_ZZZ  : sve_bfloat_matmul<"bfmmla">;
-    def BFMMLA_B_ZZZ  : sve_bfloat_matmul_longvecl<0b0, "bfmlalb">;
-    def BFMMLA_T_ZZZ  : sve_bfloat_matmul_longvecl<0b1, "bfmlalt">;
-    def BFMMLA_B_ZZI  : sve_bfloat_matmul_longvecl_idx<0b0, "bfmlalb">;
-    def BFMMLA_T_ZZI  : sve_bfloat_matmul_longvecl_idx<0b1, "bfmlalt">;
-    def BFCVT_ZPmZ   : sve_bfloat_convert<0b1, "bfcvt">;
-    def BFCVTNT_ZPmZ : sve_bfloat_convert<0b0, "bfcvtnt">;
+    defm BFDOT_ZZZ    : sve_bfloat_dot<"bfdot", int_aarch64_sve_bfdot>;
+    defm BFDOT_ZZI    : sve_bfloat_dot_indexed<"bfdot", int_aarch64_sve_bfdot_lane>;
+    defm BFMMLA_ZZZ   : sve_bfloat_matmul<"bfmmla", int_aarch64_sve_bfmmla>;
+    defm BFMMLA_B_ZZZ : sve_bfloat_matmul_longvecl<0b0, "bfmlalb", int_aarch64_sve_bfmlalb>;
+    defm BFMMLA_T_ZZZ : sve_bfloat_matmul_longvecl<0b1, "bfmlalt", int_aarch64_sve_bfmlalt>;
+    defm BFMMLA_B_ZZI : sve_bfloat_matmul_longvecl_idx<0b0, "bfmlalb", int_aarch64_sve_bfmlalb_lane>;
+    defm BFMMLA_T_ZZI : sve_bfloat_matmul_longvecl_idx<0b1, "bfmlalt", int_aarch64_sve_bfmlalt_lane>;
+    defm BFCVT_ZPmZ   : sve_bfloat_convert<0b1, "bfcvt",   int_aarch64_sve_fcvt_bf16f32>;
+    defm BFCVTNT_ZPmZ : sve_bfloat_convert<0b0, "bfcvtnt", int_aarch64_sve_fcvtnt_bf16f32>;
   }
 
   // InstAliases
@@ -1359,22 +1464,22 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
             (PTEST_PP PPR:$pg, PPR:$src)>;
 
   // LD1R of 128-bit masked data
-  def : Pat<(nxv16i8 (AArch64ld1rq PPR:$gp, GPR64:$base)),
+  def : Pat<(nxv16i8 (AArch64ld1rq_z PPR:$gp, GPR64:$base)),
             (LD1RQ_B_IMM $gp, $base, (i64 0))>;
-  def : Pat<(nxv8i16 (AArch64ld1rq PPR:$gp, GPR64:$base)),
+  def : Pat<(nxv8i16 (AArch64ld1rq_z PPR:$gp, GPR64:$base)),
             (LD1RQ_H_IMM $gp, $base, (i64 0))>;
-  def : Pat<(nxv4i32 (AArch64ld1rq PPR:$gp, GPR64:$base)),
+  def : Pat<(nxv4i32 (AArch64ld1rq_z PPR:$gp, GPR64:$base)),
             (LD1RQ_W_IMM $gp, $base, (i64 0))>;
-  def : Pat<(nxv2i64 (AArch64ld1rq PPR:$gp, GPR64:$base)),
+  def : Pat<(nxv2i64 (AArch64ld1rq_z PPR:$gp, GPR64:$base)),
             (LD1RQ_D_IMM $gp, $base, (i64 0))>;
 
-  def : Pat<(nxv16i8 (AArch64ld1rq PPR:$gp, (add GPR64:$base, (i64 simm4s16:$imm)))),
+  def : Pat<(nxv16i8 (AArch64ld1rq_z PPR:$gp, (add GPR64:$base, (i64 simm4s16:$imm)))),
             (LD1RQ_B_IMM $gp, $base, simm4s16:$imm)>;
-  def : Pat<(nxv8i16 (AArch64ld1rq PPR:$gp, (add GPR64:$base, (i64 simm4s16:$imm)))),
+  def : Pat<(nxv8i16 (AArch64ld1rq_z PPR:$gp, (add GPR64:$base, (i64 simm4s16:$imm)))),
             (LD1RQ_H_IMM $gp, $base, simm4s16:$imm)>;
-  def : Pat<(nxv4i32 (AArch64ld1rq PPR:$gp, (add GPR64:$base, (i64 simm4s16:$imm)))),
+  def : Pat<(nxv4i32 (AArch64ld1rq_z PPR:$gp, (add GPR64:$base, (i64 simm4s16:$imm)))),
             (LD1RQ_W_IMM $gp, $base, simm4s16:$imm)>;
-  def : Pat<(nxv2i64 (AArch64ld1rq PPR:$gp, (add GPR64:$base, (i64 simm4s16:$imm)))),
+  def : Pat<(nxv2i64 (AArch64ld1rq_z PPR:$gp, (add GPR64:$base, (i64 simm4s16:$imm)))),
             (LD1RQ_D_IMM $gp, $base, simm4s16:$imm)>;
 
   def : Pat<(sext_inreg (nxv2i64 ZPR:$Zs), nxv2i32), (SXTW_ZPmZ_D (IMPLICIT_DEF), (PTRUE_D 31), ZPR:$Zs)>;
@@ -1435,7 +1540,6 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
 
     def : Pat<(nxv8f16 (bitconvert (nxv16i8 ZPR:$src))), (nxv8f16 ZPR:$src)>;
     def : Pat<(nxv8f16 (bitconvert (nxv8i16 ZPR:$src))), (nxv8f16 ZPR:$src)>;
-    def : Pat<(nxv8bf16 (bitconvert (nxv8i16 ZPR:$src))), (nxv8bf16 ZPR:$src)>;
     def : Pat<(nxv8f16 (bitconvert (nxv4i32 ZPR:$src))), (nxv8f16 ZPR:$src)>;
     def : Pat<(nxv8f16 (bitconvert (nxv2i64 ZPR:$src))), (nxv8f16 ZPR:$src)>;
     def : Pat<(nxv8f16 (bitconvert (nxv4f32 ZPR:$src))), (nxv8f16 ZPR:$src)>;
@@ -1454,6 +1558,31 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
     def : Pat<(nxv2f64 (bitconvert (nxv2i64 ZPR:$src))), (nxv2f64 ZPR:$src)>;
     def : Pat<(nxv2f64 (bitconvert (nxv8f16 ZPR:$src))), (nxv2f64 ZPR:$src)>;
     def : Pat<(nxv2f64 (bitconvert (nxv4f32 ZPR:$src))), (nxv2f64 ZPR:$src)>;
+
+  }
+
+  let Predicates = [IsLE, HasBF16, HasSVE] in {
+    def : Pat<(nxv2i64  (bitconvert (nxv8bf16 ZPR:$src))), (nxv2i64 ZPR:$src)>;
+    def : Pat<(nxv8bf16 (bitconvert (nxv2i64 ZPR:$src))),  (nxv8bf16 ZPR:$src)>;
+    def : Pat<(nxv8bf16 (bitconvert (nxv8i16 ZPR:$src))),  (nxv8bf16 ZPR:$src)>;
+  }
+
+  let Predicates = [IsLE, HasSVE, HasBF16] in {
+    def : Pat<(nxv8bf16 (bitconvert (nxv16i8 ZPR:$src))), (nxv8bf16 ZPR:$src)>;
+    def : Pat<(nxv8bf16 (bitconvert (nxv8i16 ZPR:$src))), (nxv8bf16 ZPR:$src)>;
+    def : Pat<(nxv8bf16 (bitconvert (nxv4i32 ZPR:$src))), (nxv8bf16 ZPR:$src)>;
+    def : Pat<(nxv8bf16 (bitconvert (nxv2i64 ZPR:$src))), (nxv8bf16 ZPR:$src)>;
+    def : Pat<(nxv8bf16 (bitconvert (nxv8f16 ZPR:$src))), (nxv8bf16 ZPR:$src)>;
+    def : Pat<(nxv8bf16 (bitconvert (nxv4f32 ZPR:$src))), (nxv8bf16 ZPR:$src)>;
+    def : Pat<(nxv8bf16 (bitconvert (nxv2f64 ZPR:$src))), (nxv8bf16 ZPR:$src)>;
+
+    def : Pat<(nxv16i8 (bitconvert (nxv8bf16 ZPR:$src))), (nxv16i8 ZPR:$src)>;
+    def : Pat<(nxv8i16 (bitconvert (nxv8bf16 ZPR:$src))), (nxv8i16 ZPR:$src)>;
+    def : Pat<(nxv4i32 (bitconvert (nxv8bf16 ZPR:$src))), (nxv4i32 ZPR:$src)>;
+    def : Pat<(nxv2i64 (bitconvert (nxv8bf16 ZPR:$src))), (nxv2i64 ZPR:$src)>;
+    def : Pat<(nxv8f16 (bitconvert (nxv8bf16 ZPR:$src))), (nxv8f16 ZPR:$src)>;
+    def : Pat<(nxv4f32 (bitconvert (nxv8bf16 ZPR:$src))), (nxv4f32 ZPR:$src)>;
+    def : Pat<(nxv2f64 (bitconvert (nxv8bf16 ZPR:$src))), (nxv2f64 ZPR:$src)>;
   }
 
   def : Pat<(nxv16i1 (reinterpret_cast (nxv16i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
@@ -1518,10 +1647,14 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   defm : pred_load<nxv4f32, nxv4i1, nonext_masked_load,    LD1W,    LD1W_IMM,    am_sve_regreg_lsl2>;
 
   // 8-element contiguous loads
-  defm : pred_load<nxv8i16, nxv8i1, zext_masked_load_i8,  LD1B_H,  LD1B_H_IMM,  am_sve_regreg_lsl0>;
-  defm : pred_load<nxv8i16, nxv8i1, asext_masked_load_i8, LD1SB_H, LD1SB_H_IMM, am_sve_regreg_lsl0>;
-  defm : pred_load<nxv8i16, nxv8i1, nonext_masked_load,   LD1H,    LD1H_IMM,    am_sve_regreg_lsl1>;
-  defm : pred_load<nxv8f16, nxv8i1, nonext_masked_load,   LD1H,    LD1H_IMM,    am_sve_regreg_lsl1>;
+  defm : pred_load<nxv8i16,  nxv8i1, zext_masked_load_i8,  LD1B_H,  LD1B_H_IMM,  am_sve_regreg_lsl0>;
+  defm : pred_load<nxv8i16,  nxv8i1, asext_masked_load_i8, LD1SB_H, LD1SB_H_IMM, am_sve_regreg_lsl0>;
+  defm : pred_load<nxv8i16,  nxv8i1, nonext_masked_load,   LD1H,    LD1H_IMM,    am_sve_regreg_lsl1>;
+  defm : pred_load<nxv8f16,  nxv8i1, nonext_masked_load,   LD1H,    LD1H_IMM,    am_sve_regreg_lsl1>;
+
+  let Predicates = [HasBF16, HasSVE] in {
+    defm : pred_load<nxv8bf16, nxv8i1, nonext_masked_load,   LD1H,    LD1H_IMM,    am_sve_regreg_lsl1>;
+  }
 
   // 16-element contiguous loads
   defm : pred_load<nxv16i8, nxv16i1, nonext_masked_load, LD1B, LD1B_IMM, am_sve_regreg_lsl0>;
@@ -1563,6 +1696,10 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   defm : pred_store<nxv8i16, nxv8i1, nontrunc_masked_store, ST1H,   ST1H_IMM,   am_sve_regreg_lsl1>;
   defm : pred_store<nxv8f16, nxv8i1, nontrunc_masked_store, ST1H,   ST1H_IMM,   am_sve_regreg_lsl1>;
 
+  let Predicates = [HasBF16, HasSVE] in {
+    defm : pred_store<nxv8bf16, nxv8i1, nontrunc_masked_store, ST1H,   ST1H_IMM,   am_sve_regreg_lsl1>;
+  }
+
   // 16-element contiguous stores
   defm : pred_store<nxv16i8, nxv16i1, nontrunc_masked_store, ST1B, ST1B_IMM, am_sve_regreg_lsl0>;
 
@@ -1578,8 +1715,14 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
 
   multiclass unpred_store<PatFrag Store, ValueType Ty, Instruction RegImmInst,
                           Instruction PTrue> {
-    def : Pat<(Store (Ty ZPR:$val), (am_sve_fi GPR64sp:$base, simm4s1:$offset)),
-              (RegImmInst ZPR:$val, (PTrue 31), GPR64sp:$base, simm4s1:$offset)>;
+    let AddedComplexity = 1 in {
+      def _imm : Pat<(Store (Ty ZPR:$val), (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset)),
+                     (RegImmInst ZPR:$val, (PTrue 31), GPR64sp:$base, simm4s1:$offset)>;
+    }
+    let AddedComplexity = 2 in {
+      def _fi : Pat<(Store (Ty ZPR:$val), (am_sve_fi GPR64sp:$base, simm4s1:$offset)),
+                    (RegImmInst ZPR:$val, (PTrue 31), GPR64sp:$base, simm4s1:$offset)>;
+    }
 
     def : Pat<(Store (Ty ZPR:$val), GPR64:$base),
               (RegImmInst ZPR:$val, (PTrue 31), GPR64:$base, (i64 0))>;
@@ -1596,6 +1739,7 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   defm : unpred_store<truncstorevi32, nxv2i64, ST1W_D_IMM, PTRUE_D>;
   defm : unpred_store<         store, nxv2i64,   ST1D_IMM, PTRUE_D>;
   defm : unpred_store<         store, nxv8f16,   ST1H_IMM, PTRUE_H>;
+  defm : unpred_store<         store, nxv8bf16,  ST1H_IMM, PTRUE_H>;
   defm : unpred_store<         store, nxv4f16, ST1H_S_IMM, PTRUE_S>;
   defm : unpred_store<         store, nxv2f16, ST1H_D_IMM, PTRUE_D>;
   defm : unpred_store<         store, nxv4f32,   ST1W_IMM, PTRUE_S>;
@@ -1604,8 +1748,15 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
 
   multiclass unpred_load<PatFrag Load, ValueType Ty, Instruction RegImmInst,
                          Instruction PTrue> {
-    def : Pat<(Ty (Load  (am_sve_fi GPR64sp:$base, simm4s1:$offset))),
-              (RegImmInst (PTrue 31), GPR64sp:$base, simm4s1:$offset)>;
+    let AddedComplexity = 1 in {
+      def _imm: Pat<(Ty (Load  (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset))),
+                    (RegImmInst (PTrue 31), GPR64sp:$base, simm4s1:$offset)>;
+    }
+
+    let AddedComplexity = 2 in {
+      def _fi : Pat<(Ty (Load  (am_sve_fi GPR64sp:$base, simm4s1:$offset))),
+                    (RegImmInst (PTrue 31), GPR64sp:$base, simm4s1:$offset)>;
+    }
 
     def : Pat<(Ty (Load GPR64:$base)),
               (RegImmInst (PTrue 31), GPR64:$base, (i64 0))>;
@@ -1634,6 +1785,7 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   defm : unpred_load<sextloadvi32, nxv2i64, LD1SW_D_IMM, PTRUE_D>;
   defm : unpred_load<        load, nxv2i64,    LD1D_IMM, PTRUE_D>;
   defm : unpred_load<        load, nxv8f16,    LD1H_IMM, PTRUE_H>;
+  defm : unpred_load<        load, nxv8bf16,   LD1H_IMM, PTRUE_H>;
   defm : unpred_load<        load, nxv4f16,  LD1H_S_IMM, PTRUE_S>;
   defm : unpred_load<        load, nxv2f16,  LD1H_D_IMM, PTRUE_D>;
   defm : unpred_load<        load, nxv4f32,    LD1W_IMM, PTRUE_S>;
@@ -1686,31 +1838,35 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   }
 
   // 2-element contiguous loads
-  defm : ld1<LD1B_D,  LD1B_D_IMM,  nxv2i64, AArch64ld1,  nxv2i1, nxv2i8,  am_sve_regreg_lsl0>;
-  defm : ld1<LD1SB_D, LD1SB_D_IMM, nxv2i64, AArch64ld1s, nxv2i1, nxv2i8,  am_sve_regreg_lsl0>;
-  defm : ld1<LD1H_D,  LD1H_D_IMM,  nxv2i64, AArch64ld1,  nxv2i1, nxv2i16, am_sve_regreg_lsl1>;
-  defm : ld1<LD1SH_D, LD1SH_D_IMM, nxv2i64, AArch64ld1s, nxv2i1, nxv2i16, am_sve_regreg_lsl1>;
-  defm : ld1<LD1W_D,  LD1W_D_IMM,  nxv2i64, AArch64ld1,  nxv2i1, nxv2i32, am_sve_regreg_lsl2>;
-  defm : ld1<LD1SW_D, LD1SW_D_IMM, nxv2i64, AArch64ld1s, nxv2i1, nxv2i32, am_sve_regreg_lsl2>;
-  defm : ld1<LD1D,    LD1D_IMM,    nxv2i64, AArch64ld1,  nxv2i1, nxv2i64, am_sve_regreg_lsl3>;
-  defm : ld1<LD1D,    LD1D_IMM,    nxv2f64, AArch64ld1,  nxv2i1, nxv2f64, am_sve_regreg_lsl3>;
+  defm : ld1<LD1B_D,  LD1B_D_IMM,  nxv2i64, AArch64ld1_z,  nxv2i1, nxv2i8,  am_sve_regreg_lsl0>;
+  defm : ld1<LD1SB_D, LD1SB_D_IMM, nxv2i64, AArch64ld1s_z, nxv2i1, nxv2i8,  am_sve_regreg_lsl0>;
+  defm : ld1<LD1H_D,  LD1H_D_IMM,  nxv2i64, AArch64ld1_z,  nxv2i1, nxv2i16, am_sve_regreg_lsl1>;
+  defm : ld1<LD1SH_D, LD1SH_D_IMM, nxv2i64, AArch64ld1s_z, nxv2i1, nxv2i16, am_sve_regreg_lsl1>;
+  defm : ld1<LD1W_D,  LD1W_D_IMM,  nxv2i64, AArch64ld1_z,  nxv2i1, nxv2i32, am_sve_regreg_lsl2>;
+  defm : ld1<LD1SW_D, LD1SW_D_IMM, nxv2i64, AArch64ld1s_z, nxv2i1, nxv2i32, am_sve_regreg_lsl2>;
+  defm : ld1<LD1D,    LD1D_IMM,    nxv2i64, AArch64ld1_z,  nxv2i1, nxv2i64, am_sve_regreg_lsl3>;
+  defm : ld1<LD1D,    LD1D_IMM,    nxv2f64, AArch64ld1_z,  nxv2i1, nxv2f64, am_sve_regreg_lsl3>;
 
   // 4-element contiguous loads
-  defm : ld1<LD1B_S,  LD1B_S_IMM,  nxv4i32, AArch64ld1,  nxv4i1, nxv4i8,  am_sve_regreg_lsl0>;
-  defm : ld1<LD1SB_S, LD1SB_S_IMM, nxv4i32, AArch64ld1s, nxv4i1, nxv4i8,  am_sve_regreg_lsl0>;
-  defm : ld1<LD1H_S,  LD1H_S_IMM,  nxv4i32, AArch64ld1,  nxv4i1, nxv4i16, am_sve_regreg_lsl1>;
-  defm : ld1<LD1SH_S, LD1SH_S_IMM, nxv4i32, AArch64ld1s, nxv4i1, nxv4i16, am_sve_regreg_lsl1>;
-  defm : ld1<LD1W,    LD1W_IMM,    nxv4i32, AArch64ld1,  nxv4i1, nxv4i32, am_sve_regreg_lsl2>;
-  defm : ld1<LD1W,    LD1W_IMM,    nxv4f32, AArch64ld1,  nxv4i1, nxv4f32, am_sve_regreg_lsl2>;
+  defm : ld1<LD1B_S,  LD1B_S_IMM,  nxv4i32, AArch64ld1_z,  nxv4i1, nxv4i8,  am_sve_regreg_lsl0>;
+  defm : ld1<LD1SB_S, LD1SB_S_IMM, nxv4i32, AArch64ld1s_z, nxv4i1, nxv4i8,  am_sve_regreg_lsl0>;
+  defm : ld1<LD1H_S,  LD1H_S_IMM,  nxv4i32, AArch64ld1_z,  nxv4i1, nxv4i16, am_sve_regreg_lsl1>;
+  defm : ld1<LD1SH_S, LD1SH_S_IMM, nxv4i32, AArch64ld1s_z, nxv4i1, nxv4i16, am_sve_regreg_lsl1>;
+  defm : ld1<LD1W,    LD1W_IMM,    nxv4i32, AArch64ld1_z,  nxv4i1, nxv4i32, am_sve_regreg_lsl2>;
+  defm : ld1<LD1W,    LD1W_IMM,    nxv4f32, AArch64ld1_z,  nxv4i1, nxv4f32, am_sve_regreg_lsl2>;
 
   // 8-element contiguous loads
-  defm : ld1<LD1B_H,  LD1B_H_IMM,  nxv8i16, AArch64ld1,  nxv8i1, nxv8i8,  am_sve_regreg_lsl0>;
-  defm : ld1<LD1SB_H, LD1SB_H_IMM, nxv8i16, AArch64ld1s, nxv8i1, nxv8i8,  am_sve_regreg_lsl0>;
-  defm : ld1<LD1H,    LD1H_IMM,    nxv8i16, AArch64ld1,  nxv8i1, nxv8i16, am_sve_regreg_lsl1>;
-  defm : ld1<LD1H,    LD1H_IMM,    nxv8f16, AArch64ld1,  nxv8i1, nxv8f16, am_sve_regreg_lsl1>;
+  defm : ld1<LD1B_H,  LD1B_H_IMM,  nxv8i16,  AArch64ld1_z,  nxv8i1, nxv8i8,   am_sve_regreg_lsl0>;
+  defm : ld1<LD1SB_H, LD1SB_H_IMM, nxv8i16,  AArch64ld1s_z, nxv8i1, nxv8i8,   am_sve_regreg_lsl0>;
+  defm : ld1<LD1H,    LD1H_IMM,    nxv8i16,  AArch64ld1_z,  nxv8i1, nxv8i16,  am_sve_regreg_lsl1>;
+  defm : ld1<LD1H,    LD1H_IMM,    nxv8f16,  AArch64ld1_z,  nxv8i1, nxv8f16,  am_sve_regreg_lsl1>;
+
+  let Predicates = [HasBF16, HasSVE] in {
+    defm : ld1<LD1H,    LD1H_IMM,    nxv8bf16, AArch64ld1_z,  nxv8i1, nxv8bf16, am_sve_regreg_lsl1>;
+  }
 
   // 16-element contiguous loads
-  defm : ld1<LD1B, LD1B_IMM, nxv16i8, AArch64ld1, nxv16i1, nxv16i8, am_sve_regreg_lsl0>;
+  defm : ld1<LD1B, LD1B_IMM, nxv16i8, AArch64ld1_z, nxv16i1, nxv16i8, am_sve_regreg_lsl0>;
 
   multiclass ldnf1<Instruction I, ValueType Ty, SDPatternOperator Load, ValueType PredTy, ValueType MemVT> {
     // scalar + immediate (mul vl)
@@ -1725,31 +1881,35 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   }
 
   // 2-element contiguous non-faulting loads
-  defm : ldnf1<LDNF1B_D_IMM,  nxv2i64, AArch64ldnf1,  nxv2i1, nxv2i8>;
-  defm : ldnf1<LDNF1SB_D_IMM, nxv2i64, AArch64ldnf1s, nxv2i1, nxv2i8>;
-  defm : ldnf1<LDNF1H_D_IMM,  nxv2i64, AArch64ldnf1,  nxv2i1, nxv2i16>;
-  defm : ldnf1<LDNF1SH_D_IMM, nxv2i64, AArch64ldnf1s, nxv2i1, nxv2i16>;
-  defm : ldnf1<LDNF1W_D_IMM,  nxv2i64, AArch64ldnf1,  nxv2i1, nxv2i32>;
-  defm : ldnf1<LDNF1SW_D_IMM, nxv2i64, AArch64ldnf1s, nxv2i1, nxv2i32>;
-  defm : ldnf1<LDNF1D_IMM,    nxv2i64, AArch64ldnf1,  nxv2i1, nxv2i64>;
-  defm : ldnf1<LDNF1D_IMM,    nxv2f64, AArch64ldnf1,  nxv2i1, nxv2f64>;
+  defm : ldnf1<LDNF1B_D_IMM,  nxv2i64,  AArch64ldnf1_z,  nxv2i1, nxv2i8>;
+  defm : ldnf1<LDNF1SB_D_IMM, nxv2i64,  AArch64ldnf1s_z, nxv2i1, nxv2i8>;
+  defm : ldnf1<LDNF1H_D_IMM,  nxv2i64,  AArch64ldnf1_z,  nxv2i1, nxv2i16>;
+  defm : ldnf1<LDNF1SH_D_IMM, nxv2i64,  AArch64ldnf1s_z, nxv2i1, nxv2i16>;
+  defm : ldnf1<LDNF1W_D_IMM,  nxv2i64,  AArch64ldnf1_z,  nxv2i1, nxv2i32>;
+  defm : ldnf1<LDNF1SW_D_IMM, nxv2i64,  AArch64ldnf1s_z, nxv2i1, nxv2i32>;
+  defm : ldnf1<LDNF1D_IMM,    nxv2i64,  AArch64ldnf1_z,  nxv2i1, nxv2i64>;
+  defm : ldnf1<LDNF1D_IMM,    nxv2f64,  AArch64ldnf1_z,  nxv2i1, nxv2f64>;
 
   // 4-element contiguous non-faulting loads
-  defm : ldnf1<LDNF1B_S_IMM,  nxv4i32, AArch64ldnf1,  nxv4i1, nxv4i8>;
-  defm : ldnf1<LDNF1SB_S_IMM, nxv4i32, AArch64ldnf1s, nxv4i1, nxv4i8>;
-  defm : ldnf1<LDNF1H_S_IMM,  nxv4i32, AArch64ldnf1,  nxv4i1, nxv4i16>;
-  defm : ldnf1<LDNF1SH_S_IMM, nxv4i32, AArch64ldnf1s, nxv4i1, nxv4i16>;
-  defm : ldnf1<LDNF1W_IMM,    nxv4i32, AArch64ldnf1,  nxv4i1, nxv4i32>;
-  defm : ldnf1<LDNF1W_IMM,    nxv4f32, AArch64ldnf1,  nxv4i1, nxv4f32>;
+  defm : ldnf1<LDNF1B_S_IMM,  nxv4i32,  AArch64ldnf1_z,  nxv4i1, nxv4i8>;
+  defm : ldnf1<LDNF1SB_S_IMM, nxv4i32,  AArch64ldnf1s_z, nxv4i1, nxv4i8>;
+  defm : ldnf1<LDNF1H_S_IMM,  nxv4i32,  AArch64ldnf1_z,  nxv4i1, nxv4i16>;
+  defm : ldnf1<LDNF1SH_S_IMM, nxv4i32,  AArch64ldnf1s_z, nxv4i1, nxv4i16>;
+  defm : ldnf1<LDNF1W_IMM,    nxv4i32,  AArch64ldnf1_z,  nxv4i1, nxv4i32>;
+  defm : ldnf1<LDNF1W_IMM,    nxv4f32,  AArch64ldnf1_z,  nxv4i1, nxv4f32>;
 
   // 8-element contiguous non-faulting loads
-  defm : ldnf1<LDNF1B_H_IMM,  nxv8i16, AArch64ldnf1,  nxv8i1, nxv8i8>;
-  defm : ldnf1<LDNF1SB_H_IMM, nxv8i16, AArch64ldnf1s, nxv8i1, nxv8i8>;
-  defm : ldnf1<LDNF1H_IMM,    nxv8i16, AArch64ldnf1,  nxv8i1, nxv8i16>;
-  defm : ldnf1<LDNF1H_IMM,    nxv8f16, AArch64ldnf1,  nxv8i1, nxv8f16>;
+  defm : ldnf1<LDNF1B_H_IMM,  nxv8i16,  AArch64ldnf1_z,  nxv8i1, nxv8i8>;
+  defm : ldnf1<LDNF1SB_H_IMM, nxv8i16,  AArch64ldnf1s_z, nxv8i1, nxv8i8>;
+  defm : ldnf1<LDNF1H_IMM,    nxv8i16,  AArch64ldnf1_z,  nxv8i1, nxv8i16>;
+  defm : ldnf1<LDNF1H_IMM,    nxv8f16,  AArch64ldnf1_z,  nxv8i1, nxv8f16>;
+
+  let Predicates = [HasBF16, HasSVE] in {
+    defm : ldnf1<LDNF1H_IMM,    nxv8bf16, AArch64ldnf1_z,  nxv8i1, nxv8bf16>;
+  }
 
   // 16-element contiguous non-faulting loads
-  defm : ldnf1<LDNF1B_IMM,    nxv16i8, AArch64ldnf1, nxv16i1, nxv16i8>;
+  defm : ldnf1<LDNF1B_IMM,    nxv16i8,  AArch64ldnf1_z, nxv16i1, nxv16i8>;
 
   multiclass ldff1<Instruction I, ValueType Ty, SDPatternOperator Load, ValueType PredTy, ValueType MemVT, ComplexPattern AddrCP> {
     // reg + reg
@@ -1764,32 +1924,36 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   }
 
   // 2-element contiguous first faulting loads
-  defm : ldff1<LDFF1B_D,  nxv2i64, AArch64ldff1,  nxv2i1, nxv2i8,  am_sve_regreg_lsl0>;
-  defm : ldff1<LDFF1SB_D, nxv2i64, AArch64ldff1s, nxv2i1, nxv2i8,  am_sve_regreg_lsl0>;
-  defm : ldff1<LDFF1H_D,  nxv2i64, AArch64ldff1,  nxv2i1, nxv2i16, am_sve_regreg_lsl1>;
-  defm : ldff1<LDFF1SH_D, nxv2i64, AArch64ldff1s, nxv2i1, nxv2i16, am_sve_regreg_lsl1>;
-  defm : ldff1<LDFF1W_D,  nxv2i64, AArch64ldff1,  nxv2i1, nxv2i32, am_sve_regreg_lsl2>;
-  defm : ldff1<LDFF1SW_D, nxv2i64, AArch64ldff1s, nxv2i1, nxv2i32, am_sve_regreg_lsl2>;
-  defm : ldff1<LDFF1D,    nxv2i64, AArch64ldff1,  nxv2i1, nxv2i64, am_sve_regreg_lsl3>;
-  defm : ldff1<LDFF1W_D,  nxv2f32, AArch64ldff1,  nxv2i1, nxv2f32, am_sve_regreg_lsl2>;
-  defm : ldff1<LDFF1D,    nxv2f64, AArch64ldff1,  nxv2i1, nxv2f64, am_sve_regreg_lsl3>;
+  defm : ldff1<LDFF1B_D,  nxv2i64,  AArch64ldff1_z,  nxv2i1, nxv2i8,   am_sve_regreg_lsl0>;
+  defm : ldff1<LDFF1SB_D, nxv2i64,  AArch64ldff1s_z, nxv2i1, nxv2i8,   am_sve_regreg_lsl0>;
+  defm : ldff1<LDFF1H_D,  nxv2i64,  AArch64ldff1_z,  nxv2i1, nxv2i16,  am_sve_regreg_lsl1>;
+  defm : ldff1<LDFF1SH_D, nxv2i64,  AArch64ldff1s_z, nxv2i1, nxv2i16,  am_sve_regreg_lsl1>;
+  defm : ldff1<LDFF1W_D,  nxv2i64,  AArch64ldff1_z,  nxv2i1, nxv2i32,  am_sve_regreg_lsl2>;
+  defm : ldff1<LDFF1SW_D, nxv2i64,  AArch64ldff1s_z, nxv2i1, nxv2i32,  am_sve_regreg_lsl2>;
+  defm : ldff1<LDFF1D,    nxv2i64,  AArch64ldff1_z,  nxv2i1, nxv2i64,  am_sve_regreg_lsl3>;
+  defm : ldff1<LDFF1W_D,  nxv2f32,  AArch64ldff1_z,  nxv2i1, nxv2f32,  am_sve_regreg_lsl2>;
+  defm : ldff1<LDFF1D,    nxv2f64,  AArch64ldff1_z,  nxv2i1, nxv2f64,  am_sve_regreg_lsl3>;
 
   // 4-element contiguous first faulting loads
-  defm : ldff1<LDFF1B_S,  nxv4i32, AArch64ldff1,  nxv4i1, nxv4i8,  am_sve_regreg_lsl0>;
-  defm : ldff1<LDFF1SB_S, nxv4i32, AArch64ldff1s, nxv4i1, nxv4i8,  am_sve_regreg_lsl0>;
-  defm : ldff1<LDFF1H_S,  nxv4i32, AArch64ldff1,  nxv4i1, nxv4i16, am_sve_regreg_lsl1>;
-  defm : ldff1<LDFF1SH_S, nxv4i32, AArch64ldff1s, nxv4i1, nxv4i16, am_sve_regreg_lsl1>;
-  defm : ldff1<LDFF1W,    nxv4i32, AArch64ldff1,  nxv4i1, nxv4i32, am_sve_regreg_lsl2>;
-  defm : ldff1<LDFF1W,    nxv4f32, AArch64ldff1,  nxv4i1, nxv4f32, am_sve_regreg_lsl2>;
+  defm : ldff1<LDFF1B_S,  nxv4i32,  AArch64ldff1_z,  nxv4i1, nxv4i8,   am_sve_regreg_lsl0>;
+  defm : ldff1<LDFF1SB_S, nxv4i32,  AArch64ldff1s_z, nxv4i1, nxv4i8,   am_sve_regreg_lsl0>;
+  defm : ldff1<LDFF1H_S,  nxv4i32,  AArch64ldff1_z,  nxv4i1, nxv4i16,  am_sve_regreg_lsl1>;
+  defm : ldff1<LDFF1SH_S, nxv4i32,  AArch64ldff1s_z, nxv4i1, nxv4i16,  am_sve_regreg_lsl1>;
+  defm : ldff1<LDFF1W,    nxv4i32,  AArch64ldff1_z,  nxv4i1, nxv4i32,  am_sve_regreg_lsl2>;
+  defm : ldff1<LDFF1W,    nxv4f32,  AArch64ldff1_z,  nxv4i1, nxv4f32,  am_sve_regreg_lsl2>;
 
   // 8-element contiguous first faulting loads
-  defm : ldff1<LDFF1B_H,  nxv8i16, AArch64ldff1,  nxv8i1, nxv8i8,  am_sve_regreg_lsl0>;
-  defm : ldff1<LDFF1SB_H, nxv8i16, AArch64ldff1s, nxv8i1, nxv8i8,  am_sve_regreg_lsl0>;
-  defm : ldff1<LDFF1H,    nxv8i16, AArch64ldff1,  nxv8i1, nxv8i16, am_sve_regreg_lsl1>;
-  defm : ldff1<LDFF1H,    nxv8f16, AArch64ldff1,  nxv8i1, nxv8f16, am_sve_regreg_lsl1>;
+  defm : ldff1<LDFF1B_H,  nxv8i16,  AArch64ldff1_z,  nxv8i1, nxv8i8,   am_sve_regreg_lsl0>;
+  defm : ldff1<LDFF1SB_H, nxv8i16,  AArch64ldff1s_z, nxv8i1, nxv8i8,   am_sve_regreg_lsl0>;
+  defm : ldff1<LDFF1H,    nxv8i16,  AArch64ldff1_z,  nxv8i1, nxv8i16,  am_sve_regreg_lsl1>;
+  defm : ldff1<LDFF1H,    nxv8f16,  AArch64ldff1_z,  nxv8i1, nxv8f16,  am_sve_regreg_lsl1>;
+
+  let Predicates = [HasBF16, HasSVE] in {
+    defm : ldff1<LDFF1H,    nxv8bf16, AArch64ldff1_z,  nxv8i1, nxv8bf16, am_sve_regreg_lsl1>;
+  }
 
   // 16-element contiguous first faulting loads
-  defm : ldff1<LDFF1B, nxv16i8, AArch64ldff1, nxv16i1, nxv16i8, am_sve_regreg_lsl0>;
+  defm : ldff1<LDFF1B, nxv16i8, AArch64ldff1_z, nxv16i1, nxv16i8, am_sve_regreg_lsl0>;
 
   multiclass st1<Instruction RegRegInst, Instruction RegImmInst, ValueType Ty,
                  SDPatternOperator Store, ValueType PredTy, ValueType MemVT, ComplexPattern AddrCP> {
@@ -1956,14 +2120,14 @@ let Predicates = [HasSVE, HasMatMulFP32] in {
 
 let Predicates = [HasSVE, HasMatMulFP64] in {
   defm FMMLA_ZZZ_D : sve_fp_matrix_mla<1, "fmmla", ZPR64, int_aarch64_sve_fmmla, nxv2f64>;
-  defm LD1RO_B_IMM : sve_mem_ldor_si<0b00, "ld1rob", Z_b, ZPR8,  nxv16i8, nxv16i1, AArch64ld1ro>;
-  defm LD1RO_H_IMM : sve_mem_ldor_si<0b01, "ld1roh", Z_h, ZPR16, nxv8i16, nxv8i1,  AArch64ld1ro>;
-  defm LD1RO_W_IMM : sve_mem_ldor_si<0b10, "ld1row", Z_s, ZPR32, nxv4i32, nxv4i1,  AArch64ld1ro>;
-  defm LD1RO_D_IMM : sve_mem_ldor_si<0b11, "ld1rod", Z_d, ZPR64, nxv2i64, nxv2i1,  AArch64ld1ro>;
-  defm LD1RO_B     : sve_mem_ldor_ss<0b00, "ld1rob", Z_b, ZPR8,  GPR64NoXZRshifted8,  nxv16i8, nxv16i1, AArch64ld1ro, am_sve_regreg_lsl0>;
-  defm LD1RO_H     : sve_mem_ldor_ss<0b01, "ld1roh", Z_h, ZPR16, GPR64NoXZRshifted16, nxv8i16, nxv8i1,  AArch64ld1ro, am_sve_regreg_lsl1>;
-  defm LD1RO_W     : sve_mem_ldor_ss<0b10, "ld1row", Z_s, ZPR32, GPR64NoXZRshifted32, nxv4i32, nxv4i1,  AArch64ld1ro, am_sve_regreg_lsl2>;
-  defm LD1RO_D     : sve_mem_ldor_ss<0b11, "ld1rod", Z_d, ZPR64, GPR64NoXZRshifted64, nxv2i64, nxv2i1,  AArch64ld1ro, am_sve_regreg_lsl3>;
+  defm LD1RO_B_IMM : sve_mem_ldor_si<0b00, "ld1rob", Z_b, ZPR8,  nxv16i8, nxv16i1, AArch64ld1ro_z>;
+  defm LD1RO_H_IMM : sve_mem_ldor_si<0b01, "ld1roh", Z_h, ZPR16, nxv8i16, nxv8i1,  AArch64ld1ro_z>;
+  defm LD1RO_W_IMM : sve_mem_ldor_si<0b10, "ld1row", Z_s, ZPR32, nxv4i32, nxv4i1,  AArch64ld1ro_z>;
+  defm LD1RO_D_IMM : sve_mem_ldor_si<0b11, "ld1rod", Z_d, ZPR64, nxv2i64, nxv2i1,  AArch64ld1ro_z>;
+  defm LD1RO_B     : sve_mem_ldor_ss<0b00, "ld1rob", Z_b, ZPR8,  GPR64NoXZRshifted8,  nxv16i8, nxv16i1, AArch64ld1ro_z, am_sve_regreg_lsl0>;
+  defm LD1RO_H     : sve_mem_ldor_ss<0b01, "ld1roh", Z_h, ZPR16, GPR64NoXZRshifted16, nxv8i16, nxv8i1,  AArch64ld1ro_z, am_sve_regreg_lsl1>;
+  defm LD1RO_W     : sve_mem_ldor_ss<0b10, "ld1row", Z_s, ZPR32, GPR64NoXZRshifted32, nxv4i32, nxv4i1,  AArch64ld1ro_z, am_sve_regreg_lsl2>;
+  defm LD1RO_D     : sve_mem_ldor_ss<0b11, "ld1rod", Z_d, ZPR64, GPR64NoXZRshifted64, nxv2i64, nxv2i1,  AArch64ld1ro_z, am_sve_regreg_lsl3>;
   defm ZIP1_ZZZ_Q  : sve_int_perm_bin_perm_128_zz<0b00, 0, "zip1", int_aarch64_sve_zip1q>;
   defm ZIP2_ZZZ_Q  : sve_int_perm_bin_perm_128_zz<0b00, 1, "zip2", int_aarch64_sve_zip2q>;
   defm UZP1_ZZZ_Q  : sve_int_perm_bin_perm_128_zz<0b01, 0, "uzp1", int_aarch64_sve_uzp1q>;
@@ -1972,6 +2136,15 @@ let Predicates = [HasSVE, HasMatMulFP64] in {
   defm TRN2_ZZZ_Q  : sve_int_perm_bin_perm_128_zz<0b11, 1, "trn2", int_aarch64_sve_trn2q>;
 }
 
+let Predicates = [HasSVE, HasMatMulFP64, HasBF16] in {
+  def : SVE_2_Op_Pat<nxv8bf16, int_aarch64_sve_zip1q, nxv8bf16, nxv8bf16, ZIP1_ZZZ_Q>;
+  def : SVE_2_Op_Pat<nxv8bf16, int_aarch64_sve_zip2q, nxv8bf16, nxv8bf16, ZIP2_ZZZ_Q>;
+  def : SVE_2_Op_Pat<nxv8bf16, int_aarch64_sve_uzp1q, nxv8bf16, nxv8bf16, UZP1_ZZZ_Q>;
+  def : SVE_2_Op_Pat<nxv8bf16, int_aarch64_sve_uzp2q, nxv8bf16, nxv8bf16, UZP2_ZZZ_Q>;
+  def : SVE_2_Op_Pat<nxv8bf16, int_aarch64_sve_trn1q, nxv8bf16, nxv8bf16, TRN1_ZZZ_Q>;
+  def : SVE_2_Op_Pat<nxv8bf16, int_aarch64_sve_trn2q, nxv8bf16, nxv8bf16, TRN2_ZZZ_Q>;
+}
+
 let Predicates = [HasSVE2] in {
   // SVE2 integer multiply-add (indexed)
   defm MLA_ZZZI : sve2_int_mla_by_indexed_elem<0b01, 0b0, "mla", int_aarch64_sve_mla_lane>;
@@ -2131,6 +2304,14 @@ let Predicates = [HasSVE2] in {
   defm SQRSHLR_ZPmZ : sve2_int_arith_pred<0b011100, "sqrshlr", null_frag>;
   defm UQRSHLR_ZPmZ : sve2_int_arith_pred<0b011110, "uqrshlr", null_frag>;
 
+  let Predicates = [HasSVE2, UseExperimentalZeroingPseudos] in {
+    defm SQSHL_ZPZI  : sve_int_bin_pred_shift_imm_left_zeroing_bhsd<null_frag>;
+    defm UQSHL_ZPZI  : sve_int_bin_pred_shift_imm_left_zeroing_bhsd<null_frag>;
+    defm SRSHR_ZPZI  : sve_int_bin_pred_shift_imm_right_zeroing_bhsd<int_aarch64_sve_srshr>;
+    defm URSHR_ZPZI  : sve_int_bin_pred_shift_imm_right_zeroing_bhsd<int_aarch64_sve_urshr>;
+    defm SQSHLU_ZPZI : sve_int_bin_pred_shift_imm_left_zeroing_bhsd<int_aarch64_sve_sqshlu>;
+  }
+
   // SVE2 predicated shifts
   defm SQSHL_ZPmI  : sve_int_bin_pred_shift_imm_left< 0b0110, "sqshl", "SQSHL_ZPZI">;
   defm UQSHL_ZPmI  : sve_int_bin_pred_shift_imm_left< 0b0111, "uqshl", "UQSHL_ZPZI">;
@@ -2312,19 +2493,19 @@ let Predicates = [HasSVE2] in {
   def EXT_ZZI_B : sve2_int_perm_extract_i_cons<"ext">;
 
   // SVE2 non-temporal gather loads
-  defm LDNT1SB_ZZR_S : sve2_mem_gldnt_vs_32_ptrs<0b00000, "ldnt1sb", AArch64ldnt1s_gather, nxv4i8>;
-  defm LDNT1B_ZZR_S  : sve2_mem_gldnt_vs_32_ptrs<0b00001, "ldnt1b",  AArch64ldnt1_gather,  nxv4i8>;
-  defm LDNT1SH_ZZR_S : sve2_mem_gldnt_vs_32_ptrs<0b00100, "ldnt1sh", AArch64ldnt1s_gather, nxv4i16>;
-  defm LDNT1H_ZZR_S  : sve2_mem_gldnt_vs_32_ptrs<0b00101, "ldnt1h",  AArch64ldnt1_gather,  nxv4i16>;
-  defm LDNT1W_ZZR_S  : sve2_mem_gldnt_vs_32_ptrs<0b01001, "ldnt1w",  AArch64ldnt1_gather,  nxv4i32>;
-
-  defm LDNT1SB_ZZR_D : sve2_mem_gldnt_vs_64_ptrs<0b10000, "ldnt1sb", AArch64ldnt1s_gather, nxv2i8>;
-  defm LDNT1B_ZZR_D  : sve2_mem_gldnt_vs_64_ptrs<0b10010, "ldnt1b",  AArch64ldnt1_gather,  nxv2i8>;
-  defm LDNT1SH_ZZR_D : sve2_mem_gldnt_vs_64_ptrs<0b10100, "ldnt1sh", AArch64ldnt1s_gather, nxv2i16>;
-  defm LDNT1H_ZZR_D  : sve2_mem_gldnt_vs_64_ptrs<0b10110, "ldnt1h",  AArch64ldnt1_gather,  nxv2i16>;
-  defm LDNT1SW_ZZR_D : sve2_mem_gldnt_vs_64_ptrs<0b11000, "ldnt1sw", AArch64ldnt1s_gather, nxv2i32>;
-  defm LDNT1W_ZZR_D  : sve2_mem_gldnt_vs_64_ptrs<0b11010, "ldnt1w",  AArch64ldnt1_gather,  nxv2i32>;
-  defm LDNT1D_ZZR_D  : sve2_mem_gldnt_vs_64_ptrs<0b11110, "ldnt1d",  AArch64ldnt1_gather,  nxv2i64>;
+  defm LDNT1SB_ZZR_S : sve2_mem_gldnt_vs_32_ptrs<0b00000, "ldnt1sb", AArch64ldnt1s_gather_z, nxv4i8>;
+  defm LDNT1B_ZZR_S  : sve2_mem_gldnt_vs_32_ptrs<0b00001, "ldnt1b",  AArch64ldnt1_gather_z,  nxv4i8>;
+  defm LDNT1SH_ZZR_S : sve2_mem_gldnt_vs_32_ptrs<0b00100, "ldnt1sh", AArch64ldnt1s_gather_z, nxv4i16>;
+  defm LDNT1H_ZZR_S  : sve2_mem_gldnt_vs_32_ptrs<0b00101, "ldnt1h",  AArch64ldnt1_gather_z,  nxv4i16>;
+  defm LDNT1W_ZZR_S  : sve2_mem_gldnt_vs_32_ptrs<0b01001, "ldnt1w",  AArch64ldnt1_gather_z,  nxv4i32>;
+
+  defm LDNT1SB_ZZR_D : sve2_mem_gldnt_vs_64_ptrs<0b10000, "ldnt1sb", AArch64ldnt1s_gather_z, nxv2i8>;
+  defm LDNT1B_ZZR_D  : sve2_mem_gldnt_vs_64_ptrs<0b10010, "ldnt1b",  AArch64ldnt1_gather_z,  nxv2i8>;
+  defm LDNT1SH_ZZR_D : sve2_mem_gldnt_vs_64_ptrs<0b10100, "ldnt1sh", AArch64ldnt1s_gather_z, nxv2i16>;
+  defm LDNT1H_ZZR_D  : sve2_mem_gldnt_vs_64_ptrs<0b10110, "ldnt1h",  AArch64ldnt1_gather_z,  nxv2i16>;
+  defm LDNT1SW_ZZR_D : sve2_mem_gldnt_vs_64_ptrs<0b11000, "ldnt1sw", AArch64ldnt1s_gather_z, nxv2i32>;
+  defm LDNT1W_ZZR_D  : sve2_mem_gldnt_vs_64_ptrs<0b11010, "ldnt1w",  AArch64ldnt1_gather_z,  nxv2i32>;
+  defm LDNT1D_ZZR_D  : sve2_mem_gldnt_vs_64_ptrs<0b11110, "ldnt1d",  AArch64ldnt1_gather_z,  nxv2i64>;
 
   // SVE2 vector splice (constructive)
   defm SPLICE_ZPZZ : sve2_int_perm_splice_cons<"splice">;
@@ -2343,6 +2524,13 @@ let Predicates = [HasSVE2] in {
   defm TBL_ZZZZ : sve2_int_perm_tbl<"tbl", int_aarch64_sve_tbl2>;
   defm TBX_ZZZ  : sve2_int_perm_tbx<"tbx", int_aarch64_sve_tbx>;
 
+  let Predicates = [HasSVE, HasBF16] in {
+    def : SVE_3_Op_Pat<nxv8bf16, int_aarch64_sve_tbx, nxv8bf16, nxv8bf16, nxv8i16, TBX_ZZZ_H>;
+    def : Pat<(nxv8bf16 (int_aarch64_sve_tbl2 nxv8bf16:$Op1, nxv8bf16:$Op2, nxv8i16:$Op3)),
+              (nxv8bf16 (TBL_ZZZZ_H (REG_SEQUENCE ZPR2, nxv8bf16:$Op1, zsub0, nxv8bf16:$Op2, zsub1),
+                        nxv8i16:$Op3))>;
+  }
+
   // SVE2 integer compare scalar count and limit
   defm WHILEGE_PWW : sve_int_while4_rr<0b000, "whilege", int_aarch64_sve_whilege>;
   defm WHILEGT_PWW : sve_int_while4_rr<0b001, "whilegt", int_aarch64_sve_whilegt>;
diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
index ea7b87f0c7663..8f814d185e859 100644
--- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
@@ -17,7 +17,7 @@ using namespace llvm;
 
 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(
     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
-    SDValue Size, unsigned Align, bool isVolatile,
+    SDValue Size, Align Alignment, bool isVolatile,
     MachinePointerInfo DstPtrInfo) const {
   // Check to see if there is a specialized entry-point for memory zeroing.
   ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src);
diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.h b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.h
index d0967fb973cc3..d94fd8471b7b9 100644
--- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.h
@@ -21,7 +21,8 @@ class AArch64SelectionDAGInfo : public SelectionDAGTargetInfo {
 public:
   SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl,
                                   SDValue Chain, SDValue Dst, SDValue Src,
-                                  SDValue Size, unsigned Align, bool isVolatile,
+                                  SDValue Size, Align Alignment,
+                                  bool isVolatile,
                                   MachinePointerInfo DstPtrInfo) const override;
   SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl,
                                   SDValue Chain, SDValue Op1, SDValue Op2,
diff --git a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
index cb69a66103990..61f27cbc3b29d 100644
--- a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
+++ b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/PostDominators.h"
@@ -264,8 +265,9 @@ class InitializerBuilder {
       Type *EltTy = VecTy->getElementType();
       if (EltTy->isPointerTy()) {
         uint32_t EltSize = DL->getTypeSizeInBits(EltTy);
-        auto *NewTy = FixedVectorType::get(IntegerType::get(Ctx, EltSize),
-                                           VecTy->getNumElements());
+        auto *NewTy = FixedVectorType::get(
+            IntegerType::get(Ctx, EltSize),
+            cast<FixedVectorType>(VecTy)->getNumElements());
         V = IRB.CreatePointerCast(V, NewTy);
       }
     }
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
index 9bb533f8efdee..029535cb98b57 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
@@ -101,6 +101,9 @@ void AArch64Subtarget::initializeProperties() {
   case CortexA73:
   case CortexA75:
   case CortexA76:
+  case CortexA77:
+  case CortexA78:
+  case CortexX1:
     PrefFunctionLogAlignment = 4;
     break;
   case A64FX:
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index 221c103cff8f5..b111f00169488 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -55,6 +55,9 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
     CortexA73,
     CortexA75,
     CortexA76,
+    CortexA77,
+    CortexA78,
+    CortexX1,
     ExynosM3,
     Falkor,
     Kryo,
@@ -104,6 +107,10 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   bool HasPAN_RWV = false;
   bool HasCCPP = false;
 
+  // SVE extensions
+  bool HasSVE = false;
+  bool UseExperimentalZeroingPseudos = false;
+
   // Armv8.2 Crypto extensions
   bool HasSM4 = false;
   bool HasSHA3 = false;
@@ -130,8 +137,6 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   bool HasRCPC_IMMO = false;
 
   bool HasLSLFast = false;
-  bool HasSVE = false;
-  bool HasSVE2 = false;
   bool HasRCPC = false;
   bool HasAggressiveFMA = false;
 
@@ -158,6 +163,7 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   bool HasEnhancedCounterVirtualization = false;
 
   // Arm SVE2 extensions
+  bool HasSVE2 = false;
   bool HasSVE2AES = false;
   bool HasSVE2SM4 = false;
   bool HasSVE2SHA3 = false;
@@ -398,6 +404,10 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
 
   unsigned getWideningBaseCost() const { return WideningBaseCost; }
 
+  bool useExperimentalZeroingPseudos() const {
+    return UseExperimentalZeroingPseudos;
+  }
+
   /// CPU has TBI (top byte of addresses is ignored during HW address
   /// translation) and OS enables it.
   bool supportsAddressTopByteIgnored() const;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 18e7976b4f007..cf6de797727be 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -212,7 +212,7 @@ bool AArch64TTIImpl::isWideningInstruction(Type *DstTy, unsigned Opcode,
   // elements in type Ty determine the vector width.
   auto toVectorTy = [&](Type *ArgTy) {
     return FixedVectorType::get(ArgTy->getScalarType(),
-                                cast<VectorType>(DstTy)->getNumElements());
+                                cast<FixedVectorType>(DstTy)->getNumElements());
   };
 
   // Exit early if DstTy is not a vector type whose elements are at least
@@ -469,6 +469,15 @@ int AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode, Type *Dst,
   return Cost + getCastInstrCost(Opcode, Dst, Src, CostKind);
 }
 
+unsigned AArch64TTIImpl::getCFInstrCost(unsigned Opcode,
+                                        TTI::TargetCostKind CostKind) {
+  if (CostKind != TTI::TCK_RecipThroughput)
+    return Opcode == Instruction::PHI ? 0 : 1;
+  assert(CostKind == TTI::TCK_RecipThroughput && "unexpected CostKind");
+  // Branches are assumed to be predicted.
+  return 0;
+}
+
 int AArch64TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
                                        unsigned Index) {
   assert(Val->isVectorTy() && "This must be a vector type");
@@ -601,6 +610,16 @@ int AArch64TTIImpl::getArithmeticInstrCost(
     // These nodes are marked as 'custom' for combining purposes only.
     // We know that they are legal. See LowerAdd in ISelLowering.
     return (Cost + 1) * LT.first;
+
+  case ISD::FADD:
+    // These nodes are marked as 'custom' just to lower them to SVE.
+    // We know said lowering will incur no additional cost.
+    if (isa<FixedVectorType>(Ty) && !Ty->getScalarType()->isFP128Ty())
+      return (Cost + 2) * LT.first;
+
+    return Cost + BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info,
+                                                Opd2Info,
+                                                Opd1PropInfo, Opd2PropInfo);
   }
 }
 
@@ -714,8 +733,8 @@ int AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
       // have to promote the elements to v.2.
       ProfitableNumElements = 8;
 
-    if (cast<VectorType>(Ty)->getNumElements() < ProfitableNumElements) {
-      unsigned NumVecElts = cast<VectorType>(Ty)->getNumElements();
+    if (cast<FixedVectorType>(Ty)->getNumElements() < ProfitableNumElements) {
+      unsigned NumVecElts = cast<FixedVectorType>(Ty)->getNumElements();
       unsigned NumVectorizableInstsToAmortize = NumVecElts * 2;
       // We generate 2 instructions per vector element.
       return NumVectorizableInstsToAmortize * NumVecElts * 2;
@@ -725,16 +744,12 @@ int AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
   return LT.first;
 }
 
-int AArch64TTIImpl::getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
-                                               unsigned Factor,
-                                               ArrayRef<unsigned> Indices,
-                                               unsigned Alignment,
-                                               unsigned AddressSpace,
-                                               TTI::TargetCostKind CostKind,
-                                               bool UseMaskForCond,
-                                               bool UseMaskForGaps) {
+int AArch64TTIImpl::getInterleavedMemoryOpCost(
+    unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+    Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
+    bool UseMaskForCond, bool UseMaskForGaps) {
   assert(Factor >= 2 && "Invalid interleave factor");
-  auto *VecVTy = cast<VectorType>(VecTy);
+  auto *VecVTy = cast<FixedVectorType>(VecTy);
 
   if (!UseMaskForCond && !UseMaskForGaps &&
       Factor <= TLI->getMaxSupportedInterleaveFactor()) {
@@ -761,7 +776,8 @@ int AArch64TTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) {
   for (auto *I : Tys) {
     if (!I->isVectorTy())
       continue;
-    if (I->getScalarSizeInBits() * cast<VectorType>(I)->getNumElements() == 128)
+    if (I->getScalarSizeInBits() * cast<FixedVectorType>(I)->getNumElements() ==
+        128)
       Cost += getMemoryOpCost(Instruction::Store, I, Align(128), 0, CostKind) +
               getMemoryOpCost(Instruction::Load, I, Align(128), 0, CostKind);
   }
@@ -843,6 +859,11 @@ void AArch64TTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
     getFalkorUnrollingPreferences(L, SE, UP);
 }
 
+void AArch64TTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                                           TTI::PeelingPreferences &PP) {
+  BaseT::getPeelingPreferences(L, SE, PP);
+}
+
 Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
                                                          Type *ExpectedType) {
   switch (Inst->getIntrinsicID()) {
@@ -964,9 +985,10 @@ bool AArch64TTIImpl::useReductionIntrinsic(unsigned Opcode, Type *Ty,
   case Instruction::Mul:
     return false;
   case Instruction::Add:
-    return ScalarBits * VTy->getNumElements() >= 128;
+    return ScalarBits * cast<FixedVectorType>(VTy)->getNumElements() >= 128;
   case Instruction::ICmp:
-    return (ScalarBits < 64) && (ScalarBits * VTy->getNumElements() >= 128);
+    return (ScalarBits < 64) &&
+           (ScalarBits * cast<FixedVectorType>(VTy)->getNumElements() >= 128);
   case Instruction::FCmp:
     return Flags.NoNaN;
   default:
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 8e85f92776780..1f029689a60e6 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -120,6 +120,8 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   int getExtractWithExtendCost(unsigned Opcode, Type *Dst, VectorType *VecTy,
                                unsigned Index);
 
+  unsigned getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind);
+
   int getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index);
 
   int getArithmeticInstrCost(
@@ -151,17 +153,21 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP);
 
+  void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                             TTI::PeelingPreferences &PP);
+
   Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
                                            Type *ExpectedType);
 
   bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info);
 
-  bool isLegalMaskedLoadStore(Type *DataType, MaybeAlign Alignment) {
-    if (!isa<VectorType>(DataType) || !ST->hasSVE())
+  bool isLegalMaskedLoadStore(Type *DataType, Align Alignment) {
+    if (!isa<ScalableVectorType>(DataType) || !ST->hasSVE())
       return false;
 
-    Type *Ty = cast<VectorType>(DataType)->getElementType();
-    if (Ty->isHalfTy() || Ty->isFloatTy() || Ty->isDoubleTy())
+    Type *Ty = cast<ScalableVectorType>(DataType)->getElementType();
+    if (Ty->isBFloatTy() || Ty->isHalfTy() ||
+        Ty->isFloatTy() || Ty->isDoubleTy())
       return true;
 
     if (Ty->isIntegerTy(8) || Ty->isIntegerTy(16) ||
@@ -171,11 +177,11 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
     return false;
   }
 
-  bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment) {
+  bool isLegalMaskedLoad(Type *DataType, Align Alignment) {
     return isLegalMaskedLoadStore(DataType, Alignment);
   }
 
-  bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment) {
+  bool isLegalMaskedStore(Type *DataType, Align Alignment) {
     return isLegalMaskedLoadStore(DataType, Alignment);
   }
 
@@ -188,7 +194,8 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
     // the element type fits into a register and the number of elements is a
     // power of 2 > 1.
     if (auto *DataTypeVTy = dyn_cast<VectorType>(DataType)) {
-      unsigned NumElements = DataTypeVTy->getNumElements();
+      unsigned NumElements =
+          cast<FixedVectorType>(DataTypeVTy)->getNumElements();
       unsigned EltSize = DataTypeVTy->getElementType()->getScalarSizeInBits();
       return NumElements > 1 && isPowerOf2_64(NumElements) && EltSize >= 8 &&
              EltSize <= 128 && isPowerOf2_64(EltSize);
@@ -196,12 +203,11 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
     return BaseT::isLegalNTStore(DataType, Alignment);
   }
 
-  int getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor,
-                                 ArrayRef<unsigned> Indices, unsigned Alignment,
-                                 unsigned AddressSpace,
-                                 TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
-                                 bool UseMaskForCond = false,
-                                 bool UseMaskForGaps = false);
+  int getInterleavedMemoryOpCost(
+      unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+      Align Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
+      bool UseMaskForCond = false, bool UseMaskForGaps = false);
 
   bool
   shouldConsiderAddressTypePromotion(const Instruction &I,
diff --git a/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp b/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp
index d6db88603429f..1ff4abb340540 100644
--- a/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp
+++ b/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp
@@ -146,6 +146,9 @@ static DecodeStatus DecodeExclusiveLdStInstruction(MCInst &Inst, uint32_t insn,
 static DecodeStatus DecodePairLdStInstruction(MCInst &Inst, uint32_t insn,
                                               uint64_t Address,
                                               const void *Decoder);
+static DecodeStatus DecodeAuthLoadInstruction(MCInst &Inst, uint32_t insn,
+                                              uint64_t Address,
+                                              const void *Decoder);
 static DecodeStatus DecodeAddSubERegInstruction(MCInst &Inst, uint32_t insn,
                                                 uint64_t Address,
                                                 const void *Decoder);
@@ -1501,6 +1504,39 @@ static DecodeStatus DecodePairLdStInstruction(MCInst &Inst, uint32_t insn,
   return Success;
 }
 
+static DecodeStatus DecodeAuthLoadInstruction(MCInst &Inst, uint32_t insn,
+                                              uint64_t Addr,
+                                              const void *Decoder) {
+  unsigned Rt = fieldFromInstruction(insn, 0, 5);
+  unsigned Rn = fieldFromInstruction(insn, 5, 5);
+  uint64_t offset = fieldFromInstruction(insn, 22, 1) << 9 |
+                    fieldFromInstruction(insn, 12, 9);
+  unsigned writeback = fieldFromInstruction(insn, 11, 1);
+
+  switch (Inst.getOpcode()) {
+  default:
+    return Fail;
+  case AArch64::LDRAAwriteback:
+  case AArch64::LDRABwriteback:
+    DecodeGPR64spRegisterClass(Inst, Rn /* writeback register */, Addr,
+                               Decoder);
+    break;
+  case AArch64::LDRAAindexed:
+  case AArch64::LDRABindexed:
+    break;
+  }
+
+  DecodeGPR64RegisterClass(Inst, Rt, Addr, Decoder);
+  DecodeGPR64spRegisterClass(Inst, Rn, Addr, Decoder);
+  DecodeSImm<10>(Inst, offset, Addr, Decoder);
+
+  if (writeback && Rt == Rn && Rn != 31) {
+    return SoftFail;
+  }
+
+  return Success;
+}
+
 static DecodeStatus DecodeAddSubERegInstruction(MCInst &Inst, uint32_t insn,
                                                 uint64_t Addr,
                                                 const void *Decoder) {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
index 7903299fcc33c..11a8d5def4296 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
@@ -427,6 +427,14 @@ static void handleMustTailForwardedRegisters(MachineIRBuilder &MIRBuilder,
   }
 }
 
+bool AArch64CallLowering::fallBackToDAGISel(const Function &F) const {
+  if (isa<ScalableVectorType>(F.getReturnType()))
+    return true;
+  return llvm::any_of(F.args(), [](const Argument &A) {
+    return isa<ScalableVectorType>(A.getType());
+  });
+}
+
 bool AArch64CallLowering::lowerFormalArguments(
     MachineIRBuilder &MIRBuilder, const Function &F,
     ArrayRef<ArrayRef<Register>> VRegs) const {
@@ -438,9 +446,6 @@ bool AArch64CallLowering::lowerFormalArguments(
   SmallVector<ArgInfo, 8> SplitArgs;
   unsigned i = 0;
   for (auto &Arg : F.args()) {
-    if (isa<ScalableVectorType>(Arg.getType()))
-      return false;
-
     if (DL.getTypeStoreSize(Arg.getType()).isZero())
       continue;
 
@@ -880,7 +885,6 @@ bool AArch64CallLowering::lowerTailCall(
   const auto &Forwards = FuncInfo->getForwardedMustTailRegParms();
 
   // Do the actual argument marshalling.
-  SmallVector<unsigned, 8> PhysRegs;
   OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, AssignFnFixed,
                              AssignFnVarArg, true, FPDiff);
   if (!handleAssignments(MIRBuilder, OutArgs, Handler))
@@ -998,7 +1002,6 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
     TRI->emitReservedArgRegCallError(MF);
 
   // Do the actual argument marshalling.
-  SmallVector<unsigned, 8> PhysRegs;
   OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, AssignFnFixed,
                              AssignFnVarArg, false);
   if (!handleAssignments(MIRBuilder, OutArgs, Handler))
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.h b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.h
index b0c601c7062c0..640a862530596 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.h
@@ -37,6 +37,8 @@ class AArch64CallLowering: public CallLowering {
                    ArrayRef<Register> VRegs,
                    Register SwiftErrorVReg) const override;
 
+  bool fallBackToDAGISel(const Function &F) const override;
+
   bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F,
                             ArrayRef<ArrayRef<Register>> VRegs) const override;
 
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 349def06f04a5..408f0cb77e738 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -32,6 +32,7 @@
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
+#include "llvm/IR/Constants.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/IntrinsicsAArch64.h"
 #include "llvm/Support/Debug.h"
@@ -126,6 +127,8 @@ class AArch64InstructionSelector : public InstructionSelector {
                                const RegisterBank &RB,
                                MachineIRBuilder &MIRBuilder) const;
   bool selectInsertElt(MachineInstr &I, MachineRegisterInfo &MRI) const;
+  bool tryOptConstantBuildVec(MachineInstr &MI, LLT DstTy,
+                              MachineRegisterInfo &MRI) const;
   bool selectBuildVector(MachineInstr &I, MachineRegisterInfo &MRI) const;
   bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const;
   bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const;
@@ -4486,6 +4489,44 @@ bool AArch64InstructionSelector::selectInsertElt(
   return true;
 }
 
+bool AArch64InstructionSelector::tryOptConstantBuildVec(
+    MachineInstr &I, LLT DstTy, MachineRegisterInfo &MRI) const {
+  assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
+  assert(DstTy.getSizeInBits() <= 128 && "Unexpected build_vec type!");
+  if (DstTy.getSizeInBits() < 32)
+    return false;
+  // Check if we're building a constant vector, in which case we want to
+  // generate a constant pool load instead of a vector insert sequence.
+  SmallVector<Constant *, 16> Csts;
+  for (unsigned Idx = 1; Idx < I.getNumOperands(); ++Idx) {
+    // Try to find G_CONSTANT or G_FCONSTANT
+    auto *OpMI =
+        getOpcodeDef(TargetOpcode::G_CONSTANT, I.getOperand(Idx).getReg(), MRI);
+    if (OpMI)
+      Csts.emplace_back(
+          const_cast<ConstantInt *>(OpMI->getOperand(1).getCImm()));
+    else if ((OpMI = getOpcodeDef(TargetOpcode::G_FCONSTANT,
+                                  I.getOperand(Idx).getReg(), MRI)))
+      Csts.emplace_back(
+          const_cast<ConstantFP *>(OpMI->getOperand(1).getFPImm()));
+    else
+      return false;
+  }
+  Constant *CV = ConstantVector::get(Csts);
+  MachineIRBuilder MIB(I);
+  auto *CPLoad = emitLoadFromConstantPool(CV, MIB);
+  if (!CPLoad) {
+    LLVM_DEBUG(dbgs() << "Could not generate cp load for build_vector");
+    return false;
+  }
+  MIB.buildCopy(I.getOperand(0), CPLoad->getOperand(0));
+  RBI.constrainGenericRegister(I.getOperand(0).getReg(),
+                               *MRI.getRegClass(CPLoad->getOperand(0).getReg()),
+                               MRI);
+  I.eraseFromParent();
+  return true;
+}
+
 bool AArch64InstructionSelector::selectBuildVector(
     MachineInstr &I, MachineRegisterInfo &MRI) const {
   assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
@@ -4494,6 +4535,9 @@ bool AArch64InstructionSelector::selectBuildVector(
   const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
   const LLT EltTy = MRI.getType(I.getOperand(1).getReg());
   unsigned EltSize = EltTy.getSizeInBits();
+
+  if (tryOptConstantBuildVec(I, DstTy, MRI))
+    return true;
   if (EltSize < 16 || EltSize > 64)
     return false; // Don't support all element types yet.
   const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);
@@ -4836,6 +4880,17 @@ bool AArch64InstructionSelector::isWorthFoldingIntoExtendedReg(
                 [](MachineInstr &Use) { return Use.mayLoadOrStore(); });
 }
 
+static bool isSignExtendShiftType(AArch64_AM::ShiftExtendType Type) {
+  switch (Type) {
+  case AArch64_AM::SXTB:
+  case AArch64_AM::SXTH:
+  case AArch64_AM::SXTW:
+    return true;
+  default:
+    return false;
+  }
+}
+
 InstructionSelector::ComplexRendererFns
 AArch64InstructionSelector::selectExtendedSHL(
     MachineOperand &Root, MachineOperand &Base, MachineOperand &Offset,
@@ -4908,7 +4963,10 @@ AArch64InstructionSelector::selectExtendedSHL(
     if (Ext == AArch64_AM::InvalidShiftExtend)
       return None;
 
-    SignExtend = Ext == AArch64_AM::SXTW;
+    SignExtend = isSignExtendShiftType(Ext) ? 1 : 0;
+    // We only support SXTW for signed extension here.
+    if (SignExtend && Ext != AArch64_AM::SXTW)
+      return None;
 
     // Need a 32-bit wide register here.
     MachineIRBuilder MIB(*MRI.getVRegDef(Root.getReg()));
@@ -5161,13 +5219,8 @@ AArch64InstructionSelector::tryFoldAddLowIntoImm(MachineInstr &RootDef,
   if (GV->isThreadLocal())
     return None;
 
-  unsigned Alignment = GV->getAlignment();
-  Type *Ty = GV->getValueType();
   auto &MF = *RootDef.getParent()->getParent();
-  if (Alignment == 0 && Ty->isSized())
-    Alignment = MF.getDataLayout().getABITypeAlignment(Ty);
-
-  if (Alignment < Size)
+  if (GV->getPointerAlignment(MF.getDataLayout()) < Size)
     return None;
 
   unsigned OpFlags = STI.ClassifyGlobalReference(GV, MF.getTarget());
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
index 9fbe185dce2a5..baa8515baf3ea 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
@@ -271,11 +271,11 @@ static bool matchZip(MachineInstr &MI, MachineRegisterInfo &MRI,
   return true;
 }
 
-static bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI,
-                     ShuffleVectorPseudo &MatchInfo) {
-  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
-  auto Lane = getSplatIndex(MI);
-  if (!Lane || *Lane != 0)
+/// Helper function for matchDup.
+static bool matchDupFromInsertVectorElt(int Lane, MachineInstr &MI,
+                                        MachineRegisterInfo &MRI,
+                                        ShuffleVectorPseudo &MatchInfo) {
+  if (Lane != 0)
     return false;
 
   // Try to match a vector splat operation into a dup instruction.
@@ -296,10 +296,9 @@ static bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI,
                              MI.getOperand(1).getReg(), MRI);
   if (!InsMI)
     return false;
-
   // Match the undef vector operand.
-  if (!getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF,
-                               InsMI->getOperand(1).getReg(), MRI))
+  if (!getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, InsMI->getOperand(1).getReg(),
+                    MRI))
     return false;
 
   // Match the index constant 0.
@@ -307,12 +306,45 @@ static bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI,
   if (!mi_match(InsMI->getOperand(3).getReg(), MRI, m_ICst(Index)) || Index)
     return false;
 
-  Register Dst = MI.getOperand(0).getReg();
+  MatchInfo = ShuffleVectorPseudo(AArch64::G_DUP, MI.getOperand(0).getReg(),
+                                  {InsMI->getOperand(2).getReg()});
+  return true;
+}
+
+/// Helper function for matchDup.
+static bool matchDupFromBuildVector(int Lane, MachineInstr &MI,
+                                    MachineRegisterInfo &MRI,
+                                    ShuffleVectorPseudo &MatchInfo) {
+  assert(Lane >= 0 && "Expected positive lane?");
+  // Test if the LHS is a BUILD_VECTOR. If it is, then we can just reference the
+  // lane's definition directly.
+  auto *BuildVecMI = getOpcodeDef(TargetOpcode::G_BUILD_VECTOR,
+                                  MI.getOperand(1).getReg(), MRI);
+  if (!BuildVecMI)
+    return false;
+  Register Reg = BuildVecMI->getOperand(Lane + 1).getReg();
   MatchInfo =
-      ShuffleVectorPseudo(AArch64::G_DUP, Dst, {InsMI->getOperand(2).getReg()});
+      ShuffleVectorPseudo(AArch64::G_DUP, MI.getOperand(0).getReg(), {Reg});
   return true;
 }
 
+static bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI,
+                     ShuffleVectorPseudo &MatchInfo) {
+  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
+  auto MaybeLane = getSplatIndex(MI);
+  if (!MaybeLane)
+    return false;
+  int Lane = *MaybeLane;
+  // If this is undef splat, generate it via "just" vdup, if possible.
+  if (Lane < 0)
+    Lane = 0;
+  if (matchDupFromInsertVectorElt(Lane, MI, MRI, MatchInfo))
+    return true;
+  if (matchDupFromBuildVector(Lane, MI, MRI, MatchInfo))
+    return true;
+  return false;
+}
+
 static bool matchEXT(MachineInstr &MI, MachineRegisterInfo &MRI,
                      ShuffleVectorPseudo &MatchInfo) {
   assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 01e66961cb813..a005d1e65abe1 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -16,7 +16,7 @@ def SDT_AArch64Setcc : SDTypeProfile<1, 4, [
   SDTCisVT<4, OtherVT>
 ]>;
 
-def AArch64setcc_pred : SDNode<"AArch64ISD::SETCC_PRED", SDT_AArch64Setcc>;
+def AArch64setcc_z : SDNode<"AArch64ISD::SETCC_MERGE_ZERO", SDT_AArch64Setcc>;
 
 def SVEPatternOperand : AsmOperandClass {
   let Name = "SVEPattern";
@@ -419,25 +419,27 @@ class SVEPseudo2Instr<string name, bit instr> {
   bit IsInstr = instr;
 }
 
+// Lookup e.g. DIV -> DIVR
 def getSVERevInstr : InstrMapping {
   let FilterClass = "SVEInstr2Rev";
   let RowFields = ["InstrName"];
-  let ColFields = ["IsOrig"];
-  let KeyCol = ["1"];
-  let ValueCols = [["0"]];
+  let ColFields = ["isReverseInstr"];
+  let KeyCol = ["0"];
+  let ValueCols = [["1"]];
 }
 
-def getSVEOrigInstr : InstrMapping {
+// Lookup e.g. DIVR -> DIV
+def getSVENonRevInstr : InstrMapping {
   let FilterClass = "SVEInstr2Rev";
   let RowFields = ["InstrName"];
-  let ColFields = ["IsOrig"];
-  let KeyCol = ["0"];
-  let ValueCols = [["1"]];
+  let ColFields = ["isReverseInstr"];
+  let KeyCol = ["1"];
+  let ValueCols = [["0"]];
 }
 
-class SVEInstr2Rev<string name, string revname, bit nameIsOrig> {
-  string InstrName = !if(nameIsOrig, name, revname);
-  bit IsOrig = nameIsOrig;
+class SVEInstr2Rev<string name1, string name2, bit name1IsReverseInstr> {
+  string InstrName = !if(name1IsReverseInstr, name1, name2);
+  bit isReverseInstr = name1IsReverseInstr;
 }
 
 //
@@ -1570,14 +1572,14 @@ class sve_fp_2op_p_zds<bits<2> sz, bits<4> opc, string asm,
 
 multiclass sve_fp_2op_p_zds<bits<4> opc, string asm, string Ps,
                             SDPatternOperator op, DestructiveInstTypeEnum flags,
-                            string revname="", bit isOrig=0> {
+                            string revname="", bit isReverseInstr=0> {
   let DestructiveInstType = flags in {
   def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>,
-           SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isOrig>;
+           SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;
   def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>,
-           SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isOrig>;
+           SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;
   def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>,
-           SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isOrig>;
+           SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;
   }
 
   def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
@@ -1596,7 +1598,7 @@ multiclass sve_fp_2op_p_zds_fscale<bits<4> opc, string asm,
   def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;
 }
 
-multiclass sve_fp_2op_p_zds_zx<SDPatternOperator op> {
+multiclass sve_fp_2op_p_zds_zeroing_hsd<SDPatternOperator op> {
   def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>;
   def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>;
   def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>;
@@ -2352,16 +2354,16 @@ multiclass sve_int_bin_pred_log<bits<3> opc, string asm, SDPatternOperator op> {
 multiclass sve_int_bin_pred_arit_0<bits<3> opc, string asm, string Ps,
                                    SDPatternOperator op,
                                    DestructiveInstTypeEnum flags,
-                                   string revname="", bit isOrig=0> {
+                                   string revname="", bit isReverseInstr=0> {
   let DestructiveInstType = flags in {
   def _B : sve_int_bin_pred_arit_log<0b00, 0b00, opc, asm, ZPR8>,
-             SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isOrig>;
+             SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>;
   def _H : sve_int_bin_pred_arit_log<0b01, 0b00, opc, asm, ZPR16>,
-             SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isOrig>;
+             SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;
   def _S : sve_int_bin_pred_arit_log<0b10, 0b00, opc, asm, ZPR32>,
-             SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isOrig>;
+             SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;
   def _D : sve_int_bin_pred_arit_log<0b11, 0b00, opc, asm, ZPR64>,
-             SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isOrig>;
+             SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;
   }
 
   def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
@@ -2395,9 +2397,16 @@ multiclass sve_int_bin_pred_arit_2<bits<3> opc, string asm, SDPatternOperator op
 }
 
 // Special case for divides which are not defined for 8b/16b elements.
-multiclass sve_int_bin_pred_arit_2_div<bits<3> opc, string asm, SDPatternOperator op> {
-  def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>;
-  def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>;
+multiclass sve_int_bin_pred_arit_2_div<bits<3> opc, string asm, string Ps,
+                                       SDPatternOperator op,
+                                       DestructiveInstTypeEnum flags,
+                                       string revname="", bit isReverseInstr=0> {
+  let DestructiveInstType = flags in {
+  def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>,
+             SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;
+  def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>,
+             SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;
+  }
 
   def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
   def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
@@ -4157,9 +4166,9 @@ class sve_int_cmp<bit cmp_1, bits<2> sz8_64, bits<3> opc, string asm,
 
 multiclass SVE_SETCC_Pat<CondCode cc, CondCode invcc, ValueType predvt,
                          ValueType intvt, sve_int_cmp cmp> {
-  def : Pat<(predvt (AArch64setcc_pred predvt:$Op1, intvt:$Op2, intvt:$Op3, cc)),
+  def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, cc)),
             (cmp $Op1, $Op2, $Op3)>;
-  def : Pat<(predvt (AArch64setcc_pred predvt:$Op1, intvt:$Op2, intvt:$Op3, invcc)),
+  def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, invcc)),
             (cmp $Op1, $Op3, $Op2)>;
 }
 
@@ -4230,12 +4239,12 @@ class sve_int_scmp_vi<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty,
 multiclass SVE_SETCC_Imm_Pat<CondCode cc, CondCode commuted_cc,
                              ValueType predvt, ValueType intvt,
                              Operand immtype, Instruction cmp> {
-  def : Pat<(predvt (AArch64setcc_pred (predvt PPR_3b:$Pg),
+  def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg),
                                        (intvt ZPR:$Zs1),
                                        (intvt (AArch64dup (immtype:$imm))),
                                        cc)),
             (cmp $Pg, $Zs1, immtype:$imm)>;
-  def : Pat<(predvt (AArch64setcc_pred (predvt PPR_3b:$Pg),
+  def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg),
                                        (intvt (AArch64dup (immtype:$imm))),
                                        (intvt ZPR:$Zs1),
                                        commuted_cc)),
@@ -4757,27 +4766,24 @@ multiclass sve2_int_bin_pred_shift_imm_left<bits<4> opc, string asm,
     let Inst{9-8} = imm{4-3};
   }
 
-  def _B_Z_UNDEF : PredTwoOpImmPseudo<psName # _B, ZPR8,  tvecshiftL8,  FalseLanesUndef>;
-  def _H_Z_UNDEF : PredTwoOpImmPseudo<psName # _H, ZPR16, tvecshiftL16, FalseLanesUndef>;
-  def _S_Z_UNDEF : PredTwoOpImmPseudo<psName # _S, ZPR32, tvecshiftL32, FalseLanesUndef>;
-  def _D_Z_UNDEF : PredTwoOpImmPseudo<psName # _D, ZPR64, tvecshiftL64, FalseLanesUndef>;
-
-  def _B_Z_ZERO : PredTwoOpImmPseudo<psName # _B, ZPR8,  tvecshiftL8,  FalseLanesZero>;
-  def _H_Z_ZERO : PredTwoOpImmPseudo<psName # _H, ZPR16, tvecshiftL16, FalseLanesZero>;
-  def _S_Z_ZERO : PredTwoOpImmPseudo<psName # _S, ZPR32, tvecshiftL32, FalseLanesZero>;
-  def _D_Z_ZERO : PredTwoOpImmPseudo<psName # _D, ZPR64, tvecshiftL64, FalseLanesZero>;
-
-  def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftL8,  !cast<Pseudo>(NAME # _B_Z_ZERO)>;
-  def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1,  nxv8i16, tvecshiftL16, !cast<Pseudo>(NAME # _H_Z_ZERO)>;
-  def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1,  nxv4i32, tvecshiftL32, !cast<Pseudo>(NAME # _S_Z_ZERO)>;
-  def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1,  nxv2i64, tvecshiftL64, !cast<Pseudo>(NAME # _D_Z_ZERO)>;
-
   def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftL8,  !cast<Instruction>(NAME # _B)>;
   def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1,  nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>;
   def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1,  nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>;
   def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1,  nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>;
 }
 
+multiclass sve_int_bin_pred_shift_imm_left_zeroing_bhsd<SDPatternOperator op> {
+  def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8,  tvecshiftL8,  FalseLanesZero>;
+  def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, tvecshiftL16, FalseLanesZero>;
+  def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, tvecshiftL32, FalseLanesZero>;
+  def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, tvecshiftL64, FalseLanesZero>;
+
+  def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftL8,  !cast<Pseudo>(NAME # _ZERO_B)>;
+  def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1,  nxv8i16, tvecshiftL16, !cast<Pseudo>(NAME # _ZERO_H)>;
+  def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1,  nxv4i32, tvecshiftL32, !cast<Pseudo>(NAME # _ZERO_S)>;
+  def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1,  nxv2i64, tvecshiftL64, !cast<Pseudo>(NAME # _ZERO_D)>;
+}
+
 multiclass sve_int_bin_pred_shift_imm_right<bits<4> opc, string asm, string Ps,
                                             SDPatternOperator op = null_frag> {
   def _B : SVEPseudo2Instr<Ps # _B, 1>,
@@ -4802,7 +4808,7 @@ multiclass sve_int_bin_pred_shift_imm_right<bits<4> opc, string asm, string Ps,
   def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1,  nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>;
 }
 
-multiclass sve_int_bin_pred_shift_0_right_zx<SDPatternOperator op = null_frag> {
+multiclass sve_int_bin_pred_shift_imm_right_zeroing_bhsd<SDPatternOperator op = null_frag> {
   def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, vecshiftR8, FalseLanesZero>;
   def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, vecshiftR16, FalseLanesZero>;
   def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, vecshiftR32, FalseLanesZero>;
@@ -4839,16 +4845,16 @@ class sve_int_bin_pred_shift<bits<2> sz8_64, bit wide, bits<3> opc,
 }
 
 multiclass sve_int_bin_pred_shift<bits<3> opc, string asm, string Ps,
-                                  SDPatternOperator op, string revname, bit isOrig> {
+                                  SDPatternOperator op, string revname, bit isReverseInstr = 0> {
   let DestructiveInstType = DestructiveBinaryCommWithRev in {
   def _B : sve_int_bin_pred_shift<0b00, 0b0, opc, asm, ZPR8, ZPR8>,
-           SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isOrig>;
+           SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>;
   def _H : sve_int_bin_pred_shift<0b01, 0b0, opc, asm, ZPR16, ZPR16>,
-           SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isOrig>;
+           SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;
   def _S : sve_int_bin_pred_shift<0b10, 0b0, opc, asm, ZPR32, ZPR32>,
-           SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isOrig>;
+           SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;
   def _D : sve_int_bin_pred_shift<0b11, 0b0, opc, asm, ZPR64, ZPR64>,
-           SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isOrig>;
+           SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;
   }
   def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
   def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1,  nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
@@ -4856,7 +4862,7 @@ multiclass sve_int_bin_pred_shift<bits<3> opc, string asm, string Ps,
   def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1,  nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
 }
 
-multiclass sve_int_bin_pred_zx<SDPatternOperator op> {
+multiclass sve_int_bin_pred_zeroing_bhsd<SDPatternOperator op> {
   def _ZERO_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesZero>;
   def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>;
   def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>;
@@ -5806,9 +5812,9 @@ multiclass sve_int_perm_splice<string asm, SDPatternOperator op> {
   def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1,  nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
   def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1,  nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
 
-  def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
-  def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
-  def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
+  def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1,  nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1,  nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
+  def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1,  nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
 }
 
 class sve2_int_perm_splice_cons<bits<2> sz8_64, string asm,
@@ -5936,10 +5942,14 @@ multiclass sve_int_perm_cpy_r<string asm, SDPatternOperator op> {
   def : InstAlias<"mov $Zd, $Pg/m, $Rn",
                   (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, GPR64sp:$Rn), 1>;
 
-  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, i32, !cast<Instruction>(NAME # _B)>;
-  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1,  i32, !cast<Instruction>(NAME # _H)>;
-  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1,  i32, !cast<Instruction>(NAME # _S)>;
-  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1,  i64, !cast<Instruction>(NAME # _D)>;
+  def : Pat<(nxv16i8 (op nxv16i1:$pg, i32:$splat, nxv16i8:$passthru)),
+            (!cast<Instruction>(NAME # _B) $passthru, $pg, $splat)>;
+  def : Pat<(nxv8i16 (op nxv8i1:$pg, i32:$splat, nxv8i16:$passthru)),
+            (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>;
+  def : Pat<(nxv4i32 (op nxv4i1:$pg, i32:$splat, nxv4i32:$passthru)),
+            (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>;
+  def : Pat<(nxv2i64 (op nxv2i1:$pg, i64:$splat, nxv2i64:$passthru)),
+            (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>;
 }
 
 class sve_int_perm_cpy_v<bits<2> sz8_64, string asm, ZPRRegOp zprty,
@@ -5978,10 +5988,15 @@ multiclass sve_int_perm_cpy_v<string asm, SDPatternOperator op> {
   def : InstAlias<"mov $Zd, $Pg/m, $Vn",
                   (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, FPR64:$Vn), 1>;
 
-  def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8i1, f16, !cast<Instruction>(NAME # _H)>;
-  def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4i1, f32, !cast<Instruction>(NAME # _S)>;
-  def : SVE_3_Op_Pat<nxv2f32, op, nxv2f32, nxv2i1, f32, !cast<Instruction>(NAME # _S)>;
-  def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2i1, f64, !cast<Instruction>(NAME # _D)>;
+
+  def : Pat<(nxv8f16 (op nxv8i1:$pg, f16:$splat, nxv8f16:$passthru)),
+            (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>;
+  def : Pat<(nxv2f32 (op nxv2i1:$pg, f32:$splat, nxv2f32:$passthru)),
+            (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>;
+  def : Pat<(nxv4f32 (op nxv4i1:$pg, f32:$splat, nxv4f32:$passthru)),
+            (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>;
+  def : Pat<(nxv2f64 (op nxv2i1:$pg, f64:$splat, nxv2f64:$passthru)),
+            (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>;
 }
 
 class sve_int_perm_compact<bit sz, string asm, ZPRRegOp zprty>
@@ -7460,6 +7475,11 @@ class sve_bfloat_dot<string asm>
   let Inst{20-16} = Zm;
 }
 
+multiclass sve_bfloat_dot<string asm, SDPatternOperator op> {
+  def NAME : sve_bfloat_dot<asm>;
+  def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>;
+}
+
 class sve_bfloat_dot_indexed<string asm>
 : sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop",
   (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS:$iop)> {
@@ -7469,6 +7489,11 @@ class sve_bfloat_dot_indexed<string asm>
   let Inst{18-16} = Zm;
 }
 
+multiclass sve_bfloat_dot_indexed<string asm, SDPatternOperator op> {
+  def NAME : sve_bfloat_dot_indexed<asm>;
+  def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexS_timm, !cast<Instruction>(NAME)>;
+}
+
 class sve_bfloat_matmul<string asm>
 : I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm),
   asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {
@@ -7486,6 +7511,11 @@ class sve_bfloat_matmul<string asm>
   let ElementSize = ElementSizeH;
 }
 
+multiclass sve_bfloat_matmul<string asm, SDPatternOperator op> {
+  def NAME : sve_bfloat_matmul<asm>;
+  def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>;
+}
+
 class sve_bfloat_matmul_longvecl<bit BT, string asm>
 : sve_bfloat_matmul<asm> {
   let Inst{23}    = 0b1;
@@ -7493,6 +7523,11 @@ class sve_bfloat_matmul_longvecl<bit BT, string asm>
   let Inst{10}    = BT;
 }
 
+multiclass sve_bfloat_matmul_longvecl<bit BT, string asm, SDPatternOperator op> {
+  def NAME : sve_bfloat_matmul_longvecl<BT, asm>;
+  def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>;
+}
+
 class sve_bfloat_matmul_longvecl_idx<bit BT, string asm>
 : sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop",
   (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexH:$iop)> {
@@ -7505,6 +7540,11 @@ class sve_bfloat_matmul_longvecl_idx<bit BT, string asm>
   let Inst{10}    = BT;
 }
 
+multiclass sve_bfloat_matmul_longvecl_idx<bit BT, string asm, SDPatternOperator op> {
+  def NAME : sve_bfloat_matmul_longvecl_idx<BT, asm>;
+  def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexH_timm, !cast<Instruction>(NAME)>;
+}
+
 class sve_bfloat_convert<bit N, string asm>
 : I<(outs ZPR16:$Zd), (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn),
   asm, "\t$Zd, $Pg/m, $Zn", "", []>, Sched<[]> {
@@ -7524,6 +7564,11 @@ class sve_bfloat_convert<bit N, string asm>
   let ElementSize = ElementSizeS;
 }
 
+multiclass sve_bfloat_convert<bit N, string asm, SDPatternOperator op> {
+  def NAME : sve_bfloat_convert<N, asm>;
+  def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i1, nxv4f32, !cast<Instruction>(NAME)>;
+}
+
 //===----------------------------------------------------------------------===//
 // SVE Integer Matrix Multiply Group
 //===----------------------------------------------------------------------===//
@@ -7737,7 +7782,6 @@ multiclass sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm, SDPatter
   def : SVE_2_Op_Pat<nxv16i8,  op, nxv16i8,  nxv16i8,  !cast<Instruction>(NAME)>;
   def : SVE_2_Op_Pat<nxv8i16,  op, nxv8i16,  nxv8i16,  !cast<Instruction>(NAME)>;
   def : SVE_2_Op_Pat<nxv8f16,  op, nxv8f16,  nxv8f16,  !cast<Instruction>(NAME)>;
-  def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME)>;
   def : SVE_2_Op_Pat<nxv4i32,  op, nxv4i32,  nxv4i32,  !cast<Instruction>(NAME)>;
   def : SVE_2_Op_Pat<nxv4f32,  op, nxv4f32,  nxv4f32,  !cast<Instruction>(NAME)>;
   def : SVE_2_Op_Pat<nxv2i64,  op, nxv2i64,  nxv2i64,  !cast<Instruction>(NAME)>;
@@ -7752,3 +7796,36 @@ def am_sve_regreg_lsl0 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<0>", []
 def am_sve_regreg_lsl1 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<1>", []>;
 def am_sve_regreg_lsl2 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<2>", []>;
 def am_sve_regreg_lsl3 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<3>", []>;
+
+// Predicated pseudo floating point two operand instructions.
+multiclass sve_fp_bin_pred_hfd<SDPatternOperator op> {
+  def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>;
+  def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
+  def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
+
+  def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _UNDEF_H)>;
+  def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _UNDEF_S)>;
+  def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _UNDEF_D)>;
+}
+
+// Predicated pseudo integer two operand instructions.
+multiclass sve_int_bin_pred_bhsd<SDPatternOperator op> {
+  def _UNDEF_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>;
+  def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>;
+  def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
+  def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
+
+  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>;
+  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1,  nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>;
+  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1,  nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>;
+  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1,  nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>;
+}
+
+// As sve_int_bin_pred but when only i32 and i64 vector types are required.
+multiclass sve_int_bin_pred_sd<SDPatternOperator op> {
+  def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
+  def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
+
+  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>;
+  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>;
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h b/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h
index fb722920900f0..fd8889ea5c0dd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h
@@ -48,10 +48,6 @@ class AMDGPUAAResult : public AAResultBase<AMDGPUAAResult> {
                     AAQueryInfo &AAQI);
   bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI,
                               bool OrLocal);
-
-private:
-  bool Aliases(const MDNode *A, const MDNode *B) const;
-  bool PathAliases(const MDNode *A, const MDNode *B) const;
 };
 
 /// Analysis pass providing a never-invalidated alias analysis result.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp
index 69e48227e7320..d078fc147a36a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp
@@ -83,78 +83,83 @@ void AMDGPUArgumentUsageInfo::print(raw_ostream &OS, const Module *M) const {
   }
 }
 
-std::pair<const ArgDescriptor *, const TargetRegisterClass *>
+std::tuple<const ArgDescriptor *, const TargetRegisterClass *, LLT>
 AMDGPUFunctionArgInfo::getPreloadedValue(
-  AMDGPUFunctionArgInfo::PreloadedValue Value) const {
+    AMDGPUFunctionArgInfo::PreloadedValue Value) const {
   switch (Value) {
   case AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER: {
-    return std::make_pair(
-      PrivateSegmentBuffer ? &PrivateSegmentBuffer : nullptr,
-      &AMDGPU::SGPR_128RegClass);
+    return std::make_tuple(PrivateSegmentBuffer ? &PrivateSegmentBuffer
+                                                : nullptr,
+                           &AMDGPU::SGPR_128RegClass, LLT::vector(4, 32));
   }
   case AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR:
-    return std::make_pair(ImplicitBufferPtr ? &ImplicitBufferPtr : nullptr,
-                          &AMDGPU::SGPR_64RegClass);
+    return std::make_tuple(ImplicitBufferPtr ? &ImplicitBufferPtr : nullptr,
+                           &AMDGPU::SGPR_64RegClass,
+                           LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
   case AMDGPUFunctionArgInfo::WORKGROUP_ID_X:
-    return std::make_pair(WorkGroupIDX ? &WorkGroupIDX : nullptr,
-                          &AMDGPU::SGPR_32RegClass);
-
+    return std::make_tuple(WorkGroupIDX ? &WorkGroupIDX : nullptr,
+                           &AMDGPU::SGPR_32RegClass, LLT::scalar(32));
   case AMDGPUFunctionArgInfo::WORKGROUP_ID_Y:
-    return std::make_pair(WorkGroupIDY ? &WorkGroupIDY : nullptr,
-                          &AMDGPU::SGPR_32RegClass);
+    return std::make_tuple(WorkGroupIDY ? &WorkGroupIDY : nullptr,
+                           &AMDGPU::SGPR_32RegClass, LLT::scalar(32));
   case AMDGPUFunctionArgInfo::WORKGROUP_ID_Z:
-    return std::make_pair(WorkGroupIDZ ? &WorkGroupIDZ : nullptr,
-                          &AMDGPU::SGPR_32RegClass);
+    return std::make_tuple(WorkGroupIDZ ? &WorkGroupIDZ : nullptr,
+                           &AMDGPU::SGPR_32RegClass, LLT::scalar(32));
   case AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_WAVE_BYTE_OFFSET:
-    return std::make_pair(
-      PrivateSegmentWaveByteOffset ? &PrivateSegmentWaveByteOffset : nullptr,
-      &AMDGPU::SGPR_32RegClass);
+    return std::make_tuple(
+        PrivateSegmentWaveByteOffset ? &PrivateSegmentWaveByteOffset : nullptr,
+        &AMDGPU::SGPR_32RegClass, LLT::scalar(32));
   case AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR:
-    return std::make_pair(KernargSegmentPtr ? &KernargSegmentPtr : nullptr,
-                          &AMDGPU::SGPR_64RegClass);
+    return std::make_tuple(KernargSegmentPtr ? &KernargSegmentPtr : nullptr,
+                           &AMDGPU::SGPR_64RegClass,
+                           LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
   case AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR:
-    return std::make_pair(ImplicitArgPtr ? &ImplicitArgPtr : nullptr,
-                          &AMDGPU::SGPR_64RegClass);
+    return std::make_tuple(ImplicitArgPtr ? &ImplicitArgPtr : nullptr,
+                           &AMDGPU::SGPR_64RegClass,
+                           LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
   case AMDGPUFunctionArgInfo::DISPATCH_ID:
-    return std::make_pair(DispatchID ? &DispatchID : nullptr,
-                          &AMDGPU::SGPR_64RegClass);
+    return std::make_tuple(DispatchID ? &DispatchID : nullptr,
+                           &AMDGPU::SGPR_64RegClass, LLT::scalar(64));
   case AMDGPUFunctionArgInfo::FLAT_SCRATCH_INIT:
-    return std::make_pair(FlatScratchInit ? &FlatScratchInit : nullptr,
-                          &AMDGPU::SGPR_64RegClass);
+    return std::make_tuple(FlatScratchInit ? &FlatScratchInit : nullptr,
+                           &AMDGPU::SGPR_64RegClass, LLT::scalar(64));
   case AMDGPUFunctionArgInfo::DISPATCH_PTR:
-    return std::make_pair(DispatchPtr ? &DispatchPtr : nullptr,
-                          &AMDGPU::SGPR_64RegClass);
+    return std::make_tuple(DispatchPtr ? &DispatchPtr : nullptr,
+                           &AMDGPU::SGPR_64RegClass,
+                           LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
   case AMDGPUFunctionArgInfo::QUEUE_PTR:
-    return std::make_pair(QueuePtr ? &QueuePtr : nullptr,
-                          &AMDGPU::SGPR_64RegClass);
+    return std::make_tuple(QueuePtr ? &QueuePtr : nullptr,
+                           &AMDGPU::SGPR_64RegClass,
+                           LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
   case AMDGPUFunctionArgInfo::WORKITEM_ID_X:
-    return std::make_pair(WorkItemIDX ? &WorkItemIDX : nullptr,
-                          &AMDGPU::VGPR_32RegClass);
+    return std::make_tuple(WorkItemIDX ? &WorkItemIDX : nullptr,
+                           &AMDGPU::VGPR_32RegClass, LLT::scalar(32));
   case AMDGPUFunctionArgInfo::WORKITEM_ID_Y:
-    return std::make_pair(WorkItemIDY ? &WorkItemIDY : nullptr,
-                          &AMDGPU::VGPR_32RegClass);
+    return std::make_tuple(WorkItemIDY ? &WorkItemIDY : nullptr,
+                           &AMDGPU::VGPR_32RegClass, LLT::scalar(32));
   case AMDGPUFunctionArgInfo::WORKITEM_ID_Z:
-    return std::make_pair(WorkItemIDZ ? &WorkItemIDZ : nullptr,
-                          &AMDGPU::VGPR_32RegClass);
+    return std::make_tuple(WorkItemIDZ ? &WorkItemIDZ : nullptr,
+                           &AMDGPU::VGPR_32RegClass, LLT::scalar(32));
   }
   llvm_unreachable("unexpected preloaded value type");
 }
 
 constexpr AMDGPUFunctionArgInfo AMDGPUFunctionArgInfo::fixedABILayout() {
   AMDGPUFunctionArgInfo AI;
-  AI.PrivateSegmentBuffer = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
-  AI.DispatchPtr = AMDGPU::SGPR4_SGPR5;
-  AI.QueuePtr = AMDGPU::SGPR6_SGPR7;
+  AI.PrivateSegmentBuffer
+    = ArgDescriptor::createRegister(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3);
+  AI.DispatchPtr = ArgDescriptor::createRegister(AMDGPU::SGPR4_SGPR5);
+  AI.QueuePtr = ArgDescriptor::createRegister(AMDGPU::SGPR6_SGPR7);
 
   // Do not pass kernarg segment pointer, only pass increment version in its
   // place.
-  AI.ImplicitArgPtr = AMDGPU::SGPR8_SGPR9;
-  AI.DispatchID = AMDGPU::SGPR10_SGPR11;
+  AI.ImplicitArgPtr = ArgDescriptor::createRegister(AMDGPU::SGPR8_SGPR9);
+  AI.DispatchID = ArgDescriptor::createRegister(AMDGPU::SGPR10_SGPR11);
 
   // Skip FlatScratchInit/PrivateSegmentSize
-  AI.WorkGroupIDX = AMDGPU::SGPR12;
-  AI.WorkGroupIDY = AMDGPU::SGPR13;
-  AI.WorkGroupIDZ = AMDGPU::SGPR14;
+  AI.WorkGroupIDX = ArgDescriptor::createRegister(AMDGPU::SGPR12);
+  AI.WorkGroupIDY = ArgDescriptor::createRegister(AMDGPU::SGPR13);
+  AI.WorkGroupIDZ = ArgDescriptor::createRegister(AMDGPU::SGPR14);
 
   const unsigned Mask = 0x3ff;
   AI.WorkItemIDX = ArgDescriptor::createRegister(AMDGPU::VGPR31, Mask);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h
index b4ef0c5533dfa..576e6cfe929e2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/CodeGen/Register.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/LowLevelTypeImpl.h"
 
 namespace llvm {
 
@@ -148,7 +149,7 @@ struct AMDGPUFunctionArgInfo {
   ArgDescriptor WorkItemIDY;
   ArgDescriptor WorkItemIDZ;
 
-  std::pair<const ArgDescriptor *, const TargetRegisterClass *>
+  std::tuple<const ArgDescriptor *, const TargetRegisterClass *, LLT>
   getPreloadedValue(PreloadedValue Value) const;
 
   static constexpr AMDGPUFunctionArgInfo fixedABILayout();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index 42bf515256bb2..eef8fe2fc3b70 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -315,14 +315,12 @@ void AMDGPUAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
 
     const DataLayout &DL = GV->getParent()->getDataLayout();
     uint64_t Size = DL.getTypeAllocSize(GV->getValueType());
-    unsigned Align = GV->getAlignment();
-    if (!Align)
-      Align = 4;
+    Align Alignment = GV->getAlign().getValueOr(Align(4));
 
     emitVisibility(GVSym, GV->getVisibility(), !GV->isDeclaration());
     emitLinkage(GV, GVSym);
     if (auto TS = getTargetStreamer())
-      TS->emitAMDGPULDS(GVSym, Size, Align);
+      TS->emitAMDGPULDS(GVSym, Size, Alignment);
     return;
   }
 
@@ -1367,9 +1365,9 @@ bool AMDGPUAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
     if (AMDGPU::isInlinableIntLiteral(Val)) {
       O << Val;
     } else if (isUInt<16>(Val)) {
-      O << format("0x%" PRIx64, static_cast<uint16_t>(Val));
+      O << format("0x%" PRIx16, static_cast<uint16_t>(Val));
     } else if (isUInt<32>(Val)) {
-      O << format("0x%" PRIx64, static_cast<uint32_t>(Val));
+      O << format("0x%" PRIx32, static_cast<uint32_t>(Val));
     } else {
       O << format("0x%" PRIx64, static_cast<uint64_t>(Val));
     }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index 63da7137db600..05a4e3462a263 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -16,6 +16,7 @@
 #include "AMDGPU.h"
 #include "AMDGPUISelLowering.h"
 #include "AMDGPUSubtarget.h"
+#include "AMDGPUTargetMachine.h"
 #include "SIISelLowering.h"
 #include "SIMachineFunctionInfo.h"
 #include "SIRegisterInfo.h"
@@ -225,19 +226,6 @@ void AMDGPUCallLowering::splitToValueTypes(
     MVT RegVT = TLI.getRegisterTypeForCallingConv(Ctx, CallConv, VT);
 
     if (NumParts == 1) {
-      // Fixup EVTs to an MVT.
-      //
-      // FIXME: This is pretty hacky. Why do we have to split the type
-      // legalization logic between here and handleAssignments?
-      if (OrigArgIdx != AttributeList::ReturnIndex && VT != RegVT) {
-        assert(VT.getSizeInBits() < 32 &&
-               "unexpected illegal type");
-        Ty = Type::getInt32Ty(Ctx);
-        Register OrigReg = Reg;
-        Reg = B.getMRI()->createGenericVirtualRegister(LLT::scalar(32));
-        B.buildTrunc(OrigReg, Reg);
-      }
-
       // No splitting to do, but we want to replace the original type (e.g. [1 x
       // double] -> double).
       SplitArgs.emplace_back(Reg, Ty, OrigArg.Flags, OrigArg.IsFixed);
@@ -517,18 +505,23 @@ bool AMDGPUCallLowering::lowerFormalArgumentsKernel(
     if (AllocSize == 0)
       continue;
 
-    unsigned ABIAlign = DL.getABITypeAlignment(ArgTy);
+    Align ABIAlign = DL.getABITypeAlign(ArgTy);
 
     uint64_t ArgOffset = alignTo(ExplicitArgOffset, ABIAlign) + BaseOffset;
     ExplicitArgOffset = alignTo(ExplicitArgOffset, ABIAlign) + AllocSize;
 
+    if (Arg.use_empty()) {
+      ++i;
+      continue;
+    }
+
     ArrayRef<Register> OrigArgRegs = VRegs[i];
     Register ArgReg =
       OrigArgRegs.size() == 1
       ? OrigArgRegs[0]
       : MRI.createGenericVirtualRegister(getLLTForType(*ArgTy, DL));
+
     Align Alignment = commonAlignment(KernArgBaseAlign, ArgOffset);
-    ArgOffset = alignTo(ArgOffset, DL.getABITypeAlignment(ArgTy));
     lowerParameter(B, ArgTy, ArgOffset, Alignment, ArgReg);
     if (OrigArgRegs.size() > 1)
       unpackRegs(OrigArgRegs, ArgReg, ArgTy, B);
@@ -797,11 +790,17 @@ bool AMDGPUCallLowering::lowerFormalArguments(
   if (!MBB.empty())
     B.setInstr(*MBB.begin());
 
+  if (!IsEntryFunc) {
+    // For the fixed ABI, pass workitem IDs in the last argument register.
+    if (AMDGPUTargetMachine::EnableFixedFunctionABI)
+      TLI.allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info);
+  }
+
   FormalArgHandler Handler(B, MRI, AssignFn);
   if (!handleAssignments(CCInfo, ArgLocs, B, SplitArgs, Handler))
     return false;
 
-  if (!IsEntryFunc) {
+  if (!IsEntryFunc && !AMDGPUTargetMachine::EnableFixedFunctionABI) {
     // Special inputs come after user arguments.
     TLI.allocateSpecialInputVGPRs(CCInfo, MF, *TRI, *Info);
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 0978d31b6d76b..a795493017402 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -317,10 +317,9 @@ bool AMDGPUCodeGenPrepare::canWidenScalarExtLoad(LoadInst &I) const {
   Type *Ty = I.getType();
   const DataLayout &DL = Mod->getDataLayout();
   int TySize = DL.getTypeSizeInBits(Ty);
-  unsigned Align = I.getAlignment() ?
-                   I.getAlignment() : DL.getABITypeAlignment(Ty);
+  Align Alignment = DL.getValueOrABITypeAlignment(I.getAlign(), Ty);
 
-  return I.isSimple() && TySize < 32 && Align >= 4 && DA->isUniform(&I);
+  return I.isSimple() && TySize < 32 && Alignment >= 4 && DA->isUniform(&I);
 }
 
 bool AMDGPUCodeGenPrepare::promoteUniformOpToI32(BinaryOperator &I) const {
@@ -1018,9 +1017,9 @@ static Value *getSign32(Value *V, IRBuilder<> &Builder, const DataLayout *DL) {
   return Builder.CreateAShr(V, Builder.getInt32(31));
 }
 
-Value* AMDGPUCodeGenPrepare::expandDivRem32(IRBuilder<> &Builder,
-                                            BinaryOperator &I,
-                                            Value *Num, Value *Den) const {
+Value *AMDGPUCodeGenPrepare::expandDivRem32(IRBuilder<> &Builder,
+                                            BinaryOperator &I, Value *X,
+                                            Value *Y) const {
   Instruction::BinaryOps Opc = I.getOpcode();
   assert(Opc == Instruction::URem || Opc == Instruction::UDiv ||
          Opc == Instruction::SRem || Opc == Instruction::SDiv);
@@ -1029,27 +1028,27 @@ Value* AMDGPUCodeGenPrepare::expandDivRem32(IRBuilder<> &Builder,
   FMF.setFast();
   Builder.setFastMathFlags(FMF);
 
-  if (divHasSpecialOptimization(I, Num, Den))
+  if (divHasSpecialOptimization(I, X, Y))
     return nullptr;  // Keep it for later optimization.
 
   bool IsDiv = Opc == Instruction::UDiv || Opc == Instruction::SDiv;
   bool IsSigned = Opc == Instruction::SRem || Opc == Instruction::SDiv;
 
-  Type *Ty = Num->getType();
+  Type *Ty = X->getType();
   Type *I32Ty = Builder.getInt32Ty();
   Type *F32Ty = Builder.getFloatTy();
 
   if (Ty->getScalarSizeInBits() < 32) {
     if (IsSigned) {
-      Num = Builder.CreateSExt(Num, I32Ty);
-      Den = Builder.CreateSExt(Den, I32Ty);
+      X = Builder.CreateSExt(X, I32Ty);
+      Y = Builder.CreateSExt(Y, I32Ty);
     } else {
-      Num = Builder.CreateZExt(Num, I32Ty);
-      Den = Builder.CreateZExt(Den, I32Ty);
+      X = Builder.CreateZExt(X, I32Ty);
+      Y = Builder.CreateZExt(Y, I32Ty);
     }
   }
 
-  if (Value *Res = expandDivRem24(Builder, I, Num, Den, IsDiv, IsSigned)) {
+  if (Value *Res = expandDivRem24(Builder, I, X, Y, IsDiv, IsSigned)) {
     return IsSigned ? Builder.CreateSExtOrTrunc(Res, Ty) :
                       Builder.CreateZExtOrTrunc(Res, Ty);
   }
@@ -1059,97 +1058,79 @@ Value* AMDGPUCodeGenPrepare::expandDivRem32(IRBuilder<> &Builder,
 
   Value *Sign = nullptr;
   if (IsSigned) {
-    Value *LHSign = getSign32(Num, Builder, DL);
-    Value *RHSign = getSign32(Den, Builder, DL);
+    Value *SignX = getSign32(X, Builder, DL);
+    Value *SignY = getSign32(Y, Builder, DL);
     // Remainder sign is the same as LHS
-    Sign = IsDiv ? Builder.CreateXor(LHSign, RHSign) : LHSign;
+    Sign = IsDiv ? Builder.CreateXor(SignX, SignY) : SignX;
 
-    Num = Builder.CreateAdd(Num, LHSign);
-    Den = Builder.CreateAdd(Den, RHSign);
+    X = Builder.CreateAdd(X, SignX);
+    Y = Builder.CreateAdd(Y, SignY);
 
-    Num = Builder.CreateXor(Num, LHSign);
-    Den = Builder.CreateXor(Den, RHSign);
+    X = Builder.CreateXor(X, SignX);
+    Y = Builder.CreateXor(Y, SignY);
   }
 
-  // RCP =  URECIP(Den) = 2^32 / Den + e
-  // e is rounding error.
-  Value *DEN_F32 = Builder.CreateUIToFP(Den, F32Ty);
-
-  Function *RcpDecl = Intrinsic::getDeclaration(Mod, Intrinsic::amdgcn_rcp,
-                                                Builder.getFloatTy());
-  Value *RCP_F32 = Builder.CreateCall(RcpDecl, { DEN_F32 });
-  Constant *UINT_MAX_PLUS_1 = ConstantFP::get(F32Ty, BitsToFloat(0x4f800000));
-  Value *RCP_SCALE = Builder.CreateFMul(RCP_F32, UINT_MAX_PLUS_1);
-  Value *RCP = Builder.CreateFPToUI(RCP_SCALE, I32Ty);
-
-  // RCP_LO, RCP_HI = mul(RCP, Den) */
-  Value *RCP_LO, *RCP_HI;
-  std::tie(RCP_LO, RCP_HI) = getMul64(Builder, RCP, Den);
-
-  // NEG_RCP_LO = -RCP_LO
-  Value *NEG_RCP_LO = Builder.CreateNeg(RCP_LO);
-
-  // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO)
-  Value *RCP_HI_0_CC = Builder.CreateICmpEQ(RCP_HI, Zero);
-  Value *ABS_RCP_LO = Builder.CreateSelect(RCP_HI_0_CC, NEG_RCP_LO, RCP_LO);
-
-  // Calculate the rounding error from the URECIP instruction
-  // E = mulhu(ABS_RCP_LO, RCP)
-  Value *E = getMulHu(Builder, ABS_RCP_LO, RCP);
-
-  // RCP_A_E = RCP + E
-  Value *RCP_A_E = Builder.CreateAdd(RCP, E);
-
-  // RCP_S_E = RCP - E
-  Value *RCP_S_E = Builder.CreateSub(RCP, E);
-
-  // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E)
-  Value *Tmp0 = Builder.CreateSelect(RCP_HI_0_CC, RCP_A_E, RCP_S_E);
-
-  // Quotient = mulhu(Tmp0, Num)
-  Value *Quotient = getMulHu(Builder, Tmp0, Num);
-
-  // Num_S_Remainder = Quotient * Den
-  Value *Num_S_Remainder = Builder.CreateMul(Quotient, Den);
-
-  // Remainder = Num - Num_S_Remainder
-  Value *Remainder = Builder.CreateSub(Num, Num_S_Remainder);
-
-  // Remainder_GE_Den = Remainder >= Den;
-  Value *Remainder_GE_Den = Builder.CreateICmpUGE(Remainder, Den);
-
-  // Remainder_GE_Zero = Num >= Num_S_Remainder
-  Value *Remainder_GE_Zero = Builder.CreateICmpUGE(Num, Num_S_Remainder);
-
-  // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero
-  Value *Tmp1 = Builder.CreateAnd(Remainder_GE_Den, Remainder_GE_Zero);
-
+  // The algorithm here is based on ideas from "Software Integer Division", Tom
+  // Rodeheffer, August 2008.
+  //
+  // unsigned udiv(unsigned x, unsigned y) {
+  //   // Initial estimate of inv(y). The constant is less than 2^32 to ensure
+  //   // that this is a lower bound on inv(y), even if some of the calculations
+  //   // round up.
+  //   unsigned z = (unsigned)((4294967296.0 - 512.0) * v_rcp_f32((float)y));
+  //
+  //   // One round of UNR (Unsigned integer Newton-Raphson) to improve z.
+  //   // Empirically this is guaranteed to give a "two-y" lower bound on
+  //   // inv(y).
+  //   z += umulh(z, -y * z);
+  //
+  //   // Quotient/remainder estimate.
+  //   unsigned q = umulh(x, z);
+  //   unsigned r = x - q * y;
+  //
+  //   // Two rounds of quotient/remainder refinement.
+  //   if (r >= y) {
+  //     ++q;
+  //     r -= y;
+  //   }
+  //   if (r >= y) {
+  //     ++q;
+  //     r -= y;
+  //   }
+  //
+  //   return q;
+  // }
+
+  // Initial estimate of inv(y).
+  Value *FloatY = Builder.CreateUIToFP(Y, F32Ty);
+  Function *Rcp = Intrinsic::getDeclaration(Mod, Intrinsic::amdgcn_rcp, F32Ty);
+  Value *RcpY = Builder.CreateCall(Rcp, {FloatY});
+  Constant *Scale = ConstantFP::get(F32Ty, BitsToFloat(0x4F7FFFFE));
+  Value *ScaledY = Builder.CreateFMul(RcpY, Scale);
+  Value *Z = Builder.CreateFPToUI(ScaledY, I32Ty);
+
+  // One round of UNR.
+  Value *NegY = Builder.CreateSub(Zero, Y);
+  Value *NegYZ = Builder.CreateMul(NegY, Z);
+  Z = Builder.CreateAdd(Z, getMulHu(Builder, Z, NegYZ));
+
+  // Quotient/remainder estimate.
+  Value *Q = getMulHu(Builder, X, Z);
+  Value *R = Builder.CreateSub(X, Builder.CreateMul(Q, Y));
+
+  // First quotient/remainder refinement.
+  Value *Cond = Builder.CreateICmpUGE(R, Y);
+  if (IsDiv)
+    Q = Builder.CreateSelect(Cond, Builder.CreateAdd(Q, One), Q);
+  R = Builder.CreateSelect(Cond, Builder.CreateSub(R, Y), R);
+
+  // Second quotient/remainder refinement.
+  Cond = Builder.CreateICmpUGE(R, Y);
   Value *Res;
-  if (IsDiv) {
-    // Quotient_A_One = Quotient + 1
-    Value *Quotient_A_One = Builder.CreateAdd(Quotient, One);
-
-    // Quotient_S_One = Quotient - 1
-    Value *Quotient_S_One = Builder.CreateSub(Quotient, One);
-
-    // Div = (Tmp1 ? Quotient_A_One : Quotient)
-    Value *Div = Builder.CreateSelect(Tmp1, Quotient_A_One, Quotient);
-
-    // Div = (Remainder_GE_Zero ? Div : Quotient_S_One)
-    Res = Builder.CreateSelect(Remainder_GE_Zero, Div, Quotient_S_One);
-  } else {
-    // Remainder_S_Den = Remainder - Den
-    Value *Remainder_S_Den = Builder.CreateSub(Remainder, Den);
-
-    // Remainder_A_Den = Remainder + Den
-    Value *Remainder_A_Den = Builder.CreateAdd(Remainder, Den);
-
-    // Rem = (Tmp1 ?  Remainder_S_Den : Remainder)
-    Value *Rem = Builder.CreateSelect(Tmp1, Remainder_S_Den, Remainder);
-
-    // Rem = (Remainder_GE_Zero ? Rem : Remainder_A_Den)
-    Res = Builder.CreateSelect(Remainder_GE_Zero, Rem, Remainder_A_Den);
-  }
+  if (IsDiv)
+    Res = Builder.CreateSelect(Cond, Builder.CreateAdd(Q, One), Q);
+  else
+    Res = Builder.CreateSelect(Cond, Builder.CreateSub(R, Y), R);
 
   if (IsSigned) {
     Res = Builder.CreateXor(Res, Sign);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
index a077364896b64..c6f6a3b84e367 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
@@ -127,38 +127,6 @@ ValueKind MetadataStreamerV2::getValueKind(Type *Ty, StringRef TypeQual,
                       ValueKind::ByValue);
 }
 
-ValueType MetadataStreamerV2::getValueType(Type *Ty, StringRef TypeName) const {
-  switch (Ty->getTypeID()) {
-  case Type::IntegerTyID: {
-    auto Signed = !TypeName.startswith("u");
-    switch (Ty->getIntegerBitWidth()) {
-    case 8:
-      return Signed ? ValueType::I8 : ValueType::U8;
-    case 16:
-      return Signed ? ValueType::I16 : ValueType::U16;
-    case 32:
-      return Signed ? ValueType::I32 : ValueType::U32;
-    case 64:
-      return Signed ? ValueType::I64 : ValueType::U64;
-    default:
-      return ValueType::Struct;
-    }
-  }
-  case Type::HalfTyID:
-    return ValueType::F16;
-  case Type::FloatTyID:
-    return ValueType::F32;
-  case Type::DoubleTyID:
-    return ValueType::F64;
-  case Type::PointerTyID:
-    return getValueType(Ty->getPointerElementType(), TypeName);
-  case Type::FixedVectorTyID:
-    return getValueType(cast<VectorType>(Ty)->getElementType(), TypeName);
-  default:
-    return ValueType::Struct;
-  }
-}
-
 std::string MetadataStreamerV2::getTypeName(Type *Ty, bool Signed) const {
   switch (Ty->getTypeID()) {
   case Type::IntegerTyID: {
@@ -346,12 +314,11 @@ void MetadataStreamerV2::emitKernelArg(const Argument &Arg) {
   Type *Ty = Arg.getType();
   const DataLayout &DL = Func->getParent()->getDataLayout();
 
-  unsigned PointeeAlign = 0;
+  MaybeAlign PointeeAlign;
   if (auto PtrTy = dyn_cast<PointerType>(Ty)) {
     if (PtrTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
-      PointeeAlign = Arg.getParamAlignment();
-      if (PointeeAlign == 0)
-        PointeeAlign = DL.getABITypeAlignment(PtrTy->getElementType());
+      PointeeAlign = DL.getValueOrABITypeAlignment(Arg.getParamAlign(),
+                                                   PtrTy->getElementType());
     }
   }
 
@@ -361,7 +328,7 @@ void MetadataStreamerV2::emitKernelArg(const Argument &Arg) {
 
 void MetadataStreamerV2::emitKernelArg(const DataLayout &DL, Type *Ty,
                                        ValueKind ValueKind,
-                                       unsigned PointeeAlign, StringRef Name,
+                                       MaybeAlign PointeeAlign, StringRef Name,
                                        StringRef TypeName,
                                        StringRef BaseTypeName,
                                        StringRef AccQual, StringRef TypeQual) {
@@ -371,10 +338,9 @@ void MetadataStreamerV2::emitKernelArg(const DataLayout &DL, Type *Ty,
   Arg.mName = std::string(Name);
   Arg.mTypeName = std::string(TypeName);
   Arg.mSize = DL.getTypeAllocSize(Ty);
-  Arg.mAlign = DL.getABITypeAlignment(Ty);
+  Arg.mAlign = DL.getABITypeAlign(Ty).value();
   Arg.mValueKind = ValueKind;
-  Arg.mValueType = getValueType(Ty, BaseTypeName);
-  Arg.mPointeeAlign = PointeeAlign;
+  Arg.mPointeeAlign = PointeeAlign ? PointeeAlign->value() : 0;
 
   if (auto PtrTy = dyn_cast<PointerType>(Ty))
     Arg.mAddrSpaceQual = getAddressSpaceQualifier(PtrTy->getAddressSpace());
@@ -574,38 +540,6 @@ StringRef MetadataStreamerV3::getValueKind(Type *Ty, StringRef TypeQual,
                    : "by_value");
 }
 
-StringRef MetadataStreamerV3::getValueType(Type *Ty, StringRef TypeName) const {
-  switch (Ty->getTypeID()) {
-  case Type::IntegerTyID: {
-    auto Signed = !TypeName.startswith("u");
-    switch (Ty->getIntegerBitWidth()) {
-    case 8:
-      return Signed ? "i8" : "u8";
-    case 16:
-      return Signed ? "i16" : "u16";
-    case 32:
-      return Signed ? "i32" : "u32";
-    case 64:
-      return Signed ? "i64" : "u64";
-    default:
-      return "struct";
-    }
-  }
-  case Type::HalfTyID:
-    return "f16";
-  case Type::FloatTyID:
-    return "f32";
-  case Type::DoubleTyID:
-    return "f64";
-  case Type::PointerTyID:
-    return getValueType(Ty->getPointerElementType(), TypeName);
-  case Type::FixedVectorTyID:
-    return getValueType(cast<VectorType>(Ty)->getElementType(), TypeName);
-  default:
-    return "struct";
-  }
-}
-
 std::string MetadataStreamerV3::getTypeName(Type *Ty, bool Signed) const {
   switch (Ty->getTypeID()) {
   case Type::IntegerTyID: {
@@ -768,12 +702,11 @@ void MetadataStreamerV3::emitKernelArg(const Argument &Arg, unsigned &Offset,
   Type *Ty = Arg.getType();
   const DataLayout &DL = Func->getParent()->getDataLayout();
 
-  unsigned PointeeAlign = 0;
+  MaybeAlign PointeeAlign;
   if (auto PtrTy = dyn_cast<PointerType>(Ty)) {
     if (PtrTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
-      PointeeAlign = Arg.getParamAlignment();
-      if (PointeeAlign == 0)
-        PointeeAlign = DL.getABITypeAlignment(PtrTy->getElementType());
+      PointeeAlign = DL.getValueOrABITypeAlignment(Arg.getParamAlign(),
+                                                   PtrTy->getElementType());
     }
   }
 
@@ -786,7 +719,7 @@ void MetadataStreamerV3::emitKernelArg(const Argument &Arg, unsigned &Offset,
 void MetadataStreamerV3::emitKernelArg(const DataLayout &DL, Type *Ty,
                                        StringRef ValueKind, unsigned &Offset,
                                        msgpack::ArrayDocNode Args,
-                                       unsigned PointeeAlign, StringRef Name,
+                                       MaybeAlign PointeeAlign, StringRef Name,
                                        StringRef TypeName,
                                        StringRef BaseTypeName,
                                        StringRef AccQual, StringRef TypeQual) {
@@ -797,16 +730,14 @@ void MetadataStreamerV3::emitKernelArg(const DataLayout &DL, Type *Ty,
   if (!TypeName.empty())
     Arg[".type_name"] = Arg.getDocument()->getNode(TypeName, /*Copy=*/true);
   auto Size = DL.getTypeAllocSize(Ty);
-  auto Align = DL.getABITypeAlignment(Ty);
+  Align Alignment = DL.getABITypeAlign(Ty);
   Arg[".size"] = Arg.getDocument()->getNode(Size);
-  Offset = alignTo(Offset, Align);
+  Offset = alignTo(Offset, Alignment);
   Arg[".offset"] = Arg.getDocument()->getNode(Offset);
   Offset += Size;
   Arg[".value_kind"] = Arg.getDocument()->getNode(ValueKind, /*Copy=*/true);
-  Arg[".value_type"] =
-      Arg.getDocument()->getNode(getValueType(Ty, BaseTypeName), /*Copy=*/true);
   if (PointeeAlign)
-    Arg[".pointee_align"] = Arg.getDocument()->getNode(PointeeAlign);
+    Arg[".pointee_align"] = Arg.getDocument()->getNode(PointeeAlign->value());
 
   if (auto PtrTy = dyn_cast<PointerType>(Ty))
     if (auto Qualifier = getAddressSpaceQualifier(PtrTy->getAddressSpace()))
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
index 2618380a17599..9534fffd228d3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
@@ -20,6 +20,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/MsgPackDocument.h"
 #include "llvm/Support/AMDGPUMetadata.h"
+#include "llvm/Support/Alignment.h"
 
 namespace llvm {
 
@@ -66,8 +67,6 @@ class MetadataStreamerV3 final : public MetadataStreamer {
   StringRef getValueKind(Type *Ty, StringRef TypeQual,
                          StringRef BaseTypeName) const;
 
-  StringRef getValueType(Type *Ty, StringRef TypeName) const;
-
   std::string getTypeName(Type *Ty, bool Signed) const;
 
   msgpack::ArrayDocNode getWorkGroupDimensions(MDNode *Node) const;
@@ -90,7 +89,7 @@ class MetadataStreamerV3 final : public MetadataStreamer {
 
   void emitKernelArg(const DataLayout &DL, Type *Ty, StringRef ValueKind,
                      unsigned &Offset, msgpack::ArrayDocNode Args,
-                     unsigned PointeeAlign = 0, StringRef Name = "",
+                     MaybeAlign PointeeAlign = None, StringRef Name = "",
                      StringRef TypeName = "", StringRef BaseTypeName = "",
                      StringRef AccQual = "", StringRef TypeQual = "");
 
@@ -134,8 +133,6 @@ class MetadataStreamerV2 final : public MetadataStreamer {
   ValueKind getValueKind(Type *Ty, StringRef TypeQual,
                          StringRef BaseTypeName) const;
 
-  ValueType getValueType(Type *Ty, StringRef TypeName) const;
-
   std::string getTypeName(Type *Ty, bool Signed) const;
 
   std::vector<uint32_t> getWorkGroupDimensions(MDNode *Node) const;
@@ -160,10 +157,9 @@ class MetadataStreamerV2 final : public MetadataStreamer {
   void emitKernelArg(const Argument &Arg);
 
   void emitKernelArg(const DataLayout &DL, Type *Ty, ValueKind ValueKind,
-                     unsigned PointeeAlign = 0,
-                     StringRef Name = "", StringRef TypeName = "",
-                     StringRef BaseTypeName = "", StringRef AccQual = "",
-                     StringRef TypeQual = "");
+                     MaybeAlign PointeeAlign = None, StringRef Name = "",
+                     StringRef TypeName = "", StringRef BaseTypeName = "",
+                     StringRef AccQual = "", StringRef TypeQual = "");
 
   void emitHiddenKernelArgs(const Function &Func);
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 998e7f050a346..aaf448346b533 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -1628,6 +1628,37 @@ static MemSDNode* findMemSDNode(SDNode *N) {
   llvm_unreachable("cannot find MemSDNode in the pattern!");
 }
 
+static bool getBaseWithOffsetUsingSplitOR(SelectionDAG &DAG, SDValue Addr,
+                                          SDValue &N0, SDValue &N1) {
+  if (Addr.getValueType() == MVT::i64 && Addr.getOpcode() == ISD::BITCAST &&
+      Addr.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
+    // As we split 64-bit `or` earlier, it's complicated pattern to match, i.e.
+    // (i64 (bitcast (v2i32 (build_vector
+    //                        (or (extract_vector_elt V, 0), OFFSET),
+    //                        (extract_vector_elt V, 1)))))
+    SDValue Lo = Addr.getOperand(0).getOperand(0);
+    if (Lo.getOpcode() == ISD::OR && DAG.isBaseWithConstantOffset(Lo)) {
+      SDValue BaseLo = Lo.getOperand(0);
+      SDValue BaseHi = Addr.getOperand(0).getOperand(1);
+      // Check that split base (Lo and Hi) are extracted from the same one.
+      if (BaseLo.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+          BaseHi.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+          BaseLo.getOperand(0) == BaseHi.getOperand(0) &&
+          // Lo is statically extracted from index 0.
+          isa<ConstantSDNode>(BaseLo.getOperand(1)) &&
+          BaseLo.getConstantOperandVal(1) == 0 &&
+          // Hi is statically extracted from index 0.
+          isa<ConstantSDNode>(BaseHi.getOperand(1)) &&
+          BaseHi.getConstantOperandVal(1) == 1) {
+        N0 = BaseLo.getOperand(0).getOperand(0);
+        N1 = Lo.getOperand(1);
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
 template <bool IsSigned>
 bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N,
                                           SDValue Addr,
@@ -1638,84 +1669,91 @@ bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N,
 
   if (Subtarget->hasFlatInstOffsets() &&
       (!Subtarget->hasFlatSegmentOffsetBug() ||
-       findMemSDNode(N)->getAddressSpace() != AMDGPUAS::FLAT_ADDRESS) &&
-      CurDAG->isBaseWithConstantOffset(Addr)) {
-    SDValue N0 = Addr.getOperand(0);
-    SDValue N1 = Addr.getOperand(1);
-    uint64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue();
-
-    const SIInstrInfo *TII = Subtarget->getInstrInfo();
-    unsigned AS = findMemSDNode(N)->getAddressSpace();
-    if (TII->isLegalFLATOffset(COffsetVal, AS, IsSigned)) {
-      Addr = N0;
-      OffsetVal = COffsetVal;
-    } else {
-      // If the offset doesn't fit, put the low bits into the offset field and
-      // add the rest.
-
-      SDLoc DL(N);
-      uint64_t ImmField;
-      const unsigned NumBits = TII->getNumFlatOffsetBits(AS, IsSigned);
-      if (IsSigned) {
-        ImmField = SignExtend64(COffsetVal, NumBits);
-
-        // Don't use a negative offset field if the base offset is positive.
-        // Since the scheduler currently relies on the offset field, doing so
-        // could result in strange scheduling decisions.
-
-        // TODO: Should we not do this in the opposite direction as well?
-        if (static_cast<int64_t>(COffsetVal) > 0) {
-          if (static_cast<int64_t>(ImmField) < 0) {
-            const uint64_t OffsetMask = maskTrailingOnes<uint64_t>(NumBits - 1);
-            ImmField = COffsetVal & OffsetMask;
+       findMemSDNode(N)->getAddressSpace() != AMDGPUAS::FLAT_ADDRESS)) {
+    SDValue N0, N1;
+    if (CurDAG->isBaseWithConstantOffset(Addr)) {
+      N0 = Addr.getOperand(0);
+      N1 = Addr.getOperand(1);
+    } else if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, N0, N1)) {
+      assert(N0 && N1 && isa<ConstantSDNode>(N1));
+    }
+    if (N0 && N1) {
+      uint64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue();
+
+      const SIInstrInfo *TII = Subtarget->getInstrInfo();
+      unsigned AS = findMemSDNode(N)->getAddressSpace();
+      if (TII->isLegalFLATOffset(COffsetVal, AS, IsSigned)) {
+        Addr = N0;
+        OffsetVal = COffsetVal;
+      } else {
+        // If the offset doesn't fit, put the low bits into the offset field and
+        // add the rest.
+
+        SDLoc DL(N);
+        uint64_t ImmField;
+        const unsigned NumBits = TII->getNumFlatOffsetBits(AS, IsSigned);
+        if (IsSigned) {
+          ImmField = SignExtend64(COffsetVal, NumBits);
+
+          // Don't use a negative offset field if the base offset is positive.
+          // Since the scheduler currently relies on the offset field, doing so
+          // could result in strange scheduling decisions.
+
+          // TODO: Should we not do this in the opposite direction as well?
+          if (static_cast<int64_t>(COffsetVal) > 0) {
+            if (static_cast<int64_t>(ImmField) < 0) {
+              const uint64_t OffsetMask =
+                  maskTrailingOnes<uint64_t>(NumBits - 1);
+              ImmField = COffsetVal & OffsetMask;
+            }
           }
+        } else {
+          // TODO: Should we do this for a negative offset?
+          const uint64_t OffsetMask = maskTrailingOnes<uint64_t>(NumBits);
+          ImmField = COffsetVal & OffsetMask;
         }
-      } else {
-        // TODO: Should we do this for a negative offset?
-        const uint64_t OffsetMask = maskTrailingOnes<uint64_t>(NumBits);
-        ImmField = COffsetVal & OffsetMask;
-      }
 
-      uint64_t RemainderOffset = COffsetVal - ImmField;
+        uint64_t RemainderOffset = COffsetVal - ImmField;
 
-      assert(TII->isLegalFLATOffset(ImmField, AS, IsSigned));
-      assert(RemainderOffset + ImmField == COffsetVal);
+        assert(TII->isLegalFLATOffset(ImmField, AS, IsSigned));
+        assert(RemainderOffset + ImmField == COffsetVal);
 
-      OffsetVal = ImmField;
+        OffsetVal = ImmField;
 
-      // TODO: Should this try to use a scalar add pseudo if the base address is
-      // uniform and saddr is usable?
-      SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
-      SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
+        // TODO: Should this try to use a scalar add pseudo if the base address
+        // is uniform and saddr is usable?
+        SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
+        SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
 
-      SDNode *N0Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
-                                            DL, MVT::i32, N0, Sub0);
-      SDNode *N0Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
-                                            DL, MVT::i32, N0, Sub1);
+        SDNode *N0Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
+                                              MVT::i32, N0, Sub0);
+        SDNode *N0Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
+                                              MVT::i32, N0, Sub1);
 
-      SDValue AddOffsetLo
-        = getMaterializedScalarImm32(Lo_32(RemainderOffset), DL);
-      SDValue AddOffsetHi
-        = getMaterializedScalarImm32(Hi_32(RemainderOffset), DL);
+        SDValue AddOffsetLo =
+            getMaterializedScalarImm32(Lo_32(RemainderOffset), DL);
+        SDValue AddOffsetHi =
+            getMaterializedScalarImm32(Hi_32(RemainderOffset), DL);
 
-      SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i1);
-      SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
+        SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i1);
+        SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
 
-      SDNode *Add = CurDAG->getMachineNode(
-        AMDGPU::V_ADD_I32_e64, DL, VTs,
-        {AddOffsetLo, SDValue(N0Lo, 0), Clamp});
+        SDNode *Add =
+            CurDAG->getMachineNode(AMDGPU::V_ADD_I32_e64, DL, VTs,
+                                   {AddOffsetLo, SDValue(N0Lo, 0), Clamp});
 
-      SDNode *Addc = CurDAG->getMachineNode(
-        AMDGPU::V_ADDC_U32_e64, DL, VTs,
-        {AddOffsetHi, SDValue(N0Hi, 0), SDValue(Add, 1), Clamp});
+        SDNode *Addc = CurDAG->getMachineNode(
+            AMDGPU::V_ADDC_U32_e64, DL, VTs,
+            {AddOffsetHi, SDValue(N0Hi, 0), SDValue(Add, 1), Clamp});
 
-      SDValue RegSequenceArgs[] = {
-        CurDAG->getTargetConstant(AMDGPU::VReg_64RegClassID, DL, MVT::i32),
-        SDValue(Add, 0), Sub0, SDValue(Addc, 0), Sub1
-      };
+        SDValue RegSequenceArgs[] = {
+            CurDAG->getTargetConstant(AMDGPU::VReg_64RegClassID, DL, MVT::i32),
+            SDValue(Add, 0), Sub0, SDValue(Addc, 0), Sub1};
 
-      Addr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
-                                            MVT::i64, RegSequenceArgs), 0);
+        Addr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
+                                              MVT::i64, RegSequenceArgs),
+                       0);
+      }
     }
   }
 
@@ -1824,15 +1862,21 @@ bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
   // A 32-bit (address + offset) should not cause unsigned 32-bit integer
   // wraparound, because s_load instructions perform the addition in 64 bits.
   if ((Addr.getValueType() != MVT::i32 ||
-       Addr->getFlags().hasNoUnsignedWrap()) &&
-      (CurDAG->isBaseWithConstantOffset(Addr) ||
-       Addr.getOpcode() == ISD::ADD)) {
-    SDValue N0 = Addr.getOperand(0);
-    SDValue N1 = Addr.getOperand(1);
-
-    if (SelectSMRDOffset(N1, Offset, Imm)) {
-      SBase = Expand32BitAddress(N0);
-      return true;
+       Addr->getFlags().hasNoUnsignedWrap())) {
+    SDValue N0, N1;
+    // Extract the base and offset if possible.
+    if (CurDAG->isBaseWithConstantOffset(Addr) ||
+        Addr.getOpcode() == ISD::ADD) {
+      N0 = Addr.getOperand(0);
+      N1 = Addr.getOperand(1);
+    } else if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, N0, N1)) {
+      assert(N0 && N1 && isa<ConstantSDNode>(N1));
+    }
+    if (N0 && N1) {
+      if (SelectSMRDOffset(N1, Offset, Imm)) {
+        SBase = Expand32BitAddress(N0);
+        return true;
+      }
     }
   }
   SBase = Expand32BitAddress(Addr);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index d856649092b59..9f49136c986f9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -443,7 +443,7 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::UREM, VT, Expand);
     setOperationAction(ISD::SMUL_LOHI, VT, Expand);
     setOperationAction(ISD::UMUL_LOHI, VT, Expand);
-    setOperationAction(ISD::SDIVREM, VT, Custom);
+    setOperationAction(ISD::SDIVREM, VT, Expand);
     setOperationAction(ISD::UDIVREM, VT, Expand);
     setOperationAction(ISD::SELECT, VT, Expand);
     setOperationAction(ISD::VSELECT, VT, Expand);
@@ -1010,7 +1010,7 @@ void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
   const unsigned ExplicitOffset = ST.getExplicitKernelArgOffset(Fn);
   CallingConv::ID CC = Fn.getCallingConv();
 
-  unsigned MaxAlign = 1;
+  Align MaxAlign = Align(1);
   uint64_t ExplicitArgOffset = 0;
   const DataLayout &DL = Fn.getParent()->getDataLayout();
 
@@ -1018,12 +1018,12 @@ void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
 
   for (const Argument &Arg : Fn.args()) {
     Type *BaseArgTy = Arg.getType();
-    unsigned Align = DL.getABITypeAlignment(BaseArgTy);
-    MaxAlign = std::max(Align, MaxAlign);
+    Align Alignment = DL.getABITypeAlign(BaseArgTy);
+    MaxAlign = std::max(Alignment, MaxAlign);
     unsigned AllocSize = DL.getTypeAllocSize(BaseArgTy);
 
-    uint64_t ArgOffset = alignTo(ExplicitArgOffset, Align) + ExplicitOffset;
-    ExplicitArgOffset = alignTo(ExplicitArgOffset, Align) + AllocSize;
+    uint64_t ArgOffset = alignTo(ExplicitArgOffset, Alignment) + ExplicitOffset;
+    ExplicitArgOffset = alignTo(ExplicitArgOffset, Alignment) + AllocSize;
 
     // We're basically throwing away everything passed into us and starting over
     // to get accurate in-memory offsets. The "PartOffset" is completely useless
@@ -1321,7 +1321,7 @@ SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI,
 
     // TODO: We could emit code to handle the initialization somewhere.
     if (!hasDefinedInitializer(GV)) {
-      unsigned Offset = MFI->allocateLDSGlobal(DL, *GV);
+      unsigned Offset = MFI->allocateLDSGlobal(DL, *cast<GlobalVariable>(GV));
       return DAG.getConstant(Offset, SDLoc(Op), Op.getValueType());
     }
   }
@@ -1976,104 +1976,43 @@ SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
       return Res;
   }
 
-  SDValue Num = Op.getOperand(0);
-  SDValue Den = Op.getOperand(1);
-
-  // RCP =  URECIP(Den) = 2^32 / Den + e
-  // e is rounding error.
-  SDValue RCP = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Den);
-
-  // RCP_LO = mul(RCP, Den) */
-  SDValue RCP_LO = DAG.getNode(ISD::MUL, DL, VT, RCP, Den);
-
-  // RCP_HI = mulhu (RCP, Den) */
-  SDValue RCP_HI = DAG.getNode(ISD::MULHU, DL, VT, RCP, Den);
-
-  // NEG_RCP_LO = -RCP_LO
-  SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
-                                                     RCP_LO);
-
-  const SDValue Zero = DAG.getConstant(0, DL, VT);
-  const EVT CCVT = getSetCCResultType(DAG.getDataLayout(),
-                                      *DAG.getContext(), VT);
-
-  // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO)
-  SDValue CmpRcpHiZero = DAG.getSetCC(DL, CCVT, RCP_HI, Zero, ISD::SETEQ);
-  SDValue ABS_RCP_LO = DAG.getNode(ISD::SELECT,
-                                   DL, VT, CmpRcpHiZero, NEG_RCP_LO, RCP_LO);
-
-  // Calculate the rounding error from the URECIP instruction
-  // E = mulhu(ABS_RCP_LO, RCP)
-  SDValue E = DAG.getNode(ISD::MULHU, DL, VT, ABS_RCP_LO, RCP);
-
-  // RCP_A_E = RCP + E
-  SDValue RCP_A_E = DAG.getNode(ISD::ADD, DL, VT, RCP, E);
-
-  // RCP_S_E = RCP - E
-  SDValue RCP_S_E = DAG.getNode(ISD::SUB, DL, VT, RCP, E);
-
-  // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E)
-  SDValue Tmp0 = DAG.getNode(ISD::SELECT, DL, VT,
-                             CmpRcpHiZero, RCP_A_E, RCP_S_E);
-
-  // Quotient = mulhu(Tmp0, Num)
-  SDValue Quotient = DAG.getNode(ISD::MULHU, DL, VT, Tmp0, Num);
-
-  // Num_S_Remainder = Quotient * Den
-  SDValue Num_S_Remainder = DAG.getNode(ISD::MUL, DL, VT, Quotient, Den);
-
-  // Remainder = Num - Num_S_Remainder
-  SDValue Remainder = DAG.getNode(ISD::SUB, DL, VT, Num, Num_S_Remainder);
-
-  // Remainder_GE_Den = (Remainder >= Den)
-  SDValue Remainder_GE_Den = DAG.getSetCC(DL, CCVT, Remainder, Den, ISD::SETUGE);
-
-  // Remainder_GE_Zero = (Num >= Num_S_Remainder)
-  SDValue Remainder_GE_Zero = DAG.getSetCC(DL, CCVT, Num, Num_S_Remainder,
-                                           ISD::SETUGE);
-
-  // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero
-  SDValue Tmp1 = DAG.getNode(ISD::AND, DL, CCVT, Remainder_GE_Den,
-                             Remainder_GE_Zero);
-
-  // Calculate Division result:
-
-  // Quotient_A_One = Quotient + 1
-  SDValue Quotient_A_One = DAG.getNode(ISD::ADD, DL, VT, Quotient,
-                                       DAG.getConstant(1, DL, VT));
-
-  // Quotient_S_One = Quotient - 1
-  SDValue Quotient_S_One = DAG.getNode(ISD::SUB, DL, VT, Quotient,
-                                       DAG.getConstant(1, DL, VT));
-
-  // Div = (Tmp1 ? Quotient_A_One : Quotient)
-  SDValue Div = DAG.getNode(ISD::SELECT, DL, VT, Tmp1,
-                            Quotient_A_One, Quotient);
-
-  // Div = (Remainder_GE_Zero ? Div : Quotient_S_One)
-  Div = DAG.getNode(ISD::SELECT, DL, VT, Remainder_GE_Zero,
-                    Div, Quotient_S_One);
-
-  // Calculate Rem result:
-
-  // Remainder_S_Den = Remainder - Den
-  SDValue Remainder_S_Den = DAG.getNode(ISD::SUB, DL, VT, Remainder, Den);
-
-  // Remainder_A_Den = Remainder + Den
-  SDValue Remainder_A_Den = DAG.getNode(ISD::ADD, DL, VT, Remainder, Den);
-
-  // Rem = (Tmp1 ? Remainder_S_Den : Remainder)
-  SDValue Rem = DAG.getNode(ISD::SELECT, DL, VT, Tmp1,
-                            Remainder_S_Den, Remainder);
+  SDValue X = Op.getOperand(0);
+  SDValue Y = Op.getOperand(1);
 
-  // Rem = (Remainder_GE_Zero ? Rem : Remainder_A_Den)
-  Rem = DAG.getNode(ISD::SELECT, DL, VT,
-                    Remainder_GE_Zero, Rem, Remainder_A_Den);
-  SDValue Ops[2] = {
-    Div,
-    Rem
-  };
-  return DAG.getMergeValues(Ops, DL);
+  // See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
+  // algorithm used here.
+
+  // Initial estimate of inv(y).
+  SDValue Z = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Y);
+
+  // One round of UNR.
+  SDValue NegY = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Y);
+  SDValue NegYZ = DAG.getNode(ISD::MUL, DL, VT, NegY, Z);
+  Z = DAG.getNode(ISD::ADD, DL, VT, Z,
+                  DAG.getNode(ISD::MULHU, DL, VT, Z, NegYZ));
+
+  // Quotient/remainder estimate.
+  SDValue Q = DAG.getNode(ISD::MULHU, DL, VT, X, Z);
+  SDValue R =
+      DAG.getNode(ISD::SUB, DL, VT, X, DAG.getNode(ISD::MUL, DL, VT, Q, Y));
+
+  // First quotient/remainder refinement.
+  EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
+  SDValue One = DAG.getConstant(1, DL, VT);
+  SDValue Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE);
+  Q = DAG.getNode(ISD::SELECT, DL, VT, Cond,
+                  DAG.getNode(ISD::ADD, DL, VT, Q, One), Q);
+  R = DAG.getNode(ISD::SELECT, DL, VT, Cond,
+                  DAG.getNode(ISD::SUB, DL, VT, R, Y), R);
+
+  // Second quotient/remainder refinement.
+  Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE);
+  Q = DAG.getNode(ISD::SELECT, DL, VT, Cond,
+                  DAG.getNode(ISD::ADD, DL, VT, Q, One), Q);
+  R = DAG.getNode(ISD::SELECT, DL, VT, Cond,
+                  DAG.getNode(ISD::SUB, DL, VT, R, Y), R);
+
+  return DAG.getMergeValues({Q, R}, DL);
 }
 
 SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
@@ -2931,16 +2870,17 @@ SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
   EVT VT = LN->getMemoryVT();
 
   unsigned Size = VT.getStoreSize();
-  unsigned Align = LN->getAlignment();
-  if (Align < Size && isTypeLegal(VT)) {
+  Align Alignment = LN->getAlign();
+  if (Alignment < Size && isTypeLegal(VT)) {
     bool IsFast;
     unsigned AS = LN->getAddressSpace();
 
     // Expand unaligned loads earlier than legalization. Due to visitation order
     // problems during legalization, the emitted instructions to pack and unpack
     // the bytes again are not eliminated in the case of an unaligned copy.
-    if (!allowsMisalignedMemoryAccesses(
-            VT, AS, Align, LN->getMemOperand()->getFlags(), &IsFast)) {
+    if (!allowsMisalignedMemoryAccesses(VT, AS, Alignment.value(),
+                                        LN->getMemOperand()->getFlags(),
+                                        &IsFast)) {
       SDValue Ops[2];
 
       if (VT.isVector())
@@ -2985,8 +2925,8 @@ SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
 
   SDLoc SL(N);
   SelectionDAG &DAG = DCI.DAG;
-  unsigned Align = SN->getAlignment();
-  if (Align < Size && isTypeLegal(VT)) {
+  Align Alignment = SN->getAlign();
+  if (Alignment < Size && isTypeLegal(VT)) {
     bool IsFast;
     unsigned AS = SN->getAddressSpace();
 
@@ -2994,8 +2934,9 @@ SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
     // order problems during legalization, the emitted instructions to pack and
     // unpack the bytes again are not eliminated in the case of an unaligned
     // copy.
-    if (!allowsMisalignedMemoryAccesses(
-            VT, AS, Align, SN->getMemOperand()->getFlags(), &IsFast)) {
+    if (!allowsMisalignedMemoryAccesses(VT, AS, Alignment.value(),
+                                        SN->getMemOperand()->getFlags(),
+                                        &IsFast)) {
       if (VT.isVector())
         return scalarizeVectorStore(SN, DAG);
 
@@ -4589,11 +4530,10 @@ void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
   }
   case AMDGPUISD::LDS: {
     auto GA = cast<GlobalAddressSDNode>(Op.getOperand(0).getNode());
-    unsigned Align = GA->getGlobal()->getAlignment();
+    Align Alignment = GA->getGlobal()->getPointerAlignment(DAG.getDataLayout());
 
     Known.Zero.setHighBits(16);
-    if (Align)
-      Known.Zero.setLowBits(Log2_32(Align));
+    Known.Zero.setLowBits(Log2(Alignment));
     break;
   }
   case ISD::INTRINSIC_WO_CHAIN: {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 6380d6b91d617..c3d5e78964c87 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -889,6 +889,8 @@ bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const {
     return constrainCopyLikeIntrin(I, AMDGPU::WWM);
   case Intrinsic::amdgcn_div_scale:
     return selectDivScale(I);
+  case Intrinsic::amdgcn_icmp:
+    return selectIntrinsicIcmp(I);
   default:
     return selectImpl(I, *CoverageInfo);
   }
@@ -1009,6 +1011,34 @@ bool AMDGPUInstructionSelector::selectG_ICMP(MachineInstr &I) const {
   return Ret;
 }
 
+bool AMDGPUInstructionSelector::selectIntrinsicIcmp(MachineInstr &I) const {
+  Register Dst = I.getOperand(0).getReg();
+  if (isVCC(Dst, *MRI))
+    return false;
+
+  if (MRI->getType(Dst).getSizeInBits() != STI.getWavefrontSize())
+    return false;
+
+  MachineBasicBlock *BB = I.getParent();
+  const DebugLoc &DL = I.getDebugLoc();
+  Register SrcReg = I.getOperand(2).getReg();
+  unsigned Size = RBI.getSizeInBits(SrcReg, *MRI, TRI);
+  auto Pred = static_cast<CmpInst::Predicate>(I.getOperand(4).getImm());
+
+  int Opcode = getV_CMPOpcode(Pred, Size);
+  if (Opcode == -1)
+    return false;
+
+  MachineInstr *ICmp = BuildMI(*BB, &I, DL, TII.get(Opcode), Dst)
+                           .add(I.getOperand(2))
+                           .add(I.getOperand(3));
+  RBI.constrainGenericRegister(ICmp->getOperand(0).getReg(), *TRI.getBoolRC(),
+                               *MRI);
+  bool Ret = constrainSelectedInstRegOperands(*ICmp, TII, TRI, RBI);
+  I.eraseFromParent();
+  return Ret;
+}
+
 bool AMDGPUInstructionSelector::selectEndCfIntrinsic(MachineInstr &MI) const {
   // FIXME: Manually selecting to avoid dealiing with the SReg_1 trick
   // SelectionDAG uses for wave32 vs wave64.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index 0ac6788c69b8a..f8a8b5db4b556 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -106,6 +106,7 @@ class AMDGPUInstructionSelector : public InstructionSelector {
 
   bool selectInterpP1F16(MachineInstr &MI) const;
   bool selectDivScale(MachineInstr &MI) const;
+  bool selectIntrinsicIcmp(MachineInstr &MI) const;
   bool selectG_INTRINSIC(MachineInstr &I) const;
 
   bool selectEndCfIntrinsic(MachineInstr &MI) const;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
index 706053a4e3157..5cb7ac320d2fb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -528,7 +528,7 @@ class Constants {
 int TWO_PI = 0x40c90fdb;
 int PI = 0x40490fdb;
 int TWO_PI_INV = 0x3e22f983;
-int FP_UINT_MAX_PLUS_1 = 0x4f800000;    // 1 << 32 in floating point encoding
+int FP_4294966784 = 0x4f7ffffe; // 4294966784 = 4294967296 - 512 = 2^32 - 2^9
 int FP16_ONE = 0x3C00;
 int FP16_NEG_ONE = 0xBC00;
 int FP32_ONE = 0x3f800000;
@@ -815,3 +815,8 @@ def any_fmad : PatFrags<(ops node:$src0, node:$src1, node:$src2),
   [(fmad node:$src0, node:$src1, node:$src2),
    (AMDGPUfmad_ftz node:$src0, node:$src1, node:$src2)]
 >;
+
+// FIXME: fsqrt should not select directly
+def any_amdgcn_sqrt : PatFrags<(ops node:$src0),
+  [(fsqrt node:$src0), (int_amdgcn_sqrt node:$src0)]
+>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 68d0012e33143..0802f2a2d08a7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -121,10 +121,10 @@ static LegalizeMutation bitcastToRegisterType(unsigned TypeIdx) {
     unsigned Size = Ty.getSizeInBits();
 
     LLT CoercedTy;
-    if (Size < 32) {
+    if (Size <= 32) {
       // <2 x s8> -> s16
-      assert(Size == 16);
-      CoercedTy = LLT::scalar(16);
+      // <4 x s8> -> s32
+      CoercedTy = LLT::scalar(Size);
     } else
       CoercedTy = LLT::scalarOrVector(Size / 32, 32);
 
@@ -982,15 +982,14 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     Actions.bitcastIf(
       [=](const LegalityQuery &Query) -> bool {
         const LLT Ty = Query.Types[0];
+        const unsigned Size = Ty.getSizeInBits();
 
-        // Do not cast an extload/truncstore.
-        if (Ty.getSizeInBits() != Query.MMODescrs[0].SizeInBits)
-          return false;
+        if (Size != Query.MMODescrs[0].SizeInBits)
+          return Size <= 32 && Ty.isVector();
 
         if (loadStoreBitcastWorkaround(Ty) && isRegisterType(Ty))
           return true;
-        const unsigned Size = Ty.getSizeInBits();
-        return Ty.isVector() && isRegisterSize(Size) &&
+        return Ty.isVector() && (Size <= 32 || isRegisterSize(Size)) &&
                !isRegisterVectorElementType(Ty.getElementType());
       }, bitcastToRegisterType(0));
 
@@ -2003,10 +2002,12 @@ bool AMDGPULegalizerInfo::legalizeSinCos(
   return true;
 }
 
-bool AMDGPULegalizerInfo::buildPCRelGlobalAddress(
-  Register DstReg, LLT PtrTy,
-  MachineIRBuilder &B, const GlobalValue *GV,
-  unsigned Offset, unsigned GAFlags) const {
+bool AMDGPULegalizerInfo::buildPCRelGlobalAddress(Register DstReg, LLT PtrTy,
+                                                  MachineIRBuilder &B,
+                                                  const GlobalValue *GV,
+                                                  int64_t Offset,
+                                                  unsigned GAFlags) const {
+  assert(isInt<32>(Offset + 4) && "32-bit offset is expected!");
   // In order to support pc-relative addressing, SI_PC_ADD_REL_OFFSET is lowered
   // to the following code sequence:
   //
@@ -2097,7 +2098,9 @@ bool AMDGPULegalizerInfo::legalizeGlobalValue(
         return true; // Leave in place;
       }
 
-      B.buildConstant(DstReg, MFI->allocateLDSGlobal(B.getDataLayout(), *GV));
+      B.buildConstant(
+          DstReg,
+          MFI->allocateLDSGlobal(B.getDataLayout(), *cast<GlobalVariable>(GV)));
       MI.eraseFromParent();
       return true;
     }
@@ -2439,7 +2442,8 @@ const ArgDescriptor *AMDGPULegalizerInfo::getArgDescriptor(
   const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
   const ArgDescriptor *Arg;
   const TargetRegisterClass *RC;
-  std::tie(Arg, RC) = MFI->getPreloadedValue(ArgType);
+  LLT ArgTy;
+  std::tie(Arg, RC, ArgTy) = MFI->getPreloadedValue(ArgType);
   if (!Arg) {
     LLVM_DEBUG(dbgs() << "Required arg register missing\n");
     return nullptr;
@@ -2519,114 +2523,56 @@ bool AMDGPULegalizerInfo::legalizeFDIV(MachineInstr &MI,
   return false;
 }
 
-static Register buildDivRCP(MachineIRBuilder &B, Register Src) {
-  const LLT S32 = LLT::scalar(32);
-
-  auto Cvt0 = B.buildUITOFP(S32, Src);
-  auto RcpIFlag = B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {S32}, {Cvt0});
-  auto FPUIntMaxPlus1 = B.buildFConstant(S32, BitsToFloat(0x4f800000));
-  auto Mul = B.buildFMul(S32, RcpIFlag, FPUIntMaxPlus1);
-  return B.buildFPTOUI(S32, Mul).getReg(0);
-}
-
 void AMDGPULegalizerInfo::legalizeUDIV_UREM32Impl(MachineIRBuilder &B,
                                                   Register DstReg,
-                                                  Register Num,
-                                                  Register Den,
-                                                  bool IsRem) const {
+                                                  Register X,
+                                                  Register Y,
+                                                  bool IsDiv) const {
   const LLT S1 = LLT::scalar(1);
   const LLT S32 = LLT::scalar(32);
 
-  // RCP =  URECIP(Den) = 2^32 / Den + e
-  // e is rounding error.
-  auto RCP = buildDivRCP(B, Den);
-
-  // RCP_LO = mul(RCP, Den)
-  auto RCP_LO = B.buildMul(S32, RCP, Den);
-
-  // RCP_HI = mulhu (RCP, Den) */
-  auto RCP_HI = B.buildUMulH(S32, RCP, Den);
-
-  // NEG_RCP_LO = -RCP_LO
-  auto Zero = B.buildConstant(S32, 0);
-  auto NEG_RCP_LO = B.buildSub(S32, Zero, RCP_LO);
-
-  // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO)
-  auto CmpRcpHiZero = B.buildICmp(CmpInst::ICMP_EQ, S1, RCP_HI, Zero);
-  auto ABS_RCP_LO = B.buildSelect(S32, CmpRcpHiZero, NEG_RCP_LO, RCP_LO);
-
-  // Calculate the rounding error from the URECIP instruction
-  // E = mulhu(ABS_RCP_LO, RCP)
-  auto E = B.buildUMulH(S32, ABS_RCP_LO, RCP);
-
-  // RCP_A_E = RCP + E
-  auto RCP_A_E = B.buildAdd(S32, RCP, E);
-
-  // RCP_S_E = RCP - E
-  auto RCP_S_E = B.buildSub(S32, RCP, E);
-
-  // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E)
-  auto Tmp0 = B.buildSelect(S32, CmpRcpHiZero, RCP_A_E, RCP_S_E);
-
-  // Quotient = mulhu(Tmp0, Num)stmp
-  auto Quotient = B.buildUMulH(S32, Tmp0, Num);
-
-  // Num_S_Remainder = Quotient * Den
-  auto Num_S_Remainder = B.buildMul(S32, Quotient, Den);
-
-  // Remainder = Num - Num_S_Remainder
-  auto Remainder = B.buildSub(S32, Num, Num_S_Remainder);
-
-  // Remainder_GE_Den = Remainder >= Den
-  auto Remainder_GE_Den = B.buildICmp(CmpInst::ICMP_UGE, S1, Remainder, Den);
+  // See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
+  // algorithm used here.
 
-  // Remainder_GE_Zero = Num >= Num_S_Remainder;
-  auto Remainder_GE_Zero = B.buildICmp(CmpInst::ICMP_UGE, S1,
-                                       Num, Num_S_Remainder);
+  // Initial estimate of inv(y).
+  auto FloatY = B.buildUITOFP(S32, Y);
+  auto RcpIFlag = B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {S32}, {FloatY});
+  auto Scale = B.buildFConstant(S32, BitsToFloat(0x4f7ffffe));
+  auto ScaledY = B.buildFMul(S32, RcpIFlag, Scale);
+  auto Z = B.buildFPTOUI(S32, ScaledY);
 
-  // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero
-  auto Tmp1 = B.buildAnd(S1, Remainder_GE_Den, Remainder_GE_Zero);
+  // One round of UNR.
+  auto NegY = B.buildSub(S32, B.buildConstant(S32, 0), Y);
+  auto NegYZ = B.buildMul(S32, NegY, Z);
+  Z = B.buildAdd(S32, Z, B.buildUMulH(S32, Z, NegYZ));
 
-  // Calculate Division result:
+  // Quotient/remainder estimate.
+  auto Q = B.buildUMulH(S32, X, Z);
+  auto R = B.buildSub(S32, X, B.buildMul(S32, Q, Y));
 
-  // Quotient_A_One = Quotient + 1
+  // First quotient/remainder refinement.
   auto One = B.buildConstant(S32, 1);
-  auto Quotient_A_One = B.buildAdd(S32, Quotient, One);
-
-  // Quotient_S_One = Quotient - 1
-  auto Quotient_S_One = B.buildSub(S32, Quotient, One);
-
-  // Div = (Tmp1 ? Quotient_A_One : Quotient)
-  auto Div = B.buildSelect(S32, Tmp1, Quotient_A_One, Quotient);
-
-  // Div = (Remainder_GE_Zero ? Div : Quotient_S_One)
-  if (IsRem) {
-    Div = B.buildSelect(S32, Remainder_GE_Zero, Div, Quotient_S_One);
-
-    // Calculate Rem result:
-    auto Remainder_S_Den = B.buildSub(S32, Remainder, Den);
-
-    // Remainder_A_Den = Remainder + Den
-    auto Remainder_A_Den = B.buildAdd(S32, Remainder, Den);
-
-    // Rem = (Tmp1 ? Remainder_S_Den : Remainder)
-    auto Rem = B.buildSelect(S32, Tmp1, Remainder_S_Den, Remainder);
-
-    // Rem = (Remainder_GE_Zero ? Rem : Remainder_A_Den)
-    B.buildSelect(DstReg, Remainder_GE_Zero, Rem, Remainder_A_Den);
-  } else {
-    B.buildSelect(DstReg, Remainder_GE_Zero, Div, Quotient_S_One);
-  }
+  auto Cond = B.buildICmp(CmpInst::ICMP_UGE, S1, R, Y);
+  if (IsDiv)
+    Q = B.buildSelect(S32, Cond, B.buildAdd(S32, Q, One), Q);
+  R = B.buildSelect(S32, Cond, B.buildSub(S32, R, Y), R);
+
+  // Second quotient/remainder refinement.
+  Cond = B.buildICmp(CmpInst::ICMP_UGE, S1, R, Y);
+  if (IsDiv)
+    B.buildSelect(DstReg, Cond, B.buildAdd(S32, Q, One), Q);
+  else
+    B.buildSelect(DstReg, Cond, B.buildSub(S32, R, Y), R);
 }
 
 bool AMDGPULegalizerInfo::legalizeUDIV_UREM32(MachineInstr &MI,
                                               MachineRegisterInfo &MRI,
                                               MachineIRBuilder &B) const {
-  const bool IsRem = MI.getOpcode() == AMDGPU::G_UREM;
+  const bool IsDiv = MI.getOpcode() == AMDGPU::G_UDIV;
   Register DstReg = MI.getOperand(0).getReg();
   Register Num = MI.getOperand(1).getReg();
   Register Den = MI.getOperand(2).getReg();
-  legalizeUDIV_UREM32Impl(B, DstReg, Num, Den, IsRem);
+  legalizeUDIV_UREM32Impl(B, DstReg, Num, Den, IsDiv);
   MI.eraseFromParent();
   return true;
 }
@@ -2674,15 +2620,14 @@ static std::pair<Register, Register> emitReciprocalU64(MachineIRBuilder &B,
   return {ResultLo.getReg(0), ResultHi.getReg(0)};
 }
 
-bool AMDGPULegalizerInfo::legalizeUDIV_UREM64(MachineInstr &MI,
-                                              MachineRegisterInfo &MRI,
-                                              MachineIRBuilder &B) const {
-  const bool IsDiv = MI.getOpcode() == TargetOpcode::G_UDIV;
+void AMDGPULegalizerInfo::legalizeUDIV_UREM64Impl(MachineIRBuilder &B,
+                                                  Register DstReg,
+                                                  Register Numer,
+                                                  Register Denom,
+                                                  bool IsDiv) const {
   const LLT S32 = LLT::scalar(32);
   const LLT S64 = LLT::scalar(64);
   const LLT S1 = LLT::scalar(1);
-  Register Numer = MI.getOperand(1).getReg();
-  Register Denom = MI.getOperand(2).getReg();
   Register RcpLo, RcpHi;
 
   std::tie(RcpLo, RcpHi) = emitReciprocalU64(B, Denom);
@@ -2778,75 +2723,84 @@ bool AMDGPULegalizerInfo::legalizeUDIV_UREM64(MachineInstr &MI,
   if (IsDiv) {
     auto Sel1 = B.buildSelect(
         S64, B.buildICmp(CmpInst::ICMP_NE, S1, C6, Zero32), Add4, Add3);
-    B.buildSelect(MI.getOperand(0),
+    B.buildSelect(DstReg,
                   B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32), Sel1, MulHi3);
   } else {
     auto Sel2 = B.buildSelect(
         S64, B.buildICmp(CmpInst::ICMP_NE, S1, C6, Zero32), Sub3, Sub2);
-    B.buildSelect(MI.getOperand(0),
+    B.buildSelect(DstReg,
                   B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32), Sel2, Sub1);
   }
-
-  MI.eraseFromParent();
-  return true;
 }
 
 bool AMDGPULegalizerInfo::legalizeUDIV_UREM(MachineInstr &MI,
                                             MachineRegisterInfo &MRI,
                                             MachineIRBuilder &B) const {
-  LLT Ty = MRI.getType(MI.getOperand(0).getReg());
-  if (Ty == LLT::scalar(32))
-    return legalizeUDIV_UREM32(MI, MRI, B);
-  if (Ty == LLT::scalar(64))
-    return legalizeUDIV_UREM64(MI, MRI, B);
-  return false;
+  const LLT S64 = LLT::scalar(64);
+  const LLT S32 = LLT::scalar(32);
+  const bool IsDiv = MI.getOpcode() == AMDGPU::G_UDIV;
+  Register DstReg = MI.getOperand(0).getReg();
+  Register Num = MI.getOperand(1).getReg();
+  Register Den = MI.getOperand(2).getReg();
+  LLT Ty = MRI.getType(DstReg);
+
+  if (Ty == S32)
+    legalizeUDIV_UREM32Impl(B, DstReg, Num, Den, IsDiv);
+  else if (Ty == S64)
+    legalizeUDIV_UREM64Impl(B, DstReg, Num, Den, IsDiv);
+  else
+    return false;
+
+  MI.eraseFromParent();
+  return true;
+
 }
 
-bool AMDGPULegalizerInfo::legalizeSDIV_SREM32(MachineInstr &MI,
-                                              MachineRegisterInfo &MRI,
-                                              MachineIRBuilder &B) const {
+bool AMDGPULegalizerInfo::legalizeSDIV_SREM(MachineInstr &MI,
+                                            MachineRegisterInfo &MRI,
+                                            MachineIRBuilder &B) const {
+  const LLT S64 = LLT::scalar(64);
   const LLT S32 = LLT::scalar(32);
 
-  const bool IsRem = MI.getOpcode() == AMDGPU::G_SREM;
   Register DstReg = MI.getOperand(0).getReg();
+  const LLT Ty = MRI.getType(DstReg);
+  if (Ty != S32 && Ty != S64)
+    return false;
+
+  const bool IsDiv = MI.getOpcode() == AMDGPU::G_SDIV;
+
   Register LHS = MI.getOperand(1).getReg();
   Register RHS = MI.getOperand(2).getReg();
 
-  auto ThirtyOne = B.buildConstant(S32, 31);
-  auto LHSign = B.buildAShr(S32, LHS, ThirtyOne);
-  auto RHSign = B.buildAShr(S32, RHS, ThirtyOne);
+  auto SignBitOffset = B.buildConstant(S32, Ty.getSizeInBits() - 1);
+  auto LHSign = B.buildAShr(Ty, LHS, SignBitOffset);
+  auto RHSign = B.buildAShr(Ty, RHS, SignBitOffset);
 
-  LHS = B.buildAdd(S32, LHS, LHSign).getReg(0);
-  RHS = B.buildAdd(S32, RHS, RHSign).getReg(0);
+  LHS = B.buildAdd(Ty, LHS, LHSign).getReg(0);
+  RHS = B.buildAdd(Ty, RHS, RHSign).getReg(0);
 
-  LHS = B.buildXor(S32, LHS, LHSign).getReg(0);
-  RHS = B.buildXor(S32, RHS, RHSign).getReg(0);
+  LHS = B.buildXor(Ty, LHS, LHSign).getReg(0);
+  RHS = B.buildXor(Ty, RHS, RHSign).getReg(0);
 
-  Register UDivRem = MRI.createGenericVirtualRegister(S32);
-  legalizeUDIV_UREM32Impl(B, UDivRem, LHS, RHS, IsRem);
+  Register UDivRem = MRI.createGenericVirtualRegister(Ty);
+  if (Ty == S32)
+    legalizeUDIV_UREM32Impl(B, UDivRem, LHS, RHS, IsDiv);
+  else
+    legalizeUDIV_UREM64Impl(B, UDivRem, LHS, RHS, IsDiv);
 
-  if (IsRem) {
-    auto RSign = LHSign; // Remainder sign is the same as LHS
-    UDivRem = B.buildXor(S32, UDivRem, RSign).getReg(0);
-    B.buildSub(DstReg, UDivRem, RSign);
-  } else {
-    auto DSign = B.buildXor(S32, LHSign, RHSign);
-    UDivRem = B.buildXor(S32, UDivRem, DSign).getReg(0);
-    B.buildSub(DstReg, UDivRem, DSign);
-  }
+  Register Sign;
+  if (IsDiv)
+    Sign = B.buildXor(Ty, LHSign, RHSign).getReg(0);
+  else
+    Sign = LHSign.getReg(0); // Remainder sign is the same as LHS
+
+  UDivRem = B.buildXor(Ty, UDivRem, Sign).getReg(0);
+  B.buildSub(DstReg, UDivRem, Sign);
 
   MI.eraseFromParent();
   return true;
 }
 
-bool AMDGPULegalizerInfo::legalizeSDIV_SREM(MachineInstr &MI,
-                                            MachineRegisterInfo &MRI,
-                                            MachineIRBuilder &B) const {
-  if (MRI.getType(MI.getOperand(0).getReg()) == LLT::scalar(32))
-    return legalizeSDIV_SREM32(MI, MRI, B);
-  return false;
-}
-
 bool AMDGPULegalizerInfo::legalizeFastUnsafeFDIV(MachineInstr &MI,
                                                  MachineRegisterInfo &MRI,
                                                  MachineIRBuilder &B) const {
@@ -3167,8 +3121,9 @@ bool AMDGPULegalizerInfo::legalizeImplicitArgPtr(MachineInstr &MI,
 
   const ArgDescriptor *Arg;
   const TargetRegisterClass *RC;
-  std::tie(Arg, RC)
-    = MFI->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
+  LLT ArgTy;
+  std::tie(Arg, RC, ArgTy) =
+      MFI->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
   if (!Arg)
     return false;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index 35427443ab4c8..ce32bbf76b34f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -61,9 +61,9 @@ class AMDGPULegalizerInfo : public LegalizerInfo {
   bool legalizeSinCos(MachineInstr &MI, MachineRegisterInfo &MRI,
                       MachineIRBuilder &B) const;
 
-  bool buildPCRelGlobalAddress(
-    Register DstReg, LLT PtrTy, MachineIRBuilder &B, const GlobalValue *GV,
-    unsigned Offset, unsigned GAFlags = SIInstrInfo::MO_NONE) const;
+  bool buildPCRelGlobalAddress(Register DstReg, LLT PtrTy, MachineIRBuilder &B,
+                               const GlobalValue *GV, int64_t Offset,
+                               unsigned GAFlags = SIInstrInfo::MO_NONE) const;
 
   bool legalizeGlobalValue(MachineInstr &MI, MachineRegisterInfo &MRI,
                            MachineIRBuilder &B) const;
@@ -111,6 +111,10 @@ class AMDGPULegalizerInfo : public LegalizerInfo {
   bool legalizeSDIV_SREM32(MachineInstr &MI, MachineRegisterInfo &MRI,
                            MachineIRBuilder &B) const;
 
+  void legalizeUDIV_UREM64Impl(MachineIRBuilder &B,
+                               Register DstReg, Register Numer, Register Denom,
+                               bool IsDiv) const;
+
   bool legalizeUDIV_UREM64(MachineInstr &MI, MachineRegisterInfo &MRI,
                            MachineIRBuilder &B) const;
   bool legalizeSDIV_SREM(MachineInstr &MI, MachineRegisterInfo &MRI,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp b/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
index e81a68c86546b..4a14259f1bdb1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
@@ -590,8 +590,8 @@ bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,
   if (!isa<ConstantInt>(PacketSize) || !isa<ConstantInt>(PacketAlign))
     return false;
   unsigned Size = cast<ConstantInt>(PacketSize)->getZExtValue();
-  unsigned Align = cast<ConstantInt>(PacketAlign)->getZExtValue();
-  if (Size != Align || !isPowerOf2_32(Size))
+  Align Alignment = cast<ConstantInt>(PacketAlign)->getAlignValue();
+  if (Alignment != Size)
     return false;
 
   Type *PtrElemTy;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
index 58bd6e5f3b2b7..62ab5bb55a16a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
@@ -109,7 +109,7 @@ bool AMDGPULowerKernelArguments::runOnFunction(Function &F) {
 
   for (Argument &Arg : F.args()) {
     Type *ArgTy = Arg.getType();
-    unsigned ABITypeAlign = DL.getABITypeAlignment(ArgTy);
+    Align ABITypeAlign = DL.getABITypeAlign(ArgTy);
     unsigned Size = DL.getTypeSizeInBits(ArgTy);
     unsigned AllocSize = DL.getTypeAllocSize(ArgTy);
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
index 2c3619b7df21e..64acd6efe0280 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
@@ -38,19 +38,18 @@ AMDGPUMachineFunction::AMDGPUMachineFunction(const MachineFunction &MF) :
 }
 
 unsigned AMDGPUMachineFunction::allocateLDSGlobal(const DataLayout &DL,
-                                                  const GlobalValue &GV) {
+                                                  const GlobalVariable &GV) {
   auto Entry = LocalMemoryObjects.insert(std::make_pair(&GV, 0));
   if (!Entry.second)
     return Entry.first->second;
 
-  unsigned Align = GV.getAlignment();
-  if (Align == 0)
-    Align = DL.getABITypeAlignment(GV.getValueType());
+  Align Alignment =
+      DL.getValueOrABITypeAlignment(GV.getAlign(), GV.getValueType());
 
   /// TODO: We should sort these to minimize wasted space due to alignment
   /// padding. Currently the padding is decided by the first encountered use
   /// during lowering.
-  unsigned Offset = LDSSize = alignTo(LDSSize, Align);
+  unsigned Offset = LDSSize = alignTo(LDSSize, Alignment);
 
   Entry.first->second = Offset;
   LDSSize += DL.getTypeAllocSize(GV.getValueType());
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h
index d5298c62e66d3..c504dd76bc658 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h
@@ -77,7 +77,7 @@ class AMDGPUMachineFunction : public MachineFunctionInfo {
     return WaveLimiter;
   }
 
-  unsigned allocateLDSGlobal(const DataLayout &DL, const GlobalValue &GV);
+  unsigned allocateLDSGlobal(const DataLayout &DL, const GlobalVariable &GV);
 };
 
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
index 29203eb7df222..098b0e9938861 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
@@ -137,9 +137,11 @@ static bool matchUCharToFloat(MachineInstr &MI, MachineRegisterInfo &MRI,
   // about in practice.
   LLT Ty = MRI.getType(DstReg);
   if (Ty == LLT::scalar(32) || Ty == LLT::scalar(16)) {
-    const APInt Mask = APInt::getHighBitsSet(32, 24);
-    return Helper.getKnownBits()->maskedValueIsZero(MI.getOperand(1).getReg(),
-                                                    Mask);
+    Register SrcReg = MI.getOperand(1).getReg();
+    unsigned SrcSize = MRI.getType(SrcReg).getSizeInBits();
+    assert(SrcSize == 16 || SrcSize == 32 || SrcSize == 64);
+    const APInt Mask = APInt::getHighBitsSet(SrcSize, SrcSize - 8);
+    return Helper.getKnownBits()->maskedValueIsZero(SrcReg, Mask);
   }
 
   return false;
@@ -151,14 +153,18 @@ static void applyUCharToFloat(MachineInstr &MI) {
   const LLT S32 = LLT::scalar(32);
 
   Register DstReg = MI.getOperand(0).getReg();
+  Register SrcReg = MI.getOperand(1).getReg();
   LLT Ty = B.getMRI()->getType(DstReg);
+  LLT SrcTy = B.getMRI()->getType(SrcReg);
+  if (SrcTy != S32)
+    SrcReg = B.buildAnyExtOrTrunc(S32, SrcReg).getReg(0);
 
   if (Ty == S32) {
     B.buildInstr(AMDGPU::G_AMDGPU_CVT_F32_UBYTE0, {DstReg},
-                   {MI.getOperand(1)}, MI.getFlags());
+                   {SrcReg}, MI.getFlags());
   } else {
     auto Cvt0 = B.buildInstr(AMDGPU::G_AMDGPU_CVT_F32_UBYTE0, {S32},
-                             {MI.getOperand(1)}, MI.getFlags());
+                             {SrcReg}, MI.getFlags());
     B.buildFPTrunc(DstReg, Cvt0, MI.getFlags());
   }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index f26fb9a7ff6b7..727f71b350490 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -76,6 +76,11 @@ static cl::opt<bool> DisablePromoteAllocaToLDS(
   cl::desc("Disable promote alloca to LDS"),
   cl::init(false));
 
+static cl::opt<unsigned> PromoteAllocaToVectorLimit(
+  "amdgpu-promote-alloca-to-vector-limit",
+  cl::desc("Maximum byte size to consider promote alloca to vector"),
+  cl::init(0));
+
 // FIXME: This can create globals so should be a module pass.
 class AMDGPUPromoteAlloca : public FunctionPass {
 private:
@@ -86,6 +91,7 @@ class AMDGPUPromoteAlloca : public FunctionPass {
   // FIXME: This should be per-kernel.
   uint32_t LocalMemLimit = 0;
   uint32_t CurrentLocalMemUsage = 0;
+  unsigned MaxVGPRs;
 
   bool IsAMDGCN = false;
   bool IsAMDHSA = false;
@@ -129,6 +135,9 @@ class AMDGPUPromoteAlloca : public FunctionPass {
 };
 
 class AMDGPUPromoteAllocaToVector : public FunctionPass {
+private:
+  unsigned MaxVGPRs;
+
 public:
   static char ID;
 
@@ -186,6 +195,13 @@ bool AMDGPUPromoteAlloca::runOnFunction(Function &F) {
   if (!ST.isPromoteAllocaEnabled())
     return false;
 
+  if (IsAMDGCN) {
+    const GCNSubtarget &ST = TM->getSubtarget<GCNSubtarget>(F);
+    MaxVGPRs = ST.getMaxNumVGPRs(ST.getWavesPerEU(F).first);
+  } else {
+    MaxVGPRs = 128;
+  }
+
   bool SufficientLDS = hasSufficientLocalMem(F);
   bool Changed = false;
   BasicBlock &EntryBB = *F.begin();
@@ -409,7 +425,8 @@ static bool canVectorizeInst(Instruction *Inst, User *User,
   }
 }
 
-static bool tryPromoteAllocaToVector(AllocaInst *Alloca, const DataLayout &DL) {
+static bool tryPromoteAllocaToVector(AllocaInst *Alloca, const DataLayout &DL,
+                                     unsigned MaxVGPRs) {
 
   if (DisablePromoteAllocaToVector) {
     LLVM_DEBUG(dbgs() << "  Promotion alloca to vector is disabled\n");
@@ -424,6 +441,16 @@ static bool tryPromoteAllocaToVector(AllocaInst *Alloca, const DataLayout &DL) {
       VectorTy = arrayTypeToVecType(ArrayTy);
   }
 
+  // Use up to 1/4 of available register budget for vectorization.
+  unsigned Limit = PromoteAllocaToVectorLimit ? PromoteAllocaToVectorLimit * 8
+                                              : (MaxVGPRs * 32);
+
+  if (DL.getTypeSizeInBits(AllocaTy) * 4 > Limit) {
+    LLVM_DEBUG(dbgs() << "  Alloca too big for vectorization with "
+                      << MaxVGPRs << " registers available\n");
+    return false;
+  }
+
   LLVM_DEBUG(dbgs() << "Alloca candidate for vectorization\n");
 
   // FIXME: There is no reason why we can't support larger arrays, we
@@ -736,16 +763,15 @@ bool AMDGPUPromoteAlloca::hasSufficientLocalMem(const Function &F) {
         continue;
 
       if (Use->getParent()->getParent() == &F) {
-        unsigned Align = GV.getAlignment();
-        if (Align == 0)
-          Align = DL.getABITypeAlignment(GV.getValueType());
+        Align Alignment =
+            DL.getValueOrABITypeAlignment(GV.getAlign(), GV.getValueType());
 
         // FIXME: Try to account for padding here. The padding is currently
         // determined from the inverse order of uses in the function. I'm not
         // sure if the use list order is in any way connected to this, so the
         // total reported size is likely incorrect.
         uint64_t AllocSize = DL.getTypeAllocSize(GV.getValueType());
-        CurrentLocalMemUsage = alignTo(CurrentLocalMemUsage, Align);
+        CurrentLocalMemUsage = alignTo(CurrentLocalMemUsage, Alignment);
         CurrentLocalMemUsage += AllocSize;
         break;
       }
@@ -807,7 +833,7 @@ bool AMDGPUPromoteAlloca::handleAlloca(AllocaInst &I, bool SufficientLDS) {
 
   LLVM_DEBUG(dbgs() << "Trying to promote " << I << '\n');
 
-  if (tryPromoteAllocaToVector(&I, DL))
+  if (tryPromoteAllocaToVector(&I, DL, MaxVGPRs))
     return true; // Promoted to vector.
 
   if (DisablePromoteAllocaToLDS)
@@ -837,9 +863,8 @@ bool AMDGPUPromoteAlloca::handleAlloca(AllocaInst &I, bool SufficientLDS) {
   const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(*TM, ContainingFunction);
   unsigned WorkGroupSize = ST.getFlatWorkGroupSizes(ContainingFunction).second;
 
-  unsigned Align = I.getAlignment();
-  if (Align == 0)
-    Align = DL.getABITypeAlignment(I.getAllocatedType());
+  Align Alignment =
+      DL.getValueOrABITypeAlignment(I.getAlign(), I.getAllocatedType());
 
   // FIXME: This computed padding is likely wrong since it depends on inverse
   // usage order.
@@ -847,7 +872,7 @@ bool AMDGPUPromoteAlloca::handleAlloca(AllocaInst &I, bool SufficientLDS) {
   // FIXME: It is also possible that if we're allowed to use all of the memory
   // could could end up using more than the maximum due to alignment padding.
 
-  uint32_t NewSize = alignTo(CurrentLocalMemUsage, Align);
+  uint32_t NewSize = alignTo(CurrentLocalMemUsage, Alignment);
   uint32_t AllocSize = WorkGroupSize * DL.getTypeAllocSize(AllocaTy);
   NewSize += AllocSize;
 
@@ -1018,6 +1043,23 @@ bool AMDGPUPromoteAllocaToVector::runOnFunction(Function &F) {
   if (skipFunction(F) || DisablePromoteAllocaToVector)
     return false;
 
+  const TargetMachine *TM;
+  if (auto *TPC = getAnalysisIfAvailable<TargetPassConfig>())
+    TM = &TPC->getTM<TargetMachine>();
+  else
+    return false;
+
+  const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(*TM, F);
+  if (!ST.isPromoteAllocaEnabled())
+    return false;
+
+  if (TM->getTargetTriple().getArch() == Triple::amdgcn) {
+    const GCNSubtarget &ST = TM->getSubtarget<GCNSubtarget>(F);
+    MaxVGPRs = ST.getMaxNumVGPRs(ST.getWavesPerEU(F).first);
+  } else {
+    MaxVGPRs = 128;
+  }
+
   bool Changed = false;
   BasicBlock &EntryBB = *F.begin();
 
@@ -1044,7 +1086,7 @@ bool AMDGPUPromoteAllocaToVector::handleAlloca(AllocaInst &I) {
   LLVM_DEBUG(dbgs() << "Trying to promote " << I << '\n');
 
   Module *Mod = I.getParent()->getParent()->getParent();
-  return tryPromoteAllocaToVector(&I, Mod->getDataLayout());
+  return tryPromoteAllocaToVector(&I, Mod->getDataLayout(), MaxVGPRs);
 }
 
 FunctionPass *llvm::createAMDGPUPromoteAlloca() {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index b8f338df9213d..56bc0c44779d8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -1303,18 +1303,16 @@ static Register getSrcRegIgnoringCopies(const MachineRegisterInfo &MRI,
 // the three offsets (voffset, soffset and instoffset)
 static unsigned setBufferOffsets(MachineIRBuilder &B,
                                  const AMDGPURegisterBankInfo &RBI,
-                                 Register CombinedOffset,
-                                 Register &VOffsetReg,
-                                 Register &SOffsetReg,
-                                 int64_t &InstOffsetVal,
-                                 unsigned Align) {
+                                 Register CombinedOffset, Register &VOffsetReg,
+                                 Register &SOffsetReg, int64_t &InstOffsetVal,
+                                 Align Alignment) {
   const LLT S32 = LLT::scalar(32);
   MachineRegisterInfo *MRI = B.getMRI();
 
   if (Optional<int64_t> Imm = getConstantVRegVal(CombinedOffset, *MRI)) {
     uint32_t SOffset, ImmOffset;
-    if (AMDGPU::splitMUBUFOffset(*Imm, SOffset, ImmOffset,
-                                 &RBI.Subtarget, Align)) {
+    if (AMDGPU::splitMUBUFOffset(*Imm, SOffset, ImmOffset, &RBI.Subtarget,
+                                 Alignment)) {
       VOffsetReg = B.buildConstant(S32, 0).getReg(0);
       SOffsetReg = B.buildConstant(S32, SOffset).getReg(0);
       InstOffsetVal = ImmOffset;
@@ -1334,7 +1332,7 @@ static unsigned setBufferOffsets(MachineIRBuilder &B,
 
   uint32_t SOffset, ImmOffset;
   if (Offset > 0 && AMDGPU::splitMUBUFOffset(Offset, SOffset, ImmOffset,
-                                             &RBI.Subtarget, Align)) {
+                                             &RBI.Subtarget, Alignment)) {
     if (RBI.getRegBank(Base, *MRI, *RBI.TRI) == &AMDGPU::VGPRRegBank) {
       VOffsetReg = Base;
       SOffsetReg = B.buildConstant(S32, SOffset).getReg(0);
@@ -1417,7 +1415,7 @@ bool AMDGPURegisterBankInfo::applyMappingSBufferLoad(
 
   // Use the alignment to ensure that the required offsets will fit into the
   // immediate offsets.
-  const unsigned Alignment = NumLoads > 1 ? 16 * NumLoads : 1;
+  const Align Alignment = NumLoads > 1 ? Align(16 * NumLoads) : Align(1);
 
   MachineIRBuilder B(MI);
   MachineFunction &MF = B.getMF();
@@ -3841,6 +3839,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
   case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
   case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
+  case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT:
+  case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16:
   case AMDGPU::G_AMDGPU_BUFFER_STORE:
   case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE:
   case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT:
@@ -3952,6 +3952,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_log_clamp:
     case Intrinsic::amdgcn_rcp:
     case Intrinsic::amdgcn_rcp_legacy:
+    case Intrinsic::amdgcn_sqrt:
     case Intrinsic::amdgcn_rsq:
     case Intrinsic::amdgcn_rsq_legacy:
     case Intrinsic::amdgcn_rsq_clamp:
@@ -4231,8 +4232,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
       OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
       break;
     }
-    case Intrinsic::amdgcn_end_cf:
-    case Intrinsic::amdgcn_init_exec: {
+    case Intrinsic::amdgcn_end_cf: {
       unsigned Size = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
       OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
       break;
@@ -4288,7 +4288,6 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_init_exec_from_input: {
       unsigned Size = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
       OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
-      OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
       break;
     }
     case Intrinsic::amdgcn_ds_gws_init:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index a30e98172845e..2849645863a57 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -410,10 +410,10 @@ std::pair<unsigned, unsigned> AMDGPUSubtarget::getWavesPerEU(
   // minimum/maximum flat work group sizes.
   unsigned MinImpliedByFlatWorkGroupSize =
     getWavesPerEUForWorkGroup(FlatWorkGroupSizes.second);
+  Default.first = MinImpliedByFlatWorkGroupSize;
   bool RequestedFlatWorkGroupSize = false;
 
   if (F.hasFnAttribute("amdgpu-flat-work-group-size")) {
-    Default.first = MinImpliedByFlatWorkGroupSize;
     RequestedFlatWorkGroupSize = true;
   }
 
@@ -515,7 +515,7 @@ uint64_t AMDGPUSubtarget::getExplicitKernArgSize(const Function &F,
   for (const Argument &Arg : F.args()) {
     Type *ArgTy = Arg.getType();
 
-    const Align Alignment(DL.getABITypeAlignment(ArgTy));
+    const Align Alignment = DL.getABITypeAlign(ArgTy);
     uint64_t AllocSize = DL.getTypeAllocSize(ArgTy);
     ExplicitArgBytes = alignTo(ExplicitArgBytes, Alignment) + AllocSize;
     MaxAlign = std::max(MaxAlign, Alignment);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 5dff9a7fad35c..b4b10835837cd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -787,10 +787,15 @@ void AMDGPUPassConfig::addCodeGenPrepare() {
 
   if (EnableLoadStoreVectorizer)
     addPass(createLoadStoreVectorizerPass());
+
+  // LowerSwitch pass may introduce unreachable blocks that can
+  // cause unexpected behavior for subsequent passes. Placing it
+  // here seems better that these blocks would get cleaned up by
+  // UnreachableBlockElim inserted next in the pass flow.
+  addPass(createLowerSwitchPass());
 }
 
 bool AMDGPUPassConfig::addPreISel() {
-  addPass(createLowerSwitchPass());
   addPass(createFlattenCFGPass());
   return false;
 }
@@ -1052,8 +1057,8 @@ void GCNPassConfig::addPreEmitPass() {
     addPass(&SIInsertHardClausesID);
 
   addPass(&SIRemoveShortExecBranchesID);
-  addPass(&SIPreEmitPeepholeID);
   addPass(&SIInsertSkipsPassID);
+  addPass(&SIPreEmitPeepholeID);
   addPass(&BranchRelaxationPassID);
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index d3bba0fb522bd..542a5f006c0f7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -236,10 +236,14 @@ void AMDGPUTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
   }
 }
 
+void AMDGPUTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                                          TTI::PeelingPreferences &PP) {
+  BaseT::getPeelingPreferences(L, SE, PP);
+}
 unsigned GCNTTIImpl::getHardwareNumberOfRegisters(bool Vec) const {
   // The concept of vector registers doesn't really exist. Some packed vector
   // operations operate on the normal 32-bit registers.
-  return 256;
+  return MaxVGPRs;
 }
 
 unsigned GCNTTIImpl::getNumberOfRegisters(bool Vec) const {
@@ -248,6 +252,13 @@ unsigned GCNTTIImpl::getNumberOfRegisters(bool Vec) const {
   return getHardwareNumberOfRegisters(Vec) >> 3;
 }
 
+unsigned GCNTTIImpl::getNumberOfRegisters(unsigned RCID) const {
+  const SIRegisterInfo *TRI = ST->getRegisterInfo();
+  const TargetRegisterClass *RC = TRI->getRegClass(RCID);
+  unsigned NumVGPRs = (TRI->getRegSizeInBits(*RC) + 31) / 32;
+  return getHardwareNumberOfRegisters(false) / NumVGPRs;
+}
+
 unsigned GCNTTIImpl::getRegisterBitWidth(bool Vector) const {
   return 32;
 }
@@ -293,8 +304,8 @@ unsigned GCNTTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) const {
 }
 
 bool GCNTTIImpl::isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,
-                                               unsigned Alignment,
-                                               unsigned AddrSpace) const {
+                                            Align Alignment,
+                                            unsigned AddrSpace) const {
   // We allow vectorization of flat stores, even though we may need to decompose
   // them later if they may access private memory. We don't have enough context
   // here, and legalization can handle it.
@@ -306,14 +317,14 @@ bool GCNTTIImpl::isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,
 }
 
 bool GCNTTIImpl::isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
-                                                unsigned Alignment,
-                                                unsigned AddrSpace) const {
+                                             Align Alignment,
+                                             unsigned AddrSpace) const {
   return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
 }
 
 bool GCNTTIImpl::isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
-                                                 unsigned Alignment,
-                                                 unsigned AddrSpace) const {
+                                              Align Alignment,
+                                              unsigned AddrSpace) const {
   return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
 }
 
@@ -990,6 +1001,11 @@ void GCNTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
   CommonTTI.getUnrollingPreferences(L, SE, UP);
 }
 
+void GCNTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                                       TTI::PeelingPreferences &PP) {
+  CommonTTI.getPeelingPreferences(L, SE, PP);
+}
+
 unsigned R600TTIImpl::getHardwareNumberOfRegisters(bool Vec) const {
   return 4 * 128; // XXX - 4 channels. Should these count as vector instead?
 }
@@ -1025,7 +1041,7 @@ unsigned R600TTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) const {
 }
 
 bool R600TTIImpl::isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,
-                                             unsigned Alignment,
+                                             Align Alignment,
                                              unsigned AddrSpace) const {
   // We allow vectorization of flat stores, even though we may need to decompose
   // them later if they may access private memory. We don't have enough context
@@ -1034,13 +1050,13 @@ bool R600TTIImpl::isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,
 }
 
 bool R600TTIImpl::isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
-                                              unsigned Alignment,
+                                              Align Alignment,
                                               unsigned AddrSpace) const {
   return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
 }
 
 bool R600TTIImpl::isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
-                                               unsigned Alignment,
+                                               Align Alignment,
                                                unsigned AddrSpace) const {
   return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
 }
@@ -1096,3 +1112,8 @@ void R600TTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                           TTI::UnrollingPreferences &UP) {
   CommonTTI.getUnrollingPreferences(L, SE, UP);
 }
+
+void R600TTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                                        TTI::PeelingPreferences &PP) {
+  CommonTTI.getPeelingPreferences(L, SE, PP);
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 5c1f81bfce17b..3364a9bcaccbb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -61,6 +61,9 @@ class AMDGPUTTIImpl final : public BasicTTIImplBase<AMDGPUTTIImpl> {
 
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP);
+
+  void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                             TTI::PeelingPreferences &PP);
 };
 
 class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
@@ -74,6 +77,7 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
   AMDGPUTTIImpl CommonTTI;
   bool IsGraphicsShader;
   bool HasFP32Denormals;
+  unsigned MaxVGPRs;
 
   const FeatureBitset InlineFeatureIgnoreList = {
     // Codegen control options which don't matter.
@@ -133,7 +137,11 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
       TLI(ST->getTargetLowering()),
       CommonTTI(TM, F),
       IsGraphicsShader(AMDGPU::isShader(F.getCallingConv())),
-      HasFP32Denormals(AMDGPU::SIModeRegisterDefaults(F).allFP32Denormals()) {}
+      HasFP32Denormals(AMDGPU::SIModeRegisterDefaults(F).allFP32Denormals()),
+      MaxVGPRs(ST->getMaxNumVGPRs(
+          std::max(ST->getWavesPerEU(F).first,
+                   ST->getWavesPerEUForWorkGroup(
+                       ST->getFlatWorkGroupSizes(F).second)))) {}
 
   bool hasBranchDivergence() { return true; }
   bool useGPUDivergenceAnalysis() const;
@@ -141,6 +149,9 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP);
 
+  void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                             TTI::PeelingPreferences &PP);
+
   TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) {
     assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
     return TTI::PSK_FastHardware;
@@ -148,6 +159,7 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
 
   unsigned getHardwareNumberOfRegisters(bool Vector) const;
   unsigned getNumberOfRegisters(bool Vector) const;
+  unsigned getNumberOfRegisters(unsigned RCID) const;
   unsigned getRegisterBitWidth(bool Vector) const;
   unsigned getMinVectorRegisterBitWidth() const;
   unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize,
@@ -158,14 +170,11 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
                                 VectorType *VecTy) const;
   unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const;
 
-  bool isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,
-                                  unsigned Alignment,
+  bool isLegalToVectorizeMemChain(unsigned ChainSizeInBytes, Align Alignment,
                                   unsigned AddrSpace) const;
-  bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
-                                   unsigned Alignment,
+  bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, Align Alignment,
                                    unsigned AddrSpace) const;
-  bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
-                                    unsigned Alignment,
+  bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, Align Alignment,
                                     unsigned AddrSpace) const;
   Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length,
                                   unsigned SrcAddrSpace, unsigned DestAddrSpace,
@@ -261,18 +270,18 @@ class R600TTIImpl final : public BasicTTIImplBase<R600TTIImpl> {
 
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP);
+  void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                             TTI::PeelingPreferences &PP);
   unsigned getHardwareNumberOfRegisters(bool Vec) const;
   unsigned getNumberOfRegisters(bool Vec) const;
   unsigned getRegisterBitWidth(bool Vector) const;
   unsigned getMinVectorRegisterBitWidth() const;
   unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const;
-  bool isLegalToVectorizeMemChain(unsigned ChainSizeInBytes, unsigned Alignment,
+  bool isLegalToVectorizeMemChain(unsigned ChainSizeInBytes, Align Alignment,
                                   unsigned AddrSpace) const;
-  bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
-                                   unsigned Alignment,
+  bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, Align Alignment,
                                    unsigned AddrSpace) const;
-  bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
-                                    unsigned Alignment,
+  bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, Align Alignment,
                                     unsigned AddrSpace) const;
   unsigned getMaxInterleaveFactor(unsigned VF);
   unsigned getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp
index ab0d2169e390d..418296684d765 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp
@@ -199,8 +199,7 @@ bool AMDGPUUnifyDivergentExitNodes::runOnFunction(Function &F) {
   // If there's only one exit, we don't need to do anything, unless this is a
   // pixel shader and that exit is an infinite loop, since we still have to
   // insert an export in that case.
-  if (PDT.getRoots().size() <= 1 &&
-      F.getCallingConv() != CallingConv::AMDGPU_PS)
+  if (PDT.root_size() <= 1 && F.getCallingConv() != CallingConv::AMDGPU_PS)
     return false;
 
   LegacyDivergenceAnalysis &DA = getAnalysis<LegacyDivergenceAnalysis>();
@@ -217,7 +216,7 @@ bool AMDGPUUnifyDivergentExitNodes::runOnFunction(Function &F) {
   bool InsertExport = false;
 
   bool Changed = false;
-  for (BasicBlock *BB : PDT.getRoots()) {
+  for (BasicBlock *BB : PDT.roots()) {
     if (isa<ReturnInst>(BB->getTerminator())) {
       if (!isUniformlyReached(DA, *BB))
         ReturningBlocks.push_back(BB);
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 1b566ee5c4f11..013b7a0cf25d1 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -4460,19 +4460,19 @@ bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
   if (Size > LocalMemorySize)
     return Error(SizeLoc, "size is too large");
 
-  int64_t Align = 4;
+  int64_t Alignment = 4;
   if (getLexer().is(AsmToken::Comma)) {
     Lex();
     SMLoc AlignLoc = getLexer().getLoc();
-    if (getParser().parseAbsoluteExpression(Align))
+    if (getParser().parseAbsoluteExpression(Alignment))
       return true;
-    if (Align < 0 || !isPowerOf2_64(Align))
+    if (Alignment < 0 || !isPowerOf2_64(Alignment))
       return Error(AlignLoc, "alignment must be a power of two");
 
     // Alignment larger than the size of LDS is possible in theory, as long
     // as the linker manages to place to symbol at address 0, but we do want
     // to make sure the alignment fits nicely into a 32-bit integer.
-    if (Align >= 1u << 31)
+    if (Alignment >= 1u << 31)
       return Error(AlignLoc, "alignment is too large");
   }
 
@@ -4484,7 +4484,7 @@ bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
   if (!Symbol->isUndefined())
     return Error(NameLoc, "invalid symbol redefinition");
 
-  getTargetStreamer().emitAMDGPULDS(Symbol, Size, Align);
+  getTargetStreamer().emitAMDGPULDS(Symbol, Size, Align(Alignment));
   return false;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 4bc9fd04b3de4..fa42ddc54b565 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -1799,12 +1799,14 @@ defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZ
 
 let SubtargetPredicate = HasUnpackedD16VMem in {
   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
+  defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32,   "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">;
   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
 } // End HasUnpackedD16VMem.
 
 let SubtargetPredicate = HasPackedD16VMem in {
   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X">;
+  defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32,   "TBUFFER_STORE_FORMAT_D16_X">;
   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">;
   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">;
 } // End HasPackedD16VMem.
diff --git a/llvm/lib/Target/AMDGPU/CaymanInstructions.td b/llvm/lib/Target/AMDGPU/CaymanInstructions.td
index e2978624811d1..f4ddbf1131c34 100644
--- a/llvm/lib/Target/AMDGPU/CaymanInstructions.td
+++ b/llvm/lib/Target/AMDGPU/CaymanInstructions.td
@@ -57,11 +57,12 @@ def : POW_Common <LOG_IEEE_cm, EXP_IEEE_cm, MUL>;
 defm DIV_cm : DIV_Common<RECIP_IEEE_cm>;
 
 // RECIP_UINT emulation for Cayman
-// The multiplication scales from [0,1] to the unsigned integer range
+// The multiplication scales from [0,1) to the unsigned integer range,
+// rounding down a bit to avoid unwanted overflow.
 def : R600Pat <
   (AMDGPUurecip i32:$src0),
   (FLT_TO_UINT_eg (MUL_IEEE (RECIP_IEEE_cm (UINT_TO_FLT_eg $src0)),
-                            (MOV_IMM_I32 CONST.FP_UINT_MAX_PLUS_1)))
+                            (MOV_IMM_I32 CONST.FP_4294966784)))
 >;
 
 def CF_END_CM : CF_CLAUSE_EG<32, (ins), "CF_END"> {
diff --git a/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp b/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp
index 1fa75504493b0..719a968b83147 100644
--- a/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp
@@ -105,6 +105,11 @@ class GCNDPPCombine : public MachineFunctionPass {
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 
+  MachineFunctionProperties getRequiredProperties() const override {
+    return MachineFunctionProperties()
+      .set(MachineFunctionProperties::Property::IsSSA);
+  }
+
 private:
   int getDPPOp(unsigned Op) const;
 };
@@ -508,15 +513,32 @@ bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
       break;
     }
 
+    auto *Src0 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src0);
+    auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1);
+    if (Use != Src0 && !(Use == Src1 && OrigMI.isCommutable())) { // [1]
+      LLVM_DEBUG(dbgs() << "  failed: no suitable operands\n");
+      break;
+    }
+
+    assert(Src0 && "Src1 without Src0?");
+    if (Src1 && Src1->isIdenticalTo(*Src0)) {
+      assert(Src1->isReg());
+      LLVM_DEBUG(
+          dbgs()
+          << "  " << OrigMI
+          << "  failed: DPP register is used more than once per instruction\n");
+      break;
+    }
+
     LLVM_DEBUG(dbgs() << "  combining: " << OrigMI);
-    if (Use == TII->getNamedOperand(OrigMI, AMDGPU::OpName::src0)) {
+    if (Use == Src0) {
       if (auto *DPPInst = createDPPInst(OrigMI, MovMI, CombOldVGPR,
                                         OldOpndValue, CombBCZ)) {
         DPPMIs.push_back(DPPInst);
         Rollback = false;
       }
-    } else if (OrigMI.isCommutable() &&
-               Use == TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1)) {
+    } else {
+      assert(Use == Src1 && OrigMI.isCommutable()); // by check [1]
       auto *BB = OrigMI.getParent();
       auto *NewMI = BB->getParent()->CloneMachineInstr(&OrigMI);
       BB->insert(OrigMI, NewMI);
@@ -530,8 +552,7 @@ bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
       } else
         LLVM_DEBUG(dbgs() << "  failed: cannot be commuted\n");
       NewMI->eraseFromParent();
-    } else
-      LLVM_DEBUG(dbgs() << "  failed: no suitable operands\n");
+    }
     if (Rollback)
       break;
     OrigMIs.push_back(&OrigMI);
@@ -564,8 +585,6 @@ bool GCNDPPCombine::runOnMachineFunction(MachineFunction &MF) {
   MRI = &MF.getRegInfo();
   TII = ST.getInstrInfo();
 
-  assert(MRI->isSSA() && "Must be run on SSA");
-
   bool Changed = false;
   for (auto &MBB : MF) {
     for (auto I = MBB.rbegin(), E = MBB.rend(); I != E;) {
diff --git a/llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp b/llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp
index e4017be41e727..98d971630ca4f 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp
@@ -170,11 +170,13 @@ class GCNRegBankReassign : public MachineFunctionPass {
   // If Bank is not -1 assume Reg:SubReg to belong to that Bank.
   uint32_t getRegBankMask(unsigned Reg, unsigned SubReg, int Bank);
 
-  // Return number of stalls in the instructions.
-  // UsedBanks has bits set for the banks used by all operands.
-  // If Reg and Bank provided substitute the Reg with the Bank.
-  unsigned analyzeInst(const MachineInstr& MI, unsigned& UsedBanks,
-                       unsigned Reg = AMDGPU::NoRegister, int Bank = -1);
+  // Analyze one instruction returning the number of stalls and a mask of the
+  // banks used by all operands.
+  // If Reg and Bank are provided, assume all uses of Reg will be replaced with
+  // a register chosen from Bank.
+  std::pair<unsigned, unsigned> analyzeInst(const MachineInstr &MI,
+                                            unsigned Reg = AMDGPU::NoRegister,
+                                            int Bank = -1);
 
   // Return true if register is regular VGPR or SGPR or their tuples.
   // Returns false for special registers like m0, vcc etc.
@@ -356,15 +358,14 @@ uint32_t GCNRegBankReassign::getRegBankMask(unsigned Reg, unsigned SubReg,
   return Mask << SGPR_BANK_OFFSET;
 }
 
-unsigned GCNRegBankReassign::analyzeInst(const MachineInstr& MI,
-                                         unsigned& UsedBanks,
-                                         unsigned Reg,
-                                         int Bank) {
+std::pair<unsigned, unsigned>
+GCNRegBankReassign::analyzeInst(const MachineInstr &MI, unsigned Reg,
+                                int Bank) {
   unsigned StallCycles = 0;
-  UsedBanks = 0;
+  unsigned UsedBanks = 0;
 
   if (MI.isDebugValue())
-    return 0;
+    return std::make_pair(StallCycles, UsedBanks);
 
   RegsUsed.reset();
   OperandMasks.clear();
@@ -404,7 +405,7 @@ unsigned GCNRegBankReassign::analyzeInst(const MachineInstr& MI,
     OperandMasks.push_back(OperandMask(Op.getReg(), Op.getSubReg(), Mask));
   }
 
-  return StallCycles;
+  return std::make_pair(StallCycles, UsedBanks);
 }
 
 unsigned GCNRegBankReassign::getOperandGatherWeight(const MachineInstr& MI,
@@ -588,7 +589,6 @@ unsigned GCNRegBankReassign::computeStallCycles(unsigned SrcReg,
                                                 unsigned Reg, int Bank,
                                                 bool Collect) {
   unsigned TotalStallCycles = 0;
-  unsigned UsedBanks = 0;
   SmallSet<const MachineInstr *, 16> Visited;
 
   for (auto &MI : MRI->use_nodbg_instructions(SrcReg)) {
@@ -596,7 +596,9 @@ unsigned GCNRegBankReassign::computeStallCycles(unsigned SrcReg,
       continue;
     if (!Visited.insert(&MI).second)
       continue;
-    unsigned StallCycles = analyzeInst(MI, UsedBanks, Reg, Bank);
+    unsigned StallCycles;
+    unsigned UsedBanks;
+    std::tie(StallCycles, UsedBanks) = analyzeInst(MI, Reg, Bank);
     TotalStallCycles += StallCycles;
     if (Collect)
       collectCandidates(MI, UsedBanks, StallCycles);
@@ -717,8 +719,9 @@ unsigned GCNRegBankReassign::collectCandidates(MachineFunction &MF,
       if (MI.isBundle())
           continue; // we analyze the instructions inside the bundle individually
 
-      unsigned UsedBanks = 0;
-      unsigned StallCycles = analyzeInst(MI, UsedBanks);
+      unsigned StallCycles;
+      unsigned UsedBanks;
+      std::tie(StallCycles, UsedBanks) = analyzeInst(MI);
 
       if (Collect)
         collectCandidates(MI, UsedBanks, StallCycles);
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
index f3a6b77938b48..3d202d7960d65 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -212,9 +212,9 @@ void AMDGPUTargetAsmStreamer::EmitAMDGPUSymbolType(StringRef SymbolName,
 }
 
 void AMDGPUTargetAsmStreamer::emitAMDGPULDS(MCSymbol *Symbol, unsigned Size,
-                                            unsigned Align) {
-  OS << "\t.amdgpu_lds " << Symbol->getName() << ", " << Size << ", " << Align
-     << '\n';
+                                            Align Alignment) {
+  OS << "\t.amdgpu_lds " << Symbol->getName() << ", " << Size << ", "
+     << Alignment.value() << '\n';
 }
 
 bool AMDGPUTargetAsmStreamer::EmitISAVersion(StringRef IsaVersionString) {
@@ -515,9 +515,7 @@ void AMDGPUTargetELFStreamer::EmitAMDGPUSymbolType(StringRef SymbolName,
 }
 
 void AMDGPUTargetELFStreamer::emitAMDGPULDS(MCSymbol *Symbol, unsigned Size,
-                                            unsigned Align) {
-  assert(isPowerOf2_32(Align));
-
+                                            Align Alignment) {
   MCSymbolELF *SymbolELF = cast<MCSymbolELF>(Symbol);
   SymbolELF->setType(ELF::STT_OBJECT);
 
@@ -526,7 +524,7 @@ void AMDGPUTargetELFStreamer::emitAMDGPULDS(MCSymbol *Symbol, unsigned Size,
     SymbolELF->setExternal(true);
   }
 
-  if (SymbolELF->declareCommon(Size, Align, true)) {
+  if (SymbolELF->declareCommon(Size, Alignment.value(), true)) {
     report_fatal_error("Symbol: " + Symbol->getName() +
                        " redeclared as different type");
   }
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
index 5f89da365de05..a19d4646deb26 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
@@ -54,7 +54,7 @@ class AMDGPUTargetStreamer : public MCTargetStreamer {
   virtual void EmitAMDGPUSymbolType(StringRef SymbolName, unsigned Type) = 0;
 
   virtual void emitAMDGPULDS(MCSymbol *Symbol, unsigned Size,
-                             unsigned Align) = 0;
+                             Align Alignment) = 0;
 
   /// \returns True on success, false on failure.
   virtual bool EmitISAVersion(StringRef IsaVersionString) = 0;
@@ -110,7 +110,7 @@ class AMDGPUTargetAsmStreamer final : public AMDGPUTargetStreamer {
 
   void EmitAMDGPUSymbolType(StringRef SymbolName, unsigned Type) override;
 
-  void emitAMDGPULDS(MCSymbol *Sym, unsigned Size, unsigned Align) override;
+  void emitAMDGPULDS(MCSymbol *Sym, unsigned Size, Align Alignment) override;
 
   /// \returns True on success, false on failure.
   bool EmitISAVersion(StringRef IsaVersionString) override;
@@ -158,7 +158,7 @@ class AMDGPUTargetELFStreamer final : public AMDGPUTargetStreamer {
 
   void EmitAMDGPUSymbolType(StringRef SymbolName, unsigned Type) override;
 
-  void emitAMDGPULDS(MCSymbol *Sym, unsigned Size, unsigned Align) override;
+  void emitAMDGPULDS(MCSymbol *Sym, unsigned Size, Align Alignment) override;
 
   /// \returns True on success, false on failure.
   bool EmitISAVersion(StringRef IsaVersionString) override;
diff --git a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
index e0c63bc5e8e18..dc2e73e1f94e0 100644
--- a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
@@ -1265,10 +1265,11 @@ SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
     return scalarizeVectorStore(StoreNode, DAG);
   }
 
-  unsigned Align = StoreNode->getAlignment();
-  if (Align < MemVT.getStoreSize() &&
-      !allowsMisalignedMemoryAccesses(
-          MemVT, AS, Align, StoreNode->getMemOperand()->getFlags(), nullptr)) {
+  Align Alignment = StoreNode->getAlign();
+  if (Alignment < MemVT.getStoreSize() &&
+      !allowsMisalignedMemoryAccesses(MemVT, AS, Alignment.value(),
+                                      StoreNode->getMemOperand()->getFlags(),
+                                      nullptr)) {
     return expandUnalignedStore(StoreNode, DAG);
   }
 
diff --git a/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp b/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
index cec7f563f4800..b0620663a2300 100644
--- a/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
+++ b/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
@@ -56,9 +56,8 @@ using namespace llvm;
 
 #define DEBUG_TYPE "vec-merger"
 
-static bool isImplicitlyDef(MachineRegisterInfo &MRI, unsigned Reg) {
-  assert(MRI.isSSA());
-  if (Register::isPhysicalRegister(Reg))
+static bool isImplicitlyDef(MachineRegisterInfo &MRI, Register Reg) {
+  if (Reg.isPhysical())
     return false;
   const MachineInstr *MI = MRI.getUniqueVRegDef(Reg);
   return MI && MI->isImplicitDef();
@@ -69,8 +68,8 @@ namespace {
 class RegSeqInfo {
 public:
   MachineInstr *Instr;
-  DenseMap<unsigned, unsigned> RegToChan;
-  std::vector<unsigned> UndefReg;
+  DenseMap<Register, unsigned> RegToChan;
+  std::vector<Register> UndefReg;
 
   RegSeqInfo(MachineRegisterInfo &MRI, MachineInstr *MI) : Instr(MI) {
     assert(MI->getOpcode() == R600::REG_SEQUENCE);
@@ -102,7 +101,7 @@ class R600VectorRegMerger : public MachineFunctionPass {
   InstructionSetMap PreviousRegSeqByUndefCount;
 
   bool canSwizzle(const MachineInstr &MI) const;
-  bool areAllUsesSwizzeable(unsigned Reg) const;
+  bool areAllUsesSwizzeable(Register Reg) const;
   void SwizzleInput(MachineInstr &,
       const std::vector<std::pair<unsigned, unsigned>> &RemapChan) const;
   bool tryMergeVector(const RegSeqInfo *Untouched, RegSeqInfo *ToMerge,
@@ -130,6 +129,11 @@ class R600VectorRegMerger : public MachineFunctionPass {
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 
+  MachineFunctionProperties getRequiredProperties() const override {
+    return MachineFunctionProperties()
+      .set(MachineFunctionProperties::Property::IsSSA);
+  }
+
   StringRef getPassName() const override {
     return "R600 Vector Registers Merge Pass";
   }
@@ -165,9 +169,9 @@ bool R600VectorRegMerger::tryMergeVector(const RegSeqInfo *Untouched,
     RegSeqInfo *ToMerge, std::vector< std::pair<unsigned, unsigned>> &Remap)
     const {
   unsigned CurrentUndexIdx = 0;
-  for (DenseMap<unsigned, unsigned>::iterator It = ToMerge->RegToChan.begin(),
+  for (DenseMap<Register, unsigned>::iterator It = ToMerge->RegToChan.begin(),
       E = ToMerge->RegToChan.end(); It != E; ++It) {
-    DenseMap<unsigned, unsigned>::const_iterator PosInUntouched =
+    DenseMap<Register, unsigned>::const_iterator PosInUntouched =
         Untouched->RegToChan.find((*It).first);
     if (PosInUntouched != Untouched->RegToChan.end()) {
       Remap.push_back(std::pair<unsigned, unsigned>
@@ -203,9 +207,9 @@ MachineInstr *R600VectorRegMerger::RebuildVector(
   DebugLoc DL = Pos->getDebugLoc();
 
   Register SrcVec = BaseRSI->Instr->getOperand(0).getReg();
-  DenseMap<unsigned, unsigned> UpdatedRegToChan = BaseRSI->RegToChan;
-  std::vector<unsigned> UpdatedUndef = BaseRSI->UndefReg;
-  for (DenseMap<unsigned, unsigned>::iterator It = RSI->RegToChan.begin(),
+  DenseMap<Register, unsigned> UpdatedRegToChan = BaseRSI->RegToChan;
+  std::vector<Register> UpdatedUndef = BaseRSI->UndefReg;
+  for (DenseMap<Register, unsigned>::iterator It = RSI->RegToChan.begin(),
       E = RSI->RegToChan.end(); It != E; ++It) {
     Register DstReg = MRI->createVirtualRegister(&R600::R600_Reg128RegClass);
     unsigned SubReg = (*It).first;
@@ -218,7 +222,7 @@ MachineInstr *R600VectorRegMerger::RebuildVector(
         .addReg(SubReg)
         .addImm(Chan);
     UpdatedRegToChan[SubReg] = Chan;
-    std::vector<unsigned>::iterator ChanPos = llvm::find(UpdatedUndef, Chan);
+    std::vector<Register>::iterator ChanPos = llvm::find(UpdatedUndef, Chan);
     if (ChanPos != UpdatedUndef.end())
       UpdatedUndef.erase(ChanPos);
     assert(!is_contained(UpdatedUndef, Chan) &&
@@ -279,7 +283,7 @@ void R600VectorRegMerger::SwizzleInput(MachineInstr &MI,
   }
 }
 
-bool R600VectorRegMerger::areAllUsesSwizzeable(unsigned Reg) const {
+bool R600VectorRegMerger::areAllUsesSwizzeable(Register Reg) const {
   for (MachineRegisterInfo::use_instr_iterator It = MRI->use_instr_begin(Reg),
       E = MRI->use_instr_end(); It != E; ++It) {
     if (!canSwizzle(*It))
@@ -322,7 +326,7 @@ bool R600VectorRegMerger::tryMergeUsingFreeSlot(RegSeqInfo &RSI,
 }
 
 void R600VectorRegMerger::trackRSI(const RegSeqInfo &RSI) {
-  for (DenseMap<unsigned, unsigned>::const_iterator
+  for (DenseMap<Register, unsigned>::const_iterator
   It = RSI.RegToChan.begin(), E = RSI.RegToChan.end(); It != E; ++It) {
     PreviousRegSeqByReg[(*It).first].push_back(RSI.Instr);
   }
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 472d3e440fe05..ffcf4c30bc70d 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -282,6 +282,9 @@ static bool updateOperand(FoldCandidate &Fold,
   assert(!Fold.needsShrink() && "not handled");
 
   if (Fold.isImm()) {
+    // FIXME: ChangeToImmediate should probably clear the subreg flags. It's
+    // reinterpreted as TargetFlags.
+    Old.setSubReg(0);
     Old.ChangeToImmediate(Fold.ImmToFold);
     return true;
   }
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index f3aa764666a63..a2e802009d098 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -85,7 +85,7 @@ static void getVGPRSpillLaneOrTempRegister(MachineFunction &MF,
   // 1: If there is already a VGPR with free lanes, use it. We
   // may already have to pay the penalty for spilling a CSR VGPR.
   if (MFI->haveFreeLanesForSGPRSpill(MF, 1)) {
-    int NewFI = FrameInfo.CreateStackObject(4, 4, true, nullptr,
+    int NewFI = FrameInfo.CreateStackObject(4, Align(4), true, nullptr,
                                             TargetStackID::SGPRSpill);
 
     if (!MFI->allocateSGPRSpillToVGPR(MF, NewFI))
@@ -105,7 +105,7 @@ static void getVGPRSpillLaneOrTempRegister(MachineFunction &MF,
       MF.getRegInfo(), LiveRegs, AMDGPU::SReg_32_XM0_XEXECRegClass, true);
 
   if (!TempSGPR) {
-    int NewFI = FrameInfo.CreateStackObject(4, 4, true, nullptr,
+    int NewFI = FrameInfo.CreateStackObject(4, Align(4), true, nullptr,
                                             TargetStackID::SGPRSpill);
 
     if (MFI->allocateSGPRSpillToVGPR(MF, NewFI)) {
@@ -1119,9 +1119,8 @@ void SIFrameLowering::processFunctionBeforeFrameFinalized(
       RS->addScavengingFrameIndex(ScavengeFI);
     } else {
       int ScavengeFI = MFI.CreateStackObject(
-        TRI->getSpillSize(AMDGPU::SGPR_32RegClass),
-        TRI->getSpillAlignment(AMDGPU::SGPR_32RegClass),
-        false);
+          TRI->getSpillSize(AMDGPU::SGPR_32RegClass),
+          TRI->getSpillAlign(AMDGPU::SGPR_32RegClass), false);
       RS->addScavengingFrameIndex(ScavengeFI);
     }
   }
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index e7cc5afdef9be..d035aa8f72bd7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1527,9 +1527,10 @@ SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
 
   const ArgDescriptor *InputPtrReg;
   const TargetRegisterClass *RC;
+  LLT ArgTy;
 
-  std::tie(InputPtrReg, RC)
-    = Info->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
+  std::tie(InputPtrReg, RC, ArgTy) =
+      Info->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
 
   MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
   MVT PtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS);
@@ -1578,16 +1579,15 @@ SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
 }
 
 SDValue SITargetLowering::lowerKernargMemParameter(
-  SelectionDAG &DAG, EVT VT, EVT MemVT,
-  const SDLoc &SL, SDValue Chain,
-  uint64_t Offset, unsigned Align, bool Signed,
-  const ISD::InputArg *Arg) const {
+    SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain,
+    uint64_t Offset, Align Alignment, bool Signed,
+    const ISD::InputArg *Arg) const {
   MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
 
   // Try to avoid using an extload by loading earlier than the argument address,
   // and extracting the relevant bits. The load should hopefully be merged with
   // the previous argument.
-  if (MemVT.getStoreSize() < 4 && Align < 4) {
+  if (MemVT.getStoreSize() < 4 && Alignment < 4) {
     // TODO: Handle align < 4 and size >= 4 (can happen with packed structs).
     int64_t AlignDownOffset = alignDown(Offset, 4);
     int64_t OffsetDiff = Offset - AlignDownOffset;
@@ -1613,9 +1613,9 @@ SDValue SITargetLowering::lowerKernargMemParameter(
   }
 
   SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
-  SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Align,
+  SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Alignment,
                              MachineMemOperand::MODereferenceable |
-                             MachineMemOperand::MOInvariant);
+                                 MachineMemOperand::MOInvariant);
 
   SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg);
   return DAG.getMergeValues({ Val, Load.getValue(1) }, SL);
@@ -1676,8 +1676,9 @@ SDValue SITargetLowering::getPreloadedValue(SelectionDAG &DAG,
   AMDGPUFunctionArgInfo::PreloadedValue PVID) const {
   const ArgDescriptor *Reg;
   const TargetRegisterClass *RC;
+  LLT Ty;
 
-  std::tie(Reg, RC) = MFI.getPreloadedValue(PVID);
+  std::tie(Reg, RC, Ty) = MFI.getPreloadedValue(PVID);
   return CreateLiveInRegister(DAG, RC, Reg->getRegister(), VT);
 }
 
@@ -2233,9 +2234,9 @@ SDValue SITargetLowering::LowerFormalArguments(
   //
   // FIXME: Alignment of explicit arguments totally broken with non-0 explicit
   // kern arg offset.
-  const unsigned KernelArgBaseAlign = 16;
+  const Align KernelArgBaseAlign = Align(16);
 
-   for (unsigned i = 0, e = Ins.size(), ArgIdx = 0; i != e; ++i) {
+  for (unsigned i = 0, e = Ins.size(), ArgIdx = 0; i != e; ++i) {
     const ISD::InputArg &Arg = Ins[i];
     if (Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) {
       InVals.push_back(DAG.getUNDEF(Arg.VT));
@@ -2250,10 +2251,11 @@ SDValue SITargetLowering::LowerFormalArguments(
       EVT MemVT = VA.getLocVT();
 
       const uint64_t Offset = VA.getLocMemOffset();
-      unsigned Align = MinAlign(KernelArgBaseAlign, Offset);
+      Align Alignment = commonAlignment(KernelArgBaseAlign, Offset);
 
-      SDValue Arg = lowerKernargMemParameter(
-        DAG, VT, MemVT, DL, Chain, Offset, Align, Ins[i].Flags.isSExt(), &Ins[i]);
+      SDValue Arg =
+          lowerKernargMemParameter(DAG, VT, MemVT, DL, Chain, Offset, Alignment,
+                                   Ins[i].Flags.isSExt(), &Ins[i]);
       Chains.push_back(Arg.getValue(1));
 
       auto *ParamTy =
@@ -2580,15 +2582,18 @@ void SITargetLowering::passSpecialInputs(
   for (auto InputID : InputRegs) {
     const ArgDescriptor *OutgoingArg;
     const TargetRegisterClass *ArgRC;
+    LLT ArgTy;
 
-    std::tie(OutgoingArg, ArgRC) = CalleeArgInfo->getPreloadedValue(InputID);
+    std::tie(OutgoingArg, ArgRC, ArgTy) =
+        CalleeArgInfo->getPreloadedValue(InputID);
     if (!OutgoingArg)
       continue;
 
     const ArgDescriptor *IncomingArg;
     const TargetRegisterClass *IncomingArgRC;
-    std::tie(IncomingArg, IncomingArgRC)
-      = CallerArgInfo.getPreloadedValue(InputID);
+    LLT Ty;
+    std::tie(IncomingArg, IncomingArgRC, Ty) =
+        CallerArgInfo.getPreloadedValue(InputID);
     assert(IncomingArgRC == ArgRC);
 
     // All special arguments are ints for now.
@@ -2621,24 +2626,25 @@ void SITargetLowering::passSpecialInputs(
   // packed.
   const ArgDescriptor *OutgoingArg;
   const TargetRegisterClass *ArgRC;
+  LLT Ty;
 
-  std::tie(OutgoingArg, ArgRC) =
-    CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);
+  std::tie(OutgoingArg, ArgRC, Ty) =
+      CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);
   if (!OutgoingArg)
-    std::tie(OutgoingArg, ArgRC) =
-      CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
+    std::tie(OutgoingArg, ArgRC, Ty) =
+        CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
   if (!OutgoingArg)
-    std::tie(OutgoingArg, ArgRC) =
-      CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
+    std::tie(OutgoingArg, ArgRC, Ty) =
+        CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
   if (!OutgoingArg)
     return;
 
-  const ArgDescriptor *IncomingArgX
-    = CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X).first;
-  const ArgDescriptor *IncomingArgY
-    = CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y).first;
-  const ArgDescriptor *IncomingArgZ
-    = CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z).first;
+  const ArgDescriptor *IncomingArgX = std::get<0>(
+      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X));
+  const ArgDescriptor *IncomingArgY = std::get<0>(
+      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y));
+  const ArgDescriptor *IncomingArgZ = std::get<0>(
+      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z));
 
   SDValue InputReg;
   SDLoc SL;
@@ -3127,7 +3133,7 @@ SDValue SITargetLowering::lowerDYNAMIC_STACKALLOCImpl(
   SDValue Size  = Tmp2.getOperand(1);
   SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
   Chain = SP.getValue(1);
-  unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
+  MaybeAlign Alignment = cast<ConstantSDNode>(Tmp3)->getMaybeAlignValue();
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   const TargetFrameLowering *TFL = ST.getFrameLowering();
   unsigned Opc =
@@ -3138,12 +3144,13 @@ SDValue SITargetLowering::lowerDYNAMIC_STACKALLOCImpl(
       ISD::SHL, dl, VT, Size,
       DAG.getConstant(ST.getWavefrontSizeLog2(), dl, MVT::i32));
 
-  unsigned StackAlign = TFL->getStackAlignment();
+  Align StackAlign = TFL->getStackAlign();
   Tmp1 = DAG.getNode(Opc, dl, VT, SP, ScaledSize); // Value
-  if (Align > StackAlign) {
-    Tmp1 = DAG.getNode(
-      ISD::AND, dl, VT, Tmp1,
-      DAG.getConstant(-(uint64_t)Align << ST.getWavefrontSizeLog2(), dl, VT));
+  if (Alignment && *Alignment > StackAlign) {
+    Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
+                       DAG.getConstant(-(uint64_t)Alignment->value()
+                                           << ST.getWavefrontSizeLog2(),
+                                       dl, VT));
   }
 
   Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);    // Output chain
@@ -3879,6 +3886,7 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter(
     MachineBasicBlock::iterator MII = MI;
     const DebugLoc &DL = MI.getDebugLoc();
     MachineOperand &Dest = MI.getOperand(0);
+    MachineOperand &CarryDest = MI.getOperand(1);
     MachineOperand &Src0 = MI.getOperand(2);
     MachineOperand &Src1 = MI.getOperand(3);
     MachineOperand &Src2 = MI.getOperand(4);
@@ -3915,6 +3923,9 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter(
     }
 
     BuildMI(*BB, MII, DL, TII->get(Opc), Dest.getReg()).add(Src0).add(Src1);
+
+    BuildMI(*BB, MII, DL, TII->get(AMDGPU::COPY), CarryDest.getReg())
+      .addReg(AMDGPU::SCC);
     MI.eraseFromParent();
     return BB;
   }
@@ -4266,10 +4277,13 @@ bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
 
   switch (VT.getSimpleVT().SimpleTy) {
   case MVT::f32: {
-    // This is as fast on some subtargets. However, we always have full rate f32
-    // mad available which returns the same result as the separate operations
-    // which we should prefer over fma. We can't use this if we want to support
-    // denormals, so only report this in these cases.
+    // If mad is not available this depends only on if f32 fma is full rate.
+    if (!Subtarget->hasMadMacF32Insts())
+      return Subtarget->hasFastFMAF32();
+
+    // Otherwise f32 mad is always full rate and returns the same result as
+    // the separate operations so should be preferred over fma.
+    // However does not support denomals.
     if (hasFP32Denormals(MF))
       return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts();
 
@@ -4552,9 +4566,8 @@ static SDValue lowerICMPIntrinsic(const SITargetLowering &TLI,
                                   SDNode *N, SelectionDAG &DAG) {
   EVT VT = N->getValueType(0);
   const auto *CD = cast<ConstantSDNode>(N->getOperand(3));
-  int CondCode = CD->getSExtValue();
-  if (CondCode < ICmpInst::Predicate::FIRST_ICMP_PREDICATE ||
-      CondCode > ICmpInst::Predicate::LAST_ICMP_PREDICATE)
+  unsigned CondCode = CD->getZExtValue();
+  if (!ICmpInst::isIntPredicate(static_cast<ICmpInst::Predicate>(CondCode)))
     return DAG.getUNDEF(VT);
 
   ICmpInst::Predicate IcInput = static_cast<ICmpInst::Predicate>(CondCode);
@@ -4589,11 +4602,9 @@ static SDValue lowerFCMPIntrinsic(const SITargetLowering &TLI,
   EVT VT = N->getValueType(0);
   const auto *CD = cast<ConstantSDNode>(N->getOperand(3));
 
-  int CondCode = CD->getSExtValue();
-  if (CondCode < FCmpInst::Predicate::FIRST_FCMP_PREDICATE ||
-      CondCode > FCmpInst::Predicate::LAST_FCMP_PREDICATE) {
+  unsigned CondCode = CD->getZExtValue();
+  if (!FCmpInst::isFPPredicate(static_cast<FCmpInst::Predicate>(CondCode)))
     return DAG.getUNDEF(VT);
-  }
 
   SDValue Src0 = N->getOperand(1);
   SDValue Src1 = N->getOperand(2);
@@ -5461,8 +5472,9 @@ SITargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
 
 static SDValue
 buildPCRelGlobalAddress(SelectionDAG &DAG, const GlobalValue *GV,
-                        const SDLoc &DL, unsigned Offset, EVT PtrVT,
+                        const SDLoc &DL, int64_t Offset, EVT PtrVT,
                         unsigned GAFlags = SIInstrInfo::MO_NONE) {
+  assert(isInt<32>(Offset + 4) && "32-bit offset is expected!");
   // In order to support pc-relative addressing, the PC_ADD_REL_OFFSET SDNode is
   // lowered to the following code sequence:
   //
@@ -5537,11 +5549,11 @@ SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunction *MFI,
   Type *Ty = PtrVT.getTypeForEVT(*DAG.getContext());
   PointerType *PtrTy = PointerType::get(Ty, AMDGPUAS::CONSTANT_ADDRESS);
   const DataLayout &DataLayout = DAG.getDataLayout();
-  unsigned Align = DataLayout.getABITypeAlignment(PtrTy);
+  Align Alignment = DataLayout.getABITypeAlign(PtrTy);
   MachinePointerInfo PtrInfo
     = MachinePointerInfo::getGOT(DAG.getMachineFunction());
 
-  return DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), GOTAddr, PtrInfo, Align,
+  return DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), GOTAddr, PtrInfo, Alignment,
                      MachineMemOperand::MODereferenceable |
                          MachineMemOperand::MOInvariant);
 }
@@ -5567,8 +5579,8 @@ SDValue SITargetLowering::lowerImplicitZextParam(SelectionDAG &DAG,
                                                  MVT VT,
                                                  unsigned Offset) const {
   SDLoc SL(Op);
-  SDValue Param = lowerKernargMemParameter(DAG, MVT::i32, MVT::i32, SL,
-                                           DAG.getEntryNode(), Offset, 4, false);
+  SDValue Param = lowerKernargMemParameter(
+      DAG, MVT::i32, MVT::i32, SL, DAG.getEntryNode(), Offset, Align(4), false);
   // The local size values will have the hi 16-bits as zero.
   return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Param,
                      DAG.getValueType(VT));
@@ -6202,7 +6214,8 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
 
   // Use the alignment to ensure that the required offsets will fit into the
   // immediate offsets.
-  setBufferOffsets(Offset, DAG, &Ops[3], NumLoads > 1 ? 16 * NumLoads : 4);
+  setBufferOffsets(Offset, DAG, &Ops[3],
+                   NumLoads > 1 ? Align(16 * NumLoads) : Align(4));
 
   uint64_t InstOffset = cast<ConstantSDNode>(Ops[5])->getZExtValue();
   for (unsigned i = 0; i < NumLoads; ++i) {
@@ -6298,37 +6311,43 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
       return emitNonHSAIntrinsicError(DAG, DL, VT);
 
     return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(),
-                                    SI::KernelInputOffsets::NGROUPS_X, 4, false);
+                                    SI::KernelInputOffsets::NGROUPS_X, Align(4),
+                                    false);
   case Intrinsic::r600_read_ngroups_y:
     if (Subtarget->isAmdHsaOS())
       return emitNonHSAIntrinsicError(DAG, DL, VT);
 
     return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(),
-                                    SI::KernelInputOffsets::NGROUPS_Y, 4, false);
+                                    SI::KernelInputOffsets::NGROUPS_Y, Align(4),
+                                    false);
   case Intrinsic::r600_read_ngroups_z:
     if (Subtarget->isAmdHsaOS())
       return emitNonHSAIntrinsicError(DAG, DL, VT);
 
     return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(),
-                                    SI::KernelInputOffsets::NGROUPS_Z, 4, false);
+                                    SI::KernelInputOffsets::NGROUPS_Z, Align(4),
+                                    false);
   case Intrinsic::r600_read_global_size_x:
     if (Subtarget->isAmdHsaOS())
       return emitNonHSAIntrinsicError(DAG, DL, VT);
 
     return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(),
-                                    SI::KernelInputOffsets::GLOBAL_SIZE_X, 4, false);
+                                    SI::KernelInputOffsets::GLOBAL_SIZE_X,
+                                    Align(4), false);
   case Intrinsic::r600_read_global_size_y:
     if (Subtarget->isAmdHsaOS())
       return emitNonHSAIntrinsicError(DAG, DL, VT);
 
     return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(),
-                                    SI::KernelInputOffsets::GLOBAL_SIZE_Y, 4, false);
+                                    SI::KernelInputOffsets::GLOBAL_SIZE_Y,
+                                    Align(4), false);
   case Intrinsic::r600_read_global_size_z:
     if (Subtarget->isAmdHsaOS())
       return emitNonHSAIntrinsicError(DAG, DL, VT);
 
     return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(),
-                                    SI::KernelInputOffsets::GLOBAL_SIZE_Z, 4, false);
+                                    SI::KernelInputOffsets::GLOBAL_SIZE_Z,
+                                    Align(4), false);
   case Intrinsic::r600_read_local_size_x:
     if (Subtarget->isAmdHsaOS())
       return emitNonHSAIntrinsicError(DAG, DL, VT);
@@ -7617,13 +7636,14 @@ std::pair<SDValue, SDValue> SITargetLowering::splitBufferOffsets(
 // three offsets (voffset, soffset and instoffset) into the SDValue[3] array
 // pointed to by Offsets.
 unsigned SITargetLowering::setBufferOffsets(SDValue CombinedOffset,
-                                        SelectionDAG &DAG, SDValue *Offsets,
-                                        unsigned Align) const {
+                                            SelectionDAG &DAG, SDValue *Offsets,
+                                            Align Alignment) const {
   SDLoc DL(CombinedOffset);
   if (auto C = dyn_cast<ConstantSDNode>(CombinedOffset)) {
     uint32_t Imm = C->getZExtValue();
     uint32_t SOffset, ImmOffset;
-    if (AMDGPU::splitMUBUFOffset(Imm, SOffset, ImmOffset, Subtarget, Align)) {
+    if (AMDGPU::splitMUBUFOffset(Imm, SOffset, ImmOffset, Subtarget,
+                                 Alignment)) {
       Offsets[0] = DAG.getConstant(0, DL, MVT::i32);
       Offsets[1] = DAG.getConstant(SOffset, DL, MVT::i32);
       Offsets[2] = DAG.getTargetConstant(ImmOffset, DL, MVT::i32);
@@ -7636,7 +7656,7 @@ unsigned SITargetLowering::setBufferOffsets(SDValue CombinedOffset,
     uint32_t SOffset, ImmOffset;
     int Offset = cast<ConstantSDNode>(N1)->getSExtValue();
     if (Offset >= 0 && AMDGPU::splitMUBUFOffset(Offset, SOffset, ImmOffset,
-                                                Subtarget, Align)) {
+                                                Subtarget, Alignment)) {
       Offsets[0] = N0;
       Offsets[1] = DAG.getConstant(SOffset, DL, MVT::i32);
       Offsets[2] = DAG.getTargetConstant(ImmOffset, DL, MVT::i32);
@@ -11153,6 +11173,24 @@ SITargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
   return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
 }
 
+static bool isImmConstraint(StringRef Constraint) {
+  if (Constraint.size() == 1) {
+    switch (Constraint[0]) {
+    default: break;
+    case 'I':
+    case 'J':
+    case 'A':
+    case 'B':
+    case 'C':
+      return true;
+    }
+  } else if (Constraint == "DA" ||
+             Constraint == "DB") {
+    return true;
+  }
+  return false;
+}
+
 SITargetLowering::ConstraintType
 SITargetLowering::getConstraintType(StringRef Constraint) const {
   if (Constraint.size() == 1) {
@@ -11162,67 +11200,115 @@ SITargetLowering::getConstraintType(StringRef Constraint) const {
     case 'v':
     case 'a':
       return C_RegisterClass;
-    case 'A':
-      return C_Other;
     }
   }
+  if (isImmConstraint(Constraint)) {
+    return C_Other;
+  }
   return TargetLowering::getConstraintType(Constraint);
 }
 
+static uint64_t clearUnusedBits(uint64_t Val, unsigned Size) {
+  if (!AMDGPU::isInlinableIntLiteral(Val)) {
+    Val = Val & maskTrailingOnes<uint64_t>(Size);
+  }
+  return Val;
+}
+
 void SITargetLowering::LowerAsmOperandForConstraint(SDValue Op,
                                                     std::string &Constraint,
                                                     std::vector<SDValue> &Ops,
                                                     SelectionDAG &DAG) const {
-  if (Constraint.length() == 1 && Constraint[0] == 'A') {
-    LowerAsmOperandForConstraintA(Op, Ops, DAG);
+  if (isImmConstraint(Constraint)) {
+    uint64_t Val;
+    if (getAsmOperandConstVal(Op, Val) &&
+        checkAsmConstraintVal(Op, Constraint, Val)) {
+      Val = clearUnusedBits(Val, Op.getScalarValueSizeInBits());
+      Ops.push_back(DAG.getTargetConstant(Val, SDLoc(Op), MVT::i64));
+    }
   } else {
     TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
   }
 }
 
-void SITargetLowering::LowerAsmOperandForConstraintA(SDValue Op,
-                                                     std::vector<SDValue> &Ops,
-                                                     SelectionDAG &DAG) const {
+bool SITargetLowering::getAsmOperandConstVal(SDValue Op, uint64_t &Val) const {
   unsigned Size = Op.getScalarValueSizeInBits();
   if (Size > 64)
-    return;
+    return false;
+
+  if (Size == 16 && !Subtarget->has16BitInsts())
+    return false;
 
-  uint64_t Val;
-  bool IsConst = false;
   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
     Val = C->getSExtValue();
-    IsConst = true;
-  } else if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) {
+    return true;
+  }
+  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) {
     Val = C->getValueAPF().bitcastToAPInt().getSExtValue();
-    IsConst = true;
-  } else if (BuildVectorSDNode *V = dyn_cast<BuildVectorSDNode>(Op)) {
+    return true;
+  }
+  if (BuildVectorSDNode *V = dyn_cast<BuildVectorSDNode>(Op)) {
     if (Size != 16 || Op.getNumOperands() != 2)
-      return;
+      return false;
     if (Op.getOperand(0).isUndef() || Op.getOperand(1).isUndef())
-      return;
+      return false;
     if (ConstantSDNode *C = V->getConstantSplatNode()) {
       Val = C->getSExtValue();
-      IsConst = true;
-    } else if (ConstantFPSDNode *C = V->getConstantFPSplatNode()) {
+      return true;
+    }
+    if (ConstantFPSDNode *C = V->getConstantFPSplatNode()) {
       Val = C->getValueAPF().bitcastToAPInt().getSExtValue();
-      IsConst = true;
+      return true;
     }
   }
 
-  if (IsConst) {
-    bool HasInv2Pi = Subtarget->hasInv2PiInlineImm();
-    if ((Size == 16 && AMDGPU::isInlinableLiteral16(Val, HasInv2Pi)) ||
-        (Size == 32 && AMDGPU::isInlinableLiteral32(Val, HasInv2Pi)) ||
-        (Size == 64 && AMDGPU::isInlinableLiteral64(Val, HasInv2Pi))) {
-      // Clear unused bits of fp constants
-      if (!AMDGPU::isInlinableIntLiteral(Val)) {
-        unsigned UnusedBits = 64 - Size;
-        Val = (Val << UnusedBits) >> UnusedBits;
-      }
-      auto Res = DAG.getTargetConstant(Val, SDLoc(Op), MVT::i64);
-      Ops.push_back(Res);
+  return false;
+}
+
+bool SITargetLowering::checkAsmConstraintVal(SDValue Op,
+                                             const std::string &Constraint,
+                                             uint64_t Val) const {
+  if (Constraint.size() == 1) {
+    switch (Constraint[0]) {
+    case 'I':
+      return AMDGPU::isInlinableIntLiteral(Val);
+    case 'J':
+      return isInt<16>(Val);
+    case 'A':
+      return checkAsmConstraintValA(Op, Val);
+    case 'B':
+      return isInt<32>(Val);
+    case 'C':
+      return isUInt<32>(clearUnusedBits(Val, Op.getScalarValueSizeInBits())) ||
+             AMDGPU::isInlinableIntLiteral(Val);
+    default:
+      break;
+    }
+  } else if (Constraint.size() == 2) {
+    if (Constraint == "DA") {
+      int64_t HiBits = static_cast<int32_t>(Val >> 32);
+      int64_t LoBits = static_cast<int32_t>(Val);
+      return checkAsmConstraintValA(Op, HiBits, 32) &&
+             checkAsmConstraintValA(Op, LoBits, 32);
     }
+    if (Constraint == "DB") {
+      return true;
+    }
+  }
+  llvm_unreachable("Invalid asm constraint");
+}
+
+bool SITargetLowering::checkAsmConstraintValA(SDValue Op,
+                                              uint64_t Val,
+                                              unsigned MaxSize) const {
+  unsigned Size = std::min<unsigned>(Op.getScalarValueSizeInBits(), MaxSize);
+  bool HasInv2Pi = Subtarget->hasInv2PiInlineImm();
+  if ((Size == 16 && AMDGPU::isInlinableLiteral16(Val, HasInv2Pi)) ||
+      (Size == 32 && AMDGPU::isInlinableLiteral32(Val, HasInv2Pi)) ||
+      (Size == 64 && AMDGPU::isInlinableLiteral64(Val, HasInv2Pi))) {
+    return true;
   }
+  return false;
 }
 
 // Figure out which registers should be reserved for stack access. Only after
@@ -11604,3 +11690,18 @@ bool SITargetLowering::requiresUniformRegister(MachineFunction &MF,
   SmallPtrSet<const Value *, 16> Visited;
   return hasCFUser(V, Visited, Subtarget->getWavefrontSize());
 }
+
+std::pair<int, MVT>
+SITargetLowering::getTypeLegalizationCost(const DataLayout &DL,
+                                          Type *Ty) const {
+  auto Cost = TargetLoweringBase::getTypeLegalizationCost(DL, Ty);
+  auto Size = DL.getTypeSizeInBits(Ty);
+  // Maximum load or store can handle 8 dwords for scalar and 4 for
+  // vector ALU. Let's assume anything above 8 dwords is expensive
+  // even if legal.
+  if (Size <= 256)
+    return Cost;
+
+  Cost.first = (Size + 255) / 256;
+  return Cost;
+}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 3365c08b9e76c..f4c0764640575 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -42,7 +42,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
   SDValue getImplicitArgPtr(SelectionDAG &DAG, const SDLoc &SL) const;
   SDValue lowerKernargMemParameter(SelectionDAG &DAG, EVT VT, EVT MemVT,
                                    const SDLoc &SL, SDValue Chain,
-                                   uint64_t Offset, unsigned Align, bool Signed,
+                                   uint64_t Offset, Align Alignment,
+                                   bool Signed,
                                    const ISD::InputArg *Arg = nullptr) const;
 
   SDValue lowerStackParameter(SelectionDAG &DAG, CCValAssign &VA,
@@ -216,7 +217,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
   /// \returns 0 If there is a non-constant offset or if the offset is 0.
   /// Otherwise returns the constant offset.
   unsigned setBufferOffsets(SDValue CombinedOffset, SelectionDAG &DAG,
-                           SDValue *Offsets, unsigned Align = 4) const;
+                            SDValue *Offsets, Align Alignment = Align(4)) const;
 
   // Handle 8 bit and 16 bit buffer loads
   SDValue handleByteShortBufferLoads(SelectionDAG &DAG, EVT LoadVT, SDLoc DL,
@@ -396,9 +397,13 @@ class SITargetLowering final : public AMDGPUTargetLowering {
                                     std::string &Constraint,
                                     std::vector<SDValue> &Ops,
                                     SelectionDAG &DAG) const override;
-  void LowerAsmOperandForConstraintA(SDValue Op,
-                                     std::vector<SDValue> &Ops,
-                                     SelectionDAG &DAG) const;
+  bool getAsmOperandConstVal(SDValue Op, uint64_t &Val) const;
+  bool checkAsmConstraintVal(SDValue Op,
+                             const std::string &Constraint,
+                             uint64_t Val) const;
+  bool checkAsmConstraintValA(SDValue Op,
+                              uint64_t Val,
+                              unsigned MaxSize = 64) const;
   SDValue copyToM0(SelectionDAG &DAG, SDValue Chain, const SDLoc &DL,
                    SDValue V) const;
 
@@ -459,6 +464,9 @@ class SITargetLowering final : public AMDGPUTargetLowering {
                                       MachineFunction &MF,
                                       const SIRegisterInfo &TRI,
                                       SIMachineFunctionInfo &Info) const;
+
+  std::pair<int, MVT> getTypeLegalizationCost(const DataLayout &DL,
+                                              Type *Ty) const;
 };
 
 } // End namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp b/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp
index de4d7904caa40..052db5f6ea718 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp
@@ -57,10 +57,13 @@ class SIInsertSkips : public MachineFunctionPass {
   unsigned SkipThreshold = 0;
   MachineDominatorTree *MDT = nullptr;
 
+  MachineBasicBlock *EarlyExitBlock = nullptr;
+
   bool shouldSkip(const MachineBasicBlock &From,
                   const MachineBasicBlock &To) const;
 
   bool dominatesAllReachable(MachineBasicBlock &MBB);
+  void createEarlyExitBlock(MachineBasicBlock &MBB);
   void skipIfDead(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                   DebugLoc DL);
 
@@ -161,6 +164,33 @@ bool SIInsertSkips::dominatesAllReachable(MachineBasicBlock &MBB) {
   return true;
 }
 
+static void generatePsEndPgm(MachineBasicBlock &MBB,
+                             MachineBasicBlock::iterator I, DebugLoc DL,
+                             const SIInstrInfo *TII) {
+  // Generate "null export; s_endpgm".
+  BuildMI(MBB, I, DL, TII->get(AMDGPU::EXP_DONE))
+      .addImm(0x09) // V_008DFC_SQ_EXP_NULL
+      .addReg(AMDGPU::VGPR0, RegState::Undef)
+      .addReg(AMDGPU::VGPR0, RegState::Undef)
+      .addReg(AMDGPU::VGPR0, RegState::Undef)
+      .addReg(AMDGPU::VGPR0, RegState::Undef)
+      .addImm(1)  // vm
+      .addImm(0)  // compr
+      .addImm(0); // en
+  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ENDPGM)).addImm(0);
+}
+
+void SIInsertSkips::createEarlyExitBlock(MachineBasicBlock &MBB) {
+  MachineFunction *MF = MBB.getParent();
+  DebugLoc DL;
+
+  assert(!EarlyExitBlock);
+  EarlyExitBlock = MF->CreateMachineBasicBlock();
+  MF->insert(MF->end(), EarlyExitBlock);
+
+  generatePsEndPgm(*EarlyExitBlock, EarlyExitBlock->end(), DL, TII);
+}
+
 /// Insert an "if exec=0 { null export; s_endpgm }" sequence before the given
 /// iterator. Only applies to pixel shaders.
 void SIInsertSkips::skipIfDead(MachineBasicBlock &MBB,
@@ -168,11 +198,6 @@ void SIInsertSkips::skipIfDead(MachineBasicBlock &MBB,
   MachineFunction *MF = MBB.getParent();
   assert(MF->getFunction().getCallingConv() == CallingConv::AMDGPU_PS);
 
-  // Currently, SI_KILL_*_TERMINATOR is expected to occur only as the last
-  // terminator of a basic block. If this ever changes, we need to optionally
-  // split MBB here.
-  assert(I == MBB.end());
-
   // It is possible for an SI_KILL_*_TERMINATOR to sit at the bottom of a
   // basic block that has no further successors (e.g., there was an
   // `unreachable` there in IR). This can happen with original source of the
@@ -186,34 +211,49 @@ void SIInsertSkips::skipIfDead(MachineBasicBlock &MBB,
   // In this case, we write the "null_export; s_endpgm" skip code in the
   // already-existing basic block.
   auto NextBBI = std::next(MBB.getIterator());
-  bool NoSuccessor = llvm::find(MBB.successors(), &*NextBBI) == MBB.succ_end();
-  MachineBasicBlock *SkipBB;
+  bool NoSuccessor = I == MBB.end() &&
+                     llvm::find(MBB.successors(), &*NextBBI) == MBB.succ_end();
 
   if (NoSuccessor) {
-    SkipBB = &MBB;
+    generatePsEndPgm(MBB, I, DL, TII);
   } else {
-    // Create a new basic block that will contain the "null export; s_endpgm"
-    // and set up the branching to go around it.
-    SkipBB = MF->CreateMachineBasicBlock();
-    MF->insert(NextBBI, SkipBB);
+    if (!EarlyExitBlock) {
+      createEarlyExitBlock(MBB);
+      // Update next block pointer to reflect any new blocks
+      NextBBI = std::next(MBB.getIterator());
+    }
 
-    BuildMI(&MBB, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&*NextBBI);
-    MBB.addSuccessor(SkipBB);
+    auto BranchMI = BuildMI(MBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ))
+                        .addMBB(EarlyExitBlock);
+
+    // Split the block if the branch will not come at the end.
+    auto Next = std::next(BranchMI->getIterator());
+    if (Next != MBB.end() && !Next->isTerminator()) {
+      MachineBasicBlock *SplitBB =
+          MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+      MF->insert(NextBBI, SplitBB);
+      SplitBB->splice(SplitBB->begin(), &MBB, I, MBB.end());
+      SplitBB->transferSuccessorsAndUpdatePHIs(&MBB);
+      // FIXME: the expectation is that this will be used near the beginning
+      //        of a block so just assume all registers are still live.
+      for (auto LiveIn : MBB.liveins())
+        SplitBB->addLiveIn(LiveIn);
+      MBB.addSuccessor(SplitBB);
+
+      // Update dominator tree
+      using DomTreeT = DomTreeBase<MachineBasicBlock>;
+      SmallVector<DomTreeT::UpdateType, 16> DTUpdates;
+      for (MachineBasicBlock *Succ : SplitBB->successors()) {
+        DTUpdates.push_back({DomTreeT::Insert, SplitBB, Succ});
+        DTUpdates.push_back({DomTreeT::Delete, &MBB, Succ});
+      }
+      DTUpdates.push_back({DomTreeT::Insert, &MBB, SplitBB});
+      MDT->getBase().applyUpdates(DTUpdates);
+    }
 
-    MDT->addNewBlock(SkipBB, &MBB);
+    MBB.addSuccessor(EarlyExitBlock);
+    MDT->getBase().insertEdge(&MBB, EarlyExitBlock);
   }
-
-  // Generate "null export; s_endpgm".
-  BuildMI(SkipBB, DL, TII->get(AMDGPU::EXP_DONE))
-      .addImm(0x09) // V_008DFC_SQ_EXP_NULL
-      .addReg(AMDGPU::VGPR0, RegState::Undef)
-      .addReg(AMDGPU::VGPR0, RegState::Undef)
-      .addReg(AMDGPU::VGPR0, RegState::Undef)
-      .addReg(AMDGPU::VGPR0, RegState::Undef)
-      .addImm(1)  // vm
-      .addImm(0)  // compr
-      .addImm(0); // en
-  BuildMI(SkipBB, DL, TII->get(AMDGPU::S_ENDPGM)).addImm(0);
 }
 
 /// Translate a SI_KILL_*_TERMINATOR into exec-manipulating instructions.
@@ -366,7 +406,6 @@ bool SIInsertSkips::runOnMachineFunction(MachineFunction &MF) {
   MDT = &getAnalysis<MachineDominatorTree>();
   SkipThreshold = SkipThresholdFlag;
 
-  MachineBasicBlock *EmptyMBBAtEnd = nullptr;
   SmallVector<MachineInstr *, 4> KillInstrs;
   bool MadeChange = false;
 
@@ -417,26 +456,12 @@ bool SIInsertSkips::runOnMachineFunction(MachineFunction &MF) {
         break;
       }
 
-      case AMDGPU::SI_RETURN_TO_EPILOG:
-        // FIXME: Should move somewhere else
-        assert(!MF.getInfo<SIMachineFunctionInfo>()->returnsVoid());
-
-        // Graphics shaders returning non-void shouldn't contain S_ENDPGM,
-        // because external bytecode will be appended at the end.
-        if (&MBB != &MF.back() || &MI != &MBB.back()) {
-          // SI_RETURN_TO_EPILOG is not the last instruction. Add an empty block at
-          // the end and jump there.
-          if (!EmptyMBBAtEnd) {
-            EmptyMBBAtEnd = MF.CreateMachineBasicBlock();
-            MF.insert(MF.end(), EmptyMBBAtEnd);
-          }
-
-          MBB.addSuccessor(EmptyMBBAtEnd);
-          BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(AMDGPU::S_BRANCH))
-            .addMBB(EmptyMBBAtEnd);
+      case AMDGPU::SI_KILL_CLEANUP:
+        if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS &&
+            dominatesAllReachable(MBB)) {
+          KillInstrs.push_back(&MI);
+        } else {
           MI.eraseFromParent();
-
-          MDT->getBase().insertEdge(&MBB, EmptyMBBAtEnd);
         }
         break;
 
@@ -452,6 +477,7 @@ bool SIInsertSkips::runOnMachineFunction(MachineFunction &MF) {
     Kill->eraseFromParent();
   }
   KillInstrs.clear();
+  EarlyExitBlock = nullptr;
 
   return MadeChange;
 }
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 0dcaecd844e51..9af8ffedce0f3 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -144,6 +144,8 @@ bool SIInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
   case AMDGPU::V_MOV_B32_e32:
   case AMDGPU::V_MOV_B32_e64:
   case AMDGPU::V_MOV_B64_PSEUDO:
+  case AMDGPU::V_ACCVGPR_READ_B32:
+  case AMDGPU::V_ACCVGPR_WRITE_B32:
     // No implicit operands.
     return MI.getNumOperands() == MI.getDesc().getNumOperands();
   default:
@@ -396,6 +398,9 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
       BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr));
     }
     Offset = 0;
+    // Get appropriate operand, and compute width accordingly.
+    DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
+    Width = getOpSize(LdSt, DataOpIdx);
     return true;
   }
 
@@ -469,65 +474,27 @@ bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
                                       ArrayRef<const MachineOperand *> BaseOps2,
                                       unsigned NumLoads,
                                       unsigned NumBytes) const {
+  // If current mem ops pair do not have same base pointer, then they cannot be
+  // clustered.
   assert(!BaseOps1.empty() && !BaseOps2.empty());
   const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
   const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
-
   if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2))
     return false;
 
-  const MachineOperand *FirstDst = nullptr;
-  const MachineOperand *SecondDst = nullptr;
-
-  if ((isMUBUF(FirstLdSt) && isMUBUF(SecondLdSt)) ||
-      (isMTBUF(FirstLdSt) && isMTBUF(SecondLdSt)) ||
-      (isMIMG(FirstLdSt) && isMIMG(SecondLdSt)) ||
-      (isFLAT(FirstLdSt) && isFLAT(SecondLdSt))) {
-    const unsigned MaxGlobalLoadCluster = 7;
-    if (NumLoads > MaxGlobalLoadCluster)
-      return false;
-
-    FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdata);
-    if (!FirstDst)
-      FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
-    SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdata);
-    if (!SecondDst)
-      SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
-  } else if (isSMRD(FirstLdSt) && isSMRD(SecondLdSt)) {
-    FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::sdst);
-    SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::sdst);
-  } else if (isDS(FirstLdSt) && isDS(SecondLdSt)) {
-    FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
-    SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
-  }
-
-  if (!FirstDst || !SecondDst)
-    return false;
-
-  // Try to limit clustering based on the total number of bytes loaded
-  // rather than the number of instructions.  This is done to help reduce
-  // register pressure.  The method used is somewhat inexact, though,
-  // because it assumes that all loads in the cluster will load the
-  // same number of bytes as FirstLdSt.
-
-  // The unit of this value is bytes.
-  // FIXME: This needs finer tuning.
-  unsigned LoadClusterThreshold = 16;
-
-  const MachineRegisterInfo &MRI =
-      FirstLdSt.getParent()->getParent()->getRegInfo();
-
-  const Register Reg = FirstDst->getReg();
-
-  const TargetRegisterClass *DstRC = Register::isVirtualRegister(Reg)
-                                         ? MRI.getRegClass(Reg)
-                                         : RI.getPhysRegClass(Reg);
-
-  // FIXME: NumLoads should not be subtracted 1. This is to match behavior
-  // of clusterNeighboringMemOps which was previosly passing cluster length
-  // less 1. LoadClusterThreshold should be tuned instead.
-  return ((NumLoads - 1) * (RI.getRegSizeInBits(*DstRC) / 8)) <=
-         LoadClusterThreshold;
+  // Compute max cluster size based on average number bytes clustered till now,
+  // and decide based on it, if current mem ops pair can be clustered or not.
+  assert((NumLoads > 0) && (NumBytes > 0) && (NumBytes >= NumLoads) &&
+         "Invalid NumLoads/NumBytes values");
+  unsigned MaxNumLoads;
+  if (NumBytes <= 4 * NumLoads) {
+    // Loads are dword or smaller (on average).
+    MaxNumLoads = 5;
+  } else {
+    // Loads are bigger than a dword (on average).
+    MaxNumLoads = 4;
+  }
+  return NumLoads <= MaxNumLoads;
 }
 
 // FIXME: This behaves strangely. If, for example, you have 32 load + stores,
@@ -637,6 +604,13 @@ void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
   }
 
   if (RC == &AMDGPU::SReg_64RegClass) {
+    if (SrcReg == AMDGPU::SCC) {
+      BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg)
+          .addImm(1)
+          .addImm(0);
+      return;
+    }
+
     if (DestReg == AMDGPU::VCC) {
       if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
         BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
@@ -663,10 +637,18 @@ void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
   }
 
   if (DestReg == AMDGPU::SCC) {
+    // Copying 64-bit or 32-bit sources to SCC barely makes sense,
+    // but SelectionDAG emits such copies for i1 sources.
+    // TODO: Use S_BITCMP0_B32 instead and only consider the 0th bit.
+    if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
+      SrcReg = RI.getSubReg(SrcReg, AMDGPU::sub0);
+    }
     assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
+
     BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
-      .addReg(SrcReg, getKillRegState(KillSrc))
-      .addImm(0);
+        .addReg(SrcReg, getKillRegState(KillSrc))
+        .addImm(0);
+
     return;
   }
 
@@ -1877,9 +1859,15 @@ static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI,
     RegOp.ChangeToImmediate(NonRegOp.getImm());
   else if (NonRegOp.isFI())
     RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
-  else
+  else if (NonRegOp.isGlobal()) {
+    RegOp.ChangeToGA(NonRegOp.getGlobal(), NonRegOp.getOffset(),
+                     NonRegOp.getTargetFlags());
+  } else
     return nullptr;
 
+  // Make sure we don't reinterpret a subreg index in the target flags.
+  RegOp.setTargetFlags(NonRegOp.getTargetFlags());
+
   NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
   NonRegOp.setSubReg(SubReg);
 
@@ -2300,6 +2288,7 @@ unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB,
 
     // Copy the flags onto the implicit condition register operand.
     preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
+    fixImplicitOperands(*CondBr);
 
     if (BytesAdded)
       *BytesAdded = 4;
@@ -3007,6 +2996,10 @@ bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
   if (MI.isTerminator() || MI.isPosition())
     return true;
 
+  // INLINEASM_BR can jump to another block
+  if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
+    return true;
+
   // Target-independent instructions do not have an implicit-use of EXEC, even
   // when they operate on VGPRs. Treating EXEC modifications as scheduling
   // boundaries prevents incorrect movements of such instructions.
@@ -3361,7 +3354,8 @@ static void copyFlagsToImplicitVCC(MachineInstr &MI,
                                    const MachineOperand &Orig) {
 
   for (MachineOperand &Use : MI.implicit_operands()) {
-    if (Use.isUse() && Use.getReg() == AMDGPU::VCC) {
+    if (Use.isUse() &&
+        (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) {
       Use.setIsUndef(Orig.isUndef());
       Use.setIsKill(Orig.isKill());
       return;
@@ -3793,7 +3787,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
     }
 
     if (isVOP3(MI) && LiteralCount) {
-      if (LiteralCount && !ST.hasVOP3Literal()) {
+      if (!ST.hasVOP3Literal()) {
         ErrInfo = "VOP3 instruction uses literal";
         return false;
       }
@@ -5401,6 +5395,12 @@ void SIInstrInfo::moveToVALU(MachineInstr &TopInst,
       Inst.eraseFromParent();
     }
       continue;
+
+    case AMDGPU::S_CSELECT_B32:
+    case AMDGPU::S_CSELECT_B64:
+      lowerSelect(Worklist, Inst, MDT);
+      Inst.eraseFromParent();
+      continue;
     }
 
     if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
@@ -5541,6 +5541,78 @@ bool SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst,
   return false;
 }
 
+void SIInstrInfo::lowerSelect(SetVectorType &Worklist, MachineInstr &Inst,
+                              MachineDominatorTree *MDT) const {
+
+  MachineBasicBlock &MBB = *Inst.getParent();
+  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
+  MachineBasicBlock::iterator MII = Inst;
+  DebugLoc DL = Inst.getDebugLoc();
+
+  MachineOperand &Dest = Inst.getOperand(0);
+  MachineOperand &Src0 = Inst.getOperand(1);
+  MachineOperand &Src1 = Inst.getOperand(2);
+  MachineOperand &Cond = Inst.getOperand(3);
+
+  Register SCCSource = Cond.getReg();
+  // Find SCC def, and if that is a copy (SCC = COPY reg) then use reg instead.
+  if (!Cond.isUndef()) {
+    for (MachineInstr &CandI :
+         make_range(std::next(MachineBasicBlock::reverse_iterator(Inst)),
+                    Inst.getParent()->rend())) {
+      if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) !=
+          -1) {
+        if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) {
+          SCCSource = CandI.getOperand(1).getReg();
+        }
+        break;
+      }
+    }
+  }
+
+  // If this is a trivial select where the condition is effectively not SCC
+  // (SCCSource is a source of copy to SCC), then the select is semantically
+  // equivalent to copying SCCSource. Hence, there is no need to create
+  // V_CNDMASK, we can just use that and bail out.
+  if ((SCCSource != AMDGPU::SCC) && Src0.isImm() && (Src0.getImm() == -1) &&
+      Src1.isImm() && (Src1.getImm() == 0)) {
+    MRI.replaceRegWith(Dest.getReg(), SCCSource);
+    return;
+  }
+
+  const TargetRegisterClass *TC = ST.getWavefrontSize() == 64
+                                      ? &AMDGPU::SReg_64_XEXECRegClass
+                                      : &AMDGPU::SReg_32_XM0_XEXECRegClass;
+  Register CopySCC = MRI.createVirtualRegister(TC);
+
+  if (SCCSource == AMDGPU::SCC) {
+    // Insert a trivial select instead of creating a copy, because a copy from
+    // SCC would semantically mean just copying a single bit, but we may need
+    // the result to be a vector condition mask that needs preserving.
+    unsigned Opcode = (ST.getWavefrontSize() == 64) ? AMDGPU::S_CSELECT_B64
+                                                    : AMDGPU::S_CSELECT_B32;
+    auto NewSelect =
+        BuildMI(MBB, MII, DL, get(Opcode), CopySCC).addImm(-1).addImm(0);
+    NewSelect->getOperand(3).setIsUndef(Cond.isUndef());
+  } else {
+    BuildMI(MBB, MII, DL, get(AMDGPU::COPY), CopySCC).addReg(SCCSource);
+  }
+
+  Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+
+  auto UpdatedInst =
+      BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B32_e64), ResultReg)
+          .addImm(0)
+          .add(Src1) // False
+          .addImm(0)
+          .add(Src0) // True
+          .addReg(CopySCC);
+
+  MRI.replaceRegWith(Dest.getReg(), ResultReg);
+  legalizeOperands(*UpdatedInst, MDT);
+  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
+}
+
 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist,
                                  MachineInstr &Inst) const {
   MachineBasicBlock &MBB = *Inst.getParent();
@@ -6122,6 +6194,8 @@ void SIInstrInfo::movePackToVALU(SetVectorType &Worklist,
 void SIInstrInfo::addSCCDefUsersToVALUWorklist(MachineOperand &Op,
                                                MachineInstr &SCCDefInst,
                                                SetVectorType &Worklist) const {
+  bool SCCUsedImplicitly = false;
+
   // Ensure that def inst defines SCC, which is still live.
   assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() &&
          !Op.isDead() && Op.getParent() == &SCCDefInst);
@@ -6136,19 +6210,32 @@ void SIInstrInfo::addSCCDefUsersToVALUWorklist(MachineOperand &Op,
       if (MI.isCopy()) {
         MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
         unsigned DestReg = MI.getOperand(0).getReg();
-        SmallVector<MachineInstr *, 4> Users;
+
         for (auto &User : MRI.use_nodbg_instructions(DestReg)) {
           if ((User.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) ||
               (User.getOpcode() == AMDGPU::S_SUB_CO_PSEUDO)) {
-            Users.push_back(&User);
+            User.getOperand(4).setReg(RI.getVCC());
             Worklist.insert(&User);
+          } else if (User.getOpcode() == AMDGPU::V_CNDMASK_B32_e64) {
+            User.getOperand(5).setReg(RI.getVCC());
+            // No need to add to Worklist.
           }
         }
-        for (auto &U : Users)
-          U->getOperand(4).setReg(RI.getVCC());
         CopyToDelete.push_back(&MI);
-      } else
+      } else {
+        if (MI.getOpcode() == AMDGPU::S_CSELECT_B32 ||
+            MI.getOpcode() == AMDGPU::S_CSELECT_B64) {
+          // This is an implicit use of SCC and it is really expected by
+          // the SCC users to handle.
+          // We cannot preserve the edge to the user so add the explicit
+          // copy: SCC = COPY VCC.
+          // The copy will be cleaned up during the processing of the user
+          // in lowerSelect.
+          SCCUsedImplicitly = true;
+        }
+
         Worklist.insert(&MI);
+      }
     }
     // Exit if we find another SCC def.
     if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1)
@@ -6156,6 +6243,12 @@ void SIInstrInfo::addSCCDefUsersToVALUWorklist(MachineOperand &Op,
   }
   for (auto &Copy : CopyToDelete)
     Copy->eraseFromParent();
+
+  if (SCCUsedImplicitly) {
+    BuildMI(*SCCDefInst.getParent(), std::next(SCCDefInst.getIterator()),
+            SCCDefInst.getDebugLoc(), get(AMDGPU::COPY), AMDGPU::SCC)
+        .addReg(RI.getVCC());
+  }
 }
 
 const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 0f0e8420f9cf2..53e2ffba0f656 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -84,6 +84,9 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
   bool moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst,
                         MachineDominatorTree *MDT = nullptr) const;
 
+  void lowerSelect(SetVectorType &Worklist, MachineInstr &Inst,
+                   MachineDominatorTree *MDT = nullptr) const;
+
   void lowerScalarAbs(SetVectorType &Worklist,
                       MachineInstr &Inst) const;
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 24113cf69f7e6..0c4c9e0e9df2b 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -379,6 +379,9 @@ multiclass PseudoInstKill <dag ins> {
 defm SI_KILL_I1 : PseudoInstKill <(ins SCSrc_i1:$src, i1imm:$killvalue)>;
 defm SI_KILL_F32_COND_IMM : PseudoInstKill <(ins VSrc_b32:$src0, i32imm:$src1, i32imm:$cond)>;
 
+let Defs = [EXEC] in
+def SI_KILL_CLEANUP : SPseudoInstSI <(outs), (ins)>;
+
 let Defs = [EXEC,VCC] in
 def SI_ILLEGAL_COPY : SPseudoInstSI <
   (outs unknown:$dst), (ins unknown:$src),
@@ -447,7 +450,7 @@ def SI_INIT_EXEC_FROM_INPUT : SPseudoInstSI <
 
 def : GCNPat <
   (int_amdgcn_init_exec timm:$src),
-  (SI_INIT_EXEC_LO (as_i32imm imm:$src))> {
+  (SI_INIT_EXEC_LO (as_i32timm timm:$src))> {
   let WaveSizePredicate = isWave32;
 }
 
@@ -1549,11 +1552,12 @@ class Ext32Pat <SDNode ext> : GCNPat <
 def : Ext32Pat <zext>;
 def : Ext32Pat <anyext>;
 
-// The multiplication scales from [0,1] to the unsigned integer range
+// The multiplication scales from [0,1) to the unsigned integer range,
+// rounding down a bit to avoid unwanted overflow.
 def : GCNPat <
   (AMDGPUurecip i32:$src0),
   (V_CVT_U32_F32_e32
-    (V_MUL_F32_e32 (i32 CONST.FP_UINT_MAX_PLUS_1),
+    (V_MUL_F32_e32 (i32 CONST.FP_4294966784),
                    (V_RCP_IFLAG_F32_e32 (V_CVT_F32_U32_e32 $src0))))
 >;
 
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index 62a7d9cd0ed21..2eb1c52f1b595 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -103,15 +103,19 @@ enum InstClassEnum {
   TBUFFER_STORE,
 };
 
-enum RegisterEnum {
-  SBASE = 0x1,
-  SRSRC = 0x2,
-  SOFFSET = 0x4,
-  VADDR = 0x8,
-  ADDR = 0x10,
-  SSAMP = 0x20,
+struct AddressRegs {
+  unsigned char NumVAddrs = 0;
+  bool SBase = false;
+  bool SRsrc = false;
+  bool SOffset = false;
+  bool VAddr = false;
+  bool Addr = false;
+  bool SSamp = false;
 };
 
+// GFX10 image_sample instructions can have 12 vaddrs + srsrc + ssamp.
+const unsigned MaxAddressRegs = 12 + 1 + 1;
+
 class SILoadStoreOptimizer : public MachineFunctionPass {
   struct CombineInfo {
     MachineBasicBlock::iterator I;
@@ -126,8 +130,8 @@ class SILoadStoreOptimizer : public MachineFunctionPass {
     bool SLC;
     bool DLC;
     bool UseST64;
-    int AddrIdx[5];
-    const MachineOperand *AddrReg[5];
+    int AddrIdx[MaxAddressRegs];
+    const MachineOperand *AddrReg[MaxAddressRegs];
     unsigned NumAddresses;
     unsigned Order;
 
@@ -292,6 +296,11 @@ class SILoadStoreOptimizer : public MachineFunctionPass {
 
     MachineFunctionPass::getAnalysisUsage(AU);
   }
+
+  MachineFunctionProperties getRequiredProperties() const override {
+    return MachineFunctionProperties()
+      .set(MachineFunctionProperties::Property::IsSSA);
+  }
 };
 
 static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
@@ -344,7 +353,8 @@ static InstClassEnum getInstClass(unsigned Opc, const SIInstrInfo &TII) {
     }
     if (TII.isMIMG(Opc)) {
       // Ignore instructions encoded without vaddr.
-      if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr) == -1)
+      if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr) == -1 &&
+          AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0) == -1)
         return UNKNOWN;
       // TODO: Support IMAGE_GET_RESINFO and IMAGE_GET_LOD.
       if (TII.get(Opc).mayStore() || !TII.get(Opc).mayLoad() ||
@@ -417,58 +427,54 @@ static unsigned getInstSubclass(unsigned Opc, const SIInstrInfo &TII) {
   }
 }
 
-static unsigned getRegs(unsigned Opc, const SIInstrInfo &TII) {
-  if (TII.isMUBUF(Opc)) {
-    unsigned result = 0;
-
-    if (AMDGPU::getMUBUFHasVAddr(Opc)) {
-      result |= VADDR;
-    }
-
-    if (AMDGPU::getMUBUFHasSrsrc(Opc)) {
-      result |= SRSRC;
-    }
-
-    if (AMDGPU::getMUBUFHasSoffset(Opc)) {
-      result |= SOFFSET;
-    }
+static AddressRegs getRegs(unsigned Opc, const SIInstrInfo &TII) {
+  AddressRegs Result;
 
-    return result;
+  if (TII.isMUBUF(Opc)) {
+    if (AMDGPU::getMUBUFHasVAddr(Opc))
+      Result.VAddr = true;
+    if (AMDGPU::getMUBUFHasSrsrc(Opc))
+      Result.SRsrc = true;
+    if (AMDGPU::getMUBUFHasSoffset(Opc))
+      Result.SOffset = true;
+
+    return Result;
   }
 
   if (TII.isMIMG(Opc)) {
-    unsigned result = VADDR | SRSRC;
+    int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
+    if (VAddr0Idx >= 0) {
+      int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);
+      Result.NumVAddrs = SRsrcIdx - VAddr0Idx;
+    } else {
+      Result.VAddr = true;
+    }
+    Result.SRsrc = true;
     const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
     if (Info && AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode)->Sampler)
-      result |= SSAMP;
+      Result.SSamp = true;
 
-    return result;
+    return Result;
   }
   if (TII.isMTBUF(Opc)) {
-    unsigned result = 0;
-
-    if (AMDGPU::getMTBUFHasVAddr(Opc)) {
-      result |= VADDR;
-    }
-
-    if (AMDGPU::getMTBUFHasSrsrc(Opc)) {
-      result |= SRSRC;
-    }
-
-    if (AMDGPU::getMTBUFHasSoffset(Opc)) {
-      result |= SOFFSET;
-    }
-
-    return result;
+    if (AMDGPU::getMTBUFHasVAddr(Opc))
+      Result.VAddr = true;
+    if (AMDGPU::getMTBUFHasSrsrc(Opc))
+      Result.SRsrc = true;
+    if (AMDGPU::getMTBUFHasSoffset(Opc))
+      Result.SOffset = true;
+
+    return Result;
   }
 
   switch (Opc) {
   default:
-    return 0;
+    return Result;
   case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
   case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
   case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
-    return SBASE;
+    Result.SBase = true;
+    return Result;
   case AMDGPU::DS_READ_B32:
   case AMDGPU::DS_READ_B64:
   case AMDGPU::DS_READ_B32_gfx9:
@@ -477,7 +483,8 @@ static unsigned getRegs(unsigned Opc, const SIInstrInfo &TII) {
   case AMDGPU::DS_WRITE_B64:
   case AMDGPU::DS_WRITE_B32_gfx9:
   case AMDGPU::DS_WRITE_B64_gfx9:
-    return ADDR;
+    Result.Addr = true;
+    return Result;
   }
 }
 
@@ -534,38 +541,34 @@ void SILoadStoreOptimizer::CombineInfo::setMI(MachineBasicBlock::iterator MI,
     DLC = TII.getNamedOperand(*I, AMDGPU::OpName::dlc)->getImm();
   }
 
-  unsigned AddrOpName[5] = {0};
-  NumAddresses = 0;
-  const unsigned Regs = getRegs(I->getOpcode(), TII);
-
-  if (Regs & ADDR) {
-    AddrOpName[NumAddresses++] = AMDGPU::OpName::addr;
-  }
-
-  if (Regs & SBASE) {
-    AddrOpName[NumAddresses++] = AMDGPU::OpName::sbase;
-  }
-
-  if (Regs & SRSRC) {
-    AddrOpName[NumAddresses++] = AMDGPU::OpName::srsrc;
-  }
-
-  if (Regs & SOFFSET) {
-    AddrOpName[NumAddresses++] = AMDGPU::OpName::soffset;
-  }
+  AddressRegs Regs = getRegs(Opc, TII);
 
-  if (Regs & VADDR) {
-    AddrOpName[NumAddresses++] = AMDGPU::OpName::vaddr;
-  }
-
-  if (Regs & SSAMP) {
-    AddrOpName[NumAddresses++] = AMDGPU::OpName::ssamp;
-  }
-
-  for (unsigned i = 0; i < NumAddresses; i++) {
-    AddrIdx[i] = AMDGPU::getNamedOperandIdx(I->getOpcode(), AddrOpName[i]);
-    AddrReg[i] = &I->getOperand(AddrIdx[i]);
-  }
+  NumAddresses = 0;
+  for (unsigned J = 0; J < Regs.NumVAddrs; J++)
+    AddrIdx[NumAddresses++] =
+        AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0) + J;
+  if (Regs.Addr)
+    AddrIdx[NumAddresses++] =
+        AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::addr);
+  if (Regs.SBase)
+    AddrIdx[NumAddresses++] =
+        AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sbase);
+  if (Regs.SRsrc)
+    AddrIdx[NumAddresses++] =
+        AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);
+  if (Regs.SOffset)
+    AddrIdx[NumAddresses++] =
+        AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::soffset);
+  if (Regs.VAddr)
+    AddrIdx[NumAddresses++] =
+        AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr);
+  if (Regs.SSamp)
+    AddrIdx[NumAddresses++] =
+        AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::ssamp);
+  assert(NumAddresses <= MaxAddressRegs);
+
+  for (unsigned J = 0; J < NumAddresses; J++)
+    AddrReg[J] = &I->getOperand(AddrIdx[J]);
 }
 
 } // end anonymous namespace.
@@ -689,7 +692,7 @@ bool SILoadStoreOptimizer::dmasksCanBeCombined(const CombineInfo &CI,
   unsigned OperandsToMatch[] = {AMDGPU::OpName::glc, AMDGPU::OpName::slc,
                                 AMDGPU::OpName::d16, AMDGPU::OpName::unorm,
                                 AMDGPU::OpName::da,  AMDGPU::OpName::r128,
-                                AMDGPU::OpName::a16};
+                                AMDGPU::OpName::a16, AMDGPU::OpName::dlc};
 
   for (auto op : OperandsToMatch) {
     int Idx = AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), op);
@@ -1283,9 +1286,9 @@ MachineBasicBlock::iterator SILoadStoreOptimizer::mergeBufferLoadPair(
 
   auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode), DestReg);
 
-  const unsigned Regs = getRegs(Opcode, *TII);
+  AddressRegs Regs = getRegs(Opcode, *TII);
 
-  if (Regs & VADDR)
+  if (Regs.VAddr)
     MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr));
 
   // It shouldn't be possible to get this far if the two instructions
@@ -1346,9 +1349,9 @@ MachineBasicBlock::iterator SILoadStoreOptimizer::mergeTBufferLoadPair(
 
   auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode), DestReg);
 
-  const unsigned Regs = getRegs(Opcode, *TII);
+  AddressRegs Regs = getRegs(Opcode, *TII);
 
-  if (Regs & VADDR)
+  if (Regs.VAddr)
     MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr));
 
   unsigned JoinedFormat =
@@ -1426,9 +1429,9 @@ MachineBasicBlock::iterator SILoadStoreOptimizer::mergeTBufferStorePair(
   auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode))
                  .addReg(SrcReg, RegState::Kill);
 
-  const unsigned Regs = getRegs(Opcode, *TII);
+  AddressRegs Regs = getRegs(Opcode, *TII);
 
-  if (Regs & VADDR)
+  if (Regs.VAddr)
     MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr));
 
   unsigned JoinedFormat =
@@ -1589,9 +1592,9 @@ MachineBasicBlock::iterator SILoadStoreOptimizer::mergeBufferStorePair(
   auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode))
                  .addReg(SrcReg, RegState::Kill);
 
-  const unsigned Regs = getRegs(Opcode, *TII);
+  AddressRegs Regs = getRegs(Opcode, *TII);
 
-  if (Regs & VADDR)
+  if (Regs.VAddr)
     MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr));
 
 
@@ -1986,6 +1989,8 @@ SILoadStoreOptimizer::collectMergeableInsts(
     if (!CI.hasMergeableAddress(*MRI))
       continue;
 
+    LLVM_DEBUG(dbgs() << "Mergeable: " << MI);
+
     addInstToMergeableList(CI, MergeableInsts);
   }
 
@@ -2082,6 +2087,8 @@ SILoadStoreOptimizer::optimizeInstsWithSameBaseAddr(
 
     Modified = true;
 
+    LLVM_DEBUG(dbgs() << "Merging: " << *CI.I << "   with: " << *Paired.I);
+
     switch (CI.InstClass) {
     default:
       llvm_unreachable("unknown InstClass");
@@ -2165,8 +2172,6 @@ bool SILoadStoreOptimizer::runOnMachineFunction(MachineFunction &MF) {
   MRI = &MF.getRegInfo();
   AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
 
-  assert(MRI->isSSA() && "Must be run on SSA");
-
   LLVM_DEBUG(dbgs() << "Running SILoadStoreOptimizer\n");
 
   bool Modified = false;
diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
index 1e90e6ba54183..36d52ac3ee891 100644
--- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -89,8 +89,10 @@ class SILowerControlFlow : public MachineFunctionPass {
   MachineRegisterInfo *MRI = nullptr;
   SetVector<MachineInstr*> LoweredEndCf;
   DenseSet<Register> LoweredIf;
+  SmallSet<MachineInstr *, 16> NeedsKillCleanup;
 
   const TargetRegisterClass *BoolRC = nullptr;
+  bool InsertKillCleanups;
   unsigned AndOpc;
   unsigned OrOpc;
   unsigned XorOpc;
@@ -111,6 +113,8 @@ class SILowerControlFlow : public MachineFunctionPass {
 
   void combineMasks(MachineInstr &MI);
 
+  void process(MachineInstr &MI);
+
   // Skip to the next instruction, ignoring debug instructions, and trivial
   // block boundaries (blocks that have one (typically fallthrough) successor,
   // and the successor has one predecessor.
@@ -160,36 +164,36 @@ static void setImpSCCDefDead(MachineInstr &MI, bool IsDead) {
 
 char &llvm::SILowerControlFlowID = SILowerControlFlow::ID;
 
-static bool isSimpleIf(const MachineInstr &MI, const MachineRegisterInfo *MRI,
-                       const SIInstrInfo *TII) {
-  Register SaveExecReg = MI.getOperand(0).getReg();
-  auto U = MRI->use_instr_nodbg_begin(SaveExecReg);
-
-  if (U == MRI->use_instr_nodbg_end() ||
-      std::next(U) != MRI->use_instr_nodbg_end() ||
-      U->getOpcode() != AMDGPU::SI_END_CF)
-    return false;
-
-  // Check for SI_KILL_*_TERMINATOR on path from if to endif.
-  // if there is any such terminator simplififcations are not safe.
-  auto SMBB = MI.getParent();
-  auto EMBB = U->getParent();
+static bool hasKill(const MachineBasicBlock *Begin,
+                    const MachineBasicBlock *End, const SIInstrInfo *TII) {
   DenseSet<const MachineBasicBlock*> Visited;
-  SmallVector<MachineBasicBlock*, 4> Worklist(SMBB->succ_begin(),
-                                              SMBB->succ_end());
+  SmallVector<MachineBasicBlock *, 4> Worklist(Begin->succ_begin(),
+                                               Begin->succ_end());
 
   while (!Worklist.empty()) {
     MachineBasicBlock *MBB = Worklist.pop_back_val();
 
-    if (MBB == EMBB || !Visited.insert(MBB).second)
+    if (MBB == End || !Visited.insert(MBB).second)
       continue;
-    for(auto &Term : MBB->terminators())
+    for (auto &Term : MBB->terminators())
       if (TII->isKillTerminator(Term.getOpcode()))
-        return false;
+        return true;
 
     Worklist.append(MBB->succ_begin(), MBB->succ_end());
   }
 
+  return false;
+}
+
+static bool isSimpleIf(const MachineInstr &MI, const MachineRegisterInfo *MRI) {
+  Register SaveExecReg = MI.getOperand(0).getReg();
+  auto U = MRI->use_instr_nodbg_begin(SaveExecReg);
+
+  if (U == MRI->use_instr_nodbg_end() ||
+      std::next(U) != MRI->use_instr_nodbg_end() ||
+      U->getOpcode() != AMDGPU::SI_END_CF)
+    return false;
+
   return true;
 }
 
@@ -207,7 +211,35 @@ void SILowerControlFlow::emitIf(MachineInstr &MI) {
   // If there is only one use of save exec register and that use is SI_END_CF,
   // we can optimize SI_IF by returning the full saved exec mask instead of
   // just cleared bits.
-  bool SimpleIf = isSimpleIf(MI, MRI, TII);
+  bool SimpleIf = isSimpleIf(MI, MRI);
+
+  if (InsertKillCleanups) {
+    // Check for SI_KILL_*_TERMINATOR on full path of control flow and
+    // flag the associated SI_END_CF for insertion of a kill cleanup.
+    auto UseMI = MRI->use_instr_nodbg_begin(SaveExecReg);
+    while (UseMI->getOpcode() != AMDGPU::SI_END_CF) {
+      assert(std::next(UseMI) == MRI->use_instr_nodbg_end());
+      assert(UseMI->getOpcode() == AMDGPU::SI_ELSE);
+      MachineOperand &NextExec = UseMI->getOperand(0);
+      Register NextExecReg = NextExec.getReg();
+      if (NextExec.isDead()) {
+        assert(!SimpleIf);
+        break;
+      }
+      UseMI = MRI->use_instr_nodbg_begin(NextExecReg);
+    }
+    if (UseMI->getOpcode() == AMDGPU::SI_END_CF) {
+      if (hasKill(MI.getParent(), UseMI->getParent(), TII)) {
+        NeedsKillCleanup.insert(&*UseMI);
+        SimpleIf = false;
+      }
+    }
+  } else if (SimpleIf) {
+    // Check for SI_KILL_*_TERMINATOR on path from if to endif.
+    // if there is any such terminator simplifications are not safe.
+    auto UseMI = MRI->use_instr_nodbg_begin(SaveExecReg);
+    SimpleIf = !hasKill(MI.getParent(), UseMI->getParent(), TII);
+  }
 
   // Add an implicit def of exec to discourage scheduling VALU after this which
   // will interfere with trying to form s_and_saveexec_b64 later.
@@ -427,6 +459,8 @@ SILowerControlFlow::skipIgnoreExecInstsTrivialSucc(
 
     auto E = B->end();
     for ( ; It != E; ++It) {
+      if (It->getOpcode() == AMDGPU::SI_KILL_CLEANUP)
+        continue;
       if (TII->mayReadEXEC(*MRI, *It))
         break;
     }
@@ -461,8 +495,18 @@ void SILowerControlFlow::emitEndCf(MachineInstr &MI) {
 
   LoweredEndCf.insert(NewMI);
 
-  if (LIS)
+  // If this ends control flow which contains kills (as flagged in emitIf)
+  // then insert an SI_KILL_CLEANUP immediately following the exec mask
+  // manipulation.  This can be lowered to early termination if appropriate.
+  MachineInstr *CleanUpMI = nullptr;
+  if (NeedsKillCleanup.count(&MI))
+    CleanUpMI = BuildMI(MBB, InsPt, DL, TII->get(AMDGPU::SI_KILL_CLEANUP));
+
+  if (LIS) {
     LIS->ReplaceMachineInstrInMaps(MI, *NewMI);
+    if (CleanUpMI)
+      LIS->InsertMachineInstrInMaps(*CleanUpMI);
+  }
 
   MI.eraseFromParent();
 
@@ -553,6 +597,56 @@ void SILowerControlFlow::optimizeEndCf() {
   }
 }
 
+void SILowerControlFlow::process(MachineInstr &MI) {
+  MachineBasicBlock &MBB = *MI.getParent();
+  MachineBasicBlock::iterator I(MI);
+  MachineInstr *Prev = (I != MBB.begin()) ? &*(std::prev(I)) : nullptr;
+
+  switch (MI.getOpcode()) {
+  case AMDGPU::SI_IF:
+    emitIf(MI);
+    break;
+
+  case AMDGPU::SI_ELSE:
+    emitElse(MI);
+    break;
+
+  case AMDGPU::SI_IF_BREAK:
+    emitIfBreak(MI);
+    break;
+
+  case AMDGPU::SI_LOOP:
+    emitLoop(MI);
+    break;
+
+  case AMDGPU::SI_END_CF:
+    emitEndCf(MI);
+    break;
+
+  default:
+    assert(false && "Attempt to process unsupported instruction");
+    break;
+  }
+
+  MachineBasicBlock::iterator Next;
+  for (I = Prev ? Prev->getIterator() : MBB.begin(); I != MBB.end(); I = Next) {
+    Next = std::next(I);
+    MachineInstr &MaskMI = *I;
+    switch (MaskMI.getOpcode()) {
+    case AMDGPU::S_AND_B64:
+    case AMDGPU::S_OR_B64:
+    case AMDGPU::S_AND_B32:
+    case AMDGPU::S_OR_B32:
+      // Cleanup bit manipulations on exec mask
+      combineMasks(MaskMI);
+      break;
+    default:
+      I = MBB.end();
+      break;
+    }
+  }
+}
+
 bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   TII = ST.getInstrInfo();
@@ -562,6 +656,8 @@ bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
   LIS = getAnalysisIfAvailable<LiveIntervals>();
   MRI = &MF.getRegInfo();
   BoolRC = TRI->getBoolRC();
+  InsertKillCleanups =
+      MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS;
 
   if (ST.isWave32()) {
     AndOpc = AMDGPU::S_AND_B32;
@@ -583,62 +679,49 @@ bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
     Exec = AMDGPU::EXEC;
   }
 
+  SmallVector<MachineInstr *, 32> Worklist;
+
   MachineFunction::iterator NextBB;
   for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
        BI != BE; BI = NextBB) {
     NextBB = std::next(BI);
     MachineBasicBlock &MBB = *BI;
 
-    MachineBasicBlock::iterator I, Next, Last;
-
-    for (I = MBB.begin(), Last = MBB.end(); I != MBB.end(); I = Next) {
+    MachineBasicBlock::iterator I, Next;
+    for (I = MBB.begin(); I != MBB.end(); I = Next) {
       Next = std::next(I);
       MachineInstr &MI = *I;
 
       switch (MI.getOpcode()) {
       case AMDGPU::SI_IF:
-        emitIf(MI);
+        process(MI);
         break;
 
       case AMDGPU::SI_ELSE:
-        emitElse(MI);
-        break;
-
       case AMDGPU::SI_IF_BREAK:
-        emitIfBreak(MI);
-        break;
-
       case AMDGPU::SI_LOOP:
-        emitLoop(MI);
-        break;
-
       case AMDGPU::SI_END_CF:
-        emitEndCf(MI);
+        // Only build worklist if SI_IF instructions must be processed first.
+        if (InsertKillCleanups)
+          Worklist.push_back(&MI);
+        else
+          process(MI);
         break;
 
-      case AMDGPU::S_AND_B64:
-      case AMDGPU::S_OR_B64:
-      case AMDGPU::S_AND_B32:
-      case AMDGPU::S_OR_B32:
-        // Cleanup bit manipulations on exec mask
-        combineMasks(MI);
-        Last = I;
-        continue;
-
       default:
-        Last = I;
-        continue;
+        break;
       }
-
-      // Replay newly inserted code to combine masks
-      Next = (Last == MBB.end()) ? MBB.begin() : Last;
     }
   }
 
+  for (MachineInstr *MI : Worklist)
+    process(*MI);
+
   optimizeEndCf();
 
   LoweredEndCf.clear();
   LoweredIf.clear();
+  NeedsKillCleanup.clear();
 
   return true;
 }
diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
index 478a0d0ca9029..1349d3b6bf3f6 100644
--- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
@@ -211,8 +211,7 @@ bool SILowerSGPRSpills::spillCalleeSavedRegs(MachineFunction &MF) {
         const TargetRegisterClass *RC =
           TRI->getMinimalPhysRegClass(Reg, MVT::i32);
         int JunkFI = MFI.CreateStackObject(TRI->getSpillSize(*RC),
-                                           TRI->getSpillAlignment(*RC),
-                                           true);
+                                           TRI->getSpillAlign(*RC), true);
 
         CSI.push_back(CalleeSavedInfo(Reg, JunkFI));
       }
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
index d33193e10c8c4..788e9873f780f 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
@@ -303,7 +303,16 @@ bool SIMachineFunctionInfo::allocateSGPRSpillToVGPR(MachineFunction &MF,
     Register LaneVGPR;
     unsigned VGPRIndex = (NumVGPRSpillLanes % WaveSize);
 
-    if (VGPRIndex == 0 && !FuncInfo->VGPRReservedForSGPRSpill) {
+    // Reserve a VGPR (when NumVGPRSpillLanes = 0, WaveSize, 2*WaveSize, ..) and
+    // when one of the two conditions is true:
+    // 1. One reserved VGPR being tracked by VGPRReservedForSGPRSpill is not yet
+    // reserved.
+    // 2. All spill lanes of reserved VGPR(s) are full and another spill lane is
+    // required.
+    if (FuncInfo->VGPRReservedForSGPRSpill && NumVGPRSpillLanes < WaveSize) {
+      assert(FuncInfo->VGPRReservedForSGPRSpill == SpillVGPRs.back().VGPR);
+      LaneVGPR = FuncInfo->VGPRReservedForSGPRSpill;
+    } else if (VGPRIndex == 0) {
       LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);
       if (LaneVGPR == AMDGPU::NoRegister) {
         // We have no VGPRs left for spilling SGPRs. Reset because we will not
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
index 7221e01575226..cf1629fda0aff 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
@@ -679,13 +679,13 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunction {
     return ArgInfo;
   }
 
-  std::pair<const ArgDescriptor *, const TargetRegisterClass *>
+  std::tuple<const ArgDescriptor *, const TargetRegisterClass *, LLT>
   getPreloadedValue(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
     return ArgInfo.getPreloadedValue(Value);
   }
 
   Register getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
-    auto Arg = ArgInfo.getPreloadedValue(Value).first;
+    auto Arg = std::get<0>(ArgInfo.getPreloadedValue(Value));
     return Arg ? Arg->getRegister() : Register();
   }
 
diff --git a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
index 52989a280e806..0e162ac42c111 100644
--- a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
+++ b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
@@ -83,9 +83,7 @@ struct Status {
     return ((Mask & S.Mask) == S.Mask) && ((Mode & S.Mask) == S.Mode);
   }
 
-  bool isCombinable(Status &S) {
-    return !(Mask & S.Mask) || isCompatible(S);
-  }
+  bool isCombinable(Status &S) { return !(Mask & S.Mask) || isCompatible(S); }
 };
 
 class BlockData {
@@ -110,7 +108,11 @@ class BlockData {
   // which is used in Phase 3 if we need to insert a mode change.
   MachineInstr *FirstInsertionPoint;
 
-  BlockData() : FirstInsertionPoint(nullptr) {};
+  // A flag to indicate whether an Exit value has been set (we can't tell by
+  // examining the Exit value itself as all values may be valid results).
+  bool ExitSet;
+
+  BlockData() : FirstInsertionPoint(nullptr), ExitSet(false){};
 };
 
 namespace {
@@ -131,6 +133,8 @@ class SIModeRegister : public MachineFunctionPass {
   Status DefaultStatus =
       Status(FP_ROUND_MODE_DP(0x3), FP_ROUND_MODE_DP(DefaultMode));
 
+  bool Changed = false;
+
 public:
   SIModeRegister() : MachineFunctionPass(ID) {}
 
@@ -201,6 +205,7 @@ void SIModeRegister::insertSetreg(MachineBasicBlock &MBB, MachineInstr *MI,
                 (Offset << AMDGPU::Hwreg::OFFSET_SHIFT_) |
                 (AMDGPU::Hwreg::ID_MODE << AMDGPU::Hwreg::ID_SHIFT_));
     ++NumSetregInserted;
+    Changed = true;
     InstrMode.Mask &= ~(((1 << Width) - 1) << Offset);
   }
 }
@@ -325,24 +330,53 @@ void SIModeRegister::processBlockPhase1(MachineBasicBlock &MBB,
 // exit value is propagated.
 void SIModeRegister::processBlockPhase2(MachineBasicBlock &MBB,
                                         const SIInstrInfo *TII) {
-//  BlockData *BI = BlockInfo[MBB.getNumber()];
+  bool RevisitRequired = false;
+  bool ExitSet = false;
   unsigned ThisBlock = MBB.getNumber();
   if (MBB.pred_empty()) {
     // There are no predecessors, so use the default starting status.
     BlockInfo[ThisBlock]->Pred = DefaultStatus;
+    ExitSet = true;
   } else {
     // Build a status that is common to all the predecessors by intersecting
     // all the predecessor exit status values.
+    // Mask bits (which represent the Mode bits with a known value) can only be
+    // added by explicit SETREG instructions or the initial default value -
+    // the intersection process may remove Mask bits.
+    // If we find a predecessor that has not yet had an exit value determined
+    // (this can happen for example if a block is its own predecessor) we defer
+    // use of that value as the Mask will be all zero, and we will revisit this
+    // block again later (unless the only predecessor without an exit value is
+    // this block).
     MachineBasicBlock::pred_iterator P = MBB.pred_begin(), E = MBB.pred_end();
     MachineBasicBlock &PB = *(*P);
-    BlockInfo[ThisBlock]->Pred = BlockInfo[PB.getNumber()]->Exit;
+    unsigned PredBlock = PB.getNumber();
+    if ((ThisBlock == PredBlock) && (std::next(P) == E)) {
+      BlockInfo[ThisBlock]->Pred = DefaultStatus;
+      ExitSet = true;
+    } else if (BlockInfo[PredBlock]->ExitSet) {
+      BlockInfo[ThisBlock]->Pred = BlockInfo[PredBlock]->Exit;
+      ExitSet = true;
+    } else if (PredBlock != ThisBlock)
+      RevisitRequired = true;
 
     for (P = std::next(P); P != E; P = std::next(P)) {
       MachineBasicBlock *Pred = *P;
-      BlockInfo[ThisBlock]->Pred = BlockInfo[ThisBlock]->Pred.intersect(BlockInfo[Pred->getNumber()]->Exit);
+      unsigned PredBlock = Pred->getNumber();
+      if (BlockInfo[PredBlock]->ExitSet) {
+        if (BlockInfo[ThisBlock]->ExitSet) {
+          BlockInfo[ThisBlock]->Pred =
+              BlockInfo[ThisBlock]->Pred.intersect(BlockInfo[PredBlock]->Exit);
+        } else {
+          BlockInfo[ThisBlock]->Pred = BlockInfo[PredBlock]->Exit;
+        }
+        ExitSet = true;
+      } else if (PredBlock != ThisBlock)
+        RevisitRequired = true;
     }
   }
-  Status TmpStatus = BlockInfo[ThisBlock]->Pred.merge(BlockInfo[ThisBlock]->Change);
+  Status TmpStatus =
+      BlockInfo[ThisBlock]->Pred.merge(BlockInfo[ThisBlock]->Change);
   if (BlockInfo[ThisBlock]->Exit != TmpStatus) {
     BlockInfo[ThisBlock]->Exit = TmpStatus;
     // Add the successors to the work list so we can propagate the changed exit
@@ -354,6 +388,9 @@ void SIModeRegister::processBlockPhase2(MachineBasicBlock &MBB,
       Phase2List.push(&B);
     }
   }
+  BlockInfo[ThisBlock]->ExitSet = ExitSet;
+  if (RevisitRequired)
+    Phase2List.push(&MBB);
 }
 
 // In Phase 3 we revisit each block and if it has an insertion point defined we
@@ -361,10 +398,10 @@ void SIModeRegister::processBlockPhase2(MachineBasicBlock &MBB,
 // not we insert an appropriate setreg instruction to modify the Mode register.
 void SIModeRegister::processBlockPhase3(MachineBasicBlock &MBB,
                                         const SIInstrInfo *TII) {
-//  BlockData *BI = BlockInfo[MBB.getNumber()];
   unsigned ThisBlock = MBB.getNumber();
   if (!BlockInfo[ThisBlock]->Pred.isCompatible(BlockInfo[ThisBlock]->Require)) {
-    Status Delta = BlockInfo[ThisBlock]->Pred.delta(BlockInfo[ThisBlock]->Require);
+    Status Delta =
+        BlockInfo[ThisBlock]->Pred.delta(BlockInfo[ThisBlock]->Require);
     if (BlockInfo[ThisBlock]->FirstInsertionPoint)
       insertSetreg(MBB, BlockInfo[ThisBlock]->FirstInsertionPoint, TII, Delta);
     else
@@ -401,5 +438,5 @@ bool SIModeRegister::runOnMachineFunction(MachineFunction &MF) {
 
   BlockInfo.clear();
 
-  return NumSetregInserted > 0;
+  return Changed;
 }
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index 62df8ef488c60..1bb66907f9ce4 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -15,6 +15,7 @@
 #include "AMDGPUSubtarget.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "SIInstrInfo.h"
+#include "SIMachineFunctionInfo.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/Support/CommandLine.h"
 
@@ -198,6 +199,7 @@ bool SIPreEmitPeephole::runOnMachineFunction(MachineFunction &MF) {
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   TII = ST.getInstrInfo();
   TRI = &TII->getRegisterInfo();
+  MachineBasicBlock *EmptyMBBAtEnd = nullptr;
   bool Changed = false;
 
   for (MachineBasicBlock &MBB : MF) {
@@ -209,6 +211,28 @@ bool SIPreEmitPeephole::runOnMachineFunction(MachineFunction &MF) {
       case AMDGPU::S_CBRANCH_VCCNZ:
         Changed |= optimizeVccBranch(MI);
         continue;
+      case AMDGPU::SI_RETURN_TO_EPILOG:
+        // FIXME: This is not an optimization and should be
+        // moved somewhere else.
+        assert(!MF.getInfo<SIMachineFunctionInfo>()->returnsVoid());
+
+        // Graphics shaders returning non-void shouldn't contain S_ENDPGM,
+        // because external bytecode will be appended at the end.
+        if (&MBB != &MF.back() || &MI != &MBB.back()) {
+          // SI_RETURN_TO_EPILOG is not the last instruction. Add an empty block
+          // at the end and jump there.
+          if (!EmptyMBBAtEnd) {
+            EmptyMBBAtEnd = MF.CreateMachineBasicBlock();
+            MF.insert(MF.end(), EmptyMBBAtEnd);
+          }
+
+          MBB.addSuccessor(EmptyMBBAtEnd);
+          BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(AMDGPU::S_BRANCH))
+              .addMBB(EmptyMBBAtEnd);
+          MI.eraseFromParent();
+          MBBE = MBB.getFirstTerminator();
+        }
+        break;
       default:
         break;
       }
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 591545ff8b39f..5d6009ebf3843 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -987,7 +987,7 @@ bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI,
   MachineBasicBlock *MBB = MI->getParent();
   MachineFunction *MF = MBB->getParent();
   SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
-  DenseSet<unsigned> SGPRSpillVGPRDefinedSet;
+  DenseSet<Register> SGPRSpillVGPRDefinedSet;
 
   ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills
     = MFI->getSGPRToVGPRSpills(Index);
diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
index 6b2b54d896fa3..b1c73df269fb2 100644
--- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
+++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
@@ -762,18 +762,23 @@ void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg,
       if (State == StateWWM) {
         assert(SavedNonWWMReg);
         fromWWM(MBB, Before, SavedNonWWMReg);
+        LIS->createAndComputeVirtRegInterval(SavedNonWWMReg);
+        SavedNonWWMReg = 0;
         State = NonWWMState;
       }
 
       if (Needs == StateWWM) {
         NonWWMState = State;
+        assert(!SavedNonWWMReg);
         SavedNonWWMReg = MRI->createVirtualRegister(BoolRC);
         toWWM(MBB, Before, SavedNonWWMReg);
         State = StateWWM;
       } else {
         if (State == StateWQM && (Needs & StateExact) && !(Needs & StateWQM)) {
-          if (!WQMFromExec && (OutNeeds & StateWQM))
+          if (!WQMFromExec && (OutNeeds & StateWQM)) {
+            assert(!SavedWQMReg);
             SavedWQMReg = MRI->createVirtualRegister(BoolRC);
+          }
 
           toExact(MBB, Before, SavedWQMReg, LiveMaskReg);
           State = StateExact;
@@ -806,6 +811,8 @@ void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg,
       break;
     II = Next;
   }
+  assert(!SavedWQMReg);
+  assert(!SavedNonWWMReg);
 }
 
 void SIWholeQuadMode::lowerLiveMaskQueries(unsigned LiveMaskReg) {
@@ -898,10 +905,12 @@ bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
 
     if (GlobalFlags == StateWQM) {
       // For a shader that needs only WQM, we can just set it once.
-      BuildMI(Entry, EntryMI, DebugLoc(), TII->get(ST->isWave32() ?
-                AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64),
-              Exec)
-          .addReg(Exec);
+      auto MI = BuildMI(Entry, EntryMI, DebugLoc(),
+                        TII->get(ST->isWave32() ? AMDGPU::S_WQM_B32
+                                                : AMDGPU::S_WQM_B64),
+                        Exec)
+                    .addReg(Exec);
+      LIS->InsertMachineInstrInMaps(*MI);
 
       lowerCopyInstrs();
       // EntryMI may become invalid here
@@ -917,6 +926,9 @@ bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
   for (auto BII : Blocks)
     processBlock(*BII.first, LiveMaskReg, BII.first == &*MF.begin());
 
+  if (LiveMaskReg)
+    LIS->createAndComputeVirtRegInterval(LiveMaskReg);
+
   // Physical registers like SCC aren't tracked by default anyway, so just
   // removing the ranges we computed is the simplest option for maintaining
   // the analysis results.
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index dd6363b1439fe..9d7b25d552170 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -465,9 +465,19 @@ def S_MAX_U32 : SOP2_32 <"s_max_u32",
 } // End isCommutable = 1
 } // End Defs = [SCC]
 
+class SelectPat<SDPatternOperator select> : PatFrag <
+  (ops node:$src1, node:$src2),
+  (select SCC, $src1, $src2),
+  [{ return N->getOperand(0)->hasOneUse() && !N->isDivergent(); }]
+>;
 
 let Uses = [SCC] in {
-  def S_CSELECT_B32 : SOP2_32 <"s_cselect_b32">;
+  let AddedComplexity = 20 in {
+    def S_CSELECT_B32 : SOP2_32 <"s_cselect_b32",
+      [(set i32:$sdst, (SelectPat<select> i32:$src0, i32:$src1))]
+    >;
+  }
+
   def S_CSELECT_B64 : SOP2_64 <"s_cselect_b64">;
 } // End Uses = [SCC]
 
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 708c922da84e1..00e6d517bde58 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -1377,8 +1377,8 @@ Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
 // aligned if they are aligned to begin with. It also ensures that additional
 // offsets within the given alignment can be added to the resulting ImmOffset.
 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
-                      const GCNSubtarget *Subtarget, uint32_t Align) {
-  const uint32_t MaxImm = alignDown(4095, Align);
+                      const GCNSubtarget *Subtarget, Align Alignment) {
+  const uint32_t MaxImm = alignDown(4095, Alignment.value());
   uint32_t Overflow = 0;
 
   if (Imm > MaxImm) {
@@ -1396,10 +1396,10 @@ bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
       //
       // Atomic operations fail to work correctly when individual address
       // components are unaligned, even if their sum is aligned.
-      uint32_t High = (Imm + Align) & ~4095;
-      uint32_t Low = (Imm + Align) & 4095;
+      uint32_t High = (Imm + Alignment.value()) & ~4095;
+      uint32_t Low = (Imm + Alignment.value()) & 4095;
       Imm = Low;
-      Overflow = High - Align;
+      Overflow = High - Alignment.value();
     }
   }
 
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index 6766abbdc30f2..e71554575f6af 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -15,6 +15,7 @@
 #include "llvm/IR/CallingConv.h"
 #include "llvm/MC/MCInstrDesc.h"
 #include "llvm/Support/AMDHSAKernelDescriptor.h"
+#include "llvm/Support/Alignment.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/TargetParser.h"
@@ -692,7 +693,8 @@ Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
 bool isLegalSMRDImmOffset(const MCSubtargetInfo &ST, int64_t ByteOffset);
 
 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
-                      const GCNSubtarget *Subtarget, uint32_t Align = 4);
+                      const GCNSubtarget *Subtarget,
+                      Align Alignment = Align(4));
 
 /// \returns true if the intrinsic is divergent
 bool isIntrinsicSourceOfDivergence(unsigned IntrID);
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index c83442acf5e16..17f334f62a30b 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -248,13 +248,13 @@ defm V_LOG_F32 : VOP1Inst <"v_log_f32", VOP_F32_F32, flog2>;
 defm V_RCP_F32 : VOP1Inst <"v_rcp_f32", VOP_F32_F32, AMDGPUrcp>;
 defm V_RCP_IFLAG_F32 : VOP1Inst <"v_rcp_iflag_f32", VOP_F32_F32, AMDGPUrcp_iflag>;
 defm V_RSQ_F32 : VOP1Inst <"v_rsq_f32", VOP_F32_F32, AMDGPUrsq>;
-defm V_SQRT_F32 : VOP1Inst <"v_sqrt_f32", VOP_F32_F32, fsqrt>;
+defm V_SQRT_F32 : VOP1Inst <"v_sqrt_f32", VOP_F32_F32, any_amdgcn_sqrt>;
 } // End SchedRW = [WriteTrans32]
 
 let SchedRW = [WriteTrans64] in {
 defm V_RCP_F64 : VOP1Inst <"v_rcp_f64", VOP_F64_F64, AMDGPUrcp>;
 defm V_RSQ_F64 : VOP1Inst <"v_rsq_f64", VOP_F64_F64, AMDGPUrsq>;
-defm V_SQRT_F64 : VOP1Inst <"v_sqrt_f64", VOP_F64_F64, fsqrt>;
+defm V_SQRT_F64 : VOP1Inst <"v_sqrt_f64", VOP_F64_F64, any_amdgcn_sqrt>;
 } // End SchedRW = [WriteTrans64]
 
 let SchedRW = [WriteTrans32] in {
@@ -388,7 +388,7 @@ defm V_CVT_U16_F16 : VOP1Inst <"v_cvt_u16_f16", VOP_I16_F16, fp_to_uint>;
 defm V_CVT_I16_F16 : VOP1Inst <"v_cvt_i16_f16", VOP_I16_F16, fp_to_sint>;
 let SchedRW = [WriteTrans32] in {
 defm V_RCP_F16 : VOP1Inst <"v_rcp_f16", VOP_F16_F16, AMDGPUrcp>;
-defm V_SQRT_F16 : VOP1Inst <"v_sqrt_f16", VOP_F16_F16, fsqrt>;
+defm V_SQRT_F16 : VOP1Inst <"v_sqrt_f16", VOP_F16_F16, any_amdgcn_sqrt>;
 defm V_RSQ_F16 : VOP1Inst <"v_rsq_f16", VOP_F16_F16, AMDGPUrsq>;
 defm V_LOG_F16 : VOP1Inst <"v_log_f16", VOP_F16_F16, flog2>;
 defm V_EXP_F16 : VOP1Inst <"v_exp_f16", VOP_F16_F16, fexp2>;
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index 47238c6923410..fc457ad212d48 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -366,10 +366,13 @@ def VOPProfileMAI_F32_V4F16_X16 : VOPProfileMAI<VOP_V16F32_V4F16_V4F16_V16F32, A
 def VOPProfileMAI_F32_V4F16_X32 : VOPProfileMAI<VOP_V32F32_V4F16_V4F16_V32F32, AISrc_1024_b32, ADst_1024, AVSrc_64>;
 
 let Predicates = [HasMAIInsts] in {
+
+let isAsCheapAsAMove = 1, isReMaterializable = 1 in {
 def V_ACCVGPR_READ_B32  : VOP3Inst<"v_accvgpr_read_b32",  VOPProfileAccRead>;
 def V_ACCVGPR_WRITE_B32 : VOP3Inst<"v_accvgpr_write_b32", VOPProfileAccWrite> {
   let isMoveImm = 1;
 }
+}
 
 // FP32 denorm mode is respected, rounding mode is not. Exceptions are not supported.
 let isConvergent = 1, mayRaiseFPException = 0, ReadsModeReg = 1 in {
diff --git a/llvm/lib/Target/ARC/ARCAsmPrinter.cpp b/llvm/lib/Target/ARC/ARCAsmPrinter.cpp
index 03024e0ea09c1..025b920ff7b4e 100644
--- a/llvm/lib/Target/ARC/ARCAsmPrinter.cpp
+++ b/llvm/lib/Target/ARC/ARCAsmPrinter.cpp
@@ -66,7 +66,7 @@ void ARCAsmPrinter::emitInstruction(const MachineInstr *MI) {
 bool ARCAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
   // Functions are 4-byte aligned.
   MF.ensureAlignment(Align(4));
-  AsmPrinter::runOnMachineFunction(MF);
+  return AsmPrinter::runOnMachineFunction(MF);
 }
 
 // Force static initialization.
diff --git a/llvm/lib/Target/ARC/ARCFrameLowering.cpp b/llvm/lib/Target/ARC/ARCFrameLowering.cpp
index f0439357ceb8d..ead5931062628 100644
--- a/llvm/lib/Target/ARC/ARCFrameLowering.cpp
+++ b/llvm/lib/Target/ARC/ARCFrameLowering.cpp
@@ -439,8 +439,8 @@ void ARCFrameLowering::processFunctionBeforeFrameFinalized(
   LLVM_DEBUG(dbgs() << "Current stack size: " << MFI.getStackSize() << "\n");
   const TargetRegisterClass *RC = &ARC::GPR32RegClass;
   if (MFI.hasStackObjects()) {
-    int RegScavFI = MFI.CreateStackObject(
-        RegInfo->getSpillSize(*RC), RegInfo->getSpillAlignment(*RC), false);
+    int RegScavFI = MFI.CreateStackObject(RegInfo->getSpillSize(*RC),
+                                          RegInfo->getSpillAlign(*RC), false);
     RS->addScavengingFrameIndex(RegScavFI);
     LLVM_DEBUG(dbgs() << "Created scavenging index RegScavFI=" << RegScavFI
                       << "\n");
diff --git a/llvm/lib/Target/ARM/ARM.td b/llvm/lib/Target/ARM/ARM.td
index 2b4c3e13014ca..0468f7f1cf8e8 100644
--- a/llvm/lib/Target/ARM/ARM.td
+++ b/llvm/lib/Target/ARM/ARM.td
@@ -594,6 +594,12 @@ def ProcA75     : SubtargetFeature<"a75", "ARMProcFamily", "CortexA75",
                                    "Cortex-A75 ARM processors", []>;
 def ProcA76     : SubtargetFeature<"a76", "ARMProcFamily", "CortexA76",
                                    "Cortex-A76 ARM processors", []>;
+def ProcA77     : SubtargetFeature<"a77", "ARMProcFamily", "CortexA77",
+                                   "Cortex-A77 ARM processors", []>;
+def ProcA78     : SubtargetFeature<"cortex-a78", "ARMProcFamily", "CortexA78",
+                                   "Cortex-A78 ARM processors", []>;
+def ProcX1      : SubtargetFeature<"cortex-x1", "ARMProcFamily", "CortexX1",
+                                   "Cortex-X1 ARM processors", []>;
 
 def ProcKrait   : SubtargetFeature<"krait", "ARMProcFamily", "Krait",
                                    "Qualcomm Krait processors", []>;
@@ -1224,6 +1230,30 @@ def : ProcNoItin<"cortex-a76ae",                        [ARMv82a, ProcA76,
                                                          FeatureFullFP16,
                                                          FeatureDotProd]>;
 
+def : ProcNoItin<"cortex-a77",                          [ARMv82a, ProcA77,
+                                                         FeatureHWDivThumb,
+                                                         FeatureHWDivARM,
+                                                         FeatureCrypto,
+                                                         FeatureCRC,
+                                                         FeatureFullFP16,
+                                                         FeatureDotProd]>;
+
+def : ProcNoItin<"cortex-a78",                          [ARMv82a, ProcA78,
+                                                         FeatureHWDivThumb,
+                                                         FeatureHWDivARM,
+                                                         FeatureCrypto,
+                                                         FeatureCRC,
+                                                         FeatureFullFP16,
+                                                         FeatureDotProd]>;
+
+def : ProcNoItin<"cortex-x1",                           [ARMv82a, ProcX1,
+                                                         FeatureHWDivThumb,
+                                                         FeatureHWDivARM,
+                                                         FeatureCrypto,
+                                                         FeatureCRC,
+                                                         FeatureFullFP16,
+                                                         FeatureDotProd]>;
+
 def : ProcNoItin<"neoverse-n1",                         [ARMv82a,
                                                          FeatureHWDivThumb,
                                                          FeatureHWDivARM,
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 504d0cc4c08ab..4cc2b6bf7e7e0 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -2015,6 +2015,10 @@ bool ARMBaseInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
   if (MI.isTerminator() || MI.isPosition())
     return true;
 
+  // INLINEASM_BR can jump to another block
+  if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
+    return true;
+
   // Treat the start of the IT block as a scheduling boundary, but schedule
   // t2IT along with all instructions following it.
   // FIXME: This is a big hammer. But the alternative is to add all potential
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
index 615d72dab3c99..1a75b011ca59d 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
@@ -679,21 +679,21 @@ static inline bool isSubImmOpcode(int Opc) {
 static inline bool isMovRegOpcode(int Opc) {
   return Opc == ARM::MOVr || Opc == ARM::tMOVr || Opc == ARM::t2MOVr;
 }
-
 /// isValidCoprocessorNumber - decide whether an explicit coprocessor
 /// number is legal in generic instructions like CDP. The answer can
 /// vary with the subtarget.
 static inline bool isValidCoprocessorNumber(unsigned Num,
                                             const FeatureBitset& featureBits) {
+  // In Armv7 and Armv8-M CP10 and CP11 clash with VFP/NEON, however, the
+  // coprocessor is still valid for CDP/MCR/MRC and friends. Allowing it is
+  // useful for code which is shared with older architectures which do not know
+  // the new VFP/NEON mnemonics.
+
   // Armv8-A disallows everything *other* than 111x (CP14 and CP15).
   if (featureBits[ARM::HasV8Ops] && (Num & 0xE) != 0xE)
     return false;
 
-  // Armv7 disallows 101x (CP10 and CP11), which clash with VFP/NEON.
-  if (featureBits[ARM::HasV7Ops] && (Num & 0xE) == 0xA)
-    return false;
-
-  // Armv8.1-M also disallows 100x (CP8,CP9) and 111x (CP14,CP15)
+  // Armv8.1-M disallows 100x (CP8,CP9) and 111x (CP14,CP15)
   // which clash with MVE.
   if (featureBits[ARM::HasV8_1MMainlineOps] &&
       ((Num & 0xE) == 0x8 || (Num & 0xE) == 0xE))
diff --git a/llvm/lib/Target/ARM/ARMCallLowering.cpp b/llvm/lib/Target/ARM/ARMCallLowering.cpp
index 4fbb3b6993e4e..d860473011e77 100644
--- a/llvm/lib/Target/ARM/ARMCallLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMCallLowering.cpp
@@ -203,7 +203,7 @@ void ARMCallLowering::splitToValueTypes(const ArgInfo &OrigArg,
     // Even if there is no splitting to do, we still want to replace the
     // original type (e.g. pointer type -> integer).
     auto Flags = OrigArg.Flags[0];
-    Flags.setOrigAlign(Align(DL.getABITypeAlignment(OrigArg.Ty)));
+    Flags.setOrigAlign(DL.getABITypeAlign(OrigArg.Ty));
     SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx),
                            Flags, OrigArg.IsFixed);
     return;
@@ -215,7 +215,7 @@ void ARMCallLowering::splitToValueTypes(const ArgInfo &OrigArg,
     Type *SplitTy = SplitVT.getTypeForEVT(Ctx);
     auto Flags = OrigArg.Flags[0];
 
-    Flags.setOrigAlign(Align(DL.getABITypeAlignment(SplitTy)));
+    Flags.setOrigAlign(DL.getABITypeAlign(SplitTy));
 
     bool NeedsConsecutiveRegisters =
         TLI.functionArgumentNeedsConsecutiveRegisters(
diff --git a/llvm/lib/Target/ARM/ARMCallingConv.cpp b/llvm/lib/Target/ARM/ARMCallingConv.cpp
index d98edc268773e..67c822a5b6ef7 100644
--- a/llvm/lib/Target/ARM/ARMCallingConv.cpp
+++ b/llvm/lib/Target/ARM/ARMCallingConv.cpp
@@ -181,8 +181,8 @@ static bool CC_ARM_AAPCS_Custom_Aggregate(unsigned ValNo, MVT ValVT,
   // aggregate. Store the type's required alignment as extra info for later: in
   // the [N x i64] case all trace has been removed by the time we actually get
   // to do allocation.
-  PendingMembers.push_back(CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo,
-                                                   ArgFlags.getOrigAlign()));
+  PendingMembers.push_back(CCValAssign::getPending(
+      ValNo, ValVT, LocVT, LocInfo, ArgFlags.getNonZeroOrigAlign().value()));
 
   if (!ArgFlags.isInConsecutiveRegsLast())
     return true;
diff --git a/llvm/lib/Target/ARM/ARMCallingConv.td b/llvm/lib/Target/ARM/ARMCallingConv.td
index 65fc9d1cf7cdb..3517274e4c5c0 100644
--- a/llvm/lib/Target/ARM/ARMCallingConv.td
+++ b/llvm/lib/Target/ARM/ARMCallingConv.td
@@ -10,7 +10,7 @@
 
 /// CCIfAlign - Match of the original alignment of the arg
 class CCIfAlign<string Align, CCAction A>:
-  CCIf<!strconcat("ArgFlags.getOrigAlign() == ", Align), A>;
+  CCIf<!strconcat("ArgFlags.getNonZeroOrigAlign() == ", Align), A>;
 
 //===----------------------------------------------------------------------===//
 // ARM APCS Calling Convention
@@ -134,7 +134,7 @@ def CC_ARM_AAPCS_Common : CallingConv<[
   // i64 is 8-aligned i32 here, so we may need to eat R1 as a pad register
   // (and the same is true for f64 if VFP is not enabled)
   CCIfType<[i32], CCIfAlign<"8", CCAssignToRegWithShadow<[R0, R2], [R0, R1]>>>,
-  CCIfType<[i32], CCIf<"ArgFlags.getOrigAlign() != 8",
+  CCIfType<[i32], CCIf<"ArgFlags.getNonZeroOrigAlign() != Align(8)",
                        CCAssignToReg<[R0, R1, R2, R3]>>>,
 
   CCIfType<[i32], CCIfAlign<"8", CCAssignToStackWithShadow<4, 8, [R0, R1, R2, R3]>>>,
diff --git a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
index 49056d7830289..48622aae3cb41 100644
--- a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
@@ -85,12 +85,15 @@ namespace {
                                           const BitVector &ClearRegs);
     void CMSESaveClearFPRegs(MachineBasicBlock &MBB,
                              MachineBasicBlock::iterator MBBI, DebugLoc &DL,
+                             const LivePhysRegs &LiveRegs,
                              SmallVectorImpl<unsigned> &AvailableRegs);
     void CMSESaveClearFPRegsV8(MachineBasicBlock &MBB,
                                MachineBasicBlock::iterator MBBI, DebugLoc &DL,
+                               const LivePhysRegs &LiveRegs,
                                SmallVectorImpl<unsigned> &ScratchRegs);
     void CMSESaveClearFPRegsV81(MachineBasicBlock &MBB,
-                                MachineBasicBlock::iterator MBBI, DebugLoc &DL);
+                                MachineBasicBlock::iterator MBBI, DebugLoc &DL,
+                                const LivePhysRegs &LiveRegs);
     void CMSERestoreFPRegs(MachineBasicBlock &MBB,
                            MachineBasicBlock::iterator MBBI, DebugLoc &DL,
                            SmallVectorImpl<unsigned> &AvailableRegs);
@@ -1202,17 +1205,17 @@ ARMExpandPseudo::CMSEClearFPRegsV81(MachineBasicBlock &MBB,
 
 void ARMExpandPseudo::CMSESaveClearFPRegs(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,
-    SmallVectorImpl<unsigned> &ScratchRegs) {
+    const LivePhysRegs &LiveRegs, SmallVectorImpl<unsigned> &ScratchRegs) {
   if (STI->hasV8_1MMainlineOps())
-    CMSESaveClearFPRegsV81(MBB, MBBI, DL);
+    CMSESaveClearFPRegsV81(MBB, MBBI, DL, LiveRegs);
   else
-    CMSESaveClearFPRegsV8(MBB, MBBI, DL, ScratchRegs);
+    CMSESaveClearFPRegsV8(MBB, MBBI, DL, LiveRegs, ScratchRegs);
 }
 
 // Save and clear FP registers if present
 void ARMExpandPseudo::CMSESaveClearFPRegsV8(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,
-    SmallVectorImpl<unsigned> &ScratchRegs) {
+    const LivePhysRegs &LiveRegs, SmallVectorImpl<unsigned> &ScratchRegs) {
   if (!STI->hasFPRegs())
     return;
 
@@ -1271,7 +1274,11 @@ void ARMExpandPseudo::CMSESaveClearFPRegsV8(
   // Lazy store all fp registers to the stack
   MachineInstrBuilder VLSTM = BuildMI(MBB, MBBI, DL, TII->get(ARM::VLSTM))
                                   .addReg(ARM::SP)
-          .add(predOps(ARMCC::AL));
+                                  .add(predOps(ARMCC::AL));
+  for (auto R : {ARM::VPR, ARM::FPSCR, ARM::FPSCR_NZCV, ARM::Q0, ARM::Q1,
+                 ARM::Q2, ARM::Q3, ARM::Q4, ARM::Q5, ARM::Q6, ARM::Q7})
+    VLSTM.addReg(R, RegState::Implicit |
+                        (LiveRegs.contains(R) ? 0 : RegState::Undef));
 
   // Restore all arguments
   for (const auto &Regs : ClearedFPRegs) {
@@ -1343,7 +1350,8 @@ void ARMExpandPseudo::CMSESaveClearFPRegsV8(
 
 void ARMExpandPseudo::CMSESaveClearFPRegsV81(MachineBasicBlock &MBB,
                                              MachineBasicBlock::iterator MBBI,
-                                             DebugLoc &DL) {
+                                             DebugLoc &DL,
+                                             const LivePhysRegs &LiveRegs) {
   BitVector ClearRegs(32, true);
   bool DefFP = determineFPRegsToClear(*MBBI, ClearRegs);
 
@@ -1358,9 +1366,13 @@ void ARMExpandPseudo::CMSESaveClearFPRegsV81(MachineBasicBlock &MBB,
         .add(predOps(ARMCC::AL));
 
     // Lazy store all FP registers to the stack
-    BuildMI(MBB, MBBI, DL, TII->get(ARM::VLSTM))
-        .addReg(ARM::SP)
-        .add(predOps(ARMCC::AL));
+    MachineInstrBuilder VLSTM = BuildMI(MBB, MBBI, DL, TII->get(ARM::VLSTM))
+                                    .addReg(ARM::SP)
+                                    .add(predOps(ARMCC::AL));
+    for (auto R : {ARM::VPR, ARM::FPSCR, ARM::FPSCR_NZCV, ARM::Q0, ARM::Q1,
+                   ARM::Q2, ARM::Q3, ARM::Q4, ARM::Q5, ARM::Q6, ARM::Q7})
+      VLSTM.addReg(R, RegState::Implicit |
+                          (LiveRegs.contains(R) ? 0 : RegState::Undef));
   } else {
     // Push all the callee-saved registers (s16-s31).
     MachineInstrBuilder VPUSH =
@@ -1754,13 +1766,15 @@ bool ARMExpandPseudo::ExpandCMP_SWAP_64(MachineBasicBlock &MBB,
 static void CMSEPushCalleeSaves(const TargetInstrInfo &TII,
                                 MachineBasicBlock &MBB,
                                 MachineBasicBlock::iterator MBBI, int JumpReg,
-                                bool Thumb1Only) {
+                                const LivePhysRegs &LiveRegs, bool Thumb1Only) {
   const DebugLoc &DL = MBBI->getDebugLoc();
   if (Thumb1Only) { // push Lo and Hi regs separately
     MachineInstrBuilder PushMIB =
         BuildMI(MBB, MBBI, DL, TII.get(ARM::tPUSH)).add(predOps(ARMCC::AL));
-    for (int Reg = ARM::R4; Reg < ARM::R8; ++Reg)
-      PushMIB.addReg(Reg, Reg != JumpReg ? RegState::Undef : 0);
+    for (int Reg = ARM::R4; Reg < ARM::R8; ++Reg) {
+      PushMIB.addReg(
+          Reg, Reg == JumpReg || LiveRegs.contains(Reg) ? 0 : RegState::Undef);
+    }
 
     // Thumb1 can only tPUSH low regs, so we copy the high regs to the low
     // regs that we just saved and push the low regs again, taking care to
@@ -1773,7 +1787,7 @@ static void CMSEPushCalleeSaves(const TargetInstrInfo &TII,
       if (JumpReg == LoReg)
         continue;
       BuildMI(MBB, MBBI, DL, TII.get(ARM::tMOVr), LoReg)
-          .addReg(HiReg, RegState::Undef)
+          .addReg(HiReg, LiveRegs.contains(HiReg) ? 0 : RegState::Undef)
           .add(predOps(ARMCC::AL));
       --HiReg;
     }
@@ -1791,19 +1805,21 @@ static void CMSEPushCalleeSaves(const TargetInstrInfo &TII,
     if (JumpReg >= ARM::R4 && JumpReg <= ARM::R7) {
       int LoReg = JumpReg == ARM::R4 ? ARM::R5 : ARM::R4;
       BuildMI(MBB, MBBI, DL, TII.get(ARM::tMOVr), LoReg)
-          .addReg(ARM::R8)
+          .addReg(ARM::R8, LiveRegs.contains(ARM::R8) ? 0 : RegState::Undef)
           .add(predOps(ARMCC::AL));
       BuildMI(MBB, MBBI, DL, TII.get(ARM::tPUSH))
           .add(predOps(ARMCC::AL))
-          .addReg(LoReg);
+          .addReg(LoReg, RegState::Kill);
     }
   } else { // push Lo and Hi registers with a single instruction
     MachineInstrBuilder PushMIB =
         BuildMI(MBB, MBBI, DL, TII.get(ARM::t2STMDB_UPD), ARM::SP)
             .addReg(ARM::SP)
             .add(predOps(ARMCC::AL));
-    for (int Reg = ARM::R4; Reg < ARM::R12; ++Reg)
-      PushMIB.addReg(Reg, Reg != JumpReg ? RegState::Undef : 0);
+    for (int Reg = ARM::R4; Reg < ARM::R12; ++Reg) {
+      PushMIB.addReg(
+          Reg, Reg == JumpReg || LiveRegs.contains(Reg) ? 0 : RegState::Undef);
+    }
   }
 }
 
@@ -1934,7 +1950,18 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
     case ARM::tBLXNS_CALL: {
       DebugLoc DL = MBBI->getDebugLoc();
       unsigned JumpReg = MBBI->getOperand(0).getReg();
-      CMSEPushCalleeSaves(*TII, MBB, MBBI, JumpReg,
+
+      // Figure out which registers are live at the point immediately before the
+      // call. When we indiscriminately push a set of registers, the live
+      // registers are added as ordinary use operands, whereas dead registers
+      // are "undef".
+      LivePhysRegs LiveRegs(*TRI);
+      LiveRegs.addLiveOuts(MBB);
+      for (const MachineInstr &MI : make_range(MBB.rbegin(), MBBI.getReverse()))
+        LiveRegs.stepBackward(MI);
+      LiveRegs.stepBackward(*MBBI);
+
+      CMSEPushCalleeSaves(*TII, MBB, MBBI, JumpReg, LiveRegs,
                           AFI->isThumb1OnlyFunction());
 
       SmallVector<unsigned, 16> ClearRegs;
@@ -1971,7 +1998,7 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
             .add(predOps(ARMCC::AL));
       }
 
-      CMSESaveClearFPRegs(MBB, MBBI, DL,
+      CMSESaveClearFPRegs(MBB, MBBI, DL, LiveRegs,
                           ClearRegs); // save+clear FP regs with ClearRegs
       CMSEClearGPRegs(MBB, MBBI, DL, ClearRegs, JumpReg);
 
diff --git a/llvm/lib/Target/ARM/ARMFastISel.cpp b/llvm/lib/Target/ARM/ARMFastISel.cpp
index f47e9ff9f3b65..4bfca8a803ca4 100644
--- a/llvm/lib/Target/ARM/ARMFastISel.cpp
+++ b/llvm/lib/Target/ARM/ARMFastISel.cpp
@@ -2236,7 +2236,7 @@ bool ARMFastISel::ARMEmitLibcall(const Instruction *I, RTLIB::Libcall Call) {
     if (!isTypeLegal(ArgTy, ArgVT)) return false;
 
     ISD::ArgFlagsTy Flags;
-    Flags.setOrigAlign(Align(DL.getABITypeAlignment(ArgTy)));
+    Flags.setOrigAlign(DL.getABITypeAlign(ArgTy));
 
     Args.push_back(Op);
     ArgRegs.push_back(Arg);
@@ -2367,7 +2367,7 @@ bool ARMFastISel::SelectCall(const Instruction *I,
     if (!Arg.isValid())
       return false;
 
-    Flags.setOrigAlign(Align(DL.getABITypeAlignment(ArgTy)));
+    Flags.setOrigAlign(DL.getABITypeAlign(ArgTy));
 
     Args.push_back(*ArgI);
     ArgRegs.push_back(Arg);
diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
index 95696ea619603..8a8f3237bb6f1 100644
--- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
@@ -2144,8 +2144,9 @@ void ARMFrameLowering::determineCalleeSaves(MachineFunction &MF,
         LLVM_DEBUG(dbgs() << "Reserving emergency spill slot\n");
         const TargetRegisterClass &RC = ARM::GPRRegClass;
         unsigned Size = TRI->getSpillSize(RC);
-        unsigned Align = TRI->getSpillAlignment(RC);
-        RS->addScavengingFrameIndex(MFI.CreateStackObject(Size, Align, false));
+        Align Alignment = TRI->getSpillAlign(RC);
+        RS->addScavengingFrameIndex(
+            MFI.CreateStackObject(Size, Alignment, false));
       }
     }
   }
diff --git a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
index bc990690515df..2a9a31dab74fa 100644
--- a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
+++ b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
@@ -1726,7 +1726,7 @@ bool ARMDAGToDAGISel::tryMVEIndexedLoad(SDNode *N) {
   EVT LoadedVT;
   unsigned Opcode = 0;
   bool isSExtLd, isPre;
-  unsigned Align;
+  Align Alignment;
   ARMVCC::VPTCodes Pred;
   SDValue PredReg;
   SDValue Chain, Base, Offset;
@@ -1742,7 +1742,7 @@ bool ARMDAGToDAGISel::tryMVEIndexedLoad(SDNode *N) {
     Chain = LD->getChain();
     Base = LD->getBasePtr();
     Offset = LD->getOffset();
-    Align = LD->getAlignment();
+    Alignment = LD->getAlign();
     isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
     isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
     Pred = ARMVCC::None;
@@ -1758,7 +1758,7 @@ bool ARMDAGToDAGISel::tryMVEIndexedLoad(SDNode *N) {
     Chain = LD->getChain();
     Base = LD->getBasePtr();
     Offset = LD->getOffset();
-    Align = LD->getAlignment();
+    Alignment = LD->getAlign();
     isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
     isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
     Pred = ARMVCC::Then;
@@ -1772,7 +1772,7 @@ bool ARMDAGToDAGISel::tryMVEIndexedLoad(SDNode *N) {
   bool CanChangeType = Subtarget->isLittle() && !isa<MaskedLoadSDNode>(N);
 
   SDValue NewOffset;
-  if (Align >= 2 && LoadedVT == MVT::v4i16 &&
+  if (Alignment >= Align(2) && LoadedVT == MVT::v4i16 &&
       SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 1)) {
     if (isSExtLd)
       Opcode = isPre ? ARM::MVE_VLDRHS32_pre : ARM::MVE_VLDRHS32_post;
@@ -1790,12 +1790,12 @@ bool ARMDAGToDAGISel::tryMVEIndexedLoad(SDNode *N) {
       Opcode = isPre ? ARM::MVE_VLDRBS32_pre : ARM::MVE_VLDRBS32_post;
     else
       Opcode = isPre ? ARM::MVE_VLDRBU32_pre : ARM::MVE_VLDRBU32_post;
-  } else if (Align >= 4 &&
+  } else if (Alignment >= Align(4) &&
              (CanChangeType || LoadedVT == MVT::v4i32 ||
               LoadedVT == MVT::v4f32) &&
              SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 2))
     Opcode = isPre ? ARM::MVE_VLDRWU32_pre : ARM::MVE_VLDRWU32_post;
-  else if (Align >= 2 &&
+  else if (Alignment >= Align(2) &&
            (CanChangeType || LoadedVT == MVT::v8i16 ||
             LoadedVT == MVT::v8f16) &&
            SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 1))
@@ -4743,6 +4743,29 @@ void ARMDAGToDAGISel::Select(SDNode *N) {
     default:
       break;
 
+    // Scalar f32 -> bf16
+    case Intrinsic::arm_neon_vcvtbfp2bf: {
+      SDLoc dl(N);
+      const SDValue &Src = N->getOperand(1);
+      llvm::EVT DestTy = N->getValueType(0);
+      SDValue Pred = getAL(CurDAG, dl);
+      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
+      SDValue Ops[] = { Src, Src, Pred, Reg0 };
+      CurDAG->SelectNodeTo(N, ARM::BF16_VCVTB, DestTy, Ops);
+      return;
+    }
+
+    // Vector v4f32 -> v4bf16
+    case Intrinsic::arm_neon_vcvtfp2bf: {
+      SDLoc dl(N);
+      const SDValue &Src = N->getOperand(1);
+      SDValue Pred = getAL(CurDAG, dl);
+      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
+      SDValue Ops[] = { Src, Pred, Reg0 };
+      CurDAG->SelectNodeTo(N, ARM::BF16_VCVT, MVT::v4bf16, Ops);
+      return;
+    }
+
     case Intrinsic::arm_mve_urshrl:
       SelectMVE_LongShift(N, ARM::MVE_URSHRL, true, false);
       return;
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index a646f63e0839b..287e2e60e572c 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -210,6 +210,8 @@ void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT,
   setOperationAction(ISD::SREM, VT, Expand);
   setOperationAction(ISD::UREM, VT, Expand);
   setOperationAction(ISD::FREM, VT, Expand);
+  setOperationAction(ISD::SDIVREM, VT, Expand);
+  setOperationAction(ISD::UDIVREM, VT, Expand);
 
   if (!VT.isFloatingPoint() &&
       VT != MVT::v2i64 && VT != MVT::v1i64)
@@ -284,6 +286,8 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
     setOperationAction(ISD::SDIV, VT, Expand);
     setOperationAction(ISD::UREM, VT, Expand);
     setOperationAction(ISD::SREM, VT, Expand);
+    setOperationAction(ISD::UDIVREM, VT, Expand);
+    setOperationAction(ISD::SDIVREM, VT, Expand);
     setOperationAction(ISD::CTPOP, VT, Expand);
 
     // Vector reductions
@@ -292,6 +296,10 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
     setOperationAction(ISD::VECREDUCE_UMAX, VT, Legal);
     setOperationAction(ISD::VECREDUCE_SMIN, VT, Legal);
     setOperationAction(ISD::VECREDUCE_UMIN, VT, Legal);
+    setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
+    setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
+    setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
+    setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
 
     if (!HasMVEFP) {
       setOperationAction(ISD::SINT_TO_FP, VT, Expand);
@@ -341,6 +349,10 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
       setOperationAction(ISD::FMINNUM, VT, Legal);
       setOperationAction(ISD::FMAXNUM, VT, Legal);
       setOperationAction(ISD::FROUND, VT, Legal);
+      setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
+      setOperationAction(ISD::VECREDUCE_FMUL, VT, Custom);
+      setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
+      setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
 
       // No native support for these.
       setOperationAction(ISD::FDIV, VT, Expand);
@@ -358,6 +370,17 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
     }
   }
 
+  // Custom Expand smaller than legal vector reductions to prevent false zero
+  // items being added.
+  setOperationAction(ISD::VECREDUCE_FADD, MVT::v4f16, Custom);
+  setOperationAction(ISD::VECREDUCE_FMUL, MVT::v4f16, Custom);
+  setOperationAction(ISD::VECREDUCE_FMIN, MVT::v4f16, Custom);
+  setOperationAction(ISD::VECREDUCE_FMAX, MVT::v4f16, Custom);
+  setOperationAction(ISD::VECREDUCE_FADD, MVT::v2f16, Custom);
+  setOperationAction(ISD::VECREDUCE_FMUL, MVT::v2f16, Custom);
+  setOperationAction(ISD::VECREDUCE_FMIN, MVT::v2f16, Custom);
+  setOperationAction(ISD::VECREDUCE_FMAX, MVT::v2f16, Custom);
+
   // We 'support' these types up to bitcast/load/store level, regardless of
   // MVE integer-only / float support. Only doing FP data processing on the FP
   // vector types is inhibited at integer-only level.
@@ -721,10 +744,13 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
 
     setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
     setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
+  }
 
-    // For the time being bfloat is only supported when fullfp16 is present.
-    if (Subtarget->hasBF16())
-      addRegisterClass(MVT::bf16, &ARM::HPRRegClass);
+  if (Subtarget->hasBF16()) {
+    addRegisterClass(MVT::bf16, &ARM::HPRRegClass);
+    setAllExpand(MVT::bf16);
+    if (!Subtarget->hasFullFP16())
+      setOperationAction(ISD::BITCAST, MVT::bf16, Custom);
   }
 
   for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
@@ -960,6 +986,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
     setTargetDAGCombine(ISD::UMIN);
     setTargetDAGCombine(ISD::SMAX);
     setTargetDAGCombine(ISD::UMAX);
+    setTargetDAGCombine(ISD::FP_EXTEND);
   }
 
   if (!Subtarget->hasFP64()) {
@@ -1685,6 +1712,8 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case ARMISD::VMOVN:         return "ARMISD::VMOVN";
   case ARMISD::VQMOVNs:       return "ARMISD::VQMOVNs";
   case ARMISD::VQMOVNu:       return "ARMISD::VQMOVNu";
+  case ARMISD::VCVTN:         return "ARMISD::VCVTN";
+  case ARMISD::VCVTL:         return "ARMISD::VCVTL";
   case ARMISD::VMULLs:        return "ARMISD::VMULLs";
   case ARMISD::VMULLu:        return "ARMISD::VMULLu";
   case ARMISD::VADDVs:        return "ARMISD::VADDVs";
@@ -2006,6 +2035,35 @@ CCAssignFn *ARMTargetLowering::CCAssignFnForNode(CallingConv::ID CC,
   }
 }
 
+SDValue ARMTargetLowering::MoveToHPR(const SDLoc &dl, SelectionDAG &DAG,
+                                     MVT LocVT, MVT ValVT, SDValue Val) const {
+  Val = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocVT.getSizeInBits()),
+                    Val);
+  if (Subtarget->hasFullFP16()) {
+    Val = DAG.getNode(ARMISD::VMOVhr, dl, ValVT, Val);
+  } else {
+    Val = DAG.getNode(ISD::TRUNCATE, dl,
+                      MVT::getIntegerVT(ValVT.getSizeInBits()), Val);
+    Val = DAG.getNode(ISD::BITCAST, dl, ValVT, Val);
+  }
+  return Val;
+}
+
+SDValue ARMTargetLowering::MoveFromHPR(const SDLoc &dl, SelectionDAG &DAG,
+                                       MVT LocVT, MVT ValVT,
+                                       SDValue Val) const {
+  if (Subtarget->hasFullFP16()) {
+    Val = DAG.getNode(ARMISD::VMOVrh, dl,
+                      MVT::getIntegerVT(LocVT.getSizeInBits()), Val);
+  } else {
+    Val = DAG.getNode(ISD::BITCAST, dl,
+                      MVT::getIntegerVT(ValVT.getSizeInBits()), Val);
+    Val = DAG.getNode(ISD::ZERO_EXTEND, dl,
+                      MVT::getIntegerVT(LocVT.getSizeInBits()), Val);
+  }
+  return DAG.getNode(ISD::BITCAST, dl, LocVT, Val);
+}
+
 /// LowerCallResult - Lower the result values of a call into the
 /// appropriate copies out of appropriate physical registers.
 SDValue ARMTargetLowering::LowerCallResult(
@@ -2087,13 +2145,8 @@ SDValue ARMTargetLowering::LowerCallResult(
     // had been copied to the LSBs of a 32-bit register.
     // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
     if (VA.needsCustom() &&
-        (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16)) {
-      assert(Subtarget->hasFullFP16() &&
-             "Lowering half precision fp return without full fp16 support");
-      Val = DAG.getNode(ISD::BITCAST, dl,
-                        MVT::getIntegerVT(VA.getLocVT().getSizeInBits()), Val);
-      Val = DAG.getNode(ARMISD::VMOVhr, dl, VA.getValVT(), Val);
-    }
+        (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
+      Val = MoveToHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Val);
 
     InVals.push_back(Val);
   }
@@ -2268,11 +2321,7 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
     if (VA.needsCustom() &&
         (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16)) {
-      assert(Subtarget->hasFullFP16() &&
-             "Lowering half precision fp argument without full fp16 support");
-      Arg = DAG.getNode(ARMISD::VMOVrh, dl,
-                        MVT::getIntegerVT(VA.getLocVT().getSizeInBits()), Arg);
-      Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
+      Arg = MoveFromHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Arg);
     } else {
       // f16 arguments could have been extended prior to argument lowering.
       // Mask them arguments if this is a CMSE nonsecure call.
@@ -2956,12 +3005,7 @@ ARMTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
     auto RetVT = Outs[realRVLocIdx].ArgVT;
     if (AFI->isCmseNSEntryFunction() && (RetVT == MVT::f16)) {
       if (VA.needsCustom() && VA.getValVT() == MVT::f16) {
-        assert(Subtarget->hasFullFP16() &&
-               "Lowering f16 type argument without full fp16 support");
-        Arg =
-            DAG.getNode(ARMISD::VMOVrh, dl,
-                        MVT::getIntegerVT(VA.getLocVT().getSizeInBits()), Arg);
-        Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
+        Arg = MoveFromHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Arg);
       } else {
         auto LocBits = VA.getLocVT().getSizeInBits();
         auto MaskValue = APInt::getLowBitsSet(LocBits, RetVT.getSizeInBits());
@@ -3546,7 +3590,7 @@ static SDValue promoteToConstantPool(const ARMTargetLowering *TLI,
   // that are strings for simplicity.
   auto *CDAInit = dyn_cast<ConstantDataArray>(Init);
   unsigned Size = DAG.getDataLayout().getTypeAllocSize(Init->getType());
-  unsigned PrefAlign = DAG.getDataLayout().getPreferredAlignment(GVar);
+  Align PrefAlign = DAG.getDataLayout().getPreferredAlign(GVar);
   unsigned RequiredPadding = 4 - (Size % 4);
   bool PaddingPossible =
     RequiredPadding == 4 || (CDAInit && CDAInit->isString());
@@ -4331,14 +4375,8 @@ SDValue ARMTargetLowering::LowerFormalArguments(
       // had been copied to the LSBs of a 32-bit register.
       // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
       if (VA.needsCustom() &&
-          (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16)) {
-        assert(Subtarget->hasFullFP16() &&
-               "Lowering half precision fp argument without full fp16 support");
-        ArgValue = DAG.getNode(ISD::BITCAST, dl,
-                               MVT::getIntegerVT(VA.getLocVT().getSizeInBits()),
-                               ArgValue);
-        ArgValue = DAG.getNode(ARMISD::VMOVhr, dl, VA.getValVT(), ArgValue);
-      }
+          (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
+        ArgValue = MoveToHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), ArgValue);
 
       InVals.push_back(ArgValue);
     } else { // VA.isRegLoc()
@@ -5918,8 +5956,8 @@ static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC,
 /// use a VMOVDRR or VMOVRRD node.  This should not be done when the non-i64
 /// operand type is illegal (e.g., v2f32 for a target that doesn't support
 /// vectors), since the legalizer won't know what to do with that.
-static SDValue ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
-                             const ARMSubtarget *Subtarget) {
+SDValue ARMTargetLowering::ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
+                                         const ARMSubtarget *Subtarget) const {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDLoc dl(N);
   SDValue Op = N->getOperand(0);
@@ -5929,21 +5967,16 @@ static SDValue ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
   EVT SrcVT = Op.getValueType();
   EVT DstVT = N->getValueType(0);
 
-  if (SrcVT == MVT::i16 && (DstVT == MVT::f16 || DstVT == MVT::bf16)) {
-    if (!Subtarget->hasFullFP16())
-      return SDValue();
-    // (b)f16 bitcast i16 -> VMOVhr
-    return DAG.getNode(ARMISD::VMOVhr, SDLoc(N), DstVT,
-                       DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), MVT::i32, Op));
-  }
+  if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
+      (DstVT == MVT::f16 || DstVT == MVT::bf16))
+    return MoveToHPR(SDLoc(N), DAG, MVT::i32, DstVT.getSimpleVT(),
+                     DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), MVT::i32, Op));
 
-  if ((SrcVT == MVT::f16 || SrcVT == MVT::bf16) && DstVT == MVT::i16) {
-    if (!Subtarget->hasFullFP16())
-      return SDValue();
-    // i16 bitcast (b)f16 -> VMOVrh
-    return DAG.getNode(ISD::TRUNCATE, SDLoc(N), MVT::i16,
-                       DAG.getNode(ARMISD::VMOVrh, SDLoc(N), MVT::i32, Op));
-  }
+  if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
+      (SrcVT == MVT::f16 || SrcVT == MVT::bf16))
+    return DAG.getNode(
+        ISD::TRUNCATE, SDLoc(N), DstVT,
+        MoveFromHPR(SDLoc(N), DAG, MVT::i32, SrcVT.getSimpleVT(), Op));
 
   if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
     return SDValue();
@@ -7232,6 +7265,104 @@ static bool isVMOVNMask(ArrayRef<int> M, EVT VT, bool Top) {
   return true;
 }
 
+// Reconstruct an MVE VCVT from a BuildVector of scalar fptrunc, all extracted
+// from a pair of inputs. For example:
+// BUILDVECTOR(FP_ROUND(EXTRACT_ELT(X, 0),
+//             FP_ROUND(EXTRACT_ELT(Y, 0),
+//             FP_ROUND(EXTRACT_ELT(X, 1),
+//             FP_ROUND(EXTRACT_ELT(Y, 1), ...)
+static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG,
+                                         const ARMSubtarget *ST) {
+  assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
+  if (!ST->hasMVEFloatOps())
+    return SDValue();
+
+  SDLoc dl(BV);
+  EVT VT = BV.getValueType();
+  if (VT != MVT::v8f16)
+    return SDValue();
+
+  // We are looking for a buildvector of fptrunc elements, where all the
+  // elements are interleavingly extracted from two sources. Check the first two
+  // items are valid enough and extract some info from them (they are checked
+  // properly in the loop below).
+  if (BV.getOperand(0).getOpcode() != ISD::FP_ROUND ||
+      BV.getOperand(0).getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
+      BV.getOperand(0).getOperand(0).getConstantOperandVal(1) != 0)
+    return SDValue();
+  if (BV.getOperand(1).getOpcode() != ISD::FP_ROUND ||
+      BV.getOperand(1).getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
+      BV.getOperand(1).getOperand(0).getConstantOperandVal(1) != 0)
+    return SDValue();
+  SDValue Op0 = BV.getOperand(0).getOperand(0).getOperand(0);
+  SDValue Op1 = BV.getOperand(1).getOperand(0).getOperand(0);
+  if (Op0.getValueType() != MVT::v4f32 || Op1.getValueType() != MVT::v4f32)
+    return SDValue();
+
+  // Check all the values in the BuildVector line up with our expectations.
+  for (unsigned i = 1; i < 4; i++) {
+    auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
+      return Trunc.getOpcode() == ISD::FP_ROUND &&
+             Trunc.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+             Trunc.getOperand(0).getOperand(0) == Op &&
+             Trunc.getOperand(0).getConstantOperandVal(1) == Idx;
+    };
+    if (!Check(BV.getOperand(i * 2 + 0), Op0, i))
+      return SDValue();
+    if (!Check(BV.getOperand(i * 2 + 1), Op1, i))
+      return SDValue();
+  }
+
+  SDValue N1 = DAG.getNode(ARMISD::VCVTN, dl, VT, DAG.getUNDEF(VT), Op0,
+                           DAG.getConstant(0, dl, MVT::i32));
+  return DAG.getNode(ARMISD::VCVTN, dl, VT, N1, Op1,
+                     DAG.getConstant(1, dl, MVT::i32));
+}
+
+// Reconstruct an MVE VCVT from a BuildVector of scalar fpext, all extracted
+// from a single input on alternating lanes. For example:
+// BUILDVECTOR(FP_ROUND(EXTRACT_ELT(X, 0),
+//             FP_ROUND(EXTRACT_ELT(X, 2),
+//             FP_ROUND(EXTRACT_ELT(X, 4), ...)
+static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG,
+                                       const ARMSubtarget *ST) {
+  assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
+  if (!ST->hasMVEFloatOps())
+    return SDValue();
+
+  SDLoc dl(BV);
+  EVT VT = BV.getValueType();
+  if (VT != MVT::v4f32)
+    return SDValue();
+
+  // We are looking for a buildvector of fptext elements, where all the
+  // elements are alternating lanes from a single source. For example <0,2,4,6>
+  // or <1,3,5,7>. Check the first two items are valid enough and extract some
+  // info from them (they are checked properly in the loop below).
+  if (BV.getOperand(0).getOpcode() != ISD::FP_EXTEND ||
+      BV.getOperand(0).getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
+    return SDValue();
+  SDValue Op0 = BV.getOperand(0).getOperand(0).getOperand(0);
+  int Offset = BV.getOperand(0).getOperand(0).getConstantOperandVal(1);
+  if (Op0.getValueType() != MVT::v8f16 || (Offset != 0 && Offset != 1))
+    return SDValue();
+
+  // Check all the values in the BuildVector line up with our expectations.
+  for (unsigned i = 1; i < 4; i++) {
+    auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
+      return Trunc.getOpcode() == ISD::FP_EXTEND &&
+             Trunc.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+             Trunc.getOperand(0).getOperand(0) == Op &&
+             Trunc.getOperand(0).getConstantOperandVal(1) == Idx;
+    };
+    if (!Check(BV.getOperand(i), Op0, 2 * i + Offset))
+      return SDValue();
+  }
+
+  return DAG.getNode(ARMISD::VCVTL, dl, VT, Op0,
+                     DAG.getConstant(Offset, dl, MVT::i32));
+}
+
 // If N is an integer constant that can be moved into a register in one
 // instruction, return an SDValue of such a constant (will become a MOV
 // instruction).  Otherwise return null.
@@ -7487,12 +7618,19 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
   if (isConstant)
     return SDValue();
 
-  // Empirical tests suggest this is rarely worth it for vectors of length <= 2.
-  if (NumElts >= 4) {
-    SDValue shuffle = ReconstructShuffle(Op, DAG);
-    if (shuffle != SDValue())
+  // Reconstruct the BUILDVECTOR to one of the legal shuffles (such as vext and
+  // vmovn). Empirical tests suggest this is rarely worth it for vectors of
+  // length <= 2.
+  if (NumElts >= 4)
+    if (SDValue shuffle = ReconstructShuffle(Op, DAG))
       return shuffle;
-  }
+
+  // Attempt to turn a buildvector of scalar fptrunc's or fpext's back into
+  // VCVT's
+  if (SDValue VCVT = LowerBuildVectorOfFPTrunc(Op, DAG, Subtarget))
+    return VCVT;
+  if (SDValue VCVT = LowerBuildVectorOfFPExt(Op, DAG, Subtarget))
+    return VCVT;
 
   if (ST->hasNEON() && VT.is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
     // If we haven't found an efficient lowering, try splitting a 128-bit vector
@@ -9058,7 +9196,7 @@ SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
   if (ShouldUseSRet) {
     // Create stack object for sret.
     const uint64_t ByteSize = DL.getTypeAllocSize(RetTy);
-    const unsigned StackAlign = DL.getPrefTypeAlignment(RetTy);
+    const Align StackAlign = DL.getPrefTypeAlign(RetTy);
     int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false);
     SRet = DAG.getFrameIndex(FrameIdx, TLI.getPointerTy(DL));
 
@@ -9379,6 +9517,79 @@ static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) {
   return DAG.getMergeValues({Combo, NewLoad.getValue(1)}, dl);
 }
 
+static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG,
+                              const ARMSubtarget *ST) {
+  if (!ST->hasMVEIntegerOps())
+    return SDValue();
+
+  SDLoc dl(Op);
+  unsigned BaseOpcode = 0;
+  switch (Op->getOpcode()) {
+  default: llvm_unreachable("Expected VECREDUCE opcode");
+  case ISD::VECREDUCE_FADD: BaseOpcode = ISD::FADD; break;
+  case ISD::VECREDUCE_FMUL: BaseOpcode = ISD::FMUL; break;
+  case ISD::VECREDUCE_MUL:  BaseOpcode = ISD::MUL; break;
+  case ISD::VECREDUCE_AND:  BaseOpcode = ISD::AND; break;
+  case ISD::VECREDUCE_OR:   BaseOpcode = ISD::OR; break;
+  case ISD::VECREDUCE_XOR:  BaseOpcode = ISD::XOR; break;
+  case ISD::VECREDUCE_FMAX: BaseOpcode = ISD::FMAXNUM; break;
+  case ISD::VECREDUCE_FMIN: BaseOpcode = ISD::FMINNUM; break;
+  }
+
+  SDValue Op0 = Op->getOperand(0);
+  EVT VT = Op0.getValueType();
+  EVT EltVT = VT.getVectorElementType();
+  unsigned NumElts = VT.getVectorNumElements();
+  unsigned NumActiveLanes = NumElts;
+
+  assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
+          NumActiveLanes == 2) &&
+         "Only expected a power 2 vector size");
+
+  // Use Mul(X, Rev(X)) until 4 items remain. Going down to 4 vector elements
+  // allows us to easily extract vector elements from the lanes.
+  while (NumActiveLanes > 4) {
+    unsigned RevOpcode = NumActiveLanes == 16 ? ARMISD::VREV16 : ARMISD::VREV32;
+    SDValue Rev = DAG.getNode(RevOpcode, dl, VT, Op0);
+    Op0 = DAG.getNode(BaseOpcode, dl, VT, Op0, Rev);
+    NumActiveLanes /= 2;
+  }
+
+  SDValue Res;
+  if (NumActiveLanes == 4) {
+    // The remaining 4 elements are summed sequentially
+    SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
+                              DAG.getConstant(0 * NumElts / 4, dl, MVT::i32));
+    SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
+                              DAG.getConstant(1 * NumElts / 4, dl, MVT::i32));
+    SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
+                              DAG.getConstant(2 * NumElts / 4, dl, MVT::i32));
+    SDValue Ext3 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
+                              DAG.getConstant(3 * NumElts / 4, dl, MVT::i32));
+    SDValue Res0 = DAG.getNode(BaseOpcode, dl, EltVT, Ext0, Ext1, Op->getFlags());
+    SDValue Res1 = DAG.getNode(BaseOpcode, dl, EltVT, Ext2, Ext3, Op->getFlags());
+    Res = DAG.getNode(BaseOpcode, dl, EltVT, Res0, Res1, Op->getFlags());
+  } else {
+    SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
+                              DAG.getConstant(0, dl, MVT::i32));
+    SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
+                              DAG.getConstant(1, dl, MVT::i32));
+    Res = DAG.getNode(BaseOpcode, dl, EltVT, Ext0, Ext1, Op->getFlags());
+  }
+
+  // Result type may be wider than element type.
+  if (EltVT != Op->getValueType(0))
+    Res = DAG.getNode(ISD::ANY_EXTEND, dl, Op->getValueType(0), Res);
+  return Res;
+}
+
+static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG,
+                               const ARMSubtarget *ST) {
+  if (!ST->hasMVEFloatOps())
+    return SDValue();
+  return LowerVecReduce(Op, DAG, ST);
+}
+
 static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) {
   if (isStrongerThanMonotonic(cast<AtomicSDNode>(Op)->getOrdering()))
     // Acquire/Release load/store is not legal for targets without a dmb or
@@ -9583,6 +9794,16 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
     return LowerSTORE(Op, DAG, Subtarget);
   case ISD::MLOAD:
     return LowerMLOAD(Op, DAG);
+  case ISD::VECREDUCE_MUL:
+  case ISD::VECREDUCE_AND:
+  case ISD::VECREDUCE_OR:
+  case ISD::VECREDUCE_XOR:
+    return LowerVecReduce(Op, DAG, Subtarget);
+  case ISD::VECREDUCE_FADD:
+  case ISD::VECREDUCE_FMUL:
+  case ISD::VECREDUCE_FMIN:
+  case ISD::VECREDUCE_FMAX:
+    return LowerVecReduceF(Op, DAG, Subtarget);
   case ISD::ATOMIC_LOAD:
   case ISD::ATOMIC_STORE:  return LowerAtomicLoadStore(Op, DAG);
   case ISD::FSINCOS:       return LowerFSINCOS(Op, DAG);
@@ -14198,7 +14419,7 @@ static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St,
   if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
     return SDValue();
   SDValue Trunc = St->getValue();
-  if (Trunc->getOpcode() != ISD::TRUNCATE)
+  if (Trunc->getOpcode() != ISD::TRUNCATE && Trunc->getOpcode() != ISD::FP_ROUND)
     return SDValue();
   EVT FromVT = Trunc->getOperand(0).getValueType();
   EVT ToVT = Trunc.getValueType();
@@ -14213,7 +14434,10 @@ static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St,
     NumElements = 4;
   if (FromEltVT == MVT::i16 && ToEltVT == MVT::i8)
     NumElements = 8;
-  if (NumElements == 0 || FromVT.getVectorNumElements() == NumElements ||
+  if (FromEltVT == MVT::f32 && ToEltVT == MVT::f16)
+    NumElements = 4;
+  if (NumElements == 0 ||
+      (FromEltVT != MVT::f32 && FromVT.getVectorNumElements() == NumElements) ||
       FromVT.getVectorNumElements() % NumElements != 0)
     return SDValue();
 
@@ -14223,7 +14447,7 @@ static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St,
   //  rev: N 0 N+1 1 N+2 2 ...
   auto isVMOVNOriginalMask = [&](ArrayRef<int> M, bool rev) {
     unsigned NumElts = ToVT.getVectorNumElements();
-    if (NumElts != M.size() || (ToVT != MVT::v8i16 && ToVT != MVT::v16i8))
+    if (NumElts != M.size())
       return false;
 
     unsigned Off0 = rev ? NumElts : 0;
@@ -14244,6 +14468,7 @@ static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St,
         isVMOVNOriginalMask(Shuffle->getMask(), true))
       return SDValue();
 
+  LLVMContext &C = *DAG.getContext();
   SDLoc DL(St);
   // Details about the old store
   SDValue Ch = St->getChain();
@@ -14252,8 +14477,11 @@ static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St,
   MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
   AAMDNodes AAInfo = St->getAAInfo();
 
-  EVT NewFromVT = EVT::getVectorVT(*DAG.getContext(), FromEltVT, NumElements);
-  EVT NewToVT = EVT::getVectorVT(*DAG.getContext(), ToEltVT, NumElements);
+  // We split the store into slices of NumElements. fp16 trunc stores are vcvt
+  // and then stored as truncating integer stores.
+  EVT NewFromVT = EVT::getVectorVT(C, FromEltVT, NumElements);
+  EVT NewToVT = EVT::getVectorVT(
+      C, EVT::getIntegerVT(C, ToEltVT.getSizeInBits()), NumElements);
 
   SmallVector<SDValue, 4> Stores;
   for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
@@ -14263,6 +14491,14 @@ static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St,
     SDValue Extract =
         DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewFromVT, Trunc.getOperand(0),
                     DAG.getConstant(i * NumElements, DL, MVT::i32));
+
+    if (ToEltVT == MVT::f16) {
+      SDValue FPTrunc =
+          DAG.getNode(ARMISD::VCVTN, DL, MVT::v8f16, DAG.getUNDEF(MVT::v8f16),
+                      Extract, DAG.getConstant(0, DL, MVT::i32));
+      Extract = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, MVT::v4i32, FPTrunc);
+    }
+
     SDValue Store = DAG.getTruncStore(
         Ch, DL, Extract, NewPtr, St->getPointerInfo().getWithOffset(NewOffset),
         NewToVT, Alignment.value(), MMOFlags, AAInfo);
@@ -14929,9 +15165,10 @@ static SDValue PerformShiftCombine(SDNode *N,
   return SDValue();
 }
 
-// Look for a sign/zero extend of a larger than legal load. This can be split
-// into two extending loads, which are simpler to deal with than an arbitrary
-// sign extend.
+// Look for a sign/zero/fpextend extend of a larger than legal load. This can be
+// split into multiple extending loads, which are simpler to deal with than an
+// arbitrary extend. For fp extends we use an integer extending load and a VCVTL
+// to convert the type to an f32.
 static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG) {
   SDValue N0 = N->getOperand(0);
   if (N0.getOpcode() != ISD::LOAD)
@@ -14953,12 +15190,15 @@ static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG) {
     NumElements = 4;
   if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
     NumElements = 8;
+  if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16)
+    NumElements = 4;
   if (NumElements == 0 ||
-      FromVT.getVectorNumElements() == NumElements ||
+      (FromEltVT != MVT::f16 && FromVT.getVectorNumElements() == NumElements) ||
       FromVT.getVectorNumElements() % NumElements != 0 ||
       !isPowerOf2_32(NumElements))
     return SDValue();
 
+  LLVMContext &C = *DAG.getContext();
   SDLoc DL(LD);
   // Details about the old load
   SDValue Ch = LD->getChain();
@@ -14970,28 +15210,43 @@ static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG) {
   ISD::LoadExtType NewExtType =
       N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
   SDValue Offset = DAG.getUNDEF(BasePtr.getValueType());
-  EVT NewFromVT = FromVT.getHalfNumVectorElementsVT(*DAG.getContext());
-  EVT NewToVT = ToVT.getHalfNumVectorElementsVT(*DAG.getContext());
-  unsigned NewOffset = NewFromVT.getSizeInBits() / 8;
-  SDValue NewPtr = DAG.getObjectPtrOffset(DL, BasePtr, NewOffset);
-
-  // Split the load in half, each side of which is extended separately. This
-  // is good enough, as legalisation will take it from there. They are either
-  // already legal or they will be split further into something that is
-  // legal.
-  SDValue NewLoad1 = DAG.getLoad(
-      ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, BasePtr, Offset,
-      LD->getPointerInfo(), NewFromVT, Alignment.value(), MMOFlags, AAInfo);
-  SDValue NewLoad2 =
-      DAG.getLoad(ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, NewPtr, Offset,
-                  LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
-                  Alignment.value(), MMOFlags, AAInfo);
-
-  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
-                                 SDValue(NewLoad1.getNode(), 1),
-                                 SDValue(NewLoad2.getNode(), 1));
+  EVT NewFromVT = EVT::getVectorVT(
+      C, EVT::getIntegerVT(C, FromEltVT.getScalarSizeInBits()), NumElements);
+  EVT NewToVT = EVT::getVectorVT(
+      C, EVT::getIntegerVT(C, ToEltVT.getScalarSizeInBits()), NumElements);
+
+  SmallVector<SDValue, 4> Loads;
+  SmallVector<SDValue, 4> Chains;
+  for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
+    unsigned NewOffset = (i * NewFromVT.getSizeInBits()) / 8;
+    SDValue NewPtr = DAG.getObjectPtrOffset(DL, BasePtr, NewOffset);
+
+    SDValue NewLoad =
+        DAG.getLoad(ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, NewPtr, Offset,
+                    LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
+                    Alignment.value(), MMOFlags, AAInfo);
+    Loads.push_back(NewLoad);
+    Chains.push_back(SDValue(NewLoad.getNode(), 1));
+  }
+
+  // Float truncs need to extended with VCVTB's into their floating point types.
+  if (FromEltVT == MVT::f16) {
+    SmallVector<SDValue, 4> Extends;
+
+    for (unsigned i = 0; i < Loads.size(); i++) {
+      SDValue LoadBC =
+          DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, MVT::v8f16, Loads[i]);
+      SDValue FPExt = DAG.getNode(ARMISD::VCVTL, DL, MVT::v4f32, LoadBC,
+                                  DAG.getConstant(0, DL, MVT::i32));
+      Extends.push_back(FPExt);
+    }
+
+    Loads = Extends;
+  }
+
+  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
   DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewChain);
-  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ToVT, NewLoad1, NewLoad2);
+  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ToVT, Loads);
 }
 
 /// PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND,
@@ -15039,6 +15294,15 @@ static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
+static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG,
+                                      const ARMSubtarget *ST) {
+  if (ST->hasMVEFloatOps())
+    if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
+      return NewLoad;
+
+  return SDValue();
+}
+
 /// PerformMinMaxCombine - Target-specific DAG combining for creating truncating
 /// saturates.
 static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG,
@@ -15697,6 +15961,8 @@ SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::ZERO_EXTEND:
   case ISD::ANY_EXTEND:
     return PerformExtendCombine(N, DCI.DAG, Subtarget);
+  case ISD::FP_EXTEND:
+    return PerformFPExtendCombine(N, DCI.DAG, Subtarget);
   case ISD::SMIN:
   case ISD::UMIN:
   case ISD::SMAX:
@@ -16561,7 +16827,7 @@ static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT,
   return false;
 }
 
-static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, unsigned Align,
+static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment,
                                       bool isSEXTLoad, bool IsMasked, bool isLE,
                                       SDValue &Base, SDValue &Offset,
                                       bool &isInc, SelectionDAG &DAG) {
@@ -16596,16 +16862,16 @@ static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, unsigned Align,
   // (in BE/masked) type.
   Base = Ptr->getOperand(0);
   if (VT == MVT::v4i16) {
-    if (Align >= 2 && IsInRange(RHSC, 0x80, 2))
+    if (Alignment >= 2 && IsInRange(RHSC, 0x80, 2))
       return true;
   } else if (VT == MVT::v4i8 || VT == MVT::v8i8) {
     if (IsInRange(RHSC, 0x80, 1))
       return true;
-  } else if (Align >= 4 &&
+  } else if (Alignment >= 4 &&
              (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) &&
              IsInRange(RHSC, 0x80, 4))
     return true;
-  else if (Align >= 2 &&
+  else if (Alignment >= 2 &&
            (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) &&
            IsInRange(RHSC, 0x80, 2))
     return true;
@@ -16627,28 +16893,28 @@ ARMTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
 
   EVT VT;
   SDValue Ptr;
-  unsigned Align;
+  Align Alignment;
   bool isSEXTLoad = false;
   bool IsMasked = false;
   if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
     Ptr = LD->getBasePtr();
     VT = LD->getMemoryVT();
-    Align = LD->getAlignment();
+    Alignment = LD->getAlign();
     isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
   } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
     Ptr = ST->getBasePtr();
     VT = ST->getMemoryVT();
-    Align = ST->getAlignment();
+    Alignment = ST->getAlign();
   } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(N)) {
     Ptr = LD->getBasePtr();
     VT = LD->getMemoryVT();
-    Align = LD->getAlignment();
+    Alignment = LD->getAlign();
     isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
     IsMasked = true;
   } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(N)) {
     Ptr = ST->getBasePtr();
     VT = ST->getMemoryVT();
-    Align = ST->getAlignment();
+    Alignment = ST->getAlign();
     IsMasked = true;
   } else
     return false;
@@ -16657,9 +16923,9 @@ ARMTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
   bool isLegal = false;
   if (VT.isVector())
     isLegal = Subtarget->hasMVEIntegerOps() &&
-              getMVEIndexedAddressParts(Ptr.getNode(), VT, Align, isSEXTLoad,
-                                        IsMasked, Subtarget->isLittle(), Base,
-                                        Offset, isInc, DAG);
+              getMVEIndexedAddressParts(
+                  Ptr.getNode(), VT, Alignment, isSEXTLoad, IsMasked,
+                  Subtarget->isLittle(), Base, Offset, isInc, DAG);
   else {
     if (Subtarget->isThumb2())
       isLegal = getT2IndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base,
@@ -16685,31 +16951,31 @@ bool ARMTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
                                                    SelectionDAG &DAG) const {
   EVT VT;
   SDValue Ptr;
-  unsigned Align;
+  Align Alignment;
   bool isSEXTLoad = false, isNonExt;
   bool IsMasked = false;
   if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
     VT = LD->getMemoryVT();
     Ptr = LD->getBasePtr();
-    Align = LD->getAlignment();
+    Alignment = LD->getAlign();
     isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
     isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD;
   } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
     VT = ST->getMemoryVT();
     Ptr = ST->getBasePtr();
-    Align = ST->getAlignment();
+    Alignment = ST->getAlign();
     isNonExt = !ST->isTruncatingStore();
   } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(N)) {
     VT = LD->getMemoryVT();
     Ptr = LD->getBasePtr();
-    Align = LD->getAlignment();
+    Alignment = LD->getAlign();
     isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
     isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD;
     IsMasked = true;
   } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(N)) {
     VT = ST->getMemoryVT();
     Ptr = ST->getBasePtr();
-    Align = ST->getAlignment();
+    Alignment = ST->getAlign();
     isNonExt = !ST->isTruncatingStore();
     IsMasked = true;
   } else
@@ -16735,7 +17001,7 @@ bool ARMTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
   bool isLegal = false;
   if (VT.isVector())
     isLegal = Subtarget->hasMVEIntegerOps() &&
-              getMVEIndexedAddressParts(Op, VT, Align, isSEXTLoad, IsMasked,
+              getMVEIndexedAddressParts(Op, VT, Alignment, isSEXTLoad, IsMasked,
                                         Subtarget->isLittle(), Base, Offset,
                                         isInc, DAG);
   else {
@@ -16864,10 +17130,9 @@ void ARMTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
   }
 }
 
-bool
-ARMTargetLowering::targetShrinkDemandedConstant(SDValue Op,
-                                                const APInt &DemandedAPInt,
-                                                TargetLoweringOpt &TLO) const {
+bool ARMTargetLowering::targetShrinkDemandedConstant(
+    SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
+    TargetLoweringOpt &TLO) const {
   // Delay optimization, so we don't have to deal with illegal types, or block
   // optimizations.
   if (!TLO.LegalOps)
@@ -16892,7 +17157,7 @@ ARMTargetLowering::targetShrinkDemandedConstant(SDValue Op,
 
   unsigned Mask = C->getZExtValue();
 
-  unsigned Demanded = DemandedAPInt.getZExtValue();
+  unsigned Demanded = DemandedBits.getZExtValue();
   unsigned ShrunkMask = Mask & Demanded;
   unsigned ExpandedMask = Mask | ~Demanded;
 
@@ -17493,13 +17758,15 @@ ARMTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const
 
   if (DAG.getMachineFunction().getFunction().hasFnAttribute(
           "no-stack-arg-probe")) {
-    unsigned Align = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
+    MaybeAlign Align =
+        cast<ConstantSDNode>(Op.getOperand(2))->getMaybeAlignValue();
     SDValue SP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32);
     Chain = SP.getValue(1);
     SP = DAG.getNode(ISD::SUB, DL, MVT::i32, SP, Size);
     if (Align)
-      SP = DAG.getNode(ISD::AND, DL, MVT::i32, SP.getValue(0),
-                       DAG.getConstant(-(uint64_t)Align, DL, MVT::i32));
+      SP =
+          DAG.getNode(ISD::AND, DL, MVT::i32, SP.getValue(0),
+                      DAG.getConstant(-(uint64_t)Align->value(), DL, MVT::i32));
     Chain = DAG.getCopyToReg(Chain, DL, ARM::SP, SP);
     SDValue Ops[2] = { SP, Chain };
     return DAG.getMergeValues(Ops, DL);
@@ -17703,7 +17970,7 @@ bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = 0;
     Value *AlignArg = I.getArgOperand(I.getNumArgOperands() - 1);
-    Info.align = MaybeAlign(cast<ConstantInt>(AlignArg)->getZExtValue());
+    Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
     // volatile loads with NEON intrinsics not supported
     Info.flags = MachineMemOperand::MOLoad;
     return true;
@@ -17744,7 +18011,7 @@ bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = 0;
     Value *AlignArg = I.getArgOperand(I.getNumArgOperands() - 1);
-    Info.align = MaybeAlign(cast<ConstantInt>(AlignArg)->getZExtValue());
+    Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
     // volatile stores with NEON intrinsics not supported
     Info.flags = MachineMemOperand::MOStore;
     return true;
@@ -18572,7 +18839,7 @@ static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base,
 /// Return the correct alignment for the current calling convention.
 Align ARMTargetLowering::getABIAlignmentForCallingConv(Type *ArgTy,
                                                        DataLayout DL) const {
-  const Align ABITypeAlign(DL.getABITypeAlignment(ArgTy));
+  const Align ABITypeAlign = DL.getABITypeAlign(ArgTy);
   if (!ArgTy->isVectorTy())
     return ABITypeAlign;
 
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 3bf50b1da129a..8b1f4183032eb 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -208,6 +208,10 @@ class VectorType;
       VQMOVNs,      // Vector (V) Saturating (Q) Move and Narrow (N), signed (s)
       VQMOVNu,      // Vector (V) Saturating (Q) Move and Narrow (N), unsigned (u)
 
+      // MVE float <> half converts
+      VCVTN,        // MVE vcvt f32 -> f16, truncating into either the bottom or top lanes
+      VCVTL,        // MVE vcvt f16 -> f32, extending from either the bottom or top lanes
+
       // Vector multiply long:
       VMULLs,       // ...signed
       VMULLu,       // ...unsigned
@@ -449,10 +453,10 @@ class VectorType;
                                        const SelectionDAG &DAG,
                                        unsigned Depth) const override;
 
-    bool targetShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
+    bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,
+                                      const APInt &DemandedElts,
                                       TargetLoweringOpt &TLO) const override;
 
-
     bool ExpandInlineAsm(CallInst *CI) const override;
 
     ConstraintType getConstraintType(StringRef Constraint) const override;
@@ -764,6 +768,8 @@ class VectorType;
     SDValue LowerDIV_Windows(SDValue Op, SelectionDAG &DAG, bool Signed) const;
     void ExpandDIV_Windows(SDValue Op, SelectionDAG &DAG, bool Signed,
                            SmallVectorImpl<SDValue> &Results) const;
+    SDValue ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
+                          const ARMSubtarget *Subtarget) const;
     SDValue LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG, bool Signed,
                                    SDValue &Chain) const;
     SDValue LowerREM(SDNode *N, SelectionDAG &DAG) const;
@@ -787,6 +793,11 @@ class VectorType;
     bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
                                     EVT VT) const override;
 
+    SDValue MoveToHPR(const SDLoc &dl, SelectionDAG &DAG, MVT LocVT, MVT ValVT,
+                      SDValue Val) const;
+    SDValue MoveFromHPR(const SDLoc &dl, SelectionDAG &DAG, MVT LocVT,
+                        MVT ValVT, SDValue Val) const;
+
     SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const;
 
     SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.td b/llvm/lib/Target/ARM/ARMInstrInfo.td
index 6b990a59ed0e9..da0a836c8f95d 100644
--- a/llvm/lib/Target/ARM/ARMInstrInfo.td
+++ b/llvm/lib/Target/ARM/ARMInstrInfo.td
@@ -443,6 +443,62 @@ def fsub_mlx : PatFrag<(ops node:$lhs, node:$rhs),(fsub node:$lhs, node:$rhs),[{
   return hasNoVMLxHazardUse(N);
 }]>;
 
+def imm_even : ImmLeaf<i32, [{ return (Imm & 1) == 0; }]>;
+def imm_odd : ImmLeaf<i32, [{ return (Imm & 1) == 1; }]>;
+
+//===----------------------------------------------------------------------===//
+// NEON/MVE pattern fragments
+//
+
+// Extract D sub-registers of Q registers.
+def DSubReg_i8_reg  : SDNodeXForm<imm, [{
+  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
+  return CurDAG->getTargetConstant(ARM::dsub_0 + N->getZExtValue()/8, SDLoc(N),
+                                   MVT::i32);
+}]>;
+def DSubReg_i16_reg : SDNodeXForm<imm, [{
+  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
+  return CurDAG->getTargetConstant(ARM::dsub_0 + N->getZExtValue()/4, SDLoc(N),
+                                   MVT::i32);
+}]>;
+def DSubReg_i32_reg : SDNodeXForm<imm, [{
+  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
+  return CurDAG->getTargetConstant(ARM::dsub_0 + N->getZExtValue()/2, SDLoc(N),
+                                   MVT::i32);
+}]>;
+def DSubReg_f64_reg : SDNodeXForm<imm, [{
+  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
+  return CurDAG->getTargetConstant(ARM::dsub_0 + N->getZExtValue(), SDLoc(N),
+                                   MVT::i32);
+}]>;
+
+// Extract S sub-registers of Q/D registers.
+def SSubReg_f32_reg : SDNodeXForm<imm, [{
+  assert(ARM::ssub_3 == ARM::ssub_0+3 && "Unexpected subreg numbering");
+  return CurDAG->getTargetConstant(ARM::ssub_0 + N->getZExtValue(), SDLoc(N),
+                                   MVT::i32);
+}]>;
+
+// Extract S sub-registers of Q/D registers containing a given f16/bf16 lane.
+def SSubReg_f16_reg : SDNodeXForm<imm, [{
+  assert(ARM::ssub_3 == ARM::ssub_0+3 && "Unexpected subreg numbering");
+  return CurDAG->getTargetConstant(ARM::ssub_0 + N->getZExtValue()/2, SDLoc(N),
+                                   MVT::i32);
+}]>;
+
+// Translate lane numbers from Q registers to D subregs.
+def SubReg_i8_lane  : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(N->getZExtValue() & 7, SDLoc(N), MVT::i32);
+}]>;
+def SubReg_i16_lane : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(N->getZExtValue() & 3, SDLoc(N), MVT::i32);
+}]>;
+def SubReg_i32_lane : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(N->getZExtValue() & 1, SDLoc(N), MVT::i32);
+}]>;
+
+
+
 //===----------------------------------------------------------------------===//
 // Operand Definitions.
 //
@@ -5479,7 +5535,8 @@ def : ARMInstAlias<"mcr${p} $cop, $opc1, $Rt, $CRn, $CRm",
 def MRC : MovRCopro<"mrc", 1 /* from coprocessor to ARM core register */,
                     (outs GPRwithAPSR:$Rt),
                     (ins p_imm:$cop, imm0_7:$opc1, c_imm:$CRn, c_imm:$CRm,
-                         imm0_7:$opc2), []>;
+                         imm0_7:$opc2), []>,
+                    ComplexDeprecationPredicate<"MRC">;
 def : ARMInstAlias<"mrc${p} $cop, $opc1, $Rt, $CRn, $CRm",
                    (MRC GPRwithAPSR:$Rt, p_imm:$cop, imm0_7:$opc1, c_imm:$CRn,
                         c_imm:$CRm, 0, pred:$p)>;
diff --git a/llvm/lib/Target/ARM/ARMInstrMVE.td b/llvm/lib/Target/ARM/ARMInstrMVE.td
index d3fbf7c7f77db..2a1f50d97e3b3 100644
--- a/llvm/lib/Target/ARM/ARMInstrMVE.td
+++ b/llvm/lib/Target/ARM/ARMInstrMVE.td
@@ -4782,6 +4782,11 @@ class MVE_VCVT_ff<string iname, string suffix, bit op, bit T,
   let retainsPreviousHalfElement = 1;
 }
 
+def SDTARMVCVTL    : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>,
+                                         SDTCisVT<2, i32>]>;
+def MVEvcvtn       : SDNode<"ARMISD::VCVTN", SDTARMVMOVNQ>;
+def MVEvcvtl       : SDNode<"ARMISD::VCVTL", SDTARMVCVTL>;
+
 multiclass MVE_VCVT_f2h_m<string iname, int half> {
   def "": MVE_VCVT_ff<iname, "f16.f32", 0b0, half,
                       (ins MQPR:$Qd_src), vpred_n, "$Qd = $Qd_src">;
@@ -4796,6 +4801,9 @@ multiclass MVE_VCVT_f2h_m<string iname, int half> {
                          (v4i1 VCCR:$mask))),
               (v8f16 (Inst (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm),
                            ARMVCCThen, (v4i1 VCCR:$mask)))>;
+
+    def : Pat<(v8f16 (MVEvcvtn (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm), (i32 half))),
+              (v8f16 (Inst (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm)))>;
   }
 }
 
@@ -4811,6 +4819,9 @@ multiclass MVE_VCVT_h2f_m<string iname, int half> {
                          (v4i1 VCCR:$mask))),
               (v4f32 (Inst (v8f16 MQPR:$Qm), ARMVCCThen,
                            (v4i1 VCCR:$mask), (v4f32 MQPR:$inactive)))>;
+
+    def : Pat<(v4f32 (MVEvcvtl (v8f16 MQPR:$Qm), (i32 half))),
+              (v4f32 (Inst (v8f16 MQPR:$Qm)))>;
   }
 }
 
diff --git a/llvm/lib/Target/ARM/ARMInstrNEON.td b/llvm/lib/Target/ARM/ARMInstrNEON.td
index ac609600d92a3..1b3f6075c0e9d 100644
--- a/llvm/lib/Target/ARM/ARMInstrNEON.td
+++ b/llvm/lib/Target/ARM/ARMInstrNEON.td
@@ -2454,57 +2454,6 @@ def : Pat<(byte_alignedstore (v2f64 QPR:$value), addrmode6:$addr),
           (VST1q8 addrmode6:$addr, QPR:$value)>;
 }
 
-//===----------------------------------------------------------------------===//
-// NEON pattern fragments
-//===----------------------------------------------------------------------===//
-
-// Extract D sub-registers of Q registers.
-def DSubReg_i8_reg  : SDNodeXForm<imm, [{
-  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
-  return CurDAG->getTargetConstant(ARM::dsub_0 + N->getZExtValue()/8, SDLoc(N),
-                                   MVT::i32);
-}]>;
-def DSubReg_i16_reg : SDNodeXForm<imm, [{
-  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
-  return CurDAG->getTargetConstant(ARM::dsub_0 + N->getZExtValue()/4, SDLoc(N),
-                                   MVT::i32);
-}]>;
-def DSubReg_i32_reg : SDNodeXForm<imm, [{
-  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
-  return CurDAG->getTargetConstant(ARM::dsub_0 + N->getZExtValue()/2, SDLoc(N),
-                                   MVT::i32);
-}]>;
-def DSubReg_f64_reg : SDNodeXForm<imm, [{
-  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
-  return CurDAG->getTargetConstant(ARM::dsub_0 + N->getZExtValue(), SDLoc(N),
-                                   MVT::i32);
-}]>;
-
-// Extract S sub-registers of Q/D registers.
-def SSubReg_f32_reg : SDNodeXForm<imm, [{
-  assert(ARM::ssub_3 == ARM::ssub_0+3 && "Unexpected subreg numbering");
-  return CurDAG->getTargetConstant(ARM::ssub_0 + N->getZExtValue(), SDLoc(N),
-                                   MVT::i32);
-}]>;
-
-// Extract S sub-registers of Q/D registers containing a given f16/bf16 lane.
-def SSubReg_f16_reg : SDNodeXForm<imm, [{
-  assert(ARM::ssub_3 == ARM::ssub_0+3 && "Unexpected subreg numbering");
-  return CurDAG->getTargetConstant(ARM::ssub_0 + N->getZExtValue()/2, SDLoc(N),
-                                   MVT::i32);
-}]>;
-
-// Translate lane numbers from Q registers to D subregs.
-def SubReg_i8_lane  : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getZExtValue() & 7, SDLoc(N), MVT::i32);
-}]>;
-def SubReg_i16_lane : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getZExtValue() & 3, SDLoc(N), MVT::i32);
-}]>;
-def SubReg_i32_lane : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getZExtValue() & 1, SDLoc(N), MVT::i32);
-}]>;
-
 //===----------------------------------------------------------------------===//
 // Instruction Classes
 //===----------------------------------------------------------------------===//
@@ -6410,9 +6359,6 @@ def : Pat<(extractelt (v2f64 QPR:$src1), imm:$src2),
           (EXTRACT_SUBREG QPR:$src1, (DSubReg_f64_reg imm:$src2))>;
 }
 
-def imm_even : ImmLeaf<i32, [{ return (Imm & 1) == 0; }]>;
-def imm_odd : ImmLeaf<i32, [{ return (Imm & 1) == 1; }]>;
-
 multiclass ExtractEltEvenF16<ValueType VT4, ValueType VT8> {
   def : Pat<(extractelt (VT4 DPR:$src), imm_even:$lane),
               (EXTRACT_SUBREG
@@ -9099,15 +9045,20 @@ def : NEONInstAlias<"vmov${p}.f32 $Vd, $imm",
 // ARMv8.6a BFloat16 instructions.
 let Predicates = [HasBF16, HasNEON] in {
 class BF16VDOT<bits<5> op27_23, bits<2> op21_20, bit op6,
-               dag oops, dag iops>
+               dag oops, dag iops, list<dag> pattern>
    : N3Vnp<op27_23, op21_20, 0b1101, op6, 0, oops, iops,
-           N3RegFrm, IIC_VDOTPROD, "", "", []> {
+           N3RegFrm, IIC_VDOTPROD, "", "", pattern>
+{
     let DecoderNamespace = "VFPV8";
 }
 
 class BF16VDOTS<bit Q, RegisterClass RegTy, string opc, ValueType AccumTy, ValueType InputTy>
    : BF16VDOT<0b11000, 0b00,  Q, (outs RegTy:$dst),
-              (ins RegTy:$Vd, RegTy:$Vn, RegTy:$Vm)> {
+              (ins RegTy:$Vd, RegTy:$Vn, RegTy:$Vm),
+            [(set (AccumTy RegTy:$dst),
+                  (int_arm_neon_bfdot (AccumTy RegTy:$Vd),
+                                      (InputTy RegTy:$Vn),
+                                      (InputTy RegTy:$Vm)))]> {
   let Constraints = "$dst = $Vd";
   let AsmString = !strconcat(opc, ".bf16", "\t$Vd, $Vn, $Vm");
     let DecoderNamespace = "VFPV8";
@@ -9118,7 +9069,7 @@ multiclass BF16VDOTI<bit Q, RegisterClass RegTy, string opc, ValueType AccumTy,
 
   def "" : BF16VDOT<0b11100, 0b00, Q, (outs RegTy:$dst),
                     (ins RegTy:$Vd, RegTy:$Vn,
-                    DPR_VFP2:$Vm, VectorIndex32:$lane)> {
+                    DPR_VFP2:$Vm, VectorIndex32:$lane), []> {
     bit lane;
     let Inst{5} = lane;
     let Constraints = "$dst = $Vd";
@@ -9126,6 +9077,13 @@ multiclass BF16VDOTI<bit Q, RegisterClass RegTy, string opc, ValueType AccumTy,
     let DecoderNamespace = "VFPV8";
   }
 
+  def : Pat<
+    (AccumTy (int_arm_neon_bfdot (AccumTy RegTy:$Vd),
+                                 (InputTy RegTy:$Vn),
+                                 (InputTy (bitconvert (AccumTy
+                                          (ARMvduplane (AccumTy RegTy:$Vm),
+                                                        VectorIndex32:$lane)))))),
+    (!cast<Instruction>(NAME) RegTy:$Vd, RegTy:$Vn, RHS, VectorIndex32:$lane)>;
 }
 
 def BF16VDOTS_VDOTD : BF16VDOTS<0, DPR, "vdot", v2f32, v8i8>;
@@ -9138,7 +9096,10 @@ class BF16MM<bit Q, RegisterClass RegTy,
              string opc>
    : N3Vnp<0b11000, 0b00, 0b1100, Q, 0,
            (outs RegTy:$dst), (ins RegTy:$Vd, RegTy:$Vn, RegTy:$Vm),
-           N3RegFrm, IIC_VDOTPROD, "", "", []> {
+           N3RegFrm, IIC_VDOTPROD, "", "",
+                [(set (v4f32 QPR:$dst), (int_arm_neon_bfmmla (v4f32 QPR:$Vd),
+                                                (v16i8 QPR:$Vn),
+                                                (v16i8 QPR:$Vm)))]> {
    let Constraints = "$dst = $Vd";
    let AsmString = !strconcat(opc, ".bf16", "\t$Vd, $Vn, $Vm");
    let DecoderNamespace = "VFPV8";
@@ -9146,19 +9107,22 @@ class BF16MM<bit Q, RegisterClass RegTy,
 
 def VMMLA : BF16MM<1, QPR, "vmmla">;
 
-class VBF16MALQ<bit T, string suffix>
+class VBF16MALQ<bit T, string suffix, SDPatternOperator OpNode>
   : N3VCP8<0b00, 0b11, T, 1,
            (outs QPR:$dst), (ins QPR:$Vd, QPR:$Vn, QPR:$Vm),
            NoItinerary, "vfma" # suffix, "bf16", "$Vd, $Vn, $Vm", "",
-              []> { // TODO: Add intrinsics
+                [(set (v4f32 QPR:$dst),
+                      (OpNode (v4f32 QPR:$Vd),
+                              (v16i8 QPR:$Vn),
+                              (v16i8 QPR:$Vm)))]> {
   let Constraints = "$dst = $Vd";
   let DecoderNamespace = "VFPV8";
 }
 
-def VBF16MALTQ: VBF16MALQ<1, "t">;
-def VBF16MALBQ: VBF16MALQ<0, "b">;
+def VBF16MALTQ: VBF16MALQ<1, "t", int_arm_neon_bfmlalt>;
+def VBF16MALBQ: VBF16MALQ<0, "b", int_arm_neon_bfmlalb>;
 
-multiclass VBF16MALQI<bit T, string suffix> {
+multiclass VBF16MALQI<bit T, string suffix, SDPatternOperator OpNode> {
   def "" : N3VLaneCP8<0, 0b11, T, 1, (outs QPR:$dst),
               (ins QPR:$Vd, QPR:$Vn, DPR_8:$Vm, VectorIndex16:$idx),
                IIC_VMACD, "vfma" # suffix, "bf16", "$Vd, $Vn, $Vm$idx", "", []> {
@@ -9169,10 +9133,20 @@ multiclass VBF16MALQI<bit T, string suffix> {
   let DecoderNamespace = "VFPV8";
   }
 
-}
-
-defm VBF16MALTQI: VBF16MALQI<1, "t">;
-defm VBF16MALBQI: VBF16MALQI<0, "b">;
+  def : Pat<
+    (v4f32 (OpNode (v4f32 QPR:$Vd),
+                   (v16i8 QPR:$Vn),
+                   (v16i8 (bitconvert (v8bf16 (ARMvduplane (v8bf16 QPR:$Vm),
+                                                           VectorIndex16:$lane)))))),
+    (!cast<Instruction>(NAME) QPR:$Vd,
+                              QPR:$Vn,
+                              (EXTRACT_SUBREG QPR:$Vm,
+                                (DSubReg_i16_reg VectorIndex16:$lane)),
+                              (SubReg_i16_lane VectorIndex16:$lane))>;
+}
+
+defm VBF16MALTQI: VBF16MALQI<1, "t", int_arm_neon_bfmlalt>;
+defm VBF16MALBQI: VBF16MALQI<0, "b", int_arm_neon_bfmlalb>;
 
 def BF16_VCVT :  N2V<0b11, 0b11, 0b01, 0b10, 0b01100, 1, 0,
                     (outs DPR:$Vd), (ins QPR:$Vm),
diff --git a/llvm/lib/Target/ARM/ARMInstrThumb2.td b/llvm/lib/Target/ARM/ARMInstrThumb2.td
index e2235b1c25013..7137e8ee66b8f 100644
--- a/llvm/lib/Target/ARM/ARMInstrThumb2.td
+++ b/llvm/lib/Target/ARM/ARMInstrThumb2.td
@@ -1748,7 +1748,7 @@ def t2STRHT  : T2IstT<0b01, "strht", IIC_iStore_bh_i>;
 
 // ldrd / strd pre / post variants
 
-let mayLoad = 1 in
+let mayLoad = 1, hasSideEffects = 0 in
 def t2LDRD_PRE  : T2Ii8s4<1, 1, 1, (outs rGPR:$Rt, rGPR:$Rt2, GPR:$wb),
                  (ins t2addrmode_imm8s4_pre:$addr), IIC_iLoad_d_ru,
                  "ldrd", "\t$Rt, $Rt2, $addr!", "$addr.base = $wb", []>,
@@ -1756,13 +1756,13 @@ def t2LDRD_PRE  : T2Ii8s4<1, 1, 1, (outs rGPR:$Rt, rGPR:$Rt2, GPR:$wb),
   let DecoderMethod = "DecodeT2LDRDPreInstruction";
 }
 
-let mayLoad = 1 in
+let mayLoad = 1, hasSideEffects = 0 in
 def t2LDRD_POST : T2Ii8s4post<0, 1, 1, (outs rGPR:$Rt, rGPR:$Rt2, GPR:$wb),
                  (ins addr_offset_none:$addr, t2am_imm8s4_offset:$imm),
                  IIC_iLoad_d_ru, "ldrd", "\t$Rt, $Rt2, $addr$imm",
                  "$addr.base = $wb", []>, Sched<[WriteLd]>;
 
-let mayStore = 1 in
+let mayStore = 1, hasSideEffects = 0 in
 def t2STRD_PRE  : T2Ii8s4<1, 1, 0, (outs GPR:$wb),
                  (ins rGPR:$Rt, rGPR:$Rt2, t2addrmode_imm8s4_pre:$addr),
                  IIC_iStore_d_ru, "strd", "\t$Rt, $Rt2, $addr!",
@@ -1770,7 +1770,7 @@ def t2STRD_PRE  : T2Ii8s4<1, 1, 0, (outs GPR:$wb),
   let DecoderMethod = "DecodeT2STRDPreInstruction";
 }
 
-let mayStore = 1 in
+let mayStore = 1, hasSideEffects = 0 in
 def t2STRD_POST : T2Ii8s4post<0, 1, 0, (outs GPR:$wb),
                  (ins rGPR:$Rt, rGPR:$Rt2, addr_offset_none:$addr,
                       t2am_imm8s4_offset:$imm),
@@ -2765,6 +2765,8 @@ def t2SBFX: T2TwoRegBitFI<
   let Inst{25} = 1;
   let Inst{24-20} = 0b10100;
   let Inst{15} = 0;
+
+  let hasSideEffects = 0;
 }
 
 def t2UBFX: T2TwoRegBitFI<
@@ -2774,6 +2776,8 @@ def t2UBFX: T2TwoRegBitFI<
   let Inst{25} = 1;
   let Inst{24-20} = 0b11100;
   let Inst{15} = 0;
+
+  let hasSideEffects = 0;
 }
 
 // A8.8.247  UDF - Undefined (Encoding T2)
diff --git a/llvm/lib/Target/ARM/ARMInstrVFP.td b/llvm/lib/Target/ARM/ARMInstrVFP.td
index 5611ddb57541a..8a652c1d90f6e 100644
--- a/llvm/lib/Target/ARM/ARMInstrVFP.td
+++ b/llvm/lib/Target/ARM/ARMInstrVFP.td
@@ -158,15 +158,23 @@ def VLDRS : ASI5<0b1101, 0b01, (outs SPR:$Sd), (ins addrmode5:$addr),
 let isUnpredicable = 1 in
 def VLDRH : AHI5<0b1101, 0b01, (outs HPR:$Sd), (ins addrmode5fp16:$addr),
                  IIC_fpLoad16, "vldr", ".16\t$Sd, $addr",
-                 []>,
+                 [(set HPR:$Sd, (f16 (alignedload16 addrmode5fp16:$addr)))]>,
             Requires<[HasFPRegs16]>;
 
 } // End of 'let canFoldAsLoad = 1, isReMaterializable = 1 in'
 
-def : FPRegs16Pat<(f16 (alignedload16 addrmode5fp16:$addr)),
-                  (VLDRH addrmode5fp16:$addr)>;
-def : FPRegs16Pat<(bf16 (alignedload16 addrmode5fp16:$addr)),
-                  (VLDRH addrmode5fp16:$addr)>;
+def : Pat<(bf16 (alignedload16 addrmode5fp16:$addr)),
+          (VLDRH addrmode5fp16:$addr)> {
+  let Predicates = [HasFPRegs16];
+}
+def : Pat<(bf16 (alignedload16 addrmode3:$addr)),
+          (COPY_TO_REGCLASS (LDRH addrmode3:$addr), HPR)> {
+  let Predicates = [HasNoFPRegs16, IsARM];
+}
+def : Pat<(bf16 (alignedload16 t2addrmode_imm12:$addr)),
+          (COPY_TO_REGCLASS (t2LDRHi12 t2addrmode_imm12:$addr), HPR)> {
+  let Predicates = [HasNoFPRegs16, IsThumb];
+}
 
 def VSTRD : ADI5<0b1101, 0b00, (outs), (ins DPR:$Dd, addrmode5:$addr),
                  IIC_fpStore64, "vstr", "\t$Dd, $addr",
@@ -185,13 +193,21 @@ def VSTRS : ASI5<0b1101, 0b00, (outs), (ins SPR:$Sd, addrmode5:$addr),
 let isUnpredicable = 1 in
 def VSTRH : AHI5<0b1101, 0b00, (outs), (ins HPR:$Sd, addrmode5fp16:$addr),
                  IIC_fpStore16, "vstr", ".16\t$Sd, $addr",
-                 []>,
+                 [(alignedstore16 (f16 HPR:$Sd), addrmode5fp16:$addr)]>,
             Requires<[HasFPRegs16]>;
 
-def : FPRegs16Pat<(alignedstore16 (f16 HPR:$Sd), addrmode5fp16:$addr),
-                  (VSTRH (f16 HPR:$Sd), addrmode5fp16:$addr)>;
-def : FPRegs16Pat<(alignedstore16 (bf16 HPR:$Sd), addrmode5fp16:$addr),
-                  (VSTRH (bf16 HPR:$Sd), addrmode5fp16:$addr)>;
+def : Pat<(alignedstore16 (bf16 HPR:$Sd), addrmode5fp16:$addr),
+          (VSTRH (bf16 HPR:$Sd), addrmode5fp16:$addr)> {
+  let Predicates = [HasFPRegs16];
+}
+def : Pat<(alignedstore16 (bf16 HPR:$Sd), addrmode3:$addr),
+          (STRH (COPY_TO_REGCLASS $Sd, GPR), addrmode3:$addr)> {
+  let Predicates = [HasNoFPRegs16, IsARM];
+}
+def : Pat<(alignedstore16 (bf16 HPR:$Sd), t2addrmode_imm12:$addr),
+          (t2STRHi12 (COPY_TO_REGCLASS $Sd, GPR), t2addrmode_imm12:$addr)> {
+  let Predicates = [HasNoFPRegs16, IsThumb];
+}
 
 //===----------------------------------------------------------------------===//
 //  Load / store multiple Instructions.
@@ -287,7 +303,6 @@ def : MnemonicAlias<"vstm", "vstmia">;
 //===----------------------------------------------------------------------===//
 //  Lazy load / store multiple Instructions
 //
-let mayLoad = 1 in
 def VLLDM : AXSI4<(outs), (ins GPRnopc:$Rn, pred:$p), IndexModeNone,
                   NoItinerary, "vlldm${p}\t$Rn", "", []>,
             Requires<[HasV8MMainline, Has8MSecExt]> {
@@ -298,9 +313,9 @@ def VLLDM : AXSI4<(outs), (ins GPRnopc:$Rn, pred:$p), IndexModeNone,
     let Inst{15-12} = 0;
     let Inst{7-0}   = 0;
     let mayLoad     = 1;
+    let Defs = [Q0, Q1, Q2, Q3, Q4, Q5, Q6, Q7, VPR, FPSCR, FPSCR_NZCV];
 }
 
-let mayStore = 1 in
 def VLSTM : AXSI4<(outs), (ins GPRnopc:$Rn, pred:$p), IndexModeNone,
                   NoItinerary, "vlstm${p}\t$Rn", "", []>,
             Requires<[HasV8MMainline, Has8MSecExt]> {
@@ -693,6 +708,7 @@ def VCVTDS  : ASuI<0b11101, 0b11, 0b0111, 0b11, 0,
   let Inst{22}    = Dd{4};
 
   let Predicates = [HasVFP2, HasDPVFP];
+  let hasSideEffects = 0;
 }
 
 // Special case encoding: bits 11-8 is 0b1011.
@@ -717,9 +733,11 @@ def VCVTSD  : VFPAI<(outs SPR:$Sd), (ins DPR:$Dm), VFPUnaryFrm,
   let Inst{4}     = 0;
 
   let Predicates = [HasVFP2, HasDPVFP];
+  let hasSideEffects = 0;
 }
 
 // Between half, single and double-precision.
+let hasSideEffects = 0 in
 def VCVTBHS: ASuI<0b11101, 0b11, 0b0010, 0b01, 0, (outs SPR:$Sd), (ins SPR:$Sm),
                  /* FIXME */ IIC_fpCVTSH, "vcvtb", ".f32.f16\t$Sd, $Sm",
                  [/* Intentionally left blank, see patterns below */]>,
@@ -731,6 +749,7 @@ def : FP16Pat<(f32 (fpextend (f16 HPR:$Sm))),
 def : FP16Pat<(f16_to_fp GPR:$a),
               (VCVTBHS (COPY_TO_REGCLASS GPR:$a, SPR))>;
 
+let hasSideEffects = 0 in
 def VCVTBSH: ASuI<0b11101, 0b11, 0b0011, 0b01, 0, (outs SPR:$Sd), (ins SPR:$Sm),
                  /* FIXME */ IIC_fpCVTHS, "vcvtb", ".f16.f32\t$Sd, $Sm",
                  [/* Intentionally left blank, see patterns below */]>,
@@ -741,19 +760,41 @@ def : FP16Pat<(f16 (fpround SPR:$Sm)),
               (COPY_TO_REGCLASS (VCVTBSH SPR:$Sm), HPR)>;
 def : FP16Pat<(fp_to_f16 SPR:$a),
               (i32 (COPY_TO_REGCLASS (VCVTBSH SPR:$a), GPR))>;
+def : FP16Pat<(insertelt (v8f16 MQPR:$src1), (f16 (fpround (f32 SPR:$src2))), imm_even:$lane),
+              (v8f16 (INSERT_SUBREG (v8f16 MQPR:$src1), (VCVTBSH SPR:$src2),
+                                    (SSubReg_f16_reg imm:$lane)))>;
+def : FP16Pat<(insertelt (v4f16 DPR:$src1), (f16 (fpround (f32 SPR:$src2))), imm_even:$lane),
+              (v4f16 (INSERT_SUBREG (v4f16 DPR:$src1), (VCVTBSH SPR:$src2),
+                                    (SSubReg_f16_reg imm:$lane)))>;
 
+let hasSideEffects = 0 in
 def VCVTTHS: ASuI<0b11101, 0b11, 0b0010, 0b11, 0, (outs SPR:$Sd), (ins SPR:$Sm),
                  /* FIXME */ IIC_fpCVTSH, "vcvtt", ".f32.f16\t$Sd, $Sm",
-                 [/* For disassembly only; pattern left blank */]>,
+                 [/* Intentionally left blank, see patterns below */]>,
                  Requires<[HasFP16]>,
              Sched<[WriteFPCVT]>;
 
+def : FP16Pat<(f32 (fpextend (extractelt (v8f16 MQPR:$src), imm_odd:$lane))),
+              (VCVTTHS (EXTRACT_SUBREG MQPR:$src, (SSubReg_f16_reg imm_odd:$lane)))>;
+def : FP16Pat<(f32 (fpextend (extractelt (v4f16 DPR:$src), imm_odd:$lane))),
+              (VCVTTHS (EXTRACT_SUBREG
+                (v2f32 (COPY_TO_REGCLASS (v4f16 DPR:$src), DPR_VFP2)),
+                (SSubReg_f16_reg imm_odd:$lane)))>;
+
+let hasSideEffects = 0 in
 def VCVTTSH: ASuI<0b11101, 0b11, 0b0011, 0b11, 0, (outs SPR:$Sd), (ins SPR:$Sm),
                  /* FIXME */ IIC_fpCVTHS, "vcvtt", ".f16.f32\t$Sd, $Sm",
-                 [/* For disassembly only; pattern left blank */]>,
+                 [/* Intentionally left blank, see patterns below */]>,
                  Requires<[HasFP16]>,
             Sched<[WriteFPCVT]>;
 
+def : FP16Pat<(insertelt (v8f16 MQPR:$src1), (f16 (fpround (f32 SPR:$src2))), imm_odd:$lane),
+              (v8f16 (INSERT_SUBREG (v8f16 MQPR:$src1), (VCVTTSH SPR:$src2),
+                                    (SSubReg_f16_reg imm:$lane)))>;
+def : FP16Pat<(insertelt (v4f16 DPR:$src1), (f16 (fpround (f32 SPR:$src2))), imm_odd:$lane),
+              (v4f16 (INSERT_SUBREG (v4f16 DPR:$src1), (VCVTTSH SPR:$src2),
+                                    (SSubReg_f16_reg imm:$lane)))>;
+
 def VCVTBHD : ADuI<0b11101, 0b11, 0b0010, 0b01, 0,
                    (outs DPR:$Dd), (ins SPR:$Sm),
                    NoItinerary, "vcvtb", ".f64.f16\t$Dd, $Sm",
@@ -766,6 +807,8 @@ def VCVTBHD : ADuI<0b11101, 0b11, 0b0010, 0b01, 0,
   // Encode instruction operands.
   let Inst{3-0} = Sm{4-1};
   let Inst{5}   = Sm{0};
+
+  let hasSideEffects = 0;
 }
 
 def : FullFP16Pat<(f64 (fpextend (f16 HPR:$Sm))),
@@ -789,6 +832,8 @@ def VCVTBDH : ADuI<0b11101, 0b11, 0b0011, 0b01, 0,
   let Inst{5}       = Dm{4};
   let Inst{15-12}   = Sd{4-1};
   let Inst{22}      = Sd{0};
+
+  let hasSideEffects = 0;
 }
 
 def : FullFP16Pat<(f16 (fpround DPR:$Dm)),
@@ -808,6 +853,8 @@ def VCVTTHD : ADuI<0b11101, 0b11, 0b0010, 0b11, 0,
   // Encode instruction operands.
   let Inst{3-0} = Sm{4-1};
   let Inst{5}   = Sm{0};
+
+  let hasSideEffects = 0;
 }
 
 def VCVTTDH : ADuI<0b11101, 0b11, 0b0011, 0b11, 0,
@@ -823,11 +870,13 @@ def VCVTTDH : ADuI<0b11101, 0b11, 0b0011, 0b11, 0,
   let Inst{22}    = Sd{0};
   let Inst{3-0}   = Dm{3-0};
   let Inst{5}     = Dm{4};
+
+  let hasSideEffects = 0;
 }
 
 multiclass vcvt_inst<string opc, bits<2> rm,
                      SDPatternOperator node = null_frag> {
-  let PostEncoderMethod = "", DecoderNamespace = "VFPV8" in {
+  let PostEncoderMethod = "", DecoderNamespace = "VFPV8", hasSideEffects = 0 in {
     def SH : AHuInp<0b11101, 0b11, 0b1100, 0b11, 0,
                     (outs SPR:$Sd), (ins HPR:$Sm),
                     NoItinerary, !strconcat("vcvt", opc, ".s32.f16\t$Sd, $Sm"),
@@ -1332,6 +1381,7 @@ class AVConv1IDs_Encode<bits<5> opcod1, bits<2> opcod2, bits<4> opcod3,
   let Inst{22}    = Dd{4};
 
   let Predicates = [HasVFP2, HasDPVFP];
+  let hasSideEffects = 0;
 }
 
 class AVConv1InSs_Encode<bits<5> opcod1, bits<2> opcod2, bits<4> opcod3,
@@ -1348,6 +1398,8 @@ class AVConv1InSs_Encode<bits<5> opcod1, bits<2> opcod2, bits<4> opcod3,
   let Inst{5}     = Sm{0};
   let Inst{15-12} = Sd{4-1};
   let Inst{22}    = Sd{0};
+
+  let hasSideEffects = 0;
 }
 
 class AVConv1IHs_Encode<bits<5> opcod1, bits<2> opcod2, bits<4> opcod3,
@@ -1367,6 +1419,7 @@ class AVConv1IHs_Encode<bits<5> opcod1, bits<2> opcod2, bits<4> opcod3,
   let Inst{22}    = Sd{0};
 
   let Predicates = [HasFullFP16];
+  let hasSideEffects = 0;
 }
 
 def VSITOD : AVConv1IDs_Encode<0b11101, 0b11, 0b1000, 0b1011,
@@ -1480,6 +1533,7 @@ class AVConv1IsD_Encode<bits<5> opcod1, bits<2> opcod2, bits<4> opcod3,
   let Inst{22}    = Sd{0};
 
   let Predicates = [HasVFP2, HasDPVFP];
+  let hasSideEffects = 0;
 }
 
 class AVConv1InsS_Encode<bits<5> opcod1, bits<2> opcod2, bits<4> opcod3,
@@ -1516,6 +1570,7 @@ class AVConv1IsH_Encode<bits<5> opcod1, bits<2> opcod2, bits<4> opcod3,
   let Inst{22}    = Sd{0};
 
   let Predicates = [HasFullFP16];
+  let hasSideEffects = 0;
 }
 
 // Always set Z bit in the instruction, i.e. "round towards zero" variants.
@@ -1695,6 +1750,8 @@ class AVConv1XInsS_Encode<bits<5> op1, bits<2> op2, bits<4> op3, bits<4> op4,
   // if dp_operation then UInt(D:Vd) else UInt(Vd:D);
   let Inst{22} = dst{0};
   let Inst{15-12} = dst{4-1};
+
+  let hasSideEffects = 0;
 }
 
 // Double Precision register
@@ -1707,6 +1764,7 @@ class AVConv1XInsD_Encode<bits<5> op1, bits<2> op2, bits<4> op3, bits<4> op4,
   let Inst{22} = dst{4};
   let Inst{15-12} = dst{3-0};
 
+  let hasSideEffects = 0;
   let Predicates = [HasVFP2, HasDPVFP];
 }
 
@@ -1905,7 +1963,9 @@ class BF16_VCVT<string opc, bits<2> op7_6>
   let Inst{11-8}  = 0b1001;
   let Inst{7-6}   = op7_6;
   let Inst{4}     = 0;
+
   let DecoderNamespace = "VFPV8";
+  let hasSideEffects = 0;
 }
 
 def BF16_VCVTB : BF16_VCVT<"vcvtb", 0b01>;
diff --git a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
index bb57fa5d0ea8e..be75d6bef08c4 100644
--- a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
+++ b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
@@ -201,12 +201,25 @@ namespace {
     PredicatedMI *getDivergent() const { return Divergent; }
   };
 
+  struct Reduction {
+    MachineInstr *Init;
+    MachineInstr &Copy;
+    MachineInstr &Reduce;
+    MachineInstr &VPSEL;
+
+    Reduction(MachineInstr *Init, MachineInstr *Mov, MachineInstr *Add,
+              MachineInstr *Sel)
+      : Init(Init), Copy(*Mov), Reduce(*Add), VPSEL(*Sel) { }
+  };
+
   struct LowOverheadLoop {
 
     MachineLoop &ML;
+    MachineBasicBlock *Preheader = nullptr;
     MachineLoopInfo &MLI;
     ReachingDefAnalysis &RDA;
     const TargetRegisterInfo &TRI;
+    const ARMBaseInstrInfo &TII;
     MachineFunction *MF = nullptr;
     MachineInstr *InsertPt = nullptr;
     MachineInstr *Start = nullptr;
@@ -218,14 +231,20 @@ namespace {
     SetVector<MachineInstr*> CurrentPredicate;
     SmallVector<VPTBlock, 4> VPTBlocks;
     SmallPtrSet<MachineInstr*, 4> ToRemove;
+    SmallVector<std::unique_ptr<Reduction>, 1> Reductions;
     SmallPtrSet<MachineInstr*, 4> BlockMasksToRecompute;
     bool Revert = false;
     bool CannotTailPredicate = false;
 
     LowOverheadLoop(MachineLoop &ML, MachineLoopInfo &MLI,
-                    ReachingDefAnalysis &RDA, const TargetRegisterInfo &TRI)
-      : ML(ML), MLI(MLI), RDA(RDA), TRI(TRI) {
+                    ReachingDefAnalysis &RDA, const TargetRegisterInfo &TRI,
+                    const ARMBaseInstrInfo &TII)
+      : ML(ML), MLI(MLI), RDA(RDA), TRI(TRI), TII(TII) {
       MF = ML.getHeader()->getParent();
+      if (auto *MBB = ML.getLoopPreheader())
+        Preheader = MBB;
+      else if (auto *MBB = MLI.findLoopPreheader(&ML, true))
+        Preheader = MBB;
     }
 
     // If this is an MVE instruction, check that we know how to use tail
@@ -249,9 +268,13 @@ namespace {
     // of elements to the loop start instruction.
     bool ValidateTailPredicate(MachineInstr *StartInsertPt);
 
+    // See whether the live-out instructions are a reduction that we can fixup
+    // later.
+    bool FindValidReduction(InstSet &LiveMIs, InstSet &LiveOutUsers);
+
     // Check that any values available outside of the loop will be the same
     // after tail predication conversion.
-    bool ValidateLiveOuts() const;
+    bool ValidateLiveOuts();
 
     // Is it safe to define LR with DLS/WLS?
     // LR can be defined if it is the operand to start, because it's the same
@@ -341,6 +364,8 @@ namespace {
 
     void ConvertVPTBlocks(LowOverheadLoop &LoLoop);
 
+    void FixupReductions(LowOverheadLoop &LoLoop) const;
+
     MachineInstr *ExpandLoopStart(LowOverheadLoop &LoLoop);
 
     void Expand(LowOverheadLoop &LoLoop);
@@ -481,7 +506,7 @@ bool LowOverheadLoop::ValidateTailPredicate(MachineInstr *StartInsertPt) {
   };
 
   // First, find the block that looks like the preheader.
-  MachineBasicBlock *MBB = MLI.findLoopPreheader(&ML, true);
+  MachineBasicBlock *MBB = Preheader;
   if (!MBB) {
     LLVM_DEBUG(dbgs() << "ARM Loops: Didn't find preheader.\n");
     return false;
@@ -626,7 +651,109 @@ static bool producesFalseLanesZero(MachineInstr &MI,
   return true;
 }
 
-bool LowOverheadLoop::ValidateLiveOuts() const {
+bool
+LowOverheadLoop::FindValidReduction(InstSet &LiveMIs, InstSet &LiveOutUsers) {
+  // Also check for reductions where the operation needs to be merging values
+  // from the last and previous loop iterations. This means an instruction
+  // producing a value and a vmov storing the value calculated in the previous
+  // iteration. So we can have two live-out regs, one produced by a vmov and
+  // both being consumed by a vpsel.
+  LLVM_DEBUG(dbgs() << "ARM Loops: Looking for reduction live-outs:\n";
+             for (auto *MI : LiveMIs)
+               dbgs() << " - " << *MI);
+
+  if (!Preheader)
+    return false;
+
+  // Expect a vmov, a vadd and a single vpsel user.
+  // TODO: This means we can't currently support multiple reductions in the
+  // loop.
+  if (LiveMIs.size() != 2 || LiveOutUsers.size() != 1)
+    return false;
+
+  MachineInstr *VPSEL = *LiveOutUsers.begin();
+  if (VPSEL->getOpcode() != ARM::MVE_VPSEL)
+    return false;
+
+  unsigned VPRIdx = llvm::findFirstVPTPredOperandIdx(*VPSEL) + 1;
+  MachineInstr *Pred = RDA.getMIOperand(VPSEL, VPRIdx);
+  if (!Pred || Pred != VCTP) {
+    LLVM_DEBUG(dbgs() << "ARM Loops: Not using equivalent predicate.\n");
+    return false;
+  }
+
+  MachineInstr *Reduce = RDA.getMIOperand(VPSEL, 1);
+  if (!Reduce)
+    return false;
+
+  assert(LiveMIs.count(Reduce) && "Expected MI to be live-out");
+
+  // TODO: Support more operations than VADD.
+  switch (VCTP->getOpcode()) {
+  default:
+    return false;
+  case ARM::MVE_VCTP8:
+    if (Reduce->getOpcode() != ARM::MVE_VADDi8)
+      return false;
+    break;
+  case ARM::MVE_VCTP16:
+    if (Reduce->getOpcode() != ARM::MVE_VADDi16)
+      return false;
+    break;
+  case ARM::MVE_VCTP32:
+    if (Reduce->getOpcode() != ARM::MVE_VADDi32)
+      return false;
+    break;
+  }
+
+  // Test that the reduce op is overwriting ones of its operands.
+  if (Reduce->getOperand(0).getReg() != Reduce->getOperand(1).getReg() &&
+      Reduce->getOperand(0).getReg() != Reduce->getOperand(2).getReg()) {
+    LLVM_DEBUG(dbgs() << "ARM Loops: Reducing op isn't overwriting itself.\n");
+    return false;
+  }
+
+  // Check that the VORR is actually a VMOV.
+  MachineInstr *Copy = RDA.getMIOperand(VPSEL, 2);
+  if (!Copy || Copy->getOpcode() != ARM::MVE_VORR ||
+      !Copy->getOperand(1).isReg() || !Copy->getOperand(2).isReg() ||
+      Copy->getOperand(1).getReg() != Copy->getOperand(2).getReg())
+    return false;
+
+  assert(LiveMIs.count(Copy) && "Expected MI to be live-out");
+
+  // Check that the vadd and vmov are only used by each other and the vpsel.
+  SmallPtrSet<MachineInstr*, 2> CopyUsers;
+  RDA.getGlobalUses(Copy, Copy->getOperand(0).getReg(), CopyUsers);
+  if (CopyUsers.size() > 2 || !CopyUsers.count(Reduce)) {
+    LLVM_DEBUG(dbgs() << "ARM Loops: Copy users unsupported.\n");
+    return false;
+  }
+
+  SmallPtrSet<MachineInstr*, 2> ReduceUsers;
+  RDA.getGlobalUses(Reduce, Reduce->getOperand(0).getReg(), ReduceUsers);
+  if (ReduceUsers.size() > 2 || !ReduceUsers.count(Copy)) {
+    LLVM_DEBUG(dbgs() << "ARM Loops: Reduce users unsupported.\n");
+    return false;
+  }
+
+  // Then find whether there's an instruction initialising the register that
+  // is storing the reduction.
+  SmallPtrSet<MachineInstr*, 2> Incoming;
+  RDA.getLiveOuts(Preheader, Copy->getOperand(1).getReg(), Incoming);
+  if (Incoming.size() > 1)
+    return false;
+
+  MachineInstr *Init = Incoming.empty() ? nullptr : *Incoming.begin();
+  LLVM_DEBUG(dbgs() << "ARM Loops: Found a reduction:\n"
+             << " - " << *Copy
+             << " - " << *Reduce
+             << " - " << *VPSEL);
+  Reductions.push_back(std::make_unique<Reduction>(Init, Copy, Reduce, VPSEL));
+  return true;
+}
+
+bool LowOverheadLoop::ValidateLiveOuts() {
   // We want to find out if the tail-predicated version of this loop will
   // produce the same values as the loop in its original form. For this to
   // be true, the newly inserted implicit predication must not change the
@@ -652,9 +779,9 @@ bool LowOverheadLoop::ValidateLiveOuts() const {
   SetVector<MachineInstr *> FalseLanesUnknown;
   SmallPtrSet<MachineInstr *, 4> FalseLanesZero;
   SmallPtrSet<MachineInstr *, 4> Predicated;
-  MachineBasicBlock *MBB = ML.getHeader();
+  MachineBasicBlock *Header = ML.getHeader();
 
-  for (auto &MI : *MBB) {
+  for (auto &MI : *Header) {
     const MCInstrDesc &MCID = MI.getDesc();
     uint64_t Flags = MCID.TSFlags;
     if ((Flags & ARMII::DomainMask) != ARMII::DomainMVE)
@@ -702,6 +829,9 @@ bool LowOverheadLoop::ValidateLiveOuts() const {
   // stored and then we can work towards the leaves, hopefully adding more
   // instructions to Predicated. Successfully terminating the loop means that
   // all the unknown values have to found to be masked by predicated user(s).
+  // For any unpredicated values, we store them in NonPredicated so that we
+  // can later check whether these form a reduction.
+  SmallPtrSet<MachineInstr*, 2> NonPredicated;
   for (auto *MI : reverse(FalseLanesUnknown)) {
     for (auto &MO : MI->operands()) {
       if (!isRegInClass(MO, QPRs) || !MO.isDef())
@@ -709,39 +839,48 @@ bool LowOverheadLoop::ValidateLiveOuts() const {
       if (!HasPredicatedUsers(MI, MO, Predicated)) {
         LLVM_DEBUG(dbgs() << "ARM Loops: Found an unknown def of : "
                           << TRI.getRegAsmName(MO.getReg()) << " at " << *MI);
-        return false;
+        NonPredicated.insert(MI);
+        continue;
       }
     }
     // Any unknown false lanes have been masked away by the user(s).
     Predicated.insert(MI);
   }
 
-  // Collect Q-regs that are live in the exit blocks. We don't collect scalars
-  // because they won't be affected by lane predication.
-  SmallSet<Register, 2> LiveOuts;
+  SmallPtrSet<MachineInstr *, 2> LiveOutMIs;
+  SmallPtrSet<MachineInstr*, 2> LiveOutUsers;
   SmallVector<MachineBasicBlock *, 2> ExitBlocks;
   ML.getExitBlocks(ExitBlocks);
-  for (auto *MBB : ExitBlocks)
-    for (const MachineBasicBlock::RegisterMaskPair &RegMask : MBB->liveins())
-      if (QPRs->contains(RegMask.PhysReg))
-        LiveOuts.insert(RegMask.PhysReg);
-
-  // Collect the instructions in the loop body that define the live-out values.
-  SmallPtrSet<MachineInstr *, 2> LiveMIs;
   assert(ML.getNumBlocks() == 1 && "Expected single block loop!");
-  for (auto Reg : LiveOuts)
-    if (auto *MI = RDA.getLocalLiveOutMIDef(MBB, Reg))
-      LiveMIs.insert(MI);
+  assert(ExitBlocks.size() == 1 && "Expected a single exit block");
+  MachineBasicBlock *ExitBB = ExitBlocks.front();
+  for (const MachineBasicBlock::RegisterMaskPair &RegMask : ExitBB->liveins()) {
+    // Check Q-regs that are live in the exit blocks. We don't collect scalars
+    // because they won't be affected by lane predication.
+    if (QPRs->contains(RegMask.PhysReg)) {
+      if (auto *MI = RDA.getLocalLiveOutMIDef(Header, RegMask.PhysReg))
+        LiveOutMIs.insert(MI);
+      RDA.getLiveInUses(ExitBB, RegMask.PhysReg, LiveOutUsers);
+    }
+  }
+
+  // If we have any non-predicated live-outs, they need to be part of a
+  // reduction that we can fixup later. The reduction that the form of an
+  // operation that uses its previous values through a vmov and then a vpsel
+  // resides in the exit blocks to select the final bytes from n and n-1
+  // iterations.
+  if (!NonPredicated.empty() &&
+      !FindValidReduction(NonPredicated, LiveOutUsers))
+    return false;
 
-  LLVM_DEBUG(dbgs() << "ARM Loops: Found loop live-outs:\n";
-             for (auto *MI : LiveMIs)
-               dbgs() << " - " << *MI);
   // We've already validated that any VPT predication within the loop will be
   // equivalent when we perform the predication transformation; so we know that
   // any VPT predicated instruction is predicated upon VCTP. Any live-out
-  // instruction needs to be predicated, so check this here.
-  for (auto *MI : LiveMIs)
-    if (!isVectorPredicated(MI))
+  // instruction needs to be predicated, so check this here. The instructions
+  // in NonPredicated have been found to be a reduction that we can ensure its
+  // legality.
+  for (auto *MI : LiveOutMIs)
+    if (!isVectorPredicated(MI) && !NonPredicated.count(MI))
       return false;
 
   return true;
@@ -949,14 +1088,12 @@ bool ARMLowOverheadLoops::ProcessLoop(MachineLoop *ML) {
     return nullptr;
   };
 
-  LowOverheadLoop LoLoop(*ML, *MLI, *RDA, *TRI);
+  LowOverheadLoop LoLoop(*ML, *MLI, *RDA, *TRI, *TII);
   // Search the preheader for the start intrinsic.
   // FIXME: I don't see why we shouldn't be supporting multiple predecessors
   // with potentially multiple set.loop.iterations, so we need to enable this.
-  if (auto *Preheader = ML->getLoopPreheader())
-    LoLoop.Start = SearchForStart(Preheader);
-  else if (auto *Preheader = MLI->findLoopPreheader(ML, true))
-    LoLoop.Start = SearchForStart(Preheader);
+  if (LoLoop.Preheader)
+    LoLoop.Start = SearchForStart(LoLoop.Preheader);
   else
     return false;
 
@@ -1210,6 +1347,61 @@ MachineInstr* ARMLowOverheadLoops::ExpandLoopStart(LowOverheadLoop &LoLoop) {
   return &*MIB;
 }
 
+void ARMLowOverheadLoops::FixupReductions(LowOverheadLoop &LoLoop) const {
+  LLVM_DEBUG(dbgs() << "ARM Loops: Fixing up reduction(s).\n");
+  auto BuildMov = [this](MachineInstr &InsertPt, Register To, Register From) {
+    MachineBasicBlock *MBB = InsertPt.getParent();
+    MachineInstrBuilder MIB =
+      BuildMI(*MBB, &InsertPt, InsertPt.getDebugLoc(), TII->get(ARM::MVE_VORR));
+    MIB.addDef(To);
+    MIB.addReg(From);
+    MIB.addReg(From);
+    MIB.addImm(0);
+    MIB.addReg(0);
+    MIB.addReg(To);
+    LLVM_DEBUG(dbgs() << "ARM Loops: Inserted VMOV: " << *MIB);
+  };
+
+  for (auto &Reduction : LoLoop.Reductions) {
+    MachineInstr &Copy = Reduction->Copy;
+    MachineInstr &Reduce = Reduction->Reduce;
+    Register DestReg = Copy.getOperand(0).getReg();
+
+    // Change the initialiser if present
+    if (Reduction->Init) {
+      MachineInstr *Init = Reduction->Init;
+
+      for (unsigned i = 0; i < Init->getNumOperands(); ++i) {
+        MachineOperand &MO = Init->getOperand(i);
+        if (MO.isReg() && MO.isUse() && MO.isTied() &&
+            Init->findTiedOperandIdx(i) == 0)
+          Init->getOperand(i).setReg(DestReg);
+      }
+      Init->getOperand(0).setReg(DestReg);
+      LLVM_DEBUG(dbgs() << "ARM Loops: Changed init regs: " << *Init);
+    } else
+      BuildMov(LoLoop.Preheader->instr_back(), DestReg, Copy.getOperand(1).getReg());
+
+    // Change the reducing op to write to the register that is used to copy
+    // its value on the next iteration. Also update the tied-def operand.
+    Reduce.getOperand(0).setReg(DestReg);
+    Reduce.getOperand(5).setReg(DestReg);
+    LLVM_DEBUG(dbgs() << "ARM Loops: Changed reduction regs: " << Reduce);
+
+    // Instead of a vpsel, just copy the register into the necessary one.
+    MachineInstr &VPSEL = Reduction->VPSEL;
+    if (VPSEL.getOperand(0).getReg() != DestReg)
+      BuildMov(VPSEL, VPSEL.getOperand(0).getReg(), DestReg);
+
+    // Remove the unnecessary instructions.
+    LLVM_DEBUG(dbgs() << "ARM Loops: Removing:\n"
+               << " - " << Copy
+               << " - " << VPSEL << "\n");
+    Copy.eraseFromParent();
+    VPSEL.eraseFromParent();
+  }
+}
+
 void ARMLowOverheadLoops::ConvertVPTBlocks(LowOverheadLoop &LoLoop) {
   auto RemovePredicate = [](MachineInstr *MI) {
     LLVM_DEBUG(dbgs() << "ARM Loops: Removing predicate from: " << *MI);
@@ -1363,8 +1555,10 @@ void ARMLowOverheadLoops::Expand(LowOverheadLoop &LoLoop) {
     RemoveDeadBranch(LoLoop.Start);
     LoLoop.End = ExpandLoopEnd(LoLoop);
     RemoveDeadBranch(LoLoop.End);
-    if (LoLoop.IsTailPredicationLegal())
+    if (LoLoop.IsTailPredicationLegal()) {
       ConvertVPTBlocks(LoLoop);
+      FixupReductions(LoLoop);
+    }
     for (auto *I : LoLoop.ToRemove) {
       LLVM_DEBUG(dbgs() << "ARM Loops: Erasing " << *I);
       I->eraseFromParent();
diff --git a/llvm/lib/Target/ARM/ARMPredicates.td b/llvm/lib/Target/ARM/ARMPredicates.td
index 19106265c6d11..1ae71be9f7606 100644
--- a/llvm/lib/Target/ARM/ARMPredicates.td
+++ b/llvm/lib/Target/ARM/ARMPredicates.td
@@ -44,6 +44,9 @@ def HasFPRegs        : Predicate<"Subtarget->hasFPRegs()">,
 def HasFPRegs16      : Predicate<"Subtarget->hasFPRegs16()">,
                                  AssemblerPredicate<(all_of FeatureFPRegs16),
                                                     "16-bit fp registers">;
+def HasNoFPRegs16    : Predicate<"!Subtarget->hasFPRegs16()">,
+                                 AssemblerPredicate<(all_of (not FeatureFPRegs16)),
+                                                    "16-bit fp registers">;
 def HasFPRegs64      : Predicate<"Subtarget->hasFPRegs64()">,
                                  AssemblerPredicate<(all_of FeatureFPRegs64),
                                                     "64-bit fp registers">;
diff --git a/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp b/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp
index cade06e8c109b..7e06229b60c3e 100644
--- a/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp
@@ -126,24 +126,24 @@ SDValue ARMSelectionDAGInfo::EmitSpecializedLibcall(
 
 SDValue ARMSelectionDAGInfo::EmitTargetCodeForMemcpy(
     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
-    SDValue Size, unsigned Align, bool isVolatile, bool AlwaysInline,
+    SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
     MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
   const ARMSubtarget &Subtarget =
       DAG.getMachineFunction().getSubtarget<ARMSubtarget>();
   // Do repeated 4-byte loads and stores. To be improved.
   // This requires 4-byte alignment.
-  if ((Align & 3) != 0)
+  if (Alignment < Align(4))
     return SDValue();
   // This requires the copy size to be a constant, preferably
   // within a subtarget-specific limit.
   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
   if (!ConstantSize)
-    return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size, Align,
-                                  RTLIB::MEMCPY);
+    return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,
+                                  Alignment.value(), RTLIB::MEMCPY);
   uint64_t SizeVal = ConstantSize->getZExtValue();
   if (!AlwaysInline && SizeVal > Subtarget.getMaxInlineSizeThreshold())
-    return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size, Align,
-                                  RTLIB::MEMCPY);
+    return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,
+                                  Alignment.value(), RTLIB::MEMCPY);
 
   unsigned BytesLeft = SizeVal & 3;
   unsigned NumMemOps = SizeVal >> 2;
@@ -240,16 +240,16 @@ SDValue ARMSelectionDAGInfo::EmitTargetCodeForMemcpy(
 
 SDValue ARMSelectionDAGInfo::EmitTargetCodeForMemmove(
     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
-    SDValue Size, unsigned Align, bool isVolatile,
+    SDValue Size, Align Alignment, bool isVolatile,
     MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
-  return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size, Align,
-                                RTLIB::MEMMOVE);
+  return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,
+                                Alignment.value(), RTLIB::MEMMOVE);
 }
 
 SDValue ARMSelectionDAGInfo::EmitTargetCodeForMemset(
     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
-    SDValue Size, unsigned Align, bool isVolatile,
+    SDValue Size, Align Alignment, bool isVolatile,
     MachinePointerInfo DstPtrInfo) const {
-  return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size, Align,
-                                RTLIB::MEMSET);
+  return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,
+                                Alignment.value(), RTLIB::MEMSET);
 }
diff --git a/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h b/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h
index b8a86ae7310fa..7aa831c09248d 100644
--- a/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h
+++ b/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h
@@ -39,22 +39,22 @@ class ARMSelectionDAGInfo : public SelectionDAGTargetInfo {
 public:
   SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
                                   SDValue Chain, SDValue Dst, SDValue Src,
-                                  SDValue Size, unsigned Align, bool isVolatile,
-                                  bool AlwaysInline,
+                                  SDValue Size, Align Alignment,
+                                  bool isVolatile, bool AlwaysInline,
                                   MachinePointerInfo DstPtrInfo,
                                   MachinePointerInfo SrcPtrInfo) const override;
 
   SDValue
   EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
                            SDValue Dst, SDValue Src, SDValue Size,
-                           unsigned Align, bool isVolatile,
+                           Align Alignment, bool isVolatile,
                            MachinePointerInfo DstPtrInfo,
                            MachinePointerInfo SrcPtrInfo) const override;
 
   // Adjust parameters for memset, see RTABI section 4.3.4
   SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl,
                                   SDValue Chain, SDValue Op1, SDValue Op2,
-                                  SDValue Op3, unsigned Align, bool isVolatile,
+                                  SDValue Op3, Align Alignment, bool isVolatile,
                                   MachinePointerInfo DstPtrInfo) const override;
 
   SDValue EmitSpecializedLibcall(SelectionDAG &DAG, const SDLoc &dl,
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.cpp b/llvm/lib/Target/ARM/ARMSubtarget.cpp
index e87b248123f67..46802037c2aa3 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.cpp
+++ b/llvm/lib/Target/ARM/ARMSubtarget.cpp
@@ -292,12 +292,15 @@ void ARMSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   case CortexA73:
   case CortexA75:
   case CortexA76:
+  case CortexA77:
+  case CortexA78:
   case CortexR4:
   case CortexR4F:
   case CortexR5:
   case CortexR7:
   case CortexM3:
   case CortexR52:
+  case CortexX1:
     break;
   case Exynos:
     LdStMultipleTiming = SingleIssuePlusExtras;
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h
index 5667652058990..2703e385dd818 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.h
+++ b/llvm/lib/Target/ARM/ARMSubtarget.h
@@ -61,6 +61,8 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
     CortexA73,
     CortexA75,
     CortexA76,
+    CortexA77,
+    CortexA78,
     CortexA8,
     CortexA9,
     CortexM3,
@@ -69,6 +71,7 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
     CortexR5,
     CortexR52,
     CortexR7,
+    CortexX1,
     Exynos,
     Krait,
     Kryo,
diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.cpp b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
index 538ec446e0d88..9ead5fa4308c3 100644
--- a/llvm/lib/Target/ARM/ARMTargetMachine.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
@@ -518,9 +518,12 @@ void ARMPassConfig::addPreSched2() {
   addPass(createARMExpandPseudoPass());
 
   if (getOptLevel() != CodeGenOpt::None) {
-    // in v8, IfConversion depends on Thumb instruction widths
+    // When optimising for size, always run the Thumb2SizeReduction pass before
+    // IfConversion. Otherwise, check whether IT blocks are restricted
+    // (e.g. in v8, IfConversion depends on Thumb instruction widths)
     addPass(createThumb2SizeReductionPass([this](const Function &F) {
-      return this->TM->getSubtarget<ARMSubtarget>(F).restrictIT();
+      return this->TM->getSubtarget<ARMSubtarget>(F).hasMinSize() ||
+             this->TM->getSubtarget<ARMSubtarget>(F).restrictIT();
     }));
 
     addPass(createIfConverter([](const MachineFunction &MF) {
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 680f239c752c2..74b1331216a05 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -180,21 +180,6 @@ int ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
     return Cost;
   };
 
-  // Single to/from double precision conversions.
-  static const CostTblEntry NEONFltDblTbl[] = {
-    // Vector fptrunc/fpext conversions.
-    { ISD::FP_ROUND,   MVT::v2f64, 2 },
-    { ISD::FP_EXTEND,  MVT::v2f32, 2 },
-    { ISD::FP_EXTEND,  MVT::v4f32, 4 }
-  };
-
-  if (Src->isVectorTy() && ST->hasNEON() && (ISD == ISD::FP_ROUND ||
-                                          ISD == ISD::FP_EXTEND)) {
-    std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Src);
-    if (const auto *Entry = CostTableLookup(NEONFltDblTbl, ISD, LT.second))
-      return AdjustCost(LT.first * Entry->Cost);
-  }
-
   EVT SrcTy = TLI->getValueType(DL, Src);
   EVT DstTy = TLI->getValueType(DL, Dst);
 
@@ -228,12 +213,62 @@ int ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
         {ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i8, 0},
         {ISD::SIGN_EXTEND, MVT::v8i16, MVT::v8i8, 0},
         {ISD::ZERO_EXTEND, MVT::v8i16, MVT::v8i8, 0},
+        // The following extend from a legal type to an illegal type, so need to
+        // split the load. This introduced an extra load operation, but the
+        // extend is still "free".
+        {ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i16, 1},
+        {ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i16, 1},
+        {ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i8, 3},
+        {ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i8, 3},
+        {ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8, 1},
+        {ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8, 1},
     };
     if (SrcTy.isVector() && ST->hasMVEIntegerOps()) {
       if (const auto *Entry =
               ConvertCostTableLookup(MVELoadConversionTbl, ISD,
                                      DstTy.getSimpleVT(), SrcTy.getSimpleVT()))
-        return AdjustCost(Entry->Cost);
+        return AdjustCost(Entry->Cost * ST->getMVEVectorCostFactor());
+    }
+
+    static const TypeConversionCostTblEntry MVEFLoadConversionTbl[] = {
+        // FPExtends are similar but also require the VCVT instructions.
+        {ISD::FP_EXTEND, MVT::v4f32, MVT::v4f16, 1},
+        {ISD::FP_EXTEND, MVT::v8f32, MVT::v8f16, 3},
+    };
+    if (SrcTy.isVector() && ST->hasMVEFloatOps()) {
+      if (const auto *Entry =
+              ConvertCostTableLookup(MVEFLoadConversionTbl, ISD,
+                                     DstTy.getSimpleVT(), SrcTy.getSimpleVT()))
+        return AdjustCost(Entry->Cost * ST->getMVEVectorCostFactor());
+    }
+  }
+
+  // The truncate of a store is free. This is the mirror of extends above.
+  if (I && I->hasOneUse() && isa<StoreInst>(*I->user_begin())) {
+    static const TypeConversionCostTblEntry MVELoadConversionTbl[] = {
+        {ISD::TRUNCATE, MVT::v4i32, MVT::v4i16, 0},
+        {ISD::TRUNCATE, MVT::v4i32, MVT::v4i8, 0},
+        {ISD::TRUNCATE, MVT::v8i16, MVT::v8i8, 0},
+        {ISD::TRUNCATE, MVT::v8i32, MVT::v8i16, 1},
+        {ISD::TRUNCATE, MVT::v16i32, MVT::v16i8, 3},
+        {ISD::TRUNCATE, MVT::v16i16, MVT::v16i8, 1},
+    };
+    if (SrcTy.isVector() && ST->hasMVEIntegerOps()) {
+      if (const auto *Entry =
+              ConvertCostTableLookup(MVELoadConversionTbl, ISD, SrcTy.getSimpleVT(),
+                                     DstTy.getSimpleVT()))
+        return AdjustCost(Entry->Cost * ST->getMVEVectorCostFactor());
+    }
+
+    static const TypeConversionCostTblEntry MVEFLoadConversionTbl[] = {
+        {ISD::FP_ROUND, MVT::v4f32, MVT::v4f16, 1},
+        {ISD::FP_ROUND, MVT::v8f32, MVT::v8f16, 3},
+    };
+    if (SrcTy.isVector() && ST->hasMVEFloatOps()) {
+      if (const auto *Entry =
+              ConvertCostTableLookup(MVEFLoadConversionTbl, ISD, SrcTy.getSimpleVT(),
+                                     DstTy.getSimpleVT()))
+        return AdjustCost(Entry->Cost * ST->getMVEVectorCostFactor());
     }
   }
 
@@ -264,6 +299,23 @@ int ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
     }
   }
 
+  // Single to/from double precision conversions.
+  if (Src->isVectorTy() && ST->hasNEON() &&
+      ((ISD == ISD::FP_ROUND && SrcTy.getScalarType() == MVT::f64 &&
+        DstTy.getScalarType() == MVT::f32) ||
+       (ISD == ISD::FP_EXTEND && SrcTy.getScalarType() == MVT::f32 &&
+        DstTy.getScalarType() == MVT::f64))) {
+    static const CostTblEntry NEONFltDblTbl[] = {
+        // Vector fptrunc/fpext conversions.
+        {ISD::FP_ROUND, MVT::v2f64, 2},
+        {ISD::FP_EXTEND, MVT::v2f32, 2},
+        {ISD::FP_EXTEND, MVT::v4f32, 4}};
+
+    std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Src);
+    if (const auto *Entry = CostTableLookup(NEONFltDblTbl, ISD, LT.second))
+      return AdjustCost(LT.first * Entry->Cost);
+  }
+
   // Some arithmetic, load and store operations have specific instructions
   // to cast up/down their types automatically at no extra cost.
   // TODO: Get these tables to know at least what the related operations are.
@@ -443,6 +495,27 @@ int ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
       return AdjustCost(Entry->Cost * ST->getMVEVectorCostFactor());
   }
 
+  if (ISD == ISD::FP_ROUND || ISD == ISD::FP_EXTEND) {
+    // As general rule, fp converts that were not matched above are scalarized
+    // and cost 1 vcvt for each lane, so long as the instruction is available.
+    // If not it will become a series of function calls.
+    const int CallCost = getCallInstrCost(nullptr, Dst, {Src}, CostKind);
+    int Lanes = 1;
+    if (SrcTy.isFixedLengthVector())
+      Lanes = SrcTy.getVectorNumElements();
+    auto IsLegal = [this](EVT VT) {
+      EVT EltVT = VT.getScalarType();
+      return (EltVT == MVT::f32 && ST->hasVFP2Base()) ||
+             (EltVT == MVT::f64 && ST->hasFP64()) ||
+             (EltVT == MVT::f16 && ST->hasFullFP16());
+    };
+
+    if (IsLegal(SrcTy) && IsLegal(DstTy))
+      return Lanes;
+    else
+      return Lanes * CallCost;
+  }
+
   // Scalar integer conversion costs.
   static const TypeConversionCostTblEntry ARMIntegerConversionTbl[] = {
     // i16 -> i64 requires two dependent operations.
@@ -580,7 +653,7 @@ bool ARMTTIImpl::isProfitableLSRChainElement(Instruction *I) {
   return false;
 }
 
-bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, MaybeAlign Alignment) {
+bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment) {
   if (!EnableMaskedLoadStores || !ST->hasMVEIntegerOps())
     return false;
 
@@ -596,12 +669,11 @@ bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, MaybeAlign Alignment) {
   }
 
   unsigned EltWidth = DataTy->getScalarSizeInBits();
-  return (EltWidth == 32 && (!Alignment || *Alignment >= 4)) ||
-         (EltWidth == 16 && (!Alignment || *Alignment >= 2)) ||
-         (EltWidth == 8);
+  return (EltWidth == 32 && Alignment >= 4) ||
+         (EltWidth == 16 && Alignment >= 2) || (EltWidth == 8);
 }
 
-bool ARMTTIImpl::isLegalMaskedGather(Type *Ty, MaybeAlign Alignment) {
+bool ARMTTIImpl::isLegalMaskedGather(Type *Ty, Align Alignment) {
   if (!EnableMaskedGatherScatters || !ST->hasMVEIntegerOps())
     return false;
 
@@ -618,8 +690,8 @@ bool ARMTTIImpl::isLegalMaskedGather(Type *Ty, MaybeAlign Alignment) {
     return false;
 
   unsigned EltWidth = Ty->getScalarSizeInBits();
-  return ((EltWidth == 32 && (!Alignment || *Alignment >= 4)) ||
-          (EltWidth == 16 && (!Alignment || *Alignment >= 2)) || EltWidth == 8);
+  return ((EltWidth == 32 && Alignment >= 4) ||
+          (EltWidth == 16 && Alignment >= 2) || EltWidth == 8);
 }
 
 int ARMTTIImpl::getMemcpyCost(const Instruction *I) {
@@ -893,29 +965,45 @@ int ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
     return 1;
 
   // Type legalization can't handle structs
-  if (TLI->getValueType(DL, Src,  true) == MVT::Other)
+  if (TLI->getValueType(DL, Src, true) == MVT::Other)
     return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
                                   CostKind);
 
-  std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Src);
-
   if (ST->hasNEON() && Src->isVectorTy() &&
       (Alignment && *Alignment != Align(16)) &&
       cast<VectorType>(Src)->getElementType()->isDoubleTy()) {
     // Unaligned loads/stores are extremely inefficient.
     // We need 4 uops for vst.1/vld.1 vs 1uop for vldr/vstr.
+    std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Src);
     return LT.first * 4;
   }
+
+  // MVE can optimize a fpext(load(4xhalf)) using an extending integer load.
+  // Same for stores.
+  if (ST->hasMVEFloatOps() && isa<FixedVectorType>(Src) && I &&
+      ((Opcode == Instruction::Load && I->hasOneUse() &&
+        isa<FPExtInst>(*I->user_begin())) ||
+       (Opcode == Instruction::Store && isa<FPTruncInst>(I->getOperand(0))))) {
+    FixedVectorType *SrcVTy = cast<FixedVectorType>(Src);
+    Type *DstTy =
+        Opcode == Instruction::Load
+            ? (*I->user_begin())->getType()
+            : cast<Instruction>(I->getOperand(0))->getOperand(0)->getType();
+    if (SrcVTy->getNumElements() == 4 && SrcVTy->getScalarType()->isHalfTy() &&
+        DstTy->getScalarType()->isFloatTy())
+      return ST->getMVEVectorCostFactor();
+  }
+
   int BaseCost = ST->hasMVEIntegerOps() && Src->isVectorTy()
                      ? ST->getMVEVectorCostFactor()
                      : 1;
-  return BaseCost * LT.first;
+  return BaseCost * BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
+                                           CostKind, I);
 }
 
 int ARMTTIImpl::getInterleavedMemoryOpCost(
     unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
-    unsigned Alignment, unsigned AddressSpace,
-    TTI::TargetCostKind CostKind,
+    Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
     bool UseMaskForCond, bool UseMaskForGaps) {
   assert(Factor >= 2 && "Invalid interleave factor");
   assert(isa<VectorType>(VecTy) && "Expect a vector type");
@@ -953,8 +1041,8 @@ int ARMTTIImpl::getInterleavedMemoryOpCost(
 }
 
 unsigned ARMTTIImpl::getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
-                                            Value *Ptr, bool VariableMask,
-                                            unsigned Alignment,
+                                            const Value *Ptr, bool VariableMask,
+                                            Align Alignment,
                                             TTI::TargetCostKind CostKind,
                                             const Instruction *I) {
   using namespace PatternMatch;
@@ -1032,9 +1120,9 @@ unsigned ARMTTIImpl::getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
   if (ExtSize != 8 && ExtSize != 16)
     return ScalarCost;
 
-  if (auto BC = dyn_cast<BitCastInst>(Ptr))
+  if (const auto *BC = dyn_cast<BitCastInst>(Ptr))
     Ptr = BC->getOperand(0);
-  if (auto *GEP = dyn_cast<GetElementPtrInst>(Ptr)) {
+  if (const auto *GEP = dyn_cast<GetElementPtrInst>(Ptr)) {
     if (GEP->getNumOperands() != 2)
       return ScalarCost;
     unsigned Scale = DL.getTypeAllocSize(GEP->getResultElementType());
@@ -1042,7 +1130,7 @@ unsigned ARMTTIImpl::getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
     if (Scale != 1 && Scale * 8 != ExtSize)
       return ScalarCost;
     // And we need to zext (not sext) the indexes from a small enough type.
-    if (auto ZExt = dyn_cast<ZExtInst>(GEP->getOperand(1))) {
+    if (const auto *ZExt = dyn_cast<ZExtInst>(GEP->getOperand(1))) {
       if (ZExt->getOperand(0)->getType()->getScalarSizeInBits() <= ExtSize)
         return VectorCost;
     }
@@ -1494,6 +1582,11 @@ void ARMTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
     UP.Force = true;
 }
 
+void ARMTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                                       TTI::PeelingPreferences &PP) {
+  BaseT::getPeelingPreferences(L, SE, PP);
+}
+
 bool ARMTTIImpl::useReductionIntrinsic(unsigned Opcode, Type *Ty,
                                        TTI::ReductionFlags Flags) const {
   return ST->hasMVEIntegerOps();
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 7a2db740f185a..537a546361eeb 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -153,15 +153,15 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
 
   bool isProfitableLSRChainElement(Instruction *I);
 
-  bool isLegalMaskedLoad(Type *DataTy, MaybeAlign Alignment);
+  bool isLegalMaskedLoad(Type *DataTy, Align Alignment);
 
-  bool isLegalMaskedStore(Type *DataTy, MaybeAlign Alignment) {
+  bool isLegalMaskedStore(Type *DataTy, Align Alignment) {
     return isLegalMaskedLoad(DataTy, Alignment);
   }
 
-  bool isLegalMaskedGather(Type *Ty, MaybeAlign Alignment);
+  bool isLegalMaskedGather(Type *Ty, Align Alignment);
 
-  bool isLegalMaskedScatter(Type *Ty, MaybeAlign Alignment) {
+  bool isLegalMaskedScatter(Type *Ty, Align Alignment) {
     return isLegalMaskedGather(Ty, Alignment);
   }
 
@@ -224,17 +224,16 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
                       TTI::TargetCostKind CostKind,
                       const Instruction *I = nullptr);
 
-  int getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor,
-                                 ArrayRef<unsigned> Indices, unsigned Alignment,
-                                 unsigned AddressSpace,
-                                 TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
-                                 bool UseMaskForCond = false,
-                                 bool UseMaskForGaps = false);
+  int getInterleavedMemoryOpCost(
+      unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+      Align Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
+      bool UseMaskForCond = false, bool UseMaskForGaps = false);
 
-  unsigned getGatherScatterOpCost(
-    unsigned Opcode, Type *DataTy, Value *Ptr, bool VariableMask,
-    unsigned Alignment, TTI::TargetCostKind CostKind,
-    const Instruction *I = nullptr);
+  unsigned getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
+                                  const Value *Ptr, bool VariableMask,
+                                  Align Alignment, TTI::TargetCostKind CostKind,
+                                  const Instruction *I = nullptr);
 
   bool isLoweredToCall(const Function *F);
   bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
@@ -252,6 +251,8 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
 
   bool emitGetActiveLaneMask() const;
 
+  void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                             TTI::PeelingPreferences &PP);
   bool shouldBuildLookupTablesForConstant(Constant *C) const {
     // In the ROPI and RWPI relocation models we can't have pointers to global
     // variables or functions in constant data, so don't convert switches to
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
index 6ce5a19a5e708..05d73ccf6ff2a 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
@@ -63,6 +63,25 @@ static bool getMCRDeprecationInfo(MCInst &MI, const MCSubtargetInfo &STI,
       return true;
     }
   }
+  if (STI.getFeatureBits()[llvm::ARM::HasV7Ops] &&
+      ((MI.getOperand(0).isImm() && MI.getOperand(0).getImm() == 10) ||
+       (MI.getOperand(0).isImm() && MI.getOperand(0).getImm() == 11))) {
+    Info = "since v7, cp10 and cp11 are reserved for advanced SIMD or floating "
+           "point instructions";
+    return true;
+  }
+  return false;
+}
+
+static bool getMRCDeprecationInfo(MCInst &MI, const MCSubtargetInfo &STI,
+                                  std::string &Info) {
+  if (STI.getFeatureBits()[llvm::ARM::HasV7Ops] &&
+      ((MI.getOperand(0).isImm() && MI.getOperand(0).getImm() == 10) ||
+       (MI.getOperand(0).isImm() && MI.getOperand(0).getImm() == 11))) {
+    Info = "since v7, cp10 and cp11 are reserved for advanced SIMD or floating "
+           "point instructions";
+    return true;
+  }
   return false;
 }
 
diff --git a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
index 5a0c4ca4cb3db..4d7ad6cd60cb9 100644
--- a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
+++ b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
@@ -78,7 +78,7 @@ class MVEGatherScatterLowering : public FunctionPass {
 
   // Check this is a valid gather with correct alignment
   bool isLegalTypeAndAlignment(unsigned NumElements, unsigned ElemSize,
-                               unsigned Alignment);
+                               Align Alignment);
   // Check whether Ptr is hidden behind a bitcast and look through it
   void lookThroughBitcast(Value *&Ptr);
   // Check for a getelementptr and deduce base and offsets from it, on success
@@ -155,12 +155,12 @@ Pass *llvm::createMVEGatherScatterLoweringPass() {
 
 bool MVEGatherScatterLowering::isLegalTypeAndAlignment(unsigned NumElements,
                                                        unsigned ElemSize,
-                                                       unsigned Alignment) {
+                                                       Align Alignment) {
   if (((NumElements == 4 &&
         (ElemSize == 32 || ElemSize == 16 || ElemSize == 8)) ||
        (NumElements == 8 && (ElemSize == 16 || ElemSize == 8)) ||
        (NumElements == 16 && ElemSize == 8)) &&
-      ElemSize / 8 <= Alignment)
+      Alignment >= ElemSize / 8)
     return true;
   LLVM_DEBUG(dbgs() << "masked gathers/scatters: instruction does not have "
                     << "valid alignment or vector type \n");
@@ -306,7 +306,7 @@ Value *MVEGatherScatterLowering::lowerGather(IntrinsicInst *I) {
   // Potentially optimising the addressing modes as we do so.
   auto *Ty = cast<FixedVectorType>(I->getType());
   Value *Ptr = I->getArgOperand(0);
-  unsigned Alignment = cast<ConstantInt>(I->getArgOperand(1))->getZExtValue();
+  Align Alignment = cast<ConstantInt>(I->getArgOperand(1))->getAlignValue();
   Value *Mask = I->getArgOperand(2);
   Value *PassThru = I->getArgOperand(3);
 
@@ -466,7 +466,7 @@ Value *MVEGatherScatterLowering::lowerScatter(IntrinsicInst *I) {
   // Potentially optimising the addressing modes as we do so.
   Value *Input = I->getArgOperand(0);
   Value *Ptr = I->getArgOperand(1);
-  unsigned Alignment = cast<ConstantInt>(I->getArgOperand(2))->getZExtValue();
+  Align Alignment = cast<ConstantInt>(I->getArgOperand(2))->getAlignValue();
   auto *Ty = cast<FixedVectorType>(Input->getType());
 
   if (!isLegalTypeAndAlignment(Ty->getNumElements(), Ty->getScalarSizeInBits(),
@@ -495,7 +495,7 @@ Value *MVEGatherScatterLowering::tryCreateMaskedScatterBase(
     IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
   using namespace PatternMatch;
   Value *Input = I->getArgOperand(0);
-  auto *Ty = cast<VectorType>(Input->getType());
+  auto *Ty = cast<FixedVectorType>(Input->getType());
   // Only QR variants allow truncating
   if (!(Ty->getNumElements() == 4 && Ty->getScalarSizeInBits() == 32)) {
     // Can't build an intrinsic for this
@@ -519,7 +519,7 @@ Value *MVEGatherScatterLowering::tryCreateMaskedScatterBaseWB(
     IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
   using namespace PatternMatch;
   Value *Input = I->getArgOperand(0);
-  auto *Ty = cast<VectorType>(Input->getType());
+  auto *Ty = cast<FixedVectorType>(Input->getType());
   LLVM_DEBUG(
       dbgs()
       << "masked scatters: storing to a vector of pointers with writeback\n");
@@ -990,26 +990,27 @@ bool MVEGatherScatterLowering::runOnFunction(Function &F) {
   SmallVector<IntrinsicInst *, 4> Gathers;
   SmallVector<IntrinsicInst *, 4> Scatters;
 
+  bool Changed = false;
+
   for (BasicBlock &BB : F) {
     for (Instruction &I : BB) {
       IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I);
       if (II && II->getIntrinsicID() == Intrinsic::masked_gather) {
         Gathers.push_back(II);
         if (isa<GetElementPtrInst>(II->getArgOperand(0)))
-          optimiseOffsets(
+          Changed |= optimiseOffsets(
               cast<Instruction>(II->getArgOperand(0))->getOperand(1),
               II->getParent(), LI);
       } else if (II && II->getIntrinsicID() == Intrinsic::masked_scatter) {
         Scatters.push_back(II);
         if (isa<GetElementPtrInst>(II->getArgOperand(1)))
-          optimiseOffsets(
+          Changed |= optimiseOffsets(
               cast<Instruction>(II->getArgOperand(1))->getOperand(1),
               II->getParent(), LI);
       }
     }
   }
 
-  bool Changed = false;
   for (unsigned i = 0; i < Gathers.size(); i++) {
     IntrinsicInst *I = Gathers[i];
     Value *L = lowerGather(I);
diff --git a/llvm/lib/Target/ARM/MVETailPredication.cpp b/llvm/lib/Target/ARM/MVETailPredication.cpp
index 5c00c07a243a7..6583dcb77e1ed 100644
--- a/llvm/lib/Target/ARM/MVETailPredication.cpp
+++ b/llvm/lib/Target/ARM/MVETailPredication.cpp
@@ -46,6 +46,7 @@
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/IR/IRBuilder.h"
@@ -78,15 +79,9 @@ namespace {
 class MVETailPredication : public LoopPass {
   SmallVector<IntrinsicInst*, 4> MaskedInsts;
   Loop *L = nullptr;
-  LoopInfo *LI = nullptr;
-  const DataLayout *DL;
-  DominatorTree *DT = nullptr;
   ScalarEvolution *SE = nullptr;
   TargetTransformInfo *TTI = nullptr;
-  TargetLibraryInfo *TLI = nullptr;
-  bool ClonedVCTPInExitBlock = false;
-  IntrinsicInst *ActiveLaneMask = nullptr;
-  FixedVectorType *VecTy = nullptr;
+  const ARMSubtarget *ST = nullptr;
 
 public:
   static char ID;
@@ -98,8 +93,6 @@ class MVETailPredication : public LoopPass {
     AU.addRequired<LoopInfoWrapperPass>();
     AU.addRequired<TargetPassConfig>();
     AU.addRequired<TargetTransformInfoWrapperPass>();
-    AU.addRequired<DominatorTreeWrapperPass>();
-    AU.addRequired<TargetLibraryInfoWrapperPass>();
     AU.addPreserved<LoopInfoWrapperPass>();
     AU.setPreservesCFG();
   }
@@ -118,21 +111,17 @@ class MVETailPredication : public LoopPass {
   /// intrinsic: check if the first is a loop induction variable, and for the
   /// the second check that no overflow can occur in the expression that use
   /// this backedge-taken count.
-  bool IsSafeActiveMask(Value *TripCount, FixedVectorType *VecTy);
+  bool IsSafeActiveMask(IntrinsicInst *ActiveLaneMask, Value *TripCount,
+                        FixedVectorType *VecTy);
 
   /// Insert the intrinsic to represent the effect of tail predication.
   void InsertVCTPIntrinsic(IntrinsicInst *ActiveLaneMask, Value *TripCount,
-                           FixedVectorType *VecTy,
-                           DenseMap<Instruction *, Instruction *> &NewPredicates);
+                           FixedVectorType *VecTy);
 
   /// Rematerialize the iteration count in exit blocks, which enables
   /// ARMLowOverheadLoops to better optimise away loop update statements inside
   /// hardware-loops.
   void RematerializeIterCount();
-
-  /// If it is not safe to lower @llvm.get.active.lane.mask to a VCTP, it needs
-  /// to be lowered to an icmp.
-  void RevertActiveLaneMask();
 };
 
 } // end namespace
@@ -156,93 +145,6 @@ static bool IsMasked(Instruction *I) {
   return ID == Intrinsic::masked_store || ID == Intrinsic::masked_load;
 }
 
-void MVETailPredication::RematerializeIterCount() {
-  SmallVector<WeakTrackingVH, 16> DeadInsts;
-  SCEVExpander Rewriter(*SE, *DL, "mvetp");
-  ReplaceExitVal ReplaceExitValue = AlwaysRepl;
-
-  formLCSSARecursively(*L, *DT, LI, SE);
-  rewriteLoopExitValues(L, LI, TLI, SE, TTI, Rewriter, DT, ReplaceExitValue,
-                        DeadInsts);
-}
-
-void MVETailPredication::RevertActiveLaneMask() {
-  if (!ActiveLaneMask)
-    return;
-
-  int VectorWidth = VecTy->getElementCount().Min;
-  IRBuilder<> Builder(ActiveLaneMask->getParent()->getFirstNonPHI());
-
-  // 1. Create the vector induction step. This %induction will be the LHS of
-  // the icmp:
-  //
-  // %splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
-  // %splat = shufflevector <4 x i32> %splatinsert, <4 x i32> undef, <4 x i32> 0
-  // %induction = add <4 x i32> %splat, <i32 0, i32 1, i32 2, i32 3>
-  //
-  Value *Index = ActiveLaneMask->getOperand(0);
-  Value *SplatIndex =
-      Builder.CreateVectorSplat(VectorWidth, Index, "lane.mask");
-
-  SmallVector<Constant *, 8> Indices;
-  for (int i = 0; i < VectorWidth; ++i)
-    Indices.push_back(ConstantInt::get(Index->getType(), i));
-
-  Constant *CV = ConstantVector::get(Indices);
-  Value *Induction = Builder.CreateAdd(SplatIndex, CV, "lane.mask.induction");
-
-  LLVM_DEBUG(dbgs() << "ARM TP: New index: " << *SplatIndex << "\n";
-             dbgs() << "ARM TP: New Induction: " << *Induction << "\n");
-
-  // 2. In the Preheader, first look if the splat BTC already exists. Find this
-  //    %splat, which will be the RHS of the icmp:
-  //
-  //    %TC.minus.1 = add i32 %N, -1
-  //    %splatinsert = insertelement <4 x i32> undef, i32 %TC.minus.1, i32 0
-  //    %splat = shufflevector <4 x i32> %splatinsert, <4 x i32> undef, <16 x i32> 0
-  //
-  auto *Preheader = L->getLoopPreheader();
-  auto *BTC = ActiveLaneMask->getOperand(1);
-  Value *SplatBTC = nullptr;
-
-  if (auto *C = dyn_cast<ConstantInt>(BTC)) {
-    Builder.SetInsertPoint(Preheader->getTerminator());
-    SplatBTC = Builder.CreateVectorSplat(VectorWidth, C);
-    LLVM_DEBUG(dbgs() << "ARM TCP: New splat BTC: " << *SplatBTC << "\n");
-  } else {
-    Instruction *InsertElem;
-    for (auto &V : *Preheader) {
-      InsertElem = dyn_cast<InsertElementInst>(&V);
-      if (!InsertElem)
-        continue;
-      ConstantInt *CI = dyn_cast<ConstantInt>(InsertElem->getOperand(2));
-      if (!CI)
-        continue;
-      if (InsertElem->getOperand(1) != BTC || CI->getSExtValue() != 0)
-        continue;
-      if ((SplatBTC = dyn_cast<ShuffleVectorInst>(*InsertElem->users().begin())))
-         break;
-    }
-  }
-  // Or create the splat BTC if it doesn't exist.
-  if (!SplatBTC) {
-    Builder.SetInsertPoint(Preheader->getTerminator());
-    Value *Undef =
-        UndefValue::get(FixedVectorType::get(BTC->getType(), VectorWidth));
-    Value *Insert = Builder.CreateInsertElement(Undef,
-        BTC, Builder.getInt32(0), "insert.btc");
-    Value *Zero = ConstantInt::get(Insert->getType(), 0);
-    SplatBTC = Builder.CreateShuffleVector (Insert, Undef, Zero, "splat.btc");
-    LLVM_DEBUG(dbgs() << "ARM TCP: New splat BTC: " << *SplatBTC << "\n");
-  }
-
-  Builder.SetInsertPoint(ActiveLaneMask);
-  Value *ICmp = Builder.CreateICmp(ICmpInst::ICMP_ULE, Induction, SplatBTC);
-  LLVM_DEBUG(dbgs() << "ARM TP: New compare: " << *ICmp << "\n");
-  ActiveLaneMask->replaceAllUsesWith(ICmp);
-  ActiveLaneMask->eraseFromParent();
-}
-
 bool MVETailPredication::runOnLoop(Loop *L, LPPassManager&) {
   if (skipLoop(L) || DisableTailPredication)
     return false;
@@ -251,16 +153,10 @@ bool MVETailPredication::runOnLoop(Loop *L, LPPassManager&) {
   Function &F = *L->getHeader()->getParent();
   auto &TPC = getAnalysis<TargetPassConfig>();
   auto &TM = TPC.getTM<TargetMachine>();
-  auto *ST = &TM.getSubtarget<ARMSubtarget>(F);
-  DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
-  LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+  ST = &TM.getSubtarget<ARMSubtarget>(F);
   TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
   SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
-  auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
-  TLI = TLIP ? &TLIP->getTLI(*L->getHeader()->getParent()) : nullptr;
-  DL = &L->getHeader()->getModule()->getDataLayout();
   this->L = L;
-  ActiveLaneMask = nullptr;
 
   // The MVE and LOB extensions are combined to enable tail-predication, but
   // there's nothing preventing us from generating VCTP instructions for v8.1m.
@@ -313,19 +209,15 @@ bool MVETailPredication::runOnLoop(Loop *L, LPPassManager&) {
   if (!Decrement)
     return false;
 
-  ClonedVCTPInExitBlock = false;
   LLVM_DEBUG(dbgs() << "ARM TP: Running on Loop: " << *L << *Setup << "\n"
              << *Decrement << "\n");
 
-  if (TryConvert(Setup->getArgOperand(0))) {
-    if (ClonedVCTPInExitBlock)
-      RematerializeIterCount();
-    return true;
-  } else
-    RevertActiveLaneMask();
+  if (!TryConvert(Setup->getArgOperand(0))) {
+    LLVM_DEBUG(dbgs() << "ARM TP: Can't tail-predicate this loop.\n");
+    return false;
+  }
 
-  LLVM_DEBUG(dbgs() << "ARM TP: Can't tail-predicate this loop.\n");
-  return false;
+  return true;
 }
 
 static FixedVectorType *getVectorType(IntrinsicInst *I) {
@@ -340,10 +232,38 @@ bool MVETailPredication::IsPredicatedVectorLoop() {
   // Check that the loop contains at least one masked load/store intrinsic.
   // We only support 'normal' vector instructions - other than masked
   // load/stores.
+  bool ActiveLaneMask = false;
   for (auto *BB : L->getBlocks()) {
     for (auto &I : *BB) {
+      auto *Int = dyn_cast<IntrinsicInst>(&I);
+      if (!Int)
+        continue;
+
+      switch (Int->getIntrinsicID()) {
+      case Intrinsic::get_active_lane_mask:
+        ActiveLaneMask = true;
+        LLVM_FALLTHROUGH;
+      case Intrinsic::sadd_sat:
+      case Intrinsic::uadd_sat:
+      case Intrinsic::ssub_sat:
+      case Intrinsic::usub_sat:
+        continue;
+      case Intrinsic::fma:
+      case Intrinsic::trunc:
+      case Intrinsic::rint:
+      case Intrinsic::round:
+      case Intrinsic::floor:
+      case Intrinsic::ceil:
+      case Intrinsic::fabs:
+        if (ST->hasMVEFloatOps())
+          continue;
+        LLVM_FALLTHROUGH;
+      default:
+        break;
+      }
+
       if (IsMasked(&I)) {
-        FixedVectorType *VecTy = getVectorType(cast<IntrinsicInst>(&I));
+        auto *VecTy = getVectorType(Int);
         unsigned Lanes = VecTy->getNumElements();
         unsigned ElementWidth = VecTy->getScalarSizeInBits();
         // MVE vectors are 128-bit, but don't support 128 x i1.
@@ -352,17 +272,20 @@ bool MVETailPredication::IsPredicatedVectorLoop() {
         if (Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth)
           return false;
         MaskedInsts.push_back(cast<IntrinsicInst>(&I));
-      } else if (auto *Int = dyn_cast<IntrinsicInst>(&I)) {
-        if (Int->getIntrinsicID() == Intrinsic::fma)
-          continue;
-        for (auto &U : Int->args()) {
-          if (isa<VectorType>(U->getType()))
-            return false;
-        }
+        continue;
+      }
+
+      for (const Use &U : Int->args()) {
+        if (isa<VectorType>(U->getType()))
+          return false;
       }
     }
   }
 
+  if (!ActiveLaneMask) {
+    LLVM_DEBUG(dbgs() << "ARM TP: No get.active.lane.mask intrinsic found.\n");
+    return false;
+  }
   return !MaskedInsts.empty();
 }
 
@@ -373,32 +296,11 @@ bool MVETailPredication::IsPredicatedVectorLoop() {
 // in the block. This means that the VPR doesn't have to be live into the
 // exit block which should make it easier to convert this loop into a proper
 // tail predicated loop.
-static bool Cleanup(DenseMap<Instruction*, Instruction*> &NewPredicates,
-                    SetVector<Instruction*> &MaybeDead, Loop *L) {
+static void Cleanup(SetVector<Instruction*> &MaybeDead, Loop *L) {
   BasicBlock *Exit = L->getUniqueExitBlock();
   if (!Exit) {
     LLVM_DEBUG(dbgs() << "ARM TP: can't find loop exit block\n");
-    return false;
-  }
-
-  bool ClonedVCTPInExitBlock = false;
-
-  for (auto &Pair : NewPredicates) {
-    Instruction *OldPred = Pair.first;
-    Instruction *NewPred = Pair.second;
-
-    for (auto &I : *Exit) {
-      if (I.isSameOperationAs(OldPred)) {
-        Instruction *PredClone = NewPred->clone();
-        PredClone->insertBefore(&I);
-        I.replaceAllUsesWith(PredClone);
-        MaybeDead.insert(&I);
-        ClonedVCTPInExitBlock = true;
-        LLVM_DEBUG(dbgs() << "ARM TP: replacing: "; I.dump();
-                   dbgs() << "ARM TP: with:      "; PredClone->dump());
-        break;
-      }
-    }
+    return;
   }
 
   // Drop references and add operands to check for dead.
@@ -423,8 +325,6 @@ static bool Cleanup(DenseMap<Instruction*, Instruction*> &NewPredicates,
 
   for (auto I : L->blocks())
     DeleteDeadPHIs(I);
-
-  return ClonedVCTPInExitBlock;
 }
 
 // The active lane intrinsic has this form:
@@ -444,14 +344,15 @@ static bool Cleanup(DenseMap<Instruction*, Instruction*> &NewPredicates,
 //        (((ElementCount + (VectorWidth - 1)) / VectorWidth) - TripCount
 // 3) The IV must be an induction phi with an increment equal to the
 //    vector width.
-bool MVETailPredication::IsSafeActiveMask(Value *TripCount,
-    FixedVectorType *VecTy) {
+bool MVETailPredication::IsSafeActiveMask(IntrinsicInst *ActiveLaneMask,
+    Value *TripCount, FixedVectorType *VecTy) {
   // 1) Test whether entry to the loop is protected by a conditional
   // BTC + 1 < 0. In other words, if the scalar trip count overflows,
   // becomes negative, we shouldn't enter the loop and creating
   // tripcount expression BTC + 1 is not safe. So, check that BTC
   // isn't max. This is evaluated in unsigned, because the semantics
   // of @get.active.lane.mask is a ULE comparison.
+
   int VectorWidth = VecTy->getNumElements();
   auto *BackedgeTakenCount = ActiveLaneMask->getOperand(1);
   auto *BTC = SE->getSCEV(BackedgeTakenCount);
@@ -563,8 +464,8 @@ bool MVETailPredication::IsSafeActiveMask(Value *TripCount,
   if (VectorWidth == StepValue)
     return true;
 
-  LLVM_DEBUG(dbgs() << "ARM TP: step value " << StepValue << " doesn't match "
-             "vector width : " << VectorWidth << "\n");
+  LLVM_DEBUG(dbgs() << "ARM TP: Step value " << StepValue << " doesn't match "
+             "vector width " << VectorWidth << "\n");
 
   return false;
 }
@@ -602,11 +503,11 @@ static Value *getNumElements(BasicBlock *Preheader, Value *BTC) {
 }
 
 void MVETailPredication::InsertVCTPIntrinsic(IntrinsicInst *ActiveLaneMask,
-    Value *TripCount, FixedVectorType *VecTy,
-    DenseMap<Instruction*, Instruction*> &NewPredicates) {
+    Value *TripCount, FixedVectorType *VecTy) {
   IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
   Module *M = L->getHeader()->getModule();
   Type *Ty = IntegerType::get(M->getContext(), 32);
+  unsigned VectorWidth = VecTy->getNumElements();
 
   // The backedge-taken count in @llvm.get.active.lane.mask, its 2nd operand,
   // is one less than the trip count. So we need to find or create
@@ -624,10 +525,10 @@ void MVETailPredication::InsertVCTPIntrinsic(IntrinsicInst *ActiveLaneMask,
   // represent the effect of tail predication.
   Builder.SetInsertPoint(ActiveLaneMask);
   ConstantInt *Factor =
-    ConstantInt::get(cast<IntegerType>(Ty), VecTy->getNumElements());
+    ConstantInt::get(cast<IntegerType>(Ty), VectorWidth);
 
   Intrinsic::ID VCTPID;
-  switch (VecTy->getNumElements()) {
+  switch (VectorWidth) {
   default:
     llvm_unreachable("unexpected number of lanes");
   case 4:  VCTPID = Intrinsic::arm_mve_vctp32; break;
@@ -643,7 +544,6 @@ void MVETailPredication::InsertVCTPIntrinsic(IntrinsicInst *ActiveLaneMask,
   Function *VCTP = Intrinsic::getDeclaration(M, VCTPID);
   Value *VCTPCall = Builder.CreateCall(VCTP, Processed);
   ActiveLaneMask->replaceAllUsesWith(VCTPCall);
-  NewPredicates[ActiveLaneMask] = cast<Instruction>(VCTPCall);
 
   // Add the incoming value to the new phi.
   // TODO: This add likely already exists in the loop.
@@ -661,9 +561,7 @@ bool MVETailPredication::TryConvert(Value *TripCount) {
   }
 
   LLVM_DEBUG(dbgs() << "ARM TP: Found predicated vector loop.\n");
-
   SetVector<Instruction*> Predicates;
-  DenseMap<Instruction*, Instruction*> NewPredicates;
 
   // Walk through the masked intrinsics and try to find whether the predicate
   // operand is generated by intrinsic @llvm.get.active.lane.mask().
@@ -673,7 +571,7 @@ bool MVETailPredication::TryConvert(Value *TripCount) {
     if (!Predicate || Predicates.count(Predicate))
       continue;
 
-    ActiveLaneMask = dyn_cast<IntrinsicInst>(Predicate);
+    auto *ActiveLaneMask = dyn_cast<IntrinsicInst>(Predicate);
     if (!ActiveLaneMask ||
         ActiveLaneMask->getIntrinsicID() != Intrinsic::get_active_lane_mask)
       continue;
@@ -682,17 +580,16 @@ bool MVETailPredication::TryConvert(Value *TripCount) {
     LLVM_DEBUG(dbgs() << "ARM TP: Found active lane mask: "
                       << *ActiveLaneMask << "\n");
 
-    VecTy = getVectorType(I);
-    if (!IsSafeActiveMask(TripCount, VecTy)) {
+    auto *VecTy = getVectorType(I);
+    if (!IsSafeActiveMask(ActiveLaneMask, TripCount, VecTy)) {
       LLVM_DEBUG(dbgs() << "ARM TP: Not safe to insert VCTP.\n");
       return false;
     }
     LLVM_DEBUG(dbgs() << "ARM TP: Safe to insert VCTP.\n");
-    InsertVCTPIntrinsic(ActiveLaneMask, TripCount, VecTy, NewPredicates);
+    InsertVCTPIntrinsic(ActiveLaneMask, TripCount, VecTy);
   }
 
-  // Now clean up.
-  ClonedVCTPInExitBlock = Cleanup(NewPredicates, Predicates, L);
+  Cleanup(Predicates, L);
   return true;
 }
 
diff --git a/llvm/lib/Target/AVR/AVRCallingConv.td b/llvm/lib/Target/AVR/AVRCallingConv.td
index 213e35fca66d1..65545e531a882 100644
--- a/llvm/lib/Target/AVR/AVRCallingConv.td
+++ b/llvm/lib/Target/AVR/AVRCallingConv.td
@@ -6,21 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 // This describes the calling conventions for AVR architecture.
+// Normal functions use a special calling convention, solved in code.
 //===----------------------------------------------------------------------===//
 
 //===----------------------------------------------------------------------===//
 // AVR Return Value Calling Convention
 //===----------------------------------------------------------------------===//
 
-def RetCC_AVR : CallingConv
-<[
-  // i8 is returned in R24.
-  CCIfType<[i8], CCAssignToReg<[R24]>>,
-
-  // i16 are returned in R25:R24, R23:R22, R21:R20 and R19:R18.
-  CCIfType<[i16], CCAssignToReg<[R25R24, R23R22, R21R20, R19R18]>>
-]>;
-
 // Special return value calling convention for runtime functions.
 def RetCC_AVR_BUILTIN : CallingConv
 <[
@@ -41,14 +33,6 @@ def ArgCC_AVR_Vararg : CallingConv
   CCAssignToStack<2, 1>
 ]>;
 
-// Special argument calling convention for
-// division runtime functions.
-def ArgCC_AVR_BUILTIN_DIV : CallingConv
-<[
-  CCIfType<[i8], CCAssignToReg<[R24,R22]>>,
-  CCIfType<[i16], CCAssignToReg<[R25R24, R23R22]>>
-]>;
-
 //===----------------------------------------------------------------------===//
 // Callee-saved register lists.
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AVR/AVRISelLowering.cpp b/llvm/lib/Target/AVR/AVRISelLowering.cpp
index 15efb62ebafb5..bf9b32e1278e3 100644
--- a/llvm/lib/Target/AVR/AVRISelLowering.cpp
+++ b/llvm/lib/Target/AVR/AVRISelLowering.cpp
@@ -14,6 +14,7 @@
 #include "AVRISelLowering.h"
 
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -881,172 +882,145 @@ bool AVRTargetLowering::isOffsetFoldingLegal(
 
 #include "AVRGenCallingConv.inc"
 
-/// For each argument in a function store the number of pieces it is composed
-/// of.
-static void parseFunctionArgs(const SmallVectorImpl<ISD::InputArg> &Ins,
-                              SmallVectorImpl<unsigned> &Out) {
-  for (const ISD::InputArg &Arg : Ins) {
-    if(Arg.PartOffset > 0) continue;
-    unsigned Bytes = ((Arg.ArgVT.getSizeInBits()) + 7) / 8;
-
-    Out.push_back((Bytes + 1) / 2);
-  }
-}
-
-/// For external symbols there is no function prototype information so we
-/// have to rely directly on argument sizes.
-static void parseExternFuncCallArgs(const SmallVectorImpl<ISD::OutputArg> &In,
-                                    SmallVectorImpl<unsigned> &Out) {
-  for (unsigned i = 0, e = In.size(); i != e;) {
-    unsigned Size = 0;
-    unsigned Offset = 0;
-    while ((i != e) && (In[i].PartOffset == Offset)) {
-      Offset += In[i].VT.getStoreSize();
-      ++i;
-      ++Size;
-    }
-    Out.push_back(Size);
-  }
-}
-
-static StringRef getFunctionName(TargetLowering::CallLoweringInfo &CLI) {
-  SDValue Callee = CLI.Callee;
-
-  if (const ExternalSymbolSDNode *G = dyn_cast<ExternalSymbolSDNode>(Callee)) {
-    return G->getSymbol();
-  }
-
-  if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
-    return G->getGlobal()->getName();
-  }
-
-  llvm_unreachable("don't know how to get the name for this callee");
-}
+/// Registers for calling conventions, ordered in reverse as required by ABI.
+/// Both arrays must be of the same length.
+static const MCPhysReg RegList8[] = {
+    AVR::R25, AVR::R24, AVR::R23, AVR::R22, AVR::R21, AVR::R20,
+    AVR::R19, AVR::R18, AVR::R17, AVR::R16, AVR::R15, AVR::R14,
+    AVR::R13, AVR::R12, AVR::R11, AVR::R10, AVR::R9,  AVR::R8};
+static const MCPhysReg RegList16[] = {
+    AVR::R26R25, AVR::R25R24, AVR::R24R23, AVR::R23R22,
+    AVR::R22R21, AVR::R21R20, AVR::R20R19, AVR::R19R18,
+    AVR::R18R17, AVR::R17R16, AVR::R16R15, AVR::R15R14,
+    AVR::R14R13, AVR::R13R12, AVR::R12R11, AVR::R11R10,
+    AVR::R10R9,  AVR::R9R8};
+
+static_assert(array_lengthof(RegList8) == array_lengthof(RegList16),
+        "8-bit and 16-bit register arrays must be of equal length");
 
 /// Analyze incoming and outgoing function arguments. We need custom C++ code
-/// to handle special constraints in the ABI like reversing the order of the
-/// pieces of splitted arguments. In addition, all pieces of a certain argument
-/// have to be passed either using registers or the stack but never mixing both.
-static void analyzeStandardArguments(TargetLowering::CallLoweringInfo *CLI,
-                                     const Function *F, const DataLayout *TD,
-                                     const SmallVectorImpl<ISD::OutputArg> *Outs,
-                                     const SmallVectorImpl<ISD::InputArg> *Ins,
-                                     CallingConv::ID CallConv,
-                                     SmallVectorImpl<CCValAssign> &ArgLocs,
-                                     CCState &CCInfo, bool IsCall, bool IsVarArg) {
-  static const MCPhysReg RegList8[] = {AVR::R24, AVR::R22, AVR::R20,
-                                       AVR::R18, AVR::R16, AVR::R14,
-                                       AVR::R12, AVR::R10, AVR::R8};
-  static const MCPhysReg RegList16[] = {AVR::R25R24, AVR::R23R22, AVR::R21R20,
-                                        AVR::R19R18, AVR::R17R16, AVR::R15R14,
-                                        AVR::R13R12, AVR::R11R10, AVR::R9R8};
-  if (IsVarArg) {
-    // Variadic functions do not need all the analysis below.
-    if (IsCall) {
-      CCInfo.AnalyzeCallOperands(*Outs, ArgCC_AVR_Vararg);
-    } else {
-      CCInfo.AnalyzeFormalArguments(*Ins, ArgCC_AVR_Vararg);
+/// to handle special constraints in the ABI.
+/// In addition, all pieces of a certain argument have to be passed either
+/// using registers or the stack but never mixing both.
+template <typename ArgT>
+static void
+analyzeArguments(TargetLowering::CallLoweringInfo *CLI, const Function *F,
+                 const DataLayout *TD, const SmallVectorImpl<ArgT> &Args,
+                 SmallVectorImpl<CCValAssign> &ArgLocs, CCState &CCInfo) {
+  unsigned NumArgs = Args.size();
+  // This is the index of the last used register, in RegList*.
+  // -1 means R26 (R26 is never actually used in CC).
+  int RegLastIdx = -1;
+  // Once a value is passed to the stack it will always be used
+  bool UseStack = false;
+  for (unsigned i = 0; i != NumArgs;) {
+    MVT VT = Args[i].VT;
+    // We have to count the number of bytes for each function argument, that is
+    // those Args with the same OrigArgIndex. This is important in case the
+    // function takes an aggregate type.
+    // Current argument will be between [i..j).
+    unsigned ArgIndex = Args[i].OrigArgIndex;
+    unsigned TotalBytes = VT.getStoreSize();
+    unsigned j = i + 1;
+    for (; j != NumArgs; ++j) {
+      if (Args[j].OrigArgIndex != ArgIndex)
+        break;
+      TotalBytes += Args[j].VT.getStoreSize();
     }
-    return;
-  }
-
-  // Fill in the Args array which will contain original argument sizes.
-  SmallVector<unsigned, 8> Args;
-  if (IsCall) {
-    parseExternFuncCallArgs(*Outs, Args);
-  } else {
-    assert(F != nullptr && "function should not be null");
-    parseFunctionArgs(*Ins, Args);
-  }
-
-  unsigned RegsLeft = array_lengthof(RegList8), ValNo = 0;
-  // Variadic functions always use the stack.
-  bool UsesStack = false;
-  for (unsigned i = 0, pos = 0, e = Args.size(); i != e; ++i) {
-    unsigned Size = Args[i];
-
-    // If we have a zero-sized argument, don't attempt to lower it.
-    // AVR-GCC does not support zero-sized arguments and so we need not
-    // worry about ABI compatibility.
-    if (Size == 0) continue;
-
-    MVT LocVT = (IsCall) ? (*Outs)[pos].VT : (*Ins)[pos].VT;
-
-    // If we have plenty of regs to pass the whole argument do it.
-    if (!UsesStack && (Size <= RegsLeft)) {
-      const MCPhysReg *RegList = (LocVT == MVT::i16) ? RegList16 : RegList8;
+    // Round up to even number of bytes.
+    TotalBytes = alignTo(TotalBytes, 2);
+    // Skip zero sized arguments
+    if (TotalBytes == 0)
+      continue;
+    // The index of the first register to be used
+    unsigned RegIdx = RegLastIdx + TotalBytes;
+    RegLastIdx = RegIdx;
+    // If there are not enough registers, use the stack
+    if (RegIdx >= array_lengthof(RegList8)) {
+      UseStack = true;
+    }
+    for (; i != j; ++i) {
+      MVT VT = Args[i].VT;
 
-      for (unsigned j = 0; j != Size; ++j) {
-        unsigned Reg = CCInfo.AllocateReg(
-            ArrayRef<MCPhysReg>(RegList, array_lengthof(RegList8)));
+      if (UseStack) {
+        auto evt = EVT(VT).getTypeForEVT(CCInfo.getContext());
+        unsigned Offset = CCInfo.AllocateStack(TD->getTypeAllocSize(evt),
+                                               TD->getABITypeAlign(evt));
         CCInfo.addLoc(
-            CCValAssign::getReg(ValNo++, LocVT, Reg, LocVT, CCValAssign::Full));
-        --RegsLeft;
-      }
-
-      // Reverse the order of the pieces to agree with the "big endian" format
-      // required in the calling convention ABI.
-      std::reverse(ArgLocs.begin() + pos, ArgLocs.begin() + pos + Size);
-    } else {
-      // Pass the rest of arguments using the stack.
-      UsesStack = true;
-      for (unsigned j = 0; j != Size; ++j) {
-        unsigned Offset = CCInfo.AllocateStack(
-            TD->getTypeAllocSize(EVT(LocVT).getTypeForEVT(CCInfo.getContext())),
-            TD->getABITypeAlign(EVT(LocVT).getTypeForEVT(CCInfo.getContext())));
-        CCInfo.addLoc(CCValAssign::getMem(ValNo++, LocVT, Offset, LocVT,
-                                          CCValAssign::Full));
+            CCValAssign::getMem(i, VT, Offset, VT, CCValAssign::Full));
+      } else {
+        unsigned Reg;
+        if (VT == MVT::i8) {
+          Reg = CCInfo.AllocateReg(RegList8[RegIdx]);
+        } else if (VT == MVT::i16) {
+          Reg = CCInfo.AllocateReg(RegList16[RegIdx]);
+        } else {
+          llvm_unreachable(
+              "calling convention can only manage i8 and i16 types");
+        }
+        assert(Reg && "register not available in calling convention");
+        CCInfo.addLoc(CCValAssign::getReg(i, VT, Reg, VT, CCValAssign::Full));
+        // Registers inside a particular argument are sorted in increasing order
+        // (remember the array is reversed).
+        RegIdx -= VT.getStoreSize();
       }
     }
-    pos += Size;
   }
 }
 
-static void analyzeBuiltinArguments(TargetLowering::CallLoweringInfo &CLI,
-                                    const Function *F, const DataLayout *TD,
-                                    const SmallVectorImpl<ISD::OutputArg> *Outs,
-                                    const SmallVectorImpl<ISD::InputArg> *Ins,
-                                    CallingConv::ID CallConv,
-                                    SmallVectorImpl<CCValAssign> &ArgLocs,
-                                    CCState &CCInfo, bool IsCall, bool IsVarArg) {
-  StringRef FuncName = getFunctionName(CLI);
-
-  if (FuncName.startswith("__udivmod") || FuncName.startswith("__divmod")) {
-    CCInfo.AnalyzeCallOperands(*Outs, ArgCC_AVR_BUILTIN_DIV);
-  } else {
-    analyzeStandardArguments(&CLI, F, TD, Outs, Ins,
-                             CallConv, ArgLocs, CCInfo,
-                             IsCall, IsVarArg);
+/// Count the total number of bytes needed to pass or return these arguments.
+template <typename ArgT>
+static unsigned getTotalArgumentsSizeInBytes(const SmallVectorImpl<ArgT> &Args) {
+  unsigned TotalBytes = 0;
+
+  for (const ArgT& Arg : Args) {
+    TotalBytes += Arg.VT.getStoreSize();
   }
+  return TotalBytes;
 }
 
-static void analyzeArguments(TargetLowering::CallLoweringInfo *CLI,
-                             const Function *F, const DataLayout *TD,
-                             const SmallVectorImpl<ISD::OutputArg> *Outs,
-                             const SmallVectorImpl<ISD::InputArg> *Ins,
-                             CallingConv::ID CallConv,
-                             SmallVectorImpl<CCValAssign> &ArgLocs,
-                             CCState &CCInfo, bool IsCall, bool IsVarArg) {
-  switch (CallConv) {
-    case CallingConv::AVR_BUILTIN: {
-      analyzeBuiltinArguments(*CLI, F, TD, Outs, Ins,
-                              CallConv, ArgLocs, CCInfo,
-                              IsCall, IsVarArg);
-      return;
-    }
-    default: {
-      analyzeStandardArguments(CLI, F, TD, Outs, Ins,
-                               CallConv, ArgLocs, CCInfo,
-                               IsCall, IsVarArg);
-      return;
+/// Analyze incoming and outgoing value of returning from a function.
+/// The algorithm is similar to analyzeArguments, but there can only be
+/// one value, possibly an aggregate, and it is limited to 8 bytes.
+template <typename ArgT>
+static void analyzeReturnValues(const SmallVectorImpl<ArgT> &Args,
+                                CCState &CCInfo) {
+  unsigned NumArgs = Args.size();
+  unsigned TotalBytes = getTotalArgumentsSizeInBytes(Args);
+  // CanLowerReturn() guarantees this assertion.
+  assert(TotalBytes <= 8 && "return values greater than 8 bytes cannot be lowered");
+
+  // GCC-ABI says that the size is rounded up to the next even number,
+  // but actually once it is more than 4 it will always round up to 8.
+  if (TotalBytes > 4) {
+    TotalBytes = 8;
+  } else {
+    TotalBytes = alignTo(TotalBytes, 2);
+  }
+
+  // The index of the first register to use.
+  int RegIdx = TotalBytes - 1;
+  for (unsigned i = 0; i != NumArgs; ++i) {
+    MVT VT = Args[i].VT;
+    unsigned Reg;
+    if (VT == MVT::i8) {
+      Reg = CCInfo.AllocateReg(RegList8[RegIdx]);
+    } else if (VT == MVT::i16) {
+      Reg = CCInfo.AllocateReg(RegList16[RegIdx]);
+    } else {
+      llvm_unreachable("calling convention can only manage i8 and i16 types");
     }
+    assert(Reg && "register not available in calling convention");
+    CCInfo.addLoc(CCValAssign::getReg(i, VT, Reg, VT, CCValAssign::Full));
+    // Registers sort in increasing order
+    RegIdx -= VT.getStoreSize();
   }
 }
 
 SDValue AVRTargetLowering::LowerFormalArguments(
     SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
-    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, SelectionDAG &DAG,
-    SmallVectorImpl<SDValue> &InVals) const {
+    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
+    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   auto DL = DAG.getDataLayout();
@@ -1056,8 +1030,12 @@ SDValue AVRTargetLowering::LowerFormalArguments(
   CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
                  *DAG.getContext());
 
-  analyzeArguments(nullptr, &MF.getFunction(), &DL, 0, &Ins, CallConv, ArgLocs, CCInfo,
-                   false, isVarArg);
+  // Variadic functions do not need all the analysis below.
+  if (isVarArg) {
+    CCInfo.AnalyzeFormalArguments(Ins, ArgCC_AVR_Vararg);
+  } else {
+    analyzeArguments(nullptr, &MF.getFunction(), &DL, Ins, ArgLocs, CCInfo);
+  }
 
   SDValue ArgValue;
   for (CCValAssign &VA : ArgLocs) {
@@ -1178,8 +1156,12 @@ SDValue AVRTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                                          getPointerTy(DAG.getDataLayout()));
   }
 
-  analyzeArguments(&CLI, F, &DAG.getDataLayout(), &Outs, 0, CallConv, ArgLocs, CCInfo,
-                   true, isVarArg);
+  // Variadic functions do not need all the analysis below.
+  if (isVarArg) {
+    CCInfo.AnalyzeCallOperands(Outs, ArgCC_AVR_Vararg);
+  } else {
+    analyzeArguments(&CLI, F, &DAG.getDataLayout(), Outs, ArgLocs, CCInfo);
+  }
 
   // Get a count of how many bytes are to be pushed on the stack.
   unsigned NumBytes = CCInfo.getNextStackOffset();
@@ -1316,13 +1298,10 @@ SDValue AVRTargetLowering::LowerCallResult(
                  *DAG.getContext());
 
   // Handle runtime calling convs.
-  auto CCFunction = CCAssignFnForReturn(CallConv);
-  CCInfo.AnalyzeCallResult(Ins, CCFunction);
-
-  if (CallConv != CallingConv::AVR_BUILTIN && RVLocs.size() > 1) {
-    // Reverse splitted return values to get the "big endian" format required
-    // to agree with the calling convention ABI.
-    std::reverse(RVLocs.begin(), RVLocs.end());
+  if (CallConv == CallingConv::AVR_BUILTIN) {
+    CCInfo.AnalyzeCallResult(Ins, RetCC_AVR_BUILTIN);
+  } else {
+    analyzeReturnValues(Ins, CCInfo);
   }
 
   // Copy all of the result registers out of their specified physreg.
@@ -1341,26 +1320,17 @@ SDValue AVRTargetLowering::LowerCallResult(
 //               Return Value Calling Convention Implementation
 //===----------------------------------------------------------------------===//
 
-CCAssignFn *AVRTargetLowering::CCAssignFnForReturn(CallingConv::ID CC) const {
-  switch (CC) {
-  case CallingConv::AVR_BUILTIN:
-    return RetCC_AVR_BUILTIN;
-  default:
-    return RetCC_AVR;
+bool AVRTargetLowering::CanLowerReturn(
+    CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg,
+    const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
+  if (CallConv == CallingConv::AVR_BUILTIN) {
+    SmallVector<CCValAssign, 16> RVLocs;
+    CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
+    return CCInfo.CheckReturn(Outs, RetCC_AVR_BUILTIN);
   }
-}
-
-bool
-AVRTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
-                                  MachineFunction &MF, bool isVarArg,
-                                  const SmallVectorImpl<ISD::OutputArg> &Outs,
-                                  LLVMContext &Context) const
-{
-  SmallVector<CCValAssign, 16> RVLocs;
-  CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
 
-  auto CCFunction = CCAssignFnForReturn(CallConv);
-  return CCInfo.CheckReturn(Outs, CCFunction);
+  unsigned TotalBytes = getTotalArgumentsSizeInBytes(Outs);
+  return TotalBytes <= 8;
 }
 
 SDValue
@@ -1376,25 +1346,19 @@ AVRTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
   CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
                  *DAG.getContext());
 
-  // Analyze return values.
-  auto CCFunction = CCAssignFnForReturn(CallConv);
-  CCInfo.AnalyzeReturn(Outs, CCFunction);
-
-  // If this is the first return lowered for this function, add the regs to
-  // the liveout set for the function.
   MachineFunction &MF = DAG.getMachineFunction();
-  unsigned e = RVLocs.size();
 
-  // Reverse splitted return values to get the "big endian" format required
-  // to agree with the calling convention ABI.
-  if (e > 1) {
-    std::reverse(RVLocs.begin(), RVLocs.end());
+  // Analyze return values.
+  if (CallConv == CallingConv::AVR_BUILTIN) {
+    CCInfo.AnalyzeReturn(Outs, RetCC_AVR_BUILTIN);
+  } else {
+    analyzeReturnValues(Outs, CCInfo);
   }
 
   SDValue Flag;
   SmallVector<SDValue, 4> RetOps(1, Chain);
   // Copy the result values into the output registers.
-  for (unsigned i = 0; i != e; ++i) {
+  for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
     CCValAssign &VA = RVLocs[i];
     assert(VA.isRegLoc() && "Can only return in registers!");
 
diff --git a/llvm/lib/Target/AVR/AVRISelLowering.h b/llvm/lib/Target/AVR/AVRISelLowering.h
index aca1ea1d50e54..d1eaf53b15e9c 100644
--- a/llvm/lib/Target/AVR/AVRISelLowering.h
+++ b/llvm/lib/Target/AVR/AVRISelLowering.h
@@ -146,10 +146,8 @@ class AVRTargetLowering : public TargetLowering {
   SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
 
-  CCAssignFn *CCAssignFnForReturn(CallingConv::ID CC) const;
-
-  bool CanLowerReturn(CallingConv::ID CallConv,
-                      MachineFunction &MF, bool isVarArg,
+  bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
+                      bool isVarArg,
                       const SmallVectorImpl<ISD::OutputArg> &Outs,
                       LLVMContext &Context) const override;
 
diff --git a/llvm/lib/Target/AVR/AVRInstrFormats.td b/llvm/lib/Target/AVR/AVRInstrFormats.td
index 159c143804863..6eb49076efb09 100644
--- a/llvm/lib/Target/AVR/AVRInstrFormats.td
+++ b/llvm/lib/Target/AVR/AVRInstrFormats.td
@@ -148,6 +148,8 @@ class FRd<bits<4> opcode, bits<7> f, dag outs, dag ins, string asmstr,
   let Inst{11-9} = f{6-4};
   let Inst{8-4} = d;
   let Inst{3-0} = f{3-0};
+
+  let DecoderMethod = "decodeFRd";
 }
 
 //===----------------------------------------------------------------------===//
@@ -235,6 +237,8 @@ class FLPMX<bit e, bit p, dag outs, dag ins, string asmstr, list<dag> pattern>
    let Inst{3-2} = 0b01;
    let Inst{1} = e;
    let Inst{0} = p;
+
+   let DecoderMethod = "decodeFLPMX";
 }
 
 //===----------------------------------------------------------------------===//
@@ -252,6 +256,8 @@ class FMOVWRdRr<dag outs, dag ins, string asmstr, list<dag> pattern>
   let Inst{15-8} = 0b00000001;
   let Inst{7-4} = d{4-1};
   let Inst{3-0} = r{4-1};
+
+  let DecoderMethod = "decodeFMOVWRdRr";
 }
 
 //===----------------------------------------------------------------------===//
@@ -270,6 +276,8 @@ class FMUL2RdRr<bit f, dag outs, dag ins, string asmstr, list<dag> pattern>
   let Inst{8} = f;
   let Inst{7-4} = rd{3-0};
   let Inst{3-0} = rr{3-0};
+
+  let DecoderMethod = "decodeFMUL2RdRr";
 }
 
 // Special encoding for the FMUL family of instructions.
@@ -293,6 +301,8 @@ class FFMULRdRr<bits<2> f, dag outs, dag ins, string asmstr, list<dag> pattern>
   let Inst{6-4} = rd;
   let Inst{3} = f{0};
   let Inst{2-0} = rr;
+
+  let DecoderMethod = "decodeFFMULRdRr";
 }
 
 
@@ -314,6 +324,8 @@ class FWRdK<bit f, dag outs, dag ins, string asmstr, list<dag> pattern>
   let Inst{7-6} = k{5-4};
   let Inst{5-4} = dst{2-1};
   let Inst{3-0} = k{3-0};
+
+  let DecoderMethod = "decodeFWRdK";
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AVR/AVRInstrInfo.cpp b/llvm/lib/Target/AVR/AVRInstrInfo.cpp
index ae268b1ddf083..06f07696bde3e 100644
--- a/llvm/lib/Target/AVR/AVRInstrInfo.cpp
+++ b/llvm/lib/Target/AVR/AVRInstrInfo.cpp
@@ -48,7 +48,7 @@ void AVRInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
 
   // Not all AVR devices support the 16-bit `MOVW` instruction.
   if (AVR::DREGSRegClass.contains(DestReg, SrcReg)) {
-    if (STI.hasMOVW()) {
+    if (STI.hasMOVW() && AVR::DREGSMOVWRegClass.contains(DestReg, SrcReg)) {
       BuildMI(MBB, MI, DL, get(AVR::MOVWRdRr), DestReg)
           .addReg(SrcReg, getKillRegState(KillSrc));
     } else {
diff --git a/llvm/lib/Target/AVR/AVRRegisterInfo.td b/llvm/lib/Target/AVR/AVRRegisterInfo.td
index ea38fedd22ce9..ab5d02356c9db 100644
--- a/llvm/lib/Target/AVR/AVRRegisterInfo.td
+++ b/llvm/lib/Target/AVR/AVRRegisterInfo.td
@@ -103,6 +103,17 @@ CoveredBySubRegs = 1 in
   def R5R4   : AVRReg<4,  "r5:r4",   [R4, R5]>,   DwarfRegNum<[4]>;
   def R3R2   : AVRReg<2,  "r3:r2",   [R2, R3]>,   DwarfRegNum<[2]>;
   def R1R0   : AVRReg<0,  "r1:r0",   [R0, R1]>,   DwarfRegNum<[0]>;
+
+  // Pseudo registers for unaligned i16
+  def R26R25 : AVRReg<25, "r26:r25", [R25, R26]>, DwarfRegNum<[25]>;
+  def R24R23 : AVRReg<23, "r24:r23", [R23, R24]>, DwarfRegNum<[23]>;
+  def R22R21 : AVRReg<21, "r22:r21", [R21, R22]>, DwarfRegNum<[21]>;
+  def R20R19 : AVRReg<19, "r20:r19", [R19, R20]>, DwarfRegNum<[19]>;
+  def R18R17 : AVRReg<17, "r18:r17", [R17, R18]>, DwarfRegNum<[17]>;
+  def R16R15 : AVRReg<15, "r16:r15", [R15, R16]>, DwarfRegNum<[15]>;
+  def R14R13 : AVRReg<13, "r14:r13", [R13, R14]>, DwarfRegNum<[13]>;
+  def R12R11 : AVRReg<11, "r12:r11", [R11, R12]>, DwarfRegNum<[11]>;
+  def R10R9  : AVRReg<9,  "r10:r9",  [R9,  R10]>, DwarfRegNum<[9]>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -146,6 +157,22 @@ def LD8lo : RegisterClass<"AVR", [i8], 8,
 
 // Main 16-bit pair register class.
 def DREGS : RegisterClass<"AVR", [i16], 8,
+  (
+    // Return value and arguments.
+    add R25R24, R19R18, R21R20, R23R22,
+    // Scratch registers.
+    R31R30, R27R26,
+    // Callee saved registers.
+    R29R28, R17R16, R15R14, R13R12, R11R10,
+    R9R8, R7R6, R5R4, R3R2, R1R0,
+    // Pseudo regs for unaligned 16-bits
+    R26R25, R24R23, R22R21,
+    R20R19, R18R17, R16R15,
+    R14R13, R12R11, R10R9
+  )>;
+
+// 16-bit pair register class for movw
+def DREGSMOVW : RegisterClass<"AVR", [i16], 8,
   (
     // Return value and arguments.
     add R25R24, R19R18, R21R20, R23R22,
diff --git a/llvm/lib/Target/AVR/Disassembler/AVRDisassembler.cpp b/llvm/lib/Target/AVR/Disassembler/AVRDisassembler.cpp
index 093e6a45e2255..8e7251a74dfdf 100644
--- a/llvm/lib/Target/AVR/Disassembler/AVRDisassembler.cpp
+++ b/llvm/lib/Target/AVR/Disassembler/AVRDisassembler.cpp
@@ -107,6 +107,24 @@ static DecodeStatus decodeFIOBIT(MCInst &Inst, unsigned Insn,
 static DecodeStatus decodeCallTarget(MCInst &Inst, unsigned Insn,
                                      uint64_t Address, const void *Decoder);
 
+static DecodeStatus decodeFRd(MCInst &Inst, unsigned Insn,
+                              uint64_t Address, const void *Decoder);
+
+static DecodeStatus decodeFLPMX(MCInst &Inst, unsigned Insn,
+                                uint64_t Address, const void *Decoder);
+
+static DecodeStatus decodeFFMULRdRr(MCInst &Inst, unsigned Insn,
+                                    uint64_t Address, const void *Decoder);
+
+static DecodeStatus decodeFMOVWRdRr(MCInst &Inst, unsigned Insn,
+                                    uint64_t Address, const void *Decoder);
+
+static DecodeStatus decodeFWRdK(MCInst &Inst, unsigned Insn,
+                                uint64_t Address, const void *Decoder);
+
+static DecodeStatus decodeFMUL2RdRr(MCInst &Inst, unsigned Insn,
+                                    uint64_t Address, const void *Decoder);
+
 #include "AVRGenDisassemblerTables.inc"
 
 static DecodeStatus decodeFIOARr(MCInst &Inst, unsigned Insn,
@@ -150,6 +168,69 @@ static DecodeStatus decodeCallTarget(MCInst &Inst, unsigned Field,
   return MCDisassembler::Success;
 }
 
+static DecodeStatus decodeFRd(MCInst &Inst, unsigned Insn,
+                              uint64_t Address, const void *Decoder) {
+  unsigned d = fieldFromInstruction(Insn, 4, 5);
+  if (DecodeGPR8RegisterClass(Inst, d, Address, Decoder) == MCDisassembler::Fail)
+    return MCDisassembler::Fail;
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus decodeFLPMX(MCInst &Inst, unsigned Insn,
+                                uint64_t Address, const void *Decoder) {
+  if (decodeFRd(Inst, Insn, Address, Decoder) == MCDisassembler::Fail)
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::createReg(AVR::R31R30));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus decodeFFMULRdRr(MCInst &Inst, unsigned Insn,
+                                    uint64_t Address, const void *Decoder) {
+  unsigned d = fieldFromInstruction(Insn, 4, 3) + 16;
+  unsigned r = fieldFromInstruction(Insn, 0, 3) + 16;
+  if (DecodeGPR8RegisterClass(Inst, d, Address, Decoder) == MCDisassembler::Fail)
+    return MCDisassembler::Fail;
+  if (DecodeGPR8RegisterClass(Inst, r, Address, Decoder) == MCDisassembler::Fail)
+    return MCDisassembler::Fail;
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus decodeFMOVWRdRr(MCInst &Inst, unsigned Insn,
+                                    uint64_t Address, const void *Decoder) {
+  unsigned r = fieldFromInstruction(Insn, 4, 4) * 2;
+  unsigned d = fieldFromInstruction(Insn, 0, 4) * 2;
+  if (DecodeGPR8RegisterClass(Inst, r, Address, Decoder) == MCDisassembler::Fail)
+    return MCDisassembler::Fail;
+  if (DecodeGPR8RegisterClass(Inst, d, Address, Decoder) == MCDisassembler::Fail)
+    return MCDisassembler::Fail;
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus decodeFWRdK(MCInst &Inst, unsigned Insn,
+                                    uint64_t Address, const void *Decoder) {
+  unsigned d = fieldFromInstruction(Insn, 4, 2) * 2 + 24; // starts at r24:r25
+  unsigned k = 0;
+  k |= fieldFromInstruction(Insn, 0, 4);
+  k |= fieldFromInstruction(Insn, 6, 2) << 4;
+  if (DecodeGPR8RegisterClass(Inst, d, Address, Decoder) == MCDisassembler::Fail)
+    return MCDisassembler::Fail;
+  if (DecodeGPR8RegisterClass(Inst, d, Address, Decoder) == MCDisassembler::Fail)
+    return MCDisassembler::Fail;
+  Inst.addOperand(MCOperand::createImm(k));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus decodeFMUL2RdRr(MCInst &Inst, unsigned Insn,
+                                    uint64_t Address, const void *Decoder) {
+  unsigned rd = fieldFromInstruction(Insn, 4, 4) + 16;
+  unsigned rr = fieldFromInstruction(Insn, 0, 4) + 16;
+  if (DecodeGPR8RegisterClass(Inst, rd, Address, Decoder) == MCDisassembler::Fail)
+    return MCDisassembler::Fail;
+  if (DecodeGPR8RegisterClass(Inst, rr, Address, Decoder) == MCDisassembler::Fail)
+    return MCDisassembler::Fail;
+  return MCDisassembler::Success;
+}
+
 static DecodeStatus readInstruction16(ArrayRef<uint8_t> Bytes, uint64_t Address,
                                       uint64_t &Size, uint32_t &Insn) {
   if (Bytes.size() < 2) {
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
index d7ee96ed6470c..ac72abe0d9f68 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
@@ -91,8 +91,6 @@ static void adjustRelativeBranch(unsigned Size, const MCFixup &Fixup,
   // one.
   signed_width(Size + 1, Value, std::string("branch target"), Fixup, Ctx);
 
-  Value -= 2;
-
   // Rightshifts the value by one.
   AVR::fixups::adjustBranchTarget(Value);
 }
@@ -249,27 +247,6 @@ void AVRAsmBackend::adjustFixupValue(const MCFixup &Fixup,
   uint64_t Size = AVRAsmBackend::getFixupKindInfo(Fixup.getKind()).TargetSize;
 
   unsigned Kind = Fixup.getKind();
-
-  // Parsed LLVM-generated temporary labels are already
-  // adjusted for instruction size, but normal labels aren't.
-  //
-  // To handle both cases, we simply un-adjust the temporary label
-  // case so it acts like all other labels.
-  if (const MCSymbolRefExpr *A = Target.getSymA()) {
-    if (A->getSymbol().isTemporary()) {
-      switch (Kind) {
-      case FK_Data_1:
-      case FK_Data_2:
-      case FK_Data_4:
-      case FK_Data_8:
-        // Don't shift value for absolute addresses.
-        break;
-      default:
-        Value += 2;
-      }
-    }
-  }
-
   switch (Kind) {
   default:
     llvm_unreachable("unhandled fixup");
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp
index 3283f74e8a204..42fac5e2e000e 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp
@@ -100,6 +100,14 @@ const char *AVRInstPrinter::getPrettyRegisterName(unsigned RegNum,
 
 void AVRInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
                                   raw_ostream &O) {
+  const MCOperandInfo &MOI = this->MII.get(MI->getOpcode()).OpInfo[OpNo];
+  if (MOI.RegClass == AVR::ZREGRegClassID) {
+    // Special case for the Z register, which sometimes doesn't have an operand
+    // in the MCInst.
+    O << "Z";
+    return;
+  }
+
   if (OpNo >= MI->size()) {
     // Not all operands are correctly disassembled at the moment. This means
     // that some machine instructions won't have all the necessary operands
@@ -111,7 +119,6 @@ void AVRInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   }
 
   const MCOperand &Op = MI->getOperand(OpNo);
-  const MCOperandInfo &MOI = this->MII.get(MI->getOpcode()).OpInfo[OpNo];
 
   if (Op.isReg()) {
     bool isPtrReg = (MOI.RegClass == AVR::PTRREGSRegClassID) ||
@@ -124,7 +131,7 @@ void AVRInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
       O << getPrettyRegisterName(Op.getReg(), MRI);
     }
   } else if (Op.isImm()) {
-    O << Op.getImm();
+    O << formatImm(Op.getImm());
   } else {
     assert(Op.isExpr() && "Unknown operand kind in printOperand");
     O << *Op.getExpr();
diff --git a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
index b213b15796617..16708c4d1ce6f 100644
--- a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
+++ b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
@@ -117,7 +117,7 @@ class BPFAbstractMemberAccess final : public ModulePass {
   struct CallInfo {
     uint32_t Kind;
     uint32_t AccessIndex;
-    uint32_t RecordAlignment;
+    Align RecordAlignment;
     MDNode *Metadata;
     Value *Base;
   };
@@ -157,11 +157,11 @@ class BPFAbstractMemberAccess final : public ModulePass {
   void replaceWithGEP(std::vector<CallInst *> &CallList,
                       uint32_t NumOfZerosIndex, uint32_t DIIndex);
   bool HasPreserveFieldInfoCall(CallInfoStack &CallStack);
-  void GetStorageBitRange(DIDerivedType *MemberTy, uint32_t RecordAlignment,
+  void GetStorageBitRange(DIDerivedType *MemberTy, Align RecordAlignment,
                           uint32_t &StartBitOffset, uint32_t &EndBitOffset);
   uint32_t GetFieldInfo(uint32_t InfoKind, DICompositeType *CTy,
                         uint32_t AccessIndex, uint32_t PatchImm,
-                        uint32_t RecordAlignment);
+                        Align RecordAlignment);
 
   Value *computeBaseAndAccessKey(CallInst *Call, CallInfo &CInfo,
                                  std::string &AccessKey, MDNode *&BaseMeta);
@@ -250,7 +250,7 @@ bool BPFAbstractMemberAccess::IsPreserveDIAccessIndexCall(const CallInst *Call,
     CInfo.AccessIndex = getConstant(Call->getArgOperand(2));
     CInfo.Base = Call->getArgOperand(0);
     CInfo.RecordAlignment =
-        DL->getABITypeAlignment(CInfo.Base->getType()->getPointerElementType());
+        DL->getABITypeAlign(CInfo.Base->getType()->getPointerElementType());
     return true;
   }
   if (GV->getName().startswith("llvm.preserve.union.access.index")) {
@@ -261,7 +261,7 @@ bool BPFAbstractMemberAccess::IsPreserveDIAccessIndexCall(const CallInst *Call,
     CInfo.AccessIndex = getConstant(Call->getArgOperand(1));
     CInfo.Base = Call->getArgOperand(0);
     CInfo.RecordAlignment =
-        DL->getABITypeAlignment(CInfo.Base->getType()->getPointerElementType());
+        DL->getABITypeAlign(CInfo.Base->getType()->getPointerElementType());
     return true;
   }
   if (GV->getName().startswith("llvm.preserve.struct.access.index")) {
@@ -272,7 +272,7 @@ bool BPFAbstractMemberAccess::IsPreserveDIAccessIndexCall(const CallInst *Call,
     CInfo.AccessIndex = getConstant(Call->getArgOperand(2));
     CInfo.Base = Call->getArgOperand(0);
     CInfo.RecordAlignment =
-        DL->getABITypeAlignment(CInfo.Base->getType()->getPointerElementType());
+        DL->getABITypeAlign(CInfo.Base->getType()->getPointerElementType());
     return true;
   }
   if (GV->getName().startswith("llvm.bpf.preserve.field.info")) {
@@ -522,12 +522,12 @@ uint64_t BPFAbstractMemberAccess::getConstant(const Value *IndexValue) {
 
 /// Get the start and the end of storage offset for \p MemberTy.
 void BPFAbstractMemberAccess::GetStorageBitRange(DIDerivedType *MemberTy,
-                                                 uint32_t RecordAlignment,
+                                                 Align RecordAlignment,
                                                  uint32_t &StartBitOffset,
                                                  uint32_t &EndBitOffset) {
   uint32_t MemberBitSize = MemberTy->getSizeInBits();
   uint32_t MemberBitOffset = MemberTy->getOffsetInBits();
-  uint32_t AlignBits = RecordAlignment * 8;
+  uint32_t AlignBits = RecordAlignment.value() * 8;
   if (RecordAlignment > 8 || MemberBitSize > AlignBits)
     report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info, "
                        "requiring too big alignment");
@@ -543,7 +543,7 @@ uint32_t BPFAbstractMemberAccess::GetFieldInfo(uint32_t InfoKind,
                                                DICompositeType *CTy,
                                                uint32_t AccessIndex,
                                                uint32_t PatchImm,
-                                               uint32_t RecordAlignment) {
+                                               Align RecordAlignment) {
   if (InfoKind == BPFCoreSharedInfo::FIELD_EXISTENCE)
       return 1;
 
@@ -808,8 +808,10 @@ Value *BPFAbstractMemberAccess::computeBaseAndAccessKey(CallInst *Call,
     CInfo = StackElem.second;
     CallStack.pop();
 
-    if (CInfo.Kind == BPFPreserveFieldInfoAI)
+    if (CInfo.Kind == BPFPreserveFieldInfoAI) {
+      InfoKind = CInfo.AccessIndex;
       break;
+    }
 
     // If the next Call (the top of the stack) is a BPFPreserveFieldInfoAI,
     // the action will be extracting field info.
@@ -827,11 +829,10 @@ Value *BPFAbstractMemberAccess::computeBaseAndAccessKey(CallInst *Call,
     AccessKey += ":" + std::to_string(AccessIndex);
 
     MDNode *MDN = CInfo.Metadata;
-    uint32_t RecordAlignment = CInfo.RecordAlignment;
     // At this stage, it cannot be pointer type.
     auto *CTy = cast<DICompositeType>(stripQualifiers(cast<DIType>(MDN)));
     PatchImm = GetFieldInfo(InfoKind, CTy, AccessIndex, PatchImm,
-                            RecordAlignment);
+                            CInfo.RecordAlignment);
   }
 
   // Access key is the
diff --git a/llvm/lib/Target/BPF/BPFSelectionDAGInfo.cpp b/llvm/lib/Target/BPF/BPFSelectionDAGInfo.cpp
index a711294048ba3..4c36c0edcef6f 100644
--- a/llvm/lib/Target/BPF/BPFSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/BPF/BPFSelectionDAGInfo.cpp
@@ -19,7 +19,7 @@ using namespace llvm;
 
 SDValue BPFSelectionDAGInfo::EmitTargetCodeForMemcpy(
     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
-    SDValue Size, unsigned Align, bool isVolatile, bool AlwaysInline,
+    SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
     MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
   // Requires the copy size to be a constant.
   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
@@ -27,7 +27,7 @@ SDValue BPFSelectionDAGInfo::EmitTargetCodeForMemcpy(
     return SDValue();
 
   unsigned CopyLen = ConstantSize->getZExtValue();
-  unsigned StoresNumEstimate = alignTo(CopyLen, Align) >> Log2_32(Align);
+  unsigned StoresNumEstimate = alignTo(CopyLen, Alignment) >> Log2(Alignment);
   // Impose the same copy length limit as MaxStoresPerMemcpy.
   if (StoresNumEstimate > getCommonMaxStoresPerMemFunc())
     return SDValue();
@@ -36,7 +36,7 @@ SDValue BPFSelectionDAGInfo::EmitTargetCodeForMemcpy(
 
   Dst = DAG.getNode(BPFISD::MEMCPY, dl, VTs, Chain, Dst, Src,
                     DAG.getConstant(CopyLen, dl, MVT::i64),
-                    DAG.getConstant(Align, dl, MVT::i64));
+                    DAG.getConstant(Alignment.value(), dl, MVT::i64));
 
   return Dst.getValue(0);
 }
diff --git a/llvm/lib/Target/BPF/BPFSelectionDAGInfo.h b/llvm/lib/Target/BPF/BPFSelectionDAGInfo.h
index fb88c32ceb0cf..79f05e57bb5cd 100644
--- a/llvm/lib/Target/BPF/BPFSelectionDAGInfo.h
+++ b/llvm/lib/Target/BPF/BPFSelectionDAGInfo.h
@@ -21,8 +21,8 @@ class BPFSelectionDAGInfo : public SelectionDAGTargetInfo {
 public:
   SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
                                   SDValue Chain, SDValue Dst, SDValue Src,
-                                  SDValue Size, unsigned Align, bool isVolatile,
-                                  bool AlwaysInline,
+                                  SDValue Size, Align Alignment,
+                                  bool isVolatile, bool AlwaysInline,
                                   MachinePointerInfo DstPtrInfo,
                                   MachinePointerInfo SrcPtrInfo) const override;
 
diff --git a/llvm/lib/Target/BPF/BTFDebug.cpp b/llvm/lib/Target/BPF/BTFDebug.cpp
index f0996d637d365..4510e93574892 100644
--- a/llvm/lib/Target/BPF/BTFDebug.cpp
+++ b/llvm/lib/Target/BPF/BTFDebug.cpp
@@ -568,7 +568,7 @@ void BTFDebug::visitDerivedType(const DIDerivedType *DTy, uint32_t &TypeId,
         auto CTag = CTy->getTag();
         if ((CTag == dwarf::DW_TAG_structure_type ||
              CTag == dwarf::DW_TAG_union_type) &&
-            !CTy->isForwardDecl()) {
+            !CTy->getName().empty() && !CTy->isForwardDecl()) {
           /// Find a candidate, generate a fixup. Later on the struct/union
           /// pointee type will be replaced with either a real type or
           /// a forward declaration.
@@ -664,7 +664,17 @@ void BTFDebug::visitMapDefType(const DIType *Ty, uint32_t &TypeId) {
     return;
   }
 
-  // MapDef type is a struct type
+  // MapDef type may be a struct type or a non-pointer derived type
+  const DIType *OrigTy = Ty;
+  while (auto *DTy = dyn_cast<DIDerivedType>(Ty)) {
+    auto Tag = DTy->getTag();
+    if (Tag != dwarf::DW_TAG_typedef && Tag != dwarf::DW_TAG_const_type &&
+        Tag != dwarf::DW_TAG_volatile_type &&
+        Tag != dwarf::DW_TAG_restrict_type)
+      break;
+    Ty = DTy->getBaseType();
+  }
+
   const auto *CTy = dyn_cast<DICompositeType>(Ty);
   if (!CTy)
     return;
@@ -673,27 +683,15 @@ void BTFDebug::visitMapDefType(const DIType *Ty, uint32_t &TypeId) {
   if (Tag != dwarf::DW_TAG_structure_type || CTy->isForwardDecl())
     return;
 
-  // Record this type
+  // Visit all struct members to ensure pointee type is visited
   const DINodeArray Elements = CTy->getElements();
-  bool HasBitField = false;
-  for (const auto *Element : Elements) {
-    auto E = cast<DIDerivedType>(Element);
-    if (E->isBitField()) {
-      HasBitField = true;
-      break;
-    }
-  }
-
-  auto TypeEntry =
-      std::make_unique<BTFTypeStruct>(CTy, true, HasBitField, Elements.size());
-  StructTypes.push_back(TypeEntry.get());
-  TypeId = addType(std::move(TypeEntry), CTy);
-
-  // Visit all struct members
   for (const auto *Element : Elements) {
     const auto *MemberType = cast<DIDerivedType>(Element);
     visitTypeEntry(MemberType->getBaseType());
   }
+
+  // Visit this type, struct or a const/typedef/volatile/restrict type
+  visitTypeEntry(OrigTy, TypeId, false, false);
 }
 
 /// Read file contents from the actual file or from the source
diff --git a/llvm/lib/Target/Hexagon/BitTracker.cpp b/llvm/lib/Target/Hexagon/BitTracker.cpp
index 8a07b991ff5aa..7ef23ef35a745 100644
--- a/llvm/lib/Target/Hexagon/BitTracker.cpp
+++ b/llvm/lib/Target/Hexagon/BitTracker.cpp
@@ -954,6 +954,9 @@ void BT::visitBranchesFrom(const MachineInstr &BI) {
     ++It;
   } while (FallsThrough && It != End);
 
+  if (B.mayHaveInlineAsmBr())
+    DefaultToAll = true;
+
   if (!DefaultToAll) {
     // Need to add all CFG successors that lead to EH landing pads.
     // There won't be explicit branches to these blocks, but they must
diff --git a/llvm/lib/Target/Hexagon/HexagonCallingConv.td b/llvm/lib/Target/Hexagon/HexagonCallingConv.td
index 5c31a81a1e871..93e17e608dd10 100644
--- a/llvm/lib/Target/Hexagon/HexagonCallingConv.td
+++ b/llvm/lib/Target/Hexagon/HexagonCallingConv.td
@@ -18,7 +18,7 @@ def CC_HexagonStack: CallingConv<[
     CCAssignToStack<8,8>>
 ]>;
 
-def CC_Hexagon: CallingConv<[
+def CC_Hexagon_Legacy: CallingConv<[
   CCIfType<[i1,i8,i16],
     CCPromoteToType<i32>>,
   CCIfType<[f32],
@@ -48,6 +48,36 @@ def CC_Hexagon: CallingConv<[
   CCDelegateTo<CC_HexagonStack>
 ]>;
 
+def CC_Hexagon: CallingConv<[
+  CCIfType<[i1,i8,i16],
+    CCPromoteToType<i32>>,
+  CCIfType<[f32],
+    CCBitConvertToType<i32>>,
+  CCIfType<[f64],
+    CCBitConvertToType<i64>>,
+
+  CCIfByVal<
+    CCPassByVal<8,1>>,
+  CCIfArgIsVarArg<
+    CCDelegateTo<CC_HexagonStack>>,
+
+  // Pass split values in pairs, allocate odd register if necessary.
+  CCIfType<[i32],
+    CCIfSplit<
+      CCCustom<"CC_SkipOdd">>>,
+
+  CCIfType<[i32,v2i16,v4i8],
+    CCAssignToReg<[R0,R1,R2,R3,R4,R5]>>,
+  // Make sure to allocate any skipped 32-bit register, so it does not get
+  // allocated to a subsequent 32-bit value.
+  CCIfType<[i64,v2i32,v4i16,v8i8],
+    CCCustom<"CC_SkipOdd">>,
+  CCIfType<[i64,v2i32,v4i16,v8i8],
+    CCAssignToReg<[D0,D1,D2]>>,
+
+  CCDelegateTo<CC_HexagonStack>
+]>;
+
 def RetCC_Hexagon: CallingConv<[
   CCIfType<[i1,i8,i16],
     CCPromoteToType<i32>>,
diff --git a/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp b/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp
index 5821e72227bcb..77578378b0588 100644
--- a/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp
@@ -754,6 +754,9 @@ void MachineConstPropagator::visitBranchesFrom(const MachineInstr &BrI) {
     ++It;
   }
 
+  if (B.mayHaveInlineAsmBr())
+    EvalOk = false;
+
   if (EvalOk) {
     // Need to add all CFG successors that lead to EH landing pads.
     // There won't be explicit branches to these blocks, but they must
@@ -810,8 +813,12 @@ void MachineConstPropagator::visitUsesOf(unsigned Reg) {
 
 bool MachineConstPropagator::computeBlockSuccessors(const MachineBasicBlock *MB,
       SetVector<const MachineBasicBlock*> &Targets) {
+  Targets.clear();
+
   MachineBasicBlock::const_iterator FirstBr = MB->end();
   for (const MachineInstr &MI : *MB) {
+    if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
+      return false;
     if (MI.isDebugInstr())
       continue;
     if (MI.isBranch()) {
@@ -820,7 +827,6 @@ bool MachineConstPropagator::computeBlockSuccessors(const MachineBasicBlock *MB,
     }
   }
 
-  Targets.clear();
   MachineBasicBlock::const_iterator End = MB->end();
 
   bool DoNext = true;
diff --git a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
index 388c5feb0e11b..b4b389a7b9568 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
@@ -1315,28 +1315,28 @@ bool HexagonDAGToDAGISel::SelectAddrFI(SDValue &N, SDValue &R) {
 }
 
 inline bool HexagonDAGToDAGISel::SelectAddrGA(SDValue &N, SDValue &R) {
-  return SelectGlobalAddress(N, R, false, 0);
+  return SelectGlobalAddress(N, R, false, Align(1));
 }
 
 inline bool HexagonDAGToDAGISel::SelectAddrGP(SDValue &N, SDValue &R) {
-  return SelectGlobalAddress(N, R, true, 0);
+  return SelectGlobalAddress(N, R, true, Align(1));
 }
 
 inline bool HexagonDAGToDAGISel::SelectAnyImm(SDValue &N, SDValue &R) {
-  return SelectAnyImmediate(N, R, 0);
+  return SelectAnyImmediate(N, R, Align(1));
 }
 
 inline bool HexagonDAGToDAGISel::SelectAnyImm0(SDValue &N, SDValue &R) {
-  return SelectAnyImmediate(N, R, 0);
+  return SelectAnyImmediate(N, R, Align(1));
 }
 inline bool HexagonDAGToDAGISel::SelectAnyImm1(SDValue &N, SDValue &R) {
-  return SelectAnyImmediate(N, R, 1);
+  return SelectAnyImmediate(N, R, Align(2));
 }
 inline bool HexagonDAGToDAGISel::SelectAnyImm2(SDValue &N, SDValue &R) {
-  return SelectAnyImmediate(N, R, 2);
+  return SelectAnyImmediate(N, R, Align(4));
 }
 inline bool HexagonDAGToDAGISel::SelectAnyImm3(SDValue &N, SDValue &R) {
-  return SelectAnyImmediate(N, R, 3);
+  return SelectAnyImmediate(N, R, Align(8));
 }
 
 inline bool HexagonDAGToDAGISel::SelectAnyInt(SDValue &N, SDValue &R) {
@@ -1348,17 +1348,13 @@ inline bool HexagonDAGToDAGISel::SelectAnyInt(SDValue &N, SDValue &R) {
 }
 
 bool HexagonDAGToDAGISel::SelectAnyImmediate(SDValue &N, SDValue &R,
-                                             uint32_t LogAlign) {
-  auto IsAligned = [LogAlign] (uint64_t V) -> bool {
-    return alignTo(V, (uint64_t)1 << LogAlign) == V;
-  };
-
+                                             Align Alignment) {
   switch (N.getOpcode()) {
   case ISD::Constant: {
     if (N.getValueType() != MVT::i32)
       return false;
     int32_t V = cast<const ConstantSDNode>(N)->getZExtValue();
-    if (!IsAligned(V))
+    if (!isAligned(Alignment, V))
       return false;
     R = CurDAG->getTargetConstant(V, SDLoc(N), N.getValueType());
     return true;
@@ -1366,37 +1362,34 @@ bool HexagonDAGToDAGISel::SelectAnyImmediate(SDValue &N, SDValue &R,
   case HexagonISD::JT:
   case HexagonISD::CP:
     // These are assumed to always be aligned at least 8-byte boundary.
-    if (LogAlign > 3)
+    if (Alignment > Align(8))
       return false;
     R = N.getOperand(0);
     return true;
   case ISD::ExternalSymbol:
     // Symbols may be aligned at any boundary.
-    if (LogAlign > 0)
+    if (Alignment > Align(1))
       return false;
     R = N;
     return true;
   case ISD::BlockAddress:
     // Block address is always aligned at least 4-byte boundary.
-    if (LogAlign > 2 || !IsAligned(cast<BlockAddressSDNode>(N)->getOffset()))
+    if (Alignment > Align(4) ||
+        !isAligned(Alignment, cast<BlockAddressSDNode>(N)->getOffset()))
       return false;
     R = N;
     return true;
   }
 
-  if (SelectGlobalAddress(N, R, false, LogAlign) ||
-      SelectGlobalAddress(N, R, true, LogAlign))
+  if (SelectGlobalAddress(N, R, false, Alignment) ||
+      SelectGlobalAddress(N, R, true, Alignment))
     return true;
 
   return false;
 }
 
 bool HexagonDAGToDAGISel::SelectGlobalAddress(SDValue &N, SDValue &R,
-                                              bool UseGP, uint32_t LogAlign) {
-  auto IsAligned = [LogAlign] (uint64_t V) -> bool {
-    return alignTo(V, (uint64_t)1 << LogAlign) == V;
-  };
-
+                                              bool UseGP, Align Alignment) {
   switch (N.getOpcode()) {
   case ISD::ADD: {
     SDValue N0 = N.getOperand(0);
@@ -1407,10 +1400,9 @@ bool HexagonDAGToDAGISel::SelectGlobalAddress(SDValue &N, SDValue &R,
     if (!UseGP && GAOpc != HexagonISD::CONST32)
       return false;
     if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N1)) {
-      SDValue Addr = N0.getOperand(0);
-      // For the purpose of alignment, sextvalue and zextvalue are the same.
-      if (!IsAligned(Const->getZExtValue()))
+      if (!isAligned(Alignment, Const->getZExtValue()))
         return false;
+      SDValue Addr = N0.getOperand(0);
       if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Addr)) {
         if (GA->getOpcode() == ISD::TargetGlobalAddress) {
           uint64_t NewOff = GA->getOffset() + (uint64_t)Const->getSExtValue();
diff --git a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.h b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.h
index d60de4cfc2eeb..1e50385a7b4be 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.h
+++ b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.h
@@ -59,9 +59,8 @@ class HexagonDAGToDAGISel : public SelectionDAGISel {
   inline bool SelectAddrGP(SDValue &N, SDValue &R);
   inline bool SelectAnyImm(SDValue &N, SDValue &R);
   inline bool SelectAnyInt(SDValue &N, SDValue &R);
-  bool SelectAnyImmediate(SDValue &N, SDValue &R, uint32_t LogAlign);
-  bool SelectGlobalAddress(SDValue &N, SDValue &R, bool UseGP,
-                           uint32_t LogAlign);
+  bool SelectAnyImmediate(SDValue &N, SDValue &R, Align Alignment);
+  bool SelectGlobalAddress(SDValue &N, SDValue &R, bool UseGP, Align Alignment);
   bool SelectAddrFI(SDValue &N, SDValue &R);
   bool DetectUseSxtw(SDValue &N, SDValue &R);
 
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 99d70a0e16285..768fea639cf9c 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -109,6 +109,11 @@ static cl::opt<bool> AlignLoads("hexagon-align-loads",
   cl::Hidden, cl::init(false),
   cl::desc("Rewrite unaligned loads as a pair of aligned loads"));
 
+static cl::opt<bool>
+    DisableArgsMinAlignment("hexagon-disable-args-min-alignment", cl::Hidden,
+                            cl::init(false),
+                            cl::desc("Disable minimum alignment of 1 for "
+                                     "arguments passed by value on stack"));
 
 namespace {
 
@@ -401,6 +406,8 @@ HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
   if (Subtarget.useHVXOps())
     CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_HVX);
+  else if (DisableArgsMinAlignment)
+    CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_Legacy);
   else
     CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon);
 
@@ -762,6 +769,8 @@ SDValue HexagonTargetLowering::LowerFormalArguments(
 
   if (Subtarget.useHVXOps())
     CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_HVX);
+  else if (DisableArgsMinAlignment)
+    CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_Legacy);
   else
     CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon);
 
@@ -1981,8 +1990,7 @@ bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     // The offset value comes through Modifier register. For now, assume the
     // offset is 0.
     Info.offset = 0;
-    Info.align =
-        MaybeAlign(DL.getABITypeAlignment(Info.memVT.getTypeForEVT(Cont)));
+    Info.align = DL.getABITypeAlign(Info.memVT.getTypeForEVT(Cont));
     Info.flags = MachineMemOperand::MOLoad;
     return true;
   }
@@ -3269,12 +3277,12 @@ bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL,
     // The type Ty passed here would then be "void". Skip the alignment
     // checks, but do not return false right away, since that confuses
     // LSR into crashing.
-    unsigned A = DL.getABITypeAlignment(Ty);
+    Align A = DL.getABITypeAlign(Ty);
     // The base offset must be a multiple of the alignment.
-    if ((AM.BaseOffs % A) != 0)
+    if (!isAligned(A, AM.BaseOffs))
       return false;
     // The shifted offset must fit in 11 bits.
-    if (!isInt<11>(AM.BaseOffs >> Log2_32(A)))
+    if (!isInt<11>(AM.BaseOffs >> Log2(A)))
       return false;
   }
 
@@ -3385,12 +3393,12 @@ EVT HexagonTargetLowering::getOptimalMemOpType(
   return MVT::Other;
 }
 
-bool HexagonTargetLowering::allowsMemoryAccess(LLVMContext &Context,
-      const DataLayout &DL, EVT VT, unsigned AddrSpace, unsigned Alignment,
-      MachineMemOperand::Flags Flags, bool *Fast) const {
+bool HexagonTargetLowering::allowsMemoryAccess(
+    LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
+    Align Alignment, MachineMemOperand::Flags Flags, bool *Fast) const {
   MVT SVT = VT.getSimpleVT();
   if (Subtarget.isHVXVectorType(SVT, true))
-    return allowsHvxMemoryAccess(SVT, Alignment, Flags, Fast);
+    return allowsHvxMemoryAccess(SVT, Flags, Fast);
   return TargetLoweringBase::allowsMemoryAccess(
               Context, DL, VT, AddrSpace, Alignment, Flags, Fast);
 }
@@ -3400,7 +3408,7 @@ bool HexagonTargetLowering::allowsMisalignedMemoryAccesses(
       MachineMemOperand::Flags Flags, bool *Fast) const {
   MVT SVT = VT.getSimpleVT();
   if (Subtarget.isHVXVectorType(SVT, true))
-    return allowsHvxMisalignedMemoryAccesses(SVT, Alignment, Flags, Fast);
+    return allowsHvxMisalignedMemoryAccesses(SVT, Flags, Fast);
   if (Fast)
     *Fast = false;
   return false;
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
index 1c123c06bf321..7d6e6b6185c87 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
@@ -306,8 +306,9 @@ namespace HexagonISD {
                             const AttributeList &FuncAttributes) const override;
 
     bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
-        unsigned AddrSpace, unsigned Alignment, MachineMemOperand::Flags Flags,
-        bool *Fast) const override;
+                            unsigned AddrSpace, Align Alignment,
+                            MachineMemOperand::Flags Flags,
+                            bool *Fast) const override;
 
     bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace,
         unsigned Alignment, MachineMemOperand::Flags Flags, bool *Fast)
@@ -408,10 +409,11 @@ namespace HexagonISD {
     VectorPair opSplit(SDValue Vec, const SDLoc &dl, SelectionDAG &DAG) const;
     SDValue opCastElem(SDValue Vec, MVT ElemTy, SelectionDAG &DAG) const;
 
-    bool allowsHvxMemoryAccess(MVT VecTy, unsigned Alignment,
-        MachineMemOperand::Flags Flags, bool *Fast) const;
-    bool allowsHvxMisalignedMemoryAccesses(MVT VecTy, unsigned Align,
-        MachineMemOperand::Flags Flags, bool *Fast) const;
+    bool allowsHvxMemoryAccess(MVT VecTy, MachineMemOperand::Flags Flags,
+                               bool *Fast) const;
+    bool allowsHvxMisalignedMemoryAccesses(MVT VecTy,
+                                           MachineMemOperand::Flags Flags,
+                                           bool *Fast) const;
 
     bool isHvxSingleTy(MVT Ty) const;
     bool isHvxPairTy(MVT Ty) const;
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
index bf9ddef12882b..7cda915fffe91 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
@@ -299,9 +299,8 @@ HexagonTargetLowering::isHvxBoolTy(MVT Ty) const {
          Ty.getVectorElementType() == MVT::i1;
 }
 
-bool
-HexagonTargetLowering::allowsHvxMemoryAccess(MVT VecTy, unsigned Alignment,
-        MachineMemOperand::Flags Flags, bool *Fast) const {
+bool HexagonTargetLowering::allowsHvxMemoryAccess(
+    MVT VecTy, MachineMemOperand::Flags Flags, bool *Fast) const {
   // Bool vectors are excluded by default, but make it explicit to
   // emphasize that bool vectors cannot be loaded or stored.
   // Also, disallow double vector stores (to prevent unnecessary
@@ -315,9 +314,8 @@ HexagonTargetLowering::allowsHvxMemoryAccess(MVT VecTy, unsigned Alignment,
   return true;
 }
 
-bool
-HexagonTargetLowering::allowsHvxMisalignedMemoryAccesses(MVT VecTy,
-        unsigned Align, MachineMemOperand::Flags Flags, bool *Fast) const {
+bool HexagonTargetLowering::allowsHvxMisalignedMemoryAccesses(
+    MVT VecTy, MachineMemOperand::Flags Flags, bool *Fast) const {
   if (!Subtarget.isHVXVectorType(VecTy))
     return false;
   // XXX Should this be false?  vmemu are a bit slower than vmem.
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
index 64922d30c4151..d1cd23c3be3e5 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
@@ -1730,6 +1730,10 @@ bool HexagonInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
   if (MI.getDesc().isTerminator() || MI.isPosition())
     return true;
 
+  // INLINEASM_BR can jump to another block
+  if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
+    return true;
+
   if (MI.isInlineAsm() && !ScheduleInlineAsm)
     return true;
 
diff --git a/llvm/lib/Target/Hexagon/HexagonNewValueJump.cpp b/llvm/lib/Target/Hexagon/HexagonNewValueJump.cpp
index e3579dfa9ba97..8dc1113194a85 100644
--- a/llvm/lib/Target/Hexagon/HexagonNewValueJump.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonNewValueJump.cpp
@@ -291,7 +291,7 @@ static bool canCompareBeNewValueJump(const HexagonInstrInfo *QII,
     // at machine code level, we don't need this, but if we decide
     // to move new value jump prior to RA, we would be needing this.
     MachineRegisterInfo &MRI = MF.getRegInfo();
-    if (secondReg && !Register::isPhysicalRegister(cmpOp2)) {
+    if (!Register::isPhysicalRegister(cmpOp2)) {
       MachineInstr *def = MRI.getVRegDef(cmpOp2);
       if (def->getOpcode() == TargetOpcode::COPY)
         return false;
diff --git a/llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp b/llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp
index c5ba7ced4c308..1b724e8fcae91 100644
--- a/llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp
@@ -18,10 +18,10 @@ using namespace llvm;
 
 SDValue HexagonSelectionDAGInfo::EmitTargetCodeForMemcpy(
     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
-    SDValue Size, unsigned Align, bool isVolatile, bool AlwaysInline,
+    SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
     MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
-  if (AlwaysInline || (Align & 0x3) != 0 || !ConstantSize)
+  if (AlwaysInline || Alignment < Align(4) || !ConstantSize)
     return SDValue();
 
   uint64_t SizeVal = ConstantSize->getZExtValue();
diff --git a/llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.h b/llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.h
index af8b8318b059e..0d3b1725d1bc4 100644
--- a/llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.h
@@ -23,8 +23,8 @@ class HexagonSelectionDAGInfo : public SelectionDAGTargetInfo {
 
   SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
                                   SDValue Chain, SDValue Dst, SDValue Src,
-                                  SDValue Size, unsigned Align, bool isVolatile,
-                                  bool AlwaysInline,
+                                  SDValue Size, Align Alignment,
+                                  bool isVolatile, bool AlwaysInline,
                                   MachinePointerInfo DstPtrInfo,
                                   MachinePointerInfo SrcPtrInfo) const override;
 };
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index cc1aaf919434a..80c8736cb74a0 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -78,12 +78,17 @@ HexagonTTIImpl::getPopcntSupport(unsigned IntTyWidthInBit) const {
 void HexagonTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                              TTI::UnrollingPreferences &UP) {
   UP.Runtime = UP.Partial = true;
+}
+
+void HexagonTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                                           TTI::PeelingPreferences &PP) {
+  BaseT::getPeelingPreferences(L, SE, PP);
   // Only try to peel innermost loops with small runtime trip counts.
   if (L && L->empty() && canPeel(L) &&
       SE.getSmallConstantTripCount(L) == 0 &&
       SE.getSmallConstantMaxTripCount(L) > 0 &&
       SE.getSmallConstantMaxTripCount(L) <= 5) {
-    UP.PeelCount = 2;
+    PP.PeelCount = 2;
   }
 }
 
@@ -200,9 +205,10 @@ unsigned HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                                 CostKind, I);
 }
 
-unsigned HexagonTTIImpl::getMaskedMemoryOpCost(unsigned Opcode,
-      Type *Src, unsigned Alignment, unsigned AddressSpace,
-      TTI::TargetCostKind CostKind) {
+unsigned HexagonTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
+                                               Align Alignment,
+                                               unsigned AddressSpace,
+                                               TTI::TargetCostKind CostKind) {
   return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
                                       CostKind);
 }
@@ -213,18 +219,16 @@ unsigned HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp,
 }
 
 unsigned HexagonTTIImpl::getGatherScatterOpCost(
-    unsigned Opcode, Type *DataTy, Value *Ptr, bool VariableMask,
-    unsigned Alignment, TTI::TargetCostKind CostKind,
-    const Instruction *I) {
+    unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
+    Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) {
   return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
                                        Alignment, CostKind, I);
 }
 
-unsigned HexagonTTIImpl::getInterleavedMemoryOpCost(unsigned Opcode,
-      Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
-      unsigned Alignment, unsigned AddressSpace,
-      TTI::TargetCostKind CostKind, bool UseMaskForCond,
-      bool UseMaskForGaps) {
+unsigned HexagonTTIImpl::getInterleavedMemoryOpCost(
+    unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+    Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
+    bool UseMaskForCond, bool UseMaskForGaps) {
   if (Indices.size() != Factor || UseMaskForCond || UseMaskForGaps)
     return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
                                              Alignment, AddressSpace,
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 3735844538510..5fe397486402e 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -64,6 +64,9 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP);
 
+  void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                             TTI::PeelingPreferences &PP);
+
   /// Bias LSR towards creating post-increment opportunities.
   bool shouldFavorPostInc() const;
 
@@ -115,20 +118,21 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
                            unsigned AddressSpace,
                            TTI::TargetCostKind CostKind,
                            const Instruction *I = nullptr);
-  unsigned getMaskedMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment,
-                                 unsigned AddressSpace,
-                           TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency);
+  unsigned
+  getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
+                        unsigned AddressSpace,
+                        TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency);
   unsigned getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
             Type *SubTp);
-  unsigned getGatherScatterOpCost(unsigned Opcode, Type *DataTy, Value *Ptr,
-                                  bool VariableMask, unsigned Alignment,
-                                  TTI::TargetCostKind CostKind,
+  unsigned getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
+                                  const Value *Ptr, bool VariableMask,
+                                  Align Alignment, TTI::TargetCostKind CostKind,
                                   const Instruction *I);
-  unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
-            unsigned Factor, ArrayRef<unsigned> Indices, unsigned Alignment,
-            unsigned AddressSpace,
-            TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
-            bool UseMaskForCond = false, bool UseMaskForGaps = false);
+  unsigned getInterleavedMemoryOpCost(
+      unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+      Align Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
+      bool UseMaskForCond = false, bool UseMaskForGaps = false);
   unsigned getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
                               TTI::TargetCostKind CostKind,
                               const Instruction *I = nullptr);
diff --git a/llvm/lib/Target/Hexagon/HexagonVExtract.cpp b/llvm/lib/Target/Hexagon/HexagonVExtract.cpp
index 83887260b9d95..b5f06ebd3189f 100644
--- a/llvm/lib/Target/Hexagon/HexagonVExtract.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonVExtract.cpp
@@ -107,7 +107,7 @@ bool HexagonVExtract::runOnMachineFunction(MachineFunction &MF) {
   Register AR =
       MF.getInfo<HexagonMachineFunctionInfo>()->getStackAlignBaseVReg();
   std::map<unsigned, SmallVector<MachineInstr*,4>> VExtractMap;
-  unsigned MaxAlign = 0;
+  MaybeAlign MaxAlign;
   bool Changed = false;
 
   for (MachineBasicBlock &MBB : MF) {
@@ -137,14 +137,14 @@ bool HexagonVExtract::runOnMachineFunction(MachineFunction &MF) {
       continue;
 
     const auto &VecRC = *MRI.getRegClass(VecR);
-    unsigned Align = HRI.getSpillAlignment(VecRC);
-    MaxAlign = std::max(MaxAlign, Align);
+    Align Alignment = HRI.getSpillAlign(VecRC);
+    MaxAlign = max(MaxAlign, Alignment);
     // Make sure this is not a spill slot: spill slots cannot be aligned
     // if there are variable-sized objects on the stack. They must be
     // accessible via FP (which is not aligned), because SP is unknown,
     // and AP may not be available at the location of the load/store.
-    int FI = MFI.CreateStackObject(HRI.getSpillSize(VecRC), Align,
-                                   /*isSpillSlot*/false);
+    int FI = MFI.CreateStackObject(HRI.getSpillSize(VecRC), Alignment,
+                                   /*isSpillSlot*/ false);
 
     MachineInstr *DefI = MRI.getVRegDef(VecR);
     MachineBasicBlock::iterator At = std::next(DefI->getIterator());
@@ -178,13 +178,13 @@ bool HexagonVExtract::runOnMachineFunction(MachineFunction &MF) {
     }
   }
 
-  if (AR) {
+  if (AR && MaxAlign) {
     // Update the required stack alignment.
     MachineInstr *AlignaI = MRI.getVRegDef(AR);
     assert(AlignaI->getOpcode() == Hexagon::PS_aligna);
     MachineOperand &Op = AlignaI->getOperand(1);
-    if (MaxAlign > Op.getImm())
-      Op.setImm(MaxAlign);
+    if (*MaxAlign > Op.getImm())
+      Op.setImm(MaxAlign->value());
   }
 
   return Changed;
diff --git a/llvm/lib/Target/Lanai/LanaiSelectionDAGInfo.cpp b/llvm/lib/Target/Lanai/LanaiSelectionDAGInfo.cpp
index dff87a3e264d9..307619c044818 100644
--- a/llvm/lib/Target/Lanai/LanaiSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/Lanai/LanaiSelectionDAGInfo.cpp
@@ -20,7 +20,7 @@ namespace llvm {
 
 SDValue LanaiSelectionDAGInfo::EmitTargetCodeForMemcpy(
     SelectionDAG & /*DAG*/, const SDLoc & /*dl*/, SDValue /*Chain*/,
-    SDValue /*Dst*/, SDValue /*Src*/, SDValue Size, unsigned /*Align*/,
+    SDValue /*Dst*/, SDValue /*Src*/, SDValue Size, Align /*Alignment*/,
     bool /*isVolatile*/, bool /*AlwaysInline*/,
     MachinePointerInfo /*DstPtrInfo*/,
     MachinePointerInfo /*SrcPtrInfo*/) const {
diff --git a/llvm/lib/Target/Lanai/LanaiSelectionDAGInfo.h b/llvm/lib/Target/Lanai/LanaiSelectionDAGInfo.h
index c5650a7c1f53b..8355168a7396f 100644
--- a/llvm/lib/Target/Lanai/LanaiSelectionDAGInfo.h
+++ b/llvm/lib/Target/Lanai/LanaiSelectionDAGInfo.h
@@ -24,8 +24,8 @@ class LanaiSelectionDAGInfo : public SelectionDAGTargetInfo {
 
   SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
                                   SDValue Chain, SDValue Dst, SDValue Src,
-                                  SDValue Size, unsigned Align, bool isVolatile,
-                                  bool AlwaysInline,
+                                  SDValue Size, Align Alignment,
+                                  bool isVolatile, bool AlwaysInline,
                                   MachinePointerInfo DstPtrInfo,
                                   MachinePointerInfo SrcPtrInfo) const override;
 };
diff --git a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
index 5079f16ea7340..cbff8d1773ff0 100644
--- a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
+++ b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
@@ -562,7 +562,7 @@ static unsigned convertGR16ToGR8(unsigned Reg) {
   case MSP430::SP:  return MSP430::SPB;
   case MSP430::SR:  return MSP430::SRB;
   case MSP430::CG:  return MSP430::CGB;
-  case MSP430::FP:  return MSP430::FPB;
+  case MSP430::R4:  return MSP430::R4B;
   case MSP430::R5:  return MSP430::R5B;
   case MSP430::R6:  return MSP430::R6B;
   case MSP430::R7:  return MSP430::R7B;
diff --git a/llvm/lib/Target/MSP430/Disassembler/MSP430Disassembler.cpp b/llvm/lib/Target/MSP430/Disassembler/MSP430Disassembler.cpp
index 6aa76156bf140..d2902189ec403 100644
--- a/llvm/lib/Target/MSP430/Disassembler/MSP430Disassembler.cpp
+++ b/llvm/lib/Target/MSP430/Disassembler/MSP430Disassembler.cpp
@@ -65,7 +65,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeMSP430Disassembler() {
 
 static const unsigned GR8DecoderTable[] = {
   MSP430::PCB,  MSP430::SPB,  MSP430::SRB,  MSP430::CGB,
-  MSP430::FPB,  MSP430::R5B,  MSP430::R6B,  MSP430::R7B,
+  MSP430::R4B,  MSP430::R5B,  MSP430::R6B,  MSP430::R7B,
   MSP430::R8B,  MSP430::R9B,  MSP430::R10B, MSP430::R11B,
   MSP430::R12B, MSP430::R13B, MSP430::R14B, MSP430::R15B
 };
@@ -83,7 +83,7 @@ static DecodeStatus DecodeGR8RegisterClass(MCInst &MI, uint64_t RegNo,
 
 static const unsigned GR16DecoderTable[] = {
   MSP430::PC,  MSP430::SP,  MSP430::SR,  MSP430::CG,
-  MSP430::FP,  MSP430::R5,  MSP430::R6,  MSP430::R7,
+  MSP430::R4,  MSP430::R5,  MSP430::R6,  MSP430::R7,
   MSP430::R8,  MSP430::R9,  MSP430::R10, MSP430::R11,
   MSP430::R12, MSP430::R13, MSP430::R14, MSP430::R15
 };
diff --git a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp
index 6117567dcaea6..de07b47096d34 100644
--- a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp
+++ b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp
@@ -24,4 +24,6 @@ MSP430MCAsmInfo::MSP430MCAsmInfo(const Triple &TT,
 
   AlignmentIsInBytes = false;
   UsesELFSectionDirectiveForBSS = true;
+
+  SupportsDebugInformation = true;
 }
diff --git a/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp b/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp
index 567180476ae0e..4be8d0760e689 100644
--- a/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp
+++ b/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp
@@ -64,16 +64,16 @@ void MSP430FrameLowering::emitPrologue(MachineFunction &MF,
 
     // Save FP into the appropriate stack slot...
     BuildMI(MBB, MBBI, DL, TII.get(MSP430::PUSH16r))
-      .addReg(MSP430::FP, RegState::Kill);
+      .addReg(MSP430::R4, RegState::Kill);
 
     // Update FP with the new base value...
-    BuildMI(MBB, MBBI, DL, TII.get(MSP430::MOV16rr), MSP430::FP)
+    BuildMI(MBB, MBBI, DL, TII.get(MSP430::MOV16rr), MSP430::R4)
       .addReg(MSP430::SP);
 
     // Mark the FramePtr as live-in in every block except the entry.
     for (MachineFunction::iterator I = std::next(MF.begin()), E = MF.end();
          I != E; ++I)
-      I->addLiveIn(MSP430::FP);
+      I->addLiveIn(MSP430::R4);
 
   } else
     NumBytes = StackSize - MSP430FI->getCalleeSavedFrameSize();
@@ -132,7 +132,7 @@ void MSP430FrameLowering::emitEpilogue(MachineFunction &MF,
     NumBytes = FrameSize - CSSize;
 
     // pop FP.
-    BuildMI(MBB, MBBI, DL, TII.get(MSP430::POP16r), MSP430::FP);
+    BuildMI(MBB, MBBI, DL, TII.get(MSP430::POP16r), MSP430::R4);
   } else
     NumBytes = StackSize - CSSize;
 
@@ -154,7 +154,7 @@ void MSP430FrameLowering::emitEpilogue(MachineFunction &MF,
 
   if (MFI.hasVarSizedObjects()) {
     BuildMI(MBB, MBBI, DL,
-            TII.get(MSP430::MOV16rr), MSP430::SP).addReg(MSP430::FP);
+            TII.get(MSP430::MOV16rr), MSP430::SP).addReg(MSP430::R4);
     if (CSSize) {
       MachineInstr *MI =
         BuildMI(MBB, MBBI, DL,
diff --git a/llvm/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp b/llvm/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp
index 91b0f25aeede1..7dabb9b4abaeb 100644
--- a/llvm/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp
@@ -50,7 +50,7 @@ namespace {
     const BlockAddress *BlockAddr = nullptr;
     const char *ES = nullptr;
     int JT = -1;
-    unsigned Align = 0;  // CP alignment.
+    Align Alignment; // CP alignment.
 
     MSP430ISelAddressMode() = default;
 
@@ -74,12 +74,12 @@ namespace {
       } else if (CP) {
         errs() << " CP ";
         CP->dump();
-        errs() << " Align" << Align << '\n';
+        errs() << " Align" << Alignment.value() << '\n';
       } else if (ES) {
         errs() << "ES ";
         errs() << ES << '\n';
       } else if (JT != -1)
-        errs() << " JT" << JT << " Align" << Align << '\n';
+        errs() << " JT" << JT << " Align" << Alignment.value() << '\n';
     }
 #endif
   };
@@ -146,7 +146,7 @@ bool MSP430DAGToDAGISel::MatchWrapper(SDValue N, MSP430ISelAddressMode &AM) {
     //AM.SymbolFlags = G->getTargetFlags();
   } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) {
     AM.CP = CP->getConstVal();
-    AM.Align = CP->getAlign().value();
+    AM.Alignment = CP->getAlign();
     AM.Disp += CP->getOffset();
     //AM.SymbolFlags = CP->getTargetFlags();
   } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) {
@@ -263,8 +263,8 @@ bool MSP430DAGToDAGISel::SelectAddr(SDValue N,
                                           MVT::i16, AM.Disp,
                                           0/*AM.SymbolFlags*/);
   else if (AM.CP)
-    Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i16, Align(AM.Align),
-                                         AM.Disp, 0 /*AM.SymbolFlags*/);
+    Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i16, AM.Alignment, AM.Disp,
+                                         0 /*AM.SymbolFlags*/);
   else if (AM.ES)
     Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i16, 0/*AM.SymbolFlags*/);
   else if (AM.JT != -1)
diff --git a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
index 34f3d8ea61f65..821339f503551 100644
--- a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
+++ b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
@@ -1300,7 +1300,7 @@ SDValue MSP430TargetLowering::LowerFRAMEADDR(SDValue Op,
   SDLoc dl(Op);  // FIXME probably not meaningful
   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl,
-                                         MSP430::FP, VT);
+                                         MSP430::R4, VT);
   while (Depth--)
     FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
                             MachinePointerInfo());
diff --git a/llvm/lib/Target/MSP430/MSP430ISelLowering.h b/llvm/lib/Target/MSP430/MSP430ISelLowering.h
index 650f9a7040628..f23042a369fd6 100644
--- a/llvm/lib/Target/MSP430/MSP430ISelLowering.h
+++ b/llvm/lib/Target/MSP430/MSP430ISelLowering.h
@@ -79,6 +79,10 @@ namespace llvm {
       return MVT::i8;
     }
 
+    MVT::SimpleValueType getCmpLibcallReturnType() const override {
+      return MVT::i16;
+    }
+
     /// LowerOperation - Provide custom lowering hooks for some operations.
     SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
 
diff --git a/llvm/lib/Target/MSP430/MSP430RegisterInfo.cpp b/llvm/lib/Target/MSP430/MSP430RegisterInfo.cpp
index bec357a1548d6..5583ebee2f311 100644
--- a/llvm/lib/Target/MSP430/MSP430RegisterInfo.cpp
+++ b/llvm/lib/Target/MSP430/MSP430RegisterInfo.cpp
@@ -39,7 +39,7 @@ MSP430RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
   const MSP430FrameLowering *TFI = getFrameLowering(*MF);
   const Function* F = &MF->getFunction();
   static const MCPhysReg CalleeSavedRegs[] = {
-    MSP430::FP, MSP430::R5, MSP430::R6, MSP430::R7,
+    MSP430::R4, MSP430::R5, MSP430::R6, MSP430::R7,
     MSP430::R8, MSP430::R9, MSP430::R10,
     0
   };
@@ -49,7 +49,7 @@ MSP430RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
     0
   };
   static const MCPhysReg CalleeSavedRegsIntr[] = {
-    MSP430::FP,  MSP430::R5,  MSP430::R6,  MSP430::R7,
+    MSP430::R4,  MSP430::R5,  MSP430::R6,  MSP430::R7,
     MSP430::R8,  MSP430::R9,  MSP430::R10, MSP430::R11,
     MSP430::R12, MSP430::R13, MSP430::R14, MSP430::R15,
     0
@@ -86,8 +86,8 @@ BitVector MSP430RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
 
   // Mark frame pointer as reserved if needed.
   if (TFI->hasFP(MF)) {
-    Reserved.set(MSP430::FPB);
-    Reserved.set(MSP430::FP);
+    Reserved.set(MSP430::R4B);
+    Reserved.set(MSP430::R4);
   }
 
   return Reserved;
@@ -112,7 +112,7 @@ MSP430RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   DebugLoc dl = MI.getDebugLoc();
   int FrameIndex = MI.getOperand(FIOperandNum).getIndex();
 
-  unsigned BasePtr = (TFI->hasFP(MF) ? MSP430::FP : MSP430::SP);
+  unsigned BasePtr = (TFI->hasFP(MF) ? MSP430::R4 : MSP430::SP);
   int Offset = MF.getFrameInfo().getObjectOffset(FrameIndex);
 
   // Skip the saved PC
@@ -156,5 +156,5 @@ MSP430RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
 
 Register MSP430RegisterInfo::getFrameRegister(const MachineFunction &MF) const {
   const MSP430FrameLowering *TFI = getFrameLowering(MF);
-  return TFI->hasFP(MF) ? MSP430::FP : MSP430::SP;
+  return TFI->hasFP(MF) ? MSP430::R4 : MSP430::SP;
 }
diff --git a/llvm/lib/Target/MSP430/MSP430RegisterInfo.td b/llvm/lib/Target/MSP430/MSP430RegisterInfo.td
index 11003dba383f4..61cc72d494b5b 100644
--- a/llvm/lib/Target/MSP430/MSP430RegisterInfo.td
+++ b/llvm/lib/Target/MSP430/MSP430RegisterInfo.td
@@ -15,6 +15,7 @@ class MSP430Reg<bits<4> num, string n, list<string> alt = []> : Register<n> {
   let Namespace = "MSP430";
   let HWEncoding{3-0} = num;
   let AltNames = alt;
+  let DwarfNumbers = [num];
 }
 
 class MSP430RegWithSubregs<bits<4> num, string n, list<Register> subregs,
@@ -24,6 +25,7 @@ class MSP430RegWithSubregs<bits<4> num, string n, list<Register> subregs,
   let Namespace = "MSP430";
   let HWEncoding{3-0} = num;
   let AltNames = alt;
+  let DwarfNumbers = [num];
 }
 
 //===----------------------------------------------------------------------===//
@@ -34,7 +36,7 @@ def PCB  : MSP430Reg<0,  "r0", ["pc"]>;
 def SPB  : MSP430Reg<1,  "r1", ["sp"]>;
 def SRB  : MSP430Reg<2,  "r2", ["sr"]>;
 def CGB  : MSP430Reg<3,  "r3", ["cg"]>;
-def FPB  : MSP430Reg<4,  "r4", ["fp"]>;
+def R4B  : MSP430Reg<4,  "r4", ["fp"]>;
 def R5B  : MSP430Reg<5,  "r5">;
 def R6B  : MSP430Reg<6,  "r6">;
 def R7B  : MSP430Reg<7,  "r7">;
@@ -54,7 +56,7 @@ def PC  : MSP430RegWithSubregs<0,  "r0",  [PCB], ["pc"]>;
 def SP  : MSP430RegWithSubregs<1,  "r1",  [SPB], ["sp"]>;
 def SR  : MSP430RegWithSubregs<2,  "r2",  [SRB], ["sr"]>;
 def CG  : MSP430RegWithSubregs<3,  "r3",  [CGB], ["cg"]>;
-def FP  : MSP430RegWithSubregs<4,  "r4",  [FPB], ["fp"]>;
+def R4  : MSP430RegWithSubregs<4,  "r4",  [R4B], ["fp"]>;
 def R5  : MSP430RegWithSubregs<5,  "r5",  [R5B]>;
 def R6  : MSP430RegWithSubregs<6,  "r6",  [R6B]>;
 def R7  : MSP430RegWithSubregs<7,  "r7",  [R7B]>;
@@ -72,7 +74,7 @@ def GR8 : RegisterClass<"MSP430", [i8], 8,
    // Volatile registers
   (add R12B, R13B, R14B, R15B, R11B, R10B, R9B, R8B, R7B, R6B, R5B,
    // Frame pointer, sometimes allocable
-   FPB,
+   R4B,
    // Volatile, but not allocable
    PCB, SPB, SRB, CGB)>;
 
@@ -80,6 +82,6 @@ def GR16 : RegisterClass<"MSP430", [i16], 16,
    // Volatile registers
   (add R12, R13, R14, R15, R11, R10, R9, R8, R7, R6, R5,
    // Frame pointer, sometimes allocable
-   FP,
+   R4,
    // Volatile, but not allocable
    PC, SP, SR, CG)>;
diff --git a/llvm/lib/Target/Mips/Mips16ISelDAGToDAG.cpp b/llvm/lib/Target/Mips/Mips16ISelDAGToDAG.cpp
index 768d54fc9c24e..ddd28d095e513 100644
--- a/llvm/lib/Target/Mips/Mips16ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/Mips/Mips16ISelDAGToDAG.cpp
@@ -72,7 +72,7 @@ void Mips16DAGToDAGISel::initGlobalBaseReg(MachineFunction &MF) {
   MachineRegisterInfo &RegInfo = MF.getRegInfo();
   const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
   DebugLoc DL;
-  Register V0, V1, V2, GlobalBaseReg = MipsFI->getGlobalBaseReg();
+  Register V0, V1, V2, GlobalBaseReg = MipsFI->getGlobalBaseReg(MF);
   const TargetRegisterClass *RC = &Mips::CPU16RegsRegClass;
 
   V0 = RegInfo.createVirtualRegister(RC);
diff --git a/llvm/lib/Target/Mips/Mips16ISelLowering.cpp b/llvm/lib/Target/Mips/Mips16ISelLowering.cpp
index 5425df77d9b8f..a3b86bdc2ca01 100644
--- a/llvm/lib/Target/Mips/Mips16ISelLowering.cpp
+++ b/llvm/lib/Target/Mips/Mips16ISelLowering.cpp
@@ -492,7 +492,7 @@ getOpndList(SmallVectorImpl<SDValue> &Ops,
       ExternalSymbolSDNode *S = cast<ExternalSymbolSDNode>(JumpTarget);
       JumpTarget = getAddrGlobal(S, CLI.DL, JumpTarget.getValueType(), DAG,
                                  MipsII::MO_GOT, Chain,
-                                 FuncInfo->callPtrInfo(S->getSymbol()));
+                                 FuncInfo->callPtrInfo(MF, S->getSymbol()));
     } else
       RegsToPass.push_front(std::make_pair((unsigned)Mips::T9, Callee));
   }
diff --git a/llvm/lib/Target/Mips/MipsCallLowering.cpp b/llvm/lib/Target/Mips/MipsCallLowering.cpp
index d7f02683abf2c..cffd99affac10 100644
--- a/llvm/lib/Target/Mips/MipsCallLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsCallLowering.cpp
@@ -593,7 +593,7 @@ bool MipsCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   if (IsCalleeGlobalPIC) {
     MIRBuilder.buildCopy(
       Register(Mips::GP),
-      MF.getInfo<MipsFunctionInfo>()->getGlobalBaseRegForGlobalISel());
+      MF.getInfo<MipsFunctionInfo>()->getGlobalBaseRegForGlobalISel(MF));
     MIB.addDef(Mips::GP, RegState::Implicit);
   }
   MIRBuilder.insertInstr(MIB);
diff --git a/llvm/lib/Target/Mips/MipsFastISel.cpp b/llvm/lib/Target/Mips/MipsFastISel.cpp
index 1b6a75e7c383e..8a847eaf66184 100644
--- a/llvm/lib/Target/Mips/MipsFastISel.cpp
+++ b/llvm/lib/Target/Mips/MipsFastISel.cpp
@@ -420,7 +420,7 @@ unsigned MipsFastISel::materializeGV(const GlobalValue *GV, MVT VT) {
   if (IsThreadLocal)
     return 0;
   emitInst(Mips::LW, DestReg)
-      .addReg(MFI->getGlobalBaseReg())
+      .addReg(MFI->getGlobalBaseReg(*MF))
       .addGlobalAddress(GV, 0, MipsII::MO_GOT);
   if ((GV->hasInternalLinkage() ||
        (GV->hasLocalLinkage() && !isa<Function>(GV)))) {
@@ -437,7 +437,7 @@ unsigned MipsFastISel::materializeExternalCallSym(MCSymbol *Sym) {
   const TargetRegisterClass *RC = &Mips::GPR32RegClass;
   unsigned DestReg = createResultReg(RC);
   emitInst(Mips::LW, DestReg)
-      .addReg(MFI->getGlobalBaseReg())
+      .addReg(MFI->getGlobalBaseReg(*MF))
       .addSym(Sym, MipsII::MO_GOT);
   return DestReg;
 }
diff --git a/llvm/lib/Target/Mips/MipsISelDAGToDAG.cpp b/llvm/lib/Target/Mips/MipsISelDAGToDAG.cpp
index 37df04e3cbb51..d88696525e9e5 100644
--- a/llvm/lib/Target/Mips/MipsISelDAGToDAG.cpp
+++ b/llvm/lib/Target/Mips/MipsISelDAGToDAG.cpp
@@ -65,7 +65,7 @@ bool MipsDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
 /// getGlobalBaseReg - Output the instructions required to put the
 /// GOT address into a register.
 SDNode *MipsDAGToDAGISel::getGlobalBaseReg() {
-  Register GlobalBaseReg = MF->getInfo<MipsFunctionInfo>()->getGlobalBaseReg();
+  Register GlobalBaseReg = MF->getInfo<MipsFunctionInfo>()->getGlobalBaseReg(*MF);
   return CurDAG->getRegister(GlobalBaseReg, getTargetLowering()->getPointerTy(
                                                 CurDAG->getDataLayout()))
       .getNode();
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index 6302c87106028..2da35020006e2 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -142,8 +142,9 @@ unsigned MipsTargetLowering::getVectorTypeBreakdownForCallingConv(
 }
 
 SDValue MipsTargetLowering::getGlobalReg(SelectionDAG &DAG, EVT Ty) const {
-  MipsFunctionInfo *FI = DAG.getMachineFunction().getInfo<MipsFunctionInfo>();
-  return DAG.getRegister(FI->getGlobalBaseReg(), Ty);
+  MachineFunction &MF = DAG.getMachineFunction();
+  MipsFunctionInfo *FI = MF.getInfo<MipsFunctionInfo>();
+  return DAG.getRegister(FI->getGlobalBaseReg(MF), Ty);
 }
 
 SDValue MipsTargetLowering::getTargetNode(GlobalAddressSDNode *N, EVT Ty,
@@ -2908,8 +2909,8 @@ static bool CC_MipsO32(unsigned ValNo, MVT ValVT, MVT LocVT,
   // argument which is not f32 or f64.
   bool AllocateFloatsInIntReg = State.isVarArg() || ValNo > 1 ||
                                 State.getFirstUnallocated(F32Regs) != ValNo;
-  unsigned OrigAlign = ArgFlags.getOrigAlign();
-  bool isI64 = (ValVT == MVT::i32 && OrigAlign == 8);
+  Align OrigAlign = ArgFlags.getNonZeroOrigAlign();
+  bool isI64 = (ValVT == MVT::i32 && OrigAlign == Align(8));
   bool isVectorFloat = MipsState->WasOriginalArgVectorFloat(ValNo);
 
   // The MIPS vector ABI for floats passes them in a pair of registers
@@ -2963,8 +2964,7 @@ static bool CC_MipsO32(unsigned ValNo, MVT ValVT, MVT LocVT,
     llvm_unreachable("Cannot handle this ValVT.");
 
   if (!Reg) {
-    unsigned Offset =
-        State.AllocateStack(ValVT.getStoreSize(), Align(OrigAlign));
+    unsigned Offset = State.AllocateStack(ValVT.getStoreSize(), OrigAlign);
     State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
   } else
     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
@@ -3421,11 +3421,11 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       else if (Subtarget.useXGOT()) {
         Callee = getAddrGlobalLargeGOT(G, DL, Ty, DAG, MipsII::MO_CALL_HI16,
                                        MipsII::MO_CALL_LO16, Chain,
-                                       FuncInfo->callPtrInfo(Val));
+                                       FuncInfo->callPtrInfo(MF, Val));
         IsCallReloc = true;
       } else {
         Callee = getAddrGlobal(G, DL, Ty, DAG, MipsII::MO_GOT_CALL, Chain,
-                               FuncInfo->callPtrInfo(Val));
+                               FuncInfo->callPtrInfo(MF, Val));
         IsCallReloc = true;
       }
     } else
@@ -3443,11 +3443,11 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     else if (Subtarget.useXGOT()) {
       Callee = getAddrGlobalLargeGOT(S, DL, Ty, DAG, MipsII::MO_CALL_HI16,
                                      MipsII::MO_CALL_LO16, Chain,
-                                     FuncInfo->callPtrInfo(Sym));
+                                     FuncInfo->callPtrInfo(MF, Sym));
       IsCallReloc = true;
     } else { // PIC
       Callee = getAddrGlobal(S, DL, Ty, DAG, MipsII::MO_GOT_CALL, Chain,
-                             FuncInfo->callPtrInfo(Sym));
+                             FuncInfo->callPtrInfo(MF, Sym));
       IsCallReloc = true;
     }
 
diff --git a/llvm/lib/Target/Mips/MipsInstrInfo.cpp b/llvm/lib/Target/Mips/MipsInstrInfo.cpp
index 836fdcd6c4859..0c6080258a3a2 100644
--- a/llvm/lib/Target/Mips/MipsInstrInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsInstrInfo.cpp
@@ -859,6 +859,14 @@ MipsInstrInfo::describeLoadedValue(const MachineInstr &MI, Register Reg) const {
     }
     Expr = DIExpression::prepend(Expr, DIExpression::ApplyOffset, Offset);
     return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr);
+  } else if (auto DestSrc = isCopyInstr(MI)) {
+    const MachineFunction *MF = MI.getMF();
+    const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
+    Register DestReg = DestSrc->Destination->getReg();
+    // TODO: Handle cases where the Reg is sub- or super-register of the
+    // DestReg.
+    if (TRI->isSuperRegister(Reg, DestReg) || TRI->isSubRegister(Reg, DestReg))
+      return None;
   }
 
   return TargetInstrInfo::describeLoadedValue(MI, Reg);
diff --git a/llvm/lib/Target/Mips/MipsInstructionSelector.cpp b/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
index 3f63f40b90176..256fb74c1d6c9 100644
--- a/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
+++ b/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
@@ -404,7 +404,7 @@ bool MipsInstructionSelector::select(MachineInstr &I) {
                                .addDef(Dest)
                                .addUse(DestTmp)
                                .addUse(MF.getInfo<MipsFunctionInfo>()
-                                           ->getGlobalBaseRegForGlobalISel());
+                                           ->getGlobalBaseRegForGlobalISel(MF));
       if (!constrainSelectedInstRegOperands(*ADDu, TII, TRI, RBI))
         return false;
     }
@@ -669,7 +669,7 @@ bool MipsInstructionSelector::select(MachineInstr &I) {
       MachineInstr *LWGOT = BuildMI(MBB, I, I.getDebugLoc(), TII.get(Mips::LW))
                                 .addDef(I.getOperand(0).getReg())
                                 .addReg(MF.getInfo<MipsFunctionInfo>()
-                                            ->getGlobalBaseRegForGlobalISel())
+                                            ->getGlobalBaseRegForGlobalISel(MF))
                                 .addGlobalAddress(GVal);
       // Global Values that don't have local linkage are handled differently
       // when they are part of call sequence. MipsCallLowering::lowerCall
@@ -725,7 +725,7 @@ bool MipsInstructionSelector::select(MachineInstr &I) {
       MI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(Mips::LW))
                .addDef(I.getOperand(0).getReg())
                .addReg(MF.getInfo<MipsFunctionInfo>()
-                           ->getGlobalBaseRegForGlobalISel())
+                           ->getGlobalBaseRegForGlobalISel(MF))
                .addJumpTableIndex(I.getOperand(1).getIndex(), MipsII::MO_GOT)
                .addMemOperand(MF.getMachineMemOperand(
                    MachinePointerInfo::getGOT(MF), MachineMemOperand::MOLoad, 4,
diff --git a/llvm/lib/Target/Mips/MipsMachineFunction.cpp b/llvm/lib/Target/Mips/MipsMachineFunction.cpp
index 7e5c37b07976c..a7a2be30f58a4 100644
--- a/llvm/lib/Target/Mips/MipsMachineFunction.cpp
+++ b/llvm/lib/Target/Mips/MipsMachineFunction.cpp
@@ -44,22 +44,22 @@ static const TargetRegisterClass &getGlobalBaseRegClass(MachineFunction &MF) {
   return Mips::GPR32RegClass;
 }
 
-Register MipsFunctionInfo::getGlobalBaseReg() {
+Register MipsFunctionInfo::getGlobalBaseReg(MachineFunction &MF) {
   if (!GlobalBaseReg)
     GlobalBaseReg =
         MF.getRegInfo().createVirtualRegister(&getGlobalBaseRegClass(MF));
   return GlobalBaseReg;
 }
 
-Register MipsFunctionInfo::getGlobalBaseRegForGlobalISel() {
+Register MipsFunctionInfo::getGlobalBaseRegForGlobalISel(MachineFunction &MF) {
   if (!GlobalBaseReg) {
-    getGlobalBaseReg();
-    initGlobalBaseReg();
+    getGlobalBaseReg(MF);
+    initGlobalBaseReg(MF);
   }
   return GlobalBaseReg;
 }
 
-void MipsFunctionInfo::initGlobalBaseReg() {
+void MipsFunctionInfo::initGlobalBaseReg(MachineFunction &MF) {
   if (!GlobalBaseReg)
     return;
 
@@ -146,7 +146,7 @@ void MipsFunctionInfo::initGlobalBaseReg() {
       .addReg(Mips::V0).addReg(Mips::T9);
 }
 
-void MipsFunctionInfo::createEhDataRegsFI() {
+void MipsFunctionInfo::createEhDataRegsFI(MachineFunction &MF) {
   const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
   for (int I = 0; I < 4; ++I) {
     const TargetRegisterClass &RC =
@@ -154,12 +154,12 @@ void MipsFunctionInfo::createEhDataRegsFI() {
             ? Mips::GPR64RegClass
             : Mips::GPR32RegClass;
 
-    EhDataRegFI[I] = MF.getFrameInfo().CreateStackObject(TRI.getSpillSize(RC),
-        TRI.getSpillAlignment(RC), false);
+    EhDataRegFI[I] = MF.getFrameInfo().CreateStackObject(
+        TRI.getSpillSize(RC), TRI.getSpillAlign(RC), false);
   }
 }
 
-void MipsFunctionInfo::createISRRegFI() {
+void MipsFunctionInfo::createISRRegFI(MachineFunction &MF) {
   // ISRs require spill slots for Status & ErrorPC Coprocessor 0 registers.
   // The current implementation only supports Mips32r2+ not Mips64rX. Status
   // is always 32 bits, ErrorPC is 32 or 64 bits dependent on architecture,
@@ -169,7 +169,7 @@ void MipsFunctionInfo::createISRRegFI() {
 
   for (int I = 0; I < 2; ++I)
     ISRDataRegFI[I] = MF.getFrameInfo().CreateStackObject(
-        TRI.getSpillSize(RC), TRI.getSpillAlignment(RC), false);
+        TRI.getSpillSize(RC), TRI.getSpillAlign(RC), false);
 }
 
 bool MipsFunctionInfo::isEhDataRegFI(int FI) const {
@@ -180,19 +180,22 @@ bool MipsFunctionInfo::isEhDataRegFI(int FI) const {
 bool MipsFunctionInfo::isISRRegFI(int FI) const {
   return IsISR && (FI == ISRDataRegFI[0] || FI == ISRDataRegFI[1]);
 }
-MachinePointerInfo MipsFunctionInfo::callPtrInfo(const char *ES) {
+MachinePointerInfo MipsFunctionInfo::callPtrInfo(MachineFunction &MF,
+                                                 const char *ES) {
   return MachinePointerInfo(MF.getPSVManager().getExternalSymbolCallEntry(ES));
 }
 
-MachinePointerInfo MipsFunctionInfo::callPtrInfo(const GlobalValue *GV) {
+MachinePointerInfo MipsFunctionInfo::callPtrInfo(MachineFunction &MF,
+                                                 const GlobalValue *GV) {
   return MachinePointerInfo(MF.getPSVManager().getGlobalValueCallEntry(GV));
 }
 
-int MipsFunctionInfo::getMoveF64ViaSpillFI(const TargetRegisterClass *RC) {
+int MipsFunctionInfo::getMoveF64ViaSpillFI(MachineFunction &MF,
+                                           const TargetRegisterClass *RC) {
   const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
   if (MoveF64ViaSpillFI == -1) {
     MoveF64ViaSpillFI = MF.getFrameInfo().CreateStackObject(
-        TRI.getSpillSize(*RC), TRI.getSpillAlignment(*RC), false);
+        TRI.getSpillSize(*RC), TRI.getSpillAlign(*RC), false);
   }
   return MoveF64ViaSpillFI;
 }
diff --git a/llvm/lib/Target/Mips/MipsMachineFunction.h b/llvm/lib/Target/Mips/MipsMachineFunction.h
index abb5dde3cf476..786d210e2aaa7 100644
--- a/llvm/lib/Target/Mips/MipsMachineFunction.h
+++ b/llvm/lib/Target/Mips/MipsMachineFunction.h
@@ -24,7 +24,7 @@ namespace llvm {
 /// Mips target-specific information for each MachineFunction.
 class MipsFunctionInfo : public MachineFunctionInfo {
 public:
-  MipsFunctionInfo(MachineFunction &MF) : MF(MF) {}
+  MipsFunctionInfo(MachineFunction &MF) {}
 
   ~MipsFunctionInfo() override;
 
@@ -32,12 +32,12 @@ class MipsFunctionInfo : public MachineFunctionInfo {
   void setSRetReturnReg(unsigned Reg) { SRetReturnReg = Reg; }
 
   bool globalBaseRegSet() const;
-  Register getGlobalBaseReg();
-  Register getGlobalBaseRegForGlobalISel();
+  Register getGlobalBaseReg(MachineFunction &MF);
+  Register getGlobalBaseRegForGlobalISel(MachineFunction &MF);
 
   // Insert instructions to initialize the global base register in the
   // first MBB of the function.
-  void initGlobalBaseReg();
+  void initGlobalBaseReg(MachineFunction &MF);
 
   int getVarArgsFrameIndex() const { return VarArgsFrameIndex; }
   void setVarArgsFrameIndex(int Index) { VarArgsFrameIndex = Index; }
@@ -53,30 +53,30 @@ class MipsFunctionInfo : public MachineFunctionInfo {
   bool callsEhReturn() const { return CallsEhReturn; }
   void setCallsEhReturn() { CallsEhReturn = true; }
 
-  void createEhDataRegsFI();
+  void createEhDataRegsFI(MachineFunction &MF);
   int getEhDataRegFI(unsigned Reg) const { return EhDataRegFI[Reg]; }
   bool isEhDataRegFI(int FI) const;
 
   /// Create a MachinePointerInfo that has an ExternalSymbolPseudoSourceValue
   /// object representing a GOT entry for an external function.
-  MachinePointerInfo callPtrInfo(const char *ES);
+  MachinePointerInfo callPtrInfo(MachineFunction &MF, const char *ES);
 
   // Functions with the "interrupt" attribute require special prologues,
   // epilogues and additional spill slots.
   bool isISR() const { return IsISR; }
   void setISR() { IsISR = true; }
-  void createISRRegFI();
+  void createISRRegFI(MachineFunction &MF);
   int getISRRegFI(Register Reg) const { return ISRDataRegFI[Reg]; }
   bool isISRRegFI(int FI) const;
 
   /// Create a MachinePointerInfo that has a GlobalValuePseudoSourceValue object
   /// representing a GOT entry for a global function.
-  MachinePointerInfo callPtrInfo(const GlobalValue *GV);
+  MachinePointerInfo callPtrInfo(MachineFunction &MF, const GlobalValue *GV);
 
   void setSaveS2() { SaveS2 = true; }
   bool hasSaveS2() const { return SaveS2; }
 
-  int getMoveF64ViaSpillFI(const TargetRegisterClass *RC);
+  int getMoveF64ViaSpillFI(MachineFunction &MF, const TargetRegisterClass *RC);
 
   std::map<const char *, const Mips16HardFloatInfo::FuncSignature *>
   StubsNeeded;
@@ -84,8 +84,6 @@ class MipsFunctionInfo : public MachineFunctionInfo {
 private:
   virtual void anchor();
 
-  MachineFunction& MF;
-
   /// SRetReturnReg - Some subtargets require that sret lowering includes
   /// returning the value of the returned struct in a register. This field
   /// holds the virtual register into which the sret argument is passed.
diff --git a/llvm/lib/Target/Mips/MipsOptimizePICCall.cpp b/llvm/lib/Target/Mips/MipsOptimizePICCall.cpp
index 8bd64ff6cb271..2823d300dc6ee 100644
--- a/llvm/lib/Target/Mips/MipsOptimizePICCall.cpp
+++ b/llvm/lib/Target/Mips/MipsOptimizePICCall.cpp
@@ -218,8 +218,7 @@ bool OptimizePICCall::runOnMachineFunction(MachineFunction &F) {
     MBBI.preVisit(ScopedHT);
     Changed |= visitNode(MBBI);
     const MachineDomTreeNode *Node = MBBI.getNode();
-    const std::vector<MachineDomTreeNode *> &Children = Node->getChildren();
-    WorkList.append(Children.begin(), Children.end());
+    WorkList.append(Node->begin(), Node->end());
   }
 
   return Changed;
diff --git a/llvm/lib/Target/Mips/MipsSEFrameLowering.cpp b/llvm/lib/Target/Mips/MipsSEFrameLowering.cpp
index f145aefc04192..a657bb44ac789 100644
--- a/llvm/lib/Target/Mips/MipsSEFrameLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsSEFrameLowering.cpp
@@ -320,7 +320,7 @@ bool ExpandPseudo::expandBuildPairF64(MachineBasicBlock &MBB,
 
     // We re-use the same spill slot each time so that the stack frame doesn't
     // grow too much in functions with a large number of moves.
-    int FI = MF.getInfo<MipsFunctionInfo>()->getMoveF64ViaSpillFI(RC2);
+    int FI = MF.getInfo<MipsFunctionInfo>()->getMoveF64ViaSpillFI(MF, RC2);
     if (!Subtarget.isLittle())
       std::swap(LoReg, HiReg);
     TII.storeRegToStack(MBB, I, LoReg, I->getOperand(1).isKill(), FI, RC,
@@ -386,7 +386,7 @@ bool ExpandPseudo::expandExtractElementF64(MachineBasicBlock &MBB,
 
     // We re-use the same spill slot each time so that the stack frame doesn't
     // grow too much in functions with a large number of moves.
-    int FI = MF.getInfo<MipsFunctionInfo>()->getMoveF64ViaSpillFI(RC);
+    int FI = MF.getInfo<MipsFunctionInfo>()->getMoveF64ViaSpillFI(MF, RC);
     TII.storeRegToStack(MBB, I, SrcReg, Op1.isKill(), FI, RC, &RegInfo, 0);
     TII.loadRegFromStack(MBB, I, DstReg, FI, RC2, &RegInfo, Offset);
     return true;
@@ -878,11 +878,11 @@ void MipsSEFrameLowering::determineCalleeSaves(MachineFunction &MF,
 
   // Create spill slots for eh data registers if function calls eh_return.
   if (MipsFI->callsEhReturn())
-    MipsFI->createEhDataRegsFI();
+    MipsFI->createEhDataRegsFI(MF);
 
   // Create spill slots for Coprocessor 0 registers if function is an ISR.
   if (MipsFI->isISR())
-    MipsFI->createISRRegFI();
+    MipsFI->createISRRegFI(MF);
 
   // Expand pseudo instructions which load, store or copy accumulators.
   // Add an emergency spill slot if a pseudo was expanded.
@@ -893,8 +893,7 @@ void MipsSEFrameLowering::determineCalleeSaves(MachineFunction &MF,
     const TargetRegisterClass &RC = STI.isGP64bit() ?
       Mips::GPR64RegClass : Mips::GPR32RegClass;
     int FI = MF.getFrameInfo().CreateStackObject(TRI->getSpillSize(RC),
-                                                 TRI->getSpillAlignment(RC),
-                                                 false);
+                                                 TRI->getSpillAlign(RC), false);
     RS->addScavengingFrameIndex(FI);
   }
 
@@ -910,8 +909,7 @@ void MipsSEFrameLowering::determineCalleeSaves(MachineFunction &MF,
   const TargetRegisterClass &RC =
       ABI.ArePtrs64bit() ? Mips::GPR64RegClass : Mips::GPR32RegClass;
   int FI = MF.getFrameInfo().CreateStackObject(TRI->getSpillSize(RC),
-                                               TRI->getSpillAlignment(RC),
-                                               false);
+                                               TRI->getSpillAlign(RC), false);
   RS->addScavengingFrameIndex(FI);
 }
 
diff --git a/llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp b/llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp
index 3ddba25fcef61..7be5fc33a0aff 100644
--- a/llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp
+++ b/llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp
@@ -153,7 +153,7 @@ void MipsSEDAGToDAGISel::emitMCountABI(MachineInstr &MI, MachineBasicBlock &MBB,
 }
 
 void MipsSEDAGToDAGISel::processFunctionAfterISel(MachineFunction &MF) {
-  MF.getInfo<MipsFunctionInfo>()->initGlobalBaseReg();
+  MF.getInfo<MipsFunctionInfo>()->initGlobalBaseReg(MF);
 
   MachineRegisterInfo *MRI = &MF.getRegInfo();
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index c68ac5cd024ab..da1a398a68f0d 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -762,13 +762,21 @@ static bool isEmptyXXStructor(GlobalVariable *GV) {
   return InitList->getNumOperands() == 0;
 }
 
-bool NVPTXAsmPrinter::doInitialization(Module &M) {
+void NVPTXAsmPrinter::emitStartOfAsmFile(Module &M) {
   // Construct a default subtarget off of the TargetMachine defaults. The
   // rest of NVPTX isn't friendly to change subtargets per function and
   // so the default TargetMachine will have all of the options.
   const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
   const auto* STI = static_cast<const NVPTXSubtarget*>(NTM.getSubtargetImpl());
+  SmallString<128> Str1;
+  raw_svector_ostream OS1(Str1);
 
+  // Emit header before any dwarf directives are emitted below.
+  emitHeader(M, OS1, *STI);
+  OutStreamer->emitRawText(OS1.str());
+}
+
+bool NVPTXAsmPrinter::doInitialization(Module &M) {
   if (M.alias_size()) {
     report_fatal_error("Module has aliases, which NVPTX does not support.");
     return true; // error
@@ -784,26 +792,9 @@ bool NVPTXAsmPrinter::doInitialization(Module &M) {
     return true;  // error
   }
 
-  SmallString<128> Str1;
-  raw_svector_ostream OS1(Str1);
-
   // We need to call the parent's one explicitly.
   bool Result = AsmPrinter::doInitialization(M);
 
-  // Emit header before any dwarf directives are emitted below.
-  emitHeader(M, OS1, *STI);
-  OutStreamer->emitRawText(OS1.str());
-
-  // Emit module-level inline asm if it exists.
-  if (!M.getModuleInlineAsm().empty()) {
-    OutStreamer->AddComment("Start of file scope inline assembly");
-    OutStreamer->AddBlankLine();
-    OutStreamer->emitRawText(StringRef(M.getModuleInlineAsm()));
-    OutStreamer->AddBlankLine();
-    OutStreamer->AddComment("End of file scope inline assembly");
-    OutStreamer->AddBlankLine();
-  }
-
   GlobalsEmitted = false;
 
   return Result;
diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
index ae2731b025ccb..5c3a4eb470c1c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
@@ -200,6 +200,7 @@ class LLVM_LIBRARY_VISIBILITY NVPTXAsmPrinter : public AsmPrinter {
   const Function *F;
   std::string CurrentFnName;
 
+  void emitStartOfAsmFile(Module &M) override;
   void emitBasicBlockStart(const MachineBasicBlock &MBB) override;
   void emitFunctionEntryLabel() override;
   void emitFunctionBodyStart() override;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 9669fb5964e38..f45cc06e0a0a3 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -2302,10 +2302,10 @@ NVPTXTargetLowering::LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const {
     MemSDNode *MemSD = cast<MemSDNode>(N);
     const DataLayout &TD = DAG.getDataLayout();
 
-    unsigned Align = MemSD->getAlignment();
-    unsigned PrefAlign =
-        TD.getPrefTypeAlignment(ValVT.getTypeForEVT(*DAG.getContext()));
-    if (Align < PrefAlign) {
+    Align Alignment = MemSD->getAlign();
+    Align PrefAlign =
+        TD.getPrefTypeAlign(ValVT.getTypeForEVT(*DAG.getContext()));
+    if (Alignment < PrefAlign) {
       // This store is not sufficiently aligned, so bail out and let this vector
       // store be scalarized.  Note that we may still be able to emit smaller
       // vector stores.  For example, if we are storing a <4 x float> with an
@@ -3780,8 +3780,7 @@ bool NVPTXTargetLowering::getTgtMemIntrinsic(
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = 0;
     Info.flags = MachineMemOperand::MOLoad;
-    Info.align =
-        MaybeAlign(cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
+    Info.align = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();
 
     return true;
   }
@@ -3800,8 +3799,7 @@ bool NVPTXTargetLowering::getTgtMemIntrinsic(
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = 0;
     Info.flags = MachineMemOperand::MOLoad;
-    Info.align =
-        MaybeAlign(cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
+    Info.align = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();
 
     return true;
   }
@@ -4791,11 +4789,10 @@ static void ReplaceLoadVector(SDNode *N, SelectionDAG &DAG,
 
   LoadSDNode *LD = cast<LoadSDNode>(N);
 
-  unsigned Align = LD->getAlignment();
+  Align Alignment = LD->getAlign();
   auto &TD = DAG.getDataLayout();
-  unsigned PrefAlign =
-      TD.getPrefTypeAlignment(ResVT.getTypeForEVT(*DAG.getContext()));
-  if (Align < PrefAlign) {
+  Align PrefAlign = TD.getPrefTypeAlign(ResVT.getTypeForEVT(*DAG.getContext()));
+  if (Alignment < PrefAlign) {
     // This load is not sufficiently aligned, so bail out and let this vector
     // load be scalarized.  Note that we may still be able to emit smaller
     // vector loads.  For example, if we are loading a <4 x float> with an
diff --git a/llvm/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp b/llvm/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp
index b4b5d2b396d00..ea2274f394e61 100644
--- a/llvm/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp
@@ -74,7 +74,7 @@ bool NVPTXPrologEpilogPass::runOnMachineFunction(MachineFunction &MF) {
           MI.getOperand(0).setIsDebug();
           auto *DIExpr = DIExpression::prepend(
               MI.getDebugExpression(), DIExpression::ApplyOffset, Offset);
-          MI.getOperand(3).setMetadata(DIExpr);
+          MI.getDebugExpressionOp().setMetadata(DIExpr);
           continue;
         }
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index 5c14d0f1a24d5..3873c73fb2e03 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -155,3 +155,8 @@ void NVPTXTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
   UP.Partial = UP.Runtime = true;
   UP.PartialThreshold = UP.Threshold / 4;
 }
+
+void NVPTXTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                                         TTI::PeelingPreferences &PP) {
+  BaseT::getPeelingPreferences(L, SE, PP);
+}
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
index e28a8220c33e7..cb832031f1add 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
@@ -50,13 +50,11 @@ class NVPTXTTIImpl : public BasicTTIImplBase<NVPTXTTIImpl> {
 
   // Loads and stores can be vectorized if the alignment is at least as big as
   // the load/store we want to vectorize.
-  bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
-                                   unsigned Alignment,
+  bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, Align Alignment,
                                    unsigned AddrSpace) const {
     return Alignment >= ChainSizeInBytes;
   }
-  bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
-                                    unsigned Alignment,
+  bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, Align Alignment,
                                     unsigned AddrSpace) const {
     return isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment, AddrSpace);
   }
@@ -97,6 +95,10 @@ class NVPTXTTIImpl : public BasicTTIImplBase<NVPTXTTIImpl> {
 
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP);
+
+  void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                             TTI::PeelingPreferences &PP);
+
   bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) {
     // Volatile loads/stores are only supported for shared and global address
     // spaces, or for generic AS that maps to them.
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
index d492fd848c7ea..3092d56da1c59 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
@@ -120,14 +120,17 @@ class PPCTargetAsmStreamer : public PPCTargetStreamer {
       : PPCTargetStreamer(S), OS(OS) {}
 
   void emitTCEntry(const MCSymbol &S) override {
-    const MCAsmInfo *MAI = Streamer.getContext().getAsmInfo();
-    OS << "\t.tc ";
-    OS << (MAI->getSymbolsHaveSMC()
-               ? cast<MCSymbolXCOFF>(S).getUnqualifiedName()
-               : S.getName());
-    OS << "[TC],";
-    OS << S.getName();
-    OS << '\n';
+    if (const MCSymbolXCOFF *XSym = dyn_cast<MCSymbolXCOFF>(&S)) {
+      MCSymbolXCOFF *TCSym =
+          cast<MCSymbolXCOFF>(Streamer.getContext().getOrCreateSymbol(
+              XSym->getSymbolTableName() + "[TC]"));
+      if (TCSym->hasRename())
+        Streamer.emitXCOFFRenameDirective(TCSym, TCSym->getSymbolTableName());
+      OS << "\t.tc " << TCSym->getName() << "," << XSym->getName() << '\n';
+      return;
+    }
+
+    OS << "\t.tc " << S.getName() << "[TC]," << S.getName() << '\n';
   }
 
   void emitMachine(StringRef CPU) override {
diff --git a/llvm/lib/Target/PowerPC/PPC.td b/llvm/lib/Target/PowerPC/PPC.td
index 95ab6edd9f3f6..9ad78bf67fe6c 100644
--- a/llvm/lib/Target/PowerPC/PPC.td
+++ b/llvm/lib/Target/PowerPC/PPC.td
@@ -205,10 +205,11 @@ def DeprecatedDST    : SubtargetFeature<"", "DeprecatedDST", "true",
 
 def FeatureISA3_0 : SubtargetFeature<"isa-v30-instructions", "IsISA3_0",
                                      "true",
-                                     "Enable instructions added in ISA 3.0.">;
+                                     "Enable instructions in ISA 3.0.">;
 def FeatureISA3_1 : SubtargetFeature<"isa-v31-instructions", "IsISA3_1",
                                      "true",
-                                     "Enable instructions added in ISA 3.1.">;
+                                     "Enable instructions in ISA 3.1.",
+                                     [FeatureISA3_0]>;
 def FeatureP9Altivec : SubtargetFeature<"power9-altivec", "HasP9Altivec", "true",
                                         "Enable POWER9 Altivec instructions",
                                         [FeatureISA3_0, FeatureP8Altivec]>;
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index 4b51cb94adb54..bf5fe741bac8a 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -631,7 +631,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
       return;
     } else {
       MCSymbol *PICOffset =
-        MF->getInfo<PPCFunctionInfo>()->getPICOffsetSymbol();
+        MF->getInfo<PPCFunctionInfo>()->getPICOffsetSymbol(*MF);
       TmpInst.setOpcode(PPC::LWZ);
       const MCExpr *Exp =
         MCSymbolRefExpr::create(PICOffset, MCSymbolRefExpr::VK_None, OutContext);
@@ -1143,8 +1143,11 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     // can be enabled for those subtargets.
     unsigned OpNum = (MI->getOpcode() == PPC::STD) ? 2 : 1;
     const MachineOperand &MO = MI->getOperand(OpNum);
-    if (MO.isGlobal() && MO.getGlobal()->getAlignment() < 4)
-      llvm_unreachable("Global must be word-aligned for LD, STD, LWA!");
+    if (MO.isGlobal()) {
+      const DataLayout &DL = MO.getGlobal()->getParent()->getDataLayout();
+      if (MO.getGlobal()->getPointerAlignment(DL) < 4)
+        llvm_unreachable("Global must be word-aligned for LD, STD, LWA!");
+    }
     // Now process the instruction normally.
     break;
   }
@@ -1338,7 +1341,7 @@ void PPCLinuxAsmPrinter::emitFunctionEntryLabel() {
   if (!Subtarget->isPPC64()) {
     const PPCFunctionInfo *PPCFI = MF->getInfo<PPCFunctionInfo>();
     if (PPCFI->usesPICBase() && !Subtarget->isSecurePlt()) {
-      MCSymbol *RelocSymbol = PPCFI->getPICOffsetSymbol();
+      MCSymbol *RelocSymbol = PPCFI->getPICOffsetSymbol(*MF);
       MCSymbol *PICBase = MF->getPICBaseSymbol();
       OutStreamer->emitLabel(RelocSymbol);
 
@@ -1366,14 +1369,14 @@ void PPCLinuxAsmPrinter::emitFunctionEntryLabel() {
       const PPCFunctionInfo *PPCFI = MF->getInfo<PPCFunctionInfo>();
 
       MCSymbol *TOCSymbol = OutContext.getOrCreateSymbol(StringRef(".TOC."));
-      MCSymbol *GlobalEPSymbol = PPCFI->getGlobalEPSymbol();
+      MCSymbol *GlobalEPSymbol = PPCFI->getGlobalEPSymbol(*MF);
       const MCExpr *TOCDeltaExpr =
         MCBinaryExpr::createSub(MCSymbolRefExpr::create(TOCSymbol, OutContext),
                                 MCSymbolRefExpr::create(GlobalEPSymbol,
                                                         OutContext),
                                 OutContext);
 
-      OutStreamer->emitLabel(PPCFI->getTOCOffsetSymbol());
+      OutStreamer->emitLabel(PPCFI->getTOCOffsetSymbol(*MF));
       OutStreamer->emitValue(TOCDeltaExpr, 8);
     }
     return AsmPrinter::emitFunctionEntryLabel();
@@ -1480,7 +1483,7 @@ void PPCLinuxAsmPrinter::emitFunctionBodyStart() {
     // Note: The logic here must be synchronized with the code in the
     // branch-selection pass which sets the offset of the first block in the
     // function. This matters because it affects the alignment.
-    MCSymbol *GlobalEntryLabel = PPCFI->getGlobalEPSymbol();
+    MCSymbol *GlobalEntryLabel = PPCFI->getGlobalEPSymbol(*MF);
     OutStreamer->emitLabel(GlobalEntryLabel);
     const MCSymbolRefExpr *GlobalEntryLabelExp =
       MCSymbolRefExpr::create(GlobalEntryLabel, OutContext);
@@ -1503,7 +1506,7 @@ void PPCLinuxAsmPrinter::emitFunctionBodyStart() {
                                    .addReg(PPC::X2)
                                    .addExpr(TOCDeltaLo));
     } else {
-      MCSymbol *TOCOffset = PPCFI->getTOCOffsetSymbol();
+      MCSymbol *TOCOffset = PPCFI->getTOCOffsetSymbol(*MF);
       const MCExpr *TOCOffsetDeltaExpr =
         MCBinaryExpr::createSub(MCSymbolRefExpr::create(TOCOffset, OutContext),
                                 GlobalEntryLabelExp, OutContext);
@@ -1518,7 +1521,7 @@ void PPCLinuxAsmPrinter::emitFunctionBodyStart() {
                                    .addReg(PPC::X12));
     }
 
-    MCSymbol *LocalEntryLabel = PPCFI->getLocalEPSymbol();
+    MCSymbol *LocalEntryLabel = PPCFI->getLocalEPSymbol(*MF);
     OutStreamer->emitLabel(LocalEntryLabel);
     const MCSymbolRefExpr *LocalEntryLabelExp =
        MCSymbolRefExpr::create(LocalEntryLabel, OutContext);
@@ -1605,8 +1608,10 @@ void PPCAIXAsmPrinter::emitLinkage(const GlobalValue *GV,
   case GlobalValue::PrivateLinkage:
     return;
   case GlobalValue::InternalLinkage:
-    OutStreamer->emitSymbolAttribute(GVSym, MCSA_LGlobal);
-    return;
+    assert(GV->getVisibility() == GlobalValue::DefaultVisibility &&
+           "InternalLinkage should not have other visibility setting.");
+    LinkageAttr = MCSA_LGlobal;
+    break;
   case GlobalValue::AppendingLinkage:
     llvm_unreachable("Should never emit this");
   case GlobalValue::CommonLinkage:
@@ -1618,8 +1623,7 @@ void PPCAIXAsmPrinter::emitLinkage(const GlobalValue *GV,
   MCSymbolAttr VisibilityAttr = MCSA_Invalid;
   switch (GV->getVisibility()) {
 
-    // TODO: "exported" and "internal" Visibility needs to go here.
-
+  // TODO: "exported" and "internal" Visibility needs to go here.
   case GlobalValue::DefaultVisibility:
     break;
   case GlobalValue::HiddenVisibility:
@@ -1699,16 +1703,15 @@ void PPCAIXAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
 
   // Handle common symbols.
   if (GVKind.isCommon() || GVKind.isBSSLocal()) {
-    unsigned Align =
-      GV->getAlignment() ? GV->getAlignment() : DL.getPreferredAlignment(GV);
+    Align Alignment = GV->getAlign().getValueOr(DL.getPreferredAlign(GV));
     uint64_t Size = DL.getTypeAllocSize(GV->getType()->getElementType());
 
     if (GVKind.isBSSLocal())
       OutStreamer->emitXCOFFLocalCommonSymbol(
           OutContext.getOrCreateSymbol(GVSym->getUnqualifiedName()), Size,
-          GVSym, Align);
+          GVSym, Alignment.value());
     else
-      OutStreamer->emitCommonSymbol(GVSym, Size, Align);
+      OutStreamer->emitCommonSymbol(GVSym, Size, Alignment.value());
     return;
   }
 
diff --git a/llvm/lib/Target/PowerPC/PPCBranchCoalescing.cpp b/llvm/lib/Target/PowerPC/PPCBranchCoalescing.cpp
index 109b665e0d57c..50ae4450a8375 100644
--- a/llvm/lib/Target/PowerPC/PPCBranchCoalescing.cpp
+++ b/llvm/lib/Target/PowerPC/PPCBranchCoalescing.cpp
@@ -272,6 +272,11 @@ bool PPCBranchCoalescing::canCoalesceBranch(CoalescingCandidateInfo &Cand) {
     return false;
   }
 
+  if (Cand.BranchBlock->mayHaveInlineAsmBr()) {
+    LLVM_DEBUG(dbgs() << "Inline Asm Br - skip\n");
+    return false;
+  }
+
   // For now only consider triangles (i.e, BranchTargetBlock is set,
   // FalseMBB is null, and BranchTargetBlock is a successor to BranchBlock)
   if (!Cand.BranchTargetBlock || FalseMBB ||
diff --git a/llvm/lib/Target/PowerPC/PPCFastISel.cpp b/llvm/lib/Target/PowerPC/PPCFastISel.cpp
index 6608fd359d9a4..39790ac9a8aab 100644
--- a/llvm/lib/Target/PowerPC/PPCFastISel.cpp
+++ b/llvm/lib/Target/PowerPC/PPCFastISel.cpp
@@ -87,6 +87,7 @@ class PPCFastISel final : public FastISel {
 
   const TargetMachine &TM;
   const PPCSubtarget *PPCSubTarget;
+  const PPCSubtarget *Subtarget;
   PPCFunctionInfo *PPCFuncInfo;
   const TargetInstrInfo &TII;
   const TargetLowering &TLI;
@@ -97,12 +98,12 @@ class PPCFastISel final : public FastISel {
                          const TargetLibraryInfo *LibInfo)
         : FastISel(FuncInfo, LibInfo), TM(FuncInfo.MF->getTarget()),
           PPCSubTarget(&FuncInfo.MF->getSubtarget<PPCSubtarget>()),
+          Subtarget(&FuncInfo.MF->getSubtarget<PPCSubtarget>()),
           PPCFuncInfo(FuncInfo.MF->getInfo<PPCFunctionInfo>()),
-          TII(*PPCSubTarget->getInstrInfo()),
-          TLI(*PPCSubTarget->getTargetLowering()),
+          TII(*Subtarget->getInstrInfo()), TLI(*Subtarget->getTargetLowering()),
           Context(&FuncInfo.Fn->getContext()) {}
 
-  // Backend specific FastISel code.
+    // Backend specific FastISel code.
   private:
     bool fastSelectInstruction(const Instruction *I) override;
     unsigned fastMaterializeConstant(const Constant *C) override;
@@ -456,7 +457,7 @@ bool PPCFastISel::PPCEmitLoad(MVT VT, Register &ResultReg, Address &Addr,
                               bool IsZExt, unsigned FP64LoadOpc) {
   unsigned Opc;
   bool UseOffset = true;
-  bool HasSPE = PPCSubTarget->hasSPE();
+  bool HasSPE = Subtarget->hasSPE();
 
   // If ResultReg is given, it determines the register class of the load.
   // Otherwise, RC is the register class to use.  If the result of the
@@ -498,7 +499,7 @@ bool PPCFastISel::PPCEmitLoad(MVT VT, Register &ResultReg, Address &Addr,
       UseOffset = ((Addr.Offset & 3) == 0);
       break;
     case MVT::f32:
-      Opc = PPCSubTarget->hasSPE() ? PPC::SPELWZ : PPC::LFS;
+      Opc = Subtarget->hasSPE() ? PPC::SPELWZ : PPC::LFS;
       break;
     case MVT::f64:
       Opc = FP64LoadOpc;
@@ -614,7 +615,7 @@ bool PPCFastISel::SelectLoad(const Instruction *I) {
 
   Register ResultReg = 0;
   if (!PPCEmitLoad(VT, ResultReg, Addr, RC, true,
-      PPCSubTarget->hasSPE() ? PPC::EVLDD : PPC::LFD))
+                   Subtarget->hasSPE() ? PPC::EVLDD : PPC::LFD))
     return false;
   updateValueMap(I, ResultReg);
   return true;
@@ -647,10 +648,10 @@ bool PPCFastISel::PPCEmitStore(MVT VT, unsigned SrcReg, Address &Addr) {
       UseOffset = ((Addr.Offset & 3) == 0);
       break;
     case MVT::f32:
-      Opc = PPCSubTarget->hasSPE() ? PPC::SPESTW : PPC::STFS;
+      Opc = Subtarget->hasSPE() ? PPC::SPESTW : PPC::STFS;
       break;
     case MVT::f64:
-      Opc = PPCSubTarget->hasSPE() ? PPC::EVSTDD : PPC::STFD;
+      Opc = Subtarget->hasSPE() ? PPC::EVSTDD : PPC::STFD;
       break;
   }
 
@@ -794,8 +795,9 @@ bool PPCFastISel::SelectBranch(const Instruction *I) {
         return false;
 
       BuildMI(*BrBB, FuncInfo.InsertPt, DbgLoc, TII.get(PPC::BCC))
-        .addImm(PPCSubTarget->hasSPE() ? PPC::PRED_SPE : PPCPred)
-        .addReg(CondReg).addMBB(TBB);
+          .addImm(Subtarget->hasSPE() ? PPC::PRED_SPE : PPCPred)
+          .addReg(CondReg)
+          .addMBB(TBB);
       finishCondBranch(BI->getParent(), TBB, FBB);
       return true;
     }
@@ -827,7 +829,7 @@ bool PPCFastISel::PPCEmitCmp(const Value *SrcValue1, const Value *SrcValue2,
     return false;
   MVT SrcVT = SrcEVT.getSimpleVT();
 
-  if (SrcVT == MVT::i1 && PPCSubTarget->useCRBits())
+  if (SrcVT == MVT::i1 && Subtarget->useCRBits())
     return false;
 
   // See if operand 2 is an immediate encodeable in the compare.
@@ -836,7 +838,7 @@ bool PPCFastISel::PPCEmitCmp(const Value *SrcValue1, const Value *SrcValue2,
   // similar to ARM in this regard.
   long Imm = 0;
   bool UseImm = false;
-  const bool HasSPE = PPCSubTarget->hasSPE();
+  const bool HasSPE = Subtarget->hasSPE();
 
   // Only 16-bit integer constants can be represented in compares for
   // PowerPC.  Others will be materialized into a register.
@@ -988,7 +990,7 @@ bool PPCFastISel::SelectFPTrunc(const Instruction *I) {
   // Round the result to single precision.
   unsigned DestReg;
   auto RC = MRI.getRegClass(SrcReg);
-  if (PPCSubTarget->hasSPE()) {
+  if (Subtarget->hasSPE()) {
     DestReg = createResultReg(&PPC::GPRCRegClass);
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
       TII.get(PPC::EFSCFD), DestReg)
@@ -1030,7 +1032,7 @@ unsigned PPCFastISel::PPCMoveToFPReg(MVT SrcVT, unsigned SrcReg,
   // Get a stack slot 8 bytes wide, aligned on an 8-byte boundary.
   Address Addr;
   Addr.BaseType = Address::FrameIndexBase;
-  Addr.Base.FI = MFI.CreateStackObject(8, 8, false);
+  Addr.Base.FI = MFI.CreateStackObject(8, Align(8), false);
 
   // Store the value from the GPR.
   if (!PPCEmitStore(MVT::i64, SrcReg, Addr))
@@ -1043,10 +1045,10 @@ unsigned PPCFastISel::PPCMoveToFPReg(MVT SrcVT, unsigned SrcReg,
   if (SrcVT == MVT::i32) {
     if (!IsSigned) {
       LoadOpc = PPC::LFIWZX;
-      Addr.Offset = (PPCSubTarget->isLittleEndian()) ? 0 : 4;
-    } else if (PPCSubTarget->hasLFIWAX()) {
+      Addr.Offset = (Subtarget->isLittleEndian()) ? 0 : 4;
+    } else if (Subtarget->hasLFIWAX()) {
       LoadOpc = PPC::LFIWAX;
-      Addr.Offset = (PPCSubTarget->isLittleEndian()) ? 0 : 4;
+      Addr.Offset = (Subtarget->isLittleEndian()) ? 0 : 4;
     }
   }
 
@@ -1086,7 +1088,7 @@ bool PPCFastISel::SelectIToFP(const Instruction *I, bool IsSigned) {
     return false;
 
   // Shortcut for SPE.  Doesn't need to store/load, since it's all in the GPRs
-  if (PPCSubTarget->hasSPE()) {
+  if (Subtarget->hasSPE()) {
     unsigned Opc;
     if (DstVT == MVT::f32)
       Opc = IsSigned ? PPC::EFSCFSI : PPC::EFSCFUI;
@@ -1103,7 +1105,7 @@ bool PPCFastISel::SelectIToFP(const Instruction *I, bool IsSigned) {
 
   // We can only lower an unsigned convert if we have the newer
   // floating-point conversion operations.
-  if (!IsSigned && !PPCSubTarget->hasFPCVT())
+  if (!IsSigned && !Subtarget->hasFPCVT())
     return false;
 
   // FIXME: For now we require the newer floating-point conversion operations
@@ -1111,7 +1113,7 @@ bool PPCFastISel::SelectIToFP(const Instruction *I, bool IsSigned) {
   // to single-precision float.  Otherwise we have to generate a lot of
   // fiddly code to avoid double rounding.  If necessary, the fiddly code
   // can be found in PPCTargetLowering::LowerINT_TO_FP().
-  if (DstVT == MVT::f32 && !PPCSubTarget->hasFPCVT())
+  if (DstVT == MVT::f32 && !Subtarget->hasFPCVT())
     return false;
 
   // Extend the input if necessary.
@@ -1159,7 +1161,7 @@ unsigned PPCFastISel::PPCMoveToIntReg(const Instruction *I, MVT VT,
   // easiest code gen possible.
   Address Addr;
   Addr.BaseType = Address::FrameIndexBase;
-  Addr.Base.FI = MFI.CreateStackObject(8, 8, false);
+  Addr.Base.FI = MFI.CreateStackObject(8, Align(8), false);
 
   // Store the value from the FPR.
   if (!PPCEmitStore(MVT::f64, SrcReg, Addr))
@@ -1168,7 +1170,7 @@ unsigned PPCFastISel::PPCMoveToIntReg(const Instruction *I, MVT VT,
   // Reload it into a GPR.  If we want an i32 on big endian, modify the
   // address to have a 4-byte offset so we load from the right place.
   if (VT == MVT::i32)
-    Addr.Offset = (PPCSubTarget->isLittleEndian()) ? 0 : 4;
+    Addr.Offset = (Subtarget->isLittleEndian()) ? 0 : 4;
 
   // Look at the currently assigned register for this instruction
   // to determine the required register class.
@@ -1196,8 +1198,8 @@ bool PPCFastISel::SelectFPToI(const Instruction *I, bool IsSigned) {
     return false;
 
   // If we don't have FCTIDUZ, or SPE, and we need it, punt to SelectionDAG.
-  if (DstVT == MVT::i64 && !IsSigned &&
-      !PPCSubTarget->hasFPCVT() && !PPCSubTarget->hasSPE())
+  if (DstVT == MVT::i64 && !IsSigned && !Subtarget->hasFPCVT() &&
+      !Subtarget->hasSPE())
     return false;
 
   Value *Src = I->getOperand(0);
@@ -1226,7 +1228,7 @@ bool PPCFastISel::SelectFPToI(const Instruction *I, bool IsSigned) {
   unsigned Opc;
   auto RC = MRI.getRegClass(SrcReg);
 
-  if (PPCSubTarget->hasSPE()) {
+  if (Subtarget->hasSPE()) {
     DestReg = createResultReg(&PPC::GPRCRegClass);
     if (IsSigned)
       Opc = InRC == &PPC::GPRCRegClass ? PPC::EFSCTSIZ : PPC::EFDCTSIZ;
@@ -1234,7 +1236,7 @@ bool PPCFastISel::SelectFPToI(const Instruction *I, bool IsSigned) {
       Opc = InRC == &PPC::GPRCRegClass ? PPC::EFSCTUIZ : PPC::EFDCTUIZ;
   } else if (isVSFRCRegClass(RC)) {
     DestReg = createResultReg(&PPC::VSFRCRegClass);
-    if (DstVT == MVT::i32) 
+    if (DstVT == MVT::i32)
       Opc = IsSigned ? PPC::XSCVDPSXWS : PPC::XSCVDPUXWS;
     else
       Opc = IsSigned ? PPC::XSCVDPSXDS : PPC::XSCVDPUXDS;
@@ -1244,7 +1246,7 @@ bool PPCFastISel::SelectFPToI(const Instruction *I, bool IsSigned) {
       if (IsSigned)
         Opc = PPC::FCTIWZ;
       else
-        Opc = PPCSubTarget->hasFPCVT() ? PPC::FCTIWUZ : PPC::FCTIDZ;
+        Opc = Subtarget->hasFPCVT() ? PPC::FCTIWUZ : PPC::FCTIDZ;
     else
       Opc = IsSigned ? PPC::FCTIDZ : PPC::FCTIDUZ;
   }
@@ -1254,8 +1256,9 @@ bool PPCFastISel::SelectFPToI(const Instruction *I, bool IsSigned) {
     .addReg(SrcReg);
 
   // Now move the integer value from a float register to an integer register.
-  unsigned IntReg = PPCSubTarget->hasSPE() ? DestReg :
-    PPCMoveToIntReg(I, DstVT, DestReg, IsSigned);
+  unsigned IntReg = Subtarget->hasSPE()
+                        ? DestReg
+                        : PPCMoveToIntReg(I, DstVT, DestReg, IsSigned);
 
   if (IntReg == 0)
     return false;
@@ -1383,7 +1386,7 @@ bool PPCFastISel::processCallArgs(SmallVectorImpl<Value*> &Args,
   CCState CCInfo(CC, IsVarArg, *FuncInfo.MF, ArgLocs, *Context);
 
   // Reserve space for the linkage area on the stack.
-  unsigned LinkageSize = PPCSubTarget->getFrameLowering()->getLinkageSize();
+  unsigned LinkageSize = Subtarget->getFrameLowering()->getLinkageSize();
   CCInfo.AllocateStack(LinkageSize, Align(8));
 
   CCInfo.AnalyzeCallOperands(ArgVTs, ArgFlags, CC_PPC64_ELF_FIS);
@@ -1573,7 +1576,7 @@ bool PPCFastISel::fastLowerCall(CallLoweringInfo &CLI) {
   else if (!isTypeLegal(RetTy, RetVT) && RetVT != MVT::i16 &&
            RetVT != MVT::i8)
     return false;
-  else if (RetVT == MVT::i1 && PPCSubTarget->useCRBits())
+  else if (RetVT == MVT::i1 && Subtarget->useCRBits())
     // We can't handle boolean returns when CR bits are in use.
     return false;
 
@@ -1995,7 +1998,7 @@ unsigned PPCFastISel::PPCMaterializeFP(const ConstantFP *CFP, MVT VT) {
   // All FP constants are loaded from the constant pool.
   Align Alignment = DL.getPrefTypeAlign(CFP->getType());
   unsigned Idx = MCP.getConstantPoolIndex(cast<Constant>(CFP), Alignment);
-  const bool HasSPE = PPCSubTarget->hasSPE();
+  const bool HasSPE = Subtarget->hasSPE();
   const TargetRegisterClass *RC;
   if (HasSPE)
     RC = ((VT == MVT::f32) ? &PPC::GPRCRegClass : &PPC::SPERCRegClass);
@@ -2089,7 +2092,7 @@ unsigned PPCFastISel::PPCMaterializeGV(const GlobalValue *GV, MVT VT) {
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(PPC::ADDIStocHA8),
             HighPartReg).addReg(PPC::X2).addGlobalAddress(GV);
 
-    if (PPCSubTarget->isGVIndirectSymbol(GV)) {
+    if (Subtarget->isGVIndirectSymbol(GV)) {
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(PPC::LDtocL),
               DestReg).addGlobalAddress(GV).addReg(HighPartReg);
     } else {
@@ -2196,7 +2199,7 @@ unsigned PPCFastISel::PPCMaterializeInt(const ConstantInt *CI, MVT VT,
                                         bool UseSExt) {
   // If we're using CR bit registers for i1 values, handle that as a special
   // case first.
-  if (VT == MVT::i1 && PPCSubTarget->useCRBits()) {
+  if (VT == MVT::i1 && Subtarget->useCRBits()) {
     unsigned ImmReg = createResultReg(&PPC::CRBITRCRegClass);
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
             TII.get(CI->isZero() ? PPC::CRUNSET : PPC::CRSET), ImmReg);
@@ -2351,7 +2354,7 @@ bool PPCFastISel::tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
   Register ResultReg = MI->getOperand(0).getReg();
 
   if (!PPCEmitLoad(VT, ResultReg, Addr, nullptr, IsZExt,
-        PPCSubTarget->hasSPE() ? PPC::EVLDD : PPC::LFD))
+                   Subtarget->hasSPE() ? PPC::EVLDD : PPC::LFD))
     return false;
 
   MachineBasicBlock::iterator I(MI);
@@ -2378,7 +2381,7 @@ unsigned PPCFastISel::fastEmit_i(MVT Ty, MVT VT, unsigned Opc, uint64_t Imm) {
 
   // If we're using CR bit registers for i1 values, handle that as a special
   // case first.
-  if (VT == MVT::i1 && PPCSubTarget->useCRBits()) {
+  if (VT == MVT::i1 && Subtarget->useCRBits()) {
     unsigned ImmReg = createResultReg(&PPC::CRBITRCRegClass);
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
             TII.get(Imm == 0 ? PPC::CRUNSET : PPC::CRSET), ImmReg);
diff --git a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
index 3ec6788d077b5..93ea3154e47f2 100644
--- a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "MCTargetDesc/PPCPredicates.h"
 #include "PPCFrameLowering.h"
 #include "PPCInstrBuilder.h"
 #include "PPCInstrInfo.h"
@@ -31,6 +32,7 @@ using namespace llvm;
 #define DEBUG_TYPE "framelowering"
 STATISTIC(NumPESpillVSR, "Number of spills to vector in prologue");
 STATISTIC(NumPEReloadVSR, "Number of reloads from vector in epilogue");
+STATISTIC(NumPrologProbed, "Number of prologues probed");
 
 static cl::opt<bool>
 EnablePEVectorSpills("ppc-enable-pe-vector-spills",
@@ -73,10 +75,12 @@ static unsigned computeLinkageSize(const PPCSubtarget &STI) {
 }
 
 static unsigned computeBasePointerSaveOffset(const PPCSubtarget &STI) {
-  // SVR4 ABI: First slot in the general register save area.
-  return STI.isPPC64()
-             ? -16U
-             : STI.getTargetMachine().isPositionIndependent() ? -12U : -8U;
+  // Third slot in the general purpose register save area.
+  if (STI.is32BitELFABI() && STI.getTargetMachine().isPositionIndependent())
+    return -12U;
+
+  // Second slot in the general purpose register save area.
+  return STI.isPPC64() ? -16U : -8U;
 }
 
 static unsigned computeCRSaveOffset(const PPCSubtarget &STI) {
@@ -755,6 +759,7 @@ void PPCFrameLowering::emitPrologue(MachineFunction &MF,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   const PPCInstrInfo &TII = *Subtarget.getInstrInfo();
   const PPCRegisterInfo *RegInfo = Subtarget.getRegisterInfo();
+  const PPCTargetLowering &TLI = *Subtarget.getTargetLowering();
 
   MachineModuleInfo &MMI = MF.getMMI();
   const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
@@ -843,9 +848,12 @@ void PPCFrameLowering::emitPrologue(MachineFunction &MF,
          "FrameSize must be >0 to save/restore the FP or LR for 32-bit SVR4.");
 
   // Using the same bool variable as below to suppress compiler warnings.
-  bool SingleScratchReg =
-    findScratchRegister(&MBB, false, twoUniqueScratchRegsRequired(&MBB),
-                        &ScratchReg, &TempReg);
+  // Stack probe requires two scratch registers, one for old sp, one for large
+  // frame and large probe size.
+  bool SingleScratchReg = findScratchRegister(
+      &MBB, false,
+      twoUniqueScratchRegsRequired(&MBB) || TLI.hasInlineStackProbe(MF),
+      &ScratchReg, &TempReg);
   assert(SingleScratchReg &&
          "Required number of registers not available in this block");
 
@@ -1021,58 +1029,81 @@ void PPCFrameLowering::emitPrologue(MachineFunction &MF,
   // the negated frame size will be placed in ScratchReg.
   bool HasSTUX = false;
 
-  // This condition must be kept in sync with canUseAsPrologue.
-  if (HasBP && MaxAlign > 1) {
-    if (isPPC64)
-      BuildMI(MBB, MBBI, dl, TII.get(PPC::RLDICL), ScratchReg)
-          .addReg(SPReg)
-          .addImm(0)
-          .addImm(64 - Log2(MaxAlign));
-    else // PPC32...
-      BuildMI(MBB, MBBI, dl, TII.get(PPC::RLWINM), ScratchReg)
-          .addReg(SPReg)
-          .addImm(0)
-          .addImm(32 - Log2(MaxAlign))
-          .addImm(31);
-    if (!isLargeFrame) {
-      BuildMI(MBB, MBBI, dl, SubtractImmCarryingInst, ScratchReg)
-        .addReg(ScratchReg, RegState::Kill)
+  // If FrameSize <= TLI.getStackProbeSize(MF), as POWER ABI requires backchain
+  // pointer is always stored at SP, we will get a free probe due to an essential
+  // STU(X) instruction.
+  if (TLI.hasInlineStackProbe(MF) && FrameSize > TLI.getStackProbeSize(MF)) {
+    // To be consistent with other targets, a pseudo instruction is emitted and
+    // will be later expanded in `inlineStackProbe`.
+    BuildMI(MBB, MBBI, dl,
+            TII.get(isPPC64 ? PPC::PROBED_STACKALLOC_64
+                            : PPC::PROBED_STACKALLOC_32))
+        .addDef(ScratchReg)
+        .addDef(TempReg) // TempReg stores the old sp.
         .addImm(NegFrameSize);
-    } else {
-      assert(!SingleScratchReg && "Only a single scratch reg available");
-      BuildMI(MBB, MBBI, dl, LoadImmShiftedInst, TempReg)
-        .addImm(NegFrameSize >> 16);
-      BuildMI(MBB, MBBI, dl, OrImmInst, TempReg)
-        .addReg(TempReg, RegState::Kill)
-        .addImm(NegFrameSize & 0xFFFF);
-      BuildMI(MBB, MBBI, dl, SubtractCarryingInst, ScratchReg)
-        .addReg(ScratchReg, RegState::Kill)
-        .addReg(TempReg, RegState::Kill);
+    // FIXME: HasSTUX is only read if HasRedZone is not set, in such case, we
+    // update the ScratchReg to meet the assumption that ScratchReg contains
+    // the NegFrameSize. This solution is rather tricky.
+    if (!HasRedZone) {
+      BuildMI(MBB, MBBI, dl, TII.get(PPC::SUBF), ScratchReg)
+          .addReg(TempReg)
+          .addReg(SPReg);
+      HasSTUX = true;
     }
+  } else {
+    // This condition must be kept in sync with canUseAsPrologue.
+    if (HasBP && MaxAlign > 1) {
+      if (isPPC64)
+        BuildMI(MBB, MBBI, dl, TII.get(PPC::RLDICL), ScratchReg)
+            .addReg(SPReg)
+            .addImm(0)
+            .addImm(64 - Log2(MaxAlign));
+      else // PPC32...
+        BuildMI(MBB, MBBI, dl, TII.get(PPC::RLWINM), ScratchReg)
+            .addReg(SPReg)
+            .addImm(0)
+            .addImm(32 - Log2(MaxAlign))
+            .addImm(31);
+      if (!isLargeFrame) {
+        BuildMI(MBB, MBBI, dl, SubtractImmCarryingInst, ScratchReg)
+            .addReg(ScratchReg, RegState::Kill)
+            .addImm(NegFrameSize);
+      } else {
+        assert(!SingleScratchReg && "Only a single scratch reg available");
+        BuildMI(MBB, MBBI, dl, LoadImmShiftedInst, TempReg)
+            .addImm(NegFrameSize >> 16);
+        BuildMI(MBB, MBBI, dl, OrImmInst, TempReg)
+            .addReg(TempReg, RegState::Kill)
+            .addImm(NegFrameSize & 0xFFFF);
+        BuildMI(MBB, MBBI, dl, SubtractCarryingInst, ScratchReg)
+            .addReg(ScratchReg, RegState::Kill)
+            .addReg(TempReg, RegState::Kill);
+      }
 
-    BuildMI(MBB, MBBI, dl, StoreUpdtIdxInst, SPReg)
-      .addReg(SPReg, RegState::Kill)
-      .addReg(SPReg)
-      .addReg(ScratchReg);
-    HasSTUX = true;
+      BuildMI(MBB, MBBI, dl, StoreUpdtIdxInst, SPReg)
+          .addReg(SPReg, RegState::Kill)
+          .addReg(SPReg)
+          .addReg(ScratchReg);
+      HasSTUX = true;
 
-  } else if (!isLargeFrame) {
-    BuildMI(MBB, StackUpdateLoc, dl, StoreUpdtInst, SPReg)
-      .addReg(SPReg)
-      .addImm(NegFrameSize)
-      .addReg(SPReg);
+    } else if (!isLargeFrame) {
+      BuildMI(MBB, StackUpdateLoc, dl, StoreUpdtInst, SPReg)
+          .addReg(SPReg)
+          .addImm(NegFrameSize)
+          .addReg(SPReg);
 
-  } else {
-    BuildMI(MBB, MBBI, dl, LoadImmShiftedInst, ScratchReg)
-      .addImm(NegFrameSize >> 16);
-    BuildMI(MBB, MBBI, dl, OrImmInst, ScratchReg)
-      .addReg(ScratchReg, RegState::Kill)
-      .addImm(NegFrameSize & 0xFFFF);
-    BuildMI(MBB, MBBI, dl, StoreUpdtIdxInst, SPReg)
-      .addReg(SPReg, RegState::Kill)
-      .addReg(SPReg)
-      .addReg(ScratchReg);
-    HasSTUX = true;
+    } else {
+      BuildMI(MBB, MBBI, dl, LoadImmShiftedInst, ScratchReg)
+          .addImm(NegFrameSize >> 16);
+      BuildMI(MBB, MBBI, dl, OrImmInst, ScratchReg)
+          .addReg(ScratchReg, RegState::Kill)
+          .addImm(NegFrameSize & 0xFFFF);
+      BuildMI(MBB, MBBI, dl, StoreUpdtIdxInst, SPReg)
+          .addReg(SPReg, RegState::Kill)
+          .addReg(SPReg)
+          .addReg(ScratchReg);
+      HasSTUX = true;
+    }
   }
 
   // Save the TOC register after the stack pointer update if a prologue TOC
@@ -1333,6 +1364,142 @@ void PPCFrameLowering::emitPrologue(MachineFunction &MF,
   }
 }
 
+void PPCFrameLowering::inlineStackProbe(MachineFunction &MF,
+                                        MachineBasicBlock &PrologMBB) const {
+  // TODO: Generate CFI instructions.
+  bool isPPC64 = Subtarget.isPPC64();
+  const PPCTargetLowering &TLI = *Subtarget.getTargetLowering();
+  const PPCInstrInfo &TII = *Subtarget.getInstrInfo();
+  MachineFrameInfo &MFI = MF.getFrameInfo();
+  auto StackAllocMIPos = llvm::find_if(PrologMBB, [](MachineInstr &MI) {
+    int Opc = MI.getOpcode();
+    return Opc == PPC::PROBED_STACKALLOC_64 || Opc == PPC::PROBED_STACKALLOC_32;
+  });
+  if (StackAllocMIPos == PrologMBB.end())
+    return;
+  const BasicBlock *ProbedBB = PrologMBB.getBasicBlock();
+  DebugLoc DL = PrologMBB.findDebugLoc(StackAllocMIPos);
+  MachineInstr &MI = *StackAllocMIPos;
+  int64_t NegFrameSize = MI.getOperand(2).getImm();
+  int64_t NegProbeSize = -(int64_t)TLI.getStackProbeSize(MF);
+  assert(isInt<32>(NegProbeSize) && "Unhandled probe size");
+  int64_t NumBlocks = NegFrameSize / NegProbeSize;
+  int64_t NegResidualSize = NegFrameSize % NegProbeSize;
+  Register SPReg = isPPC64 ? PPC::X1 : PPC::R1;
+  Register ScratchReg = MI.getOperand(0).getReg();
+  Register FPReg = MI.getOperand(1).getReg();
+  const PPCRegisterInfo *RegInfo = Subtarget.getRegisterInfo();
+  bool HasBP = RegInfo->hasBasePointer(MF);
+  Align MaxAlign = MFI.getMaxAlign();
+  // Initialize current frame pointer.
+  const MCInstrDesc &CopyInst = TII.get(isPPC64 ? PPC::OR8 : PPC::OR);
+  BuildMI(PrologMBB, {MI}, DL, CopyInst, FPReg).addReg(SPReg).addReg(SPReg);
+  // Subroutine to determine if we can use the Imm as part of d-form.
+  auto CanUseDForm = [](int64_t Imm) { return isInt<16>(Imm) && Imm % 4 == 0; };
+  // Subroutine to materialize the Imm into TempReg.
+  auto MaterializeImm = [&](MachineBasicBlock &MBB,
+                            MachineBasicBlock::iterator MBBI, int64_t Imm,
+                            Register &TempReg) {
+    assert(isInt<32>(Imm) && "Unhandled imm");
+    if (isInt<16>(Imm))
+      BuildMI(MBB, MBBI, DL, TII.get(isPPC64 ? PPC::LI8 : PPC::LI), TempReg)
+          .addImm(Imm);
+    else {
+      BuildMI(MBB, MBBI, DL, TII.get(isPPC64 ? PPC::LIS8 : PPC::LIS), TempReg)
+          .addImm(Imm >> 16);
+      BuildMI(MBB, MBBI, DL, TII.get(isPPC64 ? PPC::ORI8 : PPC::ORI), TempReg)
+          .addReg(TempReg)
+          .addImm(Imm & 0xFFFF);
+    }
+  };
+  // Subroutine to store frame pointer and decrease stack pointer by probe size.
+  auto allocateAndProbe = [&](MachineBasicBlock &MBB,
+                              MachineBasicBlock::iterator MBBI, int64_t NegSize,
+                              Register NegSizeReg, bool UseDForm) {
+    if (UseDForm)
+      BuildMI(MBB, MBBI, DL, TII.get(isPPC64 ? PPC::STDU : PPC::STWU), SPReg)
+          .addReg(FPReg)
+          .addImm(NegSize)
+          .addReg(SPReg);
+    else
+      BuildMI(MBB, MBBI, DL, TII.get(isPPC64 ? PPC::STDUX : PPC::STWUX), SPReg)
+          .addReg(FPReg)
+          .addReg(SPReg)
+          .addReg(NegSizeReg);
+  };
+  // For case HasBP && MaxAlign > 1, we have to align the SP by performing
+  // SP = SP - SP % MaxAlign.
+  if (HasBP && MaxAlign > 1) {
+    if (isPPC64)
+      BuildMI(PrologMBB, {MI}, DL, TII.get(PPC::RLDICL), ScratchReg)
+          .addReg(FPReg)
+          .addImm(0)
+          .addImm(64 - Log2(MaxAlign));
+    else
+      BuildMI(PrologMBB, {MI}, DL, TII.get(PPC::RLWINM), ScratchReg)
+          .addReg(FPReg)
+          .addImm(0)
+          .addImm(32 - Log2(MaxAlign))
+          .addImm(31);
+    BuildMI(PrologMBB, {MI}, DL, TII.get(isPPC64 ? PPC::STDUX : PPC::STWUX),
+            SPReg)
+        .addReg(FPReg)
+        .addReg(SPReg)
+        .addReg(ScratchReg);
+  }
+  // Probe residual part.
+  if (NegResidualSize) {
+    bool ResidualUseDForm = CanUseDForm(NegResidualSize);
+    if (!ResidualUseDForm)
+      MaterializeImm(PrologMBB, {MI}, NegResidualSize, ScratchReg);
+    allocateAndProbe(PrologMBB, {MI}, NegResidualSize, ScratchReg,
+                     ResidualUseDForm);
+  }
+  bool UseDForm = CanUseDForm(NegProbeSize);
+  // If number of blocks is small, just probe them directly.
+  if (NumBlocks < 3) {
+    if (!UseDForm)
+      MaterializeImm(PrologMBB, {MI}, NegProbeSize, ScratchReg);
+    for (int i = 0; i < NumBlocks; ++i)
+      allocateAndProbe(PrologMBB, {MI}, NegProbeSize, ScratchReg, UseDForm);
+  } else {
+    // Since CTR is a volatile register and current shrinkwrap implementation
+    // won't choose an MBB in a loop as the PrologMBB, it's safe to synthesize a
+    // CTR loop to probe.
+    // Calculate trip count and stores it in CTRReg.
+    MaterializeImm(PrologMBB, {MI}, NumBlocks, ScratchReg);
+    BuildMI(PrologMBB, {MI}, DL, TII.get(isPPC64 ? PPC::MTCTR8 : PPC::MTCTR))
+        .addReg(ScratchReg, RegState::Kill);
+    if (!UseDForm)
+      MaterializeImm(PrologMBB, {MI}, NegProbeSize, ScratchReg);
+    // Create MBBs of the loop.
+    MachineFunction::iterator MBBInsertPoint =
+        std::next(PrologMBB.getIterator());
+    MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(ProbedBB);
+    MF.insert(MBBInsertPoint, LoopMBB);
+    MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(ProbedBB);
+    MF.insert(MBBInsertPoint, ExitMBB);
+    // Synthesize the loop body.
+    allocateAndProbe(*LoopMBB, LoopMBB->end(), NegProbeSize, ScratchReg,
+                     UseDForm);
+    BuildMI(LoopMBB, DL, TII.get(isPPC64 ? PPC::BDNZ8 : PPC::BDNZ))
+        .addMBB(LoopMBB);
+    LoopMBB->addSuccessor(ExitMBB);
+    LoopMBB->addSuccessor(LoopMBB);
+    // Synthesize the exit MBB.
+    ExitMBB->splice(ExitMBB->end(), &PrologMBB,
+                    std::next(MachineBasicBlock::iterator(MI)),
+                    PrologMBB.end());
+    ExitMBB->transferSuccessorsAndUpdatePHIs(&PrologMBB);
+    PrologMBB.addSuccessor(LoopMBB);
+    // Update liveins.
+    recomputeLiveIns(*LoopMBB);
+    recomputeLiveIns(*ExitMBB);
+  }
+  ++NumPrologProbed;
+  MI.eraseFromParent();
+}
+
 void PPCFrameLowering::emitEpilogue(MachineFunction &MF,
                                     MachineBasicBlock &MBB) const {
   MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
@@ -2081,8 +2248,8 @@ PPCFrameLowering::addScavengingSpillSlot(MachineFunction &MF,
     const TargetRegisterClass &RC = Subtarget.isPPC64() ? G8RC : GPRC;
     const TargetRegisterInfo &TRI = *Subtarget.getRegisterInfo();
     unsigned Size = TRI.getSpillSize(RC);
-    unsigned Align = TRI.getSpillAlignment(RC);
-    RS->addScavengingFrameIndex(MFI.CreateStackObject(Size, Align, false));
+    Align Alignment = TRI.getSpillAlign(RC);
+    RS->addScavengingFrameIndex(MFI.CreateStackObject(Size, Alignment, false));
 
     // Might we have over-aligned allocas?
     bool HasAlVars =
@@ -2090,8 +2257,8 @@ PPCFrameLowering::addScavengingSpillSlot(MachineFunction &MF,
 
     // These kinds of spills might need two registers.
     if (spillsCR(MF) || spillsVRSAVE(MF) || HasAlVars)
-      RS->addScavengingFrameIndex(MFI.CreateStackObject(Size, Align, false));
-
+      RS->addScavengingFrameIndex(
+          MFI.CreateStackObject(Size, Alignment, false));
   }
 }
 
@@ -2419,8 +2586,6 @@ unsigned PPCFrameLowering::getFramePointerSaveOffset() const {
 }
 
 unsigned PPCFrameLowering::getBasePointerSaveOffset() const {
-  if (Subtarget.isAIXABI())
-    report_fatal_error("BasePointer is not implemented on AIX yet.");
   return BasePointerSaveOffset;
 }
 
diff --git a/llvm/lib/Target/PowerPC/PPCFrameLowering.h b/llvm/lib/Target/PowerPC/PPCFrameLowering.h
index dd832ffba6434..8bf52c0ed01ab 100644
--- a/llvm/lib/Target/PowerPC/PPCFrameLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCFrameLowering.h
@@ -100,6 +100,8 @@ class PPCFrameLowering: public TargetFrameLowering {
   /// the function.
   void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
+  void inlineStackProbe(MachineFunction &MF,
+                        MachineBasicBlock &PrologMBB) const override;
 
   bool hasFP(const MachineFunction &MF) const override;
   bool needsFP(const MachineFunction &MF) const;
diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index 71fad23b78c6c..8ffd89ef5ccd2 100644
--- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -139,6 +139,7 @@ namespace {
   class PPCDAGToDAGISel : public SelectionDAGISel {
     const PPCTargetMachine &TM;
     const PPCSubtarget *PPCSubTarget = nullptr;
+    const PPCSubtarget *Subtarget = nullptr;
     const PPCTargetLowering *PPCLowering = nullptr;
     unsigned GlobalBaseReg = 0;
 
@@ -150,10 +151,11 @@ namespace {
       // Make sure we re-emit a set of the global base reg if necessary
       GlobalBaseReg = 0;
       PPCSubTarget = &MF.getSubtarget<PPCSubtarget>();
-      PPCLowering = PPCSubTarget->getTargetLowering();
+      Subtarget = &MF.getSubtarget<PPCSubtarget>();
+      PPCLowering = Subtarget->getTargetLowering();
       SelectionDAGISel::runOnMachineFunction(MF);
 
-      if (!PPCSubTarget->isSVR4ABI())
+      if (!Subtarget->isSVR4ABI())
         InsertVRSaveCode(MF);
 
       return true;
@@ -238,7 +240,7 @@ namespace {
     /// bit signed displacement.
     /// Returns false if it can be represented by [r+imm], which are preferred.
     bool SelectAddrIdx(SDValue N, SDValue &Base, SDValue &Index) {
-      return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, 0);
+      return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, None);
     }
 
     /// SelectAddrIdx4 - Given the specified address, check to see if it can be
@@ -248,7 +250,8 @@ namespace {
     /// displacement must be a multiple of 4.
     /// Returns false if it can be represented by [r+imm], which are preferred.
     bool SelectAddrIdxX4(SDValue N, SDValue &Base, SDValue &Index) {
-      return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, 4);
+      return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG,
+                                              Align(4));
     }
 
     /// SelectAddrIdx16 - Given the specified address, check to see if it can be
@@ -258,7 +261,8 @@ namespace {
     /// displacement must be a multiple of 16.
     /// Returns false if it can be represented by [r+imm], which are preferred.
     bool SelectAddrIdxX16(SDValue N, SDValue &Base, SDValue &Index) {
-      return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, 16);
+      return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG,
+                                              Align(16));
     }
 
     /// SelectAddrIdxOnly - Given the specified address, force it to be
@@ -266,28 +270,29 @@ namespace {
     bool SelectAddrIdxOnly(SDValue N, SDValue &Base, SDValue &Index) {
       return PPCLowering->SelectAddressRegRegOnly(N, Base, Index, *CurDAG);
     }
-    
+
     /// SelectAddrImm - Returns true if the address N can be represented by
     /// a base register plus a signed 16-bit displacement [r+imm].
     /// The last parameter \p 0 means D form has no requirment for 16 bit signed
     /// displacement.
     bool SelectAddrImm(SDValue N, SDValue &Disp,
                        SDValue &Base) {
-      return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, 0);
+      return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, None);
     }
 
     /// SelectAddrImmX4 - Returns true if the address N can be represented by
     /// a base register plus a signed 16-bit displacement that is a multiple of
     /// 4 (last parameter). Suitable for use by STD and friends.
     bool SelectAddrImmX4(SDValue N, SDValue &Disp, SDValue &Base) {
-      return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, 4);
+      return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, Align(4));
     }
 
     /// SelectAddrImmX16 - Returns true if the address N can be represented by
     /// a base register plus a signed 16-bit displacement that is a multiple of
     /// 16(last parameter). Suitable for use by STXV and friends.
     bool SelectAddrImmX16(SDValue N, SDValue &Disp, SDValue &Base) {
-      return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, 16);
+      return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG,
+                                              Align(16));
     }
 
     // Select an address into a single register.
@@ -320,7 +325,7 @@ namespace {
       case InlineAsm::Constraint_Zy:
         // We need to make sure that this one operand does not end up in r0
         // (because we might end up lowering this as 0(%op)).
-        const TargetRegisterInfo *TRI = PPCSubTarget->getRegisterInfo();
+        const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
         const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF, /*Kind=*/1);
         SDLoc dl(Op);
         SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32);
@@ -404,7 +409,7 @@ void PPCDAGToDAGISel::InsertVRSaveCode(MachineFunction &Fn) {
   Register InVRSAVE = RegInfo->createVirtualRegister(&PPC::GPRCRegClass);
   Register UpdatedVRSAVE = RegInfo->createVirtualRegister(&PPC::GPRCRegClass);
 
-  const TargetInstrInfo &TII = *PPCSubTarget->getInstrInfo();
+  const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
   MachineBasicBlock &EntryBB = *Fn.begin();
   DebugLoc dl;
   // Emit the following code into the entry block:
@@ -439,7 +444,7 @@ void PPCDAGToDAGISel::InsertVRSaveCode(MachineFunction &Fn) {
 ///
 SDNode *PPCDAGToDAGISel::getGlobalBaseReg() {
   if (!GlobalBaseReg) {
-    const TargetInstrInfo &TII = *PPCSubTarget->getInstrInfo();
+    const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
     // Insert the set of GlobalBaseReg into the first MBB of the function
     MachineBasicBlock &FirstMBB = MF->front();
     MachineBasicBlock::iterator MBBI = FirstMBB.begin();
@@ -447,9 +452,9 @@ SDNode *PPCDAGToDAGISel::getGlobalBaseReg() {
     DebugLoc dl;
 
     if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) == MVT::i32) {
-      if (PPCSubTarget->isTargetELF()) {
+      if (Subtarget->isTargetELF()) {
         GlobalBaseReg = PPC::R30;
-        if (!PPCSubTarget->isSecurePlt() &&
+        if (!Subtarget->isSecurePlt() &&
             M->getPICLevel() == PICLevel::SmallPIC) {
           BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MoveGOTtoLR));
           BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg);
@@ -3798,7 +3803,7 @@ SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
       Opc = PPC::CMPD;
     }
   } else if (LHS.getValueType() == MVT::f32) {
-    if (PPCSubTarget->hasSPE()) {
+    if (Subtarget->hasSPE()) {
       switch (CC) {
         default:
         case ISD::SETEQ:
@@ -3825,7 +3830,7 @@ SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
     } else
       Opc = PPC::FCMPUS;
   } else if (LHS.getValueType() == MVT::f64) {
-    if (PPCSubTarget->hasSPE()) {
+    if (Subtarget->hasSPE()) {
       switch (CC) {
         default:
         case ISD::SETEQ:
@@ -3850,10 +3855,10 @@ SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
           break;
       }
     } else
-      Opc = PPCSubTarget->hasVSX() ? PPC::XSCMPUDP : PPC::FCMPUD;
+      Opc = Subtarget->hasVSX() ? PPC::XSCMPUDP : PPC::FCMPUD;
   } else {
     assert(LHS.getValueType() == MVT::f128 && "Unknown vt!");
-    assert(PPCSubTarget->hasVSX() && "__float128 requires VSX");
+    assert(Subtarget->hasVSX() && "__float128 requires VSX");
     Opc = PPC::XSCMPUQP;
   }
   return SDValue(CurDAG->getMachineNode(Opc, dl, MVT::i32, LHS, RHS), 0);
@@ -4048,8 +4053,7 @@ bool PPCDAGToDAGISel::trySETCC(SDNode *N) {
       CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout());
   bool isPPC64 = (PtrVT == MVT::i64);
 
-  if (!PPCSubTarget->useCRBits() &&
-      isInt32Immediate(N->getOperand(1), Imm)) {
+  if (!Subtarget->useCRBits() && isInt32Immediate(N->getOperand(1), Imm)) {
     // We can codegen setcc op, imm very efficiently compared to a brcond.
     // Check for those cases here.
     // setcc op, 0
@@ -4138,20 +4142,20 @@ bool PPCDAGToDAGISel::trySETCC(SDNode *N) {
   // Altivec Vector compare instructions do not set any CR register by default and
   // vector compare operations return the same type as the operands.
   if (LHS.getValueType().isVector()) {
-    if (PPCSubTarget->hasQPX() || PPCSubTarget->hasSPE())
+    if (Subtarget->hasQPX() || Subtarget->hasSPE())
       return false;
 
     EVT VecVT = LHS.getValueType();
     bool Swap, Negate;
-    unsigned int VCmpInst = getVCmpInst(VecVT.getSimpleVT(), CC,
-                                        PPCSubTarget->hasVSX(), Swap, Negate);
+    unsigned int VCmpInst =
+        getVCmpInst(VecVT.getSimpleVT(), CC, Subtarget->hasVSX(), Swap, Negate);
     if (Swap)
       std::swap(LHS, RHS);
 
     EVT ResVT = VecVT.changeVectorElementTypeToInteger();
     if (Negate) {
       SDValue VCmp(CurDAG->getMachineNode(VCmpInst, dl, ResVT, LHS, RHS), 0);
-      CurDAG->SelectNodeTo(N, PPCSubTarget->hasVSX() ? PPC::XXLNOR : PPC::VNOR,
+      CurDAG->SelectNodeTo(N, Subtarget->hasVSX() ? PPC::XXLNOR : PPC::VNOR,
                            ResVT, VCmp, VCmp);
       return true;
     }
@@ -4160,7 +4164,7 @@ bool PPCDAGToDAGISel::trySETCC(SDNode *N) {
     return true;
   }
 
-  if (PPCSubTarget->useCRBits())
+  if (Subtarget->useCRBits())
     return false;
 
   bool Inv;
@@ -4170,7 +4174,7 @@ bool PPCDAGToDAGISel::trySETCC(SDNode *N) {
 
   // SPE e*cmp* instructions only set the 'gt' bit, so hard-code that
   // The correct compare instruction is already set by SelectCC()
-  if (PPCSubTarget->hasSPE() && LHS.getValueType().isFloatingPoint()) {
+  if (Subtarget->hasSPE() && LHS.getValueType().isFloatingPoint()) {
     Idx = 1;
   }
 
@@ -4667,7 +4671,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
   case PPCISD::ADDI_TLSGD_L_ADDR: {
     const Module *Mod = MF->getFunction().getParent();
     if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 ||
-        !PPCSubTarget->isSecurePlt() || !PPCSubTarget->isTargetELF() ||
+        !Subtarget->isSecurePlt() || !Subtarget->isTargetELF() ||
         Mod->getPICLevel() == PICLevel::SmallPIC)
       break;
     // Attach global base pointer on GETtlsADDR32 node in order to
@@ -4676,8 +4680,8 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
   } break;
   case PPCISD::CALL: {
     if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 ||
-        !TM.isPositionIndependent() || !PPCSubTarget->isSecurePlt() ||
-        !PPCSubTarget->isTargetELF())
+        !TM.isPositionIndependent() || !Subtarget->isSecurePlt() ||
+        !Subtarget->isTargetELF())
       break;
 
     SDValue Op = N->getOperand(1);
@@ -4741,7 +4745,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
   case ISD::STORE: {
     // Change TLS initial-exec D-form stores to X-form stores.
     StoreSDNode *ST = cast<StoreSDNode>(N);
-    if (EnableTLSOpt && PPCSubTarget->isELFv2ABI() &&
+    if (EnableTLSOpt && Subtarget->isELFv2ABI() &&
         ST->getAddressingMode() != ISD::PRE_INC)
       if (tryTLSXFormStore(ST))
         return;
@@ -4755,7 +4759,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
     // Normal loads are handled by code generated from the .td file.
     if (LD->getAddressingMode() != ISD::PRE_INC) {
       // Change TLS initial-exec D-form loads to X-form loads.
-      if (EnableTLSOpt && PPCSubTarget->isELFv2ABI())
+      if (EnableTLSOpt && Subtarget->isELFv2ABI())
         if (tryTLSXFormLoad(LD))
           return;
       break;
@@ -4878,7 +4882,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
     // OR with a 32-bit immediate can be handled by ori + oris
     // without creating an immediate in a GPR.
     uint64_t Imm64 = 0;
-    bool IsPPC64 = PPCSubTarget->isPPC64();
+    bool IsPPC64 = Subtarget->isPPC64();
     if (IsPPC64 && isInt64Immediate(N->getOperand(1), Imm64) &&
         (Imm64 & ~0xFFFFFFFFuLL) == 0) {
       // If ImmHi (ImmHi) is zero, only one ori (oris) is generated later.
@@ -4901,7 +4905,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
     // XOR with a 32-bit immediate can be handled by xori + xoris
     // without creating an immediate in a GPR.
     uint64_t Imm64 = 0;
-    bool IsPPC64 = PPCSubTarget->isPPC64();
+    bool IsPPC64 = Subtarget->isPPC64();
     if (IsPPC64 && isInt64Immediate(N->getOperand(1), Imm64) &&
         (Imm64 & ~0xFFFFFFFFuLL) == 0) {
       // If ImmHi (ImmHi) is zero, only one xori (xoris) is generated later.
@@ -4988,11 +4992,10 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
     bool isPPC64 = (PtrVT == MVT::i64);
 
     // If this is a select of i1 operands, we'll pattern match it.
-    if (PPCSubTarget->useCRBits() &&
-        N->getOperand(0).getValueType() == MVT::i1)
+    if (Subtarget->useCRBits() && N->getOperand(0).getValueType() == MVT::i1)
       break;
 
-    if (PPCSubTarget->isISA3_0() && PPCSubTarget->isPPC64()) {
+    if (Subtarget->isISA3_0() && Subtarget->isPPC64()) {
       bool NeedSwapOps = false;
       bool IsUnCmp = false;
       if (mayUseP9Setb(N, CC, CurDAG, NeedSwapOps, IsUnCmp)) {
@@ -5067,7 +5070,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
     }
 
     unsigned BROpc =
-        getPredicateForSetCC(CC, N->getOperand(0).getValueType(), PPCSubTarget);
+        getPredicateForSetCC(CC, N->getOperand(0).getValueType(), Subtarget);
 
     unsigned SelectCCOp;
     if (N->getValueType(0) == MVT::i32)
@@ -5075,28 +5078,28 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
     else if (N->getValueType(0) == MVT::i64)
       SelectCCOp = PPC::SELECT_CC_I8;
     else if (N->getValueType(0) == MVT::f32) {
-      if (PPCSubTarget->hasP8Vector())
+      if (Subtarget->hasP8Vector())
         SelectCCOp = PPC::SELECT_CC_VSSRC;
-      else if (PPCSubTarget->hasSPE())
+      else if (Subtarget->hasSPE())
         SelectCCOp = PPC::SELECT_CC_SPE4;
       else
         SelectCCOp = PPC::SELECT_CC_F4;
     } else if (N->getValueType(0) == MVT::f64) {
-      if (PPCSubTarget->hasVSX())
+      if (Subtarget->hasVSX())
         SelectCCOp = PPC::SELECT_CC_VSFRC;
-      else if (PPCSubTarget->hasSPE())
+      else if (Subtarget->hasSPE())
         SelectCCOp = PPC::SELECT_CC_SPE;
       else
         SelectCCOp = PPC::SELECT_CC_F8;
     } else if (N->getValueType(0) == MVT::f128)
       SelectCCOp = PPC::SELECT_CC_F16;
-    else if (PPCSubTarget->hasSPE())
+    else if (Subtarget->hasSPE())
       SelectCCOp = PPC::SELECT_CC_SPE;
-    else if (PPCSubTarget->hasQPX() && N->getValueType(0) == MVT::v4f64)
+    else if (Subtarget->hasQPX() && N->getValueType(0) == MVT::v4f64)
       SelectCCOp = PPC::SELECT_CC_QFRC;
-    else if (PPCSubTarget->hasQPX() && N->getValueType(0) == MVT::v4f32)
+    else if (Subtarget->hasQPX() && N->getValueType(0) == MVT::v4f32)
       SelectCCOp = PPC::SELECT_CC_QSRC;
-    else if (PPCSubTarget->hasQPX() && N->getValueType(0) == MVT::v4i1)
+    else if (Subtarget->hasQPX() && N->getValueType(0) == MVT::v4i1)
       SelectCCOp = PPC::SELECT_CC_QBRC;
     else if (N->getValueType(0) == MVT::v2f64 ||
              N->getValueType(0) == MVT::v2i64)
@@ -5110,8 +5113,8 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
     return;
   }
   case ISD::VECTOR_SHUFFLE:
-    if (PPCSubTarget->hasVSX() && (N->getValueType(0) == MVT::v2f64 ||
-                                  N->getValueType(0) == MVT::v2i64)) {
+    if (Subtarget->hasVSX() && (N->getValueType(0) == MVT::v2f64 ||
+                                N->getValueType(0) == MVT::v2i64)) {
       ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
 
       SDValue Op1 = N->getOperand(SVN->getMaskElt(0) < 2 ? 0 : 1),
@@ -5146,7 +5149,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
 
       // For little endian, we must swap the input operands and adjust
       // the mask elements (reverse and invert them).
-      if (PPCSubTarget->isLittleEndian()) {
+      if (Subtarget->isLittleEndian()) {
         std::swap(Op1, Op2);
         unsigned tmp = DM[0];
         DM[0] = 1 - DM[1];
@@ -5163,7 +5166,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
     break;
   case PPCISD::BDNZ:
   case PPCISD::BDZ: {
-    bool IsPPC64 = PPCSubTarget->isPPC64();
+    bool IsPPC64 = Subtarget->isPPC64();
     SDValue Ops[] = { N->getOperand(1), N->getOperand(0) };
     CurDAG->SelectNodeTo(N, N->getOpcode() == PPCISD::BDNZ
                                 ? (IsPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
@@ -5191,7 +5194,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
   case ISD::BR_CC: {
     ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get();
     unsigned PCC =
-        getPredicateForSetCC(CC, N->getOperand(2).getValueType(), PPCSubTarget);
+        getPredicateForSetCC(CC, N->getOperand(2).getValueType(), Subtarget);
 
     if (N->getOperand(2).getValueType() == MVT::i1) {
       unsigned Opc;
@@ -5244,9 +5247,9 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
     return;
   }
   case PPCISD::TOC_ENTRY: {
-    const bool isPPC64 = PPCSubTarget->isPPC64();
-    const bool isELFABI = PPCSubTarget->isSVR4ABI();
-    const bool isAIXABI = PPCSubTarget->isAIXABI();
+    const bool isPPC64 = Subtarget->isPPC64();
+    const bool isELFABI = Subtarget->isSVR4ABI();
+    const bool isAIXABI = Subtarget->isAIXABI();
 
     // PowerPC only support small, medium and large code model.
     const CodeModel::Model CModel = TM.getCodeModel();
@@ -5297,7 +5300,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
     // or 64-bit medium (ELF-only) or large (ELF and AIX) code model code. We
     // generate two instructions as described below. The first source operand
     // is a symbol reference. If it must be toc-referenced according to
-    // PPCSubTarget, we generate:
+    // Subtarget, we generate:
     // [32-bit AIX]
     //   LWZtocL(@sym, ADDIStocHA(%r2, @sym))
     // [64-bit ELF/AIX]
@@ -5329,7 +5332,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
   }
   case PPCISD::PPC32_PICGOT:
     // Generate a PIC-safe GOT reference.
-    assert(PPCSubTarget->is32BitELFABI() &&
+    assert(Subtarget->is32BitELFABI() &&
            "PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4");
     CurDAG->SelectNodeTo(N, PPC::PPC32PICGOT,
                          PPCLowering->getPointerTy(CurDAG->getDataLayout()),
@@ -5426,7 +5429,7 @@ SDValue PPCDAGToDAGISel::combineToCMPB(SDNode *N) {
          "Only OR nodes are supported for CMPB");
 
   SDValue Res;
-  if (!PPCSubTarget->hasCMPB())
+  if (!Subtarget->hasCMPB())
     return Res;
 
   if (N->getValueType(0) != MVT::i32 &&
@@ -5637,7 +5640,7 @@ SDValue PPCDAGToDAGISel::combineToCMPB(SDNode *N) {
 // only one instruction (like a zero or one), then we should fold in those
 // operations with the select.
 void PPCDAGToDAGISel::foldBoolExts(SDValue &Res, SDNode *&N) {
-  if (!PPCSubTarget->useCRBits())
+  if (!Subtarget->useCRBits())
     return;
 
   if (N->getOpcode() != ISD::ZERO_EXTEND &&
@@ -6378,7 +6381,7 @@ static bool PeepholePPC64ZExtGather(SDValue Op32,
 }
 
 void PPCDAGToDAGISel::PeepholePPC64ZExt() {
-  if (!PPCSubTarget->isPPC64())
+  if (!Subtarget->isPPC64())
     return;
 
   // When we zero-extend from i32 to i64, we use a pattern like this:
@@ -6659,7 +6662,8 @@ void PPCDAGToDAGISel::PeepholePPC64() {
     int MaxDisplacement = 7;
     if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(ImmOpnd)) {
       const GlobalValue *GV = GA->getGlobal();
-      MaxDisplacement = std::min((int) GV->getAlignment() - 1, MaxDisplacement);
+      Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout());
+      MaxDisplacement = std::min((int)Alignment.value() - 1, MaxDisplacement);
     }
 
     bool UpdateHBase = false;
@@ -6725,10 +6729,10 @@ void PPCDAGToDAGISel::PeepholePPC64() {
       if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(ImmOpnd)) {
         SDLoc dl(GA);
         const GlobalValue *GV = GA->getGlobal();
+        Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout());
         // We can't perform this optimization for data whose alignment
         // is insufficient for the instruction encoding.
-        if (GV->getAlignment() < 4 &&
-            (RequiresMod4Offset || (Offset % 4) != 0)) {
+        if (Alignment < 4 && (RequiresMod4Offset || (Offset % 4) != 0)) {
           LLVM_DEBUG(dbgs() << "Rejected this candidate for alignment.\n\n");
           continue;
         }
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 28bd80610c841..575ad68fecd99 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -117,15 +117,13 @@ cl::desc("disable sibling call optimization on ppc"), cl::Hidden);
 static cl::opt<bool> DisableInnermostLoopAlign32("disable-ppc-innermost-loop-align32",
 cl::desc("don't always align innermost loop to 32 bytes on ppc"), cl::Hidden);
 
-static cl::opt<bool> EnableQuadPrecision("enable-ppc-quad-precision",
-cl::desc("enable quad precision float support on ppc"), cl::Hidden);
-
 static cl::opt<bool> UseAbsoluteJumpTables("ppc-use-absolute-jumptables",
 cl::desc("use absolute jump tables on ppc"), cl::Hidden);
 
 STATISTIC(NumTailCalls, "Number of tail calls");
 STATISTIC(NumSiblingCalls, "Number of sibling calls");
 STATISTIC(ShufflesHandledWithVPERM, "Number of shuffles lowered to a VPERM");
+STATISTIC(NumDynamicAllocaProbed, "Number of dynamic stack allocation probed");
 
 static bool isNByteElemShuffleMask(ShuffleVectorSDNode *, unsigned, int);
 
@@ -260,15 +258,16 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
 
   // PowerPC has no SREM/UREM instructions unless we are on P9
   // On P9 we may use a hardware instruction to compute the remainder.
-  // The instructions are not legalized directly because in the cases where the
-  // result of both the remainder and the division is required it is more
-  // efficient to compute the remainder from the result of the division rather
-  // than use the remainder instruction.
+  // When the result of both the remainder and the division is required it is
+  // more efficient to compute the remainder from the result of the division
+  // rather than use the remainder instruction. The instructions are legalized
+  // directly because the DivRemPairsPass performs the transformation at the IR
+  // level.
   if (Subtarget.isISA3_0()) {
-    setOperationAction(ISD::SREM, MVT::i32, Custom);
-    setOperationAction(ISD::UREM, MVT::i32, Custom);
-    setOperationAction(ISD::SREM, MVT::i64, Custom);
-    setOperationAction(ISD::UREM, MVT::i64, Custom);
+    setOperationAction(ISD::SREM, MVT::i32, Legal);
+    setOperationAction(ISD::UREM, MVT::i32, Legal);
+    setOperationAction(ISD::SREM, MVT::i64, Legal);
+    setOperationAction(ISD::UREM, MVT::i64, Legal);
   } else {
     setOperationAction(ISD::SREM, MVT::i32, Expand);
     setOperationAction(ISD::UREM, MVT::i32, Expand);
@@ -424,6 +423,9 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
 
   if (Subtarget.hasSPE()) {
     // SPE has built-in conversions
+    setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Legal);
+    setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Legal);
+    setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Legal);
     setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal);
     setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal);
     setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal);
@@ -573,9 +575,10 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
       setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
   } else {
     // PowerPC does not have FP_TO_UINT on 32-bit implementations.
-    if (Subtarget.hasSPE())
+    if (Subtarget.hasSPE()) {
+      setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Legal);
       setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal);
-    else
+    } else
       setOperationAction(ISD::FP_TO_UINT, MVT::i32, Expand);
   }
 
@@ -1002,61 +1005,59 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
       setOperationAction(ISD::SRL, MVT::v1i128, Legal);
       setOperationAction(ISD::SRA, MVT::v1i128, Expand);
 
-      if (EnableQuadPrecision) {
-        addRegisterClass(MVT::f128, &PPC::VRRCRegClass);
-        setOperationAction(ISD::FADD, MVT::f128, Legal);
-        setOperationAction(ISD::FSUB, MVT::f128, Legal);
-        setOperationAction(ISD::FDIV, MVT::f128, Legal);
-        setOperationAction(ISD::FMUL, MVT::f128, Legal);
-        setOperationAction(ISD::FP_EXTEND, MVT::f128, Legal);
-        // No extending loads to f128 on PPC.
-        for (MVT FPT : MVT::fp_valuetypes())
-          setLoadExtAction(ISD::EXTLOAD, MVT::f128, FPT, Expand);
-        setOperationAction(ISD::FMA, MVT::f128, Legal);
-        setCondCodeAction(ISD::SETULT, MVT::f128, Expand);
-        setCondCodeAction(ISD::SETUGT, MVT::f128, Expand);
-        setCondCodeAction(ISD::SETUEQ, MVT::f128, Expand);
-        setCondCodeAction(ISD::SETOGE, MVT::f128, Expand);
-        setCondCodeAction(ISD::SETOLE, MVT::f128, Expand);
-        setCondCodeAction(ISD::SETONE, MVT::f128, Expand);
-
-        setOperationAction(ISD::FTRUNC, MVT::f128, Legal);
-        setOperationAction(ISD::FRINT, MVT::f128, Legal);
-        setOperationAction(ISD::FFLOOR, MVT::f128, Legal);
-        setOperationAction(ISD::FCEIL, MVT::f128, Legal);
-        setOperationAction(ISD::FNEARBYINT, MVT::f128, Legal);
-        setOperationAction(ISD::FROUND, MVT::f128, Legal);
-
-        setOperationAction(ISD::SELECT, MVT::f128, Expand);
-        setOperationAction(ISD::FP_ROUND, MVT::f64, Legal);
-        setOperationAction(ISD::FP_ROUND, MVT::f32, Legal);
-        setTruncStoreAction(MVT::f128, MVT::f64, Expand);
-        setTruncStoreAction(MVT::f128, MVT::f32, Expand);
-        setOperationAction(ISD::BITCAST, MVT::i128, Custom);
-        // No implementation for these ops for PowerPC.
-        setOperationAction(ISD::FSIN , MVT::f128, Expand);
-        setOperationAction(ISD::FCOS , MVT::f128, Expand);
-        setOperationAction(ISD::FPOW, MVT::f128, Expand);
-        setOperationAction(ISD::FPOWI, MVT::f128, Expand);
-        setOperationAction(ISD::FREM, MVT::f128, Expand);
-
-        // Handle constrained floating-point operations of fp128
-        setOperationAction(ISD::STRICT_FADD, MVT::f128, Legal);
-        setOperationAction(ISD::STRICT_FSUB, MVT::f128, Legal);
-        setOperationAction(ISD::STRICT_FMUL, MVT::f128, Legal);
-        setOperationAction(ISD::STRICT_FDIV, MVT::f128, Legal);
-        setOperationAction(ISD::STRICT_FMA, MVT::f128, Legal);
-        setOperationAction(ISD::STRICT_FSQRT, MVT::f128, Legal);
-        setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Legal);
-        setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Legal);
-        setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
-        setOperationAction(ISD::STRICT_FRINT, MVT::f128, Legal);
-        setOperationAction(ISD::STRICT_FNEARBYINT, MVT::f128, Legal);
-        setOperationAction(ISD::STRICT_FFLOOR, MVT::f128, Legal);
-        setOperationAction(ISD::STRICT_FCEIL,  MVT::f128, Legal);
-        setOperationAction(ISD::STRICT_FTRUNC, MVT::f128, Legal);
-        setOperationAction(ISD::STRICT_FROUND, MVT::f128, Legal);
-      }
+      addRegisterClass(MVT::f128, &PPC::VRRCRegClass);
+      setOperationAction(ISD::FADD, MVT::f128, Legal);
+      setOperationAction(ISD::FSUB, MVT::f128, Legal);
+      setOperationAction(ISD::FDIV, MVT::f128, Legal);
+      setOperationAction(ISD::FMUL, MVT::f128, Legal);
+      setOperationAction(ISD::FP_EXTEND, MVT::f128, Legal);
+      // No extending loads to f128 on PPC.
+      for (MVT FPT : MVT::fp_valuetypes())
+        setLoadExtAction(ISD::EXTLOAD, MVT::f128, FPT, Expand);
+      setOperationAction(ISD::FMA, MVT::f128, Legal);
+      setCondCodeAction(ISD::SETULT, MVT::f128, Expand);
+      setCondCodeAction(ISD::SETUGT, MVT::f128, Expand);
+      setCondCodeAction(ISD::SETUEQ, MVT::f128, Expand);
+      setCondCodeAction(ISD::SETOGE, MVT::f128, Expand);
+      setCondCodeAction(ISD::SETOLE, MVT::f128, Expand);
+      setCondCodeAction(ISD::SETONE, MVT::f128, Expand);
+
+      setOperationAction(ISD::FTRUNC, MVT::f128, Legal);
+      setOperationAction(ISD::FRINT, MVT::f128, Legal);
+      setOperationAction(ISD::FFLOOR, MVT::f128, Legal);
+      setOperationAction(ISD::FCEIL, MVT::f128, Legal);
+      setOperationAction(ISD::FNEARBYINT, MVT::f128, Legal);
+      setOperationAction(ISD::FROUND, MVT::f128, Legal);
+
+      setOperationAction(ISD::SELECT, MVT::f128, Expand);
+      setOperationAction(ISD::FP_ROUND, MVT::f64, Legal);
+      setOperationAction(ISD::FP_ROUND, MVT::f32, Legal);
+      setTruncStoreAction(MVT::f128, MVT::f64, Expand);
+      setTruncStoreAction(MVT::f128, MVT::f32, Expand);
+      setOperationAction(ISD::BITCAST, MVT::i128, Custom);
+      // No implementation for these ops for PowerPC.
+      setOperationAction(ISD::FSIN, MVT::f128, Expand);
+      setOperationAction(ISD::FCOS, MVT::f128, Expand);
+      setOperationAction(ISD::FPOW, MVT::f128, Expand);
+      setOperationAction(ISD::FPOWI, MVT::f128, Expand);
+      setOperationAction(ISD::FREM, MVT::f128, Expand);
+
+      // Handle constrained floating-point operations of fp128
+      setOperationAction(ISD::STRICT_FADD, MVT::f128, Legal);
+      setOperationAction(ISD::STRICT_FSUB, MVT::f128, Legal);
+      setOperationAction(ISD::STRICT_FMUL, MVT::f128, Legal);
+      setOperationAction(ISD::STRICT_FDIV, MVT::f128, Legal);
+      setOperationAction(ISD::STRICT_FMA, MVT::f128, Legal);
+      setOperationAction(ISD::STRICT_FSQRT, MVT::f128, Legal);
+      setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Legal);
+      setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Legal);
+      setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
+      setOperationAction(ISD::STRICT_FRINT, MVT::f128, Legal);
+      setOperationAction(ISD::STRICT_FNEARBYINT, MVT::f128, Legal);
+      setOperationAction(ISD::STRICT_FFLOOR, MVT::f128, Legal);
+      setOperationAction(ISD::STRICT_FCEIL, MVT::f128, Legal);
+      setOperationAction(ISD::STRICT_FTRUNC, MVT::f128, Legal);
+      setOperationAction(ISD::STRICT_FROUND, MVT::f128, Legal);
       setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom);
       setOperationAction(ISD::BSWAP, MVT::v8i16, Legal);
       setOperationAction(ISD::BSWAP, MVT::v4i32, Legal);
@@ -1305,20 +1306,18 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
     setTargetDAGCombine(ISD::VSELECT);
   }
 
-  if (EnableQuadPrecision) {
-    setLibcallName(RTLIB::LOG_F128, "logf128");
-    setLibcallName(RTLIB::LOG2_F128, "log2f128");
-    setLibcallName(RTLIB::LOG10_F128, "log10f128");
-    setLibcallName(RTLIB::EXP_F128, "expf128");
-    setLibcallName(RTLIB::EXP2_F128, "exp2f128");
-    setLibcallName(RTLIB::SIN_F128, "sinf128");
-    setLibcallName(RTLIB::COS_F128, "cosf128");
-    setLibcallName(RTLIB::POW_F128, "powf128");
-    setLibcallName(RTLIB::FMIN_F128, "fminf128");
-    setLibcallName(RTLIB::FMAX_F128, "fmaxf128");
-    setLibcallName(RTLIB::POWI_F128, "__powikf2");
-    setLibcallName(RTLIB::REM_F128, "fmodf128");
-  }
+  setLibcallName(RTLIB::LOG_F128, "logf128");
+  setLibcallName(RTLIB::LOG2_F128, "log2f128");
+  setLibcallName(RTLIB::LOG10_F128, "log10f128");
+  setLibcallName(RTLIB::EXP_F128, "expf128");
+  setLibcallName(RTLIB::EXP2_F128, "exp2f128");
+  setLibcallName(RTLIB::SIN_F128, "sinf128");
+  setLibcallName(RTLIB::COS_F128, "cosf128");
+  setLibcallName(RTLIB::POW_F128, "powf128");
+  setLibcallName(RTLIB::FMIN_F128, "fminf128");
+  setLibcallName(RTLIB::FMAX_F128, "fmaxf128");
+  setLibcallName(RTLIB::POWI_F128, "__powikf2");
+  setLibcallName(RTLIB::REM_F128, "fmodf128");
 
   // With 32 condition bits, we don't need to sink (and duplicate) compares
   // aggressively in CodeGenPrep.
@@ -1389,25 +1388,24 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
 
 /// getMaxByValAlign - Helper for getByValTypeAlignment to determine
 /// the desired ByVal argument alignment.
-static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign,
-                             unsigned MaxMaxAlign) {
+static void getMaxByValAlign(Type *Ty, Align &MaxAlign, Align MaxMaxAlign) {
   if (MaxAlign == MaxMaxAlign)
     return;
   if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
     if (MaxMaxAlign >= 32 &&
         VTy->getPrimitiveSizeInBits().getFixedSize() >= 256)
-      MaxAlign = 32;
+      MaxAlign = Align(32);
     else if (VTy->getPrimitiveSizeInBits().getFixedSize() >= 128 &&
              MaxAlign < 16)
-      MaxAlign = 16;
+      MaxAlign = Align(16);
   } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
-    unsigned EltAlign = 0;
+    Align EltAlign;
     getMaxByValAlign(ATy->getElementType(), EltAlign, MaxMaxAlign);
     if (EltAlign > MaxAlign)
       MaxAlign = EltAlign;
   } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
     for (auto *EltTy : STy->elements()) {
-      unsigned EltAlign = 0;
+      Align EltAlign;
       getMaxByValAlign(EltTy, EltAlign, MaxMaxAlign);
       if (EltAlign > MaxAlign)
         MaxAlign = EltAlign;
@@ -1423,10 +1421,10 @@ unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty,
                                                   const DataLayout &DL) const {
   // 16byte and wider vectors are passed on 16byte boundary.
   // The rest is 8 on PPC64 and 4 on PPC32 boundary.
-  unsigned Align = Subtarget.isPPC64() ? 8 : 4;
+  Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4);
   if (Subtarget.hasAltivec() || Subtarget.hasQPX())
-    getMaxByValAlign(Ty, Align, Subtarget.hasQPX() ? 32 : 16);
-  return Align;
+    getMaxByValAlign(Ty, Alignment, Subtarget.hasQPX() ? Align(32) : Align(16));
+  return Alignment.value();
 }
 
 bool PPCTargetLowering::useSoftFloat() const {
@@ -1474,6 +1472,10 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::STFIWX:          return "PPCISD::STFIWX";
   case PPCISD::VPERM:           return "PPCISD::VPERM";
   case PPCISD::XXSPLT:          return "PPCISD::XXSPLT";
+  case PPCISD::XXSPLTI_SP_TO_DP:
+    return "PPCISD::XXSPLTI_SP_TO_DP";
+  case PPCISD::XXSPLTI32DX:
+    return "PPCISD::XXSPLTI32DX";
   case PPCISD::VECINSERT:       return "PPCISD::VECINSERT";
   case PPCISD::XXPERMDI:        return "PPCISD::XXPERMDI";
   case PPCISD::VECSHL:          return "PPCISD::VECSHL";
@@ -1485,6 +1487,7 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::ATOMIC_CMP_SWAP_16: return "PPCISD::ATOMIC_CMP_SWAP_16";
   case PPCISD::DYNALLOC:        return "PPCISD::DYNALLOC";
   case PPCISD::DYNAREAOFFSET:   return "PPCISD::DYNAREAOFFSET";
+  case PPCISD::PROBED_ALLOCA:   return "PPCISD::PROBED_ALLOCA";
   case PPCISD::GlobalBaseReg:   return "PPCISD::GlobalBaseReg";
   case PPCISD::SRL:             return "PPCISD::SRL";
   case PPCISD::SRA:             return "PPCISD::SRA";
@@ -2435,22 +2438,22 @@ bool PPCTargetLowering::SelectAddressEVXRegReg(SDValue N, SDValue &Base,
 /// non-zero and N can be represented by a base register plus a signed 16-bit
 /// displacement, make a more precise judgement by checking (displacement % \p
 /// EncodingAlignment).
-bool PPCTargetLowering::SelectAddressRegReg(SDValue N, SDValue &Base,
-                                            SDValue &Index, SelectionDAG &DAG,
-                                            unsigned EncodingAlignment) const {
+bool PPCTargetLowering::SelectAddressRegReg(
+    SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG,
+    MaybeAlign EncodingAlignment) const {
   // If we have a PC Relative target flag don't select as [reg+reg]. It will be
   // a [pc+imm].
   if (SelectAddressPCRel(N, Base))
     return false;
 
-  int16_t imm = 0;
+  int16_t Imm = 0;
   if (N.getOpcode() == ISD::ADD) {
     // Is there any SPE load/store (f64), which can't handle 16bit offset?
     // SPE load/store can only handle 8-bit offsets.
     if (hasSPE() && SelectAddressEVXRegReg(N, Base, Index, DAG))
         return true;
-    if (isIntS16Immediate(N.getOperand(1), imm) &&
-        (!EncodingAlignment || !(imm % EncodingAlignment)))
+    if (isIntS16Immediate(N.getOperand(1), Imm) &&
+        (!EncodingAlignment || isAligned(*EncodingAlignment, Imm)))
       return false; // r+i
     if (N.getOperand(1).getOpcode() == PPCISD::Lo)
       return false;    // r+i
@@ -2459,8 +2462,8 @@ bool PPCTargetLowering::SelectAddressRegReg(SDValue N, SDValue &Base,
     Index = N.getOperand(1);
     return true;
   } else if (N.getOpcode() == ISD::OR) {
-    if (isIntS16Immediate(N.getOperand(1), imm) &&
-        (!EncodingAlignment || !(imm % EncodingAlignment)))
+    if (isIntS16Immediate(N.getOperand(1), Imm) &&
+        (!EncodingAlignment || isAligned(*EncodingAlignment, Imm)))
       return false; // r+i can fold it if we can.
 
     // If this is an or of disjoint bitfields, we can codegen this as an add
@@ -2526,10 +2529,9 @@ static void fixupFuncForFI(SelectionDAG &DAG, int FrameIdx, EVT VT) {
 /// a signed 16-bit displacement [r+imm], and if it is not better
 /// represented as reg+reg.  If \p EncodingAlignment is non-zero, only accept
 /// displacements that are multiples of that value.
-bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp,
-                                            SDValue &Base,
-                                            SelectionDAG &DAG,
-                                            unsigned EncodingAlignment) const {
+bool PPCTargetLowering::SelectAddressRegImm(
+    SDValue N, SDValue &Disp, SDValue &Base, SelectionDAG &DAG,
+    MaybeAlign EncodingAlignment) const {
   // FIXME dl should come from parent load or store, not from address
   SDLoc dl(N);
 
@@ -2545,7 +2547,7 @@ bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp,
   if (N.getOpcode() == ISD::ADD) {
     int16_t imm = 0;
     if (isIntS16Immediate(N.getOperand(1), imm) &&
-        (!EncodingAlignment || (imm % EncodingAlignment) == 0)) {
+        (!EncodingAlignment || isAligned(*EncodingAlignment, imm))) {
       Disp = DAG.getTargetConstant(imm, dl, N.getValueType());
       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0))) {
         Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
@@ -2569,7 +2571,7 @@ bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp,
   } else if (N.getOpcode() == ISD::OR) {
     int16_t imm = 0;
     if (isIntS16Immediate(N.getOperand(1), imm) &&
-        (!EncodingAlignment || (imm % EncodingAlignment) == 0)) {
+        (!EncodingAlignment || isAligned(*EncodingAlignment, imm))) {
       // If this is an or of disjoint bitfields, we can codegen this as an add
       // (for better address arithmetic) if the LHS and RHS of the OR are
       // provably disjoint.
@@ -2596,7 +2598,7 @@ bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp,
     // this as "d, 0"
     int16_t Imm;
     if (isIntS16Immediate(CN, Imm) &&
-        (!EncodingAlignment || (Imm % EncodingAlignment) == 0)) {
+        (!EncodingAlignment || isAligned(*EncodingAlignment, Imm))) {
       Disp = DAG.getTargetConstant(Imm, dl, CN->getValueType(0));
       Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
                              CN->getValueType(0));
@@ -2606,7 +2608,8 @@ bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp,
     // Handle 32-bit sext immediates with LIS + addr mode.
     if ((CN->getValueType(0) == MVT::i32 ||
          (int64_t)CN->getZExtValue() == (int)CN->getZExtValue()) &&
-        (!EncodingAlignment || (CN->getZExtValue() % EncodingAlignment) == 0)) {
+        (!EncodingAlignment ||
+         isAligned(*EncodingAlignment, CN->getZExtValue()))) {
       int Addr = (int)CN->getZExtValue();
 
       // Otherwise, break this down into an LIS + disp.
@@ -2793,14 +2796,14 @@ bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
 
   // LDU/STU can only handle immediates that are a multiple of 4.
   if (VT != MVT::i64) {
-    if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, 0))
+    if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, None))
       return false;
   } else {
     // LDU/STU need an address with at least 4-byte alignment.
     if (Alignment < 4)
       return false;
 
-    if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, 4))
+    if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, Align(4)))
       return false;
   }
 
@@ -3843,7 +3846,8 @@ SDValue PPCTargetLowering::LowerFormalArguments_32SVR4(
       MFI.CreateFixedObject(PtrVT.getSizeInBits()/8,
                             CCInfo.getNextStackOffset(), true));
 
-    FuncInfo->setVarArgsFrameIndex(MFI.CreateStackObject(Depth, 8, false));
+    FuncInfo->setVarArgsFrameIndex(
+        MFI.CreateStackObject(Depth, Align(8), false));
     SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
 
     // The fixed integer arguments of a variadic function are stored to the
@@ -4292,7 +4296,11 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
 
   // If the function takes variable number of arguments, make a frame index for
   // the start of the first vararg value... for expansion of llvm.va_start.
-  if (isVarArg) {
+  // On ELFv2ABI spec, it writes:
+  // C programs that are intended to be *portable* across different compilers
+  // and architectures must use the header file <stdarg.h> to deal with variable
+  // argument lists.
+  if (isVarArg && MFI.hasVAStart()) {
     int Depth = ArgOffset;
 
     FuncInfo->setVarArgsFrameIndex(
@@ -4700,30 +4708,67 @@ static int CalculateTailCallSPDiff(SelectionDAG& DAG, bool isTailCall,
 
 static bool isFunctionGlobalAddress(SDValue Callee);
 
-static bool
-callsShareTOCBase(const Function *Caller, SDValue Callee,
-                    const TargetMachine &TM) {
-   // Callee is either a GlobalAddress or an ExternalSymbol. ExternalSymbols
-   // don't have enough information to determine if the caller and calle share
-   // the same  TOC base, so we have to pessimistically assume they don't for
-   // correctness.
-   GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);
-   if (!G)
-     return false;
-
-   const GlobalValue *GV = G->getGlobal();
+static bool callsShareTOCBase(const Function *Caller, SDValue Callee,
+                              const TargetMachine &TM) {
+  // It does not make sense to call callsShareTOCBase() with a caller that
+  // is PC Relative since PC Relative callers do not have a TOC.
+#ifndef NDEBUG
+  const PPCSubtarget *STICaller = &TM.getSubtarget<PPCSubtarget>(*Caller);
+  assert(!STICaller->isUsingPCRelativeCalls() &&
+         "PC Relative callers do not have a TOC and cannot share a TOC Base");
+#endif
+
+  // Callee is either a GlobalAddress or an ExternalSymbol. ExternalSymbols
+  // don't have enough information to determine if the caller and callee share
+  // the same  TOC base, so we have to pessimistically assume they don't for
+  // correctness.
+  GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);
+  if (!G)
+    return false;
+
+  const GlobalValue *GV = G->getGlobal();
+
+  // If the callee is preemptable, then the static linker will use a plt-stub
+  // which saves the toc to the stack, and needs a nop after the call
+  // instruction to convert to a toc-restore.
+  if (!TM.shouldAssumeDSOLocal(*Caller->getParent(), GV))
+    return false;
+
+  // Functions with PC Relative enabled may clobber the TOC in the same DSO.
+  // We may need a TOC restore in the situation where the caller requires a
+  // valid TOC but the callee is PC Relative and does not.
+  const Function *F = dyn_cast<Function>(GV);
+  const GlobalAlias *Alias = dyn_cast<GlobalAlias>(GV);
+
+  // If we have an Alias we can try to get the function from there.
+  if (Alias) {
+    const GlobalObject *GlobalObj = Alias->getBaseObject();
+    F = dyn_cast<Function>(GlobalObj);
+  }
+
+  // If we still have no valid function pointer we do not have enough
+  // information to determine if the callee uses PC Relative calls so we must
+  // assume that it does.
+  if (!F)
+    return false;
+
+  // If the callee uses PC Relative we cannot guarantee that the callee won't
+  // clobber the TOC of the caller and so we must assume that the two
+  // functions do not share a TOC base.
+  const PPCSubtarget *STICallee = &TM.getSubtarget<PPCSubtarget>(*F);
+  if (STICallee->isUsingPCRelativeCalls())
+    return false;
+
   // The medium and large code models are expected to provide a sufficiently
   // large TOC to provide all data addressing needs of a module with a
-  // single TOC. Since each module will be addressed with a single TOC then we
-  // only need to check that caller and callee don't cross dso boundaries.
+  // single TOC.
   if (CodeModel::Medium == TM.getCodeModel() ||
       CodeModel::Large == TM.getCodeModel())
-    return TM.shouldAssumeDSOLocal(*Caller->getParent(), GV);
+    return true;
 
   // Otherwise we need to ensure callee and caller are in the same section,
   // since the linker may allocate multiple TOCs, and we don't know which
   // sections will belong to the same TOC base.
-
   if (!GV->isStrongDefinitionForLinker())
     return false;
 
@@ -4738,26 +4783,6 @@ callsShareTOCBase(const Function *Caller, SDValue Callee,
       return false;
   }
 
-  // If the callee might be interposed, then we can't assume the ultimate call
-  // target will be in the same section. Even in cases where we can assume that
-  // interposition won't happen, in any case where the linker might insert a
-  // stub to allow for interposition, we must generate code as though
-  // interposition might occur. To understand why this matters, consider a
-  // situation where: a -> b -> c where the arrows indicate calls. b and c are
-  // in the same section, but a is in a different module (i.e. has a different
-  // TOC base pointer). If the linker allows for interposition between b and c,
-  // then it will generate a stub for the call edge between b and c which will
-  // save the TOC pointer into the designated stack slot allocated by b. If we
-  // return true here, and therefore allow a tail call between b and c, that
-  // stack slot won't exist and the b -> c stub will end up saving b'c TOC base
-  // pointer into the stack slot allocated by a (where the a -> b stub saved
-  // a's TOC base pointer). If we're not considering a tail call, but rather,
-  // whether a nop is needed after the call instruction in b, because the linker
-  // will insert a stub, it might complain about a missing nop if we omit it
-  // (although many don't complain in this case).
-  if (!TM.shouldAssumeDSOLocal(*Caller->getParent(), GV))
-    return false;
-
   return true;
 }
 
@@ -5277,8 +5302,8 @@ static unsigned getCallOpcode(PPCTargetLowering::CallFlags CFlags,
   // will rewrite the nop to be a load of the TOC pointer from the linkage area
   // into gpr2.
   if (Subtarget.isAIXABI() || Subtarget.is64BitELFABI())
-      return callsShareTOCBase(&Caller, Callee, TM) ? PPCISD::CALL
-                                                    : PPCISD::CALL_NOP;
+    return callsShareTOCBase(&Caller, Callee, TM) ? PPCISD::CALL
+                                                  : PPCISD::CALL_NOP;
 
   return PPCISD::CALL;
 }
@@ -5323,7 +5348,7 @@ static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG,
           // MCSectionXCOFF to get the correct storage mapping class.
           // In this case, XCOFF::XMC_PR.
           MCSectionXCOFF *Sec = Context.getXCOFFSection(
-              S->getName(), XCOFF::XMC_PR, XCOFF::XTY_ER, SC,
+              S->getSymbolTableName(), XCOFF::XMC_PR, XCOFF::XTY_ER, SC,
               SectionKind::getMetadata());
           S->setRepresentedCsect(Sec);
         }
@@ -7200,6 +7225,46 @@ static unsigned mapArgRegToOffsetAIX(unsigned Reg, const PPCFrameLowering *FL) {
   llvm_unreachable("Only general purpose registers expected.");
 }
 
+//   AIX ABI Stack Frame Layout:
+//
+//   Low Memory +--------------------------------------------+
+//   SP   +---> | Back chain                                 | ---+
+//        |     +--------------------------------------------+    |   
+//        |     | Saved Condition Register                   |    |
+//        |     +--------------------------------------------+    |
+//        |     | Saved Linkage Register                     |    |
+//        |     +--------------------------------------------+    | Linkage Area
+//        |     | Reserved for compilers                     |    |
+//        |     +--------------------------------------------+    |
+//        |     | Reserved for binders                       |    |
+//        |     +--------------------------------------------+    |
+//        |     | Saved TOC pointer                          | ---+
+//        |     +--------------------------------------------+
+//        |     | Parameter save area                        |
+//        |     +--------------------------------------------+
+//        |     | Alloca space                               |
+//        |     +--------------------------------------------+
+//        |     | Local variable space                       |
+//        |     +--------------------------------------------+
+//        |     | Float/int conversion temporary             |
+//        |     +--------------------------------------------+
+//        |     | Save area for AltiVec registers            |
+//        |     +--------------------------------------------+
+//        |     | AltiVec alignment padding                  |
+//        |     +--------------------------------------------+
+//        |     | Save area for VRSAVE register              |
+//        |     +--------------------------------------------+
+//        |     | Save area for General Purpose registers    |
+//        |     +--------------------------------------------+
+//        |     | Save area for Floating Point registers     |
+//        |     +--------------------------------------------+
+//        +---- | Back chain                                 |
+// High Memory  +--------------------------------------------+
+//
+//  Specifications:
+//  AIX 7.2 Assembler Language Reference
+//  Subroutine linkage convention
+
 SDValue PPCTargetLowering::LowerFormalArguments_AIX(
     SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
@@ -7425,6 +7490,8 @@ SDValue PPCTargetLowering::LowerCall_AIX(
     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
     const CallBase *CB) const {
+  // See PPCTargetLowering::LowerFormalArguments_AIX() for a description of the
+  // AIX ABI stack frame layout.
 
   assert((CFlags.CallConv == CallingConv::C ||
           CFlags.CallConv == CallingConv::Cold ||
@@ -7858,6 +7925,7 @@ PPCTargetLowering::getFramePointerFrameIndex(SelectionDAG & DAG) const {
 
 SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
                                                    SelectionDAG &DAG) const {
+  MachineFunction &MF = DAG.getMachineFunction();
   // Get the inputs.
   SDValue Chain = Op.getOperand(0);
   SDValue Size  = Op.getOperand(1);
@@ -7870,9 +7938,10 @@ SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
                                 DAG.getConstant(0, dl, PtrVT), Size);
   // Construct a node for the frame pointer save index.
   SDValue FPSIdx = getFramePointerFrameIndex(DAG);
-  // Build a DYNALLOC node.
   SDValue Ops[3] = { Chain, NegSize, FPSIdx };
   SDVTList VTs = DAG.getVTList(PtrVT, MVT::Other);
+  if (hasInlineStackProbe(MF))
+    return DAG.getNode(PPCISD::PROBED_ALLOCA, dl, VTs, Ops);
   return DAG.getNode(PPCISD::DYNALLOC, dl, VTs, Ops);
 }
 
@@ -8236,7 +8305,7 @@ SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
                                           const SDLoc &dl) const {
 
   // FP to INT conversions are legal for f128.
-  if (EnableQuadPrecision && (Op->getOperand(0).getValueType() == MVT::f128))
+  if (Op->getOperand(0).getValueType() == MVT::f128)
     return Op;
 
   // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on
@@ -8504,7 +8573,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
     return LowerINT_TO_FPVector(Op, DAG, dl);
 
   // Conversions to f128 are legal.
-  if (EnableQuadPrecision && (Op.getValueType() == MVT::f128))
+  if (Op.getValueType() == MVT::f128)
     return Op;
 
   if (Subtarget.hasQPX() && Op.getOperand(0).getValueType() == MVT::v4i1) {
@@ -8643,7 +8712,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
       MachineFrameInfo &MFI = MF.getFrameInfo();
       EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
-      int FrameIdx = MFI.CreateStackObject(4, 4, false);
+      int FrameIdx = MFI.CreateStackObject(4, Align(4), false);
       SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
 
       SDValue Store =
@@ -8695,7 +8764,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
     bool ReusingLoad;
     if (!(ReusingLoad = canReuseLoadAddress(Op.getOperand(0), MVT::i32, RLI,
                                             DAG))) {
-      int FrameIdx = MFI.CreateStackObject(4, 4, false);
+      int FrameIdx = MFI.CreateStackObject(4, Align(4), false);
       SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
 
       SDValue Store =
@@ -8727,7 +8796,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
     assert(Subtarget.isPPC64() &&
            "i32->FP without LFIWAX supported only on PPC64");
 
-    int FrameIdx = MFI.CreateStackObject(8, 8, false);
+    int FrameIdx = MFI.CreateStackObject(8, Align(8), false);
     SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
 
     SDValue Ext64 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i64,
@@ -8784,7 +8853,7 @@ SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
   Chain = MFFS.getValue(1);
 
   // Save FP register to stack slot
-  int SSFI = MF.getFrameInfo().CreateStackObject(8, 8, false);
+  int SSFI = MF.getFrameInfo().CreateStackObject(8, Align(8), false);
   SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
   Chain = DAG.getStore(Chain, dl, MFFS, StackSlot, MachinePointerInfo());
 
@@ -8907,19 +8976,21 @@ SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const {
 // Vector related lowering.
 //
 
-/// BuildSplatI - Build a canonical splati of Val with an element size of
-/// SplatSize.  Cast the result to VT.
-static SDValue BuildSplatI(int Val, unsigned SplatSize, EVT VT,
-                           SelectionDAG &DAG, const SDLoc &dl) {
+/// getCanonicalConstSplat - Build a canonical splat immediate of Val with an
+/// element size of SplatSize. Cast the result to VT.
+static SDValue getCanonicalConstSplat(uint64_t Val, unsigned SplatSize, EVT VT,
+                                      SelectionDAG &DAG, const SDLoc &dl) {
   static const MVT VTys[] = { // canonical VT to use for each size.
     MVT::v16i8, MVT::v8i16, MVT::Other, MVT::v4i32
   };
 
   EVT ReqVT = VT != MVT::Other ? VT : VTys[SplatSize-1];
 
-  // Force vspltis[hw] -1 to vspltisb -1 to canonicalize.
-  if (Val == -1)
+  // For a splat with all ones, turn it to vspltisb 0xFF to canonicalize.
+  if (Val == ((1LU << (SplatSize * 8)) - 1)) {
     SplatSize = 1;
+    Val = 0xFF;
+  }
 
   EVT CanonicalVT = VTys[SplatSize-1];
 
@@ -9030,10 +9101,9 @@ SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
   SDLoc dl(Op);
   SDValue Op0 = Op->getOperand(0);
 
-  if (!EnableQuadPrecision ||
-      (Op.getValueType() != MVT::f128 ) ||
+  if ((Op.getValueType() != MVT::f128) ||
       (Op0.getOpcode() != ISD::BUILD_PAIR) ||
-      (Op0.getOperand(0).getValueType() !=  MVT::i64) ||
+      (Op0.getOperand(0).getValueType() != MVT::i64) ||
       (Op0.getOperand(1).getValueType() != MVT::i64))
     return SDValue();
 
@@ -9054,6 +9124,34 @@ static const SDValue *getNormalLoadInput(const SDValue &Op) {
   return ISD::isNormalLoad(LD) ? InputLoad : nullptr;
 }
 
+// Convert the argument APFloat to a single precision APFloat if there is no
+// loss in information during the conversion to single precision APFloat and the
+// resulting number is not a denormal number. Return true if successful.
+bool llvm::convertToNonDenormSingle(APFloat &ArgAPFloat) {
+  APFloat APFloatToConvert = ArgAPFloat;
+  bool LosesInfo = true;
+  APFloatToConvert.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven,
+                           &LosesInfo);
+  bool Success = (!LosesInfo && !APFloatToConvert.isDenormal());
+  if (Success)
+    ArgAPFloat = APFloatToConvert;
+  return Success;
+}
+
+// Bitcast the argument APInt to a double and convert it to a single precision
+// APFloat, bitcast the APFloat to an APInt and assign it to the original
+// argument if there is no loss in information during the conversion from
+// double to single precision APFloat and the resulting number is not a denormal
+// number. Return true if successful.
+bool llvm::convertToNonDenormSingle(APInt &ArgAPInt) {
+  double DpValue = ArgAPInt.bitsToDouble();
+  APFloat APFloatDp(DpValue);
+  bool Success = convertToNonDenormSingle(APFloatDp);
+  if (Success)
+    ArgAPInt = APFloatDp.bitcastToAPInt();
+  return Success;
+}
+
 // If this is a case we can't handle, return null and let the default
 // expansion code take care of it.  If we CAN select this case, and if it
 // selects to a single instruction, return Op.  Otherwise, if we can codegen
@@ -9070,7 +9168,7 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     // then convert it to a floating-point vector and compare it
     // to a zero vector to get the boolean result.
     MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
-    int FrameIdx = MFI.CreateStackObject(16, 16, false);
+    int FrameIdx = MFI.CreateStackObject(16, Align(16), false);
     MachinePointerInfo PtrInfo =
         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
     EVT PtrVT = getPointerTy(DAG.getDataLayout());
@@ -9173,9 +9271,23 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
   APInt APSplatBits, APSplatUndef;
   unsigned SplatBitSize;
   bool HasAnyUndefs;
-  if (! BVN->isConstantSplat(APSplatBits, APSplatUndef, SplatBitSize,
-                             HasAnyUndefs, 0, !Subtarget.isLittleEndian()) ||
-      SplatBitSize > 32) {
+  bool BVNIsConstantSplat =
+      BVN->isConstantSplat(APSplatBits, APSplatUndef, SplatBitSize,
+                           HasAnyUndefs, 0, !Subtarget.isLittleEndian());
+
+  // If it is a splat of a double, check if we can shrink it to a 32 bit
+  // non-denormal float which when converted back to double gives us the same
+  // double. This is to exploit the XXSPLTIDP instruction.
+  if (BVNIsConstantSplat && Subtarget.hasPrefixInstrs() &&
+      (SplatBitSize == 64) && (Op->getValueType(0) == MVT::v2f64) &&
+      convertToNonDenormSingle(APSplatBits)) {
+    SDValue SplatNode = DAG.getNode(
+        PPCISD::XXSPLTI_SP_TO_DP, dl, MVT::v2f64,
+        DAG.getTargetConstant(APSplatBits.getZExtValue(), dl, MVT::i32));
+    return DAG.getBitcast(Op.getValueType(), SplatNode);
+  }
+
+  if (!BVNIsConstantSplat || SplatBitSize > 32) {
 
     const SDValue *InputLoad = getNormalLoadInput(Op.getOperand(0));
     // Handle load-and-splat patterns as we have instructions that will do this
@@ -9214,8 +9326,8 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     return SDValue();
   }
 
-  unsigned SplatBits = APSplatBits.getZExtValue();
-  unsigned SplatUndef = APSplatUndef.getZExtValue();
+  uint64_t SplatBits = APSplatBits.getZExtValue();
+  uint64_t SplatUndef = APSplatUndef.getZExtValue();
   unsigned SplatSize = SplatBitSize / 8;
 
   // First, handle single instruction cases.
@@ -9230,17 +9342,30 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     return Op;
   }
 
-  // We have XXSPLTIB for constant splats one byte wide
-  // FIXME: SplatBits is an unsigned int being cast to an int while passing it
-  // as an argument to BuildSplatiI. Given SplatSize == 1 it is okay here.
+  // We have XXSPLTIW for constant splats four bytes wide.
+  // Given vector length is a multiple of 4, 2-byte splats can be replaced
+  // with 4-byte splats. We replicate the SplatBits in case of 2-byte splat to
+  // make a 4-byte splat element. For example: 2-byte splat of 0xABAB can be
+  // turned into a 4-byte splat of 0xABABABAB.
+  if (Subtarget.hasPrefixInstrs() && SplatSize == 2)
+    return getCanonicalConstSplat((SplatBits |= SplatBits << 16), SplatSize * 2,
+                                  Op.getValueType(), DAG, dl);
+
+  if (Subtarget.hasPrefixInstrs() && SplatSize == 4)
+    return getCanonicalConstSplat(SplatBits, SplatSize, Op.getValueType(), DAG,
+                                  dl);
+
+  // We have XXSPLTIB for constant splats one byte wide.
   if (Subtarget.hasP9Vector() && SplatSize == 1)
-    return BuildSplatI(SplatBits, SplatSize, Op.getValueType(), DAG, dl);
+    return getCanonicalConstSplat(SplatBits, SplatSize, Op.getValueType(), DAG,
+                                  dl);
 
   // If the sign extended value is in the range [-16,15], use VSPLTI[bhw].
   int32_t SextVal= (int32_t(SplatBits << (32-SplatBitSize)) >>
                     (32-SplatBitSize));
   if (SextVal >= -16 && SextVal <= 15)
-    return BuildSplatI(SextVal, SplatSize, Op.getValueType(), DAG, dl);
+    return getCanonicalConstSplat(SextVal, SplatSize, Op.getValueType(), DAG,
+                                  dl);
 
   // Two instruction sequences.
 
@@ -9271,7 +9396,7 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
   // for fneg/fabs.
   if (SplatSize == 4 && SplatBits == (0x7FFFFFFF&~SplatUndef)) {
     // Make -1 and vspltisw -1:
-    SDValue OnesV = BuildSplatI(-1, 4, MVT::v4i32, DAG, dl);
+    SDValue OnesV = getCanonicalConstSplat(-1, 4, MVT::v4i32, DAG, dl);
 
     // Make the VSLW intrinsic, computing 0x8000_0000.
     SDValue Res = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, OnesV,
@@ -9299,7 +9424,7 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
 
     // vsplti + shl self.
     if (SextVal == (int)((unsigned)i << TypeShiftAmt)) {
-      SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl);
+      SDValue Res = getCanonicalConstSplat(i, SplatSize, MVT::Other, DAG, dl);
       static const unsigned IIDs[] = { // Intrinsic to use for each size.
         Intrinsic::ppc_altivec_vslb, Intrinsic::ppc_altivec_vslh, 0,
         Intrinsic::ppc_altivec_vslw
@@ -9310,7 +9435,7 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
 
     // vsplti + srl self.
     if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) {
-      SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl);
+      SDValue Res = getCanonicalConstSplat(i, SplatSize, MVT::Other, DAG, dl);
       static const unsigned IIDs[] = { // Intrinsic to use for each size.
         Intrinsic::ppc_altivec_vsrb, Intrinsic::ppc_altivec_vsrh, 0,
         Intrinsic::ppc_altivec_vsrw
@@ -9321,7 +9446,7 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
 
     // vsplti + sra self.
     if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) {
-      SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl);
+      SDValue Res = getCanonicalConstSplat(i, SplatSize, MVT::Other, DAG, dl);
       static const unsigned IIDs[] = { // Intrinsic to use for each size.
         Intrinsic::ppc_altivec_vsrab, Intrinsic::ppc_altivec_vsrah, 0,
         Intrinsic::ppc_altivec_vsraw
@@ -9333,7 +9458,7 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     // vsplti + rol self.
     if (SextVal == (int)(((unsigned)i << TypeShiftAmt) |
                          ((unsigned)i >> (SplatBitSize-TypeShiftAmt)))) {
-      SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl);
+      SDValue Res = getCanonicalConstSplat(i, SplatSize, MVT::Other, DAG, dl);
       static const unsigned IIDs[] = { // Intrinsic to use for each size.
         Intrinsic::ppc_altivec_vrlb, Intrinsic::ppc_altivec_vrlh, 0,
         Intrinsic::ppc_altivec_vrlw
@@ -9344,19 +9469,19 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
 
     // t = vsplti c, result = vsldoi t, t, 1
     if (SextVal == (int)(((unsigned)i << 8) | (i < 0 ? 0xFF : 0))) {
-      SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl);
+      SDValue T = getCanonicalConstSplat(i, SplatSize, MVT::v16i8, DAG, dl);
       unsigned Amt = Subtarget.isLittleEndian() ? 15 : 1;
       return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl);
     }
     // t = vsplti c, result = vsldoi t, t, 2
     if (SextVal == (int)(((unsigned)i << 16) | (i < 0 ? 0xFFFF : 0))) {
-      SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl);
+      SDValue T = getCanonicalConstSplat(i, SplatSize, MVT::v16i8, DAG, dl);
       unsigned Amt = Subtarget.isLittleEndian() ? 14 : 2;
       return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl);
     }
     // t = vsplti c, result = vsldoi t, t, 3
     if (SextVal == (int)(((unsigned)i << 24) | (i < 0 ? 0xFFFFFF : 0))) {
-      SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl);
+      SDValue T = getCanonicalConstSplat(i, SplatSize, MVT::v16i8, DAG, dl);
       unsigned Amt = Subtarget.isLittleEndian() ? 13 : 3;
       return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl);
     }
@@ -9655,6 +9780,77 @@ SDValue PPCTargetLowering::lowerToVINSERTH(ShuffleVectorSDNode *N,
   return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins);
 }
 
+/// lowerToXXSPLTI32DX - Return the SDValue if this VECTOR_SHUFFLE can be
+/// handled by the XXSPLTI32DX instruction introduced in ISA 3.1, otherwise
+/// return the default SDValue.
+SDValue PPCTargetLowering::lowerToXXSPLTI32DX(ShuffleVectorSDNode *SVN,
+                                              SelectionDAG &DAG) const {
+  // The LHS and RHS may be bitcasts to v16i8 as we canonicalize shuffles
+  // to v16i8. Peek through the bitcasts to get the actual operands.
+  SDValue LHS = peekThroughBitcasts(SVN->getOperand(0));
+  SDValue RHS = peekThroughBitcasts(SVN->getOperand(1));
+
+  auto ShuffleMask = SVN->getMask();
+  SDValue VecShuffle(SVN, 0);
+  SDLoc DL(SVN);
+
+  // Check that we have a four byte shuffle.
+  if (!isNByteElemShuffleMask(SVN, 4, 1))
+    return SDValue();
+
+  // Canonicalize the RHS being a BUILD_VECTOR when lowering to xxsplti32dx.
+  if (RHS->getOpcode() != ISD::BUILD_VECTOR) {
+    std::swap(LHS, RHS);
+    VecShuffle = DAG.getCommutedVectorShuffle(*SVN);
+    ShuffleMask = cast<ShuffleVectorSDNode>(VecShuffle)->getMask();
+  }
+
+  // Ensure that the RHS is a vector of constants.
+  BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(RHS.getNode());
+  if (!BVN)
+    return SDValue();
+
+  // Check if RHS is a splat of 4-bytes (or smaller).
+  APInt APSplatValue, APSplatUndef;
+  unsigned SplatBitSize;
+  bool HasAnyUndefs;
+  if (!BVN->isConstantSplat(APSplatValue, APSplatUndef, SplatBitSize,
+                            HasAnyUndefs, 0, !Subtarget.isLittleEndian()) ||
+      SplatBitSize > 32)
+    return SDValue();
+
+  // Check that the shuffle mask matches the semantics of XXSPLTI32DX.
+  // The instruction splats a constant C into two words of the source vector
+  // producing { C, Unchanged, C, Unchanged } or { Unchanged, C, Unchanged, C }.
+  // Thus we check that the shuffle mask is the equivalent  of
+  // <0, [4-7], 2, [4-7]> or <[4-7], 1, [4-7], 3> respectively.
+  // Note: the check above of isNByteElemShuffleMask() ensures that the bytes
+  // within each word are consecutive, so we only need to check the first byte.
+  SDValue Index;
+  bool IsLE = Subtarget.isLittleEndian();
+  if ((ShuffleMask[0] == 0 && ShuffleMask[8] == 8) &&
+      (ShuffleMask[4] % 4 == 0 && ShuffleMask[12] % 4 == 0 &&
+       ShuffleMask[4] > 15 && ShuffleMask[12] > 15))
+    Index = DAG.getTargetConstant(IsLE ? 0 : 1, DL, MVT::i32);
+  else if ((ShuffleMask[4] == 4 && ShuffleMask[12] == 12) &&
+           (ShuffleMask[0] % 4 == 0 && ShuffleMask[8] % 4 == 0 &&
+            ShuffleMask[0] > 15 && ShuffleMask[8] > 15))
+    Index = DAG.getTargetConstant(IsLE ? 1 : 0, DL, MVT::i32);
+  else
+    return SDValue();
+
+  // If the splat is narrower than 32-bits, we need to get the 32-bit value
+  // for XXSPLTI32DX.
+  unsigned SplatVal = APSplatValue.getZExtValue();
+  for (; SplatBitSize < 32; SplatBitSize <<= 1)
+    SplatVal |= (SplatVal << SplatBitSize);
+
+  SDValue SplatNode = DAG.getNode(
+      PPCISD::XXSPLTI32DX, DL, MVT::v2i64, DAG.getBitcast(MVT::v2i64, LHS),
+      Index, DAG.getTargetConstant(SplatVal, DL, MVT::i32));
+  return DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, SplatNode);
+}
+
 /// LowerROTL - Custom lowering for ROTL(v1i128) to vector_shuffle(v16i8).
 /// We lower ROTL(v1i128) to vector_shuffle(v16i8) only if shift amount is
 /// a multiple of 8. Otherwise convert it to a scalar rotation(i128)
@@ -9700,9 +9896,10 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
   // to vector legalization will not be sent to the target combine. Try to
   // combine it here.
   if (SDValue NewShuffle = combineVectorShuffle(SVOp, DAG)) {
-    DAG.ReplaceAllUsesOfValueWith(Op, NewShuffle);
     Op = NewShuffle;
     SVOp = cast<ShuffleVectorSDNode>(Op);
+    V1 = Op.getOperand(0);
+    V2 = Op.getOperand(1);
   }
   EVT VT = Op.getValueType();
   bool isLittleEndian = Subtarget.isLittleEndian();
@@ -9772,6 +9969,12 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
     return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins);
   }
 
+  if (Subtarget.hasPrefixInstrs()) {
+    SDValue SplatInsertNode;
+    if ((SplatInsertNode = lowerToXXSPLTI32DX(SVOp, DAG)))
+      return SplatInsertNode;
+  }
+
   if (Subtarget.hasP9Altivec()) {
     SDValue NewISDNode;
     if ((NewISDNode = lowerToVINSERTH(SVOp, DAG)))
@@ -10370,18 +10573,6 @@ SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
   return SDValue();
 }
 
-SDValue PPCTargetLowering::LowerREM(SDValue Op, SelectionDAG &DAG) const {
-  // Check for a DIV with the same operands as this REM.
-  for (auto UI : Op.getOperand(1)->uses()) {
-    if ((Op.getOpcode() == ISD::SREM && UI->getOpcode() == ISD::SDIV) ||
-        (Op.getOpcode() == ISD::UREM && UI->getOpcode() == ISD::UDIV))
-      if (UI->getOperand(0) == Op.getOperand(0) &&
-          UI->getOperand(1) == Op.getOperand(1))
-        return SDValue();
-  }
-  return Op;
-}
-
 // Lower scalar BSWAP64 to xxbrd.
 SDValue PPCTargetLowering::LowerBSWAP(SDValue Op, SelectionDAG &DAG) const {
   SDLoc dl(Op);
@@ -10440,7 +10631,7 @@ SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
   SDLoc dl(Op);
   // Create a stack slot that is 16-byte aligned.
   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
-  int FrameIdx = MFI.CreateStackObject(16, 16, false);
+  int FrameIdx = MFI.CreateStackObject(16, Align(16), false);
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
 
@@ -10510,7 +10701,7 @@ SDValue PPCTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
     Value);
 
   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
-  int FrameIdx = MFI.CreateStackObject(16, 16, false);
+  int FrameIdx = MFI.CreateStackObject(16, Align(16), false);
   MachinePointerInfo PtrInfo =
       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
@@ -10709,7 +10900,7 @@ SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
     Value);
 
   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
-  int FrameIdx = MFI.CreateStackObject(16, 16, false);
+  int FrameIdx = MFI.CreateStackObject(16, Align(16), false);
   MachinePointerInfo PtrInfo =
       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
@@ -10758,9 +10949,9 @@ SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
   if (Op.getValueType() == MVT::v4i32) {
     SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1);
 
-    SDValue Zero  = BuildSplatI(  0, 1, MVT::v4i32, DAG, dl);
-    SDValue Neg16 = BuildSplatI(-16, 4, MVT::v4i32, DAG, dl);//+16 as shift amt.
-
+    SDValue Zero = getCanonicalConstSplat(0, 1, MVT::v4i32, DAG, dl);
+    // +16 as shift amt.
+    SDValue Neg16 = getCanonicalConstSplat(-16, 4, MVT::v4i32, DAG, dl);
     SDValue RHSSwap =   // = vrlw RHS, 16
       BuildIntrinsicOp(Intrinsic::ppc_altivec_vrlw, RHS, Neg16, DAG, dl);
 
@@ -10999,9 +11190,6 @@ SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
 
   case ISD::INTRINSIC_VOID:
     return LowerINTRINSIC_VOID(Op, DAG);
-  case ISD::SREM:
-  case ISD::UREM:
-    return LowerREM(Op, DAG);
   case ISD::BSWAP:
     return LowerBSWAP(Op, DAG);
   case ISD::ATOMIC_CMP_SWAP:
@@ -11681,6 +11869,184 @@ PPCTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
   return MBB;
 }
 
+bool PPCTargetLowering::hasInlineStackProbe(MachineFunction &MF) const {
+  // If the function specifically requests inline stack probes, emit them.
+  if (MF.getFunction().hasFnAttribute("probe-stack"))
+    return MF.getFunction().getFnAttribute("probe-stack").getValueAsString() ==
+           "inline-asm";
+  return false;
+}
+
+unsigned PPCTargetLowering::getStackProbeSize(MachineFunction &MF) const {
+  const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
+  unsigned StackAlign = TFI->getStackAlignment();
+  assert(StackAlign >= 1 && isPowerOf2_32(StackAlign) &&
+         "Unexpected stack alignment");
+  // The default stack probe size is 4096 if the function has no
+  // stack-probe-size attribute.
+  unsigned StackProbeSize = 4096;
+  const Function &Fn = MF.getFunction();
+  if (Fn.hasFnAttribute("stack-probe-size"))
+    Fn.getFnAttribute("stack-probe-size")
+        .getValueAsString()
+        .getAsInteger(0, StackProbeSize);
+  // Round down to the stack alignment.
+  StackProbeSize &= ~(StackAlign - 1);
+  return StackProbeSize ? StackProbeSize : StackAlign;
+}
+
+// Lower dynamic stack allocation with probing. `emitProbedAlloca` is splitted
+// into three phases. In the first phase, it uses pseudo instruction
+// PREPARE_PROBED_ALLOCA to get the future result of actual FramePointer and
+// FinalStackPtr. In the second phase, it generates a loop for probing blocks.
+// At last, it uses pseudo instruction DYNAREAOFFSET to get the future result of
+// MaxCallFrameSize so that it can calculate correct data area pointer.
+MachineBasicBlock *
+PPCTargetLowering::emitProbedAlloca(MachineInstr &MI,
+                                    MachineBasicBlock *MBB) const {
+  const bool isPPC64 = Subtarget.isPPC64();
+  MachineFunction *MF = MBB->getParent();
+  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
+  DebugLoc DL = MI.getDebugLoc();
+  const unsigned ProbeSize = getStackProbeSize(*MF);
+  const BasicBlock *ProbedBB = MBB->getBasicBlock();
+  MachineRegisterInfo &MRI = MF->getRegInfo();
+  // The CFG of probing stack looks as
+  //         +-----+
+  //         | MBB |
+  //         +--+--+
+  //            |
+  //       +----v----+
+  //  +--->+ TestMBB +---+
+  //  |    +----+----+   |
+  //  |         |        |
+  //  |   +-----v----+   |
+  //  +---+ BlockMBB |   |
+  //      +----------+   |
+  //                     |
+  //       +---------+   |
+  //       | TailMBB +<--+
+  //       +---------+
+  // In MBB, calculate previous frame pointer and final stack pointer.
+  // In TestMBB, test if sp is equal to final stack pointer, if so, jump to
+  // TailMBB. In BlockMBB, update the sp atomically and jump back to TestMBB.
+  // TailMBB is spliced via \p MI.
+  MachineBasicBlock *TestMBB = MF->CreateMachineBasicBlock(ProbedBB);
+  MachineBasicBlock *TailMBB = MF->CreateMachineBasicBlock(ProbedBB);
+  MachineBasicBlock *BlockMBB = MF->CreateMachineBasicBlock(ProbedBB);
+
+  MachineFunction::iterator MBBIter = ++MBB->getIterator();
+  MF->insert(MBBIter, TestMBB);
+  MF->insert(MBBIter, BlockMBB);
+  MF->insert(MBBIter, TailMBB);
+
+  const TargetRegisterClass *G8RC = &PPC::G8RCRegClass;
+  const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
+
+  Register DstReg = MI.getOperand(0).getReg();
+  Register NegSizeReg = MI.getOperand(1).getReg();
+  Register SPReg = isPPC64 ? PPC::X1 : PPC::R1;
+  Register FinalStackPtr = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
+  Register FramePointer = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
+
+  // Get the canonical FinalStackPtr like what
+  // PPCRegisterInfo::lowerDynamicAlloc does.
+  BuildMI(*MBB, {MI}, DL,
+          TII->get(isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_64
+                           : PPC::PREPARE_PROBED_ALLOCA_32),
+          FramePointer)
+      .addDef(FinalStackPtr)
+      .addReg(NegSizeReg)
+      .add(MI.getOperand(2))
+      .add(MI.getOperand(3));
+
+  // Materialize a scratch register for update.
+  int64_t NegProbeSize = -(int64_t)ProbeSize;
+  assert(isInt<32>(NegProbeSize) && "Unhandled probe size!");
+  Register ScratchReg = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
+  if (!isInt<16>(NegProbeSize)) {
+    Register TempReg = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
+    BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::LIS8 : PPC::LIS), TempReg)
+        .addImm(NegProbeSize >> 16);
+    BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::ORI8 : PPC::ORI),
+            ScratchReg)
+        .addReg(TempReg)
+        .addImm(NegProbeSize & 0xFFFF);
+  } else
+    BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::LI8 : PPC::LI), ScratchReg)
+        .addImm(NegProbeSize);
+
+  {
+    // Probing leading residual part.
+    Register Div = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
+    BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::DIVD : PPC::DIVW), Div)
+        .addReg(NegSizeReg)
+        .addReg(ScratchReg);
+    Register Mul = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
+    BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::MULLD : PPC::MULLW), Mul)
+        .addReg(Div)
+        .addReg(ScratchReg);
+    Register NegMod = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
+    BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::SUBF8 : PPC::SUBF), NegMod)
+        .addReg(Mul)
+        .addReg(NegSizeReg);
+    BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::STDUX : PPC::STWUX), SPReg)
+        .addReg(FramePointer)
+        .addReg(SPReg)
+        .addReg(NegMod);
+  }
+
+  {
+    // Remaining part should be multiple of ProbeSize.
+    Register CmpResult = MRI.createVirtualRegister(&PPC::CRRCRegClass);
+    BuildMI(TestMBB, DL, TII->get(isPPC64 ? PPC::CMPD : PPC::CMPW), CmpResult)
+        .addReg(SPReg)
+        .addReg(FinalStackPtr);
+    BuildMI(TestMBB, DL, TII->get(PPC::BCC))
+        .addImm(PPC::PRED_EQ)
+        .addReg(CmpResult)
+        .addMBB(TailMBB);
+    TestMBB->addSuccessor(BlockMBB);
+    TestMBB->addSuccessor(TailMBB);
+  }
+
+  {
+    // Touch the block.
+    // |P...|P...|P...
+    BuildMI(BlockMBB, DL, TII->get(isPPC64 ? PPC::STDUX : PPC::STWUX), SPReg)
+        .addReg(FramePointer)
+        .addReg(SPReg)
+        .addReg(ScratchReg);
+    BuildMI(BlockMBB, DL, TII->get(PPC::B)).addMBB(TestMBB);
+    BlockMBB->addSuccessor(TestMBB);
+  }
+
+  // Calculation of MaxCallFrameSize is deferred to prologepilog, use
+  // DYNAREAOFFSET pseudo instruction to get the future result.
+  Register MaxCallFrameSizeReg =
+      MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
+  BuildMI(TailMBB, DL,
+          TII->get(isPPC64 ? PPC::DYNAREAOFFSET8 : PPC::DYNAREAOFFSET),
+          MaxCallFrameSizeReg)
+      .add(MI.getOperand(2))
+      .add(MI.getOperand(3));
+  BuildMI(TailMBB, DL, TII->get(isPPC64 ? PPC::ADD8 : PPC::ADD4), DstReg)
+      .addReg(SPReg)
+      .addReg(MaxCallFrameSizeReg);
+
+  // Splice instructions after MI to TailMBB.
+  TailMBB->splice(TailMBB->end(), MBB,
+                  std::next(MachineBasicBlock::iterator(MI)), MBB->end());
+  TailMBB->transferSuccessorsAndUpdatePHIs(MBB);
+  MBB->addSuccessor(TestMBB);
+
+  // Delete the pseudo instruction.
+  MI.eraseFromParent();
+
+  ++NumDynamicAllocaProbed;
+  return TailMBB;
+}
+
 MachineBasicBlock *
 PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
                                                MachineBasicBlock *BB) const {
@@ -12370,7 +12736,7 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
         }
 
         MachineFrameInfo &MFI = F->getFrameInfo();
-        int FrameIdx = MFI.CreateStackObject(8, 8, false);
+        int FrameIdx = MFI.CreateStackObject(8, Align(8), false);
 
         MachineMemOperand *MMOStore = F->getMachineMemOperand(
             MachinePointerInfo::getFixedStack(*F, FrameIdx, 0),
@@ -12447,6 +12813,9 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
       .addReg(NewFPSCRReg)
       .addImm(0)
       .addImm(0);
+  } else if (MI.getOpcode() == PPC::PROBED_ALLOCA_32 ||
+             MI.getOpcode() == PPC::PROBED_ALLOCA_64) {
+    return emitProbedAlloca(MI, BB);
   } else {
     llvm_unreachable("Unexpected instr type to insert");
   }
@@ -14241,7 +14610,7 @@ SDValue PPCTargetLowering::combineVectorShuffle(ShuffleVectorSDNode *SVN,
 
   // None of these combines are useful on big endian systems since the ISA
   // already has a big endian bias.
-  if (!Subtarget.isLittleEndian())
+  if (!Subtarget.isLittleEndian() || !Subtarget.hasVSX())
     return Res;
 
   // If this is not a shuffle of a shuffle and the first element comes from
@@ -14323,10 +14692,21 @@ SDValue PPCTargetLowering::combineVectorShuffle(ShuffleVectorSDNode *SVN,
 
   // Adjust the mask so we are pulling in the same index from the splat
   // as the index from the interesting vector in consecutive elements.
-  // Example:
+  // Example (even elements from first vector):
   // vector_shuffle<0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23> t1, <zero>
-  for (int i = 1, e = Mask.size(); i < e; i += 2)
-    ShuffV[i] = (ShuffV[i - 1] + NumElts);
+  if (Mask[0] < NumElts)
+    for (int i = 1, e = Mask.size(); i < e; i += 2)
+      ShuffV[i] = (ShuffV[i - 1] + NumElts);
+  // Example (odd elements from first vector):
+  // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7> t1, <zero>
+  else
+    for (int i = 0, e = Mask.size(); i < e; i += 2)
+      ShuffV[i] = (ShuffV[i + 1] + NumElts);
+
+  // If the RHS has undefs, we need to remove them since we may have created
+  // a shuffle that adds those instead of the splat value.
+  SDValue SplatVal = cast<BuildVectorSDNode>(RHS.getNode())->getSplatValue();
+  RHS = DAG.getSplatBuildVector(RHS.getValueType(), dl, SplatVal);
 
   Res = DAG.getVectorShuffle(SVN->getValueType(0), dl, LHS, RHS, ShuffV);
   return Res;
@@ -14648,17 +15028,18 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
 
     EVT MemVT = LD->getMemoryVT();
     Type *Ty = MemVT.getTypeForEVT(*DAG.getContext());
-    unsigned ABIAlignment = DAG.getDataLayout().getABITypeAlignment(Ty);
+    Align ABIAlignment = DAG.getDataLayout().getABITypeAlign(Ty);
     Type *STy = MemVT.getScalarType().getTypeForEVT(*DAG.getContext());
-    unsigned ScalarABIAlignment = DAG.getDataLayout().getABITypeAlignment(STy);
+    Align ScalarABIAlignment = DAG.getDataLayout().getABITypeAlign(STy);
     if (LD->isUnindexed() && VT.isVector() &&
         ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) &&
           // P8 and later hardware should just use LOAD.
-          !Subtarget.hasP8Vector() && (VT == MVT::v16i8 || VT == MVT::v8i16 ||
-                                       VT == MVT::v4i32 || VT == MVT::v4f32)) ||
+          !Subtarget.hasP8Vector() &&
+          (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
+           VT == MVT::v4f32)) ||
          (Subtarget.hasQPX() && (VT == MVT::v4f64 || VT == MVT::v4f32) &&
-          LD->getAlignment() >= ScalarABIAlignment)) &&
-        LD->getAlignment() < ABIAlignment) {
+          LD->getAlign() >= ScalarABIAlignment)) &&
+        LD->getAlign() < ABIAlignment) {
       // This is a type-legal unaligned Altivec or QPX load.
       SDValue Chain = LD->getChain();
       SDValue Ptr = LD->getBasePtr();
@@ -15988,7 +16369,7 @@ bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
   case Type::DoubleTyID:
     return true;
   case Type::FP128TyID:
-    return EnableQuadPrecision && Subtarget.hasP9Vector();
+    return Subtarget.hasP9Vector();
   default:
     return false;
   }
@@ -16174,6 +16555,13 @@ bool PPCTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
     return false;
   case MVT::f32:
   case MVT::f64:
+    if (Subtarget.hasPrefixInstrs()) {
+      // With prefixed instructions, we can materialize anything that can be
+      // represented with a 32-bit immediate, not just positive zero.
+      APFloat APFloatOfImm = Imm;
+      return convertToNonDenormSingle(APFloatOfImm);
+    }
+    LLVM_FALLTHROUGH;
   case MVT::ppcf128:
     return Imm.isPosZero();
   }
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 9f7c6ab53a17e..768eaa43e0135 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -97,6 +97,15 @@ namespace llvm {
     ///
     XXSPLT,
 
+    /// XXSPLTI_SP_TO_DP - The PPC VSX splat instructions for immediates for
+    /// converting immediate single precision numbers to double precision
+    /// vector or scalar.
+    XXSPLTI_SP_TO_DP,
+
+    /// XXSPLTI32DX - The PPC XXSPLTI32DX instruction.
+    ///
+    XXSPLTI32DX,
+
     /// VECINSERT - The PPC vector insert instruction
     ///
     VECINSERT,
@@ -133,6 +142,10 @@ namespace llvm {
     /// dynamic alloca.
     DYNAREAOFFSET,
 
+    /// To avoid stack clash, allocation is performed by block and each block is
+    /// probed.
+    PROBED_ALLOCA,
+
     /// GlobalBaseReg - On Darwin, this node represents the result of the mflr
     /// at function entry, used for PIC code.
     GlobalBaseReg,
@@ -717,7 +730,7 @@ namespace llvm {
     /// Returns false if it can be represented by [r+imm], which are preferred.
     bool SelectAddressRegReg(SDValue N, SDValue &Base, SDValue &Index,
                              SelectionDAG &DAG,
-                             unsigned EncodingAlignment = 0) const;
+                             MaybeAlign EncodingAlignment = None) const;
 
     /// SelectAddressRegImm - Returns true if the address N can be represented
     /// by a base register plus a signed 16-bit displacement [r+imm], and if it
@@ -726,7 +739,7 @@ namespace llvm {
     /// requirement, i.e. multiples of 4 for DS form.
     bool SelectAddressRegImm(SDValue N, SDValue &Disp, SDValue &Base,
                              SelectionDAG &DAG,
-                             unsigned EncodingAlignment) const;
+                             MaybeAlign EncodingAlignment) const;
 
     /// SelectAddressRegRegOnly - Given the specified addressed, force it to be
     /// represented as an indexed [r+r] operation.
@@ -799,6 +812,13 @@ namespace llvm {
     MachineBasicBlock *emitEHSjLjLongJmp(MachineInstr &MI,
                                          MachineBasicBlock *MBB) const;
 
+    MachineBasicBlock *emitProbedAlloca(MachineInstr &MI,
+                                        MachineBasicBlock *MBB) const;
+
+    bool hasInlineStackProbe(MachineFunction &MF) const override;
+
+    unsigned getStackProbeSize(MachineFunction &MF) const;
+
     ConstraintType getConstraintType(StringRef Constraint) const override;
 
     /// Examine constraint string and operand type and determine a weight value.
@@ -1103,7 +1123,6 @@ namespace llvm {
     SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerREM(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerBSWAP(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
@@ -1255,6 +1274,10 @@ namespace llvm {
     /// essentially v16i8 vector version of VINSERTH.
     SDValue lowerToVINSERTB(ShuffleVectorSDNode *N, SelectionDAG &DAG) const;
 
+    /// lowerToXXSPLTI32DX - Return the SDValue if this VECTOR_SHUFFLE can be
+    /// handled by the XXSPLTI32DX instruction introduced in ISA 3.1.
+    SDValue lowerToXXSPLTI32DX(ShuffleVectorSDNode *N, SelectionDAG &DAG) const;
+
     // Return whether the call instruction can potentially be optimized to a
     // tail call. This will cause the optimizers to attempt to move, or
     // duplicate return instructions to help enable tail call optimizations.
@@ -1273,6 +1296,9 @@ namespace llvm {
   bool isIntS16Immediate(SDNode *N, int16_t &Imm);
   bool isIntS16Immediate(SDValue Op, int16_t &Imm);
 
+  bool convertToNonDenormSingle(APInt &ArgAPInt);
+  bool convertToNonDenormSingle(APFloat &ArgAPFloat);
+
 } // end namespace llvm
 
 #endif // LLVM_TARGET_POWERPC_PPC32ISELLOWERING_H
diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
index cc0c787c7b21f..1c457d4170d54 100644
--- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
+++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
@@ -425,6 +425,19 @@ def DYNALLOC8 : PPCEmitTimePseudo<(outs g8rc:$result), (ins g8rc:$negsize, memri
                              (PPCdynalloc i64:$negsize, iaddr:$fpsi))]>;
 def DYNAREAOFFSET8 : PPCEmitTimePseudo<(outs i64imm:$result), (ins memri:$fpsi), "#DYNAREAOFFSET8",
                        [(set i64:$result, (PPCdynareaoffset iaddr:$fpsi))]>;
+// Probed alloca to support stack clash protection.
+let Defs = [X1], Uses = [X1], hasNoSchedulingInfo = 1 in {
+def PROBED_ALLOCA_64 : PPCCustomInserterPseudo<(outs g8rc:$result),
+                         (ins g8rc:$negsize, memri:$fpsi), "#PROBED_ALLOCA_64",
+                           [(set i64:$result,
+                             (PPCprobedalloca i64:$negsize, iaddr:$fpsi))]>;
+def PREPARE_PROBED_ALLOCA_64 : PPCEmitTimePseudo<(outs g8rc:$fp,
+    g8rc:$sp),
+    (ins g8rc:$negsize, memri:$fpsi), "#PREPARE_PROBED_ALLOCA_64", []>;
+def PROBED_STACKALLOC_64 : PPCEmitTimePseudo<(outs g8rc:$scratch, g8rc:$temp),
+    (ins i64imm:$stacksize),
+    "#PROBED_STACKALLOC_64", []>;
+}
 
 let hasSideEffects = 0 in {
 let Defs = [LR8] in {
diff --git a/llvm/lib/Target/PowerPC/PPCInstrAltivec.td b/llvm/lib/Target/PowerPC/PPCInstrAltivec.td
index b550d4e4974e2..920eeed9d41ff 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrAltivec.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrAltivec.td
@@ -341,7 +341,7 @@ class VXCR_Int_Ty<bits<11> xo, string opc, Intrinsic IntID, ValueType Ty>
 //===----------------------------------------------------------------------===//
 // Instruction Definitions.
 
-def HasAltivec : Predicate<"PPCSubTarget->hasAltivec()">;
+def HasAltivec : Predicate<"Subtarget->hasAltivec()">;
 let Predicates = [HasAltivec] in {
 
 def DSS      : DSS_Form<0, 822, (outs), (ins u5imm:$STRM),
@@ -491,7 +491,7 @@ let isCommutable = 1 in {
 def VADDFP : VXForm_1<10, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
                       "vaddfp $vD, $vA, $vB", IIC_VecFP,
                       [(set v4f32:$vD, (fadd v4f32:$vA, v4f32:$vB))]>;
-                      
+
 def VADDUBM : VXForm_1<0, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
                       "vaddubm $vD, $vA, $vB", IIC_VecGeneral,
                       [(set v16i8:$vD, (add v16i8:$vA, v16i8:$vB))]>;
@@ -501,7 +501,7 @@ def VADDUHM : VXForm_1<64, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
 def VADDUWM : VXForm_1<128, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
                       "vadduwm $vD, $vA, $vB", IIC_VecGeneral,
                       [(set v4i32:$vD, (add v4i32:$vA, v4i32:$vB))]>;
-                      
+
 def VADDCUW : VX1_Int_Ty<384, "vaddcuw", int_ppc_altivec_vaddcuw, v4i32>;
 def VADDSBS : VX1_Int_Ty<768, "vaddsbs", int_ppc_altivec_vaddsbs, v16i8>;
 def VADDSHS : VX1_Int_Ty<832, "vaddshs", int_ppc_altivec_vaddshs, v8i16>;
@@ -635,7 +635,7 @@ def VMULOUB : VX1_Int_Ty2<  8, "vmuloub", int_ppc_altivec_vmuloub,
 def VMULOUH : VX1_Int_Ty2< 72, "vmulouh", int_ppc_altivec_vmulouh,
                           v4i32, v8i16>;
 } // isCommutable
-                       
+
 def VREFP     : VX2_Int_SP<266, "vrefp",     int_ppc_altivec_vrefp>;
 def VRFIM     : VX2_Int_SP<714, "vrfim",     int_ppc_altivec_vrfim>;
 def VRFIN     : VX2_Int_SP<522, "vrfin",     int_ppc_altivec_vrfin>;
@@ -657,7 +657,7 @@ def VSUBUHM : VXForm_1<1088, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
 def VSUBUWM : VXForm_1<1152, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
                       "vsubuwm $vD, $vA, $vB", IIC_VecGeneral,
                       [(set v4i32:$vD, (sub v4i32:$vA, v4i32:$vB))]>;
-                      
+
 def VSUBSBS : VX1_Int_Ty<1792, "vsubsbs" , int_ppc_altivec_vsubsbs, v16i8>;
 def VSUBSHS : VX1_Int_Ty<1856, "vsubshs" , int_ppc_altivec_vsubshs, v8i16>;
 def VSUBSWS : VX1_Int_Ty<1920, "vsubsws" , int_ppc_altivec_vsubsws, v4i32>;
@@ -1022,7 +1022,7 @@ def : Pat<(and v4i32:$A, (vnot_ppc v4i32:$B)),
 
 def : Pat<(fmul v4f32:$vA, v4f32:$vB),
           (VMADDFP $vA, $vB,
-             (v4i32 (VSLW (v4i32 (V_SETALLONES)), (v4i32 (V_SETALLONES)))))>; 
+             (v4i32 (VSLW (v4i32 (V_SETALLONES)), (v4i32 (V_SETALLONES)))))>;
 
 def : Pat<(PPCfnmsub v4f32:$A, v4f32:$B, v4f32:$C),
           (VNMSUBFP $A, $B, $C)>;
@@ -1136,8 +1136,8 @@ def : Pat<(v16i8 (srl (sub v16i8:$vA, (v16i8 (bitconvert(vnot_ppc v4i32:$vB)))),
 
 } // end HasAltivec
 
-def HasP8Altivec : Predicate<"PPCSubTarget->hasP8Altivec()">;
-def HasP8Crypto : Predicate<"PPCSubTarget->hasP8Crypto()">;
+def HasP8Altivec : Predicate<"Subtarget->hasP8Altivec()">;
+def HasP8Crypto : Predicate<"Subtarget->hasP8Crypto()">;
 let Predicates = [HasP8Altivec] in {
 
 let isCommutable = 1 in {
@@ -1158,7 +1158,7 @@ def VMINSD : VX1_Int_Ty<962, "vminsd", int_ppc_altivec_vminsd, v2i64>;
 def VMINUD : VX1_Int_Ty<706, "vminud", int_ppc_altivec_vminud, v2i64>;
 } // isCommutable
 
-// Vector merge 
+// Vector merge
 def VMRGEW : VXForm_1<1932, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
                       "vmrgew $vD, $vA, $vB", IIC_VecFP,
                       [(set v16i8:$vD,
@@ -1266,16 +1266,16 @@ def VPOPCNTD : VXForm_2<1987, (outs vrrc:$vD), (ins vrrc:$vB),
                         [(set v2i64:$vD, (ctpop v2i64:$vB))]>;
 
 let isCommutable = 1 in {
-// FIXME: Use AddedComplexity > 400 to ensure these patterns match before the 
+// FIXME: Use AddedComplexity > 400 to ensure these patterns match before the
 //        VSX equivalents. We need to fix this up at some point. Two possible
 //        solutions for this problem:
 //        1. Disable Altivec patterns that compete with VSX patterns using the
-//           !HasVSX predicate. This essentially favours VSX over Altivec, in 
-//           hopes of reducing register pressure (larger register set using VSX 
+//           !HasVSX predicate. This essentially favours VSX over Altivec, in
+//           hopes of reducing register pressure (larger register set using VSX
 //           instructions than VMX instructions)
 //        2. Employ a more disciplined use of AddedComplexity, which would provide
 //           more fine-grained control than option 1. This would be beneficial
-//           if we find situations where Altivec is really preferred over VSX. 
+//           if we find situations where Altivec is really preferred over VSX.
 def VEQV  : VXForm_1<1668, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
                      "veqv $vD, $vA, $vB", IIC_VecGeneral,
                      [(set v4i32:$vD, (vnot_ppc (xor v4i32:$vA, v4i32:$vB)))]>;
@@ -1354,7 +1354,7 @@ def VSBOX : VXBX_Int_Ty<1480, "vsbox", int_ppc_altivec_crypto_vsbox, v2i64>;
 } // HasP8Crypto
 
 // The following altivec instructions were introduced in Power ISA 3.0
-def HasP9Altivec : Predicate<"PPCSubTarget->hasP9Altivec()">;
+def HasP9Altivec : Predicate<"Subtarget->hasP9Altivec()">;
 let Predicates = [HasP9Altivec] in {
 
 // Vector Multiply-Sum
diff --git a/llvm/lib/Target/PowerPC/PPCInstrHTM.td b/llvm/lib/Target/PowerPC/PPCInstrHTM.td
index 912c0ea52a819..992ad8216f3bd 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrHTM.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrHTM.td
@@ -13,7 +13,7 @@
 
 
 
-def HasHTM : Predicate<"PPCSubTarget->hasHTM()">;
+def HasHTM : Predicate<"Subtarget->hasHTM()">;
 
 def HTM_get_imm : SDNodeXForm<imm, [{
   return getI32Imm (N->getZExtValue(), SDLoc(N));
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index 20b50a9549557..11c97210ead9b 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -19,6 +19,7 @@
 #include "PPCTargetMachine.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/LiveIntervals.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -289,12 +290,11 @@ bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const {
 // Index 0(InfoArrayIdxFMAInst): FMA instruction;
 // Index 1(InfoArrayIdxFAddInst): ADD instruction assoaicted with FMA;
 // Index 2(InfoArrayIdxFMULInst): MUL instruction assoaicted with FMA;
-// Index 3(InfoArrayIdxAddOpIdx): ADD operand index in the FMA operand list;
-// Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in the FMA operand
-//                                list;
+// Index 3(InfoArrayIdxAddOpIdx): ADD operand index in FMA operands;
+// Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in FMA operands;
 //                                second MUL operand index is plus 1.
 static const uint16_t FMAOpIdxInfo[][5] = {
-    // FIXME: add more FMA instructions like XSNMADDADP and so on.
+    // FIXME: Add more FMA instructions like XSNMADDADP and so on.
     {PPC::XSMADDADP, PPC::XSADDDP, PPC::XSMULDP, 1, 2},
     {PPC::XSMADDASP, PPC::XSADDSP, PPC::XSMULSP, 1, 2},
     {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2},
@@ -362,12 +362,12 @@ bool PPCInstrInfo::getFMAPatterns(
       return false;
 
     // Instruction can be reassociated.
-    // fast match flags may prohibit reassociation.
+    // fast math flags may prohibit reassociation.
     if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
           Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
       return false;
 
-    // Instruction operands are virtual registers for reassociating.
+    // Instruction operands are virtual registers for reassociation.
     if (!IsAllOpsVirtualReg(Instr))
       return false;
 
@@ -544,7 +544,7 @@ void PPCInstrInfo::reassociateFMA(
             .addReg(RegY, getKillRegState(KillY))
             .addReg(RegM31, getKillRegState(KillM31))
             .addReg(RegM32, getKillRegState(KillM32));
-    // if AddOpIdx is not 1, adjust the order.
+    // If AddOpIdx is not 1, adjust the order.
     if (AddOpIdx != 1) {
       AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
       AdjustOperandOrder(MINewA, RegY, KillY, RegM31, KillM31, RegM32, KillM32);
@@ -556,7 +556,7 @@ void PPCInstrInfo::reassociateFMA(
             .addReg(NewVRB, getKillRegState(true))
             .addReg(NewVRA, getKillRegState(true));
 
-    // update flags for new created instructions.
+    // Update flags for newly created instructions.
     setSpecialOperandAttr(*MINewA, IntersectedFlags);
     setSpecialOperandAttr(*MINewB, IntersectedFlags);
     setSpecialOperandAttr(*MINewC, IntersectedFlags);
@@ -596,7 +596,7 @@ void PPCInstrInfo::reassociateFMA(
             .addReg(NewVRB, getKillRegState(true))
             .addReg(NewVRD, getKillRegState(true));
 
-    // update flags for new created instructions.
+    // Update flags for newly created instructions.
     setSpecialOperandAttr(*MINewA, IntersectedFlags);
     setSpecialOperandAttr(*MINewB, IntersectedFlags);
     setSpecialOperandAttr(*MINewD, IntersectedFlags);
@@ -2563,7 +2563,8 @@ MachineInstr *PPCInstrInfo::getForwardingDefMI(
   MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
   const TargetRegisterInfo *TRI = &getRegisterInfo();
   // If we're in SSA, get the defs through the MRI. Otherwise, only look
-  // within the basic block to see if the register is defined using an LI/LI8.
+  // within the basic block to see if the register is defined using an
+  // LI/LI8/ADDI/ADDI8.
   if (MRI->isSSA()) {
     for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
       if (!MI.getOperand(i).isReg())
@@ -2574,9 +2575,16 @@ MachineInstr *PPCInstrInfo::getForwardingDefMI(
       unsigned TrueReg = TRI->lookThruCopyLike(Reg, MRI);
       if (Register::isVirtualRegister(TrueReg)) {
         DefMI = MRI->getVRegDef(TrueReg);
-        if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8) {
+        if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8 ||
+            DefMI->getOpcode() == PPC::ADDI ||
+            DefMI->getOpcode() == PPC::ADDI8) {
           OpNoForForwarding = i;
-          break;
+          // The ADDI and LI operand maybe exist in one instruction at same
+          // time. we prefer to fold LI operand as LI only has one Imm operand
+          // and is more possible to be converted. So if current DefMI is
+          // ADDI/ADDI8, we continue to find possible LI/LI8.
+          if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8)
+            break;
         }
       }
     }
@@ -2647,10 +2655,6 @@ const unsigned *PPCInstrInfo::getLoadOpcodesForSpillArray() const {
 
 void PPCInstrInfo::fixupIsDeadOrKill(MachineInstr &StartMI, MachineInstr &EndMI,
                                      unsigned RegNo) const {
-  const MachineRegisterInfo &MRI =
-      StartMI.getParent()->getParent()->getRegInfo();
-  if (MRI.isSSA())
-    return;
 
   // Instructions between [StartMI, EndMI] should be in same basic block.
   assert((StartMI.getParent() == EndMI.getParent()) &&
@@ -2973,6 +2977,13 @@ bool PPCInstrInfo::convertToImmediateForm(MachineInstr &MI,
   if (KilledDef && KillFwdDefMI)
     *KilledDef = DefMI;
 
+  // If this is a imm instruction and its register operands is produced by ADDI,
+  // put the imm into imm inst directly.
+  if (RI.getMappedIdxOpcForImmOpc(MI.getOpcode()) !=
+          PPC::INSTRUCTION_LIST_END &&
+      transformToNewImmFormFedByAdd(MI, *DefMI, ForwardingOperand))
+    return true;
+
   ImmInstrInfo III;
   bool IsVFReg = MI.getOperand(0).isReg()
                      ? isVFRegister(MI.getOperand(0).getReg())
@@ -3513,6 +3524,10 @@ bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI,
   RegMO = &DefMI.getOperand(1);
   ImmMO = &DefMI.getOperand(2);
 
+  // Before RA, ADDI first operand could be a frame index.
+  if (!RegMO->isReg())
+    return false;
+
   // This DefMI is elgible for forwarding if it is:
   // 1. add inst
   // 2. one of the operands is Imm/CPI/Global.
@@ -3561,7 +3576,8 @@ bool PPCInstrInfo::isRegElgibleForForwarding(
 bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
                                              const MachineInstr &DefMI,
                                              const ImmInstrInfo &III,
-                                             int64_t &Imm) const {
+                                             int64_t &Imm,
+                                             int64_t BaseImm) const {
   assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate");
   if (DefMI.getOpcode() == PPC::ADDItocL) {
     // The operand for ADDItocL is CPI, which isn't imm at compiling time,
@@ -3575,19 +3591,21 @@ bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
     // not just an immediate but also a multiple of 4, or 16 depending on the
     // load. A DForm load cannot be represented if it is a multiple of say 2.
     // XForm loads do not have this restriction.
-    if (ImmMO.isGlobal() &&
-        ImmMO.getGlobal()->getAlignment() < III.ImmMustBeMultipleOf)
-      return false;
+    if (ImmMO.isGlobal()) {
+      const DataLayout &DL = ImmMO.getGlobal()->getParent()->getDataLayout();
+      if (ImmMO.getGlobal()->getPointerAlignment(DL) < III.ImmMustBeMultipleOf)
+        return false;
+    }
 
     return true;
   }
 
   if (ImmMO.isImm()) {
     // It is Imm, we need to check if the Imm fit the range.
-    int64_t Immediate = ImmMO.getImm();
     // Sign-extend to 64-bits.
-    Imm = ((uint64_t)Immediate & ~0x7FFFuLL) != 0 ?
-      (Immediate | 0xFFFFFFFFFFFF0000) : Immediate;
+    // DefMI may be folded with another imm form instruction, the result Imm is
+    // the sum of Imm of DefMI and BaseImm which is from imm form instruction.
+    Imm = SignExtend64<16>(ImmMO.getImm() + BaseImm);
 
     if (Imm % III.ImmMustBeMultipleOf)
       return false;
@@ -3840,6 +3858,99 @@ bool PPCInstrInfo::simplifyToLI(MachineInstr &MI, MachineInstr &DefMI,
   return false;
 }
 
+bool PPCInstrInfo::transformToNewImmFormFedByAdd(
+    MachineInstr &MI, MachineInstr &DefMI, unsigned OpNoForForwarding) const {
+  MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
+  bool PostRA = !MRI->isSSA();
+  // FIXME: extend this to post-ra. Need to do some change in getForwardingDefMI
+  // for post-ra.
+  if (PostRA)
+    return false;
+
+  // Only handle load/store.
+  if (!MI.mayLoadOrStore())
+    return false;
+
+  unsigned XFormOpcode = RI.getMappedIdxOpcForImmOpc(MI.getOpcode());
+
+  assert((XFormOpcode != PPC::INSTRUCTION_LIST_END) &&
+         "MI must have x-form opcode");
+
+  // get Imm Form info.
+  ImmInstrInfo III;
+  bool IsVFReg = MI.getOperand(0).isReg()
+                     ? isVFRegister(MI.getOperand(0).getReg())
+                     : false;
+
+  if (!instrHasImmForm(XFormOpcode, IsVFReg, III, PostRA))
+    return false;
+
+  if (!III.IsSummingOperands)
+    return false;
+
+  if (OpNoForForwarding != III.OpNoForForwarding)
+    return false;
+
+  MachineOperand ImmOperandMI = MI.getOperand(III.ImmOpNo);
+  if (!ImmOperandMI.isImm())
+    return false;
+
+  // Check DefMI.
+  MachineOperand *ImmMO = nullptr;
+  MachineOperand *RegMO = nullptr;
+  if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
+    return false;
+  assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
+
+  // Check Imm.
+  // Set ImmBase from imm instruction as base and get new Imm inside
+  // isImmElgibleForForwarding.
+  int64_t ImmBase = ImmOperandMI.getImm();
+  int64_t Imm = 0;
+  if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm, ImmBase))
+    return false;
+
+  // Get killed info in case fixup needed after transformation.
+  unsigned ForwardKilledOperandReg = ~0U;
+  if (MI.getOperand(III.OpNoForForwarding).isKill())
+    ForwardKilledOperandReg = MI.getOperand(III.OpNoForForwarding).getReg();
+
+  // Do the transform
+  LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
+  LLVM_DEBUG(MI.dump());
+  LLVM_DEBUG(dbgs() << "Fed by:\n");
+  LLVM_DEBUG(DefMI.dump());
+
+  MI.getOperand(III.OpNoForForwarding).setReg(RegMO->getReg());
+  MI.getOperand(III.OpNoForForwarding).setIsKill(RegMO->isKill());
+  MI.getOperand(III.ImmOpNo).setImm(Imm);
+
+  // FIXME: fix kill/dead flag if MI and DefMI are not in same basic block.
+  if (DefMI.getParent() == MI.getParent()) {
+    // Check if reg is killed between MI and DefMI.
+    auto IsKilledFor = [&](unsigned Reg) {
+      MachineBasicBlock::const_reverse_iterator It = MI;
+      MachineBasicBlock::const_reverse_iterator E = DefMI;
+      It++;
+      for (; It != E; ++It) {
+        if (It->killsRegister(Reg))
+          return true;
+      }
+      return false;
+    };
+
+    // Update kill flag
+    if (RegMO->isKill() || IsKilledFor(RegMO->getReg()))
+      fixupIsDeadOrKill(DefMI, MI, RegMO->getReg());
+    if (ForwardKilledOperandReg != ~0U)
+      fixupIsDeadOrKill(DefMI, MI, ForwardKilledOperandReg);
+  }
+
+  LLVM_DEBUG(dbgs() << "With:\n");
+  LLVM_DEBUG(MI.dump());
+  return true;
+}
+
 // If an X-Form instruction is fed by an add-immediate and one of its operands
 // is the literal zero, attempt to forward the source of the add-immediate to
 // the corresponding D-Form instruction with the displacement coming from
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index ca30ced9d4041..d98597f483406 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -191,6 +191,10 @@ class PPCInstrInfo : public PPCGenInstrInfo {
   // LI8.
   bool simplifyToLI(MachineInstr &MI, MachineInstr &DefMI,
                     unsigned OpNoForForwarding, MachineInstr **KilledDef) const;
+  // If the inst is imm-form and its register operand is produced by a ADDI, put
+  // the imm into the inst directly and remove the ADDI if possible.
+  bool transformToNewImmFormFedByAdd(MachineInstr &MI, MachineInstr &DefMI,
+                                     unsigned OpNoForForwarding) const;
   // If the inst is x-form and has imm-form and one of its operand is produced
   // by a LI, put the imm into the inst directly and remove the LI if possible.
   bool transformToImmFormFedByLI(MachineInstr &MI, const ImmInstrInfo &III,
@@ -221,7 +225,8 @@ class PPCInstrInfo : public PPCGenInstrInfo {
   bool isImmElgibleForForwarding(const MachineOperand &ImmMO,
                                  const MachineInstr &DefMI,
                                  const ImmInstrInfo &III,
-                                 int64_t &Imm) const;
+                                 int64_t &Imm,
+                                 int64_t BaseImm = 0) const;
   bool isRegElgibleForForwarding(const MachineOperand &RegMO,
                                  const MachineInstr &DefMI,
                                  const MachineInstr &MI, bool KillDefMI,
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index ffc13aabcddc3..673ab63039cf7 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -50,6 +50,10 @@ def SDT_PPCVecSplat : SDTypeProfile<1, 2, [ SDTCisVec<0>,
   SDTCisVec<1>, SDTCisInt<2>
 ]>;
 
+def SDT_PPCSpToDp : SDTypeProfile<1, 1, [ SDTCisVT<0, v2f64>,
+  SDTCisInt<1>
+]>;
+
 def SDT_PPCVecShift : SDTypeProfile<1, 3, [ SDTCisVec<0>,
   SDTCisVec<1>, SDTCisVec<2>, SDTCisPtrTy<3>
 ]>;
@@ -158,9 +162,9 @@ def PPCmffs   : SDNode<"PPCISD::MFFS",
 def PPCfaddrtz: SDNode<"PPCISD::FADDRTZ", SDTFPBinOp, []>;
 
 
-def PPCfsel   : SDNode<"PPCISD::FSEL",  
+def PPCfsel   : SDNode<"PPCISD::FSEL",
    // Type constraint for fsel.
-   SDTypeProfile<1, 3, [SDTCisSameAs<0, 2>, SDTCisSameAs<0, 3>, 
+   SDTypeProfile<1, 3, [SDTCisSameAs<0, 2>, SDTCisSameAs<0, 3>,
                         SDTCisFP<0>, SDTCisVT<1, f64>]>, []>;
 def PPCxsmaxc : SDNode<"PPCISD::XSMAXCDP", SDT_PPCFPMinMax, []>;
 def PPCxsminc : SDNode<"PPCISD::XSMINCDP", SDT_PPCFPMinMax, []>;
@@ -194,6 +198,7 @@ def PPCaddiDtprelL   : SDNode<"PPCISD::ADDI_DTPREL_L", SDTIntBinOp>;
 
 def PPCvperm     : SDNode<"PPCISD::VPERM", SDT_PPCvperm, []>;
 def PPCxxsplt    : SDNode<"PPCISD::XXSPLT", SDT_PPCVecSplat, []>;
+def PPCxxspltidp : SDNode<"PPCISD::XXSPLTI_SP_TO_DP", SDT_PPCSpToDp, []>;
 def PPCvecinsert : SDNode<"PPCISD::VECINSERT", SDT_PPCVecInsert, []>;
 def PPCxxpermdi  : SDNode<"PPCISD::XXPERMDI", SDT_PPCxxpermdi, []>;
 def PPCvecshl    : SDNode<"PPCISD::VECSHL", SDT_PPCVecShift, []>;
@@ -318,6 +323,7 @@ def SDTDynOp  : SDTypeProfile<1, 2, []>;
 def SDTDynAreaOp  : SDTypeProfile<1, 1, []>;
 def PPCdynalloc   : SDNode<"PPCISD::DYNALLOC", SDTDynOp, [SDNPHasChain]>;
 def PPCdynareaoffset   : SDNode<"PPCISD::DYNAREAOFFSET", SDTDynAreaOp, [SDNPHasChain]>;
+def PPCprobedalloca : SDNode<"PPCISD::PROBED_ALLOCA", SDTDynOp, [SDNPHasChain]>;
 
 // PC Relative Specific Nodes
 def PPCmatpcreladdr : SDNode<"PPCISD::MAT_PCREL_ADDR", SDTIntUnaryOp, []>;
@@ -326,6 +332,23 @@ def PPCmatpcreladdr : SDNode<"PPCISD::MAT_PCREL_ADDR", SDTIntUnaryOp, []>;
 // PowerPC specific transformation functions and pattern fragments.
 //
 
+// A floating point immediate that is not a positive zero and can be converted
+// to a single precision floating point non-denormal immediate without loss of
+// information.
+def nzFPImmAsi32 : PatLeaf<(fpimm), [{
+  APFloat APFloatOfN = N->getValueAPF();
+  return convertToNonDenormSingle(APFloatOfN) && !N->isExactlyValue(+0.0);
+}]>;
+
+// Convert the floating point immediate into a 32 bit floating point immediate
+// and get a i32 with the resulting bits.
+def getFPAs32BitInt : SDNodeXForm<fpimm, [{
+  APFloat APFloatOfN = N->getValueAPF();
+  convertToNonDenormSingle(APFloatOfN);
+  return CurDAG->getTargetConstant(APFloatOfN.bitcastToAPInt().getZExtValue(),
+                                   SDLoc(N), MVT::i32);
+}]>;
+
 def SHL32 : SDNodeXForm<imm, [{
   // Transformation function: 31 - imm
   return getI32Imm(31 - N->getZExtValue(), SDLoc(N));
@@ -389,9 +412,10 @@ def immZExt16  : PatLeaf<(imm), [{
   // field.  Used by instructions like 'ori'.
   return (uint64_t)N->getZExtValue() == (unsigned short)N->getZExtValue();
 }], LO16>;
-def immNonAllOneAnyExt8 : ImmLeaf<i32, [{ 
+def immNonAllOneAnyExt8 : ImmLeaf<i32, [{
   return (isInt<8>(Imm) && (Imm != -1)) || (isUInt<8>(Imm) && (Imm != 0xFF));
 }]>;
+def i32immNonAllOneNonZero : ImmLeaf<i32, [{ return Imm && (Imm != -1); }]>;
 def immSExt5NonZero : ImmLeaf<i32, [{ return Imm && isInt<5>(Imm); }]>;
 
 // imm16Shifted* - These match immediates where the low 16-bits are zero.  There
@@ -407,7 +431,7 @@ def imm16ShiftedZExt : PatLeaf<(imm), [{
 
 def imm16ShiftedSExt : PatLeaf<(imm), [{
   // imm16ShiftedSExt predicate - True if only bits in the top 16-bits of the
-  // immediate are set.  Used by instructions like 'addis'.  Identical to 
+  // immediate are set.  Used by instructions like 'addis'.  Identical to
   // imm16ShiftedZExt in 32-bit mode.
   if (N->getZExtValue() & 0xFFFF) return false;
   if (N->getValueType(0) == MVT::i32)
@@ -986,25 +1010,27 @@ def pcreladdr : ComplexPattern<iPTR, 1, "SelectAddrPCRel", [], []>;
 
 //===----------------------------------------------------------------------===//
 // PowerPC Instruction Predicate Definitions.
-def In32BitMode  : Predicate<"!PPCSubTarget->isPPC64()">;
-def In64BitMode  : Predicate<"PPCSubTarget->isPPC64()">;
-def IsBookE  : Predicate<"PPCSubTarget->isBookE()">;
-def IsNotBookE  : Predicate<"!PPCSubTarget->isBookE()">;
-def HasOnlyMSYNC : Predicate<"PPCSubTarget->hasOnlyMSYNC()">;
-def HasSYNC   : Predicate<"!PPCSubTarget->hasOnlyMSYNC()">;
-def IsPPC4xx  : Predicate<"PPCSubTarget->isPPC4xx()">;
-def IsPPC6xx  : Predicate<"PPCSubTarget->isPPC6xx()">;
-def IsE500  : Predicate<"PPCSubTarget->isE500()">;
-def HasSPE  : Predicate<"PPCSubTarget->hasSPE()">;
-def HasICBT : Predicate<"PPCSubTarget->hasICBT()">;
-def HasPartwordAtomics : Predicate<"PPCSubTarget->hasPartwordAtomics()">;
-def NoNaNsFPMath : Predicate<"TM.Options.NoNaNsFPMath">;
-def NaNsFPMath   : Predicate<"!TM.Options.NoNaNsFPMath">;
-def HasBPERMD : Predicate<"PPCSubTarget->hasBPERMD()">;
-def HasExtDiv : Predicate<"PPCSubTarget->hasExtDiv()">;
-def IsISA3_0 : Predicate<"PPCSubTarget->isISA3_0()">;
-def HasFPU : Predicate<"PPCSubTarget->hasFPU()">;
-def PCRelativeMemops : Predicate<"PPCSubTarget->hasPCRelativeMemops()">;
+def In32BitMode  : Predicate<"!Subtarget->isPPC64()">;
+def In64BitMode  : Predicate<"Subtarget->isPPC64()">;
+def IsBookE  : Predicate<"Subtarget->isBookE()">;
+def IsNotBookE  : Predicate<"!Subtarget->isBookE()">;
+def HasOnlyMSYNC : Predicate<"Subtarget->hasOnlyMSYNC()">;
+def HasSYNC   : Predicate<"!Subtarget->hasOnlyMSYNC()">;
+def IsPPC4xx  : Predicate<"Subtarget->isPPC4xx()">;
+def IsPPC6xx  : Predicate<"Subtarget->isPPC6xx()">;
+def IsE500  : Predicate<"Subtarget->isE500()">;
+def HasSPE  : Predicate<"Subtarget->hasSPE()">;
+def HasICBT : Predicate<"Subtarget->hasICBT()">;
+def HasPartwordAtomics : Predicate<"Subtarget->hasPartwordAtomics()">;
+def NoNaNsFPMath
+    : Predicate<"Subtarget->getTargetMachine().Options.NoNaNsFPMath">;
+def NaNsFPMath
+    : Predicate<"!Subtarget->getTargetMachine().Options.NoNaNsFPMath">;
+def HasBPERMD : Predicate<"Subtarget->hasBPERMD()">;
+def HasExtDiv : Predicate<"Subtarget->hasExtDiv()">;
+def IsISA3_0 : Predicate<"Subtarget->isISA3_0()">;
+def HasFPU : Predicate<"Subtarget->hasFPU()">;
+def PCRelativeMemops : Predicate<"Subtarget->hasPCRelativeMemops()">;
 
 //===----------------------------------------------------------------------===//
 // PowerPC Multiclass Definitions.
@@ -1374,7 +1400,20 @@ def DYNALLOC : PPCEmitTimePseudo<(outs gprc:$result), (ins gprc:$negsize, memri:
                              (PPCdynalloc i32:$negsize, iaddr:$fpsi))]>;
 def DYNAREAOFFSET : PPCEmitTimePseudo<(outs i32imm:$result), (ins memri:$fpsi), "#DYNAREAOFFSET",
                        [(set i32:$result, (PPCdynareaoffset iaddr:$fpsi))]>;
-                         
+// Probed alloca to support stack clash protection.
+let Defs = [R1], Uses = [R1], hasNoSchedulingInfo = 1 in {
+def PROBED_ALLOCA_32 : PPCCustomInserterPseudo<(outs gprc:$result),
+                         (ins gprc:$negsize, memri:$fpsi), "#PROBED_ALLOCA_32",
+                           [(set i32:$result,
+                             (PPCprobedalloca i32:$negsize, iaddr:$fpsi))]>;
+def PREPARE_PROBED_ALLOCA_32 : PPCEmitTimePseudo<(outs gprc:$fp,
+    gprc:$sp),
+    (ins gprc:$negsize, memri:$fpsi), "#PREPARE_PROBED_ALLOCA_32", []>;
+def PROBED_STACKALLOC_32 : PPCEmitTimePseudo<(outs gprc:$scratch, gprc:$temp),
+    (ins i64imm:$stacksize),
+    "#PROBED_STACKALLOC_32", []>;
+}
+
 // SELECT_CC_* - Used to implement the SELECT_CC DAG operation.  Expanded after
 // instruction selection into a branch sequence.
 let PPC970_Single = 1 in {
@@ -1468,7 +1507,7 @@ let isTerminator = 1, isBarrier = 1, PPC970_Unit = 7 in {
 }
 
 // Set the float rounding mode.
-let Uses = [RM], Defs = [RM] in { 
+let Uses = [RM], Defs = [RM] in {
 def SETRNDi : PPCCustomInserterPseudo<(outs f8rc:$FRT), (ins u2imm:$RND),
                     "#SETRNDi", [(set f64:$FRT, (int_ppc_setrnd (i32 imm:$RND)))]>;
 
@@ -1835,6 +1874,11 @@ def : Pat<(prefetch xoaddr:$dst, (i32 1), imm, (i32 1)),
 def : Pat<(prefetch xoaddr:$dst, (i32 0), imm, (i32 0)),
           (ICBT 0, xoaddr:$dst)>, Requires<[HasICBT]>; // inst prefetch (for read)
 
+def : Pat<(int_ppc_dcbt_with_hint xoaddr:$dst, i32:$TH),
+          (DCBT i32:$TH, xoaddr:$dst)>;
+def : Pat<(int_ppc_dcbtst_with_hint xoaddr:$dst, i32:$TH),
+          (DCBTST i32:$TH, xoaddr:$dst)>;
+
 // Atomic operations
 // FIXME: some of these might be used with constant operands. This will result
 // in constant materialization instructions that may be redundant. We currently
@@ -2027,7 +2071,7 @@ def TD : XForm_1<31, 68, (outs), (ins u5imm:$to, g8rc:$rA, g8rc:$rB),
 // PPC32 Load Instructions.
 //
 
-// Unindexed (r+i) Loads. 
+// Unindexed (r+i) Loads.
 let PPC970_Unit = 2 in {
 def LBZ : DForm_1<34, (outs gprc:$rD), (ins memri:$src),
                   "lbz $rD, $src", IIC_LdStLoad,
@@ -2170,7 +2214,7 @@ def LFIWZX : XForm_25_memOp<31, 887, (outs f8rc:$frD), (ins memrr:$src),
 }
 
 // Load Multiple
-let mayLoad = 1, mayStore = 0, hasSideEffects = 0 in 
+let mayLoad = 1, mayStore = 0, hasSideEffects = 0 in
 def LMW : DForm_1<46, (outs gprc:$rD), (ins memri:$src),
                   "lmw $rD, $src", IIC_LdStLMW, []>;
 
@@ -2871,7 +2915,7 @@ let Uses = [RM] in {
 
 // The above pseudo gets expanded to make use of the following instructions
 // to manipulate FPSCR.  Note that FPSCR is not modeled at the DAG level.
-let Uses = [RM], Defs = [RM] in { 
+let Uses = [RM], Defs = [RM] in {
   def MTFSB0 : XForm_43<63, 70, (outs), (ins u5imm:$FM),
                         "mtfsb0 $FM", IIC_IntMTFSB0, []>,
                PPC970_DGroup_Single, PPC970_Unit_FPU;
@@ -3018,7 +3062,7 @@ def : InstAlias<"subc. $rA, $rB, $rC", (SUBFC_rec gprc:$rA, gprc:$rC, gprc:$rB)>
 let PPC970_Unit = 3, hasSideEffects = 0, Predicates = [HasFPU] in {  // FPU Operations.
 let Uses = [RM] in {
 let isCommutable = 1 in {
-  defm FMADD : AForm_1r<63, 29, 
+  defm FMADD : AForm_1r<63, 29,
                       (outs f8rc:$FRT), (ins f8rc:$FRA, f8rc:$FRC, f8rc:$FRB),
                       "fmadd", "$FRT, $FRA, $FRC, $FRB", IIC_FPFused,
                       [(set f64:$FRT, (any_fma f64:$FRA, f64:$FRC, f64:$FRB))]>;
@@ -3239,13 +3283,13 @@ def : Pat<(add i32:$in, (PPChi tblockaddress:$g, 0)),
           (ADDIS $in, tblockaddress:$g)>;
 
 // Support for thread-local storage.
-def PPC32GOT: PPCEmitTimePseudo<(outs gprc:$rD), (ins), "#PPC32GOT", 
+def PPC32GOT: PPCEmitTimePseudo<(outs gprc:$rD), (ins), "#PPC32GOT",
                 [(set i32:$rD, (PPCppc32GOT))]>;
 
 // Get the _GLOBAL_OFFSET_TABLE_ in PIC mode.
 // This uses two output registers, the first as the real output, the second as a
 // temporary register, used internally in code generation.
-def PPC32PICGOT: PPCEmitTimePseudo<(outs gprc:$rD, gprc:$rT), (ins), "#PPC32PICGOT", 
+def PPC32PICGOT: PPCEmitTimePseudo<(outs gprc:$rD, gprc:$rT), (ins), "#PPC32PICGOT",
                 []>, NoEncode<"$rT">;
 
 def LDgotTprelL32: PPCEmitTimePseudo<(outs gprc_nor0:$rD), (ins s16imm:$disp, gprc_nor0:$reg),
@@ -3599,7 +3643,7 @@ defm : ExtSetCCPat<SETEQ,
                               (RLWINM (CNTLZW $in), 27, 31, 31)>,
                    OutPatFrag<(ops node:$in),
                               (RLDICL (CNTLZD $in), 58, 63)> >;
- 
+
 defm : ExtSetCCPat<SETNE,
                    PatFrag<(ops node:$in, node:$cc),
                            (setcc $in, 0, $cc)>,
@@ -3607,7 +3651,7 @@ defm : ExtSetCCPat<SETNE,
                               (RLWINM (i32not (CNTLZW $in)), 27, 31, 31)>,
                    OutPatFrag<(ops node:$in),
                               (RLDICL (i64not (CNTLZD $in)), 58, 63)> >;
-                 
+
 defm : ExtSetCCPat<SETLT,
                    PatFrag<(ops node:$in, node:$cc),
                            (setcc $in, 0, $cc)>,
@@ -4506,6 +4550,7 @@ def DCBFx  : PPCAsmPseudo<"dcbf $dst", (ins memrr:$dst)>;
 def DCBFL  : PPCAsmPseudo<"dcbfl $dst", (ins memrr:$dst)>;
 def DCBFLP : PPCAsmPseudo<"dcbflp $dst", (ins memrr:$dst)>;
 
+def : Pat<(int_ppc_isync),  (ISYNC)>;
 def : Pat<(int_ppc_dcbfl xoaddr:$dst),
           (DCBF 1, xoaddr:$dst)>;
 def : Pat<(int_ppc_dcbflp xoaddr:$dst),
diff --git a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
index 31fe57150bff2..2d12a72e29aee 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
@@ -1,3 +1,19 @@
+//===----------------------------------------------------------------------===//
+// PowerPC ISA 3.1 specific type constraints.
+//
+
+def SDT_PPCSplat32 : SDTypeProfile<1, 3, [ SDTCisVT<0, v2i64>,
+  SDTCisVec<1>, SDTCisInt<2>, SDTCisInt<3>
+]>;
+
+//===----------------------------------------------------------------------===//
+// ISA 3.1 specific PPCISD nodes.
+//
+
+def PPCxxsplti32dx : SDNode<"PPCISD::XXSPLTI32DX", SDT_PPCSplat32, []>;
+
+//===----------------------------------------------------------------------===//
+
 // PC Relative flag (for instructions that use the address of the prefix for
 // address computations).
 class isPCRel { bit PCRel = 1; }
@@ -161,6 +177,225 @@ class 8LS_DForm_R_SI34_XT6_RA5<bits<5> opcode, dag OOL, dag IOL, string asmstr,
   let Inst{48-63} = D_RA{15-0}; // d1
 }
 
+// X-Form: [PO T IMM VRB XO TX]
+class XForm_XT6_IMM5_VB5<bits<6> opcode, bits<10> xo, dag OOL, dag IOL,
+                         string asmstr, InstrItinClass itin, list<dag> pattern>
+  : I<opcode, OOL, IOL, asmstr, itin> {
+  bits<6> XT;
+  bits<5> VRB;
+  bits<5> IMM;
+
+  let Pattern = pattern;
+  let Inst{6-10} = XT{4-0};
+  let Inst{11-15} = IMM;
+  let Inst{16-20} = VRB;
+  let Inst{21-30} = xo;
+  let Inst{31} = XT{5};
+}
+
+class 8RR_XX4Form_IMM8_XTAB6<bits<6> opcode, bits<2> xo,
+                             dag OOL, dag IOL, string asmstr,
+                             InstrItinClass itin, list<dag> pattern>
+  : PI<1, opcode, OOL, IOL, asmstr, itin> {
+    bits<6> XT;
+    bits<6> XA;
+    bits<6> XB;
+    bits<6> XC;
+    bits<8> IMM;
+
+    let Pattern = pattern;
+
+    // The prefix.
+    let Inst{6-7} = 1;
+    let Inst{8} = 0;
+    let Inst{9-11} = 0;
+    let Inst{12-13} = 0;
+    let Inst{14-23} = 0;
+    let Inst{24-31} = IMM;
+
+    // The instruction.
+    let Inst{38-42} = XT{4-0};
+    let Inst{43-47} = XA{4-0};
+    let Inst{48-52} = XB{4-0};
+    let Inst{53-57} = XC{4-0};
+    let Inst{58-59} = xo;
+    let Inst{60} = XC{5};
+    let Inst{61} = XA{5};
+    let Inst{62} = XB{5};
+    let Inst{63} = XT{5};
+}
+
+class VXForm_RD5_N3_VB5<bits<11> xo, dag OOL, dag IOL, string asmstr,
+                        InstrItinClass itin, list<dag> pattern>
+  : I<4, OOL, IOL, asmstr, itin> {
+  bits<5> RD;
+  bits<5> VB;
+  bits<3> N;
+
+  let Pattern = pattern;
+
+  let Inst{6-10}  = RD;
+  let Inst{11-12} = 0;
+  let Inst{13-15} = N;
+  let Inst{16-20} = VB;
+  let Inst{21-31} = xo;
+}
+
+
+// VX-Form: [PO VRT / UIM RB XO].
+// We use VXForm_1 to implement it, that is, we use "VRA" (5 bit) to represent
+// "/ UIM" (unused bit followed by a 4-bit immediate)
+// Destructive (insert) forms are suffixed with _ins.
+class VXForm_VRT5_UIM5_RB5_ins<bits<11> xo, string opc, list<dag> pattern>
+  : VXForm_1<xo, (outs vrrc:$vD), (ins vrrc:$vDi, u4imm:$UIM, g8rc:$rB),
+             !strconcat(opc, " $vD, $rB, $UIM"), IIC_VecGeneral, pattern>,
+             RegConstraint<"$vDi = $vD">, NoEncode<"$vDi">;
+
+// VX-Form: [PO VRT RA VRB XO].
+// Destructive (insert) forms are suffixed with _ins.
+class VXForm_VTB5_RA5_ins<bits<11> xo, string opc, list<dag> pattern>
+  : VXForm_1<xo, (outs vrrc:$vD), (ins vrrc:$vDi, g8rc:$rA, vrrc:$vB),
+             !strconcat(opc, " $vD, $rA, $vB"), IIC_VecGeneral, pattern>,
+             RegConstraint<"$vDi = $vD">, NoEncode<"$vDi">;
+
+// VX-Form: [PO VRT RA RB XO].
+// Destructive (insert) forms are suffixed with _ins.
+class VXForm_VRT5_RAB5_ins<bits<11> xo, string opc, list<dag> pattern>
+  : VXForm_1<xo, (outs vrrc:$vD), (ins vrrc:$vDi, g8rc:$rA, g8rc:$rB),
+             !strconcat(opc, " $vD, $rA, $rB"), IIC_VecGeneral, pattern>,
+             RegConstraint<"$vDi = $vD">, NoEncode<"$vDi">;
+
+// VN-Form: [PO VRT VRA VRB PS SD XO]
+// SD is "Shift Direction"
+class VNForm_VTAB5_SD3<bits<6> xo, bits<2> ps, dag OOL, dag IOL, string asmstr,
+                       InstrItinClass itin, list<dag> pattern>
+    : I<4, OOL, IOL, asmstr, itin> {
+  bits<5> VRT;
+  bits<5> VRA;
+  bits<5> VRB;
+  bits<3> SD;
+
+  let Pattern = pattern;
+
+  let Inst{6-10}  = VRT;
+  let Inst{11-15} = VRA;
+  let Inst{16-20} = VRB;
+  let Inst{21-22} = ps;
+  let Inst{23-25} = SD;
+  let Inst{26-31} = xo;
+}
+
+// 8RR:D-Form: [ 1 1 0 // // imm0
+//               PO T XO TX imm1 ].
+class 8RR_DForm_IMM32_XT6<bits<6> opcode, bits<4> xo, dag OOL, dag IOL,
+                          string asmstr, InstrItinClass itin,
+                          list<dag> pattern>
+  : PI<1, opcode, OOL, IOL, asmstr, itin> {
+  bits<6> XT;
+  bits<32> IMM32;
+
+  let Pattern = pattern;
+
+  // The prefix.
+  let Inst{6-7} = 1;
+  let Inst{8-11} = 0;
+  let Inst{12-13} = 0; // reserved
+  let Inst{14-15} = 0; // reserved
+  let Inst{16-31} = IMM32{31-16};
+
+  // The instruction.
+  let Inst{38-42} = XT{4-0};
+  let Inst{43-46} = xo;
+  let Inst{47} = XT{5};
+  let Inst{48-63} = IMM32{15-0};
+}
+
+// 8RR:D-Form: [ 1 1 0 // // imm0
+//               PO T XO IX TX imm1 ].
+class 8RR_DForm_IMM32_XT6_IX<bits<6> opcode, bits<3> xo, dag OOL, dag IOL,
+                             string asmstr, InstrItinClass itin,
+                             list<dag> pattern>
+  : PI<1, opcode, OOL, IOL, asmstr, itin> {
+  bits<6> XT;
+  bit IX;
+  bits<32> IMM32;
+
+  let Pattern = pattern;
+
+  // The prefix.
+  let Inst{6-7} = 1;
+  let Inst{8-11} = 0;
+  let Inst{12-13} = 0; // reserved
+  let Inst{14-15} = 0; // reserved
+  let Inst{16-31} = IMM32{31-16};
+
+  // The instruction.
+  let Inst{38-42} = XT{4-0};
+  let Inst{43-45} = xo;
+  let Inst{46} = IX;
+  let Inst{47} = XT{5};
+  let Inst{48-63} = IMM32{15-0};
+}
+
+class 8RR_XX4Form_XTABC6<bits<6> opcode, bits<2> xo, dag OOL, dag IOL,
+                         string asmstr, InstrItinClass itin, list<dag> pattern>
+  : PI<1, opcode, OOL, IOL, asmstr, itin> {
+  bits<6> XT;
+  bits<6> XA;
+  bits<6> XB;
+  bits<6> XC;
+
+  let Pattern = pattern;
+
+  // The prefix.
+  let Inst{6-7} = 1;
+  let Inst{8-11} = 0;
+  let Inst{12-13} = 0;
+  let Inst{14-31} = 0;
+
+  // The instruction.
+  let Inst{38-42} = XT{4-0};
+  let Inst{43-47} = XA{4-0};
+  let Inst{48-52} = XB{4-0};
+  let Inst{53-57} = XC{4-0};
+  let Inst{58-59} = xo;
+  let Inst{60} = XC{5};
+  let Inst{61} = XA{5};
+  let Inst{62} = XB{5};
+  let Inst{63} = XT{5};
+}
+
+class 8RR_XX4Form_IMM3_XTABC6<bits<6> opcode, bits<2> xo, dag OOL, dag IOL,
+                              string asmstr, InstrItinClass itin,
+                              list<dag> pattern>
+  : PI<1, opcode, OOL, IOL, asmstr, itin> {
+  bits<6> XT;
+  bits<6> XA;
+  bits<6> XB;
+  bits<6> XC;
+  bits<3> IMM;
+
+  let Pattern = pattern;
+
+  // The prefix.
+  let Inst{6-7} = 1;
+  let Inst{8-11} = 0;
+  let Inst{12-13} = 0;
+  let Inst{14-28} = 0;
+  let Inst{29-31} = IMM;
+
+  // The instruction.
+  let Inst{38-42} = XT{4-0};
+  let Inst{43-47} = XA{4-0};
+  let Inst{48-52} = XB{4-0};
+  let Inst{53-57} = XC{4-0};
+  let Inst{58-59} = xo;
+  let Inst{60} = XC{5};
+  let Inst{61} = XA{5};
+  let Inst{62} = XB{5};
+  let Inst{63} = XT{5};
+}
+
 multiclass MLS_DForm_R_SI34_RTA5_MEM_p<bits<6> opcode, dag OOL, dag IOL,
                                        dag PCRel_IOL, string asmstr,
                                        InstrItinClass itin> {
@@ -190,8 +425,8 @@ multiclass 8LS_DForm_R_SI34_XT6_RA5_p<bits<5> opcode, dag OOL, dag IOL,
                                     isPCRel;
 }
 
-def PrefixInstrs : Predicate<"PPCSubTarget->hasPrefixInstrs()">;
-def IsISA3_1 : Predicate<"PPCSubTarget->isISA3_1()">;
+def PrefixInstrs : Predicate<"Subtarget->hasPrefixInstrs()">;
+def IsISA3_1 : Predicate<"Subtarget->isISA3_1()">;
 
 let Predicates = [PrefixInstrs] in {
   let Interpretation64Bit = 1, isCodeGenOnly = 1 in {
@@ -501,7 +736,145 @@ let Predicates = [PCRelativeMemops], AddedComplexity = 500 in {
   def : Pat<(PPCmatpcreladdr pcreladdr:$addr), (PADDI8pc 0, $addr)>;
 }
 
+let Predicates = [PrefixInstrs] in {
+  def XXSPLTIW : 8RR_DForm_IMM32_XT6<32, 3, (outs vsrc:$XT),
+                                     (ins i32imm:$IMM32),
+                                     "xxspltiw $XT, $IMM32", IIC_VecGeneral,
+                                     []>;
+  def XXSPLTIDP : 8RR_DForm_IMM32_XT6<32, 2, (outs vsrc:$XT),
+                                      (ins i32imm:$IMM32),
+                                      "xxspltidp $XT, $IMM32", IIC_VecGeneral,
+                                      [(set v2f64:$XT,
+                                            (PPCxxspltidp i32:$IMM32))]>;
+  def XXSPLTI32DX :
+      8RR_DForm_IMM32_XT6_IX<32, 0, (outs vsrc:$XT),
+                             (ins vsrc:$XTi, u1imm:$IX, i32imm:$IMM32),
+                             "xxsplti32dx $XT, $IX, $IMM32", IIC_VecGeneral,
+                             [(set v2i64:$XT,
+                                   (PPCxxsplti32dx v2i64:$XTi, i32:$IX,
+                                                   i32:$IMM32))]>,
+                             RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">;
+  def XXPERMX :
+    8RR_XX4Form_IMM3_XTABC6<34, 0, (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB,
+                            vsrc:$XC, u3imm:$UIM),
+                            "xxpermx $XT, $XA, $XB, $XC, $UIM",
+                            IIC_VecPerm, []>;
+  def XXBLENDVB :
+    8RR_XX4Form_XTABC6<33, 0, (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB,
+                       vsrc:$XC), "xxblendvb $XT, $XA, $XB, $XC",
+                       IIC_VecGeneral, []>;
+  def XXBLENDVH :
+    8RR_XX4Form_XTABC6<33, 1, (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB,
+                       vsrc:$XC), "xxblendvh $XT, $XA, $XB, $XC",
+                       IIC_VecGeneral, []>;
+  def XXBLENDVW :
+    8RR_XX4Form_XTABC6<33, 2, (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB,
+                       vsrc:$XC), "xxblendvw $XT, $XA, $XB, $XC",
+                       IIC_VecGeneral, []>;
+  def XXBLENDVD :
+    8RR_XX4Form_XTABC6<33, 3, (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB,
+                       vsrc:$XC), "xxblendvd $XT, $XA, $XB, $XC",
+                       IIC_VecGeneral, []>;
+}
+
 let Predicates = [IsISA3_1] in {
+  def VSLDBI : VNForm_VTAB5_SD3<22, 0, (outs vrrc:$VRT),
+                                (ins vrrc:$VRA, vrrc:$VRB, u3imm:$SH),
+                                "vsldbi $VRT, $VRA, $VRB, $SH",
+                                IIC_VecGeneral, 
+                                [(set v16i8:$VRT,
+                                      (int_ppc_altivec_vsldbi v16i8:$VRA,
+                                                              v16i8:$VRB,
+                                                              i32:$SH))]>;
+  def VSRDBI : VNForm_VTAB5_SD3<22, 1, (outs vrrc:$VRT),
+                                (ins vrrc:$VRA, vrrc:$VRB, u3imm:$SH),
+                                "vsrdbi $VRT, $VRA, $VRB, $SH",
+                                IIC_VecGeneral,
+                                [(set v16i8:$VRT,
+                                      (int_ppc_altivec_vsrdbi v16i8:$VRA,
+                                                              v16i8:$VRB, 
+                                                              i32:$SH))]>;
+  def VINSW : 
+    VXForm_VRT5_UIM5_RB5_ins<207, "vinsw",
+                             [(set v4i32:$vD,
+                                   (int_ppc_altivec_vinsw v4i32:$vDi, i64:$rB,
+                                                          timm:$UIM))]>;
+  def VINSD :
+    VXForm_VRT5_UIM5_RB5_ins<463, "vinsd",
+                             [(set v2i64:$vD,
+                                   (int_ppc_altivec_vinsd v2i64:$vDi, i64:$rB,
+                                                          timm:$UIM))]>;
+  def VINSBVLX :
+    VXForm_VTB5_RA5_ins<15, "vinsbvlx",
+                        [(set v16i8:$vD,
+                              (int_ppc_altivec_vinsbvlx v16i8:$vDi, i64:$rA,
+                                                        v16i8:$vB))]>;
+  def VINSBVRX :
+    VXForm_VTB5_RA5_ins<271, "vinsbvrx",
+                        [(set v16i8:$vD,
+                              (int_ppc_altivec_vinsbvrx v16i8:$vDi, i64:$rA,
+                                                        v16i8:$vB))]>;
+  def VINSHVLX :
+    VXForm_VTB5_RA5_ins<79, "vinshvlx",
+                        [(set v8i16:$vD,
+                              (int_ppc_altivec_vinshvlx v8i16:$vDi, i64:$rA,
+                                                        v8i16:$vB))]>;
+  def VINSHVRX :
+    VXForm_VTB5_RA5_ins<335, "vinshvrx",
+                        [(set v8i16:$vD,
+                              (int_ppc_altivec_vinshvrx v8i16:$vDi, i64:$rA,
+                                                        v8i16:$vB))]>;
+  def VINSWVLX :
+    VXForm_VTB5_RA5_ins<143, "vinswvlx",
+                        [(set v4i32:$vD,
+                              (int_ppc_altivec_vinswvlx v4i32:$vDi, i64:$rA,
+                                                        v4i32:$vB))]>;
+  def VINSWVRX :
+    VXForm_VTB5_RA5_ins<399, "vinswvrx",
+                        [(set v4i32:$vD,
+                              (int_ppc_altivec_vinswvrx v4i32:$vDi, i64:$rA,
+                                                        v4i32:$vB))]>;
+  def VINSBLX :
+    VXForm_VRT5_RAB5_ins<527, "vinsblx",
+                         [(set v16i8:$vD,
+                               (int_ppc_altivec_vinsblx v16i8:$vDi, i64:$rA,
+                                                        i64:$rB))]>;
+  def VINSBRX :
+    VXForm_VRT5_RAB5_ins<783, "vinsbrx",
+                         [(set v16i8:$vD,
+                               (int_ppc_altivec_vinsbrx v16i8:$vDi, i64:$rA,
+                                                        i64:$rB))]>;
+  def VINSHLX :
+    VXForm_VRT5_RAB5_ins<591, "vinshlx",
+                         [(set v8i16:$vD,
+                               (int_ppc_altivec_vinshlx v8i16:$vDi, i64:$rA,
+                                                        i64:$rB))]>;
+  def VINSHRX :
+    VXForm_VRT5_RAB5_ins<847, "vinshrx",
+                         [(set v8i16:$vD,
+                               (int_ppc_altivec_vinshrx v8i16:$vDi, i64:$rA,
+                                                        i64:$rB))]>;
+  def VINSWLX :
+    VXForm_VRT5_RAB5_ins<655, "vinswlx",
+                         [(set v4i32:$vD,
+                               (int_ppc_altivec_vinswlx v4i32:$vDi, i64:$rA,
+                                                        i64:$rB))]>;
+  def VINSWRX :
+    VXForm_VRT5_RAB5_ins<911, "vinswrx",
+                         [(set v4i32:$vD,
+                               (int_ppc_altivec_vinswrx v4i32:$vDi, i64:$rA,
+                                                        i64:$rB))]>;
+  def VINSDLX :
+    VXForm_VRT5_RAB5_ins<719, "vinsdlx",
+                         [(set v2i64:$vD,
+                               (int_ppc_altivec_vinsdlx v2i64:$vDi, i64:$rA,
+                                                        i64:$rB))]>;
+  def VINSDRX :
+    VXForm_VRT5_RAB5_ins<975, "vinsdrx",
+                         [(set v2i64:$vD,
+                               (int_ppc_altivec_vinsdrx v2i64:$vDi, i64:$rA,
+                                                        i64:$rB))]>;
+
    def VPDEPD : VXForm_1<1485, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
                          "vpdepd $vD, $vA, $vB", IIC_VecGeneral,
                          [(set v2i64:$vD,
@@ -516,6 +889,51 @@ let Predicates = [IsISA3_1] in {
    def PEXTD : XForm_6<31, 188, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB),
                        "pextd $rA, $rS, $rB", IIC_IntGeneral,
                        [(set i64:$rA, (int_ppc_pextd i64:$rS, i64:$rB))]>;
+   def VCFUGED : VXForm_1<1357, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                          "vcfuged $vD, $vA, $vB", IIC_VecGeneral,
+                          [(set v2i64:$vD,
+                          (int_ppc_altivec_vcfuged v2i64:$vA, v2i64:$vB))]>;
+   def VGNB : VXForm_RD5_N3_VB5<1228, (outs g8rc:$rD), (ins vrrc:$vB, u3imm:$N),
+                                "vgnb $rD, $vB, $N", IIC_VecGeneral,
+                                [(set i64:$rD,
+                                (int_ppc_altivec_vgnb v1i128:$vB, timm:$N))]>;
+   def CFUGED : XForm_6<31, 220, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB),
+                        "cfuged $rA, $rS, $rB", IIC_IntGeneral,
+                        [(set i64:$rA, (int_ppc_cfuged i64:$rS, i64:$rB))]>;
+   def XXEVAL :
+     8RR_XX4Form_IMM8_XTAB6<34, 1, (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB,
+                            vsrc:$XC, u8imm:$IMM),
+                            "xxeval $XT, $XA, $XB, $XC, $IMM", IIC_VecGeneral,
+                            [(set v2i64:$XT, (int_ppc_vsx_xxeval v2i64:$XA,
+                                  v2i64:$XB, v2i64:$XC, timm:$IMM))]>;
+   def VCLZDM : VXForm_1<1924, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                         "vclzdm $vD, $vA, $vB", IIC_VecGeneral,
+                         [(set v2i64:$vD,
+                         (int_ppc_altivec_vclzdm v2i64:$vA, v2i64:$vB))]>;
+   def VCTZDM : VXForm_1<1988, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                         "vctzdm $vD, $vA, $vB", IIC_VecGeneral,
+                         [(set v2i64:$vD,
+                         (int_ppc_altivec_vctzdm v2i64:$vA, v2i64:$vB))]>;
+   def CNTLZDM : XForm_6<31, 59, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB),
+                         "cntlzdm $rA, $rS, $rB", IIC_IntGeneral,
+                         [(set i64:$rA,
+                         (int_ppc_cntlzdm i64:$rS, i64:$rB))]>;
+   def CNTTZDM : XForm_6<31, 571, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB),
+                         "cnttzdm $rA, $rS, $rB", IIC_IntGeneral,
+                         [(set i64:$rA,
+                         (int_ppc_cnttzdm i64:$rS, i64:$rB))]>;
+   def XXGENPCVBM :
+     XForm_XT6_IMM5_VB5<60, 916, (outs vsrc:$XT), (ins vrrc:$VRB, s5imm:$IMM),
+                        "xxgenpcvbm $XT, $VRB, $IMM", IIC_VecGeneral, []>;
+   def XXGENPCVHM :
+     XForm_XT6_IMM5_VB5<60, 917, (outs vsrc:$XT), (ins vrrc:$VRB, s5imm:$IMM),
+                        "xxgenpcvhm $XT, $VRB, $IMM", IIC_VecGeneral, []>;
+   def XXGENPCVWM :
+     XForm_XT6_IMM5_VB5<60, 948, (outs vsrc:$XT), (ins vrrc:$VRB, s5imm:$IMM),
+                        "xxgenpcvwm $XT, $VRB, $IMM", IIC_VecGeneral, []>;
+   def XXGENPCVDM :
+     XForm_XT6_IMM5_VB5<60, 949, (outs vsrc:$XT), (ins vrrc:$VRB, s5imm:$IMM),
+                        "xxgenpcvdm $XT, $VRB, $IMM", IIC_VecGeneral, []>;
    def VCLRLB : VXForm_1<397, (outs vrrc:$vD), (ins vrrc:$vA, gprc:$rB),
                          "vclrlb $vD, $vA, $rB", IIC_VecGeneral,
                          [(set v16i8:$vD,
@@ -524,4 +942,70 @@ let Predicates = [IsISA3_1] in {
                          "vclrrb $vD, $vA, $rB", IIC_VecGeneral,
                          [(set v16i8:$vD,
                                (int_ppc_altivec_vclrrb v16i8:$vA, i32:$rB))]>;
+
+  // The XFormMemOp flag for the following 8 instructions is set on
+  // the instruction format.
+  let mayLoad = 1, mayStore = 0 in {
+    def LXVRBX : X_XT6_RA5_RB5<31, 13, "lxvrbx", vsrc, []>;
+    def LXVRHX : X_XT6_RA5_RB5<31, 45, "lxvrhx", vsrc, []>;
+    def LXVRWX : X_XT6_RA5_RB5<31, 77, "lxvrwx", vsrc, []>;
+    def LXVRDX : X_XT6_RA5_RB5<31, 109, "lxvrdx", vsrc, []>;
+  }
+
+  let mayLoad = 0, mayStore = 1 in {
+    def STXVRBX : X_XS6_RA5_RB5<31, 141, "stxvrbx", vsrc, []>;
+    def STXVRHX : X_XS6_RA5_RB5<31, 173, "stxvrhx", vsrc, []>;
+    def STXVRWX : X_XS6_RA5_RB5<31, 205, "stxvrwx", vsrc, []>;
+    def STXVRDX : X_XS6_RA5_RB5<31, 237, "stxvrdx", vsrc, []>;
+  }
+}
+
+
+
+//---------------------------- Anonymous Patterns ----------------------------//
+let Predicates = [IsISA3_1] in {
+  def : Pat<(v16i8 (int_ppc_vsx_xxgenpcvbm v16i8:$VRB, imm:$IMM)),
+            (v16i8 (COPY_TO_REGCLASS (XXGENPCVBM $VRB, imm:$IMM), VRRC))>;
+  def : Pat<(v8i16 (int_ppc_vsx_xxgenpcvhm v8i16:$VRB, imm:$IMM)),
+            (v8i16 (COPY_TO_REGCLASS (XXGENPCVHM $VRB, imm:$IMM), VRRC))>;
+  def : Pat<(v4i32 (int_ppc_vsx_xxgenpcvwm v4i32:$VRB, imm:$IMM)),
+            (v4i32 (COPY_TO_REGCLASS (XXGENPCVWM $VRB, imm:$IMM), VRRC))>;
+  def : Pat<(v2i64 (int_ppc_vsx_xxgenpcvdm v2i64:$VRB, imm:$IMM)),
+            (v2i64 (COPY_TO_REGCLASS (XXGENPCVDM $VRB, imm:$IMM), VRRC))>;
 }
+
+let AddedComplexity = 400, Predicates = [PrefixInstrs] in {
+ def : Pat<(v4i32 (build_vector i32immNonAllOneNonZero:$A,
+                                i32immNonAllOneNonZero:$A,
+                                i32immNonAllOneNonZero:$A,
+                                i32immNonAllOneNonZero:$A)),
+           (v4i32 (XXSPLTIW imm:$A))>;
+ def : Pat<(f32 nzFPImmAsi32:$A),
+           (COPY_TO_REGCLASS (XXSPLTIDP (getFPAs32BitInt fpimm:$A)),
+                             VSFRC)>;
+ def : Pat<(f64 nzFPImmAsi32:$A),
+           (COPY_TO_REGCLASS (XXSPLTIDP (getFPAs32BitInt fpimm:$A)),
+                             VSFRC)>;
+}
+
+let Predicates = [PrefixInstrs] in {
+  def : Pat<(v16i8 (int_ppc_vsx_xxpermx v16i8:$A, v16i8:$B, v16i8:$C, timm:$D)),
+            (COPY_TO_REGCLASS (XXPERMX (COPY_TO_REGCLASS $A, VSRC),
+                                       (COPY_TO_REGCLASS $B, VSRC),
+                                       (COPY_TO_REGCLASS $C, VSRC), $D), VSRC)>;
+  def : Pat<(v16i8 (int_ppc_vsx_xxblendvb v16i8:$A, v16i8:$B, v16i8:$C)),
+            (COPY_TO_REGCLASS
+                   (XXBLENDVB (COPY_TO_REGCLASS $A, VSRC),
+                              (COPY_TO_REGCLASS $B, VSRC),
+                              (COPY_TO_REGCLASS $C, VSRC)), VSRC)>;
+  def : Pat<(v8i16 (int_ppc_vsx_xxblendvh v8i16:$A, v8i16:$B, v8i16:$C)),
+            (COPY_TO_REGCLASS
+                   (XXBLENDVH (COPY_TO_REGCLASS $A, VSRC),
+                              (COPY_TO_REGCLASS $B, VSRC),
+                              (COPY_TO_REGCLASS $C, VSRC)), VSRC)>;
+  def : Pat<(int_ppc_vsx_xxblendvw v4i32:$A, v4i32:$B, v4i32:$C),
+            (XXBLENDVW $A, $B, $C)>;
+  def : Pat<(int_ppc_vsx_xxblendvd v2i64:$A, v2i64:$B, v2i64:$C),
+            (XXBLENDVD $A, $B, $C)>;
+}
+
diff --git a/llvm/lib/Target/PowerPC/PPCInstrQPX.td b/llvm/lib/Target/PowerPC/PPCInstrQPX.td
index d565ed433a38c..2265af2815cb5 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrQPX.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrQPX.td
@@ -1,9 +1,9 @@
 //===- PPCInstrQPX.td - The PowerPC QPX Extension --*- tablegen -*-===//
-// 
+//
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-// 
+//
 //===----------------------------------------------------------------------===//
 //
 // This file describes the QPX extension to the PowerPC instruction set.
@@ -101,7 +101,7 @@ let FastIselShouldIgnore = 1 in // FastIsel should ignore all u12 instrs.
 //===----------------------------------------------------------------------===//
 // Instruction Definitions.
 
-def HasQPX : Predicate<"PPCSubTarget->hasQPX()">;
+def HasQPX : Predicate<"Subtarget->hasQPX()">;
 let Predicates = [HasQPX] in {
 let DecoderNamespace = "QPX" in {
 let hasSideEffects = 0 in { // QPX instructions don't have side effects.
@@ -1210,4 +1210,3 @@ def : Pat<(fmaxnum v4f32:$FRA, v4f32:$FRB),
                                  (QVFTSTNANbs $FRB, $FRB), (i32 7)),
                    $FRB, $FRA)>;
 }
-
diff --git a/llvm/lib/Target/PowerPC/PPCInstrSPE.td b/llvm/lib/Target/PowerPC/PPCInstrSPE.td
index 935c3044ae470..858eb0c9fe500 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrSPE.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrSPE.td
@@ -158,7 +158,7 @@ def EFDCFSF        : EFXForm_2a<755, (outs sperc:$RT), (ins spe4rc:$RB),
 
 def EFDCFSI        : EFXForm_2a<753, (outs sperc:$RT), (ins gprc:$RB),
                                 "efdcfsi $RT, $RB", IIC_FPDGeneral,
-                                [(set f64:$RT, (sint_to_fp i32:$RB))]>;
+                                [(set f64:$RT, (any_sint_to_fp i32:$RB))]>;
 
 def EFDCFSID       : EFXForm_2a<739, (outs sperc:$RT), (ins gprc:$RB),
                                 "efdcfsid $RT, $RB", IIC_FPDGeneral,
@@ -169,7 +169,7 @@ def EFDCFUF        : EFXForm_2a<754, (outs sperc:$RT), (ins spe4rc:$RB),
 
 def EFDCFUI        : EFXForm_2a<752, (outs sperc:$RT), (ins gprc:$RB),
                                 "efdcfui $RT, $RB", IIC_FPDGeneral,
-                                [(set f64:$RT, (uint_to_fp i32:$RB))]>;
+                                [(set f64:$RT, (any_uint_to_fp i32:$RB))]>;
 
 def EFDCFUID       : EFXForm_2a<738, (outs sperc:$RT), (ins gprc:$RB),
                                 "efdcfuid $RT, $RB", IIC_FPDGeneral,
@@ -197,7 +197,7 @@ def EFDCTSIDZ      : EFXForm_2a<747, (outs gprc:$RT), (ins sperc:$RB),
 
 def EFDCTSIZ       : EFXForm_2a<762, (outs gprc:$RT), (ins sperc:$RB),
                                 "efdctsiz $RT, $RB", IIC_FPDGeneral,
-                                [(set i32:$RT, (fp_to_sint f64:$RB))]>;
+                                [(set i32:$RT, (any_fp_to_sint f64:$RB))]>;
 
 def EFDCTUF        : EFXForm_2a<758, (outs sperc:$RT), (ins spe4rc:$RB),
                                 "efdctuf $RT, $RB", IIC_FPDGeneral, []>;
@@ -212,7 +212,7 @@ def EFDCTUIDZ      : EFXForm_2a<746, (outs gprc:$RT), (ins sperc:$RB),
 
 def EFDCTUIZ       : EFXForm_2a<760, (outs gprc:$RT), (ins sperc:$RB),
                                 "efdctuiz $RT, $RB", IIC_FPDGeneral,
-                                [(set i32:$RT, (fp_to_uint f64:$RB))]>;
+                                [(set i32:$RT, (any_fp_to_uint f64:$RB))]>;
 
 def EFDDIV         : EFXForm_1<745, (outs sperc:$RT), (ins sperc:$RA, sperc:$RB),
                                "efddiv $RT, $RA, $RB", IIC_FPDivD,
@@ -261,14 +261,14 @@ def EFSCFSF        : EFXForm_2a<723, (outs spe4rc:$RT), (ins spe4rc:$RB),
 
 def EFSCFSI        : EFXForm_2a<721, (outs spe4rc:$RT), (ins gprc:$RB),
                                 "efscfsi $RT, $RB", IIC_FPSGeneral,
-                                [(set f32:$RT, (sint_to_fp i32:$RB))]>;
+                                [(set f32:$RT, (any_sint_to_fp i32:$RB))]>;
 
 def EFSCFUF        : EFXForm_2a<722, (outs spe4rc:$RT), (ins spe4rc:$RB),
                                 "efscfuf $RT, $RB", IIC_FPSGeneral, []>;
 
 def EFSCFUI        : EFXForm_2a<720, (outs spe4rc:$RT), (ins gprc:$RB),
                                 "efscfui $RT, $RB", IIC_FPSGeneral,
-                                [(set f32:$RT, (uint_to_fp i32:$RB))]>;
+                                [(set f32:$RT, (any_uint_to_fp i32:$RB))]>;
 
 let isCompare = 1 in {
 def EFSCMPEQ       : EFXForm_3<718, (outs crrc:$crD), (ins spe4rc:$RA, spe4rc:$RB),
@@ -288,7 +288,7 @@ def EFSCTSI        : EFXForm_2a<725, (outs gprc:$RT), (ins spe4rc:$RB),
 
 def EFSCTSIZ       : EFXForm_2a<730, (outs gprc:$RT), (ins spe4rc:$RB),
                                 "efsctsiz $RT, $RB", IIC_FPSGeneral,
-                                [(set i32:$RT, (fp_to_sint f32:$RB))]>;
+                                [(set i32:$RT, (any_fp_to_sint f32:$RB))]>;
 
 def EFSCTUF        : EFXForm_2a<726, (outs sperc:$RT), (ins spe4rc:$RB),
                                 "efsctuf $RT, $RB", IIC_FPSGeneral, []>;
@@ -299,7 +299,7 @@ def EFSCTUI        : EFXForm_2a<724, (outs gprc:$RT), (ins spe4rc:$RB),
 
 def EFSCTUIZ       : EFXForm_2a<728, (outs gprc:$RT), (ins spe4rc:$RB),
                                 "efsctuiz $RT, $RB", IIC_FPSGeneral,
-                                [(set i32:$RT, (fp_to_uint f32:$RB))]>;
+                                [(set i32:$RT, (any_fp_to_uint f32:$RB))]>;
 
 def EFSDIV         : EFXForm_1<713, (outs spe4rc:$RT), (ins spe4rc:$RA, spe4rc:$RB),
                                "efsdiv $RT, $RA, $RB", IIC_FPDivD,
diff --git a/llvm/lib/Target/PowerPC/PPCInstrVSX.td b/llvm/lib/Target/PowerPC/PPCInstrVSX.td
index c43b2716cb370..9ba5058a6f812 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrVSX.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrVSX.td
@@ -1,9 +1,9 @@
 //===- PPCInstrVSX.td - The PowerPC VSX Extension --*- tablegen -*-===//
-// 
+//
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-// 
+//
 //===----------------------------------------------------------------------===//
 //
 // This file describes the VSX extension to the PowerPC instruction set.
@@ -142,15 +142,15 @@ def PPCSToV : SDNode<"PPCISD::SCALAR_TO_VECTOR_PERMUTED",
                      SDTypeProfile<1, 1, []>, []>;
 
 //-------------------------- Predicate definitions ---------------------------//
-def HasVSX : Predicate<"PPCSubTarget->hasVSX()">;
-def IsLittleEndian : Predicate<"PPCSubTarget->isLittleEndian()">;
-def IsBigEndian : Predicate<"!PPCSubTarget->isLittleEndian()">;
-def HasOnlySwappingMemOps : Predicate<"!PPCSubTarget->hasP9Vector()">;
-def HasP8Vector : Predicate<"PPCSubTarget->hasP8Vector()">;
-def HasDirectMove : Predicate<"PPCSubTarget->hasDirectMove()">;
-def NoP9Vector : Predicate<"!PPCSubTarget->hasP9Vector()">;
-def HasP9Vector : Predicate<"PPCSubTarget->hasP9Vector()">;
-def NoP9Altivec : Predicate<"!PPCSubTarget->hasP9Altivec()">;
+def HasVSX : Predicate<"Subtarget->hasVSX()">;
+def IsLittleEndian : Predicate<"Subtarget->isLittleEndian()">;
+def IsBigEndian : Predicate<"!Subtarget->isLittleEndian()">;
+def HasOnlySwappingMemOps : Predicate<"!Subtarget->hasP9Vector()">;
+def HasP8Vector : Predicate<"Subtarget->hasP8Vector()">;
+def HasDirectMove : Predicate<"Subtarget->hasDirectMove()">;
+def NoP9Vector : Predicate<"!Subtarget->hasP9Vector()">;
+def HasP9Vector : Predicate<"Subtarget->hasP9Vector()">;
+def NoP9Altivec : Predicate<"!Subtarget->hasP9Altivec()">;
 
 //--------------------- VSX-specific instruction formats ---------------------//
 // By default, all VSX instructions are to be selected over their Altivec
diff --git a/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp b/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp
index 1a9b50cb945fc..daf88589bb52b 100644
--- a/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp
@@ -19,31 +19,31 @@ static cl::opt<bool> PPCDisableNonVolatileCR(
     cl::init(false), cl::Hidden);
 
 void PPCFunctionInfo::anchor() {}
-PPCFunctionInfo::PPCFunctionInfo(MachineFunction &MF)
-    : DisableNonVolatileCR(PPCDisableNonVolatileCR), MF(MF) {}
+PPCFunctionInfo::PPCFunctionInfo(const MachineFunction &MF)
+    : DisableNonVolatileCR(PPCDisableNonVolatileCR) {}
 
-MCSymbol *PPCFunctionInfo::getPICOffsetSymbol() const {
+MCSymbol *PPCFunctionInfo::getPICOffsetSymbol(MachineFunction &MF) const {
   const DataLayout &DL = MF.getDataLayout();
   return MF.getContext().getOrCreateSymbol(Twine(DL.getPrivateGlobalPrefix()) +
                                            Twine(MF.getFunctionNumber()) +
                                            "$poff");
 }
 
-MCSymbol *PPCFunctionInfo::getGlobalEPSymbol() const {
+MCSymbol *PPCFunctionInfo::getGlobalEPSymbol(MachineFunction &MF) const {
   const DataLayout &DL = MF.getDataLayout();
   return MF.getContext().getOrCreateSymbol(Twine(DL.getPrivateGlobalPrefix()) +
                                            "func_gep" +
                                            Twine(MF.getFunctionNumber()));
 }
 
-MCSymbol *PPCFunctionInfo::getLocalEPSymbol() const {
+MCSymbol *PPCFunctionInfo::getLocalEPSymbol(MachineFunction &MF) const {
   const DataLayout &DL = MF.getDataLayout();
   return MF.getContext().getOrCreateSymbol(Twine(DL.getPrivateGlobalPrefix()) +
                                            "func_lep" +
                                            Twine(MF.getFunctionNumber()));
 }
 
-MCSymbol *PPCFunctionInfo::getTOCOffsetSymbol() const {
+MCSymbol *PPCFunctionInfo::getTOCOffsetSymbol(MachineFunction &MF) const {
   const DataLayout &DL = MF.getDataLayout();
   return MF.getContext().getOrCreateSymbol(Twine(DL.getPrivateGlobalPrefix()) +
                                            "func_toc" +
diff --git a/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.h b/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.h
index 3bbac5316efa9..29ca53e273d75 100644
--- a/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.h
@@ -118,9 +118,6 @@ class PPCFunctionInfo : public MachineFunctionInfo {
   /// 64-bit SVR4 ABI.
   SmallVector<Register, 3> MustSaveCRs;
 
-  /// Hold onto our MachineFunction context.
-  MachineFunction &MF;
-
   /// Whether this uses the PIC Base register or not.
   bool UsesPICBase = false;
 
@@ -129,7 +126,7 @@ class PPCFunctionInfo : public MachineFunctionInfo {
   std::vector<std::pair<Register, ISD::ArgFlagsTy>> LiveInAttrs;
 
 public:
-  explicit PPCFunctionInfo(MachineFunction &MF);
+  explicit PPCFunctionInfo(const MachineFunction &MF);
 
   int getFramePointerSaveIndex() const { return FramePointerSaveIndex; }
   void setFramePointerSaveIndex(int Idx) { FramePointerSaveIndex = Idx; }
@@ -225,11 +222,11 @@ class PPCFunctionInfo : public MachineFunctionInfo {
   void setUsesPICBase(bool uses) { UsesPICBase = uses; }
   bool usesPICBase() const { return UsesPICBase; }
 
-  MCSymbol *getPICOffsetSymbol() const;
+  MCSymbol *getPICOffsetSymbol(MachineFunction &MF) const;
 
-  MCSymbol *getGlobalEPSymbol() const;
-  MCSymbol *getLocalEPSymbol() const;
-  MCSymbol *getTOCOffsetSymbol() const;
+  MCSymbol *getGlobalEPSymbol(MachineFunction &MF) const;
+  MCSymbol *getLocalEPSymbol(MachineFunction &MF) const;
+  MCSymbol *getTOCOffsetSymbol(MachineFunction &MF) const;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp b/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
index 2b1ea8c2ec403..35f5e1fbebcdf 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
@@ -480,6 +480,63 @@ void PPCRegisterInfo::lowerDynamicAlloc(MachineBasicBlock::iterator II) const {
 
   // Get the maximum call stack size.
   unsigned maxCallFrameSize = MFI.getMaxCallFrameSize();
+  Align MaxAlign = MFI.getMaxAlign();
+  assert(isAligned(MaxAlign, maxCallFrameSize) &&
+         "Maximum call-frame size not sufficiently aligned");
+  (void)MaxAlign;
+
+  const TargetRegisterClass *G8RC = &PPC::G8RCRegClass;
+  const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
+  Register Reg = MF.getRegInfo().createVirtualRegister(LP64 ? G8RC : GPRC);
+  bool KillNegSizeReg = MI.getOperand(1).isKill();
+  Register NegSizeReg = MI.getOperand(1).getReg();
+
+  prepareDynamicAlloca(II, NegSizeReg, KillNegSizeReg, Reg);
+  // Grow the stack and update the stack pointer link, then determine the
+  // address of new allocated space.
+  if (LP64) {
+    BuildMI(MBB, II, dl, TII.get(PPC::STDUX), PPC::X1)
+        .addReg(Reg, RegState::Kill)
+        .addReg(PPC::X1)
+        .addReg(NegSizeReg, getKillRegState(KillNegSizeReg));
+    BuildMI(MBB, II, dl, TII.get(PPC::ADDI8), MI.getOperand(0).getReg())
+        .addReg(PPC::X1)
+        .addImm(maxCallFrameSize);
+  } else {
+    BuildMI(MBB, II, dl, TII.get(PPC::STWUX), PPC::R1)
+        .addReg(Reg, RegState::Kill)
+        .addReg(PPC::R1)
+        .addReg(NegSizeReg, getKillRegState(KillNegSizeReg));
+    BuildMI(MBB, II, dl, TII.get(PPC::ADDI), MI.getOperand(0).getReg())
+        .addReg(PPC::R1)
+        .addImm(maxCallFrameSize);
+  }
+
+  // Discard the DYNALLOC instruction.
+  MBB.erase(II);
+}
+
+/// To accomplish dynamic stack allocation, we have to calculate exact size
+/// subtracted from the stack pointer according alignment information and get
+/// previous frame pointer.
+void PPCRegisterInfo::prepareDynamicAlloca(MachineBasicBlock::iterator II,
+                                           Register &NegSizeReg,
+                                           bool &KillNegSizeReg,
+                                           Register &FramePointer) const {
+  // Get the instruction.
+  MachineInstr &MI = *II;
+  // Get the instruction's basic block.
+  MachineBasicBlock &MBB = *MI.getParent();
+  // Get the basic block's function.
+  MachineFunction &MF = *MBB.getParent();
+  // Get the frame info.
+  MachineFrameInfo &MFI = MF.getFrameInfo();
+  const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>();
+  // Get the instruction info.
+  const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
+  // Determine whether 64-bit pointers are used.
+  bool LP64 = TM.isPPC64();
+  DebugLoc dl = MI.getDebugLoc();
   // Get the total frame size.
   unsigned FrameSize = MFI.getStackSize();
 
@@ -487,8 +544,6 @@ void PPCRegisterInfo::lowerDynamicAlloc(MachineBasicBlock::iterator II) const {
   const PPCFrameLowering *TFI = getFrameLowering(MF);
   Align TargetAlign = TFI->getStackAlign();
   Align MaxAlign = MFI.getMaxAlign();
-  assert(isAligned(MaxAlign, maxCallFrameSize) &&
-         "Maximum call-frame size not sufficiently aligned");
 
   // Determine the previous frame's address.  If FrameSize can't be
   // represented as 16 bits or we need special alignment, then we load the
@@ -498,32 +553,26 @@ void PPCRegisterInfo::lowerDynamicAlloc(MachineBasicBlock::iterator II) const {
   // Fortunately, a frame greater than 32K is rare.
   const TargetRegisterClass *G8RC = &PPC::G8RCRegClass;
   const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
-  Register Reg = MF.getRegInfo().createVirtualRegister(LP64 ? G8RC : GPRC);
 
   if (MaxAlign < TargetAlign && isInt<16>(FrameSize)) {
     if (LP64)
-      BuildMI(MBB, II, dl, TII.get(PPC::ADDI8), Reg)
-        .addReg(PPC::X31)
-        .addImm(FrameSize);
+      BuildMI(MBB, II, dl, TII.get(PPC::ADDI8), FramePointer)
+          .addReg(PPC::X31)
+          .addImm(FrameSize);
     else
-      BuildMI(MBB, II, dl, TII.get(PPC::ADDI), Reg)
-        .addReg(PPC::R31)
-        .addImm(FrameSize);
+      BuildMI(MBB, II, dl, TII.get(PPC::ADDI), FramePointer)
+          .addReg(PPC::R31)
+          .addImm(FrameSize);
   } else if (LP64) {
-    BuildMI(MBB, II, dl, TII.get(PPC::LD), Reg)
-      .addImm(0)
-      .addReg(PPC::X1);
+    BuildMI(MBB, II, dl, TII.get(PPC::LD), FramePointer)
+        .addImm(0)
+        .addReg(PPC::X1);
   } else {
-    BuildMI(MBB, II, dl, TII.get(PPC::LWZ), Reg)
-      .addImm(0)
-      .addReg(PPC::R1);
+    BuildMI(MBB, II, dl, TII.get(PPC::LWZ), FramePointer)
+        .addImm(0)
+        .addReg(PPC::R1);
   }
-
-  bool KillNegSizeReg = MI.getOperand(1).isKill();
-  Register NegSizeReg = MI.getOperand(1).getReg();
-
-  // Grow the stack and update the stack pointer link, then determine the
-  // address of new allocated space.
+  // Determine the actual NegSizeReg according to alignment info.
   if (LP64) {
     if (MaxAlign > TargetAlign) {
       unsigned UnalNegSizeReg = NegSizeReg;
@@ -537,18 +586,10 @@ void PPCRegisterInfo::lowerDynamicAlloc(MachineBasicBlock::iterator II) const {
       unsigned NegSizeReg1 = NegSizeReg;
       NegSizeReg = MF.getRegInfo().createVirtualRegister(G8RC);
       BuildMI(MBB, II, dl, TII.get(PPC::AND8), NegSizeReg)
-        .addReg(UnalNegSizeReg, getKillRegState(KillNegSizeReg))
-        .addReg(NegSizeReg1, RegState::Kill);
+          .addReg(UnalNegSizeReg, getKillRegState(KillNegSizeReg))
+          .addReg(NegSizeReg1, RegState::Kill);
       KillNegSizeReg = true;
     }
-
-    BuildMI(MBB, II, dl, TII.get(PPC::STDUX), PPC::X1)
-      .addReg(Reg, RegState::Kill)
-      .addReg(PPC::X1)
-      .addReg(NegSizeReg, getKillRegState(KillNegSizeReg));
-    BuildMI(MBB, II, dl, TII.get(PPC::ADDI8), MI.getOperand(0).getReg())
-      .addReg(PPC::X1)
-      .addImm(maxCallFrameSize);
   } else {
     if (MaxAlign > TargetAlign) {
       unsigned UnalNegSizeReg = NegSizeReg;
@@ -562,21 +603,42 @@ void PPCRegisterInfo::lowerDynamicAlloc(MachineBasicBlock::iterator II) const {
       unsigned NegSizeReg1 = NegSizeReg;
       NegSizeReg = MF.getRegInfo().createVirtualRegister(GPRC);
       BuildMI(MBB, II, dl, TII.get(PPC::AND), NegSizeReg)
-        .addReg(UnalNegSizeReg, getKillRegState(KillNegSizeReg))
-        .addReg(NegSizeReg1, RegState::Kill);
+          .addReg(UnalNegSizeReg, getKillRegState(KillNegSizeReg))
+          .addReg(NegSizeReg1, RegState::Kill);
       KillNegSizeReg = true;
     }
+  }
+}
 
-    BuildMI(MBB, II, dl, TII.get(PPC::STWUX), PPC::R1)
-      .addReg(Reg, RegState::Kill)
-      .addReg(PPC::R1)
-      .addReg(NegSizeReg, getKillRegState(KillNegSizeReg));
-    BuildMI(MBB, II, dl, TII.get(PPC::ADDI), MI.getOperand(0).getReg())
-      .addReg(PPC::R1)
-      .addImm(maxCallFrameSize);
+void PPCRegisterInfo::lowerPrepareProbedAlloca(
+    MachineBasicBlock::iterator II) const {
+  MachineInstr &MI = *II;
+  // Get the instruction's basic block.
+  MachineBasicBlock &MBB = *MI.getParent();
+  // Get the basic block's function.
+  MachineFunction &MF = *MBB.getParent();
+  const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>();
+  // Get the instruction info.
+  const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
+  // Determine whether 64-bit pointers are used.
+  bool LP64 = TM.isPPC64();
+  DebugLoc dl = MI.getDebugLoc();
+  Register FramePointer = MI.getOperand(0).getReg();
+  Register FinalStackPtr = MI.getOperand(1).getReg();
+  bool KillNegSizeReg = MI.getOperand(2).isKill();
+  Register NegSizeReg = MI.getOperand(2).getReg();
+  prepareDynamicAlloca(II, NegSizeReg, KillNegSizeReg, FramePointer);
+  if (LP64) {
+    BuildMI(MBB, II, dl, TII.get(PPC::ADD8), FinalStackPtr)
+        .addReg(PPC::X1)
+        .addReg(NegSizeReg, getKillRegState(KillNegSizeReg));
+
+  } else {
+    BuildMI(MBB, II, dl, TII.get(PPC::ADD4), FinalStackPtr)
+        .addReg(PPC::R1)
+        .addReg(NegSizeReg, getKillRegState(KillNegSizeReg));
   }
 
-  // Discard the DYNALLOC instruction.
   MBB.erase(II);
 }
 
@@ -1020,6 +1082,13 @@ PPCRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
     return;
   }
 
+  if (FPSI && FrameIndex == FPSI &&
+      (OpC == PPC::PREPARE_PROBED_ALLOCA_64 ||
+       OpC == PPC::PREPARE_PROBED_ALLOCA_32)) {
+    lowerPrepareProbedAlloca(II);
+    return;
+  }
+
   // Special case for pseudo-ops SPILL_CR and RESTORE_CR, etc.
   if (OpC == PPC::SPILL_CR) {
     lowerCRSpilling(II, FrameIndex);
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.h b/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
index 46aaa8b4e1539..61acd955e1cba 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
@@ -107,6 +107,10 @@ class PPCRegisterInfo : public PPCGenRegisterInfo {
 
   void lowerDynamicAlloc(MachineBasicBlock::iterator II) const;
   void lowerDynamicAreaOffset(MachineBasicBlock::iterator II) const;
+  void prepareDynamicAlloca(MachineBasicBlock::iterator II,
+                            Register &NegSizeReg, bool &KillNegSizeReg,
+                            Register &FramePointer) const;
+  void lowerPrepareProbedAlloca(MachineBasicBlock::iterator II) const;
   void lowerCRSpilling(MachineBasicBlock::iterator II,
                        unsigned FrameIndex) const;
   void lowerCRRestore(MachineBasicBlock::iterator II,
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
index 2acff5d11ce07..3836cc960394f 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
@@ -180,7 +180,7 @@ void PPCSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
 bool PPCSubtarget::enableMachineScheduler() const { return true; }
 
 bool PPCSubtarget::enableMachinePipeliner() const {
-  return (CPUDirective == PPC::DIR_PWR9) && EnableMachinePipeliner;
+  return getSchedModel().hasInstrSchedModel() && EnableMachinePipeliner;
 }
 
 bool PPCSubtarget::useDFAforSMS() const { return false; }
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index a13050ec11514..53556ffc267df 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -568,6 +568,10 @@ void PPCTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
   BaseT::getUnrollingPreferences(L, SE, UP);
 }
 
+void PPCTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                                       TTI::PeelingPreferences &PP) {
+  BaseT::getPeelingPreferences(L, SE, PP);
+}
 // This function returns true to allow using coldcc calling convention.
 // Returning true results in coldcc being used for functions which are cold at
 // all call sites when the callers of the functions are not calling any other
@@ -954,14 +958,10 @@ int PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
   return Cost;
 }
 
-int PPCTTIImpl::getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
-                                           unsigned Factor,
-                                           ArrayRef<unsigned> Indices,
-                                           unsigned Alignment,
-                                           unsigned AddressSpace,
-                                           TTI::TargetCostKind CostKind,
-                                           bool UseMaskForCond,
-                                           bool UseMaskForGaps) {
+int PPCTTIImpl::getInterleavedMemoryOpCost(
+    unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+    Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
+    bool UseMaskForCond, bool UseMaskForGaps) {
   if (UseMaskForCond || UseMaskForGaps)
     return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
                                              Alignment, AddressSpace, CostKind,
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index 1bc9970e9cc53..d998521084e1c 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -66,6 +66,8 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
                   TargetLibraryInfo *LibInfo);
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP);
+  void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                             TTI::PeelingPreferences &PP);
   bool isLSRCostLess(TargetTransformInfo::LSRCost &C1,
                      TargetTransformInfo::LSRCost &C2);
 
@@ -112,14 +114,11 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
                       unsigned AddressSpace,
                       TTI::TargetCostKind CostKind,
                       const Instruction *I = nullptr);
-  int getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
-                                 unsigned Factor,
-                                 ArrayRef<unsigned> Indices,
-                                 unsigned Alignment,
-                                 unsigned AddressSpace,
-                                 TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
-                                 bool UseMaskForCond = false,
-                                 bool UseMaskForGaps = false);
+  int getInterleavedMemoryOpCost(
+      unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+      Align Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
+      bool UseMaskForCond = false, bool UseMaskForGaps = false);
   unsigned getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                  TTI::TargetCostKind CostKind);
 
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index e239870662bb0..407f980bd35e8 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -10,10 +10,13 @@
 #include "MCTargetDesc/RISCVMCExpr.h"
 #include "MCTargetDesc/RISCVMCTargetDesc.h"
 #include "MCTargetDesc/RISCVTargetStreamer.h"
+#include "RISCVInstrInfo.h"
 #include "TargetInfo/RISCVTargetInfo.h"
 #include "Utils/RISCVBaseInfo.h"
 #include "Utils/RISCVMatInt.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallBitVector.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -128,6 +131,9 @@ class RISCVAsmParser : public MCTargetAsmParser {
   // 'add' is an overloaded mnemonic.
   bool checkPseudoAddTPRel(MCInst &Inst, OperandVector &Operands);
 
+  // Check instruction constraints.
+  bool validateInstruction(MCInst &Inst, OperandVector &Operands);
+
   /// Helper for processing MC instructions that have been successfully matched
   /// by MatchAndEmitInstruction. Modifications to the emitted instructions,
   /// like the expansion of pseudo instructions (e.g., "li"), can be performed
@@ -150,6 +156,8 @@ class RISCVAsmParser : public MCTargetAsmParser {
   OperandMatchResultTy parseCallSymbol(OperandVector &Operands);
   OperandMatchResultTy parsePseudoJumpSymbol(OperandVector &Operands);
   OperandMatchResultTy parseJALOffset(OperandVector &Operands);
+  OperandMatchResultTy parseVTypeI(OperandVector &Operands);
+  OperandMatchResultTy parseMaskReg(OperandVector &Operands);
 
   bool parseOperand(OperandVector &Operands, StringRef Mnemonic);
 
@@ -198,6 +206,8 @@ class RISCVAsmParser : public MCTargetAsmParser {
     return false;
   }
 
+  std::unique_ptr<RISCVOperand> defaultMaskRegOp() const;
+
 public:
   enum RISCVMatchResultTy {
     Match_Dummy = FIRST_TARGET_MATCH_RESULT_TY,
@@ -245,7 +255,8 @@ struct RISCVOperand : public MCParsedAsmOperand {
     Token,
     Register,
     Immediate,
-    SystemRegister
+    SystemRegister,
+    VType,
   } Kind;
 
   bool IsRV64;
@@ -266,12 +277,32 @@ struct RISCVOperand : public MCParsedAsmOperand {
     // e.g.: read/write or user/supervisor/machine privileges.
   };
 
+  enum class VSEW {
+    SEW_8 = 0,
+    SEW_16,
+    SEW_32,
+    SEW_64,
+    SEW_128,
+    SEW_256,
+    SEW_512,
+    SEW_1024,
+  };
+
+  enum class VLMUL { LMUL_1 = 0, LMUL_2, LMUL_4, LMUL_8 };
+
+  struct VTypeOp {
+    VSEW Sew;
+    VLMUL Lmul;
+    unsigned Encoding;
+  };
+
   SMLoc StartLoc, EndLoc;
   union {
     StringRef Tok;
     RegOp Reg;
     ImmOp Imm;
     struct SysRegOp SysReg;
+    struct VTypeOp VType;
   };
 
   RISCVOperand(KindTy K) : MCParsedAsmOperand(), Kind(K) {}
@@ -295,14 +326,21 @@ struct RISCVOperand : public MCParsedAsmOperand {
     case KindTy::SystemRegister:
       SysReg = o.SysReg;
       break;
+    case KindTy::VType:
+      VType = o.VType;
+      break;
     }
   }
 
   bool isToken() const override { return Kind == KindTy::Token; }
   bool isReg() const override { return Kind == KindTy::Register; }
+  bool isV0Reg() const {
+    return Kind == KindTy::Register && Reg.RegNum == RISCV::V0;
+  }
   bool isImm() const override { return Kind == KindTy::Immediate; }
   bool isMem() const override { return false; }
   bool isSystemRegister() const { return Kind == KindTy::SystemRegister; }
+  bool isVType() const { return Kind == KindTy::VType; }
 
   bool isGPR() const {
     return Kind == KindTy::Register &&
@@ -386,6 +424,8 @@ struct RISCVOperand : public MCParsedAsmOperand {
 
   bool isCSRSystemRegister() const { return isSystemRegister(); }
 
+  bool isVTypeI() const { return isVType(); }
+
   /// Return true if the operand is a valid for the fence instruction e.g.
   /// ('iorw').
   bool isFenceArg() const {
@@ -493,6 +533,15 @@ struct RISCVOperand : public MCParsedAsmOperand {
            VK == RISCVMCExpr::VK_RISCV_None;
   }
 
+  bool isSImm5() const {
+    if (!isImm())
+      return false;
+    RISCVMCExpr::VariantKind VK = RISCVMCExpr::VK_RISCV_None;
+    int64_t Imm;
+    bool IsConstantImm = evaluateConstantImm(getImm(), Imm, VK);
+    return IsConstantImm && isInt<5>(Imm) && VK == RISCVMCExpr::VK_RISCV_None;
+  }
+
   bool isSImm6() const {
     if (!isImm())
       return false;
@@ -658,6 +707,16 @@ struct RISCVOperand : public MCParsedAsmOperand {
     return IsConstantImm && (Imm == 0) && VK == RISCVMCExpr::VK_RISCV_None;
   }
 
+  bool isSImm5Plus1() const {
+    if (!isImm())
+      return false;
+    RISCVMCExpr::VariantKind VK = RISCVMCExpr::VK_RISCV_None;
+    int64_t Imm;
+    bool IsConstantImm = evaluateConstantImm(getImm(), Imm, VK);
+    return IsConstantImm && isInt<5>(Imm - 1) &&
+           VK == RISCVMCExpr::VK_RISCV_None;
+  }
+
   /// getStartLoc - Gets location of the first token of this operand
   SMLoc getStartLoc() const override { return StartLoc; }
   /// getEndLoc - Gets location of the last token of this operand
@@ -685,6 +744,51 @@ struct RISCVOperand : public MCParsedAsmOperand {
     return Tok;
   }
 
+  static StringRef getSEWStr(VSEW Sew) {
+    switch (Sew) {
+    case VSEW::SEW_8:
+      return "e8";
+    case VSEW::SEW_16:
+      return "e16";
+    case VSEW::SEW_32:
+      return "e32";
+    case VSEW::SEW_64:
+      return "e64";
+    case VSEW::SEW_128:
+      return "e128";
+    case VSEW::SEW_256:
+      return "e256";
+    case VSEW::SEW_512:
+      return "e512";
+    case VSEW::SEW_1024:
+      return "e1024";
+    }
+    return "";
+  }
+
+  static StringRef getLMULStr(VLMUL Lmul) {
+    switch (Lmul) {
+    case VLMUL::LMUL_1:
+      return "m1";
+    case VLMUL::LMUL_2:
+      return "m2";
+    case VLMUL::LMUL_4:
+      return "m4";
+    case VLMUL::LMUL_8:
+      return "m8";
+    }
+    return "";
+  }
+
+  StringRef getVType(SmallString<32> &Buf) const {
+    assert(Kind == KindTy::VType && "Invalid access!");
+    Buf.append(getSEWStr(VType.Sew));
+    Buf.append(",");
+    Buf.append(getLMULStr(VType.Lmul));
+
+    return Buf.str();
+  }
+
   void print(raw_ostream &OS) const override {
     switch (Kind) {
     case KindTy::Immediate:
@@ -700,6 +804,10 @@ struct RISCVOperand : public MCParsedAsmOperand {
     case KindTy::SystemRegister:
       OS << "<sysreg: " << getSysReg() << '>';
       break;
+    case KindTy::VType:
+      SmallString<32> VTypeBuf;
+      OS << "<vtype: " << getVType(VTypeBuf) << '>';
+      break;
     }
   }
 
@@ -744,6 +852,20 @@ struct RISCVOperand : public MCParsedAsmOperand {
     return Op;
   }
 
+  static std::unique_ptr<RISCVOperand> createVType(APInt Sew, APInt Lmul,
+                                                   SMLoc S, bool IsRV64) {
+    auto Op = std::make_unique<RISCVOperand>(KindTy::VType);
+    Sew.ashrInPlace(3);
+    unsigned SewLog2 = Sew.logBase2();
+    unsigned LmulLog2 = Lmul.logBase2();
+    Op->VType.Sew = static_cast<VSEW>(SewLog2);
+    Op->VType.Lmul = static_cast<VLMUL>(LmulLog2);
+    Op->VType.Encoding = (SewLog2 << 2) | LmulLog2;
+    Op->StartLoc = S;
+    Op->IsRV64 = IsRV64;
+    return Op;
+  }
+
   void addExpr(MCInst &Inst, const MCExpr *Expr) const {
     assert(Expr && "Expr shouldn't be null!");
     int64_t Imm = 0;
@@ -767,6 +889,16 @@ struct RISCVOperand : public MCParsedAsmOperand {
     addExpr(Inst, getImm());
   }
 
+  void addSImm5Plus1Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    int64_t Imm = 0;
+    RISCVMCExpr::VariantKind VK = RISCVMCExpr::VK_RISCV_None;
+    bool IsConstant = evaluateConstantImm(getImm(), Imm, VK);
+    assert(IsConstant && "Expect constant value!");
+    (void)IsConstant;
+    Inst.addOperand(MCOperand::createImm(Imm - 1));
+  }
+
   void addFenceArgOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     // isFenceArg has validated the operand, meaning this cast is safe
@@ -791,6 +923,11 @@ struct RISCVOperand : public MCParsedAsmOperand {
     Inst.addOperand(MCOperand::createImm(SysReg.Encoding));
   }
 
+  void addVTypeIOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::createImm(VType.Encoding));
+  }
+
   // Returns the rounding mode represented by this RISCVOperand. Should only
   // be called after checking isFRMArg.
   RISCVFPRndMode::RoundingMode getRoundingMode() const {
@@ -868,6 +1005,8 @@ bool RISCVAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
   default:
     break;
   case Match_Success:
+    if (validateInstruction(Inst, Operands))
+      return true;
     return processInstruction(Inst, IDLoc, Operands, Out);
   case Match_MissingFeature: {
     assert(MissingFeatures.any() && "Unknown missing features!");
@@ -1040,6 +1179,20 @@ bool RISCVAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
     return Error(ErrorLoc, "operand must be a symbol with %tprel_add modifier");
   }
+  case Match_InvalidVTypeI: {
+    SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
+    return Error(ErrorLoc,
+                 "operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8]");
+  }
+  case Match_InvalidVMaskRegister: {
+    SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
+    return Error(ErrorLoc, "operand must be v0.t");
+  }
+  case Match_InvalidSImm5Plus1: {
+    return generateImmOutOfRangeError(Operands, ErrorInfo, -(1 << 4) + 1,
+                                      (1 << 4),
+                                      "immediate must be in the range");
+  }
   }
 
   llvm_unreachable("Unknown match type detected!");
@@ -1391,6 +1544,74 @@ OperandMatchResultTy RISCVAsmParser::parseJALOffset(OperandVector &Operands) {
   return parseImmediate(Operands);
 }
 
+OperandMatchResultTy RISCVAsmParser::parseVTypeI(OperandVector &Operands) {
+  SMLoc S = getLoc();
+  if (getLexer().getKind() != AsmToken::Identifier)
+    return MatchOperand_NoMatch;
+
+  // Parse "e8,m1"
+  StringRef Name = getLexer().getTok().getIdentifier();
+  if (!Name.consume_front("e"))
+    return MatchOperand_NoMatch;
+  APInt Sew(16, Name, 10);
+  if (Sew != 8 && Sew != 16 && Sew != 32 && Sew != 64 && Sew != 128 &&
+      Sew != 256 && Sew != 512 && Sew != 1024)
+    return MatchOperand_NoMatch;
+  getLexer().Lex();
+
+  if (getLexer().getKind() == AsmToken::EndOfStatement) {
+    Operands.push_back(
+        RISCVOperand::createVType(Sew, APInt(16, 1), S, isRV64()));
+
+    return MatchOperand_Success;
+  }
+
+  if (!getLexer().is(AsmToken::Comma))
+    return MatchOperand_NoMatch;
+  getLexer().Lex();
+
+  Name = getLexer().getTok().getIdentifier();
+  if (!Name.consume_front("m"))
+    return MatchOperand_NoMatch;
+  APInt Lmul(16, Name, 10);
+  if (Lmul != 1 && Lmul != 2 && Lmul != 4 && Lmul != 8)
+    return MatchOperand_NoMatch;
+  getLexer().Lex();
+
+  if (getLexer().getKind() != AsmToken::EndOfStatement)
+    return MatchOperand_NoMatch;
+
+  Operands.push_back(RISCVOperand::createVType(Sew, Lmul, S, isRV64()));
+
+  return MatchOperand_Success;
+}
+
+OperandMatchResultTy RISCVAsmParser::parseMaskReg(OperandVector &Operands) {
+  switch (getLexer().getKind()) {
+  default:
+    return MatchOperand_NoMatch;
+  case AsmToken::Identifier:
+    StringRef Name = getLexer().getTok().getIdentifier();
+    if (!Name.consume_back(".t")) {
+      Error(getLoc(), "expected '.t' suffix");
+      return MatchOperand_ParseFail;
+    }
+    Register RegNo;
+    matchRegisterNameHelper(isRV32E(), RegNo, Name);
+
+    if (RegNo == RISCV::NoRegister)
+      return MatchOperand_NoMatch;
+    if (RegNo != RISCV::V0)
+      return MatchOperand_NoMatch;
+    SMLoc S = getLoc();
+    SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1);
+    getLexer().Lex();
+    Operands.push_back(RISCVOperand::createReg(RegNo, S, E, isRV64()));
+  }
+
+  return MatchOperand_Success;
+}
+
 OperandMatchResultTy
 RISCVAsmParser::parseMemOpBaseReg(OperandVector &Operands) {
   if (getLexer().isNot(AsmToken::LParen)) {
@@ -2046,6 +2267,89 @@ bool RISCVAsmParser::checkPseudoAddTPRel(MCInst &Inst,
   return false;
 }
 
+std::unique_ptr<RISCVOperand> RISCVAsmParser::defaultMaskRegOp() const {
+  return RISCVOperand::createReg(RISCV::NoRegister, llvm::SMLoc(),
+                                 llvm::SMLoc(), isRV64());
+}
+
+bool RISCVAsmParser::validateInstruction(MCInst &Inst,
+                                         OperandVector &Operands) {
+  const MCInstrDesc &MCID = MII.get(Inst.getOpcode());
+  unsigned TargetFlags =
+      (MCID.TSFlags >> RISCV::ConstraintOffset) & RISCV::ConstraintMask;
+  if (TargetFlags == RISCV::NoConstraint)
+    return false;
+
+  unsigned DestReg = Inst.getOperand(0).getReg();
+  // Operands[1] will be the first operand, DestReg.
+  SMLoc Loc = Operands[1]->getStartLoc();
+  if ((TargetFlags == RISCV::WidenV) || (TargetFlags == RISCV::WidenW) ||
+      (TargetFlags == RISCV::SlideUp) || (TargetFlags == RISCV::Vrgather) ||
+      (TargetFlags == RISCV::Vcompress)) {
+    if (TargetFlags != RISCV::WidenW) {
+      unsigned Src2Reg = Inst.getOperand(1).getReg();
+      if (DestReg == Src2Reg)
+        return Error(Loc, "The destination vector register group cannot overlap"
+                          " the source vector register group.");
+      if (TargetFlags == RISCV::WidenV) {
+        // Assume DestReg LMUL is 2 at least for widening/narrowing operations.
+        if (DestReg + 1 == Src2Reg)
+          return Error(Loc,
+                       "The destination vector register group cannot overlap"
+                       " the source vector register group.");
+      }
+    }
+    if (Inst.getOperand(2).isReg()) {
+      unsigned Src1Reg = Inst.getOperand(2).getReg();
+      if (DestReg == Src1Reg)
+        return Error(Loc, "The destination vector register group cannot overlap"
+                          " the source vector register group.");
+      if (TargetFlags == RISCV::WidenV || TargetFlags == RISCV::WidenW) {
+        // Assume DestReg LMUL is 2 at least for widening/narrowing operations.
+        if (DestReg + 1 == Src1Reg)
+          return Error(Loc,
+                       "The destination vector register group cannot overlap"
+                       " the source vector register group.");
+      }
+    }
+    if (Inst.getNumOperands() == 4) {
+      unsigned MaskReg = Inst.getOperand(3).getReg();
+
+      if (DestReg == MaskReg)
+        return Error(Loc, "The destination vector register group cannot overlap"
+                          " the mask register.");
+    }
+  } else if (TargetFlags == RISCV::Narrow) {
+    unsigned Src2Reg = Inst.getOperand(1).getReg();
+    if (DestReg == Src2Reg)
+      return Error(Loc, "The destination vector register group cannot overlap"
+                        " the source vector register group.");
+    // Assume Src2Reg LMUL is 2 at least for widening/narrowing operations.
+    if (DestReg == Src2Reg + 1)
+      return Error(Loc, "The destination vector register group cannot overlap"
+                        " the source vector register group.");
+  } else if (TargetFlags == RISCV::WidenCvt || TargetFlags == RISCV::Iota) {
+    unsigned Src2Reg = Inst.getOperand(1).getReg();
+    if (DestReg == Src2Reg)
+      return Error(Loc, "The destination vector register group cannot overlap"
+                        " the source vector register group.");
+    if (TargetFlags == RISCV::WidenCvt) {
+      // Assume DestReg LMUL is 2 at least for widening/narrowing operations.
+      if (DestReg + 1 == Src2Reg)
+        return Error(Loc, "The destination vector register group cannot overlap"
+                          " the source vector register group.");
+    }
+    if (Inst.getNumOperands() == 3) {
+      unsigned MaskReg = Inst.getOperand(2).getReg();
+
+      if (DestReg == MaskReg)
+        return Error(Loc, "The destination vector register group cannot overlap"
+                          " the mask register.");
+    }
+  }
+  return false;
+}
+
 bool RISCVAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
                                         OperandVector &Operands,
                                         MCStreamer &Out) {
diff --git a/llvm/lib/Target/RISCV/CMakeLists.txt b/llvm/lib/Target/RISCV/CMakeLists.txt
index 31a82be198123..9a0c2209c8855 100644
--- a/llvm/lib/Target/RISCV/CMakeLists.txt
+++ b/llvm/lib/Target/RISCV/CMakeLists.txt
@@ -19,6 +19,7 @@ add_public_tablegen_target(RISCVCommonTableGen)
 add_llvm_target(RISCVCodeGen
   RISCVAsmPrinter.cpp
   RISCVCallLowering.cpp
+  RISCVExpandAtomicPseudoInsts.cpp
   RISCVExpandPseudoInsts.cpp
   RISCVFrameLowering.cpp
   RISCVInstrInfo.cpp
diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index 92519be870157..37edc19398a55 100644
--- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -18,6 +18,7 @@
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCFixedLenDisassembler.h"
 #include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/Endian.h"
@@ -31,10 +32,12 @@ typedef MCDisassembler::DecodeStatus DecodeStatus;
 
 namespace {
 class RISCVDisassembler : public MCDisassembler {
+  std::unique_ptr<MCInstrInfo const> const MCII;
 
 public:
-  RISCVDisassembler(const MCSubtargetInfo &STI, MCContext &Ctx)
-      : MCDisassembler(STI, Ctx) {}
+  RISCVDisassembler(const MCSubtargetInfo &STI, MCContext &Ctx,
+                    MCInstrInfo const *MCII)
+      : MCDisassembler(STI, Ctx), MCII(MCII) {}
 
   DecodeStatus getInstruction(MCInst &Instr, uint64_t &Size,
                               ArrayRef<uint8_t> Bytes, uint64_t Address,
@@ -45,7 +48,7 @@ class RISCVDisassembler : public MCDisassembler {
 static MCDisassembler *createRISCVDisassembler(const Target &T,
                                                const MCSubtargetInfo &STI,
                                                MCContext &Ctx) {
-  return new RISCVDisassembler(STI, Ctx);
+  return new RISCVDisassembler(STI, Ctx, T.createMCInstrInfo());
 }
 
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVDisassembler() {
@@ -148,6 +151,33 @@ static DecodeStatus DecodeGPRCRegisterClass(MCInst &Inst, uint64_t RegNo,
   return MCDisassembler::Success;
 }
 
+static DecodeStatus DecodeVRRegisterClass(MCInst &Inst, uint64_t RegNo,
+                                          uint64_t Address,
+                                          const void *Decoder) {
+  if (RegNo >= 32)
+    return MCDisassembler::Fail;
+
+  Register Reg = RISCV::V0 + RegNo;
+  Inst.addOperand(MCOperand::createReg(Reg));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus decodeVMaskReg(MCInst &Inst, uint64_t RegNo,
+                                   uint64_t Address, const void *Decoder) {
+  Register Reg = RISCV::NoRegister;
+  switch (RegNo) {
+  default:
+    return MCDisassembler::Fail;
+  case 0:
+    Reg = RISCV::V0;
+    break;
+  case 1:
+    break;
+  }
+  Inst.addOperand(MCOperand::createReg(Reg));
+  return MCDisassembler::Success;
+}
+
 // Add implied SP operand for instructions *SP compressed instructions. The SP
 // operand isn't explicitly encoded in the instruction.
 static void addImplySP(MCInst &Inst, int64_t Address, const void *Decoder) {
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
index 6cf97dd51baa2..eae3e13dbe409 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
@@ -150,6 +150,39 @@ void RISCVInstPrinter::printAtomicMemOp(const MCInst *MI, unsigned OpNo,
   return;
 }
 
+void RISCVInstPrinter::printVTypeI(const MCInst *MI, unsigned OpNo,
+                                   const MCSubtargetInfo &STI, raw_ostream &O) {
+  unsigned Imm = MI->getOperand(OpNo).getImm();
+  unsigned Sew = (Imm >> 2) & 0x7;
+  unsigned Lmul = Imm & 0x3;
+
+  Lmul = 0x1 << Lmul;
+  Sew = 0x1 << (Sew + 3);
+  O << "e" << Sew << ",m" << Lmul;
+}
+
+void RISCVInstPrinter::printVMaskReg(const MCInst *MI, unsigned OpNo,
+                                     const MCSubtargetInfo &STI,
+                                     raw_ostream &O) {
+  const MCOperand &MO = MI->getOperand(OpNo);
+
+  assert(MO.isReg() && "printVMaskReg can only print register operands");
+  if (MO.getReg() == RISCV::NoRegister)
+    return;
+  O << ", ";
+  printRegName(O, MO.getReg());
+  O << ".t";
+}
+
+void RISCVInstPrinter::printSImm5Plus1(const MCInst *MI, unsigned OpNo,
+                                       const MCSubtargetInfo &STI,
+                                       raw_ostream &O) {
+  const MCOperand &MO = MI->getOperand(OpNo);
+
+  assert(MO.isImm() && "printSImm5Plus1 can only print constant operands");
+  O << MO.getImm() + 1;
+}
+
 const char *RISCVInstPrinter::getRegisterName(unsigned RegNo) {
   return getRegisterName(RegNo, ArchRegNames ? RISCV::NoRegAltName
                                              : RISCV::ABIRegAltName);
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h
index 893524af5466c..fdaa00c5f8ebc 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h
@@ -40,6 +40,12 @@ class RISCVInstPrinter : public MCInstPrinter {
                    raw_ostream &O);
   void printAtomicMemOp(const MCInst *MI, unsigned OpNo,
                         const MCSubtargetInfo &STI, raw_ostream &O);
+  void printVTypeI(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
+                   raw_ostream &O);
+  void printVMaskReg(const MCInst *MI, unsigned OpNo,
+                     const MCSubtargetInfo &STI, raw_ostream &O);
+  void printSImm5Plus1(const MCInst *MI, unsigned OpNo,
+                       const MCSubtargetInfo &STI, raw_ostream &O);
 
   // Autogenerated by tblgen.
   void printInstruction(const MCInst *MI, uint64_t Address,
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp
index 8db1738566ac8..089a2def4c210 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp
@@ -27,7 +27,6 @@ RISCVMCAsmInfo::RISCVMCAsmInfo(const Triple &TT) {
   ExceptionsType = ExceptionHandling::DwarfCFI;
   Data16bitsDirective = "\t.half\t";
   Data32bitsDirective = "\t.word\t";
-  UseIntegratedAssembler = false;
 }
 
 const MCExpr *RISCVMCAsmInfo::getExprForFDESymbol(const MCSymbol *Sym,
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
index 5139ec5873d8f..ae13c00565c43 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
@@ -80,6 +80,10 @@ class RISCVMCCodeEmitter : public MCCodeEmitter {
   unsigned getImmOpValue(const MCInst &MI, unsigned OpNo,
                          SmallVectorImpl<MCFixup> &Fixups,
                          const MCSubtargetInfo &STI) const;
+
+  unsigned getVMaskReg(const MCInst &MI, unsigned OpNo,
+                       SmallVectorImpl<MCFixup> &Fixups,
+                       const MCSubtargetInfo &STI) const;
 };
 } // end anonymous namespace
 
@@ -374,4 +378,20 @@ unsigned RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
   return 0;
 }
 
+unsigned RISCVMCCodeEmitter::getVMaskReg(const MCInst &MI, unsigned OpNo,
+                                         SmallVectorImpl<MCFixup> &Fixups,
+                                         const MCSubtargetInfo &STI) const {
+  MCOperand MO = MI.getOperand(OpNo);
+  assert(MO.isReg() && "Expected a register.");
+
+  switch (MO.getReg()) {
+  default:
+    llvm_unreachable("Invalid mask register.");
+  case RISCV::V0:
+    return 0;
+  case RISCV::NoRegister:
+    return 1;
+  }
+}
+
 #include "RISCVGenMCCodeEmitter.inc"
diff --git a/llvm/lib/Target/RISCV/RISCV.h b/llvm/lib/Target/RISCV/RISCV.h
index f23f742a47822..9baa2cc2741a8 100644
--- a/llvm/lib/Target/RISCV/RISCV.h
+++ b/llvm/lib/Target/RISCV/RISCV.h
@@ -43,6 +43,9 @@ void initializeRISCVMergeBaseOffsetOptPass(PassRegistry &);
 FunctionPass *createRISCVExpandPseudoPass();
 void initializeRISCVExpandPseudoPass(PassRegistry &);
 
+FunctionPass *createRISCVExpandAtomicPseudoPass();
+void initializeRISCVExpandAtomicPseudoPass(PassRegistry &);
+
 InstructionSelector *createRISCVInstructionSelector(const RISCVTargetMachine &,
                                                     RISCVSubtarget &,
                                                     RISCVRegisterBankInfo &);
diff --git a/llvm/lib/Target/RISCV/RISCV.td b/llvm/lib/Target/RISCV/RISCV.td
index 6564e0aa49f14..f0583f6919361 100644
--- a/llvm/lib/Target/RISCV/RISCV.td
+++ b/llvm/lib/Target/RISCV/RISCV.td
@@ -140,12 +140,20 @@ def HasStdExtB : Predicate<"Subtarget->hasStdExtB()">,
                            AssemblerPredicate<(all_of FeatureStdExtB),
                            "'B' (Bit Manipulation Instructions)">;
 
-def FeatureRVCHints
-    : SubtargetFeature<"rvc-hints", "EnableRVCHintInstrs", "true",
-                       "Enable RVC Hint Instructions.">;
+def FeatureNoRVCHints
+    : SubtargetFeature<"no-rvc-hints", "EnableRVCHintInstrs", "false",
+                       "Disable RVC Hint Instructions.">;
 def HasRVCHints : Predicate<"Subtarget->enableRVCHintInstrs()">,
-                            AssemblerPredicate<(all_of FeatureRVCHints),
-                            "RVC Hint Instructions">;
+                  AssemblerPredicate<(all_of(not FeatureNoRVCHints)),
+                                     "RVC Hint Instructions">;
+
+def FeatureStdExtV
+    : SubtargetFeature<"experimental-v", "HasStdExtV", "true",
+                       "'V' (Vector Instructions)",
+                       [FeatureStdExtF]>;
+def HasStdExtV : Predicate<"Subtarget->hasStdExtV()">,
+                           AssemblerPredicate<(all_of FeatureStdExtV),
+                           "'V' (Vector Instructions)">;
 
 def Feature64Bit
     : SubtargetFeature<"64bit", "HasRV64", "true", "Implements RV64">;
@@ -199,15 +207,13 @@ include "RISCVSchedRocket64.td"
 // RISC-V processors supported.
 //===----------------------------------------------------------------------===//
 
-def : ProcessorModel<"generic-rv32", NoSchedModel, [FeatureRVCHints]>;
+def : ProcessorModel<"generic-rv32", NoSchedModel, []>;
 
-def : ProcessorModel<"generic-rv64", NoSchedModel, [Feature64Bit,
-                     FeatureRVCHints]>;
+def : ProcessorModel<"generic-rv64", NoSchedModel, [Feature64Bit]>;
 
-def : ProcessorModel<"rocket-rv32", Rocket32Model, [FeatureRVCHints]>;
+def : ProcessorModel<"rocket-rv32", Rocket32Model, []>;
 
-def : ProcessorModel<"rocket-rv64", Rocket64Model, [Feature64Bit,
-                     FeatureRVCHints]>;
+def : ProcessorModel<"rocket-rv64", Rocket64Model, [Feature64Bit]>;
 
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp
new file mode 100644
index 0000000000000..b49c767ff2ca6
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp
@@ -0,0 +1,615 @@
+//===-- RISCVExpandAtomicPseudoInsts.cpp - Expand atomic pseudo instrs. ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains a pass that expands atomic pseudo instructions into
+// target instructions. This pass should be run at the last possible moment,
+// avoiding the possibility for other passes to break the requirements for
+// forward progress in the LR/SC block.
+//
+//===----------------------------------------------------------------------===//
+
+#include "RISCV.h"
+#include "RISCVInstrInfo.h"
+#include "RISCVTargetMachine.h"
+
+#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+
+using namespace llvm;
+
+#define RISCV_EXPAND_ATOMIC_PSEUDO_NAME                                        \
+  "RISCV atomic pseudo instruction expansion pass"
+
+namespace {
+
+class RISCVExpandAtomicPseudo : public MachineFunctionPass {
+public:
+  const RISCVInstrInfo *TII;
+  static char ID;
+
+  RISCVExpandAtomicPseudo() : MachineFunctionPass(ID) {
+    initializeRISCVExpandAtomicPseudoPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  StringRef getPassName() const override {
+    return RISCV_EXPAND_ATOMIC_PSEUDO_NAME;
+  }
+
+private:
+  bool expandMBB(MachineBasicBlock &MBB);
+  bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+                MachineBasicBlock::iterator &NextMBBI);
+  bool expandAtomicBinOp(MachineBasicBlock &MBB,
+                         MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
+                         bool IsMasked, int Width,
+                         MachineBasicBlock::iterator &NextMBBI);
+  bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
+                            MachineBasicBlock::iterator MBBI,
+                            AtomicRMWInst::BinOp, bool IsMasked, int Width,
+                            MachineBasicBlock::iterator &NextMBBI);
+  bool expandAtomicCmpXchg(MachineBasicBlock &MBB,
+                           MachineBasicBlock::iterator MBBI, bool IsMasked,
+                           int Width, MachineBasicBlock::iterator &NextMBBI);
+};
+
+char RISCVExpandAtomicPseudo::ID = 0;
+
+bool RISCVExpandAtomicPseudo::runOnMachineFunction(MachineFunction &MF) {
+  TII = static_cast<const RISCVInstrInfo *>(MF.getSubtarget().getInstrInfo());
+  bool Modified = false;
+  for (auto &MBB : MF)
+    Modified |= expandMBB(MBB);
+  return Modified;
+}
+
+bool RISCVExpandAtomicPseudo::expandMBB(MachineBasicBlock &MBB) {
+  bool Modified = false;
+
+  MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
+  while (MBBI != E) {
+    MachineBasicBlock::iterator NMBBI = std::next(MBBI);
+    Modified |= expandMI(MBB, MBBI, NMBBI);
+    MBBI = NMBBI;
+  }
+
+  return Modified;
+}
+
+bool RISCVExpandAtomicPseudo::expandMI(MachineBasicBlock &MBB,
+                                       MachineBasicBlock::iterator MBBI,
+                                       MachineBasicBlock::iterator &NextMBBI) {
+  switch (MBBI->getOpcode()) {
+  case RISCV::PseudoAtomicLoadNand32:
+    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
+                             NextMBBI);
+  case RISCV::PseudoAtomicLoadNand64:
+    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
+                             NextMBBI);
+  case RISCV::PseudoMaskedAtomicSwap32:
+    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
+                             NextMBBI);
+  case RISCV::PseudoMaskedAtomicLoadAdd32:
+    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI);
+  case RISCV::PseudoMaskedAtomicLoadSub32:
+    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI);
+  case RISCV::PseudoMaskedAtomicLoadNand32:
+    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, true, 32,
+                             NextMBBI);
+  case RISCV::PseudoMaskedAtomicLoadMax32:
+    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, true, 32,
+                                NextMBBI);
+  case RISCV::PseudoMaskedAtomicLoadMin32:
+    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, true, 32,
+                                NextMBBI);
+  case RISCV::PseudoMaskedAtomicLoadUMax32:
+    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32,
+                                NextMBBI);
+  case RISCV::PseudoMaskedAtomicLoadUMin32:
+    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32,
+                                NextMBBI);
+  case RISCV::PseudoCmpXchg32:
+    return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI);
+  case RISCV::PseudoCmpXchg64:
+    return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI);
+  case RISCV::PseudoMaskedCmpXchg32:
+    return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI);
+  }
+
+  return false;
+}
+
+static unsigned getLRForRMW32(AtomicOrdering Ordering) {
+  switch (Ordering) {
+  default:
+    llvm_unreachable("Unexpected AtomicOrdering");
+  case AtomicOrdering::Monotonic:
+    return RISCV::LR_W;
+  case AtomicOrdering::Acquire:
+    return RISCV::LR_W_AQ;
+  case AtomicOrdering::Release:
+    return RISCV::LR_W;
+  case AtomicOrdering::AcquireRelease:
+    return RISCV::LR_W_AQ;
+  case AtomicOrdering::SequentiallyConsistent:
+    return RISCV::LR_W_AQ_RL;
+  }
+}
+
+static unsigned getSCForRMW32(AtomicOrdering Ordering) {
+  switch (Ordering) {
+  default:
+    llvm_unreachable("Unexpected AtomicOrdering");
+  case AtomicOrdering::Monotonic:
+    return RISCV::SC_W;
+  case AtomicOrdering::Acquire:
+    return RISCV::SC_W;
+  case AtomicOrdering::Release:
+    return RISCV::SC_W_RL;
+  case AtomicOrdering::AcquireRelease:
+    return RISCV::SC_W_RL;
+  case AtomicOrdering::SequentiallyConsistent:
+    return RISCV::SC_W_AQ_RL;
+  }
+}
+
+static unsigned getLRForRMW64(AtomicOrdering Ordering) {
+  switch (Ordering) {
+  default:
+    llvm_unreachable("Unexpected AtomicOrdering");
+  case AtomicOrdering::Monotonic:
+    return RISCV::LR_D;
+  case AtomicOrdering::Acquire:
+    return RISCV::LR_D_AQ;
+  case AtomicOrdering::Release:
+    return RISCV::LR_D;
+  case AtomicOrdering::AcquireRelease:
+    return RISCV::LR_D_AQ;
+  case AtomicOrdering::SequentiallyConsistent:
+    return RISCV::LR_D_AQ_RL;
+  }
+}
+
+static unsigned getSCForRMW64(AtomicOrdering Ordering) {
+  switch (Ordering) {
+  default:
+    llvm_unreachable("Unexpected AtomicOrdering");
+  case AtomicOrdering::Monotonic:
+    return RISCV::SC_D;
+  case AtomicOrdering::Acquire:
+    return RISCV::SC_D;
+  case AtomicOrdering::Release:
+    return RISCV::SC_D_RL;
+  case AtomicOrdering::AcquireRelease:
+    return RISCV::SC_D_RL;
+  case AtomicOrdering::SequentiallyConsistent:
+    return RISCV::SC_D_AQ_RL;
+  }
+}
+
+static unsigned getLRForRMW(AtomicOrdering Ordering, int Width) {
+  if (Width == 32)
+    return getLRForRMW32(Ordering);
+  if (Width == 64)
+    return getLRForRMW64(Ordering);
+  llvm_unreachable("Unexpected LR width\n");
+}
+
+static unsigned getSCForRMW(AtomicOrdering Ordering, int Width) {
+  if (Width == 32)
+    return getSCForRMW32(Ordering);
+  if (Width == 64)
+    return getSCForRMW64(Ordering);
+  llvm_unreachable("Unexpected SC width\n");
+}
+
+static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
+                                   DebugLoc DL, MachineBasicBlock *ThisMBB,
+                                   MachineBasicBlock *LoopMBB,
+                                   MachineBasicBlock *DoneMBB,
+                                   AtomicRMWInst::BinOp BinOp, int Width) {
+  Register DestReg = MI.getOperand(0).getReg();
+  Register ScratchReg = MI.getOperand(1).getReg();
+  Register AddrReg = MI.getOperand(2).getReg();
+  Register IncrReg = MI.getOperand(3).getReg();
+  AtomicOrdering Ordering =
+      static_cast<AtomicOrdering>(MI.getOperand(4).getImm());
+
+  // .loop:
+  //   lr.[w|d] dest, (addr)
+  //   binop scratch, dest, val
+  //   sc.[w|d] scratch, scratch, (addr)
+  //   bnez scratch, loop
+  BuildMI(LoopMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
+      .addReg(AddrReg);
+  switch (BinOp) {
+  default:
+    llvm_unreachable("Unexpected AtomicRMW BinOp");
+  case AtomicRMWInst::Nand:
+    BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
+        .addReg(DestReg)
+        .addReg(IncrReg);
+    BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
+        .addReg(ScratchReg)
+        .addImm(-1);
+    break;
+  }
+  BuildMI(LoopMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
+      .addReg(AddrReg)
+      .addReg(ScratchReg);
+  BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
+      .addReg(ScratchReg)
+      .addReg(RISCV::X0)
+      .addMBB(LoopMBB);
+}
+
+static void insertMaskedMerge(const RISCVInstrInfo *TII, DebugLoc DL,
+                              MachineBasicBlock *MBB, Register DestReg,
+                              Register OldValReg, Register NewValReg,
+                              Register MaskReg, Register ScratchReg) {
+  assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique");
+  assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique");
+  assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique");
+
+  // We select bits from newval and oldval using:
+  // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge
+  // r = oldval ^ ((oldval ^ newval) & masktargetdata);
+  BuildMI(MBB, DL, TII->get(RISCV::XOR), ScratchReg)
+      .addReg(OldValReg)
+      .addReg(NewValReg);
+  BuildMI(MBB, DL, TII->get(RISCV::AND), ScratchReg)
+      .addReg(ScratchReg)
+      .addReg(MaskReg);
+  BuildMI(MBB, DL, TII->get(RISCV::XOR), DestReg)
+      .addReg(OldValReg)
+      .addReg(ScratchReg);
+}
+
+static void doMaskedAtomicBinOpExpansion(
+    const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
+    MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopMBB,
+    MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) {
+  assert(Width == 32 && "Should never need to expand masked 64-bit operations");
+  Register DestReg = MI.getOperand(0).getReg();
+  Register ScratchReg = MI.getOperand(1).getReg();
+  Register AddrReg = MI.getOperand(2).getReg();
+  Register IncrReg = MI.getOperand(3).getReg();
+  Register MaskReg = MI.getOperand(4).getReg();
+  AtomicOrdering Ordering =
+      static_cast<AtomicOrdering>(MI.getOperand(5).getImm());
+
+  // .loop:
+  //   lr.w destreg, (alignedaddr)
+  //   binop scratch, destreg, incr
+  //   xor scratch, destreg, scratch
+  //   and scratch, scratch, masktargetdata
+  //   xor scratch, destreg, scratch
+  //   sc.w scratch, scratch, (alignedaddr)
+  //   bnez scratch, loop
+  BuildMI(LoopMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
+      .addReg(AddrReg);
+  switch (BinOp) {
+  default:
+    llvm_unreachable("Unexpected AtomicRMW BinOp");
+  case AtomicRMWInst::Xchg:
+    BuildMI(LoopMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
+        .addReg(IncrReg)
+        .addImm(0);
+    break;
+  case AtomicRMWInst::Add:
+    BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg)
+        .addReg(DestReg)
+        .addReg(IncrReg);
+    break;
+  case AtomicRMWInst::Sub:
+    BuildMI(LoopMBB, DL, TII->get(RISCV::SUB), ScratchReg)
+        .addReg(DestReg)
+        .addReg(IncrReg);
+    break;
+  case AtomicRMWInst::Nand:
+    BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
+        .addReg(DestReg)
+        .addReg(IncrReg);
+    BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
+        .addReg(ScratchReg)
+        .addImm(-1);
+    break;
+  }
+
+  insertMaskedMerge(TII, DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
+                    ScratchReg);
+
+  BuildMI(LoopMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
+      .addReg(AddrReg)
+      .addReg(ScratchReg);
+  BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
+      .addReg(ScratchReg)
+      .addReg(RISCV::X0)
+      .addMBB(LoopMBB);
+}
+
+bool RISCVExpandAtomicPseudo::expandAtomicBinOp(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+    AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
+    MachineBasicBlock::iterator &NextMBBI) {
+  MachineInstr &MI = *MBBI;
+  DebugLoc DL = MI.getDebugLoc();
+
+  MachineFunction *MF = MBB.getParent();
+  auto LoopMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+  auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+
+  // Insert new MBBs.
+  MF->insert(++MBB.getIterator(), LoopMBB);
+  MF->insert(++LoopMBB->getIterator(), DoneMBB);
+
+  // Set up successors and transfer remaining instructions to DoneMBB.
+  LoopMBB->addSuccessor(LoopMBB);
+  LoopMBB->addSuccessor(DoneMBB);
+  DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
+  DoneMBB->transferSuccessors(&MBB);
+  MBB.addSuccessor(LoopMBB);
+
+  if (!IsMasked)
+    doAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, Width);
+  else
+    doMaskedAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp,
+                                 Width);
+
+  NextMBBI = MBB.end();
+  MI.eraseFromParent();
+
+  LivePhysRegs LiveRegs;
+  computeAndAddLiveIns(LiveRegs, *LoopMBB);
+  computeAndAddLiveIns(LiveRegs, *DoneMBB);
+
+  return true;
+}
+
+static void insertSext(const RISCVInstrInfo *TII, DebugLoc DL,
+                       MachineBasicBlock *MBB, Register ValReg,
+                       Register ShamtReg) {
+  BuildMI(MBB, DL, TII->get(RISCV::SLL), ValReg)
+      .addReg(ValReg)
+      .addReg(ShamtReg);
+  BuildMI(MBB, DL, TII->get(RISCV::SRA), ValReg)
+      .addReg(ValReg)
+      .addReg(ShamtReg);
+}
+
+bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+    AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
+    MachineBasicBlock::iterator &NextMBBI) {
+  assert(IsMasked == true &&
+         "Should only need to expand masked atomic max/min");
+  assert(Width == 32 && "Should never need to expand masked 64-bit operations");
+
+  MachineInstr &MI = *MBBI;
+  DebugLoc DL = MI.getDebugLoc();
+  MachineFunction *MF = MBB.getParent();
+  auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+  auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+  auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+  auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+
+  // Insert new MBBs.
+  MF->insert(++MBB.getIterator(), LoopHeadMBB);
+  MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
+  MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
+  MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
+
+  // Set up successors and transfer remaining instructions to DoneMBB.
+  LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
+  LoopHeadMBB->addSuccessor(LoopTailMBB);
+  LoopIfBodyMBB->addSuccessor(LoopTailMBB);
+  LoopTailMBB->addSuccessor(LoopHeadMBB);
+  LoopTailMBB->addSuccessor(DoneMBB);
+  DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
+  DoneMBB->transferSuccessors(&MBB);
+  MBB.addSuccessor(LoopHeadMBB);
+
+  Register DestReg = MI.getOperand(0).getReg();
+  Register Scratch1Reg = MI.getOperand(1).getReg();
+  Register Scratch2Reg = MI.getOperand(2).getReg();
+  Register AddrReg = MI.getOperand(3).getReg();
+  Register IncrReg = MI.getOperand(4).getReg();
+  Register MaskReg = MI.getOperand(5).getReg();
+  bool IsSigned = BinOp == AtomicRMWInst::Min || BinOp == AtomicRMWInst::Max;
+  AtomicOrdering Ordering =
+      static_cast<AtomicOrdering>(MI.getOperand(IsSigned ? 7 : 6).getImm());
+
+  //
+  // .loophead:
+  //   lr.w destreg, (alignedaddr)
+  //   and scratch2, destreg, mask
+  //   mv scratch1, destreg
+  //   [sext scratch2 if signed min/max]
+  //   ifnochangeneeded scratch2, incr, .looptail
+  BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
+      .addReg(AddrReg);
+  BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), Scratch2Reg)
+      .addReg(DestReg)
+      .addReg(MaskReg);
+  BuildMI(LoopHeadMBB, DL, TII->get(RISCV::ADDI), Scratch1Reg)
+      .addReg(DestReg)
+      .addImm(0);
+
+  switch (BinOp) {
+  default:
+    llvm_unreachable("Unexpected AtomicRMW BinOp");
+  case AtomicRMWInst::Max: {
+    insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
+    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
+        .addReg(Scratch2Reg)
+        .addReg(IncrReg)
+        .addMBB(LoopTailMBB);
+    break;
+  }
+  case AtomicRMWInst::Min: {
+    insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
+    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
+        .addReg(IncrReg)
+        .addReg(Scratch2Reg)
+        .addMBB(LoopTailMBB);
+    break;
+  }
+  case AtomicRMWInst::UMax:
+    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
+        .addReg(Scratch2Reg)
+        .addReg(IncrReg)
+        .addMBB(LoopTailMBB);
+    break;
+  case AtomicRMWInst::UMin:
+    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
+        .addReg(IncrReg)
+        .addReg(Scratch2Reg)
+        .addMBB(LoopTailMBB);
+    break;
+  }
+
+  // .loopifbody:
+  //   xor scratch1, destreg, incr
+  //   and scratch1, scratch1, mask
+  //   xor scratch1, destreg, scratch1
+  insertMaskedMerge(TII, DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
+                    MaskReg, Scratch1Reg);
+
+  // .looptail:
+  //   sc.w scratch1, scratch1, (addr)
+  //   bnez scratch1, loop
+  BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), Scratch1Reg)
+      .addReg(AddrReg)
+      .addReg(Scratch1Reg);
+  BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
+      .addReg(Scratch1Reg)
+      .addReg(RISCV::X0)
+      .addMBB(LoopHeadMBB);
+
+  NextMBBI = MBB.end();
+  MI.eraseFromParent();
+
+  LivePhysRegs LiveRegs;
+  computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
+  computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB);
+  computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
+  computeAndAddLiveIns(LiveRegs, *DoneMBB);
+
+  return true;
+}
+
+bool RISCVExpandAtomicPseudo::expandAtomicCmpXchg(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
+    int Width, MachineBasicBlock::iterator &NextMBBI) {
+  MachineInstr &MI = *MBBI;
+  DebugLoc DL = MI.getDebugLoc();
+  MachineFunction *MF = MBB.getParent();
+  auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+  auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+  auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+
+  // Insert new MBBs.
+  MF->insert(++MBB.getIterator(), LoopHeadMBB);
+  MF->insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
+  MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
+
+  // Set up successors and transfer remaining instructions to DoneMBB.
+  LoopHeadMBB->addSuccessor(LoopTailMBB);
+  LoopHeadMBB->addSuccessor(DoneMBB);
+  LoopTailMBB->addSuccessor(DoneMBB);
+  LoopTailMBB->addSuccessor(LoopHeadMBB);
+  DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
+  DoneMBB->transferSuccessors(&MBB);
+  MBB.addSuccessor(LoopHeadMBB);
+
+  Register DestReg = MI.getOperand(0).getReg();
+  Register ScratchReg = MI.getOperand(1).getReg();
+  Register AddrReg = MI.getOperand(2).getReg();
+  Register CmpValReg = MI.getOperand(3).getReg();
+  Register NewValReg = MI.getOperand(4).getReg();
+  AtomicOrdering Ordering =
+      static_cast<AtomicOrdering>(MI.getOperand(IsMasked ? 6 : 5).getImm());
+
+  if (!IsMasked) {
+    // .loophead:
+    //   lr.[w|d] dest, (addr)
+    //   bne dest, cmpval, done
+    BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
+        .addReg(AddrReg);
+    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
+        .addReg(DestReg)
+        .addReg(CmpValReg)
+        .addMBB(DoneMBB);
+    // .looptail:
+    //   sc.[w|d] scratch, newval, (addr)
+    //   bnez scratch, loophead
+    BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
+        .addReg(AddrReg)
+        .addReg(NewValReg);
+    BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
+        .addReg(ScratchReg)
+        .addReg(RISCV::X0)
+        .addMBB(LoopHeadMBB);
+  } else {
+    // .loophead:
+    //   lr.w dest, (addr)
+    //   and scratch, dest, mask
+    //   bne scratch, cmpval, done
+    Register MaskReg = MI.getOperand(5).getReg();
+    BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
+        .addReg(AddrReg);
+    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), ScratchReg)
+        .addReg(DestReg)
+        .addReg(MaskReg);
+    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
+        .addReg(ScratchReg)
+        .addReg(CmpValReg)
+        .addMBB(DoneMBB);
+
+    // .looptail:
+    //   xor scratch, dest, newval
+    //   and scratch, scratch, mask
+    //   xor scratch, dest, scratch
+    //   sc.w scratch, scratch, (adrr)
+    //   bnez scratch, loophead
+    insertMaskedMerge(TII, DL, LoopTailMBB, ScratchReg, DestReg, NewValReg,
+                      MaskReg, ScratchReg);
+    BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
+        .addReg(AddrReg)
+        .addReg(ScratchReg);
+    BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
+        .addReg(ScratchReg)
+        .addReg(RISCV::X0)
+        .addMBB(LoopHeadMBB);
+  }
+
+  NextMBBI = MBB.end();
+  MI.eraseFromParent();
+
+  LivePhysRegs LiveRegs;
+  computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
+  computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
+  computeAndAddLiveIns(LiveRegs, *DoneMBB);
+
+  return true;
+}
+
+} // end of anonymous namespace
+
+INITIALIZE_PASS(RISCVExpandAtomicPseudo, "riscv-expand-atomic-pseudo",
+                RISCV_EXPAND_ATOMIC_PSEUDO_NAME, false, false)
+
+namespace llvm {
+
+FunctionPass *createRISCVExpandAtomicPseudoPass() {
+  return new RISCVExpandAtomicPseudo();
+}
+
+} // end of namespace llvm
diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
index 84bce0f485625..33db8f231c7db 100644
--- a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
@@ -19,6 +19,7 @@
 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/MC/MCContext.h"
 
 using namespace llvm;
 
@@ -41,35 +42,18 @@ class RISCVExpandPseudo : public MachineFunctionPass {
 
 private:
   bool expandMBB(MachineBasicBlock &MBB);
-  bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
-                MachineBasicBlock::iterator &NextMBBI);
-  bool expandAtomicBinOp(MachineBasicBlock &MBB,
-                         MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
-                         bool IsMasked, int Width,
-                         MachineBasicBlock::iterator &NextMBBI);
-  bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
-                            MachineBasicBlock::iterator MBBI,
-                            AtomicRMWInst::BinOp, bool IsMasked, int Width,
-                            MachineBasicBlock::iterator &NextMBBI);
-  bool expandAtomicCmpXchg(MachineBasicBlock &MBB,
-                           MachineBasicBlock::iterator MBBI, bool IsMasked,
-                           int Width, MachineBasicBlock::iterator &NextMBBI);
+  bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
   bool expandAuipcInstPair(MachineBasicBlock &MBB,
                            MachineBasicBlock::iterator MBBI,
-                           MachineBasicBlock::iterator &NextMBBI,
                            unsigned FlagsHi, unsigned SecondOpcode);
   bool expandLoadLocalAddress(MachineBasicBlock &MBB,
-                              MachineBasicBlock::iterator MBBI,
-                              MachineBasicBlock::iterator &NextMBBI);
+                              MachineBasicBlock::iterator MBBI);
   bool expandLoadAddress(MachineBasicBlock &MBB,
-                         MachineBasicBlock::iterator MBBI,
-                         MachineBasicBlock::iterator &NextMBBI);
+                         MachineBasicBlock::iterator MBBI);
   bool expandLoadTLSIEAddress(MachineBasicBlock &MBB,
-                              MachineBasicBlock::iterator MBBI,
-                              MachineBasicBlock::iterator &NextMBBI);
+                              MachineBasicBlock::iterator MBBI);
   bool expandLoadTLSGDAddress(MachineBasicBlock &MBB,
-                              MachineBasicBlock::iterator MBBI,
-                              MachineBasicBlock::iterator &NextMBBI);
+                              MachineBasicBlock::iterator MBBI);
 };
 
 char RISCVExpandPseudo::ID = 0;
@@ -88,7 +72,7 @@ bool RISCVExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
   MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
   while (MBBI != E) {
     MachineBasicBlock::iterator NMBBI = std::next(MBBI);
-    Modified |= expandMI(MBB, MBBI, NMBBI);
+    Modified |= expandMI(MBB, MBBI);
     MBBI = NMBBI;
   }
 
@@ -96,582 +80,56 @@ bool RISCVExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
 }
 
 bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
-                                 MachineBasicBlock::iterator MBBI,
-                                 MachineBasicBlock::iterator &NextMBBI) {
+                                 MachineBasicBlock::iterator MBBI) {
   switch (MBBI->getOpcode()) {
-  case RISCV::PseudoAtomicLoadNand32:
-    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
-                             NextMBBI);
-  case RISCV::PseudoAtomicLoadNand64:
-    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
-                             NextMBBI);
-  case RISCV::PseudoMaskedAtomicSwap32:
-    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
-                             NextMBBI);
-  case RISCV::PseudoMaskedAtomicLoadAdd32:
-    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI);
-  case RISCV::PseudoMaskedAtomicLoadSub32:
-    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI);
-  case RISCV::PseudoMaskedAtomicLoadNand32:
-    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, true, 32,
-                             NextMBBI);
-  case RISCV::PseudoMaskedAtomicLoadMax32:
-    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, true, 32,
-                                NextMBBI);
-  case RISCV::PseudoMaskedAtomicLoadMin32:
-    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, true, 32,
-                                NextMBBI);
-  case RISCV::PseudoMaskedAtomicLoadUMax32:
-    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32,
-                                NextMBBI);
-  case RISCV::PseudoMaskedAtomicLoadUMin32:
-    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32,
-                                NextMBBI);
-  case RISCV::PseudoCmpXchg32:
-    return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI);
-  case RISCV::PseudoCmpXchg64:
-    return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI);
-  case RISCV::PseudoMaskedCmpXchg32:
-    return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI);
   case RISCV::PseudoLLA:
-    return expandLoadLocalAddress(MBB, MBBI, NextMBBI);
+    return expandLoadLocalAddress(MBB, MBBI);
   case RISCV::PseudoLA:
-    return expandLoadAddress(MBB, MBBI, NextMBBI);
+    return expandLoadAddress(MBB, MBBI);
   case RISCV::PseudoLA_TLS_IE:
-    return expandLoadTLSIEAddress(MBB, MBBI, NextMBBI);
+    return expandLoadTLSIEAddress(MBB, MBBI);
   case RISCV::PseudoLA_TLS_GD:
-    return expandLoadTLSGDAddress(MBB, MBBI, NextMBBI);
+    return expandLoadTLSGDAddress(MBB, MBBI);
   }
 
   return false;
 }
 
-static unsigned getLRForRMW32(AtomicOrdering Ordering) {
-  switch (Ordering) {
-  default:
-    llvm_unreachable("Unexpected AtomicOrdering");
-  case AtomicOrdering::Monotonic:
-    return RISCV::LR_W;
-  case AtomicOrdering::Acquire:
-    return RISCV::LR_W_AQ;
-  case AtomicOrdering::Release:
-    return RISCV::LR_W;
-  case AtomicOrdering::AcquireRelease:
-    return RISCV::LR_W_AQ;
-  case AtomicOrdering::SequentiallyConsistent:
-    return RISCV::LR_W_AQ_RL;
-  }
-}
-
-static unsigned getSCForRMW32(AtomicOrdering Ordering) {
-  switch (Ordering) {
-  default:
-    llvm_unreachable("Unexpected AtomicOrdering");
-  case AtomicOrdering::Monotonic:
-    return RISCV::SC_W;
-  case AtomicOrdering::Acquire:
-    return RISCV::SC_W;
-  case AtomicOrdering::Release:
-    return RISCV::SC_W_RL;
-  case AtomicOrdering::AcquireRelease:
-    return RISCV::SC_W_RL;
-  case AtomicOrdering::SequentiallyConsistent:
-    return RISCV::SC_W_AQ_RL;
-  }
-}
-
-static unsigned getLRForRMW64(AtomicOrdering Ordering) {
-  switch (Ordering) {
-  default:
-    llvm_unreachable("Unexpected AtomicOrdering");
-  case AtomicOrdering::Monotonic:
-    return RISCV::LR_D;
-  case AtomicOrdering::Acquire:
-    return RISCV::LR_D_AQ;
-  case AtomicOrdering::Release:
-    return RISCV::LR_D;
-  case AtomicOrdering::AcquireRelease:
-    return RISCV::LR_D_AQ;
-  case AtomicOrdering::SequentiallyConsistent:
-    return RISCV::LR_D_AQ_RL;
-  }
-}
-
-static unsigned getSCForRMW64(AtomicOrdering Ordering) {
-  switch (Ordering) {
-  default:
-    llvm_unreachable("Unexpected AtomicOrdering");
-  case AtomicOrdering::Monotonic:
-    return RISCV::SC_D;
-  case AtomicOrdering::Acquire:
-    return RISCV::SC_D;
-  case AtomicOrdering::Release:
-    return RISCV::SC_D_RL;
-  case AtomicOrdering::AcquireRelease:
-    return RISCV::SC_D_RL;
-  case AtomicOrdering::SequentiallyConsistent:
-    return RISCV::SC_D_AQ_RL;
-  }
-}
-
-static unsigned getLRForRMW(AtomicOrdering Ordering, int Width) {
-  if (Width == 32)
-    return getLRForRMW32(Ordering);
-  if (Width == 64)
-    return getLRForRMW64(Ordering);
-  llvm_unreachable("Unexpected LR width\n");
-}
-
-static unsigned getSCForRMW(AtomicOrdering Ordering, int Width) {
-  if (Width == 32)
-    return getSCForRMW32(Ordering);
-  if (Width == 64)
-    return getSCForRMW64(Ordering);
-  llvm_unreachable("Unexpected SC width\n");
-}
-
-static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
-                                   DebugLoc DL, MachineBasicBlock *ThisMBB,
-                                   MachineBasicBlock *LoopMBB,
-                                   MachineBasicBlock *DoneMBB,
-                                   AtomicRMWInst::BinOp BinOp, int Width) {
-  Register DestReg = MI.getOperand(0).getReg();
-  Register ScratchReg = MI.getOperand(1).getReg();
-  Register AddrReg = MI.getOperand(2).getReg();
-  Register IncrReg = MI.getOperand(3).getReg();
-  AtomicOrdering Ordering =
-      static_cast<AtomicOrdering>(MI.getOperand(4).getImm());
-
-  // .loop:
-  //   lr.[w|d] dest, (addr)
-  //   binop scratch, dest, val
-  //   sc.[w|d] scratch, scratch, (addr)
-  //   bnez scratch, loop
-  BuildMI(LoopMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
-      .addReg(AddrReg);
-  switch (BinOp) {
-  default:
-    llvm_unreachable("Unexpected AtomicRMW BinOp");
-  case AtomicRMWInst::Nand:
-    BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
-        .addReg(DestReg)
-        .addReg(IncrReg);
-    BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
-        .addReg(ScratchReg)
-        .addImm(-1);
-    break;
-  }
-  BuildMI(LoopMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
-      .addReg(AddrReg)
-      .addReg(ScratchReg);
-  BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
-      .addReg(ScratchReg)
-      .addReg(RISCV::X0)
-      .addMBB(LoopMBB);
-}
-
-static void insertMaskedMerge(const RISCVInstrInfo *TII, DebugLoc DL,
-                              MachineBasicBlock *MBB, Register DestReg,
-                              Register OldValReg, Register NewValReg,
-                              Register MaskReg, Register ScratchReg) {
-  assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique");
-  assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique");
-  assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique");
-
-  // We select bits from newval and oldval using:
-  // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge
-  // r = oldval ^ ((oldval ^ newval) & masktargetdata);
-  BuildMI(MBB, DL, TII->get(RISCV::XOR), ScratchReg)
-      .addReg(OldValReg)
-      .addReg(NewValReg);
-  BuildMI(MBB, DL, TII->get(RISCV::AND), ScratchReg)
-      .addReg(ScratchReg)
-      .addReg(MaskReg);
-  BuildMI(MBB, DL, TII->get(RISCV::XOR), DestReg)
-      .addReg(OldValReg)
-      .addReg(ScratchReg);
-}
-
-static void doMaskedAtomicBinOpExpansion(
-    const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
-    MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopMBB,
-    MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) {
-  assert(Width == 32 && "Should never need to expand masked 64-bit operations");
-  Register DestReg = MI.getOperand(0).getReg();
-  Register ScratchReg = MI.getOperand(1).getReg();
-  Register AddrReg = MI.getOperand(2).getReg();
-  Register IncrReg = MI.getOperand(3).getReg();
-  Register MaskReg = MI.getOperand(4).getReg();
-  AtomicOrdering Ordering =
-      static_cast<AtomicOrdering>(MI.getOperand(5).getImm());
-
-  // .loop:
-  //   lr.w destreg, (alignedaddr)
-  //   binop scratch, destreg, incr
-  //   xor scratch, destreg, scratch
-  //   and scratch, scratch, masktargetdata
-  //   xor scratch, destreg, scratch
-  //   sc.w scratch, scratch, (alignedaddr)
-  //   bnez scratch, loop
-  BuildMI(LoopMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
-      .addReg(AddrReg);
-  switch (BinOp) {
-  default:
-    llvm_unreachable("Unexpected AtomicRMW BinOp");
-  case AtomicRMWInst::Xchg:
-    BuildMI(LoopMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
-        .addReg(IncrReg)
-        .addImm(0);
-    break;
-  case AtomicRMWInst::Add:
-    BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg)
-        .addReg(DestReg)
-        .addReg(IncrReg);
-    break;
-  case AtomicRMWInst::Sub:
-    BuildMI(LoopMBB, DL, TII->get(RISCV::SUB), ScratchReg)
-        .addReg(DestReg)
-        .addReg(IncrReg);
-    break;
-  case AtomicRMWInst::Nand:
-    BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
-        .addReg(DestReg)
-        .addReg(IncrReg);
-    BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
-        .addReg(ScratchReg)
-        .addImm(-1);
-    break;
-  }
-
-  insertMaskedMerge(TII, DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
-                    ScratchReg);
-
-  BuildMI(LoopMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
-      .addReg(AddrReg)
-      .addReg(ScratchReg);
-  BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
-      .addReg(ScratchReg)
-      .addReg(RISCV::X0)
-      .addMBB(LoopMBB);
-}
-
-bool RISCVExpandPseudo::expandAtomicBinOp(
-    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
-    AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
-    MachineBasicBlock::iterator &NextMBBI) {
-  MachineInstr &MI = *MBBI;
-  DebugLoc DL = MI.getDebugLoc();
-
+bool RISCVExpandPseudo::expandAuipcInstPair(MachineBasicBlock &MBB,
+                                            MachineBasicBlock::iterator MBBI,
+                                            unsigned FlagsHi,
+                                            unsigned SecondOpcode) {
   MachineFunction *MF = MBB.getParent();
-  auto LoopMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
-  auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
-
-  // Insert new MBBs.
-  MF->insert(++MBB.getIterator(), LoopMBB);
-  MF->insert(++LoopMBB->getIterator(), DoneMBB);
-
-  // Set up successors and transfer remaining instructions to DoneMBB.
-  LoopMBB->addSuccessor(LoopMBB);
-  LoopMBB->addSuccessor(DoneMBB);
-  DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
-  DoneMBB->transferSuccessors(&MBB);
-  MBB.addSuccessor(LoopMBB);
-
-  if (!IsMasked)
-    doAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, Width);
-  else
-    doMaskedAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp,
-                                 Width);
-
-  NextMBBI = MBB.end();
-  MI.eraseFromParent();
-
-  LivePhysRegs LiveRegs;
-  computeAndAddLiveIns(LiveRegs, *LoopMBB);
-  computeAndAddLiveIns(LiveRegs, *DoneMBB);
-
-  return true;
-}
-
-static void insertSext(const RISCVInstrInfo *TII, DebugLoc DL,
-                       MachineBasicBlock *MBB, Register ValReg,
-                       Register ShamtReg) {
-  BuildMI(MBB, DL, TII->get(RISCV::SLL), ValReg)
-      .addReg(ValReg)
-      .addReg(ShamtReg);
-  BuildMI(MBB, DL, TII->get(RISCV::SRA), ValReg)
-      .addReg(ValReg)
-      .addReg(ShamtReg);
-}
-
-bool RISCVExpandPseudo::expandAtomicMinMaxOp(
-    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
-    AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
-    MachineBasicBlock::iterator &NextMBBI) {
-  assert(IsMasked == true &&
-         "Should only need to expand masked atomic max/min");
-  assert(Width == 32 && "Should never need to expand masked 64-bit operations");
-
-  MachineInstr &MI = *MBBI;
-  DebugLoc DL = MI.getDebugLoc();
-  MachineFunction *MF = MBB.getParent();
-  auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
-  auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
-  auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
-  auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
-
-  // Insert new MBBs.
-  MF->insert(++MBB.getIterator(), LoopHeadMBB);
-  MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
-  MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
-  MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
-
-  // Set up successors and transfer remaining instructions to DoneMBB.
-  LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
-  LoopHeadMBB->addSuccessor(LoopTailMBB);
-  LoopIfBodyMBB->addSuccessor(LoopTailMBB);
-  LoopTailMBB->addSuccessor(LoopHeadMBB);
-  LoopTailMBB->addSuccessor(DoneMBB);
-  DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
-  DoneMBB->transferSuccessors(&MBB);
-  MBB.addSuccessor(LoopHeadMBB);
-
-  Register DestReg = MI.getOperand(0).getReg();
-  Register Scratch1Reg = MI.getOperand(1).getReg();
-  Register Scratch2Reg = MI.getOperand(2).getReg();
-  Register AddrReg = MI.getOperand(3).getReg();
-  Register IncrReg = MI.getOperand(4).getReg();
-  Register MaskReg = MI.getOperand(5).getReg();
-  bool IsSigned = BinOp == AtomicRMWInst::Min || BinOp == AtomicRMWInst::Max;
-  AtomicOrdering Ordering =
-      static_cast<AtomicOrdering>(MI.getOperand(IsSigned ? 7 : 6).getImm());
-
-  //
-  // .loophead:
-  //   lr.w destreg, (alignedaddr)
-  //   and scratch2, destreg, mask
-  //   mv scratch1, destreg
-  //   [sext scratch2 if signed min/max]
-  //   ifnochangeneeded scratch2, incr, .looptail
-  BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
-      .addReg(AddrReg);
-  BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), Scratch2Reg)
-      .addReg(DestReg)
-      .addReg(MaskReg);
-  BuildMI(LoopHeadMBB, DL, TII->get(RISCV::ADDI), Scratch1Reg)
-      .addReg(DestReg)
-      .addImm(0);
-
-  switch (BinOp) {
-  default:
-    llvm_unreachable("Unexpected AtomicRMW BinOp");
-  case AtomicRMWInst::Max: {
-    insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
-    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
-        .addReg(Scratch2Reg)
-        .addReg(IncrReg)
-        .addMBB(LoopTailMBB);
-    break;
-  }
-  case AtomicRMWInst::Min: {
-    insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
-    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
-        .addReg(IncrReg)
-        .addReg(Scratch2Reg)
-        .addMBB(LoopTailMBB);
-    break;
-  }
-  case AtomicRMWInst::UMax:
-    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
-        .addReg(Scratch2Reg)
-        .addReg(IncrReg)
-        .addMBB(LoopTailMBB);
-    break;
-  case AtomicRMWInst::UMin:
-    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
-        .addReg(IncrReg)
-        .addReg(Scratch2Reg)
-        .addMBB(LoopTailMBB);
-    break;
-  }
-
-  // .loopifbody:
-  //   xor scratch1, destreg, incr
-  //   and scratch1, scratch1, mask
-  //   xor scratch1, destreg, scratch1
-  insertMaskedMerge(TII, DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
-                    MaskReg, Scratch1Reg);
-
-  // .looptail:
-  //   sc.w scratch1, scratch1, (addr)
-  //   bnez scratch1, loop
-  BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), Scratch1Reg)
-      .addReg(AddrReg)
-      .addReg(Scratch1Reg);
-  BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
-      .addReg(Scratch1Reg)
-      .addReg(RISCV::X0)
-      .addMBB(LoopHeadMBB);
-
-  NextMBBI = MBB.end();
-  MI.eraseFromParent();
-
-  LivePhysRegs LiveRegs;
-  computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
-  computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB);
-  computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
-  computeAndAddLiveIns(LiveRegs, *DoneMBB);
-
-  return true;
-}
-
-bool RISCVExpandPseudo::expandAtomicCmpXchg(
-    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
-    int Width, MachineBasicBlock::iterator &NextMBBI) {
   MachineInstr &MI = *MBBI;
   DebugLoc DL = MI.getDebugLoc();
-  MachineFunction *MF = MBB.getParent();
-  auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
-  auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
-  auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
-
-  // Insert new MBBs.
-  MF->insert(++MBB.getIterator(), LoopHeadMBB);
-  MF->insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
-  MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
-
-  // Set up successors and transfer remaining instructions to DoneMBB.
-  LoopHeadMBB->addSuccessor(LoopTailMBB);
-  LoopHeadMBB->addSuccessor(DoneMBB);
-  LoopTailMBB->addSuccessor(DoneMBB);
-  LoopTailMBB->addSuccessor(LoopHeadMBB);
-  DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
-  DoneMBB->transferSuccessors(&MBB);
-  MBB.addSuccessor(LoopHeadMBB);
 
   Register DestReg = MI.getOperand(0).getReg();
-  Register ScratchReg = MI.getOperand(1).getReg();
-  Register AddrReg = MI.getOperand(2).getReg();
-  Register CmpValReg = MI.getOperand(3).getReg();
-  Register NewValReg = MI.getOperand(4).getReg();
-  AtomicOrdering Ordering =
-      static_cast<AtomicOrdering>(MI.getOperand(IsMasked ? 6 : 5).getImm());
-
-  if (!IsMasked) {
-    // .loophead:
-    //   lr.[w|d] dest, (addr)
-    //   bne dest, cmpval, done
-    BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
-        .addReg(AddrReg);
-    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
-        .addReg(DestReg)
-        .addReg(CmpValReg)
-        .addMBB(DoneMBB);
-    // .looptail:
-    //   sc.[w|d] scratch, newval, (addr)
-    //   bnez scratch, loophead
-    BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
-        .addReg(AddrReg)
-        .addReg(NewValReg);
-    BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
-        .addReg(ScratchReg)
-        .addReg(RISCV::X0)
-        .addMBB(LoopHeadMBB);
-  } else {
-    // .loophead:
-    //   lr.w dest, (addr)
-    //   and scratch, dest, mask
-    //   bne scratch, cmpval, done
-    Register MaskReg = MI.getOperand(5).getReg();
-    BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
-        .addReg(AddrReg);
-    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), ScratchReg)
-        .addReg(DestReg)
-        .addReg(MaskReg);
-    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
-        .addReg(ScratchReg)
-        .addReg(CmpValReg)
-        .addMBB(DoneMBB);
-
-    // .looptail:
-    //   xor scratch, dest, newval
-    //   and scratch, scratch, mask
-    //   xor scratch, dest, scratch
-    //   sc.w scratch, scratch, (adrr)
-    //   bnez scratch, loophead
-    insertMaskedMerge(TII, DL, LoopTailMBB, ScratchReg, DestReg, NewValReg,
-                      MaskReg, ScratchReg);
-    BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
-        .addReg(AddrReg)
-        .addReg(ScratchReg);
-    BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
-        .addReg(ScratchReg)
-        .addReg(RISCV::X0)
-        .addMBB(LoopHeadMBB);
-  }
-
-  NextMBBI = MBB.end();
-  MI.eraseFromParent();
+  Register ScratchReg =
+      MF->getRegInfo().createVirtualRegister(&RISCV::GPRRegClass);
 
-  LivePhysRegs LiveRegs;
-  computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
-  computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
-  computeAndAddLiveIns(LiveRegs, *DoneMBB);
+  MachineOperand &Symbol = MI.getOperand(1);
+  Symbol.setTargetFlags(FlagsHi);
+  MCSymbol *AUIPCSymbol = MF->getContext().createTempSymbol(false);
 
-  return true;
-}
+  MachineInstr *MIAUIPC =
+      BuildMI(MBB, MBBI, DL, TII->get(RISCV::AUIPC), ScratchReg).add(Symbol);
+  MIAUIPC->setPreInstrSymbol(*MF, AUIPCSymbol);
 
-bool RISCVExpandPseudo::expandAuipcInstPair(
-    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
-    MachineBasicBlock::iterator &NextMBBI, unsigned FlagsHi,
-    unsigned SecondOpcode) {
-  MachineFunction *MF = MBB.getParent();
-  MachineInstr &MI = *MBBI;
-  DebugLoc DL = MI.getDebugLoc();
-
-  Register DestReg = MI.getOperand(0).getReg();
-  const MachineOperand &Symbol = MI.getOperand(1);
-
-  MachineBasicBlock *NewMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
-
-  // Tell AsmPrinter that we unconditionally want the symbol of this label to be
-  // emitted.
-  NewMBB->setLabelMustBeEmitted();
-
-  MF->insert(++MBB.getIterator(), NewMBB);
-
-  BuildMI(NewMBB, DL, TII->get(RISCV::AUIPC), DestReg)
-      .addDisp(Symbol, 0, FlagsHi);
-  BuildMI(NewMBB, DL, TII->get(SecondOpcode), DestReg)
-      .addReg(DestReg)
-      .addMBB(NewMBB, RISCVII::MO_PCREL_LO);
-
-  // Move all the rest of the instructions to NewMBB.
-  NewMBB->splice(NewMBB->end(), &MBB, std::next(MBBI), MBB.end());
-  // Update machine-CFG edges.
-  NewMBB->transferSuccessorsAndUpdatePHIs(&MBB);
-  // Make the original basic block fall-through to the new.
-  MBB.addSuccessor(NewMBB);
-
-  // Make sure live-ins are correctly attached to this new basic block.
-  LivePhysRegs LiveRegs;
-  computeAndAddLiveIns(LiveRegs, *NewMBB);
+  BuildMI(MBB, MBBI, DL, TII->get(SecondOpcode), DestReg)
+      .addReg(ScratchReg)
+      .addSym(AUIPCSymbol, RISCVII::MO_PCREL_LO);
 
-  NextMBBI = MBB.end();
   MI.eraseFromParent();
   return true;
 }
 
 bool RISCVExpandPseudo::expandLoadLocalAddress(
-    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
-    MachineBasicBlock::iterator &NextMBBI) {
-  return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_PCREL_HI,
-                             RISCV::ADDI);
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
+  return expandAuipcInstPair(MBB, MBBI, RISCVII::MO_PCREL_HI, RISCV::ADDI);
 }
 
-bool RISCVExpandPseudo::expandLoadAddress(
-    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
-    MachineBasicBlock::iterator &NextMBBI) {
+bool RISCVExpandPseudo::expandLoadAddress(MachineBasicBlock &MBB,
+                                          MachineBasicBlock::iterator MBBI) {
   MachineFunction *MF = MBB.getParent();
 
   unsigned SecondOpcode;
@@ -684,25 +142,21 @@ bool RISCVExpandPseudo::expandLoadAddress(
     SecondOpcode = RISCV::ADDI;
     FlagsHi = RISCVII::MO_PCREL_HI;
   }
-  return expandAuipcInstPair(MBB, MBBI, NextMBBI, FlagsHi, SecondOpcode);
+  return expandAuipcInstPair(MBB, MBBI, FlagsHi, SecondOpcode);
 }
 
 bool RISCVExpandPseudo::expandLoadTLSIEAddress(
-    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
-    MachineBasicBlock::iterator &NextMBBI) {
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
   MachineFunction *MF = MBB.getParent();
 
   const auto &STI = MF->getSubtarget<RISCVSubtarget>();
   unsigned SecondOpcode = STI.is64Bit() ? RISCV::LD : RISCV::LW;
-  return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_TLS_GOT_HI,
-                             SecondOpcode);
+  return expandAuipcInstPair(MBB, MBBI, RISCVII::MO_TLS_GOT_HI, SecondOpcode);
 }
 
 bool RISCVExpandPseudo::expandLoadTLSGDAddress(
-    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
-    MachineBasicBlock::iterator &NextMBBI) {
-  return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_TLS_GD_HI,
-                             RISCV::ADDI);
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
+  return expandAuipcInstPair(MBB, MBBI, RISCVII::MO_TLS_GD_HI, RISCV::ADDI);
 }
 
 } // end of anonymous namespace
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 8c9f2c893e9bf..43adc7426c79d 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -31,7 +31,7 @@ static int getLibCallID(const MachineFunction &MF,
                         const std::vector<CalleeSavedInfo> &CSI) {
   const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
 
-  if (CSI.empty() || !RVFI->useSaveRestoreLibCalls())
+  if (CSI.empty() || !RVFI->useSaveRestoreLibCalls(MF))
     return -1;
 
   Register MaxReg = RISCV::NoRegister;
@@ -573,8 +573,8 @@ void RISCVFrameLowering::processFunctionBeforeFrameFinalized(
   // still needs an emergency spill slot for branch relaxation. This case
   // would currently be missed.
   if (!isInt<11>(MFI.estimateStackSize(MF))) {
-    int RegScavFI = MFI.CreateStackObject(
-        RegInfo->getSpillSize(*RC), RegInfo->getSpillAlignment(*RC), false);
+    int RegScavFI = MFI.CreateStackObject(RegInfo->getSpillSize(*RC),
+                                          RegInfo->getSpillAlign(*RC), false);
     RS->addScavengingFrameIndex(RegScavFI);
   }
 }
@@ -731,9 +731,10 @@ bool RISCVFrameLowering::restoreCalleeSavedRegisters(
 
 bool RISCVFrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const {
   MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
-  const auto *RVFI = MBB.getParent()->getInfo<RISCVMachineFunctionInfo>();
+  const MachineFunction *MF = MBB.getParent();
+  const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
 
-  if (!RVFI->useSaveRestoreLibCalls())
+  if (!RVFI->useSaveRestoreLibCalls(*MF))
     return true;
 
   // Inserting a call to a __riscv_save libcall requires the use of the register
@@ -746,10 +747,11 @@ bool RISCVFrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const {
 }
 
 bool RISCVFrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
+  const MachineFunction *MF = MBB.getParent();
   MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
-  const auto *RVFI = MBB.getParent()->getInfo<RISCVMachineFunctionInfo>();
+  const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
 
-  if (!RVFI->useSaveRestoreLibCalls())
+  if (!RVFI->useSaveRestoreLibCalls(*MF))
     return true;
 
   // Using the __riscv_restore libcalls to restore CSRs requires a tail call.
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 7a86d5e80bce3..a0ae05081adcb 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -14,6 +14,7 @@
 #include "MCTargetDesc/RISCVMCTargetDesc.h"
 #include "Utils/RISCVMatInt.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/Support/Alignment.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
@@ -74,6 +75,30 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
   EVT VT = Node->getValueType(0);
 
   switch (Opcode) {
+  case ISD::ADD: {
+    // Optimize (add r, imm) to (addi (addi r, imm0) imm1) if applicable. The
+    // immediate must be in specific ranges and have a single use.
+    if (auto *ConstOp = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
+      if (!(ConstOp->hasOneUse()))
+        break;
+      // The imm must be in range [-4096,-2049] or [2048,4094].
+      int64_t Imm = ConstOp->getSExtValue();
+      if (!(-4096 <= Imm && Imm <= -2049) && !(2048 <= Imm && Imm <= 4094))
+        break;
+      // Break the imm to imm0+imm1.
+      SDLoc DL(Node);
+      EVT VT = Node->getValueType(0);
+      const SDValue ImmOp0 = CurDAG->getTargetConstant(Imm - Imm / 2, DL, VT);
+      const SDValue ImmOp1 = CurDAG->getTargetConstant(Imm / 2, DL, VT);
+      auto *NodeAddi0 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
+                                               Node->getOperand(0), ImmOp0);
+      auto *NodeAddi1 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
+                                               SDValue(NodeAddi0, 0), ImmOp1);
+      ReplaceNode(Node, NodeAddi1);
+      return;
+    }
+    break;
+  }
   case ISD::Constant: {
     auto ConstNode = cast<ConstantSDNode>(Node);
     if (VT == XLenVT && ConstNode->isNullValue()) {
@@ -160,8 +185,9 @@ bool RISCVDAGToDAGISel::SelectAddrFI(SDValue Addr, SDValue &Base) {
 }
 
 // Merge an ADDI into the offset of a load/store instruction where possible.
-// (load (add base, off), 0) -> (load base, off)
-// (store val, (add base, off)) -> (store val, base, off)
+// (load (addi base, off1), off2) -> (load base, off1+off2)
+// (store val, (addi base, off1), off2) -> (store val, base, off1+off2)
+// This is possible when off1+off2 fits a 12-bit immediate.
 void RISCVDAGToDAGISel::doPeepholeLoadStoreADDI() {
   SelectionDAG::allnodes_iterator Position(CurDAG->getRoot().getNode());
   ++Position;
@@ -202,10 +228,7 @@ void RISCVDAGToDAGISel::doPeepholeLoadStoreADDI() {
       break;
     }
 
-    // Currently, the load/store offset must be 0 to be considered for this
-    // peephole optimisation.
-    if (!isa<ConstantSDNode>(N->getOperand(OffsetOpIdx)) ||
-        N->getConstantOperandVal(OffsetOpIdx) != 0)
+    if (!isa<ConstantSDNode>(N->getOperand(OffsetOpIdx)))
       continue;
 
     SDValue Base = N->getOperand(BaseOpIdx);
@@ -215,18 +238,39 @@ void RISCVDAGToDAGISel::doPeepholeLoadStoreADDI() {
       continue;
 
     SDValue ImmOperand = Base.getOperand(1);
+    uint64_t Offset2 = N->getConstantOperandVal(OffsetOpIdx);
 
     if (auto Const = dyn_cast<ConstantSDNode>(ImmOperand)) {
-      ImmOperand = CurDAG->getTargetConstant(
-          Const->getSExtValue(), SDLoc(ImmOperand), ImmOperand.getValueType());
+      int64_t Offset1 = Const->getSExtValue();
+      int64_t CombinedOffset = Offset1 + Offset2;
+      if (!isInt<12>(CombinedOffset))
+        continue;
+      ImmOperand = CurDAG->getTargetConstant(CombinedOffset, SDLoc(ImmOperand),
+                                             ImmOperand.getValueType());
     } else if (auto GA = dyn_cast<GlobalAddressSDNode>(ImmOperand)) {
+      // If the off1 in (addi base, off1) is a global variable's address (its
+      // low part, really), then we can rely on the alignment of that variable
+      // to provide a margin of safety before off1 can overflow the 12 bits.
+      // Check if off2 falls within that margin; if so off1+off2 can't overflow.
+      const DataLayout &DL = CurDAG->getDataLayout();
+      Align Alignment = GA->getGlobal()->getPointerAlignment(DL);
+      if (Offset2 != 0 && Alignment <= Offset2)
+        continue;
+      int64_t Offset1 = GA->getOffset();
+      int64_t CombinedOffset = Offset1 + Offset2;
       ImmOperand = CurDAG->getTargetGlobalAddress(
           GA->getGlobal(), SDLoc(ImmOperand), ImmOperand.getValueType(),
-          GA->getOffset(), GA->getTargetFlags());
+          CombinedOffset, GA->getTargetFlags());
     } else if (auto CP = dyn_cast<ConstantPoolSDNode>(ImmOperand)) {
+      // Ditto.
+      Align Alignment = CP->getAlign();
+      if (Offset2 != 0 && Alignment <= Offset2)
+        continue;
+      int64_t Offset1 = CP->getOffset();
+      int64_t CombinedOffset = Offset1 + Offset2;
       ImmOperand = CurDAG->getTargetConstantPool(
           CP->getConstVal(), ImmOperand.getValueType(), CP->getAlign(),
-          CP->getOffset(), CP->getTargetFlags());
+          CombinedOffset, CP->getTargetFlags());
     } else {
       continue;
     }
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 6abd41531daeb..91fc69b5bc10e 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -33,6 +33,7 @@
 #include "llvm/IR/IntrinsicsRISCV.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
@@ -236,6 +237,12 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
   // Effectively disable jump table generation.
   setMinimumJumpTableEntries(INT_MAX);
+
+  // Jumps are expensive, compared to logic
+  setJumpIsExpensive();
+
+  // We can use any register for comparisons
+  setHasMultipleConditionRegisters();
 }
 
 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &,
@@ -1234,7 +1241,7 @@ static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
   Register HiReg = MI.getOperand(1).getReg();
   Register SrcReg = MI.getOperand(2).getReg();
   const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
-  int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex();
+  int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
 
   TII.storeRegToStackSlot(*BB, MI, SrcReg, MI.getOperand(2).isKill(), FI, SrcRC,
                           RI);
@@ -1266,7 +1273,7 @@ static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI,
   Register LoReg = MI.getOperand(1).getReg();
   Register HiReg = MI.getOperand(2).getReg();
   const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
-  int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex();
+  int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
 
   MachineMemOperand *MMO =
       MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI),
@@ -1492,10 +1499,11 @@ static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
                                      VA1.getLocVT(), CCValAssign::Full));
   } else {
     // Both halves must be passed on the stack, with proper alignment.
-    unsigned StackAlign = std::max(XLenInBytes, ArgFlags1.getOrigAlign());
+    Align StackAlign =
+        std::max(Align(XLenInBytes), ArgFlags1.getNonZeroOrigAlign());
     State.addLoc(
         CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(),
-                            State.AllocateStack(XLenInBytes, Align(StackAlign)),
+                            State.AllocateStack(XLenInBytes, StackAlign),
                             VA1.getLocVT(), CCValAssign::Full));
     State.addLoc(CCValAssign::getMem(
         ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
@@ -1579,7 +1587,7 @@ static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
   // original type is larger than 2*XLEN, so the register alignment rule does
   // not apply.
   unsigned TwoXLenInBytes = (2 * XLen) / 8;
-  if (!IsFixed && ArgFlags.getOrigAlign() == TwoXLenInBytes &&
+  if (!IsFixed && ArgFlags.getNonZeroOrigAlign() == TwoXLenInBytes &&
       DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes) {
     unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
     // Skip 'odd' register if necessary.
@@ -2971,6 +2979,26 @@ bool RISCVTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
   return true;
 }
 
+bool RISCVTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
+                                                 SDValue C) const {
+  // Check integral scalar types.
+  if (VT.isScalarInteger()) {
+    // Do not perform the transformation on riscv32 with the M extension.
+    if (!Subtarget.is64Bit() && Subtarget.hasStdExtM())
+      return false;
+    if (auto *ConstNode = dyn_cast<ConstantSDNode>(C.getNode())) {
+      if (ConstNode->getAPIntValue().getBitWidth() > 8 * sizeof(int64_t))
+        return false;
+      int64_t Imm = ConstNode->getSExtValue();
+      if (isPowerOf2_64(Imm + 1) || isPowerOf2_64(Imm - 1) ||
+          isPowerOf2_64(1 - Imm) || isPowerOf2_64(-1 - Imm))
+        return true;
+    }
+  }
+
+  return false;
+}
+
 #define GET_REGISTER_MATCHER
 #include "RISCVGenAsmMatcher.inc"
 
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 691bb6d75d139..e420e879efc91 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -185,6 +185,9 @@ class RISCVTargetLowering : public TargetLowering {
   bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
   bool shouldConsiderGEPOffsetSplit() const override { return true; }
 
+  bool decomposeMulByConstant(LLVMContext &Context, EVT VT,
+                              SDValue C) const override;
+
   TargetLowering::AtomicExpansionKind
   shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
   Value *emitMaskedAtomicRMWIntrinsic(IRBuilder<> &Builder, AtomicRMWInst *AI,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormats.td b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
index 3ed10cca5377e..a47945a6a5154 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrFormats.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
@@ -49,6 +49,19 @@ def InstFormatCB     : InstFormat<15>;
 def InstFormatCJ     : InstFormat<16>;
 def InstFormatOther  : InstFormat<17>;
 
+class RISCVVConstraint<bits<4> val> {
+  bits<4> Value = val;
+}
+def NoConstraint : RISCVVConstraint<0>;
+def WidenV       : RISCVVConstraint<1>;
+def WidenW       : RISCVVConstraint<2>;
+def WidenCvt     : RISCVVConstraint<3>;
+def Narrow       : RISCVVConstraint<4>;
+def Iota         : RISCVVConstraint<5>;
+def SlideUp      : RISCVVConstraint<6>;
+def Vrgather     : RISCVVConstraint<7>;
+def Vcompress    : RISCVVConstraint<8>;
+
 // The following opcode names match those given in Table 19.1 in the
 // RISC-V User-level ISA specification ("RISC-V base opcode map").
 class RISCVOpcode<bits<7> val> {
@@ -71,6 +84,7 @@ def OPC_MSUB      : RISCVOpcode<0b1000111>;
 def OPC_NMSUB     : RISCVOpcode<0b1001011>;
 def OPC_NMADD     : RISCVOpcode<0b1001111>;
 def OPC_OP_FP     : RISCVOpcode<0b1010011>;
+def OPC_OP_V      : RISCVOpcode<0b1010111>;
 def OPC_BRANCH    : RISCVOpcode<0b1100011>;
 def OPC_JALR      : RISCVOpcode<0b1100111>;
 def OPC_JAL       : RISCVOpcode<0b1101111>;
@@ -99,6 +113,10 @@ class RVInst<dag outs, dag ins, string opcodestr, string argstr,
   let Pattern = pattern;
 
   let TSFlags{4-0} = format.Value;
+
+  // Defaults
+  RISCVVConstraint RVVConstraint = NoConstraint;
+  let TSFlags{8-5} = RVVConstraint.Value;
 }
 
 // Pseudo instructions
diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td b/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td
new file mode 100644
index 0000000000000..e5f154966ba61
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td
@@ -0,0 +1,300 @@
+//===-- RISCVInstrFormatsV.td - RISCV V Instruction Formats --*- tablegen -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file describes the RISC-V V extension instruction formats.
+//
+//===----------------------------------------------------------------------===//
+
+class RISCVVFormat<bits<3> val> {
+  bits<3> Value = val;
+}
+def OPIVV : RISCVVFormat<0b000>;
+def OPFVV : RISCVVFormat<0b001>;
+def OPMVV : RISCVVFormat<0b010>;
+def OPIVI : RISCVVFormat<0b011>;
+def OPIVX : RISCVVFormat<0b100>;
+def OPFVF : RISCVVFormat<0b101>;
+def OPMVX : RISCVVFormat<0b110>;
+
+class RISCVMOP<bits<3> val> {
+  bits<3> Value = val;
+}
+def MOPLDUnitStrideU : RISCVMOP<0b000>;
+def MOPLDStridedU    : RISCVMOP<0b010>;
+def MOPLDIndexedU    : RISCVMOP<0b011>;
+def MOPLDUnitStrideS : RISCVMOP<0b100>;
+def MOPLDStridedS    : RISCVMOP<0b110>;
+def MOPLDIndexedS    : RISCVMOP<0b111>;
+
+def MOPSTUnitStride  : RISCVMOP<0b000>;
+def MOPSTStrided     : RISCVMOP<0b010>;
+def MOPSTIndexedOrder: RISCVMOP<0b011>;
+def MOPSTIndexedUnOrd: RISCVMOP<0b111>;
+
+class RISCVLSUMOP<bits<5> val> {
+  bits<5> Value = val;
+}
+def LUMOPUnitStride  : RISCVLSUMOP<0b00000>;
+def LUMOPUnitStrideWholeReg : RISCVLSUMOP<0b01000>;
+def LUMOPUnitStrideFF: RISCVLSUMOP<0b10000>;
+def SUMOPUnitStride  : RISCVLSUMOP<0b00000>;
+def SUMOPUnitStrideWholeReg : RISCVLSUMOP<0b01000>;
+
+class RISCVWidth<bits<3> val> {
+  bits<3> Value = val;
+}
+def LSWidthVByte : RISCVWidth<0b000>;
+def LSWidthVHalf : RISCVWidth<0b101>;
+def LSWidthVWord : RISCVWidth<0b110>;
+def LSWidthVSEW  : RISCVWidth<0b111>;
+
+class RVInstSetVLi<dag outs, dag ins, string opcodestr, string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatI> {
+  bits<5> rs1;
+  bits<5> rd;
+  bits<11> vtypei;
+
+  let Inst{31} = 0;
+  let Inst{30-20} = vtypei;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = 0b111;
+  let Inst{11-7} = rd;
+  let Opcode = OPC_OP_V.Value;
+
+  let Defs = [VTYPE, VL];
+}
+
+class RVInstSetVL<dag outs, dag ins, string opcodestr, string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
+  bits<5> rs2;
+  bits<5> rs1;
+  bits<5> rd;
+
+  let Inst{31} = 1;
+  let Inst{30-25} = 0b000000;
+  let Inst{24-20} = rs2;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = 0b111;
+  let Inst{11-7} = rd;
+  let Opcode = OPC_OP_V.Value;
+
+  let Defs = [VTYPE, VL];
+}
+
+class RVInstVV<bits<6> funct6, RISCVVFormat opv, dag outs, dag ins,
+               string opcodestr, string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
+  bits<5> vs2;
+  bits<5> vs1;
+  bits<5> vd;
+  bit vm;
+
+  let Inst{31-26} = funct6;
+  let Inst{25} = vm;
+  let Inst{24-20} = vs2;
+  let Inst{19-15} = vs1;
+  let Inst{14-12} = opv.Value;
+  let Inst{11-7} = vd;
+  let Opcode = OPC_OP_V.Value;
+
+  let Uses = [VTYPE, VL];
+}
+
+class RVInstVX<bits<6> funct6, RISCVVFormat opv, dag outs, dag ins,
+                string opcodestr, string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
+  bits<5> vs2;
+  bits<5> rs1;
+  bits<5> vd;
+  bit vm;
+
+  let Inst{31-26} = funct6;
+  let Inst{25} = vm;
+  let Inst{24-20} = vs2;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = opv.Value;
+  let Inst{11-7} = vd;
+  let Opcode = OPC_OP_V.Value;
+
+  let Uses = [VTYPE, VL];
+}
+
+class RVInstV2<bits<6> funct6, bits<5> vs2, RISCVVFormat opv, dag outs, dag ins,
+                string opcodestr, string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
+  bits<5> rs1;
+  bits<5> vd;
+  bit vm;
+
+  let Inst{31-26} = funct6;
+  let Inst{25} = vm;
+  let Inst{24-20} = vs2;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = opv.Value;
+  let Inst{11-7} = vd;
+  let Opcode = OPC_OP_V.Value;
+
+  let Uses = [VTYPE, VL];
+}
+
+class RVInstIVI<bits<6> funct6, dag outs, dag ins, string opcodestr,
+                string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
+  bits<5> vs2;
+  bits<5> imm;
+  bits<5> vd;
+  bit vm;
+
+  let Inst{31-26} = funct6;
+  let Inst{25} = vm;
+  let Inst{24-20} = vs2;
+  let Inst{19-15} = imm;
+  let Inst{14-12} = 0b011;
+  let Inst{11-7} = vd;
+  let Opcode = OPC_OP_V.Value;
+
+  let Uses = [VTYPE, VL];
+}
+
+class RVInstV<bits<6> funct6, bits<5> vs1, RISCVVFormat opv, dag outs,
+              dag ins, string opcodestr, string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
+  bits<5> vs2;
+  bits<5> vd;
+  bit vm;
+
+  let Inst{31-26} = funct6;
+  let Inst{25} = vm;
+  let Inst{24-20} = vs2;
+  let Inst{19-15} = vs1;
+  let Inst{14-12} = opv.Value;
+  let Inst{11-7} = vd;
+  let Opcode = OPC_OP_V.Value;
+
+  let Uses = [VTYPE, VL];
+}
+
+class RVInstVLU<bits<3> nf, RISCVMOP mop, RISCVLSUMOP lumop,
+                RISCVWidth width, dag outs, dag ins, string opcodestr,
+                string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
+  bits<5> rs1;
+  bits<5> vd;
+  bit vm;
+
+  let Inst{31-29} = nf;
+  let Inst{28-26} = mop.Value;
+  let Inst{25} = vm;
+  let Inst{24-20} = lumop.Value;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = width.Value;
+  let Inst{11-7} = vd;
+  let Opcode = OPC_LOAD_FP.Value;
+
+  let Uses = [VTYPE, VL];
+}
+
+class RVInstVLS<bits<3> nf, RISCVMOP mop, RISCVWidth width,
+                dag outs, dag ins, string opcodestr, string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
+  bits<5> rs2;
+  bits<5> rs1;
+  bits<5> vd;
+  bit vm;
+
+  let Inst{31-29} = nf;
+  let Inst{28-26} = mop.Value;
+  let Inst{25} = vm;
+  let Inst{24-20} = rs2;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = width.Value;
+  let Inst{11-7} = vd;
+  let Opcode = OPC_LOAD_FP.Value;
+
+  let Uses = [VTYPE, VL];
+}
+
+class RVInstVLX<bits<3> nf, RISCVMOP mop, RISCVWidth width,
+                dag outs, dag ins, string opcodestr, string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
+  bits<5> vs2;
+  bits<5> rs1;
+  bits<5> vd;
+  bit vm;
+
+  let Inst{31-29} = nf;
+  let Inst{28-26} = mop.Value;
+  let Inst{25} = vm;
+  let Inst{24-20} = vs2;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = width.Value;
+  let Inst{11-7} = vd;
+  let Opcode = OPC_LOAD_FP.Value;
+
+  let Uses = [VTYPE, VL];
+}
+
+class RVInstVSU<bits<3> nf, RISCVMOP mop, RISCVLSUMOP sumop,
+                RISCVWidth width, dag outs, dag ins, string opcodestr,
+                string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
+  bits<5> rs1;
+  bits<5> vs3;
+  bit vm;
+
+  let Inst{31-29} = nf;
+  let Inst{28-26} = mop.Value;
+  let Inst{25} = vm;
+  let Inst{24-20} = sumop.Value;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = width.Value;
+  let Inst{11-7} = vs3;
+  let Opcode = OPC_STORE_FP.Value;
+
+  let Uses = [VTYPE, VL];
+}
+
+class RVInstVSS<bits<3> nf, RISCVMOP mop, RISCVWidth width,
+                dag outs, dag ins, string opcodestr, string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
+  bits<5> rs2;
+  bits<5> rs1;
+  bits<5> vs3;
+  bit vm;
+
+  let Inst{31-29} = nf;
+  let Inst{28-26} = mop.Value;
+  let Inst{25} = vm;
+  let Inst{24-20} = rs2;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = width.Value;
+  let Inst{11-7} = vs3;
+  let Opcode = OPC_STORE_FP.Value;
+
+  let Uses = [VTYPE, VL];
+}
+
+class RVInstVSX<bits<3> nf, RISCVMOP mop, RISCVWidth width,
+                dag outs, dag ins, string opcodestr, string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
+  bits<5> vs2;
+  bits<5> rs1;
+  bits<5> vs3;
+  bit vm;
+
+  let Inst{31-29} = nf;
+  let Inst{28-26} = mop.Value;
+  let Inst{25} = vm;
+  let Inst{24-20} = vs2;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = width.Value;
+  let Inst{11-7} = vs3;
+  let Opcode = OPC_STORE_FP.Value;
+
+  let Uses = [VTYPE, VL];
+}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index f1f4e6573df85..21bc508cdc9ca 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -133,5 +133,24 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
 protected:
   const RISCVSubtarget &STI;
 };
-}
+
+namespace RISCV {
+// Match with the definitions in RISCVInstrFormatsV.td
+enum RVVConstraintType {
+  NoConstraint = 0,
+  WidenV = 1,
+  WidenW = 2,
+  WidenCvt = 3,
+  Narrow = 4,
+  Iota = 5,
+  SlideUp = 6,
+  Vrgather = 7,
+  Vcompress = 8,
+
+  ConstraintOffset = 5,
+  ConstraintMask = 0b1111
+};
+} // end namespace RISCV
+
+} // end namespace llvm
 #endif
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index 1f75dd184cf05..b9483062ddeb1 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -1177,3 +1177,4 @@ include "RISCVInstrInfoF.td"
 include "RISCVInstrInfoD.td"
 include "RISCVInstrInfoC.td"
 include "RISCVInstrInfoB.td"
+include "RISCVInstrInfoV.td"
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
new file mode 100644
index 0000000000000..1c7f53fecb8c9
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
@@ -0,0 +1,873 @@
+//===-- RISCVInstrInfoV.td - RISC-V 'V' instructions -------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// This file describes the RISC-V instructions from the standard 'V' Vector
+/// extension, version 0.8.
+/// This version is still experimental as the 'V' extension hasn't been
+/// ratified yet.
+///
+//===----------------------------------------------------------------------===//
+
+include "RISCVInstrFormatsV.td"
+
+//===----------------------------------------------------------------------===//
+// Operand and SDNode transformation definitions.
+//===----------------------------------------------------------------------===//
+
+def VTypeIAsmOperand : AsmOperandClass {
+  let Name = "VTypeI";
+  let ParserMethod = "parseVTypeI";
+  let DiagnosticType = "InvalidVTypeI";
+}
+
+def VTypeIOp : Operand<XLenVT> {
+  let ParserMatchClass = VTypeIAsmOperand;
+  let PrintMethod = "printVTypeI";
+  let DecoderMethod = "decodeUImmOperand<11>";
+}
+
+def VRegAsmOperand : AsmOperandClass {
+  let Name = "RVVRegOpOperand";
+  let RenderMethod = "addRegOperands";
+  let PredicateMethod = "isReg";
+  let ParserMethod = "parseRegister";
+}
+
+def VRegOp : RegisterOperand<VR> {
+  let ParserMatchClass = VRegAsmOperand;
+  let PrintMethod = "printOperand";
+}
+
+def VMaskAsmOperand : AsmOperandClass {
+  let Name = "RVVMaskRegOpOperand";
+  let RenderMethod = "addRegOperands";
+  let PredicateMethod = "isV0Reg";
+  let ParserMethod = "parseMaskReg";
+  let IsOptional = 1;
+  let DefaultMethod = "defaultMaskRegOp";
+  let DiagnosticType = "InvalidVMaskRegister";
+}
+
+def VMaskOp : RegisterOperand<VMV0> {
+  let ParserMatchClass = VMaskAsmOperand;
+  let PrintMethod = "printVMaskReg";
+  let EncoderMethod = "getVMaskReg";
+  let DecoderMethod = "decodeVMaskReg";
+}
+
+def simm5 : Operand<XLenVT>, ImmLeaf<XLenVT, [{return isInt<5>(Imm);}]> {
+  let ParserMatchClass = SImmAsmOperand<5>;
+  let EncoderMethod = "getImmOpValue";
+  let DecoderMethod = "decodeSImmOperand<5>";
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isInt<5>(Imm);
+    return MCOp.isBareSymbolRef();
+  }];
+}
+
+def SImm5Plus1AsmOperand : AsmOperandClass {
+  let Name = "SImm5Plus1";
+  let RenderMethod = "addSImm5Plus1Operands";
+  let DiagnosticType = "InvalidSImm5Plus1";
+}
+
+def simm5_plus1 : Operand<XLenVT>, ImmLeaf<XLenVT,
+                                           [{return isInt<5>(Imm - 1);}]> {
+  let ParserMatchClass = SImm5Plus1AsmOperand;
+  let PrintMethod = "printSImm5Plus1";
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isInt<5>(Imm - 1);
+    return MCOp.isBareSymbolRef();
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// Instruction class templates
+//===----------------------------------------------------------------------===//
+
+let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in {
+// load vd, (rs1), vm
+class VUnitStrideLoad<RISCVMOP mop, RISCVLSUMOP lumop, RISCVWidth width,
+                        string opcodestr>
+    : RVInstVLU<0b000, mop, lumop, width, (outs VRegOp:$vd),
+                (ins GPR:$rs1, VMaskOp:$vm), opcodestr, "$vd, (${rs1})$vm">;
+
+// load vd, (rs1), rs2, vm
+class VStridedLoad<RISCVMOP mop, RISCVWidth width, string opcodestr>
+    : RVInstVLS<0b000, mop, width, (outs VRegOp:$vd),
+                (ins GPR:$rs1, GPR:$rs2, VMaskOp:$vm), opcodestr,
+                "$vd, (${rs1}), $rs2$vm">;
+
+// load vd, (rs1), vs2, vm
+class VIndexedLoad<RISCVMOP mop, RISCVWidth width, string opcodestr>
+    : RVInstVLX<0b000, mop, width, (outs VRegOp:$vd),
+                (ins GPR:$rs1, VRegOp:$vs2, VMaskOp:$vm), opcodestr,
+                "$vd, (${rs1}), $vs2$vm">;
+
+// vl<nf>r.v vd, (rs1)
+class VWholeLoad<bits<3> nf, string opcodestr>
+    : RVInstVLU<nf, MOPLDUnitStrideU, LUMOPUnitStrideWholeReg,
+                LSWidthVSEW, (outs VRegOp:$vd), (ins GPR:$rs1),
+                opcodestr, "$vd, (${rs1})"> {
+  let vm = 1;
+  let Uses = [];
+}
+} // hasSideEffects = 0, mayLoad = 1, mayStore = 0
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 1 in {
+// store vd, vs3, (rs1), vm
+class VUnitStrideStore<RISCVMOP mop, RISCVLSUMOP sumop, RISCVWidth width,
+                         string opcodestr>
+    : RVInstVSU<0b000, mop, sumop, width, (outs),
+                (ins VRegOp:$vs3, GPR:$rs1, VMaskOp:$vm), opcodestr,
+                "$vs3, (${rs1})$vm">;
+
+// store vd, vs3, (rs1), rs2, vm
+class VStridedStore<RISCVMOP mop, RISCVWidth width, string opcodestr>
+    : RVInstVSS<0b000, mop, width, (outs),
+                (ins VRegOp:$vs3, GPR:$rs1, GPR:$rs2, VMaskOp:$vm),
+                opcodestr, "$vs3, (${rs1}), $rs2$vm">;
+
+// store vd, vs3, (rs1), vs2, vm
+class VIndexedStore<RISCVMOP mop, RISCVWidth width, string opcodestr>
+    : RVInstVSX<0b000, mop, width, (outs),
+                (ins VRegOp:$vs3, GPR:$rs1, VRegOp:$vs2, VMaskOp:$vm),
+                opcodestr, "$vs3, (${rs1}), $vs2$vm">;
+
+// vs<nf>r.v vd, (rs1)
+class VWholeStore<bits<3> nf, string opcodestr>
+    : RVInstVSU<nf, MOPSTUnitStride, SUMOPUnitStrideWholeReg,
+                LSWidthVSEW, (outs), (ins VRegOp:$vs3, GPR:$rs1),
+                opcodestr, "$vs3, (${rs1})"> {
+  let vm = 1;
+  let Uses = [];
+}
+} // hasSideEffects = 0, mayLoad = 0, mayStore = 1
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+// op vd, vs2, vs1, vm
+class VALUVV<bits<6> funct6, RISCVVFormat opv, string opcodestr>
+    : RVInstVV<funct6, opv, (outs VRegOp:$vd),
+                (ins VRegOp:$vs2, VRegOp:$vs1, VMaskOp:$vm),
+                opcodestr, "$vd, $vs2, $vs1$vm">;
+
+// op vd, vs2, vs1, v0 (without mask, use v0 as carry input)
+class VALUmVV<bits<6> funct6, RISCVVFormat opv, string opcodestr>
+    : RVInstVV<funct6, opv, (outs VRegOp:$vd),
+                (ins VRegOp:$vs2, VRegOp:$vs1, VMV0:$v0),
+                opcodestr, "$vd, $vs2, $vs1, v0"> {
+  let vm = 0;
+}
+
+// op vd, vs1, vs2, vm (reverse the order of vs1 and vs2)
+class VALUrVV<bits<6> funct6, RISCVVFormat opv, string opcodestr>
+    : RVInstVV<funct6, opv, (outs VRegOp:$vd),
+                (ins VRegOp:$vs1, VRegOp:$vs2, VMaskOp:$vm),
+                opcodestr, "$vd, $vs1, $vs2$vm">;
+
+// op vd, vs1, vs2
+class VALUVVNoVm<bits<6> funct6, RISCVVFormat opv, string opcodestr>
+    : RVInstVV<funct6, opv, (outs VRegOp:$vd),
+               (ins VRegOp:$vs2, VRegOp:$vs1),
+               opcodestr, "$vd, $vs2, $vs1"> {
+  let vm = 1;
+}
+
+// op vd, vs2, rs1, vm
+class VALUVX<bits<6> funct6, RISCVVFormat opv, string opcodestr>
+    : RVInstVX<funct6, opv, (outs VRegOp:$vd),
+                (ins VRegOp:$vs2, GPR:$rs1, VMaskOp:$vm),
+                opcodestr, "$vd, $vs2, $rs1$vm">;
+
+// op vd, vs2, rs1, v0 (without mask, use v0 as carry input)
+class VALUmVX<bits<6> funct6, RISCVVFormat opv, string opcodestr>
+    : RVInstVX<funct6, opv, (outs VRegOp:$vd),
+                (ins VRegOp:$vs2, GPR:$rs1, VMV0:$v0),
+                opcodestr, "$vd, $vs2, $rs1, v0"> {
+  let vm = 0;
+}
+
+// op vd, rs1, vs2, vm (reverse the order of rs1 and vs2)
+class VALUrVX<bits<6> funct6, RISCVVFormat opv, string opcodestr>
+    : RVInstVX<funct6, opv, (outs VRegOp:$vd),
+                (ins GPR:$rs1, VRegOp:$vs2, VMaskOp:$vm),
+                opcodestr, "$vd, $rs1, $vs2$vm">;
+
+// op vd, vs1, vs2
+class VALUVXNoVm<bits<6> funct6, RISCVVFormat opv, string opcodestr>
+    : RVInstVX<funct6, opv, (outs VRegOp:$vd),
+               (ins VRegOp:$vs2, GPR:$rs1),
+               opcodestr, "$vd, $vs2, $rs1"> {
+  let vm = 1;
+}
+
+// op vd, vs2, imm, vm
+class VALUVI<bits<6> funct6, string opcodestr, Operand optype = simm5>
+    : RVInstIVI<funct6, (outs VRegOp:$vd),
+                (ins VRegOp:$vs2, optype:$imm, VMaskOp:$vm),
+                opcodestr, "$vd, $vs2, $imm$vm">;
+
+// op vd, vs2, imm, v0 (without mask, use v0 as carry input)
+class VALUmVI<bits<6> funct6, string opcodestr, Operand optype = simm5>
+    : RVInstIVI<funct6, (outs VRegOp:$vd),
+                (ins VRegOp:$vs2, optype:$imm, VMV0:$v0),
+                opcodestr, "$vd, $vs2, $imm, v0"> {
+  let vm = 0;
+}
+
+// op vd, vs2, imm, vm
+class VALUVINoVm<bits<6> funct6, string opcodestr, Operand optype = simm5>
+    : RVInstIVI<funct6, (outs VRegOp:$vd),
+                (ins VRegOp:$vs2, optype:$imm),
+                opcodestr, "$vd, $vs2, $imm"> {
+  let vm = 1;
+}
+
+// op vd, vs2, rs1, vm (Float)
+class VALUVF<bits<6> funct6, RISCVVFormat opv, string opcodestr>
+    : RVInstVX<funct6, opv, (outs VRegOp:$vd),
+                (ins VRegOp:$vs2, FPR32:$rs1, VMaskOp:$vm),
+                opcodestr, "$vd, $vs2, $rs1$vm">;
+
+// op vd, rs1, vs2, vm (Float) (with mask, reverse the order of rs1 and vs2)
+class VALUrVF<bits<6> funct6, RISCVVFormat opv, string opcodestr>
+    : RVInstVX<funct6, opv, (outs VRegOp:$vd),
+                (ins FPR32:$rs1, VRegOp:$vs2, VMaskOp:$vm),
+                opcodestr, "$vd, $rs1, $vs2$vm">;
+
+// op vd, vs2, vm (use vs1 as instruction encoding)
+class VALUVs2<bits<6> funct6, bits<5> vs1, RISCVVFormat opv, string opcodestr>
+    : RVInstV<funct6, vs1, opv, (outs VRegOp:$vd),
+               (ins VRegOp:$vs2, VMaskOp:$vm),
+               opcodestr, "$vd, $vs2$vm">;
+} // hasSideEffects = 0, mayLoad = 0, mayStore = 0
+
+//===----------------------------------------------------------------------===//
+// Combination of instruction classes.
+// Use these multiclasses to define instructions more easily.
+//===----------------------------------------------------------------------===//
+multiclass VALU_IV_V_X_I<string opcodestr, bits<6> funct6, Operand optype = simm5, string vw = "v"> {
+  def V  : VALUVV<funct6, OPIVV, opcodestr # "." # vw # "v">;
+  def X  : VALUVX<funct6, OPIVX, opcodestr # "." # vw # "x">;
+  def I  : VALUVI<funct6, opcodestr # "." # vw # "i", optype>;
+}
+
+multiclass VALU_IV_V_X<string opcodestr, bits<6> funct6, string vw = "v"> {
+  def V  : VALUVV<funct6, OPIVV, opcodestr # "." # vw # "v">;
+  def X  : VALUVX<funct6, OPIVX, opcodestr # "." # vw # "x">;
+}
+
+multiclass VALUr_IV_V_X<string opcodestr, bits<6> funct6, string vw = "v"> {
+  def V : VALUrVV<funct6, OPIVV, opcodestr # "." # vw # "v">;
+  def X : VALUrVX<funct6, OPIVX, opcodestr # "." # vw # "x">;
+}
+
+multiclass VALU_IV_X_I<string opcodestr, bits<6> funct6, Operand optype = simm5, string vw = "v"> {
+  def X  : VALUVX<funct6, OPIVX, opcodestr # "." # vw # "x">;
+  def I  : VALUVI<funct6, opcodestr # "." # vw # "i", optype>;
+}
+
+multiclass VALU_IV_V<string opcodestr, bits<6> funct6> {
+  def _VS  : VALUVV<funct6, OPIVV, opcodestr # ".vs">;
+}
+
+multiclass VALUr_IV_X<string opcodestr, bits<6> funct6, string vw = "v"> {
+  def X : VALUrVX<funct6, OPIVX, opcodestr # "." # vw # "x">;
+}
+
+multiclass VALU_MV_V_X<string opcodestr, bits<6> funct6, string vw = "v"> {
+  def V : VALUVV<funct6, OPMVV, opcodestr # "." # vw # "v">;
+  def X  : VALUVX<funct6, OPMVX, opcodestr # "." # vw # "x">;
+}
+
+multiclass VALU_MV_V<string opcodestr, bits<6> funct6> {
+  def _VS : VALUVV<funct6, OPMVV, opcodestr # ".vs">;
+}
+
+multiclass VALU_MV_Mask<string opcodestr, bits<6> funct6, string vm = "v"> {
+  def M : VALUVVNoVm<funct6, OPMVV, opcodestr # "." # vm # "m">;
+}
+
+multiclass VALU_MV_X<string opcodestr, bits<6> funct6, string vw = "v"> {
+  def X  : VALUVX<funct6, OPMVX, opcodestr # "." # vw # "x">;
+}
+
+multiclass VALUr_MV_V_X<string opcodestr, bits<6> funct6, string vw = "v"> {
+  def V : VALUrVV<funct6, OPMVV, opcodestr # "." # vw # "v">;
+  def X : VALUrVX<funct6, OPMVX, opcodestr # "." # vw # "x">;
+}
+
+multiclass VALUr_MV_X<string opcodestr, bits<6> funct6, string vw = "v"> {
+  def X : VALUrVX<funct6, OPMVX, opcodestr # "." # vw # "x">;
+}
+
+multiclass VALU_MV_VS2<string opcodestr, bits<6> funct6, bits<5> vs1> {
+  def "" : VALUVs2<funct6, vs1, OPMVV, opcodestr>;
+}
+
+multiclass VALUm_IV_V_X_I<string opcodestr, bits<6> funct6> {
+  def VM : VALUmVV<funct6, OPIVV, opcodestr # ".vvm">;
+  def XM : VALUmVX<funct6, OPIVX, opcodestr # ".vxm">;
+  def IM : VALUmVI<funct6, opcodestr # ".vim">;
+}
+
+multiclass VALUm_IV_V_X<string opcodestr, bits<6> funct6> {
+  def VM : VALUmVV<funct6, OPIVV, opcodestr # ".vvm">;
+  def XM : VALUmVX<funct6, OPIVX, opcodestr # ".vxm">;
+}
+
+multiclass VALUNoVm_IV_V_X_I<string opcodestr, bits<6> funct6, Operand optype = simm5> {
+  def V : VALUVVNoVm<funct6, OPIVV, opcodestr # ".vv">;
+  def X : VALUVXNoVm<funct6, OPIVX, opcodestr # ".vx">;
+  def I : VALUVINoVm<funct6, opcodestr # ".vi", optype>;
+}
+
+multiclass VALUNoVm_IV_V_X<string opcodestr, bits<6> funct6> {
+  def V : VALUVVNoVm<funct6, OPIVV, opcodestr # ".vv">;
+  def X : VALUVXNoVm<funct6, OPIVX, opcodestr # ".vx">;
+}
+
+multiclass VALU_FV_V_F<string opcodestr, bits<6> funct6, string vw = "v"> {
+  def V : VALUVV<funct6, OPFVV, opcodestr # "." # vw # "v">;
+  def F : VALUVF<funct6, OPFVF, opcodestr # "." # vw # "f">;
+}
+
+multiclass VALU_FV_F<string opcodestr, bits<6> funct6, string vw = "v"> {
+  def F : VALUVF<funct6, OPFVF, opcodestr # "." # vw # "f">;
+}
+
+multiclass VALUr_FV_V_F<string opcodestr, bits<6> funct6, string vw = "v"> {
+  def V : VALUrVV<funct6, OPFVV, opcodestr # "." # vw # "v">;
+  def F : VALUrVF<funct6, OPFVF, opcodestr # "." # vw # "f">;
+}
+
+multiclass VALU_FV_V<string opcodestr, bits<6> funct6> {
+  def _VS : VALUVV<funct6, OPFVV, opcodestr # ".vs">;
+}
+
+multiclass VALU_FV_VS2<string opcodestr, bits<6> funct6, bits<5> vs1> {
+  def "" : VALUVs2<funct6, vs1, OPFVV, opcodestr>;
+}
+
+//===----------------------------------------------------------------------===//
+// Instructions
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasStdExtV] in {
+let hasSideEffects = 1, mayLoad = 0, mayStore = 0 in {
+def VSETVLI : RVInstSetVLi<(outs GPR:$rd), (ins GPR:$rs1, VTypeIOp:$vtypei),
+                           "vsetvli", "$rd, $rs1, $vtypei">;
+
+def VSETVL : RVInstSetVL<(outs GPR:$rd), (ins GPR:$rs1, GPR:$rs2),
+                         "vsetvl", "$rd, $rs1, $rs2">;
+} // hasSideEffects = 1, mayLoad = 0, mayStore = 0
+
+// Vector Unit-Stride Instructions
+def VLB_V : VUnitStrideLoad<MOPLDUnitStrideS, LUMOPUnitStride, LSWidthVByte, "vlb.v">;
+def VLH_V : VUnitStrideLoad<MOPLDUnitStrideS, LUMOPUnitStride, LSWidthVHalf, "vlh.v">;
+def VLW_V : VUnitStrideLoad<MOPLDUnitStrideS, LUMOPUnitStride, LSWidthVWord, "vlw.v">;
+
+def VLBU_V : VUnitStrideLoad<MOPLDUnitStrideU, LUMOPUnitStride, LSWidthVByte, "vlbu.v">;
+def VLHU_V : VUnitStrideLoad<MOPLDUnitStrideU, LUMOPUnitStride, LSWidthVHalf, "vlhu.v">;
+def VLWU_V : VUnitStrideLoad<MOPLDUnitStrideU, LUMOPUnitStride, LSWidthVWord, "vlwu.v">;
+
+def VLE_V : VUnitStrideLoad<MOPLDUnitStrideU, LUMOPUnitStride, LSWidthVSEW, "vle.v">;
+
+def VLBFF_V : VUnitStrideLoad<MOPLDUnitStrideS, LUMOPUnitStrideFF, LSWidthVByte, "vlbff.v">;
+def VLHFF_V : VUnitStrideLoad<MOPLDUnitStrideS, LUMOPUnitStrideFF, LSWidthVHalf, "vlhff.v">;
+def VLWFF_V : VUnitStrideLoad<MOPLDUnitStrideS, LUMOPUnitStrideFF, LSWidthVWord, "vlwff.v">;
+
+def VLBUFF_V : VUnitStrideLoad<MOPLDUnitStrideU, LUMOPUnitStrideFF, LSWidthVByte, "vlbuff.v">;
+def VLHUFF_V : VUnitStrideLoad<MOPLDUnitStrideU, LUMOPUnitStrideFF, LSWidthVHalf, "vlhuff.v">;
+def VLWUFF_V : VUnitStrideLoad<MOPLDUnitStrideU, LUMOPUnitStrideFF, LSWidthVWord, "vlwuff.v">;
+
+def VLEFF_V : VUnitStrideLoad<MOPLDUnitStrideU, LUMOPUnitStrideFF, LSWidthVSEW, "vleff.v">;
+
+def VSB_V : VUnitStrideStore<MOPSTUnitStride, SUMOPUnitStride, LSWidthVByte, "vsb.v">;
+def VSH_V : VUnitStrideStore<MOPSTUnitStride, SUMOPUnitStride, LSWidthVHalf, "vsh.v">;
+def VSW_V : VUnitStrideStore<MOPSTUnitStride, SUMOPUnitStride, LSWidthVWord, "vsw.v">;
+
+def VSE_V : VUnitStrideStore<MOPSTUnitStride, SUMOPUnitStride, LSWidthVSEW, "vse.v">;
+
+// Vector Strided Instructions
+def VLSB_V : VStridedLoad<MOPLDStridedS, LSWidthVByte, "vlsb.v">;
+def VLSH_V : VStridedLoad<MOPLDStridedS, LSWidthVHalf, "vlsh.v">;
+def VLSW_V : VStridedLoad<MOPLDStridedS, LSWidthVWord, "vlsw.v">;
+
+def VLSBU_V : VStridedLoad<MOPLDStridedU, LSWidthVByte, "vlsbu.v">;
+def VLSHU_V : VStridedLoad<MOPLDStridedU, LSWidthVHalf, "vlshu.v">;
+def VLSWU_V : VStridedLoad<MOPLDStridedU, LSWidthVWord, "vlswu.v">;
+
+def VLSE_V : VStridedLoad<MOPLDStridedU, LSWidthVSEW, "vlse.v">;
+
+def VSSB_V : VStridedStore<MOPSTStrided, LSWidthVByte, "vssb.v">;
+def VSSH_V : VStridedStore<MOPSTStrided, LSWidthVHalf, "vssh.v">;
+def VSSW_V : VStridedStore<MOPSTStrided, LSWidthVWord, "vssw.v">;
+def VSSE_V : VStridedStore<MOPSTStrided, LSWidthVSEW, "vsse.v">;
+
+// Vector Indexed Instructions
+def VLXB_V : VIndexedLoad<MOPLDIndexedS, LSWidthVByte, "vlxb.v">;
+def VLXH_V : VIndexedLoad<MOPLDIndexedS, LSWidthVHalf, "vlxh.v">;
+def VLXW_V : VIndexedLoad<MOPLDIndexedS, LSWidthVWord, "vlxw.v">;
+
+def VLXBU_V : VIndexedLoad<MOPLDIndexedU, LSWidthVByte, "vlxbu.v">;
+def VLXHU_V : VIndexedLoad<MOPLDIndexedU, LSWidthVHalf, "vlxhu.v">;
+def VLXWU_V : VIndexedLoad<MOPLDIndexedU, LSWidthVWord, "vlxwu.v">;
+
+def VLXE_V : VIndexedLoad<MOPLDIndexedU, LSWidthVSEW, "vlxe.v">;
+
+def VSXB_V : VIndexedStore<MOPSTIndexedOrder, LSWidthVByte, "vsxb.v">;
+def VSXH_V : VIndexedStore<MOPSTIndexedOrder, LSWidthVHalf, "vsxh.v">;
+def VSXW_V : VIndexedStore<MOPSTIndexedOrder, LSWidthVWord, "vsxw.v">;
+def VSXE_V : VIndexedStore<MOPSTIndexedOrder, LSWidthVSEW, "vsxe.v">;
+
+def VSUXB_V : VIndexedStore<MOPSTIndexedUnOrd, LSWidthVByte, "vsuxb.v">;
+def VSUXH_V : VIndexedStore<MOPSTIndexedUnOrd, LSWidthVHalf, "vsuxh.v">;
+def VSUXW_V : VIndexedStore<MOPSTIndexedUnOrd, LSWidthVWord, "vsuxw.v">;
+def VSUXE_V : VIndexedStore<MOPSTIndexedUnOrd, LSWidthVSEW, "vsuxe.v">;
+
+def VL1R_V : VWholeLoad<0, "vl1r.v">;
+def VS1R_V : VWholeStore<0, "vs1r.v">;
+
+// Vector Single-Width Integer Add and Subtract
+defm VADD_V : VALU_IV_V_X_I<"vadd", 0b000000>;
+defm VSUB_V : VALU_IV_V_X<"vsub", 0b000010>;
+defm VRSUB_V : VALU_IV_X_I<"vrsub", 0b000011>;
+
+// Vector Widening Integer Add/Subtract
+// Refer to 11.2 Widening Vector Arithmetic Instructions
+// The destination vector register group cannot overlap a source vector
+// register group of a different element width (including the mask register
+// if masked), otherwise an illegal instruction exception is raised.
+let Constraints = "@earlyclobber $vd" in {
+let RVVConstraint = WidenV in {
+defm VWADDU_V : VALU_MV_V_X<"vwaddu", 0b110000>;
+defm VWSUBU_V : VALU_MV_V_X<"vwsubu", 0b110010>;
+defm VWADD_V : VALU_MV_V_X<"vwadd", 0b110001>;
+defm VWSUB_V : VALU_MV_V_X<"vwsub", 0b110011>;
+} // RVVConstraint = WidenV
+// Set earlyclobber for following instructions for second and mask operands.
+// This has the downside that the earlyclobber constraint is too coarse and
+// will impose unnecessary restrictions by not allowing the destination to
+// overlap with the first (wide) operand.
+let RVVConstraint = WidenW in {
+defm VWADDU_W : VALU_MV_V_X<"vwaddu", 0b110100, "w">;
+defm VWSUBU_W : VALU_MV_V_X<"vwsubu", 0b110110, "w">;
+defm VWADD_W : VALU_MV_V_X<"vwadd", 0b110101, "w">;
+defm VWSUB_W : VALU_MV_V_X<"vwsub", 0b110111, "w">;
+} // RVVConstraint = WidenW
+} // Constraints = "@earlyclobber $vd"
+
+def : InstAlias<"vwcvt.x.x.v $vd, $vs$vm",
+                (VWADD_VX VRegOp:$vd, VRegOp:$vs, X0, VMaskOp:$vm)>;
+def : InstAlias<"vwcvtu.x.x.v $vd, $vs$vm",
+                (VWADDU_VX VRegOp:$vd, VRegOp:$vs, X0, VMaskOp:$vm)>;
+
+// Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
+defm VADC_V : VALUm_IV_V_X_I<"vadc", 0b010000>;
+defm VMADC_V : VALUm_IV_V_X_I<"vmadc", 0b010001>;
+defm VMADC_V : VALUNoVm_IV_V_X_I<"vmadc", 0b010001>;
+defm VSBC_V : VALUm_IV_V_X<"vsbc", 0b010010>;
+defm VMSBC_V : VALUm_IV_V_X<"vmsbc", 0b010011>;
+defm VMSBC_V : VALUNoVm_IV_V_X<"vmsbc", 0b010011>;
+
+// Vector Bitwise Logical Instructions
+defm VAND_V : VALU_IV_V_X_I<"vand", 0b001001>;
+defm VOR_V : VALU_IV_V_X_I<"vor", 0b001010>;
+defm VXOR_V : VALU_IV_V_X_I<"vxor", 0b001011>;
+
+def : InstAlias<"vnot.v $vd, $vs$vm",
+                (VXOR_VI VRegOp:$vd, VRegOp:$vs, -1, VMaskOp:$vm)>;
+
+// Vector Single-Width Bit Shift Instructions
+defm VSLL_V : VALU_IV_V_X_I<"vsll", 0b100101, uimm5>;
+defm VSRL_V : VALU_IV_V_X_I<"vsrl", 0b101000, uimm5>;
+defm VSRA_V : VALU_IV_V_X_I<"vsra", 0b101001, uimm5>;
+
+// Vector Narrowing Integer Right Shift Instructions
+// Refer to 11.3. Narrowing Vector Arithmetic Instructions
+// The destination vector register group cannot overlap the first source
+// vector register group (specified by vs2). The destination vector register
+// group cannot overlap the mask register if used, unless LMUL=1.
+let Constraints = "@earlyclobber $vd", RVVConstraint = Narrow in {
+defm VNSRL_W : VALU_IV_V_X_I<"vnsrl", 0b101100, uimm5, "w">;
+defm VNSRA_W : VALU_IV_V_X_I<"vnsra", 0b101101, uimm5, "w">;
+} // Constraints = "@earlyclobber $vd", RVVConstraint = Narrow
+
+// Vector Integer Comparison Instructions
+defm VMSEQ_V : VALU_IV_V_X_I<"vmseq", 0b011000>;
+defm VMSNE_V : VALU_IV_V_X_I<"vmsne", 0b011001>;
+defm VMSLTU_V : VALU_IV_V_X<"vmsltu", 0b011010>;
+defm VMSLT_V : VALU_IV_V_X<"vmslt", 0b011011>;
+defm VMSLEU_V : VALU_IV_V_X_I<"vmsleu", 0b011100>;
+defm VMSLE_V : VALU_IV_V_X_I<"vmsle", 0b011101>;
+defm VMSGTU_V : VALU_IV_X_I<"vmsgtu", 0b011110>;
+defm VMSGT_V : VALU_IV_X_I<"vmsgt", 0b011111>;
+
+def : InstAlias<"vmsgtu.vv $vd, $va, $vb$vm",
+                (VMSLTU_VV VRegOp:$vd, VRegOp:$vb, VRegOp:$va, VMaskOp:$vm), 0>;
+def : InstAlias<"vmsgt.vv $vd, $va, $vb$vm",
+                (VMSLT_VV VRegOp:$vd, VRegOp:$vb, VRegOp:$va, VMaskOp:$vm), 0>;
+def : InstAlias<"vmsgeu.vv $vd, $va, $vb$vm",
+                (VMSLEU_VV VRegOp:$vd, VRegOp:$vb, VRegOp:$va, VMaskOp:$vm), 0>;
+def : InstAlias<"vmsge.vv $vd, $va, $vb$vm",
+                (VMSLE_VV VRegOp:$vd, VRegOp:$vb, VRegOp:$va, VMaskOp:$vm), 0>;
+def : InstAlias<"vmsltu.vi $vd, $va, $imm$vm",
+                (VMSLEU_VI VRegOp:$vd, VRegOp:$va, simm5_plus1:$imm,
+                 VMaskOp:$vm), 0>;
+def : InstAlias<"vmslt.vi $vd, $va, $imm$vm",
+                (VMSLE_VI VRegOp:$vd, VRegOp:$va, simm5_plus1:$imm,
+                 VMaskOp:$vm), 0>;
+def : InstAlias<"vmsgeu.vi $vd, $va, $imm$vm",
+                (VMSGTU_VI VRegOp:$vd, VRegOp:$va, simm5_plus1:$imm,
+                 VMaskOp:$vm), 0>;
+def : InstAlias<"vmsge.vi $vd, $va, $imm$vm",
+                (VMSGT_VI VRegOp:$vd, VRegOp:$va, simm5_plus1:$imm,
+                 VMaskOp:$vm), 0>;
+
+// Vector Integer Min/Max Instructions
+defm VMINU_V : VALU_IV_V_X<"vminu", 0b000100>;
+defm VMIN_V : VALU_IV_V_X<"vmin", 0b000101>;
+defm VMAXU_V : VALU_IV_V_X<"vmaxu", 0b000110>;
+defm VMAX_V : VALU_IV_V_X<"vmax", 0b000111>;
+
+// Vector Single-Width Integer Multiply Instructions
+defm VMUL_V : VALU_MV_V_X<"vmul", 0b100101>;
+defm VMULH_V : VALU_MV_V_X<"vmulh", 0b100111>;
+defm VMULHU_V : VALU_MV_V_X<"vmulhu", 0b100100>;
+defm VMULHSU_V : VALU_MV_V_X<"vmulhsu", 0b100110>;
+
+// Vector Integer Divide Instructions
+defm VDIVU_V : VALU_MV_V_X<"vdivu", 0b100000>;
+defm VDIV_V : VALU_MV_V_X<"vdiv", 0b100001>;
+defm VREMU_V : VALU_MV_V_X<"vremu", 0b100010>;
+defm VREM_V : VALU_MV_V_X<"vrem", 0b100011>;
+
+// Vector Widening Integer Multiply Instructions
+let Constraints = "@earlyclobber $vd", RVVConstraint = WidenV in {
+defm VWMUL_V : VALU_MV_V_X<"vwmul", 0b111011>;
+defm VWMULU_V : VALU_MV_V_X<"vwmulu", 0b111000>;
+defm VWMULSU_V : VALU_MV_V_X<"vwmulsu", 0b111010>;
+} // Constraints = "@earlyclobber $vd", RVVConstraint = WidenV
+
+// Vector Single-Width Integer Multiply-Add Instructions
+defm VMACC_V : VALUr_MV_V_X<"vmacc", 0b101101>;
+defm VNMSAC_V : VALUr_MV_V_X<"vnmsac", 0b101111>;
+defm VMADD_V : VALUr_MV_V_X<"vmadd", 0b101001>;
+defm VNMSUB_V : VALUr_MV_V_X<"vnmsub", 0b101011>;
+
+// Vector Widening Integer Multiply-Add Instructions
+let Constraints = "@earlyclobber $vd", RVVConstraint = WidenV in {
+defm VWMACCU_V : VALUr_MV_V_X<"vwmaccu", 0b111100>;
+defm VWMACC_V : VALUr_MV_V_X<"vwmacc", 0b111101>;
+defm VWMACCSU_V : VALUr_MV_V_X<"vwmaccsu", 0b111111>;
+defm VWMACCUS_V : VALUr_MV_X<"vwmaccus", 0b111110>;
+} // Constraints = "@earlyclobber $vd", RVVConstraint = WidenV
+
+// Vector Integer Merge Instructions
+defm VMERGE_V : VALUm_IV_V_X_I<"vmerge", 0b010111>;
+
+// Vector Integer Move Instructions
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0, vs2 = 0, vm = 1 in {
+// op vd, vs1
+def VMV_V_V : RVInstVV<0b010111, OPIVV, (outs VRegOp:$vd),
+                       (ins VRegOp:$vs1), "vmv.v.v", "$vd, $vs1">;
+// op vd, rs1
+def VMV_V_X : RVInstVX<0b010111, OPIVX, (outs VRegOp:$vd),
+                       (ins GPR:$rs1), "vmv.v.x", "$vd, $rs1">;
+// op vd, imm
+def VMV_V_I : RVInstIVI<0b010111, (outs VRegOp:$vd),
+                       (ins simm5:$imm), "vmv.v.i", "$vd, $imm">;
+} // hasSideEffects = 0, mayLoad = 0, mayStore = 0
+
+// Vector Fixed-Point Arithmetic Instructions
+defm VSADDU_V : VALU_IV_V_X_I<"vsaddu", 0b100000>;
+defm VSADD_V : VALU_IV_V_X_I<"vsadd", 0b100001>;
+defm VSSUBU_V : VALU_IV_V_X<"vssubu", 0b100010>;
+defm VSSUB_V : VALU_IV_V_X<"vssub", 0b100011>;
+
+// Vector Single-Width Averaging Add and Subtract
+defm VAADDU_V : VALU_MV_V_X<"vaaddu", 0b001000>;
+defm VAADD_V : VALU_MV_V_X<"vaadd", 0b001001>;
+defm VASUBU_V : VALU_MV_V_X<"vasubu", 0b001010>;
+defm VASUB_V : VALU_MV_V_X<"vasub", 0b001011>;
+
+// Vector Single-Width Fractional Multiply with Rounding and Saturation
+defm VSMUL_V : VALU_IV_V_X<"vsmul", 0b100111>;
+
+// Vector Single-Width Scaling Shift Instructions
+defm VSSRL_V : VALU_IV_V_X_I<"vssrl", 0b101010, uimm5>;
+defm VSSRA_V : VALU_IV_V_X_I<"vssra", 0b101011, uimm5>;
+
+// Vector Narrowing Fixed-Point Clip Instructions
+let Constraints = "@earlyclobber $vd", RVVConstraint = Narrow in {
+defm VNCLIPU_W : VALU_IV_V_X_I<"vnclipu", 0b101110, uimm5, "w">;
+defm VNCLIP_W : VALU_IV_V_X_I<"vnclip", 0b101111, uimm5, "w">;
+} // Constraints = "@earlyclobber $vd", RVVConstraint = Narrow
+
+// Vector Single-Width Floating-Point Add/Subtract Instructions
+defm VFADD_V : VALU_FV_V_F<"vfadd", 0b000000>;
+defm VFSUB_V : VALU_FV_V_F<"vfsub", 0b000010>;
+defm VFRSUB_V : VALU_FV_F<"vfrsub", 0b100111>;
+
+// Vector Widening Floating-Point Add/Subtract Instructions
+let Constraints = "@earlyclobber $vd" in {
+let RVVConstraint = WidenV in {
+defm VFWADD_V : VALU_FV_V_F<"vfwadd", 0b110000>;
+defm VFWSUB_V : VALU_FV_V_F<"vfwsub", 0b110010>;
+} // RVVConstraint = WidenV
+// Set earlyclobber for following instructions for second and mask operands.
+// This has the downside that the earlyclobber constraint is too coarse and
+// will impose unnecessary restrictions by not allowing the destination to
+// overlap with the first (wide) operand.
+let RVVConstraint = WidenW in {
+defm VFWADD_W : VALU_FV_V_F<"vfwadd", 0b110100, "w">;
+defm VFWSUB_W : VALU_FV_V_F<"vfwsub", 0b110110, "w">;
+} // RVVConstraint = WidenW
+} // Constraints = "@earlyclobber $vd"
+
+// Vector Single-Width Floating-Point Multiply/Divide Instructions
+defm VFMUL_V : VALU_FV_V_F<"vfmul", 0b100100>;
+defm VFDIV_V : VALU_FV_V_F<"vfdiv", 0b100000>;
+defm VFRDIV_V : VALU_FV_F<"vfrdiv", 0b100001>;
+
+// Vector Widening Floating-Point Multiply
+let Constraints = "@earlyclobber $vd", RVVConstraint = WidenV in {
+defm VFWMUL_V : VALU_FV_V_F<"vfwmul", 0b111000>;
+} // Constraints = "@earlyclobber $vd", RVVConstraint = WidenV
+
+// Vector Single-Width Floating-Point Fused Multiply-Add Instructions
+defm VFMACC_V : VALUr_FV_V_F<"vfmacc", 0b101100>;
+defm VFNMACC_V : VALUr_FV_V_F<"vfnmacc", 0b101101>;
+defm VFMSAC_V : VALUr_FV_V_F<"vfmsac", 0b101110>;
+defm VFNMSAC_V : VALUr_FV_V_F<"vfnmsac", 0b101111>;
+defm VFMADD_V : VALUr_FV_V_F<"vfmadd", 0b101000>;
+defm VFNMADD_V : VALUr_FV_V_F<"vfnmadd", 0b101001>;
+defm VFMSUB_V : VALUr_FV_V_F<"vfmsub", 0b101010>;
+defm VFNMSUB_V : VALUr_FV_V_F<"vfnmsub", 0b101011>;
+
+// Vector Widening Floating-Point Fused Multiply-Add Instructions
+let Constraints = "@earlyclobber $vd", RVVConstraint = WidenV in {
+defm VFWMACC_V : VALUr_FV_V_F<"vfwmacc", 0b111100>;
+defm VFWNMACC_V : VALUr_FV_V_F<"vfwnmacc", 0b111101>;
+defm VFWMSAC_V : VALUr_FV_V_F<"vfwmsac", 0b111110>;
+defm VFWNMSAC_V : VALUr_FV_V_F<"vfwnmsac", 0b111111>;
+} // Constraints = "@earlyclobber $vd", RVVConstraint = WidenV
+
+// Vector Floating-Point Square-Root Instruction
+defm VFSQRT_V : VALU_FV_VS2<"vfsqrt.v", 0b100011, 0b00000>;
+
+// Vector Floating-Point MIN/MAX Instructions
+defm VFMIN_V : VALU_FV_V_F<"vfmin", 0b000100>;
+defm VFMAX_V : VALU_FV_V_F<"vfmax", 0b000110>;
+
+// Vector Floating-Point Sign-Injection Instructions
+defm VFSGNJ_V : VALU_FV_V_F<"vfsgnj", 0b001000>;
+defm VFSGNJN_V : VALU_FV_V_F<"vfsgnjn", 0b001001>;
+defm VFSGNJX_V : VALU_FV_V_F<"vfsgnjx", 0b001010>;
+
+// Vector Floating-Point Compare Instructions
+defm VMFEQ_V : VALU_FV_V_F<"vmfeq", 0b011000>;
+defm VMFNE_V : VALU_FV_V_F<"vmfne", 0b011100>;
+defm VMFLT_V : VALU_FV_V_F<"vmflt", 0b011011>;
+defm VMFLE_V : VALU_FV_V_F<"vmfle", 0b011001>;
+defm VMFGT_V : VALU_FV_F<"vmfgt", 0b011101>;
+defm VMFGE_V : VALU_FV_F<"vmfge", 0b011111>;
+
+def : InstAlias<"vmfgt.vv $vd, $va, $vb$vm",
+                (VMFLT_VV VRegOp:$vd, VRegOp:$vb, VRegOp:$va, VMaskOp:$vm), 0>;
+def : InstAlias<"vmfge.vv $vd, $va, $vb$vm",
+                (VMFLE_VV VRegOp:$vd, VRegOp:$vb, VRegOp:$va, VMaskOp:$vm), 0>;
+
+// Vector Floating-Point Classify Instruction
+defm VFCLASS_V : VALU_FV_VS2<"vfclass.v", 0b100011, 0b10000>;
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+// Vector Floating-Point Merge Instruction
+def VFMERGE_VFM : RVInstVX<0b010111, OPFVF, (outs VRegOp:$vd),
+                           (ins VRegOp:$vs2, FPR32:$rs1, VMV0:$v0),
+                           "vfmerge.vfm", "$vd, $vs2, $rs1, v0"> {
+  let vm = 0;
+}
+
+// Vector Floating-Point Move Instruction
+def VFMV_V_F : RVInstVX<0b010111, OPFVF, (outs VRegOp:$vd),
+                       (ins FPR32:$rs1), "vfmv.v.f", "$vd, $rs1"> {
+  let vs2 = 0;
+  let vm = 1;
+}
+} // hasSideEffects = 0, mayLoad = 0, mayStore = 0
+
+// Single-Width Floating-Point/Integer Type-Convert Instructions
+defm VFCVT_XU_F_V : VALU_FV_VS2<"vfcvt.xu.f.v", 0b100010, 0b00000>;
+defm VFCVT_X_F_V : VALU_FV_VS2<"vfcvt.x.f.v", 0b100010, 0b00001>;
+defm VFCVT_F_XU_V : VALU_FV_VS2<"vfcvt.f.xu.v", 0b100010, 0b00010>;
+defm VFCVT_F_X_V : VALU_FV_VS2<"vfcvt.f.x.v", 0b100010, 0b00011>;
+
+// Widening Floating-Point/Integer Type-Convert Instructions
+let Constraints = "@earlyclobber $vd", RVVConstraint = WidenCvt in {
+defm VFWCVT_XU_F_V : VALU_FV_VS2<"vfwcvt.xu.f.v", 0b100010, 0b01000>;
+defm VFWCVT_X_F_V : VALU_FV_VS2<"vfwcvt.x.f.v", 0b100010, 0b01001>;
+defm VFWCVT_F_XU_V : VALU_FV_VS2<"vfwcvt.f.xu.v", 0b100010, 0b01010>;
+defm VFWCVT_F_X_V : VALU_FV_VS2<"vfwcvt.f.x.v", 0b100010, 0b01011>;
+defm VFWCVT_F_F_V : VALU_FV_VS2<"vfwcvt.f.f.v", 0b100010, 0b01100>;
+} // Constraints = "@earlyclobber $vd", RVVConstraint = WidenCvt
+
+// Narrowing Floating-Point/Integer Type-Convert Instructions
+let Constraints = "@earlyclobber $vd", RVVConstraint = Narrow in {
+defm VFNCVT_XU_F_W : VALU_FV_VS2<"vfncvt.xu.f.w", 0b100010, 0b10000>;
+defm VFNCVT_X_F_W : VALU_FV_VS2<"vfncvt.x.f.w", 0b100010, 0b10001>;
+defm VFNCVT_F_XU_W : VALU_FV_VS2<"vfncvt.f.xu.w", 0b100010, 0b10010>;
+defm VFNCVT_F_X_W : VALU_FV_VS2<"vfncvt.f.x.w", 0b100010, 0b10011>;
+defm VFNCVT_F_F_W : VALU_FV_VS2<"vfncvt.f.f.w", 0b100010, 0b10100>;
+defm VFNCVT_ROD_F_F_W : VALU_FV_VS2<"vfncvt.rod.f.f.w", 0b100010, 0b10101>;
+} // Constraints = "@earlyclobber $vd", RVVConstraint = Narrow
+
+// Vector Single-Width Integer Reduction Instructions
+defm VREDSUM : VALU_MV_V<"vredsum", 0b000000>;
+defm VREDMAXU : VALU_MV_V<"vredmaxu", 0b000110>;
+defm VREDMAX : VALU_MV_V<"vredmax", 0b000111>;
+defm VREDMINU : VALU_MV_V<"vredminu", 0b000100>;
+defm VREDMIN : VALU_MV_V<"vredmin", 0b000101>;
+defm VREDAND : VALU_MV_V<"vredand", 0b000001>;
+defm VREDOR : VALU_MV_V<"vredor", 0b000010>;
+defm VREDXOR : VALU_MV_V<"vredxor", 0b000011>;
+
+// Vector Widening Integer Reduction Instructions
+let Constraints = "@earlyclobber $vd" in {
+// Set earlyclobber for following instructions for second and mask operands.
+// This has the downside that the earlyclobber constraint is too coarse and
+// will impose unnecessary restrictions by not allowing the destination to
+// overlap with the first (wide) operand.
+defm VWREDSUMU : VALU_IV_V<"vwredsumu", 0b110000>;
+defm VWREDSUM : VALU_IV_V<"vwredsum", 0b110001>;
+} // Constraints = "@earlyclobber $vd"
+
+// Vector Single-Width Floating-Point Reduction Instructions
+defm VFREDOSUM : VALU_FV_V<"vfredosum", 0b000011>;
+defm VFREDSUM : VALU_FV_V<"vfredsum", 0b000001>;
+defm VFREDMAX : VALU_FV_V<"vfredmax", 0b000111>;
+defm VFREDMIN : VALU_FV_V<"vfredmin", 0b000101>;
+
+// Vector Widening Floating-Point Reduction Instructions
+let Constraints = "@earlyclobber $vd" in {
+// Set earlyclobber for following instructions for second and mask operands.
+// This has the downside that the earlyclobber constraint is too coarse and
+// will impose unnecessary restrictions by not allowing the destination to
+// overlap with the first (wide) operand.
+defm VFWREDOSUM : VALU_FV_V<"vfwredosum", 0b110011>;
+defm VFWREDSUM : VALU_FV_V<"vfwredsum", 0b110001>;
+} // Constraints = "@earlyclobber $vd"
+
+// Vector Mask-Register Logical Instructions
+defm VMAND_M : VALU_MV_Mask<"vmand", 0b011001, "m">;
+defm VMNAND_M : VALU_MV_Mask<"vmnand", 0b011101, "m">;
+defm VMANDNOT_M : VALU_MV_Mask<"vmandnot", 0b011000, "m">;
+defm VMXOR_M : VALU_MV_Mask<"vmxor", 0b011011, "m">;
+defm VMOR_M : VALU_MV_Mask<"vmor", 0b011010, "m">;
+defm VMNOR_M : VALU_MV_Mask<"vmnor", 0b011110, "m">;
+defm VMORNOT_M : VALU_MV_Mask<"vmornot", 0b011100, "m">;
+defm VMXNOR_M : VALU_MV_Mask<"vmxnor", 0b011111, "m">;
+
+def : InstAlias<"vmcpy.m $vd, $vs",
+                (VMAND_MM VRegOp:$vd, VRegOp:$vs, VRegOp:$vs)>;
+def : InstAlias<"vmclr.m $vd",
+                (VMXOR_MM VRegOp:$vd, VRegOp:$vd, VRegOp:$vd)>;
+def : InstAlias<"vmset.m $vd",
+                (VMXNOR_MM VRegOp:$vd, VRegOp:$vd, VRegOp:$vd)>;
+def : InstAlias<"vmnot.m $vd, $vs",
+                (VMNAND_MM VRegOp:$vd, VRegOp:$vs, VRegOp:$vs)>;
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+// Vector mask population count vpopc
+def VPOPC_M : RVInstV<0b010000, 0b10000, OPMVV, (outs GPR:$vd),
+                        (ins VRegOp:$vs2, VMaskOp:$vm),
+                        "vpopc.m", "$vd, $vs2$vm">;
+
+// vfirst find-first-set mask bit
+def VFIRST_M : RVInstV<0b010000, 0b10001, OPMVV, (outs GPR:$vd),
+                        (ins VRegOp:$vs2, VMaskOp:$vm),
+                        "vfirst.m", "$vd, $vs2$vm">;
+} // hasSideEffects = 0, mayLoad = 0, mayStore = 0
+
+// vmsbf.m set-before-first mask bit
+defm VMSBF_M : VALU_MV_VS2<"vmsbf.m", 0b010100, 0b00001>;
+
+// vmsif.m set-including-first mask bit
+defm VMSIF_M : VALU_MV_VS2<"vmsif.m", 0b010100, 0b00011>;
+
+// vmsof.m set-only-first mask bit
+defm VMSOF_M : VALU_MV_VS2<"vmsof.m", 0b010100, 0b00010>;
+
+// Vector Iota Instruction
+let Constraints = "@earlyclobber $vd", RVVConstraint = Iota in {
+defm VIOTA_M : VALU_MV_VS2<"viota.m", 0b010100, 0b10000>;
+} // Constraints = "@earlyclobber $vd", RVVConstraint = Iota
+
+// Vector Element Index Instruction
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+def VID_V : RVInstV<0b010100, 0b10001, OPMVV, (outs VRegOp:$vd),
+                      (ins VMaskOp:$vm), "vid.v", "$vd$vm"> {
+  let vs2 = 0;
+}
+
+// Integer Scalar Move Instructions
+let vm = 1 in {
+def VMV_X_S : RVInstV<0b010000, 0b00000, OPMVV, (outs GPR:$vd),
+                      (ins VRegOp:$vs2), "vmv.x.s", "$vd, $vs2">;
+def VMV_S_X : RVInstV2<0b010000, 0b00000, OPMVX, (outs VRegOp:$vd),
+                      (ins GPR:$rs1), "vmv.s.x", "$vd, $rs1">;
+
+}
+} // hasSideEffects = 0, mayLoad = 0, mayStore = 0
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0, vm = 1 in {
+// Floating-Point Scalar Move Instructions
+def VFMV_F_S : RVInstV<0b010000, 0b00000, OPFVV, (outs FPR32:$vd),
+                      (ins VRegOp:$vs2), "vfmv.f.s", "$vd, $vs2">;
+def VFMV_S_F : RVInstV2<0b010000, 0b00000, OPFVF, (outs VRegOp:$vd),
+                      (ins FPR32:$rs1), "vfmv.s.f", "$vd, $rs1">;
+
+} // hasSideEffects = 0, mayLoad = 0, mayStore = 0, vm = 1
+
+// Vector Slide Instructions
+let Constraints = "@earlyclobber $vd", RVVConstraint = SlideUp in {
+defm VSLIDEUP_V : VALU_IV_X_I<"vslideup", 0b001110, uimm5>;
+} // Constraints = "@earlyclobber $vd", RVVConstraint = SlideUp
+defm VSLIDEDOWN_V : VALU_IV_X_I<"vslidedown", 0b001111, uimm5>;
+
+let Constraints = "@earlyclobber $vd", RVVConstraint = SlideUp in {
+defm VSLIDE1UP_V : VALU_MV_X<"vslide1up", 0b001110>;
+} // Constraints = "@earlyclobber $vd", RVVConstraint = SlideUp
+defm VSLIDE1DOWN_V : VALU_MV_X<"vslide1down", 0b001111>;
+
+// Vector Register Gather Instruction
+let Constraints = "@earlyclobber $vd", RVVConstraint = Vrgather in {
+defm VRGATHER_V : VALU_IV_V_X_I<"vrgather", 0b001100, uimm5>;
+} // Constraints = "@earlyclobber $vd", RVVConstraint = Vrgather
+
+// Vector Compress Instruction
+let Constraints = "@earlyclobber $vd", RVVConstraint = Vcompress in {
+defm VCOMPRESS_V : VALU_MV_Mask<"vcompress", 0b010111>;
+} // Constraints = "@earlyclobber $vd", RVVConstraint = Vcompress
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+foreach nf = [1, 2, 4, 8] in {
+  def VMV#nf#R_V  : RVInstV<0b100111, !add(nf, -1), OPIVI, (outs VRegOp:$vd),
+                            (ins VRegOp:$vs2), "vmv" # nf # "r.v",
+                            "$vd, $vs2"> {
+    let Uses = [];
+    let vm = 1;
+  }
+}
+} // hasSideEffects = 0, mayLoad = 0, mayStore = 0
+} // Predicates = [HasStdExtV]
diff --git a/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp b/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp
index b1dbcfa7f7387..8ddcf757c97e7 100644
--- a/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp
@@ -121,6 +121,9 @@ bool llvm::LowerRISCVMachineOperandToMCOperand(const MachineOperand &MO,
   case MachineOperand::MO_ConstantPoolIndex:
     MCOp = lowerSymbolOperand(MO, AP.GetCPISymbol(MO.getIndex()), AP);
     break;
+  case MachineOperand::MO_MCSymbol:
+    MCOp = lowerSymbolOperand(MO, MO.getMCSymbol(), AP);
+    break;
   }
   return true;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
index 593dabc958906..c379a8d8f0d6d 100644
--- a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
@@ -23,7 +23,6 @@ namespace llvm {
 /// and contains private RISCV-specific information for each MachineFunction.
 class RISCVMachineFunctionInfo : public MachineFunctionInfo {
 private:
-  MachineFunction &MF;
   /// FrameIndex for start of varargs area
   int VarArgsFrameIndex = 0;
   /// Size of the save area used for varargs
@@ -35,7 +34,7 @@ class RISCVMachineFunctionInfo : public MachineFunctionInfo {
   unsigned LibCallStackSize = 0;
 
 public:
-  RISCVMachineFunctionInfo(MachineFunction &MF) : MF(MF) {}
+  RISCVMachineFunctionInfo(const MachineFunction &MF) {}
 
   int getVarArgsFrameIndex() const { return VarArgsFrameIndex; }
   void setVarArgsFrameIndex(int Index) { VarArgsFrameIndex = Index; }
@@ -43,16 +42,17 @@ class RISCVMachineFunctionInfo : public MachineFunctionInfo {
   unsigned getVarArgsSaveSize() const { return VarArgsSaveSize; }
   void setVarArgsSaveSize(int Size) { VarArgsSaveSize = Size; }
 
-  int getMoveF64FrameIndex() {
+  int getMoveF64FrameIndex(MachineFunction &MF) {
     if (MoveF64FrameIndex == -1)
-      MoveF64FrameIndex = MF.getFrameInfo().CreateStackObject(8, 8, false);
+      MoveF64FrameIndex =
+          MF.getFrameInfo().CreateStackObject(8, Align(8), false);
     return MoveF64FrameIndex;
   }
 
   unsigned getLibCallStackSize() const { return LibCallStackSize; }
   void setLibCallStackSize(unsigned Size) { LibCallStackSize = Size; }
 
-  bool useSaveRestoreLibCalls() const {
+  bool useSaveRestoreLibCalls(const MachineFunction &MF) const {
     // We cannot use fixed locations for the callee saved spill slots if the
     // function uses a varargs save area.
     return MF.getSubtarget<RISCVSubtarget>().enableSaveRestore() &&
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index fd234c45ae9c3..cb7d55eb0f0c3 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -35,6 +35,8 @@ static_assert(RISCV::F31_F == RISCV::F0_F + 31,
 static_assert(RISCV::F1_D == RISCV::F0_D + 1, "Register list not consecutive");
 static_assert(RISCV::F31_D == RISCV::F0_D + 31,
               "Register list not consecutive");
+static_assert(RISCV::V1 == RISCV::V0 + 1, "Register list not consecutive");
+static_assert(RISCV::V31 == RISCV::V0 + 31, "Register list not consecutive");
 
 RISCVRegisterInfo::RISCVRegisterInfo(unsigned HwMode)
     : RISCVGenRegisterInfo(RISCV::X1, /*DwarfFlavour*/0, /*EHFlavor*/0,
@@ -126,7 +128,7 @@ bool RISCVRegisterInfo::hasReservedSpillSlot(const MachineFunction &MF,
                                              Register Reg,
                                              int &FrameIdx) const {
   const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
-  if (!RVFI->useSaveRestoreLibCalls())
+  if (!RVFI->useSaveRestoreLibCalls(MF))
     return false;
 
   auto FII = FixedCSRFIMap.find(Reg);
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
index 82b37afd0805f..7544b4b3b8455 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
@@ -33,7 +33,21 @@ class RISCVReg64<RISCVReg32 subreg> : Register<""> {
   let AltNames = subreg.AltNames;
 }
 
+class RISCVRegWithSubRegs<bits<5> Enc, string n, list<Register> subregs,
+                          list<string> alt = []>
+      : RegisterWithSubRegs<n, subregs> {
+  let HWEncoding{4-0} = Enc;
+  let AltNames = alt;
+}
+
 def ABIRegAltName : RegAltNameIndex;
+
+def sub_vrm2    : SubRegIndex<64, -1>;
+def sub_vrm2_hi : SubRegIndex<64, -1>;
+def sub_vrm4    : SubRegIndex<128, -1>;
+def sub_vrm4_hi : SubRegIndex<128, -1>;
+def sub_vrm8    : SubRegIndex<256, -1>;
+def sub_vrm8_hi : SubRegIndex<256, -1>;
 } // Namespace = "RISCV"
 
 // Integer registers
@@ -233,3 +247,88 @@ def FPR64C : RegisterClass<"RISCV", [f64], 64, (add
   (sequence "F%u_D", 10, 15),
   (sequence "F%u_D", 8, 9)
 )>;
+
+// Vector registers
+let RegAltNameIndices = [ABIRegAltName] in {
+  foreach Index = 0-31 in {
+    def V#Index : RISCVReg<Index, "v"#Index, ["v"#Index]>, DwarfRegNum<[!add(Index, 64)]>;
+  }
+
+  foreach Index = [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22,
+                   24, 26, 28, 30] in {
+    def V#Index#M2 : RISCVRegWithSubRegs<Index, "v"#Index,
+                       [!cast<Register>("V"#Index),
+                        !cast<Register>("V"#!add(Index, 1))],
+                       ["v"#Index]>,
+                     DwarfRegAlias<!cast<Register>("V"#Index)> {
+      let SubRegIndices = [sub_vrm2, sub_vrm2_hi];
+    }
+  }
+
+  foreach Index = [0, 4, 8, 12, 16, 20, 24, 28] in {
+    def V#Index#M4 : RISCVRegWithSubRegs<Index, "v"#Index,
+                       [!cast<Register>("V"#Index#"M2"),
+                        !cast<Register>("V"#!add(Index, 2)#"M2")],
+                       ["v"#Index]>,
+                     DwarfRegAlias<!cast<Register>("V"#Index)> {
+      let SubRegIndices = [sub_vrm4, sub_vrm4_hi];
+    }
+  }
+
+  foreach Index = [0, 8, 16, 24] in {
+    def V#Index#M8 : RISCVRegWithSubRegs<Index, "v"#Index,
+                       [!cast<Register>("V"#Index#"M4"),
+                        !cast<Register>("V"#!add(Index, 4)#"M4")],
+                       ["v"#Index]>,
+                     DwarfRegAlias<!cast<Register>("V"#Index)> {
+      let SubRegIndices = [sub_vrm8, sub_vrm8_hi];
+    }
+  }
+
+  def VTYPE  : RISCVReg<0, "vtype", ["vtype"]>;
+  def VL     : RISCVReg<0, "vl", ["vl"]>;
+}
+
+class RegisterTypes<list<ValueType> reg_types> {
+  list<ValueType> types = reg_types;
+}
+
+// The order of registers represents the preferred allocation sequence,
+// meaning caller-save regs are listed before callee-save.
+def VR : RegisterClass<"RISCV", [nxv8i8, nxv4i16, nxv2i32, nxv1i64],
+                         64, (add
+    (sequence "V%u", 25, 31),
+    (sequence "V%u", 8, 24),
+    (sequence "V%u", 0, 7)
+  )> {
+  let Size = 64;
+}
+
+def VRM2 : RegisterClass<"RISCV", [nxv16i8, nxv8i16, nxv4i32, nxv2i64], 64,
+                         (add V26M2, V28M2, V30M2, V8M2, V10M2, V12M2, V14M2, V16M2,
+                              V18M2, V20M2, V22M2, V24M2, V0M2, V2M2, V4M2, V6M2)> {
+  let Size = 128;
+}
+
+def VRM4 : RegisterClass<"RISCV", [nxv32i8, nxv16i16, nxv8i32, nxv4i64], 64,
+                         (add V28M4, V8M4, V12M4, V16M4, V20M4, V24M4, V0M4, V4M4)> {
+  let Size = 256;
+}
+
+def VRM8 : RegisterClass<"RISCV", [nxv32i16, nxv16i32, nxv8i64], 64,
+                         (add V8M8, V16M8, V24M8, V0M8)> {
+  let Size = 512;
+}
+
+def VMaskVT : RegisterTypes<[nxv1i1, nxv2i1, nxv4i1, nxv8i1, nxv16i1, nxv32i1]>;
+
+def VM : RegisterClass<"RISCV", VMaskVT.types, 64, (add
+    (sequence "V%u", 25, 31),
+    (sequence "V%u", 8, 24),
+    (sequence "V%u", 0, 7))> {
+  let Size = 64;
+}
+
+def VMV0 : RegisterClass<"RISCV", VMaskVT.types, 64, (add V0)> {
+  let Size = 64;
+}
diff --git a/llvm/lib/Target/RISCV/RISCVSchedRocket32.td b/llvm/lib/Target/RISCV/RISCVSchedRocket32.td
index 487f72ebadc8c..305e2b9b5927a 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedRocket32.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedRocket32.td
@@ -17,6 +17,7 @@ def Rocket32Model : SchedMachineModel {
   let LoadLatency = 3;
   let MispredictPenalty = 3;
   let CompleteModel = 1;
+  let UnsupportedFeatures = [HasStdExtV];
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVSchedRocket64.td b/llvm/lib/Target/RISCV/RISCVSchedRocket64.td
index da33443cc94aa..e8514a275c45e 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedRocket64.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedRocket64.td
@@ -16,6 +16,7 @@ def Rocket64Model : SchedMachineModel {
   let IssueWidth = 1;        // 1 micro-ops are dispatched per cycle.
   let LoadLatency = 3;
   let MispredictPenalty = 3;
+  let UnsupportedFeatures = [HasStdExtV];
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h
index 8b339c3e46603..fe1285f23b159 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.h
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h
@@ -50,10 +50,11 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
   bool HasStdExtZbs = false;
   bool HasStdExtZbt = false;
   bool HasStdExtZbproposedc = false;
+  bool HasStdExtV = false;
   bool HasRV64 = false;
   bool IsRV32E = false;
   bool EnableLinkerRelax = false;
-  bool EnableRVCHintInstrs = false;
+  bool EnableRVCHintInstrs = true;
   bool EnableSaveRestore = false;
   unsigned XLen = 32;
   MVT XLenVT = MVT::i32;
@@ -110,6 +111,7 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
   bool hasStdExtZbs() const { return HasStdExtZbs; }
   bool hasStdExtZbt() const { return HasStdExtZbt; }
   bool hasStdExtZbproposedc() const { return HasStdExtZbproposedc; }
+  bool hasStdExtV() const { return HasStdExtV; }
   bool is64Bit() const { return HasRV64; }
   bool isRV32E() const { return IsRV32E; }
   bool enableLinkerRelax() const { return EnableLinkerRelax; }
diff --git a/llvm/lib/Target/RISCV/RISCVSystemOperands.td b/llvm/lib/Target/RISCV/RISCVSystemOperands.td
index 853dca0295c97..8e75647bd4a9e 100644
--- a/llvm/lib/Target/RISCV/RISCVSystemOperands.td
+++ b/llvm/lib/Target/RISCV/RISCVSystemOperands.td
@@ -310,6 +310,7 @@ def: SysReg<"mhpmcounter31h", 0xB9F>;
 //===--------------------------
 // Machine Counter Setup
 //===--------------------------
+def : SysReg<"mcountinhibit", 0x320>;
 def : SysReg<"mhpmevent3", 0x323>;
 def : SysReg<"mhpmevent4", 0x324>;
 def : SysReg<"mhpmevent5", 0x325>;
@@ -353,8 +354,19 @@ def : SysReg<"tdata3", 0x7A3>;
 //===-----------------------------------------------
 def : SysReg<"dcsr", 0x7B0>;
 def : SysReg<"dpc", 0x7B1>;
+
 // "dscratch" is an alternative name for "dscratch0" which appeared in earlier
 // drafts of the RISC-V debug spec
 let AltName = "dscratch" in
 def : SysReg<"dscratch0", 0x7B2>;
 def : SysReg<"dscratch1", 0x7B3>;
+
+//===-----------------------------------------------
+// User Vector CSRs
+//===-----------------------------------------------
+def : SysReg<"vstart", 0x008>;
+def : SysReg<"vxsat", 0x009>;
+def : SysReg<"vxrm", 0x00A>;
+def : SysReg<"vl", 0xC20>;
+def : SysReg<"vtype", 0xC21>;
+def : SysReg<"vlenb", 0xC22>;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index de71c01753dea..63f607a9c352c 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -128,6 +128,7 @@ class RISCVPassConfig : public TargetPassConfig {
   bool addGlobalInstructionSelect() override;
   void addPreEmitPass() override;
   void addPreEmitPass2() override;
+  void addPreSched2() override;
   void addPreRegAlloc() override;
 };
 }
@@ -167,15 +168,18 @@ bool RISCVPassConfig::addGlobalInstructionSelect() {
   return false;
 }
 
+void RISCVPassConfig::addPreSched2() {}
+
 void RISCVPassConfig::addPreEmitPass() { addPass(&BranchRelaxationPassID); }
 
 void RISCVPassConfig::addPreEmitPass2() {
   // Schedule the expansion of AMOs at the last possible moment, avoiding the
   // possibility for other passes to break the requirements for forward
   // progress in the LR/SC block.
-  addPass(createRISCVExpandPseudoPass());
+  addPass(createRISCVExpandAtomicPseudoPass());
 }
 
 void RISCVPassConfig::addPreRegAlloc() {
+  addPass(createRISCVExpandPseudoPass());
   addPass(createRISCVMergeBaseOffsetOptPass());
 }
diff --git a/llvm/lib/Target/RISCV/Utils/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/Utils/RISCVBaseInfo.h
index 2aa1b23d24f00..4e6cdd8606b16 100644
--- a/llvm/lib/Target/RISCV/Utils/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/Utils/RISCVBaseInfo.h
@@ -45,7 +45,7 @@ enum {
   InstFormatCJ = 16,
   InstFormatOther = 17,
 
-  InstFormatMask = 31
+  InstFormatMask = 31,
 };
 
 // RISC-V Specific Machine Operand Flags
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index c9743c5170749..116352e083829 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -59,7 +59,7 @@ static bool CC_Sparc_Assign_Split_64(unsigned &ValNo, MVT &ValVT,
     SP::I0, SP::I1, SP::I2, SP::I3, SP::I4, SP::I5
   };
   // Try to get first reg.
-  if (unsigned Reg = State.AllocateReg(RegList)) {
+  if (Register Reg = State.AllocateReg(RegList)) {
     State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
   } else {
     // Assign whole thing in stack.
@@ -69,7 +69,7 @@ static bool CC_Sparc_Assign_Split_64(unsigned &ValNo, MVT &ValVT,
   }
 
   // Try to get second reg.
-  if (unsigned Reg = State.AllocateReg(RegList))
+  if (Register Reg = State.AllocateReg(RegList))
     State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
   else
     State.addLoc(CCValAssign::getCustomMem(
@@ -86,13 +86,13 @@ static bool CC_Sparc_Assign_Ret_Split_64(unsigned &ValNo, MVT &ValVT,
   };
 
   // Try to get first reg.
-  if (unsigned Reg = State.AllocateReg(RegList))
+  if (Register Reg = State.AllocateReg(RegList))
     State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
   else
     return false;
 
   // Try to get second reg.
-  if (unsigned Reg = State.AllocateReg(RegList))
+  if (Register Reg = State.AllocateReg(RegList))
     State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
   else
     return false;
@@ -264,7 +264,7 @@ SparcTargetLowering::LowerReturn_32(SDValue Chain, CallingConv::ID CallConv,
   // If the function returns a struct, copy the SRetReturnReg to I0
   if (MF.getFunction().hasStructRetAttr()) {
     SparcMachineFunctionInfo *SFI = MF.getInfo<SparcMachineFunctionInfo>();
-    unsigned Reg = SFI->getSRetReturnReg();
+    Register Reg = SFI->getSRetReturnReg();
     if (!Reg)
       llvm_unreachable("sret virtual register not created in the entry block");
     auto PtrVT = getPointerTy(DAG.getDataLayout());
@@ -429,7 +429,7 @@ SDValue SparcTargetLowering::LowerFormalArguments_32(
           SDValue FIPtr = DAG.getFrameIndex(FrameIdx, MVT::i32);
           LoVal = DAG.getLoad(MVT::i32, dl, Chain, FIPtr, MachinePointerInfo());
         } else {
-          unsigned loReg = MF.addLiveIn(NextVA.getLocReg(),
+          Register loReg = MF.addLiveIn(NextVA.getLocReg(),
                                         &SP::IntRegsRegClass);
           LoVal = DAG.getCopyFromReg(Chain, dl, loReg, MVT::i32);
         }
@@ -520,7 +520,7 @@ SDValue SparcTargetLowering::LowerFormalArguments_32(
   if (MF.getFunction().hasStructRetAttr()) {
     // Copy the SRet Argument to SRetReturnReg.
     SparcMachineFunctionInfo *SFI = MF.getInfo<SparcMachineFunctionInfo>();
-    unsigned Reg = SFI->getSRetReturnReg();
+    Register Reg = SFI->getSRetReturnReg();
     if (!Reg) {
       Reg = MF.getRegInfo().createVirtualRegister(&SP::IntRegsRegClass);
       SFI->setSRetReturnReg(Reg);
@@ -595,7 +595,7 @@ SDValue SparcTargetLowering::LowerFormalArguments_64(
       // All integer register arguments are promoted by the caller to i64.
 
       // Create a virtual register for the promoted live-in value.
-      unsigned VReg = MF.addLiveIn(VA.getLocReg(),
+      Register VReg = MF.addLiveIn(VA.getLocReg(),
                                    getRegClassFor(VA.getLocVT()));
       SDValue Arg = DAG.getCopyFromReg(Chain, DL, VReg, VA.getLocVT());
 
@@ -666,7 +666,7 @@ SDValue SparcTargetLowering::LowerFormalArguments_64(
   // of how many arguments were actually passed.
   SmallVector<SDValue, 8> OutChains;
   for (; ArgOffset < 6*8; ArgOffset += 8) {
-    unsigned VReg = MF.addLiveIn(SP::I0 + ArgOffset/8, &SP::I64RegsRegClass);
+    Register VReg = MF.addLiveIn(SP::I0 + ArgOffset/8, &SP::I64RegsRegClass);
     SDValue VArg = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
     int FI = MF.getFrameInfo().CreateFixedObject(8, ArgOffset + ArgArea, true);
     auto PtrVT = getPointerTy(MF.getDataLayout());
@@ -929,7 +929,7 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
   // stuck together.
   SDValue InFlag;
   for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    unsigned Reg = toCallerWindow(RegsToPass[i].first);
+    Register Reg = toCallerWindow(RegsToPass[i].first);
     Chain = DAG.getCopyToReg(Chain, dl, Reg, RegsToPass[i].second, InFlag);
     InFlag = Chain.getValue(1);
   }
@@ -1016,7 +1016,7 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
 // this table could be generated automatically from RegInfo.
 Register SparcTargetLowering::getRegisterByName(const char* RegName, LLT VT,
                                                 const MachineFunction &MF) const {
-  Register Reg = StringSwitch<unsigned>(RegName)
+  Register Reg = StringSwitch<Register>(RegName)
     .Case("i0", SP::I0).Case("i1", SP::I1).Case("i2", SP::I2).Case("i3", SP::I3)
     .Case("i4", SP::I4).Case("i5", SP::I5).Case("i6", SP::I6).Case("i7", SP::I7)
     .Case("o0", SP::O0).Case("o1", SP::O1).Case("o2", SP::O2).Case("o3", SP::O3)
@@ -1058,7 +1058,7 @@ static void fixupVariableFloatArgs(SmallVectorImpl<CCValAssign> &ArgLocs,
     CCValAssign NewVA;
 
     // Determine the offset into the argument array.
-    unsigned firstReg = (ValTy == MVT::f64) ? SP::D0 : SP::Q0;
+    Register firstReg = (ValTy == MVT::f64) ? SP::D0 : SP::Q0;
     unsigned argSize  = (ValTy == MVT::f64) ? 8 : 16;
     unsigned Offset = argSize * (VA.getLocReg() - firstReg);
     assert(Offset < 16*8 && "Offset out of range, bad register enum?");
@@ -1125,7 +1125,7 @@ SparcTargetLowering::LowerCall_64(TargetLowering::CallLoweringInfo &CLI,
   // Collect the set of registers to pass to the function and their values.
   // This will be emitted as a sequence of CopyToReg nodes glued to the call
   // instruction.
-  SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
+  SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
 
   // Collect chains from all the memory opeations that copy arguments to the
   // stack. They must follow the stack pointer adjustment above and precede the
@@ -2136,7 +2136,7 @@ SDValue SparcTargetLowering::LowerF128_LibCallArg(SDValue Chain,
 
   if (ArgTy->isFP128Ty()) {
     // Create a stack object and pass the pointer to the library function.
-    int FI = MFI.CreateStackObject(16, 8, false);
+    int FI = MFI.CreateStackObject(16, Align(8), false);
     SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
     Chain = DAG.getStore(Chain, DL, Entry.Node, FIPtr, MachinePointerInfo(),
                          /* Alignment = */ 8);
@@ -2167,7 +2167,7 @@ SparcTargetLowering::LowerF128Op(SDValue Op, SelectionDAG &DAG,
   if (RetTy->isFP128Ty()) {
     // Create a Stack Object to receive the return value of type f128.
     ArgListEntry Entry;
-    int RetFI = MFI.CreateStackObject(16, 8, false);
+    int RetFI = MFI.CreateStackObject(16, Align(8), false);
     RetPtr = DAG.getFrameIndex(RetFI, PtrVT);
     Entry.Node = RetPtr;
     Entry.Ty   = PointerType::getUnqual(RetTy);
@@ -2549,7 +2549,8 @@ static SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG,
                                        const SparcSubtarget *Subtarget) {
   SDValue Chain = Op.getOperand(0);  // Legalize the chain.
   SDValue Size  = Op.getOperand(1);  // Legalize the size.
-  MaybeAlign Alignment(cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());
+  MaybeAlign Alignment =
+      cast<ConstantSDNode>(Op.getOperand(2))->getMaybeAlignValue();
   Align StackAlign = Subtarget->getFrameLowering()->getStackAlign();
   EVT VT = Size->getValueType(0);
   SDLoc dl(Op);
diff --git a/llvm/lib/Target/Sparc/SparcInstrInfo.cpp b/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
index b31830de517f1..dc3a41c630986 100644
--- a/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
+++ b/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
@@ -468,11 +468,10 @@ loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
     llvm_unreachable("Can't load this register from stack slot");
 }
 
-unsigned SparcInstrInfo::getGlobalBaseReg(MachineFunction *MF) const
-{
+Register SparcInstrInfo::getGlobalBaseReg(MachineFunction *MF) const {
   SparcMachineFunctionInfo *SparcFI = MF->getInfo<SparcMachineFunctionInfo>();
-  unsigned GlobalBaseReg = SparcFI->getGlobalBaseReg();
-  if (GlobalBaseReg != 0)
+  Register GlobalBaseReg = SparcFI->getGlobalBaseReg();
+  if (GlobalBaseReg)
     return GlobalBaseReg;
 
   // Insert the set of GlobalBaseReg into the first MBB of the function
diff --git a/llvm/lib/Target/Sparc/SparcInstrInfo.h b/llvm/lib/Target/Sparc/SparcInstrInfo.h
index 5347c7150dd5e..b25de8e5a6909 100644
--- a/llvm/lib/Target/Sparc/SparcInstrInfo.h
+++ b/llvm/lib/Target/Sparc/SparcInstrInfo.h
@@ -96,7 +96,7 @@ class SparcInstrInfo : public SparcGenInstrInfo {
                             const TargetRegisterClass *RC,
                             const TargetRegisterInfo *TRI) const override;
 
-  unsigned getGlobalBaseReg(MachineFunction *MF) const;
+  Register getGlobalBaseReg(MachineFunction *MF) const;
 
   // Lower pseudo instructions after register allocation.
   bool expandPostRAPseudo(MachineInstr &MI) const override;
diff --git a/llvm/lib/Target/Sparc/SparcMachineFunctionInfo.h b/llvm/lib/Target/Sparc/SparcMachineFunctionInfo.h
index fe57058786933..d557c8ea22e20 100644
--- a/llvm/lib/Target/Sparc/SparcMachineFunctionInfo.h
+++ b/llvm/lib/Target/Sparc/SparcMachineFunctionInfo.h
@@ -19,14 +19,14 @@ namespace llvm {
   class SparcMachineFunctionInfo : public MachineFunctionInfo {
     virtual void anchor();
   private:
-    unsigned GlobalBaseReg;
+    Register GlobalBaseReg;
 
     /// VarArgsFrameOffset - Frame offset to start of varargs area.
     int VarArgsFrameOffset;
 
     /// SRetReturnReg - Holds the virtual register into which the sret
     /// argument is passed.
-    unsigned SRetReturnReg;
+    Register SRetReturnReg;
 
     /// IsLeafProc - True if the function is a leaf procedure.
     bool IsLeafProc;
@@ -38,14 +38,14 @@ namespace llvm {
       : GlobalBaseReg(0), VarArgsFrameOffset(0), SRetReturnReg(0),
         IsLeafProc(false) {}
 
-    unsigned getGlobalBaseReg() const { return GlobalBaseReg; }
-    void setGlobalBaseReg(unsigned Reg) { GlobalBaseReg = Reg; }
+    Register getGlobalBaseReg() const { return GlobalBaseReg; }
+    void setGlobalBaseReg(Register Reg) { GlobalBaseReg = Reg; }
 
     int getVarArgsFrameOffset() const { return VarArgsFrameOffset; }
     void setVarArgsFrameOffset(int Offset) { VarArgsFrameOffset = Offset; }
 
-    unsigned getSRetReturnReg() const { return SRetReturnReg; }
-    void setSRetReturnReg(unsigned Reg) { SRetReturnReg = Reg; }
+    Register getSRetReturnReg() const { return SRetReturnReg; }
+    void setSRetReturnReg(Register Reg) { SRetReturnReg = Reg; }
 
     void setLeafProc(bool rhs) { IsLeafProc = rhs; }
     bool isLeafProc() const { return IsLeafProc; }
diff --git a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
index a3110248e8e05..d5a3a19446c7a 100644
--- a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
+++ b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
@@ -405,14 +405,16 @@ class SystemZAsmParser : public MCTargetAsmParser {
 
   bool parseRegister(Register &Reg, bool RestoreOnFailure = false);
 
+  bool parseIntegerRegister(Register &Reg, RegisterGroup Group);
+
   OperandMatchResultTy parseRegister(OperandVector &Operands,
                                      RegisterKind Kind);
 
   OperandMatchResultTy parseAnyRegister(OperandVector &Operands);
 
-  bool parseAddress(bool &HaveReg1, Register &Reg1,
-                    bool &HaveReg2, Register &Reg2,
-                    const MCExpr *&Disp, const MCExpr *&Length);
+  bool parseAddress(bool &HaveReg1, Register &Reg1, bool &HaveReg2,
+                    Register &Reg2, const MCExpr *&Disp, const MCExpr *&Length,
+                    bool HasLength = false, bool HasVectorIndex = false);
   bool parseAddressRegister(Register &Reg);
 
   bool ParseDirectiveInsn(SMLoc L);
@@ -748,82 +750,60 @@ bool SystemZAsmParser::parseRegister(Register &Reg, bool RestoreOnFailure) {
 // Parse a register of kind Kind and add it to Operands.
 OperandMatchResultTy
 SystemZAsmParser::parseRegister(OperandVector &Operands, RegisterKind Kind) {
-  SMLoc StartLoc, EndLoc;
-  unsigned RegNum;
+  Register Reg;
+  RegisterGroup Group;
+  switch (Kind) {
+  case GR32Reg:
+  case GRH32Reg:
+  case GR64Reg:
+  case GR128Reg:
+    Group = RegGR;
+    break;
+  case FP32Reg:
+  case FP64Reg:
+  case FP128Reg:
+    Group = RegFP;
+    break;
+  case VR32Reg:
+  case VR64Reg:
+  case VR128Reg:
+    Group = RegV;
+    break;
+  case AR32Reg:
+    Group = RegAR;
+    break;
+  case CR64Reg:
+    Group = RegCR;
+    break;
+  }
 
-  // Handle register names of the form %<prefix><number>.
+  // Handle register names of the form %<prefix><number>
   if (Parser.getTok().is(AsmToken::Percent)) {
-    Register Reg;
     if (parseRegister(Reg))
       return MatchOperand_ParseFail;
 
-    // Verify that a register prefix appropriate for Kind was used.
-    bool PrefixMatch;
-    switch (Kind) {
-    case GR32Reg:
-    case GRH32Reg:
-    case GR64Reg:
-    case GR128Reg:
-      PrefixMatch = Reg.Group == RegGR;
-      break;
-    case FP32Reg:
-    case FP64Reg:
-    case FP128Reg:
-      PrefixMatch = Reg.Group == RegFP;
-      break;
-    case VR32Reg:
-    case VR64Reg:
-    case VR128Reg:
-      // It is OK to use the %f prefix with vector instructions that
-      // expect some VR..Reg kind, so accept the RegFP group as well.
-      PrefixMatch = Reg.Group == RegV || Reg.Group == RegFP;
-      break;
-    case AR32Reg:
-      PrefixMatch = Reg.Group == RegAR;
-      break;
-    case CR64Reg:
-      PrefixMatch = Reg.Group == RegCR;
-      break;
-    }
-    if (!PrefixMatch) {
-      Error(Reg.StartLoc, "invalid operand for instruction");
-      return MatchOperand_ParseFail;
-    }
-
-    RegNum = Reg.Num;
-    StartLoc = Reg.StartLoc;
-    EndLoc = Reg.EndLoc;
-  }
-  // Also allow specifying just a plain register number as integer.
-  else if (Parser.getTok().is(AsmToken::Integer)) {
-    const MCExpr *Register;
-    StartLoc = Parser.getTok().getLoc();
-    if (Parser.parseExpression(Register))
-      return MatchOperand_ParseFail;
-
-    auto *CE = dyn_cast<MCConstantExpr>(Register);
-    if (!CE)
-      return MatchOperand_ParseFail;
-
-    int64_t MaxRegNum;
-    switch (Kind) {
-    case VR32Reg:
-    case VR64Reg:
-    case VR128Reg:
-      MaxRegNum = 31;
+    // Check the parsed register group "Reg.Group" with the expected "Group"
+    // Have to error out if user specified wrong prefix.
+    switch (Group) {
+    case RegGR:
+    case RegFP:
+    case RegAR:
+    case RegCR:
+      if (Group != Reg.Group) {
+        Error(Reg.StartLoc, "invalid operand for instruction");
+        return MatchOperand_ParseFail;
+      }
       break;
-    default:
-      MaxRegNum = 15;
+    case RegV:
+      if (Reg.Group != RegV && Reg.Group != RegFP) {
+        Error(Reg.StartLoc, "invalid operand for instruction");
+        return MatchOperand_ParseFail;
+      }
       break;
     }
-    int64_t Value = CE->getValue();
-    if (Value < 0 || Value > MaxRegNum) {
-      Error(StartLoc, "invalid register");
+  } else if (Parser.getTok().is(AsmToken::Integer)) {
+    if (parseIntegerRegister(Reg, Group))
       return MatchOperand_ParseFail;
-    }
-    RegNum = (unsigned) Value;
-
-    EndLoc = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
   }
   // Otherwise we didn't match a register operand.
   else
@@ -845,13 +825,13 @@ SystemZAsmParser::parseRegister(OperandVector &Operands, RegisterKind Kind) {
   case AR32Reg:  Regs = SystemZMC::AR32Regs;  break;
   case CR64Reg:  Regs = SystemZMC::CR64Regs;  break;
   }
-  if (Regs[RegNum] == 0) {
-    Error(StartLoc, "invalid register pair");
+  if (Regs[Reg.Num] == 0) {
+    Error(Reg.StartLoc, "invalid register pair");
     return MatchOperand_ParseFail;
   }
 
-  Operands.push_back(SystemZOperand::createReg(Kind, Regs[RegNum],
-                                               StartLoc, EndLoc));
+  Operands.push_back(
+      SystemZOperand::createReg(Kind, Regs[Reg.Num], Reg.StartLoc, Reg.EndLoc));
   return MatchOperand_Success;
 }
 
@@ -916,11 +896,39 @@ SystemZAsmParser::parseAnyRegister(OperandVector &Operands) {
   return MatchOperand_Success;
 }
 
+bool SystemZAsmParser::parseIntegerRegister(Register &Reg,
+                                            RegisterGroup Group) {
+  Reg.StartLoc = Parser.getTok().getLoc();
+  // We have an integer token
+  const MCExpr *Register;
+  if (Parser.parseExpression(Register))
+    return true;
+
+  const auto *CE = dyn_cast<MCConstantExpr>(Register);
+  if (!CE)
+    return true;
+
+  int64_t MaxRegNum = (Group == RegV) ? 31 : 15;
+  int64_t Value = CE->getValue();
+  if (Value < 0 || Value > MaxRegNum) {
+    Error(Parser.getTok().getLoc(), "invalid register");
+    return true;
+  }
+
+  // Assign the Register Number
+  Reg.Num = (unsigned)Value;
+  Reg.Group = Group;
+  Reg.EndLoc = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
+
+  // At this point, successfully parsed an integer register.
+  return false;
+}
+
 // Parse a memory operand into Reg1, Reg2, Disp, and Length.
 bool SystemZAsmParser::parseAddress(bool &HaveReg1, Register &Reg1,
                                     bool &HaveReg2, Register &Reg2,
-                                    const MCExpr *&Disp,
-                                    const MCExpr *&Length) {
+                                    const MCExpr *&Disp, const MCExpr *&Length,
+                                    bool HasLength, bool HasVectorIndex) {
   // Parse the displacement, which must always be present.
   if (getParser().parseExpression(Disp))
     return true;
@@ -929,6 +937,27 @@ bool SystemZAsmParser::parseAddress(bool &HaveReg1, Register &Reg1,
   HaveReg1 = false;
   HaveReg2 = false;
   Length = nullptr;
+
+  // If we have a scenario as below:
+  //   vgef %v0, 0(0), 0
+  // This is an example of a "BDVMem" instruction type.
+  //
+  // So when we parse this as an integer register, the register group
+  // needs to be tied to "RegV". Usually when the prefix is passed in
+  // as %<prefix><reg-number> its easy to check which group it should belong to
+  // However, if we're passing in just the integer there's no real way to
+  // "check" what register group it should belong to.
+  //
+  // When the user passes in the register as an integer, the user assumes that
+  // the compiler is responsible for substituting it as the right kind of
+  // register. Whereas, when the user specifies a "prefix", the onus is on
+  // the user to make sure they pass in the right kind of register.
+  //
+  // The restriction only applies to the first Register (i.e. Reg1). Reg2 is
+  // always a general register. Reg1 should be of group RegV if "HasVectorIndex"
+  // (i.e. insn is of type BDVMem) is true.
+  RegisterGroup RegGroup = HasVectorIndex ? RegV : RegGR;
+
   if (getLexer().is(AsmToken::LParen)) {
     Parser.Lex();
 
@@ -937,18 +966,47 @@ bool SystemZAsmParser::parseAddress(bool &HaveReg1, Register &Reg1,
       HaveReg1 = true;
       if (parseRegister(Reg1))
         return true;
+    }
+    // So if we have an integer as the first token in ([tok1], ..), it could:
+    // 1. Refer to a "Register" (i.e X,R,V fields in BD[X|R|V]Mem type of
+    // instructions)
+    // 2. Refer to a "Length" field (i.e L field in BDLMem type of instructions)
+    else if (getLexer().is(AsmToken::Integer)) {
+      if (HasLength) {
+        // Instruction has a "Length" field, safe to parse the first token as
+        // the "Length" field
+        if (getParser().parseExpression(Length))
+          return true;
+      } else {
+        // Otherwise, if the instruction has no "Length" field, parse the
+        // token as a "Register". We don't have to worry about whether the
+        // instruction is invalid here, because the caller will take care of
+        // error reporting.
+        HaveReg1 = true;
+        if (parseIntegerRegister(Reg1, RegGroup))
+          return true;
+      }
     } else {
-      // Parse the length.
-      if (getParser().parseExpression(Length))
-        return true;
+      // If its not an integer or a percent token, then if the instruction
+      // is reported to have a "Length" then, parse it as "Length".
+      if (HasLength) {
+        if (getParser().parseExpression(Length))
+          return true;
+      }
     }
 
     // Check whether there's a second register.
     if (getLexer().is(AsmToken::Comma)) {
       Parser.Lex();
       HaveReg2 = true;
-      if (parseRegister(Reg2))
-        return true;
+
+      if (getLexer().is(AsmToken::Integer)) {
+        if (parseIntegerRegister(Reg2, RegGR))
+          return true;
+      } else {
+        if (parseRegister(Reg2))
+          return true;
+      }
     }
 
     // Consume the closing bracket.
@@ -983,7 +1041,11 @@ SystemZAsmParser::parseAddress(OperandVector &Operands, MemoryKind MemKind,
   bool HaveReg1, HaveReg2;
   const MCExpr *Disp;
   const MCExpr *Length;
-  if (parseAddress(HaveReg1, Reg1, HaveReg2, Reg2, Disp, Length))
+
+  bool HasLength = (MemKind == BDLMem) ? true : false;
+  bool HasVectorIndex = (MemKind == BDVMem) ? true : false;
+  if (parseAddress(HaveReg1, Reg1, HaveReg2, Reg2, Disp, Length, HasLength,
+                   HasVectorIndex))
     return MatchOperand_ParseFail;
 
   const unsigned *Regs;
@@ -1001,11 +1063,7 @@ SystemZAsmParser::parseAddress(OperandVector &Operands, MemoryKind MemKind,
         return MatchOperand_ParseFail;
       Base = Regs[Reg1.Num];
     }
-    // There must be no Reg2 or length.
-    if (Length) {
-      Error(StartLoc, "invalid use of length addressing");
-      return MatchOperand_ParseFail;
-    }
+    // There must be no Reg2.
     if (HaveReg2) {
       Error(StartLoc, "invalid use of indexed addressing");
       return MatchOperand_ParseFail;
@@ -1029,11 +1087,6 @@ SystemZAsmParser::parseAddress(OperandVector &Operands, MemoryKind MemKind,
         return MatchOperand_ParseFail;
       Base = Regs[Reg2.Num];
     }
-    // There must be no length.
-    if (Length) {
-      Error(StartLoc, "invalid use of length addressing");
-      return MatchOperand_ParseFail;
-    }
     break;
   case BDLMem:
     // If we have Reg2, it must be an address register.
@@ -1066,11 +1119,6 @@ SystemZAsmParser::parseAddress(OperandVector &Operands, MemoryKind MemKind,
         return MatchOperand_ParseFail;
       Base = Regs[Reg2.Num];
     }
-    // There must be no length.
-    if (Length) {
-      Error(StartLoc, "invalid use of length addressing");
-      return MatchOperand_ParseFail;
-    }
     break;
   case BDVMem:
     // We must have Reg1, and it must be a vector register.
@@ -1085,16 +1133,11 @@ SystemZAsmParser::parseAddress(OperandVector &Operands, MemoryKind MemKind,
         return MatchOperand_ParseFail;
       Base = Regs[Reg2.Num];
     }
-    // There must be no length.
-    if (Length) {
-      Error(StartLoc, "invalid use of length addressing");
-      return MatchOperand_ParseFail;
-    }
     break;
   }
 
   SMLoc EndLoc =
-    SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
+      SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
   Operands.push_back(SystemZOperand::createMem(MemKind, RegKind, Base, Disp,
                                                Index, Length, LengthReg,
                                                StartLoc, EndLoc));
@@ -1323,7 +1366,8 @@ bool SystemZAsmParser::parseOperand(OperandVector &Operands,
   bool HaveReg1, HaveReg2;
   const MCExpr *Expr;
   const MCExpr *Length;
-  if (parseAddress(HaveReg1, Reg1, HaveReg2, Reg2, Expr, Length))
+  if (parseAddress(HaveReg1, Reg1, HaveReg2, Reg2, Expr, Length,
+                   /*HasLength*/ true, /*HasVectorIndex*/ true))
     return true;
   // If the register combination is not valid for any instruction, reject it.
   // Otherwise, fall back to reporting an unrecognized instruction.
diff --git a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
index f86725a5fde89..985722fdcab4a 100644
--- a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
@@ -341,8 +341,8 @@ processFunctionBeforeFrameFinalized(MachineFunction &MF,
     // are outside the reach of an unsigned 12-bit displacement.
     // Create 2 for the case where both addresses in an MVC are
     // out of range.
-    RS->addScavengingFrameIndex(MFFrame.CreateStackObject(8, 8, false));
-    RS->addScavengingFrameIndex(MFFrame.CreateStackObject(8, 8, false));
+    RS->addScavengingFrameIndex(MFFrame.CreateStackObject(8, Align(8), false));
+    RS->addScavengingFrameIndex(MFFrame.CreateStackObject(8, Align(8), false));
   }
 }
 
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index a753f2a14a35a..eb1e51341ec4d 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -1675,6 +1675,7 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
   if (IsTailCall)
     return DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, Ops);
   Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, Ops);
+  DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
   Glue = Chain.getValue(1);
 
   // Mark the end of the call, which is glued to the call itself.
@@ -4467,12 +4468,22 @@ static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
 }
 
 static bool isZeroVector(SDValue N) {
+  if (N->getOpcode() == ISD::BITCAST)
+    N = N->getOperand(0);
   if (N->getOpcode() == ISD::SPLAT_VECTOR)
     if (auto *Op = dyn_cast<ConstantSDNode>(N->getOperand(0)))
       return Op->getZExtValue() == 0;
   return ISD::isBuildVectorAllZeros(N.getNode());
 }
 
+// Return the index of the zero/undef vector, or UINT32_MAX if not found.
+static uint32_t findZeroVectorIdx(SDValue *Ops, unsigned Num) {
+  for (unsigned I = 0; I < Num ; I++)
+    if (isZeroVector(Ops[I]))
+      return I;
+  return UINT32_MAX;
+}
+
 // Bytes is a VPERM-like permute vector, except that -1 is used for
 // undefined bytes.  Implement it on operands Ops[0] and Ops[1] using
 // VSLDB or VPERM.
@@ -4491,9 +4502,8 @@ static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
 
   // Fall back on VPERM.  Construct an SDNode for the permute vector.  Try to
   // eliminate a zero vector by reusing any zero index in the permute vector.
-  unsigned ZeroVecIdx =
-    isZeroVector(Ops[0]) ? 0 : (isZeroVector(Ops[1]) ? 1 : UINT_MAX);
-  if (ZeroVecIdx != UINT_MAX) {
+  unsigned ZeroVecIdx = findZeroVectorIdx(&Ops[0], 2);
+  if (ZeroVecIdx != UINT32_MAX) {
     bool MaskFirst = true;
     int ZeroIdx = -1;
     for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
@@ -4551,10 +4561,13 @@ static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
 namespace {
 // Describes a general N-operand vector shuffle.
 struct GeneralShuffle {
-  GeneralShuffle(EVT vt) : VT(vt) {}
+  GeneralShuffle(EVT vt) : VT(vt), UnpackFromEltSize(UINT_MAX) {}
   void addUndef();
   bool add(SDValue, unsigned);
   SDValue getNode(SelectionDAG &, const SDLoc &);
+  void tryPrepareForUnpack();
+  bool unpackWasPrepared() { return UnpackFromEltSize <= 4; }
+  SDValue insertUnpackIfPrepared(SelectionDAG &DAG, const SDLoc &DL, SDValue Op);
 
   // The operands of the shuffle.
   SmallVector<SDValue, SystemZ::VectorBytes> Ops;
@@ -4566,6 +4579,9 @@ struct GeneralShuffle {
 
   // The type of the shuffle result.
   EVT VT;
+
+  // Holds a value of 1, 2 or 4 if a final unpack has been prepared for.
+  unsigned UnpackFromEltSize;
 };
 }
 
@@ -4648,6 +4664,9 @@ SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) {
   if (Ops.size() == 0)
     return DAG.getUNDEF(VT);
 
+  // Use a single unpack if possible as the last operation.
+  tryPrepareForUnpack();
+
   // Make sure that there are at least two shuffle operands.
   if (Ops.size() == 1)
     Ops.push_back(DAG.getUNDEF(MVT::v16i8));
@@ -4713,13 +4732,117 @@ SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) {
   // to VPERM.
   unsigned OpNo0, OpNo1;
   SDValue Op;
-  if (const Permute *P = matchPermute(Bytes, OpNo0, OpNo1))
+  if (unpackWasPrepared() && Ops[1].isUndef())
+    Op = Ops[0];
+  else if (const Permute *P = matchPermute(Bytes, OpNo0, OpNo1))
     Op = getPermuteNode(DAG, DL, *P, Ops[OpNo0], Ops[OpNo1]);
   else
     Op = getGeneralPermuteNode(DAG, DL, &Ops[0], Bytes);
+
+  Op = insertUnpackIfPrepared(DAG, DL, Op);
+
   return DAG.getNode(ISD::BITCAST, DL, VT, Op);
 }
 
+#ifndef NDEBUG
+static void dumpBytes(const SmallVectorImpl<int> &Bytes, std::string Msg) {
+  dbgs() << Msg.c_str() << " { ";
+  for (unsigned i = 0; i < Bytes.size(); i++)
+    dbgs() << Bytes[i] << " ";
+  dbgs() << "}\n";
+}
+#endif
+
+// If the Bytes vector matches an unpack operation, prepare to do the unpack
+// after all else by removing the zero vector and the effect of the unpack on
+// Bytes.
+void GeneralShuffle::tryPrepareForUnpack() {
+  uint32_t ZeroVecOpNo = findZeroVectorIdx(&Ops[0], Ops.size());
+  if (ZeroVecOpNo == UINT32_MAX || Ops.size() == 1)
+    return;
+
+  // Only do this if removing the zero vector reduces the depth, otherwise
+  // the critical path will increase with the final unpack.
+  if (Ops.size() > 2 &&
+      Log2_32_Ceil(Ops.size()) == Log2_32_Ceil(Ops.size() - 1))
+    return;
+
+  // Find an unpack that would allow removing the zero vector from Ops.
+  UnpackFromEltSize = 1;
+  for (; UnpackFromEltSize <= 4; UnpackFromEltSize *= 2) {
+    bool MatchUnpack = true;
+    SmallVector<int, SystemZ::VectorBytes> SrcBytes;
+    for (unsigned Elt = 0; Elt < SystemZ::VectorBytes; Elt++) {
+      unsigned ToEltSize = UnpackFromEltSize * 2;
+      bool IsZextByte = (Elt % ToEltSize) < UnpackFromEltSize;
+      if (!IsZextByte)
+        SrcBytes.push_back(Bytes[Elt]);
+      if (Bytes[Elt] != -1) {
+        unsigned OpNo = unsigned(Bytes[Elt]) / SystemZ::VectorBytes;
+        if (IsZextByte != (OpNo == ZeroVecOpNo)) {
+          MatchUnpack = false;
+          break;
+        }
+      }
+    }
+    if (MatchUnpack) {
+      if (Ops.size() == 2) {
+        // Don't use unpack if a single source operand needs rearrangement.
+        for (unsigned i = 0; i < SystemZ::VectorBytes / 2; i++)
+          if (SrcBytes[i] != -1 && SrcBytes[i] % 16 != int(i)) {
+            UnpackFromEltSize = UINT_MAX;
+            return;
+          }
+      }
+      break;
+    }
+  }
+  if (UnpackFromEltSize > 4)
+    return;
+
+  LLVM_DEBUG(dbgs() << "Preparing for final unpack of element size "
+             << UnpackFromEltSize << ". Zero vector is Op#" << ZeroVecOpNo
+             << ".\n";
+             dumpBytes(Bytes, "Original Bytes vector:"););
+
+  // Apply the unpack in reverse to the Bytes array.
+  unsigned B = 0;
+  for (unsigned Elt = 0; Elt < SystemZ::VectorBytes;) {
+    Elt += UnpackFromEltSize;
+    for (unsigned i = 0; i < UnpackFromEltSize; i++, Elt++, B++)
+      Bytes[B] = Bytes[Elt];
+  }
+  while (B < SystemZ::VectorBytes)
+    Bytes[B++] = -1;
+
+  // Remove the zero vector from Ops
+  Ops.erase(&Ops[ZeroVecOpNo]);
+  for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
+    if (Bytes[I] >= 0) {
+      unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
+      if (OpNo > ZeroVecOpNo)
+        Bytes[I] -= SystemZ::VectorBytes;
+    }
+
+  LLVM_DEBUG(dumpBytes(Bytes, "Resulting Bytes vector, zero vector removed:");
+             dbgs() << "\n";);
+}
+
+SDValue GeneralShuffle::insertUnpackIfPrepared(SelectionDAG &DAG,
+                                               const SDLoc &DL,
+                                               SDValue Op) {
+  if (!unpackWasPrepared())
+    return Op;
+  unsigned InBits = UnpackFromEltSize * 8;
+  EVT InVT = MVT::getVectorVT(MVT::getIntegerVT(InBits),
+                                SystemZ::VectorBits / InBits);
+  SDValue PackedOp = DAG.getNode(ISD::BITCAST, DL, InVT, Op);
+  unsigned OutBits = InBits * 2;
+  EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(OutBits),
+                               SystemZ::VectorBits / OutBits);
+  return DAG.getNode(SystemZISD::UNPACKL_HIGH, DL, OutVT, PackedOp);
+}
+
 // Return true if the given BUILD_VECTOR is a scalar-to-vector conversion.
 static bool isScalarToVector(SDValue Op) {
   for (unsigned I = 1, E = Op.getNumOperands(); I != E; ++I)
@@ -5114,9 +5237,8 @@ SystemZTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
   return DAG.getNode(ISD::BITCAST, DL, VT, Res);
 }
 
-SDValue
-SystemZTargetLowering::lowerExtendVectorInreg(SDValue Op, SelectionDAG &DAG,
-                                              unsigned UnpackHigh) const {
+SDValue SystemZTargetLowering::
+lowerSIGN_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
   SDValue PackedOp = Op.getOperand(0);
   EVT OutVT = Op.getValueType();
   EVT InVT = PackedOp.getValueType();
@@ -5126,11 +5248,39 @@ SystemZTargetLowering::lowerExtendVectorInreg(SDValue Op, SelectionDAG &DAG,
     FromBits *= 2;
     EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(FromBits),
                                  SystemZ::VectorBits / FromBits);
-    PackedOp = DAG.getNode(UnpackHigh, SDLoc(PackedOp), OutVT, PackedOp);
+    PackedOp =
+      DAG.getNode(SystemZISD::UNPACK_HIGH, SDLoc(PackedOp), OutVT, PackedOp);
   } while (FromBits != ToBits);
   return PackedOp;
 }
 
+// Lower a ZERO_EXTEND_VECTOR_INREG to a vector shuffle with a zero vector.
+SDValue SystemZTargetLowering::
+lowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
+  SDValue PackedOp = Op.getOperand(0);
+  SDLoc DL(Op);
+  EVT OutVT = Op.getValueType();
+  EVT InVT = PackedOp.getValueType();
+  unsigned InNumElts = InVT.getVectorNumElements();
+  unsigned OutNumElts = OutVT.getVectorNumElements();
+  unsigned NumInPerOut = InNumElts / OutNumElts;
+
+  SDValue ZeroVec =
+    DAG.getSplatVector(InVT, DL, DAG.getConstant(0, DL, InVT.getScalarType()));
+
+  SmallVector<int, 16> Mask(InNumElts);
+  unsigned ZeroVecElt = InNumElts;
+  for (unsigned PackedElt = 0; PackedElt < OutNumElts; PackedElt++) {
+    unsigned MaskElt = PackedElt * NumInPerOut;
+    unsigned End = MaskElt + NumInPerOut - 1;
+    for (; MaskElt < End; MaskElt++)
+      Mask[MaskElt] = ZeroVecElt++;
+    Mask[MaskElt] = PackedElt;
+  }
+  SDValue Shuf = DAG.getVectorShuffle(InVT, DL, PackedOp, ZeroVec, Mask);
+  return DAG.getNode(ISD::BITCAST, DL, OutVT, Shuf);
+}
+
 SDValue SystemZTargetLowering::lowerShift(SDValue Op, SelectionDAG &DAG,
                                           unsigned ByScalar) const {
   // Look for cases where a vector shift can use the *_BY_SCALAR form.
@@ -5296,9 +5446,9 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
   case ISD::EXTRACT_VECTOR_ELT:
     return lowerEXTRACT_VECTOR_ELT(Op, DAG);
   case ISD::SIGN_EXTEND_VECTOR_INREG:
-    return lowerExtendVectorInreg(Op, DAG, SystemZISD::UNPACK_HIGH);
+    return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
   case ISD::ZERO_EXTEND_VECTOR_INREG:
-    return lowerExtendVectorInreg(Op, DAG, SystemZISD::UNPACKL_HIGH);
+    return lowerZERO_EXTEND_VECTOR_INREG(Op, DAG);
   case ISD::SHL:
     return lowerShift(Op, DAG, SystemZISD::VSHL_BY_SCALAR);
   case ISD::SRL:
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index e60deaedbdfb6..27637762296a4 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -627,8 +627,8 @@ class SystemZTargetLowering : public TargetLowering {
   SDValue lowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
-  SDValue lowerExtendVectorInreg(SDValue Op, SelectionDAG &DAG,
-                                 unsigned UnpackHigh) const;
+  SDValue lowerSIGN_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerShift(SDValue Op, SelectionDAG &DAG, unsigned ByScalar) const;
 
   bool canTreatAsByteVector(EVT VT) const;
diff --git a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
index e655bf8329aad..fe2aaca8429a8 100644
--- a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
@@ -271,7 +271,7 @@ SystemZRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
   // Special handling of dbg_value instructions.
   if (MI->isDebugValue()) {
     MI->getOperand(FIOperandNum).ChangeToRegister(BasePtr, /*isDef*/ false);
-    MI->getOperand(FIOperandNum + 1).ChangeToImmediate(Offset);
+    MI->getDebugOffset().ChangeToImmediate(Offset);
     return;
   }
 
diff --git a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
index 47c925dcf730f..6b4f35e5ba2b4 100644
--- a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
@@ -47,7 +47,7 @@ static SDValue emitMemMem(SelectionDAG &DAG, const SDLoc &DL, unsigned Sequence,
 
 SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemcpy(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
-    SDValue Size, unsigned Align, bool IsVolatile, bool AlwaysInline,
+    SDValue Size, Align Alignment, bool IsVolatile, bool AlwaysInline,
     MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
   if (IsVolatile)
     return SDValue();
@@ -74,7 +74,7 @@ static SDValue memsetStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
 
 SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemset(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst,
-    SDValue Byte, SDValue Size, unsigned Align, bool IsVolatile,
+    SDValue Byte, SDValue Size, Align Alignment, bool IsVolatile,
     MachinePointerInfo DstPtrInfo) const {
   EVT PtrVT = Dst.getValueType();
 
@@ -97,20 +97,22 @@ SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemset(
         unsigned Size1 = Bytes == 16 ? 8 : 1 << findLastSet(Bytes);
         unsigned Size2 = Bytes - Size1;
         SDValue Chain1 = memsetStore(DAG, DL, Chain, Dst, ByteVal, Size1,
-                                     Align, DstPtrInfo);
+                                     Alignment.value(), DstPtrInfo);
         if (Size2 == 0)
           return Chain1;
         Dst = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
                           DAG.getConstant(Size1, DL, PtrVT));
         DstPtrInfo = DstPtrInfo.getWithOffset(Size1);
-        SDValue Chain2 = memsetStore(DAG, DL, Chain, Dst, ByteVal, Size2,
-                                     std::min(Align, Size1), DstPtrInfo);
+        SDValue Chain2 = memsetStore(
+            DAG, DL, Chain, Dst, ByteVal, Size2,
+            std::min((unsigned)Alignment.value(), Size1), DstPtrInfo);
         return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chain1, Chain2);
       }
     } else {
       // Handle one and two bytes using STC.
       if (Bytes <= 2) {
-        SDValue Chain1 = DAG.getStore(Chain, DL, Byte, Dst, DstPtrInfo, Align);
+        SDValue Chain1 =
+            DAG.getStore(Chain, DL, Byte, Dst, DstPtrInfo, Alignment);
         if (Bytes == 1)
           return Chain1;
         SDValue Dst2 = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
@@ -131,7 +133,7 @@ SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemset(
 
     // Copy the byte to the first location and then use MVC to copy
     // it to the rest.
-    Chain = DAG.getStore(Chain, DL, Byte, Dst, DstPtrInfo, Align);
+    Chain = DAG.getStore(Chain, DL, Byte, Dst, DstPtrInfo, Alignment);
     SDValue DstPlus1 = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
                                    DAG.getConstant(1, DL, PtrVT));
     return emitMemMem(DAG, DL, SystemZISD::MVC, SystemZISD::MVC_LOOP,
diff --git a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h
index 7d63bae83cf33..a4a5b1fbdf905 100644
--- a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h
@@ -25,14 +25,15 @@ class SystemZSelectionDAGInfo : public SelectionDAGTargetInfo {
 
   SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &DL,
                                   SDValue Chain, SDValue Dst, SDValue Src,
-                                  SDValue Size, unsigned Align, bool IsVolatile,
-                                  bool AlwaysInline,
+                                  SDValue Size, Align Alignment,
+                                  bool IsVolatile, bool AlwaysInline,
                                   MachinePointerInfo DstPtrInfo,
                                   MachinePointerInfo SrcPtrInfo) const override;
 
   SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &DL,
                                   SDValue Chain, SDValue Dst, SDValue Byte,
-                                  SDValue Size, unsigned Align, bool IsVolatile,
+                                  SDValue Size, Align Alignment,
+                                  bool IsVolatile,
                                   MachinePointerInfo DstPtrInfo) const override;
 
   std::pair<SDValue, SDValue>
diff --git a/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp b/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp
index bebfe72a1dcc4..68e0b7ae66a4d 100644
--- a/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp
@@ -74,9 +74,12 @@ bool SystemZSubtarget::enableSubRegLiveness() const {
 
 bool SystemZSubtarget::isPC32DBLSymbol(const GlobalValue *GV,
                                        CodeModel::Model CM) const {
-  // PC32DBL accesses require the low bit to be clear.  Note that a zero
-  // value selects the default alignment and is therefore OK.
-  if (GV->getAlignment() == 1)
+  // PC32DBL accesses require the low bit to be clear.
+  //
+  // FIXME: Explicitly check for functions: the datalayout is currently
+  // missing information about function pointers.
+  const DataLayout &DL = GV->getParent()->getDataLayout();
+  if (GV->getPointerAlignment(DL) == 1 && !GV->getValueType()->isFunctionTy())
     return false;
 
   // For the small model, all locally-binding symbols are in range.
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index cebc3fdc52c1f..864200e5f71cc 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -294,6 +294,10 @@ void SystemZTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
   UP.Force = true;
 }
 
+void SystemZTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                                           TTI::PeelingPreferences &PP) {
+  BaseT::getPeelingPreferences(L, SE, PP);
+}
 
 bool SystemZTTIImpl::isLSRCostLess(TargetTransformInfo::LSRCost &C1,
                                    TargetTransformInfo::LSRCost &C2) {
@@ -1102,14 +1106,10 @@ int SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
 // needed for using / defining the vector operands. The SystemZ version does
 // roughly the same but bases the computations on vector permutations
 // instead.
-int SystemZTTIImpl::getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
-                                               unsigned Factor,
-                                               ArrayRef<unsigned> Indices,
-                                               unsigned Alignment,
-                                               unsigned AddressSpace,
-                                               TTI::TargetCostKind CostKind,
-                                               bool UseMaskForCond,
-                                               bool UseMaskForGaps) {
+int SystemZTTIImpl::getInterleavedMemoryOpCost(
+    unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+    Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
+    bool UseMaskForCond, bool UseMaskForGaps) {
   if (UseMaskForCond || UseMaskForGaps)
     return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
                                              Alignment, AddressSpace, CostKind,
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index 99a11d9b5b782..7f8f7f6f923ff 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -50,6 +50,9 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP);
 
+  void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                             TTI::PeelingPreferences &PP);
+
   bool isLSRCostLess(TargetTransformInfo::LSRCost &C1,
                      TargetTransformInfo::LSRCost &C2);
   /// @}
@@ -101,14 +104,11 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
                       unsigned AddressSpace, TTI::TargetCostKind CostKind,
                       const Instruction *I = nullptr);
 
-  int getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
-                                 unsigned Factor,
-                                 ArrayRef<unsigned> Indices,
-                                 unsigned Alignment,
-                                 unsigned AddressSpace,
-                                 TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
-                                 bool UseMaskForCond = false,
-                                 bool UseMaskForGaps = false);
+  int getInterleavedMemoryOpCost(
+      unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+      Align Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
+      bool UseMaskForCond = false, bool UseMaskForGaps = false);
 
   int getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                             TTI::TargetCostKind CostKind);
diff --git a/llvm/lib/Target/Target.cpp b/llvm/lib/Target/Target.cpp
index 8a46c77492c58..7411687e2ca3d 100644
--- a/llvm/lib/Target/Target.cpp
+++ b/llvm/lib/Target/Target.cpp
@@ -111,11 +111,11 @@ unsigned long long LLVMABISizeOfType(LLVMTargetDataRef TD, LLVMTypeRef Ty) {
 }
 
 unsigned LLVMABIAlignmentOfType(LLVMTargetDataRef TD, LLVMTypeRef Ty) {
-  return unwrap(TD)->getABITypeAlignment(unwrap(Ty));
+  return unwrap(TD)->getABITypeAlign(unwrap(Ty)).value();
 }
 
 unsigned LLVMCallFrameAlignmentOfType(LLVMTargetDataRef TD, LLVMTypeRef Ty) {
-  return unwrap(TD)->getABITypeAlignment(unwrap(Ty));
+  return unwrap(TD)->getABITypeAlign(unwrap(Ty)).value();
 }
 
 unsigned LLVMPreferredAlignmentOfType(LLVMTargetDataRef TD, LLVMTypeRef Ty) {
@@ -124,7 +124,9 @@ unsigned LLVMPreferredAlignmentOfType(LLVMTargetDataRef TD, LLVMTypeRef Ty) {
 
 unsigned LLVMPreferredAlignmentOfGlobal(LLVMTargetDataRef TD,
                                         LLVMValueRef GlobalVar) {
-  return unwrap(TD)->getPreferredAlignment(unwrap<GlobalVariable>(GlobalVar));
+  return unwrap(TD)
+      ->getPreferredAlign(unwrap<GlobalVariable>(GlobalVar))
+      .value();
 }
 
 unsigned LLVMElementAtOffset(LLVMTargetDataRef TD, LLVMTypeRef StructTy,
diff --git a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
index 214adf4c3c5b7..7a899b4b38e2a 100644
--- a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
+++ b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
@@ -65,8 +65,6 @@ class VEAsmParser : public MCTargetAsmParser {
   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
                                       unsigned Kind) override;
 
-  // Helper function to parse and generate identifier with relocation.
-  const MCExpr *parseIdentifier(StringRef Identifier);
   // Custom parse functions for VE specific operands.
   OperandMatchResultTy parseMEMOperand(OperandVector &Operands);
   OperandMatchResultTy parseMEMAsOperand(OperandVector &Operands);
@@ -75,6 +73,13 @@ class VEAsmParser : public MCTargetAsmParser {
   OperandMatchResultTy parseMImmOperand(OperandVector &Operands);
   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Name);
   OperandMatchResultTy parseVEAsmOperand(std::unique_ptr<VEOperand> &Operand);
+
+  // Helper function to parse expression with a symbol.
+  const MCExpr *extractModifierFromExpr(const MCExpr *E,
+                                        VEMCExpr::VariantKind &Variant);
+  const MCExpr *fixupVariantKind(const MCExpr *E);
+  bool parseExpression(const MCExpr *&EVal);
+
   // Split the mnemonic stripping conditional code and quantifiers
   StringRef splitMnemonic(StringRef Name, SMLoc NameLoc,
                           OperandVector *Operands);
@@ -948,27 +953,163 @@ bool VEAsmParser::ParseDirective(AsmToken DirectiveID) {
   return true;
 }
 
-const MCExpr *VEAsmParser::parseIdentifier(StringRef Identifier) {
-  StringRef Modifier;
-  // Search @modifiers like "symbol@hi".
-  size_t at = Identifier.rfind('@');
-  if (at != 0 || at != StringRef::npos) {
-    std::pair<StringRef, StringRef> Pair = Identifier.rsplit("@");
-    if (!Pair.first.empty() && !Pair.second.empty()) {
-      Identifier = Pair.first;
-      Modifier = Pair.second;
+/// Extract \code @lo32/@hi32/etc \endcode modifier from expression.
+/// Recursively scan the expression and check for VK_VE_HI32/LO32/etc
+/// symbol variants.  If all symbols with modifier use the same
+/// variant, return the corresponding VEMCExpr::VariantKind,
+/// and a modified expression using the default symbol variant.
+/// Otherwise, return NULL.
+const MCExpr *
+VEAsmParser::extractModifierFromExpr(const MCExpr *E,
+                                     VEMCExpr::VariantKind &Variant) {
+  MCContext &Context = getParser().getContext();
+  Variant = VEMCExpr::VK_VE_None;
+
+  switch (E->getKind()) {
+  case MCExpr::Target:
+  case MCExpr::Constant:
+    return nullptr;
+
+  case MCExpr::SymbolRef: {
+    const MCSymbolRefExpr *SRE = cast<MCSymbolRefExpr>(E);
+
+    switch (SRE->getKind()) {
+    case MCSymbolRefExpr::VK_None:
+      // Use VK_VE_REFLONG to a symbol without modifiers.
+      Variant = VEMCExpr::VK_VE_REFLONG;
+      break;
+    case MCSymbolRefExpr::VK_VE_HI32:
+      Variant = VEMCExpr::VK_VE_HI32;
+      break;
+    case MCSymbolRefExpr::VK_VE_LO32:
+      Variant = VEMCExpr::VK_VE_LO32;
+      break;
+    case MCSymbolRefExpr::VK_VE_PC_HI32:
+      Variant = VEMCExpr::VK_VE_PC_HI32;
+      break;
+    case MCSymbolRefExpr::VK_VE_PC_LO32:
+      Variant = VEMCExpr::VK_VE_PC_LO32;
+      break;
+    case MCSymbolRefExpr::VK_VE_GOT_HI32:
+      Variant = VEMCExpr::VK_VE_GOT_HI32;
+      break;
+    case MCSymbolRefExpr::VK_VE_GOT_LO32:
+      Variant = VEMCExpr::VK_VE_GOT_LO32;
+      break;
+    case MCSymbolRefExpr::VK_VE_GOTOFF_HI32:
+      Variant = VEMCExpr::VK_VE_GOTOFF_HI32;
+      break;
+    case MCSymbolRefExpr::VK_VE_GOTOFF_LO32:
+      Variant = VEMCExpr::VK_VE_GOTOFF_LO32;
+      break;
+    case MCSymbolRefExpr::VK_VE_PLT_HI32:
+      Variant = VEMCExpr::VK_VE_PLT_HI32;
+      break;
+    case MCSymbolRefExpr::VK_VE_PLT_LO32:
+      Variant = VEMCExpr::VK_VE_PLT_LO32;
+      break;
+    case MCSymbolRefExpr::VK_VE_TLS_GD_HI32:
+      Variant = VEMCExpr::VK_VE_TLS_GD_HI32;
+      break;
+    case MCSymbolRefExpr::VK_VE_TLS_GD_LO32:
+      Variant = VEMCExpr::VK_VE_TLS_GD_LO32;
+      break;
+    case MCSymbolRefExpr::VK_VE_TPOFF_HI32:
+      Variant = VEMCExpr::VK_VE_TPOFF_HI32;
+      break;
+    case MCSymbolRefExpr::VK_VE_TPOFF_LO32:
+      Variant = VEMCExpr::VK_VE_TPOFF_LO32;
+      break;
+    default:
+      return nullptr;
     }
+
+    return MCSymbolRefExpr::create(&SRE->getSymbol(), Context);
   }
-  MCSymbol *Sym = getContext().getOrCreateSymbol(Identifier);
-  const MCExpr *Res =
-      MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, getContext());
-  VEMCExpr::VariantKind VK = VEMCExpr::parseVariantKind(Modifier);
-  if (VK == VEMCExpr::VK_VE_None) {
-    // Create identifier using default variant kind
-    VEMCExpr::VariantKind Kind = VEMCExpr::VK_VE_REFLONG;
-    return VEMCExpr::create(Kind, Res, getContext());
+
+  case MCExpr::Unary: {
+    const MCUnaryExpr *UE = cast<MCUnaryExpr>(E);
+    const MCExpr *Sub = extractModifierFromExpr(UE->getSubExpr(), Variant);
+    if (!Sub)
+      return nullptr;
+    return MCUnaryExpr::create(UE->getOpcode(), Sub, Context);
   }
-  return VEMCExpr::create(VK, Res, getContext());
+
+  case MCExpr::Binary: {
+    const MCBinaryExpr *BE = cast<MCBinaryExpr>(E);
+    VEMCExpr::VariantKind LHSVariant, RHSVariant;
+    const MCExpr *LHS = extractModifierFromExpr(BE->getLHS(), LHSVariant);
+    const MCExpr *RHS = extractModifierFromExpr(BE->getRHS(), RHSVariant);
+
+    if (!LHS && !RHS)
+      return nullptr;
+
+    if (!LHS)
+      LHS = BE->getLHS();
+    if (!RHS)
+      RHS = BE->getRHS();
+
+    if (LHSVariant == VEMCExpr::VK_VE_None)
+      Variant = RHSVariant;
+    else if (RHSVariant == VEMCExpr::VK_VE_None)
+      Variant = LHSVariant;
+    else if (LHSVariant == RHSVariant)
+      Variant = LHSVariant;
+    else
+      return nullptr;
+
+    return MCBinaryExpr::create(BE->getOpcode(), LHS, RHS, Context);
+  }
+  }
+
+  llvm_unreachable("Invalid expression kind!");
+}
+
+const MCExpr *VEAsmParser::fixupVariantKind(const MCExpr *E) {
+  MCContext &Context = getParser().getContext();
+
+  switch (E->getKind()) {
+  case MCExpr::Target:
+  case MCExpr::Constant:
+  case MCExpr::SymbolRef:
+    return E;
+
+  case MCExpr::Unary: {
+    const MCUnaryExpr *UE = cast<MCUnaryExpr>(E);
+    const MCExpr *Sub = fixupVariantKind(UE->getSubExpr());
+    if (Sub == UE->getSubExpr())
+      return E;
+    return MCUnaryExpr::create(UE->getOpcode(), Sub, Context);
+  }
+
+  case MCExpr::Binary: {
+    const MCBinaryExpr *BE = cast<MCBinaryExpr>(E);
+    const MCExpr *LHS = fixupVariantKind(BE->getLHS());
+    const MCExpr *RHS = fixupVariantKind(BE->getRHS());
+    if (LHS == BE->getLHS() && RHS == BE->getRHS())
+      return E;
+    return MCBinaryExpr::create(BE->getOpcode(), LHS, RHS, Context);
+  }
+  }
+
+  llvm_unreachable("Invalid expression kind!");
+}
+
+/// ParseExpression.  This differs from the default "parseExpression" in that
+/// it handles modifiers.
+bool VEAsmParser::parseExpression(const MCExpr *&EVal) {
+  // Handle \code symbol @lo32/@hi32/etc \endcode.
+  if (getParser().parseExpression(EVal))
+    return true;
+
+  // Convert MCSymbolRefExpr with VK_* to MCExpr with VK_*.
+  EVal = fixupVariantKind(EVal);
+  VEMCExpr::VariantKind Variant;
+  const MCExpr *E = extractModifierFromExpr(EVal, Variant);
+  if (E)
+    EVal = VEMCExpr::create(Variant, E, getParser().getContext());
+
+  return false;
 }
 
 OperandMatchResultTy VEAsmParser::parseMEMOperand(OperandVector &Operands) {
@@ -992,27 +1133,16 @@ OperandMatchResultTy VEAsmParser::parseMEMOperand(OperandVector &Operands) {
 
   case AsmToken::Minus:
   case AsmToken::Integer:
-  case AsmToken::Dot: {
+  case AsmToken::Dot:
+  case AsmToken::Identifier: {
     const MCExpr *EVal;
-    if (!getParser().parseExpression(EVal, E))
+    if (!parseExpression(EVal))
       Offset = VEOperand::CreateImm(EVal, S, E);
     else
       return MatchOperand_NoMatch;
     break;
   }
 
-  case AsmToken::Identifier: {
-    StringRef Identifier;
-    if (!getParser().parseIdentifier(Identifier)) {
-      E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
-      const MCExpr *EVal = parseIdentifier(Identifier);
-
-      E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
-      Offset = VEOperand::CreateImm(EVal, S, E);
-    }
-    break;
-  }
-
   case AsmToken::LParen:
     // empty disp (= 0)
     Offset =
@@ -1110,9 +1240,10 @@ OperandMatchResultTy VEAsmParser::parseMEMAsOperand(OperandVector &Operands) {
 
   case AsmToken::Minus:
   case AsmToken::Integer:
-  case AsmToken::Dot: {
+  case AsmToken::Dot:
+  case AsmToken::Identifier: {
     const MCExpr *EVal;
-    if (!getParser().parseExpression(EVal, E))
+    if (!parseExpression(EVal))
       Offset = VEOperand::CreateImm(EVal, S, E);
     else
       return MatchOperand_NoMatch;
@@ -1275,22 +1406,10 @@ VEAsmParser::parseVEAsmOperand(std::unique_ptr<VEOperand> &Op) {
   case AsmToken::Minus:
   case AsmToken::Integer:
   case AsmToken::Dot:
-    if (!getParser().parseExpression(EVal, E))
+  case AsmToken::Identifier:
+    if (!parseExpression(EVal))
       Op = VEOperand::CreateImm(EVal, S, E);
     break;
-
-  case AsmToken::Identifier: {
-    StringRef Identifier;
-    if (!getParser().parseIdentifier(Identifier)) {
-      E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
-      MCSymbol *Sym = getContext().getOrCreateSymbol(Identifier);
-
-      const MCExpr *Res =
-          MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, getContext());
-      Op = VEOperand::CreateImm(Res, S, E);
-    }
-    break;
-  }
   }
   return (Op) ? MatchOperand_Success : MatchOperand_ParseFail;
 }
diff --git a/llvm/lib/Target/VE/VEISelLowering.cpp b/llvm/lib/Target/VE/VEISelLowering.cpp
index 9abffae413d4e..ab720545dd831 100644
--- a/llvm/lib/Target/VE/VEISelLowering.cpp
+++ b/llvm/lib/Target/VE/VEISelLowering.cpp
@@ -548,7 +548,7 @@ bool VETargetLowering::hasAndNot(SDValue Y) const {
   // for all immediate values now.
   // FIXME: Change hasAndNot function to have two operands to make it work
   //        correctly with Aurora VE.
-  if (auto *C = dyn_cast<ConstantSDNode>(Y))
+  if (isa<ConstantSDNode>(Y))
     return false;
 
   // It's ok for generic registers.
diff --git a/llvm/lib/Target/VE/VEMCInstLower.cpp b/llvm/lib/Target/VE/VEMCInstLower.cpp
index 389e77f913fe6..9815610510e14 100644
--- a/llvm/lib/Target/VE/VEMCInstLower.cpp
+++ b/llvm/lib/Target/VE/VEMCInstLower.cpp
@@ -30,9 +30,14 @@ static MCOperand LowerSymbolOperand(const MachineInstr *MI,
                                     const MCSymbol *Symbol, AsmPrinter &AP) {
   VEMCExpr::VariantKind Kind = (VEMCExpr::VariantKind)MO.getTargetFlags();
 
-  const MCSymbolRefExpr *MCSym = MCSymbolRefExpr::create(Symbol, AP.OutContext);
-  const VEMCExpr *expr = VEMCExpr::create(Kind, MCSym, AP.OutContext);
-  return MCOperand::createExpr(expr);
+  const MCExpr *Expr = MCSymbolRefExpr::create(Symbol, AP.OutContext);
+  // Add offset iff MO is not jump table info or machine basic block.
+  if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())
+    Expr = MCBinaryExpr::createAdd(
+        Expr, MCConstantExpr::create(MO.getOffset(), AP.OutContext),
+        AP.OutContext);
+  Expr = VEMCExpr::create(Kind, Expr, AP.OutContext);
+  return MCOperand::createExpr(Expr);
 }
 
 static MCOperand LowerOperand(const MachineInstr *MI, const MachineOperand &MO,
diff --git a/llvm/lib/Target/VE/VETargetMachine.cpp b/llvm/lib/Target/VE/VETargetMachine.cpp
index a0c8ae0c82d71..08b55eebbc98d 100644
--- a/llvm/lib/Target/VE/VETargetMachine.cpp
+++ b/llvm/lib/Target/VE/VETargetMachine.cpp
@@ -41,8 +41,8 @@ static std::string computeDataLayout(const Triple &T) {
   // VE supports 32 bit and 64 bits integer on registers
   Ret += "-n32:64";
 
-  // Stack alignment is 64 bits
-  Ret += "-S64";
+  // Stack alignment is 128 bits
+  Ret += "-S128";
 
   return Ret;
 }
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
index 111211652adf9..e29d85d7588d2 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
@@ -322,6 +322,8 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
       return wasm::ValType::V128;
     if (Type == "exnref")
       return wasm::ValType::EXNREF;
+    if (Type == "externref")
+      return wasm::ValType::EXTERNREF;
     return Optional<wasm::ValType>();
   }
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyDebugValueManager.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyDebugValueManager.cpp
index eb67d438e9a4c..159fb4c00ddc7 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyDebugValueManager.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyDebugValueManager.cpp
@@ -31,7 +31,7 @@ void WebAssemblyDebugValueManager::move(MachineInstr *Insert) {
 
 void WebAssemblyDebugValueManager::updateReg(unsigned Reg) {
   for (auto *DBI : DbgValues)
-    DBI->getOperand(0).setReg(Reg);
+    DBI->getDebugOperand(0).setReg(Reg);
 }
 
 void WebAssemblyDebugValueManager::clone(MachineInstr *Insert,
@@ -40,14 +40,14 @@ void WebAssemblyDebugValueManager::clone(MachineInstr *Insert,
   MachineFunction *MF = MBB->getParent();
   for (MachineInstr *DBI : reverse(DbgValues)) {
     MachineInstr *Clone = MF->CloneMachineInstr(DBI);
-    Clone->getOperand(0).setReg(NewReg);
+    Clone->getDebugOperand(0).setReg(NewReg);
     MBB->insert(Insert, Clone);
   }
 }
 
 void WebAssemblyDebugValueManager::replaceWithLocal(unsigned LocalId) {
   for (auto *DBI : DbgValues) {
-    MachineOperand &Op = DBI->getOperand(0);
+    MachineOperand &Op = DBI->getDebugOperand(0);
     Op.ChangeToTargetIndex(llvm::WebAssembly::TI_LOCAL, LocalId);
   }
 }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
index 160bc7420f270..55925bcbe7711 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
@@ -266,7 +266,7 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
           BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(Opc), NewReg)
               .addImm(LocalId);
           MI.getOperand(2).setReg(NewReg);
-          MFI.stackifyVReg(NewReg);
+          MFI.stackifyVReg(MRI, NewReg);
         }
 
         // Replace the TEE with a LOCAL_TEE.
@@ -316,7 +316,7 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
           // yet.
           Def.setReg(NewReg);
           Def.setIsDead(false);
-          MFI.stackifyVReg(NewReg);
+          MFI.stackifyVReg(MRI, NewReg);
           Changed = true;
         }
       }
@@ -368,7 +368,7 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
             BuildMI(MBB, InsertPt, MI.getDebugLoc(), TII->get(Opc), NewReg)
                 .addImm(LocalId);
         MO.setReg(NewReg);
-        MFI.stackifyVReg(NewReg);
+        MFI.stackifyVReg(MRI, NewReg);
         Changed = true;
       }
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp
index 539343af9eb24..7f805b34b4995 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp
@@ -41,9 +41,11 @@ class WebAssemblyFixBrTableDefaults final : public MachineFunctionPass {
 
 char WebAssemblyFixBrTableDefaults::ID = 0;
 
-// `MI` is a br_table instruction missing its default target argument. This
+// `MI` is a br_table instruction with a dummy default target argument. This
 // function finds and adds the default target argument and removes any redundant
-// range check preceding the br_table.
+// range check preceding the br_table. Returns the MBB that the br_table is
+// moved into so it can be removed from further consideration, or nullptr if the
+// br_table cannot be optimized.
 MachineBasicBlock *fixBrTable(MachineInstr &MI, MachineBasicBlock *MBB,
                               MachineFunction &MF) {
   // Get the header block, which contains the redundant range check.
@@ -51,11 +53,14 @@ MachineBasicBlock *fixBrTable(MachineInstr &MI, MachineBasicBlock *MBB,
   auto *HeaderMBB = *MBB->pred_begin();
 
   // Find the conditional jump to the default target. If it doesn't exist, the
-  // default target is unreachable anyway, so we can choose anything.
+  // default target is unreachable anyway, so we can keep the existing dummy
+  // target.
   MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
   SmallVector<MachineOperand, 2> Cond;
   const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
-  TII.analyzeBranch(*HeaderMBB, TBB, FBB, Cond);
+  bool Analyzed = !TII.analyzeBranch(*HeaderMBB, TBB, FBB, Cond);
+  assert(Analyzed && "Could not analyze jump header branches");
+  (void)Analyzed;
 
   // Here are the possible outcomes. '_' is nullptr, `J` is the jump table block
   // aka MBB, 'D' is the default block.
@@ -66,14 +71,25 @@ MachineBasicBlock *fixBrTable(MachineInstr &MI, MachineBasicBlock *MBB,
   //  D  |  _  | Header jumps to the default and falls through to the jump table
   //  D  |  J  | Header jumps to the default and also to the jump table
   if (TBB && TBB != MBB) {
-    // Install the default target.
     assert((FBB == nullptr || FBB == MBB) &&
            "Expected jump or fallthrough to br_table block");
+    assert(Cond.size() == 2 && Cond[1].isReg() && "Unexpected condition info");
+
+    // If the range check checks an i64 value, we cannot optimize it out because
+    // the i64 index is truncated to an i32, making values over 2^32
+    // indistinguishable from small numbers. There are also other strange edge
+    // cases that can arise in practice that we don't want to reason about, so
+    // conservatively only perform the optimization if the range check is the
+    // normal case of an i32.gt_u.
+    MachineRegisterInfo &MRI = MF.getRegInfo();
+    auto *RangeCheck = MRI.getVRegDef(Cond[1].getReg());
+    assert(RangeCheck != nullptr);
+    if (RangeCheck->getOpcode() != WebAssembly::GT_U_I32)
+      return nullptr;
+
+    // Remove the dummy default target and install the real one.
+    MI.RemoveOperand(MI.getNumExplicitOperands() - 1);
     MI.addOperand(MF, MachineOperand::CreateMBB(TBB));
-  } else {
-    // Arbitrarily choose the first jump target as the default.
-    auto *SomeMBB = MI.getOperand(1).getMBB();
-    MI.addOperand(MachineOperand::CreateMBB(SomeMBB));
   }
 
   // Remove any branches from the header and splice in the jump table instead
@@ -110,8 +126,10 @@ bool WebAssemblyFixBrTableDefaults::runOnMachineFunction(MachineFunction &MF) {
     for (auto &MI : *MBB) {
       if (WebAssembly::isBrTable(MI)) {
         auto *Fixed = fixBrTable(MI, MBB, MF);
-        MBBSet.erase(Fixed);
-        Changed = true;
+        if (Fixed != nullptr) {
+          MBBSet.erase(Fixed);
+          Changed = true;
+        }
         break;
       }
     }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyFrameLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyFrameLowering.cpp
index 30647fd6c5bbf..95669932e73f5 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyFrameLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyFrameLowering.cpp
@@ -87,8 +87,8 @@ bool WebAssemblyFrameLowering::needsSPForLocalFrame(
 }
 
 // In function with EH pads, we need to make a copy of the value of
-// __stack_pointer global in SP32 register, in order to use it when restoring
-// __stack_pointer after an exception is caught.
+// __stack_pointer global in SP32/64 register, in order to use it when
+// restoring __stack_pointer after an exception is caught.
 bool WebAssemblyFrameLowering::needsPrologForEH(
     const MachineFunction &MF) const {
   auto EHType = MF.getTarget().getMCAsmInfo()->getExceptionHandlingType();
@@ -123,6 +123,57 @@ bool WebAssemblyFrameLowering::needsSPWriteback(
   return needsSPForLocalFrame(MF) && !CanUseRedZone;
 }
 
+unsigned WebAssemblyFrameLowering::getSPReg(const MachineFunction &MF) {
+  return MF.getSubtarget<WebAssemblySubtarget>().hasAddr64()
+             ? WebAssembly::SP64
+             : WebAssembly::SP32;
+}
+
+unsigned WebAssemblyFrameLowering::getFPReg(const MachineFunction &MF) {
+  return MF.getSubtarget<WebAssemblySubtarget>().hasAddr64()
+             ? WebAssembly::FP64
+             : WebAssembly::FP32;
+}
+
+unsigned
+WebAssemblyFrameLowering::getOpcConst(const MachineFunction &MF) {
+  return MF.getSubtarget<WebAssemblySubtarget>().hasAddr64()
+             ? WebAssembly::CONST_I64
+             : WebAssembly::CONST_I32;
+}
+
+unsigned WebAssemblyFrameLowering::getOpcAdd(const MachineFunction &MF) {
+  return MF.getSubtarget<WebAssemblySubtarget>().hasAddr64()
+             ? WebAssembly::ADD_I64
+             : WebAssembly::ADD_I32;
+}
+
+unsigned WebAssemblyFrameLowering::getOpcSub(const MachineFunction &MF) {
+  return MF.getSubtarget<WebAssemblySubtarget>().hasAddr64()
+             ? WebAssembly::SUB_I64
+             : WebAssembly::SUB_I32;
+}
+
+unsigned WebAssemblyFrameLowering::getOpcAnd(const MachineFunction &MF) {
+  return MF.getSubtarget<WebAssemblySubtarget>().hasAddr64()
+             ? WebAssembly::AND_I64
+             : WebAssembly::AND_I32;
+}
+
+unsigned
+WebAssemblyFrameLowering::getOpcGlobGet(const MachineFunction &MF) {
+  return MF.getSubtarget<WebAssemblySubtarget>().hasAddr64()
+             ? WebAssembly::GLOBAL_GET_I64
+             : WebAssembly::GLOBAL_GET_I32;
+}
+
+unsigned
+WebAssemblyFrameLowering::getOpcGlobSet(const MachineFunction &MF) {
+  return MF.getSubtarget<WebAssemblySubtarget>().hasAddr64()
+             ? WebAssembly::GLOBAL_SET_I64
+             : WebAssembly::GLOBAL_SET_I32;
+}
+
 void WebAssemblyFrameLowering::writeSPToGlobal(
     unsigned SrcReg, MachineFunction &MF, MachineBasicBlock &MBB,
     MachineBasicBlock::iterator &InsertStore, const DebugLoc &DL) const {
@@ -130,7 +181,8 @@ void WebAssemblyFrameLowering::writeSPToGlobal(
 
   const char *ES = "__stack_pointer";
   auto *SPSymbol = MF.createExternalSymbolName(ES);
-  BuildMI(MBB, InsertStore, DL, TII->get(WebAssembly::GLOBAL_SET_I32))
+
+  BuildMI(MBB, InsertStore, DL, TII->get(getOpcGlobSet(MF)))
       .addExternalSymbol(SPSymbol)
       .addReg(SrcReg);
 }
@@ -141,11 +193,12 @@ WebAssemblyFrameLowering::eliminateCallFramePseudoInstr(
     MachineBasicBlock::iterator I) const {
   assert(!I->getOperand(0).getImm() && (hasFP(MF) || hasBP(MF)) &&
          "Call frame pseudos should only be used for dynamic stack adjustment");
-  const auto *TII = MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
+  auto &ST = MF.getSubtarget<WebAssemblySubtarget>();
+  const auto *TII = ST.getInstrInfo();
   if (I->getOpcode() == TII->getCallFrameDestroyOpcode() &&
       needsSPWriteback(MF)) {
     DebugLoc DL = I->getDebugLoc();
-    writeSPToGlobal(WebAssembly::SP32, MF, MBB, I, DL);
+    writeSPToGlobal(getSPReg(MF), MF, MBB, I, DL);
   }
   return MBB.erase(I);
 }
@@ -161,7 +214,8 @@ void WebAssemblyFrameLowering::emitPrologue(MachineFunction &MF,
     return;
   uint64_t StackSize = MFI.getStackSize();
 
-  const auto *TII = MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
+  auto &ST = MF.getSubtarget<WebAssemblySubtarget>();
+  const auto *TII = ST.getInstrInfo();
   auto &MRI = MF.getRegInfo();
 
   auto InsertPt = MBB.begin();
@@ -172,13 +226,13 @@ void WebAssemblyFrameLowering::emitPrologue(MachineFunction &MF,
 
   const TargetRegisterClass *PtrRC =
       MRI.getTargetRegisterInfo()->getPointerRegClass(MF);
-  unsigned SPReg = WebAssembly::SP32;
+  unsigned SPReg = getSPReg(MF);
   if (StackSize)
     SPReg = MRI.createVirtualRegister(PtrRC);
 
   const char *ES = "__stack_pointer";
   auto *SPSymbol = MF.createExternalSymbolName(ES);
-  BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::GLOBAL_GET_I32), SPReg)
+  BuildMI(MBB, InsertPt, DL, TII->get(getOpcGlobGet(MF)), SPReg)
       .addExternalSymbol(SPSymbol);
 
   bool HasBP = hasBP(MF);
@@ -192,32 +246,30 @@ void WebAssemblyFrameLowering::emitPrologue(MachineFunction &MF,
   if (StackSize) {
     // Subtract the frame size
     Register OffsetReg = MRI.createVirtualRegister(PtrRC);
-    BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::CONST_I32), OffsetReg)
+    BuildMI(MBB, InsertPt, DL, TII->get(getOpcConst(MF)), OffsetReg)
         .addImm(StackSize);
-    BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::SUB_I32),
-            WebAssembly::SP32)
+    BuildMI(MBB, InsertPt, DL, TII->get(getOpcSub(MF)), getSPReg(MF))
         .addReg(SPReg)
         .addReg(OffsetReg);
   }
   if (HasBP) {
     Register BitmaskReg = MRI.createVirtualRegister(PtrRC);
     Align Alignment = MFI.getMaxAlign();
-    BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::CONST_I32), BitmaskReg)
-        .addImm((int)~(Alignment.value() - 1));
-    BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::AND_I32),
-            WebAssembly::SP32)
-        .addReg(WebAssembly::SP32)
+    BuildMI(MBB, InsertPt, DL, TII->get(getOpcConst(MF)), BitmaskReg)
+        .addImm((int64_t) ~(Alignment.value() - 1));
+    BuildMI(MBB, InsertPt, DL, TII->get(getOpcAnd(MF)), getSPReg(MF))
+        .addReg(getSPReg(MF))
         .addReg(BitmaskReg);
   }
   if (hasFP(MF)) {
     // Unlike most conventional targets (where FP points to the saved FP),
     // FP points to the bottom of the fixed-size locals, so we can use positive
     // offsets in load/store instructions.
-    BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::COPY), WebAssembly::FP32)
-        .addReg(WebAssembly::SP32);
+    BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::COPY), getFPReg(MF))
+        .addReg(getSPReg(MF));
   }
   if (StackSize && needsSPWriteback(MF)) {
-    writeSPToGlobal(WebAssembly::SP32, MF, MBB, InsertPt, DL);
+    writeSPToGlobal(getSPReg(MF), MF, MBB, InsertPt, DL);
   }
 }
 
@@ -226,7 +278,8 @@ void WebAssemblyFrameLowering::emitEpilogue(MachineFunction &MF,
   uint64_t StackSize = MF.getFrameInfo().getStackSize();
   if (!needsSP(MF) || !needsSPWriteback(MF))
     return;
-  const auto *TII = MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
+  auto &ST = MF.getSubtarget<WebAssemblySubtarget>();
+  const auto *TII = ST.getInstrInfo();
   auto &MRI = MF.getRegInfo();
   auto InsertPt = MBB.getFirstTerminator();
   DebugLoc DL;
@@ -237,6 +290,7 @@ void WebAssemblyFrameLowering::emitEpilogue(MachineFunction &MF,
   // Restore the stack pointer. If we had fixed-size locals, add the offset
   // subtracted in the prolog.
   unsigned SPReg = 0;
+  unsigned SPFPReg = hasFP(MF) ? getFPReg(MF) : getSPReg(MF);
   if (hasBP(MF)) {
     auto FI = MF.getInfo<WebAssemblyFunctionInfo>();
     SPReg = FI->getBasePointerVreg();
@@ -244,16 +298,17 @@ void WebAssemblyFrameLowering::emitEpilogue(MachineFunction &MF,
     const TargetRegisterClass *PtrRC =
         MRI.getTargetRegisterInfo()->getPointerRegClass(MF);
     Register OffsetReg = MRI.createVirtualRegister(PtrRC);
-    BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::CONST_I32), OffsetReg)
+    BuildMI(MBB, InsertPt, DL, TII->get(getOpcConst(MF)), OffsetReg)
         .addImm(StackSize);
-    // In the epilog we don't need to write the result back to the SP32 physreg
-    // because it won't be used again. We can use a stackified register instead.
+    // In the epilog we don't need to write the result back to the SP32/64
+    // physreg because it won't be used again. We can use a stackified register
+    // instead.
     SPReg = MRI.createVirtualRegister(PtrRC);
-    BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::ADD_I32), SPReg)
-        .addReg(hasFP(MF) ? WebAssembly::FP32 : WebAssembly::SP32)
+    BuildMI(MBB, InsertPt, DL, TII->get(getOpcAdd(MF)), SPReg)
+        .addReg(SPFPReg)
         .addReg(OffsetReg);
   } else {
-    SPReg = hasFP(MF) ? WebAssembly::FP32 : WebAssembly::SP32;
+    SPReg = SPFPReg;
   }
 
   writeSPToGlobal(SPReg, MF, MBB, InsertPt, DL);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyFrameLowering.h b/llvm/lib/Target/WebAssembly/WebAssemblyFrameLowering.h
index cf57247b38781..e16f639ff22b0 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyFrameLowering.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyFrameLowering.h
@@ -53,6 +53,15 @@ class WebAssemblyFrameLowering final : public TargetFrameLowering {
                        MachineBasicBlock::iterator &InsertStore,
                        const DebugLoc &DL) const;
 
+  static unsigned getSPReg(const MachineFunction &MF);
+  static unsigned getFPReg(const MachineFunction &MF);
+  static unsigned getOpcConst(const MachineFunction &MF);
+  static unsigned getOpcAdd(const MachineFunction &MF);
+  static unsigned getOpcSub(const MachineFunction &MF);
+  static unsigned getOpcAnd(const MachineFunction &MF);
+  static unsigned getOpcGlobGet(const MachineFunction &MF);
+  static unsigned getOpcGlobSet(const MachineFunction &MF);
+
 private:
   bool hasBP(const MachineFunction &MF) const;
   bool needsSPForLocalFrame(const MachineFunction &MF) const;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
index 510e7e80cc1a1..d1a696f854f8b 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
@@ -77,6 +77,13 @@ void WebAssemblyDAGToDAGISel::Select(SDNode *Node) {
     return;
   }
 
+  MVT PtrVT = TLI->getPointerTy(CurDAG->getDataLayout());
+  auto GlobalGetIns = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64
+                                        : WebAssembly::GLOBAL_GET_I32;
+  auto ConstIns =
+      PtrVT == MVT::i64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32;
+  auto AddIns = PtrVT == MVT::i64 ? WebAssembly::ADD_I64 : WebAssembly::ADD_I32;
+
   // Few custom selection stuff.
   SDLoc DL(Node);
   MachineFunction &MF = CurDAG->getMachineFunction();
@@ -140,20 +147,16 @@ void WebAssemblyDAGToDAGISel::Select(SDNode *Node) {
                          false);
     }
 
-    MVT PtrVT = TLI->getPointerTy(CurDAG->getDataLayout());
-    assert(PtrVT == MVT::i32 && "only wasm32 is supported for now");
-
     SDValue TLSBaseSym = CurDAG->getTargetExternalSymbol("__tls_base", PtrVT);
     SDValue TLSOffsetSym = CurDAG->getTargetGlobalAddress(
         GA->getGlobal(), DL, PtrVT, GA->getOffset(), 0);
 
-    MachineSDNode *TLSBase = CurDAG->getMachineNode(WebAssembly::GLOBAL_GET_I32,
-                                                    DL, MVT::i32, TLSBaseSym);
-    MachineSDNode *TLSOffset = CurDAG->getMachineNode(
-        WebAssembly::CONST_I32, DL, MVT::i32, TLSOffsetSym);
-    MachineSDNode *TLSAddress =
-        CurDAG->getMachineNode(WebAssembly::ADD_I32, DL, MVT::i32,
-                               SDValue(TLSBase, 0), SDValue(TLSOffset, 0));
+    MachineSDNode *TLSBase =
+        CurDAG->getMachineNode(GlobalGetIns, DL, PtrVT, TLSBaseSym);
+    MachineSDNode *TLSOffset =
+        CurDAG->getMachineNode(ConstIns, DL, PtrVT, TLSOffsetSym);
+    MachineSDNode *TLSAddress = CurDAG->getMachineNode(
+        AddIns, DL, PtrVT, SDValue(TLSBase, 0), SDValue(TLSOffset, 0));
     ReplaceNode(Node, TLSAddress);
     return;
   }
@@ -162,22 +165,16 @@ void WebAssemblyDAGToDAGISel::Select(SDNode *Node) {
     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(0))->getZExtValue();
     switch (IntNo) {
     case Intrinsic::wasm_tls_size: {
-      MVT PtrVT = TLI->getPointerTy(CurDAG->getDataLayout());
-      assert(PtrVT == MVT::i32 && "only wasm32 is supported for now");
-
       MachineSDNode *TLSSize = CurDAG->getMachineNode(
-          WebAssembly::GLOBAL_GET_I32, DL, PtrVT,
-          CurDAG->getTargetExternalSymbol("__tls_size", MVT::i32));
+          GlobalGetIns, DL, PtrVT,
+          CurDAG->getTargetExternalSymbol("__tls_size", PtrVT));
       ReplaceNode(Node, TLSSize);
       return;
     }
     case Intrinsic::wasm_tls_align: {
-      MVT PtrVT = TLI->getPointerTy(CurDAG->getDataLayout());
-      assert(PtrVT == MVT::i32 && "only wasm32 is supported for now");
-
       MachineSDNode *TLSAlign = CurDAG->getMachineNode(
-          WebAssembly::GLOBAL_GET_I32, DL, PtrVT,
-          CurDAG->getTargetExternalSymbol("__tls_align", MVT::i32));
+          GlobalGetIns, DL, PtrVT,
+          CurDAG->getTargetExternalSymbol("__tls_align", PtrVT));
       ReplaceNode(Node, TLSAlign);
       return;
     }
@@ -188,11 +185,8 @@ void WebAssemblyDAGToDAGISel::Select(SDNode *Node) {
     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
     switch (IntNo) {
     case Intrinsic::wasm_tls_base: {
-      MVT PtrVT = TLI->getPointerTy(CurDAG->getDataLayout());
-      assert(PtrVT == MVT::i32 && "only wasm32 is supported for now");
-
       MachineSDNode *TLSBase = CurDAG->getMachineNode(
-          WebAssembly::GLOBAL_GET_I32, DL, MVT::i32, MVT::Other,
+          GlobalGetIns, DL, PtrVT, MVT::Other,
           CurDAG->getTargetExternalSymbol("__tls_base", PtrVT),
           Node->getOperand(0));
       ReplaceNode(Node, TLSBase);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index 4c71137d56748..a9b9eceb41304 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -209,6 +209,7 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
   setOperationAction(ISD::DYNAMIC_STACKALLOC, MVTPtr, Expand);
 
   setOperationAction(ISD::FrameIndex, MVT::i32, Custom);
+  setOperationAction(ISD::FrameIndex, MVT::i64, Custom);
   setOperationAction(ISD::CopyToReg, MVT::Other, Custom);
 
   // Expand these forms; we pattern-match the forms that we can handle in isel.
@@ -613,7 +614,11 @@ EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
   if (VT.isVector())
     return VT.changeVectorElementTypeToInteger();
 
-  return TargetLowering::getSetCCResultType(DL, C, VT);
+  // So far, all branch instructions in Wasm take an I32 condition.
+  // The default TargetLowering::getSetCCResultType returns the pointer size,
+  // which would be useful to reduce instruction counts when testing
+  // against 64-bit pointers/values if at some point Wasm supports that.
+  return EVT::getIntegerVT(C, 32);
 }
 
 bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
@@ -841,7 +846,7 @@ WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
       assert(VT != MVT::iPTR && "Legalized args should be concrete");
       Type *Ty = VT.getTypeForEVT(*DAG.getContext());
       Align Alignment =
-          std::max(Align(Out.Flags.getOrigAlign()), Layout.getABITypeAlign(Ty));
+          std::max(Out.Flags.getNonZeroOrigAlign(), Layout.getABITypeAlign(Ty));
       unsigned Offset =
           CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty), Alignment);
       CCInfo.addLoc(CCValAssign::getMem(ArgLocs.size(), VT.getSimpleVT(),
@@ -915,7 +920,7 @@ WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
     if (In.Flags.isInConsecutiveRegsLast())
       fail(DL, DAG,
            "WebAssembly hasn't implemented cons regs last return values");
-    // Ignore In.getOrigAlign() because all our arguments are passed in
+    // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
     // registers.
     InTys.push_back(In.VT);
   }
@@ -1002,7 +1007,7 @@ SDValue WebAssemblyTargetLowering::LowerFormalArguments(
       fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
     if (In.Flags.isInConsecutiveRegsLast())
       fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
-    // Ignore In.getOrigAlign() because all our arguments are passed in
+    // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
     // registers.
     InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT,
                                            DAG.getTargetConstant(InVals.size(),
@@ -1280,8 +1285,10 @@ SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
   for (auto MBB : MBBs)
     Ops.push_back(DAG.getBasicBlock(MBB));
 
-  // Do not add the default case for now. It will be added in
-  // WebAssemblyFixBrTableDefaults.
+  // Add the first MBB as a dummy default target for now. This will be replaced
+  // with the proper default target (and the preceding range check eliminated)
+  // if possible by WebAssemblyFixBrTableDefaults.
+  Ops.push_back(DAG.getBasicBlock(*MBBs.begin()));
   return DAG.getNode(WebAssemblyISD::BR_TABLE, DL, MVT::Other, Ops);
 }
 
@@ -1670,19 +1677,13 @@ SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
   // Only manually lower vector shifts
   assert(Op.getSimpleValueType().isVector());
 
-  // Unroll non-splat vector shifts
-  BuildVectorSDNode *ShiftVec;
-  SDValue SplatVal;
-  if (!(ShiftVec = dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode())) ||
-      !(SplatVal = ShiftVec->getSplatValue()))
+  auto ShiftVal = DAG.getSplatValue(Op.getOperand(1));
+  if (!ShiftVal)
     return unrollVectorShift(Op, DAG);
 
-  // All splats except i64x2 const splats are handled by patterns
-  auto *SplatConst = dyn_cast<ConstantSDNode>(SplatVal);
-  if (!SplatConst || Op.getSimpleValueType() != MVT::v2i64)
-    return Op;
+  // Use anyext because none of the high bits can affect the shift
+  ShiftVal = DAG.getAnyExtOrTrunc(ShiftVal, DL, MVT::i32);
 
-  // i64x2 const splats are custom lowered to avoid unnecessary wraps
   unsigned Opcode;
   switch (Op.getOpcode()) {
   case ISD::SHL:
@@ -1697,9 +1698,8 @@ SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
   default:
     llvm_unreachable("unexpected opcode");
   }
-  APInt Shift = SplatConst->getAPIntValue().zextOrTrunc(32);
-  return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0),
-                     DAG.getConstant(Shift, DL, MVT::i32));
+
+  return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0), ShiftVal);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td
index 05735cf6d31f7..3e9ef6fbc7eaa 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td
@@ -33,39 +33,43 @@ def wasm_memset_t : SDTypeProfile<0, 4,
 def wasm_memset : SDNode<"WebAssemblyISD::MEMORY_FILL", wasm_memset_t,
                          [SDNPHasChain, SDNPMayStore]>;
 
+multiclass BulkMemoryOps<WebAssemblyRegClass rc, string B> {
+
 let mayStore = 1, hasSideEffects = 1 in
-defm MEMORY_INIT :
+defm MEMORY_INIT_A#B :
   BULK_I<(outs),
-         (ins i32imm_op:$seg, i32imm_op:$idx, I32:$dest,
-              I32:$offset, I32:$size),
+         (ins i32imm_op:$seg, i32imm_op:$idx, rc:$dest,
+              rc:$offset, rc:$size),
          (outs), (ins i32imm_op:$seg, i32imm_op:$idx),
-         [(int_wasm_memory_init (i32 timm:$seg), (i32 timm:$idx), I32:$dest,
-            I32:$offset, I32:$size
-          )],
+         [],
          "memory.init\t$seg, $idx, $dest, $offset, $size",
          "memory.init\t$seg, $idx", 0x08>;
 
 let hasSideEffects = 1 in
 defm DATA_DROP :
   BULK_I<(outs), (ins i32imm_op:$seg), (outs), (ins i32imm_op:$seg),
-         [(int_wasm_data_drop (i32 timm:$seg))],
+         [],
          "data.drop\t$seg", "data.drop\t$seg", 0x09>;
 
 let mayLoad = 1, mayStore = 1 in
-defm MEMORY_COPY :
+defm MEMORY_COPY_A#B :
   BULK_I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx,
-                      I32:$dst, I32:$src, I32:$len),
+                      rc:$dst, rc:$src, rc:$len),
          (outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx),
          [(wasm_memcpy (i32 imm:$src_idx), (i32 imm:$dst_idx),
-           I32:$dst, I32:$src, I32:$len
+           rc:$dst, rc:$src, rc:$len
          )],
          "memory.copy\t$src_idx, $dst_idx, $dst, $src, $len",
          "memory.copy\t$src_idx, $dst_idx", 0x0a>;
 
 let mayStore = 1 in
-defm MEMORY_FILL :
-  BULK_I<(outs), (ins i32imm_op:$idx, I32:$dst, I32:$value, I32:$size),
+defm MEMORY_FILL_A#B :
+  BULK_I<(outs), (ins i32imm_op:$idx, rc:$dst, I32:$value, rc:$size),
          (outs), (ins i32imm_op:$idx),
-         [(wasm_memset (i32 imm:$idx), I32:$dst, I32:$value, I32:$size)],
+         [(wasm_memset (i32 imm:$idx), rc:$dst, I32:$value, rc:$size)],
          "memory.fill\t$idx, $dst, $value, $size",
          "memory.fill\t$idx", 0x0b>;
+}
+
+defm : BulkMemoryOps<I32, "32">;
+defm : BulkMemoryOps<I64, "64">;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
index 686c017593c8f..b3c63cc1f884b 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
@@ -365,19 +365,24 @@ defm : StorePatGlobalAddrOffOnly<i64, truncstorei8, "STORE8_I64">;
 defm : StorePatGlobalAddrOffOnly<i64, truncstorei16, "STORE16_I64">;
 defm : StorePatGlobalAddrOffOnly<i64, truncstorei32, "STORE32_I64">;
 
+multiclass MemoryOps<WebAssemblyRegClass rc, string B> {
 // Current memory size.
-defm MEMORY_SIZE_I32 : I<(outs I32:$dst), (ins i32imm:$flags),
+defm MEMORY_SIZE_A#B : I<(outs rc:$dst), (ins i32imm:$flags),
                          (outs), (ins i32imm:$flags),
-                         [(set I32:$dst,
+                         [(set rc:$dst,
                            (int_wasm_memory_size (i32 imm:$flags)))],
                          "memory.size\t$dst, $flags", "memory.size\t$flags",
                          0x3f>;
 
 // Grow memory.
-defm MEMORY_GROW_I32 : I<(outs I32:$dst), (ins i32imm:$flags, I32:$delta),
+defm MEMORY_GROW_A#B : I<(outs rc:$dst), (ins i32imm:$flags, rc:$delta),
                          (outs), (ins i32imm:$flags),
-                         [(set I32:$dst,
+                         [(set rc:$dst,
                            (int_wasm_memory_grow (i32 imm:$flags),
-                             I32:$delta))],
+                             rc:$delta))],
                          "memory.grow\t$dst, $flags, $delta",
                          "memory.grow\t$flags", 0x40>;
+}
+
+defm : MemoryOps<I32, "32">;
+defm : MemoryOps<I64, "64">;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
index b4a8a7bc42ae8..4f3da2f35c61d 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
@@ -328,8 +328,6 @@ def splat16 : PatFrag<(ops node:$x), (build_vector
 
 multiclass Splat<ValueType vec_t, string vec, WebAssemblyRegClass reg_t,
                  PatFrag splat_pat, bits<32> simdop> {
-  // Prefer splats over v128.const for const splats (65 is lowest that works)
-  let AddedComplexity = 65 in
   defm SPLAT_#vec_t : SIMD_I<(outs V128:$dst), (ins reg_t:$x), (outs), (ins),
                              [(set (vec_t V128:$dst), (splat_pat reg_t:$x))],
                              vec#".splat\t$dst, $x", vec#".splat", simdop>;
@@ -654,55 +652,36 @@ defm BITMASK : SIMDBitmask<v4i32, "i32x4", 164>;
 // Bit shifts
 //===----------------------------------------------------------------------===//
 
-multiclass SIMDShift<ValueType vec_t, string vec, SDNode node, dag shift_vec,
-                     string name, bits<32> simdop> {
+multiclass SIMDShift<ValueType vec_t, string vec, SDNode node, string name,
+                     bits<32> simdop> {
   defm _#vec_t : SIMD_I<(outs V128:$dst), (ins V128:$vec, I32:$x),
                         (outs), (ins),
-                        [(set (vec_t V128:$dst),
-                          (node V128:$vec, (vec_t shift_vec)))],
+                        [(set (vec_t V128:$dst), (node V128:$vec, I32:$x))],
                         vec#"."#name#"\t$dst, $vec, $x", vec#"."#name, simdop>;
 }
 
 multiclass SIMDShiftInt<SDNode node, string name, bits<32> baseInst> {
-  defm "" : SIMDShift<v16i8, "i8x16", node, (splat16 I32:$x), name, baseInst>;
-  defm "" : SIMDShift<v8i16, "i16x8", node, (splat8 I32:$x), name,
-                      !add(baseInst, 32)>;
-  defm "" : SIMDShift<v4i32, "i32x4", node, (splat4 I32:$x), name,
-                      !add(baseInst, 64)>;
-  defm "" : SIMDShift<v2i64, "i64x2", node, (splat2 (i64 (zext I32:$x))),
-                      name, !add(baseInst, 96)>;
+  defm "" : SIMDShift<v16i8, "i8x16", node, name, baseInst>;
+  defm "" : SIMDShift<v8i16, "i16x8", node, name, !add(baseInst, 32)>;
+  defm "" : SIMDShift<v4i32, "i32x4", node, name, !add(baseInst, 64)>;
+  defm "" : SIMDShift<v2i64, "i64x2", node, name, !add(baseInst, 96)>;
 }
 
-// Left shift by scalar: shl
-defm SHL : SIMDShiftInt<shl, "shl", 107>;
-
-// Right shift by scalar: shr_s / shr_u
-defm SHR_S : SIMDShiftInt<sra, "shr_s", 108>;
-defm SHR_U : SIMDShiftInt<srl, "shr_u", 109>;
-
-// Truncate i64 shift operands to i32s, except if they are already i32s
-foreach shifts = [[shl, SHL_v2i64], [sra, SHR_S_v2i64], [srl, SHR_U_v2i64]] in {
-def : Pat<(v2i64 (shifts[0]
-            (v2i64 V128:$vec),
-            (v2i64 (splat2 (i64 (sext I32:$x))))
-          )),
-          (v2i64 (shifts[1] (v2i64 V128:$vec), (i32 I32:$x)))>;
-def : Pat<(v2i64 (shifts[0] (v2i64 V128:$vec), (v2i64 (splat2 I64:$x)))),
-          (v2i64 (shifts[1] (v2i64 V128:$vec), (I32_WRAP_I64 I64:$x)))>;
-}
-
-// 2xi64 shifts with constant shift amounts are custom lowered to avoid wrapping
+// WebAssembly SIMD shifts are nonstandard in that the shift amount is
+// an i32 rather than a vector, so they need custom nodes.
 def wasm_shift_t : SDTypeProfile<1, 2,
   [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisVT<2, i32>]
 >;
 def wasm_shl : SDNode<"WebAssemblyISD::VEC_SHL", wasm_shift_t>;
 def wasm_shr_s : SDNode<"WebAssemblyISD::VEC_SHR_S", wasm_shift_t>;
 def wasm_shr_u : SDNode<"WebAssemblyISD::VEC_SHR_U", wasm_shift_t>;
-foreach shifts = [[wasm_shl, SHL_v2i64],
-                  [wasm_shr_s, SHR_S_v2i64],
-                  [wasm_shr_u, SHR_U_v2i64]] in
-def : Pat<(v2i64 (shifts[0] (v2i64 V128:$vec), I32:$x)),
-          (v2i64 (shifts[1] (v2i64 V128:$vec), I32:$x))>;
+
+// Left shift by scalar: shl
+defm SHL : SIMDShiftInt<wasm_shl, "shl", 107>;
+
+// Right shift by scalar: shr_s / shr_u
+defm SHR_S : SIMDShiftInt<wasm_shr_s, "shr_s", 108>;
+defm SHR_U : SIMDShiftInt<wasm_shr_u, "shr_u", 109>;
 
 //===----------------------------------------------------------------------===//
 // Integer binary arithmetic
@@ -766,12 +745,12 @@ def add_nuw : PatFrag<(ops node:$lhs, node:$rhs),
                       "return N->getFlags().hasNoUnsignedWrap();">;
 
 foreach nodes = [[v16i8, splat16], [v8i16, splat8]] in
-def : Pat<(srl
+def : Pat<(wasm_shr_u
             (add_nuw
               (add_nuw (nodes[0] V128:$lhs), (nodes[0] V128:$rhs)),
               (nodes[1] (i32 1))
             ),
-            (nodes[0] (nodes[1] (i32 1)))
+            (i32 1)
           ),
           (!cast<NI>("AVGR_U_"#nodes[0]) V128:$lhs, V128:$rhs)>;
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp
index 2280ec38e5b90..346938daf1aa2 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp
@@ -408,8 +408,8 @@ bool WebAssemblyLateEHPrepare::restoreStackPointer(MachineFunction &MF) {
       ++InsertPos;
     if (InsertPos->getOpcode() == WebAssembly::CATCH)
       ++InsertPos;
-    FrameLowering->writeSPToGlobal(WebAssembly::SP32, MF, MBB, InsertPos,
-                                   MBB.begin()->getDebugLoc());
+    FrameLowering->writeSPToGlobal(FrameLowering->getSPReg(MF), MF, MBB,
+                                   InsertPos, MBB.begin()->getDebugLoc());
   }
   return Changed;
 }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyLowerBrUnless.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyLowerBrUnless.cpp
index 4314aa6115492..01b3aa887738e 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyLowerBrUnless.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyLowerBrUnless.cpp
@@ -191,7 +191,7 @@ bool WebAssemblyLowerBrUnless::runOnMachineFunction(MachineFunction &MF) {
         Register Tmp = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
         BuildMI(MBB, MI, MI->getDebugLoc(), TII.get(WebAssembly::EQZ_I32), Tmp)
             .addReg(Cond);
-        MFI.stackifyVReg(Tmp);
+        MFI.stackifyVReg(MRI, Tmp);
         Cond = Tmp;
         Inverted = true;
       }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
index 07830c130d7aa..304dca2ebfe4a 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
@@ -81,8 +81,9 @@ MCSymbol *WebAssemblyMCInstLower::GetExternalSymbolSymbol(
         strcmp(Name, "__stack_pointer") == 0 || strcmp(Name, "__tls_base") == 0;
     WasmSym->setType(wasm::WASM_SYMBOL_TYPE_GLOBAL);
     WasmSym->setGlobalType(wasm::WasmGlobalType{
-        uint8_t(Subtarget.hasAddr64() ? wasm::WASM_TYPE_I64
-                                      : wasm::WASM_TYPE_I32),
+        uint8_t(Subtarget.hasAddr64() && strcmp(Name, "__table_base") != 0
+                    ? wasm::WASM_TYPE_I64
+                    : wasm::WASM_TYPE_I32),
         Mutable});
     return WasmSym;
   }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.cpp
index e05ff6ab8719a..adee2f0553f97 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.cpp
@@ -21,10 +21,10 @@ using namespace llvm;
 
 WebAssemblyFunctionInfo::~WebAssemblyFunctionInfo() = default; // anchor.
 
-void WebAssemblyFunctionInfo::initWARegs() {
+void WebAssemblyFunctionInfo::initWARegs(MachineRegisterInfo &MRI) {
   assert(WARegs.empty());
   unsigned Reg = UnusedReg;
-  WARegs.resize(MF.getRegInfo().getNumVirtRegs(), Reg);
+  WARegs.resize(MRI.getNumVirtRegs(), Reg);
 }
 
 void llvm::computeLegalValueVTs(const Function &F, const TargetMachine &TM,
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h
index 7d2279df5b283..ca164fdd182cd 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h
@@ -30,8 +30,6 @@ struct WebAssemblyFunctionInfo;
 /// This class is derived from MachineFunctionInfo and contains private
 /// WebAssembly-specific information for each MachineFunction.
 class WebAssemblyFunctionInfo final : public MachineFunctionInfo {
-  MachineFunction &MF;
-
   std::vector<MVT> Params;
   std::vector<MVT> Results;
   std::vector<MVT> Locals;
@@ -66,7 +64,7 @@ class WebAssemblyFunctionInfo final : public MachineFunctionInfo {
   bool CFGStackified = false;
 
 public:
-  explicit WebAssemblyFunctionInfo(MachineFunction &MF) : MF(MF) {}
+  explicit WebAssemblyFunctionInfo(MachineFunction &MF) {}
   ~WebAssemblyFunctionInfo() override;
   void initializeBaseYamlFields(const yaml::WebAssemblyFunctionInfo &YamlMFI);
 
@@ -113,8 +111,8 @@ class WebAssemblyFunctionInfo final : public MachineFunctionInfo {
 
   static const unsigned UnusedReg = -1u;
 
-  void stackifyVReg(unsigned VReg) {
-    assert(MF.getRegInfo().getUniqueVRegDef(VReg));
+  void stackifyVReg(MachineRegisterInfo &MRI, unsigned VReg) {
+    assert(MRI.getUniqueVRegDef(VReg));
     auto I = Register::virtReg2Index(VReg);
     if (I >= VRegStackified.size())
       VRegStackified.resize(I + 1);
@@ -132,7 +130,7 @@ class WebAssemblyFunctionInfo final : public MachineFunctionInfo {
     return VRegStackified.test(I);
   }
 
-  void initWARegs();
+  void initWARegs(MachineRegisterInfo &MRI);
   void setWAReg(unsigned VReg, unsigned WAReg) {
     assert(WAReg != UnusedReg);
     auto I = Register::virtReg2Index(VReg);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyPeephole.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyPeephole.cpp
index 2b369857f9d93..a587c9d23d2b7 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyPeephole.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyPeephole.cpp
@@ -66,7 +66,7 @@ static bool maybeRewriteToDrop(unsigned OldReg, unsigned NewReg,
     Register NewReg = MRI.createVirtualRegister(MRI.getRegClass(OldReg));
     MO.setReg(NewReg);
     MO.setIsDead();
-    MFI.stackifyVReg(NewReg);
+    MFI.stackifyVReg(MRI, NewReg);
   }
   return Changed;
 }
@@ -121,7 +121,7 @@ static bool maybeRewriteToFallthrough(MachineInstr &MI, MachineBasicBlock &MBB,
       BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(CopyLocalOpc), NewReg)
           .addReg(Reg);
       MO.setReg(NewReg);
-      MFI.stackifyVReg(NewReg);
+      MFI.stackifyVReg(MRI, NewReg);
     }
   }
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyRegNumbering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyRegNumbering.cpp
index 72e7a7cf50425..b655014f4a90f 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyRegNumbering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyRegNumbering.cpp
@@ -66,7 +66,7 @@ bool WebAssemblyRegNumbering::runOnMachineFunction(MachineFunction &MF) {
   WebAssemblyFunctionInfo &MFI = *MF.getInfo<WebAssemblyFunctionInfo>();
   MachineRegisterInfo &MRI = MF.getRegInfo();
 
-  MFI.initWARegs();
+  MFI.initWARegs(MRI);
 
   // WebAssembly argument registers are in the same index space as local
   // variables. Assign the numbers for them first.
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp
index 82f752f63680b..1d4e2e3a8f9e5 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp
@@ -248,7 +248,8 @@ static void query(const MachineInstr &MI, AliasAnalysis &AA, bool &Read,
   }
 
   // Check for writes to __stack_pointer global.
-  if (MI.getOpcode() == WebAssembly::GLOBAL_SET_I32 &&
+  if ((MI.getOpcode() == WebAssembly::GLOBAL_SET_I32 ||
+       MI.getOpcode() == WebAssembly::GLOBAL_SET_I64) &&
       strcmp(MI.getOperand(0).getSymbolName(), "__stack_pointer") == 0)
     StackPointer = true;
 
@@ -530,7 +531,7 @@ static MachineInstr *moveForSingleUse(unsigned Reg, MachineOperand &Op,
   if (MRI.hasOneDef(Reg) && MRI.hasOneUse(Reg)) {
     // No one else is using this register for anything so we can just stackify
     // it in place.
-    MFI.stackifyVReg(Reg);
+    MFI.stackifyVReg(MRI, Reg);
   } else {
     // The register may have unrelated uses or defs; create a new register for
     // just our one def and use so that we can stackify it.
@@ -547,7 +548,7 @@ static MachineInstr *moveForSingleUse(unsigned Reg, MachineOperand &Op,
                      LIS.getInstructionIndex(*Op.getParent()).getRegSlot(),
                      /*RemoveDeadValNo=*/true);
 
-    MFI.stackifyVReg(NewReg);
+    MFI.stackifyVReg(MRI, NewReg);
 
     DefDIs.updateReg(NewReg);
 
@@ -576,7 +577,7 @@ static MachineInstr *rematerializeCheapDef(
   MachineInstr *Clone = &*std::prev(Insert);
   LIS.InsertMachineInstrInMaps(*Clone);
   LIS.createAndComputeVirtRegInterval(NewReg);
-  MFI.stackifyVReg(NewReg);
+  MFI.stackifyVReg(MRI, NewReg);
   imposeStackOrdering(Clone);
 
   LLVM_DEBUG(dbgs() << " - Cloned to "; Clone->dump());
@@ -667,8 +668,8 @@ static MachineInstr *moveAndTeeForMultiUse(
   // Finish stackifying the new regs.
   LIS.createAndComputeVirtRegInterval(TeeReg);
   LIS.createAndComputeVirtRegInterval(DefReg);
-  MFI.stackifyVReg(DefReg);
-  MFI.stackifyVReg(TeeReg);
+  MFI.stackifyVReg(MRI, DefReg);
+  MFI.stackifyVReg(MRI, TeeReg);
   imposeStackOrdering(Def);
   imposeStackOrdering(Tee);
 
@@ -934,7 +935,7 @@ bool WebAssemblyRegStackify::runOnMachineFunction(MachineFunction &MF) {
           // TODO: This single-use restriction could be relaxed by using tees
           if (DefReg != UseReg || !MRI.hasOneUse(DefReg))
             break;
-          MFI.stackifyVReg(DefReg);
+          MFI.stackifyVReg(MRI, DefReg);
           ++SubsequentDef;
           ++SubsequentUse;
         }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp
index 205fb6e834e9c..130589c9df8c5 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp
@@ -88,16 +88,17 @@ void WebAssemblyRegisterInfo::eliminateFrameIndex(
 
   // If this is an address being added to a constant, fold the frame offset
   // into the constant.
-  if (MI.getOpcode() == WebAssembly::ADD_I32) {
+  if (MI.getOpcode() == WebAssemblyFrameLowering::getOpcAdd(MF)) {
     MachineOperand &OtherMO = MI.getOperand(3 - FIOperandNum);
     if (OtherMO.isReg()) {
       Register OtherMOReg = OtherMO.getReg();
       if (Register::isVirtualRegister(OtherMOReg)) {
         MachineInstr *Def = MF.getRegInfo().getUniqueVRegDef(OtherMOReg);
         // TODO: For now we just opportunistically do this in the case where
-        // the CONST_I32 happens to have exactly one def and one use. We
+        // the CONST_I32/64 happens to have exactly one def and one use. We
         // should generalize this to optimize in more cases.
-        if (Def && Def->getOpcode() == WebAssembly::CONST_I32 &&
+        if (Def && Def->getOpcode() ==
+              WebAssemblyFrameLowering::getOpcConst(MF) &&
             MRI.hasOneNonDBGUse(Def->getOperand(0).getReg())) {
           MachineOperand &ImmMO = Def->getOperand(1);
           ImmMO.setImm(ImmMO.getImm() + uint32_t(FrameOffset));
@@ -109,20 +110,22 @@ void WebAssemblyRegisterInfo::eliminateFrameIndex(
     }
   }
 
-  // Otherwise create an i32.add SP, offset and make it the operand.
+  // Otherwise create an i32/64.add SP, offset and make it the operand.
   const auto *TII = MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
 
   unsigned FIRegOperand = FrameRegister;
   if (FrameOffset) {
-    // Create i32.add SP, offset and make it the operand.
+    // Create i32/64.add SP, offset and make it the operand.
     const TargetRegisterClass *PtrRC =
         MRI.getTargetRegisterInfo()->getPointerRegClass(MF);
     Register OffsetOp = MRI.createVirtualRegister(PtrRC);
-    BuildMI(MBB, *II, II->getDebugLoc(), TII->get(WebAssembly::CONST_I32),
+    BuildMI(MBB, *II, II->getDebugLoc(),
+            TII->get(WebAssemblyFrameLowering::getOpcConst(MF)),
             OffsetOp)
         .addImm(FrameOffset);
     FIRegOperand = MRI.createVirtualRegister(PtrRC);
-    BuildMI(MBB, *II, II->getDebugLoc(), TII->get(WebAssembly::ADD_I32),
+    BuildMI(MBB, *II, II->getDebugLoc(),
+            TII->get(WebAssemblyFrameLowering::getOpcAdd(MF)),
             FIRegOperand)
         .addReg(FrameRegister)
         .addReg(OffsetOp);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp
index 890e4b8e4e2a4..16e05150c64ef 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp
@@ -20,40 +20,40 @@ WebAssemblySelectionDAGInfo::~WebAssemblySelectionDAGInfo() = default; // anchor
 
 SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemcpy(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
-    SDValue Size, unsigned Align, bool IsVolatile, bool AlwaysInline,
+    SDValue Size, Align Alignment, bool IsVolatile, bool AlwaysInline,
     MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
-  if (!DAG.getMachineFunction()
-           .getSubtarget<WebAssemblySubtarget>()
-           .hasBulkMemory())
+  auto &ST = DAG.getMachineFunction().getSubtarget<WebAssemblySubtarget>();
+  if (!ST.hasBulkMemory())
     return SDValue();
 
   SDValue MemIdx = DAG.getConstant(0, DL, MVT::i32);
+  auto LenMVT = ST.hasAddr64() ? MVT::i64 : MVT::i32;
   return DAG.getNode(WebAssemblyISD::MEMORY_COPY, DL, MVT::Other,
                      {Chain, MemIdx, MemIdx, Dst, Src,
-                      DAG.getZExtOrTrunc(Size, DL, MVT::i32)});
+                      DAG.getZExtOrTrunc(Size, DL, LenMVT)});
 }
 
 SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemmove(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Op1, SDValue Op2,
-    SDValue Op3, unsigned Align, bool IsVolatile,
+    SDValue Op3, Align Alignment, bool IsVolatile,
     MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
-  return EmitTargetCodeForMemcpy(DAG, DL, Chain, Op1, Op2, Op3, Align,
-                                 IsVolatile, false, DstPtrInfo,
-                                 SrcPtrInfo);
+  return EmitTargetCodeForMemcpy(DAG, DL, Chain, Op1, Op2, Op3,
+                                 Alignment, IsVolatile, false,
+                                 DstPtrInfo, SrcPtrInfo);
 }
 
 SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemset(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Val,
-    SDValue Size, unsigned Align, bool IsVolatile,
+    SDValue Size, Align Alignment, bool IsVolatile,
     MachinePointerInfo DstPtrInfo) const {
-  if (!DAG.getMachineFunction()
-           .getSubtarget<WebAssemblySubtarget>()
-           .hasBulkMemory())
+  auto &ST = DAG.getMachineFunction().getSubtarget<WebAssemblySubtarget>();
+  if (!ST.hasBulkMemory())
     return SDValue();
 
   SDValue MemIdx = DAG.getConstant(0, DL, MVT::i32);
+  auto LenMVT = ST.hasAddr64() ? MVT::i64 : MVT::i32;
   // Only low byte matters for val argument, so anyext the i8
   return DAG.getNode(WebAssemblyISD::MEMORY_FILL, DL, MVT::Other, Chain, MemIdx,
                      Dst, DAG.getAnyExtOrTrunc(Val, DL, MVT::i32),
-                     DAG.getZExtOrTrunc(Size, DL, MVT::i32));
+                     DAG.getZExtOrTrunc(Size, DL, LenMVT));
 }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h
index 0b90ece27dff3..f4d2132fd3af2 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h
@@ -24,18 +24,19 @@ class WebAssemblySelectionDAGInfo final : public SelectionDAGTargetInfo {
   ~WebAssemblySelectionDAGInfo() override;
   SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
                                   SDValue Chain, SDValue Op1, SDValue Op2,
-                                  SDValue Op3, unsigned Align, bool isVolatile,
+                                  SDValue Op3, Align Alignment, bool isVolatile,
                                   bool AlwaysInline,
                                   MachinePointerInfo DstPtrInfo,
                                   MachinePointerInfo SrcPtrInfo) const override;
-  SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl,
-                                   SDValue Chain, SDValue Op1, SDValue Op2,
-                                   SDValue Op3, unsigned Align, bool isVolatile,
-                                   MachinePointerInfo DstPtrInfo,
-                                   MachinePointerInfo SrcPtrInfo) const override;
+  SDValue
+  EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
+                           SDValue Op1, SDValue Op2, SDValue Op3,
+                           Align Alignment, bool isVolatile,
+                           MachinePointerInfo DstPtrInfo,
+                           MachinePointerInfo SrcPtrInfo) const override;
   SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &DL,
                                   SDValue Chain, SDValue Op1, SDValue Op2,
-                                  SDValue Op3, unsigned Align, bool IsVolatile,
+                                  SDValue Op3, Align Alignment, bool IsVolatile,
                                   MachinePointerInfo DstPtrInfo) const override;
 };
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
index 9f767d391122a..28703a2787e0d 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
@@ -63,10 +63,11 @@ unsigned WebAssemblyTTIImpl::getArithmeticInstrCost(
       // approxmation.
       if (Opd2Info != TTI::OK_UniformValue &&
           Opd2Info != TTI::OK_UniformConstantValue)
-        Cost = VTy->getNumElements() *
-               (TargetTransformInfo::TCC_Basic +
-                getArithmeticInstrCost(Opcode, VTy->getElementType(), CostKind) +
-                TargetTransformInfo::TCC_Basic);
+        Cost =
+            cast<FixedVectorType>(VTy)->getNumElements() *
+            (TargetTransformInfo::TCC_Basic +
+             getArithmeticInstrCost(Opcode, VTy->getElementType(), CostKind) +
+             TargetTransformInfo::TCC_Basic);
       break;
     }
   }
diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index a9577e67def34..0573d4eec0599 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -864,6 +864,8 @@ class X86AsmParser : public MCTargetAsmParser {
     return nullptr;
   }
 
+  bool MatchRegisterByName(unsigned &RegNo, StringRef RegName, SMLoc StartLoc,
+                           SMLoc EndLoc);
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
                      bool RestoreOnFailure);
 
@@ -1145,6 +1147,108 @@ static bool CheckBaseRegAndIndexRegAndScale(unsigned BaseReg, unsigned IndexReg,
   return checkScale(Scale, ErrMsg);
 }
 
+bool X86AsmParser::MatchRegisterByName(unsigned &RegNo, StringRef RegName,
+                                       SMLoc StartLoc, SMLoc EndLoc) {
+  // If we encounter a %, ignore it. This code handles registers with and
+  // without the prefix, unprefixed registers can occur in cfi directives.
+  RegName.consume_front("%");
+
+  RegNo = MatchRegisterName(RegName);
+
+  // If the match failed, try the register name as lowercase.
+  if (RegNo == 0)
+    RegNo = MatchRegisterName(RegName.lower());
+
+  // The "flags" and "mxcsr" registers cannot be referenced directly.
+  // Treat it as an identifier instead.
+  if (isParsingMSInlineAsm() && isParsingIntelSyntax() &&
+      (RegNo == X86::EFLAGS || RegNo == X86::MXCSR))
+    RegNo = 0;
+
+  if (!is64BitMode()) {
+    // FIXME: This should be done using Requires<Not64BitMode> and
+    // Requires<In64BitMode> so "eiz" usage in 64-bit instructions can be also
+    // checked.
+    // FIXME: Check AH, CH, DH, BH cannot be used in an instruction requiring a
+    // REX prefix.
+    if (RegNo == X86::RIZ || RegNo == X86::RIP ||
+        X86MCRegisterClasses[X86::GR64RegClassID].contains(RegNo) ||
+        X86II::isX86_64NonExtLowByteReg(RegNo) ||
+        X86II::isX86_64ExtendedReg(RegNo)) {
+      return Error(StartLoc,
+                   "register %" + RegName + " is only available in 64-bit mode",
+                   SMRange(StartLoc, EndLoc));
+    }
+  }
+
+  // If this is "db[0-15]", match it as an alias
+  // for dr[0-15].
+  if (RegNo == 0 && RegName.startswith("db")) {
+    if (RegName.size() == 3) {
+      switch (RegName[2]) {
+      case '0':
+        RegNo = X86::DR0;
+        break;
+      case '1':
+        RegNo = X86::DR1;
+        break;
+      case '2':
+        RegNo = X86::DR2;
+        break;
+      case '3':
+        RegNo = X86::DR3;
+        break;
+      case '4':
+        RegNo = X86::DR4;
+        break;
+      case '5':
+        RegNo = X86::DR5;
+        break;
+      case '6':
+        RegNo = X86::DR6;
+        break;
+      case '7':
+        RegNo = X86::DR7;
+        break;
+      case '8':
+        RegNo = X86::DR8;
+        break;
+      case '9':
+        RegNo = X86::DR9;
+        break;
+      }
+    } else if (RegName.size() == 4 && RegName[2] == '1') {
+      switch (RegName[3]) {
+      case '0':
+        RegNo = X86::DR10;
+        break;
+      case '1':
+        RegNo = X86::DR11;
+        break;
+      case '2':
+        RegNo = X86::DR12;
+        break;
+      case '3':
+        RegNo = X86::DR13;
+        break;
+      case '4':
+        RegNo = X86::DR14;
+        break;
+      case '5':
+        RegNo = X86::DR15;
+        break;
+      }
+    }
+  }
+
+  if (RegNo == 0) {
+    if (isParsingIntelSyntax())
+      return true;
+    return Error(StartLoc, "invalid register name", SMRange(StartLoc, EndLoc));
+  }
+  return false;
+}
+
 bool X86AsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
                                  SMLoc &EndLoc, bool RestoreOnFailure) {
   MCAsmParser &Parser = getParser();
@@ -1180,37 +1284,9 @@ bool X86AsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
                  SMRange(StartLoc, EndLoc));
   }
 
-  RegNo = MatchRegisterName(Tok.getString());
-
-  // If the match failed, try the register name as lowercase.
-  if (RegNo == 0)
-    RegNo = MatchRegisterName(Tok.getString().lower());
-
-  // The "flags" and "mxcsr" registers cannot be referenced directly.
-  // Treat it as an identifier instead.
-  if (isParsingMSInlineAsm() && isParsingIntelSyntax() &&
-      (RegNo == X86::EFLAGS || RegNo == X86::MXCSR))
-    RegNo = 0;
-
-  if (!is64BitMode()) {
-    // FIXME: This should be done using Requires<Not64BitMode> and
-    // Requires<In64BitMode> so "eiz" usage in 64-bit instructions can be also
-    // checked.
-    // FIXME: Check AH, CH, DH, BH cannot be used in an instruction requiring a
-    // REX prefix.
-    if (RegNo == X86::RIZ || RegNo == X86::RIP ||
-        X86MCRegisterClasses[X86::GR64RegClassID].contains(RegNo) ||
-        X86II::isX86_64NonExtLowByteReg(RegNo) ||
-        X86II::isX86_64ExtendedReg(RegNo)) {
-      StringRef RegName = Tok.getString();
-      OnFailure();
-      if (!RestoreOnFailure) {
-        Parser.Lex(); // Eat register name.
-      }
-      return Error(StartLoc,
-                   "register %" + RegName + " is only available in 64-bit mode",
-                   SMRange(StartLoc, EndLoc));
-    }
+  if (MatchRegisterByName(RegNo, Tok.getString(), StartLoc, EndLoc)) {
+    OnFailure();
+    return true;
   }
 
   // Parse "%st" as "%st(0)" and "%st(1)", which is multiple tokens.
@@ -1259,40 +1335,6 @@ bool X86AsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
 
   EndLoc = Parser.getTok().getEndLoc();
 
-  // If this is "db[0-15]", match it as an alias
-  // for dr[0-15].
-  if (RegNo == 0 && Tok.getString().startswith("db")) {
-    if (Tok.getString().size() == 3) {
-      switch (Tok.getString()[2]) {
-      case '0': RegNo = X86::DR0; break;
-      case '1': RegNo = X86::DR1; break;
-      case '2': RegNo = X86::DR2; break;
-      case '3': RegNo = X86::DR3; break;
-      case '4': RegNo = X86::DR4; break;
-      case '5': RegNo = X86::DR5; break;
-      case '6': RegNo = X86::DR6; break;
-      case '7': RegNo = X86::DR7; break;
-      case '8': RegNo = X86::DR8; break;
-      case '9': RegNo = X86::DR9; break;
-      }
-    } else if (Tok.getString().size() == 4 && Tok.getString()[2] == '1') {
-      switch (Tok.getString()[3]) {
-      case '0': RegNo = X86::DR10; break;
-      case '1': RegNo = X86::DR11; break;
-      case '2': RegNo = X86::DR12; break;
-      case '3': RegNo = X86::DR13; break;
-      case '4': RegNo = X86::DR14; break;
-      case '5': RegNo = X86::DR15; break;
-      }
-    }
-
-    if (RegNo != 0) {
-      EndLoc = Parser.getTok().getEndLoc();
-      Parser.Lex(); // Eat it.
-      return false;
-    }
-  }
-
   if (RegNo == 0) {
     OnFailure();
     if (isParsingIntelSyntax()) return true;
@@ -1509,8 +1551,9 @@ std::unique_ptr<X86Operand> X86AsmParser::CreateMemForMSInlineAsm(
   } else {
     BaseReg = BaseReg ? BaseReg : 1;
     return X86Operand::CreateMem(getPointerWidth(), SegReg, Disp, BaseReg,
-                                 IndexReg, Scale, Start, End, Size, Identifier,
-                                 Decl, FrontendSize);
+                                 IndexReg, Scale, Start, End, Size,
+                                 /*DefaultBaseReg=*/X86::RIP, Identifier, Decl,
+                                 FrontendSize);
   }
 }
 
@@ -1589,12 +1632,41 @@ bool X86AsmParser::ParseIntelExpression(IntelExprStateMachine &SM, SMLoc &End) {
       SMLoc IdentLoc = Tok.getLoc();
       StringRef Identifier = Tok.getString();
       UpdateLocLex = false;
-      // Register
+      // Register, or (MASM only) <register>.<field>
       unsigned Reg;
-      if (Tok.is(AsmToken::Identifier) && !ParseRegister(Reg, IdentLoc, End)) {
-        if (SM.onRegister(Reg, ErrMsg))
-          return Error(Tok.getLoc(), ErrMsg);
-        break;
+      if (Tok.is(AsmToken::Identifier)) {
+        if (!ParseRegister(Reg, IdentLoc, End, /*RestoreOnFailure=*/true)) {
+          if (SM.onRegister(Reg, ErrMsg))
+            return Error(IdentLoc, ErrMsg);
+          break;
+        }
+        if (Parser.isParsingMasm()) {
+          const std::pair<StringRef, StringRef> RegField =
+              Tok.getString().split('.');
+          const StringRef RegName = RegField.first, Field = RegField.second;
+          SMLoc RegEndLoc =
+              SMLoc::getFromPointer(RegName.data() + RegName.size());
+          if (!Field.empty() &&
+              !MatchRegisterByName(Reg, RegName, IdentLoc, RegEndLoc)) {
+            if (SM.onRegister(Reg, ErrMsg))
+              return Error(IdentLoc, ErrMsg);
+
+            SMLoc FieldStartLoc = SMLoc::getFromPointer(Field.data());
+            const std::pair<StringRef, StringRef> BaseMember = Field.split('.');
+            const StringRef Base = BaseMember.first, Member = BaseMember.second;
+
+            unsigned Offset;
+            if (Parser.LookUpFieldOffset(Base, Member, Offset))
+              return Error(FieldStartLoc, "unknown offset");
+            else if (SM.onPlus(ErrMsg))
+              return Error(getTok().getLoc(), ErrMsg);
+            else if (SM.onInteger(Offset, ErrMsg))
+              return Error(IdentLoc, ErrMsg);
+
+            End = consumeToken();
+            break;
+          }
+        }
       }
       // Operator synonymous ("not", "or" etc.)
       bool ParseError = false;
@@ -1606,37 +1678,39 @@ bool X86AsmParser::ParseIntelExpression(IntelExprStateMachine &SM, SMLoc &End) {
       // Symbol reference, when parsing assembly content
       InlineAsmIdentifierInfo Info;
       const MCExpr *Val;
-      if (!isParsingMSInlineAsm()) {
-        if (getParser().parsePrimaryExpr(Val, End)) {
-          return Error(Tok.getLoc(), "Unexpected identifier!");
-        } else if (SM.onIdentifierExpr(Val, Identifier, Info, false, ErrMsg)) {
-          return Error(IdentLoc, ErrMsg);
-        } else
+      if (isParsingMSInlineAsm() || Parser.isParsingMasm()) {
+        // MS Dot Operator expression
+        if (Identifier.count('.') && PrevTK == AsmToken::RBrac) {
+          if (ParseIntelDotOperator(SM, End))
+            return true;
           break;
+        }
       }
-      // MS InlineAsm operators (TYPE/LENGTH/SIZE)
-      if (unsigned OpKind = IdentifyIntelInlineAsmOperator(Identifier)) {
-        if (int64_t Val = ParseIntelInlineAsmOperator(OpKind)) {
-          if (SM.onInteger(Val, ErrMsg))
-            return Error(IdentLoc, ErrMsg);
-        } else
-          return true;
-        break;
-      }
-      // MS Dot Operator expression
-      if (Identifier.count('.') && PrevTK == AsmToken::RBrac) {
-        if (ParseIntelDotOperator(SM, End))
+      if (isParsingMSInlineAsm()) {
+        // MS InlineAsm operators (TYPE/LENGTH/SIZE)
+        if (unsigned OpKind = IdentifyIntelInlineAsmOperator(Identifier)) {
+          if (int64_t Val = ParseIntelInlineAsmOperator(OpKind)) {
+            if (SM.onInteger(Val, ErrMsg))
+              return Error(IdentLoc, ErrMsg);
+          } else
+            return true;
+          break;
+        }
+        // MS InlineAsm identifier
+        // Call parseIdentifier() to combine @ with the identifier behind it.
+        if (TK == AsmToken::At && Parser.parseIdentifier(Identifier))
+          return Error(IdentLoc, "expected identifier");
+        if (ParseIntelInlineAsmIdentifier(Val, Identifier, Info, false, End))
           return true;
+        else if (SM.onIdentifierExpr(Val, Identifier, Info, true, ErrMsg))
+          return Error(IdentLoc, ErrMsg);
         break;
       }
-      // MS InlineAsm identifier
-      // Call parseIdentifier() to combine @ with the identifier behind it.
-      if (TK == AsmToken::At && Parser.parseIdentifier(Identifier))
-        return Error(IdentLoc, "expected identifier");
-      if (ParseIntelInlineAsmIdentifier(Val, Identifier, Info, false, End))
-        return true;
-      else if (SM.onIdentifierExpr(Val, Identifier, Info, true, ErrMsg))
+      if (getParser().parsePrimaryExpr(Val, End)) {
+        return Error(Tok.getLoc(), "Unexpected identifier!");
+      } else if (SM.onIdentifierExpr(Val, Identifier, Info, false, ErrMsg)) {
         return Error(IdentLoc, ErrMsg);
+      }
       break;
     }
     case AsmToken::Integer: {
@@ -1855,10 +1929,14 @@ bool X86AsmParser::ParseIntelDotOperator(IntelExprStateMachine &SM, SMLoc &End)
     APInt DotDisp;
     DotDispStr.getAsInteger(10, DotDisp);
     Offset = DotDisp.getZExtValue();
-  } else if (isParsingMSInlineAsm() && Tok.is(AsmToken::Identifier)) {
-    std::pair<StringRef, StringRef> BaseMember = DotDispStr.split('.');
-    if (SemaCallback->LookupInlineAsmField(BaseMember.first, BaseMember.second,
-                                           Offset))
+  } else if ((isParsingMSInlineAsm() || getParser().isParsingMasm()) &&
+             Tok.is(AsmToken::Identifier)) {
+    const std::pair<StringRef, StringRef> BaseMember = DotDispStr.split('.');
+    const StringRef Base = BaseMember.first, Member = BaseMember.second;
+    if (getParser().LookUpFieldOffset(SM.getSymName(), DotDispStr, Offset) &&
+        getParser().LookUpFieldOffset(Base, Member, Offset) &&
+        (!SemaCallback ||
+         SemaCallback->LookupInlineAsmField(Base, Member, Offset)))
       return Error(Tok.getLoc(), "Unable to lookup field reference!");
   } else
     return Error(Tok.getLoc(), "Unexpected token type!");
@@ -2073,6 +2151,14 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseIntelOperand() {
     return CreateMemForMSInlineAsm(RegNo, Disp, BaseReg, IndexReg, Scale, Start,
                                    End, Size, SM.getSymName(),
                                    SM.getIdentifierInfo());
+
+  // When parsing x64 MS-style assembly, all memory operands default to
+  // RIP-relative when interpreted as non-absolute references.
+  if (Parser.isParsingMasm() && is64BitMode())
+    return X86Operand::CreateMem(getPointerWidth(), RegNo, Disp, BaseReg,
+                                 IndexReg, Scale, Start, End, Size,
+                                 /*DefaultBaseReg=*/X86::RIP);
+
   if (!(BaseReg || IndexReg || RegNo))
     return X86Operand::CreateMem(getPointerWidth(), Disp, Start, End, Size);
   return X86Operand::CreateMem(getPointerWidth(), RegNo, Disp,
diff --git a/llvm/lib/Target/X86/AsmParser/X86Operand.h b/llvm/lib/Target/X86/AsmParser/X86Operand.h
index fb5f3355532eb..5cf4516ede974 100644
--- a/llvm/lib/Target/X86/AsmParser/X86Operand.h
+++ b/llvm/lib/Target/X86/AsmParser/X86Operand.h
@@ -58,6 +58,7 @@ struct X86Operand final : public MCParsedAsmOperand {
     unsigned SegReg;
     const MCExpr *Disp;
     unsigned BaseReg;
+    unsigned DefaultBaseReg;
     unsigned IndexReg;
     unsigned Scale;
     unsigned Size;
@@ -182,6 +183,10 @@ struct X86Operand final : public MCParsedAsmOperand {
     assert(Kind == Memory && "Invalid access!");
     return Mem.BaseReg;
   }
+  unsigned getMemDefaultBaseReg() const {
+    assert(Kind == Memory && "Invalid access!");
+    return Mem.DefaultBaseReg;
+  }
   unsigned getMemIndexReg() const {
     assert(Kind == Memory && "Invalid access!");
     return Mem.IndexReg;
@@ -310,6 +315,11 @@ struct X86Operand final : public MCParsedAsmOperand {
   bool isMem512() const {
     return Kind == Memory && (!Mem.Size || Mem.Size == 512);
   }
+
+  bool isSibMem() const {
+    return isMem() && Mem.BaseReg != X86::RIP && Mem.BaseReg != X86::EIP;
+  }
+
   bool isMemIndexReg(unsigned LowR, unsigned HighR) const {
     assert(Kind == Memory && "Invalid access!");
     return Mem.IndexReg >= LowR && Mem.IndexReg <= HighR;
@@ -546,7 +556,10 @@ struct X86Operand final : public MCParsedAsmOperand {
 
   void addMemOperands(MCInst &Inst, unsigned N) const {
     assert((N == 5) && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(getMemBaseReg()));
+    if (getMemBaseReg())
+      Inst.addOperand(MCOperand::createReg(getMemBaseReg()));
+    else
+      Inst.addOperand(MCOperand::createReg(getMemDefaultBaseReg()));
     Inst.addOperand(MCOperand::createImm(getMemScale()));
     Inst.addOperand(MCOperand::createReg(getMemIndexReg()));
     addExpr(Inst, getMemDisp());
@@ -639,6 +652,7 @@ struct X86Operand final : public MCParsedAsmOperand {
     Res->Mem.SegReg   = 0;
     Res->Mem.Disp     = Disp;
     Res->Mem.BaseReg  = 0;
+    Res->Mem.DefaultBaseReg = 0;
     Res->Mem.IndexReg = 0;
     Res->Mem.Scale    = 1;
     Res->Mem.Size     = Size;
@@ -654,11 +668,14 @@ struct X86Operand final : public MCParsedAsmOperand {
   static std::unique_ptr<X86Operand>
   CreateMem(unsigned ModeSize, unsigned SegReg, const MCExpr *Disp,
             unsigned BaseReg, unsigned IndexReg, unsigned Scale, SMLoc StartLoc,
-            SMLoc EndLoc, unsigned Size = 0, StringRef SymName = StringRef(),
-            void *OpDecl = nullptr, unsigned FrontendSize = 0) {
+            SMLoc EndLoc, unsigned Size = 0,
+            unsigned DefaultBaseReg = X86::NoRegister,
+            StringRef SymName = StringRef(), void *OpDecl = nullptr,
+            unsigned FrontendSize = 0) {
     // We should never just have a displacement, that should be parsed as an
     // absolute memory operand.
-    assert((SegReg || BaseReg || IndexReg) && "Invalid memory operand!");
+    assert((SegReg || BaseReg || IndexReg || DefaultBaseReg) &&
+           "Invalid memory operand!");
 
     // The scale should always be one of {1,2,4,8}.
     assert(((Scale == 1 || Scale == 2 || Scale == 4 || Scale == 8)) &&
@@ -667,6 +684,7 @@ struct X86Operand final : public MCParsedAsmOperand {
     Res->Mem.SegReg   = SegReg;
     Res->Mem.Disp     = Disp;
     Res->Mem.BaseReg  = BaseReg;
+    Res->Mem.DefaultBaseReg = DefaultBaseReg;
     Res->Mem.IndexReg = IndexReg;
     Res->Mem.Scale    = Scale;
     Res->Mem.Size     = Size;
diff --git a/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp b/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp
index ea8c606d15644..a7fa1eb9a5eea 100644
--- a/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp
+++ b/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp
@@ -776,6 +776,10 @@ static int readModRM(struct InternalInstruction *insn) {
       return prefix##_YMM0 + index;                                            \
     case TYPE_XMM:                                                             \
       return prefix##_XMM0 + index;                                            \
+    case TYPE_TMM:                                                             \
+      if (index > 7)                                                           \
+        *valid = 0;                                                            \
+      return prefix##_TMM0 + index;                                            \
     case TYPE_VK:                                                              \
       index &= 0xf;                                                            \
       if (index > 7)                                                           \
@@ -849,6 +853,7 @@ static int fixupReg(struct InternalInstruction *insn,
     if (!valid)
       return -1;
     break;
+  case ENCODING_SIB:
   CASE_ENCODING_RM:
     if (insn->eaBase >= insn->eaRegBase) {
       insn->eaBase = (EABase)fixupRMValue(
@@ -1533,6 +1538,15 @@ static int readOperands(struct InternalInstruction *insn) {
       if (Op.encoding != ENCODING_REG && insn->eaDisplacement == EA_DISP_8)
         insn->displacement *= 1 << (Op.encoding - ENCODING_VSIB);
       break;
+    case ENCODING_SIB:
+      // Reject if SIB wasn't used.
+      if (insn->eaBase != EA_BASE_sib && insn->eaBase != EA_BASE_sib64)
+        return -1;
+      if (readModRM(insn))
+        return -1;
+      if (fixupReg(insn, &Op))
+        return -1;
+      break;
     case ENCODING_REG:
     CASE_ENCODING_RM:
       if (readModRM(insn))
@@ -2006,9 +2020,11 @@ static bool translateRMRegister(MCInst &mcInst,
 /// @param mcInst       - The MCInst to append to.
 /// @param insn         - The instruction to extract Mod, R/M, and SIB fields
 ///                       from.
+/// @param ForceSIB     - The instruction must use SIB.
 /// @return             - 0 on success; nonzero otherwise
 static bool translateRMMemory(MCInst &mcInst, InternalInstruction &insn,
-                              const MCDisassembler *Dis) {
+                              const MCDisassembler *Dis,
+                              bool ForceSIB = false) {
   // Addresses in an MCInst are represented as five operands:
   //   1. basereg       (register)  The R/M base, or (if there is a SIB) the
   //                                SIB base
@@ -2067,11 +2083,12 @@ static bool translateRMMemory(MCInst &mcInst, InternalInstruction &insn,
       // -Any base register used other than ESP/RSP/R12D/R12. Using these as a
       //  base always requires a SIB byte.
       // -A scale other than 1 is used.
-      if (insn.sibScale != 1 ||
-          (insn.sibBase == SIB_BASE_NONE && insn.mode != MODE_64BIT) ||
-          (insn.sibBase != SIB_BASE_NONE &&
-           insn.sibBase != SIB_BASE_ESP && insn.sibBase != SIB_BASE_RSP &&
-           insn.sibBase != SIB_BASE_R12D && insn.sibBase != SIB_BASE_R12)) {
+      if (!ForceSIB &&
+          (insn.sibScale != 1 ||
+           (insn.sibBase == SIB_BASE_NONE && insn.mode != MODE_64BIT) ||
+           (insn.sibBase != SIB_BASE_NONE &&
+            insn.sibBase != SIB_BASE_ESP && insn.sibBase != SIB_BASE_RSP &&
+            insn.sibBase != SIB_BASE_R12D && insn.sibBase != SIB_BASE_R12))) {
         indexReg = MCOperand::createReg(insn.addressSize == 4 ? X86::EIZ :
                                                                 X86::RIZ);
       } else
@@ -2182,6 +2199,7 @@ static bool translateRM(MCInst &mcInst, const OperandSpecifier &operand,
   case TYPE_XMM:
   case TYPE_YMM:
   case TYPE_ZMM:
+  case TYPE_TMM:
   case TYPE_VK_PAIR:
   case TYPE_VK:
   case TYPE_DEBUGREG:
@@ -2193,6 +2211,8 @@ static bool translateRM(MCInst &mcInst, const OperandSpecifier &operand,
   case TYPE_MVSIBY:
   case TYPE_MVSIBZ:
     return translateRMMemory(mcInst, insn, Dis);
+  case TYPE_MSIB:
+    return translateRMMemory(mcInst, insn, Dis, true);
   }
 }
 
@@ -2242,6 +2262,7 @@ static bool translateOperand(MCInst &mcInst, const OperandSpecifier &operand,
     return false;
   case ENCODING_WRITEMASK:
     return translateMaskRegister(mcInst, insn.writemask);
+  case ENCODING_SIB:
   CASE_ENCODING_RM:
   CASE_ENCODING_VSIB:
     return translateRM(mcInst, operand, insn, Dis);
diff --git a/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h b/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h
index a950deb95b268..4318c17f03a08 100644
--- a/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h
+++ b/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h
@@ -380,6 +380,17 @@ namespace X86Disassembler {
   ENTRY(BND2)         \
   ENTRY(BND3)
 
+#undef  REGS_TMM
+#define REGS_TMM  \
+  ENTRY(TMM0)     \
+  ENTRY(TMM1)     \
+  ENTRY(TMM2)     \
+  ENTRY(TMM3)     \
+  ENTRY(TMM4)     \
+  ENTRY(TMM5)     \
+  ENTRY(TMM6)     \
+  ENTRY(TMM7)
+
 #define ALL_EA_BASES  \
   EA_BASES_16BIT      \
   EA_BASES_32BIT      \
@@ -404,6 +415,7 @@ namespace X86Disassembler {
   REGS_DEBUG          \
   REGS_CONTROL        \
   REGS_BOUND          \
+  REGS_TMM            \
   ENTRY(RIP)
 
 /// All possible values of the base field for effective-address
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index e49ee17923739..bf3b6bcb5463f 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -1096,7 +1096,7 @@ bool X86AsmBackend::writeNopData(raw_ostream &OS, uint64_t Count) const {
 
   // This CPU doesn't support long nops. If needed add more.
   // FIXME: We could generated something better than plain 0x90.
-  if (!STI.getFeatureBits()[X86::FeatureNOPL]) {
+  if (!STI.hasFeature(X86::FeatureNOPL) && !STI.hasFeature(X86::Mode64Bit)) {
     for (uint64_t i = 0; i < Count; ++i)
       OS << '\x90';
     return true;
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
index 01e865ac684c7..79f07d3c7792a 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
@@ -623,6 +623,15 @@ namespace X86II {
     /// information.  In the intel manual these are represented as /0, /1, ...
     ///
 
+    // Instructions operate on a register Reg/Opcode operand not the r/m field.
+    MRMr0 = 21,
+
+    /// MRMSrcMem - But force to use the SIB field.
+    MRMSrcMemFSIB  = 22,
+
+    /// MRMDestMem - But force to use the SIB field.
+    MRMDestMemFSIB = 23,
+
     /// MRMDestMem - This form is used for instructions that use the Mod/RM byte
     /// to specify a destination, which in this case is memory.
     ///
@@ -1082,8 +1091,10 @@ namespace X86II {
     case X86II::PrefixByte:
       return -1;
     case X86II::MRMDestMem:
+    case X86II::MRMDestMemFSIB:
       return 0;
     case X86II::MRMSrcMem:
+    case X86II::MRMSrcMemFSIB:
       // Start from 1, skip any registers encoded in VEX_VVVV or I8IMM, or a
       // mask register.
       return 1 + HasVEX_4V + HasEVEX_K;
@@ -1103,6 +1114,7 @@ namespace X86II {
     case X86II::MRMSrcRegOp4:
     case X86II::MRMSrcRegCC:
     case X86II::MRMXrCC:
+    case X86II::MRMr0:
     case X86II::MRMXr:
     case X86II::MRM0r: case X86II::MRM1r:
     case X86II::MRM2r: case X86II::MRM3r:
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index 98330f60e63d4..7dea0760a8310 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -78,7 +78,8 @@ class X86MCCodeEmitter : public MCCodeEmitter {
   void emitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField,
                         uint64_t TSFlags, bool HasREX, uint64_t StartByte,
                         raw_ostream &OS, SmallVectorImpl<MCFixup> &Fixups,
-                        const MCSubtargetInfo &STI) const;
+                        const MCSubtargetInfo &STI,
+                        bool ForceSIB = false) const;
 
   bool emitPrefixImpl(unsigned &CurOp, const MCInst &MI,
                       const MCSubtargetInfo &STI, raw_ostream &OS) const;
@@ -382,7 +383,8 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
                                         uint64_t TSFlags, bool HasREX,
                                         uint64_t StartByte, raw_ostream &OS,
                                         SmallVectorImpl<MCFixup> &Fixups,
-                                        const MCSubtargetInfo &STI) const {
+                                        const MCSubtargetInfo &STI,
+                                        bool ForceSIB) const {
   const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp);
   const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg);
   const MCOperand &Scale = MI.getOperand(Op + X86::AddrScaleAmt);
@@ -395,7 +397,8 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
       BaseReg == X86::EIP) { // [disp32+rIP] in X86-64 mode
     assert(STI.hasFeature(X86::Mode64Bit) &&
            "Rip-relative addressing requires 64-bit mode");
-    assert(IndexReg.getReg() == 0 && "Invalid rip-relative address");
+    assert(IndexReg.getReg() == 0 && !ForceSIB &&
+           "Invalid rip-relative address");
     emitByte(modRMByte(0, RegOpcodeField, 5), OS);
 
     unsigned Opcode = MI.getOpcode();
@@ -510,7 +513,7 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
   // 2-7) and absolute references.
 
   if ( // The SIB byte must be used if there is an index register.
-      IndexReg.getReg() == 0 &&
+      !ForceSIB && IndexReg.getReg() == 0 &&
       // The SIB byte must be used if the base is ESP/RSP/R12, all of which
       // encode to an R/M value of 4, which indicates that a SIB byte is
       // present.
@@ -883,9 +886,11 @@ void X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
   switch (TSFlags & X86II::FormMask) {
   default:
     llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!");
+  case X86II::MRM_C0:
   case X86II::RawFrm:
   case X86II::PrefixByte:
     break;
+  case X86II::MRMDestMemFSIB:
   case X86II::MRMDestMem: {
     // MRMDestMem instructions forms:
     //  MemAddr, src1(ModR/M)
@@ -916,6 +921,7 @@ void X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
     EVEX_R2 = ~(RegEnc >> 4) & 1;
     break;
   }
+  case X86II::MRMSrcMemFSIB:
   case X86II::MRMSrcMem: {
     // MRMSrcMem instructions forms:
     //  src1(ModR/M), MemAddr
@@ -1097,6 +1103,15 @@ void X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
       EncodeRC = true;
     break;
   }
+  case X86II::MRMr0: {
+    // MRMr0 instructions forms:
+    //  11:rrr:000
+    //  dst(ModR/M)
+    unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
+    VEX_R = ~(RegEnc >> 3) & 1;
+    EVEX_R2 = ~(RegEnc >> 4) & 1;
+    break;
+  }
   case X86II::MRM0r:
   case X86II::MRM1r:
   case X86II::MRM2r:
@@ -1267,6 +1282,11 @@ bool X86MCCodeEmitter::emitREXPrefix(int MemOperand, const MCInst &MI,
     case X86II::MRM7r:
       REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
       break;
+    case X86II::MRMr0:
+      REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
+      break;
+    case X86II::MRMDestMemFSIB:
+      llvm_unreachable("FSIB format never need REX prefix!");
     }
     if (REX && UsesHighByteReg)
       report_fatal_error(
@@ -1481,6 +1501,7 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
     CurOp = SrcRegNum + 1;
     break;
   }
+  case X86II::MRMDestMemFSIB:
   case X86II::MRMDestMem: {
     emitByte(BaseOpcode, OS);
     unsigned SrcRegNum = CurOp + X86::AddrNumOperands;
@@ -1491,8 +1512,9 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
       ++SrcRegNum;
 
+    bool ForceSIB = (Form == X86II::MRMDestMemFSIB);
     emitMemModRMByte(MI, CurOp, getX86RegNum(MI.getOperand(SrcRegNum)), TSFlags,
-                     HasREX, StartByte, OS, Fixups, STI);
+                     HasREX, StartByte, OS, Fixups, STI, ForceSIB);
     CurOp = SrcRegNum + 1;
     break;
   }
@@ -1553,6 +1575,7 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
                      getX86RegNum(MI.getOperand(FirstOp)), OS);
     break;
   }
+  case X86II::MRMSrcMemFSIB:
   case X86II::MRMSrcMem: {
     unsigned FirstMemOp = CurOp + 1;
 
@@ -1564,8 +1587,9 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
 
     emitByte(BaseOpcode, OS);
 
+    bool ForceSIB = (Form == X86II::MRMSrcMemFSIB);
     emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
-                     TSFlags, HasREX, StartByte, OS, Fixups, STI);
+                     TSFlags, HasREX, StartByte, OS, Fixups, STI, ForceSIB);
     CurOp = FirstMemOp + X86::AddrNumOperands;
     if (HasVEX_I8Reg)
       I8RegNum = getX86RegEncoding(MI, CurOp++);
@@ -1637,6 +1661,10 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
     emitRegModRMByte(MI.getOperand(CurOp++),
                      (Form == X86II::MRMXr) ? 0 : Form - X86II::MRM0r, OS);
     break;
+  case X86II::MRMr0:
+    emitByte(BaseOpcode, OS);
+    emitByte(modRMByte(3, getX86RegNum(MI.getOperand(CurOp++)),0), OS);
+    break;
 
   case X86II::MRMXmCC: {
     unsigned FirstMemOp = CurOp;
diff --git a/llvm/lib/Target/X86/X86.h b/llvm/lib/Target/X86/X86.h
index e7f373e5e8656..91ba4e3d091e8 100644
--- a/llvm/lib/Target/X86/X86.h
+++ b/llvm/lib/Target/X86/X86.h
@@ -141,7 +141,6 @@ InstructionSelector *createX86InstructionSelector(const X86TargetMachine &TM,
                                                   X86RegisterBankInfo &);
 
 FunctionPass *createX86LoadValueInjectionLoadHardeningPass();
-FunctionPass *createX86LoadValueInjectionLoadHardeningUnoptimizedPass();
 FunctionPass *createX86LoadValueInjectionRetHardeningPass();
 FunctionPass *createX86SpeculativeLoadHardeningPass();
 FunctionPass *createX86SpeculativeExecutionSideEffectSuppression();
@@ -161,7 +160,6 @@ void initializeX86ExecutionDomainFixPass(PassRegistry &);
 void initializeX86ExpandPseudoPass(PassRegistry &);
 void initializeX86FixupSetCCPassPass(PassRegistry &);
 void initializeX86FlagsCopyLoweringPassPass(PassRegistry &);
-void initializeX86LoadValueInjectionLoadHardeningUnoptimizedPassPass(PassRegistry &);
 void initializeX86LoadValueInjectionLoadHardeningPassPass(PassRegistry &);
 void initializeX86LoadValueInjectionRetHardeningPassPass(PassRegistry &);
 void initializeX86OptimizeLEAPassPass(PassRegistry &);
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index bcbc1ae492b21..dc1ff72add491 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -52,13 +52,16 @@ def FeatureXSAVE   : SubtargetFeature<"xsave", "HasXSAVE", "true",
                                        "Support xsave instructions">;
 
 def FeatureXSAVEOPT: SubtargetFeature<"xsaveopt", "HasXSAVEOPT", "true",
-                                       "Support xsaveopt instructions">;
+                                       "Support xsaveopt instructions",
+                                       [FeatureXSAVE]>;
 
 def FeatureXSAVEC  : SubtargetFeature<"xsavec", "HasXSAVEC", "true",
-                                       "Support xsavec instructions">;
+                                       "Support xsavec instructions",
+                                       [FeatureXSAVE]>;
 
 def FeatureXSAVES  : SubtargetFeature<"xsaves", "HasXSAVES", "true",
-                                       "Support xsaves instructions">;
+                                       "Support xsaves instructions",
+                                       [FeatureXSAVE]>;
 
 def FeatureSSE1    : SubtargetFeature<"sse", "X86SSELevel", "SSE1",
                                       "Enable SSE instructions">;
@@ -246,6 +249,14 @@ def FeaturePTWRITE  : SubtargetFeature<"ptwrite", "HasPTWRITE", "true",
 // target-feature attribute.
 def FeatureDeprecatedMPX : SubtargetFeature<"mpx", "DeprecatedHasMPX", "false",
                                       "Deprecated. Support MPX instructions">;
+def FeatureAMXTILE     : SubtargetFeature<"amx-tile", "HasAMXTILE", "true",
+                                      "Support AMX-TILE instructions">;
+def FeatureAMXINT8     : SubtargetFeature<"amx-int8", "HasAMXINT8", "true",
+                                      "Support AMX-INT8 instructions",
+                                      [FeatureAMXTILE]>;
+def FeatureAMXBF16     : SubtargetFeature<"amx-bf16", "HasAMXBF16", "true",
+                                      "Support AMX-BF16 instructions",
+                                      [FeatureAMXTILE]>;
 def FeatureLEAForSP : SubtargetFeature<"lea-sp", "UseLeaForSP", "true",
                                      "Use LEA for adjusting the stack pointer">;
 def FeatureSlowDivide32 : SubtargetFeature<"idivl-to-divb",
@@ -444,6 +455,15 @@ def FeatureLVIControlFlowIntegrity
           "LFENCE instruction to serialize control flow. Also decompose RET "
           "instructions into a POP+LFENCE+JMP sequence.">;
 
+// Enable SESES to mitigate speculative execution attacks
+def FeatureSpeculativeExecutionSideEffectSuppression
+    : SubtargetFeature<
+          "seses", "UseSpeculativeExecutionSideEffectSuppression", "true",
+          "Prevent speculative execution side channel timing attacks by "
+          "inserting a speculation barrier before memory reads, memory writes, "
+          "and conditional branches. Implies LVI Control Flow integrity.",
+          [FeatureLVIControlFlowIntegrity]>;
+
 // Mitigate LVI attacks against data loads
 def FeatureLVILoadHardening
     : SubtargetFeature<
@@ -847,6 +867,7 @@ def ProcessorFeatures {
                                                          FeatureFXSR,
                                                          FeatureNOPL,
                                                          FeatureCMPXCHG16B,
+                                                         FeaturePRFCHW,
                                                          FeatureLZCNT,
                                                          FeaturePOPCNT,
                                                          FeatureSlowSHLD,
@@ -942,6 +963,8 @@ def ProcessorFeatures {
   // Excavator
   list<SubtargetFeature> BdVer4AdditionalFeatures = [FeatureAVX2,
                                                      FeatureBMI2,
+                                                     FeatureMOVBE,
+                                                     FeatureRDRAND,
                                                      FeatureMWAITX];
   list<SubtargetFeature> BdVer4InheritableFeatures =
     !listconcat(BdVer3InheritableFeatures, BdVer4AdditionalFeatures);
diff --git a/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp b/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
index 2a37b72956e98..9f1fece1b9dd8 100644
--- a/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
+++ b/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
@@ -551,11 +551,8 @@ void X86AvoidSFBPass::findPotentiallylBlockedCopies(MachineFunction &MF) {
         if (StoreMI.getParent() == MI.getParent() &&
             isPotentialBlockedMemCpyPair(MI.getOpcode(), StoreMI.getOpcode()) &&
             isRelevantAddressingMode(&MI) &&
-            isRelevantAddressingMode(&StoreMI)) {
-          assert(MI.hasOneMemOperand() &&
-                 "Expected one memory operand for load instruction");
-          assert(StoreMI.hasOneMemOperand() &&
-                 "Expected one memory operand for store instruction");
+            isRelevantAddressingMode(&StoreMI) &&
+            MI.hasOneMemOperand() && StoreMI.hasOneMemOperand()) {
           if (!alias(**MI.memoperands_begin(), **StoreMI.memoperands_begin()))
             BlockedLoadsStoresPairs.push_back(std::make_pair(&MI, &StoreMI));
         }
@@ -695,10 +692,9 @@ bool X86AvoidSFBPass::runOnMachineFunction(MachineFunction &MF) {
     for (auto *PBInst : PotentialBlockers) {
       if (!isPotentialBlockingStoreInst(PBInst->getOpcode(),
                                         LoadInst->getOpcode()) ||
-          !isRelevantAddressingMode(PBInst))
+          !isRelevantAddressingMode(PBInst) || !PBInst->hasOneMemOperand())
         continue;
       int64_t PBstDispImm = getDispOperand(PBInst).getImm();
-      assert(PBInst->hasOneMemOperand() && "Expected One Memory Operand");
       unsigned PBstSize = (*PBInst->memoperands_begin())->getSize();
       // This check doesn't cover all cases, but it will suffice for now.
       // TODO: take branch probability into consideration, if the blocking
diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp
index 147cd1b9b32de..b305940139c06 100644
--- a/llvm/lib/Target/X86/X86FastISel.cpp
+++ b/llvm/lib/Target/X86/X86FastISel.cpp
@@ -1127,10 +1127,8 @@ bool X86FastISel::X86SelectStore(const Instruction *I) {
   if (!isTypeLegal(Val->getType(), VT, /*AllowI1=*/true))
     return false;
 
-  unsigned Alignment = S->getAlignment();
-  unsigned ABIAlignment = DL.getABITypeAlignment(Val->getType());
-  if (Alignment == 0) // Ensure that codegen never sees alignment 0
-    Alignment = ABIAlignment;
+  Align Alignment = S->getAlign();
+  Align ABIAlignment = DL.getABITypeAlign(Val->getType());
   bool Aligned = Alignment >= ABIAlignment;
 
   X86AddressMode AM;
@@ -1321,14 +1319,9 @@ bool X86FastISel::X86SelectLoad(const Instruction *I) {
   if (!X86SelectAddress(Ptr, AM))
     return false;
 
-  unsigned Alignment = LI->getAlignment();
-  unsigned ABIAlignment = DL.getABITypeAlignment(LI->getType());
-  if (Alignment == 0) // Ensure that codegen never sees alignment 0
-    Alignment = ABIAlignment;
-
   unsigned ResultReg = 0;
   if (!X86FastEmitLoad(VT, AM, createMachineMemOperandFor(LI), ResultReg,
-                       Alignment))
+                       LI->getAlign().value()))
     return false;
 
   updateValueMap(I, ResultReg);
@@ -3584,7 +3577,7 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
       EVT ResVT = VA.getValVT();
       unsigned Opc = ResVT == MVT::f32 ? X86::ST_Fp80m32 : X86::ST_Fp80m64;
       unsigned MemSize = ResVT.getSizeInBits()/8;
-      int FI = MFI.CreateStackObject(MemSize, MemSize, false);
+      int FI = MFI.CreateStackObject(MemSize, Align(MemSize), false);
       addFrameReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
                                 TII.get(Opc)), FI)
         .addReg(CopyReg);
@@ -3671,12 +3664,17 @@ X86FastISel::fastSelectInstruction(const Instruction *I)  {
       return false;
 
     Register Reg = getRegForValue(I->getOperand(0));
-    if (Reg == 0)
+    if (!Reg)
       return false;
 
-    // No instruction is needed for conversion. Reuse the register used by
-    // the fist operand.
-    updateValueMap(I, Reg);
+    // Emit a reg-reg copy so we don't propagate cached known bits information
+    // with the wrong VT if we fall out of fast isel after selecting this.
+    const TargetRegisterClass *DstClass = TLI.getRegClassFor(DstVT);
+    Register ResultReg = createResultReg(DstClass);
+    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+              TII.get(TargetOpcode::COPY), ResultReg).addReg(Reg);
+
+    updateValueMap(I, ResultReg);
     return true;
   }
   }
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index b360aa9d905c1..c1bd80c41f131 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -636,16 +636,16 @@ void X86FrameLowering::emitStackProbeInlineGenericLoop(
   MF.insert(MBBIter, testMBB);
   MF.insert(MBBIter, tailMBB);
 
-  Register FinalStackPtr = Uses64BitFramePtr ? X86::R11 : X86::R11D;
-  BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::COPY), FinalStackPtr)
+  Register FinalStackProbed = Uses64BitFramePtr ? X86::R11 : X86::R11D;
+  BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::COPY), FinalStackProbed)
       .addReg(StackPtr)
       .setMIFlag(MachineInstr::FrameSetup);
 
   // save loop bound
   {
     const unsigned Opc = getSUBriOpcode(Uses64BitFramePtr, Offset);
-    BuildMI(MBB, MBBI, DL, TII.get(Opc), FinalStackPtr)
-        .addReg(FinalStackPtr)
+    BuildMI(MBB, MBBI, DL, TII.get(Opc), FinalStackProbed)
+        .addReg(FinalStackProbed)
         .addImm(Offset / StackProbeSize * StackProbeSize)
         .setMIFlag(MachineInstr::FrameSetup);
   }
@@ -669,13 +669,13 @@ void X86FrameLowering::emitStackProbeInlineGenericLoop(
   // cmp with stack pointer bound
   BuildMI(testMBB, DL, TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
       .addReg(StackPtr)
-      .addReg(FinalStackPtr)
+      .addReg(FinalStackProbed)
       .setMIFlag(MachineInstr::FrameSetup);
 
   // jump
   BuildMI(testMBB, DL, TII.get(X86::JCC_1))
       .addMBB(testMBB)
-      .addImm(X86::COND_L)
+      .addImm(X86::COND_NE)
       .setMIFlag(MachineInstr::FrameSetup);
   testMBB->addSuccessor(testMBB);
   testMBB->addSuccessor(tailMBB);
@@ -686,10 +686,12 @@ void X86FrameLowering::emitStackProbeInlineGenericLoop(
   MBB.addSuccessor(testMBB);
 
   // handle tail
-  if (Offset % StackProbeSize) {
-    BuildMI(*tailMBB, tailMBB->begin(), DL, TII.get(TargetOpcode::COPY),
-            StackPtr)
-        .addReg(FinalStackPtr)
+  unsigned TailOffset = Offset % StackProbeSize;
+  if (TailOffset) {
+    const unsigned Opc = getSUBriOpcode(Uses64BitFramePtr, TailOffset);
+    BuildMI(*tailMBB, tailMBB->begin(), DL, TII.get(Opc), StackPtr)
+        .addReg(StackPtr)
+        .addImm(TailOffset)
         .setMIFlag(MachineInstr::FrameSetup);
   }
 
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 23a867ad03eea..fb285376c5808 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -71,14 +71,14 @@ namespace {
     const char *ES;
     MCSymbol *MCSym;
     int JT;
-    unsigned Align;    // CP alignment.
+    Align Alignment;            // CP alignment.
     unsigned char SymbolFlags;  // X86II::MO_*
     bool NegateIndex = false;
 
     X86ISelAddressMode()
         : BaseType(RegBase), Base_FrameIndex(0), Scale(1), IndexReg(), Disp(0),
           Segment(), GV(nullptr), CP(nullptr), BlockAddr(nullptr), ES(nullptr),
-          MCSym(nullptr), JT(-1), Align(0), SymbolFlags(X86II::MO_NO_FLAG) {}
+          MCSym(nullptr), JT(-1), SymbolFlags(X86II::MO_NO_FLAG) {}
 
     bool hasSymbolicDisplacement() const {
       return GV != nullptr || CP != nullptr || ES != nullptr ||
@@ -144,7 +144,7 @@ namespace {
         dbgs() << MCSym;
       else
         dbgs() << "nul";
-      dbgs() << " JT" << JT << " Align" << Align << '\n';
+      dbgs() << " JT" << JT << " Align" << Alignment.value() << '\n';
     }
 #endif
   };
@@ -296,7 +296,7 @@ namespace {
                                               MVT::i32, AM.Disp,
                                               AM.SymbolFlags);
       else if (AM.CP)
-        Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, Align(AM.Align),
+        Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Alignment,
                                              AM.Disp, AM.SymbolFlags);
       else if (AM.ES) {
         assert(!AM.Disp && "Non-zero displacement is ignored with ES.");
@@ -503,6 +503,7 @@ namespace {
     bool isMaskZeroExtended(SDNode *N) const;
     bool tryShiftAmountMod(SDNode *N);
     bool tryShrinkShlLogicImm(SDNode *N);
+    bool tryVPTERNLOG(SDNode *N);
     bool tryVPTESTM(SDNode *Root, SDValue Setcc, SDValue Mask);
     bool tryMatchBitSelect(SDNode *N);
 
@@ -1622,7 +1623,7 @@ bool X86DAGToDAGISel::matchWrapper(SDValue N, X86ISelAddressMode &AM) {
     Offset = G->getOffset();
   } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) {
     AM.CP = CP->getConstVal();
-    AM.Align = CP->getAlign().value();
+    AM.Alignment = CP->getAlign();
     AM.SymbolFlags = CP->getTargetFlags();
     Offset = CP->getOffset();
   } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) {
@@ -3929,6 +3930,85 @@ bool X86DAGToDAGISel::tryShrinkShlLogicImm(SDNode *N) {
   return true;
 }
 
+// Try to match two logic ops to a VPTERNLOG.
+// FIXME: Handle inverted inputs?
+// FIXME: Handle more complex patterns that use an operand more than once?
+bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
+  MVT NVT = N->getSimpleValueType(0);
+
+  // Make sure we support VPTERNLOG.
+  if (!NVT.isVector() || !Subtarget->hasAVX512() ||
+      NVT.getVectorElementType() == MVT::i1)
+    return false;
+
+  // We need VLX for 128/256-bit.
+  if (!(Subtarget->hasVLX() || NVT.is512BitVector()))
+    return false;
+
+  unsigned Opc1 = N->getOpcode();
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+
+  auto isLogicOp = [](unsigned Opc) {
+    return Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR ||
+           Opc == X86ISD::ANDNP;
+  };
+
+  SDValue A, B, C;
+  unsigned Opc2;
+  if (isLogicOp(N1.getOpcode()) && N1.hasOneUse()) {
+    Opc2 = N1.getOpcode();
+    A = N0;
+    B = N1.getOperand(0);
+    C = N1.getOperand(1);
+  } else if (isLogicOp(N0.getOpcode()) && N0.hasOneUse()) {
+    Opc2 = N0.getOpcode();
+    A = N1;
+    B = N0.getOperand(0);
+    C = N0.getOperand(1);
+  } else
+    return false;
+
+  uint64_t Imm;
+  switch (Opc1) {
+  default: llvm_unreachable("Unexpected opcode!");
+  case ISD::AND:
+    switch (Opc2) {
+    default: llvm_unreachable("Unexpected opcode!");
+    case ISD::AND:      Imm = 0x80; break;
+    case ISD::OR:       Imm = 0xe0; break;
+    case ISD::XOR:      Imm = 0x60; break;
+    case X86ISD::ANDNP: Imm = 0x20; break;
+    }
+    break;
+  case ISD::OR:
+    switch (Opc2) {
+    default: llvm_unreachable("Unexpected opcode!");
+    case ISD::AND:      Imm = 0xf8; break;
+    case ISD::OR:       Imm = 0xfe; break;
+    case ISD::XOR:      Imm = 0xf6; break;
+    case X86ISD::ANDNP: Imm = 0xf2; break;
+    }
+    break;
+  case ISD::XOR:
+    switch (Opc2) {
+    default: llvm_unreachable("Unexpected opcode!");
+    case ISD::AND:      Imm = 0x78; break;
+    case ISD::OR:       Imm = 0x1e; break;
+    case ISD::XOR:      Imm = 0x96; break;
+    case X86ISD::ANDNP: Imm = 0xd2; break;
+    }
+    break;
+  }
+
+  SDLoc DL(N);
+  SDValue New = CurDAG->getNode(X86ISD::VPTERNLOG, DL, NVT, A, B, C,
+                                CurDAG->getTargetConstant(Imm, DL, MVT::i8));
+  ReplaceNode(N, New.getNode());
+  SelectCode(New.getNode());
+  return true;
+}
+
 /// If the high bits of an 'and' operand are known zero, try setting the
 /// high bits of an 'and' constant operand to produce a smaller encoding by
 /// creating a small, sign-extended negative immediate rather than a large
@@ -4001,159 +4081,50 @@ bool X86DAGToDAGISel::shrinkAndImmediate(SDNode *And) {
 
 static unsigned getVPTESTMOpc(MVT TestVT, bool IsTestN, bool FoldedLoad,
                               bool FoldedBCast, bool Masked) {
-  if (Masked) {
-    if (FoldedLoad) {
-      switch (TestVT.SimpleTy) {
-      default: llvm_unreachable("Unexpected VT!");
-      case MVT::v16i8:
-        return IsTestN ? X86::VPTESTNMBZ128rmk : X86::VPTESTMBZ128rmk;
-      case MVT::v8i16:
-        return IsTestN ? X86::VPTESTNMWZ128rmk : X86::VPTESTMWZ128rmk;
-      case MVT::v4i32:
-        return IsTestN ? X86::VPTESTNMDZ128rmk : X86::VPTESTMDZ128rmk;
-      case MVT::v2i64:
-        return IsTestN ? X86::VPTESTNMQZ128rmk : X86::VPTESTMQZ128rmk;
-      case MVT::v32i8:
-        return IsTestN ? X86::VPTESTNMBZ256rmk : X86::VPTESTMBZ256rmk;
-      case MVT::v16i16:
-        return IsTestN ? X86::VPTESTNMWZ256rmk : X86::VPTESTMWZ256rmk;
-      case MVT::v8i32:
-        return IsTestN ? X86::VPTESTNMDZ256rmk : X86::VPTESTMDZ256rmk;
-      case MVT::v4i64:
-        return IsTestN ? X86::VPTESTNMQZ256rmk : X86::VPTESTMQZ256rmk;
-      case MVT::v64i8:
-        return IsTestN ? X86::VPTESTNMBZrmk : X86::VPTESTMBZrmk;
-      case MVT::v32i16:
-        return IsTestN ? X86::VPTESTNMWZrmk : X86::VPTESTMWZrmk;
-      case MVT::v16i32:
-        return IsTestN ? X86::VPTESTNMDZrmk : X86::VPTESTMDZrmk;
-      case MVT::v8i64:
-        return IsTestN ? X86::VPTESTNMQZrmk : X86::VPTESTMQZrmk;
-      }
-    }
-
-    if (FoldedBCast) {
-      switch (TestVT.SimpleTy) {
-      default: llvm_unreachable("Unexpected VT!");
-      case MVT::v4i32:
-        return IsTestN ? X86::VPTESTNMDZ128rmbk : X86::VPTESTMDZ128rmbk;
-      case MVT::v2i64:
-        return IsTestN ? X86::VPTESTNMQZ128rmbk : X86::VPTESTMQZ128rmbk;
-      case MVT::v8i32:
-        return IsTestN ? X86::VPTESTNMDZ256rmbk : X86::VPTESTMDZ256rmbk;
-      case MVT::v4i64:
-        return IsTestN ? X86::VPTESTNMQZ256rmbk : X86::VPTESTMQZ256rmbk;
-      case MVT::v16i32:
-        return IsTestN ? X86::VPTESTNMDZrmbk : X86::VPTESTMDZrmbk;
-      case MVT::v8i64:
-        return IsTestN ? X86::VPTESTNMQZrmbk : X86::VPTESTMQZrmbk;
-      }
-    }
-
-    switch (TestVT.SimpleTy) {
-    default: llvm_unreachable("Unexpected VT!");
-    case MVT::v16i8:
-      return IsTestN ? X86::VPTESTNMBZ128rrk : X86::VPTESTMBZ128rrk;
-    case MVT::v8i16:
-      return IsTestN ? X86::VPTESTNMWZ128rrk : X86::VPTESTMWZ128rrk;
-    case MVT::v4i32:
-      return IsTestN ? X86::VPTESTNMDZ128rrk : X86::VPTESTMDZ128rrk;
-    case MVT::v2i64:
-      return IsTestN ? X86::VPTESTNMQZ128rrk : X86::VPTESTMQZ128rrk;
-    case MVT::v32i8:
-      return IsTestN ? X86::VPTESTNMBZ256rrk : X86::VPTESTMBZ256rrk;
-    case MVT::v16i16:
-      return IsTestN ? X86::VPTESTNMWZ256rrk : X86::VPTESTMWZ256rrk;
-    case MVT::v8i32:
-      return IsTestN ? X86::VPTESTNMDZ256rrk : X86::VPTESTMDZ256rrk;
-    case MVT::v4i64:
-      return IsTestN ? X86::VPTESTNMQZ256rrk : X86::VPTESTMQZ256rrk;
-    case MVT::v64i8:
-      return IsTestN ? X86::VPTESTNMBZrrk : X86::VPTESTMBZrrk;
-    case MVT::v32i16:
-      return IsTestN ? X86::VPTESTNMWZrrk : X86::VPTESTMWZrrk;
-    case MVT::v16i32:
-      return IsTestN ? X86::VPTESTNMDZrrk : X86::VPTESTMDZrrk;
-    case MVT::v8i64:
-      return IsTestN ? X86::VPTESTNMQZrrk : X86::VPTESTMQZrrk;
-    }
-  }
+#define VPTESTM_CASE(VT, SUFFIX) \
+case MVT::VT: \
+  if (Masked) \
+    return IsTestN ? X86::VPTESTNM##SUFFIX##k: X86::VPTESTM##SUFFIX##k; \
+  return IsTestN ? X86::VPTESTNM##SUFFIX : X86::VPTESTM##SUFFIX;
+
+
+#define VPTESTM_BROADCAST_CASES(SUFFIX) \
+default: llvm_unreachable("Unexpected VT!"); \
+VPTESTM_CASE(v4i32, DZ128##SUFFIX) \
+VPTESTM_CASE(v2i64, QZ128##SUFFIX) \
+VPTESTM_CASE(v8i32, DZ256##SUFFIX) \
+VPTESTM_CASE(v4i64, QZ256##SUFFIX) \
+VPTESTM_CASE(v16i32, DZ##SUFFIX) \
+VPTESTM_CASE(v8i64, QZ##SUFFIX)
+
+#define VPTESTM_FULL_CASES(SUFFIX) \
+VPTESTM_BROADCAST_CASES(SUFFIX) \
+VPTESTM_CASE(v16i8, BZ128##SUFFIX) \
+VPTESTM_CASE(v8i16, WZ128##SUFFIX) \
+VPTESTM_CASE(v32i8, BZ256##SUFFIX) \
+VPTESTM_CASE(v16i16, WZ256##SUFFIX) \
+VPTESTM_CASE(v64i8, BZ##SUFFIX) \
+VPTESTM_CASE(v32i16, WZ##SUFFIX)
 
   if (FoldedLoad) {
     switch (TestVT.SimpleTy) {
-    default: llvm_unreachable("Unexpected VT!");
-    case MVT::v16i8:
-      return IsTestN ? X86::VPTESTNMBZ128rm : X86::VPTESTMBZ128rm;
-    case MVT::v8i16:
-      return IsTestN ? X86::VPTESTNMWZ128rm : X86::VPTESTMWZ128rm;
-    case MVT::v4i32:
-      return IsTestN ? X86::VPTESTNMDZ128rm : X86::VPTESTMDZ128rm;
-    case MVT::v2i64:
-      return IsTestN ? X86::VPTESTNMQZ128rm : X86::VPTESTMQZ128rm;
-    case MVT::v32i8:
-      return IsTestN ? X86::VPTESTNMBZ256rm : X86::VPTESTMBZ256rm;
-    case MVT::v16i16:
-      return IsTestN ? X86::VPTESTNMWZ256rm : X86::VPTESTMWZ256rm;
-    case MVT::v8i32:
-      return IsTestN ? X86::VPTESTNMDZ256rm : X86::VPTESTMDZ256rm;
-    case MVT::v4i64:
-      return IsTestN ? X86::VPTESTNMQZ256rm : X86::VPTESTMQZ256rm;
-    case MVT::v64i8:
-      return IsTestN ? X86::VPTESTNMBZrm : X86::VPTESTMBZrm;
-    case MVT::v32i16:
-      return IsTestN ? X86::VPTESTNMWZrm : X86::VPTESTMWZrm;
-    case MVT::v16i32:
-      return IsTestN ? X86::VPTESTNMDZrm : X86::VPTESTMDZrm;
-    case MVT::v8i64:
-      return IsTestN ? X86::VPTESTNMQZrm : X86::VPTESTMQZrm;
+    VPTESTM_FULL_CASES(rm)
     }
   }
 
   if (FoldedBCast) {
     switch (TestVT.SimpleTy) {
-    default: llvm_unreachable("Unexpected VT!");
-    case MVT::v4i32:
-      return IsTestN ? X86::VPTESTNMDZ128rmb : X86::VPTESTMDZ128rmb;
-    case MVT::v2i64:
-      return IsTestN ? X86::VPTESTNMQZ128rmb : X86::VPTESTMQZ128rmb;
-    case MVT::v8i32:
-      return IsTestN ? X86::VPTESTNMDZ256rmb : X86::VPTESTMDZ256rmb;
-    case MVT::v4i64:
-      return IsTestN ? X86::VPTESTNMQZ256rmb : X86::VPTESTMQZ256rmb;
-    case MVT::v16i32:
-      return IsTestN ? X86::VPTESTNMDZrmb : X86::VPTESTMDZrmb;
-    case MVT::v8i64:
-      return IsTestN ? X86::VPTESTNMQZrmb : X86::VPTESTMQZrmb;
+    VPTESTM_BROADCAST_CASES(rmb)
     }
   }
 
   switch (TestVT.SimpleTy) {
-  default: llvm_unreachable("Unexpected VT!");
-  case MVT::v16i8:
-    return IsTestN ? X86::VPTESTNMBZ128rr : X86::VPTESTMBZ128rr;
-  case MVT::v8i16:
-    return IsTestN ? X86::VPTESTNMWZ128rr : X86::VPTESTMWZ128rr;
-  case MVT::v4i32:
-    return IsTestN ? X86::VPTESTNMDZ128rr : X86::VPTESTMDZ128rr;
-  case MVT::v2i64:
-    return IsTestN ? X86::VPTESTNMQZ128rr : X86::VPTESTMQZ128rr;
-  case MVT::v32i8:
-    return IsTestN ? X86::VPTESTNMBZ256rr : X86::VPTESTMBZ256rr;
-  case MVT::v16i16:
-    return IsTestN ? X86::VPTESTNMWZ256rr : X86::VPTESTMWZ256rr;
-  case MVT::v8i32:
-    return IsTestN ? X86::VPTESTNMDZ256rr : X86::VPTESTMDZ256rr;
-  case MVT::v4i64:
-    return IsTestN ? X86::VPTESTNMQZ256rr : X86::VPTESTMQZ256rr;
-  case MVT::v64i8:
-    return IsTestN ? X86::VPTESTNMBZrr : X86::VPTESTMBZrr;
-  case MVT::v32i16:
-    return IsTestN ? X86::VPTESTNMWZrr : X86::VPTESTMWZrr;
-  case MVT::v16i32:
-    return IsTestN ? X86::VPTESTNMDZrr : X86::VPTESTMDZrr;
-  case MVT::v8i64:
-    return IsTestN ? X86::VPTESTNMQZrr : X86::VPTESTMQZrr;
+  VPTESTM_FULL_CASES(rr)
   }
+
+#undef VPTESTM_FULL_CASES
+#undef VPTESTM_BROADCAST_CASES
+#undef VPTESTM_CASE
 }
 
 // Try to create VPTESTM instruction. If InMask is not null, it will be used
@@ -4464,8 +4435,39 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
 
       break;
     }
+    case Intrinsic::x86_tileloadd64:
+    case Intrinsic::x86_tileloaddt164:
+    case Intrinsic::x86_tilestored64: {
+      if (!Subtarget->hasAMXTILE())
+        break;
+      unsigned Opc;
+      switch (IntNo) {
+      default: llvm_unreachable("Unexpected intrinsic!");
+      case Intrinsic::x86_tileloadd64:   Opc = X86::PTILELOADD; break;
+      case Intrinsic::x86_tileloaddt164: Opc = X86::PTILELOADDT1; break;
+      case Intrinsic::x86_tilestored64:  Opc = X86::PTILESTORED; break;
+      }
+      // FIXME: Match displacement and scale.
+      unsigned TIndex = Node->getConstantOperandVal(2);
+      SDValue TReg = getI8Imm(TIndex, dl);
+      SDValue Base = Node->getOperand(3);
+      SDValue Scale = getI8Imm(1, dl);
+      SDValue Index = Node->getOperand(4);
+      SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
+      SDValue Segment = CurDAG->getRegister(0, MVT::i16);
+      SDValue Chain = Node->getOperand(0);
+      MachineSDNode *CNode;
+      if (Opc == X86::PTILESTORED) {
+        SDValue Ops[] = { Base, Scale, Index, Disp, Segment, TReg, Chain };
+        CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
+      } else {
+        SDValue Ops[] = { TReg, Base, Scale, Index, Disp, Segment, Chain };
+        CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
+      }
+      ReplaceNode(Node, CNode);
+      return;
+    }
     }
-
     break;
   }
   case ISD::BRIND: {
@@ -4541,9 +4543,10 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
   case ISD::XOR:
     if (tryShrinkShlLogicImm(Node))
       return;
-
     if (Opcode == ISD::OR && tryMatchBitSelect(Node))
       return;
+    if (tryVPTERNLOG(Node))
+      return;
 
     LLVM_FALLTHROUGH;
   case ISD::ADD:
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 90de46b651024..450927aaf5cc7 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -1039,8 +1039,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::ROTL,               MVT::v4i32, Custom);
     setOperationAction(ISD::ROTL,               MVT::v8i16, Custom);
 
-    // With AVX512, expanding (and promoting the shifts) is better.
-    if (!Subtarget.hasAVX512())
+    // With 512-bit registers or AVX512VL+BW, expanding (and promoting the
+    // shifts) is better.
+    if (!Subtarget.useAVX512Regs() &&
+        !(Subtarget.hasBWI() && Subtarget.hasVLX()))
       setOperationAction(ISD::ROTL,             MVT::v16i8, Custom);
 
     setOperationAction(ISD::STRICT_FSQRT,       MVT::v2f64, Legal);
@@ -1227,7 +1229,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::ROTL,              MVT::v16i16, Custom);
 
     // With BWI, expanding (and promoting the shifts) is the better.
-    if (!Subtarget.hasBWI())
+    if (!Subtarget.useBWIRegs())
       setOperationAction(ISD::ROTL,            MVT::v32i8,  Custom);
 
     setOperationAction(ISD::SELECT,            MVT::v4f64, Custom);
@@ -2187,20 +2189,20 @@ EVT X86TargetLowering::getSetCCResultType(const DataLayout &DL,
 
 /// Helper for getByValTypeAlignment to determine
 /// the desired ByVal argument alignment.
-static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign) {
+static void getMaxByValAlign(Type *Ty, Align &MaxAlign) {
   if (MaxAlign == 16)
     return;
   if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
     if (VTy->getPrimitiveSizeInBits().getFixedSize() == 128)
-      MaxAlign = 16;
+      MaxAlign = Align(16);
   } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
-    unsigned EltAlign = 0;
+    Align EltAlign;
     getMaxByValAlign(ATy->getElementType(), EltAlign);
     if (EltAlign > MaxAlign)
       MaxAlign = EltAlign;
   } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
     for (auto *EltTy : STy->elements()) {
-      unsigned EltAlign = 0;
+      Align EltAlign;
       getMaxByValAlign(EltTy, EltAlign);
       if (EltAlign > MaxAlign)
         MaxAlign = EltAlign;
@@ -2218,16 +2220,16 @@ unsigned X86TargetLowering::getByValTypeAlignment(Type *Ty,
                                                   const DataLayout &DL) const {
   if (Subtarget.is64Bit()) {
     // Max of 8 and alignment of type.
-    unsigned TyAlign = DL.getABITypeAlignment(Ty);
+    Align TyAlign = DL.getABITypeAlign(Ty);
     if (TyAlign > 8)
-      return TyAlign;
+      return TyAlign.value();
     return 8;
   }
 
-  unsigned Align = 4;
+  Align Alignment(4);
   if (Subtarget.hasSSE1())
-    getMaxByValAlign(Ty, Align);
-  return Align;
+    getMaxByValAlign(Ty, Alignment);
+  return Alignment.value();
 }
 
 /// It returns EVT::Other if the type should be determined using generic
@@ -2592,7 +2594,7 @@ static SDValue lowerMasksToReg(const SDValue &ValArg, const EVT &ValLoc,
 /// Breaks v64i1 value into two registers and adds the new node to the DAG
 static void Passv64i1ArgInRegs(
     const SDLoc &Dl, SelectionDAG &DAG, SDValue &Arg,
-    SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass, CCValAssign &VA,
+    SmallVectorImpl<std::pair<Register, SDValue>> &RegsToPass, CCValAssign &VA,
     CCValAssign &NextVA, const X86Subtarget &Subtarget) {
   assert(Subtarget.hasBWI() && "Expected AVX512BW target!");
   assert(Subtarget.is32Bit() && "Expecting 32 bit target");
@@ -2637,7 +2639,7 @@ X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
   CCState CCInfo(CallConv, isVarArg, MF, RVLocs, *DAG.getContext());
   CCInfo.AnalyzeReturn(Outs, RetCC_X86);
 
-  SmallVector<std::pair<unsigned, SDValue>, 4> RetVals;
+  SmallVector<std::pair<Register, SDValue>, 4> RetVals;
   for (unsigned I = 0, OutsIndex = 0, E = RVLocs.size(); I != E;
        ++I, ++OutsIndex) {
     CCValAssign &VA = RVLocs[I];
@@ -2757,7 +2759,7 @@ X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
   // may not have an explicit sret argument. If FuncInfo.CanLowerReturn is
   // false, then an sret argument may be implicitly inserted in the SelDAG. In
   // either case FuncInfo->setSRetReturnReg() will have been called.
-  if (unsigned SRetReg = FuncInfo->getSRetReturnReg()) {
+  if (Register SRetReg = FuncInfo->getSRetReturnReg()) {
     // When we have both sret and another return value, we should use the
     // original Chain stored in RetOps[0], instead of the current Chain updated
     // in the above loop. If we only have sret, RetOps[0] equals to Chain.
@@ -2780,7 +2782,7 @@ X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
     SDValue Val = DAG.getCopyFromReg(RetOps[0], dl, SRetReg,
                                      getPointerTy(MF.getDataLayout()));
 
-    unsigned RetValReg
+    Register RetValReg
         = (Subtarget.is64Bit() && !Subtarget.isTarget64BitILP32()) ?
           X86::RAX : X86::EAX;
     Chain = DAG.getCopyToReg(Chain, dl, RetValReg, Val, Flag);
@@ -2906,7 +2908,7 @@ static SDValue getv64i1Argument(CCValAssign &VA, CCValAssign &NextVA,
   if (nullptr == InFlag) {
     // When no physical register is present,
     // create an intermediate virtual register.
-    unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
+    Register Reg = MF.addLiveIn(VA.getLocReg(), RC);
     ArgValueLo = DAG.getCopyFromReg(Root, Dl, Reg, MVT::i32);
     Reg = MF.addLiveIn(NextVA.getLocReg(), RC);
     ArgValueHi = DAG.getCopyFromReg(Root, Dl, Reg, MVT::i32);
@@ -3414,7 +3416,7 @@ void VarArgsLoweringHelper::createVarArgAreaAndStoreRegisters(
       FuncInfo->setVarArgsGPOffset(NumIntRegs * 8);
       FuncInfo->setVarArgsFPOffset(ArgGPRs.size() * 8 + NumXMMRegs * 16);
       FuncInfo->setRegSaveFrameIndex(FrameInfo.CreateStackObject(
-          ArgGPRs.size() * 8 + ArgXMMs.size() * 16, 16, false));
+          ArgGPRs.size() * 8 + ArgXMMs.size() * 16, Align(16), false));
     }
 
     SmallVector<SDValue, 6>
@@ -3425,15 +3427,15 @@ void VarArgsLoweringHelper::createVarArgAreaAndStoreRegisters(
 
     // Gather all the live in physical registers.
     for (MCPhysReg Reg : ArgGPRs.slice(NumIntRegs)) {
-      unsigned GPR = TheMachineFunction.addLiveIn(Reg, &X86::GR64RegClass);
+      Register GPR = TheMachineFunction.addLiveIn(Reg, &X86::GR64RegClass);
       LiveGPRs.push_back(DAG.getCopyFromReg(Chain, DL, GPR, MVT::i64));
     }
     const auto &AvailableXmms = ArgXMMs.slice(NumXMMRegs);
     if (!AvailableXmms.empty()) {
-      unsigned AL = TheMachineFunction.addLiveIn(X86::AL, &X86::GR8RegClass);
+      Register AL = TheMachineFunction.addLiveIn(X86::AL, &X86::GR8RegClass);
       ALVal = DAG.getCopyFromReg(Chain, DL, AL, MVT::i8);
       for (MCPhysReg Reg : AvailableXmms) {
-        unsigned XMMReg = TheMachineFunction.addLiveIn(Reg, &X86::VR128RegClass);
+        Register XMMReg = TheMachineFunction.addLiveIn(Reg, &X86::VR128RegClass);
         LiveXMMRegs.push_back(
             DAG.getCopyFromReg(Chain, DL, XMMReg, MVT::v4f32));
       }
@@ -3505,7 +3507,7 @@ void VarArgsLoweringHelper::forwardMustTailParameters(SDValue &Chain) {
 
   // Forward AL for SysV x86_64 targets, since it is used for varargs.
   if (is64Bit() && !isWin64() && !CCInfo.isAllocated(X86::AL)) {
-    unsigned ALVReg = TheMachineFunction.addLiveIn(X86::AL, &X86::GR8RegClass);
+    Register ALVReg = TheMachineFunction.addLiveIn(X86::AL, &X86::GR8RegClass);
     Forwards.push_back(ForwardedRegister(ALVReg, X86::AL, MVT::i8));
   }
 
@@ -3630,7 +3632,7 @@ SDValue X86TargetLowering::LowerFormalArguments(
         else
           llvm_unreachable("Unknown argument type!");
 
-        unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
+        Register Reg = MF.addLiveIn(VA.getLocReg(), RC);
         ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
       }
 
@@ -3684,7 +3686,7 @@ SDValue X86TargetLowering::LowerFormalArguments(
     // the argument into a virtual register so that we can access it from the
     // return points.
     if (Ins[I].Flags.isSRet()) {
-      unsigned Reg = FuncInfo->getSRetReturnReg();
+      Register Reg = FuncInfo->getSRetReturnReg();
       if (!Reg) {
         MVT PtrTy = getPointerTy(DAG.getDataLayout());
         Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrTy));
@@ -3742,7 +3744,7 @@ SDValue X86TargetLowering::LowerFormalArguments(
       // same, so the size of funclets' (mostly empty) frames is dictated by
       // how far this slot is from the bottom (since they allocate just enough
       // space to accommodate holding this slot at the correct offset).
-      int PSPSymFI = MFI.CreateStackObject(8, 8, /*isSS=*/false);
+      int PSPSymFI = MFI.CreateStackObject(8, Align(8), /*isSS=*/false);
       EHInfo->PSPSymFrameIdx = PSPSymFI;
     }
   }
@@ -3750,7 +3752,7 @@ SDValue X86TargetLowering::LowerFormalArguments(
   if (CallConv == CallingConv::X86_RegCall ||
       F.hasFnAttribute("no_caller_saved_registers")) {
     MachineRegisterInfo &MRI = MF.getRegInfo();
-    for (std::pair<unsigned, unsigned> Pair : MRI.liveins())
+    for (std::pair<Register, Register> Pair : MRI.liveins())
       MRI.disableCalleeSavedRegister(Pair.first);
   }
 
@@ -3761,12 +3763,13 @@ SDValue X86TargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,
                                             SDValue Arg, const SDLoc &dl,
                                             SelectionDAG &DAG,
                                             const CCValAssign &VA,
-                                            ISD::ArgFlagsTy Flags) const {
+                                            ISD::ArgFlagsTy Flags,
+                                            bool isByVal) const {
   unsigned LocMemOffset = VA.getLocMemOffset();
   SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
   PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
                        StackPtr, PtrOff);
-  if (Flags.isByVal())
+  if (isByVal)
     return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl);
 
   return DAG.getStore(
@@ -3971,7 +3974,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, isTailCall,
                                     Is64Bit, FPDiff, dl);
 
-  SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
+  SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
   SmallVector<SDValue, 8> MemOpChains;
   SDValue StackPtr;
 
@@ -4062,7 +4065,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       if (isVarArg && IsWin64) {
         // Win64 ABI requires argument XMM reg to be copied to the corresponding
         // shadow reg if callee is a varargs function.
-        unsigned ShadowReg = 0;
+        Register ShadowReg;
         switch (VA.getLocReg()) {
         case X86::XMM0: ShadowReg = X86::RCX; break;
         case X86::XMM1: ShadowReg = X86::RDX; break;
@@ -4078,7 +4081,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         StackPtr = DAG.getCopyFromReg(Chain, dl, RegInfo->getStackRegister(),
                                       getPointerTy(DAG.getDataLayout()));
       MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg,
-                                             dl, DAG, VA, Flags));
+                                             dl, DAG, VA, Flags, isByVal));
     }
   }
 
@@ -4090,7 +4093,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     // GOT pointer.
     if (!isTailCall) {
       RegsToPass.push_back(std::make_pair(
-          unsigned(X86::EBX), DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(),
+          Register(X86::EBX), DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(),
                                           getPointerTy(DAG.getDataLayout()))));
     } else {
       // If we are tail calling and generating PIC/GOT style code load the
@@ -4128,7 +4131,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs);
     assert((Subtarget.hasSSE1() || !NumXMMRegs)
            && "SSE registers cannot be used when SSE is disabled");
-    RegsToPass.push_back(std::make_pair(unsigned(X86::AL),
+    RegsToPass.push_back(std::make_pair(Register(X86::AL),
                                         DAG.getConstant(NumXMMRegs, dl,
                                                         MVT::i8)));
   }
@@ -4137,7 +4140,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     const auto &Forwards = X86Info->getForwardedMustTailRegParms();
     for (const auto &F : Forwards) {
       SDValue Val = DAG.getCopyFromReg(Chain, dl, F.VReg, F.VT);
-      RegsToPass.push_back(std::make_pair(unsigned(F.PReg), Val));
+      RegsToPass.push_back(std::make_pair(F.PReg, Val));
     }
   }
 
@@ -4448,7 +4451,7 @@ bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
 
   int FI = INT_MAX;
   if (Arg.getOpcode() == ISD::CopyFromReg) {
-    unsigned VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg();
+    Register VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg();
     if (!Register::isVirtualRegister(VR))
       return false;
     MachineInstr *Def = MRI->getVRegDef(VR);
@@ -5586,6 +5589,36 @@ static bool canWidenShuffleElements(ArrayRef<int> Mask) {
   return canWidenShuffleElements(Mask, WidenedMask);
 }
 
+// Attempt to narrow/widen shuffle mask until it matches the target number of
+// elements.
+static bool scaleShuffleElements(ArrayRef<int> Mask, unsigned NumDstElts,
+                                 SmallVectorImpl<int> &ScaledMask) {
+  unsigned NumSrcElts = Mask.size();
+  assert(((NumSrcElts % NumDstElts) == 0 || (NumDstElts % NumSrcElts) == 0) &&
+         "Illegal shuffle scale factor");
+
+  // Narrowing is guaranteed to work.
+  if (NumDstElts >= NumSrcElts) {
+    int Scale = NumDstElts / NumSrcElts;
+    llvm::narrowShuffleMaskElts(Scale, Mask, ScaledMask);
+    return true;
+  }
+
+  // We have to repeat the widening until we reach the target size, but we can
+  // split out the first widening as it sets up ScaledMask for us.
+  if (canWidenShuffleElements(Mask, ScaledMask)) {
+    while (ScaledMask.size() > NumDstElts) {
+      SmallVector<int, 16> WidenedMask;
+      if (!canWidenShuffleElements(ScaledMask, WidenedMask))
+        return false;
+      ScaledMask = std::move(WidenedMask);
+    }
+    return true;
+  }
+
+  return false;
+}
+
 /// Returns true if Elt is a constant zero or a floating point constant +0.0.
 bool X86::isZeroNode(SDValue Elt) {
   return isNullConstant(Elt) || isNullFPConstant(Elt);
@@ -7969,10 +8002,11 @@ static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros,
         Elt = NextElt;
     }
 
-    // If our first insertion is not the first index then insert into zero
-    // vector to break any register dependency else use SCALAR_TO_VECTOR.
+    // If our first insertion is not the first index or zeros are needed, then
+    // insert into zero vector. Otherwise, use SCALAR_TO_VECTOR (leaves high
+    // elements undefined).
     if (!V) {
-      if (i != 0)
+      if (i != 0 || NumZero)
         V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl);
       else {
         V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Elt);
@@ -9655,6 +9689,9 @@ static SDValue LowerToHorizontalOp(const BuildVectorSDNode *BV,
   return SDValue();
 }
 
+static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
+                          SelectionDAG &DAG);
+
 /// If a BUILD_VECTOR's source elements all apply the same bit operation and
 /// one of their operands is constant, lower to a pair of BUILD_VECTOR and
 /// just apply the bit to the vectors.
@@ -9662,6 +9699,7 @@ static SDValue LowerToHorizontalOp(const BuildVectorSDNode *BV,
 /// from this, but enough scalar bit operations are created from the later
 /// legalization + scalarization stages to need basic support.
 static SDValue lowerBuildVectorToBitOp(BuildVectorSDNode *Op,
+                                       const X86Subtarget &Subtarget,
                                        SelectionDAG &DAG) {
   SDLoc DL(Op);
   MVT VT = Op->getSimpleValueType(0);
@@ -9725,7 +9763,14 @@ static SDValue lowerBuildVectorToBitOp(BuildVectorSDNode *Op,
 
   SDValue LHS = DAG.getBuildVector(VT, DL, LHSElts);
   SDValue RHS = DAG.getBuildVector(VT, DL, RHSElts);
-  return DAG.getNode(Opcode, DL, VT, LHS, RHS);
+  SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS);
+
+  if (!IsShift)
+    return Res;
+
+  // Immediately lower the shift to ensure the constant build vector doesn't
+  // get converted to a constant pool before the shift is lowered.
+  return LowerShift(Res, Subtarget, DAG);
 }
 
 /// Create a vector constant without a load. SSE/AVX provide the bare minimum
@@ -10081,7 +10126,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
     return HorizontalOp;
   if (SDValue Broadcast = lowerBuildVectorAsBroadcast(BV, Subtarget, DAG))
     return Broadcast;
-  if (SDValue BitOp = lowerBuildVectorToBitOp(BV, DAG))
+  if (SDValue BitOp = lowerBuildVectorToBitOp(BV, Subtarget, DAG))
     return BitOp;
 
   unsigned EVTBits = EltVT.getSizeInBits();
@@ -11699,6 +11744,12 @@ static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
       return getVectorMaskingNode(V2, MaskNode, V1, Subtarget, DAG);
     }
 
+    // If we have VPTERNLOG, we can use that as a bit blend.
+    if (Subtarget.hasVLX())
+      if (SDValue BitBlend =
+              lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
+        return BitBlend;
+
     // Scale the blend by the number of bytes per element.
     int Scale = VT.getScalarSizeInBits() / 8;
 
@@ -17151,6 +17202,12 @@ static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask, V1, V2, DAG);
   }
 
+  // Try to create an in-lane repeating shuffle mask and then shuffle the
+  // results into the target lanes.
+  if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
+          DL, MVT::v16f32, V1, V2, Mask, Subtarget, DAG))
+    return V;
+
   // If we have a single input shuffle with different shuffle patterns in the
   // 128-bit lanes and don't lane cross, use variable mask VPERMILPS.
   if (V2.isUndef() &&
@@ -17288,6 +17345,13 @@ static SDValue lowerV16I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                                             CastV1, CastV2, DAG);
     return DAG.getBitcast(MVT::v16i32, ShufPS);
   }
+
+  // Try to create an in-lane repeating shuffle mask and then shuffle the
+  // results into the target lanes.
+  if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
+          DL, MVT::v16i32, V1, V2, Mask, Subtarget, DAG))
+    return V;
+
   // If we have AVX512F support, we can use VEXPAND.
   if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16i32, Zeroable, Mask, V1, V2,
                                        DAG, Subtarget))
@@ -17296,6 +17360,7 @@ static SDValue lowerV16I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i32, V1, V2, Mask,
                                           Zeroable, Subtarget, DAG))
     return Blend;
+
   return lowerShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, DAG);
 }
 
@@ -17404,6 +17469,11 @@ static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
             lowerShuffleAsBitRotate(DL, MVT::v64i8, V1, Mask, Subtarget, DAG))
       return Rotate;
 
+  // Lower as AND if possible.
+  if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v64i8, V1, V2, Mask,
+                                             Zeroable, Subtarget, DAG))
+    return Masked;
+
   if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v64i8, Mask, V1, V2,
                                               Zeroable, Subtarget, DAG))
     return PSHUFB;
@@ -17466,8 +17536,17 @@ static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
                                                   Subtarget, DAG))
     return Broadcast;
 
-  if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())
+  if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) {
+    // Try using bit ops for masking and blending before falling back to
+    // splitting.
+    if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
+                                          Subtarget, DAG))
+      return V;
+    if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
+      return V;
+
     return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG);
+  }
 
   // Dispatch to each element type for lowering. If we don't have support for
   // specific element type shuffles at 512 bits, immediately split them and
@@ -19439,7 +19518,8 @@ std::pair<SDValue, SDValue> X86TargetLowering::BuildFILD(
   if (useSSE) {
     MachineFunction &MF = DAG.getMachineFunction();
     unsigned SSFISize = DstVT.getStoreSize();
-    int SSFI = MF.getFrameInfo().CreateStackObject(SSFISize, SSFISize, false);
+    int SSFI =
+        MF.getFrameInfo().CreateStackObject(SSFISize, Align(SSFISize), false);
     auto PtrVT = getPointerTy(MF.getDataLayout());
     SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
     Tys = DAG.getVTList(MVT::Other);
@@ -20008,7 +20088,8 @@ X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
   // stack slot.
   MachineFunction &MF = DAG.getMachineFunction();
   unsigned MemSize = DstTy.getStoreSize();
-  int SSFI = MF.getFrameInfo().CreateStackObject(MemSize, MemSize, false);
+  int SSFI =
+      MF.getFrameInfo().CreateStackObject(MemSize, Align(MemSize), false);
   SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
 
   Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
@@ -20270,7 +20351,7 @@ static SDValue truncateVectorWithPACK(unsigned Opcode, EVT DstVT, SDValue In,
          "Unexpected PACK opcode");
   assert(DstVT.isVector() && "VT not a vector?");
 
-  // Requires SSE2 but AVX512 has fast vector truncate.
+  // Requires SSE2 for PACKSS (SSE41 PACKUSDW is handled below).
   if (!Subtarget.hasSSE2())
     return SDValue();
 
@@ -21356,19 +21437,31 @@ static bool matchScalarReduction(SDValue Op, ISD::NodeType BinOp,
 
 // Helper function for comparing all bits of a vector against zero.
 static SDValue LowerVectorAllZero(const SDLoc &DL, SDValue V, ISD::CondCode CC,
+                                  const APInt &Mask,
                                   const X86Subtarget &Subtarget,
                                   SelectionDAG &DAG, X86::CondCode &X86CC) {
   EVT VT = V.getValueType();
+  assert(Mask.getBitWidth() == VT.getScalarSizeInBits() &&
+         "Element Mask vs Vector bitwidth mismatch");
 
   assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode");
   X86CC = (CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE);
 
+  auto MaskBits = [&](SDValue Src) {
+    if (Mask.isAllOnesValue())
+      return Src;
+    EVT SrcVT = Src.getValueType();
+    SDValue MaskValue = DAG.getConstant(Mask, DL, SrcVT);
+    return DAG.getNode(ISD::AND, DL, SrcVT, Src, MaskValue);
+  };
+
   // For sub-128-bit vector, cast to (legal) integer and compare with zero.
   if (VT.getSizeInBits() < 128) {
     EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
     if (!DAG.getTargetLoweringInfo().isTypeLegal(IntVT))
       return SDValue();
-    return DAG.getNode(X86ISD::CMP, DL, MVT::i32, DAG.getBitcast(IntVT, V),
+    return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
+                       DAG.getBitcast(IntVT, MaskBits(V)),
                        DAG.getConstant(0, DL, IntVT));
   }
 
@@ -21387,11 +21480,16 @@ static SDValue LowerVectorAllZero(const SDLoc &DL, SDValue V, ISD::CondCode CC,
   bool UsePTEST = Subtarget.hasSSE41();
   if (UsePTEST) {
     MVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
-    V = DAG.getBitcast(TestVT, V);
+    V = DAG.getBitcast(TestVT, MaskBits(V));
     return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, V, V);
   }
 
-  V = DAG.getBitcast(MVT::v16i8, V);
+  // Without PTEST, a masked v2i64 or-reduction is not faster than
+  // scalarization.
+  if (!Mask.isAllOnesValue() && VT.getScalarSizeInBits() > 32)
+      return SDValue();
+
+  V = DAG.getBitcast(MVT::v16i8, MaskBits(V));
   V = DAG.getNode(X86ISD::PCMPEQ, DL, MVT::v16i8, V,
                   getZeroVector(MVT::v16i8, Subtarget, DAG, DL));
   V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
@@ -21410,6 +21508,26 @@ static SDValue MatchVectorAllZeroTest(SDValue Op, ISD::CondCode CC,
   if (!Subtarget.hasSSE2() || !Op->hasOneUse())
     return SDValue();
 
+  // Check whether we're masking/truncating an OR-reduction result, in which
+  // case track the masked bits.
+  APInt Mask = APInt::getAllOnesValue(Op.getScalarValueSizeInBits());
+  switch (Op.getOpcode()) {
+  case ISD::TRUNCATE: {
+    SDValue Src = Op.getOperand(0);
+    Mask = APInt::getLowBitsSet(Src.getScalarValueSizeInBits(),
+                                Op.getScalarValueSizeInBits());
+    Op = Src;
+    break;
+  }
+  case ISD::AND: {
+    if (auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+      Mask = Cst->getAPIntValue();
+      Op = Op.getOperand(0);
+    }
+    break;
+  }
+  }
+
   SmallVector<SDValue, 8> VecIns;
   if (Op.getOpcode() == ISD::OR && matchScalarReduction(Op, ISD::OR, VecIns)) {
     EVT VT = VecIns[0].getValueType();
@@ -21432,8 +21550,8 @@ static SDValue MatchVectorAllZeroTest(SDValue Op, ISD::CondCode CC,
     }
 
     X86::CondCode CCode;
-    if (SDValue V =
-            LowerVectorAllZero(DL, VecIns.back(), CC, Subtarget, DAG, CCode)) {
+    if (SDValue V = LowerVectorAllZero(DL, VecIns.back(), CC, Mask, Subtarget,
+                                       DAG, CCode)) {
       X86CC = DAG.getTargetConstant(CCode, DL, MVT::i8);
       return V;
     }
@@ -21445,7 +21563,7 @@ static SDValue MatchVectorAllZeroTest(SDValue Op, ISD::CondCode CC,
             DAG.matchBinOpReduction(Op.getNode(), BinOp, {ISD::OR})) {
       X86::CondCode CCode;
       if (SDValue V =
-              LowerVectorAllZero(DL, Match, CC, Subtarget, DAG, CCode)) {
+              LowerVectorAllZero(DL, Match, CC, Mask, Subtarget, DAG, CCode)) {
         X86CC = DAG.getTargetConstant(CCode, DL, MVT::i8);
         return V;
       }
@@ -25118,6 +25236,9 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
       // Clamp out of bounds shift amounts since they will otherwise be masked
       // to 8-bits which may make it no longer out of bounds.
       unsigned ShiftAmount = C->getAPIntValue().getLimitedValue(255);
+      if (ShiftAmount == 0)
+        return Op.getOperand(1);
+
       return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
                          Op.getOperand(0), Op.getOperand(1),
                          DAG.getTargetConstant(ShiftAmount, DL, MVT::i32));
@@ -26044,8 +26165,7 @@ SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
   SDLoc DL(Op);
 
   // Save FP Control Word to stack slot
-  int SSFI =
-      MF.getFrameInfo().CreateStackObject(2, 2, false);
+  int SSFI = MF.getFrameInfo().CreateStackObject(2, Align(2), false);
   SDValue StackSlot =
       DAG.getFrameIndex(SSFI, getPointerTy(DAG.getDataLayout()));
 
@@ -27452,12 +27572,13 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
                            ISD::SETGT);
         return DAG.getBitcast(SelVT, DAG.getSelect(dl, VT, Sel, V0, V1));
       } else if (Subtarget.hasSSE41()) {
-        // On SSE41 targets we make use of the fact that VSELECT lowers
-        // to PBLENDVB which selects bytes based just on the sign bit.
+        // On SSE41 targets we can use PBLENDVB which selects bytes based just
+        // on the sign bit.
         V0 = DAG.getBitcast(VT, V0);
         V1 = DAG.getBitcast(VT, V1);
         Sel = DAG.getBitcast(VT, Sel);
-        return DAG.getBitcast(SelVT, DAG.getSelect(dl, VT, Sel, V0, V1));
+        return DAG.getBitcast(SelVT,
+                              DAG.getNode(X86ISD::BLENDV, dl, VT, Sel, V0, V1));
       }
       // On pre-SSE41 targets we test for the sign bit by comparing to
       // zero - a negative value will set all bits of the lanes to true
@@ -27567,14 +27688,15 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
                     !ISD::isBuildVectorOfConstantSDNodes(Amt.getNode());
 
     auto SignBitSelect = [&](SDValue Sel, SDValue V0, SDValue V1) {
-      // On SSE41 targets we make use of the fact that VSELECT lowers
-      // to PBLENDVB which selects bytes based just on the sign bit.
+      // On SSE41 targets we can use PBLENDVB which selects bytes based just on
+      // the sign bit.
       if (UseSSE41) {
         MVT ExtVT = MVT::getVectorVT(MVT::i8, VT.getVectorNumElements() * 2);
         V0 = DAG.getBitcast(ExtVT, V0);
         V1 = DAG.getBitcast(ExtVT, V1);
         Sel = DAG.getBitcast(ExtVT, Sel);
-        return DAG.getBitcast(VT, DAG.getSelect(dl, ExtVT, Sel, V0, V1));
+        return DAG.getBitcast(
+            VT, DAG.getNode(X86ISD::BLENDV, dl, ExtVT, Sel, V0, V1));
       }
       // On pre-SSE41 targets we splat the sign bit - a negative value will
       // set all bits of the lanes to true and VSELECT uses that in
@@ -27714,12 +27836,13 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
 
     auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {
       if (Subtarget.hasSSE41()) {
-        // On SSE41 targets we make use of the fact that VSELECT lowers
-        // to PBLENDVB which selects bytes based just on the sign bit.
+        // On SSE41 targets we can use PBLENDVB which selects bytes based just
+        // on the sign bit.
         V0 = DAG.getBitcast(VT, V0);
         V1 = DAG.getBitcast(VT, V1);
         Sel = DAG.getBitcast(VT, Sel);
-        return DAG.getBitcast(SelVT, DAG.getSelect(DL, VT, Sel, V0, V1));
+        return DAG.getBitcast(SelVT,
+                              DAG.getNode(X86ISD::BLENDV, DL, VT, Sel, V0, V1));
       }
       // On pre-SSE41 targets we test for the sign bit by comparing to
       // zero - a negative value will set all bits of the lanes to true
@@ -30942,7 +31065,7 @@ X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr &MI,
   MachineOperand &Segment = MI.getOperand(5);
   unsigned ArgSize = MI.getOperand(6).getImm();
   unsigned ArgMode = MI.getOperand(7).getImm();
-  unsigned Align = MI.getOperand(8).getImm();
+  Align Alignment = Align(MI.getOperand(8).getImm());
 
   MachineFunction *MF = MBB->getParent();
 
@@ -30982,7 +31105,7 @@ X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr &MI,
 
   /* Align ArgSize to a multiple of 8 */
   unsigned ArgSizeA8 = (ArgSize + 7) & ~7;
-  bool NeedsAlign = (Align > 8);
+  bool NeedsAlign = (Alignment > 8);
 
   MachineBasicBlock *thisMBB = MBB;
   MachineBasicBlock *overflowMBB;
@@ -31130,17 +31253,16 @@ X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr &MI,
   // to OverflowDestReg.
   if (NeedsAlign) {
     // Align the overflow address
-    assert(isPowerOf2_32(Align) && "Alignment must be a power of 2");
     Register TmpReg = MRI.createVirtualRegister(AddrRegClass);
 
     // aligned_addr = (addr + (align-1)) & ~(align-1)
     BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), TmpReg)
-      .addReg(OverflowAddrReg)
-      .addImm(Align-1);
+        .addReg(OverflowAddrReg)
+        .addImm(Alignment.value() - 1);
 
     BuildMI(overflowMBB, DL, TII->get(X86::AND64ri32), OverflowDestReg)
-      .addReg(TmpReg)
-      .addImm(~(uint64_t)(Align-1));
+        .addReg(TmpReg)
+        .addImm(~(uint64_t)(Alignment.value() - 1));
   } else {
     BuildMI(overflowMBB, DL, TII->get(TargetOpcode::COPY), OverflowDestReg)
       .addReg(OverflowAddrReg);
@@ -31733,8 +31855,8 @@ X86TargetLowering::EmitLoweredProbedAlloca(MachineInstr &MI,
 
   BuildMI(testMBB, DL,
           TII->get(TFI.Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
-      .addReg(physSPReg)
-      .addReg(FinalStackPtr);
+      .addReg(FinalStackPtr)
+      .addReg(physSPReg);
 
   BuildMI(testMBB, DL, TII->get(X86::JCC_1))
       .addMBB(tailMBB)
@@ -32455,12 +32577,17 @@ X86TargetLowering::emitLongJmpShadowStackFix(MachineInstr &MI,
   MBB->addSuccessor(checkSspMBB);
 
   // Initialize a register with zero.
-  Register ZReg = MRI.createVirtualRegister(PtrRC);
-  unsigned XorRROpc = (PVT == MVT::i64) ? X86::XOR64rr : X86::XOR32rr;
-  BuildMI(checkSspMBB, DL, TII->get(XorRROpc))
-      .addDef(ZReg)
-      .addReg(ZReg, RegState::Undef)
-      .addReg(ZReg, RegState::Undef);
+  Register ZReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+  BuildMI(checkSspMBB, DL, TII->get(X86::MOV32r0), ZReg);
+
+  if (PVT == MVT::i64) {
+    Register TmpZReg = MRI.createVirtualRegister(PtrRC);
+    BuildMI(checkSspMBB, DL, TII->get(X86::SUBREG_TO_REG), TmpZReg)
+      .addImm(0)
+      .addReg(ZReg)
+      .addImm(X86::sub_32bit);
+    ZReg = TmpZReg;
+  }
 
   // Read the current SSP Register value to the zeroed register.
   Register SSPCopyReg = MRI.createVirtualRegister(PtrRC);
@@ -32589,7 +32716,7 @@ X86TargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
   Register Tmp = MRI.createVirtualRegister(RC);
   // Since FP is only updated here but NOT referenced, it's treated as GPR.
   const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
-  unsigned FP = (PVT == MVT::i64) ? X86::RBP : X86::EBP;
+  Register FP = (PVT == MVT::i64) ? X86::RBP : X86::EBP;
   Register SP = RegInfo->getStackRegister();
 
   MachineInstrBuilder MIB;
@@ -32936,6 +33063,10 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
   DebugLoc DL = MI.getDebugLoc();
 
+  auto TMMImmToTMMReg = [](unsigned Imm) {
+    assert (Imm < 8 && "Illegal tmm index");
+    return X86::TMM0 + Imm;
+  };
   switch (MI.getOpcode()) {
   default: llvm_unreachable("Unexpected instr type to insert");
   case X86::TLS_addr32:
@@ -33030,7 +33161,8 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   case X86::FP80_TO_INT64_IN_MEM: {
     // Change the floating point control register to use "round towards zero"
     // mode when truncating to an integer value.
-    int OrigCWFrameIdx = MF->getFrameInfo().CreateStackObject(2, 2, false);
+    int OrigCWFrameIdx =
+        MF->getFrameInfo().CreateStackObject(2, Align(2), false);
     addFrameReference(BuildMI(*BB, MI, DL,
                               TII->get(X86::FNSTCW16m)), OrigCWFrameIdx);
 
@@ -33051,7 +33183,8 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
       .addReg(NewCW, RegState::Kill, X86::sub_16bit);
 
     // Prepare memory for FLDCW.
-    int NewCWFrameIdx = MF->getFrameInfo().CreateStackObject(2, 2, false);
+    int NewCWFrameIdx =
+        MF->getFrameInfo().CreateStackObject(2, Align(2), false);
     addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOV16mr)),
                       NewCWFrameIdx)
       .addReg(NewCW16, RegState::Kill);
@@ -33216,6 +33349,67 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
     MI.eraseFromParent();
     return BB;
   }
+  case X86::PTDPBSSD:
+  case X86::PTDPBSUD:
+  case X86::PTDPBUSD:
+  case X86::PTDPBUUD:
+  case X86::PTDPBF16PS: {
+    const DebugLoc &DL = MI.getDebugLoc();
+    unsigned Opc;
+    switch (MI.getOpcode()) {
+    case X86::PTDPBSSD: Opc = X86::TDPBSSD; break;
+    case X86::PTDPBSUD: Opc = X86::TDPBSUD; break;
+    case X86::PTDPBUSD: Opc = X86::TDPBUSD; break;
+    case X86::PTDPBUUD: Opc = X86::TDPBUUD; break;
+    case X86::PTDPBF16PS: Opc = X86::TDPBF16PS; break;
+    }
+
+    MachineInstrBuilder MIB = BuildMI(*BB, MI, DL, TII->get(Opc));
+    MIB.addReg(TMMImmToTMMReg(MI.getOperand(0).getImm()), RegState::Define);
+    MIB.addReg(TMMImmToTMMReg(MI.getOperand(0).getImm()), RegState::Undef);
+    MIB.addReg(TMMImmToTMMReg(MI.getOperand(1).getImm()), RegState::Undef);
+    MIB.addReg(TMMImmToTMMReg(MI.getOperand(2).getImm()), RegState::Undef);
+
+    MI.eraseFromParent(); // The pseudo is gone now.
+    return BB;
+  }
+  case X86::PTILEZERO: {
+    const DebugLoc &DL = MI.getDebugLoc();
+    unsigned Imm = MI.getOperand(0).getImm();
+    BuildMI(*BB, MI, DL, TII->get(X86::TILEZERO), TMMImmToTMMReg(Imm));
+    MI.eraseFromParent(); // The pseudo is gone now.
+    return BB;
+  }
+  case X86::PTILELOADD:
+  case X86::PTILELOADDT1:
+  case X86::PTILESTORED: {
+    const DebugLoc &DL = MI.getDebugLoc();
+    unsigned Opc;
+    switch (MI.getOpcode()) {
+    case X86::PTILELOADD:   Opc = X86::TILELOADD;   break;
+    case X86::PTILELOADDT1: Opc = X86::TILELOADDT1; break;
+    case X86::PTILESTORED:  Opc = X86::TILESTORED;  break;
+    }
+
+    MachineInstrBuilder MIB = BuildMI(*BB, MI, DL, TII->get(Opc));
+    unsigned CurOp = 0;
+    if (Opc != X86::TILESTORED)
+      MIB.addReg(TMMImmToTMMReg(MI.getOperand(CurOp++).getImm()),
+                 RegState::Define);
+
+    MIB.add(MI.getOperand(CurOp++)); // base
+    MIB.add(MI.getOperand(CurOp++)); // scale
+    MIB.add(MI.getOperand(CurOp++)); // index -- stride
+    MIB.add(MI.getOperand(CurOp++)); // displacement
+    MIB.add(MI.getOperand(CurOp++)); // segment
+
+    if (Opc == X86::TILESTORED)
+      MIB.addReg(TMMImmToTMMReg(MI.getOperand(CurOp++).getImm()),
+                 RegState::Undef);
+
+    MI.eraseFromParent(); // The pseudo is gone now.
+    return BB;
+  }
   }
 }
 
@@ -33225,20 +33419,53 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
 
 bool
 X86TargetLowering::targetShrinkDemandedConstant(SDValue Op,
-                                                const APInt &Demanded,
+                                                const APInt &DemandedBits,
+                                                const APInt &DemandedElts,
                                                 TargetLoweringOpt &TLO) const {
-  // Only optimize Ands to prevent shrinking a constant that could be
-  // matched by movzx.
-  if (Op.getOpcode() != ISD::AND)
-    return false;
-
   EVT VT = Op.getValueType();
+  unsigned Opcode = Op.getOpcode();
+  unsigned EltSize = VT.getScalarSizeInBits();
 
-  // Ignore vectors.
-  if (VT.isVector())
+  if (VT.isVector()) {
+    // If the constant is only all signbits in the active bits, then we should
+    // extend it to the entire constant to allow it act as a boolean constant
+    // vector.
+    auto NeedsSignExtension = [&](SDValue V, unsigned ActiveBits) {
+      if (!ISD::isBuildVectorOfConstantSDNodes(V.getNode()))
+        return false;
+      for (unsigned i = 0, e = V.getNumOperands(); i != e; ++i) {
+        if (!DemandedElts[i] || V.getOperand(i).isUndef())
+          continue;
+        const APInt &Val = V.getConstantOperandAPInt(i);
+        if (Val.getBitWidth() > Val.getNumSignBits() &&
+            Val.trunc(ActiveBits).getNumSignBits() == ActiveBits)
+          return true;
+      }
+      return false;
+    };
+    // For vectors - if we have a constant, then try to sign extend.
+    // TODO: Handle AND/ANDN cases.
+    unsigned ActiveBits = DemandedBits.getActiveBits();
+    if (EltSize > ActiveBits && EltSize > 1 && isTypeLegal(VT) &&
+        (Opcode == ISD::OR || Opcode == ISD::XOR) &&
+        NeedsSignExtension(Op.getOperand(1), ActiveBits)) {
+      EVT ExtSVT = EVT::getIntegerVT(*TLO.DAG.getContext(), ActiveBits);
+      EVT ExtVT = EVT::getVectorVT(*TLO.DAG.getContext(), ExtSVT,
+                                    VT.getVectorNumElements());
+      SDValue NewC =
+          TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(Op), VT,
+                          Op.getOperand(1), TLO.DAG.getValueType(ExtVT));
+      SDValue NewOp =
+          TLO.DAG.getNode(Opcode, SDLoc(Op), VT, Op.getOperand(0), NewC);
+      return TLO.CombineTo(Op, NewOp);
+    }
     return false;
+  }
 
-  unsigned Size = VT.getSizeInBits();
+  // Only optimize Ands to prevent shrinking a constant that could be
+  // matched by movzx.
+  if (Opcode != ISD::AND)
+    return false;
 
   // Make sure the RHS really is a constant.
   ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
@@ -33248,7 +33475,7 @@ X86TargetLowering::targetShrinkDemandedConstant(SDValue Op,
   const APInt &Mask = C->getAPIntValue();
 
   // Clear all non-demanded bits initially.
-  APInt ShrunkMask = Mask & Demanded;
+  APInt ShrunkMask = Mask & DemandedBits;
 
   // Find the width of the shrunk mask.
   unsigned Width = ShrunkMask.getActiveBits();
@@ -33260,10 +33487,10 @@ X86TargetLowering::targetShrinkDemandedConstant(SDValue Op,
   // Find the next power of 2 width, rounding up to a byte.
   Width = PowerOf2Ceil(std::max(Width, 8U));
   // Truncate the width to size to handle illegal types.
-  Width = std::min(Width, Size);
+  Width = std::min(Width, EltSize);
 
   // Calculate a possible zero extend mask for this constant.
-  APInt ZeroExtendMask = APInt::getLowBitsSet(Size, Width);
+  APInt ZeroExtendMask = APInt::getLowBitsSet(EltSize, Width);
 
   // If we aren't changing the mask, just return true to keep it and prevent
   // the caller from optimizing.
@@ -33272,7 +33499,7 @@ X86TargetLowering::targetShrinkDemandedConstant(SDValue Op,
 
   // Make sure the new mask can be represented by a combination of mask bits
   // and non-demanded bits.
-  if (!ZeroExtendMask.isSubsetOf(Mask | ~Demanded))
+  if (!ZeroExtendMask.isSubsetOf(Mask | ~DemandedBits))
     return false;
 
   // Replace the constant with the zero extend mask.
@@ -34235,7 +34462,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
     return DAG.getBitcast(RootVT, V1);
   }
 
-  bool OptForSize = DAG.getMachineFunction().getFunction().hasOptSize();
+  bool OptForSize = DAG.shouldOptForSize();
   unsigned RootSizeInBits = RootVT.getSizeInBits();
   unsigned NumRootElts = RootVT.getVectorNumElements();
   unsigned BaseMaskEltSizeInBits = RootSizeInBits / NumBaseMaskElts;
@@ -34816,7 +35043,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
         continue;
       }
       if (M == SM_SentinelZero) {
-        PSHUFBMask.push_back(DAG.getConstant(255, DL, MVT::i8));
+        PSHUFBMask.push_back(DAG.getConstant(0x80, DL, MVT::i8));
         continue;
       }
       M = Ratio * M + i % Ratio;
@@ -34847,7 +35074,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
         continue;
       }
       if (M == SM_SentinelZero) {
-        VPPERMMask.push_back(DAG.getConstant(128, DL, MVT::i8));
+        VPPERMMask.push_back(DAG.getConstant(0x80, DL, MVT::i8));
         continue;
       }
       M = Ratio * M + i % Ratio;
@@ -36237,9 +36464,9 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
         (V.getOpcode() == X86ISD::PSHUFLW ||
          V.getOpcode() == X86ISD::PSHUFHW) &&
         V.getOpcode() != N.getOpcode() &&
-        V.hasOneUse()) {
+        V.hasOneUse() && V.getOperand(0).hasOneUse()) {
       SDValue D = peekThroughOneUseBitcasts(V.getOperand(0));
-      if (D.getOpcode() == X86ISD::PSHUFD && D.hasOneUse()) {
+      if (D.getOpcode() == X86ISD::PSHUFD) {
         SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
         SmallVector<int, 4> DMask = getPSHUFShuffleMask(D);
         int NOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
@@ -36676,10 +36903,11 @@ static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
   // insert into a zero vector. This helps get VZEXT_MOVL closer to
   // scalar_to_vectors where 256/512 are canonicalized to an insert and a
   // 128-bit scalar_to_vector. This reduces the number of isel patterns.
-  if (N->getOpcode() == X86ISD::VZEXT_MOVL && !DCI.isBeforeLegalizeOps()) {
+  if (N->getOpcode() == X86ISD::VZEXT_MOVL && !DCI.isBeforeLegalizeOps() &&
+      N->getOperand(0).hasOneUse()) {
     SDValue V = peekThroughOneUseBitcasts(N->getOperand(0));
 
-    if (V.getOpcode() == ISD::INSERT_SUBVECTOR && V.hasOneUse() &&
+    if (V.getOpcode() == ISD::INSERT_SUBVECTOR &&
         V.getOperand(0).isUndef() && isNullConstant(V.getOperand(2))) {
       SDValue In = V.getOperand(1);
       MVT SubVT =
@@ -36725,11 +36953,16 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetShuffle(
     return false;
 
   const Constant *C = getTargetConstantFromNode(Load);
-  if (!C || !C->getType()->isVectorTy())
+  if (!C)
+    return false;
+
+  Type *CTy = C->getType();
+  if (!CTy->isVectorTy() ||
+      CTy->getPrimitiveSizeInBits() != Mask.getValueSizeInBits())
     return false;
 
   // Handle scaling for i64 elements on 32-bit targets.
-  unsigned NumCstElts = cast<FixedVectorType>(C->getType())->getNumElements();
+  unsigned NumCstElts = cast<FixedVectorType>(CTy)->getNumElements();
   if (NumCstElts != NumElts && NumCstElts != (NumElts * 2))
     return false;
   unsigned Scale = NumCstElts / NumElts;
@@ -36929,12 +37162,10 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
     // Aggressively peek through ops to get at the demanded elts.
     // TODO - we should do this for all target/faux shuffles ops.
     if (!DemandedElts.isAllOnesValue()) {
-      APInt DemandedSrcBits =
-          APInt::getAllOnesValue(N0.getScalarValueSizeInBits());
-      SDValue NewN0 = SimplifyMultipleUseDemandedBits(
-          N0, DemandedSrcBits, DemandedLHS, TLO.DAG, Depth + 1);
-      SDValue NewN1 = SimplifyMultipleUseDemandedBits(
-          N1, DemandedSrcBits, DemandedRHS, TLO.DAG, Depth + 1);
+      SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts(N0, DemandedLHS,
+                                                            TLO.DAG, Depth + 1);
+      SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts(N1, DemandedRHS,
+                                                            TLO.DAG, Depth + 1);
       if (NewN0 || NewN1) {
         NewN0 = NewN0 ? NewN0 : N0;
         NewN1 = NewN1 ? NewN1 : N1;
@@ -37024,9 +37255,8 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
       return true;
     // Aggressively peek through src to get at the demanded elt.
     // TODO - we should do this for all target/faux shuffles ops.
-    APInt SrcBits = APInt::getAllOnesValue(SrcVT.getScalarSizeInBits());
-    if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(Src, SrcBits, SrcElts,
-                                                         TLO.DAG, Depth + 1))
+    if (SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
+            Src, SrcElts, TLO.DAG, Depth + 1))
       return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc));
     break;
   }
@@ -37063,18 +37293,6 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
       ExtSizeInBits = SizeInBits / 4;
 
     switch (Opc) {
-      // Zero upper elements.
-    case X86ISD::VZEXT_MOVL: {
-      SDLoc DL(Op);
-      SDValue Ext0 =
-          extractSubVector(Op.getOperand(0), 0, TLO.DAG, DL, ExtSizeInBits);
-      SDValue ExtOp =
-          TLO.DAG.getNode(Opc, DL, Ext0.getValueType(), Ext0);
-      SDValue UndefVec = TLO.DAG.getUNDEF(VT);
-      SDValue Insert =
-          insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
-      return TLO.CombineTo(Op, Insert);
-    }
       // Subvector broadcast.
     case X86ISD::SUBV_BROADCAST: {
       SDLoc DL(Op);
@@ -37090,11 +37308,6 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
       return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Src, 0,
                                                TLO.DAG, DL, ExtSizeInBits));
     }
-      // Target unary shuffles by immediate:
-    case X86ISD::PSHUFD:
-    case X86ISD::PSHUFLW:
-    case X86ISD::PSHUFHW:
-    case X86ISD::VPERMILPI:
       // Byte shifts by immediate.
     case X86ISD::VSHLDQ:
     case X86ISD::VSRLDQ:
@@ -37132,10 +37345,20 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
       }
       break;
     }
-      // Target Shuffles.
+      // Zero upper elements.
+    case X86ISD::VZEXT_MOVL:
+      // Target unary shuffles by immediate:
+    case X86ISD::PSHUFD:
+    case X86ISD::PSHUFLW:
+    case X86ISD::PSHUFHW:
+    case X86ISD::VPERMILPI:
+      // (Non-Lane Crossing) Target Shuffles.
+    case X86ISD::VPERMILPV:
+    case X86ISD::VPERMIL2:
     case X86ISD::PSHUFB:
     case X86ISD::UNPCKL:
     case X86ISD::UNPCKH:
+    case X86ISD::BLENDI:
       // Saturated Packs.
     case X86ISD::PACKSS:
     case X86ISD::PACKUS:
@@ -37145,36 +37368,25 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
     case X86ISD::FHADD:
     case X86ISD::FHSUB: {
       SDLoc DL(Op);
+      SmallVector<SDValue, 4> Ops;
+      for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
+        SDValue SrcOp = Op.getOperand(i);
+        EVT SrcVT = SrcOp.getValueType();
+        assert((!SrcVT.isVector() || SrcVT.getSizeInBits() == SizeInBits) &&
+               "Unsupported vector size");
+        Ops.push_back(SrcVT.isVector() ? extractSubVector(SrcOp, 0, TLO.DAG, DL,
+                                                          ExtSizeInBits)
+                                       : SrcOp);
+      }
       MVT ExtVT = VT.getSimpleVT();
       ExtVT = MVT::getVectorVT(ExtVT.getScalarType(),
                                ExtSizeInBits / ExtVT.getScalarSizeInBits());
-      SDValue Ext0 =
-          extractSubVector(Op.getOperand(0), 0, TLO.DAG, DL, ExtSizeInBits);
-      SDValue Ext1 =
-          extractSubVector(Op.getOperand(1), 0, TLO.DAG, DL, ExtSizeInBits);
-      SDValue ExtOp = TLO.DAG.getNode(Opc, DL, ExtVT, Ext0, Ext1);
+      SDValue ExtOp = TLO.DAG.getNode(Opc, DL, ExtVT, Ops);
       SDValue UndefVec = TLO.DAG.getUNDEF(VT);
       SDValue Insert =
           insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
       return TLO.CombineTo(Op, Insert);
     }
-      // Vector blend by immediate.
-    case X86ISD::BLENDI: {
-      SDLoc DL(Op);
-      MVT ExtVT = VT.getSimpleVT();
-      ExtVT = MVT::getVectorVT(ExtVT.getScalarType(),
-                               ExtSizeInBits / ExtVT.getScalarSizeInBits());
-      SDValue Ext0 =
-          extractSubVector(Op.getOperand(0), 0, TLO.DAG, DL, ExtSizeInBits);
-      SDValue Ext1 =
-          extractSubVector(Op.getOperand(1), 0, TLO.DAG, DL, ExtSizeInBits);
-      SDValue ExtOp =
-          TLO.DAG.getNode(Opc, DL, ExtVT, Ext0, Ext1, Op.getOperand(2));
-      SDValue UndefVec = TLO.DAG.getUNDEF(VT);
-      SDValue Insert =
-          insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
-      return TLO.CombineTo(Op, Insert);
-    } 
     }
   }
 
@@ -38774,9 +38986,6 @@ static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
   // Handle extract(bitcast(broadcast(scalar_value))).
   if (X86ISD::VBROADCAST == SrcBC.getOpcode()) {
     SDValue SrcOp = SrcBC.getOperand(0);
-    if (SrcOp.getValueSizeInBits() == VT.getSizeInBits())
-      return DAG.getBitcast(VT, SrcOp);
-
     EVT SrcOpVT = SrcOp.getValueType();
     if (SrcOpVT.isScalarInteger() && VT.isInteger() &&
         (SrcOpVT.getSizeInBits() % SrcEltBits) == 0) {
@@ -38797,7 +39006,7 @@ static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
     auto *MemIntr = cast<MemIntrinsicSDNode>(SrcBC);
     unsigned SrcBCWidth = SrcBC.getScalarValueSizeInBits();
     if (MemIntr->getMemoryVT().getSizeInBits() == SrcBCWidth &&
-        VT.getSizeInBits() == SrcBCWidth) {
+        VT.getSizeInBits() == SrcBCWidth && SrcEltBits == SrcBCWidth) {
       SDValue Load = DAG.getLoad(VT, dl, MemIntr->getChain(),
                                  MemIntr->getBasePtr(),
                                  MemIntr->getPointerInfo(),
@@ -39096,8 +39305,7 @@ static SDValue combineReductionToHorizontal(SDNode *ExtElt, SelectionDAG &DAG,
   }
 
   // Only use (F)HADD opcodes if they aren't microcoded or minimizes codesize.
-  bool OptForSize = DAG.getMachineFunction().getFunction().hasOptSize();
-  if (!Subtarget.hasFastHorizontalOps() && !OptForSize)
+  if (!shouldUseHorizontalOp(true, DAG, Subtarget))
     return SDValue();
 
   unsigned HorizOpcode = Opc == ISD::ADD ? X86ISD::HADD : X86ISD::FHADD;
@@ -40109,6 +40317,55 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
     }
   }
 
+  // If this is "((X & C) == 0) ? Y : Z" and C is a constant mask vector of
+  // single bits, then invert the predicate and swap the select operands.
+  // This can lower using a vector shift bit-hack rather than mask and compare.
+  if (DCI.isBeforeLegalize() && !Subtarget.hasAVX512() &&
+      N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC &&
+      Cond.hasOneUse() && CondVT.getVectorElementType() == MVT::i1 &&
+      Cond.getOperand(0).getOpcode() == ISD::AND &&
+      isNullOrNullSplat(Cond.getOperand(1)) &&
+      cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETEQ &&
+      Cond.getOperand(0).getValueType() == VT) {
+    // The 'and' mask must be composed of power-of-2 constants.
+    SDValue And = Cond.getOperand(0);
+    auto *C = isConstOrConstSplat(And.getOperand(1));
+    if (C && C->getAPIntValue().isPowerOf2()) {
+      // vselect (X & C == 0), LHS, RHS --> vselect (X & C != 0), RHS, LHS
+      SDValue NotCond =
+          DAG.getSetCC(DL, CondVT, And, Cond.getOperand(1), ISD::SETNE);
+      return DAG.getSelect(DL, VT, NotCond, RHS, LHS);
+    }
+
+    // If we have a non-splat but still powers-of-2 mask, AVX1 can use pmulld
+    // and AVX2 can use vpsllv{dq}. 8-bit lacks a proper shift or multiply.
+    // 16-bit lacks a proper blendv.
+    unsigned EltBitWidth = VT.getScalarSizeInBits();
+    bool CanShiftBlend =
+        TLI.isTypeLegal(VT) && ((Subtarget.hasAVX() && EltBitWidth == 32) ||
+                                (Subtarget.hasAVX2() && EltBitWidth == 64) ||
+                                (Subtarget.hasXOP()));
+    if (CanShiftBlend &&
+        ISD::matchUnaryPredicate(And.getOperand(1), [](ConstantSDNode *C) {
+          return C->getAPIntValue().isPowerOf2();
+        })) {
+      // Create a left-shift constant to get the mask bits over to the sign-bit.
+      SDValue Mask = And.getOperand(1);
+      SmallVector<int, 32> ShlVals;
+      for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
+        auto *MaskVal = cast<ConstantSDNode>(Mask.getOperand(i));
+        ShlVals.push_back(EltBitWidth - 1 -
+                          MaskVal->getAPIntValue().exactLogBase2());
+      }
+      // vsel ((X & C) == 0), LHS, RHS --> vsel ((shl X, C') < 0), RHS, LHS
+      SDValue ShlAmt = getConstVector(ShlVals, VT.getSimpleVT(), DAG, DL);
+      SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And.getOperand(0), ShlAmt);
+      SDValue NewCond =
+          DAG.getSetCC(DL, CondVT, Shl, Cond.getOperand(1), ISD::SETLT);
+      return DAG.getSelect(DL, VT, NewCond, RHS, LHS);
+    }
+  }
+
   return SDValue();
 }
 
@@ -41642,16 +41899,92 @@ static SDValue combineShiftRightLogical(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
+static SDValue combineVectorPackWithShuffle(SDNode *N, SelectionDAG &DAG) {
+  unsigned Opcode = N->getOpcode();
+  assert((X86ISD::PACKSS == Opcode || X86ISD::PACKUS == Opcode) &&
+         "Unexpected pack opcode");
+
+  EVT VT = N->getValueType(0);
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+  unsigned NumDstElts = VT.getVectorNumElements();
+
+  // Attempt to fold PACK(LOSUBVECTOR(SHUFFLE(X)),HISUBVECTOR(SHUFFLE(X)))
+  // to SHUFFLE(PACK(LOSUBVECTOR(X),HISUBVECTOR(X))), this is mainly for
+  // truncation trees that help us avoid lane crossing shuffles.
+  // TODO: There's a lot more we can do for PACK/HADD style shuffle combines.
+  if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
+      N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
+      N0.getConstantOperandAPInt(1) == 0 &&
+      N1.getConstantOperandAPInt(1) == (NumDstElts / 2) &&
+      N0.getOperand(0) == N1.getOperand(0) && VT.is128BitVector() &&
+      N0.getOperand(0).getValueType().is256BitVector()) {
+    // TODO - support target/faux shuffles.
+    SDValue Vec = peekThroughBitcasts(N0.getOperand(0));
+    if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(Vec)) {
+      // To keep the PACK LHS/RHS coherency, we must be able to scale the unary
+      // shuffle to a vXi64 width - we can probably relax this in the future.
+      SmallVector<int, 4> ShuffleMask;
+      if (SVN->getOperand(1).isUndef() &&
+          scaleShuffleElements(SVN->getMask(), 4, ShuffleMask)) {
+        SDLoc DL(N);
+        SDValue Lo, Hi;
+        std::tie(Lo, Hi) = DAG.SplitVector(SVN->getOperand(0), DL);
+        Lo = DAG.getBitcast(N0.getValueType(), Lo);
+        Hi = DAG.getBitcast(N1.getValueType(), Hi);
+        SDValue Res = DAG.getNode(Opcode, DL, VT, Lo, Hi);
+        Res = DAG.getBitcast(MVT::v4i32, Res);
+        Res = DAG.getVectorShuffle(MVT::v4i32, DL, Res, Res, ShuffleMask);
+        return DAG.getBitcast(VT, Res);
+      }
+    }
+  }
+
+  // Attempt to fold PACK(SHUFFLE(X,Y),SHUFFLE(X,Y)) -> SHUFFLE(PACK(X,Y)).
+  // TODO: Relax shuffle scaling to support sub-128-bit subvector shuffles.
+  if (VT.is256BitVector()) {
+    if (auto *SVN0 = dyn_cast<ShuffleVectorSDNode>(N0)) {
+      if (auto *SVN1 = dyn_cast<ShuffleVectorSDNode>(N1)) {
+        SmallVector<int, 2> ShuffleMask0, ShuffleMask1;
+        if (scaleShuffleElements(SVN0->getMask(), 2, ShuffleMask0) &&
+            scaleShuffleElements(SVN1->getMask(), 2, ShuffleMask1)) {
+          SDValue Op00 = SVN0->getOperand(0);
+          SDValue Op01 = SVN0->getOperand(1);
+          SDValue Op10 = SVN1->getOperand(0);
+          SDValue Op11 = SVN1->getOperand(1);
+          if ((Op00 == Op11) && (Op01 == Op10)) {
+            std::swap(Op10, Op11);
+            ShuffleVectorSDNode::commuteMask(ShuffleMask1);
+          }
+          if ((Op00 == Op10) && (Op01 == Op11)) {
+            SmallVector<int, 4> ShuffleMask;
+            ShuffleMask.append(ShuffleMask0.begin(), ShuffleMask0.end());
+            ShuffleMask.append(ShuffleMask1.begin(), ShuffleMask1.end());
+            SDLoc DL(N);
+            SDValue Res = DAG.getNode(Opcode, DL, VT, Op00, Op01);
+            Res = DAG.getBitcast(MVT::v4i64, Res);
+            Res = DAG.getVectorShuffle(MVT::v4i64, DL, Res, Res, ShuffleMask);
+            return DAG.getBitcast(VT, Res);
+          }
+        }
+      }
+    }
+  }
+
+  return SDValue();
+}
+
 static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG,
                                  TargetLowering::DAGCombinerInfo &DCI,
                                  const X86Subtarget &Subtarget) {
   unsigned Opcode = N->getOpcode();
   assert((X86ISD::PACKSS == Opcode || X86ISD::PACKUS == Opcode) &&
-         "Unexpected shift opcode");
+         "Unexpected pack opcode");
 
   EVT VT = N->getValueType(0);
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
+  unsigned NumDstElts = VT.getVectorNumElements();
   unsigned DstBitsPerElt = VT.getScalarSizeInBits();
   unsigned SrcBitsPerElt = 2 * DstBitsPerElt;
   assert(N0.getScalarValueSizeInBits() == SrcBitsPerElt &&
@@ -41668,7 +42001,6 @@ static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG,
       getTargetConstantBitsFromNode(N0, SrcBitsPerElt, UndefElts0, EltBits0) &&
       getTargetConstantBitsFromNode(N1, SrcBitsPerElt, UndefElts1, EltBits1)) {
     unsigned NumLanes = VT.getSizeInBits() / 128;
-    unsigned NumDstElts = VT.getVectorNumElements();
     unsigned NumSrcElts = NumDstElts / 2;
     unsigned NumDstEltsPerLane = NumDstElts / NumLanes;
     unsigned NumSrcEltsPerLane = NumSrcElts / NumLanes;
@@ -41715,6 +42047,10 @@ static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG,
     return getConstVector(Bits, Undefs, VT.getSimpleVT(), DAG, SDLoc(N));
   }
 
+  // Try to fold PACK(SHUFFLE(),SHUFFLE()) -> SHUFFLE(PACK()).
+  if (SDValue V = combineVectorPackWithShuffle(N, DAG))
+    return V;
+
   // Try to combine a PACKUSWB/PACKSSWB implemented truncate with a regular
   // truncate to create a larger truncate.
   if (Subtarget.hasAVX512() &&
@@ -42141,7 +42477,7 @@ static SDValue PromoteMaskArithmetic(SDNode *N, SelectionDAG &DAG,
   }
 }
 
-unsigned convertIntLogicToFPLogicOpcode(unsigned Opcode) {
+static unsigned convertIntLogicToFPLogicOpcode(unsigned Opcode) {
   unsigned FPOpcode;
   switch (Opcode) {
   default: llvm_unreachable("Unexpected input node for FP logic conversion");
@@ -42671,6 +43007,16 @@ static SDValue canonicalizeBitSelect(SDNode *N, SelectionDAG &DAG,
   }
 
   SDLoc DL(N);
+
+  if (UseVPTERNLOG) {
+    // Emit a VPTERNLOG node directly.
+    SDValue A = DAG.getBitcast(VT, N0.getOperand(1));
+    SDValue B = DAG.getBitcast(VT, N0.getOperand(0));
+    SDValue C = DAG.getBitcast(VT, N1.getOperand(0));
+    SDValue Imm = DAG.getTargetConstant(0xCA, DL, MVT::i8);
+    return DAG.getNode(X86ISD::VPTERNLOG, DL, VT, A, B, C, Imm);
+  }
+
   SDValue X = N->getOperand(0);
   SDValue Y =
       DAG.getNode(X86ISD::ANDNP, DL, VT, DAG.getBitcast(VT, N0.getOperand(1)),
@@ -42754,6 +43100,10 @@ static SDValue combineLogicBlendIntoPBLENDV(SDNode *N, SelectionDAG &DAG,
   if (!Subtarget.hasSSE41())
     return SDValue();
 
+  // If we have VPTERNLOG we should prefer that since PBLENDVB is multiple uops.
+  if (Subtarget.hasVLX())
+    return SDValue();
+
   MVT BlendVT = VT.is256BitVector() ? MVT::v32i8 : MVT::v16i8;
 
   X = DAG.getBitcast(BlendVT, X);
@@ -43453,6 +43803,20 @@ static SDValue combineLoad(SDNode *N, SelectionDAG &DAG,
     }
   }
 
+  // Cast ptr32 and ptr64 pointers to the default address space before a load.
+  unsigned AddrSpace = Ld->getAddressSpace();
+  if (AddrSpace == X86AS::PTR64 || AddrSpace == X86AS::PTR32_SPTR ||
+      AddrSpace == X86AS::PTR32_UPTR) {
+    MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
+    if (PtrVT != Ld->getBasePtr().getSimpleValueType()) {
+      SDValue Cast =
+          DAG.getAddrSpaceCast(dl, PtrVT, Ld->getBasePtr(), AddrSpace, 0);
+      return DAG.getLoad(RegVT, dl, Ld->getChain(), Cast, Ld->getPointerInfo(),
+                         Ld->getOriginalAlign(),
+                         Ld->getMemOperand()->getFlags());
+    }
+  }
+
   return SDValue();
 }
 
@@ -43880,6 +44244,20 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
     return SDValue();
   }
 
+  // Cast ptr32 and ptr64 pointers to the default address space before a store.
+  unsigned AddrSpace = St->getAddressSpace();
+  if (AddrSpace == X86AS::PTR64 || AddrSpace == X86AS::PTR32_SPTR ||
+      AddrSpace == X86AS::PTR32_UPTR) {
+    MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
+    if (PtrVT != St->getBasePtr().getSimpleValueType()) {
+      SDValue Cast =
+          DAG.getAddrSpaceCast(dl, PtrVT, St->getBasePtr(), AddrSpace, 0);
+      return DAG.getStore(St->getChain(), dl, StoredVal, Cast,
+                          St->getPointerInfo(), St->getOriginalAlign(),
+                          St->getMemOperand()->getFlags(), St->getAAInfo());
+    }
+  }
+
   // Turn load->store of MMX types into GPR load/stores.  This avoids clobbering
   // the FP state in cases where an emms may be missing.
   // A preferable solution to the general problem is to figure out the right
@@ -46089,7 +46467,7 @@ static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
     if (SDValue V = combineVectorSizedSetCCEquality(N, DAG, Subtarget))
       return V;
 
-    if (VT == MVT::i1) {
+    if (VT == MVT::i1 && isNullConstant(RHS)) {
       SDValue X86CC;
       if (SDValue V =
               MatchVectorAllZeroTest(LHS, CC, DL, Subtarget, DAG, X86CC))
@@ -47974,6 +48352,31 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG,
       InVec.getOperand(0).getValueType() == VT)
     return InVec.getOperand(0);
 
+  // Attempt to extract from the source of a shuffle vector.
+  if ((InVecVT.getSizeInBits() % VT.getSizeInBits()) == 0 &&
+      (IdxVal % VT.getVectorNumElements()) == 0) {
+    SmallVector<int, 32> ShuffleMask;
+    SmallVector<int, 32> ScaledMask;
+    SmallVector<SDValue, 2> ShuffleInputs;
+    unsigned NumSubVecs = InVecVT.getSizeInBits() / VT.getSizeInBits();
+    // Decode the shuffle mask and scale it so its shuffling subvectors.
+    if (getTargetShuffleInputs(InVecBC, ShuffleInputs, ShuffleMask, DAG) &&
+        scaleShuffleElements(ShuffleMask, NumSubVecs, ScaledMask)) {
+      unsigned SubVecIdx = IdxVal / VT.getVectorNumElements();
+      if (ScaledMask[SubVecIdx] == SM_SentinelUndef)
+        return DAG.getUNDEF(VT);
+      if (ScaledMask[SubVecIdx] == SM_SentinelZero)
+        return getZeroVector(VT, Subtarget, DAG, SDLoc(N));
+      SDValue Src = ShuffleInputs[ScaledMask[SubVecIdx] / NumSubVecs];
+      if (Src.getValueSizeInBits() == InVecVT.getSizeInBits()) {
+        unsigned SrcSubVecIdx = ScaledMask[SubVecIdx] % NumSubVecs;
+        unsigned SrcEltIdx = SrcSubVecIdx * VT.getVectorNumElements();
+        return extractSubVector(DAG.getBitcast(InVecVT, Src), SrcEltIdx, DAG,
+                                SDLoc(N), VT.getSizeInBits());
+      }
+    }
+  }
+
   // If we're extracting the lowest subvector and we're the only user,
   // we may be able to perform this with a smaller vector width.
   if (IdxVal == 0 && InVec.hasOneUse()) {
@@ -48046,13 +48449,25 @@ static SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG) {
                            Src.getOperand(1));
 
   // Reduce v2i64 to v4i32 if we don't need the upper bits.
-  // TODO: Move to DAGCombine?
-  if (VT == MVT::v2i64 && Src.getOpcode() == ISD::ANY_EXTEND &&
-      Src.getValueType() == MVT::i64 && Src.hasOneUse() &&
-      Src.getOperand(0).getScalarValueSizeInBits() <= 32)
-    return DAG.getBitcast(
-        VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32,
-                        DAG.getAnyExtOrTrunc(Src.getOperand(0), DL, MVT::i32)));
+  // TODO: Move to DAGCombine/SimplifyDemandedBits?
+  if (VT == MVT::v2i64 || VT == MVT::v2f64) {
+    auto IsAnyExt64 = [](SDValue Op) {
+      if (Op.getValueType() != MVT::i64 || !Op.hasOneUse())
+        return SDValue();
+      if (Op.getOpcode() == ISD::ANY_EXTEND &&
+          Op.getOperand(0).getScalarValueSizeInBits() <= 32)
+        return Op.getOperand(0);
+      if (auto *Ld = dyn_cast<LoadSDNode>(Op))
+        if (Ld->getExtensionType() == ISD::EXTLOAD &&
+            Ld->getMemoryVT().getScalarSizeInBits() <= 32)
+          return Op;
+      return SDValue();
+    };
+    if (SDValue ExtSrc = IsAnyExt64(peekThroughOneUseBitcasts(Src)))
+      return DAG.getBitcast(
+          VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32,
+                          DAG.getAnyExtOrTrunc(ExtSrc, DL, MVT::i32)));
+  }
 
   // Combine (v2i64 (scalar_to_vector (i64 (bitconvert (mmx))))) to MOVQ2DQ.
   if (VT == MVT::v2i64 && Src.getOpcode() == ISD::BITCAST &&
@@ -49486,7 +49901,7 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
 
   // Use the default implementation in TargetLowering to convert the register
   // constraint into a member of a register class.
-  std::pair<unsigned, const TargetRegisterClass*> Res;
+  std::pair<Register, const TargetRegisterClass*> Res;
   Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
 
   // Not found as a standard register?
@@ -49557,7 +49972,7 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
   if (isGRClass(*Class)) {
     unsigned Size = VT.getSizeInBits();
     if (Size == 1) Size = 8;
-    unsigned DestReg = getX86SubSuperRegisterOrZero(Res.first, Size);
+    Register DestReg = getX86SubSuperRegisterOrZero(Res.first, Size);
     if (DestReg > 0) {
       bool is64Bit = Subtarget.is64Bit();
       const TargetRegisterClass *RC =
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 560da449225a8..7f3dc90a2d735 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1036,7 +1036,8 @@ namespace llvm {
     EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
                            EVT VT) const override;
 
-    bool targetShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
+    bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,
+                                      const APInt &DemandedElts,
                                       TargetLoweringOpt &TLO) const override;
 
     /// Determine which of the bits specified in Mask are known to be either
@@ -1435,7 +1436,7 @@ namespace llvm {
     SDValue LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, SDValue Arg,
                              const SDLoc &dl, SelectionDAG &DAG,
                              const CCValAssign &VA,
-                             ISD::ArgFlagsTy Flags) const;
+                             ISD::ArgFlagsTy Flags, bool isByval) const;
 
     // Call lowering helpers.
 
diff --git a/llvm/lib/Target/X86/X86IndirectThunks.cpp b/llvm/lib/Target/X86/X86IndirectThunks.cpp
index 226af7ba1bc1a..828887d961293 100644
--- a/llvm/lib/Target/X86/X86IndirectThunks.cpp
+++ b/llvm/lib/Target/X86/X86IndirectThunks.cpp
@@ -110,12 +110,6 @@ class X86IndirectThunks : public MachineFunctionPass {
   bool doInitialization(Module &M) override;
   bool runOnMachineFunction(MachineFunction &MF) override;
 
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    MachineFunctionPass::getAnalysisUsage(AU);
-    AU.addRequired<MachineModuleInfoWrapperPass>();
-    AU.addPreserved<MachineModuleInfoWrapperPass>();
-  }
-
 private:
   std::tuple<RetpolineThunkInserter, LVIThunkInserter> TIs;
 
diff --git a/llvm/lib/Target/X86/X86InsertPrefetch.cpp b/llvm/lib/Target/X86/X86InsertPrefetch.cpp
index 2b1e3f23efd74..53925bbfd72fe 100644
--- a/llvm/lib/Target/X86/X86InsertPrefetch.cpp
+++ b/llvm/lib/Target/X86/X86InsertPrefetch.cpp
@@ -173,7 +173,7 @@ bool X86InsertPrefetch::doInitialization(Module &M) {
 
 void X86InsertPrefetch::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesAll();
-  AU.addRequired<MachineModuleInfoWrapperPass>();
+  MachineFunctionPass::getAnalysisUsage(AU);
 }
 
 bool X86InsertPrefetch::runOnMachineFunction(MachineFunction &MF) {
diff --git a/llvm/lib/Target/X86/X86InstrAMX.td b/llvm/lib/Target/X86/X86InstrAMX.td
new file mode 100644
index 0000000000000..e26dd5050a23c
--- /dev/null
+++ b/llvm/lib/Target/X86/X86InstrAMX.td
@@ -0,0 +1,119 @@
+//===---- X86InstrAMX.td - AMX Instruction Set Extension --*- tablegen -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the instructions that make up the Intel AMX instruction
+// set.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// AMX instructions
+
+let Predicates = [HasAMXTILE, In64BitMode] in {
+  let SchedRW = [WriteSystem] in {
+    let Defs = [TMM0,TMM1,TMM2,TMM3,TMM4,TMM5,TMM6,TMM7] in
+    def LDTILECFG : I <0x49, MRM0m, (outs), (ins opaquemem:$src),
+                       "ldtilecfg\t$src",
+                       [(int_x86_ldtilecfg addr:$src)]>, VEX, T8PS;
+    def STTILECFG : I <0x49, MRM0m, (outs), (ins opaquemem:$src),
+                       "sttilecfg\t$src",
+                       [(int_x86_sttilecfg addr:$src)]>, VEX, T8PD;
+    def TILELOADD : I<0x4b, MRMSrcMemFSIB, (outs TILE:$dst),
+                      (ins sibmem:$src),
+                      "tileloadd\t{$src, $dst|$dst, $src}", []>,
+                      VEX, T8XD;
+    def TILELOADDT1 : I<0x4b, MRMSrcMemFSIB, (outs TILE:$dst),
+                        (ins sibmem:$src),
+                        "tileloaddt1\t{$src, $dst|$dst, $src}", []>,
+                        VEX, T8PD;
+    let Defs = [TMM0,TMM1,TMM2,TMM3,TMM4,TMM5,TMM6,TMM7] in
+    def TILERELEASE : I<0x49, MRM_C0, (outs), (ins),
+                        "tilerelease", [(int_x86_tilerelease)]>, VEX, T8PS;
+    def TILESTORED : I<0x4b, MRMDestMemFSIB, (outs),
+                       (ins sibmem:$dst, TILE:$src),
+                       "tilestored\t{$src, $dst|$dst, $src}", []>,
+                       VEX, T8XS;
+    def TILEZERO : I<0x49, MRMr0, (outs TILE:$dst), (ins),
+                     "tilezero\t$dst", []>,
+                     VEX, T8XD;
+
+    let usesCustomInserter = 1 in {
+      // Pseudo instructions, using immediates instead of tile registers.
+      // To be translated to the actual instructions in X86ISelLowering.cpp
+      def PTILELOADD : PseudoI<(outs), (ins u8imm:$src1, sibmem:$src2), []>;
+      def PTILELOADDT1 : PseudoI<(outs), (ins u8imm:$src1,
+                                          sibmem:$src2), []>;
+      def PTILESTORED : PseudoI<(outs), (ins i8mem:$dst, u8imm:$src), []>;
+      def PTILEZERO : PseudoI<(outs), (ins u8imm:$src),
+                              [(int_x86_tilezero imm:$src)]>;
+    }
+  } // SchedRW
+} // HasAMXTILE
+
+let Predicates = [HasAMXINT8, In64BitMode] in {
+  let SchedRW = [WriteSystem] in {
+    let Constraints = "$src1 = $dst" in {
+      def TDPBSSD : I<0x5e, MRMSrcReg4VOp3, (outs TILE:$dst),
+                      (ins TILE:$src1, TILE:$src2, TILE:$src3),
+                      "tdpbssd\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
+                      VEX_4V, T8XD;
+      def TDPBSUD : I<0x5e, MRMSrcReg4VOp3, (outs TILE:$dst),
+                      (ins TILE:$src1, TILE:$src2, TILE:$src3),
+                      "tdpbsud\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
+                      VEX_4V, T8XS;
+      def TDPBUSD : I<0x5e, MRMSrcReg4VOp3, (outs TILE:$dst),
+                      (ins TILE:$src1, TILE:$src2, TILE:$src3),
+                      "tdpbusd\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
+                      VEX_4V, T8PD;
+      def TDPBUUD : I<0x5e, MRMSrcReg4VOp3, (outs TILE:$dst),
+                      (ins TILE:$src1, TILE:$src2, TILE:$src3),
+                      "tdpbuud\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
+                      VEX_4V, T8PS;
+    }
+
+    let usesCustomInserter = 1 in {
+      // Pseudo instructions, using immediates instead of tile registers.
+      // To be translated to the actual instructions in X86ISelLowering.cpp
+      def PTDPBSSD : PseudoI<(outs), (ins u8imm:$src1,
+                             u8imm:$src2, u8imm:$src3),
+                             [(int_x86_tdpbssd imm:$src1,
+                               imm:$src2, imm:$src3)]>;
+      def PTDPBSUD : PseudoI<(outs), (ins u8imm:$src1,
+                             u8imm:$src2, u8imm:$src3),
+                             [(int_x86_tdpbsud imm:$src1,
+                               imm:$src2, imm:$src3)]>;
+      def PTDPBUSD : PseudoI<(outs), (ins u8imm:$src1,
+                             u8imm:$src2, u8imm:$src3),
+                             [(int_x86_tdpbusd imm:$src1,
+                               imm:$src2, imm:$src3)]>;
+      def PTDPBUUD : PseudoI<(outs), (ins u8imm:$src1,
+                             u8imm:$src2, u8imm:$src3),
+                             [(int_x86_tdpbuud imm:$src1,
+                               imm:$src2, imm:$src3)]>;
+    }
+  }
+} // HasAMXTILE
+
+let Predicates = [HasAMXBF16, In64BitMode] in {
+  let SchedRW = [WriteSystem] in {
+    let Constraints = "$src1 = $dst" in
+    def TDPBF16PS : I<0x5c, MRMSrcReg4VOp3, (outs TILE:$dst),
+                      (ins TILE:$src1, TILE:$src2, TILE:$src3),
+                      "tdpbf16ps\t{$src3, $src2, $dst|$dst, $src2, $src3}",
+                      []>, VEX_4V, T8XS;
+
+    let usesCustomInserter = 1 in {
+      // Pseudo instructions, using immediates instead of tile registers.
+      // To be translated to the actual instructions in X86ISelLowering.cpp
+      def PTDPBF16PS : PseudoI<(outs), (ins u8imm:$src1,
+                               u8imm:$src2, u8imm:$src3),
+                               [(int_x86_tdpbf16ps imm:$src1,
+                                 imm:$src2, imm:$src3)]>;
+    }
+  }
+} // HasAMXTILE, HasAMXBF16
diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index 0921a0e51668a..a3ad0b1c8dd6c 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -11365,6 +11365,36 @@ let Predicates = [HasVLX] in {
             (VPTERNLOGQZ128rmi VR128X:$src1, VR128X:$src2, addr:$src3,
                                (VPTERNLOG132_imm8 timm:$src4))>;
 
+  def : Pat<(v16i8 (X86vpternlog VR128X:$src1, VR128X:$src2,
+                                 (bitconvert (v4i32 (X86VBroadcastld32 addr:$src3))),
+                                 (i8 timm:$src4))),
+            (VPTERNLOGDZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(v16i8 (X86vpternlog (bitconvert (v4i32 (X86VBroadcastld32 addr:$src3))),
+                                 VR128X:$src2, VR128X:$src1, (i8 timm:$src4))),
+            (VPTERNLOGDZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v16i8 (X86vpternlog VR128X:$src1,
+                                 (bitconvert (v4i32 (X86VBroadcastld32 addr:$src3))),
+                                 VR128X:$src2, (i8 timm:$src4))),
+            (VPTERNLOGDZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                               (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v16i8 (X86vpternlog VR128X:$src1, VR128X:$src2,
+                                 (bitconvert (v2i64 (X86VBroadcastld64 addr:$src3))),
+                                 (i8 timm:$src4))),
+            (VPTERNLOGQZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(v16i8 (X86vpternlog (bitconvert (v2i64 (X86VBroadcastld64 addr:$src3))),
+                                 VR128X:$src2, VR128X:$src1, (i8 timm:$src4))),
+            (VPTERNLOGQZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v16i8 (X86vpternlog VR128X:$src1,
+                                 (bitconvert (v2i64 (X86VBroadcastld64 addr:$src3))),
+                                 VR128X:$src2, (i8 timm:$src4))),
+            (VPTERNLOGQZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                (VPTERNLOG132_imm8 timm:$src4))>;
+
   def : Pat<(v8i16 (X86vpternlog VR128X:$src1, VR128X:$src2, VR128X:$src3,
                                  (i8 timm:$src4))),
             (VPTERNLOGQZ128rri VR128X:$src1, VR128X:$src2, VR128X:$src3,
@@ -11382,6 +11412,66 @@ let Predicates = [HasVLX] in {
             (VPTERNLOGQZ128rmi VR128X:$src1, VR128X:$src2, addr:$src3,
                                (VPTERNLOG132_imm8 timm:$src4))>;
 
+  def : Pat<(v8i16 (X86vpternlog VR128X:$src1, VR128X:$src2,
+                                 (bitconvert (v4i32 (X86VBroadcastld32 addr:$src3))),
+                                 (i8 timm:$src4))),
+            (VPTERNLOGDZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(v8i16 (X86vpternlog (bitconvert (v4i32 (X86VBroadcastld32 addr:$src3))),
+                                 VR128X:$src2, VR128X:$src1, (i8 timm:$src4))),
+            (VPTERNLOGDZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v8i16 (X86vpternlog VR128X:$src1,
+                                 (bitconvert (v4i32 (X86VBroadcastld32 addr:$src3))),
+                                 VR128X:$src2, (i8 timm:$src4))),
+            (VPTERNLOGDZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                               (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v8i16 (X86vpternlog VR128X:$src1, VR128X:$src2,
+                                 (bitconvert (v2i64 (X86VBroadcastld64 addr:$src3))),
+                                 (i8 timm:$src4))),
+            (VPTERNLOGQZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(v8i16 (X86vpternlog (bitconvert (v2i64 (X86VBroadcastld64 addr:$src3))),
+                                 VR128X:$src2, VR128X:$src1, (i8 timm:$src4))),
+            (VPTERNLOGQZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v8i16 (X86vpternlog VR128X:$src1,
+                                 (bitconvert (v2i64 (X86VBroadcastld64 addr:$src3))),
+                                 VR128X:$src2, (i8 timm:$src4))),
+            (VPTERNLOGQZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v4i32 (X86vpternlog VR128X:$src1, VR128X:$src2,
+                                 (bitconvert (v2i64 (X86VBroadcastld64 addr:$src3))),
+                                 (i8 timm:$src4))),
+            (VPTERNLOGQZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(v4i32 (X86vpternlog (bitconvert (v2i64 (X86VBroadcastld64 addr:$src3))),
+                                 VR128X:$src2, VR128X:$src1, (i8 timm:$src4))),
+            (VPTERNLOGQZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v4i32 (X86vpternlog VR128X:$src1,
+                                 (bitconvert (v2i64 (X86VBroadcastld64 addr:$src3))),
+                                 VR128X:$src2, (i8 timm:$src4))),
+            (VPTERNLOGQZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v2i64 (X86vpternlog VR128X:$src1, VR128X:$src2,
+                                 (bitconvert (v4i32 (X86VBroadcastld32 addr:$src3))),
+                                 (i8 timm:$src4))),
+            (VPTERNLOGDZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(v2i64 (X86vpternlog (bitconvert (v4i32 (X86VBroadcastld32 addr:$src3))),
+                                 VR128X:$src2, VR128X:$src1, (i8 timm:$src4))),
+            (VPTERNLOGDZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v2i64 (X86vpternlog VR128X:$src1,
+                                 (bitconvert (v4i32 (X86VBroadcastld32 addr:$src3))),
+                                 VR128X:$src2, (i8 timm:$src4))),
+            (VPTERNLOGDZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                               (VPTERNLOG132_imm8 timm:$src4))>;
+
   def : Pat<(v32i8 (X86vpternlog VR256X:$src1, VR256X:$src2, VR256X:$src3,
                                  (i8 timm:$src4))),
             (VPTERNLOGQZ256rri VR256X:$src1, VR256X:$src2, VR256X:$src3,
@@ -11399,6 +11489,36 @@ let Predicates = [HasVLX] in {
             (VPTERNLOGQZ256rmi VR256X:$src1, VR256X:$src2, addr:$src3,
                                (VPTERNLOG132_imm8 timm:$src4))>;
 
+  def : Pat<(v32i8 (X86vpternlog VR256X:$src1, VR256X:$src2,
+                                 (bitconvert (v8i32 (X86VBroadcastld32 addr:$src3))),
+                                 (i8 timm:$src4))),
+            (VPTERNLOGDZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(v32i8 (X86vpternlog (bitconvert (v8i32 (X86VBroadcastld32 addr:$src3))),
+                                 VR256X:$src2, VR256X:$src1, (i8 timm:$src4))),
+            (VPTERNLOGDZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v32i8 (X86vpternlog VR256X:$src1,
+                                 (bitconvert (v8i32 (X86VBroadcastld32 addr:$src3))),
+                                 VR256X:$src2, (i8 timm:$src4))),
+            (VPTERNLOGDZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                               (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v32i8 (X86vpternlog VR256X:$src1, VR256X:$src2,
+                                 (bitconvert (v4i64 (X86VBroadcastld64 addr:$src3))),
+                                 (i8 timm:$src4))),
+            (VPTERNLOGQZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(v32i8 (X86vpternlog (bitconvert (v4i64 (X86VBroadcastld64 addr:$src3))),
+                                 VR256X:$src2, VR256X:$src1, (i8 timm:$src4))),
+            (VPTERNLOGQZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v32i8 (X86vpternlog VR256X:$src1,
+                                 (bitconvert (v4i64 (X86VBroadcastld64 addr:$src3))),
+                                 VR256X:$src2, (i8 timm:$src4))),
+            (VPTERNLOGQZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                (VPTERNLOG132_imm8 timm:$src4))>;
+
   def : Pat<(v16i16 (X86vpternlog VR256X:$src1, VR256X:$src2, VR256X:$src3,
                                   (i8 timm:$src4))),
             (VPTERNLOGQZ256rri VR256X:$src1, VR256X:$src2, VR256X:$src3,
@@ -11415,6 +11535,66 @@ let Predicates = [HasVLX] in {
                                   VR256X:$src2, (i8 timm:$src4))),
             (VPTERNLOGQZ256rmi VR256X:$src1, VR256X:$src2, addr:$src3,
                                (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v16i16 (X86vpternlog VR256X:$src1, VR256X:$src2,
+                                  (bitconvert (v8i32 (X86VBroadcastld32 addr:$src3))),
+                                  (i8 timm:$src4))),
+            (VPTERNLOGDZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(v16i16 (X86vpternlog (bitconvert (v8i32 (X86VBroadcastld32 addr:$src3))),
+                                 VR256X:$src2, VR256X:$src1, (i8 timm:$src4))),
+            (VPTERNLOGDZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v16i16 (X86vpternlog VR256X:$src1,
+                                  (bitconvert (v8i32 (X86VBroadcastld32 addr:$src3))),
+                                  VR256X:$src2, (i8 timm:$src4))),
+            (VPTERNLOGDZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                               (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v16i16 (X86vpternlog VR256X:$src1, VR256X:$src2,
+                                  (bitconvert (v4i64 (X86VBroadcastld64 addr:$src3))),
+                                  (i8 timm:$src4))),
+            (VPTERNLOGQZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(v16i16 (X86vpternlog (bitconvert (v4i64 (X86VBroadcastld64 addr:$src3))),
+                                  VR256X:$src2, VR256X:$src1, (i8 timm:$src4))),
+            (VPTERNLOGQZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v16i16 (X86vpternlog VR256X:$src1,
+                                  (bitconvert (v4i64 (X86VBroadcastld64 addr:$src3))),
+                                  VR256X:$src2, (i8 timm:$src4))),
+            (VPTERNLOGQZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v8i32 (X86vpternlog VR256X:$src1, VR256X:$src2,
+                                 (bitconvert (v4i64 (X86VBroadcastld64 addr:$src3))),
+                                 (i8 timm:$src4))),
+            (VPTERNLOGQZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(v8i32 (X86vpternlog (bitconvert (v4i64 (X86VBroadcastld64 addr:$src3))),
+                                  VR256X:$src2, VR256X:$src1, (i8 timm:$src4))),
+            (VPTERNLOGQZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v8i32 (X86vpternlog VR256X:$src1,
+                                 (bitconvert (v4i64 (X86VBroadcastld64 addr:$src3))),
+                                 VR256X:$src2, (i8 timm:$src4))),
+            (VPTERNLOGQZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v4i64 (X86vpternlog VR256X:$src1, VR256X:$src2,
+                                 (bitconvert (v8i32 (X86VBroadcastld32 addr:$src3))),
+                                 (i8 timm:$src4))),
+            (VPTERNLOGDZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(v4i64 (X86vpternlog (bitconvert (v8i32 (X86VBroadcastld32 addr:$src3))),
+                                 VR256X:$src2, VR256X:$src1, (i8 timm:$src4))),
+            (VPTERNLOGDZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v4i64 (X86vpternlog VR256X:$src1,
+                                 (bitconvert (v8i32 (X86VBroadcastld32 addr:$src3))),
+                                 VR256X:$src2, (i8 timm:$src4))),
+            (VPTERNLOGDZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                               (VPTERNLOG132_imm8 timm:$src4))>;
 }
 
 let Predicates = [HasAVX512] in {
@@ -11435,6 +11615,36 @@ let Predicates = [HasAVX512] in {
             (VPTERNLOGQZrmi VR512:$src1, VR512:$src2, addr:$src3,
                             (VPTERNLOG132_imm8 timm:$src4))>;
 
+  def : Pat<(v64i8 (X86vpternlog VR512:$src1, VR512:$src2,
+                                 (bitconvert (v16i32 (X86VBroadcastld32 addr:$src3))),
+                                 (i8 timm:$src4))),
+            (VPTERNLOGDZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             timm:$src4)>;
+  def : Pat<(v64i8 (X86vpternlog (bitconvert (v16i32 (X86VBroadcastld32 addr:$src3))),
+                                 VR512:$src2, VR512:$src1, (i8 timm:$src4))),
+            (VPTERNLOGDZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v64i8 (X86vpternlog VR512:$src1,
+                                 (bitconvert (v16i32 (X86VBroadcastld32 addr:$src3))),
+                                 VR512:$src2, (i8 timm:$src4))),
+            (VPTERNLOGDZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v64i8 (X86vpternlog VR512:$src1, VR512:$src2,
+                                 (bitconvert (v8i64 (X86VBroadcastld64 addr:$src3))),
+                                 (i8 timm:$src4))),
+            (VPTERNLOGQZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(v64i8 (X86vpternlog (bitconvert (v8i64 (X86VBroadcastld64 addr:$src3))),
+                                 VR512:$src2, VR512:$src1, (i8 timm:$src4))),
+            (VPTERNLOGQZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v64i8 (X86vpternlog VR512:$src1,
+                                 (bitconvert (v8i64 (X86VBroadcastld64 addr:$src3))),
+                                 VR512:$src2, (i8 timm:$src4))),
+            (VPTERNLOGQZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                                (VPTERNLOG132_imm8 timm:$src4))>;
+
   def : Pat<(v32i16 (X86vpternlog VR512:$src1, VR512:$src2, VR512:$src3,
                                   (i8 timm:$src4))),
             (VPTERNLOGQZrri VR512:$src1, VR512:$src2, VR512:$src3,
@@ -11448,9 +11658,84 @@ let Predicates = [HasAVX512] in {
             (VPTERNLOGQZrmi VR512:$src1, VR512:$src2, addr:$src3,
                             (VPTERNLOG321_imm8 timm:$src4))>;
   def : Pat<(v32i16 (X86vpternlog VR512:$src1, (loadv32i16 addr:$src3),
-                                 VR512:$src2, (i8 timm:$src4))),
+                                  VR512:$src2, (i8 timm:$src4))),
             (VPTERNLOGQZrmi VR512:$src1, VR512:$src2, addr:$src3,
                             (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v32i16 (X86vpternlog VR512:$src1, VR512:$src2,
+                                  (bitconvert (v16i32 (X86VBroadcastld32 addr:$src3))),
+                                  (i8 timm:$src4))),
+            (VPTERNLOGDZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             timm:$src4)>;
+  def : Pat<(v32i16 (X86vpternlog (bitconvert (v16i32 (X86VBroadcastld32 addr:$src3))),
+                                 VR512:$src2, VR512:$src1, (i8 timm:$src4))),
+            (VPTERNLOGDZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v32i16 (X86vpternlog VR512:$src1,
+                                  (bitconvert (v16i32 (X86VBroadcastld32 addr:$src3))),
+                                  VR512:$src2, (i8 timm:$src4))),
+            (VPTERNLOGDZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                               (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v32i16 (X86vpternlog VR512:$src1, VR512:$src2,
+                                  (bitconvert (v8i64 (X86VBroadcastld64 addr:$src3))),
+                                  (i8 timm:$src4))),
+            (VPTERNLOGQZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             timm:$src4)>;
+  def : Pat<(v32i16 (X86vpternlog (bitconvert (v8i64 (X86VBroadcastld64 addr:$src3))),
+                                  VR512:$src2, VR512:$src1, (i8 timm:$src4))),
+            (VPTERNLOGQZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v32i16 (X86vpternlog VR512:$src1,
+                                  (bitconvert (v8i64 (X86VBroadcastld64 addr:$src3))),
+                                  VR512:$src2, (i8 timm:$src4))),
+            (VPTERNLOGQZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v32i16 (X86vpternlog VR512:$src1, VR512:$src2,
+                                  (bitconvert (v16i32 (X86VBroadcastld32 addr:$src3))),
+                                  (i8 timm:$src4))),
+            (VPTERNLOGDZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             timm:$src4)>;
+  def : Pat<(v32i16 (X86vpternlog (bitconvert (v16i32 (X86VBroadcastld32 addr:$src3))),
+                                 VR512:$src2, VR512:$src1, (i8 timm:$src4))),
+            (VPTERNLOGDZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v32i16 (X86vpternlog VR512:$src1,
+                                  (bitconvert (v16i32 (X86VBroadcastld32 addr:$src3))),
+                                  VR512:$src2, (i8 timm:$src4))),
+            (VPTERNLOGDZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                               (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v16i32 (X86vpternlog VR512:$src1, VR512:$src2,
+                                  (bitconvert (v8i64 (X86VBroadcastld64 addr:$src3))),
+                                  (i8 timm:$src4))),
+            (VPTERNLOGQZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             timm:$src4)>;
+  def : Pat<(v16i32 (X86vpternlog (bitconvert (v8i64 (X86VBroadcastld64 addr:$src3))),
+                                  VR512:$src2, VR512:$src1, (i8 timm:$src4))),
+            (VPTERNLOGQZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v16i32 (X86vpternlog VR512:$src1,
+                                  (bitconvert (v8i64 (X86VBroadcastld64 addr:$src3))),
+                                  VR512:$src2, (i8 timm:$src4))),
+            (VPTERNLOGQZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v8i64 (X86vpternlog VR512:$src1, VR512:$src2,
+                                  (bitconvert (v16i32 (X86VBroadcastld32 addr:$src3))),
+                                  (i8 timm:$src4))),
+            (VPTERNLOGDZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             timm:$src4)>;
+  def : Pat<(v8i64 (X86vpternlog (bitconvert (v16i32 (X86VBroadcastld32 addr:$src3))),
+                                 VR512:$src2, VR512:$src1, (i8 timm:$src4))),
+            (VPTERNLOGDZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v8i64 (X86vpternlog VR512:$src1,
+                                  (bitconvert (v16i32 (X86VBroadcastld32 addr:$src3))),
+                                  VR512:$src2, (i8 timm:$src4))),
+            (VPTERNLOGDZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                               (VPTERNLOG132_imm8 timm:$src4))>;
 }
 
 // Patterns to implement vnot using vpternlog instead of creating all ones
diff --git a/llvm/lib/Target/X86/X86InstrFPStack.td b/llvm/lib/Target/X86/X86InstrFPStack.td
index 4d007bd3090a0..67dcb8d00ea53 100644
--- a/llvm/lib/Target/X86/X86InstrFPStack.td
+++ b/llvm/lib/Target/X86/X86InstrFPStack.td
@@ -375,7 +375,8 @@ def TST_F  : FPI<0xD9, MRM_E4, (outs), (ins), "ftst">;
 
 // Versions of FP instructions that take a single memory operand.  Added for the
 //   disassembler; remove as they are included with patterns elsewhere.
-let SchedRW = [WriteFComLd], Uses = [FPCW], mayRaiseFPException = 1 in {
+let SchedRW = [WriteFComLd], Uses = [FPCW], mayRaiseFPException = 1,
+    mayLoad = 1 in {
 def FCOM32m  : FPI<0xD8, MRM2m, (outs), (ins f32mem:$src), "fcom{s}\t$src">;
 def FCOMP32m : FPI<0xD8, MRM3m, (outs), (ins f32mem:$src), "fcomp{s}\t$src">;
 
@@ -390,21 +391,22 @@ def FICOMP32m: FPI<0xDA, MRM3m, (outs), (ins i32mem:$src), "ficomp{l}\t$src">;
 } // SchedRW
 
 let SchedRW = [WriteMicrocoded] in {
-let Defs = [FPSW, FPCW] in {
+let Defs = [FPSW, FPCW], mayLoad = 1 in {
 def FLDENVm  : FPI<0xD9, MRM4m, (outs), (ins f32mem:$src), "fldenv\t$src">;
 def FRSTORm  : FPI<0xDD, MRM4m, (outs), (ins f32mem:$dst), "frstor\t$dst">;
 }
 
-let Defs = [FPSW, FPCW], Uses = [FPSW, FPCW] in {
+let Defs = [FPSW, FPCW], Uses = [FPSW, FPCW], mayStore = 1 in {
 def FSTENVm  : FPI<0xD9, MRM6m, (outs), (ins f32mem:$dst), "fnstenv\t$dst">;
 def FSAVEm   : FPI<0xDD, MRM6m, (outs), (ins f32mem:$dst), "fnsave\t$dst">;
 }
 
-let Uses = [FPSW] in
+let Uses = [FPSW], mayStore = 1 in
 def FNSTSWm  : FPI<0xDD, MRM7m, (outs), (ins i16mem:$dst), "fnstsw\t$dst">;
 
+let mayLoad = 1 in
 def FBLDm    : FPI<0xDF, MRM4m, (outs), (ins f80mem:$src), "fbld\t$src">;
-let Uses = [FPCW] ,mayRaiseFPException = 1 in
+let Uses = [FPCW] ,mayRaiseFPException = 1, mayStore = 1 in
 def FBSTPm   : FPI<0xDF, MRM6m, (outs), (ins f80mem:$dst), "fbstp\t$dst">;
 } // SchedRW
 
diff --git a/llvm/lib/Target/X86/X86InstrFoldTables.cpp b/llvm/lib/Target/X86/X86InstrFoldTables.cpp
index 11fbb37b1181c..e16382e956c57 100644
--- a/llvm/lib/Target/X86/X86InstrFoldTables.cpp
+++ b/llvm/lib/Target/X86/X86InstrFoldTables.cpp
@@ -5521,6 +5521,12 @@ static const X86MemoryFoldTableEntry BroadcastFoldTable3[] = {
   { X86::VFNMSUB231PSZ128r,    X86::VFNMSUB231PSZ128mb,   TB_BCAST_SS },
   { X86::VFNMSUB231PSZ256r,    X86::VFNMSUB231PSZ256mb,   TB_BCAST_SS },
   { X86::VFNMSUB231PSZr,       X86::VFNMSUB231PSZmb,      TB_BCAST_SS },
+  { X86::VPTERNLOGDZ128rri,    X86::VPTERNLOGDZ128rmbi,   TB_BCAST_D },
+  { X86::VPTERNLOGDZ256rri,    X86::VPTERNLOGDZ256rmbi,   TB_BCAST_D },
+  { X86::VPTERNLOGDZrri,       X86::VPTERNLOGDZrmbi,      TB_BCAST_D },
+  { X86::VPTERNLOGQZ128rri,    X86::VPTERNLOGQZ128rmbi,   TB_BCAST_Q },
+  { X86::VPTERNLOGQZ256rri,    X86::VPTERNLOGQZ256rmbi,   TB_BCAST_Q },
+  { X86::VPTERNLOGQZrri,       X86::VPTERNLOGQZrmbi,      TB_BCAST_Q },
 };
 
 static const X86MemoryFoldTableEntry *
diff --git a/llvm/lib/Target/X86/X86InstrFormats.td b/llvm/lib/Target/X86/X86InstrFormats.td
index 41326a6bbe07a..d7752e656b55c 100644
--- a/llvm/lib/Target/X86/X86InstrFormats.td
+++ b/llvm/lib/Target/X86/X86InstrFormats.td
@@ -28,6 +28,9 @@ def RawFrmImm8    : Format<7>;
 def RawFrmImm16   : Format<8>;
 def AddCCFrm      : Format<9>;
 def PrefixByte    : Format<10>;
+def MRMr0          : Format<21>;
+def MRMSrcMemFSIB  : Format<22>;
+def MRMDestMemFSIB : Format<23>;
 def MRMDestMem     : Format<24>;
 def MRMSrcMem      : Format<25>;
 def MRMSrcMem4VOp3 : Format<26>;
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 46ff62f7a4ed5..42c1111735701 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -8660,8 +8660,7 @@ namespace {
       }
 
       // Visit the children of this block in the dominator tree.
-      for (MachineDomTreeNode::iterator I = Node->begin(), E = Node->end();
-           I != E; ++I) {
+      for (auto I = Node->begin(), E = Node->end(); I != E; ++I) {
         Changed |= VisitNode(*I, TLSBaseAddrReg);
       }
 
diff --git a/llvm/lib/Target/X86/X86InstrInfo.td b/llvm/lib/Target/X86/X86InstrInfo.td
index 262961af7c51d..23841c3d7e506 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.td
+++ b/llvm/lib/Target/X86/X86InstrInfo.td
@@ -361,6 +361,8 @@ let RenderMethod = "addMemOperands", SuperClasses = [X86MemAsmOperand] in {
   def X86Mem512_RC256XOperand : AsmOperandClass { let Name = "Mem512_RC256X"; }
   def X86Mem256_RC512Operand  : AsmOperandClass { let Name = "Mem256_RC512"; }
   def X86Mem512_RC512Operand  : AsmOperandClass { let Name = "Mem512_RC512"; }
+
+  def X86SibMemOperand : AsmOperandClass { let Name = "SibMem"; }
 }
 
 def X86AbsMemAsmOperand : AsmOperandClass {
@@ -392,6 +394,8 @@ def X86any_fcmp : PatFrags<(ops node:$lhs, node:$rhs),
 // restrict to only unsized memory.
 def opaquemem : X86MemOperand<"printMemReference">;
 
+def sibmem: X86MemOperand<"printMemReference", X86SibMemOperand>;
+
 def i8mem   : X86MemOperand<"printbytemem",   X86Mem8AsmOperand>;
 def i16mem  : X86MemOperand<"printwordmem",  X86Mem16AsmOperand>;
 def i32mem  : X86MemOperand<"printdwordmem",  X86Mem32AsmOperand>;
@@ -928,11 +932,10 @@ def HasRTM       : Predicate<"Subtarget->hasRTM()">;
 def HasADX       : Predicate<"Subtarget->hasADX()">;
 def HasSHA       : Predicate<"Subtarget->hasSHA()">;
 def HasSGX       : Predicate<"Subtarget->hasSGX()">;
-def HasPRFCHW    : Predicate<"Subtarget->hasPRFCHW()">;
 def HasRDSEED    : Predicate<"Subtarget->hasRDSEED()">;
 def HasSSEPrefetch : Predicate<"Subtarget->hasSSEPrefetch()">;
 def NoSSEPrefetch : Predicate<"!Subtarget->hasSSEPrefetch()">;
-def HasPrefetchW : Predicate<"Subtarget->hasPRFCHW()">;
+def HasPrefetchW : Predicate<"Subtarget->hasPrefetchW()">;
 def HasPREFETCHWT1 : Predicate<"Subtarget->hasPREFETCHWT1()">;
 def HasLAHFSAHF  : Predicate<"Subtarget->hasLAHFSAHF()">;
 def HasMWAITX    : Predicate<"Subtarget->hasMWAITX()">;
@@ -956,6 +959,9 @@ def HasPCONFIG   : Predicate<"Subtarget->hasPCONFIG()">;
 def HasENQCMD    : Predicate<"Subtarget->hasENQCMD()">;
 def HasSERIALIZE : Predicate<"Subtarget->hasSERIALIZE()">;
 def HasTSXLDTRK  : Predicate<"Subtarget->hasTSXLDTRK()">;
+def HasAMXTILE   : Predicate<"Subtarget->hasAMXTILE()">;
+def HasAMXBF16   : Predicate<"Subtarget->hasAMXBF16()">;
+def HasAMXINT8   : Predicate<"Subtarget->hasAMXINT8()">;
 def Not64BitMode : Predicate<"!Subtarget->is64Bit()">,
                              AssemblerPredicate<(all_of (not Mode64Bit)), "Not 64-bit mode">;
 def In64BitMode  : Predicate<"Subtarget->is64Bit()">,
@@ -3071,6 +3077,9 @@ include "X86InstrSVM.td"
 include "X86InstrTSX.td"
 include "X86InstrSGX.td"
 
+// AMX instructions
+include "X86InstrAMX.td"
+
 // System instructions.
 include "X86InstrSystem.td"
 
diff --git a/llvm/lib/Target/X86/X86InstrXOP.td b/llvm/lib/Target/X86/X86InstrXOP.td
index 8b991ae3e21ed..a5976b7d2d74a 100644
--- a/llvm/lib/Target/X86/X86InstrXOP.td
+++ b/llvm/lib/Target/X86/X86InstrXOP.td
@@ -335,13 +335,13 @@ multiclass xop4op_int<bits<8> opc, string OpcodeStr, RegisterClass RC,
             [(set RC:$dst, (VT (or (and RC:$src3, RC:$src1),
                                    (X86andnp RC:$src3, RC:$src2))))]>, XOP_4V,
             Sched<[sched]>;
-  // FIXME: This pattern can't match.
+  // FIXME: We can't write a pattern for this in tablegen.
+  let hasSideEffects = 0, mayLoad = 1 in
   def rrm : IXOPi8Reg<opc, MRMSrcMemOp4, (outs RC:$dst),
             (ins RC:$src1, RC:$src2, x86memop:$src3),
             !strconcat(OpcodeStr,
             "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
-            [(set RC:$dst, (VT (or (and (load addr:$src3), RC:$src1),
-                                   (X86andnp (load addr:$src3), RC:$src2))))]>,
+            []>,
             XOP_4V, VEX_W, Sched<[sched.Folded, sched.ReadAfterFold, sched.ReadAfterFold]>;
   def rmr : IXOPi8Reg<opc, MRMSrcMem, (outs RC:$dst),
             (ins RC:$src1, x86memop:$src2, RC:$src3),
@@ -383,13 +383,13 @@ let Predicates = [HasXOP] in {
             (VPCMOVrrr VR128:$src1, VR128:$src2, VR128:$src3)>;
 
   def : Pat<(or (and VR128:$src3, VR128:$src1),
-                (X86andnp VR128:$src3, (bc_v16i8 (loadv2i64 addr:$src2)))),
+                (X86andnp VR128:$src3, (loadv16i8 addr:$src2))),
             (VPCMOVrmr VR128:$src1, addr:$src2, VR128:$src3)>;
   def : Pat<(or (and VR128:$src3, VR128:$src1),
-                (X86andnp VR128:$src3, (bc_v8i16 (loadv2i64 addr:$src2)))),
+                (X86andnp VR128:$src3, (loadv8i16 addr:$src2))),
             (VPCMOVrmr VR128:$src1, addr:$src2, VR128:$src3)>;
   def : Pat<(or (and VR128:$src3, VR128:$src1),
-                (X86andnp VR128:$src3, (bc_v4i32 (loadv2i64 addr:$src2)))),
+                (X86andnp VR128:$src3, (loadv4i32 addr:$src2))),
             (VPCMOVrmr VR128:$src1, addr:$src2, VR128:$src3)>;
 
   def : Pat<(v32i8 (or (and VR256:$src3, VR256:$src1),
@@ -403,13 +403,13 @@ let Predicates = [HasXOP] in {
             (VPCMOVYrrr VR256:$src1, VR256:$src2, VR256:$src3)>;
 
   def : Pat<(or (and VR256:$src3, VR256:$src1),
-                (X86andnp VR256:$src3, (bc_v32i8 (loadv4i64 addr:$src2)))),
+                (X86andnp VR256:$src3, (loadv32i8 addr:$src2))),
             (VPCMOVYrmr VR256:$src1, addr:$src2, VR256:$src3)>;
   def : Pat<(or (and VR256:$src3, VR256:$src1),
-                (X86andnp VR256:$src3, (bc_v16i16 (loadv4i64 addr:$src2)))),
+                (X86andnp VR256:$src3, (loadv16i16 addr:$src2))),
             (VPCMOVYrmr VR256:$src1, addr:$src2, VR256:$src3)>;
   def : Pat<(or (and VR256:$src3, VR256:$src1),
-                (X86andnp VR256:$src3, (bc_v8i32 (loadv4i64 addr:$src2)))),
+                (X86andnp VR256:$src3, (loadv8i32 addr:$src2))),
             (VPCMOVYrmr VR256:$src1, addr:$src2, VR256:$src3)>;
 }
 
diff --git a/llvm/lib/Target/X86/X86InterleavedAccess.cpp b/llvm/lib/Target/X86/X86InterleavedAccess.cpp
index 00ac238f284bd..a19e12766e101 100644
--- a/llvm/lib/Target/X86/X86InterleavedAccess.cpp
+++ b/llvm/lib/Target/X86/X86InterleavedAccess.cpp
@@ -69,7 +69,7 @@ class X86InterleavedAccessGroup {
 
   /// Breaks down a vector \p 'Inst' of N elements into \p NumSubVectors
   /// sub vectors of type \p T. Returns the sub-vectors in \p DecomposedVectors.
-  void decompose(Instruction *Inst, unsigned NumSubVectors, VectorType *T,
+  void decompose(Instruction *Inst, unsigned NumSubVectors, FixedVectorType *T,
                  SmallVectorImpl<Instruction *> &DecomposedVectors);
 
   /// Performs matrix transposition on a 4x4 matrix \p InputVectors and
@@ -165,7 +165,7 @@ bool X86InterleavedAccessGroup::isSupported() const {
 }
 
 void X86InterleavedAccessGroup::decompose(
-    Instruction *VecInst, unsigned NumSubVectors, VectorType *SubVecTy,
+    Instruction *VecInst, unsigned NumSubVectors, FixedVectorType *SubVecTy,
     SmallVectorImpl<Instruction *> &DecomposedVectors) {
   assert((isa<LoadInst>(VecInst) || isa<ShuffleVectorInst>(VecInst)) &&
          "Expected Load or Shuffle");
@@ -727,13 +727,13 @@ void X86InterleavedAccessGroup::transpose_4x4(
 bool X86InterleavedAccessGroup::lowerIntoOptimizedSequence() {
   SmallVector<Instruction *, 4> DecomposedVectors;
   SmallVector<Value *, 4> TransposedVectors;
-  VectorType *ShuffleTy = Shuffles[0]->getType();
+  auto *ShuffleTy = cast<FixedVectorType>(Shuffles[0]->getType());
 
   if (isa<LoadInst>(Inst)) {
     // Try to generate target-sized register(/instruction).
     decompose(Inst, Factor, ShuffleTy, DecomposedVectors);
 
-    auto *ShuffleEltTy = cast<VectorType>(Inst->getType());
+    auto *ShuffleEltTy = cast<FixedVectorType>(Inst->getType());
     unsigned NumSubVecElems = ShuffleEltTy->getNumElements() / Factor;
     // Perform matrix-transposition in order to compute interleaved
     // results by generating some sort of (optimized) target-specific
@@ -832,7 +832,8 @@ bool X86TargetLowering::lowerInterleavedStore(StoreInst *SI,
   assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
          "Invalid interleave factor");
 
-  assert(SVI->getType()->getNumElements() % Factor == 0 &&
+  assert(cast<FixedVectorType>(SVI->getType())->getNumElements() % Factor ==
+             0 &&
          "Invalid interleaved store");
 
   // Holds the indices of SVI that correspond to the starting index of each
diff --git a/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp b/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
index 35fc439998f90..50f8b3477acce 100644
--- a/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
+++ b/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
@@ -822,79 +822,3 @@ INITIALIZE_PASS_END(X86LoadValueInjectionLoadHardeningPass, PASS_KEY,
 FunctionPass *llvm::createX86LoadValueInjectionLoadHardeningPass() {
   return new X86LoadValueInjectionLoadHardeningPass();
 }
-
-namespace {
-
-/// The `X86LoadValueInjectionLoadHardeningPass` above depends on expensive
-/// analysis passes that add complexity to the pipeline. This complexity
-/// can cause noticable overhead when no optimizations are enabled, i.e., -O0.
-/// The purpose of `X86LoadValueInjectionLoadHardeningUnoptimizedPass` is to
-/// provide the same security as the optimized pass, but without adding
-/// unnecessary complexity to the LLVM pipeline.
-///
-/// The behavior of this pass is simply to insert an LFENCE after every load
-/// instruction.
-class X86LoadValueInjectionLoadHardeningUnoptimizedPass
-    : public MachineFunctionPass {
-public:
-  X86LoadValueInjectionLoadHardeningUnoptimizedPass()
-      : MachineFunctionPass(ID) {}
-
-  StringRef getPassName() const override {
-    return "X86 Load Value Injection (LVI) Load Hardening (Unoptimized)";
-  }
-  bool runOnMachineFunction(MachineFunction &MF) override;
-  static char ID;
-};
-
-} // end anonymous namespace
-
-char X86LoadValueInjectionLoadHardeningUnoptimizedPass::ID = 0;
-
-bool X86LoadValueInjectionLoadHardeningUnoptimizedPass::runOnMachineFunction(
-    MachineFunction &MF) {
-  LLVM_DEBUG(dbgs() << "***** " << getPassName() << " : " << MF.getName()
-                    << " *****\n");
-  const X86Subtarget *STI = &MF.getSubtarget<X86Subtarget>();
-  if (!STI->useLVILoadHardening())
-    return false;
-
-  // FIXME: support 32-bit
-  if (!STI->is64Bit())
-    report_fatal_error("LVI load hardening is only supported on 64-bit", false);
-
-  // Don't skip functions with the "optnone" attr but participate in opt-bisect.
-  const Function &F = MF.getFunction();
-  if (!F.hasOptNone() && skipFunction(F))
-    return false;
-
-  bool Modified = false;
-  ++NumFunctionsConsidered;
-
-  const TargetInstrInfo *TII = STI->getInstrInfo();
-  for (auto &MBB : MF) {
-    for (auto &MI : MBB) {
-      if (!MI.mayLoad() || MI.getOpcode() == X86::LFENCE ||
-          MI.getOpcode() == X86::MFENCE)
-        continue;
-
-      MachineBasicBlock::iterator InsertionPt =
-          MI.getNextNode() ? MI.getNextNode() : MBB.end();
-      BuildMI(MBB, InsertionPt, DebugLoc(), TII->get(X86::LFENCE));
-      ++NumFences;
-      Modified = true;
-    }
-  }
-
-  if (Modified)
-    ++NumFunctionsMitigated;
-
-  return Modified;
-}
-
-INITIALIZE_PASS(X86LoadValueInjectionLoadHardeningUnoptimizedPass, PASS_KEY,
-                "X86 LVI load hardening", false, false)
-
-FunctionPass *llvm::createX86LoadValueInjectionLoadHardeningUnoptimizedPass() {
-  return new X86LoadValueInjectionLoadHardeningUnoptimizedPass();
-}
diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index 60f29ae5e6a51..9ce2a4637e2ea 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -518,7 +518,7 @@ void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
     switch (OutMI.getOpcode()) {
     default: llvm_unreachable("Invalid opcode");
     case X86::MULX32Hrr: NewOpc = X86::MULX32rr; break;
-    case X86::MULX32Hrm: NewOpc = X86::MULX32rr; break;
+    case X86::MULX32Hrm: NewOpc = X86::MULX32rm; break;
     case X86::MULX64Hrr: NewOpc = X86::MULX64rr; break;
     case X86::MULX64Hrm: NewOpc = X86::MULX64rm; break;
     }
diff --git a/llvm/lib/Target/X86/X86OptimizeLEAs.cpp b/llvm/lib/Target/X86/X86OptimizeLEAs.cpp
index 0c791b6674dcb..c8899a85118e1 100644
--- a/llvm/lib/Target/X86/X86OptimizeLEAs.cpp
+++ b/llvm/lib/Target/X86/X86OptimizeLEAs.cpp
@@ -578,7 +578,7 @@ bool X86OptimizeLEAPass::removeRedundantAddrCalc(MemOpMap &LEAs) {
 MachineInstr *X86OptimizeLEAPass::replaceDebugValue(MachineInstr &MI,
                                                     unsigned VReg,
                                                     int64_t AddrDispShift) {
-  DIExpression *Expr = const_cast<DIExpression *>(MI.getDebugExpression());
+  const DIExpression *Expr = MI.getDebugExpression();
   if (AddrDispShift != 0)
     Expr = DIExpression::prepend(Expr, DIExpression::StackValue, AddrDispShift);
 
diff --git a/llvm/lib/Target/X86/X86PartialReduction.cpp b/llvm/lib/Target/X86/X86PartialReduction.cpp
index 65caeab1d1cf2..8784a3df17737 100644
--- a/llvm/lib/Target/X86/X86PartialReduction.cpp
+++ b/llvm/lib/Target/X86/X86PartialReduction.cpp
@@ -68,7 +68,7 @@ bool X86PartialReduction::tryMAddReplacement(Instruction *Op) {
     return false;
 
   // Need at least 8 elements.
-  if (cast<VectorType>(Op->getType())->getNumElements() < 8)
+  if (cast<FixedVectorType>(Op->getType())->getNumElements() < 8)
     return false;
 
   // Element type should be i32.
@@ -136,7 +136,7 @@ bool X86PartialReduction::tryMAddReplacement(Instruction *Op) {
 
   IRBuilder<> Builder(Mul);
 
-  auto *MulTy = cast<VectorType>(Op->getType());
+  auto *MulTy = cast<FixedVectorType>(Op->getType());
   unsigned NumElts = MulTy->getNumElements();
 
   // Extract even elements and odd elements and add them together. This will
@@ -211,7 +211,7 @@ bool X86PartialReduction::trySADReplacement(Instruction *Op) {
 
   IRBuilder<> Builder(SI);
 
-  auto *OpTy = cast<VectorType>(Op->getType());
+  auto *OpTy = cast<FixedVectorType>(Op->getType());
   unsigned NumElts = OpTy->getNumElements();
 
   unsigned IntrinsicNumElts;
@@ -265,7 +265,7 @@ bool X86PartialReduction::trySADReplacement(Instruction *Op) {
   unsigned Stages = Log2_32(NumSplits);
   for (unsigned s = Stages; s > 0; --s) {
     unsigned NumConcatElts =
-        cast<VectorType>(Ops[0]->getType())->getNumElements() * 2;
+        cast<FixedVectorType>(Ops[0]->getType())->getNumElements() * 2;
     for (unsigned i = 0; i != 1U << (s - 1); ++i) {
       SmallVector<int, 64> ConcatMask(NumConcatElts);
       std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
@@ -275,13 +275,14 @@ bool X86PartialReduction::trySADReplacement(Instruction *Op) {
 
   // At this point the final value should be in Ops[0]. Now we need to adjust
   // it to the final original type.
-  NumElts = cast<VectorType>(OpTy)->getNumElements();
+  NumElts = cast<FixedVectorType>(OpTy)->getNumElements();
   if (NumElts == 2) {
     // Extract down to 2 elements.
     Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], ArrayRef<int>{0, 1});
   } else if (NumElts >= 8) {
     SmallVector<int, 32> ConcatMask(NumElts);
-    unsigned SubElts = cast<VectorType>(Ops[0]->getType())->getNumElements();
+    unsigned SubElts =
+        cast<FixedVectorType>(Ops[0]->getType())->getNumElements();
     for (unsigned i = 0; i != SubElts; ++i)
       ConcatMask[i] = i;
     for (unsigned i = SubElts; i != NumElts; ++i)
@@ -309,7 +310,7 @@ static Value *matchAddReduction(const ExtractElementInst &EE) {
   if (!BO || BO->getOpcode() != Instruction::Add || !BO->hasOneUse())
     return nullptr;
 
-  unsigned NumElems = cast<VectorType>(BO->getType())->getNumElements();
+  unsigned NumElems = cast<FixedVectorType>(BO->getType())->getNumElements();
   // Ensure the reduction size is a power of 2.
   if (!isPowerOf2_32(NumElems))
     return nullptr;
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.td b/llvm/lib/Target/X86/X86RegisterInfo.td
index b2c9025f1f717..8de5b94bbffa9 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.td
+++ b/llvm/lib/Target/X86/X86RegisterInfo.td
@@ -265,6 +265,16 @@ let SubRegIndices = [sub_ymm] in {
   }
 }
 
+// Tile "registers".
+def TMM0:  X86Reg<"tmm0",   0>;
+def TMM1:  X86Reg<"tmm1",   1>;
+def TMM2:  X86Reg<"tmm2",   2>;
+def TMM3:  X86Reg<"tmm3",   3>;
+def TMM4:  X86Reg<"tmm4",   4>;
+def TMM5:  X86Reg<"tmm5",   5>;
+def TMM6:  X86Reg<"tmm6",   6>;
+def TMM7:  X86Reg<"tmm7",   7>;
+
 // Mask Registers, used by AVX-512 instructions.
 def K0 : X86Reg<"k0", 0>, DwarfRegNum<[118,  93,  93]>;
 def K1 : X86Reg<"k1", 1>, DwarfRegNum<[119,  94,  94]>;
@@ -621,3 +631,8 @@ def VK64WM  : RegisterClass<"X86", [v64i1], 64, (add VK32WM)> {let Size = 64;}
 
 // Bound registers
 def BNDR : RegisterClass<"X86", [v2i64], 128, (sequence "BND%u", 0, 3)>;
+
+// Tiles
+let isAllocatable = 0 in
+def TILE : RegisterClass<"X86", [untyped], 0,
+                         (sequence "TMM%u", 0, 7)> {let Size = 8192;}
diff --git a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
index 9e92a55dc479e..ce8d1d464da97 100644
--- a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
@@ -46,7 +46,7 @@ bool X86SelectionDAGInfo::isBaseRegConflictPossible(
 
 SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Val,
-    SDValue Size, unsigned Align, bool isVolatile,
+    SDValue Size, Align Alignment, bool isVolatile,
     MachinePointerInfo DstPtrInfo) const {
   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
   const X86Subtarget &Subtarget =
@@ -66,7 +66,7 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
   // If not DWORD aligned or size is more than the threshold, call the library.
   // The libc version is likely to be faster for these cases. It can use the
   // address value and run time information about the CPU.
-  if ((Align & 3) != 0 || !ConstantSize ||
+  if (Alignment < Align(4) || !ConstantSize ||
       ConstantSize->getZExtValue() > Subtarget.getMaxInlineSizeThreshold()) {
     // Check to see if there is a specialized entry-point for memory zeroing.
     ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Val);
@@ -112,28 +112,27 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
     uint64_t Val = ValC->getZExtValue() & 255;
 
     // If the value is a constant, then we can potentially use larger sets.
-    switch (Align & 3) {
-    case 2:   // WORD aligned
-      AVT = MVT::i16;
-      ValReg = X86::AX;
-      Val = (Val << 8) | Val;
-      break;
-    case 0:  // DWORD aligned
+    if (Alignment > Align(2)) {
+      // DWORD aligned
       AVT = MVT::i32;
       ValReg = X86::EAX;
       Val = (Val << 8)  | Val;
       Val = (Val << 16) | Val;
-      if (Subtarget.is64Bit() && ((Align & 0x7) == 0)) {  // QWORD aligned
+      if (Subtarget.is64Bit() && Alignment > Align(8)) { // QWORD aligned
         AVT = MVT::i64;
         ValReg = X86::RAX;
         Val = (Val << 32) | Val;
       }
-      break;
-    default:  // Byte aligned
+    } else if (Alignment == Align(2)) {
+      // WORD aligned
+      AVT = MVT::i16;
+      ValReg = X86::AX;
+      Val = (Val << 8) | Val;
+    } else {
+      // Byte aligned
       AVT = MVT::i8;
       ValReg = X86::AL;
       Count = DAG.getIntPtrConstant(SizeVal, dl);
-      break;
     }
 
     if (AVT.bitsGT(MVT::i8)) {
@@ -170,12 +169,12 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
     EVT AddrVT = Dst.getValueType();
     EVT SizeVT = Size.getValueType();
 
-    Chain = DAG.getMemset(Chain, dl,
-                          DAG.getNode(ISD::ADD, dl, AddrVT, Dst,
-                                      DAG.getConstant(Offset, dl, AddrVT)),
-                          Val, DAG.getConstant(BytesLeft, dl, SizeVT),
-                          llvm::Align(Align), isVolatile, false,
-                          DstPtrInfo.getWithOffset(Offset));
+    Chain =
+        DAG.getMemset(Chain, dl,
+                      DAG.getNode(ISD::ADD, dl, AddrVT, Dst,
+                                  DAG.getConstant(Offset, dl, AddrVT)),
+                      Val, DAG.getConstant(BytesLeft, dl, SizeVT), Alignment,
+                      isVolatile, false, DstPtrInfo.getWithOffset(Offset));
   }
 
   // TODO: Use a Tokenfactor, as in memcpy, instead of a single chain.
@@ -291,7 +290,7 @@ static SDValue emitConstantSizeRepmov(
 
 SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy(
     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
-    SDValue Size, unsigned Align, bool isVolatile, bool AlwaysInline,
+    SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
     MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
   // If to a segment-relative address space, use the default lowering.
   if (DstPtrInfo.getAddrSpace() >= 256 || SrcPtrInfo.getAddrSpace() >= 256)
@@ -309,10 +308,10 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy(
 
   /// Handle constant sizes,
   if (ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size))
-    return emitConstantSizeRepmov(DAG, Subtarget, dl, Chain, Dst, Src,
-                                  ConstantSize->getZExtValue(),
-                                  Size.getValueType(), Align, isVolatile,
-                                  AlwaysInline, DstPtrInfo, SrcPtrInfo);
+    return emitConstantSizeRepmov(
+        DAG, Subtarget, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
+        Size.getValueType(), Alignment.value(), isVolatile, AlwaysInline,
+        DstPtrInfo, SrcPtrInfo);
 
   return SDValue();
 }
diff --git a/llvm/lib/Target/X86/X86SelectionDAGInfo.h b/llvm/lib/Target/X86/X86SelectionDAGInfo.h
index 63ba44f1c86b0..dac62973636c5 100644
--- a/llvm/lib/Target/X86/X86SelectionDAGInfo.h
+++ b/llvm/lib/Target/X86/X86SelectionDAGInfo.h
@@ -28,13 +28,14 @@ class X86SelectionDAGInfo : public SelectionDAGTargetInfo {
 
   SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl,
                                   SDValue Chain, SDValue Dst, SDValue Src,
-                                  SDValue Size, unsigned Align, bool isVolatile,
+                                  SDValue Size, Align Alignment,
+                                  bool isVolatile,
                                   MachinePointerInfo DstPtrInfo) const override;
 
   SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
                                   SDValue Chain, SDValue Dst, SDValue Src,
-                                  SDValue Size, unsigned Align, bool isVolatile,
-                                  bool AlwaysInline,
+                                  SDValue Size, Align Alignment,
+                                  bool isVolatile, bool AlwaysInline,
                                   MachinePointerInfo DstPtrInfo,
                                   MachinePointerInfo SrcPtrInfo) const override;
 };
diff --git a/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp b/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp
index 3408dab036564..de528299654c1 100644
--- a/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp
+++ b/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp
@@ -36,7 +36,7 @@ static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits,
   //
   //   <4 x i32> <i32 -2147483648, i32 -2147483648,
   //              i32 -2147483648, i32 -2147483648>
-  auto *CstTy = dyn_cast<VectorType>(C->getType());
+  auto *CstTy = dyn_cast<FixedVectorType>(C->getType());
   if (!CstTy)
     return false;
 
diff --git a/llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp b/llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp
index 1f8285eadb7eb..7e91c37367d2f 100644
--- a/llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp
+++ b/llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp
@@ -22,6 +22,7 @@
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/Pass.h"
+#include "llvm/Target/TargetMachine.h"
 using namespace llvm;
 
 #define DEBUG_TYPE "x86-seses"
@@ -29,8 +30,8 @@ using namespace llvm;
 STATISTIC(NumLFENCEsInserted, "Number of lfence instructions inserted");
 
 static cl::opt<bool> EnableSpeculativeExecutionSideEffectSuppression(
-    "x86-seses-enable",
-    cl::desc("Force enable speculative execution side effect suppresion. "
+    "x86-seses-enable-without-lvi-cfi",
+    cl::desc("Force enable speculative execution side effect suppression. "
              "(Note: User must pass -mlvi-cfi in order to mitigate indirect "
              "branches and returns.)"),
     cl::init(false), cl::Hidden);
@@ -86,27 +87,44 @@ static bool hasConstantAddressingMode(const MachineInstr &MI) {
 
 bool X86SpeculativeExecutionSideEffectSuppression::runOnMachineFunction(
     MachineFunction &MF) {
-  if (!EnableSpeculativeExecutionSideEffectSuppression)
+
+  const auto &OptLevel = MF.getTarget().getOptLevel();
+  const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
+
+  // Check whether SESES needs to run as the fallback for LVI at O0, whether the
+  // user explicitly passed an SESES flag, or whether the SESES target feature
+  // was set.
+  if (!EnableSpeculativeExecutionSideEffectSuppression &&
+      !(Subtarget.useLVILoadHardening() && OptLevel == CodeGenOpt::None) &&
+      !Subtarget.useSpeculativeExecutionSideEffectSuppression())
     return false;
 
   LLVM_DEBUG(dbgs() << "********** " << getPassName() << " : " << MF.getName()
                     << " **********\n");
   bool Modified = false;
-  const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
   const X86InstrInfo *TII = Subtarget.getInstrInfo();
   for (MachineBasicBlock &MBB : MF) {
     MachineInstr *FirstTerminator = nullptr;
-
+    // Keep track of whether the previous instruction was an LFENCE to avoid
+    // adding redundant LFENCEs.
+    bool PrevInstIsLFENCE = false;
     for (auto &MI : MBB) {
+
+      if (MI.getOpcode() == X86::LFENCE) {
+        PrevInstIsLFENCE = true;
+        continue;
+      }
       // We want to put an LFENCE before any instruction that
       // may load or store. This LFENCE is intended to avoid leaking any secret
       // data due to a given load or store. This results in closing the cache
       // and memory timing side channels. We will treat terminators that load
       // or store separately.
       if (MI.mayLoadOrStore() && !MI.isTerminator()) {
-        BuildMI(MBB, MI, DebugLoc(), TII->get(X86::LFENCE));
-        NumLFENCEsInserted++;
-        Modified = true;
+        if (!PrevInstIsLFENCE) {
+          BuildMI(MBB, MI, DebugLoc(), TII->get(X86::LFENCE));
+          NumLFENCEsInserted++;
+          Modified = true;
+        }
         if (OneLFENCEPerBasicBlock)
           break;
       }
@@ -128,19 +146,25 @@ bool X86SpeculativeExecutionSideEffectSuppression::runOnMachineFunction(
 
       // Look for branch instructions that will require an LFENCE to be put
       // before this basic block's terminators.
-      if (!MI.isBranch() || OmitBranchLFENCEs)
+      if (!MI.isBranch() || OmitBranchLFENCEs) {
         // This isn't a branch or we're not putting LFENCEs before branches.
+        PrevInstIsLFENCE = false;
         continue;
+      }
 
-      if (OnlyLFENCENonConst && hasConstantAddressingMode(MI))
+      if (OnlyLFENCENonConst && hasConstantAddressingMode(MI)) {
         // This is a branch, but it only has constant addressing mode and we're
         // not adding LFENCEs before such branches.
+        PrevInstIsLFENCE = false;
         continue;
+      }
 
       // This branch requires adding an LFENCE.
-      BuildMI(MBB, FirstTerminator, DebugLoc(), TII->get(X86::LFENCE));
-      NumLFENCEsInserted++;
-      Modified = true;
+      if (!PrevInstIsLFENCE) {
+        BuildMI(MBB, FirstTerminator, DebugLoc(), TII->get(X86::LFENCE));
+        NumLFENCEsInserted++;
+        Modified = true;
+      }
       break;
     }
   }
@@ -153,5 +177,5 @@ FunctionPass *llvm::createX86SpeculativeExecutionSideEffectSuppression() {
 }
 
 INITIALIZE_PASS(X86SpeculativeExecutionSideEffectSuppression, "x86-seses",
-                "X86 Speculative Execution Side Effect Suppresion", false,
+                "X86 Speculative Execution Side Effect Suppression", false,
                 false)
diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h
index 5c514311db90f..de45d357e3c2d 100644
--- a/llvm/lib/Target/X86/X86Subtarget.h
+++ b/llvm/lib/Target/X86/X86Subtarget.h
@@ -403,6 +403,11 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   /// Processor supports TSXLDTRK instruction
   bool HasTSXLDTRK = false;
 
+  /// Processor has AMX support
+  bool HasAMXTILE = false;
+  bool HasAMXBF16 = false;
+  bool HasAMXINT8 = false;
+
   /// Processor has a single uop BEXTR implementation.
   bool HasFastBEXTR = false;
 
@@ -437,6 +442,9 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   /// POP+LFENCE+JMP sequence.
   bool UseLVIControlFlowIntegrity = false;
 
+  /// Enable Speculative Execution Side Effect Suppression
+  bool UseSpeculativeExecutionSideEffectSuppression = false;
+
   /// Insert LFENCE instructions to prevent data speculatively injected into
   /// loads from being used maliciously.
   bool UseLVILoadHardening = false;
@@ -647,8 +655,15 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   bool hasRTM() const { return HasRTM; }
   bool hasADX() const { return HasADX; }
   bool hasSHA() const { return HasSHA; }
-  bool hasPRFCHW() const { return HasPRFCHW || HasPREFETCHWT1; }
+  bool hasPRFCHW() const { return HasPRFCHW; }
   bool hasPREFETCHWT1() const { return HasPREFETCHWT1; }
+  bool hasPrefetchW() const {
+    // The PREFETCHW instruction was added with 3DNow but later CPUs gave it
+    // its own CPUID bit as part of deprecating 3DNow. Intel eventually added
+    // it and KNL has another that prefetches to L2 cache. We assume the
+    // L1 version exists if the L2 version does.
+    return has3DNow() || hasPRFCHW() || hasPREFETCHWT1();
+  }
   bool hasSSEPrefetch() const {
     // We implicitly enable these when we have a write prefix supporting cache
     // level OR if we have prfchw, but don't already have a read prefetch from
@@ -728,6 +743,9 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   bool useRetpolineIndirectBranches() const {
     return UseRetpolineIndirectBranches;
   }
+  bool hasAMXTILE() const { return HasAMXTILE; }
+  bool hasAMXBF16() const { return HasAMXBF16; }
+  bool hasAMXINT8() const { return HasAMXINT8; }
   bool useRetpolineExternalThunk() const { return UseRetpolineExternalThunk; }
 
   // These are generic getters that OR together all of the thunk types
@@ -744,6 +762,9 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   bool useGLMDivSqrtCosts() const { return UseGLMDivSqrtCosts; }
   bool useLVIControlFlowIntegrity() const { return UseLVIControlFlowIntegrity; }
   bool useLVILoadHardening() const { return UseLVILoadHardening; }
+  bool useSpeculativeExecutionSideEffectSuppression() const {
+    return UseSpeculativeExecutionSideEffectSuppression;
+  }
 
   unsigned getPreferVectorWidth() const { return PreferVectorWidth; }
   unsigned getRequiredVectorWidth() const { return RequiredVectorWidth; }
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index 7e00b30915a0c..7344116e14af6 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -489,10 +489,12 @@ void X86PassConfig::addMachineSSAOptimization() {
 
 void X86PassConfig::addPostRegAlloc() {
   addPass(createX86FloatingPointStackifierPass());
+  // When -O0 is enabled, the Load Value Injection Hardening pass will fall back
+  // to using the Speculative Execution Side Effect Suppression pass for
+  // mitigation. This is to prevent slow downs due to
+  // analyses needed by the LVIHardening pass when compiling at -O0.
   if (getOptLevel() != CodeGenOpt::None)
     addPass(createX86LoadValueInjectionLoadHardeningPass());
-  else
-    addPass(createX86LoadValueInjectionLoadHardeningUnoptimizedPass());
 }
 
 void X86PassConfig::addPreSched2() { addPass(createX86ExpandPseudoPass()); }
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index ca9f373d28ba1..cc18e55656ef5 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -973,7 +973,7 @@ int X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *BaseTp,
       // FIXME: Remove some of the alignment restrictions.
       // FIXME: We can use permq for 64-bit or larger extracts from 256-bit
       // vectors.
-      int OrigSubElts = cast<VectorType>(SubTp)->getNumElements();
+      int OrigSubElts = cast<FixedVectorType>(SubTp)->getNumElements();
       if (NumSubElts > OrigSubElts && (Index % OrigSubElts) == 0 &&
           (NumSubElts % OrigSubElts) == 0 &&
           LT.second.getVectorElementType() ==
@@ -1047,7 +1047,8 @@ int X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *BaseTp,
     if (LegalVT.isVector() &&
         LegalVT.getVectorElementType().getSizeInBits() ==
             BaseTp->getElementType()->getPrimitiveSizeInBits() &&
-        LegalVT.getVectorNumElements() < BaseTp->getNumElements()) {
+        LegalVT.getVectorNumElements() <
+            cast<FixedVectorType>(BaseTp)->getNumElements()) {
 
       unsigned VecTySize = DL.getTypeStoreSize(BaseTp);
       unsigned LegalVTSize = LegalVT.getStoreSize();
@@ -2771,7 +2772,7 @@ int X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
 
   Intrinsic::ID IID = ICA.getID();
   Type *RetTy = ICA.getReturnType();
-  const SmallVectorImpl<Value *> &Args = ICA.getArgs();
+  const SmallVectorImpl<const Value *> &Args = ICA.getArgs();
   unsigned ISD = ISD::DELETED_NODE;
   switch (IID) {
   default:
@@ -2935,7 +2936,8 @@ unsigned X86TTIImpl::getScalarizationOverhead(VectorType *Ty,
         // 128-bit vector is free.
         // NOTE: This assumes legalization widens vXf32 vectors.
         if (MScalarTy == MVT::f32)
-          for (unsigned i = 0, e = Ty->getNumElements(); i < e; i += 4)
+          for (unsigned i = 0, e = cast<FixedVectorType>(Ty)->getNumElements();
+               i < e; i += 4)
             if (DemandedElts[i])
               Cost--;
       }
@@ -2951,7 +2953,8 @@ unsigned X86TTIImpl::getScalarizationOverhead(VectorType *Ty,
       // vector elements, which represents the number of unpacks we'll end up
       // performing.
       unsigned NumElts = LT.second.getVectorNumElements();
-      unsigned Pow2Elts = PowerOf2Ceil(Ty->getNumElements());
+      unsigned Pow2Elts =
+          PowerOf2Ceil(cast<FixedVectorType>(Ty)->getNumElements());
       Cost += (std::min<unsigned>(NumElts, Pow2Elts) - 1) * LT.first;
     }
   }
@@ -2983,7 +2986,7 @@ int X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
   }
 
   // Handle non-power-of-two vectors such as <3 x float>
-  if (VectorType *VTy = dyn_cast<VectorType>(Src)) {
+  if (auto *VTy = dyn_cast<FixedVectorType>(Src)) {
     unsigned NumElem = VTy->getNumElements();
 
     // Handle a few common cases:
@@ -3031,23 +3034,21 @@ int X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
 }
 
 int X86TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *SrcTy,
-                                      unsigned Alignment,
-                                      unsigned AddressSpace,
+                                      Align Alignment, unsigned AddressSpace,
                                       TTI::TargetCostKind CostKind) {
   bool IsLoad = (Instruction::Load == Opcode);
   bool IsStore = (Instruction::Store == Opcode);
 
-  VectorType *SrcVTy = dyn_cast<VectorType>(SrcTy);
+  auto *SrcVTy = dyn_cast<FixedVectorType>(SrcTy);
   if (!SrcVTy)
     // To calculate scalar take the regular cost, without mask
-    return getMemoryOpCost(Opcode, SrcTy, MaybeAlign(Alignment), AddressSpace,
-                           CostKind);
+    return getMemoryOpCost(Opcode, SrcTy, Alignment, AddressSpace, CostKind);
 
   unsigned NumElem = SrcVTy->getNumElements();
   auto *MaskTy =
       FixedVectorType::get(Type::getInt8Ty(SrcVTy->getContext()), NumElem);
-  if ((IsLoad && !isLegalMaskedLoad(SrcVTy, MaybeAlign(Alignment))) ||
-      (IsStore && !isLegalMaskedStore(SrcVTy, MaybeAlign(Alignment))) ||
+  if ((IsLoad && !isLegalMaskedLoad(SrcVTy, Alignment)) ||
+      (IsStore && !isLegalMaskedStore(SrcVTy, Alignment)) ||
       !isPowerOf2_32(NumElem)) {
     // Scalarization
     APInt DemandedElts = APInt::getAllOnesValue(NumElem);
@@ -3062,8 +3063,7 @@ int X86TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *SrcTy,
         getScalarizationOverhead(SrcVTy, DemandedElts, IsLoad, IsStore);
     int MemopCost =
         NumElem * BaseT::getMemoryOpCost(Opcode, SrcVTy->getScalarType(),
-                                         MaybeAlign(Alignment), AddressSpace,
-                                         CostKind);
+                                         Alignment, AddressSpace, CostKind);
     return MemopCost + ValueSplitCost + MaskSplitCost + MaskCmpCost;
   }
 
@@ -3184,7 +3184,7 @@ int X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
 
   MVT MTy = LT.second;
 
-  auto *ValVTy = cast<VectorType>(ValTy);
+  auto *ValVTy = cast<FixedVectorType>(ValTy);
 
   unsigned ArithmeticCost = 0;
   if (LT.first != 1 && MTy.isVector() &&
@@ -3565,7 +3565,7 @@ int X86TTIImpl::getMinMaxReductionCost(VectorType *ValTy, VectorType *CondTy,
         return Entry->Cost;
   }
 
-  auto *ValVTy = cast<VectorType>(ValTy);
+  auto *ValVTy = cast<FixedVectorType>(ValTy);
   unsigned NumVecElts = ValVTy->getNumElements();
 
   auto *Ty = ValVTy;
@@ -3849,25 +3849,25 @@ X86TTIImpl::getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind) {
 }
 
 // Return an average cost of Gather / Scatter instruction, maybe improved later
-int X86TTIImpl::getGSVectorCost(unsigned Opcode, Type *SrcVTy, Value *Ptr,
-                                unsigned Alignment, unsigned AddressSpace) {
+int X86TTIImpl::getGSVectorCost(unsigned Opcode, Type *SrcVTy, const Value *Ptr,
+                                Align Alignment, unsigned AddressSpace) {
 
   assert(isa<VectorType>(SrcVTy) && "Unexpected type in getGSVectorCost");
-  unsigned VF = cast<VectorType>(SrcVTy)->getNumElements();
+  unsigned VF = cast<FixedVectorType>(SrcVTy)->getNumElements();
 
   // Try to reduce index size from 64 bit (default for GEP)
   // to 32. It is essential for VF 16. If the index can't be reduced to 32, the
   // operation will use 16 x 64 indices which do not fit in a zmm and needs
   // to split. Also check that the base pointer is the same for all lanes,
   // and that there's at most one variable index.
-  auto getIndexSizeInBits = [](Value *Ptr, const DataLayout& DL) {
+  auto getIndexSizeInBits = [](const Value *Ptr, const DataLayout &DL) {
     unsigned IndexSize = DL.getPointerSizeInBits();
-    GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
+    const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
     if (IndexSize < 64 || !GEP)
       return IndexSize;
 
     unsigned NumOfVarIndices = 0;
-    Value *Ptrs = GEP->getPointerOperand();
+    const Value *Ptrs = GEP->getPointerOperand();
     if (Ptrs->getType()->isVectorTy() && !getSplatValue(Ptrs))
       return IndexSize;
     for (unsigned i = 1; i < GEP->getNumOperands(); ++i) {
@@ -3884,7 +3884,6 @@ int X86TTIImpl::getGSVectorCost(unsigned Opcode, Type *SrcVTy, Value *Ptr,
     return (unsigned)32;
   };
 
-
   // Trying to reduce IndexSize to 32 bits for vector 16.
   // By default the IndexSize is equal to pointer size.
   unsigned IndexSize = (ST->hasAVX512() && VF >= 16)
@@ -3923,9 +3922,9 @@ int X86TTIImpl::getGSVectorCost(unsigned Opcode, Type *SrcVTy, Value *Ptr,
 /// AddressSpace - pointer[s] address space.
 ///
 int X86TTIImpl::getGSScalarCost(unsigned Opcode, Type *SrcVTy,
-                                bool VariableMask, unsigned Alignment,
+                                bool VariableMask, Align Alignment,
                                 unsigned AddressSpace) {
-  unsigned VF = cast<VectorType>(SrcVTy)->getNumElements();
+  unsigned VF = cast<FixedVectorType>(SrcVTy)->getNumElements();
   APInt DemandedElts = APInt::getAllOnesValue(VF);
   TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
 
@@ -3963,16 +3962,17 @@ int X86TTIImpl::getGSScalarCost(unsigned Opcode, Type *SrcVTy,
 }
 
 /// Calculate the cost of Gather / Scatter operation
-int X86TTIImpl::getGatherScatterOpCost(
-    unsigned Opcode, Type *SrcVTy, Value *Ptr, bool VariableMask,
-    unsigned Alignment, TTI::TargetCostKind CostKind,
-    const Instruction *I = nullptr) {
+int X86TTIImpl::getGatherScatterOpCost(unsigned Opcode, Type *SrcVTy,
+                                       const Value *Ptr, bool VariableMask,
+                                       Align Alignment,
+                                       TTI::TargetCostKind CostKind,
+                                       const Instruction *I = nullptr) {
 
   if (CostKind != TTI::TCK_RecipThroughput)
     return 1;
 
   assert(SrcVTy->isVectorTy() && "Unexpected data type for Gather/Scatter");
-  unsigned VF = cast<VectorType>(SrcVTy)->getNumElements();
+  unsigned VF = cast<FixedVectorType>(SrcVTy)->getNumElements();
   PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
   if (!PtrTy && Ptr->getType()->isVectorTy())
     PtrTy = dyn_cast<PointerType>(
@@ -3982,9 +3982,9 @@ int X86TTIImpl::getGatherScatterOpCost(
 
   bool Scalarize = false;
   if ((Opcode == Instruction::Load &&
-       !isLegalMaskedGather(SrcVTy, MaybeAlign(Alignment))) ||
+       !isLegalMaskedGather(SrcVTy, Align(Alignment))) ||
       (Opcode == Instruction::Store &&
-       !isLegalMaskedScatter(SrcVTy, MaybeAlign(Alignment))))
+       !isLegalMaskedScatter(SrcVTy, Align(Alignment))))
     Scalarize = true;
   // Gather / Scatter for vector 2 is not profitable on KNL / SKX
   // Vector-4 of gather/scatter instruction does not exist on KNL.
@@ -4017,13 +4017,13 @@ bool X86TTIImpl::canMacroFuseCmp() {
   return ST->hasMacroFusion() || ST->hasBranchFusion();
 }
 
-bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy, MaybeAlign Alignment) {
+bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment) {
   if (!ST->hasAVX())
     return false;
 
   // The backend can't handle a single element vector.
   if (isa<VectorType>(DataTy) &&
-      cast<VectorType>(DataTy)->getNumElements() == 1)
+      cast<FixedVectorType>(DataTy)->getNumElements() == 1)
     return false;
   Type *ScalarTy = DataTy->getScalarType();
 
@@ -4041,7 +4041,7 @@ bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy, MaybeAlign Alignment) {
          ((IntWidth == 8 || IntWidth == 16) && ST->hasBWI());
 }
 
-bool X86TTIImpl::isLegalMaskedStore(Type *DataType, MaybeAlign Alignment) {
+bool X86TTIImpl::isLegalMaskedStore(Type *DataType, Align Alignment) {
   return isLegalMaskedLoad(DataType, Alignment);
 }
 
@@ -4088,7 +4088,7 @@ bool X86TTIImpl::isLegalMaskedExpandLoad(Type *DataTy) {
     return false;
 
   // The backend can't handle a single element vector.
-  if (cast<VectorType>(DataTy)->getNumElements() == 1)
+  if (cast<FixedVectorType>(DataTy)->getNumElements() == 1)
     return false;
 
   Type *ScalarTy = cast<VectorType>(DataTy)->getElementType();
@@ -4108,7 +4108,7 @@ bool X86TTIImpl::isLegalMaskedCompressStore(Type *DataTy) {
   return isLegalMaskedExpandLoad(DataTy);
 }
 
-bool X86TTIImpl::isLegalMaskedGather(Type *DataTy, MaybeAlign Alignment) {
+bool X86TTIImpl::isLegalMaskedGather(Type *DataTy, Align Alignment) {
   // Some CPUs have better gather performance than others.
   // TODO: Remove the explicit ST->hasAVX512()?, That would mean we would only
   // enable gather with a -march.
@@ -4127,7 +4127,7 @@ bool X86TTIImpl::isLegalMaskedGather(Type *DataTy, MaybeAlign Alignment) {
   // In this case we can reject non-power-of-2 vectors.
   // We also reject single element vectors as the type legalizer can't
   // scalarize it.
-  if (auto *DataVTy = dyn_cast<VectorType>(DataTy)) {
+  if (auto *DataVTy = dyn_cast<FixedVectorType>(DataTy)) {
     unsigned NumElts = DataVTy->getNumElements();
     if (NumElts == 1 || !isPowerOf2_32(NumElts))
       return false;
@@ -4146,7 +4146,7 @@ bool X86TTIImpl::isLegalMaskedGather(Type *DataTy, MaybeAlign Alignment) {
   return IntWidth == 32 || IntWidth == 64;
 }
 
-bool X86TTIImpl::isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) {
+bool X86TTIImpl::isLegalMaskedScatter(Type *DataType, Align Alignment) {
   // AVX2 doesn't support scatter
   if (!ST->hasAVX512())
     return false;
@@ -4247,14 +4247,10 @@ bool X86TTIImpl::enableInterleavedAccessVectorization() {
 // computing the cost using a generic formula as a function of generic
 // shuffles. We therefore use a lookup table instead, filled according to
 // the instruction sequences that codegen currently generates.
-int X86TTIImpl::getInterleavedMemoryOpCostAVX2(unsigned Opcode, Type *VecTy,
-                                               unsigned Factor,
-                                               ArrayRef<unsigned> Indices,
-                                               unsigned Alignment,
-                                               unsigned AddressSpace,
-                                               TTI::TargetCostKind CostKind,
-                                               bool UseMaskForCond,
-                                               bool UseMaskForGaps) {
+int X86TTIImpl::getInterleavedMemoryOpCostAVX2(
+    unsigned Opcode, FixedVectorType *VecTy, unsigned Factor,
+    ArrayRef<unsigned> Indices, Align Alignment, unsigned AddressSpace,
+    TTI::TargetCostKind CostKind, bool UseMaskForCond, bool UseMaskForGaps) {
 
   if (UseMaskForCond || UseMaskForGaps)
     return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
@@ -4281,8 +4277,8 @@ int X86TTIImpl::getInterleavedMemoryOpCostAVX2(unsigned Opcode, Type *VecTy,
                                              Alignment, AddressSpace,
                                              CostKind);
 
-  unsigned VF = cast<VectorType>(VecTy)->getNumElements() / Factor;
-  Type *ScalarTy = cast<VectorType>(VecTy)->getElementType();
+  unsigned VF = VecTy->getNumElements() / Factor;
+  Type *ScalarTy = VecTy->getElementType();
 
   // Calculate the number of memory operations (NumOfMemOps), required
   // for load/store the VecTy.
@@ -4291,9 +4287,8 @@ int X86TTIImpl::getInterleavedMemoryOpCostAVX2(unsigned Opcode, Type *VecTy,
   unsigned NumOfMemOps = (VecTySize + LegalVTSize - 1) / LegalVTSize;
 
   // Get the cost of one memory operation.
-  auto *SingleMemOpTy =
-      FixedVectorType::get(cast<VectorType>(VecTy)->getElementType(),
-                           LegalVT.getVectorNumElements());
+  auto *SingleMemOpTy = FixedVectorType::get(VecTy->getElementType(),
+                                             LegalVT.getVectorNumElements());
   unsigned MemOpCost = getMemoryOpCost(Opcode, SingleMemOpTy,
                                        MaybeAlign(Alignment), AddressSpace,
                                        CostKind);
@@ -4369,14 +4364,10 @@ int X86TTIImpl::getInterleavedMemoryOpCostAVX2(unsigned Opcode, Type *VecTy,
 // \p Indices contains indices for strided load.
 // \p Factor - the factor of interleaving.
 // AVX-512 provides 3-src shuffles that significantly reduces the cost.
-int X86TTIImpl::getInterleavedMemoryOpCostAVX512(unsigned Opcode, Type *VecTy,
-                                                 unsigned Factor,
-                                                 ArrayRef<unsigned> Indices,
-                                                 unsigned Alignment,
-                                                 unsigned AddressSpace,
-                                                 TTI::TargetCostKind CostKind,
-                                                 bool UseMaskForCond,
-                                                 bool UseMaskForGaps) {
+int X86TTIImpl::getInterleavedMemoryOpCostAVX512(
+    unsigned Opcode, FixedVectorType *VecTy, unsigned Factor,
+    ArrayRef<unsigned> Indices, Align Alignment, unsigned AddressSpace,
+    TTI::TargetCostKind CostKind, bool UseMaskForCond, bool UseMaskForGaps) {
 
   if (UseMaskForCond || UseMaskForGaps)
     return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
@@ -4395,14 +4386,13 @@ int X86TTIImpl::getInterleavedMemoryOpCostAVX512(unsigned Opcode, Type *VecTy,
   unsigned NumOfMemOps = (VecTySize + LegalVTSize - 1) / LegalVTSize;
 
   // Get the cost of one memory operation.
-  auto *SingleMemOpTy =
-      FixedVectorType::get(cast<VectorType>(VecTy)->getElementType(),
-                           LegalVT.getVectorNumElements());
+  auto *SingleMemOpTy = FixedVectorType::get(VecTy->getElementType(),
+                                             LegalVT.getVectorNumElements());
   unsigned MemOpCost = getMemoryOpCost(Opcode, SingleMemOpTy,
                                        MaybeAlign(Alignment), AddressSpace,
                                        CostKind);
 
-  unsigned VF = cast<VectorType>(VecTy)->getNumElements() / Factor;
+  unsigned VF = VecTy->getNumElements() / Factor;
   MVT VT = MVT::getVectorVT(MVT::getVT(VecTy->getScalarType()), VF);
 
   if (Opcode == Instruction::Load) {
@@ -4434,9 +4424,8 @@ int X86TTIImpl::getInterleavedMemoryOpCostAVX512(unsigned Opcode, Type *VecTy,
 
     unsigned NumOfLoadsInInterleaveGrp =
         Indices.size() ? Indices.size() : Factor;
-    auto *ResultTy = FixedVectorType::get(
-        cast<VectorType>(VecTy)->getElementType(),
-        cast<VectorType>(VecTy)->getNumElements() / Factor);
+    auto *ResultTy = FixedVectorType::get(VecTy->getElementType(),
+                                          VecTy->getNumElements() / Factor);
     unsigned NumOfResults =
         getTLI()->getTypeLegalizationCost(DL, ResultTy).first *
         NumOfLoadsInInterleaveGrp;
@@ -4498,14 +4487,10 @@ int X86TTIImpl::getInterleavedMemoryOpCostAVX512(unsigned Opcode, Type *VecTy,
   return Cost;
 }
 
-int X86TTIImpl::getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
-                                           unsigned Factor,
-                                           ArrayRef<unsigned> Indices,
-                                           unsigned Alignment,
-                                           unsigned AddressSpace,
-                                           TTI::TargetCostKind CostKind,
-                                           bool UseMaskForCond,
-                                           bool UseMaskForGaps) {
+int X86TTIImpl::getInterleavedMemoryOpCost(
+    unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+    Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
+    bool UseMaskForCond, bool UseMaskForGaps) {
   auto isSupportedOnAVX512 = [](Type *VecTy, bool HasBW) {
     Type *EltTy = cast<VectorType>(VecTy)->getElementType();
     if (EltTy->isFloatTy() || EltTy->isDoubleTy() || EltTy->isIntegerTy(64) ||
@@ -4516,13 +4501,13 @@ int X86TTIImpl::getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
     return false;
   };
   if (ST->hasAVX512() && isSupportedOnAVX512(VecTy, ST->hasBWI()))
-    return getInterleavedMemoryOpCostAVX512(Opcode, VecTy, Factor, Indices,
-                                            Alignment, AddressSpace, CostKind,
-                                            UseMaskForCond, UseMaskForGaps);
+    return getInterleavedMemoryOpCostAVX512(
+        Opcode, cast<FixedVectorType>(VecTy), Factor, Indices, Alignment,
+        AddressSpace, CostKind, UseMaskForCond, UseMaskForGaps);
   if (ST->hasAVX2())
-    return getInterleavedMemoryOpCostAVX2(Opcode, VecTy, Factor, Indices,
-                                          Alignment, AddressSpace, CostKind,
-                                          UseMaskForCond, UseMaskForGaps);
+    return getInterleavedMemoryOpCostAVX2(
+        Opcode, cast<FixedVectorType>(VecTy), Factor, Indices, Alignment,
+        AddressSpace, CostKind, UseMaskForCond, UseMaskForGaps);
 
   return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
                                            Alignment, AddressSpace, CostKind,
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 345975b0b0eb4..b97072893cc2e 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -141,11 +141,11 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
                       unsigned AddressSpace,
                       TTI::TargetCostKind CostKind,
                       const Instruction *I = nullptr);
-  int getMaskedMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment,
-                            unsigned AddressSpace,
-                            TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency);
-  int getGatherScatterOpCost(unsigned Opcode, Type *DataTy, Value *Ptr,
-                             bool VariableMask, unsigned Alignment,
+  int getMaskedMemoryOpCost(
+      unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency);
+  int getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
+                             bool VariableMask, Align Alignment,
                              TTI::TargetCostKind CostKind,
                              const Instruction *I);
   int getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
@@ -168,24 +168,21 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
                              bool IsPairwiseForm, bool IsUnsigned,
                              TTI::TargetCostKind CostKind);
 
-  int getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
-                                 unsigned Factor, ArrayRef<unsigned> Indices,
-                                 unsigned Alignment, unsigned AddressSpace,
-                                 TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
-                                 bool UseMaskForCond = false,
-                                 bool UseMaskForGaps = false);
-  int getInterleavedMemoryOpCostAVX512(unsigned Opcode, Type *VecTy,
-                                 unsigned Factor, ArrayRef<unsigned> Indices,
-                                 unsigned Alignment, unsigned AddressSpace,
-                                 TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
-                                 bool UseMaskForCond = false,
-                                 bool UseMaskForGaps = false);
-  int getInterleavedMemoryOpCostAVX2(unsigned Opcode, Type *VecTy,
-                                 unsigned Factor, ArrayRef<unsigned> Indices,
-                                 unsigned Alignment, unsigned AddressSpace,
-                                 TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
-                                 bool UseMaskForCond = false,
-                                 bool UseMaskForGaps = false);
+  int getInterleavedMemoryOpCost(
+      unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+      Align Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
+      bool UseMaskForCond = false, bool UseMaskForGaps = false);
+  int getInterleavedMemoryOpCostAVX512(
+      unsigned Opcode, FixedVectorType *VecTy, unsigned Factor,
+      ArrayRef<unsigned> Indices, Align Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
+      bool UseMaskForCond = false, bool UseMaskForGaps = false);
+  int getInterleavedMemoryOpCostAVX2(
+      unsigned Opcode, FixedVectorType *VecTy, unsigned Factor,
+      ArrayRef<unsigned> Indices, Align Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
+      bool UseMaskForCond = false, bool UseMaskForGaps = false);
 
   int getIntImmCost(int64_t);
 
@@ -200,12 +197,12 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
   bool isLSRCostLess(TargetTransformInfo::LSRCost &C1,
                      TargetTransformInfo::LSRCost &C2);
   bool canMacroFuseCmp();
-  bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment);
-  bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment);
+  bool isLegalMaskedLoad(Type *DataType, Align Alignment);
+  bool isLegalMaskedStore(Type *DataType, Align Alignment);
   bool isLegalNTLoad(Type *DataType, Align Alignment);
   bool isLegalNTStore(Type *DataType, Align Alignment);
-  bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment);
-  bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment);
+  bool isLegalMaskedGather(Type *DataType, Align Alignment);
+  bool isLegalMaskedScatter(Type *DataType, Align Alignment);
   bool isLegalMaskedExpandLoad(Type *DataType);
   bool isLegalMaskedCompressStore(Type *DataType);
   bool hasDivRemOp(Type *DataType, bool IsSigned);
@@ -229,9 +226,9 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
 
 private:
   int getGSScalarCost(unsigned Opcode, Type *DataTy, bool VariableMask,
-                      unsigned Alignment, unsigned AddressSpace);
-  int getGSVectorCost(unsigned Opcode, Type *DataTy, Value *Ptr,
-                      unsigned Alignment, unsigned AddressSpace);
+                      Align Alignment, unsigned AddressSpace);
+  int getGSVectorCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
+                      Align Alignment, unsigned AddressSpace);
 
   /// @}
 };
diff --git a/llvm/lib/Target/XCore/XCoreFrameLowering.cpp b/llvm/lib/Target/XCore/XCoreFrameLowering.cpp
index 831f7c951b769..27ac6a4d1439b 100644
--- a/llvm/lib/Target/XCore/XCoreFrameLowering.cpp
+++ b/llvm/lib/Target/XCore/XCoreFrameLowering.cpp
@@ -575,9 +575,9 @@ processFunctionBeforeFrameFinalized(MachineFunction &MF,
   // When using SP for large frames, we may need 2 scratch registers.
   // When using FP, for large or small frames, we may need 1 scratch register.
   unsigned Size = TRI.getSpillSize(RC);
-  unsigned Align = TRI.getSpillAlignment(RC);
+  Align Alignment = TRI.getSpillAlign(RC);
   if (XFI->isLargeFrame(MF) || hasFP(MF))
-    RS->addScavengingFrameIndex(MFI.CreateStackObject(Size, Align, false));
+    RS->addScavengingFrameIndex(MFI.CreateStackObject(Size, Alignment, false));
   if (XFI->isLargeFrame(MF) && !hasFP(MF))
-    RS->addScavengingFrameIndex(MFI.CreateStackObject(Size, Align, false));
+    RS->addScavengingFrameIndex(MFI.CreateStackObject(Size, Alignment, false));
 }
diff --git a/llvm/lib/Target/XCore/XCoreISelLowering.cpp b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
index d8087282ff5d3..c32653137a10b 100644
--- a/llvm/lib/Target/XCore/XCoreISelLowering.cpp
+++ b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
@@ -435,7 +435,7 @@ SDValue XCoreTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
                                                     Offset, DAG);
     }
     if (TLI.isGAPlusOffset(BasePtr.getNode(), GV, Offset) &&
-        MinAlign(GV->getAlignment(), 4) == 4) {
+        GV->getPointerAlignment(DAG.getDataLayout()) >= 4) {
       SDValue NewBasePtr = DAG.getGlobalAddress(GV, DL,
                                                 BasePtr->getValueType(0));
       return lowerLoadWordFromAlignedBasePlusOffset(DL, Chain, NewBasePtr,
diff --git a/llvm/lib/Target/XCore/XCoreMachineFunctionInfo.cpp b/llvm/lib/Target/XCore/XCoreMachineFunctionInfo.cpp
index 0b4fcffbc6559..ec44d2899dd5c 100644
--- a/llvm/lib/Target/XCore/XCoreMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/XCore/XCoreMachineFunctionInfo.cpp
@@ -43,7 +43,7 @@ int XCoreFunctionInfo::createLRSpillSlot(MachineFunction &MF) {
     LRSpillSlot = MFI.CreateFixedObject(TRI.getSpillSize(RC), 0, true);
   } else {
     LRSpillSlot = MFI.CreateStackObject(TRI.getSpillSize(RC),
-                                        TRI.getSpillAlignment(RC), true);
+                                        TRI.getSpillAlign(RC), true);
   }
   LRSpillSlotSet = true;
   return LRSpillSlot;
@@ -56,8 +56,8 @@ int XCoreFunctionInfo::createFPSpillSlot(MachineFunction &MF) {
   const TargetRegisterClass &RC = XCore::GRRegsRegClass;
   const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  FPSpillSlot = MFI.CreateStackObject(TRI.getSpillSize(RC),
-                                      TRI.getSpillAlignment(RC), true);
+  FPSpillSlot =
+      MFI.CreateStackObject(TRI.getSpillSize(RC), TRI.getSpillAlign(RC), true);
   FPSpillSlotSet = true;
   return FPSpillSlot;
 }
@@ -70,9 +70,9 @@ const int* XCoreFunctionInfo::createEHSpillSlot(MachineFunction &MF) {
   const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   unsigned Size = TRI.getSpillSize(RC);
-  unsigned Align = TRI.getSpillAlignment(RC);
-  EHSpillSlot[0] = MFI.CreateStackObject(Size, Align, true);
-  EHSpillSlot[1] = MFI.CreateStackObject(Size, Align, true);
+  Align Alignment = TRI.getSpillAlign(RC);
+  EHSpillSlot[0] = MFI.CreateStackObject(Size, Alignment, true);
+  EHSpillSlot[1] = MFI.CreateStackObject(Size, Alignment, true);
   EHSpillSlotSet = true;
   return EHSpillSlot;
 }
diff --git a/llvm/lib/Target/XCore/XCoreSelectionDAGInfo.cpp b/llvm/lib/Target/XCore/XCoreSelectionDAGInfo.cpp
index c86756e345a94..0d097076348ca 100644
--- a/llvm/lib/Target/XCore/XCoreSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/XCore/XCoreSelectionDAGInfo.cpp
@@ -17,11 +17,11 @@ using namespace llvm;
 
 SDValue XCoreSelectionDAGInfo::EmitTargetCodeForMemcpy(
     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
-    SDValue Size, unsigned Align, bool isVolatile, bool AlwaysInline,
+    SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
     MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
   unsigned SizeBitWidth = Size.getValueSizeInBits();
   // Call __memcpy_4 if the src, dst and size are all 4 byte aligned.
-  if (!AlwaysInline && (Align & 3) == 0 &&
+  if (!AlwaysInline && Alignment >= Align(4) &&
       DAG.MaskedValueIsZero(Size, APInt(SizeBitWidth, 3))) {
     const TargetLowering &TLI = *DAG.getSubtarget().getTargetLowering();
     TargetLowering::ArgListTy Args;
diff --git a/llvm/lib/Target/XCore/XCoreSelectionDAGInfo.h b/llvm/lib/Target/XCore/XCoreSelectionDAGInfo.h
index 637e8547e6420..2abf526779785 100644
--- a/llvm/lib/Target/XCore/XCoreSelectionDAGInfo.h
+++ b/llvm/lib/Target/XCore/XCoreSelectionDAGInfo.h
@@ -21,7 +21,7 @@ class XCoreSelectionDAGInfo : public SelectionDAGTargetInfo {
 public:
   SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
                                   SDValue Chain, SDValue Op1, SDValue Op2,
-                                  SDValue Op3, unsigned Align, bool isVolatile,
+                                  SDValue Op3, Align Alignment, bool isVolatile,
                                   bool AlwaysInline,
                                   MachinePointerInfo DstPtrInfo,
                                   MachinePointerInfo SrcPtrInfo) const override;
diff --git a/llvm/lib/TextAPI/MachO/InterfaceFile.cpp b/llvm/lib/TextAPI/MachO/InterfaceFile.cpp
index 0a392a2424417..64d2c3e865ab8 100644
--- a/llvm/lib/TextAPI/MachO/InterfaceFile.cpp
+++ b/llvm/lib/TextAPI/MachO/InterfaceFile.cpp
@@ -14,9 +14,10 @@
 #include <iomanip>
 #include <sstream>
 
-namespace llvm {
-namespace MachO {
-namespace detail {
+using namespace llvm;
+using namespace llvm::MachO;
+
+namespace {
 template <typename C>
 typename C::iterator addEntry(C &Container, StringRef InstallName) {
   auto I = partition_point(Container, [=](const InterfaceFileRef &O) {
@@ -39,21 +40,21 @@ typename C::iterator addEntry(C &Container, const Target &Target_) {
 
   return Container.insert(Iter, Target_);
 }
-} // end namespace detail.
+} // end namespace
 
 void InterfaceFileRef::addTarget(const Target &Target) {
-  detail::addEntry(Targets, Target);
+  addEntry(Targets, Target);
 }
 
 void InterfaceFile::addAllowableClient(StringRef InstallName,
                                        const Target &Target) {
-  auto Client = detail::addEntry(AllowableClients, InstallName);
+  auto Client = addEntry(AllowableClients, InstallName);
   Client->addTarget(Target);
 }
 
 void InterfaceFile::addReexportedLibrary(StringRef InstallName,
                                          const Target &Target) {
-  auto Lib = detail::addEntry(ReexportedLibraries, InstallName);
+  auto Lib = addEntry(ReexportedLibraries, InstallName);
   Lib->addTarget(Target);
 }
 
@@ -97,7 +98,7 @@ void InterfaceFile::addUUID(const Target &Target, uint8_t UUID[16]) {
 }
 
 void InterfaceFile::addTarget(const Target &Target) {
-  detail::addEntry(Targets, Target);
+  addEntry(Targets, Target);
 }
 
 InterfaceFile::const_filtered_target_range
@@ -118,6 +119,3 @@ void InterfaceFile::addSymbol(SymbolKind Kind, StringRef Name,
     for (const auto &Target : Targets)
       result.first->second->addTarget(Target);
 }
-
-} // end namespace MachO.
-} // end namespace llvm.
diff --git a/llvm/lib/TextAPI/MachO/TextStub.cpp b/llvm/lib/TextAPI/MachO/TextStub.cpp
index db6762f9554da..141f897fb5647 100644
--- a/llvm/lib/TextAPI/MachO/TextStub.cpp
+++ b/llvm/lib/TextAPI/MachO/TextStub.cpp
@@ -1089,8 +1089,8 @@ struct DocumentListTraits<std::vector<const MachO::InterfaceFile *>> {
 };
 
 } // end namespace yaml.
+} // namespace llvm
 
-namespace MachO {
 static void DiagHandler(const SMDiagnostic &Diag, void *Context) {
   auto *File = static_cast<TextAPIContext *>(Context);
   SmallString<1024> Message;
@@ -1147,6 +1147,3 @@ Error TextAPIWriter::writeToStream(raw_ostream &OS, const InterfaceFile &File) {
 
   return Error::success();
 }
-
-} // end namespace MachO.
-} // end namespace llvm.
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombineInternal.h b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombineInternal.h
index 44e1c45664e7e..42bcadfc7dcdb 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombineInternal.h
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombineInternal.h
@@ -16,14 +16,8 @@
 #define LLVM_LIB_TRANSFORMS_AGGRESSIVEINSTCOMBINE_COMBINEINTERNAL_H
 
 #include "llvm/ADT/MapVector.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Analysis/BasicAliasAnalysis.h"
-#include "llvm/Analysis/ConstantFolding.h"
-#include "llvm/Analysis/GlobalsModRef.h"
-#include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/Pass.h"
+#include "llvm/ADT/SmallVector.h"
+
 using namespace llvm;
 
 //===----------------------------------------------------------------------===//
@@ -47,7 +41,12 @@ using namespace llvm;
 namespace llvm {
   class DataLayout;
   class DominatorTree;
+  class Function;
+  class Instruction;
   class TargetLibraryInfo;
+  class TruncInst;
+  class Type;
+  class Value;
 
 class TruncInstCombine {
   TargetLibraryInfo &TLI;
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/TruncInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/TruncInstCombine.cpp
index ece4bdf470d02..5cd40c66227f3 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/TruncInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/TruncInstCombine.cpp
@@ -25,13 +25,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "AggressiveInstCombineInternal.h"
-#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Dominators.h"
+#include "llvm/IR/Instruction.h"
 #include "llvm/IR/IRBuilder.h"
 
 using namespace llvm;
diff --git a/llvm/lib/Transforms/CFGuard/CFGuard.cpp b/llvm/lib/Transforms/CFGuard/CFGuard.cpp
index e4f338b2d9e92..96c083a144b23 100644
--- a/llvm/lib/Transforms/CFGuard/CFGuard.cpp
+++ b/llvm/lib/Transforms/CFGuard/CFGuard.cpp
@@ -204,14 +204,9 @@ void CFGuard::insertCFGuardDispatch(CallBase *CB) {
   Bundles.emplace_back("cfguardtarget", CalledOperand);
 
   // Create a copy of the call/invoke instruction and add the new bundle.
-  CallBase *NewCB;
-  if (CallInst *CI = dyn_cast<CallInst>(CB)) {
-    NewCB = CallInst::Create(CI, Bundles, CB);
-  } else {
-    assert(isa<InvokeInst>(CB) && "Unknown indirect call type");
-    InvokeInst *II = cast<InvokeInst>(CB);
-    NewCB = llvm::InvokeInst::Create(II, Bundles, CB);
-  }
+  assert((isa<CallInst>(CB) || isa<InvokeInst>(CB)) &&
+         "Unknown indirect call type");
+  CallBase *NewCB = CallBase::Create(CB, Bundles, CB);
 
   // Change the target of the call to be the guard dispatch function.
   NewCB->setCalledOperand(GuardDispatchLoad);
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 2d398643a9c38..f55501a05d858 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -1548,6 +1548,75 @@ static void sinkSpillUsesAfterCoroBegin(Function &F, const SpillInfo &Spills,
   return;
 }
 
+/// For each local variable that all of its user are only used inside one of
+/// suspended region, we sink their lifetime.start markers to the place where
+/// after the suspend block. Doing so minimizes the lifetime of each variable,
+/// hence minimizing the amount of data we end up putting on the frame.
+static void sinkLifetimeStartMarkers(Function &F, coro::Shape &Shape,
+                                     SuspendCrossingInfo &Checker) {
+  DominatorTree DT(F);
+
+  // Collect all possible basic blocks which may dominate all uses of allocas.
+  SmallPtrSet<BasicBlock *, 4> DomSet;
+  DomSet.insert(&F.getEntryBlock());
+  for (auto *CSI : Shape.CoroSuspends) {
+    BasicBlock *SuspendBlock = CSI->getParent();
+    assert(isSuspendBlock(SuspendBlock) && SuspendBlock->getSingleSuccessor() &&
+           "should have split coro.suspend into its own block");
+    DomSet.insert(SuspendBlock->getSingleSuccessor());
+  }
+
+  for (Instruction &I : instructions(F)) {
+    if (!isa<AllocaInst>(&I))
+      continue;
+
+    for (BasicBlock *DomBB : DomSet) {
+      bool Valid = true;
+      SmallVector<Instruction *, 1> BCInsts;
+
+      auto isUsedByLifetimeStart = [&](Instruction *I) {
+        if (isa<BitCastInst>(I) && I->hasOneUse())
+          if (auto *IT = dyn_cast<IntrinsicInst>(I->user_back()))
+            return IT->getIntrinsicID() == Intrinsic::lifetime_start;
+        return false;
+      };
+
+      for (User *U : I.users()) {
+        Instruction *UI = cast<Instruction>(U);
+        // For all users except lifetime.start markers, if they are all
+        // dominated by one of the basic blocks and do not cross
+        // suspend points as well, then there is no need to spill the
+        // instruction.
+        if (!DT.dominates(DomBB, UI->getParent()) ||
+            Checker.isDefinitionAcrossSuspend(DomBB, U)) {
+          // Skip bitcast used by lifetime.start markers.
+          if (isUsedByLifetimeStart(UI)) {
+            BCInsts.push_back(UI);
+            continue;
+          }
+          Valid = false;
+          break;
+        }
+      }
+      // Sink lifetime.start markers to dominate block when they are
+      // only used outside the region.
+      if (Valid && BCInsts.size() != 0) {
+        auto *NewBitcast = BCInsts[0]->clone();
+        auto *NewLifetime = cast<Instruction>(BCInsts[0]->user_back())->clone();
+        NewLifetime->replaceUsesOfWith(BCInsts[0], NewBitcast);
+        NewBitcast->insertBefore(DomBB->getTerminator());
+        NewLifetime->insertBefore(DomBB->getTerminator());
+
+        // All the outsided lifetime.start markers are no longer necessary.
+        for (Instruction *S : BCInsts) {
+          S->user_back()->eraseFromParent();
+        }
+        break;
+      }
+    }
+  }
+}
+
 void coro::buildCoroutineFrame(Function &F, Shape &Shape) {
   eliminateSwiftError(F, Shape);
 
@@ -1598,6 +1667,7 @@ void coro::buildCoroutineFrame(Function &F, Shape &Shape) {
     Spills.clear();
   }
 
+  sinkLifetimeStartMarkers(F, Shape, Checker);
   // Collect lifetime.start info for each alloca.
   using LifetimeStart = SmallPtrSet<Instruction *, 2>;
   llvm::DenseMap<Instruction *, std::unique_ptr<LifetimeStart>> LifetimeMap;
diff --git a/llvm/lib/Transforms/Coroutines/CoroInstr.h b/llvm/lib/Transforms/Coroutines/CoroInstr.h
index 384e20cd0a9b2..320137526db8d 100644
--- a/llvm/lib/Transforms/Coroutines/CoroInstr.h
+++ b/llvm/lib/Transforms/Coroutines/CoroInstr.h
@@ -212,7 +212,7 @@ class LLVM_LIBRARY_VISIBILITY AnyCoroIdRetconInst : public AnyCoroIdInst {
   }
 
   Align getStorageAlignment() const {
-    return Align(cast<ConstantInt>(getArgOperand(AlignArg))->getZExtValue());
+    return cast<ConstantInt>(getArgOperand(AlignArg))->getAlignValue();
   }
 
   Value *getStorage() const {
@@ -347,7 +347,7 @@ class LLVM_LIBRARY_VISIBILITY CoroPromiseInst : public IntrinsicInst {
   /// The required alignment of the promise.  This must match the
   /// alignment of the promise alloca in the coroutine.
   Align getAlignment() const {
-    return Align(cast<ConstantInt>(getArgOperand(AlignArg))->getZExtValue());
+    return cast<ConstantInt>(getArgOperand(AlignArg))->getAlignValue();
   }
 
   // Methods to support type inquiry through isa, cast, and dyn_cast:
@@ -468,8 +468,8 @@ class LLVM_LIBRARY_VISIBILITY CoroAllocaAllocInst : public IntrinsicInst {
   Value *getSize() const {
     return getArgOperand(SizeArg);
   }
-  unsigned getAlignment() const {
-    return cast<ConstantInt>(getArgOperand(AlignArg))->getZExtValue();
+  Align getAlignment() const {
+    return cast<ConstantInt>(getArgOperand(AlignArg))->getAlignValue();
   }
 
   // Methods to support type inquiry through isa, cast, and dyn_cast:
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 4fd127656f215..9c4392e7999b6 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -75,7 +75,7 @@ using namespace llvm;
 
 namespace {
 
-/// A little helper class for building 
+/// A little helper class for building
 class CoroCloner {
 public:
   enum class Kind {
@@ -563,7 +563,7 @@ void CoroCloner::replaceEntryBlock() {
   // In the original function, the AllocaSpillBlock is a block immediately
   // following the allocation of the frame object which defines GEPs for
   // all the allocas that have been moved into the frame, and it ends by
-  // branching to the original beginning of the coroutine.  Make this 
+  // branching to the original beginning of the coroutine.  Make this
   // the entry block of the cloned function.
   auto *Entry = cast<BasicBlock>(VMap[Shape.AllocaSpillBlock]);
   auto *OldEntry = &NewF->getEntryBlock();
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index 98b7b4efd46e0..6e5625d26c38b 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -23,6 +23,7 @@
 #include "llvm/IR/NoFolder.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/InitializePasses.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Local.h"
 
@@ -77,6 +78,12 @@ static cl::opt<bool>
                                   "wrappers for non-exact definitions."),
                          cl::init(false));
 
+static cl::list<std::string>
+    SeedAllowList("attributor-seed-allow-list", cl::Hidden,
+                  cl::desc("Comma seperated list of attrbute names that are "
+                           "allowed to be seeded."),
+                  cl::ZeroOrMore, cl::CommaSeparated);
+
 /// Logic operators for the change status enum class.
 ///
 ///{
@@ -1179,6 +1186,9 @@ ChangeStatus Attributor::cleanupIR() {
     }
   }
 
+  LLVM_DEBUG(dbgs() << "[Attributor] DeadInsts size: " << DeadInsts.size()
+                    << "\n");
+
   RecursivelyDeleteTriviallyDeadInstructions(DeadInsts);
 
   if (unsigned NumDeadBlocks = ToBeDeletedBlocks.size()) {
@@ -1237,6 +1247,9 @@ ChangeStatus Attributor::cleanupIR() {
 
   NumFnDeleted += ToBeDeletedFunctions.size();
 
+  LLVM_DEBUG(dbgs() << "[Attributor] Deleted " << NumFnDeleted
+                    << " functions after manifest.\n");
+
 #ifdef EXPENSIVE_CHECKS
   for (Function *F : Functions) {
     if (ToBeDeletedFunctions.count(F))
@@ -1249,6 +1262,7 @@ ChangeStatus Attributor::cleanupIR() {
 }
 
 ChangeStatus Attributor::run() {
+  SeedingPeriod = false;
   runTillFixpoint();
   ChangeStatus ManifestChange = manifestAttributes();
   ChangeStatus CleanupChange = cleanupIR();
@@ -1445,6 +1459,12 @@ bool Attributor::registerFunctionSignatureRewrite(
   return true;
 }
 
+bool Attributor::shouldSeedAttribute(AbstractAttribute &AA) {
+  if (SeedAllowList.size() == 0)
+    return true;
+  return std::count(SeedAllowList.begin(), SeedAllowList.end(), AA.getName());
+}
+
 ChangeStatus Attributor::rewriteFunctionSignatures(
     SmallPtrSetImpl<Function *> &ModifiedFns) {
   ChangeStatus Changed = ChangeStatus::UNCHANGED;
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index f641ad4498cd1..7e9fd61eeb41e 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -24,6 +24,7 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/NoFolder.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Transforms/IPO/ArgumentPromotion.h"
 #include "llvm/Transforms/Utils/Local.h"
 
@@ -4669,6 +4670,30 @@ struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating {
   AAValueSimplifyCallSiteArgument(const IRPosition &IRP, Attributor &A)
       : AAValueSimplifyFloating(IRP, A) {}
 
+  /// See AbstractAttribute::manifest(...).
+  ChangeStatus manifest(Attributor &A) override {
+    ChangeStatus Changed = ChangeStatus::UNCHANGED;
+
+    if (SimplifiedAssociatedValue.hasValue() &&
+        !SimplifiedAssociatedValue.getValue())
+      return Changed;
+
+    Value &V = getAssociatedValue();
+    auto *C = SimplifiedAssociatedValue.hasValue()
+                  ? dyn_cast<Constant>(SimplifiedAssociatedValue.getValue())
+                  : UndefValue::get(V.getType());
+    if (C) {
+      Use &U = cast<CallBase>(&getAnchorValue())->getArgOperandUse(getArgNo());
+      // We can replace the AssociatedValue with the constant.
+      if (&V != C && V.getType() == C->getType()) {
+        if (A.changeUseAfterManifest(U, *C))
+          Changed = ChangeStatus::CHANGED;
+      }
+    }
+
+    return Changed | AAValueSimplify::manifest(A);
+  }
+
   void trackStatistics() const override {
     STATS_DECLTRACK_CSARG_ATTR(value_simplify)
   }
diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt
index bec42276f27c1..3f30c0289e8c3 100644
--- a/llvm/lib/Transforms/IPO/CMakeLists.txt
+++ b/llvm/lib/Transforms/IPO/CMakeLists.txt
@@ -45,4 +45,5 @@ add_llvm_component_library(LLVMipo
 
   DEPENDS
   intrinsics_gen
+  omp_gen
   )
diff --git a/llvm/lib/Transforms/IPO/ConstantMerge.cpp b/llvm/lib/Transforms/IPO/ConstantMerge.cpp
index ea1278aa108fb..67f1438b9b6ac 100644
--- a/llvm/lib/Transforms/IPO/ConstantMerge.cpp
+++ b/llvm/lib/Transforms/IPO/ConstantMerge.cpp
@@ -84,11 +84,9 @@ static void copyDebugLocMetadata(const GlobalVariable *From,
     To->addDebugInfo(MD);
 }
 
-static unsigned getAlignment(GlobalVariable *GV) {
-  unsigned Align = GV->getAlignment();
-  if (Align)
-    return Align;
-  return GV->getParent()->getDataLayout().getPreferredAlignment(GV);
+static Align getAlign(GlobalVariable *GV) {
+  return GV->getAlign().getValueOr(
+      GV->getParent()->getDataLayout().getPreferredAlign(GV));
 }
 
 static bool
@@ -120,8 +118,8 @@ static void replace(Module &M, GlobalVariable *Old, GlobalVariable *New) {
                     << New->getName() << "\n");
 
   // Bump the alignment if necessary.
-  if (Old->getAlignment() || New->getAlignment())
-    New->setAlignment(Align(std::max(getAlignment(Old), getAlignment(New))));
+  if (Old->getAlign() || New->getAlign())
+    New->setAlignment(std::max(getAlign(Old), getAlign(New)));
 
   copyDebugLocMetadata(Old, New);
   Old->replaceAllUsesWith(NewConstant);
diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index 47ac7eb613a72..d9fb820f7cb53 100644
--- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -443,7 +443,7 @@ static bool IsSRASequential(Type *T) {
 static uint64_t GetSRASequentialNumElements(Type *T) {
   if (ArrayType *AT = dyn_cast<ArrayType>(T))
     return AT->getNumElements();
-  return cast<VectorType>(T)->getNumElements();
+  return cast<FixedVectorType>(T)->getNumElements();
 }
 static Type *GetSRASequentialElementType(Type *T) {
   if (ArrayType *AT = dyn_cast<ArrayType>(T))
@@ -509,9 +509,8 @@ static GlobalVariable *SRAGlobal(GlobalVariable *GV, const DataLayout &DL) {
   std::map<unsigned, GlobalVariable *> NewGlobals;
 
   // Get the alignment of the global, either explicit or target-specific.
-  unsigned StartAlignment = GV->getAlignment();
-  if (StartAlignment == 0)
-    StartAlignment = DL.getABITypeAlignment(GV->getType());
+  Align StartAlignment =
+      DL.getValueOrABITypeAlignment(GV->getAlign(), GV->getType());
 
   // Loop over all users and create replacement variables for used aggregate
   // elements.
@@ -554,9 +553,8 @@ static GlobalVariable *SRAGlobal(GlobalVariable *GV, const DataLayout &DL) {
       // had 256 byte alignment for example, something might depend on that:
       // propagate info to each field.
       uint64_t FieldOffset = Layout.getElementOffset(ElementIdx);
-      Align NewAlign(MinAlign(StartAlignment, FieldOffset));
-      if (NewAlign >
-          Align(DL.getABITypeAlignment(STy->getElementType(ElementIdx))))
+      Align NewAlign = commonAlignment(StartAlignment, FieldOffset);
+      if (NewAlign > DL.getABITypeAlign(STy->getElementType(ElementIdx)))
         NGV->setAlignment(NewAlign);
 
       // Copy over the debug info for the variable.
@@ -565,13 +563,13 @@ static GlobalVariable *SRAGlobal(GlobalVariable *GV, const DataLayout &DL) {
       transferSRADebugInfo(GV, NGV, FragmentOffsetInBits, Size);
     } else {
       uint64_t EltSize = DL.getTypeAllocSize(ElTy);
-      Align EltAlign(DL.getABITypeAlignment(ElTy));
+      Align EltAlign = DL.getABITypeAlign(ElTy);
       uint64_t FragmentSizeInBits = DL.getTypeAllocSizeInBits(ElTy);
 
       // Calculate the known alignment of the field.  If the original aggregate
       // had 256 byte alignment for example, something might depend on that:
       // propagate info to each field.
-      Align NewAlign(MinAlign(StartAlignment, EltSize * ElementIdx));
+      Align NewAlign = commonAlignment(StartAlignment, EltSize * ElementIdx);
       if (NewAlign > EltAlign)
         NGV->setAlignment(NewAlign);
       transferSRADebugInfo(GV, NGV, FragmentSizeInBits * ElementIdx,
@@ -2323,13 +2321,10 @@ static void RemovePreallocated(Function *F) {
     assert(PreallocatedSetup && "Did not find preallocated bundle");
     uint64_t ArgCount =
         cast<ConstantInt>(PreallocatedSetup->getArgOperand(0))->getZExtValue();
-    CallBase *NewCB = nullptr;
-    if (InvokeInst *II = dyn_cast<InvokeInst>(CB)) {
-      NewCB = InvokeInst::Create(II, OpBundles, CB);
-    } else {
-      CallInst *CI = cast<CallInst>(CB);
-      NewCB = CallInst::Create(CI, OpBundles, CB);
-    }
+
+    assert((isa<CallInst>(CB) || isa<InvokeInst>(CB)) &&
+           "Unknown indirect call type");
+    CallBase *NewCB = CallBase::Create(CB, OpBundles, CB);
     CB->replaceAllUsesWith(NewCB);
     NewCB->takeName(CB);
     CB->eraseFromParent();
@@ -2447,7 +2442,7 @@ OptimizeFunctions(Module &M,
     // FIXME: We should also hoist alloca affected by this to the entry
     // block if possible.
     if (F->getAttributes().hasAttrSomewhere(Attribute::InAlloca) &&
-        !F->hasAddressTaken()) {
+        !F->hasAddressTaken() && !hasMustTailCallers(F)) {
       RemoveAttribute(F, Attribute::InAlloca);
       Changed = true;
     }
@@ -2570,7 +2565,7 @@ static Constant *EvaluateStoreInto(Constant *Init, Constant *Val,
   if (ArrayType *ATy = dyn_cast<ArrayType>(Init->getType()))
     NumElts = ATy->getNumElements();
   else
-    NumElts = cast<VectorType>(Init->getType())->getNumElements();
+    NumElts = cast<FixedVectorType>(Init->getType())->getNumElements();
 
   // Break up the array into elements.
   for (uint64_t i = 0, e = NumElts; i != e; ++i)
@@ -2706,7 +2701,7 @@ static void BatchCommitValueTo(const DenseMap<Constant*, Constant*> &Mem) {
       else if (auto *ATy = dyn_cast<ArrayType>(Ty))
         NumElts = ATy->getNumElements();
       else
-        NumElts = cast<VectorType>(Ty)->getNumElements();
+        NumElts = cast<FixedVectorType>(Ty)->getNumElements();
       for (unsigned i = 0, e = NumElts; i != e; ++i)
         Elts.push_back(Init->getAggregateElement(i));
     }
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index b0958199ef22c..7d2260f4c169d 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -191,8 +191,8 @@ static void mergeInlinedArrayAllocas(Function *Caller, InlineFunctionInfo &IFI,
     // function.  Also, AllocasForType can be empty of course!
     bool MergedAwayAlloca = false;
     for (AllocaInst *AvailableAlloca : AllocasForType) {
-      unsigned Align1 = AI->getAlignment(),
-               Align2 = AvailableAlloca->getAlignment();
+      Align Align1 = AI->getAlign();
+      Align Align2 = AvailableAlloca->getAlign();
 
       // The available alloca has to be in the right function, not in some other
       // function in this SCC.
@@ -219,18 +219,8 @@ static void mergeInlinedArrayAllocas(Function *Caller, InlineFunctionInfo &IFI,
 
       AI->replaceAllUsesWith(AvailableAlloca);
 
-      if (Align1 != Align2) {
-        if (!Align1 || !Align2) {
-          const DataLayout &DL = Caller->getParent()->getDataLayout();
-          unsigned TypeAlign = DL.getABITypeAlignment(AI->getAllocatedType());
-
-          Align1 = Align1 ? Align1 : TypeAlign;
-          Align2 = Align2 ? Align2 : TypeAlign;
-        }
-
-        if (Align1 > Align2)
-          AvailableAlloca->setAlignment(AI->getAlign());
-      }
+      if (Align1 > Align2)
+        AvailableAlloca->setAlignment(AI->getAlign());
 
       AI->eraseFromParent();
       MergedAwayAlloca = true;
@@ -801,7 +791,9 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
     LazyCallGraph::Node &N = *CG.lookup(F);
     if (CG.lookupSCC(N) != C)
       continue;
-    if (F.hasOptNone()) {
+    if (!Calls[I].first->getCalledFunction()->hasFnAttribute(
+            Attribute::AlwaysInline) &&
+        F.hasOptNone()) {
       setInlineRemark(*Calls[I].first, "optnone attribute");
       continue;
     }
diff --git a/llvm/lib/Transforms/IPO/LoopExtractor.cpp b/llvm/lib/Transforms/IPO/LoopExtractor.cpp
index 14c39503e3858..f7f5b4cf67041 100644
--- a/llvm/lib/Transforms/IPO/LoopExtractor.cpp
+++ b/llvm/lib/Transforms/IPO/LoopExtractor.cpp
@@ -131,18 +131,19 @@ bool LoopExtractor::runOnFunction(Function &F) {
   if (F.empty())
     return false;
 
-  LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>(F).getLoopInfo();
+  bool Changed = false;
+  LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>(F, &Changed).getLoopInfo();
 
   // If there are no loops in the function.
   if (LI.empty())
-    return false;
+    return Changed;
 
   DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>(F).getDomTree();
 
   // If there is more than one top-level loop in this function, extract all of
   // the loops.
   if (std::next(LI.begin()) != LI.end())
-    return extractLoops(LI.begin(), LI.end(), LI, DT);
+    return Changed | extractLoops(LI.begin(), LI.end(), LI, DT);
 
   // Otherwise there is exactly one top-level loop.
   Loop *TLL = *LI.begin();
@@ -171,14 +172,14 @@ bool LoopExtractor::runOnFunction(Function &F) {
     }
 
     if (ShouldExtractLoop)
-      return extractLoop(TLL, LI, DT);
+      return Changed | extractLoop(TLL, LI, DT);
   }
 
   // Okay, this function is a minimal container around the specified loop.
   // If we extract the loop, we will continue to just keep extracting it
   // infinitely... so don't extract it. However, if the loop contains any
   // sub-loops, extract them.
-  return extractLoops(TLL->begin(), TLL->end(), LI, DT);
+  return Changed | extractLoops(TLL->begin(), TLL->end(), LI, DT);
 }
 
 bool LoopExtractor::extractLoops(Loop::iterator From, Loop::iterator To,
diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index 8b108f1bf50c6..fac52cb3d8459 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -835,11 +835,10 @@ void LowerTypeTestsModule::buildBitSetsFromGlobalVariables(
   uint64_t DesiredPadding = 0;
   for (GlobalTypeMember *G : Globals) {
     auto *GV = cast<GlobalVariable>(G->getGlobal());
-    MaybeAlign Alignment(GV->getAlignment());
-    if (!Alignment)
-      Alignment = Align(DL.getABITypeAlignment(GV->getValueType()));
-    MaxAlign = std::max(MaxAlign, *Alignment);
-    uint64_t GVOffset = alignTo(CurOffset + DesiredPadding, *Alignment);
+    Align Alignment =
+        DL.getValueOrABITypeAlignment(GV->getAlign(), GV->getValueType());
+    MaxAlign = std::max(MaxAlign, Alignment);
+    uint64_t GVOffset = alignTo(CurOffset + DesiredPadding, Alignment);
     GlobalLayout[G] = GVOffset;
     if (GVOffset != 0) {
       uint64_t Padding = GVOffset - CurOffset;
diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index 4a5f919aa6316..2a3b2abf61762 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -29,7 +29,6 @@
 
 using namespace llvm;
 using namespace omp;
-using namespace types;
 
 #define DEBUG_TYPE "openmp-opt"
 
@@ -38,6 +37,11 @@ static cl::opt<bool> DisableOpenMPOptimizations(
     cl::desc("Disable OpenMP specific optimizations."), cl::Hidden,
     cl::init(false));
 
+static cl::opt<bool> PrintICVValues("openmp-print-icv-values", cl::init(false),
+                                    cl::Hidden);
+static cl::opt<bool> PrintOpenMPKernels("openmp-print-gpu-kernels",
+                                        cl::init(false), cl::Hidden);
+
 STATISTIC(NumOpenMPRuntimeCallsDeduplicated,
           "Number of OpenMP runtime calls deduplicated");
 STATISTIC(NumOpenMPParallelRegionsDeleted,
@@ -46,27 +50,121 @@ STATISTIC(NumOpenMPRuntimeFunctionsIdentified,
           "Number of OpenMP runtime functions identified");
 STATISTIC(NumOpenMPRuntimeFunctionUsesIdentified,
           "Number of OpenMP runtime function uses identified");
+STATISTIC(NumOpenMPTargetRegionKernels,
+          "Number of OpenMP target region entry points (=kernels) identified");
+STATISTIC(
+    NumOpenMPParallelRegionsReplacedInGPUStateMachine,
+    "Number of OpenMP parallel regions replaced with ID in GPU state machines");
 
 #if !defined(NDEBUG)
 static constexpr auto TAG = "[" DEBUG_TYPE "]";
 #endif
 
+/// Apply \p CB to all uses of \p F. If \p LookThroughConstantExprUses is
+/// true, constant expression users are not given to \p CB but their uses are
+/// traversed transitively.
+template <typename CBTy>
+static void foreachUse(Function &F, CBTy CB,
+                       bool LookThroughConstantExprUses = true) {
+  SmallVector<Use *, 8> Worklist(make_pointer_range(F.uses()));
+
+  for (unsigned idx = 0; idx < Worklist.size(); ++idx) {
+    Use &U = *Worklist[idx];
+
+    // Allow use in constant bitcasts and simply look through them.
+    if (LookThroughConstantExprUses && isa<ConstantExpr>(U.getUser())) {
+      for (Use &CEU : cast<ConstantExpr>(U.getUser())->uses())
+        Worklist.push_back(&CEU);
+      continue;
+    }
+
+    CB(U);
+  }
+}
+
+/// Helper struct to store tracked ICV values at specif instructions.
+struct ICVValue {
+  Instruction *Inst;
+  Value *TrackedValue;
+
+  ICVValue(Instruction *I, Value *Val) : Inst(I), TrackedValue(Val) {}
+};
+
+namespace llvm {
+
+// Provide DenseMapInfo for ICVValue
+template <> struct DenseMapInfo<ICVValue> {
+  using InstInfo = DenseMapInfo<Instruction *>;
+  using ValueInfo = DenseMapInfo<Value *>;
+
+  static inline ICVValue getEmptyKey() {
+    return ICVValue(InstInfo::getEmptyKey(), ValueInfo::getEmptyKey());
+  };
+
+  static inline ICVValue getTombstoneKey() {
+    return ICVValue(InstInfo::getTombstoneKey(), ValueInfo::getTombstoneKey());
+  };
+
+  static unsigned getHashValue(const ICVValue &ICVVal) {
+    return detail::combineHashValue(
+        InstInfo::getHashValue(ICVVal.Inst),
+        ValueInfo::getHashValue(ICVVal.TrackedValue));
+  }
+
+  static bool isEqual(const ICVValue &LHS, const ICVValue &RHS) {
+    return InstInfo::isEqual(LHS.Inst, RHS.Inst) &&
+           ValueInfo::isEqual(LHS.TrackedValue, RHS.TrackedValue);
+  }
+};
+
+} // end namespace llvm
+
 namespace {
 
+struct AAICVTracker;
+
 /// OpenMP specific information. For now, stores RFIs and ICVs also needed for
 /// Attributor runs.
 struct OMPInformationCache : public InformationCache {
   OMPInformationCache(Module &M, AnalysisGetter &AG,
-                      BumpPtrAllocator &Allocator, SetVector<Function *> *CGSCC,
-                      SmallPtrSetImpl<Function *> &ModuleSlice)
-      : InformationCache(M, AG, Allocator, CGSCC), ModuleSlice(ModuleSlice),
-        OMPBuilder(M) {
-    initializeTypes(M);
-    initializeRuntimeFunctions();
+                      BumpPtrAllocator &Allocator, SetVector<Function *> &CGSCC,
+                      SmallPtrSetImpl<Kernel> &Kernels)
+      : InformationCache(M, AG, Allocator, &CGSCC), OMPBuilder(M),
+        Kernels(Kernels) {
+    initializeModuleSlice(CGSCC);
 
     OMPBuilder.initialize();
+    initializeRuntimeFunctions();
+    initializeInternalControlVars();
   }
 
+  /// Generic information that describes an internal control variable.
+  struct InternalControlVarInfo {
+    /// The kind, as described by InternalControlVar enum.
+    InternalControlVar Kind;
+
+    /// The name of the ICV.
+    StringRef Name;
+
+    /// Environment variable associated with this ICV.
+    StringRef EnvVarName;
+
+    /// Initial value kind.
+    ICVInitValue InitKind;
+
+    /// Initial value.
+    ConstantInt *InitValue;
+
+    /// Setter RTL function associated with this ICV.
+    RuntimeFunction Setter;
+
+    /// Getter RTL function associated with this ICV.
+    RuntimeFunction Getter;
+
+    /// RTL Function corresponding to the override clause of this ICV
+    RuntimeFunction Clause;
+  };
+
   /// Generic information that describes a runtime function
   struct RuntimeFunctionInfo {
 
@@ -91,11 +189,17 @@ struct OMPInformationCache : public InformationCache {
     /// Uses of this runtime function per function containing the use.
     using UseVector = SmallVector<Use *, 16>;
 
+    /// Clear UsesMap for runtime function.
+    void clearUsesMap() { UsesMap.clear(); }
+
+    /// Boolean conversion that is true if the runtime function was found.
+    operator bool() const { return Declaration; }
+
     /// Return the vector of uses in function \p F.
     UseVector &getOrCreateUseVector(Function *F) {
-      std::unique_ptr<UseVector> &UV = UsesMap[F];
+      std::shared_ptr<UseVector> &UV = UsesMap[F];
       if (!UV)
-        UV = std::make_unique<UseVector>();
+        UV = std::make_shared<UseVector>();
       return *UV;
     }
 
@@ -118,20 +222,20 @@ struct OMPInformationCache : public InformationCache {
     /// Run the callback \p CB on each use and forget the use if the result is
     /// true. The callback will be fed the function in which the use was
     /// encountered as second argument.
-    void foreachUse(function_ref<bool(Use &, Function &)> CB) {
-      for (auto &It : UsesMap)
-        foreachUse(CB, It.first, It.second.get());
+    void foreachUse(SmallVectorImpl<Function *> &SCC,
+                    function_ref<bool(Use &, Function &)> CB) {
+      for (Function *F : SCC)
+        foreachUse(CB, F);
     }
 
     /// Run the callback \p CB on each use within the function \p F and forget
     /// the use if the result is true.
-    void foreachUse(function_ref<bool(Use &, Function &)> CB, Function *F,
-                    UseVector *Uses = nullptr) {
+    void foreachUse(function_ref<bool(Use &, Function &)> CB, Function *F) {
       SmallVector<unsigned, 8> ToBeDeleted;
       ToBeDeleted.clear();
 
       unsigned Idx = 0;
-      UseVector &UV = Uses ? *Uses : getOrCreateUseVector(F);
+      UseVector &UV = getOrCreateUseVector(F);
 
       for (Use *U : UV) {
         if (CB(*U, *F))
@@ -140,7 +244,7 @@ struct OMPInformationCache : public InformationCache {
       }
 
       // Remove the to-be-deleted indices in reverse order as prior
-      // modifcations will not modify the smaller indices.
+      // modifications will not modify the smaller indices.
       while (!ToBeDeleted.empty()) {
         unsigned Idx = ToBeDeleted.pop_back_val();
         UV[Idx] = UV.back();
@@ -151,11 +255,48 @@ struct OMPInformationCache : public InformationCache {
   private:
     /// Map from functions to all uses of this runtime function contained in
     /// them.
-    DenseMap<Function *, std::unique_ptr<UseVector>> UsesMap;
+    DenseMap<Function *, std::shared_ptr<UseVector>> UsesMap;
   };
 
+  /// Initialize the ModuleSlice member based on \p SCC. ModuleSlices contains
+  /// (a subset of) all functions that we can look at during this SCC traversal.
+  /// This includes functions (transitively) called from the SCC and the
+  /// (transitive) callers of SCC functions. We also can look at a function if
+  /// there is a "reference edge", i.a., if the function somehow uses (!=calls)
+  /// a function in the SCC or a caller of a function in the SCC.
+  void initializeModuleSlice(SetVector<Function *> &SCC) {
+    ModuleSlice.insert(SCC.begin(), SCC.end());
+
+    SmallPtrSet<Function *, 16> Seen;
+    SmallVector<Function *, 16> Worklist(SCC.begin(), SCC.end());
+    while (!Worklist.empty()) {
+      Function *F = Worklist.pop_back_val();
+      ModuleSlice.insert(F);
+
+      for (Instruction &I : instructions(*F))
+        if (auto *CB = dyn_cast<CallBase>(&I))
+          if (Function *Callee = CB->getCalledFunction())
+            if (Seen.insert(Callee).second)
+              Worklist.push_back(Callee);
+    }
+
+    Seen.clear();
+    Worklist.append(SCC.begin(), SCC.end());
+    while (!Worklist.empty()) {
+      Function *F = Worklist.pop_back_val();
+      ModuleSlice.insert(F);
+
+      // Traverse all transitive uses.
+      foreachUse(*F, [&](Use &U) {
+        if (auto *UsrI = dyn_cast<Instruction>(U.getUser()))
+          if (Seen.insert(UsrI->getFunction()).second)
+            Worklist.push_back(UsrI->getFunction());
+      });
+    }
+  }
+
   /// The slice of the module we are allowed to look at.
-  SmallPtrSetImpl<Function *> &ModuleSlice;
+  SmallPtrSet<Function *, 8> ModuleSlice;
 
   /// An OpenMP-IR-Builder instance
   OpenMPIRBuilder OMPBuilder;
@@ -165,6 +306,49 @@ struct OMPInformationCache : public InformationCache {
                   RuntimeFunction::OMPRTL___last>
       RFIs;
 
+  /// Map from ICV kind to the ICV description.
+  EnumeratedArray<InternalControlVarInfo, InternalControlVar,
+                  InternalControlVar::ICV___last>
+      ICVs;
+
+  /// Helper to initialize all internal control variable information for those
+  /// defined in OMPKinds.def.
+  void initializeInternalControlVars() {
+#define ICV_RT_SET(_Name, RTL)                                                 \
+  {                                                                            \
+    auto &ICV = ICVs[_Name];                                                   \
+    ICV.Setter = RTL;                                                          \
+  }
+#define ICV_RT_GET(Name, RTL)                                                  \
+  {                                                                            \
+    auto &ICV = ICVs[Name];                                                    \
+    ICV.Getter = RTL;                                                          \
+  }
+#define ICV_DATA_ENV(Enum, _Name, _EnvVarName, Init)                           \
+  {                                                                            \
+    auto &ICV = ICVs[Enum];                                                    \
+    ICV.Name = _Name;                                                          \
+    ICV.Kind = Enum;                                                           \
+    ICV.InitKind = Init;                                                       \
+    ICV.EnvVarName = _EnvVarName;                                              \
+    switch (ICV.InitKind) {                                                    \
+    case ICV_IMPLEMENTATION_DEFINED:                                           \
+      ICV.InitValue = nullptr;                                                 \
+      break;                                                                   \
+    case ICV_ZERO:                                                             \
+      ICV.InitValue = ConstantInt::get(                                        \
+          Type::getInt32Ty(OMPBuilder.Int32->getContext()), 0);                \
+      break;                                                                   \
+    case ICV_FALSE:                                                            \
+      ICV.InitValue = ConstantInt::getFalse(OMPBuilder.Int1->getContext());    \
+      break;                                                                   \
+    case ICV_LAST:                                                             \
+      break;                                                                   \
+    }                                                                          \
+  }
+#include "llvm/Frontend/OpenMP/OMPKinds.def"
+  }
+
   /// Returns true if the function declaration \p F matches the runtime
   /// function types, that is, return type \p RTFRetType, and argument types
   /// \p RTFArgTypes.
@@ -191,49 +375,83 @@ struct OMPInformationCache : public InformationCache {
     return true;
   }
 
-  /// Helper to initialize all runtime function information for those defined
-  /// in OpenMPKinds.def.
-  void initializeRuntimeFunctions() {
-    // Helper to collect all uses of the decleration in the UsesMap.
-    auto CollectUses = [&](RuntimeFunctionInfo &RFI) {
-      unsigned NumUses = 0;
-      if (!RFI.Declaration)
-        return NumUses;
-      OMPBuilder.addAttributes(RFI.Kind, *RFI.Declaration);
+  // Helper to collect all uses of the declaration in the UsesMap.
+  unsigned collectUses(RuntimeFunctionInfo &RFI, bool CollectStats = true) {
+    unsigned NumUses = 0;
+    if (!RFI.Declaration)
+      return NumUses;
+    OMPBuilder.addAttributes(RFI.Kind, *RFI.Declaration);
 
+    if (CollectStats) {
       NumOpenMPRuntimeFunctionsIdentified += 1;
       NumOpenMPRuntimeFunctionUsesIdentified += RFI.Declaration->getNumUses();
+    }
 
-      // TODO: We directly convert uses into proper calls and unknown uses.
-      for (Use &U : RFI.Declaration->uses()) {
-        if (Instruction *UserI = dyn_cast<Instruction>(U.getUser())) {
-          if (ModuleSlice.count(UserI->getFunction())) {
-            RFI.getOrCreateUseVector(UserI->getFunction()).push_back(&U);
-            ++NumUses;
-          }
-        } else {
-          RFI.getOrCreateUseVector(nullptr).push_back(&U);
+    // TODO: We directly convert uses into proper calls and unknown uses.
+    for (Use &U : RFI.Declaration->uses()) {
+      if (Instruction *UserI = dyn_cast<Instruction>(U.getUser())) {
+        if (ModuleSlice.count(UserI->getFunction())) {
+          RFI.getOrCreateUseVector(UserI->getFunction()).push_back(&U);
           ++NumUses;
         }
+      } else {
+        RFI.getOrCreateUseVector(nullptr).push_back(&U);
+        ++NumUses;
       }
-      return NumUses;
-    };
+    }
+    return NumUses;
+  }
+
+  // Helper function to recollect uses of all runtime functions.
+  void recollectUses() {
+    for (int Idx = 0; Idx < RFIs.size(); ++Idx) {
+      auto &RFI = RFIs[static_cast<RuntimeFunction>(Idx)];
+      RFI.clearUsesMap();
+      collectUses(RFI, /*CollectStats*/ false);
+    }
+  }
 
+  /// Helper to initialize all runtime function information for those defined
+  /// in OpenMPKinds.def.
+  void initializeRuntimeFunctions() {
     Module &M = *((*ModuleSlice.begin())->getParent());
 
+    // Helper macros for handling __VA_ARGS__ in OMP_RTL
+#define OMP_TYPE(VarName, ...)                                                 \
+  Type *VarName = OMPBuilder.VarName;                                          \
+  (void)VarName;
+
+#define OMP_ARRAY_TYPE(VarName, ...)                                           \
+  ArrayType *VarName##Ty = OMPBuilder.VarName##Ty;                             \
+  (void)VarName##Ty;                                                           \
+  PointerType *VarName##PtrTy = OMPBuilder.VarName##PtrTy;                     \
+  (void)VarName##PtrTy;
+
+#define OMP_FUNCTION_TYPE(VarName, ...)                                        \
+  FunctionType *VarName = OMPBuilder.VarName;                                  \
+  (void)VarName;                                                               \
+  PointerType *VarName##Ptr = OMPBuilder.VarName##Ptr;                         \
+  (void)VarName##Ptr;
+
+#define OMP_STRUCT_TYPE(VarName, ...)                                          \
+  StructType *VarName = OMPBuilder.VarName;                                    \
+  (void)VarName;                                                               \
+  PointerType *VarName##Ptr = OMPBuilder.VarName##Ptr;                         \
+  (void)VarName##Ptr;
+
 #define OMP_RTL(_Enum, _Name, _IsVarArg, _ReturnType, ...)                     \
   {                                                                            \
     SmallVector<Type *, 8> ArgsTypes({__VA_ARGS__});                           \
     Function *F = M.getFunction(_Name);                                        \
-    if (declMatchesRTFTypes(F, _ReturnType, ArgsTypes)) {                      \
+    if (declMatchesRTFTypes(F, OMPBuilder._ReturnType, ArgsTypes)) {           \
       auto &RFI = RFIs[_Enum];                                                 \
       RFI.Kind = _Enum;                                                        \
       RFI.Name = _Name;                                                        \
       RFI.IsVarArg = _IsVarArg;                                                \
-      RFI.ReturnType = _ReturnType;                                            \
+      RFI.ReturnType = OMPBuilder._ReturnType;                                 \
       RFI.ArgumentTypes = std::move(ArgsTypes);                                \
       RFI.Declaration = F;                                                     \
-      unsigned NumUses = CollectUses(RFI);                                     \
+      unsigned NumUses = collectUses(RFI);                                     \
       (void)NumUses;                                                           \
       LLVM_DEBUG({                                                             \
         dbgs() << TAG << RFI.Name << (RFI.Declaration ? "" : " not")           \
@@ -249,6 +467,9 @@ struct OMPInformationCache : public InformationCache {
 
     // TODO: We should attach the attributes defined in OMPKinds.def.
   }
+
+  /// Collection of known kernels (\see Kernel) in the module.
+  SmallPtrSetImpl<Kernel> &Kernels;
 };
 
 struct OpenMPOpt {
@@ -258,24 +479,75 @@ struct OpenMPOpt {
 
   OpenMPOpt(SmallVectorImpl<Function *> &SCC, CallGraphUpdater &CGUpdater,
             OptimizationRemarkGetter OREGetter,
-            OMPInformationCache &OMPInfoCache)
+            OMPInformationCache &OMPInfoCache, Attributor &A)
       : M(*(*SCC.begin())->getParent()), SCC(SCC), CGUpdater(CGUpdater),
-        OREGetter(OREGetter), OMPInfoCache(OMPInfoCache) {}
+        OREGetter(OREGetter), OMPInfoCache(OMPInfoCache), A(A) {}
 
   /// Run all OpenMP optimizations on the underlying SCC/ModuleSlice.
   bool run() {
+    if (SCC.empty())
+      return false;
+
     bool Changed = false;
 
     LLVM_DEBUG(dbgs() << TAG << "Run on SCC with " << SCC.size()
                       << " functions in a slice with "
                       << OMPInfoCache.ModuleSlice.size() << " functions\n");
 
+    if (PrintICVValues)
+      printICVs();
+    if (PrintOpenMPKernels)
+      printKernels();
+
+    Changed |= rewriteDeviceCodeStateMachine();
+
+    Changed |= runAttributor();
+
+    // Recollect uses, in case Attributor deleted any.
+    OMPInfoCache.recollectUses();
+
     Changed |= deduplicateRuntimeCalls();
     Changed |= deleteParallelRegions();
 
     return Changed;
   }
 
+  /// Print initial ICV values for testing.
+  /// FIXME: This should be done from the Attributor once it is added.
+  void printICVs() const {
+    InternalControlVar ICVs[] = {ICV_nthreads, ICV_active_levels, ICV_cancel};
+
+    for (Function *F : OMPInfoCache.ModuleSlice) {
+      for (auto ICV : ICVs) {
+        auto ICVInfo = OMPInfoCache.ICVs[ICV];
+        auto Remark = [&](OptimizationRemark OR) {
+          return OR << "OpenMP ICV " << ore::NV("OpenMPICV", ICVInfo.Name)
+                    << " Value: "
+                    << (ICVInfo.InitValue
+                            ? ICVInfo.InitValue->getValue().toString(10, true)
+                            : "IMPLEMENTATION_DEFINED");
+        };
+
+        emitRemarkOnFunction(F, "OpenMPICVTracker", Remark);
+      }
+    }
+  }
+
+  /// Print OpenMP GPU kernels for testing.
+  void printKernels() const {
+    for (Function *F : SCC) {
+      if (!OMPInfoCache.Kernels.count(F))
+        continue;
+
+      auto Remark = [&](OptimizationRemark OR) {
+        return OR << "OpenMP GPU kernel "
+                  << ore::NV("OpenMPGPUKernel", F->getName()) << "\n";
+      };
+
+      emitRemarkOnFunction(F, "OpenMPGPU", Remark);
+    }
+  }
+
   /// Return the call if \p U is a callee use in a regular call. If \p RFI is
   /// given it has to be the callee or a nullptr is returned.
   static CallInst *getCallIfRegularCall(
@@ -341,12 +613,12 @@ struct OpenMPOpt {
       return true;
     };
 
-    RFI.foreachUse(DeleteCallCB);
+    RFI.foreachUse(SCC, DeleteCallCB);
 
     return Changed;
   }
 
-  /// Try to eliminiate runtime calls by reusing existing ones.
+  /// Try to eliminate runtime calls by reusing existing ones.
   bool deduplicateRuntimeCalls() {
     bool Changed = false;
 
@@ -426,7 +698,7 @@ struct OpenMPOpt {
                                   /* GlobalOnly */ true, SingleChoice);
       return false;
     };
-    RFI.foreachUse(CombineIdentStruct);
+    RFI.foreachUse(SCC, CombineIdentStruct);
 
     if (!Ident || !SingleChoice) {
       // The IRBuilder uses the insertion block to get to the module, this is
@@ -442,7 +714,7 @@ struct OpenMPOpt {
     return Ident;
   }
 
-  /// Try to eliminiate calls of \p RFI in \p F by reusing an existing one or
+  /// Try to eliminate calls of \p RFI in \p F by reusing an existing one or
   /// \p ReplVal if given.
   bool deduplicateRuntimeCalls(Function &F,
                                OMPInformationCache::RuntimeFunctionInfo &RFI,
@@ -460,11 +732,11 @@ struct OpenMPOpt {
            "Unexpected replacement value!");
 
     // TODO: Use dominance to find a good position instead.
-    auto CanBeMoved = [](CallBase &CB) {
+    auto CanBeMoved = [this](CallBase &CB) {
       unsigned NumArgs = CB.getNumArgOperands();
       if (NumArgs == 0)
         return true;
-      if (CB.getArgOperand(0)->getType() != IdentPtr)
+      if (CB.getArgOperand(0)->getType() != OMPInfoCache.OMPBuilder.IdentPtr)
         return false;
       for (unsigned u = 1; u < NumArgs; ++u)
         if (isa<Instruction>(CB.getArgOperand(u)))
@@ -499,7 +771,7 @@ struct OpenMPOpt {
     // existing and used by one of the calls, or created from scratch.
     if (CallBase *CI = dyn_cast<CallBase>(ReplVal)) {
       if (CI->getNumArgOperands() > 0 &&
-          CI->getArgOperand(0)->getType() == IdentPtr) {
+          CI->getArgOperand(0)->getType() == OMPInfoCache.OMPBuilder.IdentPtr) {
         Value *Ident = getCombinedIdentFromCallUsesIn(RFI, F,
                                                       /* GlobalOnly */ true);
         CI->setArgOperand(0, Ident);
@@ -526,7 +798,7 @@ struct OpenMPOpt {
       Changed = true;
       return true;
     };
-    RFI.foreachUse(ReplaceAndDeleteCB);
+    RFI.foreachUse(SCC, ReplaceAndDeleteCB);
 
     return Changed;
   }
@@ -569,7 +841,7 @@ struct OpenMPOpt {
     OMPInformationCache::RuntimeFunctionInfo &GlobThreadNumRFI =
         OMPInfoCache.RFIs[OMPRTL___kmpc_global_thread_num];
 
-    GlobThreadNumRFI.foreachUse([&](Use &U, Function &F) {
+    GlobThreadNumRFI.foreachUse(SCC, [&](Use &U, Function &F) {
       if (CallInst *CI = getCallIfRegularCall(U, &GlobThreadNumRFI))
         AddUserArgs(*CI);
       return false;
@@ -582,6 +854,31 @@ struct OpenMPOpt {
       AddUserArgs(*GTIdArgs[u]);
   }
 
+  /// Kernel (=GPU) optimizations and utility functions
+  ///
+  ///{{
+
+  /// Check if \p F is a kernel, hence entry point for target offloading.
+  bool isKernel(Function &F) { return OMPInfoCache.Kernels.count(&F); }
+
+  /// Cache to remember the unique kernel for a function.
+  DenseMap<Function *, Optional<Kernel>> UniqueKernelMap;
+
+  /// Find the unique kernel that will execute \p F, if any.
+  Kernel getUniqueKernelFor(Function &F);
+
+  /// Find the unique kernel that will execute \p I, if any.
+  Kernel getUniqueKernelFor(Instruction &I) {
+    return getUniqueKernelFor(*I.getFunction());
+  }
+
+  /// Rewrite the device (=GPU) code state machine create in non-SPMD mode in
+  /// the cases we can avoid taking the address of a function.
+  bool rewriteDeviceCodeStateMachine();
+
+  ///
+  ///}}
+
   /// Emit a remark generically
   ///
   /// This template function can be used to generically emit a remark. The
@@ -596,7 +893,7 @@ struct OpenMPOpt {
   template <typename RemarkKind,
             typename RemarkCallBack = function_ref<RemarkKind(RemarkKind &&)>>
   void emitRemark(Instruction *Inst, StringRef RemarkName,
-                  RemarkCallBack &&RemarkCB) {
+                  RemarkCallBack &&RemarkCB) const {
     Function *F = Inst->getParent()->getParent();
     auto &ORE = OREGetter(F);
 
@@ -604,7 +901,20 @@ struct OpenMPOpt {
         [&]() { return RemarkCB(RemarkKind(DEBUG_TYPE, RemarkName, Inst)); });
   }
 
-  /// The underyling module.
+  /// Emit a remark on a function. Since only OptimizationRemark is supporting
+  /// this, it can't be made generic.
+  void
+  emitRemarkOnFunction(Function *F, StringRef RemarkName,
+                       function_ref<OptimizationRemark(OptimizationRemark &&)>
+                           &&RemarkCB) const {
+    auto &ORE = OREGetter(F);
+
+    ORE.emit([&]() {
+      return RemarkCB(OptimizationRemark(DEBUG_TYPE, RemarkName, F));
+    });
+  }
+
+  /// The underlying module.
   Module &M;
 
   /// The SCC we are operating on.
@@ -619,9 +929,338 @@ struct OpenMPOpt {
 
   /// OpenMP-specific information cache. Also Used for Attributor runs.
   OMPInformationCache &OMPInfoCache;
+
+  /// Attributor instance.
+  Attributor &A;
+
+  /// Helper function to run Attributor on SCC.
+  bool runAttributor() {
+    if (SCC.empty())
+      return false;
+
+    registerAAs();
+
+    ChangeStatus Changed = A.run();
+
+    LLVM_DEBUG(dbgs() << "[Attributor] Done with " << SCC.size()
+                      << " functions, result: " << Changed << ".\n");
+
+    return Changed == ChangeStatus::CHANGED;
+  }
+
+  /// Populate the Attributor with abstract attribute opportunities in the
+  /// function.
+  void registerAAs() {
+    for (Function *F : SCC) {
+      if (F->isDeclaration())
+        continue;
+
+      A.getOrCreateAAFor<AAICVTracker>(IRPosition::function(*F));
+    }
+  }
+};
+
+Kernel OpenMPOpt::getUniqueKernelFor(Function &F) {
+  if (!OMPInfoCache.ModuleSlice.count(&F))
+    return nullptr;
+
+  // Use a scope to keep the lifetime of the CachedKernel short.
+  {
+    Optional<Kernel> &CachedKernel = UniqueKernelMap[&F];
+    if (CachedKernel)
+      return *CachedKernel;
+
+    // TODO: We should use an AA to create an (optimistic and callback
+    //       call-aware) call graph. For now we stick to simple patterns that
+    //       are less powerful, basically the worst fixpoint.
+    if (isKernel(F)) {
+      CachedKernel = Kernel(&F);
+      return *CachedKernel;
+    }
+
+    CachedKernel = nullptr;
+    if (!F.hasLocalLinkage())
+      return nullptr;
+  }
+
+  auto GetUniqueKernelForUse = [&](const Use &U) -> Kernel {
+    if (auto *Cmp = dyn_cast<ICmpInst>(U.getUser())) {
+      // Allow use in equality comparisons.
+      if (Cmp->isEquality())
+        return getUniqueKernelFor(*Cmp);
+      return nullptr;
+    }
+    if (auto *CB = dyn_cast<CallBase>(U.getUser())) {
+      // Allow direct calls.
+      if (CB->isCallee(&U))
+        return getUniqueKernelFor(*CB);
+      // Allow the use in __kmpc_kernel_prepare_parallel calls.
+      if (Function *Callee = CB->getCalledFunction())
+        if (Callee->getName() == "__kmpc_kernel_prepare_parallel")
+          return getUniqueKernelFor(*CB);
+      return nullptr;
+    }
+    // Disallow every other use.
+    return nullptr;
+  };
+
+  // TODO: In the future we want to track more than just a unique kernel.
+  SmallPtrSet<Kernel, 2> PotentialKernels;
+  foreachUse(F, [&](const Use &U) {
+    PotentialKernels.insert(GetUniqueKernelForUse(U));
+  });
+
+  Kernel K = nullptr;
+  if (PotentialKernels.size() == 1)
+    K = *PotentialKernels.begin();
+
+  // Cache the result.
+  UniqueKernelMap[&F] = K;
+
+  return K;
+}
+
+bool OpenMPOpt::rewriteDeviceCodeStateMachine() {
+  OMPInformationCache::RuntimeFunctionInfo &KernelPrepareParallelRFI =
+      OMPInfoCache.RFIs[OMPRTL___kmpc_kernel_prepare_parallel];
+
+  bool Changed = false;
+  if (!KernelPrepareParallelRFI)
+    return Changed;
+
+  for (Function *F : SCC) {
+
+    // Check if the function is uses in a __kmpc_kernel_prepare_parallel call at
+    // all.
+    bool UnknownUse = false;
+    unsigned NumDirectCalls = 0;
+
+    SmallVector<Use *, 2> ToBeReplacedStateMachineUses;
+    foreachUse(*F, [&](Use &U) {
+      if (auto *CB = dyn_cast<CallBase>(U.getUser()))
+        if (CB->isCallee(&U)) {
+          ++NumDirectCalls;
+          return;
+        }
+
+      if (isa<ICmpInst>(U.getUser())) {
+        ToBeReplacedStateMachineUses.push_back(&U);
+        return;
+      }
+      if (OpenMPOpt::getCallIfRegularCall(*U.getUser(),
+                                          &KernelPrepareParallelRFI)) {
+        ToBeReplacedStateMachineUses.push_back(&U);
+        return;
+      }
+      UnknownUse = true;
+    });
+
+    // If this ever hits, we should investigate.
+    if (UnknownUse || NumDirectCalls != 1)
+      continue;
+
+    // TODO: This is not a necessary restriction and should be lifted.
+    if (ToBeReplacedStateMachineUses.size() != 2)
+      continue;
+
+    // Even if we have __kmpc_kernel_prepare_parallel calls, we (for now) give
+    // up if the function is not called from a unique kernel.
+    Kernel K = getUniqueKernelFor(*F);
+    if (!K)
+      continue;
+
+    // We now know F is a parallel body function called only from the kernel K.
+    // We also identified the state machine uses in which we replace the
+    // function pointer by a new global symbol for identification purposes. This
+    // ensures only direct calls to the function are left.
+
+    Module &M = *F->getParent();
+    Type *Int8Ty = Type::getInt8Ty(M.getContext());
+
+    auto *ID = new GlobalVariable(
+        M, Int8Ty, /* isConstant */ true, GlobalValue::PrivateLinkage,
+        UndefValue::get(Int8Ty), F->getName() + ".ID");
+
+    for (Use *U : ToBeReplacedStateMachineUses)
+      U->set(ConstantExpr::getBitCast(ID, U->get()->getType()));
+
+    ++NumOpenMPParallelRegionsReplacedInGPUStateMachine;
+
+    Changed = true;
+  }
+
+  return Changed;
+}
+
+/// Abstract Attribute for tracking ICV values.
+struct AAICVTracker : public StateWrapper<BooleanState, AbstractAttribute> {
+  using Base = StateWrapper<BooleanState, AbstractAttribute>;
+  AAICVTracker(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
+
+  /// Returns true if value is assumed to be tracked.
+  bool isAssumedTracked() const { return getAssumed(); }
+
+  /// Returns true if value is known to be tracked.
+  bool isKnownTracked() const { return getAssumed(); }
+
+  /// Create an abstract attribute biew for the position \p IRP.
+  static AAICVTracker &createForPosition(const IRPosition &IRP, Attributor &A);
+
+  /// Return the value with which \p I can be replaced for specific \p ICV.
+  virtual Value *getReplacementValue(InternalControlVar ICV,
+                                     const Instruction *I, Attributor &A) = 0;
+
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AAICVTracker"; }
+
+  static const char ID;
+};
+
+struct AAICVTrackerFunction : public AAICVTracker {
+  AAICVTrackerFunction(const IRPosition &IRP, Attributor &A)
+      : AAICVTracker(IRP, A) {}
+
+  // FIXME: come up with better string.
+  const std::string getAsStr() const override { return "ICVTracker"; }
+
+  // FIXME: come up with some stats.
+  void trackStatistics() const override {}
+
+  /// TODO: decide whether to deduplicate here, or use current
+  /// deduplicateRuntimeCalls function.
+  ChangeStatus manifest(Attributor &A) override {
+    ChangeStatus Changed = ChangeStatus::UNCHANGED;
+
+    for (InternalControlVar &ICV : TrackableICVs)
+      if (deduplicateICVGetters(ICV, A))
+        Changed = ChangeStatus::CHANGED;
+
+    return Changed;
+  }
+
+  bool deduplicateICVGetters(InternalControlVar &ICV, Attributor &A) {
+    auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
+    auto &ICVInfo = OMPInfoCache.ICVs[ICV];
+    auto &GetterRFI = OMPInfoCache.RFIs[ICVInfo.Getter];
+
+    bool Changed = false;
+
+    auto ReplaceAndDeleteCB = [&](Use &U, Function &Caller) {
+      CallInst *CI = OpenMPOpt::getCallIfRegularCall(U, &GetterRFI);
+      Instruction *UserI = cast<Instruction>(U.getUser());
+      Value *ReplVal = getReplacementValue(ICV, UserI, A);
+
+      if (!ReplVal || !CI)
+        return false;
+
+      A.removeCallSite(CI);
+      CI->replaceAllUsesWith(ReplVal);
+      CI->eraseFromParent();
+      Changed = true;
+      return true;
+    };
+
+    GetterRFI.foreachUse(ReplaceAndDeleteCB, getAnchorScope());
+    return Changed;
+  }
+
+  // Map of ICV to their values at specific program point.
+  EnumeratedArray<SmallSetVector<ICVValue, 4>, InternalControlVar,
+                  InternalControlVar::ICV___last>
+      ICVValuesMap;
+
+  // Currently only nthreads is being tracked.
+  // this array will only grow with time.
+  InternalControlVar TrackableICVs[1] = {ICV_nthreads};
+
+  ChangeStatus updateImpl(Attributor &A) override {
+    ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
+
+    Function *F = getAnchorScope();
+
+    auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
+
+    for (InternalControlVar ICV : TrackableICVs) {
+      auto &SetterRFI = OMPInfoCache.RFIs[OMPInfoCache.ICVs[ICV].Setter];
+
+      auto TrackValues = [&](Use &U, Function &) {
+        CallInst *CI = OpenMPOpt::getCallIfRegularCall(U);
+        if (!CI)
+          return false;
+
+        // FIXME: handle setters with more that 1 arguments.
+        /// Track new value.
+        if (ICVValuesMap[ICV].insert(ICVValue(CI, CI->getArgOperand(0))))
+          HasChanged = ChangeStatus::CHANGED;
+
+        return false;
+      };
+
+      SetterRFI.foreachUse(TrackValues, F);
+    }
+
+    return HasChanged;
+  }
+
+  /// Return the value with which \p I can be replaced for specific \p ICV.
+  Value *getReplacementValue(InternalControlVar ICV, const Instruction *I,
+                             Attributor &A) override {
+    const BasicBlock *CurrBB = I->getParent();
+
+    auto &ValuesSet = ICVValuesMap[ICV];
+    auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
+    auto &GetterRFI = OMPInfoCache.RFIs[OMPInfoCache.ICVs[ICV].Getter];
+
+    for (const auto &ICVVal : ValuesSet) {
+      if (CurrBB == ICVVal.Inst->getParent()) {
+        if (!ICVVal.Inst->comesBefore(I))
+          continue;
+
+        // both instructions are in the same BB and at \p I we know the ICV
+        // value.
+        while (I != ICVVal.Inst) {
+          // we don't yet know if a call might update an ICV.
+          // TODO: check callsite AA for value.
+          if (const auto *CB = dyn_cast<CallBase>(I))
+            if (CB->getCalledFunction() != GetterRFI.Declaration)
+              return nullptr;
+
+          I = I->getPrevNode();
+        }
+
+        // No call in between, return the value.
+        return ICVVal.TrackedValue;
+      }
+    }
+
+    // No value was tracked.
+    return nullptr;
+  }
 };
 } // namespace
 
+const char AAICVTracker::ID = 0;
+
+AAICVTracker &AAICVTracker::createForPosition(const IRPosition &IRP,
+                                              Attributor &A) {
+  AAICVTracker *AA = nullptr;
+  switch (IRP.getPositionKind()) {
+  case IRPosition::IRP_INVALID:
+  case IRPosition::IRP_FLOAT:
+  case IRPosition::IRP_ARGUMENT:
+  case IRPosition::IRP_RETURNED:
+  case IRPosition::IRP_CALL_SITE_RETURNED:
+  case IRPosition::IRP_CALL_SITE_ARGUMENT:
+  case IRPosition::IRP_CALL_SITE:
+    llvm_unreachable("ICVTracker can only be created for function position!");
+  case IRPosition::IRP_FUNCTION:
+    AA = new (A.Allocator) AAICVTrackerFunction(IRP, A);
+    break;
+  }
+
+  return *AA;
+}
+
 PreservedAnalyses OpenMPOptPass::run(LazyCallGraph::SCC &C,
                                      CGSCCAnalysisManager &AM,
                                      LazyCallGraph &CG, CGSCCUpdateResult &UR) {
@@ -631,12 +1270,9 @@ PreservedAnalyses OpenMPOptPass::run(LazyCallGraph::SCC &C,
   if (DisableOpenMPOptimizations)
     return PreservedAnalyses::all();
 
-  SmallPtrSet<Function *, 16> ModuleSlice;
   SmallVector<Function *, 16> SCC;
-  for (LazyCallGraph::Node &N : C) {
+  for (LazyCallGraph::Node &N : C)
     SCC.push_back(&N.getFunction());
-    ModuleSlice.insert(SCC.back());
-  }
 
   if (SCC.empty())
     return PreservedAnalyses::all();
@@ -656,10 +1292,12 @@ PreservedAnalyses OpenMPOptPass::run(LazyCallGraph::SCC &C,
   SetVector<Function *> Functions(SCC.begin(), SCC.end());
   BumpPtrAllocator Allocator;
   OMPInformationCache InfoCache(*(Functions.back()->getParent()), AG, Allocator,
-                                /*CGSCC*/ &Functions, ModuleSlice);
+                                /*CGSCC*/ Functions, OMPInModule.getKernels());
+
+  Attributor A(Functions, InfoCache, CGUpdater);
 
   // TODO: Compute the module slice we are allowed to look at.
-  OpenMPOpt OMPOpt(SCC, CGUpdater, OREGetter, InfoCache);
+  OpenMPOpt OMPOpt(SCC, CGUpdater, OREGetter, InfoCache, A);
   bool Changed = OMPOpt.run();
   (void)Changed;
   return PreservedAnalyses::all();
@@ -692,14 +1330,11 @@ struct OpenMPOptLegacyPass : public CallGraphSCCPass {
     if (DisableOpenMPOptimizations || skipSCC(CGSCC))
       return false;
 
-    SmallPtrSet<Function *, 16> ModuleSlice;
     SmallVector<Function *, 16> SCC;
     for (CallGraphNode *CGN : CGSCC)
       if (Function *Fn = CGN->getFunction())
-        if (!Fn->isDeclaration()) {
+        if (!Fn->isDeclaration())
           SCC.push_back(Fn);
-          ModuleSlice.insert(Fn);
-        }
 
     if (SCC.empty())
       return false;
@@ -719,12 +1354,14 @@ struct OpenMPOptLegacyPass : public CallGraphSCCPass {
     AnalysisGetter AG;
     SetVector<Function *> Functions(SCC.begin(), SCC.end());
     BumpPtrAllocator Allocator;
-    OMPInformationCache InfoCache(*(Functions.back()->getParent()), AG,
-                                  Allocator,
-                                  /*CGSCC*/ &Functions, ModuleSlice);
+    OMPInformationCache InfoCache(
+        *(Functions.back()->getParent()), AG, Allocator,
+        /*CGSCC*/ Functions, OMPInModule.getKernels());
+
+    Attributor A(Functions, InfoCache, CGUpdater);
 
     // TODO: Compute the module slice we are allowed to look at.
-    OpenMPOpt OMPOpt(SCC, CGUpdater, OREGetter, InfoCache);
+    OpenMPOpt OMPOpt(SCC, CGUpdater, OREGetter, InfoCache, A);
     return OMPOpt.run();
   }
 
@@ -733,14 +1370,53 @@ struct OpenMPOptLegacyPass : public CallGraphSCCPass {
 
 } // end anonymous namespace
 
+void OpenMPInModule::identifyKernels(Module &M) {
+
+  NamedMDNode *MD = M.getOrInsertNamedMetadata("nvvm.annotations");
+  if (!MD)
+    return;
+
+  for (auto *Op : MD->operands()) {
+    if (Op->getNumOperands() < 2)
+      continue;
+    MDString *KindID = dyn_cast<MDString>(Op->getOperand(1));
+    if (!KindID || KindID->getString() != "kernel")
+      continue;
+
+    Function *KernelFn =
+        mdconst::dyn_extract_or_null<Function>(Op->getOperand(0));
+    if (!KernelFn)
+      continue;
+
+    ++NumOpenMPTargetRegionKernels;
+
+    Kernels.insert(KernelFn);
+  }
+}
+
 bool llvm::omp::containsOpenMP(Module &M, OpenMPInModule &OMPInModule) {
   if (OMPInModule.isKnown())
     return OMPInModule;
 
+  // MSVC doesn't like long if-else chains for some reason and instead just
+  // issues an error. Work around it..
+  do {
 #define OMP_RTL(_Enum, _Name, ...)                                             \
-  if (M.getFunction(_Name))                                                    \
-    return OMPInModule = true;
+  if (M.getFunction(_Name)) {                                                  \
+    OMPInModule = true;                                                        \
+    break;                                                                     \
+  }
 #include "llvm/Frontend/OpenMP/OMPKinds.def"
+  } while (false);
+
+  // Identify kernels once. TODO: We should split the OMPInformationCache into a
+  // module and an SCC part. The kernel information, among other things, could
+  // go into the module part.
+  if (OMPInModule.isKnown() && OMPInModule) {
+    OMPInModule.identifyKernels(M);
+    return true;
+  }
+
   return OMPInModule = false;
 }
 
diff --git a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
index 9534fb874107b..b65eb469a4923 100644
--- a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
+++ b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
@@ -195,6 +195,7 @@ PassManagerBuilder::PassManagerBuilder() {
     PrepareForThinLTO = EnablePrepareForThinLTO;
     PerformThinLTO = EnablePerformThinLTO;
     DivergentTarget = false;
+    CallGraphProfile = true;
 }
 
 PassManagerBuilder::~PassManagerBuilder() {
@@ -834,6 +835,10 @@ void PassManagerBuilder::populateModulePassManager(
   if (MergeFunctions)
     MPM.add(createMergeFunctionsPass());
 
+  // Add Module flag "CG Profile" based on Branch Frequency Information.
+  if (CallGraphProfile)
+    MPM.add(createCGProfileLegacyPass());
+
   // LoopSink pass sinks instructions hoisted by LICM, which serves as a
   // canonicalization pass that enables other optimizations. As a result,
   // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index 51fdf8f0db29c..b6871e260532d 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -149,14 +149,17 @@ static cl::opt<bool> ProfileAccurateForSymsInList(
              "be accurate. It may be overriden by profile-sample-accurate. "));
 
 static cl::opt<bool> ProfileMergeInlinee(
-    "sample-profile-merge-inlinee", cl::Hidden, cl::init(false),
+    "sample-profile-merge-inlinee", cl::Hidden, cl::init(true),
     cl::desc("Merge past inlinee's profile to outline version if sample "
-             "profile loader decided not to inline a call site."));
+             "profile loader decided not to inline a call site. It will "
+             "only be enabled when top-down order of profile loading is "
+             "enabled. "));
 
 static cl::opt<bool> ProfileTopDownLoad(
-    "sample-profile-top-down-load", cl::Hidden, cl::init(false),
+    "sample-profile-top-down-load", cl::Hidden, cl::init(true),
     cl::desc("Do profile annotation and inlining for functions in top-down "
-             "order of call graph during sample profile loading."));
+             "order of call graph during sample profile loading. It only "
+             "works for new pass manager. "));
 
 static cl::opt<bool> ProfileSizeInline(
     "sample-profile-inline-size", cl::Hidden, cl::init(false),
@@ -1785,6 +1788,15 @@ SampleProfileLoader::buildFunctionOrder(Module &M, CallGraph *CG) {
   FunctionOrderList.reserve(M.size());
 
   if (!ProfileTopDownLoad || CG == nullptr) {
+    if (ProfileMergeInlinee) {
+      // Disable ProfileMergeInlinee if profile is not loaded in top down order,
+      // because the profile for a function may be used for the profile
+      // annotation of its outline copy before the profile merging of its
+      // non-inlined inline instances, and that is not the way how
+      // ProfileMergeInlinee is supposed to work.
+      ProfileMergeInlinee = false;
+    }
+
     for (Function &F : M)
       if (!F.isDeclaration() && F.hasFnAttribute("use-sample-profile"))
         FunctionOrderList.push_back(&F);
diff --git a/llvm/lib/Transforms/IPO/SyntheticCountsPropagation.cpp b/llvm/lib/Transforms/IPO/SyntheticCountsPropagation.cpp
index 5a5e94ae1fb58..1b1e91cafa651 100644
--- a/llvm/lib/Transforms/IPO/SyntheticCountsPropagation.cpp
+++ b/llvm/lib/Transforms/IPO/SyntheticCountsPropagation.cpp
@@ -109,8 +109,7 @@ PreservedAnalyses SyntheticCountsPropagation::run(Module &M,
     Optional<Scaled64> Res = None;
     if (!Edge.first)
       return Res;
-    assert(isa<Instruction>(Edge.first));
-    CallBase &CB = cast<CallBase>(*Edge.first);
+    CallBase &CB = *cast<CallBase>(*Edge.first);
     Function *Caller = CB.getCaller();
     auto &BFI = FAM.getResult<BlockFrequencyAnalysis>(*Caller);
 
diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
index 9ea32f63fa010..7bea1503ea630 100644
--- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
@@ -1652,10 +1652,8 @@ void DevirtModule::rebuildGlobal(VTableBits &B) {
 
   // Align the before byte array to the global's minimum alignment so that we
   // don't break any alignment requirements on the global.
-  MaybeAlign Alignment(B.GV->getAlignment());
-  if (!Alignment)
-    Alignment =
-        Align(M.getDataLayout().getABITypeAlignment(B.GV->getValueType()));
+  Align Alignment = M.getDataLayout().getValueOrABITypeAlignment(
+      B.GV->getAlign(), B.GV->getValueType());
   B.Before.Bytes.resize(alignTo(B.Before.Bytes.size(), Alignment));
 
   // Before was stored in reverse order; flip it now.
@@ -1713,7 +1711,6 @@ void DevirtModule::scanTypeTestUsers(Function *TypeTestFunc) {
   // points to a member of the type identifier %md. Group calls by (type ID,
   // offset) pair (effectively the identity of the virtual function) and store
   // to CallSlots.
-  DenseSet<CallBase *> SeenCallSites;
   for (auto I = TypeTestFunc->use_begin(), E = TypeTestFunc->use_end();
        I != E;) {
     auto CI = dyn_cast<CallInst>(I->getUser());
@@ -1732,15 +1729,8 @@ void DevirtModule::scanTypeTestUsers(Function *TypeTestFunc) {
       Metadata *TypeId =
           cast<MetadataAsValue>(CI->getArgOperand(1))->getMetadata();
       Value *Ptr = CI->getArgOperand(0)->stripPointerCasts();
-      for (DevirtCallSite Call : DevirtCalls) {
-        // Only add this CallSite if we haven't seen it before. The vtable
-        // pointer may have been CSE'd with pointers from other call sites,
-        // and we don't want to process call sites multiple times. We can't
-        // just skip the vtable Ptr if it has been seen before, however, since
-        // it may be shared by type tests that dominate different calls.
-        if (SeenCallSites.insert(&Call.CB).second)
-          CallSlots[{TypeId, Call.Offset}].addCallSite(Ptr, Call.CB, nullptr);
-      }
+      for (DevirtCallSite Call : DevirtCalls)
+        CallSlots[{TypeId, Call.Offset}].addCallSite(Ptr, Call.CB, nullptr);
     }
 
     // We no longer need the assumes or the type test.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 655ba941f7c57..a7f5e0a7774d2 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -1787,6 +1787,21 @@ Instruction *InstCombiner::visitSub(BinaryOperator &I) {
     return BinaryOperator::CreateSub(XZ, YW);
   }
 
+  auto m_AddRdx = [](Value *&Vec) {
+    return m_OneUse(
+        m_Intrinsic<Intrinsic::experimental_vector_reduce_add>(m_Value(Vec)));
+  };
+  Value *V0, *V1;
+  if (match(Op0, m_AddRdx(V0)) && match(Op1, m_AddRdx(V1)) &&
+      V0->getType() == V1->getType()) {
+    // Difference of sums is sum of differences:
+    // add_rdx(V0) - add_rdx(V1) --> add_rdx(V0 - V1)
+    Value *Sub = Builder.CreateSub(V0, V1);
+    Value *Rdx = Builder.CreateIntrinsic(
+        Intrinsic::experimental_vector_reduce_add, {Sub->getType()}, {Sub});
+    return replaceInstUsesWith(I, Rdx);
+  }
+
   if (Constant *C = dyn_cast<Constant>(Op0)) {
     Value *X;
     if (match(Op1, m_ZExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1))
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 4822d2b097991..836af6234ad5c 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -22,6 +22,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AssumeBundleQueries.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/InstructionSimplify.h"
@@ -42,13 +43,13 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
-#include "llvm/IR/IntrinsicsX86.h"
-#include "llvm/IR/IntrinsicsARM.h"
 #include "llvm/IR/IntrinsicsAArch64.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/IntrinsicsARM.h"
 #include "llvm/IR/IntrinsicsHexagon.h"
 #include "llvm/IR/IntrinsicsNVPTX.h"
-#include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/IntrinsicsPowerPC.h"
+#include "llvm/IR/IntrinsicsX86.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/PatternMatch.h"
@@ -1143,7 +1144,7 @@ Instruction *InstCombiner::simplifyMaskedStore(IntrinsicInst &II) {
   // If the mask is all ones, this is a plain vector store of the 1st argument.
   if (ConstMask->isAllOnesValue()) {
     Value *StorePtr = II.getArgOperand(1);
-    Align Alignment(cast<ConstantInt>(II.getArgOperand(2))->getZExtValue());
+    Align Alignment = cast<ConstantInt>(II.getArgOperand(2))->getAlignValue();
     return new StoreInst(II.getArgOperand(0), StorePtr, false, Alignment);
   }
 
@@ -2378,6 +2379,14 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
       return FAdd;
     }
 
+    // fma x, y, 0 -> fmul x, y
+    // This is always valid for -0.0, but requires nsz for +0.0 as
+    // -0.0 + 0.0 = 0.0, which would not be the same as the fmul on its own.
+    if (match(II->getArgOperand(2), m_NegZeroFP()) ||
+        (match(II->getArgOperand(2), m_PosZeroFP()) &&
+         II->getFastMathFlags().noSignedZeros()))
+      return BinaryOperator::CreateFMulFMF(Src0, Src1, II);
+
     break;
   }
   case Intrinsic::copysign: {
@@ -3374,8 +3383,9 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
   case Intrinsic::arm_neon_vst4lane: {
     Align MemAlign = getKnownAlignment(II->getArgOperand(0), DL, II, &AC, &DT);
     unsigned AlignArg = II->getNumArgOperands() - 1;
-    ConstantInt *IntrAlign = dyn_cast<ConstantInt>(II->getArgOperand(AlignArg));
-    if (IntrAlign && IntrAlign->getZExtValue() < MemAlign.value())
+    Value *AlignArgOp = II->getArgOperand(AlignArg);
+    MaybeAlign Align = cast<ConstantInt>(AlignArgOp)->getMaybeAlignValue();
+    if (Align && *Align < MemAlign)
       return replaceOperand(*II, AlignArg,
                             ConstantInt::get(Type::getInt32Ty(II->getContext()),
                                              MemAlign.value(), false));
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
index 3750f31e3cffb..3639edb5df4d1 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -93,8 +93,8 @@ Instruction *InstCombiner::PromoteCastOfAllocation(BitCastInst &CI,
   Type *CastElTy = PTy->getElementType();
   if (!AllocElTy->isSized() || !CastElTy->isSized()) return nullptr;
 
-  unsigned AllocElTyAlign = DL.getABITypeAlignment(AllocElTy);
-  unsigned CastElTyAlign = DL.getABITypeAlignment(CastElTy);
+  Align AllocElTyAlign = DL.getABITypeAlign(AllocElTy);
+  Align CastElTyAlign = DL.getABITypeAlign(CastElTy);
   if (CastElTyAlign < AllocElTyAlign) return nullptr;
 
   // If the allocation has multiple uses, only promote it if we are strictly
@@ -377,29 +377,31 @@ static bool canEvaluateTruncated(Value *V, Type *Ty, InstCombiner &IC,
     break;
   }
   case Instruction::Shl: {
-    // If we are truncating the result of this SHL, and if it's a shift of a
-    // constant amount, we can always perform a SHL in a smaller type.
-    const APInt *Amt;
-    if (match(I->getOperand(1), m_APInt(Amt))) {
-      uint32_t BitWidth = Ty->getScalarSizeInBits();
-      if (Amt->getLimitedValue(BitWidth) < BitWidth)
-        return canEvaluateTruncated(I->getOperand(0), Ty, IC, CxtI);
-    }
+    // If we are truncating the result of this SHL, and if it's a shift of an
+    // inrange amount, we can always perform a SHL in a smaller type.
+    uint32_t BitWidth = Ty->getScalarSizeInBits();
+    KnownBits AmtKnownBits =
+        llvm::computeKnownBits(I->getOperand(1), IC.getDataLayout());
+    if (AmtKnownBits.getMaxValue().ult(BitWidth))
+      return canEvaluateTruncated(I->getOperand(0), Ty, IC, CxtI) &&
+             canEvaluateTruncated(I->getOperand(1), Ty, IC, CxtI);
     break;
   }
   case Instruction::LShr: {
     // If this is a truncate of a logical shr, we can truncate it to a smaller
     // lshr iff we know that the bits we would otherwise be shifting in are
     // already zeros.
-    const APInt *Amt;
-    if (match(I->getOperand(1), m_APInt(Amt))) {
-      uint32_t OrigBitWidth = OrigTy->getScalarSizeInBits();
-      uint32_t BitWidth = Ty->getScalarSizeInBits();
-      if (Amt->getLimitedValue(BitWidth) < BitWidth &&
-          IC.MaskedValueIsZero(I->getOperand(0),
-            APInt::getBitsSetFrom(OrigBitWidth, BitWidth), 0, CxtI)) {
-        return canEvaluateTruncated(I->getOperand(0), Ty, IC, CxtI);
-      }
+    // TODO: It is enough to check that the bits we would be shifting in are
+    //       zero - use AmtKnownBits.getMaxValue().
+    uint32_t OrigBitWidth = OrigTy->getScalarSizeInBits();
+    uint32_t BitWidth = Ty->getScalarSizeInBits();
+    KnownBits AmtKnownBits =
+        llvm::computeKnownBits(I->getOperand(1), IC.getDataLayout());
+    APInt ShiftedBits = APInt::getBitsSetFrom(OrigBitWidth, BitWidth);
+    if (AmtKnownBits.getMaxValue().ult(BitWidth) &&
+        IC.MaskedValueIsZero(I->getOperand(0), ShiftedBits, 0, CxtI)) {
+      return canEvaluateTruncated(I->getOperand(0), Ty, IC, CxtI) &&
+             canEvaluateTruncated(I->getOperand(1), Ty, IC, CxtI);
     }
     break;
   }
@@ -409,15 +411,15 @@ static bool canEvaluateTruncated(Value *V, Type *Ty, InstCombiner &IC,
     // original type and the sign bit of the truncate type are similar.
     // TODO: It is enough to check that the bits we would be shifting in are
     //       similar to sign bit of the truncate type.
-    const APInt *Amt;
-    if (match(I->getOperand(1), m_APInt(Amt))) {
-      uint32_t OrigBitWidth = OrigTy->getScalarSizeInBits();
-      uint32_t BitWidth = Ty->getScalarSizeInBits();
-      if (Amt->getLimitedValue(BitWidth) < BitWidth &&
-          OrigBitWidth - BitWidth <
-              IC.ComputeNumSignBits(I->getOperand(0), 0, CxtI))
-        return canEvaluateTruncated(I->getOperand(0), Ty, IC, CxtI);
-    }
+    uint32_t OrigBitWidth = OrigTy->getScalarSizeInBits();
+    uint32_t BitWidth = Ty->getScalarSizeInBits();
+    KnownBits AmtKnownBits =
+        llvm::computeKnownBits(I->getOperand(1), IC.getDataLayout());
+    unsigned ShiftedBits = OrigBitWidth - BitWidth;
+    if (AmtKnownBits.getMaxValue().ult(BitWidth) &&
+        ShiftedBits < IC.ComputeNumSignBits(I->getOperand(0), 0, CxtI))
+      return canEvaluateTruncated(I->getOperand(0), Ty, IC, CxtI) &&
+             canEvaluateTruncated(I->getOperand(1), Ty, IC, CxtI);
     break;
   }
   case Instruction::Trunc:
@@ -713,6 +715,25 @@ Instruction *InstCombiner::visitTrunc(TruncInst &Trunc) {
     return replaceInstUsesWith(Trunc, Res);
   }
 
+  // For integer types, check if we can shorten the entire input expression to
+  // DestWidth * 2, which won't allow removing the truncate, but reducing the
+  // width may enable further optimizations, e.g. allowing for larger
+  // vectorization factors.
+  if (auto *DestITy = dyn_cast<IntegerType>(DestTy)) {
+    if (DestWidth * 2 < SrcWidth) {
+      auto *NewDestTy = DestITy->getExtendedType();
+      if (shouldChangeType(SrcTy, NewDestTy) &&
+          canEvaluateTruncated(Src, NewDestTy, *this, &Trunc)) {
+        LLVM_DEBUG(
+            dbgs() << "ICE: EvaluateInDifferentType converting expression type"
+                      " to reduce the width of operand of"
+                   << Trunc << '\n');
+        Value *Res = EvaluateInDifferentType(Src, NewDestTy, false);
+        return new TruncInst(Res, DestTy);
+      }
+    }
+  }
+
   // Test if the trunc is the user of a select which is part of a
   // minimum or maximum operation. If so, don't do any more simplification.
   // Even simplifying demanded bits can break the canonical form of a
@@ -742,18 +763,21 @@ Instruction *InstCombiner::visitTrunc(TruncInst &Trunc) {
     // For vectors, we do not canonicalize all truncs to icmp, so optimize
     // patterns that would be covered within visitICmpInst.
     Value *X;
-    const APInt *C;
-    if (match(Src, m_OneUse(m_LShr(m_Value(X), m_APInt(C))))) {
+    Constant *C;
+    if (match(Src, m_OneUse(m_LShr(m_Value(X), m_Constant(C))))) {
       // trunc (lshr X, C) to i1 --> icmp ne (and X, C'), 0
-      APInt MaskC = APInt(SrcWidth, 1).shl(*C);
-      Value *And = Builder.CreateAnd(X, ConstantInt::get(SrcTy, MaskC));
+      Constant *One = ConstantInt::get(SrcTy, APInt(SrcWidth, 1));
+      Constant *MaskC = ConstantExpr::getShl(One, C);
+      Value *And = Builder.CreateAnd(X, MaskC);
       return new ICmpInst(ICmpInst::ICMP_NE, And, Zero);
     }
-    if (match(Src, m_OneUse(m_c_Or(m_LShr(m_Value(X), m_APInt(C)),
+    if (match(Src, m_OneUse(m_c_Or(m_LShr(m_Value(X), m_Constant(C)),
                                    m_Deferred(X))))) {
       // trunc (or (lshr X, C), X) to i1 --> icmp ne (and X, C'), 0
-      APInt MaskC = APInt(SrcWidth, 1).shl(*C) | 1;
-      Value *And = Builder.CreateAnd(X, ConstantInt::get(SrcTy, MaskC));
+      Constant *One = ConstantInt::get(SrcTy, APInt(SrcWidth, 1));
+      Constant *MaskC = ConstantExpr::getShl(One, C);
+      MaskC = ConstantExpr::getOr(MaskC, One);
+      Value *And = Builder.CreateAnd(X, MaskC);
       return new ICmpInst(ICmpInst::ICMP_NE, And, Zero);
     }
   }
@@ -1457,16 +1481,17 @@ Instruction *InstCombiner::visitSExt(SExtInst &CI) {
   //   %d = ashr i32 %a, 30
   Value *A = nullptr;
   // TODO: Eventually this could be subsumed by EvaluateInDifferentType.
-  ConstantInt *BA = nullptr, *CA = nullptr;
-  if (match(Src, m_AShr(m_Shl(m_Trunc(m_Value(A)), m_ConstantInt(BA)),
-                        m_ConstantInt(CA))) &&
+  Constant *BA = nullptr, *CA = nullptr;
+  if (match(Src, m_AShr(m_Shl(m_Trunc(m_Value(A)), m_Constant(BA)),
+                        m_Constant(CA))) &&
       BA == CA && A->getType() == CI.getType()) {
     unsigned MidSize = Src->getType()->getScalarSizeInBits();
     unsigned SrcDstSize = CI.getType()->getScalarSizeInBits();
-    unsigned ShAmt = CA->getZExtValue()+SrcDstSize-MidSize;
-    Constant *ShAmtV = ConstantInt::get(CI.getType(), ShAmt);
-    A = Builder.CreateShl(A, ShAmtV, CI.getName());
-    return BinaryOperator::CreateAShr(A, ShAmtV);
+    Constant *SizeDiff = ConstantInt::get(CA->getType(), SrcDstSize - MidSize);
+    Constant *ShAmt = ConstantExpr::getAdd(CA, SizeDiff);
+    Constant *ShAmtExt = ConstantExpr::getSExt(ShAmt, CI.getType());
+    A = Builder.CreateShl(A, ShAmtExt, CI.getName());
+    return BinaryOperator::CreateAShr(A, ShAmtExt);
   }
 
   return nullptr;
@@ -2471,8 +2496,9 @@ Instruction *InstCombiner::visitBitCast(BitCastInst &CI) {
   if (DestTy == Src->getType())
     return replaceInstUsesWith(CI, Src);
 
-  if (PointerType *DstPTy = dyn_cast<PointerType>(DestTy)) {
+  if (isa<PointerType>(SrcTy) && isa<PointerType>(DestTy)) {
     PointerType *SrcPTy = cast<PointerType>(SrcTy);
+    PointerType *DstPTy = cast<PointerType>(DestTy);
     Type *DstElTy = DstPTy->getElementType();
     Type *SrcElTy = SrcPTy->getElementType();
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 53a3af03e46f2..f1233b62445d0 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -4644,17 +4644,6 @@ static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal,
   switch (I.getPredicate()) {
   case ICmpInst::ICMP_EQ:
   case ICmpInst::ICMP_NE:
-    // Recognize pattern:
-    //   mulval = mul(zext A, zext B)
-    //   cmp eq/neq mulval, zext trunc mulval
-    if (ZExtInst *Zext = dyn_cast<ZExtInst>(OtherVal))
-      if (Zext->hasOneUse()) {
-        Value *ZextArg = Zext->getOperand(0);
-        if (TruncInst *Trunc = dyn_cast<TruncInst>(ZextArg))
-          if (Trunc->getType()->getPrimitiveSizeInBits() == MulWidth)
-            break; //Recognized
-      }
-
     // Recognize pattern:
     //   mulval = mul(zext A, zext B)
     //   cmp eq/neq mulval, and(mulval, mask), mask selects low MulWidth bits.
@@ -5283,6 +5272,47 @@ static ICmpInst *canonicalizeCmpWithConstant(ICmpInst &I) {
   return new ICmpInst(FlippedStrictness->first, Op0, FlippedStrictness->second);
 }
 
+/// If we have a comparison with a non-canonical predicate, if we can update
+/// all the users, invert the predicate and adjust all the users.
+static CmpInst *canonicalizeICmpPredicate(CmpInst &I) {
+  // Is the predicate already canonical?
+  CmpInst::Predicate Pred = I.getPredicate();
+  if (isCanonicalPredicate(Pred))
+    return nullptr;
+
+  // Can all users be adjusted to predicate inversion?
+  if (!canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr))
+    return nullptr;
+
+  // Ok, we can canonicalize comparison!
+  // Let's first invert the comparison's predicate.
+  I.setPredicate(CmpInst::getInversePredicate(Pred));
+  I.setName(I.getName() + ".not");
+
+  // And now let's adjust every user.
+  for (User *U : I.users()) {
+    switch (cast<Instruction>(U)->getOpcode()) {
+    case Instruction::Select: {
+      auto *SI = cast<SelectInst>(U);
+      SI->swapValues();
+      SI->swapProfMetadata();
+      break;
+    }
+    case Instruction::Br:
+      cast<BranchInst>(U)->swapSuccessors(); // swaps prof metadata too
+      break;
+    case Instruction::Xor:
+      U->replaceAllUsesWith(&I);
+      break;
+    default:
+      llvm_unreachable("Got unexpected user - out of sync with "
+                       "canFreelyInvertAllUsersOf() ?");
+    }
+  }
+
+  return &I;
+}
+
 /// Integer compare with boolean values can always be turned into bitwise ops.
 static Instruction *canonicalizeICmpBool(ICmpInst &I,
                                          InstCombiner::BuilderTy &Builder) {
@@ -5521,8 +5551,11 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
     if (Instruction *Res = canonicalizeICmpBool(I, Builder))
       return Res;
 
-  if (ICmpInst *NewICmp = canonicalizeCmpWithConstant(I))
-    return NewICmp;
+  if (Instruction *Res = canonicalizeCmpWithConstant(I))
+    return Res;
+
+  if (Instruction *Res = canonicalizeICmpPredicate(I))
+    return Res;
 
   if (Instruction *Res = foldICmpWithConstant(I))
     return Res;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 2ad9c7396f93f..dd2f59be08e92 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -18,7 +18,6 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/Statistic.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/TargetFolder.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -52,6 +51,7 @@ using namespace llvm::PatternMatch;
 
 namespace llvm {
 
+class AAResults;
 class APInt;
 class AssumptionCache;
 class BlockFrequencyInfo;
@@ -215,18 +215,23 @@ static inline bool isFreeToInvert(Value *V, bool WillInvertAllUses) {
 }
 
 /// Given i1 V, can every user of V be freely adapted if V is changed to !V ?
+/// InstCombine's canonicalizeICmpPredicate() must be kept in sync with this fn.
 ///
 /// See also: isFreeToInvert()
 static inline bool canFreelyInvertAllUsersOf(Value *V, Value *IgnoredUser) {
   // Look at every user of V.
-  for (User *U : V->users()) {
-    if (U == IgnoredUser)
+  for (Use &U : V->uses()) {
+    if (U.getUser() == IgnoredUser)
       continue; // Don't consider this user.
 
-    auto *I = cast<Instruction>(U);
+    auto *I = cast<Instruction>(U.getUser());
     switch (I->getOpcode()) {
     case Instruction::Select:
+      if (U.getOperandNo() != 0) // Only if the value is used as select cond.
+        return false;
+      break;
     case Instruction::Br:
+      assert(U.getOperandNo() == 0 && "Must be branching on that value.");
       break; // Free to invert by swapping true/false values/destinations.
     case Instruction::Xor: // Can invert 'xor' if it's a 'not', by ignoring it.
       if (!match(I, m_Not(m_Value())))
@@ -316,7 +321,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
   // Mode in which we are running the combiner.
   const bool MinimizeSize;
 
-  AliasAnalysis *AA;
+  AAResults *AA;
 
   // Required analyses.
   AssumptionCache &AC;
@@ -336,7 +341,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
 
 public:
   InstCombiner(InstCombineWorklist &Worklist, BuilderTy &Builder,
-               bool MinimizeSize, AliasAnalysis *AA,
+               bool MinimizeSize, AAResults *AA,
                AssumptionCache &AC, TargetLibraryInfo &TLI, DominatorTree &DT,
                OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI,
                ProfileSummaryInfo *PSI, const DataLayout &DL, LoopInfo *LI)
@@ -635,6 +640,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
   Value *getSelectCondition(Value *A, Value *B);
 
   Instruction *foldIntrinsicWithOverflowCommon(IntrinsicInst *II);
+  Instruction *foldFPSignBitOps(BinaryOperator &I);
 
 public:
   /// Inserts an instruction \p New before instruction \p Old
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index afb4b984b112b..7203850ad24d6 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -14,8 +14,8 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/Loads.h"
-#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfoMetadata.h"
@@ -24,6 +24,7 @@
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Local.h"
 using namespace llvm;
 using namespace PatternMatch;
 
@@ -40,7 +41,7 @@ STATISTIC(NumGlobalCopies, "Number of allocas copied from constant global");
 /// the alloca, and if the source pointer is a pointer to a constant global, we
 /// can optimize this.
 static bool
-isOnlyCopiedFromConstantMemory(AliasAnalysis *AA,
+isOnlyCopiedFromConstantMemory(AAResults *AA,
                                Value *V, MemTransferInst *&TheCopy,
                                SmallVectorImpl<Instruction *> &ToDelete) {
   // We track lifetime intrinsics as we encounter them.  If we decide to go
@@ -144,7 +145,7 @@ isOnlyCopiedFromConstantMemory(AliasAnalysis *AA,
 /// modified by a copy from a constant global.  If we can prove this, we can
 /// replace any uses of the alloca with uses of the global directly.
 static MemTransferInst *
-isOnlyCopiedFromConstantMemory(AliasAnalysis *AA,
+isOnlyCopiedFromConstantMemory(AAResults *AA,
                                AllocaInst *AI,
                                SmallVectorImpl<Instruction *> &ToDelete) {
   MemTransferInst *TheCopy = nullptr;
@@ -1424,18 +1425,33 @@ Instruction *InstCombiner::visitStoreInst(StoreInst &SI) {
   if (isa<UndefValue>(Val))
     return eraseInstFromFunction(SI);
 
+  auto IsNoopInstrForStoreMerging = [](BasicBlock::iterator BBI) {
+    return isa<DbgInfoIntrinsic>(BBI) ||
+           (isa<BitCastInst>(BBI) && BBI->getType()->isPointerTy());
+  };
+
   // If this store is the second-to-last instruction in the basic block
   // (excluding debug info and bitcasts of pointers) and if the block ends with
   // an unconditional branch, try to move the store to the successor block.
   BBI = SI.getIterator();
   do {
     ++BBI;
-  } while (isa<DbgInfoIntrinsic>(BBI) ||
-           (isa<BitCastInst>(BBI) && BBI->getType()->isPointerTy()));
+  } while (IsNoopInstrForStoreMerging(BBI));
 
   if (BranchInst *BI = dyn_cast<BranchInst>(BBI))
     if (BI->isUnconditional())
-      mergeStoreIntoSuccessor(SI);
+      if (mergeStoreIntoSuccessor(SI)) {
+        // Okay, we've managed to do that. Now, let's see if now-second-to-last
+        // instruction is also a store that we can also sink.
+        BasicBlock::iterator FirstInstr = BBI->getParent()->begin();
+        do {
+          if (BBI != FirstInstr)
+            --BBI;
+        } while (BBI != FirstInstr && IsNoopInstrForStoreMerging(BBI));
+        if (StoreInst *PrevStore = dyn_cast<StoreInst>(BBI))
+          Worklist.add(PrevStore);
+        return nullptr;
+      }
 
   return nullptr;
 }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index cbbde9a1aeb82..c6233a68847dd 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -366,6 +366,27 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) {
     }
   }
 
+  // (zext bool X) * (zext bool Y) --> zext (and X, Y)
+  // (sext bool X) * (sext bool Y) --> zext (and X, Y)
+  // Note: -1 * -1 == 1 * 1 == 1 (if the extends match, the result is the same)
+  if (((match(Op0, m_ZExt(m_Value(X))) && match(Op1, m_ZExt(m_Value(Y)))) ||
+       (match(Op0, m_SExt(m_Value(X))) && match(Op1, m_SExt(m_Value(Y))))) &&
+      X->getType()->isIntOrIntVectorTy(1) && X->getType() == Y->getType() &&
+      (Op0->hasOneUse() || Op1->hasOneUse())) {
+    Value *And = Builder.CreateAnd(X, Y, "mulbool");
+    return CastInst::Create(Instruction::ZExt, And, I.getType());
+  }
+  // (sext bool X) * (zext bool Y) --> sext (and X, Y)
+  // (zext bool X) * (sext bool Y) --> sext (and X, Y)
+  // Note: -1 * 1 == 1 * -1  == -1
+  if (((match(Op0, m_SExt(m_Value(X))) && match(Op1, m_ZExt(m_Value(Y)))) ||
+       (match(Op0, m_ZExt(m_Value(X))) && match(Op1, m_SExt(m_Value(Y))))) &&
+      X->getType()->isIntOrIntVectorTy(1) && X->getType() == Y->getType() &&
+      (Op0->hasOneUse() || Op1->hasOneUse())) {
+    Value *And = Builder.CreateAnd(X, Y, "mulbool");
+    return CastInst::Create(Instruction::SExt, And, I.getType());
+  }
+
   // (bool X) * Y --> X ? Y : 0
   // Y * (bool X) --> X ? Y : 0
   if (match(Op0, m_ZExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1))
@@ -402,7 +423,7 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) {
   return Changed ? &I : nullptr;
 }
 
-static Instruction *foldFPSignBitOps(BinaryOperator &I) {
+Instruction *InstCombiner::foldFPSignBitOps(BinaryOperator &I) {
   BinaryOperator::BinaryOps Opcode = I.getOpcode();
   assert((Opcode == Instruction::FMul || Opcode == Instruction::FDiv) &&
          "Expected fmul or fdiv");
@@ -420,6 +441,19 @@ static Instruction *foldFPSignBitOps(BinaryOperator &I) {
   if (Op0 == Op1 && match(Op0, m_Intrinsic<Intrinsic::fabs>(m_Value(X))))
     return BinaryOperator::CreateWithCopiedFlags(Opcode, X, X, &I);
 
+  // fabs(X) * fabs(Y) --> fabs(X * Y)
+  // fabs(X) / fabs(Y) --> fabs(X / Y)
+  if (match(Op0, m_Intrinsic<Intrinsic::fabs>(m_Value(X))) &&
+      match(Op1, m_Intrinsic<Intrinsic::fabs>(m_Value(Y))) &&
+      (Op0->hasOneUse() || Op1->hasOneUse())) {
+    IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
+    Builder.setFastMathFlags(I.getFastMathFlags());
+    Value *XY = Builder.CreateBinOp(Opcode, X, Y);
+    Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, XY);
+    Fabs->takeName(&I);
+    return replaceInstUsesWith(I, Fabs);
+  }
+
   return nullptr;
 }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index f406eb40e16b0..17124f717af79 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -1930,7 +1930,7 @@ Instruction *InstCombiner::foldSelectExtConst(SelectInst &Sel) {
   Type *SelType = Sel.getType();
   Constant *TruncC = ConstantExpr::getTrunc(C, SmallType);
   Constant *ExtC = ConstantExpr::getCast(ExtOpcode, TruncC, SelType);
-  if (ExtC == C) {
+  if (ExtC == C && ExtInst->hasOneUse()) {
     Value *TruncCVal = cast<Value>(TruncC);
     if (ExtInst == Sel.getFalseValue())
       std::swap(X, TruncCVal);
@@ -2443,6 +2443,78 @@ Instruction *InstCombiner::foldVectorSelect(SelectInst &Sel) {
   return nullptr;
 }
 
+static Instruction *foldSelectToPhiImpl(SelectInst &Sel, BasicBlock *BB,
+                                        const DominatorTree &DT,
+                                        InstCombiner::BuilderTy &Builder) {
+  // Find the block's immediate dominator that ends with a conditional branch
+  // that matches select's condition (maybe inverted).
+  auto *IDomNode = DT[BB]->getIDom();
+  if (!IDomNode)
+    return nullptr;
+  BasicBlock *IDom = IDomNode->getBlock();
+
+  Value *Cond = Sel.getCondition();
+  Value *IfTrue, *IfFalse;
+  BasicBlock *TrueSucc, *FalseSucc;
+  if (match(IDom->getTerminator(),
+            m_Br(m_Specific(Cond), m_BasicBlock(TrueSucc),
+                 m_BasicBlock(FalseSucc)))) {
+    IfTrue = Sel.getTrueValue();
+    IfFalse = Sel.getFalseValue();
+  } else if (match(IDom->getTerminator(),
+                   m_Br(m_Not(m_Specific(Cond)), m_BasicBlock(TrueSucc),
+                        m_BasicBlock(FalseSucc)))) {
+    IfTrue = Sel.getFalseValue();
+    IfFalse = Sel.getTrueValue();
+  } else
+    return nullptr;
+
+  // We want to replace select %cond, %a, %b with a phi that takes value %a
+  // for all incoming edges that are dominated by condition `%cond == true`,
+  // and value %b for edges dominated by condition `%cond == false`. If %a
+  // or %b are also phis from the same basic block, we can go further and take
+  // their incoming values from the corresponding blocks.
+  BasicBlockEdge TrueEdge(IDom, TrueSucc);
+  BasicBlockEdge FalseEdge(IDom, FalseSucc);
+  DenseMap<BasicBlock *, Value *> Inputs;
+  for (auto *Pred : predecessors(BB)) {
+    // Check implication.
+    BasicBlockEdge Incoming(Pred, BB);
+    if (DT.dominates(TrueEdge, Incoming))
+      Inputs[Pred] = IfTrue->DoPHITranslation(BB, Pred);
+    else if (DT.dominates(FalseEdge, Incoming))
+      Inputs[Pred] = IfFalse->DoPHITranslation(BB, Pred);
+    else
+      return nullptr;
+    // Check availability.
+    if (auto *Insn = dyn_cast<Instruction>(Inputs[Pred]))
+      if (!DT.dominates(Insn, Pred->getTerminator()))
+        return nullptr;
+  }
+
+  Builder.SetInsertPoint(&*BB->begin());
+  auto *PN = Builder.CreatePHI(Sel.getType(), Inputs.size());
+  for (auto *Pred : predecessors(BB))
+    PN->addIncoming(Inputs[Pred], Pred);
+  PN->takeName(&Sel);
+  return PN;
+}
+
+static Instruction *foldSelectToPhi(SelectInst &Sel, const DominatorTree &DT,
+                                    InstCombiner::BuilderTy &Builder) {
+  // Try to replace this select with Phi in one of these blocks.
+  SmallSetVector<BasicBlock *, 4> CandidateBlocks;
+  CandidateBlocks.insert(Sel.getParent());
+  for (Value *V : Sel.operands())
+    if (auto *I = dyn_cast<Instruction>(V))
+      CandidateBlocks.insert(I->getParent());
+
+  for (BasicBlock *BB : CandidateBlocks)
+    if (auto *PN = foldSelectToPhiImpl(Sel, BB, DT, Builder))
+      return PN;
+  return nullptr;
+}
+
 Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
   Value *CondVal = SI.getCondition();
   Value *TrueVal = SI.getTrueValue();
@@ -2475,21 +2547,7 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
   if (Instruction *I = canonicalizeScalarSelectOfVecs(SI, *this))
     return I;
 
-  // Canonicalize a one-use integer compare with a non-canonical predicate by
-  // inverting the predicate and swapping the select operands. This matches a
-  // compare canonicalization for conditional branches.
-  // TODO: Should we do the same for FP compares?
   CmpInst::Predicate Pred;
-  if (match(CondVal, m_OneUse(m_ICmp(Pred, m_Value(), m_Value()))) &&
-      !isCanonicalPredicate(Pred)) {
-    // Swap true/false values and condition.
-    CmpInst *Cond = cast<CmpInst>(CondVal);
-    Cond->setPredicate(CmpInst::getInversePredicate(Pred));
-    SI.swapValues();
-    SI.swapProfMetadata();
-    Worklist.push(Cond);
-    return &SI;
-  }
 
   if (SelType->isIntOrIntVectorTy(1) &&
       TrueVal->getType() == CondVal->getType()) {
@@ -2904,5 +2962,8 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
   if (Instruction *Copysign = foldSelectToCopysign(SI, Builder))
     return Copysign;
 
+  if (Instruction *PN = foldSelectToPhi(SI, DT, Builder))
+    return replaceInstUsesWith(SI, PN);
+
   return nullptr;
 }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index 49695432252d8..7cfe4c8b5892b 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -1130,17 +1130,11 @@ Value *InstCombiner::simplifyAMDGCNMemoryIntrinsicDemanded(IntrinsicInst *II,
     return nullptr;
   }
 
-  // Determine the overload types of the original intrinsic.
-  auto IID = II->getIntrinsicID();
-  SmallVector<Intrinsic::IITDescriptor, 16> Table;
-  getIntrinsicInfoTableEntries(IID, Table);
-  ArrayRef<Intrinsic::IITDescriptor> TableRef = Table;
-
   // Validate function argument and return types, extracting overloaded types
   // along the way.
-  FunctionType *FTy = II->getCalledFunction()->getFunctionType();
   SmallVector<Type *, 6> OverloadTys;
-  Intrinsic::matchIntrinsicSignature(FTy, TableRef, OverloadTys);
+  if (!Intrinsic::getIntrinsicSignature(II->getCalledFunction(), OverloadTys))
+    return nullptr;
 
   Module *M = II->getParent()->getParent()->getParent();
   Type *EltTy = IIVTy->getElementType();
@@ -1148,7 +1142,8 @@ Value *InstCombiner::simplifyAMDGCNMemoryIntrinsicDemanded(IntrinsicInst *II,
       (NewNumElts == 1) ? EltTy : FixedVectorType::get(EltTy, NewNumElts);
 
   OverloadTys[0] = NewTy;
-  Function *NewIntrin = Intrinsic::getDeclaration(M, IID, OverloadTys);
+  Function *NewIntrin =
+      Intrinsic::getDeclaration(M, II->getIntrinsicID(), OverloadTys);
 
   CallInst *NewCall = Builder.CreateCall(NewIntrin, Args);
   NewCall->takeName(II);
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 1f97f0c1ac997..d1c1e54188251 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -971,7 +971,7 @@ static Value *foldOperationIntoPhiValue(BinaryOperator *I, Value *InV,
   if (!ConstIsRHS)
     std::swap(Op0, Op1);
 
-  Value *RI = Builder.CreateBinOp(I->getOpcode(), Op0, Op1, "phitmp");
+  Value *RI = Builder.CreateBinOp(I->getOpcode(), Op0, Op1, "phi.bo");
   auto *FPInst = dyn_cast<Instruction>(RI);
   if (FPInst && isa<FPMathOperator>(FPInst))
     FPInst->copyFastMathFlags(I);
@@ -1076,7 +1076,7 @@ Instruction *InstCombiner::foldOpIntoPhi(Instruction &I, PHINode *PN) {
         // the select would be generated exactly once in the NonConstBB.
         Builder.SetInsertPoint(ThisBB->getTerminator());
         InV = Builder.CreateSelect(PN->getIncomingValue(i), TrueVInPred,
-                                   FalseVInPred, "phitmp");
+                                   FalseVInPred, "phi.sel");
       }
       NewPN->addIncoming(InV, ThisBB);
     }
@@ -1088,7 +1088,7 @@ Instruction *InstCombiner::foldOpIntoPhi(Instruction &I, PHINode *PN) {
         InV = ConstantExpr::getCompare(CI->getPredicate(), InC, C);
       else
         InV = Builder.CreateCmp(CI->getPredicate(), PN->getIncomingValue(i),
-                                C, "phitmp");
+                                C, "phi.cmp");
       NewPN->addIncoming(InV, PN->getIncomingBlock(i));
     }
   } else if (auto *BO = dyn_cast<BinaryOperator>(&I)) {
@@ -1106,7 +1106,7 @@ Instruction *InstCombiner::foldOpIntoPhi(Instruction &I, PHINode *PN) {
         InV = ConstantExpr::getCast(CI->getOpcode(), InC, RetTy);
       else
         InV = Builder.CreateCast(CI->getOpcode(), PN->getIncomingValue(i),
-                                 I.getType(), "phitmp");
+                                 I.getType(), "phi.cast");
       NewPN->addIncoming(InV, PN->getIncomingBlock(i));
     }
   }
@@ -2480,6 +2480,7 @@ static bool isAllocSiteRemovable(Instruction *AI,
               return false;
             LLVM_FALLTHROUGH;
           }
+          case Intrinsic::assume:
           case Intrinsic::invariant_start:
           case Intrinsic::invariant_end:
           case Intrinsic::lifetime_start:
@@ -2754,9 +2755,9 @@ Instruction *InstCombiner::visitBranchInst(BranchInst &BI) {
     return replaceOperand(
         BI, 0, ConstantInt::getFalse(BI.getCondition()->getType()));
 
-  // Canonicalize, for example, icmp_ne -> icmp_eq or fcmp_one -> fcmp_oeq.
+  // Canonicalize, for example, fcmp_one -> fcmp_oeq.
   CmpInst::Predicate Pred;
-  if (match(&BI, m_Br(m_OneUse(m_Cmp(Pred, m_Value(), m_Value())),
+  if (match(&BI, m_Br(m_OneUse(m_FCmp(Pred, m_Value(), m_Value())),
                       m_BasicBlock(), m_BasicBlock())) &&
       !isCanonicalPredicate(Pred)) {
     // Swap destinations and condition.
@@ -3355,6 +3356,12 @@ static bool TryToSinkInstruction(Instruction *I, BasicBlock *DestBlock) {
   I->moveBefore(&*InsertPos);
   ++NumSunkInst;
 
+  // Drop the debug loc of non-inlinable instructions. This prevents
+  // single-stepping from going backwards. See HowToUpdateDebugInfo.rst for
+  // the full rationale.
+  if (!isa<CallBase>(I))
+    I->setDebugLoc(DebugLoc());
+
   // Also sink all related debug uses from the source basic block. Otherwise we
   // get debug use before the def. Attempt to salvage debug uses first, to
   // maximise the range variables have location for. If we cannot salvage, then
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index f2f71f4cef2d6..ee09a4d9db7e1 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -10,6 +10,8 @@
 // Details of the algorithm:
 //  https://github.com/google/sanitizers/wiki/AddressSanitizerAlgorithm
 //
+// FIXME: This sanitizer does not yet handle scalable vectors
+//
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
@@ -693,7 +695,6 @@ struct AddressSanitizer {
   FunctionCallee AsanMemoryAccessCallbackSized[2][2];
 
   FunctionCallee AsanMemmove, AsanMemcpy, AsanMemset;
-  InlineAsm *EmptyAsm;
   Value *LocalDynamicShadow = nullptr;
   const GlobalsMetadata &GlobalsMD;
   DenseMap<const AllocaInst *, bool> ProcessedAllocas;
@@ -911,16 +912,14 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
   using AllocaForValueMapTy = DenseMap<Value *, AllocaInst *>;
   AllocaForValueMapTy AllocaForValue;
 
-  bool HasNonEmptyInlineAsm = false;
+  bool HasInlineAsm = false;
   bool HasReturnsTwiceCall = false;
-  std::unique_ptr<CallInst> EmptyInlineAsm;
 
   FunctionStackPoisoner(Function &F, AddressSanitizer &ASan)
       : F(F), ASan(ASan), DIB(*F.getParent(), /*AllowUnresolved*/ false),
         C(ASan.C), IntptrTy(ASan.IntptrTy),
         IntptrPtrTy(PointerType::get(IntptrTy, 0)), Mapping(ASan.Mapping),
-        StackAlignment(1 << Mapping.Scale),
-        EmptyInlineAsm(CallInst::Create(ASan.EmptyAsm)) {}
+        StackAlignment(1 << Mapping.Scale) {}
 
   bool runOnFunction() {
     if (!ClStack) return false;
@@ -1082,9 +1081,7 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
 
   void visitCallBase(CallBase &CB) {
     if (CallInst *CI = dyn_cast<CallInst>(&CB)) {
-      HasNonEmptyInlineAsm |= CI->isInlineAsm() &&
-                              !CI->isIdenticalTo(EmptyInlineAsm.get()) &&
-                              &CB != ASan.LocalDynamicShadow;
+      HasInlineAsm |= CI->isInlineAsm() && &CB != ASan.LocalDynamicShadow;
       HasReturnsTwiceCall |= CI->canReturnTwice();
     }
   }
@@ -1381,23 +1378,22 @@ void AddressSanitizer::getInterestingMemoryOperands(
     if (!ClInstrumentReads || ignoreAccess(LI->getPointerOperand()))
       return;
     Interesting.emplace_back(I, LI->getPointerOperandIndex(), false,
-                             LI->getType(), LI->getAlignment());
+                             LI->getType(), LI->getAlign());
   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
     if (!ClInstrumentWrites || ignoreAccess(SI->getPointerOperand()))
       return;
     Interesting.emplace_back(I, SI->getPointerOperandIndex(), true,
-                             SI->getValueOperand()->getType(),
-                             SI->getAlignment());
+                             SI->getValueOperand()->getType(), SI->getAlign());
   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
     if (!ClInstrumentAtomics || ignoreAccess(RMW->getPointerOperand()))
       return;
     Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
-                             RMW->getValOperand()->getType(), 0);
+                             RMW->getValOperand()->getType(), None);
   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
     if (!ClInstrumentAtomics || ignoreAccess(XCHG->getPointerOperand()))
       return;
     Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
-                             XCHG->getCompareOperand()->getType(), 0);
+                             XCHG->getCompareOperand()->getType(), None);
   } else if (auto CI = dyn_cast<CallInst>(I)) {
     auto *F = CI->getCalledFunction();
     if (F && (F->getName().startswith("llvm.masked.load.") ||
@@ -1412,11 +1408,10 @@ void AddressSanitizer::getInterestingMemoryOperands(
       if (ignoreAccess(BasePtr))
         return;
       auto Ty = cast<PointerType>(BasePtr->getType())->getElementType();
-      unsigned Alignment = 1;
+      MaybeAlign Alignment = Align(1);
       // Otherwise no alignment guarantees. We probably got Undef.
-      if (auto AlignmentConstant =
-              dyn_cast<ConstantInt>(CI->getOperand(1 + OpOffset)))
-        Alignment = (unsigned)AlignmentConstant->getZExtValue();
+      if (auto *Op = dyn_cast<ConstantInt>(CI->getOperand(1 + OpOffset)))
+        Alignment = Op->getMaybeAlignValue();
       Value *Mask = CI->getOperand(2 + OpOffset);
       Interesting.emplace_back(I, OpOffset, IsWrite, Ty, Alignment, Mask);
     } else {
@@ -1425,7 +1420,7 @@ void AddressSanitizer::getInterestingMemoryOperands(
             ignoreAccess(CI->getArgOperand(ArgNo)))
           continue;
         Type *Ty = CI->getParamByValType(ArgNo);
-        Interesting.emplace_back(I, ArgNo, false, Ty, 1);
+        Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
       }
     }
   }
@@ -1487,7 +1482,7 @@ void AddressSanitizer::instrumentPointerComparisonOrSubtraction(
 
 static void doInstrumentAddress(AddressSanitizer *Pass, Instruction *I,
                                 Instruction *InsertBefore, Value *Addr,
-                                unsigned Alignment, unsigned Granularity,
+                                MaybeAlign Alignment, unsigned Granularity,
                                 uint32_t TypeSize, bool IsWrite,
                                 Value *SizeArgument, bool UseCalls,
                                 uint32_t Exp) {
@@ -1495,7 +1490,7 @@ static void doInstrumentAddress(AddressSanitizer *Pass, Instruction *I,
   // if the data is properly aligned.
   if ((TypeSize == 8 || TypeSize == 16 || TypeSize == 32 || TypeSize == 64 ||
        TypeSize == 128) &&
-      (Alignment >= Granularity || Alignment == 0 || Alignment >= TypeSize / 8))
+      (!Alignment || *Alignment >= Granularity || *Alignment >= TypeSize / 8))
     return Pass->instrumentAddress(I, InsertBefore, Addr, TypeSize, IsWrite,
                                    nullptr, UseCalls, Exp);
   Pass->instrumentUnusualSizeOrAlignment(I, InsertBefore, Addr, TypeSize,
@@ -1505,12 +1500,12 @@ static void doInstrumentAddress(AddressSanitizer *Pass, Instruction *I,
 static void instrumentMaskedLoadOrStore(AddressSanitizer *Pass,
                                         const DataLayout &DL, Type *IntptrTy,
                                         Value *Mask, Instruction *I,
-                                        Value *Addr, unsigned Alignment,
+                                        Value *Addr, MaybeAlign Alignment,
                                         unsigned Granularity, uint32_t TypeSize,
                                         bool IsWrite, Value *SizeArgument,
                                         bool UseCalls, uint32_t Exp) {
-  auto *VTy =
-      cast<VectorType>(cast<PointerType>(Addr->getType())->getElementType());
+  auto *VTy = cast<FixedVectorType>(
+      cast<PointerType>(Addr->getType())->getElementType());
   uint64_t ElemTypeSize = DL.getTypeStoreSizeInBits(VTy->getScalarType());
   unsigned Num = VTy->getNumElements();
   auto Zero = ConstantInt::get(IntptrTy, 0);
@@ -1621,10 +1616,7 @@ Instruction *AddressSanitizer::generateCrashCode(Instruction *InsertBefore,
                             {Addr, ExpVal});
   }
 
-  // We don't do Call->setDoesNotReturn() because the BB already has
-  // UnreachableInst at the end.
-  // This EmptyAsm is required to avoid callback merge.
-  IRB.CreateCall(EmptyAsm, {});
+  Call->setCannotMerge();
   return Call;
 }
 
@@ -2598,10 +2590,6 @@ void AddressSanitizer::initializeCallbacks(Module &M) {
       M.getOrInsertFunction(kAsanPtrCmp, IRB.getVoidTy(), IntptrTy, IntptrTy);
   AsanPtrSubFunction =
       M.getOrInsertFunction(kAsanPtrSub, IRB.getVoidTy(), IntptrTy, IntptrTy);
-  // We insert an empty inline asm after __asan_report* to avoid callback merge.
-  EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
-                            StringRef(""), StringRef(""),
-                            /*hasSideEffects=*/true);
   if (Mapping.InGlobal)
     AsanShadowGlobal = M.getOrInsertGlobal("__asan_shadow",
                                            ArrayType::get(IRB.getInt8Ty(), 0));
@@ -3205,8 +3193,8 @@ void FunctionStackPoisoner::processStaticAllocas() {
   // 2) There is a returns_twice call (typically setjmp), which is
   //    optimization-hostile, and doesn't play well with introduced indirect
   //    register-relative calculation of local variable addresses.
-  DoDynamicAlloca &= !HasNonEmptyInlineAsm && !HasReturnsTwiceCall;
-  DoStackMalloc &= !HasNonEmptyInlineAsm && !HasReturnsTwiceCall;
+  DoDynamicAlloca &= !HasInlineAsm && !HasReturnsTwiceCall;
+  DoStackMalloc &= !HasInlineAsm && !HasReturnsTwiceCall;
 
   Value *StaticAlloca =
       DoDynamicAlloca ? nullptr : createAllocaForLayout(IRB, L, false);
diff --git a/llvm/lib/Transforms/Instrumentation/CFGMST.h b/llvm/lib/Transforms/Instrumentation/CFGMST.h
index 8bb6f47c4846f..9addb5d1ba938 100644
--- a/llvm/lib/Transforms/Instrumentation/CFGMST.h
+++ b/llvm/lib/Transforms/Instrumentation/CFGMST.h
@@ -20,6 +20,7 @@
 #include "llvm/Analysis/BranchProbabilityInfo.h"
 #include "llvm/Analysis/CFG.h"
 #include "llvm/Support/BranchProbability.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
@@ -28,6 +29,11 @@
 
 #define DEBUG_TYPE "cfgmst"
 
+using namespace llvm;
+static cl::opt<bool> PGOInstrumentEntry(
+    "pgo-instrument-entry", cl::init(false), cl::Hidden,
+    cl::desc("Force to instrument function entry basicblock."));
+
 namespace llvm {
 
 /// An union-find based Minimum Spanning Tree for CFG
@@ -100,8 +106,11 @@ template <class Edge, class BBInfo> class CFGMST {
 
     const BasicBlock *Entry = &(F.getEntryBlock());
     uint64_t EntryWeight = (BFI != nullptr ? BFI->getEntryFreq() : 2);
+    // If we want to instrument the entry count, lower the weight to 0.
+    if (PGOInstrumentEntry)
+      EntryWeight = 0;
     Edge *EntryIncoming = nullptr, *EntryOutgoing = nullptr,
-        *ExitOutgoing = nullptr, *ExitIncoming = nullptr;
+         *ExitOutgoing = nullptr, *ExitIncoming = nullptr;
     uint64_t MaxEntryOutWeight = 0, MaxExitOutWeight = 0, MaxExitInWeight = 0;
 
     // Add a fake edge to the entry.
@@ -135,6 +144,8 @@ template <class Edge, class BBInfo> class CFGMST {
           }
           if (BPI != nullptr)
             Weight = BPI->getEdgeProbability(&*BB, TargetBB).scale(scaleFactor);
+          if (Weight == 0)
+            Weight++;
           auto *E = &addEdge(&*BB, TargetBB, Weight);
           E->IsCritical = Critical;
           LLVM_DEBUG(dbgs() << "  Edge: from " << BB->getName() << " to "
@@ -278,6 +289,9 @@ template <class Edge, class BBInfo> class CFGMST {
     buildEdges();
     sortEdgesByWeight();
     computeMinimumSpanningTree();
+    if (PGOInstrumentEntry && (AllEdges.size() > 1))
+      std::iter_swap(std::move(AllEdges.begin()),
+                     std::move(AllEdges.begin() + AllEdges.size() - 1));
   }
 };
 
diff --git a/llvm/lib/Transforms/Instrumentation/CGProfile.cpp b/llvm/lib/Transforms/Instrumentation/CGProfile.cpp
index 2d5bd95709409..0cc0d9b07387b 100644
--- a/llvm/lib/Transforms/Instrumentation/CGProfile.cpp
+++ b/llvm/lib/Transforms/Instrumentation/CGProfile.cpp
@@ -10,11 +10,13 @@
 
 #include "llvm/ADT/MapVector.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/InitializePasses.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Transforms/Instrumentation.h"
 
@@ -22,10 +24,32 @@
 
 using namespace llvm;
 
-PreservedAnalyses CGProfilePass::run(Module &M, ModuleAnalysisManager &MAM) {
+static bool
+addModuleFlags(Module &M,
+               MapVector<std::pair<Function *, Function *>, uint64_t> &Counts) {
+  if (Counts.empty())
+    return false;
+
+  LLVMContext &Context = M.getContext();
+  MDBuilder MDB(Context);
+  std::vector<Metadata *> Nodes;
+
+  for (auto E : Counts) {
+    Metadata *Vals[] = {ValueAsMetadata::get(E.first.first),
+                        ValueAsMetadata::get(E.first.second),
+                        MDB.createConstant(ConstantInt::get(
+                            Type::getInt64Ty(Context), E.second))};
+    Nodes.push_back(MDNode::get(Context, Vals));
+  }
+
+  M.addModuleFlag(Module::Append, "CG Profile", MDNode::get(Context, Nodes));
+  return true;
+}
+
+static bool runCGProfilePass(
+    Module &M, function_ref<BlockFrequencyInfo &(Function &)> GetBFI,
+    function_ref<TargetTransformInfo &(Function &)> GetTTI, bool LazyBFI) {
   MapVector<std::pair<Function *, Function *>, uint64_t> Counts;
-  FunctionAnalysisManager &FAM =
-      MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
   InstrProfSymtab Symtab;
   auto UpdateCounts = [&](TargetTransformInfo &TTI, Function *F,
                           Function *CalledF, uint64_t NewCount) {
@@ -35,14 +59,18 @@ PreservedAnalyses CGProfilePass::run(Module &M, ModuleAnalysisManager &MAM) {
     Count = SaturatingAdd(Count, NewCount);
   };
   // Ignore error here.  Indirect calls are ignored if this fails.
-  (void)(bool)Symtab.create(M);
+  (void)(bool) Symtab.create(M);
   for (auto &F : M) {
-    if (F.isDeclaration())
+    // Avoid extra cost of running passes for BFI when the function doesn't have
+    // entry count. Since LazyBlockFrequencyInfoPass only exists in LPM, check
+    // if using LazyBlockFrequencyInfoPass.
+    // TODO: Remove LazyBFI when LazyBlockFrequencyInfoPass is available in NPM.
+    if (F.isDeclaration() || (LazyBFI && !F.getEntryCount()))
       continue;
-    auto &BFI = FAM.getResult<BlockFrequencyAnalysis>(F);
+    auto &BFI = GetBFI(F);
     if (BFI.getEntryFreq() == 0)
       continue;
-    TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(F);
+    TargetTransformInfo &TTI = GetTTI(F);
     for (auto &BB : F) {
       Optional<uint64_t> BBCount = BFI.getBlockProfileCount(&BB);
       if (!BBCount)
@@ -69,28 +97,56 @@ PreservedAnalyses CGProfilePass::run(Module &M, ModuleAnalysisManager &MAM) {
     }
   }
 
-  addModuleFlags(M, Counts);
-
-  return PreservedAnalyses::all();
+  return addModuleFlags(M, Counts);
 }
 
-void CGProfilePass::addModuleFlags(
-    Module &M,
-    MapVector<std::pair<Function *, Function *>, uint64_t> &Counts) const {
-  if (Counts.empty())
-    return;
+namespace {
+struct CGProfileLegacyPass final : public ModulePass {
+  static char ID;
+  CGProfileLegacyPass() : ModulePass(ID) {
+    initializeCGProfileLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
 
-  LLVMContext &Context = M.getContext();
-  MDBuilder MDB(Context);
-  std::vector<Metadata *> Nodes;
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    AU.addRequired<LazyBlockFrequencyInfoPass>();
+    AU.addRequired<TargetTransformInfoWrapperPass>();
+  }
 
-  for (auto E : Counts) {
-    Metadata *Vals[] = {ValueAsMetadata::get(E.first.first),
-                        ValueAsMetadata::get(E.first.second),
-                        MDB.createConstant(ConstantInt::get(
-                            Type::getInt64Ty(Context), E.second))};
-    Nodes.push_back(MDNode::get(Context, Vals));
+  bool runOnModule(Module &M) override {
+    auto GetBFI = [this](Function &F) -> BlockFrequencyInfo & {
+      return this->getAnalysis<LazyBlockFrequencyInfoPass>(F).getBFI();
+    };
+    auto GetTTI = [this](Function &F) -> TargetTransformInfo & {
+      return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
+    };
+
+    return runCGProfilePass(M, GetBFI, GetTTI, true);
   }
+};
 
-  M.addModuleFlag(Module::Append, "CG Profile", MDNode::get(Context, Nodes));
+} // namespace
+
+char CGProfileLegacyPass::ID = 0;
+
+INITIALIZE_PASS(CGProfileLegacyPass, "cg-profile", "Call Graph Profile", false,
+                false)
+
+ModulePass *llvm::createCGProfileLegacyPass() {
+  return new CGProfileLegacyPass();
+}
+
+PreservedAnalyses CGProfilePass::run(Module &M, ModuleAnalysisManager &MAM) {
+  FunctionAnalysisManager &FAM =
+      MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+  auto GetBFI = [&FAM](Function &F) -> BlockFrequencyInfo & {
+    return FAM.getResult<BlockFrequencyAnalysis>(F);
+  };
+  auto GetTTI = [&FAM](Function &F) -> TargetTransformInfo & {
+    return FAM.getResult<TargetIRAnalysis>(F);
+  };
+
+  runCGProfilePass(M, GetBFI, GetTTI, false);
+
+  return PreservedAnalyses::all();
 }
diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index f1f94df038d2c..2846319007318 100644
--- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -1369,15 +1369,9 @@ void DFSanVisitor::visitLoadInst(LoadInst &LI) {
     return;
   }
 
-  uint64_t Align;
-  if (ClPreserveAlignment) {
-    Align = LI.getAlignment();
-    if (Align == 0)
-      Align = DL.getABITypeAlignment(LI.getType());
-  } else {
-    Align = 1;
-  }
-  Value *Shadow = DFSF.loadShadow(LI.getPointerOperand(), Size, Align, &LI);
+  Align Alignment = ClPreserveAlignment ? LI.getAlign() : Align(1);
+  Value *Shadow =
+      DFSF.loadShadow(LI.getPointerOperand(), Size, Alignment.value(), &LI);
   if (ClCombinePointerLabelsOnLoad) {
     Value *PtrShadow = DFSF.getShadow(LI.getPointerOperand());
     Shadow = DFSF.combineShadows(Shadow, PtrShadow, &LI);
diff --git a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
index 3a9409601120b..d8a965a90127b 100644
--- a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
@@ -844,7 +844,6 @@ bool GCOVProfiler::emitProfileArcs() {
         continue;
       // TODO: Functions using scope-based EH are currently not supported.
       if (isUsingScopeBasedEH(F)) continue;
-      if (!Result) Result = true;
 
       DenseMap<std::pair<BasicBlock *, BasicBlock *>, unsigned> EdgeToCounter;
       unsigned Edges = 0;
@@ -939,6 +938,7 @@ bool GCOVProfiler::emitProfileArcs() {
     Builder.CreateRetVoid();
 
     appendToGlobalCtors(*M, F, 0);
+    Result = true;
   }
 
   return Result;
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index 5a90d8935f74d..2e71d613714a5 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -539,30 +539,29 @@ void HWAddressSanitizer::getInterestingMemoryOperands(
     if (!ClInstrumentReads || ignoreAccess(LI->getPointerOperand()))
       return;
     Interesting.emplace_back(I, LI->getPointerOperandIndex(), false,
-                             LI->getType(), LI->getAlignment());
+                             LI->getType(), LI->getAlign());
   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
     if (!ClInstrumentWrites || ignoreAccess(SI->getPointerOperand()))
       return;
     Interesting.emplace_back(I, SI->getPointerOperandIndex(), true,
-                             SI->getValueOperand()->getType(),
-                             SI->getAlignment());
+                             SI->getValueOperand()->getType(), SI->getAlign());
   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
     if (!ClInstrumentAtomics || ignoreAccess(RMW->getPointerOperand()))
       return;
     Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
-                             RMW->getValOperand()->getType(), 0);
+                             RMW->getValOperand()->getType(), None);
   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
     if (!ClInstrumentAtomics || ignoreAccess(XCHG->getPointerOperand()))
       return;
     Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
-                             XCHG->getCompareOperand()->getType(), 0);
+                             XCHG->getCompareOperand()->getType(), None);
   } else if (auto CI = dyn_cast<CallInst>(I)) {
     for (unsigned ArgNo = 0; ArgNo < CI->getNumArgOperands(); ArgNo++) {
       if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
           ignoreAccess(CI->getArgOperand(ArgNo)))
         continue;
       Type *Ty = CI->getParamByValType(ArgNo);
-      Interesting.emplace_back(I, ArgNo, false, Ty, 1);
+      Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
     }
   }
 }
@@ -732,9 +731,9 @@ bool HWAddressSanitizer::instrumentMemAccess(InterestingMemoryOperand &O) {
 
   IRBuilder<> IRB(O.getInsn());
   if (isPowerOf2_64(O.TypeSize) &&
-      (O.TypeSize / 8 <= (1UL << (kNumberOfAccessSizes - 1))) &&
-      (O.Alignment >= (1UL << Mapping.Scale) || O.Alignment == 0 ||
-       O.Alignment >= O.TypeSize / 8)) {
+      (O.TypeSize / 8 <= (1ULL << (kNumberOfAccessSizes - 1))) &&
+      (!O.Alignment || *O.Alignment >= (1ULL << Mapping.Scale) ||
+       *O.Alignment >= O.TypeSize / 8)) {
     size_t AccessSizeIndex = TypeSizeToSizeIndex(O.TypeSize);
     if (ClInstrumentWithCalls) {
       IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index ceceafa860300..7b03bbfcdfe4b 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -110,6 +110,12 @@ cl::opt<bool> AtomicCounterUpdatePromoted(
              " for promoted counters only"),
     cl::init(false));
 
+cl::opt<bool> AtomicFirstCounter(
+    "atomic-first-counter", cl::ZeroOrMore,
+    cl::desc("Use atomic fetch add for first counter in a function (usually "
+             "the entry counter)"),
+    cl::init(false));
+
 // If the option is not specified, the default behavior about whether
 // counter promotion is done depends on how instrumentaiton lowering
 // pipeline is setup, i.e., the default value of true of this option
@@ -696,7 +702,8 @@ void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
     Addr = Builder.CreateIntToPtr(Add, Int64PtrTy);
   }
 
-  if (Options.Atomic || AtomicCounterUpdateAll) {
+  if (Options.Atomic || AtomicCounterUpdateAll ||
+      (Index == 0 && AtomicFirstCounter)) {
     Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
                             AtomicOrdering::Monotonic);
   } else {
diff --git a/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp b/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp
index 64626225f23fe..ad238f1357c69 100644
--- a/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp
@@ -112,6 +112,7 @@ void llvm::initializeInstrumentation(PassRegistry &Registry) {
   initializePGOInstrumentationUseLegacyPassPass(Registry);
   initializePGOIndirectCallPromotionLegacyPassPass(Registry);
   initializePGOMemOPSizeOptLegacyPassPass(Registry);
+  initializeCGProfileLegacyPassPass(Registry);
   initializeInstrOrderFileLegacyPassPass(Registry);
   initializeInstrProfilingLegacyPassPass(Registry);
   initializeMemorySanitizerLegacyPassPass(Registry);
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index b69f4162e9c61..f825cf99205b3 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -137,6 +137,9 @@
 ///
 ///  KernelMemorySanitizer only supports X86_64 at the moment.
 ///
+//
+// FIXME: This sanitizer does not yet handle scalable vectors
+//
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
@@ -595,9 +598,6 @@ class MemorySanitizer {
 
   /// Branch weights for origin store.
   MDNode *OriginStoreWeights;
-
-  /// An empty volatile inline asm that prevents callback merge.
-  InlineAsm *EmptyAsm;
 };
 
 void insertModuleCtor(Module &M) {
@@ -854,10 +854,6 @@ void MemorySanitizer::initializeCallbacks(Module &M) {
   MemsetFn = M.getOrInsertFunction(
     "__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(),
     IntptrTy);
-  // We insert an empty inline asm after __msan_report* to avoid callback merge.
-  EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
-                            StringRef(""), StringRef(""),
-                            /*hasSideEffects=*/true);
 
   MsanInstrumentAsmStoreFn =
       M.getOrInsertFunction("__msan_instrument_asm_store", IRB.getVoidTy(),
@@ -1113,7 +1109,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
                    unsigned Size, Align Alignment) {
     const DataLayout &DL = F.getParent()->getDataLayout();
-    const Align IntptrAlignment = Align(DL.getABITypeAlignment(MS.IntptrTy));
+    const Align IntptrAlignment = DL.getABITypeAlign(MS.IntptrTy);
     unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
     assert(IntptrAlignment >= kMinOriginAlignment);
     assert(IntptrSize >= kOriginSize);
@@ -1211,8 +1207,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     if (!Origin)
       Origin = (Value *)IRB.getInt32(0);
     assert(Origin->getType()->isIntegerTy());
-    IRB.CreateCall(MS.WarningFn, Origin);
-    IRB.CreateCall(MS.EmptyAsm, {});
+    IRB.CreateCall(MS.WarningFn, Origin)->setCannotMerge();
     // FIXME: Insert UnreachableInst if !MS.Recover?
     // This may invalidate some of the following checks and needs to be done
     // at the very end.
@@ -1363,7 +1358,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     if (VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
       uint32_t EltSize = DL.getTypeSizeInBits(VT->getElementType());
       return FixedVectorType::get(IntegerType::get(*MS.C, EltSize),
-                                  VT->getNumElements());
+                                  cast<FixedVectorType>(VT)->getNumElements());
     }
     if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
       return ArrayType::get(getShadowTy(AT->getElementType()),
@@ -2085,7 +2080,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
     assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&
            "Vector of pointers is not a valid shadow type");
-    return Ty->isVectorTy() ? cast<VectorType>(Ty)->getNumElements() *
+    return Ty->isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() *
                                   Ty->getScalarSizeInBits()
                             : Ty->getPrimitiveSizeInBits();
   }
@@ -2103,8 +2098,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
       return IRB.CreateIntCast(V, dstTy, Signed);
     if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
-        cast<VectorType>(dstTy)->getNumElements() ==
-            cast<VectorType>(srcTy)->getNumElements())
+        cast<FixedVectorType>(dstTy)->getNumElements() ==
+            cast<FixedVectorType>(srcTy)->getNumElements())
       return IRB.CreateIntCast(V, dstTy, Signed);
     Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits));
     Value *V2 =
@@ -2149,7 +2144,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     Constant *ShadowMul;
     Type *Ty = ConstArg->getType();
     if (auto *VTy = dyn_cast<VectorType>(Ty)) {
-      unsigned NumElements = VTy->getNumElements();
+      unsigned NumElements = cast<FixedVectorType>(VTy)->getNumElements();
       Type *EltTy = VTy->getElementType();
       SmallVector<Constant *, 16> Elements;
       for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
@@ -2977,8 +2972,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
 
         Value *Acc = IRB.CreateExtractElement(
             MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
-        for (int i = 1,
-                 N = cast<VectorType>(PassThru->getType())->getNumElements();
+        for (int i = 1, N = cast<FixedVectorType>(PassThru->getType())
+                                ->getNumElements();
              i < N; ++i) {
           Value *More = IRB.CreateExtractElement(
               MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), i));
@@ -3038,7 +3033,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     IRBuilder<> IRB(&I);
     Type *ShadowTy = getShadowTy(&I);
     unsigned Width =
-        cast<VectorType>(I.getArgOperand(0)->getType())->getNumElements();
+        cast<FixedVectorType>(I.getArgOperand(0)->getType())->getNumElements();
     assert(isa<ConstantInt>(I.getArgOperand(2)) &&
            "pclmul 3rd operand must be a constant");
     unsigned Imm = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
@@ -3054,6 +3049,32 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     SOC.Done(&I);
   }
 
+  // Instrument _mm_*_sd intrinsics
+  void handleUnarySdIntrinsic(IntrinsicInst &I) {
+    IRBuilder<> IRB(&I);
+    Value *First = getShadow(&I, 0);
+    Value *Second = getShadow(&I, 1);
+    // High word of first operand, low word of second
+    Value *Shadow =
+        IRB.CreateShuffleVector(First, Second, llvm::makeArrayRef<int>({2, 1}));
+
+    setShadow(&I, Shadow);
+    setOriginForNaryOp(I);
+  }
+
+  void handleBinarySdIntrinsic(IntrinsicInst &I) {
+    IRBuilder<> IRB(&I);
+    Value *First = getShadow(&I, 0);
+    Value *Second = getShadow(&I, 1);
+    Value *OrShadow = IRB.CreateOr(First, Second);
+    // High word of first operand, low word of both OR'd together
+    Value *Shadow = IRB.CreateShuffleVector(First, OrShadow,
+                                            llvm::makeArrayRef<int>({2, 1}));
+
+    setShadow(&I, Shadow);
+    setOriginForNaryOp(I);
+  }
+
   void visitIntrinsicInst(IntrinsicInst &I) {
     switch (I.getIntrinsicID()) {
     case Intrinsic::lifetime_start:
@@ -3293,6 +3314,14 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       handlePclmulIntrinsic(I);
       break;
 
+    case Intrinsic::x86_sse41_round_sd:
+      handleUnarySdIntrinsic(I);
+      break;
+    case Intrinsic::x86_sse2_max_sd:
+    case Intrinsic::x86_sse2_min_sd:
+      handleBinarySdIntrinsic(I);
+      break;
+
     case Intrinsic::is_constant:
       // The result of llvm.is.constant() is always defined.
       setShadow(&I, getCleanShadow(&I));
diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
index ac36b147e9f85..b6a9df57e4315 100644
--- a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
+++ b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
@@ -250,7 +250,6 @@ class ModuleSanitizerCoverage {
   FunctionCallee SanCovTraceGepFunction;
   FunctionCallee SanCovTraceSwitchFunction;
   GlobalVariable *SanCovLowestStack;
-  InlineAsm *EmptyAsm;
   Type *IntptrTy, *IntptrPtrTy, *Int64Ty, *Int64PtrTy, *Int32Ty, *Int32PtrTy,
       *Int16Ty, *Int8Ty, *Int8PtrTy, *Int1Ty, *Int1PtrTy;
   Module *CurModule;
@@ -485,11 +484,6 @@ bool ModuleSanitizerCoverage::instrumentModule(
   if (Options.StackDepth && !SanCovLowestStack->isDeclaration())
     SanCovLowestStack->setInitializer(Constant::getAllOnesValue(IntptrTy));
 
-  // We insert an empty inline asm after cov callbacks to avoid callback merge.
-  EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
-                            StringRef(""), StringRef(""),
-                            /*hasSideEffects=*/true);
-
   SanCovTracePC = M.getOrInsertFunction(SanCovTracePCName, VoidTy);
   SanCovTracePCGuard =
       M.getOrInsertFunction(SanCovTracePCGuardName, VoidTy, Int32PtrTy);
@@ -921,16 +915,15 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
   IRBuilder<> IRB(&*IP);
   IRB.SetCurrentDebugLocation(EntryLoc);
   if (Options.TracePC) {
-    IRB.CreateCall(SanCovTracePC); // gets the PC using GET_CALLER_PC.
-    IRB.CreateCall(EmptyAsm, {}); // Avoids callback merge.
+    IRB.CreateCall(SanCovTracePC)
+        ->setCannotMerge(); // gets the PC using GET_CALLER_PC.
   }
   if (Options.TracePCGuard) {
     auto GuardPtr = IRB.CreateIntToPtr(
         IRB.CreateAdd(IRB.CreatePointerCast(FunctionGuardArray, IntptrTy),
                       ConstantInt::get(IntptrTy, Idx * 4)),
         Int32PtrTy);
-    IRB.CreateCall(SanCovTracePCGuard, GuardPtr);
-    IRB.CreateCall(EmptyAsm, {}); // Avoids callback merge.
+    IRB.CreateCall(SanCovTracePCGuard, GuardPtr)->setCannotMerge();
   }
   if (Options.Inline8bitCounters) {
     auto CounterPtr = IRB.CreateGEP(
diff --git a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
index 313fe747e8e12..c911b37afac7e 100644
--- a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
@@ -278,28 +278,28 @@ void ThreadSanitizer::initialize(Module &M) {
     TsanAtomicStore[i] = M.getOrInsertFunction(
         AtomicStoreName, Attr, IRB.getVoidTy(), PtrTy, Ty, OrdTy);
 
-    for (int op = AtomicRMWInst::FIRST_BINOP;
-        op <= AtomicRMWInst::LAST_BINOP; ++op) {
-      TsanAtomicRMW[op][i] = nullptr;
+    for (unsigned Op = AtomicRMWInst::FIRST_BINOP;
+         Op <= AtomicRMWInst::LAST_BINOP; ++Op) {
+      TsanAtomicRMW[Op][i] = nullptr;
       const char *NamePart = nullptr;
-      if (op == AtomicRMWInst::Xchg)
+      if (Op == AtomicRMWInst::Xchg)
         NamePart = "_exchange";
-      else if (op == AtomicRMWInst::Add)
+      else if (Op == AtomicRMWInst::Add)
         NamePart = "_fetch_add";
-      else if (op == AtomicRMWInst::Sub)
+      else if (Op == AtomicRMWInst::Sub)
         NamePart = "_fetch_sub";
-      else if (op == AtomicRMWInst::And)
+      else if (Op == AtomicRMWInst::And)
         NamePart = "_fetch_and";
-      else if (op == AtomicRMWInst::Or)
+      else if (Op == AtomicRMWInst::Or)
         NamePart = "_fetch_or";
-      else if (op == AtomicRMWInst::Xor)
+      else if (Op == AtomicRMWInst::Xor)
         NamePart = "_fetch_xor";
-      else if (op == AtomicRMWInst::Nand)
+      else if (Op == AtomicRMWInst::Nand)
         NamePart = "_fetch_nand";
       else
         continue;
       SmallString<32> RMWName("__tsan_atomic" + itostr(BitSize) + NamePart);
-      TsanAtomicRMW[op][i] =
+      TsanAtomicRMW[Op][i] =
           M.getOrInsertFunction(RMWName, Attr, Ty, PtrTy, Ty, OrdTy);
     }
 
diff --git a/llvm/lib/Transforms/ObjCARC/ObjCARC.h b/llvm/lib/Transforms/ObjCARC/ObjCARC.h
index 46b1126300557..b496842fcfc5d 100644
--- a/llvm/lib/Transforms/ObjCARC/ObjCARC.h
+++ b/llvm/lib/Transforms/ObjCARC/ObjCARC.h
@@ -22,16 +22,9 @@
 #ifndef LLVM_LIB_TRANSFORMS_OBJCARC_OBJCARC_H
 #define LLVM_LIB_TRANSFORMS_OBJCARC_OBJCARC_H
 
-#include "llvm/ADT/StringSwitch.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/ObjCARCAnalysisUtils.h"
 #include "llvm/Analysis/ObjCARCInstKind.h"
-#include "llvm/Analysis/Passes.h"
 #include "llvm/Transforms/Utils/Local.h"
-#include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/InstIterator.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Pass.h"
 #include "llvm/Transforms/ObjCARC.h"
 
 namespace llvm {
diff --git a/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp b/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp
index 789620d6b2b8d..7fd4857c44903 100644
--- a/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp
+++ b/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp
@@ -33,6 +33,7 @@
 #include "llvm/Analysis/EHPersonalities.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/InlineAsm.h"
+#include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/CommandLine.h"
diff --git a/llvm/lib/Transforms/Scalar/BDCE.cpp b/llvm/lib/Transforms/Scalar/BDCE.cpp
index c22ea0b8271e3..767c7656dcfac 100644
--- a/llvm/lib/Transforms/Scalar/BDCE.cpp
+++ b/llvm/lib/Transforms/Scalar/BDCE.cpp
@@ -9,7 +9,8 @@
 // This file implements the Bit-Tracking Dead Code Elimination pass. Some
 // instructions (shifts, some ands, ors, etc.) kill some of their input bits.
 // We track these dead bits and remove instructions that compute only these
-// dead bits.
+// dead bits. We also simplify sext that generates unused extension bits,
+// converting it to a zext.
 //
 //===----------------------------------------------------------------------===//
 
@@ -19,6 +20,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/DemandedBits.h"
 #include "llvm/Analysis/GlobalsModRef.h"
+#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/InitializePasses.h"
@@ -33,6 +35,8 @@ using namespace llvm;
 
 STATISTIC(NumRemoved, "Number of instructions removed (unused)");
 STATISTIC(NumSimplified, "Number of instructions trivialized (dead bits)");
+STATISTIC(NumSExt2ZExt,
+          "Number of sign extension instructions converted to zero extension");
 
 /// If an instruction is trivialized (dead), then the chain of users of that
 /// instruction may need to be cleared of assumptions that can no longer be
@@ -109,6 +113,24 @@ static bool bitTrackingDCE(Function &F, DemandedBits &DB) {
       continue;
     }
 
+    // Convert SExt into ZExt if none of the extension bits is required
+    if (SExtInst *SE = dyn_cast<SExtInst>(&I)) {
+      APInt Demanded = DB.getDemandedBits(SE);
+      const uint32_t SrcBitSize = SE->getSrcTy()->getScalarSizeInBits();
+      auto *const DstTy = SE->getDestTy();
+      const uint32_t DestBitSize = DstTy->getScalarSizeInBits();
+      if (Demanded.countLeadingZeros() >= (DestBitSize - SrcBitSize)) {
+        clearAssumptionsOfUsers(SE, DB);
+        IRBuilder<> Builder(SE);
+        I.replaceAllUsesWith(
+            Builder.CreateZExt(SE->getOperand(0), DstTy, SE->getName()));
+        Worklist.push_back(SE);
+        Changed = true;
+        NumSExt2ZExt++;
+        continue;
+      }
+    }
+
     for (Use &U : I.operands()) {
       // DemandedBits only detects dead integer uses.
       if (!U->getType()->isIntOrIntVectorTy())
diff --git a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
index fa3bc5f82a3d0..7c14b69d658dc 100644
--- a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
@@ -250,7 +250,7 @@ static void findBestInsertionSet(DominatorTree &DT, BlockFrequencyInfo &BFI,
   Orders.push_back(Entry);
   while (Idx != Orders.size()) {
     BasicBlock *Node = Orders[Idx++];
-    for (auto ChildDomNode : DT.getNode(Node)->getChildren()) {
+    for (auto ChildDomNode : DT.getNode(Node)->children()) {
       if (Candidates.count(ChildDomNode->getBlock()))
         Orders.push_back(ChildDomNode->getBlock());
     }
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 80da42d99b000..e58db03225eea 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -51,6 +51,7 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/Value.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
@@ -73,6 +74,7 @@
 #include <utility>
 
 using namespace llvm;
+using namespace PatternMatch;
 
 #define DEBUG_TYPE "dse"
 
@@ -210,18 +212,16 @@ static bool hasAnalyzableMemoryWrite(Instruction *I,
     }
   }
   if (auto *CB = dyn_cast<CallBase>(I)) {
-    if (Function *F = CB->getCalledFunction()) {
-      LibFunc LF;
-      if (TLI.getLibFunc(*F, LF) && TLI.has(LF)) {
-        switch (LF) {
-        case LibFunc_strcpy:
-        case LibFunc_strncpy:
-        case LibFunc_strcat:
-        case LibFunc_strncat:
-          return true;
-        default:
-          return false;
-        }
+    LibFunc LF;
+    if (TLI.getLibFunc(*CB, LF) && TLI.has(LF)) {
+      switch (LF) {
+      case LibFunc_strcpy:
+      case LibFunc_strncpy:
+      case LibFunc_strcat:
+      case LibFunc_strncat:
+        return true;
+      default:
+        return false;
       }
     }
   }
@@ -1535,7 +1535,7 @@ struct DSEState {
 
         auto *MD = dyn_cast_or_null<MemoryDef>(MA);
         if (MD && State.MemDefs.size() < MemorySSADefsPerBlockLimit &&
-            hasAnalyzableMemoryWrite(&I, TLI) && isRemovable(&I))
+            (State.getLocForWriteEx(&I) || State.isMemTerminatorInst(&I)))
           State.MemDefs.push_back(MD);
 
         // Track whether alloca and alloca-like objects are visible in the
@@ -1563,8 +1563,13 @@ struct DSEState {
     // Treat byval or inalloca arguments the same as Allocas, stores to them are
     // dead at the end of the function.
     for (Argument &AI : F.args())
-      if (AI.hasPassPointeeByValueAttr())
-        State.InvisibleToCallerBeforeRet.insert(&AI);
+      if (AI.hasPassPointeeByValueAttr()) {
+        // For byval, the caller doesn't know the address of the allocation.
+        if (AI.hasByValAttr())
+          State.InvisibleToCallerBeforeRet.insert(&AI);
+        State.InvisibleToCallerAfterRet.insert(&AI);
+      }
+
     return State;
   }
 
@@ -1576,16 +1581,17 @@ struct DSEState {
       return {MemoryLocation::getForDest(MTI)};
 
     if (auto *CB = dyn_cast<CallBase>(I)) {
-      if (Function *F = CB->getCalledFunction()) {
-        StringRef FnName = F->getName();
-        if (TLI.has(LibFunc_strcpy) && FnName == TLI.getName(LibFunc_strcpy))
-          return {MemoryLocation(CB->getArgOperand(0))};
-        if (TLI.has(LibFunc_strncpy) && FnName == TLI.getName(LibFunc_strncpy))
-          return {MemoryLocation(CB->getArgOperand(0))};
-        if (TLI.has(LibFunc_strcat) && FnName == TLI.getName(LibFunc_strcat))
-          return {MemoryLocation(CB->getArgOperand(0))};
-        if (TLI.has(LibFunc_strncat) && FnName == TLI.getName(LibFunc_strncat))
+      LibFunc LF;
+      if (TLI.getLibFunc(*CB, LF) && TLI.has(LF)) {
+        switch (LF) {
+        case LibFunc_strcpy:
+        case LibFunc_strncpy:
+        case LibFunc_strcat:
+        case LibFunc_strncat:
           return {MemoryLocation(CB->getArgOperand(0))};
+        default:
+          break;
+        }
       }
       return None;
     }
@@ -1616,7 +1622,99 @@ struct DSEState {
                        UseInst, IOL, AA, &F) == OW_Complete;
   }
 
-  /// Returns true if \p Use may read from \p DefLoc.
+  /// Returns true if \p Def is not read before returning from the function.
+  bool isWriteAtEndOfFunction(MemoryDef *Def) {
+    LLVM_DEBUG(dbgs() << "  Check if def " << *Def << " ("
+                      << *Def->getMemoryInst()
+                      << ") is at the end the function \n");
+
+    auto MaybeLoc = getLocForWriteEx(Def->getMemoryInst());
+    if (!MaybeLoc) {
+      LLVM_DEBUG(dbgs() << "  ... could not get location for write.\n");
+      return false;
+    }
+
+    SmallVector<MemoryAccess *, 4> WorkList;
+    SmallPtrSet<MemoryAccess *, 8> Visited;
+    auto PushMemUses = [&WorkList, &Visited](MemoryAccess *Acc) {
+      if (!Visited.insert(Acc).second)
+        return;
+      for (Use &U : Acc->uses())
+        WorkList.push_back(cast<MemoryAccess>(U.getUser()));
+    };
+    PushMemUses(Def);
+    for (unsigned I = 0; I < WorkList.size(); I++) {
+      if (WorkList.size() >= MemorySSAScanLimit) {
+        LLVM_DEBUG(dbgs() << "  ... hit exploration limit.\n");
+        return false;
+      }
+
+      MemoryAccess *UseAccess = WorkList[I];
+      if (isa<MemoryPhi>(UseAccess)) {
+        PushMemUses(UseAccess);
+        continue;
+      }
+
+      // TODO: Checking for aliasing is expensive. Consider reducing the amount
+      // of times this is called and/or caching it.
+      Instruction *UseInst = cast<MemoryUseOrDef>(UseAccess)->getMemoryInst();
+      if (isReadClobber(*MaybeLoc, UseInst)) {
+        LLVM_DEBUG(dbgs() << "  ... hit read clobber " << *UseInst << ".\n");
+        return false;
+      }
+
+      if (MemoryDef *UseDef = dyn_cast<MemoryDef>(UseAccess))
+        PushMemUses(UseDef);
+    }
+    return true;
+  }
+
+  /// If \p I is a memory  terminator like llvm.lifetime.end or free, return a
+  /// pair with the MemoryLocation terminated by \p I and a boolean flag
+  /// indicating whether \p I is a free-like call.
+  Optional<std::pair<MemoryLocation, bool>>
+  getLocForTerminator(Instruction *I) const {
+    uint64_t Len;
+    Value *Ptr;
+    if (match(I, m_Intrinsic<Intrinsic::lifetime_end>(m_ConstantInt(Len),
+                                                      m_Value(Ptr))))
+      return {std::make_pair(MemoryLocation(Ptr, Len), false)};
+
+    if (auto *CB = dyn_cast<CallBase>(I)) {
+      if (isFreeCall(I, &TLI))
+        return {std::make_pair(MemoryLocation(CB->getArgOperand(0)), true)};
+    }
+
+    return None;
+  }
+
+  /// Returns true if \p I is a memory terminator instruction like
+  /// llvm.lifetime.end or free.
+  bool isMemTerminatorInst(Instruction *I) const {
+    IntrinsicInst *II = dyn_cast<IntrinsicInst>(I);
+    return (II && II->getIntrinsicID() == Intrinsic::lifetime_end) ||
+           isFreeCall(I, &TLI);
+  }
+
+  /// Returns true if \p MaybeTerm is a memory terminator for the same
+  /// underlying object as \p DefLoc.
+  bool isMemTerminator(MemoryLocation DefLoc, Instruction *MaybeTerm) const {
+    Optional<std::pair<MemoryLocation, bool>> MaybeTermLoc =
+        getLocForTerminator(MaybeTerm);
+
+    if (!MaybeTermLoc)
+      return false;
+
+    // If the terminator is a free-like call, all accesses to the underlying
+    // object can be considered terminated.
+    if (MaybeTermLoc->second) {
+      DataLayout DL = MaybeTerm->getParent()->getModule()->getDataLayout();
+      DefLoc = MemoryLocation(GetUnderlyingObject(DefLoc.Ptr, DL));
+    }
+    return AA.isMustAlias(MaybeTermLoc->first, DefLoc);
+  }
+
+  // Returns true if \p Use may read from \p DefLoc.
   bool isReadClobber(MemoryLocation DefLoc, Instruction *UseInst) const {
     if (!UseInst->mayReadFromMemory())
       return false;
@@ -1721,6 +1819,11 @@ struct DSEState {
         continue;
       }
 
+      // A memory terminator kills all preceeding MemoryDefs and all succeeding
+      // MemoryAccesses. We do not have to check it's users.
+      if (isMemTerminator(DefLoc, UseInst))
+        continue;
+
       // Uses which may read the original MemoryDef mean we cannot eliminate the
       // original MD. Stop walk.
       if (isReadClobber(DefLoc, UseInst)) {
@@ -1802,7 +1905,7 @@ struct DSEState {
         if (CommonPred)
           WorkList.insert(CommonPred);
         else
-          for (BasicBlock *R : PDT.getRoots())
+          for (BasicBlock *R : PDT.roots())
             WorkList.insert(R);
 
         NumCFGTries++;
@@ -1900,41 +2003,97 @@ struct DSEState {
   //  * A memory instruction that may throw and \p SI accesses a non-stack
   //  object.
   //  * Atomic stores stronger that monotonic.
-  bool isDSEBarrier(Instruction *SI, MemoryLocation &SILoc,
-                    const Value *SILocUnd, Instruction *NI,
-                    MemoryLocation &NILoc) const {
+  bool isDSEBarrier(const Value *SILocUnd, Instruction *NI) const {
     // If NI may throw it acts as a barrier, unless we are to an alloca/alloca
     // like object that does not escape.
     if (NI->mayThrow() && !InvisibleToCallerBeforeRet.count(SILocUnd))
       return true;
 
+    // If NI is an atomic load/store stronger than monotonic, do not try to
+    // eliminate/reorder it.
     if (NI->isAtomic()) {
-      if (auto *NSI = dyn_cast<StoreInst>(NI)) {
-        if (isStrongerThanMonotonic(NSI->getOrdering()))
-          return true;
-      } else
-        llvm_unreachable(
-            "Other instructions should be modeled/skipped in MemorySSA");
+      if (auto *LI = dyn_cast<LoadInst>(NI))
+        return isStrongerThanMonotonic(LI->getOrdering());
+      if (auto *SI = dyn_cast<StoreInst>(NI))
+        return isStrongerThanMonotonic(SI->getOrdering());
+      llvm_unreachable("other instructions should be skipped in MemorySSA");
     }
-
     return false;
   }
-};
 
-/// \returns true if \p KillingDef stores the result of \p Load to the source of
-/// \p Load.
-static bool storeIsNoop(MemorySSA &MSSA, LoadInst *Load,
-                        MemoryDef *KillingDef) {
-  Instruction *Store = KillingDef->getMemoryInst();
-  // If the load's operand isn't the destination of the store, bail.
-  if (Load->getPointerOperand() != Store->getOperand(1))
-    return false;
+  /// Eliminate writes to objects that are not visible in the caller and are not
+  /// accessed before returning from the function.
+  bool eliminateDeadWritesAtEndOfFunction() {
+    const DataLayout &DL = F.getParent()->getDataLayout();
+    bool MadeChange = false;
+    LLVM_DEBUG(
+        dbgs()
+        << "Trying to eliminate MemoryDefs at the end of the function\n");
+    for (int I = MemDefs.size() - 1; I >= 0; I--) {
+      MemoryDef *Def = MemDefs[I];
+      if (SkipStores.find(Def) != SkipStores.end() ||
+          !isRemovable(Def->getMemoryInst()))
+        continue;
 
-  // Get the defining access for the load.
-  auto *LoadAccess = MSSA.getMemoryAccess(Load)->getDefiningAccess();
-  // The store is dead if the defining accesses are the same.
-  return LoadAccess == KillingDef->getDefiningAccess();
-}
+      // TODO: Consider doing the underlying object check first, if it is
+      // beneficial compile-time wise.
+      if (isWriteAtEndOfFunction(Def)) {
+        Instruction *DefI = Def->getMemoryInst();
+        // See through pointer-to-pointer bitcasts
+        SmallVector<const Value *, 4> Pointers;
+        GetUnderlyingObjects(getLocForWriteEx(DefI)->Ptr, Pointers, DL);
+
+        LLVM_DEBUG(dbgs() << "   ... MemoryDef is not accessed until the end "
+                             "of the function\n");
+        bool CanKill = true;
+        for (const Value *Pointer : Pointers) {
+          if (!InvisibleToCallerAfterRet.count(Pointer)) {
+            CanKill = false;
+            break;
+          }
+        }
+
+        if (CanKill) {
+          deleteDeadInstruction(DefI);
+          ++NumFastStores;
+          MadeChange = true;
+        }
+      }
+    }
+    return MadeChange;
+  }
+
+  /// \returns true if \p Def is a no-op store, either because it
+  /// directly stores back a loaded value or stores zero to a calloced object.
+  bool storeIsNoop(MemoryDef *Def, MemoryLocation DefLoc, const Value *DefUO) {
+    StoreInst *Store = dyn_cast<StoreInst>(Def->getMemoryInst());
+    if (!Store)
+      return false;
+
+    if (auto *LoadI = dyn_cast<LoadInst>(Store->getOperand(0))) {
+      if (LoadI->getPointerOperand() == Store->getOperand(1)) {
+        auto *LoadAccess = MSSA.getMemoryAccess(LoadI)->getDefiningAccess();
+        // If both accesses share the same defining access, no instructions
+        // between them can modify the memory location.
+        return LoadAccess == Def->getDefiningAccess();
+      }
+    }
+
+    Constant *StoredConstant = dyn_cast<Constant>(Store->getOperand(0));
+    if (StoredConstant && StoredConstant->isNullValue()) {
+      auto *DefUOInst = dyn_cast<Instruction>(DefUO);
+      if (DefUOInst && isCallocLikeFn(DefUOInst, &TLI)) {
+        auto *UnderlyingDef = cast<MemoryDef>(MSSA.getMemoryAccess(DefUOInst));
+        // If UnderlyingDef is the clobbering access of Def, no instructions
+        // between them can modify the memory location.
+        auto *ClobberDef =
+            MSSA.getSkipSelfWalker()->getClobberingMemoryAccess(Def);
+        return UnderlyingDef == ClobberDef;
+      }
+    }
+    return false;
+  }
+};
 
 bool eliminateDeadStoresMemorySSA(Function &F, AliasAnalysis &AA,
                                   MemorySSA &MSSA, DominatorTree &DT,
@@ -1951,19 +2110,13 @@ bool eliminateDeadStoresMemorySSA(Function &F, AliasAnalysis &AA,
       continue;
     Instruction *SI = KillingDef->getMemoryInst();
 
-    // Check if we're storing a value that we just loaded.
-    if (auto *Load = dyn_cast<LoadInst>(SI->getOperand(0))) {
-      if (storeIsNoop(MSSA, Load, KillingDef)) {
-        LLVM_DEBUG(dbgs() << "DSE: Remove Dead Store:\n  DEAD: " << *SI
-                          << '\n');
-        State.deleteDeadInstruction(SI);
-        NumNoopStores++;
-        MadeChange = true;
-        continue;
-      }
-    }
-
     auto MaybeSILoc = State.getLocForWriteEx(SI);
+    if (State.isMemTerminatorInst(SI))
+      MaybeSILoc = State.getLocForTerminator(SI).map(
+          [](const std::pair<MemoryLocation, bool> &P) { return P.first; });
+    else
+      MaybeSILoc = State.getLocForWriteEx(SI);
+
     if (!MaybeSILoc) {
       LLVM_DEBUG(dbgs() << "Failed to find analyzable write location for "
                         << *SI << "\n");
@@ -1972,6 +2125,16 @@ bool eliminateDeadStoresMemorySSA(Function &F, AliasAnalysis &AA,
     MemoryLocation SILoc = *MaybeSILoc;
     assert(SILoc.Ptr && "SILoc should not be null");
     const Value *SILocUnd = GetUnderlyingObject(SILoc.Ptr, DL);
+
+    // Check if the store is a no-op.
+    if (isRemovable(SI) && State.storeIsNoop(KillingDef, SILoc, SILocUnd)) {
+      LLVM_DEBUG(dbgs() << "DSE: Remove No-Op Store:\n  DEAD: " << *SI << '\n');
+      State.deleteDeadInstruction(SI);
+      NumNoopStores++;
+      MadeChange = true;
+      continue;
+    }
+
     Instruction *DefObj =
         const_cast<Instruction *>(dyn_cast<Instruction>(SILocUnd));
     bool DefVisibleToCallerBeforeRet =
@@ -2030,72 +2193,85 @@ bool eliminateDeadStoresMemorySSA(Function &F, AliasAnalysis &AA,
       Instruction *NI = NextDef->getMemoryInst();
       LLVM_DEBUG(dbgs() << " (" << *NI << ")\n");
 
-      if (!hasAnalyzableMemoryWrite(NI, TLI)) {
-        LLVM_DEBUG(dbgs() << "  ... skip, cannot analyze def\n");
-        continue;
-      }
-
-      if (!isRemovable(NI)) {
-        LLVM_DEBUG(dbgs() << "  ... skip, cannot remove def\n");
-        continue;
+      // Before we try to remove anything, check for any extra throwing
+      // instructions that block us from DSEing
+      if (State.mayThrowBetween(SI, NI, SILocUnd)) {
+        LLVM_DEBUG(dbgs() << "  ... skip, may throw!\n");
+        break;
       }
 
-      MemoryLocation NILoc = *State.getLocForWriteEx(NI);
       // Check for anything that looks like it will be a barrier to further
       // removal
-      if (State.isDSEBarrier(SI, SILoc, SILocUnd, NI, NILoc)) {
+      if (State.isDSEBarrier(SILocUnd, NI)) {
         LLVM_DEBUG(dbgs() << "  ... skip, barrier\n");
         continue;
       }
 
-      // Before we try to remove anything, check for any extra throwing
-      // instructions that block us from DSEing
-      if (State.mayThrowBetween(SI, NI, SILocUnd)) {
-        LLVM_DEBUG(dbgs() << "  ... skip, may throw!\n");
-        break;
+      ToCheck.insert(NextDef->getDefiningAccess());
+
+      if (!hasAnalyzableMemoryWrite(NI, TLI)) {
+        LLVM_DEBUG(dbgs() << "  ... skip, cannot analyze def\n");
+        continue;
       }
 
-      if (!DebugCounter::shouldExecute(MemorySSACounter))
+      if (!isRemovable(NI)) {
+        LLVM_DEBUG(dbgs() << "  ... skip, cannot remove def\n");
         continue;
+      }
 
-      // Check if NI overwrites SI.
-      int64_t InstWriteOffset, DepWriteOffset;
-      auto Iter = State.IOLs.insert(
-          std::make_pair<BasicBlock *, InstOverlapIntervalsTy>(
-              NI->getParent(), InstOverlapIntervalsTy()));
-      auto &IOL = Iter.first->second;
-      OverwriteResult OR = isOverwrite(SILoc, NILoc, DL, TLI, DepWriteOffset,
-                                       InstWriteOffset, NI, IOL, AA, &F);
-
-      if (EnablePartialStoreMerging && OR == OW_PartialEarlierWithFullLater) {
-        auto *Earlier = dyn_cast<StoreInst>(NI);
-        auto *Later = dyn_cast<StoreInst>(SI);
-        if (Constant *Merged = tryToMergePartialOverlappingStores(
-                Earlier, Later, InstWriteOffset, DepWriteOffset, DL, &AA,
-                &DT)) {
-
-          // Update stored value of earlier store to merged constant.
-          Earlier->setOperand(0, Merged);
-          ++NumModifiedStores;
-          MadeChange = true;
+      if (!DebugCounter::shouldExecute(MemorySSACounter))
+        continue;
 
-          // Remove later store and remove any outstanding overlap intervals for
-          // the updated store.
-          State.deleteDeadInstruction(Later);
-          auto I = State.IOLs.find(Earlier->getParent());
-          if (I != State.IOLs.end())
-            I->second.erase(Earlier);
-          break;
-        }
-      }
+      MemoryLocation NILoc = *State.getLocForWriteEx(NI);
 
-      ToCheck.insert(NextDef->getDefiningAccess());
-      if (OR == OW_Complete) {
+      if (State.isMemTerminatorInst(SI)) {
+        const Value *NIUnd = GetUnderlyingObject(NILoc.Ptr, DL);
+        if (!SILocUnd || SILocUnd != NIUnd)
+          continue;
         LLVM_DEBUG(dbgs() << "DSE: Remove Dead Store:\n  DEAD: " << *NI
                           << "\n  KILLER: " << *SI << '\n');
         State.deleteDeadInstruction(NI);
         ++NumFastStores;
         MadeChange = true;
+      } else {
+        // Check if NI overwrites SI.
+        int64_t InstWriteOffset, DepWriteOffset;
+        auto Iter = State.IOLs.insert(
+            std::make_pair<BasicBlock *, InstOverlapIntervalsTy>(
+                NI->getParent(), InstOverlapIntervalsTy()));
+        auto &IOL = Iter.first->second;
+        OverwriteResult OR = isOverwrite(SILoc, NILoc, DL, TLI, DepWriteOffset,
+                                         InstWriteOffset, NI, IOL, AA, &F);
+
+        if (EnablePartialStoreMerging && OR == OW_PartialEarlierWithFullLater) {
+          auto *Earlier = dyn_cast<StoreInst>(NI);
+          auto *Later = dyn_cast<StoreInst>(SI);
+          if (Constant *Merged = tryToMergePartialOverlappingStores(
+                  Earlier, Later, InstWriteOffset, DepWriteOffset, DL, &AA,
+                  &DT)) {
+
+            // Update stored value of earlier store to merged constant.
+            Earlier->setOperand(0, Merged);
+            ++NumModifiedStores;
+            MadeChange = true;
+
+            // Remove later store and remove any outstanding overlap intervals
+            // for the updated store.
+            State.deleteDeadInstruction(Later);
+            auto I = State.IOLs.find(Earlier->getParent());
+            if (I != State.IOLs.end())
+              I->second.erase(Earlier);
+            break;
+          }
+        }
+
+        if (OR == OW_Complete) {
+          LLVM_DEBUG(dbgs() << "DSE: Remove Dead Store:\n  DEAD: " << *NI
+                            << "\n  KILLER: " << *SI << '\n');
+          State.deleteDeadInstruction(NI);
+          ++NumFastStores;
+          MadeChange = true;
+        }
       }
     }
   }
@@ -2104,6 +2280,7 @@ bool eliminateDeadStoresMemorySSA(Function &F, AliasAnalysis &AA,
     for (auto &KV : State.IOLs)
       MadeChange |= removePartiallyOverlappedStores(&AA, DL, KV.second);
 
+  MadeChange |= State.eliminateDeadWritesAtEndOfFunction();
   return MadeChange;
 }
 } // end anonymous namespace
diff --git a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
index 1dac64a12d3e1..ddfc8555b0a0a 100644
--- a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -620,8 +620,8 @@ class EarlyCSE {
   public:
     StackNode(ScopedHTType &AvailableValues, LoadHTType &AvailableLoads,
               InvariantHTType &AvailableInvariants, CallHTType &AvailableCalls,
-              unsigned cg, DomTreeNode *n, DomTreeNode::iterator child,
-              DomTreeNode::iterator end)
+              unsigned cg, DomTreeNode *n, DomTreeNode::const_iterator child,
+              DomTreeNode::const_iterator end)
         : CurrentGeneration(cg), ChildGeneration(cg), Node(n), ChildIter(child),
           EndIter(end),
           Scopes(AvailableValues, AvailableLoads, AvailableInvariants,
@@ -635,7 +635,7 @@ class EarlyCSE {
     unsigned childGeneration() { return ChildGeneration; }
     void childGeneration(unsigned generation) { ChildGeneration = generation; }
     DomTreeNode *node() { return Node; }
-    DomTreeNode::iterator childIter() { return ChildIter; }
+    DomTreeNode::const_iterator childIter() { return ChildIter; }
 
     DomTreeNode *nextChild() {
       DomTreeNode *child = *ChildIter;
@@ -643,7 +643,7 @@ class EarlyCSE {
       return child;
     }
 
-    DomTreeNode::iterator end() { return EndIter; }
+    DomTreeNode::const_iterator end() { return EndIter; }
     bool isProcessed() { return Processed; }
     void process() { Processed = true; }
 
@@ -651,8 +651,8 @@ class EarlyCSE {
     unsigned CurrentGeneration;
     unsigned ChildGeneration;
     DomTreeNode *Node;
-    DomTreeNode::iterator ChildIter;
-    DomTreeNode::iterator EndIter;
+    DomTreeNode::const_iterator ChildIter;
+    DomTreeNode::const_iterator EndIter;
     NodeScope Scopes;
     bool Processed = false;
   };
diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index f6a0b6ea46372..0357d905fde52 100644
--- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -1435,8 +1435,12 @@ PHINode *WidenIV::createWideIV(SCEVExpander &Rewriter) {
   // either find an existing phi or materialize a new one. Either way, we
   // expect a well-formed cyclic phi-with-increments. i.e. any operand not part
   // of the phi-SCC dominates the loop entry.
-  Instruction *InsertPt = &L->getHeader()->front();
-  WidePhi = cast<PHINode>(Rewriter.expandCodeFor(AddRec, WideType, InsertPt));
+  Instruction *InsertPt = &*L->getHeader()->getFirstInsertionPt();
+  WidePhi = dyn_cast<PHINode>(Rewriter.expandCodeFor(AddRec, WideType, InsertPt));
+  // If the wide phi is not a phi node, for example a cast node, like bitcast,
+  // inttoptr, ptrtoint, just skip for now.
+  if (!WidePhi)
+    return nullptr;
 
   // Remembering the WideIV increment generated by SCEVExpander allows
   // widenIVUse to reuse it when widening the narrow IV's increment. We don't
diff --git a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
index d407d0439cd8d..db9cc58bbfc40 100644
--- a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
+++ b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
@@ -96,7 +96,6 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
@@ -116,11 +115,13 @@
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 #include <cassert>
 #include <iterator>
@@ -132,6 +133,11 @@
 
 using namespace llvm;
 
+static cl::opt<bool> AssumeDefaultIsFlatAddressSpace(
+    "assume-default-is-flat-addrspace", cl::init(false), cl::ReallyHidden,
+    cl::desc("The default address space is assumed as the flat address space. "
+             "This is mainly for test purpose."));
+
 static const unsigned UninitializedAddressSpace =
     std::numeric_limits<unsigned>::max();
 
@@ -143,6 +149,7 @@ using PostorderStackTy = llvm::SmallVector<PointerIntPair<Value *, 1, bool>, 4>;
 /// InferAddressSpaces
 class InferAddressSpaces : public FunctionPass {
   const TargetTransformInfo *TTI = nullptr;
+  const DataLayout *DL = nullptr;
 
   /// Target specific address space which uses of should be replaced if
   /// possible.
@@ -219,10 +226,45 @@ void initializeInferAddressSpacesPass(PassRegistry &);
 INITIALIZE_PASS(InferAddressSpaces, DEBUG_TYPE, "Infer address spaces",
                 false, false)
 
+// Check whether that's no-op pointer bicast using a pair of
+// `ptrtoint`/`inttoptr` due to the missing no-op pointer bitcast over
+// different address spaces.
+static bool isNoopPtrIntCastPair(const Operator *I2P, const DataLayout &DL,
+                                 const TargetTransformInfo *TTI) {
+  assert(I2P->getOpcode() == Instruction::IntToPtr);
+  auto *P2I = dyn_cast<Operator>(I2P->getOperand(0));
+  if (!P2I || P2I->getOpcode() != Instruction::PtrToInt)
+    return false;
+  // Check it's really safe to treat that pair of `ptrtoint`/`inttoptr` as a
+  // no-op cast. Besides checking both of them are no-op casts, as the
+  // reinterpreted pointer may be used in other pointer arithmetic, we also
+  // need to double-check that through the target-specific hook. That ensures
+  // the underlying target also agrees that's a no-op address space cast and
+  // pointer bits are preserved.
+  // The current IR spec doesn't have clear rules on address space casts,
+  // especially a clear definition for pointer bits in non-default address
+  // spaces. It would be undefined if that pointer is dereferenced after an
+  // invalid reinterpret cast. Also, due to the unclearness for the meaning of
+  // bits in non-default address spaces in the current spec, the pointer
+  // arithmetic may also be undefined after invalid pointer reinterpret cast.
+  // However, as we confirm through the target hooks that it's a no-op
+  // addrspacecast, it doesn't matter since the bits should be the same.
+  return CastInst::isNoopCast(Instruction::CastOps(I2P->getOpcode()),
+                              I2P->getOperand(0)->getType(), I2P->getType(),
+                              DL) &&
+         CastInst::isNoopCast(Instruction::CastOps(P2I->getOpcode()),
+                              P2I->getOperand(0)->getType(), P2I->getType(),
+                              DL) &&
+         TTI->isNoopAddrSpaceCast(
+             P2I->getOperand(0)->getType()->getPointerAddressSpace(),
+             I2P->getType()->getPointerAddressSpace());
+}
+
 // Returns true if V is an address expression.
 // TODO: Currently, we consider only phi, bitcast, addrspacecast, and
 // getelementptr operators.
-static bool isAddressExpression(const Value &V) {
+static bool isAddressExpression(const Value &V, const DataLayout &DL,
+                                const TargetTransformInfo *TTI) {
   const Operator *Op = dyn_cast<Operator>(&V);
   if (!Op)
     return false;
@@ -241,6 +283,8 @@ static bool isAddressExpression(const Value &V) {
     const IntrinsicInst *II = dyn_cast<IntrinsicInst>(&V);
     return II && II->getIntrinsicID() == Intrinsic::ptrmask;
   }
+  case Instruction::IntToPtr:
+    return isNoopPtrIntCastPair(Op, DL, TTI);
   default:
     return false;
   }
@@ -249,7 +293,9 @@ static bool isAddressExpression(const Value &V) {
 // Returns the pointer operands of V.
 //
 // Precondition: V is an address expression.
-static SmallVector<Value *, 2> getPointerOperands(const Value &V) {
+static SmallVector<Value *, 2>
+getPointerOperands(const Value &V, const DataLayout &DL,
+                   const TargetTransformInfo *TTI) {
   const Operator &Op = cast<Operator>(V);
   switch (Op.getOpcode()) {
   case Instruction::PHI: {
@@ -269,6 +315,11 @@ static SmallVector<Value *, 2> getPointerOperands(const Value &V) {
            "unexpected intrinsic call");
     return {II.getArgOperand(0)};
   }
+  case Instruction::IntToPtr: {
+    assert(isNoopPtrIntCastPair(&Op, DL, TTI));
+    auto *P2I = cast<Operator>(Op.getOperand(0));
+    return {P2I->getOperand(0)};
+  }
   default:
     llvm_unreachable("Unexpected instruction type.");
   }
@@ -337,13 +388,13 @@ void InferAddressSpaces::appendsFlatAddressExpressionToPostorderStack(
   // expressions.
   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V)) {
     // TODO: Look in non-address parts, like icmp operands.
-    if (isAddressExpression(*CE) && Visited.insert(CE).second)
+    if (isAddressExpression(*CE, *DL, TTI) && Visited.insert(CE).second)
       PostorderStack.emplace_back(CE, false);
 
     return;
   }
 
-  if (isAddressExpression(*V) &&
+  if (isAddressExpression(*V, *DL, TTI) &&
       V->getType()->getPointerAddressSpace() == FlatAddrSpace) {
     if (Visited.insert(V).second) {
       PostorderStack.emplace_back(V, false);
@@ -351,7 +402,7 @@ void InferAddressSpaces::appendsFlatAddressExpressionToPostorderStack(
       Operator *Op = cast<Operator>(V);
       for (unsigned I = 0, E = Op->getNumOperands(); I != E; ++I) {
         if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Op->getOperand(I))) {
-          if (isAddressExpression(*CE) && Visited.insert(CE).second)
+          if (isAddressExpression(*CE, *DL, TTI) && Visited.insert(CE).second)
             PostorderStack.emplace_back(CE, false);
         }
       }
@@ -407,6 +458,10 @@ InferAddressSpaces::collectFlatAddressExpressions(Function &F) const {
     } else if (auto *ASC = dyn_cast<AddrSpaceCastInst>(&I)) {
       if (!ASC->getType()->isVectorTy())
         PushPtrOperand(ASC->getPointerOperand());
+    } else if (auto *I2P = dyn_cast<IntToPtrInst>(&I)) {
+      if (isNoopPtrIntCastPair(cast<Operator>(I2P), *DL, TTI))
+        PushPtrOperand(
+            cast<PtrToIntInst>(I2P->getOperand(0))->getPointerOperand());
     }
   }
 
@@ -423,7 +478,7 @@ InferAddressSpaces::collectFlatAddressExpressions(Function &F) const {
     }
     // Otherwise, adds its operands to the stack and explores them.
     PostorderStack.back().setInt(true);
-    for (Value *PtrOperand : getPointerOperands(*TopVal)) {
+    for (Value *PtrOperand : getPointerOperands(*TopVal, *DL, TTI)) {
       appendsFlatAddressExpressionToPostorderStack(PtrOperand, PostorderStack,
                                                    Visited);
     }
@@ -536,6 +591,14 @@ Value *InferAddressSpaces::cloneInstructionWithNewAddressSpace(
     assert(I->getType()->isPointerTy());
     return SelectInst::Create(I->getOperand(0), NewPointerOperands[1],
                               NewPointerOperands[2], "", nullptr, I);
+  case Instruction::IntToPtr: {
+    assert(isNoopPtrIntCastPair(cast<Operator>(I), *DL, TTI));
+    Value *Src = cast<Operator>(I->getOperand(0))->getOperand(0);
+    assert(Src->getType()->getPointerAddressSpace() == NewAddrSpace);
+    if (Src->getType() != NewPtrType)
+      return new BitCastInst(Src, NewPtrType);
+    return Src;
+  }
   default:
     llvm_unreachable("Unexpected opcode");
   }
@@ -545,8 +608,9 @@ Value *InferAddressSpaces::cloneInstructionWithNewAddressSpace(
 // constant expression `CE` with its operands replaced as specified in
 // ValueWithNewAddrSpace.
 static Value *cloneConstantExprWithNewAddressSpace(
-  ConstantExpr *CE, unsigned NewAddrSpace,
-  const ValueToValueMapTy &ValueWithNewAddrSpace) {
+    ConstantExpr *CE, unsigned NewAddrSpace,
+    const ValueToValueMapTy &ValueWithNewAddrSpace, const DataLayout *DL,
+    const TargetTransformInfo *TTI) {
   Type *TargetType =
     CE->getType()->getPointerElementType()->getPointerTo(NewAddrSpace);
 
@@ -577,6 +641,13 @@ static Value *cloneConstantExprWithNewAddressSpace(
     }
   }
 
+  if (CE->getOpcode() == Instruction::IntToPtr) {
+    assert(isNoopPtrIntCastPair(cast<Operator>(CE), *DL, TTI));
+    Constant *Src = cast<ConstantExpr>(CE->getOperand(0))->getOperand(0);
+    assert(Src->getType()->getPointerAddressSpace() == NewAddrSpace);
+    return ConstantExpr::getBitCast(Src, TargetType);
+  }
+
   // Computes the operands of the new constant expression.
   bool IsNew = false;
   SmallVector<Constant *, 4> NewOperands;
@@ -594,7 +665,7 @@ static Value *cloneConstantExprWithNewAddressSpace(
     }
     if (auto CExpr = dyn_cast<ConstantExpr>(Operand))
       if (Value *NewOperand = cloneConstantExprWithNewAddressSpace(
-              CExpr, NewAddrSpace, ValueWithNewAddrSpace)) {
+              CExpr, NewAddrSpace, ValueWithNewAddrSpace, DL, TTI)) {
         IsNew = true;
         NewOperands.push_back(cast<Constant>(NewOperand));
         continue;
@@ -629,7 +700,7 @@ Value *InferAddressSpaces::cloneValueWithNewAddressSpace(
   const ValueToValueMapTy &ValueWithNewAddrSpace,
   SmallVectorImpl<const Use *> *UndefUsesToFix) const {
   // All values in Postorder are flat address expressions.
-  assert(isAddressExpression(*V) &&
+  assert(isAddressExpression(*V, *DL, TTI) &&
          V->getType()->getPointerAddressSpace() == FlatAddrSpace);
 
   if (Instruction *I = dyn_cast<Instruction>(V)) {
@@ -645,7 +716,7 @@ Value *InferAddressSpaces::cloneValueWithNewAddressSpace(
   }
 
   return cloneConstantExprWithNewAddressSpace(
-    cast<ConstantExpr>(V), NewAddrSpace, ValueWithNewAddrSpace);
+      cast<ConstantExpr>(V), NewAddrSpace, ValueWithNewAddrSpace, DL, TTI);
 }
 
 // Defines the join operation on the address space lattice (see the file header
@@ -669,6 +740,10 @@ bool InferAddressSpaces::runOnFunction(Function &F) {
     return false;
 
   TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
+  DL = &F.getParent()->getDataLayout();
+
+  if (AssumeDefaultIsFlatAddressSpace)
+    FlatAddrSpace = 0;
 
   if (FlatAddrSpace == UninitializedAddressSpace) {
     FlatAddrSpace = TTI->getFlatAddressSpace();
@@ -773,7 +848,7 @@ Optional<unsigned> InferAddressSpaces::updateAddressSpace(
     else
       NewAS = joinAddressSpaces(Src0AS, Src1AS);
   } else {
-    for (Value *PtrOperand : getPointerOperands(V)) {
+    for (Value *PtrOperand : getPointerOperands(V, *DL, TTI)) {
       auto I = InferredAddrSpace.find(PtrOperand);
       unsigned OperandAS = I != InferredAddrSpace.end() ?
         I->second : PtrOperand->getType()->getPointerAddressSpace();
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index d73b15522054d..1a22edaf87266 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -153,7 +153,7 @@ static bool isSafeToExecuteUnconditionally(Instruction &Inst,
                                            const Instruction *CtxI = nullptr);
 static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
                                      AliasSetTracker *CurAST, Loop *CurLoop,
-                                     AliasAnalysis *AA);
+                                     AAResults *AA);
 static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
                                              Loop *CurLoop,
                                              SinkAndHoistLICMFlags &Flags);
@@ -170,7 +170,7 @@ static void moveInstructionBefore(Instruction &I, Instruction &Dest,
 
 namespace {
 struct LoopInvariantCodeMotion {
-  bool runOnLoop(Loop *L, AliasAnalysis *AA, LoopInfo *LI, DominatorTree *DT,
+  bool runOnLoop(Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
                  TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
                  ScalarEvolution *SE, MemorySSA *MSSA,
                  OptimizationRemarkEmitter *ORE);
@@ -185,9 +185,9 @@ struct LoopInvariantCodeMotion {
   unsigned LicmMssaNoAccForPromotionCap;
 
   std::unique_ptr<AliasSetTracker>
-  collectAliasInfoForLoop(Loop *L, LoopInfo *LI, AliasAnalysis *AA);
+  collectAliasInfoForLoop(Loop *L, LoopInfo *LI, AAResults *AA);
   std::unique_ptr<AliasSetTracker>
-  collectAliasInfoForLoopWithMSSA(Loop *L, AliasAnalysis *AA,
+  collectAliasInfoForLoopWithMSSA(Loop *L, AAResults *AA,
                                   MemorySSAUpdater *MSSAU);
 };
 
@@ -285,7 +285,7 @@ Pass *llvm::createLICMPass(unsigned LicmMssaOptCap,
 /// loop is not preserved so it is not a good idea to run LICM multiple
 /// times on one loop.
 bool LoopInvariantCodeMotion::runOnLoop(
-    Loop *L, AliasAnalysis *AA, LoopInfo *LI, DominatorTree *DT,
+    Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
     TargetLibraryInfo *TLI, TargetTransformInfo *TTI, ScalarEvolution *SE,
     MemorySSA *MSSA, OptimizationRemarkEmitter *ORE) {
   bool Changed = false;
@@ -448,7 +448,7 @@ bool LoopInvariantCodeMotion::runOnLoop(
 /// first order w.r.t the DominatorTree.  This allows us to visit uses before
 /// definitions, allowing us to sink a loop body in one pass without iteration.
 ///
-bool llvm::sinkRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
+bool llvm::sinkRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
                       DominatorTree *DT, TargetLibraryInfo *TLI,
                       TargetTransformInfo *TTI, Loop *CurLoop,
                       AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
@@ -751,7 +751,7 @@ class ControlFlowHoister {
 /// order w.r.t the DominatorTree.  This allows us to visit definitions before
 /// uses, allowing us to hoist a loop body in one pass without iteration.
 ///
-bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
+bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
                        DominatorTree *DT, TargetLibraryInfo *TLI, Loop *CurLoop,
                        AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
                        ScalarEvolution *SE, ICFLoopSafetyInfo *SafetyInfo,
@@ -1108,11 +1108,11 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
     FunctionModRefBehavior Behavior = AA->getModRefBehavior(CI);
     if (Behavior == FMRB_DoesNotAccessMemory)
       return true;
-    if (AliasAnalysis::onlyReadsMemory(Behavior)) {
+    if (AAResults::onlyReadsMemory(Behavior)) {
       // A readonly argmemonly function only reads from memory pointed to by
       // it's arguments with arbitrary offsets.  If we can prove there are no
       // writes to this memory in the loop, we can hoist or sink.
-      if (AliasAnalysis::onlyAccessesArgPointees(Behavior)) {
+      if (AAResults::onlyAccessesArgPointees(Behavior)) {
         // TODO: expand to writeable arguments
         for (Value *Op : CI->arg_operands())
           if (Op->getType()->isPointerTy()) {
@@ -1894,7 +1894,7 @@ bool llvm::promoteLoopAccessesToScalars(
 
   // We start with an alignment of one and try to find instructions that allow
   // us to prove better alignment.
-  unsigned Alignment = 1;
+  Align Alignment;
   // Keep track of which types of access we see
   bool SawUnorderedAtomic = false;
   bool SawNotAtomic = false;
@@ -1942,10 +1942,7 @@ bool llvm::promoteLoopAccessesToScalars(
         SawUnorderedAtomic |= Load->isAtomic();
         SawNotAtomic |= !Load->isAtomic();
 
-        unsigned InstAlignment = Load->getAlignment();
-        if (!InstAlignment)
-          InstAlignment =
-              MDL.getABITypeAlignment(Load->getType());
+        Align InstAlignment = Load->getAlign();
 
         // Note that proving a load safe to speculate requires proving
         // sufficient alignment at the target location.  Proving it guaranteed
@@ -1973,10 +1970,7 @@ bool llvm::promoteLoopAccessesToScalars(
         // already know that promotion is safe, since it may have higher
         // alignment than any other guaranteed stores, in which case we can
         // raise the alignment on the promoted store.
-        unsigned InstAlignment = Store->getAlignment();
-        if (!InstAlignment)
-          InstAlignment =
-              MDL.getABITypeAlignment(Store->getValueOperand()->getType());
+        Align InstAlignment = Store->getAlign();
 
         if (!DereferenceableInPH || !SafeToInsertStore ||
             (InstAlignment > Alignment)) {
@@ -2079,7 +2073,8 @@ bool llvm::promoteLoopAccessesToScalars(
   SSAUpdater SSA(&NewPHIs);
   LoopPromoter Promoter(SomePtr, LoopUses, SSA, PointerMustAliases, ExitBlocks,
                         InsertPts, MSSAInsertPts, PIC, *CurAST, MSSAU, *LI, DL,
-                        Alignment, SawUnorderedAtomic, AATags, *SafetyInfo);
+                        Alignment.value(), SawUnorderedAtomic, AATags,
+                        *SafetyInfo);
 
   // Set up the preheader to have a definition of the value.  It is the live-out
   // value from the preheader that uses in the loop will use.
@@ -2088,7 +2083,7 @@ bool llvm::promoteLoopAccessesToScalars(
       SomePtr->getName() + ".promoted", Preheader->getTerminator());
   if (SawUnorderedAtomic)
     PreheaderLoad->setOrdering(AtomicOrdering::Unordered);
-  PreheaderLoad->setAlignment(Align(Alignment));
+  PreheaderLoad->setAlignment(Alignment);
   PreheaderLoad->setDebugLoc(DebugLoc());
   if (AATags)
     PreheaderLoad->setAAMetadata(AATags);
@@ -2120,7 +2115,7 @@ bool llvm::promoteLoopAccessesToScalars(
 /// from L and all subloops of L.
 std::unique_ptr<AliasSetTracker>
 LoopInvariantCodeMotion::collectAliasInfoForLoop(Loop *L, LoopInfo *LI,
-                                                 AliasAnalysis *AA) {
+                                                 AAResults *AA) {
   auto CurAST = std::make_unique<AliasSetTracker>(*AA);
 
   // Add everything from all the sub loops.
@@ -2138,7 +2133,7 @@ LoopInvariantCodeMotion::collectAliasInfoForLoop(Loop *L, LoopInfo *LI,
 
 std::unique_ptr<AliasSetTracker>
 LoopInvariantCodeMotion::collectAliasInfoForLoopWithMSSA(
-    Loop *L, AliasAnalysis *AA, MemorySSAUpdater *MSSAU) {
+    Loop *L, AAResults *AA, MemorySSAUpdater *MSSAU) {
   auto *MSSA = MSSAU->getMemorySSA();
   auto CurAST = std::make_unique<AliasSetTracker>(*AA, MSSA, L);
   CurAST->addAllInstructionsInLoopUsingMSSA();
@@ -2147,7 +2142,7 @@ LoopInvariantCodeMotion::collectAliasInfoForLoopWithMSSA(
 
 static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
                                      AliasSetTracker *CurAST, Loop *CurLoop,
-                                     AliasAnalysis *AA) {
+                                     AAResults *AA) {
   // First check to see if any of the basic blocks in CurLoop invalidate *V.
   bool isInvalidatedAccordingToAST = CurAST->getAliasSetFor(MemLoc).isMod();
 
diff --git a/llvm/lib/Transforms/Scalar/LoopDeletion.cpp b/llvm/lib/Transforms/Scalar/LoopDeletion.cpp
index 19493405c41c7..be209d34be422 100644
--- a/llvm/lib/Transforms/Scalar/LoopDeletion.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopDeletion.cpp
@@ -19,6 +19,7 @@
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/InitializePasses.h"
@@ -136,7 +137,8 @@ static bool isLoopNeverExecuted(Loop *L) {
 /// instructions out of the loop.
 static LoopDeletionResult deleteLoopIfDead(Loop *L, DominatorTree &DT,
                                            ScalarEvolution &SE, LoopInfo &LI,
-                                           MemorySSA *MSSA) {
+                                           MemorySSA *MSSA,
+                                           OptimizationRemarkEmitter &ORE) {
   assert(L->isLCSSAForm(DT) && "Expected LCSSA!");
 
   // We can only remove the loop if there is a preheader that we can branch from
@@ -166,6 +168,11 @@ static LoopDeletionResult deleteLoopIfDead(Loop *L, DominatorTree &DT,
       std::fill(P.incoming_values().begin(), P.incoming_values().end(),
                 UndefValue::get(P.getType()));
     }
+    ORE.emit([&]() {
+      return OptimizationRemark(DEBUG_TYPE, "NeverExecutes", L->getStartLoc(),
+                                L->getHeader())
+             << "Loop deleted because it never executes";
+    });
     deleteDeadLoop(L, &DT, &SE, &LI, MSSA);
     ++NumDeleted;
     return LoopDeletionResult::Deleted;
@@ -202,6 +209,11 @@ static LoopDeletionResult deleteLoopIfDead(Loop *L, DominatorTree &DT,
   }
 
   LLVM_DEBUG(dbgs() << "Loop is invariant, delete it!");
+  ORE.emit([&]() {
+    return OptimizationRemark(DEBUG_TYPE, "Invariant", L->getStartLoc(),
+                              L->getHeader())
+           << "Loop deleted because it is invariant";
+  });
   deleteDeadLoop(L, &DT, &SE, &LI, MSSA);
   ++NumDeleted;
 
@@ -215,7 +227,11 @@ PreservedAnalyses LoopDeletionPass::run(Loop &L, LoopAnalysisManager &AM,
   LLVM_DEBUG(dbgs() << "Analyzing Loop for deletion: ");
   LLVM_DEBUG(L.dump());
   std::string LoopName = std::string(L.getName());
-  auto Result = deleteLoopIfDead(&L, AR.DT, AR.SE, AR.LI, AR.MSSA);
+  // For the new PM, we can't use OptimizationRemarkEmitter as an analysis
+  // pass. Function analyses need to be preserved across loop transformations
+  // but ORE cannot be preserved (see comment before the pass definition).
+  OptimizationRemarkEmitter ORE(L.getHeader()->getParent());
+  auto Result = deleteLoopIfDead(&L, AR.DT, AR.SE, AR.LI, AR.MSSA, ORE);
   if (Result == LoopDeletionResult::Unmodified)
     return PreservedAnalyses::all();
 
@@ -265,11 +281,15 @@ bool LoopDeletionLegacyPass::runOnLoop(Loop *L, LPPassManager &LPM) {
   MemorySSA *MSSA = nullptr;
   if (MSSAAnalysis)
     MSSA = &MSSAAnalysis->getMSSA();
+  // For the old PM, we can't use OptimizationRemarkEmitter as an analysis
+  // pass.  Function analyses need to be preserved across loop transformations
+  // but ORE cannot be preserved (see comment before the pass definition).
+  OptimizationRemarkEmitter ORE(L->getHeader()->getParent());
 
   LLVM_DEBUG(dbgs() << "Analyzing Loop for deletion: ");
   LLVM_DEBUG(L->dump());
 
-  LoopDeletionResult Result = deleteLoopIfDead(L, DT, SE, LI, MSSA);
+  LoopDeletionResult Result = deleteLoopIfDead(L, DT, SE, LI, MSSA, ORE);
 
   if (Result == LoopDeletionResult::Deleted)
     LPM.markLoopAsDeleted(*L);
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index e29bf6325850d..20edc8699d797 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -743,8 +743,8 @@ struct LoopFuser {
           }
 
           if (!isSafeToMoveBefore(*FC1->Preheader,
-                                  *FC0->Preheader->getTerminator(), DT, PDT,
-                                  DI)) {
+                                  *FC0->Preheader->getTerminator(), DT, &PDT,
+                                  &DI)) {
             LLVM_DEBUG(dbgs() << "Fusion candidate contains unsafe "
                                  "instructions in preheader. Not fusing.\n");
             reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1,
@@ -757,7 +757,7 @@ struct LoopFuser {
 
             if (!isSafeToMoveBefore(*FC0->ExitBlock,
                                     *FC1->ExitBlock->getFirstNonPHIOrDbg(), DT,
-                                    PDT, DI)) {
+                                    &PDT, &DI)) {
               LLVM_DEBUG(dbgs() << "Fusion candidate contains unsafe "
                                    "instructions in exit block. Not fusing.\n");
               reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1,
@@ -767,8 +767,8 @@ struct LoopFuser {
 
             if (!isSafeToMoveBefore(
                     *FC1->GuardBranch->getParent(),
-                    *FC0->GuardBranch->getParent()->getTerminator(), DT, PDT,
-                    DI)) {
+                    *FC0->GuardBranch->getParent()->getTerminator(), DT, &PDT,
+                    &DI)) {
               LLVM_DEBUG(dbgs()
                          << "Fusion candidate contains unsafe "
                             "instructions in guard block. Not fusing.\n");
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index e1e6f7092b8b5..3cb4df12e9b08 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -1553,9 +1553,9 @@ bool LoopIdiomRecognize::recognizeAndInsertFFS() {
   //  %inc = add nsw %i.0, 1
   //  br i1 %tobool
 
-  Value *Args[] =
-      {InitX, ZeroCheck ? ConstantInt::getTrue(InitX->getContext())
-                        : ConstantInt::getFalse(InitX->getContext())};
+  const Value *Args[] = {
+      InitX, ZeroCheck ? ConstantInt::getTrue(InitX->getContext())
+                       : ConstantInt::getFalse(InitX->getContext())};
 
   // @llvm.dbg doesn't count as they have no semantic effect.
   auto InstWithoutDebugIt = CurLoop->getHeader()->instructionsWithoutDebug();
diff --git a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
index cc083c2fae70d..98889a9df1163 100644
--- a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
@@ -34,14 +34,14 @@ PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
   // instrumenting callbacks for the passes later.
   PassInstrumentation PI = AM.getResult<PassInstrumentationAnalysis>(L, AR);
   for (auto &Pass : Passes) {
-    if (DebugLogging)
-      dbgs() << "Running pass: " << Pass->name() << " on " << L;
-
     // Check the PassInstrumentation's BeforePass callbacks before running the
     // pass, skip its execution completely if asked to (callback returns false).
     if (!PI.runBeforePass<Loop>(*Pass, L))
       continue;
 
+    if (DebugLogging)
+      dbgs() << "Running pass: " << Pass->name() << " on " << L;
+
     PreservedAnalyses PassPA;
     {
       TimeTraceScope TimeScope(Pass->name(), L.getName());
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
index f0ece1faa5fd5..285cba6ee2054 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
@@ -158,7 +158,8 @@ static bool computeUnrollAndJamCount(
     const SmallPtrSetImpl<const Value *> &EphValues,
     OptimizationRemarkEmitter *ORE, unsigned OuterTripCount,
     unsigned OuterTripMultiple, unsigned OuterLoopSize, unsigned InnerTripCount,
-    unsigned InnerLoopSize, TargetTransformInfo::UnrollingPreferences &UP) {
+    unsigned InnerLoopSize, TargetTransformInfo::UnrollingPreferences &UP,
+    TargetTransformInfo::PeelingPreferences &PP) {
   // First up use computeUnrollCount from the loop unroller to get a count
   // for unrolling the outer loop, plus any loops requiring explicit
   // unrolling we leave to the unroller. This uses UP.Threshold /
@@ -168,7 +169,8 @@ static bool computeUnrollAndJamCount(
   bool UseUpperBound = false;
   bool ExplicitUnroll = computeUnrollCount(
       L, TTI, DT, LI, SE, EphValues, ORE, OuterTripCount, MaxTripCount,
-      /*MaxOrZero*/ false, OuterTripMultiple, OuterLoopSize, UP, UseUpperBound);
+      /*MaxOrZero*/ false, OuterTripMultiple, OuterLoopSize, UP, PP,
+      UseUpperBound);
   if (ExplicitUnroll || UseUpperBound) {
     // If the user explicitly set the loop as unrolled, dont UnJ it. Leave it
     // for the unroller instead.
@@ -282,7 +284,9 @@ tryToUnrollAndJamLoop(Loop *L, DominatorTree &DT, LoopInfo *LI,
                       OptimizationRemarkEmitter &ORE, int OptLevel) {
   TargetTransformInfo::UnrollingPreferences UP =
       gatherUnrollingPreferences(L, SE, TTI, nullptr, nullptr, OptLevel, None,
-                                 None, None, None, None, None, None, None);
+                                 None, None, None, None, None);
+  TargetTransformInfo::PeelingPreferences PP =
+      gatherPeelingPreferences(L, SE, TTI, None, None);
   if (AllowUnrollAndJam.getNumOccurrences() > 0)
     UP.UnrollAndJam = AllowUnrollAndJam;
   if (UnrollAndJamThreshold.getNumOccurrences() > 0)
@@ -367,7 +371,7 @@ tryToUnrollAndJamLoop(Loop *L, DominatorTree &DT, LoopInfo *LI,
   // Decide if, and by how much, to unroll
   bool IsCountSetExplicitly = computeUnrollAndJamCount(
       L, SubLoop, TTI, DT, LI, SE, EphValues, &ORE, OuterTripCount,
-      OuterTripMultiple, OuterLoopSize, InnerTripCount, InnerLoopSize, UP);
+      OuterTripMultiple, OuterLoopSize, InnerTripCount, InnerLoopSize, UP, PP);
   if (UP.Count <= 1)
     return LoopUnrollResult::Unmodified;
   // Unroll factor (Count) must be less or equal to TripCount.
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index ec56610e41e53..87f40bb7ba852 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -193,9 +193,7 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
     BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, int OptLevel,
     Optional<unsigned> UserThreshold, Optional<unsigned> UserCount,
     Optional<bool> UserAllowPartial, Optional<bool> UserRuntime,
-    Optional<bool> UserUpperBound, Optional<bool> UserAllowPeeling,
-    Optional<bool> UserAllowProfileBasedPeeling,
-    Optional<unsigned> UserFullUnrollMaxCount) {
+    Optional<bool> UserUpperBound, Optional<unsigned> UserFullUnrollMaxCount) {
   TargetTransformInfo::UnrollingPreferences UP;
 
   // Set up the defaults
@@ -206,7 +204,6 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
   UP.PartialThreshold = 150;
   UP.PartialOptSizeThreshold = 0;
   UP.Count = 0;
-  UP.PeelCount = 0;
   UP.DefaultUnrollRuntimeCount = 8;
   UP.MaxCount = std::numeric_limits<unsigned>::max();
   UP.FullUnrollMaxCount = std::numeric_limits<unsigned>::max();
@@ -218,10 +215,7 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
   UP.AllowExpensiveTripCount = false;
   UP.Force = false;
   UP.UpperBound = false;
-  UP.AllowPeeling = true;
-  UP.AllowLoopNestsPeeling = false;
   UP.UnrollAndJam = false;
-  UP.PeelProfiledIterations = true;
   UP.UnrollAndJamInnerLoopThreshold = 60;
   UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;
 
@@ -249,8 +243,6 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
     UP.MaxCount = UnrollMaxCount;
   if (UnrollFullMaxCount.getNumOccurrences() > 0)
     UP.FullUnrollMaxCount = UnrollFullMaxCount;
-  if (UnrollPeelCount.getNumOccurrences() > 0)
-    UP.PeelCount = UnrollPeelCount;
   if (UnrollAllowPartial.getNumOccurrences() > 0)
     UP.Partial = UnrollAllowPartial;
   if (UnrollAllowRemainder.getNumOccurrences() > 0)
@@ -259,10 +251,6 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
     UP.Runtime = UnrollRuntime;
   if (UnrollMaxUpperBound == 0)
     UP.UpperBound = false;
-  if (UnrollAllowPeeling.getNumOccurrences() > 0)
-    UP.AllowPeeling = UnrollAllowPeeling;
-  if (UnrollAllowLoopNestsPeeling.getNumOccurrences() > 0)
-    UP.AllowLoopNestsPeeling = UnrollAllowLoopNestsPeeling;
   if (UnrollUnrollRemainder.getNumOccurrences() > 0)
     UP.UnrollRemainder = UnrollUnrollRemainder;
   if (UnrollMaxIterationsCountToAnalyze.getNumOccurrences() > 0)
@@ -281,16 +269,45 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
     UP.Runtime = *UserRuntime;
   if (UserUpperBound.hasValue())
     UP.UpperBound = *UserUpperBound;
-  if (UserAllowPeeling.hasValue())
-    UP.AllowPeeling = *UserAllowPeeling;
-  if (UserAllowProfileBasedPeeling.hasValue())
-    UP.PeelProfiledIterations = *UserAllowProfileBasedPeeling;
   if (UserFullUnrollMaxCount.hasValue())
     UP.FullUnrollMaxCount = *UserFullUnrollMaxCount;
 
   return UP;
 }
 
+TargetTransformInfo::PeelingPreferences
+llvm::gatherPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                               const TargetTransformInfo &TTI,
+                               Optional<bool> UserAllowPeeling,
+                               Optional<bool> UserAllowProfileBasedPeeling) {
+  TargetTransformInfo::PeelingPreferences PP;
+
+  // Default values
+  PP.PeelCount = 0;
+  PP.AllowPeeling = true;
+  PP.AllowLoopNestsPeeling = false;
+  PP.PeelProfiledIterations = true;
+
+  // Get Target Specifc Values
+  TTI.getPeelingPreferences(L, SE, PP);
+
+  // User Specified Values using cl::opt
+  if (UnrollPeelCount.getNumOccurrences() > 0)
+    PP.PeelCount = UnrollPeelCount;
+  if (UnrollAllowPeeling.getNumOccurrences() > 0)
+    PP.AllowPeeling = UnrollAllowPeeling;
+  if (UnrollAllowLoopNestsPeeling.getNumOccurrences() > 0)
+    PP.AllowLoopNestsPeeling = UnrollAllowLoopNestsPeeling;
+
+  // User Specifed values provided by argument
+  if (UserAllowPeeling.hasValue())
+    PP.AllowPeeling = *UserAllowPeeling;
+  if (UserAllowProfileBasedPeeling.hasValue())
+    PP.PeelProfiledIterations = *UserAllowProfileBasedPeeling;
+
+  return PP;
+}
+
 namespace {
 
 /// A struct to densely store the state of an instruction after unrolling at
@@ -761,7 +778,8 @@ bool llvm::computeUnrollCount(
     ScalarEvolution &SE, const SmallPtrSetImpl<const Value *> &EphValues,
     OptimizationRemarkEmitter *ORE, unsigned &TripCount, unsigned MaxTripCount,
     bool MaxOrZero, unsigned &TripMultiple, unsigned LoopSize,
-    TargetTransformInfo::UnrollingPreferences &UP, bool &UseUpperBound) {
+    TargetTransformInfo::UnrollingPreferences &UP,
+    TargetTransformInfo::PeelingPreferences &PP, bool &UseUpperBound) {
 
   // Check for explicit Count.
   // 1st priority is unroll count set by "unroll-count" option.
@@ -863,8 +881,8 @@ bool llvm::computeUnrollCount(
   }
 
   // 4th priority is loop peeling.
-  computePeelCount(L, LoopSize, UP, TripCount, SE);
-  if (UP.PeelCount) {
+  computePeelCount(L, LoopSize, UP, PP, TripCount, SE);
+  if (PP.PeelCount) {
     UP.Runtime = false;
     UP.Count = 1;
     return ExplicitUnroll;
@@ -1067,8 +1085,9 @@ static LoopUnrollResult tryToUnrollLoop(
   TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences(
       L, SE, TTI, BFI, PSI, OptLevel, ProvidedThreshold, ProvidedCount,
       ProvidedAllowPartial, ProvidedRuntime, ProvidedUpperBound,
-      ProvidedAllowPeeling, ProvidedAllowProfileBasedPeeling,
       ProvidedFullUnrollMaxCount);
+  TargetTransformInfo::PeelingPreferences PP = gatherPeelingPreferences(
+      L, SE, TTI, ProvidedAllowPeeling, ProvidedAllowProfileBasedPeeling);
 
   // Exit early if unrolling is disabled. For OptForSize, we pick the loop size
   // as threshold later on.
@@ -1142,7 +1161,7 @@ static LoopUnrollResult tryToUnrollLoop(
   bool UseUpperBound = false;
   bool IsCountSetExplicitly = computeUnrollCount(
       L, TTI, DT, LI, SE, EphValues, &ORE, TripCount, MaxTripCount, MaxOrZero,
-      TripMultiple, LoopSize, UP, UseUpperBound);
+      TripMultiple, LoopSize, UP, PP, UseUpperBound);
   if (!UP.Count)
     return LoopUnrollResult::Unmodified;
   // Unroll factor (Count) must be less or equal to TripCount.
@@ -1157,7 +1176,7 @@ static LoopUnrollResult tryToUnrollLoop(
   LoopUnrollResult UnrollResult = UnrollLoop(
       L,
       {UP.Count, TripCount, UP.Force, UP.Runtime, UP.AllowExpensiveTripCount,
-       UseUpperBound, MaxOrZero, TripMultiple, UP.PeelCount, UP.UnrollRemainder,
+       UseUpperBound, MaxOrZero, TripMultiple, PP.PeelCount, UP.UnrollRemainder,
        ForgetAllSCEV},
       LI, &SE, &DT, &AC, &TTI, &ORE, PreserveLCSSA, &RemainderLoop);
   if (UnrollResult == LoopUnrollResult::Unmodified)
@@ -1189,7 +1208,7 @@ static LoopUnrollResult tryToUnrollLoop(
   // If the loop was peeled, we already "used up" the profile information
   // we had, so we don't want to unroll or peel again.
   if (UnrollResult != LoopUnrollResult::FullyUnrolled &&
-      (IsCountSetExplicitly || (UP.PeelProfiledIterations && UP.PeelCount)))
+      (IsCountSetExplicitly || (PP.PeelProfiledIterations && PP.PeelCount)))
     L->setLoopAlreadyUnrolled();
 
   return UnrollResult;
diff --git a/llvm/lib/Transforms/Scalar/LowerConstantIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerConstantIntrinsics.cpp
index f59280b60d8da..fddf28c281fc6 100644
--- a/llvm/lib/Transforms/Scalar/LowerConstantIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerConstantIntrinsics.cpp
@@ -15,6 +15,7 @@
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
@@ -136,8 +137,12 @@ static bool lowerConstantIntrinsics(Function &F, const TargetLibraryInfo *TLI) {
 
 PreservedAnalyses
 LowerConstantIntrinsicsPass::run(Function &F, FunctionAnalysisManager &AM) {
-  if (lowerConstantIntrinsics(F, AM.getCachedResult<TargetLibraryAnalysis>(F)))
-    return PreservedAnalyses::none();
+  if (lowerConstantIntrinsics(F,
+                              AM.getCachedResult<TargetLibraryAnalysis>(F))) {
+    PreservedAnalyses PA;
+    PA.preserve<GlobalsAA>();
+    return PA;
+  }
 
   return PreservedAnalyses::all();
 }
@@ -160,6 +165,10 @@ class LowerConstantIntrinsics : public FunctionPass {
     const TargetLibraryInfo *TLI = TLIP ? &TLIP->getTLI(F) : nullptr;
     return lowerConstantIntrinsics(F, TLI);
   }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addPreserved<GlobalsAAWrapperPass>();
+  }
 };
 } // namespace
 
diff --git a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
index 53671c7bc3d15..0fe7dd9cfb39f 100644
--- a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
@@ -55,13 +55,35 @@ static cl::opt<uint32_t> UnlikelyBranchWeight(
     "unlikely-branch-weight", cl::Hidden, cl::init(1),
     cl::desc("Weight of the branch unlikely to be taken (default = 1)"));
 
+static std::tuple<uint32_t, uint32_t>
+getBranchWeight(Intrinsic::ID IntrinsicID, CallInst *CI, int BranchCount) {
+  if (IntrinsicID == Intrinsic::expect) {
+    // __builtin_expect
+    return std::make_tuple(LikelyBranchWeight.getValue(),
+                           UnlikelyBranchWeight.getValue());
+  } else {
+    // __builtin_expect_with_probability
+    assert(CI->getNumOperands() >= 3 &&
+           "expect with probability must have 3 arguments");
+    ConstantFP *Confidence = dyn_cast<ConstantFP>(CI->getArgOperand(2));
+    double TrueProb = Confidence->getValueAPF().convertToDouble();
+    assert((TrueProb >= 0.0 && TrueProb <= 1.0) &&
+           "probability value must be in the range [0.0, 1.0]");
+    double FalseProb = (1.0 - TrueProb) / (BranchCount - 1);
+    uint32_t LikelyBW = ceil((TrueProb * (double)(INT32_MAX - 1)) + 1.0);
+    uint32_t UnlikelyBW = ceil((FalseProb * (double)(INT32_MAX - 1)) + 1.0);
+    return std::make_tuple(LikelyBW, UnlikelyBW);
+  }
+}
+
 static bool handleSwitchExpect(SwitchInst &SI) {
   CallInst *CI = dyn_cast<CallInst>(SI.getCondition());
   if (!CI)
     return false;
 
   Function *Fn = CI->getCalledFunction();
-  if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect)
+  if (!Fn || (Fn->getIntrinsicID() != Intrinsic::expect &&
+              Fn->getIntrinsicID() != Intrinsic::expect_with_probability))
     return false;
 
   Value *ArgValue = CI->getArgOperand(0);
@@ -71,15 +93,19 @@ static bool handleSwitchExpect(SwitchInst &SI) {
 
   SwitchInst::CaseHandle Case = *SI.findCaseValue(ExpectedValue);
   unsigned n = SI.getNumCases(); // +1 for default case.
-  SmallVector<uint32_t, 16> Weights(n + 1, UnlikelyBranchWeight);
+  uint32_t LikelyBranchWeightVal, UnlikelyBranchWeightVal;
+  std::tie(LikelyBranchWeightVal, UnlikelyBranchWeightVal) =
+      getBranchWeight(Fn->getIntrinsicID(), CI, n + 1);
+
+  SmallVector<uint32_t, 16> Weights(n + 1, UnlikelyBranchWeightVal);
 
   uint64_t Index = (Case == *SI.case_default()) ? 0 : Case.getCaseIndex() + 1;
-  Weights[Index] = LikelyBranchWeight;
+  Weights[Index] = LikelyBranchWeightVal;
 
-  SI.setMetadata(
-      LLVMContext::MD_misexpect,
-      MDBuilder(CI->getContext())
-          .createMisExpect(Index, LikelyBranchWeight, UnlikelyBranchWeight));
+  SI.setMetadata(LLVMContext::MD_misexpect,
+                 MDBuilder(CI->getContext())
+                     .createMisExpect(Index, LikelyBranchWeightVal,
+                                      UnlikelyBranchWeightVal));
 
   SI.setCondition(ArgValue);
   misexpect::checkFrontendInstrumentation(SI);
@@ -223,15 +249,18 @@ static void handlePhiDef(CallInst *Expect) {
         return true;
       return false;
     };
+    uint32_t LikelyBranchWeightVal, UnlikelyBranchWeightVal;
+    std::tie(LikelyBranchWeightVal, UnlikelyBranchWeightVal) = getBranchWeight(
+        Expect->getCalledFunction()->getIntrinsicID(), Expect, 2);
 
     if (IsOpndComingFromSuccessor(BI->getSuccessor(1)))
-      BI->setMetadata(
-          LLVMContext::MD_prof,
-          MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight));
+      BI->setMetadata(LLVMContext::MD_prof,
+                      MDB.createBranchWeights(LikelyBranchWeightVal,
+                                              UnlikelyBranchWeightVal));
     else if (IsOpndComingFromSuccessor(BI->getSuccessor(0)))
-      BI->setMetadata(
-          LLVMContext::MD_prof,
-          MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight));
+      BI->setMetadata(LLVMContext::MD_prof,
+                      MDB.createBranchWeights(UnlikelyBranchWeightVal,
+                                              LikelyBranchWeightVal));
   }
 }
 
@@ -277,7 +306,8 @@ template <class BrSelInst> static bool handleBrSelExpect(BrSelInst &BSI) {
   }
 
   Function *Fn = CI->getCalledFunction();
-  if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect)
+  if (!Fn || (Fn->getIntrinsicID() != Intrinsic::expect &&
+              Fn->getIntrinsicID() != Intrinsic::expect_with_probability))
     return false;
 
   Value *ArgValue = CI->getArgOperand(0);
@@ -289,13 +319,21 @@ template <class BrSelInst> static bool handleBrSelExpect(BrSelInst &BSI) {
   MDNode *Node;
   MDNode *ExpNode;
 
+  uint32_t LikelyBranchWeightVal, UnlikelyBranchWeightVal;
+  std::tie(LikelyBranchWeightVal, UnlikelyBranchWeightVal) =
+      getBranchWeight(Fn->getIntrinsicID(), CI, 2);
+
   if ((ExpectedValue->getZExtValue() == ValueComparedTo) ==
       (Predicate == CmpInst::ICMP_EQ)) {
-    Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight);
-    ExpNode = MDB.createMisExpect(0, LikelyBranchWeight, UnlikelyBranchWeight);
+    Node =
+        MDB.createBranchWeights(LikelyBranchWeightVal, UnlikelyBranchWeightVal);
+    ExpNode =
+        MDB.createMisExpect(0, LikelyBranchWeightVal, UnlikelyBranchWeightVal);
   } else {
-    Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight);
-    ExpNode = MDB.createMisExpect(1, LikelyBranchWeight, UnlikelyBranchWeight);
+    Node =
+        MDB.createBranchWeights(UnlikelyBranchWeightVal, LikelyBranchWeightVal);
+    ExpNode =
+        MDB.createMisExpect(1, LikelyBranchWeightVal, UnlikelyBranchWeightVal);
   }
 
   BSI.setMetadata(LLVMContext::MD_misexpect, ExpNode);
@@ -347,7 +385,8 @@ static bool lowerExpectIntrinsic(Function &F) {
       }
 
       Function *Fn = CI->getCalledFunction();
-      if (Fn && Fn->getIntrinsicID() == Intrinsic::expect) {
+      if (Fn && (Fn->getIntrinsicID() == Intrinsic::expect ||
+                 Fn->getIntrinsicID() == Intrinsic::expect_with_probability)) {
         // Before erasing the llvm.expect, walk backward to find
         // phi that define llvm.expect's first arg, and
         // infer branch probability:
diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index c663714ea66d6..90314b17b5e25 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -38,6 +38,7 @@
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Alignment.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
@@ -253,13 +254,13 @@ class LowerMatrixIntrinsics {
         return Vectors.size();
       else {
         assert(Vectors.size() > 0 && "Cannot call getNumRows without columns");
-        return cast<VectorType>(Vectors[0]->getType())->getNumElements();
+        return cast<FixedVectorType>(Vectors[0]->getType())->getNumElements();
       }
     }
     unsigned getNumRows() const {
       if (isColumnMajor()) {
         assert(Vectors.size() > 0 && "Cannot call getNumRows without columns");
-        return cast<VectorType>(Vectors[0]->getType())->getNumElements();
+        return cast<FixedVectorType>(Vectors[0]->getType())->getNumElements();
       } else
         return Vectors.size();
     }
@@ -400,7 +401,7 @@ class LowerMatrixIntrinsics {
   unsigned getNumOps(Type *VT) {
     assert(isa<VectorType>(VT) && "Expected vector type");
     return getNumOps(VT->getScalarType(),
-                     cast<VectorType>(VT)->getNumElements());
+                     cast<FixedVectorType>(VT)->getNumElements());
   }
 
   //
@@ -420,7 +421,8 @@ class LowerMatrixIntrinsics {
                      IRBuilder<> &Builder) {
     VectorType *VType = dyn_cast<VectorType>(MatrixVal->getType());
     assert(VType && "MatrixVal must be a vector type");
-    assert(VType->getNumElements() == SI.NumRows * SI.NumColumns &&
+    assert(cast<FixedVectorType>(VType)->getNumElements() ==
+               SI.NumRows * SI.NumColumns &&
            "The vector size must match the number of matrix elements");
 
     // Check if we lowered MatrixVal using shape information. In that case,
@@ -441,7 +443,8 @@ class LowerMatrixIntrinsics {
     // Otherwise split MatrixVal.
     SmallVector<Value *, 16> SplitVecs;
     Value *Undef = UndefValue::get(VType);
-    for (unsigned MaskStart = 0; MaskStart < VType->getNumElements();
+    for (unsigned MaskStart = 0;
+         MaskStart < cast<FixedVectorType>(VType)->getNumElements();
          MaskStart += SI.getStride()) {
       Value *V = Builder.CreateShuffleVector(
           MatrixVal, Undef, createSequentialMask(MaskStart, SI.getStride(), 0),
@@ -927,8 +930,8 @@ class LowerMatrixIntrinsics {
 
     // First, bring Block to the same size as Col
     unsigned BlockNumElts =
-        cast<VectorType>(Block->getType())->getNumElements();
-    unsigned NumElts = cast<VectorType>(Col->getType())->getNumElements();
+        cast<FixedVectorType>(Block->getType())->getNumElements();
+    unsigned NumElts = cast<FixedVectorType>(Col->getType())->getNumElements();
     assert(NumElts >= BlockNumElts && "Too few elements for current block");
 
     Value *Undef = UndefValue::get(Block->getType());
@@ -943,7 +946,8 @@ class LowerMatrixIntrinsics {
     for (i = 0; i < I; i++)
       Mask.push_back(i);
 
-    unsigned VecNumElts = cast<VectorType>(Col->getType())->getNumElements();
+    unsigned VecNumElts =
+        cast<FixedVectorType>(Col->getType())->getNumElements();
     for (; i < I + BlockNumElts; i++)
       Mask.push_back(i - I + VecNumElts);
 
diff --git a/llvm/lib/Transforms/Scalar/NewGVN.cpp b/llvm/lib/Transforms/Scalar/NewGVN.cpp
index 576d40f0e0ff5..0ed1773373a71 100644
--- a/llvm/lib/Transforms/Scalar/NewGVN.cpp
+++ b/llvm/lib/Transforms/Scalar/NewGVN.cpp
@@ -3436,7 +3436,7 @@ bool NewGVN::runGVN() {
   // Sort dominator tree children arrays into RPO.
   for (auto &B : RPOT) {
     auto *Node = DT->getNode(B);
-    if (Node->getChildren().size() > 1)
+    if (Node->getNumChildren() > 1)
       llvm::sort(Node->begin(), Node->end(),
                  [&](const DomTreeNode *A, const DomTreeNode *B) {
                    return RPOOrdering[A] < RPOOrdering[B];
diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index dbccfda246b83..dc2ad14ae61e5 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -1352,7 +1352,8 @@ static void CreateGCRelocates(ArrayRef<Value *> LiveVariables,
     auto AS = Ty->getScalarType()->getPointerAddressSpace();
     Type *NewTy = Type::getInt8PtrTy(M->getContext(), AS);
     if (auto *VT = dyn_cast<VectorType>(Ty))
-      NewTy = FixedVectorType::get(NewTy, VT->getNumElements());
+      NewTy = FixedVectorType::get(NewTy,
+                                   cast<FixedVectorType>(VT)->getNumElements());
     return Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate,
                                      {NewTy});
   };
@@ -2667,8 +2668,9 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT,
     unsigned VF = 0;
     for (unsigned i = 0; i < I.getNumOperands(); i++)
       if (auto *OpndVTy = dyn_cast<VectorType>(I.getOperand(i)->getType())) {
-        assert(VF == 0 || VF == OpndVTy->getNumElements());
-        VF = OpndVTy->getNumElements();
+        assert(VF == 0 ||
+               VF == cast<FixedVectorType>(OpndVTy)->getNumElements());
+        VF = cast<FixedVectorType>(OpndVTy)->getNumElements();
       }
 
     // It's the vector to scalar traversal through the pointer operand which
diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp
index 432a5ef03cdcd..5ebd3b71fe78c 100644
--- a/llvm/lib/Transforms/Scalar/SCCP.cpp
+++ b/llvm/lib/Transforms/Scalar/SCCP.cpp
@@ -27,6 +27,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/ConstantFolding.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
@@ -1103,11 +1104,21 @@ void SCCPSolver::visitStoreInst(StoreInst &SI) {
     TrackedGlobals.erase(I);      // No need to keep tracking this!
 }
 
+static ValueLatticeElement getValueFromMetadata(const Instruction *I) {
+  if (MDNode *Ranges = I->getMetadata(LLVMContext::MD_range))
+    if (I->getType()->isIntegerTy())
+      return ValueLatticeElement::getRange(
+          getConstantRangeFromMetadata(*Ranges));
+  // TODO: Also handle MD_nonnull.
+  return ValueLatticeElement::getOverdefined();
+}
+
 // Handle load instructions.  If the operand is a constant pointer to a constant
 // global, we can replace the load with the loaded constant value!
 void SCCPSolver::visitLoadInst(LoadInst &I) {
-  // If this load is of a struct, just mark the result overdefined.
-  if (I.getType()->isStructTy())
+  // If this load is of a struct or the load is volatile, just mark the result
+  // as overdefined.
+  if (I.getType()->isStructTy() || I.isVolatile())
     return (void)markOverdefined(&I);
 
   // ResolvedUndefsIn might mark I as overdefined. Bail out, even if we would
@@ -1121,41 +1132,39 @@ void SCCPSolver::visitLoadInst(LoadInst &I) {
 
   ValueLatticeElement &IV = ValueState[&I];
 
-  if (!isConstant(PtrVal) || I.isVolatile())
-    return (void)markOverdefined(IV, &I);
-
-  Constant *Ptr = getConstant(PtrVal);
-
-  // load null is undefined.
-  if (isa<ConstantPointerNull>(Ptr)) {
-    if (NullPointerIsDefined(I.getFunction(), I.getPointerAddressSpace()))
-      return (void)markOverdefined(IV, &I);
-    else
-      return;
-  }
+  if (isConstant(PtrVal)) {
+    Constant *Ptr = getConstant(PtrVal);
 
-  // Transform load (constant global) into the value loaded.
-  if (auto *GV = dyn_cast<GlobalVariable>(Ptr)) {
-    if (!TrackedGlobals.empty()) {
-      // If we are tracking this global, merge in the known value for it.
-      auto It = TrackedGlobals.find(GV);
-      if (It != TrackedGlobals.end()) {
-        mergeInValue(IV, &I, It->second, getMaxWidenStepsOpts());
+    // load null is undefined.
+    if (isa<ConstantPointerNull>(Ptr)) {
+      if (NullPointerIsDefined(I.getFunction(), I.getPointerAddressSpace()))
+        return (void)markOverdefined(IV, &I);
+      else
         return;
+    }
+
+    // Transform load (constant global) into the value loaded.
+    if (auto *GV = dyn_cast<GlobalVariable>(Ptr)) {
+      if (!TrackedGlobals.empty()) {
+        // If we are tracking this global, merge in the known value for it.
+        auto It = TrackedGlobals.find(GV);
+        if (It != TrackedGlobals.end()) {
+          mergeInValue(IV, &I, It->second, getMaxWidenStepsOpts());
+          return;
+        }
       }
     }
-  }
 
-  // Transform load from a constant into a constant if possible.
-  if (Constant *C = ConstantFoldLoadFromConstPtr(Ptr, I.getType(), DL)) {
-    if (isa<UndefValue>(C))
-      return;
-    return (void)markConstant(IV, &I, C);
+    // Transform load from a constant into a constant if possible.
+    if (Constant *C = ConstantFoldLoadFromConstPtr(Ptr, I.getType(), DL)) {
+      if (isa<UndefValue>(C))
+        return;
+      return (void)markConstant(IV, &I, C);
+    }
   }
 
-  // Otherwise we cannot say for certain what value this load will produce.
-  // Bail out.
-  markOverdefined(IV, &I);
+  // Fall back to metadata.
+  mergeInValue(&I, getValueFromMetadata(&I));
 }
 
 void SCCPSolver::visitCallBase(CallBase &CB) {
@@ -1170,10 +1179,13 @@ void SCCPSolver::handleCallOverdefined(CallBase &CB) {
   if (CB.getType()->isVoidTy())
     return;
 
+  // Always mark struct return as overdefined.
+  if (CB.getType()->isStructTy())
+    return (void)markOverdefined(&CB);
+
   // Otherwise, if we have a single return value case, and if the function is
   // a declaration, maybe we can constant fold it.
-  if (F && F->isDeclaration() && !CB.getType()->isStructTy() &&
-      canConstantFoldCallTo(&CB, F)) {
+  if (F && F->isDeclaration() && canConstantFoldCallTo(&CB, F)) {
     SmallVector<Constant *, 8> Operands;
     for (auto AI = CB.arg_begin(), E = CB.arg_end(); AI != E; ++AI) {
       if (AI->get()->getType()->isStructTy())
@@ -1201,8 +1213,8 @@ void SCCPSolver::handleCallOverdefined(CallBase &CB) {
     }
   }
 
-  // Otherwise, we don't know anything about this call, mark it overdefined.
-  return (void)markOverdefined(&CB);
+  // Fall back to metadata.
+  mergeInValue(&CB, getValueFromMetadata(&CB));
 }
 
 void SCCPSolver::handleCallArguments(CallBase &CB) {
@@ -1246,30 +1258,40 @@ void SCCPSolver::handleCallResult(CallBase &CB) {
         return;
 
       Value *CopyOf = CB.getOperand(0);
+      ValueLatticeElement CopyOfVal = getValueState(CopyOf);
       auto *PI = getPredicateInfoFor(&CB);
-      auto *PBranch = dyn_cast_or_null<PredicateBranch>(PI);
-      ValueLatticeElement OriginalVal = getValueState(CopyOf);
-      if (!PI || !PBranch) {
-        mergeInValue(ValueState[&CB], &CB, OriginalVal);
+      assert(PI && "Missing predicate info for ssa.copy");
+
+      CmpInst *Cmp;
+      bool TrueEdge;
+      if (auto *PBranch = dyn_cast<PredicateBranch>(PI)) {
+        Cmp = dyn_cast<CmpInst>(PBranch->Condition);
+        TrueEdge = PBranch->TrueEdge;
+      } else if (auto *PAssume = dyn_cast<PredicateAssume>(PI)) {
+        Cmp = dyn_cast<CmpInst>(PAssume->Condition);
+        TrueEdge = true;
+      } else {
+        mergeInValue(ValueState[&CB], &CB, CopyOfVal);
         return;
       }
 
       // Everything below relies on the condition being a comparison.
-      auto *Cmp = dyn_cast<CmpInst>(PBranch->Condition);
       if (!Cmp) {
-        mergeInValue(ValueState[&CB], &CB, OriginalVal);
+        mergeInValue(ValueState[&CB], &CB, CopyOfVal);
         return;
       }
 
+      Value *RenamedOp = PI->RenamedOp;
       Value *CmpOp0 = Cmp->getOperand(0);
       Value *CmpOp1 = Cmp->getOperand(1);
-      if (CopyOf != CmpOp0 && CopyOf != CmpOp1) {
-        mergeInValue(ValueState[&CB], &CB, OriginalVal);
+      // Bail out if neither of the operands matches RenamedOp.
+      if (CmpOp0 != RenamedOp && CmpOp1 != RenamedOp) {
+        mergeInValue(ValueState[&CB], &CB, getValueState(CopyOf));
         return;
       }
 
       auto Pred = Cmp->getPredicate();
-      if (CmpOp0 != CopyOf) {
+      if (CmpOp1 == RenamedOp) {
         std::swap(CmpOp0, CmpOp1);
         Pred = Cmp->getSwappedPredicate();
       }
@@ -1280,27 +1302,37 @@ void SCCPSolver::handleCallResult(CallBase &CB) {
         return;
       }
 
-      if (!PBranch->TrueEdge)
+      // The code below relies on PredicateInfo only inserting copies for the
+      // true branch when the branch condition is an AND and only inserting
+      // copies for the false branch when the branch condition is an OR. This
+      // ensures we can intersect the range from the condition with the range of
+      // CopyOf.
+      if (!TrueEdge)
         Pred = CmpInst::getInversePredicate(Pred);
 
       ValueLatticeElement CondVal = getValueState(CmpOp1);
       ValueLatticeElement &IV = ValueState[&CB];
-      if (CondVal.isConstantRange() || OriginalVal.isConstantRange()) {
-        auto NewCR =
+      if (CondVal.isConstantRange() || CopyOfVal.isConstantRange()) {
+        auto ImposedCR =
             ConstantRange::getFull(DL.getTypeSizeInBits(CopyOf->getType()));
 
         // Get the range imposed by the condition.
         if (CondVal.isConstantRange())
-          NewCR = ConstantRange::makeAllowedICmpRegion(
+          ImposedCR = ConstantRange::makeAllowedICmpRegion(
               Pred, CondVal.getConstantRange());
 
         // Combine range info for the original value with the new range from the
         // condition.
-        auto OriginalCR = OriginalVal.isConstantRange()
-                              ? OriginalVal.getConstantRange()
-                              : ConstantRange::getFull(
-                                    DL.getTypeSizeInBits(CopyOf->getType()));
-        NewCR = NewCR.intersectWith(OriginalCR);
+        auto CopyOfCR = CopyOfVal.isConstantRange()
+                            ? CopyOfVal.getConstantRange()
+                            : ConstantRange::getFull(
+                                  DL.getTypeSizeInBits(CopyOf->getType()));
+        auto NewCR = ImposedCR.intersectWith(CopyOfCR);
+        // If the existing information is != x, do not use the information from
+        // a chained predicate, as the != x information is more likely to be
+        // helpful in practice.
+        if (!CopyOfCR.contains(NewCR) && CopyOfCR.getSingleMissingElement())
+          NewCR = CopyOfCR;
 
         addAdditionalUser(CmpOp1, &CB);
         // TODO: Actually filp MayIncludeUndef for the created range to false,
@@ -1324,7 +1356,7 @@ void SCCPSolver::handleCallResult(CallBase &CB) {
         return;
       }
 
-      return (void)mergeInValue(IV, &CB, OriginalVal);
+      return (void)mergeInValue(IV, &CB, CopyOfVal);
     }
   }
 
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index ed0ca36a4914f..89f324deef9fd 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -1470,7 +1470,7 @@ static Value *getNaturalGEPRecursively(IRBuilderTy &IRB, const DataLayout &DL,
     }
     APInt ElementSize(Offset.getBitWidth(), ElementSizeInBits / 8);
     APInt NumSkippedElements = Offset.sdiv(ElementSize);
-    if (NumSkippedElements.ugt(VecTy->getNumElements()))
+    if (NumSkippedElements.ugt(cast<FixedVectorType>(VecTy)->getNumElements()))
       return nullptr;
     Offset -= NumSkippedElements * ElementSize;
     Indices.push_back(IRB.getInt(NumSkippedElements));
@@ -1703,8 +1703,15 @@ static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy) {
   NewTy = NewTy->getScalarType();
   if (NewTy->isPointerTy() || OldTy->isPointerTy()) {
     if (NewTy->isPointerTy() && OldTy->isPointerTy()) {
-      return cast<PointerType>(NewTy)->getPointerAddressSpace() ==
-        cast<PointerType>(OldTy)->getPointerAddressSpace();
+      unsigned OldAS = OldTy->getPointerAddressSpace();
+      unsigned NewAS = NewTy->getPointerAddressSpace();
+      // Convert pointers if they are pointers from the same address space or
+      // different integral (not non-integral) address spaces with the same
+      // pointer size.
+      return OldAS == NewAS ||
+             (!DL.isNonIntegralAddressSpace(OldAS) &&
+              !DL.isNonIntegralAddressSpace(NewAS) &&
+              DL.getPointerSize(OldAS) == DL.getPointerSize(NewAS));
     }
 
     // We can convert integers to integral pointers, but not to non-integral
@@ -1740,36 +1747,40 @@ static Value *convertValue(const DataLayout &DL, IRBuilderTy &IRB, Value *V,
   assert(!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) &&
          "Integer types must be the exact same to convert.");
 
-  // See if we need inttoptr for this type pair. A cast involving both scalars
-  // and vectors requires and additional bitcast.
+  // See if we need inttoptr for this type pair. May require additional bitcast.
   if (OldTy->isIntOrIntVectorTy() && NewTy->isPtrOrPtrVectorTy()) {
     // Expand <2 x i32> to i8* --> <2 x i32> to i64 to i8*
-    if (OldTy->isVectorTy() && !NewTy->isVectorTy())
-      return IRB.CreateIntToPtr(IRB.CreateBitCast(V, DL.getIntPtrType(NewTy)),
-                                NewTy);
-
     // Expand i128 to <2 x i8*> --> i128 to <2 x i64> to <2 x i8*>
-    if (!OldTy->isVectorTy() && NewTy->isVectorTy())
-      return IRB.CreateIntToPtr(IRB.CreateBitCast(V, DL.getIntPtrType(NewTy)),
-                                NewTy);
-
-    return IRB.CreateIntToPtr(V, NewTy);
+    // Expand <4 x i32> to <2 x i8*> --> <4 x i32> to <2 x i64> to <2 x i8*>
+    // Directly handle i64 to i8*
+    return IRB.CreateIntToPtr(IRB.CreateBitCast(V, DL.getIntPtrType(NewTy)),
+                              NewTy);
   }
 
-  // See if we need ptrtoint for this type pair. A cast involving both scalars
-  // and vectors requires and additional bitcast.
+  // See if we need ptrtoint for this type pair. May require additional bitcast.
   if (OldTy->isPtrOrPtrVectorTy() && NewTy->isIntOrIntVectorTy()) {
     // Expand <2 x i8*> to i128 --> <2 x i8*> to <2 x i64> to i128
-    if (OldTy->isVectorTy() && !NewTy->isVectorTy())
-      return IRB.CreateBitCast(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)),
-                               NewTy);
-
     // Expand i8* to <2 x i32> --> i8* to i64 to <2 x i32>
-    if (!OldTy->isVectorTy() && NewTy->isVectorTy())
-      return IRB.CreateBitCast(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)),
-                               NewTy);
+    // Expand <2 x i8*> to <4 x i32> --> <2 x i8*> to <2 x i64> to <4 x i32>
+    // Expand i8* to i64 --> i8* to i64 to i64
+    return IRB.CreateBitCast(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)),
+                             NewTy);
+  }
 
-    return IRB.CreatePtrToInt(V, NewTy);
+  if (OldTy->isPtrOrPtrVectorTy() && NewTy->isPtrOrPtrVectorTy()) {
+    unsigned OldAS = OldTy->getPointerAddressSpace();
+    unsigned NewAS = NewTy->getPointerAddressSpace();
+    // To convert pointers with different address spaces (they are already
+    // checked convertible, i.e. they have the same pointer size), so far we
+    // cannot use `bitcast` (which has restrict on the same address space) or
+    // `addrspacecast` (which is not always no-op casting). Instead, use a pair
+    // of no-op `ptrtoint`/`inttoptr` casts through an integer with the same bit
+    // size.
+    if (OldAS != NewAS) {
+      assert(DL.getPointerSize(OldAS) == DL.getPointerSize(NewAS));
+      return IRB.CreateIntToPtr(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)),
+                                NewTy);
+    }
   }
 
   return IRB.CreateBitCast(V, NewTy);
@@ -1788,12 +1799,13 @@ static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
       std::max(S.beginOffset(), P.beginOffset()) - P.beginOffset();
   uint64_t BeginIndex = BeginOffset / ElementSize;
   if (BeginIndex * ElementSize != BeginOffset ||
-      BeginIndex >= Ty->getNumElements())
+      BeginIndex >= cast<FixedVectorType>(Ty)->getNumElements())
     return false;
   uint64_t EndOffset =
       std::min(S.endOffset(), P.endOffset()) - P.beginOffset();
   uint64_t EndIndex = EndOffset / ElementSize;
-  if (EndIndex * ElementSize != EndOffset || EndIndex > Ty->getNumElements())
+  if (EndIndex * ElementSize != EndOffset ||
+      EndIndex > cast<FixedVectorType>(Ty)->getNumElements())
     return false;
 
   assert(EndIndex > BeginIndex && "Empty vector!");
@@ -1919,7 +1931,8 @@ static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) {
              "All non-integer types eliminated!");
       assert(LHSTy->getElementType()->isIntegerTy() &&
              "All non-integer types eliminated!");
-      return RHSTy->getNumElements() < LHSTy->getNumElements();
+      return cast<FixedVectorType>(RHSTy)->getNumElements() <
+             cast<FixedVectorType>(LHSTy)->getNumElements();
     };
     llvm::sort(CandidateTys, RankVectorTypes);
     CandidateTys.erase(
@@ -2162,7 +2175,7 @@ static Value *insertInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *Old,
 
 static Value *extractVector(IRBuilderTy &IRB, Value *V, unsigned BeginIndex,
                             unsigned EndIndex, const Twine &Name) {
-  VectorType *VecTy = cast<VectorType>(V->getType());
+  auto *VecTy = cast<FixedVectorType>(V->getType());
   unsigned NumElements = EndIndex - BeginIndex;
   assert(NumElements <= VecTy->getNumElements() && "Too many elements!");
 
@@ -2200,21 +2213,23 @@ static Value *insertVector(IRBuilderTy &IRB, Value *Old, Value *V,
     return V;
   }
 
-  assert(Ty->getNumElements() <= VecTy->getNumElements() &&
+  assert(cast<FixedVectorType>(Ty)->getNumElements() <=
+             cast<FixedVectorType>(VecTy)->getNumElements() &&
          "Too many elements!");
-  if (Ty->getNumElements() == VecTy->getNumElements()) {
+  if (cast<FixedVectorType>(Ty)->getNumElements() ==
+      cast<FixedVectorType>(VecTy)->getNumElements()) {
     assert(V->getType() == VecTy && "Vector type mismatch");
     return V;
   }
-  unsigned EndIndex = BeginIndex + Ty->getNumElements();
+  unsigned EndIndex = BeginIndex + cast<FixedVectorType>(Ty)->getNumElements();
 
   // When inserting a smaller vector into the larger to store, we first
   // use a shuffle vector to widen it with undef elements, and then
   // a second shuffle vector to select between the loaded vector and the
   // incoming vector.
   SmallVector<Constant *, 8> Mask;
-  Mask.reserve(VecTy->getNumElements());
-  for (unsigned i = 0; i != VecTy->getNumElements(); ++i)
+  Mask.reserve(cast<FixedVectorType>(VecTy)->getNumElements());
+  for (unsigned i = 0; i != cast<FixedVectorType>(VecTy)->getNumElements(); ++i)
     if (i >= BeginIndex && i < EndIndex)
       Mask.push_back(IRB.getInt32(i - BeginIndex));
     else
@@ -2224,7 +2239,7 @@ static Value *insertVector(IRBuilderTy &IRB, Value *Old, Value *V,
   LLVM_DEBUG(dbgs() << "    shuffle: " << *V << "\n");
 
   Mask.clear();
-  for (unsigned i = 0; i != VecTy->getNumElements(); ++i)
+  for (unsigned i = 0; i != cast<FixedVectorType>(VecTy)->getNumElements(); ++i)
     Mask.push_back(IRB.getInt1(i >= BeginIndex && i < EndIndex));
 
   V = IRB.CreateSelect(ConstantVector::get(Mask), V, Old, Name + "blend");
@@ -2584,7 +2599,8 @@ class llvm::sroa::AllocaSliceRewriter
       unsigned EndIndex = getIndex(NewEndOffset);
       assert(EndIndex > BeginIndex && "Empty vector!");
       unsigned NumElements = EndIndex - BeginIndex;
-      assert(NumElements <= VecTy->getNumElements() && "Too many elements!");
+      assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
+             "Too many elements!");
       Type *SliceTy = (NumElements == 1)
                           ? ElementTy
                           : FixedVectorType::get(ElementTy, NumElements);
@@ -2808,7 +2824,8 @@ class llvm::sroa::AllocaSliceRewriter
       unsigned EndIndex = getIndex(NewEndOffset);
       assert(EndIndex > BeginIndex && "Empty vector!");
       unsigned NumElements = EndIndex - BeginIndex;
-      assert(NumElements <= VecTy->getNumElements() && "Too many elements!");
+      assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
+             "Too many elements!");
 
       Value *Splat = getIntegerSplat(
           II.getValue(), DL.getTypeSizeInBits(ElementTy).getFixedSize() / 8);
@@ -2847,7 +2864,8 @@ class llvm::sroa::AllocaSliceRewriter
       V = getIntegerSplat(II.getValue(),
                           DL.getTypeSizeInBits(ScalarTy).getFixedSize() / 8);
       if (VectorType *AllocaVecTy = dyn_cast<VectorType>(AllocaTy))
-        V = getVectorSplat(V, AllocaVecTy->getNumElements());
+        V = getVectorSplat(
+            V, cast<FixedVectorType>(AllocaVecTy)->getNumElements());
 
       V = convertValue(DL, IRB, V, AllocaTy);
     }
@@ -2945,7 +2963,7 @@ class llvm::sroa::AllocaSliceRewriter
     unsigned OffsetWidth = DL.getIndexSizeInBits(OtherAS);
     APInt OtherOffset(OffsetWidth, NewBeginOffset - BeginOffset);
     Align OtherAlign =
-        assumeAligned(IsDest ? II.getSourceAlignment() : II.getDestAlignment());
+        (IsDest ? II.getSourceAlign() : II.getDestAlign()).valueOrOne();
     OtherAlign =
         commonAlignment(OtherAlign, OtherOffset.zextOrTrunc(64).getZExtValue());
 
@@ -3606,7 +3624,7 @@ static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset,
      } else {
        // FIXME: This isn't right for vectors with non-byte-sized or
        // non-power-of-two sized elements.
-       auto *VT = cast<VectorType>(Ty);
+       auto *VT = cast<FixedVectorType>(Ty);
        ElementTy = VT->getElementType();
        TyNumElements = VT->getNumElements();
     }
@@ -4256,7 +4274,7 @@ AllocaInst *SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS,
     const Align Alignment = commonAlignment(AI.getAlign(), P.beginOffset());
     // If we will get at least this much alignment from the type alone, leave
     // the alloca's alignment unconstrained.
-    const bool IsUnconstrained = Alignment <= DL.getABITypeAlignment(SliceTy);
+    const bool IsUnconstrained = Alignment <= DL.getABITypeAlign(SliceTy);
     NewAI = new AllocaInst(
         SliceTy, AI.getType()->getAddressSpace(), nullptr,
         IsUnconstrained ? DL.getPrefTypeAlign(SliceTy) : Alignment,
diff --git a/llvm/lib/Transforms/Scalar/Scalarizer.cpp b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
index cfd58bfa3bb78..851bd79cd6d83 100644
--- a/llvm/lib/Transforms/Scalar/Scalarizer.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
@@ -22,8 +22,8 @@
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
-#include "llvm/IR/Dominators.h"
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstVisitor.h"
@@ -41,6 +41,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/Local.h"
 #include <cassert>
 #include <cstdint>
 #include <iterator>
@@ -51,6 +52,11 @@ using namespace llvm;
 
 #define DEBUG_TYPE "scalarizer"
 
+static cl::opt<bool> ScalarizeVariableInsertExtract(
+    "scalarize-variable-insert-extract", cl::init(true), cl::Hidden,
+    cl::desc("Allow the scalarizer pass to scalarize "
+             "insertelement/extractelement with variable index"));
+
 // This is disabled by default because having separate loads and stores
 // makes it more likely that the -combiner-alias-analysis limits will be
 // reached.
@@ -156,8 +162,8 @@ struct VectorLayout {
   VectorLayout() = default;
 
   // Return the alignment of element I.
-  uint64_t getElemAlign(unsigned I) {
-    return MinAlign(VecAlign, I * ElemSize);
+  Align getElemAlign(unsigned I) {
+    return commonAlignment(VecAlign, I * ElemSize);
   }
 
   // The type of the vector.
@@ -167,7 +173,7 @@ struct VectorLayout {
   Type *ElemTy = nullptr;
 
   // The alignment of the vector.
-  uint64_t VecAlign = 0;
+  Align VecAlign;
 
   // The size of each element.
   uint64_t ElemSize = 0;
@@ -192,6 +198,8 @@ class ScalarizerVisitor : public InstVisitor<ScalarizerVisitor, bool> {
   bool visitGetElementPtrInst(GetElementPtrInst &GEPI);
   bool visitCastInst(CastInst &CI);
   bool visitBitCastInst(BitCastInst &BCI);
+  bool visitInsertElementInst(InsertElementInst &IEI);
+  bool visitExtractElementInst(ExtractElementInst &EEI);
   bool visitShuffleVectorInst(ShuffleVectorInst &SVI);
   bool visitPHINode(PHINode &PHI);
   bool visitLoadInst(LoadInst &LI);
@@ -203,8 +211,8 @@ class ScalarizerVisitor : public InstVisitor<ScalarizerVisitor, bool> {
   void gather(Instruction *Op, const ValueVector &CV);
   bool canTransferMetadata(unsigned Kind);
   void transferMetadataAndIRFlags(Instruction *Op, const ValueVector &CV);
-  bool getVectorLayout(Type *Ty, unsigned Alignment, VectorLayout &Layout,
-                       const DataLayout &DL);
+  Optional<VectorLayout> getVectorLayout(Type *Ty, Align Alignment,
+                                         const DataLayout &DL);
   bool finish();
 
   template<typename T> bool splitUnary(Instruction &, const T &);
@@ -215,6 +223,8 @@ class ScalarizerVisitor : public InstVisitor<ScalarizerVisitor, bool> {
   ScatterMap Scattered;
   GatherList Gathered;
 
+  SmallVector<WeakTrackingVH, 32> PotentiallyDeadInstrs;
+
   unsigned ParallelLoopAccessMDKind;
 
   DominatorTree *DT;
@@ -252,7 +262,7 @@ Scatterer::Scatterer(BasicBlock *bb, BasicBlock::iterator bbi, Value *v,
   PtrTy = dyn_cast<PointerType>(Ty);
   if (PtrTy)
     Ty = PtrTy->getElementType();
-  Size = cast<VectorType>(Ty)->getNumElements();
+  Size = cast<FixedVectorType>(Ty)->getNumElements();
   if (!CachePtr)
     Tmp.resize(Size, nullptr);
   else if (CachePtr->empty())
@@ -376,11 +386,6 @@ Scatterer ScalarizerVisitor::scatter(Instruction *Point, Value *V) {
 // so that we can avoid creating the gathered form if all uses of Op are
 // replaced with uses of CV.
 void ScalarizerVisitor::gather(Instruction *Op, const ValueVector &CV) {
-  // Since we're not deleting Op yet, stub out its operands, so that it
-  // doesn't make anything live unnecessarily.
-  for (unsigned I = 0, E = Op->getNumOperands(); I != E; ++I)
-    Op->setOperand(I, UndefValue::get(Op->getOperand(I)->getType()));
-
   transferMetadataAndIRFlags(Op, CV);
 
   // If we already have a scattered form of Op (created from ExtractElements
@@ -389,13 +394,13 @@ void ScalarizerVisitor::gather(Instruction *Op, const ValueVector &CV) {
   if (!SV.empty()) {
     for (unsigned I = 0, E = SV.size(); I != E; ++I) {
       Value *V = SV[I];
-      if (V == nullptr)
+      if (V == nullptr || SV[I] == CV[I])
         continue;
 
       Instruction *Old = cast<Instruction>(V);
       CV[I]->takeName(Old);
       Old->replaceAllUsesWith(CV[I]);
-      Old->eraseFromParent();
+      PotentiallyDeadInstrs.emplace_back(Old);
     }
   }
   SV = CV;
@@ -434,25 +439,22 @@ void ScalarizerVisitor::transferMetadataAndIRFlags(Instruction *Op,
 }
 
 // Try to fill in Layout from Ty, returning true on success.  Alignment is
-// the alignment of the vector, or 0 if the ABI default should be used.
-bool ScalarizerVisitor::getVectorLayout(Type *Ty, unsigned Alignment,
-                                 VectorLayout &Layout, const DataLayout &DL) {
+// the alignment of the vector, or None if the ABI default should be used.
+Optional<VectorLayout>
+ScalarizerVisitor::getVectorLayout(Type *Ty, Align Alignment,
+                                   const DataLayout &DL) {
+  VectorLayout Layout;
   // Make sure we're dealing with a vector.
   Layout.VecTy = dyn_cast<VectorType>(Ty);
   if (!Layout.VecTy)
-    return false;
-
+    return None;
   // Check that we're dealing with full-byte elements.
   Layout.ElemTy = Layout.VecTy->getElementType();
   if (!DL.typeSizeEqualsStoreSize(Layout.ElemTy))
-    return false;
-
-  if (Alignment)
-    Layout.VecAlign = Alignment;
-  else
-    Layout.VecAlign = DL.getABITypeAlignment(Layout.VecTy);
+    return None;
+  Layout.VecAlign = Alignment;
   Layout.ElemSize = DL.getTypeStoreSize(Layout.ElemTy);
-  return true;
+  return Layout;
 }
 
 // Scalarize one-operand instruction I, using Split(Builder, X, Name)
@@ -463,7 +465,7 @@ bool ScalarizerVisitor::splitUnary(Instruction &I, const Splitter &Split) {
   if (!VT)
     return false;
 
-  unsigned NumElems = VT->getNumElements();
+  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
   IRBuilder<> Builder(&I);
   Scatterer Op = scatter(&I, I.getOperand(0));
   assert(Op.size() == NumElems && "Mismatched unary operation");
@@ -483,7 +485,7 @@ bool ScalarizerVisitor::splitBinary(Instruction &I, const Splitter &Split) {
   if (!VT)
     return false;
 
-  unsigned NumElems = VT->getNumElements();
+  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
   IRBuilder<> Builder(&I);
   Scatterer VOp0 = scatter(&I, I.getOperand(0));
   Scatterer VOp1 = scatter(&I, I.getOperand(1));
@@ -526,7 +528,7 @@ bool ScalarizerVisitor::splitCall(CallInst &CI) {
   if (ID == Intrinsic::not_intrinsic || !isTriviallyScalariable(ID))
     return false;
 
-  unsigned NumElems = VT->getNumElements();
+  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
   unsigned NumArgs = CI.getNumArgOperands();
 
   ValueVector ScalarOperands(NumArgs);
@@ -576,7 +578,7 @@ bool ScalarizerVisitor::visitSelectInst(SelectInst &SI) {
   if (!VT)
     return false;
 
-  unsigned NumElems = VT->getNumElements();
+  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
   IRBuilder<> Builder(&SI);
   Scatterer VOp1 = scatter(&SI, SI.getOperand(1));
   Scatterer VOp2 = scatter(&SI, SI.getOperand(2));
@@ -630,7 +632,7 @@ bool ScalarizerVisitor::visitGetElementPtrInst(GetElementPtrInst &GEPI) {
     return false;
 
   IRBuilder<> Builder(&GEPI);
-  unsigned NumElems = VT->getNumElements();
+  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
   unsigned NumIndices = GEPI.getNumIndices();
 
   // The base pointer might be scalar even if it's a vector GEP. In those cases,
@@ -675,7 +677,7 @@ bool ScalarizerVisitor::visitCastInst(CastInst &CI) {
   if (!VT)
     return false;
 
-  unsigned NumElems = VT->getNumElements();
+  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
   IRBuilder<> Builder(&CI);
   Scatterer Op0 = scatter(&CI, CI.getOperand(0));
   assert(Op0.size() == NumElems && "Mismatched cast");
@@ -694,8 +696,8 @@ bool ScalarizerVisitor::visitBitCastInst(BitCastInst &BCI) {
   if (!DstVT || !SrcVT)
     return false;
 
-  unsigned DstNumElems = DstVT->getNumElements();
-  unsigned SrcNumElems = SrcVT->getNumElements();
+  unsigned DstNumElems = cast<FixedVectorType>(DstVT)->getNumElements();
+  unsigned SrcNumElems = cast<FixedVectorType>(SrcVT)->getNumElements();
   IRBuilder<> Builder(&BCI);
   Scatterer Op0 = scatter(&BCI, BCI.getOperand(0));
   ValueVector Res;
@@ -743,12 +745,79 @@ bool ScalarizerVisitor::visitBitCastInst(BitCastInst &BCI) {
   return true;
 }
 
+bool ScalarizerVisitor::visitInsertElementInst(InsertElementInst &IEI) {
+  VectorType *VT = dyn_cast<VectorType>(IEI.getType());
+  if (!VT)
+    return false;
+
+  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
+  IRBuilder<> Builder(&IEI);
+  Scatterer Op0 = scatter(&IEI, IEI.getOperand(0));
+  Value *NewElt = IEI.getOperand(1);
+  Value *InsIdx = IEI.getOperand(2);
+
+  ValueVector Res;
+  Res.resize(NumElems);
+
+  if (auto *CI = dyn_cast<ConstantInt>(InsIdx)) {
+    for (unsigned I = 0; I < NumElems; ++I)
+      Res[I] = CI->getValue().getZExtValue() == I ? NewElt : Op0[I];
+  } else {
+    if (!ScalarizeVariableInsertExtract)
+      return false;
+
+    for (unsigned I = 0; I < NumElems; ++I) {
+      Value *ShouldReplace =
+          Builder.CreateICmpEQ(InsIdx, ConstantInt::get(InsIdx->getType(), I),
+                               InsIdx->getName() + ".is." + Twine(I));
+      Value *OldElt = Op0[I];
+      Res[I] = Builder.CreateSelect(ShouldReplace, NewElt, OldElt,
+                                    IEI.getName() + ".i" + Twine(I));
+    }
+  }
+
+  gather(&IEI, Res);
+  return true;
+}
+
+bool ScalarizerVisitor::visitExtractElementInst(ExtractElementInst &EEI) {
+  VectorType *VT = dyn_cast<VectorType>(EEI.getOperand(0)->getType());
+  if (!VT)
+    return false;
+
+  unsigned NumSrcElems = cast<FixedVectorType>(VT)->getNumElements();
+  IRBuilder<> Builder(&EEI);
+  Scatterer Op0 = scatter(&EEI, EEI.getOperand(0));
+  Value *ExtIdx = EEI.getOperand(1);
+
+  if (auto *CI = dyn_cast<ConstantInt>(ExtIdx)) {
+    Value *Res = Op0[CI->getValue().getZExtValue()];
+    gather(&EEI, {Res});
+    return true;
+  }
+
+  if (!ScalarizeVariableInsertExtract)
+    return false;
+
+  Value *Res = UndefValue::get(VT->getElementType());
+  for (unsigned I = 0; I < NumSrcElems; ++I) {
+    Value *ShouldExtract =
+        Builder.CreateICmpEQ(ExtIdx, ConstantInt::get(ExtIdx->getType(), I),
+                             ExtIdx->getName() + ".is." + Twine(I));
+    Value *Elt = Op0[I];
+    Res = Builder.CreateSelect(ShouldExtract, Elt, Res,
+                               EEI.getName() + ".upto" + Twine(I));
+  }
+  gather(&EEI, {Res});
+  return true;
+}
+
 bool ScalarizerVisitor::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
   VectorType *VT = dyn_cast<VectorType>(SVI.getType());
   if (!VT)
     return false;
 
-  unsigned NumElems = VT->getNumElements();
+  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
   Scatterer Op0 = scatter(&SVI, SVI.getOperand(0));
   Scatterer Op1 = scatter(&SVI, SVI.getOperand(1));
   ValueVector Res;
@@ -772,7 +841,7 @@ bool ScalarizerVisitor::visitPHINode(PHINode &PHI) {
   if (!VT)
     return false;
 
-  unsigned NumElems = VT->getNumElements();
+  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
   IRBuilder<> Builder(&PHI);
   ValueVector Res;
   Res.resize(NumElems);
@@ -798,20 +867,20 @@ bool ScalarizerVisitor::visitLoadInst(LoadInst &LI) {
   if (!LI.isSimple())
     return false;
 
-  VectorLayout Layout;
-  if (!getVectorLayout(LI.getType(), LI.getAlignment(), Layout,
-                       LI.getModule()->getDataLayout()))
+  Optional<VectorLayout> Layout = getVectorLayout(
+      LI.getType(), LI.getAlign(), LI.getModule()->getDataLayout());
+  if (!Layout)
     return false;
 
-  unsigned NumElems = Layout.VecTy->getNumElements();
+  unsigned NumElems = cast<FixedVectorType>(Layout->VecTy)->getNumElements();
   IRBuilder<> Builder(&LI);
   Scatterer Ptr = scatter(&LI, LI.getPointerOperand());
   ValueVector Res;
   Res.resize(NumElems);
 
   for (unsigned I = 0; I < NumElems; ++I)
-    Res[I] = Builder.CreateAlignedLoad(Layout.VecTy->getElementType(), Ptr[I],
-                                       Align(Layout.getElemAlign(I)),
+    Res[I] = Builder.CreateAlignedLoad(Layout->VecTy->getElementType(), Ptr[I],
+                                       Align(Layout->getElemAlign(I)),
                                        LI.getName() + ".i" + Twine(I));
   gather(&LI, Res);
   return true;
@@ -823,13 +892,13 @@ bool ScalarizerVisitor::visitStoreInst(StoreInst &SI) {
   if (!SI.isSimple())
     return false;
 
-  VectorLayout Layout;
   Value *FullValue = SI.getValueOperand();
-  if (!getVectorLayout(FullValue->getType(), SI.getAlignment(), Layout,
-                       SI.getModule()->getDataLayout()))
+  Optional<VectorLayout> Layout = getVectorLayout(
+      FullValue->getType(), SI.getAlign(), SI.getModule()->getDataLayout());
+  if (!Layout)
     return false;
 
-  unsigned NumElems = Layout.VecTy->getNumElements();
+  unsigned NumElems = cast<FixedVectorType>(Layout->VecTy)->getNumElements();
   IRBuilder<> Builder(&SI);
   Scatterer VPtr = scatter(&SI, SI.getPointerOperand());
   Scatterer VVal = scatter(&SI, FullValue);
@@ -837,10 +906,9 @@ bool ScalarizerVisitor::visitStoreInst(StoreInst &SI) {
   ValueVector Stores;
   Stores.resize(NumElems);
   for (unsigned I = 0; I < NumElems; ++I) {
-    unsigned Align = Layout.getElemAlign(I);
     Value *Val = VVal[I];
     Value *Ptr = VPtr[I];
-    Stores[I] = Builder.CreateAlignedStore(Val, Ptr, MaybeAlign(Align));
+    Stores[I] = Builder.CreateAlignedStore(Val, Ptr, Layout->getElemAlign(I));
   }
   transferMetadataAndIRFlags(&SI, Stores);
   return true;
@@ -863,23 +931,32 @@ bool ScalarizerVisitor::finish() {
     if (!Op->use_empty()) {
       // The value is still needed, so recreate it using a series of
       // InsertElements.
-      auto *Ty = cast<VectorType>(Op->getType());
-      Value *Res = UndefValue::get(Ty);
-      BasicBlock *BB = Op->getParent();
-      unsigned Count = Ty->getNumElements();
-      IRBuilder<> Builder(Op);
-      if (isa<PHINode>(Op))
-        Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
-      for (unsigned I = 0; I < Count; ++I)
-        Res = Builder.CreateInsertElement(Res, CV[I], Builder.getInt32(I),
-                                          Op->getName() + ".upto" + Twine(I));
+      Value *Res = UndefValue::get(Op->getType());
+      if (auto *Ty = dyn_cast<VectorType>(Op->getType())) {
+        BasicBlock *BB = Op->getParent();
+        unsigned Count = cast<FixedVectorType>(Ty)->getNumElements();
+        IRBuilder<> Builder(Op);
+        if (isa<PHINode>(Op))
+          Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
+        for (unsigned I = 0; I < Count; ++I)
+          Res = Builder.CreateInsertElement(Res, CV[I], Builder.getInt32(I),
+                                            Op->getName() + ".upto" + Twine(I));
+      } else {
+        assert(CV.size() == 1 && Op->getType() == CV[0]->getType());
+        Res = CV[0];
+        if (Op == Res)
+          continue;
+      }
       Res->takeName(Op);
       Op->replaceAllUsesWith(Res);
     }
-    Op->eraseFromParent();
+    PotentiallyDeadInstrs.emplace_back(Op);
   }
   Gathered.clear();
   Scattered.clear();
+
+  RecursivelyDeleteTriviallyDeadInstructionsPermissive(PotentiallyDeadInstrs);
+
   return true;
 }
 
diff --git a/llvm/lib/Transforms/Scalar/Sink.cpp b/llvm/lib/Transforms/Scalar/Sink.cpp
index 677d86f8c7b48..48f289c8f17d6 100644
--- a/llvm/lib/Transforms/Scalar/Sink.cpp
+++ b/llvm/lib/Transforms/Scalar/Sink.cpp
@@ -166,8 +166,8 @@ static bool SinkInstruction(Instruction *Inst,
   // dominated by one of the successors.
   // Look at all the dominated blocks and see if we can sink it in one.
   DomTreeNode *DTN = DT.getNode(Inst->getParent());
-  for (DomTreeNode::iterator I = DTN->begin(), E = DTN->end();
-      I != E && SuccToSinkTo == nullptr; ++I) {
+  for (auto I = DTN->begin(), E = DTN->end(); I != E && SuccToSinkTo == nullptr;
+       ++I) {
     BasicBlock *Candidate = (*I)->getBlock();
     // A node always immediate-dominates its children on the dominator
     // tree.
diff --git a/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp b/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp
index 34e6387a238e4..f82a2936c762d 100644
--- a/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp
+++ b/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp
@@ -65,6 +65,7 @@
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/InitializePasses.h"
@@ -254,9 +255,25 @@ static unsigned ComputeSpeculationCost(const Instruction *I,
 bool SpeculativeExecutionPass::considerHoistingFromTo(
     BasicBlock &FromBlock, BasicBlock &ToBlock) {
   SmallPtrSet<const Instruction *, 8> NotHoisted;
-  const auto AllPrecedingUsesFromBlockHoisted = [&NotHoisted](User *U) {
-    for (Value* V : U->operand_values()) {
-      if (Instruction *I = dyn_cast<Instruction>(V)) {
+  const auto AllPrecedingUsesFromBlockHoisted = [&NotHoisted](const User *U) {
+    // Debug variable has special operand to check it's not hoisted.
+    if (const auto *DVI = dyn_cast<DbgVariableIntrinsic>(U)) {
+      if (const auto *I =
+              dyn_cast_or_null<Instruction>(DVI->getVariableLocation()))
+        if (NotHoisted.count(I) == 0)
+          return true;
+      return false;
+    }
+
+    // Usially debug label instrinsic corresponds to label in LLVM IR. In these
+    // cases we should not move it here.
+    // TODO: Possible special processing needed to detect it is related to a
+    // hoisted instruction.
+    if (isa<DbgLabelInst>(U))
+      return false;
+
+    for (const Value *V : U->operand_values()) {
+      if (const Instruction *I = dyn_cast<Instruction>(V)) {
         if (NotHoisted.count(I) > 0)
           return false;
       }
@@ -265,7 +282,8 @@ bool SpeculativeExecutionPass::considerHoistingFromTo(
   };
 
   unsigned TotalSpeculationCost = 0;
-  for (auto& I : FromBlock) {
+  unsigned NotHoistedInstCount = 0;
+  for (const auto &I : FromBlock) {
     const unsigned Cost = ComputeSpeculationCost(&I, *TTI);
     if (Cost != UINT_MAX && isSafeToSpeculativelyExecute(&I) &&
         AllPrecedingUsesFromBlockHoisted(&I)) {
@@ -273,9 +291,12 @@ bool SpeculativeExecutionPass::considerHoistingFromTo(
       if (TotalSpeculationCost > SpecExecMaxSpeculationCost)
         return false;  // too much to hoist
     } else {
-      NotHoisted.insert(&I);
-      if (NotHoisted.size() > SpecExecMaxNotHoisted)
+      // Debug info instrinsics should not be counted for threshold.
+      if (!isa<DbgInfoIntrinsic>(I))
+        NotHoistedInstCount++;
+      if (NotHoistedInstCount > SpecExecMaxNotHoisted)
         return false; // too much left behind
+      NotHoisted.insert(&I);
     }
   }
 
diff --git a/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
index ad457440d48a3..f2208edd5b196 100644
--- a/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
+++ b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
@@ -525,14 +525,16 @@ class AssumeSimplifyPassLegacyPass : public FunctionPass {
   bool runOnFunction(Function &F) override {
     if (skipFunction(F) || !EnableKnowledgeRetention)
       return false;
-    DominatorTreeWrapperPass *DT =
-        getAnalysisIfAvailable<DominatorTreeWrapperPass>();
     AssumptionCache &AC =
         getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
-    return simplifyAssumes(F, &AC, DT ? &DT->getDomTree() : nullptr);
+    DominatorTreeWrapperPass *DTWP =
+        getAnalysisIfAvailable<DominatorTreeWrapperPass>();
+    return simplifyAssumes(F, &AC, DTWP ? &DTWP->getDomTree() : nullptr);
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequired<AssumptionCacheTracker>();
+
     AU.setPreservesAll();
   }
 };
@@ -542,7 +544,6 @@ char AssumeSimplifyPassLegacyPass::ID = 0;
 
 INITIALIZE_PASS_BEGIN(AssumeSimplifyPassLegacyPass, "assume-simplify",
                       "Assume Simplify", false, false)
-INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
 INITIALIZE_PASS_END(AssumeSimplifyPassLegacyPass, "assume-simplify",
                     "Assume Simplify", false, false)
@@ -553,9 +554,43 @@ FunctionPass *llvm::createAssumeSimplifyPass() {
 
 PreservedAnalyses AssumeBuilderPass::run(Function &F,
                                          FunctionAnalysisManager &AM) {
-  AssumptionCache* AC = AM.getCachedResult<AssumptionAnalysis>(F);
+  AssumptionCache *AC = &AM.getResult<AssumptionAnalysis>(F);
   DominatorTree* DT = AM.getCachedResult<DominatorTreeAnalysis>(F);
   for (Instruction &I : instructions(F))
     salvageKnowledge(&I, AC, DT);
   return PreservedAnalyses::all();
 }
+
+namespace {
+class AssumeBuilderPassLegacyPass : public FunctionPass {
+public:
+  static char ID;
+
+  AssumeBuilderPassLegacyPass() : FunctionPass(ID) {
+    initializeAssumeBuilderPassLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+  bool runOnFunction(Function &F) override {
+    AssumptionCache &AC =
+        getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
+    DominatorTreeWrapperPass *DTWP =
+        getAnalysisIfAvailable<DominatorTreeWrapperPass>();
+    for (Instruction &I : instructions(F))
+      salvageKnowledge(&I, &AC, DTWP ? &DTWP->getDomTree() : nullptr);
+    return true;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequired<AssumptionCacheTracker>();
+
+    AU.setPreservesAll();
+  }
+};
+} // namespace
+
+char AssumeBuilderPassLegacyPass::ID = 0;
+
+INITIALIZE_PASS_BEGIN(AssumeBuilderPassLegacyPass, "assume-builder",
+                      "Assume Builder", false, false)
+INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
+INITIALIZE_PASS_END(AssumeBuilderPassLegacyPass, "assume-builder",
+                    "Assume Builder", false, false)
diff --git a/llvm/lib/Transforms/Utils/CallGraphUpdater.cpp b/llvm/lib/Transforms/Utils/CallGraphUpdater.cpp
index cded14b9cc0a1..52e859361c598 100644
--- a/llvm/lib/Transforms/Utils/CallGraphUpdater.cpp
+++ b/llvm/lib/Transforms/Utils/CallGraphUpdater.cpp
@@ -149,7 +149,7 @@ bool CallGraphUpdater::replaceCallSite(CallBase &OldCS, CallBase &NewCS) {
       CG->getOrInsertFunction(NewCS.getCalledFunction());
   CallGraphNode *CallerNode = (*CG)[Caller];
   if (llvm::none_of(*CallerNode, [&OldCS](const CallGraphNode::CallRecord &CR) {
-        return CR.first == &OldCS;
+        return CR.first && *CR.first == &OldCS;
       }))
     return false;
   CallerNode->replaceCallEdge(OldCS, NewCS, NewCalleeNode);
diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp
index fce926fb447c4..788983c156903 100644
--- a/llvm/lib/Transforms/Utils/CloneFunction.cpp
+++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp
@@ -46,7 +46,7 @@ BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap,
   if (BB->hasName())
     NewBB->setName(BB->getName() + NameSuffix);
 
-  bool hasCalls = false, hasDynamicAllocas = false, hasStaticAllocas = false;
+  bool hasCalls = false, hasDynamicAllocas = false;
   Module *TheModule = F ? F->getParent() : nullptr;
 
   // Loop over all instructions, and copy them over.
@@ -62,18 +62,15 @@ BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap,
 
     hasCalls |= (isa<CallInst>(I) && !isa<DbgInfoIntrinsic>(I));
     if (const AllocaInst *AI = dyn_cast<AllocaInst>(&I)) {
-      if (isa<ConstantInt>(AI->getArraySize()))
-        hasStaticAllocas = true;
-      else
+      if (!AI->isStaticAlloca()) {
         hasDynamicAllocas = true;
+      }
     }
   }
 
   if (CodeInfo) {
     CodeInfo->ContainsCalls          |= hasCalls;
     CodeInfo->ContainsDynamicAllocas |= hasDynamicAllocas;
-    CodeInfo->ContainsDynamicAllocas |= hasStaticAllocas &&
-                                        BB != &BB->getParent()->getEntryBlock();
   }
   return NewBB;
 }
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 594590a746b8d..8cdbb9d356523 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -877,6 +877,7 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
       case Attribute::NoMerge:
       case Attribute::NoReturn:
       case Attribute::NoSync:
+      case Attribute::NoUndef:
       case Attribute::None:
       case Attribute::NonNull:
       case Attribute::Preallocated:
diff --git a/llvm/lib/Transforms/Utils/CodeMoverUtils.cpp b/llvm/lib/Transforms/Utils/CodeMoverUtils.cpp
index 4583ff74167a5..08047dc0f96ee 100644
--- a/llvm/lib/Transforms/Utils/CodeMoverUtils.cpp
+++ b/llvm/lib/Transforms/Utils/CodeMoverUtils.cpp
@@ -15,7 +15,6 @@
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/DependenceAnalysis.h"
-#include "llvm/Analysis/OrderedInstructions.h"
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/Dominators.h"
@@ -94,6 +93,18 @@ class ControlConditions {
 };
 } // namespace
 
+static bool domTreeLevelBefore(DominatorTree *DT, const Instruction *InstA,
+                               const Instruction *InstB) {
+  // Use ordered basic block in case the 2 instructions are in the same
+  // block.
+  if (InstA->getParent() == InstB->getParent())
+    return InstA->comesBefore(InstB);
+
+  DomTreeNode *DA = DT->getNode(InstA->getParent());
+  DomTreeNode *DB = DT->getNode(InstB->getParent());
+  return DA->getLevel() < DB->getLevel();
+}
+
 const Optional<ControlConditions> ControlConditions::collectControlConditions(
     const BasicBlock &BB, const BasicBlock &Dominator, const DominatorTree &DT,
     const PostDominatorTree &PDT, unsigned MaxLookup) {
@@ -297,8 +308,12 @@ collectInstructionsInBetween(Instruction &StartInst, const Instruction &EndInst,
 }
 
 bool llvm::isSafeToMoveBefore(Instruction &I, Instruction &InsertPoint,
-                              DominatorTree &DT, const PostDominatorTree &PDT,
-                              DependenceInfo &DI) {
+                              DominatorTree &DT, const PostDominatorTree *PDT,
+                              DependenceInfo *DI) {
+  // Skip tests when we don't have PDT or DI
+  if (!PDT || !DI)
+    return false;
+
   // Cannot move itself before itself.
   if (&I == &InsertPoint)
     return false;
@@ -314,7 +329,7 @@ bool llvm::isSafeToMoveBefore(Instruction &I, Instruction &InsertPoint,
     return reportInvalidCandidate(I, NotMovedTerminator);
 
   // TODO remove this limitation.
-  if (!isControlFlowEquivalent(I, InsertPoint, DT, PDT))
+  if (!isControlFlowEquivalent(I, InsertPoint, DT, *PDT))
     return reportInvalidCandidate(I, NotControlFlowEquivalent);
 
   if (!DT.dominates(&InsertPoint, &I))
@@ -328,9 +343,8 @@ bool llvm::isSafeToMoveBefore(Instruction &I, Instruction &InsertPoint,
         if (&InsertPoint == OpInst || !DT.dominates(OpInst, &InsertPoint))
           return false;
 
-  OrderedInstructions OI(&DT);
   DT.updateDFSNumbers();
-  const bool MoveForward = OI.domTreeLevelBefore(&I, &InsertPoint);
+  const bool MoveForward = domTreeLevelBefore(&DT, &I, &InsertPoint);
   Instruction &StartInst = (MoveForward ? I : InsertPoint);
   Instruction &EndInst = (MoveForward ? InsertPoint : I);
   SmallPtrSet<Instruction *, 10> InstsToCheck;
@@ -363,7 +377,7 @@ bool llvm::isSafeToMoveBefore(Instruction &I, Instruction &InsertPoint,
   // StartInst to \p EndInst.
   if (std::any_of(InstsToCheck.begin(), InstsToCheck.end(),
                   [&DI, &I](Instruction *CurInst) {
-                    auto DepResult = DI.depends(&I, CurInst, true);
+                    auto DepResult = DI->depends(&I, CurInst, true);
                     if (DepResult &&
                         (DepResult->isOutput() || DepResult->isFlow() ||
                          DepResult->isAnti()))
@@ -376,8 +390,8 @@ bool llvm::isSafeToMoveBefore(Instruction &I, Instruction &InsertPoint,
 }
 
 bool llvm::isSafeToMoveBefore(BasicBlock &BB, Instruction &InsertPoint,
-                              DominatorTree &DT, const PostDominatorTree &PDT,
-                              DependenceInfo &DI) {
+                              DominatorTree &DT, const PostDominatorTree *PDT,
+                              DependenceInfo *DI) {
   return llvm::all_of(BB, [&](Instruction &I) {
     if (BB.getTerminator() == &I)
       return true;
@@ -396,7 +410,7 @@ void llvm::moveInstructionsToTheBeginning(BasicBlock &FromBB, BasicBlock &ToBB,
     // Increment the iterator before modifying FromBB.
     ++It;
 
-    if (isSafeToMoveBefore(I, *MovePos, DT, PDT, DI))
+    if (isSafeToMoveBefore(I, *MovePos, DT, &PDT, &DI))
       I.moveBefore(MovePos);
   }
 }
@@ -408,7 +422,7 @@ void llvm::moveInstructionsToTheEnd(BasicBlock &FromBB, BasicBlock &ToBB,
   Instruction *MovePos = ToBB.getTerminator();
   while (FromBB.size() > 1) {
     Instruction &I = FromBB.front();
-    if (isSafeToMoveBefore(I, *MovePos, DT, PDT, DI))
+    if (isSafeToMoveBefore(I, *MovePos, DT, &PDT, &DI))
       I.moveBefore(MovePos);
   }
 }
diff --git a/llvm/lib/Transforms/Utils/Debugify.cpp b/llvm/lib/Transforms/Utils/Debugify.cpp
index 7c178aba5ad24..8f98d81a3d797 100644
--- a/llvm/lib/Transforms/Utils/Debugify.cpp
+++ b/llvm/lib/Transforms/Utils/Debugify.cpp
@@ -330,11 +330,10 @@ bool checkDebugifyMetadata(Module &M,
       }
 
       if (!DL) {
-        dbg() << "ERROR: Instruction with empty DebugLoc in function ";
+        dbg() << "WARNING: Instruction with empty DebugLoc in function ";
         dbg() << F.getName() << " --";
         I.print(dbg());
         dbg() << "\n";
-        HasErrors = true;
       }
     }
 
diff --git a/llvm/lib/Transforms/Utils/EntryExitInstrumenter.cpp b/llvm/lib/Transforms/Utils/EntryExitInstrumenter.cpp
index 651f776a4915b..f84ff9e5aad1d 100644
--- a/llvm/lib/Transforms/Utils/EntryExitInstrumenter.cpp
+++ b/llvm/lib/Transforms/Utils/EntryExitInstrumenter.cpp
@@ -11,6 +11,7 @@
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
 #include "llvm/InitializePasses.h"
diff --git a/llvm/lib/Transforms/Utils/FixIrreducible.cpp b/llvm/lib/Transforms/Utils/FixIrreducible.cpp
index 510c033f64743..452463c9b6277 100644
--- a/llvm/lib/Transforms/Utils/FixIrreducible.cpp
+++ b/llvm/lib/Transforms/Utils/FixIrreducible.cpp
@@ -281,6 +281,9 @@ static bool makeReducible(LoopInfo &LI, DominatorTree &DT, Graph &&G) {
     LLVM_DEBUG(dbgs() << "Found headers:");
     for (auto BB : reverse(Blocks)) {
       for (const auto P : predecessors(BB)) {
+        // Skip unreachable predecessors.
+        if (!DT.isReachableFromEntry(P))
+          continue;
         if (!Blocks.count(P)) {
           LLVM_DEBUG(dbgs() << " " << BB->getName());
           Headers.insert(BB);
diff --git a/llvm/lib/Transforms/Utils/FlattenCFG.cpp b/llvm/lib/Transforms/Utils/FlattenCFG.cpp
index 99927486fc18c..0098dcaeb07a0 100644
--- a/llvm/lib/Transforms/Utils/FlattenCFG.cpp
+++ b/llvm/lib/Transforms/Utils/FlattenCFG.cpp
@@ -97,7 +97,7 @@ class FlattenCFGOpt {
 ///    br label %if.end;
 ///
 ///  Current implementation handles two cases.
-///  Case 1: \param BB is on the else-path.
+///  Case 1: BB is on the else-path.
 ///
 ///          BB1
 ///        /     |
@@ -105,7 +105,7 @@ class FlattenCFGOpt {
 ///      /   \   |
 ///     BB3   \  |     where, BB1, BB2 contain conditional branches.
 ///      \    |  /     BB3 contains unconditional branch.
-///       \   | /      BB4 corresponds to \param BB which is also the merge.
+///       \   | /      BB4 corresponds to BB which is also the merge.
 ///  BB => BB4
 ///
 ///
@@ -114,14 +114,14 @@ class FlattenCFGOpt {
 ///  if (a == b && c == d)
 ///    statement; // BB3
 ///
-///  Case 2: \param BB BB is on the then-path.
+///  Case 2: BB is on the then-path.
 ///
 ///             BB1
 ///          /      |
 ///         |      BB2
 ///         \    /    |  where BB1, BB2 contain conditional branches.
 ///  BB =>   BB3      |  BB3 contains unconditiona branch and corresponds
-///           \     /    to \param BB.  BB4 is the merge.
+///           \     /    to BB.  BB4 is the merge.
 ///             BB4
 ///
 ///  Corresponding source code:
@@ -129,9 +129,9 @@ class FlattenCFGOpt {
 ///  if (a == b || c == d)
 ///    statement;  // BB3
 ///
-///  In both cases,  \param BB is the common successor of conditional branches.
-///  In Case 1, \param BB (BB4) has an unconditional branch (BB3) as
-///  its predecessor.  In Case 2, \param BB (BB3) only has conditional branches
+///  In both cases, BB is the common successor of conditional branches.
+///  In Case 1, BB (BB4) has an unconditional branch (BB3) as
+///  its predecessor.  In Case 2, BB (BB3) only has conditional branches
 ///  as its predecessors.
 bool FlattenCFGOpt::FlattenParallelAndOr(BasicBlock *BB, IRBuilder<> &Builder) {
   PHINode *PHI = dyn_cast<PHINode>(BB->begin());
@@ -318,8 +318,8 @@ bool FlattenCFGOpt::FlattenParallelAndOr(BasicBlock *BB, IRBuilder<> &Builder) {
 /// Compare blocks from two if-regions, where \param Head2 is the entry of the
 /// 2nd if-region. \param Block1 is a block in the 1st if-region to compare.
 /// \param Block2 is a block in the 2nd if-region to compare.  \returns true if
-/// \param Block1 and \param Block2 have identical instructions and do not have
-/// memory reference alias with \param Head2.
+/// Block1 and Block2 have identical instructions and do not have
+/// memory reference alias with Head2.
 bool FlattenCFGOpt::CompareIfRegionBlock(BasicBlock *Block1, BasicBlock *Block2,
                                          BasicBlock *Head2) {
   Instruction *PTI2 = Head2->getTerminator();
@@ -540,6 +540,6 @@ bool FlattenCFGOpt::run(BasicBlock *BB) {
 /// FlattenCFG - This function is used to flatten a CFG.  For
 /// example, it uses parallel-and and parallel-or mode to collapse
 /// if-conditions and merge if-regions with identical statements.
-bool llvm::FlattenCFG(BasicBlock *BB, AliasAnalysis *AA) {
+bool llvm::FlattenCFG(BasicBlock *BB, AAResults *AA) {
   return FlattenCFGOpt(AA).run(BB);
 }
diff --git a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp
index 83616f0f99e77..9d8f59d62d6d0 100644
--- a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp
+++ b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp
@@ -14,6 +14,7 @@
 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/DemandedBits.h"
+#include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/InstIterator.h"
@@ -144,6 +145,7 @@ void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addPreserved<LoopAccessLegacyAnalysis>();
   AU.addPreserved<DemandedBitsWrapperPass>();
   AU.addPreserved<OptimizationRemarkEmitterWrapperPass>();
+  AU.addPreserved<GlobalsAAWrapperPass>();
 }
 
 ////////////////////////////////////////////////////////////////////////////////
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index ad2dd393d6a57..b0b7ca4847980 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -1291,7 +1291,11 @@ static void UpdateCallGraphAfterInlining(CallBase &CB,
   }
 
   for (; I != E; ++I) {
-    const Value *OrigCall = I->first;
+    // Skip 'refererence' call records.
+    if (!I->first)
+      continue;
+
+    const Value *OrigCall = *I->first;
 
     ValueToValueMapTy::iterator VMI = VMap.find(OrigCall);
     // Only copy the edge if the call was inlined!
@@ -1860,13 +1864,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
           OpDefs.emplace_back("deopt", std::move(MergedDeoptArgs));
         }
 
-        Instruction *NewI = nullptr;
-        if (isa<CallInst>(ICS))
-          NewI = CallInst::Create(cast<CallInst>(ICS), OpDefs, ICS);
-        else if (isa<CallBrInst>(ICS))
-          NewI = CallBrInst::Create(cast<CallBrInst>(ICS), OpDefs, ICS);
-        else
-          NewI = InvokeInst::Create(cast<InvokeInst>(ICS), OpDefs, ICS);
+        Instruction *NewI = CallBase::Create(ICS, OpDefs, ICS);
 
         // Note: the RAUW does the appropriate fixup in VMap, so we need to do
         // this even if the call returns void.
@@ -2162,13 +2160,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
         I->getOperandBundlesAsDefs(OpBundles);
         OpBundles.emplace_back("funclet", CallSiteEHPad);
 
-        Instruction *NewInst;
-        if (auto *CallI = dyn_cast<CallInst>(I))
-          NewInst = CallInst::Create(CallI, OpBundles, CallI);
-        else if (auto *CallBrI = dyn_cast<CallBrInst>(I))
-          NewInst = CallBrInst::Create(CallBrI, OpBundles, CallBrI);
-        else
-          NewInst = InvokeInst::Create(cast<InvokeInst>(I), OpBundles, I);
+        Instruction *NewInst = CallBase::Create(I, OpBundles, I);
         NewInst->takeName(I);
         I->replaceAllUsesWith(NewInst);
         I->eraseFromParent();
diff --git a/llvm/lib/Transforms/Utils/LoopSimplify.cpp b/llvm/lib/Transforms/Utils/LoopSimplify.cpp
index c33f36f70a888..a8445e94e55a0 100644
--- a/llvm/lib/Transforms/Utils/LoopSimplify.cpp
+++ b/llvm/lib/Transforms/Utils/LoopSimplify.cpp
@@ -696,10 +696,8 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
       LI->removeBlock(ExitingBlock);
 
       DomTreeNode *Node = DT->getNode(ExitingBlock);
-      const std::vector<DomTreeNodeBase<BasicBlock> *> &Children =
-        Node->getChildren();
-      while (!Children.empty()) {
-        DomTreeNode *Child = Children.front();
+      while (!Node->isLeaf()) {
+        DomTreeNode *Child = Node->back();
         DT->changeImmediateDominator(Child, Node->getIDom());
       }
       DT->eraseNode(ExitingBlock);
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index b86c62a78ce8f..3875c631f839b 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -809,7 +809,7 @@ LoopUnrollResult llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
     for (auto *BB : OriginalLoopBlocks) {
       auto *BBDomNode = DT->getNode(BB);
       SmallVector<BasicBlock *, 16> ChildrenToUpdate;
-      for (auto *ChildDomNode : BBDomNode->getChildren()) {
+      for (auto *ChildDomNode : BBDomNode->children()) {
         auto *ChildBB = ChildDomNode->getBlock();
         if (!L->contains(ChildBB))
           ChildrenToUpdate.push_back(ChildBB);
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollPeel.cpp b/llvm/lib/Transforms/Utils/LoopUnrollPeel.cpp
index 43dfaf3e50dc9..c653aacbee6cc 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollPeel.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollPeel.cpp
@@ -279,19 +279,20 @@ static unsigned countToEliminateCompares(Loop &L, unsigned MaxPeelCount,
 // Return the number of iterations we want to peel off.
 void llvm::computePeelCount(Loop *L, unsigned LoopSize,
                             TargetTransformInfo::UnrollingPreferences &UP,
+                            TargetTransformInfo::PeelingPreferences &PP,
                             unsigned &TripCount, ScalarEvolution &SE) {
   assert(LoopSize > 0 && "Zero loop size is not allowed!");
-  // Save the UP.PeelCount value set by the target in
-  // TTI.getUnrollingPreferences or by the flag -unroll-peel-count.
-  unsigned TargetPeelCount = UP.PeelCount;
-  UP.PeelCount = 0;
+  // Save the PP.PeelCount value set by the target in
+  // TTI.getPeelingPreferences or by the flag -unroll-peel-count.
+  unsigned TargetPeelCount = PP.PeelCount;
+  PP.PeelCount = 0;
   if (!canPeel(L))
     return;
 
   // Only try to peel innermost loops by default.
   // The constraint can be relaxed by the target in TTI.getUnrollingPreferences
   // or by the flag -unroll-allow-loop-nests-peeling.
-  if (!UP.AllowLoopNestsPeeling && !L->empty())
+  if (!PP.AllowLoopNestsPeeling && !L->empty())
     return;
 
   // If the user provided a peel count, use that.
@@ -299,13 +300,13 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
   if (UserPeelCount) {
     LLVM_DEBUG(dbgs() << "Force-peeling first " << UnrollForcePeelCount
                       << " iterations.\n");
-    UP.PeelCount = UnrollForcePeelCount;
-    UP.PeelProfiledIterations = true;
+    PP.PeelCount = UnrollForcePeelCount;
+    PP.PeelProfiledIterations = true;
     return;
   }
 
   // Skip peeling if it's disabled.
-  if (!UP.AllowPeeling)
+  if (!PP.AllowPeeling)
     return;
 
   unsigned AlreadyPeeled = 0;
@@ -354,8 +355,8 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
         LLVM_DEBUG(dbgs() << "Peel " << DesiredPeelCount
                           << " iteration(s) to turn"
                           << " some Phis into invariants.\n");
-        UP.PeelCount = DesiredPeelCount;
-        UP.PeelProfiledIterations = false;
+        PP.PeelCount = DesiredPeelCount;
+        PP.PeelProfiledIterations = false;
         return;
       }
     }
@@ -367,7 +368,7 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
     return;
 
   // Do not apply profile base peeling if it is disabled.
-  if (!UP.PeelProfiledIterations)
+  if (!PP.PeelProfiledIterations)
     return;
   // If we don't know the trip count, but have reason to believe the average
   // trip count is low, peeling should be beneficial, since we will usually
@@ -387,7 +388,7 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
           (LoopSize * (*PeelCount + 1) <= UP.Threshold)) {
         LLVM_DEBUG(dbgs() << "Peeling first " << *PeelCount
                           << " iterations.\n");
-        UP.PeelCount = *PeelCount;
+        PP.PeelCount = *PeelCount;
         return;
       }
       LLVM_DEBUG(dbgs() << "Requested peel count: " << *PeelCount << "\n");
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index c0d97cf7eeca3..2515b1676cb99 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -848,7 +848,7 @@ bool llvm::UnrollRuntimeLoopRemainder(
     // dominator of the exit blocks.
     for (auto *BB : L->blocks()) {
       auto *DomNodeBB = DT->getNode(BB);
-      for (auto *DomChild : DomNodeBB->getChildren()) {
+      for (auto *DomChild : DomNodeBB->children()) {
         auto *DomChildBB = DomChild->getBlock();
         if (!L->contains(LI->getLoopFor(DomChildBB)))
           ChildrenToUpdate.push_back(DomChildBB);
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 9241377012a45..43363736684ee 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -512,9 +512,10 @@ llvm::collectChildrenInLoop(DomTreeNode *N, const Loop *CurLoop) {
 
   AddRegionToWorklist(N);
 
-  for (size_t I = 0; I < Worklist.size(); I++)
-    for (DomTreeNode *Child : Worklist[I]->getChildren())
+  for (size_t I = 0; I < Worklist.size(); I++) {
+    for (DomTreeNode *Child : Worklist[I]->children())
       AddRegionToWorklist(Child);
+  }
 
   return Worklist;
 }
diff --git a/llvm/lib/Transforms/Utils/PredicateInfo.cpp b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
index a3ac208e1dca3..c81efd77aa5ff 100644
--- a/llvm/lib/Transforms/Utils/PredicateInfo.cpp
+++ b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
@@ -31,6 +31,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/InitializePasses.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/DebugCounter.h"
 #include "llvm/Support/FormattedStream.h"
@@ -599,6 +600,9 @@ Value *PredicateInfoBuilder::materializeStack(unsigned int &Counter,
         RenameIter == RenameStack.begin() ? OrigOp : (RenameIter - 1)->Def;
     ValueDFS &Result = *RenameIter;
     auto *ValInfo = Result.PInfo;
+    ValInfo->RenamedOp = (RenameStack.end() - Start) == RenameStack.begin()
+                             ? OrigOp
+                             : (RenameStack.end() - Start - 1)->Def;
     // For edge predicates, we can just place the operand in the block before
     // the terminator.  For assume, we have to place it right before the assume
     // to ensure we dominate all of our uses.  Always insert right before the
@@ -892,18 +896,21 @@ class PredicateInfoAnnotatedWriter : public AssemblyAnnotationWriter {
         PB->From->printAsOperand(OS);
         OS << ",";
         PB->To->printAsOperand(OS);
-        OS << "] }\n";
+        OS << "]";
       } else if (const auto *PS = dyn_cast<PredicateSwitch>(PI)) {
         OS << "; switch predicate info { CaseValue: " << *PS->CaseValue
            << " Switch:" << *PS->Switch << " Edge: [";
         PS->From->printAsOperand(OS);
         OS << ",";
         PS->To->printAsOperand(OS);
-        OS << "] }\n";
+        OS << "]";
       } else if (const auto *PA = dyn_cast<PredicateAssume>(PI)) {
         OS << "; assume predicate info {"
-           << " Comparison:" << *PA->Condition << " }\n";
+           << " Comparison:" << *PA->Condition;
       }
+      OS << ", RenamedOp: ";
+      PI->RenamedOp->printAsOperand(OS, false);
+      OS << " }\n";
     }
   }
 };
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 71b48482f26aa..c54ae26b53234 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -1292,7 +1292,8 @@ SCEVExpander::getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
   if (useSubtract)
     Step = SE.getNegativeSCEV(Step);
   // Expand the step somewhere that dominates the loop header.
-  Value *StepV = expandCodeFor(Step, IntTy, &L->getHeader()->front());
+  Value *StepV = expandCodeFor(Step, IntTy,
+                               &*L->getHeader()->getFirstInsertionPt());
 
   // The no-wrap behavior proved by IsIncrement(NUW|NSW) is only applicable if
   // we actually do emit an addition.  It does not apply if we emit a
@@ -1438,7 +1439,8 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
       {
         // Expand the step somewhere that dominates the loop header.
         SCEVInsertPointGuard Guard(Builder, this);
-        StepV = expandCodeFor(Step, IntTy, &L->getHeader()->front());
+        StepV = expandCodeFor(Step, IntTy,
+                              &*L->getHeader()->getFirstInsertionPt());
       }
       Result = expandIVInc(PN, StepV, L, ExpandTy, IntTy, useSubtract);
     }
@@ -1870,11 +1872,6 @@ Value *SCEVExpander::expand(const SCEV *S) {
     }
   }
 
-  // IndVarSimplify sometimes sets the insertion point at the block start, even
-  // when there are PHIs at that point.  We must correct for this.
-  if (isa<PHINode>(*InsertPt))
-    InsertPt = &*InsertPt->getParent()->getFirstInsertionPt();
-
   // Check to see if we already expanded this here.
   auto I = InsertedExpressions.find(std::make_pair(S, InsertPt));
   if (I != InsertedExpressions.end())
@@ -1945,7 +1942,8 @@ SCEVExpander::getOrInsertCanonicalInductionVariable(const Loop *L,
   // Emit code for it.
   SCEVInsertPointGuard Guard(Builder, this);
   PHINode *V =
-      cast<PHINode>(expandCodeFor(H, nullptr, &L->getHeader()->front()));
+      cast<PHINode>(expandCodeFor(H, nullptr,
+                                  &*L->getHeader()->getFirstInsertionPt()));
 
   return V;
 }
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 7b36907f70114..b450d71c996cb 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -133,6 +133,11 @@ static cl::opt<unsigned> MaxSpeculationDepth(
     cl::desc("Limit maximum recursion depth when calculating costs of "
              "speculatively executed instructions"));
 
+static cl::opt<int>
+MaxSmallBlockSize("simplifycfg-max-small-block-size", cl::Hidden, cl::init(10),
+                  cl::desc("Max size of a block which is still considered "
+                           "small enough to thread through"));
+
 STATISTIC(NumBitMaps, "Number of switch instructions turned into bitmaps");
 STATISTIC(NumLinearMaps,
           "Number of switch instructions turned into linear mapping");
@@ -2141,9 +2146,14 @@ bool SimplifyCFGOpt::SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB,
   }
 
   // Metadata can be dependent on the condition we are hoisting above.
-  // Conservatively strip all metadata on the instruction.
-  for (auto &I : *ThenBB)
+  // Conservatively strip all metadata on the instruction. Drop the debug loc
+  // to avoid making it appear as if the condition is a constant, which would
+  // be misleading while debugging.
+  for (auto &I : *ThenBB) {
+    if (!SpeculatedStoreValue || &I != SpeculatedStore)
+      I.setDebugLoc(DebugLoc());
     I.dropUnknownNonDebugMetadata();
+  }
 
   // Hoist the instructions.
   BB->getInstList().splice(BI->getIterator(), ThenBB->getInstList(),
@@ -2184,12 +2194,15 @@ bool SimplifyCFGOpt::SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB,
 
 /// Return true if we can thread a branch across this block.
 static bool BlockIsSimpleEnoughToThreadThrough(BasicBlock *BB) {
-  unsigned Size = 0;
+  int Size = 0;
 
   for (Instruction &I : BB->instructionsWithoutDebug()) {
-    if (Size > 10)
+    if (Size > MaxSmallBlockSize)
       return false; // Don't clone large BB's.
-    ++Size;
+    // We will delete Phis while threading, so Phis should not be accounted in
+    // block's size
+    if (!isa<PHINode>(I))
+      ++Size;
 
     // We can only support instructions that do not define values that are
     // live outside of the current basic block.
@@ -2640,7 +2653,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, MemorySSAUpdater *MSSAU,
             }
             // Quit if we can't remove this instruction.
             if (!tryCSEWithPredecessor(Curr, PB))
-              return false;
+              return Changed;
             Changed = true;
           }
         }
@@ -3134,29 +3147,11 @@ static bool mergeConditionalStoreToAddress(BasicBlock *PTB, BasicBlock *PFB,
   PStore->getAAMetadata(AAMD, /*Merge=*/false);
   PStore->getAAMetadata(AAMD, /*Merge=*/true);
   SI->setAAMetadata(AAMD);
-  unsigned PAlignment = PStore->getAlignment();
-  unsigned QAlignment = QStore->getAlignment();
-  unsigned TypeAlignment =
-      DL.getABITypeAlignment(SI->getValueOperand()->getType());
-  unsigned MinAlignment;
-  unsigned MaxAlignment;
-  std::tie(MinAlignment, MaxAlignment) = std::minmax(PAlignment, QAlignment);
   // Choose the minimum alignment. If we could prove both stores execute, we
   // could use biggest one.  In this case, though, we only know that one of the
   // stores executes.  And we don't know it's safe to take the alignment from a
   // store that doesn't execute.
-  if (MinAlignment != 0) {
-    // Choose the minimum of all non-zero alignments.
-    SI->setAlignment(Align(MinAlignment));
-  } else if (MaxAlignment != 0) {
-    // Choose the minimal alignment between the non-zero alignment and the ABI
-    // default alignment for the type of the stored value.
-    SI->setAlignment(Align(std::min(MaxAlignment, TypeAlignment)));
-  } else {
-    // If both alignments are zero, use ABI default alignment for the type of
-    // the stored value.
-    SI->setAlignment(Align(TypeAlignment));
-  }
+  SI->setAlignment(std::min(PStore->getAlign(), QStore->getAlign()));
 
   QStore->eraseFromParent();
   PStore->eraseFromParent();
diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
index 05632ca40fb08..d3d0c33419085 100644
--- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -657,7 +657,7 @@ static Instruction *GetLoopInvariantInsertPosition(Loop *L, Instruction *Hint) {
   return Hint;
 }
 
-/// Replace the UseInst with a constant if possible.
+/// Replace the UseInst with a loop invariant expression if it is safe.
 bool SimplifyIndvar::replaceIVUserWithLoopInvariant(Instruction *I) {
   if (!SE->isSCEVable(I->getType()))
     return false;
@@ -673,6 +673,13 @@ bool SimplifyIndvar::replaceIVUserWithLoopInvariant(Instruction *I) {
     return false;
 
   auto *IP = GetLoopInvariantInsertPosition(L, I);
+
+  if (!isSafeToExpandAt(S, IP, *SE)) {
+    LLVM_DEBUG(dbgs() << "INDVARS: Can not replace IV user: " << *I
+                      << " with non-speculable loop invariant: " << *S << '\n');
+    return false;
+  }
+
   auto *Invariant = Rewriter.expandCodeFor(S, I->getType(), IP);
 
   I->replaceAllUsesWith(Invariant);
diff --git a/llvm/lib/Transforms/Utils/SizeOpts.cpp b/llvm/lib/Transforms/Utils/SizeOpts.cpp
index f18f6b4c5a6ae..e257c5a015f51 100644
--- a/llvm/lib/Transforms/Utils/SizeOpts.cpp
+++ b/llvm/lib/Transforms/Utils/SizeOpts.cpp
@@ -39,7 +39,7 @@ cl::opt<bool> PGSOColdCodeOnlyForSamplePGO(
              "to cold code under sample PGO."));
 
 cl::opt<bool> PGSOColdCodeOnlyForPartialSamplePGO(
-    "pgso-cold-code-only-for-partial-sample-pgo", cl::Hidden, cl::init(true),
+    "pgso-cold-code-only-for-partial-sample-pgo", cl::Hidden, cl::init(false),
     cl::desc("Apply the profile guided size optimizations only "
              "to cold code under partial-profile sample PGO."));
 
diff --git a/llvm/lib/Transforms/Utils/Utils.cpp b/llvm/lib/Transforms/Utils/Utils.cpp
index fc1e824aadea7..ce98a739bea88 100644
--- a/llvm/lib/Transforms/Utils/Utils.cpp
+++ b/llvm/lib/Transforms/Utils/Utils.cpp
@@ -25,6 +25,7 @@ using namespace llvm;
 void llvm::initializeTransformUtils(PassRegistry &Registry) {
   initializeAddDiscriminatorsLegacyPassPass(Registry);
   initializeAssumeSimplifyPassLegacyPassPass(Registry);
+  initializeAssumeBuilderPassLegacyPassPass(Registry);
   initializeBreakCriticalEdgesPass(Registry);
   initializeCanonicalizeAliasesLegacyPassPass(Registry);
   initializeCanonicalizeFreezeInLoopsPass(Registry);
diff --git a/llvm/lib/Transforms/Utils/VNCoercion.cpp b/llvm/lib/Transforms/Utils/VNCoercion.cpp
index 091bf620350d2..c9839121e7f64 100644
--- a/llvm/lib/Transforms/Utils/VNCoercion.cpp
+++ b/llvm/lib/Transforms/Utils/VNCoercion.cpp
@@ -1,12 +1,11 @@
 #include "llvm/Transforms/Utils/VNCoercion.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "vncoerce"
+
 namespace llvm {
 namespace VNCoercion {
 
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 4885dd4351d5c..9b81afbb4b6cb 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -1073,7 +1073,7 @@ bool Vectorizer::vectorizeStoreChain(
       return false;
   }
 
-  if (!TTI.isLegalToVectorizeStoreChain(SzInBytes, Alignment.value(), AS)) {
+  if (!TTI.isLegalToVectorizeStoreChain(SzInBytes, Alignment, AS)) {
     auto Chains = splitOddVectorElts(Chain, Sz);
     return vectorizeStoreChain(Chains.first, InstructionsProcessed) |
            vectorizeStoreChain(Chains.second, InstructionsProcessed);
@@ -1218,7 +1218,7 @@ bool Vectorizer::vectorizeLoadChain(
       return false;
   }
 
-  if (!TTI.isLegalToVectorizeLoadChain(SzInBytes, Alignment.value(), AS)) {
+  if (!TTI.isLegalToVectorizeLoadChain(SzInBytes, Alignment, AS)) {
     auto Chains = splitOddVectorElts(Chain, Sz);
     return vectorizeLoadChain(Chains.first, InstructionsProcessed) |
            vectorizeLoadChain(Chains.second, InstructionsProcessed);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ec756671ea672..35af8e4257789 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -431,8 +431,9 @@ class InnerLoopVectorizer {
 
   /// Vectorize a single GetElementPtrInst based on information gathered and
   /// decisions taken during planning.
-  void widenGEP(GetElementPtrInst *GEP, unsigned UF, unsigned VF,
-                bool IsPtrLoopInvariant, SmallBitVector &IsIndexLoopInvariant);
+  void widenGEP(GetElementPtrInst *GEP, VPUser &Indices, unsigned UF,
+                unsigned VF, bool IsPtrLoopInvariant,
+                SmallBitVector &IsIndexLoopInvariant, VPTransformState &State);
 
   /// Vectorize a single PHINode in a block. This method handles the induction
   /// variable canonicalization. It supports both VF = 1 for unrolled loops and
@@ -686,7 +687,7 @@ class InnerLoopVectorizer {
   DominatorTree *DT;
 
   /// Alias Analysis.
-  AliasAnalysis *AA;
+  AAResults *AA;
 
   /// Target Library Info.
   const TargetLibraryInfo *TLI;
@@ -2876,6 +2877,18 @@ Value *InnerLoopVectorizer::emitTransformedIndex(
     return B.CreateMul(X, Y);
   };
 
+  // Get a suitable insert point for SCEV expansion. For blocks in the vector
+  // loop, choose the end of the vector loop header (=LoopVectorBody), because
+  // the DomTree is not kept up-to-date for additional blocks generated in the
+  // vector loop. By using the header as insertion point, we guarantee that the
+  // expanded instructions dominate all their uses.
+  auto GetInsertPoint = [this, &B]() {
+    BasicBlock *InsertBB = B.GetInsertPoint()->getParent();
+    if (InsertBB != LoopVectorBody &&
+        LI->getLoopFor(LoopVectorBody) == LI->getLoopFor(InsertBB))
+      return LoopVectorBody->getTerminator();
+    return &*B.GetInsertPoint();
+  };
   switch (ID.getKind()) {
   case InductionDescriptor::IK_IntInduction: {
     assert(Index->getType() == StartValue->getType() &&
@@ -2883,7 +2896,7 @@ Value *InnerLoopVectorizer::emitTransformedIndex(
     if (ID.getConstIntStepValue() && ID.getConstIntStepValue()->isMinusOne())
       return B.CreateSub(StartValue, Index);
     auto *Offset = CreateMul(
-        Index, Exp.expandCodeFor(Step, Index->getType(), &*B.GetInsertPoint()));
+        Index, Exp.expandCodeFor(Step, Index->getType(), GetInsertPoint()));
     return CreateAdd(StartValue, Offset);
   }
   case InductionDescriptor::IK_PtrInduction: {
@@ -2891,8 +2904,8 @@ Value *InnerLoopVectorizer::emitTransformedIndex(
            "Expected constant step for pointer induction");
     return B.CreateGEP(
         StartValue->getType()->getPointerElementType(), StartValue,
-        CreateMul(Index, Exp.expandCodeFor(Step, Index->getType(),
-                                           &*B.GetInsertPoint())));
+        CreateMul(Index,
+                  Exp.expandCodeFor(Step, Index->getType(), GetInsertPoint())));
   }
   case InductionDescriptor::IK_FpInduction: {
     assert(Step->getType()->isFloatingPointTy() && "Expected FP Step value");
@@ -4074,9 +4087,11 @@ void InnerLoopVectorizer::fixNonInductionPHIs() {
   }
 }
 
-void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, unsigned UF,
-                                   unsigned VF, bool IsPtrLoopInvariant,
-                                   SmallBitVector &IsIndexLoopInvariant) {
+void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPUser &Operands,
+                                   unsigned UF, unsigned VF,
+                                   bool IsPtrLoopInvariant,
+                                   SmallBitVector &IsIndexLoopInvariant,
+                                   VPTransformState &State) {
   // Construct a vector GEP by widening the operands of the scalar GEP as
   // necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP
   // results in a vector of pointers when at least one operand of the GEP
@@ -4113,19 +4128,18 @@ void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, unsigned UF,
     for (unsigned Part = 0; Part < UF; ++Part) {
       // The pointer operand of the new GEP. If it's loop-invariant, we
       // won't broadcast it.
-      auto *Ptr = IsPtrLoopInvariant
-                      ? GEP->getPointerOperand()
-                      : getOrCreateVectorValue(GEP->getPointerOperand(), Part);
+      auto *Ptr = IsPtrLoopInvariant ? State.get(Operands.getOperand(0), {0, 0})
+                                     : State.get(Operands.getOperand(0), Part);
 
       // Collect all the indices for the new GEP. If any index is
       // loop-invariant, we won't broadcast it.
       SmallVector<Value *, 4> Indices;
-      for (auto Index : enumerate(GEP->indices())) {
-        Value *User = Index.value().get();
-        if (IsIndexLoopInvariant[Index.index()])
-          Indices.push_back(User);
+      for (unsigned I = 1, E = Operands.getNumOperands(); I < E; I++) {
+        VPValue *Operand = Operands.getOperand(I);
+        if (IsIndexLoopInvariant[I - 1])
+          Indices.push_back(State.get(Operand, {0, 0}));
         else
-          Indices.push_back(getOrCreateVectorValue(User, Part));
+          Indices.push_back(State.get(Operand, Part));
       }
 
       // Create the new GEP. Note that this GEP may be a scalar if VF == 1,
@@ -4937,10 +4951,9 @@ bool LoopVectorizationCostModel::runtimeChecksRequired() {
 
   // FIXME: Avoid specializing for stride==1 instead of bailing out.
   if (!Legal->getLAI()->getSymbolicStrides().empty()) {
-    reportVectorizationFailure("Runtime stride check is required with -Os/-Oz",
+    reportVectorizationFailure("Runtime stride check for small trip count",
         "runtime stride == 1 checks needed. Enable vectorization of "
-        "this loop with '#pragma clang loop vectorize(enable)' when "
-        "compiling with -Os/-Oz",
+        "this loop without such check by compiling with -Os/-Oz",
         "CantVersionLoopWithOptForSize", ORE, TheLoop);
     return true;
   }
@@ -5890,8 +5903,8 @@ unsigned LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I,
   const Align Alignment = getLoadStoreAlignment(I);
   unsigned Cost = 0;
   if (Legal->isMaskRequired(I))
-    Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy,
-                                      Alignment.value(), AS, CostKind);
+    Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
+                                      CostKind);
   else
     Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
                                 CostKind, I);
@@ -5932,13 +5945,12 @@ unsigned LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
   Type *ValTy = getMemInstValueType(I);
   auto *VectorTy = cast<VectorType>(ToVectorTy(ValTy, VF));
   const Align Alignment = getLoadStoreAlignment(I);
-  Value *Ptr = getLoadStorePointerOperand(I);
+  const Value *Ptr = getLoadStorePointerOperand(I);
 
   return TTI.getAddressComputationCost(VectorTy) +
-         TTI.getGatherScatterOpCost(I->getOpcode(), VectorTy, Ptr,
-                                    Legal->isMaskRequired(I), Alignment.value(),
-                                    TargetTransformInfo::TCK_RecipThroughput,
-                                    I);
+         TTI.getGatherScatterOpCost(
+             I->getOpcode(), VectorTy, Ptr, Legal->isMaskRequired(I), Alignment,
+             TargetTransformInfo::TCK_RecipThroughput, I);
 }
 
 unsigned LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
@@ -5966,9 +5978,8 @@ unsigned LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
   bool UseMaskForGaps =
       Group->requiresScalarEpilogue() && !isScalarEpilogueAllowed();
   unsigned Cost = TTI.getInterleavedMemoryOpCost(
-      I->getOpcode(), WideVecTy, Group->getFactor(), Indices,
-      Group->getAlign().value(), AS, TTI::TCK_RecipThroughput,
-      Legal->isMaskRequired(I), UseMaskForGaps);
+      I->getOpcode(), WideVecTy, Group->getFactor(), Indices, Group->getAlign(),
+      AS, TTI::TCK_RecipThroughput, Legal->isMaskRequired(I), UseMaskForGaps);
 
   if (Group->isReverse()) {
     // TODO: Add support for reversed masked interleaved access.
@@ -7141,7 +7152,8 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
     return nullptr;
 
   if (auto GEP = dyn_cast<GetElementPtrInst>(Instr))
-    return new VPWidenGEPRecipe(GEP, OrigLoop);
+    return new VPWidenGEPRecipe(GEP, Plan->mapToVPValues(GEP->operands()),
+                                OrigLoop);
 
   if (auto *SI = dyn_cast<SelectInst>(Instr)) {
     bool InvariantCond =
@@ -7449,8 +7461,8 @@ void VPWidenRecipe::execute(VPTransformState &State) {
 }
 
 void VPWidenGEPRecipe::execute(VPTransformState &State) {
-  State.ILV->widenGEP(GEP, State.UF, State.VF, IsPtrLoopInvariant,
-                      IsIndexLoopInvariant);
+  State.ILV->widenGEP(GEP, User, State.UF, State.VF, IsPtrLoopInvariant,
+                      IsIndexLoopInvariant, State);
 }
 
 void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
@@ -7610,7 +7622,7 @@ static ScalarEpilogueLowering getScalarEpilogueLowering(
                                                      PGSOQueryType::IRPass);
   // 1) OptSize takes precedence over all other options, i.e. if this is set,
   // don't look at hints or options, and don't request a scalar epilogue.
-  if (OptSize && Hints.getForce() != LoopVectorizeHints::FK_Enabled)
+  if (OptSize)
     return CM_ScalarEpilogueNotAllowedOptSize;
 
   bool PredicateOptDisabled = PreferPredicateOverEpilog.getNumOccurrences() &&
@@ -7989,14 +8001,14 @@ bool LoopVectorizePass::processLoop(Loop *L) {
     Hints.setAlreadyVectorized();
   }
 
-  assert(!verifyFunction(*L->getHeader()->getParent()));
+  assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()));
   return true;
 }
 
 LoopVectorizeResult LoopVectorizePass::runImpl(
     Function &F, ScalarEvolution &SE_, LoopInfo &LI_, TargetTransformInfo &TTI_,
     DominatorTree &DT_, BlockFrequencyInfo &BFI_, TargetLibraryInfo *TLI_,
-    DemandedBits &DB_, AliasAnalysis &AA_, AssumptionCache &AC_,
+    DemandedBits &DB_, AAResults &AA_, AssumptionCache &AC_,
     std::function<const LoopAccessInfo &(Loop &)> &GetLAA_,
     OptimizationRemarkEmitter &ORE_, ProfileSummaryInfo *PSI_) {
   SE = &SE_;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1afd120e1e7eb..d4b16fac985d9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -7361,6 +7361,7 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
   bool Changed = false;
   SmallVector<Value *, 4> Incoming;
   SmallPtrSet<Value *, 16> VisitedInstrs;
+  unsigned MaxVecRegSize = R.getMaxVecRegSize();
 
   bool HaveVectorizedPhiNodes = true;
   while (HaveVectorizedPhiNodes) {
@@ -7387,8 +7388,18 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
 
       // Look for the next elements with the same type.
       SmallVector<Value *, 4>::iterator SameTypeIt = IncIt;
+      Type *EltTy = (*IncIt)->getType();
+      unsigned EltSize = EltTy->isSized() ? DL->getTypeSizeInBits(EltTy)
+                                          : MaxVecRegSize;
+      unsigned MaxNumElts = MaxVecRegSize / EltSize;
+      if (MaxNumElts < 2) {
+        ++IncIt;
+        continue;
+      }
+
       while (SameTypeIt != E &&
-             (*SameTypeIt)->getType() == (*IncIt)->getType()) {
+             (*SameTypeIt)->getType() == EltTy &&
+             (SameTypeIt - IncIt) < MaxNumElts) {
         VisitedInstrs.insert(*SameTypeIt);
         ++SameTypeIt;
       }
@@ -7503,9 +7514,12 @@ bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
                       << Entry.second.size() << ".\n");
 
     // Process the GEP list in chunks suitable for the target's supported
-    // vector size. If a vector register can't hold 1 element, we are done.
+    // vector size. If a vector register can't hold 1 element, we are done. We
+    // are trying to vectorize the index computations, so the maximum number of
+    // elements is based on the size of the index expression, rather than the
+    // size of the GEP itself (the target's pointer size).
     unsigned MaxVecRegSize = R.getMaxVecRegSize();
-    unsigned EltSize = R.getVectorElementSize(Entry.second[0]);
+    unsigned EltSize = R.getVectorElementSize(*Entry.second[0]->idx_begin());
     if (MaxVecRegSize < EltSize)
       continue;
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 914cb038adf40..f07c94e7a3c7d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -865,12 +865,18 @@ class VPWidenSelectRecipe : public VPRecipeBase {
 /// A recipe for handling GEP instructions.
 class VPWidenGEPRecipe : public VPRecipeBase {
   GetElementPtrInst *GEP;
+
+  /// Hold VPValues for the base and indices of the GEP.
+  VPUser User;
+
   bool IsPtrLoopInvariant;
   SmallBitVector IsIndexLoopInvariant;
 
 public:
-  VPWidenGEPRecipe(GetElementPtrInst *GEP, Loop *OrigLoop)
-      : VPRecipeBase(VPWidenGEPSC), GEP(GEP),
+  template <typename IterT>
+  VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range<IterT> Operands,
+                   Loop *OrigLoop)
+      : VPRecipeBase(VPWidenGEPSC), GEP(GEP), User(Operands),
         IsIndexLoopInvariant(GEP->getNumIndices(), false) {
     IsPtrLoopInvariant = OrigLoop->isLoopInvariant(GEP->getPointerOperand());
     for (auto Index : enumerate(GEP->indices()))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h b/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h
index 19f5d2c00c604..a42ebc9ee955f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h
@@ -30,7 +30,8 @@ using VPDomTreeNode = DomTreeNodeBase<VPBlockBase>;
 /// Template specializations of GraphTraits for VPDomTreeNode.
 template <>
 struct GraphTraits<VPDomTreeNode *>
-    : public DomTreeGraphTraitsBase<VPDomTreeNode, VPDomTreeNode::iterator> {};
+    : public DomTreeGraphTraitsBase<VPDomTreeNode,
+                                    VPDomTreeNode::const_iterator> {};
 
 template <>
 struct GraphTraits<const VPDomTreeNode *>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 2b48b2b591603..3a4872a721221 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -70,7 +70,8 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
         } else
           NewRecipe = new VPWidenPHIRecipe(Phi);
       } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
-        NewRecipe = new VPWidenGEPRecipe(GEP, OrigLoop);
+        NewRecipe = new VPWidenGEPRecipe(
+            GEP, Plan->mapToVPValues(GEP->operands()), OrigLoop);
       } else
         NewRecipe =
             new VPWidenRecipe(*Inst, Plan->mapToVPValues(Inst->operands()));
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index a5d08b4e9d461..64b41bf9cefa8 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -35,6 +35,7 @@ using namespace llvm::PatternMatch;
 #define DEBUG_TYPE "vector-combine"
 STATISTIC(NumVecCmp, "Number of vector compares formed");
 STATISTIC(NumVecBO, "Number of vector binops formed");
+STATISTIC(NumVecCmpBO, "Number of vector compare + binop formed");
 STATISTIC(NumShufOfBitcast, "Number of shuffles moved after bitcast");
 STATISTIC(NumScalarBO, "Number of scalar binops formed");
 STATISTIC(NumScalarCmp, "Number of scalar compares formed");
@@ -47,27 +48,85 @@ static cl::opt<bool> DisableBinopExtractShuffle(
     "disable-binop-extract-shuffle", cl::init(false), cl::Hidden,
     cl::desc("Disable binop extract to shuffle transforms"));
 
+static const unsigned InvalidIndex = std::numeric_limits<unsigned>::max();
+
+namespace {
 class VectorCombine {
 public:
   VectorCombine(Function &F, const TargetTransformInfo &TTI,
                 const DominatorTree &DT)
-      : F(F), TTI(TTI), DT(DT) {}
+      : F(F), Builder(F.getContext()), TTI(TTI), DT(DT) {}
 
   bool run();
 
 private:
   Function &F;
+  IRBuilder<> Builder;
   const TargetTransformInfo &TTI;
   const DominatorTree &DT;
 
+  ExtractElementInst *getShuffleExtract(ExtractElementInst *Ext0,
+                                        ExtractElementInst *Ext1,
+                                        unsigned PreferredExtractIndex) const;
   bool isExtractExtractCheap(ExtractElementInst *Ext0, ExtractElementInst *Ext1,
                              unsigned Opcode,
                              ExtractElementInst *&ConvertToShuffle,
                              unsigned PreferredExtractIndex);
+  void foldExtExtCmp(ExtractElementInst *Ext0, ExtractElementInst *Ext1,
+                     Instruction &I);
+  void foldExtExtBinop(ExtractElementInst *Ext0, ExtractElementInst *Ext1,
+                       Instruction &I);
   bool foldExtractExtract(Instruction &I);
   bool foldBitcastShuf(Instruction &I);
   bool scalarizeBinopOrCmp(Instruction &I);
+  bool foldExtractedCmps(Instruction &I);
 };
+} // namespace
+
+static void replaceValue(Value &Old, Value &New) {
+  Old.replaceAllUsesWith(&New);
+  New.takeName(&Old);
+}
+
+/// Determine which, if any, of the inputs should be replaced by a shuffle
+/// followed by extract from a different index.
+ExtractElementInst *VectorCombine::getShuffleExtract(
+    ExtractElementInst *Ext0, ExtractElementInst *Ext1,
+    unsigned PreferredExtractIndex = InvalidIndex) const {
+  assert(isa<ConstantInt>(Ext0->getIndexOperand()) &&
+         isa<ConstantInt>(Ext1->getIndexOperand()) &&
+         "Expected constant extract indexes");
+
+  unsigned Index0 = cast<ConstantInt>(Ext0->getIndexOperand())->getZExtValue();
+  unsigned Index1 = cast<ConstantInt>(Ext1->getIndexOperand())->getZExtValue();
+
+  // If the extract indexes are identical, no shuffle is needed.
+  if (Index0 == Index1)
+    return nullptr;
+
+  Type *VecTy = Ext0->getVectorOperand()->getType();
+  assert(VecTy == Ext1->getVectorOperand()->getType() && "Need matching types");
+  int Cost0 = TTI.getVectorInstrCost(Ext0->getOpcode(), VecTy, Index0);
+  int Cost1 = TTI.getVectorInstrCost(Ext1->getOpcode(), VecTy, Index1);
+
+  // We are extracting from 2 different indexes, so one operand must be shuffled
+  // before performing a vector operation and/or extract. The more expensive
+  // extract will be replaced by a shuffle.
+  if (Cost0 > Cost1)
+    return Ext0;
+  if (Cost1 > Cost0)
+    return Ext1;
+
+  // If the costs are equal and there is a preferred extract index, shuffle the
+  // opposite operand.
+  if (PreferredExtractIndex == Index0)
+    return Ext1;
+  if (PreferredExtractIndex == Index1)
+    return Ext0;
+
+  // Otherwise, replace the extract with the higher index.
+  return Index0 > Index1 ? Ext0 : Ext1;
+}
 
 /// Compare the relative costs of 2 extracts followed by scalar operation vs.
 /// vector operation(s) followed by extract. Return true if the existing
@@ -140,10 +199,8 @@ bool VectorCombine::isExtractExtractCheap(ExtractElementInst *Ext0,
               !Ext1->hasOneUse() * Extract1Cost;
   }
 
-  if (Ext0Index == Ext1Index) {
-    // If the extract indexes are identical, no shuffle is needed.
-    ConvertToShuffle = nullptr;
-  } else {
+  ConvertToShuffle = getShuffleExtract(Ext0, Ext1, PreferredExtractIndex);
+  if (ConvertToShuffle) {
     if (IsBinOp && DisableBinopExtractShuffle)
       return true;
 
@@ -156,20 +213,6 @@ bool VectorCombine::isExtractExtractCheap(ExtractElementInst *Ext0,
     //       (splat-from-element-0), but no option for a more general splat.
     NewCost +=
         TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
-
-    // The more expensive extract will be replaced by a shuffle. If the costs
-    // are equal and there is a preferred extract index, shuffle the opposite
-    // operand. Otherwise, replace the extract with the higher index.
-    if (Extract0Cost > Extract1Cost)
-      ConvertToShuffle = Ext0;
-    else if (Extract1Cost > Extract0Cost)
-      ConvertToShuffle = Ext1;
-    else if (PreferredExtractIndex == Ext0Index)
-      ConvertToShuffle = Ext1;
-    else if (PreferredExtractIndex == Ext1Index)
-      ConvertToShuffle = Ext0;
-    else
-      ConvertToShuffle = Ext0Index > Ext1Index ? Ext0 : Ext1;
   }
 
   // Aggressively form a vector op if the cost is equal because the transform
@@ -178,38 +221,45 @@ bool VectorCombine::isExtractExtractCheap(ExtractElementInst *Ext0,
   return OldCost < NewCost;
 }
 
+/// Create a shuffle that translates (shifts) 1 element from the input vector
+/// to a new element location.
+static Value *createShiftShuffle(Value *Vec, unsigned OldIndex,
+                                 unsigned NewIndex, IRBuilder<> &Builder) {
+  // The shuffle mask is undefined except for 1 lane that is being translated
+  // to the new element index. Example for OldIndex == 2 and NewIndex == 0:
+  // ShufMask = { 2, undef, undef, undef }
+  auto *VecTy = cast<FixedVectorType>(Vec->getType());
+  SmallVector<int, 32> ShufMask(VecTy->getNumElements(), UndefMaskElem);
+  ShufMask[NewIndex] = OldIndex;
+  Value *Undef = UndefValue::get(VecTy);
+  return Builder.CreateShuffleVector(Vec, Undef, ShufMask, "shift");
+}
+
 /// Given an extract element instruction with constant index operand, shuffle
 /// the source vector (shift the scalar element) to a NewIndex for extraction.
 /// Return null if the input can be constant folded, so that we are not creating
 /// unnecessary instructions.
 static ExtractElementInst *translateExtract(ExtractElementInst *ExtElt,
-                                            unsigned NewIndex) {
+                                            unsigned NewIndex,
+                                            IRBuilder<> &Builder) {
   // If the extract can be constant-folded, this code is unsimplified. Defer
   // to other passes to handle that.
   Value *X = ExtElt->getVectorOperand();
   Value *C = ExtElt->getIndexOperand();
+  assert(isa<ConstantInt>(C) && "Expected a constant index operand");
   if (isa<Constant>(X))
     return nullptr;
 
-  // The shuffle mask is undefined except for 1 lane that is being translated
-  // to the cheap extraction lane. Example:
-  // ShufMask = { 2, undef, undef, undef }
-  auto *VecTy = cast<FixedVectorType>(X->getType());
-  SmallVector<int, 32> Mask(VecTy->getNumElements(), -1);
-  assert(isa<ConstantInt>(C) && "Expected a constant index operand");
-  Mask[NewIndex] = cast<ConstantInt>(C)->getZExtValue();
-
-  // extelt X, C --> extelt (shuffle X), NewIndex
-  IRBuilder<> Builder(ExtElt);
-  Value *Shuf = Builder.CreateShuffleVector(X, UndefValue::get(VecTy), Mask);
+  Value *Shuf = createShiftShuffle(X, cast<ConstantInt>(C)->getZExtValue(),
+                                   NewIndex, Builder);
   return cast<ExtractElementInst>(Builder.CreateExtractElement(Shuf, NewIndex));
 }
 
 /// Try to reduce extract element costs by converting scalar compares to vector
 /// compares followed by extract.
 /// cmp (ext0 V0, C), (ext1 V1, C)
-static void foldExtExtCmp(ExtractElementInst *Ext0, ExtractElementInst *Ext1,
-                          Instruction &I) {
+void VectorCombine::foldExtExtCmp(ExtractElementInst *Ext0,
+                                  ExtractElementInst *Ext1, Instruction &I) {
   assert(isa<CmpInst>(&I) && "Expected a compare");
   assert(cast<ConstantInt>(Ext0->getIndexOperand())->getZExtValue() ==
              cast<ConstantInt>(Ext1->getIndexOperand())->getZExtValue() &&
@@ -217,19 +267,18 @@ static void foldExtExtCmp(ExtractElementInst *Ext0, ExtractElementInst *Ext1,
 
   // cmp Pred (extelt V0, C), (extelt V1, C) --> extelt (cmp Pred V0, V1), C
   ++NumVecCmp;
-  IRBuilder<> Builder(&I);
   CmpInst::Predicate Pred = cast<CmpInst>(&I)->getPredicate();
   Value *V0 = Ext0->getVectorOperand(), *V1 = Ext1->getVectorOperand();
   Value *VecCmp = Builder.CreateCmp(Pred, V0, V1);
   Value *NewExt = Builder.CreateExtractElement(VecCmp, Ext0->getIndexOperand());
-  I.replaceAllUsesWith(NewExt);
+  replaceValue(I, *NewExt);
 }
 
 /// Try to reduce extract element costs by converting scalar binops to vector
 /// binops followed by extract.
 /// bo (ext0 V0, C), (ext1 V1, C)
-static void foldExtExtBinop(ExtractElementInst *Ext0, ExtractElementInst *Ext1,
-                            Instruction &I) {
+void VectorCombine::foldExtExtBinop(ExtractElementInst *Ext0,
+                                    ExtractElementInst *Ext1, Instruction &I) {
   assert(isa<BinaryOperator>(&I) && "Expected a binary operator");
   assert(cast<ConstantInt>(Ext0->getIndexOperand())->getZExtValue() ==
              cast<ConstantInt>(Ext1->getIndexOperand())->getZExtValue() &&
@@ -237,7 +286,6 @@ static void foldExtExtBinop(ExtractElementInst *Ext0, ExtractElementInst *Ext1,
 
   // bo (extelt V0, C), (extelt V1, C) --> extelt (bo V0, V1), C
   ++NumVecBO;
-  IRBuilder<> Builder(&I);
   Value *V0 = Ext0->getVectorOperand(), *V1 = Ext1->getVectorOperand();
   Value *VecBO =
       Builder.CreateBinOp(cast<BinaryOperator>(&I)->getOpcode(), V0, V1);
@@ -248,7 +296,7 @@ static void foldExtExtBinop(ExtractElementInst *Ext0, ExtractElementInst *Ext1,
     VecBOInst->copyIRFlags(&I);
 
   Value *NewExt = Builder.CreateExtractElement(VecBO, Ext0->getIndexOperand());
-  I.replaceAllUsesWith(NewExt);
+  replaceValue(I, *NewExt);
 }
 
 /// Match an instruction with extracted vector operands.
@@ -278,7 +326,7 @@ bool VectorCombine::foldExtractExtract(Instruction &I) {
   //       probably becomes unnecessary.
   auto *Ext0 = cast<ExtractElementInst>(I0);
   auto *Ext1 = cast<ExtractElementInst>(I1);
-  uint64_t InsertIndex = std::numeric_limits<uint64_t>::max();
+  uint64_t InsertIndex = InvalidIndex;
   if (I.hasOneUse())
     match(I.user_back(),
           m_InsertElt(m_Value(), m_Value(), m_ConstantInt(InsertIndex)));
@@ -291,7 +339,7 @@ bool VectorCombine::foldExtractExtract(Instruction &I) {
   if (ExtractToChange) {
     unsigned CheapExtractIdx = ExtractToChange == Ext0 ? C1 : C0;
     ExtractElementInst *NewExtract =
-        translateExtract(ExtractToChange, CheapExtractIdx);
+        translateExtract(ExtractToChange, CheapExtractIdx, Builder);
     if (!NewExtract)
       return false;
     if (ExtractToChange == Ext0)
@@ -350,11 +398,10 @@ bool VectorCombine::foldBitcastShuf(Instruction &I) {
   }
   // bitcast (shuf V, MaskC) --> shuf (bitcast V), MaskC'
   ++NumShufOfBitcast;
-  IRBuilder<> Builder(&I);
   Value *CastV = Builder.CreateBitCast(V, DestTy);
   Value *Shuf =
       Builder.CreateShuffleVector(CastV, UndefValue::get(DestTy), NewMask);
-  I.replaceAllUsesWith(Shuf);
+  replaceValue(I, *Shuf);
   return true;
 }
 
@@ -451,7 +498,6 @@ bool VectorCombine::scalarizeBinopOrCmp(Instruction &I) {
     ++NumScalarBO;
 
   // For constant cases, extract the scalar element, this should constant fold.
-  IRBuilder<> Builder(&I);
   if (IsConst0)
     V0 = ConstantExpr::getExtractElement(VecC0, Builder.getInt64(Index));
   if (IsConst1)
@@ -472,8 +518,91 @@ bool VectorCombine::scalarizeBinopOrCmp(Instruction &I) {
   Constant *NewVecC = IsCmp ? ConstantExpr::getCompare(Pred, VecC0, VecC1)
                             : ConstantExpr::get(Opcode, VecC0, VecC1);
   Value *Insert = Builder.CreateInsertElement(NewVecC, Scalar, Index);
-  I.replaceAllUsesWith(Insert);
-  Insert->takeName(&I);
+  replaceValue(I, *Insert);
+  return true;
+}
+
+/// Try to combine a scalar binop + 2 scalar compares of extracted elements of
+/// a vector into vector operations followed by extract. Note: The SLP pass
+/// may miss this pattern because of implementation problems.
+bool VectorCombine::foldExtractedCmps(Instruction &I) {
+  // We are looking for a scalar binop of booleans.
+  // binop i1 (cmp Pred I0, C0), (cmp Pred I1, C1)
+  if (!I.isBinaryOp() || !I.getType()->isIntegerTy(1))
+    return false;
+
+  // The compare predicates should match, and each compare should have a
+  // constant operand.
+  // TODO: Relax the one-use constraints.
+  Value *B0 = I.getOperand(0), *B1 = I.getOperand(1);
+  Instruction *I0, *I1;
+  Constant *C0, *C1;
+  CmpInst::Predicate P0, P1;
+  if (!match(B0, m_OneUse(m_Cmp(P0, m_Instruction(I0), m_Constant(C0)))) ||
+      !match(B1, m_OneUse(m_Cmp(P1, m_Instruction(I1), m_Constant(C1)))) ||
+      P0 != P1)
+    return false;
+
+  // The compare operands must be extracts of the same vector with constant
+  // extract indexes.
+  // TODO: Relax the one-use constraints.
+  Value *X;
+  uint64_t Index0, Index1;
+  if (!match(I0, m_OneUse(m_ExtractElt(m_Value(X), m_ConstantInt(Index0)))) ||
+      !match(I1, m_OneUse(m_ExtractElt(m_Specific(X), m_ConstantInt(Index1)))))
+    return false;
+
+  auto *Ext0 = cast<ExtractElementInst>(I0);
+  auto *Ext1 = cast<ExtractElementInst>(I1);
+  ExtractElementInst *ConvertToShuf = getShuffleExtract(Ext0, Ext1);
+  if (!ConvertToShuf)
+    return false;
+
+  // The original scalar pattern is:
+  // binop i1 (cmp Pred (ext X, Index0), C0), (cmp Pred (ext X, Index1), C1)
+  CmpInst::Predicate Pred = P0;
+  unsigned CmpOpcode = CmpInst::isFPPredicate(Pred) ? Instruction::FCmp
+                                                    : Instruction::ICmp;
+  auto *VecTy = dyn_cast<FixedVectorType>(X->getType());
+  if (!VecTy)
+    return false;
+
+  int OldCost = TTI.getVectorInstrCost(Ext0->getOpcode(), VecTy, Index0);
+  OldCost += TTI.getVectorInstrCost(Ext1->getOpcode(), VecTy, Index1);
+  OldCost += TTI.getCmpSelInstrCost(CmpOpcode, I0->getType()) * 2;
+  OldCost += TTI.getArithmeticInstrCost(I.getOpcode(), I.getType());
+
+  // The proposed vector pattern is:
+  // vcmp = cmp Pred X, VecC
+  // ext (binop vNi1 vcmp, (shuffle vcmp, Index1)), Index0
+  int CheapIndex = ConvertToShuf == Ext0 ? Index1 : Index0;
+  int ExpensiveIndex = ConvertToShuf == Ext0 ? Index0 : Index1;
+  auto *CmpTy = cast<FixedVectorType>(CmpInst::makeCmpResultType(X->getType()));
+  int NewCost = TTI.getCmpSelInstrCost(CmpOpcode, X->getType());
+  NewCost +=
+      TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, CmpTy);
+  NewCost += TTI.getArithmeticInstrCost(I.getOpcode(), CmpTy);
+  NewCost += TTI.getVectorInstrCost(Ext0->getOpcode(), CmpTy, CheapIndex);
+
+  // Aggressively form vector ops if the cost is equal because the transform
+  // may enable further optimization.
+  // Codegen can reverse this transform (scalarize) if it was not profitable.
+  if (OldCost < NewCost)
+    return false;
+
+  // Create a vector constant from the 2 scalar constants.
+  SmallVector<Constant *, 32> CmpC(VecTy->getNumElements(),
+                                   UndefValue::get(VecTy->getElementType()));
+  CmpC[Index0] = C0;
+  CmpC[Index1] = C1;
+  Value *VCmp = Builder.CreateCmp(Pred, X, ConstantVector::get(CmpC));
+
+  Value *Shuf = createShiftShuffle(VCmp, ExpensiveIndex, CheapIndex, Builder);
+  Value *VecLogic = Builder.CreateBinOp(cast<BinaryOperator>(I).getOpcode(),
+                                        VCmp, Shuf);
+  Value *NewExt = Builder.CreateExtractElement(VecLogic, CheapIndex);
+  replaceValue(I, *NewExt);
+  ++NumVecCmpBO;
   return true;
 }
 
@@ -495,9 +624,11 @@ bool VectorCombine::run() {
     for (Instruction &I : BB) {
       if (isa<DbgInfoIntrinsic>(I))
         continue;
+      Builder.SetInsertPoint(&I);
       MadeChange |= foldExtractExtract(I);
       MadeChange |= foldBitcastShuf(I);
       MadeChange |= scalarizeBinopOrCmp(I);
+      MadeChange |= foldExtractedCmps(I);
     }
   }
 
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index ea73cec050eee..d56f7af583eda 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -222,6 +222,8 @@ else() # if this is included from LLVM's CMake
   if (LLVM_EXTERNAL_LIBCXX_SOURCE_DIR AND "libcxx" IN_LIST LLVM_ENABLE_RUNTIMES)
     # This looks wrong, but libcxx's build actually wants the header dir to be
     # the root build dir, not the include directory.
+    set(LIBCXX_BINARY_DIR ${LLVM_BINARY_DIR})
+    set(LIBCXX_SOURCE_DIR ${LLVM_EXTERNAL_LIBCXX_SOURCE_DIR})
     set(LIBCXX_HEADER_DIR ${LLVM_BINARY_DIR})
     set(CXX_HEADER_TARGET runtime-libcxx-headers)
     add_subdirectory(${LLVM_EXTERNAL_LIBCXX_SOURCE_DIR}/include ${CXX_HEADER_TARGET})
diff --git a/llvm/test/Analysis/AliasSet/argmemonly.ll b/llvm/test/Analysis/AliasSet/argmemonly.ll
index 25b9bef90a018..0149a6651e20e 100644
--- a/llvm/test/Analysis/AliasSet/argmemonly.ll
+++ b/llvm/test/Analysis/AliasSet/argmemonly.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-alias-sets -S -o - < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-alias-sets -S -o - < %s 2>&1 | FileCheck %s
 
 @s = global i8 1, align 1
 @d = global i8 2, align 1
diff --git a/llvm/test/Analysis/AliasSet/guards.ll b/llvm/test/Analysis/AliasSet/guards.ll
index 732c540c981cb..3a162b5c21c8d 100644
--- a/llvm/test/Analysis/AliasSet/guards.ll
+++ b/llvm/test/Analysis/AliasSet/guards.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-alias-sets -S -o - < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-alias-sets -S -o - < %s 2>&1 | FileCheck %s
 declare void @llvm.experimental.guard(i1, ...)
 
 ; CHECK: Alias sets for function 'test0':
diff --git a/llvm/test/Analysis/AliasSet/intrinsics.ll b/llvm/test/Analysis/AliasSet/intrinsics.ll
index bc4cbb43fb5f5..12c1a91e7b80f 100644
--- a/llvm/test/Analysis/AliasSet/intrinsics.ll
+++ b/llvm/test/Analysis/AliasSet/intrinsics.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-alias-sets -S -o - < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-alias-sets -S -o - < %s 2>&1 | FileCheck %s
 
 ; CHECK: Alias sets for function 'test1':
 ; CHECK: Alias Set Tracker: 2 alias sets for 2 pointer values.
diff --git a/llvm/test/Analysis/AliasSet/memset.ll b/llvm/test/Analysis/AliasSet/memset.ll
index 27f841e23b01e..bfa5e1b2fc6c7 100644
--- a/llvm/test/Analysis/AliasSet/memset.ll
+++ b/llvm/test/Analysis/AliasSet/memset.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-alias-sets -S -o - < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-alias-sets -S -o - < %s 2>&1 | FileCheck %s
 
 @s = global i8 1, align 1
 @d = global i8 2, align 1
diff --git a/llvm/test/Analysis/AliasSet/memtransfer.ll b/llvm/test/Analysis/AliasSet/memtransfer.ll
index ba22cbe2e04c6..0883db024f223 100644
--- a/llvm/test/Analysis/AliasSet/memtransfer.ll
+++ b/llvm/test/Analysis/AliasSet/memtransfer.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-alias-sets -S -o - < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-alias-sets -S -o - < %s 2>&1 | FileCheck %s
 
 @s = global i8 1, align 1
 @d = global i8 2, align 1
diff --git a/llvm/test/Analysis/AliasSet/saturation.ll b/llvm/test/Analysis/AliasSet/saturation.ll
index 32d2008109e3d..58af27b9b65ba 100644
--- a/llvm/test/Analysis/AliasSet/saturation.ll
+++ b/llvm/test/Analysis/AliasSet/saturation.ll
@@ -1,5 +1,5 @@
-; RUN: opt -basicaa -print-alias-sets -alias-set-saturation-threshold=2 -S -o - < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=NOSAT
-; RUN: opt -basicaa -print-alias-sets -alias-set-saturation-threshold=1 -S -o - < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=SAT
+; RUN: opt -basic-aa -print-alias-sets -alias-set-saturation-threshold=2 -S -o - < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=NOSAT
+; RUN: opt -basic-aa -print-alias-sets -alias-set-saturation-threshold=1 -S -o - < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=SAT
 
 ; CHECK-LABEL: 'allmust'
 ; CHECK: AliasSet[{{.*}}, 1] must alias, Mod Pointers: (i32* %a, LocationSize::precise(4))
diff --git a/llvm/test/Analysis/BasicAA/128-bit-ptr.ll b/llvm/test/Analysis/BasicAA/128-bit-ptr.ll
index 59fc99d3d3779..5b742fe473549 100644
--- a/llvm/test/Analysis/BasicAA/128-bit-ptr.ll
+++ b/llvm/test/Analysis/BasicAA/128-bit-ptr.ll
@@ -1,7 +1,7 @@
 ; This testcase consists of alias relations on 128-bit pointers that
 ; should be completely resolvable by basicaa.
 
-; RUN: opt < %s -basicaa -aa-eval -print-no-aliases -print-may-aliases -print-must-aliases -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-no-aliases -print-may-aliases -print-must-aliases -disable-output 2>&1 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-i128:128:128-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128-p100:128:64:64-p101:128:64:64"
 
diff --git a/llvm/test/Analysis/BasicAA/2003-02-26-AccessSizeTest.ll b/llvm/test/Analysis/BasicAA/2003-02-26-AccessSizeTest.ll
index d712e33b8c0ad..982f0552000f1 100644
--- a/llvm/test/Analysis/BasicAA/2003-02-26-AccessSizeTest.ll
+++ b/llvm/test/Analysis/BasicAA/2003-02-26-AccessSizeTest.ll
@@ -2,7 +2,7 @@
 ; is performed.  It is not legal to delete the second load instruction because
 ; the value computed by the first load instruction is changed by the store.
 
-; RUN: opt < %s -basicaa -gvn -instcombine -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -instcombine -S | FileCheck %s
 
 define i32 @test() {
 ; CHECK: %Y.DONOTREMOVE = load i32, i32* %A
diff --git a/llvm/test/Analysis/BasicAA/2003-03-04-GEPCrash.ll b/llvm/test/Analysis/BasicAA/2003-03-04-GEPCrash.ll
index 5a93b3da054e4..41319e0bc6c4e 100644
--- a/llvm/test/Analysis/BasicAA/2003-03-04-GEPCrash.ll
+++ b/llvm/test/Analysis/BasicAA/2003-03-04-GEPCrash.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -disable-output 2>/dev/null
+; RUN: opt < %s -basic-aa -aa-eval -disable-output 2>/dev/null
 ; Test for a bug in BasicAA which caused a crash when querying equality of P1&P2
 define void @test({[2 x i32],[2 x i32]}* %A, i64 %X, i64 %Y) {
 	%P1 = getelementptr {[2 x i32],[2 x i32]}, {[2 x i32],[2 x i32]}* %A, i64 0, i32 0, i64 %X
diff --git a/llvm/test/Analysis/BasicAA/2003-04-22-GEPProblem.ll b/llvm/test/Analysis/BasicAA/2003-04-22-GEPProblem.ll
index 96ca071587e65..8897b6878acc6 100644
--- a/llvm/test/Analysis/BasicAA/2003-04-22-GEPProblem.ll
+++ b/llvm/test/Analysis/BasicAA/2003-04-22-GEPProblem.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -instcombine -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -instcombine -S | FileCheck %s
 
 ; BasicAA was incorrectly concluding that P1 and P2 didn't conflict!
 
diff --git a/llvm/test/Analysis/BasicAA/2003-04-25-GEPCrash.ll b/llvm/test/Analysis/BasicAA/2003-04-25-GEPCrash.ll
index ea26c22007262..9c0a1276bd469 100644
--- a/llvm/test/Analysis/BasicAA/2003-04-25-GEPCrash.ll
+++ b/llvm/test/Analysis/BasicAA/2003-04-25-GEPCrash.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -disable-output 2>/dev/null
+; RUN: opt < %s -basic-aa -aa-eval -disable-output 2>/dev/null
 ; Test for a bug in BasicAA which caused a crash when querying equality of P1&P2
 define void @test([17 x i16]* %mask_bits) {
 	%P1 = getelementptr [17 x i16], [17 x i16]* %mask_bits, i64 0, i64 0
diff --git a/llvm/test/Analysis/BasicAA/2003-05-21-GEP-Problem.ll b/llvm/test/Analysis/BasicAA/2003-05-21-GEP-Problem.ll
index fb5b3bb4618e5..e7a2a169223b6 100644
--- a/llvm/test/Analysis/BasicAA/2003-05-21-GEP-Problem.ll
+++ b/llvm/test/Analysis/BasicAA/2003-05-21-GEP-Problem.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -licm -disable-output
+; RUN: opt < %s -basic-aa -licm -disable-output
 	%struct..apr_array_header_t = type { i32*, i32, i32, i32, i8* }
 	%struct..apr_table_t = type { %struct..apr_array_header_t, i32, [32 x i32], [32 x i32] }
 
diff --git a/llvm/test/Analysis/BasicAA/2003-06-01-AliasCrash.ll b/llvm/test/Analysis/BasicAA/2003-06-01-AliasCrash.ll
index ace5982afaab4..6cabcdbe7c1e5 100644
--- a/llvm/test/Analysis/BasicAA/2003-06-01-AliasCrash.ll
+++ b/llvm/test/Analysis/BasicAA/2003-06-01-AliasCrash.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -disable-output 2>/dev/null
+; RUN: opt < %s -basic-aa -aa-eval -disable-output 2>/dev/null
 
 define i32 @MTConcat([3 x i32]* %a.1) {
 	%tmp.961 = getelementptr [3 x i32], [3 x i32]* %a.1, i64 0, i64 4
diff --git a/llvm/test/Analysis/BasicAA/2003-07-03-BasicAACrash.ll b/llvm/test/Analysis/BasicAA/2003-07-03-BasicAACrash.ll
index 7aaae2a1927b0..c485f07befef5 100644
--- a/llvm/test/Analysis/BasicAA/2003-07-03-BasicAACrash.ll
+++ b/llvm/test/Analysis/BasicAA/2003-07-03-BasicAACrash.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -disable-output 2>/dev/null
+; RUN: opt < %s -basic-aa -aa-eval -disable-output 2>/dev/null
 
 %struct..RefPoint = type { i32, { i32, i8, i8 } }
 %struct..RefRect = type { %struct..RefPoint, %struct..RefPoint }
diff --git a/llvm/test/Analysis/BasicAA/2003-09-19-LocalArgument.ll b/llvm/test/Analysis/BasicAA/2003-09-19-LocalArgument.ll
index 1e75d6461d6f0..8d730d4fb50ee 100644
--- a/llvm/test/Analysis/BasicAA/2003-09-19-LocalArgument.ll
+++ b/llvm/test/Analysis/BasicAA/2003-09-19-LocalArgument.ll
@@ -1,6 +1,6 @@
 ; In this test, a local alloca cannot alias an incoming argument.
 
-; RUN: opt < %s -basicaa -gvn -instcombine -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -instcombine -S | FileCheck %s
 
 ; CHECK:      define i32 @test
 ; CHECK-NEXT: ret i32 0
diff --git a/llvm/test/Analysis/BasicAA/2003-11-04-SimpleCases.ll b/llvm/test/Analysis/BasicAA/2003-11-04-SimpleCases.ll
index f8d4195c46c6d..ec65fc1a3b768 100644
--- a/llvm/test/Analysis/BasicAA/2003-11-04-SimpleCases.ll
+++ b/llvm/test/Analysis/BasicAA/2003-11-04-SimpleCases.ll
@@ -1,7 +1,7 @@
 ; This testcase consists of alias relations which should be completely
 ; resolvable by basicaa.
 
-; RUN: opt < %s -basicaa -aa-eval -print-may-aliases -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-may-aliases -disable-output 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Analysis/BasicAA/2003-12-11-ConstExprGEP.ll b/llvm/test/Analysis/BasicAA/2003-12-11-ConstExprGEP.ll
index 957502f2d99ce..b66c4eb10df25 100644
--- a/llvm/test/Analysis/BasicAA/2003-12-11-ConstExprGEP.ll
+++ b/llvm/test/Analysis/BasicAA/2003-12-11-ConstExprGEP.ll
@@ -1,7 +1,7 @@
 ; This testcase consists of alias relations which should be completely
 ; resolvable by basicaa, but require analysis of getelementptr constant exprs.
 
-; RUN: opt < %s -basicaa -aa-eval -print-may-aliases -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-may-aliases -disable-output 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Analysis/BasicAA/2004-07-28-MustAliasbug.ll b/llvm/test/Analysis/BasicAA/2004-07-28-MustAliasbug.ll
index 16573a7f9af5d..0e30491362f48 100644
--- a/llvm/test/Analysis/BasicAA/2004-07-28-MustAliasbug.ll
+++ b/llvm/test/Analysis/BasicAA/2004-07-28-MustAliasbug.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -S | FileCheck %s
 
 define void @test({i32,i32 }* %P) {
 ; CHECK: store i32 0, i32* %X
diff --git a/llvm/test/Analysis/BasicAA/2006-03-03-BadArraySubscript.ll b/llvm/test/Analysis/BasicAA/2006-03-03-BadArraySubscript.ll
index 98161bff9273d..82fd2907f2dd0 100644
--- a/llvm/test/Analysis/BasicAA/2006-03-03-BadArraySubscript.ll
+++ b/llvm/test/Analysis/BasicAA/2006-03-03-BadArraySubscript.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -disable-output 2>&1 | FileCheck %s
 ; TEST that A[1][0] may alias A[0][i].
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
diff --git a/llvm/test/Analysis/BasicAA/2006-11-03-BasicAAVectorCrash.ll b/llvm/test/Analysis/BasicAA/2006-11-03-BasicAAVectorCrash.ll
index 99544d473a03a..635b70294a4a2 100644
--- a/llvm/test/Analysis/BasicAA/2006-11-03-BasicAAVectorCrash.ll
+++ b/llvm/test/Analysis/BasicAA/2006-11-03-BasicAAVectorCrash.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -licm -disable-output
+; RUN: opt < %s -basic-aa -licm -disable-output
 target datalayout = "E-p:32:32"
 target triple = "powerpc-unknown-linux-gnu"
 
diff --git a/llvm/test/Analysis/BasicAA/2007-01-13-BasePointerBadNoAlias.ll b/llvm/test/Analysis/BasicAA/2007-01-13-BasePointerBadNoAlias.ll
index 86bbd44c6448e..501fd6bd65027 100644
--- a/llvm/test/Analysis/BasicAA/2007-01-13-BasePointerBadNoAlias.ll
+++ b/llvm/test/Analysis/BasicAA/2007-01-13-BasePointerBadNoAlias.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -instcombine -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -instcombine -S | FileCheck %s
 ; PR1109
 
 target datalayout = "e-p:32:32"
diff --git a/llvm/test/Analysis/BasicAA/2007-08-01-NoAliasAndCalls.ll b/llvm/test/Analysis/BasicAA/2007-08-01-NoAliasAndCalls.ll
index 2a6f5b9955d7a..0205efc4ec18e 100644
--- a/llvm/test/Analysis/BasicAA/2007-08-01-NoAliasAndCalls.ll
+++ b/llvm/test/Analysis/BasicAA/2007-08-01-NoAliasAndCalls.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; CHECK: Function: foo
 ; CHECK:   MayAlias: i32* %x, i32* %y
diff --git a/llvm/test/Analysis/BasicAA/2007-08-01-NoAliasAndGEP.ll b/llvm/test/Analysis/BasicAA/2007-08-01-NoAliasAndGEP.ll
index 200e24f428ef3..f8d7d8d2a15d8 100644
--- a/llvm/test/Analysis/BasicAA/2007-08-01-NoAliasAndGEP.ll
+++ b/llvm/test/Analysis/BasicAA/2007-08-01-NoAliasAndGEP.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Analysis/BasicAA/2007-08-05-GetOverloadedModRef.ll b/llvm/test/Analysis/BasicAA/2007-08-05-GetOverloadedModRef.ll
index 5f0e1170f0448..3569f26ec0446 100644
--- a/llvm/test/Analysis/BasicAA/2007-08-05-GetOverloadedModRef.ll
+++ b/llvm/test/Analysis/BasicAA/2007-08-05-GetOverloadedModRef.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -instcombine -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -instcombine -S | FileCheck %s
 ; PR1600
 
 declare i16 @llvm.cttz.i16(i16, i1)
diff --git a/llvm/test/Analysis/BasicAA/2007-10-24-ArgumentsGlobals.ll b/llvm/test/Analysis/BasicAA/2007-10-24-ArgumentsGlobals.ll
index 2d33e94393763..7373fa087ff94 100644
--- a/llvm/test/Analysis/BasicAA/2007-10-24-ArgumentsGlobals.ll
+++ b/llvm/test/Analysis/BasicAA/2007-10-24-ArgumentsGlobals.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -dce -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -dce -S | FileCheck %s
 
         %struct.A = type { i32 }
         %struct.B = type { %struct.A }
diff --git a/llvm/test/Analysis/BasicAA/2007-11-05-SizeCrash.ll b/llvm/test/Analysis/BasicAA/2007-11-05-SizeCrash.ll
index b59ee42dec3a8..c65875bffa603 100644
--- a/llvm/test/Analysis/BasicAA/2007-11-05-SizeCrash.ll
+++ b/llvm/test/Analysis/BasicAA/2007-11-05-SizeCrash.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -disable-output
+; RUN: opt < %s -basic-aa -gvn -disable-output
 ; PR1774
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
diff --git a/llvm/test/Analysis/BasicAA/2007-12-08-OutOfBoundsCrash.ll b/llvm/test/Analysis/BasicAA/2007-12-08-OutOfBoundsCrash.ll
index ba9740028180d..4e5e2a3cbf3b1 100644
--- a/llvm/test/Analysis/BasicAA/2007-12-08-OutOfBoundsCrash.ll
+++ b/llvm/test/Analysis/BasicAA/2007-12-08-OutOfBoundsCrash.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -disable-output
+; RUN: opt < %s -basic-aa -gvn -disable-output
 ; PR1782
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
diff --git a/llvm/test/Analysis/BasicAA/2008-06-02-GEPTailCrash.ll b/llvm/test/Analysis/BasicAA/2008-06-02-GEPTailCrash.ll
index 7d2cde4864a35..f7b050c3270aa 100644
--- a/llvm/test/Analysis/BasicAA/2008-06-02-GEPTailCrash.ll
+++ b/llvm/test/Analysis/BasicAA/2008-06-02-GEPTailCrash.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -disable-output
+; RUN: opt < %s -basic-aa -gvn -disable-output
 ; PR2395
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
diff --git a/llvm/test/Analysis/BasicAA/2008-11-23-NoaliasRet.ll b/llvm/test/Analysis/BasicAA/2008-11-23-NoaliasRet.ll
index 49a742c31f588..e396f4d05425d 100644
--- a/llvm/test/Analysis/BasicAA/2008-11-23-NoaliasRet.ll
+++ b/llvm/test/Analysis/BasicAA/2008-11-23-NoaliasRet.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -disable-output 2>&1 | FileCheck %s
 
 declare noalias i32* @_Znwj(i32 %x) nounwind
 
diff --git a/llvm/test/Analysis/BasicAA/2009-03-04-GEPNoalias.ll b/llvm/test/Analysis/BasicAA/2009-03-04-GEPNoalias.ll
index 65dcf5ceab365..06f9d4b7ef73a 100644
--- a/llvm/test/Analysis/BasicAA/2009-03-04-GEPNoalias.ll
+++ b/llvm/test/Analysis/BasicAA/2009-03-04-GEPNoalias.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 declare noalias i32* @noalias()
 
diff --git a/llvm/test/Analysis/BasicAA/2009-10-13-AtomicModRef.ll b/llvm/test/Analysis/BasicAA/2009-10-13-AtomicModRef.ll
index 97a92517dca8b..646354912cdda 100644
--- a/llvm/test/Analysis/BasicAA/2009-10-13-AtomicModRef.ll
+++ b/llvm/test/Analysis/BasicAA/2009-10-13-AtomicModRef.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -gvn -instcombine -S < %s | FileCheck %s
+; RUN: opt -basic-aa -gvn -instcombine -S < %s | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 define i8 @foo(i8* %ptr) {
diff --git a/llvm/test/Analysis/BasicAA/2009-10-13-GEP-BaseNoAlias.ll b/llvm/test/Analysis/BasicAA/2009-10-13-GEP-BaseNoAlias.ll
index 43ee96ce9b3de..8e14261b49474 100644
--- a/llvm/test/Analysis/BasicAA/2009-10-13-GEP-BaseNoAlias.ll
+++ b/llvm/test/Analysis/BasicAA/2009-10-13-GEP-BaseNoAlias.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; If GEP base doesn't alias Z, then GEP doesn't alias Z.
 ; rdar://7282591
 
diff --git a/llvm/test/Analysis/BasicAA/2010-09-15-GEP-SignedArithmetic.ll b/llvm/test/Analysis/BasicAA/2010-09-15-GEP-SignedArithmetic.ll
index 79421824e6ea2..7af2330722405 100644
--- a/llvm/test/Analysis/BasicAA/2010-09-15-GEP-SignedArithmetic.ll
+++ b/llvm/test/Analysis/BasicAA/2010-09-15-GEP-SignedArithmetic.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; PR7959
 
 target datalayout = "e-p:32:32:32"
diff --git a/llvm/test/Analysis/BasicAA/2014-03-18-Maxlookup-reached.ll b/llvm/test/Analysis/BasicAA/2014-03-18-Maxlookup-reached.ll
index 08db5ec4d1d92..54870ae045b3e 100644
--- a/llvm/test/Analysis/BasicAA/2014-03-18-Maxlookup-reached.ll
+++ b/llvm/test/Analysis/BasicAA/2014-03-18-Maxlookup-reached.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn  -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn  -S | FileCheck %s
 
 ; PR15967
 ; BasicAA claims no alias when there is (due to a problem when the MaxLookup
diff --git a/llvm/test/Analysis/BasicAA/aligned-overread.ll b/llvm/test/Analysis/BasicAA/aligned-overread.ll
index aa0a83e669756..d3479b790231d 100644
--- a/llvm/test/Analysis/BasicAA/aligned-overread.ll
+++ b/llvm/test/Analysis/BasicAA/aligned-overread.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Analysis/BasicAA/args-rets-allocas-loads.ll b/llvm/test/Analysis/BasicAA/args-rets-allocas-loads.ll
index b31fb26f1c9b8..df07a8aa6df1d 100644
--- a/llvm/test/Analysis/BasicAA/args-rets-allocas-loads.ll
+++ b/llvm/test/Analysis/BasicAA/args-rets-allocas-loads.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -aa-eval -print-all-alias-modref-info -disable-output < %s 2>&1 | FileCheck  %s
+; RUN: opt -basic-aa -aa-eval -print-all-alias-modref-info -disable-output < %s 2>&1 | FileCheck  %s
 
 declare void @callee(double* %callee_arg)
 declare void @nocap_callee(double* nocapture %nocap_callee_arg)
diff --git a/llvm/test/Analysis/BasicAA/assume-index-positive.ll b/llvm/test/Analysis/BasicAA/assume-index-positive.ll
index d89738a23b034..e8a039a1018e8 100644
--- a/llvm/test/Analysis/BasicAA/assume-index-positive.ll
+++ b/llvm/test/Analysis/BasicAA/assume-index-positive.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -aa-eval -print-all-alias-modref-info %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -aa-eval -print-all-alias-modref-info %s 2>&1 | FileCheck %s
 
 ; %col.ptr.1 and %col.ptr.2 do not alias, if we know that %skip >= 0, because
 ; the distance between %col.ptr.1 and %col.ptr.2 is %skip + 6 and we load 6
diff --git a/llvm/test/Analysis/BasicAA/assume.ll b/llvm/test/Analysis/BasicAA/assume.ll
index 49189dee03504..1a7de5aa6afef 100644
--- a/llvm/test/Analysis/BasicAA/assume.ll
+++ b/llvm/test/Analysis/BasicAA/assume.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:32:64-v128:32:128-a0:0:32-n32"
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) #0
diff --git a/llvm/test/Analysis/BasicAA/bug.23540.ll b/llvm/test/Analysis/BasicAA/bug.23540.ll
index 6a00abdce3a10..46d008d904348 100644
--- a/llvm/test/Analysis/BasicAA/bug.23540.ll
+++ b/llvm/test/Analysis/BasicAA/bug.23540.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Analysis/BasicAA/bug.23626.ll b/llvm/test/Analysis/BasicAA/bug.23626.ll
index 7d5b5ad06698d..9a4641ef58e3d 100644
--- a/llvm/test/Analysis/BasicAA/bug.23626.ll
+++ b/llvm/test/Analysis/BasicAA/bug.23626.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-darwin13.4.0"
 
diff --git a/llvm/test/Analysis/BasicAA/byval.ll b/llvm/test/Analysis/BasicAA/byval.ll
index 4f90c3f4d26e9..0699436a8a0be 100644
--- a/llvm/test/Analysis/BasicAA/byval.ll
+++ b/llvm/test/Analysis/BasicAA/byval.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i686-apple-darwin8"
 	%struct.x = type { i32, i32, i32, i32 }
diff --git a/llvm/test/Analysis/BasicAA/call-attrs.ll b/llvm/test/Analysis/BasicAA/call-attrs.ll
index 8538e8b4771dd..c80f81a2aaa5c 100644
--- a/llvm/test/Analysis/BasicAA/call-attrs.ll
+++ b/llvm/test/Analysis/BasicAA/call-attrs.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 declare void @readonly_attr(i8* readonly nocapture)
 declare void @writeonly_attr(i8* writeonly nocapture)
diff --git a/llvm/test/Analysis/BasicAA/cas.ll b/llvm/test/Analysis/BasicAA/cas.ll
index b770cb73381af..ad1a87c6e931b 100644
--- a/llvm/test/Analysis/BasicAA/cas.ll
+++ b/llvm/test/Analysis/BasicAA/cas.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -instcombine -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -instcombine -S | FileCheck %s
 
 @flag0 = internal global i32 zeroinitializer
 @turn = internal global i32 zeroinitializer
diff --git a/llvm/test/Analysis/BasicAA/constant-over-index.ll b/llvm/test/Analysis/BasicAA/constant-over-index.ll
index f77156305c260..2288c709d01a4 100644
--- a/llvm/test/Analysis/BasicAA/constant-over-index.ll
+++ b/llvm/test/Analysis/BasicAA/constant-over-index.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info 2>&1 | FileCheck %s
 ; PR4267
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Analysis/BasicAA/cs-cs-arm.ll b/llvm/test/Analysis/BasicAA/cs-cs-arm.ll
index e4367bb6d61b8..4a613e448154e 100644
--- a/llvm/test/Analysis/BasicAA/cs-cs-arm.ll
+++ b/llvm/test/Analysis/BasicAA/cs-cs-arm.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; REQUIRES: arm
 
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:32:64-v128:32:128-a0:0:32-n32"
diff --git a/llvm/test/Analysis/BasicAA/cs-cs.ll b/llvm/test/Analysis/BasicAA/cs-cs.ll
index 28fc8febd608d..fa7a5f972aafe 100644
--- a/llvm/test/Analysis/BasicAA/cs-cs.ll
+++ b/llvm/test/Analysis/BasicAA/cs-cs.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -S 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -S 2>&1 | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:32:64-v128:32:128-a0:0:32-n32"
 target triple = "arm-apple-ios"
 
diff --git a/llvm/test/Analysis/BasicAA/dag.ll b/llvm/test/Analysis/BasicAA/dag.ll
index 63e2c1a8c09f0..746a03a0b072b 100644
--- a/llvm/test/Analysis/BasicAA/dag.ll
+++ b/llvm/test/Analysis/BasicAA/dag.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info 2>&1 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/llvm/test/Analysis/BasicAA/dereferenceable.ll b/llvm/test/Analysis/BasicAA/dereferenceable.ll
index efc9addbee112..5009c1e3deb34 100644
--- a/llvm/test/Analysis/BasicAA/dereferenceable.ll
+++ b/llvm/test/Analysis/BasicAA/dereferenceable.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-all-alias-modref-info -aa-eval -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-all-alias-modref-info -aa-eval -analyze < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Analysis/BasicAA/empty.ll b/llvm/test/Analysis/BasicAA/empty.ll
index dfc79f9c04209..a6fdcabfb3d5a 100644
--- a/llvm/test/Analysis/BasicAA/empty.ll
+++ b/llvm/test/Analysis/BasicAA/empty.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/llvm/test/Analysis/BasicAA/fallback-mayalias.ll b/llvm/test/Analysis/BasicAA/fallback-mayalias.ll
index a1e4b12d20ade..7e0a85c876a85 100644
--- a/llvm/test/Analysis/BasicAA/fallback-mayalias.ll
+++ b/llvm/test/Analysis/BasicAA/fallback-mayalias.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -aa-eval -print-all-alias-modref-info -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -aa-eval -print-all-alias-modref-info -disable-output < %s 2>&1 | FileCheck %s
 
 ; Check that BasicAA falls back to MayAlias (instead of PartialAlias) when none
 ; of its little tricks are applicable.
diff --git a/llvm/test/Analysis/BasicAA/featuretest.ll b/llvm/test/Analysis/BasicAA/featuretest.ll
index 04577c6255777..9c438e1ec6895 100644
--- a/llvm/test/Analysis/BasicAA/featuretest.ll
+++ b/llvm/test/Analysis/BasicAA/featuretest.ll
@@ -2,8 +2,8 @@
 ; This testcase tests for various features the basicaa test should be able to
 ; determine, as noted in the comments.
 
-; RUN: opt < %s -basicaa -gvn -instcombine -dce -S | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
-; RUN: opt < %s -basicaa -gvn -instcombine -dce --enable-knowledge-retention -S | FileCheck %s --check-prefixes=CHECK,USE_ASSUME
+; RUN: opt < %s -basic-aa -gvn -instcombine -dce -S | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
+; RUN: opt < %s -basic-aa -gvn -instcombine -dce --enable-knowledge-retention -S | FileCheck %s --check-prefixes=CHECK,USE_ASSUME
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 @Global = external global { i32 }
diff --git a/llvm/test/Analysis/BasicAA/full-store-partial-alias.ll b/llvm/test/Analysis/BasicAA/full-store-partial-alias.ll
index e1337d6805b7a..671546886650f 100644
--- a/llvm/test/Analysis/BasicAA/full-store-partial-alias.ll
+++ b/llvm/test/Analysis/BasicAA/full-store-partial-alias.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S -tbaa -gvn < %s | FileCheck -check-prefix=BASICAA %s
-; RUN: opt -S -tbaa -disable-basicaa -gvn < %s | FileCheck %s
+; RUN: opt -S -tbaa -disable-basic-aa -gvn < %s | FileCheck %s
 ; rdar://8875631, rdar://8875069
 
 ; BasicAA should notice that the store stores to the entire %u object,
diff --git a/llvm/test/Analysis/BasicAA/gcsetest.ll b/llvm/test/Analysis/BasicAA/gcsetest.ll
index cf6ab710bf2f4..2430f8ac2f805 100644
--- a/llvm/test/Analysis/BasicAA/gcsetest.ll
+++ b/llvm/test/Analysis/BasicAA/gcsetest.ll
@@ -2,7 +2,7 @@
 ; disambiguating some obvious cases.  All loads should be removable in
 ; this testcase.
 
-; RUN: opt < %s -basicaa -gvn -instcombine -dce -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -instcombine -dce -S | FileCheck %s
 
 @A = global i32 7
 @B = global i32 8
diff --git a/llvm/test/Analysis/BasicAA/gep-alias.ll b/llvm/test/Analysis/BasicAA/gep-alias.ll
index 5fd77e19e637d..3d6c5a9e4693f 100644
--- a/llvm/test/Analysis/BasicAA/gep-alias.ll
+++ b/llvm/test/Analysis/BasicAA/gep-alias.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -instcombine -S 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -instcombine -S 2>&1 | FileCheck %s
 
 target datalayout = "e-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 
diff --git a/llvm/test/Analysis/BasicAA/gep-and-alias-64.ll b/llvm/test/Analysis/BasicAA/gep-and-alias-64.ll
index c1a53540a6de7..c76ffbbee280f 100644
--- a/llvm/test/Analysis/BasicAA/gep-and-alias-64.ll
+++ b/llvm/test/Analysis/BasicAA/gep-and-alias-64.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -gvn < %s | FileCheck %s
+; RUN: opt -S -basic-aa -gvn < %s | FileCheck %s
 
 target datalayout = "e-m:o-p:64:64-f64:32:64-f80:128-n8:16:32-S128"
 target triple = "x86_64-apple-macosx10.6.0"
diff --git a/llvm/test/Analysis/BasicAA/gep-and-alias.ll b/llvm/test/Analysis/BasicAA/gep-and-alias.ll
index 0fa17b68a9d1b..f99d6daeb61c5 100644
--- a/llvm/test/Analysis/BasicAA/gep-and-alias.ll
+++ b/llvm/test/Analysis/BasicAA/gep-and-alias.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S -basicaa -gvn < %s | FileCheck %s
-; RUN: opt -S -basicaa -gvn -basicaa-force-at-least-64b=0 < %s | FileCheck %s
+; RUN: opt -S -basic-aa -gvn < %s | FileCheck %s
+; RUN: opt -S -basic-aa -gvn -basic-aa-force-at-least-64b=0 < %s | FileCheck %s
 
 target datalayout = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128"
 target triple = "i386-apple-macosx10.6.0"
diff --git a/llvm/test/Analysis/BasicAA/getmodrefinfo-cs-cs.ll b/llvm/test/Analysis/BasicAA/getmodrefinfo-cs-cs.ll
index 755a9ccb23ccc..75488f3a2bf05 100644
--- a/llvm/test/Analysis/BasicAA/getmodrefinfo-cs-cs.ll
+++ b/llvm/test/Analysis/BasicAA/getmodrefinfo-cs-cs.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 
 ; CHECK: Just Ref: call void @ro() <-> call void @f0()
diff --git a/llvm/test/Analysis/BasicAA/global-size.ll b/llvm/test/Analysis/BasicAA/global-size.ll
index bacf3bc9c1b92..32488c5420560 100644
--- a/llvm/test/Analysis/BasicAA/global-size.ll
+++ b/llvm/test/Analysis/BasicAA/global-size.ll
@@ -1,7 +1,7 @@
 ; A store or load cannot alias a global if the accessed amount is larger then
 ; the global.
 
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 target datalayout = "E-p:64:64:64-p1:16:16:16-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 @B = global i16 8
diff --git a/llvm/test/Analysis/BasicAA/guards.ll b/llvm/test/Analysis/BasicAA/guards.ll
index c5d078346a22f..e6cee5f6d1e44 100644
--- a/llvm/test/Analysis/BasicAA/guards.ll
+++ b/llvm/test/Analysis/BasicAA/guards.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:32:64-v128:32:128-a0:0:32-n32"
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) #0
diff --git a/llvm/test/Analysis/BasicAA/intrinsics-arm.ll b/llvm/test/Analysis/BasicAA/intrinsics-arm.ll
index e15ce1c65c64d..79374d82525e8 100644
--- a/llvm/test/Analysis/BasicAA/intrinsics-arm.ll
+++ b/llvm/test/Analysis/BasicAA/intrinsics-arm.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -gvn -S < %s | FileCheck %s
 ; REQUIRES: arm
 
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:32:64-v128:32:128-a0:0:32-n32"
diff --git a/llvm/test/Analysis/BasicAA/intrinsics.ll b/llvm/test/Analysis/BasicAA/intrinsics.ll
index cf792e8c66d69..9cc55ca7a3dec 100644
--- a/llvm/test/Analysis/BasicAA/intrinsics.ll
+++ b/llvm/test/Analysis/BasicAA/intrinsics.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -gvn -S < %s | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:32:64-v128:32:128-a0:0:32-n32"
 
diff --git a/llvm/test/Analysis/BasicAA/invariant_group.ll b/llvm/test/Analysis/BasicAA/invariant_group.ll
index 5396afe090b30..204c77d461bbe 100644
--- a/llvm/test/Analysis/BasicAA/invariant_group.ll
+++ b/llvm/test/Analysis/BasicAA/invariant_group.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
-; RUN: opt < %s -basicaa -gvn -S | FileCheck -check-prefix=CHECK-GVN %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck -check-prefix=CHECK-GVN %s
 
 ; The input *.ll had been adapted from bug 37458:
 ;
diff --git a/llvm/test/Analysis/BasicAA/invariant_load.ll b/llvm/test/Analysis/BasicAA/invariant_load.ll
index 722fb5b47650a..5e328583cc4f2 100644
--- a/llvm/test/Analysis/BasicAA/invariant_load.ll
+++ b/llvm/test/Analysis/BasicAA/invariant_load.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 ; The input *.ll is obtained by manually annotating "invariant.load" to the 
 ; two loads. With "invariant.load" metadata, the second load is redundant.
diff --git a/llvm/test/Analysis/BasicAA/memset_pattern.ll b/llvm/test/Analysis/BasicAA/memset_pattern.ll
index 821cbdf4bb064..1096d3896fda8 100644
--- a/llvm/test/Analysis/BasicAA/memset_pattern.ll
+++ b/llvm/test/Analysis/BasicAA/memset_pattern.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -gvn < %s | FileCheck %s
+; RUN: opt -S -basic-aa -gvn < %s | FileCheck %s
 ; PR10872
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-macosx10.7"
diff --git a/llvm/test/Analysis/BasicAA/modref.ll b/llvm/test/Analysis/BasicAA/modref.ll
index eb18f913fb9d6..9904d13296e89 100644
--- a/llvm/test/Analysis/BasicAA/modref.ll
+++ b/llvm/test/Analysis/BasicAA/modref.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -dse -S | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
diff --git a/llvm/test/Analysis/BasicAA/must-and-partial.ll b/llvm/test/Analysis/BasicAA/must-and-partial.ll
index 3b4c84a2f81b6..763d36213dec9 100644
--- a/llvm/test/Analysis/BasicAA/must-and-partial.ll
+++ b/llvm/test/Analysis/BasicAA/must-and-partial.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info 2>&1 | FileCheck %s
 
 ; When merging MustAlias and PartialAlias, merge to PartialAlias
 ; instead of MayAlias.
diff --git a/llvm/test/Analysis/BasicAA/negoffset.ll b/llvm/test/Analysis/BasicAA/negoffset.ll
index 7569e66eccfb1..ac1ddfdf5ac51 100644
--- a/llvm/test/Analysis/BasicAA/negoffset.ll
+++ b/llvm/test/Analysis/BasicAA/negoffset.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
 target triple = "i386-unknown-linux-gnu"
diff --git a/llvm/test/Analysis/BasicAA/no-escape-call.ll b/llvm/test/Analysis/BasicAA/no-escape-call.ll
index ea335325170ad..89ba821ba6a72 100644
--- a/llvm/test/Analysis/BasicAA/no-escape-call.ll
+++ b/llvm/test/Analysis/BasicAA/no-escape-call.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -instcombine -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -instcombine -S | FileCheck %s
 ; PR2436
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i386-apple-darwin8"
diff --git a/llvm/test/Analysis/BasicAA/noalias-bugs.ll b/llvm/test/Analysis/BasicAA/noalias-bugs.ll
index 71b3c443f5427..584b1177afec8 100644
--- a/llvm/test/Analysis/BasicAA/noalias-bugs.ll
+++ b/llvm/test/Analysis/BasicAA/noalias-bugs.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -dse < %s | FileCheck %s
+; RUN: opt -S -basic-aa -dse < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Analysis/BasicAA/noalias-geps.ll b/llvm/test/Analysis/BasicAA/noalias-geps.ll
index cdec9889ca88d..545df2154a8f5 100644
--- a/llvm/test/Analysis/BasicAA/noalias-geps.ll
+++ b/llvm/test/Analysis/BasicAA/noalias-geps.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 
diff --git a/llvm/test/Analysis/BasicAA/noalias-param.ll b/llvm/test/Analysis/BasicAA/noalias-param.ll
index c5b1ebf1a1d2b..81aca4a2bb97c 100644
--- a/llvm/test/Analysis/BasicAA/noalias-param.ll
+++ b/llvm/test/Analysis/BasicAA/noalias-param.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info 2>&1 | FileCheck %s
 
 declare i32* @captures(i32* %cap) nounwind readonly
 
diff --git a/llvm/test/Analysis/BasicAA/noalias-wraparound-bug.ll b/llvm/test/Analysis/BasicAA/noalias-wraparound-bug.ll
index a75322ed3e9d4..6187952766463 100644
--- a/llvm/test/Analysis/BasicAA/noalias-wraparound-bug.ll
+++ b/llvm/test/Analysis/BasicAA/noalias-wraparound-bug.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -gvn < %s | FileCheck %s
+; RUN: opt -S -basic-aa -gvn < %s | FileCheck %s
 
 target datalayout = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128"
 target triple = "i386-apple-macosx10.6.0"
diff --git a/llvm/test/Analysis/BasicAA/nocapture.ll b/llvm/test/Analysis/BasicAA/nocapture.ll
index 26cb69bffcc02..89f1b2cd2d16f 100644
--- a/llvm/test/Analysis/BasicAA/nocapture.ll
+++ b/llvm/test/Analysis/BasicAA/nocapture.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -instcombine -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -instcombine -S | FileCheck %s
 
 declare i32* @test(i32* nocapture)
 
diff --git a/llvm/test/Analysis/BasicAA/phi-aa.ll b/llvm/test/Analysis/BasicAA/phi-aa.ll
index e57dd40f00475..5acfff824014e 100644
--- a/llvm/test/Analysis/BasicAA/phi-aa.ll
+++ b/llvm/test/Analysis/BasicAA/phi-aa.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -phi-values -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -phi-values -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes='require<phi-values>,aa-eval' -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Analysis/BasicAA/phi-and-select.ll b/llvm/test/Analysis/BasicAA/phi-and-select.ll
index b8fee00ed0d04..8a950caa2bf2a 100644
--- a/llvm/test/Analysis/BasicAA/phi-and-select.ll
+++ b/llvm/test/Analysis/BasicAA/phi-and-select.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; BasicAA should detect NoAliases in PHIs and Selects.
 
diff --git a/llvm/test/Analysis/BasicAA/phi-loop.ll b/llvm/test/Analysis/BasicAA/phi-loop.ll
index 6337bfbc1cfb9..db3023c6560d0 100644
--- a/llvm/test/Analysis/BasicAA/phi-loop.ll
+++ b/llvm/test/Analysis/BasicAA/phi-loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -basicaa-recphi=1 -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -basic-aa-recphi=1 -gvn -S | FileCheck %s
 ;
 ; Check that section->word_ofs doesn't get reloaded in every iteration of the
 ; for loop.
diff --git a/llvm/test/Analysis/BasicAA/phi-spec-order.ll b/llvm/test/Analysis/BasicAA/phi-spec-order.ll
index b326dc360060a..f8586f094c2ce 100644
--- a/llvm/test/Analysis/BasicAA/phi-spec-order.ll
+++ b/llvm/test/Analysis/BasicAA/phi-spec-order.ll
@@ -1,6 +1,6 @@
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
 target triple = "powerpc64-bgq-linux"
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 @X = external global [16000 x double], align 32
 @Y = external global [16000 x double], align 32
diff --git a/llvm/test/Analysis/BasicAA/phi-speculation.ll b/llvm/test/Analysis/BasicAA/phi-speculation.ll
index ed0d49bc09941..198c5340488ec 100644
--- a/llvm/test/Analysis/BasicAA/phi-speculation.ll
+++ b/llvm/test/Analysis/BasicAA/phi-speculation.ll
@@ -1,7 +1,7 @@
 target datalayout =
 "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; ptr_phi and ptr2_phi do not alias.
 ; CHECK: test_noalias_1
diff --git a/llvm/test/Analysis/BasicAA/pr31761.ll b/llvm/test/Analysis/BasicAA/pr31761.ll
index 318dfdc9c246b..e8d059bbca911 100644
--- a/llvm/test/Analysis/BasicAA/pr31761.ll
+++ b/llvm/test/Analysis/BasicAA/pr31761.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Analysis/BasicAA/ptrmask.ll b/llvm/test/Analysis/BasicAA/ptrmask.ll
index 27c14ebb77cf9..7dded02f88fac 100644
--- a/llvm/test/Analysis/BasicAA/ptrmask.ll
+++ b/llvm/test/Analysis/BasicAA/ptrmask.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -aa-eval -print-no-aliases -disable-output %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -aa-eval -print-no-aliases -disable-output %s 2>&1 | FileCheck %s
 
 %struct = type <{ [20 x i64] }>
 
diff --git a/llvm/test/Analysis/BasicAA/pure-const-dce.ll b/llvm/test/Analysis/BasicAA/pure-const-dce.ll
index e48992860a604..57a1c73cd706d 100644
--- a/llvm/test/Analysis/BasicAA/pure-const-dce.ll
+++ b/llvm/test/Analysis/BasicAA/pure-const-dce.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 @g = global i32 0
 
diff --git a/llvm/test/Analysis/BasicAA/q.bad.ll b/llvm/test/Analysis/BasicAA/q.bad.ll
index 2c7bc1d8591ee..0d22f37cc251d 100644
--- a/llvm/test/Analysis/BasicAA/q.bad.ll
+++ b/llvm/test/Analysis/BasicAA/q.bad.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 target triple = "thumbv7--linux-gnueabi"
 
diff --git a/llvm/test/Analysis/BasicAA/recphi.ll b/llvm/test/Analysis/BasicAA/recphi.ll
new file mode 100644
index 0000000000000..130058c745604
--- /dev/null
+++ b/llvm/test/Analysis/BasicAA/recphi.ll
@@ -0,0 +1,85 @@
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -basic-aa-recphi -disable-output 2>&1 | FileCheck %s
+
+; CHECK-LABEL: Function: simple: 5 pointers, 0 call sites
+; CHECK:         NoAlias:      float* %src1, float* %src2
+; CHECK:         NoAlias:      float* %phi, float* %src1
+; CHECK:         MayAlias:     float* %phi, float* %src2
+; CHECK:         NoAlias:      float* %next, float* %src1
+; CHECK:         MayAlias:     float* %next, float* %src2
+; CHECK:         NoAlias:      float* %next, float* %phi
+; CHECK:         NoAlias:      float* %g, float* %src1
+; CHECK:         NoAlias:      float* %g, float* %src2
+; CHECK:         NoAlias:      float* %g, float* %phi
+; CHECK:         NoAlias:      float* %g, float* %next
+define void @simple(float *%src1, float * noalias %src2, i32 %n) nounwind {
+entry:
+  br label %loop
+
+loop:
+  %phi = phi float* [ %src2, %entry ], [ %next, %loop ]
+  %idx = phi i32 [ 0, %entry ], [ %idxn, %loop ]
+  %next = getelementptr inbounds float, float* %phi, i32 1
+  %g = getelementptr inbounds float, float* %src1, i32 3
+  %l = load float, float* %phi
+  %a = fadd float %l, 1.0
+  store float %a, float* %g
+  %idxn = add nsw nuw i32 %idx, 1
+  %cmp5 = icmp eq i32 %idxn, %n
+  br i1 %cmp5, label %end, label %loop
+
+end:
+  ret void
+}
+
+; CHECK-LABEL: Function: notmust: 6 pointers, 0 call sites
+; CHECK:        MustAlias:    [2 x i32]* %tab, i8* %0
+; CHECK:        PartialAlias: [2 x i32]* %tab, i32* %arrayidx
+; CHECK:        NoAlias:      i32* %arrayidx, i8* %0
+; CHECK:        MustAlias:    [2 x i32]* %tab, i32* %arrayidx1
+; CHECK:        MustAlias:    i32* %arrayidx1, i8* %0
+; CHECK:        NoAlias:      i32* %arrayidx, i32* %arrayidx1
+; CHECK:        MayAlias:     [2 x i32]* %tab, i32* %p.addr.05.i
+; CHECK:        MayAlias:     i32* %p.addr.05.i, i8* %0
+; CHECK:        MayAlias:     i32* %arrayidx, i32* %p.addr.05.i
+; CHECK:        MayAlias:     i32* %arrayidx1, i32* %p.addr.05.i
+; CHECK:        MayAlias:     [2 x i32]* %tab, i32* %incdec.ptr.i
+; CHECK:        NoAlias:      i32* %incdec.ptr.i, i8* %0
+; CHECK:        MayAlias:     i32* %arrayidx, i32* %incdec.ptr.i
+; CHECK:        NoAlias:      i32* %arrayidx1, i32* %incdec.ptr.i
+; CHECK:        NoAlias:      i32* %incdec.ptr.i, i32* %p.addr.05.i
+define i32 @notmust() nounwind {
+entry:
+  %tab = alloca [2 x i32], align 4
+  %0 = bitcast [2 x i32]* %tab to i8*
+  %arrayidx = getelementptr inbounds [2 x i32], [2 x i32]* %tab, i32 0, i32 1
+  store i32 0, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds [2 x i32], [2 x i32]* %tab, i32 0, i32 0
+  store i32 0, i32* %arrayidx1, align 4
+  %1 = add i32 1, 1
+  %cmp4.i = icmp slt i32 %1, 2
+  br i1 %cmp4.i, label %while.body.i, label %f.exit
+
+while.body.i: ; preds = %while.body.i, %entry
+  %2 = phi i32 [ 1, %while.body.i ], [ %1, %entry ]
+  %foo.06.i = phi i32 [ %sub.i, %while.body.i ], [ 2, %entry ]
+  %p.addr.05.i = phi i32* [ %incdec.ptr.i, %while.body.i ], [ %arrayidx1, %entry ]
+  %sub.i = sub nsw i32 %foo.06.i, %2
+  %incdec.ptr.i = getelementptr inbounds i32, i32* %p.addr.05.i, i32 1
+  store i32 %sub.i, i32* %p.addr.05.i, align 4
+  %cmp.i = icmp sgt i32 %sub.i, 1
+  br i1 %cmp.i, label %while.body.i, label %f.exit
+
+f.exit: ; preds = %entry, %while.body.i
+  %3 = load i32, i32* %arrayidx1, align 4
+  %cmp = icmp eq i32 %3, 2
+  %4 = load i32, i32* %arrayidx, align 4
+  %cmp4 = icmp eq i32 %4, 1
+  %or.cond = and i1 %cmp, %cmp4
+  br i1 %or.cond, label %if.end, label %if.then
+
+if.then: ; preds = %f.exit
+  unreachable
+
+if.end: ; preds = %f.exit
+  ret i32 0
+}
diff --git a/llvm/test/Analysis/BasicAA/returned.ll b/llvm/test/Analysis/BasicAA/returned.ll
index f0d0f1ec1fe9b..1c158e4d18897 100644
--- a/llvm/test/Analysis/BasicAA/returned.ll
+++ b/llvm/test/Analysis/BasicAA/returned.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Analysis/BasicAA/sequential-gep.ll b/llvm/test/Analysis/BasicAA/sequential-gep.ll
index 5bedab61e17dc..ab97c6492d360 100644
--- a/llvm/test/Analysis/BasicAA/sequential-gep.ll
+++ b/llvm/test/Analysis/BasicAA/sequential-gep.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; CHECK: Function: t1
 ; CHECK: NoAlias: i32* %gep1, i32* %gep2
diff --git a/llvm/test/Analysis/BasicAA/store-promote.ll b/llvm/test/Analysis/BasicAA/store-promote.ll
index 23b74bc1aef52..c4a5feba65ead 100644
--- a/llvm/test/Analysis/BasicAA/store-promote.ll
+++ b/llvm/test/Analysis/BasicAA/store-promote.ll
@@ -2,8 +2,8 @@
 ; disambiguating some obvious cases.  If LICM is able to disambiguate the
 ; two pointers, then the load should be hoisted, and the store sunk.
 
-; RUN: opt < %s -basicaa -licm -enable-mssa-loop-dependency=false -S | FileCheck %s -check-prefixes=CHECK,AST
-; RUN: opt < %s -basicaa -licm -enable-mssa-loop-dependency=true  -S | FileCheck %s -check-prefixes=CHECK,MSSA
+; RUN: opt < %s -basic-aa -licm -enable-mssa-loop-dependency=false -S | FileCheck %s -check-prefixes=CHECK,AST
+; RUN: opt < %s -basic-aa -licm -enable-mssa-loop-dependency=true  -S | FileCheck %s -check-prefixes=CHECK,MSSA
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 @A = global i32 7               ; <i32*> [#uses=3]
diff --git a/llvm/test/Analysis/BasicAA/struct-geps.ll b/llvm/test/Analysis/BasicAA/struct-geps.ll
index e048baf4c64a9..989634bc2a728 100644
--- a/llvm/test/Analysis/BasicAA/struct-geps.ll
+++ b/llvm/test/Analysis/BasicAA/struct-geps.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Analysis/BasicAA/tail-byval.ll b/llvm/test/Analysis/BasicAA/tail-byval.ll
index 0aa8dfdaedff2..2fdedb2ae60b0 100644
--- a/llvm/test/Analysis/BasicAA/tail-byval.ll
+++ b/llvm/test/Analysis/BasicAA/tail-byval.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -aa-eval -print-all-alias-modref-info -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -aa-eval -print-all-alias-modref-info -disable-output < %s 2>&1 | FileCheck %s
 
 declare void @takebyval(i32* byval %p)
 
diff --git a/llvm/test/Analysis/BasicAA/tailcall-modref.ll b/llvm/test/Analysis/BasicAA/tailcall-modref.ll
index 5857e684bc579..ed4f3a1834e02 100644
--- a/llvm/test/Analysis/BasicAA/tailcall-modref.ll
+++ b/llvm/test/Analysis/BasicAA/tailcall-modref.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -instcombine -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -instcombine -S | FileCheck %s
 
 define i32 @test() {
 ; CHECK: ret i32 0
diff --git a/llvm/test/Analysis/BasicAA/underlying-value.ll b/llvm/test/Analysis/BasicAA/underlying-value.ll
index 0cfbdb893b308..49ceb89737900 100644
--- a/llvm/test/Analysis/BasicAA/underlying-value.ll
+++ b/llvm/test/Analysis/BasicAA/underlying-value.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -licm -S < %s
+; RUN: opt -basic-aa -licm -S < %s
 ; PR9931
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/llvm/test/Analysis/BasicAA/unreachable-block.ll b/llvm/test/Analysis/BasicAA/unreachable-block.ll
index d6c149f816618..db9c513397a88 100644
--- a/llvm/test/Analysis/BasicAA/unreachable-block.ll
+++ b/llvm/test/Analysis/BasicAA/unreachable-block.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -aa-eval -disable-output < %s > /dev/null 2>&1
+; RUN: opt -basic-aa -aa-eval -disable-output < %s > /dev/null 2>&1
 
 ; BasicAA shouldn't infinitely recurse on the use-def cycles in
 ; unreachable code.
diff --git a/llvm/test/Analysis/BasicAA/vscale.ll b/llvm/test/Analysis/BasicAA/vscale.ll
index e0bafbf72e856..ecd3091e92fb0 100644
--- a/llvm/test/Analysis/BasicAA/vscale.ll
+++ b/llvm/test/Analysis/BasicAA/vscale.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; getelementptr
 
diff --git a/llvm/test/Analysis/BasicAA/zext.ll b/llvm/test/Analysis/BasicAA/zext.ll
index f8f02353db2ec..ebba92638b408 100644
--- a/llvm/test/Analysis/BasicAA/zext.ll
+++ b/llvm/test/Analysis/BasicAA/zext.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/assign.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/assign.ll
index 237fe72732e8d..8f20c96c5e8f5 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/assign.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/assign.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA handles assignment in an inclusion-based 
 ; manner
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; CHECK-LABEL: Function: test_assign
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/assign2.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/assign2.ll
index 8bd98f388f05e..d2654523a95b8 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/assign2.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/assign2.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA handles assignment in an inclusion-based 
 ; manner
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; CHECK-LABEL: Function: test_assign2
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/attrs-below.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/attrs-below.ll
index 4cf918081bb50..b320fb5b2ebd5 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/attrs-below.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/attrs-below.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that AliasAttrs are propagated not only on the same 
 ; level but also downward.
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; CHECK-LABEL: Function: test_attr_below
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/attrs.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/attrs.ll
index 4ba3d1510be9e..14a2bfea7674f 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/attrs.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/attrs.ll
@@ -1,6 +1,6 @@
 ; This testcase ensures that CFL AA handles escaped values no more conservative than it should
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; CHECK-LABEL: Function: test_local
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/basic-interproc.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/basic-interproc.ll
index 9ec52521a0461..367c4e82fff74 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/basic-interproc.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/basic-interproc.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA won't be too conservative when trying to do
 ; interprocedural analysis on simple callee
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; CHECK-LABEL: Function: noop_callee
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/cycle.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/cycle.ll
index 4ce9467ddc374..d9a769a66eabb 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/cycle.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/cycle.ll
@@ -1,6 +1,6 @@
 ; This testcase ensures that CFL AA handles assignment cycles correctly
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; CHECK-LABEL: Function: test_cycle
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-arg-deref-escape.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-arg-deref-escape.ll
index c71f40403f6e0..e720d1a3540ad 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-arg-deref-escape.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-arg-deref-escape.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to escape the memory pointed to by its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 declare void @opaque(i32*)
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-arg-escape.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-arg-escape.ll
index a87764fa7f711..b8fc6e6e10c82 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-arg-escape.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-arg-escape.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to escape its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 declare void @opaque(i32*)
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-arg.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-arg.ll
index 002ff173e7c6a..73a72c7509fac 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-arg.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-arg.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to return one of its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 define i32* @return_arg_callee(i32* %arg1, i32* %arg2) {
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-deref-arg-multilevel.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-deref-arg-multilevel.ll
index 764d14449104f..ce273ab793208 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-deref-arg-multilevel.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-deref-arg-multilevel.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to return the multi-level dereference of one of its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 define i32* @return_deref_arg_multilevel_callee(i32*** %arg1) {
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-deref-arg.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-deref-arg.ll
index 76e7c77bf87ba..3f6b50103a047 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-deref-arg.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-deref-arg.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to return the dereference of one of its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 define i32* @return_deref_arg_callee(i32** %arg1) {
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-escape.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-escape.ll
index 5eb8f30dbaaba..65a1d0f998c97 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-escape.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-escape.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to return an escaped pointer
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 declare noalias i8* @malloc(i64)
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-ref-arg-multilevel.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-ref-arg-multilevel.ll
index 4e3222dff5264..472c53e101413 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-ref-arg-multilevel.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-ref-arg-multilevel.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to return the multi-level reference of one of its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 declare noalias i8* @malloc(i64)
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-ref-arg.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-ref-arg.ll
index 623468585162d..62c743bac33a5 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-ref-arg.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-ref-arg.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to return the reference of one of its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 declare noalias i8* @malloc(i64)
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-unknown.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-unknown.ll
index 4c20269896ab7..4843d57bea67c 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-unknown.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-ret-unknown.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to return an unknown pointer
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 @g = external global i32
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-store-arg-multilevel.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-store-arg-multilevel.ll
index bf19e270b6463..0aa994a983d8f 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-store-arg-multilevel.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-store-arg-multilevel.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to mutate the memory pointed to by its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 declare noalias i8* @malloc(i64)
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-store-arg-unknown.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-store-arg-unknown.ll
index 05717b5ddb30d..253fbe597e992 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-store-arg-unknown.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-store-arg-unknown.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to mutate the memory pointed to by its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 @g = external global i32
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-store-arg.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-store-arg.ll
index 89c02198f73ba..235996d6fa5a2 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-store-arg.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/interproc-store-arg.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to mutate the memory pointed to by its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 define void @store_arg_callee(i32** %arg1, i32* %arg2) {
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/memalias.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/memalias.ll
index d3fb09091aa0c..9f9d37384830d 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/memalias.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/memalias.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA correctly handles simple memory alias 
 ; pattern
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; CHECK-LABEL: Function: test_memalias
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/struct.ll b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/struct.ll
index c1d25c1e3c214..2f20eb5b8575d 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Andersen/struct.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Andersen/struct.ll
@@ -1,6 +1,6 @@
 ; Ensures that our struct ops are sane.
 
-; RUN: opt < %s -disable-basicaa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-anders-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-anders-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; Since we ignore non-pointer values, we effectively ignore extractvalue
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/arguments-globals.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/arguments-globals.ll
index aa3cada3bec6d..579f1629bad9d 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/arguments-globals.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/arguments-globals.ll
@@ -3,7 +3,7 @@
 ; (Everything should alias everything, because args can alias globals, so the
 ; aliasing sets should of args+alloca+global should be combined)
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-may-aliases -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-may-aliases -disable-output 2>&1 | FileCheck %s
 
 ; CHECK:     Function: test
 
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/attr-escape.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/attr-escape.ll
index 3adce6a923b25..f943e2225cbac 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/attr-escape.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/attr-escape.ll
@@ -1,6 +1,6 @@
 ; This testcase ensures that CFL AA handles escaped values no more conservative than it should
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; CHECK-LABEL: Function: test_local
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/basic-interproc.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/basic-interproc.ll
index 8a1d62ebd247b..80876881c887d 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/basic-interproc.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/basic-interproc.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA won't be too conservative when trying to do
 ; interprocedural analysis on simple callee
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; CHECK-LABEL: Function: noop_callee
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/branch-alias.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/branch-alias.ll
index 4158d4a6464f2..a5f783546b80f 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/branch-alias.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/branch-alias.ll
@@ -13,7 +13,7 @@
 ;    int* ShouldAliasA = *AliasA1;
 ;  }
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-may-aliases -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-may-aliases -disable-output 2>&1 | FileCheck %s
 
 ; CHECK: Function: ptr_test
 define void @ptr_test() #0 {
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/const-expr-gep.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/const-expr-gep.ll
index c35f6a03cdb9b..57dcfb8392c38 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/const-expr-gep.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/const-expr-gep.ll
@@ -2,7 +2,7 @@
 ; resolvable by cfl-steens-aa, but require analysis of getelementptr constant exprs.
 ; Derived from BasicAA/2003-12-11-ConstExprGEP.ll
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-may-aliases -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-may-aliases -disable-output 2>&1 | FileCheck %s
 
 %T = type { i32, [10 x i8] }
 
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/const-exprs.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/const-exprs.ll
index 2e8d7abb0f26c..63dc8de1dda69 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/const-exprs.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/const-exprs.ll
@@ -1,4 +1,4 @@
-; RUN: opt %s -S -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info 2>&1 | FileCheck %s
+; RUN: opt %s -S -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info 2>&1 | FileCheck %s
 ;
 ; Regression: we weren't properly checking constexpr selects.
 
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/full-store-partial-alias.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/full-store-partial-alias.ll
index 2f9ac96c0cc62..3f494c05e80e4 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/full-store-partial-alias.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/full-store-partial-alias.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S -disable-basicaa -tbaa -cfl-steens-aa -gvn < %s | FileCheck -check-prefix=CFLSteensAA %s
-; RUN: opt -S -disable-basicaa -tbaa -gvn < %s | FileCheck %s
+; RUN: opt -S -disable-basic-aa -tbaa -cfl-steens-aa -gvn < %s | FileCheck -check-prefix=CFLSteensAA %s
+; RUN: opt -S -disable-basic-aa -tbaa -gvn < %s | FileCheck %s
 ; Adapted from the BasicAA full-store-partial-alias.ll test.
 
 ; CFL AA could notice that the store stores to the entire %u object,
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/gep-index-no-alias.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/gep-index-no-alias.ll
index 906b71c5aab32..929bb1fef090b 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/gep-index-no-alias.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/gep-index-no-alias.ll
@@ -1,6 +1,6 @@
 ; This testcase ensures that gep result does not alias gep indices
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-no-aliases -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-no-aliases -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-no-aliases -disable-output 2>&1 | FileCheck %s
 
 ; CHECK: Function: foo
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/gep-signed-arithmetic.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/gep-signed-arithmetic.ll
index 34095cedf7387..2190ec31fb764 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/gep-signed-arithmetic.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/gep-signed-arithmetic.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; Derived from BasicAA/2010-09-15-GEP-SignedArithmetic.ll
 
 target datalayout = "e-p:32:32:32"
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-arg-deref-escape.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-arg-deref-escape.ll
index 8c92d281a80b7..de8c6a149313d 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-arg-deref-escape.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-arg-deref-escape.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to escape the memory pointed to by its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 declare void @opaque(i32*)
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-arg-escape.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-arg-escape.ll
index 0dfcae7920335..872f4129c3e2f 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-arg-escape.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-arg-escape.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to escape its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 declare void @opaque(i32*)
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-arg.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-arg.ll
index e6e2065e73100..470e8563c2784 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-arg.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-arg.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to return one of its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 define i32* @return_arg_callee(i32* %arg1, i32* %arg2) {
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-deref-arg-multilevel.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-deref-arg-multilevel.ll
index 3c7c614998296..0c2eef2fba79c 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-deref-arg-multilevel.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-deref-arg-multilevel.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to return the multi-level dereference of one of its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 define i32* @return_deref_arg_multilevel_callee(i32*** %arg1) {
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-deref-arg.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-deref-arg.ll
index e1db5d34f3490..39f00800769bc 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-deref-arg.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-deref-arg.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to return the dereference of one of its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 define i32* @return_deref_arg_callee(i32** %arg1) {
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-escape.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-escape.ll
index 5019e51d7b8bf..2fe09c6d1ba35 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-escape.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-escape.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to return an escaped pointer
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 declare noalias i8* @malloc(i64)
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-ref-arg-multilevel.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-ref-arg-multilevel.ll
index e94cbbbdf30e1..ec8355858acce 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-ref-arg-multilevel.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-ref-arg-multilevel.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to return the multi-level reference of one of its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 declare noalias i8* @malloc(i64)
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-ref-arg.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-ref-arg.ll
index aae642c424da3..ae0134c237b51 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-ref-arg.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-ref-arg.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to return the reference of one of its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 declare noalias i8* @malloc(i64)
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-unknown.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-unknown.ll
index bc4a9bc901bc4..169b25ccb4967 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-unknown.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-ret-unknown.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to return an unknown pointer
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 @g = external global i32
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-store-arg-multilevel.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-store-arg-multilevel.ll
index 54e39d87127df..de32e12ea0eaa 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-store-arg-multilevel.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-store-arg-multilevel.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to mutate the memory pointed to by its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 declare noalias i8* @malloc(i64)
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-store-arg-unknown.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-store-arg-unknown.ll
index a90514c58fa95..a82a7a50a153f 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-store-arg-unknown.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-store-arg-unknown.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to mutate the memory pointed to by its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 @g = external global i32
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-store-arg.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-store-arg.ll
index e520de4d0b1c1..92ec7c1cd6900 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-store-arg.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/interproc-store-arg.ll
@@ -1,7 +1,7 @@
 ; This testcase ensures that CFL AA answers queries soundly when callee tries 
 ; to mutate the memory pointed to by its parameters
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 define void @store_arg_callee(i32** %arg1, i32* %arg2) {
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/malloc-and-free.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/malloc-and-free.ll
index 76c97db8038c7..0530805aea707 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/malloc-and-free.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/malloc-and-free.ll
@@ -1,6 +1,6 @@
 ; This testcase ensures that CFL AA handles malloc and free in a sound and precise manner
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-no-aliases -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-no-aliases -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-no-aliases -disable-output 2>&1 | FileCheck %s
 
 declare noalias i8* @malloc(i64)
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/multilevel-combine.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/multilevel-combine.ll
index 32abe9d8f744c..b87f5e05a98c1 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/multilevel-combine.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/multilevel-combine.ll
@@ -8,7 +8,7 @@
 ; }
 ;
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-may-aliases -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-may-aliases -disable-output 2>&1 | FileCheck %s
 
 %T = type { i32, [10 x i8] }
 
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/must-and-partial.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/must-and-partial.ll
index b1f96220662a2..8a1c1f6651047 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/must-and-partial.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/must-and-partial.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info 2>&1 | FileCheck %s
 ; When merging MustAlias and PartialAlias, merge to PartialAlias
 ; instead of MayAlias.
 
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/opaque-call-alias.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/opaque-call-alias.ll
index a03b104778bdf..e46c3b315588b 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/opaque-call-alias.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/opaque-call-alias.ll
@@ -2,7 +2,7 @@
 ; its own stratified set. This would make cases like the one in @test say that
 ; nothing (except %Escapes and %Arg) can alias
 
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; CHECK:     Function: test
 ; CHECK:     NoAlias: i8* %Arg, i8* %Escapes
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/pr27213.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/pr27213.ll
index d685cab6275e8..f244893742fe2 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/pr27213.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/pr27213.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-may-aliases -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-may-aliases -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=cfl-steens-aa -passes=aa-eval -print-may-aliases -disable-output 2>&1 | FileCheck %s
 
 ; CHECK-LABEL:     Function: foo
diff --git a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/va.ll b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/va.ll
index 43fca92074127..75347b905f1ec 100644
--- a/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/va.ll
+++ b/llvm/test/Analysis/CFLAliasAnalysis/Steensgaard/va.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -disable-basicaa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -cfl-steens-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; CHECK-LABEL: Function: test1
 ; CHECK: MayAlias: i32* %X, i32* %tmp
diff --git a/llvm/test/Analysis/CallGraph/callback-calls.ll b/llvm/test/Analysis/CallGraph/callback-calls.ll
new file mode 100644
index 0000000000000..b152dea330ffc
--- /dev/null
+++ b/llvm/test/Analysis/CallGraph/callback-calls.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -print-callgraph -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=print-callgraph -disable-output 2>&1 | FileCheck %s
+
+; CHECK: Call graph node for function: 'caller'
+; CHECK-NEXT:   CS<{{.*}}> calls function 'broker'
+; CHECK-NEXT:   CS<None> calls function 'callback'
+
+define void @caller(i32* %arg) {
+  call void @broker(void (i32*)* @callback, i32* %arg)
+  ret void
+}
+
+define void @callback(i32* %arg) {
+  ret void
+}
+
+declare !callback !0 void @broker(void (i32*)*, i32*)
+
+!0 = !{!1}
+!1 = !{i64 0, i64 1, i1 false}
diff --git a/llvm/test/Analysis/CallGraph/non-leaf-intrinsics.ll b/llvm/test/Analysis/CallGraph/non-leaf-intrinsics.ll
index de03012e4c547..48efcd96a575c 100644
--- a/llvm/test/Analysis/CallGraph/non-leaf-intrinsics.ll
+++ b/llvm/test/Analysis/CallGraph/non-leaf-intrinsics.ll
@@ -23,7 +23,7 @@ entry:
 
 
 ; CHECK: Call graph node <<null function>>
-; CHECK:  CS<0x0> calls function 'f'
+; CHECK:  CS<None> calls function 'f'
 
 ; CHECK: Call graph node for function: 'calls_patchpoint'
 ; CHECK-NEXT:  CS<[[addr_1:[^>]+]]> calls external node
diff --git a/llvm/test/Analysis/ConstantFolding/vectorgep-crash.ll b/llvm/test/Analysis/ConstantFolding/vectorgep-crash.ll
index b2faf89c2b7f6..1562d74d83aae 100644
--- a/llvm/test/Analysis/ConstantFolding/vectorgep-crash.ll
+++ b/llvm/test/Analysis/ConstantFolding/vectorgep-crash.ll
@@ -24,7 +24,7 @@ top:
 
 @G = internal global [65 x %struct.A] zeroinitializer, align 16
 ; CHECK-LABEL: @test
-; CHECK: ret <16 x i32*> getelementptr ([65 x %struct.A], [65 x %struct.A]* @G, <16 x i64> zeroinitializer, <16 x i64> <i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16>, <16 x i32> zeroinitializer)
+; CHECK: ret <16 x i32*> getelementptr ([65 x %struct.A], [65 x %struct.A]* @G, <16 x i64> zeroinitializer, <16 x i64> <i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16>, i32 0)
 define <16 x i32*> @test() {
 vector.body:
   %VectorGep = getelementptr [65 x %struct.A], [65 x %struct.A]* @G, <16 x i64> zeroinitializer, <16 x i64> <i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16>, <16 x i32> zeroinitializer
diff --git a/llvm/test/Analysis/CostModel/AArch64/aggregates.ll b/llvm/test/Analysis/CostModel/AArch64/aggregates.ll
index 6ce57f966ed1a..35d232b3b69ab 100644
--- a/llvm/test/Analysis/CostModel/AArch64/aggregates.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/aggregates.ll
@@ -4,63 +4,119 @@
 ; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -cost-model -cost-kind=code-size -analyze | FileCheck %s --check-prefixes=ALL,CODESIZE
 
 define i32 @extract_first_i32({i32, i32} %agg) {
-; ALL-LABEL: 'extract_first_i32'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 0
-; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+; THROUGHPUT-LABEL: 'extract_first_i32'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 0
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; LATENCY-LABEL: 'extract_first_i32'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 0
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+; CODESIZE-LABEL: 'extract_first_i32'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 0
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
 ;
   %r = extractvalue {i32, i32} %agg, 0
   ret i32 %r
 }
 
 define i32 @extract_second_i32({i32, i32} %agg) {
-; ALL-LABEL: 'extract_second_i32'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 1
-; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+; THROUGHPUT-LABEL: 'extract_second_i32'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; LATENCY-LABEL: 'extract_second_i32'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+; CODESIZE-LABEL: 'extract_second_i32'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
 ;
   %r = extractvalue {i32, i32} %agg, 1
   ret i32 %r
 }
 
 define i32 @extract_i32({i32, i1} %agg) {
-; ALL-LABEL: 'extract_i32'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 0
-; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+; THROUGHPUT-LABEL: 'extract_i32'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 0
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; LATENCY-LABEL: 'extract_i32'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 0
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+; CODESIZE-LABEL: 'extract_i32'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 0
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
 ;
   %r = extractvalue {i32, i1} %agg, 0
   ret i32 %r
 }
 
 define i1 @extract_i1({i32, i1} %agg) {
-; ALL-LABEL: 'extract_i1'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 1
-; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i1 %r
+; THROUGHPUT-LABEL: 'extract_i1'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i1 %r
+;
+; LATENCY-LABEL: 'extract_i1'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i1 %r
+;
+; CODESIZE-LABEL: 'extract_i1'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i1 %r
 ;
   %r = extractvalue {i32, i1} %agg, 1
   ret i1 %r
 }
 
 define float @extract_float({i32, float} %agg) {
-; ALL-LABEL: 'extract_float'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, float } %agg, 1
-; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %r
+; THROUGHPUT-LABEL: 'extract_float'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, float } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
+;
+; LATENCY-LABEL: 'extract_float'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, float } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %r
+;
+; CODESIZE-LABEL: 'extract_float'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, float } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %r
 ;
   %r = extractvalue {i32, float} %agg, 1
   ret float %r
 }
 
 define [42 x i42] @extract_array({i32, [42 x i42]} %agg) {
-; ALL-LABEL: 'extract_array'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, [42 x i42] } %agg, 1
-; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret [42 x i42] %r
+; THROUGHPUT-LABEL: 'extract_array'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, [42 x i42] } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret [42 x i42] %r
+;
+; LATENCY-LABEL: 'extract_array'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, [42 x i42] } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret [42 x i42] %r
+;
+; CODESIZE-LABEL: 'extract_array'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, [42 x i42] } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret [42 x i42] %r
 ;
   %r = extractvalue {i32, [42 x i42]} %agg, 1
   ret [42 x i42] %r
 }
 
 define <42 x i42> @extract_vector({i32, <42 x i42>} %agg) {
-; ALL-LABEL: 'extract_vector'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, <42 x i42> } %agg, 1
-; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <42 x i42> %r
+; THROUGHPUT-LABEL: 'extract_vector'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, <42 x i42> } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <42 x i42> %r
+;
+; LATENCY-LABEL: 'extract_vector'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, <42 x i42> } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <42 x i42> %r
+;
+; CODESIZE-LABEL: 'extract_vector'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, <42 x i42> } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <42 x i42> %r
 ;
   %r = extractvalue {i32, <42 x i42>} %agg, 1
   ret <42 x i42> %r
@@ -69,9 +125,17 @@ define <42 x i42> @extract_vector({i32, <42 x i42>} %agg) {
 %T1 = type { i32, float, <4 x i1> }
 
 define %T1 @extract_struct({i32, %T1} %agg) {
-; ALL-LABEL: 'extract_struct'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, %T1 } %agg, 1
-; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret %T1 %r
+; THROUGHPUT-LABEL: 'extract_struct'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, %T1 } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret %T1 %r
+;
+; LATENCY-LABEL: 'extract_struct'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, %T1 } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret %T1 %r
+;
+; CODESIZE-LABEL: 'extract_struct'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, %T1 } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret %T1 %r
 ;
   %r = extractvalue {i32, %T1} %agg, 1
   ret %T1 %r
diff --git a/llvm/test/Analysis/CostModel/AArch64/cast.ll b/llvm/test/Analysis/CostModel/AArch64/cast.ll
index 24e514b870962..2255f84f9ff4c 100644
--- a/llvm/test/Analysis/CostModel/AArch64/cast.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/cast.ll
@@ -267,7 +267,7 @@ define i32 @casts_no_users() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %r0 = sext i1 undef to i8
   %r1 = zext i1 undef to i8
@@ -609,7 +609,7 @@ define i32 @casts_with_users(i8 %a, i16 %b, i32 %c, i64 %d, i1 %e) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r23, i16* undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r24, i32* undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r25, i64* undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r12
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r12
 ;
   %r0 = sext i8 %a to i16
   %r1 = sext i8 %a to i32
@@ -683,7 +683,7 @@ define i32 @bitcasts() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = bitcast double undef to i64
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = bitcast half undef to i16
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = bitcast i16 undef to half
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %a = bitcast i32 undef to i32
   %b = bitcast float undef to float
@@ -731,7 +731,7 @@ define i32 @load_extends() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = zext <2 x i16> %loadv2i16 to <2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = sext <2 x i32> %loadv2i32 to <2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %loadi8 = load i8, i8* undef
   %loadi16 = load i16, i16* undef
@@ -786,7 +786,7 @@ define i32 @store_truncs() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r4, i16* undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = trunc i16 undef to i8
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %r5, i8* undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %r0 = trunc i64 undef to i8
   store i8 %r0, i8* undef
diff --git a/llvm/test/Analysis/CostModel/AArch64/cmp.ll b/llvm/test/Analysis/CostModel/AArch64/cmp.ll
index 7e8fc30141d68..c8512bb2664ce 100644
--- a/llvm/test/Analysis/CostModel/AArch64/cmp.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/cmp.ll
@@ -17,7 +17,7 @@ define i32 @cmps() {
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %a10 = fcmp olt <8 x half> undef, undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a11 = fcmp oge <4 x float> undef, undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a12 = fcmp oge <2 x double> undef, undef
-; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-SIZE-LABEL: 'cmps'
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a0 = icmp slt i8 undef, undef
diff --git a/llvm/test/Analysis/CostModel/AArch64/select.ll b/llvm/test/Analysis/CostModel/AArch64/select.ll
index 6ec4cff5cf84b..25af9af1c6e93 100644
--- a/llvm/test/Analysis/CostModel/AArch64/select.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/select.ll
@@ -18,7 +18,7 @@ define void @select() {
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16a = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v16b = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v16c = select <16 x i1> undef, <16 x i64> undef, <16 x i64> undef
-; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SIZE-LABEL: 'select'
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = select i1 undef, i8 undef, i8 undef
diff --git a/llvm/test/Analysis/CostModel/AArch64/shuffle-broadcast.ll b/llvm/test/Analysis/CostModel/AArch64/shuffle-broadcast.ll
index 61f28cbea5af5..355ed520575fb 100644
--- a/llvm/test/Analysis/CostModel/AArch64/shuffle-broadcast.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/shuffle-broadcast.ll
@@ -14,7 +14,7 @@ define void @broadcast() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v15 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v17 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v7 = shufflevector <2 x i8> undef, <2 x i8>undef, <2 x i32> zeroinitializer
   %v8 = shufflevector <4 x i8> undef, <4 x i8>undef, <4 x i32> zeroinitializer
diff --git a/llvm/test/Analysis/CostModel/AArch64/store.ll b/llvm/test/Analysis/CostModel/AArch64/store.ll
index 8e23268bada7f..63741756fa0fd 100644
--- a/llvm/test/Analysis/CostModel/AArch64/store.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/store.ll
@@ -27,7 +27,7 @@ define void @getMemoryOpCost() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> undef, <4 x i8>* undef, align 4
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %1 = load <2 x i8>, <2 x i8>* undef, align 2
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %2 = load <4 x i8>, <4 x i8>* undef, align 4
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'getMemoryOpCost'
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i64> undef, <4 x i64>* undef, align 4
@@ -69,7 +69,7 @@ define void @getMemoryOpCost() {
 ; SLOW_MISALIGNED_128_STORE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> undef, <4 x i8>* undef, align 4
 ; SLOW_MISALIGNED_128_STORE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %1 = load <2 x i8>, <2 x i8>* undef, align 2
 ; SLOW_MISALIGNED_128_STORE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %2 = load <4 x i8>, <4 x i8>* undef, align 4
-; SLOW_MISALIGNED_128_STORE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; SLOW_MISALIGNED_128_STORE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   store <4 x i64> undef, <4 x i64> * undef
   store <8 x i32> undef, <8 x i32> * undef
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/add-sub.ll b/llvm/test/Analysis/CostModel/AMDGPU/add-sub.ll
index 9a2c01058b28e..609769fd51488 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/add-sub.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/add-sub.ll
@@ -90,7 +90,7 @@ define amdgpu_kernel void @add_v4i64(<4 x i64> addrspace(1)* %out, <4 x i64> add
 }
 
 ; ALL: 'add_v16i64'
-; ALL: estimated cost of 32 for {{.*}} add <16 x i64>
+; ALL: estimated cost of 128 for {{.*}} add <16 x i64>
 define amdgpu_kernel void @add_v16i64(<16 x i64> addrspace(1)* %out, <16 x i64> addrspace(1)* %vaddr, <16 x i64> %b) #0 {
   %vec = load <16 x i64>, <16 x i64> addrspace(1)* %vaddr
   %add = add <16 x i64> %vec, %b
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/mul.ll b/llvm/test/Analysis/CostModel/AMDGPU/mul.ll
index 4d8a66ecd429b..fa36d391f9c3f 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/mul.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/mul.ll
@@ -90,7 +90,7 @@ define amdgpu_kernel void @mul_v4i64(<4 x i64> addrspace(1)* %out, <4 x i64> add
 
 
 ; ALL: 'mul_v8i64'
-; ALL: estimated cost of 128 for {{.*}} mul <8 x i64>
+; ALL: estimated cost of 256 for {{.*}} mul <8 x i64>
 define amdgpu_kernel void @mul_v8i64(<8 x i64> addrspace(1)* %out, <8 x i64> addrspace(1)* %vaddr, <8 x i64> %b) #0 {
   %vec = load <8 x i64>, <8 x i64> addrspace(1)* %vaddr
   %mul = mul <8 x i64> %vec, %b
diff --git a/llvm/test/Analysis/CostModel/ARM/cast.ll b/llvm/test/Analysis/CostModel/ARM/cast.ll
index b1861d7239d66..28f7c6cfcf36e 100644
--- a/llvm/test/Analysis/CostModel/ARM/cast.ll
+++ b/llvm/test/Analysis/CostModel/ARM/cast.ll
@@ -131,31 +131,31 @@ define i32 @casts() {
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r82df = fptrunc <4 x double> undef to <4 x float>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r83df = fptrunc <8 x double> undef to <8 x float>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r84df = fptrunc <16 x double> undef to <16 x float>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80dh = fptrunc double undef to half
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80fh = fptrunc float undef to half
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80dh = fptrunc double undef to half
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80fh = fptrunc float undef to half
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85fd = fpext float undef to double
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r86fd = fpext <2 x float> undef to <2 x double>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r87fd = fpext <4 x float> undef to <4 x double>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r88fd = fpext <8 x float> undef to <8 x double>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r89fd = fpext <16 x float> undef to <16 x double>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hd = fpext half undef to double
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hf = fpext half undef to float
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85hd = fpext half undef to double
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85hf = fpext half undef to float
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
@@ -396,6 +396,7 @@ define i32 @casts() {
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-MVE-RECIP-LABEL: 'casts'
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r0 = sext i1 undef to i8
@@ -512,36 +513,36 @@ define i32 @casts() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rext_b = zext <2 x i32> undef to <2 x i64>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r74 = trunc <8 x i32> undef to <8 x i8>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r75 = trunc <16 x i32> undef to <16 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80df = fptrunc double undef to float
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r81df = fptrunc <2 x double> undef to <2 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r82df = fptrunc <4 x double> undef to <4 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r83df = fptrunc <8 x double> undef to <8 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80df = fptrunc double undef to float
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81df = fptrunc <2 x double> undef to <2 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82df = fptrunc <4 x double> undef to <4 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83df = fptrunc <8 x double> undef to <8 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84df = fptrunc <16 x double> undef to <16 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80dh = fptrunc double undef to half
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 168 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80dh = fptrunc double undef to half
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80fh = fptrunc float undef to half
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85fd = fpext float undef to double
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85fd = fpext float undef to double
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86fd = fpext <2 x float> undef to <2 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r87fd = fpext <4 x float> undef to <4 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 328 for instruction: %r88fd = fpext <8 x float> undef to <8 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1312 for instruction: %r89fd = fpext <16 x float> undef to <16 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hd = fpext half undef to double
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87fd = fpext <4 x float> undef to <4 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88fd = fpext <8 x float> undef to <8 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89fd = fpext <16 x float> undef to <16 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85hd = fpext half undef to double
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1320 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hf = fpext half undef to float
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
@@ -782,6 +783,7 @@ define i32 @casts() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2088 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2048 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2048 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-MAIN-RECIP-LABEL: 'casts'
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r0 = sext i1 undef to i8
@@ -898,36 +900,36 @@ define i32 @casts() {
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %rext_b = zext <2 x i32> undef to <2 x i64>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r74 = trunc <8 x i32> undef to <8 x i8>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r75 = trunc <16 x i32> undef to <16 x i8>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80df = fptrunc double undef to float
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r81df = fptrunc <2 x double> undef to <2 x float>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %r82df = fptrunc <4 x double> undef to <4 x float>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r83df = fptrunc <8 x double> undef to <8 x float>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %r84df = fptrunc <16 x double> undef to <16 x float>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80dh = fptrunc double undef to half
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80fh = fptrunc float undef to half
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85fd = fpext float undef to double
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r86fd = fpext <2 x float> undef to <2 x double>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r87fd = fpext <4 x float> undef to <4 x double>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r88fd = fpext <8 x float> undef to <8 x double>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r89fd = fpext <16 x float> undef to <16 x double>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hd = fpext half undef to double
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hf = fpext half undef to float
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80df = fptrunc double undef to float
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81df = fptrunc <2 x double> undef to <2 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82df = fptrunc <4 x double> undef to <4 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83df = fptrunc <8 x double> undef to <8 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84df = fptrunc <16 x double> undef to <16 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80dh = fptrunc double undef to half
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80fh = fptrunc float undef to half
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85fd = fpext float undef to double
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86fd = fpext <2 x float> undef to <2 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87fd = fpext <4 x float> undef to <4 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88fd = fpext <8 x float> undef to <8 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89fd = fpext <16 x float> undef to <16 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85hd = fpext half undef to double
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85hf = fpext half undef to float
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
@@ -1168,6 +1170,7 @@ define i32 @casts() {
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-BASE-RECIP-LABEL: 'casts'
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r0 = sext i1 undef to i8
@@ -1284,36 +1287,36 @@ define i32 @casts() {
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %rext_b = zext <2 x i32> undef to <2 x i64>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r74 = trunc <8 x i32> undef to <8 x i8>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r75 = trunc <16 x i32> undef to <16 x i8>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80df = fptrunc double undef to float
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r81df = fptrunc <2 x double> undef to <2 x float>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %r82df = fptrunc <4 x double> undef to <4 x float>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r83df = fptrunc <8 x double> undef to <8 x float>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %r84df = fptrunc <16 x double> undef to <16 x float>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80dh = fptrunc double undef to half
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80fh = fptrunc float undef to half
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85fd = fpext float undef to double
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r86fd = fpext <2 x float> undef to <2 x double>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r87fd = fpext <4 x float> undef to <4 x double>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r88fd = fpext <8 x float> undef to <8 x double>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r89fd = fpext <16 x float> undef to <16 x double>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hd = fpext half undef to double
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hf = fpext half undef to float
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80df = fptrunc double undef to float
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81df = fptrunc <2 x double> undef to <2 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82df = fptrunc <4 x double> undef to <4 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83df = fptrunc <8 x double> undef to <8 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84df = fptrunc <16 x double> undef to <16 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80dh = fptrunc double undef to half
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80fh = fptrunc float undef to half
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85fd = fpext float undef to double
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86fd = fpext <2 x float> undef to <2 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87fd = fpext <4 x float> undef to <4 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88fd = fpext <8 x float> undef to <8 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89fd = fpext <16 x float> undef to <16 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85hd = fpext half undef to double
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85hf = fpext half undef to float
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
@@ -1554,6 +1557,7 @@ define i32 @casts() {
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8R-RECIP-LABEL: 'casts'
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r0 = sext i1 undef to i8
@@ -1675,31 +1679,31 @@ define i32 @casts() {
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r82df = fptrunc <4 x double> undef to <4 x float>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r83df = fptrunc <8 x double> undef to <8 x float>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r84df = fptrunc <16 x double> undef to <16 x float>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80dh = fptrunc double undef to half
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80fh = fptrunc float undef to half
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80dh = fptrunc double undef to half
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80fh = fptrunc float undef to half
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85fd = fpext float undef to double
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r86fd = fpext <2 x float> undef to <2 x double>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r87fd = fpext <4 x float> undef to <4 x double>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r88fd = fpext <8 x float> undef to <8 x double>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r89fd = fpext <16 x float> undef to <16 x double>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hd = fpext half undef to double
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hf = fpext half undef to float
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85hd = fpext half undef to double
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85hf = fpext half undef to float
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
@@ -1940,6 +1944,7 @@ define i32 @casts() {
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-MVE-SIZE-LABEL: 'casts'
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r0 = sext i1 undef to i8
@@ -2056,36 +2061,36 @@ define i32 @casts() {
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rext_b = zext <2 x i32> undef to <2 x i64>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r74 = trunc <8 x i32> undef to <8 x i8>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r75 = trunc <16 x i32> undef to <16 x i8>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80df = fptrunc double undef to float
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81df = fptrunc <2 x double> undef to <2 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82df = fptrunc <4 x double> undef to <4 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r83df = fptrunc <8 x double> undef to <8 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r84df = fptrunc <16 x double> undef to <16 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80dh = fptrunc double undef to half
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80df = fptrunc double undef to float
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81df = fptrunc <2 x double> undef to <2 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82df = fptrunc <4 x double> undef to <4 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83df = fptrunc <8 x double> undef to <8 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84df = fptrunc <16 x double> undef to <16 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80dh = fptrunc double undef to half
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80fh = fptrunc float undef to half
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85fd = fpext float undef to double
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86fd = fpext <2 x float> undef to <2 x double>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87fd = fpext <4 x float> undef to <4 x double>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r88fd = fpext <8 x float> undef to <8 x double>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r89fd = fpext <16 x float> undef to <16 x double>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hd = fpext half undef to double
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85fd = fpext float undef to double
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86fd = fpext <2 x float> undef to <2 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87fd = fpext <4 x float> undef to <4 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88fd = fpext <8 x float> undef to <8 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89fd = fpext <16 x float> undef to <16 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85hd = fpext half undef to double
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hf = fpext half undef to float
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
@@ -2326,6 +2331,7 @@ define i32 @casts() {
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-MAIN-SIZE-LABEL: 'casts'
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r0 = sext i1 undef to i8
@@ -2442,36 +2448,36 @@ define i32 @casts() {
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rext_b = zext <2 x i32> undef to <2 x i64>
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r74 = trunc <8 x i32> undef to <8 x i8>
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r75 = trunc <16 x i32> undef to <16 x i8>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80df = fptrunc double undef to float
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81df = fptrunc <2 x double> undef to <2 x float>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82df = fptrunc <4 x double> undef to <4 x float>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r83df = fptrunc <8 x double> undef to <8 x float>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r84df = fptrunc <16 x double> undef to <16 x float>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80dh = fptrunc double undef to half
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80fh = fptrunc float undef to half
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85fd = fpext float undef to double
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86fd = fpext <2 x float> undef to <2 x double>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87fd = fpext <4 x float> undef to <4 x double>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r88fd = fpext <8 x float> undef to <8 x double>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r89fd = fpext <16 x float> undef to <16 x double>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hd = fpext half undef to double
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hf = fpext half undef to float
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80df = fptrunc double undef to float
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81df = fptrunc <2 x double> undef to <2 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82df = fptrunc <4 x double> undef to <4 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83df = fptrunc <8 x double> undef to <8 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84df = fptrunc <16 x double> undef to <16 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80dh = fptrunc double undef to half
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80fh = fptrunc float undef to half
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85fd = fpext float undef to double
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86fd = fpext <2 x float> undef to <2 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87fd = fpext <4 x float> undef to <4 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88fd = fpext <8 x float> undef to <8 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89fd = fpext <16 x float> undef to <16 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85hd = fpext half undef to double
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85hf = fpext half undef to float
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
@@ -2712,6 +2718,7 @@ define i32 @casts() {
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-BASE-SIZE-LABEL: 'casts'
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r0 = sext i1 undef to i8
@@ -2828,36 +2835,36 @@ define i32 @casts() {
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rext_b = zext <2 x i32> undef to <2 x i64>
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r74 = trunc <8 x i32> undef to <8 x i8>
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r75 = trunc <16 x i32> undef to <16 x i8>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80df = fptrunc double undef to float
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81df = fptrunc <2 x double> undef to <2 x float>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82df = fptrunc <4 x double> undef to <4 x float>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r83df = fptrunc <8 x double> undef to <8 x float>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r84df = fptrunc <16 x double> undef to <16 x float>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80dh = fptrunc double undef to half
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80fh = fptrunc float undef to half
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85fd = fpext float undef to double
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86fd = fpext <2 x float> undef to <2 x double>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87fd = fpext <4 x float> undef to <4 x double>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r88fd = fpext <8 x float> undef to <8 x double>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r89fd = fpext <16 x float> undef to <16 x double>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hd = fpext half undef to double
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hf = fpext half undef to float
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80df = fptrunc double undef to float
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81df = fptrunc <2 x double> undef to <2 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82df = fptrunc <4 x double> undef to <4 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83df = fptrunc <8 x double> undef to <8 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84df = fptrunc <16 x double> undef to <16 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80dh = fptrunc double undef to half
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80fh = fptrunc float undef to half
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85fd = fpext float undef to double
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86fd = fpext <2 x float> undef to <2 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87fd = fpext <4 x float> undef to <4 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88fd = fpext <8 x float> undef to <8 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89fd = fpext <16 x float> undef to <16 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85hd = fpext half undef to double
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85hf = fpext half undef to float
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
@@ -3098,6 +3105,7 @@ define i32 @casts() {
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8R-SIZE-LABEL: 'casts'
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r0 = sext i1 undef to i8
@@ -3219,31 +3227,31 @@ define i32 @casts() {
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82df = fptrunc <4 x double> undef to <4 x float>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r83df = fptrunc <8 x double> undef to <8 x float>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r84df = fptrunc <16 x double> undef to <16 x float>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80dh = fptrunc double undef to half
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r80fh = fptrunc float undef to half
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80dh = fptrunc double undef to half
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81dh = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82dh = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83dh = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84dh = fptrunc <16 x double> undef to <16 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r80fh = fptrunc float undef to half
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r81fh = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r82fh = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r83fh = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r84fh = fptrunc <16 x float> undef to <16 x half>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85fd = fpext float undef to double
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86fd = fpext <2 x float> undef to <2 x double>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87fd = fpext <4 x float> undef to <4 x double>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r88fd = fpext <8 x float> undef to <8 x double>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r89fd = fpext <16 x float> undef to <16 x double>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hd = fpext half undef to double
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r85hf = fpext half undef to float
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85hd = fpext half undef to double
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86hd = fpext <2 x half> undef to <2 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87hd = fpext <4 x half> undef to <4 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88hd = fpext <8 x half> undef to <8 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89hd = fpext <16 x half> undef to <16 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r85hf = fpext half undef to float
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r86hf = fpext <2 x half> undef to <2 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
@@ -3484,6 +3492,7 @@ define i32 @casts() {
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %r0 = sext i1 undef to i8
   %r1 = zext i1 undef to i8
@@ -3916,1722 +3925,6 @@ define i32 @casts() {
   ret i32 undef
 }
 
-
-define i32 @load_extends() {
-; CHECK-NEON-RECIP-LABEL: 'load_extends'
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
-;
-; CHECK-MVE-RECIP-LABEL: 'load_extends'
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1322 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 298 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
-;
-; CHECK-V8M-MAIN-RECIP-LABEL: 'load_extends'
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
-;
-; CHECK-V8M-BASE-RECIP-LABEL: 'load_extends'
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
-;
-; CHECK-V8R-RECIP-LABEL: 'load_extends'
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
-;
-; CHECK-MVE-SIZE-LABEL: 'load_extends'
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
-;
-; CHECK-V8M-MAIN-SIZE-LABEL: 'load_extends'
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
-;
-; CHECK-V8M-BASE-SIZE-LABEL: 'load_extends'
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
-;
-; CHECK-V8R-SIZE-LABEL: 'load_extends'
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
-;
-  %loadi8 = load i8, i8* undef
-  %loadi16 = load i16, i16* undef
-  %loadi32 = load i32, i32* undef
-  %loadv2i8 = load <2 x i8>, <2 x i8>* undef
-  %loadv4i8 = load <4 x i8>, <4 x i8>* undef
-  %loadv8i8 = load <8 x i8>, <8 x i8>* undef
-  %loadv16i8 = load <16 x i8>, <16 x i8>* undef
-  %loadv2i16 = load <2 x i16>, <2 x i16>* undef
-  %loadv4i16 = load <4 x i16>, <4 x i16>* undef
-  %loadv8i16 = load <8 x i16>, <8 x i16>* undef
-  %loadv2i32 = load <2 x i32>, <2 x i32>* undef
-  %loadv4i32 = load <4 x i32>, <4 x i32>* undef
-
-  %r0 = sext i8 %loadi8 to i16
-  %r1 = zext i8 %loadi8 to i16
-  %r2 = sext i8 %loadi8 to i32
-  %r3 = zext i8 %loadi8 to i32
-  %r4 = sext i8 %loadi8 to i64
-  %r5 = zext i8 %loadi8 to i64
-  %r6 = sext i16 %loadi16 to i32
-  %r7 = zext i16 %loadi16 to i32
-  %r8 = sext i16 %loadi16 to i64
-  %r9 = zext i16 %loadi16 to i64
-  %r10 = sext i32 %loadi32 to i64
-  %r11 = zext i32 %loadi32 to i64
-
-  %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
-  %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
-  %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
-  %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
-  %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
-  %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
-  %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
-  %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
-  %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
-  %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
-  %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
-  %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
-  %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
-  %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
-  %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
-  %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
-  %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
-  %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
-  %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
-  %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
-  %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
-  %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
-  %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
-  %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
-
-  %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
-  %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
-  %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
-  %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
-  %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
-  %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
-  %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
-  %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
-  %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
-  %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
-  %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
-  %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
-
-  %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
-  %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
-  %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
-  %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
-
-  ret i32 undef
-}
-
-define i32 @store_trunc() {
-; CHECK-NEON-RECIP-LABEL: 'store_trunc'
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
-;
-; CHECK-MVE-RECIP-LABEL: 'store_trunc'
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
-;
-; CHECK-V8M-MAIN-RECIP-LABEL: 'store_trunc'
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
-;
-; CHECK-V8M-BASE-RECIP-LABEL: 'store_trunc'
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
-;
-; CHECK-V8R-RECIP-LABEL: 'store_trunc'
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
-;
-; CHECK-MVE-SIZE-LABEL: 'store_trunc'
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
-;
-; CHECK-V8M-MAIN-SIZE-LABEL: 'store_trunc'
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
-;
-; CHECK-V8M-BASE-SIZE-LABEL: 'store_trunc'
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
-;
-; CHECK-V8R-SIZE-LABEL: 'store_trunc'
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
-;
-  %i816 = trunc i16 undef to i8
-  %i832 = trunc i32 undef to i8
-  %i864 = trunc i64 undef to i8
-  %i1632 = trunc i32 undef to i16
-  %i1664 = trunc i64 undef to i16
-  %i3264 = trunc i64 undef to i32
-
-  %v2816 = trunc <2 x i16> undef to <2 x i8>
-  %v2832 = trunc <2 x i32> undef to <2 x i8>
-  %v2864 = trunc <2 x i64> undef to <2 x i8>
-  %v4816 = trunc <4 x i16> undef to <4 x i8>
-  %v4832 = trunc <4 x i32> undef to <4 x i8>
-  %v4864 = trunc <4 x i64> undef to <4 x i8>
-  %v8816 = trunc <8 x i16> undef to <8 x i8>
-  %v8832 = trunc <8 x i32> undef to <8 x i8>
-  %v8864 = trunc <8 x i64> undef to <8 x i8>
-  %v16816 = trunc <16 x i16> undef to <16 x i8>
-  %v16832 = trunc <16 x i32> undef to <16 x i8>
-  %v16864 = trunc <16 x i64> undef to <16 x i8>
-
-  %v21632 = trunc <2 x i32> undef to <2 x i16>
-  %v21664 = trunc <2 x i64> undef to <2 x i16>
-  %v41632 = trunc <4 x i32> undef to <4 x i16>
-  %v41664 = trunc <4 x i64> undef to <4 x i16>
-  %v81632 = trunc <8 x i32> undef to <8 x i16>
-  %v81664 = trunc <8 x i64> undef to <8 x i16>
-
-  %v23264 = trunc <2 x i64> undef to <2 x i32>
-  %v43264 = trunc <4 x i64> undef to <4 x i32>
-
-  store i8 %i816, i8* undef
-  store i8 %i832, i8* undef
-  store i8 %i864, i8* undef
-  store i16 %i1632, i16* undef
-  store i16 %i1664, i16* undef
-  store i32 %i3264, i32* undef
-  store <2 x i8> %v2816, <2 x i8>* undef
-  store <2 x i8> %v2832, <2 x i8>* undef
-  store <2 x i8> %v2864, <2 x i8>* undef
-  store <4 x i8> %v4816, <4 x i8>* undef
-  store <4 x i8> %v4832, <4 x i8>* undef
-  store <4 x i8> %v4864, <4 x i8>* undef
-  store <8 x i8> %v8816, <8 x i8>* undef
-  store <8 x i8> %v8832, <8 x i8>* undef
-  store <8 x i8> %v8864, <8 x i8>* undef
-  store <16 x i8> %v16816, <16 x i8>* undef
-  store <16 x i8> %v16832, <16 x i8>* undef
-  store <16 x i8> %v16864, <16 x i8>* undef
-  store <2 x i16> %v21632, <2 x i16>* undef
-  store <2 x i16> %v21664, <2 x i16>* undef
-  store <4 x i16> %v41632, <4 x i16>* undef
-  store <4 x i16> %v41664, <4 x i16>* undef
-  store <8 x i16> %v81632, <8 x i16>* undef
-  store <8 x i16> %v81664, <8 x i16>* undef
-  store <2 x i32> %v23264, <2 x i32>* undef
-  store <4 x i32> %v43264, <4 x i32>* undef
-
-  ret i32 undef
-}
-
-define i32 @load_fpextends() {
-; CHECK-NEON-RECIP-LABEL: 'load_fpextends'
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r1 = fpext half %loadf16 to float
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fpext half %loadf16 to double
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r3 = fpext float %loadf32 to double
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
-;
-; CHECK-MVE-RECIP-LABEL: 'load_fpextends'
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r1 = fpext half %loadf16 to float
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fpext half %loadf16 to double
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r3 = fpext float %loadf32 to double
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1320 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 328 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
-;
-; CHECK-V8M-MAIN-RECIP-LABEL: 'load_fpextends'
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r1 = fpext half %loadf16 to float
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fpext half %loadf16 to double
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r3 = fpext float %loadf32 to double
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
-;
-; CHECK-V8M-BASE-RECIP-LABEL: 'load_fpextends'
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r1 = fpext half %loadf16 to float
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fpext half %loadf16 to double
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r3 = fpext float %loadf32 to double
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
-;
-; CHECK-V8R-RECIP-LABEL: 'load_fpextends'
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r1 = fpext half %loadf16 to float
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fpext half %loadf16 to double
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r3 = fpext float %loadf32 to double
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
-;
-; CHECK-MVE-SIZE-LABEL: 'load_fpextends'
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r1 = fpext half %loadf16 to float
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fpext half %loadf16 to double
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r3 = fpext float %loadf32 to double
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
-;
-; CHECK-V8M-MAIN-SIZE-LABEL: 'load_fpextends'
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r1 = fpext half %loadf16 to float
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fpext half %loadf16 to double
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r3 = fpext float %loadf32 to double
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
-;
-; CHECK-V8M-BASE-SIZE-LABEL: 'load_fpextends'
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r1 = fpext half %loadf16 to float
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fpext half %loadf16 to double
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r3 = fpext float %loadf32 to double
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
-;
-; CHECK-V8R-SIZE-LABEL: 'load_fpextends'
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r1 = fpext half %loadf16 to float
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fpext half %loadf16 to double
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r3 = fpext float %loadf32 to double
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
-;
-  %loadf16 = load half, half* undef
-  %loadf32 = load float, float* undef
-  %loadv2f16 = load <2 x half>, <2 x half>* undef
-  %loadv4f16 = load <4 x half>, <4 x half>* undef
-  %loadv8f16 = load <8 x half>, <8 x half>* undef
-  %loadv16f16 = load <16 x half>, <16 x half>* undef
-  %loadv2f32 = load <2 x float>, <2 x float>* undef
-  %loadv4f32 = load <4 x float>, <4 x float>* undef
-  %loadv8f32 = load <8 x float>, <8 x float>* undef
-
-  %r1 = fpext half %loadf16 to float
-  %r2 = fpext half %loadf16 to double
-  %r3 = fpext float %loadf32 to double
-
-  %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
-  %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
-  %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
-  %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
-  %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
-  %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
-  %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
-  %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
-  %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
-  %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
-  %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
-
-  ret i32 undef
-}
-
-define i32 @load_fptrunc() {
-; CHECK-NEON-RECIP-LABEL: 'load_fptrunc'
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1632 = fptrunc float undef to half
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1664 = fptrunc double undef to half
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i3264 = fptrunc double undef to float
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
-;
-; CHECK-MVE-RECIP-LABEL: 'load_fptrunc'
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1632 = fptrunc float undef to half
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1664 = fptrunc double undef to half
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i3264 = fptrunc double undef to float
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
-;
-; CHECK-V8M-MAIN-RECIP-LABEL: 'load_fptrunc'
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1632 = fptrunc float undef to half
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1664 = fptrunc double undef to half
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i3264 = fptrunc double undef to float
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
-;
-; CHECK-V8M-BASE-RECIP-LABEL: 'load_fptrunc'
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1632 = fptrunc float undef to half
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1664 = fptrunc double undef to half
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i3264 = fptrunc double undef to float
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
-;
-; CHECK-V8R-RECIP-LABEL: 'load_fptrunc'
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1632 = fptrunc float undef to half
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1664 = fptrunc double undef to half
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i3264 = fptrunc double undef to float
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
-;
-; CHECK-MVE-SIZE-LABEL: 'load_fptrunc'
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1632 = fptrunc float undef to half
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1664 = fptrunc double undef to half
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i3264 = fptrunc double undef to float
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
-;
-; CHECK-V8M-MAIN-SIZE-LABEL: 'load_fptrunc'
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1632 = fptrunc float undef to half
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1664 = fptrunc double undef to half
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i3264 = fptrunc double undef to float
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
-;
-; CHECK-V8M-BASE-SIZE-LABEL: 'load_fptrunc'
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1632 = fptrunc float undef to half
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1664 = fptrunc double undef to half
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i3264 = fptrunc double undef to float
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
-;
-; CHECK-V8R-SIZE-LABEL: 'load_fptrunc'
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1632 = fptrunc float undef to half
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1664 = fptrunc double undef to half
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i3264 = fptrunc double undef to float
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
-;
-  %i1632 = fptrunc float undef to half
-  %i1664 = fptrunc double undef to half
-  %i3264 = fptrunc double undef to float
-
-  %v21632 = fptrunc <2 x float> undef to <2 x half>
-  %v21664 = fptrunc <2 x double> undef to <2 x half>
-  %v41632 = fptrunc <4 x float> undef to <4 x half>
-  %v41664 = fptrunc <4 x double> undef to <4 x half>
-  %v81632 = fptrunc <8 x float> undef to <8 x half>
-  %v81664 = fptrunc <8 x double> undef to <8 x half>
-
-  %v23264 = fptrunc <2 x double> undef to <2 x float>
-  %v43264 = fptrunc <4 x double> undef to <4 x float>
-
-  store half %i1632, half* undef
-  store half %i1664, half* undef
-  store float %i3264, float* undef
-  store <2 x half> %v21632, <2 x half>* undef
-  store <2 x half> %v21664, <2 x half>* undef
-  store <4 x half> %v41632, <4 x half>* undef
-  store <4 x half> %v41664, <4 x half>* undef
-  store <8 x half> %v81632, <8 x half>* undef
-  store <8 x half> %v81664, <8 x half>* undef
-  store <2 x float> %v23264, <2 x float>* undef
-  store <4 x float> %v43264, <4 x float>* undef
-
-  ret i32 undef
-}
-
 define i32 @bitcasts() {
 ; CHECK-NEON-RECIP-LABEL: 'bitcasts'
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = bitcast i32 undef to i32
@@ -5642,6 +3935,7 @@ define i32 @bitcasts() {
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = bitcast double undef to i64
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = bitcast half undef to i16
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = bitcast i16 undef to half
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-MVE-RECIP-LABEL: 'bitcasts'
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = bitcast i32 undef to i32
@@ -5652,6 +3946,7 @@ define i32 @bitcasts() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = bitcast double undef to i64
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = bitcast half undef to i16
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = bitcast i16 undef to half
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-MAIN-RECIP-LABEL: 'bitcasts'
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = bitcast i32 undef to i32
@@ -5662,6 +3957,7 @@ define i32 @bitcasts() {
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f = bitcast double undef to i64
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = bitcast half undef to i16
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = bitcast i16 undef to half
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-BASE-RECIP-LABEL: 'bitcasts'
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = bitcast i32 undef to i32
@@ -5672,6 +3968,7 @@ define i32 @bitcasts() {
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f = bitcast double undef to i64
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = bitcast half undef to i16
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = bitcast i16 undef to half
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8R-RECIP-LABEL: 'bitcasts'
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = bitcast i32 undef to i32
@@ -5682,6 +3979,7 @@ define i32 @bitcasts() {
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = bitcast double undef to i64
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = bitcast half undef to i16
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = bitcast i16 undef to half
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-MVE-SIZE-LABEL: 'bitcasts'
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = bitcast i32 undef to i32
@@ -5692,6 +3990,7 @@ define i32 @bitcasts() {
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = bitcast double undef to i64
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = bitcast half undef to i16
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = bitcast i16 undef to half
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-MAIN-SIZE-LABEL: 'bitcasts'
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = bitcast i32 undef to i32
@@ -5702,6 +4001,7 @@ define i32 @bitcasts() {
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = bitcast double undef to i64
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = bitcast half undef to i16
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = bitcast i16 undef to half
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-BASE-SIZE-LABEL: 'bitcasts'
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = bitcast i32 undef to i32
@@ -5712,6 +4012,7 @@ define i32 @bitcasts() {
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = bitcast double undef to i64
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = bitcast half undef to i16
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = bitcast i16 undef to half
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8R-SIZE-LABEL: 'bitcasts'
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = bitcast i32 undef to i32
@@ -5722,6 +4023,7 @@ define i32 @bitcasts() {
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = bitcast double undef to i64
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = bitcast half undef to i16
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = bitcast i16 undef to half
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %a = bitcast i32 undef to i32
   %b = bitcast float undef to float
diff --git a/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll b/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll
new file mode 100644
index 0000000000000..491e0900e08a2
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll
@@ -0,0 +1,3212 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s  -cost-model -analyze -mtriple=thumbv7-apple-ios6.0.0 -mcpu=cortex-a9 | FileCheck %s --check-prefix=CHECK-NEON-RECIP
+; RUN: opt -cost-model -analyze -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE-RECIP
+; RUN: opt -cost-model -analyze -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN-RECIP
+; RUN: opt -cost-model -analyze -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE-RECIP
+; RUN: opt -cost-model -analyze -mtriple=armv8r-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8R-RECIP
+; RUN: opt -cost-model -analyze -cost-kind=code-size -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE-SIZE
+; RUN: opt -cost-model -analyze -cost-kind=code-size -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN-SIZE
+; RUN: opt -cost-model -analyze -cost-kind=code-size -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE-SIZE
+; RUN: opt -cost-model -analyze -cost-kind=code-size -mtriple=armv8r-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8R-SIZE
+
+define i32 @load_extends() {
+; CHECK-NEON-RECIP-LABEL: 'load_extends'
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-MVE-RECIP-LABEL: 'load_extends'
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1322 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 298 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-MAIN-RECIP-LABEL: 'load_extends'
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-BASE-RECIP-LABEL: 'load_extends'
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8R-RECIP-LABEL: 'load_extends'
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-MVE-SIZE-LABEL: 'load_extends'
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-MAIN-SIZE-LABEL: 'load_extends'
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-BASE-SIZE-LABEL: 'load_extends'
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8R-SIZE-LABEL: 'load_extends'
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = sext i8 %loadi8 to i32
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = zext i8 %loadi8 to i32
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r4 = sext i8 %loadi8 to i64
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r5 = zext i8 %loadi8 to i64
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r6 = sext i16 %loadi16 to i32
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r7 = zext i16 %loadi16 to i32
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r8 = sext i16 %loadi16 to i64
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r9 = zext i16 %loadi16 to i64
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r10 = sext i32 %loadi32 to i64
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r11 = zext i32 %loadi32 to i64
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %loadi8 = load i8, i8* undef
+  %loadi16 = load i16, i16* undef
+  %loadi32 = load i32, i32* undef
+  %loadv2i8 = load <2 x i8>, <2 x i8>* undef
+  %loadv4i8 = load <4 x i8>, <4 x i8>* undef
+  %loadv8i8 = load <8 x i8>, <8 x i8>* undef
+  %loadv16i8 = load <16 x i8>, <16 x i8>* undef
+  %loadv2i16 = load <2 x i16>, <2 x i16>* undef
+  %loadv4i16 = load <4 x i16>, <4 x i16>* undef
+  %loadv8i16 = load <8 x i16>, <8 x i16>* undef
+  %loadv2i32 = load <2 x i32>, <2 x i32>* undef
+  %loadv4i32 = load <4 x i32>, <4 x i32>* undef
+
+  %r0 = sext i8 %loadi8 to i16
+  %r1 = zext i8 %loadi8 to i16
+  %r2 = sext i8 %loadi8 to i32
+  %r3 = zext i8 %loadi8 to i32
+  %r4 = sext i8 %loadi8 to i64
+  %r5 = zext i8 %loadi8 to i64
+  %r6 = sext i16 %loadi16 to i32
+  %r7 = zext i16 %loadi16 to i32
+  %r8 = sext i16 %loadi16 to i64
+  %r9 = zext i16 %loadi16 to i64
+  %r10 = sext i32 %loadi32 to i64
+  %r11 = zext i32 %loadi32 to i64
+
+  %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+  %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+  %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+  %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+  %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+  %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+  %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+  %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+  %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+  %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+  %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+  %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+  %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+  %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+  %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+  %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+  %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+  %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+  %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+  %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+  %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+  %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+  %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+  %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+
+  %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+  %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+  %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+  %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+  %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+  %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+  %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+  %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+  %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+  %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+  %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+  %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+
+  %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+  %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+  %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+  %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+
+  ret i32 undef
+}
+
+define i32 @store_trunc() {
+; CHECK-NEON-RECIP-LABEL: 'store_trunc'
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-MVE-RECIP-LABEL: 'store_trunc'
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-MAIN-RECIP-LABEL: 'store_trunc'
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-BASE-RECIP-LABEL: 'store_trunc'
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8R-RECIP-LABEL: 'store_trunc'
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-MVE-SIZE-LABEL: 'store_trunc'
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-MAIN-SIZE-LABEL: 'store_trunc'
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-BASE-SIZE-LABEL: 'store_trunc'
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8R-SIZE-LABEL: 'store_trunc'
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i816 = trunc i16 undef to i8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i832 = trunc i32 undef to i8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i864 = trunc i64 undef to i8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1632 = trunc i32 undef to i16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i1664 = trunc i64 undef to i16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i3264 = trunc i64 undef to i32
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i816, i8* undef, align 1
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i832, i8* undef, align 1
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %i864, i8* undef, align 1
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8816, <8 x i8>* undef, align 8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8832, <8 x i8>* undef, align 8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %v8864, <8 x i8>* undef, align 8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %i816 = trunc i16 undef to i8
+  %i832 = trunc i32 undef to i8
+  %i864 = trunc i64 undef to i8
+  %i1632 = trunc i32 undef to i16
+  %i1664 = trunc i64 undef to i16
+  %i3264 = trunc i64 undef to i32
+
+  %v2816 = trunc <2 x i16> undef to <2 x i8>
+  %v2832 = trunc <2 x i32> undef to <2 x i8>
+  %v2864 = trunc <2 x i64> undef to <2 x i8>
+  %v4816 = trunc <4 x i16> undef to <4 x i8>
+  %v4832 = trunc <4 x i32> undef to <4 x i8>
+  %v4864 = trunc <4 x i64> undef to <4 x i8>
+  %v8816 = trunc <8 x i16> undef to <8 x i8>
+  %v8832 = trunc <8 x i32> undef to <8 x i8>
+  %v8864 = trunc <8 x i64> undef to <8 x i8>
+  %v16816 = trunc <16 x i16> undef to <16 x i8>
+  %v16832 = trunc <16 x i32> undef to <16 x i8>
+  %v16864 = trunc <16 x i64> undef to <16 x i8>
+
+  %v21632 = trunc <2 x i32> undef to <2 x i16>
+  %v21664 = trunc <2 x i64> undef to <2 x i16>
+  %v41632 = trunc <4 x i32> undef to <4 x i16>
+  %v41664 = trunc <4 x i64> undef to <4 x i16>
+  %v81632 = trunc <8 x i32> undef to <8 x i16>
+  %v81664 = trunc <8 x i64> undef to <8 x i16>
+
+  %v23264 = trunc <2 x i64> undef to <2 x i32>
+  %v43264 = trunc <4 x i64> undef to <4 x i32>
+
+  store i8 %i816, i8* undef
+  store i8 %i832, i8* undef
+  store i8 %i864, i8* undef
+  store i16 %i1632, i16* undef
+  store i16 %i1664, i16* undef
+  store i32 %i3264, i32* undef
+  store <2 x i8> %v2816, <2 x i8>* undef
+  store <2 x i8> %v2832, <2 x i8>* undef
+  store <2 x i8> %v2864, <2 x i8>* undef
+  store <4 x i8> %v4816, <4 x i8>* undef
+  store <4 x i8> %v4832, <4 x i8>* undef
+  store <4 x i8> %v4864, <4 x i8>* undef
+  store <8 x i8> %v8816, <8 x i8>* undef
+  store <8 x i8> %v8832, <8 x i8>* undef
+  store <8 x i8> %v8864, <8 x i8>* undef
+  store <16 x i8> %v16816, <16 x i8>* undef
+  store <16 x i8> %v16832, <16 x i8>* undef
+  store <16 x i8> %v16864, <16 x i8>* undef
+  store <2 x i16> %v21632, <2 x i16>* undef
+  store <2 x i16> %v21664, <2 x i16>* undef
+  store <4 x i16> %v41632, <4 x i16>* undef
+  store <4 x i16> %v41664, <4 x i16>* undef
+  store <8 x i16> %v81632, <8 x i16>* undef
+  store <8 x i16> %v81664, <8 x i16>* undef
+  store <2 x i32> %v23264, <2 x i32>* undef
+  store <4 x i32> %v43264, <4 x i32>* undef
+
+  ret i32 undef
+}
+
+define i32 @load_fpextends() {
+; CHECK-NEON-RECIP-LABEL: 'load_fpextends'
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r1 = fpext half %loadf16 to float
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r2 = fpext half %loadf16 to double
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r3 = fpext float %loadf32 to double
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-MVE-RECIP-LABEL: 'load_fpextends'
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r1 = fpext half %loadf16 to float
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r2 = fpext half %loadf16 to double
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r3 = fpext float %loadf32 to double
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-MAIN-RECIP-LABEL: 'load_fpextends'
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r1 = fpext half %loadf16 to float
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r2 = fpext half %loadf16 to double
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r3 = fpext float %loadf32 to double
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-BASE-RECIP-LABEL: 'load_fpextends'
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r1 = fpext half %loadf16 to float
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r2 = fpext half %loadf16 to double
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r3 = fpext float %loadf32 to double
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8R-RECIP-LABEL: 'load_fpextends'
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r1 = fpext half %loadf16 to float
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r2 = fpext half %loadf16 to double
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r3 = fpext float %loadf32 to double
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-MVE-SIZE-LABEL: 'load_fpextends'
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r1 = fpext half %loadf16 to float
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r2 = fpext half %loadf16 to double
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r3 = fpext float %loadf32 to double
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-MAIN-SIZE-LABEL: 'load_fpextends'
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r1 = fpext half %loadf16 to float
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r2 = fpext half %loadf16 to double
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r3 = fpext float %loadf32 to double
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-BASE-SIZE-LABEL: 'load_fpextends'
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r1 = fpext half %loadf16 to float
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r2 = fpext half %loadf16 to double
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r3 = fpext float %loadf32 to double
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8R-SIZE-LABEL: 'load_fpextends'
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r1 = fpext half %loadf16 to float
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r2 = fpext half %loadf16 to double
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r3 = fpext float %loadf32 to double
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %loadf16 = load half, half* undef
+  %loadf32 = load float, float* undef
+  %loadv2f16 = load <2 x half>, <2 x half>* undef
+  %loadv4f16 = load <4 x half>, <4 x half>* undef
+  %loadv8f16 = load <8 x half>, <8 x half>* undef
+  %loadv16f16 = load <16 x half>, <16 x half>* undef
+  %loadv2f32 = load <2 x float>, <2 x float>* undef
+  %loadv4f32 = load <4 x float>, <4 x float>* undef
+  %loadv8f32 = load <8 x float>, <8 x float>* undef
+
+  %r1 = fpext half %loadf16 to float
+  %r2 = fpext half %loadf16 to double
+  %r3 = fpext float %loadf32 to double
+
+  %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+  %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+  %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+  %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+  %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+  %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+  %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+  %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+  %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+  %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+  %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+
+  %loadv4f16ou = load <4 x half>, <4 x half>* undef
+  %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+
+  ret i32 undef
+}
+
+define i32 @load_fptrunc() {
+; CHECK-NEON-RECIP-LABEL: 'load_fptrunc'
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i1632 = fptrunc float undef to half
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i1664 = fptrunc double undef to half
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i3264 = fptrunc double undef to float
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-MVE-RECIP-LABEL: 'load_fptrunc'
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1632 = fptrunc float undef to half
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i1664 = fptrunc double undef to half
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i3264 = fptrunc double undef to float
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-MAIN-RECIP-LABEL: 'load_fptrunc'
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i1632 = fptrunc float undef to half
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i1664 = fptrunc double undef to half
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i3264 = fptrunc double undef to float
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-BASE-RECIP-LABEL: 'load_fptrunc'
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i1632 = fptrunc float undef to half
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i1664 = fptrunc double undef to half
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i3264 = fptrunc double undef to float
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8R-RECIP-LABEL: 'load_fptrunc'
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i1632 = fptrunc float undef to half
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i1664 = fptrunc double undef to half
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i3264 = fptrunc double undef to float
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-MVE-SIZE-LABEL: 'load_fptrunc'
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i1632 = fptrunc float undef to half
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i1664 = fptrunc double undef to half
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i3264 = fptrunc double undef to float
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-MAIN-SIZE-LABEL: 'load_fptrunc'
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i1632 = fptrunc float undef to half
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i1664 = fptrunc double undef to half
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i3264 = fptrunc double undef to float
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-BASE-SIZE-LABEL: 'load_fptrunc'
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i1632 = fptrunc float undef to half
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i1664 = fptrunc double undef to half
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i3264 = fptrunc double undef to float
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8R-SIZE-LABEL: 'load_fptrunc'
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i1632 = fptrunc float undef to half
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %i1664 = fptrunc double undef to half
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i3264 = fptrunc double undef to float
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %i1632 = fptrunc float undef to half
+  %i1664 = fptrunc double undef to half
+  %i3264 = fptrunc double undef to float
+
+  %v21632 = fptrunc <2 x float> undef to <2 x half>
+  %v21664 = fptrunc <2 x double> undef to <2 x half>
+  %v41632 = fptrunc <4 x float> undef to <4 x half>
+  %v41664 = fptrunc <4 x double> undef to <4 x half>
+  %v81632 = fptrunc <8 x float> undef to <8 x half>
+  %v81664 = fptrunc <8 x double> undef to <8 x half>
+
+  %v23264 = fptrunc <2 x double> undef to <2 x float>
+  %v43264 = fptrunc <4 x double> undef to <4 x float>
+
+  store half %i1632, half* undef
+  store half %i1664, half* undef
+  store float %i3264, float* undef
+  store <2 x half> %v21632, <2 x half>* undef
+  store <2 x half> %v21664, <2 x half>* undef
+  store <4 x half> %v41632, <4 x half>* undef
+  store <4 x half> %v41664, <4 x half>* undef
+  store <8 x half> %v81632, <8 x half>* undef
+  store <8 x half> %v81664, <8 x half>* undef
+  store <2 x float> %v23264, <2 x float>* undef
+  store <4 x float> %v43264, <4 x float>* undef
+
+  ret i32 undef
+}
+
+define i32 @maskedload_extends() {
+; CHECK-NEON-RECIP-LABEL: 'maskedload_extends'
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-MVE-RECIP-LABEL: 'maskedload_extends'
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1322 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 298 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-MAIN-RECIP-LABEL: 'maskedload_extends'
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-BASE-RECIP-LABEL: 'maskedload_extends'
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8R-RECIP-LABEL: 'maskedload_extends'
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-MVE-SIZE-LABEL: 'maskedload_extends'
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-MAIN-SIZE-LABEL: 'maskedload_extends'
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-BASE-SIZE-LABEL: 'maskedload_extends'
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8R-SIZE-LABEL: 'maskedload_extends'
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %loadv2i8 = call <2 x i8>  @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+  %loadv4i8 = call <4 x i8>  @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+  %loadv8i8 = call <8 x i8>  @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+  %loadv16i8 = call <16 x i8>  @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+  %loadv2i16 = call <2 x i16>  @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+  %loadv4i16 = call <4 x i16>  @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+  %loadv8i16 = call <8 x i16>  @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+  %loadv2i32 = call <2 x i32>  @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+  %loadv4i32 = call <4 x i32>  @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+
+  %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
+  %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
+  %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
+  %v2832u = zext <2 x i8> %loadv2i8 to <2 x i32>
+  %v2864s = sext <2 x i8> %loadv2i8 to <2 x i64>
+  %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
+  %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
+  %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
+  %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+  %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+  %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+  %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+  %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+  %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+  %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+  %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+  %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+  %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+  %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+  %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+  %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+  %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+  %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+  %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+
+  %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
+  %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
+  %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+  %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
+  %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+  %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+  %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+  %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+  %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+  %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+  %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+  %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+
+  %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
+  %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
+  %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+  %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+
+  ret i32 undef
+}
+
+define i32 @maskedstore_trunc() {
+; CHECK-NEON-RECIP-LABEL: 'maskedstore_trunc'
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-MVE-RECIP-LABEL: 'maskedstore_trunc'
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-MAIN-RECIP-LABEL: 'maskedstore_trunc'
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-BASE-RECIP-LABEL: 'maskedstore_trunc'
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8R-RECIP-LABEL: 'maskedstore_trunc'
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-MVE-SIZE-LABEL: 'maskedstore_trunc'
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-MAIN-SIZE-LABEL: 'maskedstore_trunc'
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-BASE-SIZE-LABEL: 'maskedstore_trunc'
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8R-SIZE-LABEL: 'maskedstore_trunc'
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2816 = trunc <2 x i16> undef to <2 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2832 = trunc <2 x i32> undef to <2 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864 = trunc <2 x i64> undef to <2 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4816 = trunc <4 x i16> undef to <4 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832 = trunc <4 x i32> undef to <4 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864 = trunc <4 x i64> undef to <4 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816 = trunc <8 x i16> undef to <8 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632 = trunc <4 x i32> undef to <4 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664 = trunc <4 x i64> undef to <4 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632 = trunc <8 x i32> undef to <8 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %v2816 = trunc <2 x i16> undef to <2 x i8>
+  %v2832 = trunc <2 x i32> undef to <2 x i8>
+  %v2864 = trunc <2 x i64> undef to <2 x i8>
+  %v4816 = trunc <4 x i16> undef to <4 x i8>
+  %v4832 = trunc <4 x i32> undef to <4 x i8>
+  %v4864 = trunc <4 x i64> undef to <4 x i8>
+  %v8816 = trunc <8 x i16> undef to <8 x i8>
+  %v8832 = trunc <8 x i32> undef to <8 x i8>
+  %v8864 = trunc <8 x i64> undef to <8 x i8>
+  %v16816 = trunc <16 x i16> undef to <16 x i8>
+  %v16832 = trunc <16 x i32> undef to <16 x i8>
+  %v16864 = trunc <16 x i64> undef to <16 x i8>
+
+  %v21632 = trunc <2 x i32> undef to <2 x i16>
+  %v21664 = trunc <2 x i64> undef to <2 x i16>
+  %v41632 = trunc <4 x i32> undef to <4 x i16>
+  %v41664 = trunc <4 x i64> undef to <4 x i16>
+  %v81632 = trunc <8 x i32> undef to <8 x i16>
+  %v81664 = trunc <8 x i64> undef to <8 x i16>
+
+  %v23264 = trunc <2 x i64> undef to <2 x i32>
+  %v43264 = trunc <4 x i64> undef to <4 x i32>
+
+  call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+  call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+  call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+  call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+  call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+  call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+
+  ret i32 undef
+}
+
+define i32 @maskedload_fpextends() {
+; CHECK-NEON-RECIP-LABEL: 'maskedload_fpextends'
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-MVE-RECIP-LABEL: 'maskedload_fpextends'
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-MAIN-RECIP-LABEL: 'maskedload_fpextends'
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-BASE-RECIP-LABEL: 'maskedload_fpextends'
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8R-RECIP-LABEL: 'maskedload_fpextends'
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-MVE-SIZE-LABEL: 'maskedload_fpextends'
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-MAIN-SIZE-LABEL: 'maskedload_fpextends'
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-BASE-SIZE-LABEL: 'maskedload_fpextends'
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8R-SIZE-LABEL: 'maskedload_fpextends'
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16ou = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %loadv2f16 = call <2 x half>  @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+  %loadv4f16 = call <4 x half>  @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+  %loadv8f16 = call <8 x half>  @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+  %loadv16f16 = call <16 x half>  @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+  %loadv2f32 = call <2 x float>  @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+  %loadv4f32 = call <4 x float>  @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+  %loadv8f32 = call <8 x float>  @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+
+  %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
+  %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+  %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+  %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+  %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
+  %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
+  %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
+  %v8 = fpext <16 x half> %loadv16f16 to <16 x double>
+  %v9 = fpext <2 x float> %loadv2f32 to <2 x double>
+  %v10 = fpext <4 x float> %loadv4f32 to <4 x double>
+  %v11 = fpext <8 x float> %loadv8f32 to <8 x double>
+
+  %loadv4f16ou = load <4 x half>, <4 x half>* undef
+  %v2ou = fpext <4 x half> %loadv4f16ou to <4 x float>
+
+  ret i32 undef
+}
+
+define i32 @maskedload_fptrunc() {
+; CHECK-NEON-RECIP-LABEL: 'maskedload_fptrunc'
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-MVE-RECIP-LABEL: 'maskedload_fptrunc'
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-MAIN-RECIP-LABEL: 'maskedload_fptrunc'
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-BASE-RECIP-LABEL: 'maskedload_fptrunc'
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8R-RECIP-LABEL: 'maskedload_fptrunc'
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-MVE-SIZE-LABEL: 'maskedload_fptrunc'
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-MAIN-SIZE-LABEL: 'maskedload_fptrunc'
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8M-BASE-SIZE-LABEL: 'maskedload_fptrunc'
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; CHECK-V8R-SIZE-LABEL: 'maskedload_fptrunc'
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %v21632 = fptrunc <2 x float> undef to <2 x half>
+  %v21664 = fptrunc <2 x double> undef to <2 x half>
+  %v41632 = fptrunc <4 x float> undef to <4 x half>
+  %v41664 = fptrunc <4 x double> undef to <4 x half>
+  %v81632 = fptrunc <8 x float> undef to <8 x half>
+  %v81664 = fptrunc <8 x double> undef to <8 x half>
+
+  %v23264 = fptrunc <2 x double> undef to <2 x float>
+  %v43264 = fptrunc <4 x double> undef to <4 x float>
+
+  call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+  call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+  call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+  call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+  call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+
+  ret i32 undef
+}
+
+declare <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>*, i32, <2 x i1>, <2 x i8>)
+declare <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>*, i32, <2 x i1>, <2 x i16>)
+declare <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>*, i32, <2 x i1>, <2 x i32>)
+declare <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>*, i32, <4 x i1>, <4 x i8>)
+declare <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>*, i32, <4 x i1>, <4 x i16>)
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>)
+declare <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>*, i32, <8 x i1>, <8 x i8>)
+declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32, <8 x i1>, <8 x i16>)
+declare <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>*, i32, <8 x i1>, <8 x i32>)
+declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32, <16 x i1>, <16 x i8>)
+declare <16 x i16> @llvm.masked.load.v16i16.p0v16i16(<16 x i16>*, i32, <16 x i1>, <16 x i16>)
+declare <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>*, i32, <16 x i1>, <16 x i32>)
+declare void @llvm.masked.store.v2i8.p0v2i8(<2 x i8>, <2 x i8>*, i32 immarg, <2 x i1>)
+declare void @llvm.masked.store.v2i16.p0v2i16(<2 x i16>, <2 x i16>*, i32 immarg, <2 x i1>)
+declare void @llvm.masked.store.v2i32.p0v2i32(<2 x i32>, <2 x i32>*, i32 immarg, <2 x i1>)
+declare void @llvm.masked.store.v4i8.p0v4i8(<4 x i8>, <4 x i8>*, i32 immarg, <4 x i1>)
+declare void @llvm.masked.store.v4i16.p0v4i16(<4 x i16>, <4 x i16>*, i32 immarg, <4 x i1>)
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>)
+declare void @llvm.masked.store.v8i8.p0v8i8(<8 x i8>, <8 x i8>*, i32 immarg, <8 x i1>)
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32 immarg, <8 x i1>)
+declare void @llvm.masked.store.v8i32.p0v8i32(<8 x i32>, <8 x i32>*, i32 immarg, <8 x i1>)
+declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32 immarg, <16 x i1>)
+declare void @llvm.masked.store.v16i16.p0v16i16(<16 x i16>, <16 x i16>*, i32 immarg, <16 x i1>)
+declare void @llvm.masked.store.v16i32.p0v16i32(<16 x i32>, <16 x i32>*, i32 immarg, <16 x i1>)
+
+declare <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>*, i32, <2 x i1>, <2 x half>)
+declare <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>*, i32, <2 x i1>, <2 x float>)
+declare <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>*, i32, <4 x i1>, <4 x half>)
+declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32, <4 x i1>, <4 x float>)
+declare <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>*, i32, <8 x i1>, <8 x half>)
+declare <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>*, i32, <8 x i1>, <8 x float>)
+declare <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>*, i32, <16 x i1>, <16 x half>)
+declare <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>*, i32, <16 x i1>, <16 x float>)
+declare void @llvm.masked.store.v2f16.p0v2f16(<2 x half>, <2 x half>*, i32 immarg, <2 x i1>)
+declare void @llvm.masked.store.v2f32.p0v2f32(<2 x float>, <2 x float>*, i32 immarg, <2 x i1>)
+declare void @llvm.masked.store.v4f16.p0v4f16(<4 x half>, <4 x half>*, i32 immarg, <4 x i1>)
+declare void @llvm.masked.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32 immarg, <4 x i1>)
+declare void @llvm.masked.store.v8f16.p0v8f16(<8 x half>, <8 x half>*, i32 immarg, <8 x i1>)
+declare void @llvm.masked.store.v8f32.p0v8f32(<8 x float>, <8 x float>*, i32 immarg, <8 x i1>)
+declare void @llvm.masked.store.v16f16.p0v16f16(<16 x half>, <16 x half>*, i32 immarg, <16 x i1>)
+declare void @llvm.masked.store.v16f32.p0v16f32(<16 x float>, <16 x float>*, i32 immarg, <16 x i1>)
diff --git a/llvm/test/Analysis/CostModel/ARM/load_store.ll b/llvm/test/Analysis/CostModel/ARM/load_store.ll
index 18a094990b56b..8d346ea1c8078 100644
--- a/llvm/test/Analysis/CostModel/ARM/load_store.ll
+++ b/llvm/test/Analysis/CostModel/ARM/load_store.ll
@@ -69,16 +69,16 @@ define void @stores() {
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store i128 undef, i128* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float undef, float* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store double undef, double* undef, align 4
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> undef, <2 x i8>* undef, align 1
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> undef, <2 x i16>* undef, align 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> undef, <2 x i32>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i8> undef, <2 x i8>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i16> undef, <2 x i16>* undef, align 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i32> undef, <2 x i32>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i64> undef, <2 x i64>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> undef, <4 x i32>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i16> undef, <8 x i16>* undef, align 2
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <16 x i8> undef, <16 x i8>* undef, align 1
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x float> undef, <4 x float>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x double> undef, <4 x double>* undef, align 4
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x float> undef, <2 x float>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x float> undef, <2 x float>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x double> undef, <2 x double>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i64> undef, <2 x i64>* undef, align 1
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> undef, <4 x i32>* undef, align 1
@@ -95,8 +95,8 @@ define void @stores() {
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store i128 undef, i128* undef, align 4
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float undef, float* undef, align 4
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store double undef, double* undef, align 4
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> undef, <2 x i8>* undef, align 1
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> undef, <2 x i16>* undef, align 2
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: store <2 x i8> undef, <2 x i8>* undef, align 1
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: store <2 x i16> undef, <2 x i16>* undef, align 2
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> undef, <2 x i32>* undef, align 4
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> undef, <2 x i64>* undef, align 4
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> undef, <4 x i32>* undef, align 4
@@ -256,16 +256,16 @@ define void @loads() {
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %5 = load i128, i128* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = load float, float* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = load double, double* undef, align 4
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %8 = load <2 x i8>, <2 x i8>* undef, align 1
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %9 = load <2 x i16>, <2 x i16>* undef, align 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = load <2 x i32>, <2 x i32>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %8 = load <2 x i8>, <2 x i8>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %9 = load <2 x i16>, <2 x i16>* undef, align 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %10 = load <2 x i32>, <2 x i32>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %11 = load <2 x i64>, <2 x i64>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %12 = load <4 x i32>, <4 x i32>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %13 = load <8 x i16>, <8 x i16>* undef, align 2
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %14 = load <16 x i8>, <16 x i8>* undef, align 1
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = load <4 x float>, <4 x float>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %16 = load <4 x double>, <4 x double>* undef, align 4
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %17 = load <2 x float>, <2 x float>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %17 = load <2 x float>, <2 x float>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %18 = load <2 x double>, <2 x double>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %19 = load <2 x i64>, <2 x i64>* undef, align 1
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = load <4 x i32>, <4 x i32>* undef, align 1
diff --git a/llvm/test/Analysis/Delinearization/gcd_multiply_expr.ll b/llvm/test/Analysis/Delinearization/gcd_multiply_expr.ll
index c30a672b840be..7ebc6c39d3fac 100644
--- a/llvm/test/Analysis/Delinearization/gcd_multiply_expr.ll
+++ b/llvm/test/Analysis/Delinearization/gcd_multiply_expr.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -da -analyze -delinearize
+; RUN: opt < %s -basic-aa -da -analyze -delinearize
 ;
 ; a, b, c, d, g, h;
 ; char *f;
diff --git a/llvm/test/Analysis/Delinearization/multidim_two_accesses_different_delinearization.ll b/llvm/test/Analysis/Delinearization/multidim_two_accesses_different_delinearization.ll
index 59dfc6071e72b..6df3d58832391 100644
--- a/llvm/test/Analysis/Delinearization/multidim_two_accesses_different_delinearization.ll
+++ b/llvm/test/Analysis/Delinearization/multidim_two_accesses_different_delinearization.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -da -analyze < %s
+; RUN: opt -basic-aa -da -analyze < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Analysis/DependenceAnalysis/AA.ll b/llvm/test/Analysis/DependenceAnalysis/AA.ll
index 6f9c828798cc3..ac04378b8a85d 100644
--- a/llvm/test/Analysis/DependenceAnalysis/AA.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/AA.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>"                            \
 ; RUN: "-aa-pipeline=basic-aa,type-based-aa" 2>&1 | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -tbaa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -tbaa -da | FileCheck %s
 
 ; CHECK-LABEL: 'Dependence Analysis' for function 'test_no_noalias'
 ; CHECK: da analyze - none!
diff --git a/llvm/test/Analysis/DependenceAnalysis/Banerjee.ll b/llvm/test/Analysis/DependenceAnalysis/Banerjee.ll
index 5dda3a9030f40..06fa7ad06983f 100644
--- a/llvm/test/Analysis/DependenceAnalysis/Banerjee.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/Banerjee.ll
@@ -1,9 +1,9 @@
 ; RUN: opt < %s -disable-output -da-delinearize=false "-passes=print<da>"      \
 ; RUN: -aa-pipeline=basic-aa 2>&1 | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da -da-delinearize=false | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da -da-delinearize=false | FileCheck %s
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s -check-prefix=DELIN
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s -check-prefix=DELIN
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s -check-prefix=DELIN
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.6.0"
diff --git a/llvm/test/Analysis/DependenceAnalysis/BasePtrBug.ll b/llvm/test/Analysis/DependenceAnalysis/BasePtrBug.ll
index 694e26afa09d4..7d1e8e22b956c 100644
--- a/llvm/test/Analysis/DependenceAnalysis/BasePtrBug.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/BasePtrBug.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da  | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da  | FileCheck %s
 
 ; Test that the dependence analysis generates the correct results when using
 ; an aliased object that points to a different element in the same array.
diff --git a/llvm/test/Analysis/DependenceAnalysis/Constraints.ll b/llvm/test/Analysis/DependenceAnalysis/Constraints.ll
index 2fbee702e8e5f..d086bf37bb894 100644
--- a/llvm/test/Analysis/DependenceAnalysis/Constraints.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/Constraints.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1
-; RUN: opt < %s -analyze -basicaa -da
+; RUN: opt < %s -analyze -basic-aa -da
 ;; Check that this code doesn't abort. Test case is reduced version of lnt Polybench benchmark test case dynprog.
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Analysis/DependenceAnalysis/Coupled.ll b/llvm/test/Analysis/DependenceAnalysis/Coupled.ll
index 0480087145b5e..4e81589d3bd9c 100644
--- a/llvm/test/Analysis/DependenceAnalysis/Coupled.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/Coupled.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.6.0"
diff --git a/llvm/test/Analysis/DependenceAnalysis/DADelin.ll b/llvm/test/Analysis/DependenceAnalysis/DADelin.ll
index 5c70624068522..40054aa2187ea 100644
--- a/llvm/test/Analysis/DependenceAnalysis/DADelin.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/DADelin.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 target triple = "thumbv8m.main-arm-none-eabi"
diff --git a/llvm/test/Analysis/DependenceAnalysis/ExactRDIV.ll b/llvm/test/Analysis/DependenceAnalysis/ExactRDIV.ll
index 44b399cb191bf..40e12a784b18a 100644
--- a/llvm/test/Analysis/DependenceAnalysis/ExactRDIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/ExactRDIV.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s
 
 ; ModuleID = 'ExactRDIV.bc'
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/llvm/test/Analysis/DependenceAnalysis/ExactSIV.ll b/llvm/test/Analysis/DependenceAnalysis/ExactSIV.ll
index f3fba8258bdf5..720d4166ed1a5 100644
--- a/llvm/test/Analysis/DependenceAnalysis/ExactSIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/ExactSIV.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.6.0"
diff --git a/llvm/test/Analysis/DependenceAnalysis/GCD.ll b/llvm/test/Analysis/DependenceAnalysis/GCD.ll
index 5f734b70f5c7b..a3564b7f89553 100644
--- a/llvm/test/Analysis/DependenceAnalysis/GCD.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/GCD.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s -check-prefix=DELIN
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s -check-prefix=DELIN
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s -check-prefix=DELIN
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.6.0"
diff --git a/llvm/test/Analysis/DependenceAnalysis/Invariant.ll b/llvm/test/Analysis/DependenceAnalysis/Invariant.ll
index 6835582ac4c88..5aaa3868cf9af 100644
--- a/llvm/test/Analysis/DependenceAnalysis/Invariant.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/Invariant.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s
 
 ; Test for a bug, which caused an assert when an invalid
 ; SCEVAddRecExpr is created in addToCoefficient.
diff --git a/llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll b/llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll
index 9a3692d6127e2..e222755dd8e45 100644
--- a/llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -analyze -basicaa -da
+; RUN: opt < %s -analyze -basic-aa -da
 ; RUN: opt < %s -passes="print<da>"
 
 ; Test that the dependence analysis pass does seg-fault due to a null pointer
diff --git a/llvm/test/Analysis/DependenceAnalysis/NonAffineExpr.ll b/llvm/test/Analysis/DependenceAnalysis/NonAffineExpr.ll
index 801b3f6dd6ff8..2561df503913e 100644
--- a/llvm/test/Analysis/DependenceAnalysis/NonAffineExpr.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/NonAffineExpr.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1
-; RUN: opt < %s -analyze -basicaa -da
+; RUN: opt < %s -analyze -basic-aa -da
 ;
 ; CHECK: da analyze - consistent input [S S]!
 ; CHECK: da analyze - confused!
diff --git a/llvm/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll b/llvm/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll
index 2105023dfc4a0..d1df4ef63b542 100644
--- a/llvm/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s -check-prefix=DELIN
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s -check-prefix=DELIN
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s -check-prefix=DELIN
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.6.0"
diff --git a/llvm/test/Analysis/DependenceAnalysis/PR21585.ll b/llvm/test/Analysis/DependenceAnalysis/PR21585.ll
index ff42ba96b2343..6dd1403cd1354 100644
--- a/llvm/test/Analysis/DependenceAnalysis/PR21585.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/PR21585.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>"                            \
 ; RUN: "-aa-pipeline=basic-aa,globals-aa" 2>&1 | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -globals-aa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -globals-aa -da | FileCheck %s
 define void @i32_subscript(i32* %a) {
 entry:
   br label %for.body
diff --git a/llvm/test/Analysis/DependenceAnalysis/Preliminary.ll b/llvm/test/Analysis/DependenceAnalysis/Preliminary.ll
index 2c7e5dd0db3ba..05848a61a7378 100644
--- a/llvm/test/Analysis/DependenceAnalysis/Preliminary.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/Preliminary.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.6.0"
diff --git a/llvm/test/Analysis/DependenceAnalysis/Propagating.ll b/llvm/test/Analysis/DependenceAnalysis/Propagating.ll
index 64d6a9cfae04e..41640a0b4b657 100644
--- a/llvm/test/Analysis/DependenceAnalysis/Propagating.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/Propagating.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.6.0"
diff --git a/llvm/test/Analysis/DependenceAnalysis/Separability.ll b/llvm/test/Analysis/DependenceAnalysis/Separability.ll
index b6004b8d5eee5..bbbc0db4a609f 100644
--- a/llvm/test/Analysis/DependenceAnalysis/Separability.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/Separability.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.6.0"
diff --git a/llvm/test/Analysis/DependenceAnalysis/StrongSIV.ll b/llvm/test/Analysis/DependenceAnalysis/StrongSIV.ll
index f7528a52316f9..397ef8a2d3a03 100644
--- a/llvm/test/Analysis/DependenceAnalysis/StrongSIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/StrongSIV.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.6.0"
diff --git a/llvm/test/Analysis/DependenceAnalysis/SymbolicRDIV.ll b/llvm/test/Analysis/DependenceAnalysis/SymbolicRDIV.ll
index 34c1415aeb5e9..0151c7c78404e 100644
--- a/llvm/test/Analysis/DependenceAnalysis/SymbolicRDIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/SymbolicRDIV.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s
 
 ; ModuleID = 'SymbolicRDIV.bc'
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/llvm/test/Analysis/DependenceAnalysis/SymbolicSIV.ll b/llvm/test/Analysis/DependenceAnalysis/SymbolicSIV.ll
index b24547f73807e..7a37107baf913 100644
--- a/llvm/test/Analysis/DependenceAnalysis/SymbolicSIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/SymbolicSIV.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.6.0"
diff --git a/llvm/test/Analysis/DependenceAnalysis/UsefulGEP.ll b/llvm/test/Analysis/DependenceAnalysis/UsefulGEP.ll
index 283fd2ce2e770..c2d7765b03230 100644
--- a/llvm/test/Analysis/DependenceAnalysis/UsefulGEP.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/UsefulGEP.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1
-; RUN: opt < %s -analyze -basicaa -da
+; RUN: opt < %s -analyze -basic-aa -da
 ;; Check this doesn't crash.
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Analysis/DependenceAnalysis/WeakCrossingSIV.ll b/llvm/test/Analysis/DependenceAnalysis/WeakCrossingSIV.ll
index 119ae98bb530c..449cffc7cd036 100644
--- a/llvm/test/Analysis/DependenceAnalysis/WeakCrossingSIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/WeakCrossingSIV.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s
 
 ; ModuleID = 'WeakCrossingSIV.bc'
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/llvm/test/Analysis/DependenceAnalysis/WeakZeroDstSIV.ll b/llvm/test/Analysis/DependenceAnalysis/WeakZeroDstSIV.ll
index 37c912ca7165b..af9c0bd8f2bb1 100644
--- a/llvm/test/Analysis/DependenceAnalysis/WeakZeroDstSIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/WeakZeroDstSIV.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s
 
 ; ModuleID = 'WeakZeroDstSIV.bc'
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/llvm/test/Analysis/DependenceAnalysis/WeakZeroSrcSIV.ll b/llvm/test/Analysis/DependenceAnalysis/WeakZeroSrcSIV.ll
index 602a79194e956..70612a4b5c1c2 100644
--- a/llvm/test/Analysis/DependenceAnalysis/WeakZeroSrcSIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/WeakZeroSrcSIV.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s
 
 ; ModuleID = 'WeakZeroSrcSIV.bc'
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/llvm/test/Analysis/DependenceAnalysis/ZIV.ll b/llvm/test/Analysis/DependenceAnalysis/ZIV.ll
index c2049a6297760..4e1ea0834e9b5 100644
--- a/llvm/test/Analysis/DependenceAnalysis/ZIV.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/ZIV.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
+; RUN: opt < %s -analyze -basic-aa -da | FileCheck %s
 
 ; ModuleID = 'ZIV.bc'
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/llvm/test/Analysis/GlobalsModRef/aliastest.ll b/llvm/test/Analysis/GlobalsModRef/aliastest.ll
index 3a0eee84e881c..e0aa1613751d6 100644
--- a/llvm/test/Analysis/GlobalsModRef/aliastest.ll
+++ b/llvm/test/Analysis/GlobalsModRef/aliastest.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -globals-aa -gvn -S -enable-unsafe-globalsmodref-alias-results | FileCheck %s
+; RUN: opt < %s -basic-aa -globals-aa -gvn -S -enable-unsafe-globalsmodref-alias-results | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa,globals-aa -passes="require<globals-aa>,function(gvn)" -S -enable-unsafe-globalsmodref-alias-results | FileCheck %s
 ;
 ; Note that this test relies on an unsafe feature of GlobalsModRef. While this
diff --git a/llvm/test/Analysis/GlobalsModRef/chaining-analysis.ll b/llvm/test/Analysis/GlobalsModRef/chaining-analysis.ll
index a12d7371524a0..6f08179917d15 100644
--- a/llvm/test/Analysis/GlobalsModRef/chaining-analysis.ll
+++ b/llvm/test/Analysis/GlobalsModRef/chaining-analysis.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -globals-aa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -globals-aa -gvn -S | FileCheck %s
 
 ; This test requires the use of previous analyses to determine that
 ; doesnotmodX does not modify X (because 'sin' doesn't).
diff --git a/llvm/test/Analysis/GlobalsModRef/comdat-ipo.ll b/llvm/test/Analysis/GlobalsModRef/comdat-ipo.ll
index a9cc821e7afb2..f251e01ca69ca 100644
--- a/llvm/test/Analysis/GlobalsModRef/comdat-ipo.ll
+++ b/llvm/test/Analysis/GlobalsModRef/comdat-ipo.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -globals-aa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -globals-aa -gvn -S | FileCheck %s
 
 ; See PR26774
 
diff --git a/llvm/test/Analysis/GlobalsModRef/func-memattributes.ll b/llvm/test/Analysis/GlobalsModRef/func-memattributes.ll
index c1c269118739a..f9030cfd14093 100644
--- a/llvm/test/Analysis/GlobalsModRef/func-memattributes.ll
+++ b/llvm/test/Analysis/GlobalsModRef/func-memattributes.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -disable-basicaa -globals-aa -dse -S | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -globals-aa -dse -S | FileCheck %s
 
 @X = internal global i32 4
 
diff --git a/llvm/test/Analysis/GlobalsModRef/indirect-global.ll b/llvm/test/Analysis/GlobalsModRef/indirect-global.ll
index 39d5260b950ac..de21fe79a5283 100644
--- a/llvm/test/Analysis/GlobalsModRef/indirect-global.ll
+++ b/llvm/test/Analysis/GlobalsModRef/indirect-global.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -globals-aa -gvn -instcombine -S -enable-unsafe-globalsmodref-alias-results | FileCheck %s
+; RUN: opt < %s -basic-aa -globals-aa -gvn -instcombine -S -enable-unsafe-globalsmodref-alias-results | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa,globals-aa -passes="require<globals-aa>,function(gvn,instcombine)" -S -enable-unsafe-globalsmodref-alias-results | FileCheck %s
 ;
 ; Note that this test relies on an unsafe feature of GlobalsModRef. While this
diff --git a/llvm/test/Analysis/GlobalsModRef/modreftest.ll b/llvm/test/Analysis/GlobalsModRef/modreftest.ll
index 07497705e65a8..c275d47b0b2ca 100644
--- a/llvm/test/Analysis/GlobalsModRef/modreftest.ll
+++ b/llvm/test/Analysis/GlobalsModRef/modreftest.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -globals-aa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -globals-aa -gvn -S | FileCheck %s
 
 @X = internal global i32 4		; <i32*> [#uses=2]
 
diff --git a/llvm/test/Analysis/GlobalsModRef/no-escape.ll b/llvm/test/Analysis/GlobalsModRef/no-escape.ll
index a47ffeb537c5e..9d0f1053902f0 100644
--- a/llvm/test/Analysis/GlobalsModRef/no-escape.ll
+++ b/llvm/test/Analysis/GlobalsModRef/no-escape.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -globals-aa -S -licm | FileCheck %s
+; RUN: opt < %s -basic-aa -globals-aa -S -licm | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.10.0"
diff --git a/llvm/test/Analysis/GlobalsModRef/pr12351.ll b/llvm/test/Analysis/GlobalsModRef/pr12351.ll
index 2aa270a6b6b64..2f861cff6e78c 100644
--- a/llvm/test/Analysis/GlobalsModRef/pr12351.ll
+++ b/llvm/test/Analysis/GlobalsModRef/pr12351.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -globals-aa -gvn -S -disable-verify | FileCheck %s
+; RUN: opt < %s -basic-aa -globals-aa -gvn -S -disable-verify | FileCheck %s
 
 declare void @llvm.memcpy.p0i8.p0i8.i32(i8*, i8*, i32, i1)
 define void @foo(i8* %x, i8* %y) {
diff --git a/llvm/test/Analysis/GlobalsModRef/purecse.ll b/llvm/test/Analysis/GlobalsModRef/purecse.ll
index 8e2bfebc8af4c..f98148d5b46d9 100644
--- a/llvm/test/Analysis/GlobalsModRef/purecse.ll
+++ b/llvm/test/Analysis/GlobalsModRef/purecse.ll
@@ -1,5 +1,5 @@
 ; Test that pure functions are cse'd away
-; RUN: opt < %s -disable-basicaa -globals-aa -gvn -instcombine -S | FileCheck %s
+; RUN: opt < %s -disable-basic-aa -globals-aa -gvn -instcombine -S | FileCheck %s
 
 define i32 @pure(i32 %X) {
         %Y = add i32 %X, 1              ; <i32> [#uses=1]
diff --git a/llvm/test/Analysis/GlobalsModRef/volatile-instrs.ll b/llvm/test/Analysis/GlobalsModRef/volatile-instrs.ll
index 85d2e887e1cbb..dae1d6f4c7c07 100644
--- a/llvm/test/Analysis/GlobalsModRef/volatile-instrs.ll
+++ b/llvm/test/Analysis/GlobalsModRef/volatile-instrs.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/memcheck-wrapping-pointers.ll b/llvm/test/Analysis/LoopAccessAnalysis/memcheck-wrapping-pointers.ll
index cb1b7edb3d101..484f2b47b22a1 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/memcheck-wrapping-pointers.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/memcheck-wrapping-pointers.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-accesses -analyze < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-accesses -analyze < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/non-wrapping-pointer.ll b/llvm/test/Analysis/LoopAccessAnalysis/non-wrapping-pointer.ll
index 94080f05bde26..99ba107ed09ea 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/non-wrapping-pointer.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/non-wrapping-pointer.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-accesses -analyze < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-accesses -analyze < %s | FileCheck %s
 ; RUN: opt -passes='require<aa>,require<scalar-evolution>,require<aa>,loop(print-access-info)' -aa-pipeline='basic-aa' -disable-output < %s  2>&1 | FileCheck %s
 
 ; For this loop:
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/safe-no-checks.ll b/llvm/test/Analysis/LoopAccessAnalysis/safe-no-checks.ll
index 8018994009957..2a937cbe62f6e 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/safe-no-checks.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/safe-no-checks.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-accesses -analyze < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-accesses -analyze < %s | FileCheck %s
 ; RUN: opt -passes='require<aa>,require<scalar-evolution>,require<aa>,loop(print-access-info)' -aa-pipeline='basic-aa' -disable-output < %s  2>&1 | FileCheck %s
 
 ; If the arrays don't alias this loop is safe with no memchecks:
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/underlying-objects-1.ll b/llvm/test/Analysis/LoopAccessAnalysis/underlying-objects-1.ll
index 705453b65cc50..1204e8359a13a 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/underlying-objects-1.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/underlying-objects-1.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-accesses -analyze < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-accesses -analyze < %s | FileCheck %s
 ; RUN: opt -passes='require<scalar-evolution>,require<aa>,loop(print-access-info)' -disable-output  < %s 2>&1 | FileCheck %s
 
 ; In:
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/underlying-objects-2.ll b/llvm/test/Analysis/LoopAccessAnalysis/underlying-objects-2.ll
index f9d88e891be9b..dc2232334a7b0 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/underlying-objects-2.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/underlying-objects-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-accesses -analyze < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-accesses -analyze < %s | FileCheck %s
 ; RUN: opt -passes='require<scalar-evolution>,require<aa>,loop(print-access-info)' -disable-output  < %s 2>&1 | FileCheck %s
 
 ; This loop:
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/wrapping-pointer-versioning.ll b/llvm/test/Analysis/LoopAccessAnalysis/wrapping-pointer-versioning.ll
index fa6fccecbf187..4c058b190d69f 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/wrapping-pointer-versioning.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/wrapping-pointer-versioning.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-accesses -analyze < %s | FileCheck %s -check-prefix=LAA
+; RUN: opt -basic-aa -loop-accesses -analyze < %s | FileCheck %s -check-prefix=LAA
 ; RUN: opt -passes='require<aa>,require<scalar-evolution>,require<aa>,loop(print-access-info)' -aa-pipeline='basic-aa' -disable-output < %s  2>&1 | FileCheck %s --check-prefix=LAA
 ; RUN: opt -loop-versioning -S < %s | FileCheck %s -check-prefix=LV
 
diff --git a/llvm/test/Analysis/MemoryDependenceAnalysis/memdep-block-scan-limit.ll b/llvm/test/Analysis/MemoryDependenceAnalysis/memdep-block-scan-limit.ll
index 20ca31456fd38..ab602bf1ea0cd 100644
--- a/llvm/test/Analysis/MemoryDependenceAnalysis/memdep-block-scan-limit.ll
+++ b/llvm/test/Analysis/MemoryDependenceAnalysis/memdep-block-scan-limit.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S -memdep -gvn -basicaa < %s | FileCheck %s
-; RUN: opt -S -memdep -memdep-block-scan-limit=1 -gvn -basicaa < %s | FileCheck %s --check-prefix=WITH-LIMIT
+; RUN: opt -S -memdep -gvn -basic-aa < %s | FileCheck %s
+; RUN: opt -S -memdep -memdep-block-scan-limit=1 -gvn -basic-aa < %s | FileCheck %s --check-prefix=WITH-LIMIT
 ; CHECK-LABEL: @test(
 ; CHECK: load
 ; CHECK-NOT: load
diff --git a/llvm/test/Analysis/MemorySSA/assume.ll b/llvm/test/Analysis/MemorySSA/assume.ll
index d771c78eb1cfd..12d24df6267c4 100644
--- a/llvm/test/Analysis/MemorySSA/assume.ll
+++ b/llvm/test/Analysis/MemorySSA/assume.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -memoryssa -analyze < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Ensures that assumes are treated as not reading or writing memory.
diff --git a/llvm/test/Analysis/MemorySSA/atomic-clobber.ll b/llvm/test/Analysis/MemorySSA/atomic-clobber.ll
index acd819a89351a..79a9392d1435e 100644
--- a/llvm/test/Analysis/MemorySSA/atomic-clobber.ll
+++ b/llvm/test/Analysis/MemorySSA/atomic-clobber.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Ensures that atomic loads count as MemoryDefs
diff --git a/llvm/test/Analysis/MemorySSA/basicaa-memcpy.ll b/llvm/test/Analysis/MemorySSA/basicaa-memcpy.ll
index 28af1e5238c5f..e33169ae195ec 100644
--- a/llvm/test/Analysis/MemorySSA/basicaa-memcpy.ll
+++ b/llvm/test/Analysis/MemorySSA/basicaa-memcpy.ll
@@ -1,4 +1,4 @@
-; RUN: opt -disable-output -basicaa -print-memoryssa %s 2>&1 | FileCheck %s
+; RUN: opt -disable-output -basic-aa -print-memoryssa %s 2>&1 | FileCheck %s
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
 
diff --git a/llvm/test/Analysis/MemorySSA/constant-memory.ll b/llvm/test/Analysis/MemorySSA/constant-memory.ll
index bc970e72fc4dc..4c2551378292d 100644
--- a/llvm/test/Analysis/MemorySSA/constant-memory.ll
+++ b/llvm/test/Analysis/MemorySSA/constant-memory.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ;
 ; Things that BasicAA can prove points to constant memory should be
 ; liveOnEntry, as well.
diff --git a/llvm/test/Analysis/MemorySSA/cyclicphi.ll b/llvm/test/Analysis/MemorySSA/cyclicphi.ll
index 9db19685197ef..36d34eff4499e 100644
--- a/llvm/test/Analysis/MemorySSA/cyclicphi.ll
+++ b/llvm/test/Analysis/MemorySSA/cyclicphi.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s
 
 %struct.hoge = type { i32, %struct.widget }
diff --git a/llvm/test/Analysis/MemorySSA/debugvalue.ll b/llvm/test/Analysis/MemorySSA/debugvalue.ll
index 2d70556271ca6..3e73b98279499 100644
--- a/llvm/test/Analysis/MemorySSA/debugvalue.ll
+++ b/llvm/test/Analysis/MemorySSA/debugvalue.ll
@@ -1,4 +1,4 @@
-; RUN: opt -disable-basicaa -loop-rotate -enable-mssa-loop-dependency -verify-memoryssa -S %s | FileCheck %s
+; RUN: opt -disable-basic-aa -loop-rotate -enable-mssa-loop-dependency -verify-memoryssa -S %s | FileCheck %s
 ; REQUIRES: asserts
 
 ; CHECK-LABEL: @f_w4_i2
diff --git a/llvm/test/Analysis/MemorySSA/debugvalue2.ll b/llvm/test/Analysis/MemorySSA/debugvalue2.ll
index 63144523f4f3b..90784ae9906c3 100644
--- a/llvm/test/Analysis/MemorySSA/debugvalue2.ll
+++ b/llvm/test/Analysis/MemorySSA/debugvalue2.ll
@@ -1,4 +1,4 @@
-; RUN: opt -disable-basicaa -print-memoryssa -disable-output %s 2>&1 | FileCheck %s
+; RUN: opt -disable-basic-aa -print-memoryssa -disable-output %s 2>&1 | FileCheck %s
 
 ; Note that the test crashes the MemorySSA verification when doing loop-rotate,
 ; if debuginfo is modelled in MemorySSA, due to the fact that MemorySSA is not
diff --git a/llvm/test/Analysis/MemorySSA/function-clobber.ll b/llvm/test/Analysis/MemorySSA/function-clobber.ll
index a01893a5b954f..b9b16bd6e33d3 100644
--- a/llvm/test/Analysis/MemorySSA/function-clobber.ll
+++ b/llvm/test/Analysis/MemorySSA/function-clobber.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Ensuring that external functions without attributes are MemoryDefs
diff --git a/llvm/test/Analysis/MemorySSA/function-mem-attrs.ll b/llvm/test/Analysis/MemorySSA/function-mem-attrs.ll
index 11383771a4132..7860576bc4774 100644
--- a/llvm/test/Analysis/MemorySSA/function-mem-attrs.ll
+++ b/llvm/test/Analysis/MemorySSA/function-mem-attrs.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Test that various function attributes give us sane results.
diff --git a/llvm/test/Analysis/MemorySSA/invariant-groups.ll b/llvm/test/Analysis/MemorySSA/invariant-groups.ll
index 6d26829944b4b..455802009823d 100644
--- a/llvm/test/Analysis/MemorySSA/invariant-groups.ll
+++ b/llvm/test/Analysis/MemorySSA/invariant-groups.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ;
 ; Currently, MemorySSA doesn't support invariant groups. So, we should ignore
 ; launder.invariant.group intrinsics entirely. We'll need to pay attention to
diff --git a/llvm/test/Analysis/MemorySSA/lifetime-simple.ll b/llvm/test/Analysis/MemorySSA/lifetime-simple.ll
index f1db15cc577d5..ef274c98094ef 100644
--- a/llvm/test/Analysis/MemorySSA/lifetime-simple.ll
+++ b/llvm/test/Analysis/MemorySSA/lifetime-simple.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s
 ; This test checks a number of things:
 ; First, the lifetime markers should not clobber any uses of Q or P.
diff --git a/llvm/test/Analysis/MemorySSA/load-invariant.ll b/llvm/test/Analysis/MemorySSA/load-invariant.ll
index defb74641eaa4..13751226d0294 100644
--- a/llvm/test/Analysis/MemorySSA/load-invariant.ll
+++ b/llvm/test/Analysis/MemorySSA/load-invariant.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>' -verify-memoryssa -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Invariant loads should be considered live on entry, because, once the
diff --git a/llvm/test/Analysis/MemorySSA/loop-rotate-disablebasicaa.ll b/llvm/test/Analysis/MemorySSA/loop-rotate-disablebasicaa.ll
index f601d48f4b9ef..3845f59c68c66 100644
--- a/llvm/test/Analysis/MemorySSA/loop-rotate-disablebasicaa.ll
+++ b/llvm/test/Analysis/MemorySSA/loop-rotate-disablebasicaa.ll
@@ -1,4 +1,4 @@
-; RUN: opt -disable-basicaa -print-memoryssa -disable-output %s 2>&1 | FileCheck %s
+; RUN: opt -disable-basic-aa -print-memoryssa -disable-output %s 2>&1 | FileCheck %s
 
 ; Note: if @foo is modelled as a MemoryDef, this test will assert with -loop-rotate, due to MemorySSA not
 ; being preserved when moving instructions that may not read from or write to memory.
diff --git a/llvm/test/Analysis/MemorySSA/loop-unswitch.ll b/llvm/test/Analysis/MemorySSA/loop-unswitch.ll
index f42a9ca15c58e..8868f0cfcce30 100644
--- a/llvm/test/Analysis/MemorySSA/loop-unswitch.ll
+++ b/llvm/test/Analysis/MemorySSA/loop-unswitch.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -loop-unswitch -disable-basicaa -enable-mssa-loop-dependency -verify-memoryssa < %s | FileCheck %s
+; RUN: opt -S -loop-unswitch -disable-basic-aa -enable-mssa-loop-dependency -verify-memoryssa < %s | FileCheck %s
 ; REQUIRES: asserts
 
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Analysis/MemorySSA/many-dom-backedge.ll b/llvm/test/Analysis/MemorySSA/many-dom-backedge.ll
index 3d76f4af2d619..73f9eb2ec8498 100644
--- a/llvm/test/Analysis/MemorySSA/many-dom-backedge.ll
+++ b/llvm/test/Analysis/MemorySSA/many-dom-backedge.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; many-dom.ll, with an added back-edge back into the switch.
diff --git a/llvm/test/Analysis/MemorySSA/many-doms.ll b/llvm/test/Analysis/MemorySSA/many-doms.ll
index d2e6c6fa1e43c..1a2422c8fd5aa 100644
--- a/llvm/test/Analysis/MemorySSA/many-doms.ll
+++ b/llvm/test/Analysis/MemorySSA/many-doms.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Testing many dominators, specifically from a switch statement in C.
diff --git a/llvm/test/Analysis/MemorySSA/multi-edges.ll b/llvm/test/Analysis/MemorySSA/multi-edges.ll
index c13fc016b2c77..f8b573f165371 100644
--- a/llvm/test/Analysis/MemorySSA/multi-edges.ll
+++ b/llvm/test/Analysis/MemorySSA/multi-edges.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Makes sure we have a sane model if both successors of some block is the same
diff --git a/llvm/test/Analysis/MemorySSA/multiple-backedges-hal.ll b/llvm/test/Analysis/MemorySSA/multiple-backedges-hal.ll
index 473b3685801c9..6d1f92aca0dd0 100644
--- a/llvm/test/Analysis/MemorySSA/multiple-backedges-hal.ll
+++ b/llvm/test/Analysis/MemorySSA/multiple-backedges-hal.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; hfinkel's case
diff --git a/llvm/test/Analysis/MemorySSA/multiple-locations.ll b/llvm/test/Analysis/MemorySSA/multiple-locations.ll
index 9a3e87e4ab6d6..32a6a209af928 100644
--- a/llvm/test/Analysis/MemorySSA/multiple-locations.ll
+++ b/llvm/test/Analysis/MemorySSA/multiple-locations.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Checks that basicAA is doing some amount of disambiguation for us
diff --git a/llvm/test/Analysis/MemorySSA/no-disconnected.ll b/llvm/test/Analysis/MemorySSA/no-disconnected.ll
index d1dcb15893adb..d3b2db8bc00a7 100644
--- a/llvm/test/Analysis/MemorySSA/no-disconnected.ll
+++ b/llvm/test/Analysis/MemorySSA/no-disconnected.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; This test ensures we don't end up with multiple reaching defs for a single
diff --git a/llvm/test/Analysis/MemorySSA/optimize-use.ll b/llvm/test/Analysis/MemorySSA/optimize-use.ll
index 949c1aaa44553..4acb59e601c1f 100644
--- a/llvm/test/Analysis/MemorySSA/optimize-use.ll
+++ b/llvm/test/Analysis/MemorySSA/optimize-use.ll
@@ -1,5 +1,5 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,NOLIMIT
-; RUN: opt -memssa-check-limit=0 -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,LIMIT
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,NOLIMIT
+; RUN: opt -memssa-check-limit=0 -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,LIMIT
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,NOLIMIT
 ; RUN: opt -memssa-check-limit=0 -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,LIMIT
 
diff --git a/llvm/test/Analysis/MemorySSA/phi-translation.ll b/llvm/test/Analysis/MemorySSA/phi-translation.ll
index 588b018fdd0db..c39ccd31d24b3 100644
--- a/llvm/test/Analysis/MemorySSA/phi-translation.ll
+++ b/llvm/test/Analysis/MemorySSA/phi-translation.ll
@@ -1,5 +1,5 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,NOLIMIT
-; RUN: opt -memssa-check-limit=0 -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,LIMIT
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,NOLIMIT
+; RUN: opt -memssa-check-limit=0 -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,LIMIT
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,NOLIMIT
 ; RUN: opt -memssa-check-limit=0 -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,LIMIT
 
diff --git a/llvm/test/Analysis/MemorySSA/pr28880.ll b/llvm/test/Analysis/MemorySSA/pr28880.ll
index 80c0440f8a921..850ac04ec3b87 100644
--- a/llvm/test/Analysis/MemorySSA/pr28880.ll
+++ b/llvm/test/Analysis/MemorySSA/pr28880.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; This testcase is reduced from SingleSource/Benchmarks/Misc/fbench.c
diff --git a/llvm/test/Analysis/MemorySSA/pr36883.ll b/llvm/test/Analysis/MemorySSA/pr36883.ll
index 8411b0c228b8e..7493df47f315e 100644
--- a/llvm/test/Analysis/MemorySSA/pr36883.ll
+++ b/llvm/test/Analysis/MemorySSA/pr36883.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -memoryssa -analyze < %s 2>&1 -S | FileCheck %s
+; RUN: opt -basic-aa -memoryssa -analyze < %s 2>&1 -S | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -S < %s 2>&1 | FileCheck %s
 ;
 ; We weren't properly considering the args in callsites in equality or hashing.
diff --git a/llvm/test/Analysis/MemorySSA/ptr-const-mem.ll b/llvm/test/Analysis/MemorySSA/ptr-const-mem.ll
index a326d8d717a16..47581e3f2964a 100644
--- a/llvm/test/Analysis/MemorySSA/ptr-const-mem.ll
+++ b/llvm/test/Analysis/MemorySSA/ptr-const-mem.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze -memssa-check-limit=0 < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze -memssa-check-limit=0 < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>' -verify-memoryssa -disable-output -memssa-check-limit=0 < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64"
 target triple = "amdgcn"
diff --git a/llvm/test/Analysis/MemorySSA/scalable-vec.ll b/llvm/test/Analysis/MemorySSA/scalable-vec.ll
index 23072876a2d72..4efbf48c20971 100644
--- a/llvm/test/Analysis/MemorySSA/scalable-vec.ll
+++ b/llvm/test/Analysis/MemorySSA/scalable-vec.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>' -verify-memoryssa -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK-LABEL: define <vscale x 4 x i32> @f(
diff --git a/llvm/test/Analysis/MemorySSA/volatile-clobber.ll b/llvm/test/Analysis/MemorySSA/volatile-clobber.ll
index 5b71950d33ef8..be46af7b9dfa9 100644
--- a/llvm/test/Analysis/MemorySSA/volatile-clobber.ll
+++ b/llvm/test/Analysis/MemorySSA/volatile-clobber.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -print-memoryssa -verify-memoryssa -analyze < %s 2>&1 | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='print<memoryssa>,verify<memoryssa>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Ensures that volatile stores/loads count as MemoryDefs
diff --git a/llvm/test/Analysis/ScalarEvolution/2012-03-26-LoadConstant.ll b/llvm/test/Analysis/ScalarEvolution/2012-03-26-LoadConstant.ll
index 33fcbab929106..eba59687cf85f 100644
--- a/llvm/test/Analysis/ScalarEvolution/2012-03-26-LoadConstant.ll
+++ b/llvm/test/Analysis/ScalarEvolution/2012-03-26-LoadConstant.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -globalopt -instcombine -loop-rotate -licm -instcombine -indvars -loop-deletion -constmerge -S | FileCheck %s
+; RUN: opt < %s -basic-aa -globalopt -instcombine -loop-rotate -licm -instcombine -indvars -loop-deletion -constmerge -S | FileCheck %s
 ; PR11882: ComputeLoadConstantCompareExitLimit crash.
 ;
 ; for.body is deleted leaving a loop-invariant load.
diff --git a/llvm/test/Analysis/ScalarEvolution/add-expr-pointer-operand-sorting.ll b/llvm/test/Analysis/ScalarEvolution/add-expr-pointer-operand-sorting.ll
new file mode 100644
index 0000000000000..e164b7fc5e431
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/add-expr-pointer-operand-sorting.ll
@@ -0,0 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -S -analyze -scalar-evolution | FileCheck %s
+
+; Reduced from test-suite/MultiSource/Benchmarks/MiBench/office-ispell/correct.c
+; getelementptr, obviously, takes pointer as it's base, and returns a pointer.
+; SCEV operands are sorted in hope that it increases folding potential,
+; and at the same time SCEVAddExpr's type is the type of the last(!) operand.
+; Which means, in some exceedingly rare cases, pointer operand may happen to
+; end up not being the last operand, and as a result SCEV for GEP will suddenly
+; have a non-pointer return type. We should ensure that does not happen.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@c = dso_local local_unnamed_addr global i32* null, align 8
+@a = dso_local local_unnamed_addr global i32 0, align 4
+@b = dso_local global [1 x i32] zeroinitializer, align 4
+
+define i32 @d(i32 %base) {
+; CHECK-LABEL: 'd'
+; CHECK-NEXT:  Classifying expressions for: @d
+; CHECK-NEXT:    %e = alloca [1 x [1 x i8]], align 1
+; CHECK-NEXT:    --> %e U: full-set S: full-set
+; CHECK-NEXT:    %0 = bitcast [1 x [1 x i8]]* %e to i8*
+; CHECK-NEXT:    --> %e U: full-set S: full-set
+; CHECK-NEXT:    %f.0 = phi i32 [ %base, %entry ], [ %inc, %for.cond ]
+; CHECK-NEXT:    --> {%base,+,1}<nsw><%for.cond> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %idxprom = sext i32 %f.0 to i64
+; CHECK-NEXT:    --> {(sext i32 %base to i64),+,1}<nsw><%for.cond> U: [-2147483648,-9223372036854775808) S: [-2147483648,-9223372036854775808) Exits: <<Unknown>> LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds [1 x [1 x i8]], [1 x [1 x i8]]* %e, i64 0, i64 %idxprom
+; CHECK-NEXT:    --> {((sext i32 %base to i64) + %e)<nsw>,+,1}<nsw><%for.cond> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %1 = load i32*, i32** @c, align 8
+; CHECK-NEXT:    --> %1 U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond: Variant }
+; CHECK-NEXT:    %sub.ptr.lhs.cast = ptrtoint i32* %1 to i64
+; CHECK-NEXT:    --> %sub.ptr.lhs.cast U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond: Variant }
+; CHECK-NEXT:    %sub.ptr.sub = sub i64 %sub.ptr.lhs.cast, ptrtoint ([1 x i32]* @b to i64)
+; CHECK-NEXT:    --> ((-1 * ptrtoint ([1 x i32]* @b to i64)) + %sub.ptr.lhs.cast) U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond: Variant }
+; CHECK-NEXT:    %sub.ptr.div = sdiv exact i64 %sub.ptr.sub, 4
+; CHECK-NEXT:    --> %sub.ptr.div U: full-set S: [-2305843009213693952,2305843009213693952) Exits: <<Unknown>> LoopDispositions: { %for.cond: Variant }
+; CHECK-NEXT:    %arrayidx1 = getelementptr inbounds [1 x i8], [1 x i8]* %arrayidx, i64 0, i64 %sub.ptr.div
+; CHECK-NEXT:    --> ({((sext i32 %base to i64) + %e)<nsw>,+,1}<nsw><%for.cond> + %sub.ptr.div)<nsw> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond: Variant }
+; CHECK-NEXT:    %2 = load i8, i8* %arrayidx1, align 1
+; CHECK-NEXT:    --> %2 U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond: Variant }
+; CHECK-NEXT:    %conv = sext i8 %2 to i32
+; CHECK-NEXT:    --> (sext i8 %2 to i32) U: [-128,128) S: [-128,128) Exits: <<Unknown>> LoopDispositions: { %for.cond: Variant }
+; CHECK-NEXT:    %inc = add nsw i32 %f.0, 1
+; CHECK-NEXT:    --> {(1 + %base),+,1}<nw><%for.cond> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @d
+; CHECK-NEXT:  Loop %for.cond: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %for.cond: Unpredictable max backedge-taken count.
+; CHECK-NEXT:  Loop %for.cond: Unpredictable predicated backedge-taken count.
+;
+entry:
+  %e = alloca [1 x [1 x i8]], align 1
+  %0 = bitcast [1 x [1 x i8]]* %e to i8*
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %0) #2
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond, %entry
+  %f.0 = phi i32 [ %base, %entry ], [ %inc, %for.cond ]
+  %idxprom = sext i32 %f.0 to i64
+  %arrayidx = getelementptr inbounds [1 x [1 x i8]], [1 x [1 x i8]]* %e, i64 0, i64 %idxprom
+  %1 = load i32*, i32** @c, align 8
+  %sub.ptr.lhs.cast = ptrtoint i32* %1 to i64
+  %sub.ptr.sub = sub i64 %sub.ptr.lhs.cast, ptrtoint ([1 x i32]* @b to i64)
+  %sub.ptr.div = sdiv exact i64 %sub.ptr.sub, 4
+  %arrayidx1 = getelementptr inbounds [1 x i8], [1 x i8]* %arrayidx, i64 0, i64 %sub.ptr.div
+  %2 = load i8, i8* %arrayidx1, align 1
+  %conv = sext i8 %2 to i32
+  store i32 %conv, i32* @a, align 4
+  %inc = add nsw i32 %f.0, 1
+  br label %for.cond
+}
+
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture)
diff --git a/llvm/test/Analysis/ScalarEvolution/depth-limit-overrun.ll b/llvm/test/Analysis/ScalarEvolution/depth-limit-overrun.ll
index 0772307428aa3..f9afc952464ff 100644
--- a/llvm/test/Analysis/ScalarEvolution/depth-limit-overrun.ll
+++ b/llvm/test/Analysis/ScalarEvolution/depth-limit-overrun.ll
@@ -1,4 +1,4 @@
-; RUN: opt -passes 'strength-reduce' -scalar-evolution-max-arith-depth=2 -S < %s | FileCheck %s
+; RUN: opt -passes 'loop-reduce' -scalar-evolution-max-arith-depth=2 -S < %s | FileCheck %s
 ; RUN: opt -loop-reduce -scalar-evolution-max-arith-depth=2 -S < %s | FileCheck %s
 
 ; This test should just compile cleanly without assertions.
diff --git a/llvm/test/Analysis/ScalarEvolution/scev-aa.ll b/llvm/test/Analysis/ScalarEvolution/scev-aa.ll
index 682bd2134afc7..78aac4a26df87 100644
--- a/llvm/test/Analysis/ScalarEvolution/scev-aa.ll
+++ b/llvm/test/Analysis/ScalarEvolution/scev-aa.ll
@@ -1,9 +1,9 @@
-; RUN: opt -disable-output < %s -disable-basicaa -scev-aa -aa-eval -print-all-alias-modref-info \
+; RUN: opt -disable-output < %s -disable-basic-aa -scev-aa -aa-eval -print-all-alias-modref-info \
 ; RUN:   2>&1 | FileCheck %s
 ; RUN: opt -disable-output < %s -aa-pipeline=scev-aa -passes=aa-eval -print-all-alias-modref-info \
 ; RUN:   2>&1 | FileCheck %s
 
-; At the time of this writing, -basicaa misses the example of the form
+; At the time of this writing, -basic-aa misses the example of the form
 ; A[i+(j+1)] != A[i+j], which can arise from multi-dimensional array references,
 ; and the example of the form A[0] != A[i+1], where i+1 is known to be positive.
 
diff --git a/llvm/test/Analysis/ScalarEvolution/sdiv.ll b/llvm/test/Analysis/ScalarEvolution/sdiv.ll
new file mode 100644
index 0000000000000..106cda1b7f0ff
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/sdiv.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -scalar-evolution -analyze | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define dso_local void @_Z4loopi(i32 %width) local_unnamed_addr #0 {
+; CHECK-LABEL: '_Z4loopi'
+; CHECK-NEXT:  Classifying expressions for: @_Z4loopi
+; CHECK-NEXT:    %storage = alloca [2 x i32], align 4
+; CHECK-NEXT:    --> %storage U: [0,-3) S: [-9223372036854775808,9223372036854775805)
+; CHECK-NEXT:    %0 = bitcast [2 x i32]* %storage to i8*
+; CHECK-NEXT:    --> %storage U: [0,-3) S: [-9223372036854775808,9223372036854775805)
+; CHECK-NEXT:    %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+; CHECK-NEXT:    --> {0,+,1}<nuw><nsw><%for.cond> U: [0,-2147483648) S: [0,-2147483648) Exits: %width LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %rem = sdiv i32 %i.0, 2
+; CHECK-NEXT:    --> ({0,+,1}<nuw><nsw><%for.cond> /u 2) U: [0,1073741824) S: [0,1073741824) Exits: (%width /u 2) LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %idxprom = sext i32 %rem to i64
+; CHECK-NEXT:    --> ({0,+,1}<nuw><nsw><%for.cond> /u 2) U: [0,2147483648) S: [0,2147483648) Exits: ((zext i32 %width to i64) /u 2) LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds [2 x i32], [2 x i32]* %storage, i64 0, i64 %idxprom
+; CHECK-NEXT:    --> ((4 * ({0,+,1}<nuw><nsw><%for.cond> /u 2))<nuw><nsw> + %storage)<nsw> U: [0,-3) S: [-9223372036854775808,9223372036854775805) Exits: ((4 * ((zext i32 %width to i64) /u 2))<nuw><nsw> + %storage)<nsw> LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %1 = load i32, i32* %arrayidx, align 4
+; CHECK-NEXT:    --> %1 U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond: Variant }
+; CHECK-NEXT:    %call = call i32 @_Z3adji(i32 %1)
+; CHECK-NEXT:    --> %call U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond: Variant }
+; CHECK-NEXT:    %2 = load i32, i32* %arrayidx, align 4
+; CHECK-NEXT:    --> %2 U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond: Variant }
+; CHECK-NEXT:    %add = add nsw i32 %2, %call
+; CHECK-NEXT:    --> (%2 + %call) U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond: Variant }
+; CHECK-NEXT:    %inc = add nsw i32 %i.0, 1
+; CHECK-NEXT:    --> {1,+,1}<nuw><%for.cond> U: [1,0) S: [1,0) Exits: (1 + %width) LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @_Z4loopi
+; CHECK-NEXT:  Loop %for.cond: backedge-taken count is %width
+; CHECK-NEXT:  Loop %for.cond: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %for.cond: Predicated backedge-taken count is %width
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %for.cond: Trip multiple is 1
+;
+entry:
+  %storage = alloca [2 x i32], align 4
+  %0 = bitcast [2 x i32]* %storage to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* %0) #4
+  call void @llvm.memset.p0i8.i64(i8* align 4 %0, i8 0, i64 8, i1 false)
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp ne i32 %i.0, %width
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* %0) #4
+  ret void
+
+for.body:
+  %rem = sdiv i32 %i.0, 2
+  %idxprom = sext i32 %rem to i64
+  %arrayidx = getelementptr inbounds [2 x i32], [2 x i32]* %storage, i64 0, i64 %idxprom
+  %1 = load i32, i32* %arrayidx, align 4
+  %call = call i32 @_Z3adji(i32 %1)
+  %2 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %2, %call
+  store i32 %add, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+}
+
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg) #2
+
+declare dso_local i32 @_Z3adji(i32) local_unnamed_addr #3
+
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
diff --git a/llvm/test/Analysis/ScalarEvolution/srem.ll b/llvm/test/Analysis/ScalarEvolution/srem.ll
new file mode 100644
index 0000000000000..6debab34e3b31
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/srem.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -scalar-evolution -analyze | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define dso_local void @_Z4loopi(i32 %width) local_unnamed_addr #0 {
+; CHECK-LABEL: '_Z4loopi'
+; CHECK-NEXT:  Classifying expressions for: @_Z4loopi
+; CHECK-NEXT:    %storage = alloca [2 x i32], align 4
+; CHECK-NEXT:    --> %storage U: [0,-3) S: [-9223372036854775808,9223372036854775805)
+; CHECK-NEXT:    %0 = bitcast [2 x i32]* %storage to i8*
+; CHECK-NEXT:    --> %storage U: [0,-3) S: [-9223372036854775808,9223372036854775805)
+; CHECK-NEXT:    %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+; CHECK-NEXT:    --> {0,+,1}<nuw><nsw><%for.cond> U: [0,-2147483648) S: [0,-2147483648) Exits: %width LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %rem = srem i32 %i.0, 2
+; CHECK-NEXT:    --> (zext i1 {false,+,true}<%for.cond> to i32) U: [0,2) S: [0,2) Exits: (zext i1 (trunc i32 %width to i1) to i32) LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %idxprom = sext i32 %rem to i64
+; CHECK-NEXT:    --> (zext i1 {false,+,true}<%for.cond> to i64) U: [0,2) S: [0,2) Exits: (zext i1 (trunc i32 %width to i1) to i64) LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds [2 x i32], [2 x i32]* %storage, i64 0, i64 %idxprom
+; CHECK-NEXT:    --> ((4 * (zext i1 {false,+,true}<%for.cond> to i64))<nuw><nsw> + %storage)<nsw> U: [0,-3) S: [-9223372036854775808,9223372036854775805) Exits: ((4 * (zext i1 (trunc i32 %width to i1) to i64))<nuw><nsw> + %storage)<nsw> LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:    %1 = load i32, i32* %arrayidx, align 4
+; CHECK-NEXT:    --> %1 U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond: Variant }
+; CHECK-NEXT:    %call = call i32 @_Z3adji(i32 %1)
+; CHECK-NEXT:    --> %call U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond: Variant }
+; CHECK-NEXT:    %2 = load i32, i32* %arrayidx, align 4
+; CHECK-NEXT:    --> %2 U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond: Variant }
+; CHECK-NEXT:    %add = add nsw i32 %2, %call
+; CHECK-NEXT:    --> (%2 + %call) U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond: Variant }
+; CHECK-NEXT:    %inc = add nsw i32 %i.0, 1
+; CHECK-NEXT:    --> {1,+,1}<nuw><%for.cond> U: [1,0) S: [1,0) Exits: (1 + %width) LoopDispositions: { %for.cond: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @_Z4loopi
+; CHECK-NEXT:  Loop %for.cond: backedge-taken count is %width
+; CHECK-NEXT:  Loop %for.cond: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %for.cond: Predicated backedge-taken count is %width
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %for.cond: Trip multiple is 1
+;
+entry:
+  %storage = alloca [2 x i32], align 4
+  %0 = bitcast [2 x i32]* %storage to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* %0) #4
+  call void @llvm.memset.p0i8.i64(i8* align 4 %0, i8 0, i64 8, i1 false)
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp ne i32 %i.0, %width
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* %0) #4
+  ret void
+
+for.body:
+  %rem = srem i32 %i.0, 2
+  %idxprom = sext i32 %rem to i64
+  %arrayidx = getelementptr inbounds [2 x i32], [2 x i32]* %storage, i64 0, i64 %idxprom
+  %1 = load i32, i32* %arrayidx, align 4
+  %call = call i32 @_Z3adji(i32 %1)
+  %2 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %2, %call
+  store i32 %add, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+}
+
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg) #2
+
+declare dso_local i32 @_Z3adji(i32) local_unnamed_addr #3
+
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
diff --git a/llvm/test/Analysis/ScopedNoAliasAA/basic-domains.ll b/llvm/test/Analysis/ScopedNoAliasAA/basic-domains.ll
index 2fea2ba068a12..5c66f03b87020 100644
--- a/llvm/test/Analysis/ScopedNoAliasAA/basic-domains.ll
+++ b/llvm/test/Analysis/ScopedNoAliasAA/basic-domains.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -scoped-noalias -aa-eval -evaluate-aa-metadata -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -scoped-noalias -aa-eval -evaluate-aa-metadata -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Analysis/ScopedNoAliasAA/basic.ll b/llvm/test/Analysis/ScopedNoAliasAA/basic.ll
index 69e088e89763e..9ca06a6826bdb 100644
--- a/llvm/test/Analysis/ScopedNoAliasAA/basic.ll
+++ b/llvm/test/Analysis/ScopedNoAliasAA/basic.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -scoped-noalias -aa-eval -evaluate-aa-metadata -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -scoped-noalias -aa-eval -evaluate-aa-metadata -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa,scoped-noalias-aa -passes=aa-eval -evaluate-aa-metadata -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Analysis/ScopedNoAliasAA/basic2.ll b/llvm/test/Analysis/ScopedNoAliasAA/basic2.ll
index a8a17e5f90bf1..e0c16b1991157 100644
--- a/llvm/test/Analysis/ScopedNoAliasAA/basic2.ll
+++ b/llvm/test/Analysis/ScopedNoAliasAA/basic2.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -scoped-noalias -aa-eval -evaluate-aa-metadata -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -scoped-noalias -aa-eval -evaluate-aa-metadata -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Analysis/StackSafetyAnalysis/local.ll b/llvm/test/Analysis/StackSafetyAnalysis/local.ll
index e0f450a975abc..890d4dced3357 100644
--- a/llvm/test/Analysis/StackSafetyAnalysis/local.ll
+++ b/llvm/test/Analysis/StackSafetyAnalysis/local.ll
@@ -476,12 +476,111 @@ define void @Overflow() {
 ; CHECK-LABEL: @Overflow dso_preemptable{{$}}
 ; CHECK-NEXT: args uses:
 ; CHECK-NEXT: allocas uses:
-; LOCAL: x[1]: empty-set, @LoadMinInt64(arg0, [-9223372036854775808,-9223372036854775807)){{$}}
-; GLOBAL: x[1]: full-set, @LoadMinInt64(arg0, [-9223372036854775808,-9223372036854775807)){{$}}
+; LOCAL-NEXT: x[1]: empty-set, @LoadMinInt64(arg0, [-9223372036854775808,-9223372036854775807)){{$}}
+; GLOBAL-NEXT: x[1]: full-set, @LoadMinInt64(arg0, [-9223372036854775808,-9223372036854775807)){{$}}
 ; CHECK-NOT: ]:
 entry:
   %x = alloca i8, align 4
   %x2 = getelementptr i8, i8* %x, i64 -9223372036854775808
   %v = call i8 @LoadMinInt64(i8* %x2)
   ret void
-}
\ No newline at end of file
+}
+
+define void @DeadBlock(i64* %p) {
+; CHECK-LABEL: @DeadBlock dso_preemptable{{$}}
+; CHECK-NEXT: args uses:
+; CHECK-NEXT: p[]: empty-set{{$}}
+; CHECK-NEXT: allocas uses:
+; CHECK-NEXT: x[1]: empty-set{{$}}
+; CHECK-NOT: ]:
+entry:
+  %x = alloca i8, align 4
+  br label %end
+
+dead:
+  store i8 5, i8* %x
+  store i64 -5, i64* %p
+  br label %end
+
+end:
+  ret void
+}
+
+define void @LifeNotStarted() {
+; CHECK-LABEL: @LifeNotStarted dso_preemptable{{$}}
+; CHECK-NEXT: args uses:
+; CHECK-NEXT: allocas uses:
+; CHECK: x[1]: full-set{{$}}
+; CHECK: y[1]: full-set{{$}}
+; CHECK: z[1]: full-set{{$}}
+; CHECK-NOT: ]:
+entry:
+  %x = alloca i8, align 4
+  %y = alloca i8, align 4
+  %z = alloca i8, align 4
+
+  store i8 5, i8* %x
+  %n = load i8, i8* %y
+  call void @llvm.memset.p0i8.i32(i8* nonnull %z, i8 0, i32 1, i1 false)
+
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %x)
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %y)
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %z)
+
+  ret void
+}
+
+define void @LifeOK() {
+; CHECK-LABEL: @LifeOK dso_preemptable{{$}}
+; CHECK-NEXT: args uses:
+; CHECK-NEXT: allocas uses:
+; CHECK: x[1]: [0,1){{$}}
+; CHECK: y[1]: [0,1){{$}}
+; CHECK: z[1]: [0,1){{$}}
+; CHECK-NOT: ]:
+entry:
+  %x = alloca i8, align 4
+  %y = alloca i8, align 4
+  %z = alloca i8, align 4
+
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %x)
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %y)
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %z)
+
+  store i8 5, i8* %x
+  %n = load i8, i8* %y
+  call void @llvm.memset.p0i8.i32(i8* nonnull %z, i8 0, i32 1, i1 false)
+
+  ret void
+}
+
+define void @LifeEnded() {
+; CHECK-LABEL: @LifeEnded dso_preemptable{{$}}
+; CHECK-NEXT: args uses:
+; CHECK-NEXT: allocas uses:
+; CHECK: x[1]: full-set{{$}}
+; CHECK: y[1]: full-set{{$}}
+; CHECK: z[1]: full-set{{$}}
+; CHECK-NOT: ]:
+entry:
+  %x = alloca i8, align 4
+  %y = alloca i8, align 4
+  %z = alloca i8, align 4
+
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %x)
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %y)
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %z)
+
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %x)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %y)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %z)
+
+  store i8 5, i8* %x
+  %n = load i8, i8* %y
+  call void @llvm.memset.p0i8.i32(i8* nonnull %z, i8 0, i32 1, i1 false)
+
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
\ No newline at end of file
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/aggregates.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/aggregates.ll
index 120a1d52f275f..bf6fddb4909fb 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/aggregates.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/aggregates.ll
@@ -1,7 +1,7 @@
-; RUN: opt < %s -tbaa -basicaa -aa-eval -evaluate-aa-metadata \
+; RUN: opt < %s -tbaa -basic-aa -aa-eval -evaluate-aa-metadata \
 ; RUN:     -print-no-aliases -print-may-aliases -disable-output 2>&1 | \
 ; RUN:     FileCheck %s
-; RUN: opt < %s -tbaa -basicaa -gvn -S | FileCheck %s --check-prefix=OPT
+; RUN: opt < %s -tbaa -basic-aa -gvn -S | FileCheck %s --check-prefix=OPT
 ;
 ; Check that TBAA handles access tags with aggregate final access types
 ; correctly.
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/aliastest.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/aliastest.ll
index 3bdd9d8a2ef53..b50d9e8bd411b 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/aliastest.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/aliastest.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -tbaa -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -tbaa -basic-aa -gvn -S | FileCheck %s
 
 ; Test that basic alias queries work.
 
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/argument-promotion.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/argument-promotion.ll
index a7987f7d7f064..f0d07bff3829a 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/argument-promotion.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/argument-promotion.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -tbaa -basicaa -argpromotion -mem2reg -S | FileCheck %s
+; RUN: opt < %s -tbaa -basic-aa -argpromotion -mem2reg -S | FileCheck %s
 
 target datalayout = "E-p:64:64:64"
 
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/dse.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/dse.ll
index 8c51e99e3101d..de7b11efc2b60 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/dse.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/dse.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -tbaa -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -tbaa -basic-aa -dse -S | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; DSE should make use of TBAA.
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/dynamic-indices.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/dynamic-indices.ll
index 8ac6301f93184..a7de205d94307 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/dynamic-indices.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/dynamic-indices.ll
@@ -1,4 +1,4 @@
-; RUN: opt -tbaa -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -tbaa -basic-aa -gvn -S < %s | FileCheck %s
 ; PR9971
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/functionattrs.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/functionattrs.ll
index 28dc4342dd888..ba893fedee269 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/functionattrs.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/functionattrs.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -tbaa -basicaa -functionattrs -S | FileCheck %s
+; RUN: opt < %s -tbaa -basic-aa -functionattrs -S | FileCheck %s
 
 ; FunctionAttrs should make use of TBAA.
 
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/gvn-nonlocal-type-mismatch.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/gvn-nonlocal-type-mismatch.ll
index 1b541a5284361..f8856c0c8cbb0 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/gvn-nonlocal-type-mismatch.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/gvn-nonlocal-type-mismatch.ll
@@ -1,4 +1,4 @@
-; RUN: opt -tbaa -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -tbaa -basic-aa -gvn -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64"
 
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/intrinsics.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/intrinsics.ll
index 61b92b5bd123a..648fcf707f9f6 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/intrinsics.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/intrinsics.ll
@@ -1,4 +1,4 @@
-; RUN: opt -tbaa -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -tbaa -basic-aa -gvn -S < %s | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:32:64-v128:32:128-a0:0:32-n32"
 
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/memcpyopt.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/memcpyopt.ll
index c1f6f18f629bc..c24a721697235 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/memcpyopt.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/memcpyopt.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -tbaa -basicaa -memcpyopt -instcombine < %s | FileCheck %s
+; RUN: opt -S -tbaa -basic-aa -memcpyopt -instcombine < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64"
 
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/placement-tbaa.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/placement-tbaa.ll
index 54d57133b99c6..16cb3b741307f 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/placement-tbaa.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/placement-tbaa.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -tbaa -basicaa -aa-eval -evaluate-aa-metadata -print-no-aliases -print-may-aliases -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -tbaa -basic-aa -aa-eval -evaluate-aa-metadata -print-no-aliases -print-may-aliases -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=type-based-aa,basic-aa -passes=aa-eval -evaluate-aa-metadata -print-no-aliases -print-may-aliases -disable-output 2>&1 | FileCheck %s
 
 ; Generated with "clang -cc1 -disable-llvm-optzns -O1 -emit-llvm"
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/precedence.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/precedence.ll
index c7cd2e274dff8..b7dc0506836e7 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/precedence.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/precedence.ll
@@ -1,4 +1,4 @@
-; RUN: opt -tbaa -disable-basicaa -gvn -instcombine -S < %s | FileCheck %s --check-prefix=TBAA
+; RUN: opt -tbaa -disable-basic-aa -gvn -instcombine -S < %s | FileCheck %s --check-prefix=TBAA
 ; RUN: opt -tbaa -gvn -instcombine -S < %s | FileCheck %s --check-prefix=BASICAA
 
 ; According to the TBAA metadata the load and store don't alias. However,
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/tbaa-path-new.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/tbaa-path-new.ll
index 94fc12a8b4cb2..3f933c24e332e 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/tbaa-path-new.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/tbaa-path-new.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -tbaa -basicaa -aa-eval -evaluate-aa-metadata -print-no-aliases -print-may-aliases -disable-output 2>&1 | FileCheck %s
-; RUN: opt < %s -tbaa -basicaa -gvn -S | FileCheck %s --check-prefix=OPT
+; RUN: opt < %s -tbaa -basic-aa -aa-eval -evaluate-aa-metadata -print-no-aliases -print-may-aliases -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -tbaa -basic-aa -gvn -S | FileCheck %s --check-prefix=OPT
 ; Generated from clang/test/CodeGen/tbaa.cpp with "-O1 -new-struct-path-tbaa".
 
 %struct.StructA = type { i16, i32, i16, i32 }
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/tbaa-path.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/tbaa-path.ll
index 0b6e8245e0e01..b28867248e32a 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/tbaa-path.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/tbaa-path.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -tbaa -basicaa -aa-eval -evaluate-aa-metadata -print-no-aliases -print-may-aliases -disable-output 2>&1 | FileCheck %s
-; RUN: opt < %s -tbaa -basicaa -gvn -S | FileCheck %s --check-prefix=OPT
+; RUN: opt < %s -tbaa -basic-aa -aa-eval -evaluate-aa-metadata -print-no-aliases -print-may-aliases -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -tbaa -basic-aa -gvn -S | FileCheck %s --check-prefix=OPT
 ; Generated from clang/test/CodeGen/tbaa.cpp with "-O1 -struct-path-tbaa -disable-llvm-optzns".
 
 %struct.StructA = type { i16, i32, i16, i32 }
diff --git a/llvm/test/Assembler/align-param-attr-error0.ll b/llvm/test/Assembler/align-param-attr-error0.ll
new file mode 100644
index 0000000000000..7760c014998b7
--- /dev/null
+++ b/llvm/test/Assembler/align-param-attr-error0.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+; Test parse errors when using form of align attribute with parentheses
+
+; CHECK:  <stdin>:[[@LINE+1]]:38: error: expected ')'
+define void @missing_rparen(i8* align(4 %ptr) {
+  ret void
+}
diff --git a/llvm/test/Assembler/align-param-attr-error1.ll b/llvm/test/Assembler/align-param-attr-error1.ll
new file mode 100644
index 0000000000000..465fb2bfc6290
--- /dev/null
+++ b/llvm/test/Assembler/align-param-attr-error1.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+; Test parse errors when using form of align attribute with parentheses
+
+; CHECK:  <stdin>:[[@LINE+1]]:42: error: expected '{' in function body
+define void @missing_Lparen(i8* align 4) %ptr) {
+  ret void
+}
diff --git a/llvm/test/Assembler/align-param-attr-error2.ll b/llvm/test/Assembler/align-param-attr-error2.ll
new file mode 100644
index 0000000000000..7bfcd614a740e
--- /dev/null
+++ b/llvm/test/Assembler/align-param-attr-error2.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+; Test parse errors when using form of align attribute with parentheses
+
+; CHECK:  <stdin>:[[@LINE+1]]:39: error: expected integer
+define void @missing_value(i8* align () %ptr) {
+  ret void
+}
diff --git a/llvm/test/Assembler/align-param-attr-format.ll b/llvm/test/Assembler/align-param-attr-format.ll
new file mode 100644
index 0000000000000..91ec0c6f006c1
--- /dev/null
+++ b/llvm/test/Assembler/align-param-attr-format.ll
@@ -0,0 +1,13 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+
+; Test that align(N) is accepted as an alternative syntax to align N
+
+; CHECK: define void @param_align4(i8* align 4 %ptr) {
+define void @param_align4(i8* align(4) %ptr) {
+  ret void
+}
+
+; CHECK: define void @param_align128(i8* align 128 %0) {
+define void @param_align128(i8* align(128)) {
+  ret void
+}
diff --git a/llvm/test/Assembler/getelementptr_vscale_struct.ll b/llvm/test/Assembler/getelementptr_vscale_struct.ll
new file mode 100644
index 0000000000000..c6b592c0bf8bc
--- /dev/null
+++ b/llvm/test/Assembler/getelementptr_vscale_struct.ll
@@ -0,0 +1,10 @@
+; RUN: not llvm-as < %s >/dev/null 2> %t
+; RUN: FileCheck %s < %t
+; Test that a scalable vector struct index is rejected.
+
+; CHECK: invalid getelementptr indices
+
+define <vscale x 2 x i32*> @test7(<vscale x 2 x {i32, i32}*> %a) {
+  %w = getelementptr {i32, i32}, <vscale x 2 x {i32, i32}*> %a, <vscale x 2 x i32> zeroinitializer, <vscale x 2 x i32> zeroinitializer
+  ret <vscale x 2 x i32*> %w
+}
diff --git a/llvm/test/Bindings/Go/lit.local.cfg b/llvm/test/Bindings/Go/lit.local.cfg
index 3021fc64a750a..e32737079e4a0 100644
--- a/llvm/test/Bindings/Go/lit.local.cfg
+++ b/llvm/test/Bindings/Go/lit.local.cfg
@@ -9,6 +9,9 @@ if not 'go' in config.root.llvm_bindings:
 if not config.root.include_go_tests:
     config.unsupported = True
 
+if config.have_tf_aot:
+    config.unsupported = True
+
 def find_executable(executable, path=None):
     if path is None:
         path = os.environ['PATH']
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index 71c1fb59ebaaa..fbbe4a80f31e0 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -386,6 +386,12 @@ define void @f65() null_pointer_is_valid
   ret void;
 }
 
+; CHECK: define noundef i32 @f66(i32 noundef %a)
+define noundef i32 @f66(i32 noundef %a)
+{
+  ret i32 %a
+}
+
 ; CHECK: attributes #0 = { noreturn }
 ; CHECK: attributes #1 = { nounwind }
 ; CHECK: attributes #2 = { readnone }
diff --git a/llvm/test/Bitcode/vscale-shuffle.ll b/llvm/test/Bitcode/vscale-shuffle.ll
new file mode 100644
index 0000000000000..3f36209c7aaf5
--- /dev/null
+++ b/llvm/test/Bitcode/vscale-shuffle.ll
@@ -0,0 +1,10 @@
+; RUN: llvm-as < %s | llvm-dis -disable-output
+; RUN: verify-uselistorder < %s
+
+define void @f() {
+  %l = call <vscale x 16 x i8> @l(<vscale x 16 x i1> shufflevector (<vscale x 16 x i1> insertelement (<vscale x 16 x i1> undef, i1 true, i32 0), <vscale x 16 x i1> undef, <vscale x 16 x i32> zeroinitializer))
+  %i = add <vscale x 2 x i64> undef, shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> undef, i64 1, i32 0), <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer)
+  unreachable
+}
+
+declare <vscale x 16 x i8> @l(<vscale x 16 x i1>)
diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index 7e6c66e249fce..6f826d54a4b9c 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -15,6 +15,7 @@ llvm_canonicalize_cmake_booleans(
   LLVM_BUILD_EXAMPLES
   LLVM_ENABLE_PLUGINS
   LLVM_BYE_LINK_INTO_TOOLS
+  LLVM_HAVE_TF_AOT
   )
 
 configure_lit_site_cfg(
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
index ca628fa45f6da..cf596c98d4628 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
@@ -220,21 +220,40 @@ entry:
   ret void
 }
 
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to lower arguments{{.*}}scalable_arg
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to lower function{{.*}}scalable_arg
 ; FALLBACK-WITH-REPORT-OUT-LABEL: scalable_arg
 define <vscale x 16 x i8> @scalable_arg(<vscale x 16 x i1> %pred, i8* %addr) #1 {
   %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pred, i8* %addr)
   ret <vscale x 16 x i8> %res
 }
 
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to translate instruction{{.*}}scalable_call
-; FALLBACK-WITH-REPORT-OUT-LABEL: scalable_call
-define <vscale x 16 x i8> @scalable_call(i8* %addr) #1 {
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to lower function{{.*}}scalable_ret
+; FALLBACK-WITH-REPORT-OUT-LABEL: scalable_ret
+define <vscale x 16 x i8> @scalable_ret(i8* %addr) #1 {
   %pred = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 0)
   %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pred, i8* %addr)
   ret <vscale x 16 x i8> %res
 }
 
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to translate instruction{{.*}}scalable_call
+; FALLBACK-WITH-REPORT-OUT-LABEL: scalable_call
+define i8 @scalable_call(i8* %addr) #1 {
+  %pred = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 0)
+  %vec = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pred, i8* %addr)
+  %res = extractelement <vscale x 16 x i8> %vec, i32 0
+  ret i8 %res
+}
+
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to translate instruction{{.*}}asm_indirect_output
+; FALLBACK-WITH-REPORT-OUT-LABEL: asm_indirect_output
+define void @asm_indirect_output() {
+entry:
+  %ap = alloca i8*, align 8
+  %0 = load i8*, i8** %ap, align 8
+  call void asm sideeffect "", "=*r|m,0,~{memory}"(i8** %ap, i8* %0)
+  ret void
+}
+
 attributes #1 = { "target-features"="+sve" }
 
 declare <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 %pattern)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
index 8e312d864798e..7e806e5d0ed95 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
@@ -1968,6 +1968,32 @@ done:
   ret i32 %value_loaded
 }
 
+; Try one cmpxchg
+define i32 @test_weak_atomic_cmpxchg_1(i32* %addr) {
+; CHECK-LABEL: name: test_weak_atomic_cmpxchg_1
+; CHECK:       bb.1.entry:
+; CHECK-NEXT:  successors: %bb.{{[^)]+}}
+; CHECK-NEXT:  liveins: $x0
+; CHECK:         [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT:    [[OLDVAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+; CHECK-NEXT:    [[NEWVAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+; CHECK:       bb.2.repeat:
+; CHECK-NEXT:    successors: %bb.3({{[^)]+}}), %bb.2({{[^)]+}})
+; CHECK:         [[OLDVALRES:%[0-9]+]]:_(s32), [[SUCCESS:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[ADDR]](p0), [[OLDVAL]], [[NEWVAL]] :: (load store monotonic monotonic 4 on %ir.addr)
+; CHECK-NEXT:    G_BRCOND [[SUCCESS]](s1), %bb.3
+; CHECK-NEXT:    G_BR %bb.2
+; CHECK:       bb.3.done:
+entry:
+  br label %repeat
+repeat:
+  %val_success = cmpxchg weak i32* %addr, i32 0, i32 1 monotonic monotonic
+  %value_loaded = extractvalue { i32, i1 } %val_success, 0
+  %success = extractvalue { i32, i1 } %val_success, 1
+  br i1 %success, label %done, label %repeat
+done:
+  ret i32 %value_loaded
+}
+
 ; Try one cmpxchg with a small type and high atomic ordering.
 define i16 @test_atomic_cmpxchg_2(i16* %addr) {
 ; CHECK-LABEL: name: test_atomic_cmpxchg_2
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-regbankselect.mir b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-regbankselect.mir
index 463d6c1ae76f9..37d00dfb31747 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-regbankselect.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-regbankselect.mir
@@ -398,20 +398,23 @@ legalized:       true
 
 # CHECK:      registers:
 # CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '' }
-# CHECK-NEXT:  - { id: 1, class: gpr, preferred-register: '' }
+# FAST-NEXT:   - { id: 1, class: fpr, preferred-register: '' }
+# GREEDY-NEXT: - { id: 1, class: gpr, preferred-register: '' }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
 
 # CHECK:  body:
 # CHECK:    %0:gpr(s32) = COPY $w0
-# CHECK:    %1:gpr(s32) = G_BITCAST %0
+# FAST-NEXT:   %1:fpr(<4 x s8>) = G_BITCAST %0
+# GREEDY-NEXT: %1:gpr(<4 x s8>) = G_BITCAST %0
+# The greedy check is incorrect and should produce fpr.
 body:             |
   bb.0:
     liveins: $w0
 
     %0(s32) = COPY $w0
-    %1(s32) = G_BITCAST %0
+    %1(<4 x s8>) = G_BITCAST %0
 ...
 
 ---
@@ -421,20 +424,22 @@ legalized:       true
 
 # CHECK:      registers:
 # CHECK-NEXT:  - { id: 0, class: fpr, preferred-register: '' }
-# CHECK-NEXT:  - { id: 1, class: fpr, preferred-register: '' }
+# FAST-NEXT:   - { id: 1, class: gpr, preferred-register: '' }
+# GREEDY-NEXT: - { id: 1, class: fpr, preferred-register: '' }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
 
 # CHECK:  body:
 # CHECK:    %0:fpr(<2 x s16>) = COPY $s0
-# CHECK:    %1:fpr(<2 x s16>) = G_BITCAST %0
+# FAST:     %1:gpr(s32) = G_BITCAST %0
+# GREEDY:   %1:fpr(s32) = G_BITCAST %0
 body:             |
   bb.0:
     liveins: $s0
 
     %0(<2 x s16>) = COPY $s0
-    %1(<2 x s16>) = G_BITCAST %0
+    %1(s32) = G_BITCAST %0
 ...
 
 ---
@@ -490,13 +495,14 @@ registers:
   - { id: 1, class: _ }
 # CHECK:  body:
 # CHECK:    %0:gpr(s64) = COPY $x0
-# CHECK:    %1:gpr(s64) = G_BITCAST %0
+# FAST:    %1:fpr(<2 x s32>) = G_BITCAST %0
+# GREEDY:  %1:gpr(<2 x s32>) = G_BITCAST %0
 body:             |
   bb.0:
     liveins: $x0
 
     %0(s64) = COPY $x0
-    %1(s64) = G_BITCAST %0
+    %1(<2 x s32>) = G_BITCAST %0
 ...
 
 ---
@@ -508,13 +514,14 @@ registers:
   - { id: 1, class: _ }
 # CHECK:  body:
 # CHECK:    %0:fpr(<2 x s32>) = COPY $d0
-# CHECK:    %1:fpr(<2 x s32>) = G_BITCAST %0
+# FAST:     %1:gpr(s64) = G_BITCAST %0
+# GREEDY:   %1:fpr(s64) = G_BITCAST %0
 body:             |
   bb.0:
     liveins: $d0
 
     %0(<2 x s32>) = COPY $d0
-    %1(<2 x s32>) = G_BITCAST %0
+    %1(s64) = G_BITCAST %0
 ...
 
 ---
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store-vector-of-ptr-debugloc.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store-vector-of-ptr-debugloc.mir
new file mode 100644
index 0000000000000..4ab9a6c3ab06a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store-vector-of-ptr-debugloc.mir
@@ -0,0 +1,52 @@
+# RUN: llc -mtriple=aarch64-- -run-pass=legalizer -verify-machineinstrs -O0 %s -o - | FileCheck %s
+--- |
+  target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+  target triple = "arm64-apple-ios13.0.0"
+
+  define void @test_debugloc() {
+    ret void
+  }
+
+  !llvm.module.flags = !{!0, !1, !2, !3, !4}
+  !llvm.dbg.cu = !{!5}
+  !llvm.ident = !{!8}
+
+  !0 = !{i32 2, !"SDK Version", [2 x i32] [i32 14, i32 0]}
+  !1 = !{i32 7, !"Dwarf Version", i32 4}
+  !2 = !{i32 2, !"Debug Info Version", i32 3}
+  !3 = !{i32 1, !"wchar_size", i32 4}
+  !4 = !{i32 7, !"PIC Level", i32 2}
+  !5 = distinct !DICompileUnit(language: DW_LANG_C99, file: !6, producer: "clang")
+  !6 = !DIFile(filename: "foo.c", directory: "/")
+  !7 = !{}
+  !8 = !{!"clang"}
+  !9 = distinct !DISubprogram(name: "test_debugloc", scope: !6, file: !6, line: 3, type: !10, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !5, retainedNodes: !7)
+  !10 = !DISubroutineType(types: !7)
+  !11 = !DILocation(line: 4, column: 3, scope: !9)
+  !12 = !DILocation(line: 5, column: 1, scope: !9)
+
+...
+---
+name:            test_debugloc
+alignment:       4
+tracksRegLiveness: true
+liveins:
+  - { reg: '$x0' }
+  - { reg: '$q0' }
+body:             |
+  bb.1:
+    liveins: $q0, $x0
+
+    ; CHECK-LABEL: name: test_debugloc
+    ; CHECK: liveins: $q0, $x0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x p0>) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(<2 x s64>) = G_BITCAST [[COPY]](<2 x p0>), debug-location !DILocation(line: 4, column: 3
+    ; CHECK: G_STORE [[BITCAST]](<2 x s64>), [[COPY1]](p0), debug-location !DILocation(line: 4, column: 3
+    ; CHECK: RET_ReallyLR debug-location !DILocation(line: 5, column: 1
+    %0:_(<2 x p0>) = COPY $q0
+    %1:_(p0) = COPY $x0
+    G_STORE %0(<2 x p0>), %1(p0), debug-location !11 :: (store 16)
+    RET_ReallyLR debug-location !12
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-memlib-debug-loc.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-memlib-debug-loc.mir
new file mode 100644
index 0000000000000..6a5df883acd09
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-memlib-debug-loc.mir
@@ -0,0 +1,60 @@
+# RUN: llc -mtriple=aarch64-- -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+--- |
+  target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+  target triple = "arm64-apple-ios13.0.0"
+
+  define void @test_memset_debug(i8* %ptr, i32 %c, i32 %len) local_unnamed_addr!dbg !9 {
+  entry:
+    %conv = zext i32 %len to i64, !dbg !11
+    %0 = trunc i32 %c to i8, !dbg !11
+    call void @llvm.memset.p0i8.i64(i8* align 1 %ptr, i8 %0, i64 %conv, i1 false) #3, !dbg !11
+    ret void, !dbg !12
+  }
+
+  declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg) #1
+  attributes #1 = { argmemonly nounwind willreturn writeonly }
+
+  !llvm.module.flags = !{!0, !1, !2, !3, !4}
+  !llvm.dbg.cu = !{!5}
+  !llvm.ident = !{!8}
+
+  !0 = !{i32 2, !"SDK Version", [2 x i32] [i32 14, i32 0]}
+  !1 = !{i32 7, !"Dwarf Version", i32 4}
+  !2 = !{i32 2, !"Debug Info Version", i32 3}
+  !3 = !{i32 1, !"wchar_size", i32 4}
+  !4 = !{i32 7, !"PIC Level", i32 2}
+  !5 = distinct !DICompileUnit(language: DW_LANG_C99, file: !6, producer: "clang")
+  !6 = !DIFile(filename: "foo.c", directory: "/")
+  !7 = !{}
+  !8 = !{!"clang"}
+  !9 = distinct !DISubprogram(name: "test_memset_debug", scope: !6, file: !6, line: 3, type: !10, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !5, retainedNodes: !7)
+  !10 = !DISubroutineType(types: !7)
+  !11 = !DILocation(line: 4, column: 3, scope: !9)
+  !12 = !DILocation(line: 5, column: 1, scope: !9)
+
+...
+---
+name:            test_memset_debug
+alignment:       4
+tracksRegLiveness: true
+liveins:
+  - { reg: '$x0' }
+  - { reg: '$w1' }
+  - { reg: '$w2' }
+body:             |
+  bb.1.entry:
+    liveins: $w1, $w2, $x0
+
+    ; We're checking that the BL call has the debug loc of the original intrinsic call.
+    ; CHECK-LABEL: name: test_memset_debug
+    ; CHECK: BL &memset, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $w1, implicit $x2, debug-location !11
+    ; CHECK: RET_ReallyLR debug-location !12
+    %0:_(p0) = COPY $x0
+    %1:_(s32) = COPY $w1
+    %2:_(s32) = COPY $w2
+    %3:_(s64) = G_ZEXT %2(s32), debug-location !11
+    %4:_(s8) = G_TRUNC %1(s32), debug-location !11
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %4(s8), %3(s64), 0, debug-location !11 :: (store 1 into %ir.ptr)
+    RET_ReallyLR debug-location !12
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shift-imm-promote-dloc.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shift-imm-promote-dloc.mir
new file mode 100644
index 0000000000000..54464fe6a6101
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shift-imm-promote-dloc.mir
@@ -0,0 +1,58 @@
+# RUN: llc -mtriple=aarch64-- -run-pass=legalizer -verify-machineinstrs -O0 %s -o - | FileCheck %s
+--- |
+  target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+  target triple = "arm64-apple-ios13.0.0"
+
+  define void @test_shl_imm_promote_debug() {
+    ret void
+  }
+
+  !llvm.module.flags = !{!0, !1, !2, !3, !4}
+  !llvm.dbg.cu = !{!5}
+  !llvm.ident = !{!8}
+
+  !0 = !{i32 2, !"SDK Version", [2 x i32] [i32 14, i32 0]}
+  !1 = !{i32 7, !"Dwarf Version", i32 4}
+  !2 = !{i32 2, !"Debug Info Version", i32 3}
+  !3 = !{i32 1, !"wchar_size", i32 4}
+  !4 = !{i32 7, !"PIC Level", i32 2}
+  !5 = distinct !DICompileUnit(language: DW_LANG_C99, file: !6, producer: "clang")
+  !6 = !DIFile(filename: "foo.c", directory: "/")
+  !7 = !{}
+  !8 = !{!"clang"}
+  !9 = distinct !DISubprogram(name: "test_shl_imm_promote_debug", scope: !6, file: !6, line: 3, type: !10, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !5, retainedNodes: !7)
+  !10 = !DISubroutineType(types: !7)
+  !11 = !DILocation(line: 4, column: 3, scope: !9)
+  !12 = !DILocation(line: 5, column: 1, scope: !9)
+
+...
+---
+name:            test_shl_imm_promote_debug
+alignment:       4
+tracksRegLiveness: true
+liveins:
+  - { reg: '$x0' }
+  - { reg: '$w1' }
+  - { reg: '$w2' }
+body:             |
+  bb.1:
+    liveins: $w0, $w1
+
+    ; Check that the G_LSHR has the right debug loc. This should also check that the G_ZEXT of the constant
+    ; also has the right DL too, but it gets optimized away.
+    ; CHECK-LABEL: name: test_shl_imm_promote_debug
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s64), debug-location !DILocation(line: 4, column: 3
+    ; CHECK: $w0 = COPY [[LSHR]](s32)
+    ; CHECK: RET_ReallyLR debug-location !DILocation(line: 5, column: 1
+    %0:_(p0) = COPY $x0
+    %1:_(s32) = COPY $w1
+    %2:_(s32) = G_CONSTANT i32 8
+    %3:_(s32) = G_LSHR %1(s32), %2(s32), debug-location !11
+    $w0 = COPY %3(s32)
+    RET_ReallyLR debug-location !12
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-simple.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-simple.mir
index 5b76737edc2ba..89e310cc9ecd8 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-simple.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-simple.mir
@@ -35,8 +35,8 @@ body:             |
   ; CHECK:   [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](s64)
   ; CHECK:   [[BITCAST1:%[0-9]+]]:_(s64) = G_BITCAST [[BITCAST]](<2 x s32>)
   ; CHECK:   $x0 = COPY [[BITCAST1]](s64)
-  ; CHECK:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[SELECT3]](s32)
-  ; CHECK:   $w0 = COPY [[BITCAST2]](s32)
+  ; CHECK:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[SELECT3]](s32)
+  ; CHECK:   $w0 = COPY [[BITCAST2]](<2 x s16>)
   ; CHECK:   [[BITCAST3:%[0-9]+]]:_(<4 x s8>) = G_BITCAST [[TRUNC1]](s32)
   ; CHECK:   [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST3]](<4 x s8>)
   ; CHECK:   $w0 = COPY [[BITCAST4]](s32)
@@ -72,8 +72,8 @@ body:             |
     %12:_(<2 x s32>) = G_BITCAST %0(s64)
     %13:_(s64) = G_BITCAST %12(<2 x s32>)
     $x0 = COPY %13(s64)
-    %14:_(s32) = G_BITCAST %10(s32)
-    $w0 = COPY %14(s32)
+    %14:_(<2 x s16>) = G_BITCAST %10(s32)
+    $w0 = COPY %14
     %15:_(<4 x s8>) = G_BITCAST %4(s32)
     %20:_(s32) = G_BITCAST %15(<4 x s8>)
     $w0 = COPY %20(s32)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-vaarg.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-vaarg.mir
index a8aac0210b181..12be076e14cb1 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-vaarg.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-vaarg.mir
@@ -1,10 +1,30 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -O0 -run-pass=legalizer --debugify-and-strip-all-safe --debugify-level=locations %s -o - | FileCheck %s
+# RUN: llc -O0 -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
 
 --- |
   target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
   target triple = "aarch64--"
   define void @test_vaarg() { ret void }
+
+
+  !llvm.module.flags = !{!0, !1, !2, !3, !4}
+  !llvm.dbg.cu = !{!5}
+  !llvm.ident = !{!8}
+
+  !0 = !{i32 2, !"SDK Version", [2 x i32] [i32 14, i32 0]}
+  !1 = !{i32 7, !"Dwarf Version", i32 4}
+  !2 = !{i32 2, !"Debug Info Version", i32 3}
+  !3 = !{i32 1, !"wchar_size", i32 4}
+  !4 = !{i32 7, !"PIC Level", i32 2}
+  !5 = distinct !DICompileUnit(language: DW_LANG_C99, file: !6, producer: "clang")
+  !6 = !DIFile(filename: "foo.c", directory: "/")
+  !7 = !{}
+  !8 = !{!"clang"}
+  !9 = distinct !DISubprogram(name: "test_vaarg", scope: !6, file: !6, line: 3, type: !10, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !5, retainedNodes: !7)
+  !10 = !DISubroutineType(types: !7)
+  !11 = !DILocation(line: 4, column: 3, scope: !9)
+  !12 = !DILocation(line: 5, column: 1, scope: !9)
+
 ...
 
 ---
@@ -13,25 +33,25 @@ body: |
   bb.0:
     ; CHECK-LABEL: name: test_vaarg
     ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
-    ; CHECK: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[COPY]](p0) :: (load 8)
+    ; CHECK: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[COPY]](p0), debug-location !DILocation(line: 4, column: 3, scope: {{.*}}) :: (load 8)
     ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD]], [[C]](s64)
-    ; CHECK: G_STORE [[PTR_ADD]](p0), [[COPY]](p0) :: (store 8)
-    ; CHECK: [[LOAD1:%[0-9]+]]:_(p0) = G_LOAD [[COPY]](p0) :: (load 8)
-    ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD1]], [[C]](s64)
-    ; CHECK: G_STORE [[PTR_ADD1]](p0), [[COPY]](p0) :: (store 8)
-    ; CHECK: [[LOAD2:%[0-9]+]]:_(p0) = G_LOAD [[COPY]](p0) :: (load 8)
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD]], [[C]](s64), debug-location !DILocation(line: 4, column: 3, scope: {{.*}})
+    ; CHECK: G_STORE [[PTR_ADD]](p0), [[COPY]](p0), debug-location !DILocation(line: 4, column: 3, scope: {{.*}}) :: (store 8)
+    ; CHECK: [[LOAD1:%[0-9]+]]:_(p0) = G_LOAD [[COPY]](p0), debug-location !DILocation(line: 5, column: 1, scope: {{.*}}) :: (load 8)
+    ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD1]], [[C]](s64), debug-location !DILocation(line: 5, column: 1, scope: {{.*}})
+    ; CHECK: G_STORE [[PTR_ADD1]](p0), [[COPY]](p0), debug-location !DILocation(line: 5, column: 1, scope: {{.*}}) :: (store 8)
+    ; CHECK: [[LOAD2:%[0-9]+]]:_(p0) = G_LOAD [[COPY]](p0), debug-location !DILocation(line: 4, column: 3, scope: {{.*}}) :: (load 8)
     ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 15
-    ; CHECK: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD2]], [[C1]](s64)
+    ; CHECK: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD2]], [[C1]](s64), debug-location !DILocation(line: 4, column: 3, scope: {{.*}})
     ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -16
-    ; CHECK: [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[PTR_ADD2]], [[C2]](s64)
-    ; CHECK: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTRMASK]], [[C]](s64)
-    ; CHECK: G_STORE [[PTR_ADD3]](p0), [[COPY]](p0) :: (store 8)
+    ; CHECK: [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[PTR_ADD2]], [[C2]](s64), debug-location !DILocation(line: 4, column: 3, scope: {{.*}})
+    ; CHECK: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTRMASK]], [[C]](s64), debug-location !DILocation(line: 4, column: 3, scope: {{.*}})
+    ; CHECK: G_STORE [[PTR_ADD3]](p0), [[COPY]](p0), debug-location !DILocation(line: 4, column: 3, scope: {{.*}}) :: (store 8)
     %0:_(p0) = COPY $x0
 
-    %1:_(s8) = G_VAARG %0(p0), 1
+    %1:_(s8) = G_VAARG %0(p0), 1, debug-location !11
 
-    %2:_(s64) = G_VAARG %0(p0), 8
+    %2:_(s64) = G_VAARG %0(p0), 8, debug-location !12
 
-    %3:_(s64) = G_VAARG %0(p0), 16
+    %3:_(s64) = G_VAARG %0(p0), 16, debug-location !11
 ...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-shuffle-splat.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-shuffle-splat.mir
index 2325b0fb03413..b910e9953193f 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-shuffle-splat.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-shuffle-splat.mir
@@ -328,3 +328,31 @@ body:             |
     %splat:_(<8 x s8>) = G_SHUFFLE_VECTOR %ins(<8 x s8>), %undef, shufflemask(0, 0, 0, 0, 0, 0, 0, 0)
     $d0 = COPY %splat(<8 x s8>)
     RET_ReallyLR implicit $d0
+
+...
+---
+name:            build_vector
+alignment:       4
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $w1, $w2, $w3
+    ; The G_SHUFFLE_VECTOR is fed by a G_BUILD_VECTOR, and the 0th input
+    ; operand is not a constant. We should get a G_DUP.
+    ;
+    ; CHECK-LABEL: name: build_vector
+    ; CHECK: liveins: $w0, $w1, $w2, $w3
+    ; CHECK: %lane:_(s32) = COPY $w0
+    ; CHECK: %shuf:_(<4 x s32>) = G_DUP %lane(s32)
+    ; CHECK: $q0 = COPY %shuf(<4 x s32>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lane:_(s32) = COPY $w0
+    %b:_(s32) = COPY $w1
+    %c:_(s32) = COPY $w2
+    %d:_(s32) = COPY $w3
+    %undef:_(<4 x s32>) = G_IMPLICIT_DEF
+    %buildvec:_(<4 x s32>) = G_BUILD_VECTOR %lane, %b, %c, %d
+    %shuf:_(<4 x s32>) = G_SHUFFLE_VECTOR %buildvec(<4 x s32>), %undef, shufflemask(0, 0, 0, 0)
+    $q0 = COPY %shuf(<4 x s32>)
+    RET_ReallyLR implicit $q0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-bitcast.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-bitcast.mir
index 0846f54289d7d..d9ee37e312b99 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-bitcast.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-bitcast.mir
@@ -22,18 +22,21 @@ legalized:       true
 regBankSelected: true
 registers:
   - { id: 0, class: gpr }
-  - { id: 1, class: gpr }
-
+  - { id: 1, class: fpr }
+  - { id: 2, class: gpr }
 body:             |
   bb.0:
     liveins: $w0
 
     ; CHECK-LABEL: name: bitcast_s32_gpr
     ; CHECK: [[COPY:%[0-9]+]]:gpr32all = COPY $w0
-    ; CHECK: $w0 = COPY [[COPY]]
+    ; CHECK: [[COPY1:%[0-9]+]]:fpr32 = COPY [[COPY]]
+    ; CHECK: [[COPY2:%[0-9]+]]:gpr32all = COPY [[COPY1]]
+    ; CHECK: $w0 = COPY [[COPY2]]
     %0(s32) = COPY $w0
-    %1(s32) = G_BITCAST %0
-    $w0 = COPY %1(s32)
+    %1(<2 x s16>) = G_BITCAST %0
+    %2(s32) = G_BITCAST %1
+    $w0 = COPY %2
 ...
 
 ---
@@ -43,18 +46,21 @@ regBankSelected: true
 
 registers:
   - { id: 0, class: fpr }
-  - { id: 1, class: fpr }
-
+  - { id: 1, class: gpr }
+  - { id: 2, class: fpr }
 body:             |
   bb.0:
     liveins: $s0
 
     ; CHECK-LABEL: name: bitcast_s32_fpr
     ; CHECK: [[COPY:%[0-9]+]]:fpr32 = COPY $s0
-    ; CHECK: $s0 = COPY [[COPY]]
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr32all = COPY [[COPY]]
+    ; CHECK: [[COPY2:%[0-9]+]]:fpr32 = COPY [[COPY1]]
+    ; CHECK: $s0 = COPY [[COPY2]]
     %0(s32) = COPY $s0
-    %1(s32) = G_BITCAST %0
-    $s0 = COPY %1(s32)
+    %1(<2 x s16>) = G_BITCAST %0
+    %2(s32) = G_BITCAST %1
+    $s0 = COPY %2
 ...
 
 ---
@@ -75,8 +81,8 @@ body:             |
     ; CHECK: [[COPY1:%[0-9]+]]:fpr32 = COPY [[COPY]]
     ; CHECK: $s0 = COPY [[COPY1]]
     %0(s32) = COPY $w0
-    %1(s32) = G_BITCAST %0
-    $s0 = COPY %1(s32)
+    %1(<2 x s16>) = G_BITCAST %0
+    $s0 = COPY %1
 ...
 
 ---
@@ -94,9 +100,9 @@ body:             |
 
     ; CHECK-LABEL: name: bitcast_s32_fpr_gpr
     ; CHECK: [[COPY:%[0-9]+]]:fpr32 = COPY $s0
-    ; CHECK: [[COPY1:%[0-9]+]]:gpr32 = COPY [[COPY]]
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr32all = COPY [[COPY]]
     ; CHECK: $w0 = COPY [[COPY1]]
-    %0(s32) = COPY $s0
+    %0(<2 x s16>) = COPY $s0
     %1(s32) = G_BITCAST %0
     $w0 = COPY %1(s32)
 ...
@@ -108,7 +114,8 @@ regBankSelected: true
 
 registers:
   - { id: 0, class: gpr }
-  - { id: 1, class: gpr }
+  - { id: 1, class: fpr }
+  - { id: 2, class: gpr }
 
 body:             |
   bb.0:
@@ -116,10 +123,13 @@ body:             |
 
     ; CHECK-LABEL: name: bitcast_s64_gpr
     ; CHECK: [[COPY:%[0-9]+]]:gpr64all = COPY $x0
-    ; CHECK: $x0 = COPY [[COPY]]
+    ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY [[COPY]]
+    ; CHECK: [[COPY2:%[0-9]+]]:gpr64 = COPY [[COPY1]]
+    ; CHECK: $x0 = COPY [[COPY2]]
     %0(s64) = COPY $x0
-    %1(s64) = G_BITCAST %0
-    $x0 = COPY %1(s64)
+    %1(<2 x s32>) = G_BITCAST %0
+    %2(s64) = G_BITCAST %1
+    $x0 = COPY %2(s64)
 ...
 
 ---
@@ -139,8 +149,8 @@ body:             |
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
     ; CHECK: $d0 = COPY [[COPY]]
     %0(s64) = COPY $d0
-    %1(s64) = G_BITCAST %0
-    $d0 = COPY %1(s64)
+    %1(<2 x s32>) = G_BITCAST %0
+    $d0 = COPY %1
 ...
 
 ---
@@ -160,8 +170,8 @@ body:             |
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY [[COPY]]
     ; CHECK: $d0 = COPY [[COPY1]]
     %0(s64) = COPY $x0
-    %1(s64) = G_BITCAST %0
-    $d0 = COPY %1(s64)
+    %1(<2 x s32>) = G_BITCAST %0
+    $d0 = COPY %1
 ...
 
 ---
@@ -179,11 +189,11 @@ body:             |
 
     ; CHECK-LABEL: name: bitcast_s64_fpr_gpr
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
-    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY [[COPY]]
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64all = COPY [[COPY]]
     ; CHECK: $x0 = COPY [[COPY1]]
     %0(s64) = COPY $d0
-    %1(s64) = G_BITCAST %0
-    $x0 = COPY %1(s64)
+    %1(<2 x s32>) = G_BITCAST %0
+    $x0 = COPY %1
 ...
 
 ---
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-const-vector.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-const-vector.mir
new file mode 100644
index 0000000000000..6a34bea17a422
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-const-vector.mir
@@ -0,0 +1,149 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64-- -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+---
+name:            test_constant_vec_pool_v2f64
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+liveins:
+  - { reg: '$x0' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.1:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: test_constant_vec_pool_v2f64
+    ; CHECK: liveins: $x0
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRQui:%[0-9]+]]:fpr128 = LDRQui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: STRQui [[LDRQui]], [[COPY]], 0 :: (store 16)
+    ; CHECK: RET_ReallyLR
+    %0:gpr(p0) = COPY $x0
+    %3:fpr(s64) = G_FCONSTANT double 5.000000e-01
+    %2:fpr(s64) = G_FCONSTANT double 1.600000e+01
+    %1:fpr(<2 x s64>) = G_BUILD_VECTOR %2(s64), %3(s64)
+    G_STORE %1(<2 x s64>), %0(p0) :: (store 16)
+    RET_ReallyLR
+
+...
+---
+name:            test_constant_vec_pool_v4f32
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+liveins:
+  - { reg: '$x0' }
+frameInfo:
+  maxAlignment:    1
+body:             |
+  bb.1:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: test_constant_vec_pool_v4f32
+    ; CHECK: liveins: $x0
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRQui:%[0-9]+]]:fpr128 = LDRQui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: STRQui [[LDRQui]], [[COPY]], 0 :: (store 16)
+    ; CHECK: RET_ReallyLR
+    %0:gpr(p0) = COPY $x0
+    %3:fpr(s32) = G_FCONSTANT float 5.000000e-01
+    %2:fpr(s32) = G_FCONSTANT float 1.600000e+01
+    %1:fpr(<4 x s32>) = G_BUILD_VECTOR %2(s32), %3(s32), %3(s32), %3(s32)
+    G_STORE %1(<4 x s32>), %0(p0) :: (store 16)
+    RET_ReallyLR
+
+...
+---
+name:            test_constant_vec_pool_v2i64
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+liveins:
+  - { reg: '$x0' }
+frameInfo:
+  maxAlignment:    1
+body:             |
+  bb.1:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: test_constant_vec_pool_v2i64
+    ; CHECK: liveins: $x0
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRQui:%[0-9]+]]:fpr128 = LDRQui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: STRQui [[LDRQui]], [[COPY]], 0 :: (store 16)
+    ; CHECK: RET_ReallyLR
+    %0:gpr(p0) = COPY $x0
+    %3:gpr(s64) = G_CONSTANT i64 67839
+    %2:gpr(s64) = G_CONSTANT i64 12375
+    %1:fpr(<2 x s64>) = G_BUILD_VECTOR %2(s64), %3(s64)
+    G_STORE %1(<2 x s64>), %0(p0) :: (store 16)
+    RET_ReallyLR
+
+...
+---
+name:            test_constant_vec_pool_v4i32
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+liveins:
+  - { reg: '$x0' }
+frameInfo:
+  maxAlignment:    1
+body:             |
+  bb.1:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: test_constant_vec_pool_v4i32
+    ; CHECK: liveins: $x0
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRQui:%[0-9]+]]:fpr128 = LDRQui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: STRQui [[LDRQui]], [[COPY]], 0 :: (store 16)
+    ; CHECK: RET_ReallyLR
+    %0:gpr(p0) = COPY $x0
+    %3:gpr(s32) = G_CONSTANT i32 67839
+    %2:gpr(s32) = G_CONSTANT i32 12375
+    %1:fpr(<4 x s32>) = G_BUILD_VECTOR %2(s32), %3(s32), %2(s32), %3(s32)
+    G_STORE %1(<4 x s32>), %0(p0) :: (store 16)
+    RET_ReallyLR
+
+...
+
+---
+name:            test_constant_vec_pool_v2i32
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+liveins:
+  - { reg: '$x0' }
+frameInfo:
+  maxAlignment:    1
+body:             |
+  bb.1:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: test_constant_vec_pool_v2i32
+    ; CHECK: liveins: $x0
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: STRDui [[LDRDui]], [[COPY]], 0 :: (store 8)
+    ; CHECK: RET_ReallyLR
+    %0:gpr(p0) = COPY $x0
+    %3:gpr(s32) = G_CONSTANT i32 67839
+    %2:gpr(s32) = G_CONSTANT i32 12375
+    %1:fpr(<2 x s32>) = G_BUILD_VECTOR %2(s32), %3(s32)
+    G_STORE %1(<2 x s32>), %0(p0) :: (store 8)
+    RET_ReallyLR
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-redundant-zext.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-redundant-zext.mir
index cda64271c0afa..36132e0badbc0 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-redundant-zext.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-redundant-zext.mir
@@ -93,14 +93,15 @@ body:             |
     ; CHECK-LABEL: name: dont_fold_bitcast
     ; CHECK: liveins: $w0
     ; CHECK: %copy:gpr32all = COPY $w0
-    ; CHECK: %bitcast:gpr32 = COPY %copy
-    ; CHECK: [[SUBREG_TO_REG:%[0-9]+]]:gpr64 = SUBREG_TO_REG 0, %bitcast, %subreg.sub_32
+    ; CHECK: %bitcast1:gpr32 = COPY %copy
+    ; CHECK: [[SUBREG_TO_REG:%[0-9]+]]:gpr64 = SUBREG_TO_REG 0, %bitcast1, %subreg.sub_32
     ; CHECK: %zext:gpr64 = UBFMXri [[SUBREG_TO_REG]], 0, 31
     ; CHECK: $x0 = COPY %zext
     ; CHECK: RET_ReallyLR implicit $x0
     %copy:gpr(s32) = COPY $w0
-    %bitcast:gpr(s32) = G_BITCAST %copy(s32)
-    %zext:gpr(s64) = G_ZEXT %bitcast(s32)
+    %bitcast0:gpr(<4 x s8>) = G_BITCAST %copy(s32)
+    %bitcast1:gpr(s32) = G_BITCAST %bitcast0
+    %zext:gpr(s64) = G_ZEXT %bitcast1(s32)
     $x0 = COPY %zext(s64)
     RET_ReallyLR implicit $x0
 
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-vector-icmp.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-vector-icmp.mir
index 696d8300bbd1d..55313a4313675 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-vector-icmp.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-vector-icmp.mir
@@ -538,18 +538,12 @@ body:             |
     ; CHECK: liveins: $d0, $d1
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
-    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 65535
-    ; CHECK: [[DEF:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF]], [[MOVi32imm]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG]], 1, [[MOVi32imm]]
-    ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr]].dsub
-    ; CHECK: [[ANDv8i8_:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY]], [[COPY2]]
-    ; CHECK: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 65535
-    ; CHECK: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG1:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF1]], [[MOVi32imm1]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr1:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG1]], 1, [[MOVi32imm1]]
-    ; CHECK: [[COPY3:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr1]].dsub
-    ; CHECK: [[ANDv8i8_1:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY1]], [[COPY3]]
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: [[ANDv8i8_:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY]], [[LDRDui]]
+    ; CHECK: [[ADRP1:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui1:%[0-9]+]]:fpr64 = LDRDui [[ADRP1]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: [[ANDv8i8_1:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY1]], [[LDRDui1]]
     ; CHECK: [[CMEQv2i32_:%[0-9]+]]:fpr64 = CMEQv2i32 [[ANDv8i8_]], [[ANDv8i8_1]]
     ; CHECK: $d0 = COPY [[CMEQv2i32_]]
     ; CHECK: RET_ReallyLR implicit $d0
@@ -831,18 +825,12 @@ body:             |
     ; CHECK: liveins: $d0, $d1
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
-    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 65535
-    ; CHECK: [[DEF:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF]], [[MOVi32imm]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG]], 1, [[MOVi32imm]]
-    ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr]].dsub
-    ; CHECK: [[ANDv8i8_:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY]], [[COPY2]]
-    ; CHECK: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 65535
-    ; CHECK: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG1:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF1]], [[MOVi32imm1]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr1:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG1]], 1, [[MOVi32imm1]]
-    ; CHECK: [[COPY3:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr1]].dsub
-    ; CHECK: [[ANDv8i8_1:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY1]], [[COPY3]]
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: [[ANDv8i8_:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY]], [[LDRDui]]
+    ; CHECK: [[ADRP1:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui1:%[0-9]+]]:fpr64 = LDRDui [[ADRP1]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: [[ANDv8i8_1:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY1]], [[LDRDui1]]
     ; CHECK: [[CMEQv2i32_:%[0-9]+]]:fpr64 = CMEQv2i32 [[ANDv8i8_]], [[ANDv8i8_1]]
     ; CHECK: [[NOTv8i8_:%[0-9]+]]:fpr64 = NOTv8i8 [[CMEQv2i32_]]
     ; CHECK: $d0 = COPY [[NOTv8i8_]]
@@ -1126,18 +1114,12 @@ body:             |
     ; CHECK: liveins: $d0, $d1
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
-    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 65535
-    ; CHECK: [[DEF:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF]], [[MOVi32imm]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG]], 1, [[MOVi32imm]]
-    ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr]].dsub
-    ; CHECK: [[ANDv8i8_:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY]], [[COPY2]]
-    ; CHECK: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 65535
-    ; CHECK: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG1:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF1]], [[MOVi32imm1]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr1:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG1]], 1, [[MOVi32imm1]]
-    ; CHECK: [[COPY3:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr1]].dsub
-    ; CHECK: [[ANDv8i8_1:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY1]], [[COPY3]]
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: [[ANDv8i8_:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY]], [[LDRDui]]
+    ; CHECK: [[ADRP1:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui1:%[0-9]+]]:fpr64 = LDRDui [[ADRP1]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: [[ANDv8i8_1:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY1]], [[LDRDui1]]
     ; CHECK: [[CMHIv2i32_:%[0-9]+]]:fpr64 = CMHIv2i32 [[ANDv8i8_]], [[ANDv8i8_1]]
     ; CHECK: $d0 = COPY [[CMHIv2i32_]]
     ; CHECK: RET_ReallyLR implicit $d0
@@ -1416,18 +1398,12 @@ body:             |
     ; CHECK: liveins: $d0, $d1
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
-    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 65535
-    ; CHECK: [[DEF:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF]], [[MOVi32imm]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG]], 1, [[MOVi32imm]]
-    ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr]].dsub
-    ; CHECK: [[ANDv8i8_:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY]], [[COPY2]]
-    ; CHECK: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 65535
-    ; CHECK: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG1:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF1]], [[MOVi32imm1]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr1:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG1]], 1, [[MOVi32imm1]]
-    ; CHECK: [[COPY3:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr1]].dsub
-    ; CHECK: [[ANDv8i8_1:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY1]], [[COPY3]]
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: [[ANDv8i8_:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY]], [[LDRDui]]
+    ; CHECK: [[ADRP1:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui1:%[0-9]+]]:fpr64 = LDRDui [[ADRP1]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: [[ANDv8i8_1:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY1]], [[LDRDui1]]
     ; CHECK: [[CMHSv2i32_:%[0-9]+]]:fpr64 = CMHSv2i32 [[ANDv8i8_]], [[ANDv8i8_1]]
     ; CHECK: $d0 = COPY [[CMHSv2i32_]]
     ; CHECK: RET_ReallyLR implicit $d0
@@ -1706,18 +1682,12 @@ body:             |
     ; CHECK: liveins: $d0, $d1
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
-    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 65535
-    ; CHECK: [[DEF:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF]], [[MOVi32imm]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG]], 1, [[MOVi32imm]]
-    ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr]].dsub
-    ; CHECK: [[ANDv8i8_:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY]], [[COPY2]]
-    ; CHECK: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 65535
-    ; CHECK: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG1:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF1]], [[MOVi32imm1]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr1:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG1]], 1, [[MOVi32imm1]]
-    ; CHECK: [[COPY3:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr1]].dsub
-    ; CHECK: [[ANDv8i8_1:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY1]], [[COPY3]]
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: [[ANDv8i8_:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY]], [[LDRDui]]
+    ; CHECK: [[ADRP1:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui1:%[0-9]+]]:fpr64 = LDRDui [[ADRP1]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: [[ANDv8i8_1:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY1]], [[LDRDui1]]
     ; CHECK: [[CMHIv2i32_:%[0-9]+]]:fpr64 = CMHIv2i32 [[ANDv8i8_1]], [[ANDv8i8_]]
     ; CHECK: $d0 = COPY [[CMHIv2i32_]]
     ; CHECK: RET_ReallyLR implicit $d0
@@ -1996,18 +1966,12 @@ body:             |
     ; CHECK: liveins: $d0, $d1
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
-    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 65535
-    ; CHECK: [[DEF:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF]], [[MOVi32imm]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG]], 1, [[MOVi32imm]]
-    ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr]].dsub
-    ; CHECK: [[ANDv8i8_:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY]], [[COPY2]]
-    ; CHECK: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 65535
-    ; CHECK: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG1:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF1]], [[MOVi32imm1]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr1:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG1]], 1, [[MOVi32imm1]]
-    ; CHECK: [[COPY3:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr1]].dsub
-    ; CHECK: [[ANDv8i8_1:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY1]], [[COPY3]]
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: [[ANDv8i8_:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY]], [[LDRDui]]
+    ; CHECK: [[ADRP1:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui1:%[0-9]+]]:fpr64 = LDRDui [[ADRP1]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: [[ANDv8i8_1:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY1]], [[LDRDui1]]
     ; CHECK: [[CMHSv2i32_:%[0-9]+]]:fpr64 = CMHSv2i32 [[ANDv8i8_1]], [[ANDv8i8_]]
     ; CHECK: $d0 = COPY [[CMHSv2i32_]]
     ; CHECK: RET_ReallyLR implicit $d0
@@ -2288,21 +2252,15 @@ body:             |
     ; CHECK: liveins: $d0, $d1
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
-    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 16
-    ; CHECK: [[DEF:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF]], [[MOVi32imm]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG]], 1, [[MOVi32imm]]
-    ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr]].dsub
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
     ; CHECK: [[SHLv2i32_shift:%[0-9]+]]:fpr64 = SHLv2i32_shift [[COPY]], 16
-    ; CHECK: [[NEGv2i32_:%[0-9]+]]:fpr64 = NEGv2i32 [[COPY2]]
+    ; CHECK: [[NEGv2i32_:%[0-9]+]]:fpr64 = NEGv2i32 [[LDRDui]]
     ; CHECK: [[SSHLv2i32_:%[0-9]+]]:fpr64 = SSHLv2i32 [[SHLv2i32_shift]], [[NEGv2i32_]]
-    ; CHECK: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 16
-    ; CHECK: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG1:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF1]], [[MOVi32imm1]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr1:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG1]], 1, [[MOVi32imm1]]
-    ; CHECK: [[COPY3:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr1]].dsub
+    ; CHECK: [[ADRP1:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui1:%[0-9]+]]:fpr64 = LDRDui [[ADRP1]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
     ; CHECK: [[SHLv2i32_shift1:%[0-9]+]]:fpr64 = SHLv2i32_shift [[COPY1]], 16
-    ; CHECK: [[NEGv2i32_1:%[0-9]+]]:fpr64 = NEGv2i32 [[COPY3]]
+    ; CHECK: [[NEGv2i32_1:%[0-9]+]]:fpr64 = NEGv2i32 [[LDRDui1]]
     ; CHECK: [[SSHLv2i32_1:%[0-9]+]]:fpr64 = SSHLv2i32 [[SHLv2i32_shift1]], [[NEGv2i32_1]]
     ; CHECK: [[CMGTv2i32_:%[0-9]+]]:fpr64 = CMGTv2i32 [[SSHLv2i32_]], [[SSHLv2i32_1]]
     ; CHECK: $d0 = COPY [[CMGTv2i32_]]
@@ -2586,21 +2544,15 @@ body:             |
     ; CHECK: liveins: $d0, $d1
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
-    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 16
-    ; CHECK: [[DEF:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF]], [[MOVi32imm]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG]], 1, [[MOVi32imm]]
-    ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr]].dsub
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
     ; CHECK: [[SHLv2i32_shift:%[0-9]+]]:fpr64 = SHLv2i32_shift [[COPY]], 16
-    ; CHECK: [[NEGv2i32_:%[0-9]+]]:fpr64 = NEGv2i32 [[COPY2]]
+    ; CHECK: [[NEGv2i32_:%[0-9]+]]:fpr64 = NEGv2i32 [[LDRDui]]
     ; CHECK: [[SSHLv2i32_:%[0-9]+]]:fpr64 = SSHLv2i32 [[SHLv2i32_shift]], [[NEGv2i32_]]
-    ; CHECK: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 16
-    ; CHECK: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG1:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF1]], [[MOVi32imm1]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr1:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG1]], 1, [[MOVi32imm1]]
-    ; CHECK: [[COPY3:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr1]].dsub
+    ; CHECK: [[ADRP1:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui1:%[0-9]+]]:fpr64 = LDRDui [[ADRP1]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
     ; CHECK: [[SHLv2i32_shift1:%[0-9]+]]:fpr64 = SHLv2i32_shift [[COPY1]], 16
-    ; CHECK: [[NEGv2i32_1:%[0-9]+]]:fpr64 = NEGv2i32 [[COPY3]]
+    ; CHECK: [[NEGv2i32_1:%[0-9]+]]:fpr64 = NEGv2i32 [[LDRDui1]]
     ; CHECK: [[SSHLv2i32_1:%[0-9]+]]:fpr64 = SSHLv2i32 [[SHLv2i32_shift1]], [[NEGv2i32_1]]
     ; CHECK: [[CMGEv2i32_:%[0-9]+]]:fpr64 = CMGEv2i32 [[SSHLv2i32_]], [[SSHLv2i32_1]]
     ; CHECK: $d0 = COPY [[CMGEv2i32_]]
@@ -2884,21 +2836,15 @@ body:             |
     ; CHECK: liveins: $d0, $d1
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
-    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 16
-    ; CHECK: [[DEF:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF]], [[MOVi32imm]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG]], 1, [[MOVi32imm]]
-    ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr]].dsub
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
     ; CHECK: [[SHLv2i32_shift:%[0-9]+]]:fpr64 = SHLv2i32_shift [[COPY]], 16
-    ; CHECK: [[NEGv2i32_:%[0-9]+]]:fpr64 = NEGv2i32 [[COPY2]]
+    ; CHECK: [[NEGv2i32_:%[0-9]+]]:fpr64 = NEGv2i32 [[LDRDui]]
     ; CHECK: [[SSHLv2i32_:%[0-9]+]]:fpr64 = SSHLv2i32 [[SHLv2i32_shift]], [[NEGv2i32_]]
-    ; CHECK: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 16
-    ; CHECK: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG1:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF1]], [[MOVi32imm1]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr1:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG1]], 1, [[MOVi32imm1]]
-    ; CHECK: [[COPY3:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr1]].dsub
+    ; CHECK: [[ADRP1:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui1:%[0-9]+]]:fpr64 = LDRDui [[ADRP1]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
     ; CHECK: [[SHLv2i32_shift1:%[0-9]+]]:fpr64 = SHLv2i32_shift [[COPY1]], 16
-    ; CHECK: [[NEGv2i32_1:%[0-9]+]]:fpr64 = NEGv2i32 [[COPY3]]
+    ; CHECK: [[NEGv2i32_1:%[0-9]+]]:fpr64 = NEGv2i32 [[LDRDui1]]
     ; CHECK: [[SSHLv2i32_1:%[0-9]+]]:fpr64 = SSHLv2i32 [[SHLv2i32_shift1]], [[NEGv2i32_1]]
     ; CHECK: [[CMGTv2i32_:%[0-9]+]]:fpr64 = CMGTv2i32 [[SSHLv2i32_1]], [[SSHLv2i32_]]
     ; CHECK: $d0 = COPY [[CMGTv2i32_]]
@@ -3182,21 +3128,15 @@ body:             |
     ; CHECK: liveins: $d0, $d1
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
-    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 16
-    ; CHECK: [[DEF:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF]], [[MOVi32imm]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG]], 1, [[MOVi32imm]]
-    ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr]].dsub
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
     ; CHECK: [[SHLv2i32_shift:%[0-9]+]]:fpr64 = SHLv2i32_shift [[COPY]], 16
-    ; CHECK: [[NEGv2i32_:%[0-9]+]]:fpr64 = NEGv2i32 [[COPY2]]
+    ; CHECK: [[NEGv2i32_:%[0-9]+]]:fpr64 = NEGv2i32 [[LDRDui]]
     ; CHECK: [[SSHLv2i32_:%[0-9]+]]:fpr64 = SSHLv2i32 [[SHLv2i32_shift]], [[NEGv2i32_]]
-    ; CHECK: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 16
-    ; CHECK: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG1:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF1]], [[MOVi32imm1]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr1:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG1]], 1, [[MOVi32imm1]]
-    ; CHECK: [[COPY3:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr1]].dsub
+    ; CHECK: [[ADRP1:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui1:%[0-9]+]]:fpr64 = LDRDui [[ADRP1]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
     ; CHECK: [[SHLv2i32_shift1:%[0-9]+]]:fpr64 = SHLv2i32_shift [[COPY1]], 16
-    ; CHECK: [[NEGv2i32_1:%[0-9]+]]:fpr64 = NEGv2i32 [[COPY3]]
+    ; CHECK: [[NEGv2i32_1:%[0-9]+]]:fpr64 = NEGv2i32 [[LDRDui1]]
     ; CHECK: [[SSHLv2i32_1:%[0-9]+]]:fpr64 = SSHLv2i32 [[SHLv2i32_shift1]], [[NEGv2i32_1]]
     ; CHECK: [[CMGEv2i32_:%[0-9]+]]:fpr64 = CMGEv2i32 [[SSHLv2i32_1]], [[SSHLv2i32_]]
     ; CHECK: $d0 = COPY [[CMGEv2i32_]]
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-vector-shift.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-vector-shift.mir
index bbdcdec39bbe6..29e6f442542e2 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-vector-shift.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-vector-shift.mir
@@ -86,12 +86,9 @@ body:             |
     ; CHECK-LABEL: name: shl_v2i32_imm_out_of_range
     ; CHECK: liveins: $d0
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
-    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 40
-    ; CHECK: [[DEF:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF]], [[MOVi32imm]], %subreg.ssub
-    ; CHECK: [[INSvi32gpr:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG]], 1, [[MOVi32imm]]
-    ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr]].dsub
-    ; CHECK: [[USHLv2i32_:%[0-9]+]]:fpr64 = USHLv2i32 [[COPY]], [[COPY1]]
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: [[USHLv2i32_:%[0-9]+]]:fpr64 = USHLv2i32 [[COPY]], [[LDRDui]]
     ; CHECK: $d0 = COPY [[USHLv2i32_]]
     ; CHECK: RET_ReallyLR implicit $d0
     %0:fpr(<2 x s32>) = COPY $d0
@@ -251,12 +248,9 @@ body:             |
     ; CHECK-LABEL: name: shl_v2i64_imm_out_of_range
     ; CHECK: liveins: $q0
     ; CHECK: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
-    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 70
-    ; CHECK: [[SUBREG_TO_REG:%[0-9]+]]:gpr64 = SUBREG_TO_REG 0, [[MOVi32imm]], %subreg.sub_32
-    ; CHECK: [[DEF:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; CHECK: [[INSERT_SUBREG:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF]], [[SUBREG_TO_REG]], %subreg.dsub
-    ; CHECK: [[INSvi64gpr:%[0-9]+]]:fpr128 = INSvi64gpr [[INSERT_SUBREG]], 1, [[SUBREG_TO_REG]]
-    ; CHECK: [[USHLv2i64_:%[0-9]+]]:fpr128 = USHLv2i64 [[COPY]], [[INSvi64gpr]]
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) %const.0
+    ; CHECK: [[LDRQui:%[0-9]+]]:fpr128 = LDRQui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) %const.0
+    ; CHECK: [[USHLv2i64_:%[0-9]+]]:fpr128 = USHLv2i64 [[COPY]], [[LDRQui]]
     ; CHECK: $q0 = COPY [[USHLv2i64_]]
     ; CHECK: RET_ReallyLR implicit $q0
     %0:fpr(<2 x s64>) = COPY $q0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/sext-inreg-ldrow-16b.mir b/llvm/test/CodeGen/AArch64/GlobalISel/sext-inreg-ldrow-16b.mir
new file mode 100644
index 0000000000000..34b471d4dc99a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/sext-inreg-ldrow-16b.mir
@@ -0,0 +1,92 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64-- -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+--- |
+  target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64"
+
+  @x = global i32 -32768, align 4
+
+  define i32 @check_sext_not_lost(i32* %ptr) {
+  entry:
+    %ptr.addr = alloca i32*, align 8
+    store i32* %ptr, i32** %ptr.addr, align 8
+    %0 = load i32*, i32** %ptr.addr, align 8
+    %1 = load i32, i32* @x, align 4
+    %sub = sub nsw i32 %1, 32768
+    %conv = trunc i32 %sub to i16
+    %idxprom = sext i16 %conv to i64
+    %arrayidx = getelementptr inbounds i32, i32* %0, i64 %idxprom
+    %2 = load i32, i32* %arrayidx, align 4
+    ret i32 %2
+  }
+
+...
+---
+name:            check_sext_not_lost
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gpr }
+  - { id: 1, class: gpr }
+  - { id: 2, class: gpr }
+  - { id: 3, class: gpr }
+  - { id: 4, class: gpr }
+  - { id: 5, class: gpr }
+  - { id: 6, class: gpr }
+  - { id: 7, class: _ }
+  - { id: 8, class: gpr }
+  - { id: 9, class: _ }
+  - { id: 10, class: gpr }
+  - { id: 11, class: gpr }
+  - { id: 12, class: _ }
+  - { id: 13, class: gpr }
+  - { id: 14, class: gpr }
+  - { id: 15, class: gpr64 }
+  - { id: 16, class: gpr }
+liveins:
+  - { reg: '$x0' }
+frameInfo:
+  maxAlignment:    8
+  maxCallFrameSize: 0
+stack:
+  - { id: 0, name: ptr.addr, size: 8, alignment: 8 }
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: check_sext_not_lost
+    ; CHECK: liveins: $x0
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK: STRXui [[COPY]], %stack.0.ptr.addr, 0 :: (store 8 into %ir.ptr.addr)
+    ; CHECK: [[LDRXui:%[0-9]+]]:gpr64common = LDRXui %stack.0.ptr.addr, 0 :: (dereferenceable load 8 from %ir.ptr.addr)
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64common = ADRP target-flags(aarch64-page) @x
+    ; CHECK: [[LDRWui:%[0-9]+]]:gpr32common = LDRWui [[ADRP]], target-flags(aarch64-pageoff, aarch64-nc) @x :: (dereferenceable load 4 from @x)
+    ; CHECK: [[SUBSWri:%[0-9]+]]:gpr32 = nsw SUBSWri [[LDRWui]], 8, 12, implicit-def $nzcv
+    ; CHECK: [[DEF:%[0-9]+]]:gpr64all = IMPLICIT_DEF
+    ; CHECK: [[INSERT_SUBREG:%[0-9]+]]:gpr64 = INSERT_SUBREG [[DEF]], [[SUBSWri]], %subreg.sub_32
+    ; CHECK: [[SBFMXri:%[0-9]+]]:gpr64 = SBFMXri [[INSERT_SUBREG]], 0, 15
+    ; CHECK: [[LDRWroX:%[0-9]+]]:gpr32 = LDRWroX [[LDRXui]], [[SBFMXri]], 0, 1 :: (load 4 from %ir.arrayidx)
+    ; CHECK: $w0 = COPY [[LDRWroX]]
+    ; CHECK: RET_ReallyLR implicit $w0
+    %0:gpr(p0) = COPY $x0
+    %1:gpr(p0) = G_FRAME_INDEX %stack.0.ptr.addr
+    G_STORE %0(p0), %1(p0) :: (store 8 into %ir.ptr.addr)
+    %2:gpr(p0) = G_LOAD %1(p0) :: (dereferenceable load 8 from %ir.ptr.addr)
+    %15:gpr64(p0) = ADRP target-flags(aarch64-page) @x
+    %4:gpr(p0) = G_ADD_LOW %15(p0), target-flags(aarch64-pageoff, aarch64-nc) @x
+    %3:gpr(s32) = G_LOAD %4(p0) :: (dereferenceable load 4 from @x)
+    %5:gpr(s32) = G_CONSTANT i32 32768
+    %6:gpr(s32) = nsw G_SUB %3, %5
+    %16:gpr(s64) = G_ANYEXT %6(s32)
+    %8:gpr(s64) = G_SEXT_INREG %16, 16
+    %14:gpr(s64) = G_CONSTANT i64 2
+    %10:gpr(s64) = G_SHL %8, %14(s64)
+    %11:gpr(p0) = G_PTR_ADD %2, %10(s64)
+    %13:gpr(s32) = G_LOAD %11(p0) :: (load 4 from %ir.arrayidx)
+    $w0 = COPY %13(s32)
+    RET_ReallyLR implicit $w0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll b/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
index 05a273f5f2d9b..2878811b063ab 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
@@ -211,7 +211,7 @@ define <2 x double> @ins1f2(<1 x double> %tmp1, <2 x double> %tmp2) {
 define <2 x double> @ins1f2_args_flipped(<2 x double> %tmp2, <1 x double> %tmp1) {
 ; CHECK-LABEL: ins1f2_args_flipped:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill:  def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
 ; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    ret
   %tmp3 = extractelement <1 x double> %tmp1, i32 0
@@ -1088,56 +1088,73 @@ define <2 x float> @test_bitcastv1f64tov2f32(<1 x i64> %a) #0 {
 ; Test insert element into an undef vector
 define <8 x i8> @scalar_to_vector.v8i8(i8 %a) {
 ; CHECK-LABEL: scalar_to_vector.v8i8:
-; CHECK: fmov {{s[0-9]+}}, {{w[0-9]+}}
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    ret
   %b = insertelement <8 x i8> undef, i8 %a, i32 0
   ret <8 x i8> %b
 }
 
 define <16 x i8> @scalar_to_vector.v16i8(i8 %a) {
 ; CHECK-LABEL: scalar_to_vector.v16i8:
-; CHECK: fmov {{s[0-9]+}}, {{w[0-9]+}}
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    ret
   %b = insertelement <16 x i8> undef, i8 %a, i32 0
   ret <16 x i8> %b
 }
 
 define <4 x i16> @scalar_to_vector.v4i16(i16 %a) {
 ; CHECK-LABEL: scalar_to_vector.v4i16:
-; CHECK: fmov {{s[0-9]+}}, {{w[0-9]+}}
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    ret
   %b = insertelement <4 x i16> undef, i16 %a, i32 0
   ret <4 x i16> %b
 }
 
 define <8 x i16> @scalar_to_vector.v8i16(i16 %a) {
 ; CHECK-LABEL: scalar_to_vector.v8i16:
-; CHECK: fmov {{s[0-9]+}}, {{w[0-9]+}}
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    ret
   %b = insertelement <8 x i16> undef, i16 %a, i32 0
   ret <8 x i16> %b
 }
 
 define <2 x i32> @scalar_to_vector.v2i32(i32 %a) {
 ; CHECK-LABEL: scalar_to_vector.v2i32:
-; CHECK: fmov {{s[0-9]+}}, {{w[0-9]+}}
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    ret
   %b = insertelement <2 x i32> undef, i32 %a, i32 0
   ret <2 x i32> %b
 }
 
 define <4 x i32> @scalar_to_vector.v4i32(i32 %a) {
 ; CHECK-LABEL: scalar_to_vector.v4i32:
-; CHECK: fmov {{s[0-9]+}}, {{w[0-9]+}}
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    ret
   %b = insertelement <4 x i32> undef, i32 %a, i32 0
   ret <4 x i32> %b
 }
 
 define <2 x i64> @scalar_to_vector.v2i64(i64 %a) {
 ; CHECK-LABEL: scalar_to_vector.v2i64:
-; CHECK: fmov {{d[0-9]+}}, {{x[0-9]+}}
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ret
   %b = insertelement <2 x i64> undef, i64 %a, i32 0
   ret <2 x i64> %b
 }
 
 define <8 x i8> @testDUP.v1i8(<1 x i8> %a) {
 ; CHECK-LABEL: testDUP.v1i8:
-; CHECK: dup v0.8b, v0.b[0]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v0.8b, v0.b[0]
+; CHECK-NEXT:    ret
   %b = extractelement <1 x i8> %a, i32 0
   %c = insertelement <8 x i8> undef, i8 %b, i32 0
   %d = insertelement <8 x i8> %c, i8 %b, i32 1
@@ -1152,7 +1169,10 @@ define <8 x i8> @testDUP.v1i8(<1 x i8> %a) {
 
 define <8 x i16> @testDUP.v1i16(<1 x i16> %a) {
 ; CHECK-LABEL: testDUP.v1i16:
-; CHECK: dup v0.8h, v0.h[0]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v0.8h, v0.h[0]
+; CHECK-NEXT:    ret
   %b = extractelement <1 x i16> %a, i32 0
   %c = insertelement <8 x i16> undef, i16 %b, i32 0
   %d = insertelement <8 x i16> %c, i16 %b, i32 1
@@ -1167,7 +1187,10 @@ define <8 x i16> @testDUP.v1i16(<1 x i16> %a) {
 
 define <4 x i32> @testDUP.v1i32(<1 x i32> %a) {
 ; CHECK-LABEL: testDUP.v1i32:
-; CHECK: dup v0.4s, v0.s[0]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-NEXT:    ret
   %b = extractelement <1 x i32> %a, i32 0
   %c = insertelement <4 x i32> undef, i32 %b, i32 0
   %d = insertelement <4 x i32> %c, i32 %b, i32 1
diff --git a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
index cd650e1debf86..a5d223cc8aefb 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
@@ -1,8 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s
 
 define <8 x i8> @shadd8b(<8 x i8>* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: shadd8b:
-;CHECK: shadd.8b
+; CHECK-LABEL: shadd8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    shadd.8b v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = load <8 x i8>, <8 x i8>* %B
 	%tmp3 = call <8 x i8> @llvm.aarch64.neon.shadd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
@@ -10,8 +15,12 @@ define <8 x i8> @shadd8b(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 }
 
 define <16 x i8> @shadd16b(<16 x i8>* %A, <16 x i8>* %B) nounwind {
-;CHECK-LABEL: shadd16b:
-;CHECK: shadd.16b
+; CHECK-LABEL: shadd16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    shadd.16b v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = load <16 x i8>, <16 x i8>* %B
 	%tmp3 = call <16 x i8> @llvm.aarch64.neon.shadd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
@@ -19,8 +28,12 @@ define <16 x i8> @shadd16b(<16 x i8>* %A, <16 x i8>* %B) nounwind {
 }
 
 define <4 x i16> @shadd4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: shadd4h:
-;CHECK: shadd.4h
+; CHECK-LABEL: shadd4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    shadd.4h v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = load <4 x i16>, <4 x i16>* %B
 	%tmp3 = call <4 x i16> @llvm.aarch64.neon.shadd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
@@ -28,8 +41,12 @@ define <4 x i16> @shadd4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 }
 
 define <8 x i16> @shadd8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: shadd8h:
-;CHECK: shadd.8h
+; CHECK-LABEL: shadd8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    shadd.8h v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = load <8 x i16>, <8 x i16>* %B
 	%tmp3 = call <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
@@ -37,8 +54,12 @@ define <8 x i16> @shadd8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 }
 
 define <2 x i32> @shadd2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: shadd2s:
-;CHECK: shadd.2s
+; CHECK-LABEL: shadd2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    shadd.2s v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <2 x i32>, <2 x i32>* %A
 	%tmp2 = load <2 x i32>, <2 x i32>* %B
 	%tmp3 = call <2 x i32> @llvm.aarch64.neon.shadd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
@@ -46,8 +67,12 @@ define <2 x i32> @shadd2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 }
 
 define <4 x i32> @shadd4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: shadd4s:
-;CHECK: shadd.4s
+; CHECK-LABEL: shadd4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    shadd.4s v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = load <4 x i32>, <4 x i32>* %B
 	%tmp3 = call <4 x i32> @llvm.aarch64.neon.shadd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
@@ -55,8 +80,12 @@ define <4 x i32> @shadd4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
 }
 
 define <8 x i8> @uhadd8b(<8 x i8>* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: uhadd8b:
-;CHECK: uhadd.8b
+; CHECK-LABEL: uhadd8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    uhadd.8b v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = load <8 x i8>, <8 x i8>* %B
 	%tmp3 = call <8 x i8> @llvm.aarch64.neon.uhadd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
@@ -64,8 +93,12 @@ define <8 x i8> @uhadd8b(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 }
 
 define <16 x i8> @uhadd16b(<16 x i8>* %A, <16 x i8>* %B) nounwind {
-;CHECK-LABEL: uhadd16b:
-;CHECK: uhadd.16b
+; CHECK-LABEL: uhadd16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    uhadd.16b v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = load <16 x i8>, <16 x i8>* %B
 	%tmp3 = call <16 x i8> @llvm.aarch64.neon.uhadd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
@@ -73,8 +106,12 @@ define <16 x i8> @uhadd16b(<16 x i8>* %A, <16 x i8>* %B) nounwind {
 }
 
 define <4 x i16> @uhadd4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: uhadd4h:
-;CHECK: uhadd.4h
+; CHECK-LABEL: uhadd4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    uhadd.4h v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = load <4 x i16>, <4 x i16>* %B
 	%tmp3 = call <4 x i16> @llvm.aarch64.neon.uhadd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
@@ -82,8 +119,12 @@ define <4 x i16> @uhadd4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 }
 
 define <8 x i16> @uhadd8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: uhadd8h:
-;CHECK: uhadd.8h
+; CHECK-LABEL: uhadd8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    uhadd.8h v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = load <8 x i16>, <8 x i16>* %B
 	%tmp3 = call <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
@@ -91,8 +132,12 @@ define <8 x i16> @uhadd8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 }
 
 define <2 x i32> @uhadd2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: uhadd2s:
-;CHECK: uhadd.2s
+; CHECK-LABEL: uhadd2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    uhadd.2s v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <2 x i32>, <2 x i32>* %A
 	%tmp2 = load <2 x i32>, <2 x i32>* %B
 	%tmp3 = call <2 x i32> @llvm.aarch64.neon.uhadd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
@@ -100,8 +145,12 @@ define <2 x i32> @uhadd2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 }
 
 define <4 x i32> @uhadd4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: uhadd4s:
-;CHECK: uhadd.4s
+; CHECK-LABEL: uhadd4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    uhadd.4s v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = load <4 x i32>, <4 x i32>* %B
 	%tmp3 = call <4 x i32> @llvm.aarch64.neon.uhadd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
@@ -125,8 +174,12 @@ declare <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16>, <8 x i16>) nounwind
 declare <4 x i32> @llvm.aarch64.neon.uhadd.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i8> @srhadd8b(<8 x i8>* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: srhadd8b:
-;CHECK: srhadd.8b
+; CHECK-LABEL: srhadd8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    srhadd.8b v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = load <8 x i8>, <8 x i8>* %B
 	%tmp3 = call <8 x i8> @llvm.aarch64.neon.srhadd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
@@ -134,8 +187,12 @@ define <8 x i8> @srhadd8b(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 }
 
 define <16 x i8> @srhadd16b(<16 x i8>* %A, <16 x i8>* %B) nounwind {
-;CHECK-LABEL: srhadd16b:
-;CHECK: srhadd.16b
+; CHECK-LABEL: srhadd16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    srhadd.16b v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = load <16 x i8>, <16 x i8>* %B
 	%tmp3 = call <16 x i8> @llvm.aarch64.neon.srhadd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
@@ -143,8 +200,12 @@ define <16 x i8> @srhadd16b(<16 x i8>* %A, <16 x i8>* %B) nounwind {
 }
 
 define <4 x i16> @srhadd4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: srhadd4h:
-;CHECK: srhadd.4h
+; CHECK-LABEL: srhadd4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    srhadd.4h v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = load <4 x i16>, <4 x i16>* %B
 	%tmp3 = call <4 x i16> @llvm.aarch64.neon.srhadd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
@@ -152,8 +213,12 @@ define <4 x i16> @srhadd4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 }
 
 define <8 x i16> @srhadd8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: srhadd8h:
-;CHECK: srhadd.8h
+; CHECK-LABEL: srhadd8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    srhadd.8h v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = load <8 x i16>, <8 x i16>* %B
 	%tmp3 = call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
@@ -161,8 +226,12 @@ define <8 x i16> @srhadd8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 }
 
 define <2 x i32> @srhadd2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: srhadd2s:
-;CHECK: srhadd.2s
+; CHECK-LABEL: srhadd2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    srhadd.2s v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <2 x i32>, <2 x i32>* %A
 	%tmp2 = load <2 x i32>, <2 x i32>* %B
 	%tmp3 = call <2 x i32> @llvm.aarch64.neon.srhadd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
@@ -170,8 +239,12 @@ define <2 x i32> @srhadd2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 }
 
 define <4 x i32> @srhadd4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: srhadd4s:
-;CHECK: srhadd.4s
+; CHECK-LABEL: srhadd4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    srhadd.4s v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = load <4 x i32>, <4 x i32>* %B
 	%tmp3 = call <4 x i32> @llvm.aarch64.neon.srhadd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
@@ -179,8 +252,12 @@ define <4 x i32> @srhadd4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
 }
 
 define <8 x i8> @urhadd8b(<8 x i8>* %A, <8 x i8>* %B) nounwind {
-;CHECK-LABEL: urhadd8b:
-;CHECK: urhadd.8b
+; CHECK-LABEL: urhadd8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    urhadd.8b v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = load <8 x i8>, <8 x i8>* %B
 	%tmp3 = call <8 x i8> @llvm.aarch64.neon.urhadd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
@@ -188,8 +265,12 @@ define <8 x i8> @urhadd8b(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 }
 
 define <16 x i8> @urhadd16b(<16 x i8>* %A, <16 x i8>* %B) nounwind {
-;CHECK-LABEL: urhadd16b:
-;CHECK: urhadd.16b
+; CHECK-LABEL: urhadd16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    urhadd.16b v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = load <16 x i8>, <16 x i8>* %B
 	%tmp3 = call <16 x i8> @llvm.aarch64.neon.urhadd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
@@ -197,8 +278,12 @@ define <16 x i8> @urhadd16b(<16 x i8>* %A, <16 x i8>* %B) nounwind {
 }
 
 define <4 x i16> @urhadd4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
-;CHECK-LABEL: urhadd4h:
-;CHECK: urhadd.4h
+; CHECK-LABEL: urhadd4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    urhadd.4h v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = load <4 x i16>, <4 x i16>* %B
 	%tmp3 = call <4 x i16> @llvm.aarch64.neon.urhadd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
@@ -206,8 +291,12 @@ define <4 x i16> @urhadd4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 }
 
 define <8 x i16> @urhadd8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: urhadd8h:
-;CHECK: urhadd.8h
+; CHECK-LABEL: urhadd8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    urhadd.8h v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = load <8 x i16>, <8 x i16>* %B
 	%tmp3 = call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
@@ -215,8 +304,12 @@ define <8 x i16> @urhadd8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 }
 
 define <2 x i32> @urhadd2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
-;CHECK-LABEL: urhadd2s:
-;CHECK: urhadd.2s
+; CHECK-LABEL: urhadd2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    urhadd.2s v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <2 x i32>, <2 x i32>* %A
 	%tmp2 = load <2 x i32>, <2 x i32>* %B
 	%tmp3 = call <2 x i32> @llvm.aarch64.neon.urhadd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
@@ -224,14 +317,210 @@ define <2 x i32> @urhadd2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 }
 
 define <4 x i32> @urhadd4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
-;CHECK-LABEL: urhadd4s:
-;CHECK: urhadd.4s
+; CHECK-LABEL: urhadd4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    urhadd.4s v0, v0, v1
+; CHECK-NEXT:    ret
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = load <4 x i32>, <4 x i32>* %B
 	%tmp3 = call <4 x i32> @llvm.aarch64.neon.urhadd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
 	ret <4 x i32> %tmp3
 }
 
+define void @testLowerToSRHADD8b(<8 x i8> %src1, <8 x i8> %src2, <8 x i8>* %dest) nounwind {
+; CHECK-LABEL: testLowerToSRHADD8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    srhadd.8b v0, v0, v1
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+  %sextsrc1 = sext <8 x i8> %src1 to <8 x i16>
+  %sextsrc2 = sext <8 x i8> %src2 to <8 x i16>
+  %add1 = add <8 x i16> %sextsrc1, %sextsrc2
+  %add2 = add <8 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %resulti16 = lshr <8 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %result = trunc <8 x i16> %resulti16 to <8 x i8>
+  store <8 x i8> %result, <8 x i8>* %dest, align 8
+  ret void
+}
+
+define void @testLowerToSRHADD4h(<4 x i16> %src1, <4 x i16> %src2, <4 x i16>* %dest) nounwind {
+; CHECK-LABEL: testLowerToSRHADD4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    srhadd.4h v0, v0, v1
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+  %sextsrc1 = sext <4 x i16> %src1 to <4 x i32>
+  %sextsrc2 = sext <4 x i16> %src2 to <4 x i32>
+  %add1 = add <4 x i32> %sextsrc1, %sextsrc2
+  %add2 = add <4 x i32> %add1, <i32 1, i32 1, i32 1, i32 1>
+  %resulti16 = lshr <4 x i32> %add2, <i32 1, i32 1, i32 1, i32 1>
+  %result = trunc <4 x i32> %resulti16 to <4 x i16>
+  store <4 x i16> %result, <4 x i16>* %dest, align 8
+  ret void
+}
+
+define void @testLowerToSRHADD2s(<2 x i32> %src1, <2 x i32> %src2, <2 x i32>* %dest) nounwind {
+; CHECK-LABEL: testLowerToSRHADD2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    srhadd.2s v0, v0, v1
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+  %sextsrc1 = sext <2 x i32> %src1 to <2 x i64>
+  %sextsrc2 = sext <2 x i32> %src2 to <2 x i64>
+  %add1 = add <2 x i64> %sextsrc1, %sextsrc2
+  %add2 = add <2 x i64> %add1, <i64 1, i64 1>
+  %resulti16 = lshr <2 x i64> %add2, <i64 1, i64 1>
+  %result = trunc <2 x i64> %resulti16 to <2 x i32>
+  store <2 x i32> %result, <2 x i32>* %dest, align 8
+  ret void
+}
+
+define void @testLowerToSRHADD16b(<16 x i8> %src1, <16 x i8> %src2, <16 x i8>* %dest) nounwind {
+; CHECK-LABEL: testLowerToSRHADD16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    srhadd.16b v0, v0, v1
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+  %sextsrc1 = sext <16 x i8> %src1 to <16 x i16>
+  %sextsrc2 = sext <16 x i8> %src2 to <16 x i16>
+  %add1 = add <16 x i16> %sextsrc1, %sextsrc2
+  %add2 = add <16 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %resulti16 = lshr <16 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %result = trunc <16 x i16> %resulti16 to <16 x i8>
+  store <16 x i8> %result, <16 x i8>* %dest, align 16
+  ret void
+}
+
+define void @testLowerToSRHADD8h(<8 x i16> %src1, <8 x i16> %src2, <8 x i16>* %dest) nounwind {
+; CHECK-LABEL: testLowerToSRHADD8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    srhadd.8h v0, v0, v1
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+  %sextsrc1 = sext <8 x i16> %src1 to <8 x i32>
+  %sextsrc2 = sext <8 x i16> %src2 to <8 x i32>
+  %add1 = add <8 x i32> %sextsrc1, %sextsrc2
+  %add2 = add <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %resulti16 = lshr <8 x i32> %add2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %result = trunc <8 x i32> %resulti16 to <8 x i16>
+  store <8 x i16> %result, <8 x i16>* %dest, align 16
+  ret void
+}
+
+define void @testLowerToSRHADD4s(<4 x i32> %src1, <4 x i32> %src2, <4 x i32>* %dest) nounwind {
+; CHECK-LABEL: testLowerToSRHADD4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    srhadd.4s v0, v0, v1
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+  %sextsrc1 = sext <4 x i32> %src1 to <4 x i64>
+  %sextsrc2 = sext <4 x i32> %src2 to <4 x i64>
+  %add1 = add <4 x i64> %sextsrc1, %sextsrc2
+  %add2 = add <4 x i64> %add1, <i64 1, i64 1, i64 1, i64 1>
+  %resulti16 = lshr <4 x i64> %add2, <i64 1, i64 1, i64 1, i64 1>
+  %result = trunc <4 x i64> %resulti16 to <4 x i32>
+  store <4 x i32> %result, <4 x i32>* %dest, align 16
+  ret void
+}
+
+define void @testLowerToURHADD8b(<8 x i8> %src1, <8 x i8> %src2, <8 x i8>* %dest) nounwind {
+; CHECK-LABEL: testLowerToURHADD8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    urhadd.8b v0, v0, v1
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+  %zextsrc1 = zext <8 x i8> %src1 to <8 x i16>
+  %zextsrc2 = zext <8 x i8> %src2 to <8 x i16>
+  %add1 = add <8 x i16> %zextsrc1, %zextsrc2
+  %add2 = add <8 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %resulti16 = lshr <8 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %result = trunc <8 x i16> %resulti16 to <8 x i8>
+  store <8 x i8> %result, <8 x i8>* %dest, align 8
+  ret void
+}
+
+define void @testLowerToURHADD4h(<4 x i16> %src1, <4 x i16> %src2, <4 x i16>* %dest) nounwind {
+; CHECK-LABEL: testLowerToURHADD4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    urhadd.4h v0, v0, v1
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+  %zextsrc1 = zext <4 x i16> %src1 to <4 x i32>
+  %zextsrc2 = zext <4 x i16> %src2 to <4 x i32>
+  %add1 = add <4 x i32> %zextsrc1, %zextsrc2
+  %add2 = add <4 x i32> %add1, <i32 1, i32 1, i32 1, i32 1>
+  %resulti16 = lshr <4 x i32> %add2, <i32 1, i32 1, i32 1, i32 1>
+  %result = trunc <4 x i32> %resulti16 to <4 x i16>
+  store <4 x i16> %result, <4 x i16>* %dest, align 8
+  ret void
+}
+
+define void @testLowerToURHADD2s(<2 x i32> %src1, <2 x i32> %src2, <2 x i32>* %dest) nounwind {
+; CHECK-LABEL: testLowerToURHADD2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    urhadd.2s v0, v0, v1
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+  %zextsrc1 = zext <2 x i32> %src1 to <2 x i64>
+  %zextsrc2 = zext <2 x i32> %src2 to <2 x i64>
+  %add1 = add <2 x i64> %zextsrc1, %zextsrc2
+  %add2 = add <2 x i64> %add1, <i64 1, i64 1>
+  %resulti16 = lshr <2 x i64> %add2, <i64 1, i64 1>
+  %result = trunc <2 x i64> %resulti16 to <2 x i32>
+  store <2 x i32> %result, <2 x i32>* %dest, align 8
+  ret void
+}
+
+define void @testLowerToURHADD16b(<16 x i8> %src1, <16 x i8> %src2, <16 x i8>* %dest) nounwind {
+; CHECK-LABEL: testLowerToURHADD16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    urhadd.16b v0, v0, v1
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+  %zextsrc1 = zext <16 x i8> %src1 to <16 x i16>
+  %zextsrc2 = zext <16 x i8> %src2 to <16 x i16>
+  %add1 = add <16 x i16> %zextsrc1, %zextsrc2
+  %add2 = add <16 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %resulti16 = lshr <16 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %result = trunc <16 x i16> %resulti16 to <16 x i8>
+  store <16 x i8> %result, <16 x i8>* %dest, align 16
+  ret void
+}
+
+define void @testLowerToURHADD8h(<8 x i16> %src1, <8 x i16> %src2, <8 x i16>* %dest) nounwind {
+; CHECK-LABEL: testLowerToURHADD8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    urhadd.8h v0, v0, v1
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+  %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
+  %zextsrc2 = zext <8 x i16> %src2 to <8 x i32>
+  %add1 = add <8 x i32> %zextsrc1, %zextsrc2
+  %add2 = add <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %resulti16 = lshr <8 x i32> %add2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %result = trunc <8 x i32> %resulti16 to <8 x i16>
+  store <8 x i16> %result, <8 x i16>* %dest, align 16
+  ret void
+}
+
+define void @testLowerToURHADD4s(<4 x i32> %src1, <4 x i32> %src2, <4 x i32>* %dest) nounwind {
+; CHECK-LABEL: testLowerToURHADD4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    urhadd.4s v0, v0, v1
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+  %zextsrc1 = zext <4 x i32> %src1 to <4 x i64>
+  %zextsrc2 = zext <4 x i32> %src2 to <4 x i64>
+  %add1 = add <4 x i64> %zextsrc1, %zextsrc2
+  %add2 = add <4 x i64> %add1, <i64 1, i64 1, i64 1, i64 1>
+  %resulti16 = lshr <4 x i64> %add2, <i64 1, i64 1, i64 1, i64 1>
+  %result = trunc <4 x i64> %resulti16 to <4 x i32>
+  store <4 x i32> %result, <4 x i32>* %dest, align 16
+  ret void
+}
+
 declare <8 x i8>  @llvm.aarch64.neon.srhadd.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
 declare <4 x i16> @llvm.aarch64.neon.srhadd.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
 declare <2 x i32> @llvm.aarch64.neon.srhadd.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
diff --git a/llvm/test/CodeGen/AArch64/bf16-convert-intrinsics.ll b/llvm/test/CodeGen/AArch64/bf16-convert-intrinsics.ll
new file mode 100644
index 0000000000000..f95c2957f8637
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/bf16-convert-intrinsics.ll
@@ -0,0 +1,34 @@
+; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-arm-none-eabi -mattr=+neon -mattr=+bf16 | FileCheck %s
+
+declare bfloat @llvm.aarch64.neon.bfcvt(float)
+declare <8 x bfloat> @llvm.aarch64.neon.bfcvtn(<4 x float>)
+declare <8 x bfloat> @llvm.aarch64.neon.bfcvtn2(<8 x bfloat>, <4 x float>)
+
+; CHECK-LABEL: test_vcvth_bf16_f32
+; CHECK:      bfcvt h0, s0
+; CHECK-NEXT: ret
+define bfloat @test_vcvth_bf16_f32(float %a) {
+entry:
+  %vcvth_bf16_f32 = call bfloat @llvm.aarch64.neon.bfcvt(float %a)
+  ret bfloat %vcvth_bf16_f32
+}
+
+; CHECK-LABEL: test_vcvtq_low_bf16_f32
+; CHECK:      bfcvtn v0.4h, v0.4s
+; CHECK-NEXT: ret
+define <8 x bfloat> @test_vcvtq_low_bf16_f32(<4 x float> %a) {
+entry:
+  %cvt = call <8 x bfloat> @llvm.aarch64.neon.bfcvtn(<4 x float> %a)
+  ret <8 x bfloat> %cvt
+}
+
+; CHECK-LABEL: test_vcvtq_high_bf16_f32
+; CHECK:      bfcvtn2 v1.8h, v0.4s
+; CHECK-NEXT: mov v0.16b, v1.16b
+; CHECK-NEXT: ret
+define <8 x bfloat> @test_vcvtq_high_bf16_f32(<4 x float> %a, <8 x bfloat> %inactive) {
+entry:
+  %cvt = call <8 x bfloat> @llvm.aarch64.neon.bfcvtn2(<8 x bfloat> %inactive, <4 x float> %a)
+  ret <8 x bfloat> %cvt
+}
+
diff --git a/llvm/test/CodeGen/AArch64/bf16-vector-shuffle.ll b/llvm/test/CodeGen/AArch64/bf16-vector-shuffle.ll
index df078030e08b1..5aca916a49791 100644
--- a/llvm/test/CodeGen/AArch64/bf16-vector-shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-vector-shuffle.ll
@@ -163,3 +163,87 @@ entry:
   %vgetq_lane = extractelement <8 x bfloat> %v, i32 7
   ret bfloat %vgetq_lane
 }
+
+; vcopy_lane_bf16(a, 1, b, 3);
+define <4 x bfloat> @test_vcopy_lane_bf16_v1(<4 x bfloat> %a, <4 x bfloat> %b) nounwind {
+; CHECK-LABEL: test_vcopy_lane_bf16_v1:
+; CHECK-NEXT:    mov v0.h[1], v1.h[3]
+; CHECK-NEXT:    ret
+entry:
+  %vset_lane = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 0, i32 7, i32 2, i32 3>
+  ret <4 x bfloat> %vset_lane
+}
+
+; vcopy_lane_bf16(a, 2, b, 0);
+define <4 x bfloat> @test_vcopy_lane_bf16_v2(<4 x bfloat> %a, <4 x bfloat> %b) nounwind {
+; CHECK-LABEL: test_vcopy_lane_bf16_v2:
+; CHECK-NEXT:    mov v0.h[2], v1.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %vset_lane = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 0, i32 1, i32 4, i32 3>
+  ret <4 x bfloat> %vset_lane
+}
+
+; vcopyq_lane_bf16(a, 0, b, 2);
+define <8 x bfloat> @test_vcopyq_lane_bf16_v1(<8 x bfloat> %a, <4 x bfloat> %b) nounwind {
+; CHECK-LABEL: test_vcopyq_lane_bf16_v1:
+; CHECK-NEXT:    mov v0.h[0], v1.h[2]
+; CHECK-NEXT:    ret
+entry:
+  %0 = shufflevector <4 x bfloat> %b, <4 x bfloat> undef, <8 x i32> <i32 undef, i32 undef, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %vset_lane = shufflevector <8 x bfloat> %a, <8 x bfloat> %0, <8 x i32> <i32 10, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x bfloat> %vset_lane
+}
+
+; vcopyq_lane_bf16(a, 6, b, 0);
+define <8 x bfloat> @test_vcopyq_lane_bf16_v2(<8 x bfloat> %a, <4 x bfloat> %b) nounwind {
+; CHECK-LABEL: test_vcopyq_lane_bf16_v2:
+; CHECK-NEXT:    mov v0.h[6], v1.h[0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = shufflevector <4 x bfloat> %b, <4 x bfloat> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %vset_lane = shufflevector <8 x bfloat> %a, <8 x bfloat> %0, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 7>
+  ret <8 x bfloat> %vset_lane
+}
+
+; vcopy_laneq_bf16(a, 0, b, 7);
+define <4 x bfloat> @test_vcopy_laneq_bf16_v1(<4 x bfloat> %a, <8 x bfloat> %b) nounwind {
+; CHECK-LABEL: test_vcopy_laneq_bf16_v1:
+; CHECK-NEXT:    mov v0.h[0], v1.h[7]
+; CHECK-NEXT:    ret
+entry:
+  %vgetq_lane = extractelement <8 x bfloat> %b, i32 7
+  %vset_lane = insertelement <4 x bfloat> %a, bfloat %vgetq_lane, i32 0
+  ret <4 x bfloat> %vset_lane
+}
+
+; vcopy_laneq_bf16(a, 3, b, 4);
+define <4 x bfloat> @test_vcopy_laneq_bf16_v2(<4 x bfloat> %a, <8 x bfloat> %b) nounwind {
+; CHECK-LABEL: test_vcopy_laneq_bf16_v2:
+; CHECK-NEXT:    mov v0.h[3], v1.h[4]
+; CHECK-NEXT:    ret
+entry:
+  %vgetq_lane = extractelement <8 x bfloat> %b, i32 4
+  %vset_lane = insertelement <4 x bfloat> %a, bfloat %vgetq_lane, i32 3
+  ret <4 x bfloat> %vset_lane
+}
+
+; vcopyq_laneq_bf16(a, 3, b, 7);
+define <8 x bfloat> @test_vcopyq_laneq_bf16_v1(<8 x bfloat> %a, <8 x bfloat> %b) nounwind {
+; CHECK-LABEL: test_vcopyq_laneq_bf16_v1:
+; CHECK-NEXT:    mov v0.h[3], v1.h[7]
+; CHECK-NEXT:    ret
+entry:
+  %vset_lane = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 15, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x bfloat> %vset_lane
+}
+
+; vcopyq_laneq_bf16(a, 6, b, 2);
+define <8 x bfloat> @test_vcopyq_laneq_bf16_v2(<8 x bfloat> %a, <8 x bfloat> %b) nounwind {
+; CHECK-LABEL: test_vcopyq_laneq_bf16_v2:
+; CHECK-NEXT:    mov v0.h[6], v1.h[2]
+; CHECK-NEXT:    ret
+entry:
+  %vset_lane = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 10, i32 7>
+  ret <8 x bfloat> %vset_lane
+}
diff --git a/llvm/test/CodeGen/AArch64/callbr-asm-label.ll b/llvm/test/CodeGen/AArch64/callbr-asm-label.ll
index 1ff1464da8db4..5a0656e48bb9a 100644
--- a/llvm/test/CodeGen/AArch64/callbr-asm-label.ll
+++ b/llvm/test/CodeGen/AArch64/callbr-asm-label.ll
@@ -6,9 +6,9 @@ define i32 @test1() {
 ; CHECK-LABEL: test1:
 ; CHECK:         .word b
 ; CHECK-NEXT:    .word .Ltmp0
-; CHECK-LABEL: .LBB0_1: // %cleanup
-; CHECK-LABEL: .Ltmp0:
-; CHECK-LABEL: .LBB0_2: // %indirect
+; CHECK: // %bb.1:
+; CHECK: .Ltmp0:
+; CHECK: .LBB0_2: // %indirect
 entry:
   callbr void asm sideeffect "1:\0A\09.word b, ${0:l}\0A\09", "X"(i8* blockaddress(@test1, %indirect))
           to label %cleanup [label %indirect]
@@ -32,7 +32,7 @@ entry:
 if.then:
 ; CHECK:       .word b
 ; CHECK-NEXT:  .word .Ltmp2
-; CHECK-LABEL: .Ltmp2:
+; CHECK:       .Ltmp2:
 ; CHECK-NEXT:  .LBB1_3: // %if.end6
   callbr void asm sideeffect "1:\0A\09.word b, ${0:l}\0A\09", "X"(i8* blockaddress(@test2, %if.end6))
           to label %if.then4 [label %if.end6]
@@ -48,7 +48,7 @@ if.end6:
   br i1 %phitmp, label %if.end10, label %if.then9
 
 if.then9:
-; CHECK-LABEL: .Ltmp4:
+; CHECK: .Ltmp4:
 ; CHECK-NEXT:  .LBB1_5: // %l_yes
   callbr void asm sideeffect "", "X"(i8* blockaddress(@test2, %l_yes))
           to label %if.end10 [label %l_yes]
diff --git a/llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll b/llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll
index 91ea4edf489b5..2891143ce58fb 100644
--- a/llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll
+++ b/llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll
@@ -9,9 +9,8 @@
 ; CHECK-LABEL: <test1>:
 ; CHECK-LABEL: <$d.1>:
 ; CHECK-LABEL: <$x.2>:
-; CHECK-NEXT:    b 0x30 <$x.4+0x4>
+; CHECK-NEXT:    b 0x2c <$x.4>
 ; CHECK-LABEL: <$x.4>:
-; CHECK-NEXT:    b 0x30 <$x.4+0x4>
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ldr x30, [sp], #16
 ; CHECK-NEXT:    ret
@@ -75,7 +74,6 @@ define hidden i32 @test2() local_unnamed_addr {
 ; CHECK-LABEL: <$x.10>:
 ; CHECK-NEXT:    b {{.*}} <test3+0x18>
 ; CHECK-LABEL: <$x.12>:
-; CHECK-NEXT:    b {{.*}} <$x.12+0x4>
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ldr x30, [sp], #16
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/combine-loads.ll b/llvm/test/CodeGen/AArch64/combine-loads.ll
index fc0b921b0b3a1..22a71f5701f12 100644
--- a/llvm/test/CodeGen/AArch64/combine-loads.ll
+++ b/llvm/test/CodeGen/AArch64/combine-loads.ll
@@ -4,10 +4,8 @@
 define <2 x i64> @z(i64* nocapture nonnull readonly %p) {
 ; CHECK-LABEL: z:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    // implicit-def: $q0
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    mov v0.d[1], x8
+; CHECK-NEXT:    adrp x8, .LCPI0_0
+; CHECK-NEXT:    ldr q0, [x8, :lo12:.LCPI0_0]
 ; CHECK-NEXT:    ldr x8, [x0]
 ; CHECK-NEXT:    ldr x9, [x0, #8]
 ; CHECK-NEXT:    mov v0.d[0], x8
diff --git a/llvm/test/CodeGen/AArch64/cpus.ll b/llvm/test/CodeGen/AArch64/cpus.ll
index 83eb14ae76873..3d4ad97b7fb29 100644
--- a/llvm/test/CodeGen/AArch64/cpus.ll
+++ b/llvm/test/CodeGen/AArch64/cpus.ll
@@ -15,6 +15,9 @@
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=cortex-a75 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=cortex-a76ae 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=cortex-a76 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=cortex-a77 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=cortex-a78 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=cortex-x1 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=neoverse-e1 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=neoverse-n1 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=exynos-m3 2>&1 | FileCheck %s
diff --git a/llvm/test/CodeGen/AArch64/fadd-combines.ll b/llvm/test/CodeGen/AArch64/fadd-combines.ll
index 61d0ecc04e03d..2ff4858307802 100644
--- a/llvm/test/CodeGen/AArch64/fadd-combines.ll
+++ b/llvm/test/CodeGen/AArch64/fadd-combines.ll
@@ -207,13 +207,31 @@ define double @fadd_fma_fmul_1(double %a, double %b, double %c, double %d, doubl
   ret double %a2
 }
 
-; Minimum FMF, commute final add operands, change type.
+; Minimum FMF - the 1st fadd is contracted because that combines
+; fmul+fadd as specified by the order of operations; the 2nd fadd
+; requires reassociation to fuse with c*d.
+
+define float @fadd_fma_fmul_fmf(float %a, float %b, float %c, float %d, float %n0) nounwind {
+; CHECK-LABEL: fadd_fma_fmul_fmf:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmadd s2, s2, s3, s4
+; CHECK-NEXT:    fmadd s0, s0, s1, s2
+; CHECK-NEXT:    ret
+  %m1 = fmul float %a, %b
+  %m2 = fmul float %c, %d
+  %a1 = fadd contract float %m1, %m2
+  %a2 = fadd reassoc float %n0, %a1
+  ret float %a2
+}
+
+; Not minimum FMF.
 
 define float @fadd_fma_fmul_2(float %a, float %b, float %c, float %d, float %n0) nounwind {
 ; CHECK-LABEL: fadd_fma_fmul_2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmadd s2, s2, s3, s4
+; CHECK-NEXT:    fmul s2, s2, s3
 ; CHECK-NEXT:    fmadd s0, s0, s1, s2
+; CHECK-NEXT:    fadd s0, s4, s0
 ; CHECK-NEXT:    ret
   %m1 = fmul float %a, %b
   %m2 = fmul float %c, %d
diff --git a/llvm/test/CodeGen/AArch64/funcptr_cast.ll b/llvm/test/CodeGen/AArch64/funcptr_cast.ll
index 506485ec5b0cb..19a20c8bcf4fb 100644
--- a/llvm/test/CodeGen/AArch64/funcptr_cast.ll
+++ b/llvm/test/CodeGen/AArch64/funcptr_cast.ll
@@ -1,10 +1,12 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=aarch64-none-linux-gnu | FileCheck %s
 
 define i8 @test() {
-; CHECK-LABEL: @test
-; CHECK: adrp {{x[0-9]+}}, foo
-; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, :lo12:foo
-; CHECK: ldrb w0, [{{x[0-9]+}}]
+; CHECK-LABEL: test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adrp x8, foo
+; CHECK-NEXT:    ldrb w0, [x8, :lo12:foo]
+; CHECK-NEXT:    ret
 entry:
   %0 = load i8, i8* bitcast (void (...)* @foo to i8*), align 1
   ret i8 %0
diff --git a/llvm/test/CodeGen/AArch64/global-alignment.ll b/llvm/test/CodeGen/AArch64/global-alignment.ll
index 5e820b8bb3037..84bf20d673650 100644
--- a/llvm/test/CodeGen/AArch64/global-alignment.ll
+++ b/llvm/test/CodeGen/AArch64/global-alignment.ll
@@ -51,11 +51,11 @@ define i64 @test_var32_alias() {
 ; CHECK-LABEL: test_var32_alias:
   %addr = bitcast [3 x i32]* @alias to i64*
 
-  ; Test that we can find the alignment for aliases.
+  ; We don't know anything about the alignment of aliases.
   %val = load i64, i64* %addr
 ; CHECK: adrp x[[HIBITS:[0-9]+]], alias
-; CHECK-NOT: add x[[HIBITS]]
-; CHECK: ldr x0, [x[[HIBITS]], {{#?}}:lo12:alias]
+; CHECK: add x[[ADDR:[0-9]+]], x[[HIBITS]], {{#?}}:lo12:alias
+; CHECK: ldr x0, [x[[ADDR]]]
 
   ret i64 %val
 }
diff --git a/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll b/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll
index 786dc4bd0aef1..816465f9eaa17 100644
--- a/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll
+++ b/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll
@@ -59,6 +59,36 @@ define <vscale x 4 x i64> @sdiv_split_i64(<vscale x 4 x i64> %a, <vscale x 4 x i
   ret <vscale x 4 x i64> %div
 }
 
+;
+; SREM
+;
+
+define <vscale x 4 x i32> @srem_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: srem_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    sdiv z2.s, p0/m, z2.s, z1.s
+; CHECK-NEXT:    mul z2.s, p0/m, z2.s, z1.s
+; CHECK-NEXT:    sub z0.s, z0.s, z2.s
+; CHECK-NEXT:    ret
+  %div = srem <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %div
+}
+
+define <vscale x 2 x i64> @srem_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: srem_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    sdiv z2.d, p0/m, z2.d, z1.d
+; CHECK-NEXT:    mul z2.d, p0/m, z2.d, z1.d
+; CHECK-NEXT:    sub z0.d, z0.d, z2.d
+; CHECK-NEXT:    ret
+  %div = srem <vscale x 2 x i64> %a, %b
+  ret <vscale x 2 x i64> %div
+}
+
 ;
 ; UDIV
 ;
@@ -117,6 +147,37 @@ define <vscale x 4 x i64> @udiv_split_i64(<vscale x 4 x i64> %a, <vscale x 4 x i
   ret <vscale x 4 x i64> %div
 }
 
+
+;
+; UREM
+;
+
+define <vscale x 4 x i32> @urem_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: urem_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    udiv z2.s, p0/m, z2.s, z1.s
+; CHECK-NEXT:    mul z2.s, p0/m, z2.s, z1.s
+; CHECK-NEXT:    sub z0.s, z0.s, z2.s
+; CHECK-NEXT:    ret
+  %div = urem <vscale x 4 x i32> %a, %b
+  ret <vscale x 4 x i32> %div
+}
+
+define <vscale x 2 x i64> @urem_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: urem_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    udiv z2.d, p0/m, z2.d, z1.d
+; CHECK-NEXT:    mul z2.d, p0/m, z2.d, z1.d
+; CHECK-NEXT:    sub z0.d, z0.d, z2.d
+; CHECK-NEXT:    ret
+  %div = urem <vscale x 2 x i64> %a, %b
+  ret <vscale x 2 x i64> %div
+}
+
 ;
 ; SMIN
 ;
@@ -165,6 +226,95 @@ define <vscale x 2 x i64> @smin_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b
   ret <vscale x 2 x i64> %min
 }
 
+define <vscale x 32 x i8> @smin_split_i8(<vscale x 32 x i8> %a, <vscale x 32 x i8> %b, <vscale x 32 x i8> %c) {
+; CHECK-LABEL: smin_split_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    smin z0.b, p0/m, z0.b, z2.b
+; CHECK-NEXT:    smin z1.b, p0/m, z1.b, z3.b
+; CHECK-NEXT:    ret
+  %cmp = icmp slt <vscale x 32 x i8> %a, %b
+  %min = select <vscale x 32 x i1> %cmp, <vscale x 32 x i8> %a, <vscale x 32 x i8> %b
+  ret <vscale x 32 x i8> %min
+}
+
+define <vscale x 32 x i16> @smin_split_i16(<vscale x 32 x i16> %a, <vscale x 32 x i16> %b, <vscale x 32 x i16> %c) {
+; CHECK-LABEL: smin_split_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    smin z0.h, p0/m, z0.h, z4.h
+; CHECK-NEXT:    smin z1.h, p0/m, z1.h, z5.h
+; CHECK-NEXT:    smin z2.h, p0/m, z2.h, z6.h
+; CHECK-NEXT:    smin z3.h, p0/m, z3.h, z7.h
+; CHECK-NEXT:    ret
+  %cmp = icmp slt <vscale x 32 x i16> %a, %b
+  %min = select <vscale x 32 x i1> %cmp, <vscale x 32 x i16> %a, <vscale x 32 x i16> %b
+  ret <vscale x 32 x i16> %min
+}
+
+define <vscale x 8 x i32> @smin_split_i32(<vscale x 8 x i32> %a, <vscale x 8 x i32> %b, <vscale x 8 x i32> %c) {
+; CHECK-LABEL: smin_split_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    smin z0.s, p0/m, z0.s, z2.s
+; CHECK-NEXT:    smin z1.s, p0/m, z1.s, z3.s
+; CHECK-NEXT:    ret
+  %cmp = icmp slt <vscale x 8 x i32> %a, %b
+  %min = select <vscale x 8 x i1> %cmp, <vscale x 8 x i32> %a, <vscale x 8 x i32> %b
+  ret <vscale x 8 x i32> %min
+}
+
+define <vscale x 4 x i64> @smin_split_i64(<vscale x 4 x i64> %a, <vscale x 4 x i64> %b, <vscale x 4 x i64> %c) {
+; CHECK-LABEL: smin_split_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    smin z0.d, p0/m, z0.d, z2.d
+; CHECK-NEXT:    smin z1.d, p0/m, z1.d, z3.d
+; CHECK-NEXT:    ret
+  %cmp = icmp slt <vscale x 4 x i64> %a, %b
+  %min = select <vscale x 4 x i1> %cmp, <vscale x 4 x i64> %a, <vscale x 4 x i64> %b
+  ret <vscale x 4 x i64> %min
+}
+
+define <vscale x 8 x i8> @smin_promote_i8(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b, <vscale x 8 x i8> %c) {
+; CHECK-LABEL: smin_promote_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    sxtb z1.h, p0/m, z1.h
+; CHECK-NEXT:    sxtb z0.h, p0/m, z0.h
+; CHECK-NEXT:    smin z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %cmp = icmp slt <vscale x 8 x i8> %a, %b
+  %min = select <vscale x 8 x i1> %cmp, <vscale x 8 x i8> %a, <vscale x 8 x i8> %b
+  ret <vscale x 8 x i8> %min
+}
+
+define <vscale x 4 x i16> @smin_promote_i16(<vscale x 4 x i16> %a, <vscale x 4 x i16> %b, <vscale x 4 x i16> %c) {
+; CHECK-LABEL: smin_promote_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    sxth z1.s, p0/m, z1.s
+; CHECK-NEXT:    sxth z0.s, p0/m, z0.s
+; CHECK-NEXT:    smin z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %cmp = icmp slt <vscale x 4 x i16> %a, %b
+  %min = select <vscale x 4 x i1> %cmp, <vscale x 4 x i16> %a, <vscale x 4 x i16> %b
+  ret <vscale x 4 x i16> %min
+}
+
+define <vscale x 2 x i32> @smin_promote_i32(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscale x 2 x i32> %c) {
+; CHECK-LABEL: smin_promote_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    sxtw z1.d, p0/m, z1.d
+; CHECK-NEXT:    sxtw z0.d, p0/m, z0.d
+; CHECK-NEXT:    smin z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %cmp = icmp slt <vscale x 2 x i32> %a, %b
+  %min = select <vscale x 2 x i1> %cmp, <vscale x 2 x i32> %a, <vscale x 2 x i32> %b
+  ret <vscale x 2 x i32> %min
+}
+
 ;
 ; UMIN
 ;
@@ -213,6 +363,31 @@ define <vscale x 2 x i64> @umin_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b
   ret <vscale x 2 x i64> %min
 }
 
+define <vscale x 4 x i64> @umin_split_i64(<vscale x 4 x i64> %a, <vscale x 4 x i64> %b, <vscale x 4 x i64> %c) {
+; CHECK-LABEL: umin_split_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    umin z0.d, p0/m, z0.d, z2.d
+; CHECK-NEXT:    umin z1.d, p0/m, z1.d, z3.d
+; CHECK-NEXT:    ret
+  %cmp = icmp ult <vscale x 4 x i64> %a, %b
+  %min = select <vscale x 4 x i1> %cmp, <vscale x 4 x i64> %a, <vscale x 4 x i64> %b
+  ret <vscale x 4 x i64> %min
+}
+
+define <vscale x 8 x i8> @umin_promote_i8(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b, <vscale x 8 x i8> %c) {
+; CHECK-LABEL: umin_promote_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    and z1.h, z1.h, #0xff
+; CHECK-NEXT:    and z0.h, z0.h, #0xff
+; CHECK-NEXT:    umin z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %cmp = icmp ult <vscale x 8 x i8> %a, %b
+  %min = select <vscale x 8 x i1> %cmp, <vscale x 8 x i8> %a, <vscale x 8 x i8> %b
+  ret <vscale x 8 x i8> %min
+}
+
 ;
 ; SMAX
 ;
@@ -224,8 +399,8 @@ define <vscale x 16 x i8> @smax_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b,
 ; CHECK-NEXT:    smax z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    ret
   %cmp = icmp sgt <vscale x 16 x i8> %a, %b
-  %min = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b
-  ret <vscale x 16 x i8> %min
+  %max = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b
+  ret <vscale x 16 x i8> %max
 }
 
 define <vscale x 8 x i16> @smax_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
@@ -235,8 +410,8 @@ define <vscale x 8 x i16> @smax_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b
 ; CHECK-NEXT:    smax z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
   %cmp = icmp sgt <vscale x 8 x i16> %a, %b
-  %min = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b
-  ret <vscale x 8 x i16> %min
+  %max = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b
+  ret <vscale x 8 x i16> %max
 }
 
 define <vscale x 4 x i32> @smax_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
@@ -246,8 +421,8 @@ define <vscale x 4 x i32> @smax_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b
 ; CHECK-NEXT:    smax z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
   %cmp = icmp sgt <vscale x 4 x i32> %a, %b
-  %min = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b
-  ret <vscale x 4 x i32> %min
+  %max = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b
+  ret <vscale x 4 x i32> %max
 }
 
 define <vscale x 2 x i64> @smax_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) {
@@ -257,8 +432,33 @@ define <vscale x 2 x i64> @smax_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b
 ; CHECK-NEXT:    smax z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    ret
   %cmp = icmp sgt <vscale x 2 x i64> %a, %b
-  %min = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b
-  ret <vscale x 2 x i64> %min
+  %max = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b
+  ret <vscale x 2 x i64> %max
+}
+
+define <vscale x 8 x i32> @smax_split_i32(<vscale x 8 x i32> %a, <vscale x 8 x i32> %b, <vscale x 8 x i32> %c) {
+; CHECK-LABEL: smax_split_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    smax z0.s, p0/m, z0.s, z2.s
+; CHECK-NEXT:    smax z1.s, p0/m, z1.s, z3.s
+; CHECK-NEXT:    ret
+  %cmp = icmp sgt <vscale x 8 x i32> %a, %b
+  %max = select <vscale x 8 x i1> %cmp, <vscale x 8 x i32> %a, <vscale x 8 x i32> %b
+  ret <vscale x 8 x i32> %max
+}
+
+define <vscale x 4 x i16> @smax_promote_i16(<vscale x 4 x i16> %a, <vscale x 4 x i16> %b, <vscale x 4 x i16> %c) {
+; CHECK-LABEL: smax_promote_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    sxth z1.s, p0/m, z1.s
+; CHECK-NEXT:    sxth z0.s, p0/m, z0.s
+; CHECK-NEXT:    smax z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %cmp = icmp sgt <vscale x 4 x i16> %a, %b
+  %max = select <vscale x 4 x i1> %cmp, <vscale x 4 x i16> %a, <vscale x 4 x i16> %b
+  ret <vscale x 4 x i16> %max
 }
 
 ;
@@ -272,8 +472,8 @@ define <vscale x 16 x i8> @umax_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b,
 ; CHECK-NEXT:    umax z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    ret
   %cmp = icmp ugt <vscale x 16 x i8> %a, %b
-  %min = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b
-  ret <vscale x 16 x i8> %min
+  %max = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b
+  ret <vscale x 16 x i8> %max
 }
 
 define <vscale x 8 x i16> @umax_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
@@ -283,8 +483,8 @@ define <vscale x 8 x i16> @umax_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b
 ; CHECK-NEXT:    umax z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
   %cmp = icmp ugt <vscale x 8 x i16> %a, %b
-  %min = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b
-  ret <vscale x 8 x i16> %min
+  %max = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b
+  ret <vscale x 8 x i16> %max
 }
 
 define <vscale x 4 x i32> @umax_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
@@ -294,8 +494,8 @@ define <vscale x 4 x i32> @umax_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b
 ; CHECK-NEXT:    umax z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
   %cmp = icmp ugt <vscale x 4 x i32> %a, %b
-  %min = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b
-  ret <vscale x 4 x i32> %min
+  %max = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b
+  ret <vscale x 4 x i32> %max
 }
 
 define <vscale x 2 x i64> @umax_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) {
@@ -305,8 +505,33 @@ define <vscale x 2 x i64> @umax_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b
 ; CHECK-NEXT:    umax z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    ret
   %cmp = icmp ugt <vscale x 2 x i64> %a, %b
-  %min = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b
-  ret <vscale x 2 x i64> %min
+  %max = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b
+  ret <vscale x 2 x i64> %max
+}
+
+define <vscale x 16 x i16> @umax_split_i16(<vscale x 16 x i16> %a, <vscale x 16 x i16> %b, <vscale x 16 x i16> %c) {
+; CHECK-LABEL: umax_split_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    umax z0.h, p0/m, z0.h, z2.h
+; CHECK-NEXT:    umax z1.h, p0/m, z1.h, z3.h
+; CHECK-NEXT:    ret
+  %cmp = icmp ugt <vscale x 16 x i16> %a, %b
+  %max = select <vscale x 16 x i1> %cmp, <vscale x 16 x i16> %a, <vscale x 16 x i16> %b
+  ret <vscale x 16 x i16> %max
+}
+
+define <vscale x 2 x i32> @umax_promote_i32(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscale x 2 x i32> %c) {
+; CHECK-LABEL: umax_promote_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    and z1.d, z1.d, #0xffffffff
+; CHECK-NEXT:    and z0.d, z0.d, #0xffffffff
+; CHECK-NEXT:    umax z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %cmp = icmp ugt <vscale x 2 x i32> %a, %b
+  %max = select <vscale x 2 x i1> %cmp, <vscale x 2 x i32> %a, <vscale x 2 x i32> %b
+  ret <vscale x 2 x i32> %max
 }
 
 ;
@@ -508,3 +733,31 @@ define <vscale x 8 x i32> @lsr_split_i32(<vscale x 8 x i32> %a, <vscale x 8 x i3
   %shr = lshr <vscale x 8 x i32> %a, %b
   ret <vscale x 8 x i32> %shr
 }
+
+;
+; CMP
+;
+
+define <vscale x 32 x i1> @cmp_split_32(<vscale x 32 x i8> %a, <vscale x 32 x i8> %b) {
+; CHECK-LABEL: cmp_split_32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    cmpgt p0.b, p1/z, z2.b, z0.b
+; CHECK-NEXT:    cmpgt p1.b, p1/z, z3.b, z1.b
+; CHECK-NEXT:    ret
+  %cmp = icmp slt <vscale x 32 x i8> %a, %b
+  ret <vscale x 32 x i1> %cmp
+}
+
+define <vscale x 64 x i1> @cmp_split_64(<vscale x 64 x i8> %a, <vscale x 64 x i8> %b) {
+; CHECK-LABEL: cmp_split_64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p3.b
+; CHECK-NEXT:    cmpgt p0.b, p3/z, z0.b, z4.b
+; CHECK-NEXT:    cmpgt p1.b, p3/z, z1.b, z5.b
+; CHECK-NEXT:    cmpgt p2.b, p3/z, z2.b, z6.b
+; CHECK-NEXT:    cmpgt p3.b, p3/z, z3.b, z7.b
+; CHECK-NEXT:    ret
+  %cmp = icmp sgt <vscale x 64 x i8> %a, %b
+  ret <vscale x 64 x i1> %cmp
+}
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-cfi.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-cfi.ll
index 49e538bce1721..3b61588f41ed9 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-cfi.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-cfi.ll
@@ -82,10 +82,8 @@ define void @c() "sign-return-address"="all" "sign-return-address-key"="b_key" n
 
 ; CHECK:            [[OUTLINED_FUNC]]
 ; CHECK:            // %bb.0:
-; CHECK-NEXT:           .cfi_b_key_frame
 ; V8A-NEXT:             hint #27
 ; V83A-NEXT:            pacibsp
-; CHECK-NEXT:           .cfi_negate_ra_state
 ; V8A:                  hint #31
 ; V83A:                 autibsp
 ; V8A-NEXT, V83A-NEXT:  ret
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-same-key-b.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-same-key-b.ll
index de32f059b807e..128ac6dc4b456 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-same-key-b.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-same-scope-same-key-b.ll
@@ -72,10 +72,8 @@ define void @c() "sign-return-address"="all" "sign-return-address-key"="b_key" n
 
 ; CHECK-LABEL:      OUTLINED_FUNCTION_0:
 ; CHECK:            // %bb.0:
-; CHECK-NEXT:           .cfi_b_key_frame
 ; V8A-NEXT:             hint #27
 ; V83A-NEXT:            pacibsp
-; CHECK-NEXT:           .cfi_negate_ra_state
 ; V8A:                  hint #31
 ; V83A:                 autibsp
 ; CHECK-NEXT:           ret
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-v8-3.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-v8-3.ll
index 5d728aa378f86..2c66bcb6f4e0e 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-v8-3.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-v8-3.ll
@@ -77,7 +77,6 @@ attributes #0 = { "sign-return-address"="all"
 
 ; CHECK:            OUTLINED_FUNC
 ; CHECK:            // %bb.0:
-; CHECK-NEXT:               .cfi_b_key_frame
 ; CHECK-NEXT:               pacibsp
 ; CHECK:                    retab
 ; CHECK-NOT:                auti[a,b]sp
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-throw.ll b/llvm/test/CodeGen/AArch64/machine-outliner-throw.ll
new file mode 100644
index 0000000000000..52d05aa900e3c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-throw.ll
@@ -0,0 +1,71 @@
+; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple=aarch64-arm-none-eabi < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple=aarch64-arm-none-eabi -stop-after=machine-outliner < %s | FileCheck %s -check-prefix=TARGET_FEATURES
+
+; Make sure that we haven't added nouwind.
+; TARGET_FEATURES: define internal void @OUTLINED_FUNCTION_0()
+; TARGET_FEATURES-SAME: #[[ATTR_NUM:[0-9]+]]
+; TARGET_FEATURES: attributes #[[ATTR_NUM]] = { minsize optsize }
+
+define dso_local i32 @_Z5func1i(i32 %x) #0 {
+; CHECK-LABEL: _Z5func1i:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    orr w8, wzr, #0x1
+; CHECK-NEXT:    madd w19, w0, w0, w8
+; CHECK-NEXT:    mov w0, #4
+; CHECK-NEXT:    bl __cxa_allocate_exception
+; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
+entry:
+  %mul = mul nsw i32 %x, %x
+  %add = add nuw nsw i32 %mul, 1
+  %exception = tail call i8* @__cxa_allocate_exception(i64 4) #1
+  %0 = bitcast i8* %exception to i32*
+  store i32 %add, i32* %0, align 16
+  tail call void @__cxa_throw(i8* %exception, i8* bitcast (i8** @_ZTIi to i8*), i8* null) #2
+  unreachable
+}
+
+define dso_local i32 @_Z5func2c(i8 %x) #0 {
+; CHECK-LABEL: _Z5func2c:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    and w8, w0, #0xff
+; CHECK-NEXT:    orr w9, wzr, #0x1
+; CHECK-NEXT:    mov w0, #4
+; CHECK-NEXT:    madd w19, w8, w8, w9
+; CHECK-NEXT:    bl __cxa_allocate_exception
+; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
+entry:
+  %conv = zext i8 %x to i32
+  %mul = mul nuw nsw i32 %conv, %conv
+  %add = add nuw nsw i32 %mul, 1
+  %exception = tail call i8* @__cxa_allocate_exception(i64 4) #1
+  %0 = bitcast i8* %exception to i32*
+  store i32 %add, i32* %0, align 16
+  tail call void @__cxa_throw(i8* %exception, i8* bitcast (i8** @_ZTIi to i8*), i8* null) #2
+  unreachable
+}
+
+; CHECK-LABEL: OUTLINED_FUNCTION_0:
+; CHECK:      .cfi_startproc
+; CHECK:        adrp    x1, _ZTIi
+; CHECK-NEXT:   add     x1, x1, :lo12:_ZTIi
+; CHECK-NEXT:   mov     x2, xzr
+; CHECK-NEXT:   str     w19, [x0]
+; CHECK-NEXT:   b       __cxa_throw
+; CHECK:      .cfi_endproc
+
+
+@_ZTIi = external dso_local constant i8*
+declare dso_local i8* @__cxa_allocate_exception(i64) local_unnamed_addr
+declare dso_local void @__cxa_throw(i8*, i8*, i8*) local_unnamed_addr
+
+attributes #0 = { minsize noreturn optsize }
+attributes #1 = { nounwind }
+attributes #2 = { noreturn }
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-throw2.ll b/llvm/test/CodeGen/AArch64/machine-outliner-throw2.ll
new file mode 100644
index 0000000000000..2ea906fa47ba9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-throw2.ll
@@ -0,0 +1,101 @@
+; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple=aarch64-arm-none-eabi -frame-pointer=non-leaf < %s | FileCheck %s --check-prefix=NOOMIT
+; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple=aarch64-arm-none-eabi -frame-pointer=none < %s | FileCheck %s --check-prefix=OMITFP
+
+define void @_Z1giii(i32 %x, i32 %y, i32 %z) minsize {
+; NOOMIT-LABEL: _Z1giii:
+; NOOMIT:       // %bb.0: // %entry
+; NOOMIT-NEXT:    b _Z1hiii
+;
+; OMITFP-LABEL: _Z1giii:
+; OMITFP:       // %bb.0: // %entry
+; OMITFP-NEXT:    b _Z1hiii
+entry:
+  tail call void @_Z1hiii(i32 %x, i32 %y, i32 %z)
+  ret void
+}
+
+declare void @_Z1hiii(i32, i32, i32) minsize
+
+define void @_Z2f1v() minsize {
+; NOOMIT-LABEL: _Z2f1v:
+; NOOMIT:       // %bb.0: // %entry
+; NOOMIT-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; NOOMIT-NEXT:    mov x29, sp
+; NOOMIT-NEXT:    .cfi_def_cfa w29, 16
+; NOOMIT-NEXT:    .cfi_offset w30, -8
+; NOOMIT-NEXT:    .cfi_offset w29, -16
+; NOOMIT-NEXT:    bl OUTLINED_FUNCTION_0
+; NOOMIT-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; NOOMIT-NEXT:    b _Z1giii
+;
+; OMITFP-LABEL: _Z2f1v:
+; OMITFP:       // %bb.0: // %entry
+; OMITFP-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; OMITFP-NEXT:    .cfi_def_cfa_offset 16
+; OMITFP-NEXT:    .cfi_offset w30, -16
+; OMITFP-NEXT:    bl OUTLINED_FUNCTION_0
+; OMITFP-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; OMITFP-NEXT:    b _Z1giii
+entry:
+  tail call void @_Z1giii(i32 1, i32 2, i32 3)
+  tail call void @_Z1giii(i32 1, i32 2, i32 3)
+  ret void
+}
+
+define void @_Z2f2v() minsize {
+; NOOMIT-LABEL: _Z2f2v:
+; NOOMIT:       // %bb.0: // %entry
+; NOOMIT-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; NOOMIT-NEXT:    mov x29, sp
+; NOOMIT-NEXT:    .cfi_def_cfa w29, 16
+; NOOMIT-NEXT:    .cfi_offset w30, -8
+; NOOMIT-NEXT:    .cfi_offset w29, -16
+; NOOMIT-NEXT:    bl OUTLINED_FUNCTION_0
+; NOOMIT-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; NOOMIT-NEXT:    b _Z1giii
+;
+; OMITFP-LABEL: _Z2f2v:
+; OMITFP:       // %bb.0: // %entry
+; OMITFP-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; OMITFP-NEXT:    .cfi_def_cfa_offset 16
+; OMITFP-NEXT:    .cfi_offset w30, -16
+; OMITFP-NEXT:    bl OUTLINED_FUNCTION_0
+; OMITFP-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; OMITFP-NEXT:    b _Z1giii
+entry:
+  tail call void @_Z1giii(i32 1, i32 2, i32 3)
+  tail call void @_Z1giii(i32 1, i32 2, i32 3)
+  ret void
+}
+
+; OMITFP-LABEL: OUTLINED_FUNCTION_0:
+; OMITFP:              .cfi_startproc
+; OMITFP-NEXT: // %bb.0:
+; OMITFP-NEXT:         .cfi_def_cfa_offset 16
+; OMITFP-NEXT:         .cfi_offset w30, -16
+; OMITFP-NEXT:         str     x30, [sp, #-16]!
+; OMITFP-NEXT:         mov     w0, #1
+; OMITFP-NEXT:         mov     w1, #2
+; OMITFP-NEXT:         mov     w2, #3
+; OMITFP-NEXT:         bl      _Z1giii
+; OMITFP-NEXT:         mov     w0, #1
+; OMITFP-NEXT:         mov     w1, #2
+; OMITFP-NEXT:         mov     w2, #3
+; OMITFP-NEXT:         ldr     x30, [sp], #16
+; OMITFP-NEXT:         ret
+
+; NOOMIT-LABEL: OUTLINED_FUNCTION_0:
+; NOOMIT:              .cfi_startproc
+; NOOMIT-NEXT: // %bb.0:
+; NOOMIT-NEXT:         .cfi_def_cfa_offset 16
+; NOOMIT-NEXT:         .cfi_offset w30, -16
+; NOOMIT-NEXT:         str     x30, [sp, #-16]!
+; NOOMIT-NEXT:         mov     w0, #1
+; NOOMIT-NEXT:         mov     w1, #2
+; NOOMIT-NEXT:         mov     w2, #3
+; NOOMIT-NEXT:         bl      _Z1giii
+; NOOMIT-NEXT:         mov     w0, #1
+; NOOMIT-NEXT:         mov     w1, #2
+; NOOMIT-NEXT:         mov     w2, #3
+; NOOMIT-NEXT:         ldr     x30, [sp], #16
+; NOOMIT-NEXT:         ret
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner.ll b/llvm/test/CodeGen/AArch64/machine-outliner.ll
index 6c76f894c8567..3d1c7357b9678 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner.ll
@@ -9,6 +9,7 @@
 ; TARGET_FEATURES-SAME: #[[ATTR_NUM:[0-9]+]]
 ; TARGET_FEATURES-DAG: attributes #[[ATTR_NUM]] = {
 ; TARGET_FEATURES-SAME: minsize
+; TARGET_FEATURES-SAME: nounwind
 ; TARGET_FEATURES-SAME: optsize
 ; TARGET_FEATURES-SAME: "target-features"="+sse"
 
@@ -103,4 +104,4 @@ define void @dog() #0 {
 ; CHECK-DAG: add     sp, sp, #32
 ; CHECK-DAG: ret
 
-attributes #0 = { noredzone "target-cpu"="cyclone" "target-features"="+sse" }
+attributes #0 = { nounwind noredzone "target-cpu"="cyclone" "target-features"="+sse" }
diff --git a/llvm/test/CodeGen/AArch64/merge-store-dependency.ll b/llvm/test/CodeGen/AArch64/merge-store-dependency.ll
index 5613db1e52144..77b7012d2ed1e 100644
--- a/llvm/test/CodeGen/AArch64/merge-store-dependency.ll
+++ b/llvm/test/CodeGen/AArch64/merge-store-dependency.ll
@@ -50,7 +50,6 @@ define void @test(%struct1* %fde, i32 %fd, void (i32, i32, i8*)* %func, i8* %arg
 ; A53-NEXT:    // =>This Inner Loop Header: Depth=1
 ; A53-NEXT:    b .LBB0_4
 entry:
-
   %0 = bitcast %struct1* %fde to i8*
   tail call void @llvm.memset.p0i8.i64(i8* align 8 %0, i8 0, i64 40, i1 false)
   %state = getelementptr inbounds %struct1, %struct1* %fde, i64 0, i32 4
@@ -96,6 +95,110 @@ exit:
   ret void
 }
 
+define void @rotate16_in_place(i8* %p) {
+; A53-LABEL: rotate16_in_place:
+; A53:       // %bb.0:
+; A53-NEXT:    ldrb w8, [x0, #1]
+; A53-NEXT:    ldrb w9, [x0]
+; A53-NEXT:    strb w8, [x0]
+; A53-NEXT:    strb w9, [x0, #1]
+; A53-NEXT:    ret
+  %p0 = getelementptr i8, i8* %p, i64 0
+  %p1 = getelementptr i8, i8* %p, i64 1
+  %i0 = load i8, i8* %p0, align 1
+  %i1 = load i8, i8* %p1, align 1
+  store i8 %i1, i8* %p0, align 1
+  store i8 %i0, i8* %p1, align 1
+  ret void
+}
+
+define void @rotate16(i8* %p, i8* %q) {
+; A53-LABEL: rotate16:
+; A53:       // %bb.0:
+; A53-NEXT:    ldrb w8, [x0, #1]
+; A53-NEXT:    ldrb w9, [x0]
+; A53-NEXT:    strb w8, [x1]
+; A53-NEXT:    strb w9, [x1, #1]
+; A53-NEXT:    ret
+  %p0 = getelementptr i8, i8* %p, i64 0
+  %p1 = getelementptr i8, i8* %p, i64 1
+  %q0 = getelementptr i8, i8* %q, i64 0
+  %q1 = getelementptr i8, i8* %q, i64 1
+  %i0 = load i8, i8* %p0, align 1
+  %i1 = load i8, i8* %p1, align 1
+  store i8 %i1, i8* %q0, align 1
+  store i8 %i0, i8* %q1, align 1
+  ret void
+}
+
+define void @rotate32_in_place(i16* %p) {
+; A53-LABEL: rotate32_in_place:
+; A53:       // %bb.0:
+; A53-NEXT:    ldrh w8, [x0, #2]
+; A53-NEXT:    ldrh w9, [x0]
+; A53-NEXT:    strh w8, [x0]
+; A53-NEXT:    strh w9, [x0, #2]
+; A53-NEXT:    ret
+  %p0 = getelementptr i16, i16* %p, i64 0
+  %p1 = getelementptr i16, i16* %p, i64 1
+  %i0 = load i16, i16* %p0, align 2
+  %i1 = load i16, i16* %p1, align 2
+  store i16 %i1, i16* %p0, align 2
+  store i16 %i0, i16* %p1, align 2
+  ret void
+}
+
+define void @rotate32(i16* %p) {
+; A53-LABEL: rotate32:
+; A53:       // %bb.0:
+; A53-NEXT:    ldrh w8, [x0, #2]
+; A53-NEXT:    ldrh w9, [x0]
+; A53-NEXT:    strh w8, [x0, #84]
+; A53-NEXT:    strh w9, [x0, #86]
+; A53-NEXT:    ret
+  %p0 = getelementptr i16, i16* %p, i64 0
+  %p1 = getelementptr i16, i16* %p, i64 1
+  %p42 = getelementptr i16, i16* %p, i64 42
+  %p43 = getelementptr i16, i16* %p, i64 43
+  %i0 = load i16, i16* %p0, align 2
+  %i1 = load i16, i16* %p1, align 2
+  store i16 %i1, i16* %p42, align 2
+  store i16 %i0, i16* %p43, align 2
+  ret void
+}
+
+define void @rotate64_in_place(i32* %p) {
+; A53-LABEL: rotate64_in_place:
+; A53:       // %bb.0:
+; A53-NEXT:    ldp w9, w8, [x0]
+; A53-NEXT:    stp w8, w9, [x0]
+; A53-NEXT:    ret
+  %p0 = getelementptr i32, i32* %p, i64 0
+  %p1 = getelementptr i32, i32* %p, i64 1
+  %i0 = load i32, i32* %p0, align 4
+  %i1 = load i32, i32* %p1, align 4
+  store i32 %i1, i32* %p0, align 4
+  store i32 %i0, i32* %p1, align 4
+  ret void
+}
+
+define void @rotate64(i32* %p) {
+; A53-LABEL: rotate64:
+; A53:       // %bb.0:
+; A53-NEXT:    ldp w9, w8, [x0]
+; A53-NEXT:    stp w8, w9, [x0, #8]
+; A53-NEXT:    ret
+  %p0 = getelementptr i32, i32* %p, i64 0
+  %p1 = getelementptr i32, i32* %p, i64 1
+  %p2 = getelementptr i32, i32* %p, i64 2
+  %p3 = getelementptr i32, i32* %p, i64 3
+  %i0 = load i32, i32* %p0, align 4
+  %i1 = load i32, i32* %p1, align 4
+  store i32 %i1, i32* %p2, align 4
+  store i32 %i0, i32* %p3, align 4
+  ret void
+}
+
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1)
 declare i32 @fcntl(i32, i32, ...)
 declare noalias i8* @foo()
diff --git a/llvm/test/CodeGen/AArch64/regress-combine-extract-vectors.ll b/llvm/test/CodeGen/AArch64/regress-combine-extract-vectors.ll
new file mode 100644
index 0000000000000..1662e27ecdefd
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/regress-combine-extract-vectors.ll
@@ -0,0 +1,17 @@
+; RUN: llc -asm-verbose=0 -mtriple aarch64-arm-none-eabi < %s | FileCheck %s
+
+; The following code previously broke in the DAGCombiner. Specifically, trying to combine:
+; extract_vector_elt (concat_vectors v4i16:a, v4i16:b), x
+;   -> extract_vector_elt a, x
+
+define half @test_combine_extract_concat_vectors(<4 x i16> %a) nounwind {
+entry:
+  %0 = shufflevector <4 x i16> %a, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %1 = bitcast <8 x i16> %0 to <8 x half>
+  %2 = extractelement <8 x half> %1, i32 3
+  ret half %2
+}
+
+; CHECK-LABEL: test_combine_extract_concat_vectors:
+; CHECK-NEXT: mov h0, v0.h[3]
+; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/remat.ll b/llvm/test/CodeGen/AArch64/remat.ll
index 676a605b11f97..90ad2508fe170 100644
--- a/llvm/test/CodeGen/AArch64/remat.ll
+++ b/llvm/test/CodeGen/AArch64/remat.ll
@@ -8,6 +8,9 @@
 ; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=cortex-a72 -o - %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=cortex-a73 -o - %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=cortex-a75 -o - %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=cortex-a77 -o - %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=cortex-a78 -o - %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=cortex-x1 -o - %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=neoverse-e1 -o - %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=neoverse-n1 -o - %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnuabi -mcpu=exynos-m3 -o - %s | FileCheck %s
diff --git a/llvm/test/CodeGen/AArch64/spillfill-sve.ll b/llvm/test/CodeGen/AArch64/spillfill-sve.ll
index 92c827b41c3a2..a703c510952f6 100644
--- a/llvm/test/CodeGen/AArch64/spillfill-sve.ll
+++ b/llvm/test/CodeGen/AArch64/spillfill-sve.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+sve -mattr=+bf16 < %s | FileCheck %s
 
 ; This file checks that unpredicated load/store instructions to locals
 ; use the right instructions and offsets.
@@ -199,6 +199,50 @@ define void @fill_nxv2i64() {
   ret void
 }
 
+define void @fill_nxv8bf16() {
+; CHECK-LABEL: fill_nxv8bf16
+; CHECK-DAG: ld1h    { z{{[01]}}.h }, p0/z, [sp]
+; CHECK-DAG: ld1h    { z{{[01]}}.h }, p0/z, [sp, #1, mul vl]
+  %local0 = alloca <vscale x 8 x bfloat>
+  %local1 = alloca <vscale x 8 x bfloat>
+  load volatile <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %local0
+  load volatile <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %local1
+  ret void
+}
+
+define void @fill_nxv8f16() {
+; CHECK-LABEL: fill_nxv8f16
+; CHECK-DAG: ld1h    { z{{[01]}}.h }, p0/z, [sp]
+; CHECK-DAG: ld1h    { z{{[01]}}.h }, p0/z, [sp, #1, mul vl]
+  %local0 = alloca <vscale x 8 x half>
+  %local1 = alloca <vscale x 8 x half>
+  load volatile <vscale x 8 x half>, <vscale x 8 x half>* %local0
+  load volatile <vscale x 8 x half>, <vscale x 8 x half>* %local1
+  ret void
+}
+
+define void @fill_nxv4f32() {
+; CHECK-LABEL: fill_nxv4f32
+; CHECK-DAG: ld1w    { z{{[01]}}.s }, p0/z, [sp]
+; CHECK-DAG: ld1w    { z{{[01]}}.s }, p0/z, [sp, #1, mul vl]
+  %local0 = alloca <vscale x 4 x float>
+  %local1 = alloca <vscale x 4 x float>
+  load volatile <vscale x 4 x float>, <vscale x 4 x float>* %local0
+  load volatile <vscale x 4 x float>, <vscale x 4 x float>* %local1
+  ret void
+}
+
+define void @fill_nxv2f64() {
+; CHECK-LABEL: fill_nxv2f64
+; CHECK-DAG: ld1d    { z{{[01]}}.d }, p0/z, [sp]
+; CHECK-DAG: ld1d    { z{{[01]}}.d }, p0/z, [sp, #1, mul vl]
+  %local0 = alloca <vscale x 2 x double>
+  %local1 = alloca <vscale x 2 x double>
+  load volatile <vscale x 2 x double>, <vscale x 2 x double>* %local0
+  load volatile <vscale x 2 x double>, <vscale x 2 x double>* %local1
+  ret void
+}
+
 
 ; Data spills
 
@@ -312,6 +356,50 @@ define void @spill_nxv2i64(<vscale x 2 x i64> %v0, <vscale x 2 x i64> %v1) {
   ret void
 }
 
+define void @spill_nxv8f16(<vscale x 8 x half> %v0, <vscale x 8 x half> %v1) {
+; CHECK-LABEL: spill_nxv8f16
+; CHECK-DAG: st1h    { z{{[01]}}.h }, p0, [sp]
+; CHECK-DAG: st1h    { z{{[01]}}.h }, p0, [sp, #1, mul vl]
+  %local0 = alloca <vscale x 8 x half>
+  %local1 = alloca <vscale x 8 x half>
+  store volatile <vscale x 8 x half> %v0, <vscale x 8 x half>* %local0
+  store volatile <vscale x 8 x half> %v1, <vscale x 8 x half>* %local1
+  ret void
+}
+
+define void @spill_nxv8bf16(<vscale x 8 x bfloat> %v0, <vscale x 8 x bfloat> %v1) {
+; CHECK-LABEL: spill_nxv8bf16
+; CHECK-DAG: st1h    { z{{[01]}}.h }, p0, [sp]
+; CHECK-DAG: st1h    { z{{[01]}}.h }, p0, [sp, #1, mul vl]
+  %local0 = alloca <vscale x 8 x bfloat>
+  %local1 = alloca <vscale x 8 x bfloat>
+  store volatile <vscale x 8 x bfloat> %v0, <vscale x 8 x bfloat>* %local0
+  store volatile <vscale x 8 x bfloat> %v1, <vscale x 8 x bfloat>* %local1
+  ret void
+}
+
+define void @spill_nxv4f32(<vscale x 4 x float> %v0, <vscale x 4 x float> %v1) {
+; CHECK-LABEL: spill_nxv4f32
+; CHECK-DAG: st1w    { z{{[01]}}.s }, p0, [sp]
+; CHECK-DAG: st1w    { z{{[01]}}.s }, p0, [sp, #1, mul vl]
+  %local0 = alloca <vscale x 4 x float>
+  %local1 = alloca <vscale x 4 x float>
+  store volatile <vscale x 4 x float> %v0, <vscale x 4 x float>* %local0
+  store volatile <vscale x 4 x float> %v1, <vscale x 4 x float>* %local1
+  ret void
+}
+
+define void @spill_nxv2f64(<vscale x 2 x double> %v0, <vscale x 2 x double> %v1) {
+; CHECK-LABEL: spill_nxv2f64
+; CHECK-DAG: st1d    { z{{[01]}}.d }, p0, [sp]
+; CHECK-DAG: st1d    { z{{[01]}}.d }, p0, [sp, #1, mul vl]
+  %local0 = alloca <vscale x 2 x double>
+  %local1 = alloca <vscale x 2 x double>
+  store volatile <vscale x 2 x double> %v0, <vscale x 2 x double>* %local0
+  store volatile <vscale x 2 x double> %v1, <vscale x 2 x double>* %local1
+  ret void
+}
+
 ; Predicate fills
 
 define void @fill_nxv16i1() {
diff --git a/llvm/test/CodeGen/AArch64/sve-bitcast.ll b/llvm/test/CodeGen/AArch64/sve-bitcast.ll
index ab70dea81d458..670c9864c85ab 100644
--- a/llvm/test/CodeGen/AArch64/sve-bitcast.ll
+++ b/llvm/test/CodeGen/AArch64/sve-bitcast.ll
@@ -340,3 +340,118 @@ define <vscale x 2 x double> @bitcast_float_to_double(<vscale x 4 x float> %v) {
   %bc = bitcast <vscale x 4 x float> %v to <vscale x 2 x double>
   ret <vscale x 2 x double> %bc
 }
+
+define <vscale x 16 x i8> @bitcast_bfloat_to_i8(<vscale x 8 x bfloat> %v) #0 {
+; CHECK-LABEL: bitcast_bfloat_to_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %bc = bitcast <vscale x 8 x bfloat> %v to <vscale x 16 x i8>
+  ret <vscale x 16 x i8> %bc
+}
+
+define <vscale x 8 x i16> @bitcast_bfloat_to_i16(<vscale x 8 x bfloat> %v) #0 {
+; CHECK-LABEL: bitcast_bfloat_to_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %bc = bitcast <vscale x 8 x bfloat> %v to <vscale x 8 x i16>
+  ret <vscale x 8 x i16> %bc
+}
+
+define <vscale x 4 x i32> @bitcast_bfloat_to_i32(<vscale x 8 x bfloat> %v) #0 {
+; CHECK-LABEL: bitcast_bfloat_to_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %bc = bitcast <vscale x 8 x bfloat> %v to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %bc
+}
+
+define <vscale x 2 x i64> @bitcast_bfloat_to_i64(<vscale x 8 x bfloat> %v) #0 {
+; CHECK-LABEL: bitcast_bfloat_to_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %bc = bitcast <vscale x 8 x bfloat> %v to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %bc
+}
+
+define <vscale x 8 x half> @bitcast_bfloat_to_half(<vscale x 8 x bfloat> %v) #0 {
+; CHECK-LABEL: bitcast_bfloat_to_half:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %bc = bitcast <vscale x 8 x bfloat> %v to <vscale x 8 x half>
+  ret <vscale x 8 x half> %bc
+}
+
+define <vscale x 4 x float> @bitcast_bfloat_to_float(<vscale x 8 x bfloat> %v) #0 {
+; CHECK-LABEL: bitcast_bfloat_to_float:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %bc = bitcast <vscale x 8 x bfloat> %v to <vscale x 4 x float>
+  ret <vscale x 4 x float> %bc
+}
+
+define <vscale x 2 x double> @bitcast_bfloat_to_double(<vscale x 8 x bfloat> %v) #0 {
+; CHECK-LABEL: bitcast_bfloat_to_double:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %bc = bitcast <vscale x 8 x bfloat> %v to <vscale x 2 x double>
+  ret <vscale x 2 x double> %bc
+}
+
+define <vscale x 8 x bfloat> @bitcast_i8_to_bfloat(<vscale x 16 x i8> %v) #0 {
+; CHECK-LABEL: bitcast_i8_to_bfloat:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %bc = bitcast <vscale x 16 x i8> %v to <vscale x 8 x bfloat>
+  ret <vscale x 8 x bfloat> %bc
+}
+
+define <vscale x 8 x bfloat> @bitcast_i16_to_bfloat(<vscale x 8 x i16> %v) #0 {
+; CHECK-LABEL: bitcast_i16_to_bfloat:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %bc = bitcast <vscale x 8 x i16> %v to <vscale x 8 x bfloat>
+  ret <vscale x 8 x bfloat> %bc
+}
+
+define <vscale x 8 x bfloat> @bitcast_i32_to_bfloat(<vscale x 4 x i32> %v) #0 {
+; CHECK-LABEL: bitcast_i32_to_bfloat:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %bc = bitcast <vscale x 4 x i32> %v to <vscale x 8 x bfloat>
+  ret <vscale x 8 x bfloat> %bc
+}
+
+define <vscale x 8 x bfloat> @bitcast_i64_to_bfloat(<vscale x 2 x i64> %v) #0 {
+; CHECK-LABEL: bitcast_i64_to_bfloat:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %bc = bitcast <vscale x 2 x i64> %v to <vscale x 8 x bfloat>
+  ret <vscale x 8 x bfloat> %bc
+}
+
+define <vscale x 8 x bfloat> @bitcast_half_to_bfloat(<vscale x 8 x half> %v) #0 {
+; CHECK-LABEL: bitcast_half_to_bfloat:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %bc = bitcast <vscale x 8 x half> %v to <vscale x 8 x bfloat>
+  ret <vscale x 8 x bfloat> %bc
+}
+
+define <vscale x 8 x bfloat> @bitcast_float_to_bfloat(<vscale x 4 x float> %v) #0 {
+; CHECK-LABEL: bitcast_float_to_bfloat:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %bc = bitcast <vscale x 4 x float> %v to <vscale x 8 x bfloat>
+  ret <vscale x 8 x bfloat> %bc
+}
+
+define <vscale x 8 x bfloat> @bitcast_double_to_bfloat(<vscale x 2 x double> %v) #0 {
+; CHECK-LABEL: bitcast_double_to_bfloat:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %bc = bitcast <vscale x 2 x double> %v to <vscale x 8 x bfloat>
+  ret <vscale x 8 x bfloat> %bc
+}
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-breakdown-scalable-vectortype.ll b/llvm/test/CodeGen/AArch64/sve-breakdown-scalable-vectortype.ll
index 14123b476704a..92395a6502746 100644
--- a/llvm/test/CodeGen/AArch64/sve-breakdown-scalable-vectortype.ll
+++ b/llvm/test/CodeGen/AArch64/sve-breakdown-scalable-vectortype.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -asm-verbose=0 < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ; Test that scalable vectors that are a multiple of the legal vector size
 ; can be properly broken down into part vectors.
diff --git a/llvm/test/CodeGen/AArch64/sve-callbyref-notailcall.ll b/llvm/test/CodeGen/AArch64/sve-callbyref-notailcall.ll
index ca29e15697fe0..caa8d32186f4c 100644
--- a/llvm/test/CodeGen/AArch64/sve-callbyref-notailcall.ll
+++ b/llvm/test/CodeGen/AArch64/sve-callbyref-notailcall.ll
@@ -1,6 +1,9 @@
 ; Because some arguments are passed by reference (through stack),
 ; the compiler should not do tail-call optimization.
-; RUN: llc -mtriple=aarch64 -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64 -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ; CHECK-LABEL: caller:
 ; CHECK:       addvl sp, sp, #-[[STACKSIZE:[0-9]+]]
diff --git a/llvm/test/CodeGen/AArch64/sve-calling-convention-byref.ll b/llvm/test/CodeGen/AArch64/sve-calling-convention-byref.ll
index bbb8209941b0e..d579ba08b59b6 100644
--- a/llvm/test/CodeGen/AArch64/sve-calling-convention-byref.ll
+++ b/llvm/test/CodeGen/AArch64/sve-calling-convention-byref.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -stop-after=finalize-isel < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -stop-after=finalize-isel < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ; Test that z8 and z9, passed in by reference, are correctly loaded from x0 and x1.
 ; i.e. z0 =  %z0
diff --git a/llvm/test/CodeGen/AArch64/sve-calling-convention-tuple-types.ll b/llvm/test/CodeGen/AArch64/sve-calling-convention-tuple-types.ll
index 6c702b64a4554..6700c27eb1096 100644
--- a/llvm/test/CodeGen/AArch64/sve-calling-convention-tuple-types.ll
+++ b/llvm/test/CodeGen/AArch64/sve-calling-convention-tuple-types.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple aarch64 -mattr=+sve -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -mtriple aarch64 -mattr=+sve -asm-verbose=0 < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; svint8x2_t
diff --git a/llvm/test/CodeGen/AArch64/sve-extract-subvector.ll b/llvm/test/CodeGen/AArch64/sve-extract-subvector.ll
new file mode 100644
index 0000000000000..29ad1273e352b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-extract-subvector.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
+
+; Test that DAGCombiner doesn't drop the scalable flag when it tries to fold:
+;   extract_subv (bitcast X), Index --> bitcast (extract_subv X, Index')
+define <vscale x 16 x i8> @extract_nxv16i8_nxv4i64(<vscale x 4 x i64> %z0_z1) {
+; CHECK-LABEL: extract_nxv16i8_nxv4i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ret
+  %z0_z1_bc = bitcast <vscale x 4 x i64> %z0_z1 to <vscale x 32 x i8>
+  %ext = call <vscale x 16 x i8> @llvm.aarch64.sve.tuple.get.nxv32i8(<vscale x 32 x i8> %z0_z1_bc, i32 1)
+  ret <vscale x 16 x i8> %ext
+}
+
+
+define <vscale x 2 x i64> @extract_nxv2i64_nxv32i8(<vscale x 32 x i8> %z0_z1) {
+; CHECK-LABEL: extract_nxv2i64_nxv32i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ret
+  %z0_z1_bc = bitcast <vscale x 32 x i8> %z0_z1 to <vscale x 4 x i64>
+  %ext = call <vscale x 2 x i64> @llvm.aarch64.sve.tuple.get.nxv4i64(<vscale x 4 x i64> %z0_z1_bc, i32 1)
+  ret <vscale x 2 x i64> %ext
+}
+
+declare <vscale x 2 x i64> @llvm.aarch64.sve.tuple.get.nxv4i64(<vscale x 4 x i64>, i32)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.tuple.get.nxv32i8(<vscale x 32 x i8>, i32)
diff --git a/llvm/test/CodeGen/AArch64/sve-fcmp.ll b/llvm/test/CodeGen/AArch64/sve-fcmp.ll
index cbafae608262d..703e86d9f4539 100644
--- a/llvm/test/CodeGen/AArch64/sve-fcmp.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fcmp.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 define <vscale x 4 x i1> @oeq(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
 ; CHECK-LABEL: oeq:
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll
new file mode 100644
index 0000000000000..9c32748e3b8ef
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll
@@ -0,0 +1,300 @@
+; RUN: llc -aarch64-sve-vector-bits-min=128  < %s | FileCheck %s -D#VBYTES=16  -check-prefix=NO_SVE
+; RUN: llc -aarch64-sve-vector-bits-min=256  < %s | FileCheck %s -D#VBYTES=32  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512,VBITS_LE_256
+; RUN: llc -aarch64-sve-vector-bits-min=384  < %s | FileCheck %s -D#VBYTES=32  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512,VBITS_LE_256
+; RUN: llc -aarch64-sve-vector-bits-min=512  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=640  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=768  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=896  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=1024 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1152 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1280 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1408 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1536 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1664 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1792 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1920 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -D#VBYTES=256 -check-prefixes=CHECK
+
+; VBYTES represents the useful byte size of a vector register from the code
+; generator's point of view. It is clamped to power-of-2 values because
+; only power-of-2 vector lengths are considered legal, regardless of the
+; user specified vector length.
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Don't use SVE when its registers are no bigger than NEON.
+; NO_SVE-NOT: ptrue
+
+; Don't use SVE for 64-bit vectors.
+define <4 x half> @fadd_v4f16(<4 x half> %op1, <4 x half> %op2) #0 {
+; CHECK-LABEL: @fadd_v4f16
+; CHECK: fadd v0.4h, v0.4h, v1.4h
+; CHECK: ret
+  %res = fadd <4 x half> %op1, %op2
+  ret <4 x half> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <8 x half> @fadd_v8f16(<8 x half> %op1, <8 x half> %op2) #0 {
+; CHECK-LABEL: @fadd_v8f16
+; CHECK: fadd v0.8h, v0.8h, v1.8h
+; CHECK: ret
+  %res = fadd <8 x half> %op1, %op2
+  ret <8 x half> %res
+}
+
+define void @fadd_v16f16(<16 x half>* %a, <16 x half>* %b) #0 {
+; CHECK-LABEL: @fadd_v16f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),16)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: fadd [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x half>, <16 x half>* %a
+  %op2 = load <16 x half>, <16 x half>* %b
+  %res = fadd <16 x half> %op1, %op2
+  store <16 x half> %res, <16 x half>* %a
+  ret void
+}
+
+define void @fadd_v32f16(<32 x half>* %a, <32 x half>* %b) #0 {
+; CHECK-LABEL: @fadd_v32f16
+; CHECK-DAG: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),32)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK-DAG: fadd [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK-DAG: st1h { [[RES]].h }, [[PG]], [x0]
+; VBITS_LE_256-DAG: add x[[A1:[0-9]+]], x0, #[[#VBYTES]]
+; VBITS_LE_256-DAG: add x[[B1:[0-9]+]], x1, #[[#VBYTES]]
+; VBITS_LE_256-DAG: ld1h { [[OP1_1:z[0-9]+]].h }, [[PG]]/z, [x[[A1]]]
+; VBITS_LE_256-DAG: ld1h { [[OP2_1:z[0-9]+]].h }, [[PG]]/z, [x[[B1]]]
+; VBITS_LE_256-DAG: fadd [[RES_1:z[0-9]+]].h, [[PG]]/m, [[OP1_1]].h, [[OP2_1]].h
+; VBITS_LE_256-DAG: st1h { [[RES_1]].h }, [[PG]], [x[[A1]]]
+; CHECK: ret
+  %op1 = load <32 x half>, <32 x half>* %a
+  %op2 = load <32 x half>, <32 x half>* %b
+  %res = fadd <32 x half> %op1, %op2
+  store <32 x half> %res, <32 x half>* %a
+  ret void
+}
+
+define void @fadd_v64f16(<64 x half>* %a, <64 x half>* %b) #0 {
+; CHECK-LABEL: @fadd_v64f16
+; CHECK-DAG: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),64)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK-DAG: fadd [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK-DAG: st1h { [[RES]].h }, [[PG]], [x0]
+; VBITS_LE_512-DAG: add x[[A1:[0-9]+]], x0, #[[#VBYTES]]
+; VBITS_LE_512-DAG: add x[[B1:[0-9]+]], x1, #[[#VBYTES]]
+; VBITS_LE_512-DAG: ld1h { [[OP1_1:z[0-9]+]].h }, [[PG]]/z, [x[[A1]]]
+; VBITS_LE_512-DAG: ld1h { [[OP2_1:z[0-9]+]].h }, [[PG]]/z, [x[[B1]]]
+; VBITS_LE_512-DAG: fadd [[RES_1:z[0-9]+]].h, [[PG]]/m, [[OP1_1]].h, [[OP2_1]].h
+; VBITS_LE_512-DAG: st1h { [[RES_1]].h }, [[PG]], [x[[A1]]]
+; VBITS_LE_256-DAG: add x[[A2:[0-9]+]], x0, #[[#mul(VBYTES,2)]]
+; VBITS_LE_256-DAG: add x[[B2:[0-9]+]], x1, #[[#mul(VBYTES,2)]]
+; VBITS_LE_256-DAG: ld1h { [[OP1_2:z[0-9]+]].h }, [[PG]]/z, [x[[A2]]]
+; VBITS_LE_256-DAG: ld1h { [[OP2_2:z[0-9]+]].h }, [[PG]]/z, [x[[B2]]]
+; VBITS_LE_256-DAG: fadd [[RES_2:z[0-9]+]].h, [[PG]]/m, [[OP1_2]].h, [[OP2_2]].h
+; VBITS_LE_256-DAG: st1h { [[RES_2]].h }, [[PG]], [x[[A2]]]
+; VBITS_LE_256-DAG: add x[[A3:[0-9]+]], x0, #[[#mul(VBYTES,3)]]
+; VBITS_LE_256-DAG: add x[[B3:[0-9]+]], x1, #[[#mul(VBYTES,3)]]
+; VBITS_LE_256-DAG: ld1h { [[OP1_3:z[0-9]+]].h }, [[PG]]/z, [x[[A3]]]
+; VBITS_LE_256-DAG: ld1h { [[OP2_3:z[0-9]+]].h }, [[PG]]/z, [x[[B3]]]
+; VBITS_LE_256-DAG: fadd [[RES_3:z[0-9]+]].h, [[PG]]/m, [[OP1_3]].h, [[OP2_3]].h
+; VBITS_LE_256-DAG: st1h { [[RES_3]].h }, [[PG]], [x[[A3]]]
+; CHECK: ret
+  %op1 = load <64 x half>, <64 x half>* %a
+  %op2 = load <64 x half>, <64 x half>* %b
+  %res = fadd <64 x half> %op1, %op2
+  store <64 x half> %res, <64 x half>* %a
+  ret void
+}
+
+; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests
+; already cover the general legalisation cases.
+define void @fadd_v128f16(<128 x half>* %a, <128 x half>* %b) #0 {
+; CHECK-LABEL: @fadd_v128f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),128)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: fadd [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <128 x half>, <128 x half>* %a
+  %op2 = load <128 x half>, <128 x half>* %b
+  %res = fadd <128 x half> %op1, %op2
+  store <128 x half> %res, <128 x half>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <2 x float> @fadd_v2f32(<2 x float> %op1, <2 x float> %op2) #0 {
+; CHECK-LABEL: @fadd_v2f32
+; CHECK: fadd v0.2s, v0.2s, v1.2s
+; CHECK: ret
+  %res = fadd <2 x float> %op1, %op2
+  ret <2 x float> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x float> @fadd_v4f32(<4 x float> %op1, <4 x float> %op2) #0 {
+; CHECK-LABEL: @fadd_v4f32
+; CHECK: fadd v0.4s, v0.4s, v1.4s
+; CHECK: ret
+  %res = fadd <4 x float> %op1, %op2
+  ret <4 x float> %res
+}
+
+define void @fadd_v8f32(<8 x float>* %a, <8 x float>* %b) #0 {
+; CHECK-LABEL: @fadd_v8f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: fadd [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <8 x float>, <8 x float>* %a
+  %op2 = load <8 x float>, <8 x float>* %b
+  %res = fadd <8 x float> %op1, %op2
+  store <8 x float> %res, <8 x float>* %a
+  ret void
+}
+
+; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests
+; already cover the general legalisation cases.
+define void @fadd_v16f32(<16 x float>* %a, <16 x float>* %b) #0 {
+; CHECK-LABEL: @fadd_v16f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: fadd [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x float>, <16 x float>* %a
+  %op2 = load <16 x float>, <16 x float>* %b
+  %res = fadd <16 x float> %op1, %op2
+  store <16 x float> %res, <16 x float>* %a
+  ret void
+}
+
+; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests
+; already cover the general legalisation cases.
+define void @fadd_v32f32(<32 x float>* %a, <32 x float>* %b) #0 {
+; CHECK-LABEL: @fadd_v32f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: fadd [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x float>, <32 x float>* %a
+  %op2 = load <32 x float>, <32 x float>* %b
+  %res = fadd <32 x float> %op1, %op2
+  store <32 x float> %res, <32 x float>* %a
+  ret void
+}
+
+; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests
+; already cover the general legalisation cases.
+define void @fadd_v64f32(<64 x float>* %a, <64 x float>* %b) #0 {
+; CHECK-LABEL: @fadd_v64f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: fadd [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <64 x float>, <64 x float>* %a
+  %op2 = load <64 x float>, <64 x float>* %b
+  %res = fadd <64 x float> %op1, %op2
+  store <64 x float> %res, <64 x float>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <1 x double> @fadd_v1f64(<1 x double> %op1, <1 x double> %op2) #0 {
+; CHECK-LABEL: @fadd_v1f64
+; CHECK: fadd d0, d0, d1
+; CHECK: ret
+  %res = fadd <1 x double> %op1, %op2
+  ret <1 x double> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x double> @fadd_v2f64(<2 x double> %op1, <2 x double> %op2) #0 {
+; CHECK-LABEL: @fadd_v2f64
+; CHECK: fadd v0.2d, v0.2d, v1.2d
+; CHECK: ret
+  %res = fadd <2 x double> %op1, %op2
+  ret <2 x double> %res
+}
+
+define void @fadd_v4f64(<4 x double>* %a, <4 x double>* %b) #0 {
+; CHECK-LABEL: @fadd_v4f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),4)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: fadd [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <4 x double>, <4 x double>* %a
+  %op2 = load <4 x double>, <4 x double>* %b
+  %res = fadd <4 x double> %op1, %op2
+  store <4 x double> %res, <4 x double>* %a
+  ret void
+}
+
+; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests
+; already cover the general legalisation cases.
+define void @fadd_v8f64(<8 x double>* %a, <8 x double>* %b) #0 {
+; CHECK-LABEL: @fadd_v8f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: fadd [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <8 x double>, <8 x double>* %a
+  %op2 = load <8 x double>, <8 x double>* %b
+  %res = fadd <8 x double> %op1, %op2
+  store <8 x double> %res, <8 x double>* %a
+  ret void
+}
+
+; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests
+; already cover the general legalisation cases.
+define void @fadd_v16f64(<16 x double>* %a, <16 x double>* %b) #0 {
+; CHECK-LABEL: @fadd_v16f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: fadd [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x double>, <16 x double>* %a
+  %op2 = load <16 x double>, <16 x double>* %b
+  %res = fadd <16 x double> %op1, %op2
+  store <16 x double> %res, <16 x double>* %a
+  ret void
+}
+
+; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests
+; already cover the general legalisation cases.
+define void @fadd_v32f64(<32 x double>* %a, <32 x double>* %b) #0 {
+; CHECK-LABEL: @fadd_v32f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: fadd [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x double>, <32 x double>* %a
+  %op2 = load <32 x double>, <32 x double>* %b
+  %res = fadd <32 x double> %op1, %op2
+  store <32 x double> %res, <32 x double>* %a
+  ret void
+}
+
+attributes #0 = { "target-features"="+sve" }
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll
new file mode 100644
index 0000000000000..51970b01192f7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll
@@ -0,0 +1,413 @@
+; RUN: llc -aarch64-sve-vector-bits-min=128  < %s | FileCheck %s -D#VBYTES=16  -check-prefix=NO_SVE
+; RUN: llc -aarch64-sve-vector-bits-min=256  < %s | FileCheck %s -D#VBYTES=32  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512,VBITS_LE_256
+; RUN: llc -aarch64-sve-vector-bits-min=384  < %s | FileCheck %s -D#VBYTES=32  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512,VBITS_LE_256
+; RUN: llc -aarch64-sve-vector-bits-min=512  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=640  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=768  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=896  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=1024 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1152 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1280 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1408 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1536 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1664 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1792 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1920 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -D#VBYTES=256 -check-prefixes=CHECK
+
+; VBYTES represents the useful byte size of a vector register from the code
+; generator's point of view. It is clamped to power-of-2 values because
+; only power-of-2 vector lengths are considered legal, regardless of the
+; user specified vector length.
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Don't use SVE when its registers are no bigger than NEON.
+; NO_SVE-NOT: ptrue
+
+; Don't use SVE for 64-bit vectors.
+define <8 x i8> @add_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 {
+; CHECK-LABEL: @add_v8i8
+; CHECK: add v0.8b, v0.8b, v1.8b
+; CHECK: ret
+  %res = add <8 x i8> %op1, %op2
+  ret <8 x i8> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <16 x i8> @add_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 {
+; CHECK-LABEL: @add_v16i8
+; CHECK: add v0.16b, v0.16b, v1.16b
+; CHECK: ret
+  %res = add <16 x i8> %op1, %op2
+  ret <16 x i8> %res
+}
+
+define void @add_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 {
+; CHECK-LABEL: @add_v32i8
+; CHECK: ptrue [[PG:p[0-9]+]].b, vl[[#min(VBYTES,32)]]
+; CHECK-DAG: ld1b { [[OP1:z[0-9]+]].b }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1b { [[OP2:z[0-9]+]].b }, [[PG]]/z, [x1]
+; CHECK: add [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[OP2]].b
+; CHECK: st1b { [[RES]].b }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x i8>, <32 x i8>* %a
+  %op2 = load <32 x i8>, <32 x i8>* %b
+  %res = add <32 x i8> %op1, %op2
+  store <32 x i8> %res, <32 x i8>* %a
+  ret void
+}
+
+define void @add_v64i8(<64 x i8>* %a, <64 x i8>* %b) #0 {
+; CHECK-LABEL: @add_v64i8
+; CHECK-DAG: ptrue [[PG:p[0-9]+]].b, vl[[#min(VBYTES,64)]]
+; CHECK-DAG: ld1b { [[OP1:z[0-9]+]].b }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1b { [[OP2:z[0-9]+]].b }, [[PG]]/z, [x1]
+; CHECK-DAG: add [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[OP2]].b
+; CHECK-DAG: st1b { [[RES]].b }, [[PG]], [x0]
+; VBITS_LE_256-DAG: mov w[[OFF_1:[0-9]+]], #[[#VBYTES]]
+; VBITS_LE_256-DAG: ld1b { [[OP1_1:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_1]]]
+; VBITS_LE_256-DAG: ld1b { [[OP2_1:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_1]]]
+; VBITS_LE_256-DAG: add [[RES_1:z[0-9]+]].b, [[PG]]/m, [[OP1_1]].b, [[OP2_1]].b
+; VBITS_LE_256-DAG: st1b { [[RES_1]].b }, [[PG]], [x0, x[[OFF_1]]]
+; CHECK: ret
+  %op1 = load <64 x i8>, <64 x i8>* %a
+  %op2 = load <64 x i8>, <64 x i8>* %b
+  %res = add <64 x i8> %op1, %op2
+  store <64 x i8> %res, <64 x i8>* %a
+  ret void
+}
+
+define void @add_v128i8(<128 x i8>* %a, <128 x i8>* %b) #0 {
+; CHECK-LABEL: @add_v128i8
+; CHECK-DAG: ptrue [[PG:p[0-9]+]].b, vl[[#min(VBYTES,128)]]
+; CHECK-DAG: ld1b { [[OP1:z[0-9]+]].b }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1b { [[OP2:z[0-9]+]].b }, [[PG]]/z, [x1]
+; CHECK-DAG: add [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[OP2]].b
+; CHECK-DAG: st1b { [[RES]].b }, [[PG]], [x0]
+; VBITS_LE_512-DAG: mov w[[OFF_1:[0-9]+]], #[[#VBYTES]]
+; VBITS_LE_512-DAG: ld1b { [[OP1_1:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_1]]]
+; VBITS_LE_512-DAG: ld1b { [[OP2_1:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_1]]]
+; VBITS_LE_512-DAG: add [[RES_1:z[0-9]+]].b, [[PG]]/m, [[OP1_1]].b, [[OP2_1]].b
+; VBITS_LE_512-DAG: st1b { [[RES_1]].b }, [[PG]], [x0, x[[OFF_1]]]
+; VBITS_LE_256-DAG: mov w[[OFF_2:[0-9]+]], #[[#mul(VBYTES,2)]]
+; VBITS_LE_256-DAG: ld1b { [[OP1_2:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_2]]]
+; VBITS_LE_256-DAG: ld1b { [[OP2_2:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_2]]]
+; VBITS_LE_256-DAG: add [[RES_2:z[0-9]+]].b, [[PG]]/m, [[OP1_2]].b, [[OP2_2]].b
+; VBITS_LE_256-DAG: st1b { [[RES_2]].b }, [[PG]], [x0, x[[OFF_2]]]
+; VBITS_LE_256-DAG: mov w[[OFF_3:[0-9]+]], #[[#mul(VBYTES,3)]]
+; VBITS_LE_256-DAG: ld1b { [[OP1_3:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_3]]]
+; VBITS_LE_256-DAG: ld1b { [[OP2_3:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_3]]]
+; VBITS_LE_256-DAG: add [[RES_3:z[0-9]+]].b, [[PG]]/m, [[OP1_3]].b, [[OP2_3]].b
+; VBITS_LE_256-DAG: st1b { [[RES_3]].b }, [[PG]], [x0, x[[OFF_3]]]
+; CHECK: ret
+  %op1 = load <128 x i8>, <128 x i8>* %a
+  %op2 = load <128 x i8>, <128 x i8>* %b
+  %res = add <128 x i8> %op1, %op2
+  store <128 x i8> %res, <128 x i8>* %a
+  ret void
+}
+
+define void @add_v256i8(<256 x i8>* %a, <256 x i8>* %b) #0 {
+; CHECK-LABEL: @add_v256i8
+; CHECK-DAG: ptrue [[PG:p[0-9]+]].b, vl[[#min(VBYTES,256)]]
+; CHECK-DAG: ld1b { [[OP1:z[0-9]+]].b }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1b { [[OP2:z[0-9]+]].b }, [[PG]]/z, [x1]
+; CHECK-DAG: add [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[OP2]].b
+; CHECK-DAG: st1b { [[RES]].b }, [[PG]], [x0]
+; VBITS_LE_1024-DAG: mov w[[OFF_1:[0-9]+]], #[[#VBYTES]]
+; VBITS_LE_1024-DAG: ld1b { [[OP1_1:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_1]]]
+; VBITS_LE_1024-DAG: ld1b { [[OP2_1:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_1]]]
+; VBITS_LE_1024-DAG: add [[RES_1:z[0-9]+]].b, [[PG]]/m, [[OP1_1]].b, [[OP2_1]].b
+; VBITS_LE_1024-DAG: st1b { [[RES_1]].b }, [[PG]], [x0, x[[OFF_1]]]
+; VBITS_LE_512-DAG: mov w[[OFF_2:[0-9]+]], #[[#mul(VBYTES,2)]]
+; VBITS_LE_512-DAG: ld1b { [[OP1_2:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_2]]]
+; VBITS_LE_512-DAG: ld1b { [[OP2_2:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_2]]]
+; VBITS_LE_512-DAG: add [[RES_2:z[0-9]+]].b, [[PG]]/m, [[OP1_2]].b, [[OP2_2]].b
+; VBITS_LE_512-DAG: st1b { [[RES_2]].b }, [[PG]], [x0, x[[OFF_2]]]
+; VBITS_LE_512-DAG: mov w[[OFF_3:[0-9]+]], #[[#mul(VBYTES,3)]]
+; VBITS_LE_512-DAG: ld1b { [[OP1_3:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_3]]]
+; VBITS_LE_512-DAG: ld1b { [[OP2_3:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_3]]]
+; VBITS_LE_512-DAG: add [[RES_3:z[0-9]+]].b, [[PG]]/m, [[OP1_3]].b, [[OP2_3]].b
+; VBITS_LE_512-DAG: st1b { [[RES_3]].b }, [[PG]], [x0, x[[OFF_3]]]
+; VBITS_LE_256-DAG: mov w[[OFF_4:[0-9]+]], #[[#mul(VBYTES,4)]]
+; VBITS_LE_256-DAG: ld1b { [[OP1_4:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_4]]]
+; VBITS_LE_256-DAG: ld1b { [[OP2_4:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_4]]]
+; VBITS_LE_256-DAG: add [[RES_4:z[0-9]+]].b, [[PG]]/m, [[OP1_4]].b, [[OP2_4]].b
+; VBITS_LE_256-DAG: st1b { [[RES_4]].b }, [[PG]], [x0, x[[OFF_4]]]
+; VBITS_LE_256-DAG: mov w[[OFF_5:[0-9]+]], #[[#mul(VBYTES,5)]]
+; VBITS_LE_256-DAG: ld1b { [[OP1_5:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_5]]]
+; VBITS_LE_256-DAG: ld1b { [[OP2_5:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_5]]]
+; VBITS_LE_256-DAG: add [[RES_5:z[0-9]+]].b, [[PG]]/m, [[OP1_5]].b, [[OP2_5]].b
+; VBITS_LE_256-DAG: st1b { [[RES_5]].b }, [[PG]], [x0, x[[OFF_5]]]
+; VBITS_LE_256-DAG: mov w[[OFF_6:[0-9]+]], #[[#mul(VBYTES,6)]]
+; VBITS_LE_256-DAG: ld1b { [[OP1_6:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_6]]]
+; VBITS_LE_256-DAG: ld1b { [[OP2_6:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_6]]]
+; VBITS_LE_256-DAG: add [[RES_6:z[0-9]+]].b, [[PG]]/m, [[OP1_6]].b, [[OP2_6]].b
+; VBITS_LE_256-DAG: st1b { [[RES_6]].b }, [[PG]], [x0, x[[OFF_6]]]
+; VBITS_LE_256-DAG: mov w[[OFF_7:[0-9]+]], #[[#mul(VBYTES,7)]]
+; VBITS_LE_256-DAG: ld1b { [[OP1_7:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_7]]]
+; VBITS_LE_256-DAG: ld1b { [[OP2_7:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_7]]]
+; VBITS_LE_256-DAG: add [[RES_7:z[0-9]+]].b, [[PG]]/m, [[OP1_7]].b, [[OP2_7]].b
+; VBITS_LE_256-DAG: st1b { [[RES_7]].b }, [[PG]], [x0, x[[OFF_7]]]
+; CHECK: ret
+  %op1 = load <256 x i8>, <256 x i8>* %a
+  %op2 = load <256 x i8>, <256 x i8>* %b
+  %res = add <256 x i8> %op1, %op2
+  store <256 x i8> %res, <256 x i8>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <4 x i16> @add_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 {
+; CHECK-LABEL: @add_v4i16
+; CHECK: add v0.4h, v0.4h, v1.4h
+; CHECK: ret
+  %res = add <4 x i16> %op1, %op2
+  ret <4 x i16> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <8 x i16> @add_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 {
+; CHECK-LABEL: @add_v8i16
+; CHECK: add v0.8h, v0.8h, v1.8h
+; CHECK: ret
+  %res = add <8 x i16> %op1, %op2
+  ret <8 x i16> %res
+}
+
+define void @add_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 {
+; CHECK-LABEL: @add_v16i16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),16)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: add [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x i16>, <16 x i16>* %a
+  %op2 = load <16 x i16>, <16 x i16>* %b
+  %res = add <16 x i16> %op1, %op2
+  store <16 x i16> %res, <16 x i16>* %a
+  ret void
+}
+
+; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
+; already cover the general legalisation cases.
+define void @add_v32i16(<32 x i16>* %a, <32 x i16>* %b) #0 {
+; CHECK-LABEL: @add_v32i16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),32)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: add [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x i16>, <32 x i16>* %a
+  %op2 = load <32 x i16>, <32 x i16>* %b
+  %res = add <32 x i16> %op1, %op2
+  store <32 x i16> %res, <32 x i16>* %a
+  ret void
+}
+
+; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
+; already cover the general legalisation cases.
+define void @add_v64i16(<64 x i16>* %a, <64 x i16>* %b) #0 {
+; CHECK-LABEL: @add_v64i16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),64)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: add [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <64 x i16>, <64 x i16>* %a
+  %op2 = load <64 x i16>, <64 x i16>* %b
+  %res = add <64 x i16> %op1, %op2
+  store <64 x i16> %res, <64 x i16>* %a
+  ret void
+}
+
+; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
+; already cover the general legalisation cases.
+define void @add_v128i16(<128 x i16>* %a, <128 x i16>* %b) #0 {
+; CHECK-LABEL: @add_v128i16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),128)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: add [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <128 x i16>, <128 x i16>* %a
+  %op2 = load <128 x i16>, <128 x i16>* %b
+  %res = add <128 x i16> %op1, %op2
+  store <128 x i16> %res, <128 x i16>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <2 x i32> @add_v2i32(<2 x i32> %op1, <2 x i32> %op2) #0 {
+; CHECK-LABEL: @add_v2i32
+; CHECK: add v0.2s, v0.2s, v1.2s
+; CHECK: ret
+  %res = add <2 x i32> %op1, %op2
+  ret <2 x i32> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x i32> @add_v4i32(<4 x i32> %op1, <4 x i32> %op2) #0 {
+; CHECK-LABEL: @add_v4i32
+; CHECK: add v0.4s, v0.4s, v1.4s
+; CHECK: ret
+  %res = add <4 x i32> %op1, %op2
+  ret <4 x i32> %res
+}
+
+define void @add_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 {
+; CHECK-LABEL: @add_v8i32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: add [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <8 x i32>, <8 x i32>* %a
+  %op2 = load <8 x i32>, <8 x i32>* %b
+  %res = add <8 x i32> %op1, %op2
+  store <8 x i32> %res, <8 x i32>* %a
+  ret void
+}
+
+; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
+; already cover the general legalisation cases.
+define void @add_v16i32(<16 x i32>* %a, <16 x i32>* %b) #0 {
+; CHECK-LABEL: @add_v16i32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: add [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x i32>, <16 x i32>* %a
+  %op2 = load <16 x i32>, <16 x i32>* %b
+  %res = add <16 x i32> %op1, %op2
+  store <16 x i32> %res, <16 x i32>* %a
+  ret void
+}
+
+; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
+; already cover the general legalisation cases.
+define void @add_v32i32(<32 x i32>* %a, <32 x i32>* %b) #0 {
+; CHECK-LABEL: @add_v32i32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: add [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x i32>, <32 x i32>* %a
+  %op2 = load <32 x i32>, <32 x i32>* %b
+  %res = add <32 x i32> %op1, %op2
+  store <32 x i32> %res, <32 x i32>* %a
+  ret void
+}
+
+; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
+; already cover the general legalisation cases.
+define void @add_v64i32(<64 x i32>* %a, <64 x i32>* %b) #0 {
+; CHECK-LABEL: @add_v64i32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: add [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <64 x i32>, <64 x i32>* %a
+  %op2 = load <64 x i32>, <64 x i32>* %b
+  %res = add <64 x i32> %op1, %op2
+  store <64 x i32> %res, <64 x i32>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <1 x i64> @add_v1i64(<1 x i64> %op1, <1 x i64> %op2) #0 {
+; CHECK-LABEL: @add_v1i64
+; CHECK: add d0, d0, d1
+; CHECK: ret
+  %res = add <1 x i64> %op1, %op2
+  ret <1 x i64> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x i64> @add_v2i64(<2 x i64> %op1, <2 x i64> %op2) #0 {
+; CHECK-LABEL: @add_v2i64
+; CHECK: add v0.2d, v0.2d, v1.2d
+; CHECK: ret
+  %res = add <2 x i64> %op1, %op2
+  ret <2 x i64> %res
+}
+
+define void @add_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 {
+; CHECK-LABEL: @add_v4i64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),4)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: add [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <4 x i64>, <4 x i64>* %a
+  %op2 = load <4 x i64>, <4 x i64>* %b
+  %res = add <4 x i64> %op1, %op2
+  store <4 x i64> %res, <4 x i64>* %a
+  ret void
+}
+
+; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
+; already cover the general legalisation cases.
+define void @add_v8i64(<8 x i64>* %a, <8 x i64>* %b) #0 {
+; CHECK-LABEL: @add_v8i64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: add [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <8 x i64>, <8 x i64>* %a
+  %op2 = load <8 x i64>, <8 x i64>* %b
+  %res = add <8 x i64> %op1, %op2
+  store <8 x i64> %res, <8 x i64>* %a
+  ret void
+}
+
+; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
+; already cover the general legalisation cases.
+define void @add_v16i64(<16 x i64>* %a, <16 x i64>* %b) #0 {
+; CHECK-LABEL: @add_v16i64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: add [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x i64>, <16 x i64>* %a
+  %op2 = load <16 x i64>, <16 x i64>* %b
+  %res = add <16 x i64> %op1, %op2
+  store <16 x i64> %res, <16 x i64>* %a
+  ret void
+}
+
+; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
+; already cover the general legalisation cases.
+define void @add_v32i64(<32 x i64>* %a, <32 x i64>* %b) #0 {
+; CHECK-LABEL: @add_v32i64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: add [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x i64>, <32 x i64>* %a
+  %op2 = load <32 x i64>, <32 x i64>* %b
+  %res = add <32 x i64> %op1, %op2
+  store <32 x i64> %res, <32 x i64>* %a
+  ret void
+}
+
+attributes #0 = { "target-features"="+sve" }
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-loads.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-loads.ll
new file mode 100644
index 0000000000000..8478cfa3772ee
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-loads.ll
@@ -0,0 +1,104 @@
+; RUN: llc -aarch64-sve-vector-bits-min=128  < %s | FileCheck %s -D#VBYTES=16  -check-prefix=NO_SVE
+; RUN: llc -aarch64-sve-vector-bits-min=256  < %s | FileCheck %s -D#VBYTES=32  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512,VBITS_LE_256
+; RUN: llc -aarch64-sve-vector-bits-min=384  < %s | FileCheck %s -D#VBYTES=32  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512,VBITS_LE_256
+; RUN: llc -aarch64-sve-vector-bits-min=512  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=640  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=768  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=896  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=1024 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1152 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1280 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1408 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1536 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1664 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1792 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1920 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -D#VBYTES=256 -check-prefixes=CHECK
+
+; VBYTES represents the useful byte size of a vector register from the code
+; generator's point of view. It is clamped to power-of-2 values because
+; only power-of-2 vector lengths are considered legal, regardless of the
+; user specified vector length.
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Don't use SVE when its registers are no bigger than NEON.
+; NO_SVE-NOT: ptrue
+
+; Don't use SVE for 64-bit vectors.
+define <2 x float> @load_v2f32(<2 x float>* %a) #0 {
+; CHECK-LABEL: load_v2f32:
+; CHECK: ldr d0, [x0]
+; CHECK: ret
+  %load = load <2 x float>, <2 x float>* %a
+  ret <2 x float> %load
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x float> @load_v4f32(<4 x float>* %a) #0 {
+; CHECK-LABEL: load_v4f32:
+; CHECK: ldr q0, [x0]
+; CHECK: ret
+  %load = load <4 x float>, <4 x float>* %a
+  ret <4 x float> %load
+}
+
+define <8 x float> @load_v8f32(<8 x float>* %a) #0 {
+; CHECK-LABEL: load_v8f32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]]
+; CHECK: ld1w { z{{[0-9]+}}.s }, [[PG]]/z, [x0]
+; CHECK: ret
+  %load = load <8 x float>, <8 x float>* %a
+  ret <8 x float> %load
+}
+
+define <16 x float> @load_v16f32(<16 x float>* %a) #0 {
+; CHECK-LABEL: load_v16f32:
+; CHECK-DAG: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
+; CHECK-DAG: ld1w { z{{[0-9]+}}.s }, [[PG]]/z, [x0]
+; VBITS_LE_256-DAG: add x[[A1:[0-9]+]], x0, #[[#VBYTES]]
+; VBITS_LE_256-DAG: ld1w { z{{[0-9]+}}.s }, [[PG]]/z, [x[[A1]]]
+; CHECK: ret
+  %load = load <16 x float>, <16 x float>* %a
+  ret <16 x float> %load
+}
+
+define <32 x float> @load_v32f32(<32 x float>* %a) #0 {
+; CHECK-LABEL: load_v32f32:
+; CHECK-DAG: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
+; CHECK-DAG: ld1w { z{{[0-9]+}}.s }, [[PG]]/z, [x0]
+; VBITS_LE_512-DAG: add x[[A1:[0-9]+]], x0, #[[#VBYTES]]
+; VBITS_LE_512-DAG: ld1w { z{{[0-9]+}}.s }, [[PG]]/z, [x[[A1]]]
+; VBITS_LE_256-DAG: add x[[A2:[0-9]+]], x0, #[[#mul(VBYTES,2)]]
+; VBITS_LE_256-DAG: ld1w { z{{[0-9]+}}.s }, [[PG]]/z, [x[[A2]]]
+; VBITS_LE_256-DAG: add x[[A3:[0-9]+]], x0, #[[#mul(VBYTES,3)]]
+; VBITS_LE_256-DAG: ld1w { z{{[0-9]+}}.s }, [[PG]]/z, [x[[A3]]]
+; CHECK: ret
+  %load = load <32 x float>, <32 x float>* %a
+  ret <32 x float> %load
+}
+
+define <64 x float> @load_v64f32(<64 x float>* %a) #0 {
+; CHECK-LABEL: load_v64f32:
+; CHECK-DAG: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
+; CHECK-DAG: ld1w { z{{[0-9]+}}.s }, [[PG]]/z, [x0]
+; VBITS_LE_1024-DAG: add x[[A1:[0-9]+]], x0, #[[#VBYTES]]
+; VBITS_LE_1024-DAG: ld1w { z{{[0-9]+}}.s }, [[PG]]/z, [x[[A1]]]
+; VBITS_LE_512-DAG:  add x[[A2:[0-9]+]], x0, #[[#mul(VBYTES,2)]]
+; VBITS_LE_512-DAG:  ld1w { z{{[0-9]+}}.s }, [[PG]]/z, [x[[A2]]]
+; VBITS_LE_512-DAG:  add x[[A3:[0-9]+]], x0, #[[#mul(VBYTES,3)]]
+; VBITS_LE_512-DAG:  ld1w { z{{[0-9]+}}.s }, [[PG]]/z, [x[[A3]]]
+; VBITS_LE_256-DAG:  add x[[A4:[0-9]+]], x0, #[[#mul(VBYTES,4)]]
+; VBITS_LE_256-DAG:  ld1w { z{{[0-9]+}}.s }, [[PG]]/z, [x[[A4]]]
+; VBITS_LE_256-DAG:  add x[[A5:[0-9]+]], x0, #[[#mul(VBYTES,5)]]
+; VBITS_LE_256-DAG:  ld1w { z{{[0-9]+}}.s }, [[PG]]/z, [x[[A5]]]
+; VBITS_LE_256-DAG:  add x[[A6:[0-9]+]], x0, #[[#mul(VBYTES,6)]]
+; VBITS_LE_256-DAG:  ld1w { z{{[0-9]+}}.s }, [[PG]]/z, [x[[A6]]]
+; VBITS_LE_256-DAG:  add x[[A7:[0-9]+]], x0, #[[#mul(VBYTES,7)]]
+; VBITS_LE_256-DAG:  ld1w { z{{[0-9]+}}.s }, [[PG]]/z, [x[[A7]]]
+; CHECK: ret
+  %load = load <64 x float>, <64 x float>* %a
+  ret <64 x float> %load
+}
+
+attributes #0 = { "target-features"="+sve" }
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
new file mode 100644
index 0000000000000..6c01b51a4d818
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
@@ -0,0 +1,50 @@
+; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s
+; RUN: llc -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Currently there is no custom lowering for vector shuffles operating on types
+; bigger than NEON. However, having no support opens us up to a code generator
+; hang when expanding BUILD_VECTOR. Here we just validate the promblematic case
+; successfully exits code generation.
+define void @hang_when_merging_stores_after_legalisation(<8 x i32>* %a, <2 x i32> %b) #0 {
+; CHECK-LABEL: hang_when_merging_stores_after_legalisation:
+  %splat = shufflevector <2 x i32> %b, <2 x i32> undef, <8 x i32> zeroinitializer
+  %interleaved.vec = shufflevector <8 x i32> %splat, <8 x i32> undef, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+  store <8 x i32> %interleaved.vec, <8 x i32>* %a, align 4
+  ret void
+}
+
+; NOTE: Currently all CONCAT_VECTORS get expanded so there's little point in
+; validating all combinations of vector type.
+
+define void @concat_vectors_v4i64(<2 x i64> %a, <2 x i64> %b, <4 x i64> *%c.addr) #0 {
+; CHECK-LABEL: concat_vectors_v4i64:
+; CHECK: stp q0, q1, [sp]
+; CHECK: ptrue [[OUT_PG:p[0-9]+]].d, vl4
+; CHECK: mov x[[LO_ADDR:[0-9]+]], sp
+; CHECK: ld1d { z{{[0-9]+}}.d }, [[OUT_PG]]/z, [x[[LO_ADDR]]]
+  %concat = shufflevector <2 x i64> %a, <2 x i64> %b, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  store <4 x i64> %concat, <4 x i64>* %c.addr
+  ret void
+}
+
+define void @concat_vectors_v8i64(<4 x i64> *%a.addr, <4 x i64> *%b.addr, <8 x i64> *%c.addr) #0 {
+; VBITS_GE_512-LABEL: concat_vectors_v8i64:
+; VBITS_GE_512: ptrue [[IN_PG:p[0-9]+]].d, vl4
+; VBITS_GE_512: ld1d { [[LO:z[0-9]+]].d }, [[IN_PG]]/z, [x0]
+; VBITS_GE_512: ld1d { [[HI:z[0-9]+]].d }, [[IN_PG]]/z, [x1]
+; VBITS_GE_512: mov x[[LO_ADDR:[0-9]+]], sp
+; VBITS_GE_512: orr x[[HI_ADDR:[0-9]+]], x[[LO_ADDR]], #0x20
+; VBITS_GE_512: st1d { [[LO]].d }, [[IN_PG]], [x[[LO_ADDR]]]
+; VBITS_GE_512: st1d { [[HI]].d }, [[IN_PG]], [x[[HI_ADDR]]]
+; VBITS_GE_512: ptrue [[OUT_PG:p[0-9]+]].d, vl8
+; VBITS_GE_512: ld1d { z{{[0-9]+}}.d }, [[OUT_PG]]/z, [x8]
+  %a = load <4 x i64>, <4 x i64>* %a.addr
+  %b = load <4 x i64>, <4 x i64>* %b.addr
+  %concat = shufflevector <4 x i64> %a, <4 x i64> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  store <8 x i64> %concat, <8 x i64>* %c.addr
+  ret void
+}
+
+attributes #0 = { nounwind "target-features"="+sve" }
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-stores.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-stores.ll
new file mode 100644
index 0000000000000..3182d07cc6d94
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-stores.ll
@@ -0,0 +1,104 @@
+; RUN: llc -aarch64-sve-vector-bits-min=128  < %s | FileCheck %s -D#VBYTES=16  -check-prefix=NO_SVE
+; RUN: llc -aarch64-sve-vector-bits-min=256  < %s | FileCheck %s -D#VBYTES=32  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512,VBITS_LE_256
+; RUN: llc -aarch64-sve-vector-bits-min=384  < %s | FileCheck %s -D#VBYTES=32  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512,VBITS_LE_256
+; RUN: llc -aarch64-sve-vector-bits-min=512  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=640  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=768  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=896  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=1024 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1152 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1280 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1408 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1536 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1664 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1792 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1920 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -D#VBYTES=256 -check-prefixes=CHECK
+
+; VBYTES represents the useful byte size of a vector register from the code
+; generator's point of view. It is clamped to power-of-2 values because
+; only power-of-2 vector lengths are considered legal, regardless of the
+; user specified vector length.
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Don't use SVE when its registers are no bigger than NEON.
+; NO_SVE-NOT: ptrue
+
+; Don't use SVE for 64-bit vectors.
+define void @store_v2f32(<2 x float>* %a) #0 {
+; CHECK-LABEL: store_v2f32:
+; CHECK: str xzr, [x0]
+; CHECK: ret
+  store <2 x float> zeroinitializer, <2 x float>* %a
+  ret void
+}
+
+; Don't use SVE for 128-bit vectors.
+define void @store_v4f32(<4 x float>* %a) #0 {
+; CHECK-LABEL: store_v4f32:
+; CHECK: stp xzr, xzr, [x0]
+; CHECK: ret
+  store <4 x float> zeroinitializer, <4 x float>* %a
+  ret void
+}
+
+define void @store_v8f32(<8 x float>* %a) #0 {
+; CHECK-LABEL: store_v8f32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]]
+; CHECK: st1w { z{{[0-9]+}}.s }, [[PG]], [x0]
+; CHECK: ret
+  store <8 x float> zeroinitializer, <8 x float>* %a
+  ret void
+}
+
+define void @store_v16f32(<16 x float>* %a) #0 {
+; CHECK-LABEL: store_v16f32:
+; CHECK-DAG: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
+; CHECK-DAG: st1w { z{{[0-9]+}}.s }, [[PG]], [x0]
+; VBITS_LE_256-DAG: add x[[A1:[0-9]+]], x0, #[[#VBYTES]]
+; VBITS_LE_256-DAG: st1w { z{{[0-9]+}}.s }, [[PG]], [x[[A1]]]
+; CHECK: ret
+  store <16 x float> zeroinitializer, <16 x float>* %a
+  ret void
+}
+
+define void @store_v32f32(<32 x float>* %a) #0 {
+; CHECK-LABEL: store_v32f32:
+; CHECK-DAG: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
+; CHECK-DAG: st1w { z{{[0-9]+}}.s }, [[PG]], [x0]
+; VBITS_LE_512-DAG: add x[[A1:[0-9]+]], x0, #[[#VBYTES]]
+; VBITS_LE_512-DAG: st1w { z{{[0-9]+}}.s }, [[PG]], [x[[A1]]]
+; VBITS_LE_256-DAG: add x[[A2:[0-9]+]], x0, #[[#mul(VBYTES,2)]]
+; VBITS_LE_256-DAG: st1w { z{{[0-9]+}}.s }, [[PG]], [x[[A2]]]
+; VBITS_LE_256-DAG: add x[[A3:[0-9]+]], x0, #[[#mul(VBYTES,3)]]
+; VBITS_LE_256-DAG: st1w { z{{[0-9]+}}.s }, [[PG]], [x[[A3]]]
+; CHECK: ret
+  store <32 x float> zeroinitializer, <32 x float>* %a
+  ret void
+}
+
+define void @store_v64f32(<64 x float>* %a) #0 {
+; CHECK-LABEL: store_v64f32:
+; CHECK-DAG: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
+; CHECK-DAG: st1w { z{{[0-9]+}}.s }, [[PG]], [x0]
+; VBITS_LE_1024-DAG: add x[[A1:[0-9]+]], x0, #[[#VBYTES]]
+; VBITS_LE_1024-DAG: st1w { z{{[0-9]+}}.s }, [[PG]], [x[[A1]]]
+; VBITS_LE_512-DAG:  add x[[A2:[0-9]+]], x0, #[[#mul(VBYTES,2)]]
+; VBITS_LE_512-DAG:  st1w { z{{[0-9]+}}.s }, [[PG]], [x[[A2]]]
+; VBITS_LE_512-DAG:  add x[[A3:[0-9]+]], x0, #[[#mul(VBYTES,3)]]
+; VBITS_LE_512-DAG:  st1w { z{{[0-9]+}}.s }, [[PG]], [x[[A3]]]
+; VBITS_LE_256-DAG:  add x[[A4:[0-9]+]], x0, #[[#mul(VBYTES,4)]]
+; VBITS_LE_256-DAG:  st1w { z{{[0-9]+}}.s }, [[PG]], [x[[A4]]]
+; VBITS_LE_256-DAG:  add x[[A5:[0-9]+]], x0, #[[#mul(VBYTES,5)]]
+; VBITS_LE_256-DAG:  st1w { z{{[0-9]+}}.s }, [[PG]], [x[[A5]]]
+; VBITS_LE_256-DAG:  add x[[A6:[0-9]+]], x0, #[[#mul(VBYTES,6)]]
+; VBITS_LE_256-DAG:  st1w { z{{[0-9]+}}.s }, [[PG]], [x[[A6]]]
+; VBITS_LE_256-DAG:  add x[[A7:[0-9]+]], x0, #[[#mul(VBYTES,7)]]
+; VBITS_LE_256-DAG:  st1w { z{{[0-9]+}}.s }, [[PG]], [x[[A7]]]
+; CHECK: ret
+  store <64 x float> zeroinitializer, <64 x float>* %a
+  ret void
+}
+
+attributes #0 = { "target-features"="+sve" }
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-subvector.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-subvector.ll
new file mode 100644
index 0000000000000..45ebdc78784e4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-subvector.ll
@@ -0,0 +1,88 @@
+; RUN: llc -aarch64-sve-vector-bits-min=128  -aarch64-enable-atomic-cfg-tidy=false < %s | FileCheck %s -check-prefix=NO_SVE
+; RUN: llc -aarch64-sve-vector-bits-min=256  -aarch64-enable-atomic-cfg-tidy=false < %s | FileCheck %s -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=384  -aarch64-enable-atomic-cfg-tidy=false < %s | FileCheck %s -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=512  -aarch64-enable-atomic-cfg-tidy=false < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=640  -aarch64-enable-atomic-cfg-tidy=false < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=768  -aarch64-enable-atomic-cfg-tidy=false < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=896  -aarch64-enable-atomic-cfg-tidy=false < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=1024 -aarch64-enable-atomic-cfg-tidy=false < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1152 -aarch64-enable-atomic-cfg-tidy=false < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1280 -aarch64-enable-atomic-cfg-tidy=false < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1408 -aarch64-enable-atomic-cfg-tidy=false < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1536 -aarch64-enable-atomic-cfg-tidy=false < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1664 -aarch64-enable-atomic-cfg-tidy=false < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1792 -aarch64-enable-atomic-cfg-tidy=false < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1920 -aarch64-enable-atomic-cfg-tidy=false < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=2048 -aarch64-enable-atomic-cfg-tidy=false < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024,VBITS_GE_2048
+
+; Test we can code generater patterns of the form:
+;   fixed_length_vector = ISD::EXTRACT_SUBVECTOR scalable_vector, 0
+;   scalable_vector = ISD::INSERT_SUBVECTOR scalable_vector, fixed_length_vector, 0
+;
+; NOTE: Currently shufflevector does not support scalable vectors so it cannot
+; be used to model the above operations.  Instead these tests rely on knowing
+; how fixed length operation are lowered to scalable ones, with multiple blocks
+; ensuring insert/extract sequences are not folded away.
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Don't use SVE when its registers are no bigger than NEON.
+; NO_SVE-NOT: ptrue
+
+define void @subvector_v8i32(<8 x i32> *%in, <8 x i32>* %out) #0 {
+; CHECK-LABEL: subvector_v8i32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK: ld1w { [[DATA:z[0-9]+.s]] }, [[PG]]/z, [x0]
+; CHECK: st1w { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <8 x i32>, <8 x i32>* %in
+  br label %bb1
+
+bb1:
+  store <8 x i32> %a, <8 x i32>* %out
+  ret void
+}
+
+define void @subvector_v16i32(<16 x i32> *%in, <16 x i32>* %out) #0 {
+; CHECK-LABEL: subvector_v16i32:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512: ld1w { [[DATA:z[0-9]+.s]] }, [[PG]]/z, [x0]
+; VBITS_GE_512: st1w { [[DATA]] }, [[PG]], [x1]
+; CHECKT: ret
+  %a = load <16 x i32>, <16 x i32>* %in
+  br label %bb1
+
+bb1:
+  store <16 x i32> %a, <16 x i32>* %out
+  ret void
+}
+
+define void @subvector_v32i32(<32 x i32> *%in, <32 x i32>* %out) #0 {
+; CHECK-LABEL: subvector_v32i32:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024: ld1w { [[DATA:z[0-9]+.s]] }, [[PG]]/z, [x0]
+; VBITS_GE_1024: st1w { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <32 x i32>, <32 x i32>* %in
+  br label %bb1
+
+bb1:
+  store <32 x i32> %a, <32 x i32>* %out
+  ret void
+}
+
+define void @subvector_v64i32(<64 x i32> *%in, <64 x i32>* %out) #0 {
+; CHECK-LABEL: subvector_v64i32:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048: ld1w { [[DATA:z[0-9]+.s]] }, [[PG]]/z, [x0]
+; VBITS_GE_2048: st1w { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <64 x i32>, <64 x i32>* %in
+  br label %bb1
+
+bb1:
+  store <64 x i32> %a, <64 x i32>* %out
+  ret void
+}
+
+attributes #0 = { "target-features"="+sve" }
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-trunc.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-trunc.ll
new file mode 100644
index 0000000000000..f62abc0946066
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-trunc.ll
@@ -0,0 +1,369 @@
+; RUN: llc -aarch64-sve-vector-bits-min=128  -asm-verbose=0 < %s | FileCheck %s -check-prefix=NO_SVE
+; RUN: llc -aarch64-sve-vector-bits-min=256  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=384  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=512  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=640  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=768  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=896  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=1024 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1152 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1280 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1408 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1536 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1664 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1792 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1920 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=2048 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024,VBITS_GE_2048
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Don't use SVE when its registers are no bigger than NEON.
+; NO_SVE-NOT: z{0-9}
+
+;
+; truncate i16 -> i8
+;
+
+define <16 x i8> @trunc_v16i16_v16i8(<16 x i16>* %in) #0 {
+; CHECK-LABEL: trunc_v16i16_v16i8:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK-NEXT: ld1h { [[A_HALFS:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-NEXT: uzp1 z0.b, [[A_HALFS]].b, [[A_HALFS]].b
+; CHECK-NEXT: ret
+  %a = load <16 x i16>, <16 x i16>* %in
+  %b = trunc <16 x i16> %a to <16 x i8>
+  ret <16 x i8> %b
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v32i16_v32i8(<32 x i16>* %in, <32 x i8>* %out) #0 {
+; CHECK-LABEL: trunc_v32i16_v32i8:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512: ld1h { [[A_HALFS:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_512: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_512: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+  %a = load <32 x i16>, <32 x i16>* %in
+  %b = trunc <32 x i16> %a to <32 x i8>
+  %c = add <32 x i8> %b, %b
+  store <32 x i8> %c, <32 x i8>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v64i16_v64i8(<64 x i16>* %in, <64 x i8>* %out) #0 {
+; CHECK-LABEL: trunc_v64i16_v64i8:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024: ld1h { [[A_HALFS:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_1024: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_1024: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+  %a = load <64 x i16>, <64 x i16>* %in
+  %b = trunc <64 x i16> %a to <64 x i8>
+  %c = add <64 x i8> %b, %b
+  store <64 x i8> %c, <64 x i8>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v128i16_v128i8(<128 x i16>* %in, <128 x i8>* %out) #0 {
+; CHECK-LABEL: trunc_v128i16_v128i8:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048: ld1h { [[A_HALFS:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_2048: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_2048: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+  %a = load <128 x i16>, <128 x i16>* %in
+  %b = trunc <128 x i16> %a to <128 x i8>
+  %c = add <128 x i8> %b, %b
+  store <128 x i8> %c, <128 x i8>* %out
+  ret void
+}
+
+;
+; truncate i32 -> i8
+;
+
+define <8 x i8> @trunc_v8i32_v8i8(<8 x i32>* %in) #0 {
+; CHECK-LABEL: trunc_v8i32_v8i8:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-NEXT: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; CHECK-NEXT: uzp1 z0.b, [[A_HALFS]].b, [[A_HALFS]].b
+; CHECK-NEXT: ret
+  %a = load <8 x i32>, <8 x i32>* %in
+  %b = trunc <8 x i32> %a to <8 x i8>
+  ret <8 x i8> %b
+}
+
+define <16 x i8> @trunc_v16i32_v16i8(<16 x i32>* %in) #0 {
+; CHECK-LABEL: trunc_v16i32_v16i8:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_512-NEXT: uzp1 z0.b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_512-NEXT: ret
+  %a = load <16 x i32>, <16 x i32>* %in
+  %b = trunc <16 x i32> %a to <16 x i8>
+  ret <16 x i8> %b
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v32i32_v32i8(<32 x i32>* %in, <32 x i8>* %out) #0 {
+; CHECK-LABEL: trunc_v32i32_v32i8:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_1024: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_1024: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_1024: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+  %a = load <32 x i32>, <32 x i32>* %in
+  %b = trunc <32 x i32> %a to <32 x i8>
+  %c = add <32 x i8> %b, %b
+  store <32 x i8> %c, <32 x i8>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v64i32_v64i8(<64 x i32>* %in, <64 x i8>* %out) #0 {
+; CHECK-LABEL: trunc_v64i32_v64i8:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_2048: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_2048: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_2048: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+  %a = load <64 x i32>, <64 x i32>* %in
+  %b = trunc <64 x i32> %a to <64 x i8>
+  %c = add <64 x i8> %b, %b
+  store <64 x i8> %c, <64 x i8>* %out
+  ret void
+}
+
+;
+; truncate i32 -> i16
+;
+
+define <8 x i16> @trunc_v8i32_v8i16(<8 x i32>* %in) #0 {
+; CHECK-LABEL: trunc_v8i32_v8i16:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-NEXT: uzp1 z0.h, [[A_WORDS]].h, [[A_WORDS]].h
+; CHECK-NEXT: ret
+  %a = load <8 x i32>, <8 x i32>* %in
+  %b = trunc <8 x i32> %a to <8 x i16>
+  ret <8 x i16> %b
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v16i32_v16i16(<16 x i32>* %in, <16 x i16>* %out) #0 {
+; CHECK-LABEL: trunc_v16i32_v16i16:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_512: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_512: add [[A_HALFS]].h, [[PG]]/m, [[A_HALFS]].h, [[A_HALFS]].h
+  %a = load <16 x i32>, <16 x i32>* %in
+  %b = trunc <16 x i32> %a to <16 x i16>
+  %c = add <16 x i16> %b, %b
+  store <16 x i16> %c, <16 x i16>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v32i32_v32i16(<32 x i32>* %in, <32 x i16>* %out) #0 {
+; CHECK-LABEL: trunc_v32i32_v32i16:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_1024: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_1024: add [[A_HALFS]].h, [[PG]]/m, [[A_HALFS]].h, [[A_HALFS]].h
+  %a = load <32 x i32>, <32 x i32>* %in
+  %b = trunc <32 x i32> %a to <32 x i16>
+  %c = add <32 x i16> %b, %b
+  store <32 x i16> %c, <32 x i16>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v64i32_v64i16(<64 x i32>* %in, <64 x i16>* %out) #0 {
+; CHECK-LABEL: trunc_v64i32_v64i16:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_2048: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_2048: add [[A_HALFS]].h, [[PG]]/m, [[A_HALFS]].h, [[A_HALFS]].h
+  %a = load <64 x i32>, <64 x i32>* %in
+  %b = trunc <64 x i32> %a to <64 x i16>
+  %c = add <64 x i16> %b, %b
+  store <64 x i16> %c, <64 x i16>* %out
+  ret void
+}
+
+;
+; truncate i64 -> i8
+;
+
+; NOTE: v4i8 is not legal so result i8 elements are held within i16 containers.
+define <4 x i8> @trunc_v4i64_v4i8(<4 x i64>* %in) #0 {
+; CHECK-LABEL: trunc_v4i64_v4i8:
+; VBITS_GE_256: ptrue [[PG:p[0-9]+]].d, vl4
+; VBITS_GE_256-NEXT: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_256-NEXT: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_256-NEXT: uzp1 z0.h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_256-NEXT: ret
+  %a = load <4 x i64>, <4 x i64>* %in
+  %b = trunc <4 x i64> %a to <4 x i8>
+  ret <4 x i8> %b
+}
+
+define <8 x i8> @trunc_v8i64_v8i8(<8 x i64>* %in) #0 {
+; CHECK-LABEL: trunc_v8i64_v8i8:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_512-NEXT: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_512-NEXT: uzp1 z0.b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_512-NEXT: ret
+  %a = load <8 x i64>, <8 x i64>* %in
+  %b = trunc <8 x i64> %a to <8 x i8>
+  ret <8 x i8> %b
+}
+
+define <16 x i8> @trunc_v16i64_v16i8(<16 x i64>* %in) #0 {
+; CHECK-LABEL: trunc_v16i64_v16i8:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_1024-NEXT: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_1024-NEXT: uzp1 z0.b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_1024-NEXT: ret
+  %a = load <16 x i64>, <16 x i64>* %in
+  %b = trunc <16 x i64> %a to <16 x i8>
+  ret <16 x i8> %b
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v32i64_v32i8(<32 x i64>* %in, <32 x i8>* %out) #0 {
+; CHECK-LABEL: trunc_v32i64_v32i8:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_2048: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_2048: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_2048: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_2048: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+  %a = load <32 x i64>, <32 x i64>* %in
+  %b = trunc <32 x i64> %a to <32 x i8>
+  %c = add <32 x i8> %b, %b
+  store <32 x i8> %c, <32 x i8>* %out
+  ret void
+}
+
+;
+; truncate i64 -> i16
+;
+
+define <4 x i16> @trunc_v4i64_v4i16(<4 x i64>* %in) #0 {
+; CHECK-LABEL: trunc_v4i64_v4i16:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-NEXT: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; CHECK-NEXT: uzp1 z0.h, [[A_WORDS]].h, [[A_WORDS]].h
+; CHECK-NEXT: ret
+  %a = load <4 x i64>, <4 x i64>* %in
+  %b = trunc <4 x i64> %a to <4 x i16>
+  ret <4 x i16> %b
+}
+
+define <8 x i16> @trunc_v8i64_v8i16(<8 x i64>* %in) #0 {
+; CHECK-LABEL: trunc_v8i64_v8i16:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_512-NEXT: uzp1 z0.h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_512-NEXT: ret
+  %a = load <8 x i64>, <8 x i64>* %in
+  %b = trunc <8 x i64> %a to <8 x i16>
+  ret <8 x i16> %b
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v16i64_v16i16(<16 x i64>* %in, <16 x i16>* %out) #0 {
+; CHECK-LABEL: trunc_v16i64_v16i16:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_1024: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_1024: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_1024: add [[A_HALFS]].h, [[PG]]/m, [[A_HALFS]].h, [[A_HALFS]].h
+  %a = load <16 x i64>, <16 x i64>* %in
+  %b = trunc <16 x i64> %a to <16 x i16>
+  %c = add <16 x i16> %b, %b
+  store <16 x i16> %c, <16 x i16>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v32i64_v32i16(<32 x i64>* %in, <32 x i16>* %out) #0 {
+; CHECK-LABEL: trunc_v32i64_v32i16:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_2048: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_2048: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_2048: add [[A_HALFS]].h, [[PG]]/m, [[A_HALFS]].h, [[A_HALFS]].h
+  %a = load <32 x i64>, <32 x i64>* %in
+  %b = trunc <32 x i64> %a to <32 x i16>
+  %c = add <32 x i16> %b, %b
+  store <32 x i16> %c, <32 x i16>* %out
+  ret void
+}
+
+;
+; truncate i64 -> i32
+;
+
+define <4 x i32> @trunc_v4i64_v4i32(<4 x i64>* %in) #0 {
+; CHECK-LABEL: trunc_v4i64_v4i32:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-NEXT: uzp1 z0.s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; CHECK-NEXT: ret
+  %a = load <4 x i64>, <4 x i64>* %in
+  %b = trunc <4 x i64> %a to <4 x i32>
+  ret <4 x i32> %b
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v8i64_v8i32(<8 x i64>* %in, <8 x i32>* %out) #0 {
+; CHECK-LABEL: trunc_v8i64_v8i32:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_512: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_512: add [[A_WORDS]].s, [[PG]]/m, [[A_WORDS]].s, [[A_WORDS]].s
+  %a = load <8 x i64>, <8 x i64>* %in
+  %b = trunc <8 x i64> %a to <8 x i32>
+  %c = add <8 x i32> %b, %b
+  store <8 x i32> %c, <8 x i32>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v16i64_v16i32(<16 x i64>* %in, <16 x i32>* %out) #0 {
+; CHECK-LABEL: trunc_v16i64_v16i32:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_1024: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_1024: add [[A_WORDS]].s, [[PG]]/m, [[A_WORDS]].s, [[A_WORDS]].s
+  %a = load <16 x i64>, <16 x i64>* %in
+  %b = trunc <16 x i64> %a to <16 x i32>
+  %c = add <16 x i32> %b, %b
+  store <16 x i32> %c, <16 x i32>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v32i64_v32i32(<32 x i64>* %in, <32 x i32>* %out) #0 {
+; CHECK-LABEL: trunc_v32i64_v32i32:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_2048: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_2048: add [[A_WORDS]].s, [[PG]]/m, [[A_WORDS]].s, [[A_WORDS]].s
+  %a = load <32 x i64>, <32 x i64>* %in
+  %b = trunc <32 x i64> %a to <32 x i32>
+  %c = add <32 x i32> %b, %b
+  store <32 x i32> %c, <32 x i32>* %out
+  ret void
+}
+
+attributes #0 = { nounwind "target-features"="+sve" }
diff --git a/llvm/test/CodeGen/AArch64/sve-fp.ll b/llvm/test/CodeGen/AArch64/sve-fp.ll
index 05ca330b3668a..e3c0ba72bda1f 100644
--- a/llvm/test/CodeGen/AArch64/sve-fp.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fp.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
 ; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
 
@@ -5,124 +6,208 @@
 
 define <vscale x 8 x half> @fadd_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
 ; CHECK-LABEL: fadd_h:
-; CHECK: fadd z0.h, z0.h, z1.h
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fadd z0.h, z0.h, z1.h
+; CHECK-NEXT:    ret
   %res = fadd <vscale x 8 x half> %a, %b
   ret <vscale x 8 x half> %res
 }
 
 define <vscale x 4 x float> @fadd_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: fadd_s:
-; CHECK: fadd z0.s, z0.s, z1.s
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fadd z0.s, z0.s, z1.s
+; CHECK-NEXT:    ret
   %res = fadd <vscale x 4 x float> %a, %b
   ret <vscale x 4 x float> %res
 }
 
 define <vscale x 2 x double> @fadd_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
 ; CHECK-LABEL: fadd_d:
-; CHECK: fadd z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fadd z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = fadd <vscale x 2 x double> %a, %b
   ret <vscale x 2 x double> %res
 }
 
 define <vscale x 8 x half> @fsub_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
 ; CHECK-LABEL: fsub_h:
-; CHECK: fsub z0.h, z0.h, z1.h
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fsub z0.h, z0.h, z1.h
+; CHECK-NEXT:    ret
   %res = fsub <vscale x 8 x half> %a, %b
   ret <vscale x 8 x half> %res
 }
 
 define <vscale x 4 x float> @fsub_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: fsub_s:
-; CHECK: fsub z0.s, z0.s, z1.s
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fsub z0.s, z0.s, z1.s
+; CHECK-NEXT:    ret
   %res = fsub <vscale x 4 x float> %a, %b
   ret <vscale x 4 x float> %res
 }
 
 define <vscale x 2 x double> @fsub_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
 ; CHECK-LABEL: fsub_d:
-; CHECK: fsub z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fsub z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = fsub <vscale x 2 x double> %a, %b
   ret <vscale x 2 x double> %res
 }
 
 define <vscale x 8 x half> @fmul_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
 ; CHECK-LABEL: fmul_h:
-; CHECK: fmul z0.h, z0.h, z1.h
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmul z0.h, z0.h, z1.h
+; CHECK-NEXT:    ret
   %res = fmul <vscale x 8 x half> %a, %b
   ret <vscale x 8 x half> %res
 }
 
 define <vscale x 4 x float> @fmul_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: fmul_s:
-; CHECK: fmul z0.s, z0.s, z1.s
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmul z0.s, z0.s, z1.s
+; CHECK-NEXT:    ret
   %res = fmul <vscale x 4 x float> %a, %b
   ret <vscale x 4 x float> %res
 }
 
 define <vscale x 2 x double> @fmul_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
 ; CHECK-LABEL: fmul_d:
-; CHECK: fmul z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmul z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = fmul <vscale x 2 x double> %a, %b
   ret <vscale x 2 x double> %res
 }
 
+define <vscale x 8 x half> @fma_half(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fma_half:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    fmla z2.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    mov z0.d, z2.d
+; CHECK-NEXT:    ret
+  %r = call <vscale x 8 x half> @llvm.fma.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c)
+  ret <vscale x 8 x half> %r
+}
+define <vscale x 4 x float> @fma_float(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) {
+; CHECK-LABEL: fma_float:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fmla z2.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    mov z0.d, z2.d
+; CHECK-NEXT:    ret
+  %r = call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c)
+  ret <vscale x 4 x float> %r
+}
+define <vscale x 2 x double> @fma_double_1(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) {
+; CHECK-LABEL: fma_double_1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fmla z2.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    mov z0.d, z2.d
+; CHECK-NEXT:    ret
+  %r = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c)
+  ret <vscale x 2 x double> %r
+}
+define <vscale x 2 x double> @fma_double_2(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) {
+; CHECK-LABEL: fma_double_2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fmla z2.d, p0/m, z1.d, z0.d
+; CHECK-NEXT:    mov z0.d, z2.d
+; CHECK-NEXT:    ret
+  %r = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> %b, <vscale x 2 x double> %a, <vscale x 2 x double> %c)
+  ret <vscale x 2 x double> %r
+}
+define <vscale x 2 x double> @fma_double_3(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) {
+; CHECK-LABEL: fma_double_3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fmla z0.d, p0/m, z2.d, z1.d
+; CHECK-NEXT:    ret
+  %r = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> %c, <vscale x 2 x double> %b, <vscale x 2 x double> %a)
+  ret <vscale x 2 x double> %r
+}
+
 define <vscale x 8 x half> @frecps_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
 ; CHECK-LABEL: frecps_h:
-; CHECK: frecps z0.h, z0.h, z1.h
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frecps z0.h, z0.h, z1.h
+; CHECK-NEXT:    ret
   %res = call <vscale x 8 x half> @llvm.aarch64.sve.frecps.x.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b)
   ret <vscale x 8 x half> %res
 }
 
 define <vscale x 4 x float> @frecps_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: frecps_s:
-; CHECK: frecps z0.s, z0.s, z1.s
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frecps z0.s, z0.s, z1.s
+; CHECK-NEXT:    ret
   %res = call <vscale x 4 x float> @llvm.aarch64.sve.frecps.x.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b)
   ret <vscale x 4 x float> %res
 }
 
 define <vscale x 2 x double> @frecps_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
 ; CHECK-LABEL: frecps_d:
-; CHECK: frecps z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frecps z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = call <vscale x 2 x double> @llvm.aarch64.sve.frecps.x.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
   ret <vscale x 2 x double> %res
 }
 
 define <vscale x 8 x half> @frsqrts_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
 ; CHECK-LABEL: frsqrts_h:
-; CHECK: frsqrts z0.h, z0.h, z1.h
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frsqrts z0.h, z0.h, z1.h
+; CHECK-NEXT:    ret
   %res = call <vscale x 8 x half> @llvm.aarch64.sve.frsqrts.x.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b)
   ret <vscale x 8 x half> %res
 }
 
 define <vscale x 4 x float> @frsqrts_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: frsqrts_s:
-; CHECK: frsqrts z0.s, z0.s, z1.s
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frsqrts z0.s, z0.s, z1.s
+; CHECK-NEXT:    ret
   %res = call <vscale x 4 x float> @llvm.aarch64.sve.frsqrts.x.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b)
   ret <vscale x 4 x float> %res
 }
 
 define <vscale x 2 x double> @frsqrts_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
 ; CHECK-LABEL: frsqrts_d:
-; CHECK: frsqrts z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frsqrts z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = call <vscale x 2 x double> @llvm.aarch64.sve.frsqrts.x.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
   ret <vscale x 2 x double> %res
 }
 
+%complex = type { { double, double } }
+
+define void @scalar_to_vector(%complex* %outval, <vscale x 2 x i1> %pred, <vscale x 2 x double> %in1, <vscale x 2 x double> %in2) {
+; CHECK-LABEL: scalar_to_vector:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    faddv d0, p0, z0.d
+; CHECK-NEXT:    faddv d1, p0, z1.d
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+  %realp = getelementptr inbounds %complex, %complex* %outval, i64 0, i32 0, i32 0
+  %imagp = getelementptr inbounds %complex, %complex* %outval, i64 0, i32 0, i32 1
+  %1 = call double @llvm.aarch64.sve.faddv.nxv2f64(<vscale x 2 x i1> %pred, <vscale x 2 x double> %in1)
+  %2 = call double @llvm.aarch64.sve.faddv.nxv2f64(<vscale x 2 x i1> %pred, <vscale x 2 x double> %in2)
+  store double %1, double* %realp, align 8
+  store double %2, double* %imagp, align 8
+  ret void
+}
+
 declare <vscale x 8 x half> @llvm.aarch64.sve.frecps.x.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x float>  @llvm.aarch64.sve.frecps.x.nxv4f32(<vscale x 4 x float> , <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.frecps.x.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
@@ -130,3 +215,10 @@ declare <vscale x 2 x double> @llvm.aarch64.sve.frecps.x.nxv2f64(<vscale x 2 x d
 declare <vscale x 8 x half> @llvm.aarch64.sve.frsqrts.x.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.frsqrts.x.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.frsqrts.x.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)
+declare <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 8 x half> @llvm.fma.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
+
+; Function Attrs: nounwind readnone
+declare double @llvm.aarch64.sve.faddv.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>) #2
diff --git a/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll b/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll
index dffeee2df570b..e9e34ada83d19 100644
--- a/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll
+++ b/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ; Verify that DAG combine rules for LD1 + sext/zext don't apply when the
 ; result of LD1 has multiple uses
diff --git a/llvm/test/CodeGen/AArch64/sve-gep.ll b/llvm/test/CodeGen/AArch64/sve-gep.ll
index 1b558a833f3b0..48fc3ccb48bbc 100644
--- a/llvm/test/CodeGen/AArch64/sve-gep.ll
+++ b/llvm/test/CodeGen/AArch64/sve-gep.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 define <vscale x 2 x i64>* @scalar_of_scalable_1(<vscale x 2 x i64>* %base) {
 ; CHECK-LABEL: scalar_of_scalable_1:
diff --git a/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll b/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll
index d8d44a8f56114..b721cc7b00c55 100644
--- a/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; SMAX
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-bfloat.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-bfloat.ll
new file mode 100644
index 0000000000000..faccbcc9b3ebc
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-bfloat.ll
@@ -0,0 +1,243 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+bf16 -asm-verbose=0 < %s | FileCheck %s
+
+;
+; BFDOT
+;
+
+define <vscale x 4 x float> @bfdot_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfdot_f32:
+; CHECK-NEXT:  bfdot z0.s, z1.h, z2.h
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfdot(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfdot_lane_0_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfdot_lane_0_f32:
+; CHECK-NEXT:  bfdot z0.s, z1.h, z2.h[0]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfdot.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 0)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfdot_lane_1_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfdot_lane_1_f32:
+; CHECK-NEXT:  bfdot z0.s, z1.h, z2.h[1]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfdot.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 1)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfdot_lane_2_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfdot_lane_2_f32:
+; CHECK-NEXT:  bfdot z0.s, z1.h, z2.h[2]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfdot.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 2)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfdot_lane_3_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfdot_lane_3_f32:
+; CHECK-NEXT:  bfdot z0.s, z1.h, z2.h[3]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfdot.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 3)
+  ret <vscale x 4 x float> %out
+}
+
+;
+; BFMLALB
+;
+
+define <vscale x 4 x float> @bfmlalb_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalb_f32:
+; CHECK-NEXT:  bfmlalb z0.s, z1.h, z2.h
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalb(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlalb_lane_0_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalb_lane_0_f32:
+; CHECK-NEXT:  bfmlalb z0.s, z1.h, z2.h[0]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalb.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 0)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlalb_lane_1_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalb_lane_1_f32:
+; CHECK-NEXT:  bfmlalb z0.s, z1.h, z2.h[1]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalb.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 1)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlalb_lane_2_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalb_lane_2_f32:
+; CHECK-NEXT:  bfmlalb z0.s, z1.h, z2.h[2]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalb.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 2)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlalb_lane_3_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalb_lane_3_f32:
+; CHECK-NEXT:  bfmlalb z0.s, z1.h, z2.h[3]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalb.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 3)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlalb_lane_4_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalb_lane_4_f32:
+; CHECK-NEXT:  bfmlalb z0.s, z1.h, z2.h[4]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalb.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 4)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlalb_lane_5_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalb_lane_5_f32:
+; CHECK-NEXT:  bfmlalb z0.s, z1.h, z2.h[5]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalb.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 5)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlalb_lane_6_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalb_lane_6_f32:
+; CHECK-NEXT:  bfmlalb z0.s, z1.h, z2.h[6]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalb.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 6)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlalb_lane_7_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalb_lane_7_f32:
+; CHECK-NEXT:  bfmlalb z0.s, z1.h, z2.h[7]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalb.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 7)
+  ret <vscale x 4 x float> %out
+}
+
+;
+; BFMLALT
+;
+
+define <vscale x 4 x float> @bfmlalt_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalt_f32:
+; CHECK-NEXT:  bfmlalt z0.s, z1.h, z2.h
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlalt_lane_0_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalt_lane_0_f32:
+; CHECK-NEXT:  bfmlalt z0.s, z1.h, z2.h[0]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 0)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlalt_lane_1_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalt_lane_1_f32:
+; CHECK-NEXT:  bfmlalt z0.s, z1.h, z2.h[1]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 1)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlalt_lane_2_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalt_lane_2_f32:
+; CHECK-NEXT:  bfmlalt z0.s, z1.h, z2.h[2]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 2)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlalt_lane_3_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalt_lane_3_f32:
+; CHECK-NEXT:  bfmlalt z0.s, z1.h, z2.h[3]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 3)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlalt_lane_4_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalt_lane_4_f32:
+; CHECK-NEXT:  bfmlalt z0.s, z1.h, z2.h[4]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 4)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlalt_lane_5_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalt_lane_5_f32:
+; CHECK-NEXT:  bfmlalt z0.s, z1.h, z2.h[5]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 5)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlalt_lane_6_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalt_lane_6_f32:
+; CHECK-NEXT:  bfmlalt z0.s, z1.h, z2.h[6]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 6)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlalt_lane_7_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmlalt_lane_7_f32:
+; CHECK-NEXT:  bfmlalt z0.s, z1.h, z2.h[7]
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt.lane(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, i64 7)
+  ret <vscale x 4 x float> %out
+}
+
+;
+; BFMMLA
+;
+
+define <vscale x 4 x float> @bfmmla_f32(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) nounwind {
+; CHECK-LABEL: bfmmla_f32:
+; CHECK-NEXT:  bfmmla z0.s, z1.h, z2.h
+; CHECK-NEXT:  ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmmla(<vscale x 4 x float> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c)
+  ret <vscale x 4 x float> %out
+}
+
+;
+; BFCVT
+;
+
+define <vscale x 8 x bfloat> @fcvt_bf16_f32(<vscale x 8 x bfloat> %a, <vscale x 8 x i1> %pg, <vscale x 4 x float> %b) nounwind {
+; CHECK-LABEL: fcvt_bf16_f32:
+; CHECK-NEXT: bfcvt z0.h, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> %a, <vscale x 8 x i1> %pg, <vscale x 4 x float> %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
+;
+; BFCVTNT
+;
+
+define <vscale x 8 x bfloat> @fcvtnt_bf16_f32(<vscale x 8 x bfloat> %a, <vscale x 8 x i1> %pg, <vscale x 4 x float> %b) nounwind {
+; CHECK-LABEL: fcvtnt_bf16_f32:
+; CHECK-NEXT: bfcvtnt z0.h, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32(<vscale x 8 x bfloat> %a, <vscale x 8 x i1> %pg, <vscale x 4 x float> %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
+declare <vscale x 4 x float> @llvm.aarch64.sve.bfdot(<vscale x 4 x float>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.bfdot.lane(<vscale x 4 x float>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i64)
+declare <vscale x 4 x float> @llvm.aarch64.sve.bfmlalb(<vscale x 4 x float>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.bfmlalb.lane(<vscale x 4 x float>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i64)
+declare <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt(<vscale x 4 x float>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt.lane(<vscale x 4 x float>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i64)
+declare <vscale x 4 x float> @llvm.aarch64.sve.bfmmla(<vscale x 4 x float>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat>, <vscale x 8 x i1>, <vscale x 4 x float>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32(<vscale x 8 x bfloat>, <vscale x 8 x i1>, <vscale x 4 x float>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-counting-bits.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-counting-bits.ll
index 83e985fd2a3c3..40826c26af0d0 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-counting-bits.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-counting-bits.ll
@@ -145,6 +145,16 @@ define <vscale x 8 x i16> @cnt_f16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg,
   ret <vscale x 8 x i16> %out
 }
 
+define <vscale x 8 x i16> @cnt_bf16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: cnt_bf16:
+; CHECK: cnt z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.cnt.nxv8bf16(<vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x bfloat> %b)
+  ret <vscale x 8 x i16> %out
+}
+
 define <vscale x 4 x i32> @cnt_f32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: cnt_f32:
 ; CHECK: cnt z0.s, p0/m, z1.s
@@ -180,5 +190,9 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.cnt.nxv8i16(<vscale x 8 x i16>, <vs
 declare <vscale x 4 x i32> @llvm.aarch64.sve.cnt.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.cnt.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.cnt.nxv8f16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.cnt.nxv8bf16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x bfloat>)
 declare <vscale x 4 x i32> @llvm.aarch64.sve.cnt.nxv4f32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x float>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.cnt.nxv2f64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-create-tuple.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-create-tuple.ll
index 38b05e42b962d..2f2b8f5af3b17 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-create-tuple.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-create-tuple.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple aarch64 -mattr=+sve -asm-verbose=1 < %s | FileCheck %s
+; RUN: llc -mtriple aarch64 -mattr=+sve -asm-verbose=1 < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; SVCREATE2 (i8)
@@ -93,6 +96,37 @@ L2:
   ret <vscale x 8 x half> %extract
 }
 
+;
+; SVCREATE2 (bfloat)
+;
+
+define <vscale x 8 x bfloat> @test_svcreate2_bf16_vec0(i1 %p, <vscale x 8 x bfloat> %z0, <vscale x 8 x bfloat> %z1) local_unnamed_addr #1 {
+; CHECK-LABEL: test_svcreate2_bf16_vec0:
+; CHECK: // %L2
+; CHECK-NEXT: ret
+  %tuple = tail call <vscale x 16 x bfloat> @llvm.aarch64.sve.tuple.create2.nxv16bf16.nxv8bf16(<vscale x 8 x bfloat> %z0, <vscale x 8 x bfloat> %z1)
+  br i1 %p, label %L1, label %L2
+L1:
+  ret <vscale x 8 x bfloat> undef
+L2:
+  %extract = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> %tuple, i32 0)
+  ret <vscale x 8 x bfloat> %extract
+}
+
+define <vscale x 8 x bfloat> @test_svcreate2_bf16_vec1(i1 %p, <vscale x 8 x bfloat> %z0, <vscale x 8 x bfloat> %z1) local_unnamed_addr #1 {
+; CHECK-LABEL: test_svcreate2_bf16_vec1:
+; CHECK: // %L2
+; CHECK-NEXT: mov z0.d, z1.d
+; CHECK-NEXT: ret
+  %tuple = tail call <vscale x 16 x bfloat> @llvm.aarch64.sve.tuple.create2.nxv16bf16.nxv8bf16(<vscale x 8 x bfloat> %z0, <vscale x 8 x bfloat> %z1)
+  br i1 %p, label %L1, label %L2
+L1:
+  ret <vscale x 8 x bfloat> undef
+L2:
+  %extract = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> %tuple, i32 1)
+  ret <vscale x 8 x bfloat> %extract
+}
+
 ;
 ; SVCREATE2 (i32)
 ;
@@ -278,6 +312,7 @@ L2:
   %extract = tail call <vscale x 8 x i16> @llvm.aarch64.sve.tuple.get.nxv8i16.nxv24i16(<vscale x 24 x i16> %tuple, i32 2)
   ret <vscale x 8 x i16> %extract
 }
+
 ;
 ; SVCREATE3 (half)
 ;
@@ -309,6 +344,36 @@ L2:
   ret <vscale x 8 x half> %extract
 }
 
+;
+; SVCREATE3 (bfloat)
+;
+
+define <vscale x 8 x bfloat> @test_svcreate3_bf16_vec0(i1 %p, <vscale x 8 x bfloat> %z0, <vscale x 8 x bfloat> %z1, <vscale x 8 x bfloat> %z2) local_unnamed_addr #1 {
+; CHECK-LABEL: test_svcreate3_bf16_vec0:
+; CHECK: // %L2
+; CHECK-NEXT: ret
+  %tuple = tail call <vscale x 24 x bfloat> @llvm.aarch64.sve.tuple.create3.nxv24bf16.nxv8bf16(<vscale x 8 x bfloat> %z0, <vscale x 8 x bfloat> %z1, <vscale x 8 x bfloat> %z2)
+  br i1 %p, label %L1, label %L2
+L1:
+  ret <vscale x 8 x bfloat> undef
+L2:
+  %extract = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv24bf16(<vscale x 24 x bfloat> %tuple, i32 0)
+  ret <vscale x 8 x bfloat> %extract
+}
+
+define <vscale x 8 x bfloat> @test_svcreate3_bf16_vec2(i1 %p, <vscale x 8 x bfloat> %z0, <vscale x 8 x bfloat> %z1, <vscale x 8 x bfloat> %z2) local_unnamed_addr #1 {
+; CHECK-LABEL: test_svcreate3_bf16_vec2:
+; CHECK: // %L2
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+  %tuple = tail call <vscale x 24 x bfloat> @llvm.aarch64.sve.tuple.create3.nxv24bf16.nxv8bf16(<vscale x 8 x bfloat> %z0, <vscale x 8 x bfloat> %z1, <vscale x 8 x bfloat> %z2)
+  br i1 %p, label %L1, label %L2
+L1:
+  ret <vscale x 8 x bfloat> undef
+L2:
+  %extract = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv24bf16(<vscale x 24 x bfloat> %tuple, i32 2)
+  ret <vscale x 8 x bfloat> %extract
+}
 
 ;
 ; SVCREATE3 (i32)
@@ -527,6 +592,37 @@ L2:
   ret <vscale x 8 x half> %extract
 }
 
+;
+; SVCREATE4 (bfloat)
+;
+
+define <vscale x 8 x bfloat> @test_svcreate4_bf16_vec0(i1 %p, <vscale x 8 x bfloat> %z0, <vscale x 8 x bfloat> %z1, <vscale x 8 x bfloat> %z2, <vscale x 8 x bfloat> %z3) local_unnamed_addr #1 {
+; CHECK-LABEL: test_svcreate4_bf16_vec0:
+; CHECK: // %L2
+; CHECK-NEXT: ret
+  %tuple = tail call <vscale x 32 x bfloat> @llvm.aarch64.sve.tuple.create4.nxv32bf16.nxv8bf16(<vscale x 8 x bfloat> %z0, <vscale x 8 x bfloat> %z1, <vscale x 8 x bfloat> %z2, <vscale x 8 x bfloat> %z3)
+  br i1 %p, label %L1, label %L2
+L1:
+  ret <vscale x 8 x bfloat> undef
+L2:
+  %extract = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> %tuple, i32 0)
+  ret <vscale x 8 x bfloat> %extract
+}
+
+define <vscale x 8 x bfloat> @test_svcreate4_bf16_vec3(i1 %p, <vscale x 8 x bfloat> %z0, <vscale x 8 x bfloat> %z1, <vscale x 8 x bfloat> %z2, <vscale x 8 x bfloat> %z3) local_unnamed_addr #1 {
+; CHECK-LABEL: test_svcreate4_bf16_vec3:
+; CHECK: // %L2
+; CHECK-NEXT: mov z0.d, z3.d
+; CHECK-NEXT: ret
+  %tuple = tail call <vscale x 32 x bfloat> @llvm.aarch64.sve.tuple.create4.nxv32bf16.nxv8bf16(<vscale x 8 x bfloat> %z0, <vscale x 8 x bfloat> %z1, <vscale x 8 x bfloat> %z2, <vscale x 8 x bfloat> %z3)
+  br i1 %p, label %L1, label %L2
+L1:
+  ret <vscale x 8 x bfloat> undef
+L2:
+  %extract = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> %tuple, i32 3)
+  ret <vscale x 8 x bfloat> %extract
+}
+
 ;
 ; SVCREATE4 (i32)
 ;
@@ -652,10 +748,13 @@ L2:
 }
 
 attributes #0 = { nounwind "target-features"="+sve" }
+; +bf16 is required for the bfloat version.
+attributes #1 = { nounwind "target-features"="+sve,+bf16" }
 
 declare <vscale x 4 x double>  @llvm.aarch64.sve.tuple.create2.nxv4f64.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
 declare <vscale x 8 x float>  @llvm.aarch64.sve.tuple.create2.nxv8f32.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 16 x half>  @llvm.aarch64.sve.tuple.create2.nxv16f16.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 16 x bfloat>  @llvm.aarch64.sve.tuple.create2.nxv16bf16.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
 declare <vscale x 4 x i64>  @llvm.aarch64.sve.tuple.create2.nxv4i64.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
 declare <vscale x 8 x i32>  @llvm.aarch64.sve.tuple.create2.nxv8i32.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 16 x i16> @llvm.aarch64.sve.tuple.create2.nxv16i16.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
@@ -664,6 +763,7 @@ declare <vscale x 32 x i8>  @llvm.aarch64.sve.tuple.create2.nxv32i8.nxv16i8(<vsc
 declare <vscale x 6 x double>  @llvm.aarch64.sve.tuple.create3.nxv6f64.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)
 declare <vscale x 12 x float> @llvm.aarch64.sve.tuple.create3.nxv12f32.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 24 x half> @llvm.aarch64.sve.tuple.create3.nxv24f16.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 24 x bfloat> @llvm.aarch64.sve.tuple.create3.nxv24bf16.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
 declare <vscale x 6 x i64>  @llvm.aarch64.sve.tuple.create3.nxv6i64.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
 declare <vscale x 12 x i32> @llvm.aarch64.sve.tuple.create3.nxv12i32.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 24 x i16> @llvm.aarch64.sve.tuple.create3.nxv24i16.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)
@@ -672,6 +772,7 @@ declare <vscale x 48 x i8>  @llvm.aarch64.sve.tuple.create3.nxv48i8.nxv16i8(<vsc
 declare <vscale x 8 x double>  @llvm.aarch64.sve.tuple.create4.nxv8f64.nxv2f64 (<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)
 declare <vscale x 16 x float>  @llvm.aarch64.sve.tuple.create4.nxv16f32.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 32 x half>  @llvm.aarch64.sve.tuple.create4.nxv32f16.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 32 x bfloat>  @llvm.aarch64.sve.tuple.create4.nxv32bf16.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
 declare <vscale x  8 x i64> @llvm.aarch64.sve.tuple.create4.nxv8i64.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
 declare <vscale x 16 x i32> @llvm.aarch64.sve.tuple.create4.nxv16i32.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 32 x i16> @llvm.aarch64.sve.tuple.create4.nxv32i16.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)
@@ -693,6 +794,10 @@ declare <vscale x 2 x i64> @llvm.aarch64.sve.tuple.get.nxv2i64.nxv4i64(<vscale x
 declare <vscale x 2 x i64> @llvm.aarch64.sve.tuple.get.nxv2i64.nxv6i64(<vscale x 6 x i64>, i32 immarg)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.tuple.get.nxv2i64.nxv8i64(<vscale x 8 x i64>, i32 immarg)
 
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat>, i32 immarg)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv24bf16(<vscale x 24 x bfloat>, i32 immarg)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.tuple.get.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat>, i32 immarg)
+
 declare <vscale x 8 x half> @llvm.aarch64.sve.tuple.get.nxv8f16.nxv16f16(<vscale x 16 x half>, i32 immarg)
 declare <vscale x 8 x half> @llvm.aarch64.sve.tuple.get.nxv8f16.nxv24f16(<vscale x 24 x half>, i32 immarg)
 declare <vscale x 8 x half> @llvm.aarch64.sve.tuple.get.nxv8f16.nxv32f16(<vscale x 32 x half>, i32 immarg)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-dup-x.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-dup-x.ll
index d6f240a15bed8..788592c131bc8 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-dup-x.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-dup-x.ll
@@ -81,6 +81,14 @@ define <vscale x 8 x half> @dup_f16(half %b) {
   ret <vscale x 8 x half> %out
 }
 
+define <vscale x 8 x bfloat> @dup_bf16(bfloat %b) #0 {
+; CHECK-LABEL: dup_bf16:
+; CHECK: mov z0.h, h0
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.dup.x.nxv8bf16(bfloat %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 8 x half> @dup_imm_f16(half %b) {
 ; CHECK-LABEL: dup_imm_f16:
 ; CHECK: mov z0.h, #16.00000000
@@ -126,5 +134,9 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.dup.x.nxv8i16(i16)
 declare <vscale x 4 x i32> @llvm.aarch64.sve.dup.x.nxv4i32(i32)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.dup.x.nxv2i64(i64)
 declare <vscale x 8 x half> @llvm.aarch64.sve.dup.x.nxv8f16(half)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.dup.x.nxv8bf16(bfloat)
 declare <vscale x 4 x float> @llvm.aarch64.sve.dup.x.nxv4f32(float)
 declare <vscale x 2 x double> @llvm.aarch64.sve.dup.x.nxv2f64(double)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-32bit-scaled-offsets.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-32bit-scaled-offsets.ll
index a86da9594a212..5637a6982c2ab 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-32bit-scaled-offsets.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-32bit-scaled-offsets.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LDFF1H, LDFF1W, LDFF1D: base + 32-bit scaled offset, sign (sxtw) or zero (uxtw)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-32bit-unscaled-offsets.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-32bit-unscaled-offsets.ll
index 012812fb22b06..27aa5622160d4 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-32bit-unscaled-offsets.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-32bit-unscaled-offsets.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LDFF1B, LDFF1W, LDFF1H, LDFF1D: base + 32-bit unscaled offset, sign (sxtw) or zero
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-64bit-scaled-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-64bit-scaled-offset.ll
index 4d52673560817..6cfbddf031daf 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-64bit-scaled-offset.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-64bit-scaled-offset.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LDFF1H, LDFF1W, LDFF1D: base + 64-bit scaled offset
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-64bit-unscaled-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-64bit-unscaled-offset.ll
index 570bac58cc9ac..9e17b470037a2 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-64bit-unscaled-offset.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-64bit-unscaled-offset.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LDFF1B, LDFF1W, LDFF1H, LDFF1D: base + 64-bit unscaled offset
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-vector-base-imm-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-vector-base-imm-offset.ll
index 5cb887932eff6..d591614b964ca 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-vector-base-imm-offset.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-vector-base-imm-offset.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LDFF1B, LDFF1W, LDFF1H, LDFF1D: vector base + immediate offset (index)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-vector-base-scalar-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-vector-base-scalar-offset.ll
index 4b9fba9dc2756..6534f32cfbb10 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-vector-base-scalar-offset.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ff-gather-loads-vector-base-scalar-offset.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LDFF1B, LDFF1W, LDFF1H, LDFF1D: vector base + scalar offset (index)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-arith-merging.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-arith-merging.ll
index e86bf037f79d9..1bcae7135fd36 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-arith-merging.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-arith-merging.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=sve < %s 2>%t | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=sve -mattr=+use-experimental-zeroing-pseudos < %s 2>%t | FileCheck %s
 ; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
 
 ; WARN-NOT: warning
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-scaled-offsets.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-scaled-offsets.ll
index db593413f7af6..b03e1a25f5bf6 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-scaled-offsets.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-scaled-offsets.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LD1H, LD1W, LD1D: base + 32-bit scaled offset, sign (sxtw) or zero (uxtw)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-unscaled-offsets.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-unscaled-offsets.ll
index ba8806986d690..cf38473237340 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-unscaled-offsets.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-unscaled-offsets.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LD1B, LD1W, LD1H, LD1D: base + 32-bit unscaled offset, sign (sxtw) or zero
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-scaled-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-scaled-offset.ll
index 10de34975fa0d..3818c6178faad 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-scaled-offset.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-scaled-offset.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LD1H, LD1W, LD1D: base + 64-bit scaled offset
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-unscaled-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-unscaled-offset.ll
index fddbc24e911ae..87580c92e710c 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-unscaled-offset.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-unscaled-offset.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LD1B, LD1W, LD1H, LD1D: base + 64-bit unscaled offset
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-vector-base-imm-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-vector-base-imm-offset.ll
index c7798e7f52d25..856d29aec7a48 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-vector-base-imm-offset.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-vector-base-imm-offset.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LD1B, LD1W, LD1H, LD1D: vector base + immediate offset (index)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-vector-base-scalar-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-vector-base-scalar-offset.ll
index 3d84c0bbfc719..f877d24111da5 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-vector-base-scalar-offset.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-vector-base-scalar-offset.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LD1B, LD1W, LD1H, LD1D: vector base + scalar offset (index)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll
index 60a9dd88da6e5..18d7f3515756a 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ; SMAX
 
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-merging.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-merging.ll
index 733f509838645..50bb76d2ffb9e 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-merging.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-merging.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -mattr=+use-experimental-zeroing-pseudos < %s 2>%t | FileCheck %s
 ; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
 
 ; WARN-NOT: warning
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1-addressing-mode-reg-imm.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1-addressing-mode-reg-imm.ll
index 36765ad132428..2aaf222504a70 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1-addressing-mode-reg-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1-addressing-mode-reg-imm.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LD1B
@@ -207,6 +210,17 @@ define <vscale x 8 x half> @ld1h_f16_inbound(<vscale x 8 x i1> %pg, half* %a) {
   ret <vscale x 8 x half> %load
 }
 
+define <vscale x 8 x bfloat> @ld1h_bf16_inbound(<vscale x 8 x i1> %pg, bfloat* %a) #0 {
+; CHECK-LABEL: ld1h_bf16_inbound:
+; CHECK: ld1h { z0.h }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast bfloat* %a to <vscale x 8 x bfloat>*
+  %base = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %base_scalable, i64 1
+  %base_scalar = bitcast <vscale x 8 x bfloat>* %base to bfloat*
+  %load = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1.nxv8bf16(<vscale x 8 x i1> %pg, bfloat* %base_scalar)
+  ret <vscale x 8 x bfloat> %load
+}
+
 ;
 ; LD1W
 ;
@@ -288,6 +302,7 @@ declare <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1>, i8*
 declare <vscale x 8 x i8> @llvm.aarch64.sve.ld1.nxv8i8(<vscale x 8 x i1>, i8*)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1>, i16*)
 declare <vscale x 8 x half> @llvm.aarch64.sve.ld1.nxv8f16(<vscale x 8 x i1>, half*)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1.nxv8bf16(<vscale x 8 x i1>, bfloat*)
 
 declare <vscale x 4 x i8> @llvm.aarch64.sve.ld1.nxv4i8(<vscale x 4 x i1>, i8*)
 declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.nxv4i16(<vscale x 4 x i1>, i16*)
@@ -299,3 +314,6 @@ declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.nxv2i16(<vscale x 2 x i1>, i16*
 declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.nxv2i32(<vscale x 2 x i1>, i32*)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.nxv2i64(<vscale x 2 x i1>, i64*)
 declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.nxv2f64(<vscale x 2 x i1>, double*)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1-addressing-mode-reg-reg.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1-addressing-mode-reg-reg.ll
index 29099e553906f..e66b84a741033 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1-addressing-mode-reg-reg.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1-addressing-mode-reg-reg.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LD1B
@@ -95,6 +98,15 @@ define <vscale x 8 x half> @ld1h_f16(<vscale x 8 x i1> %pg, half* %a, i64 %index
   ret <vscale x 8 x half> %load
 }
 
+define <vscale x 8 x bfloat> @ld1h_bf16(<vscale x 8 x i1> %pg, bfloat* %a, i64 %index) #0 {
+; CHECK-LABEL: ld1h_bf16
+; CHECK: ld1h { z0.h }, p0/z, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+  %base = getelementptr bfloat, bfloat* %a, i64 %index
+  %load = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1.nxv8bf16(<vscale x 8 x i1> %pg, bfloat* %base)
+  ret <vscale x 8 x bfloat> %load
+}
+
 define <vscale x 4 x i32> @ld1h_s(<vscale x 4 x i1> %pred, i16* %a, i64 %index) {
 ; CHECK-LABEL: ld1h_s:
 ; CHECK: ld1h { z0.s }, p0/z, [x0, x1, lsl #1]
@@ -204,6 +216,7 @@ declare <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1>, i8*
 declare <vscale x 8 x i8> @llvm.aarch64.sve.ld1.nxv8i8(<vscale x 8 x i1>, i8*)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1>, i16*)
 declare <vscale x 8 x half> @llvm.aarch64.sve.ld1.nxv8f16(<vscale x 8 x i1>, half*)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1.nxv8bf16(<vscale x 8 x i1>, bfloat*)
 
 declare <vscale x 4 x i8> @llvm.aarch64.sve.ld1.nxv4i8(<vscale x 4 x i1>, i8*)
 declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.nxv4i16(<vscale x 4 x i1>, i16*)
@@ -215,3 +228,6 @@ declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.nxv2i16(<vscale x 2 x i1>, i16*
 declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.nxv2i32(<vscale x 2 x i1>, i32*)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.nxv2i64(<vscale x 2 x i1>, i64*)
 declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.nxv2f64(<vscale x 2 x i1>, double*)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1.ll
index 30d6eeb8c733c..e5d200945098c 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1.ll
@@ -1,5 +1,8 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
 ; RUN: llc -O0 -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LD1B
@@ -87,6 +90,14 @@ define <vscale x 8 x half> @ld1h_f16(<vscale x 8 x i1> %pred, half* %addr) {
   ret <vscale x 8 x half> %res
 }
 
+define <vscale x 8 x bfloat> @ld1h_bf16(<vscale x 8 x i1> %pred, bfloat* %addr) #0 {
+; CHECK-LABEL: ld1h_bf16:
+; CHECK: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1.nxv8bf16(<vscale x 8 x i1> %pred, bfloat* %addr)
+  ret <vscale x 8 x bfloat> %res
+}
+
 define <vscale x 4 x i32> @ld1h_s(<vscale x 4 x i1> %pred, i16* %addr) {
 ; CHECK-LABEL: ld1h_s:
 ; CHECK: ld1h { z0.s }, p0/z, [x0]
@@ -188,6 +199,7 @@ declare <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1>, i8*
 declare <vscale x 8 x i8> @llvm.aarch64.sve.ld1.nxv8i8(<vscale x 8 x i1>, i8*)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1>, i16*)
 declare <vscale x 8 x half> @llvm.aarch64.sve.ld1.nxv8f16(<vscale x 8 x i1>, half*)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1.nxv8bf16(<vscale x 8 x i1>, bfloat*)
 
 declare <vscale x 4 x i8> @llvm.aarch64.sve.ld1.nxv4i8(<vscale x 4 x i1>, i8*)
 declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.nxv4i16(<vscale x 4 x i1>, i16*)
@@ -199,3 +211,6 @@ declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.nxv2i16(<vscale x 2 x i1>, i16*
 declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.nxv2i32(<vscale x 2 x i1>, i32*)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.nxv2i64(<vscale x 2 x i1>, i64*)
 declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.nxv2f64(<vscale x 2 x i1>, double*)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1ro-addressing-mode-reg-reg.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1ro-addressing-mode-reg-reg.ll
index b4ac587c0b794..1a4a25c83b347 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1ro-addressing-mode-reg-reg.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1ro-addressing-mode-reg-reg.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+f64mm,+bf16 -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+f64mm,+bf16 -asm-verbose=0 < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LD1ROB
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-loads-ff.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-loads-ff.ll
index a27faf3a7220b..e56ebcbde8e6a 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-loads-ff.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-loads-ff.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LDFF1B
@@ -206,6 +209,14 @@ define <vscale x 8 x half> @ldff1h_f16(<vscale x 8 x i1> %pg, half* %a) {
   ret <vscale x 8 x half> %load
 }
 
+define <vscale x 8 x bfloat> @ldff1h_bf16(<vscale x 8 x i1> %pg, bfloat* %a) #0 {
+; CHECK-LABEL: ldff1h_bf16:
+; CHECK: ldff1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldff1.nxv8bf16(<vscale x 8 x i1> %pg, bfloat* %a)
+  ret <vscale x 8 x bfloat> %load
+}
+
 define <vscale x 8 x half> @ldff1h_f16_reg(<vscale x 8 x i1> %pg, half* %a, i64 %offset) {
 ; CHECK-LABEL: ldff1h_f16_reg:
 ; CHECK: ldff1h { z0.h }, p0/z, [x0, x1, lsl #1]
@@ -215,6 +226,15 @@ define <vscale x 8 x half> @ldff1h_f16_reg(<vscale x 8 x i1> %pg, half* %a, i64
   ret <vscale x 8 x half> %load
 }
 
+define <vscale x 8 x bfloat> @ldff1h_bf16_reg(<vscale x 8 x i1> %pg, bfloat* %a, i64 %offset) #0 {
+; CHECK-LABEL: ldff1h_bf16_reg:
+; CHECK: ldff1h { z0.h }, p0/z, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+  %base = getelementptr bfloat, bfloat* %a, i64 %offset
+  %load = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldff1.nxv8bf16(<vscale x 8 x i1> %pg, bfloat* %base)
+  ret <vscale x 8 x bfloat> %load
+}
+
 ;
 ; LDFF1SH
 ;
@@ -398,6 +418,7 @@ declare <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1>, i
 declare <vscale x 8 x i8> @llvm.aarch64.sve.ldff1.nxv8i8(<vscale x 8 x i1>, i8*)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.ldff1.nxv8i16(<vscale x 8 x i1>, i16*)
 declare <vscale x 8 x half> @llvm.aarch64.sve.ldff1.nxv8f16(<vscale x 8 x i1>, half*)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.ldff1.nxv8bf16(<vscale x 8 x i1>, bfloat*)
 
 declare <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.nxv4i8(<vscale x 4 x i1>, i8*)
 declare <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.nxv4i16(<vscale x 4 x i1>, i16*)
@@ -410,3 +431,6 @@ declare <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.nxv2i16(<vscale x 2 x i1>, i1
 declare <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.nxv2i32(<vscale x 2 x i1>, i32*)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.nxv2i64(<vscale x 2 x i1>, i64*)
 declare <vscale x 2 x double> @llvm.aarch64.sve.ldff1.nxv2f64(<vscale x 2 x i1>, double*)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-loads-nf.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-loads-nf.ll
index 917d7ccab69e5..31f5c6797bbc5 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-loads-nf.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-loads-nf.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ; Range testing for the immediate in the reg+imm(mulvl) addressing
 ; mode is done only for one instruction. The rest of the instrucions
@@ -140,6 +143,14 @@ define <vscale x 8 x half> @ldnf1h_f16(<vscale x 8 x i1> %pg, half* %a) {
   ret <vscale x 8 x half> %load
 }
 
+define <vscale x 8 x bfloat> @ldnf1h_bf16(<vscale x 8 x i1> %pg, bfloat* %a) #0 {
+; CHECK-LABEL: ldnf1h_bf16:
+; CHECK: ldnf1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnf1.nxv8bf16(<vscale x 8 x i1> %pg, bfloat* %a)
+  ret <vscale x 8 x bfloat> %load
+}
+
 define <vscale x 8 x half> @ldnf1h_f16_inbound(<vscale x 8 x i1> %pg, half* %a) {
 ; CHECK-LABEL: ldnf1h_f16_inbound:
 ; CHECK: ldnf1h { z0.h }, p0/z, [x0, #1, mul vl]
@@ -151,6 +162,17 @@ define <vscale x 8 x half> @ldnf1h_f16_inbound(<vscale x 8 x i1> %pg, half* %a)
   ret <vscale x 8 x half> %load
 }
 
+define <vscale x 8 x bfloat> @ldnf1h_bf16_inbound(<vscale x 8 x i1> %pg, bfloat* %a) #0 {
+; CHECK-LABEL: ldnf1h_bf16_inbound:
+; CHECK: ldnf1h { z0.h }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast bfloat* %a to <vscale x 8 x bfloat>*
+  %base = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %base_scalable, i64 1
+  %base_scalar = bitcast <vscale x 8 x bfloat>* %base to bfloat*
+  %load = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnf1.nxv8bf16(<vscale x 8 x i1> %pg, bfloat* %base_scalar)
+  ret <vscale x 8 x bfloat> %load
+}
+
 define <vscale x 4 x i32> @ldnf1b_s(<vscale x 4 x i1> %pg, i8* %a) {
 ; CHECK-LABEL: ldnf1b_s:
 ; CHECK: ldnf1b { z0.s }, p0/z, [x0]
@@ -442,6 +464,7 @@ declare <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1>, i
 declare <vscale x 8 x i8> @llvm.aarch64.sve.ldnf1.nxv8i8(<vscale x 8 x i1>, i8*)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.ldnf1.nxv8i16(<vscale x 8 x i1>, i16*)
 declare <vscale x 8 x half> @llvm.aarch64.sve.ldnf1.nxv8f16(<vscale x 8 x i1>, half*)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnf1.nxv8bf16(<vscale x 8 x i1>, bfloat*)
 
 declare <vscale x 4 x i8> @llvm.aarch64.sve.ldnf1.nxv4i8(<vscale x 4 x i1>, i8*)
 declare <vscale x 4 x i16> @llvm.aarch64.sve.ldnf1.nxv4i16(<vscale x 4 x i1>, i16*)
@@ -453,3 +476,6 @@ declare <vscale x 2 x i16> @llvm.aarch64.sve.ldnf1.nxv2i16(<vscale x 2 x i1>, i1
 declare <vscale x 2 x i32> @llvm.aarch64.sve.ldnf1.nxv2i32(<vscale x 2 x i1>, i32*)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.ldnf1.nxv2i64(<vscale x 2 x i1>, i64*)
 declare <vscale x 2 x double> @llvm.aarch64.sve.ldnf1.nxv2f64(<vscale x 2 x i1>, double*)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-loads.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-loads.ll
index 838e93d346576..5ff497c20ef5c 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-loads.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-loads.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+bf16 -asm-verbose=0 < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;
 ; LD1RQB
@@ -97,6 +100,23 @@ define <vscale x 8 x half> @ld1rqh_f16_imm(<vscale x 8 x i1> %pred, half* %addr)
   ret <vscale x 8 x half> %res
 }
 
+define <vscale x 8 x bfloat> @ld1rqh_bf16(<vscale x 8 x i1> %pred, bfloat* %addr) {
+; CHECK-LABEL: ld1rqh_bf16:
+; CHECK: ld1rqh { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1rq.nxv8bf16(<vscale x 8 x i1> %pred, bfloat* %addr)
+  ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 8 x bfloat> @ld1rqh_bf16_imm(<vscale x 8 x i1> %pred, bfloat* %addr) {
+; CHECK-LABEL: ld1rqh_bf16_imm:
+; CHECK: ld1rqh { z0.h }, p0/z, [x0, #-16]
+; CHECK-NEXT: ret
+  %ptr = getelementptr inbounds bfloat, bfloat* %addr, i16 -8
+  %res = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1rq.nxv8bf16(<vscale x 8 x i1> %pred, bfloat* %ptr)
+  ret <vscale x 8 x bfloat> %res
+}
+
 ;
 ; LD1RQW
 ;
@@ -208,6 +228,15 @@ define <vscale x 8 x half> @ldnt1h_f16(<vscale x 8 x i1> %pred, half* %addr) {
   ret <vscale x 8 x half> %res
 }
 
+define <vscale x 8 x bfloat> @ldnt1h_bf16(<vscale x 8 x i1> %pred, bfloat* %addr) {
+; CHECK-LABEL: ldnt1h_bf16:
+; CHECK: ldnt1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnt1.nxv8bf16(<vscale x 8 x i1> %pred,
+                                                                     bfloat* %addr)
+  ret <vscale x 8 x bfloat> %res
+}
+
 ;
 ; LDNT1W
 ;
@@ -284,6 +313,14 @@ define <vscale x 16 x half> @ld2h_f16(<vscale x 8 x i1> %pred, half* %addr) {
   ret <vscale x 16 x half> %res
 }
 
+define <vscale x 16 x bfloat> @ld2h_bf16(<vscale x 8 x i1> %pred, bfloat* %addr) {
+; CHECK-LABEL: ld2h_bf16:
+; CHECK: ld2h { z0.h, z1.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x bfloat> @llvm.aarch64.sve.ld2.nxv16bf16.nxv8i1.p0bf16(<vscale x 8 x i1> %pred, bfloat* %addr)
+  ret <vscale x 16 x bfloat> %res
+}
+
 ;
 ; LD2W
 ;
@@ -356,6 +393,14 @@ define <vscale x 24 x half> @ld3h_f16(<vscale x 8 x i1> %pred, half* %addr) {
   ret <vscale x 24 x half> %res
 }
 
+define <vscale x 24 x bfloat> @ld3h_bf16(<vscale x 8 x i1> %pred, bfloat* %addr) {
+; CHECK-LABEL: ld3h_bf16:
+; CHECK: ld3h { z0.h, z1.h, z2.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 24 x bfloat> @llvm.aarch64.sve.ld3.nxv24bf16.nxv8i1.p0bf16(<vscale x 8 x i1> %pred, bfloat* %addr)
+  ret <vscale x 24 x bfloat> %res
+}
+
 ;
 ; LD3W
 ;
@@ -428,6 +473,14 @@ define <vscale x 32 x half> @ld4h_f16(<vscale x 8 x i1> %pred, half* %addr) {
   ret <vscale x 32 x half> %res
 }
 
+define <vscale x 32 x bfloat> @ld4h_bf16(<vscale x 8 x i1> %pred, bfloat* %addr) {
+; CHECK-LABEL: ld4h_bf16:
+; CHECK: ld4h { z0.h, z1.h, z2.h, z3.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 32 x bfloat> @llvm.aarch64.sve.ld4.nxv32bf16.nxv8i1.p0bf16(<vscale x 8 x i1> %pred, bfloat* %addr)
+  ret <vscale x 32 x bfloat> %res
+}
+
 ;
 ; LD4W
 ;
@@ -474,6 +527,7 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.ld1rq.nxv8i16(<vscale x 8 x i1>, i1
 declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1rq.nxv4i32(<vscale x 4 x i1>, i32*)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1rq.nxv2i64(<vscale x 2 x i1>, i64*)
 declare <vscale x 8 x half> @llvm.aarch64.sve.ld1rq.nxv8f16(<vscale x 8 x i1>, half*)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1rq.nxv8bf16(<vscale x 8 x i1>, bfloat*)
 declare <vscale x 4 x float> @llvm.aarch64.sve.ld1rq.nxv4f32(<vscale x 4 x i1>, float*)
 declare <vscale x 2 x double> @llvm.aarch64.sve.ld1rq.nxv2f64(<vscale x 2 x i1>, double*)
 
@@ -482,6 +536,7 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1>, i1
 declare <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1>, i32*)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1>, i64*)
 declare <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1>, half*)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnt1.nxv8bf16(<vscale x 8 x i1>, bfloat*)
 declare <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1>, float*)
 declare <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1>, double*)
 
@@ -490,6 +545,7 @@ declare <vscale x 16 x i16> @llvm.aarch64.sve.ld2.nxv16i16.nxv8i1.p0i16(<vscale
 declare <vscale x 8 x i32> @llvm.aarch64.sve.ld2.nxv8i32.nxv4i1.p0i32(<vscale x 4 x i1>, i32*)
 declare <vscale x 4 x i64> @llvm.aarch64.sve.ld2.nxv4i64.nxv2i1.p0i64(<vscale x 2 x i1>, i64*)
 declare <vscale x 16 x half> @llvm.aarch64.sve.ld2.nxv16f16.nxv8i1.p0f16(<vscale x 8 x i1>, half*)
+declare <vscale x 16 x bfloat> @llvm.aarch64.sve.ld2.nxv16bf16.nxv8i1.p0bf16(<vscale x 8 x i1>, bfloat*)
 declare <vscale x 8 x float> @llvm.aarch64.sve.ld2.nxv8f32.nxv4i1.p0f32(<vscale x 4 x i1>, float*)
 declare <vscale x 4 x double> @llvm.aarch64.sve.ld2.nxv4f64.nxv2i1.p0f64(<vscale x 2 x i1>, double*)
 
@@ -498,6 +554,7 @@ declare <vscale x 24 x i16> @llvm.aarch64.sve.ld3.nxv24i16.nxv8i1.p0i16(<vscale
 declare <vscale x 12 x i32> @llvm.aarch64.sve.ld3.nxv12i32.nxv4i1.p0i32(<vscale x 4 x i1>, i32*)
 declare <vscale x 6 x i64> @llvm.aarch64.sve.ld3.nxv6i64.nxv2i1.p0i64(<vscale x 2 x i1>, i64*)
 declare <vscale x 24 x half> @llvm.aarch64.sve.ld3.nxv24f16.nxv8i1.p0f16(<vscale x 8 x i1>, half*)
+declare <vscale x 24 x bfloat> @llvm.aarch64.sve.ld3.nxv24bf16.nxv8i1.p0bf16(<vscale x 8 x i1>, bfloat*)
 declare <vscale x 12 x float> @llvm.aarch64.sve.ld3.nxv12f32.nxv4i1.p0f32(<vscale x 4 x i1>, float*)
 declare <vscale x 6 x double> @llvm.aarch64.sve.ld3.nxv6f64.nxv2i1.p0f64(<vscale x 2 x i1>, double*)
 
@@ -506,5 +563,6 @@ declare <vscale x 32 x i16> @llvm.aarch64.sve.ld4.nxv32i16.nxv8i1.p0i16(<vscale
 declare <vscale x 16 x i32> @llvm.aarch64.sve.ld4.nxv16i32.nxv4i1.p0i32(<vscale x 4 x i1>, i32*)
 declare <vscale x 8 x i64> @llvm.aarch64.sve.ld4.nxv8i64.nxv2i1.p0i64(<vscale x 2 x i1>, i64*)
 declare <vscale x 32 x half> @llvm.aarch64.sve.ld4.nxv32f16.nxv8i1.p0f16(<vscale x 8 x i1>, half*)
+declare <vscale x 32 x bfloat> @llvm.aarch64.sve.ld4.nxv32bf16.nxv8i1.p0bf16(<vscale x 8 x i1>, bfloat*)
 declare <vscale x 16 x float> @llvm.aarch64.sve.ld4.nxv16f32.nxv4i1.p0f32(<vscale x 4 x i1>, float*)
 declare <vscale x 8 x double> @llvm.aarch64.sve.ld4.nxv8f64.nxv2i1.p0f64(<vscale x 2 x i1>, double*)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select-matmul-fp64.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select-matmul-fp64.ll
index 51a7b6810a28b..f4a46c20a8777 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select-matmul-fp64.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select-matmul-fp64.ll
@@ -49,7 +49,7 @@ define <vscale x 8 x half> @trn1_f16(<vscale x 8 x half> %a, <vscale x 8 x half>
   ret <vscale x 8 x half> %out
 }
 
-define <vscale x 8 x bfloat> @trn1_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) nounwind {
+define <vscale x 8 x bfloat> @trn1_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) nounwind #0 {
 ; CHECK-LABEL: trn1_bf16:
 ; CHECK-NEXT:  trn1 z0.q, z0.q, z1.q
 ; CHECK-NEXT:  ret
@@ -125,7 +125,7 @@ define <vscale x 8 x half> @trn2_f16(<vscale x 8 x half> %a, <vscale x 8 x half>
   ret <vscale x 8 x half> %out
 }
 
-define <vscale x 8 x bfloat> @trn2_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) nounwind {
+define <vscale x 8 x bfloat> @trn2_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) nounwind #0 {
 ; CHECK-LABEL: trn2_bf16:
 ; CHECK-NEXT:  trn2 z0.q, z0.q, z1.q
 ; CHECK-NEXT:  ret
@@ -201,7 +201,7 @@ define <vscale x 8 x half> @uzp1_f16(<vscale x 8 x half> %a, <vscale x 8 x half>
   ret <vscale x 8 x half> %out
 }
 
-define <vscale x 8 x bfloat> @uzp1_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) nounwind {
+define <vscale x 8 x bfloat> @uzp1_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) nounwind #0 {
 ; CHECK-LABEL: uzp1_bf16:
 ; CHECK-NEXT:  uzp1 z0.q, z0.q, z1.q
 ; CHECK-NEXT:  ret
@@ -277,7 +277,7 @@ define <vscale x 8 x half> @uzp2_f16(<vscale x 8 x half> %a, <vscale x 8 x half>
   ret <vscale x 8 x half> %out
 }
 
-define <vscale x 8 x bfloat> @uzp2_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) nounwind {
+define <vscale x 8 x bfloat> @uzp2_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) nounwind #0 {
 ; CHECK-LABEL: uzp2_bf16:
 ; CHECK-NEXT:  uzp2 z0.q, z0.q, z1.q
 ; CHECK-NEXT:  ret
@@ -353,7 +353,7 @@ define <vscale x 8 x half> @zip1_f16(<vscale x 8 x half> %a, <vscale x 8 x half>
   ret <vscale x 8 x half> %out
 }
 
-define <vscale x 8 x bfloat> @zip1_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) nounwind {
+define <vscale x 8 x bfloat> @zip1_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) nounwind #0 {
 ; CHECK-LABEL: zip1_bf16:
 ; CHECK-NEXT:  zip1 z0.q, z0.q, z1.q
 ; CHECK-NEXT:  ret
@@ -429,7 +429,7 @@ define <vscale x 8 x half> @zip2_f16(<vscale x 8 x half> %a, <vscale x 8 x half>
   ret <vscale x 8 x half> %out
 }
 
-define <vscale x 8 x bfloat> @zip2_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) nounwind {
+define <vscale x 8 x bfloat> @zip2_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) nounwind #0 {
 ; CHECK-LABEL: zip2_bf16:
 ; CHECK-NEXT:  zip2 z0.q, z0.q, z1.q
 ; CHECK-NEXT:  ret
@@ -510,3 +510,6 @@ declare <vscale x 8 x bfloat> @llvm.aarch64.sve.zip2q.nxv8bf16(<vscale x 8 x bfl
 declare <vscale x 8 x half> @llvm.aarch64.sve.zip2q.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.zip2q.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
 declare <vscale x 16 x i8> @llvm.aarch64.sve.zip2q.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+fp64mm,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll
index 22f87e787417a..433a1cdfd8e91 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll
@@ -57,6 +57,16 @@ define <vscale x 8 x half> @clasta_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half
   ret <vscale x 8 x half> %out
 }
 
+define <vscale x 8 x bfloat> @clasta_bf16(<vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: clasta_bf16:
+; CHECK: clasta z0.h, p0, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.clasta.nxv8bf16(<vscale x 8 x i1> %pg,
+                                                                      <vscale x 8 x bfloat> %a,
+                                                                      <vscale x 8 x bfloat> %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 4 x float> @clasta_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: clasta_f32:
 ; CHECK: clasta z0.s, p0, z0.s, z1.s
@@ -131,6 +141,16 @@ define half @clasta_n_f16(<vscale x 8 x i1> %pg, half %a, <vscale x 8 x half> %b
   ret half %out
 }
 
+define bfloat @clasta_n_bf16(<vscale x 8 x i1> %pg, bfloat %a, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: clasta_n_bf16:
+; CHECK: clasta h0, p0, h0, z1.h
+; CHECK-NEXT: ret
+  %out = call bfloat @llvm.aarch64.sve.clasta.n.nxv8bf16(<vscale x 8 x i1> %pg,
+                                                         bfloat %a,
+                                                         <vscale x 8 x bfloat> %b)
+  ret bfloat %out
+}
+
 define float @clasta_n_f32(<vscale x 4 x i1> %pg, float %a, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: clasta_n_f32:
 ; CHECK: clasta s0, p0, s0, z1.s
@@ -205,6 +225,16 @@ define <vscale x 8 x half> @clastb_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half
   ret <vscale x 8 x half> %out
 }
 
+define <vscale x 8 x bfloat> @clastb_bf16(<vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: clastb_bf16:
+; CHECK: clastb z0.h, p0, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.clastb.nxv8bf16(<vscale x 8 x i1> %pg,
+                                                                      <vscale x 8 x bfloat> %a,
+                                                                      <vscale x 8 x bfloat> %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 4 x float> @clastb_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: clastb_f32:
 ; CHECK: clastb z0.s, p0, z0.s, z1.s
@@ -279,6 +309,16 @@ define half @clastb_n_f16(<vscale x 8 x i1> %pg, half %a, <vscale x 8 x half> %b
   ret half %out
 }
 
+define bfloat @clastb_n_bf16(<vscale x 8 x i1> %pg, bfloat %a, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: clastb_n_bf16:
+; CHECK: clastb h0, p0, h0, z1.h
+; CHECK-NEXT: ret
+  %out = call bfloat @llvm.aarch64.sve.clastb.n.nxv8bf16(<vscale x 8 x i1> %pg,
+                                                         bfloat %a,
+                                                         <vscale x 8 x bfloat> %b)
+  ret bfloat %out
+}
+
 define float @clastb_n_f32(<vscale x 4 x i1> %pg, float %a, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: clastb_n_f32:
 ; CHECK: clastb s0, p0, s0, z1.s
@@ -343,6 +383,14 @@ define <vscale x 8 x half> @dupq_f16(<vscale x 8 x half> %a) {
   ret <vscale x 8 x half> %out
 }
 
+define <vscale x 8 x bfloat> @dupq_bf16(<vscale x 8 x bfloat> %a) #0 {
+; CHECK-LABEL: dupq_bf16:
+; CHECK: mov z0.q, q0
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.dupq.lane.nxv8bf16(<vscale x 8 x bfloat> %a, i64 0)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 4 x float> @dupq_f32(<vscale x 4 x float> %a) {
 ; CHECK-LABEL: dupq_f32:
 ; CHECK: mov z0.q, z0.q[1]
@@ -432,6 +480,20 @@ define <vscale x 8 x half> @dupq_lane_f16(<vscale x 8 x half> %a, i64 %idx) {
   ret <vscale x 8 x half> %out
 }
 
+; NOTE: Identical operation to dupq_lane_i8 (i.e. element type is irrelevant).
+define <vscale x 8 x bfloat> @dupq_lane_bf16(<vscale x 8 x bfloat> %a, i64 %idx) #0 {
+; CHECK-LABEL: dupq_lane_bf16:
+; CHECK-DAG:  index [[Z1:z[0-9]+]].d, #0, #1
+; CHECK-DAG:  and   [[Z2:z[0-9]+]].d, [[Z1]].d, #0x1
+; CHECK-DAG:  add   [[X1:x[0-9]+]], x0, x0
+; CHECK-DAG:  mov   [[Z3:z[0-9]+]].d, [[X1]]
+; CHECK:      add   [[Z4:z[0-9]+]].d, [[Z2]].d, [[Z3]].d
+; CHECK: tbl z0.d, { z0.d }, [[Z4]].d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.dupq.lane.nxv8bf16(<vscale x 8 x bfloat> %a, i64 %idx)
+  ret <vscale x 8 x bfloat> %out
+}
+
 ; NOTE: Identical operation to dupq_lane_i8 (i.e. element type is irrelevant).
 define <vscale x 4 x float> @dupq_lane_f32(<vscale x 4 x float> %a, i64 %idx) {
 ; CHECK-LABEL: dupq_lane_f32:
@@ -516,6 +578,16 @@ define <vscale x 2 x i64> @ext_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b)
   ret <vscale x 2 x i64> %out
 }
 
+define <vscale x 8 x bfloat> @ext_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: ext_bf16:
+; CHECK: ext z0.b, z0.b, z1.b, #6
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ext.nxv8bf16(<vscale x 8 x bfloat> %a,
+                                                                   <vscale x 8 x bfloat> %b,
+                                                                   i32 3)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 8 x half> @ext_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
 ; CHECK-LABEL: ext_f16:
 ; CHECK: ext z0.b, z0.b, z1.b, #6
@@ -595,6 +667,15 @@ define half @lasta_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
   ret half %res
 }
 
+define bfloat @lasta_bf16(<vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %a) #0 {
+; CHECK-LABEL: lasta_bf16
+; CHECK: lasta h0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call bfloat @llvm.aarch64.sve.lasta.nxv8bf16(<vscale x 8 x i1> %pg,
+                                                      <vscale x 8 x bfloat> %a)
+  ret bfloat %res
+}
+
 define float @lasta_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
 ; CHECK-LABEL: lasta_f32
 ; CHECK: lasta s0, p0, z0.s
@@ -671,6 +752,15 @@ define half @lastb_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
   ret half %res
 }
 
+define bfloat @lastb_bf16(<vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %a) #0 {
+; CHECK-LABEL: lastb_bf16
+; CHECK: lastb h0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call bfloat @llvm.aarch64.sve.lastb.nxv8bf16(<vscale x 8 x i1> %pg,
+                                                      <vscale x 8 x bfloat> %a)
+  ret bfloat %res
+}
+
 define float @lastb_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
 ; CHECK-LABEL: lastb_f32
 ; CHECK: lastb s0, p0, z0.s
@@ -806,6 +896,14 @@ define <vscale x 2 x i64> @rev_i64(<vscale x 2 x i64> %a) {
   ret <vscale x 2 x i64> %res
 }
 
+define <vscale x 8 x bfloat> @rev_bf16(<vscale x 8 x bfloat> %a) #0 {
+; CHECK-LABEL: rev_bf16
+; CHECK: rev z0.h, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x bfloat> @llvm.aarch64.sve.rev.nxv8bf16(<vscale x 8 x bfloat> %a)
+  ret <vscale x 8 x bfloat> %res
+}
+
 define <vscale x 8 x half> @rev_f16(<vscale x 8 x half> %a) {
 ; CHECK-LABEL: rev_f16
 ; CHECK: rev z0.h, z0.h
@@ -874,6 +972,16 @@ define <vscale x 2 x i64> @splice_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
   ret <vscale x 2 x i64> %out
 }
 
+define <vscale x 8 x bfloat> @splice_bf16(<vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: splice_bf16:
+; CHECK: splice z0.h, p0, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.splice.nxv8bf16(<vscale x 8 x i1> %pg,
+                                                                      <vscale x 8 x bfloat> %a,
+                                                                      <vscale x 8 x bfloat> %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 8 x half> @splice_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
 ; CHECK-LABEL: splice_f16:
 ; CHECK: splice z0.h, p0, z0.h, z1.h
@@ -1009,6 +1117,15 @@ define <vscale x 8 x half> @tbl_f16(<vscale x 8 x half> %a, <vscale x 8 x i16> %
   ret <vscale x 8 x half> %out
 }
 
+define <vscale x 8 x bfloat> @tbl_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x i16> %b) #0 {
+; CHECK-LABEL: tbl_bf16:
+; CHECK: tbl z0.h, { z0.h }, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tbl.nxv8bf16(<vscale x 8 x bfloat> %a,
+                                                                   <vscale x 8 x i16> %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 4 x float> @tbl_f32(<vscale x 4 x float> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: tbl_f32:
 ; CHECK: tbl z0.s, { z0.s }, z1.s
@@ -1168,6 +1285,15 @@ define <vscale x 4 x half> @trn1_f16_v4(<vscale x 4 x half> %a, <vscale x 4 x ha
   ret <vscale x 4 x half> %out
 }
 
+define <vscale x 8 x bfloat> @trn1_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: trn1_bf16:
+; CHECK: trn1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.trn1.nxv8bf16(<vscale x 8 x bfloat> %a,
+                                                                    <vscale x 8 x bfloat> %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 8 x half> @trn1_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
 ; CHECK-LABEL: trn1_f16:
 ; CHECK: trn1 z0.h, z0.h, z1.h
@@ -1280,6 +1406,15 @@ define <vscale x 4 x half> @trn2_f16_v4(<vscale x 4 x half> %a, <vscale x 4 x ha
   ret <vscale x 4 x half> %out
 }
 
+define <vscale x 8 x bfloat> @trn2_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: trn2_bf16:
+; CHECK: trn2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.trn2.nxv8bf16(<vscale x 8 x bfloat> %a,
+                                                                    <vscale x 8 x bfloat> %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 8 x half> @trn2_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
 ; CHECK-LABEL: trn2_f16:
 ; CHECK: trn2 z0.h, z0.h, z1.h
@@ -1392,6 +1527,15 @@ define <vscale x 4 x half> @uzp1_f16_v4(<vscale x 4 x half> %a, <vscale x 4 x ha
   ret <vscale x 4 x half> %out
 }
 
+define <vscale x 8 x bfloat> @uzp1_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: uzp1_bf16:
+; CHECK: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.uzp1.nxv8bf16(<vscale x 8 x bfloat> %a,
+                                                                    <vscale x 8 x bfloat> %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 8 x half> @uzp1_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
 ; CHECK-LABEL: uzp1_f16:
 ; CHECK: uzp1 z0.h, z0.h, z1.h
@@ -1504,6 +1648,15 @@ define <vscale x 4 x half> @uzp2_f16_v4(<vscale x 4 x half> %a, <vscale x 4 x ha
   ret <vscale x 4 x half> %out
 }
 
+define <vscale x 8 x bfloat> @uzp2_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: uzp2_bf16:
+; CHECK: uzp2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.uzp2.nxv8bf16(<vscale x 8 x bfloat> %a,
+                                                                    <vscale x 8 x bfloat> %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 8 x half> @uzp2_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
 ; CHECK-LABEL: uzp2_f16:
 ; CHECK: uzp2 z0.h, z0.h, z1.h
@@ -1616,6 +1769,15 @@ define <vscale x 4 x half> @zip1_f16_v4(<vscale x 4 x half> %a, <vscale x 4 x ha
   ret <vscale x 4 x half> %out
 }
 
+define <vscale x 8 x bfloat> @zip1_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: zip1_bf16:
+; CHECK: zip1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.zip1.nxv8bf16(<vscale x 8 x bfloat> %a,
+                                                                    <vscale x 8 x bfloat> %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 8 x half> @zip1_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
 ; CHECK-LABEL: zip1_f16:
 ; CHECK: zip1 z0.h, z0.h, z1.h
@@ -1728,6 +1890,15 @@ define <vscale x 4 x half> @zip2_f16_v4(<vscale x 4 x half> %a, <vscale x 4 x ha
   ret <vscale x 4 x half> %out
 }
 
+define <vscale x 8 x bfloat> @zip2_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: zip2_bf16:
+; CHECK: zip2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.zip2.nxv8bf16(<vscale x 8 x bfloat> %a,
+                                                                    <vscale x 8 x bfloat> %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 8 x half> @zip2_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
 ; CHECK-LABEL: zip2_f16:
 ; CHECK: zip2 z0.h, z0.h, z1.h
@@ -1760,6 +1931,7 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.clasta.nxv8i16(<vscale x 8 x i1>, <
 declare <vscale x 4 x i32> @llvm.aarch64.sve.clasta.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.clasta.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
 declare <vscale x 8 x half> @llvm.aarch64.sve.clasta.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.clasta.nxv8bf16(<vscale x 8 x i1>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.clasta.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.clasta.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
 
@@ -1768,6 +1940,7 @@ declare i16 @llvm.aarch64.sve.clasta.n.nxv8i16(<vscale x 8 x i1>, i16, <vscale x
 declare i32 @llvm.aarch64.sve.clasta.n.nxv4i32(<vscale x 4 x i1>, i32, <vscale x 4 x i32>)
 declare i64 @llvm.aarch64.sve.clasta.n.nxv2i64(<vscale x 2 x i1>, i64, <vscale x 2 x i64>)
 declare half @llvm.aarch64.sve.clasta.n.nxv8f16(<vscale x 8 x i1>, half, <vscale x 8 x half>)
+declare bfloat @llvm.aarch64.sve.clasta.n.nxv8bf16(<vscale x 8 x i1>, bfloat, <vscale x 8 x bfloat>)
 declare float @llvm.aarch64.sve.clasta.n.nxv4f32(<vscale x 4 x i1>, float, <vscale x 4 x float>)
 declare double @llvm.aarch64.sve.clasta.n.nxv2f64(<vscale x 2 x i1>, double, <vscale x 2 x double>)
 
@@ -1776,6 +1949,7 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.clastb.nxv8i16(<vscale x 8 x i1>, <
 declare <vscale x 4 x i32> @llvm.aarch64.sve.clastb.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.clastb.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
 declare <vscale x 8 x half> @llvm.aarch64.sve.clastb.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.clastb.nxv8bf16(<vscale x 8 x i1>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.clastb.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.clastb.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
 
@@ -1784,6 +1958,7 @@ declare i16 @llvm.aarch64.sve.clastb.n.nxv8i16(<vscale x 8 x i1>, i16, <vscale x
 declare i32 @llvm.aarch64.sve.clastb.n.nxv4i32(<vscale x 4 x i1>, i32, <vscale x 4 x i32>)
 declare i64 @llvm.aarch64.sve.clastb.n.nxv2i64(<vscale x 2 x i1>, i64, <vscale x 2 x i64>)
 declare half @llvm.aarch64.sve.clastb.n.nxv8f16(<vscale x 8 x i1>, half, <vscale x 8 x half>)
+declare bfloat @llvm.aarch64.sve.clastb.n.nxv8bf16(<vscale x 8 x i1>, bfloat, <vscale x 8 x bfloat>)
 declare float @llvm.aarch64.sve.clastb.n.nxv4f32(<vscale x 4 x i1>, float, <vscale x 4 x float>)
 declare double @llvm.aarch64.sve.clastb.n.nxv2f64(<vscale x 2 x i1>, double, <vscale x 2 x double>)
 
@@ -1797,6 +1972,7 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.dupq.lane.nxv8i16(<vscale x 8 x i16
 declare <vscale x 4 x i32> @llvm.aarch64.sve.dupq.lane.nxv4i32(<vscale x 4 x i32>, i64)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.dupq.lane.nxv2i64(<vscale x 2 x i64>, i64)
 declare <vscale x 8 x half> @llvm.aarch64.sve.dupq.lane.nxv8f16(<vscale x 8 x half>, i64)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.dupq.lane.nxv8bf16(<vscale x 8 x bfloat>, i64)
 declare <vscale x 4 x float> @llvm.aarch64.sve.dupq.lane.nxv4f32(<vscale x 4 x float>, i64)
 declare <vscale x 2 x double> @llvm.aarch64.sve.dupq.lane.nxv2f64(<vscale x 2 x double>, i64)
 
@@ -1804,6 +1980,7 @@ declare <vscale x 16 x i8> @llvm.aarch64.sve.ext.nxv16i8(<vscale x 16 x i8>, <vs
 declare <vscale x 8 x i16> @llvm.aarch64.sve.ext.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, i32)
 declare <vscale x 4 x i32> @llvm.aarch64.sve.ext.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, i32)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.ext.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, i32)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.ext.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i32)
 declare <vscale x 8 x half> @llvm.aarch64.sve.ext.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, i32)
 declare <vscale x 4 x float> @llvm.aarch64.sve.ext.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, i32)
 declare <vscale x 2 x double> @llvm.aarch64.sve.ext.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, i32)
@@ -1813,6 +1990,7 @@ declare i16 @llvm.aarch64.sve.lasta.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16
 declare i32 @llvm.aarch64.sve.lasta.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>)
 declare i64 @llvm.aarch64.sve.lasta.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>)
 declare half @llvm.aarch64.sve.lasta.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>)
+declare bfloat @llvm.aarch64.sve.lasta.nxv8bf16(<vscale x 8 x i1>, <vscale x 8 x bfloat>)
 declare float @llvm.aarch64.sve.lasta.nxv2f32(<vscale x 2 x i1>, <vscale x 2 x float>)
 declare float @llvm.aarch64.sve.lasta.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>)
 declare double @llvm.aarch64.sve.lasta.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>)
@@ -1822,6 +2000,7 @@ declare i16 @llvm.aarch64.sve.lastb.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16
 declare i32 @llvm.aarch64.sve.lastb.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>)
 declare i64 @llvm.aarch64.sve.lastb.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>)
 declare half @llvm.aarch64.sve.lastb.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>)
+declare bfloat @llvm.aarch64.sve.lastb.nxv8bf16(<vscale x 8 x i1>, <vscale x 8 x bfloat>)
 declare float @llvm.aarch64.sve.lastb.nxv2f32(<vscale x 2 x i1>, <vscale x 2 x float>)
 declare float @llvm.aarch64.sve.lastb.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>)
 declare double @llvm.aarch64.sve.lastb.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>)
@@ -1834,6 +2013,7 @@ declare <vscale x 16 x i8> @llvm.aarch64.sve.rev.nxv16i8(<vscale x 16 x i8>)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.rev.nxv8i16(<vscale x 8 x i16>)
 declare <vscale x 4 x i32> @llvm.aarch64.sve.rev.nxv4i32(<vscale x 4 x i32>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.rev.nxv2i64(<vscale x 2 x i64>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.rev.nxv8bf16(<vscale x 8 x bfloat>)
 declare <vscale x 8 x half> @llvm.aarch64.sve.rev.nxv8f16(<vscale x 8 x half>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.rev.nxv4f32(<vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.rev.nxv2f64(<vscale x 2 x double>)
@@ -1842,6 +2022,7 @@ declare <vscale x 16 x i8> @llvm.aarch64.sve.splice.nxv16i8(<vscale x 16 x i1>,
 declare <vscale x 8 x i16> @llvm.aarch64.sve.splice.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
 declare <vscale x 4 x i32> @llvm.aarch64.sve.splice.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.splice.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.splice.nxv8bf16(<vscale x 8 x i1>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
 declare <vscale x 8 x half> @llvm.aarch64.sve.splice.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.splice.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.splice.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
@@ -1859,6 +2040,7 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.tbl.nxv8i16(<vscale x 8 x i16>, <vs
 declare <vscale x 4 x i32> @llvm.aarch64.sve.tbl.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.tbl.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
 declare <vscale x 8 x half> @llvm.aarch64.sve.tbl.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i16>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.tbl.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x i16>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.tbl.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i32>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.tbl.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i64>)
 
@@ -1879,6 +2061,7 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.trn1.nxv8i16(<vscale x 8 x i16>, <v
 declare <vscale x 4 x i32> @llvm.aarch64.sve.trn1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.trn1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
 declare <vscale x 4 x half> @llvm.aarch64.sve.trn1.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.trn1.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
 declare <vscale x 8 x half> @llvm.aarch64.sve.trn1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.trn1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.trn1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
@@ -1892,6 +2075,7 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.trn2.nxv8i16(<vscale x 8 x i16>, <v
 declare <vscale x 4 x i32> @llvm.aarch64.sve.trn2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.trn2.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
 declare <vscale x 4 x half> @llvm.aarch64.sve.trn2.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.trn2.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
 declare <vscale x 8 x half> @llvm.aarch64.sve.trn2.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.trn2.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.trn2.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
@@ -1905,6 +2089,7 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.uzp1.nxv8i16(<vscale x 8 x i16>, <v
 declare <vscale x 4 x i32> @llvm.aarch64.sve.uzp1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.uzp1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
 declare <vscale x 4 x half> @llvm.aarch64.sve.uzp1.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.uzp1.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
 declare <vscale x 8 x half> @llvm.aarch64.sve.uzp1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.uzp1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.uzp1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
@@ -1918,6 +2103,7 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.uzp2.nxv8i16(<vscale x 8 x i16>, <v
 declare <vscale x 4 x i32> @llvm.aarch64.sve.uzp2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.uzp2.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
 declare <vscale x 4 x half> @llvm.aarch64.sve.uzp2.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.uzp2.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
 declare <vscale x 8 x half> @llvm.aarch64.sve.uzp2.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.uzp2.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.uzp2.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
@@ -1931,6 +2117,7 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.zip1.nxv8i16(<vscale x 8 x i16>, <v
 declare <vscale x 4 x i32> @llvm.aarch64.sve.zip1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.zip1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
 declare <vscale x 4 x half> @llvm.aarch64.sve.zip1.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.zip1.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
 declare <vscale x 8 x half> @llvm.aarch64.sve.zip1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.zip1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.zip1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
@@ -1944,6 +2131,10 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.zip2.nxv8i16(<vscale x 8 x i16>, <v
 declare <vscale x 4 x i32> @llvm.aarch64.sve.zip2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.zip2.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
 declare <vscale x 4 x half> @llvm.aarch64.sve.zip2.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.zip2.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
 declare <vscale x 8 x half> @llvm.aarch64.sve.zip2.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.zip2.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.zip2.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-scalar-to-vec.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-scalar-to-vec.ll
index a9ff1648f3b1f..9f679aa6dc4f8 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-scalar-to-vec.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-scalar-to-vec.ll
@@ -57,6 +57,16 @@ define <vscale x 8 x half> @dup_f16(<vscale x 8 x half> %a, <vscale x 8 x i1> %p
   ret <vscale x 8 x half> %out
 }
 
+define <vscale x 8 x bfloat> @dup_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x i1> %pg, bfloat %b) #0 {
+; CHECK-LABEL: dup_bf16:
+; CHECK: mov z0.h, p0/m, h1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.dup.nxv8bf16(<vscale x 8 x bfloat> %a,
+                                                                   <vscale x 8 x i1> %pg,
+                                                                   bfloat %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 4 x float> @dup_f32(<vscale x 4 x float> %a, <vscale x 4 x i1> %pg, float %b) {
 ; CHECK-LABEL: dup_f32:
 ; CHECK: mov z0.s, p0/m, s1
@@ -77,10 +87,41 @@ define <vscale x 2 x double> @dup_f64(<vscale x 2 x double> %a, <vscale x 2 x i1
   ret <vscale x 2 x double> %out
 }
 
+define <vscale x 8 x bfloat> @test_svdup_n_bf16_z(<vscale x 8 x i1> %pg, bfloat %op) #0 {
+; CHECK-LABEL: test_svdup_n_bf16_z:
+; CHECK: mov z1.h, #0
+; CHECK: mov z1.h, p0/m, h0
+; CHECK: mov z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.dup.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x i1> %pg, bfloat %op)
+  ret <vscale x 8 x bfloat> %out
+}
+
+define <vscale x 8 x bfloat> @test_svdup_n_bf16_m(<vscale x 8 x bfloat> %inactive, <vscale x 8 x i1> %pg, bfloat %op) #0 {
+; CHECK-LABEL: test_svdup_n_bf16_m:
+; CHECK: mov z0.h, p0/m, h1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.dup.nxv8bf16(<vscale x 8 x bfloat> %inactive, <vscale x 8 x i1> %pg, bfloat %op)
+  ret <vscale x 8 x bfloat> %out
+}
+
+
+define <vscale x 8 x bfloat> @test_svdup_n_bf16_x(<vscale x 8 x i1> %pg, bfloat %op) #0 {
+; CHECK-LABEL: test_svdup_n_bf16_x:
+; CHECK: mov z0.h, p0/m, h0
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.dup.nxv8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> %pg, bfloat %op)
+  ret <vscale x 8 x bfloat> %out
+}
+
 declare <vscale x 16 x i8> @llvm.aarch64.sve.dup.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i8)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.dup.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16)
 declare <vscale x 4 x i32> @llvm.aarch64.sve.dup.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64)
 declare <vscale x 8 x half> @llvm.aarch64.sve.dup.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.dup.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x i1>, bfloat)
 declare <vscale x 4 x float> @llvm.aarch64.sve.dup.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float)
 declare <vscale x 2 x double> @llvm.aarch64.sve.dup.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-sel.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-sel.ll
index 60e5216b21715..28341cb522b85 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-sel.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-sel.ll
@@ -57,6 +57,16 @@ define <vscale x 2 x i64> @sel_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a,
   ret <vscale x 2 x i64> %out
 }
 
+define <vscale x 8 x bfloat> @sel_bf16(<vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: sel_bf16:
+; CHECK: sel z0.h, p0, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.sel.nxv8bf16(<vscale x 8 x i1> %pg,
+                                                                   <vscale x 8 x bfloat> %a,
+                                                                   <vscale x 8 x bfloat> %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 8 x half> @sel_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
 ; CHECK-LABEL: sel_f16:
 ; CHECK: sel z0.h, p0, z0.h, z1.h
@@ -92,6 +102,10 @@ declare <vscale x 16 x i8> @llvm.aarch64.sve.sel.nxv16i8(<vscale x 16 x i1>, <vs
 declare <vscale x 8 x i16> @llvm.aarch64.sve.sel.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
 declare <vscale x 4 x i32> @llvm.aarch64.sve.sel.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.sel.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.sel.nxv8bf16(<vscale x 8 x i1>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
 declare <vscale x 8 x half> @llvm.aarch64.sve.sel.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.sel.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.sel.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-shifts-merging.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-shifts-merging.ll
index 63936ead22fcd..615caf18d3286 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-shifts-merging.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-shifts-merging.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -mattr=+use-experimental-zeroing-pseudos < %s 2>%t | FileCheck %s
 ; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
 
 ; WARN-NOT: warning
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-shifts.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-shifts.ll
index 0333eb79714be..dd884d5577f09 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-shifts.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-shifts.ll
@@ -165,6 +165,14 @@ define <vscale x 8 x half> @insr_f16(<vscale x 8 x half> %a, half %b) {
   ret <vscale x 8 x half> %out
 }
 
+define <vscale x 8 x bfloat> @insr_bf16(<vscale x 8 x bfloat> %a, bfloat %b) #0 {
+; CHECK-LABEL: insr_bf16:
+; CHECK: insr z0.h, h1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.insr.nxv8bf16(<vscale x 8 x bfloat> %a, bfloat %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 4 x float> @insr_f32(<vscale x 4 x float> %a, float %b) {
 ; CHECK-LABEL: insr_f32:
 ; CHECK: insr z0.s, s1
@@ -348,6 +356,7 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.insr.nxv8i16(<vscale x 8 x i16>, i1
 declare <vscale x 4 x i32> @llvm.aarch64.sve.insr.nxv4i32(<vscale x 4 x i32>, i32)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.insr.nxv2i64(<vscale x 2 x i64>, i64)
 declare <vscale x 8 x half> @llvm.aarch64.sve.insr.nxv8f16(<vscale x 8 x half>, half)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.insr.nxv8bf16(<vscale x 8 x bfloat>, bfloat)
 declare <vscale x 4 x float> @llvm.aarch64.sve.insr.nxv4f32(<vscale x 4 x float>, float)
 declare <vscale x 2 x double> @llvm.aarch64.sve.insr.nxv2f64(<vscale x 2 x double>, double)
 
@@ -368,3 +377,6 @@ declare <vscale x 2 x i64> @llvm.aarch64.sve.lsr.nxv2i64(<vscale x 2 x i1>, <vsc
 declare <vscale x 16 x i8> @llvm.aarch64.sve.lsr.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.lsr.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
 declare <vscale x 4 x i32> @llvm.aarch64.sve.lsr.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-st1-addressing-mode-reg-imm.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-st1-addressing-mode-reg-imm.ll
index e5dd40687caa4..e4e0ed8738f4f 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-st1-addressing-mode-reg-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-st1-addressing-mode-reg-imm.ll
@@ -126,6 +126,17 @@ define void @st1h_f16_inbound(<vscale x 8 x half> %data, <vscale x 8 x i1> %pg,
   ret void
 }
 
+define void @st1h_bf16_inbound(<vscale x 8 x bfloat> %data, <vscale x 8 x i1> %pg, bfloat* %a) #0 {
+; CHECK-LABEL: st1h_bf16_inbound:
+; CHECK: st1h { z0.h }, p0, [x0, #-5, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast bfloat* %a to <vscale x 8 x bfloat>*
+  %base = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %base_scalable, i64 -5
+  %base_scalar = bitcast <vscale x 8 x bfloat>* %base to bfloat*
+  call void @llvm.aarch64.sve.st1.nxv8bf16(<vscale x 8 x bfloat> %data, <vscale x 8 x i1> %pg, bfloat* %base_scalar)
+  ret void
+}
+
 define void @st1h_s_inbound(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i16* %a) {
 ; CHECK-LABEL: st1h_s_inbound:
 ; CHECK: st1h { z0.s }, p0, [x0, #2, mul vl]
@@ -219,6 +230,7 @@ declare void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1
 declare void @llvm.aarch64.sve.st1.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i1>, i8*)
 declare void @llvm.aarch64.sve.st1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16*)
 declare void @llvm.aarch64.sve.st1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half*)
+declare void @llvm.aarch64.sve.st1.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x i1>, bfloat*)
 
 declare void @llvm.aarch64.sve.st1.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i1>, i8*)
 declare void @llvm.aarch64.sve.st1.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i1>, i16*)
@@ -230,3 +242,6 @@ declare void @llvm.aarch64.sve.st1.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i1>
 declare void @llvm.aarch64.sve.st1.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i1>, i32*)
 declare void @llvm.aarch64.sve.st1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*)
 declare void @llvm.aarch64.sve.st1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-st1-addressing-mode-reg-reg.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-st1-addressing-mode-reg-reg.ll
index da075c6c32fd4..1af4ce746c8a3 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-st1-addressing-mode-reg-reg.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-st1-addressing-mode-reg-reg.ll
@@ -82,6 +82,17 @@ define void @st1h_f16(<vscale x 8 x half> %data, <vscale x 8 x i1> %pred, half*
   ret void
 }
 
+define void @st1h_bf16(<vscale x 8 x bfloat> %data, <vscale x 8 x i1> %pred, bfloat* %a, i64 %index) #0 {
+; CHECK-LABEL: st1h_bf16:
+; CHECK: st1h { z0.h }, p0, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+  %base = getelementptr bfloat, bfloat* %a, i64 %index
+  call void @llvm.aarch64.sve.st1.nxv8bf16(<vscale x 8 x bfloat> %data,
+                                           <vscale x 8 x i1> %pred,
+                                           bfloat* %base)
+  ret void
+}
+
 define void @st1h_s(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pred, i16* %addr) {
 ; CHECK-LABEL: st1h_s:
 ; CHECK: st1h { z0.s }, p0, [x0]
@@ -174,6 +185,7 @@ declare void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1
 declare void @llvm.aarch64.sve.st1.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i1>, i8*)
 declare void @llvm.aarch64.sve.st1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16*)
 declare void @llvm.aarch64.sve.st1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half*)
+declare void @llvm.aarch64.sve.st1.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x i1>, bfloat*)
 
 declare void @llvm.aarch64.sve.st1.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i1>, i8*)
 declare void @llvm.aarch64.sve.st1.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i1>, i16*)
@@ -185,3 +197,6 @@ declare void @llvm.aarch64.sve.st1.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i1>
 declare void @llvm.aarch64.sve.st1.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i1>, i32*)
 declare void @llvm.aarch64.sve.st1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*)
 declare void @llvm.aarch64.sve.st1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-st1.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-st1.ll
index 9c297bf0ac700..c541415f0c3bf 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-st1.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-st1.ll
@@ -75,6 +75,16 @@ define void @st1h_f16(<vscale x 8 x half> %data, <vscale x 8 x i1> %pred, half*
   ret void
 }
 
+define void @st1h_bf16(<vscale x 8 x bfloat> %data, <vscale x 8 x i1> %pred, bfloat* %addr) #0 {
+; CHECK-LABEL: st1h_bf16:
+; CHECK: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.nxv8bf16(<vscale x 8 x bfloat> %data,
+                                           <vscale x 8 x i1> %pred,
+                                           bfloat* %addr)
+  ret void
+}
+
 define void @st1h_s(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pred, i16* %addr) {
 ; CHECK-LABEL: st1h_s:
 ; CHECK: st1h { z0.s }, p0, [x0]
@@ -161,6 +171,7 @@ declare void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1
 declare void @llvm.aarch64.sve.st1.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i1>, i8*)
 declare void @llvm.aarch64.sve.st1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16*)
 declare void @llvm.aarch64.sve.st1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half*)
+declare void @llvm.aarch64.sve.st1.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x i1>, bfloat*)
 
 declare void @llvm.aarch64.sve.st1.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i1>, i8*)
 declare void @llvm.aarch64.sve.st1.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i1>, i16*)
@@ -172,3 +183,6 @@ declare void @llvm.aarch64.sve.st1.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i1>
 declare void @llvm.aarch64.sve.st1.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i1>, i32*)
 declare void @llvm.aarch64.sve.st1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*)
 declare void @llvm.aarch64.sve.st1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-stores.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-stores.ll
index 96fc98146effc..bda72ab2ab3a1 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-stores.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+bf16 < %s 2>%t | FileCheck %s
 ; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
 
 ; WARN-NOT: warning
@@ -44,6 +44,17 @@ define void @st2h_f16(<vscale x 8 x half> %v0, <vscale x 8 x half> %v1, <vscale
   ret void
 }
 
+define void @st2h_bf16(<vscale x 8 x bfloat> %v0, <vscale x 8 x bfloat> %v1, <vscale x 8 x i1> %pred, bfloat* %addr) #0 {
+; CHECK-LABEL: st2h_bf16:
+; CHECK: st2h { z0.h, z1.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st2.nxv8bf16(<vscale x 8 x bfloat> %v0,
+                                          <vscale x 8 x bfloat> %v1,
+                                          <vscale x 8 x i1> %pred,
+                                          bfloat* %addr)
+  ret void
+}
+
 ;
 ; ST2W
 ;
@@ -140,6 +151,18 @@ define void @st3h_f16(<vscale x 8 x half> %v0, <vscale x 8 x half> %v1, <vscale
   ret void
 }
 
+define void @st3h_bf16(<vscale x 8 x bfloat> %v0, <vscale x 8 x bfloat> %v1, <vscale x 8 x bfloat> %v2, <vscale x 8 x i1> %pred, bfloat* %addr) #0 {
+; CHECK-LABEL: st3h_bf16:
+; CHECK: st3h { z0.h, z1.h, z2.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st3.nxv8bf16(<vscale x 8 x bfloat> %v0,
+                                          <vscale x 8 x bfloat> %v1,
+                                          <vscale x 8 x bfloat> %v2,
+                                          <vscale x 8 x i1> %pred,
+                                          bfloat* %addr)
+  ret void
+}
+
 ;
 ; ST3W
 ;
@@ -243,6 +266,19 @@ define void @st4h_f16(<vscale x 8 x half> %v0, <vscale x 8 x half> %v1, <vscale
   ret void
 }
 
+define void @st4h_bf16(<vscale x 8 x bfloat> %v0, <vscale x 8 x bfloat> %v1, <vscale x 8 x bfloat> %v2, <vscale x 8 x bfloat> %v3, <vscale x 8 x i1> %pred, bfloat* %addr) #0 {
+; CHECK-LABEL: st4h_bf16:
+; CHECK: st4h { z0.h, z1.h, z2.h, z3.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st4.nxv8bf16(<vscale x 8 x bfloat> %v0,
+                                          <vscale x 8 x bfloat> %v1,
+                                          <vscale x 8 x bfloat> %v2,
+                                          <vscale x 8 x bfloat> %v3,
+                                          <vscale x 8 x i1> %pred,
+                                          bfloat* %addr)
+  ret void
+}
+
 ;
 ; ST4W
 ;
@@ -341,6 +377,16 @@ define void @stnt1h_f16(<vscale x 8 x half> %data, <vscale x 8 x i1> %pred, half
   ret void
 }
 
+define void @stnt1h_bf16(<vscale x 8 x bfloat> %data, <vscale x 8 x i1> %pred, bfloat* %addr) #0 {
+; CHECK-LABEL: stnt1h_bf16:
+; CHECK: stnt1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.stnt1.nxv8bf16(<vscale x 8 x bfloat> %data,
+                                             <vscale x 8 x i1> %pred,
+                                             bfloat* %addr)
+  ret void
+}
+
 ;
 ; STNT1W
 ;
@@ -395,6 +441,7 @@ declare void @llvm.aarch64.sve.st2.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16
 declare void @llvm.aarch64.sve.st2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i1>, i32*)
 declare void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i1>, i64*)
 declare void @llvm.aarch64.sve.st2.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x i1>, half*)
+declare void @llvm.aarch64.sve.st2.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x i1>, bfloat*)
 declare void @llvm.aarch64.sve.st2.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x i1>, float*)
 declare void @llvm.aarch64.sve.st2.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x i1>, double*)
 
@@ -403,6 +450,7 @@ declare void @llvm.aarch64.sve.st3.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16
 declare void @llvm.aarch64.sve.st3.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i1>, i32*)
 declare void @llvm.aarch64.sve.st3.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i1>, i64*)
 declare void @llvm.aarch64.sve.st3.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x i1>, half*)
+declare void @llvm.aarch64.sve.st3.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x i1>, bfloat*)
 declare void @llvm.aarch64.sve.st3.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x i1>, float*)
 declare void @llvm.aarch64.sve.st3.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x i1>, double*)
 
@@ -411,6 +459,7 @@ declare void @llvm.aarch64.sve.st4.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16
 declare void @llvm.aarch64.sve.st4.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i1>, i32*)
 declare void @llvm.aarch64.sve.st4.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i1>, i64*)
 declare void @llvm.aarch64.sve.st4.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x i1>, half*)
+declare void @llvm.aarch64.sve.st4.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x i1>, bfloat*)
 declare void @llvm.aarch64.sve.st4.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x i1>, float*)
 declare void @llvm.aarch64.sve.st4.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x i1>, double*)
 
@@ -419,5 +468,9 @@ declare void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i
 declare void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32*)
 declare void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*)
 declare void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half*)
+declare void @llvm.aarch64.sve.stnt1.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x i1>, bfloat*)
 declare void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float*)
 declare void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-ld-post-inc.ll b/llvm/test/CodeGen/AArch64/sve-ld-post-inc.ll
new file mode 100644
index 0000000000000..d0b65768b96f5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-ld-post-inc.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+; These tests are here to ensure we don't get a selection error caused
+; by performPostLD1Combine, which should bail out if the return
+; type is a scalable vector
+
+define <vscale x 4 x i32> @test_post_ld1_insert(i32* %a, i32** %ptr, i64 %inc) {
+; CHECK-LABEL: test_post_ld1_insert:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    add x9, x0, x2, lsl #2
+; CHECK-NEXT:    str x9, [x1]
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    ret
+  %load = load i32, i32* %a
+  %ins = insertelement <vscale x 4 x i32> undef, i32 %load, i32 0
+  %gep = getelementptr i32, i32* %a, i64 %inc
+  store i32* %gep, i32** %ptr
+  ret <vscale x 4 x i32> %ins
+}
+
+define <vscale x 2 x double> @test_post_ld1_dup(double* %a, double** %ptr, i64 %inc) {
+; CHECK-LABEL: test_post_ld1_dup:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    add x8, x0, x2, lsl #3
+; CHECK-NEXT:    mov z0.d, d0
+; CHECK-NEXT:    str x8, [x1]
+; CHECK-NEXT:    ret
+  %load = load double, double* %a
+  %dup = call <vscale x 2 x double> @llvm.aarch64.sve.dup.x.nxv2f64(double %load)
+  %gep = getelementptr double, double* %a, i64 %inc
+  store double* %gep, double** %ptr
+  ret <vscale x 2 x double> %dup
+}
+
+declare <vscale x 2 x double> @llvm.aarch64.sve.dup.x.nxv2f64(double)
diff --git a/llvm/test/CodeGen/AArch64/sve-ld1-addressing-mode-reg-imm.ll b/llvm/test/CodeGen/AArch64/sve-ld1-addressing-mode-reg-imm.ll
new file mode 100644
index 0000000000000..13bd864c1f23f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-ld1-addressing-mode-reg-imm.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
+
+; LD1B
+
+define <vscale x 16 x i8> @ld1b_lower_bound(<vscale x 16 x i8>* %a) {
+; CHECK-LABEL: ld1b_lower_bound:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0, #-8, mul vl]
+; CHECK-NEXT:    ret
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %a, i64 -8
+  %load = load <vscale x 16 x i8>, <vscale x 16 x i8>* %base
+  ret <vscale x 16 x i8> %load
+}
+
+define <vscale x 16 x i8> @ld1b_inbound(<vscale x 16 x i8>* %a) {
+; CHECK-LABEL: ld1b_inbound:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT:    ret
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %a, i64 2
+  %load = load <vscale x 16 x i8>, <vscale x 16 x i8>* %base
+  ret <vscale x 16 x i8> %load
+}
+
+define <vscale x 16 x i8> @ld1b_upper_bound(<vscale x 16 x i8>* %a) {
+; CHECK-LABEL: ld1b_upper_bound:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT:    ret
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %a, i64 7
+  %load = load <vscale x 16 x i8>, <vscale x 16 x i8>* %base
+  ret <vscale x 16 x i8> %load
+}
+
+define <vscale x 16 x i8> @ld1b_out_of_upper_bound(<vscale x 16 x i8>* %a) {
+; CHECK-LABEL: ld1b_out_of_upper_bound:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    rdvl x8, #8
+; CHECK-NEXT:    add x8, x0, x8
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x8]
+; CHECK-NEXT:    ret
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %a, i64 8
+  %load = load <vscale x 16 x i8>, <vscale x 16 x i8>* %base
+  ret <vscale x 16 x i8> %load
+}
+
+define <vscale x 16 x i8> @ld1b_out_of_lower_bound(<vscale x 16 x i8>* %a) {
+; CHECK-LABEL: ld1b_out_of_lower_bound:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    rdvl x8, #-9
+; CHECK-NEXT:    add x8, x0, x8
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x8]
+; CHECK-NEXT:    ret
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %a, i64 -9
+  %load = load <vscale x 16 x i8>, <vscale x 16 x i8>* %base
+  ret <vscale x 16 x i8> %load
+}
+
+; LD1H
+
+define <vscale x 8 x i16> @ld1h_inbound(<vscale x 8 x i16>* %a) {
+; CHECK-LABEL: ld1h_inbound:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0, #-2, mul vl]
+; CHECK-NEXT:    ret
+  %base = getelementptr <vscale x 8 x i16>, <vscale x 8 x i16>* %a, i64 -2
+  %load = load <vscale x 8 x i16>, <vscale x 8 x i16>* %base
+  ret <vscale x 8 x i16> %load
+}
+
+; LD1W
+
+define <vscale x 4 x i32> @ld1s_inbound(<vscale x 4 x i32>* %a) {
+; CHECK-LABEL: ld1s_inbound:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, #4, mul vl]
+; CHECK-NEXT:    ret
+  %base = getelementptr <vscale x 4 x i32>, <vscale x 4 x i32>* %a, i64 4
+  %load = load <vscale x 4 x i32>, <vscale x 4 x i32>* %base
+  ret <vscale x 4 x i32> %load
+}
+
+; LD1D
+
+define <vscale x 2 x i64> @ld1d_inbound(<vscale x 2 x i64>* %a) {
+; CHECK-LABEL: ld1d_inbound:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, #6, mul vl]
+; CHECK-NEXT:    ret
+  %base = getelementptr <vscale x 2 x i64>, <vscale x 2 x i64>* %a, i64 6
+  %load = load <vscale x 2 x i64>, <vscale x 2 x i64>* %base
+  ret <vscale x 2 x i64> %load
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll b/llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll
index 734cb19759ddc..c709cfa82373e 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll
@@ -87,6 +87,14 @@ define <vscale x 8 x half> @masked_load_nxv8f16(<vscale x 8 x half> *%a, <vscale
   ret <vscale x 8 x half> %load
 }
 
+define <vscale x 8 x bfloat> @masked_load_nxv8bf16(<vscale x 8 x bfloat> *%a, <vscale x 8 x i1> %mask) nounwind #0 {
+; CHECK-LABEL: masked_load_nxv8bf16:
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 8 x bfloat> @llvm.masked.load.nxv8bf16(<vscale x 8 x bfloat> *%a, i32 2, <vscale x 8 x i1> %mask, <vscale x 8 x bfloat> undef)
+  ret <vscale x 8 x bfloat> %load
+}
+
 ;
 ; Masked Stores
 ;
@@ -171,6 +179,14 @@ define void @masked_store_nxv8f16(<vscale x 8 x half> *%a, <vscale x 8 x half> %
   ret void
 }
 
+define void @masked_store_nxv8bf16(<vscale x 8 x bfloat> *%a, <vscale x 8 x bfloat> %val, <vscale x 8 x i1> %mask) nounwind #0 {
+; CHECK-LABEL: masked_store_nxv8bf16:
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.masked.store.nxv8bf16(<vscale x 8 x bfloat> %val, <vscale x 8 x bfloat> *%a, i32 2, <vscale x 8 x i1> %mask)
+  ret void
+}
+
 declare <vscale x 2 x i64> @llvm.masked.load.nxv2i64(<vscale x 2 x i64>*, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
 declare <vscale x 4 x i32> @llvm.masked.load.nxv4i32(<vscale x 4 x i32>*, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)
 declare <vscale x 8 x i16> @llvm.masked.load.nxv8i16(<vscale x 8 x i16>*, i32, <vscale x 8 x i1>, <vscale x 8 x i16>)
@@ -182,6 +198,7 @@ declare <vscale x 2 x half> @llvm.masked.load.nxv2f16(<vscale x 2 x half>*, i32,
 declare <vscale x 4 x float> @llvm.masked.load.nxv4f32(<vscale x 4 x float>*, i32, <vscale x 4 x i1>, <vscale x 4 x float>)
 declare <vscale x 4 x half> @llvm.masked.load.nxv4f16(<vscale x 4 x half>*, i32, <vscale x 4 x i1>, <vscale x 4 x half>)
 declare <vscale x 8 x half> @llvm.masked.load.nxv8f16(<vscale x 8 x half>*, i32, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 8 x bfloat> @llvm.masked.load.nxv8bf16(<vscale x 8 x bfloat>*, i32, <vscale x 8 x i1>, <vscale x 8 x bfloat>)
 
 declare void @llvm.masked.store.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>*, i32, <vscale x 2 x i1>)
 declare void @llvm.masked.store.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>*, i32, <vscale x 4 x i1>)
@@ -194,3 +211,7 @@ declare void @llvm.masked.store.nxv2f16(<vscale x 2 x half>, <vscale x 2 x half>
 declare void @llvm.masked.store.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>*, i32, <vscale x 4 x i1>)
 declare void @llvm.masked.store.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>*, i32, <vscale x 4 x i1>)
 declare void @llvm.masked.store.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>*, i32, <vscale x 8 x i1>)
+declare void @llvm.masked.store.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>*, i32, <vscale x 8 x i1>)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-merging-stores.ll b/llvm/test/CodeGen/AArch64/sve-merging-stores.ll
new file mode 100644
index 0000000000000..66a526f5c0d56
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-merging-stores.ll
@@ -0,0 +1,32 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+%complex = type { { double, double } }
+
+; Function Attrs: argmemonly nounwind readonly
+declare <vscale x 2 x double> @llvm.aarch64.sve.tuple.get.nxv2f64.nxv4f64(<vscale x 4 x double>, i32 immarg) #3
+
+; Function Attrs: argmemonly nounwind readonly
+declare <vscale x 4 x double> @llvm.aarch64.sve.ld2.nxv4f64.nxv2i1(<vscale x 2 x i1>, double*) #3
+
+; Function Attrs: nounwind readnone
+declare double @llvm.aarch64.sve.faddv.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>) #2
+
+define void @foo1(%complex* %outval, <vscale x 2 x i1> %pred, double *%inptr) {
+; CHECK-LABEL: foo1:
+; CHECK: ld2d { z0.d, z1.d }, p0/z, [x1]
+; CHECK-NEXT: faddv d2, p0, z0.d
+; CHECK-NEXT: faddv d0, p0, z1.d
+; CHECK-NEXT: mov v2.d[1], v0.d[0]
+; CHECK-NEXT: str q2, [x0]
+  %realp = getelementptr inbounds %complex, %complex* %outval, i64 0, i32 0, i32 0
+  %imagp = getelementptr inbounds %complex, %complex* %outval, i64 0, i32 0, i32 1
+  %1 = call <vscale x 4 x double> @llvm.aarch64.sve.ld2.nxv4f64.nxv2i1(<vscale x 2 x i1> %pred, double* nonnull %inptr)
+  %2 = call <vscale x 2 x double> @llvm.aarch64.sve.tuple.get.nxv2f64.nxv4f64(<vscale x 4 x double> %1, i32 0)
+  %3 = call double @llvm.aarch64.sve.faddv.nxv2f64(<vscale x 2 x i1> %pred, <vscale x 2 x double> %2)
+  %4 = call <vscale x 2 x double> @llvm.aarch64.sve.tuple.get.nxv2f64.nxv4f64(<vscale x 4 x double> %1, i32 1)
+  %5 = call double @llvm.aarch64.sve.faddv.nxv2f64(<vscale x 2 x i1> %pred, <vscale x 2 x double> %4)
+  store double %3, double* %realp, align 8
+  store double %5, double* %imagp, align 8
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AArch64/sve-pred-contiguous-ldst-addressing-mode-reg-imm.ll b/llvm/test/CodeGen/AArch64/sve-pred-contiguous-ldst-addressing-mode-reg-imm.ll
index ba4dd23be8560..2a6c1ec532f9b 100644
--- a/llvm/test/CodeGen/AArch64/sve-pred-contiguous-ldst-addressing-mode-reg-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-pred-contiguous-ldst-addressing-mode-reg-imm.ll
@@ -513,6 +513,24 @@ define void @test_masked_ldst_sv8f16(<vscale x 8 x half> * %base, <vscale x 8 x
   ret void
 }
 
+define void @test_masked_ldst_sv8bf16(<vscale x 8 x bfloat> * %base, <vscale x 8 x i1> %mask) nounwind #0 {
+; CHECK-LABEL: test_masked_ldst_sv8bf16:
+; CHECK-NEXT: ld1h { z[[DATA:[0-9]+]].h }, p0/z, [x0, #-1, mul vl]
+; CHECK-NEXT: st1h { z[[DATA]].h }, p0, [x0, #2, mul vl]
+; CHECK-NEXT: ret
+  %base_load = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %base, i64 -1
+  %data = call <vscale x 8 x bfloat> @llvm.masked.load.nxv8bf16(<vscale x 8 x bfloat>* %base_load,
+                                                                i32 1,
+                                                                <vscale x 8 x i1> %mask,
+                                                                <vscale x 8 x bfloat> undef)
+  %base_store = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat> * %base, i64 2
+  call void @llvm.masked.store.nxv8bf16(<vscale x 8 x bfloat> %data,
+                                        <vscale x 8 x bfloat>* %base_store,
+                                        i32 1,
+                                        <vscale x 8 x i1> %mask)
+  ret void
+}
+
 ; 8-lane zero/sign extended contiguous loads.
 
 define <vscale x 8 x i16> @masked_zload_sv8i8_to_sv8i16(<vscale x 8 x i8>* %base, <vscale x 8 x i1> %mask) nounwind {
@@ -596,6 +614,7 @@ declare <vscale x 4 x float> @llvm.masked.load.nxv4f32(<vscale x 4 x float>*, i3
 declare <vscale x 8 x i8>  @llvm.masked.load.nxv8i8 (<vscale x 8 x i8>* , i32, <vscale x 8 x i1>, <vscale x 8 x i8> )
 declare <vscale x 8 x i16> @llvm.masked.load.nxv8i16(<vscale x 8 x i16>*, i32, <vscale x 8 x i1>, <vscale x 8 x i16>)
 declare <vscale x 8 x half> @llvm.masked.load.nxv8f16(<vscale x 8 x half>*, i32, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 8 x bfloat> @llvm.masked.load.nxv8bf16(<vscale x 8 x bfloat>*, i32, <vscale x 8 x i1>, <vscale x 8 x bfloat>)
 
 ; 16-element contiguous loads.
 declare <vscale x 16 x i8> @llvm.masked.load.nxv16i8(<vscale x 16 x i8>*, i32, <vscale x 16 x i1>, <vscale x 16 x i8>)
@@ -620,6 +639,10 @@ declare void @llvm.masked.store.nxv4f32(<vscale x 4 x float>, <vscale x 4 x floa
 declare void @llvm.masked.store.nxv8i8 (<vscale x 8 x i8> , <vscale x 8 x i8>* , i32, <vscale x 8 x i1>)
 declare void @llvm.masked.store.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>*, i32, <vscale x 8 x i1>)
 declare void @llvm.masked.store.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>*, i32, <vscale x 8 x i1>)
+declare void @llvm.masked.store.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>*, i32, <vscale x 8 x i1>)
 
 ; 16-element contiguous stores.
 declare void @llvm.masked.store.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>*, i32, <vscale x 16 x i1>)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-pred-contiguous-ldst-addressing-mode-reg-reg.ll b/llvm/test/CodeGen/AArch64/sve-pred-contiguous-ldst-addressing-mode-reg-reg.ll
index e130d055d990e..d3bd88a1f9d42 100644
--- a/llvm/test/CodeGen/AArch64/sve-pred-contiguous-ldst-addressing-mode-reg-reg.ll
+++ b/llvm/test/CodeGen/AArch64/sve-pred-contiguous-ldst-addressing-mode-reg-reg.ll
@@ -498,6 +498,24 @@ define void @test_masked_ldst_sv8f16(half * %base, <vscale x 8 x i1> %mask, i64
   ret void
 }
 
+define void @test_masked_ldst_sv8bf16(bfloat * %base, <vscale x 8 x i1> %mask, i64 %offset) nounwind #0 {
+; CHECK-LABEL: test_masked_ldst_sv8bf16:
+; CHECK-NEXT: ld1h { z[[DATA:[0-9]+]].h }, p0/z, [x0, x1, lsl #1]
+; CHECK-NEXT: st1h { z[[DATA]].h }, p0, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+  %base_f16 = getelementptr bfloat, bfloat* %base, i64 %offset
+  %base_addr = bitcast bfloat* %base_f16 to <vscale x 8 x bfloat>*
+  %data = call <vscale x 8 x bfloat> @llvm.masked.load.nxv8bf16(<vscale x 8 x bfloat>* %base_addr,
+                                                               i32 1,
+                                                               <vscale x 8 x i1> %mask,
+                                                               <vscale x 8 x bfloat> undef)
+  call void @llvm.masked.store.nxv8bf16(<vscale x 8 x bfloat> %data,
+                                        <vscale x 8 x bfloat>* %base_addr,
+                                        i32 1,
+                                        <vscale x 8 x i1> %mask)
+  ret void
+}
+
 ; 8-lane zero/sign extended contiguous loads.
 
 define <vscale x 8 x i16> @masked_zload_sv8i8_to_sv8i16(i8* %base, <vscale x 8 x i1> %mask, i64 %offset) nounwind {
@@ -584,6 +602,7 @@ declare <vscale x 4 x float> @llvm.masked.load.nxv4f32(<vscale x 4 x float>*, i3
 declare <vscale x 8 x i8>  @llvm.masked.load.nxv8i8 (<vscale x 8 x i8>* , i32, <vscale x 8 x i1>, <vscale x 8 x i8> )
 declare <vscale x 8 x i16> @llvm.masked.load.nxv8i16(<vscale x 8 x i16>*, i32, <vscale x 8 x i1>, <vscale x 8 x i16>)
 declare <vscale x 8 x half> @llvm.masked.load.nxv8f16(<vscale x 8 x half>*, i32, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 8 x bfloat> @llvm.masked.load.nxv8bf16(<vscale x 8 x bfloat>*, i32, <vscale x 8 x i1>, <vscale x 8 x bfloat>)
 
 ; 16-element contiguous loads.
 declare <vscale x 16 x i8> @llvm.masked.load.nxv16i8(<vscale x 16 x i8>*, i32, <vscale x 16 x i1>, <vscale x 16 x i8>)
@@ -608,6 +627,10 @@ declare void @llvm.masked.store.nxv4f32(<vscale x 4 x float>, <vscale x 4 x floa
 declare void @llvm.masked.store.nxv8i8 (<vscale x 8 x i8> , <vscale x 8 x i8>* , i32, <vscale x 8 x i1>)
 declare void @llvm.masked.store.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>*, i32, <vscale x 8 x i1>)
 declare void @llvm.masked.store.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>*, i32, <vscale x 8 x i1>)
+declare void @llvm.masked.store.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>*, i32, <vscale x 8 x i1>)
 
 ; 16-element contiguous stores.
 declare void @llvm.masked.store.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>*, i32, <vscale x 16 x i1>)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-pred-non-temporal-ldst-addressing-mode-reg-imm.ll b/llvm/test/CodeGen/AArch64/sve-pred-non-temporal-ldst-addressing-mode-reg-imm.ll
index c2d5a4cd75e84..91c252585c662 100644
--- a/llvm/test/CodeGen/AArch64/sve-pred-non-temporal-ldst-addressing-mode-reg-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-pred-non-temporal-ldst-addressing-mode-reg-imm.ll
@@ -139,6 +139,23 @@ define void @test_masked_ldst_sv8f16(<vscale x 8 x half> * %base, <vscale x 8 x
   ret void
 }
 
+define void @test_masked_ldst_sv8bf16(<vscale x 8 x bfloat> * %base, <vscale x 8 x i1> %mask) nounwind #0 {
+; CHECK-LABEL: test_masked_ldst_sv8bf16:
+; CHECK-NEXT: ldnt1h { z[[DATA:[0-9]+]].h }, p0/z, [x0, #-1, mul vl]
+; CHECK-NEXT: stnt1h { z[[DATA]].h }, p0, [x0, #2, mul vl]
+; CHECK-NEXT: ret
+  %base_load = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %base, i64 -1
+  %base_load_bc = bitcast <vscale x 8 x bfloat>* %base_load to bfloat*
+  %data = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnt1.nxv8bf16(<vscale x 8 x i1> %mask,
+                                                                      bfloat* %base_load_bc)
+  %base_store = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat> * %base, i64 2
+  %base_store_bc = bitcast <vscale x 8 x bfloat>* %base_store to bfloat*
+  call void @llvm.aarch64.sve.stnt1.nxv8bf16(<vscale x 8 x bfloat> %data,
+                                             <vscale x 8 x i1> %mask,
+                                             bfloat* %base_store_bc)
+  ret void
+}
+
 ; 16-lane non-temporal load/stores.
 
 define void @test_masked_ldst_sv16i8(<vscale x 16 x i8> * %base, <vscale x 16 x i1> %mask) nounwind {
@@ -169,6 +186,7 @@ declare <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1>,
 ; 8-element non-temporal loads.
 declare <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1>, i16*)
 declare <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1>, half*)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnt1.nxv8bf16(<vscale x 8 x i1>, bfloat*)
 
 ; 16-element non-temporal loads.
 declare <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1>, i8*)
@@ -176,15 +194,18 @@ declare <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1>, i
 ; 2-element non-temporal stores.
 declare void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*)
 declare void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*)
-                                                                      
-; 4-element non-temporal stores.                                        
+
+; 4-element non-temporal stores.
 declare void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32*)
 declare void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float*)
-                                                                      
-; 8-element non-temporal stores.                                        
+
+; 8-element non-temporal stores.
 declare void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16*)
 declare void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half*)
+declare void @llvm.aarch64.sve.stnt1.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x i1>, bfloat*)
 
 ; 16-element non-temporal stores.
 declare void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i8*)
 
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-pred-non-temporal-ldst-addressing-mode-reg-reg.ll b/llvm/test/CodeGen/AArch64/sve-pred-non-temporal-ldst-addressing-mode-reg-reg.ll
index 1455fc7bf55be..b78dfa41eb098 100644
--- a/llvm/test/CodeGen/AArch64/sve-pred-non-temporal-ldst-addressing-mode-reg-reg.ll
+++ b/llvm/test/CodeGen/AArch64/sve-pred-non-temporal-ldst-addressing-mode-reg-reg.ll
@@ -94,6 +94,20 @@ define void @test_masked_ldst_sv8f16(half* %base, <vscale x 8 x i1> %mask, i64 %
   ret void
 }
 
+define void @test_masked_ldst_sv8bf16(bfloat* %base, <vscale x 8 x i1> %mask, i64 %offset) nounwind #0 {
+; CHECK-LABEL: test_masked_ldst_sv8bf16:
+; CHECK-NEXT: ldnt1h { z[[DATA:[0-9]+]].h }, p0/z, [x0, x1, lsl #1]
+; CHECK-NEXT: stnt1h { z[[DATA]].h }, p0, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+  %gep = getelementptr bfloat, bfloat* %base, i64 %offset
+  %data = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnt1.nxv8bf16(<vscale x 8 x i1> %mask,
+                                                                      bfloat* %gep)
+  call void @llvm.aarch64.sve.stnt1.nxv8bf16(<vscale x 8 x bfloat> %data,
+                                             <vscale x 8 x i1> %mask,
+                                             bfloat* %gep)
+  ret void
+}
+
 ; 16-lane non-temporal load/stores.
 
 define void @test_masked_ldst_sv16i8(i8* %base, <vscale x 16 x i1> %mask, i64 %offset) nounwind {
@@ -121,6 +135,7 @@ declare <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1>,
 ; 8-element non-temporal loads.
 declare <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1>, i16*)
 declare <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1>, half*)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnt1.nxv8bf16(<vscale x 8 x i1>, bfloat*)
 
 ; 16-element non-temporal loads.
 declare <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1>, i8*)
@@ -128,14 +143,18 @@ declare <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1>, i
 ; 2-element non-temporal stores.
 declare void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*)
 declare void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*)
-                                                                      
-; 4-element non-temporal stores.                                        
+
+; 4-element non-temporal stores.
 declare void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32*)
 declare void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float*)
-                                                                      
-; 8-element non-temporal stores.                                        
+
+; 8-element non-temporal stores.
 declare void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16*)
 declare void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half*)
+declare void @llvm.aarch64.sve.stnt1.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x i1>, bfloat*)
 
 ; 16-element non-temporal stores.
 declare void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i8*)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-pseudos-expand-undef.mir b/llvm/test/CodeGen/AArch64/sve-pseudos-expand-undef.mir
new file mode 100644
index 0000000000000..c81363f6133b7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-pseudos-expand-undef.mir
@@ -0,0 +1,22 @@
+# RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -run-pass=aarch64-expand-pseudo -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s
+---
+name:            add_x
+alignment:       4
+tracksRegLiveness: true
+liveins:
+  - { reg: '$p0' }
+  - { reg: '$z0' }
+frameInfo:
+  maxCallFrameSize: 0
+body:             |
+  bb.0:
+    liveins: $p0, $z0
+
+    ; CHECK: add_x
+    ; CHECK-NOT: MOVPRFX
+    ; CHECK: $z0 = FADD_ZPmZ_S renamable $p0, killed $z0, renamable $z0
+    ; CHECK-NEXT: RET
+    renamable $z0 = FADD_ZPZZ_UNDEF_S renamable $p0, renamable $z0, killed renamable $z0
+    RET_ReallyLR
+
+...
diff --git a/llvm/test/CodeGen/AArch64/sve-split-load.ll b/llvm/test/CodeGen/AArch64/sve-split-load.ll
new file mode 100644
index 0000000000000..a76b27e635574
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-split-load.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+; LOAD
+
+define <vscale x 4 x i16> @load_promote_4i8(<vscale x 4 x i16>* %a) {
+; CHECK-LABEL: load_promote_4i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    ld1h { z0.s }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %load = load <vscale x 4 x i16>, <vscale x 4 x i16>* %a
+  ret <vscale x 4 x i16> %load
+}
+
+define <vscale x 16 x i16> @load_split_i16(<vscale x 16 x i16>* %a) {
+; CHECK-LABEL: load_split_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT:    ret
+  %load = load <vscale x 16 x i16>, <vscale x 16 x i16>* %a
+  ret <vscale x 16 x i16> %load
+}
+
+define <vscale x 32 x i16> @load_split_32i16(<vscale x 32 x i16>* %a) {
+; CHECK-LABEL: load_split_32i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT:    ld1h { z2.h }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT:    ld1h { z3.h }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT:    ret
+  %load = load <vscale x 32 x i16>, <vscale x 32 x i16>* %a
+  ret <vscale x 32 x i16> %load
+}
+
+define <vscale x 16 x i64> @load_split_16i64(<vscale x 16 x i64>* %a) {
+; CHECK-LABEL: load_split_16i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT:    ld1d { z4.d }, p0/z, [x0, #4, mul vl]
+; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x0, #5, mul vl]
+; CHECK-NEXT:    ld1d { z6.d }, p0/z, [x0, #6, mul vl]
+; CHECK-NEXT:    ld1d { z7.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT:    ret
+  %load = load <vscale x 16 x i64>, <vscale x 16 x i64>* %a
+  ret <vscale x 16 x i64> %load
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-split-store.ll b/llvm/test/CodeGen/AArch64/sve-split-store.ll
new file mode 100644
index 0000000000000..2fba0404ef348
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-split-store.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define void @store_promote_4i8(<vscale x 4 x i8> %data, <vscale x 4 x i8>* %a) {
+; CHECK-LABEL: store_promote_4i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    st1b { z0.s }, p0, [x0]
+; CHECK-NEXT:    ret
+  store <vscale x 4 x i8> %data, <vscale x 4 x i8>* %a
+  ret void
+}
+
+define void @store_split_i16(<vscale x 16 x i16> %data, <vscale x 16 x i16>* %a) {
+; CHECK-LABEL: store_split_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    st1h { z1.h }, p0, [x0, #1, mul vl]
+; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
+; CHECK-NEXT:    ret
+  store <vscale x 16 x i16> %data, <vscale x 16 x i16>* %a
+  ret void
+}
+
+define void @store_split_16i32(<vscale x 16 x i32> %data, <vscale x 16 x i32>* %a) {
+; CHECK-LABEL: store_split_16i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    st1w { z3.s }, p0, [x0, #3, mul vl]
+; CHECK-NEXT:    st1w { z2.s }, p0, [x0, #2, mul vl]
+; CHECK-NEXT:    st1w { z1.s }, p0, [x0, #1, mul vl]
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
+; CHECK-NEXT:    ret
+  store <vscale x 16 x i32> %data, <vscale x 16 x i32>* %a
+  ret void
+}
+
+define void @store_split_16i64(<vscale x 16 x i64> %data, <vscale x 16 x i64>* %a) {
+; CHECK-LABEL: store_split_16i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    st1d { z7.d }, p0, [x0, #7, mul vl]
+; CHECK-NEXT:    st1d { z6.d }, p0, [x0, #6, mul vl]
+; CHECK-NEXT:    st1d { z5.d }, p0, [x0, #5, mul vl]
+; CHECK-NEXT:    st1d { z4.d }, p0, [x0, #4, mul vl]
+; CHECK-NEXT:    st1d { z3.d }, p0, [x0, #3, mul vl]
+; CHECK-NEXT:    st1d { z2.d }, p0, [x0, #2, mul vl]
+; CHECK-NEXT:    st1d { z1.d }, p0, [x0, #1, mul vl]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
+; CHECK-NEXT:    ret
+  store <vscale x 16 x i64> %data, <vscale x 16 x i64>* %a
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll b/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll
new file mode 100644
index 0000000000000..2e4f190145454
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
+
+; ST1B
+
+define void @st1b_lower_bound(<vscale x 16 x i8> %data, <vscale x 16 x i8>* %a) {
+; CHECK-LABEL: st1b_lower_bound:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0, #-8, mul vl]
+; CHECK-NEXT:    ret
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %a, i64 -8
+  store <vscale x 16 x i8> %data, <vscale x 16 x i8>* %base
+  ret void
+}
+
+define void @st1b_inbound(<vscale x 16 x i8> %data, <vscale x 16 x i8>* %a) {
+; CHECK-LABEL: st1b_inbound:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0, #1, mul vl]
+; CHECK-NEXT:    ret
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %a, i64 1
+  store <vscale x 16 x i8> %data, <vscale x 16 x i8>* %base
+  ret void
+}
+
+define void @st1b_upper_bound(<vscale x 16 x i8> %data, <vscale x 16 x i8>* %a) {
+; CHECK-LABEL: st1b_upper_bound:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0, #7, mul vl]
+; CHECK-NEXT:    ret
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %a, i64 7
+  store <vscale x 16 x i8> %data, <vscale x 16 x i8>* %base
+  ret void
+}
+
+define void @st1b_out_of_upper_bound(<vscale x 16 x i8> %data, <vscale x 16 x i8>* %a) {
+; CHECK-LABEL: st1b_out_of_upper_bound:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    rdvl x8, #8
+; CHECK-NEXT:    add x8, x0, x8
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    st1b { z0.b }, p0, [x8]
+; CHECK-NEXT:    ret
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %a, i64 8
+  store <vscale x 16 x i8> %data, <vscale x 16 x i8>* %base
+  ret void
+}
+
+define void @st1b_out_of_lower_bound(<vscale x 16 x i8> %data, <vscale x 16 x i8>* %a) {
+; CHECK-LABEL: st1b_out_of_lower_bound:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    rdvl x8, #-9
+; CHECK-NEXT:    add x8, x0, x8
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    st1b { z0.b }, p0, [x8]
+; CHECK-NEXT:    ret
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %a, i64 -9
+  store <vscale x 16 x i8> %data, <vscale x 16 x i8>* %base
+  ret void
+}
+
+; ST1H
+
+define void @st1h_inbound(<vscale x 8 x i16> %data, <vscale x 8 x i16>* %a) {
+; CHECK-LABEL: st1h_inbound:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    st1h { z0.h }, p0, [x0, #-6, mul vl]
+; CHECK-NEXT:    ret
+  %base = getelementptr <vscale x 8 x i16>, <vscale x 8 x i16>* %a, i64 -6
+  store <vscale x 8 x i16> %data, <vscale x 8 x i16>* %base
+  ret void
+}
+
+; ST1W
+
+define void @st1w_inbound(<vscale x 4 x i32> %data, <vscale x 4 x i32>* %a) {
+; CHECK-LABEL: st1w_inbound:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0, #2, mul vl]
+; CHECK-NEXT:    ret
+  %base = getelementptr <vscale x 4 x i32>, <vscale x 4 x i32>* %a, i64 2
+  store <vscale x 4 x i32> %data, <vscale x 4 x i32>* %base
+  ret void
+}
+
+; ST1D
+
+define void @st1d_inbound(<vscale x 2 x i64> %data, <vscale x 2 x i64>* %a) {
+; CHECK-LABEL: st1d_inbound:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0, #5, mul vl]
+; CHECK-NEXT:    ret
+  %base = getelementptr <vscale x 2 x i64>, <vscale x 2 x i64>* %a, i64 5
+  store <vscale x 2 x i64> %data, <vscale x 2 x i64>* %base
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-trunc.ll b/llvm/test/CodeGen/AArch64/sve-trunc.ll
index c78f036078bab..50ce4d9660879 100644
--- a/llvm/test/CodeGen/AArch64/sve-trunc.ll
+++ b/llvm/test/CodeGen/AArch64/sve-trunc.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ; For all the functions below should the operation is a nop
 define <vscale x 8 x i8> @trunc_i16toi8(<vscale x 8 x i16> %in) {
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-splat.ll b/llvm/test/CodeGen/AArch64/sve-vector-splat.ll
index 5bacbee042c09..cd7ecbeb5ca13 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-splat.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-splat.ll
@@ -1,4 +1,7 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 ;; Splats of legal integer vector types
 
@@ -172,6 +175,15 @@ define <vscale x 16 x i1> @sve_splat_16xi1(i1 %val) {
 
 ;; Splats of legal floating point vector types
 
+define <vscale x 8 x bfloat> @splat_nxv8bf16(bfloat %val) #0 {
+; CHECK-LABEL: splat_nxv8bf16:
+; CHECK: mov z0.h, h0
+; CHECK-NEXT: ret
+  %1 = insertelement <vscale x 8 x bfloat> undef, bfloat %val, i32 0
+  %2 = shufflevector <vscale x 8 x bfloat> %1, <vscale x 8 x bfloat> undef, <vscale x 8 x i32> zeroinitializer
+  ret <vscale x 8 x bfloat> %2
+}
+
 define <vscale x 8 x half> @splat_nxv8f16(half %val) {
 ; CHECK-LABEL: splat_nxv8f16:
 ; CHECK: mov z0.h, h0
@@ -233,6 +245,13 @@ define <vscale x 8 x half> @splat_nxv8f16_zero() {
   ret <vscale x 8 x half> zeroinitializer
 }
 
+define <vscale x 8 x bfloat> @splat_nxv8bf16_zero() #0 {
+; CHECK-LABEL: splat_nxv8bf16_zero:
+; CHECK: mov z0.h, #0
+; CHECK-NEXT: ret
+  ret <vscale x 8 x bfloat> zeroinitializer
+}
+
 define <vscale x 4 x half> @splat_nxv4f16_zero() {
 ; CHECK-LABEL: splat_nxv4f16_zero:
 ; CHECK: mov z0.h, #0
@@ -321,3 +340,6 @@ define <vscale x 2 x double> @splat_nxv2f64_imm() {
   %2 = shufflevector <vscale x 2 x double> %1, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   ret <vscale x 2 x double> %2
 }
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-contiguous-conflict-detection.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-contiguous-conflict-detection.ll
index d1a924475224c..67c9637b5c426 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-contiguous-conflict-detection.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-contiguous-conflict-detection.ll
@@ -36,6 +36,14 @@ define <vscale x 2 x i1> @whilerw_i64(i64* %a, i64* %b) {
   ret <vscale x 2 x i1> %out
 }
 
+define <vscale x 8 x i1> @whilerw_bfloat(bfloat* %a, bfloat* %b) {
+; CHECK-LABEL: whilerw_bfloat:
+; CHECK: whilerw  p0.h, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilerw.h.nx8i1.bf16.bf16(bfloat* %a, bfloat* %b)
+  ret <vscale x 8 x i1> %out
+}
+
 define <vscale x 8 x i1> @whilerw_half(half* %a, half* %b) {
 ; CHECK-LABEL: whilerw_half:
 ; CHECK: whilerw  p0.h, x0, x1
@@ -96,6 +104,14 @@ define <vscale x 2 x i1> @whilewr_i64(i64* %a, i64* %b) {
   ret <vscale x 2 x i1> %out
 }
 
+define <vscale x 8 x i1> @whilewr_bfloat(bfloat* %a, bfloat* %b) {
+; CHECK-LABEL: whilewr_bfloat:
+; CHECK: whilewr  p0.h, x0, x1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.whilewr.h.nx8i1.bf16.bf16(bfloat* %a, bfloat* %b)
+  ret <vscale x 8 x i1> %out
+}
+
 define <vscale x 8 x i1> @whilewr_half(half* %a, half* %b) {
 ; CHECK-LABEL: whilewr_half:
 ; CHECK: whilewr  p0.h, x0, x1
@@ -125,6 +141,7 @@ declare <vscale x 8 x i1> @llvm.aarch64.sve.whilerw.h.nx8i1(i16* %a, i16* %b)
 declare <vscale x 4 x i1> @llvm.aarch64.sve.whilerw.s.nx4i1(i32* %a, i32* %b)
 declare <vscale x 2 x i1> @llvm.aarch64.sve.whilerw.d.nx2i1(i64* %a, i64* %b)
 
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilerw.h.nx8i1.bf16.bf16(bfloat* %a, bfloat* %b)
 declare <vscale x 8 x i1> @llvm.aarch64.sve.whilerw.h.nx8i1.f16.f16(half* %a, half* %b)
 declare <vscale x 4 x i1> @llvm.aarch64.sve.whilerw.s.nx4i1.f32.f32(float* %a, float* %b)
 declare <vscale x 2 x i1> @llvm.aarch64.sve.whilerw.d.nx2i1.f64.f64(double* %a, double* %b)
@@ -134,6 +151,7 @@ declare <vscale x 8 x i1> @llvm.aarch64.sve.whilewr.h.nx8i1(i16* %a, i16* %b)
 declare <vscale x 4 x i1> @llvm.aarch64.sve.whilewr.s.nx4i1(i32* %a, i32* %b)
 declare <vscale x 2 x i1> @llvm.aarch64.sve.whilewr.d.nx2i1(i64* %a, i64* %b)
 
+declare <vscale x 8 x i1> @llvm.aarch64.sve.whilewr.h.nx8i1.bf16.bf16(bfloat* %a, bfloat* %b)
 declare <vscale x 8 x i1> @llvm.aarch64.sve.whilewr.h.nx8i1.f16.f16(half* %a, half* %b)
 declare <vscale x 4 x i1> @llvm.aarch64.sve.whilewr.s.nx4i1.f32.f32(float* %a, float* %b)
 declare <vscale x 2 x i1> @llvm.aarch64.sve.whilewr.d.nx2i1.f64.f64(double* %a, double* %b)
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-perm-tb.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-perm-tb.ll
index 122a3985e641b..bbff15fa8b3fa 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-perm-tb.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-perm-tb.ll
@@ -122,6 +122,16 @@ define <vscale x 8 x half> @ftbx_h(<vscale x 8 x half> %a, <vscale x 8 x half> %
   ret <vscale x 8 x half> %out
 }
 
+define <vscale x 8 x bfloat> @ftbx_h_bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x i16> %c) #0 {
+; CHECK-LABEL: ftbx_h_bf16:
+; CHECK: tbx z0.h, z1.h, z2.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.tbx.nxv8bf16(<vscale x 8 x bfloat> %a,
+                                                                   <vscale x 8 x bfloat> %b,
+                                                                   <vscale x 8 x i16> %c)
+  ret <vscale x 8 x bfloat> %out
+}
+
 define <vscale x 4 x i32> @tbx_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
 ; CHECK-LABEL: tbx_s:
 ; CHECK: tbx z0.s, z1.s, z2.s
@@ -179,3 +189,8 @@ declare <vscale x 2 x i64> @llvm.aarch64.sve.tbx.nxv2i64(<vscale x 2 x i64>, <vs
 declare <vscale x 8 x half> @llvm.aarch64.sve.tbx.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x i16>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.tbx.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x i32>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.tbx.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x i64>)
+
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.tbx.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x i16>)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-uniform-dsp-zeroing.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-uniform-dsp-zeroing.ll
index a83f71d54fd14..60bd2e167271f 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-uniform-dsp-zeroing.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-uniform-dsp-zeroing.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 -asm-verbose=0 -mattr=+use-experimental-zeroing-pseudos < %s | FileCheck %s
 
 ;
 ; SQSHLU
diff --git a/llvm/test/CodeGen/AArch64/vec-extract-branch.ll b/llvm/test/CodeGen/AArch64/vec-extract-branch.ll
new file mode 100644
index 0000000000000..e05213d534217
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vec-extract-branch.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -verify-machineinstrs | FileCheck %s
+
+define i32 @vec_extract_branch(<2 x double> %x, i32 %y)  {
+; CHECK-LABEL: vec_extract_branch:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcmgt v0.2d, v0.2d, #0.0
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    and w8, w9, w8
+; CHECK-NEXT:    tbz w8, #0, .LBB0_2
+; CHECK-NEXT:  // %bb.1: // %true
+; CHECK-NEXT:    mov w8, #42
+; CHECK-NEXT:    sdiv w0, w8, w0
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_2: // %false
+; CHECK-NEXT:    mov w0, #88
+; CHECK-NEXT:    ret
+  %t1 = fcmp ogt <2 x double> %x, zeroinitializer
+  %t2 = extractelement <2 x i1> %t1, i32 0
+  %t3 = extractelement <2 x i1> %t1, i32 1
+  %t4 = and i1 %t2, %t3
+  br i1 %t4, label %true, label %false
+true:
+  %y1 = sdiv i32 42, %y
+  ret i32 %y1
+false:
+  ret i32 88
+}
diff --git a/llvm/test/CodeGen/AMDGPU/32-bit-local-address-space.ll b/llvm/test/CodeGen/AMDGPU/32-bit-local-address-space.ll
index 72e62f8dbbfdf..eb6c98774ea67 100644
--- a/llvm/test/CodeGen/AMDGPU/32-bit-local-address-space.ll
+++ b/llvm/test/CodeGen/AMDGPU/32-bit-local-address-space.ll
@@ -57,9 +57,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}null_32bit_lds_ptr:
-; SI: v_cmp_ne_u32
+; SI: s_cmp_lg_u32
 ; SI-NOT: v_cmp_ne_u32
-; SI: v_cndmask_b32
+; SI: s_cselect_b32
 define amdgpu_kernel void @null_32bit_lds_ptr(i32 addrspace(1)* %out, i32 addrspace(3)* %lds) nounwind {
   %cmp = icmp ne i32 addrspace(3)* %lds, null
   %x = select i1 %cmp, i32 123, i32 456
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-itofp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-itofp.mir
index 2021107b84485..ad21521c389ff 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-itofp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-itofp.mir
@@ -173,3 +173,95 @@ body:             |
     %4:_(s32) = G_ANYEXT %3
     $vgpr0 = COPY %4
 ...
+
+---
+name: uitofp_s64_char_to_f32
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: uitofp_s64_char_to_f32
+    ; CHECK: liveins: $vgpr0_vgpr1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[AND]](s64)
+    ; CHECK: [[AMDGPU_CVT_F32_UBYTE0_:%[0-9]+]]:_(s32) = G_AMDGPU_CVT_F32_UBYTE0 [[TRUNC]]
+    ; CHECK: $vgpr0 = COPY [[AMDGPU_CVT_F32_UBYTE0_]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_CONSTANT i64 255
+    %2:_(s64) = G_AND %0, %1
+    %3:_(s32) = G_UITOFP %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: sitofp_s64_char_to_f32
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: sitofp_s64_char_to_f32
+    ; CHECK: liveins: $vgpr0_vgpr1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[AND]](s64)
+    ; CHECK: [[AMDGPU_CVT_F32_UBYTE0_:%[0-9]+]]:_(s32) = G_AMDGPU_CVT_F32_UBYTE0 [[TRUNC]]
+    ; CHECK: $vgpr0 = COPY [[AMDGPU_CVT_F32_UBYTE0_]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_CONSTANT i64 255
+    %2:_(s64) = G_AND %0, %1
+    %3:_(s32) = G_SITOFP %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: uitofp_s16_char_to_f32
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: uitofp_s16_char_to_f32
+    ; CHECK: liveins: $vgpr0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CHECK: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[AND]](s16)
+    ; CHECK: [[AMDGPU_CVT_F32_UBYTE0_:%[0-9]+]]:_(s32) = G_AMDGPU_CVT_F32_UBYTE0 [[ANYEXT]]
+    ; CHECK: $vgpr0 = COPY [[AMDGPU_CVT_F32_UBYTE0_]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s16) = G_CONSTANT i16 255
+    %3:_(s16) = G_AND %1, %2
+    %4:_(s32) = G_UITOFP %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: sitofp_s16_char_to_f32
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: sitofp_s16_char_to_f32
+    ; CHECK: liveins: $vgpr0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CHECK: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[AND]](s16)
+    ; CHECK: [[AMDGPU_CVT_F32_UBYTE0_:%[0-9]+]]:_(s32) = G_AMDGPU_CVT_F32_UBYTE0 [[ANYEXT]]
+    ; CHECK: $vgpr0 = COPY [[AMDGPU_CVT_F32_UBYTE0_]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s16) = G_CONSTANT i16 255
+    %3:_(s16) = G_AND %1, %2
+    %4:_(s32) = G_SITOFP %3
+    $vgpr0 = COPY %4
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
index 35d2df4894e70..1622edace5b22 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
@@ -503,8 +503,8 @@ define double @v_uitofp_i8_to_f64(i8 %arg0) nounwind {
 define amdgpu_kernel void @load_i8_to_f32(float addrspace(1)* noalias %out, i8 addrspace(1)* noalias %in) nounwind {
 ; SI-LABEL: load_i8_to_f32:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xa
-; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xc
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
 ; SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; SI-NEXT:    s_mov_b32 s2, 0
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
@@ -520,8 +520,8 @@ define amdgpu_kernel void @load_i8_to_f32(float addrspace(1)* noalias %out, i8 a
 ;
 ; VI-LABEL: load_i8_to_f32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x28
-; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x30
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; VI-NEXT:    v_ashrrev_i32_e32 v3, 31, v0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s1
@@ -579,8 +579,8 @@ define amdgpu_kernel void @load_i8_to_f32(float addrspace(1)* noalias %out, i8 a
 define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned(<4 x float> addrspace(1)* noalias %out, <4 x i8> addrspace(1)* noalias %in) nounwind {
 ; SI-LABEL: load_v4i8_to_v4f32_unaligned:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xa
-; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xc
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
 ; SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; SI-NEXT:    s_mov_b32 s2, 0
@@ -610,8 +610,8 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned(<4 x float> addrspace(1)
 ;
 ; VI-LABEL: load_v4i8_to_v4f32_unaligned:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x28
-; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x30
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; VI-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -686,8 +686,8 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned(<4 x float> addrspace(1)
 define amdgpu_kernel void @i8_zext_inreg_i32_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind {
 ; SI-LABEL: i8_zext_inreg_i32_to_f32:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xa
-; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xc
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
 ; SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; SI-NEXT:    s_mov_b32 s2, 0
@@ -705,8 +705,8 @@ define amdgpu_kernel void @i8_zext_inreg_i32_to_f32(float addrspace(1)* noalias
 ;
 ; VI-LABEL: i8_zext_inreg_i32_to_f32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x28
-; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x30
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; VI-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -735,8 +735,8 @@ define amdgpu_kernel void @i8_zext_inreg_i32_to_f32(float addrspace(1)* noalias
 define amdgpu_kernel void @i8_zext_inreg_hi1_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind {
 ; SI-LABEL: i8_zext_inreg_hi1_to_f32:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xa
-; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xc
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
 ; SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; SI-NEXT:    s_mov_b32 s2, 0
@@ -753,8 +753,8 @@ define amdgpu_kernel void @i8_zext_inreg_hi1_to_f32(float addrspace(1)* noalias
 ;
 ; VI-LABEL: i8_zext_inreg_hi1_to_f32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x28
-; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x30
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; VI-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -785,8 +785,8 @@ define amdgpu_kernel void @i8_zext_inreg_hi1_to_f32(float addrspace(1)* noalias
 define amdgpu_kernel void @i8_zext_i32_to_f32(float addrspace(1)* noalias %out, i8 addrspace(1)* noalias %in) nounwind {
 ; SI-LABEL: i8_zext_i32_to_f32:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xa
-; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xc
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
 ; SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; SI-NEXT:    s_mov_b32 s2, 0
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
@@ -801,8 +801,8 @@ define amdgpu_kernel void @i8_zext_i32_to_f32(float addrspace(1)* noalias %out,
 ;
 ; VI-LABEL: i8_zext_i32_to_f32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x28
-; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x30
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; VI-NEXT:    v_ashrrev_i32_e32 v3, 31, v0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s1
@@ -828,8 +828,8 @@ define amdgpu_kernel void @i8_zext_i32_to_f32(float addrspace(1)* noalias %out,
 define amdgpu_kernel void @v4i8_zext_v4i32_to_v4f32(<4 x float> addrspace(1)* noalias %out, <4 x i8> addrspace(1)* noalias %in) nounwind {
 ; SI-LABEL: v4i8_zext_v4i32_to_v4f32:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xa
-; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xc
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
 ; SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; SI-NEXT:    s_mov_b32 s2, 0
@@ -859,8 +859,8 @@ define amdgpu_kernel void @v4i8_zext_v4i32_to_v4f32(<4 x float> addrspace(1)* no
 ;
 ; VI-LABEL: v4i8_zext_v4i32_to_v4f32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x28
-; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x30
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; VI-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -902,8 +902,8 @@ define amdgpu_kernel void @v4i8_zext_v4i32_to_v4f32(<4 x float> addrspace(1)* no
 define amdgpu_kernel void @extract_byte0_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind {
 ; SI-LABEL: extract_byte0_to_f32:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xa
-; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xc
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
 ; SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; SI-NEXT:    s_mov_b32 s2, 0
@@ -920,8 +920,8 @@ define amdgpu_kernel void @extract_byte0_to_f32(float addrspace(1)* noalias %out
 ;
 ; VI-LABEL: extract_byte0_to_f32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x28
-; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x30
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; VI-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -948,8 +948,8 @@ define amdgpu_kernel void @extract_byte0_to_f32(float addrspace(1)* noalias %out
 define amdgpu_kernel void @extract_byte1_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind {
 ; SI-LABEL: extract_byte1_to_f32:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xa
-; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xc
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
 ; SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; SI-NEXT:    s_mov_b32 s2, 0
@@ -967,8 +967,8 @@ define amdgpu_kernel void @extract_byte1_to_f32(float addrspace(1)* noalias %out
 ;
 ; VI-LABEL: extract_byte1_to_f32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x28
-; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x30
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; VI-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -997,8 +997,8 @@ define amdgpu_kernel void @extract_byte1_to_f32(float addrspace(1)* noalias %out
 define amdgpu_kernel void @extract_byte2_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind {
 ; SI-LABEL: extract_byte2_to_f32:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xa
-; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xc
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
 ; SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; SI-NEXT:    s_mov_b32 s2, 0
@@ -1016,8 +1016,8 @@ define amdgpu_kernel void @extract_byte2_to_f32(float addrspace(1)* noalias %out
 ;
 ; VI-LABEL: extract_byte2_to_f32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x28
-; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x30
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; VI-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1047,8 +1047,8 @@ define amdgpu_kernel void @extract_byte2_to_f32(float addrspace(1)* noalias %out
 define amdgpu_kernel void @extract_byte3_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind {
 ; SI-LABEL: extract_byte3_to_f32:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xa
-; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xc
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
 ; SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; SI-NEXT:    s_mov_b32 s2, 0
@@ -1064,8 +1064,8 @@ define amdgpu_kernel void @extract_byte3_to_f32(float addrspace(1)* noalias %out
 ;
 ; VI-LABEL: extract_byte3_to_f32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x28
-; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x30
+; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; VI-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1142,3 +1142,187 @@ bb:
   store float %add, float addrspace(1)* %out
   ret void
 }
+
+define float @v_test_sitofp_i64_byte_to_f32(i64 %arg0) {
+; SI-LABEL: v_test_sitofp_i64_byte_to_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    s_movk_i32 s6, 0xff
+; SI-NEXT:    v_and_b32_e32 v2, s6, v0
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0, v2
+; SI-NEXT:    v_ffbh_u32_e32 v4, v2
+; SI-NEXT:    v_addc_u32_e64 v3, s[4:5], 0, 0, vcc
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 32, v4
+; SI-NEXT:    v_ffbh_u32_e32 v5, v3
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; SI-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v5, 0xbe
+; SI-NEXT:    v_sub_i32_e32 v6, vcc, v5, v4
+; SI-NEXT:    v_lshl_b64 v[4:5], v[2:3], v4
+; SI-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; SI-NEXT:    v_and_b32_e32 v3, 0x7fffffff, v5
+; SI-NEXT:    v_cndmask_b32_e32 v2, 0, v6, vcc
+; SI-NEXT:    v_and_b32_e32 v5, s6, v3
+; SI-NEXT:    v_lshrrev_b32_e32 v3, 8, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 23, v2
+; SI-NEXT:    s_mov_b32 s4, 0
+; SI-NEXT:    s_movk_i32 s5, 0x80
+; SI-NEXT:    v_or_b32_e32 v2, v2, v3
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[4:5]
+; SI-NEXT:    v_and_b32_e32 v3, 1, v2
+; SI-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; SI-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[4:5]
+; SI-NEXT:    v_mov_b32_e32 v0, 0
+; SI-NEXT:    v_cndmask_b32_e64 v3, v3, 1, vcc
+; SI-NEXT:    v_mov_b32_e32 v1, v0
+; SI-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; SI-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; SI-NEXT:    v_cndmask_b32_e64 v0, v2, -v2, vcc
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_test_sitofp_i64_byte_to_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    s_movk_i32 s6, 0xff
+; VI-NEXT:    v_and_b32_e32 v2, s6, v0
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 0, v2
+; VI-NEXT:    v_ffbh_u32_e32 v4, v2
+; VI-NEXT:    v_addc_u32_e64 v3, s[4:5], 0, 0, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 32, v4
+; VI-NEXT:    v_ffbh_u32_e32 v5, v3
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; VI-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; VI-NEXT:    v_mov_b32_e32 v5, 0xbe
+; VI-NEXT:    v_sub_u32_e32 v6, vcc, v5, v4
+; VI-NEXT:    v_lshlrev_b64 v[4:5], v4, v[2:3]
+; VI-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; VI-NEXT:    v_and_b32_e32 v3, 0x7fffffff, v5
+; VI-NEXT:    v_cndmask_b32_e32 v2, 0, v6, vcc
+; VI-NEXT:    v_and_b32_e32 v5, s6, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v3, 8, v3
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 23, v2
+; VI-NEXT:    s_mov_b32 s4, 0
+; VI-NEXT:    s_movk_i32 s5, 0x80
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[4:5]
+; VI-NEXT:    v_and_b32_e32 v3, 1, v2
+; VI-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; VI-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    v_cndmask_b32_e64 v3, v3, 1, vcc
+; VI-NEXT:    v_mov_b32_e32 v1, v0
+; VI-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
+; VI-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; VI-NEXT:    v_cndmask_b32_e64 v0, v2, -v2, vcc
+; VI-NEXT:    s_setpc_b64 s[30:31]
+  %masked = and i64 %arg0, 255
+  %itofp = sitofp i64 %masked to float
+  ret float %itofp
+}
+
+define float @v_test_uitofp_i64_byte_to_f32(i64 %arg0) {
+; SI-LABEL: v_test_uitofp_i64_byte_to_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    s_movk_i32 s4, 0xff
+; SI-NEXT:    v_and_b32_e32 v0, s4, v0
+; SI-NEXT:    v_ffbh_u32_e32 v2, v0
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 32, v2
+; SI-NEXT:    v_ffbh_u32_e32 v3, 0
+; SI-NEXT:    v_cmp_eq_u32_e64 vcc, 0, 0
+; SI-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; SI-NEXT:    v_mov_b32_e32 v3, 0xbe
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    v_sub_i32_e32 v4, vcc, v3, v2
+; SI-NEXT:    v_lshl_b64 v[2:3], v[0:1], v2
+; SI-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; SI-NEXT:    v_and_b32_e32 v1, 0x7fffffff, v3
+; SI-NEXT:    v_cndmask_b32_e32 v0, 0, v4, vcc
+; SI-NEXT:    v_and_b32_e32 v3, s4, v1
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 23, v0
+; SI-NEXT:    s_mov_b32 s4, 0
+; SI-NEXT:    s_movk_i32 s5, 0x80
+; SI-NEXT:    v_or_b32_e32 v0, v0, v1
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[2:3]
+; SI-NEXT:    v_and_b32_e32 v1, 1, v0
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
+; SI-NEXT:    v_cndmask_b32_e64 v1, v1, 1, vcc
+; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_test_uitofp_i64_byte_to_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    s_movk_i32 s4, 0xff
+; VI-NEXT:    v_and_b32_e32 v0, s4, v0
+; VI-NEXT:    v_ffbh_u32_e32 v2, v0
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 32, v2
+; VI-NEXT:    v_ffbh_u32_e32 v3, 0
+; VI-NEXT:    v_cmp_eq_u32_e64 vcc, 0, 0
+; VI-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v3, 0xbe
+; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    v_sub_u32_e32 v4, vcc, v3, v2
+; VI-NEXT:    v_lshlrev_b64 v[2:3], v2, v[0:1]
+; VI-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; VI-NEXT:    v_and_b32_e32 v1, 0x7fffffff, v3
+; VI-NEXT:    v_cndmask_b32_e32 v0, 0, v4, vcc
+; VI-NEXT:    v_and_b32_e32 v3, s4, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 23, v0
+; VI-NEXT:    s_mov_b32 s4, 0
+; VI-NEXT:    s_movk_i32 s5, 0x80
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[2:3]
+; VI-NEXT:    v_and_b32_e32 v1, 1, v0
+; VI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
+; VI-NEXT:    v_cndmask_b32_e64 v1, v1, 1, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; VI-NEXT:    s_setpc_b64 s[30:31]
+  %masked = and i64 %arg0, 255
+  %itofp = uitofp i64 %masked to float
+  ret float %itofp
+}
+
+define float @v_test_sitofp_i16_byte_to_f32(i16 %arg0) {
+; SI-LABEL: v_test_sitofp_i16_byte_to_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_test_sitofp_i16_byte_to_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; VI-NEXT:    v_cvt_f32_ubyte0_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; VI-NEXT:    s_setpc_b64 s[30:31]
+  %masked = and i16 %arg0, 255
+  %itofp = sitofp i16 %masked to float
+  ret float %itofp
+}
+
+define float @v_test_uitofp_i16_byte_to_f32(i16 %arg0) {
+; SI-LABEL: v_test_uitofp_i16_byte_to_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT:    v_bfe_u32 v0, v0, 0, 16
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_test_uitofp_i16_byte_to_f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; VI-NEXT:    v_cvt_f32_ubyte0_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; VI-NEXT:    s_setpc_b64 s[30:31]
+  %masked = and i16 %arg0, 255
+  %itofp = uitofp i16 %masked to float
+  ret float %itofp
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-divergent.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-divergent.ll
index bfec0b93f33b6..256b95d222bea 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-divergent.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-divergent.ll
@@ -1,10 +1,10 @@
-; RUN: not llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -global-isel-abort=2 -pass-remarks-missed="gisel.*" -o /dev/null 2>&1 %s | FileCheck -check-prefix=ERR %s
+; RUN: not llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -global-isel-abort=2 -pass-remarks-missed="gisel.*" -verify-machineinstrs -o /dev/null 2>&1 %s | FileCheck -check-prefix=ERR %s
 
-; ERR: remark: <unknown>:0:0: cannot select: %24:sreg_32(p5) = G_DYN_STACKALLOC %23:vgpr(s32), 1 (in function: kernel_dynamic_stackalloc_vgpr_align4)
+; ERR: remark: <unknown>:0:0: cannot select: %{{[0-9]+}}:sreg_32(p5) = G_DYN_STACKALLOC %{{[0-9]+}}:vgpr(s32), 1 (in function: kernel_dynamic_stackalloc_vgpr_align4)
 ; ERR-NEXT: warning: Instruction selection used fallback path for kernel_dynamic_stackalloc_vgpr_align4
 ; ERR-NEXT: error: <unknown>:0:0: in function kernel_dynamic_stackalloc_vgpr_align4 void (i32 addrspace(1)*): unsupported dynamic alloca
 
-; ERR: remark: <unknown>:0:0: cannot select: %8:sreg_32(p5) = G_DYN_STACKALLOC %7:vgpr(s32), 1 (in function: func_dynamic_stackalloc_vgpr_align4)
+; ERR: remark: <unknown>:0:0: cannot select: %{{[0-9]+}}:sreg_32(p5) = G_DYN_STACKALLOC %{{[0-9]+}}:vgpr(s32), 1 (in function: func_dynamic_stackalloc_vgpr_align4)
 ; ERR-NEXT: warning: Instruction selection used fallback path for func_dynamic_stackalloc_vgpr_align4
 ; ERR-NEXT: error: <unknown>:0:0: in function func_dynamic_stackalloc_vgpr_align4 void (i32): unsupported dynamic alloca
 
@@ -13,13 +13,13 @@ define amdgpu_kernel void @kernel_dynamic_stackalloc_vgpr_align4(i32 addrspace(1
   %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 %id
   %n = load i32, i32 addrspace(1)* %gep
   %alloca = alloca i32, i32 %n, align 4, addrspace(5)
-  store volatile i32 0, i32 addrspace(5)* %alloca
+  store volatile i32 addrspace(5)* %alloca, i32 addrspace(5)* addrspace(1)* undef
   ret void
 }
 
 define void @func_dynamic_stackalloc_vgpr_align4(i32 %n) {
   %alloca = alloca i32, i32 %n, align 4, addrspace(5)
-  store volatile i32 0, i32 addrspace(5)* %alloca
+  store volatile i32 addrspace(5)* %alloca, i32 addrspace(5)* addrspace(1)* undef
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.ll
index 82ecb616aa114..acd71947aeeed 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.ll
@@ -181,6 +181,21 @@ define signext i16 @i16_signext_func_void() #0 {
   ret i16 %val
 }
 
+define half @f16_func_void() #0 {
+  ; CHECK-LABEL: name: f16_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s16) = G_LOAD [[DEF]](p1) :: (load 2 from `half addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LOAD]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  %val = load half, half addrspace(1)* undef
+  ret half %val
+}
+
 define i32 @i32_func_void() #0 {
   ; CHECK-LABEL: name: i32_func_void
   ; CHECK: bb.1 (%ir-block.0):
@@ -726,6 +741,24 @@ define <2 x i16> @v2i16_func_void() #0 {
   ret <2 x i16> %val
 }
 
+define <2 x half> @v2f16_func_void() #0 {
+  ; CHECK-LABEL: name: v2f16_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[DEF]](p1) :: (load 4 from `<2 x half> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[LOAD]](<2 x s16>)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   $vgpr1 = COPY [[ANYEXT1]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  %val = load <2 x half>, <2 x half> addrspace(1)* undef
+  ret <2 x half> %val
+}
+
 define <3 x i16> @v3i16_func_void() #0 {
   ; CHECK-LABEL: name: v3i16_func_void
   ; CHECK: bb.1 (%ir-block.0):
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inline-asm.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/inline-asm.ll
new file mode 100644
index 0000000000000..587f808bc55e7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inline-asm.ll
@@ -0,0 +1,83 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -O0 -global-isel -verify-machineinstrs -o - %s | FileCheck %s
+
+define i32 @test_sgpr_reg_class_constraint() nounwind {
+; CHECK-LABEL: test_sgpr_reg_class_constraint:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    s_mov_b32 s4, 7
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    s_mov_b32 s5, 8
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    s_add_u32 s4, s4, s5
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %asm0 = tail call i32 asm "s_mov_b32 $0, 7", "=s"() nounwind
+  %asm1 = tail call i32 asm "s_mov_b32 $0, 8", "=s"() nounwind
+  %asm2 = tail call i32 asm "s_add_u32 $0, $1, $2", "=s,s,s"(i32 %asm0, i32 %asm1) nounwind
+  ret i32 %asm2
+}
+
+define i32 @test_sgpr_matching_constraint() nounwind {
+; CHECK-LABEL: test_sgpr_matching_constraint:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    s_mov_b32 s4, 7
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    s_mov_b32 s5, 8
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    s_add_u32 s5, s4, s5
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    v_mov_b32_e32 v0, s5
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %asm0 = tail call i32 asm "s_mov_b32 $0, 7", "=s"() nounwind
+  %asm1 = tail call i32 asm "s_mov_b32 $0, 8", "=s"() nounwind
+  %asm2 = tail call i32 asm "s_add_u32 $0, $1, $2", "=s,s,0"(i32 %asm0, i32 %asm1) nounwind
+  ret i32 %asm2
+}
+
+define i32 @test_sgpr_to_vgpr_move_reg_class_constraint() nounwind {
+; CHECK-LABEL: test_sgpr_to_vgpr_move_reg_class_constraint:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    s_mov_b32 s4, 7
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    v_mov_b32 v0, s4
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %asm0 = tail call i32 asm "s_mov_b32 $0, 7", "=s"() nounwind
+  %asm1 = tail call i32 asm "v_mov_b32 $0, $1", "=v,s"(i32 %asm0) nounwind
+  ret i32 %asm1
+}
+
+define i32 @test_sgpr_to_vgpr_move_matching_constraint() nounwind {
+; CHECK-LABEL: test_sgpr_to_vgpr_move_matching_constraint:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    s_mov_b32 s4, 7
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    v_mov_b32 v0, v0
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %asm0 = tail call i32 asm "s_mov_b32 $0, 7", "=s"() nounwind
+  %asm1 = tail call i32 asm "v_mov_b32 $0, $1", "=v,0"(i32 %asm0) nounwind
+  ret i32 %asm1
+}
+
+!0 = !{i32 70}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-and.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-and.mir
index 7ccdcb414c281..ecfb9b618f5ed 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-and.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-and.mir
@@ -370,65 +370,6 @@ body: |
 
 ---
 
-name:            and_s1_vcc_undef_vcc_undef_vcc
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-
-body: |
-  bb.0:
-    ; WAVE64-LABEL: name: and_s1_vcc_undef_vcc_undef_vcc
-    ; WAVE64: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 undef %1:sreg_64, undef %2:sreg_64
-    ; WAVE64: S_ENDPGM 0, implicit [[S_AND_B64_]]
-    ; WAVE32-LABEL: name: and_s1_vcc_undef_vcc_undef_vcc
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
-    ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 undef %1:sreg_32, undef %2:sreg_32
-    ; WAVE32: S_ENDPGM 0, implicit [[S_AND_B32_]]
-    %2:vcc(s1) = G_AND undef %0:vcc(s1), undef %1:vcc(s1)
-    S_ENDPGM 0, implicit %2
-...
-
----
-
-name:            and_s1_sgpr_undef_sgpr_undef_sgpr
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-
-body: |
-  bb.0:
-    ; WAVE64-LABEL: name: and_s1_sgpr_undef_sgpr_undef_sgpr
-    ; WAVE64: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 undef %1:sreg_32, undef %2:sreg_32, implicit-def dead $scc
-    ; WAVE64: S_ENDPGM 0, implicit [[S_AND_B32_]]
-    ; WAVE32-LABEL: name: and_s1_sgpr_undef_sgpr_undef_sgpr
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
-    ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 undef %1:sreg_32, undef %2:sreg_32, implicit-def dead $scc
-    ; WAVE32: S_ENDPGM 0, implicit [[S_AND_B32_]]
-    %2:sgpr(s1) = G_AND undef %0:sgpr(s1), undef %1:sgpr(s1)
-    S_ENDPGM 0, implicit %2
-...
-
----
-
-name:            and_s1_vgpr_undef_vgpr_undef_vgpr
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-
-body: |
-  bb.0:
-    ; WAVE64-LABEL: name: and_s1_vgpr_undef_vgpr_undef_vgpr
-    ; WAVE64: [[AND:%[0-9]+]]:vgpr(s1) = G_AND undef %1:vgpr, undef %2:vgpr
-    ; WAVE64: S_ENDPGM 0, implicit [[AND]](s1)
-    ; WAVE32-LABEL: name: and_s1_vgpr_undef_vgpr_undef_vgpr
-    ; WAVE32: [[AND:%[0-9]+]]:vgpr(s1) = G_AND undef %1:vgpr, undef %2:vgpr
-    ; WAVE32: S_ENDPGM 0, implicit [[AND]](s1)
-    %2:vgpr(s1) = G_AND undef %0:vgpr(s1), undef %1:vgpr(s1)
-    S_ENDPGM 0, implicit %2
-...
-
----
-
 name:            and_s1_vcc_copy_to_vcc
 legalized:       true
 regBankSelected: true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-brcond.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-brcond.mir
index 42ce52d3975e7..496d496a42c58 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-brcond.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-brcond.mir
@@ -62,28 +62,6 @@ body: |
 
 ---
 
-name:            brcond_scc_undef
-legalized:       true
-regBankSelected: true
-
-body: |
-  ; GCN-LABEL: name: brcond_scc_undef
-  ; GCN: bb.0:
-  ; GCN:   successors: %bb.1(0x80000000)
-  ; GCN:   $scc = COPY %0:sgpr_32
-  ; GCN:   S_CBRANCH_SCC1 %bb.1, implicit $scc
-  ; GCN: bb.1:
-  bb.0:
-    liveins: $sgpr0, $sgpr1
-
-    G_BRCOND undef %0:sgpr(s32), %bb.1
-
-  bb.1:
-
-...
-
----
-
 name:            brcond_scc_br
 legalized:       true
 regBankSelected: true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-build-vector.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-build-vector.mir
index b69dcf20dd3c3..9cbb2940655a4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-build-vector.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-build-vector.mir
@@ -89,46 +89,6 @@ body: |
     S_ENDPGM 0, implicit %2
 ...
 
----
-name: test_build_vector_s_v2s32_undef_s_s32_s_s32
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-
-body: |
-  bb.0:
-    liveins: $sgpr0
-
-    ; GCN-LABEL: name: test_build_vector_s_v2s32_undef_s_s32_s_s32
-    ; GCN: liveins: $sgpr0
-    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; GCN: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE undef %2:sreg_32, %subreg.sub0, [[COPY]], %subreg.sub1
-    ; GCN: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
-    %1:sgpr(s32) = COPY $sgpr0
-    %2:sgpr(<2 x s32>) = G_BUILD_VECTOR undef %0:sgpr(s32), %1
-    S_ENDPGM 0, implicit %2
-...
-
----
-name: test_build_vector_s_v2s32_s_s32_undef_s_s32
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-
-body: |
-  bb.0:
-    liveins: $sgpr0
-
-    ; GCN-LABEL: name: test_build_vector_s_v2s32_s_s32_undef_s_s32
-    ; GCN: liveins: $sgpr0
-    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; GCN: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, undef %2:sreg_32, %subreg.sub1
-    ; GCN: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
-    %0:sgpr(s32) = COPY $sgpr0
-    %2:sgpr(<2 x s32>) = G_BUILD_VECTOR %0, undef %1:sgpr(s32),
-    S_ENDPGM 0, implicit %2
-...
-
 ---
 name: test_build_vector_s_v2s64_s_s64_s_s64
 legalized:       true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-copy.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-copy.mir
index e600aa0e18054..f8a7fc8bd6f8b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-copy.mir
@@ -207,7 +207,7 @@ body: |
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_64_xexec = COPY $sgpr0_sgpr1
     ; WAVE32: S_ENDPGM 0, implicit [[COPY]]
     %0:sreg_64_xexec = COPY $sgpr0_sgpr1
-    %1:sreg_64_xexec  = COPY %0
+    %1:sreg_64_xexec = COPY %0
     S_ENDPGM 0, implicit %1
 
 ...
@@ -239,24 +239,3 @@ body: |
     S_ENDPGM 0, implicit %2
 
 ...
-
----
-
-name:            copy_s1_vcc_to_vcc_undef
-legalized:       true
-regBankSelected: true
-
-
-body: |
-  bb.0:
-    liveins: $sgpr0_sgpr1
-
-    ; WAVE64-LABEL: name: copy_s1_vcc_to_vcc_undef
-    ; WAVE64: S_ENDPGM 0, implicit %1:sreg_64_xexec
-    ; WAVE32-LABEL: name: copy_s1_vcc_to_vcc_undef
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
-    ; WAVE32: S_ENDPGM 0, implicit %1:sreg_32_xm0_xexec
-    %1:vcc(s1) = COPY undef  %0:vcc(s1)
-    S_ENDPGM 0, implicit %1
-
-...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-merge-values.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-merge-values.mir
index 495f6c8ca3809..a4c831e25810a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-merge-values.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-merge-values.mir
@@ -89,46 +89,6 @@ body: |
     S_ENDPGM 0, implicit %2
 ...
 
----
-name: test_merge_values_s_s64_undef_s_s32_s_s32
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-
-body: |
-  bb.0:
-    liveins: $sgpr0
-
-    ; GCN-LABEL: name: test_merge_values_s_s64_undef_s_s32_s_s32
-    ; GCN: liveins: $sgpr0
-    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; GCN: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE undef %2:sreg_32, %subreg.sub0, [[COPY]], %subreg.sub1
-    ; GCN: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
-    %1:sgpr(s32) = COPY $sgpr0
-    %2:sgpr(s64) = G_MERGE_VALUES undef %0:sgpr(s32), %1
-    S_ENDPGM 0, implicit %2
-...
-
----
-name: test_merge_values_s_s64_s_s32_undef_s_s32
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-
-body: |
-  bb.0:
-    liveins: $sgpr0
-
-    ; GCN-LABEL: name: test_merge_values_s_s64_s_s32_undef_s_s32
-    ; GCN: liveins: $sgpr0
-    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; GCN: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, undef %2:sreg_32, %subreg.sub1
-    ; GCN: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
-    %0:sgpr(s32) = COPY $sgpr0
-    %2:sgpr(s64) = G_MERGE_VALUES %0, undef %1:sgpr(s32),
-    S_ENDPGM 0, implicit %2
-...
-
 ---
 name: test_merge_values_s_s96_s_s32_s_s32_s_s32
 legalized:       true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir
index 2f10cd3203201..40b7b69f83b7f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir
@@ -370,65 +370,6 @@ body: |
 
 ---
 
-name:            or_s1_vcc_undef_vcc_undef_vcc
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-
-body: |
-  bb.0:
-    ; WAVE64-LABEL: name: or_s1_vcc_undef_vcc_undef_vcc
-    ; WAVE64: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 undef %1:sreg_64, undef %2:sreg_64
-    ; WAVE64: S_ENDPGM 0, implicit [[S_OR_B64_]]
-    ; WAVE32-LABEL: name: or_s1_vcc_undef_vcc_undef_vcc
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
-    ; WAVE32: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 undef %1:sreg_32, undef %2:sreg_32
-    ; WAVE32: S_ENDPGM 0, implicit [[S_OR_B32_]]
-    %2:vcc(s1) = G_OR undef %0:vcc(s1), undef %1:vcc(s1)
-    S_ENDPGM 0, implicit %2
-...
-
----
-
-name:            or_s1_sgpr_undef_sgpr_undef_sgpr
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-
-body: |
-  bb.0:
-    ; WAVE64-LABEL: name: or_s1_sgpr_undef_sgpr_undef_sgpr
-    ; WAVE64: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 undef %1:sreg_32, undef %2:sreg_32, implicit-def dead $scc
-    ; WAVE64: S_ENDPGM 0, implicit [[S_OR_B32_]]
-    ; WAVE32-LABEL: name: or_s1_sgpr_undef_sgpr_undef_sgpr
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
-    ; WAVE32: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 undef %1:sreg_32, undef %2:sreg_32, implicit-def dead $scc
-    ; WAVE32: S_ENDPGM 0, implicit [[S_OR_B32_]]
-    %2:sgpr(s1) = G_OR undef %0:sgpr(s1), undef %1:sgpr(s1)
-    S_ENDPGM 0, implicit %2
-...
-
----
-
-name:            or_s1_vgpr_undef_vgpr_undef_vgpr
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-
-body: |
-  bb.0:
-    ; WAVE64-LABEL: name: or_s1_vgpr_undef_vgpr_undef_vgpr
-    ; WAVE64: [[OR:%[0-9]+]]:vgpr(s1) = G_OR undef %1:vgpr, undef %2:vgpr
-    ; WAVE64: S_ENDPGM 0, implicit [[OR]](s1)
-    ; WAVE32-LABEL: name: or_s1_vgpr_undef_vgpr_undef_vgpr
-    ; WAVE32: [[OR:%[0-9]+]]:vgpr(s1) = G_OR undef %1:vgpr, undef %2:vgpr
-    ; WAVE32: S_ENDPGM 0, implicit [[OR]](s1)
-    %2:vgpr(s1) = G_OR undef %0:vgpr(s1), undef %1:vgpr(s1)
-    S_ENDPGM 0, implicit %2
-...
-
----
-
 name:            or_s1_vcc_copy_to_vcc
 legalized:       true
 regBankSelected: true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-trunc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-trunc.mir
index 238e1d15f0d3f..a9b9464944ab2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-trunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-trunc.mir
@@ -13,7 +13,7 @@ body: |
     ; GCN-LABEL: name: trunc_sgpr_s32_to_s1
     ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GCN: S_ENDPGM 0, implicit [[COPY]]
-    %0:sgpr(s32) =COPY $sgpr0
+    %0:sgpr(s32) = COPY $sgpr0
     %1:sgpr(s1) = G_TRUNC %0
     S_ENDPGM 0, implicit %1
 ...
@@ -29,7 +29,7 @@ body: |
     ; GCN-LABEL: name: trunc_sgpr_s32_to_s16
     ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GCN: S_ENDPGM 0, implicit [[COPY]]
-    %0:sgpr(s32) =COPY $sgpr0
+    %0:sgpr(s32) = COPY $sgpr0
     %1:sgpr(s16) = G_TRUNC %0
     S_ENDPGM 0, implicit %1
 ...
@@ -198,7 +198,7 @@ body: |
     ; GCN-LABEL: name: trunc_vgpr_s32_to_s1
     ; GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GCN: S_ENDPGM 0, implicit [[COPY]]
-    %0:vgpr(s32) =COPY $vgpr0
+    %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s1) = G_TRUNC %0
     S_ENDPGM 0, implicit %1
 ...
@@ -214,7 +214,7 @@ body: |
     ; GCN-LABEL: name: trunc_vgpr_s32_to_s16
     ; GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GCN: S_ENDPGM 0, implicit [[COPY]]
-    %0:vgpr(s32) =COPY $vgpr0
+    %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s16) = G_TRUNC %0
     S_ENDPGM 0, implicit %1
 ...
@@ -381,8 +381,8 @@ regBankSelected: true
 body: |
   bb.0:
     liveins: $sgpr0, $sgpr1
-    %0:sgpr(s32) =COPY $sgpr0
-    %1:sgpr(s32) =COPY $sgpr1
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:sgpr(s32) = COPY $sgpr1
     %2:sgpr(s1) = G_TRUNC %0
     %3:sgpr(s32) = G_SELECT %2, %0, %1
     S_ENDPGM 0, implicit %3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-trunc.v2s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-trunc.v2s16.mir
index fba87c7304659..310bd955b8ce1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-trunc.v2s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-trunc.v2s16.mir
@@ -30,7 +30,7 @@ body: |
     ; GFX8: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY1]], [[S_MOV_B32_]], implicit-def $scc
     ; GFX8: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[S_LSHL_B32_]], [[S_AND_B32_]], implicit-def $scc
     ; GFX8: S_ENDPGM 0, implicit [[S_OR_B32_]]
-    %0:sgpr(<2 x s32>) =COPY $sgpr0_sgpr1
+    %0:sgpr(<2 x s32>) = COPY $sgpr0_sgpr1
     %1:sgpr(<2 x s16>) = G_TRUNC %0
     S_ENDPGM 0, implicit %1
 ...
@@ -59,7 +59,7 @@ body: |
     ; GFX8: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
     ; GFX8: [[V_MOV_B32_sdwa:%[0-9]+]]:vgpr_32 = V_MOV_B32_sdwa 0, [[COPY2]], 0, 5, 2, 4, implicit $exec, implicit [[COPY1]](tied-def 0)
     ; GFX8: S_ENDPGM 0, implicit [[V_MOV_B32_sdwa]]
-    %0:vgpr(<2 x s32>) =COPY $vgpr0_vgpr1
+    %0:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:vgpr(<2 x s16>) = G_TRUNC %0
     S_ENDPGM 0, implicit %1
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-unmerge-values.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-unmerge-values.mir
index 7a27911288ac6..1c0644f7bf421 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-unmerge-values.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-unmerge-values.mir
@@ -85,23 +85,6 @@ body: |
     S_ENDPGM 0, implicit %1, implicit %2
 ...
 
----
-name: test_unmerge_values_s_s32_v_s32_s_s64_undef_src
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-
-body: |
-  bb.0:
-
-    ; GCN-LABEL: name: test_unmerge_values_s_s32_v_s32_s_s64_undef_src
-    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY undef %2.sub0:sreg_64
-    ; GCN: [[COPY1:%[0-9]+]]:vgpr_32 = COPY undef %2.sub1:sreg_64
-    ; GCN: S_ENDPGM 0, implicit [[COPY]], implicit [[COPY1]]
-    %1:sgpr(s32), %2:vgpr(s32) = G_UNMERGE_VALUES undef %0:sgpr(s64)
-    S_ENDPGM 0, implicit %1, implicit %2
-...
-
 ---
 name: test_unmerge_values_s_s32_s_s32_s32_s_s96
 legalized:       true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-xor.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-xor.mir
index c81348aa5d899..1c03557cc4953 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-xor.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-xor.mir
@@ -371,65 +371,6 @@ body: |
 
 ---
 
-name:            xor_s1_vcc_undef_vcc_undef_vcc
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-
-body: |
-  bb.0:
-    ; WAVE64-LABEL: name: xor_s1_vcc_undef_vcc_undef_vcc
-    ; WAVE64: [[S_XOR_B64_:%[0-9]+]]:sreg_64 = S_XOR_B64 undef %1:sreg_64, undef %2:sreg_64
-    ; WAVE64: S_ENDPGM 0, implicit [[S_XOR_B64_]]
-    ; WAVE32-LABEL: name: xor_s1_vcc_undef_vcc_undef_vcc
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
-    ; WAVE32: [[S_XOR_B32_:%[0-9]+]]:sreg_32 = S_XOR_B32 undef %1:sreg_32, undef %2:sreg_32
-    ; WAVE32: S_ENDPGM 0, implicit [[S_XOR_B32_]]
-    %2:vcc(s1) = G_XOR undef %0:vcc(s1), undef %1:vcc(s1)
-    S_ENDPGM 0, implicit %2
-...
-
----
-
-name:            xor_s1_sgpr_undef_sgpr_undef_sgpr
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-
-body: |
-  bb.0:
-    ; WAVE64-LABEL: name: xor_s1_sgpr_undef_sgpr_undef_sgpr
-    ; WAVE64: [[S_XOR_B32_:%[0-9]+]]:sreg_32 = S_XOR_B32 undef %1:sreg_32, undef %2:sreg_32, implicit-def dead $scc
-    ; WAVE64: S_ENDPGM 0, implicit [[S_XOR_B32_]]
-    ; WAVE32-LABEL: name: xor_s1_sgpr_undef_sgpr_undef_sgpr
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
-    ; WAVE32: [[S_XOR_B32_:%[0-9]+]]:sreg_32 = S_XOR_B32 undef %1:sreg_32, undef %2:sreg_32, implicit-def dead $scc
-    ; WAVE32: S_ENDPGM 0, implicit [[S_XOR_B32_]]
-    %2:sgpr(s1) = G_XOR undef %0:sgpr(s1), undef %1:sgpr(s1)
-    S_ENDPGM 0, implicit %2
-...
-
----
-
-name:            xor_s1_vgpr_undef_vgpr_undef_vgpr
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-
-body: |
-  bb.0:
-    ; WAVE64-LABEL: name: xor_s1_vgpr_undef_vgpr_undef_vgpr
-    ; WAVE64: [[XOR:%[0-9]+]]:vgpr(s1) = G_XOR undef %1:vgpr, undef %2:vgpr
-    ; WAVE64: S_ENDPGM 0, implicit [[XOR]](s1)
-    ; WAVE32-LABEL: name: xor_s1_vgpr_undef_vgpr_undef_vgpr
-    ; WAVE32: [[XOR:%[0-9]+]]:vgpr(s1) = G_XOR undef %1:vgpr, undef %2:vgpr
-    ; WAVE32: S_ENDPGM 0, implicit [[XOR]](s1)
-    %2:vgpr(s1) = G_XOR undef %0:vgpr(s1), undef %1:vgpr(s1)
-    S_ENDPGM 0, implicit %2
-...
-
----
-
 name:            xor_s1_vcc_copy_to_vcc
 legalized:       true
 regBankSelected: true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel.ll
index 36c8baa0d15cd..76a6f1732102b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -O0 -amdgpu-ir-lower-kernel-arguments=0 -stop-after=irtranslator -global-isel %s -o - | FileCheck -check-prefix=HSA-VI %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -O0 -amdgpu-ir-lower-kernel-arguments=0 -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck -check-prefix=HSA-VI %s
+; RUN: llc -global-isel -mtriple=amdgcn-- -mcpu=fiji -O0 -amdgpu-ir-lower-kernel-arguments=0 -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck -check-prefix=LEGACY-MESA-VI %s
 
 define amdgpu_kernel void @i8_arg(i32 addrspace(1)* nocapture %out, i8 %in) nounwind {
   ; HSA-VI-LABEL: name: i8_arg
@@ -15,6 +16,19 @@ define amdgpu_kernel void @i8_arg(i32 addrspace(1)* nocapture %out, i8 %in) noun
   ; HSA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s8)
   ; HSA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: i8_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 1, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s8)
+  ; LEGACY-MESA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
   %ext = zext i8 %in to i32
   store i32 %ext, i32 addrspace(1)* %out, align 4
   ret void
@@ -34,6 +48,19 @@ define amdgpu_kernel void @i8_zext_arg(i32 addrspace(1)* nocapture %out, i8 zero
   ; HSA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s8)
   ; HSA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: i8_zext_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 1, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s8)
+  ; LEGACY-MESA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
   %ext = zext i8 %in to i32
   store i32 %ext, i32 addrspace(1)* %out, align 4
   ret void
@@ -53,6 +80,19 @@ define amdgpu_kernel void @i8_sext_arg(i32 addrspace(1)* nocapture %out, i8 sign
   ; HSA-VI:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[LOAD1]](s8)
   ; HSA-VI:   G_STORE [[SEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: i8_sext_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 1, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[LOAD1]](s8)
+  ; LEGACY-MESA-VI:   G_STORE [[SEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
   %ext = sext i8 %in to i32
   store i32 %ext, i32 addrspace(1)* %out, align 4
   ret void
@@ -72,6 +112,19 @@ define amdgpu_kernel void @i16_arg(i32 addrspace(1)* nocapture %out, i16 %in) no
   ; HSA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s16)
   ; HSA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: i16_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 2, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s16)
+  ; LEGACY-MESA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
   %ext = zext i16 %in to i32
   store i32 %ext, i32 addrspace(1)* %out, align 4
   ret void
@@ -91,6 +144,19 @@ define amdgpu_kernel void @i16_zext_arg(i32 addrspace(1)* nocapture %out, i16 ze
   ; HSA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s16)
   ; HSA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: i16_zext_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 2, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s16)
+  ; LEGACY-MESA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
   %ext = zext i16 %in to i32
   store i32 %ext, i32 addrspace(1)* %out, align 4
   ret void
@@ -110,6 +176,19 @@ define amdgpu_kernel void @i16_sext_arg(i32 addrspace(1)* nocapture %out, i16 si
   ; HSA-VI:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[LOAD1]](s16)
   ; HSA-VI:   G_STORE [[SEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: i16_sext_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 2, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[LOAD1]](s16)
+  ; LEGACY-MESA-VI:   G_STORE [[SEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
   %ext = sext i16 %in to i32
   store i32 %ext, i32 addrspace(1)* %out, align 4
   ret void
@@ -128,6 +207,18 @@ define amdgpu_kernel void @i32_arg(i32 addrspace(1)* nocapture %out, i32 %in) no
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 4, align 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: i32_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store i32 %in, i32 addrspace(1)* %out, align 4
   ret void
@@ -146,6 +237,18 @@ define amdgpu_kernel void @f32_arg(float addrspace(1)* nocapture %out, float %in
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 4, align 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: f32_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store float %in, float addrspace(1)* %out, align 4
   ret void
@@ -164,6 +267,18 @@ define amdgpu_kernel void @v2i8_arg(<2 x i8> addrspace(1)* %out, <2 x i8> %in) {
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<2 x s8>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 2, align 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<2 x s8>), [[LOAD]](p1) :: (store 2 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v2i8_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<2 x s8>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 2, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<2 x s8>), [[LOAD]](p1) :: (store 2 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <2 x i8> %in, <2 x i8> addrspace(1)* %out
   ret void
@@ -182,6 +297,18 @@ define amdgpu_kernel void @v2i16_arg(<2 x i16> addrspace(1)* %out, <2 x i16> %in
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 4, align 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<2 x s16>), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v2i16_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<2 x s16>), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <2 x i16> %in, <2 x i16> addrspace(1)* %out
   ret void
@@ -200,6 +327,18 @@ define amdgpu_kernel void @v2i32_arg(<2 x i32> addrspace(1)* nocapture %out, <2
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<2 x s32>), [[LOAD]](p1) :: (store 8 into %ir.out, align 4, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v2i32_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<2 x s32>), [[LOAD]](p1) :: (store 8 into %ir.out, align 4, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <2 x i32> %in, <2 x i32> addrspace(1)* %out, align 4
   ret void
@@ -218,6 +357,18 @@ define amdgpu_kernel void @v2f32_arg(<2 x float> addrspace(1)* nocapture %out, <
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<2 x s32>), [[LOAD]](p1) :: (store 8 into %ir.out, align 4, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v2f32_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<2 x s32>), [[LOAD]](p1) :: (store 8 into %ir.out, align 4, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <2 x float> %in, <2 x float> addrspace(1)* %out, align 4
   ret void
@@ -236,6 +387,18 @@ define amdgpu_kernel void @v3i8_arg(<3 x i8> addrspace(1)* nocapture %out, <3 x
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 3, align 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<3 x s8>), [[LOAD]](p1) :: (store 3 into %ir.out, align 4, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v3i8_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 3, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<3 x s8>), [[LOAD]](p1) :: (store 3 into %ir.out, align 4, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <3 x i8> %in, <3 x i8> addrspace(1)* %out, align 4
   ret void
@@ -254,6 +417,18 @@ define amdgpu_kernel void @v3i16_arg(<3 x i16> addrspace(1)* nocapture %out, <3
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<3 x s16>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 6, align 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<3 x s16>), [[LOAD]](p1) :: (store 6 into %ir.out, align 4, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v3i16_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<3 x s16>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 6, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<3 x s16>), [[LOAD]](p1) :: (store 6 into %ir.out, align 4, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <3 x i16> %in, <3 x i16> addrspace(1)* %out, align 4
   ret void
@@ -272,6 +447,18 @@ define amdgpu_kernel void @v3i32_arg(<3 x i32> addrspace(1)* nocapture %out, <3
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 12, align 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<3 x s32>), [[LOAD]](p1) :: (store 12 into %ir.out, align 4, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v3i32_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 52
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 12, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<3 x s32>), [[LOAD]](p1) :: (store 12 into %ir.out, align 4, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <3 x i32> %in, <3 x i32> addrspace(1)* %out, align 4
   ret void
@@ -290,6 +477,18 @@ define amdgpu_kernel void @v3f32_arg(<3 x float> addrspace(1)* nocapture %out, <
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 12, align 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<3 x s32>), [[LOAD]](p1) :: (store 12 into %ir.out, align 4, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v3f32_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 52
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 12, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<3 x s32>), [[LOAD]](p1) :: (store 12 into %ir.out, align 4, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <3 x float> %in, <3 x float> addrspace(1)* %out, align 4
   ret void
@@ -308,6 +507,18 @@ define amdgpu_kernel void @v4i8_arg(<4 x i8> addrspace(1)* %out, <4 x i8> %in) {
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 4, align 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<4 x s8>), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v4i8_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<4 x s8>), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <4 x i8> %in, <4 x i8> addrspace(1)* %out
   ret void
@@ -326,6 +537,18 @@ define amdgpu_kernel void @v4i16_arg(<4 x i16> addrspace(1)* %out, <4 x i16> %in
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<4 x s16>), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v4i16_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<4 x s16>), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <4 x i16> %in, <4 x i16> addrspace(1)* %out
   ret void
@@ -344,6 +567,18 @@ define amdgpu_kernel void @v4i32_arg(<4 x i32> addrspace(1)* nocapture %out, <4
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<4 x s32>), [[LOAD]](p1) :: (store 16 into %ir.out, align 4, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v4i32_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 52
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 16, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<4 x s32>), [[LOAD]](p1) :: (store 16 into %ir.out, align 4, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <4 x i32> %in, <4 x i32> addrspace(1)* %out, align 4
   ret void
@@ -362,6 +597,18 @@ define amdgpu_kernel void @v4f32_arg(<4 x float> addrspace(1)* nocapture %out, <
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<4 x s32>), [[LOAD]](p1) :: (store 16 into %ir.out, align 4, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v4f32_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 52
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 16, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<4 x s32>), [[LOAD]](p1) :: (store 16 into %ir.out, align 4, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <4 x float> %in, <4 x float> addrspace(1)* %out, align 4
   ret void
@@ -380,6 +627,18 @@ define amdgpu_kernel void @v8i8_arg(<8 x i8> addrspace(1)* %out, <8 x i8> %in) {
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<8 x s8>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<8 x s8>), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v8i8_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<8 x s8>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<8 x s8>), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <8 x i8> %in, <8 x i8> addrspace(1)* %out
   ret void
@@ -398,6 +657,18 @@ define amdgpu_kernel void @v8i16_arg(<8 x i16> addrspace(1)* %out, <8 x i16> %in
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<8 x s16>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<8 x s16>), [[LOAD]](p1) :: (store 16 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v8i16_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 52
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<8 x s16>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 16, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<8 x s16>), [[LOAD]](p1) :: (store 16 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <8 x i16> %in, <8 x i16> addrspace(1)* %out
   ret void
@@ -416,6 +687,18 @@ define amdgpu_kernel void @v8i32_arg(<8 x i32> addrspace(1)* nocapture %out, <8
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<8 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 32, align 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<8 x s32>), [[LOAD]](p1) :: (store 32 into %ir.out, align 4, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v8i32_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 68
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<8 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 32, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<8 x s32>), [[LOAD]](p1) :: (store 32 into %ir.out, align 4, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <8 x i32> %in, <8 x i32> addrspace(1)* %out, align 4
   ret void
@@ -434,6 +717,18 @@ define amdgpu_kernel void @v8f32_arg(<8 x float> addrspace(1)* nocapture %out, <
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<8 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 32, align 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<8 x s32>), [[LOAD]](p1) :: (store 32 into %ir.out, align 4, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v8f32_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 68
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<8 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 32, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<8 x s32>), [[LOAD]](p1) :: (store 32 into %ir.out, align 4, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <8 x float> %in, <8 x float> addrspace(1)* %out, align 4
   ret void
@@ -452,6 +747,18 @@ define amdgpu_kernel void @v16i8_arg(<16 x i8> addrspace(1)* %out, <16 x i8> %in
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<16 x s8>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<16 x s8>), [[LOAD]](p1) :: (store 16 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v16i8_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 52
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<16 x s8>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 16, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<16 x s8>), [[LOAD]](p1) :: (store 16 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <16 x i8> %in, <16 x i8> addrspace(1)* %out
   ret void
@@ -470,6 +777,18 @@ define amdgpu_kernel void @v16i16_arg(<16 x i16> addrspace(1)* %out, <16 x i16>
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<16 x s16>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 32, align 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<16 x s16>), [[LOAD]](p1) :: (store 32 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v16i16_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 68
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<16 x s16>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 32, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<16 x s16>), [[LOAD]](p1) :: (store 32 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <16 x i16> %in, <16 x i16> addrspace(1)* %out
   ret void
@@ -488,6 +807,18 @@ define amdgpu_kernel void @v16i32_arg(<16 x i32> addrspace(1)* nocapture %out, <
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 64, align 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<16 x s32>), [[LOAD]](p1) :: (store 64 into %ir.out, align 4, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v16i32_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 100
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 64, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<16 x s32>), [[LOAD]](p1) :: (store 64 into %ir.out, align 4, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <16 x i32> %in, <16 x i32> addrspace(1)* %out, align 4
   ret void
@@ -506,6 +837,18 @@ define amdgpu_kernel void @v16f32_arg(<16 x float> addrspace(1)* nocapture %out,
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 64, align 16, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](<16 x s32>), [[LOAD]](p1) :: (store 64 into %ir.out, align 4, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: v16f32_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 100
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 64, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](<16 x s32>), [[LOAD]](p1) :: (store 64 into %ir.out, align 4, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store <16 x float> %in, <16 x float> addrspace(1)* %out, align 4
   ret void
@@ -524,6 +867,18 @@ define amdgpu_kernel void @kernel_arg_i64(i64 addrspace(1)* %out, i64 %a) nounwi
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](s64), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: kernel_arg_i64
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](s64), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
   store i64 %a, i64 addrspace(1)* %out, align 8
   ret void
 }
@@ -541,6 +896,18 @@ define amdgpu_kernel void @f64_kernel_arg(double addrspace(1)* %out, double  %in
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](s64), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: f64_kernel_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](s64), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
 entry:
   store double %in, double addrspace(1)* %out
   ret void
@@ -559,6 +926,18 @@ define amdgpu_kernel void @i1_arg(i1 addrspace(1)* %out, i1 %x) nounwind {
   ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s1) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 1, align 8, addrspace 4)
   ; HSA-VI:   G_STORE [[LOAD1]](s1), [[LOAD]](p1) :: (store 1 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: i1_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s1) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 1, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](s1), [[LOAD]](p1) :: (store 1 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
   store i1 %x, i1 addrspace(1)* %out, align 1
   ret void
 }
@@ -577,6 +956,19 @@ define amdgpu_kernel void @i1_arg_zext_i32(i32 addrspace(1)* %out, i1 %x) nounwi
   ; HSA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s1)
   ; HSA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: i1_arg_zext_i32
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s1) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 1, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s1)
+  ; LEGACY-MESA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
   %ext = zext i1 %x to i32
   store i32 %ext, i32 addrspace(1)* %out, align 4
   ret void
@@ -596,6 +988,19 @@ define amdgpu_kernel void @i1_arg_zext_i64(i64 addrspace(1)* %out, i1 %x) nounwi
   ; HSA-VI:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[LOAD1]](s1)
   ; HSA-VI:   G_STORE [[ZEXT]](s64), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: i1_arg_zext_i64
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s1) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 1, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[LOAD1]](s1)
+  ; LEGACY-MESA-VI:   G_STORE [[ZEXT]](s64), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
   %ext = zext i1 %x to i64
   store i64 %ext, i64 addrspace(1)* %out, align 8
   ret void
@@ -615,6 +1020,19 @@ define amdgpu_kernel void @i1_arg_sext_i32(i32 addrspace(1)* %out, i1 %x) nounwi
   ; HSA-VI:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[LOAD1]](s1)
   ; HSA-VI:   G_STORE [[SEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: i1_arg_sext_i32
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s1) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 1, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[LOAD1]](s1)
+  ; LEGACY-MESA-VI:   G_STORE [[SEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
   %ext = sext i1 %x to i32
   store i32 %ext, i32addrspace(1)* %out, align 4
   ret void
@@ -634,17 +1052,73 @@ define amdgpu_kernel void @i1_arg_sext_i64(i64 addrspace(1)* %out, i1 %x) nounwi
   ; HSA-VI:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[LOAD1]](s1)
   ; HSA-VI:   G_STORE [[SEXT]](s64), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: i1_arg_sext_i64
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s1) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 1, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[LOAD1]](s1)
+  ; LEGACY-MESA-VI:   G_STORE [[SEXT]](s64), [[LOAD]](p1) :: (store 8 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
   %ext = sext i1 %x to i64
   store i64 %ext, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-define amdgpu_kernel void @empty_struct_arg({} %in) nounwind {
+; 0-sized arguments do not add a slot to the argument register set, so
+; waste an index in the virtual register array.
+define amdgpu_kernel void @empty_struct_arg({} %arg0, i32 %arg1) nounwind {
   ; HSA-VI-LABEL: name: empty_struct_arg
   ; HSA-VI: bb.1 (%ir-block.0):
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
+  ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; HSA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 4, align 16, addrspace 4)
+  ; HSA-VI:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; HSA-VI:   G_STORE [[LOAD]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: empty_struct_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  store i32 %arg1, i32 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @empty_array_arg([0 x i8] %arg0, i32 %arg1) nounwind {
+  ; HSA-VI-LABEL: name: empty_array_arg
+  ; HSA-VI: bb.1 (%ir-block.0):
+  ; HSA-VI:   liveins: $sgpr4_sgpr5
+  ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
+  ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; HSA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 4, align 16, addrspace 4)
+  ; HSA-VI:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; HSA-VI:   G_STORE [[LOAD]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: empty_array_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  store i32 %arg1, i32 addrspace(1)* undef
   ret void
 }
 
@@ -656,20 +1130,9 @@ define amdgpu_kernel void @empty_struct_arg({} %in) nounwind {
 
 ; With the SelectionDAG argument lowering, the alignments for the
 ; struct members is not properly considered, making these wrong.
-
-; FIXME: GlobalISel extractvalue emission broken
-
-define amdgpu_kernel void @struct_argument_alignment({i32, i64} %arg0, i8, {i32, i64} %arg1) {
-  ; %val0 = extractvalue {i32, i64} %arg0, 0
-  ; %val1 = extractvalue {i32, i64} %arg0, 1
-  ; %val2 = extractvalue {i32, i64} %arg1, 0
-  ; %val3 = extractvalue {i32, i64} %arg1, 1
-  ; store volatile i32 %val0, i32 addrspace(1)* null
-  ; store volatile i64 %val1, i64 addrspace(1)* null
-  ; store volatile i32 %val2, i32 addrspace(1)* null
-  ; store volatile i64 %val3, i64 addrspace(1)* null
+define amdgpu_kernel void @struct_argument_alignment({i32, i64} %arg0, i8 %pad, {i32, i64} %arg1) {
   ; HSA-VI-LABEL: name: struct_argument_alignment
-  ; HSA-VI: bb.1 (%ir-block.1):
+  ; HSA-VI: bb.1 (%ir-block.0):
   ; HSA-VI:   liveins: $sgpr4_sgpr5
   ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
   ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
@@ -685,21 +1148,56 @@ define amdgpu_kernel void @struct_argument_alignment({i32, i64} %arg0, i8, {i32,
   ; HSA-VI:   [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD2]](p4) :: (dereferenceable invariant load 16, align 8, addrspace 4)
   ; HSA-VI:   [[EXTRACT2:%[0-9]+]]:_(s32) = G_EXTRACT [[LOAD2]](s128), 0
   ; HSA-VI:   [[EXTRACT3:%[0-9]+]]:_(s64) = G_EXTRACT [[LOAD2]](s128), 64
+  ; HSA-VI:   [[C3:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+  ; HSA-VI:   [[COPY1:%[0-9]+]]:_(p1) = COPY [[C3]](p1)
+  ; HSA-VI:   [[COPY2:%[0-9]+]]:_(p1) = COPY [[C3]](p1)
+  ; HSA-VI:   G_STORE [[EXTRACT]](s32), [[C3]](p1) :: (volatile store 4 into `i32 addrspace(1)* null`, addrspace 1)
+  ; HSA-VI:   G_STORE [[EXTRACT1]](s64), [[COPY1]](p1) :: (volatile store 8 into `i64 addrspace(1)* null`, addrspace 1)
+  ; HSA-VI:   G_STORE [[LOAD1]](s8), [[COPY2]](p1) :: (volatile store 1 into `i8 addrspace(1)* null`, addrspace 1)
+  ; HSA-VI:   G_STORE [[EXTRACT2]](s32), [[C3]](p1) :: (volatile store 4 into `i32 addrspace(1)* null`, addrspace 1)
+  ; HSA-VI:   G_STORE [[EXTRACT3]](s64), [[COPY1]](p1) :: (volatile store 8 into `i64 addrspace(1)* null`, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: struct_argument_alignment
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 16, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[LOAD]](s128), 0
+  ; LEGACY-MESA-VI:   [[EXTRACT1:%[0-9]+]]:_(s64) = G_EXTRACT [[LOAD]](s128), 64
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 52
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 1, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 60
+  ; LEGACY-MESA-VI:   [[PTR_ADD2:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C2]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD2]](p4) :: (dereferenceable invariant load 16, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[EXTRACT2:%[0-9]+]]:_(s32) = G_EXTRACT [[LOAD2]](s128), 0
+  ; LEGACY-MESA-VI:   [[EXTRACT3:%[0-9]+]]:_(s64) = G_EXTRACT [[LOAD2]](s128), 64
+  ; LEGACY-MESA-VI:   [[C3:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+  ; LEGACY-MESA-VI:   [[COPY1:%[0-9]+]]:_(p1) = COPY [[C3]](p1)
+  ; LEGACY-MESA-VI:   [[COPY2:%[0-9]+]]:_(p1) = COPY [[C3]](p1)
+  ; LEGACY-MESA-VI:   G_STORE [[EXTRACT]](s32), [[C3]](p1) :: (volatile store 4 into `i32 addrspace(1)* null`, addrspace 1)
+  ; LEGACY-MESA-VI:   G_STORE [[EXTRACT1]](s64), [[COPY1]](p1) :: (volatile store 8 into `i64 addrspace(1)* null`, addrspace 1)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](s8), [[COPY2]](p1) :: (volatile store 1 into `i8 addrspace(1)* null`, addrspace 1)
+  ; LEGACY-MESA-VI:   G_STORE [[EXTRACT2]](s32), [[C3]](p1) :: (volatile store 4 into `i32 addrspace(1)* null`, addrspace 1)
+  ; LEGACY-MESA-VI:   G_STORE [[EXTRACT3]](s64), [[COPY1]](p1) :: (volatile store 8 into `i64 addrspace(1)* null`, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  %val0 = extractvalue {i32, i64} %arg0, 0
+  %val1 = extractvalue {i32, i64} %arg0, 1
+  %val2 = extractvalue {i32, i64} %arg1, 0
+  %val3 = extractvalue {i32, i64} %arg1, 1
+  store volatile i32 %val0, i32 addrspace(1)* null
+  store volatile i64 %val1, i64 addrspace(1)* null
+  store volatile i8 %pad, i8 addrspace(1)* null
+  store volatile i32 %val2, i32 addrspace(1)* null
+  store volatile i64 %val3, i64 addrspace(1)* null
   ret void
 }
 
 ; No padding between i8 and next struct, but round up at end to 4 byte
 ; multiple.
 define amdgpu_kernel void @packed_struct_argument_alignment(<{i32, i64}> %arg0, i8, <{i32, i64}> %arg1) {
-  ; %val0 = extractvalue <{i32, i64}> %arg0, 0
-  ; %val1 = extractvalue <{i32, i64}> %arg0, 1
-  ; %val2 = extractvalue <{i32, i64}> %arg1, 0
-  ; %val3 = extractvalue <{i32, i64}> %arg1, 1
-  ; store volatile i32 %val0, i32 addrspace(1)* null
-  ; store volatile i64 %val1, i64 addrspace(1)* null
-  ; store volatile i32 %val2, i32 addrspace(1)* null
-  ; store volatile i64 %val3, i64 addrspace(1)* null
   ; HSA-VI-LABEL: name: packed_struct_argument_alignment
   ; HSA-VI: bb.1 (%ir-block.1):
   ; HSA-VI:   liveins: $sgpr4_sgpr5
@@ -709,14 +1207,61 @@ define amdgpu_kernel void @packed_struct_argument_alignment(<{i32, i64}> %arg0,
   ; HSA-VI:   [[LOAD:%[0-9]+]]:_(s96) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 12, align 16, addrspace 4)
   ; HSA-VI:   [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[LOAD]](s96), 0
   ; HSA-VI:   [[EXTRACT1:%[0-9]+]]:_(s64) = G_EXTRACT [[LOAD]](s96), 32
-  ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
+  ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 13
   ; HSA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
-  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 1, align 4, addrspace 4)
-  ; HSA-VI:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 13
-  ; HSA-VI:   [[PTR_ADD2:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C2]](s64)
-  ; HSA-VI:   [[LOAD2:%[0-9]+]]:_(s96) = G_LOAD [[PTR_ADD2]](p4) :: (dereferenceable invariant load 12, align 1, addrspace 4)
-  ; HSA-VI:   [[EXTRACT2:%[0-9]+]]:_(s32) = G_EXTRACT [[LOAD2]](s96), 0
-  ; HSA-VI:   [[EXTRACT3:%[0-9]+]]:_(s64) = G_EXTRACT [[LOAD2]](s96), 32
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s96) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 12, align 1, addrspace 4)
+  ; HSA-VI:   [[EXTRACT2:%[0-9]+]]:_(s32) = G_EXTRACT [[LOAD1]](s96), 0
+  ; HSA-VI:   [[EXTRACT3:%[0-9]+]]:_(s64) = G_EXTRACT [[LOAD1]](s96), 32
+  ; HSA-VI:   [[C2:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+  ; HSA-VI:   [[COPY1:%[0-9]+]]:_(p1) = COPY [[C2]](p1)
+  ; HSA-VI:   G_STORE [[EXTRACT]](s32), [[C2]](p1) :: (volatile store 4 into `i32 addrspace(1)* null`, addrspace 1)
+  ; HSA-VI:   G_STORE [[EXTRACT1]](s64), [[COPY1]](p1) :: (volatile store 8 into `i64 addrspace(1)* null`, addrspace 1)
+  ; HSA-VI:   G_STORE [[EXTRACT2]](s32), [[C2]](p1) :: (volatile store 4 into `i32 addrspace(1)* null`, addrspace 1)
+  ; HSA-VI:   G_STORE [[EXTRACT3]](s64), [[COPY1]](p1) :: (volatile store 8 into `i64 addrspace(1)* null`, addrspace 1)
   ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: packed_struct_argument_alignment
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.1):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(s96) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 12, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[LOAD]](s96), 0
+  ; LEGACY-MESA-VI:   [[EXTRACT1:%[0-9]+]]:_(s64) = G_EXTRACT [[LOAD]](s96), 32
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s96) = G_LOAD [[PTR_ADD1]](p4) :: (dereferenceable invariant load 12, align 1, addrspace 4)
+  ; LEGACY-MESA-VI:   [[EXTRACT2:%[0-9]+]]:_(s32) = G_EXTRACT [[LOAD1]](s96), 0
+  ; LEGACY-MESA-VI:   [[EXTRACT3:%[0-9]+]]:_(s64) = G_EXTRACT [[LOAD1]](s96), 32
+  ; LEGACY-MESA-VI:   [[C2:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+  ; LEGACY-MESA-VI:   [[COPY1:%[0-9]+]]:_(p1) = COPY [[C2]](p1)
+  ; LEGACY-MESA-VI:   G_STORE [[EXTRACT]](s32), [[C2]](p1) :: (volatile store 4 into `i32 addrspace(1)* null`, addrspace 1)
+  ; LEGACY-MESA-VI:   G_STORE [[EXTRACT1]](s64), [[COPY1]](p1) :: (volatile store 8 into `i64 addrspace(1)* null`, addrspace 1)
+  ; LEGACY-MESA-VI:   G_STORE [[EXTRACT2]](s32), [[C2]](p1) :: (volatile store 4 into `i32 addrspace(1)* null`, addrspace 1)
+  ; LEGACY-MESA-VI:   G_STORE [[EXTRACT3]](s64), [[COPY1]](p1) :: (volatile store 8 into `i64 addrspace(1)* null`, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  %val0 = extractvalue <{i32, i64}> %arg0, 0
+  %val1 = extractvalue <{i32, i64}> %arg0, 1
+  %val2 = extractvalue <{i32, i64}> %arg1, 0
+  %val3 = extractvalue <{i32, i64}> %arg1, 1
+  store volatile i32 %val0, i32 addrspace(1)* null
+  store volatile i64 %val1, i64 addrspace(1)* null
+  store volatile i32 %val2, i32 addrspace(1)* null
+  store volatile i64 %val3, i64 addrspace(1)* null
+  ret void
+}
+
+define amdgpu_kernel void @unused_i32_arg(i32 addrspace(1)* nocapture %out, i32 %unused, i32 %in) nounwind {
+  ; HSA-VI-LABEL: name: unused_i32_arg
+  ; HSA-VI: bb.1.entry:
+  ; HSA-VI:   liveins: $sgpr4_sgpr5
+  ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
+  ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: unused_i32_arg
+  ; LEGACY-MESA-VI: bb.1.entry:
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+entry:
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-fixed-function-abi-vgpr-args.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-fixed-function-abi-vgpr-args.ll
new file mode 100644
index 0000000000000..3c4bdab14e495
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-fixed-function-abi-vgpr-args.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -amdgpu-fixed-function-abi=1 -stop-after=irtranslator -o - %s | FileCheck -check-prefixes=COMMON,FIXED %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -amdgpu-fixed-function-abi=0 -stop-after=irtranslator -o - %s | FileCheck -check-prefixes=COMMON,VARABI %s
+
+; Make sure arg1 is not allocated in v31, which is reserved for
+; workitem IDs with -amdgpu-fixed-function-abi.
+
+define void @void_a31i32_i32([31 x i32] %arg0, i32 %arg1) {
+  ; FIXED-LABEL: name: void_a31i32_i32
+  ; FIXED: bb.1 (%ir-block.0):
+  ; FIXED:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $sgpr30_sgpr31
+  ; FIXED:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; FIXED:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; FIXED:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; FIXED:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; FIXED:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; FIXED:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; FIXED:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; FIXED:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; FIXED:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; FIXED:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; FIXED:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; FIXED:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; FIXED:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; FIXED:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; FIXED:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; FIXED:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; FIXED:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; FIXED:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; FIXED:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; FIXED:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; FIXED:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; FIXED:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; FIXED:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; FIXED:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; FIXED:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; FIXED:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; FIXED:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; FIXED:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; FIXED:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; FIXED:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; FIXED:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; FIXED:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
+  ; FIXED:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 4 from %fixed-stack.0, align 16, addrspace 5)
+  ; FIXED:   [[COPY31:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; FIXED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; FIXED:   G_STORE [[LOAD]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; FIXED:   [[COPY32:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY31]]
+  ; FIXED:   S_SETPC_B64_return [[COPY32]]
+  ; VARABI-LABEL: name: void_a31i32_i32
+  ; VARABI: bb.1 (%ir-block.0):
+  ; VARABI:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31, $sgpr30_sgpr31
+  ; VARABI:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; VARABI:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; VARABI:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; VARABI:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; VARABI:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; VARABI:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; VARABI:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; VARABI:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; VARABI:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; VARABI:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; VARABI:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; VARABI:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; VARABI:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; VARABI:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; VARABI:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; VARABI:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; VARABI:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; VARABI:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; VARABI:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; VARABI:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; VARABI:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; VARABI:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; VARABI:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; VARABI:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; VARABI:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; VARABI:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; VARABI:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; VARABI:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; VARABI:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; VARABI:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; VARABI:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; VARABI:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; VARABI:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; VARABI:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; VARABI:   G_STORE [[COPY31]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; VARABI:   [[COPY33:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
+  ; VARABI:   S_SETPC_B64_return [[COPY33]]
+  store i32 %arg1, i32 addrspace(1)* undef
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
index f7e6fd5cd4746..340392ea8d464 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
@@ -2,6 +2,34 @@
 ; RUN: llc -march=amdgcn -mcpu=fiji -O0 -stop-after=irtranslator -global-isel -verify-machineinstrs -o - %s | FileCheck %s
 ; FIXME: pre-VI should have same ABI without legal i16 operations.
 
+define void @void_func_empty_arg({} %arg0, i32 %arg1) #0 {
+  ; CHECK-LABEL: name: void_func_empty_arg
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[COPY]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  store i32 %arg1, i32 addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_empty_array([0 x i8] %arg0, i32 %arg1) #0 {
+  ; CHECK-LABEL: name: void_func_empty_array
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[COPY]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  store i32 %arg1, i32 addrspace(1)* undef
+  ret void
+}
+
 define void @void_func_i1(i1 %arg0) #0 {
   ; CHECK-LABEL: name: void_func_i1
   ; CHECK: bb.1 (%ir-block.0):
@@ -95,10 +123,11 @@ define void @void_func_i8(i8 %arg0) #0 {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC]](s16)
   ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
-  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
   ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
-  ; CHECK:   G_STORE [[TRUNC]](s8), [[DEF]](p1) :: (store 1 into `i8 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[TRUNC1]](s8), [[DEF]](p1) :: (store 1 into `i8 addrspace(1)* undef`, addrspace 1)
   ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
   ; CHECK:   S_SETPC_B64_return [[COPY2]]
   store i8 %arg0, i8 addrspace(1)* undef
@@ -110,8 +139,8 @@ define void @void_func_i8_zeroext(i8 zeroext %arg0) #0 {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
   ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; CHECK:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s8)
@@ -130,8 +159,8 @@ define void @void_func_i8_signext(i8 signext %arg0) #0 {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
   ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; CHECK:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[TRUNC]](s8)
@@ -205,8 +234,8 @@ define void @void_func_i24(i24 %arg0) #0 {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[TRUNC:%[0-9]+]]:_(s24) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; CHECK:   G_STORE [[TRUNC]](s24), [[DEF]](p1) :: (store 3 into `i24 addrspace(1)* undef`, align 4, addrspace 1)
   ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
@@ -220,8 +249,8 @@ define void @void_func_i24_zeroext(i24 zeroext %arg0) #0 {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[TRUNC:%[0-9]+]]:_(s24) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; CHECK:   G_STORE [[TRUNC]](s24), [[DEF]](p1) :: (store 3 into `i24 addrspace(1)* undef`, align 4, addrspace 1)
   ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
@@ -235,8 +264,8 @@ define void @void_func_i24_signext(i24 signext %arg0) #0 {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[TRUNC:%[0-9]+]]:_(s24) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; CHECK:   G_STORE [[TRUNC]](s24), [[DEF]](p1) :: (store 3 into `i24 addrspace(1)* undef`, align 4, addrspace 1)
   ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
@@ -1541,11 +1570,12 @@ define void @void_func_struct_i8_i32({ i8, i32 } %arg0) #0 {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC]](s16)
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
-  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
   ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
-  ; CHECK:   G_STORE [[TRUNC]](s8), [[DEF]](p1) :: (store 1 into `{ i8, i32 } addrspace(1)* undef`, align 4, addrspace 1)
+  ; CHECK:   G_STORE [[TRUNC1]](s8), [[DEF]](p1) :: (store 1 into `{ i8, i32 } addrspace(1)* undef`, align 4, addrspace 1)
   ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
   ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[DEF]], [[C]](s64)
   ; CHECK:   G_STORE [[COPY1]](s32), [[PTR_ADD]](p1) :: (store 4 into `{ i8, i32 } addrspace(1)* undef` + 4, addrspace 1)
@@ -1738,14 +1768,13 @@ define void @void_func_v32i32_i1_i8_i16(<32 x i32> %arg0, i1 %arg1, i8 %arg2, i1
   ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.3
   ; CHECK:   [[LOAD:%[0-9]+]]:_(s1) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 1 from %fixed-stack.3, align 16, addrspace 5)
   ; CHECK:   [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.2
-  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p5) :: (invariant load 4 from %fixed-stack.2, addrspace 5)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX1]](p5) :: (invariant load 1 from %fixed-stack.2, align 4, addrspace 5)
   ; CHECK:   [[FRAME_INDEX2:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.1
   ; CHECK:   [[LOAD2:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX2]](p5) :: (invariant load 2 from %fixed-stack.1, align 8, addrspace 5)
   ; CHECK:   [[FRAME_INDEX3:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
   ; CHECK:   [[LOAD3:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX3]](p5) :: (invariant load 2 from %fixed-stack.0, align 4, addrspace 5)
   ; CHECK:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32)
-  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[LOAD1]](s32)
   ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; CHECK:   [[COPY33:%[0-9]+]]:_(p1) = COPY [[DEF]](p1)
   ; CHECK:   [[COPY34:%[0-9]+]]:_(p1) = COPY [[DEF]](p1)
@@ -1753,7 +1782,7 @@ define void @void_func_v32i32_i1_i8_i16(<32 x i32> %arg0, i1 %arg1, i8 %arg2, i1
   ; CHECK:   [[COPY36:%[0-9]+]]:_(p1) = COPY [[DEF]](p1)
   ; CHECK:   G_STORE [[BUILD_VECTOR]](<32 x s32>), [[DEF]](p1) :: (volatile store 128 into `<32 x i32> addrspace(1)* undef`, addrspace 1)
   ; CHECK:   G_STORE [[LOAD]](s1), [[COPY33]](p1) :: (volatile store 1 into `i1 addrspace(1)* undef`, addrspace 1)
-  ; CHECK:   G_STORE [[TRUNC]](s8), [[COPY34]](p1) :: (volatile store 1 into `i8 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[LOAD1]](s8), [[COPY34]](p1) :: (volatile store 1 into `i8 addrspace(1)* undef`, addrspace 1)
   ; CHECK:   G_STORE [[LOAD2]](s16), [[COPY35]](p1) :: (volatile store 2 into `i16 addrspace(1)* undef`, addrspace 1)
   ; CHECK:   G_STORE [[LOAD3]](s16), [[COPY36]](p1) :: (volatile store 2 into `half addrspace(1)* undef`, addrspace 1)
   ; CHECK:   [[COPY37:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll
index 15320c9155efa..7dc247de3687e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll
@@ -234,4 +234,96 @@ define i32 @test_memory_constraint(i32 addrspace(3)* %a) nounwind {
   ret i32 %1
 }
 
+define i32 @test_vgpr_matching_constraint(i32 %a) nounwind {
+  ; CHECK-LABEL: name: test_vgpr_matching_constraint
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK:   [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[AND]](s32)
+  ; CHECK:   INLINEASM &";", 1 /* sideeffect attdialect */, 1835018 /* regdef:VGPR_32 */, def %4, 2147483657 /* reguse tiedto:$0 */, [[COPY2]](tied-def 3)
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY %4
+  ; CHECK:   $vgpr0 = COPY [[COPY3]](s32)
+  ; CHECK:   [[COPY4:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY4]], implicit $vgpr0
+  %and = and i32 %a, 1
+  %asm = call i32 asm sideeffect ";", "=v,0"(i32 %and)
+  ret i32 %asm
+}
+
+define i32 @test_sgpr_matching_constraint() nounwind {
+  ; CHECK-LABEL: name: test_sgpr_matching_constraint
+  ; CHECK: bb.1.entry:
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   INLINEASM &"s_mov_b32 $0, 7", 0 /* attdialect */, 1966090 /* regdef:SReg_32 */, def %1
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY %1
+  ; CHECK:   INLINEASM &"s_mov_b32 $0, 8", 0 /* attdialect */, 1966090 /* regdef:SReg_32 */, def %3
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY %3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY [[COPY1]](s32)
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY [[COPY2]](s32)
+  ; CHECK:   INLINEASM &"s_add_u32 $0, $1, $2", 0 /* attdialect */, 1966090 /* regdef:SReg_32 */, def %5, 9 /* reguse */, [[COPY3]], 2147483657 /* reguse tiedto:$0 */, [[COPY4]](tied-def 3)
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY %5
+  ; CHECK:   $vgpr0 = COPY [[COPY5]](s32)
+  ; CHECK:   [[COPY6:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY6]], implicit $vgpr0
+entry:
+  %asm0 = tail call i32 asm "s_mov_b32 $0, 7", "=s"() nounwind
+  %asm1 = tail call i32 asm "s_mov_b32 $0, 8", "=s"() nounwind
+  %asm2 = tail call i32 asm "s_add_u32 $0, $1, $2", "=s,s,0"(i32 %asm0, i32 %asm1) nounwind
+  ret i32 %asm2
+}
+
+define void @test_many_matching_constraints(i32 %a, i32 %b, i32 %c) nounwind {
+  ; CHECK-LABEL: name: test_many_matching_constraints
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY]](s32)
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY1]](s32)
+  ; CHECK:   INLINEASM &"; ", 1 /* sideeffect attdialect */, 1835018 /* regdef:VGPR_32 */, def %4, 1835018 /* regdef:VGPR_32 */, def %5, 1835018 /* regdef:VGPR_32 */, def %6, 2147483657 /* reguse tiedto:$0 */, [[COPY4]](tied-def 3), 2147614729 /* reguse tiedto:$2 */, [[COPY5]](tied-def 7), 2147549193 /* reguse tiedto:$1 */, [[COPY6]](tied-def 5)
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY %4
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY %5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY %6
+  ; CHECK:   G_STORE [[COPY7]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[COPY8]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[COPY9]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY10:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY3]]
+  ; CHECK:   S_SETPC_B64_return [[COPY10]]
+  %asm = call {i32, i32, i32} asm sideeffect "; ", "=v,=v,=v,0,2,1"(i32 %c, i32 %a, i32 %b)
+  %asmresult0 = extractvalue  {i32, i32, i32} %asm, 0
+  store i32 %asmresult0, i32 addrspace(1)* undef
+  %asmresult1 = extractvalue  {i32, i32, i32} %asm, 1
+  store i32 %asmresult1, i32 addrspace(1)* undef
+  %asmresult2 = extractvalue  {i32, i32, i32} %asm, 2
+  store i32 %asmresult2, i32 addrspace(1)* undef
+  ret void
+}
+
+define i32 @test_sgpr_to_vgpr_move_matching_constraint() nounwind {
+  ; CHECK-LABEL: name: test_sgpr_to_vgpr_move_matching_constraint
+  ; CHECK: bb.1.entry:
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   INLINEASM &"s_mov_b32 $0, 7", 0 /* attdialect */, 1966090 /* regdef:SReg_32 */, def %1
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY %1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]](s32)
+  ; CHECK:   INLINEASM &"v_mov_b32 $0, $1", 0 /* attdialect */, 1835018 /* regdef:VGPR_32 */, def %3, 2147483657 /* reguse tiedto:$0 */, [[COPY2]](tied-def 3)
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY %3
+  ; CHECK:   $vgpr0 = COPY [[COPY3]](s32)
+  ; CHECK:   [[COPY4:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY4]], implicit $vgpr0
+entry:
+  %asm0 = tail call i32 asm "s_mov_b32 $0, 7", "=s"() nounwind
+  %asm1 = tail call i32 asm "v_mov_b32 $0, $1", "=v,0"(i32 %asm0) nounwind
+  ret i32 %asm1
+}
+
 !0 = !{i32 70}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-ptrmask.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-ptrmask.ll
index cc1c75e404e05..4f56e1ae598da 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-ptrmask.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-ptrmask.ll
@@ -48,9 +48,9 @@ define i8* @ptrmask_flat_i16(i8* %ptr, i16 %mask) {
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
   ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
-  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
   ; CHECK:   [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[MV]], [[TRUNC]](s16)
   ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[PTRMASK]](p0)
   ; CHECK:   $vgpr0 = COPY [[UV]](s32)
@@ -119,8 +119,8 @@ define i8 addrspace(3)* @ptrmask_local_i16(i8 addrspace(3)* %ptr, i16 %mask) {
   ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[PTRMASK:%[0-9]+]]:_(p3) = G_PTRMASK [[COPY]], [[TRUNC]](s16)
   ; CHECK:   $vgpr0 = COPY [[PTRMASK]](p3)
   ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sat.ll
index 2d648580408f7..efdecf7fb49fe 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sat.ll
@@ -6,10 +6,10 @@ define i16 @uaddsat_i16(i16 %lhs, i16 %rhs) {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC]], [[TRUNC1]]
   ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UADDSAT]](s16)
   ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
@@ -85,10 +85,10 @@ define i16 @saddsat_i16(i16 %lhs, i16 %rhs) {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[TRUNC]], [[TRUNC1]]
   ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SADDSAT]](s16)
   ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
@@ -164,10 +164,10 @@ define i16 @usubsat_i16(i16 %lhs, i16 %rhs) {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC]], [[TRUNC1]]
   ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[USUBSAT]](s16)
   ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
@@ -243,10 +243,10 @@ define i16 @ssubsat_i16(i16 %lhs, i16 %rhs) {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[TRUNC]], [[TRUNC1]]
   ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SSUBSAT]](s16)
   ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir
index 2963458c6dd9e..c4d511ddda3ca 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir
@@ -1652,7 +1652,7 @@ body: |
     %1:_(s32) = COPY $vgpr8
     %2:_(s256) = G_ZEXT %1
     %3:_(s256) = G_ASHR %0, %2
-    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7  = COPY %3
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %3
 ...
 
 ---
@@ -1784,5 +1784,5 @@ body: |
     %0:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     %1:_(<2 x s32>) = COPY $vgpr4_vgpr5
     %2:_(<2 x s128>) = G_ASHR %0, %1
-    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7  = COPY %2
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %2
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir
index b85f3ad94c637..3233c2742d884 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir
@@ -4008,43 +4008,102 @@ body: |
 
     ; CI-LABEL: name: test_load_constant_v2s8_align4
     ; CI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; CI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p4) :: (load 2, align 4, addrspace 4)
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; CI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; CI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; CI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p4) :: (load 2, align 4, addrspace 4)
+    ; CI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; CI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; CI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; CI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; CI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; CI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; CI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; CI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; CI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; VI-LABEL: name: test_load_constant_v2s8_align4
     ; VI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; VI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p4) :: (load 2, align 4, addrspace 4)
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; VI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; VI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p4) :: (load 2, align 4, addrspace 4)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; VI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; VI: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: test_load_constant_v2s8_align4
     ; GFX9: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; GFX9: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p4) :: (load 2, align 4, addrspace 4)
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; GFX9: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p4) :: (load 2, align 4, addrspace 4)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     ; CI-MESA-LABEL: name: test_load_constant_v2s8_align4
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; CI-MESA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p4) :: (load 2, align 4, addrspace 4)
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; CI-MESA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; CI-MESA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; CI-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p4) :: (load 2, align 4, addrspace 4)
+    ; CI-MESA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI-MESA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; CI-MESA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI-MESA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; CI-MESA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CI-MESA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; CI-MESA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CI-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; CI-MESA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; CI-MESA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CI-MESA: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CI-MESA: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CI-MESA: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CI-MESA: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; CI-MESA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CI-MESA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; CI-MESA: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-MESA-LABEL: name: test_load_constant_v2s8_align4
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p4) :: (load 2, align 4, addrspace 4)
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; GFX9-MESA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; GFX9-MESA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p4) :: (load 2, align 4, addrspace 4)
+    ; GFX9-MESA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9-MESA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX9-MESA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9-MESA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; GFX9-MESA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9-MESA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; GFX9-MESA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX9-MESA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; GFX9-MESA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9-MESA: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9-MESA: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9-MESA: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9-MESA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9-MESA: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(p4) = COPY $vgpr0_vgpr1
     %1:_(<2 x s8>) = G_LOAD %0 :: (load 2, align 4, addrspace 4)
@@ -4061,43 +4120,102 @@ body: |
 
     ; CI-LABEL: name: test_load_constant_v2s8_align2
     ; CI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; CI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p4) :: (load 2, addrspace 4)
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; CI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; CI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; CI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p4) :: (load 2, addrspace 4)
+    ; CI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; CI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; CI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; CI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; CI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; CI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; CI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; CI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; CI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; VI-LABEL: name: test_load_constant_v2s8_align2
     ; VI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; VI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p4) :: (load 2, addrspace 4)
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; VI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; VI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p4) :: (load 2, addrspace 4)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; VI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; VI: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: test_load_constant_v2s8_align2
     ; GFX9: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; GFX9: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p4) :: (load 2, addrspace 4)
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; GFX9: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p4) :: (load 2, addrspace 4)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     ; CI-MESA-LABEL: name: test_load_constant_v2s8_align2
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; CI-MESA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p4) :: (load 2, addrspace 4)
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; CI-MESA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; CI-MESA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; CI-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p4) :: (load 2, addrspace 4)
+    ; CI-MESA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI-MESA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; CI-MESA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI-MESA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; CI-MESA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CI-MESA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; CI-MESA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CI-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; CI-MESA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; CI-MESA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CI-MESA: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CI-MESA: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CI-MESA: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CI-MESA: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; CI-MESA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CI-MESA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; CI-MESA: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-MESA-LABEL: name: test_load_constant_v2s8_align2
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p4) :: (load 2, addrspace 4)
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; GFX9-MESA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; GFX9-MESA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p4) :: (load 2, addrspace 4)
+    ; GFX9-MESA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9-MESA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX9-MESA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9-MESA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; GFX9-MESA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9-MESA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; GFX9-MESA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX9-MESA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; GFX9-MESA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9-MESA: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9-MESA: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9-MESA: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9-MESA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9-MESA: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(p4) = COPY $vgpr0_vgpr1
     %1:_(<2 x s8>) = G_LOAD %0 :: (load 2, align 2, addrspace 4)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
index aaa665e9984bd..9dbbede334ac5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
@@ -4028,43 +4028,102 @@ body: |
 
     ; CI-LABEL: name: test_load_flat_v2s8_align4
     ; CI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; CI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p0) :: (load 2, align 4)
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; CI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; CI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; CI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 2, align 4)
+    ; CI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; CI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; CI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; CI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; CI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; CI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; CI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; CI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; CI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; VI-LABEL: name: test_load_flat_v2s8_align4
     ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; VI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p0) :: (load 2, align 4)
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; VI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; VI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 2, align 4)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; VI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; VI: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: test_load_flat_v2s8_align4
     ; GFX9: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p0) :: (load 2, align 4)
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; GFX9: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 2, align 4)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     ; CI-MESA-LABEL: name: test_load_flat_v2s8_align4
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; CI-MESA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p0) :: (load 2, align 4)
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; CI-MESA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; CI-MESA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; CI-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 2, align 4)
+    ; CI-MESA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI-MESA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; CI-MESA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI-MESA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; CI-MESA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CI-MESA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; CI-MESA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CI-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; CI-MESA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; CI-MESA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CI-MESA: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CI-MESA: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CI-MESA: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CI-MESA: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; CI-MESA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CI-MESA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; CI-MESA: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-MESA-LABEL: name: test_load_flat_v2s8_align4
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p0) :: (load 2, align 4)
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; GFX9-MESA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; GFX9-MESA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 2, align 4)
+    ; GFX9-MESA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9-MESA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX9-MESA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9-MESA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; GFX9-MESA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9-MESA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; GFX9-MESA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX9-MESA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; GFX9-MESA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9-MESA: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9-MESA: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9-MESA: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9-MESA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9-MESA: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(p0) = COPY $vgpr0_vgpr1
     %1:_(<2 x s8>) = G_LOAD %0 :: (load 2, align 4, addrspace 0)
@@ -4081,43 +4140,102 @@ body: |
 
     ; CI-LABEL: name: test_load_flat_v2s8_align2
     ; CI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; CI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p0) :: (load 2)
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; CI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; CI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; CI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 2)
+    ; CI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; CI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; CI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; CI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; CI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; CI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; CI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; CI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; CI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; VI-LABEL: name: test_load_flat_v2s8_align2
     ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; VI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p0) :: (load 2)
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; VI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; VI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 2)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; VI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; VI: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: test_load_flat_v2s8_align2
     ; GFX9: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p0) :: (load 2)
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; GFX9: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 2)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     ; CI-MESA-LABEL: name: test_load_flat_v2s8_align2
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; CI-MESA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p0) :: (load 2)
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; CI-MESA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; CI-MESA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; CI-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 2)
+    ; CI-MESA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI-MESA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; CI-MESA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI-MESA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; CI-MESA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CI-MESA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; CI-MESA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CI-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; CI-MESA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; CI-MESA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CI-MESA: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CI-MESA: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CI-MESA: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CI-MESA: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; CI-MESA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CI-MESA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; CI-MESA: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-MESA-LABEL: name: test_load_flat_v2s8_align2
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p0) :: (load 2)
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; GFX9-MESA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; GFX9-MESA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 2)
+    ; GFX9-MESA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9-MESA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX9-MESA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9-MESA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; GFX9-MESA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9-MESA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; GFX9-MESA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX9-MESA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; GFX9-MESA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9-MESA: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9-MESA: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9-MESA: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9-MESA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9-MESA: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(p0) = COPY $vgpr0_vgpr1
     %1:_(<2 x s8>) = G_LOAD %0 :: (load 2, align 2, addrspace 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir
index d16e9631e7657..275a4ec274c09 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir
@@ -3745,51 +3745,123 @@ body: |
 
     ; SI-LABEL: name: test_load_global_v2s8_align4
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; SI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p1) :: (load 2, align 4, addrspace 1)
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; SI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; SI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 2, align 4, addrspace 1)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; SI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; CI-HSA-LABEL: name: test_load_global_v2s8_align4
     ; CI-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CI-HSA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p1) :: (load 2, align 4, addrspace 1)
-    ; CI-HSA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; CI-HSA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; CI-HSA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; CI-HSA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; CI-HSA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 2, align 4, addrspace 1)
+    ; CI-HSA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI-HSA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; CI-HSA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI-HSA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; CI-HSA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CI-HSA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; CI-HSA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CI-HSA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; CI-HSA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; CI-HSA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CI-HSA: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CI-HSA: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CI-HSA: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CI-HSA: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; CI-HSA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CI-HSA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; CI-HSA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; CI-HSA: $vgpr0 = COPY [[ANYEXT]](s32)
     ; CI-MESA-LABEL: name: test_load_global_v2s8_align4
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CI-MESA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p1) :: (load 2, align 4, addrspace 1)
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; CI-MESA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; CI-MESA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; CI-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 2, align 4, addrspace 1)
+    ; CI-MESA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI-MESA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; CI-MESA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI-MESA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; CI-MESA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CI-MESA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; CI-MESA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CI-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; CI-MESA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; CI-MESA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CI-MESA: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CI-MESA: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CI-MESA: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CI-MESA: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; CI-MESA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CI-MESA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; CI-MESA: $vgpr0 = COPY [[ANYEXT]](s32)
     ; VI-LABEL: name: test_load_global_v2s8_align4
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; VI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p1) :: (load 2, align 4, addrspace 1)
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; VI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; VI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 2, align 4, addrspace 1)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; VI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; VI: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-HSA-LABEL: name: test_load_global_v2s8_align4
     ; GFX9-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX9-HSA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p1) :: (load 2, align 4, addrspace 1)
-    ; GFX9-HSA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; GFX9-HSA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; GFX9-HSA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; GFX9-HSA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; GFX9-HSA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 2, align 4, addrspace 1)
+    ; GFX9-HSA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9-HSA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX9-HSA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9-HSA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; GFX9-HSA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9-HSA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; GFX9-HSA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9-HSA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX9-HSA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; GFX9-HSA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9-HSA: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9-HSA: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9-HSA: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9-HSA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX9-HSA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9-HSA: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-MESA-LABEL: name: test_load_global_v2s8_align4
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p1) :: (load 2, align 4, addrspace 1)
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; GFX9-MESA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; GFX9-MESA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 2, align 4, addrspace 1)
+    ; GFX9-MESA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9-MESA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX9-MESA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9-MESA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; GFX9-MESA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9-MESA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; GFX9-MESA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX9-MESA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; GFX9-MESA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9-MESA: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9-MESA: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9-MESA: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9-MESA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9-MESA: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<2 x s8>) = G_LOAD %0 :: (load 2, align 4, addrspace 1)
@@ -3806,51 +3878,123 @@ body: |
 
     ; SI-LABEL: name: test_load_global_v2s8_align2
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; SI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p1) :: (load 2, addrspace 1)
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; SI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; SI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 2, addrspace 1)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; SI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; CI-HSA-LABEL: name: test_load_global_v2s8_align2
     ; CI-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CI-HSA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p1) :: (load 2, addrspace 1)
-    ; CI-HSA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; CI-HSA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; CI-HSA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; CI-HSA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; CI-HSA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 2, addrspace 1)
+    ; CI-HSA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI-HSA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; CI-HSA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI-HSA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; CI-HSA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CI-HSA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; CI-HSA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CI-HSA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; CI-HSA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; CI-HSA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CI-HSA: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CI-HSA: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CI-HSA: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CI-HSA: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; CI-HSA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CI-HSA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; CI-HSA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; CI-HSA: $vgpr0 = COPY [[ANYEXT]](s32)
     ; CI-MESA-LABEL: name: test_load_global_v2s8_align2
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CI-MESA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p1) :: (load 2, addrspace 1)
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; CI-MESA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; CI-MESA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; CI-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 2, addrspace 1)
+    ; CI-MESA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI-MESA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; CI-MESA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI-MESA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; CI-MESA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CI-MESA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; CI-MESA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CI-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; CI-MESA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; CI-MESA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CI-MESA: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CI-MESA: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CI-MESA: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CI-MESA: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; CI-MESA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CI-MESA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; CI-MESA: $vgpr0 = COPY [[ANYEXT]](s32)
     ; VI-LABEL: name: test_load_global_v2s8_align2
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; VI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p1) :: (load 2, addrspace 1)
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; VI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; VI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 2, addrspace 1)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; VI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; VI: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-HSA-LABEL: name: test_load_global_v2s8_align2
     ; GFX9-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX9-HSA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p1) :: (load 2, addrspace 1)
-    ; GFX9-HSA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; GFX9-HSA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; GFX9-HSA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; GFX9-HSA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; GFX9-HSA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 2, addrspace 1)
+    ; GFX9-HSA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9-HSA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX9-HSA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9-HSA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; GFX9-HSA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9-HSA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; GFX9-HSA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9-HSA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX9-HSA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; GFX9-HSA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9-HSA: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9-HSA: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9-HSA: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9-HSA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX9-HSA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9-HSA: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-MESA-LABEL: name: test_load_global_v2s8_align2
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p1) :: (load 2, addrspace 1)
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; GFX9-MESA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; GFX9-MESA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 2, addrspace 1)
+    ; GFX9-MESA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9-MESA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX9-MESA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9-MESA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; GFX9-MESA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9-MESA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; GFX9-MESA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX9-MESA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; GFX9-MESA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9-MESA: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9-MESA: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9-MESA: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9-MESA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9-MESA: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<2 x s8>) = G_LOAD %0 :: (load 2, align 2, addrspace 1)
@@ -3885,11 +4029,24 @@ body: |
     ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; CI-HSA-LABEL: name: test_load_global_v2s8_align1
     ; CI-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CI-HSA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p1) :: (load 2, align 1, addrspace 1)
-    ; CI-HSA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; CI-HSA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; CI-HSA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; CI-HSA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; CI-HSA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 2, align 1, addrspace 1)
+    ; CI-HSA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI-HSA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; CI-HSA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI-HSA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; CI-HSA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CI-HSA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; CI-HSA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CI-HSA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; CI-HSA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; CI-HSA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CI-HSA: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CI-HSA: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CI-HSA: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CI-HSA: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; CI-HSA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CI-HSA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; CI-HSA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; CI-HSA: $vgpr0 = COPY [[ANYEXT]](s32)
     ; CI-MESA-LABEL: name: test_load_global_v2s8_align1
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
@@ -3927,11 +4084,22 @@ body: |
     ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-HSA-LABEL: name: test_load_global_v2s8_align1
     ; GFX9-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX9-HSA: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p1) :: (load 2, align 1, addrspace 1)
-    ; GFX9-HSA: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; GFX9-HSA: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; GFX9-HSA: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; GFX9-HSA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; GFX9-HSA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 2, align 1, addrspace 1)
+    ; GFX9-HSA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9-HSA: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX9-HSA: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9-HSA: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; GFX9-HSA: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9-HSA: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; GFX9-HSA: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9-HSA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX9-HSA: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; GFX9-HSA: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9-HSA: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9-HSA: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9-HSA: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9-HSA: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX9-HSA: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9-HSA: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-MESA-LABEL: name: test_load_global_v2s8_align1
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
index 001bfc0339db5..a465c05d921aa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
@@ -4909,43 +4909,104 @@ body: |
 
     ; SI-LABEL: name: test_load_local_v2s8_align2
     ; SI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; SI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p3) :: (load 2, addrspace 3)
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; SI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; SI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load 2, addrspace 3)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; SI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; CI-LABEL: name: test_load_local_v2s8_align2
     ; CI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; CI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p3) :: (load 2, addrspace 3)
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; CI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; CI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; CI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load 2, addrspace 3)
+    ; CI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; CI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; CI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; CI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; CI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; CI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; CI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; CI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; CI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; CI-DS128-LABEL: name: test_load_local_v2s8_align2
     ; CI-DS128: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; CI-DS128: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p3) :: (load 2, addrspace 3)
-    ; CI-DS128: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; CI-DS128: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; CI-DS128: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; CI-DS128: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; CI-DS128: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load 2, addrspace 3)
+    ; CI-DS128: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI-DS128: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; CI-DS128: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI-DS128: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; CI-DS128: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CI-DS128: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; CI-DS128: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CI-DS128: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; CI-DS128: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; CI-DS128: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CI-DS128: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CI-DS128: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CI-DS128: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CI-DS128: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; CI-DS128: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CI-DS128: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; CI-DS128: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; CI-DS128: $vgpr0 = COPY [[ANYEXT]](s32)
     ; VI-LABEL: name: test_load_local_v2s8_align2
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p3) :: (load 2, addrspace 3)
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; VI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; VI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load 2, addrspace 3)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; VI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; VI: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: test_load_local_v2s8_align2
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p3) :: (load 2, addrspace 3)
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; GFX9: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load 2, addrspace 3)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(p3) = COPY $vgpr0
     %1:_(<2 x s8>) = G_LOAD %0 :: (load 2, align 2, addrspace 3)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
index 5a598e2a6ab7a..9c43f7ae5453e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
@@ -4069,35 +4069,83 @@ body: |
 
     ; SI-LABEL: name: test_load_private_v2s8_align2
     ; SI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; SI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p5) :: (load 2, addrspace 5)
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; SI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; SI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load 2, addrspace 5)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; SI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; CI-LABEL: name: test_load_private_v2s8_align2
     ; CI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; CI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p5) :: (load 2, addrspace 5)
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; CI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; CI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; CI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load 2, addrspace 5)
+    ; CI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; CI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; CI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; CI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; CI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; CI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; CI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; CI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; CI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; VI-LABEL: name: test_load_private_v2s8_align2
     ; VI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p5) :: (load 2, addrspace 5)
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; VI: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; VI: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load 2, addrspace 5)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; VI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; VI: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: test_load_private_v2s8_align2
     ; GFX9: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p5) :: (load 2, addrspace 5)
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<2 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<2 x s8>)
-    ; GFX9: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV]](s8), [[UV1]](s8)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load 2, addrspace 5)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C2]](s32)
+    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(p5) = COPY $vgpr0
     %1:_(<2 x s8>) = G_LOAD %0 :: (load 2, align 2, addrspace 5)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir
index 4057c21296f55..f5f7baeba73a4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir
@@ -1488,7 +1488,7 @@ body: |
     %1:_(s32) = COPY $vgpr8
     %2:_(s256) = G_ZEXT %1
     %3:_(s256) = G_LSHR %0, %2
-    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7  = COPY %3
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %3
 ...
 
 ---
@@ -1614,5 +1614,5 @@ body: |
     %0:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     %1:_(<2 x s32>) = COPY $vgpr4_vgpr5
     %2:_(<2 x s128>) = G_LSHR %0, %1
-    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7  = COPY %2
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %2
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sdiv.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sdiv.mir
index 8a569c68b1f37..8290a38bcb892 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sdiv.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sdiv.mir
@@ -21,34 +21,30 @@ body: |
     ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX6: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
-    ; GFX6: $vgpr0 = COPY [[SUB4]](s32)
+    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX6: $vgpr0 = COPY [[SUB3]](s32)
     ; GFX8-LABEL: name: test_sdiv_s32
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
@@ -61,34 +57,30 @@ body: |
     ; GFX8: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX8: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX8: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
-    ; GFX8: $vgpr0 = COPY [[SUB4]](s32)
+    ; GFX8: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX8: $vgpr0 = COPY [[SUB3]](s32)
     ; GFX9-LABEL: name: test_sdiv_s32
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
@@ -101,34 +93,30 @@ body: |
     ; GFX9: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX9: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX9: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
-    ; GFX9: $vgpr0 = COPY [[SUB4]](s32)
+    ; GFX9: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX9: $vgpr0 = COPY [[SUB3]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_SDIV %0, %1
@@ -155,67 +143,59 @@ body: |
     ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX6: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
     ; GFX6: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[UV1]], [[C]](s32)
     ; GFX6: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[UV3]], [[C]](s32)
-    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[ASHR2]]
-    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UV3]], [[ASHR3]]
-    ; GFX6: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR2]]
-    ; GFX6: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR3]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[ASHR2]]
+    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[UV3]], [[ASHR3]]
+    ; GFX6: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR2]]
+    ; GFX6: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[ADD6]], [[ASHR3]]
     ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[XOR5]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C4]]
     ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[XOR5]]
-    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[XOR5]]
-    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL2]]
-    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C2]]
-    ; GFX6: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB5]], [[MUL2]]
-    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX6: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD6]], [[SUB6]]
-    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[XOR4]]
-    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[XOR5]]
-    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR4]], [[MUL3]]
-    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB7]](s32), [[XOR5]]
-    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR4]](s32), [[MUL3]]
-    ; GFX6: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[UMULH5]], [[C3]]
-    ; GFX6: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[UMULH5]], [[C3]]
-    ; GFX6: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[ADD7]], [[UMULH5]]
-    ; GFX6: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[SUB8]]
+    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR5]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB4]], [[FPTOUI1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[XOR4]], [[ADD7]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[XOR5]]
+    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[XOR4]], [[MUL3]]
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB5]](s32), [[XOR5]]
+    ; GFX6: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[UMULH3]], [[C3]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[ADD8]], [[UMULH3]]
+    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB5]], [[XOR5]]
+    ; GFX6: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB6]], [[SUB5]]
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT4]](s32), [[XOR5]]
+    ; GFX6: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[SELECT3]], [[C3]]
+    ; GFX6: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD9]], [[SELECT3]]
     ; GFX6: [[XOR6:%[0-9]+]]:_(s32) = G_XOR [[ASHR2]], [[ASHR3]]
-    ; GFX6: [[XOR7:%[0-9]+]]:_(s32) = G_XOR [[SELECT7]], [[XOR6]]
-    ; GFX6: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[XOR7]], [[XOR6]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB4]](s32), [[SUB9]](s32)
+    ; GFX6: [[XOR7:%[0-9]+]]:_(s32) = G_XOR [[SELECT5]], [[XOR6]]
+    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR7]], [[XOR6]]
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB3]](s32), [[SUB7]](s32)
     ; GFX6: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; GFX8-LABEL: name: test_sdiv_v2s32
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
@@ -231,67 +211,59 @@ body: |
     ; GFX8: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX8: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX8: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX8: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
     ; GFX8: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[UV1]], [[C]](s32)
     ; GFX8: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[UV3]], [[C]](s32)
-    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[ASHR2]]
-    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UV3]], [[ASHR3]]
-    ; GFX8: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR2]]
-    ; GFX8: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR3]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[ASHR2]]
+    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[UV3]], [[ASHR3]]
+    ; GFX8: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR2]]
+    ; GFX8: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[ADD6]], [[ASHR3]]
     ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[XOR5]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C4]]
     ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[XOR5]]
-    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[XOR5]]
-    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL2]]
-    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C2]]
-    ; GFX8: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB5]], [[MUL2]]
-    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX8: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX8: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD6]], [[SUB6]]
-    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[XOR4]]
-    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[XOR5]]
-    ; GFX8: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR4]], [[MUL3]]
-    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB7]](s32), [[XOR5]]
-    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR4]](s32), [[MUL3]]
-    ; GFX8: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[UMULH5]], [[C3]]
-    ; GFX8: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[UMULH5]], [[C3]]
-    ; GFX8: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[ADD7]], [[UMULH5]]
-    ; GFX8: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[SUB8]]
+    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR5]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB4]], [[FPTOUI1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[XOR4]], [[ADD7]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[XOR5]]
+    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[XOR4]], [[MUL3]]
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB5]](s32), [[XOR5]]
+    ; GFX8: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[UMULH3]], [[C3]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[ADD8]], [[UMULH3]]
+    ; GFX8: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB5]], [[XOR5]]
+    ; GFX8: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB6]], [[SUB5]]
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT4]](s32), [[XOR5]]
+    ; GFX8: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[SELECT3]], [[C3]]
+    ; GFX8: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD9]], [[SELECT3]]
     ; GFX8: [[XOR6:%[0-9]+]]:_(s32) = G_XOR [[ASHR2]], [[ASHR3]]
-    ; GFX8: [[XOR7:%[0-9]+]]:_(s32) = G_XOR [[SELECT7]], [[XOR6]]
-    ; GFX8: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[XOR7]], [[XOR6]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB4]](s32), [[SUB9]](s32)
+    ; GFX8: [[XOR7:%[0-9]+]]:_(s32) = G_XOR [[SELECT5]], [[XOR6]]
+    ; GFX8: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR7]], [[XOR6]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB3]](s32), [[SUB7]](s32)
     ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; GFX9-LABEL: name: test_sdiv_v2s32
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
@@ -307,67 +279,59 @@ body: |
     ; GFX9: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX9: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX9: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX9: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
     ; GFX9: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[UV1]], [[C]](s32)
     ; GFX9: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[UV3]], [[C]](s32)
-    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[ASHR2]]
-    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UV3]], [[ASHR3]]
-    ; GFX9: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR2]]
-    ; GFX9: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR3]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[ASHR2]]
+    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[UV3]], [[ASHR3]]
+    ; GFX9: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR2]]
+    ; GFX9: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[ADD6]], [[ASHR3]]
     ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[XOR5]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C4]]
     ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[XOR5]]
-    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[XOR5]]
-    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL2]]
-    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C2]]
-    ; GFX9: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB5]], [[MUL2]]
-    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX9: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD6]], [[SUB6]]
-    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[XOR4]]
-    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[XOR5]]
-    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR4]], [[MUL3]]
-    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB7]](s32), [[XOR5]]
-    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR4]](s32), [[MUL3]]
-    ; GFX9: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[UMULH5]], [[C3]]
-    ; GFX9: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[UMULH5]], [[C3]]
-    ; GFX9: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[ADD7]], [[UMULH5]]
-    ; GFX9: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[SUB8]]
+    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR5]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB4]], [[FPTOUI1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[XOR4]], [[ADD7]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[XOR5]]
+    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[XOR4]], [[MUL3]]
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB5]](s32), [[XOR5]]
+    ; GFX9: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[UMULH3]], [[C3]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[ADD8]], [[UMULH3]]
+    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB5]], [[XOR5]]
+    ; GFX9: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB6]], [[SUB5]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT4]](s32), [[XOR5]]
+    ; GFX9: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[SELECT3]], [[C3]]
+    ; GFX9: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD9]], [[SELECT3]]
     ; GFX9: [[XOR6:%[0-9]+]]:_(s32) = G_XOR [[ASHR2]], [[ASHR3]]
-    ; GFX9: [[XOR7:%[0-9]+]]:_(s32) = G_XOR [[SELECT7]], [[XOR6]]
-    ; GFX9: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[XOR7]], [[XOR6]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB4]](s32), [[SUB9]](s32)
+    ; GFX9: [[XOR7:%[0-9]+]]:_(s32) = G_XOR [[SELECT5]], [[XOR6]]
+    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR7]], [[XOR6]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB3]](s32), [[SUB7]](s32)
     ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
@@ -384,18 +348,528 @@ body: |
     ; GFX6-LABEL: name: test_sdiv_s64
     ; GFX6: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX6: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX6: [[SDIV:%[0-9]+]]:_(s64) = G_SDIV [[COPY]], [[COPY1]]
-    ; GFX6: $vgpr0_vgpr1 = COPY [[SDIV]](s64)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[C]](s32)
+    ; GFX6: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY1]], [[C]](s32)
+    ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX6: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX6: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX6: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX6: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV8]](s32)
+    ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV9]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX6: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX6: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX6: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX6: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX6: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[UV12]]
+    ; GFX6: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[UV13]], [[USUBO1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX6: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX6: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX6: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX6: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX6: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX6: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX6: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX6: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX6: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX6: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX6: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX6: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX6: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX6: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX6: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX6: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX6: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX6: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX6: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX6: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX6: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX6: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX6: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX6: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX6: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX6: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX6: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX6: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX6: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX6: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX6: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX6: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX6: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX6: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX6: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX6: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX6: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX6: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX6: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX6: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDO26]]
+    ; GFX6: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV16]], [[UADDE8]]
+    ; GFX6: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDO26]]
+    ; GFX6: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX6: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX6: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX6: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX6: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX6: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDE8]]
+    ; GFX6: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDO26]]
+    ; GFX6: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDE8]]
+    ; GFX6: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX6: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX6: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX6: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX6: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX6: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX6: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX6: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX6: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDE8]]
+    ; GFX6: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX6: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[ADD16]](s32)
+    ; GFX6: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[UADDO36]]
+    ; GFX6: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV19]], [[UADDO36]]
+    ; GFX6: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[ADD16]]
+    ; GFX6: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV18]], [[UADDO36]]
+    ; GFX6: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX6: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX6: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[MUL15]]
+    ; GFX6: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[ADD18]], [[USUBO3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV15]], [[ADD18]]
+    ; GFX6: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV21]]
+    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV20]]
+    ; GFX6: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV21]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX6: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV20]]
+    ; GFX6: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV21]], [[USUBO3]]
+    ; GFX6: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX6: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX6: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX6: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[UADDO36]], [[UV22]]
+    ; GFX6: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV23]], [[UADDO39]]
+    ; GFX6: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO38]](s32), [[UADDE10]](s32)
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV21]]
+    ; GFX6: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV20]]
+    ; GFX6: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV21]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX6: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX6: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UADDO38]], [[UV24]]
+    ; GFX6: [[UADDE12:%[0-9]+]]:_(s32), [[UADDE13:%[0-9]+]]:_(s1) = G_UADDE [[UADDE10]], [[UV25]], [[UADDO41]]
+    ; GFX6: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO40]](s32), [[UADDE12]](s32)
+    ; GFX6: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV20]]
+    ; GFX6: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV21]], [[USUBO5]]
+    ; GFX6: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX6: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX6: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX6: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[ASHR]], [[ASHR1]]
+    ; GFX6: [[XOR3:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[XOR2]]
+    ; GFX6: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR3]](s64)
+    ; GFX6: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX6: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV26]], [[UV28]]
+    ; GFX6: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV27]], [[UV29]], [[USUBO9]]
+    ; GFX6: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[MV5]](s64)
     ; GFX8-LABEL: name: test_sdiv_s64
     ; GFX8: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX8: [[SDIV:%[0-9]+]]:_(s64) = G_SDIV [[COPY]], [[COPY1]]
-    ; GFX8: $vgpr0_vgpr1 = COPY [[SDIV]](s64)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[C]](s32)
+    ; GFX8: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY1]], [[C]](s32)
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX8: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX8: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX8: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX8: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV8]](s32)
+    ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV9]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX8: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX8: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX8: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX8: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX8: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX8: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX8: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[UV12]]
+    ; GFX8: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[UV13]], [[USUBO1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX8: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX8: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX8: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX8: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX8: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX8: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX8: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX8: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX8: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX8: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX8: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX8: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX8: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX8: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX8: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX8: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX8: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX8: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX8: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX8: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX8: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX8: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX8: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX8: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX8: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX8: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX8: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX8: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX8: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX8: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX8: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX8: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX8: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX8: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX8: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX8: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX8: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX8: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDO26]]
+    ; GFX8: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV16]], [[UADDE8]]
+    ; GFX8: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDO26]]
+    ; GFX8: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX8: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX8: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX8: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX8: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX8: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDE8]]
+    ; GFX8: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDO26]]
+    ; GFX8: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDE8]]
+    ; GFX8: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX8: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX8: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX8: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX8: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX8: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX8: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX8: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX8: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDE8]]
+    ; GFX8: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX8: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[ADD16]](s32)
+    ; GFX8: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[UADDO36]]
+    ; GFX8: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV19]], [[UADDO36]]
+    ; GFX8: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[ADD16]]
+    ; GFX8: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV18]], [[UADDO36]]
+    ; GFX8: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX8: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX8: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[MUL15]]
+    ; GFX8: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[ADD18]], [[USUBO3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV15]], [[ADD18]]
+    ; GFX8: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV21]]
+    ; GFX8: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV20]]
+    ; GFX8: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV21]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX8: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV20]]
+    ; GFX8: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV21]], [[USUBO3]]
+    ; GFX8: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX8: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX8: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX8: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[UADDO36]], [[UV22]]
+    ; GFX8: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV23]], [[UADDO39]]
+    ; GFX8: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO38]](s32), [[UADDE10]](s32)
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV21]]
+    ; GFX8: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV20]]
+    ; GFX8: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV21]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX8: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX8: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UADDO38]], [[UV24]]
+    ; GFX8: [[UADDE12:%[0-9]+]]:_(s32), [[UADDE13:%[0-9]+]]:_(s1) = G_UADDE [[UADDE10]], [[UV25]], [[UADDO41]]
+    ; GFX8: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO40]](s32), [[UADDE12]](s32)
+    ; GFX8: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV20]]
+    ; GFX8: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV21]], [[USUBO5]]
+    ; GFX8: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX8: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX8: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX8: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[ASHR]], [[ASHR1]]
+    ; GFX8: [[XOR3:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[XOR2]]
+    ; GFX8: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR3]](s64)
+    ; GFX8: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX8: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV26]], [[UV28]]
+    ; GFX8: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV27]], [[UV29]], [[USUBO9]]
+    ; GFX8: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX8: $vgpr0_vgpr1 = COPY [[MV5]](s64)
     ; GFX9-LABEL: name: test_sdiv_s64
     ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX9: [[SDIV:%[0-9]+]]:_(s64) = G_SDIV [[COPY]], [[COPY1]]
-    ; GFX9: $vgpr0_vgpr1 = COPY [[SDIV]](s64)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[C]](s32)
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY1]], [[C]](s32)
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX9: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX9: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV8]](s32)
+    ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV9]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX9: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX9: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX9: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX9: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX9: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[UV12]]
+    ; GFX9: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[UV13]], [[USUBO1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX9: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX9: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX9: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX9: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX9: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX9: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX9: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX9: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX9: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX9: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX9: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX9: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX9: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX9: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX9: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX9: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX9: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX9: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX9: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX9: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX9: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX9: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX9: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX9: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX9: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX9: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX9: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX9: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX9: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX9: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX9: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX9: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX9: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX9: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX9: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDO26]]
+    ; GFX9: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV16]], [[UADDE8]]
+    ; GFX9: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDO26]]
+    ; GFX9: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX9: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX9: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX9: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX9: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX9: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDE8]]
+    ; GFX9: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDO26]]
+    ; GFX9: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDE8]]
+    ; GFX9: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX9: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX9: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX9: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX9: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX9: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX9: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX9: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX9: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDE8]]
+    ; GFX9: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX9: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[ADD16]](s32)
+    ; GFX9: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[UADDO36]]
+    ; GFX9: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV19]], [[UADDO36]]
+    ; GFX9: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[ADD16]]
+    ; GFX9: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV18]], [[UADDO36]]
+    ; GFX9: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX9: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX9: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[MUL15]]
+    ; GFX9: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[ADD18]], [[USUBO3]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV15]], [[ADD18]]
+    ; GFX9: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV21]]
+    ; GFX9: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV20]]
+    ; GFX9: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV21]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX9: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV20]]
+    ; GFX9: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV21]], [[USUBO3]]
+    ; GFX9: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX9: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX9: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX9: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[UADDO36]], [[UV22]]
+    ; GFX9: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV23]], [[UADDO39]]
+    ; GFX9: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO38]](s32), [[UADDE10]](s32)
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV21]]
+    ; GFX9: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV20]]
+    ; GFX9: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV21]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX9: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX9: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UADDO38]], [[UV24]]
+    ; GFX9: [[UADDE12:%[0-9]+]]:_(s32), [[UADDE13:%[0-9]+]]:_(s1) = G_UADDE [[UADDE10]], [[UV25]], [[UADDO41]]
+    ; GFX9: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO40]](s32), [[UADDE12]](s32)
+    ; GFX9: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV20]]
+    ; GFX9: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV21]], [[USUBO5]]
+    ; GFX9: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX9: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX9: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX9: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[ASHR]], [[ASHR1]]
+    ; GFX9: [[XOR3:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[XOR2]]
+    ; GFX9: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR3]](s64)
+    ; GFX9: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX9: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV26]], [[UV28]]
+    ; GFX9: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV27]], [[UV29]], [[USUBO9]]
+    ; GFX9: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[MV5]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s64) = G_SDIV %0, %1
@@ -413,27 +887,1035 @@ body: |
     ; GFX6: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX6: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX6: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX6: [[SDIV:%[0-9]+]]:_(s64) = G_SDIV [[UV]], [[UV2]]
-    ; GFX6: [[SDIV1:%[0-9]+]]:_(s64) = G_SDIV [[UV1]], [[UV3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SDIV]](s64), [[SDIV1]](s64)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV]], [[C]](s32)
+    ; GFX6: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV2]], [[C]](s32)
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX6: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX6: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]]
+    ; GFX6: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO3]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX6: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX6: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX6: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV12]](s32)
+    ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV13]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX6: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX6: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX6: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX6: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX6: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[UV16]]
+    ; GFX6: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[UV17]], [[USUBO1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX6: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX6: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX6: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX6: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX6: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX6: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX6: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX6: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX6: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX6: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX6: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX6: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX6: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX6: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX6: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX6: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX6: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX6: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX6: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX6: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX6: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX6: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX6: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX6: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX6: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX6: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX6: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX6: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX6: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX6: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX6: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX6: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX6: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX6: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX6: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX6: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX6: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX6: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX6: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX6: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV21]], [[UADDO26]]
+    ; GFX6: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV20]], [[UADDE8]]
+    ; GFX6: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV20]], [[UADDO26]]
+    ; GFX6: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX6: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX6: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX6: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX6: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX6: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV21]], [[UADDE8]]
+    ; GFX6: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV21]], [[UADDO26]]
+    ; GFX6: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV20]], [[UADDE8]]
+    ; GFX6: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX6: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX6: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX6: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX6: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX6: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX6: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX6: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX6: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV21]], [[UADDE8]]
+    ; GFX6: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX6: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[ADD16]](s32)
+    ; GFX6: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV22]], [[UADDO36]]
+    ; GFX6: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV23]], [[UADDO36]]
+    ; GFX6: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV22]], [[ADD16]]
+    ; GFX6: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV22]], [[UADDO36]]
+    ; GFX6: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX6: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX6: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV18]], [[MUL15]]
+    ; GFX6: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV19]], [[ADD18]], [[USUBO3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV19]], [[ADD18]]
+    ; GFX6: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV25]]
+    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV24]]
+    ; GFX6: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV25]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX6: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV24]]
+    ; GFX6: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV25]], [[USUBO3]]
+    ; GFX6: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX6: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX6: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX6: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[UADDO36]], [[UV26]]
+    ; GFX6: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV27]], [[UADDO39]]
+    ; GFX6: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO38]](s32), [[UADDE10]](s32)
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV25]]
+    ; GFX6: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV24]]
+    ; GFX6: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV25]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX6: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX6: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UADDO38]], [[UV28]]
+    ; GFX6: [[UADDE12:%[0-9]+]]:_(s32), [[UADDE13:%[0-9]+]]:_(s1) = G_UADDE [[UADDE10]], [[UV29]], [[UADDO41]]
+    ; GFX6: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO40]](s32), [[UADDE12]](s32)
+    ; GFX6: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV24]]
+    ; GFX6: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV25]], [[USUBO5]]
+    ; GFX6: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX6: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX6: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX6: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[ASHR]], [[ASHR1]]
+    ; GFX6: [[XOR3:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[XOR2]]
+    ; GFX6: [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR3]](s64)
+    ; GFX6: [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX6: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV30]], [[UV32]]
+    ; GFX6: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV31]], [[UV33]], [[USUBO9]]
+    ; GFX6: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX6: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; GFX6: [[ASHR3:%[0-9]+]]:_(s64) = G_ASHR [[UV3]], [[C]](s32)
+    ; GFX6: [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX6: [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR2]](s64)
+    ; GFX6: [[UADDO42:%[0-9]+]]:_(s32), [[UADDO43:%[0-9]+]]:_(s1) = G_UADDO [[UV34]], [[UV36]]
+    ; GFX6: [[UADDE14:%[0-9]+]]:_(s32), [[UADDE15:%[0-9]+]]:_(s1) = G_UADDE [[UV35]], [[UV37]], [[UADDO43]]
+    ; GFX6: [[MV6:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO42]](s32), [[UADDE14]](s32)
+    ; GFX6: [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX6: [[UV40:%[0-9]+]]:_(s32), [[UV41:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR3]](s64)
+    ; GFX6: [[UADDO44:%[0-9]+]]:_(s32), [[UADDO45:%[0-9]+]]:_(s1) = G_UADDO [[UV38]], [[UV40]]
+    ; GFX6: [[UADDE16:%[0-9]+]]:_(s32), [[UADDE17:%[0-9]+]]:_(s1) = G_UADDE [[UV39]], [[UV41]], [[UADDO45]]
+    ; GFX6: [[MV7:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO44]](s32), [[UADDE16]](s32)
+    ; GFX6: [[XOR4:%[0-9]+]]:_(s64) = G_XOR [[MV6]], [[ASHR2]]
+    ; GFX6: [[XOR5:%[0-9]+]]:_(s64) = G_XOR [[MV7]], [[ASHR3]]
+    ; GFX6: [[UV42:%[0-9]+]]:_(s32), [[UV43:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR5]](s64)
+    ; GFX6: [[UITOFP2:%[0-9]+]]:_(s32) = G_UITOFP [[UV42]](s32)
+    ; GFX6: [[UITOFP3:%[0-9]+]]:_(s32) = G_UITOFP [[UV43]](s32)
+    ; GFX6: [[C8:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[FMUL4:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP3]], [[C8]]
+    ; GFX6: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL4]], [[UITOFP2]]
+    ; GFX6: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD2]](s32)
+    ; GFX6: [[C9:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX6: [[FMUL5:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C9]]
+    ; GFX6: [[C10:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX6: [[FMUL6:%[0-9]+]]:_(s32) = G_FMUL [[FMUL5]], [[C10]]
+    ; GFX6: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL6]]
+    ; GFX6: [[C11:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX6: [[FMUL7:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC1]], [[C11]]
+    ; GFX6: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL7]], [[FMUL5]]
+    ; GFX6: [[FPTOUI2:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD3]](s32)
+    ; GFX6: [[FPTOUI3:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC1]](s32)
+    ; GFX6: [[UV44:%[0-9]+]]:_(s32), [[UV45:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX6: [[UV46:%[0-9]+]]:_(s32), [[UV47:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR5]](s64)
+    ; GFX6: [[USUBO10:%[0-9]+]]:_(s32), [[USUBO11:%[0-9]+]]:_(s1) = G_USUBO [[UV44]], [[UV46]]
+    ; GFX6: [[USUBE14:%[0-9]+]]:_(s32), [[USUBE15:%[0-9]+]]:_(s1) = G_USUBE [[UV45]], [[UV47]], [[USUBO11]]
+    ; GFX6: [[MUL18:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[FPTOUI2]]
+    ; GFX6: [[MUL19:%[0-9]+]]:_(s32) = G_MUL [[USUBE14]], [[FPTOUI2]]
+    ; GFX6: [[MUL20:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[FPTOUI3]]
+    ; GFX6: [[UMULH15:%[0-9]+]]:_(s32) = G_UMULH [[USUBO10]], [[FPTOUI2]]
+    ; GFX6: [[ADD19:%[0-9]+]]:_(s32) = G_ADD [[MUL19]], [[MUL20]]
+    ; GFX6: [[ADD20:%[0-9]+]]:_(s32) = G_ADD [[ADD19]], [[UMULH15]]
+    ; GFX6: [[MUL21:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[MUL18]]
+    ; GFX6: [[MUL22:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI2]], [[ADD20]]
+    ; GFX6: [[UMULH16:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[MUL18]]
+    ; GFX6: [[UADDO46:%[0-9]+]]:_(s32), [[UADDO47:%[0-9]+]]:_(s1) = G_UADDO [[MUL21]], [[MUL22]]
+    ; GFX6: [[ZEXT15:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO47]](s1)
+    ; GFX6: [[UADDO48:%[0-9]+]]:_(s32), [[UADDO49:%[0-9]+]]:_(s1) = G_UADDO [[UADDO46]], [[UMULH16]]
+    ; GFX6: [[ZEXT16:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO49]](s1)
+    ; GFX6: [[ADD21:%[0-9]+]]:_(s32) = G_ADD [[ZEXT15]], [[ZEXT16]]
+    ; GFX6: [[MUL23:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[ADD20]]
+    ; GFX6: [[UMULH17:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[MUL18]]
+    ; GFX6: [[UMULH18:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[ADD20]]
+    ; GFX6: [[UADDO50:%[0-9]+]]:_(s32), [[UADDO51:%[0-9]+]]:_(s1) = G_UADDO [[MUL23]], [[UMULH17]]
+    ; GFX6: [[ZEXT17:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO51]](s1)
+    ; GFX6: [[UADDO52:%[0-9]+]]:_(s32), [[UADDO53:%[0-9]+]]:_(s1) = G_UADDO [[UADDO50]], [[UMULH18]]
+    ; GFX6: [[ZEXT18:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO53]](s1)
+    ; GFX6: [[ADD22:%[0-9]+]]:_(s32) = G_ADD [[ZEXT17]], [[ZEXT18]]
+    ; GFX6: [[UADDO54:%[0-9]+]]:_(s32), [[UADDO55:%[0-9]+]]:_(s1) = G_UADDO [[UADDO52]], [[ADD21]]
+    ; GFX6: [[ZEXT19:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO55]](s1)
+    ; GFX6: [[ADD23:%[0-9]+]]:_(s32) = G_ADD [[ADD22]], [[ZEXT19]]
+    ; GFX6: [[UMULH19:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[ADD20]]
+    ; GFX6: [[ADD24:%[0-9]+]]:_(s32) = G_ADD [[UMULH19]], [[ADD23]]
+    ; GFX6: [[UADDO56:%[0-9]+]]:_(s32), [[UADDO57:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI2]], [[UADDO54]]
+    ; GFX6: [[UADDE18:%[0-9]+]]:_(s32), [[UADDE19:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI3]], [[ADD24]], [[UADDO57]]
+    ; GFX6: [[ADD25:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI3]], [[ADD24]]
+    ; GFX6: [[MUL24:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[UADDO56]]
+    ; GFX6: [[MUL25:%[0-9]+]]:_(s32) = G_MUL [[USUBE14]], [[UADDO56]]
+    ; GFX6: [[MUL26:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[UADDE18]]
+    ; GFX6: [[UMULH20:%[0-9]+]]:_(s32) = G_UMULH [[USUBO10]], [[UADDO56]]
+    ; GFX6: [[ADD26:%[0-9]+]]:_(s32) = G_ADD [[MUL25]], [[MUL26]]
+    ; GFX6: [[ADD27:%[0-9]+]]:_(s32) = G_ADD [[ADD26]], [[UMULH20]]
+    ; GFX6: [[MUL27:%[0-9]+]]:_(s32) = G_MUL [[UADDE18]], [[MUL24]]
+    ; GFX6: [[MUL28:%[0-9]+]]:_(s32) = G_MUL [[UADDO56]], [[ADD27]]
+    ; GFX6: [[UMULH21:%[0-9]+]]:_(s32) = G_UMULH [[UADDO56]], [[MUL24]]
+    ; GFX6: [[UADDO58:%[0-9]+]]:_(s32), [[UADDO59:%[0-9]+]]:_(s1) = G_UADDO [[MUL27]], [[MUL28]]
+    ; GFX6: [[ZEXT20:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO59]](s1)
+    ; GFX6: [[UADDO60:%[0-9]+]]:_(s32), [[UADDO61:%[0-9]+]]:_(s1) = G_UADDO [[UADDO58]], [[UMULH21]]
+    ; GFX6: [[ZEXT21:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO61]](s1)
+    ; GFX6: [[ADD28:%[0-9]+]]:_(s32) = G_ADD [[ZEXT20]], [[ZEXT21]]
+    ; GFX6: [[MUL29:%[0-9]+]]:_(s32) = G_MUL [[UADDE18]], [[ADD27]]
+    ; GFX6: [[UMULH22:%[0-9]+]]:_(s32) = G_UMULH [[UADDE18]], [[MUL24]]
+    ; GFX6: [[UMULH23:%[0-9]+]]:_(s32) = G_UMULH [[UADDO56]], [[ADD27]]
+    ; GFX6: [[UADDO62:%[0-9]+]]:_(s32), [[UADDO63:%[0-9]+]]:_(s1) = G_UADDO [[MUL29]], [[UMULH22]]
+    ; GFX6: [[ZEXT22:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO63]](s1)
+    ; GFX6: [[UADDO64:%[0-9]+]]:_(s32), [[UADDO65:%[0-9]+]]:_(s1) = G_UADDO [[UADDO62]], [[UMULH23]]
+    ; GFX6: [[ZEXT23:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO65]](s1)
+    ; GFX6: [[ADD29:%[0-9]+]]:_(s32) = G_ADD [[ZEXT22]], [[ZEXT23]]
+    ; GFX6: [[UADDO66:%[0-9]+]]:_(s32), [[UADDO67:%[0-9]+]]:_(s1) = G_UADDO [[UADDO64]], [[ADD28]]
+    ; GFX6: [[ZEXT24:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO67]](s1)
+    ; GFX6: [[ADD30:%[0-9]+]]:_(s32) = G_ADD [[ADD29]], [[ZEXT24]]
+    ; GFX6: [[UMULH24:%[0-9]+]]:_(s32) = G_UMULH [[UADDE18]], [[ADD27]]
+    ; GFX6: [[ADD31:%[0-9]+]]:_(s32) = G_ADD [[UMULH24]], [[ADD30]]
+    ; GFX6: [[UADDO68:%[0-9]+]]:_(s32), [[UADDO69:%[0-9]+]]:_(s1) = G_UADDO [[UADDO56]], [[UADDO66]]
+    ; GFX6: [[UADDE20:%[0-9]+]]:_(s32), [[UADDE21:%[0-9]+]]:_(s1) = G_UADDE [[ADD25]], [[ADD31]], [[UADDO57]]
+    ; GFX6: [[UADDE22:%[0-9]+]]:_(s32), [[UADDE23:%[0-9]+]]:_(s1) = G_UADDE [[UADDE20]], [[C6]], [[UADDO69]]
+    ; GFX6: [[UV48:%[0-9]+]]:_(s32), [[UV49:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX6: [[UV50:%[0-9]+]]:_(s32), [[UV51:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX6: [[MUL30:%[0-9]+]]:_(s32) = G_MUL [[UV51]], [[UADDO68]]
+    ; GFX6: [[MUL31:%[0-9]+]]:_(s32) = G_MUL [[UV50]], [[UADDE22]]
+    ; GFX6: [[UMULH25:%[0-9]+]]:_(s32) = G_UMULH [[UV50]], [[UADDO68]]
+    ; GFX6: [[UADDO70:%[0-9]+]]:_(s32), [[UADDO71:%[0-9]+]]:_(s1) = G_UADDO [[MUL30]], [[MUL31]]
+    ; GFX6: [[ZEXT25:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO71]](s1)
+    ; GFX6: [[UADDO72:%[0-9]+]]:_(s32), [[UADDO73:%[0-9]+]]:_(s1) = G_UADDO [[UADDO70]], [[UMULH25]]
+    ; GFX6: [[ZEXT26:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO73]](s1)
+    ; GFX6: [[ADD32:%[0-9]+]]:_(s32) = G_ADD [[ZEXT25]], [[ZEXT26]]
+    ; GFX6: [[MUL32:%[0-9]+]]:_(s32) = G_MUL [[UV51]], [[UADDE22]]
+    ; GFX6: [[UMULH26:%[0-9]+]]:_(s32) = G_UMULH [[UV51]], [[UADDO68]]
+    ; GFX6: [[UMULH27:%[0-9]+]]:_(s32) = G_UMULH [[UV50]], [[UADDE22]]
+    ; GFX6: [[UADDO74:%[0-9]+]]:_(s32), [[UADDO75:%[0-9]+]]:_(s1) = G_UADDO [[MUL32]], [[UMULH26]]
+    ; GFX6: [[ZEXT27:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO75]](s1)
+    ; GFX6: [[UADDO76:%[0-9]+]]:_(s32), [[UADDO77:%[0-9]+]]:_(s1) = G_UADDO [[UADDO74]], [[UMULH27]]
+    ; GFX6: [[ZEXT28:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO77]](s1)
+    ; GFX6: [[ADD33:%[0-9]+]]:_(s32) = G_ADD [[ZEXT27]], [[ZEXT28]]
+    ; GFX6: [[UADDO78:%[0-9]+]]:_(s32), [[UADDO79:%[0-9]+]]:_(s1) = G_UADDO [[UADDO76]], [[ADD32]]
+    ; GFX6: [[ZEXT29:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO79]](s1)
+    ; GFX6: [[ADD34:%[0-9]+]]:_(s32) = G_ADD [[ADD33]], [[ZEXT29]]
+    ; GFX6: [[UMULH28:%[0-9]+]]:_(s32) = G_UMULH [[UV51]], [[UADDE22]]
+    ; GFX6: [[ADD35:%[0-9]+]]:_(s32) = G_ADD [[UMULH28]], [[ADD34]]
+    ; GFX6: [[MV8:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO78]](s32), [[ADD35]](s32)
+    ; GFX6: [[UV52:%[0-9]+]]:_(s32), [[UV53:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR5]](s64)
+    ; GFX6: [[MUL33:%[0-9]+]]:_(s32) = G_MUL [[UV52]], [[UADDO78]]
+    ; GFX6: [[MUL34:%[0-9]+]]:_(s32) = G_MUL [[UV53]], [[UADDO78]]
+    ; GFX6: [[MUL35:%[0-9]+]]:_(s32) = G_MUL [[UV52]], [[ADD35]]
+    ; GFX6: [[UMULH29:%[0-9]+]]:_(s32) = G_UMULH [[UV52]], [[UADDO78]]
+    ; GFX6: [[ADD36:%[0-9]+]]:_(s32) = G_ADD [[MUL34]], [[MUL35]]
+    ; GFX6: [[ADD37:%[0-9]+]]:_(s32) = G_ADD [[ADD36]], [[UMULH29]]
+    ; GFX6: [[USUBO12:%[0-9]+]]:_(s32), [[USUBO13:%[0-9]+]]:_(s1) = G_USUBO [[UV48]], [[MUL33]]
+    ; GFX6: [[USUBE16:%[0-9]+]]:_(s32), [[USUBE17:%[0-9]+]]:_(s1) = G_USUBE [[UV49]], [[ADD37]], [[USUBO13]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV49]], [[ADD37]]
+    ; GFX6: [[UV54:%[0-9]+]]:_(s32), [[UV55:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR5]](s64)
+    ; GFX6: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE16]](s32), [[UV55]]
+    ; GFX6: [[SEXT4:%[0-9]+]]:_(s32) = G_SEXT [[ICMP8]](s1)
+    ; GFX6: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO12]](s32), [[UV54]]
+    ; GFX6: [[SEXT5:%[0-9]+]]:_(s32) = G_SEXT [[ICMP9]](s1)
+    ; GFX6: [[ICMP10:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE16]](s32), [[UV55]]
+    ; GFX6: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP10]](s1), [[SEXT5]], [[SEXT4]]
+    ; GFX6: [[USUBO14:%[0-9]+]]:_(s32), [[USUBO15:%[0-9]+]]:_(s1) = G_USUBO [[USUBO12]], [[UV54]]
+    ; GFX6: [[USUBE18:%[0-9]+]]:_(s32), [[USUBE19:%[0-9]+]]:_(s1) = G_USUBE [[SUB1]], [[UV55]], [[USUBO13]]
+    ; GFX6: [[USUBE20:%[0-9]+]]:_(s32), [[USUBE21:%[0-9]+]]:_(s1) = G_USUBE [[USUBE18]], [[C6]], [[USUBO15]]
+    ; GFX6: [[UV56:%[0-9]+]]:_(s32), [[UV57:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX6: [[UADDO80:%[0-9]+]]:_(s32), [[UADDO81:%[0-9]+]]:_(s1) = G_UADDO [[UADDO78]], [[UV56]]
+    ; GFX6: [[UADDE24:%[0-9]+]]:_(s32), [[UADDE25:%[0-9]+]]:_(s1) = G_UADDE [[ADD35]], [[UV57]], [[UADDO81]]
+    ; GFX6: [[MV9:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO80]](s32), [[UADDE24]](s32)
+    ; GFX6: [[ICMP11:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE20]](s32), [[UV55]]
+    ; GFX6: [[SEXT6:%[0-9]+]]:_(s32) = G_SEXT [[ICMP11]](s1)
+    ; GFX6: [[ICMP12:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO14]](s32), [[UV54]]
+    ; GFX6: [[SEXT7:%[0-9]+]]:_(s32) = G_SEXT [[ICMP12]](s1)
+    ; GFX6: [[ICMP13:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE20]](s32), [[UV55]]
+    ; GFX6: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP13]](s1), [[SEXT7]], [[SEXT6]]
+    ; GFX6: [[UV58:%[0-9]+]]:_(s32), [[UV59:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX6: [[UADDO82:%[0-9]+]]:_(s32), [[UADDO83:%[0-9]+]]:_(s1) = G_UADDO [[UADDO80]], [[UV58]]
+    ; GFX6: [[UADDE26:%[0-9]+]]:_(s32), [[UADDE27:%[0-9]+]]:_(s1) = G_UADDE [[UADDE24]], [[UV59]], [[UADDO83]]
+    ; GFX6: [[MV10:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO82]](s32), [[UADDE26]](s32)
+    ; GFX6: [[USUBO16:%[0-9]+]]:_(s32), [[USUBO17:%[0-9]+]]:_(s1) = G_USUBO [[USUBO14]], [[UV54]]
+    ; GFX6: [[USUBE22:%[0-9]+]]:_(s32), [[USUBE23:%[0-9]+]]:_(s1) = G_USUBE [[USUBE18]], [[UV55]], [[USUBO15]]
+    ; GFX6: [[USUBE24:%[0-9]+]]:_(s32), [[USUBE25:%[0-9]+]]:_(s1) = G_USUBE [[USUBE22]], [[C6]], [[USUBO17]]
+    ; GFX6: [[ICMP14:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT5]](s32), [[C6]]
+    ; GFX6: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP14]](s1), [[MV10]], [[MV9]]
+    ; GFX6: [[ICMP15:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT4]](s32), [[C6]]
+    ; GFX6: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP15]](s1), [[SELECT6]], [[MV8]]
+    ; GFX6: [[XOR6:%[0-9]+]]:_(s64) = G_XOR [[ASHR2]], [[ASHR3]]
+    ; GFX6: [[XOR7:%[0-9]+]]:_(s64) = G_XOR [[SELECT7]], [[XOR6]]
+    ; GFX6: [[UV60:%[0-9]+]]:_(s32), [[UV61:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR7]](s64)
+    ; GFX6: [[UV62:%[0-9]+]]:_(s32), [[UV63:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR6]](s64)
+    ; GFX6: [[USUBO18:%[0-9]+]]:_(s32), [[USUBO19:%[0-9]+]]:_(s1) = G_USUBO [[UV60]], [[UV62]]
+    ; GFX6: [[USUBE26:%[0-9]+]]:_(s32), [[USUBE27:%[0-9]+]]:_(s1) = G_USUBE [[UV61]], [[UV63]], [[USUBO19]]
+    ; GFX6: [[MV11:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO18]](s32), [[USUBE26]](s32)
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV5]](s64), [[MV11]](s64)
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX8-LABEL: name: test_sdiv_v2s64
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX8: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX8: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX8: [[SDIV:%[0-9]+]]:_(s64) = G_SDIV [[UV]], [[UV2]]
-    ; GFX8: [[SDIV1:%[0-9]+]]:_(s64) = G_SDIV [[UV1]], [[UV3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SDIV]](s64), [[SDIV1]](s64)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV]], [[C]](s32)
+    ; GFX8: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV2]], [[C]](s32)
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX8: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX8: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]]
+    ; GFX8: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO3]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX8: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX8: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX8: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV12]](s32)
+    ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV13]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX8: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX8: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX8: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX8: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX8: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX8: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX8: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[UV16]]
+    ; GFX8: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[UV17]], [[USUBO1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX8: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX8: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX8: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX8: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX8: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX8: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX8: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX8: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX8: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX8: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX8: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX8: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX8: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX8: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX8: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX8: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX8: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX8: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX8: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX8: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX8: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX8: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX8: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX8: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX8: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX8: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX8: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX8: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX8: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX8: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX8: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX8: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX8: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX8: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX8: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX8: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX8: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX8: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV21]], [[UADDO26]]
+    ; GFX8: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV20]], [[UADDE8]]
+    ; GFX8: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV20]], [[UADDO26]]
+    ; GFX8: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX8: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX8: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX8: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX8: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX8: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV21]], [[UADDE8]]
+    ; GFX8: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV21]], [[UADDO26]]
+    ; GFX8: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV20]], [[UADDE8]]
+    ; GFX8: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX8: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX8: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX8: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX8: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX8: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX8: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX8: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX8: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV21]], [[UADDE8]]
+    ; GFX8: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX8: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[ADD16]](s32)
+    ; GFX8: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV22]], [[UADDO36]]
+    ; GFX8: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV23]], [[UADDO36]]
+    ; GFX8: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV22]], [[ADD16]]
+    ; GFX8: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV22]], [[UADDO36]]
+    ; GFX8: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX8: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX8: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV18]], [[MUL15]]
+    ; GFX8: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV19]], [[ADD18]], [[USUBO3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV19]], [[ADD18]]
+    ; GFX8: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV25]]
+    ; GFX8: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV24]]
+    ; GFX8: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV25]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX8: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV24]]
+    ; GFX8: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV25]], [[USUBO3]]
+    ; GFX8: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX8: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX8: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX8: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[UADDO36]], [[UV26]]
+    ; GFX8: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV27]], [[UADDO39]]
+    ; GFX8: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO38]](s32), [[UADDE10]](s32)
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV25]]
+    ; GFX8: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV24]]
+    ; GFX8: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV25]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX8: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX8: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UADDO38]], [[UV28]]
+    ; GFX8: [[UADDE12:%[0-9]+]]:_(s32), [[UADDE13:%[0-9]+]]:_(s1) = G_UADDE [[UADDE10]], [[UV29]], [[UADDO41]]
+    ; GFX8: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO40]](s32), [[UADDE12]](s32)
+    ; GFX8: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV24]]
+    ; GFX8: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV25]], [[USUBO5]]
+    ; GFX8: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX8: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX8: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX8: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[ASHR]], [[ASHR1]]
+    ; GFX8: [[XOR3:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[XOR2]]
+    ; GFX8: [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR3]](s64)
+    ; GFX8: [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX8: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV30]], [[UV32]]
+    ; GFX8: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV31]], [[UV33]], [[USUBO9]]
+    ; GFX8: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX8: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; GFX8: [[ASHR3:%[0-9]+]]:_(s64) = G_ASHR [[UV3]], [[C]](s32)
+    ; GFX8: [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX8: [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR2]](s64)
+    ; GFX8: [[UADDO42:%[0-9]+]]:_(s32), [[UADDO43:%[0-9]+]]:_(s1) = G_UADDO [[UV34]], [[UV36]]
+    ; GFX8: [[UADDE14:%[0-9]+]]:_(s32), [[UADDE15:%[0-9]+]]:_(s1) = G_UADDE [[UV35]], [[UV37]], [[UADDO43]]
+    ; GFX8: [[MV6:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO42]](s32), [[UADDE14]](s32)
+    ; GFX8: [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX8: [[UV40:%[0-9]+]]:_(s32), [[UV41:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR3]](s64)
+    ; GFX8: [[UADDO44:%[0-9]+]]:_(s32), [[UADDO45:%[0-9]+]]:_(s1) = G_UADDO [[UV38]], [[UV40]]
+    ; GFX8: [[UADDE16:%[0-9]+]]:_(s32), [[UADDE17:%[0-9]+]]:_(s1) = G_UADDE [[UV39]], [[UV41]], [[UADDO45]]
+    ; GFX8: [[MV7:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO44]](s32), [[UADDE16]](s32)
+    ; GFX8: [[XOR4:%[0-9]+]]:_(s64) = G_XOR [[MV6]], [[ASHR2]]
+    ; GFX8: [[XOR5:%[0-9]+]]:_(s64) = G_XOR [[MV7]], [[ASHR3]]
+    ; GFX8: [[UV42:%[0-9]+]]:_(s32), [[UV43:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR5]](s64)
+    ; GFX8: [[UITOFP2:%[0-9]+]]:_(s32) = G_UITOFP [[UV42]](s32)
+    ; GFX8: [[UITOFP3:%[0-9]+]]:_(s32) = G_UITOFP [[UV43]](s32)
+    ; GFX8: [[C8:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[FMUL4:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP3]], [[C8]]
+    ; GFX8: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL4]], [[UITOFP2]]
+    ; GFX8: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD2]](s32)
+    ; GFX8: [[C9:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX8: [[FMUL5:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C9]]
+    ; GFX8: [[C10:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX8: [[FMUL6:%[0-9]+]]:_(s32) = G_FMUL [[FMUL5]], [[C10]]
+    ; GFX8: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL6]]
+    ; GFX8: [[C11:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX8: [[FMUL7:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC1]], [[C11]]
+    ; GFX8: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL7]], [[FMUL5]]
+    ; GFX8: [[FPTOUI2:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD3]](s32)
+    ; GFX8: [[FPTOUI3:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC1]](s32)
+    ; GFX8: [[UV44:%[0-9]+]]:_(s32), [[UV45:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX8: [[UV46:%[0-9]+]]:_(s32), [[UV47:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR5]](s64)
+    ; GFX8: [[USUBO10:%[0-9]+]]:_(s32), [[USUBO11:%[0-9]+]]:_(s1) = G_USUBO [[UV44]], [[UV46]]
+    ; GFX8: [[USUBE14:%[0-9]+]]:_(s32), [[USUBE15:%[0-9]+]]:_(s1) = G_USUBE [[UV45]], [[UV47]], [[USUBO11]]
+    ; GFX8: [[MUL18:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[FPTOUI2]]
+    ; GFX8: [[MUL19:%[0-9]+]]:_(s32) = G_MUL [[USUBE14]], [[FPTOUI2]]
+    ; GFX8: [[MUL20:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[FPTOUI3]]
+    ; GFX8: [[UMULH15:%[0-9]+]]:_(s32) = G_UMULH [[USUBO10]], [[FPTOUI2]]
+    ; GFX8: [[ADD19:%[0-9]+]]:_(s32) = G_ADD [[MUL19]], [[MUL20]]
+    ; GFX8: [[ADD20:%[0-9]+]]:_(s32) = G_ADD [[ADD19]], [[UMULH15]]
+    ; GFX8: [[MUL21:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[MUL18]]
+    ; GFX8: [[MUL22:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI2]], [[ADD20]]
+    ; GFX8: [[UMULH16:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[MUL18]]
+    ; GFX8: [[UADDO46:%[0-9]+]]:_(s32), [[UADDO47:%[0-9]+]]:_(s1) = G_UADDO [[MUL21]], [[MUL22]]
+    ; GFX8: [[ZEXT15:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO47]](s1)
+    ; GFX8: [[UADDO48:%[0-9]+]]:_(s32), [[UADDO49:%[0-9]+]]:_(s1) = G_UADDO [[UADDO46]], [[UMULH16]]
+    ; GFX8: [[ZEXT16:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO49]](s1)
+    ; GFX8: [[ADD21:%[0-9]+]]:_(s32) = G_ADD [[ZEXT15]], [[ZEXT16]]
+    ; GFX8: [[MUL23:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[ADD20]]
+    ; GFX8: [[UMULH17:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[MUL18]]
+    ; GFX8: [[UMULH18:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[ADD20]]
+    ; GFX8: [[UADDO50:%[0-9]+]]:_(s32), [[UADDO51:%[0-9]+]]:_(s1) = G_UADDO [[MUL23]], [[UMULH17]]
+    ; GFX8: [[ZEXT17:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO51]](s1)
+    ; GFX8: [[UADDO52:%[0-9]+]]:_(s32), [[UADDO53:%[0-9]+]]:_(s1) = G_UADDO [[UADDO50]], [[UMULH18]]
+    ; GFX8: [[ZEXT18:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO53]](s1)
+    ; GFX8: [[ADD22:%[0-9]+]]:_(s32) = G_ADD [[ZEXT17]], [[ZEXT18]]
+    ; GFX8: [[UADDO54:%[0-9]+]]:_(s32), [[UADDO55:%[0-9]+]]:_(s1) = G_UADDO [[UADDO52]], [[ADD21]]
+    ; GFX8: [[ZEXT19:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO55]](s1)
+    ; GFX8: [[ADD23:%[0-9]+]]:_(s32) = G_ADD [[ADD22]], [[ZEXT19]]
+    ; GFX8: [[UMULH19:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[ADD20]]
+    ; GFX8: [[ADD24:%[0-9]+]]:_(s32) = G_ADD [[UMULH19]], [[ADD23]]
+    ; GFX8: [[UADDO56:%[0-9]+]]:_(s32), [[UADDO57:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI2]], [[UADDO54]]
+    ; GFX8: [[UADDE18:%[0-9]+]]:_(s32), [[UADDE19:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI3]], [[ADD24]], [[UADDO57]]
+    ; GFX8: [[ADD25:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI3]], [[ADD24]]
+    ; GFX8: [[MUL24:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[UADDO56]]
+    ; GFX8: [[MUL25:%[0-9]+]]:_(s32) = G_MUL [[USUBE14]], [[UADDO56]]
+    ; GFX8: [[MUL26:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[UADDE18]]
+    ; GFX8: [[UMULH20:%[0-9]+]]:_(s32) = G_UMULH [[USUBO10]], [[UADDO56]]
+    ; GFX8: [[ADD26:%[0-9]+]]:_(s32) = G_ADD [[MUL25]], [[MUL26]]
+    ; GFX8: [[ADD27:%[0-9]+]]:_(s32) = G_ADD [[ADD26]], [[UMULH20]]
+    ; GFX8: [[MUL27:%[0-9]+]]:_(s32) = G_MUL [[UADDE18]], [[MUL24]]
+    ; GFX8: [[MUL28:%[0-9]+]]:_(s32) = G_MUL [[UADDO56]], [[ADD27]]
+    ; GFX8: [[UMULH21:%[0-9]+]]:_(s32) = G_UMULH [[UADDO56]], [[MUL24]]
+    ; GFX8: [[UADDO58:%[0-9]+]]:_(s32), [[UADDO59:%[0-9]+]]:_(s1) = G_UADDO [[MUL27]], [[MUL28]]
+    ; GFX8: [[ZEXT20:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO59]](s1)
+    ; GFX8: [[UADDO60:%[0-9]+]]:_(s32), [[UADDO61:%[0-9]+]]:_(s1) = G_UADDO [[UADDO58]], [[UMULH21]]
+    ; GFX8: [[ZEXT21:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO61]](s1)
+    ; GFX8: [[ADD28:%[0-9]+]]:_(s32) = G_ADD [[ZEXT20]], [[ZEXT21]]
+    ; GFX8: [[MUL29:%[0-9]+]]:_(s32) = G_MUL [[UADDE18]], [[ADD27]]
+    ; GFX8: [[UMULH22:%[0-9]+]]:_(s32) = G_UMULH [[UADDE18]], [[MUL24]]
+    ; GFX8: [[UMULH23:%[0-9]+]]:_(s32) = G_UMULH [[UADDO56]], [[ADD27]]
+    ; GFX8: [[UADDO62:%[0-9]+]]:_(s32), [[UADDO63:%[0-9]+]]:_(s1) = G_UADDO [[MUL29]], [[UMULH22]]
+    ; GFX8: [[ZEXT22:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO63]](s1)
+    ; GFX8: [[UADDO64:%[0-9]+]]:_(s32), [[UADDO65:%[0-9]+]]:_(s1) = G_UADDO [[UADDO62]], [[UMULH23]]
+    ; GFX8: [[ZEXT23:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO65]](s1)
+    ; GFX8: [[ADD29:%[0-9]+]]:_(s32) = G_ADD [[ZEXT22]], [[ZEXT23]]
+    ; GFX8: [[UADDO66:%[0-9]+]]:_(s32), [[UADDO67:%[0-9]+]]:_(s1) = G_UADDO [[UADDO64]], [[ADD28]]
+    ; GFX8: [[ZEXT24:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO67]](s1)
+    ; GFX8: [[ADD30:%[0-9]+]]:_(s32) = G_ADD [[ADD29]], [[ZEXT24]]
+    ; GFX8: [[UMULH24:%[0-9]+]]:_(s32) = G_UMULH [[UADDE18]], [[ADD27]]
+    ; GFX8: [[ADD31:%[0-9]+]]:_(s32) = G_ADD [[UMULH24]], [[ADD30]]
+    ; GFX8: [[UADDO68:%[0-9]+]]:_(s32), [[UADDO69:%[0-9]+]]:_(s1) = G_UADDO [[UADDO56]], [[UADDO66]]
+    ; GFX8: [[UADDE20:%[0-9]+]]:_(s32), [[UADDE21:%[0-9]+]]:_(s1) = G_UADDE [[ADD25]], [[ADD31]], [[UADDO57]]
+    ; GFX8: [[UADDE22:%[0-9]+]]:_(s32), [[UADDE23:%[0-9]+]]:_(s1) = G_UADDE [[UADDE20]], [[C6]], [[UADDO69]]
+    ; GFX8: [[UV48:%[0-9]+]]:_(s32), [[UV49:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX8: [[UV50:%[0-9]+]]:_(s32), [[UV51:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX8: [[MUL30:%[0-9]+]]:_(s32) = G_MUL [[UV51]], [[UADDO68]]
+    ; GFX8: [[MUL31:%[0-9]+]]:_(s32) = G_MUL [[UV50]], [[UADDE22]]
+    ; GFX8: [[UMULH25:%[0-9]+]]:_(s32) = G_UMULH [[UV50]], [[UADDO68]]
+    ; GFX8: [[UADDO70:%[0-9]+]]:_(s32), [[UADDO71:%[0-9]+]]:_(s1) = G_UADDO [[MUL30]], [[MUL31]]
+    ; GFX8: [[ZEXT25:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO71]](s1)
+    ; GFX8: [[UADDO72:%[0-9]+]]:_(s32), [[UADDO73:%[0-9]+]]:_(s1) = G_UADDO [[UADDO70]], [[UMULH25]]
+    ; GFX8: [[ZEXT26:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO73]](s1)
+    ; GFX8: [[ADD32:%[0-9]+]]:_(s32) = G_ADD [[ZEXT25]], [[ZEXT26]]
+    ; GFX8: [[MUL32:%[0-9]+]]:_(s32) = G_MUL [[UV51]], [[UADDE22]]
+    ; GFX8: [[UMULH26:%[0-9]+]]:_(s32) = G_UMULH [[UV51]], [[UADDO68]]
+    ; GFX8: [[UMULH27:%[0-9]+]]:_(s32) = G_UMULH [[UV50]], [[UADDE22]]
+    ; GFX8: [[UADDO74:%[0-9]+]]:_(s32), [[UADDO75:%[0-9]+]]:_(s1) = G_UADDO [[MUL32]], [[UMULH26]]
+    ; GFX8: [[ZEXT27:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO75]](s1)
+    ; GFX8: [[UADDO76:%[0-9]+]]:_(s32), [[UADDO77:%[0-9]+]]:_(s1) = G_UADDO [[UADDO74]], [[UMULH27]]
+    ; GFX8: [[ZEXT28:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO77]](s1)
+    ; GFX8: [[ADD33:%[0-9]+]]:_(s32) = G_ADD [[ZEXT27]], [[ZEXT28]]
+    ; GFX8: [[UADDO78:%[0-9]+]]:_(s32), [[UADDO79:%[0-9]+]]:_(s1) = G_UADDO [[UADDO76]], [[ADD32]]
+    ; GFX8: [[ZEXT29:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO79]](s1)
+    ; GFX8: [[ADD34:%[0-9]+]]:_(s32) = G_ADD [[ADD33]], [[ZEXT29]]
+    ; GFX8: [[UMULH28:%[0-9]+]]:_(s32) = G_UMULH [[UV51]], [[UADDE22]]
+    ; GFX8: [[ADD35:%[0-9]+]]:_(s32) = G_ADD [[UMULH28]], [[ADD34]]
+    ; GFX8: [[MV8:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO78]](s32), [[ADD35]](s32)
+    ; GFX8: [[UV52:%[0-9]+]]:_(s32), [[UV53:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR5]](s64)
+    ; GFX8: [[MUL33:%[0-9]+]]:_(s32) = G_MUL [[UV52]], [[UADDO78]]
+    ; GFX8: [[MUL34:%[0-9]+]]:_(s32) = G_MUL [[UV53]], [[UADDO78]]
+    ; GFX8: [[MUL35:%[0-9]+]]:_(s32) = G_MUL [[UV52]], [[ADD35]]
+    ; GFX8: [[UMULH29:%[0-9]+]]:_(s32) = G_UMULH [[UV52]], [[UADDO78]]
+    ; GFX8: [[ADD36:%[0-9]+]]:_(s32) = G_ADD [[MUL34]], [[MUL35]]
+    ; GFX8: [[ADD37:%[0-9]+]]:_(s32) = G_ADD [[ADD36]], [[UMULH29]]
+    ; GFX8: [[USUBO12:%[0-9]+]]:_(s32), [[USUBO13:%[0-9]+]]:_(s1) = G_USUBO [[UV48]], [[MUL33]]
+    ; GFX8: [[USUBE16:%[0-9]+]]:_(s32), [[USUBE17:%[0-9]+]]:_(s1) = G_USUBE [[UV49]], [[ADD37]], [[USUBO13]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV49]], [[ADD37]]
+    ; GFX8: [[UV54:%[0-9]+]]:_(s32), [[UV55:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR5]](s64)
+    ; GFX8: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE16]](s32), [[UV55]]
+    ; GFX8: [[SEXT4:%[0-9]+]]:_(s32) = G_SEXT [[ICMP8]](s1)
+    ; GFX8: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO12]](s32), [[UV54]]
+    ; GFX8: [[SEXT5:%[0-9]+]]:_(s32) = G_SEXT [[ICMP9]](s1)
+    ; GFX8: [[ICMP10:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE16]](s32), [[UV55]]
+    ; GFX8: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP10]](s1), [[SEXT5]], [[SEXT4]]
+    ; GFX8: [[USUBO14:%[0-9]+]]:_(s32), [[USUBO15:%[0-9]+]]:_(s1) = G_USUBO [[USUBO12]], [[UV54]]
+    ; GFX8: [[USUBE18:%[0-9]+]]:_(s32), [[USUBE19:%[0-9]+]]:_(s1) = G_USUBE [[SUB1]], [[UV55]], [[USUBO13]]
+    ; GFX8: [[USUBE20:%[0-9]+]]:_(s32), [[USUBE21:%[0-9]+]]:_(s1) = G_USUBE [[USUBE18]], [[C6]], [[USUBO15]]
+    ; GFX8: [[UV56:%[0-9]+]]:_(s32), [[UV57:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX8: [[UADDO80:%[0-9]+]]:_(s32), [[UADDO81:%[0-9]+]]:_(s1) = G_UADDO [[UADDO78]], [[UV56]]
+    ; GFX8: [[UADDE24:%[0-9]+]]:_(s32), [[UADDE25:%[0-9]+]]:_(s1) = G_UADDE [[ADD35]], [[UV57]], [[UADDO81]]
+    ; GFX8: [[MV9:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO80]](s32), [[UADDE24]](s32)
+    ; GFX8: [[ICMP11:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE20]](s32), [[UV55]]
+    ; GFX8: [[SEXT6:%[0-9]+]]:_(s32) = G_SEXT [[ICMP11]](s1)
+    ; GFX8: [[ICMP12:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO14]](s32), [[UV54]]
+    ; GFX8: [[SEXT7:%[0-9]+]]:_(s32) = G_SEXT [[ICMP12]](s1)
+    ; GFX8: [[ICMP13:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE20]](s32), [[UV55]]
+    ; GFX8: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP13]](s1), [[SEXT7]], [[SEXT6]]
+    ; GFX8: [[UV58:%[0-9]+]]:_(s32), [[UV59:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX8: [[UADDO82:%[0-9]+]]:_(s32), [[UADDO83:%[0-9]+]]:_(s1) = G_UADDO [[UADDO80]], [[UV58]]
+    ; GFX8: [[UADDE26:%[0-9]+]]:_(s32), [[UADDE27:%[0-9]+]]:_(s1) = G_UADDE [[UADDE24]], [[UV59]], [[UADDO83]]
+    ; GFX8: [[MV10:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO82]](s32), [[UADDE26]](s32)
+    ; GFX8: [[USUBO16:%[0-9]+]]:_(s32), [[USUBO17:%[0-9]+]]:_(s1) = G_USUBO [[USUBO14]], [[UV54]]
+    ; GFX8: [[USUBE22:%[0-9]+]]:_(s32), [[USUBE23:%[0-9]+]]:_(s1) = G_USUBE [[USUBE18]], [[UV55]], [[USUBO15]]
+    ; GFX8: [[USUBE24:%[0-9]+]]:_(s32), [[USUBE25:%[0-9]+]]:_(s1) = G_USUBE [[USUBE22]], [[C6]], [[USUBO17]]
+    ; GFX8: [[ICMP14:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT5]](s32), [[C6]]
+    ; GFX8: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP14]](s1), [[MV10]], [[MV9]]
+    ; GFX8: [[ICMP15:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT4]](s32), [[C6]]
+    ; GFX8: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP15]](s1), [[SELECT6]], [[MV8]]
+    ; GFX8: [[XOR6:%[0-9]+]]:_(s64) = G_XOR [[ASHR2]], [[ASHR3]]
+    ; GFX8: [[XOR7:%[0-9]+]]:_(s64) = G_XOR [[SELECT7]], [[XOR6]]
+    ; GFX8: [[UV60:%[0-9]+]]:_(s32), [[UV61:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR7]](s64)
+    ; GFX8: [[UV62:%[0-9]+]]:_(s32), [[UV63:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR6]](s64)
+    ; GFX8: [[USUBO18:%[0-9]+]]:_(s32), [[USUBO19:%[0-9]+]]:_(s1) = G_USUBO [[UV60]], [[UV62]]
+    ; GFX8: [[USUBE26:%[0-9]+]]:_(s32), [[USUBE27:%[0-9]+]]:_(s1) = G_USUBE [[UV61]], [[UV63]], [[USUBO19]]
+    ; GFX8: [[MV11:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO18]](s32), [[USUBE26]](s32)
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV5]](s64), [[MV11]](s64)
     ; GFX8: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX9-LABEL: name: test_sdiv_v2s64
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX9: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX9: [[SDIV:%[0-9]+]]:_(s64) = G_SDIV [[UV]], [[UV2]]
-    ; GFX9: [[SDIV1:%[0-9]+]]:_(s64) = G_SDIV [[UV1]], [[UV3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SDIV]](s64), [[SDIV1]](s64)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV]], [[C]](s32)
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV2]], [[C]](s32)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX9: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]]
+    ; GFX9: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO3]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX9: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX9: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV12]](s32)
+    ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV13]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX9: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX9: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX9: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX9: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX9: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[UV16]]
+    ; GFX9: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[UV17]], [[USUBO1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX9: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX9: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX9: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX9: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX9: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX9: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX9: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX9: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX9: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX9: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX9: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX9: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX9: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX9: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX9: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX9: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX9: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX9: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX9: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX9: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX9: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX9: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX9: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX9: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX9: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX9: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX9: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX9: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX9: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX9: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX9: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX9: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX9: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX9: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX9: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV21]], [[UADDO26]]
+    ; GFX9: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV20]], [[UADDE8]]
+    ; GFX9: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV20]], [[UADDO26]]
+    ; GFX9: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX9: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX9: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX9: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX9: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX9: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV21]], [[UADDE8]]
+    ; GFX9: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV21]], [[UADDO26]]
+    ; GFX9: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV20]], [[UADDE8]]
+    ; GFX9: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX9: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX9: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX9: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX9: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX9: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX9: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX9: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX9: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV21]], [[UADDE8]]
+    ; GFX9: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX9: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[ADD16]](s32)
+    ; GFX9: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV22]], [[UADDO36]]
+    ; GFX9: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV23]], [[UADDO36]]
+    ; GFX9: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV22]], [[ADD16]]
+    ; GFX9: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV22]], [[UADDO36]]
+    ; GFX9: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX9: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX9: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV18]], [[MUL15]]
+    ; GFX9: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV19]], [[ADD18]], [[USUBO3]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV19]], [[ADD18]]
+    ; GFX9: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV25]]
+    ; GFX9: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV24]]
+    ; GFX9: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV25]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX9: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV24]]
+    ; GFX9: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV25]], [[USUBO3]]
+    ; GFX9: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX9: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX9: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX9: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[UADDO36]], [[UV26]]
+    ; GFX9: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV27]], [[UADDO39]]
+    ; GFX9: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO38]](s32), [[UADDE10]](s32)
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV25]]
+    ; GFX9: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV24]]
+    ; GFX9: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV25]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX9: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX9: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UADDO38]], [[UV28]]
+    ; GFX9: [[UADDE12:%[0-9]+]]:_(s32), [[UADDE13:%[0-9]+]]:_(s1) = G_UADDE [[UADDE10]], [[UV29]], [[UADDO41]]
+    ; GFX9: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO40]](s32), [[UADDE12]](s32)
+    ; GFX9: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV24]]
+    ; GFX9: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV25]], [[USUBO5]]
+    ; GFX9: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX9: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX9: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX9: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[ASHR]], [[ASHR1]]
+    ; GFX9: [[XOR3:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[XOR2]]
+    ; GFX9: [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR3]](s64)
+    ; GFX9: [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX9: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV30]], [[UV32]]
+    ; GFX9: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV31]], [[UV33]], [[USUBO9]]
+    ; GFX9: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX9: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; GFX9: [[ASHR3:%[0-9]+]]:_(s64) = G_ASHR [[UV3]], [[C]](s32)
+    ; GFX9: [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX9: [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR2]](s64)
+    ; GFX9: [[UADDO42:%[0-9]+]]:_(s32), [[UADDO43:%[0-9]+]]:_(s1) = G_UADDO [[UV34]], [[UV36]]
+    ; GFX9: [[UADDE14:%[0-9]+]]:_(s32), [[UADDE15:%[0-9]+]]:_(s1) = G_UADDE [[UV35]], [[UV37]], [[UADDO43]]
+    ; GFX9: [[MV6:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO42]](s32), [[UADDE14]](s32)
+    ; GFX9: [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX9: [[UV40:%[0-9]+]]:_(s32), [[UV41:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR3]](s64)
+    ; GFX9: [[UADDO44:%[0-9]+]]:_(s32), [[UADDO45:%[0-9]+]]:_(s1) = G_UADDO [[UV38]], [[UV40]]
+    ; GFX9: [[UADDE16:%[0-9]+]]:_(s32), [[UADDE17:%[0-9]+]]:_(s1) = G_UADDE [[UV39]], [[UV41]], [[UADDO45]]
+    ; GFX9: [[MV7:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO44]](s32), [[UADDE16]](s32)
+    ; GFX9: [[XOR4:%[0-9]+]]:_(s64) = G_XOR [[MV6]], [[ASHR2]]
+    ; GFX9: [[XOR5:%[0-9]+]]:_(s64) = G_XOR [[MV7]], [[ASHR3]]
+    ; GFX9: [[UV42:%[0-9]+]]:_(s32), [[UV43:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR5]](s64)
+    ; GFX9: [[UITOFP2:%[0-9]+]]:_(s32) = G_UITOFP [[UV42]](s32)
+    ; GFX9: [[UITOFP3:%[0-9]+]]:_(s32) = G_UITOFP [[UV43]](s32)
+    ; GFX9: [[C8:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[FMUL4:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP3]], [[C8]]
+    ; GFX9: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL4]], [[UITOFP2]]
+    ; GFX9: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD2]](s32)
+    ; GFX9: [[C9:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX9: [[FMUL5:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C9]]
+    ; GFX9: [[C10:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX9: [[FMUL6:%[0-9]+]]:_(s32) = G_FMUL [[FMUL5]], [[C10]]
+    ; GFX9: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL6]]
+    ; GFX9: [[C11:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX9: [[FMUL7:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC1]], [[C11]]
+    ; GFX9: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL7]], [[FMUL5]]
+    ; GFX9: [[FPTOUI2:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD3]](s32)
+    ; GFX9: [[FPTOUI3:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC1]](s32)
+    ; GFX9: [[UV44:%[0-9]+]]:_(s32), [[UV45:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX9: [[UV46:%[0-9]+]]:_(s32), [[UV47:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR5]](s64)
+    ; GFX9: [[USUBO10:%[0-9]+]]:_(s32), [[USUBO11:%[0-9]+]]:_(s1) = G_USUBO [[UV44]], [[UV46]]
+    ; GFX9: [[USUBE14:%[0-9]+]]:_(s32), [[USUBE15:%[0-9]+]]:_(s1) = G_USUBE [[UV45]], [[UV47]], [[USUBO11]]
+    ; GFX9: [[MUL18:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[FPTOUI2]]
+    ; GFX9: [[MUL19:%[0-9]+]]:_(s32) = G_MUL [[USUBE14]], [[FPTOUI2]]
+    ; GFX9: [[MUL20:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[FPTOUI3]]
+    ; GFX9: [[UMULH15:%[0-9]+]]:_(s32) = G_UMULH [[USUBO10]], [[FPTOUI2]]
+    ; GFX9: [[ADD19:%[0-9]+]]:_(s32) = G_ADD [[MUL19]], [[MUL20]]
+    ; GFX9: [[ADD20:%[0-9]+]]:_(s32) = G_ADD [[ADD19]], [[UMULH15]]
+    ; GFX9: [[MUL21:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[MUL18]]
+    ; GFX9: [[MUL22:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI2]], [[ADD20]]
+    ; GFX9: [[UMULH16:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[MUL18]]
+    ; GFX9: [[UADDO46:%[0-9]+]]:_(s32), [[UADDO47:%[0-9]+]]:_(s1) = G_UADDO [[MUL21]], [[MUL22]]
+    ; GFX9: [[ZEXT15:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO47]](s1)
+    ; GFX9: [[UADDO48:%[0-9]+]]:_(s32), [[UADDO49:%[0-9]+]]:_(s1) = G_UADDO [[UADDO46]], [[UMULH16]]
+    ; GFX9: [[ZEXT16:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO49]](s1)
+    ; GFX9: [[ADD21:%[0-9]+]]:_(s32) = G_ADD [[ZEXT15]], [[ZEXT16]]
+    ; GFX9: [[MUL23:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[ADD20]]
+    ; GFX9: [[UMULH17:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[MUL18]]
+    ; GFX9: [[UMULH18:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[ADD20]]
+    ; GFX9: [[UADDO50:%[0-9]+]]:_(s32), [[UADDO51:%[0-9]+]]:_(s1) = G_UADDO [[MUL23]], [[UMULH17]]
+    ; GFX9: [[ZEXT17:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO51]](s1)
+    ; GFX9: [[UADDO52:%[0-9]+]]:_(s32), [[UADDO53:%[0-9]+]]:_(s1) = G_UADDO [[UADDO50]], [[UMULH18]]
+    ; GFX9: [[ZEXT18:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO53]](s1)
+    ; GFX9: [[ADD22:%[0-9]+]]:_(s32) = G_ADD [[ZEXT17]], [[ZEXT18]]
+    ; GFX9: [[UADDO54:%[0-9]+]]:_(s32), [[UADDO55:%[0-9]+]]:_(s1) = G_UADDO [[UADDO52]], [[ADD21]]
+    ; GFX9: [[ZEXT19:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO55]](s1)
+    ; GFX9: [[ADD23:%[0-9]+]]:_(s32) = G_ADD [[ADD22]], [[ZEXT19]]
+    ; GFX9: [[UMULH19:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[ADD20]]
+    ; GFX9: [[ADD24:%[0-9]+]]:_(s32) = G_ADD [[UMULH19]], [[ADD23]]
+    ; GFX9: [[UADDO56:%[0-9]+]]:_(s32), [[UADDO57:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI2]], [[UADDO54]]
+    ; GFX9: [[UADDE18:%[0-9]+]]:_(s32), [[UADDE19:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI3]], [[ADD24]], [[UADDO57]]
+    ; GFX9: [[ADD25:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI3]], [[ADD24]]
+    ; GFX9: [[MUL24:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[UADDO56]]
+    ; GFX9: [[MUL25:%[0-9]+]]:_(s32) = G_MUL [[USUBE14]], [[UADDO56]]
+    ; GFX9: [[MUL26:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[UADDE18]]
+    ; GFX9: [[UMULH20:%[0-9]+]]:_(s32) = G_UMULH [[USUBO10]], [[UADDO56]]
+    ; GFX9: [[ADD26:%[0-9]+]]:_(s32) = G_ADD [[MUL25]], [[MUL26]]
+    ; GFX9: [[ADD27:%[0-9]+]]:_(s32) = G_ADD [[ADD26]], [[UMULH20]]
+    ; GFX9: [[MUL27:%[0-9]+]]:_(s32) = G_MUL [[UADDE18]], [[MUL24]]
+    ; GFX9: [[MUL28:%[0-9]+]]:_(s32) = G_MUL [[UADDO56]], [[ADD27]]
+    ; GFX9: [[UMULH21:%[0-9]+]]:_(s32) = G_UMULH [[UADDO56]], [[MUL24]]
+    ; GFX9: [[UADDO58:%[0-9]+]]:_(s32), [[UADDO59:%[0-9]+]]:_(s1) = G_UADDO [[MUL27]], [[MUL28]]
+    ; GFX9: [[ZEXT20:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO59]](s1)
+    ; GFX9: [[UADDO60:%[0-9]+]]:_(s32), [[UADDO61:%[0-9]+]]:_(s1) = G_UADDO [[UADDO58]], [[UMULH21]]
+    ; GFX9: [[ZEXT21:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO61]](s1)
+    ; GFX9: [[ADD28:%[0-9]+]]:_(s32) = G_ADD [[ZEXT20]], [[ZEXT21]]
+    ; GFX9: [[MUL29:%[0-9]+]]:_(s32) = G_MUL [[UADDE18]], [[ADD27]]
+    ; GFX9: [[UMULH22:%[0-9]+]]:_(s32) = G_UMULH [[UADDE18]], [[MUL24]]
+    ; GFX9: [[UMULH23:%[0-9]+]]:_(s32) = G_UMULH [[UADDO56]], [[ADD27]]
+    ; GFX9: [[UADDO62:%[0-9]+]]:_(s32), [[UADDO63:%[0-9]+]]:_(s1) = G_UADDO [[MUL29]], [[UMULH22]]
+    ; GFX9: [[ZEXT22:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO63]](s1)
+    ; GFX9: [[UADDO64:%[0-9]+]]:_(s32), [[UADDO65:%[0-9]+]]:_(s1) = G_UADDO [[UADDO62]], [[UMULH23]]
+    ; GFX9: [[ZEXT23:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO65]](s1)
+    ; GFX9: [[ADD29:%[0-9]+]]:_(s32) = G_ADD [[ZEXT22]], [[ZEXT23]]
+    ; GFX9: [[UADDO66:%[0-9]+]]:_(s32), [[UADDO67:%[0-9]+]]:_(s1) = G_UADDO [[UADDO64]], [[ADD28]]
+    ; GFX9: [[ZEXT24:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO67]](s1)
+    ; GFX9: [[ADD30:%[0-9]+]]:_(s32) = G_ADD [[ADD29]], [[ZEXT24]]
+    ; GFX9: [[UMULH24:%[0-9]+]]:_(s32) = G_UMULH [[UADDE18]], [[ADD27]]
+    ; GFX9: [[ADD31:%[0-9]+]]:_(s32) = G_ADD [[UMULH24]], [[ADD30]]
+    ; GFX9: [[UADDO68:%[0-9]+]]:_(s32), [[UADDO69:%[0-9]+]]:_(s1) = G_UADDO [[UADDO56]], [[UADDO66]]
+    ; GFX9: [[UADDE20:%[0-9]+]]:_(s32), [[UADDE21:%[0-9]+]]:_(s1) = G_UADDE [[ADD25]], [[ADD31]], [[UADDO57]]
+    ; GFX9: [[UADDE22:%[0-9]+]]:_(s32), [[UADDE23:%[0-9]+]]:_(s1) = G_UADDE [[UADDE20]], [[C6]], [[UADDO69]]
+    ; GFX9: [[UV48:%[0-9]+]]:_(s32), [[UV49:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX9: [[UV50:%[0-9]+]]:_(s32), [[UV51:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX9: [[MUL30:%[0-9]+]]:_(s32) = G_MUL [[UV51]], [[UADDO68]]
+    ; GFX9: [[MUL31:%[0-9]+]]:_(s32) = G_MUL [[UV50]], [[UADDE22]]
+    ; GFX9: [[UMULH25:%[0-9]+]]:_(s32) = G_UMULH [[UV50]], [[UADDO68]]
+    ; GFX9: [[UADDO70:%[0-9]+]]:_(s32), [[UADDO71:%[0-9]+]]:_(s1) = G_UADDO [[MUL30]], [[MUL31]]
+    ; GFX9: [[ZEXT25:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO71]](s1)
+    ; GFX9: [[UADDO72:%[0-9]+]]:_(s32), [[UADDO73:%[0-9]+]]:_(s1) = G_UADDO [[UADDO70]], [[UMULH25]]
+    ; GFX9: [[ZEXT26:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO73]](s1)
+    ; GFX9: [[ADD32:%[0-9]+]]:_(s32) = G_ADD [[ZEXT25]], [[ZEXT26]]
+    ; GFX9: [[MUL32:%[0-9]+]]:_(s32) = G_MUL [[UV51]], [[UADDE22]]
+    ; GFX9: [[UMULH26:%[0-9]+]]:_(s32) = G_UMULH [[UV51]], [[UADDO68]]
+    ; GFX9: [[UMULH27:%[0-9]+]]:_(s32) = G_UMULH [[UV50]], [[UADDE22]]
+    ; GFX9: [[UADDO74:%[0-9]+]]:_(s32), [[UADDO75:%[0-9]+]]:_(s1) = G_UADDO [[MUL32]], [[UMULH26]]
+    ; GFX9: [[ZEXT27:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO75]](s1)
+    ; GFX9: [[UADDO76:%[0-9]+]]:_(s32), [[UADDO77:%[0-9]+]]:_(s1) = G_UADDO [[UADDO74]], [[UMULH27]]
+    ; GFX9: [[ZEXT28:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO77]](s1)
+    ; GFX9: [[ADD33:%[0-9]+]]:_(s32) = G_ADD [[ZEXT27]], [[ZEXT28]]
+    ; GFX9: [[UADDO78:%[0-9]+]]:_(s32), [[UADDO79:%[0-9]+]]:_(s1) = G_UADDO [[UADDO76]], [[ADD32]]
+    ; GFX9: [[ZEXT29:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO79]](s1)
+    ; GFX9: [[ADD34:%[0-9]+]]:_(s32) = G_ADD [[ADD33]], [[ZEXT29]]
+    ; GFX9: [[UMULH28:%[0-9]+]]:_(s32) = G_UMULH [[UV51]], [[UADDE22]]
+    ; GFX9: [[ADD35:%[0-9]+]]:_(s32) = G_ADD [[UMULH28]], [[ADD34]]
+    ; GFX9: [[MV8:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO78]](s32), [[ADD35]](s32)
+    ; GFX9: [[UV52:%[0-9]+]]:_(s32), [[UV53:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR5]](s64)
+    ; GFX9: [[MUL33:%[0-9]+]]:_(s32) = G_MUL [[UV52]], [[UADDO78]]
+    ; GFX9: [[MUL34:%[0-9]+]]:_(s32) = G_MUL [[UV53]], [[UADDO78]]
+    ; GFX9: [[MUL35:%[0-9]+]]:_(s32) = G_MUL [[UV52]], [[ADD35]]
+    ; GFX9: [[UMULH29:%[0-9]+]]:_(s32) = G_UMULH [[UV52]], [[UADDO78]]
+    ; GFX9: [[ADD36:%[0-9]+]]:_(s32) = G_ADD [[MUL34]], [[MUL35]]
+    ; GFX9: [[ADD37:%[0-9]+]]:_(s32) = G_ADD [[ADD36]], [[UMULH29]]
+    ; GFX9: [[USUBO12:%[0-9]+]]:_(s32), [[USUBO13:%[0-9]+]]:_(s1) = G_USUBO [[UV48]], [[MUL33]]
+    ; GFX9: [[USUBE16:%[0-9]+]]:_(s32), [[USUBE17:%[0-9]+]]:_(s1) = G_USUBE [[UV49]], [[ADD37]], [[USUBO13]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV49]], [[ADD37]]
+    ; GFX9: [[UV54:%[0-9]+]]:_(s32), [[UV55:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR5]](s64)
+    ; GFX9: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE16]](s32), [[UV55]]
+    ; GFX9: [[SEXT4:%[0-9]+]]:_(s32) = G_SEXT [[ICMP8]](s1)
+    ; GFX9: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO12]](s32), [[UV54]]
+    ; GFX9: [[SEXT5:%[0-9]+]]:_(s32) = G_SEXT [[ICMP9]](s1)
+    ; GFX9: [[ICMP10:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE16]](s32), [[UV55]]
+    ; GFX9: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP10]](s1), [[SEXT5]], [[SEXT4]]
+    ; GFX9: [[USUBO14:%[0-9]+]]:_(s32), [[USUBO15:%[0-9]+]]:_(s1) = G_USUBO [[USUBO12]], [[UV54]]
+    ; GFX9: [[USUBE18:%[0-9]+]]:_(s32), [[USUBE19:%[0-9]+]]:_(s1) = G_USUBE [[SUB1]], [[UV55]], [[USUBO13]]
+    ; GFX9: [[USUBE20:%[0-9]+]]:_(s32), [[USUBE21:%[0-9]+]]:_(s1) = G_USUBE [[USUBE18]], [[C6]], [[USUBO15]]
+    ; GFX9: [[UV56:%[0-9]+]]:_(s32), [[UV57:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX9: [[UADDO80:%[0-9]+]]:_(s32), [[UADDO81:%[0-9]+]]:_(s1) = G_UADDO [[UADDO78]], [[UV56]]
+    ; GFX9: [[UADDE24:%[0-9]+]]:_(s32), [[UADDE25:%[0-9]+]]:_(s1) = G_UADDE [[ADD35]], [[UV57]], [[UADDO81]]
+    ; GFX9: [[MV9:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO80]](s32), [[UADDE24]](s32)
+    ; GFX9: [[ICMP11:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE20]](s32), [[UV55]]
+    ; GFX9: [[SEXT6:%[0-9]+]]:_(s32) = G_SEXT [[ICMP11]](s1)
+    ; GFX9: [[ICMP12:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO14]](s32), [[UV54]]
+    ; GFX9: [[SEXT7:%[0-9]+]]:_(s32) = G_SEXT [[ICMP12]](s1)
+    ; GFX9: [[ICMP13:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE20]](s32), [[UV55]]
+    ; GFX9: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP13]](s1), [[SEXT7]], [[SEXT6]]
+    ; GFX9: [[UV58:%[0-9]+]]:_(s32), [[UV59:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX9: [[UADDO82:%[0-9]+]]:_(s32), [[UADDO83:%[0-9]+]]:_(s1) = G_UADDO [[UADDO80]], [[UV58]]
+    ; GFX9: [[UADDE26:%[0-9]+]]:_(s32), [[UADDE27:%[0-9]+]]:_(s1) = G_UADDE [[UADDE24]], [[UV59]], [[UADDO83]]
+    ; GFX9: [[MV10:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO82]](s32), [[UADDE26]](s32)
+    ; GFX9: [[USUBO16:%[0-9]+]]:_(s32), [[USUBO17:%[0-9]+]]:_(s1) = G_USUBO [[USUBO14]], [[UV54]]
+    ; GFX9: [[USUBE22:%[0-9]+]]:_(s32), [[USUBE23:%[0-9]+]]:_(s1) = G_USUBE [[USUBE18]], [[UV55]], [[USUBO15]]
+    ; GFX9: [[USUBE24:%[0-9]+]]:_(s32), [[USUBE25:%[0-9]+]]:_(s1) = G_USUBE [[USUBE22]], [[C6]], [[USUBO17]]
+    ; GFX9: [[ICMP14:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT5]](s32), [[C6]]
+    ; GFX9: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP14]](s1), [[MV10]], [[MV9]]
+    ; GFX9: [[ICMP15:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT4]](s32), [[C6]]
+    ; GFX9: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP15]](s1), [[SELECT6]], [[MV8]]
+    ; GFX9: [[XOR6:%[0-9]+]]:_(s64) = G_XOR [[ASHR2]], [[ASHR3]]
+    ; GFX9: [[XOR7:%[0-9]+]]:_(s64) = G_XOR [[SELECT7]], [[XOR6]]
+    ; GFX9: [[UV60:%[0-9]+]]:_(s32), [[UV61:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR7]](s64)
+    ; GFX9: [[UV62:%[0-9]+]]:_(s32), [[UV63:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR6]](s64)
+    ; GFX9: [[USUBO18:%[0-9]+]]:_(s32), [[USUBO19:%[0-9]+]]:_(s1) = G_USUBO [[UV60]], [[UV62]]
+    ; GFX9: [[USUBE26:%[0-9]+]]:_(s32), [[USUBE27:%[0-9]+]]:_(s1) = G_USUBE [[UV61]], [[UV63]], [[USUBO19]]
+    ; GFX9: [[MV11:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO18]](s32), [[USUBE26]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV5]](s64), [[MV11]](s64)
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
@@ -463,34 +1945,30 @@ body: |
     ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX6: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
-    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
+    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB3]](s32)
     ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: test_sdiv_s16
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
@@ -508,34 +1986,30 @@ body: |
     ; GFX8: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX8: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX8: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
-    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
+    ; GFX8: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB3]](s32)
     ; GFX8: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX9-LABEL: name: test_sdiv_s16
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
@@ -553,34 +2027,30 @@ body: |
     ; GFX9: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX9: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX9: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
-    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
+    ; GFX9: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB3]](s32)
     ; GFX9: $vgpr0 = COPY [[COPY4]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -618,77 +2088,69 @@ body: |
     ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C3]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX6: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[XOR1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C4]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C4]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C4]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C4]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
     ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
     ; GFX6: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 16
     ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
     ; GFX6: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 16
     ; GFX6: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG2]], [[C1]](s32)
     ; GFX6: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG3]], [[C1]](s32)
-    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG2]], [[ASHR2]]
-    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG3]], [[ASHR3]]
-    ; GFX6: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR2]]
-    ; GFX6: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR3]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG2]], [[ASHR2]]
+    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG3]], [[ASHR3]]
+    ; GFX6: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR2]]
+    ; GFX6: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[ADD6]], [[ASHR3]]
     ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[XOR5]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX6: [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C5]]
     ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[XOR5]]
-    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[XOR5]]
-    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL2]]
-    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C3]]
-    ; GFX6: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB5]], [[MUL2]]
-    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX6: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD6]], [[SUB6]]
-    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[XOR4]]
-    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[XOR5]]
-    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR4]], [[MUL3]]
-    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB7]](s32), [[XOR5]]
-    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR4]](s32), [[MUL3]]
-    ; GFX6: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[UMULH5]], [[C4]]
-    ; GFX6: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[UMULH5]], [[C4]]
-    ; GFX6: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[ADD7]], [[UMULH5]]
-    ; GFX6: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[SUB8]]
+    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[XOR5]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB4]], [[FPTOUI1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[XOR4]], [[ADD7]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[XOR5]]
+    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[XOR4]], [[MUL3]]
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB5]](s32), [[XOR5]]
+    ; GFX6: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[UMULH3]], [[C4]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[ADD8]], [[UMULH3]]
+    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB5]], [[XOR5]]
+    ; GFX6: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB6]], [[SUB5]]
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT4]](s32), [[XOR5]]
+    ; GFX6: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[SELECT3]], [[C4]]
+    ; GFX6: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD9]], [[SELECT3]]
     ; GFX6: [[XOR6:%[0-9]+]]:_(s32) = G_XOR [[ASHR2]], [[ASHR3]]
-    ; GFX6: [[XOR7:%[0-9]+]]:_(s32) = G_XOR [[SELECT7]], [[XOR6]]
-    ; GFX6: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[XOR7]], [[XOR6]]
+    ; GFX6: [[XOR7:%[0-9]+]]:_(s32) = G_XOR [[SELECT5]], [[XOR6]]
+    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR7]], [[XOR6]]
     ; GFX6: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
-    ; GFX6: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C6]]
-    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SUB9]](s32)
-    ; GFX6: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C6]]
-    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
-    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL]]
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SUB3]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C6]]
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SUB7]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C6]]
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX6: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX6: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX8-LABEL: name: test_sdiv_v2s16
@@ -712,77 +2174,69 @@ body: |
     ; GFX8: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C3]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX8: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[XOR1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C4]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C4]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C4]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C4]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX8: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX8: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
     ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
     ; GFX8: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 16
     ; GFX8: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
     ; GFX8: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 16
     ; GFX8: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG2]], [[C1]](s32)
     ; GFX8: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG3]], [[C1]](s32)
-    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG2]], [[ASHR2]]
-    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG3]], [[ASHR3]]
-    ; GFX8: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR2]]
-    ; GFX8: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR3]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG2]], [[ASHR2]]
+    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG3]], [[ASHR3]]
+    ; GFX8: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR2]]
+    ; GFX8: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[ADD6]], [[ASHR3]]
     ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[XOR5]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX8: [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C5]]
     ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[XOR5]]
-    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[XOR5]]
-    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL2]]
-    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C3]]
-    ; GFX8: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB5]], [[MUL2]]
-    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX8: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX8: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD6]], [[SUB6]]
-    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[XOR4]]
-    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[XOR5]]
-    ; GFX8: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR4]], [[MUL3]]
-    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB7]](s32), [[XOR5]]
-    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR4]](s32), [[MUL3]]
-    ; GFX8: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[UMULH5]], [[C4]]
-    ; GFX8: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[UMULH5]], [[C4]]
-    ; GFX8: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[ADD7]], [[UMULH5]]
-    ; GFX8: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[SUB8]]
+    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[XOR5]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB4]], [[FPTOUI1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[XOR4]], [[ADD7]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[XOR5]]
+    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[XOR4]], [[MUL3]]
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB5]](s32), [[XOR5]]
+    ; GFX8: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[UMULH3]], [[C4]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[ADD8]], [[UMULH3]]
+    ; GFX8: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB5]], [[XOR5]]
+    ; GFX8: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB6]], [[SUB5]]
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT4]](s32), [[XOR5]]
+    ; GFX8: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[SELECT3]], [[C4]]
+    ; GFX8: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD9]], [[SELECT3]]
     ; GFX8: [[XOR6:%[0-9]+]]:_(s32) = G_XOR [[ASHR2]], [[ASHR3]]
-    ; GFX8: [[XOR7:%[0-9]+]]:_(s32) = G_XOR [[SELECT7]], [[XOR6]]
-    ; GFX8: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[XOR7]], [[XOR6]]
+    ; GFX8: [[XOR7:%[0-9]+]]:_(s32) = G_XOR [[SELECT5]], [[XOR6]]
+    ; GFX8: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR7]], [[XOR6]]
     ; GFX8: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; GFX8: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
-    ; GFX8: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C6]]
-    ; GFX8: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SUB9]](s32)
-    ; GFX8: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C6]]
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
-    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL]]
+    ; GFX8: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SUB3]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C6]]
+    ; GFX8: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SUB7]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C6]]
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX9-LABEL: name: test_sdiv_v2s16
@@ -806,72 +2260,64 @@ body: |
     ; GFX9: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C3]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX9: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[XOR1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C4]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C4]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C4]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C4]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX9: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX9: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
     ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
     ; GFX9: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 16
     ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
     ; GFX9: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 16
     ; GFX9: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG2]], [[C1]](s32)
     ; GFX9: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG3]], [[C1]](s32)
-    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG2]], [[ASHR2]]
-    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG3]], [[ASHR3]]
-    ; GFX9: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR2]]
-    ; GFX9: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR3]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG2]], [[ASHR2]]
+    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG3]], [[ASHR3]]
+    ; GFX9: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR2]]
+    ; GFX9: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[ADD6]], [[ASHR3]]
     ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[XOR5]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C5]]
     ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[XOR5]]
-    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[XOR5]]
-    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL2]]
-    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C3]]
-    ; GFX9: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB5]], [[MUL2]]
-    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX9: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD6]], [[SUB6]]
-    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[XOR4]]
-    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[XOR5]]
-    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR4]], [[MUL3]]
-    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB7]](s32), [[XOR5]]
-    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR4]](s32), [[MUL3]]
-    ; GFX9: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[UMULH5]], [[C4]]
-    ; GFX9: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[UMULH5]], [[C4]]
-    ; GFX9: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[ADD7]], [[UMULH5]]
-    ; GFX9: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[SUB8]]
+    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[XOR5]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB4]], [[FPTOUI1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[XOR4]], [[ADD7]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[XOR5]]
+    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[XOR4]], [[MUL3]]
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB5]](s32), [[XOR5]]
+    ; GFX9: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[UMULH3]], [[C4]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[ADD8]], [[UMULH3]]
+    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB5]], [[XOR5]]
+    ; GFX9: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB6]], [[SUB5]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT4]](s32), [[XOR5]]
+    ; GFX9: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[SELECT3]], [[C4]]
+    ; GFX9: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD9]], [[SELECT3]]
     ; GFX9: [[XOR6:%[0-9]+]]:_(s32) = G_XOR [[ASHR2]], [[ASHR3]]
-    ; GFX9: [[XOR7:%[0-9]+]]:_(s32) = G_XOR [[SELECT7]], [[XOR6]]
-    ; GFX9: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[XOR7]], [[XOR6]]
-    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
-    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SUB9]](s32)
+    ; GFX9: [[XOR7:%[0-9]+]]:_(s32) = G_XOR [[SELECT5]], [[XOR6]]
+    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR7]], [[XOR6]]
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SUB3]](s32)
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SUB7]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[COPY7]](s32)
     ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR_TRUNC]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
@@ -902,34 +2348,30 @@ body: |
     ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX6: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
-    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
+    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB3]](s32)
     ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: test_sdiv_s7
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
@@ -947,34 +2389,30 @@ body: |
     ; GFX8: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX8: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX8: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
-    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
+    ; GFX8: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB3]](s32)
     ; GFX8: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX9-LABEL: name: test_sdiv_s7
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
@@ -992,34 +2430,30 @@ body: |
     ; GFX9: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX9: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX9: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
-    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
+    ; GFX9: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB3]](s32)
     ; GFX9: $vgpr0 = COPY [[COPY4]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -1052,34 +2486,30 @@ body: |
     ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX6: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
-    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
+    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB3]](s32)
     ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: test_sdiv_s17
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
@@ -1097,34 +2527,30 @@ body: |
     ; GFX8: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX8: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX8: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
-    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
+    ; GFX8: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB3]](s32)
     ; GFX8: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX9-LABEL: name: test_sdiv_s17
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
@@ -1142,34 +2568,30 @@ body: |
     ; GFX9: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX9: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
     ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD3]], [[UMULH2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD3]], [[UMULH1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[XOR1]]
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD4]], [[SELECT]]
     ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ASHR]], [[ASHR1]]
-    ; GFX9: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[XOR2]]
-    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
-    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
+    ; GFX9: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SELECT2]], [[XOR2]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[XOR2]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB3]](s32)
     ; GFX9: $vgpr0 = COPY [[COPY4]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -1189,36 +2611,543 @@ body: |
     ; GFX6-LABEL: name: test_sdiv_s33
     ; GFX6: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX6: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s33) = G_TRUNC [[COPY]](s64)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s33) = G_TRUNC [[COPY1]](s64)
-    ; GFX6: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[TRUNC]](s33)
-    ; GFX6: [[SEXT1:%[0-9]+]]:_(s64) = G_SEXT [[TRUNC1]](s33)
-    ; GFX6: [[SDIV:%[0-9]+]]:_(s64) = G_SDIV [[SEXT]], [[SEXT1]]
-    ; GFX6: [[TRUNC2:%[0-9]+]]:_(s33) = G_TRUNC [[SDIV]](s64)
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC2]](s33)
-    ; GFX6: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+    ; GFX6: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY2]], 33
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
+    ; GFX6: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY3]], 33
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SEXT_INREG]], [[C]](s32)
+    ; GFX6: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SEXT_INREG1]], [[C]](s32)
+    ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SEXT_INREG]](s64)
+    ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SEXT_INREG1]](s64)
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX6: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX6: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX6: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX6: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV8]](s32)
+    ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV9]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX6: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX6: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX6: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX6: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX6: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[UV12]]
+    ; GFX6: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[UV13]], [[USUBO1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX6: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX6: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX6: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX6: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX6: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX6: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX6: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX6: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX6: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX6: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX6: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX6: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX6: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX6: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX6: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX6: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX6: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX6: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX6: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX6: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX6: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX6: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX6: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX6: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX6: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX6: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX6: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX6: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX6: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX6: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX6: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX6: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX6: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX6: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX6: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX6: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX6: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX6: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX6: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX6: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDO26]]
+    ; GFX6: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV16]], [[UADDE8]]
+    ; GFX6: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDO26]]
+    ; GFX6: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX6: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX6: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX6: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX6: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX6: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDE8]]
+    ; GFX6: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDO26]]
+    ; GFX6: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDE8]]
+    ; GFX6: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX6: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX6: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX6: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX6: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX6: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX6: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX6: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX6: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDE8]]
+    ; GFX6: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX6: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[ADD16]](s32)
+    ; GFX6: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[UADDO36]]
+    ; GFX6: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV19]], [[UADDO36]]
+    ; GFX6: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[ADD16]]
+    ; GFX6: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV18]], [[UADDO36]]
+    ; GFX6: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX6: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX6: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[MUL15]]
+    ; GFX6: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[ADD18]], [[USUBO3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV15]], [[ADD18]]
+    ; GFX6: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV21]]
+    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV20]]
+    ; GFX6: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV21]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX6: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV20]]
+    ; GFX6: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV21]], [[USUBO3]]
+    ; GFX6: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX6: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX6: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX6: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[UADDO36]], [[UV22]]
+    ; GFX6: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV23]], [[UADDO39]]
+    ; GFX6: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO38]](s32), [[UADDE10]](s32)
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV21]]
+    ; GFX6: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV20]]
+    ; GFX6: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV21]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX6: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX6: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UADDO38]], [[UV24]]
+    ; GFX6: [[UADDE12:%[0-9]+]]:_(s32), [[UADDE13:%[0-9]+]]:_(s1) = G_UADDE [[UADDE10]], [[UV25]], [[UADDO41]]
+    ; GFX6: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO40]](s32), [[UADDE12]](s32)
+    ; GFX6: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV20]]
+    ; GFX6: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV21]], [[USUBO5]]
+    ; GFX6: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX6: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX6: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX6: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[ASHR]], [[ASHR1]]
+    ; GFX6: [[XOR3:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[XOR2]]
+    ; GFX6: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR3]](s64)
+    ; GFX6: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX6: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV26]], [[UV28]]
+    ; GFX6: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV27]], [[UV29]], [[USUBO9]]
+    ; GFX6: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s64) = COPY [[MV5]](s64)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[COPY4]](s64)
     ; GFX8-LABEL: name: test_sdiv_s33
     ; GFX8: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX8: [[TRUNC:%[0-9]+]]:_(s33) = G_TRUNC [[COPY]](s64)
-    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s33) = G_TRUNC [[COPY1]](s64)
-    ; GFX8: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[TRUNC]](s33)
-    ; GFX8: [[SEXT1:%[0-9]+]]:_(s64) = G_SEXT [[TRUNC1]](s33)
-    ; GFX8: [[SDIV:%[0-9]+]]:_(s64) = G_SDIV [[SEXT]], [[SEXT1]]
-    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s33) = G_TRUNC [[SDIV]](s64)
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC2]](s33)
-    ; GFX8: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+    ; GFX8: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY2]], 33
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
+    ; GFX8: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY3]], 33
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SEXT_INREG]], [[C]](s32)
+    ; GFX8: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SEXT_INREG1]], [[C]](s32)
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SEXT_INREG]](s64)
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SEXT_INREG1]](s64)
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX8: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX8: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX8: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX8: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV8]](s32)
+    ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV9]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX8: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX8: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX8: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX8: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX8: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX8: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX8: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[UV12]]
+    ; GFX8: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[UV13]], [[USUBO1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX8: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX8: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX8: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX8: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX8: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX8: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX8: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX8: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX8: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX8: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX8: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX8: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX8: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX8: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX8: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX8: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX8: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX8: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX8: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX8: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX8: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX8: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX8: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX8: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX8: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX8: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX8: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX8: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX8: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX8: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX8: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX8: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX8: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX8: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX8: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX8: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX8: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX8: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDO26]]
+    ; GFX8: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV16]], [[UADDE8]]
+    ; GFX8: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDO26]]
+    ; GFX8: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX8: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX8: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX8: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX8: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX8: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDE8]]
+    ; GFX8: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDO26]]
+    ; GFX8: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDE8]]
+    ; GFX8: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX8: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX8: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX8: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX8: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX8: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX8: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX8: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX8: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDE8]]
+    ; GFX8: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX8: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[ADD16]](s32)
+    ; GFX8: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[UADDO36]]
+    ; GFX8: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV19]], [[UADDO36]]
+    ; GFX8: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[ADD16]]
+    ; GFX8: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV18]], [[UADDO36]]
+    ; GFX8: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX8: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX8: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[MUL15]]
+    ; GFX8: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[ADD18]], [[USUBO3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV15]], [[ADD18]]
+    ; GFX8: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV21]]
+    ; GFX8: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV20]]
+    ; GFX8: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV21]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX8: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV20]]
+    ; GFX8: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV21]], [[USUBO3]]
+    ; GFX8: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX8: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX8: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX8: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[UADDO36]], [[UV22]]
+    ; GFX8: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV23]], [[UADDO39]]
+    ; GFX8: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO38]](s32), [[UADDE10]](s32)
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV21]]
+    ; GFX8: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV20]]
+    ; GFX8: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV21]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX8: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX8: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UADDO38]], [[UV24]]
+    ; GFX8: [[UADDE12:%[0-9]+]]:_(s32), [[UADDE13:%[0-9]+]]:_(s1) = G_UADDE [[UADDE10]], [[UV25]], [[UADDO41]]
+    ; GFX8: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO40]](s32), [[UADDE12]](s32)
+    ; GFX8: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV20]]
+    ; GFX8: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV21]], [[USUBO5]]
+    ; GFX8: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX8: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX8: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX8: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[ASHR]], [[ASHR1]]
+    ; GFX8: [[XOR3:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[XOR2]]
+    ; GFX8: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR3]](s64)
+    ; GFX8: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX8: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV26]], [[UV28]]
+    ; GFX8: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV27]], [[UV29]], [[USUBO9]]
+    ; GFX8: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s64) = COPY [[MV5]](s64)
+    ; GFX8: $vgpr0_vgpr1 = COPY [[COPY4]](s64)
     ; GFX9-LABEL: name: test_sdiv_s33
     ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(s33) = G_TRUNC [[COPY]](s64)
-    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s33) = G_TRUNC [[COPY1]](s64)
-    ; GFX9: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[TRUNC]](s33)
-    ; GFX9: [[SEXT1:%[0-9]+]]:_(s64) = G_SEXT [[TRUNC1]](s33)
-    ; GFX9: [[SDIV:%[0-9]+]]:_(s64) = G_SDIV [[SEXT]], [[SEXT1]]
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s33) = G_TRUNC [[SDIV]](s64)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC2]](s33)
-    ; GFX9: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+    ; GFX9: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY2]], 33
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
+    ; GFX9: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY3]], 33
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SEXT_INREG]], [[C]](s32)
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SEXT_INREG1]], [[C]](s32)
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SEXT_INREG]](s64)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SEXT_INREG1]](s64)
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX9: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX9: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV8]](s32)
+    ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV9]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX9: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX9: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX9: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX9: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX9: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[UV12]]
+    ; GFX9: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[UV13]], [[USUBO1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX9: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX9: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX9: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX9: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX9: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX9: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX9: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX9: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX9: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX9: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX9: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX9: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX9: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX9: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX9: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX9: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX9: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX9: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX9: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX9: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX9: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX9: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX9: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX9: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX9: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX9: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX9: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX9: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX9: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX9: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX9: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX9: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX9: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX9: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX9: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDO26]]
+    ; GFX9: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV16]], [[UADDE8]]
+    ; GFX9: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDO26]]
+    ; GFX9: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX9: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX9: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX9: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX9: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX9: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDE8]]
+    ; GFX9: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDO26]]
+    ; GFX9: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDE8]]
+    ; GFX9: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX9: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX9: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX9: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX9: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX9: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX9: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX9: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX9: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDE8]]
+    ; GFX9: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX9: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[ADD16]](s32)
+    ; GFX9: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[UADDO36]]
+    ; GFX9: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV19]], [[UADDO36]]
+    ; GFX9: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[ADD16]]
+    ; GFX9: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV18]], [[UADDO36]]
+    ; GFX9: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX9: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX9: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[MUL15]]
+    ; GFX9: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[ADD18]], [[USUBO3]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV15]], [[ADD18]]
+    ; GFX9: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV21]]
+    ; GFX9: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV20]]
+    ; GFX9: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV21]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX9: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV20]]
+    ; GFX9: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV21]], [[USUBO3]]
+    ; GFX9: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX9: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX9: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX9: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[UADDO36]], [[UV22]]
+    ; GFX9: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV23]], [[UADDO39]]
+    ; GFX9: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO38]](s32), [[UADDE10]](s32)
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV21]]
+    ; GFX9: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV20]]
+    ; GFX9: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV21]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX9: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX9: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UADDO38]], [[UV24]]
+    ; GFX9: [[UADDE12:%[0-9]+]]:_(s32), [[UADDE13:%[0-9]+]]:_(s1) = G_UADDE [[UADDE10]], [[UV25]], [[UADDO41]]
+    ; GFX9: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO40]](s32), [[UADDE12]](s32)
+    ; GFX9: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV20]]
+    ; GFX9: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV21]], [[USUBO5]]
+    ; GFX9: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX9: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX9: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX9: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[ASHR]], [[ASHR1]]
+    ; GFX9: [[XOR3:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[XOR2]]
+    ; GFX9: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR3]](s64)
+    ; GFX9: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX9: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV26]], [[UV28]]
+    ; GFX9: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV27]], [[UV29]], [[USUBO9]]
+    ; GFX9: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s64) = COPY [[MV5]](s64)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[COPY4]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s33) = G_TRUNC %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
index 4641c846d8ff9..5f5697f5fddc8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
@@ -1003,7 +1003,7 @@ body: |
     %3:_(s32) = G_CONSTANT i32 0
     %4:_(s1) = G_ICMP intpred(ne), %2, %3
     %5:_(<2 x s128>) = G_SELECT %4, %0, %1
-    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7   = COPY %5
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %5
 
 ...
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext-inreg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext-inreg.mir
index 55fa4fb210176..a2cbe94e56c09 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext-inreg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext-inreg.mir
@@ -1149,5 +1149,5 @@ body: |
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<2 x s128>)
     %0:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     %1:_(<2 x s128>) = G_SEXT_INREG %0, 1
-    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7  = COPY %1
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %1
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
index 0d818d850217e..10e32d7f87c90 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
@@ -1550,7 +1550,7 @@ body: |
     %1:_(s32) = COPY $vgpr8
     %2:_(s256) = G_ZEXT %1
     %3:_(s256) = G_SHL %0, %2
-    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7  = COPY %3
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %3
 ...
 
 ---
@@ -1676,5 +1676,5 @@ body: |
     %0:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     %1:_(<2 x s32>) = COPY $vgpr4_vgpr5
     %2:_(<2 x s128>) = G_SHL %0, %1
-    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7  = COPY %2
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %2
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-srem.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-srem.mir
index 0eb03135e18bd..74c6ce7db0e02 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-srem.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-srem.mir
@@ -21,30 +21,24 @@ body: |
     ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX6: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX6: $vgpr0 = COPY [[SUB4]](s32)
     ; GFX8-LABEL: name: test_srem_s32
@@ -59,30 +53,24 @@ body: |
     ; GFX8: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX8: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX8: $vgpr0 = COPY [[SUB4]](s32)
     ; GFX9-LABEL: name: test_srem_s32
@@ -97,30 +85,24 @@ body: |
     ; GFX9: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX9: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX9: $vgpr0 = COPY [[SUB4]](s32)
     %0:_(s32) = COPY $vgpr0
@@ -149,62 +131,50 @@ body: |
     ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX6: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX6: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[UV1]], [[C]](s32)
     ; GFX6: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[UV3]], [[C]](s32)
-    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[ASHR2]]
-    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UV3]], [[ASHR3]]
-    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR2]]
-    ; GFX6: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR3]]
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[ASHR2]]
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[UV3]], [[ASHR3]]
+    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[ADD3]], [[ASHR2]]
+    ; GFX6: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR3]]
     ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[XOR4]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C3]]
     ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[XOR4]]
-    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[XOR4]]
-    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL2]]
-    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C2]]
-    ; GFX6: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB5]], [[MUL2]]
-    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX6: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD6]], [[SUB6]]
-    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[XOR3]]
-    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[XOR4]]
-    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[MUL3]]
-    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB7]](s32), [[XOR4]]
-    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR3]](s32), [[MUL3]]
-    ; GFX6: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX6: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB7]], [[XOR4]]
-    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[SUB7]], [[XOR4]]
-    ; GFX6: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[SUB8]], [[SUB7]]
-    ; GFX6: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[ADD7]]
-    ; GFX6: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[SELECT7]], [[ASHR2]]
+    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR4]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB5]], [[FPTOUI1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[XOR3]], [[ADD5]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[XOR4]]
+    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[MUL3]]
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[XOR4]]
+    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SUB6]], [[XOR4]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB7]], [[SUB6]]
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT2]](s32), [[XOR4]]
+    ; GFX6: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SELECT2]], [[XOR4]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB8]], [[SELECT2]]
+    ; GFX6: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR2]]
     ; GFX6: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[XOR5]], [[ASHR2]]
     ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB4]](s32), [[SUB9]](s32)
     ; GFX6: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
@@ -222,62 +192,50 @@ body: |
     ; GFX8: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX8: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX8: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[UV1]], [[C]](s32)
     ; GFX8: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[UV3]], [[C]](s32)
-    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[ASHR2]]
-    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UV3]], [[ASHR3]]
-    ; GFX8: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR2]]
-    ; GFX8: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR3]]
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[ASHR2]]
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[UV3]], [[ASHR3]]
+    ; GFX8: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[ADD3]], [[ASHR2]]
+    ; GFX8: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR3]]
     ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[XOR4]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C3]]
     ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[XOR4]]
-    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[XOR4]]
-    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL2]]
-    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C2]]
-    ; GFX8: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB5]], [[MUL2]]
-    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX8: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX8: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD6]], [[SUB6]]
-    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[XOR3]]
-    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[XOR4]]
-    ; GFX8: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[MUL3]]
-    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB7]](s32), [[XOR4]]
-    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR3]](s32), [[MUL3]]
-    ; GFX8: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX8: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB7]], [[XOR4]]
-    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[SUB7]], [[XOR4]]
-    ; GFX8: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[SUB8]], [[SUB7]]
-    ; GFX8: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[ADD7]]
-    ; GFX8: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[SELECT7]], [[ASHR2]]
+    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR4]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB5]], [[FPTOUI1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[XOR3]], [[ADD5]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[XOR4]]
+    ; GFX8: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[MUL3]]
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[XOR4]]
+    ; GFX8: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SUB6]], [[XOR4]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB7]], [[SUB6]]
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT2]](s32), [[XOR4]]
+    ; GFX8: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SELECT2]], [[XOR4]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB8]], [[SELECT2]]
+    ; GFX8: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR2]]
     ; GFX8: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[XOR5]], [[ASHR2]]
     ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB4]](s32), [[SUB9]](s32)
     ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
@@ -295,62 +253,50 @@ body: |
     ; GFX9: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX9: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX9: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[UV1]], [[C]](s32)
     ; GFX9: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[UV3]], [[C]](s32)
-    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[ASHR2]]
-    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UV3]], [[ASHR3]]
-    ; GFX9: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR2]]
-    ; GFX9: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR3]]
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[ASHR2]]
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[UV3]], [[ASHR3]]
+    ; GFX9: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[ADD3]], [[ASHR2]]
+    ; GFX9: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR3]]
     ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[XOR4]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C3]]
     ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[XOR4]]
-    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[XOR4]]
-    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL2]]
-    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C2]]
-    ; GFX9: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB5]], [[MUL2]]
-    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX9: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD6]], [[SUB6]]
-    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[XOR3]]
-    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[XOR4]]
-    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[MUL3]]
-    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB7]](s32), [[XOR4]]
-    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR3]](s32), [[MUL3]]
-    ; GFX9: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX9: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB7]], [[XOR4]]
-    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[SUB7]], [[XOR4]]
-    ; GFX9: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[SUB8]], [[SUB7]]
-    ; GFX9: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[ADD7]]
-    ; GFX9: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[SELECT7]], [[ASHR2]]
+    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR4]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB5]], [[FPTOUI1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[XOR3]], [[ADD5]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[XOR4]]
+    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[MUL3]]
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[XOR4]]
+    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SUB6]], [[XOR4]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB7]], [[SUB6]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT2]](s32), [[XOR4]]
+    ; GFX9: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SELECT2]], [[XOR4]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB8]], [[SELECT2]]
+    ; GFX9: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR2]]
     ; GFX9: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[XOR5]], [[ASHR2]]
     ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB4]](s32), [[SUB9]](s32)
     ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
@@ -369,18 +315,504 @@ body: |
     ; GFX6-LABEL: name: test_srem_s64
     ; GFX6: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX6: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX6: [[SREM:%[0-9]+]]:_(s64) = G_SREM [[COPY]], [[COPY1]]
-    ; GFX6: $vgpr0_vgpr1 = COPY [[SREM]](s64)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[C]](s32)
+    ; GFX6: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY1]], [[C]](s32)
+    ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX6: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX6: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX6: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX6: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV8]](s32)
+    ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV9]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX6: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX6: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX6: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX6: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX6: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[UV12]]
+    ; GFX6: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[UV13]], [[USUBO1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX6: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX6: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX6: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX6: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX6: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX6: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX6: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX6: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX6: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX6: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX6: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX6: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX6: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX6: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX6: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX6: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX6: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX6: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX6: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX6: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX6: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX6: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX6: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX6: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX6: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX6: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX6: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX6: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX6: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX6: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX6: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX6: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX6: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX6: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX6: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX6: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX6: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX6: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX6: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX6: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDO26]]
+    ; GFX6: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV16]], [[UADDE8]]
+    ; GFX6: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDO26]]
+    ; GFX6: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX6: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX6: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX6: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX6: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX6: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDE8]]
+    ; GFX6: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDO26]]
+    ; GFX6: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDE8]]
+    ; GFX6: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX6: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX6: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX6: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX6: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX6: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX6: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX6: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX6: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDE8]]
+    ; GFX6: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX6: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[UADDO36]]
+    ; GFX6: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV19]], [[UADDO36]]
+    ; GFX6: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[ADD16]]
+    ; GFX6: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV18]], [[UADDO36]]
+    ; GFX6: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX6: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX6: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[MUL15]]
+    ; GFX6: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[ADD18]], [[USUBO3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV15]], [[ADD18]]
+    ; GFX6: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX6: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV21]]
+    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV20]]
+    ; GFX6: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV21]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX6: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV20]]
+    ; GFX6: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV21]], [[USUBO3]]
+    ; GFX6: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX6: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV21]]
+    ; GFX6: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV20]]
+    ; GFX6: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV21]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX6: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV20]]
+    ; GFX6: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV21]], [[USUBO5]]
+    ; GFX6: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX6: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX6: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX6: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX6: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX6: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX6: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX6: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV22]], [[UV24]]
+    ; GFX6: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV23]], [[UV25]], [[USUBO9]]
+    ; GFX6: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[MV5]](s64)
     ; GFX8-LABEL: name: test_srem_s64
     ; GFX8: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX8: [[SREM:%[0-9]+]]:_(s64) = G_SREM [[COPY]], [[COPY1]]
-    ; GFX8: $vgpr0_vgpr1 = COPY [[SREM]](s64)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[C]](s32)
+    ; GFX8: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY1]], [[C]](s32)
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX8: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX8: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX8: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX8: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV8]](s32)
+    ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV9]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX8: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX8: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX8: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX8: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX8: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX8: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX8: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[UV12]]
+    ; GFX8: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[UV13]], [[USUBO1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX8: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX8: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX8: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX8: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX8: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX8: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX8: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX8: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX8: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX8: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX8: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX8: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX8: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX8: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX8: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX8: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX8: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX8: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX8: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX8: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX8: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX8: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX8: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX8: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX8: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX8: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX8: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX8: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX8: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX8: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX8: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX8: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX8: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX8: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX8: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX8: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX8: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX8: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDO26]]
+    ; GFX8: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV16]], [[UADDE8]]
+    ; GFX8: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDO26]]
+    ; GFX8: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX8: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX8: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX8: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX8: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX8: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDE8]]
+    ; GFX8: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDO26]]
+    ; GFX8: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDE8]]
+    ; GFX8: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX8: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX8: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX8: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX8: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX8: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX8: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX8: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX8: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDE8]]
+    ; GFX8: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX8: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[UADDO36]]
+    ; GFX8: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV19]], [[UADDO36]]
+    ; GFX8: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[ADD16]]
+    ; GFX8: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV18]], [[UADDO36]]
+    ; GFX8: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX8: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX8: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[MUL15]]
+    ; GFX8: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[ADD18]], [[USUBO3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV15]], [[ADD18]]
+    ; GFX8: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX8: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV21]]
+    ; GFX8: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV20]]
+    ; GFX8: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV21]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX8: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV20]]
+    ; GFX8: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV21]], [[USUBO3]]
+    ; GFX8: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX8: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV21]]
+    ; GFX8: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV20]]
+    ; GFX8: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV21]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX8: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV20]]
+    ; GFX8: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV21]], [[USUBO5]]
+    ; GFX8: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX8: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX8: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX8: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX8: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX8: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX8: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX8: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV22]], [[UV24]]
+    ; GFX8: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV23]], [[UV25]], [[USUBO9]]
+    ; GFX8: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX8: $vgpr0_vgpr1 = COPY [[MV5]](s64)
     ; GFX9-LABEL: name: test_srem_s64
     ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX9: [[SREM:%[0-9]+]]:_(s64) = G_SREM [[COPY]], [[COPY1]]
-    ; GFX9: $vgpr0_vgpr1 = COPY [[SREM]](s64)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[C]](s32)
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY1]], [[C]](s32)
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX9: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX9: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV8]](s32)
+    ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV9]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX9: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX9: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX9: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX9: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX9: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[UV12]]
+    ; GFX9: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[UV13]], [[USUBO1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX9: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX9: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX9: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX9: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX9: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX9: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX9: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX9: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX9: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX9: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX9: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX9: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX9: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX9: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX9: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX9: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX9: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX9: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX9: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX9: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX9: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX9: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX9: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX9: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX9: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX9: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX9: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX9: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX9: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX9: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX9: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX9: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX9: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX9: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX9: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDO26]]
+    ; GFX9: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV16]], [[UADDE8]]
+    ; GFX9: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDO26]]
+    ; GFX9: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX9: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX9: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX9: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX9: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX9: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDE8]]
+    ; GFX9: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDO26]]
+    ; GFX9: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDE8]]
+    ; GFX9: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX9: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX9: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX9: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX9: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX9: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX9: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX9: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX9: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDE8]]
+    ; GFX9: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX9: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[UADDO36]]
+    ; GFX9: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV19]], [[UADDO36]]
+    ; GFX9: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[ADD16]]
+    ; GFX9: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV18]], [[UADDO36]]
+    ; GFX9: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX9: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX9: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[MUL15]]
+    ; GFX9: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[ADD18]], [[USUBO3]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV15]], [[ADD18]]
+    ; GFX9: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX9: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV21]]
+    ; GFX9: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV20]]
+    ; GFX9: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV21]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX9: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV20]]
+    ; GFX9: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV21]], [[USUBO3]]
+    ; GFX9: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX9: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV21]]
+    ; GFX9: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV20]]
+    ; GFX9: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV21]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX9: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV20]]
+    ; GFX9: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV21]], [[USUBO5]]
+    ; GFX9: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX9: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX9: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX9: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX9: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX9: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX9: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX9: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV22]], [[UV24]]
+    ; GFX9: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV23]], [[UV25]], [[USUBO9]]
+    ; GFX9: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[MV5]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s64) = G_SREM %0, %1
@@ -398,27 +830,990 @@ body: |
     ; GFX6: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX6: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX6: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX6: [[SREM:%[0-9]+]]:_(s64) = G_SREM [[UV]], [[UV2]]
-    ; GFX6: [[SREM1:%[0-9]+]]:_(s64) = G_SREM [[UV1]], [[UV3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SREM]](s64), [[SREM1]](s64)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV]], [[C]](s32)
+    ; GFX6: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV2]], [[C]](s32)
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX6: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX6: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]]
+    ; GFX6: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO3]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX6: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX6: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX6: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV12]](s32)
+    ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV13]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX6: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX6: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX6: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX6: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX6: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[UV16]]
+    ; GFX6: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[UV17]], [[USUBO1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX6: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX6: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX6: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX6: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX6: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX6: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX6: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX6: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX6: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX6: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX6: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX6: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX6: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX6: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX6: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX6: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX6: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX6: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX6: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX6: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX6: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX6: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX6: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX6: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX6: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX6: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX6: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX6: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX6: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX6: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX6: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX6: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX6: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX6: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX6: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX6: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX6: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX6: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX6: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX6: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV21]], [[UADDO26]]
+    ; GFX6: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV20]], [[UADDE8]]
+    ; GFX6: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV20]], [[UADDO26]]
+    ; GFX6: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX6: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX6: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX6: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX6: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX6: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV21]], [[UADDE8]]
+    ; GFX6: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV21]], [[UADDO26]]
+    ; GFX6: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV20]], [[UADDE8]]
+    ; GFX6: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX6: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX6: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX6: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX6: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX6: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX6: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX6: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX6: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV21]], [[UADDE8]]
+    ; GFX6: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX6: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV22]], [[UADDO36]]
+    ; GFX6: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV23]], [[UADDO36]]
+    ; GFX6: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV22]], [[ADD16]]
+    ; GFX6: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV22]], [[UADDO36]]
+    ; GFX6: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX6: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX6: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV18]], [[MUL15]]
+    ; GFX6: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV19]], [[ADD18]], [[USUBO3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV19]], [[ADD18]]
+    ; GFX6: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX6: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV25]]
+    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV24]]
+    ; GFX6: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV25]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX6: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV24]]
+    ; GFX6: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV25]], [[USUBO3]]
+    ; GFX6: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX6: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV25]]
+    ; GFX6: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV24]]
+    ; GFX6: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV25]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX6: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV24]]
+    ; GFX6: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV25]], [[USUBO5]]
+    ; GFX6: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX6: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX6: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX6: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX6: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX6: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX6: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX6: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV26]], [[UV28]]
+    ; GFX6: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV27]], [[UV29]], [[USUBO9]]
+    ; GFX6: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX6: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; GFX6: [[ASHR3:%[0-9]+]]:_(s64) = G_ASHR [[UV3]], [[C]](s32)
+    ; GFX6: [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX6: [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR2]](s64)
+    ; GFX6: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[UV30]], [[UV32]]
+    ; GFX6: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[UV31]], [[UV33]], [[UADDO39]]
+    ; GFX6: [[MV6:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO38]](s32), [[UADDE10]](s32)
+    ; GFX6: [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX6: [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR3]](s64)
+    ; GFX6: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UV34]], [[UV36]]
+    ; GFX6: [[UADDE12:%[0-9]+]]:_(s32), [[UADDE13:%[0-9]+]]:_(s1) = G_UADDE [[UV35]], [[UV37]], [[UADDO41]]
+    ; GFX6: [[MV7:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO40]](s32), [[UADDE12]](s32)
+    ; GFX6: [[XOR3:%[0-9]+]]:_(s64) = G_XOR [[MV6]], [[ASHR2]]
+    ; GFX6: [[XOR4:%[0-9]+]]:_(s64) = G_XOR [[MV7]], [[ASHR3]]
+    ; GFX6: [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX6: [[UITOFP2:%[0-9]+]]:_(s32) = G_UITOFP [[UV38]](s32)
+    ; GFX6: [[UITOFP3:%[0-9]+]]:_(s32) = G_UITOFP [[UV39]](s32)
+    ; GFX6: [[C7:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[FMUL4:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP3]], [[C7]]
+    ; GFX6: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL4]], [[UITOFP2]]
+    ; GFX6: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD2]](s32)
+    ; GFX6: [[C8:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX6: [[FMUL5:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C8]]
+    ; GFX6: [[C9:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX6: [[FMUL6:%[0-9]+]]:_(s32) = G_FMUL [[FMUL5]], [[C9]]
+    ; GFX6: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL6]]
+    ; GFX6: [[C10:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX6: [[FMUL7:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC1]], [[C10]]
+    ; GFX6: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL7]], [[FMUL5]]
+    ; GFX6: [[FPTOUI2:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD3]](s32)
+    ; GFX6: [[FPTOUI3:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC1]](s32)
+    ; GFX6: [[UV40:%[0-9]+]]:_(s32), [[UV41:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX6: [[UV42:%[0-9]+]]:_(s32), [[UV43:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX6: [[USUBO10:%[0-9]+]]:_(s32), [[USUBO11:%[0-9]+]]:_(s1) = G_USUBO [[UV40]], [[UV42]]
+    ; GFX6: [[USUBE14:%[0-9]+]]:_(s32), [[USUBE15:%[0-9]+]]:_(s1) = G_USUBE [[UV41]], [[UV43]], [[USUBO11]]
+    ; GFX6: [[MUL18:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[FPTOUI2]]
+    ; GFX6: [[MUL19:%[0-9]+]]:_(s32) = G_MUL [[USUBE14]], [[FPTOUI2]]
+    ; GFX6: [[MUL20:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[FPTOUI3]]
+    ; GFX6: [[UMULH15:%[0-9]+]]:_(s32) = G_UMULH [[USUBO10]], [[FPTOUI2]]
+    ; GFX6: [[ADD19:%[0-9]+]]:_(s32) = G_ADD [[MUL19]], [[MUL20]]
+    ; GFX6: [[ADD20:%[0-9]+]]:_(s32) = G_ADD [[ADD19]], [[UMULH15]]
+    ; GFX6: [[MUL21:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[MUL18]]
+    ; GFX6: [[MUL22:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI2]], [[ADD20]]
+    ; GFX6: [[UMULH16:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[MUL18]]
+    ; GFX6: [[UADDO42:%[0-9]+]]:_(s32), [[UADDO43:%[0-9]+]]:_(s1) = G_UADDO [[MUL21]], [[MUL22]]
+    ; GFX6: [[ZEXT15:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO43]](s1)
+    ; GFX6: [[UADDO44:%[0-9]+]]:_(s32), [[UADDO45:%[0-9]+]]:_(s1) = G_UADDO [[UADDO42]], [[UMULH16]]
+    ; GFX6: [[ZEXT16:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO45]](s1)
+    ; GFX6: [[ADD21:%[0-9]+]]:_(s32) = G_ADD [[ZEXT15]], [[ZEXT16]]
+    ; GFX6: [[MUL23:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[ADD20]]
+    ; GFX6: [[UMULH17:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[MUL18]]
+    ; GFX6: [[UMULH18:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[ADD20]]
+    ; GFX6: [[UADDO46:%[0-9]+]]:_(s32), [[UADDO47:%[0-9]+]]:_(s1) = G_UADDO [[MUL23]], [[UMULH17]]
+    ; GFX6: [[ZEXT17:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO47]](s1)
+    ; GFX6: [[UADDO48:%[0-9]+]]:_(s32), [[UADDO49:%[0-9]+]]:_(s1) = G_UADDO [[UADDO46]], [[UMULH18]]
+    ; GFX6: [[ZEXT18:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO49]](s1)
+    ; GFX6: [[ADD22:%[0-9]+]]:_(s32) = G_ADD [[ZEXT17]], [[ZEXT18]]
+    ; GFX6: [[UADDO50:%[0-9]+]]:_(s32), [[UADDO51:%[0-9]+]]:_(s1) = G_UADDO [[UADDO48]], [[ADD21]]
+    ; GFX6: [[ZEXT19:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO51]](s1)
+    ; GFX6: [[ADD23:%[0-9]+]]:_(s32) = G_ADD [[ADD22]], [[ZEXT19]]
+    ; GFX6: [[UMULH19:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[ADD20]]
+    ; GFX6: [[ADD24:%[0-9]+]]:_(s32) = G_ADD [[UMULH19]], [[ADD23]]
+    ; GFX6: [[UADDO52:%[0-9]+]]:_(s32), [[UADDO53:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI2]], [[UADDO50]]
+    ; GFX6: [[UADDE14:%[0-9]+]]:_(s32), [[UADDE15:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI3]], [[ADD24]], [[UADDO53]]
+    ; GFX6: [[ADD25:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI3]], [[ADD24]]
+    ; GFX6: [[MUL24:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[UADDO52]]
+    ; GFX6: [[MUL25:%[0-9]+]]:_(s32) = G_MUL [[USUBE14]], [[UADDO52]]
+    ; GFX6: [[MUL26:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[UADDE14]]
+    ; GFX6: [[UMULH20:%[0-9]+]]:_(s32) = G_UMULH [[USUBO10]], [[UADDO52]]
+    ; GFX6: [[ADD26:%[0-9]+]]:_(s32) = G_ADD [[MUL25]], [[MUL26]]
+    ; GFX6: [[ADD27:%[0-9]+]]:_(s32) = G_ADD [[ADD26]], [[UMULH20]]
+    ; GFX6: [[MUL27:%[0-9]+]]:_(s32) = G_MUL [[UADDE14]], [[MUL24]]
+    ; GFX6: [[MUL28:%[0-9]+]]:_(s32) = G_MUL [[UADDO52]], [[ADD27]]
+    ; GFX6: [[UMULH21:%[0-9]+]]:_(s32) = G_UMULH [[UADDO52]], [[MUL24]]
+    ; GFX6: [[UADDO54:%[0-9]+]]:_(s32), [[UADDO55:%[0-9]+]]:_(s1) = G_UADDO [[MUL27]], [[MUL28]]
+    ; GFX6: [[ZEXT20:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO55]](s1)
+    ; GFX6: [[UADDO56:%[0-9]+]]:_(s32), [[UADDO57:%[0-9]+]]:_(s1) = G_UADDO [[UADDO54]], [[UMULH21]]
+    ; GFX6: [[ZEXT21:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO57]](s1)
+    ; GFX6: [[ADD28:%[0-9]+]]:_(s32) = G_ADD [[ZEXT20]], [[ZEXT21]]
+    ; GFX6: [[MUL29:%[0-9]+]]:_(s32) = G_MUL [[UADDE14]], [[ADD27]]
+    ; GFX6: [[UMULH22:%[0-9]+]]:_(s32) = G_UMULH [[UADDE14]], [[MUL24]]
+    ; GFX6: [[UMULH23:%[0-9]+]]:_(s32) = G_UMULH [[UADDO52]], [[ADD27]]
+    ; GFX6: [[UADDO58:%[0-9]+]]:_(s32), [[UADDO59:%[0-9]+]]:_(s1) = G_UADDO [[MUL29]], [[UMULH22]]
+    ; GFX6: [[ZEXT22:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO59]](s1)
+    ; GFX6: [[UADDO60:%[0-9]+]]:_(s32), [[UADDO61:%[0-9]+]]:_(s1) = G_UADDO [[UADDO58]], [[UMULH23]]
+    ; GFX6: [[ZEXT23:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO61]](s1)
+    ; GFX6: [[ADD29:%[0-9]+]]:_(s32) = G_ADD [[ZEXT22]], [[ZEXT23]]
+    ; GFX6: [[UADDO62:%[0-9]+]]:_(s32), [[UADDO63:%[0-9]+]]:_(s1) = G_UADDO [[UADDO60]], [[ADD28]]
+    ; GFX6: [[ZEXT24:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO63]](s1)
+    ; GFX6: [[ADD30:%[0-9]+]]:_(s32) = G_ADD [[ADD29]], [[ZEXT24]]
+    ; GFX6: [[UMULH24:%[0-9]+]]:_(s32) = G_UMULH [[UADDE14]], [[ADD27]]
+    ; GFX6: [[ADD31:%[0-9]+]]:_(s32) = G_ADD [[UMULH24]], [[ADD30]]
+    ; GFX6: [[UADDO64:%[0-9]+]]:_(s32), [[UADDO65:%[0-9]+]]:_(s1) = G_UADDO [[UADDO52]], [[UADDO62]]
+    ; GFX6: [[UADDE16:%[0-9]+]]:_(s32), [[UADDE17:%[0-9]+]]:_(s1) = G_UADDE [[ADD25]], [[ADD31]], [[UADDO53]]
+    ; GFX6: [[UADDE18:%[0-9]+]]:_(s32), [[UADDE19:%[0-9]+]]:_(s1) = G_UADDE [[UADDE16]], [[C6]], [[UADDO65]]
+    ; GFX6: [[UV44:%[0-9]+]]:_(s32), [[UV45:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR3]](s64)
+    ; GFX6: [[UV46:%[0-9]+]]:_(s32), [[UV47:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR3]](s64)
+    ; GFX6: [[MUL30:%[0-9]+]]:_(s32) = G_MUL [[UV47]], [[UADDO64]]
+    ; GFX6: [[MUL31:%[0-9]+]]:_(s32) = G_MUL [[UV46]], [[UADDE18]]
+    ; GFX6: [[UMULH25:%[0-9]+]]:_(s32) = G_UMULH [[UV46]], [[UADDO64]]
+    ; GFX6: [[UADDO66:%[0-9]+]]:_(s32), [[UADDO67:%[0-9]+]]:_(s1) = G_UADDO [[MUL30]], [[MUL31]]
+    ; GFX6: [[ZEXT25:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO67]](s1)
+    ; GFX6: [[UADDO68:%[0-9]+]]:_(s32), [[UADDO69:%[0-9]+]]:_(s1) = G_UADDO [[UADDO66]], [[UMULH25]]
+    ; GFX6: [[ZEXT26:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO69]](s1)
+    ; GFX6: [[ADD32:%[0-9]+]]:_(s32) = G_ADD [[ZEXT25]], [[ZEXT26]]
+    ; GFX6: [[MUL32:%[0-9]+]]:_(s32) = G_MUL [[UV47]], [[UADDE18]]
+    ; GFX6: [[UMULH26:%[0-9]+]]:_(s32) = G_UMULH [[UV47]], [[UADDO64]]
+    ; GFX6: [[UMULH27:%[0-9]+]]:_(s32) = G_UMULH [[UV46]], [[UADDE18]]
+    ; GFX6: [[UADDO70:%[0-9]+]]:_(s32), [[UADDO71:%[0-9]+]]:_(s1) = G_UADDO [[MUL32]], [[UMULH26]]
+    ; GFX6: [[ZEXT27:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO71]](s1)
+    ; GFX6: [[UADDO72:%[0-9]+]]:_(s32), [[UADDO73:%[0-9]+]]:_(s1) = G_UADDO [[UADDO70]], [[UMULH27]]
+    ; GFX6: [[ZEXT28:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO73]](s1)
+    ; GFX6: [[ADD33:%[0-9]+]]:_(s32) = G_ADD [[ZEXT27]], [[ZEXT28]]
+    ; GFX6: [[UADDO74:%[0-9]+]]:_(s32), [[UADDO75:%[0-9]+]]:_(s1) = G_UADDO [[UADDO72]], [[ADD32]]
+    ; GFX6: [[ZEXT29:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO75]](s1)
+    ; GFX6: [[ADD34:%[0-9]+]]:_(s32) = G_ADD [[ADD33]], [[ZEXT29]]
+    ; GFX6: [[UMULH28:%[0-9]+]]:_(s32) = G_UMULH [[UV47]], [[UADDE18]]
+    ; GFX6: [[ADD35:%[0-9]+]]:_(s32) = G_ADD [[UMULH28]], [[ADD34]]
+    ; GFX6: [[UV48:%[0-9]+]]:_(s32), [[UV49:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX6: [[MUL33:%[0-9]+]]:_(s32) = G_MUL [[UV48]], [[UADDO74]]
+    ; GFX6: [[MUL34:%[0-9]+]]:_(s32) = G_MUL [[UV49]], [[UADDO74]]
+    ; GFX6: [[MUL35:%[0-9]+]]:_(s32) = G_MUL [[UV48]], [[ADD35]]
+    ; GFX6: [[UMULH29:%[0-9]+]]:_(s32) = G_UMULH [[UV48]], [[UADDO74]]
+    ; GFX6: [[ADD36:%[0-9]+]]:_(s32) = G_ADD [[MUL34]], [[MUL35]]
+    ; GFX6: [[ADD37:%[0-9]+]]:_(s32) = G_ADD [[ADD36]], [[UMULH29]]
+    ; GFX6: [[USUBO12:%[0-9]+]]:_(s32), [[USUBO13:%[0-9]+]]:_(s1) = G_USUBO [[UV44]], [[MUL33]]
+    ; GFX6: [[USUBE16:%[0-9]+]]:_(s32), [[USUBE17:%[0-9]+]]:_(s1) = G_USUBE [[UV45]], [[ADD37]], [[USUBO13]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV45]], [[ADD37]]
+    ; GFX6: [[MV8:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO12]](s32), [[USUBE16]](s32)
+    ; GFX6: [[UV50:%[0-9]+]]:_(s32), [[UV51:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX6: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE16]](s32), [[UV51]]
+    ; GFX6: [[SEXT4:%[0-9]+]]:_(s32) = G_SEXT [[ICMP8]](s1)
+    ; GFX6: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO12]](s32), [[UV50]]
+    ; GFX6: [[SEXT5:%[0-9]+]]:_(s32) = G_SEXT [[ICMP9]](s1)
+    ; GFX6: [[ICMP10:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE16]](s32), [[UV51]]
+    ; GFX6: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP10]](s1), [[SEXT5]], [[SEXT4]]
+    ; GFX6: [[USUBO14:%[0-9]+]]:_(s32), [[USUBO15:%[0-9]+]]:_(s1) = G_USUBO [[USUBO12]], [[UV50]]
+    ; GFX6: [[USUBE18:%[0-9]+]]:_(s32), [[USUBE19:%[0-9]+]]:_(s1) = G_USUBE [[SUB1]], [[UV51]], [[USUBO13]]
+    ; GFX6: [[USUBE20:%[0-9]+]]:_(s32), [[USUBE21:%[0-9]+]]:_(s1) = G_USUBE [[USUBE18]], [[C6]], [[USUBO15]]
+    ; GFX6: [[MV9:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO14]](s32), [[USUBE20]](s32)
+    ; GFX6: [[ICMP11:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE20]](s32), [[UV51]]
+    ; GFX6: [[SEXT6:%[0-9]+]]:_(s32) = G_SEXT [[ICMP11]](s1)
+    ; GFX6: [[ICMP12:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO14]](s32), [[UV50]]
+    ; GFX6: [[SEXT7:%[0-9]+]]:_(s32) = G_SEXT [[ICMP12]](s1)
+    ; GFX6: [[ICMP13:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE20]](s32), [[UV51]]
+    ; GFX6: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP13]](s1), [[SEXT7]], [[SEXT6]]
+    ; GFX6: [[USUBO16:%[0-9]+]]:_(s32), [[USUBO17:%[0-9]+]]:_(s1) = G_USUBO [[USUBO14]], [[UV50]]
+    ; GFX6: [[USUBE22:%[0-9]+]]:_(s32), [[USUBE23:%[0-9]+]]:_(s1) = G_USUBE [[USUBE18]], [[UV51]], [[USUBO15]]
+    ; GFX6: [[USUBE24:%[0-9]+]]:_(s32), [[USUBE25:%[0-9]+]]:_(s1) = G_USUBE [[USUBE22]], [[C6]], [[USUBO17]]
+    ; GFX6: [[MV10:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO16]](s32), [[USUBE24]](s32)
+    ; GFX6: [[ICMP14:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT5]](s32), [[C6]]
+    ; GFX6: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP14]](s1), [[MV10]], [[MV9]]
+    ; GFX6: [[ICMP15:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT4]](s32), [[C6]]
+    ; GFX6: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP15]](s1), [[SELECT6]], [[MV8]]
+    ; GFX6: [[XOR5:%[0-9]+]]:_(s64) = G_XOR [[SELECT7]], [[ASHR2]]
+    ; GFX6: [[UV52:%[0-9]+]]:_(s32), [[UV53:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR5]](s64)
+    ; GFX6: [[UV54:%[0-9]+]]:_(s32), [[UV55:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR2]](s64)
+    ; GFX6: [[USUBO18:%[0-9]+]]:_(s32), [[USUBO19:%[0-9]+]]:_(s1) = G_USUBO [[UV52]], [[UV54]]
+    ; GFX6: [[USUBE26:%[0-9]+]]:_(s32), [[USUBE27:%[0-9]+]]:_(s1) = G_USUBE [[UV53]], [[UV55]], [[USUBO19]]
+    ; GFX6: [[MV11:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO18]](s32), [[USUBE26]](s32)
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV5]](s64), [[MV11]](s64)
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX8-LABEL: name: test_srem_v2s64
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX8: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX8: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX8: [[SREM:%[0-9]+]]:_(s64) = G_SREM [[UV]], [[UV2]]
-    ; GFX8: [[SREM1:%[0-9]+]]:_(s64) = G_SREM [[UV1]], [[UV3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SREM]](s64), [[SREM1]](s64)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV]], [[C]](s32)
+    ; GFX8: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV2]], [[C]](s32)
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX8: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX8: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]]
+    ; GFX8: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO3]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX8: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX8: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX8: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV12]](s32)
+    ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV13]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX8: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX8: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX8: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX8: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX8: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX8: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX8: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[UV16]]
+    ; GFX8: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[UV17]], [[USUBO1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX8: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX8: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX8: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX8: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX8: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX8: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX8: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX8: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX8: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX8: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX8: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX8: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX8: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX8: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX8: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX8: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX8: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX8: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX8: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX8: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX8: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX8: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX8: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX8: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX8: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX8: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX8: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX8: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX8: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX8: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX8: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX8: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX8: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX8: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX8: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX8: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX8: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX8: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV21]], [[UADDO26]]
+    ; GFX8: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV20]], [[UADDE8]]
+    ; GFX8: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV20]], [[UADDO26]]
+    ; GFX8: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX8: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX8: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX8: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX8: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX8: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV21]], [[UADDE8]]
+    ; GFX8: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV21]], [[UADDO26]]
+    ; GFX8: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV20]], [[UADDE8]]
+    ; GFX8: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX8: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX8: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX8: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX8: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX8: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX8: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX8: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX8: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV21]], [[UADDE8]]
+    ; GFX8: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX8: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV22]], [[UADDO36]]
+    ; GFX8: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV23]], [[UADDO36]]
+    ; GFX8: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV22]], [[ADD16]]
+    ; GFX8: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV22]], [[UADDO36]]
+    ; GFX8: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX8: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX8: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV18]], [[MUL15]]
+    ; GFX8: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV19]], [[ADD18]], [[USUBO3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV19]], [[ADD18]]
+    ; GFX8: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX8: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV25]]
+    ; GFX8: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV24]]
+    ; GFX8: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV25]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX8: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV24]]
+    ; GFX8: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV25]], [[USUBO3]]
+    ; GFX8: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX8: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV25]]
+    ; GFX8: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV24]]
+    ; GFX8: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV25]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX8: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV24]]
+    ; GFX8: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV25]], [[USUBO5]]
+    ; GFX8: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX8: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX8: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX8: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX8: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX8: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX8: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX8: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV26]], [[UV28]]
+    ; GFX8: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV27]], [[UV29]], [[USUBO9]]
+    ; GFX8: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX8: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; GFX8: [[ASHR3:%[0-9]+]]:_(s64) = G_ASHR [[UV3]], [[C]](s32)
+    ; GFX8: [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX8: [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR2]](s64)
+    ; GFX8: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[UV30]], [[UV32]]
+    ; GFX8: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[UV31]], [[UV33]], [[UADDO39]]
+    ; GFX8: [[MV6:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO38]](s32), [[UADDE10]](s32)
+    ; GFX8: [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX8: [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR3]](s64)
+    ; GFX8: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UV34]], [[UV36]]
+    ; GFX8: [[UADDE12:%[0-9]+]]:_(s32), [[UADDE13:%[0-9]+]]:_(s1) = G_UADDE [[UV35]], [[UV37]], [[UADDO41]]
+    ; GFX8: [[MV7:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO40]](s32), [[UADDE12]](s32)
+    ; GFX8: [[XOR3:%[0-9]+]]:_(s64) = G_XOR [[MV6]], [[ASHR2]]
+    ; GFX8: [[XOR4:%[0-9]+]]:_(s64) = G_XOR [[MV7]], [[ASHR3]]
+    ; GFX8: [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX8: [[UITOFP2:%[0-9]+]]:_(s32) = G_UITOFP [[UV38]](s32)
+    ; GFX8: [[UITOFP3:%[0-9]+]]:_(s32) = G_UITOFP [[UV39]](s32)
+    ; GFX8: [[C7:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[FMUL4:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP3]], [[C7]]
+    ; GFX8: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL4]], [[UITOFP2]]
+    ; GFX8: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD2]](s32)
+    ; GFX8: [[C8:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX8: [[FMUL5:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C8]]
+    ; GFX8: [[C9:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX8: [[FMUL6:%[0-9]+]]:_(s32) = G_FMUL [[FMUL5]], [[C9]]
+    ; GFX8: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL6]]
+    ; GFX8: [[C10:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX8: [[FMUL7:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC1]], [[C10]]
+    ; GFX8: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL7]], [[FMUL5]]
+    ; GFX8: [[FPTOUI2:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD3]](s32)
+    ; GFX8: [[FPTOUI3:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC1]](s32)
+    ; GFX8: [[UV40:%[0-9]+]]:_(s32), [[UV41:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX8: [[UV42:%[0-9]+]]:_(s32), [[UV43:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX8: [[USUBO10:%[0-9]+]]:_(s32), [[USUBO11:%[0-9]+]]:_(s1) = G_USUBO [[UV40]], [[UV42]]
+    ; GFX8: [[USUBE14:%[0-9]+]]:_(s32), [[USUBE15:%[0-9]+]]:_(s1) = G_USUBE [[UV41]], [[UV43]], [[USUBO11]]
+    ; GFX8: [[MUL18:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[FPTOUI2]]
+    ; GFX8: [[MUL19:%[0-9]+]]:_(s32) = G_MUL [[USUBE14]], [[FPTOUI2]]
+    ; GFX8: [[MUL20:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[FPTOUI3]]
+    ; GFX8: [[UMULH15:%[0-9]+]]:_(s32) = G_UMULH [[USUBO10]], [[FPTOUI2]]
+    ; GFX8: [[ADD19:%[0-9]+]]:_(s32) = G_ADD [[MUL19]], [[MUL20]]
+    ; GFX8: [[ADD20:%[0-9]+]]:_(s32) = G_ADD [[ADD19]], [[UMULH15]]
+    ; GFX8: [[MUL21:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[MUL18]]
+    ; GFX8: [[MUL22:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI2]], [[ADD20]]
+    ; GFX8: [[UMULH16:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[MUL18]]
+    ; GFX8: [[UADDO42:%[0-9]+]]:_(s32), [[UADDO43:%[0-9]+]]:_(s1) = G_UADDO [[MUL21]], [[MUL22]]
+    ; GFX8: [[ZEXT15:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO43]](s1)
+    ; GFX8: [[UADDO44:%[0-9]+]]:_(s32), [[UADDO45:%[0-9]+]]:_(s1) = G_UADDO [[UADDO42]], [[UMULH16]]
+    ; GFX8: [[ZEXT16:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO45]](s1)
+    ; GFX8: [[ADD21:%[0-9]+]]:_(s32) = G_ADD [[ZEXT15]], [[ZEXT16]]
+    ; GFX8: [[MUL23:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[ADD20]]
+    ; GFX8: [[UMULH17:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[MUL18]]
+    ; GFX8: [[UMULH18:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[ADD20]]
+    ; GFX8: [[UADDO46:%[0-9]+]]:_(s32), [[UADDO47:%[0-9]+]]:_(s1) = G_UADDO [[MUL23]], [[UMULH17]]
+    ; GFX8: [[ZEXT17:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO47]](s1)
+    ; GFX8: [[UADDO48:%[0-9]+]]:_(s32), [[UADDO49:%[0-9]+]]:_(s1) = G_UADDO [[UADDO46]], [[UMULH18]]
+    ; GFX8: [[ZEXT18:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO49]](s1)
+    ; GFX8: [[ADD22:%[0-9]+]]:_(s32) = G_ADD [[ZEXT17]], [[ZEXT18]]
+    ; GFX8: [[UADDO50:%[0-9]+]]:_(s32), [[UADDO51:%[0-9]+]]:_(s1) = G_UADDO [[UADDO48]], [[ADD21]]
+    ; GFX8: [[ZEXT19:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO51]](s1)
+    ; GFX8: [[ADD23:%[0-9]+]]:_(s32) = G_ADD [[ADD22]], [[ZEXT19]]
+    ; GFX8: [[UMULH19:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[ADD20]]
+    ; GFX8: [[ADD24:%[0-9]+]]:_(s32) = G_ADD [[UMULH19]], [[ADD23]]
+    ; GFX8: [[UADDO52:%[0-9]+]]:_(s32), [[UADDO53:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI2]], [[UADDO50]]
+    ; GFX8: [[UADDE14:%[0-9]+]]:_(s32), [[UADDE15:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI3]], [[ADD24]], [[UADDO53]]
+    ; GFX8: [[ADD25:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI3]], [[ADD24]]
+    ; GFX8: [[MUL24:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[UADDO52]]
+    ; GFX8: [[MUL25:%[0-9]+]]:_(s32) = G_MUL [[USUBE14]], [[UADDO52]]
+    ; GFX8: [[MUL26:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[UADDE14]]
+    ; GFX8: [[UMULH20:%[0-9]+]]:_(s32) = G_UMULH [[USUBO10]], [[UADDO52]]
+    ; GFX8: [[ADD26:%[0-9]+]]:_(s32) = G_ADD [[MUL25]], [[MUL26]]
+    ; GFX8: [[ADD27:%[0-9]+]]:_(s32) = G_ADD [[ADD26]], [[UMULH20]]
+    ; GFX8: [[MUL27:%[0-9]+]]:_(s32) = G_MUL [[UADDE14]], [[MUL24]]
+    ; GFX8: [[MUL28:%[0-9]+]]:_(s32) = G_MUL [[UADDO52]], [[ADD27]]
+    ; GFX8: [[UMULH21:%[0-9]+]]:_(s32) = G_UMULH [[UADDO52]], [[MUL24]]
+    ; GFX8: [[UADDO54:%[0-9]+]]:_(s32), [[UADDO55:%[0-9]+]]:_(s1) = G_UADDO [[MUL27]], [[MUL28]]
+    ; GFX8: [[ZEXT20:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO55]](s1)
+    ; GFX8: [[UADDO56:%[0-9]+]]:_(s32), [[UADDO57:%[0-9]+]]:_(s1) = G_UADDO [[UADDO54]], [[UMULH21]]
+    ; GFX8: [[ZEXT21:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO57]](s1)
+    ; GFX8: [[ADD28:%[0-9]+]]:_(s32) = G_ADD [[ZEXT20]], [[ZEXT21]]
+    ; GFX8: [[MUL29:%[0-9]+]]:_(s32) = G_MUL [[UADDE14]], [[ADD27]]
+    ; GFX8: [[UMULH22:%[0-9]+]]:_(s32) = G_UMULH [[UADDE14]], [[MUL24]]
+    ; GFX8: [[UMULH23:%[0-9]+]]:_(s32) = G_UMULH [[UADDO52]], [[ADD27]]
+    ; GFX8: [[UADDO58:%[0-9]+]]:_(s32), [[UADDO59:%[0-9]+]]:_(s1) = G_UADDO [[MUL29]], [[UMULH22]]
+    ; GFX8: [[ZEXT22:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO59]](s1)
+    ; GFX8: [[UADDO60:%[0-9]+]]:_(s32), [[UADDO61:%[0-9]+]]:_(s1) = G_UADDO [[UADDO58]], [[UMULH23]]
+    ; GFX8: [[ZEXT23:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO61]](s1)
+    ; GFX8: [[ADD29:%[0-9]+]]:_(s32) = G_ADD [[ZEXT22]], [[ZEXT23]]
+    ; GFX8: [[UADDO62:%[0-9]+]]:_(s32), [[UADDO63:%[0-9]+]]:_(s1) = G_UADDO [[UADDO60]], [[ADD28]]
+    ; GFX8: [[ZEXT24:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO63]](s1)
+    ; GFX8: [[ADD30:%[0-9]+]]:_(s32) = G_ADD [[ADD29]], [[ZEXT24]]
+    ; GFX8: [[UMULH24:%[0-9]+]]:_(s32) = G_UMULH [[UADDE14]], [[ADD27]]
+    ; GFX8: [[ADD31:%[0-9]+]]:_(s32) = G_ADD [[UMULH24]], [[ADD30]]
+    ; GFX8: [[UADDO64:%[0-9]+]]:_(s32), [[UADDO65:%[0-9]+]]:_(s1) = G_UADDO [[UADDO52]], [[UADDO62]]
+    ; GFX8: [[UADDE16:%[0-9]+]]:_(s32), [[UADDE17:%[0-9]+]]:_(s1) = G_UADDE [[ADD25]], [[ADD31]], [[UADDO53]]
+    ; GFX8: [[UADDE18:%[0-9]+]]:_(s32), [[UADDE19:%[0-9]+]]:_(s1) = G_UADDE [[UADDE16]], [[C6]], [[UADDO65]]
+    ; GFX8: [[UV44:%[0-9]+]]:_(s32), [[UV45:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR3]](s64)
+    ; GFX8: [[UV46:%[0-9]+]]:_(s32), [[UV47:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR3]](s64)
+    ; GFX8: [[MUL30:%[0-9]+]]:_(s32) = G_MUL [[UV47]], [[UADDO64]]
+    ; GFX8: [[MUL31:%[0-9]+]]:_(s32) = G_MUL [[UV46]], [[UADDE18]]
+    ; GFX8: [[UMULH25:%[0-9]+]]:_(s32) = G_UMULH [[UV46]], [[UADDO64]]
+    ; GFX8: [[UADDO66:%[0-9]+]]:_(s32), [[UADDO67:%[0-9]+]]:_(s1) = G_UADDO [[MUL30]], [[MUL31]]
+    ; GFX8: [[ZEXT25:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO67]](s1)
+    ; GFX8: [[UADDO68:%[0-9]+]]:_(s32), [[UADDO69:%[0-9]+]]:_(s1) = G_UADDO [[UADDO66]], [[UMULH25]]
+    ; GFX8: [[ZEXT26:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO69]](s1)
+    ; GFX8: [[ADD32:%[0-9]+]]:_(s32) = G_ADD [[ZEXT25]], [[ZEXT26]]
+    ; GFX8: [[MUL32:%[0-9]+]]:_(s32) = G_MUL [[UV47]], [[UADDE18]]
+    ; GFX8: [[UMULH26:%[0-9]+]]:_(s32) = G_UMULH [[UV47]], [[UADDO64]]
+    ; GFX8: [[UMULH27:%[0-9]+]]:_(s32) = G_UMULH [[UV46]], [[UADDE18]]
+    ; GFX8: [[UADDO70:%[0-9]+]]:_(s32), [[UADDO71:%[0-9]+]]:_(s1) = G_UADDO [[MUL32]], [[UMULH26]]
+    ; GFX8: [[ZEXT27:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO71]](s1)
+    ; GFX8: [[UADDO72:%[0-9]+]]:_(s32), [[UADDO73:%[0-9]+]]:_(s1) = G_UADDO [[UADDO70]], [[UMULH27]]
+    ; GFX8: [[ZEXT28:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO73]](s1)
+    ; GFX8: [[ADD33:%[0-9]+]]:_(s32) = G_ADD [[ZEXT27]], [[ZEXT28]]
+    ; GFX8: [[UADDO74:%[0-9]+]]:_(s32), [[UADDO75:%[0-9]+]]:_(s1) = G_UADDO [[UADDO72]], [[ADD32]]
+    ; GFX8: [[ZEXT29:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO75]](s1)
+    ; GFX8: [[ADD34:%[0-9]+]]:_(s32) = G_ADD [[ADD33]], [[ZEXT29]]
+    ; GFX8: [[UMULH28:%[0-9]+]]:_(s32) = G_UMULH [[UV47]], [[UADDE18]]
+    ; GFX8: [[ADD35:%[0-9]+]]:_(s32) = G_ADD [[UMULH28]], [[ADD34]]
+    ; GFX8: [[UV48:%[0-9]+]]:_(s32), [[UV49:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX8: [[MUL33:%[0-9]+]]:_(s32) = G_MUL [[UV48]], [[UADDO74]]
+    ; GFX8: [[MUL34:%[0-9]+]]:_(s32) = G_MUL [[UV49]], [[UADDO74]]
+    ; GFX8: [[MUL35:%[0-9]+]]:_(s32) = G_MUL [[UV48]], [[ADD35]]
+    ; GFX8: [[UMULH29:%[0-9]+]]:_(s32) = G_UMULH [[UV48]], [[UADDO74]]
+    ; GFX8: [[ADD36:%[0-9]+]]:_(s32) = G_ADD [[MUL34]], [[MUL35]]
+    ; GFX8: [[ADD37:%[0-9]+]]:_(s32) = G_ADD [[ADD36]], [[UMULH29]]
+    ; GFX8: [[USUBO12:%[0-9]+]]:_(s32), [[USUBO13:%[0-9]+]]:_(s1) = G_USUBO [[UV44]], [[MUL33]]
+    ; GFX8: [[USUBE16:%[0-9]+]]:_(s32), [[USUBE17:%[0-9]+]]:_(s1) = G_USUBE [[UV45]], [[ADD37]], [[USUBO13]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV45]], [[ADD37]]
+    ; GFX8: [[MV8:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO12]](s32), [[USUBE16]](s32)
+    ; GFX8: [[UV50:%[0-9]+]]:_(s32), [[UV51:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX8: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE16]](s32), [[UV51]]
+    ; GFX8: [[SEXT4:%[0-9]+]]:_(s32) = G_SEXT [[ICMP8]](s1)
+    ; GFX8: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO12]](s32), [[UV50]]
+    ; GFX8: [[SEXT5:%[0-9]+]]:_(s32) = G_SEXT [[ICMP9]](s1)
+    ; GFX8: [[ICMP10:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE16]](s32), [[UV51]]
+    ; GFX8: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP10]](s1), [[SEXT5]], [[SEXT4]]
+    ; GFX8: [[USUBO14:%[0-9]+]]:_(s32), [[USUBO15:%[0-9]+]]:_(s1) = G_USUBO [[USUBO12]], [[UV50]]
+    ; GFX8: [[USUBE18:%[0-9]+]]:_(s32), [[USUBE19:%[0-9]+]]:_(s1) = G_USUBE [[SUB1]], [[UV51]], [[USUBO13]]
+    ; GFX8: [[USUBE20:%[0-9]+]]:_(s32), [[USUBE21:%[0-9]+]]:_(s1) = G_USUBE [[USUBE18]], [[C6]], [[USUBO15]]
+    ; GFX8: [[MV9:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO14]](s32), [[USUBE20]](s32)
+    ; GFX8: [[ICMP11:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE20]](s32), [[UV51]]
+    ; GFX8: [[SEXT6:%[0-9]+]]:_(s32) = G_SEXT [[ICMP11]](s1)
+    ; GFX8: [[ICMP12:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO14]](s32), [[UV50]]
+    ; GFX8: [[SEXT7:%[0-9]+]]:_(s32) = G_SEXT [[ICMP12]](s1)
+    ; GFX8: [[ICMP13:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE20]](s32), [[UV51]]
+    ; GFX8: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP13]](s1), [[SEXT7]], [[SEXT6]]
+    ; GFX8: [[USUBO16:%[0-9]+]]:_(s32), [[USUBO17:%[0-9]+]]:_(s1) = G_USUBO [[USUBO14]], [[UV50]]
+    ; GFX8: [[USUBE22:%[0-9]+]]:_(s32), [[USUBE23:%[0-9]+]]:_(s1) = G_USUBE [[USUBE18]], [[UV51]], [[USUBO15]]
+    ; GFX8: [[USUBE24:%[0-9]+]]:_(s32), [[USUBE25:%[0-9]+]]:_(s1) = G_USUBE [[USUBE22]], [[C6]], [[USUBO17]]
+    ; GFX8: [[MV10:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO16]](s32), [[USUBE24]](s32)
+    ; GFX8: [[ICMP14:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT5]](s32), [[C6]]
+    ; GFX8: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP14]](s1), [[MV10]], [[MV9]]
+    ; GFX8: [[ICMP15:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT4]](s32), [[C6]]
+    ; GFX8: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP15]](s1), [[SELECT6]], [[MV8]]
+    ; GFX8: [[XOR5:%[0-9]+]]:_(s64) = G_XOR [[SELECT7]], [[ASHR2]]
+    ; GFX8: [[UV52:%[0-9]+]]:_(s32), [[UV53:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR5]](s64)
+    ; GFX8: [[UV54:%[0-9]+]]:_(s32), [[UV55:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR2]](s64)
+    ; GFX8: [[USUBO18:%[0-9]+]]:_(s32), [[USUBO19:%[0-9]+]]:_(s1) = G_USUBO [[UV52]], [[UV54]]
+    ; GFX8: [[USUBE26:%[0-9]+]]:_(s32), [[USUBE27:%[0-9]+]]:_(s1) = G_USUBE [[UV53]], [[UV55]], [[USUBO19]]
+    ; GFX8: [[MV11:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO18]](s32), [[USUBE26]](s32)
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV5]](s64), [[MV11]](s64)
     ; GFX8: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX9-LABEL: name: test_srem_v2s64
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX9: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX9: [[SREM:%[0-9]+]]:_(s64) = G_SREM [[UV]], [[UV2]]
-    ; GFX9: [[SREM1:%[0-9]+]]:_(s64) = G_SREM [[UV1]], [[UV3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SREM]](s64), [[SREM1]](s64)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV]], [[C]](s32)
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV2]], [[C]](s32)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX9: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]]
+    ; GFX9: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO3]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX9: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX9: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV12]](s32)
+    ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV13]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX9: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX9: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX9: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX9: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX9: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[UV16]]
+    ; GFX9: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[UV17]], [[USUBO1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX9: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX9: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX9: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX9: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX9: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX9: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX9: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX9: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX9: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX9: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX9: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX9: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX9: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX9: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX9: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX9: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX9: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX9: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX9: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX9: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX9: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX9: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX9: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX9: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX9: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX9: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX9: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX9: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX9: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX9: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX9: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX9: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX9: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX9: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX9: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV21]], [[UADDO26]]
+    ; GFX9: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV20]], [[UADDE8]]
+    ; GFX9: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV20]], [[UADDO26]]
+    ; GFX9: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX9: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX9: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX9: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX9: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX9: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV21]], [[UADDE8]]
+    ; GFX9: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV21]], [[UADDO26]]
+    ; GFX9: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV20]], [[UADDE8]]
+    ; GFX9: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX9: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX9: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX9: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX9: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX9: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX9: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX9: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX9: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV21]], [[UADDE8]]
+    ; GFX9: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX9: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV22]], [[UADDO36]]
+    ; GFX9: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV23]], [[UADDO36]]
+    ; GFX9: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV22]], [[ADD16]]
+    ; GFX9: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV22]], [[UADDO36]]
+    ; GFX9: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX9: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX9: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV18]], [[MUL15]]
+    ; GFX9: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV19]], [[ADD18]], [[USUBO3]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV19]], [[ADD18]]
+    ; GFX9: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX9: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV25]]
+    ; GFX9: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV24]]
+    ; GFX9: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV25]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX9: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV24]]
+    ; GFX9: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV25]], [[USUBO3]]
+    ; GFX9: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX9: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV25]]
+    ; GFX9: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV24]]
+    ; GFX9: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV25]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX9: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV24]]
+    ; GFX9: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV25]], [[USUBO5]]
+    ; GFX9: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX9: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX9: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX9: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX9: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX9: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX9: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX9: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV26]], [[UV28]]
+    ; GFX9: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV27]], [[UV29]], [[USUBO9]]
+    ; GFX9: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX9: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; GFX9: [[ASHR3:%[0-9]+]]:_(s64) = G_ASHR [[UV3]], [[C]](s32)
+    ; GFX9: [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX9: [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR2]](s64)
+    ; GFX9: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[UV30]], [[UV32]]
+    ; GFX9: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[UV31]], [[UV33]], [[UADDO39]]
+    ; GFX9: [[MV6:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO38]](s32), [[UADDE10]](s32)
+    ; GFX9: [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX9: [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR3]](s64)
+    ; GFX9: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UV34]], [[UV36]]
+    ; GFX9: [[UADDE12:%[0-9]+]]:_(s32), [[UADDE13:%[0-9]+]]:_(s1) = G_UADDE [[UV35]], [[UV37]], [[UADDO41]]
+    ; GFX9: [[MV7:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO40]](s32), [[UADDE12]](s32)
+    ; GFX9: [[XOR3:%[0-9]+]]:_(s64) = G_XOR [[MV6]], [[ASHR2]]
+    ; GFX9: [[XOR4:%[0-9]+]]:_(s64) = G_XOR [[MV7]], [[ASHR3]]
+    ; GFX9: [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX9: [[UITOFP2:%[0-9]+]]:_(s32) = G_UITOFP [[UV38]](s32)
+    ; GFX9: [[UITOFP3:%[0-9]+]]:_(s32) = G_UITOFP [[UV39]](s32)
+    ; GFX9: [[C7:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[FMUL4:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP3]], [[C7]]
+    ; GFX9: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL4]], [[UITOFP2]]
+    ; GFX9: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD2]](s32)
+    ; GFX9: [[C8:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX9: [[FMUL5:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C8]]
+    ; GFX9: [[C9:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX9: [[FMUL6:%[0-9]+]]:_(s32) = G_FMUL [[FMUL5]], [[C9]]
+    ; GFX9: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL6]]
+    ; GFX9: [[C10:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX9: [[FMUL7:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC1]], [[C10]]
+    ; GFX9: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL7]], [[FMUL5]]
+    ; GFX9: [[FPTOUI2:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD3]](s32)
+    ; GFX9: [[FPTOUI3:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC1]](s32)
+    ; GFX9: [[UV40:%[0-9]+]]:_(s32), [[UV41:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX9: [[UV42:%[0-9]+]]:_(s32), [[UV43:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX9: [[USUBO10:%[0-9]+]]:_(s32), [[USUBO11:%[0-9]+]]:_(s1) = G_USUBO [[UV40]], [[UV42]]
+    ; GFX9: [[USUBE14:%[0-9]+]]:_(s32), [[USUBE15:%[0-9]+]]:_(s1) = G_USUBE [[UV41]], [[UV43]], [[USUBO11]]
+    ; GFX9: [[MUL18:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[FPTOUI2]]
+    ; GFX9: [[MUL19:%[0-9]+]]:_(s32) = G_MUL [[USUBE14]], [[FPTOUI2]]
+    ; GFX9: [[MUL20:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[FPTOUI3]]
+    ; GFX9: [[UMULH15:%[0-9]+]]:_(s32) = G_UMULH [[USUBO10]], [[FPTOUI2]]
+    ; GFX9: [[ADD19:%[0-9]+]]:_(s32) = G_ADD [[MUL19]], [[MUL20]]
+    ; GFX9: [[ADD20:%[0-9]+]]:_(s32) = G_ADD [[ADD19]], [[UMULH15]]
+    ; GFX9: [[MUL21:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[MUL18]]
+    ; GFX9: [[MUL22:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI2]], [[ADD20]]
+    ; GFX9: [[UMULH16:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[MUL18]]
+    ; GFX9: [[UADDO42:%[0-9]+]]:_(s32), [[UADDO43:%[0-9]+]]:_(s1) = G_UADDO [[MUL21]], [[MUL22]]
+    ; GFX9: [[ZEXT15:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO43]](s1)
+    ; GFX9: [[UADDO44:%[0-9]+]]:_(s32), [[UADDO45:%[0-9]+]]:_(s1) = G_UADDO [[UADDO42]], [[UMULH16]]
+    ; GFX9: [[ZEXT16:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO45]](s1)
+    ; GFX9: [[ADD21:%[0-9]+]]:_(s32) = G_ADD [[ZEXT15]], [[ZEXT16]]
+    ; GFX9: [[MUL23:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[ADD20]]
+    ; GFX9: [[UMULH17:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[MUL18]]
+    ; GFX9: [[UMULH18:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[ADD20]]
+    ; GFX9: [[UADDO46:%[0-9]+]]:_(s32), [[UADDO47:%[0-9]+]]:_(s1) = G_UADDO [[MUL23]], [[UMULH17]]
+    ; GFX9: [[ZEXT17:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO47]](s1)
+    ; GFX9: [[UADDO48:%[0-9]+]]:_(s32), [[UADDO49:%[0-9]+]]:_(s1) = G_UADDO [[UADDO46]], [[UMULH18]]
+    ; GFX9: [[ZEXT18:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO49]](s1)
+    ; GFX9: [[ADD22:%[0-9]+]]:_(s32) = G_ADD [[ZEXT17]], [[ZEXT18]]
+    ; GFX9: [[UADDO50:%[0-9]+]]:_(s32), [[UADDO51:%[0-9]+]]:_(s1) = G_UADDO [[UADDO48]], [[ADD21]]
+    ; GFX9: [[ZEXT19:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO51]](s1)
+    ; GFX9: [[ADD23:%[0-9]+]]:_(s32) = G_ADD [[ADD22]], [[ZEXT19]]
+    ; GFX9: [[UMULH19:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[ADD20]]
+    ; GFX9: [[ADD24:%[0-9]+]]:_(s32) = G_ADD [[UMULH19]], [[ADD23]]
+    ; GFX9: [[UADDO52:%[0-9]+]]:_(s32), [[UADDO53:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI2]], [[UADDO50]]
+    ; GFX9: [[UADDE14:%[0-9]+]]:_(s32), [[UADDE15:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI3]], [[ADD24]], [[UADDO53]]
+    ; GFX9: [[ADD25:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI3]], [[ADD24]]
+    ; GFX9: [[MUL24:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[UADDO52]]
+    ; GFX9: [[MUL25:%[0-9]+]]:_(s32) = G_MUL [[USUBE14]], [[UADDO52]]
+    ; GFX9: [[MUL26:%[0-9]+]]:_(s32) = G_MUL [[USUBO10]], [[UADDE14]]
+    ; GFX9: [[UMULH20:%[0-9]+]]:_(s32) = G_UMULH [[USUBO10]], [[UADDO52]]
+    ; GFX9: [[ADD26:%[0-9]+]]:_(s32) = G_ADD [[MUL25]], [[MUL26]]
+    ; GFX9: [[ADD27:%[0-9]+]]:_(s32) = G_ADD [[ADD26]], [[UMULH20]]
+    ; GFX9: [[MUL27:%[0-9]+]]:_(s32) = G_MUL [[UADDE14]], [[MUL24]]
+    ; GFX9: [[MUL28:%[0-9]+]]:_(s32) = G_MUL [[UADDO52]], [[ADD27]]
+    ; GFX9: [[UMULH21:%[0-9]+]]:_(s32) = G_UMULH [[UADDO52]], [[MUL24]]
+    ; GFX9: [[UADDO54:%[0-9]+]]:_(s32), [[UADDO55:%[0-9]+]]:_(s1) = G_UADDO [[MUL27]], [[MUL28]]
+    ; GFX9: [[ZEXT20:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO55]](s1)
+    ; GFX9: [[UADDO56:%[0-9]+]]:_(s32), [[UADDO57:%[0-9]+]]:_(s1) = G_UADDO [[UADDO54]], [[UMULH21]]
+    ; GFX9: [[ZEXT21:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO57]](s1)
+    ; GFX9: [[ADD28:%[0-9]+]]:_(s32) = G_ADD [[ZEXT20]], [[ZEXT21]]
+    ; GFX9: [[MUL29:%[0-9]+]]:_(s32) = G_MUL [[UADDE14]], [[ADD27]]
+    ; GFX9: [[UMULH22:%[0-9]+]]:_(s32) = G_UMULH [[UADDE14]], [[MUL24]]
+    ; GFX9: [[UMULH23:%[0-9]+]]:_(s32) = G_UMULH [[UADDO52]], [[ADD27]]
+    ; GFX9: [[UADDO58:%[0-9]+]]:_(s32), [[UADDO59:%[0-9]+]]:_(s1) = G_UADDO [[MUL29]], [[UMULH22]]
+    ; GFX9: [[ZEXT22:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO59]](s1)
+    ; GFX9: [[UADDO60:%[0-9]+]]:_(s32), [[UADDO61:%[0-9]+]]:_(s1) = G_UADDO [[UADDO58]], [[UMULH23]]
+    ; GFX9: [[ZEXT23:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO61]](s1)
+    ; GFX9: [[ADD29:%[0-9]+]]:_(s32) = G_ADD [[ZEXT22]], [[ZEXT23]]
+    ; GFX9: [[UADDO62:%[0-9]+]]:_(s32), [[UADDO63:%[0-9]+]]:_(s1) = G_UADDO [[UADDO60]], [[ADD28]]
+    ; GFX9: [[ZEXT24:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO63]](s1)
+    ; GFX9: [[ADD30:%[0-9]+]]:_(s32) = G_ADD [[ADD29]], [[ZEXT24]]
+    ; GFX9: [[UMULH24:%[0-9]+]]:_(s32) = G_UMULH [[UADDE14]], [[ADD27]]
+    ; GFX9: [[ADD31:%[0-9]+]]:_(s32) = G_ADD [[UMULH24]], [[ADD30]]
+    ; GFX9: [[UADDO64:%[0-9]+]]:_(s32), [[UADDO65:%[0-9]+]]:_(s1) = G_UADDO [[UADDO52]], [[UADDO62]]
+    ; GFX9: [[UADDE16:%[0-9]+]]:_(s32), [[UADDE17:%[0-9]+]]:_(s1) = G_UADDE [[ADD25]], [[ADD31]], [[UADDO53]]
+    ; GFX9: [[UADDE18:%[0-9]+]]:_(s32), [[UADDE19:%[0-9]+]]:_(s1) = G_UADDE [[UADDE16]], [[C6]], [[UADDO65]]
+    ; GFX9: [[UV44:%[0-9]+]]:_(s32), [[UV45:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR3]](s64)
+    ; GFX9: [[UV46:%[0-9]+]]:_(s32), [[UV47:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR3]](s64)
+    ; GFX9: [[MUL30:%[0-9]+]]:_(s32) = G_MUL [[UV47]], [[UADDO64]]
+    ; GFX9: [[MUL31:%[0-9]+]]:_(s32) = G_MUL [[UV46]], [[UADDE18]]
+    ; GFX9: [[UMULH25:%[0-9]+]]:_(s32) = G_UMULH [[UV46]], [[UADDO64]]
+    ; GFX9: [[UADDO66:%[0-9]+]]:_(s32), [[UADDO67:%[0-9]+]]:_(s1) = G_UADDO [[MUL30]], [[MUL31]]
+    ; GFX9: [[ZEXT25:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO67]](s1)
+    ; GFX9: [[UADDO68:%[0-9]+]]:_(s32), [[UADDO69:%[0-9]+]]:_(s1) = G_UADDO [[UADDO66]], [[UMULH25]]
+    ; GFX9: [[ZEXT26:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO69]](s1)
+    ; GFX9: [[ADD32:%[0-9]+]]:_(s32) = G_ADD [[ZEXT25]], [[ZEXT26]]
+    ; GFX9: [[MUL32:%[0-9]+]]:_(s32) = G_MUL [[UV47]], [[UADDE18]]
+    ; GFX9: [[UMULH26:%[0-9]+]]:_(s32) = G_UMULH [[UV47]], [[UADDO64]]
+    ; GFX9: [[UMULH27:%[0-9]+]]:_(s32) = G_UMULH [[UV46]], [[UADDE18]]
+    ; GFX9: [[UADDO70:%[0-9]+]]:_(s32), [[UADDO71:%[0-9]+]]:_(s1) = G_UADDO [[MUL32]], [[UMULH26]]
+    ; GFX9: [[ZEXT27:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO71]](s1)
+    ; GFX9: [[UADDO72:%[0-9]+]]:_(s32), [[UADDO73:%[0-9]+]]:_(s1) = G_UADDO [[UADDO70]], [[UMULH27]]
+    ; GFX9: [[ZEXT28:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO73]](s1)
+    ; GFX9: [[ADD33:%[0-9]+]]:_(s32) = G_ADD [[ZEXT27]], [[ZEXT28]]
+    ; GFX9: [[UADDO74:%[0-9]+]]:_(s32), [[UADDO75:%[0-9]+]]:_(s1) = G_UADDO [[UADDO72]], [[ADD32]]
+    ; GFX9: [[ZEXT29:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO75]](s1)
+    ; GFX9: [[ADD34:%[0-9]+]]:_(s32) = G_ADD [[ADD33]], [[ZEXT29]]
+    ; GFX9: [[UMULH28:%[0-9]+]]:_(s32) = G_UMULH [[UV47]], [[UADDE18]]
+    ; GFX9: [[ADD35:%[0-9]+]]:_(s32) = G_ADD [[UMULH28]], [[ADD34]]
+    ; GFX9: [[UV48:%[0-9]+]]:_(s32), [[UV49:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX9: [[MUL33:%[0-9]+]]:_(s32) = G_MUL [[UV48]], [[UADDO74]]
+    ; GFX9: [[MUL34:%[0-9]+]]:_(s32) = G_MUL [[UV49]], [[UADDO74]]
+    ; GFX9: [[MUL35:%[0-9]+]]:_(s32) = G_MUL [[UV48]], [[ADD35]]
+    ; GFX9: [[UMULH29:%[0-9]+]]:_(s32) = G_UMULH [[UV48]], [[UADDO74]]
+    ; GFX9: [[ADD36:%[0-9]+]]:_(s32) = G_ADD [[MUL34]], [[MUL35]]
+    ; GFX9: [[ADD37:%[0-9]+]]:_(s32) = G_ADD [[ADD36]], [[UMULH29]]
+    ; GFX9: [[USUBO12:%[0-9]+]]:_(s32), [[USUBO13:%[0-9]+]]:_(s1) = G_USUBO [[UV44]], [[MUL33]]
+    ; GFX9: [[USUBE16:%[0-9]+]]:_(s32), [[USUBE17:%[0-9]+]]:_(s1) = G_USUBE [[UV45]], [[ADD37]], [[USUBO13]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV45]], [[ADD37]]
+    ; GFX9: [[MV8:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO12]](s32), [[USUBE16]](s32)
+    ; GFX9: [[UV50:%[0-9]+]]:_(s32), [[UV51:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR4]](s64)
+    ; GFX9: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE16]](s32), [[UV51]]
+    ; GFX9: [[SEXT4:%[0-9]+]]:_(s32) = G_SEXT [[ICMP8]](s1)
+    ; GFX9: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO12]](s32), [[UV50]]
+    ; GFX9: [[SEXT5:%[0-9]+]]:_(s32) = G_SEXT [[ICMP9]](s1)
+    ; GFX9: [[ICMP10:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE16]](s32), [[UV51]]
+    ; GFX9: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP10]](s1), [[SEXT5]], [[SEXT4]]
+    ; GFX9: [[USUBO14:%[0-9]+]]:_(s32), [[USUBO15:%[0-9]+]]:_(s1) = G_USUBO [[USUBO12]], [[UV50]]
+    ; GFX9: [[USUBE18:%[0-9]+]]:_(s32), [[USUBE19:%[0-9]+]]:_(s1) = G_USUBE [[SUB1]], [[UV51]], [[USUBO13]]
+    ; GFX9: [[USUBE20:%[0-9]+]]:_(s32), [[USUBE21:%[0-9]+]]:_(s1) = G_USUBE [[USUBE18]], [[C6]], [[USUBO15]]
+    ; GFX9: [[MV9:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO14]](s32), [[USUBE20]](s32)
+    ; GFX9: [[ICMP11:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE20]](s32), [[UV51]]
+    ; GFX9: [[SEXT6:%[0-9]+]]:_(s32) = G_SEXT [[ICMP11]](s1)
+    ; GFX9: [[ICMP12:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO14]](s32), [[UV50]]
+    ; GFX9: [[SEXT7:%[0-9]+]]:_(s32) = G_SEXT [[ICMP12]](s1)
+    ; GFX9: [[ICMP13:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE20]](s32), [[UV51]]
+    ; GFX9: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP13]](s1), [[SEXT7]], [[SEXT6]]
+    ; GFX9: [[USUBO16:%[0-9]+]]:_(s32), [[USUBO17:%[0-9]+]]:_(s1) = G_USUBO [[USUBO14]], [[UV50]]
+    ; GFX9: [[USUBE22:%[0-9]+]]:_(s32), [[USUBE23:%[0-9]+]]:_(s1) = G_USUBE [[USUBE18]], [[UV51]], [[USUBO15]]
+    ; GFX9: [[USUBE24:%[0-9]+]]:_(s32), [[USUBE25:%[0-9]+]]:_(s1) = G_USUBE [[USUBE22]], [[C6]], [[USUBO17]]
+    ; GFX9: [[MV10:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO16]](s32), [[USUBE24]](s32)
+    ; GFX9: [[ICMP14:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT5]](s32), [[C6]]
+    ; GFX9: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP14]](s1), [[MV10]], [[MV9]]
+    ; GFX9: [[ICMP15:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT4]](s32), [[C6]]
+    ; GFX9: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP15]](s1), [[SELECT6]], [[MV8]]
+    ; GFX9: [[XOR5:%[0-9]+]]:_(s64) = G_XOR [[SELECT7]], [[ASHR2]]
+    ; GFX9: [[UV52:%[0-9]+]]:_(s32), [[UV53:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR5]](s64)
+    ; GFX9: [[UV54:%[0-9]+]]:_(s32), [[UV55:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR2]](s64)
+    ; GFX9: [[USUBO18:%[0-9]+]]:_(s32), [[USUBO19:%[0-9]+]]:_(s1) = G_USUBO [[UV52]], [[UV54]]
+    ; GFX9: [[USUBE26:%[0-9]+]]:_(s32), [[USUBE27:%[0-9]+]]:_(s1) = G_USUBE [[UV53]], [[UV55]], [[USUBO19]]
+    ; GFX9: [[MV11:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO18]](s32), [[USUBE26]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV5]](s64), [[MV11]](s64)
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
@@ -448,35 +1843,29 @@ body: |
     ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX6: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
-    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
-    ; GFX6: $vgpr0 = COPY [[AND1]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
+    ; GFX6: $vgpr0 = COPY [[AND]](s32)
     ; GFX8-LABEL: name: test_srem_s16
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
@@ -493,35 +1882,29 @@ body: |
     ; GFX8: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX8: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
-    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
-    ; GFX8: $vgpr0 = COPY [[AND1]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
+    ; GFX8: $vgpr0 = COPY [[AND]](s32)
     ; GFX9-LABEL: name: test_srem_s16
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
@@ -538,35 +1921,29 @@ body: |
     ; GFX9: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX9: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
-    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
-    ; GFX9: $vgpr0 = COPY [[AND1]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
+    ; GFX9: $vgpr0 = COPY [[AND]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s16) = G_TRUNC %0
@@ -603,30 +1980,24 @@ body: |
     ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C3]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX6: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[XOR1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
     ; GFX6: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 16
@@ -634,43 +2005,37 @@ body: |
     ; GFX6: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 16
     ; GFX6: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG2]], [[C1]](s32)
     ; GFX6: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG3]], [[C1]](s32)
-    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG2]], [[ASHR2]]
-    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG3]], [[ASHR3]]
-    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR2]]
-    ; GFX6: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR3]]
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG2]], [[ASHR2]]
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG3]], [[ASHR3]]
+    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[ADD3]], [[ASHR2]]
+    ; GFX6: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR3]]
     ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[XOR4]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C4]]
     ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[XOR4]]
-    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[XOR4]]
-    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL2]]
-    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C3]]
-    ; GFX6: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB5]], [[MUL2]]
-    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX6: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD6]], [[SUB6]]
-    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[XOR3]]
-    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[XOR4]]
-    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[MUL3]]
-    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB7]](s32), [[XOR4]]
-    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR3]](s32), [[MUL3]]
-    ; GFX6: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX6: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB7]], [[XOR4]]
-    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[SUB7]], [[XOR4]]
-    ; GFX6: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[SUB8]], [[SUB7]]
-    ; GFX6: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[ADD7]]
-    ; GFX6: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[SELECT7]], [[ASHR2]]
+    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[XOR4]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB5]], [[FPTOUI1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[XOR3]], [[ADD5]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[XOR4]]
+    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[MUL3]]
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[XOR4]]
+    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SUB6]], [[XOR4]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB7]], [[SUB6]]
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT2]](s32), [[XOR4]]
+    ; GFX6: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SELECT2]], [[XOR4]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB8]], [[SELECT2]]
+    ; GFX6: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR2]]
     ; GFX6: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[XOR5]], [[ASHR2]]
     ; GFX6: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
-    ; GFX6: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C5]]
+    ; GFX6: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C5]]
     ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SUB9]](s32)
-    ; GFX6: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C5]]
-    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
-    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL]]
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C5]]
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX6: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX6: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX8-LABEL: name: test_srem_v2s16
@@ -694,30 +2059,24 @@ body: |
     ; GFX8: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C3]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX8: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[XOR1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
     ; GFX8: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 16
@@ -725,43 +2084,37 @@ body: |
     ; GFX8: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 16
     ; GFX8: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG2]], [[C1]](s32)
     ; GFX8: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG3]], [[C1]](s32)
-    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG2]], [[ASHR2]]
-    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG3]], [[ASHR3]]
-    ; GFX8: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR2]]
-    ; GFX8: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR3]]
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG2]], [[ASHR2]]
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG3]], [[ASHR3]]
+    ; GFX8: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[ADD3]], [[ASHR2]]
+    ; GFX8: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR3]]
     ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[XOR4]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C4]]
     ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[XOR4]]
-    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[XOR4]]
-    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL2]]
-    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C3]]
-    ; GFX8: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB5]], [[MUL2]]
-    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX8: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX8: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD6]], [[SUB6]]
-    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[XOR3]]
-    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[XOR4]]
-    ; GFX8: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[MUL3]]
-    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB7]](s32), [[XOR4]]
-    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR3]](s32), [[MUL3]]
-    ; GFX8: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX8: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB7]], [[XOR4]]
-    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[SUB7]], [[XOR4]]
-    ; GFX8: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[SUB8]], [[SUB7]]
-    ; GFX8: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[ADD7]]
-    ; GFX8: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[SELECT7]], [[ASHR2]]
+    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[XOR4]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB5]], [[FPTOUI1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[XOR3]], [[ADD5]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[XOR4]]
+    ; GFX8: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[MUL3]]
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[XOR4]]
+    ; GFX8: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SUB6]], [[XOR4]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB7]], [[SUB6]]
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT2]](s32), [[XOR4]]
+    ; GFX8: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SELECT2]], [[XOR4]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB8]], [[SELECT2]]
+    ; GFX8: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR2]]
     ; GFX8: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[XOR5]], [[ASHR2]]
     ; GFX8: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; GFX8: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
-    ; GFX8: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C5]]
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C5]]
     ; GFX8: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SUB9]](s32)
-    ; GFX8: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C5]]
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
-    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL]]
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C5]]
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX9-LABEL: name: test_srem_v2s16
@@ -785,30 +2138,24 @@ body: |
     ; GFX9: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C3]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX9: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[XOR1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
     ; GFX9: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 16
@@ -816,35 +2163,29 @@ body: |
     ; GFX9: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 16
     ; GFX9: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG2]], [[C1]](s32)
     ; GFX9: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG3]], [[C1]](s32)
-    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG2]], [[ASHR2]]
-    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG3]], [[ASHR3]]
-    ; GFX9: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR2]]
-    ; GFX9: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD5]], [[ASHR3]]
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG2]], [[ASHR2]]
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[SEXT_INREG3]], [[ASHR3]]
+    ; GFX9: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[ADD3]], [[ASHR2]]
+    ; GFX9: [[XOR4:%[0-9]+]]:_(s32) = G_XOR [[ADD4]], [[ASHR3]]
     ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[XOR4]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C4]]
     ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[XOR4]]
-    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[XOR4]]
-    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL2]]
-    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C3]]
-    ; GFX9: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB5]], [[MUL2]]
-    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX9: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD6]], [[SUB6]]
-    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[XOR3]]
-    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[XOR4]]
-    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[MUL3]]
-    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB7]](s32), [[XOR4]]
-    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR3]](s32), [[MUL3]]
-    ; GFX9: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX9: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB7]], [[XOR4]]
-    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[SUB7]], [[XOR4]]
-    ; GFX9: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[SUB8]], [[SUB7]]
-    ; GFX9: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[ADD7]]
-    ; GFX9: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[SELECT7]], [[ASHR2]]
+    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[XOR4]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB5]], [[FPTOUI1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[XOR3]], [[ADD5]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[XOR4]]
+    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[XOR3]], [[MUL3]]
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[XOR4]]
+    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SUB6]], [[XOR4]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB7]], [[SUB6]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT2]](s32), [[XOR4]]
+    ; GFX9: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SELECT2]], [[XOR4]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB8]], [[SELECT2]]
+    ; GFX9: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR2]]
     ; GFX9: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[XOR5]], [[ASHR2]]
     ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
     ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SUB9]](s32)
@@ -878,30 +2219,24 @@ body: |
     ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX6: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
     ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
@@ -921,30 +2256,24 @@ body: |
     ; GFX8: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX8: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
     ; GFX8: $vgpr0 = COPY [[COPY4]](s32)
@@ -964,30 +2293,24 @@ body: |
     ; GFX9: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX9: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
     ; GFX9: $vgpr0 = COPY [[COPY4]](s32)
@@ -1022,30 +2345,24 @@ body: |
     ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX6: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
     ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
@@ -1065,30 +2382,24 @@ body: |
     ; GFX8: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX8: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX8: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
     ; GFX8: $vgpr0 = COPY [[COPY4]](s32)
@@ -1108,30 +2419,24 @@ body: |
     ; GFX9: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ADD1]], [[ASHR1]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[XOR1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[XOR1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[XOR1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD2]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[XOR]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[XOR1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[XOR1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[XOR]](s32), [[MUL1]]
-    ; GFX9: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[XOR1]]
-    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[XOR1]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD3]]
-    ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[XOR1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[XOR]], [[ADD2]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[XOR1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[XOR]], [[MUL1]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[XOR1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[XOR1]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[XOR1]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[XOR1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX9: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[ASHR]]
     ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[XOR2]], [[ASHR]]
     ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
     ; GFX9: $vgpr0 = COPY [[COPY4]](s32)
@@ -1153,36 +2458,519 @@ body: |
     ; GFX6-LABEL: name: test_srem_s33
     ; GFX6: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX6: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s33) = G_TRUNC [[COPY]](s64)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s33) = G_TRUNC [[COPY1]](s64)
-    ; GFX6: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[TRUNC]](s33)
-    ; GFX6: [[SEXT1:%[0-9]+]]:_(s64) = G_SEXT [[TRUNC1]](s33)
-    ; GFX6: [[SREM:%[0-9]+]]:_(s64) = G_SREM [[SEXT]], [[SEXT1]]
-    ; GFX6: [[TRUNC2:%[0-9]+]]:_(s33) = G_TRUNC [[SREM]](s64)
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC2]](s33)
-    ; GFX6: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+    ; GFX6: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY2]], 33
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
+    ; GFX6: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY3]], 33
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SEXT_INREG]], [[C]](s32)
+    ; GFX6: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SEXT_INREG1]], [[C]](s32)
+    ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SEXT_INREG]](s64)
+    ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SEXT_INREG1]](s64)
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX6: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX6: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX6: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX6: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV8]](s32)
+    ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV9]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX6: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX6: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX6: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX6: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX6: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[UV12]]
+    ; GFX6: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[UV13]], [[USUBO1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX6: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX6: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX6: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX6: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX6: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX6: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX6: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX6: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX6: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX6: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX6: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX6: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX6: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX6: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX6: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX6: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX6: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX6: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX6: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX6: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX6: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX6: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX6: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX6: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX6: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX6: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX6: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX6: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX6: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX6: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX6: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX6: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX6: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX6: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX6: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX6: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX6: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX6: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX6: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX6: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDO26]]
+    ; GFX6: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV16]], [[UADDE8]]
+    ; GFX6: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDO26]]
+    ; GFX6: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX6: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX6: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX6: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX6: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX6: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDE8]]
+    ; GFX6: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDO26]]
+    ; GFX6: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDE8]]
+    ; GFX6: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX6: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX6: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX6: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX6: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX6: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX6: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX6: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX6: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDE8]]
+    ; GFX6: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX6: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[UADDO36]]
+    ; GFX6: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV19]], [[UADDO36]]
+    ; GFX6: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[ADD16]]
+    ; GFX6: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV18]], [[UADDO36]]
+    ; GFX6: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX6: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX6: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[MUL15]]
+    ; GFX6: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[ADD18]], [[USUBO3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV15]], [[ADD18]]
+    ; GFX6: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX6: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV21]]
+    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV20]]
+    ; GFX6: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV21]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX6: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV20]]
+    ; GFX6: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV21]], [[USUBO3]]
+    ; GFX6: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX6: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV21]]
+    ; GFX6: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV20]]
+    ; GFX6: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV21]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX6: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV20]]
+    ; GFX6: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV21]], [[USUBO5]]
+    ; GFX6: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX6: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX6: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX6: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX6: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX6: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX6: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX6: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV22]], [[UV24]]
+    ; GFX6: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV23]], [[UV25]], [[USUBO9]]
+    ; GFX6: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s64) = COPY [[MV5]](s64)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[COPY4]](s64)
     ; GFX8-LABEL: name: test_srem_s33
     ; GFX8: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX8: [[TRUNC:%[0-9]+]]:_(s33) = G_TRUNC [[COPY]](s64)
-    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s33) = G_TRUNC [[COPY1]](s64)
-    ; GFX8: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[TRUNC]](s33)
-    ; GFX8: [[SEXT1:%[0-9]+]]:_(s64) = G_SEXT [[TRUNC1]](s33)
-    ; GFX8: [[SREM:%[0-9]+]]:_(s64) = G_SREM [[SEXT]], [[SEXT1]]
-    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s33) = G_TRUNC [[SREM]](s64)
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC2]](s33)
-    ; GFX8: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+    ; GFX8: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY2]], 33
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
+    ; GFX8: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY3]], 33
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SEXT_INREG]], [[C]](s32)
+    ; GFX8: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SEXT_INREG1]], [[C]](s32)
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SEXT_INREG]](s64)
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SEXT_INREG1]](s64)
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX8: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX8: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX8: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX8: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV8]](s32)
+    ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV9]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX8: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX8: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX8: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX8: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX8: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX8: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX8: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[UV12]]
+    ; GFX8: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[UV13]], [[USUBO1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX8: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX8: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX8: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX8: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX8: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX8: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX8: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX8: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX8: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX8: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX8: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX8: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX8: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX8: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX8: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX8: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX8: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX8: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX8: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX8: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX8: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX8: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX8: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX8: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX8: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX8: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX8: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX8: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX8: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX8: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX8: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX8: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX8: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX8: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX8: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX8: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX8: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX8: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDO26]]
+    ; GFX8: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV16]], [[UADDE8]]
+    ; GFX8: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDO26]]
+    ; GFX8: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX8: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX8: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX8: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX8: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX8: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDE8]]
+    ; GFX8: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDO26]]
+    ; GFX8: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDE8]]
+    ; GFX8: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX8: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX8: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX8: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX8: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX8: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX8: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX8: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX8: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDE8]]
+    ; GFX8: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX8: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[UADDO36]]
+    ; GFX8: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV19]], [[UADDO36]]
+    ; GFX8: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[ADD16]]
+    ; GFX8: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV18]], [[UADDO36]]
+    ; GFX8: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX8: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX8: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[MUL15]]
+    ; GFX8: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[ADD18]], [[USUBO3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV15]], [[ADD18]]
+    ; GFX8: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX8: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV21]]
+    ; GFX8: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV20]]
+    ; GFX8: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV21]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX8: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV20]]
+    ; GFX8: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV21]], [[USUBO3]]
+    ; GFX8: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX8: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV21]]
+    ; GFX8: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV20]]
+    ; GFX8: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV21]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX8: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV20]]
+    ; GFX8: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV21]], [[USUBO5]]
+    ; GFX8: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX8: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX8: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX8: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX8: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX8: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX8: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX8: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV22]], [[UV24]]
+    ; GFX8: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV23]], [[UV25]], [[USUBO9]]
+    ; GFX8: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s64) = COPY [[MV5]](s64)
+    ; GFX8: $vgpr0_vgpr1 = COPY [[COPY4]](s64)
     ; GFX9-LABEL: name: test_srem_s33
     ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(s33) = G_TRUNC [[COPY]](s64)
-    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s33) = G_TRUNC [[COPY1]](s64)
-    ; GFX9: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[TRUNC]](s33)
-    ; GFX9: [[SEXT1:%[0-9]+]]:_(s64) = G_SEXT [[TRUNC1]](s33)
-    ; GFX9: [[SREM:%[0-9]+]]:_(s64) = G_SREM [[SEXT]], [[SEXT1]]
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s33) = G_TRUNC [[SREM]](s64)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC2]](s33)
-    ; GFX9: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+    ; GFX9: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY2]], 33
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
+    ; GFX9: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY3]], 33
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SEXT_INREG]], [[C]](s32)
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SEXT_INREG1]], [[C]](s32)
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SEXT_INREG]](s64)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SEXT_INREG1]](s64)
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX9: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[MV]], [[ASHR]]
+    ; GFX9: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[MV1]], [[ASHR1]]
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV8]](s32)
+    ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV9]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX9: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX9: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX9: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX9: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX9: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[UV12]]
+    ; GFX9: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[UV13]], [[USUBO1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX9: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH1]]
+    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX9: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX9: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[UADDO8]], [[UMULH3]]
+    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO11]](s1)
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX9: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[ADD2]]
+    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX9: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO12]]
+    ; GFX9: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO15]]
+    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX9: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO14]]
+    ; GFX9: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO14]]
+    ; GFX9: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE4]]
+    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO14]]
+    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX9: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX9: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[MUL6]]
+    ; GFX9: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO14]], [[ADD8]]
+    ; GFX9: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[MUL6]]
+    ; GFX9: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX9: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH6]]
+    ; GFX9: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX9: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX9: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE4]], [[ADD8]]
+    ; GFX9: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[MUL6]]
+    ; GFX9: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO14]], [[ADD8]]
+    ; GFX9: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX9: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO20]], [[UMULH8]]
+    ; GFX9: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO23]](s1)
+    ; GFX9: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX9: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[UADDO22]], [[ADD9]]
+    ; GFX9: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX9: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX9: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE4]], [[ADD8]]
+    ; GFX9: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX9: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO14]], [[UADDO24]]
+    ; GFX9: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO15]]
+    ; GFX9: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[C6]], [[UADDO27]]
+    ; GFX9: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX9: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR]](s64)
+    ; GFX9: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDO26]]
+    ; GFX9: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV16]], [[UADDE8]]
+    ; GFX9: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDO26]]
+    ; GFX9: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX9: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX9: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH10]]
+    ; GFX9: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX9: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX9: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV17]], [[UADDE8]]
+    ; GFX9: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDO26]]
+    ; GFX9: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV16]], [[UADDE8]]
+    ; GFX9: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX9: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX9: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UMULH12]]
+    ; GFX9: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX9: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX9: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[ADD13]]
+    ; GFX9: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX9: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX9: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV17]], [[UADDE8]]
+    ; GFX9: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX9: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[UADDO36]]
+    ; GFX9: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV19]], [[UADDO36]]
+    ; GFX9: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV18]], [[ADD16]]
+    ; GFX9: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV18]], [[UADDO36]]
+    ; GFX9: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX9: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX9: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV14]], [[MUL15]]
+    ; GFX9: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV15]], [[ADD18]], [[USUBO3]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV15]], [[ADD18]]
+    ; GFX9: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX9: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR1]](s64)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV21]]
+    ; GFX9: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV20]]
+    ; GFX9: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV21]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX9: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV20]]
+    ; GFX9: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV21]], [[USUBO3]]
+    ; GFX9: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX9: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV21]]
+    ; GFX9: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV20]]
+    ; GFX9: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV21]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX9: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV20]]
+    ; GFX9: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV21]], [[USUBO5]]
+    ; GFX9: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX9: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX9: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV4]], [[MV3]]
+    ; GFX9: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV2]]
+    ; GFX9: [[XOR2:%[0-9]+]]:_(s64) = G_XOR [[SELECT3]], [[ASHR]]
+    ; GFX9: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[XOR2]](s64)
+    ; GFX9: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX9: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV22]], [[UV24]]
+    ; GFX9: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV23]], [[UV25]], [[USUBO9]]
+    ; GFX9: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO8]](s32), [[USUBE12]](s32)
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s64) = COPY [[MV5]](s64)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[COPY4]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s33) = G_TRUNC %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir
index eceec6ad8784f..55641ca31f752 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir
@@ -387,25 +387,685 @@ body: |
 ...
 
 ---
-name: test_store_global_v3s8_align4
+name: test_store_global_v2s8_align1
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_store_global_v2s8_align1
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<2 x s32>)
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; SI: G_STORE [[COPY1]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    ; SI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; SI: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; SI: G_STORE [[COPY2]](s32), [[PTR_ADD]](p1) :: (store 1 + 1, addrspace 1)
+    ; VI-LABEL: name: test_store_global_v2s8_align1
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<2 x s32>)
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; VI: G_STORE [[COPY1]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    ; VI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; VI: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; VI: G_STORE [[COPY2]](s32), [[PTR_ADD]](p1) :: (store 1 + 1, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s8>) = G_IMPLICIT_DEF
+    G_STORE %1, %0 :: (store 2, addrspace 1, align 1)
+
+...
+
+---
+name: test_store_global_v2s8_align2
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_store_global_v2s8_align2
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<2 x s32>)
+    ; SI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C2]]
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; SI: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store 2, addrspace 1)
+    ; VI-LABEL: name: test_store_global_v2s8_align2
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<2 x s32>)
+    ; VI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
+    ; VI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; VI: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store 2, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s8>) = G_IMPLICIT_DEF
+    G_STORE %1, %0 :: (store 2, addrspace 1, align 2)
+
+...
+
+---
+name: test_store_global_v2s8_align4
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
 
+    ; SI-LABEL: name: test_store_global_v2s8_align4
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<2 x s32>)
+    ; SI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C2]]
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; SI: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store 2, align 4, addrspace 1)
+    ; VI-LABEL: name: test_store_global_v2s8_align4
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<2 x s32>)
+    ; VI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
+    ; VI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; VI: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store 2, align 4, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s8>) = G_IMPLICIT_DEF
+    G_STORE %1, %0 :: (store 2, addrspace 1, align 4)
+
+...
+
+---
+name: test_store_global_v3s8_align1
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4
+
+    ; SI-LABEL: name: test_store_global_v3s8_align1
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[UV2]](s32)
+    ; SI: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; SI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY2]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC2]]
+    ; SI: [[COPY4:%[0-9]+]]:_(s16) = COPY [[OR]](s16)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[COPY4]](s16)
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT]], [[C1]](s32)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[COPY4]](s16)
+    ; SI: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    ; SI: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; SI: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
+    ; SI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: G_STORE [[COPY5]](s32), [[PTR_ADD]](p1) :: (store 1 + 1, addrspace 1)
+    ; SI: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; SI: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
+    ; SI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; SI: G_STORE [[COPY6]](s32), [[PTR_ADD1]](p1) :: (store 1 + 2, addrspace 1)
+    ; VI-LABEL: name: test_store_global_v3s8_align1
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[UV2]](s32)
+    ; VI: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; VI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C]]
+    ; VI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI: [[COPY2:%[0-9]+]]:_(s16) = COPY [[OR]](s16)
+    ; VI: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[COPY2]], [[C1]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[COPY2]](s16)
+    ; VI: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    ; VI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; VI: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
+    ; VI: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; VI: G_STORE [[ANYEXT1]](s32), [[PTR_ADD]](p1) :: (store 1 + 1, addrspace 1)
+    ; VI: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; VI: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; VI: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p1) :: (store 1 + 2, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    %2:_(<3 x s8>) = G_TRUNC %1
+    G_STORE %2, %0 :: (store 3, addrspace 1, align 1)
+
+...
+
+---
+name: test_store_global_v3s8_align2
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4
+
+    ; SI-LABEL: name: test_store_global_v3s8_align2
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[UV2]](s32)
+    ; SI: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; SI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC2]]
+    ; SI: [[COPY3:%[0-9]+]]:_(s16) = COPY [[OR]](s16)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[COPY3]](s16)
+    ; SI: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store 2, addrspace 1)
+    ; SI: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; SI: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; SI: G_STORE [[COPY4]](s32), [[PTR_ADD]](p1) :: (store 1 + 2, align 2, addrspace 1)
+    ; VI-LABEL: name: test_store_global_v3s8_align2
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[UV2]](s32)
+    ; VI: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; VI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C]]
+    ; VI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI: [[COPY2:%[0-9]+]]:_(s16) = COPY [[OR]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[COPY2]](s16)
+    ; VI: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store 2, addrspace 1)
+    ; VI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; VI: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; VI: G_STORE [[COPY3]](s32), [[PTR_ADD]](p1) :: (store 1 + 2, align 2, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    %2:_(<3 x s8>) = G_TRUNC %1
+    G_STORE %2, %0 :: (store 3, addrspace 1, align 2)
+
+...
+
+---
+name: test_store_global_v3s8_align4
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4
+
     ; SI-LABEL: name: test_store_global_v3s8_align4
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; SI: [[DEF:%[0-9]+]]:_(<3 x s8>) = G_IMPLICIT_DEF
-    ; SI: [[DEF1:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF1]], [[DEF]](<3 x s8>), 0
-    ; SI: G_STORE [[INSERT]](<4 x s8>), [[COPY]](p1) :: (store 3, align 4, addrspace 1)
+    ; SI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    ; SI: [[TRUNC:%[0-9]+]]:_(<3 x s8>) = G_TRUNC [[COPY1]](<3 x s32>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s24) = G_BITCAST [[TRUNC]](<3 x s8>)
+    ; SI: G_STORE [[BITCAST]](s24), [[COPY]](p1) :: (store 3, align 4, addrspace 1)
     ; VI-LABEL: name: test_store_global_v3s8_align4
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; VI: [[DEF:%[0-9]+]]:_(<3 x s8>) = G_IMPLICIT_DEF
-    ; VI: [[DEF1:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF1]], [[DEF]](<3 x s8>), 0
-    ; VI: G_STORE [[INSERT]](<4 x s8>), [[COPY]](p1) :: (store 3, align 4, addrspace 1)
+    ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    ; VI: [[TRUNC:%[0-9]+]]:_(<3 x s8>) = G_TRUNC [[COPY1]](<3 x s32>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s24) = G_BITCAST [[TRUNC]](<3 x s8>)
+    ; VI: G_STORE [[BITCAST]](s24), [[COPY]](p1) :: (store 3, align 4, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    %2:_(<3 x s8>) = G_TRUNC %1
+    G_STORE %2, %0 :: (store 3, addrspace 1, align 4)
+
+...
+
+---
+name: test_store_global_v4s8_align1
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_store_global_v4s8_align1
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; SI: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    ; SI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; SI: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; SI: G_STORE [[COPY3]](s32), [[PTR_ADD]](p1) :: (store 1 + 1, addrspace 1)
+    ; SI: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; SI: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C1]](s64)
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; SI: G_STORE [[COPY4]](s32), [[PTR_ADD1]](p1) :: (store 1 + 2, addrspace 1)
+    ; SI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+    ; SI: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
+    ; SI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[UV3]](s32)
+    ; SI: G_STORE [[COPY5]](s32), [[PTR_ADD2]](p1) :: (store 1 + 3, addrspace 1)
+    ; VI-LABEL: name: test_store_global_v4s8_align1
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; VI: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    ; VI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; VI: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; VI: G_STORE [[COPY3]](s32), [[PTR_ADD]](p1) :: (store 1 + 1, addrspace 1)
+    ; VI: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; VI: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C1]](s64)
+    ; VI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; VI: G_STORE [[COPY4]](s32), [[PTR_ADD1]](p1) :: (store 1 + 2, addrspace 1)
+    ; VI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+    ; VI: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
+    ; VI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[UV3]](s32)
+    ; VI: G_STORE [[COPY5]](s32), [[PTR_ADD2]](p1) :: (store 1 + 3, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    %2:_(<4 x s8>) = G_TRUNC %1
+    G_STORE %2, %0 :: (store 4, addrspace 1, align 1)
+
+...
+
+---
+name: test_store_global_v4s8_align2
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_store_global_v4s8_align2
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; SI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY2]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV2]](s32)
+    ; SI: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C]]
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[UV3]](s32)
+    ; SI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C2]]
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+    ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
+    ; SI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; SI: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store 2, addrspace 1)
+    ; SI: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; SI: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
+    ; SI: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[OR1]](s16)
+    ; SI: G_STORE [[ANYEXT1]](s32), [[PTR_ADD]](p1) :: (store 2 + 2, addrspace 1)
+    ; VI-LABEL: name: test_store_global_v4s8_align2
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; VI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
+    ; VI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV2]](s32)
+    ; VI: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C]]
+    ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[UV3]](s32)
+    ; VI: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C]]
+    ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C1]](s16)
+    ; VI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; VI: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store 2, addrspace 1)
+    ; VI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; VI: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
+    ; VI: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[OR1]](s16)
+    ; VI: G_STORE [[ANYEXT1]](s32), [[PTR_ADD]](p1) :: (store 2 + 2, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    %2:_(<4 x s8>) = G_TRUNC %1
+    G_STORE %2, %0 :: (store 4, addrspace 1, align 2)
+
+...
+
+---
+name: test_store_global_v4s8_align4
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_store_global_v4s8_align4
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; SI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
+    ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; SI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[UV3]](s32)
+    ; SI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]]
+    ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; SI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
+    ; SI: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; SI: G_STORE [[OR2]](s32), [[COPY]](p1) :: (store 4, addrspace 1)
+    ; VI-LABEL: name: test_store_global_v4s8_align4
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; VI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; VI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
+    ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; VI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[UV3]](s32)
+    ; VI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]]
+    ; VI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; VI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
+    ; VI: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; VI: G_STORE [[OR2]](s32), [[COPY]](p1) :: (store 4, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    %2:_(<4 x s8>) = G_TRUNC %1
+    G_STORE %2, %0 :: (store 4, addrspace 1, align 4)
+
+...
+
+---
+name: test_truncstore_global_v2s8_to_1_align1
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_truncstore_global_v2s8_to_1_align1
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<2 x s32>)
+    ; SI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C2]]
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; SI: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    ; VI-LABEL: name: test_truncstore_global_v2s8_to_1_align1
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<2 x s32>)
+    ; VI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
+    ; VI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; VI: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s8>) = G_IMPLICIT_DEF
+    G_STORE %1, %0 :: (store 1, addrspace 1, align 1)
+
+...
+
+---
+name: test_truncstore_global_v3s8_to_1_align1
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4
+
+    ; SI-LABEL: name: test_truncstore_global_v3s8_to_1_align1
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    ; SI: [[TRUNC:%[0-9]+]]:_(<3 x s8>) = G_TRUNC [[COPY1]](<3 x s32>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s24) = G_BITCAST [[TRUNC]](<3 x s8>)
+    ; SI: G_STORE [[BITCAST]](s24), [[COPY]](p1) :: (store 1, addrspace 1)
+    ; VI-LABEL: name: test_truncstore_global_v3s8_to_1_align1
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    ; VI: [[TRUNC:%[0-9]+]]:_(<3 x s8>) = G_TRUNC [[COPY1]](<3 x s32>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s24) = G_BITCAST [[TRUNC]](<3 x s8>)
+    ; VI: G_STORE [[BITCAST]](s24), [[COPY]](p1) :: (store 1, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    %2:_(<3 x s8>) = G_TRUNC %1
+    G_STORE %2, %0 :: (store 1, addrspace 1, align 1)
+
+...
+
+---
+name: test_truncstore_global_v3s8_to_2_align2
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4
+
+    ; SI-LABEL: name: test_truncstore_global_v3s8_to_2_align2
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    ; SI: [[TRUNC:%[0-9]+]]:_(<3 x s8>) = G_TRUNC [[COPY1]](<3 x s32>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s24) = G_BITCAST [[TRUNC]](<3 x s8>)
+    ; SI: G_STORE [[BITCAST]](s24), [[COPY]](p1) :: (store 2, addrspace 1)
+    ; VI-LABEL: name: test_truncstore_global_v3s8_to_2_align2
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    ; VI: [[TRUNC:%[0-9]+]]:_(<3 x s8>) = G_TRUNC [[COPY1]](<3 x s32>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s24) = G_BITCAST [[TRUNC]](<3 x s8>)
+    ; VI: G_STORE [[BITCAST]](s24), [[COPY]](p1) :: (store 2, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    %2:_(<3 x s8>) = G_TRUNC %1
+    G_STORE %2, %0 :: (store 2, addrspace 1, align 2)
+
+...
+
+
+---
+name: test_truncstore_global_v4s8_to_3_align1
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_truncstore_global_v4s8_to_3_align1
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[UV2]](s32)
+    ; SI: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; SI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY2]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC2]]
+    ; SI: [[COPY4:%[0-9]+]]:_(s16) = COPY [[OR]](s16)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[COPY4]](s16)
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT]], [[C1]](s32)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[COPY4]](s16)
+    ; SI: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    ; SI: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; SI: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
+    ; SI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: G_STORE [[COPY5]](s32), [[PTR_ADD]](p1) :: (store 1 + 1, addrspace 1)
+    ; SI: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; SI: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
+    ; SI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; SI: G_STORE [[COPY6]](s32), [[PTR_ADD1]](p1) :: (store 1 + 2, addrspace 1)
+    ; VI-LABEL: name: test_truncstore_global_v4s8_to_3_align1
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[UV2]](s32)
+    ; VI: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; VI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C]]
+    ; VI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI: [[COPY2:%[0-9]+]]:_(s16) = COPY [[OR]](s16)
+    ; VI: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[COPY2]], [[C1]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[COPY2]](s16)
+    ; VI: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    ; VI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; VI: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
+    ; VI: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; VI: G_STORE [[ANYEXT1]](s32), [[PTR_ADD]](p1) :: (store 1 + 1, addrspace 1)
+    ; VI: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; VI: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; VI: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p1) :: (store 1 + 2, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    %2:_(<4 x s8>) = G_TRUNC %1
+    G_STORE %2, %0 :: (store 3, addrspace 1, align 1)
+
+...
+
+---
+name: test_truncstore_global_v4s8_to_3_align2
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_truncstore_global_v4s8_to_3_align2
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[UV2]](s32)
+    ; SI: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; SI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC2]]
+    ; SI: [[COPY3:%[0-9]+]]:_(s16) = COPY [[OR]](s16)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[COPY3]](s16)
+    ; SI: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store 2, addrspace 1)
+    ; SI: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; SI: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; SI: G_STORE [[COPY4]](s32), [[PTR_ADD]](p1) :: (store 1 + 2, align 2, addrspace 1)
+    ; VI-LABEL: name: test_truncstore_global_v4s8_to_3_align2
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[UV2]](s32)
+    ; VI: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; VI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C]]
+    ; VI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI: [[COPY2:%[0-9]+]]:_(s16) = COPY [[OR]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[COPY2]](s16)
+    ; VI: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store 2, addrspace 1)
+    ; VI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; VI: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; VI: G_STORE [[COPY3]](s32), [[PTR_ADD]](p1) :: (store 1 + 2, align 2, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    %2:_(<4 x s8>) = G_TRUNC %1
+    G_STORE %2, %0 :: (store 3, addrspace 1, align 2)
+
+...
+
+---
+name: test_truncstore_global_v4s8_to_3_align4
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_truncstore_global_v4s8_to_3_align4
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; SI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[COPY1]](<4 x s32>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[TRUNC]](<4 x s8>)
+    ; SI: G_STORE [[BITCAST]](s32), [[COPY]](p1) :: (store 3, align 4, addrspace 1)
+    ; VI-LABEL: name: test_truncstore_global_v4s8_to_3_align4
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; VI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[COPY1]](<4 x s32>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[TRUNC]](<4 x s8>)
+    ; VI: G_STORE [[BITCAST]](s32), [[COPY]](p1) :: (store 3, align 4, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
-    %1:_(<3 x s8>) = G_IMPLICIT_DEF
-    G_STORE %1, %0 :: (store 3, addrspace 1, align 4)
+    %1:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    %2:_(<4 x s8>) = G_TRUNC %1
+    G_STORE %2, %0 :: (store 3, addrspace 1, align 4)
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-udiv.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-udiv.mir
index b0615686427b7..08e01280204a3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-udiv.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-udiv.mir
@@ -14,91 +14,79 @@ body: |
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[COPY1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[COPY1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[COPY1]]
     ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C1]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[COPY]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[COPY1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[COPY1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY]](s32), [[MUL1]]
-    ; GFX6: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[COPY1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[COPY]], [[ADD]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[COPY1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[MUL1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C2]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C2]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
-    ; GFX6: $vgpr0 = COPY [[SELECT3]](s32)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[COPY1]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C2]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[COPY1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[COPY1]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C2]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
+    ; GFX6: $vgpr0 = COPY [[SELECT2]](s32)
     ; GFX8-LABEL: name: test_udiv_s32
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[COPY1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[COPY1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[COPY1]]
     ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C1]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[COPY]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[COPY1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[COPY1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY]](s32), [[MUL1]]
-    ; GFX8: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[COPY1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[COPY]], [[ADD]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[COPY1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[MUL1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C2]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C2]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
-    ; GFX8: $vgpr0 = COPY [[SELECT3]](s32)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[COPY1]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C2]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[COPY1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[COPY1]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C2]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
+    ; GFX8: $vgpr0 = COPY [[SELECT2]](s32)
     ; GFX9-LABEL: name: test_udiv_s32
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[COPY1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[COPY1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[COPY1]]
     ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C1]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[COPY]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[COPY1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[COPY1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY]](s32), [[MUL1]]
-    ; GFX9: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[COPY1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[COPY]], [[ADD]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[COPY1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[MUL1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C2]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C2]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
-    ; GFX9: $vgpr0 = COPY [[SELECT3]](s32)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[COPY1]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C2]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[COPY1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[COPY1]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C2]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
+    ; GFX9: $vgpr0 = COPY [[SELECT2]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_UDIV %0, %1
@@ -118,55 +106,47 @@ body: |
     ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV2]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[UV2]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[UV2]]
     ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C1]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[UV]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[UV2]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[UV2]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[UV]](s32), [[MUL1]]
-    ; GFX6: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[UV2]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[UV]], [[ADD]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[UV2]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[MUL1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C2]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C2]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[UV2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C2]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[UV2]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[UV2]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C2]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
     ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV3]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C3]]
     ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[UV3]]
-    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[UV3]]
-    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL2]]
-    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C1]]
-    ; GFX6: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB4]], [[MUL2]]
-    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX6: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD2]], [[SUB5]]
-    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[UV1]]
-    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[UV3]]
-    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[MUL3]]
-    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[UV3]]
-    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[UV1]](s32), [[MUL3]]
-    ; GFX6: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH5]], [[C2]]
-    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[UMULH5]], [[C2]]
-    ; GFX6: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[ADD3]], [[UMULH5]]
-    ; GFX6: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[SUB7]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SELECT3]](s32), [[SELECT7]](s32)
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[UV3]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB3]], [[FPTOUI1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[UV1]], [[ADD3]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[UV3]]
+    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[MUL3]]
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB4]](s32), [[UV3]]
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[UMULH3]], [[C2]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[ADD4]], [[UMULH3]]
+    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[SUB4]], [[UV3]]
+    ; GFX6: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB5]], [[SUB4]]
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT4]](s32), [[UV3]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[SELECT3]], [[C2]]
+    ; GFX6: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD5]], [[SELECT3]]
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SELECT2]](s32), [[SELECT5]](s32)
     ; GFX6: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; GFX8-LABEL: name: test_udiv_v2s32
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
@@ -175,55 +155,47 @@ body: |
     ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV2]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[UV2]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[UV2]]
     ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C1]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[UV]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[UV2]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[UV2]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[UV]](s32), [[MUL1]]
-    ; GFX8: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[UV2]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[UV]], [[ADD]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[UV2]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[MUL1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C2]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C2]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[UV2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C2]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[UV2]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[UV2]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C2]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
     ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV3]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C3]]
     ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[UV3]]
-    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[UV3]]
-    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL2]]
-    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C1]]
-    ; GFX8: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB4]], [[MUL2]]
-    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX8: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD2]], [[SUB5]]
-    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[UV1]]
-    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[UV3]]
-    ; GFX8: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[MUL3]]
-    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[UV3]]
-    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[UV1]](s32), [[MUL3]]
-    ; GFX8: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH5]], [[C2]]
-    ; GFX8: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[UMULH5]], [[C2]]
-    ; GFX8: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[ADD3]], [[UMULH5]]
-    ; GFX8: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[SUB7]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SELECT3]](s32), [[SELECT7]](s32)
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[UV3]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB3]], [[FPTOUI1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[UV1]], [[ADD3]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[UV3]]
+    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[MUL3]]
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB4]](s32), [[UV3]]
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[UMULH3]], [[C2]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[ADD4]], [[UMULH3]]
+    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[SUB4]], [[UV3]]
+    ; GFX8: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB5]], [[SUB4]]
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT4]](s32), [[UV3]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[SELECT3]], [[C2]]
+    ; GFX8: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD5]], [[SELECT3]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SELECT2]](s32), [[SELECT5]](s32)
     ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; GFX9-LABEL: name: test_udiv_v2s32
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
@@ -232,55 +204,47 @@ body: |
     ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV2]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[UV2]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[UV2]]
     ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C1]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[UV]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[UV2]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[UV2]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[UV]](s32), [[MUL1]]
-    ; GFX9: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[UV2]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[UV]], [[ADD]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[UV2]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[MUL1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C2]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C2]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[UV2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C2]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[UV2]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[UV2]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C2]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
     ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV3]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C3]]
     ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[UV3]]
-    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[UV3]]
-    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL2]]
-    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C1]]
-    ; GFX9: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB4]], [[MUL2]]
-    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX9: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD2]], [[SUB5]]
-    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[UV1]]
-    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[UV3]]
-    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[MUL3]]
-    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[UV3]]
-    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[UV1]](s32), [[MUL3]]
-    ; GFX9: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH5]], [[C2]]
-    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[UMULH5]], [[C2]]
-    ; GFX9: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[ADD3]], [[UMULH5]]
-    ; GFX9: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[SUB7]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SELECT3]](s32), [[SELECT7]](s32)
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[UV3]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB3]], [[FPTOUI1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[UV1]], [[ADD3]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[UV3]]
+    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[MUL3]]
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB4]](s32), [[UV3]]
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[UMULH3]], [[C2]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[ADD4]], [[UMULH3]]
+    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[SUB4]], [[UV3]]
+    ; GFX9: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB5]], [[SUB4]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT4]](s32), [[UV3]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[SELECT3]], [[C2]]
+    ; GFX9: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD5]], [[SELECT3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SELECT2]](s32), [[SELECT5]](s32)
     ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
@@ -1693,33 +1657,29 @@ body: |
     ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX6: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
     ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
-    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
-    ; GFX6: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
-    ; GFX6: $vgpr0 = COPY [[AND3]](s32)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[AND1]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT2]](s32)
+    ; GFX6: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; GFX6: $vgpr0 = COPY [[AND2]](s32)
     ; GFX8-LABEL: name: test_udiv_s16
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
@@ -1730,33 +1690,29 @@ body: |
     ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX8: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
     ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
-    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
-    ; GFX8: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
-    ; GFX8: $vgpr0 = COPY [[AND3]](s32)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[AND1]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT2]](s32)
+    ; GFX8: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; GFX8: $vgpr0 = COPY [[AND2]](s32)
     ; GFX9-LABEL: name: test_udiv_s16
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
@@ -1767,33 +1723,29 @@ body: |
     ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX9: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
     ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
-    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
-    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
-    ; GFX9: $vgpr0 = COPY [[AND3]](s32)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[AND1]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT2]](s32)
+    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; GFX9: $vgpr0 = COPY [[AND2]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s16) = G_TRUNC %0
@@ -1824,64 +1776,56 @@ body: |
     ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C3]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX6: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[AND1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
     ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C4]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C4]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C4]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[AND1]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C4]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
     ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; GFX6: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C1]]
+    ; GFX6: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C1]]
     ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
-    ; GFX6: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C1]]
-    ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[AND4]](s32)
+    ; GFX6: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C1]]
+    ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[AND3]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX6: [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C5]]
     ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[AND4]]
-    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[AND4]]
-    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL2]]
-    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C3]]
-    ; GFX6: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB4]], [[MUL2]]
-    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX6: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD2]], [[SUB5]]
-    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[AND3]]
-    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[AND4]]
-    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[AND3]], [[MUL3]]
-    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[AND4]]
-    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND3]](s32), [[MUL3]]
-    ; GFX6: [[AND5:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH5]], [[C4]]
-    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[UMULH5]], [[C4]]
-    ; GFX6: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND5]](s1), [[ADD3]], [[UMULH5]]
-    ; GFX6: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[SUB7]]
-    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
-    ; GFX6: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C1]]
-    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SELECT7]](s32)
-    ; GFX6: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C1]]
-    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C]](s32)
-    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND6]], [[SHL]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[AND3]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB3]], [[FPTOUI1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[AND2]], [[ADD3]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[AND3]]
+    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[AND2]], [[MUL3]]
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB4]](s32), [[AND3]]
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[UMULH3]], [[C4]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[ADD4]], [[UMULH3]]
+    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[SUB4]], [[AND3]]
+    ; GFX6: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB5]], [[SUB4]]
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT4]](s32), [[AND3]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[SELECT3]], [[C4]]
+    ; GFX6: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD5]], [[SELECT3]]
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT2]](s32)
+    ; GFX6: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C1]]
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SELECT5]](s32)
+    ; GFX6: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C1]]
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SHL]]
     ; GFX6: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX6: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX8-LABEL: name: test_udiv_v2s16
@@ -1899,64 +1843,56 @@ body: |
     ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C3]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX8: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[AND1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
     ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C4]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C4]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C4]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[AND1]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C4]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
     ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; GFX8: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C1]]
+    ; GFX8: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C1]]
     ; GFX8: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
-    ; GFX8: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C1]]
-    ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[AND4]](s32)
+    ; GFX8: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C1]]
+    ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[AND3]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX8: [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C5]]
     ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[AND4]]
-    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[AND4]]
-    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL2]]
-    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C3]]
-    ; GFX8: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB4]], [[MUL2]]
-    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX8: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD2]], [[SUB5]]
-    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[AND3]]
-    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[AND4]]
-    ; GFX8: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[AND3]], [[MUL3]]
-    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[AND4]]
-    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND3]](s32), [[MUL3]]
-    ; GFX8: [[AND5:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH5]], [[C4]]
-    ; GFX8: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[UMULH5]], [[C4]]
-    ; GFX8: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND5]](s1), [[ADD3]], [[UMULH5]]
-    ; GFX8: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[SUB7]]
-    ; GFX8: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
-    ; GFX8: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C1]]
-    ; GFX8: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SELECT7]](s32)
-    ; GFX8: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C1]]
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C]](s32)
-    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND6]], [[SHL]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[AND3]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB3]], [[FPTOUI1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[AND2]], [[ADD3]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[AND3]]
+    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[AND2]], [[MUL3]]
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB4]](s32), [[AND3]]
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[UMULH3]], [[C4]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[ADD4]], [[UMULH3]]
+    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[SUB4]], [[AND3]]
+    ; GFX8: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB5]], [[SUB4]]
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT4]](s32), [[AND3]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[SELECT3]], [[C4]]
+    ; GFX8: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD5]], [[SELECT3]]
+    ; GFX8: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT2]](s32)
+    ; GFX8: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C1]]
+    ; GFX8: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SELECT5]](s32)
+    ; GFX8: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C1]]
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX9-LABEL: name: test_udiv_v2s16
@@ -1974,60 +1910,52 @@ body: |
     ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C3]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX9: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[AND1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
     ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C4]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C4]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C4]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[AND1]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C4]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
     ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C1]]
+    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C1]]
     ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
-    ; GFX9: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C1]]
-    ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[AND4]](s32)
+    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C1]]
+    ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[AND3]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C5]]
     ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[AND4]]
-    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[AND4]]
-    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL2]]
-    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C3]]
-    ; GFX9: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB4]], [[MUL2]]
-    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX9: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD2]], [[SUB5]]
-    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[AND3]]
-    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[AND4]]
-    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[AND3]], [[MUL3]]
-    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[AND4]]
-    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND3]](s32), [[MUL3]]
-    ; GFX9: [[AND5:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH5]], [[C4]]
-    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[UMULH5]], [[C4]]
-    ; GFX9: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND5]](s1), [[ADD3]], [[UMULH5]]
-    ; GFX9: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[SUB7]]
-    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
-    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SELECT7]](s32)
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[AND3]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB3]], [[FPTOUI1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[AND2]], [[ADD3]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[AND3]]
+    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[AND2]], [[MUL3]]
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB4]](s32), [[AND3]]
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[UMULH3]], [[C4]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[ADD4]], [[UMULH3]]
+    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[SUB4]], [[AND3]]
+    ; GFX9: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB5]], [[SUB4]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT4]](s32), [[AND3]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[SELECT3]], [[C4]]
+    ; GFX9: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD5]], [[SELECT3]]
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT2]](s32)
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SELECT5]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[COPY7]](s32)
     ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR_TRUNC]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
@@ -2052,31 +1980,27 @@ body: |
     ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX6: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
     ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
-    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[AND1]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT2]](s32)
     ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: test_udiv_s7
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
@@ -2088,31 +2012,27 @@ body: |
     ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX8: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
     ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
-    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[AND1]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT2]](s32)
     ; GFX8: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX9-LABEL: name: test_udiv_s7
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
@@ -2124,31 +2044,27 @@ body: |
     ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX9: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
     ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
-    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[AND1]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT2]](s32)
     ; GFX9: $vgpr0 = COPY [[COPY4]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -2175,31 +2091,27 @@ body: |
     ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX6: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
     ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
-    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[AND1]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT2]](s32)
     ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: test_udiv_s17
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
@@ -2211,31 +2123,27 @@ body: |
     ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX8: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
     ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
-    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[AND1]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT2]](s32)
     ; GFX8: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX9-LABEL: name: test_udiv_s17
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
@@ -2247,31 +2155,27 @@ body: |
     ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX9: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
     ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH2]], [[C3]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[UMULH2]], [[C3]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[ADD1]], [[UMULH2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[SUB3]]
-    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UMULH1]], [[C3]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD1]], [[UMULH1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](s32), [[AND1]]
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SELECT]], [[C3]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT2]](s32)
     ; GFX9: $vgpr0 = COPY [[COPY4]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-urem.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-urem.mir
index e42fe1400477f..988d75459e011 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-urem.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-urem.mir
@@ -14,88 +14,70 @@ body: |
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[COPY1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[COPY1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[COPY1]]
     ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C1]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[COPY]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[COPY1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[COPY1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY]](s32), [[MUL1]]
-    ; GFX6: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[COPY1]]
-    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[COPY1]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
-    ; GFX6: $vgpr0 = COPY [[SELECT3]](s32)
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[COPY1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[COPY]], [[ADD]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[COPY1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[MUL1]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[COPY1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[COPY1]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[COPY1]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[COPY1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX6: $vgpr0 = COPY [[SELECT1]](s32)
     ; GFX8-LABEL: name: test_urem_s32
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[COPY1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[COPY1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[COPY1]]
     ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C1]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[COPY]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[COPY1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[COPY1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY]](s32), [[MUL1]]
-    ; GFX8: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[COPY1]]
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[COPY1]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
-    ; GFX8: $vgpr0 = COPY [[SELECT3]](s32)
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[COPY1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[COPY]], [[ADD]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[COPY1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[MUL1]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[COPY1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[COPY1]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[COPY1]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[COPY1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX8: $vgpr0 = COPY [[SELECT1]](s32)
     ; GFX9-LABEL: name: test_urem_s32
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[COPY1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[COPY1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[COPY1]]
     ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C1]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[COPY]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[COPY1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[COPY1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY]](s32), [[MUL1]]
-    ; GFX9: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[COPY1]]
-    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[COPY1]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
-    ; GFX9: $vgpr0 = COPY [[SELECT3]](s32)
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[COPY1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[COPY]], [[ADD]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[COPY1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[MUL1]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[COPY1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[COPY1]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[COPY1]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[COPY1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX9: $vgpr0 = COPY [[SELECT1]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_UREM %0, %1
@@ -115,54 +97,42 @@ body: |
     ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV2]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[UV2]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[UV2]]
     ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C1]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[UV]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[UV2]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[UV2]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[UV]](s32), [[MUL1]]
-    ; GFX6: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[UV2]]
-    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[UV2]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[UV2]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[UV]], [[ADD]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[UV2]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[MUL1]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[UV2]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[UV2]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[UV2]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[UV2]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
     ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV3]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C2]]
     ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[UV3]]
-    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[UV3]]
-    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL2]]
-    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C1]]
-    ; GFX6: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB4]], [[MUL2]]
-    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX6: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD2]], [[SUB5]]
-    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[UV1]]
-    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[UV3]]
-    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[MUL3]]
-    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[UV3]]
-    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[UV1]](s32), [[MUL3]]
-    ; GFX6: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SUB6]], [[UV3]]
-    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB6]], [[UV3]]
-    ; GFX6: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[SUB7]], [[SUB6]]
-    ; GFX6: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[ADD3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SELECT3]](s32), [[SELECT7]](s32)
+    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[UV3]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB4]], [[FPTOUI1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[UV1]], [[ADD1]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[UV3]]
+    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[MUL3]]
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB5]](s32), [[UV3]]
+    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB5]], [[UV3]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB6]], [[SUB5]]
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT2]](s32), [[UV3]]
+    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SELECT2]], [[UV3]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB7]], [[SELECT2]]
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SELECT1]](s32), [[SELECT3]](s32)
     ; GFX6: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; GFX8-LABEL: name: test_urem_v2s32
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
@@ -171,54 +141,42 @@ body: |
     ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV2]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[UV2]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[UV2]]
     ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C1]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[UV]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[UV2]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[UV2]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[UV]](s32), [[MUL1]]
-    ; GFX8: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[UV2]]
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[UV2]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[UV2]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[UV]], [[ADD]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[UV2]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[MUL1]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[UV2]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[UV2]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[UV2]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[UV2]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
     ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV3]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C2]]
     ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[UV3]]
-    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[UV3]]
-    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL2]]
-    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C1]]
-    ; GFX8: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB4]], [[MUL2]]
-    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX8: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD2]], [[SUB5]]
-    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[UV1]]
-    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[UV3]]
-    ; GFX8: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[MUL3]]
-    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[UV3]]
-    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[UV1]](s32), [[MUL3]]
-    ; GFX8: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX8: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SUB6]], [[UV3]]
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB6]], [[UV3]]
-    ; GFX8: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[SUB7]], [[SUB6]]
-    ; GFX8: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[ADD3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SELECT3]](s32), [[SELECT7]](s32)
+    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[UV3]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB4]], [[FPTOUI1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[UV1]], [[ADD1]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[UV3]]
+    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[MUL3]]
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB5]](s32), [[UV3]]
+    ; GFX8: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB5]], [[UV3]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB6]], [[SUB5]]
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT2]](s32), [[UV3]]
+    ; GFX8: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SELECT2]], [[UV3]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB7]], [[SELECT2]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SELECT1]](s32), [[SELECT3]](s32)
     ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; GFX9-LABEL: name: test_urem_v2s32
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
@@ -227,54 +185,42 @@ body: |
     ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV2]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[UV2]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[UV2]]
     ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C1]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[UV]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[UV2]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[UV2]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[UV]](s32), [[MUL1]]
-    ; GFX9: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[UV2]]
-    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[UV2]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s1), [[SUB3]], [[SUB2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[UV2]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[UV]], [[ADD]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[UV2]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[MUL1]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[UV2]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[UV2]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[UV2]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[UV2]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
     ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV3]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C2]]
     ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[UV3]]
-    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[UV3]]
-    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[MUL2]]
-    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C1]]
-    ; GFX9: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB4]], [[MUL2]]
-    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX9: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD2]], [[SUB5]]
-    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[UV1]]
-    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[UV3]]
-    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[MUL3]]
-    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[UV3]]
-    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[UV1]](s32), [[MUL3]]
-    ; GFX9: [[AND1:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SUB6]], [[UV3]]
-    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB6]], [[UV3]]
-    ; GFX9: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s1), [[SUB7]], [[SUB6]]
-    ; GFX9: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[ADD3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SELECT3]](s32), [[SELECT7]](s32)
+    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[UV3]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB4]], [[FPTOUI1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[UV1]], [[ADD1]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[UV3]]
+    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[MUL3]]
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB5]](s32), [[UV3]]
+    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB5]], [[UV3]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB6]], [[SUB5]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT2]](s32), [[UV3]]
+    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SELECT2]], [[UV3]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB7]], [[SELECT2]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SELECT1]](s32), [[SELECT3]](s32)
     ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
@@ -1627,32 +1573,26 @@ body: |
     ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX6: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[AND1]]
-    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[AND1]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[SUB3]], [[SUB2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
-    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
-    ; GFX6: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
-    ; GFX6: $vgpr0 = COPY [[AND3]](s32)
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; GFX6: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; GFX6: $vgpr0 = COPY [[AND2]](s32)
     ; GFX8-LABEL: name: test_urem_s16
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
@@ -1663,32 +1603,26 @@ body: |
     ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX8: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[AND1]]
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[AND1]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[SUB3]], [[SUB2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
-    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
-    ; GFX8: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
-    ; GFX8: $vgpr0 = COPY [[AND3]](s32)
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; GFX8: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; GFX8: $vgpr0 = COPY [[AND2]](s32)
     ; GFX9-LABEL: name: test_urem_s16
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
@@ -1699,32 +1633,26 @@ body: |
     ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX9: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[AND1]]
-    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[AND1]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[SUB3]], [[SUB2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
-    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
-    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
-    ; GFX9: $vgpr0 = COPY [[AND3]](s32)
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; GFX9: $vgpr0 = COPY [[AND2]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s16) = G_TRUNC %0
@@ -1755,63 +1683,51 @@ body: |
     ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C3]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX6: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[AND1]]
-    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[AND1]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[SUB3]], [[SUB2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[AND1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
     ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; GFX6: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C1]]
+    ; GFX6: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C1]]
     ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
-    ; GFX6: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C1]]
-    ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[AND4]](s32)
+    ; GFX6: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C1]]
+    ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[AND3]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C4]]
     ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[AND4]]
-    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[AND4]]
-    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL2]]
-    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C3]]
-    ; GFX6: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB4]], [[MUL2]]
-    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX6: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD2]], [[SUB5]]
-    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[AND3]]
-    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[AND4]]
-    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[AND3]], [[MUL3]]
-    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[AND4]]
-    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND3]](s32), [[MUL3]]
-    ; GFX6: [[AND5:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SUB6]], [[AND4]]
-    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB6]], [[AND4]]
-    ; GFX6: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND5]](s1), [[SUB7]], [[SUB6]]
-    ; GFX6: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[ADD3]]
-    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
-    ; GFX6: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C1]]
-    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SELECT7]](s32)
-    ; GFX6: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C1]]
-    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C]](s32)
-    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND6]], [[SHL]]
+    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[AND3]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB4]], [[FPTOUI1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[AND2]], [[ADD1]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[AND3]]
+    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[AND2]], [[MUL3]]
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB5]](s32), [[AND3]]
+    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB5]], [[AND3]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB6]], [[SUB5]]
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT2]](s32), [[AND3]]
+    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SELECT2]], [[AND3]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB7]], [[SELECT2]]
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; GFX6: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C1]]
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
+    ; GFX6: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C1]]
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SHL]]
     ; GFX6: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX6: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX8-LABEL: name: test_urem_v2s16
@@ -1829,63 +1745,51 @@ body: |
     ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C3]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX8: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[AND1]]
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[AND1]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[SUB3]], [[SUB2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[AND1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
     ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; GFX8: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C1]]
+    ; GFX8: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C1]]
     ; GFX8: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
-    ; GFX8: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C1]]
-    ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[AND4]](s32)
+    ; GFX8: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C1]]
+    ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[AND3]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C4]]
     ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[AND4]]
-    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[AND4]]
-    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL2]]
-    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C3]]
-    ; GFX8: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB4]], [[MUL2]]
-    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX8: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD2]], [[SUB5]]
-    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[AND3]]
-    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[AND4]]
-    ; GFX8: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[AND3]], [[MUL3]]
-    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[AND4]]
-    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND3]](s32), [[MUL3]]
-    ; GFX8: [[AND5:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX8: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SUB6]], [[AND4]]
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB6]], [[AND4]]
-    ; GFX8: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND5]](s1), [[SUB7]], [[SUB6]]
-    ; GFX8: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[ADD3]]
-    ; GFX8: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
-    ; GFX8: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C1]]
-    ; GFX8: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SELECT7]](s32)
-    ; GFX8: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C1]]
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C]](s32)
-    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND6]], [[SHL]]
+    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[AND3]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB4]], [[FPTOUI1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[AND2]], [[ADD1]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[AND3]]
+    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[AND2]], [[MUL3]]
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB5]](s32), [[AND3]]
+    ; GFX8: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB5]], [[AND3]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB6]], [[SUB5]]
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT2]](s32), [[AND3]]
+    ; GFX8: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SELECT2]], [[AND3]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB7]], [[SELECT2]]
+    ; GFX8: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; GFX8: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C1]]
+    ; GFX8: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
+    ; GFX8: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C1]]
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX9-LABEL: name: test_urem_v2s16
@@ -1903,59 +1807,47 @@ body: |
     ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C3]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX9: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[AND1]]
-    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[AND1]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[SUB3]], [[SUB2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[AND1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
     ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C1]]
+    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C1]]
     ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
-    ; GFX9: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C1]]
-    ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[AND4]](s32)
+    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C1]]
+    ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[AND3]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](s32)
-    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C4]]
     ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL1]](s32)
-    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[AND4]]
-    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[AND4]]
-    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[MUL2]]
-    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH3]](s32), [[C3]]
-    ; GFX9: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB4]], [[MUL2]]
-    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[SELECT4]], [[FPTOUI1]]
-    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH4]]
-    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI1]], [[UMULH4]]
-    ; GFX9: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD2]], [[SUB5]]
-    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[SELECT5]], [[AND3]]
-    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH5]], [[AND4]]
-    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[AND3]], [[MUL3]]
-    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB6]](s32), [[AND4]]
-    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND3]](s32), [[MUL3]]
-    ; GFX9: [[AND5:%[0-9]+]]:_(s1) = G_AND [[ICMP4]], [[ICMP5]]
-    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SUB6]], [[AND4]]
-    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SUB6]], [[AND4]]
-    ; GFX9: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[AND5]](s1), [[SUB7]], [[SUB6]]
-    ; GFX9: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT6]], [[ADD3]]
-    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
-    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SELECT7]](s32)
+    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[AND3]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SUB4]], [[FPTOUI1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[UMULH2]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[AND2]], [[ADD1]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UMULH3]], [[AND3]]
+    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[AND2]], [[MUL3]]
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB5]](s32), [[AND3]]
+    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB5]], [[AND3]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SUB6]], [[SUB5]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT2]](s32), [[AND3]]
+    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SELECT2]], [[AND3]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB7]], [[SELECT2]]
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[COPY7]](s32)
     ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR_TRUNC]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
@@ -1980,30 +1872,24 @@ body: |
     ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX6: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[AND1]]
-    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[AND1]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[SUB3]], [[SUB2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
-    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
     ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: test_urem_s7
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
@@ -2015,30 +1901,24 @@ body: |
     ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX8: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[AND1]]
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[AND1]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[SUB3]], [[SUB2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
-    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
     ; GFX8: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX9-LABEL: name: test_urem_s7
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
@@ -2050,30 +1930,24 @@ body: |
     ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX9: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[AND1]]
-    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[AND1]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[SUB3]], [[SUB2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
-    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
     ; GFX9: $vgpr0 = COPY [[COPY4]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -2100,30 +1974,24 @@ body: |
     ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX6: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[AND1]]
-    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[AND1]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[SUB3]], [[SUB2]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
-    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
     ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: test_urem_s17
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
@@ -2135,30 +2003,24 @@ body: |
     ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX8: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[AND1]]
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[AND1]]
-    ; GFX8: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[SUB3]], [[SUB2]]
-    ; GFX8: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
-    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
     ; GFX8: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX9-LABEL: name: test_urem_s17
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
@@ -2170,30 +2032,24 @@ body: |
     ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
     ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
     ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
     ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
-    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[AND1]]
-    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[AND1]]
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[MUL]]
-    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UMULH]](s32), [[C2]]
-    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB]], [[MUL]]
-    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[SELECT]], [[FPTOUI]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[FPTOUI]], [[UMULH1]]
-    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ADD]], [[SUB1]]
-    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[SELECT1]], [[AND]]
-    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH2]], [[AND1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
-    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB2]](s32), [[AND1]]
-    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[AND]](s32), [[MUL1]]
-    ; GFX9: [[AND2:%[0-9]+]]:_(s1) = G_AND [[ICMP1]], [[ICMP2]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[AND1]]
-    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SUB2]], [[AND1]]
-    ; GFX9: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s1), [[SUB3]], [[SUB2]]
-    ; GFX9: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SELECT2]], [[ADD1]]
-    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT3]](s32)
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
     ; GFX9: $vgpr0 = COPY [[COPY4]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll
index f09df63976d64..2a3034763087d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll
@@ -235,17 +235,17 @@ define amdgpu_kernel void @test_div_fmas_f32(float addrspace(1)* %out, [8 x i32]
 ;
 ; GFX8-LABEL: test_div_fmas_f32:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_load_dword s2, s[0:1], 0x4c
-; GFX8-NEXT:    s_load_dword s3, s[0:1], 0x70
-; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x94
-; GFX8-NEXT:    s_load_dword s5, s[0:1], 0xb8
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    s_and_b32 s2, 1, s5
-; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s2
+; GFX8-NEXT:	s_load_dword s2, s[0:1], 0xb8
+; GFX8-NEXT:	s_load_dword s3, s[0:1], 0x4c
+; GFX8-NEXT:	s_load_dword s4, s[0:1], 0x70
+; GFX8-NEXT:	s_load_dword s5, s[0:1], 0x94
+; GFX8-NEXT:	s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:	s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:	s_and_b32 s2, 1, s2
+; GFX8-NEXT:	v_mov_b32_e32 v0, s3
+; GFX8-NEXT:	v_mov_b32_e32 v1, s4
+; GFX8-NEXT:	v_mov_b32_e32 v2, s5
+; GFX8-NEXT:	v_cmp_ne_u32_e64 vcc, 0, s2
 ; GFX8-NEXT:    s_nop 3
 ; GFX8-NEXT:    v_div_fmas_f32 v2, v0, v1, v2
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
@@ -527,43 +527,43 @@ define amdgpu_kernel void @test_div_fmas_f32_inline_imm_2(float addrspace(1)* %o
 define amdgpu_kernel void @test_div_fmas_f64(double addrspace(1)* %out, double %a, double %b, double %c, i1 %d) {
 ; GFX7-LABEL: test_div_fmas_f64:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_load_dword s8, s[0:1], 0x11
-; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v4, s6
-; GFX7-NEXT:    s_and_b32 s2, 1, s8
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    v_mov_b32_e32 v5, s7
-; GFX7-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s2
-; GFX7-NEXT:    s_nop 3
-; GFX7-NEXT:    v_div_fmas_f64 v[0:1], v[0:1], v[2:3], v[4:5]
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GFX7-NEXT:    s_endpgm
+; GFX7-NEXT:	s_load_dwordx8 s[4:11], s[0:1], 0x9
+; GFX7-NEXT:	s_load_dword s0, s[0:1], 0x11
+; GFX7-NEXT:	s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:	v_mov_b32_e32 v0, s6
+; GFX7-NEXT:	v_mov_b32_e32 v2, s8
+; GFX7-NEXT:	v_mov_b32_e32 v4, s10
+; GFX7-NEXT:	s_and_b32 s0, 1, s0
+; GFX7-NEXT:	v_mov_b32_e32 v1, s7
+; GFX7-NEXT:	v_mov_b32_e32 v3, s9
+; GFX7-NEXT:	v_mov_b32_e32 v5, s11
+; GFX7-NEXT:	v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX7-NEXT:	s_nop 3
+; GFX7-NEXT:	v_div_fmas_f64 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX7-NEXT:	v_mov_b32_e32 v2, s4
+; GFX7-NEXT:	v_mov_b32_e32 v3, s5
+; GFX7-NEXT:	flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:	s_endpgm
 ;
 ; GFX8-LABEL: test_div_fmas_f64:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_load_dword s8, s[0:1], 0x44
-; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v4, s6
-; GFX8-NEXT:    s_and_b32 s2, 1, s8
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    v_mov_b32_e32 v5, s7
-; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s2
-; GFX8-NEXT:    s_nop 3
-; GFX8-NEXT:    v_div_fmas_f64 v[0:1], v[0:1], v[2:3], v[4:5]
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GFX8-NEXT:    s_endpgm
+; GFX8-NEXT:	s_load_dwordx8 s[4:11], s[0:1], 0x24
+; GFX8-NEXT:	s_load_dword s0, s[0:1], 0x44
+; GFX8-NEXT:	s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:	v_mov_b32_e32 v0, s6
+; GFX8-NEXT:	v_mov_b32_e32 v2, s8
+; GFX8-NEXT:	v_mov_b32_e32 v4, s10
+; GFX8-NEXT:	s_and_b32 s0, 1, s0
+; GFX8-NEXT:	v_mov_b32_e32 v1, s7
+; GFX8-NEXT:	v_mov_b32_e32 v3, s9
+; GFX8-NEXT:	v_mov_b32_e32 v5, s11
+; GFX8-NEXT:	v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:	s_nop 3
+; GFX8-NEXT:	v_div_fmas_f64 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX8-NEXT:	v_mov_b32_e32 v2, s4
+; GFX8-NEXT:	v_mov_b32_e32 v3, s5
+; GFX8-NEXT:	flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:	s_endpgm
 ;
 ; GFX10_W32-LABEL: test_div_fmas_f64:
 ; GFX10_W32:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.icmp.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.icmp.ll
new file mode 100644
index 0000000000000..dc04f15967a7c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.icmp.ll
@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -global-isel-abort=1 -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+define amdgpu_ps void @test_intr_icmp_eq_i64(i64 addrspace(1)* %out, i32 %src) #0 {
+; GCN-LABEL: test_intr_icmp_eq_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0x64, v2
+; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
+; GCN-NEXT:    s_endpgm
+  %result = call i64 @llvm.amdgcn.icmp.i64.i32(i32 %src, i32 100, i32 32)
+  store i64 %result, i64 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_ps void @test_intr_icmp_ne_i32(i32 addrspace(1)* %out, i32 %src) #1 {
+; GCN-LABEL: test_intr_icmp_ne_i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_cmp_ne_u32_e64 s0, 0x64, v2
+; GCN-NEXT:    ; implicit-def: $vcc_hi
+; GCN-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-NEXT:    global_store_dword v[0:1], v2, off
+; GCN-NEXT:    s_endpgm
+  %result = call i32 @llvm.amdgcn.icmp.i32.i32(i32 %src, i32 100, i32 33)
+  store i32 %result, i32 addrspace(1)* %out
+  ret void
+}
+declare i64 @llvm.amdgcn.icmp.i64.i32(i32, i32, i32)
+declare i32 @llvm.amdgcn.icmp.i32.i32(i32, i32, i32)
+attributes #0 = { "target-features"="+wavefrontsize64" }
+attributes #1 = { "target-features"="+wavefrontsize32" }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.ll
index 77c2f80ac02b6..22826a7e99371 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.ll
@@ -1,2 +1,2 @@
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %S/../llvm.amdgcn.init.exec.ll | FileCheck -check-prefix=GCN %S/../llvm.amdgcn.init.exec.ll
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %S/../llvm.amdgcn.init.exec.ll | FileCheck -check-prefix=GCN %S/../llvm.amdgcn.init.exec.ll
+; RUN: llc -march=amdgcn -mcpu=gfx900 -global-isel -verify-machineinstrs < %S/../llvm.amdgcn.init.exec.ll | FileCheck -check-prefix=GCN %S/../llvm.amdgcn.init.exec.ll
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -global-isel -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %S/../llvm.amdgcn.init.exec.ll | FileCheck -check-prefix=GCN %S/../llvm.amdgcn.init.exec.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
new file mode 100644
index 0000000000000..987c839bf91c4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
@@ -0,0 +1,492 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=PACKED %s
+
+define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(half %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; UNPACKED:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; PACKED:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_tbuffer_store_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<2 x half> %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_tbuffer_store_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+  ; UNPACKED:   TBUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_tbuffer_store_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; PACKED:   TBUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+; FIXME: Crashes
+; define amdgpu_ps void @raw_tbuffer_store_v3f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<3 x half> %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+;   call void @llvm.amdgcn.raw.tbuffer.store.v3f16(<3 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+;   ret void
+; }
+
+define amdgpu_ps void @raw_tbuffer_store_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x half> %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_tbuffer_store_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY]], implicit $exec
+  ; UNPACKED:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY1]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
+  ; UNPACKED:   TBUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7", align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_tbuffer_store_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; PACKED:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; PACKED:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; PACKED:   TBUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 78, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7", align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc
+define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset(half %val, <4 x i32> %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   successors: %bb.2(0x80000000)
+  ; UNPACKED:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; UNPACKED:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; UNPACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; UNPACKED: bb.2:
+  ; UNPACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY7]], implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; UNPACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; UNPACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; UNPACKED:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE3]], [[COPY6]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; UNPACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; UNPACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; UNPACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; UNPACKED: bb.3:
+  ; UNPACKED:   successors: %bb.4(0x80000000)
+  ; UNPACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; UNPACKED: bb.4:
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   successors: %bb.2(0x80000000)
+  ; PACKED:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; PACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; PACKED:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; PACKED:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; PACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; PACKED: bb.2:
+  ; PACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; PACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY7]], implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; PACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; PACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; PACKED:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE3]], [[COPY6]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; PACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; PACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; PACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; PACKED: bb.3:
+  ; PACKED:   successors: %bb.4(0x80000000)
+  ; PACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; PACKED: bb.4:
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc and soffset
+define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; UNPACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   successors: %bb.2(0x80000000)
+  ; UNPACKED:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; UNPACKED:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; UNPACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; UNPACKED: bb.2:
+  ; UNPACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY7]], implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; UNPACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; UNPACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; UNPACKED:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; UNPACKED:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[S_AND_B64_]], implicit-def $scc
+  ; UNPACKED:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; UNPACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; UNPACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; UNPACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; UNPACKED: bb.3:
+  ; UNPACKED:   successors: %bb.4(0x80000000)
+  ; UNPACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; UNPACKED: bb.4:
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   successors: %bb.2(0x80000000)
+  ; PACKED:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; PACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; PACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; PACKED:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; PACKED:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; PACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; PACKED: bb.2:
+  ; PACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; PACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY7]], implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; PACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; PACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; PACKED:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; PACKED:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; PACKED:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[S_AND_B64_]], implicit-def $scc
+  ; PACKED:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; PACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; PACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; PACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; PACKED: bb.3:
+  ; PACKED:   successors: %bb.4(0x80000000)
+  ; PACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; PACKED: bb.4:
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc and soffset, copy for voffset
+define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset(half %val, <4 x i32> %rsrc, i32 inreg %voffset, i32 %soffset) {
+  ; UNPACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   successors: %bb.2(0x80000000)
+  ; UNPACKED:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; UNPACKED:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; UNPACKED:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; UNPACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; UNPACKED: bb.2:
+  ; UNPACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY8]], implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY9]], implicit $exec
+  ; UNPACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; UNPACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; UNPACKED:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; UNPACKED:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[S_AND_B64_]], implicit-def $scc
+  ; UNPACKED:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; UNPACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; UNPACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; UNPACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; UNPACKED: bb.3:
+  ; UNPACKED:   successors: %bb.4(0x80000000)
+  ; UNPACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; UNPACKED: bb.4:
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   successors: %bb.2(0x80000000)
+  ; PACKED:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; PACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; PACKED:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; PACKED:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; PACKED:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; PACKED:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; PACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; PACKED: bb.2:
+  ; PACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; PACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY8]], implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY9]], implicit $exec
+  ; PACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; PACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; PACKED:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; PACKED:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; PACKED:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[S_AND_B64_]], implicit-def $scc
+  ; PACKED:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; PACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; PACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; PACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; PACKED: bb.3:
+  ; PACKED:   successors: %bb.4(0x80000000)
+  ; PACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; PACKED: bb.4:
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc(half %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; UNPACKED:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; PACKED:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
+  ret void
+}
+
+define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc(half %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; UNPACKED:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 1, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; PACKED:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 1, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
+  ret void
+}
+
+define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc(half %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; UNPACKED:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 1, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; PACKED:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 1, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
+  ret void
+}
+
+define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc(half %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; UNPACKED:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, 0, 1, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; PACKED:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, 0, 1, 0, implicit $exec :: (dereferenceable store 2 into custom "TargetCustom7", align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
+  ret void
+}
+
+declare void @llvm.amdgcn.raw.tbuffer.store.f16(half, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
+declare void @llvm.amdgcn.raw.tbuffer.store.v2f16(<2 x half>, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
+declare void @llvm.amdgcn.raw.tbuffer.store.v3f16(<3 x half>, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
+declare void @llvm.amdgcn.raw.tbuffer.store.v4f16(<4 x half>, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.i8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.i8.ll
new file mode 100644
index 0000000000000..e3a09d5dd3679
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.i8.ll
@@ -0,0 +1,284 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=PACKED %s
+
+define amdgpu_ps void @raw_tbuffer_store_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset(i8 %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_tbuffer_store_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; UNPACKED:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 1 into custom "TargetCustom7", addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_tbuffer_store_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; PACKED:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; PACKED:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 1 into custom "TargetCustom7", addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.i8(i8 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc
+define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffset(i8 %val, <4 x i32> %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   successors: %bb.2(0x80000000)
+  ; UNPACKED:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; UNPACKED:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; UNPACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; UNPACKED: bb.2:
+  ; UNPACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY7]], implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; UNPACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; UNPACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; UNPACKED:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE3]], [[COPY6]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 1 into custom "TargetCustom7", addrspace 4)
+  ; UNPACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; UNPACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; UNPACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; UNPACKED: bb.3:
+  ; UNPACKED:   successors: %bb.4(0x80000000)
+  ; UNPACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; UNPACKED: bb.4:
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   successors: %bb.2(0x80000000)
+  ; PACKED:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; PACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; PACKED:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; PACKED:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; PACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; PACKED: bb.2:
+  ; PACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; PACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY7]], implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; PACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; PACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; PACKED:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE3]], [[COPY6]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 1 into custom "TargetCustom7", addrspace 4)
+  ; PACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; PACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; PACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; PACKED: bb.3:
+  ; PACKED:   successors: %bb.4(0x80000000)
+  ; PACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; PACKED: bb.4:
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.i8(i8 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc and soffset
+define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffset(i8 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; UNPACKED-LABEL: name: raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffset
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   successors: %bb.2(0x80000000)
+  ; UNPACKED:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; UNPACKED:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; UNPACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; UNPACKED: bb.2:
+  ; UNPACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY7]], implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; UNPACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; UNPACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; UNPACKED:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; UNPACKED:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[S_AND_B64_]], implicit-def $scc
+  ; UNPACKED:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 1 into custom "TargetCustom7", addrspace 4)
+  ; UNPACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; UNPACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; UNPACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; UNPACKED: bb.3:
+  ; UNPACKED:   successors: %bb.4(0x80000000)
+  ; UNPACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; UNPACKED: bb.4:
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffset
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   successors: %bb.2(0x80000000)
+  ; PACKED:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; PACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; PACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; PACKED:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; PACKED:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; PACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; PACKED: bb.2:
+  ; PACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; PACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY7]], implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; PACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; PACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; PACKED:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; PACKED:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; PACKED:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[S_AND_B64_]], implicit-def $scc
+  ; PACKED:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 1 into custom "TargetCustom7", addrspace 4)
+  ; PACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; PACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; PACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; PACKED: bb.3:
+  ; PACKED:   successors: %bb.4(0x80000000)
+  ; PACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; PACKED: bb.4:
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.i8(i8 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc and soffset, copy for voffset
+define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffset(i8 %val, <4 x i32> %rsrc, i32 inreg %voffset, i32 %soffset) {
+  ; UNPACKED-LABEL: name: raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   successors: %bb.2(0x80000000)
+  ; UNPACKED:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; UNPACKED:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; UNPACKED:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; UNPACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; UNPACKED: bb.2:
+  ; UNPACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY8]], implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY9]], implicit $exec
+  ; UNPACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; UNPACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; UNPACKED:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; UNPACKED:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[S_AND_B64_]], implicit-def $scc
+  ; UNPACKED:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 1 into custom "TargetCustom7", addrspace 4)
+  ; UNPACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; UNPACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; UNPACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; UNPACKED: bb.3:
+  ; UNPACKED:   successors: %bb.4(0x80000000)
+  ; UNPACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; UNPACKED: bb.4:
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   successors: %bb.2(0x80000000)
+  ; PACKED:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; PACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; PACKED:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; PACKED:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; PACKED:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; PACKED:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; PACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; PACKED: bb.2:
+  ; PACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; PACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY8]], implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY9]], implicit $exec
+  ; PACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; PACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; PACKED:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; PACKED:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; PACKED:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[S_AND_B64_]], implicit-def $scc
+  ; PACKED:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 1 into custom "TargetCustom7", addrspace 4)
+  ; PACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; PACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; PACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; PACKED: bb.3:
+  ; PACKED:   successors: %bb.4(0x80000000)
+  ; PACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; PACKED: bb.4:
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.i8(i8 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.raw.tbuffer.store.i8(i8, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.ll
new file mode 100644
index 0000000000000..24047146ffb79
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.ll
@@ -0,0 +1,651 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck %s
+
+; Natural mapping
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(float %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+; Natural mapping
+define amdgpu_ps void @raw_tbuffer_store_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<2 x float> %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; CHECK:   TBUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 78, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+; Natural mapping
+define amdgpu_ps void @raw_tbuffer_store_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<3 x float> %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY6]], %subreg.sub3
+  ; CHECK:   TBUFFER_STORE_FORMAT_XYZ_OFFEN_exact [[REG_SEQUENCE]], [[COPY7]], [[REG_SEQUENCE1]], [[COPY8]], 0, 78, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 12 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.v3f32(<3 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+; Natural mapping
+define amdgpu_ps void @raw_tbuffer_store_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x float> %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; CHECK:   TBUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[REG_SEQUENCE]], [[COPY8]], [[REG_SEQUENCE1]], [[COPY9]], 0, 78, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+; Copies for VGPR arguments
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__sgpr_voffset__sgpr_soffset(float %val, <4 x i32> inreg %rsrc, i32 inreg %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__sgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr0
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE]], [[COPY6]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc
+define amdgpu_ps void @raw_tbuffer_store_f32__vgpr_rsrc__vgpr_voffset__sgpr_soffset(float %val, <4 x i32> %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32_term $exec_lo
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY7]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; CHECK:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE3]], [[COPY6]], 0, 94, 1, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 1)
+  ret void
+}
+
+; Waterfall for rsrc and soffset
+define amdgpu_ps void @raw_tbuffer_store_f32__vgpr_rsrc__vgpr_voffset__vgpr_soffset(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__vgpr_rsrc__vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32_term $exec_lo
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY7]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; CHECK:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; CHECK:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U32_e64_]], [[S_AND_B32_]], implicit-def $scc
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc and soffset, copy for voffset
+define amdgpu_ps void @raw_tbuffer_store_f32__vgpr_rsrc__sgpr_voffset__vgpr_soffset(float %val, <4 x i32> %rsrc, i32 inreg %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; CHECK:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32_term $exec_lo
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY8]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY9]], implicit $exec
+  ; CHECK:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; CHECK:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U32_e64_]], [[S_AND_B32_]], implicit-def $scc
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+; Natural mapping + glc
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc(float %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
+  ret void
+}
+
+; Natural mapping + slc
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc(float %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 1, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
+  ret void
+}
+
+; Natural mapping + glc + slc
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc(float %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 1, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
+  ret void
+}
+
+; Natural mapping + dlc
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc(float %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, 0, 1, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
+  ret void
+}
+
+
+
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vdpr_voffset__sgpr_soffset__voffset0(float %val, <4 x i32> inreg %rsrc, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vdpr_voffset__sgpr_soffset__voffset0
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFSET_exact [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4095(float %val, <4 x i32> inreg %rsrc, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4095
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFSET_exact [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 4095, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 4095, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4096(float %val, <4 x i32> inreg %rsrc, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 4096, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 4096, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add16(float %val, <4 x i32> inreg %rsrc, i32 %voffset.base, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 16, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset = add i32 %voffset.base, 16
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095(float %val, <4 x i32> inreg %rsrc, i32 %voffset.base, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 4095, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset = add i32 %voffset.base, 4095
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4096(float %val, <4 x i32> inreg %rsrc, i32 %voffset.base, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; CHECK:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 4096, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset = add i32 %voffset.base, 4096
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4095(float %val, <4 x i32> inreg %rsrc, i32 %voffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4095
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 4095, i32 94, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4096(float %val, <4 x i32> inreg %rsrc, i32 %voffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 4096, i32 94, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add16(float %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset.base) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; CHECK:   [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY6]], [[S_MOV_B32_]], implicit-def $scc
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %soffset = add i32 %soffset.base, 16
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4095(float %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset.base) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4095
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; CHECK:   [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY6]], [[S_MOV_B32_]], implicit-def $scc
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %soffset = add i32 %soffset.base, 4095
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4096(float %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset.base) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; CHECK:   [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY6]], [[S_MOV_B32_]], implicit-def $scc
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %soffset = add i32 %soffset.base, 4096
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+; An add of the offset is necessary, with a waterfall loop. Make sure the add is done outside of the waterfall loop.
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add5000(float %val, <4 x i32> %rsrc, i32 %voffset, i32 inreg %soffset.base) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add5000
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 5000
+  ; CHECK:   [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY6]], [[S_MOV_B32_]], implicit-def $scc
+  ; CHECK:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32_term $exec_lo
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY7]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; CHECK:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE3]], [[S_ADD_I32_]], 0, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7", align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  %soffset = add i32 %soffset.base, 5000
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+; An add of the offset is necessary, with a waterfall loop. Make sure the add is done outside of the waterfall loop.
+define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add5000(float %val, <4 x i32> %rsrc, i32 %voffset.base, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add5000
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; CHECK:   $vcc_hi = IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; CHECK:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; CHECK:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B32_term:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32_term $exec_lo
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY8]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY9]], implicit $exec
+  ; CHECK:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[V_ADD_U32_e64_]], [[REG_SEQUENCE3]], [[COPY6]], 904, 94, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom "TargetCustom7" + 5000, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  %voffset = add i32 %voffset.base, 5000
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.raw.tbuffer.store.f32(float, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
+declare void @llvm.amdgcn.raw.tbuffer.store.v2f32(<2 x float>, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
+declare void @llvm.amdgcn.raw.tbuffer.store.v3f32(<3 x float>, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
+declare void @llvm.amdgcn.raw.tbuffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bitcast.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bitcast.mir
index aedcb37e758ce..6dcb28b9826d4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bitcast.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bitcast.mir
@@ -11,9 +11,9 @@ body: |
     liveins: $sgpr0
     ; CHECK-LABEL: name: bitcast_s
     ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-    ; CHECK: [[BITCAST:%[0-9]+]]:sgpr(s32) = G_BITCAST [[COPY]](s32)
+    ; CHECK: [[BITCAST:%[0-9]+]]:sgpr(<2 x s16>) = G_BITCAST [[COPY]](s32)
     %0:_(s32) = COPY $sgpr0
-    %1:_(s32) = G_BITCAST %0
+    %1:_(<2 x s16>) = G_BITCAST %0
 ...
 
 ---
@@ -25,7 +25,7 @@ body: |
     liveins: $vgpr0
     ; CHECK-LABEL: name: bitcast_v
     ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; CHECK: [[BITCAST:%[0-9]+]]:vgpr(s32) = G_BITCAST [[COPY]](s32)
+    ; CHECK: [[BITCAST:%[0-9]+]]:vgpr(<2 x s16>) = G_BITCAST [[COPY]](s32)
     %0:_(s32) = COPY $vgpr0
-    %1:_(s32) = G_BITCAST %0
+    %1:_(<2 x s16>) = G_BITCAST %0
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll
index 19c359b9a4f7f..f68465faf61cd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll
@@ -15,28 +15,24 @@ define i32 @v_sdiv_i32(i32 %num, i32 %den) {
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v3
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, v1
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v1
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT:    v_mul_lo_u32 v5, v4, v1
-; GISEL-NEXT:    v_mul_hi_u32 v6, v4, v1
-; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v5
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
-; GISEL-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v4
-; GISEL-NEXT:    v_add_i32_e64 v6, s[4:5], v4, v5
-; GISEL-NEXT:    v_sub_i32_e64 v4, s[4:5], v4, v5
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v4, v4, v0
+; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v4
+; GISEL-NEXT:    v_mul_hi_u32 v5, v4, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT:    v_mul_hi_u32 v4, v0, v4
 ; GISEL-NEXT:    v_mul_lo_u32 v5, v4, v1
 ; GISEL-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
-; GISEL-NEXT:    v_subrev_i32_e32 v7, vcc, 1, v4
-; GISEL-NEXT:    v_sub_i32_e32 v8, vcc, v0, v5
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v5
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v1
-; GISEL-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v4, v6, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v5, s[4:5], v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, 1, v4
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v5, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v2, v3
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
@@ -53,39 +49,32 @@ define i32 @v_sdiv_i32(i32 %num, i32 %den) {
 ; CGP-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; CGP-NEXT:    v_xor_b32_e32 v1, v1, v3
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v2, v1
-; CGP-NEXT:    v_mul_lo_u32 v3, 0, v1
-; CGP-NEXT:    v_mul_lo_u32 v5, 0, v0
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; CGP-NEXT:    v_mul_lo_u32 v5, v0, 0
 ; CGP-NEXT:    v_rcp_f32_e32 v2, v2
-; CGP-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; CGP-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; CGP-NEXT:    v_mul_lo_u32 v6, v2, v1
-; CGP-NEXT:    v_mul_lo_u32 v7, v2, 0
-; CGP-NEXT:    v_mul_hi_u32 v8, v2, v1
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; CGP-NEXT:    v_sub_i32_e32 v9, vcc, 0, v6
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v8
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; CGP-NEXT:    v_cndmask_b32_e32 v3, v6, v9, vcc
-; CGP-NEXT:    v_mul_lo_u32 v6, v3, 0
-; CGP-NEXT:    v_mul_hi_u32 v3, v3, v2
-; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
-; CGP-NEXT:    v_add_i32_e64 v3, s[4:5], v6, v3
-; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v2, v3
-; CGP-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; CGP-NEXT:    v_mul_lo_u32 v3, v2, 0
-; CGP-NEXT:    v_mul_hi_u32 v2, v2, v0
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
+; CGP-NEXT:    v_mul_lo_u32 v3, v3, v2
+; CGP-NEXT:    v_mul_lo_u32 v6, v2, 0
+; CGP-NEXT:    v_mul_lo_u32 v7, 0, v3
+; CGP-NEXT:    v_mul_hi_u32 v3, v2, v3
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT:    v_mul_lo_u32 v3, 0, v2
+; CGP-NEXT:    v_mul_hi_u32 v2, v0, v2
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
 ; CGP-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; CGP-NEXT:    v_mul_lo_u32 v3, v2, v1
 ; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v2
-; CGP-NEXT:    v_subrev_i32_e32 v6, vcc, 1, v2
-; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v0, v3
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v1
-; CGP-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v2, v5, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, 1, v2
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; CGP-NEXT:    v_xor_b32_e32 v0, v0, v4
 ; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
@@ -103,31 +92,27 @@ define amdgpu_ps i32 @s_sdiv_i32(i32 inreg %num, i32 inreg %den) {
 ; GISEL-NEXT:    s_ashr_i32 s3, s1, 31
 ; GISEL-NEXT:    s_add_i32 s0, s0, s2
 ; GISEL-NEXT:    s_add_i32 s1, s1, s3
-; GISEL-NEXT:    s_xor_b32 s4, s0, s2
-; GISEL-NEXT:    s_xor_b32 s5, s1, s3
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v0, s5
+; GISEL-NEXT:    s_xor_b32 s0, s0, s2
+; GISEL-NEXT:    s_xor_b32 s4, s1, s3
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v0, s4
+; GISEL-NEXT:    s_sub_i32 s1, 0, s4
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GISEL-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GISEL-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GISEL-NEXT:    v_mul_lo_u32 v1, v0, s5
-; GISEL-NEXT:    v_mul_hi_u32 v2, v0, s5
-; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GISEL-NEXT:    v_add_i32_e64 v2, s[0:1], v0, v1
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[0:1], v0, v1
+; GISEL-NEXT:    v_mul_lo_u32 v1, s1, v0
+; GISEL-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GISEL-NEXT:    v_mul_lo_u32 v1, v0, s4
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, s0, v1
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v1
 ; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v0, v0, s4
-; GISEL-NEXT:    v_mul_lo_u32 v1, v0, s5
+; GISEL-NEXT:    v_subrev_i32_e64 v2, s[0:1], s4, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
-; GISEL-NEXT:    v_subrev_i32_e32 v3, vcc, 1, v0
-; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, s4, v1
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, s4, v1
-; GISEL-NEXT:    v_cmp_le_u32_e64 s[0:1], s5, v4
-; GISEL-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-NEXT:    s_xor_b32 s0, s2, s3
 ; GISEL-NEXT:    v_xor_b32_e32 v0, s0, v0
 ; GISEL-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
@@ -138,50 +123,41 @@ define amdgpu_ps i32 @s_sdiv_i32(i32 inreg %num, i32 inreg %den) {
 ; CGP:       ; %bb.0:
 ; CGP-NEXT:    s_ashr_i32 s2, s0, 31
 ; CGP-NEXT:    s_ashr_i32 s3, s1, 31
-; CGP-NEXT:    s_xor_b32 s5, s2, s3
+; CGP-NEXT:    s_xor_b32 s4, s2, s3
 ; CGP-NEXT:    s_add_i32 s0, s0, s2
 ; CGP-NEXT:    s_add_i32 s1, s1, s3
-; CGP-NEXT:    s_xor_b32 s2, s0, s2
-; CGP-NEXT:    s_xor_b32 s4, s1, s3
-; CGP-NEXT:    v_cvt_f32_u32_e32 v0, s4
-; CGP-NEXT:    s_bfe_u64 s[0:1], s[4:5], 0x200000
-; CGP-NEXT:    s_bfe_u64 s[6:7], s[2:3], 0x200000
+; CGP-NEXT:    s_xor_b32 s0, s0, s2
+; CGP-NEXT:    s_xor_b32 s5, s1, s3
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, s5
+; CGP-NEXT:    s_sub_i32 s1, 0, s5
+; CGP-NEXT:    s_bfe_u64 s[2:3], s[0:1], 0x200000
 ; CGP-NEXT:    v_rcp_f32_e32 v0, v0
-; CGP-NEXT:    v_mul_lo_u32 v1, 0, s0
-; CGP-NEXT:    v_mul_lo_u32 v2, 0, s6
-; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; CGP-NEXT:    v_mul_lo_u32 v1, s2, 0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; CGP-NEXT:    v_mul_lo_u32 v3, v0, s0
-; CGP-NEXT:    v_mul_lo_u32 v4, v0, s1
-; CGP-NEXT:    v_mul_hi_u32 v5, v0, s0
-; CGP-NEXT:    v_mul_lo_u32 v6, 0, v0
-; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
-; CGP-NEXT:    v_sub_i32_e32 v4, vcc, 0, v3
-; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; CGP-NEXT:    v_mul_lo_u32 v3, v1, 0
-; CGP-NEXT:    v_mul_hi_u32 v1, v1, v0
-; CGP-NEXT:    v_add_i32_e64 v3, s[0:1], v6, v3
-; CGP-NEXT:    v_add_i32_e64 v1, s[0:1], v3, v1
-; CGP-NEXT:    v_add_i32_e64 v3, s[0:1], v0, v1
-; CGP-NEXT:    v_sub_i32_e64 v0, s[0:1], v0, v1
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; CGP-NEXT:    v_mul_lo_u32 v1, v0, s7
-; CGP-NEXT:    v_mul_hi_u32 v0, v0, s6
+; CGP-NEXT:    v_mul_lo_u32 v2, s1, v0
+; CGP-NEXT:    v_mul_lo_u32 v3, v0, 0
+; CGP-NEXT:    v_mul_lo_u32 v4, 0, v2
+; CGP-NEXT:    v_mul_hi_u32 v2, v0, v2
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; CGP-NEXT:    v_mul_lo_u32 v2, s3, v0
+; CGP-NEXT:    v_mul_hi_u32 v0, s2, v0
 ; CGP-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; CGP-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
-; CGP-NEXT:    v_mul_lo_u32 v1, v0, s4
+; CGP-NEXT:    v_mul_lo_u32 v1, v0, s5
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, s0, v1
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s5, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CGP-NEXT:    v_subrev_i32_e64 v2, s[0:1], s5, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
 ; CGP-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
-; CGP-NEXT:    v_subrev_i32_e32 v3, vcc, 1, v0
-; CGP-NEXT:    v_sub_i32_e32 v4, vcc, s2, v1
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, s2, v1
-; CGP-NEXT:    v_cmp_le_u32_e64 s[0:1], s4, v4
-; CGP-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; CGP-NEXT:    v_xor_b32_e32 v0, s5, v0
-; CGP-NEXT:    v_subrev_i32_e32 v0, vcc, s5, v0
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s5, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CGP-NEXT:    v_xor_b32_e32 v0, s4, v0
+; CGP-NEXT:    v_subrev_i32_e32 v0, vcc, s4, v0
 ; CGP-NEXT:    v_readfirstlane_b32 s0, v0
 ; CGP-NEXT:    ; return to shader part epilog
   %result = sdiv i32 %num, %den
@@ -208,51 +184,43 @@ define <2 x i32> @v_sdiv_v2i32(<2 x i32> %num, <2 x i32> %den) {
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v6
 ; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v7
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v3
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v6, v3
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v3
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; GISEL-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
+; GISEL-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GISEL-NEXT:    v_mul_lo_u32 v6, v4, v2
-; GISEL-NEXT:    v_mul_hi_u32 v7, v4, v2
-; GISEL-NEXT:    v_mul_lo_u32 v10, v5, v3
-; GISEL-NEXT:    v_mul_hi_u32 v11, v5, v3
-; GISEL-NEXT:    v_sub_i32_e32 v12, vcc, 0, v6
-; GISEL-NEXT:    v_sub_i32_e32 v13, vcc, 0, v10
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v6, v6, v12, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v11
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, v10, v13, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v6, v6, v4
-; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v5
-; GISEL-NEXT:    v_add_i32_e64 v10, s[6:7], v4, v6
-; GISEL-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v6
-; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v5, v7
-; GISEL-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v4, v4, v0
-; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v1
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v4
+; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v6
+; GISEL-NEXT:    v_mul_hi_u32 v5, v4, v5
+; GISEL-NEXT:    v_mul_hi_u32 v7, v6, v7
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v6, v7
+; GISEL-NEXT:    v_mul_hi_u32 v4, v0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v5, v1, v5
 ; GISEL-NEXT:    v_mul_lo_u32 v6, v4, v2
 ; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
-; GISEL-NEXT:    v_subrev_i32_e32 v10, vcc, 1, v4
-; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v3
-; GISEL-NEXT:    v_add_i32_e32 v12, vcc, 1, v5
-; GISEL-NEXT:    v_subrev_i32_e32 v13, vcc, 1, v5
-; GISEL-NEXT:    v_sub_i32_e32 v14, vcc, v0, v6
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v6
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v11
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v11
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[6:7], v14, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[8:9], v0, v3
-; GISEL-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v4, v7, s[6:7]
-; GISEL-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v5, v12, s[6:7]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v10, v5, v3
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, 1, v5
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v10
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v6, s[4:5], v0, v2
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v7, s[6:7], v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v5
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v6, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v7, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v8
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v9
 ; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v8
@@ -277,73 +245,59 @@ define <2 x i32> @v_sdiv_v2i32(<2 x i32> %num, <2 x i32> %den) {
 ; CGP-NEXT:    v_xor_b32_e32 v1, v1, v6
 ; CGP-NEXT:    v_xor_b32_e32 v3, v3, v7
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, 0, v2
-; CGP-NEXT:    v_mul_lo_u32 v6, 0, v0
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; CGP-NEXT:    v_mul_lo_u32 v6, v0, 0
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v7, v3
-; CGP-NEXT:    v_mul_lo_u32 v10, 0, v3
-; CGP-NEXT:    v_mul_lo_u32 v11, 0, v1
+; CGP-NEXT:    v_sub_i32_e32 v10, vcc, 0, v3
+; CGP-NEXT:    v_mul_lo_u32 v11, v1, 0
 ; CGP-NEXT:    v_rcp_f32_e32 v4, v4
 ; CGP-NEXT:    v_rcp_f32_e32 v7, v7
-; CGP-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f800000, v7
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; CGP-NEXT:    v_mul_lo_u32 v12, v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v13, v4, 0
-; CGP-NEXT:    v_mul_hi_u32 v14, v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v15, v7, v3
-; CGP-NEXT:    v_mul_lo_u32 v16, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v17, v7, v3
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v13
-; CGP-NEXT:    v_sub_i32_e32 v18, vcc, 0, v12
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v16
-; CGP-NEXT:    v_sub_i32_e32 v19, vcc, 0, v15
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v14
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v17
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; CGP-NEXT:    v_cndmask_b32_e32 v5, v12, v18, vcc
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v10
-; CGP-NEXT:    v_cndmask_b32_e64 v10, v15, v19, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v12, v5, 0
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v4
-; CGP-NEXT:    v_mul_lo_u32 v14, v10, 0
-; CGP-NEXT:    v_mul_hi_u32 v10, v10, v7
-; CGP-NEXT:    v_add_i32_e64 v12, s[6:7], v13, v12
-; CGP-NEXT:    v_add_i32_e64 v13, s[6:7], v16, v14
-; CGP-NEXT:    v_add_i32_e64 v5, s[6:7], v12, v5
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v13, v10
-; CGP-NEXT:    v_add_i32_e64 v12, s[6:7], v4, v5
-; CGP-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v5
-; CGP-NEXT:    v_add_i32_e64 v5, s[6:7], v7, v10
-; CGP-NEXT:    v_sub_i32_e64 v7, s[6:7], v7, v10
-; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v7, v4, 0
-; CGP-NEXT:    v_mul_hi_u32 v4, v4, v0
-; CGP-NEXT:    v_mul_lo_u32 v10, v5, 0
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v1
-; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v11, v10
+; CGP-NEXT:    v_mul_lo_u32 v5, v5, v4
+; CGP-NEXT:    v_mul_lo_u32 v12, v4, 0
+; CGP-NEXT:    v_mul_lo_u32 v10, v10, v7
+; CGP-NEXT:    v_mul_lo_u32 v13, v7, 0
+; CGP-NEXT:    v_mul_lo_u32 v14, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v4, v5
+; CGP-NEXT:    v_mul_lo_u32 v15, 0, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v7, v10
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v14, v12
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v15, v13
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v12, v5
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v7, v10
+; CGP-NEXT:    v_mul_lo_u32 v7, 0, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v0, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v1, v5
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v10, v11
 ; CGP-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
 ; CGP-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
 ; CGP-NEXT:    v_mul_lo_u32 v6, v4, v2
 ; CGP-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
-; CGP-NEXT:    v_subrev_i32_e32 v10, vcc, 1, v4
-; CGP-NEXT:    v_mul_lo_u32 v11, v5, v3
-; CGP-NEXT:    v_add_i32_e32 v12, vcc, 1, v5
-; CGP-NEXT:    v_subrev_i32_e32 v13, vcc, 1, v5
-; CGP-NEXT:    v_sub_i32_e32 v14, vcc, v0, v6
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v6
-; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v11
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v11
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v14, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[8:9], v0, v3
-; CGP-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v4, v7, s[6:7]
-; CGP-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v5, v12, s[6:7]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v10, v5, v3
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, 1, v5
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v10
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; CGP-NEXT:    v_sub_i32_e64 v6, s[4:5], v0, v2
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v7, s[6:7], v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, 1, v5
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v4, v6, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v5, v7, vcc
 ; CGP-NEXT:    v_xor_b32_e32 v0, v0, v8
 ; CGP-NEXT:    v_xor_b32_e32 v1, v1, v9
 ; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v8
@@ -359,31 +313,27 @@ define i32 @v_sdiv_i32_pow2k_denom(i32 %num) {
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_movk_i32 s6, 0x1000
 ; CHECK-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0xfffff000
 ; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, s6
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, s6
 ; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v1
-; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; CHECK-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; CHECK-NEXT:    v_lshlrev_b32_e32 v3, 12, v2
-; CHECK-NEXT:    v_mul_hi_u32 v4, v2, s6
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v2
-; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v2, v3
-; CHECK-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v2, v2, v0
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; CHECK-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CHECK-NEXT:    v_mul_lo_u32 v2, v2, v3
+; CHECK-NEXT:    v_mul_hi_u32 v2, v3, v2
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v0, v2
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v3, 12, v2
 ; CHECK-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; CHECK-NEXT:    v_subrev_i32_e32 v5, vcc, 1, v2
-; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v0, v3
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v6
-; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CHECK-NEXT:    v_subrev_i32_e64 v3, s[4:5], s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, 1, v2
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
@@ -400,52 +350,43 @@ define <2 x i32> @v_sdiv_v2i32_pow2k_denom(<2 x i32> %num) {
 ; GISEL-NEXT:    v_ashrrev_i32_e32 v3, 31, v1
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s8
+; GISEL-NEXT:    s_sub_i32 s4, 0, s8
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v3
-; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v4
-; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f7ffffe, v4
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT:    v_mul_lo_u32 v6, v5, s8
-; GISEL-NEXT:    v_mul_hi_u32 v7, v5, s8
-; GISEL-NEXT:    v_mul_lo_u32 v8, v4, s8
-; GISEL-NEXT:    v_mul_hi_u32 v9, v4, s8
-; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, 0, v6
-; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, 0, v8
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, v8, v11, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v6, v6, v5
-; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v4
-; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v5, v6
-; GISEL-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v6
-; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v4, v7
-; GISEL-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v5, v5, v8, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v0
-; GISEL-NEXT:    v_mul_hi_u32 v4, v4, v1
+; GISEL-NEXT:    v_mul_lo_u32 v6, s4, v5
+; GISEL-NEXT:    v_mul_lo_u32 v7, s4, v4
+; GISEL-NEXT:    v_mul_hi_u32 v6, v5, v6
+; GISEL-NEXT:    v_mul_hi_u32 v7, v4, v7
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT:    v_mul_hi_u32 v5, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v4, v1, v4
 ; GISEL-NEXT:    v_mul_lo_u32 v6, v5, s8
 ; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v5
-; GISEL-NEXT:    v_subrev_i32_e32 v8, vcc, 1, v5
-; GISEL-NEXT:    v_mul_lo_u32 v9, v4, s8
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, 1, v4
-; GISEL-NEXT:    v_subrev_i32_e32 v11, vcc, 1, v4
-; GISEL-NEXT:    v_sub_i32_e32 v12, vcc, v0, v6
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v6
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v9
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v9
-; GISEL-NEXT:    v_cmp_le_u32_e64 s[6:7], s8, v12
-; GISEL-NEXT:    v_cmp_le_u32_e64 s[8:9], s8, v0
-; GISEL-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v5, v7, s[6:7]
-; GISEL-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v4, v10, s[6:7]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v11, v1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v8, v4, s8
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, 1, v4
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v8
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GISEL-NEXT:    v_subrev_i32_e64 v6, s[4:5], s8, v0
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s8, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, v4, v9, s[4:5]
+; GISEL-NEXT:    v_subrev_i32_e64 v7, s[6:7], s8, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, 1, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v5, v6, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s8, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v4, v7, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v3
 ; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
@@ -458,61 +399,53 @@ define <2 x i32> @v_sdiv_v2i32_pow2k_denom(<2 x i32> %num) {
 ; CGP-NEXT:    s_movk_i32 s4, 0x1000
 ; CGP-NEXT:    v_ashrrev_i32_e32 v2, 31, v0
 ; CGP-NEXT:    v_mov_b32_e32 v3, 0x1000
-; CGP-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
+; CGP-NEXT:    s_mov_b32 s5, 0xfffff000
+; CGP-NEXT:    v_mov_b32_e32 v4, 0xfffff000
+; CGP-NEXT:    v_ashrrev_i32_e32 v5, 31, v1
 ; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; CGP-NEXT:    v_cvt_f32_u32_e32 v5, s4
-; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
-; CGP-NEXT:    v_cvt_f32_u32_e32 v6, v3
+; CGP-NEXT:    v_cvt_f32_u32_e32 v6, s4
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
+; CGP-NEXT:    v_cvt_f32_u32_e32 v7, v3
 ; CGP-NEXT:    v_xor_b32_e32 v0, v0, v2
-; CGP-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; CGP-NEXT:    v_xor_b32_e32 v1, v1, v4
 ; CGP-NEXT:    v_rcp_iflag_f32_e32 v6, v6
-; CGP-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v5
-; CGP-NEXT:    v_mul_f32_e32 v6, 0x4f800000, v6
-; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v5
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v7, v7
+; CGP-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
-; CGP-NEXT:    v_lshlrev_b32_e32 v7, 12, v5
-; CGP-NEXT:    v_mul_hi_u32 v8, v5, s4
-; CGP-NEXT:    v_lshlrev_b32_e32 v9, 12, v6
-; CGP-NEXT:    v_mul_hi_u32 v10, v6, v3
-; CGP-NEXT:    v_sub_i32_e32 v11, vcc, 0, v7
-; CGP-NEXT:    v_sub_i32_e32 v12, vcc, 0, v9
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v8
-; CGP-NEXT:    v_cndmask_b32_e32 v7, v7, v11, vcc
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v10
-; CGP-NEXT:    v_cndmask_b32_e64 v8, v9, v12, s[4:5]
-; CGP-NEXT:    v_mul_hi_u32 v7, v7, v5
-; CGP-NEXT:    v_mul_hi_u32 v8, v8, v6
-; CGP-NEXT:    v_add_i32_e64 v9, s[6:7], v5, v7
-; CGP-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v7
-; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v6, v8
-; CGP-NEXT:    v_sub_i32_e64 v6, s[6:7], v6, v8
-; CGP-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[4:5]
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v0
-; CGP-NEXT:    v_mul_hi_u32 v6, v6, v1
-; CGP-NEXT:    v_lshlrev_b32_e32 v7, 12, v5
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, 1, v5
-; CGP-NEXT:    v_subrev_i32_e32 v9, vcc, 1, v5
-; CGP-NEXT:    v_lshlrev_b32_e32 v10, 12, v6
-; CGP-NEXT:    v_add_i32_e32 v11, vcc, 1, v6
-; CGP-NEXT:    v_subrev_i32_e32 v12, vcc, 1, v6
-; CGP-NEXT:    v_sub_i32_e32 v13, vcc, v0, v7
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v7
-; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v10
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v10
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v13, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[8:9], v0, v3
-; CGP-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v5, v8, s[6:7]
-; CGP-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v6, v11, s[6:7]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v12, v1, s[4:5]
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_mul_lo_u32 v8, s5, v6
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, v7
+; CGP-NEXT:    v_mul_hi_u32 v8, v6, v8
+; CGP-NEXT:    v_mul_hi_u32 v4, v7, v4
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
+; CGP-NEXT:    v_mul_hi_u32 v6, v0, v6
+; CGP-NEXT:    v_mul_hi_u32 v4, v1, v4
+; CGP-NEXT:    v_lshlrev_b32_e32 v7, 12, v6
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, 1, v6
+; CGP-NEXT:    v_lshlrev_b32_e32 v9, 12, v4
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, 1, v4
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v7
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v9
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v6, v6, v8, vcc
+; CGP-NEXT:    v_subrev_i32_e64 v7, s[4:5], s4, v0
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v8, s[6:7], v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, 1, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v1, v8, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v6, v7, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v4, v8, vcc
 ; CGP-NEXT:    v_xor_b32_e32 v0, v0, v2
-; CGP-NEXT:    v_xor_b32_e32 v1, v1, v4
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v5
 ; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
-; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %result = sdiv <2 x i32> %num, <i32 4096, i32 4096>
   ret <2 x i32> %result
@@ -524,31 +457,27 @@ define i32 @v_sdiv_i32_oddk_denom(i32 %num) {
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_mov_b32 s6, 0x12d8fb
 ; CHECK-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0xffed2705
 ; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, s6
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, s6
 ; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v1
-; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; CHECK-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; CHECK-NEXT:    v_mul_lo_u32 v3, v2, s6
-; CHECK-NEXT:    v_mul_hi_u32 v4, v2, s6
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v2
-; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v2, v3
-; CHECK-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v2, v2, v0
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; CHECK-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CHECK-NEXT:    v_mul_lo_u32 v2, v2, v3
+; CHECK-NEXT:    v_mul_hi_u32 v2, v3, v2
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v0, v2
 ; CHECK-NEXT:    v_mul_lo_u32 v3, v2, s6
 ; CHECK-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; CHECK-NEXT:    v_subrev_i32_e32 v5, vcc, 1, v2
-; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v0, v3
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v6
-; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CHECK-NEXT:    v_subrev_i32_e64 v3, s[4:5], s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, 1, v2
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
@@ -565,52 +494,43 @@ define <2 x i32> @v_sdiv_v2i32_oddk_denom(<2 x i32> %num) {
 ; GISEL-NEXT:    v_ashrrev_i32_e32 v3, 31, v1
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s8
+; GISEL-NEXT:    s_sub_i32 s4, 0, s8
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v3
-; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v4
-; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f7ffffe, v4
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT:    v_mul_lo_u32 v6, v5, s8
-; GISEL-NEXT:    v_mul_hi_u32 v7, v5, s8
-; GISEL-NEXT:    v_mul_lo_u32 v8, v4, s8
-; GISEL-NEXT:    v_mul_hi_u32 v9, v4, s8
-; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, 0, v6
-; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, 0, v8
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, v8, v11, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v6, v6, v5
-; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v4
-; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v5, v6
-; GISEL-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v6
-; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v4, v7
-; GISEL-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v5, v5, v8, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v0
-; GISEL-NEXT:    v_mul_hi_u32 v4, v4, v1
+; GISEL-NEXT:    v_mul_lo_u32 v6, s4, v5
+; GISEL-NEXT:    v_mul_lo_u32 v7, s4, v4
+; GISEL-NEXT:    v_mul_hi_u32 v6, v5, v6
+; GISEL-NEXT:    v_mul_hi_u32 v7, v4, v7
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT:    v_mul_hi_u32 v5, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v4, v1, v4
 ; GISEL-NEXT:    v_mul_lo_u32 v6, v5, s8
 ; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v5
-; GISEL-NEXT:    v_subrev_i32_e32 v8, vcc, 1, v5
-; GISEL-NEXT:    v_mul_lo_u32 v9, v4, s8
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, 1, v4
-; GISEL-NEXT:    v_subrev_i32_e32 v11, vcc, 1, v4
-; GISEL-NEXT:    v_sub_i32_e32 v12, vcc, v0, v6
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v6
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v9
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v9
-; GISEL-NEXT:    v_cmp_le_u32_e64 s[6:7], s8, v12
-; GISEL-NEXT:    v_cmp_le_u32_e64 s[8:9], s8, v0
-; GISEL-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v5, v7, s[6:7]
-; GISEL-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v4, v10, s[6:7]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v11, v1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v8, v4, s8
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, 1, v4
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v8
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GISEL-NEXT:    v_subrev_i32_e64 v6, s[4:5], s8, v0
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s8, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, v4, v9, s[4:5]
+; GISEL-NEXT:    v_subrev_i32_e64 v7, s[6:7], s8, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, 1, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v5, v6, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s8, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v4, v7, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v3
 ; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
@@ -620,64 +540,56 @@ define <2 x i32> @v_sdiv_v2i32_oddk_denom(<2 x i32> %num) {
 ; CGP-LABEL: v_sdiv_v2i32_oddk_denom:
 ; CGP:       ; %bb.0:
 ; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT:    s_mov_b32 s8, 0x12d8fb
+; CGP-NEXT:    s_mov_b32 s4, 0x12d8fb
 ; CGP-NEXT:    v_ashrrev_i32_e32 v2, 31, v0
 ; CGP-NEXT:    v_mov_b32_e32 v3, 0x12d8fb
-; CGP-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
+; CGP-NEXT:    s_mov_b32 s5, 0xffed2705
+; CGP-NEXT:    v_mov_b32_e32 v4, 0xffed2705
+; CGP-NEXT:    v_ashrrev_i32_e32 v5, 31, v1
 ; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; CGP-NEXT:    v_cvt_f32_u32_e32 v5, s8
-; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
-; CGP-NEXT:    v_cvt_f32_u32_e32 v6, v3
+; CGP-NEXT:    v_cvt_f32_u32_e32 v6, s4
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
+; CGP-NEXT:    v_cvt_f32_u32_e32 v7, v3
 ; CGP-NEXT:    v_xor_b32_e32 v0, v0, v2
-; CGP-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; CGP-NEXT:    v_xor_b32_e32 v1, v1, v4
 ; CGP-NEXT:    v_rcp_iflag_f32_e32 v6, v6
-; CGP-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v5
-; CGP-NEXT:    v_mul_f32_e32 v6, 0x4f800000, v6
-; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v5
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v7, v7
+; CGP-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
-; CGP-NEXT:    v_mul_lo_u32 v7, v5, s8
-; CGP-NEXT:    v_mul_hi_u32 v8, v5, s8
-; CGP-NEXT:    v_mul_lo_u32 v9, v6, v3
-; CGP-NEXT:    v_mul_hi_u32 v10, v6, v3
-; CGP-NEXT:    v_sub_i32_e32 v11, vcc, 0, v7
-; CGP-NEXT:    v_sub_i32_e32 v12, vcc, 0, v9
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v8
-; CGP-NEXT:    v_cndmask_b32_e32 v7, v7, v11, vcc
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v10
-; CGP-NEXT:    v_cndmask_b32_e64 v8, v9, v12, s[4:5]
-; CGP-NEXT:    v_mul_hi_u32 v7, v7, v5
-; CGP-NEXT:    v_mul_hi_u32 v8, v8, v6
-; CGP-NEXT:    v_add_i32_e64 v9, s[6:7], v5, v7
-; CGP-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v7
-; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v6, v8
-; CGP-NEXT:    v_sub_i32_e64 v6, s[6:7], v6, v8
-; CGP-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[4:5]
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v0
-; CGP-NEXT:    v_mul_hi_u32 v6, v6, v1
-; CGP-NEXT:    v_mul_lo_u32 v7, v5, s8
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, 1, v5
-; CGP-NEXT:    v_subrev_i32_e32 v9, vcc, 1, v5
-; CGP-NEXT:    v_mul_lo_u32 v10, v6, v3
-; CGP-NEXT:    v_add_i32_e32 v11, vcc, 1, v6
-; CGP-NEXT:    v_subrev_i32_e32 v12, vcc, 1, v6
-; CGP-NEXT:    v_sub_i32_e32 v13, vcc, v0, v7
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v7
-; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v10
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v10
-; CGP-NEXT:    v_cmp_le_u32_e64 s[6:7], s8, v13
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[8:9], v0, v3
-; CGP-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v5, v8, s[6:7]
-; CGP-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v6, v11, s[6:7]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v12, v1, s[4:5]
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_mul_lo_u32 v8, s5, v6
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, v7
+; CGP-NEXT:    v_mul_hi_u32 v8, v6, v8
+; CGP-NEXT:    v_mul_hi_u32 v4, v7, v4
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
+; CGP-NEXT:    v_mul_hi_u32 v6, v0, v6
+; CGP-NEXT:    v_mul_hi_u32 v4, v1, v4
+; CGP-NEXT:    v_mul_lo_u32 v7, v6, s4
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, 1, v6
+; CGP-NEXT:    v_mul_lo_u32 v9, v4, v3
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, 1, v4
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v7
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v9
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v6, v6, v8, vcc
+; CGP-NEXT:    v_subrev_i32_e64 v7, s[4:5], s4, v0
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v8, s[6:7], v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, 1, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v1, v8, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v6, v7, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v4, v8, vcc
 ; CGP-NEXT:    v_xor_b32_e32 v0, v0, v2
-; CGP-NEXT:    v_xor_b32_e32 v1, v1, v4
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v5
 ; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
-; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %result = sdiv <2 x i32> %num, <i32 1235195, i32 1235195>
   ret <2 x i32> %result
@@ -695,28 +607,24 @@ define i32 @v_sdiv_i32_pow2_shl_denom(i32 %x, i32 %y) {
 ; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v3
 ; CHECK-NEXT:    v_cvt_f32_u32_e32 v4, v1
+; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, 0, v1
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; CHECK-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
+; CHECK-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
 ; CHECK-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; CHECK-NEXT:    v_mul_lo_u32 v5, v4, v1
-; CHECK-NEXT:    v_mul_hi_u32 v6, v4, v1
-; CHECK-NEXT:    v_sub_i32_e32 v7, vcc, 0, v5
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
-; CHECK-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v5, v5, v4
-; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v4, v5
-; CHECK-NEXT:    v_sub_i32_e64 v4, s[4:5], v4, v5
-; CHECK-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v4, v4, v0
+; CHECK-NEXT:    v_mul_lo_u32 v5, v5, v4
+; CHECK-NEXT:    v_mul_hi_u32 v5, v4, v5
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; CHECK-NEXT:    v_mul_hi_u32 v4, v0, v4
 ; CHECK-NEXT:    v_mul_lo_u32 v5, v4, v1
 ; CHECK-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
-; CHECK-NEXT:    v_subrev_i32_e32 v7, vcc, 1, v4
-; CHECK-NEXT:    v_sub_i32_e32 v8, vcc, v0, v5
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v5
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v1
-; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v4, v6, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v5, s[4:5], v0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, 1, v4
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v4, v5, vcc
 ; CHECK-NEXT:    v_xor_b32_e32 v1, v2, v3
 ; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
@@ -748,51 +656,43 @@ define <2 x i32> @v_sdiv_v2i32_pow2_shl_denom(<2 x i32> %x, <2 x i32> %y) {
 ; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v6
 ; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v7
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v6, v2
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v7, v3
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v2
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v8, v3
+; GISEL-NEXT:    v_sub_i32_e32 v9, vcc, 0, v3
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v6, v6
-; GISEL-NEXT:    v_rcp_iflag_f32_e32 v7, v7
-; GISEL-NEXT:    v_mul_f32_e32 v6, 0x4f800000, v6
-; GISEL-NEXT:    v_mul_f32_e32 v7, 0x4f800000, v7
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v8, v8
+; GISEL-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; GISEL-NEXT:    v_mul_f32_e32 v8, 0x4f7ffffe, v8
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; GISEL-NEXT:    v_mul_lo_u32 v8, v6, v2
-; GISEL-NEXT:    v_mul_hi_u32 v9, v6, v2
-; GISEL-NEXT:    v_mul_lo_u32 v10, v7, v3
-; GISEL-NEXT:    v_mul_hi_u32 v11, v7, v3
-; GISEL-NEXT:    v_sub_i32_e32 v12, vcc, 0, v8
-; GISEL-NEXT:    v_sub_i32_e32 v13, vcc, 0, v10
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v9
-; GISEL-NEXT:    v_cndmask_b32_e32 v8, v8, v12, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v11
-; GISEL-NEXT:    v_cndmask_b32_e64 v9, v10, v13, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v8, v8, v6
-; GISEL-NEXT:    v_mul_hi_u32 v9, v9, v7
-; GISEL-NEXT:    v_add_i32_e64 v10, s[6:7], v6, v8
-; GISEL-NEXT:    v_sub_i32_e64 v6, s[6:7], v6, v8
-; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v7, v9
-; GISEL-NEXT:    v_sub_i32_e64 v7, s[6:7], v7, v9
-; GISEL-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v6, v6, v0
-; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v1
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v6
+; GISEL-NEXT:    v_mul_lo_u32 v9, v9, v8
+; GISEL-NEXT:    v_mul_hi_u32 v7, v6, v7
+; GISEL-NEXT:    v_mul_hi_u32 v9, v8, v9
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v9
+; GISEL-NEXT:    v_mul_hi_u32 v6, v0, v6
+; GISEL-NEXT:    v_mul_hi_u32 v7, v1, v7
 ; GISEL-NEXT:    v_mul_lo_u32 v8, v6, v2
 ; GISEL-NEXT:    v_add_i32_e32 v9, vcc, 1, v6
-; GISEL-NEXT:    v_subrev_i32_e32 v10, vcc, 1, v6
-; GISEL-NEXT:    v_mul_lo_u32 v11, v7, v3
-; GISEL-NEXT:    v_add_i32_e32 v12, vcc, 1, v7
-; GISEL-NEXT:    v_subrev_i32_e32 v13, vcc, 1, v7
-; GISEL-NEXT:    v_sub_i32_e32 v14, vcc, v0, v8
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v8
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v11
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v11
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[6:7], v14, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[8:9], v0, v3
-; GISEL-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v6, v9, s[6:7]
-; GISEL-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v7, v12, s[6:7]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v10, v7, v3
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, 1, v7
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v10
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v6, v6, v9, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v8, s[4:5], v0, v2
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, v7, v11, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v9, s[6:7], v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, 1, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, 1, v7
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v6, v8, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v7, v9, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v4
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v5
 ; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
@@ -815,78 +715,64 @@ define <2 x i32> @v_sdiv_v2i32_pow2_shl_denom(<2 x i32> %x, <2 x i32> %y) {
 ; CGP-NEXT:    v_xor_b32_e32 v1, v1, v5
 ; CGP-NEXT:    v_xor_b32_e32 v4, v4, v6
 ; CGP-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
-; CGP-NEXT:    v_mul_lo_u32 v8, 0, v0
+; CGP-NEXT:    v_mul_lo_u32 v8, v0, 0
 ; CGP-NEXT:    v_xor_b32_e32 v5, v5, v7
 ; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; CGP-NEXT:    v_mul_lo_u32 v9, 0, v1
+; CGP-NEXT:    v_mul_lo_u32 v9, v1, 0
 ; CGP-NEXT:    v_xor_b32_e32 v2, v2, v6
 ; CGP-NEXT:    v_xor_b32_e32 v3, v3, v7
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v6, v2
-; CGP-NEXT:    v_mul_lo_u32 v7, 0, v2
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, 0, v2
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v10, v3
-; CGP-NEXT:    v_mul_lo_u32 v11, 0, v3
+; CGP-NEXT:    v_sub_i32_e32 v11, vcc, 0, v3
 ; CGP-NEXT:    v_rcp_f32_e32 v6, v6
 ; CGP-NEXT:    v_rcp_f32_e32 v10, v10
-; CGP-NEXT:    v_mul_f32_e32 v6, 0x4f800000, v6
-; CGP-NEXT:    v_mul_f32_e32 v10, 0x4f800000, v10
+; CGP-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; CGP-NEXT:    v_mul_f32_e32 v10, 0x4f7ffffe, v10
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v10, v10
-; CGP-NEXT:    v_mul_lo_u32 v12, v6, v2
-; CGP-NEXT:    v_mul_lo_u32 v13, v6, 0
-; CGP-NEXT:    v_mul_hi_u32 v14, v6, v2
-; CGP-NEXT:    v_mul_lo_u32 v15, v10, v3
-; CGP-NEXT:    v_mul_lo_u32 v16, v10, 0
-; CGP-NEXT:    v_mul_hi_u32 v17, v10, v3
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v13
-; CGP-NEXT:    v_sub_i32_e32 v18, vcc, 0, v12
-; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v16
-; CGP-NEXT:    v_sub_i32_e32 v19, vcc, 0, v15
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v14
-; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v17
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; CGP-NEXT:    v_cndmask_b32_e32 v7, v12, v18, vcc
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v11
-; CGP-NEXT:    v_cndmask_b32_e64 v11, v15, v19, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v12, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v7, v7, v6
-; CGP-NEXT:    v_mul_lo_u32 v14, v11, 0
-; CGP-NEXT:    v_mul_hi_u32 v11, v11, v10
-; CGP-NEXT:    v_add_i32_e64 v12, s[6:7], v13, v12
-; CGP-NEXT:    v_add_i32_e64 v13, s[6:7], v16, v14
-; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v12, v7
-; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v13, v11
-; CGP-NEXT:    v_add_i32_e64 v12, s[6:7], v6, v7
-; CGP-NEXT:    v_sub_i32_e64 v6, s[6:7], v6, v7
-; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v10, v11
-; CGP-NEXT:    v_sub_i32_e64 v10, s[6:7], v10, v11
-; CGP-NEXT:    v_cndmask_b32_e32 v6, v6, v12, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v7, v10, v7, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v10, v6, 0
-; CGP-NEXT:    v_mul_hi_u32 v6, v6, v0
-; CGP-NEXT:    v_mul_lo_u32 v11, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v7, v7, v1
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_mul_lo_u32 v7, v7, v6
+; CGP-NEXT:    v_mul_lo_u32 v12, v6, 0
+; CGP-NEXT:    v_mul_lo_u32 v11, v11, v10
+; CGP-NEXT:    v_mul_lo_u32 v13, v10, 0
+; CGP-NEXT:    v_mul_lo_u32 v14, 0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v6, v7
+; CGP-NEXT:    v_mul_lo_u32 v15, 0, v11
+; CGP-NEXT:    v_mul_hi_u32 v11, v10, v11
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v14, v12
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v15, v13
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v12, v7
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v10, v11
+; CGP-NEXT:    v_mul_lo_u32 v10, 0, v6
+; CGP-NEXT:    v_mul_hi_u32 v6, v0, v6
+; CGP-NEXT:    v_mul_lo_u32 v11, 0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v1, v7
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
 ; CGP-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
 ; CGP-NEXT:    v_add_i32_e32 v7, vcc, v9, v7
 ; CGP-NEXT:    v_mul_lo_u32 v8, v6, v2
 ; CGP-NEXT:    v_add_i32_e32 v9, vcc, 1, v6
-; CGP-NEXT:    v_subrev_i32_e32 v10, vcc, 1, v6
-; CGP-NEXT:    v_mul_lo_u32 v11, v7, v3
-; CGP-NEXT:    v_add_i32_e32 v12, vcc, 1, v7
-; CGP-NEXT:    v_subrev_i32_e32 v13, vcc, 1, v7
-; CGP-NEXT:    v_sub_i32_e32 v14, vcc, v0, v8
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v8
-; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v11
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v11
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v14, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[8:9], v0, v3
-; CGP-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v6, v9, s[6:7]
-; CGP-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v7, v12, s[6:7]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v10, v7, v3
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, 1, v7
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v8
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v10
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v6, v6, v9, vcc
+; CGP-NEXT:    v_sub_i32_e64 v8, s[4:5], v0, v2
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v7, v7, v11, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v9, s[6:7], v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, 1, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, 1, v7
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v6, v8, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v7, v9, vcc
 ; CGP-NEXT:    v_xor_b32_e32 v0, v0, v4
 ; CGP-NEXT:    v_xor_b32_e32 v1, v1, v5
 ; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
@@ -911,28 +797,24 @@ define i32 @v_sdiv_i32_24bit(i32 %num, i32 %den) {
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v3
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, v1
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v1
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT:    v_mul_lo_u32 v5, v4, v1
-; GISEL-NEXT:    v_mul_hi_u32 v6, v4, v1
-; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v5
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
-; GISEL-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v4
-; GISEL-NEXT:    v_add_i32_e64 v6, s[4:5], v4, v5
-; GISEL-NEXT:    v_sub_i32_e64 v4, s[4:5], v4, v5
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v4, v4, v0
+; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v4
+; GISEL-NEXT:    v_mul_hi_u32 v5, v4, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT:    v_mul_hi_u32 v4, v0, v4
 ; GISEL-NEXT:    v_mul_lo_u32 v5, v4, v1
 ; GISEL-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
-; GISEL-NEXT:    v_subrev_i32_e32 v7, vcc, 1, v4
-; GISEL-NEXT:    v_sub_i32_e32 v8, vcc, v0, v5
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v5
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v1
-; GISEL-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v4, v6, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v5, s[4:5], v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, 1, v4
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v5, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v2, v3
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
@@ -945,39 +827,32 @@ define i32 @v_sdiv_i32_24bit(i32 %num, i32 %den) {
 ; CGP-NEXT:    v_and_b32_e32 v0, s4, v0
 ; CGP-NEXT:    v_and_b32_e32 v1, s4, v1
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v2, v1
-; CGP-NEXT:    v_mul_lo_u32 v3, 0, v1
-; CGP-NEXT:    v_mul_lo_u32 v4, 0, v0
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; CGP-NEXT:    v_mul_lo_u32 v4, v0, 0
 ; CGP-NEXT:    v_rcp_f32_e32 v2, v2
-; CGP-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; CGP-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, v2, v1
-; CGP-NEXT:    v_mul_lo_u32 v6, v2, 0
-; CGP-NEXT:    v_mul_hi_u32 v7, v2, v1
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; CGP-NEXT:    v_sub_i32_e32 v8, vcc, 0, v5
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; CGP-NEXT:    v_cndmask_b32_e32 v3, v5, v8, vcc
-; CGP-NEXT:    v_mul_lo_u32 v5, v3, 0
-; CGP-NEXT:    v_mul_hi_u32 v3, v3, v2
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v6, v5
-; CGP-NEXT:    v_add_i32_e64 v3, s[4:5], v5, v3
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v2, v3
-; CGP-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; CGP-NEXT:    v_mul_lo_u32 v3, v2, 0
-; CGP-NEXT:    v_mul_hi_u32 v2, v2, v0
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; CGP-NEXT:    v_mul_lo_u32 v3, v3, v2
+; CGP-NEXT:    v_mul_lo_u32 v5, v2, 0
+; CGP-NEXT:    v_mul_lo_u32 v6, 0, v3
+; CGP-NEXT:    v_mul_hi_u32 v3, v2, v3
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT:    v_mul_lo_u32 v3, 0, v2
+; CGP-NEXT:    v_mul_hi_u32 v2, v0, v2
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
 ; CGP-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; CGP-NEXT:    v_mul_lo_u32 v3, v2, v1
 ; CGP-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; CGP-NEXT:    v_subrev_i32_e32 v5, vcc, 1, v2
-; CGP-NEXT:    v_sub_i32_e32 v6, vcc, v0, v3
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v1
-; CGP-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, 1, v2
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %num.mask = and i32 %num, 16777215
   %den.mask = and i32 %den, 16777215
@@ -1009,51 +884,43 @@ define <2 x i32> @v_sdiv_v2i32_24bit(<2 x i32> %num, <2 x i32> %den) {
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v6
 ; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v7
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v3
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v6, v3
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v3
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; GISEL-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
+; GISEL-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GISEL-NEXT:    v_mul_lo_u32 v6, v4, v2
-; GISEL-NEXT:    v_mul_hi_u32 v7, v4, v2
-; GISEL-NEXT:    v_mul_lo_u32 v10, v5, v3
-; GISEL-NEXT:    v_mul_hi_u32 v11, v5, v3
-; GISEL-NEXT:    v_sub_i32_e32 v12, vcc, 0, v6
-; GISEL-NEXT:    v_sub_i32_e32 v13, vcc, 0, v10
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v6, v6, v12, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v11
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, v10, v13, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v6, v6, v4
-; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v5
-; GISEL-NEXT:    v_add_i32_e64 v10, s[6:7], v4, v6
-; GISEL-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v6
-; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v5, v7
-; GISEL-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v4, v4, v0
-; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v1
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v4
+; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v6
+; GISEL-NEXT:    v_mul_hi_u32 v5, v4, v5
+; GISEL-NEXT:    v_mul_hi_u32 v7, v6, v7
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v6, v7
+; GISEL-NEXT:    v_mul_hi_u32 v4, v0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v5, v1, v5
 ; GISEL-NEXT:    v_mul_lo_u32 v6, v4, v2
 ; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
-; GISEL-NEXT:    v_subrev_i32_e32 v10, vcc, 1, v4
-; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v3
-; GISEL-NEXT:    v_add_i32_e32 v12, vcc, 1, v5
-; GISEL-NEXT:    v_subrev_i32_e32 v13, vcc, 1, v5
-; GISEL-NEXT:    v_sub_i32_e32 v14, vcc, v0, v6
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v6
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v11
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v11
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[6:7], v14, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[8:9], v0, v3
-; GISEL-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v4, v7, s[6:7]
-; GISEL-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v5, v12, s[6:7]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v10, v5, v3
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, 1, v5
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v10
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v6, s[4:5], v0, v2
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v7, s[6:7], v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v5
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v6, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v7, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v8
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v9
 ; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v8
@@ -1069,73 +936,59 @@ define <2 x i32> @v_sdiv_v2i32_24bit(<2 x i32> %num, <2 x i32> %den) {
 ; CGP-NEXT:    v_and_b32_e32 v2, s4, v2
 ; CGP-NEXT:    v_and_b32_e32 v3, s4, v3
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, 0, v2
-; CGP-NEXT:    v_mul_lo_u32 v6, 0, v0
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; CGP-NEXT:    v_mul_lo_u32 v6, v0, 0
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v7, v3
-; CGP-NEXT:    v_mul_lo_u32 v8, 0, v3
-; CGP-NEXT:    v_mul_lo_u32 v9, 0, v1
+; CGP-NEXT:    v_sub_i32_e32 v8, vcc, 0, v3
+; CGP-NEXT:    v_mul_lo_u32 v9, v1, 0
 ; CGP-NEXT:    v_rcp_f32_e32 v4, v4
 ; CGP-NEXT:    v_rcp_f32_e32 v7, v7
-; CGP-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f800000, v7
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; CGP-NEXT:    v_mul_lo_u32 v10, v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v11, v4, 0
-; CGP-NEXT:    v_mul_hi_u32 v12, v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v13, v7, v3
-; CGP-NEXT:    v_mul_lo_u32 v14, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v15, v7, v3
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v11
-; CGP-NEXT:    v_sub_i32_e32 v16, vcc, 0, v10
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v14
-; CGP-NEXT:    v_sub_i32_e32 v17, vcc, 0, v13
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v12
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v15
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; CGP-NEXT:    v_cndmask_b32_e32 v5, v10, v16, vcc
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
-; CGP-NEXT:    v_cndmask_b32_e64 v8, v13, v17, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v10, v5, 0
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v4
-; CGP-NEXT:    v_mul_lo_u32 v12, v8, 0
-; CGP-NEXT:    v_mul_hi_u32 v8, v8, v7
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v11, v10
-; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v14, v12
-; CGP-NEXT:    v_add_i32_e64 v5, s[6:7], v10, v5
-; CGP-NEXT:    v_add_i32_e64 v8, s[6:7], v11, v8
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v4, v5
-; CGP-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v5
-; CGP-NEXT:    v_add_i32_e64 v5, s[6:7], v7, v8
-; CGP-NEXT:    v_sub_i32_e64 v7, s[6:7], v7, v8
-; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v7, v4, 0
-; CGP-NEXT:    v_mul_hi_u32 v4, v4, v0
-; CGP-NEXT:    v_mul_lo_u32 v8, v5, 0
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v1
-; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v9, v8
+; CGP-NEXT:    v_mul_lo_u32 v5, v5, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v4, 0
+; CGP-NEXT:    v_mul_lo_u32 v8, v8, v7
+; CGP-NEXT:    v_mul_lo_u32 v11, v7, 0
+; CGP-NEXT:    v_mul_lo_u32 v12, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v4, v5
+; CGP-NEXT:    v_mul_lo_u32 v13, 0, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v7, v8
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v10, v5
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v7, v8
+; CGP-NEXT:    v_mul_lo_u32 v7, 0, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v0, v4
+; CGP-NEXT:    v_mul_lo_u32 v8, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v1, v5
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v8, v9
 ; CGP-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
 ; CGP-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
 ; CGP-NEXT:    v_mul_lo_u32 v6, v4, v2
 ; CGP-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
-; CGP-NEXT:    v_subrev_i32_e32 v8, vcc, 1, v4
-; CGP-NEXT:    v_mul_lo_u32 v9, v5, v3
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, 1, v5
-; CGP-NEXT:    v_subrev_i32_e32 v11, vcc, 1, v5
-; CGP-NEXT:    v_sub_i32_e32 v12, vcc, v0, v6
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v6
-; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v9
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v9
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v12, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[8:9], v0, v3
-; CGP-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v4, v7, s[6:7]
-; CGP-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v5, v10, s[6:7]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v11, v1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v8, v5, v3
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, 1, v5
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v8
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; CGP-NEXT:    v_sub_i32_e64 v6, s[4:5], v0, v2
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v7, s[6:7], v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, 1, v5
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v4, v6, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v5, v7, vcc
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %num.mask = and <2 x i32> %num, <i32 16777215, i32 16777215>
   %den.mask = and <2 x i32> %den, <i32 16777215, i32 16777215>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
new file mode 100644
index 0000000000000..1813c33019ae8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
@@ -0,0 +1,3751 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=1 -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=CHECK,GISEL %s
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=CHECK,CGP %s
+
+; The same 32-bit expansion is implemented in the legalizer and in AMDGPUCodeGenPrepare.
+
+define i64 @v_sdiv_i64(i64 %num, i64 %den) {
+; CHECK-LABEL: v_sdiv_i64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v5, v1, v3
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0
+; CHECK-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CHECK-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CHECK-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CHECK-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CHECK-NEXT:    s_cbranch_execz BB0_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v2, v4
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v5, v5, v4
+; CHECK-NEXT:    v_xor_b32_e32 v3, v3, v4
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v6, v5
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v7, v3
+; CHECK-NEXT:    v_ashrrev_i32_e32 v8, 31, v1
+; CHECK-NEXT:    v_mac_f32_e32 v6, 0x4f800000, v7
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v0, v8
+; CHECK-NEXT:    v_addc_u32_e32 v1, vcc, v1, v8, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v10, vcc, 0, v5
+; CHECK-NEXT:    v_mul_f32_e32 v6, 0x5f7ffffc, v6
+; CHECK-NEXT:    v_mul_f32_e32 v9, 0x2f800000, v6
+; CHECK-NEXT:    v_trunc_f32_e32 v9, v9
+; CHECK-NEXT:    v_mac_f32_e32 v6, 0xcf800000, v9
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v9, v9
+; CHECK-NEXT:    v_subb_u32_e32 v11, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v7, v7, v8
+; CHECK-NEXT:    v_mul_lo_u32 v12, v11, v6
+; CHECK-NEXT:    v_mul_lo_u32 v13, v10, v9
+; CHECK-NEXT:    v_mul_hi_u32 v15, v10, v6
+; CHECK-NEXT:    v_mul_lo_u32 v14, v10, v6
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v8
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; CHECK-NEXT:    v_mul_lo_u32 v13, v9, v14
+; CHECK-NEXT:    v_mul_lo_u32 v15, v6, v12
+; CHECK-NEXT:    v_mul_hi_u32 v16, v6, v14
+; CHECK-NEXT:    v_mul_hi_u32 v14, v9, v14
+; CHECK-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; CHECK-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
+; CHECK-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v16, v9, v12
+; CHECK-NEXT:    v_add_i32_e32 v13, vcc, v15, v13
+; CHECK-NEXT:    v_mul_hi_u32 v15, v6, v12
+; CHECK-NEXT:    v_mul_hi_u32 v12, v9, v12
+; CHECK-NEXT:    v_add_i32_e32 v14, vcc, v16, v14
+; CHECK-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; CHECK-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v15, vcc, v16, v15
+; CHECK-NEXT:    v_add_i32_e32 v13, vcc, v14, v13
+; CHECK-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v13
+; CHECK-NEXT:    v_addc_u32_e64 v13, s[4:5], v9, v12, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v11, v11, v6
+; CHECK-NEXT:    v_mul_lo_u32 v14, v10, v13
+; CHECK-NEXT:    v_mul_lo_u32 v15, v10, v6
+; CHECK-NEXT:    v_mul_hi_u32 v10, v10, v6
+; CHECK-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; CHECK-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; CHECK-NEXT:    v_mul_hi_u32 v12, v6, v15
+; CHECK-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v10
+; CHECK-NEXT:    v_mul_lo_u32 v11, v13, v15
+; CHECK-NEXT:    v_mul_lo_u32 v14, v6, v10
+; CHECK-NEXT:    v_mul_hi_u32 v15, v13, v15
+; CHECK-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; CHECK-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CHECK-NEXT:    v_mul_lo_u32 v12, v13, v10
+; CHECK-NEXT:    v_add_i32_e64 v11, s[4:5], v14, v11
+; CHECK-NEXT:    v_mul_hi_u32 v14, v6, v10
+; CHECK-NEXT:    v_mul_hi_u32 v10, v13, v10
+; CHECK-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
+; CHECK-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; CHECK-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v14, s[4:5], v15, v14
+; CHECK-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v12, s[4:5], v14, v12
+; CHECK-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v12
+; CHECK-NEXT:    v_addc_u32_e32 v9, vcc, v9, v10, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v11
+; CHECK-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v10, v1, v6
+; CHECK-NEXT:    v_mul_lo_u32 v11, v7, v9
+; CHECK-NEXT:    v_mul_hi_u32 v12, v7, v6
+; CHECK-NEXT:    v_mul_hi_u32 v6, v1, v6
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v12, v1, v9
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT:    v_mul_hi_u32 v11, v7, v9
+; CHECK-NEXT:    v_mul_hi_u32 v9, v1, v9
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v12, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CHECK-NEXT:    v_mul_lo_u32 v10, v3, v6
+; CHECK-NEXT:    v_mul_lo_u32 v11, v5, v9
+; CHECK-NEXT:    v_mul_hi_u32 v13, v5, v6
+; CHECK-NEXT:    v_mul_lo_u32 v12, v5, v6
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v10, v13
+; CHECK-NEXT:    v_sub_i32_e32 v7, vcc, v7, v12
+; CHECK-NEXT:    v_subb_u32_e64 v11, s[4:5], v1, v10, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v10
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v11, v3
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v5
+; CHECK-NEXT:    v_sub_i32_e32 v7, vcc, v7, v5
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], v11, v3
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, 1, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, v10, v12, s[4:5]
+; CHECK-NEXT:    v_addc_u32_e32 v12, vcc, 0, v9, vcc
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v13, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v7, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v13, v5, vcc
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, 1, v11
+; CHECK-NEXT:    v_addc_u32_e32 v5, vcc, 0, v12, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v11, v3, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, v12, v5, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v5, v8, v4
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v5
+; CHECK-NEXT:    v_xor_b32_e32 v3, v3, v5
+; CHECK-NEXT:    v_sub_i32_e32 v4, vcc, v1, v5
+; CHECK-NEXT:    v_subb_u32_e32 v5, vcc, v3, v5, vcc
+; CHECK-NEXT:  BB0_2: ; %Flow
+; CHECK-NEXT:    s_or_saveexec_b64 s[6:7], s[6:7]
+; CHECK-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; CHECK-NEXT:    s_cbranch_execz BB0_4
+; CHECK-NEXT:  ; %bb.3:
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, v2
+; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, 0, v2
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; CHECK-NEXT:    v_mul_lo_u32 v3, v3, v1
+; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CHECK-NEXT:    v_mul_lo_u32 v3, v1, v2
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, 1, v1
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CHECK-NEXT:    v_sub_i32_e64 v3, s[4:5], v0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, v1, v3, vcc
+; CHECK-NEXT:  BB0_4:
+; CHECK-NEXT:    s_or_b64 exec, exec, s[6:7]
+; CHECK-NEXT:    v_mov_b32_e32 v0, v4
+; CHECK-NEXT:    v_mov_b32_e32 v1, v5
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %result = sdiv i64 %num, %den
+  ret i64 %result
+}
+
+; FIXME: This is a workaround for not handling uniform VGPR case.
+declare i32 @llvm.amdgcn.readfirstlane(i32)
+
+define amdgpu_ps i64 @s_sdiv_i64(i64 inreg %num, i64 inreg %den) {
+; CHECK-LABEL: s_sdiv_i64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_or_b64 s[0:1], s[2:3], s[4:5]
+; CHECK-NEXT:    s_mov_b32 s6, 0
+; CHECK-NEXT:    s_mov_b32 s7, -1
+; CHECK-NEXT:    s_and_b64 s[0:1], s[0:1], s[6:7]
+; CHECK-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[0:1], 0
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[8:9], 0, 1
+; CHECK-NEXT:    s_mov_b32 s0, 1
+; CHECK-NEXT:    s_xor_b64 vcc, s[6:7], s[8:9]
+; CHECK-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; CHECK-NEXT:    s_cbranch_vccz BB1_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    s_ashr_i32 s6, s3, 31
+; CHECK-NEXT:    s_ashr_i32 s8, s5, 31
+; CHECK-NEXT:    s_add_u32 s0, s2, s6
+; CHECK-NEXT:    s_cselect_b32 s1, 1, 0
+; CHECK-NEXT:    s_and_b32 s1, s1, 1
+; CHECK-NEXT:    s_cmp_lg_u32 s1, 0
+; CHECK-NEXT:    s_addc_u32 s1, s3, s6
+; CHECK-NEXT:    s_add_u32 s10, s4, s8
+; CHECK-NEXT:    s_cselect_b32 s3, 1, 0
+; CHECK-NEXT:    s_and_b32 s3, s3, 1
+; CHECK-NEXT:    s_cmp_lg_u32 s3, 0
+; CHECK-NEXT:    s_mov_b32 s9, s8
+; CHECK-NEXT:    s_addc_u32 s11, s5, s8
+; CHECK-NEXT:    s_xor_b64 s[10:11], s[10:11], s[8:9]
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v0, s10
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, s11
+; CHECK-NEXT:    s_mov_b32 s7, s6
+; CHECK-NEXT:    s_xor_b64 s[12:13], s[0:1], s[6:7]
+; CHECK-NEXT:    s_sub_u32 s3, 0, s10
+; CHECK-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT:    s_cselect_b32 s0, 1, 0
+; CHECK-NEXT:    s_and_b32 s0, s0, 1
+; CHECK-NEXT:    s_cmp_lg_u32 s0, 0
+; CHECK-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; CHECK-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; CHECK-NEXT:    v_trunc_f32_e32 v1, v1
+; CHECK-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; CHECK-NEXT:    s_subb_u32 s5, 0, s11
+; CHECK-NEXT:    v_mul_lo_u32 v2, s5, v0
+; CHECK-NEXT:    v_mul_lo_u32 v3, s3, v1
+; CHECK-NEXT:    v_mul_hi_u32 v5, s3, v0
+; CHECK-NEXT:    v_mul_lo_u32 v4, s3, v0
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_mul_lo_u32 v3, v1, v4
+; CHECK-NEXT:    v_mul_lo_u32 v5, v0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v6, v0, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v1, v4
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v6, v1, v2
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
+; CHECK-NEXT:    v_mul_hi_u32 v5, v0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT:    v_addc_u32_e64 v3, s[0:1], v1, v2, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v4, s5, v0
+; CHECK-NEXT:    v_mul_lo_u32 v5, s3, v3
+; CHECK-NEXT:    v_mul_hi_u32 v7, s3, v0
+; CHECK-NEXT:    v_mul_lo_u32 v6, s3, v0
+; CHECK-NEXT:    v_add_i32_e64 v1, s[0:1], v1, v2
+; CHECK-NEXT:    v_add_i32_e64 v4, s[0:1], v4, v5
+; CHECK-NEXT:    v_add_i32_e64 v4, s[0:1], v4, v7
+; CHECK-NEXT:    v_mul_lo_u32 v5, v3, v6
+; CHECK-NEXT:    v_mul_lo_u32 v7, v0, v4
+; CHECK-NEXT:    v_mul_hi_u32 v2, v0, v6
+; CHECK-NEXT:    v_mul_hi_u32 v6, v3, v6
+; CHECK-NEXT:    v_add_i32_e64 v5, s[0:1], v5, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[0:1]
+; CHECK-NEXT:    v_add_i32_e64 v2, s[0:1], v5, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; CHECK-NEXT:    v_mul_lo_u32 v5, v3, v4
+; CHECK-NEXT:    v_add_i32_e64 v2, s[0:1], v7, v2
+; CHECK-NEXT:    v_mul_hi_u32 v7, v0, v4
+; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v4
+; CHECK-NEXT:    v_add_i32_e64 v5, s[0:1], v5, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[0:1]
+; CHECK-NEXT:    v_add_i32_e64 v5, s[0:1], v5, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[0:1]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[0:1], v6, v7
+; CHECK-NEXT:    v_add_i32_e64 v2, s[0:1], v5, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s[0:1]
+; CHECK-NEXT:    v_add_i32_e64 v4, s[0:1], v6, v5
+; CHECK-NEXT:    v_add_i32_e64 v3, s[0:1], v3, v4
+; CHECK-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; CHECK-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v2, s13, v0
+; CHECK-NEXT:    v_mul_lo_u32 v3, s12, v1
+; CHECK-NEXT:    v_mul_hi_u32 v4, s12, v0
+; CHECK-NEXT:    v_mul_hi_u32 v0, s13, v0
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v4, s13, v1
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT:    v_mul_hi_u32 v3, s12, v1
+; CHECK-NEXT:    v_mul_hi_u32 v1, s13, v1
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v2, s11, v0
+; CHECK-NEXT:    v_mul_lo_u32 v1, s10, v1
+; CHECK-NEXT:    v_mul_hi_u32 v4, s10, v0
+; CHECK-NEXT:    v_mul_lo_u32 v3, s10, v0
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; CHECK-NEXT:    v_mov_b32_e32 v2, s13
+; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, s12, v3
+; CHECK-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v1, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[0:1], s13, v1
+; CHECK-NEXT:    v_mov_b32_e32 v4, s11
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[0:1], s11, v2
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[0:1], s10, v3
+; CHECK-NEXT:    v_subrev_i32_e32 v3, vcc, s10, v3
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
+; CHECK-NEXT:    v_cmp_eq_u32_e64 s[0:1], s11, v2
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s11, v1
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, v5, v6, s[0:1]
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s10, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, s11, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v5, v3, vcc
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, 1, v4
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CHECK-NEXT:    s_xor_b64 s[0:1], s[6:7], s[8:9]
+; CHECK-NEXT:    v_xor_b32_e32 v0, s0, v0
+; CHECK-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
+; CHECK-NEXT:    s_mov_b32 s0, 0
+; CHECK-NEXT:  BB1_2: ; %Flow
+; CHECK-NEXT:    s_and_b32 s0, s0, 1
+; CHECK-NEXT:    s_cmp_lg_u32 s0, 0
+; CHECK-NEXT:    s_cbranch_scc0 BB1_4
+; CHECK-NEXT:  ; %bb.3:
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v0, s4
+; CHECK-NEXT:    s_sub_i32 s0, 0, s4
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT:    v_mul_lo_u32 v1, s0, v0
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT:    v_mul_hi_u32 v0, s2, v0
+; CHECK-NEXT:    v_mul_lo_u32 v1, v0, s4
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, s2, v1
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s4, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CHECK-NEXT:    v_subrev_i32_e64 v2, s[0:1], s4, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s4, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CHECK-NEXT:  BB1_4:
+; CHECK-NEXT:    v_readfirstlane_b32 s0, v0
+; CHECK-NEXT:    s_mov_b32 s1, s0
+; CHECK-NEXT:    ; return to shader part epilog
+  %result = sdiv i64 %num, %den
+  %cast = bitcast i64 %result to <2 x i32>
+  %elt.0 = extractelement <2 x i32> %cast, i32 0
+  %elt.1 = extractelement <2 x i32> %cast, i32 1
+  %res.0 = call i32 @llvm.amdgcn.readfirstlane(i32 %elt.0)
+  %res.1 = call i32 @llvm.amdgcn.readfirstlane(i32 %elt.1)
+  %ins.0 = insertelement <2 x i32> undef, i32 %res.0, i32 0
+  %ins.1 = insertelement <2 x i32> %ins.0, i32 %res.0, i32 1
+  %cast.back = bitcast <2 x i32> %ins.1 to i64
+  ret i64 %cast.back
+}
+
+define <2 x i64> @v_sdiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
+; GISEL-LABEL: v_sdiv_v2i64:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    v_ashrrev_i32_e32 v8, 31, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v8, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v4, v4, v8
+; GISEL-NEXT:    v_xor_b32_e32 v5, v5, v8
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v9, v4
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v10, v5
+; GISEL-NEXT:    v_ashrrev_i32_e32 v11, 31, v1
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v11
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v11, vcc
+; GISEL-NEXT:    v_mac_f32_e32 v9, 0x4f800000, v10
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v9, v9
+; GISEL-NEXT:    v_sub_i32_e32 v12, vcc, 0, v4
+; GISEL-NEXT:    v_subb_u32_e32 v13, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v11
+; GISEL-NEXT:    v_mul_f32_e32 v9, 0x5f7ffffc, v9
+; GISEL-NEXT:    v_mul_f32_e32 v10, 0x2f800000, v9
+; GISEL-NEXT:    v_trunc_f32_e32 v10, v10
+; GISEL-NEXT:    v_mac_f32_e32 v9, 0xcf800000, v10
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v9, v9
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v10, v10
+; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v11
+; GISEL-NEXT:    v_mul_lo_u32 v14, v13, v9
+; GISEL-NEXT:    v_mul_lo_u32 v15, v12, v10
+; GISEL-NEXT:    v_mul_hi_u32 v17, v12, v9
+; GISEL-NEXT:    v_mul_lo_u32 v16, v12, v9
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; GISEL-NEXT:    v_mul_lo_u32 v15, v10, v16
+; GISEL-NEXT:    v_mul_lo_u32 v17, v9, v14
+; GISEL-NEXT:    v_mul_hi_u32 v18, v9, v16
+; GISEL-NEXT:    v_mul_hi_u32 v16, v10, v16
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v15, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v15, v18
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v18, v10, v14
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v17, v15
+; GISEL-NEXT:    v_mul_hi_u32 v17, v9, v14
+; GISEL-NEXT:    v_mul_hi_u32 v14, v10, v14
+; GISEL-NEXT:    v_add_i32_e32 v16, vcc, v18, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v16, vcc, v16, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v17, vcc, v18, v17
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v16, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v16, vcc, v17, v16
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; GISEL-NEXT:    v_addc_u32_e64 v15, s[4:5], v10, v14, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v13, v13, v9
+; GISEL-NEXT:    v_mul_lo_u32 v16, v12, v15
+; GISEL-NEXT:    v_mul_lo_u32 v17, v12, v9
+; GISEL-NEXT:    v_mul_hi_u32 v12, v12, v9
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v14
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v16
+; GISEL-NEXT:    v_mul_hi_u32 v14, v9, v17
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v13, v12
+; GISEL-NEXT:    v_mul_lo_u32 v13, v15, v17
+; GISEL-NEXT:    v_mul_lo_u32 v16, v9, v12
+; GISEL-NEXT:    v_mul_hi_u32 v17, v15, v17
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v14, v15, v12
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v16, v13
+; GISEL-NEXT:    v_mul_hi_u32 v16, v9, v12
+; GISEL-NEXT:    v_mul_hi_u32 v12, v15, v12
+; GISEL-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v16, s[4:5], v17, v16
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[4:5], v16, v14
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; GISEL-NEXT:    v_addc_u32_e32 v10, vcc, v10, v12, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v13
+; GISEL-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v1, v9
+; GISEL-NEXT:    v_mul_lo_u32 v13, v0, v10
+; GISEL-NEXT:    v_mul_hi_u32 v14, v0, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v1, v9
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v14, v1, v10
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT:    v_mul_hi_u32 v13, v0, v10
+; GISEL-NEXT:    v_mul_hi_u32 v10, v1, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v14, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v14, v13
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT:    v_mul_lo_u32 v12, v5, v9
+; GISEL-NEXT:    v_mul_lo_u32 v13, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v15, v4, v9
+; GISEL-NEXT:    v_mul_lo_u32 v14, v4, v9
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v14
+; GISEL-NEXT:    v_subb_u32_e64 v13, s[4:5], v1, v12, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v12
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v13, v5
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v4
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v13, v5
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, 1, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, v12, v14, s[4:5]
+; GISEL-NEXT:    v_addc_u32_e32 v14, vcc, 0, v10, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v15, v0, vcc
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, 1, v13
+; GISEL-NEXT:    v_addc_u32_e32 v4, vcc, 0, v14, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v13, v1, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v14, v4, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GISEL-NEXT:    v_ashrrev_i32_e32 v5, 31, v7
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v5
+; GISEL-NEXT:    v_addc_u32_e32 v7, vcc, v7, v5, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v7, v7, v5
+; GISEL-NEXT:    v_xor_b32_e32 v6, v6, v5
+; GISEL-NEXT:    v_xor_b32_e32 v4, v11, v8
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v8, v6
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v9, v7
+; GISEL-NEXT:    v_ashrrev_i32_e32 v10, 31, v3
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v10
+; GISEL-NEXT:    v_addc_u32_e32 v3, vcc, v3, v10, vcc
+; GISEL-NEXT:    v_mac_f32_e32 v8, 0x4f800000, v9
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v8, v8
+; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, 0, v6
+; GISEL-NEXT:    v_subb_u32_e32 v12, vcc, 0, v7, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v4
+; GISEL-NEXT:    v_mul_f32_e32 v8, 0x5f7ffffc, v8
+; GISEL-NEXT:    v_mul_f32_e32 v9, 0x2f800000, v8
+; GISEL-NEXT:    v_trunc_f32_e32 v9, v9
+; GISEL-NEXT:    v_mac_f32_e32 v8, 0xcf800000, v9
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v9, v9
+; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v10
+; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v10
+; GISEL-NEXT:    v_mul_lo_u32 v13, v12, v8
+; GISEL-NEXT:    v_mul_lo_u32 v14, v11, v9
+; GISEL-NEXT:    v_mul_hi_u32 v16, v11, v8
+; GISEL-NEXT:    v_mul_lo_u32 v15, v11, v8
+; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v4
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
+; GISEL-NEXT:    v_mul_lo_u32 v14, v9, v15
+; GISEL-NEXT:    v_mul_lo_u32 v16, v8, v13
+; GISEL-NEXT:    v_mul_hi_u32 v17, v8, v15
+; GISEL-NEXT:    v_mul_hi_u32 v15, v9, v15
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v17, v9, v13
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v16, v14
+; GISEL-NEXT:    v_mul_hi_u32 v16, v8, v13
+; GISEL-NEXT:    v_mul_hi_u32 v13, v9, v13
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v17, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v15, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v16, vcc, v17, v16
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v16, v15
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v14
+; GISEL-NEXT:    v_addc_u32_e64 v14, s[4:5], v9, v13, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v12, v8
+; GISEL-NEXT:    v_mul_lo_u32 v15, v11, v14
+; GISEL-NEXT:    v_mul_lo_u32 v16, v11, v8
+; GISEL-NEXT:    v_mul_hi_u32 v11, v11, v8
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v13
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
+; GISEL-NEXT:    v_mul_hi_u32 v13, v8, v16
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_mul_lo_u32 v12, v14, v16
+; GISEL-NEXT:    v_mul_lo_u32 v15, v8, v11
+; GISEL-NEXT:    v_mul_hi_u32 v16, v14, v16
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v13, v14, v11
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v15, v12
+; GISEL-NEXT:    v_mul_hi_u32 v15, v8, v11
+; GISEL-NEXT:    v_mul_hi_u32 v11, v14, v11
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v15, s[4:5], v16, v15
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v13, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v15, v13
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_addc_u32_e32 v9, vcc, v9, v11, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, v3, v8
+; GISEL-NEXT:    v_mul_lo_u32 v12, v2, v9
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; GISEL-NEXT:    v_mul_hi_u32 v4, v2, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, v3, v8
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v11, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v11, v3, v9
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v12, v4
+; GISEL-NEXT:    v_mul_hi_u32 v12, v2, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v3, v9
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_mul_lo_u32 v9, v7, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, v6, v8
+; GISEL-NEXT:    v_mul_hi_u32 v13, v6, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, v6, v4
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v13
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v12
+; GISEL-NEXT:    v_subb_u32_e64 v11, s[4:5], v3, v9, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v9
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v11, v7
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v6
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v6
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v11, v7
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, 1, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, v9, v12, s[4:5]
+; GISEL-NEXT:    v_addc_u32_e32 v12, vcc, 0, v8, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v7
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v13, v2, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, 1, v11
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, 0, v12, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v11, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v12, v6, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v4, v10, v5
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v4
+; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v4
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v4
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_sdiv_v2i64:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    v_mov_b32_e32 v9, v1
+; CGP-NEXT:    v_mov_b32_e32 v8, v0
+; CGP-NEXT:    v_or_b32_e32 v1, v9, v5
+; CGP-NEXT:    v_mov_b32_e32 v0, 0
+; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CGP-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CGP-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; CGP-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CGP-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execz BB2_2
+; CGP-NEXT:  ; %bb.1:
+; CGP-NEXT:    v_ashrrev_i32_e32 v0, 31, v5
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, v4, v0
+; CGP-NEXT:    v_addc_u32_e32 v5, vcc, v5, v0, vcc
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v0
+; CGP-NEXT:    v_xor_b32_e32 v5, v5, v0
+; CGP-NEXT:    v_cvt_f32_u32_e32 v10, v1
+; CGP-NEXT:    v_cvt_f32_u32_e32 v11, v5
+; CGP-NEXT:    v_ashrrev_i32_e32 v12, 31, v9
+; CGP-NEXT:    v_mac_f32_e32 v10, 0x4f800000, v11
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v10, v10
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v8, v12
+; CGP-NEXT:    v_addc_u32_e32 v9, vcc, v9, v12, vcc
+; CGP-NEXT:    v_sub_i32_e32 v14, vcc, 0, v1
+; CGP-NEXT:    v_mul_f32_e32 v10, 0x5f7ffffc, v10
+; CGP-NEXT:    v_mul_f32_e32 v13, 0x2f800000, v10
+; CGP-NEXT:    v_trunc_f32_e32 v13, v13
+; CGP-NEXT:    v_mac_f32_e32 v10, 0xcf800000, v13
+; CGP-NEXT:    v_cvt_u32_f32_e32 v10, v10
+; CGP-NEXT:    v_cvt_u32_f32_e32 v13, v13
+; CGP-NEXT:    v_subb_u32_e32 v15, vcc, 0, v5, vcc
+; CGP-NEXT:    v_xor_b32_e32 v11, v11, v12
+; CGP-NEXT:    v_mul_lo_u32 v16, v15, v10
+; CGP-NEXT:    v_mul_lo_u32 v17, v14, v13
+; CGP-NEXT:    v_mul_hi_u32 v19, v14, v10
+; CGP-NEXT:    v_mul_lo_u32 v18, v14, v10
+; CGP-NEXT:    v_xor_b32_e32 v9, v9, v12
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v16, v17
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v16, v19
+; CGP-NEXT:    v_mul_lo_u32 v17, v13, v18
+; CGP-NEXT:    v_mul_lo_u32 v19, v10, v16
+; CGP-NEXT:    v_mul_hi_u32 v20, v10, v18
+; CGP-NEXT:    v_mul_hi_u32 v18, v13, v18
+; CGP-NEXT:    v_add_i32_e32 v17, vcc, v17, v19
+; CGP-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v17, vcc, v17, v20
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v20, v13, v16
+; CGP-NEXT:    v_add_i32_e32 v17, vcc, v19, v17
+; CGP-NEXT:    v_mul_hi_u32 v19, v10, v16
+; CGP-NEXT:    v_mul_hi_u32 v16, v13, v16
+; CGP-NEXT:    v_add_i32_e32 v18, vcc, v20, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v18, vcc, v18, v19
+; CGP-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v19, vcc, v20, v19
+; CGP-NEXT:    v_add_i32_e32 v17, vcc, v18, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v18, vcc, v19, v18
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v16, v18
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v17
+; CGP-NEXT:    v_addc_u32_e64 v17, s[4:5], v13, v16, vcc
+; CGP-NEXT:    v_mul_lo_u32 v15, v15, v10
+; CGP-NEXT:    v_mul_lo_u32 v18, v14, v17
+; CGP-NEXT:    v_mul_lo_u32 v19, v14, v10
+; CGP-NEXT:    v_mul_hi_u32 v14, v14, v10
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v16
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v15, v18
+; CGP-NEXT:    v_mul_hi_u32 v16, v10, v19
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v15, v14
+; CGP-NEXT:    v_mul_lo_u32 v15, v17, v19
+; CGP-NEXT:    v_mul_lo_u32 v18, v10, v14
+; CGP-NEXT:    v_mul_hi_u32 v19, v17, v19
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v15, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v15, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v16, v17, v14
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v18, v15
+; CGP-NEXT:    v_mul_hi_u32 v18, v10, v14
+; CGP-NEXT:    v_mul_hi_u32 v14, v17, v14
+; CGP-NEXT:    v_add_i32_e64 v16, s[4:5], v16, v19
+; CGP-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v16, s[4:5], v16, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v18, s[4:5], v19, v18
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v16, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v16, s[4:5], v18, v16
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v16
+; CGP-NEXT:    v_addc_u32_e32 v13, vcc, v13, v14, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v15
+; CGP-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; CGP-NEXT:    v_mul_lo_u32 v14, v9, v10
+; CGP-NEXT:    v_mul_lo_u32 v15, v11, v13
+; CGP-NEXT:    v_mul_hi_u32 v16, v11, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v9, v10
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v16, v9, v13
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT:    v_mul_hi_u32 v15, v11, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v9, v13
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v16, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v16, v15
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; CGP-NEXT:    v_mul_lo_u32 v14, v5, v10
+; CGP-NEXT:    v_mul_lo_u32 v15, v1, v13
+; CGP-NEXT:    v_mul_hi_u32 v17, v1, v10
+; CGP-NEXT:    v_mul_lo_u32 v16, v1, v10
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; CGP-NEXT:    v_sub_i32_e32 v11, vcc, v11, v16
+; CGP-NEXT:    v_subb_u32_e64 v15, s[4:5], v9, v14, vcc
+; CGP-NEXT:    v_sub_i32_e64 v9, s[4:5], v9, v14
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v15, v5
+; CGP-NEXT:    v_subb_u32_e32 v9, vcc, v9, v5, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v11, v1
+; CGP-NEXT:    v_sub_i32_e32 v11, vcc, v11, v1
+; CGP-NEXT:    v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], v15, v5
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, 1, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v14, v14, v16, s[4:5]
+; CGP-NEXT:    v_addc_u32_e32 v16, vcc, 0, v13, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v9, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, -1, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v11, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v9, v5
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v15
+; CGP-NEXT:    v_addc_u32_e32 v9, vcc, 0, v16, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v15, v5, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v5, v16, v9, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
+; CGP-NEXT:    v_xor_b32_e32 v9, v12, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc
+; CGP-NEXT:    v_xor_b32_e32 v0, v1, v9
+; CGP-NEXT:    v_xor_b32_e32 v1, v5, v9
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v9
+; CGP-NEXT:    v_subb_u32_e32 v1, vcc, v1, v9, vcc
+; CGP-NEXT:  BB2_2: ; %Flow2
+; CGP-NEXT:    s_or_saveexec_b64 s[6:7], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execz BB2_4
+; CGP-NEXT:  ; %bb.3:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v4
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, 0, v4
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_lo_u32 v1, v1, v0
+; CGP-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; CGP-NEXT:    v_mul_hi_u32 v0, v8, v0
+; CGP-NEXT:    v_mul_lo_u32 v1, v0, v4
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v0
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v8, v1
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; CGP-NEXT:    v_sub_i32_e64 v5, s[4:5], v1, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v0
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; CGP-NEXT:    v_mov_b32_e32 v1, 0
+; CGP-NEXT:  BB2_4:
+; CGP-NEXT:    s_or_b64 exec, exec, s[6:7]
+; CGP-NEXT:    v_or_b32_e32 v5, v3, v7
+; CGP-NEXT:    v_mov_b32_e32 v4, 0
+; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CGP-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CGP-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; CGP-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CGP-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execz BB2_6
+; CGP-NEXT:  ; %bb.5:
+; CGP-NEXT:    v_ashrrev_i32_e32 v4, 31, v7
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v6, v4
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, v7, v4, vcc
+; CGP-NEXT:    v_xor_b32_e32 v5, v5, v4
+; CGP-NEXT:    v_xor_b32_e32 v7, v7, v4
+; CGP-NEXT:    v_cvt_f32_u32_e32 v8, v5
+; CGP-NEXT:    v_cvt_f32_u32_e32 v9, v7
+; CGP-NEXT:    v_ashrrev_i32_e32 v10, 31, v3
+; CGP-NEXT:    v_mac_f32_e32 v8, 0x4f800000, v9
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v8, v8
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v2, v10
+; CGP-NEXT:    v_addc_u32_e32 v3, vcc, v3, v10, vcc
+; CGP-NEXT:    v_sub_i32_e32 v12, vcc, 0, v5
+; CGP-NEXT:    v_mul_f32_e32 v8, 0x5f7ffffc, v8
+; CGP-NEXT:    v_mul_f32_e32 v11, 0x2f800000, v8
+; CGP-NEXT:    v_trunc_f32_e32 v11, v11
+; CGP-NEXT:    v_mac_f32_e32 v8, 0xcf800000, v11
+; CGP-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; CGP-NEXT:    v_cvt_u32_f32_e32 v11, v11
+; CGP-NEXT:    v_subb_u32_e32 v13, vcc, 0, v7, vcc
+; CGP-NEXT:    v_xor_b32_e32 v9, v9, v10
+; CGP-NEXT:    v_mul_lo_u32 v14, v13, v8
+; CGP-NEXT:    v_mul_lo_u32 v15, v12, v11
+; CGP-NEXT:    v_mul_hi_u32 v17, v12, v8
+; CGP-NEXT:    v_mul_lo_u32 v16, v12, v8
+; CGP-NEXT:    v_xor_b32_e32 v3, v3, v10
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; CGP-NEXT:    v_mul_lo_u32 v15, v11, v16
+; CGP-NEXT:    v_mul_lo_u32 v17, v8, v14
+; CGP-NEXT:    v_mul_hi_u32 v18, v8, v16
+; CGP-NEXT:    v_mul_hi_u32 v16, v11, v16
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v15, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v15, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v18, v11, v14
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v17, v15
+; CGP-NEXT:    v_mul_hi_u32 v17, v8, v14
+; CGP-NEXT:    v_mul_hi_u32 v14, v11, v14
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v18, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v16, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v17, vcc, v18, v17
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v16, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v17, v16
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v15
+; CGP-NEXT:    v_addc_u32_e64 v15, s[4:5], v11, v14, vcc
+; CGP-NEXT:    v_mul_lo_u32 v13, v13, v8
+; CGP-NEXT:    v_mul_lo_u32 v16, v12, v15
+; CGP-NEXT:    v_mul_lo_u32 v17, v12, v8
+; CGP-NEXT:    v_mul_hi_u32 v12, v12, v8
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v16
+; CGP-NEXT:    v_mul_hi_u32 v14, v8, v17
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v13, v12
+; CGP-NEXT:    v_mul_lo_u32 v13, v15, v17
+; CGP-NEXT:    v_mul_lo_u32 v16, v8, v12
+; CGP-NEXT:    v_mul_hi_u32 v17, v15, v17
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v14, v15, v12
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v16, v13
+; CGP-NEXT:    v_mul_hi_u32 v16, v8, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v15, v12
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v16, s[4:5], v17, v16
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v16, v14
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; CGP-NEXT:    v_addc_u32_e32 v11, vcc, v11, v12, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v13
+; CGP-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; CGP-NEXT:    v_mul_lo_u32 v12, v3, v8
+; CGP-NEXT:    v_mul_lo_u32 v13, v9, v11
+; CGP-NEXT:    v_mul_hi_u32 v14, v9, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v3, v8
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v14, v3, v11
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_mul_hi_u32 v13, v9, v11
+; CGP-NEXT:    v_mul_hi_u32 v11, v3, v11
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v14, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v14, v13
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT:    v_mul_lo_u32 v12, v7, v8
+; CGP-NEXT:    v_mul_lo_u32 v13, v5, v11
+; CGP-NEXT:    v_mul_hi_u32 v15, v5, v8
+; CGP-NEXT:    v_mul_lo_u32 v14, v5, v8
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; CGP-NEXT:    v_sub_i32_e32 v9, vcc, v9, v14
+; CGP-NEXT:    v_subb_u32_e64 v13, s[4:5], v3, v12, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v12
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v13, v7
+; CGP-NEXT:    v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v5
+; CGP-NEXT:    v_sub_i32_e32 v9, vcc, v9, v5
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], v13, v7
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, 1, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v12, v12, v14, s[4:5]
+; CGP-NEXT:    v_addc_u32_e32 v14, vcc, 0, v11, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, -1, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v9, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v15, v5, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v13
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, 0, v14, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v13, v5, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v5, v14, v7, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
+; CGP-NEXT:    v_xor_b32_e32 v7, v10, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v5, v11, v5, vcc
+; CGP-NEXT:    v_xor_b32_e32 v3, v3, v7
+; CGP-NEXT:    v_xor_b32_e32 v5, v5, v7
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v3, v7
+; CGP-NEXT:    v_subb_u32_e32 v5, vcc, v5, v7, vcc
+; CGP-NEXT:  BB2_6: ; %Flow
+; CGP-NEXT:    s_or_saveexec_b64 s[6:7], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execz BB2_8
+; CGP-NEXT:  ; %bb.7:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v3, v6
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, 0, v6
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; CGP-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, v3
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v3, v2, v3
+; CGP-NEXT:    v_mul_lo_u32 v4, v3, v6
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v4
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v6
+; CGP-NEXT:    v_sub_i32_e64 v4, s[4:5], v2, v6
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, 1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v6
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v3, v4, vcc
+; CGP-NEXT:    v_mov_b32_e32 v5, 0
+; CGP-NEXT:  BB2_8:
+; CGP-NEXT:    s_or_b64 exec, exec, s[6:7]
+; CGP-NEXT:    v_mov_b32_e32 v2, v4
+; CGP-NEXT:    v_mov_b32_e32 v3, v5
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %result = sdiv <2 x i64> %num, %den
+  ret <2 x i64> %result
+}
+
+define i64 @v_sdiv_i64_pow2k_denom(i64 %num) {
+; CHECK-LABEL: v_sdiv_i64_pow2k_denom:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_movk_i32 s6, 0x1000
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, s6
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, 0
+; CHECK-NEXT:    s_mov_b32 s7, 0xfffff000
+; CHECK-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; CHECK-NEXT:    v_mac_f32_e32 v2, 0x4f800000, v3
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; CHECK-NEXT:    v_addc_u32_e32 v1, vcc, v1, v4, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v4
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v4
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; CHECK-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
+; CHECK-NEXT:    v_trunc_f32_e32 v3, v3
+; CHECK-NEXT:    v_mac_f32_e32 v2, 0xcf800000, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CHECK-NEXT:    v_mul_lo_u32 v5, -1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, s7, v3
+; CHECK-NEXT:    v_mul_hi_u32 v8, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v7, s7, v2
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; CHECK-NEXT:    v_mul_lo_u32 v6, v3, v7
+; CHECK-NEXT:    v_mul_lo_u32 v8, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v9, v2, v7
+; CHECK-NEXT:    v_mul_hi_u32 v7, v3, v7
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v9, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CHECK-NEXT:    v_mul_hi_u32 v8, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v5, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v9, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT:    v_addc_u32_e64 v6, s[4:5], v3, v5, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v7, -1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v8, s7, v6
+; CHECK-NEXT:    v_mul_hi_u32 v10, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v9, s7, v2
+; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v3, v5
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v8
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v10
+; CHECK-NEXT:    v_mul_lo_u32 v8, v6, v9
+; CHECK-NEXT:    v_mul_lo_u32 v10, v2, v7
+; CHECK-NEXT:    v_mul_hi_u32 v5, v2, v9
+; CHECK-NEXT:    v_mul_hi_u32 v9, v6, v9
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v8, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s[4:5]
+; CHECK-NEXT:    v_mul_lo_u32 v8, v6, v7
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v10, v5
+; CHECK-NEXT:    v_mul_hi_u32 v10, v2, v7
+; CHECK-NEXT:    v_mul_hi_u32 v6, v6, v7
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v10
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v8, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v9, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v7
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, v3, v6, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v5, v1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, v0, v3
+; CHECK-NEXT:    v_mul_hi_u32 v7, v0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v7, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT:    v_mul_hi_u32 v6, v0, v3
+; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; CHECK-NEXT:    v_mul_lo_u32 v5, 0, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, s6, v3
+; CHECK-NEXT:    v_mul_hi_u32 v8, s6, v2
+; CHECK-NEXT:    v_mul_lo_u32 v7, s6, v2
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v7
+; CHECK-NEXT:    v_subb_u32_e64 v6, s[4:5], v1, v5, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v5
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], 0, v6
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v0
+; CHECK-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v0
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v6
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, 1, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, v5, v7, s[4:5]
+; CHECK-NEXT:    v_addc_u32_e32 v7, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, 1, v6
+; CHECK-NEXT:    v_addc_u32_e32 v8, vcc, 0, v7, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v6, v1, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v7, v8, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v4
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v4
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %result = sdiv i64 %num, 4096
+  ret i64 %result
+}
+
+define <2 x i64> @v_sdiv_v2i64_pow2k_denom(<2 x i64> %num) {
+; GISEL-LABEL: v_sdiv_v2i64_pow2k_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_movk_i32 s10, 0x1000
+; GISEL-NEXT:    s_add_u32 s4, s10, 0
+; GISEL-NEXT:    s_cselect_b32 s5, 1, 0
+; GISEL-NEXT:    s_and_b32 s5, s5, 1
+; GISEL-NEXT:    s_cmp_lg_u32 s5, 0
+; GISEL-NEXT:    s_mov_b32 s6, 0
+; GISEL-NEXT:    s_mov_b32 s7, s6
+; GISEL-NEXT:    s_addc_u32 s5, 0, 0
+; GISEL-NEXT:    s_xor_b64 s[8:9], s[4:5], s[6:7]
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s8
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, s9
+; GISEL-NEXT:    s_sub_u32 s11, 0, s8
+; GISEL-NEXT:    s_cselect_b32 s4, 1, 0
+; GISEL-NEXT:    s_and_b32 s4, s4, 1
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
+; GISEL-NEXT:    s_subb_u32 s12, 0, s9
+; GISEL-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; GISEL-NEXT:    v_trunc_f32_e32 v5, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v7, s12, v4
+; GISEL-NEXT:    v_mul_lo_u32 v8, s11, v5
+; GISEL-NEXT:    v_mul_hi_u32 v10, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, s11, v4
+; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v6
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; GISEL-NEXT:    v_mul_lo_u32 v8, v5, v9
+; GISEL-NEXT:    v_mul_lo_u32 v10, v4, v7
+; GISEL-NEXT:    v_mul_hi_u32 v11, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v9
+; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v6
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v7
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; GISEL-NEXT:    v_mul_hi_u32 v10, v4, v7
+; GISEL-NEXT:    v_mul_hi_u32 v7, v5, v7
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_addc_u32_e64 v8, s[4:5], v5, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, s12, v4
+; GISEL-NEXT:    v_mul_lo_u32 v10, s11, v8
+; GISEL-NEXT:    v_mul_hi_u32 v12, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, s11, v4
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v7
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v10
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_mul_lo_u32 v10, v8, v11
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v7, v4, v11
+; GISEL-NEXT:    v_mul_hi_u32 v11, v8, v11
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v10, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v10, v8, v9
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v12, v7
+; GISEL-NEXT:    v_mul_hi_u32 v12, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v8, v8, v9
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v10, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v11, v10
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v9
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v8, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v7, v1, v4
+; GISEL-NEXT:    v_mul_lo_u32 v8, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v9, v0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, v1, v5
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; GISEL-NEXT:    v_mul_hi_u32 v8, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v9, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT:    v_mul_lo_u32 v7, s9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v8, s8, v5
+; GISEL-NEXT:    v_mul_hi_u32 v10, s8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, s8, v4
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT:    v_subb_u32_e64 v8, s[4:5], v1, v7, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v7
+; GISEL-NEXT:    v_mov_b32_e32 v7, s9
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s9, v8
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v7, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s8, v0
+; GISEL-NEXT:    v_subrev_i32_e32 v0, vcc, s8, v0
+; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], s9, v8
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, v9, v10, s[4:5]
+; GISEL-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s9, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
+; GISEL-NEXT:    s_add_u32 s4, s10, 0
+; GISEL-NEXT:    s_cselect_b32 s5, 1, 0
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, s9, v1
+; GISEL-NEXT:    s_and_b32 s5, s5, 1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, 1, v7
+; GISEL-NEXT:    s_cmp_lg_u32 s5, 0
+; GISEL-NEXT:    v_addc_u32_e32 v10, vcc, 0, v9, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GISEL-NEXT:    s_addc_u32 s5, 0, 0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v7, v1, vcc
+; GISEL-NEXT:    s_xor_b64 s[6:7], s[4:5], s[6:7]
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v9, v10, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s6
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, s7
+; GISEL-NEXT:    s_sub_u32 s8, 0, s6
+; GISEL-NEXT:    s_cselect_b32 s4, 1, 0
+; GISEL-NEXT:    s_and_b32 s4, s4, 1
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
+; GISEL-NEXT:    s_subb_u32 s9, 0, s7
+; GISEL-NEXT:    v_ashrrev_i32_e32 v7, 31, v3
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; GISEL-NEXT:    v_trunc_f32_e32 v5, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v7
+; GISEL-NEXT:    v_addc_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v8, s9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, s8, v5
+; GISEL-NEXT:    v_mul_hi_u32 v11, s8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v10, s8, v4
+; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v6
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; GISEL-NEXT:    v_mul_lo_u32 v9, v5, v10
+; GISEL-NEXT:    v_mul_lo_u32 v11, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v12, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v10, v5, v10
+; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v7
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v5, v8
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_mul_hi_u32 v11, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, v5, v8
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; GISEL-NEXT:    v_addc_u32_e64 v9, s[4:5], v5, v8, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v10, s9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, s8, v9
+; GISEL-NEXT:    v_mul_hi_u32 v13, s8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, s8, v4
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v8
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; GISEL-NEXT:    v_mul_lo_u32 v11, v9, v12
+; GISEL-NEXT:    v_mul_lo_u32 v13, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v8, v4, v12
+; GISEL-NEXT:    v_mul_hi_u32 v12, v9, v12
+; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v7
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v11, v9, v10
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v13, v8
+; GISEL-NEXT:    v_mul_hi_u32 v13, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v9, v9, v10
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v12, v11
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v10
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v9, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v6
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v2, v5
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT:    v_mul_hi_u32 v6, v2, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v5
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
+; GISEL-NEXT:    v_mul_hi_u32 v9, v2, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v3, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; GISEL-NEXT:    v_mul_lo_u32 v6, s7, v4
+; GISEL-NEXT:    v_mul_lo_u32 v8, s6, v5
+; GISEL-NEXT:    v_mul_hi_u32 v10, s6, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, s6, v4
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v10
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v9
+; GISEL-NEXT:    v_subb_u32_e64 v8, s[4:5], v3, v6, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v6
+; GISEL-NEXT:    v_mov_b32_e32 v6, s7
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s7, v8
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v6, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v2
+; GISEL-NEXT:    v_subrev_i32_e32 v2, vcc, s6, v2
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], s7, v8
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, v9, v10, s[4:5]
+; GISEL-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s7, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s6, v2
+; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, s7, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, 1, v6
+; GISEL-NEXT:    v_addc_u32_e32 v10, vcc, 0, v9, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v9, v10, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v7
+; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v7
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v7
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_sdiv_v2i64_pow2k_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_movk_i32 s6, 0x1000
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, s6
+; CGP-NEXT:    v_cvt_f32_u32_e32 v5, 0
+; CGP-NEXT:    s_mov_b32 s7, 0xfffff000
+; CGP-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
+; CGP-NEXT:    v_mov_b32_e32 v7, v4
+; CGP-NEXT:    v_mac_f32_e32 v7, 0x4f800000, v5
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v7, v7
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_addc_u32_e32 v1, vcc, v1, v6, vcc
+; CGP-NEXT:    v_xor_b32_e32 v0, v0, v6
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x5f7ffffc, v7
+; CGP-NEXT:    v_mul_f32_e32 v8, 0x2f800000, v7
+; CGP-NEXT:    v_trunc_f32_e32 v8, v8
+; CGP-NEXT:    v_mac_f32_e32 v7, 0xcf800000, v8
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v6
+; CGP-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; CGP-NEXT:    v_mul_lo_u32 v9, -1, v7
+; CGP-NEXT:    v_mul_lo_u32 v10, s7, v8
+; CGP-NEXT:    v_mul_hi_u32 v12, s7, v7
+; CGP-NEXT:    v_mul_lo_u32 v11, s7, v7
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; CGP-NEXT:    v_mul_lo_u32 v10, v8, v11
+; CGP-NEXT:    v_mul_lo_u32 v12, v7, v9
+; CGP-NEXT:    v_mul_hi_u32 v13, v7, v11
+; CGP-NEXT:    v_mul_hi_u32 v11, v8, v11
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v13, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_mul_hi_u32 v12, v7, v9
+; CGP-NEXT:    v_mul_hi_u32 v9, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; CGP-NEXT:    v_addc_u32_e64 v10, s[4:5], v8, v9, vcc
+; CGP-NEXT:    v_mul_lo_u32 v11, -1, v7
+; CGP-NEXT:    v_mul_lo_u32 v12, s7, v10
+; CGP-NEXT:    v_mul_hi_u32 v14, s7, v7
+; CGP-NEXT:    v_mul_lo_u32 v13, s7, v7
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v9
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; CGP-NEXT:    v_mul_lo_u32 v12, v10, v13
+; CGP-NEXT:    v_mul_lo_u32 v14, v7, v11
+; CGP-NEXT:    v_mul_hi_u32 v9, v7, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v10, v13
+; CGP-NEXT:    v_ashrrev_i32_e32 v5, 31, v3
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v12, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v12, v10, v11
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v14, v9
+; CGP-NEXT:    v_mul_hi_u32 v14, v7, v11
+; CGP-NEXT:    v_mul_hi_u32 v10, v10, v11
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v14
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v12, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v13, v12
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; CGP-NEXT:    v_addc_u32_e32 v8, vcc, v8, v10, vcc
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; CGP-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; CGP-NEXT:    v_mul_lo_u32 v9, v1, v7
+; CGP-NEXT:    v_mul_lo_u32 v10, v0, v8
+; CGP-NEXT:    v_mul_hi_u32 v11, v0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v1, v7
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v11, v1, v8
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT:    v_mul_hi_u32 v10, v0, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v1, v8
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v11, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_mul_lo_u32 v9, 0, v7
+; CGP-NEXT:    v_mul_lo_u32 v10, s6, v8
+; CGP-NEXT:    v_mul_hi_u32 v12, s6, v7
+; CGP-NEXT:    v_mul_lo_u32 v11, s6, v7
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v11
+; CGP-NEXT:    v_subb_u32_e64 v10, s[4:5], v1, v9, vcc
+; CGP-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v9
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], 0, v10
+; CGP-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v0
+; CGP-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v0
+; CGP-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v10
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, 1, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v9, v9, v11, s[4:5]
+; CGP-NEXT:    v_addc_u32_e32 v11, vcc, 0, v8, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, 1, v10
+; CGP-NEXT:    v_addc_u32_e32 v12, vcc, 0, v11, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v10, v1, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v11, v12, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x2f800000, v4
+; CGP-NEXT:    v_trunc_f32_e32 v7, v7
+; CGP-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT:    v_mul_lo_u32 v8, -1, v4
+; CGP-NEXT:    v_mul_lo_u32 v9, s7, v7
+; CGP-NEXT:    v_mul_hi_u32 v11, s7, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, s7, v4
+; CGP-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; CGP-NEXT:    v_mul_lo_u32 v9, v7, v10
+; CGP-NEXT:    v_mul_lo_u32 v11, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v12, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v7, v10
+; CGP-NEXT:    v_xor_b32_e32 v0, v0, v6
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v12, v7, v8
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT:    v_mul_hi_u32 v11, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v7, v8
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; CGP-NEXT:    v_addc_u32_e64 v9, s[4:5], v7, v8, vcc
+; CGP-NEXT:    v_mul_lo_u32 v10, -1, v4
+; CGP-NEXT:    v_mul_lo_u32 v11, s7, v9
+; CGP-NEXT:    v_mul_hi_u32 v13, s7, v4
+; CGP-NEXT:    v_mul_lo_u32 v12, s7, v4
+; CGP-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v8
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; CGP-NEXT:    v_mul_lo_u32 v11, v9, v12
+; CGP-NEXT:    v_mul_lo_u32 v13, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v8, v4, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v9, v12
+; CGP-NEXT:    v_xor_b32_e32 v2, v2, v5
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v11, v9, v10
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v13, v8
+; CGP-NEXT:    v_mul_hi_u32 v13, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v9, v9, v10
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v12, v11
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v10
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, v7, v9, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT:    v_xor_b32_e32 v3, v3, v5
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v6
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_mul_lo_u32 v8, v3, v4
+; CGP-NEXT:    v_mul_lo_u32 v9, v2, v7
+; CGP-NEXT:    v_subb_u32_e32 v1, vcc, v1, v6, vcc
+; CGP-NEXT:    v_mul_hi_u32 v6, v2, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v8, v3, v7
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
+; CGP-NEXT:    v_mul_hi_u32 v9, v2, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v3, v7
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_mul_lo_u32 v7, 0, v4
+; CGP-NEXT:    v_mul_lo_u32 v8, s6, v6
+; CGP-NEXT:    v_mul_hi_u32 v10, s6, v4
+; CGP-NEXT:    v_mul_lo_u32 v9, s6, v4
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v9
+; CGP-NEXT:    v_subb_u32_e64 v8, s[4:5], v3, v7, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v7
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], 0, v8
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v2
+; CGP-NEXT:    v_subrev_i32_e32 v2, vcc, s6, v2
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v7, v7, v9, s[4:5]
+; CGP-NEXT:    v_addc_u32_e32 v9, vcc, 0, v6, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s6, v2
+; CGP-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, 1, v8
+; CGP-NEXT:    v_addc_u32_e32 v10, vcc, 0, v9, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v8, v3, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v9, v10, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; CGP-NEXT:    v_xor_b32_e32 v2, v2, v5
+; CGP-NEXT:    v_xor_b32_e32 v3, v3, v5
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT:    v_subb_u32_e32 v3, vcc, v3, v5, vcc
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %result = sdiv <2 x i64> %num, <i64 4096, i64 4096>
+  ret <2 x i64> %result
+}
+
+define i64 @v_sdiv_i64_oddk_denom(i64 %num) {
+; CHECK-LABEL: v_sdiv_i64_oddk_denom:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s6, 0x12d8fb
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, s6
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, 0
+; CHECK-NEXT:    s_mov_b32 s7, 0xffed2705
+; CHECK-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; CHECK-NEXT:    v_mac_f32_e32 v2, 0x4f800000, v3
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; CHECK-NEXT:    v_addc_u32_e32 v1, vcc, v1, v4, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v4
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v4
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; CHECK-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
+; CHECK-NEXT:    v_trunc_f32_e32 v3, v3
+; CHECK-NEXT:    v_mac_f32_e32 v2, 0xcf800000, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CHECK-NEXT:    v_mul_lo_u32 v5, -1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, s7, v3
+; CHECK-NEXT:    v_mul_hi_u32 v8, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v7, s7, v2
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; CHECK-NEXT:    v_mul_lo_u32 v6, v3, v7
+; CHECK-NEXT:    v_mul_lo_u32 v8, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v9, v2, v7
+; CHECK-NEXT:    v_mul_hi_u32 v7, v3, v7
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v9, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CHECK-NEXT:    v_mul_hi_u32 v8, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v5, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v9, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT:    v_addc_u32_e64 v6, s[4:5], v3, v5, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v7, -1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v8, s7, v6
+; CHECK-NEXT:    v_mul_hi_u32 v10, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v9, s7, v2
+; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v3, v5
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v8
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v10
+; CHECK-NEXT:    v_mul_lo_u32 v8, v6, v9
+; CHECK-NEXT:    v_mul_lo_u32 v10, v2, v7
+; CHECK-NEXT:    v_mul_hi_u32 v5, v2, v9
+; CHECK-NEXT:    v_mul_hi_u32 v9, v6, v9
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v8, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s[4:5]
+; CHECK-NEXT:    v_mul_lo_u32 v8, v6, v7
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v10, v5
+; CHECK-NEXT:    v_mul_hi_u32 v10, v2, v7
+; CHECK-NEXT:    v_mul_hi_u32 v6, v6, v7
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v10
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v8, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v9, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v7
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, v3, v6, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v5, v1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, v0, v3
+; CHECK-NEXT:    v_mul_hi_u32 v7, v0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v7, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT:    v_mul_hi_u32 v6, v0, v3
+; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; CHECK-NEXT:    v_mul_lo_u32 v5, 0, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, s6, v3
+; CHECK-NEXT:    v_mul_hi_u32 v8, s6, v2
+; CHECK-NEXT:    v_mul_lo_u32 v7, s6, v2
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v7
+; CHECK-NEXT:    v_subb_u32_e64 v6, s[4:5], v1, v5, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v5
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], 0, v6
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v0
+; CHECK-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v0
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v6
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, 1, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, v5, v7, s[4:5]
+; CHECK-NEXT:    v_addc_u32_e32 v7, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, 1, v6
+; CHECK-NEXT:    v_addc_u32_e32 v8, vcc, 0, v7, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v6, v1, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v7, v8, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v4
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v4
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %result = sdiv i64 %num, 1235195
+  ret i64 %result
+}
+
+define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
+; GISEL-LABEL: v_sdiv_v2i64_oddk_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_mov_b32 s10, 0x12d8fb
+; GISEL-NEXT:    s_add_u32 s4, s10, 0
+; GISEL-NEXT:    s_cselect_b32 s5, 1, 0
+; GISEL-NEXT:    s_and_b32 s5, s5, 1
+; GISEL-NEXT:    s_cmp_lg_u32 s5, 0
+; GISEL-NEXT:    s_mov_b32 s6, 0
+; GISEL-NEXT:    s_mov_b32 s7, s6
+; GISEL-NEXT:    s_addc_u32 s5, 0, 0
+; GISEL-NEXT:    s_xor_b64 s[8:9], s[4:5], s[6:7]
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s8
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, s9
+; GISEL-NEXT:    s_sub_u32 s11, 0, s8
+; GISEL-NEXT:    s_cselect_b32 s4, 1, 0
+; GISEL-NEXT:    s_and_b32 s4, s4, 1
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
+; GISEL-NEXT:    s_subb_u32 s12, 0, s9
+; GISEL-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; GISEL-NEXT:    v_trunc_f32_e32 v5, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v7, s12, v4
+; GISEL-NEXT:    v_mul_lo_u32 v8, s11, v5
+; GISEL-NEXT:    v_mul_hi_u32 v10, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, s11, v4
+; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v6
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; GISEL-NEXT:    v_mul_lo_u32 v8, v5, v9
+; GISEL-NEXT:    v_mul_lo_u32 v10, v4, v7
+; GISEL-NEXT:    v_mul_hi_u32 v11, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v9
+; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v6
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v7
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; GISEL-NEXT:    v_mul_hi_u32 v10, v4, v7
+; GISEL-NEXT:    v_mul_hi_u32 v7, v5, v7
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_addc_u32_e64 v8, s[4:5], v5, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, s12, v4
+; GISEL-NEXT:    v_mul_lo_u32 v10, s11, v8
+; GISEL-NEXT:    v_mul_hi_u32 v12, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, s11, v4
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v7
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v10
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_mul_lo_u32 v10, v8, v11
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v7, v4, v11
+; GISEL-NEXT:    v_mul_hi_u32 v11, v8, v11
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v10, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v10, v8, v9
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v12, v7
+; GISEL-NEXT:    v_mul_hi_u32 v12, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v8, v8, v9
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v10, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v11, v10
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v9
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v8, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v7, v1, v4
+; GISEL-NEXT:    v_mul_lo_u32 v8, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v9, v0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, v1, v5
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; GISEL-NEXT:    v_mul_hi_u32 v8, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v9, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT:    v_mul_lo_u32 v7, s9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v8, s8, v5
+; GISEL-NEXT:    v_mul_hi_u32 v10, s8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, s8, v4
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT:    v_subb_u32_e64 v8, s[4:5], v1, v7, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v7
+; GISEL-NEXT:    v_mov_b32_e32 v7, s9
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s9, v8
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v7, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s8, v0
+; GISEL-NEXT:    v_subrev_i32_e32 v0, vcc, s8, v0
+; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], s9, v8
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, v9, v10, s[4:5]
+; GISEL-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s9, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
+; GISEL-NEXT:    s_add_u32 s4, s10, 0
+; GISEL-NEXT:    s_cselect_b32 s5, 1, 0
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, s9, v1
+; GISEL-NEXT:    s_and_b32 s5, s5, 1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, 1, v7
+; GISEL-NEXT:    s_cmp_lg_u32 s5, 0
+; GISEL-NEXT:    v_addc_u32_e32 v10, vcc, 0, v9, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GISEL-NEXT:    s_addc_u32 s5, 0, 0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v7, v1, vcc
+; GISEL-NEXT:    s_xor_b64 s[6:7], s[4:5], s[6:7]
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v9, v10, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s6
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, s7
+; GISEL-NEXT:    s_sub_u32 s8, 0, s6
+; GISEL-NEXT:    s_cselect_b32 s4, 1, 0
+; GISEL-NEXT:    s_and_b32 s4, s4, 1
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
+; GISEL-NEXT:    s_subb_u32 s9, 0, s7
+; GISEL-NEXT:    v_ashrrev_i32_e32 v7, 31, v3
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; GISEL-NEXT:    v_trunc_f32_e32 v5, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v7
+; GISEL-NEXT:    v_addc_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v8, s9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, s8, v5
+; GISEL-NEXT:    v_mul_hi_u32 v11, s8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v10, s8, v4
+; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v6
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; GISEL-NEXT:    v_mul_lo_u32 v9, v5, v10
+; GISEL-NEXT:    v_mul_lo_u32 v11, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v12, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v10, v5, v10
+; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v7
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v5, v8
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_mul_hi_u32 v11, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, v5, v8
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; GISEL-NEXT:    v_addc_u32_e64 v9, s[4:5], v5, v8, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v10, s9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, s8, v9
+; GISEL-NEXT:    v_mul_hi_u32 v13, s8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, s8, v4
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v8
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; GISEL-NEXT:    v_mul_lo_u32 v11, v9, v12
+; GISEL-NEXT:    v_mul_lo_u32 v13, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v8, v4, v12
+; GISEL-NEXT:    v_mul_hi_u32 v12, v9, v12
+; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v7
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v11, v9, v10
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v13, v8
+; GISEL-NEXT:    v_mul_hi_u32 v13, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v9, v9, v10
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v12, v11
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v10
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v9, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v6
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v2, v5
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT:    v_mul_hi_u32 v6, v2, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v5
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
+; GISEL-NEXT:    v_mul_hi_u32 v9, v2, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v3, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; GISEL-NEXT:    v_mul_lo_u32 v6, s7, v4
+; GISEL-NEXT:    v_mul_lo_u32 v8, s6, v5
+; GISEL-NEXT:    v_mul_hi_u32 v10, s6, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, s6, v4
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v10
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v9
+; GISEL-NEXT:    v_subb_u32_e64 v8, s[4:5], v3, v6, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v6
+; GISEL-NEXT:    v_mov_b32_e32 v6, s7
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s7, v8
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v6, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v2
+; GISEL-NEXT:    v_subrev_i32_e32 v2, vcc, s6, v2
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], s7, v8
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, v9, v10, s[4:5]
+; GISEL-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s7, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s6, v2
+; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, s7, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, 1, v6
+; GISEL-NEXT:    v_addc_u32_e32 v10, vcc, 0, v9, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v9, v10, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v7
+; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v7
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v7
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_sdiv_v2i64_oddk_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_mov_b32 s6, 0x12d8fb
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, s6
+; CGP-NEXT:    v_cvt_f32_u32_e32 v5, 0
+; CGP-NEXT:    s_mov_b32 s7, 0xffed2705
+; CGP-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
+; CGP-NEXT:    v_mov_b32_e32 v7, v4
+; CGP-NEXT:    v_mac_f32_e32 v7, 0x4f800000, v5
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v7, v7
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_addc_u32_e32 v1, vcc, v1, v6, vcc
+; CGP-NEXT:    v_xor_b32_e32 v0, v0, v6
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x5f7ffffc, v7
+; CGP-NEXT:    v_mul_f32_e32 v8, 0x2f800000, v7
+; CGP-NEXT:    v_trunc_f32_e32 v8, v8
+; CGP-NEXT:    v_mac_f32_e32 v7, 0xcf800000, v8
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v6
+; CGP-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; CGP-NEXT:    v_mul_lo_u32 v9, -1, v7
+; CGP-NEXT:    v_mul_lo_u32 v10, s7, v8
+; CGP-NEXT:    v_mul_hi_u32 v12, s7, v7
+; CGP-NEXT:    v_mul_lo_u32 v11, s7, v7
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; CGP-NEXT:    v_mul_lo_u32 v10, v8, v11
+; CGP-NEXT:    v_mul_lo_u32 v12, v7, v9
+; CGP-NEXT:    v_mul_hi_u32 v13, v7, v11
+; CGP-NEXT:    v_mul_hi_u32 v11, v8, v11
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v13, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_mul_hi_u32 v12, v7, v9
+; CGP-NEXT:    v_mul_hi_u32 v9, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; CGP-NEXT:    v_addc_u32_e64 v10, s[4:5], v8, v9, vcc
+; CGP-NEXT:    v_mul_lo_u32 v11, -1, v7
+; CGP-NEXT:    v_mul_lo_u32 v12, s7, v10
+; CGP-NEXT:    v_mul_hi_u32 v14, s7, v7
+; CGP-NEXT:    v_mul_lo_u32 v13, s7, v7
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v9
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; CGP-NEXT:    v_mul_lo_u32 v12, v10, v13
+; CGP-NEXT:    v_mul_lo_u32 v14, v7, v11
+; CGP-NEXT:    v_mul_hi_u32 v9, v7, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v10, v13
+; CGP-NEXT:    v_ashrrev_i32_e32 v5, 31, v3
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v12, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v12, v10, v11
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v14, v9
+; CGP-NEXT:    v_mul_hi_u32 v14, v7, v11
+; CGP-NEXT:    v_mul_hi_u32 v10, v10, v11
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v14
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v12, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v13, v12
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; CGP-NEXT:    v_addc_u32_e32 v8, vcc, v8, v10, vcc
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; CGP-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; CGP-NEXT:    v_mul_lo_u32 v9, v1, v7
+; CGP-NEXT:    v_mul_lo_u32 v10, v0, v8
+; CGP-NEXT:    v_mul_hi_u32 v11, v0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v1, v7
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v11, v1, v8
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT:    v_mul_hi_u32 v10, v0, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v1, v8
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v11, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_mul_lo_u32 v9, 0, v7
+; CGP-NEXT:    v_mul_lo_u32 v10, s6, v8
+; CGP-NEXT:    v_mul_hi_u32 v12, s6, v7
+; CGP-NEXT:    v_mul_lo_u32 v11, s6, v7
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v11
+; CGP-NEXT:    v_subb_u32_e64 v10, s[4:5], v1, v9, vcc
+; CGP-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v9
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], 0, v10
+; CGP-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v0
+; CGP-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v0
+; CGP-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v10
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, 1, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v9, v9, v11, s[4:5]
+; CGP-NEXT:    v_addc_u32_e32 v11, vcc, 0, v8, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, 1, v10
+; CGP-NEXT:    v_addc_u32_e32 v12, vcc, 0, v11, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v10, v1, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v11, v12, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x2f800000, v4
+; CGP-NEXT:    v_trunc_f32_e32 v7, v7
+; CGP-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT:    v_mul_lo_u32 v8, -1, v4
+; CGP-NEXT:    v_mul_lo_u32 v9, s7, v7
+; CGP-NEXT:    v_mul_hi_u32 v11, s7, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, s7, v4
+; CGP-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; CGP-NEXT:    v_mul_lo_u32 v9, v7, v10
+; CGP-NEXT:    v_mul_lo_u32 v11, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v12, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v7, v10
+; CGP-NEXT:    v_xor_b32_e32 v0, v0, v6
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v12, v7, v8
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT:    v_mul_hi_u32 v11, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v7, v8
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; CGP-NEXT:    v_addc_u32_e64 v9, s[4:5], v7, v8, vcc
+; CGP-NEXT:    v_mul_lo_u32 v10, -1, v4
+; CGP-NEXT:    v_mul_lo_u32 v11, s7, v9
+; CGP-NEXT:    v_mul_hi_u32 v13, s7, v4
+; CGP-NEXT:    v_mul_lo_u32 v12, s7, v4
+; CGP-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v8
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; CGP-NEXT:    v_mul_lo_u32 v11, v9, v12
+; CGP-NEXT:    v_mul_lo_u32 v13, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v8, v4, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v9, v12
+; CGP-NEXT:    v_xor_b32_e32 v2, v2, v5
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v11, v9, v10
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v13, v8
+; CGP-NEXT:    v_mul_hi_u32 v13, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v9, v9, v10
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v12, v11
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v10
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, v7, v9, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT:    v_xor_b32_e32 v3, v3, v5
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v6
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_mul_lo_u32 v8, v3, v4
+; CGP-NEXT:    v_mul_lo_u32 v9, v2, v7
+; CGP-NEXT:    v_subb_u32_e32 v1, vcc, v1, v6, vcc
+; CGP-NEXT:    v_mul_hi_u32 v6, v2, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v8, v3, v7
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
+; CGP-NEXT:    v_mul_hi_u32 v9, v2, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v3, v7
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_mul_lo_u32 v7, 0, v4
+; CGP-NEXT:    v_mul_lo_u32 v8, s6, v6
+; CGP-NEXT:    v_mul_hi_u32 v10, s6, v4
+; CGP-NEXT:    v_mul_lo_u32 v9, s6, v4
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v9
+; CGP-NEXT:    v_subb_u32_e64 v8, s[4:5], v3, v7, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v7
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], 0, v8
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v2
+; CGP-NEXT:    v_subrev_i32_e32 v2, vcc, s6, v2
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v7, v7, v9, s[4:5]
+; CGP-NEXT:    v_addc_u32_e32 v9, vcc, 0, v6, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s6, v2
+; CGP-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, 1, v8
+; CGP-NEXT:    v_addc_u32_e32 v10, vcc, 0, v9, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v8, v3, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v9, v10, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; CGP-NEXT:    v_xor_b32_e32 v2, v2, v5
+; CGP-NEXT:    v_xor_b32_e32 v3, v3, v5
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT:    v_subb_u32_e32 v3, vcc, v3, v5, vcc
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %result = sdiv <2 x i64> %num, <i64 1235195, i64 1235195>
+  ret <2 x i64> %result
+}
+
+define i64 @v_sdiv_i64_pow2_shl_denom(i64 %x, i64 %y) {
+; CHECK-LABEL: v_sdiv_i64_pow2_shl_denom:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_movk_i32 s4, 0x1000
+; CHECK-NEXT:    s_mov_b32 s5, 0
+; CHECK-NEXT:    v_lshl_b64 v[4:5], s[4:5], v2
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0
+; CHECK-NEXT:    v_or_b32_e32 v3, v1, v5
+; CHECK-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CHECK-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CHECK-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; CHECK-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CHECK-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CHECK-NEXT:    s_cbranch_execz BB7_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    v_ashrrev_i32_e32 v2, 31, v5
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v4, v2
+; CHECK-NEXT:    v_addc_u32_e32 v5, vcc, v5, v2, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v3, v3, v2
+; CHECK-NEXT:    v_xor_b32_e32 v5, v5, v2
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v6, v3
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v7, v5
+; CHECK-NEXT:    v_ashrrev_i32_e32 v8, 31, v1
+; CHECK-NEXT:    v_mac_f32_e32 v6, 0x4f800000, v7
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v0, v8
+; CHECK-NEXT:    v_addc_u32_e32 v1, vcc, v1, v8, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v10, vcc, 0, v3
+; CHECK-NEXT:    v_mul_f32_e32 v6, 0x5f7ffffc, v6
+; CHECK-NEXT:    v_mul_f32_e32 v9, 0x2f800000, v6
+; CHECK-NEXT:    v_trunc_f32_e32 v9, v9
+; CHECK-NEXT:    v_mac_f32_e32 v6, 0xcf800000, v9
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v9, v9
+; CHECK-NEXT:    v_subb_u32_e32 v11, vcc, 0, v5, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v7, v7, v8
+; CHECK-NEXT:    v_mul_lo_u32 v12, v11, v6
+; CHECK-NEXT:    v_mul_lo_u32 v13, v10, v9
+; CHECK-NEXT:    v_mul_hi_u32 v15, v10, v6
+; CHECK-NEXT:    v_mul_lo_u32 v14, v10, v6
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v8
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; CHECK-NEXT:    v_mul_lo_u32 v13, v9, v14
+; CHECK-NEXT:    v_mul_lo_u32 v15, v6, v12
+; CHECK-NEXT:    v_mul_hi_u32 v16, v6, v14
+; CHECK-NEXT:    v_mul_hi_u32 v14, v9, v14
+; CHECK-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; CHECK-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
+; CHECK-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v16, v9, v12
+; CHECK-NEXT:    v_add_i32_e32 v13, vcc, v15, v13
+; CHECK-NEXT:    v_mul_hi_u32 v15, v6, v12
+; CHECK-NEXT:    v_mul_hi_u32 v12, v9, v12
+; CHECK-NEXT:    v_add_i32_e32 v14, vcc, v16, v14
+; CHECK-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; CHECK-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v15, vcc, v16, v15
+; CHECK-NEXT:    v_add_i32_e32 v13, vcc, v14, v13
+; CHECK-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v13
+; CHECK-NEXT:    v_addc_u32_e64 v13, s[4:5], v9, v12, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v11, v11, v6
+; CHECK-NEXT:    v_mul_lo_u32 v14, v10, v13
+; CHECK-NEXT:    v_mul_lo_u32 v15, v10, v6
+; CHECK-NEXT:    v_mul_hi_u32 v10, v10, v6
+; CHECK-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; CHECK-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; CHECK-NEXT:    v_mul_hi_u32 v12, v6, v15
+; CHECK-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v10
+; CHECK-NEXT:    v_mul_lo_u32 v11, v13, v15
+; CHECK-NEXT:    v_mul_lo_u32 v14, v6, v10
+; CHECK-NEXT:    v_mul_hi_u32 v15, v13, v15
+; CHECK-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; CHECK-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CHECK-NEXT:    v_mul_lo_u32 v12, v13, v10
+; CHECK-NEXT:    v_add_i32_e64 v11, s[4:5], v14, v11
+; CHECK-NEXT:    v_mul_hi_u32 v14, v6, v10
+; CHECK-NEXT:    v_mul_hi_u32 v10, v13, v10
+; CHECK-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
+; CHECK-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; CHECK-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v14, s[4:5], v15, v14
+; CHECK-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v12, s[4:5], v14, v12
+; CHECK-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v12
+; CHECK-NEXT:    v_addc_u32_e32 v9, vcc, v9, v10, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v11
+; CHECK-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v10, v1, v6
+; CHECK-NEXT:    v_mul_lo_u32 v11, v7, v9
+; CHECK-NEXT:    v_mul_hi_u32 v12, v7, v6
+; CHECK-NEXT:    v_mul_hi_u32 v6, v1, v6
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v12, v1, v9
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT:    v_mul_hi_u32 v11, v7, v9
+; CHECK-NEXT:    v_mul_hi_u32 v9, v1, v9
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v12, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CHECK-NEXT:    v_mul_lo_u32 v10, v5, v6
+; CHECK-NEXT:    v_mul_lo_u32 v11, v3, v9
+; CHECK-NEXT:    v_mul_hi_u32 v13, v3, v6
+; CHECK-NEXT:    v_mul_lo_u32 v12, v3, v6
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v10, v13
+; CHECK-NEXT:    v_sub_i32_e32 v7, vcc, v7, v12
+; CHECK-NEXT:    v_subb_u32_e64 v11, s[4:5], v1, v10, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v10
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v11, v5
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v3
+; CHECK-NEXT:    v_sub_i32_e32 v7, vcc, v7, v3
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], v11, v5
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, 1, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, v10, v12, s[4:5]
+; CHECK-NEXT:    v_addc_u32_e32 v12, vcc, 0, v9, vcc
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v13, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v7, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v13, v3, vcc
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, 1, v11
+; CHECK-NEXT:    v_addc_u32_e32 v5, vcc, 0, v12, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v11, v3, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, v12, v5, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v5, v8, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v5
+; CHECK-NEXT:    v_xor_b32_e32 v3, v3, v5
+; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, v1, v5
+; CHECK-NEXT:    v_subb_u32_e32 v3, vcc, v3, v5, vcc
+; CHECK-NEXT:  BB7_2: ; %Flow
+; CHECK-NEXT:    s_or_saveexec_b64 s[6:7], s[6:7]
+; CHECK-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; CHECK-NEXT:    s_cbranch_execz BB7_4
+; CHECK-NEXT:  ; %bb.3:
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, v4
+; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, 0, v4
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; CHECK-NEXT:    v_mul_lo_u32 v2, v2, v1
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CHECK-NEXT:    v_mul_lo_u32 v2, v1, v4
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; CHECK-NEXT:    v_sub_i32_e64 v2, s[4:5], v0, v4
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, 1, v1
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:  BB7_4:
+; CHECK-NEXT:    s_or_b64 exec, exec, s[6:7]
+; CHECK-NEXT:    v_mov_b32_e32 v0, v2
+; CHECK-NEXT:    v_mov_b32_e32 v1, v3
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shl.y = shl i64 4096, %y
+  %r = sdiv i64 %x, %shl.y
+  ret i64 %r
+}
+
+define <2 x i64> @v_sdiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
+; GISEL-LABEL: v_sdiv_v2i64_pow2_shl_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_movk_i32 s6, 0x1000
+; GISEL-NEXT:    s_mov_b32 s7, 0
+; GISEL-NEXT:    v_lshl_b64 v[4:5], s[6:7], v4
+; GISEL-NEXT:    v_ashrrev_i32_e32 v10, 31, v1
+; GISEL-NEXT:    v_ashrrev_i32_e32 v7, 31, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v7, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v4, v4, v7
+; GISEL-NEXT:    v_xor_b32_e32 v5, v5, v7
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v8, v4
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v9, v5
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v10, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, 0, v4
+; GISEL-NEXT:    v_mac_f32_e32 v8, 0x4f800000, v9
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v8, v8
+; GISEL-NEXT:    v_xor_b32_e32 v9, v0, v10
+; GISEL-NEXT:    v_subb_u32_e32 v12, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v17, v1, v10
+; GISEL-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v8
+; GISEL-NEXT:    v_mul_f32_e32 v8, 0x2f800000, v0
+; GISEL-NEXT:    v_trunc_f32_e32 v8, v8
+; GISEL-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v8
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; GISEL-NEXT:    v_mul_lo_u32 v13, v12, v0
+; GISEL-NEXT:    v_mul_lo_u32 v14, v11, v8
+; GISEL-NEXT:    v_mul_hi_u32 v16, v11, v0
+; GISEL-NEXT:    v_mul_lo_u32 v15, v11, v0
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
+; GISEL-NEXT:    v_mul_lo_u32 v14, v8, v15
+; GISEL-NEXT:    v_mul_lo_u32 v16, v0, v13
+; GISEL-NEXT:    v_mul_hi_u32 v1, v0, v15
+; GISEL-NEXT:    v_mul_hi_u32 v15, v8, v15
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v14, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v14, v8, v13
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v16, v1
+; GISEL-NEXT:    v_mul_hi_u32 v16, v0, v13
+; GISEL-NEXT:    v_mul_hi_u32 v13, v8, v13
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v15, v16
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v14, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT:    v_addc_u32_e64 v1, s[4:5], v8, v13, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v12, v0
+; GISEL-NEXT:    v_mul_lo_u32 v14, v11, v1
+; GISEL-NEXT:    v_mul_lo_u32 v15, v11, v0
+; GISEL-NEXT:    v_mul_hi_u32 v11, v11, v0
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v13
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; GISEL-NEXT:    v_mul_hi_u32 v13, v0, v15
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_mul_lo_u32 v12, v1, v15
+; GISEL-NEXT:    v_mul_lo_u32 v14, v0, v11
+; GISEL-NEXT:    v_mul_hi_u32 v15, v1, v15
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v13, v1, v11
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v14, v12
+; GISEL-NEXT:    v_mul_hi_u32 v14, v0, v11
+; GISEL-NEXT:    v_mul_hi_u32 v1, v1, v11
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[4:5], v15, v14
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v13, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v14, v13
+; GISEL-NEXT:    v_add_i32_e64 v1, s[4:5], v1, v11
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v8, v1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v0, v12
+; GISEL-NEXT:    v_addc_u32_e32 v11, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v17, v8
+; GISEL-NEXT:    v_mul_lo_u32 v13, v9, v11
+; GISEL-NEXT:    v_lshl_b64 v[0:1], s[6:7], v6
+; GISEL-NEXT:    v_mul_hi_u32 v6, v9, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, v17, v8
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v12, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v17, v11
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v13, v6
+; GISEL-NEXT:    v_mul_hi_u32 v13, v9, v11
+; GISEL-NEXT:    v_mul_hi_u32 v11, v17, v11
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v6
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v14, v4, v6
+; GISEL-NEXT:    v_mul_lo_u32 v13, v4, v6
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v14
+; GISEL-NEXT:    v_sub_i32_e32 v9, vcc, v9, v13
+; GISEL-NEXT:    v_subb_u32_e64 v12, s[4:5], v17, v11, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v11, s[4:5], v17, v11
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v12, v5
+; GISEL-NEXT:    v_subb_u32_e32 v11, vcc, v11, v5, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v4
+; GISEL-NEXT:    v_sub_i32_e32 v9, vcc, v9, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v12, v5
+; GISEL-NEXT:    v_subbrev_u32_e32 v11, vcc, 0, v11, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, v13, v14, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, 1, v6
+; GISEL-NEXT:    v_addc_u32_e32 v14, vcc, 0, v8, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v11, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v9, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v11, v5
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v15, v4, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, 1, v13
+; GISEL-NEXT:    v_addc_u32_e32 v9, vcc, 0, v14, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v13, v5, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v5, v14, v9, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v6, v10, v7
+; GISEL-NEXT:    v_ashrrev_i32_e32 v7, 31, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v7
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v7, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v8, v0, v7
+; GISEL-NEXT:    v_xor_b32_e32 v9, v1, v7
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v0, v8
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v1, v9
+; GISEL-NEXT:    v_ashrrev_i32_e32 v10, 31, v3
+; GISEL-NEXT:    v_xor_b32_e32 v4, v4, v6
+; GISEL-NEXT:    v_xor_b32_e32 v5, v5, v6
+; GISEL-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v2, v10
+; GISEL-NEXT:    v_addc_u32_e32 v2, vcc, v3, v10, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v3, v1, v10
+; GISEL-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GISEL-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GISEL-NEXT:    v_trunc_f32_e32 v1, v1
+; GISEL-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, 0, v8
+; GISEL-NEXT:    v_subb_u32_e32 v12, vcc, 0, v9, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v13, v12, v0
+; GISEL-NEXT:    v_mul_lo_u32 v14, v11, v1
+; GISEL-NEXT:    v_mul_hi_u32 v16, v11, v0
+; GISEL-NEXT:    v_mul_lo_u32 v15, v11, v0
+; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v10
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
+; GISEL-NEXT:    v_mul_lo_u32 v14, v1, v15
+; GISEL-NEXT:    v_mul_lo_u32 v16, v0, v13
+; GISEL-NEXT:    v_mul_hi_u32 v17, v0, v15
+; GISEL-NEXT:    v_mul_hi_u32 v15, v1, v15
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v17, v1, v13
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v16, v14
+; GISEL-NEXT:    v_mul_hi_u32 v16, v0, v13
+; GISEL-NEXT:    v_mul_hi_u32 v13, v1, v13
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v17, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v15, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v16, vcc, v17, v16
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v16, v15
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v14
+; GISEL-NEXT:    v_addc_u32_e64 v14, s[4:5], v1, v13, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v12, v0
+; GISEL-NEXT:    v_mul_lo_u32 v15, v11, v14
+; GISEL-NEXT:    v_mul_lo_u32 v16, v11, v0
+; GISEL-NEXT:    v_mul_hi_u32 v11, v11, v0
+; GISEL-NEXT:    v_add_i32_e64 v1, s[4:5], v1, v13
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
+; GISEL-NEXT:    v_mul_hi_u32 v13, v0, v16
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_mul_lo_u32 v12, v14, v16
+; GISEL-NEXT:    v_mul_lo_u32 v15, v0, v11
+; GISEL-NEXT:    v_mul_hi_u32 v16, v14, v16
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v13, v14, v11
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v15, v12
+; GISEL-NEXT:    v_mul_hi_u32 v15, v0, v11
+; GISEL-NEXT:    v_mul_hi_u32 v11, v14, v11
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v15, s[4:5], v16, v15
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v13, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v15, v13
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v11, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v0, v12
+; GISEL-NEXT:    v_addc_u32_e32 v12, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v13, v2, v11
+; GISEL-NEXT:    v_mul_lo_u32 v14, v3, v12
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v4, v6
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v11
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v5, v6, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v13, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v5, v2, v12
+; GISEL-NEXT:    v_mul_hi_u32 v11, v2, v11
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; GISEL-NEXT:    v_mul_hi_u32 v6, v3, v12
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v11, v6
+; GISEL-NEXT:    v_mul_hi_u32 v11, v2, v12
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v11, v5
+; GISEL-NEXT:    v_mul_lo_u32 v6, v9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, v8, v5
+; GISEL-NEXT:    v_mul_hi_u32 v13, v8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, v8, v4
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v11
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v13
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, v3, v12
+; GISEL-NEXT:    v_subb_u32_e64 v11, s[4:5], v2, v6, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v6
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v11, v9
+; GISEL-NEXT:    v_subb_u32_e32 v2, vcc, v2, v9, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v3, v8
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, v3, v8
+; GISEL-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v11, v9
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, 1, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, v6, v12, s[4:5]
+; GISEL-NEXT:    v_addc_u32_e32 v12, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v9
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v13, v3, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, 1, v11
+; GISEL-NEXT:    v_addc_u32_e32 v8, vcc, 0, v12, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v11, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v12, v8, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v4, v10, v7
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v4
+; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v4
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v4
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_sdiv_v2i64_pow2_shl_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_movk_i32 s4, 0x1000
+; CGP-NEXT:    s_mov_b32 s5, 0
+; CGP-NEXT:    v_lshl_b64 v[10:11], s[4:5], v4
+; CGP-NEXT:    v_mov_b32_e32 v7, v1
+; CGP-NEXT:    v_mov_b32_e32 v5, v0
+; CGP-NEXT:    v_or_b32_e32 v1, v7, v11
+; CGP-NEXT:    v_mov_b32_e32 v0, 0
+; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; CGP-NEXT:    v_lshl_b64 v[8:9], s[4:5], v6
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CGP-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CGP-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; CGP-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CGP-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execz BB8_2
+; CGP-NEXT:  ; %bb.1:
+; CGP-NEXT:    v_ashrrev_i32_e32 v0, 31, v11
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, v10, v0
+; CGP-NEXT:    v_addc_u32_e32 v4, vcc, v11, v0, vcc
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v0
+; CGP-NEXT:    v_xor_b32_e32 v4, v4, v0
+; CGP-NEXT:    v_cvt_f32_u32_e32 v6, v1
+; CGP-NEXT:    v_cvt_f32_u32_e32 v11, v4
+; CGP-NEXT:    v_ashrrev_i32_e32 v12, 31, v7
+; CGP-NEXT:    v_mac_f32_e32 v6, 0x4f800000, v11
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v5, v12
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, v7, v12, vcc
+; CGP-NEXT:    v_sub_i32_e32 v14, vcc, 0, v1
+; CGP-NEXT:    v_mul_f32_e32 v6, 0x5f7ffffc, v6
+; CGP-NEXT:    v_mul_f32_e32 v13, 0x2f800000, v6
+; CGP-NEXT:    v_trunc_f32_e32 v13, v13
+; CGP-NEXT:    v_mac_f32_e32 v6, 0xcf800000, v13
+; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; CGP-NEXT:    v_cvt_u32_f32_e32 v13, v13
+; CGP-NEXT:    v_subb_u32_e32 v15, vcc, 0, v4, vcc
+; CGP-NEXT:    v_xor_b32_e32 v11, v11, v12
+; CGP-NEXT:    v_mul_lo_u32 v16, v15, v6
+; CGP-NEXT:    v_mul_lo_u32 v17, v14, v13
+; CGP-NEXT:    v_mul_hi_u32 v19, v14, v6
+; CGP-NEXT:    v_mul_lo_u32 v18, v14, v6
+; CGP-NEXT:    v_xor_b32_e32 v7, v7, v12
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v16, v17
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v16, v19
+; CGP-NEXT:    v_mul_lo_u32 v17, v13, v18
+; CGP-NEXT:    v_mul_lo_u32 v19, v6, v16
+; CGP-NEXT:    v_mul_hi_u32 v20, v6, v18
+; CGP-NEXT:    v_mul_hi_u32 v18, v13, v18
+; CGP-NEXT:    v_add_i32_e32 v17, vcc, v17, v19
+; CGP-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v17, vcc, v17, v20
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v20, v13, v16
+; CGP-NEXT:    v_add_i32_e32 v17, vcc, v19, v17
+; CGP-NEXT:    v_mul_hi_u32 v19, v6, v16
+; CGP-NEXT:    v_mul_hi_u32 v16, v13, v16
+; CGP-NEXT:    v_add_i32_e32 v18, vcc, v20, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v18, vcc, v18, v19
+; CGP-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v19, vcc, v20, v19
+; CGP-NEXT:    v_add_i32_e32 v17, vcc, v18, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v18, vcc, v19, v18
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v16, v18
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v17
+; CGP-NEXT:    v_addc_u32_e64 v17, s[4:5], v13, v16, vcc
+; CGP-NEXT:    v_mul_lo_u32 v15, v15, v6
+; CGP-NEXT:    v_mul_lo_u32 v18, v14, v17
+; CGP-NEXT:    v_mul_lo_u32 v19, v14, v6
+; CGP-NEXT:    v_mul_hi_u32 v14, v14, v6
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v16
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v15, v18
+; CGP-NEXT:    v_mul_hi_u32 v16, v6, v19
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v15, v14
+; CGP-NEXT:    v_mul_lo_u32 v15, v17, v19
+; CGP-NEXT:    v_mul_lo_u32 v18, v6, v14
+; CGP-NEXT:    v_mul_hi_u32 v19, v17, v19
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v15, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v15, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v16, v17, v14
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v18, v15
+; CGP-NEXT:    v_mul_hi_u32 v18, v6, v14
+; CGP-NEXT:    v_mul_hi_u32 v14, v17, v14
+; CGP-NEXT:    v_add_i32_e64 v16, s[4:5], v16, v19
+; CGP-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v16, s[4:5], v16, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v18, s[4:5], v19, v18
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v16, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v16, s[4:5], v18, v16
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v16
+; CGP-NEXT:    v_addc_u32_e32 v13, vcc, v13, v14, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v15
+; CGP-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; CGP-NEXT:    v_mul_lo_u32 v14, v7, v6
+; CGP-NEXT:    v_mul_lo_u32 v15, v11, v13
+; CGP-NEXT:    v_mul_hi_u32 v16, v11, v6
+; CGP-NEXT:    v_mul_hi_u32 v6, v7, v6
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v16, v7, v13
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT:    v_mul_hi_u32 v15, v11, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v7, v13
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v16, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v16, v15
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; CGP-NEXT:    v_mul_lo_u32 v14, v4, v6
+; CGP-NEXT:    v_mul_lo_u32 v15, v1, v13
+; CGP-NEXT:    v_mul_hi_u32 v17, v1, v6
+; CGP-NEXT:    v_mul_lo_u32 v16, v1, v6
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; CGP-NEXT:    v_sub_i32_e32 v11, vcc, v11, v16
+; CGP-NEXT:    v_subb_u32_e64 v15, s[4:5], v7, v14, vcc
+; CGP-NEXT:    v_sub_i32_e64 v7, s[4:5], v7, v14
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v15, v4
+; CGP-NEXT:    v_subb_u32_e32 v7, vcc, v7, v4, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v11, v1
+; CGP-NEXT:    v_sub_i32_e32 v11, vcc, v11, v1
+; CGP-NEXT:    v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], v15, v4
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, 1, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v14, v14, v16, s[4:5]
+; CGP-NEXT:    v_addc_u32_e32 v16, vcc, 0, v13, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v7, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, -1, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v11, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v7, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, 1, v15
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, 0, v16, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v15, v4, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v16, v7, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; CGP-NEXT:    v_xor_b32_e32 v6, v12, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
+; CGP-NEXT:    v_xor_b32_e32 v0, v1, v6
+; CGP-NEXT:    v_xor_b32_e32 v1, v4, v6
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_subb_u32_e32 v1, vcc, v1, v6, vcc
+; CGP-NEXT:  BB8_2: ; %Flow2
+; CGP-NEXT:    s_or_saveexec_b64 s[6:7], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execz BB8_4
+; CGP-NEXT:  ; %bb.3:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v10
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, 0, v10
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_lo_u32 v1, v1, v0
+; CGP-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; CGP-NEXT:    v_mul_hi_u32 v0, v5, v0
+; CGP-NEXT:    v_mul_lo_u32 v1, v0, v10
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v5, v1
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v10
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_sub_i32_e64 v4, s[4:5], v1, v10
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v10
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_mov_b32_e32 v1, 0
+; CGP-NEXT:  BB8_4:
+; CGP-NEXT:    s_or_b64 exec, exec, s[6:7]
+; CGP-NEXT:    v_or_b32_e32 v5, v3, v9
+; CGP-NEXT:    v_mov_b32_e32 v4, 0
+; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CGP-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CGP-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; CGP-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CGP-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execz BB8_6
+; CGP-NEXT:  ; %bb.5:
+; CGP-NEXT:    v_ashrrev_i32_e32 v4, 31, v9
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v8, v4
+; CGP-NEXT:    v_addc_u32_e32 v6, vcc, v9, v4, vcc
+; CGP-NEXT:    v_xor_b32_e32 v5, v5, v4
+; CGP-NEXT:    v_xor_b32_e32 v6, v6, v4
+; CGP-NEXT:    v_cvt_f32_u32_e32 v7, v5
+; CGP-NEXT:    v_cvt_f32_u32_e32 v9, v6
+; CGP-NEXT:    v_ashrrev_i32_e32 v10, 31, v3
+; CGP-NEXT:    v_mac_f32_e32 v7, 0x4f800000, v9
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v7, v7
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v2, v10
+; CGP-NEXT:    v_addc_u32_e32 v3, vcc, v3, v10, vcc
+; CGP-NEXT:    v_sub_i32_e32 v12, vcc, 0, v5
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x5f7ffffc, v7
+; CGP-NEXT:    v_mul_f32_e32 v11, 0x2f800000, v7
+; CGP-NEXT:    v_trunc_f32_e32 v11, v11
+; CGP-NEXT:    v_mac_f32_e32 v7, 0xcf800000, v11
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v11, v11
+; CGP-NEXT:    v_subb_u32_e32 v13, vcc, 0, v6, vcc
+; CGP-NEXT:    v_xor_b32_e32 v9, v9, v10
+; CGP-NEXT:    v_mul_lo_u32 v14, v13, v7
+; CGP-NEXT:    v_mul_lo_u32 v15, v12, v11
+; CGP-NEXT:    v_mul_hi_u32 v17, v12, v7
+; CGP-NEXT:    v_mul_lo_u32 v16, v12, v7
+; CGP-NEXT:    v_xor_b32_e32 v3, v3, v10
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; CGP-NEXT:    v_mul_lo_u32 v15, v11, v16
+; CGP-NEXT:    v_mul_lo_u32 v17, v7, v14
+; CGP-NEXT:    v_mul_hi_u32 v18, v7, v16
+; CGP-NEXT:    v_mul_hi_u32 v16, v11, v16
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v15, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v15, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v18, v11, v14
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v17, v15
+; CGP-NEXT:    v_mul_hi_u32 v17, v7, v14
+; CGP-NEXT:    v_mul_hi_u32 v14, v11, v14
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v18, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v16, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v17, vcc, v18, v17
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v16, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v17, v16
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v15
+; CGP-NEXT:    v_addc_u32_e64 v15, s[4:5], v11, v14, vcc
+; CGP-NEXT:    v_mul_lo_u32 v13, v13, v7
+; CGP-NEXT:    v_mul_lo_u32 v16, v12, v15
+; CGP-NEXT:    v_mul_lo_u32 v17, v12, v7
+; CGP-NEXT:    v_mul_hi_u32 v12, v12, v7
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v16
+; CGP-NEXT:    v_mul_hi_u32 v14, v7, v17
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v13, v12
+; CGP-NEXT:    v_mul_lo_u32 v13, v15, v17
+; CGP-NEXT:    v_mul_lo_u32 v16, v7, v12
+; CGP-NEXT:    v_mul_hi_u32 v17, v15, v17
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v14, v15, v12
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v16, v13
+; CGP-NEXT:    v_mul_hi_u32 v16, v7, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v15, v12
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v16, s[4:5], v17, v16
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v16, v14
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; CGP-NEXT:    v_addc_u32_e32 v11, vcc, v11, v12, vcc
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v13
+; CGP-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; CGP-NEXT:    v_mul_lo_u32 v12, v3, v7
+; CGP-NEXT:    v_mul_lo_u32 v13, v9, v11
+; CGP-NEXT:    v_mul_hi_u32 v14, v9, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v3, v7
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v14, v3, v11
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_mul_hi_u32 v13, v9, v11
+; CGP-NEXT:    v_mul_hi_u32 v11, v3, v11
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v14, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v14, v13
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT:    v_mul_lo_u32 v12, v6, v7
+; CGP-NEXT:    v_mul_lo_u32 v13, v5, v11
+; CGP-NEXT:    v_mul_hi_u32 v15, v5, v7
+; CGP-NEXT:    v_mul_lo_u32 v14, v5, v7
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; CGP-NEXT:    v_sub_i32_e32 v9, vcc, v9, v14
+; CGP-NEXT:    v_subb_u32_e64 v13, s[4:5], v3, v12, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v12
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v13, v6
+; CGP-NEXT:    v_subb_u32_e32 v3, vcc, v3, v6, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v5
+; CGP-NEXT:    v_sub_i32_e32 v9, vcc, v9, v5
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], v13, v6
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, 1, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v12, v12, v14, s[4:5]
+; CGP-NEXT:    v_addc_u32_e32 v14, vcc, 0, v11, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, -1, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v9, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v6
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v15, v5, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v13
+; CGP-NEXT:    v_addc_u32_e32 v6, vcc, 0, v14, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v13, v5, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v5, v14, v6, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; CGP-NEXT:    v_xor_b32_e32 v6, v10, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v5, v11, v5, vcc
+; CGP-NEXT:    v_xor_b32_e32 v3, v3, v6
+; CGP-NEXT:    v_xor_b32_e32 v5, v5, v6
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v3, v6
+; CGP-NEXT:    v_subb_u32_e32 v5, vcc, v5, v6, vcc
+; CGP-NEXT:  BB8_6: ; %Flow
+; CGP-NEXT:    s_or_saveexec_b64 s[6:7], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execz BB8_8
+; CGP-NEXT:  ; %bb.7:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v3, v8
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, 0, v8
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; CGP-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, v3
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v3, v2, v3
+; CGP-NEXT:    v_mul_lo_u32 v4, v3, v8
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v4
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v8
+; CGP-NEXT:    v_sub_i32_e64 v4, s[4:5], v2, v8
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, 1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v8
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v3, v4, vcc
+; CGP-NEXT:    v_mov_b32_e32 v5, 0
+; CGP-NEXT:  BB8_8:
+; CGP-NEXT:    s_or_b64 exec, exec, s[6:7]
+; CGP-NEXT:    v_mov_b32_e32 v2, v4
+; CGP-NEXT:    v_mov_b32_e32 v3, v5
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %shl.y = shl <2 x i64> <i64 4096, i64 4096>, %y
+  %r = sdiv <2 x i64> %x, %shl.y
+  ret <2 x i64> %r
+}
+
+define i64 @v_sdiv_i64_24bit(i64 %num, i64 %den) {
+; GISEL-LABEL: v_sdiv_i64_24bit:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_mov_b32 s4, 0xffffff
+; GISEL-NEXT:    v_and_b32_e32 v1, s4, v2
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v2, v1
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; GISEL-NEXT:    v_and_b32_e32 v0, s4, v0
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GISEL-NEXT:    v_mul_lo_u32 v3, v3, v2
+; GISEL-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GISEL-NEXT:    v_mul_hi_u32 v2, v0, v2
+; GISEL-NEXT:    v_mul_lo_u32 v3, v2, v1
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, 1, v2
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
+; GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_sdiv_i64_24bit:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_mov_b32 s4, 0xffffff
+; CGP-NEXT:    v_and_b32_e32 v1, s4, v2
+; CGP-NEXT:    v_cvt_f32_i32_e32 v2, v1
+; CGP-NEXT:    v_and_b32_e32 v0, s4, v0
+; CGP-NEXT:    v_cvt_f32_i32_e32 v3, v0
+; CGP-NEXT:    v_xor_b32_e32 v0, v0, v1
+; CGP-NEXT:    v_rcp_f32_e32 v4, v2
+; CGP-NEXT:    v_ashrrev_i32_e32 v0, 30, v0
+; CGP-NEXT:    v_or_b32_e32 v0, 1, v0
+; CGP-NEXT:    v_mul_f32_e32 v1, v3, v4
+; CGP-NEXT:    v_trunc_f32_e32 v1, v1
+; CGP-NEXT:    v_mad_f32 v3, -v1, v2, v3
+; CGP-NEXT:    v_cvt_i32_f32_e32 v1, v1
+; CGP-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
+; CGP-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; CGP-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
+; CGP-NEXT:    v_ashrrev_i32_e32 v0, 7, v0
+; CGP-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %num.mask = and i64 %num, 16777215
+  %den.mask = and i64 %den, 16777215
+  %result = sdiv i64 %num.mask, %den.mask
+  ret i64 %result
+}
+
+define <2 x i64> @v_sdiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
+; GISEL-LABEL: v_sdiv_v2i64_24bit:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_mov_b32 s6, 0xffffff
+; GISEL-NEXT:    v_and_b32_e32 v1, s6, v4
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, 0, v1
+; GISEL-NEXT:    v_addc_u32_e64 v3, s[4:5], 0, 0, vcc
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, v1
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v3
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v1
+; GISEL-NEXT:    v_subb_u32_e32 v8, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_and_b32_e32 v0, s6, v0
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    v_and_b32_e32 v6, s6, v6
+; GISEL-NEXT:    v_and_b32_e32 v2, s6, v2
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; GISEL-NEXT:    v_trunc_f32_e32 v5, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_mul_lo_u32 v9, v8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v10, v7, v5
+; GISEL-NEXT:    v_mul_hi_u32 v12, v7, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, v7, v4
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT:    v_mul_lo_u32 v10, v5, v11
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v14, v4, v11
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
+; GISEL-NEXT:    v_addc_u32_e64 v13, s[4:5], 0, 0, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v14, v5, v9
+; GISEL-NEXT:    v_mul_hi_u32 v11, v5, v11
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; GISEL-NEXT:    v_mul_hi_u32 v12, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v9
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v14, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v14, v12
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; GISEL-NEXT:    v_addc_u32_e64 v10, s[4:5], v5, v9, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v8, v8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, v7, v10
+; GISEL-NEXT:    v_mul_lo_u32 v12, v7, v4
+; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v4
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v9
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; GISEL-NEXT:    v_mul_hi_u32 v9, v4, v12
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v8, v7
+; GISEL-NEXT:    v_mul_lo_u32 v8, v10, v12
+; GISEL-NEXT:    v_mul_lo_u32 v11, v4, v7
+; GISEL-NEXT:    v_mul_hi_u32 v12, v10, v12
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v9, v10, v7
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; GISEL-NEXT:    v_mul_hi_u32 v11, v4, v7
+; GISEL-NEXT:    v_mul_hi_u32 v7, v10, v7
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v9, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v11, v9
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v9
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v7, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v7, v13, v4
+; GISEL-NEXT:    v_mul_lo_u32 v8, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v9, v0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v13, v4
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, v13, v5
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; GISEL-NEXT:    v_mul_hi_u32 v8, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v13, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v9, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT:    v_mul_lo_u32 v7, v3, v4
+; GISEL-NEXT:    v_mul_lo_u32 v8, v1, v5
+; GISEL-NEXT:    v_mul_hi_u32 v10, v1, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v1, v4
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT:    v_subb_u32_e64 v8, s[4:5], v13, v7, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v7, s[4:5], v13, v7
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v3
+; GISEL-NEXT:    v_subb_u32_e32 v7, vcc, v7, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v1
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v8, v3
+; GISEL-NEXT:    v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, v9, v10, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, 1, v4
+; GISEL-NEXT:    v_addc_u32_e32 v10, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v7, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v7, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, 1, v9
+; GISEL-NEXT:    v_addc_u32_e32 v3, vcc, 0, v10, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, 0, v6
+; GISEL-NEXT:    v_addc_u32_e64 v7, s[4:5], 0, 0, vcc
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v11, v6
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v12, v7
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v9, v1, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v10, v3, vcc
+; GISEL-NEXT:    v_mac_f32_e32 v11, 0x4f800000, v12
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v3, v11
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GISEL-NEXT:    v_mul_f32_e32 v3, 0x5f7ffffc, v3
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x2f800000, v3
+; GISEL-NEXT:    v_trunc_f32_e32 v4, v4
+; GISEL-NEXT:    v_mac_f32_e32 v3, 0xcf800000, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v6
+; GISEL-NEXT:    v_subb_u32_e32 v8, vcc, 0, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, v8, v3
+; GISEL-NEXT:    v_mul_lo_u32 v10, v5, v4
+; GISEL-NEXT:    v_mul_hi_u32 v12, v5, v3
+; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v3
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT:    v_mul_lo_u32 v10, v4, v11
+; GISEL-NEXT:    v_mul_lo_u32 v12, v3, v9
+; GISEL-NEXT:    v_mul_hi_u32 v14, v3, v11
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, 0, v2
+; GISEL-NEXT:    v_addc_u32_e64 v13, s[4:5], 0, 0, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v14, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v11, v4, v11
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; GISEL-NEXT:    v_mul_hi_u32 v12, v3, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v4, v9
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v14, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v14, v12
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v10
+; GISEL-NEXT:    v_addc_u32_e64 v10, s[4:5], v4, v9, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v8, v8, v3
+; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v10
+; GISEL-NEXT:    v_mul_lo_u32 v12, v5, v3
+; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v3
+; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v9
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; GISEL-NEXT:    v_mul_hi_u32 v9, v3, v12
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v8, v5
+; GISEL-NEXT:    v_mul_lo_u32 v8, v10, v12
+; GISEL-NEXT:    v_mul_lo_u32 v11, v3, v5
+; GISEL-NEXT:    v_mul_hi_u32 v12, v10, v12
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v9, v10, v5
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; GISEL-NEXT:    v_mul_hi_u32 v11, v3, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v10, v5
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v9, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v11, v9
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v9
+; GISEL-NEXT:    v_addc_u32_e32 v4, vcc, v4, v5, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v8
+; GISEL-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v5, v13, v3
+; GISEL-NEXT:    v_mul_lo_u32 v8, v2, v4
+; GISEL-NEXT:    v_mul_hi_u32 v9, v2, v3
+; GISEL-NEXT:    v_subrev_i32_e32 v0, vcc, 0, v0
+; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, v13, v4
+; GISEL-NEXT:    v_mul_hi_u32 v3, v13, v3
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
+; GISEL-NEXT:    v_mul_hi_u32 v8, v2, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v13, v4
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v9, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT:    v_mul_lo_u32 v5, v7, v3
+; GISEL-NEXT:    v_mul_lo_u32 v8, v6, v4
+; GISEL-NEXT:    v_mul_hi_u32 v10, v6, v3
+; GISEL-NEXT:    v_mul_lo_u32 v9, v6, v3
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v9
+; GISEL-NEXT:    v_subb_u32_e64 v8, s[4:5], v13, v5, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v5, s[4:5], v13, v5
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v7
+; GISEL-NEXT:    v_subb_u32_e32 v5, vcc, v5, v7, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v6
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v8, v7
+; GISEL-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, v9, v10, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, 1, v3
+; GISEL-NEXT:    v_addc_u32_e32 v10, vcc, 0, v4, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v5, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v7
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, 1, v9
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, 0, v10, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v9, v5, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v5, v10, v6, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v2, vcc, 0, v2
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_sdiv_v2i64_24bit:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_mov_b32 s4, 0xffffff
+; CGP-NEXT:    v_and_b32_e32 v1, s4, v4
+; CGP-NEXT:    v_cvt_f32_i32_e32 v3, v1
+; CGP-NEXT:    v_and_b32_e32 v0, s4, v0
+; CGP-NEXT:    v_and_b32_e32 v4, s4, v6
+; CGP-NEXT:    v_cvt_f32_i32_e32 v5, v0
+; CGP-NEXT:    v_rcp_f32_e32 v6, v3
+; CGP-NEXT:    v_xor_b32_e32 v0, v0, v1
+; CGP-NEXT:    v_and_b32_e32 v2, s4, v2
+; CGP-NEXT:    v_ashrrev_i32_e32 v0, 30, v0
+; CGP-NEXT:    v_mul_f32_e32 v1, v5, v6
+; CGP-NEXT:    v_trunc_f32_e32 v1, v1
+; CGP-NEXT:    v_mad_f32 v5, -v1, v3, v5
+; CGP-NEXT:    v_cmp_ge_f32_e64 vcc, |v5|, |v3|
+; CGP-NEXT:    v_cvt_f32_i32_e32 v3, v4
+; CGP-NEXT:    v_cvt_f32_i32_e32 v5, v2
+; CGP-NEXT:    v_cvt_i32_f32_e32 v1, v1
+; CGP-NEXT:    v_xor_b32_e32 v2, v2, v4
+; CGP-NEXT:    v_rcp_f32_e32 v6, v3
+; CGP-NEXT:    v_or_b32_e32 v0, 1, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; CGP-NEXT:    v_mul_f32_e32 v4, v5, v6
+; CGP-NEXT:    v_trunc_f32_e32 v4, v4
+; CGP-NEXT:    v_mad_f32 v5, -v4, v3, v5
+; CGP-NEXT:    v_cvt_i32_f32_e32 v4, v4
+; CGP-NEXT:    v_ashrrev_i32_e32 v2, 30, v2
+; CGP-NEXT:    v_or_b32_e32 v2, 1, v2
+; CGP-NEXT:    v_cmp_ge_f32_e64 vcc, |v5|, |v3|
+; CGP-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; CGP-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
+; CGP-NEXT:    v_lshlrev_b32_e32 v2, 7, v2
+; CGP-NEXT:    v_ashrrev_i32_e32 v0, 7, v0
+; CGP-NEXT:    v_ashrrev_i32_e32 v2, 7, v2
+; CGP-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CGP-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %num.mask = and <2 x i64> %num, <i64 16777215, i64 16777215>
+  %den.mask = and <2 x i64> %den, <i64 16777215, i64 16777215>
+  %result = sdiv <2 x i64> %num.mask, %den.mask
+  ret <2 x i64> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll
index c53d5627d9feb..43f79f4b207d0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll
@@ -15,28 +15,22 @@ define i32 @v_srem_i32(i32 %num, i32 %den) {
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v3
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v3, v1
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, 0, v1
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; GISEL-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; GISEL-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GISEL-NEXT:    v_mul_lo_u32 v4, v3, v1
-; GISEL-NEXT:    v_mul_hi_u32 v5, v3, v1
-; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v4, v4, v3
-; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v3, v4
-; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v3, v3, v0
+; GISEL-NEXT:    v_mul_lo_u32 v4, v4, v3
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; GISEL-NEXT:    v_mul_hi_u32 v3, v0, v3
 ; GISEL-NEXT:    v_mul_lo_u32 v3, v3, v1
-; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v0, v3
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v1
-; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v4, v1
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v3
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[6:7], v4, v1
-; GISEL-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, v0, v1
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, v0, v1
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
@@ -51,39 +45,30 @@ define i32 @v_srem_i32(i32 %num, i32 %den) {
 ; CGP-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; CGP-NEXT:    v_xor_b32_e32 v1, v1, v3
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v3, v1
-; CGP-NEXT:    v_mul_lo_u32 v4, 0, v1
-; CGP-NEXT:    v_mul_lo_u32 v5, 0, v0
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, 0, v1
+; CGP-NEXT:    v_mul_lo_u32 v5, v0, 0
 ; CGP-NEXT:    v_rcp_f32_e32 v3, v3
-; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; CGP-NEXT:    v_mul_lo_u32 v6, v3, v1
-; CGP-NEXT:    v_mul_lo_u32 v7, v3, 0
-; CGP-NEXT:    v_mul_hi_u32 v8, v3, v1
-; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
-; CGP-NEXT:    v_sub_i32_e32 v9, vcc, 0, v6
-; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; CGP-NEXT:    v_cndmask_b32_e32 v4, v6, v9, vcc
-; CGP-NEXT:    v_mul_lo_u32 v6, v4, 0
-; CGP-NEXT:    v_mul_hi_u32 v4, v4, v3
-; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
-; CGP-NEXT:    v_add_i32_e64 v4, s[4:5], v6, v4
-; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v3, v4
-; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
-; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
-; CGP-NEXT:    v_mul_lo_u32 v4, v3, 0
-; CGP-NEXT:    v_mul_hi_u32 v3, v3, v0
-; CGP-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, v3
+; CGP-NEXT:    v_mul_lo_u32 v6, v3, 0
+; CGP-NEXT:    v_mul_lo_u32 v7, 0, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT:    v_mul_lo_u32 v4, 0, v3
+; CGP-NEXT:    v_mul_hi_u32 v3, v0, v3
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
 ; CGP-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
 ; CGP-NEXT:    v_mul_lo_u32 v3, v3, v1
-; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v0, v3
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v1
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v4, v1
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v3
-; CGP-NEXT:    v_sub_i32_e64 v0, s[6:7], v4, v1
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, v0, v1
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, v0, v1
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; CGP-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
@@ -97,87 +82,70 @@ declare i32 @llvm.amdgcn.readfirstlane(i32)
 define amdgpu_ps i32 @s_srem_i32(i32 inreg %num, i32 inreg %den) {
 ; GISEL-LABEL: s_srem_i32:
 ; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_ashr_i32 s4, s0, 31
-; GISEL-NEXT:    s_ashr_i32 s2, s1, 31
-; GISEL-NEXT:    s_add_i32 s0, s0, s4
-; GISEL-NEXT:    s_add_i32 s1, s1, s2
-; GISEL-NEXT:    s_xor_b32 s3, s0, s4
-; GISEL-NEXT:    s_xor_b32 s2, s1, s2
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GISEL-NEXT:    s_ashr_i32 s2, s0, 31
+; GISEL-NEXT:    s_ashr_i32 s3, s1, 31
+; GISEL-NEXT:    s_add_i32 s0, s0, s2
+; GISEL-NEXT:    s_add_i32 s1, s1, s3
+; GISEL-NEXT:    s_xor_b32 s0, s0, s2
+; GISEL-NEXT:    s_xor_b32 s1, s1, s3
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v0, s1
+; GISEL-NEXT:    s_sub_i32 s3, 0, s1
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GISEL-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GISEL-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GISEL-NEXT:    v_mul_lo_u32 v1, v0, s2
-; GISEL-NEXT:    v_mul_hi_u32 v2, v0, s2
-; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GISEL-NEXT:    v_add_i32_e64 v2, s[0:1], v0, v1
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[0:1], v0, v1
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v0, v0, s3
-; GISEL-NEXT:    v_mul_lo_u32 v0, v0, s2
-; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, s3, v0
-; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s2, v1
-; GISEL-NEXT:    v_add_i32_e64 v2, s[0:1], s2, v1
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[0:1], s3, v0
-; GISEL-NEXT:    v_subrev_i32_e64 v0, s[2:3], s2, v1
-; GISEL-NEXT:    s_and_b64 vcc, vcc, s[0:1]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[0:1]
-; GISEL-NEXT:    v_xor_b32_e32 v0, s4, v0
-; GISEL-NEXT:    v_subrev_i32_e32 v0, vcc, s4, v0
+; GISEL-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GISEL-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GISEL-NEXT:    v_mul_lo_u32 v0, v0, s1
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GISEL-NEXT:    v_subrev_i32_e32 v1, vcc, s1, v0
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s1, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v1, vcc, s1, v0
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s1, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v0, s2, v0
+; GISEL-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
 ; GISEL-NEXT:    v_readfirstlane_b32 s0, v0
 ; GISEL-NEXT:    ; return to shader part epilog
 ;
 ; CGP-LABEL: s_srem_i32:
 ; CGP:       ; %bb.0:
-; CGP-NEXT:    s_ashr_i32 s5, s0, 31
-; CGP-NEXT:    s_ashr_i32 s3, s1, 31
-; CGP-NEXT:    s_add_i32 s0, s0, s5
-; CGP-NEXT:    s_add_i32 s1, s1, s3
-; CGP-NEXT:    s_xor_b32 s2, s0, s5
-; CGP-NEXT:    s_xor_b32 s4, s1, s3
-; CGP-NEXT:    v_cvt_f32_u32_e32 v0, s4
-; CGP-NEXT:    s_bfe_u64 s[0:1], s[4:5], 0x200000
-; CGP-NEXT:    s_bfe_u64 s[6:7], s[2:3], 0x200000
+; CGP-NEXT:    s_ashr_i32 s4, s0, 31
+; CGP-NEXT:    s_ashr_i32 s2, s1, 31
+; CGP-NEXT:    s_add_i32 s0, s0, s4
+; CGP-NEXT:    s_add_i32 s1, s1, s2
+; CGP-NEXT:    s_xor_b32 s0, s0, s4
+; CGP-NEXT:    s_xor_b32 s1, s1, s2
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, s1
+; CGP-NEXT:    s_sub_i32 s5, 0, s1
+; CGP-NEXT:    s_bfe_u64 s[2:3], s[0:1], 0x200000
 ; CGP-NEXT:    v_rcp_f32_e32 v0, v0
-; CGP-NEXT:    v_mul_lo_u32 v1, 0, s0
-; CGP-NEXT:    v_mul_lo_u32 v2, 0, s6
-; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; CGP-NEXT:    v_mul_lo_u32 v1, s2, 0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; CGP-NEXT:    v_mul_lo_u32 v3, v0, s0
-; CGP-NEXT:    v_mul_lo_u32 v4, v0, s1
-; CGP-NEXT:    v_mul_hi_u32 v5, v0, s0
-; CGP-NEXT:    v_mul_lo_u32 v6, 0, v0
-; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
-; CGP-NEXT:    v_sub_i32_e32 v4, vcc, 0, v3
-; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; CGP-NEXT:    v_mul_lo_u32 v3, v1, 0
-; CGP-NEXT:    v_mul_hi_u32 v1, v1, v0
-; CGP-NEXT:    v_add_i32_e64 v3, s[0:1], v6, v3
-; CGP-NEXT:    v_add_i32_e64 v1, s[0:1], v3, v1
-; CGP-NEXT:    v_add_i32_e64 v3, s[0:1], v0, v1
-; CGP-NEXT:    v_sub_i32_e64 v0, s[0:1], v0, v1
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; CGP-NEXT:    v_mul_lo_u32 v1, v0, s7
-; CGP-NEXT:    v_mul_hi_u32 v0, v0, s6
+; CGP-NEXT:    v_mul_lo_u32 v2, s5, v0
+; CGP-NEXT:    v_mul_lo_u32 v3, v0, 0
+; CGP-NEXT:    v_mul_lo_u32 v4, 0, v2
+; CGP-NEXT:    v_mul_hi_u32 v2, v0, v2
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; CGP-NEXT:    v_mul_lo_u32 v2, s3, v0
+; CGP-NEXT:    v_mul_hi_u32 v0, s2, v0
 ; CGP-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; CGP-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
-; CGP-NEXT:    v_mul_lo_u32 v0, v0, s4
-; CGP-NEXT:    v_sub_i32_e32 v1, vcc, s2, v0
-; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s4, v1
-; CGP-NEXT:    v_add_i32_e64 v2, s[0:1], s4, v1
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[0:1], s2, v0
-; CGP-NEXT:    v_subrev_i32_e64 v0, s[2:3], s4, v1
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[0:1]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[0:1]
-; CGP-NEXT:    v_xor_b32_e32 v0, s5, v0
-; CGP-NEXT:    v_subrev_i32_e32 v0, vcc, s5, v0
+; CGP-NEXT:    v_mul_lo_u32 v0, v0, s1
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; CGP-NEXT:    v_subrev_i32_e32 v1, vcc, s1, v0
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s1, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v1, vcc, s1, v0
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s1, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CGP-NEXT:    v_xor_b32_e32 v0, s4, v0
+; CGP-NEXT:    v_subrev_i32_e32 v0, vcc, s4, v0
 ; CGP-NEXT:    v_readfirstlane_b32 s0, v0
 ; CGP-NEXT:    ; return to shader part epilog
   %result = srem i32 %num, %den
@@ -202,51 +170,39 @@ define <2 x i32> @v_srem_v2i32(<2 x i32> %num, <2 x i32> %den) {
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v6
 ; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v7
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v2
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v7, v3
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v2
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v8, v3
+; GISEL-NEXT:    v_sub_i32_e32 v9, vcc, 0, v3
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; GISEL-NEXT:    v_rcp_iflag_f32_e32 v7, v7
-; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v5
-; GISEL-NEXT:    v_mul_f32_e32 v7, 0x4f800000, v7
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v8, v8
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f7ffffe, v5
+; GISEL-NEXT:    v_mul_f32_e32 v8, 0x4f7ffffe, v8
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; GISEL-NEXT:    v_mul_lo_u32 v8, v5, v2
-; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v2
-; GISEL-NEXT:    v_mul_lo_u32 v10, v7, v3
-; GISEL-NEXT:    v_mul_hi_u32 v11, v7, v3
-; GISEL-NEXT:    v_sub_i32_e32 v12, vcc, 0, v8
-; GISEL-NEXT:    v_sub_i32_e32 v13, vcc, 0, v10
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v9
-; GISEL-NEXT:    v_cndmask_b32_e32 v8, v8, v12, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v11
-; GISEL-NEXT:    v_cndmask_b32_e64 v9, v10, v13, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v8, v8, v5
-; GISEL-NEXT:    v_mul_hi_u32 v9, v9, v7
-; GISEL-NEXT:    v_add_i32_e64 v10, s[6:7], v5, v8
-; GISEL-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v8
-; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v7, v9
-; GISEL-NEXT:    v_sub_i32_e64 v7, s[6:7], v7, v9
-; GISEL-NEXT:    v_cndmask_b32_e32 v5, v5, v10, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v0
-; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v1
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v5
+; GISEL-NEXT:    v_mul_lo_u32 v9, v9, v8
+; GISEL-NEXT:    v_mul_hi_u32 v7, v5, v7
+; GISEL-NEXT:    v_mul_hi_u32 v9, v8, v9
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v9
+; GISEL-NEXT:    v_mul_hi_u32 v5, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v7, v1, v7
 ; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v2
 ; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v3
-; GISEL-NEXT:    v_sub_i32_e32 v8, vcc, v0, v5
-; GISEL-NEXT:    v_sub_i32_e32 v9, vcc, v1, v7
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v8, v2
-; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v8, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v5
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[6:7], v8, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[6:7], v9, v3
-; GISEL-NEXT:    v_add_i32_e64 v2, s[8:9], v9, v3
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v7
-; GISEL-NEXT:    v_sub_i32_e64 v1, s[10:11], v9, v3
-; GISEL-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GISEL-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v10, v0, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[8:9]
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v7
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, v0, v2
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, v1, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, v0, v2
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, v1, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v4
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v6
 ; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
@@ -269,73 +225,55 @@ define <2 x i32> @v_srem_v2i32(<2 x i32> %num, <2 x i32> %den) {
 ; CGP-NEXT:    v_xor_b32_e32 v1, v1, v6
 ; CGP-NEXT:    v_xor_b32_e32 v3, v3, v7
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v5, v2
-; CGP-NEXT:    v_mul_lo_u32 v7, 0, v2
-; CGP-NEXT:    v_mul_lo_u32 v8, 0, v0
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, 0, v2
+; CGP-NEXT:    v_mul_lo_u32 v8, v0, 0
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v9, v3
-; CGP-NEXT:    v_mul_lo_u32 v10, 0, v3
-; CGP-NEXT:    v_mul_lo_u32 v11, 0, v1
+; CGP-NEXT:    v_sub_i32_e32 v10, vcc, 0, v3
+; CGP-NEXT:    v_mul_lo_u32 v11, v1, 0
 ; CGP-NEXT:    v_rcp_f32_e32 v5, v5
 ; CGP-NEXT:    v_rcp_f32_e32 v9, v9
-; CGP-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v5
-; CGP-NEXT:    v_mul_f32_e32 v9, 0x4f800000, v9
+; CGP-NEXT:    v_mul_f32_e32 v5, 0x4f7ffffe, v5
+; CGP-NEXT:    v_mul_f32_e32 v9, 0x4f7ffffe, v9
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v9, v9
-; CGP-NEXT:    v_mul_lo_u32 v12, v5, v2
-; CGP-NEXT:    v_mul_lo_u32 v13, v5, 0
-; CGP-NEXT:    v_mul_hi_u32 v14, v5, v2
-; CGP-NEXT:    v_mul_lo_u32 v15, v9, v3
-; CGP-NEXT:    v_mul_lo_u32 v16, v9, 0
-; CGP-NEXT:    v_mul_hi_u32 v17, v9, v3
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v13
-; CGP-NEXT:    v_sub_i32_e32 v18, vcc, 0, v12
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v16
-; CGP-NEXT:    v_sub_i32_e32 v19, vcc, 0, v15
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v14
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v17
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; CGP-NEXT:    v_cndmask_b32_e32 v7, v12, v18, vcc
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v10
-; CGP-NEXT:    v_cndmask_b32_e64 v10, v15, v19, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v12, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v7, v7, v5
-; CGP-NEXT:    v_mul_lo_u32 v14, v10, 0
-; CGP-NEXT:    v_mul_hi_u32 v10, v10, v9
-; CGP-NEXT:    v_add_i32_e64 v12, s[6:7], v13, v12
-; CGP-NEXT:    v_add_i32_e64 v13, s[6:7], v16, v14
-; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v12, v7
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v13, v10
-; CGP-NEXT:    v_add_i32_e64 v12, s[6:7], v5, v7
-; CGP-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v7
-; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v9, v10
-; CGP-NEXT:    v_sub_i32_e64 v9, s[6:7], v9, v10
-; CGP-NEXT:    v_cndmask_b32_e32 v5, v5, v12, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v7, v9, v7, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v9, v5, 0
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v0
-; CGP-NEXT:    v_mul_lo_u32 v10, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v7, v7, v1
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v11, v10
+; CGP-NEXT:    v_mul_lo_u32 v7, v7, v5
+; CGP-NEXT:    v_mul_lo_u32 v12, v5, 0
+; CGP-NEXT:    v_mul_lo_u32 v10, v10, v9
+; CGP-NEXT:    v_mul_lo_u32 v13, v9, 0
+; CGP-NEXT:    v_mul_lo_u32 v14, 0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v5, v7
+; CGP-NEXT:    v_mul_lo_u32 v15, 0, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v9, v10
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v14, v12
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v15, v13
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v12, v7
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v9, v10
+; CGP-NEXT:    v_mul_lo_u32 v9, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v0, v5
+; CGP-NEXT:    v_mul_lo_u32 v10, 0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v1, v7
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v11
 ; CGP-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
 ; CGP-NEXT:    v_add_i32_e32 v7, vcc, v9, v7
 ; CGP-NEXT:    v_mul_lo_u32 v5, v5, v2
 ; CGP-NEXT:    v_mul_lo_u32 v7, v7, v3
-; CGP-NEXT:    v_sub_i32_e32 v8, vcc, v0, v5
-; CGP-NEXT:    v_sub_i32_e32 v9, vcc, v1, v7
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v8, v2
-; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v8, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v5
-; CGP-NEXT:    v_sub_i32_e64 v0, s[6:7], v8, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v9, v3
-; CGP-NEXT:    v_add_i32_e64 v2, s[8:9], v9, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v7
-; CGP-NEXT:    v_sub_i32_e64 v1, s[10:11], v9, v3
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; CGP-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v10, v0, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[8:9]
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v7
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, v0, v2
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, v0, v2
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
 ; CGP-NEXT:    v_xor_b32_e32 v0, v0, v4
 ; CGP-NEXT:    v_xor_b32_e32 v1, v1, v6
 ; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
@@ -349,34 +287,27 @@ define i32 @v_srem_i32_pow2k_denom(i32 %num) {
 ; CHECK-LABEL: v_srem_i32_pow2k_denom:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_movk_i32 s6, 0x1000
+; CHECK-NEXT:    s_movk_i32 s4, 0x1000
 ; CHECK-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CHECK-NEXT:    v_mov_b32_e32 v2, 0x1000
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0xfffff000
 ; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, s6
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, s4
 ; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; CHECK-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; CHECK-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; CHECK-NEXT:    v_lshlrev_b32_e32 v4, 12, v3
-; CHECK-NEXT:    v_mul_hi_u32 v5, v3, s6
-; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; CHECK-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v4, v4, v3
-; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v3, v4
-; CHECK-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v0
-; CHECK-NEXT:    v_lshlrev_b32_e32 v3, 12, v3
-; CHECK-NEXT:    v_sub_i32_e32 v4, vcc, v0, v3
-; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v4
-; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v4, v2
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v3
-; CHECK-NEXT:    v_sub_i32_e64 v0, s[6:7], v4, v2
-; CHECK-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
+; CHECK-NEXT:    v_mul_lo_u32 v2, v2, v3
+; CHECK-NEXT:    v_mul_hi_u32 v2, v3, v2
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v0, v2
+; CHECK-NEXT:    v_lshlrev_b32_e32 v2, 12, v2
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; CHECK-NEXT:    v_subrev_i32_e32 v2, vcc, s4, v0
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v2, vcc, s4, v0
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
@@ -389,56 +320,43 @@ define <2 x i32> @v_srem_v2i32_pow2k_denom(<2 x i32> %num) {
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_ashrrev_i32_e32 v2, 31, v0
-; GISEL-NEXT:    s_add_i32 s10, 0x1000, 0
+; GISEL-NEXT:    s_add_i32 s4, 0x1000, 0
 ; GISEL-NEXT:    v_ashrrev_i32_e32 v3, 31, v1
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s10
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s4
+; GISEL-NEXT:    s_sub_i32 s5, 0, s4
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v3
-; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v4
-; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f7ffffe, v4
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT:    v_mul_lo_u32 v6, v5, s10
-; GISEL-NEXT:    v_mul_hi_u32 v7, v5, s10
-; GISEL-NEXT:    v_mul_lo_u32 v8, v4, s10
-; GISEL-NEXT:    v_mul_hi_u32 v9, v4, s10
-; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, 0, v6
-; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, 0, v8
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, v8, v11, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v6, v6, v5
-; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v4
-; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v5, v6
-; GISEL-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v6
-; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v4, v7
-; GISEL-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v5, v5, v8, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v0
-; GISEL-NEXT:    v_mul_hi_u32 v4, v4, v1
-; GISEL-NEXT:    v_mul_lo_u32 v5, v5, s10
-; GISEL-NEXT:    v_mul_lo_u32 v4, v4, s10
-; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, v0, v5
-; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, v1, v4
-; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s10, v6
-; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], s10, v6
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v5
-; GISEL-NEXT:    v_subrev_i32_e64 v0, s[6:7], s10, v6
-; GISEL-NEXT:    v_cmp_le_u32_e64 s[6:7], s10, v7
-; GISEL-NEXT:    v_add_i32_e64 v5, s[8:9], s10, v7
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v4
-; GISEL-NEXT:    v_subrev_i32_e64 v1, s[10:11], s10, v7
-; GISEL-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GISEL-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[8:9]
+; GISEL-NEXT:    v_mul_lo_u32 v6, s5, v5
+; GISEL-NEXT:    v_mul_lo_u32 v7, s5, v4
+; GISEL-NEXT:    v_mul_hi_u32 v6, v5, v6
+; GISEL-NEXT:    v_mul_hi_u32 v7, v4, v7
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT:    v_mul_hi_u32 v5, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GISEL-NEXT:    v_mul_lo_u32 v5, v5, s4
+; GISEL-NEXT:    v_mul_lo_u32 v4, v4, s4
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; GISEL-NEXT:    v_subrev_i32_e32 v4, vcc, s4, v0
+; GISEL-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v1
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v4, vcc, s4, v0
+; GISEL-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v1
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v3
 ; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
@@ -451,61 +369,49 @@ define <2 x i32> @v_srem_v2i32_pow2k_denom(<2 x i32> %num) {
 ; CGP-NEXT:    s_movk_i32 s4, 0x1000
 ; CGP-NEXT:    v_ashrrev_i32_e32 v2, 31, v0
 ; CGP-NEXT:    v_mov_b32_e32 v3, 0x1000
-; CGP-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
+; CGP-NEXT:    s_mov_b32 s5, 0xfffff000
+; CGP-NEXT:    v_mov_b32_e32 v4, 0xfffff000
+; CGP-NEXT:    v_ashrrev_i32_e32 v5, 31, v1
 ; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; CGP-NEXT:    v_cvt_f32_u32_e32 v5, s4
-; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
-; CGP-NEXT:    v_cvt_f32_u32_e32 v6, v3
+; CGP-NEXT:    v_cvt_f32_u32_e32 v6, s4
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
+; CGP-NEXT:    v_cvt_f32_u32_e32 v7, v3
 ; CGP-NEXT:    v_xor_b32_e32 v0, v0, v2
-; CGP-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; CGP-NEXT:    v_xor_b32_e32 v1, v1, v4
 ; CGP-NEXT:    v_rcp_iflag_f32_e32 v6, v6
-; CGP-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v5
-; CGP-NEXT:    v_mul_f32_e32 v6, 0x4f800000, v6
-; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v5
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v7, v7
+; CGP-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
-; CGP-NEXT:    v_lshlrev_b32_e32 v7, 12, v5
-; CGP-NEXT:    v_mul_hi_u32 v8, v5, s4
-; CGP-NEXT:    v_lshlrev_b32_e32 v9, 12, v6
-; CGP-NEXT:    v_mul_hi_u32 v10, v6, v3
-; CGP-NEXT:    v_sub_i32_e32 v11, vcc, 0, v7
-; CGP-NEXT:    v_sub_i32_e32 v12, vcc, 0, v9
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v8
-; CGP-NEXT:    v_cndmask_b32_e32 v7, v7, v11, vcc
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v10
-; CGP-NEXT:    v_cndmask_b32_e64 v8, v9, v12, s[4:5]
-; CGP-NEXT:    v_mul_hi_u32 v7, v7, v5
-; CGP-NEXT:    v_mul_hi_u32 v8, v8, v6
-; CGP-NEXT:    v_add_i32_e64 v9, s[6:7], v5, v7
-; CGP-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v7
-; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v6, v8
-; CGP-NEXT:    v_sub_i32_e64 v6, s[6:7], v6, v8
-; CGP-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[4:5]
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v0
-; CGP-NEXT:    v_mul_hi_u32 v6, v6, v1
-; CGP-NEXT:    v_lshlrev_b32_e32 v5, 12, v5
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_mul_lo_u32 v8, s5, v6
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, v7
+; CGP-NEXT:    v_mul_hi_u32 v8, v6, v8
+; CGP-NEXT:    v_mul_hi_u32 v4, v7, v4
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
+; CGP-NEXT:    v_mul_hi_u32 v6, v0, v6
+; CGP-NEXT:    v_mul_hi_u32 v4, v1, v4
 ; CGP-NEXT:    v_lshlrev_b32_e32 v6, 12, v6
-; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v0, v5
-; CGP-NEXT:    v_sub_i32_e32 v8, vcc, v1, v6
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v7, v3
-; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v7, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v5
-; CGP-NEXT:    v_sub_i32_e64 v0, s[6:7], v7, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v8, v3
-; CGP-NEXT:    v_add_i32_e64 v5, s[8:9], v8, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v6
-; CGP-NEXT:    v_sub_i32_e64 v1, s[10:11], v8, v3
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
-; CGP-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v9, v0, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[8:9]
+; CGP-NEXT:    v_lshlrev_b32_e32 v4, 12, v4
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; CGP-NEXT:    v_subrev_i32_e32 v4, vcc, s4, v0
+; CGP-NEXT:    v_sub_i32_e32 v6, vcc, v1, v3
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v4, vcc, s4, v0
+; CGP-NEXT:    v_sub_i32_e32 v6, vcc, v1, v3
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
 ; CGP-NEXT:    v_xor_b32_e32 v0, v0, v2
-; CGP-NEXT:    v_xor_b32_e32 v1, v1, v4
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v5
 ; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
-; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %result = srem <2 x i32> %num, <i32 4096, i32 4096>
   ret <2 x i32> %result
@@ -515,34 +421,27 @@ define i32 @v_srem_i32_oddk_denom(i32 %num) {
 ; CHECK-LABEL: v_srem_i32_oddk_denom:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_mov_b32 s6, 0x12d8fb
+; CHECK-NEXT:    s_mov_b32 s4, 0x12d8fb
 ; CHECK-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CHECK-NEXT:    v_mov_b32_e32 v2, 0x12d8fb
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0xffed2705
 ; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, s6
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, s4
 ; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; CHECK-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; CHECK-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; CHECK-NEXT:    v_mul_lo_u32 v4, v3, s6
-; CHECK-NEXT:    v_mul_hi_u32 v5, v3, s6
-; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; CHECK-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v4, v4, v3
-; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v3, v4
-; CHECK-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v0
-; CHECK-NEXT:    v_mul_lo_u32 v3, v3, s6
-; CHECK-NEXT:    v_sub_i32_e32 v4, vcc, v0, v3
-; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v4
-; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v4, v2
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v3
-; CHECK-NEXT:    v_sub_i32_e64 v0, s[6:7], v4, v2
-; CHECK-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
+; CHECK-NEXT:    v_mul_lo_u32 v2, v2, v3
+; CHECK-NEXT:    v_mul_hi_u32 v2, v3, v2
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v0, v2
+; CHECK-NEXT:    v_mul_lo_u32 v2, v2, s4
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; CHECK-NEXT:    v_subrev_i32_e32 v2, vcc, s4, v0
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v2, vcc, s4, v0
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
@@ -555,56 +454,43 @@ define <2 x i32> @v_srem_v2i32_oddk_denom(<2 x i32> %num) {
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_ashrrev_i32_e32 v2, 31, v0
-; GISEL-NEXT:    s_add_i32 s10, 0x12d8fb, 0
+; GISEL-NEXT:    s_add_i32 s4, 0x12d8fb, 0
 ; GISEL-NEXT:    v_ashrrev_i32_e32 v3, 31, v1
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s10
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s4
+; GISEL-NEXT:    s_sub_i32 s5, 0, s4
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v3
-; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v4
-; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f7ffffe, v4
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT:    v_mul_lo_u32 v6, v5, s10
-; GISEL-NEXT:    v_mul_hi_u32 v7, v5, s10
-; GISEL-NEXT:    v_mul_lo_u32 v8, v4, s10
-; GISEL-NEXT:    v_mul_hi_u32 v9, v4, s10
-; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, 0, v6
-; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, 0, v8
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, v8, v11, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v6, v6, v5
-; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v4
-; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v5, v6
-; GISEL-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v6
-; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v4, v7
-; GISEL-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v5, v5, v8, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v0
-; GISEL-NEXT:    v_mul_hi_u32 v4, v4, v1
-; GISEL-NEXT:    v_mul_lo_u32 v5, v5, s10
-; GISEL-NEXT:    v_mul_lo_u32 v4, v4, s10
-; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, v0, v5
-; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, v1, v4
-; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s10, v6
-; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], s10, v6
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v5
-; GISEL-NEXT:    v_subrev_i32_e64 v0, s[6:7], s10, v6
-; GISEL-NEXT:    v_cmp_le_u32_e64 s[6:7], s10, v7
-; GISEL-NEXT:    v_add_i32_e64 v5, s[8:9], s10, v7
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v4
-; GISEL-NEXT:    v_subrev_i32_e64 v1, s[10:11], s10, v7
-; GISEL-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GISEL-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[8:9]
+; GISEL-NEXT:    v_mul_lo_u32 v6, s5, v5
+; GISEL-NEXT:    v_mul_lo_u32 v7, s5, v4
+; GISEL-NEXT:    v_mul_hi_u32 v6, v5, v6
+; GISEL-NEXT:    v_mul_hi_u32 v7, v4, v7
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT:    v_mul_hi_u32 v5, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GISEL-NEXT:    v_mul_lo_u32 v5, v5, s4
+; GISEL-NEXT:    v_mul_lo_u32 v4, v4, s4
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; GISEL-NEXT:    v_subrev_i32_e32 v4, vcc, s4, v0
+; GISEL-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v1
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v4, vcc, s4, v0
+; GISEL-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v1
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v3
 ; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
@@ -614,64 +500,52 @@ define <2 x i32> @v_srem_v2i32_oddk_denom(<2 x i32> %num) {
 ; CGP-LABEL: v_srem_v2i32_oddk_denom:
 ; CGP:       ; %bb.0:
 ; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT:    s_mov_b32 s8, 0x12d8fb
+; CGP-NEXT:    s_mov_b32 s4, 0x12d8fb
 ; CGP-NEXT:    v_ashrrev_i32_e32 v2, 31, v0
 ; CGP-NEXT:    v_mov_b32_e32 v3, 0x12d8fb
-; CGP-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
+; CGP-NEXT:    s_mov_b32 s5, 0xffed2705
+; CGP-NEXT:    v_mov_b32_e32 v4, 0xffed2705
+; CGP-NEXT:    v_ashrrev_i32_e32 v5, 31, v1
 ; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; CGP-NEXT:    v_cvt_f32_u32_e32 v5, s8
-; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
-; CGP-NEXT:    v_cvt_f32_u32_e32 v6, v3
+; CGP-NEXT:    v_cvt_f32_u32_e32 v6, s4
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
+; CGP-NEXT:    v_cvt_f32_u32_e32 v7, v3
 ; CGP-NEXT:    v_xor_b32_e32 v0, v0, v2
-; CGP-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; CGP-NEXT:    v_xor_b32_e32 v1, v1, v4
 ; CGP-NEXT:    v_rcp_iflag_f32_e32 v6, v6
-; CGP-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v5
-; CGP-NEXT:    v_mul_f32_e32 v6, 0x4f800000, v6
-; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v5
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v7, v7
+; CGP-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
-; CGP-NEXT:    v_mul_lo_u32 v7, v5, s8
-; CGP-NEXT:    v_mul_hi_u32 v8, v5, s8
-; CGP-NEXT:    v_mul_lo_u32 v9, v6, v3
-; CGP-NEXT:    v_mul_hi_u32 v10, v6, v3
-; CGP-NEXT:    v_sub_i32_e32 v11, vcc, 0, v7
-; CGP-NEXT:    v_sub_i32_e32 v12, vcc, 0, v9
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v8
-; CGP-NEXT:    v_cndmask_b32_e32 v7, v7, v11, vcc
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v10
-; CGP-NEXT:    v_cndmask_b32_e64 v8, v9, v12, s[4:5]
-; CGP-NEXT:    v_mul_hi_u32 v7, v7, v5
-; CGP-NEXT:    v_mul_hi_u32 v8, v8, v6
-; CGP-NEXT:    v_add_i32_e64 v9, s[6:7], v5, v7
-; CGP-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v7
-; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v6, v8
-; CGP-NEXT:    v_sub_i32_e64 v6, s[6:7], v6, v8
-; CGP-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[4:5]
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v0
-; CGP-NEXT:    v_mul_hi_u32 v6, v6, v1
-; CGP-NEXT:    v_mul_lo_u32 v5, v5, s8
-; CGP-NEXT:    v_mul_lo_u32 v6, v6, v3
-; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v0, v5
-; CGP-NEXT:    v_sub_i32_e32 v8, vcc, v1, v6
-; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s8, v7
-; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v7, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v5
-; CGP-NEXT:    v_sub_i32_e64 v0, s[6:7], v7, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v8, v3
-; CGP-NEXT:    v_add_i32_e64 v5, s[8:9], v8, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v6
-; CGP-NEXT:    v_sub_i32_e64 v1, s[10:11], v8, v3
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
-; CGP-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v9, v0, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[8:9]
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_mul_lo_u32 v8, s5, v6
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, v7
+; CGP-NEXT:    v_mul_hi_u32 v8, v6, v8
+; CGP-NEXT:    v_mul_hi_u32 v4, v7, v4
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
+; CGP-NEXT:    v_mul_hi_u32 v6, v0, v6
+; CGP-NEXT:    v_mul_hi_u32 v4, v1, v4
+; CGP-NEXT:    v_mul_lo_u32 v6, v6, s4
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, v3
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; CGP-NEXT:    v_subrev_i32_e32 v4, vcc, s4, v0
+; CGP-NEXT:    v_sub_i32_e32 v6, vcc, v1, v3
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v4, vcc, s4, v0
+; CGP-NEXT:    v_sub_i32_e32 v6, vcc, v1, v3
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
 ; CGP-NEXT:    v_xor_b32_e32 v0, v0, v2
-; CGP-NEXT:    v_xor_b32_e32 v1, v1, v4
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v5
 ; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
-; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %result = srem <2 x i32> %num, <i32 1235195, i32 1235195>
   ret <2 x i32> %result
@@ -689,28 +563,22 @@ define i32 @v_srem_i32_pow2_shl_denom(i32 %x, i32 %y) {
 ; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v3
 ; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, v1
+; CHECK-NEXT:    v_sub_i32_e32 v4, vcc, 0, v1
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; CHECK-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; CHECK-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; CHECK-NEXT:    v_mul_lo_u32 v4, v3, v1
-; CHECK-NEXT:    v_mul_hi_u32 v5, v3, v1
-; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; CHECK-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v4, v4, v3
-; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v3, v4
-; CHECK-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v0
+; CHECK-NEXT:    v_mul_lo_u32 v4, v4, v3
+; CHECK-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; CHECK-NEXT:    v_mul_hi_u32 v3, v0, v3
 ; CHECK-NEXT:    v_mul_lo_u32 v3, v3, v1
-; CHECK-NEXT:    v_sub_i32_e32 v4, vcc, v0, v3
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v1
-; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v4, v1
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v3
-; CHECK-NEXT:    v_sub_i32_e64 v0, s[6:7], v4, v1
-; CHECK-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, v0, v1
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, v0, v1
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
@@ -739,51 +607,39 @@ define <2 x i32> @v_srem_v2i32_pow2_shl_denom(<2 x i32> %x, <2 x i32> %y) {
 ; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v6
 ; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v7
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v6, v2
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v7, v3
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v2
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v8, v3
+; GISEL-NEXT:    v_sub_i32_e32 v9, vcc, 0, v3
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v6, v6
-; GISEL-NEXT:    v_rcp_iflag_f32_e32 v7, v7
-; GISEL-NEXT:    v_mul_f32_e32 v6, 0x4f800000, v6
-; GISEL-NEXT:    v_mul_f32_e32 v7, 0x4f800000, v7
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v8, v8
+; GISEL-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; GISEL-NEXT:    v_mul_f32_e32 v8, 0x4f7ffffe, v8
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; GISEL-NEXT:    v_mul_lo_u32 v8, v6, v2
-; GISEL-NEXT:    v_mul_hi_u32 v9, v6, v2
-; GISEL-NEXT:    v_mul_lo_u32 v10, v7, v3
-; GISEL-NEXT:    v_mul_hi_u32 v11, v7, v3
-; GISEL-NEXT:    v_sub_i32_e32 v12, vcc, 0, v8
-; GISEL-NEXT:    v_sub_i32_e32 v13, vcc, 0, v10
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v9
-; GISEL-NEXT:    v_cndmask_b32_e32 v8, v8, v12, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v11
-; GISEL-NEXT:    v_cndmask_b32_e64 v9, v10, v13, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v8, v8, v6
-; GISEL-NEXT:    v_mul_hi_u32 v9, v9, v7
-; GISEL-NEXT:    v_add_i32_e64 v10, s[6:7], v6, v8
-; GISEL-NEXT:    v_sub_i32_e64 v6, s[6:7], v6, v8
-; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v7, v9
-; GISEL-NEXT:    v_sub_i32_e64 v7, s[6:7], v7, v9
-; GISEL-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v6, v6, v0
-; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v1
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v6
+; GISEL-NEXT:    v_mul_lo_u32 v9, v9, v8
+; GISEL-NEXT:    v_mul_hi_u32 v7, v6, v7
+; GISEL-NEXT:    v_mul_hi_u32 v9, v8, v9
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v9
+; GISEL-NEXT:    v_mul_hi_u32 v6, v0, v6
+; GISEL-NEXT:    v_mul_hi_u32 v7, v1, v7
 ; GISEL-NEXT:    v_mul_lo_u32 v6, v6, v2
 ; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v3
-; GISEL-NEXT:    v_sub_i32_e32 v8, vcc, v0, v6
-; GISEL-NEXT:    v_sub_i32_e32 v9, vcc, v1, v7
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v8, v2
-; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v8, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v6
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[6:7], v8, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[6:7], v9, v3
-; GISEL-NEXT:    v_add_i32_e64 v2, s[8:9], v9, v3
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v7
-; GISEL-NEXT:    v_sub_i32_e64 v1, s[10:11], v9, v3
-; GISEL-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GISEL-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v10, v0, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[8:9]
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v7
+; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, v0, v2
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, v1, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, v0, v2
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, v1, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v4
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v5
 ; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
@@ -805,77 +661,59 @@ define <2 x i32> @v_srem_v2i32_pow2_shl_denom(<2 x i32> %x, <2 x i32> %y) {
 ; CGP-NEXT:    v_ashrrev_i32_e32 v7, 31, v3
 ; CGP-NEXT:    v_xor_b32_e32 v1, v1, v5
 ; CGP-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
-; CGP-NEXT:    v_mul_lo_u32 v8, 0, v0
+; CGP-NEXT:    v_mul_lo_u32 v8, v0, 0
 ; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; CGP-NEXT:    v_mul_lo_u32 v9, 0, v1
+; CGP-NEXT:    v_mul_lo_u32 v9, v1, 0
 ; CGP-NEXT:    v_xor_b32_e32 v2, v2, v6
 ; CGP-NEXT:    v_xor_b32_e32 v3, v3, v7
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v6, v2
-; CGP-NEXT:    v_mul_lo_u32 v7, 0, v2
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, 0, v2
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v10, v3
-; CGP-NEXT:    v_mul_lo_u32 v11, 0, v3
+; CGP-NEXT:    v_sub_i32_e32 v11, vcc, 0, v3
 ; CGP-NEXT:    v_rcp_f32_e32 v6, v6
 ; CGP-NEXT:    v_rcp_f32_e32 v10, v10
-; CGP-NEXT:    v_mul_f32_e32 v6, 0x4f800000, v6
-; CGP-NEXT:    v_mul_f32_e32 v10, 0x4f800000, v10
+; CGP-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; CGP-NEXT:    v_mul_f32_e32 v10, 0x4f7ffffe, v10
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v10, v10
-; CGP-NEXT:    v_mul_lo_u32 v12, v6, v2
-; CGP-NEXT:    v_mul_lo_u32 v13, v6, 0
-; CGP-NEXT:    v_mul_hi_u32 v14, v6, v2
-; CGP-NEXT:    v_mul_lo_u32 v15, v10, v3
-; CGP-NEXT:    v_mul_lo_u32 v16, v10, 0
-; CGP-NEXT:    v_mul_hi_u32 v17, v10, v3
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v13
-; CGP-NEXT:    v_sub_i32_e32 v18, vcc, 0, v12
-; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v16
-; CGP-NEXT:    v_sub_i32_e32 v19, vcc, 0, v15
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v14
-; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v17
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; CGP-NEXT:    v_cndmask_b32_e32 v7, v12, v18, vcc
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v11
-; CGP-NEXT:    v_cndmask_b32_e64 v11, v15, v19, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v12, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v7, v7, v6
-; CGP-NEXT:    v_mul_lo_u32 v14, v11, 0
-; CGP-NEXT:    v_mul_hi_u32 v11, v11, v10
-; CGP-NEXT:    v_add_i32_e64 v12, s[6:7], v13, v12
-; CGP-NEXT:    v_add_i32_e64 v13, s[6:7], v16, v14
-; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v12, v7
-; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v13, v11
-; CGP-NEXT:    v_add_i32_e64 v12, s[6:7], v6, v7
-; CGP-NEXT:    v_sub_i32_e64 v6, s[6:7], v6, v7
-; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v10, v11
-; CGP-NEXT:    v_sub_i32_e64 v10, s[6:7], v10, v11
-; CGP-NEXT:    v_cndmask_b32_e32 v6, v6, v12, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v7, v10, v7, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v10, v6, 0
-; CGP-NEXT:    v_mul_hi_u32 v6, v6, v0
-; CGP-NEXT:    v_mul_lo_u32 v11, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v7, v7, v1
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_mul_lo_u32 v7, v7, v6
+; CGP-NEXT:    v_mul_lo_u32 v12, v6, 0
+; CGP-NEXT:    v_mul_lo_u32 v11, v11, v10
+; CGP-NEXT:    v_mul_lo_u32 v13, v10, 0
+; CGP-NEXT:    v_mul_lo_u32 v14, 0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v6, v7
+; CGP-NEXT:    v_mul_lo_u32 v15, 0, v11
+; CGP-NEXT:    v_mul_hi_u32 v11, v10, v11
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v14, v12
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v15, v13
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v12, v7
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v10, v11
+; CGP-NEXT:    v_mul_lo_u32 v10, 0, v6
+; CGP-NEXT:    v_mul_hi_u32 v6, v0, v6
+; CGP-NEXT:    v_mul_lo_u32 v11, 0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v1, v7
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
 ; CGP-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
 ; CGP-NEXT:    v_add_i32_e32 v7, vcc, v9, v7
 ; CGP-NEXT:    v_mul_lo_u32 v6, v6, v2
 ; CGP-NEXT:    v_mul_lo_u32 v7, v7, v3
-; CGP-NEXT:    v_sub_i32_e32 v8, vcc, v0, v6
-; CGP-NEXT:    v_sub_i32_e32 v9, vcc, v1, v7
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v8, v2
-; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v8, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v6
-; CGP-NEXT:    v_sub_i32_e64 v0, s[6:7], v8, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v9, v3
-; CGP-NEXT:    v_add_i32_e64 v2, s[8:9], v9, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v7
-; CGP-NEXT:    v_sub_i32_e64 v1, s[10:11], v9, v3
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; CGP-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v10, v0, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[8:9]
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v7
+; CGP-NEXT:    v_sub_i32_e32 v6, vcc, v0, v2
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
+; CGP-NEXT:    v_sub_i32_e32 v6, vcc, v0, v2
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
 ; CGP-NEXT:    v_xor_b32_e32 v0, v0, v4
 ; CGP-NEXT:    v_xor_b32_e32 v1, v1, v5
 ; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
@@ -900,28 +738,22 @@ define i32 @v_srem_i32_24bit(i32 %num, i32 %den) {
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v3
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v3, v1
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, 0, v1
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; GISEL-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; GISEL-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GISEL-NEXT:    v_mul_lo_u32 v4, v3, v1
-; GISEL-NEXT:    v_mul_hi_u32 v5, v3, v1
-; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v4, v4, v3
-; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v3, v4
-; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v3, v3, v0
+; GISEL-NEXT:    v_mul_lo_u32 v4, v4, v3
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; GISEL-NEXT:    v_mul_hi_u32 v3, v0, v3
 ; GISEL-NEXT:    v_mul_lo_u32 v3, v3, v1
-; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v0, v3
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v1
-; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v4, v1
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v3
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[6:7], v4, v1
-; GISEL-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, v0, v1
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, v0, v1
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
@@ -933,39 +765,30 @@ define i32 @v_srem_i32_24bit(i32 %num, i32 %den) {
 ; CGP-NEXT:    v_and_b32_e32 v0, s4, v0
 ; CGP-NEXT:    v_and_b32_e32 v1, s4, v1
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v2, v1
-; CGP-NEXT:    v_mul_lo_u32 v3, 0, v1
-; CGP-NEXT:    v_mul_lo_u32 v4, 0, v0
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; CGP-NEXT:    v_mul_lo_u32 v4, v0, 0
 ; CGP-NEXT:    v_rcp_f32_e32 v2, v2
-; CGP-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; CGP-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, v2, v1
-; CGP-NEXT:    v_mul_lo_u32 v6, v2, 0
-; CGP-NEXT:    v_mul_hi_u32 v7, v2, v1
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; CGP-NEXT:    v_sub_i32_e32 v8, vcc, 0, v5
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; CGP-NEXT:    v_cndmask_b32_e32 v3, v5, v8, vcc
-; CGP-NEXT:    v_mul_lo_u32 v5, v3, 0
-; CGP-NEXT:    v_mul_hi_u32 v3, v3, v2
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v6, v5
-; CGP-NEXT:    v_add_i32_e64 v3, s[4:5], v5, v3
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v2, v3
-; CGP-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; CGP-NEXT:    v_mul_lo_u32 v3, v2, 0
-; CGP-NEXT:    v_mul_hi_u32 v2, v2, v0
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; CGP-NEXT:    v_mul_lo_u32 v3, v3, v2
+; CGP-NEXT:    v_mul_lo_u32 v5, v2, 0
+; CGP-NEXT:    v_mul_lo_u32 v6, 0, v3
+; CGP-NEXT:    v_mul_hi_u32 v3, v2, v3
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT:    v_mul_lo_u32 v3, 0, v2
+; CGP-NEXT:    v_mul_hi_u32 v2, v0, v2
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
 ; CGP-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; CGP-NEXT:    v_mul_lo_u32 v2, v2, v1
-; CGP-NEXT:    v_sub_i32_e32 v3, vcc, v0, v2
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v1
-; CGP-NEXT:    v_add_i32_e64 v4, s[4:5], v3, v1
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v2
-; CGP-NEXT:    v_sub_i32_e64 v0, s[6:7], v3, v1
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[4:5]
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %num.mask = and i32 %num, 16777215
   %den.mask = and i32 %den, 16777215
@@ -995,51 +818,39 @@ define <2 x i32> @v_srem_v2i32_24bit(<2 x i32> %num, <2 x i32> %den) {
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v6
 ; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v7
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v2
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v7, v3
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v2
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v8, v3
+; GISEL-NEXT:    v_sub_i32_e32 v9, vcc, 0, v3
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; GISEL-NEXT:    v_rcp_iflag_f32_e32 v7, v7
-; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v5
-; GISEL-NEXT:    v_mul_f32_e32 v7, 0x4f800000, v7
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v8, v8
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f7ffffe, v5
+; GISEL-NEXT:    v_mul_f32_e32 v8, 0x4f7ffffe, v8
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; GISEL-NEXT:    v_mul_lo_u32 v8, v5, v2
-; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v2
-; GISEL-NEXT:    v_mul_lo_u32 v10, v7, v3
-; GISEL-NEXT:    v_mul_hi_u32 v11, v7, v3
-; GISEL-NEXT:    v_sub_i32_e32 v12, vcc, 0, v8
-; GISEL-NEXT:    v_sub_i32_e32 v13, vcc, 0, v10
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v9
-; GISEL-NEXT:    v_cndmask_b32_e32 v8, v8, v12, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v11
-; GISEL-NEXT:    v_cndmask_b32_e64 v9, v10, v13, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v8, v8, v5
-; GISEL-NEXT:    v_mul_hi_u32 v9, v9, v7
-; GISEL-NEXT:    v_add_i32_e64 v10, s[6:7], v5, v8
-; GISEL-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v8
-; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v7, v9
-; GISEL-NEXT:    v_sub_i32_e64 v7, s[6:7], v7, v9
-; GISEL-NEXT:    v_cndmask_b32_e32 v5, v5, v10, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v0
-; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v1
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v5
+; GISEL-NEXT:    v_mul_lo_u32 v9, v9, v8
+; GISEL-NEXT:    v_mul_hi_u32 v7, v5, v7
+; GISEL-NEXT:    v_mul_hi_u32 v9, v8, v9
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v9
+; GISEL-NEXT:    v_mul_hi_u32 v5, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v7, v1, v7
 ; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v2
 ; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v3
-; GISEL-NEXT:    v_sub_i32_e32 v8, vcc, v0, v5
-; GISEL-NEXT:    v_sub_i32_e32 v9, vcc, v1, v7
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v8, v2
-; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v8, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v5
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[6:7], v8, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[6:7], v9, v3
-; GISEL-NEXT:    v_add_i32_e64 v2, s[8:9], v9, v3
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v7
-; GISEL-NEXT:    v_sub_i32_e64 v1, s[10:11], v9, v3
-; GISEL-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GISEL-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v10, v0, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[8:9]
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v7
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, v0, v2
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, v1, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, v0, v2
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, v1, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v4
 ; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v6
 ; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
@@ -1055,73 +866,55 @@ define <2 x i32> @v_srem_v2i32_24bit(<2 x i32> %num, <2 x i32> %den) {
 ; CGP-NEXT:    v_and_b32_e32 v2, s4, v2
 ; CGP-NEXT:    v_and_b32_e32 v3, s4, v3
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, 0, v2
-; CGP-NEXT:    v_mul_lo_u32 v6, 0, v0
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; CGP-NEXT:    v_mul_lo_u32 v6, v0, 0
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v7, v3
-; CGP-NEXT:    v_mul_lo_u32 v8, 0, v3
-; CGP-NEXT:    v_mul_lo_u32 v9, 0, v1
+; CGP-NEXT:    v_sub_i32_e32 v8, vcc, 0, v3
+; CGP-NEXT:    v_mul_lo_u32 v9, v1, 0
 ; CGP-NEXT:    v_rcp_f32_e32 v4, v4
 ; CGP-NEXT:    v_rcp_f32_e32 v7, v7
-; CGP-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f800000, v7
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; CGP-NEXT:    v_mul_lo_u32 v10, v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v11, v4, 0
-; CGP-NEXT:    v_mul_hi_u32 v12, v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v13, v7, v3
-; CGP-NEXT:    v_mul_lo_u32 v14, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v15, v7, v3
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v11
-; CGP-NEXT:    v_sub_i32_e32 v16, vcc, 0, v10
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v14
-; CGP-NEXT:    v_sub_i32_e32 v17, vcc, 0, v13
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v12
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v15
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; CGP-NEXT:    v_cndmask_b32_e32 v5, v10, v16, vcc
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
-; CGP-NEXT:    v_cndmask_b32_e64 v8, v13, v17, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v10, v5, 0
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v4
-; CGP-NEXT:    v_mul_lo_u32 v12, v8, 0
-; CGP-NEXT:    v_mul_hi_u32 v8, v8, v7
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v11, v10
-; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v14, v12
-; CGP-NEXT:    v_add_i32_e64 v5, s[6:7], v10, v5
-; CGP-NEXT:    v_add_i32_e64 v8, s[6:7], v11, v8
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v4, v5
-; CGP-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v5
-; CGP-NEXT:    v_add_i32_e64 v5, s[6:7], v7, v8
-; CGP-NEXT:    v_sub_i32_e64 v7, s[6:7], v7, v8
-; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v7, v4, 0
-; CGP-NEXT:    v_mul_hi_u32 v4, v4, v0
-; CGP-NEXT:    v_mul_lo_u32 v8, v5, 0
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v1
-; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v9, v8
+; CGP-NEXT:    v_mul_lo_u32 v5, v5, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v4, 0
+; CGP-NEXT:    v_mul_lo_u32 v8, v8, v7
+; CGP-NEXT:    v_mul_lo_u32 v11, v7, 0
+; CGP-NEXT:    v_mul_lo_u32 v12, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v4, v5
+; CGP-NEXT:    v_mul_lo_u32 v13, 0, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v7, v8
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v10, v5
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v7, v8
+; CGP-NEXT:    v_mul_lo_u32 v7, 0, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v0, v4
+; CGP-NEXT:    v_mul_lo_u32 v8, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v1, v5
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v8, v9
 ; CGP-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
 ; CGP-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
 ; CGP-NEXT:    v_mul_lo_u32 v4, v4, v2
 ; CGP-NEXT:    v_mul_lo_u32 v5, v5, v3
-; CGP-NEXT:    v_sub_i32_e32 v6, vcc, v0, v4
-; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v1, v5
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v6, v2
-; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v6, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v4
-; CGP-NEXT:    v_sub_i32_e64 v0, s[6:7], v6, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v7, v3
-; CGP-NEXT:    v_add_i32_e64 v2, s[8:9], v7, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v5
-; CGP-NEXT:    v_sub_i32_e64 v1, s[10:11], v7, v3
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; CGP-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[8:9]
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, v1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, v1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %num.mask = and <2 x i32> %num, <i32 16777215, i32 16777215>
   %den.mask = and <2 x i32> %den, <i32 16777215, i32 16777215>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
new file mode 100644
index 0000000000000..438388ebf7136
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
@@ -0,0 +1,3701 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=1 -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=CHECK,GISEL %s
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=CHECK,CGP %s
+
+; The same 32-bit expansion is implemented in the legalizer and in AMDGPUCodeGenPrepare.
+
+define i64 @v_srem_i64(i64 %num, i64 %den) {
+; CHECK-LABEL: v_srem_i64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v5, v1, v3
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0
+; CHECK-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CHECK-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CHECK-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CHECK-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CHECK-NEXT:    s_cbranch_execz BB0_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v2, v4
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v3, v3, v4
+; CHECK-NEXT:    v_xor_b32_e32 v5, v5, v4
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v4, v5
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v6, v3
+; CHECK-NEXT:    v_ashrrev_i32_e32 v7, 31, v1
+; CHECK-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v6
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v0, v7
+; CHECK-NEXT:    v_addc_u32_e32 v1, vcc, v1, v7, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v9, vcc, 0, v5
+; CHECK-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CHECK-NEXT:    v_mul_f32_e32 v8, 0x2f800000, v4
+; CHECK-NEXT:    v_trunc_f32_e32 v8, v8
+; CHECK-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v8
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; CHECK-NEXT:    v_subb_u32_e32 v10, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v6, v6, v7
+; CHECK-NEXT:    v_mul_lo_u32 v11, v10, v4
+; CHECK-NEXT:    v_mul_lo_u32 v12, v9, v8
+; CHECK-NEXT:    v_mul_hi_u32 v14, v9, v4
+; CHECK-NEXT:    v_mul_lo_u32 v13, v9, v4
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v7
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, v11, v14
+; CHECK-NEXT:    v_mul_lo_u32 v12, v8, v13
+; CHECK-NEXT:    v_mul_lo_u32 v14, v4, v11
+; CHECK-NEXT:    v_mul_hi_u32 v15, v4, v13
+; CHECK-NEXT:    v_mul_hi_u32 v13, v8, v13
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; CHECK-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; CHECK-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v15, v8, v11
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, v14, v12
+; CHECK-NEXT:    v_mul_hi_u32 v14, v4, v11
+; CHECK-NEXT:    v_mul_hi_u32 v11, v8, v11
+; CHECK-NEXT:    v_add_i32_e32 v13, vcc, v15, v13
+; CHECK-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; CHECK-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CHECK-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v13, vcc, v14, v13
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, v11, v13
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v12
+; CHECK-NEXT:    v_addc_u32_e64 v12, s[4:5], v8, v11, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v10, v10, v4
+; CHECK-NEXT:    v_mul_lo_u32 v13, v9, v12
+; CHECK-NEXT:    v_mul_lo_u32 v14, v9, v4
+; CHECK-NEXT:    v_mul_hi_u32 v9, v9, v4
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; CHECK-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; CHECK-NEXT:    v_mul_hi_u32 v11, v4, v14
+; CHECK-NEXT:    v_add_i32_e64 v9, s[4:5], v10, v9
+; CHECK-NEXT:    v_mul_lo_u32 v10, v12, v14
+; CHECK-NEXT:    v_mul_lo_u32 v13, v4, v9
+; CHECK-NEXT:    v_mul_hi_u32 v14, v12, v14
+; CHECK-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; CHECK-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CHECK-NEXT:    v_mul_lo_u32 v11, v12, v9
+; CHECK-NEXT:    v_add_i32_e64 v10, s[4:5], v13, v10
+; CHECK-NEXT:    v_mul_hi_u32 v13, v4, v9
+; CHECK-NEXT:    v_mul_hi_u32 v9, v12, v9
+; CHECK-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; CHECK-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; CHECK-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; CHECK-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v11, s[4:5], v13, v11
+; CHECK-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v11
+; CHECK-NEXT:    v_addc_u32_e32 v8, vcc, v8, v9, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; CHECK-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v9, v1, v4
+; CHECK-NEXT:    v_mul_lo_u32 v10, v6, v8
+; CHECK-NEXT:    v_mul_hi_u32 v11, v6, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v1, v4
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v11, v1, v8
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CHECK-NEXT:    v_mul_hi_u32 v10, v6, v8
+; CHECK-NEXT:    v_mul_hi_u32 v8, v1, v8
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v11, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CHECK-NEXT:    v_mul_lo_u32 v9, v3, v4
+; CHECK-NEXT:    v_mul_lo_u32 v8, v5, v8
+; CHECK-NEXT:    v_mul_lo_u32 v10, v5, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v5, v4
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
+; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v6, v10
+; CHECK-NEXT:    v_subb_u32_e64 v8, s[4:5], v1, v4, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v4
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], v8, v3
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, v4, v9, s[4:5]
+; CHECK-NEXT:    v_sub_i32_e32 v9, vcc, v6, v5
+; CHECK-NEXT:    v_subbrev_u32_e64 v10, s[4:5], 0, v1, vcc
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v5
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], v10, v3
+; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, v9, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, v11, v12, s[4:5]
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v3, v3, v7
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v7
+; CHECK-NEXT:    v_sub_i32_e32 v4, vcc, v3, v7
+; CHECK-NEXT:    v_subb_u32_e32 v5, vcc, v1, v7, vcc
+; CHECK-NEXT:  BB0_2: ; %Flow
+; CHECK-NEXT:    s_or_saveexec_b64 s[4:5], s[6:7]
+; CHECK-NEXT:    s_xor_b64 exec, exec, s[4:5]
+; CHECK-NEXT:    s_cbranch_execz BB0_4
+; CHECK-NEXT:  ; %bb.3:
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, v2
+; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, 0, v2
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; CHECK-NEXT:    v_mul_lo_u32 v3, v3, v1
+; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CHECK-NEXT:    v_mul_lo_u32 v1, v1, v2
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, v0, v2
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, v0, v2
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, v0, v1, vcc
+; CHECK-NEXT:  BB0_4:
+; CHECK-NEXT:    s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT:    v_mov_b32_e32 v0, v4
+; CHECK-NEXT:    v_mov_b32_e32 v1, v5
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %result = srem i64 %num, %den
+  ret i64 %result
+}
+
+; FIXME: This is a workaround for not handling uniform VGPR case.
+declare i32 @llvm.amdgcn.readfirstlane(i32)
+
+define amdgpu_ps i64 @s_srem_i64(i64 inreg %num, i64 inreg %den) {
+; CHECK-LABEL: s_srem_i64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_or_b64 s[0:1], s[2:3], s[4:5]
+; CHECK-NEXT:    s_mov_b32 s6, 0
+; CHECK-NEXT:    s_mov_b32 s7, -1
+; CHECK-NEXT:    s_and_b64 s[0:1], s[0:1], s[6:7]
+; CHECK-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[0:1], 0
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[8:9], 0, 1
+; CHECK-NEXT:    s_mov_b32 s0, 1
+; CHECK-NEXT:    s_xor_b64 vcc, s[6:7], s[8:9]
+; CHECK-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; CHECK-NEXT:    s_cbranch_vccz BB1_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    s_ashr_i32 s0, s5, 31
+; CHECK-NEXT:    s_ashr_i32 s6, s3, 31
+; CHECK-NEXT:    s_add_u32 s8, s2, s6
+; CHECK-NEXT:    s_cselect_b32 s7, 1, 0
+; CHECK-NEXT:    s_and_b32 s7, s7, 1
+; CHECK-NEXT:    s_cmp_lg_u32 s7, 0
+; CHECK-NEXT:    s_addc_u32 s9, s3, s6
+; CHECK-NEXT:    s_add_u32 s10, s4, s0
+; CHECK-NEXT:    s_cselect_b32 s3, 1, 0
+; CHECK-NEXT:    s_and_b32 s3, s3, 1
+; CHECK-NEXT:    s_cmp_lg_u32 s3, 0
+; CHECK-NEXT:    s_mov_b32 s1, s0
+; CHECK-NEXT:    s_addc_u32 s11, s5, s0
+; CHECK-NEXT:    s_xor_b64 s[10:11], s[10:11], s[0:1]
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v0, s10
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, s11
+; CHECK-NEXT:    s_mov_b32 s7, s6
+; CHECK-NEXT:    s_xor_b64 s[8:9], s[8:9], s[6:7]
+; CHECK-NEXT:    s_sub_u32 s3, 0, s10
+; CHECK-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT:    s_cselect_b32 s0, 1, 0
+; CHECK-NEXT:    s_and_b32 s0, s0, 1
+; CHECK-NEXT:    s_cmp_lg_u32 s0, 0
+; CHECK-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; CHECK-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; CHECK-NEXT:    v_trunc_f32_e32 v1, v1
+; CHECK-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; CHECK-NEXT:    s_subb_u32 s5, 0, s11
+; CHECK-NEXT:    v_mul_lo_u32 v2, s5, v0
+; CHECK-NEXT:    v_mul_lo_u32 v3, s3, v1
+; CHECK-NEXT:    v_mul_hi_u32 v5, s3, v0
+; CHECK-NEXT:    v_mul_lo_u32 v4, s3, v0
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_mul_lo_u32 v3, v1, v4
+; CHECK-NEXT:    v_mul_lo_u32 v5, v0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v6, v0, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v1, v4
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v6, v1, v2
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
+; CHECK-NEXT:    v_mul_hi_u32 v5, v0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT:    v_addc_u32_e64 v3, s[0:1], v1, v2, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v4, s5, v0
+; CHECK-NEXT:    v_mul_lo_u32 v5, s3, v3
+; CHECK-NEXT:    v_mul_hi_u32 v7, s3, v0
+; CHECK-NEXT:    v_mul_lo_u32 v6, s3, v0
+; CHECK-NEXT:    v_add_i32_e64 v1, s[0:1], v1, v2
+; CHECK-NEXT:    v_add_i32_e64 v4, s[0:1], v4, v5
+; CHECK-NEXT:    v_add_i32_e64 v4, s[0:1], v4, v7
+; CHECK-NEXT:    v_mul_lo_u32 v5, v3, v6
+; CHECK-NEXT:    v_mul_lo_u32 v7, v0, v4
+; CHECK-NEXT:    v_mul_hi_u32 v2, v0, v6
+; CHECK-NEXT:    v_mul_hi_u32 v6, v3, v6
+; CHECK-NEXT:    v_add_i32_e64 v5, s[0:1], v5, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[0:1]
+; CHECK-NEXT:    v_add_i32_e64 v2, s[0:1], v5, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; CHECK-NEXT:    v_mul_lo_u32 v5, v3, v4
+; CHECK-NEXT:    v_add_i32_e64 v2, s[0:1], v7, v2
+; CHECK-NEXT:    v_mul_hi_u32 v7, v0, v4
+; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v4
+; CHECK-NEXT:    v_add_i32_e64 v5, s[0:1], v5, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[0:1]
+; CHECK-NEXT:    v_add_i32_e64 v5, s[0:1], v5, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[0:1]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[0:1], v6, v7
+; CHECK-NEXT:    v_add_i32_e64 v2, s[0:1], v5, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s[0:1]
+; CHECK-NEXT:    v_add_i32_e64 v4, s[0:1], v6, v5
+; CHECK-NEXT:    v_add_i32_e64 v3, s[0:1], v3, v4
+; CHECK-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; CHECK-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v2, s9, v0
+; CHECK-NEXT:    v_mul_lo_u32 v3, s8, v1
+; CHECK-NEXT:    v_mul_hi_u32 v4, s8, v0
+; CHECK-NEXT:    v_mul_hi_u32 v0, s9, v0
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v4, s9, v1
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT:    v_mul_hi_u32 v3, s8, v1
+; CHECK-NEXT:    v_mul_hi_u32 v1, s9, v1
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v2, s11, v0
+; CHECK-NEXT:    v_mul_lo_u32 v1, s10, v1
+; CHECK-NEXT:    v_mul_lo_u32 v3, s10, v0
+; CHECK-NEXT:    v_mul_hi_u32 v0, s10, v0
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, s8, v3
+; CHECK-NEXT:    v_mov_b32_e32 v1, s9
+; CHECK-NEXT:    v_subb_u32_e64 v1, s[0:1], v1, v0, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v0, s[0:1], s9, v0
+; CHECK-NEXT:    v_mov_b32_e32 v3, s11
+; CHECK-NEXT:    v_subb_u32_e32 v0, vcc, v0, v3, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v3, vcc, s10, v2
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[0:1], s11, v1
+; CHECK-NEXT:    v_subbrev_u32_e32 v0, vcc, 0, v0, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[0:1]
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[0:1], s10, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
+; CHECK-NEXT:    v_cmp_eq_u32_e64 s[0:1], s11, v1
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s11, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v1, v4, v5, s[0:1]
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s10, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, s11, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v4, v5, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v4, vcc, s10, v3
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v3, v4, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v0, s6, v0
+; CHECK-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v0
+; CHECK-NEXT:    s_mov_b32 s0, 0
+; CHECK-NEXT:  BB1_2: ; %Flow
+; CHECK-NEXT:    s_and_b32 s0, s0, 1
+; CHECK-NEXT:    s_cmp_lg_u32 s0, 0
+; CHECK-NEXT:    s_cbranch_scc0 BB1_4
+; CHECK-NEXT:  ; %bb.3:
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v0, s4
+; CHECK-NEXT:    s_sub_i32 s0, 0, s4
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT:    v_mul_lo_u32 v1, s0, v0
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT:    v_mul_hi_u32 v0, s2, v0
+; CHECK-NEXT:    v_mul_lo_u32 v0, v0, s4
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
+; CHECK-NEXT:    v_subrev_i32_e32 v1, vcc, s4, v0
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v1, vcc, s4, v0
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CHECK-NEXT:  BB1_4:
+; CHECK-NEXT:    v_readfirstlane_b32 s0, v0
+; CHECK-NEXT:    s_mov_b32 s1, s0
+; CHECK-NEXT:    ; return to shader part epilog
+  %result = srem i64 %num, %den
+  %cast = bitcast i64 %result to <2 x i32>
+  %elt.0 = extractelement <2 x i32> %cast, i32 0
+  %elt.1 = extractelement <2 x i32> %cast, i32 1
+  %res.0 = call i32 @llvm.amdgcn.readfirstlane(i32 %elt.0)
+  %res.1 = call i32 @llvm.amdgcn.readfirstlane(i32 %elt.1)
+  %ins.0 = insertelement <2 x i32> undef, i32 %res.0, i32 0
+  %ins.1 = insertelement <2 x i32> %ins.0, i32 %res.0, i32 1
+  %cast.back = bitcast <2 x i32> %ins.1 to i64
+  ret i64 %cast.back
+}
+
+define <2 x i64> @v_srem_v2i64(<2 x i64> %num, <2 x i64> %den) {
+; GISEL-LABEL: v_srem_v2i64:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    v_ashrrev_i32_e32 v8, 31, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v8, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v5, v5, v8
+; GISEL-NEXT:    v_xor_b32_e32 v4, v4, v8
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v8, v4
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v9, v5
+; GISEL-NEXT:    v_ashrrev_i32_e32 v10, 31, v1
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v10, vcc
+; GISEL-NEXT:    v_mac_f32_e32 v8, 0x4f800000, v9
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v8, v8
+; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, 0, v4
+; GISEL-NEXT:    v_subb_u32_e32 v12, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v10
+; GISEL-NEXT:    v_mul_f32_e32 v8, 0x5f7ffffc, v8
+; GISEL-NEXT:    v_mul_f32_e32 v9, 0x2f800000, v8
+; GISEL-NEXT:    v_trunc_f32_e32 v9, v9
+; GISEL-NEXT:    v_mac_f32_e32 v8, 0xcf800000, v9
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v9, v9
+; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v10
+; GISEL-NEXT:    v_mul_lo_u32 v13, v12, v8
+; GISEL-NEXT:    v_mul_lo_u32 v14, v11, v9
+; GISEL-NEXT:    v_mul_hi_u32 v16, v11, v8
+; GISEL-NEXT:    v_mul_lo_u32 v15, v11, v8
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
+; GISEL-NEXT:    v_mul_lo_u32 v14, v9, v15
+; GISEL-NEXT:    v_mul_lo_u32 v16, v8, v13
+; GISEL-NEXT:    v_mul_hi_u32 v17, v8, v15
+; GISEL-NEXT:    v_mul_hi_u32 v15, v9, v15
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v17, v9, v13
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v16, v14
+; GISEL-NEXT:    v_mul_hi_u32 v16, v8, v13
+; GISEL-NEXT:    v_mul_hi_u32 v13, v9, v13
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v17, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v15, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v16, vcc, v17, v16
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v16, v15
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v14
+; GISEL-NEXT:    v_addc_u32_e64 v14, s[4:5], v9, v13, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v12, v8
+; GISEL-NEXT:    v_mul_lo_u32 v15, v11, v14
+; GISEL-NEXT:    v_mul_lo_u32 v16, v11, v8
+; GISEL-NEXT:    v_mul_hi_u32 v11, v11, v8
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v13
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
+; GISEL-NEXT:    v_mul_hi_u32 v13, v8, v16
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_mul_lo_u32 v12, v14, v16
+; GISEL-NEXT:    v_mul_lo_u32 v15, v8, v11
+; GISEL-NEXT:    v_mul_hi_u32 v16, v14, v16
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v13, v14, v11
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v15, v12
+; GISEL-NEXT:    v_mul_hi_u32 v15, v8, v11
+; GISEL-NEXT:    v_mul_hi_u32 v11, v14, v11
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v15, s[4:5], v16, v15
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v13, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v15, v13
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_addc_u32_e32 v9, vcc, v9, v11, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v11, v1, v8
+; GISEL-NEXT:    v_mul_lo_u32 v12, v0, v9
+; GISEL-NEXT:    v_mul_hi_u32 v13, v0, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, v1, v8
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v13, v1, v9
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_mul_hi_u32 v12, v0, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v1, v9
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v13, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v8
+; GISEL-NEXT:    v_mul_lo_u32 v9, v4, v9
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, v4, v8
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v12
+; GISEL-NEXT:    v_subb_u32_e64 v9, s[4:5], v1, v8, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v8
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v9, v5
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, v8, v11, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, v0, v4
+; GISEL-NEXT:    v_subbrev_u32_e64 v12, s[4:5], 0, v1, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v12, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v11, v4
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v11, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v12, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, v13, v14, s[4:5]
+; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v13
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v11, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v12, v1, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_ashrrev_i32_e32 v4, 31, v7
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v6, v4
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, v7, v4, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v5, v5, v4
+; GISEL-NEXT:    v_xor_b32_e32 v4, v6, v4
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v6, v5
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v7, v4
+; GISEL-NEXT:    v_ashrrev_i32_e32 v8, 31, v3
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v8
+; GISEL-NEXT:    v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GISEL-NEXT:    v_mac_f32_e32 v6, 0x4f800000, v7
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GISEL-NEXT:    v_sub_i32_e32 v9, vcc, 0, v5
+; GISEL-NEXT:    v_subb_u32_e32 v11, vcc, 0, v4, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v10
+; GISEL-NEXT:    v_mul_f32_e32 v6, 0x5f7ffffc, v6
+; GISEL-NEXT:    v_mul_f32_e32 v7, 0x2f800000, v6
+; GISEL-NEXT:    v_trunc_f32_e32 v7, v7
+; GISEL-NEXT:    v_mac_f32_e32 v6, 0xcf800000, v7
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v8
+; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v8
+; GISEL-NEXT:    v_mul_lo_u32 v12, v11, v6
+; GISEL-NEXT:    v_mul_lo_u32 v13, v9, v7
+; GISEL-NEXT:    v_mul_hi_u32 v15, v9, v6
+; GISEL-NEXT:    v_mul_lo_u32 v14, v9, v6
+; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v10
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; GISEL-NEXT:    v_mul_lo_u32 v13, v7, v14
+; GISEL-NEXT:    v_mul_lo_u32 v15, v6, v12
+; GISEL-NEXT:    v_mul_hi_u32 v16, v6, v14
+; GISEL-NEXT:    v_mul_hi_u32 v14, v7, v14
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v16, v7, v12
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v15, v13
+; GISEL-NEXT:    v_mul_hi_u32 v15, v6, v12
+; GISEL-NEXT:    v_mul_hi_u32 v12, v7, v12
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v16, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v16, v15
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v14, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v13
+; GISEL-NEXT:    v_addc_u32_e64 v13, s[4:5], v7, v12, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v11, v11, v6
+; GISEL-NEXT:    v_mul_lo_u32 v14, v9, v13
+; GISEL-NEXT:    v_mul_lo_u32 v15, v9, v6
+; GISEL-NEXT:    v_mul_hi_u32 v9, v9, v6
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v12
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; GISEL-NEXT:    v_mul_hi_u32 v12, v6, v15
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v11, v9
+; GISEL-NEXT:    v_mul_lo_u32 v11, v13, v15
+; GISEL-NEXT:    v_mul_lo_u32 v14, v6, v9
+; GISEL-NEXT:    v_mul_hi_u32 v15, v13, v15
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v12, v13, v9
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v14, v11
+; GISEL-NEXT:    v_mul_hi_u32 v14, v6, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v13, v9
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[4:5], v15, v14
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v14, v12
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_addc_u32_e32 v7, vcc, v7, v9, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v11
+; GISEL-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT:    v_mul_lo_u32 v9, v3, v6
+; GISEL-NEXT:    v_mul_lo_u32 v11, v2, v7
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v10, vcc
+; GISEL-NEXT:    v_mul_hi_u32 v10, v2, v6
+; GISEL-NEXT:    v_mul_hi_u32 v6, v3, v6
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v10, v3, v7
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_mul_hi_u32 v11, v2, v7
+; GISEL-NEXT:    v_mul_hi_u32 v7, v3, v7
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT:    v_mul_lo_u32 v9, v4, v6
+; GISEL-NEXT:    v_mul_lo_u32 v7, v5, v7
+; GISEL-NEXT:    v_mul_lo_u32 v10, v5, v6
+; GISEL-NEXT:    v_mul_hi_u32 v6, v5, v6
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v9, v7
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v10
+; GISEL-NEXT:    v_subb_u32_e64 v7, s[4:5], v3, v6, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v6
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v7, v4
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e32 v9, vcc, v2, v5
+; GISEL-NEXT:    v_subbrev_u32_e64 v10, s[4:5], 0, v3, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v5
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v10, v4
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v9, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, v11, v12, s[4:5]
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v8
+; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v8
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v8
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v8, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_srem_v2i64:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    v_mov_b32_e32 v9, v1
+; CGP-NEXT:    v_mov_b32_e32 v8, v0
+; CGP-NEXT:    v_or_b32_e32 v1, v9, v5
+; CGP-NEXT:    v_mov_b32_e32 v0, 0
+; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CGP-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CGP-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; CGP-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CGP-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execz BB2_2
+; CGP-NEXT:  ; %bb.1:
+; CGP-NEXT:    v_ashrrev_i32_e32 v0, 31, v5
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, v4, v0
+; CGP-NEXT:    v_addc_u32_e32 v5, vcc, v5, v0, vcc
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v0
+; CGP-NEXT:    v_xor_b32_e32 v0, v5, v0
+; CGP-NEXT:    v_cvt_f32_u32_e32 v5, v1
+; CGP-NEXT:    v_cvt_f32_u32_e32 v10, v0
+; CGP-NEXT:    v_ashrrev_i32_e32 v11, 31, v9
+; CGP-NEXT:    v_mac_f32_e32 v5, 0x4f800000, v10
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v5, v5
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v8, v11
+; CGP-NEXT:    v_addc_u32_e32 v9, vcc, v9, v11, vcc
+; CGP-NEXT:    v_sub_i32_e32 v13, vcc, 0, v1
+; CGP-NEXT:    v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; CGP-NEXT:    v_mul_f32_e32 v12, 0x2f800000, v5
+; CGP-NEXT:    v_trunc_f32_e32 v12, v12
+; CGP-NEXT:    v_mac_f32_e32 v5, 0xcf800000, v12
+; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CGP-NEXT:    v_cvt_u32_f32_e32 v12, v12
+; CGP-NEXT:    v_subb_u32_e32 v14, vcc, 0, v0, vcc
+; CGP-NEXT:    v_xor_b32_e32 v10, v10, v11
+; CGP-NEXT:    v_mul_lo_u32 v15, v14, v5
+; CGP-NEXT:    v_mul_lo_u32 v16, v13, v12
+; CGP-NEXT:    v_mul_hi_u32 v18, v13, v5
+; CGP-NEXT:    v_mul_lo_u32 v17, v13, v5
+; CGP-NEXT:    v_xor_b32_e32 v9, v9, v11
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v15, v16
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v15, v18
+; CGP-NEXT:    v_mul_lo_u32 v16, v12, v17
+; CGP-NEXT:    v_mul_lo_u32 v18, v5, v15
+; CGP-NEXT:    v_mul_hi_u32 v19, v5, v17
+; CGP-NEXT:    v_mul_hi_u32 v17, v12, v17
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v16, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v16, v19
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v19, v12, v15
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v18, v16
+; CGP-NEXT:    v_mul_hi_u32 v18, v5, v15
+; CGP-NEXT:    v_mul_hi_u32 v15, v12, v15
+; CGP-NEXT:    v_add_i32_e32 v17, vcc, v19, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v17, vcc, v17, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v18, vcc, v19, v18
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v17, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v17, vcc, v18, v17
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v15, v17
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v16
+; CGP-NEXT:    v_addc_u32_e64 v16, s[4:5], v12, v15, vcc
+; CGP-NEXT:    v_mul_lo_u32 v14, v14, v5
+; CGP-NEXT:    v_mul_lo_u32 v17, v13, v16
+; CGP-NEXT:    v_mul_lo_u32 v18, v13, v5
+; CGP-NEXT:    v_mul_hi_u32 v13, v13, v5
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v17
+; CGP-NEXT:    v_mul_hi_u32 v15, v5, v18
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; CGP-NEXT:    v_mul_lo_u32 v14, v16, v18
+; CGP-NEXT:    v_mul_lo_u32 v17, v5, v13
+; CGP-NEXT:    v_mul_hi_u32 v18, v16, v18
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v15, v16, v13
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v17, v14
+; CGP-NEXT:    v_mul_hi_u32 v17, v5, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v16, v13
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v15, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v15, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v17, s[4:5], v18, v17
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v15, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v17, v15
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v15
+; CGP-NEXT:    v_addc_u32_e32 v12, vcc, v12, v13, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v14
+; CGP-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; CGP-NEXT:    v_mul_lo_u32 v13, v9, v5
+; CGP-NEXT:    v_mul_lo_u32 v14, v10, v12
+; CGP-NEXT:    v_mul_hi_u32 v15, v10, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v9, v5
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v15, v9, v12
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v14, v13
+; CGP-NEXT:    v_mul_hi_u32 v14, v10, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v9, v12
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v15, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v14, v13
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT:    v_mul_lo_u32 v13, v0, v5
+; CGP-NEXT:    v_mul_lo_u32 v12, v1, v12
+; CGP-NEXT:    v_mul_lo_u32 v14, v1, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v1, v5
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v12, v5
+; CGP-NEXT:    v_sub_i32_e32 v10, vcc, v10, v14
+; CGP-NEXT:    v_subb_u32_e64 v12, s[4:5], v9, v5, vcc
+; CGP-NEXT:    v_sub_i32_e64 v5, s[4:5], v9, v5
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v12, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], v12, v0
+; CGP-NEXT:    v_subb_u32_e32 v5, vcc, v5, v0, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v9, v9, v13, s[4:5]
+; CGP-NEXT:    v_sub_i32_e32 v13, vcc, v10, v1
+; CGP-NEXT:    v_subbrev_u32_e64 v14, s[4:5], 0, v5, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v14, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v13, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], v14, v0
+; CGP-NEXT:    v_subb_u32_e32 v0, vcc, v5, v0, vcc
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v13, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v15, v15, v16, s[4:5]
+; CGP-NEXT:    v_subbrev_u32_e32 v0, vcc, 0, v0, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v15
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v14, v0, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v11
+; CGP-NEXT:    v_xor_b32_e32 v5, v0, v11
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v1, v11
+; CGP-NEXT:    v_subb_u32_e32 v1, vcc, v5, v11, vcc
+; CGP-NEXT:  BB2_2: ; %Flow2
+; CGP-NEXT:    s_or_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[4:5]
+; CGP-NEXT:    s_cbranch_execz BB2_4
+; CGP-NEXT:  ; %bb.3:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v4
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, 0, v4
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_lo_u32 v1, v1, v0
+; CGP-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; CGP-NEXT:    v_mul_hi_u32 v0, v8, v0
+; CGP-NEXT:    v_mul_lo_u32 v0, v0, v4
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v8, v0
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v0, v4
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v0, v4
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CGP-NEXT:    v_mov_b32_e32 v1, 0
+; CGP-NEXT:  BB2_4:
+; CGP-NEXT:    s_or_b64 exec, exec, s[4:5]
+; CGP-NEXT:    v_or_b32_e32 v5, v3, v7
+; CGP-NEXT:    v_mov_b32_e32 v4, 0
+; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CGP-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CGP-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; CGP-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CGP-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execz BB2_6
+; CGP-NEXT:  ; %bb.5:
+; CGP-NEXT:    v_ashrrev_i32_e32 v4, 31, v7
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v6, v4
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, v7, v4, vcc
+; CGP-NEXT:    v_xor_b32_e32 v5, v5, v4
+; CGP-NEXT:    v_xor_b32_e32 v4, v7, v4
+; CGP-NEXT:    v_cvt_f32_u32_e32 v7, v5
+; CGP-NEXT:    v_cvt_f32_u32_e32 v8, v4
+; CGP-NEXT:    v_ashrrev_i32_e32 v9, 31, v3
+; CGP-NEXT:    v_mac_f32_e32 v7, 0x4f800000, v8
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v7, v7
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v2, v9
+; CGP-NEXT:    v_addc_u32_e32 v3, vcc, v3, v9, vcc
+; CGP-NEXT:    v_sub_i32_e32 v11, vcc, 0, v5
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x5f7ffffc, v7
+; CGP-NEXT:    v_mul_f32_e32 v10, 0x2f800000, v7
+; CGP-NEXT:    v_trunc_f32_e32 v10, v10
+; CGP-NEXT:    v_mac_f32_e32 v7, 0xcf800000, v10
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v10, v10
+; CGP-NEXT:    v_subb_u32_e32 v12, vcc, 0, v4, vcc
+; CGP-NEXT:    v_xor_b32_e32 v8, v8, v9
+; CGP-NEXT:    v_mul_lo_u32 v13, v12, v7
+; CGP-NEXT:    v_mul_lo_u32 v14, v11, v10
+; CGP-NEXT:    v_mul_hi_u32 v16, v11, v7
+; CGP-NEXT:    v_mul_lo_u32 v15, v11, v7
+; CGP-NEXT:    v_xor_b32_e32 v3, v3, v9
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
+; CGP-NEXT:    v_mul_lo_u32 v14, v10, v15
+; CGP-NEXT:    v_mul_lo_u32 v16, v7, v13
+; CGP-NEXT:    v_mul_hi_u32 v17, v7, v15
+; CGP-NEXT:    v_mul_hi_u32 v15, v10, v15
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v17, v10, v13
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v16, v14
+; CGP-NEXT:    v_mul_hi_u32 v16, v7, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v10, v13
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v17, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v15, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v17, v16
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v16, v15
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v14
+; CGP-NEXT:    v_addc_u32_e64 v14, s[4:5], v10, v13, vcc
+; CGP-NEXT:    v_mul_lo_u32 v12, v12, v7
+; CGP-NEXT:    v_mul_lo_u32 v15, v11, v14
+; CGP-NEXT:    v_mul_lo_u32 v16, v11, v7
+; CGP-NEXT:    v_mul_hi_u32 v11, v11, v7
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
+; CGP-NEXT:    v_mul_hi_u32 v13, v7, v16
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; CGP-NEXT:    v_mul_lo_u32 v12, v14, v16
+; CGP-NEXT:    v_mul_lo_u32 v15, v7, v11
+; CGP-NEXT:    v_mul_hi_u32 v16, v14, v16
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v13, v14, v11
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v15, v12
+; CGP-NEXT:    v_mul_hi_u32 v15, v7, v11
+; CGP-NEXT:    v_mul_hi_u32 v11, v14, v11
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v16, v15
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v13, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v15, v13
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; CGP-NEXT:    v_addc_u32_e32 v10, vcc, v10, v11, vcc
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v12
+; CGP-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; CGP-NEXT:    v_mul_lo_u32 v11, v3, v7
+; CGP-NEXT:    v_mul_lo_u32 v12, v8, v10
+; CGP-NEXT:    v_mul_hi_u32 v13, v8, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v3, v7
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v13, v3, v10
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_mul_hi_u32 v12, v8, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v3, v10
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v13, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT:    v_mul_lo_u32 v11, v4, v7
+; CGP-NEXT:    v_mul_lo_u32 v10, v5, v10
+; CGP-NEXT:    v_mul_lo_u32 v12, v5, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v5, v7
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
+; CGP-NEXT:    v_sub_i32_e32 v8, vcc, v8, v12
+; CGP-NEXT:    v_subb_u32_e64 v10, s[4:5], v3, v7, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v7
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], v10, v4
+; CGP-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v7, v7, v11, s[4:5]
+; CGP-NEXT:    v_sub_i32_e32 v11, vcc, v8, v5
+; CGP-NEXT:    v_subbrev_u32_e64 v12, s[4:5], 0, v3, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v12, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v11, v5
+; CGP-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], v12, v4
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v11, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v13, v13, v14, s[4:5]
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v13
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v11, v4, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
+; CGP-NEXT:    v_xor_b32_e32 v4, v4, v9
+; CGP-NEXT:    v_xor_b32_e32 v3, v3, v9
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v4, v9
+; CGP-NEXT:    v_subb_u32_e32 v5, vcc, v3, v9, vcc
+; CGP-NEXT:  BB2_6: ; %Flow
+; CGP-NEXT:    s_or_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[4:5]
+; CGP-NEXT:    s_cbranch_execz BB2_8
+; CGP-NEXT:  ; %bb.7:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v3, v6
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, 0, v6
+; CGP-NEXT:    v_mov_b32_e32 v5, 0
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; CGP-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, v3
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v3, v2, v3
+; CGP-NEXT:    v_mul_lo_u32 v3, v3, v6
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, v2, v6
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v6
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, v2, v6
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v6
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
+; CGP-NEXT:  BB2_8:
+; CGP-NEXT:    s_or_b64 exec, exec, s[4:5]
+; CGP-NEXT:    v_mov_b32_e32 v2, v4
+; CGP-NEXT:    v_mov_b32_e32 v3, v5
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %result = srem <2 x i64> %num, %den
+  ret <2 x i64> %result
+}
+
+define i64 @v_srem_i64_pow2k_denom(i64 %num) {
+; CHECK-LABEL: v_srem_i64_pow2k_denom:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_movk_i32 s6, 0x1000
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, s6
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, 0
+; CHECK-NEXT:    s_mov_b32 s7, 0xfffff000
+; CHECK-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; CHECK-NEXT:    v_mac_f32_e32 v2, 0x4f800000, v3
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; CHECK-NEXT:    v_addc_u32_e32 v1, vcc, v1, v4, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v4
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v4
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; CHECK-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
+; CHECK-NEXT:    v_trunc_f32_e32 v3, v3
+; CHECK-NEXT:    v_mac_f32_e32 v2, 0xcf800000, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CHECK-NEXT:    v_mul_lo_u32 v5, -1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, s7, v3
+; CHECK-NEXT:    v_mul_hi_u32 v8, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v7, s7, v2
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; CHECK-NEXT:    v_mul_lo_u32 v6, v3, v7
+; CHECK-NEXT:    v_mul_lo_u32 v8, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v9, v2, v7
+; CHECK-NEXT:    v_mul_hi_u32 v7, v3, v7
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v9, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CHECK-NEXT:    v_mul_hi_u32 v8, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v5, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v9, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT:    v_addc_u32_e64 v6, s[4:5], v3, v5, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v7, -1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v8, s7, v6
+; CHECK-NEXT:    v_mul_hi_u32 v10, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v9, s7, v2
+; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v3, v5
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v8
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v10
+; CHECK-NEXT:    v_mul_lo_u32 v8, v6, v9
+; CHECK-NEXT:    v_mul_lo_u32 v10, v2, v7
+; CHECK-NEXT:    v_mul_hi_u32 v5, v2, v9
+; CHECK-NEXT:    v_mul_hi_u32 v9, v6, v9
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v8, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s[4:5]
+; CHECK-NEXT:    v_mul_lo_u32 v8, v6, v7
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v10, v5
+; CHECK-NEXT:    v_mul_hi_u32 v10, v2, v7
+; CHECK-NEXT:    v_mul_hi_u32 v6, v6, v7
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v10
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v8, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v9, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v7
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, v3, v6, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v5, v1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, v0, v3
+; CHECK-NEXT:    v_mul_hi_u32 v7, v0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v7, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT:    v_mul_hi_u32 v6, v0, v3
+; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; CHECK-NEXT:    v_mul_lo_u32 v5, 0, v2
+; CHECK-NEXT:    v_mul_lo_u32 v3, s6, v3
+; CHECK-NEXT:    v_mul_lo_u32 v6, s6, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, s6, v2
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; CHECK-NEXT:    v_subb_u32_e64 v3, s[4:5], v1, v2, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v2
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], 0, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v3
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
+; CHECK-NEXT:    v_subrev_i32_e32 v5, vcc, s6, v0
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v7, vcc, s6, v5
+; CHECK-NEXT:    v_subbrev_u32_e32 v8, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; CHECK-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v8, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v4
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v4
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %result = srem i64 %num, 4096
+  ret i64 %result
+}
+
+define <2 x i64> @v_srem_v2i64_pow2k_denom(<2 x i64> %num) {
+; GISEL-LABEL: v_srem_v2i64_pow2k_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_movk_i32 s10, 0x1000
+; GISEL-NEXT:    s_add_u32 s4, s10, 0
+; GISEL-NEXT:    s_cselect_b32 s5, 1, 0
+; GISEL-NEXT:    s_and_b32 s5, s5, 1
+; GISEL-NEXT:    s_cmp_lg_u32 s5, 0
+; GISEL-NEXT:    s_mov_b32 s6, 0
+; GISEL-NEXT:    s_mov_b32 s7, s6
+; GISEL-NEXT:    s_addc_u32 s5, 0, 0
+; GISEL-NEXT:    s_xor_b64 s[8:9], s[4:5], s[6:7]
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s8
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, s9
+; GISEL-NEXT:    s_sub_u32 s11, 0, s8
+; GISEL-NEXT:    s_cselect_b32 s4, 1, 0
+; GISEL-NEXT:    s_and_b32 s4, s4, 1
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
+; GISEL-NEXT:    s_subb_u32 s12, 0, s9
+; GISEL-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; GISEL-NEXT:    v_trunc_f32_e32 v5, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v7, s12, v4
+; GISEL-NEXT:    v_mul_lo_u32 v8, s11, v5
+; GISEL-NEXT:    v_mul_hi_u32 v10, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, s11, v4
+; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v6
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; GISEL-NEXT:    v_mul_lo_u32 v8, v5, v9
+; GISEL-NEXT:    v_mul_lo_u32 v10, v4, v7
+; GISEL-NEXT:    v_mul_hi_u32 v11, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v9
+; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v6
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v7
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; GISEL-NEXT:    v_mul_hi_u32 v10, v4, v7
+; GISEL-NEXT:    v_mul_hi_u32 v7, v5, v7
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_addc_u32_e64 v8, s[4:5], v5, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, s12, v4
+; GISEL-NEXT:    v_mul_lo_u32 v10, s11, v8
+; GISEL-NEXT:    v_mul_hi_u32 v12, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, s11, v4
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v7
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v10
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_mul_lo_u32 v10, v8, v11
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v7, v4, v11
+; GISEL-NEXT:    v_mul_hi_u32 v11, v8, v11
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v10, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v10, v8, v9
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v12, v7
+; GISEL-NEXT:    v_mul_hi_u32 v12, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v8, v8, v9
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v10, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v11, v10
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v9
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v8, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v7, v1, v4
+; GISEL-NEXT:    v_mul_lo_u32 v8, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v9, v0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, v1, v5
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; GISEL-NEXT:    v_mul_hi_u32 v8, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v9, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT:    v_mul_lo_u32 v7, s9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v5, s8, v5
+; GISEL-NEXT:    v_mul_lo_u32 v8, s8, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, s8, v4
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT:    v_subb_u32_e64 v5, s[4:5], v1, v4, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v4
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s9, v5
+; GISEL-NEXT:    v_mov_b32_e32 v4, s9
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s8, v0
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], s9, v5
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s[4:5]
+; GISEL-NEXT:    v_subrev_i32_e32 v8, vcc, s8, v0
+; GISEL-NEXT:    v_subbrev_u32_e64 v9, s[4:5], 0, v1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s9, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s8, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], s9, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, v10, v11, s[4:5]
+; GISEL-NEXT:    s_add_u32 s4, s10, 0
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; GISEL-NEXT:    s_cselect_b32 s5, 1, 0
+; GISEL-NEXT:    v_subrev_i32_e32 v4, vcc, s8, v8
+; GISEL-NEXT:    s_and_b32 s5, s5, 1
+; GISEL-NEXT:    s_cmp_lg_u32 s5, 0
+; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT:    s_addc_u32 s5, 0, 0
+; GISEL-NEXT:    s_xor_b64 s[6:7], s[4:5], s[6:7]
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s6
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, s7
+; GISEL-NEXT:    s_sub_u32 s8, 0, s6
+; GISEL-NEXT:    s_cselect_b32 s4, 1, 0
+; GISEL-NEXT:    s_and_b32 s4, s4, 1
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
+; GISEL-NEXT:    s_subb_u32 s9, 0, s7
+; GISEL-NEXT:    v_ashrrev_i32_e32 v7, 31, v3
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; GISEL-NEXT:    v_trunc_f32_e32 v5, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v7
+; GISEL-NEXT:    v_addc_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v8, s9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, s8, v5
+; GISEL-NEXT:    v_mul_hi_u32 v11, s8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v10, s8, v4
+; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v6
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; GISEL-NEXT:    v_mul_lo_u32 v9, v5, v10
+; GISEL-NEXT:    v_mul_lo_u32 v11, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v12, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v10, v5, v10
+; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v7
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v5, v8
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_mul_hi_u32 v11, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, v5, v8
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; GISEL-NEXT:    v_addc_u32_e64 v9, s[4:5], v5, v8, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v10, s9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, s8, v9
+; GISEL-NEXT:    v_mul_hi_u32 v13, s8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, s8, v4
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v8
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; GISEL-NEXT:    v_mul_lo_u32 v11, v9, v12
+; GISEL-NEXT:    v_mul_lo_u32 v13, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v8, v4, v12
+; GISEL-NEXT:    v_mul_hi_u32 v12, v9, v12
+; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v7
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v11, v9, v10
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v13, v8
+; GISEL-NEXT:    v_mul_hi_u32 v13, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v9, v9, v10
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v12, v11
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v10
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v9, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v6
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v2, v5
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT:    v_mul_hi_u32 v6, v2, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v5
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
+; GISEL-NEXT:    v_mul_hi_u32 v9, v2, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v3, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; GISEL-NEXT:    v_mul_lo_u32 v6, s7, v4
+; GISEL-NEXT:    v_mul_lo_u32 v5, s6, v5
+; GISEL-NEXT:    v_mul_lo_u32 v8, s6, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, s6, v4
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v8
+; GISEL-NEXT:    v_subb_u32_e64 v5, s[4:5], v3, v4, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s7, v5
+; GISEL-NEXT:    v_mov_b32_e32 v4, s7
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v2
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], s7, v5
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, v6, v8, s[4:5]
+; GISEL-NEXT:    v_subrev_i32_e32 v8, vcc, s6, v2
+; GISEL-NEXT:    v_subbrev_u32_e64 v9, s[4:5], 0, v3, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s7, v9
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v8
+; GISEL-NEXT:    v_subrev_i32_e32 v4, vcc, s6, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], s7, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, v10, v11, s[4:5]
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v7
+; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v7
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v7
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_srem_v2i64_pow2k_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_movk_i32 s6, 0x1000
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, s6
+; CGP-NEXT:    v_cvt_f32_u32_e32 v5, 0
+; CGP-NEXT:    s_mov_b32 s7, 0xfffff000
+; CGP-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
+; CGP-NEXT:    v_mov_b32_e32 v7, v4
+; CGP-NEXT:    v_mac_f32_e32 v7, 0x4f800000, v5
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v7, v7
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_addc_u32_e32 v1, vcc, v1, v6, vcc
+; CGP-NEXT:    v_xor_b32_e32 v0, v0, v6
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x5f7ffffc, v7
+; CGP-NEXT:    v_mul_f32_e32 v8, 0x2f800000, v7
+; CGP-NEXT:    v_trunc_f32_e32 v8, v8
+; CGP-NEXT:    v_mac_f32_e32 v7, 0xcf800000, v8
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v6
+; CGP-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; CGP-NEXT:    v_mul_lo_u32 v9, -1, v7
+; CGP-NEXT:    v_mul_lo_u32 v10, s7, v8
+; CGP-NEXT:    v_mul_hi_u32 v12, s7, v7
+; CGP-NEXT:    v_mul_lo_u32 v11, s7, v7
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; CGP-NEXT:    v_mul_lo_u32 v10, v8, v11
+; CGP-NEXT:    v_mul_lo_u32 v12, v7, v9
+; CGP-NEXT:    v_mul_hi_u32 v13, v7, v11
+; CGP-NEXT:    v_mul_hi_u32 v11, v8, v11
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v13, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_mul_hi_u32 v12, v7, v9
+; CGP-NEXT:    v_mul_hi_u32 v9, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; CGP-NEXT:    v_addc_u32_e64 v10, s[4:5], v8, v9, vcc
+; CGP-NEXT:    v_mul_lo_u32 v11, -1, v7
+; CGP-NEXT:    v_mul_lo_u32 v12, s7, v10
+; CGP-NEXT:    v_mul_hi_u32 v14, s7, v7
+; CGP-NEXT:    v_mul_lo_u32 v13, s7, v7
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v9
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; CGP-NEXT:    v_mul_lo_u32 v12, v10, v13
+; CGP-NEXT:    v_mul_lo_u32 v14, v7, v11
+; CGP-NEXT:    v_mul_hi_u32 v9, v7, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v10, v13
+; CGP-NEXT:    v_ashrrev_i32_e32 v5, 31, v3
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v12, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v12, v10, v11
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v14, v9
+; CGP-NEXT:    v_mul_hi_u32 v14, v7, v11
+; CGP-NEXT:    v_mul_hi_u32 v10, v10, v11
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v14
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v12, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v13, v12
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; CGP-NEXT:    v_addc_u32_e32 v8, vcc, v8, v10, vcc
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; CGP-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; CGP-NEXT:    v_mul_lo_u32 v9, v1, v7
+; CGP-NEXT:    v_mul_lo_u32 v10, v0, v8
+; CGP-NEXT:    v_mul_hi_u32 v11, v0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v1, v7
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v11, v1, v8
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT:    v_mul_hi_u32 v10, v0, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v1, v8
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v11, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_mul_lo_u32 v9, 0, v7
+; CGP-NEXT:    v_mul_lo_u32 v8, s6, v8
+; CGP-NEXT:    v_mul_lo_u32 v10, s6, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, s6, v7
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v10
+; CGP-NEXT:    v_subb_u32_e64 v8, s[4:5], v1, v7, vcc
+; CGP-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v7
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], 0, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
+; CGP-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v7, v7, v9, s[4:5]
+; CGP-NEXT:    v_subrev_i32_e32 v9, vcc, s6, v0
+; CGP-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s6, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v10, v10, v11, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v11, vcc, s6, v9
+; CGP-NEXT:    v_subbrev_u32_e32 v12, vcc, 0, v1, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; CGP-NEXT:    v_cndmask_b32_e32 v9, v9, v11, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v12, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x2f800000, v4
+; CGP-NEXT:    v_trunc_f32_e32 v7, v7
+; CGP-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v9, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v8, -1, v4
+; CGP-NEXT:    v_mul_lo_u32 v9, s7, v7
+; CGP-NEXT:    v_mul_hi_u32 v11, s7, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, s7, v4
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; CGP-NEXT:    v_mul_lo_u32 v9, v7, v10
+; CGP-NEXT:    v_mul_lo_u32 v11, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v12, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v7, v10
+; CGP-NEXT:    v_xor_b32_e32 v0, v0, v6
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v12, v7, v8
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT:    v_mul_hi_u32 v11, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v7, v8
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; CGP-NEXT:    v_addc_u32_e64 v9, s[4:5], v7, v8, vcc
+; CGP-NEXT:    v_mul_lo_u32 v10, -1, v4
+; CGP-NEXT:    v_mul_lo_u32 v11, s7, v9
+; CGP-NEXT:    v_mul_hi_u32 v13, s7, v4
+; CGP-NEXT:    v_mul_lo_u32 v12, s7, v4
+; CGP-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v8
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; CGP-NEXT:    v_mul_lo_u32 v11, v9, v12
+; CGP-NEXT:    v_mul_lo_u32 v13, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v8, v4, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v9, v12
+; CGP-NEXT:    v_xor_b32_e32 v2, v2, v5
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v11, v9, v10
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v13, v8
+; CGP-NEXT:    v_mul_hi_u32 v13, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v9, v9, v10
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v12, v11
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v10
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, v7, v9, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT:    v_xor_b32_e32 v3, v3, v5
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v6
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_mul_lo_u32 v8, v3, v4
+; CGP-NEXT:    v_mul_lo_u32 v9, v2, v7
+; CGP-NEXT:    v_subb_u32_e32 v1, vcc, v1, v6, vcc
+; CGP-NEXT:    v_mul_hi_u32 v6, v2, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v8, v3, v7
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
+; CGP-NEXT:    v_mul_hi_u32 v9, v2, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v3, v7
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_mul_lo_u32 v7, 0, v4
+; CGP-NEXT:    v_mul_lo_u32 v6, s6, v6
+; CGP-NEXT:    v_mul_lo_u32 v8, s6, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, s6, v4
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v8
+; CGP-NEXT:    v_subb_u32_e64 v6, s[4:5], v3, v4, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], 0, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v2
+; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v6
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
+; CGP-NEXT:    v_subrev_i32_e32 v7, vcc, s6, v2
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s6, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v9, vcc, s6, v7
+; CGP-NEXT:    v_subbrev_u32_e32 v10, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; CGP-NEXT:    v_cndmask_b32_e32 v7, v7, v9, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v10, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v7, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; CGP-NEXT:    v_xor_b32_e32 v2, v2, v5
+; CGP-NEXT:    v_xor_b32_e32 v3, v3, v5
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT:    v_subb_u32_e32 v3, vcc, v3, v5, vcc
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %result = srem <2 x i64> %num, <i64 4096, i64 4096>
+  ret <2 x i64> %result
+}
+
+define i64 @v_srem_i64_oddk_denom(i64 %num) {
+; CHECK-LABEL: v_srem_i64_oddk_denom:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s6, 0x12d8fb
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, s6
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, 0
+; CHECK-NEXT:    s_mov_b32 s7, 0xffed2705
+; CHECK-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; CHECK-NEXT:    v_mac_f32_e32 v2, 0x4f800000, v3
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; CHECK-NEXT:    v_addc_u32_e32 v1, vcc, v1, v4, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v4
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v4
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; CHECK-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
+; CHECK-NEXT:    v_trunc_f32_e32 v3, v3
+; CHECK-NEXT:    v_mac_f32_e32 v2, 0xcf800000, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CHECK-NEXT:    v_mul_lo_u32 v5, -1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, s7, v3
+; CHECK-NEXT:    v_mul_hi_u32 v8, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v7, s7, v2
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; CHECK-NEXT:    v_mul_lo_u32 v6, v3, v7
+; CHECK-NEXT:    v_mul_lo_u32 v8, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v9, v2, v7
+; CHECK-NEXT:    v_mul_hi_u32 v7, v3, v7
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v9, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CHECK-NEXT:    v_mul_hi_u32 v8, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v5, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v9, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT:    v_addc_u32_e64 v6, s[4:5], v3, v5, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v7, -1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v8, s7, v6
+; CHECK-NEXT:    v_mul_hi_u32 v10, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v9, s7, v2
+; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v3, v5
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v8
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v10
+; CHECK-NEXT:    v_mul_lo_u32 v8, v6, v9
+; CHECK-NEXT:    v_mul_lo_u32 v10, v2, v7
+; CHECK-NEXT:    v_mul_hi_u32 v5, v2, v9
+; CHECK-NEXT:    v_mul_hi_u32 v9, v6, v9
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v8, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s[4:5]
+; CHECK-NEXT:    v_mul_lo_u32 v8, v6, v7
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v10, v5
+; CHECK-NEXT:    v_mul_hi_u32 v10, v2, v7
+; CHECK-NEXT:    v_mul_hi_u32 v6, v6, v7
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v10
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v8, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v9, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v7
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, v3, v6, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v5, v1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, v0, v3
+; CHECK-NEXT:    v_mul_hi_u32 v7, v0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v7, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT:    v_mul_hi_u32 v6, v0, v3
+; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; CHECK-NEXT:    v_mul_lo_u32 v5, 0, v2
+; CHECK-NEXT:    v_mul_lo_u32 v3, s6, v3
+; CHECK-NEXT:    v_mul_lo_u32 v6, s6, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, s6, v2
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; CHECK-NEXT:    v_subb_u32_e64 v3, s[4:5], v1, v2, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v2
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], 0, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v3
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
+; CHECK-NEXT:    v_subrev_i32_e32 v5, vcc, s6, v0
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v7, vcc, s6, v5
+; CHECK-NEXT:    v_subbrev_u32_e32 v8, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; CHECK-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v8, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v4
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v4
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %result = srem i64 %num, 1235195
+  ret i64 %result
+}
+
+define <2 x i64> @v_srem_v2i64_oddk_denom(<2 x i64> %num) {
+; GISEL-LABEL: v_srem_v2i64_oddk_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_mov_b32 s10, 0x12d8fb
+; GISEL-NEXT:    s_add_u32 s4, s10, 0
+; GISEL-NEXT:    s_cselect_b32 s5, 1, 0
+; GISEL-NEXT:    s_and_b32 s5, s5, 1
+; GISEL-NEXT:    s_cmp_lg_u32 s5, 0
+; GISEL-NEXT:    s_mov_b32 s6, 0
+; GISEL-NEXT:    s_mov_b32 s7, s6
+; GISEL-NEXT:    s_addc_u32 s5, 0, 0
+; GISEL-NEXT:    s_xor_b64 s[8:9], s[4:5], s[6:7]
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s8
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, s9
+; GISEL-NEXT:    s_sub_u32 s11, 0, s8
+; GISEL-NEXT:    s_cselect_b32 s4, 1, 0
+; GISEL-NEXT:    s_and_b32 s4, s4, 1
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
+; GISEL-NEXT:    s_subb_u32 s12, 0, s9
+; GISEL-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; GISEL-NEXT:    v_trunc_f32_e32 v5, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v7, s12, v4
+; GISEL-NEXT:    v_mul_lo_u32 v8, s11, v5
+; GISEL-NEXT:    v_mul_hi_u32 v10, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, s11, v4
+; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v6
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; GISEL-NEXT:    v_mul_lo_u32 v8, v5, v9
+; GISEL-NEXT:    v_mul_lo_u32 v10, v4, v7
+; GISEL-NEXT:    v_mul_hi_u32 v11, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v9
+; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v6
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v7
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; GISEL-NEXT:    v_mul_hi_u32 v10, v4, v7
+; GISEL-NEXT:    v_mul_hi_u32 v7, v5, v7
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_addc_u32_e64 v8, s[4:5], v5, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, s12, v4
+; GISEL-NEXT:    v_mul_lo_u32 v10, s11, v8
+; GISEL-NEXT:    v_mul_hi_u32 v12, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, s11, v4
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v7
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v10
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_mul_lo_u32 v10, v8, v11
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v7, v4, v11
+; GISEL-NEXT:    v_mul_hi_u32 v11, v8, v11
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v10, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v10, v8, v9
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v12, v7
+; GISEL-NEXT:    v_mul_hi_u32 v12, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v8, v8, v9
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v10, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v11, v10
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v9
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v8, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v7, v1, v4
+; GISEL-NEXT:    v_mul_lo_u32 v8, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v9, v0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, v1, v5
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; GISEL-NEXT:    v_mul_hi_u32 v8, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v9, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT:    v_mul_lo_u32 v7, s9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v5, s8, v5
+; GISEL-NEXT:    v_mul_lo_u32 v8, s8, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, s8, v4
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT:    v_subb_u32_e64 v5, s[4:5], v1, v4, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v4
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s9, v5
+; GISEL-NEXT:    v_mov_b32_e32 v4, s9
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s8, v0
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], s9, v5
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s[4:5]
+; GISEL-NEXT:    v_subrev_i32_e32 v8, vcc, s8, v0
+; GISEL-NEXT:    v_subbrev_u32_e64 v9, s[4:5], 0, v1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s9, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s8, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], s9, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, v10, v11, s[4:5]
+; GISEL-NEXT:    s_add_u32 s4, s10, 0
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; GISEL-NEXT:    s_cselect_b32 s5, 1, 0
+; GISEL-NEXT:    v_subrev_i32_e32 v4, vcc, s8, v8
+; GISEL-NEXT:    s_and_b32 s5, s5, 1
+; GISEL-NEXT:    s_cmp_lg_u32 s5, 0
+; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT:    s_addc_u32 s5, 0, 0
+; GISEL-NEXT:    s_xor_b64 s[6:7], s[4:5], s[6:7]
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s6
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, s7
+; GISEL-NEXT:    s_sub_u32 s8, 0, s6
+; GISEL-NEXT:    s_cselect_b32 s4, 1, 0
+; GISEL-NEXT:    s_and_b32 s4, s4, 1
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
+; GISEL-NEXT:    s_subb_u32 s9, 0, s7
+; GISEL-NEXT:    v_ashrrev_i32_e32 v7, 31, v3
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; GISEL-NEXT:    v_trunc_f32_e32 v5, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v7
+; GISEL-NEXT:    v_addc_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v8, s9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, s8, v5
+; GISEL-NEXT:    v_mul_hi_u32 v11, s8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v10, s8, v4
+; GISEL-NEXT:    v_xor_b32_e32 v0, v0, v6
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; GISEL-NEXT:    v_mul_lo_u32 v9, v5, v10
+; GISEL-NEXT:    v_mul_lo_u32 v11, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v12, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v10, v5, v10
+; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v7
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v5, v8
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_mul_hi_u32 v11, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, v5, v8
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; GISEL-NEXT:    v_addc_u32_e64 v9, s[4:5], v5, v8, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v10, s9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, s8, v9
+; GISEL-NEXT:    v_mul_hi_u32 v13, s8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, s8, v4
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v8
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; GISEL-NEXT:    v_mul_lo_u32 v11, v9, v12
+; GISEL-NEXT:    v_mul_lo_u32 v13, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v8, v4, v12
+; GISEL-NEXT:    v_mul_hi_u32 v12, v9, v12
+; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v7
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v11, v9, v10
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v13, v8
+; GISEL-NEXT:    v_mul_hi_u32 v13, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v9, v9, v10
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v12, v11
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v10
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v9, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v1, v1, v6
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v2, v5
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT:    v_mul_hi_u32 v6, v2, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v5
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
+; GISEL-NEXT:    v_mul_hi_u32 v9, v2, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v3, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; GISEL-NEXT:    v_mul_lo_u32 v6, s7, v4
+; GISEL-NEXT:    v_mul_lo_u32 v5, s6, v5
+; GISEL-NEXT:    v_mul_lo_u32 v8, s6, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, s6, v4
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v8
+; GISEL-NEXT:    v_subb_u32_e64 v5, s[4:5], v3, v4, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s7, v5
+; GISEL-NEXT:    v_mov_b32_e32 v4, s7
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v2
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], s7, v5
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, v6, v8, s[4:5]
+; GISEL-NEXT:    v_subrev_i32_e32 v8, vcc, s6, v2
+; GISEL-NEXT:    v_subbrev_u32_e64 v9, s[4:5], 0, v3, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s7, v9
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v8
+; GISEL-NEXT:    v_subrev_i32_e32 v4, vcc, s6, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], s7, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, v10, v11, s[4:5]
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v7
+; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v7
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v7
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_srem_v2i64_oddk_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_mov_b32 s6, 0x12d8fb
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, s6
+; CGP-NEXT:    v_cvt_f32_u32_e32 v5, 0
+; CGP-NEXT:    s_mov_b32 s7, 0xffed2705
+; CGP-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
+; CGP-NEXT:    v_mov_b32_e32 v7, v4
+; CGP-NEXT:    v_mac_f32_e32 v7, 0x4f800000, v5
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v7, v7
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_addc_u32_e32 v1, vcc, v1, v6, vcc
+; CGP-NEXT:    v_xor_b32_e32 v0, v0, v6
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x5f7ffffc, v7
+; CGP-NEXT:    v_mul_f32_e32 v8, 0x2f800000, v7
+; CGP-NEXT:    v_trunc_f32_e32 v8, v8
+; CGP-NEXT:    v_mac_f32_e32 v7, 0xcf800000, v8
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v6
+; CGP-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; CGP-NEXT:    v_mul_lo_u32 v9, -1, v7
+; CGP-NEXT:    v_mul_lo_u32 v10, s7, v8
+; CGP-NEXT:    v_mul_hi_u32 v12, s7, v7
+; CGP-NEXT:    v_mul_lo_u32 v11, s7, v7
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; CGP-NEXT:    v_mul_lo_u32 v10, v8, v11
+; CGP-NEXT:    v_mul_lo_u32 v12, v7, v9
+; CGP-NEXT:    v_mul_hi_u32 v13, v7, v11
+; CGP-NEXT:    v_mul_hi_u32 v11, v8, v11
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v13, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_mul_hi_u32 v12, v7, v9
+; CGP-NEXT:    v_mul_hi_u32 v9, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; CGP-NEXT:    v_addc_u32_e64 v10, s[4:5], v8, v9, vcc
+; CGP-NEXT:    v_mul_lo_u32 v11, -1, v7
+; CGP-NEXT:    v_mul_lo_u32 v12, s7, v10
+; CGP-NEXT:    v_mul_hi_u32 v14, s7, v7
+; CGP-NEXT:    v_mul_lo_u32 v13, s7, v7
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v9
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; CGP-NEXT:    v_mul_lo_u32 v12, v10, v13
+; CGP-NEXT:    v_mul_lo_u32 v14, v7, v11
+; CGP-NEXT:    v_mul_hi_u32 v9, v7, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v10, v13
+; CGP-NEXT:    v_ashrrev_i32_e32 v5, 31, v3
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v12, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v12, v10, v11
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v14, v9
+; CGP-NEXT:    v_mul_hi_u32 v14, v7, v11
+; CGP-NEXT:    v_mul_hi_u32 v10, v10, v11
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v14
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v12, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v13, v12
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; CGP-NEXT:    v_addc_u32_e32 v8, vcc, v8, v10, vcc
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; CGP-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; CGP-NEXT:    v_mul_lo_u32 v9, v1, v7
+; CGP-NEXT:    v_mul_lo_u32 v10, v0, v8
+; CGP-NEXT:    v_mul_hi_u32 v11, v0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v1, v7
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v11, v1, v8
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT:    v_mul_hi_u32 v10, v0, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v1, v8
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v11, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_mul_lo_u32 v9, 0, v7
+; CGP-NEXT:    v_mul_lo_u32 v8, s6, v8
+; CGP-NEXT:    v_mul_lo_u32 v10, s6, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, s6, v7
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v10
+; CGP-NEXT:    v_subb_u32_e64 v8, s[4:5], v1, v7, vcc
+; CGP-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v7
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], 0, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
+; CGP-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v7, v7, v9, s[4:5]
+; CGP-NEXT:    v_subrev_i32_e32 v9, vcc, s6, v0
+; CGP-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s6, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v10, v10, v11, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v11, vcc, s6, v9
+; CGP-NEXT:    v_subbrev_u32_e32 v12, vcc, 0, v1, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; CGP-NEXT:    v_cndmask_b32_e32 v9, v9, v11, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v12, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x2f800000, v4
+; CGP-NEXT:    v_trunc_f32_e32 v7, v7
+; CGP-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v9, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v8, -1, v4
+; CGP-NEXT:    v_mul_lo_u32 v9, s7, v7
+; CGP-NEXT:    v_mul_hi_u32 v11, s7, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, s7, v4
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; CGP-NEXT:    v_mul_lo_u32 v9, v7, v10
+; CGP-NEXT:    v_mul_lo_u32 v11, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v12, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v7, v10
+; CGP-NEXT:    v_xor_b32_e32 v0, v0, v6
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v12, v7, v8
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT:    v_mul_hi_u32 v11, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v7, v8
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; CGP-NEXT:    v_addc_u32_e64 v9, s[4:5], v7, v8, vcc
+; CGP-NEXT:    v_mul_lo_u32 v10, -1, v4
+; CGP-NEXT:    v_mul_lo_u32 v11, s7, v9
+; CGP-NEXT:    v_mul_hi_u32 v13, s7, v4
+; CGP-NEXT:    v_mul_lo_u32 v12, s7, v4
+; CGP-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v8
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; CGP-NEXT:    v_mul_lo_u32 v11, v9, v12
+; CGP-NEXT:    v_mul_lo_u32 v13, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v8, v4, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v9, v12
+; CGP-NEXT:    v_xor_b32_e32 v2, v2, v5
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v11, v9, v10
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v13, v8
+; CGP-NEXT:    v_mul_hi_u32 v13, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v9, v9, v10
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v12, v11
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v10
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, v7, v9, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT:    v_xor_b32_e32 v3, v3, v5
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v6
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_mul_lo_u32 v8, v3, v4
+; CGP-NEXT:    v_mul_lo_u32 v9, v2, v7
+; CGP-NEXT:    v_subb_u32_e32 v1, vcc, v1, v6, vcc
+; CGP-NEXT:    v_mul_hi_u32 v6, v2, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v8, v3, v7
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
+; CGP-NEXT:    v_mul_hi_u32 v9, v2, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v3, v7
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_mul_lo_u32 v7, 0, v4
+; CGP-NEXT:    v_mul_lo_u32 v6, s6, v6
+; CGP-NEXT:    v_mul_lo_u32 v8, s6, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, s6, v4
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v8
+; CGP-NEXT:    v_subb_u32_e64 v6, s[4:5], v3, v4, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], 0, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v2
+; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v6
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
+; CGP-NEXT:    v_subrev_i32_e32 v7, vcc, s6, v2
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s6, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v9, vcc, s6, v7
+; CGP-NEXT:    v_subbrev_u32_e32 v10, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; CGP-NEXT:    v_cndmask_b32_e32 v7, v7, v9, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v10, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v7, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; CGP-NEXT:    v_xor_b32_e32 v2, v2, v5
+; CGP-NEXT:    v_xor_b32_e32 v3, v3, v5
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v5
+; CGP-NEXT:    v_subb_u32_e32 v3, vcc, v3, v5, vcc
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %result = srem <2 x i64> %num, <i64 1235195, i64 1235195>
+  ret <2 x i64> %result
+}
+
+define i64 @v_srem_i64_pow2_shl_denom(i64 %x, i64 %y) {
+; CHECK-LABEL: v_srem_i64_pow2_shl_denom:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_movk_i32 s4, 0x1000
+; CHECK-NEXT:    s_mov_b32 s5, 0
+; CHECK-NEXT:    v_lshl_b64 v[4:5], s[4:5], v2
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0
+; CHECK-NEXT:    v_or_b32_e32 v3, v1, v5
+; CHECK-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CHECK-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CHECK-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; CHECK-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CHECK-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CHECK-NEXT:    s_cbranch_execz BB7_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    v_ashrrev_i32_e32 v2, 31, v5
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v4, v2
+; CHECK-NEXT:    v_addc_u32_e32 v5, vcc, v5, v2, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v3, v3, v2
+; CHECK-NEXT:    v_xor_b32_e32 v2, v5, v2
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v5, v3
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v6, v2
+; CHECK-NEXT:    v_ashrrev_i32_e32 v7, 31, v1
+; CHECK-NEXT:    v_mac_f32_e32 v5, 0x4f800000, v6
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v5, v5
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v0, v7
+; CHECK-NEXT:    v_addc_u32_e32 v1, vcc, v1, v7, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v9, vcc, 0, v3
+; CHECK-NEXT:    v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; CHECK-NEXT:    v_mul_f32_e32 v8, 0x2f800000, v5
+; CHECK-NEXT:    v_trunc_f32_e32 v8, v8
+; CHECK-NEXT:    v_mac_f32_e32 v5, 0xcf800000, v8
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; CHECK-NEXT:    v_subb_u32_e32 v10, vcc, 0, v2, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v6, v6, v7
+; CHECK-NEXT:    v_mul_lo_u32 v11, v10, v5
+; CHECK-NEXT:    v_mul_lo_u32 v12, v9, v8
+; CHECK-NEXT:    v_mul_hi_u32 v14, v9, v5
+; CHECK-NEXT:    v_mul_lo_u32 v13, v9, v5
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v7
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, v11, v14
+; CHECK-NEXT:    v_mul_lo_u32 v12, v8, v13
+; CHECK-NEXT:    v_mul_lo_u32 v14, v5, v11
+; CHECK-NEXT:    v_mul_hi_u32 v15, v5, v13
+; CHECK-NEXT:    v_mul_hi_u32 v13, v8, v13
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; CHECK-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; CHECK-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v15, v8, v11
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, v14, v12
+; CHECK-NEXT:    v_mul_hi_u32 v14, v5, v11
+; CHECK-NEXT:    v_mul_hi_u32 v11, v8, v11
+; CHECK-NEXT:    v_add_i32_e32 v13, vcc, v15, v13
+; CHECK-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; CHECK-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CHECK-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v13, vcc, v14, v13
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, v11, v13
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v12
+; CHECK-NEXT:    v_addc_u32_e64 v12, s[4:5], v8, v11, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v10, v10, v5
+; CHECK-NEXT:    v_mul_lo_u32 v13, v9, v12
+; CHECK-NEXT:    v_mul_lo_u32 v14, v9, v5
+; CHECK-NEXT:    v_mul_hi_u32 v9, v9, v5
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; CHECK-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; CHECK-NEXT:    v_mul_hi_u32 v11, v5, v14
+; CHECK-NEXT:    v_add_i32_e64 v9, s[4:5], v10, v9
+; CHECK-NEXT:    v_mul_lo_u32 v10, v12, v14
+; CHECK-NEXT:    v_mul_lo_u32 v13, v5, v9
+; CHECK-NEXT:    v_mul_hi_u32 v14, v12, v14
+; CHECK-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; CHECK-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CHECK-NEXT:    v_mul_lo_u32 v11, v12, v9
+; CHECK-NEXT:    v_add_i32_e64 v10, s[4:5], v13, v10
+; CHECK-NEXT:    v_mul_hi_u32 v13, v5, v9
+; CHECK-NEXT:    v_mul_hi_u32 v9, v12, v9
+; CHECK-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; CHECK-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; CHECK-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; CHECK-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v11, s[4:5], v13, v11
+; CHECK-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v11
+; CHECK-NEXT:    v_addc_u32_e32 v8, vcc, v8, v9, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
+; CHECK-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v9, v1, v5
+; CHECK-NEXT:    v_mul_lo_u32 v10, v6, v8
+; CHECK-NEXT:    v_mul_hi_u32 v11, v6, v5
+; CHECK-NEXT:    v_mul_hi_u32 v5, v1, v5
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v11, v1, v8
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CHECK-NEXT:    v_mul_hi_u32 v10, v6, v8
+; CHECK-NEXT:    v_mul_hi_u32 v8, v1, v8
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v11, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CHECK-NEXT:    v_mul_lo_u32 v9, v2, v5
+; CHECK-NEXT:    v_mul_lo_u32 v8, v3, v8
+; CHECK-NEXT:    v_mul_lo_u32 v10, v3, v5
+; CHECK-NEXT:    v_mul_hi_u32 v5, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
+; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v6, v10
+; CHECK-NEXT:    v_subb_u32_e64 v8, s[4:5], v1, v5, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v5
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], v8, v2
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[4:5]
+; CHECK-NEXT:    v_sub_i32_e32 v9, vcc, v6, v3
+; CHECK-NEXT:    v_subbrev_u32_e64 v10, s[4:5], 0, v1, vcc
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v3
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], v10, v2
+; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, v9, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, v11, v12, s[4:5]
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
+; CHECK-NEXT:    v_xor_b32_e32 v2, v2, v7
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v7
+; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, v2, v7
+; CHECK-NEXT:    v_subb_u32_e32 v3, vcc, v1, v7, vcc
+; CHECK-NEXT:  BB7_2: ; %Flow
+; CHECK-NEXT:    s_or_saveexec_b64 s[4:5], s[6:7]
+; CHECK-NEXT:    s_xor_b64 exec, exec, s[4:5]
+; CHECK-NEXT:    s_cbranch_execz BB7_4
+; CHECK-NEXT:  ; %bb.3:
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, v4
+; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, 0, v4
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; CHECK-NEXT:    v_mul_lo_u32 v2, v2, v1
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CHECK-NEXT:    v_mul_lo_u32 v1, v1, v4
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, v0, v4
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, v0, v4
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
+; CHECK-NEXT:  BB7_4:
+; CHECK-NEXT:    s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT:    v_mov_b32_e32 v0, v2
+; CHECK-NEXT:    v_mov_b32_e32 v1, v3
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shl.y = shl i64 4096, %y
+  %r = srem i64 %x, %shl.y
+  ret i64 %r
+}
+
+define <2 x i64> @v_srem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
+; GISEL-LABEL: v_srem_v2i64_pow2_shl_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_movk_i32 s6, 0x1000
+; GISEL-NEXT:    s_mov_b32 s7, 0
+; GISEL-NEXT:    v_lshl_b64 v[4:5], s[6:7], v4
+; GISEL-NEXT:    v_ashrrev_i32_e32 v9, 31, v1
+; GISEL-NEXT:    v_ashrrev_i32_e32 v7, 31, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v7, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v5, v5, v7
+; GISEL-NEXT:    v_xor_b32_e32 v4, v4, v7
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v7, v4
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v8, v5
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v9, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, 0, v4
+; GISEL-NEXT:    v_mac_f32_e32 v7, 0x4f800000, v8
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v7, v7
+; GISEL-NEXT:    v_xor_b32_e32 v8, v0, v9
+; GISEL-NEXT:    v_subb_u32_e32 v11, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v16, v1, v9
+; GISEL-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v7
+; GISEL-NEXT:    v_mul_f32_e32 v7, 0x2f800000, v0
+; GISEL-NEXT:    v_trunc_f32_e32 v7, v7
+; GISEL-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v7
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; GISEL-NEXT:    v_mul_lo_u32 v12, v11, v0
+; GISEL-NEXT:    v_mul_lo_u32 v13, v10, v7
+; GISEL-NEXT:    v_mul_hi_u32 v15, v10, v0
+; GISEL-NEXT:    v_mul_lo_u32 v14, v10, v0
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; GISEL-NEXT:    v_mul_lo_u32 v13, v7, v14
+; GISEL-NEXT:    v_mul_lo_u32 v15, v0, v12
+; GISEL-NEXT:    v_mul_hi_u32 v1, v0, v14
+; GISEL-NEXT:    v_mul_hi_u32 v14, v7, v14
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v13, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v13, v7, v12
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v15, v1
+; GISEL-NEXT:    v_mul_hi_u32 v15, v0, v12
+; GISEL-NEXT:    v_mul_hi_u32 v12, v7, v12
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v13, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v14, v13
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT:    v_addc_u32_e64 v1, s[4:5], v7, v12, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v11, v11, v0
+; GISEL-NEXT:    v_mul_lo_u32 v13, v10, v1
+; GISEL-NEXT:    v_mul_lo_u32 v14, v10, v0
+; GISEL-NEXT:    v_mul_hi_u32 v10, v10, v0
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v12
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_mul_hi_u32 v12, v0, v14
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v10
+; GISEL-NEXT:    v_mul_lo_u32 v11, v1, v14
+; GISEL-NEXT:    v_mul_lo_u32 v13, v0, v10
+; GISEL-NEXT:    v_mul_hi_u32 v14, v1, v14
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v12, v1, v10
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v13, v11
+; GISEL-NEXT:    v_mul_hi_u32 v13, v0, v10
+; GISEL-NEXT:    v_mul_hi_u32 v1, v1, v10
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v13, v12
+; GISEL-NEXT:    v_add_i32_e64 v1, s[4:5], v1, v10
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v7, v1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v0, v11
+; GISEL-NEXT:    v_addc_u32_e32 v10, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v11, v16, v7
+; GISEL-NEXT:    v_mul_lo_u32 v12, v8, v10
+; GISEL-NEXT:    v_lshl_b64 v[0:1], s[6:7], v6
+; GISEL-NEXT:    v_mul_hi_u32 v6, v8, v7
+; GISEL-NEXT:    v_mul_hi_u32 v7, v16, v7
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v11, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v11, v16, v10
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v12, v6
+; GISEL-NEXT:    v_mul_hi_u32 v12, v8, v10
+; GISEL-NEXT:    v_mul_hi_u32 v10, v16, v10
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v11, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v11, v7
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
+; GISEL-NEXT:    v_mul_lo_u32 v10, v5, v6
+; GISEL-NEXT:    v_mul_lo_u32 v7, v4, v7
+; GISEL-NEXT:    v_mul_lo_u32 v11, v4, v6
+; GISEL-NEXT:    v_mul_hi_u32 v6, v4, v6
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, v8, v11
+; GISEL-NEXT:    v_subb_u32_e64 v8, s[4:5], v16, v6, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v6, s[4:5], v16, v6
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v8, v5
+; GISEL-NEXT:    v_subb_u32_e32 v6, vcc, v6, v5, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, v10, v11, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, v7, v4
+; GISEL-NEXT:    v_subbrev_u32_e64 v12, s[4:5], 0, v6, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v12, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v11, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v12, v5
+; GISEL-NEXT:    v_subb_u32_e32 v5, vcc, v6, v5, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v11, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, v13, v14, s[4:5]
+; GISEL-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v13
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v11, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v5, v12, v5, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v7, v0, v6
+; GISEL-NEXT:    v_xor_b32_e32 v6, v1, v6
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v0, v7
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v1, v6
+; GISEL-NEXT:    v_ashrrev_i32_e32 v8, 31, v3
+; GISEL-NEXT:    v_xor_b32_e32 v4, v4, v9
+; GISEL-NEXT:    v_xor_b32_e32 v5, v5, v9
+; GISEL-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v2, v8
+; GISEL-NEXT:    v_addc_u32_e32 v2, vcc, v3, v8, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v3, v1, v8
+; GISEL-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GISEL-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GISEL-NEXT:    v_trunc_f32_e32 v1, v1
+; GISEL-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, 0, v7
+; GISEL-NEXT:    v_subb_u32_e32 v11, vcc, 0, v6, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v11, v0
+; GISEL-NEXT:    v_mul_lo_u32 v13, v10, v1
+; GISEL-NEXT:    v_mul_hi_u32 v15, v10, v0
+; GISEL-NEXT:    v_mul_lo_u32 v14, v10, v0
+; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v8
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; GISEL-NEXT:    v_mul_lo_u32 v13, v1, v14
+; GISEL-NEXT:    v_mul_lo_u32 v15, v0, v12
+; GISEL-NEXT:    v_mul_hi_u32 v16, v0, v14
+; GISEL-NEXT:    v_mul_hi_u32 v14, v1, v14
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v16, v1, v12
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v15, v13
+; GISEL-NEXT:    v_mul_hi_u32 v15, v0, v12
+; GISEL-NEXT:    v_mul_hi_u32 v12, v1, v12
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v16, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v16, v15
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v14, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v13
+; GISEL-NEXT:    v_addc_u32_e64 v13, s[4:5], v1, v12, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v11, v11, v0
+; GISEL-NEXT:    v_mul_lo_u32 v14, v10, v13
+; GISEL-NEXT:    v_mul_lo_u32 v15, v10, v0
+; GISEL-NEXT:    v_mul_hi_u32 v10, v10, v0
+; GISEL-NEXT:    v_add_i32_e64 v1, s[4:5], v1, v12
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; GISEL-NEXT:    v_mul_hi_u32 v12, v0, v15
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v10
+; GISEL-NEXT:    v_mul_lo_u32 v11, v13, v15
+; GISEL-NEXT:    v_mul_lo_u32 v14, v0, v10
+; GISEL-NEXT:    v_mul_hi_u32 v15, v13, v15
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v12, v13, v10
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v14, v11
+; GISEL-NEXT:    v_mul_hi_u32 v14, v0, v10
+; GISEL-NEXT:    v_mul_hi_u32 v10, v13, v10
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[4:5], v15, v14
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v14, v12
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v12
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v10, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v0, v11
+; GISEL-NEXT:    v_addc_u32_e32 v11, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v2, v10
+; GISEL-NEXT:    v_mul_lo_u32 v13, v3, v11
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v10
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v5, v9, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v12, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v5, v2, v11
+; GISEL-NEXT:    v_mul_hi_u32 v10, v2, v10
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v9, v4
+; GISEL-NEXT:    v_mul_hi_u32 v9, v3, v11
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT:    v_mul_hi_u32 v10, v2, v11
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v10, v5
+; GISEL-NEXT:    v_mul_lo_u32 v9, v6, v4
+; GISEL-NEXT:    v_mul_lo_u32 v5, v7, v5
+; GISEL-NEXT:    v_mul_lo_u32 v10, v7, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v7, v4
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, v3, v10
+; GISEL-NEXT:    v_subb_u32_e64 v5, s[4:5], v2, v4, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v4
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v5, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v3, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v5, v6
+; GISEL-NEXT:    v_subb_u32_e32 v2, vcc, v2, v6, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, v4, v9, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e32 v9, vcc, v3, v7
+; GISEL-NEXT:    v_subbrev_u32_e64 v10, s[4:5], 0, v2, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v7
+; GISEL-NEXT:    v_subb_u32_e32 v2, vcc, v2, v6, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v10, v6
+; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, v9, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, v11, v12, s[4:5]
+; GISEL-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GISEL-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v8
+; GISEL-NEXT:    v_xor_b32_e32 v4, v2, v8
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v3, v8
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v4, v8, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_srem_v2i64_pow2_shl_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_movk_i32 s4, 0x1000
+; CGP-NEXT:    s_mov_b32 s5, 0
+; CGP-NEXT:    v_lshl_b64 v[10:11], s[4:5], v4
+; CGP-NEXT:    v_mov_b32_e32 v7, v1
+; CGP-NEXT:    v_mov_b32_e32 v5, v0
+; CGP-NEXT:    v_or_b32_e32 v1, v7, v11
+; CGP-NEXT:    v_mov_b32_e32 v0, 0
+; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; CGP-NEXT:    v_lshl_b64 v[8:9], s[4:5], v6
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CGP-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CGP-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; CGP-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CGP-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execz BB8_2
+; CGP-NEXT:  ; %bb.1:
+; CGP-NEXT:    v_ashrrev_i32_e32 v0, 31, v11
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, v10, v0
+; CGP-NEXT:    v_addc_u32_e32 v4, vcc, v11, v0, vcc
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v0
+; CGP-NEXT:    v_xor_b32_e32 v0, v4, v0
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, v1
+; CGP-NEXT:    v_cvt_f32_u32_e32 v6, v0
+; CGP-NEXT:    v_ashrrev_i32_e32 v11, 31, v7
+; CGP-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v6
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v5, v11
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, v7, v11, vcc
+; CGP-NEXT:    v_sub_i32_e32 v13, vcc, 0, v1
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CGP-NEXT:    v_mul_f32_e32 v12, 0x2f800000, v4
+; CGP-NEXT:    v_trunc_f32_e32 v12, v12
+; CGP-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v12
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_cvt_u32_f32_e32 v12, v12
+; CGP-NEXT:    v_subb_u32_e32 v14, vcc, 0, v0, vcc
+; CGP-NEXT:    v_xor_b32_e32 v6, v6, v11
+; CGP-NEXT:    v_mul_lo_u32 v15, v14, v4
+; CGP-NEXT:    v_mul_lo_u32 v16, v13, v12
+; CGP-NEXT:    v_mul_hi_u32 v18, v13, v4
+; CGP-NEXT:    v_mul_lo_u32 v17, v13, v4
+; CGP-NEXT:    v_xor_b32_e32 v7, v7, v11
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v15, v16
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v15, v18
+; CGP-NEXT:    v_mul_lo_u32 v16, v12, v17
+; CGP-NEXT:    v_mul_lo_u32 v18, v4, v15
+; CGP-NEXT:    v_mul_hi_u32 v19, v4, v17
+; CGP-NEXT:    v_mul_hi_u32 v17, v12, v17
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v16, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v16, v19
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v19, v12, v15
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v18, v16
+; CGP-NEXT:    v_mul_hi_u32 v18, v4, v15
+; CGP-NEXT:    v_mul_hi_u32 v15, v12, v15
+; CGP-NEXT:    v_add_i32_e32 v17, vcc, v19, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v17, vcc, v17, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v18, vcc, v19, v18
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v17, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v17, vcc, v18, v17
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v15, v17
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v16
+; CGP-NEXT:    v_addc_u32_e64 v16, s[4:5], v12, v15, vcc
+; CGP-NEXT:    v_mul_lo_u32 v14, v14, v4
+; CGP-NEXT:    v_mul_lo_u32 v17, v13, v16
+; CGP-NEXT:    v_mul_lo_u32 v18, v13, v4
+; CGP-NEXT:    v_mul_hi_u32 v13, v13, v4
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v17
+; CGP-NEXT:    v_mul_hi_u32 v15, v4, v18
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; CGP-NEXT:    v_mul_lo_u32 v14, v16, v18
+; CGP-NEXT:    v_mul_lo_u32 v17, v4, v13
+; CGP-NEXT:    v_mul_hi_u32 v18, v16, v18
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v15, v16, v13
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v17, v14
+; CGP-NEXT:    v_mul_hi_u32 v17, v4, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v16, v13
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v15, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v15, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v17, s[4:5], v18, v17
+; CGP-NEXT:    v_add_i32_e64 v14, s[4:5], v15, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v17, v15
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v15
+; CGP-NEXT:    v_addc_u32_e32 v12, vcc, v12, v13, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v14
+; CGP-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; CGP-NEXT:    v_mul_lo_u32 v13, v7, v4
+; CGP-NEXT:    v_mul_lo_u32 v14, v6, v12
+; CGP-NEXT:    v_mul_hi_u32 v15, v6, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v7, v4
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v15, v7, v12
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v14, v13
+; CGP-NEXT:    v_mul_hi_u32 v14, v6, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v7, v12
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v15, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v14, v13
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT:    v_mul_lo_u32 v13, v0, v4
+; CGP-NEXT:    v_mul_lo_u32 v12, v1, v12
+; CGP-NEXT:    v_mul_lo_u32 v14, v1, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v1, v4
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v12, v4
+; CGP-NEXT:    v_sub_i32_e32 v6, vcc, v6, v14
+; CGP-NEXT:    v_subb_u32_e64 v12, s[4:5], v7, v4, vcc
+; CGP-NEXT:    v_sub_i32_e64 v4, s[4:5], v7, v4
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v12, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], v12, v0
+; CGP-NEXT:    v_subb_u32_e32 v4, vcc, v4, v0, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v7, v7, v13, s[4:5]
+; CGP-NEXT:    v_sub_i32_e32 v13, vcc, v6, v1
+; CGP-NEXT:    v_subbrev_u32_e64 v14, s[4:5], 0, v4, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v14, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v13, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], v14, v0
+; CGP-NEXT:    v_subb_u32_e32 v0, vcc, v4, v0, vcc
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v13, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v15, v15, v16, s[4:5]
+; CGP-NEXT:    v_subbrev_u32_e32 v0, vcc, 0, v0, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v15
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v14, v0, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
+; CGP-NEXT:    v_xor_b32_e32 v1, v1, v11
+; CGP-NEXT:    v_xor_b32_e32 v4, v0, v11
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v1, v11
+; CGP-NEXT:    v_subb_u32_e32 v1, vcc, v4, v11, vcc
+; CGP-NEXT:  BB8_2: ; %Flow2
+; CGP-NEXT:    s_or_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[4:5]
+; CGP-NEXT:    s_cbranch_execz BB8_4
+; CGP-NEXT:  ; %bb.3:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v10
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, 0, v10
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_lo_u32 v1, v1, v0
+; CGP-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; CGP-NEXT:    v_mul_hi_u32 v0, v5, v0
+; CGP-NEXT:    v_mul_lo_u32 v0, v0, v10
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v5, v0
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v0, v10
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v10
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v0, v10
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v10
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CGP-NEXT:    v_mov_b32_e32 v1, 0
+; CGP-NEXT:  BB8_4:
+; CGP-NEXT:    s_or_b64 exec, exec, s[4:5]
+; CGP-NEXT:    v_or_b32_e32 v5, v3, v9
+; CGP-NEXT:    v_mov_b32_e32 v4, 0
+; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CGP-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CGP-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; CGP-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CGP-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execz BB8_6
+; CGP-NEXT:  ; %bb.5:
+; CGP-NEXT:    v_ashrrev_i32_e32 v4, 31, v9
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v8, v4
+; CGP-NEXT:    v_addc_u32_e32 v6, vcc, v9, v4, vcc
+; CGP-NEXT:    v_xor_b32_e32 v5, v5, v4
+; CGP-NEXT:    v_xor_b32_e32 v4, v6, v4
+; CGP-NEXT:    v_cvt_f32_u32_e32 v6, v5
+; CGP-NEXT:    v_cvt_f32_u32_e32 v7, v4
+; CGP-NEXT:    v_ashrrev_i32_e32 v9, 31, v3
+; CGP-NEXT:    v_mac_f32_e32 v6, 0x4f800000, v7
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v2, v9
+; CGP-NEXT:    v_addc_u32_e32 v3, vcc, v3, v9, vcc
+; CGP-NEXT:    v_sub_i32_e32 v11, vcc, 0, v5
+; CGP-NEXT:    v_mul_f32_e32 v6, 0x5f7ffffc, v6
+; CGP-NEXT:    v_mul_f32_e32 v10, 0x2f800000, v6
+; CGP-NEXT:    v_trunc_f32_e32 v10, v10
+; CGP-NEXT:    v_mac_f32_e32 v6, 0xcf800000, v10
+; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; CGP-NEXT:    v_cvt_u32_f32_e32 v10, v10
+; CGP-NEXT:    v_subb_u32_e32 v12, vcc, 0, v4, vcc
+; CGP-NEXT:    v_xor_b32_e32 v7, v7, v9
+; CGP-NEXT:    v_mul_lo_u32 v13, v12, v6
+; CGP-NEXT:    v_mul_lo_u32 v14, v11, v10
+; CGP-NEXT:    v_mul_hi_u32 v16, v11, v6
+; CGP-NEXT:    v_mul_lo_u32 v15, v11, v6
+; CGP-NEXT:    v_xor_b32_e32 v3, v3, v9
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
+; CGP-NEXT:    v_mul_lo_u32 v14, v10, v15
+; CGP-NEXT:    v_mul_lo_u32 v16, v6, v13
+; CGP-NEXT:    v_mul_hi_u32 v17, v6, v15
+; CGP-NEXT:    v_mul_hi_u32 v15, v10, v15
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v17, v10, v13
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v16, v14
+; CGP-NEXT:    v_mul_hi_u32 v16, v6, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v10, v13
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v17, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v15, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v16, vcc, v17, v16
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v16, v15
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v14
+; CGP-NEXT:    v_addc_u32_e64 v14, s[4:5], v10, v13, vcc
+; CGP-NEXT:    v_mul_lo_u32 v12, v12, v6
+; CGP-NEXT:    v_mul_lo_u32 v15, v11, v14
+; CGP-NEXT:    v_mul_lo_u32 v16, v11, v6
+; CGP-NEXT:    v_mul_hi_u32 v11, v11, v6
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
+; CGP-NEXT:    v_mul_hi_u32 v13, v6, v16
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; CGP-NEXT:    v_mul_lo_u32 v12, v14, v16
+; CGP-NEXT:    v_mul_lo_u32 v15, v6, v11
+; CGP-NEXT:    v_mul_hi_u32 v16, v14, v16
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v13, v14, v11
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v15, v12
+; CGP-NEXT:    v_mul_hi_u32 v15, v6, v11
+; CGP-NEXT:    v_mul_hi_u32 v11, v14, v11
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v15, s[4:5], v16, v15
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v13, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v15, v13
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; CGP-NEXT:    v_addc_u32_e32 v10, vcc, v10, v11, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v12
+; CGP-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; CGP-NEXT:    v_mul_lo_u32 v11, v3, v6
+; CGP-NEXT:    v_mul_lo_u32 v12, v7, v10
+; CGP-NEXT:    v_mul_hi_u32 v13, v7, v6
+; CGP-NEXT:    v_mul_hi_u32 v6, v3, v6
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v13, v3, v10
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_mul_hi_u32 v12, v7, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v3, v10
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v13, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT:    v_mul_lo_u32 v11, v4, v6
+; CGP-NEXT:    v_mul_lo_u32 v10, v5, v10
+; CGP-NEXT:    v_mul_lo_u32 v12, v5, v6
+; CGP-NEXT:    v_mul_hi_u32 v6, v5, v6
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v7, v12
+; CGP-NEXT:    v_subb_u32_e64 v10, s[4:5], v3, v6, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v6
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], v10, v4
+; CGP-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v6, v6, v11, s[4:5]
+; CGP-NEXT:    v_sub_i32_e32 v11, vcc, v7, v5
+; CGP-NEXT:    v_subbrev_u32_e64 v12, s[4:5], 0, v3, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v12, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v11, v5
+; CGP-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], v12, v4
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v11, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v13, v13, v14, s[4:5]
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v13
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v11, v4, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
+; CGP-NEXT:    v_xor_b32_e32 v4, v4, v9
+; CGP-NEXT:    v_xor_b32_e32 v3, v3, v9
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v4, v9
+; CGP-NEXT:    v_subb_u32_e32 v5, vcc, v3, v9, vcc
+; CGP-NEXT:  BB8_6: ; %Flow
+; CGP-NEXT:    s_or_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[4:5]
+; CGP-NEXT:    s_cbranch_execz BB8_8
+; CGP-NEXT:  ; %bb.7:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v3, v8
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, 0, v8
+; CGP-NEXT:    v_mov_b32_e32 v5, 0
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; CGP-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, v3
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v3, v2, v3
+; CGP-NEXT:    v_mul_lo_u32 v3, v3, v8
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, v2, v8
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v8
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, v2, v8
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v8
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
+; CGP-NEXT:  BB8_8:
+; CGP-NEXT:    s_or_b64 exec, exec, s[4:5]
+; CGP-NEXT:    v_mov_b32_e32 v2, v4
+; CGP-NEXT:    v_mov_b32_e32 v3, v5
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %shl.y = shl <2 x i64> <i64 4096, i64 4096>, %y
+  %r = srem <2 x i64> %x, %shl.y
+  ret <2 x i64> %r
+}
+
+define i64 @v_srem_i64_24bit(i64 %num, i64 %den) {
+; GISEL-LABEL: v_srem_i64_24bit:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_mov_b32 s4, 0xffffff
+; GISEL-NEXT:    v_and_b32_e32 v1, s4, v2
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v2, v1
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; GISEL-NEXT:    v_and_b32_e32 v0, s4, v0
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GISEL-NEXT:    v_mul_lo_u32 v3, v3, v2
+; GISEL-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GISEL-NEXT:    v_mul_hi_u32 v2, v0, v2
+; GISEL-NEXT:    v_mul_lo_u32 v2, v2, v1
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_srem_i64_24bit:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_mov_b32 s4, 0xffffff
+; CGP-NEXT:    v_and_b32_e32 v1, s4, v2
+; CGP-NEXT:    v_cvt_f32_i32_e32 v2, v1
+; CGP-NEXT:    v_and_b32_e32 v0, s4, v0
+; CGP-NEXT:    v_cvt_f32_i32_e32 v3, v0
+; CGP-NEXT:    v_xor_b32_e32 v5, v0, v1
+; CGP-NEXT:    v_rcp_f32_e32 v4, v2
+; CGP-NEXT:    v_ashrrev_i32_e32 v5, 30, v5
+; CGP-NEXT:    v_or_b32_e32 v5, 1, v5
+; CGP-NEXT:    v_mul_f32_e32 v4, v3, v4
+; CGP-NEXT:    v_trunc_f32_e32 v4, v4
+; CGP-NEXT:    v_mad_f32 v3, -v4, v2, v3
+; CGP-NEXT:    v_cvt_i32_f32_e32 v4, v4
+; CGP-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
+; CGP-NEXT:    v_cndmask_b32_e32 v2, 0, v5, vcc
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; CGP-NEXT:    v_mul_lo_u32 v1, v2, v1
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; CGP-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
+; CGP-NEXT:    v_ashrrev_i32_e32 v0, 7, v0
+; CGP-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %num.mask = and i64 %num, 16777215
+  %den.mask = and i64 %den, 16777215
+  %result = srem i64 %num.mask, %den.mask
+  ret i64 %result
+}
+
+define <2 x i64> @v_srem_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
+; GISEL-LABEL: v_srem_v2i64_24bit:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_mov_b32 s6, 0xffffff
+; GISEL-NEXT:    v_and_b32_e32 v1, s6, v4
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, 0, v1
+; GISEL-NEXT:    v_addc_u32_e64 v3, s[4:5], 0, 0, vcc
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, v1
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v3
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v1
+; GISEL-NEXT:    v_subb_u32_e32 v8, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_and_b32_e32 v0, s6, v0
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    v_and_b32_e32 v6, s6, v6
+; GISEL-NEXT:    v_and_b32_e32 v2, s6, v2
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; GISEL-NEXT:    v_trunc_f32_e32 v5, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_mul_lo_u32 v9, v8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v10, v7, v5
+; GISEL-NEXT:    v_mul_hi_u32 v12, v7, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, v7, v4
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT:    v_mul_lo_u32 v10, v5, v11
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v14, v4, v11
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
+; GISEL-NEXT:    v_addc_u32_e64 v13, s[4:5], 0, 0, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v14, v5, v9
+; GISEL-NEXT:    v_mul_hi_u32 v11, v5, v11
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; GISEL-NEXT:    v_mul_hi_u32 v12, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v9
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v14, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v14, v12
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; GISEL-NEXT:    v_addc_u32_e64 v10, s[4:5], v5, v9, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v8, v8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, v7, v10
+; GISEL-NEXT:    v_mul_lo_u32 v12, v7, v4
+; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v4
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v9
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; GISEL-NEXT:    v_mul_hi_u32 v9, v4, v12
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v8, v7
+; GISEL-NEXT:    v_mul_lo_u32 v8, v10, v12
+; GISEL-NEXT:    v_mul_lo_u32 v11, v4, v7
+; GISEL-NEXT:    v_mul_hi_u32 v12, v10, v12
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v9, v10, v7
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; GISEL-NEXT:    v_mul_hi_u32 v11, v4, v7
+; GISEL-NEXT:    v_mul_hi_u32 v7, v10, v7
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v9, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v11, v9
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v9
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v7, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v7, v13, v4
+; GISEL-NEXT:    v_mul_lo_u32 v8, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v9, v0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v13, v4
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, v13, v5
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; GISEL-NEXT:    v_mul_hi_u32 v8, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v13, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v9, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT:    v_mul_lo_u32 v7, v3, v4
+; GISEL-NEXT:    v_mul_lo_u32 v5, v1, v5
+; GISEL-NEXT:    v_mul_lo_u32 v8, v1, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT:    v_subb_u32_e64 v5, s[4:5], v13, v4, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v4, s[4:5], v13, v4
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v5, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v5, v3
+; GISEL-NEXT:    v_subb_u32_e32 v4, vcc, v4, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e32 v8, vcc, v0, v1
+; GISEL-NEXT:    v_subbrev_u32_e64 v9, s[4:5], 0, v4, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v9, v3
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v4, v3, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v8, v1
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, 0, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, v10, v11, s[4:5]
+; GISEL-NEXT:    v_addc_u32_e64 v6, s[4:5], 0, 0, vcc
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v11, v4
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v12, v6
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
+; GISEL-NEXT:    v_mac_f32_e32 v11, 0x4f800000, v12
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v8, v11
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v3, vcc
+; GISEL-NEXT:    v_mul_f32_e32 v3, 0x5f7ffffc, v8
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v3
+; GISEL-NEXT:    v_trunc_f32_e32 v5, v5
+; GISEL-NEXT:    v_mac_f32_e32 v3, 0xcf800000, v5
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v4
+; GISEL-NEXT:    v_subb_u32_e32 v8, vcc, 0, v6, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, v8, v3
+; GISEL-NEXT:    v_mul_lo_u32 v10, v7, v5
+; GISEL-NEXT:    v_mul_hi_u32 v12, v7, v3
+; GISEL-NEXT:    v_mul_lo_u32 v11, v7, v3
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT:    v_mul_lo_u32 v10, v5, v11
+; GISEL-NEXT:    v_mul_lo_u32 v12, v3, v9
+; GISEL-NEXT:    v_mul_hi_u32 v14, v3, v11
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, 0, v2
+; GISEL-NEXT:    v_addc_u32_e64 v13, s[4:5], 0, 0, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v14, v5, v9
+; GISEL-NEXT:    v_mul_hi_u32 v11, v5, v11
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; GISEL-NEXT:    v_mul_hi_u32 v12, v3, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v9
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v14, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v14, v12
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v10
+; GISEL-NEXT:    v_addc_u32_e64 v10, s[4:5], v5, v9, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v8, v8, v3
+; GISEL-NEXT:    v_mul_lo_u32 v11, v7, v10
+; GISEL-NEXT:    v_mul_lo_u32 v12, v7, v3
+; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v3
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v9
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; GISEL-NEXT:    v_mul_hi_u32 v9, v3, v12
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v8, v7
+; GISEL-NEXT:    v_mul_lo_u32 v8, v10, v12
+; GISEL-NEXT:    v_mul_lo_u32 v11, v3, v7
+; GISEL-NEXT:    v_mul_hi_u32 v12, v10, v12
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v9, v10, v7
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v11, v8
+; GISEL-NEXT:    v_mul_hi_u32 v11, v3, v7
+; GISEL-NEXT:    v_mul_hi_u32 v7, v10, v7
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v9, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v11, v9
+; GISEL-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v9
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v7, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v8
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v7, v13, v3
+; GISEL-NEXT:    v_mul_lo_u32 v8, v2, v5
+; GISEL-NEXT:    v_mul_hi_u32 v9, v2, v3
+; GISEL-NEXT:    v_subrev_i32_e32 v0, vcc, 0, v0
+; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, v13, v5
+; GISEL-NEXT:    v_mul_hi_u32 v3, v13, v3
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; GISEL-NEXT:    v_mul_hi_u32 v8, v2, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v13, v5
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v9, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT:    v_mul_lo_u32 v7, v6, v3
+; GISEL-NEXT:    v_mul_lo_u32 v5, v4, v5
+; GISEL-NEXT:    v_mul_lo_u32 v8, v4, v3
+; GISEL-NEXT:    v_mul_hi_u32 v3, v4, v3
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v8
+; GISEL-NEXT:    v_subb_u32_e64 v5, s[4:5], v13, v3, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v13, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v5, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v5, v6
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v6, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e32 v8, vcc, v2, v4
+; GISEL-NEXT:    v_subbrev_u32_e64 v9, s[4:5], 0, v3, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v4
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v6, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v8, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v9, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, v10, v11, s[4:5]
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v2, vcc, 0, v2
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_srem_v2i64_24bit:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_mov_b32 s4, 0xffffff
+; CGP-NEXT:    v_and_b32_e32 v1, s4, v4
+; CGP-NEXT:    v_cvt_f32_i32_e32 v3, v1
+; CGP-NEXT:    v_and_b32_e32 v0, s4, v0
+; CGP-NEXT:    v_cvt_f32_i32_e32 v4, v0
+; CGP-NEXT:    v_xor_b32_e32 v7, v0, v1
+; CGP-NEXT:    v_rcp_f32_e32 v5, v3
+; CGP-NEXT:    v_ashrrev_i32_e32 v7, 30, v7
+; CGP-NEXT:    v_or_b32_e32 v7, 1, v7
+; CGP-NEXT:    v_and_b32_e32 v2, s4, v2
+; CGP-NEXT:    v_mul_f32_e32 v5, v4, v5
+; CGP-NEXT:    v_trunc_f32_e32 v5, v5
+; CGP-NEXT:    v_mad_f32 v4, -v5, v3, v4
+; CGP-NEXT:    v_cvt_i32_f32_e32 v5, v5
+; CGP-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, |v3|
+; CGP-NEXT:    v_cndmask_b32_e32 v3, 0, v7, vcc
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
+; CGP-NEXT:    v_mul_lo_u32 v1, v3, v1
+; CGP-NEXT:    v_and_b32_e32 v3, s4, v6
+; CGP-NEXT:    v_cvt_f32_i32_e32 v4, v3
+; CGP-NEXT:    v_xor_b32_e32 v6, v2, v3
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; CGP-NEXT:    v_cvt_f32_i32_e32 v1, v2
+; CGP-NEXT:    v_rcp_f32_e32 v5, v4
+; CGP-NEXT:    v_ashrrev_i32_e32 v6, 30, v6
+; CGP-NEXT:    v_or_b32_e32 v6, 1, v6
+; CGP-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
+; CGP-NEXT:    v_mul_f32_e32 v5, v1, v5
+; CGP-NEXT:    v_trunc_f32_e32 v5, v5
+; CGP-NEXT:    v_mad_f32 v1, -v5, v4, v1
+; CGP-NEXT:    v_cvt_i32_f32_e32 v5, v5
+; CGP-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v4|
+; CGP-NEXT:    v_cndmask_b32_e32 v1, 0, v6, vcc
+; CGP-NEXT:    v_ashrrev_i32_e32 v0, 7, v0
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, v5, v1
+; CGP-NEXT:    v_mul_lo_u32 v3, v1, v3
+; CGP-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT:    v_lshlrev_b32_e32 v2, 7, v2
+; CGP-NEXT:    v_ashrrev_i32_e32 v2, 7, v2
+; CGP-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %num.mask = and <2 x i64> %num, <i64 16777215, i64 16777215>
+  %den.mask = and <2 x i64> %den, <i64 16777215, i64 16777215>
+  %result = srem <2 x i64> %num.mask, %den.mask
+  ret <2 x i64> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll
index 31ce2d033eeaf..25eafb45f9309 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll
@@ -9,67 +9,56 @@ define i32 @v_udiv_i32(i32 %num, i32 %den) {
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v2, v1
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GISEL-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GISEL-NEXT:    v_mul_hi_u32 v4, v2, v1
-; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v3, v3, v2
-; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v2, v3
-; GISEL-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v2, v2, v0
+; GISEL-NEXT:    v_mul_lo_u32 v3, v3, v2
+; GISEL-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GISEL-NEXT:    v_mul_hi_u32 v2, v0, v2
 ; GISEL-NEXT:    v_mul_lo_u32 v3, v2, v1
 ; GISEL-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; GISEL-NEXT:    v_subrev_i32_e32 v5, vcc, 1, v2
-; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, v0, v3
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v1
-; GISEL-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, 1, v2
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; CGP-LABEL: v_udiv_i32:
 ; CGP:       ; %bb.0:
 ; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v2, v1
-; CGP-NEXT:    v_mul_lo_u32 v3, 0, v1
-; CGP-NEXT:    v_mul_lo_u32 v4, 0, v0
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; CGP-NEXT:    v_mul_lo_u32 v4, v0, 0
 ; CGP-NEXT:    v_rcp_f32_e32 v2, v2
-; CGP-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; CGP-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, v2, v1
-; CGP-NEXT:    v_mul_lo_u32 v6, v2, 0
-; CGP-NEXT:    v_mul_hi_u32 v7, v2, v1
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; CGP-NEXT:    v_sub_i32_e32 v8, vcc, 0, v5
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; CGP-NEXT:    v_cndmask_b32_e32 v3, v5, v8, vcc
-; CGP-NEXT:    v_mul_lo_u32 v5, v3, 0
-; CGP-NEXT:    v_mul_hi_u32 v3, v3, v2
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v6, v5
-; CGP-NEXT:    v_add_i32_e64 v3, s[4:5], v5, v3
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v2, v3
-; CGP-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; CGP-NEXT:    v_mul_lo_u32 v3, v2, 0
-; CGP-NEXT:    v_mul_hi_u32 v2, v2, v0
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; CGP-NEXT:    v_mul_lo_u32 v3, v3, v2
+; CGP-NEXT:    v_mul_lo_u32 v5, v2, 0
+; CGP-NEXT:    v_mul_lo_u32 v6, 0, v3
+; CGP-NEXT:    v_mul_hi_u32 v3, v2, v3
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT:    v_mul_lo_u32 v3, 0, v2
+; CGP-NEXT:    v_mul_hi_u32 v2, v0, v2
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
 ; CGP-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; CGP-NEXT:    v_mul_lo_u32 v3, v2, v1
 ; CGP-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; CGP-NEXT:    v_subrev_i32_e32 v5, vcc, 1, v2
-; CGP-NEXT:    v_sub_i32_e32 v6, vcc, v0, v3
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v1
-; CGP-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, 1, v2
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %result = udiv i32 %num, %den
   ret i32 %result
@@ -82,71 +71,57 @@ define amdgpu_ps i32 @s_udiv_i32(i32 inreg %num, i32 inreg %den) {
 ; GISEL-LABEL: s_udiv_i32:
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v0, s1
+; GISEL-NEXT:    s_sub_i32 s2, 0, s1
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GISEL-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GISEL-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GISEL-NEXT:    v_mul_lo_u32 v1, s2, v0
+; GISEL-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT:    v_mul_hi_u32 v0, s0, v0
 ; GISEL-NEXT:    v_mul_lo_u32 v1, v0, s1
-; GISEL-NEXT:    v_mul_hi_u32 v2, v0, s1
-; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GISEL-NEXT:    v_add_i32_e64 v2, s[2:3], v0, v1
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[2:3], v0, v1
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, s0, v1
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s1, v1
 ; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v0, v0, s0
-; GISEL-NEXT:    v_mul_lo_u32 v1, v0, s1
+; GISEL-NEXT:    v_subrev_i32_e64 v2, s[2:3], s1, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
-; GISEL-NEXT:    v_subrev_i32_e32 v3, vcc, 1, v0
-; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, s0, v1
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, s0, v1
-; GISEL-NEXT:    v_cmp_le_u32_e64 s[0:1], s1, v4
-; GISEL-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s1, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-NEXT:    v_readfirstlane_b32 s0, v0
 ; GISEL-NEXT:    ; return to shader part epilog
 ;
 ; CGP-LABEL: s_udiv_i32:
 ; CGP:       ; %bb.0:
-; CGP-NEXT:    s_mov_b32 s4, s1
-; CGP-NEXT:    v_cvt_f32_u32_e32 v0, s4
-; CGP-NEXT:    s_bfe_u64 s[2:3], s[4:5], 0x200000
-; CGP-NEXT:    s_bfe_u64 s[6:7], s[0:1], 0x200000
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, s1
+; CGP-NEXT:    s_sub_i32 s4, 0, s1
+; CGP-NEXT:    s_bfe_u64 s[2:3], s[0:1], 0x200000
 ; CGP-NEXT:    v_rcp_f32_e32 v0, v0
-; CGP-NEXT:    v_mul_lo_u32 v1, 0, s2
-; CGP-NEXT:    v_mul_lo_u32 v2, 0, s6
-; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; CGP-NEXT:    v_mul_lo_u32 v1, s2, 0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; CGP-NEXT:    v_mul_lo_u32 v3, v0, s2
-; CGP-NEXT:    v_mul_lo_u32 v4, v0, s3
-; CGP-NEXT:    v_mul_hi_u32 v5, v0, s2
-; CGP-NEXT:    v_mul_lo_u32 v6, 0, v0
-; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
-; CGP-NEXT:    v_sub_i32_e32 v4, vcc, 0, v3
-; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; CGP-NEXT:    v_mul_lo_u32 v3, v1, 0
-; CGP-NEXT:    v_mul_hi_u32 v1, v1, v0
-; CGP-NEXT:    v_add_i32_e64 v3, s[2:3], v6, v3
-; CGP-NEXT:    v_add_i32_e64 v1, s[2:3], v3, v1
-; CGP-NEXT:    v_add_i32_e64 v3, s[2:3], v0, v1
-; CGP-NEXT:    v_sub_i32_e64 v0, s[2:3], v0, v1
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; CGP-NEXT:    v_mul_lo_u32 v1, v0, s7
-; CGP-NEXT:    v_mul_hi_u32 v0, v0, s6
+; CGP-NEXT:    v_mul_lo_u32 v2, s4, v0
+; CGP-NEXT:    v_mul_lo_u32 v3, v0, 0
+; CGP-NEXT:    v_mul_lo_u32 v4, 0, v2
+; CGP-NEXT:    v_mul_hi_u32 v2, v0, v2
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; CGP-NEXT:    v_mul_lo_u32 v2, s3, v0
+; CGP-NEXT:    v_mul_hi_u32 v0, s2, v0
 ; CGP-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; CGP-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
-; CGP-NEXT:    v_mul_lo_u32 v1, v0, s4
+; CGP-NEXT:    v_mul_lo_u32 v1, v0, s1
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, s0, v1
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s1, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CGP-NEXT:    v_subrev_i32_e64 v2, s[2:3], s1, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
 ; CGP-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
-; CGP-NEXT:    v_subrev_i32_e32 v3, vcc, 1, v0
-; CGP-NEXT:    v_sub_i32_e32 v4, vcc, s0, v1
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, s0, v1
-; CGP-NEXT:    v_cmp_le_u32_e64 s[0:1], s4, v4
-; CGP-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s1, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CGP-NEXT:    v_readfirstlane_b32 s0, v0
 ; CGP-NEXT:    ; return to shader part epilog
   %result = udiv i32 %num, %den
@@ -159,124 +134,102 @@ define <2 x i32> @v_udiv_v2i32(<2 x i32> %num, <2 x i32> %den) {
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v3
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v6, v3
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v3
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; GISEL-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
+; GISEL-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GISEL-NEXT:    v_mul_lo_u32 v6, v4, v2
-; GISEL-NEXT:    v_mul_hi_u32 v7, v4, v2
-; GISEL-NEXT:    v_mul_lo_u32 v8, v5, v3
-; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v3
-; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, 0, v6
-; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, 0, v8
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, v8, v11, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v6, v6, v4
-; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v5
-; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v4, v6
-; GISEL-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v6
-; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v5, v7
-; GISEL-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v4, v4, v0
-; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v1
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v4
+; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v6
+; GISEL-NEXT:    v_mul_hi_u32 v5, v4, v5
+; GISEL-NEXT:    v_mul_hi_u32 v7, v6, v7
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v6, v7
+; GISEL-NEXT:    v_mul_hi_u32 v4, v0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v5, v1, v5
 ; GISEL-NEXT:    v_mul_lo_u32 v6, v4, v2
 ; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
-; GISEL-NEXT:    v_subrev_i32_e32 v8, vcc, 1, v4
-; GISEL-NEXT:    v_mul_lo_u32 v9, v5, v3
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, 1, v5
-; GISEL-NEXT:    v_subrev_i32_e32 v11, vcc, 1, v5
-; GISEL-NEXT:    v_sub_i32_e32 v12, vcc, v0, v6
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v6
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v9
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v9
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[6:7], v12, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[8:9], v0, v3
-; GISEL-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v4, v7, s[6:7]
-; GISEL-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v5, v10, s[6:7]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v11, v1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v8, v5, v3
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, 1, v5
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v8
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v6, s[4:5], v0, v2
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v7, s[6:7], v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v5
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v6, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v7, vcc
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; CGP-LABEL: v_udiv_v2i32:
 ; CGP:       ; %bb.0:
 ; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, 0, v2
-; CGP-NEXT:    v_mul_lo_u32 v6, 0, v0
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; CGP-NEXT:    v_mul_lo_u32 v6, v0, 0
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v7, v3
-; CGP-NEXT:    v_mul_lo_u32 v8, 0, v3
-; CGP-NEXT:    v_mul_lo_u32 v9, 0, v1
+; CGP-NEXT:    v_sub_i32_e32 v8, vcc, 0, v3
+; CGP-NEXT:    v_mul_lo_u32 v9, v1, 0
 ; CGP-NEXT:    v_rcp_f32_e32 v4, v4
 ; CGP-NEXT:    v_rcp_f32_e32 v7, v7
-; CGP-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f800000, v7
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; CGP-NEXT:    v_mul_lo_u32 v10, v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v11, v4, 0
-; CGP-NEXT:    v_mul_hi_u32 v12, v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v13, v7, v3
-; CGP-NEXT:    v_mul_lo_u32 v14, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v15, v7, v3
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v11
-; CGP-NEXT:    v_sub_i32_e32 v16, vcc, 0, v10
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v14
-; CGP-NEXT:    v_sub_i32_e32 v17, vcc, 0, v13
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v12
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v15
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; CGP-NEXT:    v_cndmask_b32_e32 v5, v10, v16, vcc
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
-; CGP-NEXT:    v_cndmask_b32_e64 v8, v13, v17, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v10, v5, 0
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v4
-; CGP-NEXT:    v_mul_lo_u32 v12, v8, 0
-; CGP-NEXT:    v_mul_hi_u32 v8, v8, v7
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v11, v10
-; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v14, v12
-; CGP-NEXT:    v_add_i32_e64 v5, s[6:7], v10, v5
-; CGP-NEXT:    v_add_i32_e64 v8, s[6:7], v11, v8
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v4, v5
-; CGP-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v5
-; CGP-NEXT:    v_add_i32_e64 v5, s[6:7], v7, v8
-; CGP-NEXT:    v_sub_i32_e64 v7, s[6:7], v7, v8
-; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v7, v4, 0
-; CGP-NEXT:    v_mul_hi_u32 v4, v4, v0
-; CGP-NEXT:    v_mul_lo_u32 v8, v5, 0
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v1
-; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v9, v8
+; CGP-NEXT:    v_mul_lo_u32 v5, v5, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v4, 0
+; CGP-NEXT:    v_mul_lo_u32 v8, v8, v7
+; CGP-NEXT:    v_mul_lo_u32 v11, v7, 0
+; CGP-NEXT:    v_mul_lo_u32 v12, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v4, v5
+; CGP-NEXT:    v_mul_lo_u32 v13, 0, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v7, v8
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v10, v5
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v7, v8
+; CGP-NEXT:    v_mul_lo_u32 v7, 0, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v0, v4
+; CGP-NEXT:    v_mul_lo_u32 v8, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v1, v5
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v8, v9
 ; CGP-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
 ; CGP-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
 ; CGP-NEXT:    v_mul_lo_u32 v6, v4, v2
 ; CGP-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
-; CGP-NEXT:    v_subrev_i32_e32 v8, vcc, 1, v4
-; CGP-NEXT:    v_mul_lo_u32 v9, v5, v3
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, 1, v5
-; CGP-NEXT:    v_subrev_i32_e32 v11, vcc, 1, v5
-; CGP-NEXT:    v_sub_i32_e32 v12, vcc, v0, v6
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v6
-; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v9
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v9
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v12, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[8:9], v0, v3
-; CGP-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v4, v7, s[6:7]
-; CGP-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v5, v10, s[6:7]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v11, v1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v8, v5, v3
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, 1, v5
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v8
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; CGP-NEXT:    v_sub_i32_e64 v6, s[4:5], v0, v2
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v7, s[6:7], v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, 1, v5
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v4, v6, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v5, v7, vcc
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %result = udiv <2 x i32> %num, %den
   ret <2 x i32> %result
@@ -286,89 +239,117 @@ define i32 @v_udiv_i32_pow2k_denom(i32 %num) {
 ; CHECK-LABEL: v_udiv_i32_pow2k_denom:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_movk_i32 s4, 0x1000
-; CHECK-NEXT:    v_mov_b32_e32 v1, 0x1000
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, s4
+; CHECK-NEXT:    s_movk_i32 s6, 0x1000
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0xfffff000
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, s6
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; CHECK-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; CHECK-NEXT:    v_lshlrev_b32_e32 v3, 12, v2
-; CHECK-NEXT:    v_mul_hi_u32 v4, v2, s4
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v2
-; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v2, v3
-; CHECK-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v2, v2, v0
-; CHECK-NEXT:    v_lshlrev_b32_e32 v3, 12, v2
-; CHECK-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; CHECK-NEXT:    v_subrev_i32_e32 v5, vcc, 1, v2
-; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v0, v3
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v1
-; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v1, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v1, v2, v1
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CHECK-NEXT:    v_lshlrev_b32_e32 v2, 12, v1
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; CHECK-NEXT:    v_subrev_i32_e64 v2, s[4:5], s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, 1, v1
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %result = udiv i32 %num, 4096
   ret i32 %result
 }
 
 define <2 x i32> @v_udiv_v2i32_pow2k_denom(<2 x i32> %num) {
-; CHECK-LABEL: v_udiv_v2i32_pow2k_denom:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_movk_i32 s4, 0x1000
-; CHECK-NEXT:    v_mov_b32_e32 v2, 0x1000
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, s4
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; CHECK-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; CHECK-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; CHECK-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
-; CHECK-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; CHECK-NEXT:    v_lshlrev_b32_e32 v5, 12, v3
-; CHECK-NEXT:    v_mul_hi_u32 v6, v3, s4
-; CHECK-NEXT:    v_lshlrev_b32_e32 v7, 12, v4
-; CHECK-NEXT:    v_mul_hi_u32 v8, v4, v2
-; CHECK-NEXT:    v_sub_i32_e32 v9, vcc, 0, v5
-; CHECK-NEXT:    v_sub_i32_e32 v10, vcc, 0, v7
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
-; CHECK-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
-; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
-; CHECK-NEXT:    v_cndmask_b32_e64 v6, v7, v10, s[4:5]
-; CHECK-NEXT:    v_mul_hi_u32 v5, v5, v3
-; CHECK-NEXT:    v_mul_hi_u32 v6, v6, v4
-; CHECK-NEXT:    v_add_i32_e64 v7, s[6:7], v3, v5
-; CHECK-NEXT:    v_sub_i32_e64 v3, s[6:7], v3, v5
-; CHECK-NEXT:    v_add_i32_e64 v5, s[6:7], v4, v6
-; CHECK-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v6
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v4, v4, v5, s[4:5]
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v0
-; CHECK-NEXT:    v_mul_hi_u32 v4, v4, v1
-; CHECK-NEXT:    v_lshlrev_b32_e32 v5, 12, v3
-; CHECK-NEXT:    v_add_i32_e32 v6, vcc, 1, v3
-; CHECK-NEXT:    v_subrev_i32_e32 v7, vcc, 1, v3
-; CHECK-NEXT:    v_lshlrev_b32_e32 v8, 12, v4
-; CHECK-NEXT:    v_add_i32_e32 v9, vcc, 1, v4
-; CHECK-NEXT:    v_subrev_i32_e32 v10, vcc, 1, v4
-; CHECK-NEXT:    v_sub_i32_e32 v11, vcc, v0, v5
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v5
-; CHECK-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v8
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v8
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[6:7], v11, v2
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[8:9], v0, v2
-; CHECK-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v3, v6, s[6:7]
-; CHECK-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e64 v1, v4, v9, s[6:7]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v1, v10, v1, s[4:5]
-; CHECK-NEXT:    s_setpc_b64 s[30:31]
+; GISEL-LABEL: v_udiv_v2i32_pow2k_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_movk_i32 s8, 0x1000
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v2, s8
+; GISEL-NEXT:    s_sub_i32 s4, 0, s8
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GISEL-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v2
+; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GISEL-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GISEL-NEXT:    v_mul_lo_u32 v5, s4, v2
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT:    v_mul_hi_u32 v5, v2, v5
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GISEL-NEXT:    v_mul_hi_u32 v3, v0, v3
+; GISEL-NEXT:    v_mul_hi_u32 v2, v1, v2
+; GISEL-NEXT:    v_lshlrev_b32_e32 v4, 12, v3
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
+; GISEL-NEXT:    v_lshlrev_b32_e32 v6, 12, v2
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v2
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v6
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GISEL-NEXT:    v_subrev_i32_e64 v4, s[4:5], s8, v0
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s8, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v2, v2, v7, s[4:5]
+; GISEL-NEXT:    v_subrev_i32_e64 v5, s[6:7], s8, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, 1, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, 1, v2
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v3, v4, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s8, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v2, v5, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_udiv_v2i32_pow2k_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_movk_i32 s4, 0x1000
+; CGP-NEXT:    v_mov_b32_e32 v2, 0x1000
+; CGP-NEXT:    s_mov_b32 s5, 0xfffff000
+; CGP-NEXT:    v_mov_b32_e32 v3, 0xfffff000
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, s4
+; CGP-NEXT:    v_cvt_f32_u32_e32 v5, v2
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v5, v5
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
+; CGP-NEXT:    v_mul_f32_e32 v5, 0x4f7ffffe, v5
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CGP-NEXT:    v_mul_lo_u32 v6, s5, v4
+; CGP-NEXT:    v_mul_lo_u32 v3, v3, v5
+; CGP-NEXT:    v_mul_hi_u32 v6, v4, v6
+; CGP-NEXT:    v_mul_hi_u32 v3, v5, v3
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
+; CGP-NEXT:    v_mul_hi_u32 v4, v0, v4
+; CGP-NEXT:    v_mul_hi_u32 v3, v1, v3
+; CGP-NEXT:    v_lshlrev_b32_e32 v5, 12, v4
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; CGP-NEXT:    v_lshlrev_b32_e32 v7, 12, v3
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, 1, v3
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v7
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; CGP-NEXT:    v_subrev_i32_e64 v5, s[4:5], s4, v0
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v2
+; CGP-NEXT:    v_cndmask_b32_e64 v3, v3, v8, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v6, s[6:7], v1, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v1, v6, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, 1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v4, v5, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v3, v6, vcc
+; CGP-NEXT:    s_setpc_b64 s[30:31]
   %result = udiv <2 x i32> %num, <i32 4096, i32 4096>
   ret <2 x i32> %result
 }
@@ -378,87 +359,115 @@ define i32 @v_udiv_i32_oddk_denom(i32 %num) {
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_mov_b32 s6, 0x12d8fb
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, s6
-; CHECK-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; CHECK-NEXT:    v_mul_lo_u32 v2, v1, s6
-; CHECK-NEXT:    v_mul_hi_u32 v3, v1, s6
-; CHECK-NEXT:    v_sub_i32_e32 v4, vcc, 0, v2
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v2, v2, v1
-; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v1, v2
-; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v2
-; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v1, v1, v0
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0xffed2705
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, s6
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; CHECK-NEXT:    v_mul_lo_u32 v1, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v1, v2, v1
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; CHECK-NEXT:    v_mul_lo_u32 v2, v1, s6
 ; CHECK-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
-; CHECK-NEXT:    v_subrev_i32_e32 v4, vcc, 1, v1
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, v0, v2
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
-; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v5
-; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v1, v3, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; CHECK-NEXT:    v_subrev_i32_e64 v2, s[4:5], s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, 1, v1
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %result = udiv i32 %num, 1235195
   ret i32 %result
 }
 
 define <2 x i32> @v_udiv_v2i32_oddk_denom(<2 x i32> %num) {
-; CHECK-LABEL: v_udiv_v2i32_oddk_denom:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_mov_b32 s8, 0x12d8fb
-; CHECK-NEXT:    v_mov_b32_e32 v2, 0x12d8fb
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, s8
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; CHECK-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; CHECK-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; CHECK-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
-; CHECK-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; CHECK-NEXT:    v_mul_lo_u32 v5, v3, s8
-; CHECK-NEXT:    v_mul_hi_u32 v6, v3, s8
-; CHECK-NEXT:    v_mul_lo_u32 v7, v4, v2
-; CHECK-NEXT:    v_mul_hi_u32 v8, v4, v2
-; CHECK-NEXT:    v_sub_i32_e32 v9, vcc, 0, v5
-; CHECK-NEXT:    v_sub_i32_e32 v10, vcc, 0, v7
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
-; CHECK-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
-; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
-; CHECK-NEXT:    v_cndmask_b32_e64 v6, v7, v10, s[4:5]
-; CHECK-NEXT:    v_mul_hi_u32 v5, v5, v3
-; CHECK-NEXT:    v_mul_hi_u32 v6, v6, v4
-; CHECK-NEXT:    v_add_i32_e64 v7, s[6:7], v3, v5
-; CHECK-NEXT:    v_sub_i32_e64 v3, s[6:7], v3, v5
-; CHECK-NEXT:    v_add_i32_e64 v5, s[6:7], v4, v6
-; CHECK-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v6
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v4, v4, v5, s[4:5]
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v0
-; CHECK-NEXT:    v_mul_hi_u32 v4, v4, v1
-; CHECK-NEXT:    v_mul_lo_u32 v5, v3, s8
-; CHECK-NEXT:    v_add_i32_e32 v6, vcc, 1, v3
-; CHECK-NEXT:    v_subrev_i32_e32 v7, vcc, 1, v3
-; CHECK-NEXT:    v_mul_lo_u32 v8, v4, v2
-; CHECK-NEXT:    v_add_i32_e32 v9, vcc, 1, v4
-; CHECK-NEXT:    v_subrev_i32_e32 v10, vcc, 1, v4
-; CHECK-NEXT:    v_sub_i32_e32 v11, vcc, v0, v5
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v5
-; CHECK-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v8
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v8
-; CHECK-NEXT:    v_cmp_le_u32_e64 s[6:7], s8, v11
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[8:9], v0, v2
-; CHECK-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v3, v6, s[6:7]
-; CHECK-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e64 v1, v4, v9, s[6:7]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v1, v10, v1, s[4:5]
-; CHECK-NEXT:    s_setpc_b64 s[30:31]
+; GISEL-LABEL: v_udiv_v2i32_oddk_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_mov_b32 s8, 0x12d8fb
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v2, s8
+; GISEL-NEXT:    s_sub_i32 s4, 0, s8
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GISEL-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v2
+; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GISEL-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GISEL-NEXT:    v_mul_lo_u32 v5, s4, v2
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT:    v_mul_hi_u32 v5, v2, v5
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GISEL-NEXT:    v_mul_hi_u32 v3, v0, v3
+; GISEL-NEXT:    v_mul_hi_u32 v2, v1, v2
+; GISEL-NEXT:    v_mul_lo_u32 v4, v3, s8
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
+; GISEL-NEXT:    v_mul_lo_u32 v6, v2, s8
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v2
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v6
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GISEL-NEXT:    v_subrev_i32_e64 v4, s[4:5], s8, v0
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s8, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v2, v2, v7, s[4:5]
+; GISEL-NEXT:    v_subrev_i32_e64 v5, s[6:7], s8, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, 1, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, 1, v2
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v3, v4, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s8, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v2, v5, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_udiv_v2i32_oddk_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_mov_b32 s4, 0x12d8fb
+; CGP-NEXT:    v_mov_b32_e32 v2, 0x12d8fb
+; CGP-NEXT:    s_mov_b32 s5, 0xffed2705
+; CGP-NEXT:    v_cvt_f32_u32_e32 v3, s4
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, v2
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
+; CGP-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_mul_lo_u32 v5, s5, v3
+; CGP-NEXT:    v_mul_lo_u32 v6, s5, v4
+; CGP-NEXT:    v_mul_hi_u32 v5, v3, v5
+; CGP-NEXT:    v_mul_hi_u32 v6, v4, v6
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CGP-NEXT:    v_mul_hi_u32 v3, v0, v3
+; CGP-NEXT:    v_mul_hi_u32 v4, v1, v4
+; CGP-NEXT:    v_mul_lo_u32 v5, v3, s4
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, 1, v3
+; CGP-NEXT:    v_mul_lo_u32 v7, v4, v2
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v7
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; CGP-NEXT:    v_subrev_i32_e64 v5, s[4:5], s4, v0
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v2
+; CGP-NEXT:    v_cndmask_b32_e64 v4, v4, v8, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v6, s[6:7], v1, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v1, v6, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v4, v6, vcc
+; CGP-NEXT:    s_setpc_b64 s[30:31]
   %result = udiv <2 x i32> %num, <i32 1235195, i32 1235195>
   ret <2 x i32> %result
 }
@@ -469,28 +478,24 @@ define i32 @v_udiv_i32_pow2_shl_denom(i32 %x, i32 %y) {
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    v_lshl_b32_e32 v1, 0x1000, v1
 ; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, v1
+; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; CHECK-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; CHECK-NEXT:    v_mul_lo_u32 v3, v2, v1
-; CHECK-NEXT:    v_mul_hi_u32 v4, v2, v1
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v2
-; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v2, v3
-; CHECK-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v2, v2, v0
+; CHECK-NEXT:    v_mul_lo_u32 v3, v3, v2
+; CHECK-NEXT:    v_mul_hi_u32 v3, v2, v3
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; CHECK-NEXT:    v_mul_hi_u32 v2, v0, v2
 ; CHECK-NEXT:    v_mul_lo_u32 v3, v2, v1
 ; CHECK-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; CHECK-NEXT:    v_subrev_i32_e32 v5, vcc, 1, v2
-; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v0, v3
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v1
-; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v3, s[4:5], v0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, 1, v2
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %shl.y = shl i32 4096, %y
   %r = udiv i32 %x, %shl.y
@@ -505,127 +510,105 @@ define <2 x i32> @v_udiv_v2i32_pow2_shl_denom(<2 x i32> %x, <2 x i32> %y) {
 ; GISEL-NEXT:    v_lshl_b32_e32 v2, s4, v2
 ; GISEL-NEXT:    v_lshl_b32_e32 v3, s4, v3
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v3
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v6, v3
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v3
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; GISEL-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
+; GISEL-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GISEL-NEXT:    v_mul_lo_u32 v6, v4, v2
-; GISEL-NEXT:    v_mul_hi_u32 v7, v4, v2
-; GISEL-NEXT:    v_mul_lo_u32 v8, v5, v3
-; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v3
-; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, 0, v6
-; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, 0, v8
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, v8, v11, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v6, v6, v4
-; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v5
-; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v4, v6
-; GISEL-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v6
-; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v5, v7
-; GISEL-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v4, v4, v0
-; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v1
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v4
+; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v6
+; GISEL-NEXT:    v_mul_hi_u32 v5, v4, v5
+; GISEL-NEXT:    v_mul_hi_u32 v7, v6, v7
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v6, v7
+; GISEL-NEXT:    v_mul_hi_u32 v4, v0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v5, v1, v5
 ; GISEL-NEXT:    v_mul_lo_u32 v6, v4, v2
 ; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
-; GISEL-NEXT:    v_subrev_i32_e32 v8, vcc, 1, v4
-; GISEL-NEXT:    v_mul_lo_u32 v9, v5, v3
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, 1, v5
-; GISEL-NEXT:    v_subrev_i32_e32 v11, vcc, 1, v5
-; GISEL-NEXT:    v_sub_i32_e32 v12, vcc, v0, v6
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v6
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v9
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v9
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[6:7], v12, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[8:9], v0, v3
-; GISEL-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v4, v7, s[6:7]
-; GISEL-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v5, v10, s[6:7]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v11, v1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v8, v5, v3
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, 1, v5
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v8
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v6, s[4:5], v0, v2
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v7, s[6:7], v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v5
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v6, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v7, vcc
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; CGP-LABEL: v_udiv_v2i32_pow2_shl_denom:
 ; CGP:       ; %bb.0:
 ; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CGP-NEXT:    s_movk_i32 s4, 0x1000
-; CGP-NEXT:    v_mul_lo_u32 v4, 0, v0
-; CGP-NEXT:    v_mul_lo_u32 v5, 0, v1
+; CGP-NEXT:    v_mul_lo_u32 v4, v0, 0
+; CGP-NEXT:    v_mul_lo_u32 v5, v1, 0
 ; CGP-NEXT:    v_lshl_b32_e32 v2, s4, v2
 ; CGP-NEXT:    v_lshl_b32_e32 v3, s4, v3
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v6, v2
-; CGP-NEXT:    v_mul_lo_u32 v7, 0, v2
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, 0, v2
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v8, v3
-; CGP-NEXT:    v_mul_lo_u32 v9, 0, v3
+; CGP-NEXT:    v_sub_i32_e32 v9, vcc, 0, v3
 ; CGP-NEXT:    v_rcp_f32_e32 v6, v6
 ; CGP-NEXT:    v_rcp_f32_e32 v8, v8
-; CGP-NEXT:    v_mul_f32_e32 v6, 0x4f800000, v6
-; CGP-NEXT:    v_mul_f32_e32 v8, 0x4f800000, v8
+; CGP-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; CGP-NEXT:    v_mul_f32_e32 v8, 0x4f7ffffe, v8
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v8, v8
-; CGP-NEXT:    v_mul_lo_u32 v10, v6, v2
-; CGP-NEXT:    v_mul_lo_u32 v11, v6, 0
-; CGP-NEXT:    v_mul_hi_u32 v12, v6, v2
-; CGP-NEXT:    v_mul_lo_u32 v13, v8, v3
-; CGP-NEXT:    v_mul_lo_u32 v14, v8, 0
-; CGP-NEXT:    v_mul_hi_u32 v15, v8, v3
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v11
-; CGP-NEXT:    v_sub_i32_e32 v16, vcc, 0, v10
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v14
-; CGP-NEXT:    v_sub_i32_e32 v17, vcc, 0, v13
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v12
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; CGP-NEXT:    v_cndmask_b32_e32 v7, v10, v16, vcc
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v9
-; CGP-NEXT:    v_cndmask_b32_e64 v9, v13, v17, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v10, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v7, v7, v6
-; CGP-NEXT:    v_mul_lo_u32 v12, v9, 0
-; CGP-NEXT:    v_mul_hi_u32 v9, v9, v8
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v11, v10
-; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v14, v12
-; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v10, v7
-; CGP-NEXT:    v_add_i32_e64 v9, s[6:7], v11, v9
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v6, v7
-; CGP-NEXT:    v_sub_i32_e64 v6, s[6:7], v6, v7
-; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v8, v9
-; CGP-NEXT:    v_sub_i32_e64 v8, s[6:7], v8, v9
-; CGP-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v7, v8, v7, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v8, v6, 0
-; CGP-NEXT:    v_mul_hi_u32 v6, v6, v0
-; CGP-NEXT:    v_mul_lo_u32 v9, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v7, v7, v1
-; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT:    v_mul_lo_u32 v7, v7, v6
+; CGP-NEXT:    v_mul_lo_u32 v10, v6, 0
+; CGP-NEXT:    v_mul_lo_u32 v9, v9, v8
+; CGP-NEXT:    v_mul_lo_u32 v11, v8, 0
+; CGP-NEXT:    v_mul_lo_u32 v12, 0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v6, v7
+; CGP-NEXT:    v_mul_lo_u32 v13, 0, v9
+; CGP-NEXT:    v_mul_hi_u32 v9, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v8, v9
+; CGP-NEXT:    v_mul_lo_u32 v8, 0, v6
+; CGP-NEXT:    v_mul_hi_u32 v6, v0, v6
+; CGP-NEXT:    v_mul_lo_u32 v9, 0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v1, v7
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
 ; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
 ; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
 ; CGP-NEXT:    v_mul_lo_u32 v6, v4, v2
 ; CGP-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
-; CGP-NEXT:    v_subrev_i32_e32 v8, vcc, 1, v4
-; CGP-NEXT:    v_mul_lo_u32 v9, v5, v3
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, 1, v5
-; CGP-NEXT:    v_subrev_i32_e32 v11, vcc, 1, v5
-; CGP-NEXT:    v_sub_i32_e32 v12, vcc, v0, v6
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v6
-; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v9
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v9
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v12, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[8:9], v0, v3
-; CGP-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v4, v7, s[6:7]
-; CGP-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v5, v10, s[6:7]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v11, v1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v8, v5, v3
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, 1, v5
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v8
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; CGP-NEXT:    v_sub_i32_e64 v6, s[4:5], v0, v2
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v7, s[6:7], v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, 1, v5
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v4, v6, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v5, v7, vcc
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
   %r = udiv <2 x i32> %x, %shl.y
@@ -640,28 +623,24 @@ define i32 @v_udiv_i32_24bit(i32 %num, i32 %den) {
 ; GISEL-NEXT:    v_and_b32_e32 v0, s4, v0
 ; GISEL-NEXT:    v_and_b32_e32 v1, s4, v1
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v2, v1
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GISEL-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GISEL-NEXT:    v_mul_hi_u32 v4, v2, v1
-; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v3, v3, v2
-; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v2, v3
-; GISEL-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v2, v2, v0
+; GISEL-NEXT:    v_mul_lo_u32 v3, v3, v2
+; GISEL-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GISEL-NEXT:    v_mul_hi_u32 v2, v0, v2
 ; GISEL-NEXT:    v_mul_lo_u32 v3, v2, v1
 ; GISEL-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; GISEL-NEXT:    v_subrev_i32_e32 v5, vcc, 1, v2
-; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, v0, v3
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v1
-; GISEL-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, 1, v2
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; CGP-LABEL: v_udiv_i32_24bit:
@@ -671,39 +650,32 @@ define i32 @v_udiv_i32_24bit(i32 %num, i32 %den) {
 ; CGP-NEXT:    v_and_b32_e32 v0, s4, v0
 ; CGP-NEXT:    v_and_b32_e32 v1, s4, v1
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v2, v1
-; CGP-NEXT:    v_mul_lo_u32 v3, 0, v1
-; CGP-NEXT:    v_mul_lo_u32 v4, 0, v0
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; CGP-NEXT:    v_mul_lo_u32 v4, v0, 0
 ; CGP-NEXT:    v_rcp_f32_e32 v2, v2
-; CGP-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; CGP-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, v2, v1
-; CGP-NEXT:    v_mul_lo_u32 v6, v2, 0
-; CGP-NEXT:    v_mul_hi_u32 v7, v2, v1
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; CGP-NEXT:    v_sub_i32_e32 v8, vcc, 0, v5
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; CGP-NEXT:    v_cndmask_b32_e32 v3, v5, v8, vcc
-; CGP-NEXT:    v_mul_lo_u32 v5, v3, 0
-; CGP-NEXT:    v_mul_hi_u32 v3, v3, v2
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v6, v5
-; CGP-NEXT:    v_add_i32_e64 v3, s[4:5], v5, v3
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v2, v3
-; CGP-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; CGP-NEXT:    v_mul_lo_u32 v3, v2, 0
-; CGP-NEXT:    v_mul_hi_u32 v2, v2, v0
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; CGP-NEXT:    v_mul_lo_u32 v3, v3, v2
+; CGP-NEXT:    v_mul_lo_u32 v5, v2, 0
+; CGP-NEXT:    v_mul_lo_u32 v6, 0, v3
+; CGP-NEXT:    v_mul_hi_u32 v3, v2, v3
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT:    v_mul_lo_u32 v3, 0, v2
+; CGP-NEXT:    v_mul_hi_u32 v2, v0, v2
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
 ; CGP-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; CGP-NEXT:    v_mul_lo_u32 v3, v2, v1
 ; CGP-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; CGP-NEXT:    v_subrev_i32_e32 v5, vcc, 1, v2
-; CGP-NEXT:    v_sub_i32_e32 v6, vcc, v0, v3
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v1
-; CGP-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, 1, v2
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %num.mask = and i32 %num, 16777215
   %den.mask = and i32 %den, 16777215
@@ -721,51 +693,43 @@ define <2 x i32> @v_udiv_v2i32_24bit(<2 x i32> %num, <2 x i32> %den) {
 ; GISEL-NEXT:    v_and_b32_e32 v2, s4, v2
 ; GISEL-NEXT:    v_and_b32_e32 v3, s4, v3
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v3
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v6, v3
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v3
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; GISEL-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
+; GISEL-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GISEL-NEXT:    v_mul_lo_u32 v6, v4, v2
-; GISEL-NEXT:    v_mul_hi_u32 v7, v4, v2
-; GISEL-NEXT:    v_mul_lo_u32 v8, v5, v3
-; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v3
-; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, 0, v6
-; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, 0, v8
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, v8, v11, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v6, v6, v4
-; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v5
-; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v4, v6
-; GISEL-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v6
-; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v5, v7
-; GISEL-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v4, v4, v0
-; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v1
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v4
+; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v6
+; GISEL-NEXT:    v_mul_hi_u32 v5, v4, v5
+; GISEL-NEXT:    v_mul_hi_u32 v7, v6, v7
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v6, v7
+; GISEL-NEXT:    v_mul_hi_u32 v4, v0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v5, v1, v5
 ; GISEL-NEXT:    v_mul_lo_u32 v6, v4, v2
 ; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
-; GISEL-NEXT:    v_subrev_i32_e32 v8, vcc, 1, v4
-; GISEL-NEXT:    v_mul_lo_u32 v9, v5, v3
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, 1, v5
-; GISEL-NEXT:    v_subrev_i32_e32 v11, vcc, 1, v5
-; GISEL-NEXT:    v_sub_i32_e32 v12, vcc, v0, v6
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v6
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v9
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v9
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[6:7], v12, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[8:9], v0, v3
-; GISEL-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v4, v7, s[6:7]
-; GISEL-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v5, v10, s[6:7]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v11, v1, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v8, v5, v3
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, 1, v5
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v8
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v6, s[4:5], v0, v2
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v7, s[6:7], v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, 1, v5
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v6, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v7, vcc
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; CGP-LABEL: v_udiv_v2i32_24bit:
@@ -777,73 +741,59 @@ define <2 x i32> @v_udiv_v2i32_24bit(<2 x i32> %num, <2 x i32> %den) {
 ; CGP-NEXT:    v_and_b32_e32 v2, s4, v2
 ; CGP-NEXT:    v_and_b32_e32 v3, s4, v3
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, 0, v2
-; CGP-NEXT:    v_mul_lo_u32 v6, 0, v0
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; CGP-NEXT:    v_mul_lo_u32 v6, v0, 0
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v7, v3
-; CGP-NEXT:    v_mul_lo_u32 v8, 0, v3
-; CGP-NEXT:    v_mul_lo_u32 v9, 0, v1
+; CGP-NEXT:    v_sub_i32_e32 v8, vcc, 0, v3
+; CGP-NEXT:    v_mul_lo_u32 v9, v1, 0
 ; CGP-NEXT:    v_rcp_f32_e32 v4, v4
 ; CGP-NEXT:    v_rcp_f32_e32 v7, v7
-; CGP-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f800000, v7
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; CGP-NEXT:    v_mul_lo_u32 v10, v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v11, v4, 0
-; CGP-NEXT:    v_mul_hi_u32 v12, v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v13, v7, v3
-; CGP-NEXT:    v_mul_lo_u32 v14, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v15, v7, v3
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v11
-; CGP-NEXT:    v_sub_i32_e32 v16, vcc, 0, v10
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v14
-; CGP-NEXT:    v_sub_i32_e32 v17, vcc, 0, v13
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v12
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v15
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; CGP-NEXT:    v_cndmask_b32_e32 v5, v10, v16, vcc
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
-; CGP-NEXT:    v_cndmask_b32_e64 v8, v13, v17, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v10, v5, 0
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v4
-; CGP-NEXT:    v_mul_lo_u32 v12, v8, 0
-; CGP-NEXT:    v_mul_hi_u32 v8, v8, v7
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v11, v10
-; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v14, v12
-; CGP-NEXT:    v_add_i32_e64 v5, s[6:7], v10, v5
-; CGP-NEXT:    v_add_i32_e64 v8, s[6:7], v11, v8
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v4, v5
-; CGP-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v5
-; CGP-NEXT:    v_add_i32_e64 v5, s[6:7], v7, v8
-; CGP-NEXT:    v_sub_i32_e64 v7, s[6:7], v7, v8
-; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v7, v4, 0
-; CGP-NEXT:    v_mul_hi_u32 v4, v4, v0
-; CGP-NEXT:    v_mul_lo_u32 v8, v5, 0
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v1
-; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v9, v8
+; CGP-NEXT:    v_mul_lo_u32 v5, v5, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v4, 0
+; CGP-NEXT:    v_mul_lo_u32 v8, v8, v7
+; CGP-NEXT:    v_mul_lo_u32 v11, v7, 0
+; CGP-NEXT:    v_mul_lo_u32 v12, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v4, v5
+; CGP-NEXT:    v_mul_lo_u32 v13, 0, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v7, v8
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v10, v5
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v7, v8
+; CGP-NEXT:    v_mul_lo_u32 v7, 0, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v0, v4
+; CGP-NEXT:    v_mul_lo_u32 v8, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v1, v5
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v8, v9
 ; CGP-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
 ; CGP-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
 ; CGP-NEXT:    v_mul_lo_u32 v6, v4, v2
 ; CGP-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
-; CGP-NEXT:    v_subrev_i32_e32 v8, vcc, 1, v4
-; CGP-NEXT:    v_mul_lo_u32 v9, v5, v3
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, 1, v5
-; CGP-NEXT:    v_subrev_i32_e32 v11, vcc, 1, v5
-; CGP-NEXT:    v_sub_i32_e32 v12, vcc, v0, v6
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v6
-; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v1, v9
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v9
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v12, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[8:9], v0, v3
-; CGP-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v4, v7, s[6:7]
-; CGP-NEXT:    s_and_b64 s[6:7], s[8:9], s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v5, v10, s[6:7]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v11, v1, s[4:5]
+; CGP-NEXT:    v_mul_lo_u32 v8, v5, v3
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, 1, v5
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v8
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; CGP-NEXT:    v_sub_i32_e64 v6, s[4:5], v0, v2
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v7, s[6:7], v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, 1, v5
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v4, v6, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v5, v7, vcc
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %num.mask = and <2 x i32> %num, <i32 16777215, i32 16777215>
   %den.mask = and <2 x i32> %den, <i32 16777215, i32 16777215>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
index 219bcce04da14..e956af93bc6f1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
@@ -151,28 +151,24 @@ define i64 @v_udiv_i64(i64 %num, i64 %den) {
 ; CHECK-NEXT:    s_cbranch_execz BB0_4
 ; CHECK-NEXT:  ; %bb.3:
 ; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, v2
+; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, 0, v2
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
+; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
 ; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; CHECK-NEXT:    v_mul_lo_u32 v3, v1, v2
-; CHECK-NEXT:    v_mul_hi_u32 v4, v1, v2
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v1
-; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v1, v3
-; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v1, v1, v0
+; CHECK-NEXT:    v_mul_lo_u32 v3, v3, v1
+; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; CHECK-NEXT:    v_mul_lo_u32 v3, v1, v2
 ; CHECK-NEXT:    v_add_i32_e32 v4, vcc, 1, v1
-; CHECK-NEXT:    v_subrev_i32_e32 v5, vcc, 1, v1
-; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v0, v3
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v2
-; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v1, v4, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v4, v5, v0, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v3, s[4:5], v0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, v1, v3, vcc
 ; CHECK-NEXT:    v_mov_b32_e32 v5, 0
 ; CHECK-NEXT:  BB0_4:
 ; CHECK-NEXT:    s_or_b64 exec, exec, s[6:7]
@@ -335,28 +331,24 @@ define amdgpu_ps i64 @s_udiv_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-NEXT:    s_cbranch_scc0 BB1_4
 ; CHECK-NEXT:  ; %bb.3:
 ; CHECK-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; CHECK-NEXT:    s_sub_i32 s1, 0, s2
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; CHECK-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; CHECK-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; CHECK-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT:    v_mul_lo_u32 v1, s1, v0
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT:    v_mul_hi_u32 v0, s0, v0
 ; CHECK-NEXT:    v_mul_lo_u32 v1, v0, s2
-; CHECK-NEXT:    v_mul_hi_u32 v2, v0, s2
-; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v1, v1, v0
-; CHECK-NEXT:    v_add_i32_e64 v2, s[4:5], v0, v1
-; CHECK-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v1
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, s0, v1
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s2, v1
 ; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v0, v0, s0
-; CHECK-NEXT:    v_mul_lo_u32 v1, v0, s2
+; CHECK-NEXT:    v_subrev_i32_e64 v2, s[0:1], s2, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
 ; CHECK-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
-; CHECK-NEXT:    v_subrev_i32_e32 v3, vcc, 1, v0
-; CHECK-NEXT:    v_sub_i32_e32 v4, vcc, s0, v1
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, s0, v1
-; CHECK-NEXT:    v_cmp_le_u32_e64 s[0:1], s2, v4
-; CHECK-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s2, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CHECK-NEXT:  BB1_4:
 ; CHECK-NEXT:    v_readfirstlane_b32 s0, v0
 ; CHECK-NEXT:    s_mov_b32 s1, s0
@@ -781,28 +773,24 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
 ; CGP-NEXT:    s_cbranch_execz BB2_4
 ; CGP-NEXT:  ; %bb.3:
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v4
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, 0, v4
 ; CGP-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_lo_u32 v1, v1, v0
+; CGP-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; CGP-NEXT:    v_mul_hi_u32 v0, v8, v0
 ; CGP-NEXT:    v_mul_lo_u32 v1, v0, v4
-; CGP-NEXT:    v_mul_hi_u32 v5, v0, v4
-; CGP-NEXT:    v_sub_i32_e32 v9, vcc, 0, v1
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; CGP-NEXT:    v_mul_hi_u32 v1, v1, v0
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v0, v1
-; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v1
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v0
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v8, v1
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v4
 ; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
-; CGP-NEXT:    v_mul_hi_u32 v0, v0, v8
-; CGP-NEXT:    v_mul_lo_u32 v1, v0, v4
+; CGP-NEXT:    v_sub_i32_e64 v5, s[4:5], v1, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v0
-; CGP-NEXT:    v_subrev_i32_e32 v9, vcc, 1, v0
-; CGP-NEXT:    v_sub_i32_e32 v10, vcc, v8, v1
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v8, v1
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v4
-; CGP-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v0, v5, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
 ; CGP-NEXT:    v_mov_b32_e32 v1, 0
 ; CGP-NEXT:  BB2_4:
 ; CGP-NEXT:    s_or_b64 exec, exec, s[6:7]
@@ -949,28 +937,24 @@ define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
 ; CGP-NEXT:    s_cbranch_execz BB2_8
 ; CGP-NEXT:  ; %bb.7:
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v3, v6
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, 0, v6
 ; CGP-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; CGP-NEXT:    v_mul_lo_u32 v4, v3, v6
-; CGP-NEXT:    v_mul_hi_u32 v5, v3, v6
-; CGP-NEXT:    v_sub_i32_e32 v7, vcc, 0, v4
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
-; CGP-NEXT:    v_mul_hi_u32 v4, v4, v3
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v3, v4
-; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
-; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; CGP-NEXT:    v_mul_hi_u32 v3, v3, v2
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, v3
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v3, v2, v3
 ; CGP-NEXT:    v_mul_lo_u32 v4, v3, v6
 ; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
-; CGP-NEXT:    v_subrev_i32_e32 v7, vcc, 1, v3
-; CGP-NEXT:    v_sub_i32_e32 v8, vcc, v2, v4
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v4
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v6
-; CGP-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v2, v3, v5, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v4, v7, v2, vcc
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v4
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v6
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; CGP-NEXT:    v_sub_i32_e64 v4, s[4:5], v2, v6
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, 1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v6
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v3, v4, vcc
 ; CGP-NEXT:    v_mov_b32_e32 v5, 0
 ; CGP-NEXT:  BB2_8:
 ; CGP-NEXT:    s_or_b64 exec, exec, s[6:7]
@@ -2453,28 +2437,24 @@ define i64 @v_udiv_i64_pow2_shl_denom(i64 %x, i64 %y) {
 ; CHECK-NEXT:    s_cbranch_execz BB7_4
 ; CHECK-NEXT:  ; %bb.3:
 ; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, v4
+; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, 0, v4
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
+; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
 ; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; CHECK-NEXT:    v_mul_lo_u32 v2, v1, v4
-; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v4
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v2, v2, v1
-; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v1, v2
-; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v2
-; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v1, v1, v0
+; CHECK-NEXT:    v_mul_lo_u32 v2, v2, v1
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; CHECK-NEXT:    v_mul_lo_u32 v2, v1, v4
 ; CHECK-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
-; CHECK-NEXT:    v_subrev_i32_e32 v5, vcc, 1, v1
-; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v0, v2
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v4
-; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v1, v3, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v2, v5, v0, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v2, s[4:5], v0, v4
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, 1, v1
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
 ; CHECK-NEXT:    v_mov_b32_e32 v3, 0
 ; CHECK-NEXT:  BB7_4:
 ; CHECK-NEXT:    s_or_b64 exec, exec, s[6:7]
@@ -2902,28 +2882,24 @@ define <2 x i64> @v_udiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
 ; CGP-NEXT:    s_cbranch_execz BB8_4
 ; CGP-NEXT:  ; %bb.3:
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v10
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, 0, v10
 ; CGP-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_lo_u32 v1, v1, v0
+; CGP-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; CGP-NEXT:    v_mul_hi_u32 v0, v5, v0
 ; CGP-NEXT:    v_mul_lo_u32 v1, v0, v10
-; CGP-NEXT:    v_mul_hi_u32 v4, v0, v10
-; CGP-NEXT:    v_sub_i32_e32 v6, vcc, 0, v1
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
-; CGP-NEXT:    v_mul_hi_u32 v1, v1, v0
-; CGP-NEXT:    v_add_i32_e64 v4, s[4:5], v0, v1
-; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v1
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v5, v1
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v10
 ; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; CGP-NEXT:    v_mul_hi_u32 v0, v0, v5
-; CGP-NEXT:    v_mul_lo_u32 v1, v0, v10
+; CGP-NEXT:    v_sub_i32_e64 v4, s[4:5], v1, v10
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; CGP-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; CGP-NEXT:    v_subrev_i32_e32 v6, vcc, 1, v0
-; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v5, v1
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v5, v1
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v10
-; CGP-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v10
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
 ; CGP-NEXT:    v_mov_b32_e32 v1, 0
 ; CGP-NEXT:  BB8_4:
 ; CGP-NEXT:    s_or_b64 exec, exec, s[6:7]
@@ -3070,28 +3046,24 @@ define <2 x i64> @v_udiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
 ; CGP-NEXT:    s_cbranch_execz BB8_8
 ; CGP-NEXT:  ; %bb.7:
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v3, v8
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, 0, v8
 ; CGP-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; CGP-NEXT:    v_mul_lo_u32 v4, v3, v8
-; CGP-NEXT:    v_mul_hi_u32 v5, v3, v8
-; CGP-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; CGP-NEXT:    v_mul_hi_u32 v4, v4, v3
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v3, v4
-; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
-; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; CGP-NEXT:    v_mul_hi_u32 v3, v3, v2
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, v3
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v3, v2, v3
 ; CGP-NEXT:    v_mul_lo_u32 v4, v3, v8
 ; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
-; CGP-NEXT:    v_subrev_i32_e32 v6, vcc, 1, v3
-; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v2, v4
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v4
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v8
-; CGP-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v2, v3, v5, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v4
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v8
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; CGP-NEXT:    v_sub_i32_e64 v4, s[4:5], v2, v8
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, 1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v8
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v3, v4, vcc
 ; CGP-NEXT:    v_mov_b32_e32 v5, 0
 ; CGP-NEXT:  BB8_8:
 ; CGP-NEXT:    s_or_b64 exec, exec, s[6:7]
@@ -3111,28 +3083,24 @@ define i64 @v_udiv_i64_24bit(i64 %num, i64 %den) {
 ; GISEL-NEXT:    v_and_b32_e32 v0, s4, v0
 ; GISEL-NEXT:    v_and_b32_e32 v1, s4, v2
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v2, v1
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GISEL-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GISEL-NEXT:    v_mul_hi_u32 v4, v2, v1
-; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v3, v3, v2
-; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v2, v3
-; GISEL-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v2, v2, v0
+; GISEL-NEXT:    v_mul_lo_u32 v3, v3, v2
+; GISEL-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GISEL-NEXT:    v_mul_hi_u32 v2, v0, v2
 ; GISEL-NEXT:    v_mul_lo_u32 v3, v2, v1
 ; GISEL-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; GISEL-NEXT:    v_subrev_i32_e32 v5, vcc, 1, v2
-; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, v0, v3
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v1
-; GISEL-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, 1, v2
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; GISEL-NEXT:    v_mov_b32_e32 v1, 0
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
index 45ce6cdf42106..68a83a91c62f8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
@@ -9,67 +9,52 @@ define i32 @v_urem_i32(i32 %num, i32 %den) {
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v2, v1
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GISEL-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GISEL-NEXT:    v_mul_hi_u32 v4, v2, v1
-; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v3, v3, v2
-; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v2, v3
-; GISEL-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v2, v2, v0
+; GISEL-NEXT:    v_mul_lo_u32 v3, v3, v2
+; GISEL-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GISEL-NEXT:    v_mul_hi_u32 v2, v0, v2
 ; GISEL-NEXT:    v_mul_lo_u32 v2, v2, v1
-; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, v0, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v1
-; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v3, v1
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v2
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[6:7], v3, v1
-; GISEL-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; CGP-LABEL: v_urem_i32:
 ; CGP:       ; %bb.0:
 ; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v2, v1
-; CGP-NEXT:    v_mul_lo_u32 v3, 0, v1
-; CGP-NEXT:    v_mul_lo_u32 v4, 0, v0
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; CGP-NEXT:    v_mul_lo_u32 v4, v0, 0
 ; CGP-NEXT:    v_rcp_f32_e32 v2, v2
-; CGP-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; CGP-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, v2, v1
-; CGP-NEXT:    v_mul_lo_u32 v6, v2, 0
-; CGP-NEXT:    v_mul_hi_u32 v7, v2, v1
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; CGP-NEXT:    v_sub_i32_e32 v8, vcc, 0, v5
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; CGP-NEXT:    v_cndmask_b32_e32 v3, v5, v8, vcc
-; CGP-NEXT:    v_mul_lo_u32 v5, v3, 0
-; CGP-NEXT:    v_mul_hi_u32 v3, v3, v2
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v6, v5
-; CGP-NEXT:    v_add_i32_e64 v3, s[4:5], v5, v3
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v2, v3
-; CGP-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; CGP-NEXT:    v_mul_lo_u32 v3, v2, 0
-; CGP-NEXT:    v_mul_hi_u32 v2, v2, v0
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; CGP-NEXT:    v_mul_lo_u32 v3, v3, v2
+; CGP-NEXT:    v_mul_lo_u32 v5, v2, 0
+; CGP-NEXT:    v_mul_lo_u32 v6, 0, v3
+; CGP-NEXT:    v_mul_hi_u32 v3, v2, v3
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT:    v_mul_lo_u32 v3, 0, v2
+; CGP-NEXT:    v_mul_hi_u32 v2, v0, v2
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
 ; CGP-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; CGP-NEXT:    v_mul_lo_u32 v2, v2, v1
-; CGP-NEXT:    v_sub_i32_e32 v3, vcc, v0, v2
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v1
-; CGP-NEXT:    v_add_i32_e64 v4, s[4:5], v3, v1
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v2
-; CGP-NEXT:    v_sub_i32_e64 v0, s[6:7], v3, v1
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[4:5]
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %result = urem i32 %num, %den
   ret i32 %result
@@ -82,71 +67,53 @@ define amdgpu_ps i32 @s_urem_i32(i32 inreg %num, i32 inreg %den) {
 ; GISEL-LABEL: s_urem_i32:
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v0, s1
+; GISEL-NEXT:    s_sub_i32 s2, 0, s1
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GISEL-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GISEL-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GISEL-NEXT:    v_mul_lo_u32 v1, v0, s1
-; GISEL-NEXT:    v_mul_hi_u32 v2, v0, s1
-; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GISEL-NEXT:    v_add_i32_e64 v2, s[2:3], v0, v1
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[2:3], v0, v1
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v0, v0, s0
+; GISEL-NEXT:    v_mul_lo_u32 v1, s2, v0
+; GISEL-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT:    v_mul_hi_u32 v0, s0, v0
 ; GISEL-NEXT:    v_mul_lo_u32 v0, v0, s1
-; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, s0, v0
-; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s1, v1
-; GISEL-NEXT:    v_add_i32_e64 v2, s[2:3], s1, v1
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], s0, v0
-; GISEL-NEXT:    v_subrev_i32_e64 v0, s[2:3], s1, v1
-; GISEL-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GISEL-NEXT:    v_subrev_i32_e32 v1, vcc, s1, v0
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s1, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v1, vcc, s1, v0
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s1, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; GISEL-NEXT:    v_readfirstlane_b32 s0, v0
 ; GISEL-NEXT:    ; return to shader part epilog
 ;
 ; CGP-LABEL: s_urem_i32:
 ; CGP:       ; %bb.0:
-; CGP-NEXT:    s_mov_b32 s4, s1
-; CGP-NEXT:    v_cvt_f32_u32_e32 v0, s4
-; CGP-NEXT:    s_bfe_u64 s[2:3], s[4:5], 0x200000
-; CGP-NEXT:    s_bfe_u64 s[6:7], s[0:1], 0x200000
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, s1
+; CGP-NEXT:    s_sub_i32 s4, 0, s1
+; CGP-NEXT:    s_bfe_u64 s[2:3], s[0:1], 0x200000
 ; CGP-NEXT:    v_rcp_f32_e32 v0, v0
-; CGP-NEXT:    v_mul_lo_u32 v1, 0, s2
-; CGP-NEXT:    v_mul_lo_u32 v2, 0, s6
-; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; CGP-NEXT:    v_mul_lo_u32 v1, s2, 0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; CGP-NEXT:    v_mul_lo_u32 v3, v0, s2
-; CGP-NEXT:    v_mul_lo_u32 v4, v0, s3
-; CGP-NEXT:    v_mul_hi_u32 v5, v0, s2
-; CGP-NEXT:    v_mul_lo_u32 v6, 0, v0
-; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
-; CGP-NEXT:    v_sub_i32_e32 v4, vcc, 0, v3
-; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; CGP-NEXT:    v_mul_lo_u32 v3, v1, 0
-; CGP-NEXT:    v_mul_hi_u32 v1, v1, v0
-; CGP-NEXT:    v_add_i32_e64 v3, s[2:3], v6, v3
-; CGP-NEXT:    v_add_i32_e64 v1, s[2:3], v3, v1
-; CGP-NEXT:    v_add_i32_e64 v3, s[2:3], v0, v1
-; CGP-NEXT:    v_sub_i32_e64 v0, s[2:3], v0, v1
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; CGP-NEXT:    v_mul_lo_u32 v1, v0, s7
-; CGP-NEXT:    v_mul_hi_u32 v0, v0, s6
+; CGP-NEXT:    v_mul_lo_u32 v2, s4, v0
+; CGP-NEXT:    v_mul_lo_u32 v3, v0, 0
+; CGP-NEXT:    v_mul_lo_u32 v4, 0, v2
+; CGP-NEXT:    v_mul_hi_u32 v2, v0, v2
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; CGP-NEXT:    v_mul_lo_u32 v2, s3, v0
+; CGP-NEXT:    v_mul_hi_u32 v0, s2, v0
 ; CGP-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; CGP-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
-; CGP-NEXT:    v_mul_lo_u32 v0, v0, s4
-; CGP-NEXT:    v_sub_i32_e32 v1, vcc, s0, v0
-; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s4, v1
-; CGP-NEXT:    v_add_i32_e64 v2, s[2:3], s4, v1
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[0:1], s0, v0
-; CGP-NEXT:    v_subrev_i32_e64 v0, s[2:3], s4, v1
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[0:1]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[0:1]
+; CGP-NEXT:    v_mul_lo_u32 v0, v0, s1
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; CGP-NEXT:    v_subrev_i32_e32 v1, vcc, s1, v0
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s1, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v1, vcc, s1, v0
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s1, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; CGP-NEXT:    v_readfirstlane_b32 s0, v0
 ; CGP-NEXT:    ; return to shader part epilog
   %result = urem i32 %num, %den
@@ -159,124 +126,95 @@ define <2 x i32> @v_urem_v2i32(<2 x i32> %num, <2 x i32> %den) {
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v3
+; GISEL-NEXT:    s_mov_b32 s4, 0x4f7ffffe
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v6, v3
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v3
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; GISEL-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GISEL-NEXT:    v_mul_f32_e32 v4, s4, v4
+; GISEL-NEXT:    v_mul_f32_e32 v6, s4, v6
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GISEL-NEXT:    v_mul_lo_u32 v6, v4, v2
-; GISEL-NEXT:    v_mul_hi_u32 v7, v4, v2
-; GISEL-NEXT:    v_mul_lo_u32 v8, v5, v3
-; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v3
-; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, 0, v6
-; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, 0, v8
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, v8, v11, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v6, v6, v4
-; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v5
-; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v4, v6
-; GISEL-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v6
-; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v5, v7
-; GISEL-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v4, v4, v0
-; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v1
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v4
+; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v6
+; GISEL-NEXT:    v_mul_hi_u32 v5, v4, v5
+; GISEL-NEXT:    v_mul_hi_u32 v7, v6, v7
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v6, v7
+; GISEL-NEXT:    v_mul_hi_u32 v4, v0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v5, v1, v5
 ; GISEL-NEXT:    v_mul_lo_u32 v4, v4, v2
 ; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v3
-; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, v0, v4
-; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, v1, v5
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v6, v2
-; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v6, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v4
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[6:7], v6, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[6:7], v7, v3
-; GISEL-NEXT:    v_add_i32_e64 v2, s[8:9], v7, v3
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v5
-; GISEL-NEXT:    v_sub_i32_e64 v1, s[10:11], v7, v3
-; GISEL-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GISEL-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[8:9]
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, v1, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, v1, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; CGP-LABEL: v_urem_v2i32:
 ; CGP:       ; %bb.0:
 ; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, 0, v2
-; CGP-NEXT:    v_mul_lo_u32 v6, 0, v0
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; CGP-NEXT:    v_mul_lo_u32 v6, v0, 0
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v7, v3
-; CGP-NEXT:    v_mul_lo_u32 v8, 0, v3
-; CGP-NEXT:    v_mul_lo_u32 v9, 0, v1
+; CGP-NEXT:    v_sub_i32_e32 v8, vcc, 0, v3
+; CGP-NEXT:    v_mul_lo_u32 v9, v1, 0
 ; CGP-NEXT:    v_rcp_f32_e32 v4, v4
 ; CGP-NEXT:    v_rcp_f32_e32 v7, v7
-; CGP-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f800000, v7
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; CGP-NEXT:    v_mul_lo_u32 v10, v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v11, v4, 0
-; CGP-NEXT:    v_mul_hi_u32 v12, v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v13, v7, v3
-; CGP-NEXT:    v_mul_lo_u32 v14, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v15, v7, v3
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v11
-; CGP-NEXT:    v_sub_i32_e32 v16, vcc, 0, v10
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v14
-; CGP-NEXT:    v_sub_i32_e32 v17, vcc, 0, v13
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v12
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v15
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; CGP-NEXT:    v_cndmask_b32_e32 v5, v10, v16, vcc
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
-; CGP-NEXT:    v_cndmask_b32_e64 v8, v13, v17, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v10, v5, 0
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v4
-; CGP-NEXT:    v_mul_lo_u32 v12, v8, 0
-; CGP-NEXT:    v_mul_hi_u32 v8, v8, v7
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v11, v10
-; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v14, v12
-; CGP-NEXT:    v_add_i32_e64 v5, s[6:7], v10, v5
-; CGP-NEXT:    v_add_i32_e64 v8, s[6:7], v11, v8
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v4, v5
-; CGP-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v5
-; CGP-NEXT:    v_add_i32_e64 v5, s[6:7], v7, v8
-; CGP-NEXT:    v_sub_i32_e64 v7, s[6:7], v7, v8
-; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v7, v4, 0
-; CGP-NEXT:    v_mul_hi_u32 v4, v4, v0
-; CGP-NEXT:    v_mul_lo_u32 v8, v5, 0
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v1
-; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v9, v8
+; CGP-NEXT:    v_mul_lo_u32 v5, v5, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v4, 0
+; CGP-NEXT:    v_mul_lo_u32 v8, v8, v7
+; CGP-NEXT:    v_mul_lo_u32 v11, v7, 0
+; CGP-NEXT:    v_mul_lo_u32 v12, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v4, v5
+; CGP-NEXT:    v_mul_lo_u32 v13, 0, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v7, v8
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v10, v5
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v7, v8
+; CGP-NEXT:    v_mul_lo_u32 v7, 0, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v0, v4
+; CGP-NEXT:    v_mul_lo_u32 v8, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v1, v5
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v8, v9
 ; CGP-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
 ; CGP-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
 ; CGP-NEXT:    v_mul_lo_u32 v4, v4, v2
 ; CGP-NEXT:    v_mul_lo_u32 v5, v5, v3
-; CGP-NEXT:    v_sub_i32_e32 v6, vcc, v0, v4
-; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v1, v5
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v6, v2
-; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v6, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v4
-; CGP-NEXT:    v_sub_i32_e64 v0, s[6:7], v6, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v7, v3
-; CGP-NEXT:    v_add_i32_e64 v2, s[8:9], v7, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v5
-; CGP-NEXT:    v_sub_i32_e64 v1, s[10:11], v7, v3
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; CGP-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[8:9]
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, v1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, v1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %result = urem <2 x i32> %num, %den
   ret <2 x i32> %result
@@ -287,88 +225,101 @@ define i32 @v_urem_i32_pow2k_denom(i32 %num) {
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_movk_i32 s4, 0x1000
-; CHECK-NEXT:    v_mov_b32_e32 v1, 0x1000
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0xfffff000
 ; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, s4
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; CHECK-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; CHECK-NEXT:    v_lshlrev_b32_e32 v3, 12, v2
-; CHECK-NEXT:    v_mul_hi_u32 v4, v2, s4
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v2
-; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v2, v3
-; CHECK-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v2, v2, v0
-; CHECK-NEXT:    v_lshlrev_b32_e32 v2, 12, v2
-; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, v0, v2
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v1
-; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v3, v1
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v2
-; CHECK-NEXT:    v_sub_i32_e64 v0, s[6:7], v3, v1
-; CHECK-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[4:5]
+; CHECK-NEXT:    v_mul_lo_u32 v1, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v1, v2, v1
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CHECK-NEXT:    v_lshlrev_b32_e32 v1, 12, v1
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT:    v_subrev_i32_e32 v1, vcc, s4, v0
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v1, vcc, s4, v0
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %result = urem i32 %num, 4096
   ret i32 %result
 }
 
 define <2 x i32> @v_urem_v2i32_pow2k_denom(<2 x i32> %num) {
-; CHECK-LABEL: v_urem_v2i32_pow2k_denom:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_movk_i32 s4, 0x1000
-; CHECK-NEXT:    v_mov_b32_e32 v2, 0x1000
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, s4
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; CHECK-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; CHECK-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; CHECK-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
-; CHECK-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; CHECK-NEXT:    v_lshlrev_b32_e32 v5, 12, v3
-; CHECK-NEXT:    v_mul_hi_u32 v6, v3, s4
-; CHECK-NEXT:    v_lshlrev_b32_e32 v7, 12, v4
-; CHECK-NEXT:    v_mul_hi_u32 v8, v4, v2
-; CHECK-NEXT:    v_sub_i32_e32 v9, vcc, 0, v5
-; CHECK-NEXT:    v_sub_i32_e32 v10, vcc, 0, v7
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
-; CHECK-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
-; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
-; CHECK-NEXT:    v_cndmask_b32_e64 v6, v7, v10, s[4:5]
-; CHECK-NEXT:    v_mul_hi_u32 v5, v5, v3
-; CHECK-NEXT:    v_mul_hi_u32 v6, v6, v4
-; CHECK-NEXT:    v_add_i32_e64 v7, s[6:7], v3, v5
-; CHECK-NEXT:    v_sub_i32_e64 v3, s[6:7], v3, v5
-; CHECK-NEXT:    v_add_i32_e64 v5, s[6:7], v4, v6
-; CHECK-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v6
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v4, v4, v5, s[4:5]
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v0
-; CHECK-NEXT:    v_mul_hi_u32 v4, v4, v1
-; CHECK-NEXT:    v_lshlrev_b32_e32 v3, 12, v3
-; CHECK-NEXT:    v_lshlrev_b32_e32 v4, 12, v4
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, v0, v3
-; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v1, v4
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v5, v2
-; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v5, v2
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v3
-; CHECK-NEXT:    v_sub_i32_e64 v0, s[6:7], v5, v2
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[6:7], v6, v2
-; CHECK-NEXT:    v_add_i32_e64 v3, s[8:9], v6, v2
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v4
-; CHECK-NEXT:    v_sub_i32_e64 v1, s[10:11], v6, v2
-; CHECK-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; CHECK-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; CHECK-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v7, v0, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[8:9]
-; CHECK-NEXT:    s_setpc_b64 s[30:31]
+; GISEL-LABEL: v_urem_v2i32_pow2k_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_movk_i32 s4, 0x1000
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v2, s4
+; GISEL-NEXT:    s_sub_i32 s5, 0, s4
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GISEL-NEXT:    v_mul_lo_u32 v3, s5, v2
+; GISEL-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GISEL-NEXT:    v_mul_hi_u32 v3, v0, v2
+; GISEL-NEXT:    v_mul_hi_u32 v2, v1, v2
+; GISEL-NEXT:    v_lshlrev_b32_e32 v3, 12, v3
+; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 12, v2
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; GISEL-NEXT:    v_subrev_i32_e32 v2, vcc, s4, v0
+; GISEL-NEXT:    v_subrev_i32_e32 v3, vcc, s4, v1
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v2, vcc, s4, v0
+; GISEL-NEXT:    v_subrev_i32_e32 v3, vcc, s4, v1
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_urem_v2i32_pow2k_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_movk_i32 s4, 0x1000
+; CGP-NEXT:    v_mov_b32_e32 v2, 0x1000
+; CGP-NEXT:    s_mov_b32 s5, 0x4f7ffffe
+; CGP-NEXT:    s_mov_b32 s6, 0xfffff000
+; CGP-NEXT:    v_cvt_f32_u32_e32 v3, s4
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, v2
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; CGP-NEXT:    v_mul_f32_e32 v3, s5, v3
+; CGP-NEXT:    v_mul_f32_e32 v4, s5, v4
+; CGP-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_mul_lo_u32 v5, s6, v3
+; CGP-NEXT:    v_mul_lo_u32 v6, s6, v4
+; CGP-NEXT:    v_mul_hi_u32 v5, v3, v5
+; CGP-NEXT:    v_mul_hi_u32 v6, v4, v6
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CGP-NEXT:    v_mul_hi_u32 v3, v0, v3
+; CGP-NEXT:    v_mul_hi_u32 v4, v1, v4
+; CGP-NEXT:    v_lshlrev_b32_e32 v3, 12, v3
+; CGP-NEXT:    v_lshlrev_b32_e32 v4, 12, v4
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; CGP-NEXT:    v_subrev_i32_e32 v3, vcc, s4, v0
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v1, v2
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v3, vcc, s4, v0
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v1, v2
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; CGP-NEXT:    s_setpc_b64 s[30:31]
   %result = urem <2 x i32> %num, <i32 4096, i32 4096>
   ret <2 x i32> %result
 }
@@ -377,89 +328,94 @@ define i32 @v_urem_i32_oddk_denom(i32 %num) {
 ; CHECK-LABEL: v_urem_i32_oddk_denom:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_mov_b32 s6, 0x12d8fb
-; CHECK-NEXT:    v_mov_b32_e32 v1, 0x12d8fb
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, s6
+; CHECK-NEXT:    s_mov_b32 s4, 0x12d8fb
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0xffed2705
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, s4
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; CHECK-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; CHECK-NEXT:    v_mul_lo_u32 v3, v2, s6
-; CHECK-NEXT:    v_mul_hi_u32 v4, v2, s6
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v2
-; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v2, v3
-; CHECK-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v2, v2, v0
-; CHECK-NEXT:    v_mul_lo_u32 v2, v2, s6
-; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, v0, v2
-; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v3
-; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v3, v1
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v2
-; CHECK-NEXT:    v_sub_i32_e64 v0, s[6:7], v3, v1
-; CHECK-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[4:5]
+; CHECK-NEXT:    v_mul_lo_u32 v1, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v1, v2, v1
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CHECK-NEXT:    v_mul_lo_u32 v1, v1, s4
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT:    v_subrev_i32_e32 v1, vcc, s4, v0
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v1, vcc, s4, v0
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %result = urem i32 %num, 1235195
   ret i32 %result
 }
 
 define <2 x i32> @v_urem_v2i32_oddk_denom(<2 x i32> %num) {
-; CHECK-LABEL: v_urem_v2i32_oddk_denom:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_mov_b32 s8, 0x12d8fb
-; CHECK-NEXT:    v_mov_b32_e32 v2, 0x12d8fb
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, s8
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; CHECK-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; CHECK-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; CHECK-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
-; CHECK-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; CHECK-NEXT:    v_mul_lo_u32 v5, v3, s8
-; CHECK-NEXT:    v_mul_hi_u32 v6, v3, s8
-; CHECK-NEXT:    v_mul_lo_u32 v7, v4, v2
-; CHECK-NEXT:    v_mul_hi_u32 v8, v4, v2
-; CHECK-NEXT:    v_sub_i32_e32 v9, vcc, 0, v5
-; CHECK-NEXT:    v_sub_i32_e32 v10, vcc, 0, v7
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
-; CHECK-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
-; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
-; CHECK-NEXT:    v_cndmask_b32_e64 v6, v7, v10, s[4:5]
-; CHECK-NEXT:    v_mul_hi_u32 v5, v5, v3
-; CHECK-NEXT:    v_mul_hi_u32 v6, v6, v4
-; CHECK-NEXT:    v_add_i32_e64 v7, s[6:7], v3, v5
-; CHECK-NEXT:    v_sub_i32_e64 v3, s[6:7], v3, v5
-; CHECK-NEXT:    v_add_i32_e64 v5, s[6:7], v4, v6
-; CHECK-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v6
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v4, v4, v5, s[4:5]
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v0
-; CHECK-NEXT:    v_mul_hi_u32 v4, v4, v1
-; CHECK-NEXT:    v_mul_lo_u32 v3, v3, s8
-; CHECK-NEXT:    v_mul_lo_u32 v4, v4, v2
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, v0, v3
-; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v1, v4
-; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s8, v5
-; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v5, v2
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v3
-; CHECK-NEXT:    v_sub_i32_e64 v0, s[6:7], v5, v2
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[6:7], v6, v2
-; CHECK-NEXT:    v_add_i32_e64 v3, s[8:9], v6, v2
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v4
-; CHECK-NEXT:    v_sub_i32_e64 v1, s[10:11], v6, v2
-; CHECK-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; CHECK-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; CHECK-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v7, v0, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[8:9]
-; CHECK-NEXT:    s_setpc_b64 s[30:31]
+; GISEL-LABEL: v_urem_v2i32_oddk_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_mov_b32 s4, 0x12d8fb
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v2, s4
+; GISEL-NEXT:    s_sub_i32 s5, 0, s4
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GISEL-NEXT:    v_mul_lo_u32 v3, s5, v2
+; GISEL-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GISEL-NEXT:    v_mul_hi_u32 v3, v0, v2
+; GISEL-NEXT:    v_mul_hi_u32 v2, v1, v2
+; GISEL-NEXT:    v_mul_lo_u32 v3, v3, s4
+; GISEL-NEXT:    v_mul_lo_u32 v2, v2, s4
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; GISEL-NEXT:    v_subrev_i32_e32 v2, vcc, s4, v0
+; GISEL-NEXT:    v_subrev_i32_e32 v3, vcc, s4, v1
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v2, vcc, s4, v0
+; GISEL-NEXT:    v_subrev_i32_e32 v3, vcc, s4, v1
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s4, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_urem_v2i32_oddk_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_mov_b32 s4, 0x12d8fb
+; CGP-NEXT:    v_mov_b32_e32 v2, 0x12d8fb
+; CGP-NEXT:    s_mov_b32 s5, 0xffed2705
+; CGP-NEXT:    v_cvt_f32_u32_e32 v3, s4
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; CGP-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_lo_u32 v4, s5, v3
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v0, v3
+; CGP-NEXT:    v_mul_hi_u32 v3, v1, v3
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, s4
+; CGP-NEXT:    v_mul_lo_u32 v3, v3, v2
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v3
+; CGP-NEXT:    v_subrev_i32_e32 v3, vcc, s4, v0
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v1, v2
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v3, vcc, s4, v0
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v1, v2
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; CGP-NEXT:    s_setpc_b64 s[30:31]
   %result = urem <2 x i32> %num, <i32 1235195, i32 1235195>
   ret <2 x i32> %result
 }
@@ -470,28 +426,22 @@ define i32 @v_urem_i32_pow2_shl_denom(i32 %x, i32 %y) {
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    v_lshl_b32_e32 v1, 0x1000, v1
 ; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, v1
+; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; CHECK-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; CHECK-NEXT:    v_mul_lo_u32 v3, v2, v1
-; CHECK-NEXT:    v_mul_hi_u32 v4, v2, v1
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v2
-; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v2, v3
-; CHECK-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v2, v2, v0
+; CHECK-NEXT:    v_mul_lo_u32 v3, v3, v2
+; CHECK-NEXT:    v_mul_hi_u32 v3, v2, v3
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; CHECK-NEXT:    v_mul_hi_u32 v2, v0, v2
 ; CHECK-NEXT:    v_mul_lo_u32 v2, v2, v1
-; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, v0, v2
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v1
-; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v3, v1
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v2
-; CHECK-NEXT:    v_sub_i32_e64 v0, s[6:7], v3, v1
-; CHECK-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[4:5]
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %shl.y = shl i32 4096, %y
   %r = urem i32 %x, %shl.y
@@ -503,130 +453,101 @@ define <2 x i32> @v_urem_v2i32_pow2_shl_denom(<2 x i32> %x, <2 x i32> %y) {
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    s_movk_i32 s4, 0x1000
+; GISEL-NEXT:    s_mov_b32 s5, 0x4f7ffffe
 ; GISEL-NEXT:    v_lshl_b32_e32 v2, s4, v2
 ; GISEL-NEXT:    v_lshl_b32_e32 v3, s4, v3
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v3
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v6, v3
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v3
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; GISEL-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GISEL-NEXT:    v_mul_f32_e32 v4, s5, v4
+; GISEL-NEXT:    v_mul_f32_e32 v6, s5, v6
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GISEL-NEXT:    v_mul_lo_u32 v6, v4, v2
-; GISEL-NEXT:    v_mul_hi_u32 v7, v4, v2
-; GISEL-NEXT:    v_mul_lo_u32 v8, v5, v3
-; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v3
-; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, 0, v6
-; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, 0, v8
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, v8, v11, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v6, v6, v4
-; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v5
-; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v4, v6
-; GISEL-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v6
-; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v5, v7
-; GISEL-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v4, v4, v0
-; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v1
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v4
+; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v6
+; GISEL-NEXT:    v_mul_hi_u32 v5, v4, v5
+; GISEL-NEXT:    v_mul_hi_u32 v7, v6, v7
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v6, v7
+; GISEL-NEXT:    v_mul_hi_u32 v4, v0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v5, v1, v5
 ; GISEL-NEXT:    v_mul_lo_u32 v4, v4, v2
 ; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v3
-; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, v0, v4
-; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, v1, v5
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v6, v2
-; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v6, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v4
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[6:7], v6, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[6:7], v7, v3
-; GISEL-NEXT:    v_add_i32_e64 v2, s[8:9], v7, v3
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v5
-; GISEL-NEXT:    v_sub_i32_e64 v1, s[10:11], v7, v3
-; GISEL-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GISEL-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[8:9]
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, v1, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, v1, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; CGP-LABEL: v_urem_v2i32_pow2_shl_denom:
 ; CGP:       ; %bb.0:
 ; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CGP-NEXT:    s_movk_i32 s4, 0x1000
-; CGP-NEXT:    v_mul_lo_u32 v4, 0, v0
-; CGP-NEXT:    v_mul_lo_u32 v5, 0, v1
+; CGP-NEXT:    v_mul_lo_u32 v4, v0, 0
+; CGP-NEXT:    v_mul_lo_u32 v5, v1, 0
 ; CGP-NEXT:    v_lshl_b32_e32 v2, s4, v2
 ; CGP-NEXT:    v_lshl_b32_e32 v3, s4, v3
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v6, v2
-; CGP-NEXT:    v_mul_lo_u32 v7, 0, v2
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, 0, v2
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v8, v3
-; CGP-NEXT:    v_mul_lo_u32 v9, 0, v3
+; CGP-NEXT:    v_sub_i32_e32 v9, vcc, 0, v3
 ; CGP-NEXT:    v_rcp_f32_e32 v6, v6
 ; CGP-NEXT:    v_rcp_f32_e32 v8, v8
-; CGP-NEXT:    v_mul_f32_e32 v6, 0x4f800000, v6
-; CGP-NEXT:    v_mul_f32_e32 v8, 0x4f800000, v8
+; CGP-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; CGP-NEXT:    v_mul_f32_e32 v8, 0x4f7ffffe, v8
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v8, v8
-; CGP-NEXT:    v_mul_lo_u32 v10, v6, v2
-; CGP-NEXT:    v_mul_lo_u32 v11, v6, 0
-; CGP-NEXT:    v_mul_hi_u32 v12, v6, v2
-; CGP-NEXT:    v_mul_lo_u32 v13, v8, v3
-; CGP-NEXT:    v_mul_lo_u32 v14, v8, 0
-; CGP-NEXT:    v_mul_hi_u32 v15, v8, v3
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v11
-; CGP-NEXT:    v_sub_i32_e32 v16, vcc, 0, v10
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v14
-; CGP-NEXT:    v_sub_i32_e32 v17, vcc, 0, v13
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v12
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; CGP-NEXT:    v_cndmask_b32_e32 v7, v10, v16, vcc
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v9
-; CGP-NEXT:    v_cndmask_b32_e64 v9, v13, v17, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v10, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v7, v7, v6
-; CGP-NEXT:    v_mul_lo_u32 v12, v9, 0
-; CGP-NEXT:    v_mul_hi_u32 v9, v9, v8
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v11, v10
-; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v14, v12
-; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v10, v7
-; CGP-NEXT:    v_add_i32_e64 v9, s[6:7], v11, v9
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v6, v7
-; CGP-NEXT:    v_sub_i32_e64 v6, s[6:7], v6, v7
-; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v8, v9
-; CGP-NEXT:    v_sub_i32_e64 v8, s[6:7], v8, v9
-; CGP-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v7, v8, v7, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v8, v6, 0
-; CGP-NEXT:    v_mul_hi_u32 v6, v6, v0
-; CGP-NEXT:    v_mul_lo_u32 v9, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v7, v7, v1
-; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT:    v_mul_lo_u32 v7, v7, v6
+; CGP-NEXT:    v_mul_lo_u32 v10, v6, 0
+; CGP-NEXT:    v_mul_lo_u32 v9, v9, v8
+; CGP-NEXT:    v_mul_lo_u32 v11, v8, 0
+; CGP-NEXT:    v_mul_lo_u32 v12, 0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v6, v7
+; CGP-NEXT:    v_mul_lo_u32 v13, 0, v9
+; CGP-NEXT:    v_mul_hi_u32 v9, v8, v9
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v8, v9
+; CGP-NEXT:    v_mul_lo_u32 v8, 0, v6
+; CGP-NEXT:    v_mul_hi_u32 v6, v0, v6
+; CGP-NEXT:    v_mul_lo_u32 v9, 0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v1, v7
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
 ; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
 ; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
 ; CGP-NEXT:    v_mul_lo_u32 v4, v4, v2
 ; CGP-NEXT:    v_mul_lo_u32 v5, v5, v3
-; CGP-NEXT:    v_sub_i32_e32 v6, vcc, v0, v4
-; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v1, v5
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v6, v2
-; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v6, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v4
-; CGP-NEXT:    v_sub_i32_e64 v0, s[6:7], v6, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v7, v3
-; CGP-NEXT:    v_add_i32_e64 v2, s[8:9], v7, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v5
-; CGP-NEXT:    v_sub_i32_e64 v1, s[10:11], v7, v3
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; CGP-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[8:9]
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, v1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, v1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
   %r = urem <2 x i32> %x, %shl.y
@@ -641,28 +562,22 @@ define i32 @v_urem_i32_24bit(i32 %num, i32 %den) {
 ; GISEL-NEXT:    v_and_b32_e32 v0, s4, v0
 ; GISEL-NEXT:    v_and_b32_e32 v1, s4, v1
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v2, v1
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GISEL-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GISEL-NEXT:    v_mul_hi_u32 v4, v2, v1
-; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v3, v3, v2
-; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v2, v3
-; GISEL-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v2, v2, v0
+; GISEL-NEXT:    v_mul_lo_u32 v3, v3, v2
+; GISEL-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GISEL-NEXT:    v_mul_hi_u32 v2, v0, v2
 ; GISEL-NEXT:    v_mul_lo_u32 v2, v2, v1
-; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, v0, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v1
-; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v3, v1
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v2
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[6:7], v3, v1
-; GISEL-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; CGP-LABEL: v_urem_i32_24bit:
@@ -672,39 +587,30 @@ define i32 @v_urem_i32_24bit(i32 %num, i32 %den) {
 ; CGP-NEXT:    v_and_b32_e32 v0, s4, v0
 ; CGP-NEXT:    v_and_b32_e32 v1, s4, v1
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v2, v1
-; CGP-NEXT:    v_mul_lo_u32 v3, 0, v1
-; CGP-NEXT:    v_mul_lo_u32 v4, 0, v0
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; CGP-NEXT:    v_mul_lo_u32 v4, v0, 0
 ; CGP-NEXT:    v_rcp_f32_e32 v2, v2
-; CGP-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; CGP-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, v2, v1
-; CGP-NEXT:    v_mul_lo_u32 v6, v2, 0
-; CGP-NEXT:    v_mul_hi_u32 v7, v2, v1
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; CGP-NEXT:    v_sub_i32_e32 v8, vcc, 0, v5
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; CGP-NEXT:    v_cndmask_b32_e32 v3, v5, v8, vcc
-; CGP-NEXT:    v_mul_lo_u32 v5, v3, 0
-; CGP-NEXT:    v_mul_hi_u32 v3, v3, v2
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v6, v5
-; CGP-NEXT:    v_add_i32_e64 v3, s[4:5], v5, v3
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v2, v3
-; CGP-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; CGP-NEXT:    v_mul_lo_u32 v3, v2, 0
-; CGP-NEXT:    v_mul_hi_u32 v2, v2, v0
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; CGP-NEXT:    v_mul_lo_u32 v3, v3, v2
+; CGP-NEXT:    v_mul_lo_u32 v5, v2, 0
+; CGP-NEXT:    v_mul_lo_u32 v6, 0, v3
+; CGP-NEXT:    v_mul_hi_u32 v3, v2, v3
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT:    v_mul_lo_u32 v3, 0, v2
+; CGP-NEXT:    v_mul_hi_u32 v2, v0, v2
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
 ; CGP-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; CGP-NEXT:    v_mul_lo_u32 v2, v2, v1
-; CGP-NEXT:    v_sub_i32_e32 v3, vcc, v0, v2
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v1
-; CGP-NEXT:    v_add_i32_e64 v4, s[4:5], v3, v1
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v2
-; CGP-NEXT:    v_sub_i32_e64 v0, s[6:7], v3, v1
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[4:5]
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %num.mask = and i32 %num, 16777215
   %den.mask = and i32 %den, 16777215
@@ -717,56 +623,45 @@ define <2 x i32> @v_urem_v2i32_24bit(<2 x i32> %num, <2 x i32> %den) {
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    s_mov_b32 s4, 0xffffff
+; GISEL-NEXT:    s_mov_b32 s5, 0x4f7ffffe
 ; GISEL-NEXT:    v_and_b32_e32 v0, s4, v0
 ; GISEL-NEXT:    v_and_b32_e32 v1, s4, v1
 ; GISEL-NEXT:    v_and_b32_e32 v2, s4, v2
 ; GISEL-NEXT:    v_and_b32_e32 v3, s4, v3
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v3
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v6, v3
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, 0, v3
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; GISEL-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; GISEL-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; GISEL-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GISEL-NEXT:    v_mul_f32_e32 v4, s5, v4
+; GISEL-NEXT:    v_mul_f32_e32 v6, s5, v6
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GISEL-NEXT:    v_mul_lo_u32 v6, v4, v2
-; GISEL-NEXT:    v_mul_hi_u32 v7, v4, v2
-; GISEL-NEXT:    v_mul_lo_u32 v8, v5, v3
-; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v3
-; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, 0, v6
-; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, 0, v8
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, v8, v11, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v6, v6, v4
-; GISEL-NEXT:    v_mul_hi_u32 v7, v7, v5
-; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v4, v6
-; GISEL-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v6
-; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v5, v7
-; GISEL-NEXT:    v_sub_i32_e64 v5, s[6:7], v5, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s[4:5]
-; GISEL-NEXT:    v_mul_hi_u32 v4, v4, v0
-; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v1
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v4
+; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v6
+; GISEL-NEXT:    v_mul_hi_u32 v5, v4, v5
+; GISEL-NEXT:    v_mul_hi_u32 v7, v6, v7
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v6, v7
+; GISEL-NEXT:    v_mul_hi_u32 v4, v0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v5, v1, v5
 ; GISEL-NEXT:    v_mul_lo_u32 v4, v4, v2
 ; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v3
-; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, v0, v4
-; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, v1, v5
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v6, v2
-; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v6, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v4
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[6:7], v6, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[6:7], v7, v3
-; GISEL-NEXT:    v_add_i32_e64 v2, s[8:9], v7, v3
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v5
-; GISEL-NEXT:    v_sub_i32_e64 v1, s[10:11], v7, v3
-; GISEL-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GISEL-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[8:9]
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, v1, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, v1, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; CGP-LABEL: v_urem_v2i32_24bit:
@@ -778,73 +673,55 @@ define <2 x i32> @v_urem_v2i32_24bit(<2 x i32> %num, <2 x i32> %den) {
 ; CGP-NEXT:    v_and_b32_e32 v2, s4, v2
 ; CGP-NEXT:    v_and_b32_e32 v3, s4, v3
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, 0, v2
-; CGP-NEXT:    v_mul_lo_u32 v6, 0, v0
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; CGP-NEXT:    v_mul_lo_u32 v6, v0, 0
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v7, v3
-; CGP-NEXT:    v_mul_lo_u32 v8, 0, v3
-; CGP-NEXT:    v_mul_lo_u32 v9, 0, v1
+; CGP-NEXT:    v_sub_i32_e32 v8, vcc, 0, v3
+; CGP-NEXT:    v_mul_lo_u32 v9, v1, 0
 ; CGP-NEXT:    v_rcp_f32_e32 v4, v4
 ; CGP-NEXT:    v_rcp_f32_e32 v7, v7
-; CGP-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
-; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f800000, v7
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x4f7ffffe, v4
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; CGP-NEXT:    v_mul_lo_u32 v10, v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v11, v4, 0
-; CGP-NEXT:    v_mul_hi_u32 v12, v4, v2
-; CGP-NEXT:    v_mul_lo_u32 v13, v7, v3
-; CGP-NEXT:    v_mul_lo_u32 v14, v7, 0
-; CGP-NEXT:    v_mul_hi_u32 v15, v7, v3
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v11
-; CGP-NEXT:    v_sub_i32_e32 v16, vcc, 0, v10
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v14
-; CGP-NEXT:    v_sub_i32_e32 v17, vcc, 0, v13
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v12
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v15
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; CGP-NEXT:    v_cndmask_b32_e32 v5, v10, v16, vcc
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
-; CGP-NEXT:    v_cndmask_b32_e64 v8, v13, v17, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v10, v5, 0
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v4
-; CGP-NEXT:    v_mul_lo_u32 v12, v8, 0
-; CGP-NEXT:    v_mul_hi_u32 v8, v8, v7
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v11, v10
-; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v14, v12
-; CGP-NEXT:    v_add_i32_e64 v5, s[6:7], v10, v5
-; CGP-NEXT:    v_add_i32_e64 v8, s[6:7], v11, v8
-; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v4, v5
-; CGP-NEXT:    v_sub_i32_e64 v4, s[6:7], v4, v5
-; CGP-NEXT:    v_add_i32_e64 v5, s[6:7], v7, v8
-; CGP-NEXT:    v_sub_i32_e64 v7, s[6:7], v7, v8
-; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[4:5]
-; CGP-NEXT:    v_mul_lo_u32 v7, v4, 0
-; CGP-NEXT:    v_mul_hi_u32 v4, v4, v0
-; CGP-NEXT:    v_mul_lo_u32 v8, v5, 0
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v1
-; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v9, v8
+; CGP-NEXT:    v_mul_lo_u32 v5, v5, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v4, 0
+; CGP-NEXT:    v_mul_lo_u32 v8, v8, v7
+; CGP-NEXT:    v_mul_lo_u32 v11, v7, 0
+; CGP-NEXT:    v_mul_lo_u32 v12, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v4, v5
+; CGP-NEXT:    v_mul_lo_u32 v13, 0, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v7, v8
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v10, v5
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v7, v8
+; CGP-NEXT:    v_mul_lo_u32 v7, 0, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v0, v4
+; CGP-NEXT:    v_mul_lo_u32 v8, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v1, v5
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v8, v9
 ; CGP-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
 ; CGP-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
 ; CGP-NEXT:    v_mul_lo_u32 v4, v4, v2
 ; CGP-NEXT:    v_mul_lo_u32 v5, v5, v3
-; CGP-NEXT:    v_sub_i32_e32 v6, vcc, v0, v4
-; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v1, v5
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v6, v2
-; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v6, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v4
-; CGP-NEXT:    v_sub_i32_e64 v0, s[6:7], v6, v2
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v7, v3
-; CGP-NEXT:    v_add_i32_e64 v2, s[8:9], v7, v3
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[8:9], v1, v5
-; CGP-NEXT:    v_sub_i32_e64 v1, s[10:11], v7, v3
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; CGP-NEXT:    s_and_b64 vcc, s[6:7], s[8:9]
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[8:9]
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, v1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, v1, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
   %num.mask = and <2 x i32> %num, <i32 16777215, i32 16777215>
   %den.mask = and <2 x i32> %den, <i32 16777215, i32 16777215>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
index 3e43bcf0409c4..6b9357043b3ca 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
@@ -145,36 +145,30 @@ define i64 @v_urem_i64(i64 %num, i64 %den) {
 ; CHECK-NEXT:    v_cndmask_b32_e32 v4, v5, v3, vcc
 ; CHECK-NEXT:    v_cndmask_b32_e32 v5, v6, v1, vcc
 ; CHECK-NEXT:  BB0_2: ; %Flow
-; CHECK-NEXT:    s_or_saveexec_b64 s[8:9], s[6:7]
-; CHECK-NEXT:    s_xor_b64 exec, exec, s[8:9]
+; CHECK-NEXT:    s_or_saveexec_b64 s[4:5], s[6:7]
+; CHECK-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; CHECK-NEXT:    s_cbranch_execz BB0_4
 ; CHECK-NEXT:  ; %bb.3:
 ; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, v2
+; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, 0, v2
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
+; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
 ; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; CHECK-NEXT:    v_mul_lo_u32 v3, v1, v2
-; CHECK-NEXT:    v_mul_hi_u32 v4, v1, v2
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v1
-; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v1, v3
-; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v1, v1, v0
+; CHECK-NEXT:    v_mul_lo_u32 v3, v3, v1
+; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; CHECK-NEXT:    v_mul_lo_u32 v1, v1, v2
-; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, v0, v1
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v2
-; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v3, v2
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v1
-; CHECK-NEXT:    v_sub_i32_e64 v0, s[6:7], v3, v2
-; CHECK-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v4, v4, v0, s[4:5]
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, v0, v2
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, v0, v2
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, v0, v1, vcc
 ; CHECK-NEXT:    v_mov_b32_e32 v5, 0
 ; CHECK-NEXT:  BB0_4:
-; CHECK-NEXT:    s_or_b64 exec, exec, s[8:9]
+; CHECK-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; CHECK-NEXT:    v_mov_b32_e32 v0, v4
 ; CHECK-NEXT:    v_mov_b32_e32 v1, v5
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
@@ -333,28 +327,22 @@ define amdgpu_ps i64 @s_urem_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-NEXT:    s_cbranch_scc0 BB1_4
 ; CHECK-NEXT:  ; %bb.3:
 ; CHECK-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; CHECK-NEXT:    s_sub_i32 s1, 0, s2
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; CHECK-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; CHECK-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; CHECK-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; CHECK-NEXT:    v_mul_lo_u32 v1, v0, s2
-; CHECK-NEXT:    v_mul_hi_u32 v2, v0, s2
-; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v1, v1, v0
-; CHECK-NEXT:    v_add_i32_e64 v2, s[4:5], v0, v1
-; CHECK-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v1
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v0, v0, s0
+; CHECK-NEXT:    v_mul_lo_u32 v1, s1, v0
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT:    v_mul_hi_u32 v0, s0, v0
 ; CHECK-NEXT:    v_mul_lo_u32 v0, v0, s2
-; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, s0, v0
-; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s2, v1
-; CHECK-NEXT:    v_add_i32_e64 v2, s[4:5], s2, v1
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[0:1], s0, v0
-; CHECK-NEXT:    v_subrev_i32_e64 v0, s[2:3], s2, v1
-; CHECK-NEXT:    s_and_b64 vcc, vcc, s[0:1]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[0:1]
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; CHECK-NEXT:    v_subrev_i32_e32 v1, vcc, s2, v0
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s2, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v1, vcc, s2, v0
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s2, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; CHECK-NEXT:  BB1_4:
 ; CHECK-NEXT:    v_readfirstlane_b32 s0, v0
 ; CHECK-NEXT:    s_mov_b32 s1, s0
@@ -771,36 +759,30 @@ define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
 ; CGP-NEXT:    v_cndmask_b32_e32 v0, v1, v5, vcc
 ; CGP-NEXT:    v_cndmask_b32_e32 v1, v10, v11, vcc
 ; CGP-NEXT:  BB2_2: ; %Flow2
-; CGP-NEXT:    s_or_saveexec_b64 s[8:9], s[6:7]
-; CGP-NEXT:    s_xor_b64 exec, exec, s[8:9]
+; CGP-NEXT:    s_or_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; CGP-NEXT:    s_cbranch_execz BB2_4
 ; CGP-NEXT:  ; %bb.3:
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v4
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, 0, v4
 ; CGP-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; CGP-NEXT:    v_mul_lo_u32 v1, v0, v4
-; CGP-NEXT:    v_mul_hi_u32 v5, v0, v4
-; CGP-NEXT:    v_sub_i32_e32 v9, vcc, 0, v1
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; CGP-NEXT:    v_mul_hi_u32 v1, v1, v0
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v0, v1
-; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v1
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
-; CGP-NEXT:    v_mul_hi_u32 v0, v0, v8
+; CGP-NEXT:    v_mul_lo_u32 v1, v1, v0
+; CGP-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; CGP-NEXT:    v_mul_hi_u32 v0, v8, v0
 ; CGP-NEXT:    v_mul_lo_u32 v0, v0, v4
-; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v8, v0
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v4
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v1, v4
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v0
-; CGP-NEXT:    v_sub_i32_e64 v0, s[6:7], v1, v4
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v8, v0
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v0, v4
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v0, v4
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; CGP-NEXT:    v_mov_b32_e32 v1, 0
 ; CGP-NEXT:  BB2_4:
-; CGP-NEXT:    s_or_b64 exec, exec, s[8:9]
+; CGP-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; CGP-NEXT:    v_or_b32_e32 v5, v3, v7
 ; CGP-NEXT:    v_mov_b32_e32 v4, 0
 ; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
@@ -938,36 +920,30 @@ define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
 ; CGP-NEXT:    v_cndmask_b32_e32 v4, v5, v7, vcc
 ; CGP-NEXT:    v_cndmask_b32_e32 v5, v8, v3, vcc
 ; CGP-NEXT:  BB2_6: ; %Flow
-; CGP-NEXT:    s_or_saveexec_b64 s[8:9], s[6:7]
-; CGP-NEXT:    s_xor_b64 exec, exec, s[8:9]
+; CGP-NEXT:    s_or_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; CGP-NEXT:    s_cbranch_execz BB2_8
 ; CGP-NEXT:  ; %bb.7:
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v3, v6
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, 0, v6
 ; CGP-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; CGP-NEXT:    v_mul_lo_u32 v4, v3, v6
-; CGP-NEXT:    v_mul_hi_u32 v5, v3, v6
-; CGP-NEXT:    v_sub_i32_e32 v7, vcc, 0, v4
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
-; CGP-NEXT:    v_mul_hi_u32 v4, v4, v3
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v3, v4
-; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
-; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; CGP-NEXT:    v_mul_hi_u32 v3, v3, v2
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, v3
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v3, v2, v3
 ; CGP-NEXT:    v_mul_lo_u32 v3, v3, v6
-; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v2, v3
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v6
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v4, v6
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v3
-; CGP-NEXT:    v_sub_i32_e64 v2, s[6:7], v4, v6
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v4, v5, v2, s[4:5]
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, v2, v6
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v6
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, v2, v6
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v6
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
 ; CGP-NEXT:    v_mov_b32_e32 v5, 0
 ; CGP-NEXT:  BB2_8:
-; CGP-NEXT:    s_or_b64 exec, exec, s[8:9]
+; CGP-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; CGP-NEXT:    v_mov_b32_e32 v2, v4
 ; CGP-NEXT:    v_mov_b32_e32 v3, v5
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
@@ -2421,36 +2397,30 @@ define i64 @v_urem_i64_pow2_shl_denom(i64 %x, i64 %y) {
 ; CHECK-NEXT:    v_cndmask_b32_e32 v2, v3, v5, vcc
 ; CHECK-NEXT:    v_cndmask_b32_e32 v3, v6, v1, vcc
 ; CHECK-NEXT:  BB7_2: ; %Flow
-; CHECK-NEXT:    s_or_saveexec_b64 s[8:9], s[6:7]
-; CHECK-NEXT:    s_xor_b64 exec, exec, s[8:9]
+; CHECK-NEXT:    s_or_saveexec_b64 s[4:5], s[6:7]
+; CHECK-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; CHECK-NEXT:    s_cbranch_execz BB7_4
 ; CHECK-NEXT:  ; %bb.3:
 ; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, v4
+; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, 0, v4
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
+; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
 ; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; CHECK-NEXT:    v_mul_lo_u32 v2, v1, v4
-; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v4
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
-; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v2, v2, v1
-; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v1, v2
-; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v2
-; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; CHECK-NEXT:    v_mul_hi_u32 v1, v1, v0
+; CHECK-NEXT:    v_mul_lo_u32 v2, v2, v1
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; CHECK-NEXT:    v_mul_lo_u32 v1, v1, v4
-; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v4
-; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v2, v4
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v1
-; CHECK-NEXT:    v_sub_i32_e64 v0, s[6:7], v2, v4
-; CHECK-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v2, v3, v0, s[4:5]
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, v0, v4
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, v0, v4
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
 ; CHECK-NEXT:    v_mov_b32_e32 v3, 0
 ; CHECK-NEXT:  BB7_4:
-; CHECK-NEXT:    s_or_b64 exec, exec, s[8:9]
+; CHECK-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; CHECK-NEXT:    v_mov_b32_e32 v0, v2
 ; CHECK-NEXT:    v_mov_b32_e32 v1, v3
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
@@ -2867,36 +2837,30 @@ define <2 x i64> @v_urem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
 ; CGP-NEXT:    v_cndmask_b32_e32 v0, v1, v7, vcc
 ; CGP-NEXT:    v_cndmask_b32_e32 v1, v4, v11, vcc
 ; CGP-NEXT:  BB8_2: ; %Flow2
-; CGP-NEXT:    s_or_saveexec_b64 s[8:9], s[6:7]
-; CGP-NEXT:    s_xor_b64 exec, exec, s[8:9]
+; CGP-NEXT:    s_or_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; CGP-NEXT:    s_cbranch_execz BB8_4
 ; CGP-NEXT:  ; %bb.3:
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v10
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, 0, v10
 ; CGP-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; CGP-NEXT:    v_mul_lo_u32 v1, v0, v10
-; CGP-NEXT:    v_mul_hi_u32 v4, v0, v10
-; CGP-NEXT:    v_sub_i32_e32 v6, vcc, 0, v1
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
-; CGP-NEXT:    v_mul_hi_u32 v1, v1, v0
-; CGP-NEXT:    v_add_i32_e64 v4, s[4:5], v0, v1
-; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v1
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; CGP-NEXT:    v_mul_hi_u32 v0, v0, v5
+; CGP-NEXT:    v_mul_lo_u32 v1, v1, v0
+; CGP-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; CGP-NEXT:    v_mul_hi_u32 v0, v5, v0
 ; CGP-NEXT:    v_mul_lo_u32 v0, v0, v10
-; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v5, v0
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v10
-; CGP-NEXT:    v_add_i32_e64 v4, s[4:5], v1, v10
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v5, v0
-; CGP-NEXT:    v_sub_i32_e64 v0, s[6:7], v1, v10
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[4:5]
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v5, v0
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v0, v10
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v10
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v0, v10
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v10
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; CGP-NEXT:    v_mov_b32_e32 v1, 0
 ; CGP-NEXT:  BB8_4:
-; CGP-NEXT:    s_or_b64 exec, exec, s[8:9]
+; CGP-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; CGP-NEXT:    v_or_b32_e32 v5, v3, v9
 ; CGP-NEXT:    v_mov_b32_e32 v4, 0
 ; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
@@ -3034,36 +2998,30 @@ define <2 x i64> @v_urem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
 ; CGP-NEXT:    v_cndmask_b32_e32 v4, v5, v7, vcc
 ; CGP-NEXT:    v_cndmask_b32_e32 v5, v6, v3, vcc
 ; CGP-NEXT:  BB8_6: ; %Flow
-; CGP-NEXT:    s_or_saveexec_b64 s[8:9], s[6:7]
-; CGP-NEXT:    s_xor_b64 exec, exec, s[8:9]
+; CGP-NEXT:    s_or_saveexec_b64 s[4:5], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; CGP-NEXT:    s_cbranch_execz BB8_8
 ; CGP-NEXT:  ; %bb.7:
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v3, v8
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, 0, v8
 ; CGP-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; CGP-NEXT:    v_mul_lo_u32 v4, v3, v8
-; CGP-NEXT:    v_mul_hi_u32 v5, v3, v8
-; CGP-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
-; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; CGP-NEXT:    v_mul_hi_u32 v4, v4, v3
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v3, v4
-; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
-; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; CGP-NEXT:    v_mul_hi_u32 v3, v3, v2
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, v3
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v3, v2, v3
 ; CGP-NEXT:    v_mul_lo_u32 v3, v3, v8
-; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v2, v3
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v8
-; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v4, v8
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v3
-; CGP-NEXT:    v_sub_i32_e64 v2, s[6:7], v4, v8
-; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v4, v5, v2, s[4:5]
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v3
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, v2, v8
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v8
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CGP-NEXT:    v_sub_i32_e32 v3, vcc, v2, v8
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v8
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
 ; CGP-NEXT:    v_mov_b32_e32 v5, 0
 ; CGP-NEXT:  BB8_8:
-; CGP-NEXT:    s_or_b64 exec, exec, s[8:9]
+; CGP-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; CGP-NEXT:    v_mov_b32_e32 v2, v4
 ; CGP-NEXT:    v_mov_b32_e32 v3, v5
 ; CGP-NEXT:    s_setpc_b64 s[30:31]
@@ -3080,28 +3038,22 @@ define i64 @v_urem_i64_24bit(i64 %num, i64 %den) {
 ; GISEL-NEXT:    v_and_b32_e32 v0, s4, v0
 ; GISEL-NEXT:    v_and_b32_e32 v1, s4, v2
 ; GISEL-NEXT:    v_cvt_f32_u32_e32 v2, v1
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GISEL-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GISEL-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GISEL-NEXT:    v_mul_hi_u32 v4, v2, v1
-; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v3, v3, v2
-; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v2, v3
-; GISEL-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
-; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GISEL-NEXT:    v_mul_hi_u32 v2, v2, v0
+; GISEL-NEXT:    v_mul_lo_u32 v3, v3, v2
+; GISEL-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GISEL-NEXT:    v_mul_hi_u32 v2, v0, v2
 ; GISEL-NEXT:    v_mul_lo_u32 v2, v2, v1
-; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, v0, v2
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v1
-; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v3, v1
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v2
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[6:7], v3, v1
-; GISEL-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-NEXT:    v_mov_b32_e32 v1, 0
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/InlineAsmCrash.ll b/llvm/test/CodeGen/AMDGPU/InlineAsmCrash.ll
index 0d725a1891b1c..2079613d069ad 100644
--- a/llvm/test/CodeGen/AMDGPU/InlineAsmCrash.ll
+++ b/llvm/test/CodeGen/AMDGPU/InlineAsmCrash.ll
@@ -4,9 +4,11 @@
 ; CHECK-NEXT: s_nop 0
 ; CHECK-NEXT: ;;#ASMEND
 
-define void @foo(i32 addrspace(5)* %ptr) {
+define void @foo(i32 addrspace(5)* %ptr) #0 {
   %tmp = tail call { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } asm "s_nop 0", "=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65"(i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2)
   %tmp2 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %tmp, 0
   store i32 %tmp2, i32 addrspace(5)* %ptr, align 4
   ret void
 }
+
+attributes #0 = { "amdgpu-flat-work-group-size"="1,768" }
diff --git a/llvm/test/CodeGen/AMDGPU/accvgpr-copy.mir b/llvm/test/CodeGen/AMDGPU/accvgpr-copy.mir
index 8db8e1bb017ea..ca7012607aa1c 100644
--- a/llvm/test/CodeGen/AMDGPU/accvgpr-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/accvgpr-copy.mir
@@ -1,124 +1,211 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -march=amdgcn -mcpu=gfx908 -run-pass postrapseudos -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
 
-# GCN-LABEL: name: a_to_v
-# GCN: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec
+--- |
+    define amdgpu_kernel void @a_to_v() #0 { ret void }
+    define amdgpu_kernel void @a4_to_v4() #0 { ret void }
+    define amdgpu_kernel void @a16_to_v16() #0 { ret void }
+
+    define amdgpu_kernel void @v_to_a() #0 { ret void }
+    define amdgpu_kernel void @v4_to_a4() #0 { ret void }
+    define amdgpu_kernel void @v16_to_a16() #0 { ret void }
+
+    define amdgpu_kernel void @s_to_a() #0 { ret void }
+    define amdgpu_kernel void @s2_to_a2() #0 { ret void }
+
+    define amdgpu_kernel void @a_to_a() #0 { ret void }
+    define amdgpu_kernel void @a2_to_a2() #0 { ret void }
+
+    define amdgpu_kernel void @a_to_a_spill() #0 { ret void }
+    attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
+...
+
 ---
 name:            a_to_v
+tracksRegLiveness: true
 body:             |
   bb.0:
+    liveins: $agpr0
+    ; GCN-LABEL: name: a_to_v
+    ; GCN: liveins: $agpr0
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit $vgpr0
     $vgpr0 = COPY killed $agpr0, implicit $exec
+    S_ENDPGM 0, implicit $vgpr0
 ...
 
-# GCN-LABEL: name: a4_to_v4
-# GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3, implicit $agpr0_agpr1_agpr2_agpr3
-# GCN: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
-# GCN: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
-# GCN: $vgpr3 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3
 ---
 name:            a4_to_v4
+tracksRegLiveness: true
 body:             |
   bb.0:
+    liveins: $agpr0_agpr1_agpr2_agpr3
+    ; GCN-LABEL: name: a4_to_v4
+    ; GCN: liveins: $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3, implicit $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $vgpr3 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY killed $agpr0_agpr1_agpr2_agpr3, implicit $exec
+    S_ENDPGM 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3
 ...
 
-# GCN-LABEL: name: a16_to_v16
-# GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
-# GCN: $vgpr15 = V_ACCVGPR_READ_B32 $agpr15, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
 ---
 name:            a16_to_v16
+tracksRegLiveness: true
 body:             |
   bb.0:
+    liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN-LABEL: name: a16_to_v16
+    ; GCN: liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr3 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr4 = V_ACCVGPR_READ_B32 $agpr4, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr5 = V_ACCVGPR_READ_B32 $agpr5, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr6 = V_ACCVGPR_READ_B32 $agpr6, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr7 = V_ACCVGPR_READ_B32 $agpr7, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr8 = V_ACCVGPR_READ_B32 $agpr8, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr9 = V_ACCVGPR_READ_B32 $agpr9, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr10 = V_ACCVGPR_READ_B32 $agpr10, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr11 = V_ACCVGPR_READ_B32 $agpr11, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr12 = V_ACCVGPR_READ_B32 $agpr12, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr13 = V_ACCVGPR_READ_B32 $agpr13, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr14 = V_ACCVGPR_READ_B32 $agpr14, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr15 = V_ACCVGPR_READ_B32 $agpr15, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit $exec
+    S_ENDPGM 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ...
 
-# GCN-LABEL: name: v_to_a
-# GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
 ---
 name:            v_to_a
+tracksRegLiveness: true
 body:             |
   bb.0:
+    liveins: $vgpr0
+    ; GCN-LABEL: name: v_to_a
+    ; GCN: liveins: $vgpr0
+    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit $agpr0
     $agpr0 = COPY killed $vgpr0, implicit $exec
+    S_ENDPGM 0, implicit $agpr0
 ...
 
-# GCN-LABEL: name: v4_to_a4
-# GCN: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3, implicit $vgpr0_vgpr1_vgpr2_vgpr3
-# GCN: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3
-# GCN: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3
-# GCN: $agpr3 = V_ACCVGPR_WRITE_B32 $vgpr3, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3, implicit $exec
 ---
 name:            v4_to_a4
+tracksRegLiveness: true
 body:             |
   bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GCN-LABEL: name: v4_to_a4
+    ; GCN: liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3, implicit $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GCN: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GCN: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GCN: $agpr3 = V_ACCVGPR_WRITE_B32 $vgpr3, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit $agpr0_agpr1_agpr2_agpr3
     $agpr0_agpr1_agpr2_agpr3 = COPY killed $vgpr0_vgpr1_vgpr2_vgpr3, implicit $exec
+    S_ENDPGM 0, implicit $agpr0_agpr1_agpr2_agpr3
 ...
 
-# GCN-LABEL: name: v16_to_a16
-# GCN: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-# GCN: $agpr15 = V_ACCVGPR_WRITE_B32 $vgpr15, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ---
 name:            v16_to_a16
+tracksRegLiveness: true
 body:             |
   bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN-LABEL: name: v16_to_a16
+    ; GCN: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN: $agpr3 = V_ACCVGPR_WRITE_B32 $vgpr3, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN: $agpr4 = V_ACCVGPR_WRITE_B32 $vgpr4, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN: $agpr5 = V_ACCVGPR_WRITE_B32 $vgpr5, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN: $agpr6 = V_ACCVGPR_WRITE_B32 $vgpr6, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN: $agpr7 = V_ACCVGPR_WRITE_B32 $vgpr7, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN: $agpr8 = V_ACCVGPR_WRITE_B32 $vgpr8, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN: $agpr9 = V_ACCVGPR_WRITE_B32 $vgpr9, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN: $agpr10 = V_ACCVGPR_WRITE_B32 $vgpr10, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN: $agpr11 = V_ACCVGPR_WRITE_B32 $vgpr11, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN: $agpr12 = V_ACCVGPR_WRITE_B32 $vgpr12, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN: $agpr13 = V_ACCVGPR_WRITE_B32 $vgpr13, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN: $agpr14 = V_ACCVGPR_WRITE_B32 $vgpr14, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; GCN: $agpr15 = V_ACCVGPR_WRITE_B32 $vgpr15, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $exec
+    S_ENDPGM 0, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
 ...
 
-# GCN-LABEL: name: s_to_a
-# GCN: $vgpr[[TMP:[0-9]+]] = V_MOV_B32_e32 killed $sgpr0, implicit $exec
-# GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr[[TMP]], implicit $exec
 ---
 name:            s_to_a
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $sgpr0
+    ; GCN-LABEL: name: s_to_a
+    ; GCN: liveins: $sgpr0
+    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec
+    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit $agpr0
     $agpr0 = COPY killed $sgpr0, implicit $exec
+    S_ENDPGM 0, implicit $agpr0
 ...
 
-# GCN-LABEL: name: s2_to_a2
-# GCN: $vgpr[[TMP1:[0-9]+]] = V_MOV_B32_e32 killed $sgpr0, implicit $exec
-# GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr[[TMP1]], implicit $exec
-# GCN: $vgpr[[TMP2:[0-9]+]] = V_MOV_B32_e32 killed $sgpr1, implicit $exec
-# GCN: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr[[TMP2]], implicit $exec
 ---
 name:            s2_to_a2
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $sgpr0_sgpr1
+    ; GCN-LABEL: name: s2_to_a2
+    ; GCN: liveins: $sgpr0_sgpr1
+    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec
+    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
+    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec
+    ; GCN: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit $agpr0_agpr1
     $agpr0_agpr1 = COPY killed $sgpr0_sgpr1, implicit $exec
+    S_ENDPGM 0, implicit $agpr0_agpr1
 ...
 
-# GCN-LABEL: name: a_to_a
-# GCN: $vgpr[[TMP:[0-9]+]] = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
-# GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr[[TMP]], implicit $exec
 ---
 name:            a_to_a
 tracksRegLiveness: true
 body:             |
   bb.0:
+    ; GCN-LABEL: name: a_to_a
+    ; GCN: $agpr1 = IMPLICIT_DEF
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit $agpr0
     $agpr1 = IMPLICIT_DEF
     $agpr0 = COPY killed $agpr1, implicit $exec
+    S_ENDPGM 0, implicit $agpr0
 ...
 
-# GCN-LABEL: name: a2_to_a2
-# GCN: $vgpr[[TMP1:[0-9]+]] = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
-# GCN: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr[[TMP1]], implicit $exec
-# GCN: $vgpr[[TMP2:[0-9]+]] = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec
-# GCN: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr[[TMP2]], implicit $exec
 ---
 name:            a2_to_a2
 tracksRegLiveness: true
 body:             |
   bb.0:
+    ; GCN-LABEL: name: a2_to_a2
+    ; GCN: $agpr0_agpr1 = IMPLICIT_DEF
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; GCN: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec
+    ; GCN: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit $agpr1_agpr2
     $agpr0_agpr1 = IMPLICIT_DEF
     $agpr1_agpr2 = COPY killed $agpr0_agpr1, implicit $exec
+    S_ENDPGM 0, implicit $agpr1_agpr2
 ...
 
-# GCN-LABEL: name: a_to_a_spill
-# Using last vgpr255 will raise error about absence of emergency spill slot.
 
-# GCN: $vgpr255 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
-# GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr255, implicit $exec
+# Using last vgpr255 will raise error about absence of emergency spill slot.
 
 ---
 name:            a_to_a_spill
@@ -127,6 +214,13 @@ body:             |
   bb.0:
     liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254
 
+    ; GCN-LABEL: name: a_to_a_spill
+    ; GCN: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254
+    ; GCN: $agpr1 = IMPLICIT_DEF
+    ; GCN: $vgpr255 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr255, implicit $exec, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit $agpr0
     $agpr1 = IMPLICIT_DEF
     $agpr0 = COPY killed $agpr1, implicit $exec
+    S_ENDPGM 0, implicit $agpr0
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
index d16edbac75fe2..5c6b3d32ab93d 100644
--- a/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
+++ b/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
@@ -148,10 +148,12 @@ define amdgpu_kernel void @use_constant_to_global_addrspacecast(i32 addrspace(4)
 ; HSA: enable_sgpr_queue_ptr = 0
 
 ; HSA: s_load_dwordx2 s{{\[}}[[PTR_LO:[0-9]+]]:[[PTR_HI:[0-9]+]]{{\]}}
-; HSA-DAG: v_cmp_ne_u64_e64 vcc, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0{{$}}
-; HSA-DAG: v_mov_b32_e32 v[[VPTR_LO:[0-9]+]], s[[PTR_LO]]
-; HSA-DAG: v_cndmask_b32_e32 [[CASTPTR:v[0-9]+]], -1, v[[VPTR_LO]]
 ; HSA-DAG: v_mov_b32_e32 v[[K:[0-9]+]], 0{{$}}
+; CI-DAG: v_cmp_ne_u64_e64 s{{\[}}[[CMP_LO:[0-9]+]]:[[CMP_HI:[0-9]+]]{{\]}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0{{$}}
+; CI-DAG: s_cmp_lg_u32 s[[CMP_LO]], 0
+; GFX9-DAG: s_cmp_lg_u64 s{{\[}}[[CMP_LO:[0-9]+]]:[[CMP_HI:[0-9]+]]{{\]}}, 0
+; HSA-DAG: s_cselect_b32 s[[PTR_LO]], s[[PTR_LO]], -1
+; HSA-DAG: v_mov_b32_e32 [[CASTPTR:v[0-9]+]], s[[PTR_LO]]
 ; HSA: ds_write_b32 [[CASTPTR]], v[[K]]
 define amdgpu_kernel void @use_flat_to_group_addrspacecast(i32* %ptr) #0 {
   %ftos = addrspacecast i32* %ptr to i32 addrspace(3)*
@@ -165,10 +167,12 @@ define amdgpu_kernel void @use_flat_to_group_addrspacecast(i32* %ptr) #0 {
 ; HSA: enable_sgpr_queue_ptr = 0
 
 ; HSA: s_load_dwordx2 s{{\[}}[[PTR_LO:[0-9]+]]:[[PTR_HI:[0-9]+]]{{\]}}
-; HSA-DAG: v_cmp_ne_u64_e64 vcc, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0{{$}}
-; HSA-DAG: v_mov_b32_e32 v[[VPTR_LO:[0-9]+]], s[[PTR_LO]]
-; HSA-DAG: v_cndmask_b32_e32 [[CASTPTR:v[0-9]+]], -1, v[[VPTR_LO]]
 ; HSA-DAG: v_mov_b32_e32 v[[K:[0-9]+]], 0{{$}}
+; CI-DAG: v_cmp_ne_u64_e64 s{{\[}}[[CMP_LO:[0-9]+]]:[[CMP_HI:[0-9]+]]{{\]}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0{{$}}
+; CI-DAG: s_cmp_lg_u32 s[[CMP_LO]], 0
+; GFX9-DAG: s_cmp_lg_u64 s{{\[}}[[CMP_LO:[0-9]+]]:[[CMP_HI:[0-9]+]]{{\]}}, 0
+; HSA-DAG: s_cselect_b32 s[[PTR_LO]], s[[PTR_LO]], -1
+; HSA-DAG: v_mov_b32_e32 [[CASTPTR:v[0-9]+]], s[[PTR_LO]]
 ; HSA: buffer_store_dword v[[K]], [[CASTPTR]], s{{\[[0-9]+:[0-9]+\]}}, 0 offen{{$}}
 define amdgpu_kernel void @use_flat_to_private_addrspacecast(i32* %ptr) #0 {
   %ftos = addrspacecast i32* %ptr to i32 addrspace(5)*
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-remat.ll b/llvm/test/CodeGen/AMDGPU/agpr-remat.ll
new file mode 100644
index 0000000000000..2ec1cee766622
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/agpr-remat.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX908 %s
+
+; Make sure there are no v_accvgpr_read_b32 copying back and forth
+; between AGPR and VGPR.
+define amdgpu_kernel void @remat_constant_voids_spill(i32 addrspace(1)* %p) #1 {
+; GFX908-LABEL: remat_constant_voids_spill:
+; GFX908:       ; %bb.0:
+; GFX908-NEXT:    v_accvgpr_write_b32 a1, 1
+; GFX908-NEXT:    v_accvgpr_write_b32 a5, 6
+; GFX908-NEXT:    v_accvgpr_write_b32 a6, 7
+; GFX908-NEXT:    v_accvgpr_write_b32 a7, 8
+; GFX908-NEXT:    v_accvgpr_write_b32 a0, 9
+; GFX908-NEXT:    v_accvgpr_write_b32 a2, 2
+; GFX908-NEXT:    v_accvgpr_write_b32 a3, 3
+; GFX908-NEXT:    v_accvgpr_write_b32 a4, 4
+; GFX908-NEXT:    ;;#ASMSTART
+; GFX908-NEXT:    ;;#ASMEND
+; GFX908-NEXT:    v_accvgpr_write_b32 a1, 5
+; GFX908-NEXT:    ;;#ASMSTART
+; GFX908-NEXT:    ;;#ASMEND
+; GFX908-NEXT:    s_endpgm
+  call void asm sideeffect "", "a,a,a,a"(i32 1, i32 2, i32 3, i32 4)
+  call void asm sideeffect "", "a,a,a,a,a"(i32 5, i32 6, i32 7, i32 8, i32 9)
+  ret void
+}
+
+define void @remat_regcopy_avoids_spill(i32 %v0, i32 %v1, i32 %v2, i32 %v3, i32 %v4, i32 %v5, i32 %v6, i32 %v7, i32 %v8, i32 %v9, i32 %v10) #1 {
+; GFX908-LABEL: remat_regcopy_avoids_spill:
+; GFX908:       ; %bb.0:
+; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT:    v_accvgpr_write_b32 a1, v0
+; GFX908-NEXT:    v_accvgpr_write_b32 a2, v1
+; GFX908-NEXT:    v_accvgpr_write_b32 a3, v2
+; GFX908-NEXT:    v_accvgpr_write_b32 a4, v3
+; GFX908-NEXT:    ;;#ASMSTART
+; GFX908-NEXT:    ;;#ASMEND
+; GFX908-NEXT:    v_accvgpr_write_b32 a0, v8
+; GFX908-NEXT:    v_accvgpr_write_b32 a1, v4
+; GFX908-NEXT:    v_accvgpr_write_b32 a2, v5
+; GFX908-NEXT:    v_accvgpr_write_b32 a3, v6
+; GFX908-NEXT:    v_accvgpr_write_b32 a4, v7
+; GFX908-NEXT:    ;;#ASMSTART
+; GFX908-NEXT:    ;;#ASMEND
+; GFX908-NEXT:    s_setpc_b64 s[30:31]
+  call void asm sideeffect "", "a,a,a,a"(i32 %v0, i32 %v1, i32 %v2, i32 %v3)
+  call void asm sideeffect "", "a,a,a,a,a"(i32 %v4, i32 %v5, i32 %v6, i32 %v7, i32 %v8)
+  ret void
+}
+
+attributes #1 = { nounwind "amdgpu-num-vgpr"="8" }
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-alias-analysis.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-alias-analysis.ll
index 25ec7af9d2318..583bb60c5acd2 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-alias-analysis.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-alias-analysis.ll
@@ -1,5 +1,5 @@
-; RUN: opt -mtriple=amdgcn-- -aa-eval -amdgpu-aa -amdgpu-aa-wrapper -disable-basicaa  -print-all-alias-modref-info -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt -mtriple=r600-- -aa-eval -amdgpu-aa -amdgpu-aa-wrapper -disable-basicaa  -print-all-alias-modref-info -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt -mtriple=amdgcn-- -aa-eval -amdgpu-aa -amdgpu-aa-wrapper -disable-basic-aa  -print-all-alias-modref-info -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt -mtriple=r600-- -aa-eval -amdgpu-aa -amdgpu-aa-wrapper -disable-basic-aa  -print-all-alias-modref-info -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK: NoAlias:      i8 addrspace(1)* %p1, i8 addrspace(5)* %p
 
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll
index 8b1ec0b013a60..4d1731d55ad57 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll
@@ -93,43 +93,35 @@ define i32 @select_sdiv_lhs_opaque_const0_i32(i1 %cond) {
 ; IR-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP3]], [[TMP1]]
 ; IR-NEXT:    [[TMP5:%.*]] = uitofp i32 [[TMP4]] to float
 ; IR-NEXT:    [[TMP6:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP5]])
-; IR-NEXT:    [[TMP7:%.*]] = fmul fast float [[TMP6]], 0x41F0000000000000
+; IR-NEXT:    [[TMP7:%.*]] = fmul fast float [[TMP6]], 0x41EFFFFFC0000000
 ; IR-NEXT:    [[TMP8:%.*]] = fptoui float [[TMP7]] to i32
-; IR-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP8]] to i64
-; IR-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP4]] to i64
-; IR-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP9]], [[TMP10]]
-; IR-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP11]] to i32
-; IR-NEXT:    [[TMP13:%.*]] = lshr i64 [[TMP11]], 32
+; IR-NEXT:    [[TMP9:%.*]] = sub i32 0, [[TMP4]]
+; IR-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], [[TMP8]]
+; IR-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP8]] to i64
+; IR-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP10]] to i64
+; IR-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP11]], [[TMP12]]
 ; IR-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP13]] to i32
-; IR-NEXT:    [[TMP15:%.*]] = sub i32 0, [[TMP12]]
-; IR-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP14]], 0
-; IR-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP15]], i32 [[TMP12]]
+; IR-NEXT:    [[TMP15:%.*]] = lshr i64 [[TMP13]], 32
+; IR-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32
+; IR-NEXT:    [[TMP17:%.*]] = add i32 [[TMP8]], [[TMP16]]
 ; IR-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP17]] to i64
-; IR-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP8]] to i64
-; IR-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP18]], [[TMP19]]
-; IR-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
-; IR-NEXT:    [[TMP22:%.*]] = lshr i64 [[TMP20]], 32
-; IR-NEXT:    [[TMP23:%.*]] = trunc i64 [[TMP22]] to i32
-; IR-NEXT:    [[TMP24:%.*]] = add i32 [[TMP8]], [[TMP23]]
-; IR-NEXT:    [[TMP25:%.*]] = sub i32 [[TMP8]], [[TMP23]]
-; IR-NEXT:    [[TMP26:%.*]] = select i1 [[TMP16]], i32 [[TMP24]], i32 [[TMP25]]
-; IR-NEXT:    [[TMP27:%.*]] = zext i32 [[TMP26]] to i64
-; IR-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 1000000
-; IR-NEXT:    [[TMP29:%.*]] = trunc i64 [[TMP28]] to i32
-; IR-NEXT:    [[TMP30:%.*]] = lshr i64 [[TMP28]], 32
-; IR-NEXT:    [[TMP31:%.*]] = trunc i64 [[TMP30]] to i32
-; IR-NEXT:    [[TMP32:%.*]] = mul i32 [[TMP31]], [[TMP4]]
-; IR-NEXT:    [[TMP33:%.*]] = sub i32 1000000, [[TMP32]]
-; IR-NEXT:    [[TMP34:%.*]] = icmp uge i32 [[TMP33]], [[TMP4]]
-; IR-NEXT:    [[TMP35:%.*]] = icmp uge i32 1000000, [[TMP32]]
-; IR-NEXT:    [[TMP36:%.*]] = and i1 [[TMP34]], [[TMP35]]
-; IR-NEXT:    [[TMP37:%.*]] = add i32 [[TMP31]], 1
-; IR-NEXT:    [[TMP38:%.*]] = sub i32 [[TMP31]], 1
-; IR-NEXT:    [[TMP39:%.*]] = select i1 [[TMP36]], i32 [[TMP37]], i32 [[TMP31]]
-; IR-NEXT:    [[TMP40:%.*]] = select i1 [[TMP35]], i32 [[TMP39]], i32 [[TMP38]]
-; IR-NEXT:    [[TMP41:%.*]] = xor i32 [[TMP40]], [[TMP2]]
-; IR-NEXT:    [[TMP42:%.*]] = sub i32 [[TMP41]], [[TMP2]]
-; IR-NEXT:    ret i32 [[TMP42]]
+; IR-NEXT:    [[TMP19:%.*]] = mul i64 1000000, [[TMP18]]
+; IR-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
+; IR-NEXT:    [[TMP21:%.*]] = lshr i64 [[TMP19]], 32
+; IR-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP21]] to i32
+; IR-NEXT:    [[TMP23:%.*]] = mul i32 [[TMP22]], [[TMP4]]
+; IR-NEXT:    [[TMP24:%.*]] = sub i32 1000000, [[TMP23]]
+; IR-NEXT:    [[TMP25:%.*]] = icmp uge i32 [[TMP24]], [[TMP4]]
+; IR-NEXT:    [[TMP26:%.*]] = add i32 [[TMP22]], 1
+; IR-NEXT:    [[TMP27:%.*]] = select i1 [[TMP25]], i32 [[TMP26]], i32 [[TMP22]]
+; IR-NEXT:    [[TMP28:%.*]] = sub i32 [[TMP24]], [[TMP4]]
+; IR-NEXT:    [[TMP29:%.*]] = select i1 [[TMP25]], i32 [[TMP28]], i32 [[TMP24]]
+; IR-NEXT:    [[TMP30:%.*]] = icmp uge i32 [[TMP29]], [[TMP4]]
+; IR-NEXT:    [[TMP31:%.*]] = add i32 [[TMP27]], 1
+; IR-NEXT:    [[TMP32:%.*]] = select i1 [[TMP30]], i32 [[TMP31]], i32 [[TMP27]]
+; IR-NEXT:    [[TMP33:%.*]] = xor i32 [[TMP32]], [[TMP2]]
+; IR-NEXT:    [[TMP34:%.*]] = sub i32 [[TMP33]], [[TMP2]]
+; IR-NEXT:    ret i32 [[TMP34]]
 ;
 ; GCN-LABEL: select_sdiv_lhs_opaque_const0_i32:
 ; GCN:       ; %bb.0:
@@ -140,7 +132,6 @@ define i32 @select_sdiv_lhs_opaque_const0_i32(i1 %cond) {
 ; GCN-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GCN-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GCN-NEXT:    s_mov_b32 s6, 0xf4240
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v1, s4
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 5, v1, vcc
@@ -148,29 +139,25 @@ define i32 @select_sdiv_lhs_opaque_const0_i32(i1 %cond) {
 ; GCN-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v2, v0
+; GCN-NEXT:    v_sub_u32_e32 v3, vcc, 0, v0
+; GCN-NEXT:    s_mov_b32 s4, 0xf4240
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GCN-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; GCN-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GCN-NEXT:    v_mul_lo_u32 v3, v3, v2
+; GCN-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GCN-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
+; GCN-NEXT:    v_mul_hi_u32 v2, v2, s4
 ; GCN-NEXT:    v_mul_lo_u32 v3, v2, v0
-; GCN-NEXT:    v_mul_hi_u32 v4, v2, v0
-; GCN-NEXT:    v_sub_u32_e32 v5, vcc, 0, v3
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GCN-NEXT:    v_mul_hi_u32 v3, v3, v2
-; GCN-NEXT:    v_add_u32_e64 v4, s[4:5], v2, v3
-; GCN-NEXT:    v_sub_u32_e64 v2, s[4:5], v2, v3
+; GCN-NEXT:    v_add_u32_e32 v4, vcc, 1, v2
+; GCN-NEXT:    v_sub_u32_e32 v3, vcc, s4, v3
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v0
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GCN-NEXT:    v_mul_hi_u32 v2, v2, s6
-; GCN-NEXT:    s_mov_b32 s4, 0xf4241
-; GCN-NEXT:    v_mul_lo_u32 v3, v2, v0
+; GCN-NEXT:    v_sub_u32_e64 v4, s[4:5], v3, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
 ; GCN-NEXT:    v_add_u32_e32 v4, vcc, 1, v2
-; GCN-NEXT:    v_add_u32_e32 v5, vcc, -1, v2
-; GCN-NEXT:    v_sub_u32_e32 v6, vcc, s6, v3
-; GCN-NEXT:    v_cmp_gt_u32_e32 vcc, s4, v3
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v0
-; GCN-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; GCN-NEXT:    v_sub_u32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
@@ -188,43 +175,35 @@ define i32 @select_sdiv_lhs_opaque_const1_i32(i1 %cond) {
 ; IR-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP3]], [[TMP1]]
 ; IR-NEXT:    [[TMP5:%.*]] = uitofp i32 [[TMP4]] to float
 ; IR-NEXT:    [[TMP6:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP5]])
-; IR-NEXT:    [[TMP7:%.*]] = fmul fast float [[TMP6]], 0x41F0000000000000
+; IR-NEXT:    [[TMP7:%.*]] = fmul fast float [[TMP6]], 0x41EFFFFFC0000000
 ; IR-NEXT:    [[TMP8:%.*]] = fptoui float [[TMP7]] to i32
-; IR-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP8]] to i64
-; IR-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP4]] to i64
-; IR-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP9]], [[TMP10]]
-; IR-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP11]] to i32
-; IR-NEXT:    [[TMP13:%.*]] = lshr i64 [[TMP11]], 32
+; IR-NEXT:    [[TMP9:%.*]] = sub i32 0, [[TMP4]]
+; IR-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], [[TMP8]]
+; IR-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP8]] to i64
+; IR-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP10]] to i64
+; IR-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP11]], [[TMP12]]
 ; IR-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP13]] to i32
-; IR-NEXT:    [[TMP15:%.*]] = sub i32 0, [[TMP12]]
-; IR-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP14]], 0
-; IR-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP15]], i32 [[TMP12]]
+; IR-NEXT:    [[TMP15:%.*]] = lshr i64 [[TMP13]], 32
+; IR-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32
+; IR-NEXT:    [[TMP17:%.*]] = add i32 [[TMP8]], [[TMP16]]
 ; IR-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP17]] to i64
-; IR-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP8]] to i64
-; IR-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP18]], [[TMP19]]
-; IR-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
-; IR-NEXT:    [[TMP22:%.*]] = lshr i64 [[TMP20]], 32
-; IR-NEXT:    [[TMP23:%.*]] = trunc i64 [[TMP22]] to i32
-; IR-NEXT:    [[TMP24:%.*]] = add i32 [[TMP8]], [[TMP23]]
-; IR-NEXT:    [[TMP25:%.*]] = sub i32 [[TMP8]], [[TMP23]]
-; IR-NEXT:    [[TMP26:%.*]] = select i1 [[TMP16]], i32 [[TMP24]], i32 [[TMP25]]
-; IR-NEXT:    [[TMP27:%.*]] = zext i32 [[TMP26]] to i64
-; IR-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 1000000
-; IR-NEXT:    [[TMP29:%.*]] = trunc i64 [[TMP28]] to i32
-; IR-NEXT:    [[TMP30:%.*]] = lshr i64 [[TMP28]], 32
-; IR-NEXT:    [[TMP31:%.*]] = trunc i64 [[TMP30]] to i32
-; IR-NEXT:    [[TMP32:%.*]] = mul i32 [[TMP31]], [[TMP4]]
-; IR-NEXT:    [[TMP33:%.*]] = sub i32 1000000, [[TMP32]]
-; IR-NEXT:    [[TMP34:%.*]] = icmp uge i32 [[TMP33]], [[TMP4]]
-; IR-NEXT:    [[TMP35:%.*]] = icmp uge i32 1000000, [[TMP32]]
-; IR-NEXT:    [[TMP36:%.*]] = and i1 [[TMP34]], [[TMP35]]
-; IR-NEXT:    [[TMP37:%.*]] = add i32 [[TMP31]], 1
-; IR-NEXT:    [[TMP38:%.*]] = sub i32 [[TMP31]], 1
-; IR-NEXT:    [[TMP39:%.*]] = select i1 [[TMP36]], i32 [[TMP37]], i32 [[TMP31]]
-; IR-NEXT:    [[TMP40:%.*]] = select i1 [[TMP35]], i32 [[TMP39]], i32 [[TMP38]]
-; IR-NEXT:    [[TMP41:%.*]] = xor i32 [[TMP40]], [[TMP2]]
-; IR-NEXT:    [[TMP42:%.*]] = sub i32 [[TMP41]], [[TMP2]]
-; IR-NEXT:    ret i32 [[TMP42]]
+; IR-NEXT:    [[TMP19:%.*]] = mul i64 1000000, [[TMP18]]
+; IR-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
+; IR-NEXT:    [[TMP21:%.*]] = lshr i64 [[TMP19]], 32
+; IR-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP21]] to i32
+; IR-NEXT:    [[TMP23:%.*]] = mul i32 [[TMP22]], [[TMP4]]
+; IR-NEXT:    [[TMP24:%.*]] = sub i32 1000000, [[TMP23]]
+; IR-NEXT:    [[TMP25:%.*]] = icmp uge i32 [[TMP24]], [[TMP4]]
+; IR-NEXT:    [[TMP26:%.*]] = add i32 [[TMP22]], 1
+; IR-NEXT:    [[TMP27:%.*]] = select i1 [[TMP25]], i32 [[TMP26]], i32 [[TMP22]]
+; IR-NEXT:    [[TMP28:%.*]] = sub i32 [[TMP24]], [[TMP4]]
+; IR-NEXT:    [[TMP29:%.*]] = select i1 [[TMP25]], i32 [[TMP28]], i32 [[TMP24]]
+; IR-NEXT:    [[TMP30:%.*]] = icmp uge i32 [[TMP29]], [[TMP4]]
+; IR-NEXT:    [[TMP31:%.*]] = add i32 [[TMP27]], 1
+; IR-NEXT:    [[TMP32:%.*]] = select i1 [[TMP30]], i32 [[TMP31]], i32 [[TMP27]]
+; IR-NEXT:    [[TMP33:%.*]] = xor i32 [[TMP32]], [[TMP2]]
+; IR-NEXT:    [[TMP34:%.*]] = sub i32 [[TMP33]], [[TMP2]]
+; IR-NEXT:    ret i32 [[TMP34]]
 ;
 ; GCN-LABEL: select_sdiv_lhs_opaque_const1_i32:
 ; GCN:       ; %bb.0:
@@ -235,7 +214,6 @@ define i32 @select_sdiv_lhs_opaque_const1_i32(i1 %cond) {
 ; GCN-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GCN-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GCN-NEXT:    s_mov_b32 s6, 0xf4240
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v1, s4
 ; GCN-NEXT:    v_cndmask_b32_e64 v0, v1, 5, vcc
@@ -243,29 +221,25 @@ define i32 @select_sdiv_lhs_opaque_const1_i32(i1 %cond) {
 ; GCN-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v2, v0
+; GCN-NEXT:    v_sub_u32_e32 v3, vcc, 0, v0
+; GCN-NEXT:    s_mov_b32 s4, 0xf4240
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GCN-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; GCN-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GCN-NEXT:    v_mul_lo_u32 v3, v3, v2
+; GCN-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GCN-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
+; GCN-NEXT:    v_mul_hi_u32 v2, v2, s4
 ; GCN-NEXT:    v_mul_lo_u32 v3, v2, v0
-; GCN-NEXT:    v_mul_hi_u32 v4, v2, v0
-; GCN-NEXT:    v_sub_u32_e32 v5, vcc, 0, v3
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GCN-NEXT:    v_mul_hi_u32 v3, v3, v2
-; GCN-NEXT:    v_add_u32_e64 v4, s[4:5], v2, v3
-; GCN-NEXT:    v_sub_u32_e64 v2, s[4:5], v2, v3
+; GCN-NEXT:    v_add_u32_e32 v4, vcc, 1, v2
+; GCN-NEXT:    v_sub_u32_e32 v3, vcc, s4, v3
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v0
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GCN-NEXT:    v_mul_hi_u32 v2, v2, s6
-; GCN-NEXT:    s_mov_b32 s4, 0xf4241
-; GCN-NEXT:    v_mul_lo_u32 v3, v2, v0
+; GCN-NEXT:    v_sub_u32_e64 v4, s[4:5], v3, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
 ; GCN-NEXT:    v_add_u32_e32 v4, vcc, 1, v2
-; GCN-NEXT:    v_add_u32_e32 v5, vcc, -1, v2
-; GCN-NEXT:    v_sub_u32_e32 v6, vcc, s6, v3
-; GCN-NEXT:    v_cmp_gt_u32_e32 vcc, s4, v3
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v0
-; GCN-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; GCN-NEXT:    v_sub_u32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
@@ -357,6 +331,7 @@ define float @select_fadd_lhs_const_i32_fmf(i1 %cond) {
 ; IR-LABEL: @select_fadd_lhs_const_i32_fmf(
 ; IR-NEXT:    [[OP:%.*]] = select nnan nsz i1 [[COND:%.*]], float 3.000000e+00, float 5.000000e+00
 ; IR-NEXT:    ret float [[OP]]
+;
 ; GCN-LABEL: select_fadd_lhs_const_i32_fmf:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -385,6 +360,7 @@ define i32 @select_mul_lhs_const_i32(i1 %cond) {
 ; IR-LABEL: @select_mul_lhs_const_i32(
 ; IR-NEXT:    [[OP:%.*]] = select i1 [[COND:%.*]], i32 5000, i32 8000
 ; IR-NEXT:    ret i32 [[OP]]
+;
   %select = select i1 %cond, i32 5, i32 8
   %op = mul i32 1000, %select
   ret i32 %op
@@ -404,6 +380,7 @@ define i32 @select_mul_rhs_const_i32(i1 %cond) {
 ; IR-LABEL: @select_mul_rhs_const_i32(
 ; IR-NEXT:    [[OP:%.*]] = select i1 [[COND:%.*]], i32 5000, i32 8000
 ; IR-NEXT:    ret i32 [[OP]]
+;
   %select = select i1 %cond, i32 5, i32 8
   %op = mul i32 %select, 1000
   ret i32 %op
@@ -412,8 +389,9 @@ define i32 @select_mul_rhs_const_i32(i1 %cond) {
 define amdgpu_kernel void @select_add_lhs_const_i16(i1 %cond) {
 ; IR-LABEL: @select_add_lhs_const_i16(
 ; IR-NEXT:    [[OP:%.*]] = select i1 [[COND:%.*]], i16 128, i16 131
-; IR-NEXT:    store i16 [[OP]], i16 addrspace(1)* undef
+; IR-NEXT:    store i16 [[OP]], i16 addrspace(1)* undef, align 2
 ; IR-NEXT:    ret void
+;
 ; GCN-LABEL: select_add_lhs_const_i16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dword s0, s[4:5], 0x0
@@ -442,6 +420,7 @@ define i16 @select_add_trunc_select(i1 %cond) {
 ; IR-LABEL: @select_add_trunc_select(
 ; IR-NEXT:    [[OP:%.*]] = select i1 [[COND:%.*]], i16 47, i16 50
 ; IR-NEXT:    ret i16 [[OP]]
+;
   %select = select i1 %cond, i32 5, i32 8
   %trunc = trunc i32 %select to i16
   %op = add i16 %trunc, 42
@@ -452,6 +431,7 @@ define i32 @select_add_sext_select(i1 %cond) {
 ; IR-LABEL: @select_add_sext_select(
 ; IR-NEXT:    [[OP:%.*]] = select i1 [[COND:%.*]], i32 29, i32 50
 ; IR-NEXT:    ret i32 [[OP]]
+;
 ; GCN-LABEL: select_add_sext_select:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -469,6 +449,7 @@ define i32 @select_add_zext_select(i1 %cond) {
 ; IR-LABEL: @select_add_zext_select(
 ; IR-NEXT:    [[OP:%.*]] = select i1 [[COND:%.*]], i32 47, i32 50
 ; IR-NEXT:    ret i32 [[OP]]
+;
 ; GCN-LABEL: select_add_zext_select:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
index de7521c3d30d4..53e3005910cd5 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
@@ -7,74 +7,63 @@ define amdgpu_kernel void @udiv_i32(i32 addrspace(1)* %out, i32 %x, i32 %y) {
 ; CHECK-LABEL: @udiv_i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = uitofp i32 [[Y:%.*]] to float
 ; CHECK-NEXT:    [[TMP2:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP1]])
-; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast float [[TMP2]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast float [[TMP2]], 0x41EFFFFFC0000000
 ; CHECK-NEXT:    [[TMP4:%.*]] = fptoui float [[TMP3]] to i32
-; CHECK-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
-; CHECK-NEXT:    [[TMP6:%.*]] = zext i32 [[Y]] to i64
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP5]], [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
-; CHECK-NEXT:    [[TMP9:%.*]] = lshr i64 [[TMP7]], 32
+; CHECK-NEXT:    [[TMP5:%.*]] = sub i32 0, [[Y]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP7]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; CHECK-NEXT:    [[TMP11:%.*]] = sub i32 0, [[TMP8]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP10]], 0
-; CHECK-NEXT:    [[TMP13:%.*]] = select i1 [[TMP12]], i32 [[TMP11]], i32 [[TMP8]]
-; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
-; CHECK-NEXT:    [[TMP15:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = lshr i64 [[TMP9]], 32
+; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[TMP4]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = zext i32 [[TMP13]] to i64
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP14]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32
 ; CHECK-NEXT:    [[TMP18:%.*]] = lshr i64 [[TMP16]], 32
 ; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
-; CHECK-NEXT:    [[TMP20:%.*]] = add i32 [[TMP4]], [[TMP19]]
-; CHECK-NEXT:    [[TMP21:%.*]] = sub i32 [[TMP4]], [[TMP19]]
-; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP12]], i32 [[TMP20]], i32 [[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP22]] to i64
-; CHECK-NEXT:    [[TMP24:%.*]] = zext i32 [[X:%.*]] to i64
-; CHECK-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP23]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP25]] to i32
-; CHECK-NEXT:    [[TMP27:%.*]] = lshr i64 [[TMP25]], 32
-; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i32
-; CHECK-NEXT:    [[TMP29:%.*]] = mul i32 [[TMP28]], [[Y]]
-; CHECK-NEXT:    [[TMP30:%.*]] = sub i32 [[X]], [[TMP29]]
-; CHECK-NEXT:    [[TMP31:%.*]] = icmp uge i32 [[TMP30]], [[Y]]
-; CHECK-NEXT:    [[TMP32:%.*]] = icmp uge i32 [[X]], [[TMP29]]
-; CHECK-NEXT:    [[TMP33:%.*]] = and i1 [[TMP31]], [[TMP32]]
-; CHECK-NEXT:    [[TMP34:%.*]] = add i32 [[TMP28]], 1
-; CHECK-NEXT:    [[TMP35:%.*]] = sub i32 [[TMP28]], 1
-; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP33]], i32 [[TMP34]], i32 [[TMP28]]
-; CHECK-NEXT:    [[TMP37:%.*]] = select i1 [[TMP32]], i32 [[TMP36]], i32 [[TMP35]]
-; CHECK-NEXT:    store i32 [[TMP37]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP20:%.*]] = mul i32 [[TMP19]], [[Y]]
+; CHECK-NEXT:    [[TMP21:%.*]] = sub i32 [[X]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp uge i32 [[TMP21]], [[Y]]
+; CHECK-NEXT:    [[TMP23:%.*]] = add i32 [[TMP19]], 1
+; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP22]], i32 [[TMP23]], i32 [[TMP19]]
+; CHECK-NEXT:    [[TMP25:%.*]] = sub i32 [[TMP21]], [[Y]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP22]], i32 [[TMP25]], i32 [[TMP21]]
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp uge i32 [[TMP26]], [[Y]]
+; CHECK-NEXT:    [[TMP28:%.*]] = add i32 [[TMP24]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = select i1 [[TMP27]], i32 [[TMP28]], i32 [[TMP24]]
+; CHECK-NEXT:    store i32 [[TMP29]], i32 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_i32:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xb
-; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s9
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s3
+; GCN-NEXT:    s_sub_i32 s4, 0, s3
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_mul_lo_u32 v1, v0, s9
-; GCN-NEXT:    v_mul_hi_u32 v2, v0, s9
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v1, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v0, v0, s8
-; GCN-NEXT:    v_mul_lo_u32 v1, v0, s9
+; GCN-NEXT:    v_mul_lo_u32 v1, s4, v0
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v1, v0, s3
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, -1, v0
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s8, v1
-; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, s8, v1
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v4
-; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s2, v1
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s3, v1
 ; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s3, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s3, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = udiv i32 %x, %y
@@ -86,75 +75,59 @@ define amdgpu_kernel void @urem_i32(i32 addrspace(1)* %out, i32 %x, i32 %y) {
 ; CHECK-LABEL: @urem_i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = uitofp i32 [[Y:%.*]] to float
 ; CHECK-NEXT:    [[TMP2:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP1]])
-; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast float [[TMP2]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast float [[TMP2]], 0x41EFFFFFC0000000
 ; CHECK-NEXT:    [[TMP4:%.*]] = fptoui float [[TMP3]] to i32
-; CHECK-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
-; CHECK-NEXT:    [[TMP6:%.*]] = zext i32 [[Y]] to i64
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP5]], [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
-; CHECK-NEXT:    [[TMP9:%.*]] = lshr i64 [[TMP7]], 32
+; CHECK-NEXT:    [[TMP5:%.*]] = sub i32 0, [[Y]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP7]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; CHECK-NEXT:    [[TMP11:%.*]] = sub i32 0, [[TMP8]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP10]], 0
-; CHECK-NEXT:    [[TMP13:%.*]] = select i1 [[TMP12]], i32 [[TMP11]], i32 [[TMP8]]
-; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
-; CHECK-NEXT:    [[TMP15:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = lshr i64 [[TMP9]], 32
+; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[TMP4]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = zext i32 [[TMP13]] to i64
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP14]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32
 ; CHECK-NEXT:    [[TMP18:%.*]] = lshr i64 [[TMP16]], 32
 ; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
-; CHECK-NEXT:    [[TMP20:%.*]] = add i32 [[TMP4]], [[TMP19]]
-; CHECK-NEXT:    [[TMP21:%.*]] = sub i32 [[TMP4]], [[TMP19]]
-; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP12]], i32 [[TMP20]], i32 [[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP22]] to i64
-; CHECK-NEXT:    [[TMP24:%.*]] = zext i32 [[X:%.*]] to i64
-; CHECK-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP23]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP25]] to i32
-; CHECK-NEXT:    [[TMP27:%.*]] = lshr i64 [[TMP25]], 32
-; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i32
-; CHECK-NEXT:    [[TMP29:%.*]] = mul i32 [[TMP28]], [[Y]]
-; CHECK-NEXT:    [[TMP30:%.*]] = sub i32 [[X]], [[TMP29]]
-; CHECK-NEXT:    [[TMP31:%.*]] = icmp uge i32 [[TMP30]], [[Y]]
-; CHECK-NEXT:    [[TMP32:%.*]] = icmp uge i32 [[X]], [[TMP29]]
-; CHECK-NEXT:    [[TMP33:%.*]] = and i1 [[TMP31]], [[TMP32]]
-; CHECK-NEXT:    [[TMP34:%.*]] = sub i32 [[TMP30]], [[Y]]
-; CHECK-NEXT:    [[TMP35:%.*]] = add i32 [[TMP30]], [[Y]]
-; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP33]], i32 [[TMP34]], i32 [[TMP30]]
-; CHECK-NEXT:    [[TMP37:%.*]] = select i1 [[TMP32]], i32 [[TMP36]], i32 [[TMP35]]
-; CHECK-NEXT:    store i32 [[TMP37]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP20:%.*]] = mul i32 [[TMP19]], [[Y]]
+; CHECK-NEXT:    [[TMP21:%.*]] = sub i32 [[X]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp uge i32 [[TMP21]], [[Y]]
+; CHECK-NEXT:    [[TMP23:%.*]] = sub i32 [[TMP21]], [[Y]]
+; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP22]], i32 [[TMP23]], i32 [[TMP21]]
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp uge i32 [[TMP24]], [[Y]]
+; CHECK-NEXT:    [[TMP26:%.*]] = sub i32 [[TMP24]], [[Y]]
+; CHECK-NEXT:    [[TMP27:%.*]] = select i1 [[TMP25]], i32 [[TMP26]], i32 [[TMP24]]
+; CHECK-NEXT:    store i32 [[TMP27]], i32 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_i32:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xb
-; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; GCN-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s9
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s5
+; GCN-NEXT:    s_sub_i32 s2, 0, s5
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_mul_lo_u32 v1, v0, s9
-; GCN-NEXT:    v_mul_hi_u32 v2, v0, s9
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v1, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v0, v0, s8
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s9
-; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s8, v0
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s8, v0
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v1
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, s9, v1
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s9, v1
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[2:3]
-; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    v_mul_lo_u32 v1, s2, v0
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s5
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s4, v0
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s5, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s5, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s5, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s5, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
   %r = urem i32 %x, %y
   store i32 %r, i32 addrspace(1)* %out
@@ -172,44 +145,36 @@ define amdgpu_kernel void @sdiv_i32(i32 addrspace(1)* %out, i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = xor i32 [[TMP5]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = uitofp i32 [[TMP7]] to float
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP8]])
-; CHECK-NEXT:    [[TMP10:%.*]] = fmul fast float [[TMP9]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul fast float [[TMP9]], 0x41EFFFFFC0000000
 ; CHECK-NEXT:    [[TMP11:%.*]] = fptoui float [[TMP10]] to i32
-; CHECK-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
-; CHECK-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP7]] to i64
-; CHECK-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP12]], [[TMP13]]
-; CHECK-NEXT:    [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; CHECK-NEXT:    [[TMP16:%.*]] = lshr i64 [[TMP14]], 32
+; CHECK-NEXT:    [[TMP12:%.*]] = sub i32 0, [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i32 [[TMP12]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP14]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32
-; CHECK-NEXT:    [[TMP18:%.*]] = sub i32 0, [[TMP15]]
-; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[TMP17]], 0
-; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP18]], i32 [[TMP15]]
-; CHECK-NEXT:    [[TMP21:%.*]] = zext i32 [[TMP20]] to i64
-; CHECK-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = lshr i64 [[TMP16]], 32
+; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
+; CHECK-NEXT:    [[TMP20:%.*]] = add i32 [[TMP11]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP20]] to i64
 ; CHECK-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP21]], [[TMP22]]
 ; CHECK-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP23]] to i32
 ; CHECK-NEXT:    [[TMP25:%.*]] = lshr i64 [[TMP23]], 32
 ; CHECK-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP25]] to i32
-; CHECK-NEXT:    [[TMP27:%.*]] = add i32 [[TMP11]], [[TMP26]]
-; CHECK-NEXT:    [[TMP28:%.*]] = sub i32 [[TMP11]], [[TMP26]]
-; CHECK-NEXT:    [[TMP29:%.*]] = select i1 [[TMP19]], i32 [[TMP27]], i32 [[TMP28]]
-; CHECK-NEXT:    [[TMP30:%.*]] = zext i32 [[TMP29]] to i64
-; CHECK-NEXT:    [[TMP31:%.*]] = zext i32 [[TMP6]] to i64
-; CHECK-NEXT:    [[TMP32:%.*]] = mul i64 [[TMP30]], [[TMP31]]
-; CHECK-NEXT:    [[TMP33:%.*]] = trunc i64 [[TMP32]] to i32
-; CHECK-NEXT:    [[TMP34:%.*]] = lshr i64 [[TMP32]], 32
-; CHECK-NEXT:    [[TMP35:%.*]] = trunc i64 [[TMP34]] to i32
-; CHECK-NEXT:    [[TMP36:%.*]] = mul i32 [[TMP35]], [[TMP7]]
-; CHECK-NEXT:    [[TMP37:%.*]] = sub i32 [[TMP6]], [[TMP36]]
-; CHECK-NEXT:    [[TMP38:%.*]] = icmp uge i32 [[TMP37]], [[TMP7]]
-; CHECK-NEXT:    [[TMP39:%.*]] = icmp uge i32 [[TMP6]], [[TMP36]]
-; CHECK-NEXT:    [[TMP40:%.*]] = and i1 [[TMP38]], [[TMP39]]
-; CHECK-NEXT:    [[TMP41:%.*]] = add i32 [[TMP35]], 1
-; CHECK-NEXT:    [[TMP42:%.*]] = sub i32 [[TMP35]], 1
-; CHECK-NEXT:    [[TMP43:%.*]] = select i1 [[TMP40]], i32 [[TMP41]], i32 [[TMP35]]
-; CHECK-NEXT:    [[TMP44:%.*]] = select i1 [[TMP39]], i32 [[TMP43]], i32 [[TMP42]]
-; CHECK-NEXT:    [[TMP45:%.*]] = xor i32 [[TMP44]], [[TMP3]]
-; CHECK-NEXT:    [[TMP46:%.*]] = sub i32 [[TMP45]], [[TMP3]]
-; CHECK-NEXT:    store i32 [[TMP46]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP27:%.*]] = mul i32 [[TMP26]], [[TMP7]]
+; CHECK-NEXT:    [[TMP28:%.*]] = sub i32 [[TMP6]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp uge i32 [[TMP28]], [[TMP7]]
+; CHECK-NEXT:    [[TMP30:%.*]] = add i32 [[TMP26]], 1
+; CHECK-NEXT:    [[TMP31:%.*]] = select i1 [[TMP29]], i32 [[TMP30]], i32 [[TMP26]]
+; CHECK-NEXT:    [[TMP32:%.*]] = sub i32 [[TMP28]], [[TMP7]]
+; CHECK-NEXT:    [[TMP33:%.*]] = select i1 [[TMP29]], i32 [[TMP32]], i32 [[TMP28]]
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp uge i32 [[TMP33]], [[TMP7]]
+; CHECK-NEXT:    [[TMP35:%.*]] = add i32 [[TMP31]], 1
+; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP34]], i32 [[TMP35]], i32 [[TMP31]]
+; CHECK-NEXT:    [[TMP37:%.*]] = xor i32 [[TMP36]], [[TMP3]]
+; CHECK-NEXT:    [[TMP38:%.*]] = sub i32 [[TMP37]], [[TMP3]]
+; CHECK-NEXT:    store i32 [[TMP38]], i32 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_i32:
@@ -223,34 +188,30 @@ define amdgpu_kernel void @sdiv_i32(i32 addrspace(1)* %out, i32 %x, i32 %y) {
 ; GCN-NEXT:    s_add_i32 s3, s3, s8
 ; GCN-NEXT:    s_xor_b32 s9, s3, s8
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s9
-; GCN-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-NEXT:    s_add_i32 s2, s2, s3
-; GCN-NEXT:    s_xor_b32 s2, s2, s3
+; GCN-NEXT:    s_sub_i32 s3, 0, s9
+; GCN-NEXT:    s_ashr_i32 s0, s2, 31
+; GCN-NEXT:    s_add_i32 s1, s2, s0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT:    s_xor_b32 s3, s3, s8
-; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GCN-NEXT:    s_xor_b32 s1, s1, s0
+; GCN-NEXT:    s_xor_b32 s2, s0, s8
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_mul_lo_u32 v1, v0, s9
-; GCN-NEXT:    v_mul_hi_u32 v2, v0, s9
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v1, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v0, v0, s2
+; GCN-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GCN-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s1, v0
 ; GCN-NEXT:    v_mul_lo_u32 v1, v0, s9
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, -1, v0
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s2, v1
-; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, s2, v1
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v4
-; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s1, v1
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v1
 ; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GCN-NEXT:    v_xor_b32_e32 v0, s3, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s3, v0
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s9, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    v_xor_b32_e32 v0, s2, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
 ; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = sdiv i32 %x, %y
@@ -268,86 +229,69 @@ define amdgpu_kernel void @srem_i32(i32 addrspace(1)* %out, i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP4]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = uitofp i32 [[TMP6]] to float
 ; CHECK-NEXT:    [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
-; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast float [[TMP8]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast float [[TMP8]], 0x41EFFFFFC0000000
 ; CHECK-NEXT:    [[TMP10:%.*]] = fptoui float [[TMP9]] to i32
-; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP10]] to i64
-; CHECK-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP6]] to i64
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP13]] to i32
-; CHECK-NEXT:    [[TMP15:%.*]] = lshr i64 [[TMP13]], 32
+; CHECK-NEXT:    [[TMP11:%.*]] = sub i32 0, [[TMP6]]
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i32 [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32
-; CHECK-NEXT:    [[TMP17:%.*]] = sub i32 0, [[TMP14]]
-; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i32 [[TMP16]], 0
-; CHECK-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP17]], i32 [[TMP14]]
-; CHECK-NEXT:    [[TMP20:%.*]] = zext i32 [[TMP19]] to i64
-; CHECK-NEXT:    [[TMP21:%.*]] = zext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = lshr i64 [[TMP15]], 32
+; CHECK-NEXT:    [[TMP18:%.*]] = trunc i64 [[TMP17]] to i32
+; CHECK-NEXT:    [[TMP19:%.*]] = add i32 [[TMP10]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP21:%.*]] = zext i32 [[TMP19]] to i64
 ; CHECK-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP20]], [[TMP21]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = trunc i64 [[TMP22]] to i32
 ; CHECK-NEXT:    [[TMP24:%.*]] = lshr i64 [[TMP22]], 32
 ; CHECK-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP24]] to i32
-; CHECK-NEXT:    [[TMP26:%.*]] = add i32 [[TMP10]], [[TMP25]]
-; CHECK-NEXT:    [[TMP27:%.*]] = sub i32 [[TMP10]], [[TMP25]]
-; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP18]], i32 [[TMP26]], i32 [[TMP27]]
-; CHECK-NEXT:    [[TMP29:%.*]] = zext i32 [[TMP28]] to i64
-; CHECK-NEXT:    [[TMP30:%.*]] = zext i32 [[TMP5]] to i64
-; CHECK-NEXT:    [[TMP31:%.*]] = mul i64 [[TMP29]], [[TMP30]]
-; CHECK-NEXT:    [[TMP32:%.*]] = trunc i64 [[TMP31]] to i32
-; CHECK-NEXT:    [[TMP33:%.*]] = lshr i64 [[TMP31]], 32
-; CHECK-NEXT:    [[TMP34:%.*]] = trunc i64 [[TMP33]] to i32
-; CHECK-NEXT:    [[TMP35:%.*]] = mul i32 [[TMP34]], [[TMP6]]
-; CHECK-NEXT:    [[TMP36:%.*]] = sub i32 [[TMP5]], [[TMP35]]
-; CHECK-NEXT:    [[TMP37:%.*]] = icmp uge i32 [[TMP36]], [[TMP6]]
-; CHECK-NEXT:    [[TMP38:%.*]] = icmp uge i32 [[TMP5]], [[TMP35]]
-; CHECK-NEXT:    [[TMP39:%.*]] = and i1 [[TMP37]], [[TMP38]]
-; CHECK-NEXT:    [[TMP40:%.*]] = sub i32 [[TMP36]], [[TMP6]]
-; CHECK-NEXT:    [[TMP41:%.*]] = add i32 [[TMP36]], [[TMP6]]
-; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP39]], i32 [[TMP40]], i32 [[TMP36]]
-; CHECK-NEXT:    [[TMP43:%.*]] = select i1 [[TMP38]], i32 [[TMP42]], i32 [[TMP41]]
-; CHECK-NEXT:    [[TMP44:%.*]] = xor i32 [[TMP43]], [[TMP1]]
-; CHECK-NEXT:    [[TMP45:%.*]] = sub i32 [[TMP44]], [[TMP1]]
-; CHECK-NEXT:    store i32 [[TMP45]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP26:%.*]] = mul i32 [[TMP25]], [[TMP6]]
+; CHECK-NEXT:    [[TMP27:%.*]] = sub i32 [[TMP5]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp uge i32 [[TMP27]], [[TMP6]]
+; CHECK-NEXT:    [[TMP29:%.*]] = sub i32 [[TMP27]], [[TMP6]]
+; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP28]], i32 [[TMP29]], i32 [[TMP27]]
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp uge i32 [[TMP30]], [[TMP6]]
+; CHECK-NEXT:    [[TMP32:%.*]] = sub i32 [[TMP30]], [[TMP6]]
+; CHECK-NEXT:    [[TMP33:%.*]] = select i1 [[TMP31]], i32 [[TMP32]], i32 [[TMP30]]
+; CHECK-NEXT:    [[TMP34:%.*]] = xor i32 [[TMP33]], [[TMP1]]
+; CHECK-NEXT:    [[TMP35:%.*]] = sub i32 [[TMP34]], [[TMP1]]
+; CHECK-NEXT:    store i32 [[TMP35]], i32 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_i32:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
-; GCN-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_ashr_i32 s2, s5, 31
-; GCN-NEXT:    s_add_i32 s3, s5, s2
-; GCN-NEXT:    s_xor_b32 s10, s3, s2
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s10
-; GCN-NEXT:    s_ashr_i32 s8, s4, 31
-; GCN-NEXT:    s_add_i32 s4, s4, s8
-; GCN-NEXT:    s_xor_b32 s9, s4, s8
+; GCN-NEXT:    s_ashr_i32 s4, s3, 31
+; GCN-NEXT:    s_add_i32 s3, s3, s4
+; GCN-NEXT:    s_xor_b32 s4, s3, s4
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s4
+; GCN-NEXT:    s_sub_i32 s3, 0, s4
+; GCN-NEXT:    s_ashr_i32 s5, s2, 31
+; GCN-NEXT:    s_add_i32 s2, s2, s5
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GCN-NEXT:    s_xor_b32 s6, s2, s5
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_mul_lo_u32 v1, v0, s10
-; GCN-NEXT:    v_mul_hi_u32 v2, v0, s10
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v1, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v0, v0, s9
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s10
-; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s9, v0
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s9, v0
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s10, v1
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, s10, v1
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s10, v1
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[2:3]
-; GCN-NEXT:    v_xor_b32_e32 v0, s8, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s8, v0
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s6, v0
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s4
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s4, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s4, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-NEXT:    v_xor_b32_e32 v0, s5, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s5, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
   %r = srem i32 %x, %y
   store i32 %r, i32 addrspace(1)* %out
@@ -373,7 +317,7 @@ define amdgpu_kernel void @udiv_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP10]], [[TMP14]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP15]], 65535
 ; CHECK-NEXT:    [[TMP17:%.*]] = trunc i32 [[TMP16]] to i16
-; CHECK-NEXT:    store i16 [[TMP17]], i16 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i16 [[TMP17]], i16 addrspace(1)* [[OUT:%.*]], align 2
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_i16:
@@ -422,7 +366,7 @@ define amdgpu_kernel void @urem_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; CHECK-NEXT:    [[TMP17:%.*]] = sub i32 [[TMP1]], [[TMP16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP17]], 65535
 ; CHECK-NEXT:    [[TMP19:%.*]] = trunc i32 [[TMP18]] to i16
-; CHECK-NEXT:    store i16 [[TMP19]], i16 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i16 [[TMP19]], i16 addrspace(1)* [[OUT:%.*]], align 2
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_i16:
@@ -475,7 +419,7 @@ define amdgpu_kernel void @sdiv_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; CHECK-NEXT:    [[TMP19:%.*]] = shl i32 [[TMP18]], 16
 ; CHECK-NEXT:    [[TMP20:%.*]] = ashr i32 [[TMP19]], 16
 ; CHECK-NEXT:    [[TMP21:%.*]] = trunc i32 [[TMP20]] to i16
-; CHECK-NEXT:    store i16 [[TMP21]], i16 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i16 [[TMP21]], i16 addrspace(1)* [[OUT:%.*]], align 2
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_i16:
@@ -492,15 +436,15 @@ define amdgpu_kernel void @sdiv_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; GCN-NEXT:    s_xor_b32 s0, s0, s1
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-NEXT:    s_or_b32 s0, s0, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s0
+; GCN-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cselect_b32 s0, s2, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
 ; GCN-NEXT:    buffer_store_short v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = sdiv i16 %x, %y
@@ -533,7 +477,7 @@ define amdgpu_kernel void @srem_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; CHECK-NEXT:    [[TMP21:%.*]] = shl i32 [[TMP20]], 16
 ; CHECK-NEXT:    [[TMP22:%.*]] = ashr i32 [[TMP21]], 16
 ; CHECK-NEXT:    [[TMP23:%.*]] = trunc i32 [[TMP22]] to i16
-; CHECK-NEXT:    store i16 [[TMP23]], i16 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i16 [[TMP23]], i16 addrspace(1)* [[OUT:%.*]], align 2
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_i16:
@@ -541,24 +485,24 @@ define amdgpu_kernel void @srem_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; GCN-NEXT:    s_load_dword s4, s[0:1], 0xb
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_ashr_i32 s2, s4, 16
-; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s2
-; GCN-NEXT:    s_sext_i32_i16 s3, s4
-; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s3
-; GCN-NEXT:    s_xor_b32 s3, s3, s2
+; GCN-NEXT:    s_ashr_i32 s5, s4, 16
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s5
+; GCN-NEXT:    s_sext_i32_i16 s2, s4
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s2
+; GCN-NEXT:    s_xor_b32 s2, s2, s5
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-NEXT:    s_ashr_i32 s3, s3, 30
-; GCN-NEXT:    s_or_b32 s3, s3, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s3
+; GCN-NEXT:    s_ashr_i32 s2, s2, 30
+; GCN-NEXT:    s_or_b32 s6, s2, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s2, 0
+; GCN-NEXT:    s_cselect_b32 s2, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s5
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
 ; GCN-NEXT:    s_mov_b32 s2, -1
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s4, v0
 ; GCN-NEXT:    buffer_store_short v0, off, s[0:3], 0
@@ -587,7 +531,7 @@ define amdgpu_kernel void @udiv_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP10]], [[TMP14]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP15]], 255
 ; CHECK-NEXT:    [[TMP17:%.*]] = trunc i32 [[TMP16]] to i8
-; CHECK-NEXT:    store i8 [[TMP17]], i8 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i8 [[TMP17]], i8 addrspace(1)* [[OUT:%.*]], align 1
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_i8:
@@ -634,7 +578,7 @@ define amdgpu_kernel void @urem_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; CHECK-NEXT:    [[TMP17:%.*]] = sub i32 [[TMP1]], [[TMP16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP17]], 255
 ; CHECK-NEXT:    [[TMP19:%.*]] = trunc i32 [[TMP18]] to i8
-; CHECK-NEXT:    store i8 [[TMP19]], i8 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i8 [[TMP19]], i8 addrspace(1)* [[OUT:%.*]], align 1
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_i8:
@@ -686,7 +630,7 @@ define amdgpu_kernel void @sdiv_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; CHECK-NEXT:    [[TMP19:%.*]] = shl i32 [[TMP18]], 24
 ; CHECK-NEXT:    [[TMP20:%.*]] = ashr i32 [[TMP19]], 24
 ; CHECK-NEXT:    [[TMP21:%.*]] = trunc i32 [[TMP20]] to i8
-; CHECK-NEXT:    store i8 [[TMP21]], i8 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i8 [[TMP21]], i8 addrspace(1)* [[OUT:%.*]], align 1
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_i8:
@@ -703,15 +647,15 @@ define amdgpu_kernel void @sdiv_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; GCN-NEXT:    s_xor_b32 s0, s0, s1
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-NEXT:    s_or_b32 s0, s0, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s0
+; GCN-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cselect_b32 s0, s2, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
 ; GCN-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = sdiv i8 %x, %y
@@ -744,35 +688,35 @@ define amdgpu_kernel void @srem_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; CHECK-NEXT:    [[TMP21:%.*]] = shl i32 [[TMP20]], 24
 ; CHECK-NEXT:    [[TMP22:%.*]] = ashr i32 [[TMP21]], 24
 ; CHECK-NEXT:    [[TMP23:%.*]] = trunc i32 [[TMP22]] to i8
-; CHECK-NEXT:    store i8 [[TMP23]], i8 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i8 [[TMP23]], i8 addrspace(1)* [[OUT:%.*]], align 1
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_i8:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0xb
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_bfe_i32 s1, s0, 0x80008
-; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s1
-; GCN-NEXT:    s_sext_i32_i8 s3, s0
-; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s3
-; GCN-NEXT:    s_xor_b32 s1, s3, s1
+; GCN-NEXT:    s_bfe_i32 s0, s2, 0x80008
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s0
+; GCN-NEXT:    s_sext_i32_i8 s1, s2
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s1
+; GCN-NEXT:    s_xor_b32 s0, s1, s0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-NEXT:    s_ashr_i32 s1, s1, 30
-; GCN-NEXT:    s_or_b32 s1, s1, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_lshr_b32 s3, s2, 8
+; GCN-NEXT:    s_or_b32 s6, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    s_lshr_b32 s2, s0, 8
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cselect_b32 s0, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s3
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
 ; GCN-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = srem i8 %x, %y
@@ -786,266 +730,219 @@ define amdgpu_kernel void @udiv_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %x
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[Y:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = uitofp i32 [[TMP2]] to float
 ; CHECK-NEXT:    [[TMP4:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[TMP4]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[TMP4]], 0x41EFFFFFC0000000
 ; CHECK-NEXT:    [[TMP6:%.*]] = fptoui float [[TMP5]] to i32
-; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
-; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP2]] to i64
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; CHECK-NEXT:    [[TMP11:%.*]] = lshr i64 [[TMP9]], 32
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i32 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP9]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP11]] to i32
-; CHECK-NEXT:    [[TMP13:%.*]] = sub i32 0, [[TMP10]]
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP12]], 0
-; CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP13]], i32 [[TMP10]]
-; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP15]] to i64
-; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = lshr i64 [[TMP11]], 32
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP6]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP15]] to i64
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP16]], [[TMP17]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
 ; CHECK-NEXT:    [[TMP20:%.*]] = lshr i64 [[TMP18]], 32
 ; CHECK-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
-; CHECK-NEXT:    [[TMP22:%.*]] = add i32 [[TMP6]], [[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = sub i32 [[TMP6]], [[TMP21]]
-; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP14]], i32 [[TMP22]], i32 [[TMP23]]
-; CHECK-NEXT:    [[TMP25:%.*]] = zext i32 [[TMP24]] to i64
-; CHECK-NEXT:    [[TMP26:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP25]], [[TMP26]]
-; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i32
-; CHECK-NEXT:    [[TMP29:%.*]] = lshr i64 [[TMP27]], 32
-; CHECK-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP29]] to i32
-; CHECK-NEXT:    [[TMP31:%.*]] = mul i32 [[TMP30]], [[TMP2]]
-; CHECK-NEXT:    [[TMP32:%.*]] = sub i32 [[TMP1]], [[TMP31]]
-; CHECK-NEXT:    [[TMP33:%.*]] = icmp uge i32 [[TMP32]], [[TMP2]]
-; CHECK-NEXT:    [[TMP34:%.*]] = icmp uge i32 [[TMP1]], [[TMP31]]
-; CHECK-NEXT:    [[TMP35:%.*]] = and i1 [[TMP33]], [[TMP34]]
-; CHECK-NEXT:    [[TMP36:%.*]] = add i32 [[TMP30]], 1
-; CHECK-NEXT:    [[TMP37:%.*]] = sub i32 [[TMP30]], 1
-; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP35]], i32 [[TMP36]], i32 [[TMP30]]
-; CHECK-NEXT:    [[TMP39:%.*]] = select i1 [[TMP34]], i32 [[TMP38]], i32 [[TMP37]]
-; CHECK-NEXT:    [[TMP40:%.*]] = insertelement <4 x i32> undef, i32 [[TMP39]], i64 0
-; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <4 x i32> [[X]], i64 1
-; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <4 x i32> [[Y]], i64 1
-; CHECK-NEXT:    [[TMP43:%.*]] = uitofp i32 [[TMP42]] to float
-; CHECK-NEXT:    [[TMP44:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP43]])
-; CHECK-NEXT:    [[TMP45:%.*]] = fmul fast float [[TMP44]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP46:%.*]] = fptoui float [[TMP45]] to i32
-; CHECK-NEXT:    [[TMP47:%.*]] = zext i32 [[TMP46]] to i64
-; CHECK-NEXT:    [[TMP48:%.*]] = zext i32 [[TMP42]] to i64
-; CHECK-NEXT:    [[TMP49:%.*]] = mul i64 [[TMP47]], [[TMP48]]
-; CHECK-NEXT:    [[TMP50:%.*]] = trunc i64 [[TMP49]] to i32
-; CHECK-NEXT:    [[TMP51:%.*]] = lshr i64 [[TMP49]], 32
-; CHECK-NEXT:    [[TMP52:%.*]] = trunc i64 [[TMP51]] to i32
-; CHECK-NEXT:    [[TMP53:%.*]] = sub i32 0, [[TMP50]]
-; CHECK-NEXT:    [[TMP54:%.*]] = icmp eq i32 [[TMP52]], 0
-; CHECK-NEXT:    [[TMP55:%.*]] = select i1 [[TMP54]], i32 [[TMP53]], i32 [[TMP50]]
-; CHECK-NEXT:    [[TMP56:%.*]] = zext i32 [[TMP55]] to i64
-; CHECK-NEXT:    [[TMP57:%.*]] = zext i32 [[TMP46]] to i64
-; CHECK-NEXT:    [[TMP58:%.*]] = mul i64 [[TMP56]], [[TMP57]]
-; CHECK-NEXT:    [[TMP59:%.*]] = trunc i64 [[TMP58]] to i32
-; CHECK-NEXT:    [[TMP60:%.*]] = lshr i64 [[TMP58]], 32
-; CHECK-NEXT:    [[TMP61:%.*]] = trunc i64 [[TMP60]] to i32
-; CHECK-NEXT:    [[TMP62:%.*]] = add i32 [[TMP46]], [[TMP61]]
-; CHECK-NEXT:    [[TMP63:%.*]] = sub i32 [[TMP46]], [[TMP61]]
-; CHECK-NEXT:    [[TMP64:%.*]] = select i1 [[TMP54]], i32 [[TMP62]], i32 [[TMP63]]
-; CHECK-NEXT:    [[TMP65:%.*]] = zext i32 [[TMP64]] to i64
-; CHECK-NEXT:    [[TMP66:%.*]] = zext i32 [[TMP41]] to i64
-; CHECK-NEXT:    [[TMP67:%.*]] = mul i64 [[TMP65]], [[TMP66]]
-; CHECK-NEXT:    [[TMP68:%.*]] = trunc i64 [[TMP67]] to i32
-; CHECK-NEXT:    [[TMP69:%.*]] = lshr i64 [[TMP67]], 32
-; CHECK-NEXT:    [[TMP70:%.*]] = trunc i64 [[TMP69]] to i32
-; CHECK-NEXT:    [[TMP71:%.*]] = mul i32 [[TMP70]], [[TMP42]]
-; CHECK-NEXT:    [[TMP72:%.*]] = sub i32 [[TMP41]], [[TMP71]]
-; CHECK-NEXT:    [[TMP73:%.*]] = icmp uge i32 [[TMP72]], [[TMP42]]
-; CHECK-NEXT:    [[TMP74:%.*]] = icmp uge i32 [[TMP41]], [[TMP71]]
-; CHECK-NEXT:    [[TMP75:%.*]] = and i1 [[TMP73]], [[TMP74]]
-; CHECK-NEXT:    [[TMP76:%.*]] = add i32 [[TMP70]], 1
-; CHECK-NEXT:    [[TMP77:%.*]] = sub i32 [[TMP70]], 1
-; CHECK-NEXT:    [[TMP78:%.*]] = select i1 [[TMP75]], i32 [[TMP76]], i32 [[TMP70]]
-; CHECK-NEXT:    [[TMP79:%.*]] = select i1 [[TMP74]], i32 [[TMP78]], i32 [[TMP77]]
-; CHECK-NEXT:    [[TMP80:%.*]] = insertelement <4 x i32> [[TMP40]], i32 [[TMP79]], i64 1
-; CHECK-NEXT:    [[TMP81:%.*]] = extractelement <4 x i32> [[X]], i64 2
-; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <4 x i32> [[Y]], i64 2
-; CHECK-NEXT:    [[TMP83:%.*]] = uitofp i32 [[TMP82]] to float
-; CHECK-NEXT:    [[TMP84:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP83]])
-; CHECK-NEXT:    [[TMP85:%.*]] = fmul fast float [[TMP84]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP86:%.*]] = fptoui float [[TMP85]] to i32
-; CHECK-NEXT:    [[TMP87:%.*]] = zext i32 [[TMP86]] to i64
-; CHECK-NEXT:    [[TMP88:%.*]] = zext i32 [[TMP82]] to i64
-; CHECK-NEXT:    [[TMP89:%.*]] = mul i64 [[TMP87]], [[TMP88]]
-; CHECK-NEXT:    [[TMP90:%.*]] = trunc i64 [[TMP89]] to i32
-; CHECK-NEXT:    [[TMP91:%.*]] = lshr i64 [[TMP89]], 32
-; CHECK-NEXT:    [[TMP92:%.*]] = trunc i64 [[TMP91]] to i32
-; CHECK-NEXT:    [[TMP93:%.*]] = sub i32 0, [[TMP90]]
-; CHECK-NEXT:    [[TMP94:%.*]] = icmp eq i32 [[TMP92]], 0
-; CHECK-NEXT:    [[TMP95:%.*]] = select i1 [[TMP94]], i32 [[TMP93]], i32 [[TMP90]]
-; CHECK-NEXT:    [[TMP96:%.*]] = zext i32 [[TMP95]] to i64
-; CHECK-NEXT:    [[TMP97:%.*]] = zext i32 [[TMP86]] to i64
-; CHECK-NEXT:    [[TMP98:%.*]] = mul i64 [[TMP96]], [[TMP97]]
-; CHECK-NEXT:    [[TMP99:%.*]] = trunc i64 [[TMP98]] to i32
-; CHECK-NEXT:    [[TMP100:%.*]] = lshr i64 [[TMP98]], 32
-; CHECK-NEXT:    [[TMP101:%.*]] = trunc i64 [[TMP100]] to i32
-; CHECK-NEXT:    [[TMP102:%.*]] = add i32 [[TMP86]], [[TMP101]]
-; CHECK-NEXT:    [[TMP103:%.*]] = sub i32 [[TMP86]], [[TMP101]]
-; CHECK-NEXT:    [[TMP104:%.*]] = select i1 [[TMP94]], i32 [[TMP102]], i32 [[TMP103]]
-; CHECK-NEXT:    [[TMP105:%.*]] = zext i32 [[TMP104]] to i64
-; CHECK-NEXT:    [[TMP106:%.*]] = zext i32 [[TMP81]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = mul i32 [[TMP21]], [[TMP2]]
+; CHECK-NEXT:    [[TMP23:%.*]] = sub i32 [[TMP1]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp uge i32 [[TMP23]], [[TMP2]]
+; CHECK-NEXT:    [[TMP25:%.*]] = add i32 [[TMP21]], 1
+; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP24]], i32 [[TMP25]], i32 [[TMP21]]
+; CHECK-NEXT:    [[TMP27:%.*]] = sub i32 [[TMP23]], [[TMP2]]
+; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP24]], i32 [[TMP27]], i32 [[TMP23]]
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp uge i32 [[TMP28]], [[TMP2]]
+; CHECK-NEXT:    [[TMP30:%.*]] = add i32 [[TMP26]], 1
+; CHECK-NEXT:    [[TMP31:%.*]] = select i1 [[TMP29]], i32 [[TMP30]], i32 [[TMP26]]
+; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <4 x i32> undef, i32 [[TMP31]], i64 0
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <4 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <4 x i32> [[Y]], i64 1
+; CHECK-NEXT:    [[TMP35:%.*]] = uitofp i32 [[TMP34]] to float
+; CHECK-NEXT:    [[TMP36:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP35]])
+; CHECK-NEXT:    [[TMP37:%.*]] = fmul fast float [[TMP36]], 0x41EFFFFFC0000000
+; CHECK-NEXT:    [[TMP38:%.*]] = fptoui float [[TMP37]] to i32
+; CHECK-NEXT:    [[TMP39:%.*]] = sub i32 0, [[TMP34]]
+; CHECK-NEXT:    [[TMP40:%.*]] = mul i32 [[TMP39]], [[TMP38]]
+; CHECK-NEXT:    [[TMP41:%.*]] = zext i32 [[TMP38]] to i64
+; CHECK-NEXT:    [[TMP42:%.*]] = zext i32 [[TMP40]] to i64
+; CHECK-NEXT:    [[TMP43:%.*]] = mul i64 [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP44:%.*]] = trunc i64 [[TMP43]] to i32
+; CHECK-NEXT:    [[TMP45:%.*]] = lshr i64 [[TMP43]], 32
+; CHECK-NEXT:    [[TMP46:%.*]] = trunc i64 [[TMP45]] to i32
+; CHECK-NEXT:    [[TMP47:%.*]] = add i32 [[TMP38]], [[TMP46]]
+; CHECK-NEXT:    [[TMP48:%.*]] = zext i32 [[TMP33]] to i64
+; CHECK-NEXT:    [[TMP49:%.*]] = zext i32 [[TMP47]] to i64
+; CHECK-NEXT:    [[TMP50:%.*]] = mul i64 [[TMP48]], [[TMP49]]
+; CHECK-NEXT:    [[TMP51:%.*]] = trunc i64 [[TMP50]] to i32
+; CHECK-NEXT:    [[TMP52:%.*]] = lshr i64 [[TMP50]], 32
+; CHECK-NEXT:    [[TMP53:%.*]] = trunc i64 [[TMP52]] to i32
+; CHECK-NEXT:    [[TMP54:%.*]] = mul i32 [[TMP53]], [[TMP34]]
+; CHECK-NEXT:    [[TMP55:%.*]] = sub i32 [[TMP33]], [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = icmp uge i32 [[TMP55]], [[TMP34]]
+; CHECK-NEXT:    [[TMP57:%.*]] = add i32 [[TMP53]], 1
+; CHECK-NEXT:    [[TMP58:%.*]] = select i1 [[TMP56]], i32 [[TMP57]], i32 [[TMP53]]
+; CHECK-NEXT:    [[TMP59:%.*]] = sub i32 [[TMP55]], [[TMP34]]
+; CHECK-NEXT:    [[TMP60:%.*]] = select i1 [[TMP56]], i32 [[TMP59]], i32 [[TMP55]]
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp uge i32 [[TMP60]], [[TMP34]]
+; CHECK-NEXT:    [[TMP62:%.*]] = add i32 [[TMP58]], 1
+; CHECK-NEXT:    [[TMP63:%.*]] = select i1 [[TMP61]], i32 [[TMP62]], i32 [[TMP58]]
+; CHECK-NEXT:    [[TMP64:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP63]], i64 1
+; CHECK-NEXT:    [[TMP65:%.*]] = extractelement <4 x i32> [[X]], i64 2
+; CHECK-NEXT:    [[TMP66:%.*]] = extractelement <4 x i32> [[Y]], i64 2
+; CHECK-NEXT:    [[TMP67:%.*]] = uitofp i32 [[TMP66]] to float
+; CHECK-NEXT:    [[TMP68:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP67]])
+; CHECK-NEXT:    [[TMP69:%.*]] = fmul fast float [[TMP68]], 0x41EFFFFFC0000000
+; CHECK-NEXT:    [[TMP70:%.*]] = fptoui float [[TMP69]] to i32
+; CHECK-NEXT:    [[TMP71:%.*]] = sub i32 0, [[TMP66]]
+; CHECK-NEXT:    [[TMP72:%.*]] = mul i32 [[TMP71]], [[TMP70]]
+; CHECK-NEXT:    [[TMP73:%.*]] = zext i32 [[TMP70]] to i64
+; CHECK-NEXT:    [[TMP74:%.*]] = zext i32 [[TMP72]] to i64
+; CHECK-NEXT:    [[TMP75:%.*]] = mul i64 [[TMP73]], [[TMP74]]
+; CHECK-NEXT:    [[TMP76:%.*]] = trunc i64 [[TMP75]] to i32
+; CHECK-NEXT:    [[TMP77:%.*]] = lshr i64 [[TMP75]], 32
+; CHECK-NEXT:    [[TMP78:%.*]] = trunc i64 [[TMP77]] to i32
+; CHECK-NEXT:    [[TMP79:%.*]] = add i32 [[TMP70]], [[TMP78]]
+; CHECK-NEXT:    [[TMP80:%.*]] = zext i32 [[TMP65]] to i64
+; CHECK-NEXT:    [[TMP81:%.*]] = zext i32 [[TMP79]] to i64
+; CHECK-NEXT:    [[TMP82:%.*]] = mul i64 [[TMP80]], [[TMP81]]
+; CHECK-NEXT:    [[TMP83:%.*]] = trunc i64 [[TMP82]] to i32
+; CHECK-NEXT:    [[TMP84:%.*]] = lshr i64 [[TMP82]], 32
+; CHECK-NEXT:    [[TMP85:%.*]] = trunc i64 [[TMP84]] to i32
+; CHECK-NEXT:    [[TMP86:%.*]] = mul i32 [[TMP85]], [[TMP66]]
+; CHECK-NEXT:    [[TMP87:%.*]] = sub i32 [[TMP65]], [[TMP86]]
+; CHECK-NEXT:    [[TMP88:%.*]] = icmp uge i32 [[TMP87]], [[TMP66]]
+; CHECK-NEXT:    [[TMP89:%.*]] = add i32 [[TMP85]], 1
+; CHECK-NEXT:    [[TMP90:%.*]] = select i1 [[TMP88]], i32 [[TMP89]], i32 [[TMP85]]
+; CHECK-NEXT:    [[TMP91:%.*]] = sub i32 [[TMP87]], [[TMP66]]
+; CHECK-NEXT:    [[TMP92:%.*]] = select i1 [[TMP88]], i32 [[TMP91]], i32 [[TMP87]]
+; CHECK-NEXT:    [[TMP93:%.*]] = icmp uge i32 [[TMP92]], [[TMP66]]
+; CHECK-NEXT:    [[TMP94:%.*]] = add i32 [[TMP90]], 1
+; CHECK-NEXT:    [[TMP95:%.*]] = select i1 [[TMP93]], i32 [[TMP94]], i32 [[TMP90]]
+; CHECK-NEXT:    [[TMP96:%.*]] = insertelement <4 x i32> [[TMP64]], i32 [[TMP95]], i64 2
+; CHECK-NEXT:    [[TMP97:%.*]] = extractelement <4 x i32> [[X]], i64 3
+; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <4 x i32> [[Y]], i64 3
+; CHECK-NEXT:    [[TMP99:%.*]] = uitofp i32 [[TMP98]] to float
+; CHECK-NEXT:    [[TMP100:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP99]])
+; CHECK-NEXT:    [[TMP101:%.*]] = fmul fast float [[TMP100]], 0x41EFFFFFC0000000
+; CHECK-NEXT:    [[TMP102:%.*]] = fptoui float [[TMP101]] to i32
+; CHECK-NEXT:    [[TMP103:%.*]] = sub i32 0, [[TMP98]]
+; CHECK-NEXT:    [[TMP104:%.*]] = mul i32 [[TMP103]], [[TMP102]]
+; CHECK-NEXT:    [[TMP105:%.*]] = zext i32 [[TMP102]] to i64
+; CHECK-NEXT:    [[TMP106:%.*]] = zext i32 [[TMP104]] to i64
 ; CHECK-NEXT:    [[TMP107:%.*]] = mul i64 [[TMP105]], [[TMP106]]
 ; CHECK-NEXT:    [[TMP108:%.*]] = trunc i64 [[TMP107]] to i32
 ; CHECK-NEXT:    [[TMP109:%.*]] = lshr i64 [[TMP107]], 32
 ; CHECK-NEXT:    [[TMP110:%.*]] = trunc i64 [[TMP109]] to i32
-; CHECK-NEXT:    [[TMP111:%.*]] = mul i32 [[TMP110]], [[TMP82]]
-; CHECK-NEXT:    [[TMP112:%.*]] = sub i32 [[TMP81]], [[TMP111]]
-; CHECK-NEXT:    [[TMP113:%.*]] = icmp uge i32 [[TMP112]], [[TMP82]]
-; CHECK-NEXT:    [[TMP114:%.*]] = icmp uge i32 [[TMP81]], [[TMP111]]
-; CHECK-NEXT:    [[TMP115:%.*]] = and i1 [[TMP113]], [[TMP114]]
-; CHECK-NEXT:    [[TMP116:%.*]] = add i32 [[TMP110]], 1
-; CHECK-NEXT:    [[TMP117:%.*]] = sub i32 [[TMP110]], 1
-; CHECK-NEXT:    [[TMP118:%.*]] = select i1 [[TMP115]], i32 [[TMP116]], i32 [[TMP110]]
-; CHECK-NEXT:    [[TMP119:%.*]] = select i1 [[TMP114]], i32 [[TMP118]], i32 [[TMP117]]
-; CHECK-NEXT:    [[TMP120:%.*]] = insertelement <4 x i32> [[TMP80]], i32 [[TMP119]], i64 2
-; CHECK-NEXT:    [[TMP121:%.*]] = extractelement <4 x i32> [[X]], i64 3
-; CHECK-NEXT:    [[TMP122:%.*]] = extractelement <4 x i32> [[Y]], i64 3
-; CHECK-NEXT:    [[TMP123:%.*]] = uitofp i32 [[TMP122]] to float
-; CHECK-NEXT:    [[TMP124:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP123]])
-; CHECK-NEXT:    [[TMP125:%.*]] = fmul fast float [[TMP124]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP126:%.*]] = fptoui float [[TMP125]] to i32
-; CHECK-NEXT:    [[TMP127:%.*]] = zext i32 [[TMP126]] to i64
-; CHECK-NEXT:    [[TMP128:%.*]] = zext i32 [[TMP122]] to i64
-; CHECK-NEXT:    [[TMP129:%.*]] = mul i64 [[TMP127]], [[TMP128]]
-; CHECK-NEXT:    [[TMP130:%.*]] = trunc i64 [[TMP129]] to i32
-; CHECK-NEXT:    [[TMP131:%.*]] = lshr i64 [[TMP129]], 32
-; CHECK-NEXT:    [[TMP132:%.*]] = trunc i64 [[TMP131]] to i32
-; CHECK-NEXT:    [[TMP133:%.*]] = sub i32 0, [[TMP130]]
-; CHECK-NEXT:    [[TMP134:%.*]] = icmp eq i32 [[TMP132]], 0
-; CHECK-NEXT:    [[TMP135:%.*]] = select i1 [[TMP134]], i32 [[TMP133]], i32 [[TMP130]]
-; CHECK-NEXT:    [[TMP136:%.*]] = zext i32 [[TMP135]] to i64
-; CHECK-NEXT:    [[TMP137:%.*]] = zext i32 [[TMP126]] to i64
-; CHECK-NEXT:    [[TMP138:%.*]] = mul i64 [[TMP136]], [[TMP137]]
-; CHECK-NEXT:    [[TMP139:%.*]] = trunc i64 [[TMP138]] to i32
-; CHECK-NEXT:    [[TMP140:%.*]] = lshr i64 [[TMP138]], 32
-; CHECK-NEXT:    [[TMP141:%.*]] = trunc i64 [[TMP140]] to i32
-; CHECK-NEXT:    [[TMP142:%.*]] = add i32 [[TMP126]], [[TMP141]]
-; CHECK-NEXT:    [[TMP143:%.*]] = sub i32 [[TMP126]], [[TMP141]]
-; CHECK-NEXT:    [[TMP144:%.*]] = select i1 [[TMP134]], i32 [[TMP142]], i32 [[TMP143]]
-; CHECK-NEXT:    [[TMP145:%.*]] = zext i32 [[TMP144]] to i64
-; CHECK-NEXT:    [[TMP146:%.*]] = zext i32 [[TMP121]] to i64
-; CHECK-NEXT:    [[TMP147:%.*]] = mul i64 [[TMP145]], [[TMP146]]
-; CHECK-NEXT:    [[TMP148:%.*]] = trunc i64 [[TMP147]] to i32
-; CHECK-NEXT:    [[TMP149:%.*]] = lshr i64 [[TMP147]], 32
-; CHECK-NEXT:    [[TMP150:%.*]] = trunc i64 [[TMP149]] to i32
-; CHECK-NEXT:    [[TMP151:%.*]] = mul i32 [[TMP150]], [[TMP122]]
-; CHECK-NEXT:    [[TMP152:%.*]] = sub i32 [[TMP121]], [[TMP151]]
-; CHECK-NEXT:    [[TMP153:%.*]] = icmp uge i32 [[TMP152]], [[TMP122]]
-; CHECK-NEXT:    [[TMP154:%.*]] = icmp uge i32 [[TMP121]], [[TMP151]]
-; CHECK-NEXT:    [[TMP155:%.*]] = and i1 [[TMP153]], [[TMP154]]
-; CHECK-NEXT:    [[TMP156:%.*]] = add i32 [[TMP150]], 1
-; CHECK-NEXT:    [[TMP157:%.*]] = sub i32 [[TMP150]], 1
-; CHECK-NEXT:    [[TMP158:%.*]] = select i1 [[TMP155]], i32 [[TMP156]], i32 [[TMP150]]
-; CHECK-NEXT:    [[TMP159:%.*]] = select i1 [[TMP154]], i32 [[TMP158]], i32 [[TMP157]]
-; CHECK-NEXT:    [[TMP160:%.*]] = insertelement <4 x i32> [[TMP120]], i32 [[TMP159]], i64 3
-; CHECK-NEXT:    store <4 x i32> [[TMP160]], <4 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP111:%.*]] = add i32 [[TMP102]], [[TMP110]]
+; CHECK-NEXT:    [[TMP112:%.*]] = zext i32 [[TMP97]] to i64
+; CHECK-NEXT:    [[TMP113:%.*]] = zext i32 [[TMP111]] to i64
+; CHECK-NEXT:    [[TMP114:%.*]] = mul i64 [[TMP112]], [[TMP113]]
+; CHECK-NEXT:    [[TMP115:%.*]] = trunc i64 [[TMP114]] to i32
+; CHECK-NEXT:    [[TMP116:%.*]] = lshr i64 [[TMP114]], 32
+; CHECK-NEXT:    [[TMP117:%.*]] = trunc i64 [[TMP116]] to i32
+; CHECK-NEXT:    [[TMP118:%.*]] = mul i32 [[TMP117]], [[TMP98]]
+; CHECK-NEXT:    [[TMP119:%.*]] = sub i32 [[TMP97]], [[TMP118]]
+; CHECK-NEXT:    [[TMP120:%.*]] = icmp uge i32 [[TMP119]], [[TMP98]]
+; CHECK-NEXT:    [[TMP121:%.*]] = add i32 [[TMP117]], 1
+; CHECK-NEXT:    [[TMP122:%.*]] = select i1 [[TMP120]], i32 [[TMP121]], i32 [[TMP117]]
+; CHECK-NEXT:    [[TMP123:%.*]] = sub i32 [[TMP119]], [[TMP98]]
+; CHECK-NEXT:    [[TMP124:%.*]] = select i1 [[TMP120]], i32 [[TMP123]], i32 [[TMP119]]
+; CHECK-NEXT:    [[TMP125:%.*]] = icmp uge i32 [[TMP124]], [[TMP98]]
+; CHECK-NEXT:    [[TMP126:%.*]] = add i32 [[TMP122]], 1
+; CHECK-NEXT:    [[TMP127:%.*]] = select i1 [[TMP125]], i32 [[TMP126]], i32 [[TMP122]]
+; CHECK-NEXT:    [[TMP128:%.*]] = insertelement <4 x i32> [[TMP96]], i32 [[TMP127]], i64 3
+; CHECK-NEXT:    store <4 x i32> [[TMP128]], <4 x i32> addrspace(1)* [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_v4i32:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx8 s[8:15], s[0:1], 0xd
-; GCN-NEXT:    s_mov_b32 s6, 0x4f800000
-; GCN-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x9
-; GCN-NEXT:    s_mov_b32 s19, 0xf000
-; GCN-NEXT:    s_mov_b32 s18, -1
+; GCN-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s12, 0x4f7ffffe
+; GCN-NEXT:    s_mov_b32 s15, 0xf000
+; GCN-NEXT:    s_mov_b32 s14, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s12
-; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s13
-; GCN-NEXT:    v_cvt_f32_u32_e32 v7, s15
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s8
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s9
+; GCN-NEXT:    s_sub_i32 s2, 0, s8
+; GCN-NEXT:    v_cvt_f32_u32_e32 v3, s10
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; GCN-NEXT:    v_mul_f32_e32 v0, s6, v0
+; GCN-NEXT:    v_mul_f32_e32 v0, s12, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_mul_f32_e32 v1, s6, v1
+; GCN-NEXT:    v_mul_f32_e32 v1, s12, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    v_mul_hi_u32 v2, v0, s12
-; GCN-NEXT:    v_mul_lo_u32 v3, v0, s12
-; GCN-NEXT:    v_mul_hi_u32 v4, v1, s13
-; GCN-NEXT:    v_mul_lo_u32 v5, v1, s13
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_sub_i32_e32 v6, vcc, 0, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v3, v6, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v2, v2, v0
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v5
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v2, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v2, v0
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v0, v0, s8
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v3, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v2, v2, v1
-; GCN-NEXT:    v_mul_lo_u32 v3, v0, s12
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, -1, v0
-; GCN-NEXT:    v_sub_i32_e32 v5, vcc, s8, v3
-; GCN-NEXT:    v_cmp_le_u32_e64 s[4:5], s12, v5
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v2, v1
-; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v2, v1
-; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s14
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, s9
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s8, v3
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
-; GCN-NEXT:    s_and_b64 vcc, s[4:5], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GCN-NEXT:    v_mul_f32_e32 v2, s6, v2
+; GCN-NEXT:    v_mul_lo_u32 v2, s2, v0
+; GCN-NEXT:    s_sub_i32 s2, 0, s9
+; GCN-NEXT:    v_mul_lo_u32 v4, s2, v1
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v3
+; GCN-NEXT:    v_mul_hi_u32 v3, v1, v4
+; GCN-NEXT:    v_mul_lo_u32 v4, v0, s8
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v0
+; GCN-NEXT:    v_mul_f32_e32 v2, s12, v2
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GCN-NEXT:    v_mul_lo_u32 v3, v1, s13
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v6, v2, s14
-; GCN-NEXT:    v_mul_lo_u32 v5, v2, s14
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s9, v3
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s9, v3
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v6
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v3, v3, v2
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v4
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, -1, v1
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v1
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v3, v2
-; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v7
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v2, v2, s10
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_mul_f32_e32 v3, s6, v3
-; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GCN-NEXT:    v_mul_lo_u32 v5, v2, s14
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v7, v3, s15
-; GCN-NEXT:    v_mul_lo_u32 v6, v3, s15
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s10, v5
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s14, v4
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v7
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v4, v4, v3
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, -1, v2
-; GCN-NEXT:    v_add_i32_e32 v7, vcc, v4, v3
-; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, v4, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v3, v3, s11
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s10, v5
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_mul_lo_u32 v5, v3, s15
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[2:3]
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s11, v5
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s15, v4
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s11, v5
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, -1, v3
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s4, v4
+; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s8, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v5, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s8, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v5, s[2:3]
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s8, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
+; GCN-NEXT:    v_mul_hi_u32 v1, s5, v1
+; GCN-NEXT:    s_sub_i32 s4, 0, s10
+; GCN-NEXT:    v_mul_lo_u32 v5, s4, v2
+; GCN-NEXT:    s_sub_i32 s4, 0, s11
+; GCN-NEXT:    v_mul_lo_u32 v3, v1, s9
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v1
+; GCN-NEXT:    v_mul_hi_u32 v5, v2, v5
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s5, v3
+; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s9, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v4, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, s9, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v4, s[2:3]
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v1
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; GCN-NEXT:    v_cvt_f32_u32_e32 v4, s11
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_mul_hi_u32 v2, s6, v2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GCN-NEXT:    v_mul_lo_u32 v3, v2, s10
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v2
+; GCN-NEXT:    v_mul_f32_e32 v4, s12, v4
+; GCN-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s10, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[2:3]
+; GCN-NEXT:    v_mul_lo_u32 v6, s4, v4
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s10, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v5, s[2:3]
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v4, v6
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s10, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GCN-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x9
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v4
+; GCN-NEXT:    v_mul_hi_u32 v3, s7, v3
+; GCN-NEXT:    v_mul_lo_u32 v4, v3, s11
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s7, v4
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s11, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v5, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s11, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v5, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s11, v4
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v3, v4, v3, s[2:3]
-; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[16:19], 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[12:15], 0
 ; GCN-NEXT:    s_endpgm
   %r = udiv <4 x i32> %x, %y
   store <4 x i32> %r, <4 x i32> addrspace(1)* %out
@@ -1058,266 +955,202 @@ define amdgpu_kernel void @urem_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %x
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[Y:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = uitofp i32 [[TMP2]] to float
 ; CHECK-NEXT:    [[TMP4:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[TMP4]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[TMP4]], 0x41EFFFFFC0000000
 ; CHECK-NEXT:    [[TMP6:%.*]] = fptoui float [[TMP5]] to i32
-; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
-; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP2]] to i64
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; CHECK-NEXT:    [[TMP11:%.*]] = lshr i64 [[TMP9]], 32
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i32 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP9]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP11]] to i32
-; CHECK-NEXT:    [[TMP13:%.*]] = sub i32 0, [[TMP10]]
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP12]], 0
-; CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP13]], i32 [[TMP10]]
-; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP15]] to i64
-; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = lshr i64 [[TMP11]], 32
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP6]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP15]] to i64
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP16]], [[TMP17]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
 ; CHECK-NEXT:    [[TMP20:%.*]] = lshr i64 [[TMP18]], 32
 ; CHECK-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
-; CHECK-NEXT:    [[TMP22:%.*]] = add i32 [[TMP6]], [[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = sub i32 [[TMP6]], [[TMP21]]
-; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP14]], i32 [[TMP22]], i32 [[TMP23]]
-; CHECK-NEXT:    [[TMP25:%.*]] = zext i32 [[TMP24]] to i64
-; CHECK-NEXT:    [[TMP26:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP25]], [[TMP26]]
-; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i32
-; CHECK-NEXT:    [[TMP29:%.*]] = lshr i64 [[TMP27]], 32
-; CHECK-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP29]] to i32
-; CHECK-NEXT:    [[TMP31:%.*]] = mul i32 [[TMP30]], [[TMP2]]
-; CHECK-NEXT:    [[TMP32:%.*]] = sub i32 [[TMP1]], [[TMP31]]
-; CHECK-NEXT:    [[TMP33:%.*]] = icmp uge i32 [[TMP32]], [[TMP2]]
-; CHECK-NEXT:    [[TMP34:%.*]] = icmp uge i32 [[TMP1]], [[TMP31]]
-; CHECK-NEXT:    [[TMP35:%.*]] = and i1 [[TMP33]], [[TMP34]]
-; CHECK-NEXT:    [[TMP36:%.*]] = sub i32 [[TMP32]], [[TMP2]]
-; CHECK-NEXT:    [[TMP37:%.*]] = add i32 [[TMP32]], [[TMP2]]
-; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP35]], i32 [[TMP36]], i32 [[TMP32]]
-; CHECK-NEXT:    [[TMP39:%.*]] = select i1 [[TMP34]], i32 [[TMP38]], i32 [[TMP37]]
-; CHECK-NEXT:    [[TMP40:%.*]] = insertelement <4 x i32> undef, i32 [[TMP39]], i64 0
-; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <4 x i32> [[X]], i64 1
-; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <4 x i32> [[Y]], i64 1
-; CHECK-NEXT:    [[TMP43:%.*]] = uitofp i32 [[TMP42]] to float
-; CHECK-NEXT:    [[TMP44:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP43]])
-; CHECK-NEXT:    [[TMP45:%.*]] = fmul fast float [[TMP44]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP46:%.*]] = fptoui float [[TMP45]] to i32
-; CHECK-NEXT:    [[TMP47:%.*]] = zext i32 [[TMP46]] to i64
-; CHECK-NEXT:    [[TMP48:%.*]] = zext i32 [[TMP42]] to i64
-; CHECK-NEXT:    [[TMP49:%.*]] = mul i64 [[TMP47]], [[TMP48]]
-; CHECK-NEXT:    [[TMP50:%.*]] = trunc i64 [[TMP49]] to i32
-; CHECK-NEXT:    [[TMP51:%.*]] = lshr i64 [[TMP49]], 32
-; CHECK-NEXT:    [[TMP52:%.*]] = trunc i64 [[TMP51]] to i32
-; CHECK-NEXT:    [[TMP53:%.*]] = sub i32 0, [[TMP50]]
-; CHECK-NEXT:    [[TMP54:%.*]] = icmp eq i32 [[TMP52]], 0
-; CHECK-NEXT:    [[TMP55:%.*]] = select i1 [[TMP54]], i32 [[TMP53]], i32 [[TMP50]]
-; CHECK-NEXT:    [[TMP56:%.*]] = zext i32 [[TMP55]] to i64
-; CHECK-NEXT:    [[TMP57:%.*]] = zext i32 [[TMP46]] to i64
-; CHECK-NEXT:    [[TMP58:%.*]] = mul i64 [[TMP56]], [[TMP57]]
-; CHECK-NEXT:    [[TMP59:%.*]] = trunc i64 [[TMP58]] to i32
-; CHECK-NEXT:    [[TMP60:%.*]] = lshr i64 [[TMP58]], 32
-; CHECK-NEXT:    [[TMP61:%.*]] = trunc i64 [[TMP60]] to i32
-; CHECK-NEXT:    [[TMP62:%.*]] = add i32 [[TMP46]], [[TMP61]]
-; CHECK-NEXT:    [[TMP63:%.*]] = sub i32 [[TMP46]], [[TMP61]]
-; CHECK-NEXT:    [[TMP64:%.*]] = select i1 [[TMP54]], i32 [[TMP62]], i32 [[TMP63]]
-; CHECK-NEXT:    [[TMP65:%.*]] = zext i32 [[TMP64]] to i64
-; CHECK-NEXT:    [[TMP66:%.*]] = zext i32 [[TMP41]] to i64
-; CHECK-NEXT:    [[TMP67:%.*]] = mul i64 [[TMP65]], [[TMP66]]
-; CHECK-NEXT:    [[TMP68:%.*]] = trunc i64 [[TMP67]] to i32
-; CHECK-NEXT:    [[TMP69:%.*]] = lshr i64 [[TMP67]], 32
-; CHECK-NEXT:    [[TMP70:%.*]] = trunc i64 [[TMP69]] to i32
-; CHECK-NEXT:    [[TMP71:%.*]] = mul i32 [[TMP70]], [[TMP42]]
-; CHECK-NEXT:    [[TMP72:%.*]] = sub i32 [[TMP41]], [[TMP71]]
-; CHECK-NEXT:    [[TMP73:%.*]] = icmp uge i32 [[TMP72]], [[TMP42]]
-; CHECK-NEXT:    [[TMP74:%.*]] = icmp uge i32 [[TMP41]], [[TMP71]]
-; CHECK-NEXT:    [[TMP75:%.*]] = and i1 [[TMP73]], [[TMP74]]
-; CHECK-NEXT:    [[TMP76:%.*]] = sub i32 [[TMP72]], [[TMP42]]
-; CHECK-NEXT:    [[TMP77:%.*]] = add i32 [[TMP72]], [[TMP42]]
-; CHECK-NEXT:    [[TMP78:%.*]] = select i1 [[TMP75]], i32 [[TMP76]], i32 [[TMP72]]
-; CHECK-NEXT:    [[TMP79:%.*]] = select i1 [[TMP74]], i32 [[TMP78]], i32 [[TMP77]]
-; CHECK-NEXT:    [[TMP80:%.*]] = insertelement <4 x i32> [[TMP40]], i32 [[TMP79]], i64 1
-; CHECK-NEXT:    [[TMP81:%.*]] = extractelement <4 x i32> [[X]], i64 2
-; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <4 x i32> [[Y]], i64 2
-; CHECK-NEXT:    [[TMP83:%.*]] = uitofp i32 [[TMP82]] to float
-; CHECK-NEXT:    [[TMP84:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP83]])
-; CHECK-NEXT:    [[TMP85:%.*]] = fmul fast float [[TMP84]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP86:%.*]] = fptoui float [[TMP85]] to i32
-; CHECK-NEXT:    [[TMP87:%.*]] = zext i32 [[TMP86]] to i64
-; CHECK-NEXT:    [[TMP88:%.*]] = zext i32 [[TMP82]] to i64
-; CHECK-NEXT:    [[TMP89:%.*]] = mul i64 [[TMP87]], [[TMP88]]
-; CHECK-NEXT:    [[TMP90:%.*]] = trunc i64 [[TMP89]] to i32
-; CHECK-NEXT:    [[TMP91:%.*]] = lshr i64 [[TMP89]], 32
-; CHECK-NEXT:    [[TMP92:%.*]] = trunc i64 [[TMP91]] to i32
-; CHECK-NEXT:    [[TMP93:%.*]] = sub i32 0, [[TMP90]]
-; CHECK-NEXT:    [[TMP94:%.*]] = icmp eq i32 [[TMP92]], 0
-; CHECK-NEXT:    [[TMP95:%.*]] = select i1 [[TMP94]], i32 [[TMP93]], i32 [[TMP90]]
-; CHECK-NEXT:    [[TMP96:%.*]] = zext i32 [[TMP95]] to i64
-; CHECK-NEXT:    [[TMP97:%.*]] = zext i32 [[TMP86]] to i64
-; CHECK-NEXT:    [[TMP98:%.*]] = mul i64 [[TMP96]], [[TMP97]]
-; CHECK-NEXT:    [[TMP99:%.*]] = trunc i64 [[TMP98]] to i32
-; CHECK-NEXT:    [[TMP100:%.*]] = lshr i64 [[TMP98]], 32
-; CHECK-NEXT:    [[TMP101:%.*]] = trunc i64 [[TMP100]] to i32
-; CHECK-NEXT:    [[TMP102:%.*]] = add i32 [[TMP86]], [[TMP101]]
-; CHECK-NEXT:    [[TMP103:%.*]] = sub i32 [[TMP86]], [[TMP101]]
-; CHECK-NEXT:    [[TMP104:%.*]] = select i1 [[TMP94]], i32 [[TMP102]], i32 [[TMP103]]
-; CHECK-NEXT:    [[TMP105:%.*]] = zext i32 [[TMP104]] to i64
-; CHECK-NEXT:    [[TMP106:%.*]] = zext i32 [[TMP81]] to i64
-; CHECK-NEXT:    [[TMP107:%.*]] = mul i64 [[TMP105]], [[TMP106]]
-; CHECK-NEXT:    [[TMP108:%.*]] = trunc i64 [[TMP107]] to i32
-; CHECK-NEXT:    [[TMP109:%.*]] = lshr i64 [[TMP107]], 32
-; CHECK-NEXT:    [[TMP110:%.*]] = trunc i64 [[TMP109]] to i32
-; CHECK-NEXT:    [[TMP111:%.*]] = mul i32 [[TMP110]], [[TMP82]]
-; CHECK-NEXT:    [[TMP112:%.*]] = sub i32 [[TMP81]], [[TMP111]]
-; CHECK-NEXT:    [[TMP113:%.*]] = icmp uge i32 [[TMP112]], [[TMP82]]
-; CHECK-NEXT:    [[TMP114:%.*]] = icmp uge i32 [[TMP81]], [[TMP111]]
-; CHECK-NEXT:    [[TMP115:%.*]] = and i1 [[TMP113]], [[TMP114]]
-; CHECK-NEXT:    [[TMP116:%.*]] = sub i32 [[TMP112]], [[TMP82]]
-; CHECK-NEXT:    [[TMP117:%.*]] = add i32 [[TMP112]], [[TMP82]]
-; CHECK-NEXT:    [[TMP118:%.*]] = select i1 [[TMP115]], i32 [[TMP116]], i32 [[TMP112]]
-; CHECK-NEXT:    [[TMP119:%.*]] = select i1 [[TMP114]], i32 [[TMP118]], i32 [[TMP117]]
-; CHECK-NEXT:    [[TMP120:%.*]] = insertelement <4 x i32> [[TMP80]], i32 [[TMP119]], i64 2
-; CHECK-NEXT:    [[TMP121:%.*]] = extractelement <4 x i32> [[X]], i64 3
-; CHECK-NEXT:    [[TMP122:%.*]] = extractelement <4 x i32> [[Y]], i64 3
-; CHECK-NEXT:    [[TMP123:%.*]] = uitofp i32 [[TMP122]] to float
-; CHECK-NEXT:    [[TMP124:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP123]])
-; CHECK-NEXT:    [[TMP125:%.*]] = fmul fast float [[TMP124]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP126:%.*]] = fptoui float [[TMP125]] to i32
-; CHECK-NEXT:    [[TMP127:%.*]] = zext i32 [[TMP126]] to i64
-; CHECK-NEXT:    [[TMP128:%.*]] = zext i32 [[TMP122]] to i64
-; CHECK-NEXT:    [[TMP129:%.*]] = mul i64 [[TMP127]], [[TMP128]]
-; CHECK-NEXT:    [[TMP130:%.*]] = trunc i64 [[TMP129]] to i32
-; CHECK-NEXT:    [[TMP131:%.*]] = lshr i64 [[TMP129]], 32
-; CHECK-NEXT:    [[TMP132:%.*]] = trunc i64 [[TMP131]] to i32
-; CHECK-NEXT:    [[TMP133:%.*]] = sub i32 0, [[TMP130]]
-; CHECK-NEXT:    [[TMP134:%.*]] = icmp eq i32 [[TMP132]], 0
-; CHECK-NEXT:    [[TMP135:%.*]] = select i1 [[TMP134]], i32 [[TMP133]], i32 [[TMP130]]
-; CHECK-NEXT:    [[TMP136:%.*]] = zext i32 [[TMP135]] to i64
-; CHECK-NEXT:    [[TMP137:%.*]] = zext i32 [[TMP126]] to i64
-; CHECK-NEXT:    [[TMP138:%.*]] = mul i64 [[TMP136]], [[TMP137]]
-; CHECK-NEXT:    [[TMP139:%.*]] = trunc i64 [[TMP138]] to i32
-; CHECK-NEXT:    [[TMP140:%.*]] = lshr i64 [[TMP138]], 32
-; CHECK-NEXT:    [[TMP141:%.*]] = trunc i64 [[TMP140]] to i32
-; CHECK-NEXT:    [[TMP142:%.*]] = add i32 [[TMP126]], [[TMP141]]
-; CHECK-NEXT:    [[TMP143:%.*]] = sub i32 [[TMP126]], [[TMP141]]
-; CHECK-NEXT:    [[TMP144:%.*]] = select i1 [[TMP134]], i32 [[TMP142]], i32 [[TMP143]]
-; CHECK-NEXT:    [[TMP145:%.*]] = zext i32 [[TMP144]] to i64
-; CHECK-NEXT:    [[TMP146:%.*]] = zext i32 [[TMP121]] to i64
-; CHECK-NEXT:    [[TMP147:%.*]] = mul i64 [[TMP145]], [[TMP146]]
-; CHECK-NEXT:    [[TMP148:%.*]] = trunc i64 [[TMP147]] to i32
-; CHECK-NEXT:    [[TMP149:%.*]] = lshr i64 [[TMP147]], 32
-; CHECK-NEXT:    [[TMP150:%.*]] = trunc i64 [[TMP149]] to i32
-; CHECK-NEXT:    [[TMP151:%.*]] = mul i32 [[TMP150]], [[TMP122]]
-; CHECK-NEXT:    [[TMP152:%.*]] = sub i32 [[TMP121]], [[TMP151]]
-; CHECK-NEXT:    [[TMP153:%.*]] = icmp uge i32 [[TMP152]], [[TMP122]]
-; CHECK-NEXT:    [[TMP154:%.*]] = icmp uge i32 [[TMP121]], [[TMP151]]
-; CHECK-NEXT:    [[TMP155:%.*]] = and i1 [[TMP153]], [[TMP154]]
-; CHECK-NEXT:    [[TMP156:%.*]] = sub i32 [[TMP152]], [[TMP122]]
-; CHECK-NEXT:    [[TMP157:%.*]] = add i32 [[TMP152]], [[TMP122]]
-; CHECK-NEXT:    [[TMP158:%.*]] = select i1 [[TMP155]], i32 [[TMP156]], i32 [[TMP152]]
-; CHECK-NEXT:    [[TMP159:%.*]] = select i1 [[TMP154]], i32 [[TMP158]], i32 [[TMP157]]
-; CHECK-NEXT:    [[TMP160:%.*]] = insertelement <4 x i32> [[TMP120]], i32 [[TMP159]], i64 3
-; CHECK-NEXT:    store <4 x i32> [[TMP160]], <4 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP22:%.*]] = mul i32 [[TMP21]], [[TMP2]]
+; CHECK-NEXT:    [[TMP23:%.*]] = sub i32 [[TMP1]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp uge i32 [[TMP23]], [[TMP2]]
+; CHECK-NEXT:    [[TMP25:%.*]] = sub i32 [[TMP23]], [[TMP2]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP24]], i32 [[TMP25]], i32 [[TMP23]]
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp uge i32 [[TMP26]], [[TMP2]]
+; CHECK-NEXT:    [[TMP28:%.*]] = sub i32 [[TMP26]], [[TMP2]]
+; CHECK-NEXT:    [[TMP29:%.*]] = select i1 [[TMP27]], i32 [[TMP28]], i32 [[TMP26]]
+; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <4 x i32> undef, i32 [[TMP29]], i64 0
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <4 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <4 x i32> [[Y]], i64 1
+; CHECK-NEXT:    [[TMP33:%.*]] = uitofp i32 [[TMP32]] to float
+; CHECK-NEXT:    [[TMP34:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP33]])
+; CHECK-NEXT:    [[TMP35:%.*]] = fmul fast float [[TMP34]], 0x41EFFFFFC0000000
+; CHECK-NEXT:    [[TMP36:%.*]] = fptoui float [[TMP35]] to i32
+; CHECK-NEXT:    [[TMP37:%.*]] = sub i32 0, [[TMP32]]
+; CHECK-NEXT:    [[TMP38:%.*]] = mul i32 [[TMP37]], [[TMP36]]
+; CHECK-NEXT:    [[TMP39:%.*]] = zext i32 [[TMP36]] to i64
+; CHECK-NEXT:    [[TMP40:%.*]] = zext i32 [[TMP38]] to i64
+; CHECK-NEXT:    [[TMP41:%.*]] = mul i64 [[TMP39]], [[TMP40]]
+; CHECK-NEXT:    [[TMP42:%.*]] = trunc i64 [[TMP41]] to i32
+; CHECK-NEXT:    [[TMP43:%.*]] = lshr i64 [[TMP41]], 32
+; CHECK-NEXT:    [[TMP44:%.*]] = trunc i64 [[TMP43]] to i32
+; CHECK-NEXT:    [[TMP45:%.*]] = add i32 [[TMP36]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = zext i32 [[TMP31]] to i64
+; CHECK-NEXT:    [[TMP47:%.*]] = zext i32 [[TMP45]] to i64
+; CHECK-NEXT:    [[TMP48:%.*]] = mul i64 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP49:%.*]] = trunc i64 [[TMP48]] to i32
+; CHECK-NEXT:    [[TMP50:%.*]] = lshr i64 [[TMP48]], 32
+; CHECK-NEXT:    [[TMP51:%.*]] = trunc i64 [[TMP50]] to i32
+; CHECK-NEXT:    [[TMP52:%.*]] = mul i32 [[TMP51]], [[TMP32]]
+; CHECK-NEXT:    [[TMP53:%.*]] = sub i32 [[TMP31]], [[TMP52]]
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp uge i32 [[TMP53]], [[TMP32]]
+; CHECK-NEXT:    [[TMP55:%.*]] = sub i32 [[TMP53]], [[TMP32]]
+; CHECK-NEXT:    [[TMP56:%.*]] = select i1 [[TMP54]], i32 [[TMP55]], i32 [[TMP53]]
+; CHECK-NEXT:    [[TMP57:%.*]] = icmp uge i32 [[TMP56]], [[TMP32]]
+; CHECK-NEXT:    [[TMP58:%.*]] = sub i32 [[TMP56]], [[TMP32]]
+; CHECK-NEXT:    [[TMP59:%.*]] = select i1 [[TMP57]], i32 [[TMP58]], i32 [[TMP56]]
+; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <4 x i32> [[TMP30]], i32 [[TMP59]], i64 1
+; CHECK-NEXT:    [[TMP61:%.*]] = extractelement <4 x i32> [[X]], i64 2
+; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <4 x i32> [[Y]], i64 2
+; CHECK-NEXT:    [[TMP63:%.*]] = uitofp i32 [[TMP62]] to float
+; CHECK-NEXT:    [[TMP64:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP63]])
+; CHECK-NEXT:    [[TMP65:%.*]] = fmul fast float [[TMP64]], 0x41EFFFFFC0000000
+; CHECK-NEXT:    [[TMP66:%.*]] = fptoui float [[TMP65]] to i32
+; CHECK-NEXT:    [[TMP67:%.*]] = sub i32 0, [[TMP62]]
+; CHECK-NEXT:    [[TMP68:%.*]] = mul i32 [[TMP67]], [[TMP66]]
+; CHECK-NEXT:    [[TMP69:%.*]] = zext i32 [[TMP66]] to i64
+; CHECK-NEXT:    [[TMP70:%.*]] = zext i32 [[TMP68]] to i64
+; CHECK-NEXT:    [[TMP71:%.*]] = mul i64 [[TMP69]], [[TMP70]]
+; CHECK-NEXT:    [[TMP72:%.*]] = trunc i64 [[TMP71]] to i32
+; CHECK-NEXT:    [[TMP73:%.*]] = lshr i64 [[TMP71]], 32
+; CHECK-NEXT:    [[TMP74:%.*]] = trunc i64 [[TMP73]] to i32
+; CHECK-NEXT:    [[TMP75:%.*]] = add i32 [[TMP66]], [[TMP74]]
+; CHECK-NEXT:    [[TMP76:%.*]] = zext i32 [[TMP61]] to i64
+; CHECK-NEXT:    [[TMP77:%.*]] = zext i32 [[TMP75]] to i64
+; CHECK-NEXT:    [[TMP78:%.*]] = mul i64 [[TMP76]], [[TMP77]]
+; CHECK-NEXT:    [[TMP79:%.*]] = trunc i64 [[TMP78]] to i32
+; CHECK-NEXT:    [[TMP80:%.*]] = lshr i64 [[TMP78]], 32
+; CHECK-NEXT:    [[TMP81:%.*]] = trunc i64 [[TMP80]] to i32
+; CHECK-NEXT:    [[TMP82:%.*]] = mul i32 [[TMP81]], [[TMP62]]
+; CHECK-NEXT:    [[TMP83:%.*]] = sub i32 [[TMP61]], [[TMP82]]
+; CHECK-NEXT:    [[TMP84:%.*]] = icmp uge i32 [[TMP83]], [[TMP62]]
+; CHECK-NEXT:    [[TMP85:%.*]] = sub i32 [[TMP83]], [[TMP62]]
+; CHECK-NEXT:    [[TMP86:%.*]] = select i1 [[TMP84]], i32 [[TMP85]], i32 [[TMP83]]
+; CHECK-NEXT:    [[TMP87:%.*]] = icmp uge i32 [[TMP86]], [[TMP62]]
+; CHECK-NEXT:    [[TMP88:%.*]] = sub i32 [[TMP86]], [[TMP62]]
+; CHECK-NEXT:    [[TMP89:%.*]] = select i1 [[TMP87]], i32 [[TMP88]], i32 [[TMP86]]
+; CHECK-NEXT:    [[TMP90:%.*]] = insertelement <4 x i32> [[TMP60]], i32 [[TMP89]], i64 2
+; CHECK-NEXT:    [[TMP91:%.*]] = extractelement <4 x i32> [[X]], i64 3
+; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <4 x i32> [[Y]], i64 3
+; CHECK-NEXT:    [[TMP93:%.*]] = uitofp i32 [[TMP92]] to float
+; CHECK-NEXT:    [[TMP94:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP93]])
+; CHECK-NEXT:    [[TMP95:%.*]] = fmul fast float [[TMP94]], 0x41EFFFFFC0000000
+; CHECK-NEXT:    [[TMP96:%.*]] = fptoui float [[TMP95]] to i32
+; CHECK-NEXT:    [[TMP97:%.*]] = sub i32 0, [[TMP92]]
+; CHECK-NEXT:    [[TMP98:%.*]] = mul i32 [[TMP97]], [[TMP96]]
+; CHECK-NEXT:    [[TMP99:%.*]] = zext i32 [[TMP96]] to i64
+; CHECK-NEXT:    [[TMP100:%.*]] = zext i32 [[TMP98]] to i64
+; CHECK-NEXT:    [[TMP101:%.*]] = mul i64 [[TMP99]], [[TMP100]]
+; CHECK-NEXT:    [[TMP102:%.*]] = trunc i64 [[TMP101]] to i32
+; CHECK-NEXT:    [[TMP103:%.*]] = lshr i64 [[TMP101]], 32
+; CHECK-NEXT:    [[TMP104:%.*]] = trunc i64 [[TMP103]] to i32
+; CHECK-NEXT:    [[TMP105:%.*]] = add i32 [[TMP96]], [[TMP104]]
+; CHECK-NEXT:    [[TMP106:%.*]] = zext i32 [[TMP91]] to i64
+; CHECK-NEXT:    [[TMP107:%.*]] = zext i32 [[TMP105]] to i64
+; CHECK-NEXT:    [[TMP108:%.*]] = mul i64 [[TMP106]], [[TMP107]]
+; CHECK-NEXT:    [[TMP109:%.*]] = trunc i64 [[TMP108]] to i32
+; CHECK-NEXT:    [[TMP110:%.*]] = lshr i64 [[TMP108]], 32
+; CHECK-NEXT:    [[TMP111:%.*]] = trunc i64 [[TMP110]] to i32
+; CHECK-NEXT:    [[TMP112:%.*]] = mul i32 [[TMP111]], [[TMP92]]
+; CHECK-NEXT:    [[TMP113:%.*]] = sub i32 [[TMP91]], [[TMP112]]
+; CHECK-NEXT:    [[TMP114:%.*]] = icmp uge i32 [[TMP113]], [[TMP92]]
+; CHECK-NEXT:    [[TMP115:%.*]] = sub i32 [[TMP113]], [[TMP92]]
+; CHECK-NEXT:    [[TMP116:%.*]] = select i1 [[TMP114]], i32 [[TMP115]], i32 [[TMP113]]
+; CHECK-NEXT:    [[TMP117:%.*]] = icmp uge i32 [[TMP116]], [[TMP92]]
+; CHECK-NEXT:    [[TMP118:%.*]] = sub i32 [[TMP116]], [[TMP92]]
+; CHECK-NEXT:    [[TMP119:%.*]] = select i1 [[TMP117]], i32 [[TMP118]], i32 [[TMP116]]
+; CHECK-NEXT:    [[TMP120:%.*]] = insertelement <4 x i32> [[TMP90]], i32 [[TMP119]], i64 3
+; CHECK-NEXT:    store <4 x i32> [[TMP120]], <4 x i32> addrspace(1)* [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_v4i32:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx8 s[8:15], s[0:1], 0xd
-; GCN-NEXT:    s_mov_b32 s6, 0x4f800000
-; GCN-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x9
-; GCN-NEXT:    s_mov_b32 s19, 0xf000
-; GCN-NEXT:    s_mov_b32 s18, -1
+; GCN-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s12, 0x4f7ffffe
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s12
-; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s13
-; GCN-NEXT:    v_cvt_f32_u32_e32 v7, s15
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s8
+; GCN-NEXT:    s_sub_i32 s2, 0, s8
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s9
+; GCN-NEXT:    v_cvt_f32_u32_e32 v4, s11
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    s_sub_i32 s3, 0, s9
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; GCN-NEXT:    v_mul_f32_e32 v0, s6, v0
+; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s10
+; GCN-NEXT:    v_mul_f32_e32 v0, s12, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_mul_f32_e32 v1, s6, v1
+; GCN-NEXT:    v_mul_f32_e32 v1, s12, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    v_mul_lo_u32 v2, v0, s12
-; GCN-NEXT:    v_mul_hi_u32 v3, v0, s12
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v2
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v4, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v2, v2, v0
-; GCN-NEXT:    v_mul_lo_u32 v3, v1, s13
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v2, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v2, v0
-; GCN-NEXT:    v_mul_hi_u32 v2, v1, s13
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[0:1]
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v3
-; GCN-NEXT:    v_mul_hi_u32 v0, v0, s8
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v2, v2, v1
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s12
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v2, v1
-; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v2, v1
-; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s14
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, s9
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s8, v0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[4:5], s8, v0
-; GCN-NEXT:    v_mul_lo_u32 v1, v1, s13
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s12, v3
-; GCN-NEXT:    v_mul_f32_e32 v2, s6, v2
+; GCN-NEXT:    v_mul_lo_u32 v3, s2, v0
+; GCN-NEXT:    s_sub_i32 s2, 0, s10
+; GCN-NEXT:    v_mul_f32_e32 v2, s12, v2
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v3
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, s12, v3
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s12, v3
-; GCN-NEXT:    s_and_b64 vcc, s[2:3], s[4:5]
-; GCN-NEXT:    v_mul_lo_u32 v5, v2, s14
-; GCN-NEXT:    v_mul_hi_u32 v6, v2, s14
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[4:5]
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s9, v1
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s9, v1
-; GCN-NEXT:    v_sub_i32_e32 v1, vcc, 0, v5
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, v2
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, s13, v3
-; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s13, v3
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v1, v2
-; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v1, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v6, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, s10
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v7
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GCN-NEXT:    v_mul_lo_u32 v5, v1, s14
-; GCN-NEXT:    v_mul_f32_e32 v1, s6, v2
-; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v1
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v4, v3, s[2:3]
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s10, v5
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s14, v3
-; GCN-NEXT:    v_mul_lo_u32 v4, v2, s15
-; GCN-NEXT:    v_mul_hi_u32 v6, v2, s15
-; GCN-NEXT:    v_sub_i32_e32 v7, vcc, 0, v4
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v4, v4, v2
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, s14, v3
-; GCN-NEXT:    v_add_i32_e32 v7, vcc, v4, v2
-; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, v4, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v7, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v2, v2, s11
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s10, v5
-; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, s14, v3
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_mul_lo_u32 v5, v2, s15
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[2:3]
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s11, v5
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s11, v5
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s15, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, s15, v3
-; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s15, v3
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v3, v4, v3, s[2:3]
-; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[16:19], 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v4
+; GCN-NEXT:    v_mul_lo_u32 v4, s3, v1
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s8
+; GCN-NEXT:    v_mul_f32_e32 v3, s12, v3
+; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s4, v0
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s8, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s8, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v4, v1
+; GCN-NEXT:    v_mul_hi_u32 v1, s5, v1
+; GCN-NEXT:    v_mul_lo_u32 v4, s2, v2
+; GCN-NEXT:    s_sub_i32 s2, 0, s11
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s9
+; GCN-NEXT:    v_mul_hi_u32 v4, v2, v4
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s5, v1
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s9, v1
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s9, v1
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; GCN-NEXT:    v_mul_hi_u32 v2, s6, v2
+; GCN-NEXT:    v_mul_lo_u32 v4, s2, v3
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, s10
+; GCN-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s10, v2
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s10, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s10, v2
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s10, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_mul_hi_u32 v3, s7, v3
+; GCN-NEXT:    v_mul_lo_u32 v3, v3, s11
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s7, v3
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, s11, v3
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s11, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, s11, v3
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s11, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
   %r = urem <4 x i32> %x, %y
   store <4 x i32> %r, <4 x i32> addrspace(1)* %out
@@ -1337,331 +1170,284 @@ define amdgpu_kernel void @sdiv_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %x
 ; CHECK-NEXT:    [[TMP9:%.*]] = xor i32 [[TMP7]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = uitofp i32 [[TMP9]] to float
 ; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP10]])
-; CHECK-NEXT:    [[TMP12:%.*]] = fmul fast float [[TMP11]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP12:%.*]] = fmul fast float [[TMP11]], 0x41EFFFFFC0000000
 ; CHECK-NEXT:    [[TMP13:%.*]] = fptoui float [[TMP12]] to i32
-; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
-; CHECK-NEXT:    [[TMP15:%.*]] = zext i32 [[TMP9]] to i64
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP14]], [[TMP15]]
-; CHECK-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32
-; CHECK-NEXT:    [[TMP18:%.*]] = lshr i64 [[TMP16]], 32
+; CHECK-NEXT:    [[TMP14:%.*]] = sub i32 0, [[TMP9]]
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP15]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP16]], [[TMP17]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
-; CHECK-NEXT:    [[TMP20:%.*]] = sub i32 0, [[TMP17]]
-; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i32 [[TMP19]], 0
-; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 [[TMP17]]
-; CHECK-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP22]] to i64
-; CHECK-NEXT:    [[TMP24:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP20:%.*]] = lshr i64 [[TMP18]], 32
+; CHECK-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
+; CHECK-NEXT:    [[TMP22:%.*]] = add i32 [[TMP13]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP24:%.*]] = zext i32 [[TMP22]] to i64
 ; CHECK-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP23]], [[TMP24]]
 ; CHECK-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP25]] to i32
 ; CHECK-NEXT:    [[TMP27:%.*]] = lshr i64 [[TMP25]], 32
 ; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i32
-; CHECK-NEXT:    [[TMP29:%.*]] = add i32 [[TMP13]], [[TMP28]]
-; CHECK-NEXT:    [[TMP30:%.*]] = sub i32 [[TMP13]], [[TMP28]]
-; CHECK-NEXT:    [[TMP31:%.*]] = select i1 [[TMP21]], i32 [[TMP29]], i32 [[TMP30]]
-; CHECK-NEXT:    [[TMP32:%.*]] = zext i32 [[TMP31]] to i64
-; CHECK-NEXT:    [[TMP33:%.*]] = zext i32 [[TMP8]] to i64
-; CHECK-NEXT:    [[TMP34:%.*]] = mul i64 [[TMP32]], [[TMP33]]
-; CHECK-NEXT:    [[TMP35:%.*]] = trunc i64 [[TMP34]] to i32
-; CHECK-NEXT:    [[TMP36:%.*]] = lshr i64 [[TMP34]], 32
-; CHECK-NEXT:    [[TMP37:%.*]] = trunc i64 [[TMP36]] to i32
-; CHECK-NEXT:    [[TMP38:%.*]] = mul i32 [[TMP37]], [[TMP9]]
-; CHECK-NEXT:    [[TMP39:%.*]] = sub i32 [[TMP8]], [[TMP38]]
-; CHECK-NEXT:    [[TMP40:%.*]] = icmp uge i32 [[TMP39]], [[TMP9]]
-; CHECK-NEXT:    [[TMP41:%.*]] = icmp uge i32 [[TMP8]], [[TMP38]]
-; CHECK-NEXT:    [[TMP42:%.*]] = and i1 [[TMP40]], [[TMP41]]
-; CHECK-NEXT:    [[TMP43:%.*]] = add i32 [[TMP37]], 1
-; CHECK-NEXT:    [[TMP44:%.*]] = sub i32 [[TMP37]], 1
-; CHECK-NEXT:    [[TMP45:%.*]] = select i1 [[TMP42]], i32 [[TMP43]], i32 [[TMP37]]
-; CHECK-NEXT:    [[TMP46:%.*]] = select i1 [[TMP41]], i32 [[TMP45]], i32 [[TMP44]]
-; CHECK-NEXT:    [[TMP47:%.*]] = xor i32 [[TMP46]], [[TMP5]]
-; CHECK-NEXT:    [[TMP48:%.*]] = sub i32 [[TMP47]], [[TMP5]]
-; CHECK-NEXT:    [[TMP49:%.*]] = insertelement <4 x i32> undef, i32 [[TMP48]], i64 0
-; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <4 x i32> [[X]], i64 1
-; CHECK-NEXT:    [[TMP51:%.*]] = extractelement <4 x i32> [[Y]], i64 1
-; CHECK-NEXT:    [[TMP52:%.*]] = ashr i32 [[TMP50]], 31
-; CHECK-NEXT:    [[TMP53:%.*]] = ashr i32 [[TMP51]], 31
-; CHECK-NEXT:    [[TMP54:%.*]] = xor i32 [[TMP52]], [[TMP53]]
-; CHECK-NEXT:    [[TMP55:%.*]] = add i32 [[TMP50]], [[TMP52]]
-; CHECK-NEXT:    [[TMP56:%.*]] = add i32 [[TMP51]], [[TMP53]]
-; CHECK-NEXT:    [[TMP57:%.*]] = xor i32 [[TMP55]], [[TMP52]]
-; CHECK-NEXT:    [[TMP58:%.*]] = xor i32 [[TMP56]], [[TMP53]]
-; CHECK-NEXT:    [[TMP59:%.*]] = uitofp i32 [[TMP58]] to float
-; CHECK-NEXT:    [[TMP60:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP59]])
-; CHECK-NEXT:    [[TMP61:%.*]] = fmul fast float [[TMP60]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP62:%.*]] = fptoui float [[TMP61]] to i32
-; CHECK-NEXT:    [[TMP63:%.*]] = zext i32 [[TMP62]] to i64
-; CHECK-NEXT:    [[TMP64:%.*]] = zext i32 [[TMP58]] to i64
-; CHECK-NEXT:    [[TMP65:%.*]] = mul i64 [[TMP63]], [[TMP64]]
-; CHECK-NEXT:    [[TMP66:%.*]] = trunc i64 [[TMP65]] to i32
-; CHECK-NEXT:    [[TMP67:%.*]] = lshr i64 [[TMP65]], 32
-; CHECK-NEXT:    [[TMP68:%.*]] = trunc i64 [[TMP67]] to i32
-; CHECK-NEXT:    [[TMP69:%.*]] = sub i32 0, [[TMP66]]
-; CHECK-NEXT:    [[TMP70:%.*]] = icmp eq i32 [[TMP68]], 0
-; CHECK-NEXT:    [[TMP71:%.*]] = select i1 [[TMP70]], i32 [[TMP69]], i32 [[TMP66]]
-; CHECK-NEXT:    [[TMP72:%.*]] = zext i32 [[TMP71]] to i64
-; CHECK-NEXT:    [[TMP73:%.*]] = zext i32 [[TMP62]] to i64
-; CHECK-NEXT:    [[TMP74:%.*]] = mul i64 [[TMP72]], [[TMP73]]
-; CHECK-NEXT:    [[TMP75:%.*]] = trunc i64 [[TMP74]] to i32
-; CHECK-NEXT:    [[TMP76:%.*]] = lshr i64 [[TMP74]], 32
-; CHECK-NEXT:    [[TMP77:%.*]] = trunc i64 [[TMP76]] to i32
-; CHECK-NEXT:    [[TMP78:%.*]] = add i32 [[TMP62]], [[TMP77]]
-; CHECK-NEXT:    [[TMP79:%.*]] = sub i32 [[TMP62]], [[TMP77]]
-; CHECK-NEXT:    [[TMP80:%.*]] = select i1 [[TMP70]], i32 [[TMP78]], i32 [[TMP79]]
-; CHECK-NEXT:    [[TMP81:%.*]] = zext i32 [[TMP80]] to i64
-; CHECK-NEXT:    [[TMP82:%.*]] = zext i32 [[TMP57]] to i64
-; CHECK-NEXT:    [[TMP83:%.*]] = mul i64 [[TMP81]], [[TMP82]]
-; CHECK-NEXT:    [[TMP84:%.*]] = trunc i64 [[TMP83]] to i32
-; CHECK-NEXT:    [[TMP85:%.*]] = lshr i64 [[TMP83]], 32
-; CHECK-NEXT:    [[TMP86:%.*]] = trunc i64 [[TMP85]] to i32
-; CHECK-NEXT:    [[TMP87:%.*]] = mul i32 [[TMP86]], [[TMP58]]
-; CHECK-NEXT:    [[TMP88:%.*]] = sub i32 [[TMP57]], [[TMP87]]
-; CHECK-NEXT:    [[TMP89:%.*]] = icmp uge i32 [[TMP88]], [[TMP58]]
-; CHECK-NEXT:    [[TMP90:%.*]] = icmp uge i32 [[TMP57]], [[TMP87]]
-; CHECK-NEXT:    [[TMP91:%.*]] = and i1 [[TMP89]], [[TMP90]]
-; CHECK-NEXT:    [[TMP92:%.*]] = add i32 [[TMP86]], 1
-; CHECK-NEXT:    [[TMP93:%.*]] = sub i32 [[TMP86]], 1
-; CHECK-NEXT:    [[TMP94:%.*]] = select i1 [[TMP91]], i32 [[TMP92]], i32 [[TMP86]]
-; CHECK-NEXT:    [[TMP95:%.*]] = select i1 [[TMP90]], i32 [[TMP94]], i32 [[TMP93]]
-; CHECK-NEXT:    [[TMP96:%.*]] = xor i32 [[TMP95]], [[TMP54]]
-; CHECK-NEXT:    [[TMP97:%.*]] = sub i32 [[TMP96]], [[TMP54]]
-; CHECK-NEXT:    [[TMP98:%.*]] = insertelement <4 x i32> [[TMP49]], i32 [[TMP97]], i64 1
-; CHECK-NEXT:    [[TMP99:%.*]] = extractelement <4 x i32> [[X]], i64 2
-; CHECK-NEXT:    [[TMP100:%.*]] = extractelement <4 x i32> [[Y]], i64 2
-; CHECK-NEXT:    [[TMP101:%.*]] = ashr i32 [[TMP99]], 31
-; CHECK-NEXT:    [[TMP102:%.*]] = ashr i32 [[TMP100]], 31
-; CHECK-NEXT:    [[TMP103:%.*]] = xor i32 [[TMP101]], [[TMP102]]
-; CHECK-NEXT:    [[TMP104:%.*]] = add i32 [[TMP99]], [[TMP101]]
-; CHECK-NEXT:    [[TMP105:%.*]] = add i32 [[TMP100]], [[TMP102]]
-; CHECK-NEXT:    [[TMP106:%.*]] = xor i32 [[TMP104]], [[TMP101]]
-; CHECK-NEXT:    [[TMP107:%.*]] = xor i32 [[TMP105]], [[TMP102]]
-; CHECK-NEXT:    [[TMP108:%.*]] = uitofp i32 [[TMP107]] to float
-; CHECK-NEXT:    [[TMP109:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP108]])
-; CHECK-NEXT:    [[TMP110:%.*]] = fmul fast float [[TMP109]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP111:%.*]] = fptoui float [[TMP110]] to i32
-; CHECK-NEXT:    [[TMP112:%.*]] = zext i32 [[TMP111]] to i64
-; CHECK-NEXT:    [[TMP113:%.*]] = zext i32 [[TMP107]] to i64
-; CHECK-NEXT:    [[TMP114:%.*]] = mul i64 [[TMP112]], [[TMP113]]
-; CHECK-NEXT:    [[TMP115:%.*]] = trunc i64 [[TMP114]] to i32
-; CHECK-NEXT:    [[TMP116:%.*]] = lshr i64 [[TMP114]], 32
-; CHECK-NEXT:    [[TMP117:%.*]] = trunc i64 [[TMP116]] to i32
-; CHECK-NEXT:    [[TMP118:%.*]] = sub i32 0, [[TMP115]]
-; CHECK-NEXT:    [[TMP119:%.*]] = icmp eq i32 [[TMP117]], 0
-; CHECK-NEXT:    [[TMP120:%.*]] = select i1 [[TMP119]], i32 [[TMP118]], i32 [[TMP115]]
-; CHECK-NEXT:    [[TMP121:%.*]] = zext i32 [[TMP120]] to i64
-; CHECK-NEXT:    [[TMP122:%.*]] = zext i32 [[TMP111]] to i64
-; CHECK-NEXT:    [[TMP123:%.*]] = mul i64 [[TMP121]], [[TMP122]]
-; CHECK-NEXT:    [[TMP124:%.*]] = trunc i64 [[TMP123]] to i32
-; CHECK-NEXT:    [[TMP125:%.*]] = lshr i64 [[TMP123]], 32
-; CHECK-NEXT:    [[TMP126:%.*]] = trunc i64 [[TMP125]] to i32
-; CHECK-NEXT:    [[TMP127:%.*]] = add i32 [[TMP111]], [[TMP126]]
-; CHECK-NEXT:    [[TMP128:%.*]] = sub i32 [[TMP111]], [[TMP126]]
-; CHECK-NEXT:    [[TMP129:%.*]] = select i1 [[TMP119]], i32 [[TMP127]], i32 [[TMP128]]
-; CHECK-NEXT:    [[TMP130:%.*]] = zext i32 [[TMP129]] to i64
-; CHECK-NEXT:    [[TMP131:%.*]] = zext i32 [[TMP106]] to i64
-; CHECK-NEXT:    [[TMP132:%.*]] = mul i64 [[TMP130]], [[TMP131]]
-; CHECK-NEXT:    [[TMP133:%.*]] = trunc i64 [[TMP132]] to i32
-; CHECK-NEXT:    [[TMP134:%.*]] = lshr i64 [[TMP132]], 32
-; CHECK-NEXT:    [[TMP135:%.*]] = trunc i64 [[TMP134]] to i32
-; CHECK-NEXT:    [[TMP136:%.*]] = mul i32 [[TMP135]], [[TMP107]]
-; CHECK-NEXT:    [[TMP137:%.*]] = sub i32 [[TMP106]], [[TMP136]]
-; CHECK-NEXT:    [[TMP138:%.*]] = icmp uge i32 [[TMP137]], [[TMP107]]
-; CHECK-NEXT:    [[TMP139:%.*]] = icmp uge i32 [[TMP106]], [[TMP136]]
-; CHECK-NEXT:    [[TMP140:%.*]] = and i1 [[TMP138]], [[TMP139]]
-; CHECK-NEXT:    [[TMP141:%.*]] = add i32 [[TMP135]], 1
-; CHECK-NEXT:    [[TMP142:%.*]] = sub i32 [[TMP135]], 1
-; CHECK-NEXT:    [[TMP143:%.*]] = select i1 [[TMP140]], i32 [[TMP141]], i32 [[TMP135]]
-; CHECK-NEXT:    [[TMP144:%.*]] = select i1 [[TMP139]], i32 [[TMP143]], i32 [[TMP142]]
-; CHECK-NEXT:    [[TMP145:%.*]] = xor i32 [[TMP144]], [[TMP103]]
-; CHECK-NEXT:    [[TMP146:%.*]] = sub i32 [[TMP145]], [[TMP103]]
-; CHECK-NEXT:    [[TMP147:%.*]] = insertelement <4 x i32> [[TMP98]], i32 [[TMP146]], i64 2
-; CHECK-NEXT:    [[TMP148:%.*]] = extractelement <4 x i32> [[X]], i64 3
-; CHECK-NEXT:    [[TMP149:%.*]] = extractelement <4 x i32> [[Y]], i64 3
-; CHECK-NEXT:    [[TMP150:%.*]] = ashr i32 [[TMP148]], 31
-; CHECK-NEXT:    [[TMP151:%.*]] = ashr i32 [[TMP149]], 31
-; CHECK-NEXT:    [[TMP152:%.*]] = xor i32 [[TMP150]], [[TMP151]]
-; CHECK-NEXT:    [[TMP153:%.*]] = add i32 [[TMP148]], [[TMP150]]
-; CHECK-NEXT:    [[TMP154:%.*]] = add i32 [[TMP149]], [[TMP151]]
-; CHECK-NEXT:    [[TMP155:%.*]] = xor i32 [[TMP153]], [[TMP150]]
-; CHECK-NEXT:    [[TMP156:%.*]] = xor i32 [[TMP154]], [[TMP151]]
-; CHECK-NEXT:    [[TMP157:%.*]] = uitofp i32 [[TMP156]] to float
-; CHECK-NEXT:    [[TMP158:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP157]])
-; CHECK-NEXT:    [[TMP159:%.*]] = fmul fast float [[TMP158]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP160:%.*]] = fptoui float [[TMP159]] to i32
-; CHECK-NEXT:    [[TMP161:%.*]] = zext i32 [[TMP160]] to i64
-; CHECK-NEXT:    [[TMP162:%.*]] = zext i32 [[TMP156]] to i64
-; CHECK-NEXT:    [[TMP163:%.*]] = mul i64 [[TMP161]], [[TMP162]]
-; CHECK-NEXT:    [[TMP164:%.*]] = trunc i64 [[TMP163]] to i32
-; CHECK-NEXT:    [[TMP165:%.*]] = lshr i64 [[TMP163]], 32
-; CHECK-NEXT:    [[TMP166:%.*]] = trunc i64 [[TMP165]] to i32
-; CHECK-NEXT:    [[TMP167:%.*]] = sub i32 0, [[TMP164]]
-; CHECK-NEXT:    [[TMP168:%.*]] = icmp eq i32 [[TMP166]], 0
-; CHECK-NEXT:    [[TMP169:%.*]] = select i1 [[TMP168]], i32 [[TMP167]], i32 [[TMP164]]
-; CHECK-NEXT:    [[TMP170:%.*]] = zext i32 [[TMP169]] to i64
-; CHECK-NEXT:    [[TMP171:%.*]] = zext i32 [[TMP160]] to i64
-; CHECK-NEXT:    [[TMP172:%.*]] = mul i64 [[TMP170]], [[TMP171]]
-; CHECK-NEXT:    [[TMP173:%.*]] = trunc i64 [[TMP172]] to i32
-; CHECK-NEXT:    [[TMP174:%.*]] = lshr i64 [[TMP172]], 32
-; CHECK-NEXT:    [[TMP175:%.*]] = trunc i64 [[TMP174]] to i32
-; CHECK-NEXT:    [[TMP176:%.*]] = add i32 [[TMP160]], [[TMP175]]
-; CHECK-NEXT:    [[TMP177:%.*]] = sub i32 [[TMP160]], [[TMP175]]
-; CHECK-NEXT:    [[TMP178:%.*]] = select i1 [[TMP168]], i32 [[TMP176]], i32 [[TMP177]]
-; CHECK-NEXT:    [[TMP179:%.*]] = zext i32 [[TMP178]] to i64
-; CHECK-NEXT:    [[TMP180:%.*]] = zext i32 [[TMP155]] to i64
-; CHECK-NEXT:    [[TMP181:%.*]] = mul i64 [[TMP179]], [[TMP180]]
-; CHECK-NEXT:    [[TMP182:%.*]] = trunc i64 [[TMP181]] to i32
-; CHECK-NEXT:    [[TMP183:%.*]] = lshr i64 [[TMP181]], 32
-; CHECK-NEXT:    [[TMP184:%.*]] = trunc i64 [[TMP183]] to i32
-; CHECK-NEXT:    [[TMP185:%.*]] = mul i32 [[TMP184]], [[TMP156]]
-; CHECK-NEXT:    [[TMP186:%.*]] = sub i32 [[TMP155]], [[TMP185]]
-; CHECK-NEXT:    [[TMP187:%.*]] = icmp uge i32 [[TMP186]], [[TMP156]]
-; CHECK-NEXT:    [[TMP188:%.*]] = icmp uge i32 [[TMP155]], [[TMP185]]
-; CHECK-NEXT:    [[TMP189:%.*]] = and i1 [[TMP187]], [[TMP188]]
-; CHECK-NEXT:    [[TMP190:%.*]] = add i32 [[TMP184]], 1
-; CHECK-NEXT:    [[TMP191:%.*]] = sub i32 [[TMP184]], 1
-; CHECK-NEXT:    [[TMP192:%.*]] = select i1 [[TMP189]], i32 [[TMP190]], i32 [[TMP184]]
-; CHECK-NEXT:    [[TMP193:%.*]] = select i1 [[TMP188]], i32 [[TMP192]], i32 [[TMP191]]
-; CHECK-NEXT:    [[TMP194:%.*]] = xor i32 [[TMP193]], [[TMP152]]
-; CHECK-NEXT:    [[TMP195:%.*]] = sub i32 [[TMP194]], [[TMP152]]
-; CHECK-NEXT:    [[TMP196:%.*]] = insertelement <4 x i32> [[TMP147]], i32 [[TMP195]], i64 3
-; CHECK-NEXT:    store <4 x i32> [[TMP196]], <4 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP29:%.*]] = mul i32 [[TMP28]], [[TMP9]]
+; CHECK-NEXT:    [[TMP30:%.*]] = sub i32 [[TMP8]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp uge i32 [[TMP30]], [[TMP9]]
+; CHECK-NEXT:    [[TMP32:%.*]] = add i32 [[TMP28]], 1
+; CHECK-NEXT:    [[TMP33:%.*]] = select i1 [[TMP31]], i32 [[TMP32]], i32 [[TMP28]]
+; CHECK-NEXT:    [[TMP34:%.*]] = sub i32 [[TMP30]], [[TMP9]]
+; CHECK-NEXT:    [[TMP35:%.*]] = select i1 [[TMP31]], i32 [[TMP34]], i32 [[TMP30]]
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp uge i32 [[TMP35]], [[TMP9]]
+; CHECK-NEXT:    [[TMP37:%.*]] = add i32 [[TMP33]], 1
+; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP36]], i32 [[TMP37]], i32 [[TMP33]]
+; CHECK-NEXT:    [[TMP39:%.*]] = xor i32 [[TMP38]], [[TMP5]]
+; CHECK-NEXT:    [[TMP40:%.*]] = sub i32 [[TMP39]], [[TMP5]]
+; CHECK-NEXT:    [[TMP41:%.*]] = insertelement <4 x i32> undef, i32 [[TMP40]], i64 0
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <4 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <4 x i32> [[Y]], i64 1
+; CHECK-NEXT:    [[TMP44:%.*]] = ashr i32 [[TMP42]], 31
+; CHECK-NEXT:    [[TMP45:%.*]] = ashr i32 [[TMP43]], 31
+; CHECK-NEXT:    [[TMP46:%.*]] = xor i32 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = add i32 [[TMP42]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = add i32 [[TMP43]], [[TMP45]]
+; CHECK-NEXT:    [[TMP49:%.*]] = xor i32 [[TMP47]], [[TMP44]]
+; CHECK-NEXT:    [[TMP50:%.*]] = xor i32 [[TMP48]], [[TMP45]]
+; CHECK-NEXT:    [[TMP51:%.*]] = uitofp i32 [[TMP50]] to float
+; CHECK-NEXT:    [[TMP52:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP51]])
+; CHECK-NEXT:    [[TMP53:%.*]] = fmul fast float [[TMP52]], 0x41EFFFFFC0000000
+; CHECK-NEXT:    [[TMP54:%.*]] = fptoui float [[TMP53]] to i32
+; CHECK-NEXT:    [[TMP55:%.*]] = sub i32 0, [[TMP50]]
+; CHECK-NEXT:    [[TMP56:%.*]] = mul i32 [[TMP55]], [[TMP54]]
+; CHECK-NEXT:    [[TMP57:%.*]] = zext i32 [[TMP54]] to i64
+; CHECK-NEXT:    [[TMP58:%.*]] = zext i32 [[TMP56]] to i64
+; CHECK-NEXT:    [[TMP59:%.*]] = mul i64 [[TMP57]], [[TMP58]]
+; CHECK-NEXT:    [[TMP60:%.*]] = trunc i64 [[TMP59]] to i32
+; CHECK-NEXT:    [[TMP61:%.*]] = lshr i64 [[TMP59]], 32
+; CHECK-NEXT:    [[TMP62:%.*]] = trunc i64 [[TMP61]] to i32
+; CHECK-NEXT:    [[TMP63:%.*]] = add i32 [[TMP54]], [[TMP62]]
+; CHECK-NEXT:    [[TMP64:%.*]] = zext i32 [[TMP49]] to i64
+; CHECK-NEXT:    [[TMP65:%.*]] = zext i32 [[TMP63]] to i64
+; CHECK-NEXT:    [[TMP66:%.*]] = mul i64 [[TMP64]], [[TMP65]]
+; CHECK-NEXT:    [[TMP67:%.*]] = trunc i64 [[TMP66]] to i32
+; CHECK-NEXT:    [[TMP68:%.*]] = lshr i64 [[TMP66]], 32
+; CHECK-NEXT:    [[TMP69:%.*]] = trunc i64 [[TMP68]] to i32
+; CHECK-NEXT:    [[TMP70:%.*]] = mul i32 [[TMP69]], [[TMP50]]
+; CHECK-NEXT:    [[TMP71:%.*]] = sub i32 [[TMP49]], [[TMP70]]
+; CHECK-NEXT:    [[TMP72:%.*]] = icmp uge i32 [[TMP71]], [[TMP50]]
+; CHECK-NEXT:    [[TMP73:%.*]] = add i32 [[TMP69]], 1
+; CHECK-NEXT:    [[TMP74:%.*]] = select i1 [[TMP72]], i32 [[TMP73]], i32 [[TMP69]]
+; CHECK-NEXT:    [[TMP75:%.*]] = sub i32 [[TMP71]], [[TMP50]]
+; CHECK-NEXT:    [[TMP76:%.*]] = select i1 [[TMP72]], i32 [[TMP75]], i32 [[TMP71]]
+; CHECK-NEXT:    [[TMP77:%.*]] = icmp uge i32 [[TMP76]], [[TMP50]]
+; CHECK-NEXT:    [[TMP78:%.*]] = add i32 [[TMP74]], 1
+; CHECK-NEXT:    [[TMP79:%.*]] = select i1 [[TMP77]], i32 [[TMP78]], i32 [[TMP74]]
+; CHECK-NEXT:    [[TMP80:%.*]] = xor i32 [[TMP79]], [[TMP46]]
+; CHECK-NEXT:    [[TMP81:%.*]] = sub i32 [[TMP80]], [[TMP46]]
+; CHECK-NEXT:    [[TMP82:%.*]] = insertelement <4 x i32> [[TMP41]], i32 [[TMP81]], i64 1
+; CHECK-NEXT:    [[TMP83:%.*]] = extractelement <4 x i32> [[X]], i64 2
+; CHECK-NEXT:    [[TMP84:%.*]] = extractelement <4 x i32> [[Y]], i64 2
+; CHECK-NEXT:    [[TMP85:%.*]] = ashr i32 [[TMP83]], 31
+; CHECK-NEXT:    [[TMP86:%.*]] = ashr i32 [[TMP84]], 31
+; CHECK-NEXT:    [[TMP87:%.*]] = xor i32 [[TMP85]], [[TMP86]]
+; CHECK-NEXT:    [[TMP88:%.*]] = add i32 [[TMP83]], [[TMP85]]
+; CHECK-NEXT:    [[TMP89:%.*]] = add i32 [[TMP84]], [[TMP86]]
+; CHECK-NEXT:    [[TMP90:%.*]] = xor i32 [[TMP88]], [[TMP85]]
+; CHECK-NEXT:    [[TMP91:%.*]] = xor i32 [[TMP89]], [[TMP86]]
+; CHECK-NEXT:    [[TMP92:%.*]] = uitofp i32 [[TMP91]] to float
+; CHECK-NEXT:    [[TMP93:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP92]])
+; CHECK-NEXT:    [[TMP94:%.*]] = fmul fast float [[TMP93]], 0x41EFFFFFC0000000
+; CHECK-NEXT:    [[TMP95:%.*]] = fptoui float [[TMP94]] to i32
+; CHECK-NEXT:    [[TMP96:%.*]] = sub i32 0, [[TMP91]]
+; CHECK-NEXT:    [[TMP97:%.*]] = mul i32 [[TMP96]], [[TMP95]]
+; CHECK-NEXT:    [[TMP98:%.*]] = zext i32 [[TMP95]] to i64
+; CHECK-NEXT:    [[TMP99:%.*]] = zext i32 [[TMP97]] to i64
+; CHECK-NEXT:    [[TMP100:%.*]] = mul i64 [[TMP98]], [[TMP99]]
+; CHECK-NEXT:    [[TMP101:%.*]] = trunc i64 [[TMP100]] to i32
+; CHECK-NEXT:    [[TMP102:%.*]] = lshr i64 [[TMP100]], 32
+; CHECK-NEXT:    [[TMP103:%.*]] = trunc i64 [[TMP102]] to i32
+; CHECK-NEXT:    [[TMP104:%.*]] = add i32 [[TMP95]], [[TMP103]]
+; CHECK-NEXT:    [[TMP105:%.*]] = zext i32 [[TMP90]] to i64
+; CHECK-NEXT:    [[TMP106:%.*]] = zext i32 [[TMP104]] to i64
+; CHECK-NEXT:    [[TMP107:%.*]] = mul i64 [[TMP105]], [[TMP106]]
+; CHECK-NEXT:    [[TMP108:%.*]] = trunc i64 [[TMP107]] to i32
+; CHECK-NEXT:    [[TMP109:%.*]] = lshr i64 [[TMP107]], 32
+; CHECK-NEXT:    [[TMP110:%.*]] = trunc i64 [[TMP109]] to i32
+; CHECK-NEXT:    [[TMP111:%.*]] = mul i32 [[TMP110]], [[TMP91]]
+; CHECK-NEXT:    [[TMP112:%.*]] = sub i32 [[TMP90]], [[TMP111]]
+; CHECK-NEXT:    [[TMP113:%.*]] = icmp uge i32 [[TMP112]], [[TMP91]]
+; CHECK-NEXT:    [[TMP114:%.*]] = add i32 [[TMP110]], 1
+; CHECK-NEXT:    [[TMP115:%.*]] = select i1 [[TMP113]], i32 [[TMP114]], i32 [[TMP110]]
+; CHECK-NEXT:    [[TMP116:%.*]] = sub i32 [[TMP112]], [[TMP91]]
+; CHECK-NEXT:    [[TMP117:%.*]] = select i1 [[TMP113]], i32 [[TMP116]], i32 [[TMP112]]
+; CHECK-NEXT:    [[TMP118:%.*]] = icmp uge i32 [[TMP117]], [[TMP91]]
+; CHECK-NEXT:    [[TMP119:%.*]] = add i32 [[TMP115]], 1
+; CHECK-NEXT:    [[TMP120:%.*]] = select i1 [[TMP118]], i32 [[TMP119]], i32 [[TMP115]]
+; CHECK-NEXT:    [[TMP121:%.*]] = xor i32 [[TMP120]], [[TMP87]]
+; CHECK-NEXT:    [[TMP122:%.*]] = sub i32 [[TMP121]], [[TMP87]]
+; CHECK-NEXT:    [[TMP123:%.*]] = insertelement <4 x i32> [[TMP82]], i32 [[TMP122]], i64 2
+; CHECK-NEXT:    [[TMP124:%.*]] = extractelement <4 x i32> [[X]], i64 3
+; CHECK-NEXT:    [[TMP125:%.*]] = extractelement <4 x i32> [[Y]], i64 3
+; CHECK-NEXT:    [[TMP126:%.*]] = ashr i32 [[TMP124]], 31
+; CHECK-NEXT:    [[TMP127:%.*]] = ashr i32 [[TMP125]], 31
+; CHECK-NEXT:    [[TMP128:%.*]] = xor i32 [[TMP126]], [[TMP127]]
+; CHECK-NEXT:    [[TMP129:%.*]] = add i32 [[TMP124]], [[TMP126]]
+; CHECK-NEXT:    [[TMP130:%.*]] = add i32 [[TMP125]], [[TMP127]]
+; CHECK-NEXT:    [[TMP131:%.*]] = xor i32 [[TMP129]], [[TMP126]]
+; CHECK-NEXT:    [[TMP132:%.*]] = xor i32 [[TMP130]], [[TMP127]]
+; CHECK-NEXT:    [[TMP133:%.*]] = uitofp i32 [[TMP132]] to float
+; CHECK-NEXT:    [[TMP134:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP133]])
+; CHECK-NEXT:    [[TMP135:%.*]] = fmul fast float [[TMP134]], 0x41EFFFFFC0000000
+; CHECK-NEXT:    [[TMP136:%.*]] = fptoui float [[TMP135]] to i32
+; CHECK-NEXT:    [[TMP137:%.*]] = sub i32 0, [[TMP132]]
+; CHECK-NEXT:    [[TMP138:%.*]] = mul i32 [[TMP137]], [[TMP136]]
+; CHECK-NEXT:    [[TMP139:%.*]] = zext i32 [[TMP136]] to i64
+; CHECK-NEXT:    [[TMP140:%.*]] = zext i32 [[TMP138]] to i64
+; CHECK-NEXT:    [[TMP141:%.*]] = mul i64 [[TMP139]], [[TMP140]]
+; CHECK-NEXT:    [[TMP142:%.*]] = trunc i64 [[TMP141]] to i32
+; CHECK-NEXT:    [[TMP143:%.*]] = lshr i64 [[TMP141]], 32
+; CHECK-NEXT:    [[TMP144:%.*]] = trunc i64 [[TMP143]] to i32
+; CHECK-NEXT:    [[TMP145:%.*]] = add i32 [[TMP136]], [[TMP144]]
+; CHECK-NEXT:    [[TMP146:%.*]] = zext i32 [[TMP131]] to i64
+; CHECK-NEXT:    [[TMP147:%.*]] = zext i32 [[TMP145]] to i64
+; CHECK-NEXT:    [[TMP148:%.*]] = mul i64 [[TMP146]], [[TMP147]]
+; CHECK-NEXT:    [[TMP149:%.*]] = trunc i64 [[TMP148]] to i32
+; CHECK-NEXT:    [[TMP150:%.*]] = lshr i64 [[TMP148]], 32
+; CHECK-NEXT:    [[TMP151:%.*]] = trunc i64 [[TMP150]] to i32
+; CHECK-NEXT:    [[TMP152:%.*]] = mul i32 [[TMP151]], [[TMP132]]
+; CHECK-NEXT:    [[TMP153:%.*]] = sub i32 [[TMP131]], [[TMP152]]
+; CHECK-NEXT:    [[TMP154:%.*]] = icmp uge i32 [[TMP153]], [[TMP132]]
+; CHECK-NEXT:    [[TMP155:%.*]] = add i32 [[TMP151]], 1
+; CHECK-NEXT:    [[TMP156:%.*]] = select i1 [[TMP154]], i32 [[TMP155]], i32 [[TMP151]]
+; CHECK-NEXT:    [[TMP157:%.*]] = sub i32 [[TMP153]], [[TMP132]]
+; CHECK-NEXT:    [[TMP158:%.*]] = select i1 [[TMP154]], i32 [[TMP157]], i32 [[TMP153]]
+; CHECK-NEXT:    [[TMP159:%.*]] = icmp uge i32 [[TMP158]], [[TMP132]]
+; CHECK-NEXT:    [[TMP160:%.*]] = add i32 [[TMP156]], 1
+; CHECK-NEXT:    [[TMP161:%.*]] = select i1 [[TMP159]], i32 [[TMP160]], i32 [[TMP156]]
+; CHECK-NEXT:    [[TMP162:%.*]] = xor i32 [[TMP161]], [[TMP128]]
+; CHECK-NEXT:    [[TMP163:%.*]] = sub i32 [[TMP162]], [[TMP128]]
+; CHECK-NEXT:    [[TMP164:%.*]] = insertelement <4 x i32> [[TMP123]], i32 [[TMP163]], i64 3
+; CHECK-NEXT:    store <4 x i32> [[TMP164]], <4 x i32> addrspace(1)* [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_v4i32:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx8 s[12:19], s[0:1], 0xd
-; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x9
-; GCN-NEXT:    s_mov_b32 s11, 0xf000
-; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s16, 0x4f7ffffe
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_ashr_i32 s2, s16, 31
-; GCN-NEXT:    s_add_i32 s3, s16, s2
-; GCN-NEXT:    s_xor_b32 s5, s3, s2
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s5
-; GCN-NEXT:    s_mov_b32 s16, 0x4f800000
-; GCN-NEXT:    s_ashr_i32 s6, s17, 31
-; GCN-NEXT:    s_add_i32 s0, s17, s6
+; GCN-NEXT:    s_ashr_i32 s14, s8, 31
+; GCN-NEXT:    s_add_i32 s2, s8, s14
+; GCN-NEXT:    s_xor_b32 s12, s2, s14
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s12
+; GCN-NEXT:    s_ashr_i32 s8, s9, 31
+; GCN-NEXT:    s_add_i32 s2, s9, s8
+; GCN-NEXT:    s_xor_b32 s15, s2, s8
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT:    s_xor_b32 s17, s0, s6
-; GCN-NEXT:    v_cvt_f32_u32_e32 v3, s17
-; GCN-NEXT:    s_ashr_i32 s3, s12, 31
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s15
+; GCN-NEXT:    s_sub_i32 s3, 0, s12
+; GCN-NEXT:    s_ashr_i32 s9, s4, 31
 ; GCN-NEXT:    v_mul_f32_e32 v0, s16, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    s_add_i32 s4, s12, s3
-; GCN-NEXT:    s_xor_b32 s4, s4, s3
-; GCN-NEXT:    s_xor_b32 s7, s3, s2
-; GCN-NEXT:    v_mul_lo_u32 v1, v0, s5
-; GCN-NEXT:    v_mul_hi_u32 v2, v0, s5
-; GCN-NEXT:    s_ashr_i32 s12, s13, 31
-; GCN-NEXT:    s_add_i32 s13, s13, s12
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v1
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v4, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v3
-; GCN-NEXT:    s_xor_b32 s13, s13, s12
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v1, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v3, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v0, v0, s4
-; GCN-NEXT:    v_mul_f32_e32 v1, s16, v2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GCN-NEXT:    s_add_i32 s2, s4, s9
+; GCN-NEXT:    s_xor_b32 s2, s2, s9
+; GCN-NEXT:    v_mul_lo_u32 v2, s3, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, s16, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    v_mul_lo_u32 v2, v0, s5
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, -1, v0
-; GCN-NEXT:    v_mul_hi_u32 v5, v1, s17
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s4, v2
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s5, v4
-; GCN-NEXT:    v_mul_lo_u32 v4, v1, s17
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], s4, v2
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v5
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
-; GCN-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v4, v4, v1
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v4, v1
-; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v4, v1
-; GCN-NEXT:    s_and_b64 vcc, s[2:3], s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    s_ashr_i32 s5, s18, 31
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v3, v0, s[0:1]
-; GCN-NEXT:    s_add_i32 s0, s18, s5
-; GCN-NEXT:    s_xor_b32 s4, s12, s6
-; GCN-NEXT:    s_xor_b32 s12, s0, s5
-; GCN-NEXT:    v_cvt_f32_u32_e32 v4, s12
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, s13
-; GCN-NEXT:    v_xor_b32_e32 v0, s7, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s7, v0
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; GCN-NEXT:    v_mul_lo_u32 v2, v1, s17
-; GCN-NEXT:    s_ashr_i32 s6, s19, 31
-; GCN-NEXT:    v_mul_f32_e32 v4, s16, v4
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s13, v2
-; GCN-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s17, v3
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s13, v2
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, -1, v1
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v1
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[2:3]
-; GCN-NEXT:    v_mul_lo_u32 v2, v4, s12
-; GCN-NEXT:    v_mul_hi_u32 v3, v4, s12
-; GCN-NEXT:    s_ashr_i32 s2, s14, 31
-; GCN-NEXT:    s_add_i32 s3, s14, s2
-; GCN-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v3
+; GCN-NEXT:    s_sub_i32 s3, 0, s15
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, v2
+; GCN-NEXT:    s_ashr_i32 s4, s5, 31
+; GCN-NEXT:    v_mul_lo_u32 v3, s3, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GCN-NEXT:    v_mul_hi_u32 v2, v1, v3
+; GCN-NEXT:    v_mul_lo_u32 v3, v0, s12
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s2, v3
+; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s12, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, s12, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v4, s[2:3]
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v3
+; GCN-NEXT:    s_add_i32 s2, s5, s4
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GCN-NEXT:    s_xor_b32 s2, s2, s4
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GCN-NEXT:    v_mul_hi_u32 v1, s2, v1
+; GCN-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x9
+; GCN-NEXT:    s_xor_b32 s0, s9, s14
+; GCN-NEXT:    v_xor_b32_e32 v0, s0, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, s15
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v1
+; GCN-NEXT:    s_ashr_i32 s3, s6, 31
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s2, v2
+; GCN-NEXT:    s_ashr_i32 s2, s10, 31
+; GCN-NEXT:    s_add_i32 s0, s10, s2
+; GCN-NEXT:    s_xor_b32 s5, s0, s2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v3, s5
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s15, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v4, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, s15, v2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v4, s[0:1]
+; GCN-NEXT:    s_sub_i32 s0, 0, s5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v1
+; GCN-NEXT:    v_mul_f32_e32 v3, s16, v3
+; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s15, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; GCN-NEXT:    s_xor_b32 s1, s4, s8
+; GCN-NEXT:    v_mul_lo_u32 v5, s0, v3
+; GCN-NEXT:    s_add_i32 s0, s6, s3
+; GCN-NEXT:    s_xor_b32 s0, s0, s3
+; GCN-NEXT:    s_ashr_i32 s4, s11, 31
+; GCN-NEXT:    v_mul_hi_u32 v2, v3, v5
+; GCN-NEXT:    v_xor_b32_e32 v1, s1, v1
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s1, v1
+; GCN-NEXT:    s_xor_b32 s2, s3, s2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GCN-NEXT:    v_mul_hi_u32 v2, s0, v2
+; GCN-NEXT:    s_mov_b32 s15, 0xf000
+; GCN-NEXT:    s_mov_b32 s14, -1
+; GCN-NEXT:    v_mul_lo_u32 v3, v2, s5
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v2
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s0, v3
+; GCN-NEXT:    s_add_i32 s0, s11, s4
+; GCN-NEXT:    s_xor_b32 s6, s0, s4
+; GCN-NEXT:    v_cvt_f32_u32_e32 v4, s6
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s5, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v2, v2, v4
-; GCN-NEXT:    s_xor_b32 s3, s3, s2
-; GCN-NEXT:    v_xor_b32_e32 v1, s4, v1
-; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s4, v1
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v2, v4
-; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; GCN-NEXT:    s_add_i32 s0, s19, s6
-; GCN-NEXT:    s_xor_b32 s14, s0, s6
-; GCN-NEXT:    v_cvt_f32_u32_e32 v4, s14
-; GCN-NEXT:    v_mul_hi_u32 v2, v2, s3
-; GCN-NEXT:    s_xor_b32 s7, s2, s5
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s5, v3
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v4
-; GCN-NEXT:    v_mul_lo_u32 v3, v2, s12
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v5, s[0:1]
+; GCN-NEXT:    s_sub_i32 s0, 0, s6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v2
 ; GCN-NEXT:    v_mul_f32_e32 v4, s16, v4
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GCN-NEXT:    v_sub_i32_e32 v5, vcc, s3, v3
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s12, v5
-; GCN-NEXT:    s_ashr_i32 s12, s15, 31
-; GCN-NEXT:    v_mul_lo_u32 v6, v4, s14
-; GCN-NEXT:    v_mul_hi_u32 v7, v4, s14
-; GCN-NEXT:    s_add_i32 s13, s15, s12
-; GCN-NEXT:    s_xor_b32 s13, s13, s12
-; GCN-NEXT:    v_sub_i32_e32 v8, vcc, 0, v6
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v7
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v8, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v6, v6, v4
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s3, v3
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, -1, v2
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, 1, v2
-; GCN-NEXT:    v_add_i32_e32 v7, vcc, v6, v4
-; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, v6, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v4, v4, s13
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
-; GCN-NEXT:    v_mul_lo_u32 v3, v4, s14
-; GCN-NEXT:    v_xor_b32_e32 v2, s7, v2
-; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s7, v2
-; GCN-NEXT:    s_xor_b32 s4, s12, s6
-; GCN-NEXT:    v_sub_i32_e32 v5, vcc, s13, v3
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s14, v5
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s13, v3
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, -1, v4
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, 1, v4
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[2:3]
-; GCN-NEXT:    v_xor_b32_e32 v3, s4, v3
-; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s4, v3
-; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s5, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GCN-NEXT:    v_xor_b32_e32 v2, s2, v2
+; GCN-NEXT:    v_mul_lo_u32 v6, s0, v4
+; GCN-NEXT:    s_ashr_i32 s0, s7, 31
+; GCN-NEXT:    s_add_i32 s1, s7, s0
+; GCN-NEXT:    s_xor_b32 s1, s1, s0
+; GCN-NEXT:    v_mul_hi_u32 v3, v4, v6
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s2, v2
+; GCN-NEXT:    s_xor_b32 s2, s0, s4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; GCN-NEXT:    v_mul_hi_u32 v3, s1, v3
+; GCN-NEXT:    v_mul_lo_u32 v4, v3, s6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s1, v4
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s6, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v5, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s6, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v5, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s6, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GCN-NEXT:    v_xor_b32_e32 v3, s2, v3
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s2, v3
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[12:15], 0
 ; GCN-NEXT:    s_endpgm
   %r = sdiv <4 x i32> %x, %y
   store <4 x i32> %r, <4 x i32> addrspace(1)* %out
@@ -1680,324 +1466,260 @@ define amdgpu_kernel void @srem_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %x
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i32 [[TMP6]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = uitofp i32 [[TMP8]] to float
 ; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
-; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP10]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP10]], 0x41EFFFFFC0000000
 ; CHECK-NEXT:    [[TMP12:%.*]] = fptoui float [[TMP11]] to i32
-; CHECK-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP12]] to i64
-; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP8]] to i64
-; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP13]], [[TMP14]]
-; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32
-; CHECK-NEXT:    [[TMP17:%.*]] = lshr i64 [[TMP15]], 32
+; CHECK-NEXT:    [[TMP13:%.*]] = sub i32 0, [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = mul i32 [[TMP13]], [[TMP12]]
+; CHECK-NEXT:    [[TMP15:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP15]], [[TMP16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = trunc i64 [[TMP17]] to i32
-; CHECK-NEXT:    [[TMP19:%.*]] = sub i32 0, [[TMP16]]
-; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i32 [[TMP18]], 0
-; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP20]], i32 [[TMP19]], i32 [[TMP16]]
-; CHECK-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP21]] to i64
-; CHECK-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = lshr i64 [[TMP17]], 32
+; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
+; CHECK-NEXT:    [[TMP21:%.*]] = add i32 [[TMP12]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP21]] to i64
 ; CHECK-NEXT:    [[TMP24:%.*]] = mul i64 [[TMP22]], [[TMP23]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP24]] to i32
 ; CHECK-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP24]], 32
 ; CHECK-NEXT:    [[TMP27:%.*]] = trunc i64 [[TMP26]] to i32
-; CHECK-NEXT:    [[TMP28:%.*]] = add i32 [[TMP12]], [[TMP27]]
-; CHECK-NEXT:    [[TMP29:%.*]] = sub i32 [[TMP12]], [[TMP27]]
-; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP20]], i32 [[TMP28]], i32 [[TMP29]]
-; CHECK-NEXT:    [[TMP31:%.*]] = zext i32 [[TMP30]] to i64
-; CHECK-NEXT:    [[TMP32:%.*]] = zext i32 [[TMP7]] to i64
-; CHECK-NEXT:    [[TMP33:%.*]] = mul i64 [[TMP31]], [[TMP32]]
-; CHECK-NEXT:    [[TMP34:%.*]] = trunc i64 [[TMP33]] to i32
-; CHECK-NEXT:    [[TMP35:%.*]] = lshr i64 [[TMP33]], 32
-; CHECK-NEXT:    [[TMP36:%.*]] = trunc i64 [[TMP35]] to i32
-; CHECK-NEXT:    [[TMP37:%.*]] = mul i32 [[TMP36]], [[TMP8]]
-; CHECK-NEXT:    [[TMP38:%.*]] = sub i32 [[TMP7]], [[TMP37]]
-; CHECK-NEXT:    [[TMP39:%.*]] = icmp uge i32 [[TMP38]], [[TMP8]]
-; CHECK-NEXT:    [[TMP40:%.*]] = icmp uge i32 [[TMP7]], [[TMP37]]
-; CHECK-NEXT:    [[TMP41:%.*]] = and i1 [[TMP39]], [[TMP40]]
-; CHECK-NEXT:    [[TMP42:%.*]] = sub i32 [[TMP38]], [[TMP8]]
-; CHECK-NEXT:    [[TMP43:%.*]] = add i32 [[TMP38]], [[TMP8]]
-; CHECK-NEXT:    [[TMP44:%.*]] = select i1 [[TMP41]], i32 [[TMP42]], i32 [[TMP38]]
-; CHECK-NEXT:    [[TMP45:%.*]] = select i1 [[TMP40]], i32 [[TMP44]], i32 [[TMP43]]
-; CHECK-NEXT:    [[TMP46:%.*]] = xor i32 [[TMP45]], [[TMP3]]
-; CHECK-NEXT:    [[TMP47:%.*]] = sub i32 [[TMP46]], [[TMP3]]
-; CHECK-NEXT:    [[TMP48:%.*]] = insertelement <4 x i32> undef, i32 [[TMP47]], i64 0
-; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <4 x i32> [[X]], i64 1
-; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <4 x i32> [[Y]], i64 1
-; CHECK-NEXT:    [[TMP51:%.*]] = ashr i32 [[TMP49]], 31
-; CHECK-NEXT:    [[TMP52:%.*]] = ashr i32 [[TMP50]], 31
-; CHECK-NEXT:    [[TMP53:%.*]] = add i32 [[TMP49]], [[TMP51]]
-; CHECK-NEXT:    [[TMP54:%.*]] = add i32 [[TMP50]], [[TMP52]]
-; CHECK-NEXT:    [[TMP55:%.*]] = xor i32 [[TMP53]], [[TMP51]]
-; CHECK-NEXT:    [[TMP56:%.*]] = xor i32 [[TMP54]], [[TMP52]]
-; CHECK-NEXT:    [[TMP57:%.*]] = uitofp i32 [[TMP56]] to float
-; CHECK-NEXT:    [[TMP58:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP57]])
-; CHECK-NEXT:    [[TMP59:%.*]] = fmul fast float [[TMP58]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP60:%.*]] = fptoui float [[TMP59]] to i32
-; CHECK-NEXT:    [[TMP61:%.*]] = zext i32 [[TMP60]] to i64
-; CHECK-NEXT:    [[TMP62:%.*]] = zext i32 [[TMP56]] to i64
-; CHECK-NEXT:    [[TMP63:%.*]] = mul i64 [[TMP61]], [[TMP62]]
-; CHECK-NEXT:    [[TMP64:%.*]] = trunc i64 [[TMP63]] to i32
-; CHECK-NEXT:    [[TMP65:%.*]] = lshr i64 [[TMP63]], 32
-; CHECK-NEXT:    [[TMP66:%.*]] = trunc i64 [[TMP65]] to i32
-; CHECK-NEXT:    [[TMP67:%.*]] = sub i32 0, [[TMP64]]
-; CHECK-NEXT:    [[TMP68:%.*]] = icmp eq i32 [[TMP66]], 0
-; CHECK-NEXT:    [[TMP69:%.*]] = select i1 [[TMP68]], i32 [[TMP67]], i32 [[TMP64]]
-; CHECK-NEXT:    [[TMP70:%.*]] = zext i32 [[TMP69]] to i64
-; CHECK-NEXT:    [[TMP71:%.*]] = zext i32 [[TMP60]] to i64
-; CHECK-NEXT:    [[TMP72:%.*]] = mul i64 [[TMP70]], [[TMP71]]
-; CHECK-NEXT:    [[TMP73:%.*]] = trunc i64 [[TMP72]] to i32
-; CHECK-NEXT:    [[TMP74:%.*]] = lshr i64 [[TMP72]], 32
-; CHECK-NEXT:    [[TMP75:%.*]] = trunc i64 [[TMP74]] to i32
-; CHECK-NEXT:    [[TMP76:%.*]] = add i32 [[TMP60]], [[TMP75]]
-; CHECK-NEXT:    [[TMP77:%.*]] = sub i32 [[TMP60]], [[TMP75]]
-; CHECK-NEXT:    [[TMP78:%.*]] = select i1 [[TMP68]], i32 [[TMP76]], i32 [[TMP77]]
-; CHECK-NEXT:    [[TMP79:%.*]] = zext i32 [[TMP78]] to i64
-; CHECK-NEXT:    [[TMP80:%.*]] = zext i32 [[TMP55]] to i64
-; CHECK-NEXT:    [[TMP81:%.*]] = mul i64 [[TMP79]], [[TMP80]]
-; CHECK-NEXT:    [[TMP82:%.*]] = trunc i64 [[TMP81]] to i32
-; CHECK-NEXT:    [[TMP83:%.*]] = lshr i64 [[TMP81]], 32
-; CHECK-NEXT:    [[TMP84:%.*]] = trunc i64 [[TMP83]] to i32
-; CHECK-NEXT:    [[TMP85:%.*]] = mul i32 [[TMP84]], [[TMP56]]
-; CHECK-NEXT:    [[TMP86:%.*]] = sub i32 [[TMP55]], [[TMP85]]
-; CHECK-NEXT:    [[TMP87:%.*]] = icmp uge i32 [[TMP86]], [[TMP56]]
-; CHECK-NEXT:    [[TMP88:%.*]] = icmp uge i32 [[TMP55]], [[TMP85]]
-; CHECK-NEXT:    [[TMP89:%.*]] = and i1 [[TMP87]], [[TMP88]]
-; CHECK-NEXT:    [[TMP90:%.*]] = sub i32 [[TMP86]], [[TMP56]]
-; CHECK-NEXT:    [[TMP91:%.*]] = add i32 [[TMP86]], [[TMP56]]
-; CHECK-NEXT:    [[TMP92:%.*]] = select i1 [[TMP89]], i32 [[TMP90]], i32 [[TMP86]]
-; CHECK-NEXT:    [[TMP93:%.*]] = select i1 [[TMP88]], i32 [[TMP92]], i32 [[TMP91]]
-; CHECK-NEXT:    [[TMP94:%.*]] = xor i32 [[TMP93]], [[TMP51]]
-; CHECK-NEXT:    [[TMP95:%.*]] = sub i32 [[TMP94]], [[TMP51]]
-; CHECK-NEXT:    [[TMP96:%.*]] = insertelement <4 x i32> [[TMP48]], i32 [[TMP95]], i64 1
-; CHECK-NEXT:    [[TMP97:%.*]] = extractelement <4 x i32> [[X]], i64 2
-; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <4 x i32> [[Y]], i64 2
-; CHECK-NEXT:    [[TMP99:%.*]] = ashr i32 [[TMP97]], 31
-; CHECK-NEXT:    [[TMP100:%.*]] = ashr i32 [[TMP98]], 31
-; CHECK-NEXT:    [[TMP101:%.*]] = add i32 [[TMP97]], [[TMP99]]
-; CHECK-NEXT:    [[TMP102:%.*]] = add i32 [[TMP98]], [[TMP100]]
-; CHECK-NEXT:    [[TMP103:%.*]] = xor i32 [[TMP101]], [[TMP99]]
-; CHECK-NEXT:    [[TMP104:%.*]] = xor i32 [[TMP102]], [[TMP100]]
-; CHECK-NEXT:    [[TMP105:%.*]] = uitofp i32 [[TMP104]] to float
-; CHECK-NEXT:    [[TMP106:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP105]])
-; CHECK-NEXT:    [[TMP107:%.*]] = fmul fast float [[TMP106]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP108:%.*]] = fptoui float [[TMP107]] to i32
-; CHECK-NEXT:    [[TMP109:%.*]] = zext i32 [[TMP108]] to i64
-; CHECK-NEXT:    [[TMP110:%.*]] = zext i32 [[TMP104]] to i64
-; CHECK-NEXT:    [[TMP111:%.*]] = mul i64 [[TMP109]], [[TMP110]]
-; CHECK-NEXT:    [[TMP112:%.*]] = trunc i64 [[TMP111]] to i32
-; CHECK-NEXT:    [[TMP113:%.*]] = lshr i64 [[TMP111]], 32
-; CHECK-NEXT:    [[TMP114:%.*]] = trunc i64 [[TMP113]] to i32
-; CHECK-NEXT:    [[TMP115:%.*]] = sub i32 0, [[TMP112]]
-; CHECK-NEXT:    [[TMP116:%.*]] = icmp eq i32 [[TMP114]], 0
-; CHECK-NEXT:    [[TMP117:%.*]] = select i1 [[TMP116]], i32 [[TMP115]], i32 [[TMP112]]
-; CHECK-NEXT:    [[TMP118:%.*]] = zext i32 [[TMP117]] to i64
-; CHECK-NEXT:    [[TMP119:%.*]] = zext i32 [[TMP108]] to i64
-; CHECK-NEXT:    [[TMP120:%.*]] = mul i64 [[TMP118]], [[TMP119]]
-; CHECK-NEXT:    [[TMP121:%.*]] = trunc i64 [[TMP120]] to i32
-; CHECK-NEXT:    [[TMP122:%.*]] = lshr i64 [[TMP120]], 32
-; CHECK-NEXT:    [[TMP123:%.*]] = trunc i64 [[TMP122]] to i32
-; CHECK-NEXT:    [[TMP124:%.*]] = add i32 [[TMP108]], [[TMP123]]
-; CHECK-NEXT:    [[TMP125:%.*]] = sub i32 [[TMP108]], [[TMP123]]
-; CHECK-NEXT:    [[TMP126:%.*]] = select i1 [[TMP116]], i32 [[TMP124]], i32 [[TMP125]]
-; CHECK-NEXT:    [[TMP127:%.*]] = zext i32 [[TMP126]] to i64
-; CHECK-NEXT:    [[TMP128:%.*]] = zext i32 [[TMP103]] to i64
-; CHECK-NEXT:    [[TMP129:%.*]] = mul i64 [[TMP127]], [[TMP128]]
-; CHECK-NEXT:    [[TMP130:%.*]] = trunc i64 [[TMP129]] to i32
-; CHECK-NEXT:    [[TMP131:%.*]] = lshr i64 [[TMP129]], 32
+; CHECK-NEXT:    [[TMP28:%.*]] = mul i32 [[TMP27]], [[TMP8]]
+; CHECK-NEXT:    [[TMP29:%.*]] = sub i32 [[TMP7]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp uge i32 [[TMP29]], [[TMP8]]
+; CHECK-NEXT:    [[TMP31:%.*]] = sub i32 [[TMP29]], [[TMP8]]
+; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP30]], i32 [[TMP31]], i32 [[TMP29]]
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp uge i32 [[TMP32]], [[TMP8]]
+; CHECK-NEXT:    [[TMP34:%.*]] = sub i32 [[TMP32]], [[TMP8]]
+; CHECK-NEXT:    [[TMP35:%.*]] = select i1 [[TMP33]], i32 [[TMP34]], i32 [[TMP32]]
+; CHECK-NEXT:    [[TMP36:%.*]] = xor i32 [[TMP35]], [[TMP3]]
+; CHECK-NEXT:    [[TMP37:%.*]] = sub i32 [[TMP36]], [[TMP3]]
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <4 x i32> undef, i32 [[TMP37]], i64 0
+; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <4 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <4 x i32> [[Y]], i64 1
+; CHECK-NEXT:    [[TMP41:%.*]] = ashr i32 [[TMP39]], 31
+; CHECK-NEXT:    [[TMP42:%.*]] = ashr i32 [[TMP40]], 31
+; CHECK-NEXT:    [[TMP43:%.*]] = add i32 [[TMP39]], [[TMP41]]
+; CHECK-NEXT:    [[TMP44:%.*]] = add i32 [[TMP40]], [[TMP42]]
+; CHECK-NEXT:    [[TMP45:%.*]] = xor i32 [[TMP43]], [[TMP41]]
+; CHECK-NEXT:    [[TMP46:%.*]] = xor i32 [[TMP44]], [[TMP42]]
+; CHECK-NEXT:    [[TMP47:%.*]] = uitofp i32 [[TMP46]] to float
+; CHECK-NEXT:    [[TMP48:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP47]])
+; CHECK-NEXT:    [[TMP49:%.*]] = fmul fast float [[TMP48]], 0x41EFFFFFC0000000
+; CHECK-NEXT:    [[TMP50:%.*]] = fptoui float [[TMP49]] to i32
+; CHECK-NEXT:    [[TMP51:%.*]] = sub i32 0, [[TMP46]]
+; CHECK-NEXT:    [[TMP52:%.*]] = mul i32 [[TMP51]], [[TMP50]]
+; CHECK-NEXT:    [[TMP53:%.*]] = zext i32 [[TMP50]] to i64
+; CHECK-NEXT:    [[TMP54:%.*]] = zext i32 [[TMP52]] to i64
+; CHECK-NEXT:    [[TMP55:%.*]] = mul i64 [[TMP53]], [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = trunc i64 [[TMP55]] to i32
+; CHECK-NEXT:    [[TMP57:%.*]] = lshr i64 [[TMP55]], 32
+; CHECK-NEXT:    [[TMP58:%.*]] = trunc i64 [[TMP57]] to i32
+; CHECK-NEXT:    [[TMP59:%.*]] = add i32 [[TMP50]], [[TMP58]]
+; CHECK-NEXT:    [[TMP60:%.*]] = zext i32 [[TMP45]] to i64
+; CHECK-NEXT:    [[TMP61:%.*]] = zext i32 [[TMP59]] to i64
+; CHECK-NEXT:    [[TMP62:%.*]] = mul i64 [[TMP60]], [[TMP61]]
+; CHECK-NEXT:    [[TMP63:%.*]] = trunc i64 [[TMP62]] to i32
+; CHECK-NEXT:    [[TMP64:%.*]] = lshr i64 [[TMP62]], 32
+; CHECK-NEXT:    [[TMP65:%.*]] = trunc i64 [[TMP64]] to i32
+; CHECK-NEXT:    [[TMP66:%.*]] = mul i32 [[TMP65]], [[TMP46]]
+; CHECK-NEXT:    [[TMP67:%.*]] = sub i32 [[TMP45]], [[TMP66]]
+; CHECK-NEXT:    [[TMP68:%.*]] = icmp uge i32 [[TMP67]], [[TMP46]]
+; CHECK-NEXT:    [[TMP69:%.*]] = sub i32 [[TMP67]], [[TMP46]]
+; CHECK-NEXT:    [[TMP70:%.*]] = select i1 [[TMP68]], i32 [[TMP69]], i32 [[TMP67]]
+; CHECK-NEXT:    [[TMP71:%.*]] = icmp uge i32 [[TMP70]], [[TMP46]]
+; CHECK-NEXT:    [[TMP72:%.*]] = sub i32 [[TMP70]], [[TMP46]]
+; CHECK-NEXT:    [[TMP73:%.*]] = select i1 [[TMP71]], i32 [[TMP72]], i32 [[TMP70]]
+; CHECK-NEXT:    [[TMP74:%.*]] = xor i32 [[TMP73]], [[TMP41]]
+; CHECK-NEXT:    [[TMP75:%.*]] = sub i32 [[TMP74]], [[TMP41]]
+; CHECK-NEXT:    [[TMP76:%.*]] = insertelement <4 x i32> [[TMP38]], i32 [[TMP75]], i64 1
+; CHECK-NEXT:    [[TMP77:%.*]] = extractelement <4 x i32> [[X]], i64 2
+; CHECK-NEXT:    [[TMP78:%.*]] = extractelement <4 x i32> [[Y]], i64 2
+; CHECK-NEXT:    [[TMP79:%.*]] = ashr i32 [[TMP77]], 31
+; CHECK-NEXT:    [[TMP80:%.*]] = ashr i32 [[TMP78]], 31
+; CHECK-NEXT:    [[TMP81:%.*]] = add i32 [[TMP77]], [[TMP79]]
+; CHECK-NEXT:    [[TMP82:%.*]] = add i32 [[TMP78]], [[TMP80]]
+; CHECK-NEXT:    [[TMP83:%.*]] = xor i32 [[TMP81]], [[TMP79]]
+; CHECK-NEXT:    [[TMP84:%.*]] = xor i32 [[TMP82]], [[TMP80]]
+; CHECK-NEXT:    [[TMP85:%.*]] = uitofp i32 [[TMP84]] to float
+; CHECK-NEXT:    [[TMP86:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP85]])
+; CHECK-NEXT:    [[TMP87:%.*]] = fmul fast float [[TMP86]], 0x41EFFFFFC0000000
+; CHECK-NEXT:    [[TMP88:%.*]] = fptoui float [[TMP87]] to i32
+; CHECK-NEXT:    [[TMP89:%.*]] = sub i32 0, [[TMP84]]
+; CHECK-NEXT:    [[TMP90:%.*]] = mul i32 [[TMP89]], [[TMP88]]
+; CHECK-NEXT:    [[TMP91:%.*]] = zext i32 [[TMP88]] to i64
+; CHECK-NEXT:    [[TMP92:%.*]] = zext i32 [[TMP90]] to i64
+; CHECK-NEXT:    [[TMP93:%.*]] = mul i64 [[TMP91]], [[TMP92]]
+; CHECK-NEXT:    [[TMP94:%.*]] = trunc i64 [[TMP93]] to i32
+; CHECK-NEXT:    [[TMP95:%.*]] = lshr i64 [[TMP93]], 32
+; CHECK-NEXT:    [[TMP96:%.*]] = trunc i64 [[TMP95]] to i32
+; CHECK-NEXT:    [[TMP97:%.*]] = add i32 [[TMP88]], [[TMP96]]
+; CHECK-NEXT:    [[TMP98:%.*]] = zext i32 [[TMP83]] to i64
+; CHECK-NEXT:    [[TMP99:%.*]] = zext i32 [[TMP97]] to i64
+; CHECK-NEXT:    [[TMP100:%.*]] = mul i64 [[TMP98]], [[TMP99]]
+; CHECK-NEXT:    [[TMP101:%.*]] = trunc i64 [[TMP100]] to i32
+; CHECK-NEXT:    [[TMP102:%.*]] = lshr i64 [[TMP100]], 32
+; CHECK-NEXT:    [[TMP103:%.*]] = trunc i64 [[TMP102]] to i32
+; CHECK-NEXT:    [[TMP104:%.*]] = mul i32 [[TMP103]], [[TMP84]]
+; CHECK-NEXT:    [[TMP105:%.*]] = sub i32 [[TMP83]], [[TMP104]]
+; CHECK-NEXT:    [[TMP106:%.*]] = icmp uge i32 [[TMP105]], [[TMP84]]
+; CHECK-NEXT:    [[TMP107:%.*]] = sub i32 [[TMP105]], [[TMP84]]
+; CHECK-NEXT:    [[TMP108:%.*]] = select i1 [[TMP106]], i32 [[TMP107]], i32 [[TMP105]]
+; CHECK-NEXT:    [[TMP109:%.*]] = icmp uge i32 [[TMP108]], [[TMP84]]
+; CHECK-NEXT:    [[TMP110:%.*]] = sub i32 [[TMP108]], [[TMP84]]
+; CHECK-NEXT:    [[TMP111:%.*]] = select i1 [[TMP109]], i32 [[TMP110]], i32 [[TMP108]]
+; CHECK-NEXT:    [[TMP112:%.*]] = xor i32 [[TMP111]], [[TMP79]]
+; CHECK-NEXT:    [[TMP113:%.*]] = sub i32 [[TMP112]], [[TMP79]]
+; CHECK-NEXT:    [[TMP114:%.*]] = insertelement <4 x i32> [[TMP76]], i32 [[TMP113]], i64 2
+; CHECK-NEXT:    [[TMP115:%.*]] = extractelement <4 x i32> [[X]], i64 3
+; CHECK-NEXT:    [[TMP116:%.*]] = extractelement <4 x i32> [[Y]], i64 3
+; CHECK-NEXT:    [[TMP117:%.*]] = ashr i32 [[TMP115]], 31
+; CHECK-NEXT:    [[TMP118:%.*]] = ashr i32 [[TMP116]], 31
+; CHECK-NEXT:    [[TMP119:%.*]] = add i32 [[TMP115]], [[TMP117]]
+; CHECK-NEXT:    [[TMP120:%.*]] = add i32 [[TMP116]], [[TMP118]]
+; CHECK-NEXT:    [[TMP121:%.*]] = xor i32 [[TMP119]], [[TMP117]]
+; CHECK-NEXT:    [[TMP122:%.*]] = xor i32 [[TMP120]], [[TMP118]]
+; CHECK-NEXT:    [[TMP123:%.*]] = uitofp i32 [[TMP122]] to float
+; CHECK-NEXT:    [[TMP124:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP123]])
+; CHECK-NEXT:    [[TMP125:%.*]] = fmul fast float [[TMP124]], 0x41EFFFFFC0000000
+; CHECK-NEXT:    [[TMP126:%.*]] = fptoui float [[TMP125]] to i32
+; CHECK-NEXT:    [[TMP127:%.*]] = sub i32 0, [[TMP122]]
+; CHECK-NEXT:    [[TMP128:%.*]] = mul i32 [[TMP127]], [[TMP126]]
+; CHECK-NEXT:    [[TMP129:%.*]] = zext i32 [[TMP126]] to i64
+; CHECK-NEXT:    [[TMP130:%.*]] = zext i32 [[TMP128]] to i64
+; CHECK-NEXT:    [[TMP131:%.*]] = mul i64 [[TMP129]], [[TMP130]]
 ; CHECK-NEXT:    [[TMP132:%.*]] = trunc i64 [[TMP131]] to i32
-; CHECK-NEXT:    [[TMP133:%.*]] = mul i32 [[TMP132]], [[TMP104]]
-; CHECK-NEXT:    [[TMP134:%.*]] = sub i32 [[TMP103]], [[TMP133]]
-; CHECK-NEXT:    [[TMP135:%.*]] = icmp uge i32 [[TMP134]], [[TMP104]]
-; CHECK-NEXT:    [[TMP136:%.*]] = icmp uge i32 [[TMP103]], [[TMP133]]
-; CHECK-NEXT:    [[TMP137:%.*]] = and i1 [[TMP135]], [[TMP136]]
-; CHECK-NEXT:    [[TMP138:%.*]] = sub i32 [[TMP134]], [[TMP104]]
-; CHECK-NEXT:    [[TMP139:%.*]] = add i32 [[TMP134]], [[TMP104]]
-; CHECK-NEXT:    [[TMP140:%.*]] = select i1 [[TMP137]], i32 [[TMP138]], i32 [[TMP134]]
-; CHECK-NEXT:    [[TMP141:%.*]] = select i1 [[TMP136]], i32 [[TMP140]], i32 [[TMP139]]
-; CHECK-NEXT:    [[TMP142:%.*]] = xor i32 [[TMP141]], [[TMP99]]
-; CHECK-NEXT:    [[TMP143:%.*]] = sub i32 [[TMP142]], [[TMP99]]
-; CHECK-NEXT:    [[TMP144:%.*]] = insertelement <4 x i32> [[TMP96]], i32 [[TMP143]], i64 2
-; CHECK-NEXT:    [[TMP145:%.*]] = extractelement <4 x i32> [[X]], i64 3
-; CHECK-NEXT:    [[TMP146:%.*]] = extractelement <4 x i32> [[Y]], i64 3
-; CHECK-NEXT:    [[TMP147:%.*]] = ashr i32 [[TMP145]], 31
-; CHECK-NEXT:    [[TMP148:%.*]] = ashr i32 [[TMP146]], 31
-; CHECK-NEXT:    [[TMP149:%.*]] = add i32 [[TMP145]], [[TMP147]]
-; CHECK-NEXT:    [[TMP150:%.*]] = add i32 [[TMP146]], [[TMP148]]
-; CHECK-NEXT:    [[TMP151:%.*]] = xor i32 [[TMP149]], [[TMP147]]
-; CHECK-NEXT:    [[TMP152:%.*]] = xor i32 [[TMP150]], [[TMP148]]
-; CHECK-NEXT:    [[TMP153:%.*]] = uitofp i32 [[TMP152]] to float
-; CHECK-NEXT:    [[TMP154:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP153]])
-; CHECK-NEXT:    [[TMP155:%.*]] = fmul fast float [[TMP154]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP156:%.*]] = fptoui float [[TMP155]] to i32
-; CHECK-NEXT:    [[TMP157:%.*]] = zext i32 [[TMP156]] to i64
-; CHECK-NEXT:    [[TMP158:%.*]] = zext i32 [[TMP152]] to i64
-; CHECK-NEXT:    [[TMP159:%.*]] = mul i64 [[TMP157]], [[TMP158]]
-; CHECK-NEXT:    [[TMP160:%.*]] = trunc i64 [[TMP159]] to i32
-; CHECK-NEXT:    [[TMP161:%.*]] = lshr i64 [[TMP159]], 32
-; CHECK-NEXT:    [[TMP162:%.*]] = trunc i64 [[TMP161]] to i32
-; CHECK-NEXT:    [[TMP163:%.*]] = sub i32 0, [[TMP160]]
-; CHECK-NEXT:    [[TMP164:%.*]] = icmp eq i32 [[TMP162]], 0
-; CHECK-NEXT:    [[TMP165:%.*]] = select i1 [[TMP164]], i32 [[TMP163]], i32 [[TMP160]]
-; CHECK-NEXT:    [[TMP166:%.*]] = zext i32 [[TMP165]] to i64
-; CHECK-NEXT:    [[TMP167:%.*]] = zext i32 [[TMP156]] to i64
-; CHECK-NEXT:    [[TMP168:%.*]] = mul i64 [[TMP166]], [[TMP167]]
-; CHECK-NEXT:    [[TMP169:%.*]] = trunc i64 [[TMP168]] to i32
-; CHECK-NEXT:    [[TMP170:%.*]] = lshr i64 [[TMP168]], 32
-; CHECK-NEXT:    [[TMP171:%.*]] = trunc i64 [[TMP170]] to i32
-; CHECK-NEXT:    [[TMP172:%.*]] = add i32 [[TMP156]], [[TMP171]]
-; CHECK-NEXT:    [[TMP173:%.*]] = sub i32 [[TMP156]], [[TMP171]]
-; CHECK-NEXT:    [[TMP174:%.*]] = select i1 [[TMP164]], i32 [[TMP172]], i32 [[TMP173]]
-; CHECK-NEXT:    [[TMP175:%.*]] = zext i32 [[TMP174]] to i64
-; CHECK-NEXT:    [[TMP176:%.*]] = zext i32 [[TMP151]] to i64
-; CHECK-NEXT:    [[TMP177:%.*]] = mul i64 [[TMP175]], [[TMP176]]
-; CHECK-NEXT:    [[TMP178:%.*]] = trunc i64 [[TMP177]] to i32
-; CHECK-NEXT:    [[TMP179:%.*]] = lshr i64 [[TMP177]], 32
-; CHECK-NEXT:    [[TMP180:%.*]] = trunc i64 [[TMP179]] to i32
-; CHECK-NEXT:    [[TMP181:%.*]] = mul i32 [[TMP180]], [[TMP152]]
-; CHECK-NEXT:    [[TMP182:%.*]] = sub i32 [[TMP151]], [[TMP181]]
-; CHECK-NEXT:    [[TMP183:%.*]] = icmp uge i32 [[TMP182]], [[TMP152]]
-; CHECK-NEXT:    [[TMP184:%.*]] = icmp uge i32 [[TMP151]], [[TMP181]]
-; CHECK-NEXT:    [[TMP185:%.*]] = and i1 [[TMP183]], [[TMP184]]
-; CHECK-NEXT:    [[TMP186:%.*]] = sub i32 [[TMP182]], [[TMP152]]
-; CHECK-NEXT:    [[TMP187:%.*]] = add i32 [[TMP182]], [[TMP152]]
-; CHECK-NEXT:    [[TMP188:%.*]] = select i1 [[TMP185]], i32 [[TMP186]], i32 [[TMP182]]
-; CHECK-NEXT:    [[TMP189:%.*]] = select i1 [[TMP184]], i32 [[TMP188]], i32 [[TMP187]]
-; CHECK-NEXT:    [[TMP190:%.*]] = xor i32 [[TMP189]], [[TMP147]]
-; CHECK-NEXT:    [[TMP191:%.*]] = sub i32 [[TMP190]], [[TMP147]]
-; CHECK-NEXT:    [[TMP192:%.*]] = insertelement <4 x i32> [[TMP144]], i32 [[TMP191]], i64 3
-; CHECK-NEXT:    store <4 x i32> [[TMP192]], <4 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP133:%.*]] = lshr i64 [[TMP131]], 32
+; CHECK-NEXT:    [[TMP134:%.*]] = trunc i64 [[TMP133]] to i32
+; CHECK-NEXT:    [[TMP135:%.*]] = add i32 [[TMP126]], [[TMP134]]
+; CHECK-NEXT:    [[TMP136:%.*]] = zext i32 [[TMP121]] to i64
+; CHECK-NEXT:    [[TMP137:%.*]] = zext i32 [[TMP135]] to i64
+; CHECK-NEXT:    [[TMP138:%.*]] = mul i64 [[TMP136]], [[TMP137]]
+; CHECK-NEXT:    [[TMP139:%.*]] = trunc i64 [[TMP138]] to i32
+; CHECK-NEXT:    [[TMP140:%.*]] = lshr i64 [[TMP138]], 32
+; CHECK-NEXT:    [[TMP141:%.*]] = trunc i64 [[TMP140]] to i32
+; CHECK-NEXT:    [[TMP142:%.*]] = mul i32 [[TMP141]], [[TMP122]]
+; CHECK-NEXT:    [[TMP143:%.*]] = sub i32 [[TMP121]], [[TMP142]]
+; CHECK-NEXT:    [[TMP144:%.*]] = icmp uge i32 [[TMP143]], [[TMP122]]
+; CHECK-NEXT:    [[TMP145:%.*]] = sub i32 [[TMP143]], [[TMP122]]
+; CHECK-NEXT:    [[TMP146:%.*]] = select i1 [[TMP144]], i32 [[TMP145]], i32 [[TMP143]]
+; CHECK-NEXT:    [[TMP147:%.*]] = icmp uge i32 [[TMP146]], [[TMP122]]
+; CHECK-NEXT:    [[TMP148:%.*]] = sub i32 [[TMP146]], [[TMP122]]
+; CHECK-NEXT:    [[TMP149:%.*]] = select i1 [[TMP147]], i32 [[TMP148]], i32 [[TMP146]]
+; CHECK-NEXT:    [[TMP150:%.*]] = xor i32 [[TMP149]], [[TMP117]]
+; CHECK-NEXT:    [[TMP151:%.*]] = sub i32 [[TMP150]], [[TMP117]]
+; CHECK-NEXT:    [[TMP152:%.*]] = insertelement <4 x i32> [[TMP114]], i32 [[TMP151]], i64 3
+; CHECK-NEXT:    store <4 x i32> [[TMP152]], <4 x i32> addrspace(1)* [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_v4i32:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx8 s[12:19], s[0:1], 0xd
-; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x9
-; GCN-NEXT:    s_mov_b32 s11, 0xf000
-; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s14, 0x4f7ffffe
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_ashr_i32 s2, s16, 31
-; GCN-NEXT:    s_add_i32 s3, s16, s2
-; GCN-NEXT:    s_xor_b32 s5, s3, s2
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s5
-; GCN-NEXT:    s_mov_b32 s16, 0x4f800000
-; GCN-NEXT:    s_ashr_i32 s6, s12, 31
-; GCN-NEXT:    s_ashr_i32 s2, s17, 31
+; GCN-NEXT:    s_ashr_i32 s2, s8, 31
+; GCN-NEXT:    s_add_i32 s3, s8, s2
+; GCN-NEXT:    s_xor_b32 s2, s3, s2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GCN-NEXT:    s_sub_i32 s13, 0, s2
+; GCN-NEXT:    s_ashr_i32 s12, s9, 31
+; GCN-NEXT:    s_add_i32 s9, s9, s12
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT:    s_add_i32 s0, s12, s6
-; GCN-NEXT:    s_add_i32 s3, s17, s2
-; GCN-NEXT:    s_xor_b32 s4, s0, s6
-; GCN-NEXT:    v_mul_f32_e32 v0, s16, v0
+; GCN-NEXT:    s_xor_b32 s9, s9, s12
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s9
+; GCN-NEXT:    s_ashr_i32 s3, s4, 31
+; GCN-NEXT:    v_mul_f32_e32 v0, s14, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    s_xor_b32 s17, s3, s2
-; GCN-NEXT:    s_ashr_i32 s7, s13, 31
-; GCN-NEXT:    s_add_i32 s12, s13, s7
-; GCN-NEXT:    v_mul_lo_u32 v1, v0, s5
-; GCN-NEXT:    v_mul_hi_u32 v2, v0, s5
-; GCN-NEXT:    s_xor_b32 s12, s12, s7
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s17
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v1, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v3, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v0, v0, s4
-; GCN-NEXT:    v_mul_f32_e32 v1, s16, v1
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GCN-NEXT:    s_add_i32 s4, s4, s3
+; GCN-NEXT:    s_xor_b32 s4, s4, s3
+; GCN-NEXT:    v_mul_lo_u32 v2, s13, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, s14, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s5
-; GCN-NEXT:    v_mul_lo_u32 v4, v1, s17
-; GCN-NEXT:    v_mul_hi_u32 v5, v1, s17
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s4, v0
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s4, v0
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s5, v2
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, s5, v2
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s5, v2
-; GCN-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v4, v4, v1
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v4, v1
-; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v4, v1
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    s_ashr_i32 s0, s18, 31
-; GCN-NEXT:    s_add_i32 s1, s18, s0
-; GCN-NEXT:    s_xor_b32 s13, s1, s0
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s13
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, s12
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v3, v0, s[2:3]
+; GCN-NEXT:    s_sub_i32 s13, 0, s9
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, v2
+; GCN-NEXT:    s_ashr_i32 s12, s10, 31
+; GCN-NEXT:    s_ashr_i32 s8, s5, 31
+; GCN-NEXT:    s_add_i32 s5, s5, s8
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s13, v1
+; GCN-NEXT:    s_xor_b32 s5, s5, s8
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
+; GCN-NEXT:    v_mul_hi_u32 v2, v1, v2
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s4, v0
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s2, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s2, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s2, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s2, v0
+; GCN-NEXT:    s_add_i32 s2, s10, s12
+; GCN-NEXT:    s_xor_b32 s2, s2, s12
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s2
+; GCN-NEXT:    v_mul_hi_u32 v1, s5, v1
+; GCN-NEXT:    v_xor_b32_e32 v0, s3, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s3, v0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GCN-NEXT:    v_xor_b32_e32 v0, s6, v0
-; GCN-NEXT:    v_mul_lo_u32 v1, v1, s17
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v0
-; GCN-NEXT:    v_mul_f32_e32 v2, s16, v2
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s9
+; GCN-NEXT:    s_sub_i32 s3, 0, s2
+; GCN-NEXT:    s_ashr_i32 s4, s6, 31
+; GCN-NEXT:    v_mul_f32_e32 v2, s14, v2
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s12, v1
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s12, v1
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s17, v3
-; GCN-NEXT:    v_mul_lo_u32 v5, v2, s13
-; GCN-NEXT:    v_mul_hi_u32 v6, v2, s13
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, s17, v3
-; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s17, v3
-; GCN-NEXT:    v_sub_i32_e32 v7, vcc, 0, v5
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v5, v5, v7, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v5, v5, v2
-; GCN-NEXT:    s_ashr_i32 s6, s14, 31
-; GCN-NEXT:    s_add_i32 s12, s14, s6
-; GCN-NEXT:    s_xor_b32 s12, s12, s6
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v5, v2
-; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, v5, v2
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    s_ashr_i32 s0, s19, 31
-; GCN-NEXT:    s_add_i32 s1, s19, s0
-; GCN-NEXT:    s_xor_b32 s14, s1, s0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GCN-NEXT:    v_cvt_f32_u32_e32 v3, s14
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v2, v2, s12
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[2:3]
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s5, v1
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s9, v1
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v1
+; GCN-NEXT:    v_mul_lo_u32 v4, s3, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s9, v1
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GCN-NEXT:    v_mul_hi_u32 v3, v2, v4
+; GCN-NEXT:    s_ashr_i32 s5, s11, 31
+; GCN-NEXT:    s_add_i32 s3, s6, s4
+; GCN-NEXT:    s_add_i32 s6, s11, s5
+; GCN-NEXT:    s_xor_b32 s5, s6, s5
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v3, s5
+; GCN-NEXT:    s_xor_b32 s3, s3, s4
+; GCN-NEXT:    v_mul_hi_u32 v2, s3, v2
+; GCN-NEXT:    v_xor_b32_e32 v1, s8, v1
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; GCN-NEXT:    v_xor_b32_e32 v1, s7, v1
-; GCN-NEXT:    v_mul_lo_u32 v2, v2, s13
-; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s7, v1
-; GCN-NEXT:    v_mul_f32_e32 v3, s16, v3
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s8, v1
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, s2
+; GCN-NEXT:    s_ashr_i32 s6, s7, 31
+; GCN-NEXT:    v_mul_f32_e32 v3, s14, v3
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GCN-NEXT:    s_ashr_i32 s7, s15, 31
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s12, v2
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s12, v2
-; GCN-NEXT:    v_mul_lo_u32 v6, v3, s14
-; GCN-NEXT:    v_mul_hi_u32 v7, v3, s14
-; GCN-NEXT:    s_add_i32 s12, s15, s7
-; GCN-NEXT:    s_xor_b32 s12, s12, s7
-; GCN-NEXT:    v_sub_i32_e32 v8, vcc, 0, v6
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v7
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v8, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v6, v6, v3
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v4
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, s13, v4
-; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s13, v4
-; GCN-NEXT:    v_add_i32_e32 v7, vcc, v6, v3
-; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, v6, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v3, v3, s12
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
-; GCN-NEXT:    v_mul_lo_u32 v3, v3, s14
-; GCN-NEXT:    v_xor_b32_e32 v2, s6, v2
-; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s6, v2
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s12, v3
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s12, v3
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s14, v4
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, s14, v4
-; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s14, v4
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[2:3]
-; GCN-NEXT:    v_xor_b32_e32 v3, s7, v3
-; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s7, v3
-; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s3, v2
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, s2, v2
+; GCN-NEXT:    s_sub_i32 s3, 0, s5
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s2, v2
+; GCN-NEXT:    v_mul_lo_u32 v5, s3, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, s2, v2
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s2, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN-NEXT:    v_mul_hi_u32 v4, v3, v5
+; GCN-NEXT:    s_add_i32 s2, s7, s6
+; GCN-NEXT:    s_xor_b32 s7, s2, s6
+; GCN-NEXT:    v_xor_b32_e32 v2, s4, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_mul_hi_u32 v3, s7, v3
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s4, v2
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mul_lo_u32 v3, v3, s5
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s7, v3
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, s5, v3
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s5, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, s5, v3
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s5, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GCN-NEXT:    v_xor_b32_e32 v3, s6, v3
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
   %r = srem <4 x i32> %x, %y
   store <4 x i32> %r, <4 x i32> addrspace(1)* %out
@@ -2086,7 +1808,7 @@ define amdgpu_kernel void @udiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP78:%.*]] = and i32 [[TMP77]], 65535
 ; CHECK-NEXT:    [[TMP79:%.*]] = trunc i32 [[TMP78]] to i16
 ; CHECK-NEXT:    [[TMP80:%.*]] = insertelement <4 x i16> [[TMP60]], i16 [[TMP79]], i64 3
-; CHECK-NEXT:    store <4 x i16> [[TMP80]], <4 x i16> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <4 x i16> [[TMP80]], <4 x i16> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_v4i16:
@@ -2244,7 +1966,7 @@ define amdgpu_kernel void @urem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP86:%.*]] = and i32 [[TMP85]], 65535
 ; CHECK-NEXT:    [[TMP87:%.*]] = trunc i32 [[TMP86]] to i16
 ; CHECK-NEXT:    [[TMP88:%.*]] = insertelement <4 x i16> [[TMP66]], i16 [[TMP87]], i64 3
-; CHECK-NEXT:    store <4 x i16> [[TMP88]], <4 x i16> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <4 x i16> [[TMP88]], <4 x i16> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_v4i16:
@@ -2418,7 +2140,7 @@ define amdgpu_kernel void @sdiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP94:%.*]] = ashr i32 [[TMP93]], 16
 ; CHECK-NEXT:    [[TMP95:%.*]] = trunc i32 [[TMP94]] to i16
 ; CHECK-NEXT:    [[TMP96:%.*]] = insertelement <4 x i16> [[TMP72]], i16 [[TMP95]], i64 3
-; CHECK-NEXT:    store <4 x i16> [[TMP96]], <4 x i16> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <4 x i16> [[TMP96]], <4 x i16> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_v4i16:
@@ -2434,71 +2156,71 @@ define amdgpu_kernel void @sdiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s9
 ; GCN-NEXT:    s_xor_b32 s8, s9, s8
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-NEXT:    s_ashr_i32 s2, s2, 16
 ; GCN-NEXT:    s_ashr_i32 s8, s8, 30
-; GCN-NEXT:    s_or_b32 s8, s8, 1
+; GCN-NEXT:    s_or_b32 s10, s8, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s2
-; GCN-NEXT:    v_mov_b32_e32 v3, s8
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b32 s8, s10, 0
+; GCN-NEXT:    s_ashr_i32 s2, s2, 16
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s2
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 16
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v1
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s0
+; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v0
 ; GCN-NEXT:    s_xor_b32 s0, s0, s2
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-NEXT:    s_or_b32 s0, s0, 1
-; GCN-NEXT:    v_mul_f32_e32 v3, v2, v3
+; GCN-NEXT:    s_sext_i32_i16 s2, s3
+; GCN-NEXT:    v_mul_f32_e32 v3, v1, v3
 ; GCN-NEXT:    v_trunc_f32_e32 v3, v3
-; GCN-NEXT:    v_mad_f32 v2, -v3, v1, v2
-; GCN-NEXT:    v_mov_b32_e32 v4, s0
-; GCN-NEXT:    s_sext_i32_i16 s0, s3
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v1|
+; GCN-NEXT:    v_mad_f32 v1, -v3, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v3, v3
-; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
-; GCN-NEXT:    s_sext_i32_i16 s2, s1
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v1, v3
-; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s2
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
-; GCN-NEXT:    s_xor_b32 s0, s2, s0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s8, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s2
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b32 s0, s0, 0
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s0, v3
+; GCN-NEXT:    s_sext_i32_i16 s0, s1
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v0
+; GCN-NEXT:    s_xor_b32 s0, s0, s2
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
 ; GCN-NEXT:    s_or_b32 s0, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v4, v1, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v4, v4
-; GCN-NEXT:    v_mad_f32 v1, -v4, v2, v1
-; GCN-NEXT:    v_mov_b32_e32 v5, s0
-; GCN-NEXT:    s_ashr_i32 s0, s3, 16
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v2|
+; GCN-NEXT:    v_mad_f32 v1, -v4, v0, v1
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b32 s0, s0, 0
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v5, vcc
-; GCN-NEXT:    s_ashr_i32 s1, s1, 16
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
-; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s1
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v2
-; GCN-NEXT:    s_xor_b32 s0, s1, s0
+; GCN-NEXT:    s_ashr_i32 s2, s3, 16
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s0, v4
+; GCN-NEXT:    s_ashr_i32 s0, s1, 16
+; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v0
+; GCN-NEXT:    s_xor_b32 s0, s0, s2
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v5, v4, v5
 ; GCN-NEXT:    v_trunc_f32_e32 v5, v5
-; GCN-NEXT:    v_mad_f32 v4, -v5, v2, v4
+; GCN-NEXT:    v_mad_f32 v4, -v5, v0, v4
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
-; GCN-NEXT:    v_mov_b32_e32 v6, s0
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, |v2|
-; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v4|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cselect_b32 s0, s2, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v5
 ; GCN-NEXT:    s_mov_b32 s0, 0xffff
-; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GCN-NEXT:    v_and_b32_e32 v1, s0, v1
-; GCN-NEXT:    v_or_b32_e32 v1, v1, v2
-; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
-; GCN-NEXT:    v_and_b32_e32 v0, s0, v0
-; GCN-NEXT:    v_or_b32_e32 v0, v0, v2
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v3
+; GCN-NEXT:    v_and_b32_e32 v2, s0, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v2, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = sdiv <4 x i16> %x, %y
@@ -2612,7 +2334,7 @@ define amdgpu_kernel void @srem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP102:%.*]] = ashr i32 [[TMP101]], 16
 ; CHECK-NEXT:    [[TMP103:%.*]] = trunc i32 [[TMP102]] to i16
 ; CHECK-NEXT:    [[TMP104:%.*]] = insertelement <4 x i16> [[TMP78]], i16 [[TMP103]], i64 3
-; CHECK-NEXT:    store <4 x i16> [[TMP104]], <4 x i16> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <4 x i16> [[TMP104]], <4 x i16> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_v4i16:
@@ -2629,15 +2351,15 @@ define amdgpu_kernel void @srem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; GCN-NEXT:    s_xor_b32 s8, s9, s8
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_ashr_i32 s8, s8, 30
-; GCN-NEXT:    s_or_b32 s8, s8, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s8
+; GCN-NEXT:    s_or_b32 s10, s8, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b32 s8, s10, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s8, v2
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
 ; GCN-NEXT:    s_ashr_i32 s2, s2, 16
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s2
@@ -2647,15 +2369,15 @@ define amdgpu_kernel void @srem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v1
 ; GCN-NEXT:    s_xor_b32 s8, s0, s2
 ; GCN-NEXT:    s_ashr_i32 s8, s8, 30
-; GCN-NEXT:    s_or_b32 s8, s8, 1
+; GCN-NEXT:    s_or_b32 s10, s8, 1
 ; GCN-NEXT:    v_mul_f32_e32 v3, v2, v3
 ; GCN-NEXT:    v_trunc_f32_e32 v3, v3
 ; GCN-NEXT:    v_mad_f32 v2, -v3, v1, v2
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v3, v3
-; GCN-NEXT:    v_mov_b32_e32 v4, s8
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v1|
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v2|, |v1|
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b32 s8, s10, 0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s8, v3
 ; GCN-NEXT:    v_mul_lo_u32 v1, v1, s2
 ; GCN-NEXT:    s_sext_i32_i16 s2, s3
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s2
@@ -2669,33 +2391,33 @@ define amdgpu_kernel void @srem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; GCN-NEXT:    v_mul_f32_e32 v4, v1, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v4, v4
 ; GCN-NEXT:    v_mad_f32 v1, -v4, v2, v1
-; GCN-NEXT:    v_mov_b32_e32 v5, s0
-; GCN-NEXT:    s_ashr_i32 s0, s3, 16
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v2|
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v2|
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b32 s0, s0, 0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s0, v4
+; GCN-NEXT:    s_ashr_i32 s0, s3, 16
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v5, vcc
-; GCN-NEXT:    s_ashr_i32 s2, s1, 16
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
-; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s2
+; GCN-NEXT:    s_ashr_i32 s8, s1, 16
+; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s8
+; GCN-NEXT:    s_xor_b32 s2, s8, s0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v2
+; GCN-NEXT:    s_ashr_i32 s2, s2, 30
 ; GCN-NEXT:    v_mul_lo_u32 v1, v1, s3
-; GCN-NEXT:    s_xor_b32 s3, s2, s0
-; GCN-NEXT:    s_ashr_i32 s3, s3, 30
+; GCN-NEXT:    s_or_b32 s9, s2, 1
 ; GCN-NEXT:    v_mul_f32_e32 v5, v4, v5
 ; GCN-NEXT:    v_trunc_f32_e32 v5, v5
 ; GCN-NEXT:    v_mad_f32 v4, -v5, v2, v4
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
-; GCN-NEXT:    s_or_b32 s3, s3, 1
-; GCN-NEXT:    v_mov_b32_e32 v6, s3
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, |v2|
-; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[2:3], |v4|, |v2|
+; GCN-NEXT:    s_cmp_lg_u32 s2, 0
+; GCN-NEXT:    s_cselect_b32 s2, s9, 0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s2, v5
 ; GCN-NEXT:    v_mul_lo_u32 v2, v2, s0
 ; GCN-NEXT:    s_mov_b32 s0, 0xffff
 ; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s1, v1
 ; GCN-NEXT:    v_and_b32_e32 v1, s0, v1
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s2, v2
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s8, v2
 ; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GCN-NEXT:    v_or_b32_e32 v1, v1, v2
 ; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
@@ -2727,7 +2449,7 @@ define amdgpu_kernel void @udiv_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP10]], [[TMP14]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP15]], 7
 ; CHECK-NEXT:    [[TMP17:%.*]] = trunc i32 [[TMP16]] to i3
-; CHECK-NEXT:    store i3 [[TMP17]], i3 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i3 [[TMP17]], i3 addrspace(1)* [[OUT:%.*]], align 1
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_i3:
@@ -2777,7 +2499,7 @@ define amdgpu_kernel void @urem_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; CHECK-NEXT:    [[TMP17:%.*]] = sub i32 [[TMP1]], [[TMP16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP17]], 7
 ; CHECK-NEXT:    [[TMP19:%.*]] = trunc i32 [[TMP18]] to i3
-; CHECK-NEXT:    store i3 [[TMP19]], i3 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i3 [[TMP19]], i3 addrspace(1)* [[OUT:%.*]], align 1
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_i3:
@@ -2832,7 +2554,7 @@ define amdgpu_kernel void @sdiv_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; CHECK-NEXT:    [[TMP19:%.*]] = shl i32 [[TMP18]], 29
 ; CHECK-NEXT:    [[TMP20:%.*]] = ashr i32 [[TMP19]], 29
 ; CHECK-NEXT:    [[TMP21:%.*]] = trunc i32 [[TMP20]] to i3
-; CHECK-NEXT:    store i3 [[TMP21]], i3 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i3 [[TMP21]], i3 addrspace(1)* [[OUT:%.*]], align 1
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_i3:
@@ -2849,15 +2571,15 @@ define amdgpu_kernel void @sdiv_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; GCN-NEXT:    s_xor_b32 s0, s0, s1
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-NEXT:    s_or_b32 s0, s0, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s0
+; GCN-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cselect_b32 s0, s2, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
 ; GCN-NEXT:    v_and_b32_e32 v0, 7, v0
 ; GCN-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
@@ -2891,35 +2613,35 @@ define amdgpu_kernel void @srem_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; CHECK-NEXT:    [[TMP21:%.*]] = shl i32 [[TMP20]], 29
 ; CHECK-NEXT:    [[TMP22:%.*]] = ashr i32 [[TMP21]], 29
 ; CHECK-NEXT:    [[TMP23:%.*]] = trunc i32 [[TMP22]] to i3
-; CHECK-NEXT:    store i3 [[TMP23]], i3 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i3 [[TMP23]], i3 addrspace(1)* [[OUT:%.*]], align 1
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_i3:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0xb
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_bfe_i32 s1, s0, 0x30008
-; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s1
-; GCN-NEXT:    s_bfe_i32 s3, s0, 0x30000
-; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s3
-; GCN-NEXT:    s_xor_b32 s1, s3, s1
+; GCN-NEXT:    s_bfe_i32 s0, s2, 0x30008
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s0
+; GCN-NEXT:    s_bfe_i32 s1, s2, 0x30000
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s1
+; GCN-NEXT:    s_xor_b32 s0, s1, s0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-NEXT:    s_ashr_i32 s1, s1, 30
-; GCN-NEXT:    s_or_b32 s1, s1, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_lshr_b32 s3, s2, 8
+; GCN-NEXT:    s_or_b32 s6, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    s_lshr_b32 s2, s0, 8
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cselect_b32 s0, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s3
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
 ; GCN-NEXT:    v_and_b32_e32 v0, 7, v0
 ; GCN-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
@@ -2990,7 +2712,7 @@ define amdgpu_kernel void @udiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP58:%.*]] = and i32 [[TMP57]], 65535
 ; CHECK-NEXT:    [[TMP59:%.*]] = trunc i32 [[TMP58]] to i16
 ; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <3 x i16> [[TMP40]], i16 [[TMP59]], i64 2
-; CHECK-NEXT:    store <3 x i16> [[TMP60]], <3 x i16> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <3 x i16> [[TMP60]], <3 x i16> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_v3i16:
@@ -3114,7 +2836,7 @@ define amdgpu_kernel void @urem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP64:%.*]] = and i32 [[TMP63]], 65535
 ; CHECK-NEXT:    [[TMP65:%.*]] = trunc i32 [[TMP64]] to i16
 ; CHECK-NEXT:    [[TMP66:%.*]] = insertelement <3 x i16> [[TMP44]], i16 [[TMP65]], i64 2
-; CHECK-NEXT:    store <3 x i16> [[TMP66]], <3 x i16> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <3 x i16> [[TMP66]], <3 x i16> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_v3i16:
@@ -3254,7 +2976,7 @@ define amdgpu_kernel void @sdiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP70:%.*]] = ashr i32 [[TMP69]], 16
 ; CHECK-NEXT:    [[TMP71:%.*]] = trunc i32 [[TMP70]] to i16
 ; CHECK-NEXT:    [[TMP72:%.*]] = insertelement <3 x i16> [[TMP48]], i16 [[TMP71]], i64 2
-; CHECK-NEXT:    store <3 x i16> [[TMP72]], <3 x i16> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <3 x i16> [[TMP72]], <3 x i16> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_v3i16:
@@ -3270,54 +2992,54 @@ define amdgpu_kernel void @sdiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s8
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s9
 ; GCN-NEXT:    s_xor_b32 s8, s9, s8
-; GCN-NEXT:    s_ashr_i32 s0, s0, 16
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_ashr_i32 s8, s8, 30
-; GCN-NEXT:    s_or_b32 s8, s8, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s8
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    s_or_b32 s10, s8, 1
+; GCN-NEXT:    s_sext_i32_i16 s1, s1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b32 s8, s10, 0
+; GCN-NEXT:    s_ashr_i32 s0, s0, 16
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s0
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s0
 ; GCN-NEXT:    s_ashr_i32 s2, s2, 16
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s2
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v1
 ; GCN-NEXT:    s_xor_b32 s0, s2, s0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s8, v2
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v0
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
 ; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GCN-NEXT:    v_mul_f32_e32 v3, v2, v3
 ; GCN-NEXT:    v_trunc_f32_e32 v3, v3
-; GCN-NEXT:    v_mad_f32 v2, -v3, v1, v2
-; GCN-NEXT:    v_mov_b32_e32 v4, s0
-; GCN-NEXT:    s_sext_i32_i16 s0, s1
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v1|
+; GCN-NEXT:    v_mad_f32 v2, -v3, v0, v2
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v3, v3
-; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
-; GCN-NEXT:    s_sext_i32_i16 s1, s3
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
-; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s1
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
-; GCN-NEXT:    s_xor_b32 s0, s1, s0
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v2|, |v0|
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s1
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b32 s0, s0, 0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s0, v3
+; GCN-NEXT:    s_sext_i32_i16 s0, s3
+; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v0
+; GCN-NEXT:    s_xor_b32 s0, s0, s1
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v4, v3, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v4, v4
-; GCN-NEXT:    v_mad_f32 v3, -v4, v2, v3
+; GCN-NEXT:    v_mad_f32 v3, -v4, v0, v3
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GCN-NEXT:    v_mov_b32_e32 v5, s0
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v5, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
-; GCN-NEXT:    buffer_store_short v2, off, s[4:7], 0 offset:4
-; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v3|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cselect_b32 s0, s2, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v4
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v2
+; GCN-NEXT:    buffer_store_short v0, off, s[4:7], 0 offset:4
+; GCN-NEXT:    buffer_store_dword v1, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = sdiv <3 x i16> %x, %y
   store <3 x i16> %r, <3 x i16> addrspace(1)* %out
@@ -3404,7 +3126,7 @@ define amdgpu_kernel void @srem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP76:%.*]] = ashr i32 [[TMP75]], 16
 ; CHECK-NEXT:    [[TMP77:%.*]] = trunc i32 [[TMP76]] to i16
 ; CHECK-NEXT:    [[TMP78:%.*]] = insertelement <3 x i16> [[TMP52]], i16 [[TMP77]], i64 2
-; CHECK-NEXT:    store <3 x i16> [[TMP78]], <3 x i16> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <3 x i16> [[TMP78]], <3 x i16> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_v3i16:
@@ -3422,20 +3144,19 @@ define amdgpu_kernel void @srem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; GCN-NEXT:    s_ashr_i32 s6, s6, 30
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_or_b32 s6, s6, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s6
-; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_mov_b32_e32 v1, s2
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b32 s6, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s6, v2
 ; GCN-NEXT:    v_mov_b32_e32 v2, s0
 ; GCN-NEXT:    v_alignbit_b32 v2, s1, v2, 16
 ; GCN-NEXT:    v_bfe_i32 v3, v2, 0, 16
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v4, v3
+; GCN-NEXT:    v_mov_b32_e32 v1, s2
 ; GCN-NEXT:    v_alignbit_b32 v1, s3, v1, 16
 ; GCN-NEXT:    v_bfe_i32 v5, v1, 0, 16
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v6, v5
@@ -3453,7 +3174,7 @@ define amdgpu_kernel void @srem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s0
 ; GCN-NEXT:    v_or_b32_e32 v3, 1, v3
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
 ; GCN-NEXT:    s_sext_i32_i16 s2, s3
 ; GCN-NEXT:    v_mul_lo_u32 v2, v3, v2
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s2
@@ -3465,12 +3186,13 @@ define amdgpu_kernel void @srem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; GCN-NEXT:    v_trunc_f32_e32 v5, v5
 ; GCN-NEXT:    v_mad_f32 v3, -v5, v4, v3
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
-; GCN-NEXT:    v_mov_b32_e32 v6, s0
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v4|
-; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v3|, |v4|
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b32 s0, s0, 0
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s0, v5
 ; GCN-NEXT:    v_mul_lo_u32 v3, v3, s1
 ; GCN-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s3, v3
@@ -3545,7 +3267,7 @@ define amdgpu_kernel void @udiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP58:%.*]] = and i32 [[TMP57]], 32767
 ; CHECK-NEXT:    [[TMP59:%.*]] = trunc i32 [[TMP58]] to i15
 ; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <3 x i15> [[TMP40]], i15 [[TMP59]], i64 2
-; CHECK-NEXT:    store <3 x i15> [[TMP60]], <3 x i15> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <3 x i15> [[TMP60]], <3 x i15> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_v3i15:
@@ -3677,7 +3399,7 @@ define amdgpu_kernel void @urem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP64:%.*]] = and i32 [[TMP63]], 32767
 ; CHECK-NEXT:    [[TMP65:%.*]] = trunc i32 [[TMP64]] to i15
 ; CHECK-NEXT:    [[TMP66:%.*]] = insertelement <3 x i15> [[TMP44]], i15 [[TMP65]], i64 2
-; CHECK-NEXT:    store <3 x i15> [[TMP66]], <3 x i15> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <3 x i15> [[TMP66]], <3 x i15> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_v3i15:
@@ -3823,7 +3545,7 @@ define amdgpu_kernel void @sdiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP70:%.*]] = ashr i32 [[TMP69]], 17
 ; CHECK-NEXT:    [[TMP71:%.*]] = trunc i32 [[TMP70]] to i15
 ; CHECK-NEXT:    [[TMP72:%.*]] = insertelement <3 x i15> [[TMP48]], i15 [[TMP71]], i64 2
-; CHECK-NEXT:    store <3 x i15> [[TMP72]], <3 x i15> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <3 x i15> [[TMP72]], <3 x i15> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_v3i15:
@@ -3844,51 +3566,51 @@ define amdgpu_kernel void @sdiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s1
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
 ; GCN-NEXT:    s_xor_b32 s1, s1, s3
-; GCN-NEXT:    s_bfe_i32 s0, s0, 0xf000f
 ; GCN-NEXT:    s_ashr_i32 s1, s1, 30
+; GCN-NEXT:    s_or_b32 s1, s1, 1
 ; GCN-NEXT:    v_mul_f32_e32 v4, v3, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v4, v4
 ; GCN-NEXT:    v_mad_f32 v3, -v4, v2, v3
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v3|, |v2|
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b32 s1, s1, 0
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s0
-; GCN-NEXT:    s_or_b32 s1, s1, 1
-; GCN-NEXT:    v_mov_b32_e32 v5, s1
-; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v5, vcc
+; GCN-NEXT:    s_bfe_i32 s0, s0, 0xf000f
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
+; GCN-NEXT:    v_bfe_i32 v1, v1, 0, 15
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s1, v4
 ; GCN-NEXT:    s_bfe_i32 s1, s2, 0xf000f
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s1
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v3
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v2
 ; GCN-NEXT:    s_xor_b32 s0, s1, s0
-; GCN-NEXT:    v_bfe_i32 v1, v1, 0, 15
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v5, v4, v5
 ; GCN-NEXT:    v_trunc_f32_e32 v5, v5
-; GCN-NEXT:    v_mad_f32 v4, -v5, v3, v4
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, |v3|
+; GCN-NEXT:    v_mad_f32 v4, -v5, v2, v4
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v4|, |v2|
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
-; GCN-NEXT:    v_cvt_f32_i32_e32 v4, v1
-; GCN-NEXT:    s_or_b32 s0, s0, 1
-; GCN-NEXT:    v_mov_b32_e32 v6, s0
-; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v6, vcc
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cselect_b32 s0, s2, 0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 15
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s0, v5
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v5, v0
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v6, v4
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v6, v2
 ; GCN-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; GCN-NEXT:    v_ashrrev_i32_e32 v0, 30, v0
 ; GCN-NEXT:    v_or_b32_e32 v0, 1, v0
 ; GCN-NEXT:    v_mul_f32_e32 v1, v5, v6
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
-; GCN-NEXT:    v_mad_f32 v5, -v1, v4, v5
+; GCN-NEXT:    v_mad_f32 v5, -v1, v2, v5
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v5|, |v4|
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v5|, |v2|
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
 ; GCN-NEXT:    s_movk_i32 s0, 0x7fff
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-NEXT:    v_and_b32_e32 v3, s0, v3
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_and_b32_e32 v2, s0, v3
+; GCN-NEXT:    v_and_b32_e32 v3, s0, v4
 ; GCN-NEXT:    v_lshl_b64 v[0:1], v[0:1], 30
-; GCN-NEXT:    v_and_b32_e32 v2, s0, v2
 ; GCN-NEXT:    v_lshlrev_b32_e32 v3, 15, v3
 ; GCN-NEXT:    v_or_b32_e32 v2, v2, v3
 ; GCN-NEXT:    v_or_b32_e32 v0, v2, v0
@@ -3981,7 +3703,7 @@ define amdgpu_kernel void @srem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP76:%.*]] = ashr i32 [[TMP75]], 17
 ; CHECK-NEXT:    [[TMP77:%.*]] = trunc i32 [[TMP76]] to i15
 ; CHECK-NEXT:    [[TMP78:%.*]] = insertelement <3 x i15> [[TMP52]], i15 [[TMP77]], i64 2
-; CHECK-NEXT:    store <3 x i15> [[TMP78]], <3 x i15> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <3 x i15> [[TMP78]], <3 x i15> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_v3i15:
@@ -3993,52 +3715,52 @@ define amdgpu_kernel void @srem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_mov_b32_e32 v1, s0
 ; GCN-NEXT:    v_alignbit_b32 v0, s3, v0, 30
 ; GCN-NEXT:    s_movk_i32 s3, 0x7fff
-; GCN-NEXT:    s_and_b32 s11, s0, s3
-; GCN-NEXT:    s_bfe_i32 s11, s11, 0xf0000
-; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s11
-; GCN-NEXT:    s_and_b32 s9, s2, s3
-; GCN-NEXT:    s_bfe_i32 s9, s9, 0xf0000
-; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s9
+; GCN-NEXT:    v_alignbit_b32 v1, s1, v1, 30
+; GCN-NEXT:    s_and_b32 s1, s0, s3
+; GCN-NEXT:    s_bfe_i32 s1, s1, 0xf0000
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s1
+; GCN-NEXT:    s_and_b32 s8, s2, s3
+; GCN-NEXT:    s_bfe_i32 s8, s8, 0xf0000
+; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s8
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
-; GCN-NEXT:    s_xor_b32 s9, s9, s11
-; GCN-NEXT:    s_ashr_i32 s9, s9, 30
-; GCN-NEXT:    s_or_b32 s9, s9, 1
+; GCN-NEXT:    s_xor_b32 s1, s8, s1
+; GCN-NEXT:    s_ashr_i32 s1, s1, 30
+; GCN-NEXT:    s_lshr_b32 s10, s2, 15
 ; GCN-NEXT:    v_mul_f32_e32 v4, v3, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v4, v4
 ; GCN-NEXT:    v_mad_f32 v3, -v4, v2, v3
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GCN-NEXT:    v_mov_b32_e32 v5, s9
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v5, vcc
-; GCN-NEXT:    v_mov_b32_e32 v1, s0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    s_bfe_u32 s12, s0, 0xf000f
-; GCN-NEXT:    v_alignbit_b32 v1, s1, v1, 30
+; GCN-NEXT:    s_bfe_u32 s11, s2, 0xf000f
+; GCN-NEXT:    s_lshr_b32 s12, s0, 15
+; GCN-NEXT:    s_bfe_u32 s13, s0, 0xf000f
+; GCN-NEXT:    s_or_b32 s1, s1, 1
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v3|, |v2|
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s1, v4
 ; GCN-NEXT:    v_mul_lo_u32 v2, v2, s0
-; GCN-NEXT:    s_lshr_b32 s1, s0, 15
-; GCN-NEXT:    s_bfe_i32 s0, s12, 0xf0000
+; GCN-NEXT:    s_bfe_i32 s0, s13, 0xf0000
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s0
-; GCN-NEXT:    s_bfe_u32 s10, s2, 0xf000f
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s2, v2
-; GCN-NEXT:    s_lshr_b32 s8, s2, 15
-; GCN-NEXT:    s_bfe_i32 s2, s10, 0xf0000
-; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s2
+; GCN-NEXT:    s_bfe_i32 s1, s11, 0xf0000
+; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s1
+; GCN-NEXT:    s_xor_b32 s0, s1, s0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v3
-; GCN-NEXT:    s_xor_b32 s0, s2, s0
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s2, v2
+; GCN-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v5, v4, v5
 ; GCN-NEXT:    v_trunc_f32_e32 v5, v5
 ; GCN-NEXT:    v_mad_f32 v4, -v5, v3, v4
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v4|, |v3|
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
 ; GCN-NEXT:    v_and_b32_e32 v1, s3, v1
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, |v3|
-; GCN-NEXT:    v_mov_b32_e32 v6, s0
-; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v6, vcc
+; GCN-NEXT:    s_cselect_b32 s0, s2, 0
 ; GCN-NEXT:    v_bfe_i32 v4, v1, 0, 15
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s0, v5
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v5, v4
 ; GCN-NEXT:    v_and_b32_e32 v0, s3, v0
 ; GCN-NEXT:    v_bfe_i32 v6, v0, 0, 15
@@ -4053,11 +3775,11 @@ define amdgpu_kernel void @srem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v6, v6
 ; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v7|, |v5|
 ; GCN-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GCN-NEXT:    v_mul_lo_u32 v3, v3, s1
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GCN-NEXT:    v_mul_lo_u32 v3, v3, s12
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
 ; GCN-NEXT:    v_mul_lo_u32 v1, v4, v1
 ; GCN-NEXT:    v_and_b32_e32 v2, s3, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s8, v3
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s10, v3
 ; GCN-NEXT:    v_and_b32_e32 v3, s3, v3
 ; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
 ; GCN-NEXT:    v_lshl_b64 v[0:1], v[0:1], 30
@@ -4076,7 +3798,7 @@ define amdgpu_kernel void @srem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 define amdgpu_kernel void @udiv_i32_oddk_denom(i32 addrspace(1)* %out, i32 %x) {
 ; CHECK-LABEL: @udiv_i32_oddk_denom(
 ; CHECK-NEXT:    [[R:%.*]] = udiv i32 [[X:%.*]], 1235195
-; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_i32_oddk_denom:
@@ -4102,7 +3824,7 @@ define amdgpu_kernel void @udiv_i32_oddk_denom(i32 addrspace(1)* %out, i32 %x) {
 define amdgpu_kernel void @udiv_i32_pow2k_denom(i32 addrspace(1)* %out, i32 %x) {
 ; CHECK-LABEL: @udiv_i32_pow2k_denom(
 ; CHECK-NEXT:    [[R:%.*]] = udiv i32 [[X:%.*]], 4096
-; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_i32_pow2k_denom:
@@ -4125,7 +3847,7 @@ define amdgpu_kernel void @udiv_i32_pow2_shl_denom(i32 addrspace(1)* %out, i32 %
 ; CHECK-LABEL: @udiv_i32_pow2_shl_denom(
 ; CHECK-NEXT:    [[SHL_Y:%.*]] = shl i32 4096, [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = udiv i32 [[X:%.*]], [[SHL_Y]]
-; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_i32_pow2_shl_denom:
@@ -4154,7 +3876,7 @@ define amdgpu_kernel void @udiv_v2i32_pow2k_denom(<2 x i32> addrspace(1)* %out,
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[X]], i64 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = udiv i32 [[TMP4]], 4096
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP5]], i64 1
-; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_v2i32_pow2k_denom:
@@ -4183,7 +3905,7 @@ define amdgpu_kernel void @udiv_v2i32_mixed_pow2k_denom(<2 x i32> addrspace(1)*
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[X]], i64 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = udiv i32 [[TMP4]], 4095
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP5]], i64 1
-; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_v2i32_mixed_pow2k_denom:
@@ -4215,144 +3937,120 @@ define amdgpu_kernel void @udiv_v2i32_pow2_shl_denom(<2 x i32> addrspace(1)* %ou
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = uitofp i32 [[TMP2]] to float
 ; CHECK-NEXT:    [[TMP4:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[TMP4]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[TMP4]], 0x41EFFFFFC0000000
 ; CHECK-NEXT:    [[TMP6:%.*]] = fptoui float [[TMP5]] to i32
-; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
-; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP2]] to i64
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; CHECK-NEXT:    [[TMP11:%.*]] = lshr i64 [[TMP9]], 32
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i32 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP9]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP11]] to i32
-; CHECK-NEXT:    [[TMP13:%.*]] = sub i32 0, [[TMP10]]
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP12]], 0
-; CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP13]], i32 [[TMP10]]
-; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP15]] to i64
-; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = lshr i64 [[TMP11]], 32
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP6]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP15]] to i64
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP16]], [[TMP17]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
 ; CHECK-NEXT:    [[TMP20:%.*]] = lshr i64 [[TMP18]], 32
 ; CHECK-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
-; CHECK-NEXT:    [[TMP22:%.*]] = add i32 [[TMP6]], [[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = sub i32 [[TMP6]], [[TMP21]]
-; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP14]], i32 [[TMP22]], i32 [[TMP23]]
-; CHECK-NEXT:    [[TMP25:%.*]] = zext i32 [[TMP24]] to i64
-; CHECK-NEXT:    [[TMP26:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP25]], [[TMP26]]
-; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i32
-; CHECK-NEXT:    [[TMP29:%.*]] = lshr i64 [[TMP27]], 32
-; CHECK-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP29]] to i32
-; CHECK-NEXT:    [[TMP31:%.*]] = mul i32 [[TMP30]], [[TMP2]]
-; CHECK-NEXT:    [[TMP32:%.*]] = sub i32 [[TMP1]], [[TMP31]]
-; CHECK-NEXT:    [[TMP33:%.*]] = icmp uge i32 [[TMP32]], [[TMP2]]
-; CHECK-NEXT:    [[TMP34:%.*]] = icmp uge i32 [[TMP1]], [[TMP31]]
-; CHECK-NEXT:    [[TMP35:%.*]] = and i1 [[TMP33]], [[TMP34]]
-; CHECK-NEXT:    [[TMP36:%.*]] = add i32 [[TMP30]], 1
-; CHECK-NEXT:    [[TMP37:%.*]] = sub i32 [[TMP30]], 1
-; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP35]], i32 [[TMP36]], i32 [[TMP30]]
-; CHECK-NEXT:    [[TMP39:%.*]] = select i1 [[TMP34]], i32 [[TMP38]], i32 [[TMP37]]
-; CHECK-NEXT:    [[TMP40:%.*]] = insertelement <2 x i32> undef, i32 [[TMP39]], i64 0
-; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <2 x i32> [[X]], i64 1
-; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 1
-; CHECK-NEXT:    [[TMP43:%.*]] = uitofp i32 [[TMP42]] to float
-; CHECK-NEXT:    [[TMP44:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP43]])
-; CHECK-NEXT:    [[TMP45:%.*]] = fmul fast float [[TMP44]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP46:%.*]] = fptoui float [[TMP45]] to i32
-; CHECK-NEXT:    [[TMP47:%.*]] = zext i32 [[TMP46]] to i64
-; CHECK-NEXT:    [[TMP48:%.*]] = zext i32 [[TMP42]] to i64
-; CHECK-NEXT:    [[TMP49:%.*]] = mul i64 [[TMP47]], [[TMP48]]
-; CHECK-NEXT:    [[TMP50:%.*]] = trunc i64 [[TMP49]] to i32
-; CHECK-NEXT:    [[TMP51:%.*]] = lshr i64 [[TMP49]], 32
-; CHECK-NEXT:    [[TMP52:%.*]] = trunc i64 [[TMP51]] to i32
-; CHECK-NEXT:    [[TMP53:%.*]] = sub i32 0, [[TMP50]]
-; CHECK-NEXT:    [[TMP54:%.*]] = icmp eq i32 [[TMP52]], 0
-; CHECK-NEXT:    [[TMP55:%.*]] = select i1 [[TMP54]], i32 [[TMP53]], i32 [[TMP50]]
-; CHECK-NEXT:    [[TMP56:%.*]] = zext i32 [[TMP55]] to i64
-; CHECK-NEXT:    [[TMP57:%.*]] = zext i32 [[TMP46]] to i64
-; CHECK-NEXT:    [[TMP58:%.*]] = mul i64 [[TMP56]], [[TMP57]]
-; CHECK-NEXT:    [[TMP59:%.*]] = trunc i64 [[TMP58]] to i32
-; CHECK-NEXT:    [[TMP60:%.*]] = lshr i64 [[TMP58]], 32
-; CHECK-NEXT:    [[TMP61:%.*]] = trunc i64 [[TMP60]] to i32
-; CHECK-NEXT:    [[TMP62:%.*]] = add i32 [[TMP46]], [[TMP61]]
-; CHECK-NEXT:    [[TMP63:%.*]] = sub i32 [[TMP46]], [[TMP61]]
-; CHECK-NEXT:    [[TMP64:%.*]] = select i1 [[TMP54]], i32 [[TMP62]], i32 [[TMP63]]
-; CHECK-NEXT:    [[TMP65:%.*]] = zext i32 [[TMP64]] to i64
-; CHECK-NEXT:    [[TMP66:%.*]] = zext i32 [[TMP41]] to i64
-; CHECK-NEXT:    [[TMP67:%.*]] = mul i64 [[TMP65]], [[TMP66]]
-; CHECK-NEXT:    [[TMP68:%.*]] = trunc i64 [[TMP67]] to i32
-; CHECK-NEXT:    [[TMP69:%.*]] = lshr i64 [[TMP67]], 32
-; CHECK-NEXT:    [[TMP70:%.*]] = trunc i64 [[TMP69]] to i32
-; CHECK-NEXT:    [[TMP71:%.*]] = mul i32 [[TMP70]], [[TMP42]]
-; CHECK-NEXT:    [[TMP72:%.*]] = sub i32 [[TMP41]], [[TMP71]]
-; CHECK-NEXT:    [[TMP73:%.*]] = icmp uge i32 [[TMP72]], [[TMP42]]
-; CHECK-NEXT:    [[TMP74:%.*]] = icmp uge i32 [[TMP41]], [[TMP71]]
-; CHECK-NEXT:    [[TMP75:%.*]] = and i1 [[TMP73]], [[TMP74]]
-; CHECK-NEXT:    [[TMP76:%.*]] = add i32 [[TMP70]], 1
-; CHECK-NEXT:    [[TMP77:%.*]] = sub i32 [[TMP70]], 1
-; CHECK-NEXT:    [[TMP78:%.*]] = select i1 [[TMP75]], i32 [[TMP76]], i32 [[TMP70]]
-; CHECK-NEXT:    [[TMP79:%.*]] = select i1 [[TMP74]], i32 [[TMP78]], i32 [[TMP77]]
-; CHECK-NEXT:    [[TMP80:%.*]] = insertelement <2 x i32> [[TMP40]], i32 [[TMP79]], i64 1
-; CHECK-NEXT:    store <2 x i32> [[TMP80]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP22:%.*]] = mul i32 [[TMP21]], [[TMP2]]
+; CHECK-NEXT:    [[TMP23:%.*]] = sub i32 [[TMP1]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp uge i32 [[TMP23]], [[TMP2]]
+; CHECK-NEXT:    [[TMP25:%.*]] = add i32 [[TMP21]], 1
+; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP24]], i32 [[TMP25]], i32 [[TMP21]]
+; CHECK-NEXT:    [[TMP27:%.*]] = sub i32 [[TMP23]], [[TMP2]]
+; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP24]], i32 [[TMP27]], i32 [[TMP23]]
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp uge i32 [[TMP28]], [[TMP2]]
+; CHECK-NEXT:    [[TMP30:%.*]] = add i32 [[TMP26]], 1
+; CHECK-NEXT:    [[TMP31:%.*]] = select i1 [[TMP29]], i32 [[TMP30]], i32 [[TMP26]]
+; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <2 x i32> undef, i32 [[TMP31]], i64 0
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <2 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 1
+; CHECK-NEXT:    [[TMP35:%.*]] = uitofp i32 [[TMP34]] to float
+; CHECK-NEXT:    [[TMP36:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP35]])
+; CHECK-NEXT:    [[TMP37:%.*]] = fmul fast float [[TMP36]], 0x41EFFFFFC0000000
+; CHECK-NEXT:    [[TMP38:%.*]] = fptoui float [[TMP37]] to i32
+; CHECK-NEXT:    [[TMP39:%.*]] = sub i32 0, [[TMP34]]
+; CHECK-NEXT:    [[TMP40:%.*]] = mul i32 [[TMP39]], [[TMP38]]
+; CHECK-NEXT:    [[TMP41:%.*]] = zext i32 [[TMP38]] to i64
+; CHECK-NEXT:    [[TMP42:%.*]] = zext i32 [[TMP40]] to i64
+; CHECK-NEXT:    [[TMP43:%.*]] = mul i64 [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP44:%.*]] = trunc i64 [[TMP43]] to i32
+; CHECK-NEXT:    [[TMP45:%.*]] = lshr i64 [[TMP43]], 32
+; CHECK-NEXT:    [[TMP46:%.*]] = trunc i64 [[TMP45]] to i32
+; CHECK-NEXT:    [[TMP47:%.*]] = add i32 [[TMP38]], [[TMP46]]
+; CHECK-NEXT:    [[TMP48:%.*]] = zext i32 [[TMP33]] to i64
+; CHECK-NEXT:    [[TMP49:%.*]] = zext i32 [[TMP47]] to i64
+; CHECK-NEXT:    [[TMP50:%.*]] = mul i64 [[TMP48]], [[TMP49]]
+; CHECK-NEXT:    [[TMP51:%.*]] = trunc i64 [[TMP50]] to i32
+; CHECK-NEXT:    [[TMP52:%.*]] = lshr i64 [[TMP50]], 32
+; CHECK-NEXT:    [[TMP53:%.*]] = trunc i64 [[TMP52]] to i32
+; CHECK-NEXT:    [[TMP54:%.*]] = mul i32 [[TMP53]], [[TMP34]]
+; CHECK-NEXT:    [[TMP55:%.*]] = sub i32 [[TMP33]], [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = icmp uge i32 [[TMP55]], [[TMP34]]
+; CHECK-NEXT:    [[TMP57:%.*]] = add i32 [[TMP53]], 1
+; CHECK-NEXT:    [[TMP58:%.*]] = select i1 [[TMP56]], i32 [[TMP57]], i32 [[TMP53]]
+; CHECK-NEXT:    [[TMP59:%.*]] = sub i32 [[TMP55]], [[TMP34]]
+; CHECK-NEXT:    [[TMP60:%.*]] = select i1 [[TMP56]], i32 [[TMP59]], i32 [[TMP55]]
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp uge i32 [[TMP60]], [[TMP34]]
+; CHECK-NEXT:    [[TMP62:%.*]] = add i32 [[TMP58]], 1
+; CHECK-NEXT:    [[TMP63:%.*]] = select i1 [[TMP61]], i32 [[TMP62]], i32 [[TMP58]]
+; CHECK-NEXT:    [[TMP64:%.*]] = insertelement <2 x i32> [[TMP32]], i32 [[TMP63]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP64]], <2 x i32> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_v2i32_pow2_shl_denom:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
 ; GCN-NEXT:    s_movk_i32 s4, 0x1000
+; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xb
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_lshl_b32 s2, s4, s2
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GCN-NEXT:    s_lshl_b32 s5, s4, s2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s5
 ; GCN-NEXT:    s_lshl_b32 s10, s4, s3
-; GCN-NEXT:    s_mov_b32 s3, 0x4f800000
+; GCN-NEXT:    s_mov_b32 s3, 0x4f7ffffe
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s10
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xb
+; GCN-NEXT:    s_sub_i32 s2, 0, s5
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v1
 ; GCN-NEXT:    v_mul_f32_e32 v0, s3, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_mul_f32_e32 v1, s3, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    v_mul_lo_u32 v2, v0, s2
-; GCN-NEXT:    v_mul_hi_u32 v3, v0, s2
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v2
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v4, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v2, v2, v0
-; GCN-NEXT:    v_mul_lo_u32 v3, v1, s10
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v2, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v2, v0
-; GCN-NEXT:    v_mul_hi_u32 v2, v1, s10
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[0:1]
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v3
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_mul_hi_u32 v0, v0, s8
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v2, v2, v1
-; GCN-NEXT:    v_mul_lo_u32 v5, v0, s2
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v2, v1
-; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v2, v1
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v4, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, s9
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s8, v5
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s2, v3
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, -1, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, v1, s10
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], s8, v5
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
-; GCN-NEXT:    s_and_b64 vcc, s[2:3], s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s9, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v3, v0, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v2, s2, v0
+; GCN-NEXT:    s_sub_i32 s2, 0, s10
+; GCN-NEXT:    v_mul_lo_u32 v3, s2, v1
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s8, v0
+; GCN-NEXT:    v_mul_hi_u32 v2, v1, v3
+; GCN-NEXT:    v_mul_lo_u32 v3, v0, s5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s8, v3
+; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s5, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, s5, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v4, s[2:3]
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s5, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GCN-NEXT:    v_mul_hi_u32 v1, s9, v1
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, s10
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s9, v2
 ; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s10, v2
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, -1, v1
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s9, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s10, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s10, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[2:3]
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
@@ -4364,7 +4062,7 @@ define amdgpu_kernel void @udiv_v2i32_pow2_shl_denom(<2 x i32> addrspace(1)* %ou
 define amdgpu_kernel void @urem_i32_oddk_denom(i32 addrspace(1)* %out, i32 %x) {
 ; CHECK-LABEL: @urem_i32_oddk_denom(
 ; CHECK-NEXT:    [[R:%.*]] = urem i32 [[X:%.*]], 1235195
-; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_i32_oddk_denom:
@@ -4392,7 +4090,7 @@ define amdgpu_kernel void @urem_i32_oddk_denom(i32 addrspace(1)* %out, i32 %x) {
 define amdgpu_kernel void @urem_i32_pow2k_denom(i32 addrspace(1)* %out, i32 %x) {
 ; CHECK-LABEL: @urem_i32_pow2k_denom(
 ; CHECK-NEXT:    [[R:%.*]] = urem i32 [[X:%.*]], 4096
-; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_i32_pow2k_denom:
@@ -4415,7 +4113,7 @@ define amdgpu_kernel void @urem_i32_pow2_shl_denom(i32 addrspace(1)* %out, i32 %
 ; CHECK-LABEL: @urem_i32_pow2_shl_denom(
 ; CHECK-NEXT:    [[SHL_Y:%.*]] = shl i32 4096, [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = urem i32 [[X:%.*]], [[SHL_Y]]
-; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_i32_pow2_shl_denom:
@@ -4445,7 +4143,7 @@ define amdgpu_kernel void @urem_v2i32_pow2k_denom(<2 x i32> addrspace(1)* %out,
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[X]], i64 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = urem i32 [[TMP4]], 4096
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP5]], i64 1
-; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_v2i32_pow2k_denom:
@@ -4474,145 +4172,113 @@ define amdgpu_kernel void @urem_v2i32_pow2_shl_denom(<2 x i32> addrspace(1)* %ou
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = uitofp i32 [[TMP2]] to float
 ; CHECK-NEXT:    [[TMP4:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[TMP4]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[TMP4]], 0x41EFFFFFC0000000
 ; CHECK-NEXT:    [[TMP6:%.*]] = fptoui float [[TMP5]] to i32
-; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
-; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP2]] to i64
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; CHECK-NEXT:    [[TMP11:%.*]] = lshr i64 [[TMP9]], 32
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i32 0, [[TMP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP9]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP11]] to i32
-; CHECK-NEXT:    [[TMP13:%.*]] = sub i32 0, [[TMP10]]
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP12]], 0
-; CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP13]], i32 [[TMP10]]
-; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP15]] to i64
-; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = lshr i64 [[TMP11]], 32
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP6]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP15]] to i64
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP16]], [[TMP17]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
 ; CHECK-NEXT:    [[TMP20:%.*]] = lshr i64 [[TMP18]], 32
 ; CHECK-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
-; CHECK-NEXT:    [[TMP22:%.*]] = add i32 [[TMP6]], [[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = sub i32 [[TMP6]], [[TMP21]]
-; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP14]], i32 [[TMP22]], i32 [[TMP23]]
-; CHECK-NEXT:    [[TMP25:%.*]] = zext i32 [[TMP24]] to i64
-; CHECK-NEXT:    [[TMP26:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP25]], [[TMP26]]
-; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i32
-; CHECK-NEXT:    [[TMP29:%.*]] = lshr i64 [[TMP27]], 32
-; CHECK-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP29]] to i32
-; CHECK-NEXT:    [[TMP31:%.*]] = mul i32 [[TMP30]], [[TMP2]]
-; CHECK-NEXT:    [[TMP32:%.*]] = sub i32 [[TMP1]], [[TMP31]]
-; CHECK-NEXT:    [[TMP33:%.*]] = icmp uge i32 [[TMP32]], [[TMP2]]
-; CHECK-NEXT:    [[TMP34:%.*]] = icmp uge i32 [[TMP1]], [[TMP31]]
-; CHECK-NEXT:    [[TMP35:%.*]] = and i1 [[TMP33]], [[TMP34]]
-; CHECK-NEXT:    [[TMP36:%.*]] = sub i32 [[TMP32]], [[TMP2]]
-; CHECK-NEXT:    [[TMP37:%.*]] = add i32 [[TMP32]], [[TMP2]]
-; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP35]], i32 [[TMP36]], i32 [[TMP32]]
-; CHECK-NEXT:    [[TMP39:%.*]] = select i1 [[TMP34]], i32 [[TMP38]], i32 [[TMP37]]
-; CHECK-NEXT:    [[TMP40:%.*]] = insertelement <2 x i32> undef, i32 [[TMP39]], i64 0
-; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <2 x i32> [[X]], i64 1
-; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 1
-; CHECK-NEXT:    [[TMP43:%.*]] = uitofp i32 [[TMP42]] to float
-; CHECK-NEXT:    [[TMP44:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP43]])
-; CHECK-NEXT:    [[TMP45:%.*]] = fmul fast float [[TMP44]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP46:%.*]] = fptoui float [[TMP45]] to i32
-; CHECK-NEXT:    [[TMP47:%.*]] = zext i32 [[TMP46]] to i64
-; CHECK-NEXT:    [[TMP48:%.*]] = zext i32 [[TMP42]] to i64
-; CHECK-NEXT:    [[TMP49:%.*]] = mul i64 [[TMP47]], [[TMP48]]
-; CHECK-NEXT:    [[TMP50:%.*]] = trunc i64 [[TMP49]] to i32
-; CHECK-NEXT:    [[TMP51:%.*]] = lshr i64 [[TMP49]], 32
-; CHECK-NEXT:    [[TMP52:%.*]] = trunc i64 [[TMP51]] to i32
-; CHECK-NEXT:    [[TMP53:%.*]] = sub i32 0, [[TMP50]]
-; CHECK-NEXT:    [[TMP54:%.*]] = icmp eq i32 [[TMP52]], 0
-; CHECK-NEXT:    [[TMP55:%.*]] = select i1 [[TMP54]], i32 [[TMP53]], i32 [[TMP50]]
-; CHECK-NEXT:    [[TMP56:%.*]] = zext i32 [[TMP55]] to i64
-; CHECK-NEXT:    [[TMP57:%.*]] = zext i32 [[TMP46]] to i64
-; CHECK-NEXT:    [[TMP58:%.*]] = mul i64 [[TMP56]], [[TMP57]]
-; CHECK-NEXT:    [[TMP59:%.*]] = trunc i64 [[TMP58]] to i32
-; CHECK-NEXT:    [[TMP60:%.*]] = lshr i64 [[TMP58]], 32
-; CHECK-NEXT:    [[TMP61:%.*]] = trunc i64 [[TMP60]] to i32
-; CHECK-NEXT:    [[TMP62:%.*]] = add i32 [[TMP46]], [[TMP61]]
-; CHECK-NEXT:    [[TMP63:%.*]] = sub i32 [[TMP46]], [[TMP61]]
-; CHECK-NEXT:    [[TMP64:%.*]] = select i1 [[TMP54]], i32 [[TMP62]], i32 [[TMP63]]
-; CHECK-NEXT:    [[TMP65:%.*]] = zext i32 [[TMP64]] to i64
-; CHECK-NEXT:    [[TMP66:%.*]] = zext i32 [[TMP41]] to i64
-; CHECK-NEXT:    [[TMP67:%.*]] = mul i64 [[TMP65]], [[TMP66]]
-; CHECK-NEXT:    [[TMP68:%.*]] = trunc i64 [[TMP67]] to i32
-; CHECK-NEXT:    [[TMP69:%.*]] = lshr i64 [[TMP67]], 32
-; CHECK-NEXT:    [[TMP70:%.*]] = trunc i64 [[TMP69]] to i32
-; CHECK-NEXT:    [[TMP71:%.*]] = mul i32 [[TMP70]], [[TMP42]]
-; CHECK-NEXT:    [[TMP72:%.*]] = sub i32 [[TMP41]], [[TMP71]]
-; CHECK-NEXT:    [[TMP73:%.*]] = icmp uge i32 [[TMP72]], [[TMP42]]
-; CHECK-NEXT:    [[TMP74:%.*]] = icmp uge i32 [[TMP41]], [[TMP71]]
-; CHECK-NEXT:    [[TMP75:%.*]] = and i1 [[TMP73]], [[TMP74]]
-; CHECK-NEXT:    [[TMP76:%.*]] = sub i32 [[TMP72]], [[TMP42]]
-; CHECK-NEXT:    [[TMP77:%.*]] = add i32 [[TMP72]], [[TMP42]]
-; CHECK-NEXT:    [[TMP78:%.*]] = select i1 [[TMP75]], i32 [[TMP76]], i32 [[TMP72]]
-; CHECK-NEXT:    [[TMP79:%.*]] = select i1 [[TMP74]], i32 [[TMP78]], i32 [[TMP77]]
-; CHECK-NEXT:    [[TMP80:%.*]] = insertelement <2 x i32> [[TMP40]], i32 [[TMP79]], i64 1
-; CHECK-NEXT:    store <2 x i32> [[TMP80]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP22:%.*]] = mul i32 [[TMP21]], [[TMP2]]
+; CHECK-NEXT:    [[TMP23:%.*]] = sub i32 [[TMP1]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp uge i32 [[TMP23]], [[TMP2]]
+; CHECK-NEXT:    [[TMP25:%.*]] = sub i32 [[TMP23]], [[TMP2]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP24]], i32 [[TMP25]], i32 [[TMP23]]
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp uge i32 [[TMP26]], [[TMP2]]
+; CHECK-NEXT:    [[TMP28:%.*]] = sub i32 [[TMP26]], [[TMP2]]
+; CHECK-NEXT:    [[TMP29:%.*]] = select i1 [[TMP27]], i32 [[TMP28]], i32 [[TMP26]]
+; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <2 x i32> undef, i32 [[TMP29]], i64 0
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <2 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 1
+; CHECK-NEXT:    [[TMP33:%.*]] = uitofp i32 [[TMP32]] to float
+; CHECK-NEXT:    [[TMP34:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP33]])
+; CHECK-NEXT:    [[TMP35:%.*]] = fmul fast float [[TMP34]], 0x41EFFFFFC0000000
+; CHECK-NEXT:    [[TMP36:%.*]] = fptoui float [[TMP35]] to i32
+; CHECK-NEXT:    [[TMP37:%.*]] = sub i32 0, [[TMP32]]
+; CHECK-NEXT:    [[TMP38:%.*]] = mul i32 [[TMP37]], [[TMP36]]
+; CHECK-NEXT:    [[TMP39:%.*]] = zext i32 [[TMP36]] to i64
+; CHECK-NEXT:    [[TMP40:%.*]] = zext i32 [[TMP38]] to i64
+; CHECK-NEXT:    [[TMP41:%.*]] = mul i64 [[TMP39]], [[TMP40]]
+; CHECK-NEXT:    [[TMP42:%.*]] = trunc i64 [[TMP41]] to i32
+; CHECK-NEXT:    [[TMP43:%.*]] = lshr i64 [[TMP41]], 32
+; CHECK-NEXT:    [[TMP44:%.*]] = trunc i64 [[TMP43]] to i32
+; CHECK-NEXT:    [[TMP45:%.*]] = add i32 [[TMP36]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = zext i32 [[TMP31]] to i64
+; CHECK-NEXT:    [[TMP47:%.*]] = zext i32 [[TMP45]] to i64
+; CHECK-NEXT:    [[TMP48:%.*]] = mul i64 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP49:%.*]] = trunc i64 [[TMP48]] to i32
+; CHECK-NEXT:    [[TMP50:%.*]] = lshr i64 [[TMP48]], 32
+; CHECK-NEXT:    [[TMP51:%.*]] = trunc i64 [[TMP50]] to i32
+; CHECK-NEXT:    [[TMP52:%.*]] = mul i32 [[TMP51]], [[TMP32]]
+; CHECK-NEXT:    [[TMP53:%.*]] = sub i32 [[TMP31]], [[TMP52]]
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp uge i32 [[TMP53]], [[TMP32]]
+; CHECK-NEXT:    [[TMP55:%.*]] = sub i32 [[TMP53]], [[TMP32]]
+; CHECK-NEXT:    [[TMP56:%.*]] = select i1 [[TMP54]], i32 [[TMP55]], i32 [[TMP53]]
+; CHECK-NEXT:    [[TMP57:%.*]] = icmp uge i32 [[TMP56]], [[TMP32]]
+; CHECK-NEXT:    [[TMP58:%.*]] = sub i32 [[TMP56]], [[TMP32]]
+; CHECK-NEXT:    [[TMP59:%.*]] = select i1 [[TMP57]], i32 [[TMP58]], i32 [[TMP56]]
+; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <2 x i32> [[TMP30]], i32 [[TMP59]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP60]], <2 x i32> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_v2i32_pow2_shl_denom:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
 ; GCN-NEXT:    s_movk_i32 s4, 0x1000
-; GCN-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_mov_b32 s7, 0x4f7ffffe
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_lshl_b32 s10, s4, s2
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s10
-; GCN-NEXT:    s_mov_b32 s2, 0x4f800000
-; GCN-NEXT:    s_lshl_b32 s11, s4, s3
-; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s11
+; GCN-NEXT:    s_lshl_b32 s2, s4, s2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GCN-NEXT:    s_lshl_b32 s6, s4, s3
+; GCN-NEXT:    s_sub_i32 s3, 0, s2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s6
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; GCN-NEXT:    v_mul_f32_e32 v0, s2, v0
+; GCN-NEXT:    v_mul_f32_e32 v0, s7, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_mul_f32_e32 v1, s2, v1
+; GCN-NEXT:    v_mul_f32_e32 v1, s7, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    v_mul_lo_u32 v2, v0, s10
-; GCN-NEXT:    v_mul_hi_u32 v3, v0, s10
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v2
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v4, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v2, v2, v0
-; GCN-NEXT:    v_mul_lo_u32 v3, v1, s11
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v2, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v2, v0
-; GCN-NEXT:    v_mul_hi_u32 v2, v1, s11
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[0:1]
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v3
+; GCN-NEXT:    v_mul_lo_u32 v2, s3, v0
+; GCN-NEXT:    s_sub_i32 s3, 0, s6
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_mul_hi_u32 v0, v0, s8
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v2, v2, v1
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s10
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v2, v1
-; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v2, v1
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, s9
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s8, v0
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], s8, v0
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s10, v3
-; GCN-NEXT:    v_mul_lo_u32 v1, v1, s11
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, s10, v3
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s10, v3
-; GCN-NEXT:    s_and_b64 vcc, s[2:3], s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s9, v1
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[0:1]
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s9, v1
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s11, v2
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, s11, v2
-; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s11, v2
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[2:3]
-; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s3, v1
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
+; GCN-NEXT:    v_mul_hi_u32 v2, v1, v2
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s4, v0
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s2, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s2, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s2, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s2, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GCN-NEXT:    v_mul_hi_u32 v1, s5, v1
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s6
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s5, v1
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s6, v1
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s6, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s6, v1
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s6, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
   %shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
   %r = urem <2 x i32> %x, %shl.y
@@ -4623,7 +4289,7 @@ define amdgpu_kernel void @urem_v2i32_pow2_shl_denom(<2 x i32> addrspace(1)* %ou
 define amdgpu_kernel void @sdiv_i32_oddk_denom(i32 addrspace(1)* %out, i32 %x) {
 ; CHECK-LABEL: @sdiv_i32_oddk_denom(
 ; CHECK-NEXT:    [[R:%.*]] = sdiv i32 [[X:%.*]], 1235195
-; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_i32_oddk_denom:
@@ -4649,7 +4315,7 @@ define amdgpu_kernel void @sdiv_i32_oddk_denom(i32 addrspace(1)* %out, i32 %x) {
 define amdgpu_kernel void @sdiv_i32_pow2k_denom(i32 addrspace(1)* %out, i32 %x) {
 ; CHECK-LABEL: @sdiv_i32_pow2k_denom(
 ; CHECK-NEXT:    [[R:%.*]] = sdiv i32 [[X:%.*]], 4096
-; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_i32_pow2k_denom:
@@ -4675,7 +4341,7 @@ define amdgpu_kernel void @sdiv_i32_pow2_shl_denom(i32 addrspace(1)* %out, i32 %
 ; CHECK-LABEL: @sdiv_i32_pow2_shl_denom(
 ; CHECK-NEXT:    [[SHL_Y:%.*]] = shl i32 4096, [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = sdiv i32 [[X:%.*]], [[SHL_Y]]
-; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_i32_pow2_shl_denom:
@@ -4690,34 +4356,30 @@ define amdgpu_kernel void @sdiv_i32_pow2_shl_denom(i32 addrspace(1)* %out, i32 %
 ; GCN-NEXT:    s_add_i32 s3, s3, s8
 ; GCN-NEXT:    s_xor_b32 s9, s3, s8
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s9
-; GCN-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-NEXT:    s_add_i32 s2, s2, s3
-; GCN-NEXT:    s_xor_b32 s2, s2, s3
+; GCN-NEXT:    s_sub_i32 s3, 0, s9
+; GCN-NEXT:    s_ashr_i32 s0, s2, 31
+; GCN-NEXT:    s_add_i32 s1, s2, s0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT:    s_xor_b32 s3, s3, s8
-; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GCN-NEXT:    s_xor_b32 s1, s1, s0
+; GCN-NEXT:    s_xor_b32 s2, s0, s8
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_mul_lo_u32 v1, v0, s9
-; GCN-NEXT:    v_mul_hi_u32 v2, v0, s9
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v1, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v0, v0, s2
+; GCN-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GCN-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s1, v0
 ; GCN-NEXT:    v_mul_lo_u32 v1, v0, s9
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, -1, v0
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], s2, v1
-; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s2, v1
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s1, v1
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s9, v1
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v1
-; GCN-NEXT:    s_and_b64 vcc, vcc, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v3, v0, s[0:1]
-; GCN-NEXT:    v_xor_b32_e32 v0, s3, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s3, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GCN-NEXT:    v_xor_b32_e32 v0, s2, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
 ; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %shl.y = shl i32 4096, %y
@@ -4734,7 +4396,7 @@ define amdgpu_kernel void @sdiv_v2i32_pow2k_denom(<2 x i32> addrspace(1)* %out,
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[X]], i64 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = sdiv i32 [[TMP4]], 4096
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP5]], i64 1
-; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_v2i32_pow2k_denom:
@@ -4769,7 +4431,7 @@ define amdgpu_kernel void @ssdiv_v2i32_mixed_pow2k_denom(<2 x i32> addrspace(1)*
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[X]], i64 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = sdiv i32 [[TMP4]], 4095
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP5]], i64 1
-; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: ssdiv_v2i32_mixed_pow2k_denom:
@@ -4811,173 +4473,149 @@ define amdgpu_kernel void @sdiv_v2i32_pow2_shl_denom(<2 x i32> addrspace(1)* %ou
 ; CHECK-NEXT:    [[TMP9:%.*]] = xor i32 [[TMP7]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = uitofp i32 [[TMP9]] to float
 ; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP10]])
-; CHECK-NEXT:    [[TMP12:%.*]] = fmul fast float [[TMP11]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP12:%.*]] = fmul fast float [[TMP11]], 0x41EFFFFFC0000000
 ; CHECK-NEXT:    [[TMP13:%.*]] = fptoui float [[TMP12]] to i32
-; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
-; CHECK-NEXT:    [[TMP15:%.*]] = zext i32 [[TMP9]] to i64
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP14]], [[TMP15]]
-; CHECK-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32
-; CHECK-NEXT:    [[TMP18:%.*]] = lshr i64 [[TMP16]], 32
+; CHECK-NEXT:    [[TMP14:%.*]] = sub i32 0, [[TMP9]]
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP15]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP16]], [[TMP17]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
-; CHECK-NEXT:    [[TMP20:%.*]] = sub i32 0, [[TMP17]]
-; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i32 [[TMP19]], 0
-; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 [[TMP17]]
-; CHECK-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP22]] to i64
-; CHECK-NEXT:    [[TMP24:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP20:%.*]] = lshr i64 [[TMP18]], 32
+; CHECK-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
+; CHECK-NEXT:    [[TMP22:%.*]] = add i32 [[TMP13]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP24:%.*]] = zext i32 [[TMP22]] to i64
 ; CHECK-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP23]], [[TMP24]]
 ; CHECK-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP25]] to i32
 ; CHECK-NEXT:    [[TMP27:%.*]] = lshr i64 [[TMP25]], 32
 ; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i32
-; CHECK-NEXT:    [[TMP29:%.*]] = add i32 [[TMP13]], [[TMP28]]
-; CHECK-NEXT:    [[TMP30:%.*]] = sub i32 [[TMP13]], [[TMP28]]
-; CHECK-NEXT:    [[TMP31:%.*]] = select i1 [[TMP21]], i32 [[TMP29]], i32 [[TMP30]]
-; CHECK-NEXT:    [[TMP32:%.*]] = zext i32 [[TMP31]] to i64
-; CHECK-NEXT:    [[TMP33:%.*]] = zext i32 [[TMP8]] to i64
-; CHECK-NEXT:    [[TMP34:%.*]] = mul i64 [[TMP32]], [[TMP33]]
-; CHECK-NEXT:    [[TMP35:%.*]] = trunc i64 [[TMP34]] to i32
-; CHECK-NEXT:    [[TMP36:%.*]] = lshr i64 [[TMP34]], 32
-; CHECK-NEXT:    [[TMP37:%.*]] = trunc i64 [[TMP36]] to i32
-; CHECK-NEXT:    [[TMP38:%.*]] = mul i32 [[TMP37]], [[TMP9]]
-; CHECK-NEXT:    [[TMP39:%.*]] = sub i32 [[TMP8]], [[TMP38]]
-; CHECK-NEXT:    [[TMP40:%.*]] = icmp uge i32 [[TMP39]], [[TMP9]]
-; CHECK-NEXT:    [[TMP41:%.*]] = icmp uge i32 [[TMP8]], [[TMP38]]
-; CHECK-NEXT:    [[TMP42:%.*]] = and i1 [[TMP40]], [[TMP41]]
-; CHECK-NEXT:    [[TMP43:%.*]] = add i32 [[TMP37]], 1
-; CHECK-NEXT:    [[TMP44:%.*]] = sub i32 [[TMP37]], 1
-; CHECK-NEXT:    [[TMP45:%.*]] = select i1 [[TMP42]], i32 [[TMP43]], i32 [[TMP37]]
-; CHECK-NEXT:    [[TMP46:%.*]] = select i1 [[TMP41]], i32 [[TMP45]], i32 [[TMP44]]
-; CHECK-NEXT:    [[TMP47:%.*]] = xor i32 [[TMP46]], [[TMP5]]
-; CHECK-NEXT:    [[TMP48:%.*]] = sub i32 [[TMP47]], [[TMP5]]
-; CHECK-NEXT:    [[TMP49:%.*]] = insertelement <2 x i32> undef, i32 [[TMP48]], i64 0
-; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <2 x i32> [[X]], i64 1
-; CHECK-NEXT:    [[TMP51:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 1
-; CHECK-NEXT:    [[TMP52:%.*]] = ashr i32 [[TMP50]], 31
-; CHECK-NEXT:    [[TMP53:%.*]] = ashr i32 [[TMP51]], 31
-; CHECK-NEXT:    [[TMP54:%.*]] = xor i32 [[TMP52]], [[TMP53]]
-; CHECK-NEXT:    [[TMP55:%.*]] = add i32 [[TMP50]], [[TMP52]]
-; CHECK-NEXT:    [[TMP56:%.*]] = add i32 [[TMP51]], [[TMP53]]
-; CHECK-NEXT:    [[TMP57:%.*]] = xor i32 [[TMP55]], [[TMP52]]
-; CHECK-NEXT:    [[TMP58:%.*]] = xor i32 [[TMP56]], [[TMP53]]
-; CHECK-NEXT:    [[TMP59:%.*]] = uitofp i32 [[TMP58]] to float
-; CHECK-NEXT:    [[TMP60:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP59]])
-; CHECK-NEXT:    [[TMP61:%.*]] = fmul fast float [[TMP60]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP62:%.*]] = fptoui float [[TMP61]] to i32
-; CHECK-NEXT:    [[TMP63:%.*]] = zext i32 [[TMP62]] to i64
-; CHECK-NEXT:    [[TMP64:%.*]] = zext i32 [[TMP58]] to i64
-; CHECK-NEXT:    [[TMP65:%.*]] = mul i64 [[TMP63]], [[TMP64]]
-; CHECK-NEXT:    [[TMP66:%.*]] = trunc i64 [[TMP65]] to i32
-; CHECK-NEXT:    [[TMP67:%.*]] = lshr i64 [[TMP65]], 32
-; CHECK-NEXT:    [[TMP68:%.*]] = trunc i64 [[TMP67]] to i32
-; CHECK-NEXT:    [[TMP69:%.*]] = sub i32 0, [[TMP66]]
-; CHECK-NEXT:    [[TMP70:%.*]] = icmp eq i32 [[TMP68]], 0
-; CHECK-NEXT:    [[TMP71:%.*]] = select i1 [[TMP70]], i32 [[TMP69]], i32 [[TMP66]]
-; CHECK-NEXT:    [[TMP72:%.*]] = zext i32 [[TMP71]] to i64
-; CHECK-NEXT:    [[TMP73:%.*]] = zext i32 [[TMP62]] to i64
-; CHECK-NEXT:    [[TMP74:%.*]] = mul i64 [[TMP72]], [[TMP73]]
-; CHECK-NEXT:    [[TMP75:%.*]] = trunc i64 [[TMP74]] to i32
-; CHECK-NEXT:    [[TMP76:%.*]] = lshr i64 [[TMP74]], 32
-; CHECK-NEXT:    [[TMP77:%.*]] = trunc i64 [[TMP76]] to i32
-; CHECK-NEXT:    [[TMP78:%.*]] = add i32 [[TMP62]], [[TMP77]]
-; CHECK-NEXT:    [[TMP79:%.*]] = sub i32 [[TMP62]], [[TMP77]]
-; CHECK-NEXT:    [[TMP80:%.*]] = select i1 [[TMP70]], i32 [[TMP78]], i32 [[TMP79]]
-; CHECK-NEXT:    [[TMP81:%.*]] = zext i32 [[TMP80]] to i64
-; CHECK-NEXT:    [[TMP82:%.*]] = zext i32 [[TMP57]] to i64
-; CHECK-NEXT:    [[TMP83:%.*]] = mul i64 [[TMP81]], [[TMP82]]
-; CHECK-NEXT:    [[TMP84:%.*]] = trunc i64 [[TMP83]] to i32
-; CHECK-NEXT:    [[TMP85:%.*]] = lshr i64 [[TMP83]], 32
-; CHECK-NEXT:    [[TMP86:%.*]] = trunc i64 [[TMP85]] to i32
-; CHECK-NEXT:    [[TMP87:%.*]] = mul i32 [[TMP86]], [[TMP58]]
-; CHECK-NEXT:    [[TMP88:%.*]] = sub i32 [[TMP57]], [[TMP87]]
-; CHECK-NEXT:    [[TMP89:%.*]] = icmp uge i32 [[TMP88]], [[TMP58]]
-; CHECK-NEXT:    [[TMP90:%.*]] = icmp uge i32 [[TMP57]], [[TMP87]]
-; CHECK-NEXT:    [[TMP91:%.*]] = and i1 [[TMP89]], [[TMP90]]
-; CHECK-NEXT:    [[TMP92:%.*]] = add i32 [[TMP86]], 1
-; CHECK-NEXT:    [[TMP93:%.*]] = sub i32 [[TMP86]], 1
-; CHECK-NEXT:    [[TMP94:%.*]] = select i1 [[TMP91]], i32 [[TMP92]], i32 [[TMP86]]
-; CHECK-NEXT:    [[TMP95:%.*]] = select i1 [[TMP90]], i32 [[TMP94]], i32 [[TMP93]]
-; CHECK-NEXT:    [[TMP96:%.*]] = xor i32 [[TMP95]], [[TMP54]]
-; CHECK-NEXT:    [[TMP97:%.*]] = sub i32 [[TMP96]], [[TMP54]]
-; CHECK-NEXT:    [[TMP98:%.*]] = insertelement <2 x i32> [[TMP49]], i32 [[TMP97]], i64 1
-; CHECK-NEXT:    store <2 x i32> [[TMP98]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP29:%.*]] = mul i32 [[TMP28]], [[TMP9]]
+; CHECK-NEXT:    [[TMP30:%.*]] = sub i32 [[TMP8]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp uge i32 [[TMP30]], [[TMP9]]
+; CHECK-NEXT:    [[TMP32:%.*]] = add i32 [[TMP28]], 1
+; CHECK-NEXT:    [[TMP33:%.*]] = select i1 [[TMP31]], i32 [[TMP32]], i32 [[TMP28]]
+; CHECK-NEXT:    [[TMP34:%.*]] = sub i32 [[TMP30]], [[TMP9]]
+; CHECK-NEXT:    [[TMP35:%.*]] = select i1 [[TMP31]], i32 [[TMP34]], i32 [[TMP30]]
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp uge i32 [[TMP35]], [[TMP9]]
+; CHECK-NEXT:    [[TMP37:%.*]] = add i32 [[TMP33]], 1
+; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP36]], i32 [[TMP37]], i32 [[TMP33]]
+; CHECK-NEXT:    [[TMP39:%.*]] = xor i32 [[TMP38]], [[TMP5]]
+; CHECK-NEXT:    [[TMP40:%.*]] = sub i32 [[TMP39]], [[TMP5]]
+; CHECK-NEXT:    [[TMP41:%.*]] = insertelement <2 x i32> undef, i32 [[TMP40]], i64 0
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <2 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 1
+; CHECK-NEXT:    [[TMP44:%.*]] = ashr i32 [[TMP42]], 31
+; CHECK-NEXT:    [[TMP45:%.*]] = ashr i32 [[TMP43]], 31
+; CHECK-NEXT:    [[TMP46:%.*]] = xor i32 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = add i32 [[TMP42]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = add i32 [[TMP43]], [[TMP45]]
+; CHECK-NEXT:    [[TMP49:%.*]] = xor i32 [[TMP47]], [[TMP44]]
+; CHECK-NEXT:    [[TMP50:%.*]] = xor i32 [[TMP48]], [[TMP45]]
+; CHECK-NEXT:    [[TMP51:%.*]] = uitofp i32 [[TMP50]] to float
+; CHECK-NEXT:    [[TMP52:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP51]])
+; CHECK-NEXT:    [[TMP53:%.*]] = fmul fast float [[TMP52]], 0x41EFFFFFC0000000
+; CHECK-NEXT:    [[TMP54:%.*]] = fptoui float [[TMP53]] to i32
+; CHECK-NEXT:    [[TMP55:%.*]] = sub i32 0, [[TMP50]]
+; CHECK-NEXT:    [[TMP56:%.*]] = mul i32 [[TMP55]], [[TMP54]]
+; CHECK-NEXT:    [[TMP57:%.*]] = zext i32 [[TMP54]] to i64
+; CHECK-NEXT:    [[TMP58:%.*]] = zext i32 [[TMP56]] to i64
+; CHECK-NEXT:    [[TMP59:%.*]] = mul i64 [[TMP57]], [[TMP58]]
+; CHECK-NEXT:    [[TMP60:%.*]] = trunc i64 [[TMP59]] to i32
+; CHECK-NEXT:    [[TMP61:%.*]] = lshr i64 [[TMP59]], 32
+; CHECK-NEXT:    [[TMP62:%.*]] = trunc i64 [[TMP61]] to i32
+; CHECK-NEXT:    [[TMP63:%.*]] = add i32 [[TMP54]], [[TMP62]]
+; CHECK-NEXT:    [[TMP64:%.*]] = zext i32 [[TMP49]] to i64
+; CHECK-NEXT:    [[TMP65:%.*]] = zext i32 [[TMP63]] to i64
+; CHECK-NEXT:    [[TMP66:%.*]] = mul i64 [[TMP64]], [[TMP65]]
+; CHECK-NEXT:    [[TMP67:%.*]] = trunc i64 [[TMP66]] to i32
+; CHECK-NEXT:    [[TMP68:%.*]] = lshr i64 [[TMP66]], 32
+; CHECK-NEXT:    [[TMP69:%.*]] = trunc i64 [[TMP68]] to i32
+; CHECK-NEXT:    [[TMP70:%.*]] = mul i32 [[TMP69]], [[TMP50]]
+; CHECK-NEXT:    [[TMP71:%.*]] = sub i32 [[TMP49]], [[TMP70]]
+; CHECK-NEXT:    [[TMP72:%.*]] = icmp uge i32 [[TMP71]], [[TMP50]]
+; CHECK-NEXT:    [[TMP73:%.*]] = add i32 [[TMP69]], 1
+; CHECK-NEXT:    [[TMP74:%.*]] = select i1 [[TMP72]], i32 [[TMP73]], i32 [[TMP69]]
+; CHECK-NEXT:    [[TMP75:%.*]] = sub i32 [[TMP71]], [[TMP50]]
+; CHECK-NEXT:    [[TMP76:%.*]] = select i1 [[TMP72]], i32 [[TMP75]], i32 [[TMP71]]
+; CHECK-NEXT:    [[TMP77:%.*]] = icmp uge i32 [[TMP76]], [[TMP50]]
+; CHECK-NEXT:    [[TMP78:%.*]] = add i32 [[TMP74]], 1
+; CHECK-NEXT:    [[TMP79:%.*]] = select i1 [[TMP77]], i32 [[TMP78]], i32 [[TMP74]]
+; CHECK-NEXT:    [[TMP80:%.*]] = xor i32 [[TMP79]], [[TMP46]]
+; CHECK-NEXT:    [[TMP81:%.*]] = sub i32 [[TMP80]], [[TMP46]]
+; CHECK-NEXT:    [[TMP82:%.*]] = insertelement <2 x i32> [[TMP41]], i32 [[TMP81]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP82]], <2 x i32> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_v2i32_pow2_shl_denom:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
-; GCN-NEXT:    s_movk_i32 s4, 0x1000
-; GCN-NEXT:    s_mov_b32 s14, 0x4f800000
-; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x9
-; GCN-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0xb
-; GCN-NEXT:    s_mov_b32 s11, 0xf000
+; GCN-NEXT:    s_movk_i32 s6, 0x1000
+; GCN-NEXT:    s_mov_b32 s12, 0x4f7ffffe
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_lshl_b32 s2, s4, s2
-; GCN-NEXT:    s_ashr_i32 s5, s2, 31
-; GCN-NEXT:    s_add_i32 s2, s2, s5
-; GCN-NEXT:    s_xor_b32 s13, s2, s5
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s13
-; GCN-NEXT:    s_ashr_i32 s2, s6, 31
-; GCN-NEXT:    s_lshl_b32 s0, s4, s3
-; GCN-NEXT:    s_add_i32 s1, s6, s2
+; GCN-NEXT:    s_lshl_b32 s2, s6, s2
+; GCN-NEXT:    s_ashr_i32 s10, s2, 31
+; GCN-NEXT:    s_add_i32 s2, s2, s10
+; GCN-NEXT:    s_xor_b32 s11, s2, s10
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s11
+; GCN-NEXT:    s_sub_i32 s1, 0, s11
+; GCN-NEXT:    s_lshl_b32 s0, s6, s3
+; GCN-NEXT:    s_ashr_i32 s3, s0, 31
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT:    s_ashr_i32 s6, s0, 31
-; GCN-NEXT:    s_add_i32 s4, s0, s6
-; GCN-NEXT:    s_xor_b32 s3, s1, s2
-; GCN-NEXT:    v_mul_f32_e32 v0, s14, v0
+; GCN-NEXT:    s_add_i32 s0, s0, s3
+; GCN-NEXT:    s_xor_b32 s13, s0, s3
+; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s13
+; GCN-NEXT:    v_mul_f32_e32 v0, s12, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    s_xor_b32 s15, s4, s6
-; GCN-NEXT:    s_xor_b32 s12, s2, s5
-; GCN-NEXT:    s_mov_b32 s10, -1
-; GCN-NEXT:    v_mul_lo_u32 v1, v0, s13
-; GCN-NEXT:    v_mul_hi_u32 v2, v0, s13
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s15
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v1, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v3, s[0:1]
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v2
-; GCN-NEXT:    v_mul_hi_u32 v0, v0, s3
-; GCN-NEXT:    v_mul_f32_e32 v1, s14, v1
-; GCN-NEXT:    v_mul_lo_u32 v2, v0, s13
+; GCN-NEXT:    s_ashr_i32 s2, s8, 31
+; GCN-NEXT:    s_add_i32 s0, s8, s2
+; GCN-NEXT:    s_xor_b32 s0, s0, s2
+; GCN-NEXT:    v_mul_lo_u32 v1, s1, v0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GCN-NEXT:    s_xor_b32 s2, s2, s10
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, s12, v2
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, -1, v0
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s3, v2
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v4
-; GCN-NEXT:    v_mul_lo_u32 v4, v1, s15
-; GCN-NEXT:    v_mul_hi_u32 v5, v1, s15
-; GCN-NEXT:    s_ashr_i32 s13, s7, 31
-; GCN-NEXT:    s_add_i32 s7, s7, s13
-; GCN-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v4, v4, v1
-; GCN-NEXT:    s_xor_b32 s7, s7, s13
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s3, v2
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v4, v1
-; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v4, v1
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, s7
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v3, v0, s[2:3]
-; GCN-NEXT:    v_mul_lo_u32 v2, v1, s15
-; GCN-NEXT:    v_xor_b32_e32 v0, s12, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s12, v0
-; GCN-NEXT:    s_xor_b32 s4, s13, s6
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s7, v2
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s15, v3
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s7, v2
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, -1, v1
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v1
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[2:3]
-; GCN-NEXT:    v_xor_b32_e32 v1, s4, v1
-; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s4, v1
-; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NEXT:    v_mul_lo_u32 v2, v0, s11
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s0, v2
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s11, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s11, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; GCN-NEXT:    s_sub_i32 s0, 0, s13
+; GCN-NEXT:    v_mul_lo_u32 v4, s0, v1
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s11, v2
+; GCN-NEXT:    s_ashr_i32 s0, s9, 31
+; GCN-NEXT:    v_mul_hi_u32 v2, v1, v4
+; GCN-NEXT:    s_add_i32 s1, s9, s0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GCN-NEXT:    s_xor_b32 s1, s1, s0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GCN-NEXT:    v_mul_hi_u32 v1, s1, v1
+; GCN-NEXT:    v_xor_b32_e32 v0, s2, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
+; GCN-NEXT:    s_xor_b32 s2, s0, s3
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, s13
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s1, v2
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s13, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GCN-NEXT:    v_xor_b32_e32 v1, s2, v1
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s2, v1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
   %r = sdiv <2 x i32> %x, %shl.y
@@ -4988,7 +4626,7 @@ define amdgpu_kernel void @sdiv_v2i32_pow2_shl_denom(<2 x i32> addrspace(1)* %ou
 define amdgpu_kernel void @srem_i32_oddk_denom(i32 addrspace(1)* %out, i32 %x) {
 ; CHECK-LABEL: @srem_i32_oddk_denom(
 ; CHECK-NEXT:    [[R:%.*]] = srem i32 [[X:%.*]], 1235195
-; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_i32_oddk_denom:
@@ -5016,7 +4654,7 @@ define amdgpu_kernel void @srem_i32_oddk_denom(i32 addrspace(1)* %out, i32 %x) {
 define amdgpu_kernel void @srem_i32_pow2k_denom(i32 addrspace(1)* %out, i32 %x) {
 ; CHECK-LABEL: @srem_i32_pow2k_denom(
 ; CHECK-NEXT:    [[R:%.*]] = srem i32 [[X:%.*]], 4096
-; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_i32_pow2k_denom:
@@ -5043,50 +4681,43 @@ define amdgpu_kernel void @srem_i32_pow2_shl_denom(i32 addrspace(1)* %out, i32 %
 ; CHECK-LABEL: @srem_i32_pow2_shl_denom(
 ; CHECK-NEXT:    [[SHL_Y:%.*]] = shl i32 4096, [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = srem i32 [[X:%.*]], [[SHL_Y]]
-; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_i32_pow2_shl_denom:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
-; GCN-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_lshl_b32 s2, 0x1000, s5
-; GCN-NEXT:    s_ashr_i32 s3, s2, 31
-; GCN-NEXT:    s_add_i32 s2, s2, s3
-; GCN-NEXT:    s_xor_b32 s10, s2, s3
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s10
-; GCN-NEXT:    s_ashr_i32 s8, s4, 31
-; GCN-NEXT:    s_add_i32 s4, s4, s8
-; GCN-NEXT:    s_xor_b32 s9, s4, s8
+; GCN-NEXT:    s_lshl_b32 s3, 0x1000, s3
+; GCN-NEXT:    s_ashr_i32 s4, s3, 31
+; GCN-NEXT:    s_add_i32 s3, s3, s4
+; GCN-NEXT:    s_xor_b32 s4, s3, s4
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s4
+; GCN-NEXT:    s_sub_i32 s3, 0, s4
+; GCN-NEXT:    s_ashr_i32 s5, s2, 31
+; GCN-NEXT:    s_add_i32 s2, s2, s5
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GCN-NEXT:    s_xor_b32 s6, s2, s5
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_mul_lo_u32 v1, v0, s10
-; GCN-NEXT:    v_mul_hi_u32 v2, v0, s10
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v1, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v0, v0, s9
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s10
-; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s9, v0
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], s9, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, s10, v1
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s10, v1
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s10, v1
-; GCN-NEXT:    s_and_b64 vcc, s[2:3], s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[0:1]
-; GCN-NEXT:    v_xor_b32_e32 v0, s8, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s8, v0
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s6, v0
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s4
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s4, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s4, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s4, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-NEXT:    v_xor_b32_e32 v0, s5, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s5, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
   %shl.y = shl i32 4096, %y
   %r = srem i32 %x, %shl.y
@@ -5102,7 +4733,7 @@ define amdgpu_kernel void @srem_v2i32_pow2k_denom(<2 x i32> addrspace(1)* %out,
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[X]], i64 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = srem i32 [[TMP4]], 4096
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP5]], i64 1
-; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_v2i32_pow2k_denom:
@@ -5145,170 +4776,139 @@ define amdgpu_kernel void @srem_v2i32_pow2_shl_denom(<2 x i32> addrspace(1)* %ou
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i32 [[TMP6]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = uitofp i32 [[TMP8]] to float
 ; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
-; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP10]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP10]], 0x41EFFFFFC0000000
 ; CHECK-NEXT:    [[TMP12:%.*]] = fptoui float [[TMP11]] to i32
-; CHECK-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP12]] to i64
-; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP8]] to i64
-; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP13]], [[TMP14]]
-; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32
-; CHECK-NEXT:    [[TMP17:%.*]] = lshr i64 [[TMP15]], 32
+; CHECK-NEXT:    [[TMP13:%.*]] = sub i32 0, [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = mul i32 [[TMP13]], [[TMP12]]
+; CHECK-NEXT:    [[TMP15:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP15]], [[TMP16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = trunc i64 [[TMP17]] to i32
-; CHECK-NEXT:    [[TMP19:%.*]] = sub i32 0, [[TMP16]]
-; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i32 [[TMP18]], 0
-; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP20]], i32 [[TMP19]], i32 [[TMP16]]
-; CHECK-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP21]] to i64
-; CHECK-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = lshr i64 [[TMP17]], 32
+; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
+; CHECK-NEXT:    [[TMP21:%.*]] = add i32 [[TMP12]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP21]] to i64
 ; CHECK-NEXT:    [[TMP24:%.*]] = mul i64 [[TMP22]], [[TMP23]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP24]] to i32
 ; CHECK-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP24]], 32
 ; CHECK-NEXT:    [[TMP27:%.*]] = trunc i64 [[TMP26]] to i32
-; CHECK-NEXT:    [[TMP28:%.*]] = add i32 [[TMP12]], [[TMP27]]
-; CHECK-NEXT:    [[TMP29:%.*]] = sub i32 [[TMP12]], [[TMP27]]
-; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP20]], i32 [[TMP28]], i32 [[TMP29]]
-; CHECK-NEXT:    [[TMP31:%.*]] = zext i32 [[TMP30]] to i64
-; CHECK-NEXT:    [[TMP32:%.*]] = zext i32 [[TMP7]] to i64
-; CHECK-NEXT:    [[TMP33:%.*]] = mul i64 [[TMP31]], [[TMP32]]
-; CHECK-NEXT:    [[TMP34:%.*]] = trunc i64 [[TMP33]] to i32
-; CHECK-NEXT:    [[TMP35:%.*]] = lshr i64 [[TMP33]], 32
-; CHECK-NEXT:    [[TMP36:%.*]] = trunc i64 [[TMP35]] to i32
-; CHECK-NEXT:    [[TMP37:%.*]] = mul i32 [[TMP36]], [[TMP8]]
-; CHECK-NEXT:    [[TMP38:%.*]] = sub i32 [[TMP7]], [[TMP37]]
-; CHECK-NEXT:    [[TMP39:%.*]] = icmp uge i32 [[TMP38]], [[TMP8]]
-; CHECK-NEXT:    [[TMP40:%.*]] = icmp uge i32 [[TMP7]], [[TMP37]]
-; CHECK-NEXT:    [[TMP41:%.*]] = and i1 [[TMP39]], [[TMP40]]
-; CHECK-NEXT:    [[TMP42:%.*]] = sub i32 [[TMP38]], [[TMP8]]
-; CHECK-NEXT:    [[TMP43:%.*]] = add i32 [[TMP38]], [[TMP8]]
-; CHECK-NEXT:    [[TMP44:%.*]] = select i1 [[TMP41]], i32 [[TMP42]], i32 [[TMP38]]
-; CHECK-NEXT:    [[TMP45:%.*]] = select i1 [[TMP40]], i32 [[TMP44]], i32 [[TMP43]]
-; CHECK-NEXT:    [[TMP46:%.*]] = xor i32 [[TMP45]], [[TMP3]]
-; CHECK-NEXT:    [[TMP47:%.*]] = sub i32 [[TMP46]], [[TMP3]]
-; CHECK-NEXT:    [[TMP48:%.*]] = insertelement <2 x i32> undef, i32 [[TMP47]], i64 0
-; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <2 x i32> [[X]], i64 1
-; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 1
-; CHECK-NEXT:    [[TMP51:%.*]] = ashr i32 [[TMP49]], 31
-; CHECK-NEXT:    [[TMP52:%.*]] = ashr i32 [[TMP50]], 31
-; CHECK-NEXT:    [[TMP53:%.*]] = add i32 [[TMP49]], [[TMP51]]
-; CHECK-NEXT:    [[TMP54:%.*]] = add i32 [[TMP50]], [[TMP52]]
-; CHECK-NEXT:    [[TMP55:%.*]] = xor i32 [[TMP53]], [[TMP51]]
-; CHECK-NEXT:    [[TMP56:%.*]] = xor i32 [[TMP54]], [[TMP52]]
-; CHECK-NEXT:    [[TMP57:%.*]] = uitofp i32 [[TMP56]] to float
-; CHECK-NEXT:    [[TMP58:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP57]])
-; CHECK-NEXT:    [[TMP59:%.*]] = fmul fast float [[TMP58]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP60:%.*]] = fptoui float [[TMP59]] to i32
-; CHECK-NEXT:    [[TMP61:%.*]] = zext i32 [[TMP60]] to i64
-; CHECK-NEXT:    [[TMP62:%.*]] = zext i32 [[TMP56]] to i64
-; CHECK-NEXT:    [[TMP63:%.*]] = mul i64 [[TMP61]], [[TMP62]]
-; CHECK-NEXT:    [[TMP64:%.*]] = trunc i64 [[TMP63]] to i32
-; CHECK-NEXT:    [[TMP65:%.*]] = lshr i64 [[TMP63]], 32
-; CHECK-NEXT:    [[TMP66:%.*]] = trunc i64 [[TMP65]] to i32
-; CHECK-NEXT:    [[TMP67:%.*]] = sub i32 0, [[TMP64]]
-; CHECK-NEXT:    [[TMP68:%.*]] = icmp eq i32 [[TMP66]], 0
-; CHECK-NEXT:    [[TMP69:%.*]] = select i1 [[TMP68]], i32 [[TMP67]], i32 [[TMP64]]
-; CHECK-NEXT:    [[TMP70:%.*]] = zext i32 [[TMP69]] to i64
-; CHECK-NEXT:    [[TMP71:%.*]] = zext i32 [[TMP60]] to i64
-; CHECK-NEXT:    [[TMP72:%.*]] = mul i64 [[TMP70]], [[TMP71]]
-; CHECK-NEXT:    [[TMP73:%.*]] = trunc i64 [[TMP72]] to i32
-; CHECK-NEXT:    [[TMP74:%.*]] = lshr i64 [[TMP72]], 32
-; CHECK-NEXT:    [[TMP75:%.*]] = trunc i64 [[TMP74]] to i32
-; CHECK-NEXT:    [[TMP76:%.*]] = add i32 [[TMP60]], [[TMP75]]
-; CHECK-NEXT:    [[TMP77:%.*]] = sub i32 [[TMP60]], [[TMP75]]
-; CHECK-NEXT:    [[TMP78:%.*]] = select i1 [[TMP68]], i32 [[TMP76]], i32 [[TMP77]]
-; CHECK-NEXT:    [[TMP79:%.*]] = zext i32 [[TMP78]] to i64
-; CHECK-NEXT:    [[TMP80:%.*]] = zext i32 [[TMP55]] to i64
-; CHECK-NEXT:    [[TMP81:%.*]] = mul i64 [[TMP79]], [[TMP80]]
-; CHECK-NEXT:    [[TMP82:%.*]] = trunc i64 [[TMP81]] to i32
-; CHECK-NEXT:    [[TMP83:%.*]] = lshr i64 [[TMP81]], 32
-; CHECK-NEXT:    [[TMP84:%.*]] = trunc i64 [[TMP83]] to i32
-; CHECK-NEXT:    [[TMP85:%.*]] = mul i32 [[TMP84]], [[TMP56]]
-; CHECK-NEXT:    [[TMP86:%.*]] = sub i32 [[TMP55]], [[TMP85]]
-; CHECK-NEXT:    [[TMP87:%.*]] = icmp uge i32 [[TMP86]], [[TMP56]]
-; CHECK-NEXT:    [[TMP88:%.*]] = icmp uge i32 [[TMP55]], [[TMP85]]
-; CHECK-NEXT:    [[TMP89:%.*]] = and i1 [[TMP87]], [[TMP88]]
-; CHECK-NEXT:    [[TMP90:%.*]] = sub i32 [[TMP86]], [[TMP56]]
-; CHECK-NEXT:    [[TMP91:%.*]] = add i32 [[TMP86]], [[TMP56]]
-; CHECK-NEXT:    [[TMP92:%.*]] = select i1 [[TMP89]], i32 [[TMP90]], i32 [[TMP86]]
-; CHECK-NEXT:    [[TMP93:%.*]] = select i1 [[TMP88]], i32 [[TMP92]], i32 [[TMP91]]
-; CHECK-NEXT:    [[TMP94:%.*]] = xor i32 [[TMP93]], [[TMP51]]
-; CHECK-NEXT:    [[TMP95:%.*]] = sub i32 [[TMP94]], [[TMP51]]
-; CHECK-NEXT:    [[TMP96:%.*]] = insertelement <2 x i32> [[TMP48]], i32 [[TMP95]], i64 1
-; CHECK-NEXT:    store <2 x i32> [[TMP96]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP28:%.*]] = mul i32 [[TMP27]], [[TMP8]]
+; CHECK-NEXT:    [[TMP29:%.*]] = sub i32 [[TMP7]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp uge i32 [[TMP29]], [[TMP8]]
+; CHECK-NEXT:    [[TMP31:%.*]] = sub i32 [[TMP29]], [[TMP8]]
+; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP30]], i32 [[TMP31]], i32 [[TMP29]]
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp uge i32 [[TMP32]], [[TMP8]]
+; CHECK-NEXT:    [[TMP34:%.*]] = sub i32 [[TMP32]], [[TMP8]]
+; CHECK-NEXT:    [[TMP35:%.*]] = select i1 [[TMP33]], i32 [[TMP34]], i32 [[TMP32]]
+; CHECK-NEXT:    [[TMP36:%.*]] = xor i32 [[TMP35]], [[TMP3]]
+; CHECK-NEXT:    [[TMP37:%.*]] = sub i32 [[TMP36]], [[TMP3]]
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <2 x i32> undef, i32 [[TMP37]], i64 0
+; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <2 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 1
+; CHECK-NEXT:    [[TMP41:%.*]] = ashr i32 [[TMP39]], 31
+; CHECK-NEXT:    [[TMP42:%.*]] = ashr i32 [[TMP40]], 31
+; CHECK-NEXT:    [[TMP43:%.*]] = add i32 [[TMP39]], [[TMP41]]
+; CHECK-NEXT:    [[TMP44:%.*]] = add i32 [[TMP40]], [[TMP42]]
+; CHECK-NEXT:    [[TMP45:%.*]] = xor i32 [[TMP43]], [[TMP41]]
+; CHECK-NEXT:    [[TMP46:%.*]] = xor i32 [[TMP44]], [[TMP42]]
+; CHECK-NEXT:    [[TMP47:%.*]] = uitofp i32 [[TMP46]] to float
+; CHECK-NEXT:    [[TMP48:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP47]])
+; CHECK-NEXT:    [[TMP49:%.*]] = fmul fast float [[TMP48]], 0x41EFFFFFC0000000
+; CHECK-NEXT:    [[TMP50:%.*]] = fptoui float [[TMP49]] to i32
+; CHECK-NEXT:    [[TMP51:%.*]] = sub i32 0, [[TMP46]]
+; CHECK-NEXT:    [[TMP52:%.*]] = mul i32 [[TMP51]], [[TMP50]]
+; CHECK-NEXT:    [[TMP53:%.*]] = zext i32 [[TMP50]] to i64
+; CHECK-NEXT:    [[TMP54:%.*]] = zext i32 [[TMP52]] to i64
+; CHECK-NEXT:    [[TMP55:%.*]] = mul i64 [[TMP53]], [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = trunc i64 [[TMP55]] to i32
+; CHECK-NEXT:    [[TMP57:%.*]] = lshr i64 [[TMP55]], 32
+; CHECK-NEXT:    [[TMP58:%.*]] = trunc i64 [[TMP57]] to i32
+; CHECK-NEXT:    [[TMP59:%.*]] = add i32 [[TMP50]], [[TMP58]]
+; CHECK-NEXT:    [[TMP60:%.*]] = zext i32 [[TMP45]] to i64
+; CHECK-NEXT:    [[TMP61:%.*]] = zext i32 [[TMP59]] to i64
+; CHECK-NEXT:    [[TMP62:%.*]] = mul i64 [[TMP60]], [[TMP61]]
+; CHECK-NEXT:    [[TMP63:%.*]] = trunc i64 [[TMP62]] to i32
+; CHECK-NEXT:    [[TMP64:%.*]] = lshr i64 [[TMP62]], 32
+; CHECK-NEXT:    [[TMP65:%.*]] = trunc i64 [[TMP64]] to i32
+; CHECK-NEXT:    [[TMP66:%.*]] = mul i32 [[TMP65]], [[TMP46]]
+; CHECK-NEXT:    [[TMP67:%.*]] = sub i32 [[TMP45]], [[TMP66]]
+; CHECK-NEXT:    [[TMP68:%.*]] = icmp uge i32 [[TMP67]], [[TMP46]]
+; CHECK-NEXT:    [[TMP69:%.*]] = sub i32 [[TMP67]], [[TMP46]]
+; CHECK-NEXT:    [[TMP70:%.*]] = select i1 [[TMP68]], i32 [[TMP69]], i32 [[TMP67]]
+; CHECK-NEXT:    [[TMP71:%.*]] = icmp uge i32 [[TMP70]], [[TMP46]]
+; CHECK-NEXT:    [[TMP72:%.*]] = sub i32 [[TMP70]], [[TMP46]]
+; CHECK-NEXT:    [[TMP73:%.*]] = select i1 [[TMP71]], i32 [[TMP72]], i32 [[TMP70]]
+; CHECK-NEXT:    [[TMP74:%.*]] = xor i32 [[TMP73]], [[TMP41]]
+; CHECK-NEXT:    [[TMP75:%.*]] = sub i32 [[TMP74]], [[TMP41]]
+; CHECK-NEXT:    [[TMP76:%.*]] = insertelement <2 x i32> [[TMP38]], i32 [[TMP75]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP76]], <2 x i32> addrspace(1)* [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_v2i32_pow2_shl_denom:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
-; GCN-NEXT:    s_movk_i32 s4, 0x1000
-; GCN-NEXT:    s_mov_b32 s14, 0x4f800000
-; GCN-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0xb
-; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x9
+; GCN-NEXT:    s_movk_i32 s6, 0x1000
+; GCN-NEXT:    s_mov_b32 s7, 0x4f7ffffe
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_lshl_b32 s2, s4, s2
-; GCN-NEXT:    s_ashr_i32 s5, s2, 31
-; GCN-NEXT:    s_add_i32 s2, s2, s5
-; GCN-NEXT:    s_xor_b32 s13, s2, s5
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s13
-; GCN-NEXT:    s_lshl_b32 s2, s4, s3
-; GCN-NEXT:    s_ashr_i32 s12, s6, 31
-; GCN-NEXT:    s_add_i32 s3, s6, s12
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    s_lshl_b32 s2, s6, s2
 ; GCN-NEXT:    s_ashr_i32 s4, s2, 31
-; GCN-NEXT:    s_add_i32 s6, s2, s4
-; GCN-NEXT:    s_xor_b32 s5, s3, s12
-; GCN-NEXT:    v_mul_f32_e32 v0, s14, v0
+; GCN-NEXT:    s_add_i32 s2, s2, s4
+; GCN-NEXT:    s_xor_b32 s2, s2, s4
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GCN-NEXT:    s_lshl_b32 s3, s6, s3
+; GCN-NEXT:    s_ashr_i32 s6, s3, 31
+; GCN-NEXT:    s_add_i32 s3, s3, s6
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    s_xor_b32 s3, s3, s6
+; GCN-NEXT:    s_sub_i32 s6, 0, s2
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GCN-NEXT:    v_mul_f32_e32 v0, s7, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    s_xor_b32 s15, s6, s4
-; GCN-NEXT:    s_ashr_i32 s6, s7, 31
-; GCN-NEXT:    s_add_i32 s7, s7, s6
-; GCN-NEXT:    v_mul_lo_u32 v1, v0, s13
-; GCN-NEXT:    v_mul_hi_u32 v2, v0, s13
-; GCN-NEXT:    s_xor_b32 s7, s7, s6
-; GCN-NEXT:    s_mov_b32 s11, 0xf000
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s15
-; GCN-NEXT:    s_mov_b32 s10, -1
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v1, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v3, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v0, v0, s5
-; GCN-NEXT:    v_mul_f32_e32 v1, s14, v1
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s3
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s8, s0, 31
+; GCN-NEXT:    v_mul_lo_u32 v2, s6, v0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GCN-NEXT:    s_add_i32 s0, s0, s8
+; GCN-NEXT:    s_xor_b32 s0, s0, s8
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, v2
+; GCN-NEXT:    v_mul_f32_e32 v1, s7, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s13
-; GCN-NEXT:    v_mul_lo_u32 v4, v1, s15
-; GCN-NEXT:    v_mul_hi_u32 v5, v1, s15
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s5, v0
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s5, v0
-; GCN-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v4, v4, v1
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v2
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, s13, v2
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s13, v2
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v4, v1
-; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v4, v1
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v1, v1, s7
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v3, v0, s[2:3]
-; GCN-NEXT:    v_mul_lo_u32 v1, v1, s15
-; GCN-NEXT:    v_xor_b32_e32 v0, s12, v0
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s12, v0
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s7, v1
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s7, v1
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s15, v2
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, s15, v2
-; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s15, v2
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[2:3]
-; GCN-NEXT:    v_xor_b32_e32 v1, s6, v1
-; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s6, v1
-; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NEXT:    s_sub_i32 s6, 0, s3
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
+; GCN-NEXT:    s_ashr_i32 s9, s1, 31
+; GCN-NEXT:    s_add_i32 s1, s1, s9
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
+; GCN-NEXT:    v_mul_hi_u32 v2, v1, v2
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s2, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s2, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s2, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s2, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GCN-NEXT:    s_xor_b32 s0, s1, s9
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GCN-NEXT:    v_mul_hi_u32 v1, s0, v1
+; GCN-NEXT:    v_xor_b32_e32 v0, s8, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s8, v0
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s3
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s0, v1
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s3, v1
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s3, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s3, v1
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s3, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-NEXT:    v_xor_b32_e32 v1, s9, v1
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s9, v1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
   %r = srem <2 x i32> %x, %shl.y
@@ -5319,7 +4919,7 @@ define amdgpu_kernel void @srem_v2i32_pow2_shl_denom(<2 x i32> addrspace(1)* %ou
 define amdgpu_kernel void @udiv_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {
 ; CHECK-LABEL: @udiv_i64_oddk_denom(
 ; CHECK-NEXT:    [[R:%.*]] = udiv i64 [[X:%.*]], 1235195949943
-; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_i64_oddk_denom:
@@ -5330,121 +4930,121 @@ define amdgpu_kernel void @udiv_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
 ; GCN-NEXT:    s_movk_i32 s2, 0xfee0
 ; GCN-NEXT:    s_mov_b32 s3, 0x68958c89
-; GCN-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
 ; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    v_mov_b32_e32 v7, 0
-; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s4, s8
+; GCN-NEXT:    s_movk_i32 s8, 0x11f
 ; GCN-NEXT:    v_mul_lo_u32 v2, v0, s2
 ; GCN-NEXT:    v_mul_hi_u32 v3, v0, s3
 ; GCN-NEXT:    v_mul_lo_u32 v4, v1, s3
-; GCN-NEXT:    s_mov_b32 s11, 0xf000
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s8, s4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, s3
+; GCN-NEXT:    s_mov_b32 s5, s9
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_mul_lo_u32 v3, v0, s3
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
-; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v4, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v6, v0, v3
-; GCN-NEXT:    v_mul_hi_u32 v9, v1, v2
+; GCN-NEXT:    v_mul_lo_u32 v3, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v4, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GCN-NEXT:    s_movk_i32 s4, 0x11e
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; GCN-NEXT:    v_mul_lo_u32 v6, v1, v3
-; GCN-NEXT:    v_mul_hi_u32 v3, v1, v3
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v8, v4, vcc
-; GCN-NEXT:    s_mov_b32 s10, -1
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v4, v3, vcc
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v9, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v6, vcc
+; GCN-NEXT:    v_mul_lo_u32 v6, v1, v5
+; GCN-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GCN-NEXT:    s_movk_i32 s9, 0x11e
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v4, v5, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
 ; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, v0, s2
-; GCN-NEXT:    v_mul_hi_u32 v5, v0, s3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, s2
+; GCN-NEXT:    v_mul_hi_u32 v7, v0, s3
 ; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v6, v2, s3
+; GCN-NEXT:    v_mul_lo_u32 v8, v2, s3
 ; GCN-NEXT:    s_mov_b32 s2, 0x976a7377
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_mul_lo_u32 v5, v0, s3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
-; GCN-NEXT:    v_mul_lo_u32 v6, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v10, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v9, v0, v5
-; GCN-NEXT:    v_mul_hi_u32 v11, v2, v4
-; GCN-NEXT:    s_movk_i32 s3, 0x11f
-; GCN-NEXT:    s_mov_b32 s9, s5
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
-; GCN-NEXT:    v_addc_u32_e32 v9, vcc, v8, v10, vcc
-; GCN-NEXT:    v_mul_lo_u32 v10, v2, v5
-; GCN-NEXT:    v_mul_hi_u32 v5, v2, v5
-; GCN-NEXT:    v_mul_lo_u32 v2, v2, v4
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v6, v10
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v9, v5, vcc
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v11, v7, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GCN-NEXT:    v_mul_lo_u32 v7, v0, s3
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; GCN-NEXT:    v_mul_lo_u32 v8, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v10, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v9, v0, v7
+; GCN-NEXT:    v_mul_hi_u32 v11, v2, v5
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v10, vcc
+; GCN-NEXT:    v_mul_lo_u32 v10, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v9, v7, vcc
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v11, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
-; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v4, s[0:1]
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
-; GCN-NEXT:    v_mul_hi_u32 v4, s6, v1
-; GCN-NEXT:    v_mul_hi_u32 v5, s7, v1
-; GCN-NEXT:    v_mul_lo_u32 v1, s7, v1
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
-; GCN-NEXT:    v_mul_hi_u32 v0, s7, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v7, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v8, v2, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, v0, s3
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v0, s8
 ; GCN-NEXT:    v_mul_hi_u32 v3, v0, s2
 ; GCN-NEXT:    v_mul_lo_u32 v4, v1, s2
-; GCN-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-NEXT:    v_mov_b32_e32 v5, s8
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_mul_lo_u32 v3, v0, s2
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s7, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s6, v3
-; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s2, v3
-; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s4, v4
-; GCN-NEXT:    s_mov_b32 s2, 0x976a7376
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s2, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s3, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v5, s[0:1], 2, v0
-; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 1, v0
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v6, s7
-; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v6, v2, vcc
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v2
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s11, v2
+; GCN-NEXT:    v_sub_i32_e64 v3, s[0:1], s10, v3
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v4, v5, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s2, v3
+; GCN-NEXT:    v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s9, v4
+; GCN-NEXT:    s_mov_b32 s10, 0x976a7376
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s10, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s8, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[2:3]
+; GCN-NEXT:    v_mov_b32_e32 v6, s11
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v6, v2, s[0:1]
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s9, v2
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s2, v3
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s10, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s8, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = udiv i64 %x, 1235195949943
   store i64 %r, i64 addrspace(1)* %out
@@ -5454,7 +5054,7 @@ define amdgpu_kernel void @udiv_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {
 define amdgpu_kernel void @udiv_i64_pow2k_denom(i64 addrspace(1)* %out, i64 %x) {
 ; CHECK-LABEL: @udiv_i64_pow2k_denom(
 ; CHECK-NEXT:    [[R:%.*]] = udiv i64 [[X:%.*]], 4096
-; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_i64_pow2k_denom:
@@ -5479,7 +5079,7 @@ define amdgpu_kernel void @udiv_i64_pow2_shl_denom(i64 addrspace(1)* %out, i64 %
 ; CHECK-LABEL: @udiv_i64_pow2_shl_denom(
 ; CHECK-NEXT:    [[SHL_Y:%.*]] = shl i64 4096, [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = udiv i64 [[X:%.*]], [[SHL_Y]]
-; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_i64_pow2_shl_denom:
@@ -5511,7 +5111,7 @@ define amdgpu_kernel void @udiv_v2i64_pow2k_denom(<2 x i64> addrspace(1)* %out,
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[X]], i64 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = udiv i64 [[TMP4]], 4096
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[TMP3]], i64 [[TMP5]], i64 1
-; CHECK-NEXT:    store <2 x i64> [[TMP6]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <2 x i64> [[TMP6]], <2 x i64> addrspace(1)* [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_v2i64_pow2k_denom:
@@ -5542,7 +5142,7 @@ define amdgpu_kernel void @udiv_v2i64_mixed_pow2k_denom(<2 x i64> addrspace(1)*
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[X]], i64 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = udiv i64 [[TMP4]], 4095
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[TMP3]], i64 [[TMP5]], i64 1
-; CHECK-NEXT:    store <2 x i64> [[TMP6]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <2 x i64> [[TMP6]], <2 x i64> addrspace(1)* [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_v2i64_mixed_pow2k_denom:
@@ -5619,6 +5219,7 @@ define amdgpu_kernel void @udiv_v2i64_mixed_pow2k_denom(<2 x i64> addrspace(1)*
 ; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
 ; GCN-NEXT:    s_lshr_b64 s[2:3], s[8:9], 12
+; GCN-NEXT:    s_movk_i32 s8, 0xffe
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v4, v0, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v6, v2, vcc
@@ -5633,25 +5234,24 @@ define amdgpu_kernel void @udiv_v2i64_mixed_pow2k_denom(<2 x i64> addrspace(1)*
 ; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v3, v2, vcc
 ; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s0, v4
 ; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v2, vcc
-; GCN-NEXT:    s_movk_i32 s0, 0xffe
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s0, v3
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s8, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, -1, v3, vcc
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s0, v4
 ; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v2, -1, v4, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v8, v6, vcc
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v3, v1, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v7, v5, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v0, v1, s[0:1]
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s8, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v2, -1, v4, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v8, v6, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v1, v3, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
 ; GCN-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
@@ -5672,7 +5272,7 @@ define amdgpu_kernel void @udiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[SHL_Y]], i64 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = udiv i64 [[TMP5]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> [[TMP4]], i64 [[TMP7]], i64 1
-; CHECK-NEXT:    store <2 x i64> [[TMP8]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <2 x i64> [[TMP8]], <2 x i64> addrspace(1)* [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: udiv_v2i64_pow2_shl_denom:
@@ -5702,7 +5302,7 @@ define amdgpu_kernel void @udiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 define amdgpu_kernel void @urem_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {
 ; CHECK-LABEL: @urem_i64_oddk_denom(
 ; CHECK-NEXT:    [[R:%.*]] = urem i64 [[X:%.*]], 1235195393993
-; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_i64_oddk_denom:
@@ -5713,120 +5313,120 @@ define amdgpu_kernel void @urem_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
 ; GCN-NEXT:    s_movk_i32 s2, 0xfee0
 ; GCN-NEXT:    s_mov_b32 s3, 0x689e0837
-; GCN-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
 ; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    v_mov_b32_e32 v7, 0
-; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s4, s8
+; GCN-NEXT:    s_movk_i32 s8, 0x11f
 ; GCN-NEXT:    v_mul_lo_u32 v2, v0, s2
 ; GCN-NEXT:    v_mul_hi_u32 v3, v0, s3
 ; GCN-NEXT:    v_mul_lo_u32 v4, v1, s3
-; GCN-NEXT:    s_movk_i32 s12, 0x11f
-; GCN-NEXT:    s_mov_b32 s13, 0x9761f7c9
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, s3
+; GCN-NEXT:    s_mov_b32 s12, 0x9761f7c9
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_mul_lo_u32 v3, v0, s3
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
-; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v4, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v6, v0, v3
-; GCN-NEXT:    v_mul_hi_u32 v9, v1, v2
+; GCN-NEXT:    v_mul_lo_u32 v3, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v4, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s9, s5
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; GCN-NEXT:    v_mul_lo_u32 v6, v1, v3
-; GCN-NEXT:    v_mul_hi_u32 v3, v1, v3
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v8, v4, vcc
-; GCN-NEXT:    s_movk_i32 s5, 0x11e
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v4, v3, vcc
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v9, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v6, vcc
+; GCN-NEXT:    v_mul_lo_u32 v6, v1, v5
+; GCN-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GCN-NEXT:    s_mov_b32 s5, s9
+; GCN-NEXT:    s_movk_i32 s9, 0x11e
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v4, v5, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
 ; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, v0, s2
-; GCN-NEXT:    v_mul_hi_u32 v5, v0, s3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, s2
+; GCN-NEXT:    v_mul_hi_u32 v7, v0, s3
 ; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v6, v2, s3
-; GCN-NEXT:    s_mov_b32 s8, s4
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_mul_lo_u32 v5, v0, s3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
-; GCN-NEXT:    v_mul_lo_u32 v6, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v10, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v9, v0, v5
-; GCN-NEXT:    v_mul_hi_u32 v11, v2, v4
-; GCN-NEXT:    s_mov_b32 s4, 0x9761f7c8
-; GCN-NEXT:    s_mov_b32 s11, 0xf000
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
-; GCN-NEXT:    v_addc_u32_e32 v9, vcc, v8, v10, vcc
-; GCN-NEXT:    v_mul_lo_u32 v10, v2, v5
-; GCN-NEXT:    v_mul_hi_u32 v5, v2, v5
-; GCN-NEXT:    v_mul_lo_u32 v2, v2, v4
-; GCN-NEXT:    s_mov_b32 s10, -1
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v6, v10
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v9, v5, vcc
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v11, v7, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v8, v4, vcc
+; GCN-NEXT:    v_mul_lo_u32 v8, v2, s3
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GCN-NEXT:    v_mul_lo_u32 v7, v0, s3
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; GCN-NEXT:    v_mul_lo_u32 v8, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v10, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v9, v0, v7
+; GCN-NEXT:    v_mul_hi_u32 v11, v2, v5
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v10, vcc
+; GCN-NEXT:    v_mul_lo_u32 v10, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v9, v7, vcc
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v11, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
-; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v4, s[0:1]
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
-; GCN-NEXT:    v_mul_hi_u32 v4, s6, v1
-; GCN-NEXT:    v_mul_hi_u32 v5, s7, v1
-; GCN-NEXT:    v_mul_lo_u32 v1, s7, v1
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
-; GCN-NEXT:    v_mul_hi_u32 v0, s7, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v7, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v8, v2, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, v0, s12
-; GCN-NEXT:    v_mul_hi_u32 v3, v0, s13
-; GCN-NEXT:    v_mul_lo_u32 v1, v1, s13
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s13
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v0, s8
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, s12
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s12
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s12
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s7, v1
-; GCN-NEXT:    v_mov_b32_e32 v3, s12
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s13, v0
-; GCN-NEXT:    v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[2:3], s5, v5
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[2:3], s4, v4
-; GCN-NEXT:    v_subrev_i32_e64 v3, s[0:1], s13, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s12, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[2:3]
-; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v5, s7
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v5, v1, vcc
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s5, v1
+; GCN-NEXT:    v_sub_i32_e64 v0, s[0:1], s10, v0
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s11, v1
+; GCN-NEXT:    v_mov_b32_e32 v3, s8
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v4, s[2:3], s12, v0
+; GCN-NEXT:    v_subbrev_u32_e64 v5, vcc, 0, v2, s[2:3]
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s9, v5
+; GCN-NEXT:    s_mov_b32 s10, 0x9761f7c8
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s10, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s8, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s12, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
+; GCN-NEXT:    v_mov_b32_e32 v5, s11
+; GCN-NEXT:    v_subb_u32_e64 v1, vcc, v5, v1, s[0:1]
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s9, v1
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v0
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s10, v0
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s12, v1
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s8, v1
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = urem i64 %x, 1235195393993
   store i64 %r, i64 addrspace(1)* %out
@@ -5836,7 +5436,7 @@ define amdgpu_kernel void @urem_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {
 define amdgpu_kernel void @urem_i64_pow2k_denom(i64 addrspace(1)* %out, i64 %x) {
 ; CHECK-LABEL: @urem_i64_pow2k_denom(
 ; CHECK-NEXT:    [[R:%.*]] = urem i64 [[X:%.*]], 4096
-; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_i64_pow2k_denom:
@@ -5861,7 +5461,7 @@ define amdgpu_kernel void @urem_i64_pow2_shl_denom(i64 addrspace(1)* %out, i64 %
 ; CHECK-LABEL: @urem_i64_pow2_shl_denom(
 ; CHECK-NEXT:    [[SHL_Y:%.*]] = shl i64 4096, [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = urem i64 [[X:%.*]], [[SHL_Y]]
-; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_i64_pow2_shl_denom:
@@ -5897,7 +5497,7 @@ define amdgpu_kernel void @urem_v2i64_pow2k_denom(<2 x i64> addrspace(1)* %out,
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[X]], i64 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = urem i64 [[TMP4]], 4096
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[TMP3]], i64 [[TMP5]], i64 1
-; CHECK-NEXT:    store <2 x i64> [[TMP6]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <2 x i64> [[TMP6]], <2 x i64> addrspace(1)* [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_v2i64_pow2k_denom:
@@ -5932,7 +5532,7 @@ define amdgpu_kernel void @urem_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[SHL_Y]], i64 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = urem i64 [[TMP5]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> [[TMP4]], i64 [[TMP7]], i64 1
-; CHECK-NEXT:    store <2 x i64> [[TMP8]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <2 x i64> [[TMP8]], <2 x i64> addrspace(1)* [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: urem_v2i64_pow2_shl_denom:
@@ -5968,7 +5568,7 @@ define amdgpu_kernel void @urem_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 define amdgpu_kernel void @sdiv_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {
 ; CHECK-LABEL: @sdiv_i64_oddk_denom(
 ; CHECK-NEXT:    [[R:%.*]] = sdiv i64 [[X:%.*]], 1235195
-; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_i64_oddk_denom:
@@ -6059,30 +5659,30 @@ define amdgpu_kernel void @sdiv_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {
 ; GCN-NEXT:    v_mul_hi_u32 v3, s3, v0
 ; GCN-NEXT:    v_mul_lo_u32 v4, v0, s3
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s0, v4
 ; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s0, v4
 ; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v3, v2, vcc
 ; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s3, v4
 ; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v2, vcc
-; GCN-NEXT:    s_mov_b32 s0, 0x12d8fa
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s0, v3
+; GCN-NEXT:    s_mov_b32 s3, 0x12d8fa
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s3, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, -1, v3, vcc
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s0, v4
 ; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v2, -1, v4, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v7, v5, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v8, v6, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s3, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v2, -1, v4, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v8, v6, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v0, s2, v0
 ; GCN-NEXT:    v_xor_b32_e32 v1, s2, v1
 ; GCN-NEXT:    v_mov_b32_e32 v2, s2
@@ -6098,7 +5698,7 @@ define amdgpu_kernel void @sdiv_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {
 define amdgpu_kernel void @sdiv_i64_pow2k_denom(i64 addrspace(1)* %out, i64 %x) {
 ; CHECK-LABEL: @sdiv_i64_pow2k_denom(
 ; CHECK-NEXT:    [[R:%.*]] = sdiv i64 [[X:%.*]], 4096
-; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_i64_pow2k_denom:
@@ -6127,7 +5727,7 @@ define amdgpu_kernel void @sdiv_i64_pow2_shl_denom(i64 addrspace(1)* %out, i64 %
 ; CHECK-LABEL: @sdiv_i64_pow2_shl_denom(
 ; CHECK-NEXT:    [[SHL_Y:%.*]] = shl i64 4096, [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = sdiv i64 [[X:%.*]], [[SHL_Y]]
-; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_i64_pow2_shl_denom:
@@ -6143,26 +5743,28 @@ define amdgpu_kernel void @sdiv_i64_pow2_shl_denom(i64 addrspace(1)* %out, i64 %
 ; GCN-NEXT:    s_add_u32 s2, s2, s12
 ; GCN-NEXT:    s_mov_b32 s13, s12
 ; GCN-NEXT:    s_addc_u32 s3, s3, s12
-; GCN-NEXT:    s_xor_b64 s[2:3], s[2:3], s[12:13]
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s2
-; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s3
-; GCN-NEXT:    s_sub_u32 s4, 0, s2
-; GCN-NEXT:    s_subb_u32 s5, 0, s3
-; GCN-NEXT:    s_ashr_i32 s14, s11, 31
+; GCN-NEXT:    s_xor_b64 s[14:15], s[2:3], s[12:13]
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s14
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s15
+; GCN-NEXT:    s_sub_u32 s2, 0, s14
+; GCN-NEXT:    s_subb_u32 s3, 0, s15
+; GCN-NEXT:    s_ashr_i32 s16, s11, 31
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-NEXT:    s_mov_b32 s15, s14
+; GCN-NEXT:    s_mov_b32 s17, s16
 ; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_mov_b32 s4, s8
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
 ; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, s4, v0
-; GCN-NEXT:    v_mul_lo_u32 v2, s4, v1
-; GCN-NEXT:    v_mul_lo_u32 v5, s5, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s4, v0
+; GCN-NEXT:    s_mov_b32 s5, s9
+; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
+; GCN-NEXT:    v_mul_lo_u32 v5, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s2, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
 ; GCN-NEXT:    v_mul_hi_u32 v3, v0, v4
@@ -6183,12 +5785,11 @@ define amdgpu_kernel void @sdiv_i64_pow2_shl_denom(i64 addrspace(1)* %out, i64 %
 ; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
 ; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v5, s4, v2
-; GCN-NEXT:    v_mul_hi_u32 v7, s4, v0
-; GCN-NEXT:    v_mul_lo_u32 v8, s5, v0
-; GCN-NEXT:    s_mov_b32 s5, s9
+; GCN-NEXT:    v_mul_lo_u32 v5, s2, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v8, s3, v0
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
-; GCN-NEXT:    v_mul_lo_u32 v7, s4, v0
+; GCN-NEXT:    v_mul_lo_u32 v7, s2, v0
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
 ; GCN-NEXT:    v_mul_lo_u32 v10, v0, v5
 ; GCN-NEXT:    v_mul_hi_u32 v12, v0, v5
@@ -6206,10 +5807,10 @@ define amdgpu_kernel void @sdiv_i64_pow2_shl_denom(i64 addrspace(1)* %out, i64 %
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
-; GCN-NEXT:    s_add_u32 s0, s10, s14
-; GCN-NEXT:    s_addc_u32 s1, s11, s14
+; GCN-NEXT:    s_add_u32 s0, s10, s16
+; GCN-NEXT:    s_addc_u32 s1, s11, s16
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    s_xor_b64 s[10:11], s[0:1], s[14:15]
+; GCN-NEXT:    s_xor_b64 s[10:11], s[0:1], s[16:17]
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
 ; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
@@ -6220,48 +5821,47 @@ define amdgpu_kernel void @sdiv_i64_pow2_shl_denom(i64 addrspace(1)* %out, i64 %
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
-; GCN-NEXT:    s_mov_b32 s4, s8
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s3, v0
-; GCN-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-NEXT:    v_mul_lo_u32 v2, s14, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s14, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s15, v0
+; GCN-NEXT:    v_mov_b32_e32 v5, s15
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_mul_lo_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v3, s14, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s11, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s10, v3
-; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s2, v3
-; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s3, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s2, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s3, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v5, s[0:1], 2, v0
-; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 1, v0
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[0:1]
+; GCN-NEXT:    v_sub_i32_e64 v3, s[0:1], s10, v3
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v4, v5, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s14, v3
+; GCN-NEXT:    v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s15, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s14, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s15, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v6, s11
-; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v6, v2, vcc
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v6, v2, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s15, v2
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s2, v3
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s14, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s15, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    s_xor_b64 s[0:1], s[14:15], s[12:13]
+; GCN-NEXT:    s_xor_b64 s[0:1], s[16:17], s[12:13]
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v0, s0, v0
 ; GCN-NEXT:    v_xor_b32_e32 v1, s1, v1
@@ -6284,7 +5884,7 @@ define amdgpu_kernel void @sdiv_v2i64_pow2k_denom(<2 x i64> addrspace(1)* %out,
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[X]], i64 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = sdiv i64 [[TMP4]], 4096
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[TMP3]], i64 [[TMP5]], i64 1
-; CHECK-NEXT:    store <2 x i64> [[TMP6]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <2 x i64> [[TMP6]], <2 x i64> addrspace(1)* [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_v2i64_pow2k_denom:
@@ -6323,7 +5923,7 @@ define amdgpu_kernel void @ssdiv_v2i64_mixed_pow2k_denom(<2 x i64> addrspace(1)*
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[X]], i64 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = sdiv i64 [[TMP4]], 4095
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[TMP3]], i64 [[TMP5]], i64 1
-; CHECK-NEXT:    store <2 x i64> [[TMP6]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <2 x i64> [[TMP6]], <2 x i64> addrspace(1)* [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: ssdiv_v2i64_mixed_pow2k_denom:
@@ -6419,30 +6019,30 @@ define amdgpu_kernel void @ssdiv_v2i64_mixed_pow2k_denom(<2 x i64> addrspace(1)*
 ; GCN-NEXT:    v_mul_hi_u32 v3, s9, v0
 ; GCN-NEXT:    v_mul_lo_u32 v4, v0, s9
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s0, v4
 ; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s0, v4
 ; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v3, v2, vcc
 ; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s9, v4
 ; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v2, vcc
-; GCN-NEXT:    s_movk_i32 s0, 0xffe
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s0, v3
+; GCN-NEXT:    s_movk_i32 s9, 0xffe
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s9, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, -1, v3, vcc
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s0, v4
 ; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v2, -1, v4, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v7, v5, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v8, v6, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s9, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v2, -1, v4, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v8, v6, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v0, s8, v0
 ; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s8, v0
 ; GCN-NEXT:    v_xor_b32_e32 v1, s8, v1
@@ -6468,7 +6068,7 @@ define amdgpu_kernel void @sdiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[SHL_Y]], i64 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = sdiv i64 [[TMP5]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> [[TMP4]], i64 [[TMP7]], i64 1
-; CHECK-NEXT:    store <2 x i64> [[TMP8]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <2 x i64> [[TMP8]], <2 x i64> addrspace(1)* [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_v2i64_pow2_shl_denom:
@@ -6476,8 +6076,8 @@ define amdgpu_kernel void @sdiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x11
 ; GCN-NEXT:    s_mov_b32 s3, 0
 ; GCN-NEXT:    s_movk_i32 s2, 0x1000
-; GCN-NEXT:    s_mov_b32 s18, 0x4f800000
-; GCN-NEXT:    s_mov_b32 s19, 0x5f7ffffc
+; GCN-NEXT:    s_mov_b32 s20, 0x4f800000
+; GCN-NEXT:    s_mov_b32 s21, 0x5f7ffffc
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_lshl_b64 s[12:13], s[2:3], s6
 ; GCN-NEXT:    s_lshl_b64 s[2:3], s[2:3], s4
@@ -6488,24 +6088,28 @@ define amdgpu_kernel void @sdiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; GCN-NEXT:    s_xor_b64 s[14:15], s[2:3], s[16:17]
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s14
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s15
-; GCN-NEXT:    s_mov_b32 s20, 0x2f800000
-; GCN-NEXT:    s_mov_b32 s21, 0xcf800000
+; GCN-NEXT:    s_mov_b32 s22, 0x2f800000
+; GCN-NEXT:    s_mov_b32 s23, 0xcf800000
 ; GCN-NEXT:    s_sub_u32 s6, 0, s14
-; GCN-NEXT:    v_mac_f32_e32 v0, s18, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, s20, v1
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
 ; GCN-NEXT:    s_subb_u32 s7, 0, s15
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0xd
-; GCN-NEXT:    v_mul_f32_e32 v0, s19, v0
-; GCN-NEXT:    v_mul_f32_e32 v1, s20, v0
+; GCN-NEXT:    v_mul_f32_e32 v0, s21, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, s22, v0
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
-; GCN-NEXT:    v_mac_f32_e32 v0, s21, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, s23, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s18, s9, 31
+; GCN-NEXT:    s_add_u32 s0, s8, s18
 ; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
 ; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
 ; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
 ; GCN-NEXT:    v_mul_lo_u32 v5, s6, v0
+; GCN-NEXT:    s_mov_b32 s19, s18
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_mul_lo_u32 v3, v0, v2
@@ -6517,6 +6121,8 @@ define amdgpu_kernel void @sdiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v6, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v6, v1, v5
 ; GCN-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GCN-NEXT:    s_addc_u32 s1, s9, s18
+; GCN-NEXT:    s_xor_b64 s[8:9], s[0:1], s[18:19]
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v4, v5, vcc
 ; GCN-NEXT:    v_mov_b32_e32 v4, 0
@@ -6549,13 +6155,7 @@ define amdgpu_kernel void @sdiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[2:3]
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_ashr_i32 s2, s9, 31
-; GCN-NEXT:    s_add_u32 s0, s8, s2
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    s_mov_b32 s3, s2
-; GCN-NEXT:    s_addc_u32 s1, s9, s2
-; GCN-NEXT:    s_xor_b64 s[8:9], s[0:1], s[2:3]
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v2, s8, v1
 ; GCN-NEXT:    v_mul_hi_u32 v3, s8, v0
@@ -6566,7 +6166,6 @@ define amdgpu_kernel void @sdiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v5, s9, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s9, v0
-; GCN-NEXT:    s_xor_b64 s[2:3], s[2:3], s[16:17]
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
@@ -6581,68 +6180,71 @@ define amdgpu_kernel void @sdiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; GCN-NEXT:    v_mul_lo_u32 v3, s14, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
 ; GCN-NEXT:    v_sub_i32_e32 v5, vcc, s9, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s8, v3
-; GCN-NEXT:    v_subb_u32_e64 v5, s[0:1], v5, v7, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v7, s[0:1], s14, v3
-; GCN-NEXT:    v_subbrev_u32_e64 v5, s[0:1], 0, v5, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s15, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s14, v7
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s15, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v5, v8, v7, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 2, v0
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v9, s[0:1], 1, v0
-; GCN-NEXT:    v_addc_u32_e64 v10, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    s_ashr_i32 s8, s13, 31
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v5
-; GCN-NEXT:    s_add_u32 s12, s12, s8
-; GCN-NEXT:    v_cndmask_b32_e64 v5, v10, v8, s[0:1]
+; GCN-NEXT:    v_sub_i32_e64 v3, s[0:1], s8, v3
+; GCN-NEXT:    v_subb_u32_e64 v5, vcc, v5, v7, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v7, vcc, s14, v3
+; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s15, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s14, v7
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s15, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v5, v8, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 1, v0
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v1, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v10, v8, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v8, s9
-; GCN-NEXT:    s_mov_b32 s9, s8
-; GCN-NEXT:    s_addc_u32 s13, s13, s8
-; GCN-NEXT:    s_xor_b64 s[12:13], s[12:13], s[8:9]
+; GCN-NEXT:    s_xor_b64 s[8:9], s[18:19], s[16:17]
+; GCN-NEXT:    s_ashr_i32 s16, s13, 31
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v8, v2, s[0:1]
+; GCN-NEXT:    s_add_u32 s0, s12, s16
+; GCN-NEXT:    s_mov_b32 s17, s16
+; GCN-NEXT:    s_addc_u32 s1, s13, s16
+; GCN-NEXT:    s_xor_b64 s[12:13], s[0:1], s[16:17]
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v10, s12
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v11, s13
-; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v8, v2, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s15, v2
 ; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s14, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s15, v2
-; GCN-NEXT:    v_mac_f32_e32 v10, s18, v11
+; GCN-NEXT:    v_mac_f32_e32 v10, s20, v11
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v8, v3, vcc
 ; GCN-NEXT:    v_rcp_f32_e32 v3, v10
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GCN-NEXT:    s_sub_u32 s14, 0, s12
-; GCN-NEXT:    v_mul_f32_e32 v3, s19, v3
-; GCN-NEXT:    v_mul_f32_e32 v5, s20, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v9, v7, s[2:3]
+; GCN-NEXT:    v_mul_f32_e32 v3, s21, v3
+; GCN-NEXT:    v_mul_f32_e32 v5, s22, v3
 ; GCN-NEXT:    v_trunc_f32_e32 v5, v5
-; GCN-NEXT:    v_mac_f32_e32 v3, s21, v5
+; GCN-NEXT:    v_mac_f32_e32 v3, s23, v5
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v9, v7, s[0:1]
+; GCN-NEXT:    s_sub_u32 s2, 0, s12
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    v_mul_hi_u32 v2, s14, v3
-; GCN-NEXT:    v_mul_lo_u32 v7, s14, v5
-; GCN-NEXT:    s_subb_u32 s15, 0, s13
-; GCN-NEXT:    v_mul_lo_u32 v8, s15, v3
-; GCN-NEXT:    v_xor_b32_e32 v0, s2, v0
+; GCN-NEXT:    v_mul_hi_u32 v2, s2, v3
+; GCN-NEXT:    v_mul_lo_u32 v7, s2, v5
+; GCN-NEXT:    s_subb_u32 s3, 0, s13
+; GCN-NEXT:    v_mul_lo_u32 v8, s3, v3
+; GCN-NEXT:    s_ashr_i32 s14, s11, 31
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v7
-; GCN-NEXT:    v_mul_lo_u32 v7, s14, v3
+; GCN-NEXT:    v_mul_lo_u32 v7, s2, v3
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v8
 ; GCN-NEXT:    v_mul_lo_u32 v8, v3, v2
 ; GCN-NEXT:    v_mul_hi_u32 v10, v3, v2
 ; GCN-NEXT:    v_mul_hi_u32 v9, v3, v7
 ; GCN-NEXT:    v_mul_hi_u32 v11, v5, v2
 ; GCN-NEXT:    v_mul_lo_u32 v2, v5, v2
-; GCN-NEXT:    v_xor_b32_e32 v1, s3, v1
+; GCN-NEXT:    s_mov_b32 s15, s14
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
 ; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v10, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v10, v5, v7
 ; GCN-NEXT:    v_mul_hi_u32 v7, v5, v7
+; GCN-NEXT:    v_xor_b32_e32 v0, s8, v0
+; GCN-NEXT:    v_xor_b32_e32 v1, s9, v1
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v9, v7, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v8, vcc, v11, v4, vcc
@@ -6650,11 +6252,11 @@ define amdgpu_kernel void @sdiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; GCN-NEXT:    v_add_i32_e64 v2, s[0:1], v3, v2
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v6, v8, vcc
 ; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v5, v7, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v8, s14, v3
-; GCN-NEXT:    v_mul_hi_u32 v9, s14, v2
-; GCN-NEXT:    v_mul_lo_u32 v10, s15, v2
+; GCN-NEXT:    v_mul_lo_u32 v8, s2, v3
+; GCN-NEXT:    v_mul_hi_u32 v9, s2, v2
+; GCN-NEXT:    v_mul_lo_u32 v10, s3, v2
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
-; GCN-NEXT:    v_mul_lo_u32 v9, s14, v2
+; GCN-NEXT:    v_mul_lo_u32 v9, s2, v2
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
 ; GCN-NEXT:    v_mul_lo_u32 v12, v2, v8
 ; GCN-NEXT:    v_mul_hi_u32 v14, v2, v8
@@ -6671,11 +6273,9 @@ define amdgpu_kernel void @sdiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v9, v3
 ; GCN-NEXT:    v_addc_u32_e32 v8, vcc, v6, v8, vcc
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
-; GCN-NEXT:    s_ashr_i32 s14, s11, 31
 ; GCN-NEXT:    v_addc_u32_e64 v5, vcc, v5, v8, s[0:1]
 ; GCN-NEXT:    s_add_u32 s0, s10, s14
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
-; GCN-NEXT:    s_mov_b32 s15, s14
 ; GCN-NEXT:    s_addc_u32 s1, s11, s14
 ; GCN-NEXT:    s_xor_b64 s[10:11], s[0:1], s[14:15]
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
@@ -6688,7 +6288,7 @@ define amdgpu_kernel void @sdiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v9, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v9, s11, v2
 ; GCN-NEXT:    v_mul_hi_u32 v2, s11, v2
-; GCN-NEXT:    v_mov_b32_e32 v8, s3
+; GCN-NEXT:    v_mov_b32_e32 v8, s9
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v2, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v10, v4, vcc
@@ -6696,32 +6296,32 @@ define amdgpu_kernel void @sdiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v4, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v4, s12, v3
 ; GCN-NEXT:    v_mul_hi_u32 v5, s12, v2
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s8, v0
 ; GCN-NEXT:    v_mul_lo_u32 v6, s13, v2
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v8, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
 ; GCN-NEXT:    v_mul_lo_u32 v5, s12, v2
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
 ; GCN-NEXT:    v_sub_i32_e32 v6, vcc, s11, v4
 ; GCN-NEXT:    v_mov_b32_e32 v7, s13
-; GCN-NEXT:    v_sub_i32_e32 v5, vcc, s10, v5
-; GCN-NEXT:    v_subb_u32_e64 v6, s[0:1], v6, v7, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v7, s[0:1], s12, v5
-; GCN-NEXT:    v_subbrev_u32_e64 v6, s[0:1], 0, v6, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s12, v7
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s13, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v8, v7, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 2, v2
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v3, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v9, s[0:1], 1, v2
-; GCN-NEXT:    v_addc_u32_e64 v10, s[0:1], 0, v3, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v10, v8, s[0:1]
+; GCN-NEXT:    v_sub_i32_e64 v5, s[0:1], s10, v5
+; GCN-NEXT:    v_subb_u32_e64 v6, vcc, v6, v7, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v7, vcc, s12, v5
+; GCN-NEXT:    v_subbrev_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v7
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v6
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v8, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 2, v2
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 1, v2
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v3, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v6, v10, v8, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v8, s11
-; GCN-NEXT:    v_subb_u32_e32 v4, vcc, v8, v4, vcc
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v8, v4, s[0:1]
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v4
 ; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v5
@@ -6729,9 +6329,9 @@ define amdgpu_kernel void @sdiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v4
 ; GCN-NEXT:    v_cndmask_b32_e32 v4, v8, v5, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v9, v7, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v9, v7, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GCN-NEXT:    s_xor_b64 s[0:1], s[14:15], s[8:9]
+; GCN-NEXT:    s_xor_b64 s[0:1], s[14:15], s[16:17]
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v2, s0, v2
 ; GCN-NEXT:    v_xor_b32_e32 v3, s1, v3
@@ -6749,7 +6349,7 @@ define amdgpu_kernel void @sdiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 define amdgpu_kernel void @srem_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {
 ; CHECK-LABEL: @srem_i64_oddk_denom(
 ; CHECK-NEXT:    [[R:%.*]] = srem i64 [[X:%.*]], 1235195
-; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_i64_oddk_denom:
@@ -6840,28 +6440,28 @@ define amdgpu_kernel void @srem_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {
 ; GCN-NEXT:    v_mul_lo_u32 v1, v1, s3
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s3
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
 ; GCN-NEXT:    v_mov_b32_e32 v2, s1
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
 ; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s3, v0
 ; GCN-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, s3, v2
 ; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v3, vcc
-; GCN-NEXT:    s_mov_b32 s0, 0x12d8fa
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s0, v2
+; GCN-NEXT:    s_mov_b32 s3, 0x12d8fa
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s3, v2
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GCN-NEXT:    v_cndmask_b32_e32 v6, -1, v6, vcc
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s0, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v1
-; GCN-NEXT:    v_cndmask_b32_e64 v5, -1, v5, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v5
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s3, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v5, -1, v5, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v4, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v0, s2, v0
 ; GCN-NEXT:    v_xor_b32_e32 v1, s2, v1
 ; GCN-NEXT:    v_mov_b32_e32 v2, s2
@@ -6877,7 +6477,7 @@ define amdgpu_kernel void @srem_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {
 define amdgpu_kernel void @srem_i64_pow2k_denom(i64 addrspace(1)* %out, i64 %x) {
 ; CHECK-LABEL: @srem_i64_pow2k_denom(
 ; CHECK-NEXT:    [[R:%.*]] = srem i64 [[X:%.*]], 4096
-; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_i64_pow2k_denom:
@@ -6908,7 +6508,7 @@ define amdgpu_kernel void @srem_i64_pow2_shl_denom(i64 addrspace(1)* %out, i64 %
 ; CHECK-LABEL: @srem_i64_pow2_shl_denom(
 ; CHECK-NEXT:    [[SHL_Y:%.*]] = shl i64 4096, [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = srem i64 [[X:%.*]], [[SHL_Y]]
-; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_i64_pow2_shl_denom:
@@ -7013,25 +6613,25 @@ define amdgpu_kernel void @srem_i64_pow2_shl_denom(i64 addrspace(1)* %out, i64 %
 ; GCN-NEXT:    v_mul_lo_u32 v0, s12, v0
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e64 v0, s[0:1], s10, v0
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s11, v1
 ; GCN-NEXT:    v_mov_b32_e32 v3, s13
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s10, v0
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s12, v0
-; GCN-NEXT:    v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s13, v5
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s12, v4
-; GCN-NEXT:    v_subrev_i32_e64 v3, s[0:1], s12, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s13, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[2:3]
-; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v4, s[2:3], s12, v0
+; GCN-NEXT:    v_subbrev_u32_e64 v5, vcc, 0, v2, s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s12, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v5, s11
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v5, v1, vcc
+; GCN-NEXT:    v_subb_u32_e64 v1, vcc, v5, v1, s[0:1]
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v1
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v0
@@ -7040,7 +6640,7 @@ define amdgpu_kernel void @srem_i64_pow2_shl_denom(i64 addrspace(1)* %out, i64 %
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v0, s14, v0
 ; GCN-NEXT:    v_xor_b32_e32 v1, s14, v1
@@ -7063,7 +6663,7 @@ define amdgpu_kernel void @srem_v2i64_pow2k_denom(<2 x i64> addrspace(1)* %out,
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[X]], i64 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = srem i64 [[TMP4]], 4096
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[TMP3]], i64 [[TMP5]], i64 1
-; CHECK-NEXT:    store <2 x i64> [[TMP6]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <2 x i64> [[TMP6]], <2 x i64> addrspace(1)* [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_v2i64_pow2k_denom:
@@ -7110,7 +6710,7 @@ define amdgpu_kernel void @srem_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[SHL_Y]], i64 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = srem i64 [[TMP5]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> [[TMP4]], i64 [[TMP7]], i64 1
-; CHECK-NEXT:    store <2 x i64> [[TMP8]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    store <2 x i64> [[TMP8]], <2 x i64> addrspace(1)* [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_v2i64_pow2_shl_denom:
@@ -7220,43 +6820,43 @@ define amdgpu_kernel void @srem_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; GCN-NEXT:    v_mul_lo_u32 v0, s16, v0
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e64 v0, s[0:1], s8, v0
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s9, v1
 ; GCN-NEXT:    v_mov_b32_e32 v3, s17
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s8, v0
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s16, v0
-; GCN-NEXT:    v_subbrev_u32_e64 v7, s[2:3], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s17, v7
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s16, v5
-; GCN-NEXT:    v_subrev_i32_e64 v3, s[0:1], s16, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s17, v7
-; GCN-NEXT:    v_cndmask_b32_e64 v8, v8, v9, s[2:3]
-; GCN-NEXT:    s_ashr_i32 s2, s15, 31
-; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v8
-; GCN-NEXT:    s_add_u32 s8, s14, s2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v2, s[0:1]
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v5, s[2:3], s16, v0
+; GCN-NEXT:    v_subbrev_u32_e64 v7, vcc, 0, v2, s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s17, v7
+; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s16, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v9, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s17, v7
+; GCN-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s16, v5
+; GCN-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v8
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v2, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v7, s9
-; GCN-NEXT:    s_mov_b32 s3, s2
-; GCN-NEXT:    s_addc_u32 s9, s15, s2
-; GCN-NEXT:    s_xor_b64 s[8:9], s[8:9], s[2:3]
-; GCN-NEXT:    v_cvt_f32_u32_e32 v8, s8
-; GCN-NEXT:    v_cvt_f32_u32_e32 v9, s9
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v7, v1, vcc
+; GCN-NEXT:    v_subb_u32_e64 v1, vcc, v7, v1, s[0:1]
+; GCN-NEXT:    s_ashr_i32 s0, s15, 31
+; GCN-NEXT:    s_add_u32 s8, s14, s0
+; GCN-NEXT:    s_mov_b32 s1, s0
+; GCN-NEXT:    s_addc_u32 s9, s15, s0
+; GCN-NEXT:    s_xor_b64 s[8:9], s[8:9], s[0:1]
+; GCN-NEXT:    v_cvt_f32_u32_e32 v9, s8
+; GCN-NEXT:    v_cvt_f32_u32_e32 v10, s9
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s17, v1
 ; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
-; GCN-NEXT:    v_mac_f32_e32 v8, s18, v9
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s16, v0
-; GCN-NEXT:    v_rcp_f32_e32 v8, v8
-; GCN-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s17, v1
-; GCN-NEXT:    v_cndmask_b32_e32 v7, v7, v10, vcc
+; GCN-NEXT:    v_mac_f32_e32 v9, s18, v10
+; GCN-NEXT:    v_cndmask_b32_e32 v7, v7, v8, vcc
+; GCN-NEXT:    v_rcp_f32_e32 v8, v9
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v3, s[2:3]
 ; GCN-NEXT:    v_mul_f32_e32 v3, s19, v8
 ; GCN-NEXT:    v_mul_f32_e32 v5, s20, v3
 ; GCN-NEXT:    v_trunc_f32_e32 v5, v5
@@ -7337,30 +6937,30 @@ define amdgpu_kernel void @srem_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; GCN-NEXT:    v_mul_lo_u32 v3, s8, v3
 ; GCN-NEXT:    v_mul_hi_u32 v4, s8, v2
 ; GCN-NEXT:    v_mul_lo_u32 v5, s9, v2
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s12, v0
 ; GCN-NEXT:    v_mul_lo_u32 v2, s8, v2
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s12, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v8, vcc
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_sub_i32_e64 v2, s[0:1], s10, v2
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s11, v3
 ; GCN-NEXT:    v_mov_b32_e32 v5, s9
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s10, v2
-; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v6, s[0:1], s8, v2
-; GCN-NEXT:    v_subbrev_u32_e64 v7, s[2:3], 0, v4, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s9, v7
-; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s8, v6
-; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s8, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s9, v7
-; GCN-NEXT:    v_cndmask_b32_e64 v8, v8, v9, s[2:3]
-; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v8
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v7, v4, s[0:1]
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v4, v5, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v6, s[2:3], s8, v2
+; GCN-NEXT:    v_subbrev_u32_e64 v7, vcc, 0, v4, s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v7
+; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s8, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v9, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s9, v7
+; GCN-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v4, v5, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s8, v6
+; GCN-NEXT:    v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v8
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v7, v4, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v7, s11
-; GCN-NEXT:    v_subb_u32_e32 v3, vcc, v7, v3, vcc
+; GCN-NEXT:    v_subb_u32_e64 v3, vcc, v7, v3, s[0:1]
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s8, v2
@@ -7369,7 +6969,7 @@ define amdgpu_kernel void @srem_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %ou
 ; GCN-NEXT:    v_cndmask_b32_e32 v7, v7, v8, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v2, s14, v2
 ; GCN-NEXT:    v_xor_b32_e32 v3, s14, v3
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-unroll-threshold.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-unroll-threshold.ll
index 4da11ca77956b..edc93455ef7d9 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-unroll-threshold.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-unroll-threshold.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -S -mtriple=amdgcn-- -basicaa -loop-unroll | FileCheck %s
+; RUN: opt < %s -S -mtriple=amdgcn-- -basic-aa -loop-unroll | FileCheck %s
 
 ; Check that the loop in unroll_default is not fully unrolled using the default
 ; unroll threshold
diff --git a/llvm/test/CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll b/llvm/test/CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll
index 3d75eca93cb48..8f4f1c3915351 100644
--- a/llvm/test/CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll
@@ -3,7 +3,7 @@
 
 ; GCN-LABEL: {{^}}amdhsa_trap_num_sgprs
 ; TRAP-HANDLER-ENABLE:  NumSgprs: 61
-; TRAP-HANDLER-DISABLE: NumSgprs: 79
+; TRAP-HANDLER-DISABLE: NumSgprs: 77
 define amdgpu_kernel void @amdhsa_trap_num_sgprs(
     i32 addrspace(1)* %out0, i32 %in0,
     i32 addrspace(1)* %out1, i32 %in1,
diff --git a/llvm/test/CodeGen/AMDGPU/bypass-div.ll b/llvm/test/CodeGen/AMDGPU/bypass-div.ll
index 5cc320a3658b4..d10d911aeac07 100644
--- a/llvm/test/CodeGen/AMDGPU/bypass-div.ll
+++ b/llvm/test/CodeGen/AMDGPU/bypass-div.ll
@@ -138,36 +138,32 @@ define i64 @sdiv64(i64 %a, i64 %b) {
 ; GFX9-NEXT:    v_sub_co_u32_e32 v3, vcc, v3, v5
 ; GFX9-NEXT:    v_subb_co_u32_e32 v4, vcc, v1, v5, vcc
 ; GFX9-NEXT:  BB0_2: ; %Flow
-; GFX9-NEXT:    s_or_saveexec_b64 s[6:7], s[6:7]
-; GFX9-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], s[6:7]
+; GFX9-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz BB0_4
 ; GFX9-NEXT:  ; %bb.3:
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, v2
+; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v2
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
+; GFX9-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT:    v_mul_lo_u32 v3, v1, v2
-; GFX9-NEXT:    v_mul_hi_u32 v4, v1, v2
-; GFX9-NEXT:    v_sub_u32_e32 v5, 0, v3
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v3, v3, v1
-; GFX9-NEXT:    v_add_u32_e32 v4, v1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v1
+; GFX9-NEXT:    v_mul_hi_u32 v3, v1, v3
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; GFX9-NEXT:    v_mul_lo_u32 v3, v1, v2
 ; GFX9-NEXT:    v_add_u32_e32 v4, 1, v1
-; GFX9-NEXT:    v_add_u32_e32 v5, -1, v1
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v3
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v2
-; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v1, v4, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v0, vcc
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v3, v0, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT:    v_add_u32_e32 v3, 1, v1
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v1, v3, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:  BB0_4:
-; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -293,36 +289,32 @@ define i64 @udiv64(i64 %a, i64 %b) {
 ; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
 ; GFX9-NEXT:  BB1_2: ; %Flow
-; GFX9-NEXT:    s_or_saveexec_b64 s[6:7], s[6:7]
-; GFX9-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], s[6:7]
+; GFX9-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz BB1_4
 ; GFX9-NEXT:  ; %bb.3:
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, v2
+; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
+; GFX9-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT:    v_mul_lo_u32 v3, v1, v2
-; GFX9-NEXT:    v_mul_hi_u32 v4, v1, v2
-; GFX9-NEXT:    v_sub_u32_e32 v5, 0, v3
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v3, v3, v1
-; GFX9-NEXT:    v_add_u32_e32 v4, v1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v1
+; GFX9-NEXT:    v_mul_hi_u32 v3, v1, v3
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; GFX9-NEXT:    v_mul_lo_u32 v3, v1, v2
 ; GFX9-NEXT:    v_add_u32_e32 v4, 1, v1
-; GFX9-NEXT:    v_add_u32_e32 v5, -1, v1
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v3
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v2
-; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v1, v4, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v0, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v5, 0
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v3, v0, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; GFX9-NEXT:    v_add_u32_e32 v3, 1, v1
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v1, v3, vcc
 ; GFX9-NEXT:  BB1_4:
-; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -462,36 +454,30 @@ define i64 @srem64(i64 %a, i64 %b) {
 ; GFX9-NEXT:    v_sub_co_u32_e32 v3, vcc, v3, v7
 ; GFX9-NEXT:    v_subb_co_u32_e32 v4, vcc, v1, v7, vcc
 ; GFX9-NEXT:  BB2_2: ; %Flow
-; GFX9-NEXT:    s_or_saveexec_b64 s[6:7], s[8:9]
-; GFX9-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], s[8:9]
+; GFX9-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz BB2_4
 ; GFX9-NEXT:  ; %bb.3:
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, v2
+; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
+; GFX9-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT:    v_mul_lo_u32 v3, v1, v2
-; GFX9-NEXT:    v_mul_hi_u32 v4, v1, v2
-; GFX9-NEXT:    v_sub_u32_e32 v5, 0, v3
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v3, v3, v1
-; GFX9-NEXT:    v_add_u32_e32 v4, v1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v1
+; GFX9-NEXT:    v_mul_hi_u32 v3, v1, v3
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; GFX9-NEXT:    v_mul_lo_u32 v1, v1, v2
-; GFX9-NEXT:    v_sub_u32_e32 v3, v0, v1
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v3, v2
-; GFX9-NEXT:    v_sub_u32_e32 v0, v3, v2
-; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; GFX9-NEXT:    v_add_u32_e32 v4, v3, v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, v0, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v0, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v1, v0, v2
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v1, v0, v2
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
 ; GFX9-NEXT:  BB2_4:
-; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -616,36 +602,30 @@ define i64 @urem64(i64 %a, i64 %b) {
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v7, v10, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
 ; GFX9-NEXT:  BB3_2: ; %Flow
-; GFX9-NEXT:    s_or_saveexec_b64 s[6:7], s[8:9]
-; GFX9-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], s[8:9]
+; GFX9-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz BB3_4
 ; GFX9-NEXT:  ; %bb.3:
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, v2
+; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
+; GFX9-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT:    v_mul_lo_u32 v3, v1, v2
-; GFX9-NEXT:    v_mul_hi_u32 v4, v1, v2
-; GFX9-NEXT:    v_sub_u32_e32 v5, 0, v3
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v3, v3, v1
-; GFX9-NEXT:    v_mov_b32_e32 v5, 0
-; GFX9-NEXT:    v_add_u32_e32 v4, v1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v1
+; GFX9-NEXT:    v_mul_hi_u32 v3, v1, v3
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; GFX9-NEXT:    v_mul_lo_u32 v1, v1, v2
-; GFX9-NEXT:    v_sub_u32_e32 v3, v0, v1
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v3, v2
-; GFX9-NEXT:    v_sub_u32_e32 v0, v3, v2
-; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; GFX9-NEXT:    v_add_u32_e32 v4, v3, v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, v0, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v0, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v1, v0, v2
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v1, v0, v2
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v0, v1, vcc
 ; GFX9-NEXT:  BB3_4:
-; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -661,32 +641,28 @@ define i32 @sdiv32(i32 %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u32_e32 v1, v1, v2
 ; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v2
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, v1
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; GFX9-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GFX9-NEXT:    v_mul_lo_u32 v4, v3, v1
-; GFX9-NEXT:    v_mul_hi_u32 v5, v3, v1
-; GFX9-NEXT:    v_sub_u32_e32 v6, 0, v4
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v4, v4, v3
+; GFX9-NEXT:    v_sub_u32_e32 v4, 0, v1
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v5, 31, v0
 ; GFX9-NEXT:    v_add_u32_e32 v0, v0, v5
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v3
 ; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v5
-; GFX9-NEXT:    v_add_u32_e32 v6, v3, v4
-; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v3, v3, v0
 ; GFX9-NEXT:    v_xor_b32_e32 v2, v5, v2
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT:    v_mul_lo_u32 v4, v4, v3
+; GFX9-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v4
+; GFX9-NEXT:    v_mul_hi_u32 v3, v0, v3
 ; GFX9-NEXT:    v_mul_lo_u32 v4, v3, v1
 ; GFX9-NEXT:    v_add_u32_e32 v5, 1, v3
-; GFX9-NEXT:    v_add_u32_e32 v6, -1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v7, v0, v4
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v1
-; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, v5, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v4
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v4, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX9-NEXT:    v_add_u32_e32 v4, 1, v3
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v4, vcc
 ; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v2
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -699,28 +675,24 @@ define i32 @udiv32(i32 %a, i32 %b) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, v1
+; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v1
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GFX9-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; GFX9-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GFX9-NEXT:    v_mul_hi_u32 v4, v2, v1
-; GFX9-NEXT:    v_sub_u32_e32 v5, 0, v3
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v3, v3, v2
-; GFX9-NEXT:    v_add_u32_e32 v4, v2, v3
-; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v2, v2, v0
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v2
+; GFX9-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
+; GFX9-NEXT:    v_mul_hi_u32 v2, v0, v2
 ; GFX9-NEXT:    v_mul_lo_u32 v3, v2, v1
 ; GFX9-NEXT:    v_add_u32_e32 v4, 1, v2
-; GFX9-NEXT:    v_add_u32_e32 v5, -1, v2
-; GFX9-NEXT:    v_sub_u32_e32 v6, v0, v3
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v1
-; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v3
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v3, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT:    v_add_u32_e32 v3, 1, v2
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %d = udiv i32 %a, %b
   ret i32 %d
@@ -734,31 +706,25 @@ define i32 @srem32(i32 %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u32_e32 v1, v1, v2
 ; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v2
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, v1
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GFX9-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GFX9-NEXT:    v_mul_hi_u32 v4, v2, v1
-; GFX9-NEXT:    v_sub_u32_e32 v5, 0, v3
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v3, v3, v2
+; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v1
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v4, 31, v0
 ; GFX9-NEXT:    v_add_u32_e32 v0, v0, v4
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v2, v2
 ; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v4
-; GFX9-NEXT:    v_add_u32_e32 v5, v2, v3
-; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v2, v2, v0
+; GFX9-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v2
+; GFX9-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
+; GFX9-NEXT:    v_mul_hi_u32 v2, v0, v2
 ; GFX9-NEXT:    v_mul_lo_u32 v2, v2, v1
-; GFX9-NEXT:    v_sub_u32_e32 v3, v0, v2
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v2
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v1
-; GFX9-NEXT:    v_sub_u32_e32 v0, v3, v1
-; GFX9-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX9-NEXT:    v_add_u32_e32 v5, v3, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v2, v0, v1
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v2, v0, v1
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v4
 ; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -771,28 +737,22 @@ define i32 @urem32(i32 %a, i32 %b) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, v1
+; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v1
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GFX9-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; GFX9-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GFX9-NEXT:    v_mul_hi_u32 v4, v2, v1
-; GFX9-NEXT:    v_sub_u32_e32 v5, 0, v3
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v3, v3, v2
-; GFX9-NEXT:    v_add_u32_e32 v4, v2, v3
-; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v2, v2, v0
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v2
+; GFX9-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
+; GFX9-NEXT:    v_mul_hi_u32 v2, v0, v2
 ; GFX9-NEXT:    v_mul_lo_u32 v2, v2, v1
-; GFX9-NEXT:    v_sub_u32_e32 v3, v0, v2
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v2
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v1
-; GFX9-NEXT:    v_sub_u32_e32 v0, v3, v1
-; GFX9-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX9-NEXT:    v_add_u32_e32 v4, v3, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[4:5]
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v2, v0, v1
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v2, v0, v1
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %d = urem i32 %a, %b
   ret i32 %d
@@ -944,41 +904,35 @@ define <2 x i64> @sdivrem64(i64 %a, i64 %b) {
 ; GFX9-NEXT:    v_subb_co_u32_e64 v4, s[8:9], v7, v10, s[8:9]
 ; GFX9-NEXT:    v_subb_co_u32_e32 v6, vcc, v1, v8, vcc
 ; GFX9-NEXT:  BB8_2: ; %Flow
-; GFX9-NEXT:    s_or_saveexec_b64 s[6:7], s[10:11]
-; GFX9-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], s[10:11]
+; GFX9-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz BB8_4
 ; GFX9-NEXT:  ; %bb.3:
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, v2
+; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
+; GFX9-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v1
+; GFX9-NEXT:    v_mul_hi_u32 v3, v1, v3
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; GFX9-NEXT:    v_mul_lo_u32 v3, v1, v2
-; GFX9-NEXT:    v_mul_hi_u32 v4, v1, v2
-; GFX9-NEXT:    v_sub_u32_e32 v5, 0, v3
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v3, v3, v1
-; GFX9-NEXT:    v_add_u32_e32 v4, v1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mul_lo_u32 v3, v1, v2
+; GFX9-NEXT:    v_add_u32_e32 v5, 1, v1
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v3
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v3, v0, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v3, v0, v2
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
 ; GFX9-NEXT:    v_add_u32_e32 v6, 1, v1
-; GFX9-NEXT:    v_add_u32_e32 v7, -1, v1
-; GFX9-NEXT:    v_sub_u32_e32 v5, v0, v3
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v5, v2
-; GFX9-NEXT:    v_sub_u32_e32 v0, v5, v2
-; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; GFX9-NEXT:    v_add_u32_e32 v8, v5, v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v8, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v1, v6, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v0, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v1, v6, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v6, v4
 ; GFX9-NEXT:  BB8_4:
-; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v2, v5
@@ -1117,41 +1071,35 @@ define <2 x i64> @udivrem64(i64 %a, i64 %b) {
 ; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v11, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v6, v8, v1, vcc
 ; GFX9-NEXT:  BB9_2: ; %Flow
-; GFX9-NEXT:    s_or_saveexec_b64 s[6:7], s[8:9]
-; GFX9-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], s[8:9]
+; GFX9-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz BB9_4
 ; GFX9-NEXT:  ; %bb.3:
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, v2
+; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v1
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
+; GFX9-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v1
+; GFX9-NEXT:    v_mul_hi_u32 v3, v1, v3
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; GFX9-NEXT:    v_mul_lo_u32 v3, v1, v2
-; GFX9-NEXT:    v_mul_hi_u32 v4, v1, v2
-; GFX9-NEXT:    v_sub_u32_e32 v5, 0, v3
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v3, v3, v1
-; GFX9-NEXT:    v_mov_b32_e32 v5, 0
-; GFX9-NEXT:    v_add_u32_e32 v4, v1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v3
+; GFX9-NEXT:    v_add_u32_e32 v4, 1, v1
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v3
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v3, v0, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GFX9-NEXT:    v_mul_lo_u32 v3, v1, v2
+; GFX9-NEXT:    v_sub_u32_e32 v3, v0, v2
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
 ; GFX9-NEXT:    v_add_u32_e32 v4, 1, v1
-; GFX9-NEXT:    v_add_u32_e32 v7, -1, v1
-; GFX9-NEXT:    v_sub_u32_e32 v6, v0, v3
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v2
-; GFX9-NEXT:    v_sub_u32_e32 v0, v6, v2
-; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
-; GFX9-NEXT:    v_add_u32_e32 v8, v6, v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v6, v0, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v8, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v1, v4, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v7, v0, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v0, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
 ; GFX9-NEXT:  BB9_4:
-; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    v_mov_b32_e32 v2, v6
diff --git a/llvm/test/CodeGen/AMDGPU/cluster_stores.ll b/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
index 67d6f686f459a..bc3bcfe6089af 100644
--- a/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
+++ b/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
@@ -5,6 +5,14 @@
 ; CHECK-LABEL: {{^}}cluster_load_cluster_store:
 define amdgpu_kernel void @cluster_load_cluster_store(i32* noalias %lb, i32* noalias %sb) {
 bb:
+
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 8
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 8
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+
 ; DBG: Cluster ld/st SU(1) - SU(2)
 
 ; DBG: Cluster ld/st SU([[L1:[0-9]+]]) - SU([[L2:[0-9]+]])
@@ -45,6 +53,13 @@ bb:
 ; CHECK-LABEL: {{^}}cluster_load_valu_cluster_store:
 define amdgpu_kernel void @cluster_load_valu_cluster_store(i32* noalias %lb, i32* noalias %sb) {
 bb:
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 8
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 8
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+
 ; DBG: Cluster ld/st SU(1) - SU(2)
 
 ; DBG: Cluster ld/st SU([[L1:[0-9]+]]) - SU([[L2:[0-9]+]])
@@ -86,6 +101,8 @@ bb:
 
 ; Cluster loads from the same texture with different coordinates
 ; CHECK-LABEL: {{^}}cluster_image_load:
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
 ; DBG: {{^}}Cluster ld/st [[SU1:SU\([0-9]+\)]] - [[SU2:SU\([0-9]+\)]]
 ; DBG: {{^}}[[SU1]]: {{.*}} IMAGE_LOAD
 ; DBG: {{^}}[[SU2]]: {{.*}} IMAGE_LOAD
@@ -106,6 +123,9 @@ entry:
 
 ; Don't cluster loads from different textures
 ; CHECK-LABEL: {{^}}no_cluster_image_load:
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
 ; DBG-NOT: {{^}}Cluster ld/st
 define amdgpu_ps void @no_cluster_image_load(<8 x i32> inreg %src1, <8 x i32> inreg %src2, <8 x i32> inreg %dst, i32 %x, i32 %y) {
 entry:
@@ -118,6 +138,8 @@ entry:
 
 ; Cluster loads from the same texture and sampler with different coordinates
 ; CHECK-LABEL: {{^}}cluster_image_sample:
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
 ; DBG: {{^}}Cluster ld/st [[SU1:SU\([0-9]+\)]] - [[SU2:SU\([0-9]+\)]]
 ; DBG: {{^}}[[SU1]]: {{.*}} IMAGE_SAMPLE
 ; DBG: {{^}}[[SU2]]: {{.*}} IMAGE_SAMPLE
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz.ll b/llvm/test/CodeGen/AMDGPU/ctlz.ll
index dc8d630bc7d4a..c26af864e7fe3 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz.ll
@@ -25,10 +25,10 @@ define amdgpu_kernel void @s_ctlz_i32(i32 addrspace(1)* noalias %out, i32 %val)
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_flbit_i32_b32 s0, s2
+; SI-NEXT:    s_cmp_lg_u32 s2, 0
+; SI-NEXT:    s_cselect_b32 s0, s0, 32
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    v_mov_b32_e32 v0, s0
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s2, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, 32, v0, vcc
 ; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
@@ -40,9 +40,9 @@ define amdgpu_kernel void @s_ctlz_i32(i32 addrspace(1)* noalias %out, i32 %val)
 ; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_flbit_i32_b32 s1, s0
-; VI-NEXT:    v_mov_b32_e32 v0, s1
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s0, 0
-; VI-NEXT:    v_cndmask_b32_e32 v0, 32, v0, vcc
+; VI-NEXT:    s_cmp_lg_u32 s0, 0
+; VI-NEXT:    s_cselect_b32 s0, s1, 32
+; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
@@ -384,14 +384,13 @@ define amdgpu_kernel void @s_ctlz_i64(i64 addrspace(1)* noalias %out, [8 x i32],
 ; SI-NEXT:    s_flbit_i32_b32 s0, s2
 ; SI-NEXT:    s_flbit_i32_b32 s1, s3
 ; SI-NEXT:    s_add_i32 s0, s0, 32
-; SI-NEXT:    s_or_b32 s2, s2, s3
-; SI-NEXT:    v_mov_b32_e32 v0, s1
-; SI-NEXT:    v_mov_b32_e32 v1, s0
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s2, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, 64, v0, vcc
+; SI-NEXT:    s_cmp_eq_u32 s3, 0
+; SI-NEXT:    s_cselect_b32 s0, s0, s1
+; SI-NEXT:    s_or_b32 s1, s2, s3
+; SI-NEXT:    s_cmp_lg_u32 s1, 0
+; SI-NEXT:    s_cselect_b32 s0, s0, 64
 ; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    v_mov_b32_e32 v0, s0
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
@@ -401,18 +400,17 @@ define amdgpu_kernel void @s_ctlz_i64(i64 addrspace(1)* noalias %out, [8 x i32],
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x4c
 ; VI-NEXT:    s_mov_b32 s7, 0xf000
 ; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_flbit_i32_b32 s2, s0
-; VI-NEXT:    s_flbit_i32_b32 s3, s1
 ; VI-NEXT:    s_add_i32 s2, s2, 32
-; VI-NEXT:    v_mov_b32_e32 v0, s3
-; VI-NEXT:    v_mov_b32_e32 v1, s2
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; VI-NEXT:    s_flbit_i32_b32 s3, s1
+; VI-NEXT:    s_cmp_eq_u32 s1, 0
+; VI-NEXT:    s_cselect_b32 s2, s2, s3
 ; VI-NEXT:    s_or_b32 s0, s0, s1
-; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s0, 0
-; VI-NEXT:    v_cndmask_b32_e32 v0, 64, v0, vcc
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_cmp_lg_u32 s0, 0
+; VI-NEXT:    s_cselect_b32 s0, s2, 64
+; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
@@ -444,18 +442,17 @@ define amdgpu_kernel void @s_ctlz_i64_trunc(i32 addrspace(1)* noalias %out, i64
 ; SI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_flbit_i32_b32 s0, s2
 ; SI-NEXT:    s_flbit_i32_b32 s1, s3
 ; SI-NEXT:    s_add_i32 s0, s0, 32
-; SI-NEXT:    s_or_b32 s2, s2, s3
-; SI-NEXT:    v_mov_b32_e32 v0, s1
-; SI-NEXT:    v_mov_b32_e32 v1, s0
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s2, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, 64, v0, vcc
+; SI-NEXT:    s_cmp_eq_u32 s3, 0
+; SI-NEXT:    s_cselect_b32 s0, s0, s1
+; SI-NEXT:    s_or_b32 s1, s2, s3
+; SI-NEXT:    s_cmp_lg_u32 s1, 0
+; SI-NEXT:    s_cselect_b32 s0, s0, 64
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s0
 ; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
@@ -467,15 +464,14 @@ define amdgpu_kernel void @s_ctlz_i64_trunc(i32 addrspace(1)* noalias %out, i64
 ; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_flbit_i32_b32 s2, s0
-; VI-NEXT:    s_flbit_i32_b32 s3, s1
 ; VI-NEXT:    s_add_i32 s2, s2, 32
-; VI-NEXT:    v_mov_b32_e32 v0, s3
-; VI-NEXT:    v_mov_b32_e32 v1, s2
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; VI-NEXT:    s_flbit_i32_b32 s3, s1
+; VI-NEXT:    s_cmp_eq_u32 s1, 0
+; VI-NEXT:    s_cselect_b32 s2, s2, s3
 ; VI-NEXT:    s_or_b32 s0, s0, s1
-; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s0, 0
-; VI-NEXT:    v_cndmask_b32_e32 v0, 64, v0, vcc
+; VI-NEXT:    s_cmp_lg_u32 s0, 0
+; VI-NEXT:    s_cselect_b32 s0, s2, 64
+; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
index 081cb188cfe05..a34fb5c503d6b 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
@@ -99,14 +99,13 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i8(i8 addrspace(1)* noalias %out, i
 
 ; FUNC-LABEL: {{^}}s_ctlz_zero_undef_i64:
 ; GCN: s_load_dwordx2 s{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x13|0x4c}}
-; GCN-DAG: v_cmp_eq_u32_e64 vcc, s[[HI]], 0{{$}}
+; GCN-DAG: s_cmp_eq_u32 s[[HI]], 0{{$}}
 ; GCN-DAG: s_flbit_i32_b32 [[FFBH_LO:s[0-9]+]], s[[LO]]
 ; GCN-DAG: s_add_i32 [[ADD:s[0-9]+]], [[FFBH_LO]], 32
 ; GCN-DAG: s_flbit_i32_b32 [[FFBH_HI:s[0-9]+]], s[[HI]]
-; GCN-DAG: v_mov_b32_e32 [[VFFBH_LO:v[0-9]+]], [[ADD]]
-; GCN-DAG: v_mov_b32_e32 [[VFFBH_HI:v[0-9]+]], [[FFBH_HI]]
-; GCN-DAG: v_cndmask_b32_e32 v[[CTLZ:[0-9]+]], [[VFFBH_HI]], [[VFFBH_LO]]
+; GCN-DAG: s_cselect_b32 [[RES:s[0-9]+]], [[ADD]], [[FFBH_HI]]
 ; GCN-DAG: v_mov_b32_e32 v[[CTLZ_HI:[0-9]+]], 0{{$}}
+; GCN-DAG: v_mov_b32_e32 v[[CTLZ:[0-9]+]], [[RES]]
 ; GCN: {{buffer|flat}}_store_dwordx2 v{{\[}}[[CTLZ]]:[[CTLZ_HI]]{{\]}}
 define amdgpu_kernel void @s_ctlz_zero_undef_i64(i64 addrspace(1)* noalias %out, [8 x i32], i64 %val) nounwind {
   %ctlz = call i64 @llvm.ctlz.i64(i64 %val, i1 true)
diff --git a/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
index cbcb29f1e83a7..7b4d6f8d2f925 100644
--- a/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
@@ -156,8 +156,8 @@ define amdgpu_kernel void @v_cttz_zero_undef_i16_with_select(i16 addrspace(1)* n
 }
 
 ; FUNC-LABEL: {{^}}v_cttz_zero_undef_i32_with_select:
-; SI: v_ffbl_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_ne_u32_e32 vcc, 0
+; SI-DAG: v_ffbl_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; SI-DAG: v_cmp_ne_u32_e32 vcc, 0
 ; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+\.[XYZW]]]
 define amdgpu_kernel void @v_cttz_zero_undef_i32_with_select(i32 addrspace(1)* noalias %out, i32 addrspace(1)* nocapture readonly %arrayidx) nounwind {
   %val = load i32, i32 addrspace(1)* %arrayidx, align 1
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-select.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-select.ll
index 4e44968596960..e9233632c9da7 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-select.ll
@@ -1,9 +1,10 @@
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}select_and1:
-; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], 0, v{{[0-9]+}},
+; GCN:     s_cselect_b32 [[SEL:s[0-9]+]], s{{[0-9]+}},
+; GCN:     v_mov_b32_e32 [[VSEL:v[0-9]+]], [[SEL]]
 ; GCN-NOT: v_and_b32
-; GCN:     store_dword v[{{[0-9:]+}}], [[SEL]],
+; GCN:     store_dword v[{{[0-9:]+}}], [[VSEL]],
 define amdgpu_kernel void @select_and1(i32 addrspace(1)* %p, i32 %x, i32 %y) {
   %c = icmp slt i32 %x, 11
   %s = select i1 %c, i32 0, i32 -1
@@ -13,9 +14,10 @@ define amdgpu_kernel void @select_and1(i32 addrspace(1)* %p, i32 %x, i32 %y) {
 }
 
 ; GCN-LABEL: {{^}}select_and2:
-; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], 0, v{{[0-9]+}},
+; GCN:     s_cselect_b32 [[SEL:s[0-9]+]], s{{[0-9]+}},
+; GCN:     v_mov_b32_e32 [[VSEL:v[0-9]+]], [[SEL]]
 ; GCN-NOT: v_and_b32
-; GCN:     store_dword v[{{[0-9:]+}}], [[SEL]],
+; GCN:     store_dword v[{{[0-9:]+}}], [[VSEL]],
 define amdgpu_kernel void @select_and2(i32 addrspace(1)* %p, i32 %x, i32 %y) {
   %c = icmp slt i32 %x, 11
   %s = select i1 %c, i32 0, i32 -1
@@ -25,9 +27,10 @@ define amdgpu_kernel void @select_and2(i32 addrspace(1)* %p, i32 %x, i32 %y) {
 }
 
 ; GCN-LABEL: {{^}}select_and3:
-; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], 0, v{{[0-9]+}},
+; GCN:     s_cselect_b32 [[SEL:s[0-9]+]], s{{[0-9]+}},
+; GCN:     v_mov_b32_e32 [[VSEL:v[0-9]+]], [[SEL]]
 ; GCN-NOT: v_and_b32
-; GCN:     store_dword v[{{[0-9:]+}}], [[SEL]],
+; GCN:     store_dword v[{{[0-9:]+}}], [[VSEL]],
 define amdgpu_kernel void @select_and3(i32 addrspace(1)* %p, i32 %x, i32 %y) {
   %c = icmp slt i32 %x, 11
   %s = select i1 %c, i32 -1, i32 0
@@ -52,9 +55,10 @@ define amdgpu_kernel void @select_and_v4(<4 x i32> addrspace(1)* %p, i32 %x, <4
 }
 
 ; GCN-LABEL: {{^}}select_or1:
-; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], -1, v{{[0-9]+}},
+; GCN:     s_cselect_b32 [[SEL:s[0-9]+]], s{{[0-9]+}},
+; GCN:     v_mov_b32_e32 [[VSEL:v[0-9]+]], [[SEL]]
 ; GCN-NOT: v_or_b32
-; GCN:     store_dword v[{{[0-9:]+}}], [[SEL]],
+; GCN:     store_dword v[{{[0-9:]+}}], [[VSEL]],
 define amdgpu_kernel void @select_or1(i32 addrspace(1)* %p, i32 %x, i32 %y) {
   %c = icmp slt i32 %x, 11
   %s = select i1 %c, i32 0, i32 -1
@@ -64,9 +68,10 @@ define amdgpu_kernel void @select_or1(i32 addrspace(1)* %p, i32 %x, i32 %y) {
 }
 
 ; GCN-LABEL: {{^}}select_or2:
-; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], -1, v{{[0-9]+}},
+; GCN:     s_cselect_b32 [[SEL:s[0-9]+]], s{{[0-9]+}},
+; GCN:     v_mov_b32_e32 [[VSEL:v[0-9]+]], [[SEL]]
 ; GCN-NOT: v_or_b32
-; GCN:     store_dword v[{{[0-9:]+}}], [[SEL]],
+; GCN:     store_dword v[{{[0-9:]+}}], [[VSEL]],
 define amdgpu_kernel void @select_or2(i32 addrspace(1)* %p, i32 %x, i32 %y) {
   %c = icmp slt i32 %x, 11
   %s = select i1 %c, i32 0, i32 -1
@@ -76,9 +81,10 @@ define amdgpu_kernel void @select_or2(i32 addrspace(1)* %p, i32 %x, i32 %y) {
 }
 
 ; GCN-LABEL: {{^}}select_or3:
-; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], -1, v{{[0-9]+}},
+; GCN:     s_cselect_b32 [[SEL:s[0-9]+]], s{{[0-9]+}},
+; GCN:     v_mov_b32_e32 [[VSEL:v[0-9]+]], [[SEL]]
 ; GCN-NOT: v_or_b32
-; GCN:     store_dword v[{{[0-9:]+}}], [[SEL]],
+; GCN:     store_dword v[{{[0-9:]+}}], [[VSEL]],
 define amdgpu_kernel void @select_or3(i32 addrspace(1)* %p, i32 %x, i32 %y) {
   %c = icmp slt i32 %x, 11
   %s = select i1 %c, i32 -1, i32 0
@@ -103,7 +109,7 @@ define amdgpu_kernel void @select_or_v4(<4 x i32> addrspace(1)* %p, i32 %x, <4 x
 }
 
 ; GCN-LABEL: {{^}}sel_constants_sub_constant_sel_constants:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 2, 9,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 9, 2
 define amdgpu_kernel void @sel_constants_sub_constant_sel_constants(i32 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i32 -4, i32 3
   %bo = sub i32 5, %sel
@@ -131,9 +137,8 @@ define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_i16_neg(i16
 }
 
 ; GCN-LABEL: {{^}}sel_constants_sub_constant_sel_constants_v2i16:
-; GCN-DAG: v_mov_b32_e32 [[F:v[0-9]+]], 0x60002
-; GCN-DAG: v_mov_b32_e32 [[T:v[0-9]+]], 0x50009
-; GCN:     v_cndmask_b32_e32 v{{[0-9]+}}, [[F]], [[T]],
+; GCN-DAG: s_mov_b32 [[T:s[0-9]+]], 0x50009
+; GCN:     s_cselect_b32 s{{[0-9]+}}, [[T]], 0x60002
 define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_v2i16(<2 x i16> addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, <2 x i16> <i16 -4, i16 2>, <2 x i16> <i16 3, i16 1>
   %bo = sub <2 x i16> <i16 5, i16 7>, %sel
@@ -154,7 +159,7 @@ define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_v4i32(<4 x i
 }
 
 ; GCN-LABEL: {{^}}sdiv_constant_sel_constants_i64:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 5, 0,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 0, 5
 define amdgpu_kernel void @sdiv_constant_sel_constants_i64(i64 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i64 121, i64 23
   %bo = sdiv i64 120, %sel
@@ -163,7 +168,7 @@ define amdgpu_kernel void @sdiv_constant_sel_constants_i64(i64 addrspace(1)* %p,
 }
 
 ; GCN-LABEL: {{^}}sdiv_constant_sel_constants_i32:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 8, 26,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 26, 8
 define amdgpu_kernel void @sdiv_constant_sel_constants_i32(i32 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i32 7, i32 23
   %bo = sdiv i32 184, %sel
@@ -172,7 +177,7 @@ define amdgpu_kernel void @sdiv_constant_sel_constants_i32(i32 addrspace(1)* %p,
 }
 
 ; GCN-LABEL: {{^}}udiv_constant_sel_constants_i64:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 5, 0,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 0, 5
 define amdgpu_kernel void @udiv_constant_sel_constants_i64(i64 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i64 -4, i64 23
   %bo = udiv i64 120, %sel
@@ -181,7 +186,7 @@ define amdgpu_kernel void @udiv_constant_sel_constants_i64(i64 addrspace(1)* %p,
 }
 
 ; GCN-LABEL: {{^}}srem_constant_sel_constants:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 3, 33,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 33, 3
 define amdgpu_kernel void @srem_constant_sel_constants(i64 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i64 34, i64 15
   %bo = srem i64 33, %sel
@@ -190,7 +195,7 @@ define amdgpu_kernel void @srem_constant_sel_constants(i64 addrspace(1)* %p, i1
 }
 
 ; GCN-LABEL: {{^}}urem_constant_sel_constants:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 3, 33,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 33, 3
 define amdgpu_kernel void @urem_constant_sel_constants(i64 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i64 34, i64 15
   %bo = urem i64 33, %sel
@@ -199,7 +204,7 @@ define amdgpu_kernel void @urem_constant_sel_constants(i64 addrspace(1)* %p, i1
 }
 
 ; GCN-LABEL: {{^}}shl_constant_sel_constants:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 8, 4,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 4, 8
 define amdgpu_kernel void @shl_constant_sel_constants(i32 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i32 2, i32 3
   %bo = shl i32 1, %sel
@@ -208,7 +213,7 @@ define amdgpu_kernel void @shl_constant_sel_constants(i32 addrspace(1)* %p, i1 %
 }
 
 ; GCN-LABEL: {{^}}lshr_constant_sel_constants:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 8, 16,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 16, 8
 define amdgpu_kernel void @lshr_constant_sel_constants(i32 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i32 2, i32 3
   %bo = lshr i32 64, %sel
@@ -217,7 +222,7 @@ define amdgpu_kernel void @lshr_constant_sel_constants(i32 addrspace(1)* %p, i1
 }
 
 ; GCN-LABEL: {{^}}ashr_constant_sel_constants:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 16, 32,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 32, 16
 define amdgpu_kernel void @ashr_constant_sel_constants(i32 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i32 2, i32 3
   %bo = ashr i32 128, %sel
@@ -247,8 +252,7 @@ define amdgpu_kernel void @fsub_constant_sel_constants_f16(half addrspace(1)* %p
 }
 
 ; GCN-LABEL: {{^}}fsub_constant_sel_constants_v2f16:
-; GCN-DAG: v_mov_b32_e32 [[T:v[0-9]+]], 0x45003c00
-; GCN:     v_cndmask_b32_e32 v{{[0-9]+}}, -2.0, [[T]],
+; GCN:     s_cselect_b32 s{{[0-9]+}}, 0x45003c00, -2.0
 define amdgpu_kernel void @fsub_constant_sel_constants_v2f16(<2 x half> addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, <2 x half> <half -2.0, half -3.0>, <2 x half> <half -1.0, half 4.0>
   %bo = fsub <2 x half> <half -1.0, half 2.0>, %sel
diff --git a/llvm/test/CodeGen/AMDGPU/dpp_combine.mir b/llvm/test/CodeGen/AMDGPU/dpp_combine.mir
index 358a331da1a5d..6194515c3bec4 100644
--- a/llvm/test/CodeGen/AMDGPU/dpp_combine.mir
+++ b/llvm/test/CodeGen/AMDGPU/dpp_combine.mir
@@ -833,3 +833,33 @@ body:             |
     S_ENDPGM 0, implicit %4
 
 ...
+
+# GCN-LABEL: name: dont_combine_more_than_one_operand
+# GCN: %3:vgpr_32 = V_MAX_F32_e64 0, %2, 0, %2, 0, 0, implicit $mode, implicit $exec
+name: dont_combine_more_than_one_operand
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    %2:vgpr_32 = V_MOV_B32_dpp %0, %1, 1, 15, 15, 1, implicit $exec
+    %3:vgpr_32 = V_MAX_F32_e64 0, %2, 0, %2, 0, 0, implicit $mode, implicit $exec
+...
+
+# GCN-LABEL: name: dont_combine_more_than_one_operand_dpp_reg_sequence
+# GCN: %5:vgpr_32 = V_ADD_I32_e32 %4.sub0, %4.sub0, implicit-def $vcc, implicit $exec
+# GCN: %6:vgpr_32 = V_ADDC_U32_e32 %4.sub1, %4.sub1, implicit-def $vcc, implicit $vcc, implicit $exec
+name: dont_combine_more_than_one_operand_dpp_reg_sequence
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    %0:vreg_64 = COPY $vgpr0_vgpr1
+    %1:vreg_64 = COPY $vgpr2_vgpr3
+    %2:vgpr_32 = V_MOV_B32_dpp %0.sub0, %1.sub0, 1, 15, 15, 1, implicit $exec
+    %3:vgpr_32 = V_MOV_B32_dpp %0.sub1, %1.sub1, 1, 15, 15, 1, implicit $exec
+    %4:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %3, %subreg.sub1
+    %5:vgpr_32 = V_ADD_I32_e32 %4.sub0, %4.sub0, implicit-def $vcc, implicit $exec
+    %6:vgpr_32 = V_ADDC_U32_e32 %4.sub1, %4.sub1, implicit-def $vcc, implicit $vcc, implicit $exec
+...
diff --git a/llvm/test/CodeGen/AMDGPU/expand-scalar-carry-out-select-user.ll b/llvm/test/CodeGen/AMDGPU/expand-scalar-carry-out-select-user.ll
new file mode 100644
index 0000000000000..aad3ea52ab813
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/expand-scalar-carry-out-select-user.ll
@@ -0,0 +1,121 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GCN,GFX10 %s
+
+define i32 @s_add_co_select_user() {
+; GFX9-LABEL: s_add_co_select_user:
+; GFX9:       ; %bb.0: ; %bb
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e64 v0, s[4:5], s6, s6
+; GFX9-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GFX9-NEXT:    s_addc_u32 s4, s6, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, 1, 0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    s_cmp_gt_u32 s6, 31
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: s_add_co_select_user:
+; GFX10:       ; %bb.0: ; %bb
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_mov_b64 s[4:5], 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s5, s4, s4
+; GFX10-NEXT:    s_cmpk_lg_u32 s5, 0x0
+; GFX10-NEXT:    s_addc_u32 s5, s4, 0
+; GFX10-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX10-NEXT:    s_cmp_gt_u32 s4, 31
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, s5, s6
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, -1, 0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+bb:
+  %i = load volatile i32, i32 addrspace(4)* null, align 8
+  %i1 = add i32 %i, %i
+  %i2 = icmp ult i32 %i1, %i
+  %i3 = zext i1 %i2 to i32
+  %i4 = add nuw nsw i32 %i3, 0
+  %i5 = add i32 %i4, %i
+  %i6 = icmp ult i32 %i5, %i4
+  %i7 = select i1 %i6, i32 %i5, i32 0
+  %i8 = icmp ugt i32 %i, 31
+  %i9 = select i1 %i8, i32 %i1, i32 %i7
+  ret i32 %i9
+}
+
+define amdgpu_kernel void @s_add_co_br_user(i32 %i) {
+; GFX9-LABEL: s_add_co_br_user:
+; GFX9:       ; %bb.0: ; %bb
+; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_add_i32 s1, s0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, s1, v0
+; GFX9-NEXT:    s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    s_addc_u32 s0, s0, 0
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, s0, v0
+; GFX9-NEXT:    s_and_b64 vcc, exec, vcc
+; GFX9-NEXT:    s_cbranch_vccnz BB1_2
+; GFX9-NEXT:  ; %bb.1: ; %bb0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, 9
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:  BB1_2: ; %bb1
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, 10
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: s_add_co_br_user:
+; GFX10:       ; %bb.0: ; %bb
+; GFX10-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_add_i32 s1, s0, s0
+; GFX10-NEXT:    v_cmp_lt_u32_e64 s1, s1, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s1
+; GFX10-NEXT:    s_cmpk_lg_u32 s1, 0x0
+; GFX10-NEXT:    s_addc_u32 s0, s0, 0
+; GFX10-NEXT:    v_cmp_ge_u32_e32 vcc_lo, s0, v0
+; GFX10-NEXT:    s_and_b32 vcc_lo, exec_lo, vcc_lo
+; GFX10-NEXT:    s_cbranch_vccnz BB1_2
+; GFX10-NEXT:  ; %bb.1: ; %bb0
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    v_mov_b32_e32 v2, 9
+; GFX10-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:  BB1_2: ; %bb1
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    v_mov_b32_e32 v2, 10
+; GFX10-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
+bb:
+  %i1 = add i32 %i, %i
+  %i2 = icmp ult i32 %i1, %i
+  %i3 = zext i1 %i2 to i32
+  %i4 = add nuw nsw i32 %i3, 0
+  %i5 = add i32 %i4, %i
+  %i6 = icmp ult i32 %i5, %i4
+  %i7 = select i1 %i6, i32 %i5, i32 0
+  br i1 %i6, label %bb0, label %bb1
+
+bb0:
+  store volatile i32 9, i32 addrspace(1)* null
+  br label %bb1
+
+bb1:
+  store volatile i32 10, i32 addrspace(1)* null
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll b/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
index 1ec749059452b..a14e2ae48504f 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
@@ -18,12 +18,12 @@ entry:
 
 ; GCN-LABEL: {{^}}int4_extelt:
 ; GCN-NOT: buffer_
-; GCN-DAG: v_cmp_eq_u32_e64 [[C1:[^,]+]], [[IDX:s[0-9]+]], 1
-; GCN-DAG: v_cmp_ne_u32_e64 [[C2:[^,]+]], [[IDX]], 2
-; GCN-DAG: v_cmp_ne_u32_e64 [[C3:[^,]+]], [[IDX]], 3
+; GCN-DAG: s_cmp_lg_u32 [[IDX:s[0-9]+]], 2
+; GCN-DAG: v_cmp_eq_u32_e64 [[C1:[^,]+]], [[IDX]], 1
+; GCN-DAG: s_cmp_lg_u32 [[IDX]], 3
 ; GCN-DAG: v_cndmask_b32_e{{32|64}} [[V1:v[0-9]+]], 0, 1, [[C1]]
-; GCN-DAG: v_cndmask_b32_e{{32|64}} [[V2:v[0-9]+]], 2, [[V1]], [[C2]]
-; GCN-DAG: v_cndmask_b32_e{{32|64}} [[V3:v[0-9]+]], 4, [[V2]], [[C3]]
+; GCN-DAG: v_cndmask_b32_e{{32|64}} [[V2:v[0-9]+]], 2, [[V1]], vcc
+; GCN-DAG: v_cndmask_b32_e{{32|64}} [[V3:v[0-9]+]], 4, [[V2]], vcc
 ; GCN: store_dword v[{{[0-9:]+}}], [[V3]]
 define amdgpu_kernel void @int4_extelt(i32 addrspace(1)* %out, i32 %sel) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/extractelt-to-trunc.ll b/llvm/test/CodeGen/AMDGPU/extractelt-to-trunc.ll
index 1165cd82a2993..8dd0f46c93036 100644
--- a/llvm/test/CodeGen/AMDGPU/extractelt-to-trunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/extractelt-to-trunc.ll
@@ -147,23 +147,26 @@ define amdgpu_kernel void @no_extract_volatile_load_dynextract(i32 addrspace(1)*
 ; GCN-LABEL: no_extract_volatile_load_dynextract:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s12, s[0:1], 0xd
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    s_mov_b32 s2, -1
-; GCN-NEXT:    s_load_dword s12, s[0:1], 0xd
 ; GCN-NEXT:    s_mov_b32 s10, s2
 ; GCN-NEXT:    s_mov_b32 s11, s3
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    s_mov_b32 s8, s6
 ; GCN-NEXT:    s_mov_b32 s9, s7
 ; GCN-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; GCN-NEXT:    s_mov_b32 s0, s4
-; GCN-NEXT:    s_mov_b32 s1, s5
-; GCN-NEXT:    v_cmp_eq_u32_e64 vcc, s12, 1
+; GCN-NEXT:    s_cmp_eq_u32 s12, 1
+; GCN-NEXT:    s_cselect_b64 vcc, -1, 0
+; GCN-NEXT:    s_cmp_eq_u32 s12, 2
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e64 vcc, s12, 2
+; GCN-NEXT:    s_cselect_b64 vcc, -1, 0
+; GCN-NEXT:    s_cmp_eq_u32 s12, 3
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e64 vcc, s12, 3
+; GCN-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fceil64.ll b/llvm/test/CodeGen/AMDGPU/fceil64.ll
index da852af3f2303..f0cd6a1df9ef8 100644
--- a/llvm/test/CodeGen/AMDGPU/fceil64.ll
+++ b/llvm/test/CodeGen/AMDGPU/fceil64.ll
@@ -27,7 +27,7 @@ declare <16 x double> @llvm.ceil.v16f64(<16 x double>) nounwind readnone
 ; SI-DAG: v_cmp_gt_f64
 ; SI-DAG: v_cmp_lg_f64
 ; SI-DAG: v_cndmask_b32
-; SI: v_cndmask_b32
+; SI: s_cselect_b32
 ; SI: v_add_f64
 ; SI: s_endpgm
 define amdgpu_kernel void @fceil_f64(double addrspace(1)* %out, double %x) {
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies.mir b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies.mir
index 7bc14939624d8..22775ec82714f 100644
--- a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies.mir
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=amdgcn -run-pass=si-fix-sgpr-copies -o - %s | FileCheck --check-prefix=GCN %s
+# RUN: llc -march=amdgcn -run-pass=si-fix-sgpr-copies -verify-machineinstrs -o - %s | FileCheck --check-prefix=GCN %s
 
 # GCN-LABEL: name: fix-sgpr-copies
 # GCN: V_ADD_I32_e32
@@ -110,3 +110,19 @@ body:               |
     %3:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %0, %subreg.sub1
     $vgpr3 = COPY %3.sub0
 ...
+
+# Test to ensure that undef SCC gets properly propagated.
+# GCN-LABEL: name: scc_undef
+# GCN: S_CSELECT_B64 -1, 0, implicit undef $scc
+# GCN: V_CNDMASK
+---
+name: scc_undef
+tracksRegLiveness: true
+
+body:               |
+  bb.0:
+  %1:vgpr_32 = IMPLICIT_DEF
+  %2:sreg_32 = S_MOV_B32 1
+  %3:sreg_32 = COPY %1:vgpr_32
+  %4:sreg_32 = S_CSELECT_B32 killed %2:sreg_32, killed %3:sreg_32, implicit undef $scc
+---
diff --git a/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir b/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
index 86a634b81013b..c53b817344c50 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
@@ -26,17 +26,36 @@ body:             |
 ---
 # GCN-LABEL:       name: no_extra_fold_on_same_opnd
 # The first XOR needs commuting to fold that immediate operand.
-# GCN:             V_XOR_B32_e32 {{.*}} 0, %1
+# GCN:             V_XOR_B32_e32 0, %1
 # GCN:             V_XOR_B32_e32 %2, %4.sub0
 name: no_extra_fold_on_same_opnd
 tracksRegLiveness: true
 body:             |
   bb.0:
-    %0:vgpr_32  = IMPLICIT_DEF
-    %1:vgpr_32  = IMPLICIT_DEF
-    %2:vgpr_32  = IMPLICIT_DEF
-    %3:vgpr_32  = V_MOV_B32_e32 0, implicit $exec
-    %4:vreg_64  = REG_SEQUENCE killed %0, %subreg.sub0, killed %3, %subreg.sub1
-    %5:vgpr_32  = V_XOR_B32_e32 %1, %4.sub1, implicit $exec
-    %6:vgpr_32  = V_XOR_B32_e32 %2, %4.sub0, implicit $exec
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:vgpr_32 = IMPLICIT_DEF
+    %3:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %4:vreg_64 = REG_SEQUENCE killed %0, %subreg.sub0, killed %3, %subreg.sub1
+    %5:vgpr_32 = V_XOR_B32_e32 %1, %4.sub1, implicit $exec
+    %6:vgpr_32 = V_XOR_B32_e32 %2, %4.sub0, implicit $exec
+...
+
+---
+
+# Make sure the subreg index is not reinterpreted when folding
+# immediates
+#
+# GCN-LABEL: name: clear_subreg_imm_fold{{$}}
+# GCN: %1:sgpr_32 = S_MOV_B32 4294967288
+# GCN: %2:sgpr_32 = S_MOV_B32 4294967295
+name: clear_subreg_imm_fold
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %0:sreg_64 = S_MOV_B64 -8
+    %1:sgpr_32 = COPY %0.sub0
+    %2:sgpr_32 = COPY %0.sub1
+    S_ENDPGM 0, implicit %1, implicit %2
+
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/fshl.ll b/llvm/test/CodeGen/AMDGPU/fshl.ll
index 0f2d16ebcc2cc..9f3255f4e500f 100644
--- a/llvm/test/CodeGen/AMDGPU/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/fshl.ll
@@ -18,11 +18,12 @@ define amdgpu_kernel void @fshl_i32(i32 addrspace(1)* %in, i32 %x, i32 %y, i32 %
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_sub_i32 s3, 32, s2
 ; SI-NEXT:    v_mov_b32_e32 v0, s1
-; SI-NEXT:    v_mov_b32_e32 v1, s3
 ; SI-NEXT:    s_and_b32 s1, s2, 31
+; SI-NEXT:    v_mov_b32_e32 v1, s3
+; SI-NEXT:    s_cmp_eq_u32 s1, 0
 ; SI-NEXT:    v_alignbit_b32 v0, s0, v0, v1
 ; SI-NEXT:    v_mov_b32_e32 v1, s0
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
@@ -35,10 +36,11 @@ define amdgpu_kernel void @fshl_i32(i32 addrspace(1)* %in, i32 %x, i32 %y, i32 %
 ; VI-NEXT:    s_sub_i32 s3, 32, s2
 ; VI-NEXT:    v_mov_b32_e32 v0, s1
 ; VI-NEXT:    s_and_b32 s1, s2, 31
-; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    s_cmp_eq_u32 s1, 0
+; VI-NEXT:    v_alignbit_b32 v0, s0, v0, v1
 ; VI-NEXT:    v_mov_b32_e32 v1, s0
-; VI-NEXT:    v_alignbit_b32 v0, s0, v0, v2
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
@@ -53,10 +55,11 @@ define amdgpu_kernel void @fshl_i32(i32 addrspace(1)* %in, i32 %x, i32 %y, i32 %
 ; GFX9-NEXT:    s_sub_i32 s3, 32, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX9-NEXT:    s_and_b32 s1, s2, 31
-; GFX9-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_cmp_eq_u32 s1, 0
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, v2
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
@@ -150,19 +153,21 @@ define amdgpu_kernel void @fshl_v2i32(<2 x i32> addrspace(1)* %in, <2 x i32> %x,
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v0, s9
 ; SI-NEXT:    s_sub_i32 s10, 32, s1
-; SI-NEXT:    v_mov_b32_e32 v1, s10
 ; SI-NEXT:    s_and_b32 s1, s1, 31
+; SI-NEXT:    v_mov_b32_e32 v1, s10
+; SI-NEXT:    s_cmp_eq_u32 s1, 0
 ; SI-NEXT:    v_alignbit_b32 v0, s3, v0, v1
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_mov_b32_e32 v1, s3
 ; SI-NEXT:    s_sub_i32 s1, 32, s0
-; SI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
 ; SI-NEXT:    s_and_b32 s0, s0, 31
+; SI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-NEXT:    s_cmp_eq_u32 s0, 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s8
 ; SI-NEXT:    v_mov_b32_e32 v2, s1
 ; SI-NEXT:    v_alignbit_b32 v0, s2, v0, v2
 ; SI-NEXT:    v_mov_b32_e32 v2, s2
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
@@ -176,19 +181,21 @@ define amdgpu_kernel void @fshl_v2i32(<2 x i32> addrspace(1)* %in, <2 x i32> %x,
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s7
 ; VI-NEXT:    s_sub_i32 s8, 32, s1
-; VI-NEXT:    v_mov_b32_e32 v1, s8
 ; VI-NEXT:    s_and_b32 s1, s1, 31
+; VI-NEXT:    v_mov_b32_e32 v1, s8
+; VI-NEXT:    s_cmp_eq_u32 s1, 0
 ; VI-NEXT:    v_alignbit_b32 v0, s5, v0, v1
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    s_sub_i32 s1, 32, s0
-; VI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
 ; VI-NEXT:    s_and_b32 s0, s0, 31
+; VI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; VI-NEXT:    s_cmp_eq_u32 s0, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s6
 ; VI-NEXT:    v_mov_b32_e32 v2, s1
 ; VI-NEXT:    v_alignbit_b32 v0, s4, v0, v2
 ; VI-NEXT:    v_mov_b32_e32 v2, s4
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
@@ -204,19 +211,21 @@ define amdgpu_kernel void @fshl_v2i32(<2 x i32> addrspace(1)* %in, <2 x i32> %x,
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s7
 ; GFX9-NEXT:    s_sub_i32 s8, 32, s1
-; GFX9-NEXT:    v_mov_b32_e32 v1, s8
 ; GFX9-NEXT:    s_and_b32 s1, s1, 31
+; GFX9-NEXT:    v_mov_b32_e32 v1, s8
+; GFX9-NEXT:    s_cmp_eq_u32 s1, 0
 ; GFX9-NEXT:    v_alignbit_b32 v0, s5, v0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-NEXT:    s_sub_i32 s1, 32, s0
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
 ; GFX9-NEXT:    s_and_b32 s0, s0, 31
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX9-NEXT:    v_alignbit_b32 v0, s4, v0, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s3
@@ -327,35 +336,39 @@ define amdgpu_kernel void @fshl_v4i32(<4 x i32> addrspace(1)* %in, <4 x i32> %x,
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v0, s15
 ; SI-NEXT:    s_sub_i32 s16, 32, s3
-; SI-NEXT:    v_mov_b32_e32 v1, s16
 ; SI-NEXT:    s_and_b32 s3, s3, 31
+; SI-NEXT:    v_mov_b32_e32 v1, s16
+; SI-NEXT:    s_cmp_eq_u32 s3, 0
 ; SI-NEXT:    v_alignbit_b32 v0, s11, v0, v1
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_mov_b32_e32 v1, s11
 ; SI-NEXT:    s_sub_i32 s3, 32, s2
-; SI-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
 ; SI-NEXT:    s_and_b32 s2, s2, 31
+; SI-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
+; SI-NEXT:    s_cmp_eq_u32 s2, 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s14
 ; SI-NEXT:    v_mov_b32_e32 v1, s3
 ; SI-NEXT:    v_alignbit_b32 v0, s10, v0, v1
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_mov_b32_e32 v1, s10
 ; SI-NEXT:    s_sub_i32 s2, 32, s1
-; SI-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
 ; SI-NEXT:    s_and_b32 s1, s1, 31
+; SI-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
+; SI-NEXT:    s_cmp_eq_u32 s1, 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s13
 ; SI-NEXT:    v_mov_b32_e32 v1, s2
 ; SI-NEXT:    v_alignbit_b32 v0, s9, v0, v1
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_mov_b32_e32 v1, s9
 ; SI-NEXT:    s_sub_i32 s1, 32, s0
-; SI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
 ; SI-NEXT:    s_and_b32 s0, s0, 31
+; SI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-NEXT:    s_cmp_eq_u32 s0, 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s12
 ; SI-NEXT:    v_mov_b32_e32 v4, s1
 ; SI-NEXT:    v_alignbit_b32 v0, s8, v0, v4
 ; SI-NEXT:    v_mov_b32_e32 v4, s8
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
@@ -369,35 +382,39 @@ define amdgpu_kernel void @fshl_v4i32(<4 x i32> addrspace(1)* %in, <4 x i32> %x,
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s11
 ; VI-NEXT:    s_sub_i32 s14, 32, s3
-; VI-NEXT:    v_mov_b32_e32 v1, s14
 ; VI-NEXT:    s_and_b32 s3, s3, 31
+; VI-NEXT:    v_mov_b32_e32 v1, s14
+; VI-NEXT:    s_cmp_eq_u32 s3, 0
 ; VI-NEXT:    v_alignbit_b32 v0, s7, v0, v1
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, s7
 ; VI-NEXT:    s_sub_i32 s3, 32, s2
-; VI-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
 ; VI-NEXT:    s_and_b32 s2, s2, 31
+; VI-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
+; VI-NEXT:    s_cmp_eq_u32 s2, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s10
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_alignbit_b32 v0, s6, v0, v1
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, s6
 ; VI-NEXT:    s_sub_i32 s2, 32, s1
-; VI-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
 ; VI-NEXT:    s_and_b32 s1, s1, 31
+; VI-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
+; VI-NEXT:    s_cmp_eq_u32 s1, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s9
 ; VI-NEXT:    v_mov_b32_e32 v1, s2
 ; VI-NEXT:    v_alignbit_b32 v0, s5, v0, v1
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    s_sub_i32 s1, 32, s0
-; VI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
 ; VI-NEXT:    s_and_b32 s0, s0, 31
+; VI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; VI-NEXT:    s_cmp_eq_u32 s0, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s8
 ; VI-NEXT:    v_mov_b32_e32 v4, s1
 ; VI-NEXT:    v_alignbit_b32 v0, s4, v0, v4
 ; VI-NEXT:    v_mov_b32_e32 v4, s4
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
 ; VI-NEXT:    v_mov_b32_e32 v4, s12
 ; VI-NEXT:    v_mov_b32_e32 v5, s13
@@ -413,35 +430,39 @@ define amdgpu_kernel void @fshl_v4i32(<4 x i32> addrspace(1)* %in, <4 x i32> %x,
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s11
 ; GFX9-NEXT:    s_sub_i32 s14, 32, s3
-; GFX9-NEXT:    v_mov_b32_e32 v1, s14
 ; GFX9-NEXT:    s_and_b32 s3, s3, 31
+; GFX9-NEXT:    v_mov_b32_e32 v1, s14
+; GFX9-NEXT:    s_cmp_eq_u32 s3, 0
 ; GFX9-NEXT:    v_alignbit_b32 v0, s7, v0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX9-NEXT:    s_sub_i32 s3, 32, s2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
 ; GFX9-NEXT:    s_and_b32 s2, s2, 31
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
+; GFX9-NEXT:    s_cmp_eq_u32 s2, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s10
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9-NEXT:    v_alignbit_b32 v0, s6, v0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-NEXT:    s_sub_i32 s2, 32, s1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
 ; GFX9-NEXT:    s_and_b32 s1, s1, 31
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
+; GFX9-NEXT:    s_cmp_eq_u32 s1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s9
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    v_alignbit_b32 v0, s5, v0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-NEXT:    s_sub_i32 s1, 32, s0
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
 ; GFX9-NEXT:    s_and_b32 s0, s0, 31
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s8
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s1
 ; GFX9-NEXT:    v_alignbit_b32 v0, s4, v0, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s12
 ; GFX9-NEXT:    v_mov_b32_e32 v5, s13
diff --git a/llvm/test/CodeGen/AMDGPU/fshr.ll b/llvm/test/CodeGen/AMDGPU/fshr.ll
index e27cbe73a0bd3..2dadb5e80dfd7 100644
--- a/llvm/test/CodeGen/AMDGPU/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/fshr.ll
@@ -140,14 +140,16 @@ define amdgpu_kernel void @fshr_v2i32(<2 x i32> addrspace(1)* %in, <2 x i32> %x,
 ; SI-NEXT:    v_mov_b32_e32 v0, s9
 ; SI-NEXT:    s_and_b32 s1, s1, 31
 ; SI-NEXT:    v_mov_b32_e32 v1, s1
+; SI-NEXT:    s_cmp_eq_u32 s1, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    s_and_b32 s0, s0, 31
 ; SI-NEXT:    v_alignbit_b32 v1, s3, v0, v1
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
 ; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc
+; SI-NEXT:    s_cmp_eq_u32 s0, 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s8
 ; SI-NEXT:    v_mov_b32_e32 v2, s0
 ; SI-NEXT:    v_alignbit_b32 v2, s2, v0, v2
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
@@ -162,14 +164,16 @@ define amdgpu_kernel void @fshr_v2i32(<2 x i32> addrspace(1)* %in, <2 x i32> %x,
 ; VI-NEXT:    v_mov_b32_e32 v0, s7
 ; VI-NEXT:    s_and_b32 s1, s1, 31
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_cmp_eq_u32 s1, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    s_and_b32 s0, s0, 31
 ; VI-NEXT:    v_alignbit_b32 v1, s5, v0, v1
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
 ; VI-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc
+; VI-NEXT:    s_cmp_eq_u32 s0, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s6
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    v_alignbit_b32 v2, s4, v0, v2
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
@@ -186,14 +190,16 @@ define amdgpu_kernel void @fshr_v2i32(<2 x i32> addrspace(1)* %in, <2 x i32> %x,
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s7
 ; GFX9-NEXT:    s_and_b32 s1, s1, 31
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_cmp_eq_u32 s1, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    s_and_b32 s0, s0, 31
 ; GFX9-NEXT:    v_alignbit_b32 v1, s5, v0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX9-NEXT:    v_alignbit_b32 v2, s4, v0, v2
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s3
@@ -303,26 +309,30 @@ define amdgpu_kernel void @fshr_v4i32(<4 x i32> addrspace(1)* %in, <4 x i32> %x,
 ; SI-NEXT:    v_mov_b32_e32 v0, s15
 ; SI-NEXT:    s_and_b32 s3, s3, 31
 ; SI-NEXT:    v_mov_b32_e32 v1, s3
-; SI-NEXT:    v_alignbit_b32 v1, s11, v0, v1
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
+; SI-NEXT:    s_cmp_eq_u32 s3, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    s_and_b32 s2, s2, 31
+; SI-NEXT:    v_alignbit_b32 v1, s11, v0, v1
 ; SI-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
+; SI-NEXT:    s_cmp_eq_u32 s2, 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s14
 ; SI-NEXT:    v_mov_b32_e32 v1, s2
-; SI-NEXT:    v_alignbit_b32 v1, s10, v0, v1
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    s_and_b32 s1, s1, 31
+; SI-NEXT:    v_alignbit_b32 v1, s10, v0, v1
 ; SI-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
+; SI-NEXT:    s_cmp_eq_u32 s1, 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s13
 ; SI-NEXT:    v_mov_b32_e32 v1, s1
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    s_and_b32 s0, s0, 31
 ; SI-NEXT:    v_alignbit_b32 v1, s9, v0, v1
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
 ; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc
+; SI-NEXT:    s_cmp_eq_u32 s0, 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s12
 ; SI-NEXT:    v_mov_b32_e32 v4, s0
 ; SI-NEXT:    v_alignbit_b32 v4, s8, v0, v4
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
@@ -337,26 +347,30 @@ define amdgpu_kernel void @fshr_v4i32(<4 x i32> addrspace(1)* %in, <4 x i32> %x,
 ; VI-NEXT:    v_mov_b32_e32 v0, s11
 ; VI-NEXT:    s_and_b32 s3, s3, 31
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
-; VI-NEXT:    v_alignbit_b32 v1, s7, v0, v1
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
+; VI-NEXT:    s_cmp_eq_u32 s3, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    s_and_b32 s2, s2, 31
+; VI-NEXT:    v_alignbit_b32 v1, s7, v0, v1
 ; VI-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
+; VI-NEXT:    s_cmp_eq_u32 s2, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s10
 ; VI-NEXT:    v_mov_b32_e32 v1, s2
-; VI-NEXT:    v_alignbit_b32 v1, s6, v0, v1
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    s_and_b32 s1, s1, 31
+; VI-NEXT:    v_alignbit_b32 v1, s6, v0, v1
 ; VI-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
+; VI-NEXT:    s_cmp_eq_u32 s1, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s9
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    s_and_b32 s0, s0, 31
 ; VI-NEXT:    v_alignbit_b32 v1, s5, v0, v1
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
 ; VI-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc
+; VI-NEXT:    s_cmp_eq_u32 s0, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s8
 ; VI-NEXT:    v_mov_b32_e32 v4, s0
 ; VI-NEXT:    v_alignbit_b32 v4, s4, v0, v4
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; VI-NEXT:    v_mov_b32_e32 v4, s12
 ; VI-NEXT:    v_mov_b32_e32 v5, s13
@@ -373,26 +387,30 @@ define amdgpu_kernel void @fshr_v4i32(<4 x i32> addrspace(1)* %in, <4 x i32> %x,
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s11
 ; GFX9-NEXT:    s_and_b32 s3, s3, 31
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GFX9-NEXT:    v_alignbit_b32 v1, s7, v0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s3, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    s_and_b32 s2, s2, 31
+; GFX9-NEXT:    v_alignbit_b32 v1, s7, v0, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
+; GFX9-NEXT:    s_cmp_eq_u32 s2, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s10
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; GFX9-NEXT:    v_alignbit_b32 v1, s6, v0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    s_and_b32 s1, s1, 31
+; GFX9-NEXT:    v_alignbit_b32 v1, s6, v0, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
+; GFX9-NEXT:    s_cmp_eq_u32 s1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s9
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    s_and_b32 s0, s0, 31
 ; GFX9-NEXT:    v_alignbit_b32 v1, s5, v0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s8
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s0
 ; GFX9-NEXT:    v_alignbit_b32 v4, s4, v0, v4
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s12
 ; GFX9-NEXT:    v_mov_b32_e32 v5, s13
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr.ll b/llvm/test/CodeGen/AMDGPU/global-saddr.ll
index 136cfd63686c8..8a3d1d3053f9b 100644
--- a/llvm/test/CodeGen/AMDGPU/global-saddr.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr.ll
@@ -46,8 +46,8 @@ entry:
 
 ; Test various offset boundaries.
 ; GFX9: global_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:4088{{$}}
-; GFX9: global_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+}}:{{[0-9]+}}] offset:2040{{$}}
 ; GFX9: global_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+}}:{{[0-9]+}}] offset:4088{{$}}
+; GFX9: global_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+}}:{{[0-9]+}}] offset:2056{{$}}
   %gep11 = getelementptr inbounds i64, i64 addrspace(1)* %gep, i64 511
   %load11 = load i64, i64 addrspace(1)* %gep11
   %gep12 = getelementptr inbounds i64, i64 addrspace(1)* %gep, i64 1023
diff --git a/llvm/test/CodeGen/AMDGPU/gws-hazards.mir b/llvm/test/CodeGen/AMDGPU/gws-hazards.mir
index e1ce2ad9bb641..29e330caee546 100644
--- a/llvm/test/CodeGen/AMDGPU/gws-hazards.mir
+++ b/llvm/test/CodeGen/AMDGPU/gws-hazards.mir
@@ -58,7 +58,7 @@ body: |
     ; SI: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
     ; SI: $m0 = S_MOV_B32 -1
     ; SI: DS_GWS_INIT $vgpr0, 0, 1, implicit $m0, implicit $exec
-    $vgpr0  = V_MOV_B32_e32 0, implicit $exec
+    $vgpr0 = V_MOV_B32_e32 0, implicit $exec
     $m0 = S_MOV_B32 -1
     DS_GWS_INIT  $vgpr0, 0, 1, implicit $m0, implicit $exec
 
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg-v3.ll
index 5ed07dc6e3cc3..4afa9f35499f1 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg-v3.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
 
-; CHECK:        - .args:           
+; CHECK:        - .args:
 ; CHECK-NEXT:       - .access:         read_only
 ; CHECK-NEXT:         .address_space:  global
 ; CHECK-NEXT:         .is_const:       true
@@ -10,14 +10,12 @@
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      'float*'
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f32
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .name:           out
 ; CHECK-NEXT:         .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      'float*'
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f32
 ; CHECK:          .name:           test_ro_arg
 ; CHECK:          .symbol:         test_ro_arg.kd
 
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg.ll
index 053c21b8eb796..6e22f85f453f1 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg.ll
@@ -8,7 +8,6 @@
 ; CHECK-NEXT:   Size:            8
 ; CHECK-NEXT:   Align:           8
 ; CHECK-NEXT:   ValueKind:       GlobalBuffer
-; CHECK-NEXT:   ValueType:       F32
 ; CHECK-NEXT:   AddrSpaceQual:   Global
 ; CHECK-NEXT:   AccQual:         ReadOnly
 ; CHECK-NEXT:   IsConst:         true
@@ -18,7 +17,6 @@
 ; CHECK-NEXT:   Size:            8
 ; CHECK-NEXT:   Align:           8
 ; CHECK-NEXT:   ValueKind:       GlobalBuffer
-; CHECK-NEXT:   ValueType:       F32
 ; CHECK-NEXT:   AddrSpaceQual:   Global
 ; CHECK-NEXT:   AccQual:         Default
 
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-v3.ll
index 850c6f1a6f63a..5f77fef31bfa8 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-v3.ll
@@ -9,19 +9,15 @@
 ; CHECK-NEXT:         .size:           1
 ; CHECK-NEXT:         .type_name:      char
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NOT:        .value_kind:    hidden_default_queue
 ; CHECK-NOT:        .value_kind:    hidden_completion_action
 ; CHECK:          .language:       OpenCL C
@@ -42,34 +38,27 @@ define amdgpu_kernel void @test_non_enqueue_kernel_caller(i8 %a) #0
 ; CHECK-NEXT:         .size:           1
 ; CHECK-NEXT:         .type_name:      char
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_default_queue
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_completion_action
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel.ll
index 9879a741ad43c..086c810c9ded3 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel.ll
@@ -16,20 +16,16 @@
 ; CHECK-NEXT:       Size:          1
 ; CHECK-NEXT:       Align:         1
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NOT:        ValueKind:     HiddenDefaultQueue
 ; CHECK-NOT:        ValueKind:     HiddenCompletionAction
 define amdgpu_kernel void @test_non_enqueue_kernel_caller(i8 %a) #0
@@ -48,34 +44,27 @@ define amdgpu_kernel void @test_non_enqueue_kernel_caller(i8 %a) #0
 ; CHECK-NEXT:       Size:          1
 ; CHECK-NEXT:       Align:         1
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenDefaultQueue
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenCompletionAction
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_enqueue_kernel_caller(i8 %a) #1
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full-v3.ll
index 74e80ec432084..ef441b9c3cc26 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full-v3.ll
@@ -24,19 +24,15 @@
 ; CHECK-NEXT:         .size:           1
 ; CHECK-NEXT:         .type_name:      char
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
@@ -44,9 +40,7 @@
 ; CHECK-NOT:          .value_kind:     hidden_completion_action
 ; CHECK-NOT:          .value_kind:     hidden_hostcall_buffer
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:              .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -65,39 +59,31 @@ define amdgpu_kernel void @test_char(i8 %a) #0
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      ushort2
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     u16
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -116,39 +102,31 @@ define amdgpu_kernel void @test_ushort2(<2 x i16> %a) #0
 ; CHECK-NEXT:         .size:           16
 ; CHECK-NEXT:         .type_name:      int3
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         64
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -167,39 +145,31 @@ define amdgpu_kernel void @test_int3(<3 x i32> %a) #0
 ; CHECK-NEXT:         .size:           32
 ; CHECK-NEXT:         .type_name:      ulong4
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     u64
 ; CHECK-NEXT:       - .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         64
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         72
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         80
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -218,39 +188,31 @@ define amdgpu_kernel void @test_ulong4(<4 x i64> %a) #0
 ; CHECK-NEXT:         .size:           16
 ; CHECK-NEXT:         .type_name:      half8
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         64
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -269,39 +231,31 @@ define amdgpu_kernel void @test_half8(<8 x half> %a) #0
 ; CHECK-NEXT:         .size:           64
 ; CHECK-NEXT:         .type_name:      float16
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     f32
 ; CHECK-NEXT:       - .offset:         64
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         72
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         80
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         88
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         96
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         104
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         112
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -320,39 +274,31 @@ define amdgpu_kernel void @test_float16(<16 x float> %a) #0
 ; CHECK-NEXT:         .size:           128
 ; CHECK-NEXT:         .type_name:      double16
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     f64
 ; CHECK-NEXT:       - .offset:         128
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         136
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         144
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         152
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         160
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         168
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         176
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -372,39 +318,31 @@ define amdgpu_kernel void @test_double16(<16 x double> %a) #0
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      'int  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -424,39 +362,31 @@ define amdgpu_kernel void @test_pointer(i32 addrspace(1)* %a) #0
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      image2d_t
 ; CHECK-NEXT:         .value_kind:     image
-; CHECK-NEXT:         .value_type:     struct
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -475,39 +405,31 @@ define amdgpu_kernel void @test_image(%opencl.image2d_t addrspace(1)* %a) #0
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      sampler_t
 ; CHECK-NEXT:         .value_kind:     sampler
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -527,39 +449,31 @@ define amdgpu_kernel void @test_sampler(i32 %a) #0
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      queue_t
 ; CHECK-NEXT:         .value_kind:     queue
-; CHECK-NEXT:         .value_type:     struct
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -578,39 +492,31 @@ define amdgpu_kernel void @test_queue(%opencl.queue_t addrspace(1)* %a) #0
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      struct A
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     struct
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -629,39 +535,31 @@ define amdgpu_kernel void @test_struct(%struct.A %a) #0
 ; CHECK-NEXT:         .size:           32
 ; CHECK-NEXT:         .type_name:      struct A
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     struct
 ; CHECK-NEXT:       - .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         64
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         72
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         80
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -680,39 +578,31 @@ define amdgpu_kernel void @test_array([32 x i8] %a) #0
 ; CHECK-NEXT:         .size:           16
 ; CHECK-NEXT:         .type_name:      i128
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     struct
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         64
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -731,51 +621,41 @@ define amdgpu_kernel void @test_i128(i128 %a) #0
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      int
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .name:           b
 ; CHECK-NEXT:         .offset:         4
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      short2
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i16
 ; CHECK-NEXT:       - .name:           c
 ; CHECK-NEXT:         .offset:         8
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      char3
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         64
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -795,14 +675,12 @@ define amdgpu_kernel void @test_multi_arg(i32 %a, <2 x i16> %b, <3 x i8> %c) #0
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      'int  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .address_space:  constant
 ; CHECK-NEXT:         .name:           c
 ; CHECK-NEXT:         .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      'int  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .address_space:  local
 ; CHECK-NEXT:         .name:           l
 ; CHECK-NEXT:         .offset:         16
@@ -810,39 +688,31 @@ define amdgpu_kernel void @test_multi_arg(i32 %a, <2 x i16> %b, <3 x i8> %c) #0
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      'int  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     dynamic_shared_pointer
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         64
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         72
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -865,7 +735,6 @@ define amdgpu_kernel void @test_addr_space(i32 addrspace(1)* %g,
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      'int  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .is_const:       true
 ; CHECK-NEXT:         .is_restrict:    true
@@ -874,7 +743,6 @@ define amdgpu_kernel void @test_addr_space(i32 addrspace(1)* %g,
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      'int  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .is_pipe:        true
 ; CHECK-NEXT:         .name:           c
@@ -882,39 +750,31 @@ define amdgpu_kernel void @test_addr_space(i32 addrspace(1)* %g,
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      'int  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     pipe
-; CHECK-NEXT:         .value_type:     struct
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         64
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         72
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -937,7 +797,6 @@ define amdgpu_kernel void @test_type_qual(i32 addrspace(1)* %a,
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      image1d_t
 ; CHECK-NEXT:         .value_kind:     image
-; CHECK-NEXT:         .value_type:     struct
 ; CHECK-NEXT:       - .access:         write_only
 ; CHECK-NEXT:         .address_space:  global
 ; CHECK-NEXT:         .name:           wo
@@ -945,7 +804,6 @@ define amdgpu_kernel void @test_type_qual(i32 addrspace(1)* %a,
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      image2d_t
 ; CHECK-NEXT:         .value_kind:     image
-; CHECK-NEXT:         .value_type:     struct
 ; CHECK-NEXT:       - .access:         read_write
 ; CHECK-NEXT:         .address_space:  global
 ; CHECK-NEXT:         .name:           rw
@@ -953,39 +811,31 @@ define amdgpu_kernel void @test_type_qual(i32 addrspace(1)* %a,
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      image3d_t
 ; CHECK-NEXT:         .value_kind:     image
-; CHECK-NEXT:         .value_type:     struct
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         64
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         72
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -1006,39 +856,31 @@ define amdgpu_kernel void @test_access_qual(%opencl.image1d_t addrspace(1)* %ro,
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      int
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -1058,39 +900,31 @@ define amdgpu_kernel void @test_vec_type_hint_half(i32 %a) #0
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      int
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -1110,39 +944,31 @@ define amdgpu_kernel void @test_vec_type_hint_float(i32 %a) #0
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      int
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -1162,39 +988,31 @@ define amdgpu_kernel void @test_vec_type_hint_double(i32 %a) #0
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      int
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -1214,39 +1032,31 @@ define amdgpu_kernel void @test_vec_type_hint_char(i32 %a) #0
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      int
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -1266,39 +1076,31 @@ define amdgpu_kernel void @test_vec_type_hint_short(i32 %a) #0
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      int
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -1318,39 +1120,31 @@ define amdgpu_kernel void @test_vec_type_hint_long(i32 %a) #0
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      int
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -1370,39 +1164,31 @@ define amdgpu_kernel void @test_vec_type_hint_unknown(i32 %a) #0
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      int
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -1427,39 +1213,31 @@ define amdgpu_kernel void @test_reqd_wgs_vec_type_hint(i32 %a) #0
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      int
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -1485,39 +1263,31 @@ define amdgpu_kernel void @test_wgs_hint_vec_type_hint(i32 %a) #0
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      'int  addrspace(5)* addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -1536,39 +1306,31 @@ define amdgpu_kernel void @test_arg_ptr_to_ptr(i32 addrspace(5)* addrspace(1)* %
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      struct B
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     struct
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -1587,39 +1349,31 @@ define amdgpu_kernel void @test_arg_struct_contains_ptr(%struct.B %a) #0
 ; CHECK-NEXT:         .size:           16
 ; CHECK-NEXT:         .type_name:      'global int addrspace(5)* __attribute__((ext_vector_type(2)))'
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         64
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -1639,39 +1393,31 @@ define amdgpu_kernel void @test_arg_vector_of_ptr(<2 x i32 addrspace(1)*> %a) #0
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      clk_event_t
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     struct
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -1692,7 +1438,6 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      'long  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  local
 ; CHECK-NEXT:         .name:           b
 ; CHECK-NEXT:         .offset:         8
@@ -1700,7 +1445,6 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      'char  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     dynamic_shared_pointer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  local
 ; CHECK-NEXT:         .name:           c
 ; CHECK-NEXT:         .offset:         12
@@ -1708,7 +1452,6 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      'char2  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     dynamic_shared_pointer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  local
 ; CHECK-NEXT:         .name:           d
 ; CHECK-NEXT:         .offset:         16
@@ -1716,7 +1459,6 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      'char3  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     dynamic_shared_pointer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  local
 ; CHECK-NEXT:         .name:           e
 ; CHECK-NEXT:         .offset:         20
@@ -1724,7 +1466,6 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      'char4  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     dynamic_shared_pointer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  local
 ; CHECK-NEXT:         .name:           f
 ; CHECK-NEXT:         .offset:         24
@@ -1732,7 +1473,6 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      'char8  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     dynamic_shared_pointer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  local
 ; CHECK-NEXT:         .name:           g
 ; CHECK-NEXT:         .offset:         28
@@ -1740,46 +1480,37 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      'char16  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     dynamic_shared_pointer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  local
 ; CHECK-NEXT:         .name:           h
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .pointee_align:  1
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .value_kind:     dynamic_shared_pointer
-; CHECK-NEXT:         .value_type:     struct
 ; CHECK-NEXT:       - .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         64
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         72
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         80
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         88
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -1806,7 +1537,6 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .type_name:      'long  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  local
 ; CHECK-NEXT:         .name:           b
 ; CHECK-NEXT:         .offset:         8
@@ -1814,7 +1544,6 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      'char  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     dynamic_shared_pointer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  local
 ; CHECK-NEXT:         .name:           c
 ; CHECK-NEXT:         .offset:         12
@@ -1822,7 +1551,6 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      'char2  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     dynamic_shared_pointer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  local
 ; CHECK-NEXT:         .name:           d
 ; CHECK-NEXT:         .offset:         16
@@ -1830,7 +1558,6 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      'char3  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     dynamic_shared_pointer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  local
 ; CHECK-NEXT:         .name:           e
 ; CHECK-NEXT:         .offset:         20
@@ -1838,7 +1565,6 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      'char4  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     dynamic_shared_pointer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  local
 ; CHECK-NEXT:         .name:           f
 ; CHECK-NEXT:         .offset:         24
@@ -1846,7 +1572,6 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      'char8  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     dynamic_shared_pointer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  local
 ; CHECK-NEXT:         .name:           g
 ; CHECK-NEXT:         .offset:         28
@@ -1854,46 +1579,37 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .type_name:      'char16  addrspace(5)*'
 ; CHECK-NEXT:         .value_kind:     dynamic_shared_pointer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  local
 ; CHECK-NEXT:         .name:           h
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .pointee_align:  16
 ; CHECK-NEXT:         .size:           4
 ; CHECK-NEXT:         .value_kind:     dynamic_shared_pointer
-; CHECK-NEXT:         .value_type:     struct
 ; CHECK-NEXT:       - .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         64
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         72
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         80
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         88
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -1918,39 +1634,31 @@ define amdgpu_kernel void @test_pointee_align_attribute(i64 addrspace(1)* align
 ; CHECK-NEXT:         .size:           25
 ; CHECK-NEXT:         .type_name:      __block_literal
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     struct
 ; CHECK-NEXT:       - .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         64
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         72
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         80
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .device_enqueue_symbol: __test_block_invoke_kernel_runtime_handle
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
@@ -1971,39 +1679,31 @@ define amdgpu_kernel void @__test_block_invoke_kernel(
 ; CHECK-NEXT:         .size:           1
 ; CHECK-NEXT:         .type_name:      char
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_default_queue
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_completion_action
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
@@ -2021,7 +1721,6 @@ define amdgpu_kernel void @test_enqueue_kernel_caller(i8 %a) #2
 ; CHECK-NEXT:         .offset:         0
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     i32
 ; CHECK:          .name:           unknown_addrspace_kernarg
 ; CHECK:          .symbol:         unknown_addrspace_kernarg.kd
 define amdgpu_kernel void @unknown_addrspace_kernarg(i32 addrspace(12345)* %ptr) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll
index d5a5fb630d712..8969c7b45bbbd 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll
@@ -33,30 +33,24 @@
 ; CHECK-NEXT:       Size:          1
 ; CHECK-NEXT:       Align:         1
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NOT:        ValueKind:     HiddenHostcallBuffer
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NOT:        ValueKind:     HiddenDefaultQueue
 ; CHECK-NOT:        ValueKind:     HiddenCompletionAction
 ; CHECK:            ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_char(i8 %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !9
@@ -74,39 +68,31 @@ define amdgpu_kernel void @test_char(i8 %a) #0
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     U16
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_ushort2(<2 x i16> %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !10
@@ -124,39 +110,31 @@ define amdgpu_kernel void @test_ushort2(<2 x i16> %a) #0
 ; CHECK-NEXT:       Size:          16
 ; CHECK-NEXT:       Align:         16
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_int3(<3 x i32> %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !11
@@ -174,39 +152,31 @@ define amdgpu_kernel void @test_int3(<3 x i32> %a) #0
 ; CHECK-NEXT:       Size:          32
 ; CHECK-NEXT:       Align:         32
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     U64
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_ulong4(<4 x i64> %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !12
@@ -224,39 +194,31 @@ define amdgpu_kernel void @test_ulong4(<4 x i64> %a) #0
 ; CHECK-NEXT:       Size:          16
 ; CHECK-NEXT:       Align:         16
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     F16
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_half8(<8 x half> %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !13
@@ -274,39 +236,31 @@ define amdgpu_kernel void @test_half8(<8 x half> %a) #0
 ; CHECK-NEXT:       Size:          64
 ; CHECK-NEXT:       Align:         64
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     F32
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_float16(<16 x float> %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !14
@@ -324,39 +278,31 @@ define amdgpu_kernel void @test_float16(<16 x float> %a) #0
 ; CHECK-NEXT:       Size:          128
 ; CHECK-NEXT:       Align:         128
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     F64
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_double16(<16 x double> %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !15
@@ -374,40 +320,32 @@ define amdgpu_kernel void @test_double16(<16 x double> %a) #0
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_pointer(i32 addrspace(1)* %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !16
@@ -425,40 +363,32 @@ define amdgpu_kernel void @test_pointer(i32 addrspace(1)* %a) #0
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     Image
-; CHECK-NEXT:       ValueType:     Struct
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_image(%opencl.image2d_t addrspace(1)* %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !17
@@ -476,39 +406,31 @@ define amdgpu_kernel void @test_image(%opencl.image2d_t addrspace(1)* %a) #0
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     Sampler
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_sampler(i32 %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !18
@@ -526,40 +448,32 @@ define amdgpu_kernel void @test_sampler(i32 %a) #0
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     Queue
-; CHECK-NEXT:       ValueType:     Struct
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_queue(%opencl.queue_t addrspace(1)* %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !19
@@ -577,39 +491,31 @@ define amdgpu_kernel void @test_queue(%opencl.queue_t addrspace(1)* %a) #0
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     Struct
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_struct(%struct.A %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !20
@@ -627,39 +533,31 @@ define amdgpu_kernel void @test_struct(%struct.A %a) #0
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         1
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     Struct
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_array([8 x i8] %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !20
@@ -677,39 +575,31 @@ define amdgpu_kernel void @test_array([8 x i8] %a) #0
 ; CHECK-NEXT:       Size:          16
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     Struct
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_i128(i128 %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !21
@@ -727,53 +617,43 @@ define amdgpu_kernel void @test_i128(i128 %a) #0
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Name:          b
 ; CHECK-NEXT:       TypeName:      short2
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I16
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Name:          c
 ; CHECK-NEXT:       TypeName:      char3
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_multi_arg(i32 %a, <2 x i16> %b, <3 x i8> %c) #0
     !kernel_arg_addr_space !22 !kernel_arg_access_qual !23 !kernel_arg_type !24
@@ -791,7 +671,6 @@ define amdgpu_kernel void @test_multi_arg(i32 %a, <2 x i16> %b, <3 x i8> %c) #0
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Name:          c
@@ -799,7 +678,6 @@ define amdgpu_kernel void @test_multi_arg(i32 %a, <2 x i16> %b, <3 x i8> %c) #0
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AddrSpaceQual: Constant
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Name:          l
@@ -807,41 +685,33 @@ define amdgpu_kernel void @test_multi_arg(i32 %a, <2 x i16> %b, <3 x i8> %c) #0
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       PointeeAlign:  4
 ; CHECK-NEXT:       AddrSpaceQual: Local
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_addr_space(i32 addrspace(1)* %g,
                                            i32 addrspace(4)* %c,
@@ -861,7 +731,6 @@ define amdgpu_kernel void @test_addr_space(i32 addrspace(1)* %g,
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       IsVolatile:    true
@@ -870,7 +739,6 @@ define amdgpu_kernel void @test_addr_space(i32 addrspace(1)* %g,
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       IsConst:       true
@@ -880,41 +748,33 @@ define amdgpu_kernel void @test_addr_space(i32 addrspace(1)* %g,
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     Pipe
-; CHECK-NEXT:       ValueType:     Struct
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       IsPipe:        true
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_type_qual(i32 addrspace(1)* %a,
                                           i32 addrspace(1)* %b,
@@ -934,7 +794,6 @@ define amdgpu_kernel void @test_type_qual(i32 addrspace(1)* %a,
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     Image
-; CHECK-NEXT:       ValueType:     Struct
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:       AccQual:       ReadOnly
 ; CHECK-NEXT:     - Name:          wo
@@ -942,7 +801,6 @@ define amdgpu_kernel void @test_type_qual(i32 addrspace(1)* %a,
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     Image
-; CHECK-NEXT:       ValueType:     Struct
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:       AccQual:       WriteOnly
 ; CHECK-NEXT:     - Name:          rw
@@ -950,40 +808,32 @@ define amdgpu_kernel void @test_type_qual(i32 addrspace(1)* %a,
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     Image
-; CHECK-NEXT:       ValueType:     Struct
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:       AccQual:       ReadWrite
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_access_qual(%opencl.image1d_t addrspace(1)* %ro,
                                             %opencl.image2d_t addrspace(1)* %wo,
@@ -1005,39 +855,31 @@ define amdgpu_kernel void @test_access_qual(%opencl.image1d_t addrspace(1)* %ro,
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_vec_type_hint_half(i32 %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3
@@ -1057,39 +899,31 @@ define amdgpu_kernel void @test_vec_type_hint_half(i32 %a) #0
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_vec_type_hint_float(i32 %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3
@@ -1109,39 +943,31 @@ define amdgpu_kernel void @test_vec_type_hint_float(i32 %a) #0
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_vec_type_hint_double(i32 %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3
@@ -1161,39 +987,31 @@ define amdgpu_kernel void @test_vec_type_hint_double(i32 %a) #0
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_vec_type_hint_char(i32 %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3
@@ -1213,39 +1031,31 @@ define amdgpu_kernel void @test_vec_type_hint_char(i32 %a) #0
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_vec_type_hint_short(i32 %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3
@@ -1265,39 +1075,31 @@ define amdgpu_kernel void @test_vec_type_hint_short(i32 %a) #0
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_vec_type_hint_long(i32 %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3
@@ -1317,39 +1119,31 @@ define amdgpu_kernel void @test_vec_type_hint_long(i32 %a) #0
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_vec_type_hint_unknown(i32 %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3
@@ -1370,39 +1164,31 @@ define amdgpu_kernel void @test_vec_type_hint_unknown(i32 %a) #0
 ; CHECK-NEXT:       Size:              4
 ; CHECK-NEXT:       Align:             4
 ; CHECK-NEXT:       ValueKind:         ByValue
-; CHECK-NEXT:       ValueType:         I32
 ; CHECK-NEXT:       AccQual:           Default
 ; CHECK-NEXT:     - Size:              8
 ; CHECK-NEXT:       Align:             8
 ; CHECK-NEXT:       ValueKind:         HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:         I64
 ; CHECK-NEXT:     - Size:              8
 ; CHECK-NEXT:       Align:             8
 ; CHECK-NEXT:       ValueKind:         HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:         I64
 ; CHECK-NEXT:     - Size:              8
 ; CHECK-NEXT:       Align:             8
 ; CHECK-NEXT:       ValueKind:         HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:         I64
 ; CHECK-NEXT:     - Size:              8
 ; CHECK-NEXT:       Align:             8
 ; CHECK-NEXT:       ValueKind:         HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:         I8
 ; CHECK-NEXT:       AddrSpaceQual:     Global
 ; CHECK-NEXT:     - Size:              8
 ; CHECK-NEXT:       Align:             8
 ; CHECK-NEXT:       ValueKind:         HiddenNone
-; CHECK-NEXT:       ValueType:         I8
 ; CHECK-NEXT:       AddrSpaceQual:     Global
 ; CHECK-NEXT:     - Size:              8
 ; CHECK-NEXT:       Align:             8
 ; CHECK-NEXT:       ValueKind:         HiddenNone
-; CHECK-NEXT:       ValueType:         I8
 ; CHECK-NEXT:       AddrSpaceQual:     Global
 ; CHECK-NEXT:     - Size:              8
 ; CHECK-NEXT:       Align:             8
 ; CHECK-NEXT:       ValueKind:         HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:         I8
 ; CHECK-NEXT:       AddrSpaceQual:     Global
 define amdgpu_kernel void @test_reqd_wgs_vec_type_hint(i32 %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3
@@ -1424,39 +1210,31 @@ define amdgpu_kernel void @test_reqd_wgs_vec_type_hint(i32 %a) #0
 ; CHECK-NEXT:       Size:              4
 ; CHECK-NEXT:       Align:             4
 ; CHECK-NEXT:       ValueKind:         ByValue
-; CHECK-NEXT:       ValueType:         I32
 ; CHECK-NEXT:       AccQual:           Default
 ; CHECK-NEXT:     - Size:              8
 ; CHECK-NEXT:       Align:             8
 ; CHECK-NEXT:       ValueKind:         HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:         I64
 ; CHECK-NEXT:     - Size:              8
 ; CHECK-NEXT:       Align:             8
 ; CHECK-NEXT:       ValueKind:         HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:         I64
 ; CHECK-NEXT:     - Size:              8
 ; CHECK-NEXT:       Align:             8
 ; CHECK-NEXT:       ValueKind:         HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:         I64
 ; CHECK-NEXT:     - Size:              8
 ; CHECK-NEXT:       Align:             8
 ; CHECK-NEXT:       ValueKind:         HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:         I8
 ; CHECK-NEXT:       AddrSpaceQual:     Global
 ; CHECK-NEXT:     - Size:              8
 ; CHECK-NEXT:       Align:             8
 ; CHECK-NEXT:       ValueKind:         HiddenNone
-; CHECK-NEXT:       ValueType:         I8
 ; CHECK-NEXT:       AddrSpaceQual:     Global
 ; CHECK-NEXT:     - Size:              8
 ; CHECK-NEXT:       Align:             8
 ; CHECK-NEXT:       ValueKind:         HiddenNone
-; CHECK-NEXT:       ValueType:         I8
 ; CHECK-NEXT:       AddrSpaceQual:     Global
 ; CHECK-NEXT:     - Size:              8
 ; CHECK-NEXT:       Align:             8
 ; CHECK-NEXT:       ValueKind:         HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:         I8
 ; CHECK-NEXT:       AddrSpaceQual:     Global
 define amdgpu_kernel void @test_wgs_hint_vec_type_hint(i32 %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3
@@ -1475,40 +1253,32 @@ define amdgpu_kernel void @test_wgs_hint_vec_type_hint(i32 %a) #0
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_arg_ptr_to_ptr(i32 addrspace(5)* addrspace(1)* %a) #0
     !kernel_arg_addr_space !81 !kernel_arg_access_qual !2 !kernel_arg_type !80
@@ -1526,24 +1296,19 @@ define amdgpu_kernel void @test_arg_ptr_to_ptr(i32 addrspace(5)* addrspace(1)* %
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     Struct
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_arg_struct_contains_ptr(%struct.B %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !82
@@ -1561,39 +1326,31 @@ define amdgpu_kernel void @test_arg_struct_contains_ptr(%struct.B %a) #0
 ; CHECK-NEXT:       Size:          16
 ; CHECK-NEXT:       Align:         16
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_arg_vector_of_ptr(<2 x i32 addrspace(1)*> %a) #0
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !83
@@ -1611,40 +1368,32 @@ define amdgpu_kernel void @test_arg_vector_of_ptr(<2 x i32 addrspace(1)*> %a) #0
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
-; CHECK-NEXT:       ValueType:     Struct
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_arg_unknown_builtin_type(
     %opencl.clk_event_t addrspace(1)* %a) #0
@@ -1663,7 +1412,6 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Name:          b
@@ -1671,7 +1419,6 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  1
 ; CHECK-NEXT:       AddrSpaceQual: Local
 ; CHECK-NEXT:       AccQual:       Default
@@ -1680,7 +1427,6 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  2
 ; CHECK-NEXT:       AddrSpaceQual: Local
 ; CHECK-NEXT:       AccQual:       Default
@@ -1689,7 +1435,6 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  4
 ; CHECK-NEXT:       AddrSpaceQual: Local
 ; CHECK-NEXT:       AccQual:       Default
@@ -1698,7 +1443,6 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  4
 ; CHECK-NEXT:       AddrSpaceQual: Local
 ; CHECK-NEXT:       AccQual:       Default
@@ -1707,7 +1451,6 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  8
 ; CHECK-NEXT:       AddrSpaceQual: Local
 ; CHECK-NEXT:       AccQual:       Default
@@ -1716,7 +1459,6 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  16
 ; CHECK-NEXT:       AddrSpaceQual: Local
 ; CHECK-NEXT:       AccQual:       Default
@@ -1724,40 +1466,32 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
-; CHECK-NEXT:       ValueType:     Struct
 ; CHECK-NEXT:       PointeeAlign:  1
 ; CHECK-NEXT:       AddrSpaceQual: Local
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
                                               i8 addrspace(3)* %b,
@@ -1782,7 +1516,6 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
 ; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Name:          b
@@ -1790,7 +1523,6 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  8
 ; CHECK-NEXT:       AddrSpaceQual: Local
 ; CHECK-NEXT:       AccQual:       Default
@@ -1799,7 +1531,6 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  32
 ; CHECK-NEXT:       AddrSpaceQual: Local
 ; CHECK-NEXT:       AccQual:       Default
@@ -1808,7 +1539,6 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  64
 ; CHECK-NEXT:       AddrSpaceQual: Local
 ; CHECK-NEXT:       AccQual:       Default
@@ -1817,7 +1547,6 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  256
 ; CHECK-NEXT:       AddrSpaceQual: Local
 ; CHECK-NEXT:       AccQual:       Default
@@ -1826,7 +1555,6 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  128
 ; CHECK-NEXT:       AddrSpaceQual: Local
 ; CHECK-NEXT:       AccQual:       Default
@@ -1835,7 +1563,6 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
 ; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  1024
 ; CHECK-NEXT:       AddrSpaceQual: Local
 ; CHECK-NEXT:       AccQual:       Default
@@ -1843,40 +1570,32 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
 ; CHECK-NEXT:        Size:            4
 ; CHECK-NEXT:        Align:           4
 ; CHECK-NEXT:        ValueKind:       DynamicSharedPointer
-; CHECK-NEXT:        ValueType:       Struct
 ; CHECK-NEXT:        PointeeAlign:    16
 ; CHECK-NEXT:        AddrSpaceQual:   Local
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_pointee_align_attribute(i64 addrspace(1)* align 16 %a,
                                                         i8 addrspace(3)* align 8 %b,
@@ -1904,39 +1623,31 @@ define amdgpu_kernel void @test_pointee_align_attribute(i64 addrspace(1)* align
 ; CHECK-NEXT:       Size:          25
 ; CHECK-NEXT:       Align:         1
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     Struct
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenNone
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @__test_block_invoke_kernel(
     <{ i32, i32, i8*, i8 addrspace(1)*, i8 }> %arg) #1
@@ -1955,39 +1666,31 @@ define amdgpu_kernel void @__test_block_invoke_kernel(
 ; CHECK-NEXT:       Size:          1
 ; CHECK-NEXT:       Align:         1
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenDefaultQueue
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenCompletionAction
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
 define amdgpu_kernel void @test_enqueue_kernel_caller(i8 %a) #2
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !9
@@ -2001,7 +1704,6 @@ define amdgpu_kernel void @test_enqueue_kernel_caller(i8 %a) #2
 ; CHECK-NEXT: Size:            8
 ; CHECK-NEXT: Align:           8
 ; CHECK-NEXT: ValueKind:       GlobalBuffer
-; CHECK-NEXT: ValueType:       I32
 define amdgpu_kernel void @unknown_addrspace_kernarg(i32 addrspace(12345)* %ptr) #0 {
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v3.ll
index 20368f6643ade..67e2bfce50356 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v3.ll
@@ -11,19 +11,16 @@
 ; CHECK-NEXT:         .offset:         0
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .name:           a
 ; CHECK-NEXT:         .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .name:           b
 ; CHECK-NEXT:         .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK:          .name:           test0
 ; CHECK:          .symbol:         test0.kd
 define amdgpu_kernel void @test0(
@@ -44,23 +41,19 @@ entry:
 ; CHECK-NEXT:         .offset:         0
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .name:           a
 ; CHECK-NEXT:         .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .name:           b
 ; CHECK-NEXT:         .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK:          .name:           test8
 ; CHECK:          .symbol:         test8.kd
 define amdgpu_kernel void @test8(
@@ -81,27 +74,22 @@ entry:
 ; CHECK-NEXT:         .offset:         0
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .name:           a
 ; CHECK-NEXT:         .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .name:           b
 ; CHECK-NEXT:         .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK:          .name:           test16
 ; CHECK:          .symbol:         test16.kd
 define amdgpu_kernel void @test16(
@@ -122,31 +110,25 @@ entry:
 ; CHECK-NEXT:         .offset:         0
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .name:           a
 ; CHECK-NEXT:         .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .name:           b
 ; CHECK-NEXT:         .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK:          .name:           test24
 ; CHECK:          .symbol:         test24.kd
 define amdgpu_kernel void @test24(
@@ -167,36 +149,29 @@ entry:
 ; CHECK-NEXT:         .offset:         0
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .name:           a
 ; CHECK-NEXT:         .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .name:           b
 ; CHECK-NEXT:         .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .name:           test32
 ; CHECK:          .symbol:         test32.kd
 define amdgpu_kernel void @test32(
@@ -217,46 +192,37 @@ entry:
 ; CHECK-NEXT:         .offset:         0
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .name:           a
 ; CHECK-NEXT:         .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .name:           b
 ; CHECK-NEXT:         .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         64
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .name:           test48
 ; CHECK:          .symbol:         test48.kd
 define amdgpu_kernel void @test48(
@@ -277,51 +243,41 @@ entry:
 ; CHECK-NEXT:         .offset:         0
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .name:           a
 ; CHECK-NEXT:         .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .name:           b
 ; CHECK-NEXT:         .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     global_buffer
-; CHECK-NEXT:         .value_type:     f16
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         40
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         48
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         56
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         64
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_none
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         72
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .name:           test56
 ; CHECK:          .symbol:         test56.kd
 define amdgpu_kernel void @test56(
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args.ll
index 8b59cdde8458f..91e382dcce5fb 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args.ll
@@ -13,19 +13,16 @@
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Name:            a
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Name:            b
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:   CodeProps:
 define amdgpu_kernel void @test0(
@@ -47,24 +44,20 @@ entry:
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Name:            a
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Name:            b
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:       I64
 ; CHECK-NEXT:   CodeProps:
 define amdgpu_kernel void @test8(
     half addrspace(1)* %r,
@@ -85,28 +78,23 @@ entry:
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Name:            a
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Name:            b
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:       I64
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:       I64
 ; CHECK-NEXT:   CodeProps:
 define amdgpu_kernel void @test16(
     half addrspace(1)* %r,
@@ -127,32 +115,26 @@ entry:
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Name:            a
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Name:            b
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:       I64
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:       I64
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:       I64
 ; CHECK-NEXT:   CodeProps:
 define amdgpu_kernel void @test24(
     half addrspace(1)* %r,
@@ -173,36 +155,29 @@ entry:
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Name:            a
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Name:            b
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:       I64
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:       I64
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:       I64
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenNone
-; CHECK-NEXT:       ValueType:       I8
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:   CodeProps:
 define amdgpu_kernel void @test32(
@@ -224,46 +199,37 @@ entry:
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Name:            a
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Name:            b
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:       I64
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:       I64
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:       I64
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenNone
-; CHECK-NEXT:       ValueType:       I8
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenNone
-; CHECK-NEXT:       ValueType:       I8
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenNone
-; CHECK-NEXT:       ValueType:       I8
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:   CodeProps:
 define amdgpu_kernel void @test48(
@@ -285,51 +251,41 @@ entry:
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Name:            a
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Name:            b
 ; CHECK-NEXT:       Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       GlobalBuffer
-; CHECK-NEXT:       ValueType:       F16
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:       I64
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:       I64
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:       I64
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenNone
-; CHECK-NEXT:       ValueType:       I8
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenNone
-; CHECK-NEXT:       ValueType:       I8
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenNone
-; CHECK-NEXT:       ValueType:       I8
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenMultiGridSyncArg
-; CHECK-NEXT:       ValueType:       I8
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NEXT:   CodeProps:
 define amdgpu_kernel void @test56(
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent-v3.ll
index 8741bfbc1bb60..c52e09158eeb8 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent-v3.ll
@@ -9,19 +9,15 @@
 ; CHECK-NEXT:         .size:           1
 ; CHECK-NEXT:         .type_name:      char
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 
 ; CHECK-NOT:          .value_kind:     hidden_hostcall_buffer
 
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent.ll
index 5f1cda0fd216d..fb5874acbe7d7 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent.ll
@@ -15,20 +15,16 @@
 ; CHECK-NEXT:       Size:          1
 ; CHECK-NEXT:       Align:         1
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NOT:        ValueKind:     HiddenHostcallBuffer
 ; CHECK-NOT:        ValueKind:     HiddenDefaultQueue
 ; CHECK-NOT:        ValueKind:     HiddenCompletionAction
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-v3.ll
index 1a75f3661bd4e..8fd62c89a6033 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-v3.ll
@@ -9,24 +9,19 @@
 ; CHECK-NEXT:         .size:           1
 ; CHECK-NEXT:         .type_name:      char
 ; CHECK-NEXT:         .value_kind:     by_value
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK-NEXT:       - .offset:         8
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         16
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .offset:         24
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
-; CHECK-NEXT:         .value_type:     i64
 ; CHECK-NEXT:       - .address_space:  global
 ; CHECK-NEXT:         .offset:         32
 ; CHECK-NEXT:         .size:           8
 ; CHECK-NEXT:         .value_kind:     hidden_hostcall_buffer
-; CHECK-NEXT:         .value_type:     i8
 ; CHECK:          .language:       OpenCL C
 ; CHECK-NEXT:     .language_version:
 ; CHECK-NEXT:       - 2
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present.ll
index b0428638e2541..e8db4d2a866eb 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present.ll
@@ -15,24 +15,19 @@
 ; CHECK-NEXT:       Size:          1
 ; CHECK-NEXT:       Align:         1
 ; CHECK-NEXT:       ValueKind:     ByValue
-; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
-; CHECK-NEXT:       ValueType:     I64
 ; CHECK-NEXT:     - Size:            8
 ; CHECK-NEXT:       Align:           8
 ; CHECK-NEXT:       ValueKind:       HiddenHostcallBuffer
-; CHECK-NEXT:       ValueType:       I8
 ; CHECK-NEXT:       AddrSpaceQual:   Global
 ; CHECK-NOT:        ValueKind:     HiddenDefaultQueue
 ; CHECK-NOT:        ValueKind:     HiddenCompletionAction
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-images-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-images-v3.ll
index ec048c2d02dba..5ee17a8c4855a 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-images-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-images-v3.ll
@@ -16,92 +16,80 @@
 %opencl.image3d_t = type opaque
 
 ; CHECK:         ---
-; CHECK: amdhsa.kernels:  
-; CHECK:   - .args:           
+; CHECK: amdhsa.kernels:
+; CHECK:   - .args:
 ; CHECK:       - .address_space:  global
 ; CHECK:         .name:           a
 ; CHECK:         .offset:         0
 ; CHECK:         .size:           8
 ; CHECK:         .type_name:      image1d_t
 ; CHECK:         .value_kind:     image
-; CHECK:         .value_type:     struct
 ; CHECK:       - .address_space:  global
 ; CHECK:         .name:           b
 ; CHECK:         .offset:         8
 ; CHECK:         .size:           8
 ; CHECK:         .type_name:      image1d_array_t
 ; CHECK:         .value_kind:     image
-; CHECK:         .value_type:     struct
 ; CHECK:       - .address_space:  global
 ; CHECK:         .name:           c
 ; CHECK:         .offset:         16
 ; CHECK:         .size:           8
 ; CHECK:         .type_name:      image1d_buffer_t
 ; CHECK:         .value_kind:     image
-; CHECK:         .value_type:     struct
 ; CHECK:       - .address_space:  global
 ; CHECK:         .name:           d
 ; CHECK:         .offset:         24
 ; CHECK:         .size:           8
 ; CHECK:         .type_name:      image2d_t
 ; CHECK:         .value_kind:     image
-; CHECK:         .value_type:     struct
 ; CHECK:       - .address_space:  global
 ; CHECK:         .name:           e
 ; CHECK:         .offset:         32
 ; CHECK:         .size:           8
 ; CHECK:         .type_name:      image2d_array_t
 ; CHECK:         .value_kind:     image
-; CHECK:         .value_type:     struct
 ; CHECK:       - .address_space:  global
 ; CHECK:         .name:           f
 ; CHECK:         .offset:         40
 ; CHECK:         .size:           8
 ; CHECK:         .type_name:      image2d_array_depth_t
 ; CHECK:         .value_kind:     image
-; CHECK:         .value_type:     struct
 ; CHECK:       - .address_space:  global
 ; CHECK:         .name:           g
 ; CHECK:         .offset:         48
 ; CHECK:         .size:           8
 ; CHECK:         .type_name:      image2d_array_msaa_t
 ; CHECK:         .value_kind:     image
-; CHECK:         .value_type:     struct
 ; CHECK:       - .address_space:  global
 ; CHECK:         .name:           h
 ; CHECK:         .offset:         56
 ; CHECK:         .size:           8
 ; CHECK:         .type_name:      image2d_array_msaa_depth_t
 ; CHECK:         .value_kind:     image
-; CHECK:         .value_type:     struct
 ; CHECK:       - .address_space:  global
 ; CHECK:         .name:           i
 ; CHECK:         .offset:         64
 ; CHECK:         .size:           8
 ; CHECK:         .type_name:      image2d_depth_t
 ; CHECK:         .value_kind:     image
-; CHECK:         .value_type:     struct
 ; CHECK:       - .address_space:  global
 ; CHECK:         .name:           j
 ; CHECK:         .offset:         72
 ; CHECK:         .size:           8
 ; CHECK:         .type_name:      image2d_msaa_t
 ; CHECK:         .value_kind:     image
-; CHECK:         .value_type:     struct
 ; CHECK:       - .address_space:  global
 ; CHECK:         .name:           k
 ; CHECK:         .offset:         80
 ; CHECK:         .size:           8
 ; CHECK:         .type_name:      image2d_msaa_depth_t
 ; CHECK:         .value_kind:     image
-; CHECK:         .value_type:     struct
 ; CHECK:       - .address_space:  global
 ; CHECK:         .name:           l
 ; CHECK:         .offset:         88
 ; CHECK:         .size:           8
 ; CHECK:         .type_name:      image3d_t
 ; CHECK:         .value_kind:     image
-; CHECK:         .value_type:     struct
 define amdgpu_kernel void @test(%opencl.image1d_t addrspace(1)* %a,
                                 %opencl.image1d_array_t addrspace(1)* %b,
                                 %opencl.image1d_buffer_t addrspace(1)* %c,
diff --git a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
index d9699fe4ce66e..cf17589f135d2 100644
--- a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
+++ b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
@@ -5,45 +5,41 @@ define amdgpu_kernel void @udiv32_invariant_denom(i32 addrspace(1)* nocapture %a
 ; GFX9-LABEL: udiv32_invariant_denom:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
-; GFX9-NEXT:    s_mov_b64 s[6:7], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
 ; GFX9-NEXT:    s_sub_i32 s3, 0, s2
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v0, s2
-; GFX9-NEXT:    v_mul_hi_u32 v2, v0, s2
-; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GFX9-NEXT:    v_add_u32_e32 v2, v0, v1
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
 ; GFX9-NEXT:  BB0_1: ; %bb3
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    v_mul_lo_u32 v3, v0, s7
-; GFX9-NEXT:    v_mul_hi_u32 v4, v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s4
-; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_mul_lo_u32 v3, s5, v0
+; GFX9-NEXT:    v_mul_hi_u32 v4, s4, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-NEXT:    v_add_u32_e32 v3, v4, v3
 ; GFX9-NEXT:    v_mul_lo_u32 v4, s3, v3
-; GFX9-NEXT:    v_mul_lo_u32 v5, v3, s2
-; GFX9-NEXT:    v_add_u32_e32 v6, 1, v3
-; GFX9-NEXT:    v_add_u32_e32 v7, -1, v3
-; GFX9-NEXT:    v_add_u32_e32 v4, s6, v4
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, s6, v5
-; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s2, v4
-; GFX9-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
-; GFX9-NEXT:    s_add_u32 s6, s6, 1
-; GFX9-NEXT:    s_addc_u32 s7, s7, 0
-; GFX9-NEXT:    s_add_u32 s4, s4, 4
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v6, s[0:1]
+; GFX9-NEXT:    v_not_b32_e32 v6, v3
+; GFX9-NEXT:    v_mul_lo_u32 v6, s2, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 1, v3
+; GFX9-NEXT:    v_add_u32_e32 v4, s4, v4
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s2, v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT:    v_add_u32_e32 v5, s4, v6
+; GFX9-NEXT:    s_add_u32 s4, s4, 1
 ; GFX9-NEXT:    s_addc_u32 s5, s5, 0
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX9-NEXT:    s_cmpk_eq_i32 s6, 0x400
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; GFX9-NEXT:    s_add_u32 s0, s0, 4
+; GFX9-NEXT:    s_addc_u32 s1, s1, 0
+; GFX9-NEXT:    v_add_u32_e32 v5, 1, v3
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s2, v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT:    s_cmpk_eq_i32 s4, 0x400
 ; GFX9-NEXT:    global_store_dword v[1:2], v3, off
 ; GFX9-NEXT:    s_cbranch_scc0 BB0_1
 ; GFX9-NEXT:  ; %bb.2: ; %bb2
@@ -69,49 +65,39 @@ define amdgpu_kernel void @urem32_invariant_denom(i32 addrspace(1)* nocapture %a
 ; GFX9-LABEL: urem32_invariant_denom:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
-; GFX9-NEXT:    s_mov_b64 s[6:7], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
 ; GFX9-NEXT:    s_sub_i32 s3, 0, s2
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v0, s2
-; GFX9-NEXT:    v_mul_hi_u32 v2, v0, s2
-; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GFX9-NEXT:    v_add_u32_e32 v2, v0, v1
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
 ; GFX9-NEXT:  BB1_1: ; %bb3
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    v_mul_lo_u32 v3, v0, s7
-; GFX9-NEXT:    v_mul_hi_u32 v4, v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s4
-; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_mul_lo_u32 v3, s5, v0
+; GFX9-NEXT:    v_mul_hi_u32 v4, s4, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-NEXT:    v_add_u32_e32 v3, v4, v3
-; GFX9-NEXT:    v_mul_lo_u32 v5, s3, v3
-; GFX9-NEXT:    v_mul_lo_u32 v4, v3, s2
-; GFX9-NEXT:    v_not_b32_e32 v6, v3
-; GFX9-NEXT:    v_sub_u32_e32 v3, 1, v3
+; GFX9-NEXT:    v_mul_lo_u32 v4, s3, v3
+; GFX9-NEXT:    v_not_b32_e32 v3, v3
 ; GFX9-NEXT:    v_mul_lo_u32 v3, s2, v3
-; GFX9-NEXT:    v_mul_lo_u32 v6, s2, v6
-; GFX9-NEXT:    v_add_u32_e32 v5, s6, v5
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[0:1], s6, v4
-; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s2, v5
-; GFX9-NEXT:    s_and_b64 vcc, vcc, s[0:1]
-; GFX9-NEXT:    v_add_u32_e32 v4, s6, v6
-; GFX9-NEXT:    v_add_u32_e32 v3, s6, v3
-; GFX9-NEXT:    s_add_u32 s6, s6, 1
-; GFX9-NEXT:    s_addc_u32 s7, s7, 0
-; GFX9-NEXT:    s_add_u32 s4, s4, 4
+; GFX9-NEXT:    v_add_u32_e32 v4, s4, v4
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s2, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, s4, v3
+; GFX9-NEXT:    s_add_u32 s4, s4, 1
 ; GFX9-NEXT:    s_addc_u32 s5, s5, 0
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v4, s[0:1]
-; GFX9-NEXT:    s_cmpk_eq_i32 s6, 0x400
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX9-NEXT:    s_add_u32 s0, s0, 4
+; GFX9-NEXT:    s_addc_u32 s1, s1, 0
+; GFX9-NEXT:    v_subrev_u32_e32 v4, s2, v3
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s2, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX9-NEXT:    s_cmpk_eq_i32 s4, 0x400
 ; GFX9-NEXT:    global_store_dword v[1:2], v3, off
 ; GFX9-NEXT:    s_cbranch_scc0 BB1_1
 ; GFX9-NEXT:  ; %bb.2: ; %bb2
@@ -137,45 +123,41 @@ define amdgpu_kernel void @sdiv32_invariant_denom(i32 addrspace(1)* nocapture %a
 ; GFX9-LABEL: sdiv32_invariant_denom:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x2c
-; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
-; GFX9-NEXT:    s_mov_b32 s6, 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_ashr_i32 s2, s3, 31
 ; GFX9-NEXT:    s_add_i32 s3, s3, s2
 ; GFX9-NEXT:    s_xor_b32 s3, s3, s2
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s3
+; GFX9-NEXT:    s_sub_i32 s4, 0, s3
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v0, s3
-; GFX9-NEXT:    v_mul_hi_u32 v2, v0, s3
-; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GFX9-NEXT:    v_add_u32_e32 v2, v0, v1
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v1, s4, v0
+; GFX9-NEXT:    s_mov_b32 s4, 0
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
 ; GFX9-NEXT:  BB2_1: ; %bb3
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    v_mul_hi_u32 v3, v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s4
-; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_mul_hi_u32 v3, s4, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-NEXT:    v_mul_lo_u32 v4, v3, s3
 ; GFX9-NEXT:    v_add_u32_e32 v5, 1, v3
-; GFX9-NEXT:    v_add_u32_e32 v6, -1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v7, s6, v4
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, s6, v4
-; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s3, v7
-; GFX9-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v5, s[0:1]
-; GFX9-NEXT:    s_add_i32 s6, s6, 1
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
-; GFX9-NEXT:    s_add_u32 s4, s4, 4
+; GFX9-NEXT:    v_sub_u32_e32 v4, s4, v4
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s3, v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v5, s3, v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; GFX9-NEXT:    s_add_i32 s4, s4, 1
+; GFX9-NEXT:    v_add_u32_e32 v5, 1, v3
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s3, v4
+; GFX9-NEXT:    s_add_u32 s0, s0, 4
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX9-NEXT:    v_xor_b32_e32 v3, s2, v3
-; GFX9-NEXT:    s_addc_u32 s5, s5, 0
+; GFX9-NEXT:    s_cmpk_eq_i32 s4, 0x400
 ; GFX9-NEXT:    v_subrev_u32_e32 v3, s2, v3
-; GFX9-NEXT:    s_cmpk_eq_i32 s6, 0x400
 ; GFX9-NEXT:    global_store_dword v[1:2], v3, off
 ; GFX9-NEXT:    s_cbranch_scc0 BB2_1
 ; GFX9-NEXT:  ; %bb.2: ; %bb2
@@ -201,43 +183,37 @@ define amdgpu_kernel void @srem32_invariant_denom(i32 addrspace(1)* nocapture %a
 ; GFX9-LABEL: srem32_invariant_denom:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_ashr_i32 s3, s2, 31
 ; GFX9-NEXT:    s_add_i32 s2, s2, s3
 ; GFX9-NEXT:    s_xor_b32 s2, s2, s3
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
-; GFX9-NEXT:    s_mov_b32 s3, 0
+; GFX9-NEXT:    s_sub_i32 s3, 0, s2
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v0, s2
-; GFX9-NEXT:    v_mul_hi_u32 v2, v0, s2
-; GFX9-NEXT:    v_sub_u32_e32 v3, 0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v0
-; GFX9-NEXT:    v_add_u32_e32 v2, v0, v1
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX9-NEXT:    s_mov_b32 s3, 0
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
 ; GFX9-NEXT:  BB3_1: ; %bb3
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    v_mul_hi_u32 v3, v0, s3
-; GFX9-NEXT:    v_mov_b32_e32 v1, s4
-; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_mul_hi_u32 v3, s3, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-NEXT:    v_mul_lo_u32 v3, v3, s2
-; GFX9-NEXT:    v_sub_u32_e32 v4, s3, v3
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[0:1], s3, v3
-; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s2, v4
+; GFX9-NEXT:    v_sub_u32_e32 v3, s3, v3
 ; GFX9-NEXT:    s_add_i32 s3, s3, 1
-; GFX9-NEXT:    s_and_b64 vcc, vcc, s[0:1]
-; GFX9-NEXT:    v_subrev_u32_e32 v3, s2, v4
-; GFX9-NEXT:    s_add_u32 s4, s4, 4
-; GFX9-NEXT:    s_addc_u32 s5, s5, 0
-; GFX9-NEXT:    v_add_u32_e32 v5, s2, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[0:1]
+; GFX9-NEXT:    v_subrev_u32_e32 v4, s2, v3
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s2, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX9-NEXT:    s_add_u32 s0, s0, 4
+; GFX9-NEXT:    s_addc_u32 s1, s1, 0
+; GFX9-NEXT:    v_subrev_u32_e32 v4, s2, v3
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s2, v3
 ; GFX9-NEXT:    s_cmpk_eq_i32 s3, 0x400
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
 ; GFX9-NEXT:    global_store_dword v[1:2], v3, off
 ; GFX9-NEXT:    s_cbranch_scc0 BB3_1
 ; GFX9-NEXT:  ; %bb.2: ; %bb2
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-call.ll b/llvm/test/CodeGen/AMDGPU/indirect-call.ll
index 8432d2961f04f..dacc77b499926 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-call.ll
@@ -81,16 +81,19 @@ define amdgpu_kernel void @test_indirect_call_sgpr_ptr() {
 ; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_add_u32 s0, s0, s17
 ; GCN-NEXT:    s_addc_u32 s1, s1, 0
-; GCN-NEXT:    s_getpc_b64 s[4:5]
-; GCN-NEXT:    s_add_u32 s4, s4, gv.fptr0@rel32@lo+4
-; GCN-NEXT:    s_addc_u32 s5, s5, gv.fptr0@rel32@hi+4
-; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GCN-NEXT:    s_mov_b32 s13, s15
+; GCN-NEXT:    s_mov_b32 s12, s14
+; GCN-NEXT:    s_getpc_b64 s[14:15]
+; GCN-NEXT:    s_add_u32 s14, s14, gv.fptr0@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s15, s15, gv.fptr0@rel32@hi+4
+; GCN-NEXT:    s_load_dwordx2 s[18:19], s[14:15], 0x0
 ; GCN-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GCN-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GCN-NEXT:    v_or_b32_e32 v31, v0, v2
+; GCN-NEXT:    s_mov_b32 s14, s16
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GCN-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GCN-NEXT:    s_endpgm
   %fptr = load void()*, void()* addrspace(4)* @gv.fptr0
   call void %fptr()
@@ -174,17 +177,20 @@ define amdgpu_kernel void @test_indirect_call_sgpr_ptr_arg() {
 ; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_add_u32 s0, s0, s17
 ; GCN-NEXT:    s_addc_u32 s1, s1, 0
-; GCN-NEXT:    s_getpc_b64 s[4:5]
-; GCN-NEXT:    s_add_u32 s4, s4, gv.fptr1@rel32@lo+4
-; GCN-NEXT:    s_addc_u32 s5, s5, gv.fptr1@rel32@hi+4
+; GCN-NEXT:    s_mov_b32 s13, s15
+; GCN-NEXT:    s_mov_b32 s12, s14
+; GCN-NEXT:    s_getpc_b64 s[14:15]
+; GCN-NEXT:    s_add_u32 s14, s14, gv.fptr1@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s15, s15, gv.fptr1@rel32@hi+4
 ; GCN-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; GCN-NEXT:    s_load_dwordx2 s[18:19], s[14:15], 0x0
 ; GCN-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GCN-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0x7b
+; GCN-NEXT:    s_mov_b32 s14, s16
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GCN-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; GCN-NEXT:    s_endpgm
   %fptr = load void(i32)*, void(i32)* addrspace(4)* @gv.fptr1
   call void %fptr(i32 123)
diff --git a/llvm/test/CodeGen/AMDGPU/inline-constraints.ll b/llvm/test/CodeGen/AMDGPU/inline-constraints.ll
index 63585ebc9553f..dc1d442959616 100644
--- a/llvm/test/CodeGen/AMDGPU/inline-constraints.ll
+++ b/llvm/test/CodeGen/AMDGPU/inline-constraints.ll
@@ -347,3 +347,971 @@ define i32 @inline_A_constraint_E2() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(i128 100000000000000000000)
   ret i32 %v0
 }
+
+;==============================================================================
+; 'I' constraint, 16-bit operand
+;==============================================================================
+
+; NOSI: error: invalid operand for inline asm constraint 'I'
+; VI-LABEL: {{^}}inline_I_constraint_H0:
+; VI: v_mov_b32 {{v[0-9]+}}, 64
+define i32 @inline_I_constraint_H0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(i16 64)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'I'
+; VI-LABEL: {{^}}inline_I_constraint_H1:
+; VI: v_mov_b32 {{v[0-9]+}}, -16
+define i32 @inline_I_constraint_H1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(half bitcast (i16 -16 to half))
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'I'
+define i32 @inline_I_constraint_H6() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(half 1.0)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'I'
+define i32 @inline_I_constraint_H7() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(i16 bitcast (half -1.0 to i16))
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'I'
+define i32 @inline_I_constraint_H8() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(i16 -17)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'I'
+define i32 @inline_I_constraint_H9() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(i16 65)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'I' constraint, 32-bit operand
+;==============================================================================
+
+; GCN-LABEL: {{^}}inline_I_constraint_F0:
+; GCN: v_mov_b32 {{v[0-9]+}}, -16
+define i32 @inline_I_constraint_F0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(i32 -16)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_I_constraint_F1:
+; GCN: v_mov_b32 {{v[0-9]+}}, -1
+define i32 @inline_I_constraint_F1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(float bitcast (i32 -1 to float))
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'I'
+define i32 @inline_I_constraint_F8() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(float -4.0)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'I'
+define i32 @inline_I_constraint_F9() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(i32 -17)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'I' constraint, 64-bit operand
+;==============================================================================
+
+; GCN-LABEL: {{^}}inline_I_constraint_D0:
+; GCN: v_mov_b32 {{v[0-9]+}}, -16
+define i32 @inline_I_constraint_D0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(i64 -16)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'I'
+define i32 @inline_I_constraint_D8() {
+  %v0 = tail call i32 asm "v_cvt_f32_f64 $0, $1", "=v,I"(double 0.5)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'I'
+define i32 @inline_I_constraint_D9() {
+  %v0 = tail call i32 asm "v_cvt_f32_f64 $0, $1", "=v,I"(i64 65)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'I' constraint, v2x16 operand
+;==============================================================================
+
+; NOSI: error: invalid operand for inline asm constraint 'I'
+; VI-LABEL: {{^}}inline_I_constraint_V0:
+; VI: v_mov_b32 {{v[0-9]+}}, -4
+define i32 @inline_I_constraint_V0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(<2 x i16> <i16 -4, i16 -4>)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'I'
+define i32 @inline_I_constraint_V1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(<2 x half> <half -0.5, half -0.5>)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'I'
+define i32 @inline_I_constraint_V2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(<2 x i16> <i16 -4, i16 undef>)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'I'
+define i32 @inline_I_constraint_V3() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(<2 x i16> <i16 1, i16 2>)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'I'
+define i32 @inline_I_constraint_V4() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(<4 x i16> <i16 0, i16 0, i16 0, i16 0>)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'I'
+define i32 @inline_I_constraint_V5() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(<2 x i32> <i32 0, i32 0>)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'I' constraint, type errors
+;==============================================================================
+
+; NOGCN: error: invalid operand for inline asm constraint 'I'
+define i32 @inline_I_constraint_E1(i32 %x) {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(i32 %x)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'I'
+define i32 @inline_I_constraint_E2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,I"(i128 100000000000000000000)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'J' constraint, 16-bit operand
+;==============================================================================
+
+; NOSI: error: invalid operand for inline asm constraint 'J'
+; VI-LABEL: {{^}}inline_J_constraint_H0:
+; VI: v_mov_b32 {{v[0-9]+}}, -1
+define i32 @inline_J_constraint_H0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(i16 65535)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'J'
+; VI-LABEL: {{^}}inline_J_constraint_H1:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x7fff
+define i32 @inline_J_constraint_H1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(i16 32767)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'J'
+; VI-LABEL: {{^}}inline_J_constraint_H2:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x8000
+define i32 @inline_J_constraint_H2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(i16 -32768)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'J'
+; VI-LABEL: {{^}}inline_J_constraint_H3:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x4800
+define i32 @inline_J_constraint_H3() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(half 8.0)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'J'
+; VI-LABEL: {{^}}inline_J_constraint_H4:
+; VI: v_mov_b32 {{v[0-9]+}}, -16
+define i32 @inline_J_constraint_H4() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(half bitcast (i16 -16 to half))
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'J' constraint, 32-bit operand
+;==============================================================================
+
+; GCN-LABEL: {{^}}inline_J_constraint_F0:
+; GCN: v_mov_b32 {{v[0-9]+}}, -1
+define i32 @inline_J_constraint_F0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(i32 -1)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_J_constraint_F1:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0x7fff
+define i32 @inline_J_constraint_F1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(i32 32767)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_J_constraint_F2:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0xffff8000
+define i32 @inline_J_constraint_F2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(i32 -32768)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'J'
+define i32 @inline_J_constraint_F6() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(i32 32768)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'J'
+define i32 @inline_J_constraint_F7() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(i32 -32769)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'J'
+define i32 @inline_J_constraint_F8() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(float -4.0)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'J' constraint, 64-bit operand
+;==============================================================================
+
+; GCN-LABEL: {{^}}inline_J_constraint_D0:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0x7fff
+define i32 @inline_J_constraint_D0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(i64 32767)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_J_constraint_D1:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0xffffffffffff8000
+define i32 @inline_J_constraint_D1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(i64 -32768)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'J'
+define i32 @inline_J_constraint_D8() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(i64 32768)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'J'
+define i32 @inline_J_constraint_D9() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(i64 -32769)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'J' constraint, v2x16 operand
+;==============================================================================
+
+; NOSI: error: invalid operand for inline asm constraint 'J'
+; VI-LABEL: {{^}}inline_J_constraint_V0:
+; VI: v_mov_b32 {{v[0-9]+}}, -4
+define i32 @inline_J_constraint_V0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(<2 x i16> <i16 -4, i16 -4>)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'J'
+; VI-LABEL: {{^}}inline_J_constraint_V1:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x7fff
+define i32 @inline_J_constraint_V1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(<2 x i16> <i16 32767, i16 32767>)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'J'
+; VI-LABEL: {{^}}inline_J_constraint_V2:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x8000
+define i32 @inline_J_constraint_V2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(<2 x i16> <i16 -32768, i16 -32768>)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'J'
+; VI-LABEL: {{^}}inline_J_constraint_V3:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x4c00
+define i32 @inline_J_constraint_V3() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(<2 x half> <half 16.0, half 16.0>)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'J' constraint, type errors
+;==============================================================================
+
+; NOGCN: error: invalid operand for inline asm constraint 'J'
+define i32 @inline_J_constraint_E1(i32 %x) {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(i32 %x)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'J'
+define i32 @inline_J_constraint_E2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,J"(i128 100000000000000000000)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'B' constraint, 16-bit operand
+;==============================================================================
+
+; NOSI: error: invalid operand for inline asm constraint 'B'
+; VI-LABEL: {{^}}inline_B_constraint_H0:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x7fff
+define i32 @inline_B_constraint_H0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,B"(i16 32767)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'B'
+; VI-LABEL: {{^}}inline_B_constraint_H1:
+; VI: v_mov_b32 {{v[0-9]+}}, -1
+define i32 @inline_B_constraint_H1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,B"(i16 65535)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'B'
+; VI-LABEL: {{^}}inline_B_constraint_H3:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x8000
+define i32 @inline_B_constraint_H3() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,B"(i16 -32768)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'B'
+; VI-LABEL: {{^}}inline_B_constraint_H4:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x4a80
+define i32 @inline_B_constraint_H4() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,B"(half 13.0)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'B' constraint, 32-bit operand
+;==============================================================================
+
+; GCN-LABEL: {{^}}inline_B_constraint_F0:
+; GCN: v_mov_b32 {{v[0-9]+}}, -1
+define i32 @inline_B_constraint_F0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,B"(i32 4294967295)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_B_constraint_F1:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0x80000000
+define i32 @inline_B_constraint_F1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,B"(i32 2147483648)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_B_constraint_F2:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0x42000000
+define i32 @inline_B_constraint_F2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,B"(float 32.0)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'B' constraint, 64-bit operand
+;==============================================================================
+
+; GCN-LABEL: {{^}}inline_B_constraint_D0:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0x7fffffff
+define i32 @inline_B_constraint_D0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,B"(i64 2147483647)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_B_constraint_D1:
+; GCN: v_mov_b32 {{v[0-9]+}}, -1
+define i32 @inline_B_constraint_D1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,B"(i64 -1)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_B_constraint_D2:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0xffffffff80000000
+define i32 @inline_B_constraint_D2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,B"(i64 -2147483648)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'B'
+define i32 @inline_B_constraint_D7() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,B"(i64 -2147483649)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'B'
+define i32 @inline_B_constraint_D8() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,B"(i64 4294967295)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'B'
+define i32 @inline_B_constraint_D9() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,B"(i64 2147483648)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'B' constraint, v2x16 operand
+;==============================================================================
+
+; NOSI: error: invalid operand for inline asm constraint 'B'
+; VI-LABEL: {{^}}inline_B_constraint_V0:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x7fff
+define i32 @inline_B_constraint_V0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,B"(<2 x i16> <i16 32767, i16 32767>)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'B'
+; VI-LABEL: {{^}}inline_B_constraint_V1:
+; VI: v_mov_b32 {{v[0-9]+}}, -1
+define i32 @inline_B_constraint_V1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,B"(<2 x i16> <i16 -1, i16 -1>)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'B'
+; VI-LABEL: {{^}}inline_B_constraint_V2:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x8000
+define i32 @inline_B_constraint_V2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,B"(<2 x i16> <i16 -32768, i16 -32768>)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'C' constraint, 16-bit operand
+;==============================================================================
+
+; NOSI: error: invalid operand for inline asm constraint 'C'
+; VI-LABEL: {{^}}inline_C_constraint_H0:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x7fff
+define i32 @inline_C_constraint_H0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(i16 32767)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'C'
+; VI-LABEL: {{^}}inline_C_constraint_H1:
+; VI: v_mov_b32 {{v[0-9]+}}, -1
+define i32 @inline_C_constraint_H1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(i16 65535)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'C'
+; VI-LABEL: {{^}}inline_C_constraint_H3:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x8000
+define i32 @inline_C_constraint_H3() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(i16 -32768)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'C'
+; VI-LABEL: {{^}}inline_C_constraint_H4:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x4a80
+define i32 @inline_C_constraint_H4() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(half 13.0)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'C' constraint, 32-bit operand
+;==============================================================================
+
+; GCN-LABEL: {{^}}inline_C_constraint_F0:
+; GCN: v_mov_b32 {{v[0-9]+}}, -1
+define i32 @inline_C_constraint_F0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(i32 4294967295)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_C_constraint_F1:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0x80000000
+define i32 @inline_C_constraint_F1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(i32 2147483648)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_C_constraint_F2:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0x7fffffff
+define i32 @inline_C_constraint_F2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(i32 2147483647)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_C_constraint_F3:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0x42000000
+define i32 @inline_C_constraint_F3() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(float 32.0)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_C_constraint_F4:
+; GCN: v_mov_b32 {{v[0-9]+}}, -16
+define i32 @inline_C_constraint_F4() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(i32 -16)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_C_constraint_F5:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0xffffffef
+define i32 @inline_C_constraint_F5() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(i32 -17)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'C' constraint, 64-bit operand
+;==============================================================================
+
+; GCN-LABEL: {{^}}inline_C_constraint_D0:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0xffffffff
+define i32 @inline_C_constraint_D0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(i64 4294967295)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_C_constraint_D1:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0x80000000
+define i32 @inline_C_constraint_D1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(i64 2147483648)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_C_constraint_D2:
+; GCN: v_mov_b32 {{v[0-9]+}}, -16
+define i32 @inline_C_constraint_D2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(i64 -16)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'C'
+define i32 @inline_C_constraint_D8() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(i64 -17)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'C'
+define i32 @inline_C_constraint_D9() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(i64 4294967296)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'C' constraint, v2x16 operand
+;==============================================================================
+
+; NOSI: error: invalid operand for inline asm constraint 'C'
+; VI-LABEL: {{^}}inline_C_constraint_V0:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x7fff
+define i32 @inline_C_constraint_V0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(<2 x i16> <i16 32767, i16 32767>)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'C'
+; VI-LABEL: {{^}}inline_C_constraint_V1:
+; VI: v_mov_b32 {{v[0-9]+}}, -1
+define i32 @inline_C_constraint_V1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(<2 x i16> <i16 -1, i16 -1>)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'C'
+; VI-LABEL: {{^}}inline_C_constraint_V2:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x8000
+define i32 @inline_C_constraint_V2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,C"(<2 x i16> <i16 -32768, i16 -32768>)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'DA' constraint, 16-bit operand
+;==============================================================================
+
+; NOSI: error: invalid operand for inline asm constraint 'DA'
+; VI-LABEL: {{^}}inline_DA_constraint_H0:
+; VI: v_mov_b32 {{v[0-9]+}}, 64
+define i32 @inline_DA_constraint_H0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 64)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'DA'
+; VI-LABEL: {{^}}inline_DA_constraint_H1:
+; VI: v_mov_b32 {{v[0-9]+}}, -16
+define i32 @inline_DA_constraint_H1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 -16)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'DA'
+; VI-LABEL: {{^}}inline_DA_constraint_H2:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x3c00
+define i32 @inline_DA_constraint_H2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 bitcast (half 1.0 to i16))
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'DA'
+; VI-LABEL: {{^}}inline_DA_constraint_H3:
+; VI: v_mov_b32 {{v[0-9]+}}, 0xbc00
+define i32 @inline_DA_constraint_H3() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 bitcast (half -1.0 to i16))
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'DA'
+; VI-LABEL: {{^}}inline_DA_constraint_H4:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x3118
+define i32 @inline_DA_constraint_H4() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(half 0xH3118)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'DA'
+; VI-LABEL: {{^}}inline_DA_constraint_H5:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x3118
+define i32 @inline_DA_constraint_H5() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 bitcast (half 0xH3118 to i16))
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'DA'
+; VI-LABEL: {{^}}inline_DA_constraint_H6:
+; VI: v_mov_b32 {{v[0-9]+}}, 0xb800
+define i32 @inline_DA_constraint_H6() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(half -0.5)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'DA'
+define i32 @inline_DA_constraint_H7() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 bitcast (half 0xH3119 to i16))
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'DA'
+define i32 @inline_DA_constraint_H8() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 -17)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'DA'
+define i32 @inline_DA_constraint_H9() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 65)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'DA' constraint, 32-bit operand
+;==============================================================================
+
+; GCN-LABEL: {{^}}inline_DA_constraint_F0:
+; GCN: v_mov_b32 {{v[0-9]+}}, -16
+define i32 @inline_DA_constraint_F0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i32 -16)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_DA_constraint_F1:
+; GCN: v_mov_b32 {{v[0-9]+}}, 1
+define i32 @inline_DA_constraint_F1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i32 1)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_DA_constraint_F2:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0xbf000000
+define i32 @inline_DA_constraint_F2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i32 bitcast (float -0.5 to i32))
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_DA_constraint_F3:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0x40000000
+define i32 @inline_DA_constraint_F3() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i32 bitcast (float 2.0 to i32))
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_DA_constraint_F4:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0xc0800000
+define i32 @inline_DA_constraint_F4() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(float -4.0)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'DA'
+; VI-LABEL: {{^}}inline_DA_constraint_F5:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x3e22f983
+define i32 @inline_DA_constraint_F5() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i32 1042479491)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_DA_constraint_F6:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0x3f000000
+define i32 @inline_DA_constraint_F6() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(float 0.5)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'DA'
+define i32 @inline_DA_constraint_F7() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i32 65)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'DA'
+define i32 @inline_DA_constraint_F8() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i32 -17)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'DA' constraint, 64-bit operand
+;==============================================================================
+
+; GCN-LABEL: {{^}}inline_DA_constraint_D0:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0x40fffffff0
+define i32 @inline_DA_constraint_D0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1 >> 32", "=v,^DA"(i64 bitcast (double 0x40fffffff0 to i64))
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_DA_constraint_D1:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0xfffffff000000040
+define i32 @inline_DA_constraint_D1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1 >> 32", "=v,^DA"(i64 bitcast (double 0xfffffff000000040 to i64))
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_DA_constraint_D2:
+; GCN: v_mov_b32 {{v[0-9]+}}, -1
+define i32 @inline_DA_constraint_D2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1 >> 32", "=v,^DA"(i64 -1)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_DA_constraint_D3:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0xbf000000c0800000
+define i32 @inline_DA_constraint_D3() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1 >> 32", "=v,^DA"(i64 bitcast (double 0xbf000000c0800000 to i64))
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'DA'
+; VI-LABEL: {{^}}inline_DA_constraint_D4:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x3e22f9833e22f983
+define i32 @inline_DA_constraint_D4() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1 >> 32", "=v,^DA"(i64 bitcast (double 0x3e22f9833e22f983 to i64))
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'DA'
+define i32 @inline_DA_constraint_D5() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1 >> 32", "=v,^DA"(i64 bitcast (double 0x0000004000000041 to i64))
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'DA'
+define i32 @inline_DA_constraint_D8() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1 >> 32", "=v,^DA"(i64 bitcast (double 0x0000004100000040 to i64))
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'DA'
+define i32 @inline_DA_constraint_D9() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1 >> 32", "=v,^DA"(double 100.0)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'DA' constraint, v2x16 operand
+;==============================================================================
+
+; NOSI: error: invalid operand for inline asm constraint 'DA'
+; VI-LABEL: {{^}}inline_DA_constraint_V0:
+; VI: v_mov_b32 {{v[0-9]+}}, -4
+define i32 @inline_DA_constraint_V0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(<2 x i16> <i16 -4, i16 -4>)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'DA'
+; VI-LABEL: {{^}}inline_DA_constraint_V1:
+; VI: v_mov_b32 {{v[0-9]+}}, 0xb800
+define i32 @inline_DA_constraint_V1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(<2 x half> <half -0.5, half -0.5>)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'DA'
+define i32 @inline_DA_constraint_V2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(<2 x i16> <i16 -4, i16 undef>)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'DA'
+define i32 @inline_DA_constraint_V6() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(<2 x i32> <i32 0, i32 0>)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'DA' constraint, type errors
+;==============================================================================
+
+; NOGCN: error: invalid operand for inline asm constraint 'DA'
+define i32 @inline_DA_constraint_E1(i32 %x) {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i32 %x)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'DA'
+define i32 @inline_DA_constraint_E2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i128 100000000000000000000)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'DB' constraint, 16-bit operand
+;==============================================================================
+
+; NOSI: error: invalid operand for inline asm constraint 'DB'
+; VI-LABEL: {{^}}inline_DB_constraint_H0:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x7fff
+define i32 @inline_DB_constraint_H0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DB"(i16 32767)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'DB'
+; VI-LABEL: {{^}}inline_DB_constraint_H1:
+; VI: v_mov_b32 {{v[0-9]+}}, -1
+define i32 @inline_DB_constraint_H1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DB"(i16 65535)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'DB'
+; VI-LABEL: {{^}}inline_DB_constraint_H2:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x4a80
+define i32 @inline_DB_constraint_H2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DB"(half 13.0)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'DB' constraint, 32-bit operand
+;==============================================================================
+
+; GCN-LABEL: {{^}}inline_DB_constraint_F0:
+; GCN: v_mov_b32 {{v[0-9]+}}, -1
+define i32 @inline_DB_constraint_F0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DB"(i32 4294967295)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_DB_constraint_F1:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0x80000000
+define i32 @inline_DB_constraint_F1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DB"(i32 2147483648)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_DB_constraint_F2:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0x42000000
+define i32 @inline_DB_constraint_F2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DB"(float 32.0)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'DB' constraint, 64-bit operand
+;==============================================================================
+
+; GCN-LABEL: {{^}}inline_DB_constraint_D0:
+; GCN: v_mov_b32 {{v[0-9]+}}, -1
+define i32 @inline_DB_constraint_D0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1 >> 32", "=v,^DB"(i64 -1)
+  ret i32 %v0
+}
+
+; GCN-LABEL: {{^}}inline_DB_constraint_D1:
+; GCN: v_mov_b32 {{v[0-9]+}}, 0x1234567890abcdef
+define i32 @inline_DB_constraint_D1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1 >> 32", "=v,^DB"(i64 bitcast (double 0x1234567890abcdef to i64))
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'DB' constraint, v2x16 operand
+;==============================================================================
+
+; NOSI: error: invalid operand for inline asm constraint 'DB'
+; VI-LABEL: {{^}}inline_DB_constraint_V0:
+; VI: v_mov_b32 {{v[0-9]+}}, 0x7fff
+define i32 @inline_DB_constraint_V0() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DB"(<2 x i16> <i16 32767, i16 32767>)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'DB'
+; VI-LABEL: {{^}}inline_DB_constraint_V1:
+; VI: v_mov_b32 {{v[0-9]+}}, -1
+define i32 @inline_DB_constraint_V1() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DB"(<2 x i16> <i16 65535, i16 65535>)
+  ret i32 %v0
+}
+
+; NOSI: error: invalid operand for inline asm constraint 'DB'
+; VI-LABEL: {{^}}inline_DB_constraint_V2:
+; VI: v_mov_b32 {{v[0-9]+}}, 0xd640
+define i32 @inline_DB_constraint_V2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DB"(<2 x half> <half -100.0, half -100.0>)
+  ret i32 %v0
+}
+
+;==============================================================================
+; 'DB' constraint, type errors
+;==============================================================================
+
+; NOGCN: error: invalid operand for inline asm constraint 'DB'
+define i32 @inline_DB_constraint_E1(i32 %x) {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DB"(i32 %x)
+  ret i32 %v0
+}
+
+; NOGCN: error: invalid operand for inline asm constraint 'DB'
+define i32 @inline_DB_constraint_E2() {
+  %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DB"(i128 100000000000000000000)
+  ret i32 %v0
+}
diff --git a/llvm/test/CodeGen/AMDGPU/insert-branch-w32.mir b/llvm/test/CodeGen/AMDGPU/insert-branch-w32.mir
new file mode 100644
index 0000000000000..5ccf84d3a08b5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/insert-branch-w32.mir
@@ -0,0 +1,47 @@
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -run-pass branch-folder -o - %s | FileCheck %s
+
+# Designed to provoke calling SIInstrInfo::insertBranch in wave32 mode
+# The implicit $vcc operand should be $vcc_lo in this case
+
+...
+# CHECK-LABEL: bb.1:
+# CHECK: S_CBRANCH_VCCNZ %bb.1, implicit $vcc_lo
+
+name:            _amdgpu_cs_main
+body:             |
+  bb.0:
+    $vgpr1 = V_MOV_B32_e32 1050, implicit $exec
+    $sgpr0 = S_MOV_B32 1123418112
+    $vcc_hi = IMPLICIT_DEF
+  bb.1:
+    $vgpr0 = COPY killed $vgpr1, implicit $exec
+    V_CMP_GT_U32_e32 5, $vgpr1, implicit-def $vcc_lo, implicit $exec, implicit-def $vcc
+    $vcc_lo = S_AND_B32 $exec_lo, $vcc_lo, implicit-def dead $scc
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc_lo, implicit $vcc
+    S_BRANCH %bb.2
+  
+  bb.2:
+    $sgpr1 = COPY $sgpr0
+    S_BRANCH %bb.1
+
+...
+
+# CHECK-LABEL: bb.1:
+# CHECK: S_CBRANCH_VCCNZ %bb.1, implicit undef $vcc_lo
+---
+name:            _amdgpu_cs_main_undef
+body:             |
+  bb.0:
+    $vgpr1 = V_MOV_B32_e32 1050, implicit $exec
+    $sgpr0 = S_MOV_B32 1123418112
+    $vcc_hi = IMPLICIT_DEF
+  bb.1:
+    $vgpr0 = COPY killed $vgpr1, implicit $exec
+    S_CBRANCH_VCCNZ %bb.1, implicit undef $vcc_lo, implicit undef $vcc
+    S_BRANCH %bb.2
+  
+  bb.2:
+    $sgpr1 = COPY $sgpr0
+    S_BRANCH %bb.1
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/insert-skips-kill-uncond.mir b/llvm/test/CodeGen/AMDGPU/insert-skips-kill-uncond.mir
index c8832caf61636..8887eb31989e5 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-skips-kill-uncond.mir
+++ b/llvm/test/CodeGen/AMDGPU/insert-skips-kill-uncond.mir
@@ -14,13 +14,13 @@
 
 # CHECK: bb.1:
 # CHECK: V_CMPX_LE_F32_e32
-# CHECK-NEXT: S_CBRANCH_EXECNZ %bb.2, implicit $exec
+# CHECK-NEXT: S_CBRANCH_EXECZ %bb.3, implicit $exec
 
-# CHECK: bb.3:
-# CHECK-NEXT: EXP_DONE
+# CHECK: bb.2:
 # CHECK: S_ENDPGM 0
 
-# CHECK: bb.2:
+# CHECK: bb.3:
+# CHECK-NEXT: EXP_DONE
 # CHECK: S_ENDPGM 0
 
 name: kill_uncond_branch
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
index 851d232e00a6b..1ae10d7fe48c2 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
@@ -38,15 +38,19 @@ entry:
 ; GCN-LABEL: {{^}}int4_inselt:
 ; GCN-NOT: v_movrel
 ; GCN-NOT: buffer_
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC1:[^,]+]], [[IDX:s[0-9]+]], 3
-; GCN-DAG: v_cndmask_b32_e32 v[[ELT_LAST:[0-9]+]], 1, v{{[0-9]+}}, [[CC1]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC2:[^,]+]], [[IDX]], 2
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1, v{{[0-9]+}}, [[CC2]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC3:[^,]+]], [[IDX]], 1
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1, v{{[0-9]+}}, [[CC3]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC4:[^,]+]], [[IDX]], 0
-; GCN-DAG: v_cndmask_b32_e32 v[[ELT_FIRST:[0-9]+]], 1, v{{[0-9]+}}, [[CC4]]
-; GCN:     flat_store_dwordx4 v[{{[0-9:]+}}], v{{\[}}[[ELT_FIRST]]:[[ELT_LAST]]]
+; GCN-DAG: s_cmp_lg_u32 [[IDX:s[0-9]+]], 3
+; GCN-DAG: s_cselect_b32 s[[ELT_3:[0-9]+]], s{{[0-9]+}}, 1
+; GCN-DAG: s_cmp_lg_u32 [[IDX]], 2
+; GCN-DAG: s_cselect_b32 s[[ELT_2:[0-9]+]], s{{[0-9]+}}, 1
+; GCN-DAG: s_cmp_lg_u32 [[IDX]], 1
+; GCN-DAG: s_cselect_b32 s[[ELT_1:[0-9]+]], s{{[0-9]+}}, 1
+; GCN-DAG: s_cmp_lg_u32 [[IDX]], 0
+; GCN-DAG: s_cselect_b32 s[[ELT_0:[0-9]+]], s{{[0-9]+}}, 1
+; GCN-DAG: v_mov_b32_e32 v[[VELT_0:[0-9]+]], s[[ELT_0]]
+; GCN-DAG: v_mov_b32_e32 v[[VELT_1:[0-9]+]], s[[ELT_1]]
+; GCN-DAG: v_mov_b32_e32 v[[VELT_2:[0-9]+]], s[[ELT_2]]
+; GCN-DAG: v_mov_b32_e32 v[[VELT_3:[0-9]+]], s[[ELT_3]]
+; GCN:     flat_store_dwordx4 v[{{[0-9:]+}}], v{{\[}}[[VELT_0]]:[[VELT_3]]]
 define amdgpu_kernel void @int4_inselt(<4 x i32> addrspace(1)* %out, <4 x i32> %vec, i32 %sel) {
 entry:
   %v = insertelement <4 x i32> %vec, i32 1, i32 %sel
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
index 60a2582741f17..c53f2b07aa7c9 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
@@ -579,12 +579,12 @@ define amdgpu_kernel void @dynamic_insertelement_v2i32(<2 x i32> addrspace(1)* %
 ; SI-NEXT:    s_mov_b32 s3, 0x100f000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; SI-NEXT:    s_cmp_lg_u32 s4, 1
+; SI-NEXT:    s_cselect_b32 s5, s7, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 0
+; SI-NEXT:    s_cselect_b32 s4, s6, 5
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
@@ -596,12 +596,12 @@ define amdgpu_kernel void @dynamic_insertelement_v2i32(<2 x i32> addrspace(1)* %
 ; VI-NEXT:    s_mov_b32 s3, 0x1100f000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s7
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
-; VI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
-; VI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; VI-NEXT:    s_cmp_lg_u32 s4, 1
+; VI-NEXT:    s_cselect_b32 s5, s7, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 0
+; VI-NEXT:    s_cselect_b32 s4, s6, 5
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
   %vecins = insertelement <2 x i32> %a, i32 5, i32 %b
@@ -618,15 +618,15 @@ define amdgpu_kernel void @dynamic_insertelement_v3i32(<3 x i32> addrspace(1)* %
 ; SI-NEXT:    s_mov_b32 s3, 0x100f000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, s10
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
-; SI-NEXT:    v_cndmask_b32_e32 v2, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s9
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s8
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; SI-NEXT:    s_cmp_lg_u32 s4, 2
+; SI-NEXT:    s_cselect_b32 s5, s10, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 1
+; SI-NEXT:    s_cselect_b32 s6, s9, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 0
+; SI-NEXT:    s_cselect_b32 s4, s8, 5
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s6
+; SI-NEXT:    v_mov_b32_e32 v2, s5
 ; SI-NEXT:    buffer_store_dwordx3 v[0:2], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
@@ -638,15 +638,15 @@ define amdgpu_kernel void @dynamic_insertelement_v3i32(<3 x i32> addrspace(1)* %
 ; VI-NEXT:    s_mov_b32 s3, 0x1100f000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s10
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
-; VI-NEXT:    v_cndmask_b32_e32 v2, 5, v0, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s9
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
-; VI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s8
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
-; VI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; VI-NEXT:    s_cmp_lg_u32 s4, 2
+; VI-NEXT:    s_cselect_b32 s5, s10, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 1
+; VI-NEXT:    s_cselect_b32 s6, s9, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 0
+; VI-NEXT:    s_cselect_b32 s4, s8, 5
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s6
+; VI-NEXT:    v_mov_b32_e32 v2, s5
 ; VI-NEXT:    buffer_store_dwordx3 v[0:2], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
   %vecins = insertelement <3 x i32> %a, i32 5, i32 %b
@@ -664,19 +664,18 @@ define amdgpu_kernel void @dynamic_insertelement_v4i32(<4 x i32> addrspace(1)* %
 ; SI-NEXT:    s_mov_b32 s3, 0x100f000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, s11
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 3
-; SI-NEXT:    v_mov_b32_e32 v4, s4
-; SI-NEXT:    v_cndmask_b32_e32 v3, v0, v4, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s10
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 2
-; SI-NEXT:    v_cndmask_b32_e32 v2, v0, v4, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s9
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 1
-; SI-NEXT:    v_cndmask_b32_e32 v1, v0, v4, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s8
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; SI-NEXT:    s_cmp_eq_u32 s6, 3
+; SI-NEXT:    s_cselect_b32 s5, s4, s11
+; SI-NEXT:    s_cmp_eq_u32 s6, 2
+; SI-NEXT:    s_cselect_b32 s7, s4, s10
+; SI-NEXT:    s_cmp_eq_u32 s6, 1
+; SI-NEXT:    s_cselect_b32 s9, s4, s9
+; SI-NEXT:    s_cmp_eq_u32 s6, 0
+; SI-NEXT:    s_cselect_b32 s4, s4, s8
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s9
+; SI-NEXT:    v_mov_b32_e32 v2, s7
+; SI-NEXT:    v_mov_b32_e32 v3, s5
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
@@ -689,19 +688,18 @@ define amdgpu_kernel void @dynamic_insertelement_v4i32(<4 x i32> addrspace(1)* %
 ; VI-NEXT:    s_mov_b32 s3, 0x1100f000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s11
-; VI-NEXT:    v_mov_b32_e32 v4, s4
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 3
-; VI-NEXT:    v_cndmask_b32_e32 v3, v0, v4, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s10
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 2
-; VI-NEXT:    v_cndmask_b32_e32 v2, v0, v4, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s9
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 1
-; VI-NEXT:    v_cndmask_b32_e32 v1, v0, v4, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s8
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 0
-; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; VI-NEXT:    s_cmp_eq_u32 s6, 3
+; VI-NEXT:    s_cselect_b32 s5, s4, s11
+; VI-NEXT:    s_cmp_eq_u32 s6, 2
+; VI-NEXT:    s_cselect_b32 s7, s4, s10
+; VI-NEXT:    s_cmp_eq_u32 s6, 1
+; VI-NEXT:    s_cselect_b32 s9, s4, s9
+; VI-NEXT:    s_cmp_eq_u32 s6, 0
+; VI-NEXT:    s_cselect_b32 s4, s4, s8
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s9
+; VI-NEXT:    v_mov_b32_e32 v2, s7
+; VI-NEXT:    v_mov_b32_e32 v3, s5
 ; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
   %vecins = insertelement <4 x i32> %a, i32 %val, i32 %b
@@ -718,32 +716,32 @@ define amdgpu_kernel void @dynamic_insertelement_v8i32(<8 x i32> addrspace(1)* %
 ; SI-NEXT:    s_mov_b32 s3, 0x100f000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, s11
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 3
-; SI-NEXT:    v_cndmask_b32_e32 v3, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s10
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
-; SI-NEXT:    v_cndmask_b32_e32 v2, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s9
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s8
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v4, s15
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 7
-; SI-NEXT:    v_cndmask_b32_e32 v7, 5, v4, vcc
-; SI-NEXT:    v_mov_b32_e32 v4, s14
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 6
-; SI-NEXT:    v_cndmask_b32_e32 v6, 5, v4, vcc
-; SI-NEXT:    v_mov_b32_e32 v4, s13
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 5
-; SI-NEXT:    v_cndmask_b32_e32 v5, 5, v4, vcc
-; SI-NEXT:    v_mov_b32_e32 v4, s12
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 4
-; SI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
-; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_cmp_lg_u32 s4, 3
+; SI-NEXT:    s_cselect_b32 s5, s11, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 2
+; SI-NEXT:    s_cselect_b32 s6, s10, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 1
+; SI-NEXT:    s_cselect_b32 s7, s9, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 0
+; SI-NEXT:    s_cselect_b32 s8, s8, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 7
+; SI-NEXT:    s_cselect_b32 s9, s15, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 6
+; SI-NEXT:    s_cselect_b32 s10, s14, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 5
+; SI-NEXT:    s_cselect_b32 s11, s13, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 4
+; SI-NEXT:    s_cselect_b32 s4, s12, 5
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s11
+; SI-NEXT:    v_mov_b32_e32 v2, s10
+; SI-NEXT:    v_mov_b32_e32 v3, s9
+; SI-NEXT:    v_mov_b32_e32 v4, s8
+; SI-NEXT:    v_mov_b32_e32 v5, s7
+; SI-NEXT:    v_mov_b32_e32 v6, s6
+; SI-NEXT:    v_mov_b32_e32 v7, s5
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: dynamic_insertelement_v8i32:
@@ -754,32 +752,32 @@ define amdgpu_kernel void @dynamic_insertelement_v8i32(<8 x i32> addrspace(1)* %
 ; VI-NEXT:    s_mov_b32 s3, 0x1100f000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s11
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 3
-; VI-NEXT:    v_cndmask_b32_e32 v3, 5, v0, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s10
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
-; VI-NEXT:    v_cndmask_b32_e32 v2, 5, v0, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s9
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
-; VI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s8
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
-; VI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
-; VI-NEXT:    v_mov_b32_e32 v4, s15
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 7
-; VI-NEXT:    v_cndmask_b32_e32 v7, 5, v4, vcc
-; VI-NEXT:    v_mov_b32_e32 v4, s14
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 6
-; VI-NEXT:    v_cndmask_b32_e32 v6, 5, v4, vcc
-; VI-NEXT:    v_mov_b32_e32 v4, s13
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 5
-; VI-NEXT:    v_cndmask_b32_e32 v5, 5, v4, vcc
-; VI-NEXT:    v_mov_b32_e32 v4, s12
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 4
-; VI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
-; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_cmp_lg_u32 s4, 3
+; VI-NEXT:    s_cselect_b32 s5, s11, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 2
+; VI-NEXT:    s_cselect_b32 s6, s10, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 1
+; VI-NEXT:    s_cselect_b32 s7, s9, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 0
+; VI-NEXT:    s_cselect_b32 s8, s8, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 7
+; VI-NEXT:    s_cselect_b32 s9, s15, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 6
+; VI-NEXT:    s_cselect_b32 s10, s14, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 5
+; VI-NEXT:    s_cselect_b32 s11, s13, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 4
+; VI-NEXT:    s_cselect_b32 s4, s12, 5
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s11
+; VI-NEXT:    v_mov_b32_e32 v2, s10
+; VI-NEXT:    v_mov_b32_e32 v3, s9
+; VI-NEXT:    v_mov_b32_e32 v4, s8
+; VI-NEXT:    v_mov_b32_e32 v5, s7
+; VI-NEXT:    v_mov_b32_e32 v6, s6
+; VI-NEXT:    v_mov_b32_e32 v7, s5
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
   %vecins = insertelement <8 x i32> %a, i32 5, i32 %b
   store <8 x i32> %vecins, <8 x i32> addrspace(1)* %out, align 32
@@ -1123,107 +1121,95 @@ define amdgpu_kernel void @dynamic_insertelement_v16i8(<16 x i8> addrspace(1)* %
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; SI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x4
 ; SI-NEXT:    s_load_dword s4, s[4:5], 0x8
+; SI-NEXT:    s_movk_i32 s7, 0xff
 ; SI-NEXT:    s_mov_b32 s3, 0x100f000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_lshr_b32 s5, s11, 24
-; SI-NEXT:    v_mov_b32_e32 v0, s5
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 15
-; SI-NEXT:    s_lshr_b32 s5, s11, 16
-; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v1, s5
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 14
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
-; SI-NEXT:    s_movk_i32 s5, 0xff
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; SI-NEXT:    v_and_b32_e32 v1, s5, v1
+; SI-NEXT:    s_cmp_lg_u32 s4, 15
+; SI-NEXT:    s_cselect_b32 s5, s5, 5
+; SI-NEXT:    s_lshl_b32 s5, s5, 8
+; SI-NEXT:    s_lshr_b32 s6, s11, 16
+; SI-NEXT:    s_cmp_lg_u32 s4, 14
+; SI-NEXT:    s_cselect_b32 s6, s6, 5
+; SI-NEXT:    s_and_b32 s6, s6, s7
+; SI-NEXT:    s_or_b32 s5, s6, s5
+; SI-NEXT:    s_lshl_b32 s5, s5, 16
 ; SI-NEXT:    s_lshr_b32 s6, s11, 8
-; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    v_mov_b32_e32 v1, s6
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 13
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
-; SI-NEXT:    v_mov_b32_e32 v2, s11
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 12
-; SI-NEXT:    v_cndmask_b32_e32 v2, 5, v2, vcc
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT:    v_and_b32_e32 v2, s5, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    s_mov_b32 s6, 0xffff
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_and_b32_e32 v1, s6, v1
-; SI-NEXT:    s_lshr_b32 s7, s10, 24
-; SI-NEXT:    v_or_b32_e32 v3, v1, v0
-; SI-NEXT:    v_mov_b32_e32 v0, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 11
-; SI-NEXT:    s_lshr_b32 s7, s10, 16
-; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v1, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 10
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; SI-NEXT:    v_and_b32_e32 v1, s5, v1
-; SI-NEXT:    s_lshr_b32 s7, s10, 8
-; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    v_mov_b32_e32 v1, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 9
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
-; SI-NEXT:    v_mov_b32_e32 v2, s10
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 8
-; SI-NEXT:    v_cndmask_b32_e32 v2, 5, v2, vcc
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT:    v_and_b32_e32 v2, s5, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_and_b32_e32 v1, s6, v1
-; SI-NEXT:    s_lshr_b32 s7, s9, 24
-; SI-NEXT:    v_or_b32_e32 v2, v1, v0
-; SI-NEXT:    v_mov_b32_e32 v0, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 7
-; SI-NEXT:    s_lshr_b32 s7, s9, 16
-; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v1, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 6
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; SI-NEXT:    v_and_b32_e32 v1, s5, v1
-; SI-NEXT:    s_lshr_b32 s7, s9, 8
-; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    v_mov_b32_e32 v1, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 5
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
-; SI-NEXT:    v_mov_b32_e32 v4, s9
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 4
-; SI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT:    v_and_b32_e32 v4, s5, v4
-; SI-NEXT:    v_or_b32_e32 v1, v4, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_and_b32_e32 v1, s6, v1
-; SI-NEXT:    s_lshr_b32 s7, s8, 24
-; SI-NEXT:    v_or_b32_e32 v1, v1, v0
-; SI-NEXT:    v_mov_b32_e32 v0, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 3
-; SI-NEXT:    s_lshr_b32 s7, s8, 16
-; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v4, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
-; SI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; SI-NEXT:    v_and_b32_e32 v4, s5, v4
-; SI-NEXT:    s_lshr_b32 s7, s8, 8
-; SI-NEXT:    v_or_b32_e32 v0, v4, v0
-; SI-NEXT:    v_mov_b32_e32 v4, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
-; SI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
-; SI-NEXT:    v_mov_b32_e32 v5, s8
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
-; SI-NEXT:    v_cndmask_b32_e32 v5, 5, v5, vcc
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
-; SI-NEXT:    v_and_b32_e32 v5, s5, v5
-; SI-NEXT:    v_or_b32_e32 v4, v5, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_and_b32_e32 v4, s6, v4
-; SI-NEXT:    v_or_b32_e32 v0, v4, v0
+; SI-NEXT:    s_cmp_lg_u32 s4, 13
+; SI-NEXT:    s_cselect_b32 s6, s6, 5
+; SI-NEXT:    s_lshl_b32 s6, s6, 8
+; SI-NEXT:    s_cmp_lg_u32 s4, 12
+; SI-NEXT:    s_cselect_b32 s11, s11, 5
+; SI-NEXT:    s_and_b32 s11, s11, s7
+; SI-NEXT:    s_or_b32 s6, s11, s6
+; SI-NEXT:    s_mov_b32 s11, 0xffff
+; SI-NEXT:    s_and_b32 s6, s6, s11
+; SI-NEXT:    s_or_b32 s5, s6, s5
+; SI-NEXT:    s_lshr_b32 s6, s10, 24
+; SI-NEXT:    s_cmp_lg_u32 s4, 11
+; SI-NEXT:    s_cselect_b32 s6, s6, 5
+; SI-NEXT:    s_lshl_b32 s6, s6, 8
+; SI-NEXT:    s_lshr_b32 s12, s10, 16
+; SI-NEXT:    s_cmp_lg_u32 s4, 10
+; SI-NEXT:    s_cselect_b32 s12, s12, 5
+; SI-NEXT:    s_and_b32 s12, s12, s7
+; SI-NEXT:    s_or_b32 s6, s12, s6
+; SI-NEXT:    s_lshl_b32 s6, s6, 16
+; SI-NEXT:    s_lshr_b32 s12, s10, 8
+; SI-NEXT:    s_cmp_lg_u32 s4, 9
+; SI-NEXT:    s_cselect_b32 s12, s12, 5
+; SI-NEXT:    s_lshl_b32 s12, s12, 8
+; SI-NEXT:    s_cmp_lg_u32 s4, 8
+; SI-NEXT:    s_cselect_b32 s10, s10, 5
+; SI-NEXT:    s_and_b32 s10, s10, s7
+; SI-NEXT:    s_or_b32 s10, s10, s12
+; SI-NEXT:    s_and_b32 s10, s10, s11
+; SI-NEXT:    s_or_b32 s6, s10, s6
+; SI-NEXT:    s_lshr_b32 s10, s9, 24
+; SI-NEXT:    s_cmp_lg_u32 s4, 7
+; SI-NEXT:    s_cselect_b32 s10, s10, 5
+; SI-NEXT:    s_lshl_b32 s10, s10, 8
+; SI-NEXT:    s_lshr_b32 s12, s9, 16
+; SI-NEXT:    s_cmp_lg_u32 s4, 6
+; SI-NEXT:    s_cselect_b32 s12, s12, 5
+; SI-NEXT:    s_and_b32 s12, s12, s7
+; SI-NEXT:    s_or_b32 s10, s12, s10
+; SI-NEXT:    s_lshl_b32 s10, s10, 16
+; SI-NEXT:    s_lshr_b32 s12, s9, 8
+; SI-NEXT:    s_cmp_lg_u32 s4, 5
+; SI-NEXT:    s_cselect_b32 s12, s12, 5
+; SI-NEXT:    s_lshl_b32 s12, s12, 8
+; SI-NEXT:    s_cmp_lg_u32 s4, 4
+; SI-NEXT:    s_cselect_b32 s9, s9, 5
+; SI-NEXT:    s_and_b32 s9, s9, s7
+; SI-NEXT:    s_or_b32 s9, s9, s12
+; SI-NEXT:    s_and_b32 s9, s9, s11
+; SI-NEXT:    s_or_b32 s9, s9, s10
+; SI-NEXT:    s_lshr_b32 s10, s8, 24
+; SI-NEXT:    s_cmp_lg_u32 s4, 3
+; SI-NEXT:    s_cselect_b32 s10, s10, 5
+; SI-NEXT:    s_lshl_b32 s10, s10, 8
+; SI-NEXT:    s_lshr_b32 s12, s8, 16
+; SI-NEXT:    s_cmp_lg_u32 s4, 2
+; SI-NEXT:    s_cselect_b32 s12, s12, 5
+; SI-NEXT:    s_and_b32 s12, s12, s7
+; SI-NEXT:    s_or_b32 s10, s12, s10
+; SI-NEXT:    s_lshl_b32 s10, s10, 16
+; SI-NEXT:    s_lshr_b32 s12, s8, 8
+; SI-NEXT:    s_cmp_lg_u32 s4, 1
+; SI-NEXT:    s_cselect_b32 s12, s12, 5
+; SI-NEXT:    s_lshl_b32 s12, s12, 8
+; SI-NEXT:    s_cmp_lg_u32 s4, 0
+; SI-NEXT:    s_cselect_b32 s4, s8, 5
+; SI-NEXT:    s_and_b32 s4, s4, s7
+; SI-NEXT:    s_or_b32 s4, s4, s12
+; SI-NEXT:    s_and_b32 s4, s4, s11
+; SI-NEXT:    s_or_b32 s4, s4, s10
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s9
+; SI-NEXT:    v_mov_b32_e32 v2, s6
+; SI-NEXT:    v_mov_b32_e32 v3, s5
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/kernel-args.ll b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
index 49c2bf08ba3e1..38fddcafaa290 100644
--- a/llvm/test/CodeGen/AMDGPU/kernel-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
@@ -855,10 +855,10 @@ define amdgpu_kernel void @struct_argument_alignment({i32, i64} %arg0, i8, {i32,
 ; multiple.
 ; FUNC-LABEL: {{^}}packed_struct_argument_alignment:
 ; HSA-GFX9: kernarg_segment_byte_size = 28
-; HSA-GFX9: global_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:17
-; HSA-GFX9: global_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:13
 ; HSA-GFX9: s_load_dword s{{[0-9]+}}, s[4:5], 0x0
 ; HSA-GFX9: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s[4:5], 0x4
+; HSA-GFX9: global_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:17
+; HSA-GFX9: global_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:13
 define amdgpu_kernel void @packed_struct_argument_alignment(<{i32, i64}> %arg0, i8, <{i32, i64}> %arg1) {
   %val0 = extractvalue <{i32, i64}> %arg0, 0
   %val1 = extractvalue <{i32, i64}> %arg0, 1
diff --git a/llvm/test/CodeGen/AMDGPU/kill-infinite-loop.ll b/llvm/test/CodeGen/AMDGPU/kill-infinite-loop.ll
index a2358f3a80f49..172e6bf327215 100644
--- a/llvm/test/CodeGen/AMDGPU/kill-infinite-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/kill-infinite-loop.ll
@@ -61,9 +61,11 @@ loop:
   br label %loop
 }
 
-; In case there's an epilog, we shouldn't have to do this.
+; Check that the epilog is the final block
 ; CHECK-LABEL: return_nonvoid
-; CHECK-NOT: exp null off, off, off, off done vm
+; CHECK: exp null off, off, off, off done vm
+; CHECK-NEXT: s_endpgm
+; CHECK-NEXT: BB{{[0-9]+}}_{{[0-9]+}}:
 define amdgpu_ps float @return_nonvoid(float %0) #0 {
 main_body:
   %cmp = fcmp olt float %0, 1.000000e+01
diff --git a/llvm/test/CodeGen/AMDGPU/lds-relocs.ll b/llvm/test/CodeGen/AMDGPU/lds-relocs.ll
index dd6bb1f0db2a3..cf88c235e0371 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-relocs.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-relocs.ll
@@ -35,7 +35,7 @@
 ; GCN: v_mov_b32_e32 v1, lds.external@abs32@lo ; encoding: [0xff,0x02,0x02,0x7e,A,A,A,A]
 ; GCN-NEXT:              ; fixup A - offset: 4, value: lds.external@abs32@lo, kind: FK_Data_4{{$}}
 ;
-; GCN: s_add_i32 s0, lds.defined@abs32@lo, s0 ; encoding: [0xff,0x00,0x00,0x81,A,A,A,A]
+; GCN: s_add_i32 s0, s0, lds.defined@abs32@lo ; encoding: [0x00,0xff,0x00,0x81,A,A,A,A]
 ; GCN-NEXT:          ; fixup A - offset: 4, value: lds.defined@abs32@lo, kind: FK_Data_4{{$}}
 ;
 ; GCN: .globl lds.external
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.f16.ll
new file mode 100644
index 0000000000000..bbfb88a4b22a3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.f16.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=fiji < %s | FileCheck -check-prefix=GCN %s
+
+define half @v_sqrt_f16(half %src)  {
+; GCN-LABEL: v_sqrt_f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sqrt_f16_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call half @llvm.amdgcn.sqrt.f16(half %src)
+  ret half %sqrt
+}
+
+define half @v_fabs_sqrt_f16(half %src)  {
+; GCN-LABEL: v_fabs_sqrt_f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sqrt_f16_e64 v0, |v0|
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fabs.src = call half @llvm.fabs.f16(half %src)
+  %sqrt = call half @llvm.amdgcn.sqrt.f16(half %fabs.src)
+  ret half %sqrt
+}
+
+define half @v_fneg_fabs_sqrt_f16(half %src)  {
+; GCN-LABEL: v_fneg_fabs_sqrt_f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sqrt_f16_e64 v0, -|v0|
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fabs.src = call half @llvm.fabs.f16(half %src)
+  %neg.fabs.src = fneg half %fabs.src
+  %sqrt = call half @llvm.amdgcn.sqrt.f16(half %neg.fabs.src)
+  ret half %sqrt
+}
+
+declare half @llvm.amdgcn.sqrt.f16(half) #0
+declare half @llvm.fabs.f16(half) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.ll
new file mode 100644
index 0000000000000..0257a3d111421
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=fiji < %s | FileCheck -check-prefix=GCN %s
+
+define float @v_sqrt_f32(float %src)  {
+; GCN-LABEL: v_sqrt_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sqrt_f32_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call float @llvm.amdgcn.sqrt.f32(float %src)
+  ret float %sqrt
+}
+
+define float @v_fabs_sqrt_f32(float %src)  {
+; GCN-LABEL: v_fabs_sqrt_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sqrt_f32_e64 v0, |v0|
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fabs.src = call float @llvm.fabs.f32(float %src)
+  %sqrt = call float @llvm.amdgcn.sqrt.f32(float %fabs.src)
+  ret float %sqrt
+}
+
+define float @v_fneg_fabs_sqrt_f32(float %src)  {
+; GCN-LABEL: v_fneg_fabs_sqrt_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sqrt_f32_e64 v0, -|v0|
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fabs.src = call float @llvm.fabs.f32(float %src)
+  %neg.fabs.src = fneg float %fabs.src
+  %sqrt = call float @llvm.amdgcn.sqrt.f32(float %neg.fabs.src)
+  ret float %sqrt
+}
+
+define double @v_sqrt_f64(double %src)  {
+; GCN-LABEL: v_sqrt_f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sqrt_f64_e32 v[0:1], v[0:1]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call double @llvm.amdgcn.sqrt.f64(double %src)
+  ret double %sqrt
+}
+
+define double @v_fabs_sqrt_f64(double %src)  {
+; GCN-LABEL: v_fabs_sqrt_f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sqrt_f64_e64 v[0:1], |v[0:1]|
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fabs.src = call double @llvm.fabs.f64(double %src)
+  %sqrt = call double @llvm.amdgcn.sqrt.f64(double %fabs.src)
+  ret double %sqrt
+}
+
+define double @v_fneg_fabs_sqrt_f64(double %src)  {
+; GCN-LABEL: v_fneg_fabs_sqrt_f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sqrt_f64_e64 v[0:1], -|v[0:1]|
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fabs.src = call double @llvm.fabs.f64(double %src)
+  %neg.fabs.src = fneg double %fabs.src
+  %sqrt = call double @llvm.amdgcn.sqrt.f64(double %neg.fabs.src)
+  ret double %sqrt
+}
+
+declare float @llvm.amdgcn.sqrt.f32(float) #0
+declare double @llvm.amdgcn.sqrt.f64(double) #0
+declare float @llvm.fabs.f32(float) #0
+declare double @llvm.fabs.f64(double) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/loop-idiom.ll b/llvm/test/CodeGen/AMDGPU/loop-idiom.ll
index d79a35ff6a6a9..68abd5c692f6a 100644
--- a/llvm/test/CodeGen/AMDGPU/loop-idiom.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop-idiom.ll
@@ -1,6 +1,6 @@
-; RUN: opt -basicaa -loop-idiom -S < %s -march=r600 -mcpu=redwood | FileCheck --check-prefix=R600 --check-prefix=FUNC %s
-; RUN: opt -basicaa -loop-idiom -S < %s -march=amdgcn -mcpu=tahiti -verify-machineinstrs| FileCheck --check-prefix=SI --check-prefix=FUNC %s
-; RUN: opt -basicaa -loop-idiom -S < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs| FileCheck --check-prefix=SI --check-prefix=FUNC %s
+; RUN: opt -basic-aa -loop-idiom -S < %s -march=r600 -mcpu=redwood | FileCheck --check-prefix=R600 --check-prefix=FUNC %s
+; RUN: opt -basic-aa -loop-idiom -S < %s -march=amdgcn -mcpu=tahiti -verify-machineinstrs| FileCheck --check-prefix=SI --check-prefix=FUNC %s
+; RUN: opt -basic-aa -loop-idiom -S < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs| FileCheck --check-prefix=SI --check-prefix=FUNC %s
 
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
diff --git a/llvm/test/CodeGen/AMDGPU/machine-cse-commute-target-flags.mir b/llvm/test/CodeGen/AMDGPU/machine-cse-commute-target-flags.mir
new file mode 100644
index 0000000000000..a6e6eeb3fc315
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/machine-cse-commute-target-flags.mir
@@ -0,0 +1,116 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -run-pass=machine-cse -o - %s | FileCheck %s
+
+--- |
+  define void @commute_instruction_subreg_target_flag() { ret void }
+  define void @commute_target_flag_frame_index() { ret void }
+  define void @commute_target_flag_global() { ret void }
+  define void @commute_target_flag_global_offset() { ret void }
+  define void @commute_target_flag_global_offset_mismatch() { ret void }
+
+  declare void @func()
+  @gv = external addrspace(1) global i32
+
+...
+
+# Make sure the subreg index is cleared when commuting a register and immediate.
+
+---
+name: commute_instruction_subreg_target_flag
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: commute_instruction_subreg_target_flag
+    ; CHECK: liveins: $vgpr0_vgpr1
+    ; CHECK: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; CHECK: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]].sub1, 64, 0, implicit $exec
+    ; CHECK: S_ENDPGM 0, implicit [[V_ADD_U32_e64_]], implicit [[V_ADD_U32_e64_]]
+    %0:vreg_64 = COPY $vgpr0_vgpr1
+    %1:vgpr_32 = V_ADD_U32_e64 %0.sub1, 64, 0, implicit $exec
+    %2:vgpr_32 = V_ADD_U32_e64 64, %0.sub1, 0, implicit $exec
+    S_ENDPGM 0, implicit %1, implicit %2
+
+...
+
+# FIXME: Why doesn't this CSE?
+---
+name: commute_target_flag_frame_index
+tracksRegLiveness: true
+stack:
+  - { id: 0, type: default, offset: 0, size: 4, alignment: 4 }
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: commute_target_flag_frame_index
+    ; CHECK: liveins: $vgpr0_vgpr1
+    ; CHECK: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; CHECK: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 %stack.0, [[COPY]].sub0, 0, implicit $exec
+    ; CHECK: S_ENDPGM 0, implicit [[V_ADD_U32_e64_]], implicit [[V_ADD_U32_e64_]]
+    %0:vreg_64 = COPY $vgpr0_vgpr1
+    %1:vgpr_32 = V_ADD_U32_e64 %0.sub0, %stack.0, 0, implicit $exec
+    %2:vgpr_32 = V_ADD_U32_e64 %stack.0, %0.sub0, 0, implicit $exec
+    S_ENDPGM 0, implicit %1, implicit %2
+
+...
+
+# FIXME: Handle commuting global variables
+---
+name: commute_target_flag_global
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+
+    ; CHECK-LABEL: name: commute_target_flag_global
+    ; CHECK: liveins: $sgpr0_sgpr1
+    ; CHECK: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; CHECK: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]].sub0, target-flags(amdgpu-rel32-lo) @func, implicit-def dead $scc
+    ; CHECK: S_ENDPGM 0, implicit [[S_ADD_U32_]], implicit [[S_ADD_U32_]]
+    %0:sreg_64 = COPY $sgpr0_sgpr1
+    %1:sreg_32 = S_ADD_U32 %0.sub0, target-flags(amdgpu-rel32-lo) @func, implicit-def dead $scc
+    %2:sreg_32 = S_ADD_U32 target-flags(amdgpu-rel32-lo) @func, %0.sub0, implicit-def dead $scc
+    S_ENDPGM 0, implicit %1, implicit %2
+
+...
+
+---
+name: commute_target_flag_global_offset
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+
+    ; CHECK-LABEL: name: commute_target_flag_global_offset
+    ; CHECK: liveins: $sgpr0_sgpr1
+    ; CHECK: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; CHECK: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]].sub0, target-flags(amdgpu-rel32-lo) @gv + 4, implicit-def dead $scc
+    ; CHECK: S_ENDPGM 0, implicit [[S_ADD_U32_]], implicit [[S_ADD_U32_]]
+    %0:sreg_64 = COPY $sgpr0_sgpr1
+    %1:sreg_32 = S_ADD_U32 %0.sub0, target-flags(amdgpu-rel32-lo) @gv + 4, implicit-def dead $scc
+    %2:sreg_32 = S_ADD_U32 target-flags(amdgpu-rel32-lo) @gv + 4, %0.sub0, implicit-def dead $scc
+    S_ENDPGM 0, implicit %1, implicit %2
+
+...
+
+---
+name: commute_target_flag_global_offset_mismatch
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+
+    ; CHECK-LABEL: name: commute_target_flag_global_offset_mismatch
+    ; CHECK: liveins: $sgpr0_sgpr1
+    ; CHECK: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; CHECK: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]].sub0, target-flags(amdgpu-rel32-lo) @gv + 4, implicit-def dead $scc
+    ; CHECK: [[S_ADD_U32_1:%[0-9]+]]:sreg_32 = S_ADD_U32 target-flags(amdgpu-rel32-lo) @gv + 8, [[COPY]].sub0, implicit-def dead $scc
+    ; CHECK: S_ENDPGM 0, implicit [[S_ADD_U32_]], implicit [[S_ADD_U32_1]]
+    %0:sreg_64 = COPY $sgpr0_sgpr1
+    %1:sreg_32 = S_ADD_U32 %0.sub0, target-flags(amdgpu-rel32-lo) @gv + 4, implicit-def dead $scc
+    %2:sreg_32 = S_ADD_U32 target-flags(amdgpu-rel32-lo) @gv + 8, %0.sub0, implicit-def dead $scc
+    S_ENDPGM 0, implicit %1, implicit %2
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/mad_uint24.ll b/llvm/test/CodeGen/AMDGPU/mad_uint24.ll
index 971f99dd77176..f60db1a4627ba 100644
--- a/llvm/test/CodeGen/AMDGPU/mad_uint24.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad_uint24.ll
@@ -75,7 +75,7 @@ entry:
 ; Check that the select instruction is not deleted.
 ; FUNC-LABEL: {{^}}i24_i32_i32_mad:
 ; EG: CNDE_INT
-; SI: v_cndmask
+; SI: s_cselect
 define amdgpu_kernel void @i24_i32_i32_mad(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c, i32 %d) {
 entry:
   %0 = ashr i32 %a, 8
diff --git a/llvm/test/CodeGen/AMDGPU/memory_clause.ll b/llvm/test/CodeGen/AMDGPU/memory_clause.ll
index 5a435f01925c6..bc4c0d03db932 100644
--- a/llvm/test/CodeGen/AMDGPU/memory_clause.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory_clause.ll
@@ -51,38 +51,38 @@ bb:
 define amdgpu_kernel void @scalar_clause(<4 x i32> addrspace(1)* noalias nocapture readonly %arg, <4 x i32> addrspace(1)* noalias nocapture %arg1) {
 ; GCN-LABEL: scalar_clause:
 ; GCN:       ; %bb.0: ; %bb
-; GCN-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x24
-; GCN-NEXT:    s_load_dwordx2 s[18:19], s[0:1], 0x2c
-; GCN-NEXT:    s_nop 0
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_load_dwordx4 s[0:3], s[16:17], 0x0
-; GCN-NEXT:    s_load_dwordx4 s[4:7], s[16:17], 0x10
-; GCN-NEXT:    s_load_dwordx4 s[8:11], s[16:17], 0x20
-; GCN-NEXT:    s_load_dwordx4 s[12:15], s[16:17], 0x30
-; GCN-NEXT:    v_mov_b32_e32 v12, s18
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_mov_b32_e32 v0, s0
-; GCN-NEXT:    v_mov_b32_e32 v4, s4
-; GCN-NEXT:    v_mov_b32_e32 v8, s8
-; GCN-NEXT:    v_mov_b32_e32 v13, s19
-; GCN-NEXT:    v_mov_b32_e32 v1, s1
-; GCN-NEXT:    v_mov_b32_e32 v2, s2
-; GCN-NEXT:    v_mov_b32_e32 v3, s3
-; GCN-NEXT:    v_mov_b32_e32 v5, s5
-; GCN-NEXT:    v_mov_b32_e32 v6, s6
-; GCN-NEXT:    v_mov_b32_e32 v7, s7
-; GCN-NEXT:    global_store_dwordx4 v[12:13], v[0:3], off
-; GCN-NEXT:    global_store_dwordx4 v[12:13], v[4:7], off offset:16
-; GCN-NEXT:    v_mov_b32_e32 v0, s12
-; GCN-NEXT:    v_mov_b32_e32 v9, s9
-; GCN-NEXT:    v_mov_b32_e32 v10, s10
-; GCN-NEXT:    v_mov_b32_e32 v11, s11
-; GCN-NEXT:    v_mov_b32_e32 v1, s13
-; GCN-NEXT:    v_mov_b32_e32 v2, s14
-; GCN-NEXT:    v_mov_b32_e32 v3, s15
-; GCN-NEXT:    global_store_dwordx4 v[12:13], v[8:11], off offset:32
-; GCN-NEXT:    global_store_dwordx4 v[12:13], v[0:3], off offset:48
-; GCN-NEXT:    s_endpgm
+; GCN-NEXT:	s_load_dwordx2 s[16:17], s[0:1], 0x24
+; GCN-NEXT:	s_load_dwordx2 s[18:19], s[0:1], 0x2c
+; GCN-NEXT:	s_nop 0
+; GCN-NEXT:	s_waitcnt lgkmcnt(0)
+; GCN-NEXT:	s_load_dwordx4 s[0:3], s[16:17], 0x0
+; GCN-NEXT:	s_load_dwordx4 s[4:7], s[16:17], 0x10
+; GCN-NEXT:	s_load_dwordx4 s[8:11], s[16:17], 0x20
+; GCN-NEXT:	s_load_dwordx4 s[12:15], s[16:17], 0x30
+; GCN-NEXT:	v_mov_b32_e32 v16, s18
+; GCN-NEXT:	s_waitcnt lgkmcnt(0)
+; GCN-NEXT:	v_mov_b32_e32 v0, s0
+; GCN-NEXT:	v_mov_b32_e32 v4, s4
+; GCN-NEXT:	v_mov_b32_e32 v8, s8
+; GCN-NEXT:	v_mov_b32_e32 v12, s12
+; GCN-NEXT:	v_mov_b32_e32 v17, s19
+; GCN-NEXT:	v_mov_b32_e32 v1, s1
+; GCN-NEXT:	v_mov_b32_e32 v2, s2
+; GCN-NEXT:	v_mov_b32_e32 v3, s3
+; GCN-NEXT:	v_mov_b32_e32 v5, s5
+; GCN-NEXT:	v_mov_b32_e32 v6, s6
+; GCN-NEXT:	v_mov_b32_e32 v7, s7
+; GCN-NEXT:	v_mov_b32_e32 v9, s9
+; GCN-NEXT:	v_mov_b32_e32 v10, s10
+; GCN-NEXT:	v_mov_b32_e32 v11, s11
+; GCN-NEXT:	v_mov_b32_e32 v13, s13
+; GCN-NEXT:	v_mov_b32_e32 v14, s14
+; GCN-NEXT:	v_mov_b32_e32 v15, s15
+; GCN-NEXT:	global_store_dwordx4 v[16:17], v[0:3], off
+; GCN-NEXT:	global_store_dwordx4 v[16:17], v[4:7], off offset:16
+; GCN-NEXT:	global_store_dwordx4 v[16:17], v[8:11], off offset:32
+; GCN-NEXT:	global_store_dwordx4 v[16:17], v[12:15], off offset:48
+; GCN-NEXT:	s_endpgm
 bb:
   %tmp = load <4 x i32>, <4 x i32> addrspace(1)* %arg, align 16
   %tmp2 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 1
diff --git a/llvm/test/CodeGen/AMDGPU/merge-image-load-gfx10.mir b/llvm/test/CodeGen/AMDGPU/merge-image-load-gfx10.mir
new file mode 100644
index 0000000000000..c7d297de04a21
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/merge-image-load-gfx10.mir
@@ -0,0 +1,490 @@
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -run-pass si-load-store-opt -o - %s | FileCheck -check-prefix=GFX10 %s
+
+# GFX10-LABEL: name: image_load_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_LOAD_V4_V2_gfx10 %5, %3, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_load_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5:vreg_64, %3:sgpr_256, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5:vreg_64, %3:sgpr_256, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+# GFX10-LABEL: name: image_load_merged_v1v3_reversed
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_LOAD_V4_V2_gfx10 %5, %3, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub3
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub0_sub1_sub2
+
+name: image_load_merged_v1v3_reversed
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5:vreg_64, %3:sgpr_256, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5:vreg_64, %3:sgpr_256, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_merged_v2v2
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_LOAD_V4_V2_gfx10 %5, %3, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_64 = COPY %8.sub0_sub1
+# GFX10: %{{[0-9]+}}:vreg_64 = COPY killed %8.sub2_sub3
+
+name: image_load_merged_v2v2
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vreg_64 = IMAGE_LOAD_V2_V2_gfx10 %5:vreg_64, %3:sgpr_256, 3, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 8, align 16, addrspace 4)
+    %7:vreg_64 = IMAGE_LOAD_V2_V2_gfx10 %5:vreg_64, %3:sgpr_256, 12, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 8, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_merged_v2v2_reversed
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_LOAD_V4_V2_gfx10 %5, %3, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_64 = COPY %8.sub2_sub3
+# GFX10: %{{[0-9]+}}:vreg_64 = COPY killed %8.sub0_sub1
+
+name: image_load_merged_v2v2_reversed
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vreg_64 = IMAGE_LOAD_V2_V2_gfx10 %5:vreg_64, %3:sgpr_256, 12, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 8, align 16, addrspace 4)
+    %7:vreg_64 = IMAGE_LOAD_V2_V2_gfx10 %5:vreg_64, %3:sgpr_256, 3, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 8, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_merged_v3v1
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_LOAD_V4_V2_gfx10 %5, %3, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY %8.sub0_sub1_sub2
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY killed %8.sub3
+
+name: image_load_merged_v3v1
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5:vreg_64, %3:sgpr_256, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+    %7:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5:vreg_64, %3:sgpr_256, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_merged_v3v1_reversed
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_LOAD_V4_V2_gfx10 %5, %3, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY %8.sub1_sub2_sub3
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY killed %8.sub0
+
+name: image_load_merged_v3v1_reversed
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5:vreg_64, %3:sgpr_256, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+    %7:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5:vreg_64, %3:sgpr_256, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_divided_merged
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_LOAD_V4_V2_gfx10 %5, %3, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+
+name: image_load_divided_merged
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5:vreg_64, %3:sgpr_256, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %8:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %9:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %7:vreg_64, %3:sgpr_256, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+    %10:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %11:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5:vreg_64, %3:sgpr_256, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_divided_not_merged
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_load_divided_not_merged
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vreg_128 = COPY %2
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5:vreg_64, %3:sgpr_256, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    IMAGE_STORE_V4_V2 %4:vreg_128, %5:vreg_64, %3:sgpr_256, 15, -1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 16)
+    %7:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5:vreg_64, %3:sgpr_256, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_dmask_overlapped_not_merged
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 4, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_load_dmask_overlapped_not_merged
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5:vreg_64, %3:sgpr_256, 4, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5:vreg_64, %3:sgpr_256, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_dmask_not_disjoint_not_merged
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 4, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 11, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_load_dmask_not_disjoint_not_merged
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5:vreg_64, %3:sgpr_256, 4, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5:vreg_64, %3:sgpr_256, 11, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_not_merged_0
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %6, %3, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_load_not_merged_0
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2, 1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %7:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %8:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %6, %3, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_not_merged_1
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %6, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %6, %4, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_load_not_merged_1
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %5:vgpr_32 = COPY %2.sub3
+    %6:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %7:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %6, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %8:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %6, %4, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_not_merged_3
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_load_not_merged_3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_not_merged_4
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_load_not_merged_4
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_not_merged_5
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 0, 1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_load_not_merged_5
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 0, 1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_not_merged_6
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 0, 0, 1, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_load_not_merged_6
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 0, 0, 1, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_not_merged_7
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 0, 0, 0, 1, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_load_not_merged_7
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 0, 0, 0, 1, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_not_merged_8
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_LOAD_V1_V1_gfx10 %6, %3, 8, 1, -1, 0, 0, 0, 0, 1, 0, 0, 0, implicit $exec :: (dereferenceable load 8, align 16, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_load_not_merged_8
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = COPY %5.sub0
+    %7:vgpr_32 = IMAGE_LOAD_V1_V1_gfx10 %6, %3, 8, 1, -1, 0, 0, 0, 0, 1, 0, 0, 0, implicit $exec :: (dereferenceable load 8, align 16, addrspace 4)
+    %8:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_not_merged_9
+# GFX10: %{{[0-9]+}}:vreg_64 = IMAGE_LOAD_V2_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 1, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_load_not_merged_9
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vreg_64 = IMAGE_LOAD_V2_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 1, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_not_merged_10
+# GFX10: %{{[0-9]+}}:vreg_64 = IMAGE_LOAD_V2_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 1, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_load_not_merged_10
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vreg_64 = IMAGE_LOAD_V2_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 1, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_not_merged_11
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 1, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_load_not_merged_11
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_V1_V2_gfx10 %5, %3, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 1, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_V3_V2_gfx10 %5, %3, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_mip_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_LOAD_MIP_V4_V3_gfx10 %5, %3, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_load_mip_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_96 = BUFFER_LOAD_DWORDX3_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_MIP_V1_V3_gfx10 %5:vreg_96, %3:sgpr_256, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_MIP_V3_V3_gfx10 %5:vreg_96, %3:sgpr_256, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+
+# GFX10-LABEL: name: image_load_mip_pck_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_LOAD_MIP_PCK_V4_V3_gfx10 %5, %3, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_load_mip_pck_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_96 = BUFFER_LOAD_DWORDX3_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_MIP_PCK_V1_V3_gfx10 %5:vreg_96, %3:sgpr_256, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_MIP_PCK_V3_V3_gfx10 %5:vreg_96, %3:sgpr_256, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+
+# GFX10-LABEL: name: image_load_mip_pck_sgn_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_LOAD_MIP_PCK_SGN_V4_V3_gfx10 %5, %3, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_load_mip_pck_sgn_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_96 = BUFFER_LOAD_DWORDX3_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_MIP_PCK_SGN_V1_V3_gfx10 %5:vreg_96, %3:sgpr_256, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_MIP_PCK_SGN_V3_V3_gfx10 %5:vreg_96, %3:sgpr_256, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_pck_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_LOAD_PCK_V4_V2_gfx10 %5, %3, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_load_pck_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_PCK_V1_V2_gfx10 %5:vreg_64, %3:sgpr_256, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_PCK_V3_V2_gfx10 %5:vreg_64, %3:sgpr_256, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_load_pck_sgn_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_LOAD_PCK_SGN_V4_V2_gfx10 %5, %3, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_load_pck_sgn_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_LOAD_PCK_SGN_V1_V2_gfx10 %5:vreg_64, %3:sgpr_256, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_LOAD_PCK_SGN_V3_V2_gfx10 %5:vreg_64, %3:sgpr_256, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
diff --git a/llvm/test/CodeGen/AMDGPU/merge-image-load.mir b/llvm/test/CodeGen/AMDGPU/merge-image-load.mir
index fc91d650f972b..020ccd497a115 100644
--- a/llvm/test/CodeGen/AMDGPU/merge-image-load.mir
+++ b/llvm/test/CodeGen/AMDGPU/merge-image-load.mir
@@ -10,7 +10,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -28,7 +28,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -47,7 +47,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -66,7 +66,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -85,7 +85,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -104,7 +104,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -121,7 +121,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -143,7 +143,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vreg_128 = COPY %2
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -162,7 +162,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -180,7 +180,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -198,7 +198,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -217,7 +217,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %5:vgpr_32 = COPY %2.sub3
@@ -236,7 +236,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -254,7 +254,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -272,7 +272,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -290,7 +290,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -308,7 +308,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -326,7 +326,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -344,7 +344,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -362,7 +362,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -381,7 +381,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -402,7 +402,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -423,7 +423,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -442,7 +442,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
@@ -461,7 +461,7 @@ body:             |
   bb.0.entry:
     %0:sgpr_64 = COPY $sgpr0_sgpr1
     %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
-    %2:sgpr_128 =COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
     %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
     %4:vgpr_32 = COPY %2.sub3
     %5:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
diff --git a/llvm/test/CodeGen/AMDGPU/merge-image-sample-gfx10.mir b/llvm/test/CodeGen/AMDGPU/merge-image-sample-gfx10.mir
new file mode 100644
index 0000000000000..70923145dd2d3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/merge-image-sample-gfx10.mir
@@ -0,0 +1,1173 @@
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -run-pass si-load-store-opt -o - %s | FileCheck -check-prefix=GFX10 %s
+
+# GFX10-LABEL: name: image_sample_l_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_L_V4_V3_nsa_gfx10 %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_l_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+# GFX10-LABEL: name: image_sample_l_merged_v1v3_reversed
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_L_V4_V3_nsa_gfx10 %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub3
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub0_sub1_sub2
+
+name: image_sample_l_merged_v1v3_reversed
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_merged_v2v2
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_L_V4_V3_nsa_gfx10 %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_64 = COPY %8.sub0_sub1
+# GFX10: %{{[0-9]+}}:vreg_64 = COPY killed %8.sub2_sub3
+
+name: image_sample_l_merged_v2v2
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vreg_64 = IMAGE_SAMPLE_L_V2_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 3, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 8, align 16, addrspace 4)
+    %7:vreg_64 = IMAGE_SAMPLE_L_V2_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 12, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 8, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_merged_v2v2_reversed
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_L_V4_V3_nsa_gfx10 %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_64 = COPY %8.sub2_sub3
+# GFX10: %{{[0-9]+}}:vreg_64 = COPY killed %8.sub0_sub1
+
+name: image_sample_l_merged_v2v2_reversed
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vreg_64 = IMAGE_SAMPLE_L_V2_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 12, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 8, align 16, addrspace 4)
+    %7:vreg_64 = IMAGE_SAMPLE_L_V2_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 3, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 8, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_merged_v3v1
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_L_V4_V3_nsa_gfx10 %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY %8.sub0_sub1_sub2
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY killed %8.sub3
+
+name: image_sample_l_merged_v3v1
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+    %7:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_merged_v3v1_reversed
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_L_V4_V3_nsa_gfx10 %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY %8.sub1_sub2_sub3
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY killed %8.sub0
+
+name: image_sample_l_merged_v3v1_reversed
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+    %7:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_divided_merged
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_L_V4_V3_nsa_gfx10 %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+
+name: image_sample_l_divided_merged
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %9:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %7:vgpr_32, %7:vgpr_32, %7:vgpr_32, %3:sgpr_256, %2:sgpr_128, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+    %10:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %11:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_divided_not_merged
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_sample_l_divided_not_merged
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vreg_128 = COPY %2
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    IMAGE_STORE_V4_V2_nsa_gfx10 %4:vreg_128, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 16)
+    %7:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_dmask_overlapped_not_merged
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5, %5, %5, %3, %2, 4, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_sample_l_dmask_overlapped_not_merged
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 4, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_dmask_not_disjoint_not_merged
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5, %5, %5, %3, %2, 4, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 11, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_sample_l_dmask_not_disjoint_not_merged
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 4, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 11, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_not_merged_0
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %6, %6, %6, %3, %2, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_sample_l_not_merged_0
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2, 1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %7:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %8:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %6, %6, %6, %3, %2, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_not_merged_1
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %6, %6, %6, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %6, %6, %6, %4, %2, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_sample_l_not_merged_1
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %5:vgpr_32 = COPY %2.sub3
+    %6:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %7:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %6, %6, %6, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %8:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %6, %6, %6, %4, %2, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_not_merged_2
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %6, %6, %6, %4, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %6, %6, %6, %4, %3, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_sample_l_not_merged_2
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_128 = COPY $sgpr92_sgpr93_sgpr94_sgpr95
+    %4:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %5:vgpr_32 = COPY %2.sub3
+    %6:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %7:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %6, %6, %6, %4, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %8:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %6, %6, %6, %4, %3, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_not_merged_3
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_sample_l_not_merged_3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_not_merged_4
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 0, 1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_sample_l_not_merged_4
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 0, 1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_not_merged_5
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 0, 0, 1, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_sample_l_not_merged_5
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 0, 0, 1, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_not_merged_6
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 0, 0, 0, 1, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_sample_l_not_merged_6
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 0, 0, 0, 1, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_not_merged_7
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_SAMPLE_L_V1_V2_nsa_gfx10 %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 1, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_sample_l_not_merged_7
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_L_V1_V2_nsa_gfx10 %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 1, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_not_merged_8
+# GFX10: %{{[0-9]+}}:vreg_64 = IMAGE_SAMPLE_L_V2_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 1, 0, 0, implicit $exec :: (dereferenceable load 8, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_sample_l_not_merged_8
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vreg_64 = IMAGE_SAMPLE_L_V2_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 1, 0, 0, implicit $exec :: (dereferenceable load 8, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_not_merged_9
+# GFX10: %{{[0-9]+}}:vreg_64 = IMAGE_SAMPLE_L_V2_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 1, 0, implicit $exec :: (dereferenceable load 8, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_sample_l_not_merged_9
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vreg_64 = IMAGE_SAMPLE_L_V2_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 1, 0, implicit $exec :: (dereferenceable load 8, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+# GFX10-LABEL: name: image_sample_l_not_merged_10
+# GFX10: %{{[0-9]+}}:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 1, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+
+name: image_sample_l_not_merged_10
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_L_V1_V3_nsa_gfx10 %5, %5, %5, %3, %2, 8, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_L_V3_V3_nsa_gfx10 %5, %5, %5, %3, %2, 7, 1, -1, 0, 0, 0, 0, 0, 0, 0, 1, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+
+
+# GFX10-LABEL: name: image_sample_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_V4_V2_nsa_gfx10 %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_V1_V2_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_V3_V2_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_b_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_B_V4_V3_nsa_gfx10 %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_b_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_B_V1_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_B_V3_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_b_cl_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_B_CL_V4_V4_nsa_gfx10 %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_b_cl_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_B_CL_V1_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_B_CL_V3_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_b_cl_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_B_CL_O_V4_V5_nsa_gfx10 %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_b_cl_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_B_CL_O_V1_V5_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_B_CL_O_V3_V5_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_b_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_B_O_V4_V4_nsa_gfx10 %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_b_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_B_O_V1_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_B_O_V3_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_V4_V3_nsa_gfx10 %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_V1_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_V3_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_cd_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_CD_V4_V6_nsa_gfx10 %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_cd_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_CD_V1_V6_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_CD_V3_V6_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_cd_cl_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_CD_CL_V4_V7_nsa_gfx10 %5, %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_cd_cl_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_CD_CL_V1_V7_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_CD_CL_V3_V7_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_cd_cl_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_CD_CL_O_V4_V8_nsa_gfx10 %5, %5, %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_cd_cl_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_CD_CL_O_V1_V8_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_CD_CL_O_V3_V8_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_cd_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_CD_O_V4_V7_nsa_gfx10 %5, %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_cd_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_CD_O_V1_V7_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_CD_O_V3_V7_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_cl_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_CL_V4_V3_nsa_gfx10 %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_cl_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_CL_V1_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_CL_V3_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_cl_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_CL_O_V4_V4_nsa_gfx10 %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_cl_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_CL_O_V1_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_CL_O_V3_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_b_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_B_V4_V4_nsa_gfx10 %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_b_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_B_V1_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_B_V3_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_b_cl_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_B_CL_V4_V5_nsa_gfx10 %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_b_cl_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_B_CL_V1_V5_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_B_CL_V3_V5_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_b_cl_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_B_CL_O_V4_V6_nsa_gfx10 %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_b_cl_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_B_CL_O_V1_V6_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_B_CL_O_V3_V6_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_b_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_B_O_V4_V5_nsa_gfx10 %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_b_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_B_O_V1_V5_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_B_O_V3_V5_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_cd_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_CD_V4_V7_nsa_gfx10 %5, %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_cd_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_CD_V1_V7_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_CD_V3_V7_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_cd_cl_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_CD_CL_V4_V8_nsa_gfx10 %5, %5, %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_cd_cl_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_CD_CL_V1_V8_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_CD_CL_V3_V8_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_cd_cl_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_CD_CL_O_V4_V9_nsa_gfx10 %5, %5, %5, %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_cd_cl_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_CD_CL_O_V1_V9_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_CD_CL_O_V3_V9_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_cd_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_CD_O_V4_V8_nsa_gfx10 %5, %5, %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_cd_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_CD_O_V1_V8_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_CD_O_V3_V8_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_cl_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_CL_V4_V4_nsa_gfx10 %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_cl_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_CL_V1_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_CL_V3_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_cl_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_CL_O_V4_V5_nsa_gfx10 %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_cl_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_CL_O_V1_V5_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_CL_O_V3_V5_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_d_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_D_V4_V7_nsa_gfx10 %5, %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_d_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_D_V1_V7_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_D_V3_V7_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_d_cl_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_D_CL_V4_V8_nsa_gfx10 %5, %5, %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_d_cl_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_D_CL_V1_V8_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_D_CL_V3_V8_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_d_cl_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_D_CL_O_V4_V9_nsa_gfx10 %5, %5, %5, %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_d_cl_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_D_CL_O_V1_V9_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_D_CL_O_V3_V9_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_d_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_D_O_V4_V8_nsa_gfx10 %5, %5, %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_d_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_D_O_V1_V8_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_D_O_V3_V8_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_l_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_L_V4_V4_nsa_gfx10 %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_l_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_L_V1_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_L_V3_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_lz_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_LZ_V4_V3_nsa_gfx10 %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_lz_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_LZ_V1_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_LZ_V3_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_lz_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_LZ_O_V4_V4_nsa_gfx10 %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_lz_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_LZ_O_V1_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_LZ_O_V3_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_l_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_L_O_V4_V5_nsa_gfx10 %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_l_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_L_O_V1_V5_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_L_O_V3_V5_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_c_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_C_O_V4_V4_nsa_gfx10 %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_c_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_C_O_V1_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_C_O_V3_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_d_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_D_V4_V6_nsa_gfx10 %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_d_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_D_V1_V6_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_D_V3_V6_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_d_cl_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_D_CL_V4_V7_nsa_gfx10 %5, %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_d_cl_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_D_CL_V1_V7_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_D_CL_V3_V7_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_d_cl_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_D_CL_O_V4_V8_nsa_gfx10 %5, %5, %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_d_cl_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_D_CL_O_V1_V8_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_D_CL_O_V3_V8_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_d_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_D_O_V4_V7_nsa_gfx10 %5, %5, %5, %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_d_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_D_O_V1_V7_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_D_O_V3_V7_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_lz_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_LZ_V4_V2_nsa_gfx10 %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_lz_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_LZ_V3_V2_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_lz_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_LZ_O_V4_V3_nsa_gfx10 %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_lz_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_LZ_O_V1_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_LZ_O_V3_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_l_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_L_O_V4_V4_nsa_gfx10 %5, %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_l_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_L_O_V1_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_L_O_V3_V4_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
+
+
+# GFX10-LABEL: name: image_sample_o_merged_v1v3
+# GFX10: %{{[0-9]+}}:vreg_128 = IMAGE_SAMPLE_O_V4_V3_nsa_gfx10 %5, %5, %5, %3, %2, 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $exec :: (dereferenceable load 16, align 4, addrspace 4)
+# GFX10: %{{[0-9]+}}:vgpr_32 = COPY %8.sub0
+# GFX10: %{{[0-9]+}}:vreg_96 = COPY killed %8.sub1_sub2_sub3
+
+name: image_sample_o_merged_v1v3
+body:             |
+  bb.0.entry:
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0, 36, 0, 0
+    %2:sgpr_128 = COPY $sgpr96_sgpr97_sgpr98_sgpr99
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %1, 208, 0, 0
+    %4:vgpr_32 = COPY %2.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %2:sgpr_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16)
+    %6:vgpr_32 = IMAGE_SAMPLE_O_V1_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4, addrspace 4)
+    %7:vreg_96 = IMAGE_SAMPLE_O_V3_V3_nsa_gfx10 %5:vgpr_32, %5:vgpr_32, %5:vgpr_32, %3:sgpr_256, %2:sgpr_128, 14, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 12, align 16, addrspace 4)
+...
+---
diff --git a/llvm/test/CodeGen/AMDGPU/merge-load-store-vreg.mir b/llvm/test/CodeGen/AMDGPU/merge-load-store-vreg.mir
index 0f3913e9e45fd..5e13ed178aaa0 100644
--- a/llvm/test/CodeGen/AMDGPU/merge-load-store-vreg.mir
+++ b/llvm/test/CodeGen/AMDGPU/merge-load-store-vreg.mir
@@ -115,7 +115,7 @@ body:             |
     %2:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %1, implicit $exec
     V_CMP_NE_U32_e32 1, %2, implicit-def $vcc, implicit $exec
     DS_WRITE_B32 %0.sub0, %0.sub0, 1024, 0, implicit $m0, implicit $exec :: (store 4 into %ir.tmp)
-    undef %3.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    %3:vreg_64 = V_LSHLREV_B64 0, 0, implicit $exec
     DS_WRITE_B32 %0.sub0, %3.sub0, 1056, 0, implicit $m0, implicit $exec :: (store 4 into %ir.tmp1)
     %4:vgpr_32 = DS_READ_B32 %3.sub0, 1088, 0, implicit $m0, implicit $exec :: (load 4 from %ir.tmp2)
     %5:vgpr_32 = DS_READ_B32 %3.sub0, 1120, 0, implicit $m0, implicit $exec :: (load 4 from %ir.tmp3)
@@ -146,7 +146,7 @@ body:             |
     %2:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %1, implicit $exec
     V_CMP_NE_U32_e32 1, %2, implicit-def $vcc, implicit $exec
     DS_WRITE_B32 %0.sub0, %0.sub0, 0, 0, implicit $m0, implicit $exec :: (store 4 into %ir.tmp)
-    undef %3.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    %3:vreg_64 = V_LSHLREV_B64 0, 0, implicit $exec
     DS_WRITE_B32 %0.sub0, %3.sub0, 32, 0, implicit $m0, implicit $exec :: (store 4 into %ir.tmp1)
     %4:vgpr_32 = DS_READ_B32 %3.sub0, 0, 0, implicit $m0, implicit $exec :: (load 4 from %ir.tmp2)
     %5:vgpr_32 = DS_READ_B32 %3.sub0, 32, 0, implicit $m0, implicit $exec :: (load 4 from %ir.tmp3)
diff --git a/llvm/test/CodeGen/AMDGPU/mode-register.mir b/llvm/test/CodeGen/AMDGPU/mode-register.mir
index 753e6a3ce0a75..382050bc3888f 100644
--- a/llvm/test/CodeGen/AMDGPU/mode-register.mir
+++ b/llvm/test/CodeGen/AMDGPU/mode-register.mir
@@ -457,3 +457,55 @@ body: |
   bb.4:
     S_ENDPGM 0
 ...
+---
+# checks for bug where if a block is its own predecessor it could cause mode tracking
+# to produce the wrong mode, resulting in an unnecessary setreg
+# CHECK-LABEL: name: single_block_loop
+# CHECK-LABEL: bb.0:
+# CHECK-NOT: S_SETREG
+
+name: single_block_loop
+
+body: |
+  bb.0:
+    successors: %bb.1
+    S_BRANCH %bb.1
+
+  bb.1:
+    successors: %bb.1, %bb.2
+    S_CBRANCH_VCCZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    successors: %bb.3
+    liveins: $vgpr1_vgpr2
+    $vgpr1_vgpr2 = V_FRACT_F64_e32 killed $vgpr1_vgpr2, implicit $mode, implicit $exec
+    S_BRANCH %bb.3
+
+  bb.3:
+    S_ENDPGM 0
+...
+---
+# checks for a bug where if the first block is its own predecessor the initial mode was
+# not correctly propagated, resulting in an unnecessary setreg
+# CHECK-LABEL: name: first_block_loop
+# CHECK-LABEL: bb.0:
+# CHECK-NOT: S_SETREG
+
+name: first_block_loop
+
+body: |
+  bb.0:
+    successors: %bb.0, %bb.1
+    S_CBRANCH_VCCZ %bb.0, implicit $vcc
+    S_BRANCH %bb.1
+
+  bb.1:
+    successors: %bb.2
+    liveins: $vgpr1_vgpr2
+    $vgpr1_vgpr2 = V_FRACT_F64_e32 killed $vgpr1_vgpr2, implicit $mode, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.2:
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/opt-pipeline.ll b/llvm/test/CodeGen/AMDGPU/opt-pipeline.ll
index 32d36f4e7280e..85f9d8c867bf5 100644
--- a/llvm/test/CodeGen/AMDGPU/opt-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/opt-pipeline.ll
@@ -276,6 +276,12 @@
 ; GCN-O1-NEXT:       Warn about non-applied transformations
 ; GCN-O1-NEXT:       Alignment from assumptions
 ; GCN-O1-NEXT:     Strip Unused Function Prototypes
+; GCN-O1-NEXT:     Call Graph Profile
+; GCN-O1-NEXT:       FunctionPass Manager
+; GCN-O1-NEXT:         Dominator Tree Construction
+; GCN-O1-NEXT:         Natural Loop Information
+; GCN-O1-NEXT:         Lazy Branch Probability Analysis
+; GCN-O1-NEXT:         Lazy Block Frequency Analysis
 ; GCN-O1-NEXT:     FunctionPass Manager
 ; GCN-O1-NEXT:       Dominator Tree Construction
 ; GCN-O1-NEXT:       Natural Loop Information
@@ -623,6 +629,12 @@
 ; GCN-O2-NEXT:     Strip Unused Function Prototypes
 ; GCN-O2-NEXT:     Dead Global Elimination
 ; GCN-O2-NEXT:     Merge Duplicate Global Constants
+; GCN-O2-NEXT:     Call Graph Profile
+; GCN-O2-NEXT:       FunctionPass Manager
+; GCN-O2-NEXT:         Dominator Tree Construction
+; GCN-O2-NEXT:         Natural Loop Information
+; GCN-O2-NEXT:         Lazy Branch Probability Analysis
+; GCN-O2-NEXT:         Lazy Block Frequency Analysis
 ; GCN-O2-NEXT:     FunctionPass Manager
 ; GCN-O2-NEXT:       Dominator Tree Construction
 ; GCN-O2-NEXT:       Natural Loop Information
@@ -975,6 +987,12 @@
 ; GCN-O3-NEXT:     Strip Unused Function Prototypes
 ; GCN-O3-NEXT:     Dead Global Elimination
 ; GCN-O3-NEXT:     Merge Duplicate Global Constants
+; GCN-O3-NEXT:     Call Graph Profile
+; GCN-O3-NEXT:       FunctionPass Manager
+; GCN-O3-NEXT:         Dominator Tree Construction
+; GCN-O3-NEXT:         Natural Loop Information
+; GCN-O3-NEXT:         Lazy Branch Probability Analysis
+; GCN-O3-NEXT:         Lazy Block Frequency Analysis
 ; GCN-O3-NEXT:     FunctionPass Manager
 ; GCN-O3-NEXT:       Dominator Tree Construction
 ; GCN-O3-NEXT:       Natural Loop Information
diff --git a/llvm/test/CodeGen/AMDGPU/phi-elimination-end-cf.mir b/llvm/test/CodeGen/AMDGPU/phi-elimination-end-cf.mir
index 8944ef86f6209..9059c8edf3e48 100644
--- a/llvm/test/CodeGen/AMDGPU/phi-elimination-end-cf.mir
+++ b/llvm/test/CodeGen/AMDGPU/phi-elimination-end-cf.mir
@@ -4,7 +4,7 @@
 # CHECK: bb.0:
 # CHECK:     [[COND:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64
 # CHECK:     [[IF_SOURCE0:%[0-9]+]]:sreg_64 = SI_IF [[COND]], %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
-# CHECK:     [[IF_INPUT_REG:%[0-9]+]]:sreg_64 =  S_MOV_B64_term killed [[IF_SOURCE0]], implicit $exec
+# CHECK:     [[IF_INPUT_REG:%[0-9]+]]:sreg_64 = S_MOV_B64_term killed [[IF_SOURCE0]], implicit $exec
 
 # CHECK: bb.1:
 # CHECK:     [[END_CF_ARG:%[0-9]+]]:sreg_64 = COPY killed [[IF_INPUT_REG]]
@@ -12,7 +12,7 @@
 
 # CHECK: bb.2:
 # CHECK:     [[IF_SOURCE1:%[0-9]+]]:sreg_64 = SI_IF [[COND]], %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
-# CHECK:     [[IF_INPUT_REG]]:sreg_64 =  S_MOV_B64_term killed [[IF_SOURCE1]], implicit $exec
+# CHECK:     [[IF_INPUT_REG]]:sreg_64 = S_MOV_B64_term killed [[IF_SOURCE1]], implicit $exec
 
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/phi-vgpr-input-moveimm.mir b/llvm/test/CodeGen/AMDGPU/phi-vgpr-input-moveimm.mir
index ff061dec039a8..7d26589ed4e0c 100644
--- a/llvm/test/CodeGen/AMDGPU/phi-vgpr-input-moveimm.mir
+++ b/llvm/test/CodeGen/AMDGPU/phi-vgpr-input-moveimm.mir
@@ -18,7 +18,7 @@ body:             |
 
   bb.1:
     successors: %bb.2
-    %2:sreg_32 =  S_ADD_U32 %4, %5, implicit-def $scc
+    %2:sreg_32 = S_ADD_U32 %4, %5, implicit-def $scc
     S_BRANCH %bb.2
 
   bb.2:
@@ -50,7 +50,7 @@ body:             |
 
   bb.1:
     successors: %bb.2
-    undef %2.sub0:sreg_64 =  S_ADD_U32 %4, %5, implicit-def $scc
+    undef %2.sub0:sreg_64 = S_ADD_U32 %4, %5, implicit-def $scc
     S_BRANCH %bb.2
 
   bb.2:
@@ -84,7 +84,7 @@ body:             |
   bb.1:
 
     successors: %bb.2
-    %2:sreg_32 =  S_ADD_U32 %4, %5, implicit-def $scc
+    %2:sreg_32 = S_ADD_U32 %4, %5, implicit-def $scc
     S_BRANCH %bb.2
   bb.2:
     successors: %bb.3
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
index c7ae08c839ee4..4c6fa2f0f4c8c 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
@@ -17,9 +17,9 @@ define amdgpu_kernel void @clmem_read_simplified(i8 addrspace(1)*  %buffer) {
 ;
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
@@ -86,6 +86,7 @@ define hidden amdgpu_kernel void @clmem_read(i8 addrspace(1)*  %buffer) {
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ;
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
@@ -95,10 +96,8 @@ define hidden amdgpu_kernel void @clmem_read(i8 addrspace(1)*  %buffer) {
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off
 ;
-; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
 ; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
@@ -300,9 +299,9 @@ define amdgpu_kernel void @Offset64(i8 addrspace(1)*  %buffer) {
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ;
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ;
 ; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
@@ -456,10 +455,10 @@ define amdgpu_kernel void @ReverseOrder(i8 addrspace(1)* %buffer) {
 ;
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/rel32.ll b/llvm/test/CodeGen/AMDGPU/rel32.ll
new file mode 100644
index 0000000000000..21c80c7628b88
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rel32.ll
@@ -0,0 +1,12 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck %s
+
+@g = protected local_unnamed_addr addrspace(4) externally_initialized global i32 0, align 4
+
+; CHECK-LABEL: rel32_neg_offset:
+; CHECK: s_getpc_b64 s{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{]}}
+; CHECK: s_add_u32 s[[LO]], s[[LO]], g@rel32@lo-4
+; CHECK: s_addc_u32 s[[HI]], s[[HI]], g@rel32@hi-4
+define i32 addrspace(4)* @rel32_neg_offset() {
+  %r = getelementptr i32, i32 addrspace(4)* @g, i64 -2
+  ret i32 addrspace(4)* %r
+}
diff --git a/llvm/test/CodeGen/AMDGPU/reserve-vgpr-for-sgpr-spill.ll b/llvm/test/CodeGen/AMDGPU/reserve-vgpr-for-sgpr-spill.ll
index 3858fd9eec06f..9df99aae15d49 100644
--- a/llvm/test/CodeGen/AMDGPU/reserve-vgpr-for-sgpr-spill.ll
+++ b/llvm/test/CodeGen/AMDGPU/reserve-vgpr-for-sgpr-spill.ll
@@ -47,4 +47,4 @@ define void @parent_func() #0 {
   ret void
 }
 
-attributes #0 = { nounwind noinline norecurse }
+attributes #0 = { nounwind noinline norecurse "amdgpu-flat-work-group-size"="1,256" }
diff --git a/llvm/test/CodeGen/AMDGPU/sad.ll b/llvm/test/CodeGen/AMDGPU/sad.ll
index 3a4a2d07772c1..514223d47d88b 100644
--- a/llvm/test/CodeGen/AMDGPU/sad.ll
+++ b/llvm/test/CodeGen/AMDGPU/sad.ll
@@ -134,7 +134,7 @@ define amdgpu_kernel void @v_sad_u32_multi_use_sub_pat2(i32 addrspace(1)* %out,
 
 ; GCN-LABEL: {{^}}v_sad_u32_multi_use_select_pat2:
 ; GCN: s_sub_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-; GCN-DAG: v_cmp_gt_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG: s_cmp_gt_u32  s{{[0-9]+}}, s{{[0-9]+}}
 ; GCN-DAG: s_sub_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
 define amdgpu_kernel void @v_sad_u32_multi_use_select_pat2(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c) {
   %icmp0 = icmp ugt i32 %a, %b
@@ -254,12 +254,12 @@ define amdgpu_kernel void @v_sad_u32_i8_pat2(i8 addrspace(1)* %out) {
 
 ; GCN-LABEL: {{^}}s_sad_u32_i8_pat2:
 ; GCN: s_load_dword
-; GCN: s_bfe_u32
+; GCN-DAG: s_bfe_u32
 ; GCN-DAG: s_sub_i32
 ; GCN-DAG: s_and_b32
 ; GCN-DAG: s_sub_i32
 ; GCN-DAG: s_lshr_b32
-; GCN: v_add_i32_e32
+; GCN: s_add_i32
 define amdgpu_kernel void @s_sad_u32_i8_pat2(i8 addrspace(1)* %out, i8 zeroext %a, i8 zeroext %b, i8 zeroext %c) {
   %icmp0 = icmp ugt i8 %a, %b
   %sub0 = sub i8 %a, %b
@@ -273,10 +273,10 @@ define amdgpu_kernel void @s_sad_u32_i8_pat2(i8 addrspace(1)* %out, i8 zeroext %
 }
 
 ; GCN-LABEL: {{^}}v_sad_u32_mismatched_operands_pat1:
-; GCN: v_cmp_le_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
 ; GCN: s_max_u32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-; GCN: v_sub_i32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GCN: v_add_i32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GCN: s_cmp_le_u32 s{{[0-9]+}}, s{{[0-9]+}}
+; GCN: s_sub_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+; GCN: s_add_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
 define amdgpu_kernel void @v_sad_u32_mismatched_operands_pat1(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c, i32 %d) {
   %icmp0 = icmp ugt i32 %a, %b
   %t0 = select i1 %icmp0, i32 %a, i32 %b
@@ -294,7 +294,7 @@ define amdgpu_kernel void @v_sad_u32_mismatched_operands_pat1(i32 addrspace(1)*
 ; GCN-LABEL: {{^}}v_sad_u32_mismatched_operands_pat2:
 ; GCN: s_sub_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
 ; GCN: s_sub_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-; GCN: v_add_i32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GCN: s_add_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
 define amdgpu_kernel void @v_sad_u32_mismatched_operands_pat2(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c, i32 %d) {
   %icmp0 = icmp ugt i32 %a, %b
   %sub0 = sub i32 %a, %d
diff --git a/llvm/test/CodeGen/AMDGPU/salu-to-valu.ll b/llvm/test/CodeGen/AMDGPU/salu-to-valu.ll
index 14635ab7e708f..091ed34e11121 100644
--- a/llvm/test/CodeGen/AMDGPU/salu-to-valu.ll
+++ b/llvm/test/CodeGen/AMDGPU/salu-to-valu.ll
@@ -203,10 +203,10 @@ entry:
 ; GCN-LABEL: {{^}}smrd_valu_ci_offset_x16:
 
 ; SI: s_mov_b32 {{s[0-9]+}}, 0x13480
+; SI: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+:[0-9]+}}], {{s[0-9]+}} addr64
 ; SI: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+:[0-9]+}}], 0 addr64 offset:16
 ; SI: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+:[0-9]+}}], 0 addr64 offset:32
 ; SI: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+:[0-9]+}}], 0 addr64 offset:48
-; SI: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+:[0-9]+}}], {{s[0-9]+}} addr64
 ; CI-NOHSA-DAG: s_mov_b32 [[OFFSET0:s[0-9]+]], 0x13480{{$}}
 ; CI-NOHSA-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+:[0-9]+}}], [[OFFSET0]] addr64{{$}}
 ; CI-NOHSA-DAG: s_mov_b32 [[OFFSET1:s[0-9]+]], 0x13490{{$}}
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv.ll b/llvm/test/CodeGen/AMDGPU/sdiv.ll
index 4b322a8750057..f51d152fa157d 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv.ll
@@ -16,48 +16,44 @@
 define amdgpu_kernel void @sdiv_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
 ; GCN-LABEL: sdiv_i32:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
-; GCN-NEXT:    s_mov_b32 s2, s6
-; GCN-NEXT:    s_mov_b32 s3, s7
+; GCN-NEXT:    s_mov_b32 s10, s6
+; GCN-NEXT:    s_mov_b32 s11, s7
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s0, s10
-; GCN-NEXT:    s_mov_b32 s1, s11
-; GCN-NEXT:    buffer_load_dwordx2 v[0:1], off, s[0:3], 0
-; GCN-NEXT:    s_mov_b32 s4, s8
-; GCN-NEXT:    s_mov_b32 s5, s9
+; GCN-NEXT:    s_mov_b32 s8, s2
+; GCN-NEXT:    s_mov_b32 s9, s3
+; GCN-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NEXT:    s_mov_b32 s4, s0
+; GCN-NEXT:    s_mov_b32 s5, s1
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-NEXT:    v_xor_b32_e32 v1, v1, v2
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v3, v1
-; GCN-NEXT:    v_ashrrev_i32_e32 v6, 31, v0
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v6, v0
-; GCN-NEXT:    v_xor_b32_e32 v0, v0, v6
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v1
+; GCN-NEXT:    v_ashrrev_i32_e32 v5, 31, v0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v5, v0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; GCN-NEXT:    v_xor_b32_e32 v2, v6, v2
-; GCN-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; GCN-NEXT:    v_xor_b32_e32 v0, v0, v5
+; GCN-NEXT:    v_xor_b32_e32 v2, v5, v2
+; GCN-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, v1
-; GCN-NEXT:    v_mul_hi_u32 v5, v3, v1
-; GCN-NEXT:    v_sub_i32_e32 v7, vcc, 0, v4
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v4, v4, v3
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v4, v3
-; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, v4, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v5, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v3, v3, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, v4, v3
+; GCN-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v3
 ; GCN-NEXT:    v_mul_lo_u32 v4, v3, v1
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, -1, v3
-; GCN-NEXT:    v_subrev_i32_e32 v7, vcc, v4, v0
-; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v7, v1
-; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v3, v5, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v4, v0
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v0, v1
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, v1, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v3
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v3, v4, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
 ; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
@@ -65,48 +61,44 @@ define amdgpu_kernel void @sdiv_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %i
 ;
 ; TONGA-LABEL: sdiv_i32:
 ; TONGA:       ; %bb.0:
-; TONGA-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x24
+; TONGA-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; TONGA-NEXT:    s_mov_b32 s7, 0xf000
 ; TONGA-NEXT:    s_mov_b32 s6, -1
-; TONGA-NEXT:    s_mov_b32 s2, s6
-; TONGA-NEXT:    s_mov_b32 s3, s7
+; TONGA-NEXT:    s_mov_b32 s10, s6
+; TONGA-NEXT:    s_mov_b32 s11, s7
 ; TONGA-NEXT:    s_waitcnt lgkmcnt(0)
-; TONGA-NEXT:    s_mov_b32 s0, s10
-; TONGA-NEXT:    s_mov_b32 s1, s11
-; TONGA-NEXT:    buffer_load_dwordx2 v[0:1], off, s[0:3], 0
-; TONGA-NEXT:    s_mov_b32 s4, s8
-; TONGA-NEXT:    s_mov_b32 s5, s9
+; TONGA-NEXT:    s_mov_b32 s8, s2
+; TONGA-NEXT:    s_mov_b32 s9, s3
+; TONGA-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; TONGA-NEXT:    s_mov_b32 s4, s0
+; TONGA-NEXT:    s_mov_b32 s5, s1
 ; TONGA-NEXT:    s_waitcnt vmcnt(0)
 ; TONGA-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; TONGA-NEXT:    v_add_u32_e32 v1, vcc, v2, v1
 ; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v2
 ; TONGA-NEXT:    v_cvt_f32_u32_e32 v3, v1
-; TONGA-NEXT:    v_ashrrev_i32_e32 v6, 31, v0
-; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v6, v0
-; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v6
+; TONGA-NEXT:    v_sub_u32_e32 v4, vcc, 0, v1
+; TONGA-NEXT:    v_ashrrev_i32_e32 v5, 31, v0
+; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v5, v0
 ; TONGA-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; TONGA-NEXT:    v_xor_b32_e32 v2, v6, v2
-; TONGA-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v5
+; TONGA-NEXT:    v_xor_b32_e32 v2, v5, v2
+; TONGA-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; TONGA-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; TONGA-NEXT:    v_mul_lo_u32 v4, v3, v1
-; TONGA-NEXT:    v_mul_hi_u32 v5, v3, v1
-; TONGA-NEXT:    v_sub_u32_e32 v7, vcc, 0, v4
-; TONGA-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v5
-; TONGA-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[0:1]
-; TONGA-NEXT:    v_mul_hi_u32 v4, v4, v3
-; TONGA-NEXT:    v_add_u32_e32 v5, vcc, v4, v3
-; TONGA-NEXT:    v_subrev_u32_e32 v3, vcc, v4, v3
-; TONGA-NEXT:    v_cndmask_b32_e64 v3, v3, v5, s[0:1]
-; TONGA-NEXT:    v_mul_hi_u32 v3, v3, v0
+; TONGA-NEXT:    v_mul_lo_u32 v4, v4, v3
+; TONGA-NEXT:    v_mul_hi_u32 v4, v3, v4
+; TONGA-NEXT:    v_add_u32_e32 v3, vcc, v4, v3
+; TONGA-NEXT:    v_mul_hi_u32 v3, v0, v3
 ; TONGA-NEXT:    v_mul_lo_u32 v4, v3, v1
 ; TONGA-NEXT:    v_add_u32_e32 v5, vcc, 1, v3
-; TONGA-NEXT:    v_add_u32_e32 v6, vcc, -1, v3
-; TONGA-NEXT:    v_subrev_u32_e32 v7, vcc, v4, v0
-; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
-; TONGA-NEXT:    v_cmp_ge_u32_e64 s[0:1], v7, v1
-; TONGA-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
-; TONGA-NEXT:    v_cndmask_b32_e64 v0, v3, v5, s[0:1]
-; TONGA-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; TONGA-NEXT:    v_subrev_u32_e32 v0, vcc, v4, v0
+; TONGA-NEXT:    v_cmp_ge_u32_e64 s[0:1], v0, v1
+; TONGA-NEXT:    v_subrev_u32_e32 v4, vcc, v1, v0
+; TONGA-NEXT:    v_cndmask_b32_e64 v3, v3, v5, s[0:1]
+; TONGA-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[0:1]
+; TONGA-NEXT:    v_add_u32_e32 v4, vcc, 1, v3
+; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; TONGA-NEXT:    v_cndmask_b32_e32 v0, v3, v4, vcc
 ; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v0, v2
 ; TONGA-NEXT:    buffer_store_dword v0, off, s[4:7], 0
@@ -114,58 +106,54 @@ define amdgpu_kernel void @sdiv_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %i
 ;
 ; GFX9-LABEL: sdiv_i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX9-NEXT:    s_mov_b32 s7, 0xf000
-; GFX9-NEXT:    s_mov_b32 s6, -1
-; GFX9-NEXT:    s_mov_b32 s10, s6
-; GFX9-NEXT:    s_mov_b32 s11, s7
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_mov_b32 s10, s2
+; GFX9-NEXT:    s_mov_b32 s11, s3
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s8, s2
-; GFX9-NEXT:    s_mov_b32 s9, s3
+; GFX9-NEXT:    s_mov_b32 s8, s6
+; GFX9-NEXT:    s_mov_b32 s9, s7
 ; GFX9-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
-; GFX9-NEXT:    s_mov_b32 s4, s0
-; GFX9-NEXT:    s_mov_b32 s5, s1
+; GFX9-NEXT:    s_mov_b32 s0, s4
+; GFX9-NEXT:    s_mov_b32 s1, s5
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; GFX9-NEXT:    v_add_u32_e32 v1, v1, v2
 ; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v2
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, v1
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; GFX9-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GFX9-NEXT:    v_mul_lo_u32 v4, v3, v1
-; GFX9-NEXT:    v_mul_hi_u32 v5, v3, v1
-; GFX9-NEXT:    v_sub_u32_e32 v6, 0, v4
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v4, v4, v3
+; GFX9-NEXT:    v_sub_u32_e32 v4, 0, v1
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v5, 31, v0
 ; GFX9-NEXT:    v_add_u32_e32 v0, v0, v5
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v3
 ; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v5
-; GFX9-NEXT:    v_add_u32_e32 v6, v3, v4
-; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v3, v3, v0
 ; GFX9-NEXT:    v_xor_b32_e32 v2, v5, v2
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT:    v_mul_lo_u32 v4, v4, v3
+; GFX9-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v4
+; GFX9-NEXT:    v_mul_hi_u32 v3, v0, v3
 ; GFX9-NEXT:    v_mul_lo_u32 v4, v3, v1
 ; GFX9-NEXT:    v_add_u32_e32 v5, 1, v3
-; GFX9-NEXT:    v_add_u32_e32 v6, -1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v7, v0, v4
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[0:1], v7, v1
-; GFX9-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, v5, s[0:1]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v4
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v4, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX9-NEXT:    v_add_u32_e32 v4, 1, v3
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v4, vcc
 ; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v2
-; GFX9-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
 ;
 ; EG-LABEL: sdiv_i32:
 ; EG:       ; %bb.0:
 ; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    TEX 0 @6
-; EG-NEXT:    ALU 30, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 26, @9, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
@@ -177,29 +165,25 @@ define amdgpu_kernel void @sdiv_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %i
 ; EG-NEXT:     SETGT_INT * T0.W, 0.0, T0.Y,
 ; EG-NEXT:     ADD_INT * T1.W, T0.Y, PV.W,
 ; EG-NEXT:     XOR_INT * T1.W, PV.W, T0.W,
+; EG-NEXT:     SUB_INT T2.W, 0.0, PV.W,
 ; EG-NEXT:     RECIP_UINT * T0.Y, PV.W,
-; EG-NEXT:     MULLO_INT * T0.Z, PS, T1.W,
-; EG-NEXT:     SUB_INT T2.W, 0.0, PS,
-; EG-NEXT:     MULHI * T1.X, T0.Y, T1.W,
-; EG-NEXT:     CNDE_INT T2.W, PS, PV.W, T0.Z,
-; EG-NEXT:     SETGT_INT * T3.W, 0.0, T0.X,
-; EG-NEXT:     MULHI * T0.Z, PV.W, T0.Y,
-; EG-NEXT:     ADD_INT T1.Z, T0.X, T3.W,
-; EG-NEXT:     ADD_INT T2.W, T0.Y, PS,
-; EG-NEXT:     SUB_INT * T4.W, T0.Y, PS,
-; EG-NEXT:     CNDE_INT T2.W, T1.X, PV.W, PS,
-; EG-NEXT:     XOR_INT * T4.W, PV.Z, T3.W,
-; EG-NEXT:     MULHI * T0.X, PV.W, PS,
+; EG-NEXT:     SETGT_INT T3.W, 0.0, T0.X,
+; EG-NEXT:     MULLO_INT * T0.Z, PV.W, PS,
+; EG-NEXT:     ADD_INT T2.W, T0.X, PV.W,
+; EG-NEXT:     MULHI * T0.X, T0.Y, PS,
+; EG-NEXT:     ADD_INT T4.W, T0.Y, PS,
+; EG-NEXT:     XOR_INT * T2.W, PV.W, T3.W,
+; EG-NEXT:     MULHI * T0.X, PS, PV.W,
 ; EG-NEXT:     MULLO_INT * T0.Y, PS, T1.W,
-; EG-NEXT:     SUB_INT * T2.W, T4.W, PS,
-; EG-NEXT:     SETGE_UINT T1.W, PV.W, T1.W,
-; EG-NEXT:     SETGE_UINT * T2.W, T4.W, T0.Y,
-; EG-NEXT:     AND_INT T1.W, PV.W, PS,
-; EG-NEXT:     ADD_INT * T4.W, T0.X, 1,
-; EG-NEXT:     CNDE_INT T1.W, PV.W, T0.X, PS,
-; EG-NEXT:     ADD_INT * T4.W, T0.X, literal.x,
-; EG-NEXT:    -1(nan), 0(0.000000e+00)
-; EG-NEXT:     CNDE_INT T1.W, T2.W, PS, PV.W,
+; EG-NEXT:     SUB_INT * T2.W, T2.W, PS,
+; EG-NEXT:     ADD_INT T0.Z, T0.X, 1,
+; EG-NEXT:     SETGE_UINT T4.W, PV.W, T1.W,
+; EG-NEXT:     SUB_INT * T5.W, PV.W, T1.W,
+; EG-NEXT:     CNDE_INT T2.W, PV.W, T2.W, PS,
+; EG-NEXT:     CNDE_INT * T4.W, PV.W, T0.X, PV.Z,
+; EG-NEXT:     ADD_INT T5.W, PS, 1,
+; EG-NEXT:     SETGE_UINT * T1.W, PV.W, T1.W,
+; EG-NEXT:     CNDE_INT T1.W, PS, T4.W, PV.W, BS:VEC_102/SCL_221
 ; EG-NEXT:     XOR_INT * T0.W, T3.W, T0.W,
 ; EG-NEXT:     XOR_INT * T1.W, PV.W, PS,
 ; EG-NEXT:     SUB_INT T0.X, PV.W, T0.W,
@@ -408,255 +392,233 @@ define amdgpu_kernel void @slow_sdiv_i32_3435(i32 addrspace(1)* %out, i32 addrsp
 define amdgpu_kernel void @sdiv_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in) {
 ; GCN-LABEL: sdiv_v2i32:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
-; GCN-NEXT:    s_mov_b32 s11, 0xf000
-; GCN-NEXT:    s_mov_b32 s10, -1
-; GCN-NEXT:    s_mov_b32 s4, 0x4f800000
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_mov_b32 s2, s6
+; GCN-NEXT:    s_mov_b32 s3, s7
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s8, s0
-; GCN-NEXT:    s_mov_b32 s9, s1
-; GCN-NEXT:    s_mov_b32 s0, s2
-; GCN-NEXT:    s_mov_b32 s1, s3
-; GCN-NEXT:    s_mov_b32 s2, s10
-; GCN-NEXT:    s_mov_b32 s3, s11
+; GCN-NEXT:    s_mov_b32 s0, s10
+; GCN-NEXT:    s_mov_b32 s1, s11
 ; GCN-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NEXT:    s_mov_b32 s2, 0x4f7ffffe
+; GCN-NEXT:    s_mov_b32 s4, s8
+; GCN-NEXT:    s_mov_b32 s5, s9
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_ashrrev_i32_e32 v5, 31, v2
-; GCN-NEXT:    v_ashrrev_i32_e32 v7, 31, v3
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
-; GCN-NEXT:    v_ashrrev_i32_e32 v4, 31, v0
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v7, v3
-; GCN-NEXT:    v_xor_b32_e32 v2, v2, v5
-; GCN-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
-; GCN-NEXT:    v_xor_b32_e32 v8, v4, v5
+; GCN-NEXT:    v_ashrrev_i32_e32 v4, 31, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; GCN-NEXT:    v_xor_b32_e32 v2, v2, v4
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v5, v2
-; GCN-NEXT:    v_xor_b32_e32 v3, v3, v7
-; GCN-NEXT:    v_xor_b32_e32 v9, v6, v7
-; GCN-NEXT:    v_cvt_f32_u32_e32 v7, v3
+; GCN-NEXT:    v_sub_i32_e32 v6, vcc, 0, v2
+; GCN-NEXT:    v_ashrrev_i32_e32 v7, 31, v0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v7, v0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
-; GCN-NEXT:    v_xor_b32_e32 v0, v0, v4
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v7, v7
-; GCN-NEXT:    v_mul_f32_e32 v4, s4, v5
-; GCN-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v6, v1
-; GCN-NEXT:    v_mul_f32_e32 v5, s4, v7
+; GCN-NEXT:    v_xor_b32_e32 v0, v0, v7
+; GCN-NEXT:    v_xor_b32_e32 v4, v7, v4
+; GCN-NEXT:    v_mul_f32_e32 v5, s2, v5
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GCN-NEXT:    v_mul_lo_u32 v6, v6, v5
+; GCN-NEXT:    v_mul_hi_u32 v6, v5, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GCN-NEXT:    v_mul_hi_u32 v5, v0, v5
+; GCN-NEXT:    v_ashrrev_i32_e32 v6, 31, v3
+; GCN-NEXT:    v_mul_lo_u32 v8, v5, v2
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 1, v5
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v8, v0
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v0, v2
+; GCN-NEXT:    v_subrev_i32_e32 v8, vcc, v2, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 1, v5
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GCN-NEXT:    s_mov_b64 s[0:1], vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v6, v3
+; GCN-NEXT:    v_xor_b32_e32 v2, v0, v6
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, v2
+; GCN-NEXT:    v_sub_i32_e32 v9, vcc, 0, v2
+; GCN-NEXT:    v_ashrrev_i32_e32 v3, 31, v1
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    v_xor_b32_e32 v1, v1, v3
+; GCN-NEXT:    v_xor_b32_e32 v6, v3, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v5, v8, s[0:1]
+; GCN-NEXT:    v_mul_f32_e32 v0, s2, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_lo_u32 v9, v9, v0
+; GCN-NEXT:    v_mul_hi_u32 v7, v0, v9
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v7, v0
+; GCN-NEXT:    v_mul_hi_u32 v3, v1, v0
+; GCN-NEXT:    v_xor_b32_e32 v0, v5, v4
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v4, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v4, v1
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v1, v2
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, v2, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v4, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v3
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v1, v1, v6
-; GCN-NEXT:    v_mul_hi_u32 v6, v4, v2
-; GCN-NEXT:    v_mul_lo_u32 v7, v4, v2
-; GCN-NEXT:    v_mul_hi_u32 v10, v5, v3
-; GCN-NEXT:    v_mul_lo_u32 v11, v5, v3
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v6
-; GCN-NEXT:    v_sub_i32_e32 v12, vcc, 0, v7
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v7, v12, s[0:1]
-; GCN-NEXT:    v_sub_i32_e32 v13, vcc, 0, v11
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v10
-; GCN-NEXT:    v_cndmask_b32_e64 v7, v11, v13, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v6, v6, v4
-; GCN-NEXT:    v_mul_hi_u32 v7, v7, v5
-; GCN-NEXT:    v_add_i32_e32 v10, vcc, v6, v4
-; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, v6, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[0:1]
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v7, v5
-; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, v7, v5
-; GCN-NEXT:    v_mul_hi_u32 v4, v4, v0
-; GCN-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v5, v5, v1
-; GCN-NEXT:    v_mul_lo_u32 v6, v4, v2
-; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
-; GCN-NEXT:    v_mul_lo_u32 v11, v5, v3
-; GCN-NEXT:    v_add_i32_e32 v10, vcc, -1, v4
-; GCN-NEXT:    v_subrev_i32_e32 v14, vcc, v6, v0
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v0, v6
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], v14, v2
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v11, v1
-; GCN-NEXT:    v_add_i32_e32 v12, vcc, 1, v5
-; GCN-NEXT:    v_add_i32_e32 v13, vcc, -1, v5
-; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v11
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v3
-; GCN-NEXT:    s_and_b64 s[2:3], s[2:3], s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v4, v7, s[2:3]
-; GCN-NEXT:    s_and_b64 s[2:3], s[4:5], vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v5, v12, s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v10, v0, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; GCN-NEXT:    v_xor_b32_e32 v0, v0, v8
-; GCN-NEXT:    v_xor_b32_e32 v1, v1, v9
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v0, v8
-; GCN-NEXT:    v_sub_i32_e32 v1, vcc, v1, v9
-; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, v1, v6
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; TONGA-LABEL: sdiv_v2i32:
 ; TONGA:       ; %bb.0:
-; TONGA-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; TONGA-NEXT:    s_mov_b32 s11, 0xf000
-; TONGA-NEXT:    s_mov_b32 s10, -1
-; TONGA-NEXT:    s_mov_b32 s4, 0x4f800000
+; TONGA-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x24
+; TONGA-NEXT:    s_mov_b32 s7, 0xf000
+; TONGA-NEXT:    s_mov_b32 s6, -1
+; TONGA-NEXT:    s_mov_b32 s2, s6
+; TONGA-NEXT:    s_mov_b32 s3, s7
 ; TONGA-NEXT:    s_waitcnt lgkmcnt(0)
-; TONGA-NEXT:    s_mov_b32 s8, s0
-; TONGA-NEXT:    s_mov_b32 s9, s1
-; TONGA-NEXT:    s_mov_b32 s0, s2
-; TONGA-NEXT:    s_mov_b32 s1, s3
-; TONGA-NEXT:    s_mov_b32 s2, s10
-; TONGA-NEXT:    s_mov_b32 s3, s11
+; TONGA-NEXT:    s_mov_b32 s0, s10
+; TONGA-NEXT:    s_mov_b32 s1, s11
 ; TONGA-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
+; TONGA-NEXT:    s_mov_b32 s2, 0x4f7ffffe
+; TONGA-NEXT:    s_mov_b32 s4, s8
+; TONGA-NEXT:    s_mov_b32 s5, s9
 ; TONGA-NEXT:    s_waitcnt vmcnt(0)
-; TONGA-NEXT:    v_ashrrev_i32_e32 v5, 31, v2
-; TONGA-NEXT:    v_ashrrev_i32_e32 v7, 31, v3
-; TONGA-NEXT:    v_add_u32_e32 v2, vcc, v5, v2
-; TONGA-NEXT:    v_ashrrev_i32_e32 v4, 31, v0
-; TONGA-NEXT:    v_add_u32_e32 v3, vcc, v7, v3
-; TONGA-NEXT:    v_xor_b32_e32 v2, v2, v5
-; TONGA-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
-; TONGA-NEXT:    v_xor_b32_e32 v8, v4, v5
+; TONGA-NEXT:    v_ashrrev_i32_e32 v4, 31, v2
+; TONGA-NEXT:    v_add_u32_e32 v2, vcc, v4, v2
+; TONGA-NEXT:    v_xor_b32_e32 v2, v2, v4
 ; TONGA-NEXT:    v_cvt_f32_u32_e32 v5, v2
-; TONGA-NEXT:    v_xor_b32_e32 v3, v3, v7
-; TONGA-NEXT:    v_xor_b32_e32 v9, v6, v7
-; TONGA-NEXT:    v_cvt_f32_u32_e32 v7, v3
+; TONGA-NEXT:    v_sub_u32_e32 v6, vcc, 0, v2
+; TONGA-NEXT:    v_ashrrev_i32_e32 v7, 31, v0
+; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v7, v0
 ; TONGA-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v4, v0
-; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v4
-; TONGA-NEXT:    v_rcp_iflag_f32_e32 v7, v7
-; TONGA-NEXT:    v_mul_f32_e32 v4, s4, v5
-; TONGA-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; TONGA-NEXT:    v_add_u32_e32 v1, vcc, v6, v1
-; TONGA-NEXT:    v_mul_f32_e32 v5, s4, v7
+; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v7
+; TONGA-NEXT:    v_xor_b32_e32 v4, v7, v4
+; TONGA-NEXT:    v_mul_f32_e32 v5, s2, v5
 ; TONGA-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; TONGA-NEXT:    v_mul_lo_u32 v6, v6, v5
+; TONGA-NEXT:    v_mul_hi_u32 v6, v5, v6
+; TONGA-NEXT:    v_add_u32_e32 v5, vcc, v6, v5
+; TONGA-NEXT:    v_mul_hi_u32 v5, v0, v5
+; TONGA-NEXT:    v_ashrrev_i32_e32 v6, 31, v3
+; TONGA-NEXT:    v_mul_lo_u32 v8, v5, v2
+; TONGA-NEXT:    v_add_u32_e32 v9, vcc, 1, v5
+; TONGA-NEXT:    v_subrev_u32_e32 v0, vcc, v8, v0
+; TONGA-NEXT:    v_cmp_ge_u32_e64 s[0:1], v0, v2
+; TONGA-NEXT:    v_subrev_u32_e32 v8, vcc, v2, v0
+; TONGA-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[0:1]
+; TONGA-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[0:1]
+; TONGA-NEXT:    v_add_u32_e32 v8, vcc, 1, v5
+; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; TONGA-NEXT:    s_mov_b64 s[0:1], vcc
+; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v6, v3
+; TONGA-NEXT:    v_xor_b32_e32 v2, v0, v6
+; TONGA-NEXT:    v_cvt_f32_u32_e32 v0, v2
+; TONGA-NEXT:    v_sub_u32_e32 v9, vcc, 0, v2
+; TONGA-NEXT:    v_ashrrev_i32_e32 v3, 31, v1
+; TONGA-NEXT:    v_add_u32_e32 v1, vcc, v3, v1
+; TONGA-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v3
+; TONGA-NEXT:    v_xor_b32_e32 v6, v3, v6
+; TONGA-NEXT:    v_cndmask_b32_e64 v5, v5, v8, s[0:1]
+; TONGA-NEXT:    v_mul_f32_e32 v0, s2, v0
+; TONGA-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; TONGA-NEXT:    v_mul_lo_u32 v9, v9, v0
+; TONGA-NEXT:    v_mul_hi_u32 v7, v0, v9
+; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v7, v0
+; TONGA-NEXT:    v_mul_hi_u32 v3, v1, v0
+; TONGA-NEXT:    v_xor_b32_e32 v0, v5, v4
+; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v0, v4
+; TONGA-NEXT:    v_mul_lo_u32 v4, v3, v2
+; TONGA-NEXT:    v_add_u32_e32 v5, vcc, 1, v3
+; TONGA-NEXT:    v_subrev_u32_e32 v1, vcc, v4, v1
+; TONGA-NEXT:    v_cmp_ge_u32_e64 s[0:1], v1, v2
+; TONGA-NEXT:    v_subrev_u32_e32 v4, vcc, v2, v1
+; TONGA-NEXT:    v_cndmask_b32_e64 v3, v3, v5, s[0:1]
+; TONGA-NEXT:    v_cndmask_b32_e64 v1, v1, v4, s[0:1]
+; TONGA-NEXT:    v_add_u32_e32 v4, vcc, 1, v3
+; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
+; TONGA-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
 ; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v6
-; TONGA-NEXT:    v_mul_hi_u32 v6, v4, v2
-; TONGA-NEXT:    v_mul_lo_u32 v7, v4, v2
-; TONGA-NEXT:    v_mul_hi_u32 v10, v5, v3
-; TONGA-NEXT:    v_mul_lo_u32 v11, v5, v3
-; TONGA-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v6
-; TONGA-NEXT:    v_sub_u32_e32 v12, vcc, 0, v7
-; TONGA-NEXT:    v_cndmask_b32_e64 v6, v7, v12, s[0:1]
-; TONGA-NEXT:    v_sub_u32_e32 v13, vcc, 0, v11
-; TONGA-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v10
-; TONGA-NEXT:    v_cndmask_b32_e64 v7, v11, v13, s[2:3]
-; TONGA-NEXT:    v_mul_hi_u32 v6, v6, v4
-; TONGA-NEXT:    v_mul_hi_u32 v7, v7, v5
-; TONGA-NEXT:    v_add_u32_e32 v10, vcc, v6, v4
-; TONGA-NEXT:    v_subrev_u32_e32 v4, vcc, v6, v4
-; TONGA-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[0:1]
-; TONGA-NEXT:    v_add_u32_e32 v6, vcc, v7, v5
-; TONGA-NEXT:    v_subrev_u32_e32 v5, vcc, v7, v5
-; TONGA-NEXT:    v_mul_hi_u32 v4, v4, v0
-; TONGA-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s[2:3]
-; TONGA-NEXT:    v_mul_hi_u32 v5, v5, v1
-; TONGA-NEXT:    v_mul_lo_u32 v6, v4, v2
-; TONGA-NEXT:    v_add_u32_e32 v7, vcc, 1, v4
-; TONGA-NEXT:    v_mul_lo_u32 v11, v5, v3
-; TONGA-NEXT:    v_add_u32_e32 v10, vcc, -1, v4
-; TONGA-NEXT:    v_subrev_u32_e32 v14, vcc, v6, v0
-; TONGA-NEXT:    v_cmp_ge_u32_e64 s[0:1], v0, v6
-; TONGA-NEXT:    v_cmp_ge_u32_e64 s[2:3], v14, v2
-; TONGA-NEXT:    v_subrev_u32_e32 v0, vcc, v11, v1
-; TONGA-NEXT:    v_add_u32_e32 v12, vcc, 1, v5
-; TONGA-NEXT:    v_add_u32_e32 v13, vcc, -1, v5
-; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v11
-; TONGA-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v3
-; TONGA-NEXT:    s_and_b64 s[2:3], s[2:3], s[0:1]
-; TONGA-NEXT:    v_cndmask_b32_e64 v0, v4, v7, s[2:3]
-; TONGA-NEXT:    s_and_b64 s[2:3], s[4:5], vcc
-; TONGA-NEXT:    v_cndmask_b32_e64 v1, v5, v12, s[2:3]
-; TONGA-NEXT:    v_cndmask_b32_e64 v0, v10, v0, s[0:1]
-; TONGA-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc
-; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v8
-; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v9
-; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v0, v8
-; TONGA-NEXT:    v_sub_u32_e32 v1, vcc, v1, v9
-; TONGA-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; TONGA-NEXT:    v_sub_u32_e32 v1, vcc, v1, v6
+; TONGA-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; TONGA-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: sdiv_v2i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX9-NEXT:    s_mov_b32 s11, 0xf000
-; GFX9-NEXT:    s_mov_b32 s10, -1
-; GFX9-NEXT:    s_mov_b32 s4, 0x4f800000
+; GFX9-NEXT:    s_mov_b32 s7, 0xf000
+; GFX9-NEXT:    s_mov_b32 s6, -1
+; GFX9-NEXT:    s_mov_b32 s10, s6
+; GFX9-NEXT:    s_mov_b32 s11, s7
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s8, s0
-; GFX9-NEXT:    s_mov_b32 s9, s1
-; GFX9-NEXT:    s_mov_b32 s0, s2
-; GFX9-NEXT:    s_mov_b32 s1, s3
-; GFX9-NEXT:    s_mov_b32 s2, s10
-; GFX9-NEXT:    s_mov_b32 s3, s11
-; GFX9-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
+; GFX9-NEXT:    s_mov_b32 s8, s2
+; GFX9-NEXT:    s_mov_b32 s9, s3
+; GFX9-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GFX9-NEXT:    s_mov_b32 s2, 0x4f7ffffe
+; GFX9-NEXT:    s_mov_b32 s4, s0
+; GFX9-NEXT:    s_mov_b32 s5, s1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_ashrrev_i32_e32 v5, 31, v2
-; GFX9-NEXT:    v_ashrrev_i32_e32 v6, 31, v3
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-NEXT:    v_add_u32_e32 v3, v3, v6
-; GFX9-NEXT:    v_xor_b32_e32 v2, v2, v5
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v7, v2
-; GFX9-NEXT:    v_xor_b32_e32 v3, v3, v6
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v8, v3
-; GFX9-NEXT:    v_ashrrev_i32_e32 v4, 31, v0
+; GFX9-NEXT:    v_ashrrev_i32_e32 v4, 31, v2
+; GFX9-NEXT:    v_ashrrev_i32_e32 v5, 31, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v5
+; GFX9-NEXT:    v_xor_b32_e32 v2, v2, v4
+; GFX9-NEXT:    v_xor_b32_e32 v3, v3, v5
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v6, v2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v7, v3
+; GFX9-NEXT:    v_sub_u32_e32 v10, 0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v11, 0, v3
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v6, v6
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v7, v7
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v4
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v8, v8
-; GFX9-NEXT:    v_xor_b32_e32 v5, v4, v5
-; GFX9-NEXT:    v_mul_f32_e32 v7, s4, v7
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; GFX9-NEXT:    v_mul_f32_e32 v8, s4, v8
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v8, v8
-; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v4
-; GFX9-NEXT:    v_mul_lo_u32 v4, v7, v2
-; GFX9-NEXT:    v_mul_hi_u32 v11, v7, v2
-; GFX9-NEXT:    v_mul_lo_u32 v10, v8, v3
-; GFX9-NEXT:    v_mul_hi_u32 v12, v8, v3
-; GFX9-NEXT:    v_sub_u32_e32 v13, 0, v4
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v11
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v13, vcc
-; GFX9-NEXT:    v_sub_u32_e32 v14, 0, v10
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v12
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v10, v14, s[0:1]
-; GFX9-NEXT:    v_mul_hi_u32 v4, v4, v7
-; GFX9-NEXT:    v_mul_hi_u32 v10, v10, v8
+; GFX9-NEXT:    v_ashrrev_i32_e32 v8, 31, v0
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v9, 31, v1
+; GFX9-NEXT:    v_mul_f32_e32 v6, s2, v6
+; GFX9-NEXT:    v_mul_f32_e32 v7, s2, v7
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v8
 ; GFX9-NEXT:    v_add_u32_e32 v1, v1, v9
-; GFX9-NEXT:    v_xor_b32_e32 v6, v9, v6
+; GFX9-NEXT:    v_mul_lo_u32 v10, v10, v6
+; GFX9-NEXT:    v_mul_lo_u32 v11, v11, v7
+; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v8
 ; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v9
-; GFX9-NEXT:    v_add_u32_e32 v9, v7, v4
-; GFX9-NEXT:    v_sub_u32_e32 v4, v7, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v9, vcc
-; GFX9-NEXT:    v_add_u32_e32 v7, v8, v10
-; GFX9-NEXT:    v_sub_u32_e32 v8, v8, v10
-; GFX9-NEXT:    v_mul_hi_u32 v4, v4, v0
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v8, v7, s[0:1]
-; GFX9-NEXT:    v_mul_hi_u32 v7, v7, v1
-; GFX9-NEXT:    v_mul_lo_u32 v8, v4, v2
-; GFX9-NEXT:    v_add_u32_e32 v9, 1, v4
-; GFX9-NEXT:    v_mul_lo_u32 v11, v7, v3
-; GFX9-NEXT:    v_add_u32_e32 v12, 1, v7
-; GFX9-NEXT:    v_sub_u32_e32 v14, v0, v8
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v8
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[2:3], v14, v2
-; GFX9-NEXT:    v_sub_u32_e32 v0, v1, v11
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[0:1], v1, v11
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v3
-; GFX9-NEXT:    s_and_b64 s[2:3], s[2:3], vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, v9, s[2:3]
-; GFX9-NEXT:    s_and_b64 s[2:3], s[4:5], s[0:1]
-; GFX9-NEXT:    v_add_u32_e32 v10, -1, v4
-; GFX9-NEXT:    v_add_u32_e32 v13, -1, v7
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v7, v12, s[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v13, v1, s[0:1]
-; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v5
-; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v6
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v5
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v6
-; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GFX9-NEXT:    v_mul_hi_u32 v10, v6, v10
+; GFX9-NEXT:    v_mul_hi_u32 v11, v7, v11
+; GFX9-NEXT:    v_xor_b32_e32 v4, v8, v4
+; GFX9-NEXT:    v_xor_b32_e32 v5, v9, v5
+; GFX9-NEXT:    v_add_u32_e32 v6, v6, v10
+; GFX9-NEXT:    v_add_u32_e32 v7, v7, v11
+; GFX9-NEXT:    v_mul_hi_u32 v6, v0, v6
+; GFX9-NEXT:    v_mul_hi_u32 v7, v1, v7
+; GFX9-NEXT:    v_mul_lo_u32 v8, v6, v2
+; GFX9-NEXT:    v_mul_lo_u32 v9, v7, v3
+; GFX9-NEXT:    v_add_u32_e32 v10, 1, v6
+; GFX9-NEXT:    v_add_u32_e32 v11, 1, v7
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v8
+; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v9
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v8, v0, v2
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[0:1], v1, v3
+; GFX9-NEXT:    v_sub_u32_e32 v9, v1, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v11, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[0:1]
+; GFX9-NEXT:    v_add_u32_e32 v8, 1, v6
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v8, vcc
+; GFX9-NEXT:    v_add_u32_e32 v9, 1, v7
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v9, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v4
+; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v5
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v4
+; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v5
+; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX9-NEXT:    s_endpgm
 ;
 ; EG-LABEL: sdiv_v2i32:
 ; EG:       ; %bb.0:
 ; EG-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    TEX 1 @6
-; EG-NEXT:    ALU 59, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 51, @11, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
@@ -667,61 +629,53 @@ define amdgpu_kernel void @sdiv_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> ad
 ; EG-NEXT:     MOV * T0.X, KC0[2].Z,
 ; EG-NEXT:    ALU clause starting at 11:
 ; EG-NEXT:     SETGT_INT * T0.W, 0.0, T1.Y,
-; EG-NEXT:     ADD_INT * T1.W, T1.Y, PV.W,
-; EG-NEXT:     XOR_INT T1.W, PV.W, T0.W,
+; EG-NEXT:     ADD_INT T1.W, T1.Y, PV.W,
 ; EG-NEXT:     SETGT_INT * T2.W, 0.0, T1.X,
-; EG-NEXT:     ADD_INT T3.W, T1.X, PS,
-; EG-NEXT:     RECIP_UINT * T0.Z, PV.W,
-; EG-NEXT:     XOR_INT T3.W, PV.W, T2.W, BS:VEC_021/SCL_122
-; EG-NEXT:     MULLO_INT * T1.X, PS, T1.W,
+; EG-NEXT:     XOR_INT * T1.W, PV.W, T0.W,
+; EG-NEXT:     SUB_INT T0.Z, 0.0, PV.W,
+; EG-NEXT:     ADD_INT T3.W, T1.X, T2.W,
+; EG-NEXT:     RECIP_UINT * T1.X, PV.W,
+; EG-NEXT:     XOR_INT T3.W, PV.W, T2.W,
+; EG-NEXT:     MULLO_INT * T0.Z, PV.Z, PS,
+; EG-NEXT:     SUB_INT T4.W, 0.0, PV.W,
 ; EG-NEXT:     RECIP_UINT * T1.Y, PV.W,
-; EG-NEXT:     MULLO_INT * T1.Z, PS, T3.W,
-; EG-NEXT:     SUB_INT T4.W, 0.0, PS,
-; EG-NEXT:     MULHI * T2.X, T1.Y, T3.W,
-; EG-NEXT:     CNDE_INT T1.Z, PS, PV.W, T1.Z, BS:VEC_021/SCL_122
-; EG-NEXT:     SUB_INT T4.W, 0.0, T1.X,
-; EG-NEXT:     MULHI * T2.Y, T0.Z, T1.W,
-; EG-NEXT:     CNDE_INT T2.Z, PS, PV.W, T1.X,
-; EG-NEXT:     SETGT_INT T4.W, 0.0, T0.X,
-; EG-NEXT:     MULHI * T1.X, PV.Z, T1.Y,
-; EG-NEXT:     SETGT_INT T3.X, 0.0, T0.Y,
-; EG-NEXT:     ADD_INT T3.Y, T0.X, PV.W,
-; EG-NEXT:     ADD_INT T1.Z, T1.Y, PS,
-; EG-NEXT:     SUB_INT T5.W, T1.Y, PS,
-; EG-NEXT:     MULHI * T0.X, PV.Z, T0.Z,
-; EG-NEXT:     CNDE_INT T1.X, T2.X, PV.Z, PV.W,
-; EG-NEXT:     XOR_INT T1.Y, PV.Y, T4.W,
-; EG-NEXT:     ADD_INT T1.Z, T0.Y, PV.X,
-; EG-NEXT:     ADD_INT T5.W, T0.Z, PS,
-; EG-NEXT:     SUB_INT * T6.W, T0.Z, PS,
-; EG-NEXT:     CNDE_INT T0.Z, T2.Y, PV.W, PS,
-; EG-NEXT:     XOR_INT T5.W, PV.Z, T3.X,
-; EG-NEXT:     MULHI * T0.X, PV.X, PV.Y,
-; EG-NEXT:     MULHI * T0.Y, PV.Z, PV.W,
+; EG-NEXT:     SETGT_INT T5.W, 0.0, T0.X,
+; EG-NEXT:     MULLO_INT * T1.Z, PV.W, PS,
+; EG-NEXT:     SETGT_INT T2.Z, 0.0, T0.Y,
+; EG-NEXT:     ADD_INT T4.W, T0.X, PV.W,
+; EG-NEXT:     MULHI * T0.X, T1.Y, PS,
+; EG-NEXT:     ADD_INT T1.Y, T1.Y, PS,
+; EG-NEXT:     XOR_INT T1.Z, PV.W, T5.W,
+; EG-NEXT:     ADD_INT T4.W, T0.Y, PV.Z, BS:VEC_120/SCL_212
+; EG-NEXT:     MULHI * T0.X, T1.X, T0.Z,
+; EG-NEXT:     ADD_INT T0.Z, T1.X, PS,
+; EG-NEXT:     XOR_INT T4.W, PV.W, T2.Z,
+; EG-NEXT:     MULHI * T0.X, PV.Z, PV.Y,
+; EG-NEXT:     MULHI * T0.Y, PV.W, PV.Z,
 ; EG-NEXT:     MULLO_INT * T0.Z, PS, T1.W,
-; EG-NEXT:     SUB_INT T6.W, T5.W, PS,
-; EG-NEXT:     MULLO_INT * T1.X, T0.X, T3.W,
-; EG-NEXT:     SUB_INT T1.Z, T1.Y, PS,
-; EG-NEXT:     SETGE_UINT T1.W, PV.W, T1.W,
-; EG-NEXT:     SETGE_UINT * T5.W, T5.W, T0.Z,
-; EG-NEXT:     AND_INT T2.Y, PV.W, PS,
+; EG-NEXT:     SUB_INT T4.W, T4.W, PS,
+; EG-NEXT:     MULLO_INT * T0.Z, T0.X, T3.W,
+; EG-NEXT:     SUB_INT T1.Y, T1.Z, PS,
 ; EG-NEXT:     ADD_INT T0.Z, T0.Y, 1,
-; EG-NEXT:     SETGE_UINT T1.W, PV.Z, T3.W,
-; EG-NEXT:     SETGE_UINT * T3.W, T1.Y, T1.X,
-; EG-NEXT:     AND_INT T1.Y, PV.W, PS,
-; EG-NEXT:     ADD_INT T1.Z, T0.X, 1,
-; EG-NEXT:     CNDE_INT T1.W, PV.Y, T0.Y, PV.Z,
-; EG-NEXT:     ADD_INT * T6.W, T0.Y, literal.x,
-; EG-NEXT:    -1(nan), 0(0.000000e+00)
-; EG-NEXT:     CNDE_INT T0.Y, T5.W, PS, PV.W,
-; EG-NEXT:     XOR_INT T0.Z, T3.X, T0.W,
-; EG-NEXT:     CNDE_INT T0.W, PV.Y, T0.X, PV.Z,
-; EG-NEXT:     ADD_INT * T1.W, T0.X, literal.x,
-; EG-NEXT:    -1(nan), 0(0.000000e+00)
-; EG-NEXT:     CNDE_INT T1.Z, T3.W, PS, PV.W,
-; EG-NEXT:     XOR_INT T0.W, T4.W, T2.W, BS:VEC_120/SCL_212
+; EG-NEXT:     SETGE_UINT T6.W, PV.W, T1.W,
+; EG-NEXT:     SUB_INT * T7.W, PV.W, T1.W,
+; EG-NEXT:     CNDE_INT T1.X, PV.W, T4.W, PS, BS:VEC_021/SCL_122
+; EG-NEXT:     CNDE_INT T0.Y, PV.W, T0.Y, PV.Z,
+; EG-NEXT:     ADD_INT T0.Z, T0.X, 1,
+; EG-NEXT:     SETGE_UINT T4.W, PV.Y, T3.W,
+; EG-NEXT:     SUB_INT * T6.W, PV.Y, T3.W,
+; EG-NEXT:     CNDE_INT T1.Y, PV.W, T1.Y, PS,
+; EG-NEXT:     CNDE_INT T0.Z, PV.W, T0.X, PV.Z,
+; EG-NEXT:     ADD_INT T4.W, PV.Y, 1,
+; EG-NEXT:     SETGE_UINT * T1.W, PV.X, T1.W,
+; EG-NEXT:     CNDE_INT T0.Y, PS, T0.Y, PV.W,
+; EG-NEXT:     XOR_INT T1.Z, T2.Z, T0.W, BS:VEC_021/SCL_122
+; EG-NEXT:     ADD_INT T0.W, PV.Z, 1,
+; EG-NEXT:     SETGE_UINT * T1.W, PV.Y, T3.W,
+; EG-NEXT:     CNDE_INT T0.Z, PS, T0.Z, PV.W,
+; EG-NEXT:     XOR_INT T0.W, T5.W, T2.W,
 ; EG-NEXT:     XOR_INT * T1.W, PV.Y, PV.Z,
-; EG-NEXT:     SUB_INT T0.Y, PS, T0.Z,
+; EG-NEXT:     SUB_INT T0.Y, PS, T1.Z,
 ; EG-NEXT:     XOR_INT * T1.W, PV.Z, PV.W,
 ; EG-NEXT:     SUB_INT T0.X, PV.W, T0.W,
 ; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
@@ -846,582 +800,520 @@ define amdgpu_kernel void @sdiv_v2i32_4(<2 x i32> addrspace(1)* %out, <2 x i32>
 define amdgpu_kernel void @sdiv_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) {
 ; GCN-LABEL: sdiv_v4i32:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0x9
-; GCN-NEXT:    s_mov_b32 s11, 0xf000
-; GCN-NEXT:    s_mov_b32 s10, -1
-; GCN-NEXT:    s_mov_b32 s2, s10
-; GCN-NEXT:    s_mov_b32 s3, s11
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_mov_b32 s2, s6
+; GCN-NEXT:    s_mov_b32 s3, s7
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s0, s14
-; GCN-NEXT:    s_mov_b32 s1, s15
-; GCN-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
-; GCN-NEXT:    buffer_load_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; GCN-NEXT:    s_mov_b32 s14, 0x4f800000
-; GCN-NEXT:    s_mov_b32 s8, s12
-; GCN-NEXT:    s_mov_b32 s9, s13
-; GCN-NEXT:    s_waitcnt vmcnt(1)
-; GCN-NEXT:    v_ashrrev_i32_e32 v8, 31, v0
+; GCN-NEXT:    s_mov_b32 s0, s10
+; GCN-NEXT:    s_mov_b32 s1, s11
+; GCN-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GCN-NEXT:    s_mov_b32 s10, 0x4f7ffffe
+; GCN-NEXT:    s_mov_b32 s4, s8
+; GCN-NEXT:    s_mov_b32 s5, s9
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_ashrrev_i32_e32 v9, 31, v4
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v9, v4
-; GCN-NEXT:    v_xor_b32_e32 v4, v4, v9
-; GCN-NEXT:    v_xor_b32_e32 v15, v8, v9
-; GCN-NEXT:    v_cvt_f32_u32_e32 v9, v4
-; GCN-NEXT:    v_ashrrev_i32_e32 v11, 31, v5
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v11, v5
+; GCN-NEXT:    v_ashrrev_i32_e32 v8, 31, v0
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v8, v0
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v9, v9
-; GCN-NEXT:    v_xor_b32_e32 v5, v5, v11
 ; GCN-NEXT:    v_xor_b32_e32 v0, v0, v8
-; GCN-NEXT:    v_cvt_f32_u32_e32 v8, v5
-; GCN-NEXT:    v_mul_f32_e32 v9, s14, v9
-; GCN-NEXT:    v_cvt_u32_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_u32_e32 v4, v0
+; GCN-NEXT:    v_ashrrev_i32_e32 v14, 31, v2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GCN-NEXT:    v_mul_f32_e32 v4, s10, v4
+; GCN-NEXT:    v_cvt_u32_f32_e32 v9, v4
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v0
+; GCN-NEXT:    v_mul_lo_u32 v10, v4, v9
+; GCN-NEXT:    buffer_load_dwordx4 v[4:7], off, s[0:3], 0
+; GCN-NEXT:    v_mul_hi_u32 v10, v9, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
 ; GCN-NEXT:    v_ashrrev_i32_e32 v10, 31, v1
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v8, v8
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v10, v1
-; GCN-NEXT:    v_xor_b32_e32 v16, v10, v11
-; GCN-NEXT:    v_xor_b32_e32 v1, v1, v10
-; GCN-NEXT:    v_mul_f32_e32 v8, s14, v8
-; GCN-NEXT:    v_mul_hi_u32 v11, v9, v4
-; GCN-NEXT:    v_mul_lo_u32 v10, v9, v4
-; GCN-NEXT:    v_cvt_u32_f32_e32 v8, v8
-; GCN-NEXT:    v_ashrrev_i32_e32 v12, 31, v2
-; GCN-NEXT:    v_ashrrev_i32_e32 v13, 31, v6
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v12, v2
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v11
-; GCN-NEXT:    v_xor_b32_e32 v17, v12, v13
-; GCN-NEXT:    v_xor_b32_e32 v2, v2, v12
-; GCN-NEXT:    v_sub_i32_e32 v12, vcc, 0, v10
-; GCN-NEXT:    v_cndmask_b32_e64 v10, v10, v12, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v12, v8, v5
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v13, v6
-; GCN-NEXT:    v_xor_b32_e32 v6, v6, v13
-; GCN-NEXT:    v_mul_lo_u32 v11, v8, v5
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v12
-; GCN-NEXT:    v_cvt_f32_u32_e32 v12, v6
-; GCN-NEXT:    v_mul_hi_u32 v10, v10, v9
-; GCN-NEXT:    v_sub_i32_e32 v13, vcc, 0, v11
-; GCN-NEXT:    v_cndmask_b32_e64 v11, v11, v13, s[2:3]
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v12, v12
-; GCN-NEXT:    v_ashrrev_i32_e32 v14, 31, v7
-; GCN-NEXT:    v_add_i32_e32 v7, vcc, v14, v7
-; GCN-NEXT:    v_xor_b32_e32 v7, v7, v14
-; GCN-NEXT:    v_mul_f32_e32 v12, s14, v12
-; GCN-NEXT:    v_cvt_u32_f32_e32 v12, v12
-; GCN-NEXT:    v_mul_hi_u32 v18, v12, v6
-; GCN-NEXT:    v_mul_lo_u32 v13, v12, v6
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v18
-; GCN-NEXT:    v_add_i32_e32 v18, vcc, v10, v9
-; GCN-NEXT:    v_subrev_i32_e32 v9, vcc, v10, v9
-; GCN-NEXT:    v_mul_hi_u32 v10, v11, v8
-; GCN-NEXT:    v_cndmask_b32_e64 v9, v9, v18, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v9, v9, v0
-; GCN-NEXT:    v_sub_i32_e32 v19, vcc, 0, v13
-; GCN-NEXT:    v_add_i32_e32 v11, vcc, v10, v8
-; GCN-NEXT:    v_subrev_i32_e32 v8, vcc, v10, v8
-; GCN-NEXT:    v_cndmask_b32_e64 v13, v13, v19, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e64 v8, v8, v11, s[2:3]
-; GCN-NEXT:    v_mul_hi_u32 v10, v13, v12
-; GCN-NEXT:    v_mul_lo_u32 v11, v9, v4
-; GCN-NEXT:    v_mul_hi_u32 v8, v8, v1
-; GCN-NEXT:    v_add_i32_e32 v13, vcc, v10, v12
-; GCN-NEXT:    v_subrev_i32_e32 v10, vcc, v10, v12
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v0, v11
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v0, v11
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], v0, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v10, v10, v13, s[4:5]
-; GCN-NEXT:    v_mul_lo_u32 v0, v8, v5
-; GCN-NEXT:    v_mul_hi_u32 v4, v10, v2
-; GCN-NEXT:    v_add_i32_e32 v12, vcc, -1, v9
-; GCN-NEXT:    v_add_i32_e32 v10, vcc, -1, v8
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v0
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v1, v0
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[6:7], v0, v5
-; GCN-NEXT:    v_mul_lo_u32 v5, v4, v6
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, 1, v9
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, 1, v8
-; GCN-NEXT:    s_and_b64 vcc, s[2:3], s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GCN-NEXT:    v_sub_i32_e32 v9, vcc, v2, v5
-; GCN-NEXT:    s_and_b64 vcc, s[6:7], s[4:5]
-; GCN-NEXT:    v_cvt_f32_u32_e32 v11, v7
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v12, v1, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v10, v0, s[4:5]
-; GCN-NEXT:    v_xor_b32_e32 v1, v1, v15
-; GCN-NEXT:    v_xor_b32_e32 v8, v0, v16
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v1, v15
-; GCN-NEXT:    v_sub_i32_e32 v1, vcc, v8, v16
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v8, v11
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v9, v6
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], v2, v5
-; GCN-NEXT:    v_ashrrev_i32_e32 v10, 31, v3
-; GCN-NEXT:    v_mul_f32_e32 v8, s14, v8
-; GCN-NEXT:    v_cvt_u32_f32_e32 v8, v8
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v10, v3
-; GCN-NEXT:    v_xor_b32_e32 v3, v3, v10
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, -1, v4
-; GCN-NEXT:    v_mul_lo_u32 v5, v8, v7
-; GCN-NEXT:    v_mul_hi_u32 v9, v8, v7
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v4
-; GCN-NEXT:    v_sub_i32_e32 v11, vcc, 0, v5
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v9
-; GCN-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v5, v5, v8
-; GCN-NEXT:    v_add_i32_e32 v9, vcc, v5, v8
-; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, v5, v8
-; GCN-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[4:5]
-; GCN-NEXT:    v_mul_hi_u32 v5, v5, v3
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[2:3]
-; GCN-NEXT:    v_mul_lo_u32 v4, v5, v7
-; GCN-NEXT:    v_xor_b32_e32 v2, v2, v17
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, v2, v17
-; GCN-NEXT:    v_xor_b32_e32 v6, v10, v14
-; GCN-NEXT:    v_sub_i32_e32 v8, vcc, v3, v4
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v8, v7
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], v3, v4
-; GCN-NEXT:    v_add_i32_e32 v7, vcc, -1, v5
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, 1, v5
-; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[2:3]
-; GCN-NEXT:    v_xor_b32_e32 v3, v3, v6
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, v3, v6
-; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_ashrrev_i32_e32 v11, 31, v4
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, v11, v4
+; GCN-NEXT:    v_xor_b32_e32 v4, v4, v11
+; GCN-NEXT:    v_mul_hi_u32 v9, v4, v9
+; GCN-NEXT:    v_xor_b32_e32 v8, v11, v8
+; GCN-NEXT:    v_mul_lo_u32 v12, v9, v0
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 1, v9
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, v4, v12
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v4, v0
+; GCN-NEXT:    v_sub_i32_e32 v12, vcc, v4, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v9, v9, v13, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 1, v9
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v0
+; GCN-NEXT:    s_mov_b64 s[0:1], vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v10, v1
+; GCN-NEXT:    v_xor_b32_e32 v1, v0, v10
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, v1
+; GCN-NEXT:    v_sub_i32_e32 v13, vcc, 0, v1
+; GCN-NEXT:    v_ashrrev_i32_e32 v4, 31, v5
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v4, v5
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    v_xor_b32_e32 v5, v5, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v9, v9, v12, s[0:1]
+; GCN-NEXT:    v_xor_b32_e32 v4, v4, v10
+; GCN-NEXT:    v_mul_f32_e32 v0, s10, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_ashrrev_i32_e32 v10, 31, v6
+; GCN-NEXT:    v_mul_lo_u32 v13, v13, v0
+; GCN-NEXT:    v_mul_hi_u32 v11, v0, v13
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v11, v0
+; GCN-NEXT:    v_mul_hi_u32 v11, v5, v0
+; GCN-NEXT:    v_xor_b32_e32 v0, v9, v8
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v8, v0
+; GCN-NEXT:    v_mul_lo_u32 v8, v11, v1
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 1, v11
+; GCN-NEXT:    v_sub_i32_e32 v5, vcc, v5, v8
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v5, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v8, v11, v9, s[0:1]
+; GCN-NEXT:    v_sub_i32_e32 v9, vcc, v5, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 1, v8
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v5, v1
+; GCN-NEXT:    s_mov_b64 s[0:1], vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v14, v2
+; GCN-NEXT:    v_xor_b32_e32 v2, v1, v14
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, v2
+; GCN-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v8, v8, v9, s[0:1]
+; GCN-NEXT:    v_ashrrev_i32_e32 v9, 31, v3
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_f32_e32 v1, s10, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_lo_u32 v5, v5, v1
+; GCN-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v5, v1
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v10, v6
+; GCN-NEXT:    v_xor_b32_e32 v5, v5, v10
+; GCN-NEXT:    v_mul_hi_u32 v6, v5, v1
+; GCN-NEXT:    v_xor_b32_e32 v1, v8, v4
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v4, v1
+; GCN-NEXT:    v_xor_b32_e32 v10, v10, v14
+; GCN-NEXT:    v_mul_lo_u32 v4, v6, v2
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 1, v6
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, v5, v4
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v4, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v6, v8, s[0:1]
+; GCN-NEXT:    v_sub_i32_e32 v6, vcc, v4, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 1, v5
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v2
+; GCN-NEXT:    s_mov_b64 s[0:1], vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v9, v3
+; GCN-NEXT:    v_xor_b32_e32 v3, v2, v9
+; GCN-NEXT:    v_cvt_f32_u32_e32 v2, v3
+; GCN-NEXT:    v_sub_i32_e32 v8, vcc, 0, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s[0:1]
+; GCN-NEXT:    v_ashrrev_i32_e32 v4, 31, v7
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, v4, v7
+; GCN-NEXT:    v_xor_b32_e32 v9, v4, v9
+; GCN-NEXT:    v_xor_b32_e32 v4, v7, v4
+; GCN-NEXT:    v_mul_f32_e32 v2, s10, v2
+; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GCN-NEXT:    v_mul_lo_u32 v8, v8, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v2, v8
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v6, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v4, v2
+; GCN-NEXT:    v_xor_b32_e32 v2, v5, v10
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, v10, v2
+; GCN-NEXT:    v_mul_lo_u32 v5, v6, v3
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v6
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, v4, v5
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v4, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v6, v7, s[0:1]
+; GCN-NEXT:    v_sub_i32_e32 v6, vcc, v4, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 1, v5
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
+; GCN-NEXT:    v_xor_b32_e32 v3, v3, v9
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, v9, v3
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; TONGA-LABEL: sdiv_v4i32:
 ; TONGA:       ; %bb.0:
-; TONGA-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0x24
-; TONGA-NEXT:    s_mov_b32 s11, 0xf000
-; TONGA-NEXT:    s_mov_b32 s10, -1
-; TONGA-NEXT:    s_mov_b32 s2, s10
-; TONGA-NEXT:    s_mov_b32 s3, s11
+; TONGA-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x24
+; TONGA-NEXT:    s_mov_b32 s7, 0xf000
+; TONGA-NEXT:    s_mov_b32 s6, -1
+; TONGA-NEXT:    s_mov_b32 s2, s6
+; TONGA-NEXT:    s_mov_b32 s3, s7
 ; TONGA-NEXT:    s_waitcnt lgkmcnt(0)
-; TONGA-NEXT:    s_mov_b32 s0, s14
-; TONGA-NEXT:    s_mov_b32 s1, s15
-; TONGA-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
-; TONGA-NEXT:    buffer_load_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; TONGA-NEXT:    s_mov_b32 s14, 0x4f800000
-; TONGA-NEXT:    s_mov_b32 s8, s12
-; TONGA-NEXT:    s_mov_b32 s9, s13
-; TONGA-NEXT:    s_waitcnt vmcnt(1)
-; TONGA-NEXT:    v_ashrrev_i32_e32 v8, 31, v0
+; TONGA-NEXT:    s_mov_b32 s0, s10
+; TONGA-NEXT:    s_mov_b32 s1, s11
+; TONGA-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; TONGA-NEXT:    s_mov_b32 s10, 0x4f7ffffe
+; TONGA-NEXT:    s_mov_b32 s4, s8
+; TONGA-NEXT:    s_mov_b32 s5, s9
 ; TONGA-NEXT:    s_waitcnt vmcnt(0)
-; TONGA-NEXT:    v_ashrrev_i32_e32 v9, 31, v4
-; TONGA-NEXT:    v_add_u32_e32 v4, vcc, v9, v4
-; TONGA-NEXT:    v_xor_b32_e32 v4, v4, v9
-; TONGA-NEXT:    v_xor_b32_e32 v15, v8, v9
-; TONGA-NEXT:    v_cvt_f32_u32_e32 v9, v4
-; TONGA-NEXT:    v_ashrrev_i32_e32 v11, 31, v5
-; TONGA-NEXT:    v_add_u32_e32 v5, vcc, v11, v5
+; TONGA-NEXT:    v_ashrrev_i32_e32 v8, 31, v0
 ; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v8, v0
-; TONGA-NEXT:    v_rcp_iflag_f32_e32 v9, v9
-; TONGA-NEXT:    v_xor_b32_e32 v5, v5, v11
 ; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v8
-; TONGA-NEXT:    v_cvt_f32_u32_e32 v8, v5
-; TONGA-NEXT:    v_mul_f32_e32 v9, s14, v9
-; TONGA-NEXT:    v_cvt_u32_f32_e32 v9, v9
+; TONGA-NEXT:    v_cvt_f32_u32_e32 v4, v0
+; TONGA-NEXT:    v_ashrrev_i32_e32 v14, 31, v2
+; TONGA-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; TONGA-NEXT:    v_mul_f32_e32 v4, s10, v4
+; TONGA-NEXT:    v_cvt_u32_f32_e32 v9, v4
+; TONGA-NEXT:    v_sub_u32_e32 v4, vcc, 0, v0
+; TONGA-NEXT:    v_mul_lo_u32 v10, v4, v9
+; TONGA-NEXT:    buffer_load_dwordx4 v[4:7], off, s[0:3], 0
+; TONGA-NEXT:    v_mul_hi_u32 v10, v9, v10
+; TONGA-NEXT:    v_add_u32_e32 v9, vcc, v10, v9
 ; TONGA-NEXT:    v_ashrrev_i32_e32 v10, 31, v1
-; TONGA-NEXT:    v_rcp_iflag_f32_e32 v8, v8
-; TONGA-NEXT:    v_add_u32_e32 v1, vcc, v10, v1
-; TONGA-NEXT:    v_xor_b32_e32 v16, v10, v11
-; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v10
-; TONGA-NEXT:    v_mul_f32_e32 v8, s14, v8
-; TONGA-NEXT:    v_mul_hi_u32 v11, v9, v4
-; TONGA-NEXT:    v_mul_lo_u32 v10, v9, v4
-; TONGA-NEXT:    v_cvt_u32_f32_e32 v8, v8
-; TONGA-NEXT:    v_ashrrev_i32_e32 v12, 31, v2
-; TONGA-NEXT:    v_ashrrev_i32_e32 v13, 31, v6
-; TONGA-NEXT:    v_add_u32_e32 v2, vcc, v12, v2
-; TONGA-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v11
-; TONGA-NEXT:    v_xor_b32_e32 v17, v12, v13
-; TONGA-NEXT:    v_xor_b32_e32 v2, v2, v12
-; TONGA-NEXT:    v_sub_u32_e32 v12, vcc, 0, v10
-; TONGA-NEXT:    v_cndmask_b32_e64 v10, v10, v12, s[0:1]
-; TONGA-NEXT:    v_mul_hi_u32 v12, v8, v5
-; TONGA-NEXT:    v_add_u32_e32 v6, vcc, v13, v6
-; TONGA-NEXT:    v_xor_b32_e32 v6, v6, v13
-; TONGA-NEXT:    v_mul_lo_u32 v11, v8, v5
-; TONGA-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v12
-; TONGA-NEXT:    v_cvt_f32_u32_e32 v12, v6
-; TONGA-NEXT:    v_mul_hi_u32 v10, v10, v9
-; TONGA-NEXT:    v_sub_u32_e32 v13, vcc, 0, v11
-; TONGA-NEXT:    v_cndmask_b32_e64 v11, v11, v13, s[2:3]
-; TONGA-NEXT:    v_rcp_iflag_f32_e32 v12, v12
-; TONGA-NEXT:    v_ashrrev_i32_e32 v14, 31, v7
-; TONGA-NEXT:    v_add_u32_e32 v7, vcc, v14, v7
-; TONGA-NEXT:    v_xor_b32_e32 v7, v7, v14
-; TONGA-NEXT:    v_mul_f32_e32 v12, s14, v12
-; TONGA-NEXT:    v_cvt_u32_f32_e32 v12, v12
-; TONGA-NEXT:    v_mul_hi_u32 v18, v12, v6
-; TONGA-NEXT:    v_mul_lo_u32 v13, v12, v6
-; TONGA-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v18
-; TONGA-NEXT:    v_add_u32_e32 v18, vcc, v10, v9
-; TONGA-NEXT:    v_subrev_u32_e32 v9, vcc, v10, v9
-; TONGA-NEXT:    v_mul_hi_u32 v10, v11, v8
-; TONGA-NEXT:    v_cndmask_b32_e64 v9, v9, v18, s[0:1]
-; TONGA-NEXT:    v_mul_hi_u32 v9, v9, v0
-; TONGA-NEXT:    v_sub_u32_e32 v19, vcc, 0, v13
-; TONGA-NEXT:    v_add_u32_e32 v11, vcc, v10, v8
-; TONGA-NEXT:    v_subrev_u32_e32 v8, vcc, v10, v8
-; TONGA-NEXT:    v_cndmask_b32_e64 v13, v13, v19, s[4:5]
-; TONGA-NEXT:    v_cndmask_b32_e64 v8, v8, v11, s[2:3]
-; TONGA-NEXT:    v_mul_hi_u32 v10, v13, v12
-; TONGA-NEXT:    v_mul_lo_u32 v11, v9, v4
-; TONGA-NEXT:    v_mul_hi_u32 v8, v8, v1
-; TONGA-NEXT:    v_add_u32_e32 v13, vcc, v10, v12
-; TONGA-NEXT:    v_subrev_u32_e32 v10, vcc, v10, v12
-; TONGA-NEXT:    v_cmp_ge_u32_e64 s[0:1], v0, v11
-; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v0, v11
-; TONGA-NEXT:    v_cmp_ge_u32_e64 s[2:3], v0, v4
-; TONGA-NEXT:    v_cndmask_b32_e64 v10, v10, v13, s[4:5]
-; TONGA-NEXT:    v_mul_lo_u32 v0, v8, v5
-; TONGA-NEXT:    v_mul_hi_u32 v4, v10, v2
-; TONGA-NEXT:    v_add_u32_e32 v12, vcc, -1, v9
-; TONGA-NEXT:    v_add_u32_e32 v10, vcc, -1, v8
-; TONGA-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v0
-; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v1, v0
-; TONGA-NEXT:    v_cmp_ge_u32_e64 s[6:7], v0, v5
-; TONGA-NEXT:    v_mul_lo_u32 v5, v4, v6
-; TONGA-NEXT:    v_add_u32_e32 v1, vcc, 1, v9
-; TONGA-NEXT:    v_add_u32_e32 v0, vcc, 1, v8
-; TONGA-NEXT:    s_and_b64 vcc, s[2:3], s[0:1]
-; TONGA-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; TONGA-NEXT:    v_sub_u32_e32 v9, vcc, v2, v5
-; TONGA-NEXT:    s_and_b64 vcc, s[6:7], s[4:5]
-; TONGA-NEXT:    v_cvt_f32_u32_e32 v11, v7
-; TONGA-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; TONGA-NEXT:    v_cndmask_b32_e64 v1, v12, v1, s[0:1]
-; TONGA-NEXT:    v_cndmask_b32_e64 v0, v10, v0, s[4:5]
-; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v15
-; TONGA-NEXT:    v_xor_b32_e32 v8, v0, v16
-; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v1, v15
-; TONGA-NEXT:    v_sub_u32_e32 v1, vcc, v8, v16
-; TONGA-NEXT:    v_rcp_iflag_f32_e32 v8, v11
-; TONGA-NEXT:    v_cmp_ge_u32_e64 s[0:1], v9, v6
-; TONGA-NEXT:    v_cmp_ge_u32_e64 s[2:3], v2, v5
-; TONGA-NEXT:    v_ashrrev_i32_e32 v10, 31, v3
-; TONGA-NEXT:    v_mul_f32_e32 v8, s14, v8
-; TONGA-NEXT:    v_cvt_u32_f32_e32 v8, v8
-; TONGA-NEXT:    v_add_u32_e32 v3, vcc, v10, v3
-; TONGA-NEXT:    v_xor_b32_e32 v3, v3, v10
-; TONGA-NEXT:    v_add_u32_e32 v6, vcc, -1, v4
-; TONGA-NEXT:    v_mul_lo_u32 v5, v8, v7
-; TONGA-NEXT:    v_mul_hi_u32 v9, v8, v7
-; TONGA-NEXT:    v_add_u32_e32 v2, vcc, 1, v4
-; TONGA-NEXT:    v_sub_u32_e32 v11, vcc, 0, v5
-; TONGA-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v9
-; TONGA-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[4:5]
-; TONGA-NEXT:    v_mul_hi_u32 v5, v5, v8
-; TONGA-NEXT:    v_add_u32_e32 v9, vcc, v5, v8
-; TONGA-NEXT:    v_subrev_u32_e32 v5, vcc, v5, v8
-; TONGA-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[4:5]
-; TONGA-NEXT:    v_mul_hi_u32 v5, v5, v3
-; TONGA-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; TONGA-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; TONGA-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[2:3]
-; TONGA-NEXT:    v_mul_lo_u32 v4, v5, v7
-; TONGA-NEXT:    v_xor_b32_e32 v2, v2, v17
-; TONGA-NEXT:    v_sub_u32_e32 v2, vcc, v2, v17
-; TONGA-NEXT:    v_xor_b32_e32 v6, v10, v14
-; TONGA-NEXT:    v_sub_u32_e32 v8, vcc, v3, v4
-; TONGA-NEXT:    v_cmp_ge_u32_e64 s[0:1], v8, v7
-; TONGA-NEXT:    v_cmp_ge_u32_e64 s[2:3], v3, v4
-; TONGA-NEXT:    v_add_u32_e32 v7, vcc, -1, v5
-; TONGA-NEXT:    v_add_u32_e32 v3, vcc, 1, v5
-; TONGA-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
-; TONGA-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; TONGA-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[2:3]
-; TONGA-NEXT:    v_xor_b32_e32 v3, v3, v6
-; TONGA-NEXT:    v_sub_u32_e32 v3, vcc, v3, v6
-; TONGA-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
+; TONGA-NEXT:    s_waitcnt vmcnt(0)
+; TONGA-NEXT:    v_ashrrev_i32_e32 v11, 31, v4
+; TONGA-NEXT:    v_add_u32_e32 v4, vcc, v11, v4
+; TONGA-NEXT:    v_xor_b32_e32 v4, v4, v11
+; TONGA-NEXT:    v_mul_hi_u32 v9, v4, v9
+; TONGA-NEXT:    v_xor_b32_e32 v8, v11, v8
+; TONGA-NEXT:    v_mul_lo_u32 v12, v9, v0
+; TONGA-NEXT:    v_add_u32_e32 v13, vcc, 1, v9
+; TONGA-NEXT:    v_sub_u32_e32 v4, vcc, v4, v12
+; TONGA-NEXT:    v_cmp_ge_u32_e64 s[0:1], v4, v0
+; TONGA-NEXT:    v_sub_u32_e32 v12, vcc, v4, v0
+; TONGA-NEXT:    v_cndmask_b32_e64 v9, v9, v13, s[0:1]
+; TONGA-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s[0:1]
+; TONGA-NEXT:    v_add_u32_e32 v12, vcc, 1, v9
+; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v0
+; TONGA-NEXT:    s_mov_b64 s[0:1], vcc
+; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v10, v1
+; TONGA-NEXT:    v_xor_b32_e32 v1, v0, v10
+; TONGA-NEXT:    v_cvt_f32_u32_e32 v0, v1
+; TONGA-NEXT:    v_sub_u32_e32 v13, vcc, 0, v1
+; TONGA-NEXT:    v_ashrrev_i32_e32 v4, 31, v5
+; TONGA-NEXT:    v_add_u32_e32 v5, vcc, v4, v5
+; TONGA-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; TONGA-NEXT:    v_xor_b32_e32 v5, v5, v4
+; TONGA-NEXT:    v_cndmask_b32_e64 v9, v9, v12, s[0:1]
+; TONGA-NEXT:    v_xor_b32_e32 v4, v4, v10
+; TONGA-NEXT:    v_mul_f32_e32 v0, s10, v0
+; TONGA-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; TONGA-NEXT:    v_ashrrev_i32_e32 v10, 31, v6
+; TONGA-NEXT:    v_mul_lo_u32 v13, v13, v0
+; TONGA-NEXT:    v_mul_hi_u32 v11, v0, v13
+; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v11, v0
+; TONGA-NEXT:    v_mul_hi_u32 v11, v5, v0
+; TONGA-NEXT:    v_xor_b32_e32 v0, v9, v8
+; TONGA-NEXT:    v_subrev_u32_e32 v0, vcc, v8, v0
+; TONGA-NEXT:    v_mul_lo_u32 v8, v11, v1
+; TONGA-NEXT:    v_add_u32_e32 v9, vcc, 1, v11
+; TONGA-NEXT:    v_sub_u32_e32 v5, vcc, v5, v8
+; TONGA-NEXT:    v_cmp_ge_u32_e64 s[0:1], v5, v1
+; TONGA-NEXT:    v_cndmask_b32_e64 v8, v11, v9, s[0:1]
+; TONGA-NEXT:    v_sub_u32_e32 v9, vcc, v5, v1
+; TONGA-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[0:1]
+; TONGA-NEXT:    v_add_u32_e32 v9, vcc, 1, v8
+; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v5, v1
+; TONGA-NEXT:    s_mov_b64 s[0:1], vcc
+; TONGA-NEXT:    v_add_u32_e32 v1, vcc, v14, v2
+; TONGA-NEXT:    v_xor_b32_e32 v2, v1, v14
+; TONGA-NEXT:    v_cvt_f32_u32_e32 v1, v2
+; TONGA-NEXT:    v_sub_u32_e32 v5, vcc, 0, v2
+; TONGA-NEXT:    v_cndmask_b32_e64 v8, v8, v9, s[0:1]
+; TONGA-NEXT:    v_ashrrev_i32_e32 v9, 31, v3
+; TONGA-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; TONGA-NEXT:    v_mul_f32_e32 v1, s10, v1
+; TONGA-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; TONGA-NEXT:    v_mul_lo_u32 v5, v5, v1
+; TONGA-NEXT:    v_mul_hi_u32 v5, v1, v5
+; TONGA-NEXT:    v_add_u32_e32 v1, vcc, v5, v1
+; TONGA-NEXT:    v_add_u32_e32 v5, vcc, v10, v6
+; TONGA-NEXT:    v_xor_b32_e32 v5, v5, v10
+; TONGA-NEXT:    v_mul_hi_u32 v6, v5, v1
+; TONGA-NEXT:    v_xor_b32_e32 v1, v8, v4
+; TONGA-NEXT:    v_subrev_u32_e32 v1, vcc, v4, v1
+; TONGA-NEXT:    v_xor_b32_e32 v10, v10, v14
+; TONGA-NEXT:    v_mul_lo_u32 v4, v6, v2
+; TONGA-NEXT:    v_add_u32_e32 v8, vcc, 1, v6
+; TONGA-NEXT:    v_sub_u32_e32 v4, vcc, v5, v4
+; TONGA-NEXT:    v_cmp_ge_u32_e64 s[0:1], v4, v2
+; TONGA-NEXT:    v_cndmask_b32_e64 v5, v6, v8, s[0:1]
+; TONGA-NEXT:    v_sub_u32_e32 v6, vcc, v4, v2
+; TONGA-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[0:1]
+; TONGA-NEXT:    v_add_u32_e32 v6, vcc, 1, v5
+; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v2
+; TONGA-NEXT:    s_mov_b64 s[0:1], vcc
+; TONGA-NEXT:    v_add_u32_e32 v2, vcc, v9, v3
+; TONGA-NEXT:    v_xor_b32_e32 v3, v2, v9
+; TONGA-NEXT:    v_cvt_f32_u32_e32 v2, v3
+; TONGA-NEXT:    v_sub_u32_e32 v8, vcc, 0, v3
+; TONGA-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s[0:1]
+; TONGA-NEXT:    v_ashrrev_i32_e32 v4, 31, v7
+; TONGA-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; TONGA-NEXT:    v_add_u32_e32 v7, vcc, v4, v7
+; TONGA-NEXT:    v_xor_b32_e32 v9, v4, v9
+; TONGA-NEXT:    v_xor_b32_e32 v4, v7, v4
+; TONGA-NEXT:    v_mul_f32_e32 v2, s10, v2
+; TONGA-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; TONGA-NEXT:    v_mul_lo_u32 v8, v8, v2
+; TONGA-NEXT:    v_mul_hi_u32 v6, v2, v8
+; TONGA-NEXT:    v_add_u32_e32 v2, vcc, v6, v2
+; TONGA-NEXT:    v_mul_hi_u32 v6, v4, v2
+; TONGA-NEXT:    v_xor_b32_e32 v2, v5, v10
+; TONGA-NEXT:    v_subrev_u32_e32 v2, vcc, v10, v2
+; TONGA-NEXT:    v_mul_lo_u32 v5, v6, v3
+; TONGA-NEXT:    v_add_u32_e32 v7, vcc, 1, v6
+; TONGA-NEXT:    v_sub_u32_e32 v4, vcc, v4, v5
+; TONGA-NEXT:    v_cmp_ge_u32_e64 s[0:1], v4, v3
+; TONGA-NEXT:    v_cndmask_b32_e64 v5, v6, v7, s[0:1]
+; TONGA-NEXT:    v_sub_u32_e32 v6, vcc, v4, v3
+; TONGA-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[0:1]
+; TONGA-NEXT:    v_add_u32_e32 v6, vcc, 1, v5
+; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v3
+; TONGA-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
+; TONGA-NEXT:    v_xor_b32_e32 v3, v3, v9
+; TONGA-NEXT:    v_subrev_u32_e32 v3, vcc, v9, v3
+; TONGA-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; TONGA-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: sdiv_v4i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x24
-; GFX9-NEXT:    s_mov_b32 s15, 0xf000
-; GFX9-NEXT:    s_mov_b32 s14, -1
-; GFX9-NEXT:    s_mov_b32 s2, s14
-; GFX9-NEXT:    s_mov_b32 s3, s15
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b32 s11, 0xf000
+; GFX9-NEXT:    s_mov_b32 s10, -1
+; GFX9-NEXT:    s_mov_b32 s4, 0x4f7ffffe
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s0, s10
-; GFX9-NEXT:    s_mov_b32 s1, s11
+; GFX9-NEXT:    s_mov_b32 s8, s0
+; GFX9-NEXT:    s_mov_b32 s9, s1
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s10
+; GFX9-NEXT:    s_mov_b32 s3, s11
 ; GFX9-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
 ; GFX9-NEXT:    buffer_load_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; GFX9-NEXT:    s_mov_b32 s4, 0x4f800000
-; GFX9-NEXT:    s_mov_b32 s12, s8
-; GFX9-NEXT:    s_mov_b32 s13, s9
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v8, 31, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v9, 31, v4
 ; GFX9-NEXT:    v_add_u32_e32 v4, v4, v9
+; GFX9-NEXT:    v_ashrrev_i32_e32 v11, 31, v5
 ; GFX9-NEXT:    v_add_u32_e32 v0, v0, v8
 ; GFX9-NEXT:    v_xor_b32_e32 v4, v4, v9
+; GFX9-NEXT:    v_ashrrev_i32_e32 v10, 31, v1
+; GFX9-NEXT:    v_ashrrev_i32_e32 v13, 31, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, v5, v11
 ; GFX9-NEXT:    v_xor_b32_e32 v16, v8, v9
 ; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v8
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v8, v4
-; GFX9-NEXT:    v_ashrrev_i32_e32 v11, 31, v5
-; GFX9-NEXT:    v_add_u32_e32 v5, v5, v11
-; GFX9-NEXT:    v_xor_b32_e32 v5, v5, v11
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v9, v5
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v8, v8
-; GFX9-NEXT:    v_ashrrev_i32_e32 v13, 31, v6
-; GFX9-NEXT:    v_ashrrev_i32_e32 v10, 31, v1
-; GFX9-NEXT:    v_add_u32_e32 v6, v6, v13
+; GFX9-NEXT:    v_ashrrev_i32_e32 v12, 31, v2
+; GFX9-NEXT:    v_ashrrev_i32_e32 v15, 31, v7
 ; GFX9-NEXT:    v_add_u32_e32 v1, v1, v10
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v9, v9
-; GFX9-NEXT:    v_mul_f32_e32 v8, s4, v8
-; GFX9-NEXT:    v_xor_b32_e32 v6, v6, v13
+; GFX9-NEXT:    v_add_u32_e32 v6, v6, v13
+; GFX9-NEXT:    v_xor_b32_e32 v5, v5, v11
+; GFX9-NEXT:    v_ashrrev_i32_e32 v14, 31, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v12
+; GFX9-NEXT:    v_add_u32_e32 v7, v7, v15
 ; GFX9-NEXT:    v_xor_b32_e32 v17, v10, v11
 ; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v10
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v10, v6
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v8, v8
-; GFX9-NEXT:    v_ashrrev_i32_e32 v12, 31, v2
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v12
-; GFX9-NEXT:    v_mul_f32_e32 v9, s4, v9
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v10, v10
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v10, v5
+; GFX9-NEXT:    v_xor_b32_e32 v6, v6, v13
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v14
 ; GFX9-NEXT:    v_xor_b32_e32 v18, v12, v13
 ; GFX9-NEXT:    v_xor_b32_e32 v2, v2, v12
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v9, v9
-; GFX9-NEXT:    v_mul_hi_u32 v12, v8, v4
-; GFX9-NEXT:    v_mul_lo_u32 v11, v8, v4
-; GFX9-NEXT:    v_mul_f32_e32 v10, s4, v10
-; GFX9-NEXT:    v_mul_lo_u32 v13, v9, v5
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v12
-; GFX9-NEXT:    v_mul_hi_u32 v12, v9, v5
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v10, v10
-; GFX9-NEXT:    v_sub_u32_e32 v19, 0, v11
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v11, v19, vcc
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v12
-; GFX9-NEXT:    v_sub_u32_e32 v19, 0, v13
-; GFX9-NEXT:    v_cndmask_b32_e64 v13, v13, v19, s[0:1]
-; GFX9-NEXT:    v_mul_hi_u32 v19, v10, v6
-; GFX9-NEXT:    v_ashrrev_i32_e32 v15, 31, v7
-; GFX9-NEXT:    v_add_u32_e32 v7, v7, v15
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v12, v6
 ; GFX9-NEXT:    v_xor_b32_e32 v7, v7, v15
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v19
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v19, v7
-; GFX9-NEXT:    v_mul_hi_u32 v11, v11, v8
-; GFX9-NEXT:    v_mul_lo_u32 v12, v10, v6
-; GFX9-NEXT:    v_ashrrev_i32_e32 v14, 31, v3
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v19, v19
-; GFX9-NEXT:    v_add_u32_e32 v3, v3, v14
-; GFX9-NEXT:    v_sub_u32_e32 v20, 0, v12
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, v12, v20, s[2:3]
-; GFX9-NEXT:    v_mul_f32_e32 v19, s4, v19
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v19, v19
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v8, v8
+; GFX9-NEXT:    v_xor_b32_e32 v19, v14, v15
 ; GFX9-NEXT:    v_xor_b32_e32 v3, v3, v14
-; GFX9-NEXT:    v_mul_hi_u32 v21, v19, v7
-; GFX9-NEXT:    v_mul_lo_u32 v20, v19, v7
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v21
-; GFX9-NEXT:    v_add_u32_e32 v21, v8, v11
-; GFX9-NEXT:    v_sub_u32_e32 v8, v8, v11
-; GFX9-NEXT:    v_mul_hi_u32 v11, v13, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v8, v21, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v8, v8, v0
-; GFX9-NEXT:    v_sub_u32_e32 v22, 0, v20
-; GFX9-NEXT:    v_add_u32_e32 v13, v9, v11
-; GFX9-NEXT:    v_sub_u32_e32 v9, v9, v11
-; GFX9-NEXT:    v_mul_hi_u32 v11, v12, v10
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v9, v13, s[0:1]
-; GFX9-NEXT:    v_mul_hi_u32 v9, v9, v1
-; GFX9-NEXT:    v_cndmask_b32_e64 v20, v20, v22, s[4:5]
-; GFX9-NEXT:    v_add_u32_e32 v12, v10, v11
-; GFX9-NEXT:    v_sub_u32_e32 v10, v10, v11
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v10, v12, s[2:3]
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v14, v7
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v10, v10
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v12, v12
+; GFX9-NEXT:    v_mul_f32_e32 v8, s4, v8
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v14, v14
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; GFX9-NEXT:    v_mul_f32_e32 v10, s4, v10
+; GFX9-NEXT:    v_mul_f32_e32 v12, s4, v12
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v10, v10
+; GFX9-NEXT:    v_sub_u32_e32 v9, 0, v4
+; GFX9-NEXT:    v_mul_f32_e32 v14, s4, v14
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v12, v12
+; GFX9-NEXT:    v_mul_lo_u32 v9, v9, v8
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v14, v14
+; GFX9-NEXT:    v_sub_u32_e32 v11, 0, v5
+; GFX9-NEXT:    v_sub_u32_e32 v13, 0, v6
+; GFX9-NEXT:    v_mul_lo_u32 v11, v11, v10
+; GFX9-NEXT:    v_sub_u32_e32 v15, 0, v7
+; GFX9-NEXT:    v_mul_lo_u32 v13, v13, v12
+; GFX9-NEXT:    v_mul_lo_u32 v15, v15, v14
+; GFX9-NEXT:    v_mul_hi_u32 v9, v8, v9
+; GFX9-NEXT:    v_mul_hi_u32 v11, v10, v11
+; GFX9-NEXT:    v_mul_hi_u32 v13, v12, v13
+; GFX9-NEXT:    v_mul_hi_u32 v15, v14, v15
+; GFX9-NEXT:    v_add_u32_e32 v8, v8, v9
+; GFX9-NEXT:    v_mul_hi_u32 v8, v0, v8
+; GFX9-NEXT:    v_add_u32_e32 v9, v10, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, v12, v13
+; GFX9-NEXT:    v_mul_hi_u32 v9, v1, v9
+; GFX9-NEXT:    v_add_u32_e32 v11, v14, v15
+; GFX9-NEXT:    v_mul_hi_u32 v10, v2, v10
 ; GFX9-NEXT:    v_mul_lo_u32 v12, v8, v4
-; GFX9-NEXT:    v_mul_hi_u32 v11, v20, v19
-; GFX9-NEXT:    v_mul_hi_u32 v10, v10, v2
-; GFX9-NEXT:    v_add_u32_e32 v13, 1, v8
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v12
+; GFX9-NEXT:    v_mul_hi_u32 v11, v3, v11
+; GFX9-NEXT:    v_mul_lo_u32 v14, v9, v5
+; GFX9-NEXT:    v_mul_lo_u32 v15, v10, v6
 ; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v12
-; GFX9-NEXT:    v_mul_lo_u32 v12, v9, v5
-; GFX9-NEXT:    v_add_u32_e32 v20, v19, v11
-; GFX9-NEXT:    v_sub_u32_e32 v11, v19, v11
-; GFX9-NEXT:    v_cndmask_b32_e64 v11, v11, v20, s[4:5]
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[2:3], v1, v12
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v12
-; GFX9-NEXT:    v_mul_lo_u32 v12, v10, v6
-; GFX9-NEXT:    v_mul_hi_u32 v11, v11, v3
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[0:1], v0, v4
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v5
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[6:7], v2, v12
-; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v12
-; GFX9-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[8:9], v2, v6
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v8, v13, s[0:1]
-; GFX9-NEXT:    v_add_u32_e32 v0, 1, v9
-; GFX9-NEXT:    s_and_b64 s[0:1], s[4:5], s[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v9, v0, s[0:1]
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v10
-; GFX9-NEXT:    s_and_b64 s[0:1], s[8:9], s[6:7]
 ; GFX9-NEXT:    v_mul_lo_u32 v12, v11, v7
-; GFX9-NEXT:    v_add_u32_e32 v19, -1, v8
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v10, v1, s[0:1]
-; GFX9-NEXT:    v_add_u32_e32 v5, -1, v10
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
-; GFX9-NEXT:    v_add_u32_e32 v4, -1, v9
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[2:3]
-; GFX9-NEXT:    v_xor_b32_e32 v2, v2, v16
-; GFX9-NEXT:    v_xor_b32_e32 v5, v1, v18
-; GFX9-NEXT:    v_xor_b32_e32 v4, v0, v17
-; GFX9-NEXT:    v_sub_u32_e32 v0, v2, v16
-; GFX9-NEXT:    v_sub_u32_e32 v2, v5, v18
-; GFX9-NEXT:    v_sub_u32_e32 v5, v3, v12
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v5, v7
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[0:1], v3, v12
-; GFX9-NEXT:    v_add_u32_e32 v3, 1, v11
-; GFX9-NEXT:    s_and_b64 vcc, vcc, s[0:1]
-; GFX9-NEXT:    v_add_u32_e32 v5, -1, v11
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v11, v3, vcc
-; GFX9-NEXT:    v_sub_u32_e32 v1, v4, v17
-; GFX9-NEXT:    v_xor_b32_e32 v4, v14, v15
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[0:1]
-; GFX9-NEXT:    v_xor_b32_e32 v3, v3, v4
-; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v4
-; GFX9-NEXT:    buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 1, v8
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v15
+; GFX9-NEXT:    v_cndmask_b32_e32 v8, v8, v13, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v13, v0, v4
+; GFX9-NEXT:    v_add_u32_e32 v14, 1, v9
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[0:1], v1, v5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v13, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v12
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, v9, v14, s[0:1]
+; GFX9-NEXT:    v_sub_u32_e32 v14, v1, v5
+; GFX9-NEXT:    v_add_u32_e32 v15, 1, v10
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[2:3], v2, v6
+; GFX9-NEXT:    v_cndmask_b32_e64 v10, v10, v15, s[2:3]
+; GFX9-NEXT:    v_sub_u32_e32 v15, v2, v6
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v14, s[0:1]
+; GFX9-NEXT:    v_add_u32_e32 v12, 1, v11
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v3, v7
+; GFX9-NEXT:    v_add_u32_e32 v13, 1, v8
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v8, v13, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v11, v11, v12, s[4:5]
+; GFX9-NEXT:    v_sub_u32_e32 v12, v3, v7
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v15, s[2:3]
+; GFX9-NEXT:    v_add_u32_e32 v14, 1, v9
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v5
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v14, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v12, s[4:5]
+; GFX9-NEXT:    v_add_u32_e32 v15, 1, v10
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v6
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v10, v15, vcc
+; GFX9-NEXT:    v_add_u32_e32 v12, 1, v11
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v7
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v11, v12, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v16
+; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v17
+; GFX9-NEXT:    v_xor_b32_e32 v2, v2, v18
+; GFX9-NEXT:    v_xor_b32_e32 v3, v3, v19
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v16
+; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v17
+; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v18
+; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v19
+; GFX9-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
 ; GFX9-NEXT:    s_endpgm
 ;
 ; EG-LABEL: sdiv_v4i32:
 ; EG:       ; %bb.0:
-; EG-NEXT:    ALU 0, @12, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    TEX 0 @8
-; EG-NEXT:    ALU 2, @13, KC0[], KC1[]
-; EG-NEXT:    TEX 0 @10
-; EG-NEXT:    ALU 114, @16, KC0[CB0:0-32], KC1[]
-; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T3.XYZW, T0.X, 1
+; EG-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @6
+; EG-NEXT:    ALU 101, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T2.XYZW, T0.X, 1
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
-; EG-NEXT:    Fetch clause starting at 8:
+; EG-NEXT:    Fetch clause starting at 6:
 ; EG-NEXT:     VTX_READ_128 T1.XYZW, T0.X, 16, #1
-; EG-NEXT:    Fetch clause starting at 10:
-; EG-NEXT:     VTX_READ_128 T3.XYZW, T0.X, 0, #1
-; EG-NEXT:    ALU clause starting at 12:
+; EG-NEXT:     VTX_READ_128 T0.XYZW, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 10:
 ; EG-NEXT:     MOV * T0.X, KC0[2].Z,
-; EG-NEXT:    ALU clause starting at 13:
-; EG-NEXT:     SETGT_INT * T0.W, 0.0, T1.Z,
-; EG-NEXT:     ADD_INT * T2.W, T1.Z, PV.W,
-; EG-NEXT:     XOR_INT * T2.W, PV.W, T0.W,
-; EG-NEXT:    ALU clause starting at 16:
-; EG-NEXT:     RECIP_UINT * T0.X, T2.W,
-; EG-NEXT:     MULLO_INT * T0.Y, PS, T2.W,
-; EG-NEXT:     SUB_INT T4.W, 0.0, PS,
-; EG-NEXT:     MULHI * T0.Z, T0.X, T2.W,
-; EG-NEXT:     CNDE_INT T4.W, PS, PV.W, T0.Y,
-; EG-NEXT:     SETGT_INT * T5.W, 0.0, T3.Z,
-; EG-NEXT:     MULHI * T0.Y, PV.W, T0.X,
-; EG-NEXT:     SETGT_INT T2.Y, 0.0, T1.W,
-; EG-NEXT:     ADD_INT T1.Z, T3.Z, T5.W, BS:VEC_021/SCL_122
-; EG-NEXT:     ADD_INT T4.W, T0.X, PS,
-; EG-NEXT:     SUB_INT * T6.W, T0.X, PS,
-; EG-NEXT:     CNDE_INT T0.Z, T0.Z, PV.W, PS,
-; EG-NEXT:     XOR_INT T4.W, PV.Z, T5.W,
-; EG-NEXT:     ADD_INT * T1.W, T1.W, PV.Y,
-; EG-NEXT:     XOR_INT T1.W, PS, T2.Y,
-; EG-NEXT:     MULHI * T0.X, PV.Z, PV.W,
-; EG-NEXT:     SETGT_INT T6.W, 0.0, T1.Y,
-; EG-NEXT:     RECIP_UINT * T0.Y, PV.W,
-; EG-NEXT:     ADD_INT T7.W, T1.Y, PV.W,
-; EG-NEXT:     MULLO_INT * T0.Z, PS, T1.W,
-; EG-NEXT:     XOR_INT T1.Z, PV.W, T6.W, BS:VEC_021/SCL_122
-; EG-NEXT:     SUB_INT T7.W, 0.0, PS,
-; EG-NEXT:     MULHI * T1.Y, T0.Y, T1.W,
-; EG-NEXT:     CNDE_INT T7.W, PS, PV.W, T0.Z,
-; EG-NEXT:     RECIP_UINT * T0.Z, PV.Z,
-; EG-NEXT:     SETGT_INT T8.W, 0.0, T3.W,
-; EG-NEXT:     MULHI * T2.X, PV.W, T0.Y,
-; EG-NEXT:     ADD_INT T4.Y, T3.W, PV.W,
-; EG-NEXT:     ADD_INT T2.Z, T0.Y, PS,
-; EG-NEXT:     SUB_INT T3.W, T0.Y, PS,
-; EG-NEXT:     MULLO_INT * T0.Y, T0.Z, T1.Z,
-; EG-NEXT:     CNDE_INT T2.X, T1.Y, PV.Z, PV.W,
-; EG-NEXT:     XOR_INT T1.Y, PV.Y, T8.W,
-; EG-NEXT:     SETGT_INT T2.Z, 0.0, T1.X,
-; EG-NEXT:     SUB_INT T3.W, 0.0, PS,
-; EG-NEXT:     MULHI * T3.Z, T0.Z, T1.Z,
-; EG-NEXT:     CNDE_INT T4.Z, PS, PV.W, T0.Y,
-; EG-NEXT:     ADD_INT T3.W, T1.X, PV.Z,
-; EG-NEXT:     MULHI * T0.Y, PV.X, PV.Y,
-; EG-NEXT:     XOR_INT T3.W, PV.W, T2.Z, BS:VEC_021/SCL_122
-; EG-NEXT:     MULHI * T1.X, PV.Z, T0.Z,
+; EG-NEXT:    ALU clause starting at 11:
+; EG-NEXT:     SETGT_INT * T2.W, 0.0, T1.W,
+; EG-NEXT:     ADD_INT * T1.W, T1.W, PV.W,
+; EG-NEXT:     XOR_INT * T1.W, PV.W, T2.W,
+; EG-NEXT:     SUB_INT T3.W, 0.0, PV.W,
+; EG-NEXT:     RECIP_UINT * T2.X, PV.W,
+; EG-NEXT:     SETGT_INT T4.W, 0.0, T0.W,
+; EG-NEXT:     MULLO_INT * T2.Y, PV.W, PS,
+; EG-NEXT:     SETGT_INT T2.Z, 0.0, T1.Y,
+; EG-NEXT:     ADD_INT T0.W, T0.W, PV.W,
+; EG-NEXT:     MULHI * T2.Y, T2.X, PS,
+; EG-NEXT:     ADD_INT T3.Z, T2.X, PS,
+; EG-NEXT:     XOR_INT T0.W, PV.W, T4.W,
+; EG-NEXT:     ADD_INT * T3.W, T1.Y, PV.Z,
+; EG-NEXT:     XOR_INT T3.W, PS, T2.Z,
+; EG-NEXT:     MULHI * T1.Y, PV.W, PV.Z,
+; EG-NEXT:     SUB_INT T5.W, 0.0, PV.W,
 ; EG-NEXT:     RECIP_UINT * T2.X, PV.W,
-; EG-NEXT:     MULLO_INT * T4.X, PS, T3.W,
-; EG-NEXT:     SETGT_INT T4.Z, 0.0, T3.Y,
-; EG-NEXT:     SUB_INT T7.W, 0.0, PS,
-; EG-NEXT:     MULHI * T4.Y, T2.X, T3.W,
-; EG-NEXT:     CNDE_INT T4.X, PS, PV.W, T4.X,
-; EG-NEXT:     ADD_INT T3.Y, T3.Y, PV.Z,
-; EG-NEXT:     ADD_INT T5.Z, T0.Z, T1.X,
-; EG-NEXT:     SUB_INT T7.W, T0.Z, T1.X,
-; EG-NEXT:     MULLO_INT * T0.Z, T0.Y, T1.W,
-; EG-NEXT:     CNDE_INT T5.Y, T3.Z, PV.Z, PV.W,
-; EG-NEXT:     XOR_INT T3.Z, PV.Y, T4.Z,
-; EG-NEXT:     SUB_INT T7.W, T1.Y, PS,
-; EG-NEXT:     MULHI * T1.X, PV.X, T2.X,
-; EG-NEXT:     SETGE_UINT T5.Z, PV.W, T1.W,
-; EG-NEXT:     SETGE_UINT T1.W, T1.Y, T0.Z,
-; EG-NEXT:     MULHI * T0.Z, PV.Y, PV.Z,
-; EG-NEXT:     AND_INT T1.Y, PV.Z, PV.W,
-; EG-NEXT:     ADD_INT T5.Z, T0.Y, 1,
-; EG-NEXT:     SETGT_INT T7.W, 0.0, T3.X,
-; EG-NEXT:     MULLO_INT * T3.Y, PS, T1.Z,
-; EG-NEXT:     SUB_INT T4.X, T3.Z, PS,
-; EG-NEXT:     ADD_INT T5.Y, T3.X, PV.W,
-; EG-NEXT:     ADD_INT T6.Z, T2.X, T1.X, BS:VEC_120/SCL_212
-; EG-NEXT:     SUB_INT * T9.W, T2.X, T1.X, BS:VEC_120/SCL_212
-; EG-NEXT:     MULLO_INT * T1.X, T0.X, T2.W,
-; EG-NEXT:     CNDE_INT T2.X, T4.Y, T6.Z, T9.W,
-; EG-NEXT:     XOR_INT T4.Y, T5.Y, T7.W, BS:VEC_201
-; EG-NEXT:     SUB_INT T6.Z, T4.W, PS, BS:VEC_120/SCL_212
-; EG-NEXT:     SETGE_UINT T9.W, T4.X, T1.Z, BS:VEC_102/SCL_221
-; EG-NEXT:     SETGE_UINT * T10.W, T3.Z, T3.Y,
-; EG-NEXT:     AND_INT T3.X, PV.W, PS,
-; EG-NEXT:     ADD_INT T3.Y, T0.Z, 1,
-; EG-NEXT:     SETGE_UINT T1.Z, PV.Z, T2.W,
-; EG-NEXT:     SETGE_UINT T2.W, T4.W, T1.X,
-; EG-NEXT:     MULHI * T1.X, PV.X, PV.Y,
-; EG-NEXT:     AND_INT T2.X, PV.Z, PV.W,
-; EG-NEXT:     ADD_INT T5.Y, T0.X, 1,
-; EG-NEXT:     CNDE_INT T1.Z, PV.X, T0.Z, PV.Y,
-; EG-NEXT:     ADD_INT T4.W, T0.Z, literal.x,
-; EG-NEXT:     MULLO_INT * T0.Z, PS, T3.W,
-; EG-NEXT:    -1(nan), 0(0.000000e+00)
-; EG-NEXT:     CNDE_INT T3.X, T10.W, PV.W, PV.Z,
-; EG-NEXT:     CNDE_INT T3.Y, PV.X, T0.X, PV.Y,
-; EG-NEXT:     CNDE_INT T1.Z, T1.Y, T0.Y, T5.Z,
-; EG-NEXT:     ADD_INT T4.W, T0.Y, literal.x, BS:VEC_120/SCL_212
-; EG-NEXT:     SUB_INT * T9.W, T4.Y, PS,
-; EG-NEXT:    -1(nan), 0(0.000000e+00)
-; EG-NEXT:     ADD_INT T0.X, T0.X, literal.x,
-; EG-NEXT:     SETGE_UINT T0.Y, PS, T3.W,
-; EG-NEXT:     SETGE_UINT T0.Z, T4.Y, T0.Z,
-; EG-NEXT:     CNDE_INT T1.W, T1.W, PV.W, PV.Z,
-; EG-NEXT:     XOR_INT * T3.W, T8.W, T2.Y,
-; EG-NEXT:    -1(nan), 0(0.000000e+00)
+; EG-NEXT:     SETGT_INT T6.W, 0.0, T0.Y,
+; EG-NEXT:     MULLO_INT * T2.Y, PV.W, PS,
+; EG-NEXT:     ADD_INT T5.W, T0.Y, PV.W,
+; EG-NEXT:     MULHI * T0.Y, T2.X, PS,
+; EG-NEXT:     ADD_INT T0.Y, T2.X, PS,
+; EG-NEXT:     XOR_INT T3.Z, PV.W, T6.W, BS:VEC_021/SCL_122
+; EG-NEXT:     SETGT_INT T5.W, 0.0, T1.Z,
+; EG-NEXT:     MULLO_INT * T2.X, T1.Y, T1.W,
+; EG-NEXT:     ADD_INT T7.W, T1.Z, PV.W,
+; EG-NEXT:     MULHI * T0.Y, PV.Z, PV.Y,
+; EG-NEXT:     XOR_INT T7.W, PV.W, T5.W, BS:VEC_021/SCL_122
+; EG-NEXT:     MULLO_INT * T1.Z, PS, T3.W,
+; EG-NEXT:     SUB_INT T4.Z, 0.0, PV.W,
+; EG-NEXT:     SETGT_INT T8.W, 0.0, T1.X,
+; EG-NEXT:     RECIP_UINT * T2.Y, PV.W,
+; EG-NEXT:     ADD_INT T9.W, T1.X, PV.W,
+; EG-NEXT:     MULLO_INT * T1.X, PV.Z, PS,
+; EG-NEXT:     SETGT_INT T4.Z, 0.0, T0.Z,
+; EG-NEXT:     XOR_INT T9.W, PV.W, T8.W,
+; EG-NEXT:     MULHI * T1.X, T2.Y, PS,
+; EG-NEXT:     ADD_INT T1.X, T2.Y, PS,
+; EG-NEXT:     SUB_INT T2.Y, 0.0, PV.W,
+; EG-NEXT:     SUB_INT T1.Z, T3.Z, T1.Z,
+; EG-NEXT:     ADD_INT T10.W, T0.Z, PV.Z, BS:VEC_201
+; EG-NEXT:     RECIP_UINT * T0.Z, PV.W,
+; EG-NEXT:     XOR_INT T3.X, PV.W, T4.Z,
+; EG-NEXT:     ADD_INT T3.Y, T0.Y, 1,
+; EG-NEXT:     SETGE_UINT T3.Z, PV.Z, T3.W,
+; EG-NEXT:     SUB_INT T10.W, PV.Z, T3.W,
+; EG-NEXT:     MULLO_INT * T2.Y, PV.Y, PS,
+; EG-NEXT:     CNDE_INT T1.Z, PV.Z, T1.Z, PV.W,
+; EG-NEXT:     CNDE_INT T10.W, PV.Z, T0.Y, PV.Y,
+; EG-NEXT:     MULHI * T0.Y, PV.X, T1.X,
+; EG-NEXT:     SETGT_INT T3.Y, 0.0, T0.X,
+; EG-NEXT:     ADD_INT T3.Z, PV.W, 1,
+; EG-NEXT:     SETGE_UINT T3.W, PV.Z, T3.W, BS:VEC_021/SCL_122
+; EG-NEXT:     MULLO_INT * T1.X, PS, T7.W,
+; EG-NEXT:     CNDE_INT T4.Y, PV.W, T10.W, PV.Z,
+; EG-NEXT:     ADD_INT T1.Z, T0.X, PV.Y,
+; EG-NEXT:     SUB_INT T3.W, T3.X, PS, BS:VEC_120/SCL_212
+; EG-NEXT:     MULHI * T0.X, T0.Z, T2.Y,
+; EG-NEXT:     ADD_INT T1.X, T0.Y, 1,
+; EG-NEXT:     SETGE_UINT T2.Y, PV.W, T7.W,
+; EG-NEXT:     ADD_INT T0.Z, T0.Z, PS,
+; EG-NEXT:     XOR_INT T10.W, PV.Z, T3.Y,
+; EG-NEXT:     SUB_INT * T0.W, T0.W, T2.X,
+; EG-NEXT:     SUB_INT T0.X, T3.W, T7.W,
+; EG-NEXT:     ADD_INT T5.Y, T1.Y, 1,
+; EG-NEXT:     SETGE_UINT T1.Z, PS, T1.W, BS:VEC_021/SCL_122
+; EG-NEXT:     SUB_INT T11.W, PS, T1.W, BS:VEC_021/SCL_122
+; EG-NEXT:     MULHI * T0.Z, PV.W, PV.Z,
+; EG-NEXT:     CNDE_INT T2.X, PV.Z, T0.W, PV.W, BS:VEC_021/SCL_122
+; EG-NEXT:     CNDE_INT T1.Y, PV.Z, T1.Y, PV.Y,
+; EG-NEXT:     CNDE_INT T1.Z, T2.Y, T3.W, PV.X, BS:VEC_201
+; EG-NEXT:     CNDE_INT T0.W, T2.Y, T0.Y, T1.X, BS:VEC_201
+; EG-NEXT:     MULLO_INT * T0.X, PS, T9.W,
+; EG-NEXT:     ADD_INT T1.X, PV.W, 1,
+; EG-NEXT:     SETGE_UINT T0.Y, PV.Z, T7.W,
+; EG-NEXT:     ADD_INT T1.Z, PV.Y, 1,
+; EG-NEXT:     SETGE_UINT T1.W, PV.X, T1.W, BS:VEC_102/SCL_221
+; EG-NEXT:     SUB_INT * T3.W, T10.W, PS,
+; EG-NEXT:     ADD_INT T0.X, T0.Z, 1,
+; EG-NEXT:     SETGE_UINT T2.Y, PS, T9.W, BS:VEC_102/SCL_221
+; EG-NEXT:     SUB_INT T3.Z, PS, T9.W, BS:VEC_102/SCL_221
+; EG-NEXT:     CNDE_INT T1.W, PV.W, T1.Y, PV.Z,
+; EG-NEXT:     XOR_INT * T2.W, T4.W, T2.W,
 ; EG-NEXT:     XOR_INT T2.X, PV.W, PS,
-; EG-NEXT:     AND_INT T0.Y, PV.Y, PV.Z,
-; EG-NEXT:     ADD_INT T1.Z, T1.X, 1,
-; EG-NEXT:     CNDE_INT T1.W, T2.W, PV.X, T3.Y,
-; EG-NEXT:     XOR_INT * T0.W, T5.W, T0.W,
-; EG-NEXT:     XOR_INT T0.X, T4.Z, T6.W, BS:VEC_021/SCL_122
-; EG-NEXT:     XOR_INT T1.Y, PV.W, PS,
-; EG-NEXT:     CNDE_INT T1.Z, PV.Y, T1.X, PV.Z,
-; EG-NEXT:     ADD_INT T1.W, T1.X, literal.x,
-; EG-NEXT:     SUB_INT * T3.W, PV.X, T3.W,
-; EG-NEXT:    -1(nan), 0(0.000000e+00)
-; EG-NEXT:     CNDE_INT T0.Y, T0.Z, PV.W, PV.Z,
-; EG-NEXT:     SUB_INT T3.Z, PV.Y, T0.W,
-; EG-NEXT:     XOR_INT T0.W, T7.W, T2.Z,
-; EG-NEXT:     XOR_INT * T1.W, T3.X, PV.X,
-; EG-NEXT:     SUB_INT T3.Y, PS, T0.X,
+; EG-NEXT:     CNDE_INT T1.Y, PV.Y, T3.W, PV.Z, BS:VEC_021/SCL_122
+; EG-NEXT:     CNDE_INT T0.Z, PV.Y, T0.Z, PV.X,
+; EG-NEXT:     CNDE_INT T0.W, T0.Y, T0.W, T1.X, BS:VEC_102/SCL_221
+; EG-NEXT:     XOR_INT * T1.W, T4.Z, T5.W,
+; EG-NEXT:     XOR_INT T0.X, T6.W, T2.Z,
+; EG-NEXT:     XOR_INT T0.Y, PV.W, PS,
+; EG-NEXT:     ADD_INT T1.Z, PV.Z, 1,
+; EG-NEXT:     SETGE_UINT T0.W, PV.Y, T9.W, BS:VEC_021/SCL_122
+; EG-NEXT:     SUB_INT * T2.W, PV.X, T2.W,
+; EG-NEXT:     CNDE_INT T1.Y, PV.W, T0.Z, PV.Z,
+; EG-NEXT:     SUB_INT T2.Z, PV.Y, T1.W,
+; EG-NEXT:     XOR_INT T0.W, T3.Y, T8.W, BS:VEC_021/SCL_122
+; EG-NEXT:     XOR_INT * T1.W, T4.Y, PV.X,
+; EG-NEXT:     SUB_INT T2.Y, PS, T0.X,
 ; EG-NEXT:     XOR_INT * T1.W, PV.Y, PV.W,
-; EG-NEXT:     SUB_INT T3.X, PV.W, T0.W,
+; EG-NEXT:     SUB_INT T2.X, PV.W, T0.W,
 ; EG-NEXT:     LSHR * T0.X, KC0[2].Y, literal.x,
 ; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %den_ptr = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %in, i32 1
@@ -1605,7 +1497,7 @@ define amdgpu_kernel void @v_sdiv_i8(i32 addrspace(1)* %out, i8 addrspace(1)* %i
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v1, v1
 ; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 8
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
@@ -1638,7 +1530,7 @@ define amdgpu_kernel void @v_sdiv_i8(i32 addrspace(1)* %out, i8 addrspace(1)* %i
 ; TONGA-NEXT:    v_cvt_i32_f32_e32 v1, v1
 ; TONGA-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
 ; TONGA-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
 ; TONGA-NEXT:    v_bfe_i32 v0, v0, 0, 8
 ; TONGA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; TONGA-NEXT:    s_endpgm
@@ -1758,7 +1650,7 @@ define amdgpu_kernel void @v_sdiv_i23(i32 addrspace(1)* %out, i23 addrspace(1)*
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
 ; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v3|
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 23
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
@@ -1799,7 +1691,7 @@ define amdgpu_kernel void @v_sdiv_i23(i32 addrspace(1)* %out, i23 addrspace(1)*
 ; TONGA-NEXT:    v_cvt_i32_f32_e32 v2, v2
 ; TONGA-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v3|
 ; TONGA-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
+; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v2, v0
 ; TONGA-NEXT:    v_bfe_i32 v0, v0, 0, 23
 ; TONGA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; TONGA-NEXT:    s_endpgm
@@ -1938,7 +1830,7 @@ define amdgpu_kernel void @v_sdiv_i24(i32 addrspace(1)* %out, i24 addrspace(1)*
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v3, v3
 ; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, |v2|
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
@@ -1976,7 +1868,7 @@ define amdgpu_kernel void @v_sdiv_i24(i32 addrspace(1)* %out, i24 addrspace(1)*
 ; TONGA-NEXT:    v_cvt_i32_f32_e32 v3, v3
 ; TONGA-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, |v2|
 ; TONGA-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v3
+; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v3, v0
 ; TONGA-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; TONGA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; TONGA-NEXT:    s_endpgm
@@ -2023,7 +1915,7 @@ define amdgpu_kernel void @v_sdiv_i24(i32 addrspace(1)* %out, i24 addrspace(1)*
 ; EG:       ; %bb.0:
 ; EG-NEXT:    ALU 0, @14, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    TEX 3 @6
-; EG-NEXT:    ALU 43, @15, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 39, @15, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
@@ -2041,37 +1933,33 @@ define amdgpu_kernel void @v_sdiv_i24(i32 addrspace(1)* %out, i24 addrspace(1)*
 ; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
 ; EG-NEXT:     OR_INT * T0.W, T0.X, PV.W,
 ; EG-NEXT:     SETGT_INT * T1.W, 0.0, PV.W,
-; EG-NEXT:     ADD_INT * T0.W, T0.W, PV.W,
-; EG-NEXT:     XOR_INT * T0.W, PV.W, T1.W,
-; EG-NEXT:     RECIP_UINT * T0.X, PV.W,
 ; EG-NEXT:     BFE_INT T2.W, T3.X, 0.0, literal.x,
-; EG-NEXT:     MULLO_INT * T0.Y, PS, T0.W,
+; EG-NEXT:     ADD_INT * T0.W, T0.W, PV.W,
 ; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
-; EG-NEXT:     LSHL T0.Z, PV.W, literal.x,
-; EG-NEXT:     SUB_INT T2.W, 0.0, PS,
-; EG-NEXT:     MULHI * T1.X, T0.X, T0.W,
+; EG-NEXT:     LSHL T2.W, PV.W, literal.x,
+; EG-NEXT:     XOR_INT * T0.W, PS, T1.W,
 ; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
-; EG-NEXT:     CNDE_INT T2.W, PS, PV.W, T0.Y,
-; EG-NEXT:     OR_INT * T3.W, T2.X, PV.Z,
-; EG-NEXT:     SETGT_INT T4.W, 0.0, PS,
-; EG-NEXT:     MULHI * T0.Y, PV.W, T0.X,
-; EG-NEXT:     ADD_INT T0.Z, T3.W, PV.W,
-; EG-NEXT:     ADD_INT T2.W, T0.X, PS,
-; EG-NEXT:     SUB_INT * T3.W, T0.X, PS,
-; EG-NEXT:     CNDE_INT T2.W, T1.X, PV.W, PS,
-; EG-NEXT:     XOR_INT * T3.W, PV.Z, T4.W,
-; EG-NEXT:     MULHI * T0.X, PV.W, PS,
+; EG-NEXT:     SUB_INT T0.Z, 0.0, PS,
+; EG-NEXT:     OR_INT T2.W, T2.X, PV.W,
+; EG-NEXT:     RECIP_UINT * T0.X, PS,
+; EG-NEXT:     SETGT_INT T3.W, 0.0, PV.W,
+; EG-NEXT:     MULLO_INT * T0.Y, PV.Z, PS,
+; EG-NEXT:     ADD_INT T2.W, T2.W, PV.W,
+; EG-NEXT:     MULHI * T0.Y, T0.X, PS,
+; EG-NEXT:     ADD_INT T4.W, T0.X, PS,
+; EG-NEXT:     XOR_INT * T2.W, PV.W, T3.W,
+; EG-NEXT:     MULHI * T0.X, PS, PV.W,
 ; EG-NEXT:     MULLO_INT * T0.Y, PS, T0.W,
-; EG-NEXT:     SUB_INT * T2.W, T3.W, PS,
-; EG-NEXT:     SETGE_UINT T0.W, PV.W, T0.W,
-; EG-NEXT:     SETGE_UINT * T2.W, T3.W, T0.Y,
-; EG-NEXT:     AND_INT T0.W, PV.W, PS,
-; EG-NEXT:     ADD_INT * T3.W, T0.X, 1,
-; EG-NEXT:     CNDE_INT T0.W, PV.W, T0.X, PS,
-; EG-NEXT:     ADD_INT * T3.W, T0.X, literal.x,
-; EG-NEXT:    -1(nan), 0(0.000000e+00)
-; EG-NEXT:     CNDE_INT T0.W, T2.W, PS, PV.W,
-; EG-NEXT:     XOR_INT * T1.W, T4.W, T1.W,
+; EG-NEXT:     SUB_INT * T2.W, T2.W, PS,
+; EG-NEXT:     ADD_INT T0.Z, T0.X, 1,
+; EG-NEXT:     SETGE_UINT T4.W, PV.W, T0.W,
+; EG-NEXT:     SUB_INT * T5.W, PV.W, T0.W,
+; EG-NEXT:     CNDE_INT T2.W, PV.W, T2.W, PS,
+; EG-NEXT:     CNDE_INT * T4.W, PV.W, T0.X, PV.Z,
+; EG-NEXT:     ADD_INT T5.W, PS, 1,
+; EG-NEXT:     SETGE_UINT * T0.W, PV.W, T0.W,
+; EG-NEXT:     CNDE_INT T0.W, PS, T4.W, PV.W, BS:VEC_102/SCL_221
+; EG-NEXT:     XOR_INT * T1.W, T3.W, T1.W,
 ; EG-NEXT:     XOR_INT * T0.W, PV.W, PS,
 ; EG-NEXT:     SUB_INT * T0.W, PV.W, T1.W,
 ; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
@@ -2091,165 +1979,153 @@ define amdgpu_kernel void @v_sdiv_i24(i32 addrspace(1)* %out, i24 addrspace(1)*
 define amdgpu_kernel void @v_sdiv_i25(i32 addrspace(1)* %out, i25 addrspace(1)* %in) {
 ; GCN-LABEL: v_sdiv_i25:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
-; GCN-NEXT:    s_mov_b32 s2, s6
-; GCN-NEXT:    s_mov_b32 s3, s7
+; GCN-NEXT:    s_mov_b32 s10, s6
+; GCN-NEXT:    s_mov_b32 s11, s7
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s0, s10
-; GCN-NEXT:    s_mov_b32 s1, s11
-; GCN-NEXT:    buffer_load_dwordx2 v[0:1], off, s[0:3], 0
-; GCN-NEXT:    s_mov_b32 s4, s8
-; GCN-NEXT:    s_mov_b32 s5, s9
+; GCN-NEXT:    s_mov_b32 s8, s2
+; GCN-NEXT:    s_mov_b32 s9, s3
+; GCN-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NEXT:    s_mov_b32 s4, s0
+; GCN-NEXT:    s_mov_b32 s5, s1
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_bfe_i32 v2, v1, 0, 25
 ; GCN-NEXT:    v_bfe_i32 v1, v1, 24, 1
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v1, v2
 ; GCN-NEXT:    v_xor_b32_e32 v2, v2, v1
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v3, v2
-; GCN-NEXT:    v_bfe_i32 v4, v0, 0, 25
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v2
+; GCN-NEXT:    v_bfe_i32 v5, v0, 0, 25
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 24, 1
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v0, v4
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; GCN-NEXT:    v_xor_b32_e32 v4, v4, v0
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v0, v5
+; GCN-NEXT:    v_xor_b32_e32 v5, v5, v0
 ; GCN-NEXT:    v_xor_b32_e32 v0, v0, v1
-; GCN-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; GCN-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GCN-NEXT:    v_mul_lo_u32 v5, v3, v2
-; GCN-NEXT:    v_mul_hi_u32 v6, v3, v2
-; GCN-NEXT:    v_sub_i32_e32 v7, vcc, 0, v5
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v5, v5, v7, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v5, v5, v3
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v5, v3
-; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, v5, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v6, s[0:1]
-; GCN-NEXT:    v_mul_hi_u32 v3, v3, v4
+; GCN-NEXT:    v_mul_lo_u32 v4, v4, v3
+; GCN-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_mul_hi_u32 v3, v5, v3
 ; GCN-NEXT:    v_mul_lo_u32 v1, v3, v2
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, -1, v3
-; GCN-NEXT:    v_subrev_i32_e32 v7, vcc, v1, v4
-; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v1
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v7, v2
-; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v3, v5, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v3
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v1, v5
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v1, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v4, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, v2, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v4, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v3
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v1, v1, v0
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 25
 ; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; TONGA-LABEL: v_sdiv_i25:
 ; TONGA:       ; %bb.0:
-; TONGA-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x24
+; TONGA-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; TONGA-NEXT:    s_mov_b32 s7, 0xf000
 ; TONGA-NEXT:    s_mov_b32 s6, -1
-; TONGA-NEXT:    s_mov_b32 s2, s6
-; TONGA-NEXT:    s_mov_b32 s3, s7
+; TONGA-NEXT:    s_mov_b32 s10, s6
+; TONGA-NEXT:    s_mov_b32 s11, s7
 ; TONGA-NEXT:    s_waitcnt lgkmcnt(0)
-; TONGA-NEXT:    s_mov_b32 s0, s10
-; TONGA-NEXT:    s_mov_b32 s1, s11
-; TONGA-NEXT:    buffer_load_dwordx2 v[0:1], off, s[0:3], 0
-; TONGA-NEXT:    s_mov_b32 s4, s8
-; TONGA-NEXT:    s_mov_b32 s5, s9
+; TONGA-NEXT:    s_mov_b32 s8, s2
+; TONGA-NEXT:    s_mov_b32 s9, s3
+; TONGA-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; TONGA-NEXT:    s_mov_b32 s4, s0
+; TONGA-NEXT:    s_mov_b32 s5, s1
 ; TONGA-NEXT:    s_waitcnt vmcnt(0)
 ; TONGA-NEXT:    v_bfe_i32 v2, v1, 0, 25
 ; TONGA-NEXT:    v_bfe_i32 v1, v1, 24, 1
 ; TONGA-NEXT:    v_add_u32_e32 v2, vcc, v1, v2
 ; TONGA-NEXT:    v_xor_b32_e32 v2, v2, v1
 ; TONGA-NEXT:    v_cvt_f32_u32_e32 v3, v2
-; TONGA-NEXT:    v_bfe_i32 v4, v0, 0, 25
+; TONGA-NEXT:    v_sub_u32_e32 v4, vcc, 0, v2
+; TONGA-NEXT:    v_bfe_i32 v5, v0, 0, 25
 ; TONGA-NEXT:    v_bfe_i32 v0, v0, 24, 1
-; TONGA-NEXT:    v_add_u32_e32 v4, vcc, v0, v4
 ; TONGA-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; TONGA-NEXT:    v_xor_b32_e32 v4, v4, v0
+; TONGA-NEXT:    v_add_u32_e32 v5, vcc, v0, v5
+; TONGA-NEXT:    v_xor_b32_e32 v5, v5, v0
 ; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v1
-; TONGA-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; TONGA-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
 ; TONGA-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; TONGA-NEXT:    v_mul_lo_u32 v5, v3, v2
-; TONGA-NEXT:    v_mul_hi_u32 v6, v3, v2
-; TONGA-NEXT:    v_sub_u32_e32 v7, vcc, 0, v5
-; TONGA-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v6
-; TONGA-NEXT:    v_cndmask_b32_e64 v5, v5, v7, s[0:1]
-; TONGA-NEXT:    v_mul_hi_u32 v5, v5, v3
-; TONGA-NEXT:    v_add_u32_e32 v6, vcc, v5, v3
-; TONGA-NEXT:    v_subrev_u32_e32 v3, vcc, v5, v3
-; TONGA-NEXT:    v_cndmask_b32_e64 v3, v3, v6, s[0:1]
-; TONGA-NEXT:    v_mul_hi_u32 v3, v3, v4
+; TONGA-NEXT:    v_mul_lo_u32 v4, v4, v3
+; TONGA-NEXT:    v_mul_hi_u32 v4, v3, v4
+; TONGA-NEXT:    v_add_u32_e32 v3, vcc, v4, v3
+; TONGA-NEXT:    v_mul_hi_u32 v3, v5, v3
 ; TONGA-NEXT:    v_mul_lo_u32 v1, v3, v2
-; TONGA-NEXT:    v_add_u32_e32 v5, vcc, 1, v3
-; TONGA-NEXT:    v_add_u32_e32 v6, vcc, -1, v3
-; TONGA-NEXT:    v_subrev_u32_e32 v7, vcc, v1, v4
-; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v1
-; TONGA-NEXT:    v_cmp_ge_u32_e64 s[0:1], v7, v2
-; TONGA-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
-; TONGA-NEXT:    v_cndmask_b32_e64 v1, v3, v5, s[0:1]
-; TONGA-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; TONGA-NEXT:    v_add_u32_e32 v4, vcc, 1, v3
+; TONGA-NEXT:    v_subrev_u32_e32 v1, vcc, v1, v5
+; TONGA-NEXT:    v_cmp_ge_u32_e64 s[0:1], v1, v2
+; TONGA-NEXT:    v_cndmask_b32_e64 v3, v3, v4, s[0:1]
+; TONGA-NEXT:    v_subrev_u32_e32 v4, vcc, v2, v1
+; TONGA-NEXT:    v_cndmask_b32_e64 v1, v1, v4, s[0:1]
+; TONGA-NEXT:    v_add_u32_e32 v4, vcc, 1, v3
+; TONGA-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
+; TONGA-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
 ; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v0
-; TONGA-NEXT:    v_sub_u32_e32 v0, vcc, v1, v0
+; TONGA-NEXT:    v_subrev_u32_e32 v0, vcc, v0, v1
 ; TONGA-NEXT:    v_bfe_i32 v0, v0, 0, 25
 ; TONGA-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; TONGA-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: v_sdiv_i25:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX9-NEXT:    s_mov_b32 s7, 0xf000
-; GFX9-NEXT:    s_mov_b32 s6, -1
-; GFX9-NEXT:    s_mov_b32 s10, s6
-; GFX9-NEXT:    s_mov_b32 s11, s7
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_mov_b32 s10, s2
+; GFX9-NEXT:    s_mov_b32 s11, s3
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s8, s2
-; GFX9-NEXT:    s_mov_b32 s9, s3
+; GFX9-NEXT:    s_mov_b32 s8, s6
+; GFX9-NEXT:    s_mov_b32 s9, s7
 ; GFX9-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
-; GFX9-NEXT:    s_mov_b32 s4, s0
-; GFX9-NEXT:    s_mov_b32 s5, s1
+; GFX9-NEXT:    s_mov_b32 s0, s4
+; GFX9-NEXT:    s_mov_b32 s1, s5
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_bfe_i32 v2, v1, 0, 25
 ; GFX9-NEXT:    v_bfe_i32 v1, v1, 24, 1
 ; GFX9-NEXT:    v_add_u32_e32 v2, v2, v1
 ; GFX9-NEXT:    v_xor_b32_e32 v2, v2, v1
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, v2
-; GFX9-NEXT:    v_bfe_i32 v6, v0, 0, 25
+; GFX9-NEXT:    v_sub_u32_e32 v4, 0, v2
+; GFX9-NEXT:    v_bfe_i32 v5, v0, 0, 25
 ; GFX9-NEXT:    v_bfe_i32 v0, v0, 24, 1
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; GFX9-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GFX9-NEXT:    v_mul_lo_u32 v4, v3, v2
-; GFX9-NEXT:    v_mul_hi_u32 v5, v3, v2
-; GFX9-NEXT:    v_sub_u32_e32 v7, 0, v4
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v4, v4, v3
-; GFX9-NEXT:    v_add_u32_e32 v5, v6, v0
+; GFX9-NEXT:    v_add_u32_e32 v5, v5, v0
 ; GFX9-NEXT:    v_xor_b32_e32 v5, v5, v0
 ; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v1
-; GFX9-NEXT:    v_add_u32_e32 v6, v3, v4
-; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v3, v3, v5
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT:    v_mul_lo_u32 v4, v4, v3
+; GFX9-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v4
+; GFX9-NEXT:    v_mul_hi_u32 v3, v5, v3
 ; GFX9-NEXT:    v_mul_lo_u32 v4, v3, v2
 ; GFX9-NEXT:    v_add_u32_e32 v1, 1, v3
-; GFX9-NEXT:    v_add_u32_e32 v6, -1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v7, v5, v4
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v5, v4
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[0:1], v7, v2
-; GFX9-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[0:1]
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v4, v5, v4
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v3, v4, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX9-NEXT:    v_add_u32_e32 v4, 1, v1
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v0
 ; GFX9-NEXT:    v_sub_u32_e32 v0, v1, v0
 ; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 25
-; GFX9-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
 ;
 ; EG-LABEL: v_sdiv_i25:
 ; EG:       ; %bb.0:
 ; EG-NEXT:    ALU 1, @10, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    TEX 1 @6
-; EG-NEXT:    ALU 41, @12, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 37, @12, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
@@ -2265,36 +2141,32 @@ define amdgpu_kernel void @v_sdiv_i25(i32 addrspace(1)* %out, i25 addrspace(1)*
 ; EG-NEXT:     ASHR * T0.W, PV.W, literal.x,
 ; EG-NEXT:    7(9.809089e-45), 0(0.000000e+00)
 ; EG-NEXT:     SETGT_INT * T1.W, 0.0, PV.W,
-; EG-NEXT:     ADD_INT * T0.W, T0.W, PV.W,
+; EG-NEXT:     ADD_INT T0.W, T0.W, PV.W,
+; EG-NEXT:     LSHL * T2.W, T1.X, literal.x,
+; EG-NEXT:    7(9.809089e-45), 0(0.000000e+00)
 ; EG-NEXT:     XOR_INT * T0.W, PV.W, T1.W,
+; EG-NEXT:     SUB_INT T0.Z, 0.0, PV.W,
+; EG-NEXT:     ASHR T2.W, T2.W, literal.x,
 ; EG-NEXT:     RECIP_UINT * T0.X, PV.W,
-; EG-NEXT:     MULLO_INT * T0.Y, PS, T0.W,
-; EG-NEXT:     LSHL T0.Z, T1.X, literal.x,
-; EG-NEXT:     SUB_INT T2.W, 0.0, PS,
-; EG-NEXT:     MULHI * T1.X, T0.X, T0.W,
 ; EG-NEXT:    7(9.809089e-45), 0(0.000000e+00)
-; EG-NEXT:     CNDE_INT T2.W, PS, PV.W, T0.Y,
-; EG-NEXT:     ASHR * T3.W, PV.Z, literal.x,
-; EG-NEXT:    7(9.809089e-45), 0(0.000000e+00)
-; EG-NEXT:     SETGT_INT T4.W, 0.0, PS,
-; EG-NEXT:     MULHI * T0.Y, PV.W, T0.X,
-; EG-NEXT:     ADD_INT T0.Z, T3.W, PV.W,
-; EG-NEXT:     ADD_INT T2.W, T0.X, PS,
-; EG-NEXT:     SUB_INT * T3.W, T0.X, PS,
-; EG-NEXT:     CNDE_INT T2.W, T1.X, PV.W, PS,
-; EG-NEXT:     XOR_INT * T3.W, PV.Z, T4.W,
-; EG-NEXT:     MULHI * T0.X, PV.W, PS,
+; EG-NEXT:     SETGT_INT T3.W, 0.0, PV.W,
+; EG-NEXT:     MULLO_INT * T0.Y, PV.Z, PS,
+; EG-NEXT:     ADD_INT T2.W, T2.W, PV.W,
+; EG-NEXT:     MULHI * T0.Y, T0.X, PS,
+; EG-NEXT:     ADD_INT T4.W, T0.X, PS,
+; EG-NEXT:     XOR_INT * T2.W, PV.W, T3.W,
+; EG-NEXT:     MULHI * T0.X, PS, PV.W,
 ; EG-NEXT:     MULLO_INT * T0.Y, PS, T0.W,
-; EG-NEXT:     SUB_INT * T2.W, T3.W, PS,
-; EG-NEXT:     SETGE_UINT T0.W, PV.W, T0.W,
-; EG-NEXT:     SETGE_UINT * T2.W, T3.W, T0.Y,
-; EG-NEXT:     AND_INT T0.W, PV.W, PS,
-; EG-NEXT:     ADD_INT * T3.W, T0.X, 1,
-; EG-NEXT:     CNDE_INT T0.W, PV.W, T0.X, PS,
-; EG-NEXT:     ADD_INT * T3.W, T0.X, literal.x,
-; EG-NEXT:    -1(nan), 0(0.000000e+00)
-; EG-NEXT:     CNDE_INT T0.W, T2.W, PS, PV.W,
-; EG-NEXT:     XOR_INT * T1.W, T4.W, T1.W,
+; EG-NEXT:     SUB_INT * T2.W, T2.W, PS,
+; EG-NEXT:     ADD_INT T0.Z, T0.X, 1,
+; EG-NEXT:     SETGE_UINT T4.W, PV.W, T0.W,
+; EG-NEXT:     SUB_INT * T5.W, PV.W, T0.W,
+; EG-NEXT:     CNDE_INT T2.W, PV.W, T2.W, PS,
+; EG-NEXT:     CNDE_INT * T4.W, PV.W, T0.X, PV.Z,
+; EG-NEXT:     ADD_INT T5.W, PS, 1,
+; EG-NEXT:     SETGE_UINT * T0.W, PV.W, T0.W,
+; EG-NEXT:     CNDE_INT T0.W, PS, T4.W, PV.W, BS:VEC_102/SCL_221
+; EG-NEXT:     XOR_INT * T1.W, T3.W, T1.W,
 ; EG-NEXT:     XOR_INT * T0.W, PV.W, PS,
 ; EG-NEXT:     SUB_INT * T0.W, PV.W, T1.W,
 ; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
index 9ad18f4127082..e0a469e11ed59 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
@@ -6,7 +6,6 @@ define amdgpu_kernel void @s_test_sdiv(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ; GCN-LABEL: s_test_sdiv:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
-; GCN-NEXT:    v_mov_b32_e32 v7, 0
 ; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
@@ -15,123 +14,124 @@ define amdgpu_kernel void @s_test_sdiv(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ; GCN-NEXT:    s_add_u32 s2, s2, s12
 ; GCN-NEXT:    s_mov_b32 s13, s12
 ; GCN-NEXT:    s_addc_u32 s3, s3, s12
-; GCN-NEXT:    s_xor_b64 s[2:3], s[2:3], s[12:13]
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s2
-; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s3
-; GCN-NEXT:    s_sub_u32 s4, 0, s2
-; GCN-NEXT:    s_subb_u32 s5, 0, s3
-; GCN-NEXT:    s_ashr_i32 s14, s11, 31
+; GCN-NEXT:    s_xor_b64 s[14:15], s[2:3], s[12:13]
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s14
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s15
+; GCN-NEXT:    s_sub_u32 s2, 0, s14
+; GCN-NEXT:    s_subb_u32 s3, 0, s15
+; GCN-NEXT:    s_ashr_i32 s16, s11, 31
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-NEXT:    s_mov_b32 s15, s14
+; GCN-NEXT:    s_mov_b32 s17, s16
+; GCN-NEXT:    s_mov_b32 s4, s8
+; GCN-NEXT:    s_mov_b32 s5, s9
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GCN-NEXT:    v_mul_f32_e32 v2, 0x2f800000, v0
-; GCN-NEXT:    v_trunc_f32_e32 v2, v2
-; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v2
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GCN-NEXT:    v_mul_hi_u32 v4, s4, v0
-; GCN-NEXT:    v_mul_lo_u32 v3, s4, v2
-; GCN-NEXT:    v_mul_lo_u32 v6, s5, v0
-; GCN-NEXT:    v_mul_lo_u32 v5, s4, v0
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v4, v0, v5
-; GCN-NEXT:    v_mul_lo_u32 v6, v0, v3
-; GCN-NEXT:    v_mul_hi_u32 v8, v0, v3
-; GCN-NEXT:    v_mul_hi_u32 v9, v2, v3
-; GCN-NEXT:    v_mul_lo_u32 v3, v2, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v7, v8, vcc
-; GCN-NEXT:    v_mul_lo_u32 v8, v2, v5
-; GCN-NEXT:    v_mul_hi_u32 v5, v2, v5
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v6, v5, vcc
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v9, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
-; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v3
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v7, v5, vcc
-; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v2, v4, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v5, s4, v3
-; GCN-NEXT:    v_mul_hi_u32 v6, s4, v0
-; GCN-NEXT:    v_mul_lo_u32 v8, s5, v0
-; GCN-NEXT:    s_mov_b32 s5, s9
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; GCN-NEXT:    v_mul_lo_u32 v6, s4, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
+; GCN-NEXT:    v_mul_lo_u32 v5, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s2, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v0, v2
+; GCN-NEXT:    v_mul_lo_u32 v6, v1, v4
+; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_mul_hi_u32 v8, v1, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v5, s2, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v8, s3, v0
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GCN-NEXT:    v_mul_lo_u32 v7, s2, v0
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
 ; GCN-NEXT:    v_mul_lo_u32 v10, v0, v5
 ; GCN-NEXT:    v_mul_hi_u32 v12, v0, v5
-; GCN-NEXT:    v_mul_hi_u32 v11, v0, v6
-; GCN-NEXT:    v_mul_hi_u32 v9, v3, v6
-; GCN-NEXT:    v_mul_lo_u32 v6, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v8, v3, v5
+; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v8, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, v7, v12, vcc
-; GCN-NEXT:    v_mul_lo_u32 v3, v3, v5
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v11, v9, vcc
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v5, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v2, v5, s[0:1]
-; GCN-NEXT:    s_add_u32 s0, s10, s14
-; GCN-NEXT:    s_addc_u32 s1, s11, s14
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
-; GCN-NEXT:    s_xor_b64 s[10:11], s[0:1], s[14:15]
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
-; GCN-NEXT:    v_mul_lo_u32 v3, s10, v2
-; GCN-NEXT:    v_mul_hi_u32 v4, s10, v0
-; GCN-NEXT:    v_mul_hi_u32 v5, s10, v2
-; GCN-NEXT:    v_mul_hi_u32 v6, s11, v2
-; GCN-NEXT:    v_mul_lo_u32 v2, s11, v2
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v7, v5, vcc
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
+; GCN-NEXT:    s_add_u32 s0, s10, s16
+; GCN-NEXT:    s_addc_u32 s1, s11, s16
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    s_xor_b64 s[10:11], s[0:1], s[16:17]
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
-; GCN-NEXT:    s_mov_b32 s4, s8
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v4, v0, vcc
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v7, v1, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s3, v0
-; GCN-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s14, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s14, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s15, v0
+; GCN-NEXT:    v_mov_b32_e32 v5, s15
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_mul_lo_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v3, s14, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s11, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s10, v3
-; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s2, v3
-; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s3, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s2, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s3, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v5, s[0:1], 2, v0
-; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 1, v0
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[0:1]
+; GCN-NEXT:    v_sub_i32_e64 v3, s[0:1], s10, v3
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v4, v5, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s14, v3
+; GCN-NEXT:    v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s15, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s14, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s15, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v6, s11
-; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v6, v2, vcc
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v6, v2, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s15, v2
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s2, v3
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s14, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s15, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    s_xor_b64 s[0:1], s[14:15], s[12:13]
+; GCN-NEXT:    s_xor_b64 s[0:1], s[16:17], s[12:13]
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v0, s0, v0
 ; GCN-NEXT:    v_xor_b32_e32 v1, s1, v1
@@ -144,104 +144,98 @@ define amdgpu_kernel void @s_test_sdiv(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ; GCN-IR-LABEL: s_test_sdiv:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-IR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GCN-IR-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_ashr_i32 s2, s7, 31
+; GCN-IR-NEXT:    s_ashr_i32 s0, s7, 31
+; GCN-IR-NEXT:    s_mov_b32 s1, s0
+; GCN-IR-NEXT:    s_ashr_i32 s2, s9, 31
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[0:1], s[6:7]
+; GCN-IR-NEXT:    s_sub_u32 s10, s6, s0
 ; GCN-IR-NEXT:    s_mov_b32 s3, s2
-; GCN-IR-NEXT:    s_ashr_i32 s8, s1, 31
-; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[2:3], s[6:7]
-; GCN-IR-NEXT:    s_sub_u32 s10, s6, s2
-; GCN-IR-NEXT:    s_mov_b32 s9, s8
-; GCN-IR-NEXT:    s_subb_u32 s11, s7, s2
-; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[8:9], s[0:1]
-; GCN-IR-NEXT:    s_sub_u32 s6, s0, s8
-; GCN-IR-NEXT:    s_flbit_i32_b32 s14, s6
-; GCN-IR-NEXT:    s_subb_u32 s7, s1, s8
-; GCN-IR-NEXT:    s_add_i32 s14, s14, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s15, s7
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s14
+; GCN-IR-NEXT:    s_subb_u32 s11, s7, s0
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[2:3], s[8:9]
+; GCN-IR-NEXT:    s_sub_u32 s6, s6, s2
+; GCN-IR-NEXT:    s_subb_u32 s7, s7, s2
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[6:7], 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[14:15], s[10:11], 0
+; GCN-IR-NEXT:    s_mov_b64 s[8:9], 0
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[12:13], s[14:15]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s6
+; GCN-IR-NEXT:    s_add_i32 s12, s12, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s14, s7
+; GCN-IR-NEXT:    s_cmp_eq_u32 s7, 0
+; GCN-IR-NEXT:    s_cselect_b32 s12, s12, s14
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s14, s10
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s15
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
 ; GCN-IR-NEXT:    s_add_i32 s14, s14, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s15, s11
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s15
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s14
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s11, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v2, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[14:15], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[10:11], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[12:13]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[12:13], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[12:13], s[12:13], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[12:13]
-; GCN-IR-NEXT:    s_cbranch_vccz BB0_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[4:5], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[10:11], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_cmp_eq_u32 s11, 0
+; GCN-IR-NEXT:    s_cselect_b32 s16, s14, s15
+; GCN-IR-NEXT:    s_sub_u32 s14, s12, s16
+; GCN-IR-NEXT:    s_subb_u32 s15, 0, 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[20:21], s[14:15], 63
+; GCN-IR-NEXT:    s_mov_b32 s13, 0
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[20:21]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[20:21], s[14:15], 63
+; GCN-IR-NEXT:    s_xor_b64 s[22:23], s[18:19], -1
+; GCN-IR-NEXT:    s_and_b64 s[20:21], s[22:23], s[20:21]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[20:21]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB0_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s18, s14, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s14
+; GCN-IR-NEXT:    s_addc_u32 s19, s15, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s15
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[18:19], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s14, 63, s14
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[10:11], s14
+; GCN-IR-NEXT:    s_cbranch_vccz BB0_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_not_b32_e32 v2, v2
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[10:11], v4
+; GCN-IR-NEXT:    s_lshr_b64 s[18:19], s[10:11], s18
 ; GCN-IR-NEXT:    s_add_u32 s10, s6, -1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v2, v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
 ; GCN-IR-NEXT:    s_addc_u32 s11, s7, -1
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], -1, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_not_b64 s[8:9], s[12:13]
+; GCN-IR-NEXT:    s_mov_b32 s17, s13
+; GCN-IR-NEXT:    s_add_u32 s12, s8, s16
+; GCN-IR-NEXT:    s_addc_u32 s13, s9, s13
+; GCN-IR-NEXT:    s_mov_b64 s[16:17], 0
+; GCN-IR-NEXT:    s_mov_b32 s9, 0
 ; GCN-IR-NEXT:  BB0_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s11
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s10, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s6, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s7, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s8, s15, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[18:19], s[18:19], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[8:9]
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[16:17], s[14:15]
+; GCN-IR-NEXT:    s_sub_u32 s8, s10, s18
+; GCN-IR-NEXT:    s_subb_u32 s8, s11, s19
+; GCN-IR-NEXT:    s_ashr_i32 s16, s8, 31
+; GCN-IR-NEXT:    s_mov_b32 s17, s16
+; GCN-IR-NEXT:    s_and_b32 s8, s16, 1
+; GCN-IR-NEXT:    s_and_b64 s[20:21], s[16:17], s[6:7]
+; GCN-IR-NEXT:    s_sub_u32 s18, s18, s20
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s12
+; GCN-IR-NEXT:    s_subb_u32 s19, s19, s21
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s13
+; GCN-IR-NEXT:    s_add_u32 s12, s12, 1
+; GCN-IR-NEXT:    s_addc_u32 s13, s13, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[16:17], s[8:9]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB0_3
+; GCN-IR-NEXT:  BB0_4: ; %Flow6
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[14:15], 1
+; GCN-IR-NEXT:    s_or_b64 s[6:7], s[8:9], s[6:7]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s7
 ; GCN-IR-NEXT:    s_branch BB0_6
-; GCN-IR-NEXT:  BB0_4:
+; GCN-IR-NEXT:  BB0_5:
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[18:19]
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB0_7
-; GCN-IR-NEXT:  BB0_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB0_6: ; %Flow6
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB0_7: ; %Flow7
-; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[8:9], s[2:3]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[18:19]
+; GCN-IR-NEXT:  BB0_6: ; %Flow7
+; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[2:3], s[0:1]
 ; GCN-IR-NEXT:    v_xor_b32_e32 v0, s0, v0
 ; GCN-IR-NEXT:    v_xor_b32_e32 v1, s1, v1
 ; GCN-IR-NEXT:    v_mov_b32_e32 v2, s1
@@ -512,15 +506,15 @@ define amdgpu_kernel void @s_test_sdiv24_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_xor_b32 s4, s4, s8
 ; GCN-NEXT:    s_ashr_i32 s4, s4, 30
-; GCN-NEXT:    s_or_b32 s4, s4, 1
+; GCN-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_mov_b32_e32 v3, s4
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s4, 0
+; GCN-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -542,15 +536,15 @@ define amdgpu_kernel void @s_test_sdiv24_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-IR-NEXT:    s_xor_b32 s4, s4, s8
 ; GCN-IR-NEXT:    s_ashr_i32 s4, s4, 30
-; GCN-IR-NEXT:    s_or_b32 s4, s4, 1
+; GCN-IR-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s4
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u32 s4, 0
+; GCN-IR-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -619,16 +613,16 @@ define amdgpu_kernel void @s_test_sdiv32_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-NEXT:    s_xor_b32 s4, s7, s6
 ; GCN-NEXT:    s_ashr_i32 s4, s4, 30
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-NEXT:    s_or_b32 s4, s4, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s4
 ; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s4, 0
+; GCN-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
@@ -647,16 +641,16 @@ define amdgpu_kernel void @s_test_sdiv32_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-IR-NEXT:    s_xor_b32 s4, s7, s6
 ; GCN-IR-NEXT:    s_ashr_i32 s4, s4, 30
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-IR-NEXT:    s_or_b32 s4, s4, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s4
 ; GCN-IR-NEXT:    s_mov_b32 s1, s5
+; GCN-IR-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u32 s4, 0
+; GCN-IR-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
@@ -684,15 +678,15 @@ define amdgpu_kernel void @s_test_sdiv31_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_xor_b32 s4, s4, s8
 ; GCN-NEXT:    s_ashr_i32 s4, s4, 30
-; GCN-NEXT:    s_or_b32 s4, s4, 1
+; GCN-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_mov_b32_e32 v3, s4
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s4, 0
+; GCN-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 31
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -714,15 +708,15 @@ define amdgpu_kernel void @s_test_sdiv31_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-IR-NEXT:    s_xor_b32 s4, s4, s8
 ; GCN-IR-NEXT:    s_ashr_i32 s4, s4, 30
-; GCN-IR-NEXT:    s_or_b32 s4, s4, 1
+; GCN-IR-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s4
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u32 s4, 0
+; GCN-IR-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 31
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -751,15 +745,15 @@ define amdgpu_kernel void @s_test_sdiv23_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_xor_b32 s4, s4, s8
 ; GCN-NEXT:    s_ashr_i32 s4, s4, 30
-; GCN-NEXT:    s_or_b32 s4, s4, 1
+; GCN-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_mov_b32_e32 v3, s4
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s4, 0
+; GCN-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 23
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -781,15 +775,15 @@ define amdgpu_kernel void @s_test_sdiv23_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-IR-NEXT:    s_xor_b32 s4, s4, s8
 ; GCN-IR-NEXT:    s_ashr_i32 s4, s4, 30
-; GCN-IR-NEXT:    s_or_b32 s4, s4, 1
+; GCN-IR-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s4
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u32 s4, 0
+; GCN-IR-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 23
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -818,15 +812,15 @@ define amdgpu_kernel void @s_test_sdiv25_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_xor_b32 s4, s4, s8
 ; GCN-NEXT:    s_ashr_i32 s4, s4, 30
-; GCN-NEXT:    s_or_b32 s4, s4, 1
+; GCN-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_mov_b32_e32 v3, s4
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s4, 0
+; GCN-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 25
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -848,15 +842,15 @@ define amdgpu_kernel void @s_test_sdiv25_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-IR-NEXT:    s_xor_b32 s4, s4, s8
 ; GCN-IR-NEXT:    s_ashr_i32 s4, s4, 30
-; GCN-IR-NEXT:    s_or_b32 s4, s4, 1
+; GCN-IR-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s4
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u32 s4, 0
+; GCN-IR-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 25
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -882,35 +876,35 @@ define amdgpu_kernel void @s_test_sdiv24_v2i64(<2 x i64> addrspace(1)* %out, <2
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s0
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s8
 ; GCN-NEXT:    s_xor_b32 s0, s8, s0
-; GCN-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-NEXT:    s_or_b32 s0, s0, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s0
 ; GCN-NEXT:    s_ashr_i64 s[2:3], s[2:3], 40
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_ashr_i64 s[10:11], s[10:11], 40
+; GCN-NEXT:    s_or_b32 s3, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    s_ashr_i64 s[10:11], s[10:11], 40
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cselect_b32 s0, s3, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s2
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s10
 ; GCN-NEXT:    s_xor_b32 s0, s10, s2
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
-; GCN-NEXT:    s_or_b32 s0, s0, 1
-; GCN-NEXT:    v_mov_b32_e32 v5, s0
+; GCN-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
+; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    v_mul_f32_e32 v4, v3, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v4, v4
 ; GCN-NEXT:    v_mad_f32 v3, -v4, v2, v3
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v5, vcc
-; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v3|, |v2|
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cselect_b32 s0, s2, 0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s0, v4
 ; GCN-NEXT:    v_bfe_i32 v2, v2, 0, 24
 ; GCN-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
 ; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
@@ -929,35 +923,35 @@ define amdgpu_kernel void @s_test_sdiv24_v2i64(<2 x i64> addrspace(1)* %out, <2
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s0
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v1, s8
 ; GCN-IR-NEXT:    s_xor_b32 s0, s8, s0
-; GCN-IR-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-IR-NEXT:    s_or_b32 s0, s0, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s0
 ; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[2:3], 40
+; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-IR-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-IR-NEXT:    s_ashr_i64 s[10:11], s[10:11], 40
+; GCN-IR-NEXT:    s_or_b32 s3, s0, 1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    s_ashr_i64 s[10:11], s[10:11], 40
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-IR-NEXT:    s_cselect_b32 s0, s3, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v2, s2
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v3, s10
 ; GCN-IR-NEXT:    s_xor_b32 s0, s10, s2
 ; GCN-IR-NEXT:    s_ashr_i32 s0, s0, 30
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v4, v2
-; GCN-IR-NEXT:    s_or_b32 s0, s0, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, s0
+; GCN-IR-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 24
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    v_mul_f32_e32 v4, v3, v4
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v4, v4
 ; GCN-IR-NEXT:    v_mad_f32 v3, -v4, v2, v3
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, 0, v5, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v3|, |v2|
+; GCN-IR-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-IR-NEXT:    s_cselect_b32 s0, s2, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, s0, v4
 ; GCN-IR-NEXT:    v_bfe_i32 v2, v2, 0, 24
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
 ; GCN-IR-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
@@ -998,7 +992,7 @@ define amdgpu_kernel void @s_test_sdiv24_48(i48 addrspace(1)* %out, i48 %x, i48
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
 ; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v1|
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
@@ -1014,105 +1008,99 @@ define amdgpu_kernel void @s_test_sdiv24_48(i48 addrspace(1)* %out, i48 %x, i48
 ; GCN-IR-NEXT:    s_load_dword s0, s[0:1], 0xe
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_sext_i32_i16 s3, s3
+; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[2:3], 24
 ; GCN-IR-NEXT:    s_sext_i32_i16 s7, s0
-; GCN-IR-NEXT:    s_ashr_i64 s[0:1], s[2:3], 24
-; GCN-IR-NEXT:    s_ashr_i32 s2, s3, 31
+; GCN-IR-NEXT:    s_ashr_i32 s0, s3, 31
+; GCN-IR-NEXT:    s_mov_b32 s1, s0
+; GCN-IR-NEXT:    s_ashr_i32 s2, s7, 31
+; GCN-IR-NEXT:    s_ashr_i64 s[12:13], s[6:7], 24
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[0:1], s[8:9]
+; GCN-IR-NEXT:    s_sub_u32 s10, s6, s0
 ; GCN-IR-NEXT:    s_mov_b32 s3, s2
-; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[6:7], 24
-; GCN-IR-NEXT:    s_ashr_i32 s6, s7, 31
-; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[2:3], s[0:1]
-; GCN-IR-NEXT:    s_sub_u32 s10, s0, s2
-; GCN-IR-NEXT:    s_mov_b32 s7, s6
-; GCN-IR-NEXT:    s_subb_u32 s11, s1, s2
-; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[6:7], s[8:9]
-; GCN-IR-NEXT:    s_sub_u32 s8, s0, s6
-; GCN-IR-NEXT:    s_flbit_i32_b32 s14, s8
-; GCN-IR-NEXT:    s_subb_u32 s9, s1, s6
-; GCN-IR-NEXT:    s_add_i32 s14, s14, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s15, s9
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s14
+; GCN-IR-NEXT:    s_subb_u32 s11, s7, s0
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[2:3], s[12:13]
+; GCN-IR-NEXT:    s_sub_u32 s6, s6, s2
+; GCN-IR-NEXT:    s_subb_u32 s7, s7, s2
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[6:7], 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[14:15], s[10:11], 0
+; GCN-IR-NEXT:    s_mov_b64 s[8:9], 0
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[12:13], s[14:15]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s6
+; GCN-IR-NEXT:    s_add_i32 s12, s12, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s14, s7
+; GCN-IR-NEXT:    s_cmp_eq_u32 s7, 0
+; GCN-IR-NEXT:    s_cselect_b32 s12, s12, s14
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s14, s10
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s15
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s9, 0
 ; GCN-IR-NEXT:    s_add_i32 s14, s14, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s15, s11
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s15
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s14
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s11, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v2, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[14:15], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[8:9], 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[10:11], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[12:13]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[12:13], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[12:13], s[12:13], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[12:13]
-; GCN-IR-NEXT:    s_cbranch_vccz BB9_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[4:5], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[10:11], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_cmp_eq_u32 s11, 0
+; GCN-IR-NEXT:    s_cselect_b32 s16, s14, s15
+; GCN-IR-NEXT:    s_sub_u32 s14, s12, s16
+; GCN-IR-NEXT:    s_subb_u32 s15, 0, 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[20:21], s[14:15], 63
+; GCN-IR-NEXT:    s_mov_b32 s13, 0
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[20:21]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[20:21], s[14:15], 63
+; GCN-IR-NEXT:    s_xor_b64 s[22:23], s[18:19], -1
+; GCN-IR-NEXT:    s_and_b64 s[20:21], s[22:23], s[20:21]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[20:21]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB9_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s18, s14, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s14
+; GCN-IR-NEXT:    s_addc_u32 s19, s15, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s15
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[18:19], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s14, 63, s14
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[10:11], s14
+; GCN-IR-NEXT:    s_cbranch_vccz BB9_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_not_b32_e32 v2, v2
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[10:11], v4
-; GCN-IR-NEXT:    s_add_u32 s10, s8, -1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v2, v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s11, s9, -1
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], -1, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[18:19], s[10:11], s18
+; GCN-IR-NEXT:    s_add_u32 s10, s6, -1
+; GCN-IR-NEXT:    s_addc_u32 s11, s7, -1
+; GCN-IR-NEXT:    s_not_b64 s[8:9], s[12:13]
+; GCN-IR-NEXT:    s_mov_b32 s17, s13
+; GCN-IR-NEXT:    s_add_u32 s12, s8, s16
+; GCN-IR-NEXT:    s_addc_u32 s13, s9, s13
+; GCN-IR-NEXT:    s_mov_b64 s[16:17], 0
+; GCN-IR-NEXT:    s_mov_b32 s9, 0
 ; GCN-IR-NEXT:  BB9_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s11
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s10, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s8, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s9, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s8, s15, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[18:19], s[18:19], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[8:9]
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[16:17], s[14:15]
+; GCN-IR-NEXT:    s_sub_u32 s8, s10, s18
+; GCN-IR-NEXT:    s_subb_u32 s8, s11, s19
+; GCN-IR-NEXT:    s_ashr_i32 s16, s8, 31
+; GCN-IR-NEXT:    s_mov_b32 s17, s16
+; GCN-IR-NEXT:    s_and_b32 s8, s16, 1
+; GCN-IR-NEXT:    s_and_b64 s[20:21], s[16:17], s[6:7]
+; GCN-IR-NEXT:    s_sub_u32 s18, s18, s20
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s12
+; GCN-IR-NEXT:    s_subb_u32 s19, s19, s21
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s13
+; GCN-IR-NEXT:    s_add_u32 s12, s12, 1
+; GCN-IR-NEXT:    s_addc_u32 s13, s13, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[16:17], s[8:9]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB9_3
+; GCN-IR-NEXT:  BB9_4: ; %Flow3
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[14:15], 1
+; GCN-IR-NEXT:    s_or_b64 s[6:7], s[8:9], s[6:7]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s7
 ; GCN-IR-NEXT:    s_branch BB9_6
-; GCN-IR-NEXT:  BB9_4:
+; GCN-IR-NEXT:  BB9_5:
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[18:19]
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB9_7
-; GCN-IR-NEXT:  BB9_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB9_6: ; %Flow3
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB9_7: ; %Flow4
-; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[6:7], s[2:3]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[18:19]
+; GCN-IR-NEXT:  BB9_6: ; %Flow4
+; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[2:3], s[0:1]
 ; GCN-IR-NEXT:    v_xor_b32_e32 v0, s0, v0
 ; GCN-IR-NEXT:    v_xor_b32_e32 v1, s1, v1
 ; GCN-IR-NEXT:    v_mov_b32_e32 v2, s1
@@ -1136,15 +1124,15 @@ define amdgpu_kernel void @s_test_sdiv_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-NEXT:    v_mov_b32_e32 v2, 0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_ashr_i32 s2, s7, 31
-; GCN-NEXT:    s_add_u32 s0, s6, s2
-; GCN-NEXT:    s_addc_u32 s1, s7, s2
-; GCN-NEXT:    s_mov_b32 s3, s2
-; GCN-NEXT:    s_xor_b64 s[8:9], s[0:1], s[2:3]
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s8
-; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s9
-; GCN-NEXT:    s_sub_u32 s3, 0, s8
-; GCN-NEXT:    s_subb_u32 s10, 0, s9
+; GCN-NEXT:    s_ashr_i32 s8, s7, 31
+; GCN-NEXT:    s_add_u32 s0, s6, s8
+; GCN-NEXT:    s_addc_u32 s1, s7, s8
+; GCN-NEXT:    s_mov_b32 s9, s8
+; GCN-NEXT:    s_xor_b64 s[10:11], s[0:1], s[8:9]
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s10
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s11
+; GCN-NEXT:    s_sub_u32 s2, 0, s10
+; GCN-NEXT:    s_subb_u32 s3, 0, s11
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
@@ -1156,10 +1144,10 @@ define amdgpu_kernel void @s_test_sdiv_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v3
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GCN-NEXT:    v_mul_hi_u32 v5, s3, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s3, v3
-; GCN-NEXT:    v_mul_lo_u32 v7, s10, v0
-; GCN-NEXT:    v_mul_lo_u32 v6, s3, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s2, v3
+; GCN-NEXT:    v_mul_lo_u32 v7, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v6, s2, v0
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
 ; GCN-NEXT:    v_mul_hi_u32 v5, v0, v6
@@ -1178,11 +1166,11 @@ define amdgpu_kernel void @s_test_sdiv_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v4
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v2, v6, vcc
 ; GCN-NEXT:    v_addc_u32_e64 v4, vcc, v3, v5, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v6, s3, v4
-; GCN-NEXT:    v_mul_hi_u32 v7, s3, v0
-; GCN-NEXT:    v_mul_lo_u32 v8, s10, v0
+; GCN-NEXT:    v_mul_lo_u32 v6, s2, v4
+; GCN-NEXT:    v_mul_hi_u32 v7, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v8, s3, v0
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
-; GCN-NEXT:    v_mul_lo_u32 v7, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v7, s2, v0
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
 ; GCN-NEXT:    v_mul_lo_u32 v10, v0, v6
 ; GCN-NEXT:    v_mul_hi_u32 v12, v0, v6
@@ -1212,139 +1200,133 @@ define amdgpu_kernel void @s_test_sdiv_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0, v4
 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v2, v0, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, s8, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, s8, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s9, v0
-; GCN-NEXT:    v_mov_b32_e32 v5, s9
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s11, v0
+; GCN-NEXT:    v_mov_b32_e32 v5, s11
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_mul_lo_u32 v3, s8, v0
+; GCN-NEXT:    v_mul_lo_u32 v3, s10, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 24, v3
-; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s8, v3
-; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s8, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s9, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v5, s[0:1], 2, v0
-; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 1, v0
-; GCN-NEXT:    v_subb_u32_e32 v2, vcc, 0, v2, vcc
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[0:1]
+; GCN-NEXT:    v_sub_i32_e64 v3, s[0:1], 24, v3
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v4, v5, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s10, v3
+; GCN-NEXT:    v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s11, v4
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s8, v3
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s10, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s11, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, 0, v2, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v4
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s11, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s10, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s9, v2
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s11, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GCN-NEXT:    v_xor_b32_e32 v0, s2, v0
-; GCN-NEXT:    v_xor_b32_e32 v1, s2, v1
-; GCN-NEXT:    v_mov_b32_e32 v2, s2
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
+; GCN-NEXT:    v_xor_b32_e32 v0, s8, v0
+; GCN-NEXT:    v_xor_b32_e32 v1, s8, v1
+; GCN-NEXT:    v_mov_b32_e32 v2, s8
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s8, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_sdiv_k_num_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
-; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_ashr_i32 s2, s7, 31
-; GCN-IR-NEXT:    s_mov_b32 s3, s2
-; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[2:3], s[6:7]
-; GCN-IR-NEXT:    s_sub_u32 s6, s0, s2
-; GCN-IR-NEXT:    s_subb_u32 s7, s1, s2
-; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s6
-; GCN-IR-NEXT:    s_add_i32 s8, s8, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s9, s7
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s9
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s8
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 0xffffffc5, v2
-; GCN-IR-NEXT:    v_addc_u32_e64 v1, s[8:9], 0, -1, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[8:9], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[8:9], s[8:9], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT:    s_cbranch_vccz BB10_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[3:4], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], 24, v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_ashr_i32 s4, s3, 31
+; GCN-IR-NEXT:    s_mov_b32 s5, s4
+; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[4:5], s[2:3]
+; GCN-IR-NEXT:    s_sub_u32 s2, s2, s4
+; GCN-IR-NEXT:    s_flbit_i32_b32 s6, s2
+; GCN-IR-NEXT:    s_subb_u32 s3, s3, s4
+; GCN-IR-NEXT:    s_add_i32 s6, s6, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s7, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s10, s6, s7
+; GCN-IR-NEXT:    s_add_u32 s8, s10, 0xffffffc5
+; GCN-IR-NEXT:    s_addc_u32 s9, 0, -1
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[2:3], 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[14:15], s[8:9], 63
+; GCN-IR-NEXT:    s_mov_b64 s[6:7], 0
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[14:15]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[14:15], s[8:9], 63
+; GCN-IR-NEXT:    s_xor_b64 s[16:17], s[12:13], -1
+; GCN-IR-NEXT:    s_and_b64 s[14:15], s[16:17], s[14:15]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[14:15]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB10_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s14, s8, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_addc_u32 s15, s9, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[14:15], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s8, 63, s8
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[12:13], 24, s8
+; GCN-IR-NEXT:    s_cbranch_vccz BB10_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_add_u32 s8, s6, -1
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], 24, v3
-; GCN-IR-NEXT:    v_sub_i32_e32 v4, vcc, 58, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s9, s7, -1
-; GCN-IR-NEXT:    v_subb_u32_e64 v5, s[0:1], 0, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[16:17], 24, s14
+; GCN-IR-NEXT:    s_add_u32 s8, s2, -1
+; GCN-IR-NEXT:    s_addc_u32 s9, s3, -1
+; GCN-IR-NEXT:    s_sub_u32 s10, 58, s10
+; GCN-IR-NEXT:    s_subb_u32 s11, 0, 0
+; GCN-IR-NEXT:    s_mov_b64 s[14:15], 0
+; GCN-IR-NEXT:    s_mov_b32 s7, 0
 ; GCN-IR-NEXT:  BB10_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s9
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s8, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s6, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s7, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s6, s13, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[16:17], s[16:17], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[12:13], s[12:13], 1
+; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[6:7]
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[14:15], s[12:13]
+; GCN-IR-NEXT:    s_sub_u32 s6, s8, s16
+; GCN-IR-NEXT:    s_subb_u32 s6, s9, s17
+; GCN-IR-NEXT:    s_ashr_i32 s14, s6, 31
+; GCN-IR-NEXT:    s_mov_b32 s15, s14
+; GCN-IR-NEXT:    s_and_b32 s6, s14, 1
+; GCN-IR-NEXT:    s_and_b64 s[18:19], s[14:15], s[2:3]
+; GCN-IR-NEXT:    s_sub_u32 s16, s16, s18
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-IR-NEXT:    s_subb_u32 s17, s17, s19
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-IR-NEXT:    s_add_u32 s10, s10, 1
+; GCN-IR-NEXT:    s_addc_u32 s11, s11, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[10:11], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[14:15], s[6:7]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB10_3
+; GCN-IR-NEXT:  BB10_4: ; %Flow5
+; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[12:13], 1
+; GCN-IR-NEXT:    s_or_b64 s[2:3], s[6:7], s[2:3]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-IR-NEXT:    s_branch BB10_6
-; GCN-IR-NEXT:  BB10_4:
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, 24, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB10_7
 ; GCN-IR-NEXT:  BB10_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB10_6: ; %Flow5
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB10_7: ; %udiv-end
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, s2, v0
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, s3, v1
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s3
-; GCN-IR-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, 24, 0, s[12:13]
+; GCN-IR-NEXT:  BB10_6: ; %udiv-end
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, s4, v0
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, s5, v1
+; GCN-IR-NEXT:    v_mov_b32_e32 v2, s5
+; GCN-IR-NEXT:    v_subrev_i32_e32 v0, vcc, s4, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
-; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-IR-NEXT:    s_mov_b32 s6, -1
-; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-IR-NEXT:    s_mov_b32 s2, -1
+; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %result = sdiv i64 24, %x
   store i64 %result, i64 addrspace(1)* %out
@@ -1895,16 +1877,16 @@ define amdgpu_kernel void @s_test_sdiv24_k_num_i64(i64 addrspace(1)* %out, i64 %
 ; GCN-NEXT:    s_mov_b32 s0, s4
 ; GCN-NEXT:    s_ashr_i32 s4, s6, 30
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v0
-; GCN-NEXT:    s_or_b32 s4, s4, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s4
 ; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-NEXT:    v_mul_f32_e32 v1, s7, v1
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-NEXT:    v_mad_f32 v2, -v1, v0, s7
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s4, 0
+; GCN-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s4, v1
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -1922,16 +1904,16 @@ define amdgpu_kernel void @s_test_sdiv24_k_num_i64(i64 addrspace(1)* %out, i64 %
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
 ; GCN-IR-NEXT:    s_ashr_i32 s4, s6, 30
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v1, v0
-; GCN-IR-NEXT:    s_or_b32 s4, s4, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s4
 ; GCN-IR-NEXT:    s_mov_b32 s1, s5
+; GCN-IR-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v1, s7, v1
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-IR-NEXT:    v_mad_f32 v2, -v1, v0, s7
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u32 s4, 0
+; GCN-IR-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s4, v1
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -1954,17 +1936,17 @@ define amdgpu_kernel void @s_test_sdiv24_k_den_i64(i64 addrspace(1)* %out, i64 %
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s6
 ; GCN-NEXT:    s_mov_b32 s0, s4
 ; GCN-NEXT:    s_ashr_i32 s4, s6, 30
-; GCN-NEXT:    s_or_b32 s4, s4, 1
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_mul_f32_e32 v1, 0x38331158, v0
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-NEXT:    v_mad_f32 v0, -v1, s8, v0
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; GCN-NEXT:    v_mov_b32_e32 v2, s4
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, s8
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v2, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    s_or_b32 s6, s4, 1
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v0|, s8
+; GCN-NEXT:    s_cmp_lg_u32 s4, 0
+; GCN-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s4, v1
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
-; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
@@ -1980,17 +1962,17 @@ define amdgpu_kernel void @s_test_sdiv24_k_den_i64(i64 addrspace(1)* %out, i64 %
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s6
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
 ; GCN-IR-NEXT:    s_ashr_i32 s4, s6, 30
-; GCN-IR-NEXT:    s_or_b32 s4, s4, 1
+; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_mul_f32_e32 v1, 0x38331158, v0
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-IR-NEXT:    v_mad_f32 v0, -v1, s8, v0
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s4
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, s8
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v2, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-IR-NEXT:    s_or_b32 s6, s4, 1
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v0|, s8
+; GCN-IR-NEXT:    s_cmp_lg_u32 s4, 0
+; GCN-IR-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s4, v1
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 24
-; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/select-i1.ll b/llvm/test/CodeGen/AMDGPU/select-i1.ll
index 8e60aa1811f41..951449c463d07 100644
--- a/llvm/test/CodeGen/AMDGPU/select-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-i1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 
 ; FIXME: This should go in existing select.ll test, except the current testcase there is broken on GCN
diff --git a/llvm/test/CodeGen/AMDGPU/select-opt.ll b/llvm/test/CodeGen/AMDGPU/select-opt.ll
index 24df126e4cafc..7054b013b0a6d 100644
--- a/llvm/test/CodeGen/AMDGPU/select-opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-opt.ll
@@ -7,8 +7,10 @@
 ; GCN-LABEL: {{^}}opt_select_i32_and_cmp_i32:
 ; GCN-DAG: v_cmp_ne_u32_e32 vcc,
 ; GCN-DAG: v_cmp_ne_u32_e64 [[CMP1:s\[[0-9]+:[0-9]+\]]]
-; GCN: s_and_b64 vcc, vcc, [[CMP1]]
-; GCN: v_cndmask_b32_e32 [[RESULT:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}}, vcc
+; GCN: s_and_b64 [[CMP1]], vcc, [[CMP1]]
+; GCN: s_cselect_b32 [[SRESULT:s[0-9]+]], {{s[0-9]+}}, {{s[0-9]+}}
+; GCN-NOT: [[SRESULT]]
+; GCN: v_mov_b32_e32 [[RESULT:v[0-9]+]], [[SRESULT]]
 ; GCN-NOT: [[RESULT]]
 ; GCN: buffer_store_dword [[RESULT]]
 define amdgpu_kernel void @opt_select_i32_and_cmp_i32(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c, i32 %x, i32 %y) #0 {
@@ -23,8 +25,10 @@ define amdgpu_kernel void @opt_select_i32_and_cmp_i32(i32 addrspace(1)* %out, i3
 ; GCN-LABEL: {{^}}opt_select_i32_and_cmp_f32:
 ; GCN-DAG: v_cmp_lg_f32_e32 vcc
 ; GCN-DAG: v_cmp_lg_f32_e64 [[CMP1:s\[[0-9]+:[0-9]+\]]]
-; GCN: s_and_b64 vcc, vcc, [[CMP1]]
-; GCN: v_cndmask_b32_e32 [[RESULT:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}}, vcc
+; GCN: s_and_b64 [[CMP1]], vcc, [[CMP1]]
+; GCN: s_cselect_b32 [[SRESULT:s[0-9]+]], {{s[0-9]+}}, {{s[0-9]+}}
+; GCN-NOT: [[SRESULT]]
+; GCN: v_mov_b32_e32 [[RESULT:v[0-9]+]], [[SRESULT]]
 ; GCN-NOT: [[RESULT]]
 ; GCN: buffer_store_dword [[RESULT]]
 define amdgpu_kernel void @opt_select_i32_and_cmp_f32(i32 addrspace(1)* %out, float %a, float %b, float %c, i32 %x, i32 %y) #0 {
@@ -71,8 +75,10 @@ define amdgpu_kernel void @opt_select_i64_and_cmp_f32(i64 addrspace(1)* %out, fl
 ; GCN-LABEL: {{^}}opt_select_i32_or_cmp_i32:
 ; GCN-DAG: v_cmp_ne_u32_e32 vcc,
 ; GCN-DAG: v_cmp_ne_u32_e64 [[CMP1:s\[[0-9]+:[0-9]+\]]]
-; GCN: s_or_b64 vcc, vcc, [[CMP1]]
-; GCN: v_cndmask_b32_e32 [[RESULT:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}}, vcc
+; GCN: s_or_b64 [[CMP1]], vcc, [[CMP1]]
+; GCN: s_cselect_b32 [[SRESULT:s[0-9]+]], {{s[0-9]+}}, {{s[0-9]+}}
+; GCN-NOT: [[SRESULT]]
+; GCN: v_mov_b32_e32 [[RESULT:v[0-9]+]], [[SRESULT]]
 ; GCN-NOT: [[RESULT]]
 ; GCN: buffer_store_dword [[RESULT]]
 ; GCN: s_endpgm
@@ -88,8 +94,10 @@ define amdgpu_kernel void @opt_select_i32_or_cmp_i32(i32 addrspace(1)* %out, i32
 ; GCN-LABEL: {{^}}opt_select_i32_or_cmp_f32:
 ; GCN-DAG: v_cmp_lg_f32_e32 vcc
 ; GCN-DAG: v_cmp_lg_f32_e64 [[CMP1:s\[[0-9]+:[0-9]+\]]]
-; GCN: s_or_b64 vcc, vcc, [[CMP1]]
-; GCN: v_cndmask_b32_e32 [[RESULT:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}}, vcc
+; GCN: s_or_b64 [[CMP1]], vcc, [[CMP1]]
+; GCN: s_cselect_b32 [[SRESULT:s[0-9]+]], {{s[0-9]+}}, {{s[0-9]+}}
+; GCN-NOT: [[SRESULT]]
+; GCN: v_mov_b32_e32 [[RESULT:v[0-9]+]], [[SRESULT]]
 ; GCN-NOT: [[RESULT]]
 ; GCN: buffer_store_dword [[RESULT]]
 define amdgpu_kernel void @opt_select_i32_or_cmp_f32(i32 addrspace(1)* %out, float %a, float %b, float %c, i32 %x, i32 %y) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/select-vectors.ll b/llvm/test/CodeGen/AMDGPU/select-vectors.ll
index 4c136d09c4ab7..9e6d1a7cf76b2 100644
--- a/llvm/test/CodeGen/AMDGPU/select-vectors.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-vectors.ll
@@ -66,8 +66,8 @@ define amdgpu_kernel void @v_select_v16i8(<16 x i8> addrspace(1)* %out, <16 x i8
 }
 
 ; GCN-LABEL: {{^}}select_v4i8:
-; GCN: v_cndmask_b32
-; GCN-NOT: cndmask
+; GCN: s_cselect_b32
+; GCN-NOT: s_cselect_b32
 define amdgpu_kernel void @select_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> %a, <4 x i8> %b, i8 %c) #0 {
   %cmp = icmp eq i8 %c, 0
   %select = select i1 %cmp, <4 x i8> %a, <4 x i8> %b
@@ -79,11 +79,11 @@ define amdgpu_kernel void @select_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> %a,
 ; GFX89: s_load_dword
 ; GFX89: s_load_dword
 ; GFX89: s_load_dword
-; GFX89: v_cndmask_b32
-; GFX89-NOT: v_cndmask_b32
+; GFX89: s_cselect_b32
+; GFX89-NOT: s_cselect_b32
 
-; SI: v_cndmask_b32_e32
-; SI-NOT: v_cndmask_b32e
+; SI: s_cselect_b32
+; SI-NOT: s_cselect_b32
 define amdgpu_kernel void @select_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> %a, <2 x i16> %b, i32 %c) #0 {
   %cmp = icmp eq i32 %c, 0
   %select = select i1 %cmp, <2 x i16> %a, <2 x i16> %b
diff --git a/llvm/test/CodeGen/AMDGPU/select64.ll b/llvm/test/CodeGen/AMDGPU/select64.ll
index 0a48227f5c500..9c492dcfb3512 100644
--- a/llvm/test/CodeGen/AMDGPU/select64.ll
+++ b/llvm/test/CodeGen/AMDGPU/select64.ll
@@ -16,8 +16,8 @@ entry:
 }
 
 ; CHECK-LABEL: {{^}}select_trunc_i64:
-; CHECK: v_cndmask_b32
-; CHECK-NOT: v_cndmask_b32
+; CHECK: s_cselect_b32
+; CHECK-NOT: s_cselect_b32
 define amdgpu_kernel void @select_trunc_i64(i32 addrspace(1)* %out, i32 %cond, i64 %in) nounwind {
   %cmp = icmp ugt i32 %cond, 5
   %sel = select i1 %cmp, i64 0, i64 %in
@@ -27,8 +27,8 @@ define amdgpu_kernel void @select_trunc_i64(i32 addrspace(1)* %out, i32 %cond, i
 }
 
 ; CHECK-LABEL: {{^}}select_trunc_i64_2:
-; CHECK: v_cndmask_b32
-; CHECK-NOT: v_cndmask_b32
+; CHECK: s_cselect_b32
+; CHECK-NOT: s_cselect_b32
 define amdgpu_kernel void @select_trunc_i64_2(i32 addrspace(1)* %out, i32 %cond, i64 %a, i64 %b) nounwind {
   %cmp = icmp ugt i32 %cond, 5
   %sel = select i1 %cmp, i64 %a, i64 %b
@@ -38,8 +38,8 @@ define amdgpu_kernel void @select_trunc_i64_2(i32 addrspace(1)* %out, i32 %cond,
 }
 
 ; CHECK-LABEL: {{^}}v_select_trunc_i64_2:
-; CHECK: v_cndmask_b32
-; CHECK-NOT: v_cndmask_b32
+; CHECK: s_cselect_b32
+; CHECK-NOT: s_cselect_b32
 define amdgpu_kernel void @v_select_trunc_i64_2(i32 addrspace(1)* %out, i32 %cond, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind {
   %cmp = icmp ugt i32 %cond, 5
   %a = load i64, i64 addrspace(1)* %aptr, align 8
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll b/llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll
index 187caffdade9f..596225fac66aa 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll
@@ -11,32 +11,32 @@
 define amdgpu_kernel void @sgpr_if_else_salu_br(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) {
 ; SI-LABEL: sgpr_if_else_salu_br:
 ; SI:       ; %bb.0: ; %entry
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; SI-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0xb
-; SI-NEXT:    s_load_dword s0, s[0:1], 0xf
-; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_cmp_lg_u32 s8, 0
-; SI-NEXT:    s_cbranch_scc0 BB0_2
-; SI-NEXT:  ; %bb.1: ; %else
-; SI-NEXT:    s_add_i32 s0, s11, s0
-; SI-NEXT:    s_mov_b64 s[2:3], 0
-; SI-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
-; SI-NEXT:    s_cbranch_vccz BB0_3
-; SI-NEXT:    s_branch BB0_4
-; SI-NEXT:  BB0_2:
-; SI-NEXT:    s_mov_b64 s[2:3], -1
-; SI-NEXT:    ; implicit-def: $sgpr0
-; SI-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
-; SI-NEXT:    s_cbranch_vccnz BB0_4
-; SI-NEXT:  BB0_3: ; %if
-; SI-NEXT:    s_sub_i32 s0, s9, s10
-; SI-NEXT:  BB0_4: ; %endif
-; SI-NEXT:    s_add_i32 s0, s0, s8
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s0
-; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; SI-NEXT:    s_endpgm
+; SI-NEXT:	s_load_dwordx4 s[4:7], s[0:1], 0xb
+; SI-NEXT:	s_load_dword s2, s[0:1], 0xf
+; SI-NEXT:	s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:	s_waitcnt lgkmcnt(0)
+; SI-NEXT:	s_cmp_lg_u32 s4, 0
+; SI-NEXT:	s_cbranch_scc0 BB0_2
+; SI-NEXT:; %bb.1:                                ; %else
+; SI-NEXT:	s_add_i32 s2, s7, s2
+; SI-NEXT:	s_mov_b64 s[8:9], 0
+; SI-NEXT:	s_andn2_b64 vcc, exec, s[8:9]
+; SI-NEXT:	s_cbranch_vccz BB0_3
+; SI-NEXT:	s_branch BB0_4
+; SI-NEXT:BB0_2:
+; SI-NEXT:	s_mov_b64 s[8:9], -1
+; SI-NEXT:                                        ; implicit-def: $sgpr2
+; SI-NEXT:	s_andn2_b64 vcc, exec, s[8:9]
+; SI-NEXT:	s_cbranch_vccnz BB0_4
+; SI-NEXT:BB0_3:                                  ; %if
+; SI-NEXT:	s_sub_i32 s2, s5, s6
+; SI-NEXT:BB0_4:                                  ; %endif
+; SI-NEXT:	s_add_i32 s4, s2, s4
+; SI-NEXT:	s_mov_b32 s3, 0xf000
+; SI-NEXT:	s_mov_b32 s2, -1
+; SI-NEXT:	v_mov_b32_e32 v0, s4
+; SI-NEXT:	buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:	s_endpgm
 entry:
   %0 = icmp eq i32 %a, 0
   br i1 %0, label %if, label %else
diff --git a/llvm/test/CodeGen/AMDGPU/shift-i128.ll b/llvm/test/CodeGen/AMDGPU/shift-i128.ll
index f2077aa2a1ad0..59aebaeed56e9 100644
--- a/llvm/test/CodeGen/AMDGPU/shift-i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/shift-i128.ll
@@ -446,68 +446,68 @@ define <2 x i128> @v_ashr_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 define amdgpu_kernel void @s_shl_v2i128ss(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; GCN-LABEL: s_shl_v2i128ss:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx8 s[16:23], s[4:5], 0x8
-; GCN-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x0
-; GCN-NEXT:    v_mov_b32_e32 v10, 16
-; GCN-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-NEXT:    v_mov_b32_e32 v11, 0
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_sub_i32 s6, 64, s16
-; GCN-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[16:17], 64
-; GCN-NEXT:    v_cmp_eq_u64_e64 s[2:3], s[18:19], 0
-; GCN-NEXT:    s_sub_i32 s4, s16, 64
-; GCN-NEXT:    s_lshr_b64 s[6:7], s[8:9], s6
-; GCN-NEXT:    s_lshl_b64 s[24:25], s[10:11], s16
-; GCN-NEXT:    s_and_b64 vcc, s[2:3], s[0:1]
-; GCN-NEXT:    s_or_b64 s[0:1], s[16:17], s[18:19]
-; GCN-NEXT:    s_lshl_b64 s[4:5], s[8:9], s4
-; GCN-NEXT:    s_or_b64 s[6:7], s[24:25], s[6:7]
-; GCN-NEXT:    v_mov_b32_e32 v0, s5
-; GCN-NEXT:    v_mov_b32_e32 v1, s7
-; GCN-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[0:1], 0
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-NEXT:    v_mov_b32_e32 v1, s11
-; GCN-NEXT:    v_cndmask_b32_e64 v3, v0, v1, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v0, s4
-; GCN-NEXT:    v_mov_b32_e32 v1, s6
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-NEXT:    v_mov_b32_e32 v1, s10
-; GCN-NEXT:    s_sub_i32 s6, 64, s20
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v0, v1, s[0:1]
-; GCN-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[20:21], 64
-; GCN-NEXT:    v_cmp_eq_u64_e64 s[2:3], s[22:23], 0
-; GCN-NEXT:    s_sub_i32 s4, s20, 64
-; GCN-NEXT:    s_lshr_b64 s[6:7], s[12:13], s6
-; GCN-NEXT:    s_lshl_b64 s[10:11], s[14:15], s20
-; GCN-NEXT:    s_lshl_b64 s[4:5], s[12:13], s4
-; GCN-NEXT:    s_or_b64 s[6:7], s[10:11], s[6:7]
-; GCN-NEXT:    s_and_b64 s[0:1], s[2:3], s[0:1]
-; GCN-NEXT:    s_or_b64 s[2:3], s[20:21], s[22:23]
-; GCN-NEXT:    v_mov_b32_e32 v0, s5
-; GCN-NEXT:    v_mov_b32_e32 v1, s7
-; GCN-NEXT:    v_cmp_eq_u64_e64 s[2:3], s[2:3], 0
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v1, s15
-; GCN-NEXT:    v_cndmask_b32_e64 v7, v0, v1, s[2:3]
-; GCN-NEXT:    v_mov_b32_e32 v0, s4
-; GCN-NEXT:    v_mov_b32_e32 v1, s6
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v1, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v1, s14
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v0, v1, s[2:3]
-; GCN-NEXT:    s_lshl_b64 s[2:3], s[8:9], s16
-; GCN-NEXT:    v_mov_b32_e32 v0, s3
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v0, vcc
-; GCN-NEXT:    v_mov_b32_e32 v0, s2
-; GCN-NEXT:    s_lshl_b64 s[2:3], s[12:13], s20
-; GCN-NEXT:    v_mov_b32_e32 v4, s3
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, v4, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v4, s2
-; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, v4, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
-; GCN-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
-; GCN-NEXT:    s_endpgm
+; GCN-NEXT:	s_load_dwordx8 s[8:15], s[4:5], 0x0
+; GCN-NEXT:	s_load_dwordx8 s[16:23], s[4:5], 0x8
+; GCN-NEXT:	v_mov_b32_e32 v10, 16
+; GCN-NEXT:	v_mov_b32_e32 v8, 0
+; GCN-NEXT:	v_mov_b32_e32 v11, 0
+; GCN-NEXT:	v_mov_b32_e32 v9, 0
+; GCN-NEXT:	s_waitcnt lgkmcnt(0)
+; GCN-NEXT:	s_sub_i32 s6, 64, s16
+; GCN-NEXT:	v_cmp_lt_u64_e64 s[0:1], s[16:17], 64
+; GCN-NEXT:	v_cmp_eq_u64_e64 s[2:3], s[18:19], 0
+; GCN-NEXT:	s_sub_i32 s4, s16, 64
+; GCN-NEXT:	s_lshr_b64 s[6:7], s[8:9], s6
+; GCN-NEXT:	s_lshl_b64 s[24:25], s[10:11], s16
+; GCN-NEXT:	s_and_b64 vcc, s[2:3], s[0:1]
+; GCN-NEXT:	s_or_b64 s[0:1], s[16:17], s[18:19]
+; GCN-NEXT:	s_lshl_b64 s[4:5], s[8:9], s4
+; GCN-NEXT:	s_or_b64 s[6:7], s[24:25], s[6:7]
+; GCN-NEXT:	v_mov_b32_e32 v0, s5
+; GCN-NEXT:	v_mov_b32_e32 v1, s7
+; GCN-NEXT:	v_cmp_eq_u64_e64 s[0:1], s[0:1], 0
+; GCN-NEXT:	v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-NEXT:	v_mov_b32_e32 v1, s11
+; GCN-NEXT:	v_cndmask_b32_e64 v3, v0, v1, s[0:1]
+; GCN-NEXT:	v_mov_b32_e32 v0, s4
+; GCN-NEXT:	v_mov_b32_e32 v1, s6
+; GCN-NEXT:	v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-NEXT:	v_mov_b32_e32 v1, s10
+; GCN-NEXT:	s_sub_i32 s6, 64, s20
+; GCN-NEXT:	v_cndmask_b32_e64 v2, v0, v1, s[0:1]
+; GCN-NEXT:	v_cmp_lt_u64_e64 s[0:1], s[20:21], 64
+; GCN-NEXT:	v_cmp_eq_u64_e64 s[2:3], s[22:23], 0
+; GCN-NEXT:	s_sub_i32 s4, s20, 64
+; GCN-NEXT:	s_lshr_b64 s[6:7], s[12:13], s6
+; GCN-NEXT:	s_lshl_b64 s[10:11], s[14:15], s20
+; GCN-NEXT:	s_lshl_b64 s[4:5], s[12:13], s4
+; GCN-NEXT:	s_or_b64 s[6:7], s[10:11], s[6:7]
+; GCN-NEXT:	s_and_b64 s[0:1], s[2:3], s[0:1]
+; GCN-NEXT:	s_or_b64 s[2:3], s[20:21], s[22:23]
+; GCN-NEXT:	v_mov_b32_e32 v0, s5
+; GCN-NEXT:	v_mov_b32_e32 v1, s7
+; GCN-NEXT:	v_cmp_eq_u64_e64 s[2:3], s[2:3], 0
+; GCN-NEXT:	v_cndmask_b32_e64 v0, v0, v1, s[0:1]
+; GCN-NEXT:	v_mov_b32_e32 v1, s15
+; GCN-NEXT:	v_cndmask_b32_e64 v7, v0, v1, s[2:3]
+; GCN-NEXT:	v_mov_b32_e32 v0, s4
+; GCN-NEXT:	v_mov_b32_e32 v1, s6
+; GCN-NEXT:	v_cndmask_b32_e64 v0, v0, v1, s[0:1]
+; GCN-NEXT:	v_mov_b32_e32 v1, s14
+; GCN-NEXT:	v_cndmask_b32_e64 v6, v0, v1, s[2:3]
+; GCN-NEXT:	s_lshl_b64 s[2:3], s[8:9], s16
+; GCN-NEXT:	v_mov_b32_e32 v0, s3
+; GCN-NEXT:	v_cndmask_b32_e32 v1, 0, v0, vcc
+; GCN-NEXT:	v_mov_b32_e32 v0, s2
+; GCN-NEXT:	s_lshl_b64 s[2:3], s[12:13], s20
+; GCN-NEXT:	v_mov_b32_e32 v4, s3
+; GCN-NEXT:	v_cndmask_b32_e64 v5, 0, v4, s[0:1]
+; GCN-NEXT:	v_mov_b32_e32 v4, s2
+; GCN-NEXT:	v_cndmask_b32_e64 v4, 0, v4, s[0:1]
+; GCN-NEXT:	v_cndmask_b32_e32 v0, 0, v0, vcc
+; GCN-NEXT:	flat_store_dwordx4 v[10:11], v[4:7]
+; GCN-NEXT:	flat_store_dwordx4 v[8:9], v[0:3]
+; GCN-NEXT:	s_endpgm
   %shift = shl <2 x i128> %lhs, %rhs
   store <2 x i128> %shift, <2 x i128> addrspace(1)* null
   ret void
@@ -516,68 +516,68 @@ define amdgpu_kernel void @s_shl_v2i128ss(<2 x i128> %lhs, <2 x i128> %rhs) {
 define amdgpu_kernel void @s_lshr_v2i128_ss(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; GCN-LABEL: s_lshr_v2i128_ss:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx8 s[16:23], s[4:5], 0x8
-; GCN-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x0
-; GCN-NEXT:    v_mov_b32_e32 v10, 16
-; GCN-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-NEXT:    v_mov_b32_e32 v11, 0
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_sub_i32 s6, 64, s16
-; GCN-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[16:17], 64
-; GCN-NEXT:    v_cmp_eq_u64_e64 s[2:3], s[18:19], 0
-; GCN-NEXT:    s_sub_i32 s4, s16, 64
-; GCN-NEXT:    s_lshl_b64 s[6:7], s[10:11], s6
-; GCN-NEXT:    s_lshr_b64 s[24:25], s[8:9], s16
-; GCN-NEXT:    s_or_b64 s[6:7], s[24:25], s[6:7]
-; GCN-NEXT:    s_and_b64 vcc, s[2:3], s[0:1]
-; GCN-NEXT:    s_or_b64 s[0:1], s[16:17], s[18:19]
-; GCN-NEXT:    s_lshr_b64 s[4:5], s[10:11], s4
-; GCN-NEXT:    v_mov_b32_e32 v0, s5
-; GCN-NEXT:    v_mov_b32_e32 v1, s7
-; GCN-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[0:1], 0
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-NEXT:    v_mov_b32_e32 v1, s9
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v0, v1, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v0, s4
-; GCN-NEXT:    v_mov_b32_e32 v2, s6
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    v_mov_b32_e32 v2, s8
-; GCN-NEXT:    s_sub_i32 s6, 64, s20
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[20:21], 64
-; GCN-NEXT:    v_cmp_eq_u64_e64 s[2:3], s[22:23], 0
-; GCN-NEXT:    s_sub_i32 s4, s20, 64
-; GCN-NEXT:    s_lshl_b64 s[6:7], s[14:15], s6
-; GCN-NEXT:    s_lshr_b64 s[8:9], s[12:13], s20
-; GCN-NEXT:    s_lshr_b64 s[4:5], s[14:15], s4
-; GCN-NEXT:    s_or_b64 s[6:7], s[8:9], s[6:7]
-; GCN-NEXT:    s_and_b64 s[0:1], s[2:3], s[0:1]
-; GCN-NEXT:    s_or_b64 s[2:3], s[20:21], s[22:23]
-; GCN-NEXT:    v_mov_b32_e32 v2, s5
-; GCN-NEXT:    v_mov_b32_e32 v3, s7
-; GCN-NEXT:    v_cmp_eq_u64_e64 s[2:3], s[2:3], 0
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v3, s13
-; GCN-NEXT:    v_cndmask_b32_e64 v5, v2, v3, s[2:3]
-; GCN-NEXT:    v_mov_b32_e32 v2, s4
-; GCN-NEXT:    v_mov_b32_e32 v3, s6
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v3, s12
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v2, v3, s[2:3]
-; GCN-NEXT:    s_lshr_b64 s[2:3], s[10:11], s16
-; GCN-NEXT:    v_mov_b32_e32 v2, s3
-; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v2, vcc
-; GCN-NEXT:    v_mov_b32_e32 v2, s2
-; GCN-NEXT:    s_lshr_b64 s[2:3], s[14:15], s20
-; GCN-NEXT:    v_mov_b32_e32 v6, s3
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, v6, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v6, s2
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, v6, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
-; GCN-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
-; GCN-NEXT:    s_endpgm
+; GCN-NEXT:	s_load_dwordx8 s[8:15], s[4:5], 0x0
+; GCN-NEXT:	s_load_dwordx8 s[16:23], s[4:5], 0x8
+; GCN-NEXT:	v_mov_b32_e32 v10, 16
+; GCN-NEXT:	v_mov_b32_e32 v8, 0
+; GCN-NEXT:	v_mov_b32_e32 v11, 0
+; GCN-NEXT:	v_mov_b32_e32 v9, 0
+; GCN-NEXT:	s_waitcnt lgkmcnt(0)
+; GCN-NEXT:	s_sub_i32 s6, 64, s16
+; GCN-NEXT:	v_cmp_lt_u64_e64 s[0:1], s[16:17], 64
+; GCN-NEXT:	v_cmp_eq_u64_e64 s[2:3], s[18:19], 0
+; GCN-NEXT:	s_sub_i32 s4, s16, 64
+; GCN-NEXT:	s_lshl_b64 s[6:7], s[10:11], s6
+; GCN-NEXT:	s_lshr_b64 s[24:25], s[8:9], s16
+; GCN-NEXT:	s_or_b64 s[6:7], s[24:25], s[6:7]
+; GCN-NEXT:	s_and_b64 vcc, s[2:3], s[0:1]
+; GCN-NEXT:	s_or_b64 s[0:1], s[16:17], s[18:19]
+; GCN-NEXT:	s_lshr_b64 s[4:5], s[10:11], s4
+; GCN-NEXT:	v_mov_b32_e32 v0, s5
+; GCN-NEXT:	v_mov_b32_e32 v1, s7
+; GCN-NEXT:	v_cmp_eq_u64_e64 s[0:1], s[0:1], 0
+; GCN-NEXT:	v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-NEXT:	v_mov_b32_e32 v1, s9
+; GCN-NEXT:	v_cndmask_b32_e64 v1, v0, v1, s[0:1]
+; GCN-NEXT:	v_mov_b32_e32 v0, s4
+; GCN-NEXT:	v_mov_b32_e32 v2, s6
+; GCN-NEXT:	v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:	v_mov_b32_e32 v2, s8
+; GCN-NEXT:	s_sub_i32 s6, 64, s20
+; GCN-NEXT:	v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GCN-NEXT:	v_cmp_lt_u64_e64 s[0:1], s[20:21], 64
+; GCN-NEXT:	v_cmp_eq_u64_e64 s[2:3], s[22:23], 0
+; GCN-NEXT:	s_sub_i32 s4, s20, 64
+; GCN-NEXT:	s_lshl_b64 s[6:7], s[14:15], s6
+; GCN-NEXT:	s_lshr_b64 s[8:9], s[12:13], s20
+; GCN-NEXT:	s_lshr_b64 s[4:5], s[14:15], s4
+; GCN-NEXT:	s_or_b64 s[6:7], s[8:9], s[6:7]
+; GCN-NEXT:	s_and_b64 s[0:1], s[2:3], s[0:1]
+; GCN-NEXT:	s_or_b64 s[2:3], s[20:21], s[22:23]
+; GCN-NEXT:	v_mov_b32_e32 v2, s5
+; GCN-NEXT:	v_mov_b32_e32 v3, s7
+; GCN-NEXT:	v_cmp_eq_u64_e64 s[2:3], s[2:3], 0
+; GCN-NEXT:	v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; GCN-NEXT:	v_mov_b32_e32 v3, s13
+; GCN-NEXT:	v_cndmask_b32_e64 v5, v2, v3, s[2:3]
+; GCN-NEXT:	v_mov_b32_e32 v2, s4
+; GCN-NEXT:	v_mov_b32_e32 v3, s6
+; GCN-NEXT:	v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; GCN-NEXT:	v_mov_b32_e32 v3, s12
+; GCN-NEXT:	v_cndmask_b32_e64 v4, v2, v3, s[2:3]
+; GCN-NEXT:	s_lshr_b64 s[2:3], s[10:11], s16
+; GCN-NEXT:	v_mov_b32_e32 v2, s3
+; GCN-NEXT:	v_cndmask_b32_e32 v3, 0, v2, vcc
+; GCN-NEXT:	v_mov_b32_e32 v2, s2
+; GCN-NEXT:	s_lshr_b64 s[2:3], s[14:15], s20
+; GCN-NEXT:	v_mov_b32_e32 v6, s3
+; GCN-NEXT:	v_cndmask_b32_e64 v7, 0, v6, s[0:1]
+; GCN-NEXT:	v_mov_b32_e32 v6, s2
+; GCN-NEXT:	v_cndmask_b32_e64 v6, 0, v6, s[0:1]
+; GCN-NEXT:	v_cndmask_b32_e32 v2, 0, v2, vcc
+; GCN-NEXT:	flat_store_dwordx4 v[10:11], v[4:7]
+; GCN-NEXT:	flat_store_dwordx4 v[8:9], v[0:3]
+; GCN-NEXT:	s_endpgm
   %shift = lshr <2 x i128> %lhs, %rhs
   store <2 x i128> %shift, <2 x i128> addrspace(1)* null
   ret void
@@ -586,72 +586,72 @@ define amdgpu_kernel void @s_lshr_v2i128_ss(<2 x i128> %lhs, <2 x i128> %rhs) {
 define amdgpu_kernel void @s_ashr_v2i128_ss(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; GCN-LABEL: s_ashr_v2i128_ss:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx8 s[16:23], s[4:5], 0x8
-; GCN-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x0
-; GCN-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_sub_i32 s6, 64, s16
-; GCN-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[16:17], 64
-; GCN-NEXT:    v_cmp_eq_u64_e64 s[2:3], s[18:19], 0
-; GCN-NEXT:    s_sub_i32 s4, s16, 64
-; GCN-NEXT:    s_lshl_b64 s[6:7], s[10:11], s6
-; GCN-NEXT:    s_lshr_b64 s[24:25], s[8:9], s16
-; GCN-NEXT:    s_or_b64 s[6:7], s[24:25], s[6:7]
-; GCN-NEXT:    s_and_b64 vcc, s[2:3], s[0:1]
-; GCN-NEXT:    s_or_b64 s[0:1], s[16:17], s[18:19]
-; GCN-NEXT:    s_ashr_i64 s[4:5], s[10:11], s4
-; GCN-NEXT:    v_mov_b32_e32 v0, s5
-; GCN-NEXT:    v_mov_b32_e32 v1, s7
-; GCN-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[0:1], 0
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-NEXT:    v_mov_b32_e32 v1, s9
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v0, v1, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v0, s4
-; GCN-NEXT:    v_mov_b32_e32 v2, s6
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    v_mov_b32_e32 v2, s8
-; GCN-NEXT:    s_sub_i32 s6, 64, s20
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[20:21], 64
-; GCN-NEXT:    v_cmp_eq_u64_e64 s[2:3], s[22:23], 0
-; GCN-NEXT:    s_sub_i32 s4, s20, 64
-; GCN-NEXT:    s_lshl_b64 s[6:7], s[14:15], s6
-; GCN-NEXT:    s_lshr_b64 s[8:9], s[12:13], s20
-; GCN-NEXT:    s_ashr_i64 s[4:5], s[14:15], s4
-; GCN-NEXT:    s_or_b64 s[6:7], s[8:9], s[6:7]
-; GCN-NEXT:    s_and_b64 s[0:1], s[2:3], s[0:1]
-; GCN-NEXT:    s_or_b64 s[2:3], s[20:21], s[22:23]
-; GCN-NEXT:    v_mov_b32_e32 v2, s5
-; GCN-NEXT:    v_mov_b32_e32 v3, s7
-; GCN-NEXT:    v_cmp_eq_u64_e64 s[2:3], s[2:3], 0
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v3, s13
-; GCN-NEXT:    v_cndmask_b32_e64 v5, v2, v3, s[2:3]
-; GCN-NEXT:    v_mov_b32_e32 v2, s4
-; GCN-NEXT:    v_mov_b32_e32 v3, s6
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v3, s12
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v2, v3, s[2:3]
-; GCN-NEXT:    s_ashr_i64 s[2:3], s[10:11], s16
-; GCN-NEXT:    s_ashr_i32 s4, s11, 31
-; GCN-NEXT:    v_mov_b32_e32 v2, s4
-; GCN-NEXT:    v_mov_b32_e32 v3, s3
-; GCN-NEXT:    v_mov_b32_e32 v6, s2
-; GCN-NEXT:    s_ashr_i64 s[2:3], s[14:15], s20
-; GCN-NEXT:    s_ashr_i32 s4, s15, 31
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v2, v3, vcc
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GCN-NEXT:    v_mov_b32_e32 v6, s4
-; GCN-NEXT:    v_mov_b32_e32 v7, s3
-; GCN-NEXT:    v_mov_b32_e32 v10, s2
-; GCN-NEXT:    v_cndmask_b32_e64 v7, v6, v7, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v10, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v10, 16
-; GCN-NEXT:    v_mov_b32_e32 v11, 0
-; GCN-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
-; GCN-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
-; GCN-NEXT:    s_endpgm
+; GCN-NEXT:	s_load_dwordx8 s[8:15], s[4:5], 0x0
+; GCN-NEXT:	s_load_dwordx8 s[16:23], s[4:5], 0x8
+; GCN-NEXT:	v_mov_b32_e32 v8, 0
+; GCN-NEXT:	v_mov_b32_e32 v9, 0
+; GCN-NEXT:	s_waitcnt lgkmcnt(0)
+; GCN-NEXT:	s_sub_i32 s6, 64, s16
+; GCN-NEXT:	v_cmp_lt_u64_e64 s[0:1], s[16:17], 64
+; GCN-NEXT:	v_cmp_eq_u64_e64 s[2:3], s[18:19], 0
+; GCN-NEXT:	s_sub_i32 s4, s16, 64
+; GCN-NEXT:	s_lshl_b64 s[6:7], s[10:11], s6
+; GCN-NEXT:	s_lshr_b64 s[24:25], s[8:9], s16
+; GCN-NEXT:	s_or_b64 s[6:7], s[24:25], s[6:7]
+; GCN-NEXT:	s_and_b64 vcc, s[2:3], s[0:1]
+; GCN-NEXT:	s_or_b64 s[0:1], s[16:17], s[18:19]
+; GCN-NEXT:	s_ashr_i64 s[4:5], s[10:11], s4
+; GCN-NEXT:	v_mov_b32_e32 v0, s5
+; GCN-NEXT:	v_mov_b32_e32 v1, s7
+; GCN-NEXT:	v_cmp_eq_u64_e64 s[0:1], s[0:1], 0
+; GCN-NEXT:	v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-NEXT:	v_mov_b32_e32 v1, s9
+; GCN-NEXT:	v_cndmask_b32_e64 v1, v0, v1, s[0:1]
+; GCN-NEXT:	v_mov_b32_e32 v0, s4
+; GCN-NEXT:	v_mov_b32_e32 v2, s6
+; GCN-NEXT:	v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:	v_mov_b32_e32 v2, s8
+; GCN-NEXT:	s_sub_i32 s6, 64, s20
+; GCN-NEXT:	v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GCN-NEXT:	v_cmp_lt_u64_e64 s[0:1], s[20:21], 64
+; GCN-NEXT:	v_cmp_eq_u64_e64 s[2:3], s[22:23], 0
+; GCN-NEXT:	s_sub_i32 s4, s20, 64
+; GCN-NEXT:	s_lshl_b64 s[6:7], s[14:15], s6
+; GCN-NEXT:	s_lshr_b64 s[8:9], s[12:13], s20
+; GCN-NEXT:	s_ashr_i64 s[4:5], s[14:15], s4
+; GCN-NEXT:	s_or_b64 s[6:7], s[8:9], s[6:7]
+; GCN-NEXT:	s_and_b64 s[0:1], s[2:3], s[0:1]
+; GCN-NEXT:	s_or_b64 s[2:3], s[20:21], s[22:23]
+; GCN-NEXT:	v_mov_b32_e32 v2, s5
+; GCN-NEXT:	v_mov_b32_e32 v3, s7
+; GCN-NEXT:	v_cmp_eq_u64_e64 s[2:3], s[2:3], 0
+; GCN-NEXT:	v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; GCN-NEXT:	v_mov_b32_e32 v3, s13
+; GCN-NEXT:	v_cndmask_b32_e64 v5, v2, v3, s[2:3]
+; GCN-NEXT:	v_mov_b32_e32 v2, s4
+; GCN-NEXT:	v_mov_b32_e32 v3, s6
+; GCN-NEXT:	v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; GCN-NEXT:	v_mov_b32_e32 v3, s12
+; GCN-NEXT:	v_cndmask_b32_e64 v4, v2, v3, s[2:3]
+; GCN-NEXT:	s_ashr_i64 s[2:3], s[10:11], s16
+; GCN-NEXT:	s_ashr_i32 s4, s11, 31
+; GCN-NEXT:	v_mov_b32_e32 v2, s4
+; GCN-NEXT:	v_mov_b32_e32 v3, s3
+; GCN-NEXT:	v_mov_b32_e32 v6, s2
+; GCN-NEXT:	s_ashr_i64 s[2:3], s[14:15], s20
+; GCN-NEXT:	s_ashr_i32 s4, s15, 31
+; GCN-NEXT:	v_cndmask_b32_e32 v3, v2, v3, vcc
+; GCN-NEXT:	v_cndmask_b32_e32 v2, v2, v6, vcc
+; GCN-NEXT:	v_mov_b32_e32 v6, s4
+; GCN-NEXT:	v_mov_b32_e32 v7, s3
+; GCN-NEXT:	v_mov_b32_e32 v10, s2
+; GCN-NEXT:	v_cndmask_b32_e64 v7, v6, v7, s[0:1]
+; GCN-NEXT:	v_cndmask_b32_e64 v6, v6, v10, s[0:1]
+; GCN-NEXT:	v_mov_b32_e32 v10, 16
+; GCN-NEXT:	v_mov_b32_e32 v11, 0
+; GCN-NEXT:	flat_store_dwordx4 v[10:11], v[4:7]
+; GCN-NEXT:	flat_store_dwordx4 v[8:9], v[0:3]
+; GCN-NEXT:	s_endpgm
   %shift = ashr <2 x i128> %lhs, %rhs
   store <2 x i128> %shift, <2 x i128> addrspace(1)* null
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll b/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
index 663831f06f4b5..833a941748285 100644
--- a/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
@@ -15,9 +15,10 @@ define amdgpu_kernel void @sint_to_fp_i32_to_f64(double addrspace(1)* %out, i32
 ; uses an SGPR (implicit vcc).
 
 ; GCN-LABEL: {{^}}sint_to_fp_i1_f64:
-; GCN-DAG: v_cmp_eq_u32_e64 vcc,
-; GCN-DAG: v_cndmask_b32_e32 v[[SEL:[0-9]+]], 0, v{{[0-9]+}}
+; GCN-DAG: s_cmp_eq_u32
+; GCN-DAG: s_cselect_b32 s[[SSEL:[0-9]+]], 0xbff00000, 0
 ; GCN-DAG: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}}
+; GCN-DAG: v_mov_b32_e32 v[[SEL:[0-9]+]], s[[SSEL]]
 ; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[ZERO]]:[[SEL]]{{\]}}
 ; GCN: s_endpgm
 define amdgpu_kernel void @sint_to_fp_i1_f64(double addrspace(1)* %out, i32 %in) {
diff --git a/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll b/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll
index e6799cda00ab3..f178259f2b6f6 100644
--- a/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll
+++ b/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll
@@ -12,11 +12,11 @@ define amdgpu_ps void @test_kill_depth_0_imm_pos() #0 {
 ; CHECK-LABEL: {{^}}test_kill_depth_0_imm_neg:
 ; CHECK-NEXT: ; %bb.0:
 ; CHECK-NEXT: s_mov_b64 exec, 0
-; CHECK-NEXT: s_cbranch_execnz BB1_2
+; CHECK-NEXT: s_cbranch_execz BB1_2
 ; CHECK-NEXT: ; %bb.1:
-; CHECK-NEXT: exp null off, off, off, off done vm
 ; CHECK-NEXT: s_endpgm
 ; CHECK-NEXT: BB1_2:
+; CHECK-NEXT: exp null off, off, off, off done vm
 ; CHECK-NEXT: s_endpgm
 define amdgpu_ps void @test_kill_depth_0_imm_neg() #0 {
   call void @llvm.amdgcn.kill(i1 false)
@@ -27,15 +27,14 @@ define amdgpu_ps void @test_kill_depth_0_imm_neg() #0 {
 ; CHECK-LABEL: {{^}}test_kill_depth_0_imm_neg_x2:
 ; CHECK-NEXT: ; %bb.0:
 ; CHECK-NEXT: s_mov_b64 exec, 0
-; CHECK-NEXT: s_cbranch_execnz BB2_2
-; CHECK:      exp null
-; CHECK-NEXT: s_endpgm
-; CHECK:      BB2_2:
+; CHECK-NEXT: s_cbranch_execz BB2_3
+; CHECK-NEXT: ; %bb.1:
 ; CHECK-NEXT: s_mov_b64 exec, 0
-; CHECK-NEXT: s_cbranch_execnz BB2_4
-; CHECK:      exp null
+; CHECK-NEXT: s_cbranch_execz BB2_3
+; CHECK-NEXT: ; %bb.2:
 ; CHECK-NEXT: s_endpgm
-; CHECK-NEXT: BB2_4:
+; CHECK-NEXT: BB2_3:
+; CHECK:      exp null
 ; CHECK-NEXT: s_endpgm
 define amdgpu_ps void @test_kill_depth_0_imm_neg_x2() #0 {
   call void @llvm.amdgcn.kill(i1 false)
@@ -46,10 +45,11 @@ define amdgpu_ps void @test_kill_depth_0_imm_neg_x2() #0 {
 ; CHECK-LABEL: {{^}}test_kill_depth_var:
 ; CHECK-NEXT: ; %bb.0:
 ; CHECK-NEXT: v_cmpx_gt_f32_e32 vcc, 0, v0
-; CHECK-NEXT: s_cbranch_execnz BB3_2
-; CHECK:      exp null
+; CHECK-NEXT: s_cbranch_execz BB3_2
+; CHECK-NEXT: ; %bb.1:
 ; CHECK-NEXT: s_endpgm
 ; CHECK-NEXT: BB3_2:
+; CHECK:      exp null
 ; CHECK-NEXT: s_endpgm
 define amdgpu_ps void @test_kill_depth_var(float %x) #0 {
   %cmp = fcmp olt float %x, 0.0
@@ -61,15 +61,14 @@ define amdgpu_ps void @test_kill_depth_var(float %x) #0 {
 ; CHECK-LABEL: {{^}}test_kill_depth_var_x2_same:
 ; CHECK-NEXT: ; %bb.0:
 ; CHECK-NEXT: v_cmpx_gt_f32_e32 vcc, 0, v0
-; CHECK-NEXT: s_cbranch_execnz BB4_2
-; CHECK:      exp null
-; CHECK-NEXT: s_endpgm
-; CHECK-NEXT: BB4_2:
+; CHECK-NEXT: s_cbranch_execz BB4_3
+; CHECK-NEXT: ; %bb.1:
 ; CHECK-NEXT: v_cmpx_gt_f32_e32 vcc, 0, v0
-; CHECK-NEXT: s_cbranch_execnz BB4_4
-; CHECK:      exp null
+; CHECK-NEXT: s_cbranch_execz BB4_3
+; CHECK-NEXT: ; %bb.2:
 ; CHECK-NEXT: s_endpgm
-; CHECK-NEXT: BB4_4:
+; CHECK-NEXT: BB4_3:
+; CHECK:      exp null
 ; CHECK-NEXT: s_endpgm
 define amdgpu_ps void @test_kill_depth_var_x2_same(float %x) #0 {
   %cmp = fcmp olt float %x, 0.0
@@ -82,15 +81,14 @@ define amdgpu_ps void @test_kill_depth_var_x2_same(float %x) #0 {
 ; CHECK-LABEL: {{^}}test_kill_depth_var_x2:
 ; CHECK-NEXT: ; %bb.0:
 ; CHECK-NEXT: v_cmpx_gt_f32_e32 vcc, 0, v0
-; CHECK-NEXT: s_cbranch_execnz BB5_2
-; CHECK:      exp null
-; CHECK-NEXT: s_endpgm
-; CHECK-NEXT: BB5_2:
+; CHECK-NEXT: s_cbranch_execz BB5_3
+; CHECK-NEXT: ; %bb.1
 ; CHECK-NEXT: v_cmpx_gt_f32_e32 vcc, 0, v1
-; CHECK-NEXT: s_cbranch_execnz BB5_4
-; CHECK:      exp null
+; CHECK-NEXT: s_cbranch_execz BB5_3
+; CHECK-NEXT: ; %bb.2
 ; CHECK-NEXT: s_endpgm
-; CHECK-NEXT: BB5_4:
+; CHECK-NEXT: BB5_3:
+; CHECK:      exp null
 ; CHECK-NEXT: s_endpgm
 define amdgpu_ps void @test_kill_depth_var_x2(float %x, float %y) #0 {
   %cmp.x = fcmp olt float %x, 0.0
@@ -103,18 +101,15 @@ define amdgpu_ps void @test_kill_depth_var_x2(float %x, float %y) #0 {
 ; CHECK-LABEL: {{^}}test_kill_depth_var_x2_instructions:
 ; CHECK-NEXT: ; %bb.0:
 ; CHECK-NEXT: v_cmpx_gt_f32_e32 vcc, 0, v0
-; CHECK-NEXT: s_cbranch_execnz BB6_2
+; CHECK-NEXT: s_cbranch_execz BB6_3
 ; CHECK-NEXT: ; %bb.1:
-; CHECK-NEXT: exp
-; CHECK-NEXT: s_endpgm
-; CHECK-NEXT: BB6_2:
 ; CHECK: v_mov_b32_e64 v7, -1
 ; CHECK: v_cmpx_gt_f32_e32 vcc, 0, v7
-; CHECK-NEXT: s_cbranch_execnz BB6_4
-; CHECK-NEXT: ; %bb.3:
-; CHECK-NEXT: exp
+; CHECK-NEXT: s_cbranch_execz BB6_3
+; CHECK-NEXT: ; %bb.2:
 ; CHECK-NEXT: s_endpgm
-; CHECK-NEXT: BB6_4:
+; CHECK-NEXT: BB6_3:
+; CHECK-NEXT: exp null
 ; CHECK-NEXT: s_endpgm
 define amdgpu_ps void @test_kill_depth_var_x2_instructions(float %x) #0 {
   %cmp.x = fcmp olt float %x, 0.0
@@ -237,6 +232,62 @@ exit:
   ret void
 }
 
+; CHECK-LABEL: {{^}}test_kill_control_flow_return:
+
+; CHECK: v_cmp_eq_u32_e64 [[KILL_CC:s\[[0-9]+:[0-9]+\]]], s0, 1
+; CHECK: s_and_b64 exec, exec, s[2:3]
+; CHECK-NEXT: s_cbranch_execz [[EXIT_BB:BB[0-9]+_[0-9]+]]
+
+; CHECK: s_cmp_lg_u32 s{{[0-9]+}}, 0
+; CHECK: s_cbranch_scc0 [[COND_BB:BB[0-9]+_[0-9]+]]
+; CHECK: s_branch [[RETURN_BB:BB[0-9]+_[0-9]+]]
+
+; CHECK: [[COND_BB]]:
+; CHECK: v_mov_b32_e64 v7, -1
+; CHECK: v_nop_e64
+; CHECK: v_nop_e64
+; CHECK: v_nop_e64
+; CHECK: v_nop_e64
+; CHECK: v_nop_e64
+; CHECK: v_nop_e64
+; CHECK: v_nop_e64
+; CHECK: v_nop_e64
+; CHECK: v_nop_e64
+; CHECK: v_nop_e64
+; CHECK: v_mov_b32_e32 v0, v7
+
+; CHECK: [[EXIT_BB]]:
+; CHECK-NEXT: exp null
+; CHECK-NEXT: s_endpgm
+
+; CHECK: [[RETURN_BB]]:
+define amdgpu_ps float @test_kill_control_flow_return(i32 inreg %arg) #0 {
+entry:
+  %kill = icmp eq i32 %arg, 1
+  %cmp = icmp eq i32 %arg, 0
+  call void @llvm.amdgcn.kill(i1 %kill)
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %var = call float asm sideeffect "
+    v_mov_b32_e64 v7, -1
+    v_nop_e64
+    v_nop_e64
+    v_nop_e64
+    v_nop_e64
+    v_nop_e64
+    v_nop_e64
+    v_nop_e64
+    v_nop_e64
+    v_nop_e64
+    v_nop_e64", "={v7}"()
+  br label %exit
+
+exit:
+  %ret = phi float [ %var, %bb ], [ 0.0, %entry ]
+  ret float %ret
+}
+
 ; CHECK-LABEL: {{^}}test_kill_divergent_loop:
 ; CHECK: v_cmp_eq_u32_e32 vcc, 0, v0
 ; CHECK-NEXT: s_and_saveexec_b64 [[SAVEEXEC:s\[[0-9]+:[0-9]+\]]], vcc
@@ -295,12 +346,9 @@ exit:
 ; CHECK-LABEL: {{^}}phi_use_def_before_kill:
 ; CHECK: v_cndmask_b32_e64 [[PHIREG:v[0-9]+]], 0, -1.0,
 ; CHECK: v_cmpx_lt_f32_e32 vcc, 0,
-; CHECK-NEXT: s_cbranch_execnz [[BB4:BB[0-9]+_[0-9]+]]
-
-; CHECK: exp
-; CHECK-NEXT: s_endpgm
+; CHECK-NEXT: s_cbranch_execz [[EXITBB:BB[0-9]+_[0-9]+]]
 
-; CHECK: [[KILLBB:BB[0-9]+_[0-9]+]]:
+; CHECK: ; %[[KILLBB:bb.[0-9]+]]:
 ; CHECK-NEXT: s_cbranch_scc0 [[PHIBB:BB[0-9]+_[0-9]+]]
 
 ; CHECK: [[PHIBB]]:
@@ -313,6 +361,10 @@ exit:
 
 ; CHECK: [[ENDBB]]:
 ; CHECK-NEXT: s_endpgm
+
+; CHECK: [[EXITBB]]:
+; CHECK: exp null
+; CHECK-NEXT: s_endpgm
 define amdgpu_ps void @phi_use_def_before_kill(float inreg %x) #0 {
 bb:
   %tmp = fadd float %x, 1.000000e+00
@@ -418,7 +470,11 @@ bb9:                                              ; preds = %bb4
 }
 
 ; CHECK-LABEL: {{^}}cbranch_kill:
-; CHECK-NOT: exp null off, off, off, off done vm
+; CHECK: ; %bb.{{[0-9]+}}: ; %export
+; CHECK-NEXT: s_or_b64
+; CHECK-NEXT: s_cbranch_execz [[EXIT:BB[0-9]+_[0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: exp null off, off, off, off done vm
 define amdgpu_ps void @cbranch_kill(i32 inreg %0, <2 x float> %1) {
 .entry:
   %val0 = extractelement <2 x float> %1, i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/spill-agpr.ll b/llvm/test/CodeGen/AMDGPU/spill-agpr.ll
index 3e60879de1795..3e7b381a45fef 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-agpr.ll
@@ -64,15 +64,26 @@ st:
 ; A2M-DAG:    s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0
 ; A2M-DAG:    s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1
 ; A2V-NOT:    SCRATCH_RSRC
-; GFX908-DAG: v_accvgpr_read_b32 v[[VSPILL:[0-9]+]], a0
-; A2M:        buffer_store_dword v[[VSPILL]], off, s[{{[0-9:]+}}], 0 offset:[[FI:[0-9]+]] ; 4-byte Folded Spill
-; A2M:        buffer_load_dword v[[VSPILL:[0-9]+]], off, s[{{[0-9:]+}}], 0 offset:[[FI]] ; 4-byte Folded Reload
-; GFX908:     v_accvgpr_write_b32 a{{[0-9]+}}, v[[VSPILL]]
-; A2V:        ScratchSize: 0
-define amdgpu_kernel void @max_10_vgprs_used_9a(i32 addrspace(1)* %p) #1 {
-  %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  call void asm sideeffect "", "a,a,a,a"(i32 1, i32 2, i32 3, i32 4)
-  call void asm sideeffect "", "a,a,a,a,a"(i32 5, i32 6, i32 7, i32 8, i32 9)
+
+; A2V: v_accvgpr_read_b32 v[[VSPILL:[0-9]+]], a{{[0-9]+}}
+; A2V: v_accvgpr_write_b32 a{{[0-9]+}}, v[[VSPILL]]
+; A2V: ScratchSize: 0
+
+; A2M: buffer_store_dword v[[VSPILLSTORE:[0-9]+]], off, s[{{[0-9:]+}}], 0 offset:[[FI:[0-9]+]] ; 4-byte Folded Spill
+; A2M: buffer_load_dword v[[VSPILL_RELOAD:[0-9]+]], off, s[{{[0-9:]+}}], 0 offset:[[FI]] ; 4-byte Folded Reload
+; A2M: v_accvgpr_write_b32 a{{[0-9]+}}, v[[VSPILL_RELOAD]]
+define amdgpu_kernel void @max_10_vgprs_used_9a() #1 {
+  %v0 = load volatile i32, i32 addrspace(3)* undef
+  %v1 = load volatile i32, i32 addrspace(3)* undef
+  %v2 = load volatile i32, i32 addrspace(3)* undef
+  %v3 = load volatile i32, i32 addrspace(3)* undef
+  %v4 = load volatile i32, i32 addrspace(3)* undef
+  %v5 = load volatile i32, i32 addrspace(3)* undef
+  %v6 = load volatile i32, i32 addrspace(3)* undef
+  %v7 = load volatile i32, i32 addrspace(3)* undef
+  call void asm sideeffect "", "a,a,a,a,~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6}"(i32 %v0, i32 %v1, i32 %v2, i32 %v3)
+  %v8 = load volatile i32, i32 addrspace(3)* undef
+  call void asm sideeffect "", "a,a,a,a,a"(i32 %v4, i32 %v5, i32 %v6, i32 %v7, i32 %v8)
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vgpr-to-agpr.ll b/llvm/test/CodeGen/AMDGPU/spill-vgpr-to-agpr.ll
index e5e9516ff6f31..6fe3e617bffe3 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vgpr-to-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vgpr-to-agpr.ll
@@ -199,7 +199,7 @@ define amdgpu_kernel void @max_10_vgprs_spill_v32(<32 x float> addrspace(1)* %p)
 ; GFX908: ScratchSize: 0
 ; GCN:    VGPRBlocks: 63
 ; GCN:    NumVGPRsForWavesPerEU: 256
-define amdgpu_kernel void @max_256_vgprs_spill_9x32(<32 x float> addrspace(1)* %p) {
+define amdgpu_kernel void @max_256_vgprs_spill_9x32(<32 x float> addrspace(1)* %p) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %p1 = getelementptr inbounds <32 x float>, <32 x float> addrspace(1)* %p, i32 %tid
   %p2 = getelementptr inbounds <32 x float>, <32 x float> addrspace(1)* %p1, i32 %tid
@@ -250,7 +250,7 @@ define amdgpu_kernel void @max_256_vgprs_spill_9x32(<32 x float> addrspace(1)* %
 ; GFX908-FIXME: ScratchSize: 0
 ; GCN:    VGPRBlocks: 63
 ; GCN:    NumVGPRsForWavesPerEU: 256
-define amdgpu_kernel void @max_256_vgprs_spill_9x32_2bb(<32 x float> addrspace(1)* %p) {
+define amdgpu_kernel void @max_256_vgprs_spill_9x32_2bb(<32 x float> addrspace(1)* %p) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %p1 = getelementptr inbounds <32 x float>, <32 x float> addrspace(1)* %p, i32 %tid
   %p2 = getelementptr inbounds <32 x float>, <32 x float> addrspace(1)* %p1, i32 %tid
@@ -288,3 +288,4 @@ st:
 declare i32 @llvm.amdgcn.workitem.id.x()
 
 attributes #0 = { nounwind "amdgpu-num-vgpr"="10" }
+attributes #1 = { "amdgpu-flat-work-group-size"="1,256" }
diff --git a/llvm/test/CodeGen/AMDGPU/spill_more_than_wavesize_csr_sgprs.ll b/llvm/test/CodeGen/AMDGPU/spill_more_than_wavesize_csr_sgprs.ll
new file mode 100644
index 0000000000000..b89db9d02b569
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/spill_more_than_wavesize_csr_sgprs.ll
@@ -0,0 +1,43 @@
+; RUN: llc -mtriple amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -enable-var-scope %s
+
+; CHECK-LABEL: {{^}}spill_more_than_wavesize_csr_sgprs:
+; CHECK:        v_writelane_b32 v0, s98, 63
+; CHECK-NEXT:   v_writelane_b32 v1, s99, 0
+; CHECK:        v_readlane_b32 s99, v1, 0
+; CHECK-NEXT:   v_readlane_b32 s98, v0, 63
+
+define void @spill_more_than_wavesize_csr_sgprs() {
+  call void asm sideeffect "",
+   "~{s35},~{s36},~{s37},~{s38},~{s39},~{s40},~{s41},~{s42}
+   ,~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49},~{s50}
+   ,~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58}
+   ,~{s59},~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66}
+   ,~{s67},~{s68},~{s69},~{s70},~{s71},~{s72},~{s73},~{s74}
+   ,~{s75},~{s76},~{s77},~{s78},~{s79},~{s80},~{s81},~{s82}
+   ,~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89},~{s90}
+   ,~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98}
+   ,~{s99},~{s100},~{s101},~{s102}"()
+  ret void
+}
+
+; CHECK-LABEL: {{^}}spill_more_than_wavesize_csr_sgprs_with_stack_object:
+; CHECK:        v_writelane_b32 v1, s98, 63
+; CHECK-NEXT:   v_writelane_b32 v2, s99, 0
+; CHECK:        v_readlane_b32 s99, v2, 0
+; CHECK-NEXT:   v_readlane_b32 s98, v1, 63
+
+define void @spill_more_than_wavesize_csr_sgprs_with_stack_object() {
+  %alloca = alloca i32, align 4, addrspace(5)
+  store volatile i32 0, i32 addrspace(5)* %alloca
+  call void asm sideeffect "",
+   "~{s35},~{s36},~{s37},~{s38},~{s39},~{s40},~{s41},~{s42}
+   ,~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49},~{s50}
+   ,~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58}
+   ,~{s59},~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66}
+   ,~{s67},~{s68},~{s69},~{s70},~{s71},~{s72},~{s73},~{s74}
+   ,~{s75},~{s76},~{s77},~{s78},~{s79},~{s80},~{s81},~{s82}
+   ,~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89},~{s90}
+   ,~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98}
+   ,~{s99},~{s100},~{s101},~{s102}"()
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/srem64.ll b/llvm/test/CodeGen/AMDGPU/srem64.ll
index a0e16ae0cef6a..746afd333b367 100644
--- a/llvm/test/CodeGen/AMDGPU/srem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/srem64.ll
@@ -6,7 +6,6 @@ define amdgpu_kernel void @s_test_srem(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ; GCN-LABEL: s_test_srem:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0xd
-; GCN-NEXT:    v_mov_b32_e32 v2, 0
 ; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
@@ -18,99 +17,100 @@ define amdgpu_kernel void @s_test_srem(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ; GCN-NEXT:    s_mov_b32 s4, s8
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-NEXT:    v_mov_b32_e32 v1, 0
 ; GCN-NEXT:    s_mov_b32 s5, s9
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GCN-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v0
-; GCN-NEXT:    v_trunc_f32_e32 v3, v3
-; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v3
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GCN-NEXT:    v_mul_hi_u32 v5, s2, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s2, v3
-; GCN-NEXT:    v_mul_lo_u32 v7, s3, v0
-; GCN-NEXT:    v_mul_lo_u32 v6, s2, v0
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
-; GCN-NEXT:    v_mul_hi_u32 v5, v0, v6
-; GCN-NEXT:    v_mul_lo_u32 v7, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v9, v0, v4
-; GCN-NEXT:    v_mul_lo_u32 v8, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v6, v3, v6
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v2, v9, vcc
-; GCN-NEXT:    v_mul_hi_u32 v9, v3, v4
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v9, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v2, v6, vcc
-; GCN-NEXT:    v_addc_u32_e64 v4, vcc, v3, v5, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v6, s2, v4
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
+; GCN-NEXT:    v_mul_lo_u32 v5, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s2, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v1, v4
+; GCN-NEXT:    v_mul_lo_u32 v4, v1, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_mul_hi_u32 v8, v1, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v6, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v5, s2, v2
 ; GCN-NEXT:    v_mul_hi_u32 v7, s2, v0
 ; GCN-NEXT:    v_mul_lo_u32 v8, s3, v0
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
 ; GCN-NEXT:    v_mul_lo_u32 v7, s2, v0
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
-; GCN-NEXT:    v_mul_lo_u32 v10, v0, v6
-; GCN-NEXT:    v_mul_hi_u32 v12, v0, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
+; GCN-NEXT:    v_mul_lo_u32 v10, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v12, v0, v5
 ; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
-; GCN-NEXT:    v_mul_hi_u32 v9, v4, v7
-; GCN-NEXT:    v_mul_lo_u32 v7, v4, v7
-; GCN-NEXT:    v_mul_hi_u32 v8, v4, v6
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v8, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, v2, v12, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, v4, v6
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v8, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v2, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
-; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v6, s[0:1]
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, s10, v3
-; GCN-NEXT:    v_mul_hi_u32 v5, s10, v0
-; GCN-NEXT:    v_mul_hi_u32 v6, s10, v3
-; GCN-NEXT:    v_mul_hi_u32 v7, s11, v3
-; GCN-NEXT:    v_mul_lo_u32 v3, s11, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v2, v6, vcc
-; GCN-NEXT:    v_mul_lo_u32 v6, s11, v0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v5, v0, vcc
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v7, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v1, s12, v1
 ; GCN-NEXT:    v_mul_hi_u32 v2, s12, v0
 ; GCN-NEXT:    v_mul_lo_u32 v3, s13, v0
 ; GCN-NEXT:    v_mul_lo_u32 v0, s12, v0
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e64 v0, s[0:1], s10, v0
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s11, v1
 ; GCN-NEXT:    v_mov_b32_e32 v3, s13
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s10, v0
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s12, v0
-; GCN-NEXT:    v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s13, v5
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s12, v4
-; GCN-NEXT:    v_subrev_i32_e64 v3, s[0:1], s12, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s13, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[2:3]
-; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v4, s[2:3], s12, v0
+; GCN-NEXT:    v_subbrev_u32_e64 v5, vcc, 0, v2, s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s12, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v5, s11
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v5, v1, vcc
+; GCN-NEXT:    v_subb_u32_e64 v1, vcc, v5, v1, s[0:1]
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v1
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v0
@@ -119,7 +119,7 @@ define amdgpu_kernel void @s_test_srem(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
@@ -127,97 +127,91 @@ define amdgpu_kernel void @s_test_srem(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ; GCN-IR-LABEL: s_test_srem:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-IR-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
+; GCN-IR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GCN-IR-NEXT:    s_mov_b64 s[2:3], 0
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[6:7], 0
-; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s2
-; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s10
-; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s6
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
-; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s10
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v2, v3
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[2:3], 0
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[10:11], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[8:9]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[8:9], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[8:9], s[8:9], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT:    s_cbranch_vccz BB0_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[4:5], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[6:7], 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[0:1], 0
+; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s0
+; GCN-IR-NEXT:    s_add_i32 s14, s12, 32
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[8:9], s[10:11]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s1
+; GCN-IR-NEXT:    s_cmp_eq_u32 s1, 0
+; GCN-IR-NEXT:    s_cselect_b32 s10, s14, s8
+; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s6
+; GCN-IR-NEXT:    s_add_i32 s8, s8, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s9, s7
+; GCN-IR-NEXT:    s_cmp_eq_u32 s7, 0
+; GCN-IR-NEXT:    s_cselect_b32 s14, s8, s9
+; GCN-IR-NEXT:    s_sub_u32 s8, s10, s14
+; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[8:9], 63
+; GCN-IR-NEXT:    s_mov_b32 s11, 0
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[16:17]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[16:17], s[8:9], 63
+; GCN-IR-NEXT:    s_xor_b64 s[18:19], s[12:13], -1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[18:19], s[16:17]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[16:17]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB0_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s16, s8, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_addc_u32 s17, s9, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[16:17], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s8, 63, s8
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[12:13], s[6:7], s8
+; GCN-IR-NEXT:    s_cbranch_vccz BB0_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_not_b32_e32 v2, v2
-; GCN-IR-NEXT:    s_add_u32 s8, s2, -1
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[6:7], v4
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v2, v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s9, s3, -1
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], -1, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[16:17], s[6:7], s16
+; GCN-IR-NEXT:    s_add_u32 s8, s0, -1
+; GCN-IR-NEXT:    s_addc_u32 s9, s1, -1
+; GCN-IR-NEXT:    s_not_b64 s[2:3], s[10:11]
+; GCN-IR-NEXT:    s_mov_b32 s15, s11
+; GCN-IR-NEXT:    s_add_u32 s10, s2, s14
+; GCN-IR-NEXT:    s_addc_u32 s11, s3, s11
+; GCN-IR-NEXT:    s_mov_b64 s[14:15], 0
+; GCN-IR-NEXT:    s_mov_b32 s3, 0
 ; GCN-IR-NEXT:  BB0_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s9
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s8, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s2, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s3, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s2, s13, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[16:17], s[16:17], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[12:13], s[12:13], 1
+; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[2:3]
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[14:15], s[12:13]
+; GCN-IR-NEXT:    s_sub_u32 s2, s8, s16
+; GCN-IR-NEXT:    s_subb_u32 s2, s9, s17
+; GCN-IR-NEXT:    s_ashr_i32 s14, s2, 31
+; GCN-IR-NEXT:    s_mov_b32 s15, s14
+; GCN-IR-NEXT:    s_and_b32 s2, s14, 1
+; GCN-IR-NEXT:    s_and_b64 s[18:19], s[14:15], s[0:1]
+; GCN-IR-NEXT:    s_sub_u32 s16, s16, s18
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-IR-NEXT:    s_subb_u32 s17, s17, s19
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-IR-NEXT:    s_add_u32 s10, s10, 1
+; GCN-IR-NEXT:    s_addc_u32 s11, s11, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[10:11], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[14:15], s[2:3]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB0_3
+; GCN-IR-NEXT:  BB0_4: ; %Flow6
+; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[12:13], 1
+; GCN-IR-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-IR-NEXT:    s_branch BB0_6
-; GCN-IR-NEXT:  BB0_4:
+; GCN-IR-NEXT:  BB0_5:
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[12:13]
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB0_7
-; GCN-IR-NEXT:  BB0_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB0_6: ; %Flow6
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB0_7: ; %udiv-end
-; GCN-IR-NEXT:    v_mul_lo_u32 v1, s2, v1
-; GCN-IR-NEXT:    v_mul_hi_u32 v2, s2, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v3, s3, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[12:13]
+; GCN-IR-NEXT:  BB0_6: ; %udiv-end
+; GCN-IR-NEXT:    v_mul_lo_u32 v1, s0, v1
+; GCN-IR-NEXT:    v_mul_hi_u32 v2, s0, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v3, s1, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v0, s0, v0
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-IR-NEXT:    v_mov_b32_e32 v2, s7
@@ -490,17 +484,17 @@ define amdgpu_kernel void @s_test_srem23_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-NEXT:    s_ashr_i32 s1, s1, 30
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_or_b32 s1, s1, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 23
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
@@ -522,17 +516,17 @@ define amdgpu_kernel void @s_test_srem23_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-IR-NEXT:    s_ashr_i32 s1, s1, 30
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-IR-NEXT:    s_or_b32 s1, s1, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-IR-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s1, v2
 ; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
+; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 23
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
@@ -561,17 +555,17 @@ define amdgpu_kernel void @s_test_srem24_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-NEXT:    s_ashr_i32 s1, s1, 30
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_or_b32 s1, s1, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
@@ -593,17 +587,17 @@ define amdgpu_kernel void @s_test_srem24_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-IR-NEXT:    s_ashr_i32 s1, s1, 30
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-IR-NEXT:    s_or_b32 s1, s1, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-IR-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s1, v2
 ; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
+; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
@@ -686,17 +680,17 @@ define amdgpu_kernel void @s_test_srem25_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-NEXT:    s_ashr_i32 s1, s1, 30
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_or_b32 s1, s1, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 25
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
@@ -718,17 +712,17 @@ define amdgpu_kernel void @s_test_srem25_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-IR-NEXT:    s_ashr_i32 s1, s1, 30
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-IR-NEXT:    s_or_b32 s1, s1, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-IR-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s1, v2
 ; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
+; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 25
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
@@ -757,17 +751,17 @@ define amdgpu_kernel void @s_test_srem31_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-NEXT:    s_ashr_i32 s1, s1, 30
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_or_b32 s1, s1, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 31
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
@@ -789,17 +783,17 @@ define amdgpu_kernel void @s_test_srem31_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-IR-NEXT:    s_ashr_i32 s1, s1, 30
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-IR-NEXT:    s_or_b32 s1, s1, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-IR-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s1, v2
 ; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
+; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 31
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
@@ -817,27 +811,28 @@ define amdgpu_kernel void @s_test_srem32_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-LABEL: s_test_srem32_64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-NEXT:    s_load_dword s0, s[0:1], 0xe
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_load_dword s6, s[0:1], 0xe
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    s_mov_b32 s2, -1
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s7
-; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s0
-; GCN-NEXT:    s_xor_b32 s1, s7, s0
-; GCN-NEXT:    s_ashr_i32 s1, s1, 30
-; GCN-NEXT:    s_or_b32 s1, s1, 1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s6
+; GCN-NEXT:    s_xor_b32 s0, s7, s6
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_or_b32 s8, s0, 1
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s0
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cselect_b32 s0, s8, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s6
 ; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s7, v0
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -846,27 +841,28 @@ define amdgpu_kernel void @s_test_srem32_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-IR-LABEL: s_test_srem32_64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-IR-NEXT:    s_load_dword s0, s[0:1], 0xe
+; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IR-NEXT:    s_load_dword s6, s[0:1], 0xe
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
-; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v1, s7
-; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s0
-; GCN-IR-NEXT:    s_xor_b32 s1, s7, s0
-; GCN-IR-NEXT:    s_ashr_i32 s1, s1, 30
-; GCN-IR-NEXT:    s_or_b32 s1, s1, 1
+; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s6
+; GCN-IR-NEXT:    s_xor_b32 s0, s7, s6
+; GCN-IR-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-IR-NEXT:    s_or_b32 s8, s0, 1
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, s0
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-IR-NEXT:    s_cselect_b32 s0, s8, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, s6
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
+; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s7, v0
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -882,124 +878,124 @@ define amdgpu_kernel void @s_test_srem32_64(i64 addrspace(1)* %out, i64 %x, i64
 define amdgpu_kernel void @s_test_srem33_64(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ; GCN-LABEL: s_test_srem33_64:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GCN-NEXT:    v_mov_b32_e32 v7, 0
-; GCN-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_mov_b32 s11, 0xf000
+; GCN-NEXT:    s_mov_b32 s10, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_ashr_i64 s[2:3], s[10:11], 31
-; GCN-NEXT:    s_ashr_i64 s[4:5], s[0:1], 31
+; GCN-NEXT:    s_ashr_i64 s[2:3], s[6:7], 31
+; GCN-NEXT:    s_ashr_i64 s[8:9], s[0:1], 31
 ; GCN-NEXT:    s_ashr_i32 s0, s1, 31
-; GCN-NEXT:    s_add_u32 s4, s4, s0
+; GCN-NEXT:    s_add_u32 s8, s8, s0
 ; GCN-NEXT:    s_mov_b32 s1, s0
-; GCN-NEXT:    s_addc_u32 s5, s5, s0
-; GCN-NEXT:    s_xor_b64 s[12:13], s[4:5], s[0:1]
+; GCN-NEXT:    s_addc_u32 s9, s9, s0
+; GCN-NEXT:    s_xor_b64 s[12:13], s[8:9], s[0:1]
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s12
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s13
-; GCN-NEXT:    s_sub_u32 s4, 0, s12
-; GCN-NEXT:    s_subb_u32 s5, 0, s13
-; GCN-NEXT:    s_ashr_i32 s10, s11, 31
+; GCN-NEXT:    s_sub_u32 s6, 0, s12
+; GCN-NEXT:    s_subb_u32 s8, 0, s13
+; GCN-NEXT:    s_mov_b32 s9, s5
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-NEXT:    s_mov_b32 s11, s10
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GCN-NEXT:    v_mul_f32_e32 v2, 0x2f800000, v0
-; GCN-NEXT:    v_trunc_f32_e32 v2, v2
-; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v2
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GCN-NEXT:    v_mul_hi_u32 v4, s4, v0
-; GCN-NEXT:    v_mul_lo_u32 v3, s4, v2
-; GCN-NEXT:    v_mul_lo_u32 v6, s5, v0
-; GCN-NEXT:    v_mul_lo_u32 v5, s4, v0
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v4, v0, v5
-; GCN-NEXT:    v_mul_lo_u32 v6, v0, v3
-; GCN-NEXT:    v_mul_hi_u32 v8, v0, v3
-; GCN-NEXT:    v_mul_hi_u32 v9, v2, v3
-; GCN-NEXT:    v_mul_lo_u32 v3, v2, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v7, v8, vcc
-; GCN-NEXT:    v_mul_lo_u32 v8, v2, v5
-; GCN-NEXT:    v_mul_hi_u32 v5, v2, v5
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v6, v5, vcc
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v9, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
-; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v3
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v7, v5, vcc
-; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v2, v4, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v5, s4, v3
-; GCN-NEXT:    v_mul_hi_u32 v6, s4, v0
-; GCN-NEXT:    v_mul_lo_u32 v8, s5, v0
-; GCN-NEXT:    s_mov_b32 s5, s9
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; GCN-NEXT:    v_mul_lo_u32 v6, s4, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
+; GCN-NEXT:    v_mul_lo_u32 v5, s8, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s6, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; GCN-NEXT:    v_mul_lo_u32 v6, v1, v4
+; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v5, s6, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, s6, v0
+; GCN-NEXT:    v_mul_lo_u32 v8, s8, v0
+; GCN-NEXT:    s_mov_b32 s8, s4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GCN-NEXT:    v_mul_lo_u32 v7, s6, v0
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
 ; GCN-NEXT:    v_mul_lo_u32 v10, v0, v5
 ; GCN-NEXT:    v_mul_hi_u32 v12, v0, v5
-; GCN-NEXT:    v_mul_hi_u32 v11, v0, v6
-; GCN-NEXT:    v_mul_hi_u32 v9, v3, v6
-; GCN-NEXT:    v_mul_lo_u32 v6, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v8, v3, v5
+; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v8, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, v7, v12, vcc
-; GCN-NEXT:    v_mul_lo_u32 v3, v3, v5
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v11, v9, vcc
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v5, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v2, v5, s[0:1]
-; GCN-NEXT:    s_add_u32 s0, s2, s10
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
-; GCN-NEXT:    s_addc_u32 s1, s3, s10
-; GCN-NEXT:    s_xor_b64 s[14:15], s[0:1], s[10:11]
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
-; GCN-NEXT:    v_mul_lo_u32 v3, s14, v2
-; GCN-NEXT:    v_mul_hi_u32 v4, s14, v0
-; GCN-NEXT:    v_mul_hi_u32 v5, s14, v2
-; GCN-NEXT:    v_mul_hi_u32 v6, s15, v2
-; GCN-NEXT:    v_mul_lo_u32 v2, s15, v2
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v7, v5, vcc
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    s_ashr_i32 s6, s7, 31
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
+; GCN-NEXT:    s_add_u32 s0, s2, s6
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    s_mov_b32 s7, s6
+; GCN-NEXT:    s_addc_u32 s1, s3, s6
+; GCN-NEXT:    s_xor_b64 s[14:15], s[0:1], s[6:7]
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s14, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s14, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s14, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s15, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s15, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v5, s15, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s15, v0
-; GCN-NEXT:    s_mov_b32 s4, s8
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v4, v0, vcc
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v7, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v1, s12, v1
 ; GCN-NEXT:    v_mul_hi_u32 v2, s12, v0
 ; GCN-NEXT:    v_mul_lo_u32 v3, s13, v0
 ; GCN-NEXT:    v_mul_lo_u32 v0, s12, v0
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e64 v0, s[0:1], s14, v0
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s15, v1
 ; GCN-NEXT:    v_mov_b32_e32 v3, s13
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s14, v0
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s12, v0
-; GCN-NEXT:    v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s13, v5
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s12, v4
-; GCN-NEXT:    v_subrev_i32_e64 v3, s[0:1], s12, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s13, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[2:3]
-; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v4, s[2:3], s12, v0
+; GCN-NEXT:    v_subbrev_u32_e64 v5, vcc, 0, v2, s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s12, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v5, s15
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v5, v1, vcc
+; GCN-NEXT:    v_subb_u32_e64 v1, vcc, v5, v1, s[0:1]
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v1
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v0
@@ -1008,14 +1004,14 @@ define amdgpu_kernel void @s_test_srem33_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    v_xor_b32_e32 v0, s10, v0
-; GCN-NEXT:    v_xor_b32_e32 v1, s10, v1
-; GCN-NEXT:    v_mov_b32_e32 v2, s10
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s10, v0
+; GCN-NEXT:    v_xor_b32_e32 v0, s6, v0
+; GCN-NEXT:    v_xor_b32_e32 v1, s6, v1
+; GCN-NEXT:    v_mov_b32_e32 v2, s6
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
-; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem33_64:
@@ -1023,118 +1019,112 @@ define amdgpu_kernel void @s_test_srem33_64(i64 addrspace(1)* %out, i64 %x, i64
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_ashr_i32 s2, s7, 31
-; GCN-IR-NEXT:    s_ashr_i64 s[10:11], s[0:1], 31
-; GCN-IR-NEXT:    s_ashr_i32 s0, s1, 31
-; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[6:7], 31
-; GCN-IR-NEXT:    s_mov_b32 s3, s2
+; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[6:7], 31
+; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[0:1], 31
+; GCN-IR-NEXT:    s_ashr_i32 s10, s1, 31
+; GCN-IR-NEXT:    s_ashr_i32 s0, s7, 31
 ; GCN-IR-NEXT:    s_mov_b32 s1, s0
-; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[8:9], s[2:3]
-; GCN-IR-NEXT:    s_xor_b64 s[8:9], s[10:11], s[0:1]
-; GCN-IR-NEXT:    s_sub_u32 s6, s6, s2
-; GCN-IR-NEXT:    s_subb_u32 s7, s7, s2
-; GCN-IR-NEXT:    s_sub_u32 s8, s8, s0
-; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s8
-; GCN-IR-NEXT:    s_subb_u32 s9, s9, s0
-; GCN-IR-NEXT:    s_add_i32 s12, s12, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s13, s9
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s12
-; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s6
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s13
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s9, 0
-; GCN-IR-NEXT:    s_add_i32 s12, s12, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s13, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s13
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s12
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v2, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[12:13], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[8:9], 0
+; GCN-IR-NEXT:    s_mov_b32 s11, s10
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[8:9], s[10:11]
+; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[2:3], s[0:1]
+; GCN-IR-NEXT:    s_sub_u32 s2, s2, s0
+; GCN-IR-NEXT:    s_subb_u32 s3, s3, s0
+; GCN-IR-NEXT:    s_sub_u32 s6, s6, s10
+; GCN-IR-NEXT:    s_subb_u32 s7, s7, s10
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[10:11]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[10:11], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[10:11], s[10:11], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[10:11]
-; GCN-IR-NEXT:    s_cbranch_vccz BB8_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[4:5], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[2:3], 0
+; GCN-IR-NEXT:    s_mov_b64 s[8:9], 0
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[10:11], s[12:13]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s6
+; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s7
+; GCN-IR-NEXT:    s_cmp_eq_u32 s7, 0
+; GCN-IR-NEXT:    s_cselect_b32 s12, s10, s11
+; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s2
+; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s16, s10, s11
+; GCN-IR-NEXT:    s_sub_u32 s10, s12, s16
+; GCN-IR-NEXT:    s_subb_u32 s11, 0, 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[18:19], s[10:11], 63
+; GCN-IR-NEXT:    s_mov_b32 s13, 0
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[18:19]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[18:19], s[10:11], 63
+; GCN-IR-NEXT:    s_xor_b64 s[20:21], s[14:15], -1
+; GCN-IR-NEXT:    s_and_b64 s[18:19], s[20:21], s[18:19]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[18:19]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB8_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s18, s10, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-IR-NEXT:    s_addc_u32 s19, s11, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[18:19], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s10, 63, s10
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[2:3], s10
+; GCN-IR-NEXT:    s_cbranch_vccz BB8_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_not_b32_e32 v2, v2
-; GCN-IR-NEXT:    s_add_u32 s10, s8, -1
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[6:7], v4
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v2, v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s11, s9, -1
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], -1, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[18:19], s[2:3], s18
+; GCN-IR-NEXT:    s_add_u32 s10, s6, -1
+; GCN-IR-NEXT:    s_addc_u32 s11, s7, -1
+; GCN-IR-NEXT:    s_not_b64 s[8:9], s[12:13]
+; GCN-IR-NEXT:    s_mov_b32 s17, s13
+; GCN-IR-NEXT:    s_add_u32 s12, s8, s16
+; GCN-IR-NEXT:    s_addc_u32 s13, s9, s13
+; GCN-IR-NEXT:    s_mov_b64 s[16:17], 0
+; GCN-IR-NEXT:    s_mov_b32 s9, 0
 ; GCN-IR-NEXT:  BB8_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s11
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s10, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s8, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s9, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s8, s15, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[18:19], s[18:19], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[8:9]
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[16:17], s[14:15]
+; GCN-IR-NEXT:    s_sub_u32 s8, s10, s18
+; GCN-IR-NEXT:    s_subb_u32 s8, s11, s19
+; GCN-IR-NEXT:    s_ashr_i32 s16, s8, 31
+; GCN-IR-NEXT:    s_mov_b32 s17, s16
+; GCN-IR-NEXT:    s_and_b32 s8, s16, 1
+; GCN-IR-NEXT:    s_and_b64 s[20:21], s[16:17], s[6:7]
+; GCN-IR-NEXT:    s_sub_u32 s18, s18, s20
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s12
+; GCN-IR-NEXT:    s_subb_u32 s19, s19, s21
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s13
+; GCN-IR-NEXT:    s_add_u32 s12, s12, 1
+; GCN-IR-NEXT:    s_addc_u32 s13, s13, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[16:17], s[8:9]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB8_3
+; GCN-IR-NEXT:  BB8_4: ; %Flow6
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[14:15], 1
+; GCN-IR-NEXT:    s_or_b64 s[8:9], s[8:9], s[10:11]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
 ; GCN-IR-NEXT:    s_branch BB8_6
-; GCN-IR-NEXT:  BB8_4:
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB8_7
 ; GCN-IR-NEXT:  BB8_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB8_6: ; %Flow6
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB8_7: ; %udiv-end
-; GCN-IR-NEXT:    v_mul_lo_u32 v1, s8, v1
-; GCN-IR-NEXT:    v_mul_hi_u32 v2, s8, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v3, s9, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v0, s8, v0
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s3
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[14:15]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[14:15]
+; GCN-IR-NEXT:  BB8_6: ; %udiv-end
+; GCN-IR-NEXT:    v_mul_lo_u32 v1, s6, v1
+; GCN-IR-NEXT:    v_mul_hi_u32 v2, s6, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v3, s7, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v0, s6, v0
+; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s7
-; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, s2, v0
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, s3, v1
+; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v2, s3
-; GCN-IR-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
+; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, s0, v0
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, s1, v1
+; GCN-IR-NEXT:    v_mov_b32_e32 v2, s1
+; GCN-IR-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
-; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-IR-NEXT:    s_endpgm
@@ -1174,7 +1164,7 @@ define amdgpu_kernel void @s_test_srem24_48(i48 addrspace(1)* %out, i48 %x, i48
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
 ; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v1|
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v5, vcc
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v4, v1
 ; GCN-NEXT:    v_mul_lo_u32 v0, v1, v0
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v2, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
@@ -1192,119 +1182,113 @@ define amdgpu_kernel void @s_test_srem24_48(i48 addrspace(1)* %out, i48 %x, i48
 ; GCN-IR-NEXT:    s_load_dword s0, s[0:1], 0xe
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_sext_i32_i16 s3, s3
+; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[2:3], 24
 ; GCN-IR-NEXT:    s_sext_i32_i16 s7, s0
-; GCN-IR-NEXT:    s_ashr_i64 s[0:1], s[2:3], 24
-; GCN-IR-NEXT:    s_ashr_i32 s2, s3, 31
-; GCN-IR-NEXT:    s_ashr_i32 s10, s7, 31
-; GCN-IR-NEXT:    s_mov_b32 s3, s2
-; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[6:7], 24
-; GCN-IR-NEXT:    s_mov_b32 s11, s10
-; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
-; GCN-IR-NEXT:    s_xor_b64 s[8:9], s[8:9], s[10:11]
-; GCN-IR-NEXT:    s_sub_u32 s6, s0, s2
-; GCN-IR-NEXT:    s_subb_u32 s7, s1, s2
-; GCN-IR-NEXT:    s_sub_u32 s8, s8, s10
-; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s8
-; GCN-IR-NEXT:    s_subb_u32 s9, s9, s10
-; GCN-IR-NEXT:    s_add_i32 s12, s12, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s13, s9
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s12
-; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s6
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s13
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s9, 0
-; GCN-IR-NEXT:    s_add_i32 s12, s12, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s13, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s13
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s12
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v2, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[12:13], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[8:9], 0
+; GCN-IR-NEXT:    s_ashr_i32 s0, s3, 31
+; GCN-IR-NEXT:    s_ashr_i32 s12, s7, 31
+; GCN-IR-NEXT:    s_mov_b32 s1, s0
+; GCN-IR-NEXT:    s_ashr_i64 s[10:11], s[6:7], 24
+; GCN-IR-NEXT:    s_mov_b32 s13, s12
+; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[8:9], s[0:1]
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[10:11], s[12:13]
+; GCN-IR-NEXT:    s_sub_u32 s2, s2, s0
+; GCN-IR-NEXT:    s_subb_u32 s3, s3, s0
+; GCN-IR-NEXT:    s_sub_u32 s6, s6, s12
+; GCN-IR-NEXT:    s_subb_u32 s7, s7, s12
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[10:11]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[10:11], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[10:11], s[10:11], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[10:11]
-; GCN-IR-NEXT:    s_cbranch_vccz BB9_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[4:5], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[2:3], 0
+; GCN-IR-NEXT:    s_mov_b64 s[8:9], 0
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[10:11], s[12:13]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s6
+; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s7
+; GCN-IR-NEXT:    s_cmp_eq_u32 s7, 0
+; GCN-IR-NEXT:    s_cselect_b32 s12, s10, s11
+; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s2
+; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s16, s10, s11
+; GCN-IR-NEXT:    s_sub_u32 s10, s12, s16
+; GCN-IR-NEXT:    s_subb_u32 s11, 0, 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[18:19], s[10:11], 63
+; GCN-IR-NEXT:    s_mov_b32 s13, 0
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[18:19]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[18:19], s[10:11], 63
+; GCN-IR-NEXT:    s_xor_b64 s[20:21], s[14:15], -1
+; GCN-IR-NEXT:    s_and_b64 s[18:19], s[20:21], s[18:19]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[18:19]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB9_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s18, s10, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-IR-NEXT:    s_addc_u32 s19, s11, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[18:19], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s10, 63, s10
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[2:3], s10
+; GCN-IR-NEXT:    s_cbranch_vccz BB9_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_not_b32_e32 v2, v2
-; GCN-IR-NEXT:    s_add_u32 s10, s8, -1
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[6:7], v4
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v2, v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s11, s9, -1
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], -1, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[18:19], s[2:3], s18
+; GCN-IR-NEXT:    s_add_u32 s10, s6, -1
+; GCN-IR-NEXT:    s_addc_u32 s11, s7, -1
+; GCN-IR-NEXT:    s_not_b64 s[8:9], s[12:13]
+; GCN-IR-NEXT:    s_mov_b32 s17, s13
+; GCN-IR-NEXT:    s_add_u32 s12, s8, s16
+; GCN-IR-NEXT:    s_addc_u32 s13, s9, s13
+; GCN-IR-NEXT:    s_mov_b64 s[16:17], 0
+; GCN-IR-NEXT:    s_mov_b32 s9, 0
 ; GCN-IR-NEXT:  BB9_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s11
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s10, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s8, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s9, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s8, s15, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[18:19], s[18:19], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[8:9]
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[16:17], s[14:15]
+; GCN-IR-NEXT:    s_sub_u32 s8, s10, s18
+; GCN-IR-NEXT:    s_subb_u32 s8, s11, s19
+; GCN-IR-NEXT:    s_ashr_i32 s16, s8, 31
+; GCN-IR-NEXT:    s_mov_b32 s17, s16
+; GCN-IR-NEXT:    s_and_b32 s8, s16, 1
+; GCN-IR-NEXT:    s_and_b64 s[20:21], s[16:17], s[6:7]
+; GCN-IR-NEXT:    s_sub_u32 s18, s18, s20
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s12
+; GCN-IR-NEXT:    s_subb_u32 s19, s19, s21
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s13
+; GCN-IR-NEXT:    s_add_u32 s12, s12, 1
+; GCN-IR-NEXT:    s_addc_u32 s13, s13, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[16:17], s[8:9]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB9_3
+; GCN-IR-NEXT:  BB9_4: ; %Flow3
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[14:15], 1
+; GCN-IR-NEXT:    s_or_b64 s[8:9], s[8:9], s[10:11]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
 ; GCN-IR-NEXT:    s_branch BB9_6
-; GCN-IR-NEXT:  BB9_4:
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB9_7
 ; GCN-IR-NEXT:  BB9_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB9_6: ; %Flow3
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB9_7: ; %udiv-end
-; GCN-IR-NEXT:    v_mul_lo_u32 v1, s8, v1
-; GCN-IR-NEXT:    v_mul_hi_u32 v2, s8, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v3, s9, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v0, s8, v0
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s3
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[14:15]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[14:15]
+; GCN-IR-NEXT:  BB9_6: ; %udiv-end
+; GCN-IR-NEXT:    v_mul_lo_u32 v1, s6, v1
+; GCN-IR-NEXT:    v_mul_hi_u32 v2, s6, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v3, s7, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v0, s6, v0
+; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s7
-; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, s2, v0
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, s3, v1
+; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v2, s3
-; GCN-IR-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
+; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, s0, v0
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, s1, v1
+; GCN-IR-NEXT:    v_mov_b32_e32 v2, s1
+; GCN-IR-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
-; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    buffer_store_short v1, off, s[4:7], 0 offset:4
 ; GCN-IR-NEXT:    buffer_store_dword v0, off, s[4:7], 0
@@ -1404,25 +1388,25 @@ define amdgpu_kernel void @s_test_srem_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_mul_lo_u32 v0, s8, v0
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e64 v0, s[0:1], 24, v0
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, 0, v1
 ; GCN-NEXT:    v_mov_b32_e32 v3, s9
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, 24, v0
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s8, v0
-; GCN-NEXT:    v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s9, v5
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s8, v4
-; GCN-NEXT:    v_subrev_i32_e64 v3, s[0:1], s8, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s9, v5
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[2:3]
-; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v4, s[2:3], s8, v0
+; GCN-NEXT:    v_subbrev_u32_e64 v5, vcc, 0, v2, s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s8, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s9, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s8, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_subb_u32_e64 v1, vcc, 0, v1, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v6
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v1
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
@@ -1430,104 +1414,98 @@ define amdgpu_kernel void @s_test_srem_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem_k_num_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
-; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_ashr_i32 s0, s7, 31
-; GCN-IR-NEXT:    s_mov_b32 s1, s0
-; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[6:7], s[0:1]
-; GCN-IR-NEXT:    s_sub_u32 s2, s2, s0
-; GCN-IR-NEXT:    s_subb_u32 s3, s3, s0
-; GCN-IR-NEXT:    s_flbit_i32_b32 s6, s2
-; GCN-IR-NEXT:    s_add_i32 s6, s6, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s7, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s6
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 0xffffffc5, v2
-; GCN-IR-NEXT:    v_addc_u32_e64 v1, s[6:7], 0, -1, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[2:3], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[6:7]
-; GCN-IR-NEXT:    s_cbranch_vccz BB10_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[3:4], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], 24, v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_ashr_i32 s4, s3, 31
+; GCN-IR-NEXT:    s_mov_b32 s5, s4
+; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[2:3], s[4:5]
+; GCN-IR-NEXT:    s_sub_u32 s2, s2, s4
+; GCN-IR-NEXT:    s_subb_u32 s3, s3, s4
+; GCN-IR-NEXT:    s_flbit_i32_b32 s4, s2
+; GCN-IR-NEXT:    s_add_i32 s4, s4, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s5, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s8, s4, s5
+; GCN-IR-NEXT:    s_add_u32 s6, s8, 0xffffffc5
+; GCN-IR-NEXT:    s_addc_u32 s7, 0, -1
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[2:3], 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[6:7], 63
+; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[12:13], s[6:7], 63
+; GCN-IR-NEXT:    s_xor_b64 s[14:15], s[10:11], -1
+; GCN-IR-NEXT:    s_and_b64 s[12:13], s[14:15], s[12:13]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[12:13]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB10_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s12, s6, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT:    s_addc_u32 s13, s7, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s7
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s6, 63, s6
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], 24, s6
+; GCN-IR-NEXT:    s_cbranch_vccz BB10_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
+; GCN-IR-NEXT:    s_lshr_b64 s[14:15], 24, s12
 ; GCN-IR-NEXT:    s_add_u32 s6, s2, -1
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], 24, v3
-; GCN-IR-NEXT:    v_sub_i32_e32 v4, vcc, 58, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
 ; GCN-IR-NEXT:    s_addc_u32 s7, s3, -1
-; GCN-IR-NEXT:    v_subb_u32_e64 v5, s[0:1], 0, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_sub_u32 s8, 58, s8
+; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
+; GCN-IR-NEXT:    s_mov_b32 s5, 0
 ; GCN-IR-NEXT:  BB10_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s7
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s6, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s2, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s3, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s4, s11, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[4:5]
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[12:13], s[10:11]
+; GCN-IR-NEXT:    s_sub_u32 s4, s6, s14
+; GCN-IR-NEXT:    s_subb_u32 s4, s7, s15
+; GCN-IR-NEXT:    s_ashr_i32 s12, s4, 31
+; GCN-IR-NEXT:    s_mov_b32 s13, s12
+; GCN-IR-NEXT:    s_and_b32 s4, s12, 1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[12:13], s[2:3]
+; GCN-IR-NEXT:    s_sub_u32 s14, s14, s16
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_subb_u32 s15, s15, s17
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    s_add_u32 s8, s8, 1
+; GCN-IR-NEXT:    s_addc_u32 s9, s9, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[8:9], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], s[4:5]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB10_3
+; GCN-IR-NEXT:  BB10_4: ; %Flow5
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s5
 ; GCN-IR-NEXT:    s_branch BB10_6
-; GCN-IR-NEXT:  BB10_4:
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, 24, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB10_7
 ; GCN-IR-NEXT:  BB10_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB10_6: ; %Flow5
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB10_7: ; %udiv-end
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, 24, 0, s[10:11]
+; GCN-IR-NEXT:  BB10_6: ; %udiv-end
 ; GCN-IR-NEXT:    v_mul_lo_u32 v1, s2, v1
 ; GCN-IR-NEXT:    v_mul_hi_u32 v2, s2, v0
 ; GCN-IR-NEXT:    v_mul_lo_u32 v3, s3, v0
 ; GCN-IR-NEXT:    v_mul_lo_u32 v0, s2, v0
-; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 24, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    s_mov_b32 s6, -1
-; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-IR-NEXT:    s_mov_b32 s2, -1
+; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %result = srem i64 24, %x
   store i64 %result, i64 addrspace(1)* %out
@@ -2076,17 +2054,17 @@ define amdgpu_kernel void @s_test_srem24_k_num_i64(i64 addrspace(1)* %out, i64 %
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s4
 ; GCN-NEXT:    s_ashr_i32 s5, s4, 30
 ; GCN-NEXT:    s_or_b32 s5, s5, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s5
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v0
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v0
 ; GCN-NEXT:    s_mov_b32 s2, -1
 ; GCN-NEXT:    v_mul_f32_e32 v1, s6, v1
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-NEXT:    v_mad_f32 v2, -v1, v0, s6
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[6:7], |v2|, |v0|
+; GCN-NEXT:    s_cmp_lg_u32 s6, 0
+; GCN-NEXT:    s_cselect_b32 s5, s5, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s5, v1
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s4
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, 24, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
@@ -2103,17 +2081,17 @@ define amdgpu_kernel void @s_test_srem24_k_num_i64(i64 addrspace(1)* %out, i64 %
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s4
 ; GCN-IR-NEXT:    s_ashr_i32 s5, s4, 30
 ; GCN-IR-NEXT:    s_or_b32 s5, s5, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s5
-; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v1, v0
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v1, v0
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v1, s6, v1
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-IR-NEXT:    v_mad_f32 v2, -v1, v0, s6
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[6:7], |v2|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u32 s6, 0
+; GCN-IR-NEXT:    s_cselect_b32 s5, s5, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s5, v1
 ; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, s4
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 24, v0
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 24
@@ -2130,23 +2108,23 @@ define amdgpu_kernel void @s_test_srem24_k_den_i64(i64 addrspace(1)* %out, i64 %
 ; GCN-LABEL: s_test_srem24_k_den_i64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-NEXT:    s_mov_b32 s8, 0x46b6fe00
+; GCN-NEXT:    s_mov_b32 s1, 0x46b6fe00
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    s_mov_b32 s2, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_ashr_i64 s[6:7], s[6:7], 40
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s6
 ; GCN-NEXT:    s_ashr_i32 s0, s6, 30
-; GCN-NEXT:    s_or_b32 s0, s0, 1
-; GCN-NEXT:    v_mov_b32_e32 v1, s0
-; GCN-NEXT:    v_mul_f32_e32 v2, 0x38331158, v0
-; GCN-NEXT:    v_trunc_f32_e32 v2, v2
-; GCN-NEXT:    v_mad_f32 v0, -v2, s8, v0
-; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, s8
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_or_b32 s7, s0, 1
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x38331158, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mad_f32 v0, -v1, s1, v0
+; GCN-NEXT:    v_cvt_i32_f32_e32 v1, v1
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v0|, s1
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cselect_b32 s0, s7, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v1
 ; GCN-NEXT:    s_movk_i32 s0, 0x5b7f
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-NEXT:    s_mov_b32 s0, s4
 ; GCN-NEXT:    s_mov_b32 s1, s5
@@ -2159,23 +2137,23 @@ define amdgpu_kernel void @s_test_srem24_k_den_i64(i64 addrspace(1)* %out, i64 %
 ; GCN-IR-LABEL: s_test_srem24_k_den_i64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-IR-NEXT:    s_mov_b32 s8, 0x46b6fe00
+; GCN-IR-NEXT:    s_mov_b32 s1, 0x46b6fe00
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_ashr_i64 s[6:7], s[6:7], 40
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s6
 ; GCN-IR-NEXT:    s_ashr_i32 s0, s6, 30
-; GCN-IR-NEXT:    s_or_b32 s0, s0, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s0
-; GCN-IR-NEXT:    v_mul_f32_e32 v2, 0x38331158, v0
-; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_mad_f32 v0, -v2, s8, v0
-; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, s8
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-IR-NEXT:    s_or_b32 s7, s0, 1
+; GCN-IR-NEXT:    v_mul_f32_e32 v1, 0x38331158, v0
+; GCN-IR-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-IR-NEXT:    v_mad_f32 v0, -v1, s1, v0
+; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v1, v1
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v0|, s1
+; GCN-IR-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-IR-NEXT:    s_cselect_b32 s0, s7, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s0, v1
 ; GCN-IR-NEXT:    s_movk_i32 s0, 0x5b7f
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
 ; GCN-IR-NEXT:    s_mov_b32 s1, s5
diff --git a/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll b/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll
index 16c6676ae582a..9c673c74248b9 100644
--- a/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll
@@ -6,14 +6,14 @@
 define void @local_store_i56(i56 addrspace(3)* %ptr, i56 %arg) #0 {
 ; CIVI-LABEL: local_store_i56:
 ; CIVI:       ; %bb.0:
-; CIVI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CIVI-NEXT:    s_mov_b32 m0, -1
-; CIVI-NEXT:    ds_write_b16 v0, v2 offset:4
-; CIVI-NEXT:    ds_write_b32 v0, v1
-; CIVI-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
-; CIVI-NEXT:    ds_write_b8 v0, v1 offset:6
-; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
-; CIVI-NEXT:    s_setpc_b64 s[30:31]
+; CIVI-NEXT:	s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CIVI-NEXT:	v_lshrrev_b32_e32 v3, 16, v2
+; CIVI-NEXT:	s_mov_b32 m0, -1
+; CIVI-NEXT:	ds_write_b8 v0, v3 offset:6
+; CIVI-NEXT:	ds_write_b16 v0, v2 offset:4
+; CIVI-NEXT:	ds_write_b32 v0, v1
+; CIVI-NEXT:	s_waitcnt lgkmcnt(0)
+; CIVI-NEXT:	s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: local_store_i56:
 ; GFX9:       ; %bb.0:
@@ -30,72 +30,70 @@ define void @local_store_i56(i56 addrspace(3)* %ptr, i56 %arg) #0 {
 define amdgpu_kernel void @local_store_i55(i55 addrspace(3)* %ptr, i55 %arg) #0 {
 ; HAWAII-LABEL: local_store_i55:
 ; HAWAII:       ; %bb.0:
-; HAWAII-NEXT:    s_add_u32 s0, s4, 14
-; HAWAII-NEXT:    s_addc_u32 s1, s5, 0
-; HAWAII-NEXT:    v_mov_b32_e32 v0, s0
-; HAWAII-NEXT:    v_mov_b32_e32 v1, s1
-; HAWAII-NEXT:    flat_load_ubyte v0, v[0:1]
-; HAWAII-NEXT:    s_load_dword s0, s[4:5], 0x0
-; HAWAII-NEXT:    s_load_dword s1, s[4:5], 0x2
-; HAWAII-NEXT:    s_load_dword s2, s[4:5], 0x3
-; HAWAII-NEXT:    s_mov_b32 m0, -1
-; HAWAII-NEXT:    s_waitcnt lgkmcnt(0)
-; HAWAII-NEXT:    v_mov_b32_e32 v1, s0
-; HAWAII-NEXT:    v_mov_b32_e32 v3, s1
-; HAWAII-NEXT:    v_mov_b32_e32 v2, s2
-; HAWAII-NEXT:    ds_write_b16 v1, v2 offset:4
-; HAWAII-NEXT:    s_waitcnt vmcnt(0)
-; HAWAII-NEXT:    v_and_b32_e32 v0, 0x7f, v0
-; HAWAII-NEXT:    ds_write_b8 v1, v0 offset:6
-; HAWAII-NEXT:    ds_write_b32 v1, v3
-; HAWAII-NEXT:    s_endpgm
+; HAWAII-NEXT:	s_or_b32 s0, s4, 14
+; HAWAII-NEXT:	v_mov_b32_e32 v0, s0
+; HAWAII-NEXT:	v_mov_b32_e32 v1, s5
+; HAWAII-NEXT:	flat_load_ubyte v0, v[0:1]
+; HAWAII-NEXT:	s_load_dword s0, s[4:5], 0x0
+; HAWAII-NEXT:	s_load_dword s1, s[4:5], 0x2
+; HAWAII-NEXT:	s_load_dword s2, s[4:5], 0x3
+; HAWAII-NEXT:	s_mov_b32 m0, -1
+; HAWAII-NEXT:	s_waitcnt lgkmcnt(0)
+; HAWAII-NEXT:	v_mov_b32_e32 v1, s0
+; HAWAII-NEXT:	v_mov_b32_e32 v2, s1
+; HAWAII-NEXT:	v_mov_b32_e32 v3, s2
+; HAWAII-NEXT:	s_waitcnt vmcnt(0)
+; HAWAII-NEXT:	v_and_b32_e32 v0, 0x7f, v0
+; HAWAII-NEXT:	ds_write_b8 v1, v0 offset:6
+; HAWAII-NEXT:	ds_write_b16 v1, v3 offset:4
+; HAWAII-NEXT:	ds_write_b32 v1, v2
+; HAWAII-NEXT:	s_endpgm
 ;
 ; FIJI-LABEL: local_store_i55:
 ; FIJI:       ; %bb.0:
-; FIJI-NEXT:    s_load_dword s0, s[4:5], 0x0
-; FIJI-NEXT:    s_load_dword s2, s[4:5], 0x8
-; FIJI-NEXT:    s_load_dword s1, s[4:5], 0xc
-; FIJI-NEXT:    s_mov_b32 m0, -1
-; FIJI-NEXT:    s_waitcnt lgkmcnt(0)
-; FIJI-NEXT:    v_mov_b32_e32 v2, s0
-; FIJI-NEXT:    s_and_b32 s3, s1, 0xffff
-; FIJI-NEXT:    s_add_u32 s0, s4, 14
-; FIJI-NEXT:    v_mov_b32_e32 v3, s1
-; FIJI-NEXT:    s_addc_u32 s1, s5, 0
-; FIJI-NEXT:    v_mov_b32_e32 v0, s0
-; FIJI-NEXT:    v_mov_b32_e32 v1, s1
-; FIJI-NEXT:    flat_load_ubyte v0, v[0:1]
-; FIJI-NEXT:    ds_write_b16 v2, v3 offset:4
-; FIJI-NEXT:    v_mov_b32_e32 v3, s2
-; FIJI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)
-; FIJI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; FIJI-NEXT:    v_or_b32_e32 v0, s3, v0
-; FIJI-NEXT:    v_bfe_u32 v0, v0, 16, 7
-; FIJI-NEXT:    ds_write_b8 v2, v0 offset:6
-; FIJI-NEXT:    ds_write_b32 v2, v3
-; FIJI-NEXT:    s_endpgm
+; FIJI-NEXT:	s_or_b32 s0, s4, 14
+; FIJI-NEXT:	v_mov_b32_e32 v0, s0
+; FIJI-NEXT:	v_mov_b32_e32 v1, s5
+; FIJI-NEXT:	flat_load_ubyte v0, v[0:1]
+; FIJI-NEXT:	s_load_dword s0, s[4:5], 0x0
+; FIJI-NEXT:	s_load_dword s1, s[4:5], 0x8
+; FIJI-NEXT:	s_load_dword s2, s[4:5], 0xc
+; FIJI-NEXT:	s_mov_b32 m0, -1
+; FIJI-NEXT:	s_waitcnt lgkmcnt(0)
+; FIJI-NEXT:	v_mov_b32_e32 v1, s0
+; FIJI-NEXT:	v_mov_b32_e32 v3, s1
+; FIJI-NEXT:	s_and_b32 s3, s2, 0xffff
+; FIJI-NEXT:	v_mov_b32_e32 v2, s2
+; FIJI-NEXT:	s_waitcnt vmcnt(0)
+; FIJI-NEXT:	v_lshlrev_b32_e32 v0, 16, v0
+; FIJI-NEXT:	v_or_b32_e32 v0, s3, v0
+; FIJI-NEXT:	v_bfe_u32 v0, v0, 16, 7
+; FIJI-NEXT:	ds_write_b8 v1, v0 offset:6
+; FIJI-NEXT:	ds_write_b16 v1, v2 offset:4
+; FIJI-NEXT:	ds_write_b32 v1, v3
+; FIJI-NEXT:	s_endpgm
 ;
 ; GFX9-LABEL: local_store_i55:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_load_ubyte_d16_hi v2, v[0:1], off offset:14
-; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s1, s[4:5], 0x8
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0xc
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; GFX9-NEXT:    s_and_b32 s3, s2, 0xffff
-; GFX9-NEXT:    ds_write_b16 v0, v1 offset:4
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_e32 v1, s3, v2
-; GFX9-NEXT:    v_and_b32_e32 v1, 0x7fffff, v1
-; GFX9-NEXT:    ds_write_b8_d16_hi v0, v1 offset:6
-; GFX9-NEXT:    ds_write_b32 v0, v3
-; GFX9-NEXT:    s_endpgm
+; GFX9-NEXT:	v_mov_b32_e32 v0, s4
+; GFX9-NEXT:	v_mov_b32_e32 v1, s5
+; GFX9-NEXT:	v_mov_b32_e32 v2, 0
+; GFX9-NEXT:	global_load_ubyte_d16_hi v2, v[0:1], off offset:14
+; GFX9-NEXT:	s_load_dword s0, s[4:5], 0x0
+; GFX9-NEXT:	s_load_dword s1, s[4:5], 0x8
+; GFX9-NEXT:	s_load_dword s2, s[4:5], 0xc
+; GFX9-NEXT:	s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:	v_mov_b32_e32 v0, s0
+; GFX9-NEXT:	v_mov_b32_e32 v3, s1
+; GFX9-NEXT:	s_and_b32 s3, s2, 0xffff
+; GFX9-NEXT:	v_mov_b32_e32 v1, s2
+; GFX9-NEXT:	s_waitcnt vmcnt(0)
+; GFX9-NEXT:	v_or_b32_e32 v2, s3, v2
+; GFX9-NEXT:	v_and_b32_e32 v2, 0x7fffff, v2
+; GFX9-NEXT:	ds_write_b8_d16_hi v0, v2 offset:6
+; GFX9-NEXT:	ds_write_b16 v0, v1 offset:4
+; GFX9-NEXT:	ds_write_b32 v0, v3
+; GFX9-NEXT:	s_endpgm
   store i55 %arg, i55 addrspace(3)* %ptr, align 8
   ret void
 }
@@ -103,31 +101,31 @@ define amdgpu_kernel void @local_store_i55(i55 addrspace(3)* %ptr, i55 %arg) #0
 define amdgpu_kernel void @local_store_i48(i48 addrspace(3)* %ptr, i48 %arg) #0 {
 ; HAWAII-LABEL: local_store_i48:
 ; HAWAII:       ; %bb.0:
-; HAWAII-NEXT:    s_load_dword s0, s[4:5], 0x0
-; HAWAII-NEXT:    s_load_dword s1, s[4:5], 0x2
-; HAWAII-NEXT:    s_load_dword s2, s[4:5], 0x3
-; HAWAII-NEXT:    s_mov_b32 m0, -1
-; HAWAII-NEXT:    s_waitcnt lgkmcnt(0)
-; HAWAII-NEXT:    v_mov_b32_e32 v0, s0
-; HAWAII-NEXT:    v_mov_b32_e32 v1, s2
-; HAWAII-NEXT:    ds_write_b16 v0, v1 offset:4
-; HAWAII-NEXT:    v_mov_b32_e32 v1, s1
-; HAWAII-NEXT:    ds_write_b32 v0, v1
-; HAWAII-NEXT:    s_endpgm
+; HAWAII-NEXT:	s_load_dword s0, s[4:5], 0x0
+; HAWAII-NEXT:	s_load_dword s1, s[4:5], 0x2
+; HAWAII-NEXT:	s_load_dword s2, s[4:5], 0x3
+; HAWAII-NEXT:	s_mov_b32 m0, -1
+; HAWAII-NEXT:	s_waitcnt lgkmcnt(0)
+; HAWAII-NEXT:	v_mov_b32_e32 v0, s0
+; HAWAII-NEXT:	v_mov_b32_e32 v2, s1
+; HAWAII-NEXT:	v_mov_b32_e32 v1, s2
+; HAWAII-NEXT:	ds_write_b16 v0, v1 offset:4
+; HAWAII-NEXT:	ds_write_b32 v0, v2
+; HAWAII-NEXT:	s_endpgm
 ;
 ; FIJI-LABEL: local_store_i48:
 ; FIJI:       ; %bb.0:
-; FIJI-NEXT:    s_load_dword s0, s[4:5], 0x0
-; FIJI-NEXT:    s_load_dword s1, s[4:5], 0x8
-; FIJI-NEXT:    s_load_dword s2, s[4:5], 0xc
-; FIJI-NEXT:    s_mov_b32 m0, -1
-; FIJI-NEXT:    s_waitcnt lgkmcnt(0)
-; FIJI-NEXT:    v_mov_b32_e32 v0, s0
-; FIJI-NEXT:    v_mov_b32_e32 v1, s2
-; FIJI-NEXT:    ds_write_b16 v0, v1 offset:4
-; FIJI-NEXT:    v_mov_b32_e32 v1, s1
-; FIJI-NEXT:    ds_write_b32 v0, v1
-; FIJI-NEXT:    s_endpgm
+; FIJI-NEXT:	s_load_dword s0, s[4:5], 0x0
+; FIJI-NEXT:	s_load_dword s1, s[4:5], 0x8
+; FIJI-NEXT:	s_load_dword s2, s[4:5], 0xc
+; FIJI-NEXT:	s_mov_b32 m0, -1
+; FIJI-NEXT:	s_waitcnt lgkmcnt(0)
+; FIJI-NEXT:	v_mov_b32_e32 v0, s0
+; FIJI-NEXT:	v_mov_b32_e32 v2, s1
+; FIJI-NEXT:	v_mov_b32_e32 v1, s2
+; FIJI-NEXT:	ds_write_b16 v0, v1 offset:4
+; FIJI-NEXT:	ds_write_b32 v0, v2
+; FIJI-NEXT:	s_endpgm
 ;
 ; GFX9-LABEL: local_store_i48:
 ; GFX9:       ; %bb.0:
@@ -148,35 +146,35 @@ define amdgpu_kernel void @local_store_i48(i48 addrspace(3)* %ptr, i48 %arg) #0
 define amdgpu_kernel void @local_store_i65(i65 addrspace(3)* %ptr, i65 %arg) #0 {
 ; HAWAII-LABEL: local_store_i65:
 ; HAWAII:       ; %bb.0:
-; HAWAII-NEXT:    s_load_dword s2, s[4:5], 0x0
-; HAWAII-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
-; HAWAII-NEXT:    s_load_dword s3, s[4:5], 0x4
-; HAWAII-NEXT:    s_mov_b32 m0, -1
-; HAWAII-NEXT:    s_waitcnt lgkmcnt(0)
-; HAWAII-NEXT:    v_mov_b32_e32 v2, s2
-; HAWAII-NEXT:    s_and_b32 s3, s3, 1
-; HAWAII-NEXT:    v_mov_b32_e32 v0, s3
-; HAWAII-NEXT:    ds_write_b8 v2, v0 offset:8
-; HAWAII-NEXT:    v_mov_b32_e32 v0, s0
-; HAWAII-NEXT:    v_mov_b32_e32 v1, s1
-; HAWAII-NEXT:    ds_write_b64 v2, v[0:1]
-; HAWAII-NEXT:    s_endpgm
+; HAWAII-NEXT:	s_load_dword s2, s[4:5], 0x0
+; HAWAII-NEXT:	s_load_dwordx2 s[0:1], s[4:5], 0x2
+; HAWAII-NEXT:	s_load_dword s3, s[4:5], 0x4
+; HAWAII-NEXT:	s_mov_b32 m0, -1
+; HAWAII-NEXT:	s_waitcnt lgkmcnt(0)
+; HAWAII-NEXT:	v_mov_b32_e32 v2, s2
+; HAWAII-NEXT:	v_mov_b32_e32 v0, s0
+; HAWAII-NEXT:	s_and_b32 s3, s3, 1
+; HAWAII-NEXT:	v_mov_b32_e32 v3, s3
+; HAWAII-NEXT:	v_mov_b32_e32 v1, s1
+; HAWAII-NEXT:	ds_write_b8 v2, v3 offset:8
+; HAWAII-NEXT:	ds_write_b64 v2, v[0:1]
+; HAWAII-NEXT:	s_endpgm
 ;
 ; FIJI-LABEL: local_store_i65:
 ; FIJI:       ; %bb.0:
-; FIJI-NEXT:    s_load_dword s2, s[4:5], 0x0
-; FIJI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
-; FIJI-NEXT:    s_load_dword s3, s[4:5], 0x10
-; FIJI-NEXT:    s_mov_b32 m0, -1
-; FIJI-NEXT:    s_waitcnt lgkmcnt(0)
-; FIJI-NEXT:    v_mov_b32_e32 v2, s2
-; FIJI-NEXT:    s_and_b32 s3, s3, 1
-; FIJI-NEXT:    v_mov_b32_e32 v0, s3
-; FIJI-NEXT:    ds_write_b8 v2, v0 offset:8
-; FIJI-NEXT:    v_mov_b32_e32 v0, s0
-; FIJI-NEXT:    v_mov_b32_e32 v1, s1
-; FIJI-NEXT:    ds_write_b64 v2, v[0:1]
-; FIJI-NEXT:    s_endpgm
+; FIJI-NEXT:	s_load_dword s2, s[4:5], 0x0
+; FIJI-NEXT:	s_load_dwordx2 s[0:1], s[4:5], 0x8
+; FIJI-NEXT:	s_load_dword s3, s[4:5], 0x10
+; FIJI-NEXT:	s_mov_b32 m0, -1
+; FIJI-NEXT:	s_waitcnt lgkmcnt(0)
+; FIJI-NEXT:	v_mov_b32_e32 v2, s2
+; FIJI-NEXT:	v_mov_b32_e32 v0, s0
+; FIJI-NEXT:	s_and_b32 s3, s3, 1
+; FIJI-NEXT:	v_mov_b32_e32 v3, s3
+; FIJI-NEXT:	v_mov_b32_e32 v1, s1
+; FIJI-NEXT:	ds_write_b8 v2, v3 offset:8
+; FIJI-NEXT:	ds_write_b64 v2, v[0:1]
+; FIJI-NEXT:	s_endpgm
 ;
 ; GFX9-LABEL: local_store_i65:
 ; GFX9:       ; %bb.0:
@@ -220,22 +218,22 @@ define void @local_store_i13(i13 addrspace(3)* %ptr, i13 %arg) #0 {
 define void @local_store_i17(i17 addrspace(3)* %ptr, i17 %arg) #0 {
 ; CIVI-LABEL: local_store_i17:
 ; CIVI:       ; %bb.0:
-; CIVI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CIVI-NEXT:    s_mov_b32 m0, -1
-; CIVI-NEXT:    ds_write_b16 v0, v1
-; CIVI-NEXT:    v_bfe_u32 v1, v1, 16, 1
-; CIVI-NEXT:    ds_write_b8 v0, v1 offset:2
-; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
-; CIVI-NEXT:    s_setpc_b64 s[30:31]
+; CIVI-NEXT:	s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CIVI-NEXT:	s_mov_b32 m0, -1
+; CIVI-NEXT:	v_bfe_u32 v2, v1, 16, 1
+; CIVI-NEXT:	ds_write_b16 v0, v1
+; CIVI-NEXT:	ds_write_b8 v0, v2 offset:2
+; CIVI-NEXT:	s_waitcnt lgkmcnt(0)
+; CIVI-NEXT:	s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: local_store_i17:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    ds_write_b16 v0, v1
-; GFX9-NEXT:    v_and_b32_e32 v1, 0x1ffff, v1
-; GFX9-NEXT:    ds_write_b8_d16_hi v0, v1 offset:2
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-NEXT:	s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:	v_and_b32_e32 v2, 0x1ffff, v1
+; GFX9-NEXT:	ds_write_b16 v0, v1
+; GFX9-NEXT:	ds_write_b8_d16_hi v0, v2 offset:2
+; GFX9-NEXT:	s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:	s_setpc_b64 s[30:31]
   store i17 %arg, i17 addrspace(3)* %ptr, align 8
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/switch-default-block-unreachable.ll b/llvm/test/CodeGen/AMDGPU/switch-default-block-unreachable.ll
new file mode 100644
index 0000000000000..13c4dc80be156
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/switch-default-block-unreachable.ll
@@ -0,0 +1,60 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -verify-machineinstrs -stop-after=amdgpu-isel -o - %s | FileCheck -check-prefix=GCN %s
+define void @test() #1 {
+  ; Clean up the unreachable blocks introduced with LowerSwitch pass.
+  ; This test ensures that, in the pass flow, UnreachableBlockElim pass
+  ; follows the LowerSwitch. Otherwise, this testcase will crash
+  ; immediately after the instruction selection due to the incomplete
+  ; PHI node in an MBB whose incoming values were never codegenerated.
+  ;
+  ; GCN-LABEL: name: test
+  ; GCN: bb.{{[0-9]+}}.entry:
+  ; GCN: bb.{{[0-9]+}}.entry.true.blk:
+  ; GCN: bb.{{[0-9]+}}.entry.false.blk:
+  ; GCN: bb.{{[0-9]+}}.switch.blk:
+
+  ; GCN-NOT: bb.{{[0-9]+}}.preheader.blk
+  ; GCN-NOT: bb.{{[0-9]+}}.pre.false.blk:
+  ; GCN-NOT: bb.{{[0-9]+}}.unreach.blk:
+  ; GCN-NOT: PHI
+
+  ; GCN: bb.{{[0-9]+}}.exit:
+  entry:
+    %idx = tail call i32 @llvm.amdgcn.workitem.id.x() #0
+    br i1 undef, label %entry.true.blk, label %entry.false.blk
+
+  entry.true.blk:                                   ; preds = %entry
+    %exit.cmp = icmp ult i32 %idx, 3
+    br i1 %exit.cmp, label %switch.blk, label %exit
+
+  entry.false.blk:                                  ; preds = %entry
+    unreachable
+
+  switch.blk:                                       ; preds = %entry.true.blk
+    switch i32 %idx, label %preheader.blk [
+      i32 0, label %exit
+      i32 1, label %exit
+      i32 2, label %exit
+    ]
+
+  preheader.blk:                                    ; preds = %switch.blk
+    %pre.exit = icmp ult i32 %idx, 5
+    br i1 %pre.exit, label %unreach.blk, label %pre.false.blk
+
+  pre.false.blk:                                    ; preds = %preheader.blk
+    %call.pre.false = tail call i32 @func(i32 %idx) #0
+    br label %unreach.blk
+
+  unreach.blk:                                      ; preds = %preheader.blk, %pre.false.blk
+    %phi.val = phi i32 [ %call.pre.false, %pre.false.blk ], [ undef, %preheader.blk ]
+    store i32 %phi.val, i32* undef
+    unreachable
+
+  exit:                                             ; preds = %switch.blk
+    ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #0
+declare i32 @func(i32)#0
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/transform-block-with-return-to-epilog.ll b/llvm/test/CodeGen/AMDGPU/transform-block-with-return-to-epilog.ll
new file mode 100644
index 0000000000000..c34a62bfc31db
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/transform-block-with-return-to-epilog.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs -stop-after=si-pre-emit-peephole -o - %s | FileCheck -check-prefix=GCN %s
+; If the block containing the SI_RETURN_TO_EPILOG is not the last block, insert an empty block at the end and
+; insert an unconditional jump there.
+define amdgpu_ps float @simple_test_return_to_epilog(float %a) #0 {
+  ; GCN-LABEL: name: simple_test_return_to_epilog
+  ; GCN: bb.0.entry:
+  ; GCN:   liveins: $vgpr0
+  ; GCN:   SI_RETURN_TO_EPILOG killed $vgpr0
+entry:
+  ret float %a
+}
+
+define amdgpu_ps float @test_return_to_epilog_into_end_block(i32 inreg %a, float %b) #0 {
+  ; GCN-LABEL: name: test_return_to_epilog_into_end_block
+  ; GCN: bb.0.entry:
+  ; GCN:   successors: %bb.1(0x7fffffff), %bb.2(0x00000001)
+  ; GCN:   liveins: $sgpr2, $vgpr0
+  ; GCN:   S_CMP_LT_I32 killed renamable $sgpr2, 1, implicit-def $scc
+  ; GCN:   S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+  ; GCN: bb.1.if:
+  ; GCN:   successors: %bb.3(0x80000000)
+  ; GCN:   liveins: $vgpr0
+  ; GCN:   S_BRANCH %bb.3
+  ; GCN: bb.2.else:
+  ; GCN:   successors:
+  ; GCN:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; GCN:   GLOBAL_STORE_DWORD undef renamable $vgpr0_vgpr1, killed renamable $vgpr0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; GCN: bb.3:
+entry:
+  %cc = icmp sgt i32 %a, 0
+  br i1 %cc, label %if, label %else
+if:                                               ; preds = %entry
+  ret float %b
+else:                                             ; preds = %entry
+  store volatile i32 0, i32 addrspace(1)* undef
+  unreachable
+}
+
+define amdgpu_ps float @test_unify_return_to_epilog_into_end_block(i32 inreg %a, i32 inreg %b, float %c, float %d) #0 {
+  ; GCN-LABEL: name: test_unify_return_to_epilog_into_end_block
+  ; GCN: bb.0.entry:
+  ; GCN:   successors: %bb.1(0x50000000), %bb.2(0x30000000)
+  ; GCN:   liveins: $sgpr2, $sgpr3, $vgpr0, $vgpr1
+  ; GCN:   S_CMP_LT_I32 killed renamable $sgpr2, 1, implicit-def $scc
+  ; GCN:   S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+  ; GCN: bb.1.if:
+  ; GCN:   successors: %bb.5(0x80000000)
+  ; GCN:   liveins: $vgpr0
+  ; GCN:   S_BRANCH %bb.5
+  ; GCN: bb.2.else.if.cond:
+  ; GCN:   successors: %bb.3(0x7fffffff), %bb.4(0x00000001)
+  ; GCN:   liveins: $sgpr3, $vgpr1
+  ; GCN:   S_CMP_LT_I32 killed renamable $sgpr3, 1, implicit-def $scc
+  ; GCN:   S_CBRANCH_SCC1 %bb.4, implicit killed $scc
+  ; GCN: bb.3.else.if:
+  ; GCN:   successors: %bb.5(0x80000000)
+  ; GCN:   liveins: $vgpr1
+  ; GCN:   $vgpr0 = V_MOV_B32_e32 killed $vgpr1, implicit $exec, implicit $exec
+  ; GCN:   S_BRANCH %bb.5
+  ; GCN: bb.4.else:
+  ; GCN:   successors:
+  ; GCN:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; GCN:   GLOBAL_STORE_DWORD undef renamable $vgpr0_vgpr1, killed renamable $vgpr0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; GCN: bb.5:
+entry:
+  %cc = icmp sgt i32 %a, 0
+  br i1 %cc, label %if, label %else.if.cond
+if:                                               ; preds = %entry
+  ret float %c
+else.if.cond:                                     ; preds = %entry
+  %cc1 = icmp sgt i32 %b, 0
+  br i1 %cc1, label %else.if, label %else
+else.if:                                          ; preds = %else.if.cond
+  ret float %d
+else:                                             ; preds = %else.if.cond
+  store volatile i32 0, i32 addrspace(1)* undef
+  unreachable
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/trunc-store-i64.ll b/llvm/test/CodeGen/AMDGPU/trunc-store-i64.ll
index 627ba9e0f7170..1c52aac1068f3 100644
--- a/llvm/test/CodeGen/AMDGPU/trunc-store-i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/trunc-store-i64.ll
@@ -38,10 +38,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}trunc_store_v16i64_v16i32:
-; GCN: global_store_dwordx4 v{{\[[0-9]:[0-9]+\]}}, v{{\[[0-9]:[0-9]+\]}}, off offset:48
-; GCN: global_store_dwordx4 v{{\[[0-9]:[0-9]+\]}}, v{{\[[0-9]:[0-9]+\]}}, off offset:32
-; GCN: global_store_dwordx4 v{{\[[0-9]:[0-9]+\]}}, v{{\[[0-9]:[0-9]+\]}}, off offset:16
-; GCN: global_store_dwordx4 v{{\[[0-9]:[0-9]+\]}}, v{{\[[0-9]:[0-9]+\]}}, off
+; GCN: global_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:48
+; GCN: global_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:32
+; GCN: global_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:16
+; GCN: global_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, off
 define amdgpu_kernel void @trunc_store_v16i64_v16i32(< 16 x i32> addrspace(1)* %out, <16 x i64> %in) {
 entry:
   %trunc = trunc <16 x i64> %in to <16 x i32>
diff --git a/llvm/test/CodeGen/AMDGPU/trunc.ll b/llvm/test/CodeGen/AMDGPU/trunc.ll
index 7dc43cf485df4..3b6cda0c860e1 100644
--- a/llvm/test/CodeGen/AMDGPU/trunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/trunc.ll
@@ -98,7 +98,8 @@ define amdgpu_kernel void @sgpr_trunc_i32_to_i1(i32 addrspace(1)* %out, i32 %a)
 ; VI: s_load_dwordx2 s{{\[}}[[SLO:[0-9]+]]:{{[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0x4c
 ; GCN: s_and_b32 [[MASKED:s[0-9]+]], 1, s[[SLO]]
 ; GCN: v_cmp_eq_u32_e64 s{{\[}}[[VLO:[0-9]+]]:[[VHI:[0-9]+]]], [[MASKED]], 1{{$}}
-; GCN: v_cndmask_b32_e64 {{v[0-9]+}}, -12, 63, s{{\[}}[[VLO]]:[[VHI]]]
+; GCN: s_cmp_lg_u32 s[[VLO]], 0
+; GCN: s_cselect_b32 {{s[0-9]+}}, 63, -12
 define amdgpu_kernel void @s_trunc_i64_to_i1(i32 addrspace(1)* %out, [8 x i32], i64 %x) {
   %trunc = trunc i64 %x to i1
   %sel = select i1 %trunc, i32 63, i32 -12
diff --git a/llvm/test/CodeGen/AMDGPU/udiv64.ll b/llvm/test/CodeGen/AMDGPU/udiv64.ll
index 6b29f29620436..7ebe78c16cb7d 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv64.ll
@@ -5,121 +5,121 @@
 define amdgpu_kernel void @s_test_udiv_i64(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ; GCN-LABEL: s_test_udiv_i64:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
-; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0xd
 ; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s2
-; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s3
-; GCN-NEXT:    s_sub_u32 s4, 0, s2
-; GCN-NEXT:    s_subb_u32 s5, 0, s3
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s12
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s13
+; GCN-NEXT:    s_sub_u32 s2, 0, s12
+; GCN-NEXT:    s_subb_u32 s3, 0, s13
+; GCN-NEXT:    s_mov_b32 s4, s8
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    s_mov_b32 s5, s9
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GCN-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v0
-; GCN-NEXT:    v_trunc_f32_e32 v3, v3
-; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v3
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GCN-NEXT:    v_mul_hi_u32 v5, s4, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s4, v3
-; GCN-NEXT:    v_mul_lo_u32 v7, s5, v0
-; GCN-NEXT:    v_mul_lo_u32 v6, s4, v0
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
-; GCN-NEXT:    v_mul_hi_u32 v5, v0, v6
-; GCN-NEXT:    v_mul_lo_u32 v7, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v9, v0, v4
-; GCN-NEXT:    v_mul_lo_u32 v8, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v6, v3, v6
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v2, v9, vcc
-; GCN-NEXT:    v_mul_hi_u32 v9, v3, v4
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
+; GCN-NEXT:    v_mul_lo_u32 v5, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s2, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v1, v4
+; GCN-NEXT:    v_mul_lo_u32 v4, v1, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_mul_hi_u32 v8, v1, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v6, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v5, s2, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v8, s3, v0
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GCN-NEXT:    v_mul_lo_u32 v7, s2, v0
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v9, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v2, v6, vcc
-; GCN-NEXT:    v_addc_u32_e64 v4, vcc, v3, v5, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v6, s4, v4
-; GCN-NEXT:    v_mul_hi_u32 v7, s4, v0
-; GCN-NEXT:    v_mul_lo_u32 v8, s5, v0
-; GCN-NEXT:    s_mov_b32 s5, s9
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
-; GCN-NEXT:    v_mul_lo_u32 v7, s4, v0
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
-; GCN-NEXT:    v_mul_lo_u32 v10, v0, v6
-; GCN-NEXT:    v_mul_hi_u32 v12, v0, v6
+; GCN-NEXT:    v_mul_lo_u32 v10, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v12, v0, v5
 ; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
-; GCN-NEXT:    v_mul_hi_u32 v9, v4, v7
-; GCN-NEXT:    v_mul_lo_u32 v7, v4, v7
-; GCN-NEXT:    v_mul_hi_u32 v8, v4, v6
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v8, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, v2, v12, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, v4, v6
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v8, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v2, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
-; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v6, s[0:1]
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, s10, v3
-; GCN-NEXT:    v_mul_hi_u32 v5, s10, v0
-; GCN-NEXT:    v_mul_hi_u32 v6, s10, v3
-; GCN-NEXT:    v_mul_hi_u32 v7, s11, v3
-; GCN-NEXT:    v_mul_lo_u32 v3, s11, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v2, v6, vcc
-; GCN-NEXT:    v_mul_lo_u32 v6, s11, v0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
-; GCN-NEXT:    s_mov_b32 s4, s8
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v5, v0, vcc
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v7, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s3, v0
-; GCN-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s12, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s12, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s13, v0
+; GCN-NEXT:    v_mov_b32_e32 v5, s13
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_mul_lo_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v3, s12, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s11, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s10, v3
-; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s2, v3
-; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s3, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s2, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s3, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v5, s[0:1], 2, v0
-; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 1, v0
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[0:1]
+; GCN-NEXT:    v_sub_i32_e64 v3, s[0:1], s10, v3
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v4, v5, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s12, v3
+; GCN-NEXT:    v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v6, s11
-; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v6, v2, vcc
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v6, v2, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v2
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s2, v3
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
@@ -128,93 +128,87 @@ define amdgpu_kernel void @s_test_udiv_i64(i64 addrspace(1)* %out, i64 %x, i64 %
 ; GCN-IR-LABEL: s_test_udiv_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-IR-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
+; GCN-IR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GCN-IR-NEXT:    s_mov_b64 s[2:3], 0
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[6:7], 0
-; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s2
-; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s10
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[6:7], 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[0:1], 0
+; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s0
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[8:9], s[10:11]
+; GCN-IR-NEXT:    s_add_i32 s12, s12, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s1
+; GCN-IR-NEXT:    s_cmp_eq_u32 s1, 0
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s6
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s8, s12, s8
 ; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s10
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v2, v3
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[2:3], 0
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[10:11], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[8:9]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[8:9], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[8:9], s[8:9], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT:    s_cbranch_vccz BB0_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[4:5], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_cmp_eq_u32 s7, 0
+; GCN-IR-NEXT:    s_cselect_b32 s12, s10, s11
+; GCN-IR-NEXT:    s_sub_u32 s10, s8, s12
+; GCN-IR-NEXT:    s_subb_u32 s11, 0, 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[10:11], 63
+; GCN-IR-NEXT:    s_mov_b32 s9, 0
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[16:17]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[16:17], s[10:11], 63
+; GCN-IR-NEXT:    s_xor_b64 s[18:19], s[14:15], -1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[18:19], s[16:17]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[16:17]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB0_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s14, s10, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-IR-NEXT:    s_addc_u32 s15, s11, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[14:15], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s10, 63, s10
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[6:7], s10
+; GCN-IR-NEXT:    s_cbranch_vccz BB0_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_not_b32_e32 v2, v2
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[6:7], v4
-; GCN-IR-NEXT:    s_add_u32 s6, s2, -1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v2, v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s7, s3, -1
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], -1, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[14:15], s[6:7], s14
+; GCN-IR-NEXT:    s_add_u32 s6, s0, -1
+; GCN-IR-NEXT:    s_addc_u32 s7, s1, -1
+; GCN-IR-NEXT:    s_not_b64 s[2:3], s[8:9]
+; GCN-IR-NEXT:    s_mov_b32 s13, s9
+; GCN-IR-NEXT:    s_add_u32 s8, s2, s12
+; GCN-IR-NEXT:    s_addc_u32 s9, s3, s9
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
+; GCN-IR-NEXT:    s_mov_b32 s3, 0
 ; GCN-IR-NEXT:  BB0_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s7
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s6, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s2, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s3, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s2, s11, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[2:3]
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[12:13], s[10:11]
+; GCN-IR-NEXT:    s_sub_u32 s2, s6, s14
+; GCN-IR-NEXT:    s_subb_u32 s2, s7, s15
+; GCN-IR-NEXT:    s_ashr_i32 s12, s2, 31
+; GCN-IR-NEXT:    s_mov_b32 s13, s12
+; GCN-IR-NEXT:    s_and_b32 s2, s12, 1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[12:13], s[0:1]
+; GCN-IR-NEXT:    s_sub_u32 s14, s14, s16
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_subb_u32 s15, s15, s17
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    s_add_u32 s8, s8, 1
+; GCN-IR-NEXT:    s_addc_u32 s9, s9, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[8:9], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], s[2:3]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB0_3
+; GCN-IR-NEXT:  BB0_4: ; %Flow6
+; GCN-IR-NEXT:    s_lshl_b64 s[0:1], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[0:1], s[2:3], s[0:1]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s1
 ; GCN-IR-NEXT:    s_branch BB0_6
-; GCN-IR-NEXT:  BB0_4:
+; GCN-IR-NEXT:  BB0_5:
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[14:15]
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB0_7
-; GCN-IR-NEXT:  BB0_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB0_6: ; %Flow6
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB0_7: ; %udiv-end
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[14:15]
+; GCN-IR-NEXT:  BB0_6: ; %udiv-end
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
@@ -801,24 +795,24 @@ define amdgpu_kernel void @s_test_udiv24_i48(i48 addrspace(1)* %out, i48 %x, i48
 ; GCN-NEXT:    v_subb_u32_e32 v4, vcc, 0, v4, vcc
 ; GCN-NEXT:    v_sub_i32_e32 v5, vcc, v3, v0
 ; GCN-NEXT:    v_subbrev_u32_e32 v6, vcc, 0, v4, vcc
-; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v5, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, v0, v5
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, -1, v5, vcc
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, 2, v1
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v2, vcc
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, 1, v1
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v3, v0
 ; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v4
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v0, -1, v0, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v5, v8, v6, vcc
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v0
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v1, v5, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v9, v7, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, v0, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v8, v6, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v5, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v9, v7, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    buffer_store_short v1, off, s[4:7], 0 offset:4
 ; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
@@ -838,93 +832,87 @@ define amdgpu_kernel void @s_test_udiv24_i48(i48 addrspace(1)* %out, i48 %x, i48
 ; GCN-IR-NEXT:    s_and_b32 s0, s2, s9
 ; GCN-IR-NEXT:    s_and_b32 s3, s7, s8
 ; GCN-IR-NEXT:    s_and_b32 s2, s6, s9
-; GCN-IR-NEXT:    s_lshr_b64 s[2:3], s[2:3], 24
-; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s2
 ; GCN-IR-NEXT:    s_lshr_b64 s[6:7], s[0:1], 24
-; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s10
+; GCN-IR-NEXT:    s_lshr_b64 s[2:3], s[2:3], 24
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[2:3], 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[6:7], 0
+; GCN-IR-NEXT:    s_mov_b64 s[0:1], 0
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[8:9], s[10:11]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s2
+; GCN-IR-NEXT:    s_add_i32 s8, s8, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s8, s8, s10
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s6
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
 ; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s10
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v2, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[10:11], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[2:3], 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[8:9]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[8:9], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[8:9], s[8:9], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT:    s_cbranch_vccz BB7_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[4:5], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_cmp_eq_u32 s7, 0
+; GCN-IR-NEXT:    s_cselect_b32 s12, s10, s11
+; GCN-IR-NEXT:    s_sub_u32 s10, s8, s12
+; GCN-IR-NEXT:    s_subb_u32 s11, 0, 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[10:11], 63
+; GCN-IR-NEXT:    s_mov_b32 s9, 0
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[16:17]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[16:17], s[10:11], 63
+; GCN-IR-NEXT:    s_xor_b64 s[18:19], s[14:15], -1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[18:19], s[16:17]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[16:17]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB7_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s14, s10, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-IR-NEXT:    s_addc_u32 s15, s11, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[14:15], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s10, 63, s10
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[6:7], s10
+; GCN-IR-NEXT:    s_cbranch_vccz BB7_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_not_b32_e32 v2, v2
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[6:7], v4
+; GCN-IR-NEXT:    s_lshr_b64 s[14:15], s[6:7], s14
 ; GCN-IR-NEXT:    s_add_u32 s6, s2, -1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v2, v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
 ; GCN-IR-NEXT:    s_addc_u32 s7, s3, -1
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], -1, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_not_b64 s[0:1], s[8:9]
+; GCN-IR-NEXT:    s_mov_b32 s13, s9
+; GCN-IR-NEXT:    s_add_u32 s8, s0, s12
+; GCN-IR-NEXT:    s_addc_u32 s9, s1, s9
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
+; GCN-IR-NEXT:    s_mov_b32 s1, 0
 ; GCN-IR-NEXT:  BB7_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s7
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s6, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s2, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s3, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s0, s11, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[0:1]
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[12:13], s[10:11]
+; GCN-IR-NEXT:    s_sub_u32 s0, s6, s14
+; GCN-IR-NEXT:    s_subb_u32 s0, s7, s15
+; GCN-IR-NEXT:    s_ashr_i32 s12, s0, 31
+; GCN-IR-NEXT:    s_mov_b32 s13, s12
+; GCN-IR-NEXT:    s_and_b32 s0, s12, 1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[12:13], s[2:3]
+; GCN-IR-NEXT:    s_sub_u32 s14, s14, s16
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_subb_u32 s15, s15, s17
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    s_add_u32 s8, s8, 1
+; GCN-IR-NEXT:    s_addc_u32 s9, s9, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[8:9], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], s[0:1]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB7_3
+; GCN-IR-NEXT:  BB7_4: ; %Flow3
+; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s1
 ; GCN-IR-NEXT:    s_branch BB7_6
-; GCN-IR-NEXT:  BB7_4:
+; GCN-IR-NEXT:  BB7_5:
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[14:15]
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB7_7
-; GCN-IR-NEXT:  BB7_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB7_6: ; %Flow3
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB7_7: ; %udiv-end
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[14:15]
+; GCN-IR-NEXT:  BB7_6: ; %udiv-end
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    buffer_store_short v1, off, s[4:7], 0 offset:4
@@ -1020,31 +1008,31 @@ define amdgpu_kernel void @s_test_udiv_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_mul_lo_u32 v3, s6, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 24, v3
-; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s6, v3
-; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s7, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s6, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s7, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v5, s[0:1], 2, v0
-; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 1, v0
-; GCN-NEXT:    v_subb_u32_e32 v2, vcc, 0, v2, vcc
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
+; GCN-NEXT:    v_sub_i32_e64 v3, s[0:1], 24, v3
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v4, v5, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s6, v3
+; GCN-NEXT:    v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s7, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s6, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s7, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, 0, v2, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v4
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s7, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s6, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s7, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
@@ -1052,84 +1040,78 @@ define amdgpu_kernel void @s_test_udiv_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ;
 ; GCN-IR-LABEL: s_test_udiv_k_num_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
-; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_flbit_i32_b32 s2, s6
-; GCN-IR-NEXT:    s_flbit_i32_b32 s3, s7
-; GCN-IR-NEXT:    s_add_i32 s2, s2, 32
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s2
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 0xffffffc5, v2
-; GCN-IR-NEXT:    v_addc_u32_e64 v1, s[2:3], 0, -1, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[2:3], s[2:3], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[2:3]
-; GCN-IR-NEXT:    s_cbranch_vccz BB8_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[3:4], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], 24, v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s4, s2
+; GCN-IR-NEXT:    s_add_i32 s4, s4, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s5, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s8, s4, s5
+; GCN-IR-NEXT:    s_add_u32 s6, s8, 0xffffffc5
+; GCN-IR-NEXT:    s_addc_u32 s7, 0, -1
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[2:3], 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[6:7], 63
+; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[12:13], s[6:7], 63
+; GCN-IR-NEXT:    s_xor_b64 s[14:15], s[10:11], -1
+; GCN-IR-NEXT:    s_and_b64 s[12:13], s[14:15], s[12:13]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[12:13]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB8_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s12, s6, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT:    s_addc_u32 s13, s7, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s7
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s6, 63, s6
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], 24, s6
+; GCN-IR-NEXT:    s_cbranch_vccz BB8_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_add_u32 s2, s6, -1
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], 24, v3
-; GCN-IR-NEXT:    v_sub_i32_e32 v4, vcc, 58, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s3, s7, -1
-; GCN-IR-NEXT:    v_subb_u32_e64 v5, s[0:1], 0, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[14:15], 24, s12
+; GCN-IR-NEXT:    s_add_u32 s6, s2, -1
+; GCN-IR-NEXT:    s_addc_u32 s7, s3, -1
+; GCN-IR-NEXT:    s_sub_u32 s8, 58, s8
+; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
+; GCN-IR-NEXT:    s_mov_b32 s5, 0
 ; GCN-IR-NEXT:  BB8_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s3
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s2, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s6, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s7, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s4, s11, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[4:5]
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[12:13], s[10:11]
+; GCN-IR-NEXT:    s_sub_u32 s4, s6, s14
+; GCN-IR-NEXT:    s_subb_u32 s4, s7, s15
+; GCN-IR-NEXT:    s_ashr_i32 s12, s4, 31
+; GCN-IR-NEXT:    s_mov_b32 s13, s12
+; GCN-IR-NEXT:    s_and_b32 s4, s12, 1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[12:13], s[2:3]
+; GCN-IR-NEXT:    s_sub_u32 s14, s14, s16
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_subb_u32 s15, s15, s17
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    s_add_u32 s8, s8, 1
+; GCN-IR-NEXT:    s_addc_u32 s9, s9, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[8:9], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], s[4:5]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB8_3
+; GCN-IR-NEXT:  BB8_4: ; %Flow5
+; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[2:3], s[4:5], s[2:3]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-IR-NEXT:    s_branch BB8_6
-; GCN-IR-NEXT:  BB8_4:
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, 24, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB8_7
 ; GCN-IR-NEXT:  BB8_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB8_6: ; %Flow5
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB8_7: ; %udiv-end
-; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-IR-NEXT:    s_mov_b32 s6, -1
-; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, 24, 0, s[10:11]
+; GCN-IR-NEXT:  BB8_6: ; %udiv-end
+; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-IR-NEXT:    s_mov_b32 s2, -1
+; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %result = udiv i64 24, %x
   store i64 %result, i64 addrspace(1)* %out
@@ -1504,8 +1486,8 @@ define amdgpu_kernel void @s_test_udiv_k_den_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_mul_hi_u32 v3, v0, 24
 ; GCN-NEXT:    v_mul_lo_u32 v4, v0, 24
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s10, v4
 ; GCN-NEXT:    v_mov_b32_e32 v3, s11
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s10, v4
 ; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v3, v2, vcc
 ; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, 24, v4
 ; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v2, vcc
@@ -1516,98 +1498,93 @@ define amdgpu_kernel void @s_test_udiv_k_den_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], 23, v4
 ; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v2, -1, v4, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v8, v6, vcc
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v7, v5, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, 23, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v2, -1, v4, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v8, v6, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_udiv_k_den_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
-; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_flbit_i32_b32 s2, s6
-; GCN-IR-NEXT:    s_flbit_i32_b32 s3, s7
-; GCN-IR-NEXT:    s_add_i32 s2, s2, 32
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s2
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 59, v2
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[2:3], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[2:3], s[2:3], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[2:3]
-; GCN-IR-NEXT:    s_cbranch_vccz BB11_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[3:4], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s4, s2
+; GCN-IR-NEXT:    s_add_i32 s4, s4, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s5, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s8, s4, s5
+; GCN-IR-NEXT:    s_sub_u32 s6, 59, s8
+; GCN-IR-NEXT:    s_subb_u32 s7, 0, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[2:3], 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[6:7], 63
+; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[12:13], s[6:7], 63
+; GCN-IR-NEXT:    s_xor_b64 s[14:15], s[10:11], -1
+; GCN-IR-NEXT:    s_and_b64 s[12:13], s[14:15], s[12:13]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[12:13]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB11_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s10, s6, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT:    s_addc_u32 s11, s7, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s7
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[10:11], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s6, 63, s6
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[2:3], s6
+; GCN-IR-NEXT:    s_cbranch_vccz BB11_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[6:7], v3
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 0xffffffc4, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], 0, -1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[10:11], s[2:3], s10
+; GCN-IR-NEXT:    s_add_u32 s2, s8, 0xffffffc4
+; GCN-IR-NEXT:    s_addc_u32 s3, 0, -1
+; GCN-IR-NEXT:    s_mov_b64 s[8:9], 0
+; GCN-IR-NEXT:    s_mov_b32 s5, 0
 ; GCN-IR-NEXT:  BB11_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_sub_i32_e32 v2, vcc, 23, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, 0, v7, vcc
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, 24, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subbrev_u32_e64 v7, s[0:1], 0, v7, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s4, s7, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[10:11], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[6:7], 1
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[10:11], s[4:5]
+; GCN-IR-NEXT:    s_or_b64 s[6:7], s[8:9], s[6:7]
+; GCN-IR-NEXT:    s_sub_u32 s4, 23, s10
+; GCN-IR-NEXT:    s_subb_u32 s4, 0, s11
+; GCN-IR-NEXT:    s_ashr_i32 s8, s4, 31
+; GCN-IR-NEXT:    s_and_b32 s4, s8, 1
+; GCN-IR-NEXT:    s_and_b32 s8, s8, 24
+; GCN-IR-NEXT:    s_sub_u32 s10, s10, s8
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    s_subb_u32 s11, s11, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-IR-NEXT:    s_add_u32 s2, s2, 1
+; GCN-IR-NEXT:    s_addc_u32 s3, s3, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[8:9], s[4:5]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB11_3
+; GCN-IR-NEXT:  BB11_4: ; %Flow5
+; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[6:7], 1
+; GCN-IR-NEXT:    s_or_b64 s[2:3], s[4:5], s[2:3]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-IR-NEXT:    s_branch BB11_6
-; GCN-IR-NEXT:  BB11_4:
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB11_7
 ; GCN-IR-NEXT:  BB11_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB11_6: ; %Flow5
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB11_7: ; %udiv-end
-; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-IR-NEXT:    s_mov_b32 s6, -1
-; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s3
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[10:11]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[10:11]
+; GCN-IR-NEXT:  BB11_6: ; %udiv-end
+; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-IR-NEXT:    s_mov_b32 s2, -1
+; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %result = udiv i64 %x, 24
   store i64 %result, i64 addrspace(1)* %out
diff --git a/llvm/test/CodeGen/AMDGPU/udivrem.ll b/llvm/test/CodeGen/AMDGPU/udivrem.ll
index 326eda5296cf0..10299b314e83e 100644
--- a/llvm/test/CodeGen/AMDGPU/udivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/udivrem.ll
@@ -1,57 +1,112 @@
-; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap --check-prefix=SI --check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap --check-prefix=SI --check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -allow-deprecated-dag-overlap --check-prefix=EG --check-prefix=FUNC %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck --check-prefix=R600 %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck --check-prefix=GFX6 %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck --check-prefix=GFX8 %s
 
-; FUNC-LABEL: {{^}}test_udivrem:
-; EG: RECIP_UINT
-; EG-DAG: MULHI
-; EG-DAG: MULLO_INT
-; EG-DAG: SUB_INT
-; EG: CNDE_INT
-; EG: MULHI
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG: CNDE_INT
-; EG: MULHI
-; EG: MULLO_INT
-; EG: SUB_INT
-; EG-DAG: SETGE_UINT
-; EG-DAG: SETGE_UINT
-; EG: AND_INT
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: CNDE_INT
-
-; SI: v_rcp_iflag_f32_e32 [[RCP:v[0-9]+]]
-; SI-DAG: v_mul_hi_u32 [[RCP_HI:v[0-9]+]], [[RCP]]
-; SI-DAG: v_mul_lo_u32 [[RCP_LO:v[0-9]+]], [[RCP]]
-; SI-DAG: v_sub_{{[iu]}}32_e32 [[NEG_RCP_LO:v[0-9]+]], vcc, 0, [[RCP_LO]]
-; SI: v_cmp_eq_u32_e64 [[CC1:s\[[0-9:]+\]]], 0, [[RCP_HI]]
-; SI: v_cndmask_b32_e64 [[CND1:v[0-9]+]], [[RCP_LO]], [[NEG_RCP_LO]], [[CC1]]
-; SI: v_mul_hi_u32 [[E:v[0-9]+]], [[CND1]], [[RCP]]
-; SI-DAG: v_add_{{[iu]}}32_e32 [[RCP_A_E:v[0-9]+]], vcc, [[E]], [[RCP]]
-; SI-DAG: v_subrev_{{[iu]}}32_e32 [[RCP_S_E:v[0-9]+]], vcc, [[E]], [[RCP]]
-; SI: v_cndmask_b32_e64 [[CND2:v[0-9]+]], [[RCP_S_E]], [[RCP_A_E]], [[CC1]]
-; SI: v_mul_hi_u32 [[Quotient:v[0-9]+]], [[CND2]],
-; SI: v_mul_lo_u32 [[Num_S_Remainder:v[0-9]+]], [[CND2]]
-; SI-DAG: v_add_{{[iu]}}32_e32 [[Quotient_A_One:v[0-9]+]], vcc, 1, [[Quotient]]
-; SI-DAG: v_sub_{{[iu]}}32_e32 [[Remainder:v[0-9]+]], vcc, {{[vs][0-9]+}}, [[Num_S_Remainder]]
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_subrev_{{[iu]}}32_e32 [[Quotient_S_One:v[0-9]+]],
-; SI-DAG: v_subrev_{{[iu]}}32_e32 [[Remainder_S_Den:v[0-9]+]],
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_{{[iu]}}32_e32 [[Remainder_A_Den:v[0-9]+]],
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_cndmask_b32_e64
-; SI-NOT: v_and_b32
-; SI: s_endpgm
 define amdgpu_kernel void @test_udivrem(i32 addrspace(1)* %out0, [8 x i32], i32 addrspace(1)* %out1, [8 x i32], i32 %x, [8 x i32], i32 %y) {
+; R600-LABEL: test_udivrem:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 21, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T2.X, T3.X, 0
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     SUB_INT T0.W, 0.0, KC0[9].X,
+; R600-NEXT:     RECIP_UINT * T0.X, KC0[9].X,
+; R600-NEXT:     MULLO_INT * T0.Y, PV.W, PS,
+; R600-NEXT:     MULHI * T0.Y, T0.X, PS,
+; R600-NEXT:     ADD_INT * T0.W, T0.X, PS,
+; R600-NEXT:     MULHI * T0.X, KC0[6].W, PV.W,
+; R600-NEXT:     MULLO_INT * T0.Y, PS, KC0[9].X,
+; R600-NEXT:     SUB_INT * T0.W, KC0[6].W, PS,
+; R600-NEXT:     SUB_INT T1.W, PV.W, KC0[9].X,
+; R600-NEXT:     SETGE_UINT * T2.W, PV.W, KC0[9].X,
+; R600-NEXT:     CNDE_INT * T0.W, PS, T0.W, PV.W,
+; R600-NEXT:     ADD_INT T0.Z, T0.X, 1,
+; R600-NEXT:     SUB_INT T1.W, PV.W, KC0[9].X,
+; R600-NEXT:     SETGE_UINT * T3.W, PV.W, KC0[9].X,
+; R600-NEXT:     CNDE_INT T1.X, PS, T0.W, PV.W,
+; R600-NEXT:     CNDE_INT T0.W, T2.W, T0.X, PV.Z,
+; R600-NEXT:     LSHR * T0.X, KC0[4].Z, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; R600-NEXT:     ADD_INT * T1.W, PV.W, 1,
+; R600-NEXT:     CNDE_INT T2.X, T3.W, T0.W, PV.W,
+; R600-NEXT:     LSHR * T3.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX6-LABEL: test_udivrem:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_load_dword s3, s[0:1], 0x26
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x13
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0x1d
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_mov_b32 s10, s6
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s3
+; GFX6-NEXT:    s_sub_i32 s2, 0, s3
+; GFX6-NEXT:    s_mov_b32 s11, s7
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_lo_u32 v1, s2, v0
+; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GFX6-NEXT:    v_mul_lo_u32 v1, v0, s3
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s0, v1
+; GFX6-NEXT:    v_cmp_le_u32_e64 s[0:1], s3, v1
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, s3, v1
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; GFX6-NEXT:    v_cmp_le_u32_e64 s[0:1], s3, v1
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, s3, v1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_waitcnt expcnt(0)
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v1, v2, s[0:1]
+; GFX6-NEXT:    buffer_store_dword v0, off, s[8:11], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: test_udivrem:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dword s7, s[0:1], 0x98
+; GFX8-NEXT:    s_load_dword s6, s[0:1], 0x74
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_cvt_f32_u32_e32 v0, s7
+; GFX8-NEXT:    s_sub_i32 s2, 0, s7
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_lo_u32 v1, s2, v0
+; GFX8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x4c
+; GFX8-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
+; GFX8-NEXT:    v_mul_hi_u32 v2, s6, v0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    v_mul_lo_u32 v3, v2, s7
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 1, v2
+; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, s6, v3
+; GFX8-NEXT:    v_cmp_le_u32_e64 s[0:1], s7, v3
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v4, s[0:1]
+; GFX8-NEXT:    v_subrev_u32_e32 v4, vcc, s7, v3
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v4, s[0:1]
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 1, v2
+; GFX8-NEXT:    v_cmp_le_u32_e64 s[0:1], s7, v3
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v4, s[0:1]
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    v_subrev_u32_e32 v4, vcc, s7, v3
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
   %result0 = udiv i32 %x, %y
   store i32 %result0, i32 addrspace(1)* %out0
   %result1 = urem i32 %x, %y
@@ -59,103 +114,134 @@ define amdgpu_kernel void @test_udivrem(i32 addrspace(1)* %out0, [8 x i32], i32
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_udivrem_v2:
-; EG-DAG: RECIP_UINT
-; EG-DAG: MULHI
-; EG-DAG: MULLO_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: MULHI
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: MULHI
-; EG-DAG: MULLO_INT
-; EG-DAG: SUB_INT
-; EG-DAG: SETGE_UINT
-; EG-DAG: SETGE_UINT
-; EG-DAG: AND_INT
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: RECIP_UINT
-; EG-DAG: MULHI
-; EG-DAG: MULLO_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: MULHI
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: MULHI
-; EG-DAG: MULLO_INT
-; EG-DAG: SUB_INT
-; EG-DAG: SETGE_UINT
-; EG-DAG: SETGE_UINT
-; EG-DAG: AND_INT
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: CNDE_INT
-
-; For SI, we used to have checks for the input and output registers
-; of the instructions, but these are way too fragile.  The division for
-; the two vector elements can be intermixed which makes it impossible to
-; accurately check all the operands.
-; SI-DAG: v_rcp_iflag_f32_e32
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_u32
-; SI-DAG: v_sub_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_add_{{[iu]}}32_e32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_u32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_{{[iu]}}32_e32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_{{[iu]}}32_e32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_rcp_iflag_f32_e32
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_u32
-; SI-DAG: v_sub_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_add_{{[iu]}}32_e32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_u32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_{{[iu]}}32_e32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_{{[iu]}}32_e32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_cndmask_b32_e64
-; SI-NOT: v_and_b32
-; SI: s_endpgm
 define amdgpu_kernel void @test_udivrem_v2(<2 x i32> addrspace(1)* %out, <2 x i32> %x, <2 x i32> %y) {
+; R600-LABEL: test_udivrem_v2:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 29, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     SUB_INT T0.W, 0.0, KC0[3].Z,
+; R600-NEXT:     RECIP_UINT * T0.X, KC0[3].Z,
+; R600-NEXT:     MULLO_INT * T0.Y, PV.W, PS,
+; R600-NEXT:     SUB_INT T0.W, 0.0, KC0[3].Y,
+; R600-NEXT:     RECIP_UINT * T0.Z, KC0[3].Y,
+; R600-NEXT:     MULLO_INT * T0.W, PV.W, PS,
+; R600-NEXT:     MULHI * T0.W, T0.Z, PS,
+; R600-NEXT:     ADD_INT T0.W, T0.Z, PS,
+; R600-NEXT:     MULHI * T0.Y, T0.X, T0.Y,
+; R600-NEXT:     ADD_INT T1.W, T0.X, PS,
+; R600-NEXT:     MULHI * T0.X, KC0[2].W, PV.W,
+; R600-NEXT:     MULHI * T0.Y, KC0[3].X, PV.W,
+; R600-NEXT:     MULLO_INT * T0.Y, PS, KC0[3].Z,
+; R600-NEXT:     SUB_INT T0.W, KC0[3].X, PS,
+; R600-NEXT:     MULLO_INT * T0.X, T0.X, KC0[3].Y,
+; R600-NEXT:     SUB_INT T0.Z, KC0[2].W, PS,
+; R600-NEXT:     SETGE_UINT T1.W, PV.W, KC0[3].Z,
+; R600-NEXT:     SUB_INT * T2.W, PV.W, KC0[3].Z,
+; R600-NEXT:     CNDE_INT T1.Z, PV.W, T0.W, PS,
+; R600-NEXT:     SETGE_UINT T0.W, PV.Z, KC0[3].Y,
+; R600-NEXT:     SUB_INT * T1.W, PV.Z, KC0[3].Y,
+; R600-NEXT:     CNDE_INT T0.Z, PV.W, T0.Z, PS,
+; R600-NEXT:     SETGE_UINT T0.W, PV.Z, KC0[3].Z,
+; R600-NEXT:     SUB_INT * T1.W, PV.Z, KC0[3].Z,
+; R600-NEXT:     CNDE_INT T0.Y, PV.W, T1.Z, PS,
+; R600-NEXT:     SETGE_UINT T0.W, PV.Z, KC0[3].Y,
+; R600-NEXT:     SUB_INT * T1.W, PV.Z, KC0[3].Y,
+; R600-NEXT:     CNDE_INT T0.X, PV.W, T0.Z, PS,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX6-LABEL: test_udivrem_v2:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0x4f7ffffe
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s6
+; GFX6-NEXT:    s_sub_i32 s2, 0, s6
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v1, s7
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GFX6-NEXT:    v_mul_f32_e32 v0, s3, v0
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_f32_e32 v1, s3, v1
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    v_mul_lo_u32 v2, s2, v0
+; GFX6-NEXT:    s_sub_i32 s2, 0, s7
+; GFX6-NEXT:    v_mul_hi_u32 v2, v0, v2
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GFX6-NEXT:    v_mul_lo_u32 v2, s2, v1
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    v_mul_lo_u32 v0, v0, s6
+; GFX6-NEXT:    v_mul_hi_u32 v2, v1, v2
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s4, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, s6, v0
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, s6, v0
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GFX6-NEXT:    v_mul_hi_u32 v1, s5, v1
+; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s7
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s5, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, s7, v1
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, s7, v1
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, s7, v1
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, s7, v1
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: test_udivrem_v2:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x2c
+; GFX8-NEXT:    s_mov_b32 s3, 0x4f7ffffe
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_cvt_f32_u32_e32 v0, s6
+; GFX8-NEXT:    s_sub_i32 s2, 0, s6
+; GFX8-NEXT:    v_cvt_f32_u32_e32 v1, s7
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GFX8-NEXT:    v_mul_f32_e32 v0, s3, v0
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_f32_e32 v1, s3, v1
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX8-NEXT:    v_mul_lo_u32 v2, s2, v0
+; GFX8-NEXT:    s_sub_i32 s2, 0, s7
+; GFX8-NEXT:    v_mul_hi_u32 v2, v0, v2
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v2, v0
+; GFX8-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GFX8-NEXT:    v_mul_lo_u32 v2, s2, v1
+; GFX8-NEXT:    v_mul_lo_u32 v0, v0, s6
+; GFX8-NEXT:    v_mul_hi_u32 v2, v1, v2
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s4, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v3, vcc, s6, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v3, vcc, s6, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v2, v1
+; GFX8-NEXT:    v_mul_hi_u32 v1, s5, v1
+; GFX8-NEXT:    v_mul_lo_u32 v1, v1, s7
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, s5, v1
+; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, s7, v1
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s7, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, s7, v1
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s7, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_endpgm
   %result0 = udiv <2 x i32> %x, %y
   store <2 x i32> %result0, <2 x i32> addrspace(1)* %out
   %result1 = urem <2 x i32> %x, %y
@@ -163,176 +249,230 @@ define amdgpu_kernel void @test_udivrem_v2(<2 x i32> addrspace(1)* %out, <2 x i3
   ret void
 }
 
-
-; FUNC-LABEL: {{^}}test_udivrem_v4:
-; EG-DAG: RECIP_UINT
-; EG-DAG: MULHI
-; EG-DAG: MULLO_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: MULHI
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: MULHI
-; EG-DAG: MULLO_INT
-; EG-DAG: SUB_INT
-; EG-DAG: SETGE_UINT
-; EG-DAG: SETGE_UINT
-; EG-DAG: AND_INT
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: RECIP_UINT
-; EG-DAG: MULHI
-; EG-DAG: MULLO_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: MULHI
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: MULHI
-; EG-DAG: MULLO_INT
-; EG-DAG: SUB_INT
-; EG-DAG: SETGE_UINT
-; EG-DAG: SETGE_UINT
-; EG-DAG: AND_INT
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: RECIP_UINT
-; EG-DAG: MULHI
-; EG-DAG: MULLO_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: MULHI
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: MULHI
-; EG-DAG: MULLO_INT
-; EG-DAG: SUB_INT
-; EG-DAG: SETGE_UINT
-; EG-DAG: SETGE_UINT
-; EG-DAG: AND_INT
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: RECIP_UINT
-; EG-DAG: MULHI
-; EG-DAG: MULLO_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: MULHI
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: MULHI
-; EG-DAG: MULLO_INT
-; EG-DAG: SUB_INT
-; EG-DAG: SETGE_UINT
-; EG-DAG: SETGE_UINT
-; EG-DAG: AND_INT
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: ADD_INT
-; EG-DAG: SUB_INT
-; EG-DAG: CNDE_INT
-; EG-DAG: CNDE_INT
-
-; SI-DAG: v_rcp_iflag_f32_e32
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_u32
-; SI-DAG: v_sub_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_add_{{[iu]}}32_e32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_u32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_{{[iu]}}32_e32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_{{[iu]}}32_e32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_rcp_iflag_f32_e32
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_u32
-; SI-DAG: v_sub_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_add_{{[iu]}}32_e32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_u32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_{{[iu]}}32_e32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_{{[iu]}}32_e32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_rcp_iflag_f32_e32
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_u32
-; SI-DAG: v_sub_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_add_{{[iu]}}32_e32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_u32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_{{[iu]}}32_e32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_{{[iu]}}32_e32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_rcp_iflag_f32_e32
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_u32
-; SI-DAG: v_sub_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_add_{{[iu]}}32_e32
-; SI-DAG: v_subrev_{{[iu]}}32_e32
-; SI-DAG: v_cndmask_b32_e64
-; SI-NOT: v_and_b32
-; SI: s_endpgm
 define amdgpu_kernel void @test_udivrem_v4(<4 x i32> addrspace(1)* %out, <4 x i32> %x, <4 x i32> %y) {
+; R600-LABEL: test_udivrem_v4:
+; R600:       ; %bb.0:
+; R600-NEXT:    ALU 57, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T3.XYZW, T0.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    ALU clause starting at 4:
+; R600-NEXT:     SUB_INT T0.W, 0.0, KC0[5].X,
+; R600-NEXT:     RECIP_UINT * T0.X, KC0[5].X,
+; R600-NEXT:     MULLO_INT * T0.Y, PV.W, PS,
+; R600-NEXT:     SUB_INT T0.W, 0.0, KC0[4].Z,
+; R600-NEXT:     RECIP_UINT * T0.Z, KC0[4].Z,
+; R600-NEXT:     MULLO_INT * T0.W, PV.W, PS,
+; R600-NEXT:     MULHI * T0.W, T0.Z, PS,
+; R600-NEXT:     ADD_INT T0.W, T0.Z, PS,
+; R600-NEXT:     MULHI * T0.Y, T0.X, T0.Y,
+; R600-NEXT:     ADD_INT T1.W, T0.X, PS,
+; R600-NEXT:     MULHI * T0.X, KC0[3].Z, PV.W,
+; R600-NEXT:     MULHI * T0.Y, KC0[4].X, PV.W,
+; R600-NEXT:     MULLO_INT * T0.Y, PS, KC0[5].X,
+; R600-NEXT:     RECIP_UINT * T0.Z, KC0[4].Y,
+; R600-NEXT:     SUB_INT T0.W, 0.0, KC0[4].W,
+; R600-NEXT:     RECIP_UINT * T1.X, KC0[4].W,
+; R600-NEXT:     MULLO_INT * T0.W, PV.W, PS,
+; R600-NEXT:     SUB_INT T1.W, 0.0, KC0[4].Y,
+; R600-NEXT:     MULHI * T0.W, T1.X, PS,
+; R600-NEXT:     ADD_INT T0.W, T1.X, PS,
+; R600-NEXT:     MULLO_INT * T1.X, PV.W, T0.Z,
+; R600-NEXT:     MULHI * T0.W, KC0[3].W, PV.W,
+; R600-NEXT:     MULLO_INT * T0.W, PS, KC0[4].W,
+; R600-NEXT:     SUB_INT T0.W, KC0[3].W, PS,
+; R600-NEXT:     MULHI * T1.X, T0.Z, T1.X,
+; R600-NEXT:     SETGE_UINT T1.Y, PV.W, KC0[4].W,
+; R600-NEXT:     ADD_INT T0.Z, T0.Z, PS,
+; R600-NEXT:     SUB_INT T1.W, KC0[4].X, T0.Y,
+; R600-NEXT:     MULLO_INT * T0.X, T0.X, KC0[4].Z,
+; R600-NEXT:     SUB_INT T0.Y, KC0[3].Z, PS,
+; R600-NEXT:     SETGE_UINT T1.Z, PV.W, KC0[5].X,
+; R600-NEXT:     SUB_INT * T2.W, PV.W, KC0[5].X,
+; R600-NEXT:     MULHI * T0.X, KC0[3].Y, T0.Z,
+; R600-NEXT:     SUB_INT T1.X, T0.W, KC0[4].W,
+; R600-NEXT:     CNDE_INT T2.Y, T1.Z, T1.W, T2.W,
+; R600-NEXT:     SETGE_UINT T0.Z, T0.Y, KC0[4].Z,
+; R600-NEXT:     SUB_INT T1.W, T0.Y, KC0[4].Z,
+; R600-NEXT:     MULLO_INT * T0.X, PS, KC0[4].Y,
+; R600-NEXT:     CNDE_INT T2.X, PV.Z, T0.Y, PV.W,
+; R600-NEXT:     SETGE_UINT T0.Y, PV.Y, KC0[5].X,
+; R600-NEXT:     SUB_INT T0.Z, PV.Y, KC0[5].X,
+; R600-NEXT:     SUB_INT T1.W, KC0[3].Y, PS,
+; R600-NEXT:     CNDE_INT * T0.W, T1.Y, T0.W, PV.X,
+; R600-NEXT:     SETGE_UINT T0.X, PS, KC0[4].W,
+; R600-NEXT:     SUB_INT T1.Y, PS, KC0[4].W,
+; R600-NEXT:     SETGE_UINT T1.Z, PV.W, KC0[4].Y,
+; R600-NEXT:     SUB_INT T2.W, PV.W, KC0[4].Y,
+; R600-NEXT:     CNDE_INT * T3.W, PV.Y, T2.Y, PV.Z,
+; R600-NEXT:     CNDE_INT T0.Y, PV.Z, T1.W, PV.W,
+; R600-NEXT:     CNDE_INT T3.Z, PV.X, T0.W, PV.Y, BS:VEC_021/SCL_122
+; R600-NEXT:     SETGE_UINT T0.W, T2.X, KC0[4].Z,
+; R600-NEXT:     SUB_INT * T1.W, T2.X, KC0[4].Z,
+; R600-NEXT:     CNDE_INT T3.Y, PV.W, T2.X, PS,
+; R600-NEXT:     SETGE_UINT T0.W, PV.Y, KC0[4].Y,
+; R600-NEXT:     SUB_INT * T1.W, PV.Y, KC0[4].Y,
+; R600-NEXT:     CNDE_INT T3.X, PV.W, T0.Y, PS,
+; R600-NEXT:     LSHR * T0.X, KC0[2].Y, literal.x,
+; R600-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX6-LABEL: test_udivrem_v4:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0xd
+; GFX6-NEXT:    s_mov_b32 s12, 0x4f7ffffe
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s8
+; GFX6-NEXT:    s_sub_i32 s2, 0, s8
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v1, s9
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v4, s11
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT:    s_sub_i32 s3, 0, s9
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s10
+; GFX6-NEXT:    v_mul_f32_e32 v0, s12, v0
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_f32_e32 v1, s12, v1
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GFX6-NEXT:    v_mul_lo_u32 v3, s2, v0
+; GFX6-NEXT:    s_sub_i32 s2, 0, s10
+; GFX6-NEXT:    v_mul_f32_e32 v2, s12, v2
+; GFX6-NEXT:    v_mul_hi_u32 v3, v0, v3
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v3, v4
+; GFX6-NEXT:    v_mul_lo_u32 v4, s3, v1
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    v_mul_lo_u32 v0, v0, s8
+; GFX6-NEXT:    v_mul_f32_e32 v3, s12, v3
+; GFX6-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s4, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s8, v0
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s8, v0
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v4, v1
+; GFX6-NEXT:    v_mul_hi_u32 v1, s5, v1
+; GFX6-NEXT:    v_mul_lo_u32 v4, s2, v2
+; GFX6-NEXT:    s_sub_i32 s2, 0, s11
+; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s9
+; GFX6-NEXT:    v_mul_hi_u32 v4, v2, v4
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s5, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s9, v1
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, s9, v1
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s9, v1
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, s9, v1
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; GFX6-NEXT:    v_mul_hi_u32 v2, s6, v2
+; GFX6-NEXT:    v_mul_lo_u32 v4, s2, v3
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    v_mul_lo_u32 v2, v2, s10
+; GFX6-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, s6, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s10, v2
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, s10, v2
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s10, v2
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, s10, v2
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GFX6-NEXT:    v_mul_hi_u32 v3, s7, v3
+; GFX6-NEXT:    v_mul_lo_u32 v3, v3, s11
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, s7, v3
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, s11, v3
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, s11, v3
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, s11, v3
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, s11, v3
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX6-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: test_udivrem_v4:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x34
+; GFX8-NEXT:    s_mov_b32 s12, 0x4f7ffffe
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_cvt_f32_u32_e32 v0, s8
+; GFX8-NEXT:    s_sub_i32 s2, 0, s8
+; GFX8-NEXT:    v_cvt_f32_u32_e32 v1, s9
+; GFX8-NEXT:    v_cvt_f32_u32_e32 v4, s11
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT:    s_sub_i32 s3, 0, s9
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GFX8-NEXT:    v_cvt_f32_u32_e32 v2, s10
+; GFX8-NEXT:    v_mul_f32_e32 v0, s12, v0
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_f32_e32 v1, s12, v1
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GFX8-NEXT:    v_mul_lo_u32 v3, s2, v0
+; GFX8-NEXT:    s_sub_i32 s2, 0, s10
+; GFX8-NEXT:    v_mul_f32_e32 v2, s12, v2
+; GFX8-NEXT:    v_mul_hi_u32 v3, v0, v3
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v3, v0
+; GFX8-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v3, v4
+; GFX8-NEXT:    v_mul_lo_u32 v4, s3, v1
+; GFX8-NEXT:    v_mul_lo_u32 v0, v0, s8
+; GFX8-NEXT:    v_mul_f32_e32 v3, s12, v3
+; GFX8-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s4, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s8, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s8, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v4, v1
+; GFX8-NEXT:    v_mul_hi_u32 v1, s5, v1
+; GFX8-NEXT:    v_mul_lo_u32 v4, s2, v2
+; GFX8-NEXT:    s_sub_i32 s2, 0, s11
+; GFX8-NEXT:    v_mul_lo_u32 v1, v1, s9
+; GFX8-NEXT:    v_mul_hi_u32 v4, v2, v4
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, s5, v1
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s9, v1
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s9, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s9, v1
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s9, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v4, v2
+; GFX8-NEXT:    v_mul_hi_u32 v2, s6, v2
+; GFX8-NEXT:    v_mul_lo_u32 v4, s2, v3
+; GFX8-NEXT:    v_mul_lo_u32 v2, v2, s10
+; GFX8-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, s6, v2
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s10, v2
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s10, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s10, v2
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s10, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v4, v3
+; GFX8-NEXT:    v_mul_hi_u32 v3, s7, v3
+; GFX8-NEXT:    v_mul_lo_u32 v3, v3, s11
+; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, s7, v3
+; GFX8-NEXT:    v_subrev_u32_e32 v4, vcc, s11, v3
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s11, v3
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v4, vcc, s11, v3
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s11, v3
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT:    s_endpgm
   %result0 = udiv <4 x i32> %x, %y
   store <4 x i32> %result0, <4 x i32> addrspace(1)* %out
   %result1 = urem <4 x i32> %x, %y
diff --git a/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll b/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
index 75d6eb57cb0b7..97a6d0436ee8c 100644
--- a/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
@@ -75,8 +75,9 @@ define amdgpu_kernel void @s_uint_to_fp_v4i32_to_v4f64(<4 x double> addrspace(1)
 ; uses an SGPR (implicit vcc).
 
 ; GCN-LABEL: {{^}}uint_to_fp_i1_to_f64:
-; GCN-DAG: v_cmp_eq_u32_e64 vcc
-; GCN-DAG: v_cndmask_b32_e32 v[[SEL:[0-9]+]], 0, v{{[0-9]+}}
+; GCN-DAG: s_cmp_eq_u32
+; GCN-DAG: s_cselect_b32 s[[SSEL:[0-9]+]], 0x3ff00000, 0
+; GCN-DAG: v_mov_b32_e32 v[[SEL:[0-9]+]], s[[SSEL]]
 ; GCN-DAG: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}}
 ; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[ZERO]]:[[SEL]]{{\]}}
 ; GCN: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/undefined-subreg-liverange.ll b/llvm/test/CodeGen/AMDGPU/undefined-subreg-liverange.ll
index 0d6bb66179775..0b0389edf46be 100644
--- a/llvm/test/CodeGen/AMDGPU/undefined-subreg-liverange.ll
+++ b/llvm/test/CodeGen/AMDGPU/undefined-subreg-liverange.ll
@@ -1,10 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s
+
 ; We may have subregister live ranges that are undefined on some paths. The
 ; verifier should not complain about this.
 
-
-; CHECK-LABEL: {{^}}func:
 define amdgpu_kernel void @func() #0 {
+; CHECK-LABEL: func:
+; CHECK:       ; %bb.0: ; %B0
+; CHECK-NEXT:    s_mov_b32 s0, 0
+; CHECK-NEXT:    s_cbranch_scc1 BB0_2
+; CHECK-NEXT:  ; %bb.1: ; %B30.1
+; CHECK-NEXT:    s_mov_b32 s0, 0x7fc00000
+; CHECK-NEXT:  BB0_2: ; %B30.2
+; CHECK-NEXT:    v_mov_b32_e32 v0, s0
+; CHECK-NEXT:    s_mov_b32 m0, -1
+; CHECK-NEXT:    ds_write_b32 v0, v0
+; CHECK-NEXT:    s_endpgm
 B0:
   br i1 undef, label %B1, label %B2
 
@@ -28,8 +39,28 @@ B30.2:
 
 ; FIXME: Extra undef subregister copy should be removed before
 ; overwritten with defined copy
-; CHECK-LABEL: {{^}}valley_partially_undef_copy:
 define amdgpu_ps float @valley_partially_undef_copy() #0 {
+; CHECK-LABEL: valley_partially_undef_copy:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_mov_b32 s3, 0xf000
+; CHECK-NEXT:    s_mov_b32 s2, -1
+; CHECK-NEXT:    buffer_load_dword v1, off, s[0:3], 0
+; CHECK-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; CHECK-NEXT:    buffer_store_dword v2, off, s[0:3], 0
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v1
+; CHECK-NEXT:  BB1_1: ; %bb9
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; CHECK-NEXT:    s_cbranch_vccnz BB1_1
+; CHECK-NEXT:  ; %bb.2: ; %bb11
+; CHECK-NEXT:    s_mov_b32 s3, 0xf000
+; CHECK-NEXT:    s_mov_b32 s2, -1
+; CHECK-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; CHECK-NEXT:    ; return to shader part epilog
 bb:
   %tmp = load volatile i32, i32 addrspace(1)* undef, align 4
   %tmp1 = load volatile i32, i32 addrspace(1)* undef, align 4
@@ -54,24 +85,27 @@ bb11:                                             ; preds = %bb9
 }
 
 ; FIXME: Should be able to remove the undef copies
-
-; CHECK-LABEL: {{^}}partially_undef_copy:
-; CHECK: v_mov_b32_e32 v5, 5
-; CHECK-DAG: v_mov_b32_e32 v6, 6
-
-; CHECK-DAG: v_mov_b32_e32 v[[OUTPUT_LO:[0-9]+]], v5
-
-; Undef copy
-; CHECK-DAG: v_mov_b32_e32 v1, v6
-
-; undef copy
-; CHECK-DAG: v_mov_b32_e32 v2, v7
-
-; CHECK-DAG: v_mov_b32_e32 v[[OUTPUT_HI:[0-9]+]], v8
-; CHECK-DAG: v_mov_b32_e32 v[[OUTPUT_LO]], v6
-
-; CHECK: buffer_store_dwordx4 v{{\[}}[[OUTPUT_LO]]:[[OUTPUT_HI]]{{\]}}
 define amdgpu_kernel void @partially_undef_copy() #0 {
+; CHECK-LABEL: partially_undef_copy:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    v_mov_b32_e32 v5, 5
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    v_mov_b32_e32 v6, 6
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    v_mov_b32_e32 v0, v5
+; CHECK-NEXT:    v_mov_b32_e32 v1, v6
+; CHECK-NEXT:    v_mov_b32_e32 v2, v7
+; CHECK-NEXT:    v_mov_b32_e32 v3, v8
+; CHECK-NEXT:    s_mov_b32 s3, 0xf000
+; CHECK-NEXT:    s_mov_b32 s2, -1
+; CHECK-NEXT:    v_mov_b32_e32 v0, v6
+; CHECK-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    v_nop
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_endpgm
   %tmp0 = call i32 asm sideeffect "v_mov_b32_e32 v5, 5", "={v5}"()
   %tmp1 = call i32 asm sideeffect "v_mov_b32_e32 v6, 6", "={v6}"()
 
diff --git a/llvm/test/CodeGen/AMDGPU/urem64.ll b/llvm/test/CodeGen/AMDGPU/urem64.ll
index 0aac641dae974..ef492be025068 100644
--- a/llvm/test/CodeGen/AMDGPU/urem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/urem64.ll
@@ -6,7 +6,6 @@ define amdgpu_kernel void @s_test_urem_i64(i64 addrspace(1)* %out, i64 %x, i64 %
 ; GCN-LABEL: s_test_urem_i64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0xd
-; GCN-NEXT:    v_mov_b32_e32 v2, 0
 ; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
@@ -18,99 +17,100 @@ define amdgpu_kernel void @s_test_urem_i64(i64 addrspace(1)* %out, i64 %x, i64 %
 ; GCN-NEXT:    s_mov_b32 s4, s8
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-NEXT:    v_mov_b32_e32 v1, 0
 ; GCN-NEXT:    s_mov_b32 s5, s9
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GCN-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v0
-; GCN-NEXT:    v_trunc_f32_e32 v3, v3
-; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v3
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GCN-NEXT:    v_mul_hi_u32 v5, s2, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s2, v3
-; GCN-NEXT:    v_mul_lo_u32 v7, s3, v0
-; GCN-NEXT:    v_mul_lo_u32 v6, s2, v0
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
-; GCN-NEXT:    v_mul_hi_u32 v5, v0, v6
-; GCN-NEXT:    v_mul_lo_u32 v7, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v9, v0, v4
-; GCN-NEXT:    v_mul_lo_u32 v8, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v6, v3, v6
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v2, v9, vcc
-; GCN-NEXT:    v_mul_hi_u32 v9, v3, v4
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v9, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v2, v6, vcc
-; GCN-NEXT:    v_addc_u32_e64 v4, vcc, v3, v5, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v6, s2, v4
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
+; GCN-NEXT:    v_mul_lo_u32 v5, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s2, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v1, v4
+; GCN-NEXT:    v_mul_lo_u32 v4, v1, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_mul_hi_u32 v8, v1, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v6, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v5, s2, v2
 ; GCN-NEXT:    v_mul_hi_u32 v7, s2, v0
 ; GCN-NEXT:    v_mul_lo_u32 v8, s3, v0
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
 ; GCN-NEXT:    v_mul_lo_u32 v7, s2, v0
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
-; GCN-NEXT:    v_mul_lo_u32 v10, v0, v6
-; GCN-NEXT:    v_mul_hi_u32 v12, v0, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
+; GCN-NEXT:    v_mul_lo_u32 v10, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v12, v0, v5
 ; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
-; GCN-NEXT:    v_mul_hi_u32 v9, v4, v7
-; GCN-NEXT:    v_mul_lo_u32 v7, v4, v7
-; GCN-NEXT:    v_mul_hi_u32 v8, v4, v6
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v8, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, v2, v12, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, v4, v6
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v8, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v2, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
-; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v6, s[0:1]
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, s10, v3
-; GCN-NEXT:    v_mul_hi_u32 v5, s10, v0
-; GCN-NEXT:    v_mul_hi_u32 v6, s10, v3
-; GCN-NEXT:    v_mul_hi_u32 v7, s11, v3
-; GCN-NEXT:    v_mul_lo_u32 v3, s11, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v2, v6, vcc
-; GCN-NEXT:    v_mul_lo_u32 v6, s11, v0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v5, v0, vcc
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v7, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v1, s12, v1
 ; GCN-NEXT:    v_mul_hi_u32 v2, s12, v0
 ; GCN-NEXT:    v_mul_lo_u32 v3, s13, v0
 ; GCN-NEXT:    v_mul_lo_u32 v0, s12, v0
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e64 v0, s[0:1], s10, v0
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s11, v1
 ; GCN-NEXT:    v_mov_b32_e32 v3, s13
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s10, v0
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s12, v0
-; GCN-NEXT:    v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s13, v5
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s12, v4
-; GCN-NEXT:    v_subrev_i32_e64 v3, s[0:1], s12, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s13, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[2:3]
-; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v4, s[2:3], s12, v0
+; GCN-NEXT:    v_subbrev_u32_e64 v5, vcc, 0, v2, s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s12, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v5, s11
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v5, v1, vcc
+; GCN-NEXT:    v_subb_u32_e64 v1, vcc, v5, v1, s[0:1]
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v1
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v0
@@ -119,7 +119,7 @@ define amdgpu_kernel void @s_test_urem_i64(i64 addrspace(1)* %out, i64 %x, i64 %
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
@@ -127,97 +127,91 @@ define amdgpu_kernel void @s_test_urem_i64(i64 addrspace(1)* %out, i64 %x, i64 %
 ; GCN-IR-LABEL: s_test_urem_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-IR-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
+; GCN-IR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GCN-IR-NEXT:    s_mov_b64 s[2:3], 0
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[6:7], 0
-; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s2
-; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s10
-; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s6
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
-; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s10
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v2, v3
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[2:3], 0
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[10:11], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[8:9]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[8:9], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[8:9], s[8:9], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT:    s_cbranch_vccz BB0_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[4:5], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[6:7], 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[0:1], 0
+; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s0
+; GCN-IR-NEXT:    s_add_i32 s14, s12, 32
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[8:9], s[10:11]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s1
+; GCN-IR-NEXT:    s_cmp_eq_u32 s1, 0
+; GCN-IR-NEXT:    s_cselect_b32 s10, s14, s8
+; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s6
+; GCN-IR-NEXT:    s_add_i32 s8, s8, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s9, s7
+; GCN-IR-NEXT:    s_cmp_eq_u32 s7, 0
+; GCN-IR-NEXT:    s_cselect_b32 s14, s8, s9
+; GCN-IR-NEXT:    s_sub_u32 s8, s10, s14
+; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[8:9], 63
+; GCN-IR-NEXT:    s_mov_b32 s11, 0
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[16:17]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[16:17], s[8:9], 63
+; GCN-IR-NEXT:    s_xor_b64 s[18:19], s[12:13], -1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[18:19], s[16:17]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[16:17]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB0_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s16, s8, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_addc_u32 s17, s9, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[16:17], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s8, 63, s8
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[12:13], s[6:7], s8
+; GCN-IR-NEXT:    s_cbranch_vccz BB0_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_not_b32_e32 v2, v2
-; GCN-IR-NEXT:    s_add_u32 s8, s2, -1
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[6:7], v4
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v2, v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s9, s3, -1
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], -1, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[16:17], s[6:7], s16
+; GCN-IR-NEXT:    s_add_u32 s8, s0, -1
+; GCN-IR-NEXT:    s_addc_u32 s9, s1, -1
+; GCN-IR-NEXT:    s_not_b64 s[2:3], s[10:11]
+; GCN-IR-NEXT:    s_mov_b32 s15, s11
+; GCN-IR-NEXT:    s_add_u32 s10, s2, s14
+; GCN-IR-NEXT:    s_addc_u32 s11, s3, s11
+; GCN-IR-NEXT:    s_mov_b64 s[14:15], 0
+; GCN-IR-NEXT:    s_mov_b32 s3, 0
 ; GCN-IR-NEXT:  BB0_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s9
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s8, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s2, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s3, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s2, s13, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[16:17], s[16:17], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[12:13], s[12:13], 1
+; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[2:3]
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[14:15], s[12:13]
+; GCN-IR-NEXT:    s_sub_u32 s2, s8, s16
+; GCN-IR-NEXT:    s_subb_u32 s2, s9, s17
+; GCN-IR-NEXT:    s_ashr_i32 s14, s2, 31
+; GCN-IR-NEXT:    s_mov_b32 s15, s14
+; GCN-IR-NEXT:    s_and_b32 s2, s14, 1
+; GCN-IR-NEXT:    s_and_b64 s[18:19], s[14:15], s[0:1]
+; GCN-IR-NEXT:    s_sub_u32 s16, s16, s18
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-IR-NEXT:    s_subb_u32 s17, s17, s19
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-IR-NEXT:    s_add_u32 s10, s10, 1
+; GCN-IR-NEXT:    s_addc_u32 s11, s11, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[10:11], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[14:15], s[2:3]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB0_3
+; GCN-IR-NEXT:  BB0_4: ; %Flow6
+; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[12:13], 1
+; GCN-IR-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-IR-NEXT:    s_branch BB0_6
-; GCN-IR-NEXT:  BB0_4:
+; GCN-IR-NEXT:  BB0_5:
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[12:13]
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB0_7
-; GCN-IR-NEXT:  BB0_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB0_6: ; %Flow6
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB0_7: ; %udiv-end
-; GCN-IR-NEXT:    v_mul_lo_u32 v1, s2, v1
-; GCN-IR-NEXT:    v_mul_hi_u32 v2, s2, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v3, s3, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[12:13]
+; GCN-IR-NEXT:  BB0_6: ; %udiv-end
+; GCN-IR-NEXT:    v_mul_lo_u32 v1, s0, v1
+; GCN-IR-NEXT:    v_mul_hi_u32 v2, s0, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v3, s1, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v0, s0, v0
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-IR-NEXT:    v_mov_b32_e32 v2, s7
@@ -826,25 +820,25 @@ define amdgpu_kernel void @s_test_urem_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_mul_lo_u32 v0, s6, v0
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e64 v0, s[0:1], 24, v0
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, 0, v1
 ; GCN-NEXT:    v_mov_b32_e32 v3, s7
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, 24, v0
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s6, v0
-; GCN-NEXT:    v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s7, v5
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s6, v4
-; GCN-NEXT:    v_subrev_i32_e64 v3, s[0:1], s6, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s7, v5
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[2:3]
-; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v4, s[2:3], s6, v0
+; GCN-NEXT:    v_subbrev_u32_e64 v5, vcc, 0, v2, s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s7, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s6, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s7, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s6, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_subb_u32_e64 v1, vcc, 0, v1, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v6
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s7, v1
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
@@ -852,99 +846,93 @@ define amdgpu_kernel void @s_test_urem_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_urem_k_num_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
-; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_flbit_i32_b32 s2, s6
-; GCN-IR-NEXT:    s_flbit_i32_b32 s3, s7
-; GCN-IR-NEXT:    s_add_i32 s2, s2, 32
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s2
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 0xffffffc5, v2
-; GCN-IR-NEXT:    v_addc_u32_e64 v1, s[2:3], 0, -1, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[2:3], s[2:3], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[2:3]
-; GCN-IR-NEXT:    s_cbranch_vccz BB6_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[3:4], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], 24, v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s4, s2
+; GCN-IR-NEXT:    s_add_i32 s4, s4, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s5, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s8, s4, s5
+; GCN-IR-NEXT:    s_add_u32 s6, s8, 0xffffffc5
+; GCN-IR-NEXT:    s_addc_u32 s7, 0, -1
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[2:3], 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[6:7], 63
+; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[12:13], s[6:7], 63
+; GCN-IR-NEXT:    s_xor_b64 s[14:15], s[10:11], -1
+; GCN-IR-NEXT:    s_and_b64 s[12:13], s[14:15], s[12:13]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[12:13]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB6_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s12, s6, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT:    s_addc_u32 s13, s7, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s7
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s6, 63, s6
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], 24, s6
+; GCN-IR-NEXT:    s_cbranch_vccz BB6_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_add_u32 s2, s6, -1
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], 24, v3
-; GCN-IR-NEXT:    v_sub_i32_e32 v4, vcc, 58, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s3, s7, -1
-; GCN-IR-NEXT:    v_subb_u32_e64 v5, s[0:1], 0, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[14:15], 24, s12
+; GCN-IR-NEXT:    s_add_u32 s6, s2, -1
+; GCN-IR-NEXT:    s_addc_u32 s7, s3, -1
+; GCN-IR-NEXT:    s_sub_u32 s8, 58, s8
+; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
+; GCN-IR-NEXT:    s_mov_b32 s5, 0
 ; GCN-IR-NEXT:  BB6_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s3
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s2, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s6, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s7, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s4, s11, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[4:5]
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[12:13], s[10:11]
+; GCN-IR-NEXT:    s_sub_u32 s4, s6, s14
+; GCN-IR-NEXT:    s_subb_u32 s4, s7, s15
+; GCN-IR-NEXT:    s_ashr_i32 s12, s4, 31
+; GCN-IR-NEXT:    s_mov_b32 s13, s12
+; GCN-IR-NEXT:    s_and_b32 s4, s12, 1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[12:13], s[2:3]
+; GCN-IR-NEXT:    s_sub_u32 s14, s14, s16
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_subb_u32 s15, s15, s17
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    s_add_u32 s8, s8, 1
+; GCN-IR-NEXT:    s_addc_u32 s9, s9, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[8:9], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], s[4:5]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB6_3
+; GCN-IR-NEXT:  BB6_4: ; %Flow5
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s5
 ; GCN-IR-NEXT:    s_branch BB6_6
-; GCN-IR-NEXT:  BB6_4:
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, 24, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB6_7
 ; GCN-IR-NEXT:  BB6_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB6_6: ; %Flow5
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB6_7: ; %udiv-end
-; GCN-IR-NEXT:    v_mul_lo_u32 v1, s6, v1
-; GCN-IR-NEXT:    v_mul_hi_u32 v2, s6, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v3, s7, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v0, s6, v0
-; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, 24, 0, s[10:11]
+; GCN-IR-NEXT:  BB6_6: ; %udiv-end
+; GCN-IR-NEXT:    v_mul_lo_u32 v1, s2, v1
+; GCN-IR-NEXT:    v_mul_hi_u32 v2, s2, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v3, s3, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 24, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    s_mov_b32 s6, -1
-; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-IR-NEXT:    s_mov_b32 s2, -1
+; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %result = urem i64 24, %x
   store i64 %result, i64 addrspace(1)* %out
@@ -966,12 +954,12 @@ define amdgpu_kernel void @s_test_urem_k_den_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-NEXT:    s_mov_b32 s11, 0xf000
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    v_mul_hi_u32 v2, v0, s2
 ; GCN-NEXT:    v_mul_lo_u32 v3, v1, s2
 ; GCN-NEXT:    v_mul_lo_u32 v4, v0, s2
-; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, v0, v2
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
@@ -984,7 +972,7 @@ define amdgpu_kernel void @s_test_urem_k_den_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v3, vcc
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s8, s4
+; GCN-NEXT:    s_mov_b32 s4, s8
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v9, v7, vcc
@@ -996,7 +984,7 @@ define amdgpu_kernel void @s_test_urem_k_den_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_mul_lo_u32 v5, v2, s2
 ; GCN-NEXT:    v_mul_lo_u32 v6, v0, s2
 ; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, v0, v4
-; GCN-NEXT:    s_mov_b32 s9, s5
+; GCN-NEXT:    s_mov_b32 s5, s9
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
 ; GCN-NEXT:    v_mul_lo_u32 v5, v0, v4
 ; GCN-NEXT:    v_mul_hi_u32 v9, v0, v6
@@ -1016,15 +1004,15 @@ define amdgpu_kernel void @s_test_urem_k_den_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v4, s[0:1]
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
-; GCN-NEXT:    v_mul_hi_u32 v4, s6, v1
-; GCN-NEXT:    v_mul_hi_u32 v5, s7, v1
-; GCN-NEXT:    v_mul_lo_u32 v1, s7, v1
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v4, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v5, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
-; GCN-NEXT:    v_mul_hi_u32 v0, s7, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s11, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v7, vcc
@@ -1034,8 +1022,8 @@ define amdgpu_kernel void @s_test_urem_k_den_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_mul_lo_u32 v1, v1, 24
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, 24
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
-; GCN-NEXT:    v_mov_b32_e32 v2, s7
+; GCN-NEXT:    v_mov_b32_e32 v2, s11
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s10, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
 ; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, 24, v0
 ; GCN-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v1, vcc
@@ -1045,104 +1033,99 @@ define amdgpu_kernel void @s_test_urem_k_den_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GCN-NEXT:    v_cndmask_b32_e32 v6, -1, v6, vcc
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], 23, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v1
-; GCN-NEXT:    v_cndmask_b32_e64 v5, -1, v5, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v5
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, 23, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v5, -1, v5, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v4, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_urem_k_den_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
-; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_flbit_i32_b32 s2, s6
-; GCN-IR-NEXT:    s_flbit_i32_b32 s3, s7
-; GCN-IR-NEXT:    s_add_i32 s2, s2, 32
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s2
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 59, v2
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[2:3], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[2:3], s[2:3], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[2:3]
-; GCN-IR-NEXT:    s_cbranch_vccz BB7_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[3:4], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s4, s2
+; GCN-IR-NEXT:    s_add_i32 s4, s4, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s5, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s6, s4, s5
+; GCN-IR-NEXT:    s_sub_u32 s8, 59, s6
+; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[2:3], 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[8:9], 63
+; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[12:13], s[8:9], 63
+; GCN-IR-NEXT:    s_xor_b64 s[14:15], s[10:11], -1
+; GCN-IR-NEXT:    s_and_b64 s[12:13], s[14:15], s[12:13]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[12:13]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB7_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s10, s8, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_addc_u32 s11, s9, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[10:11], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s7, 63, s8
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[2:3], s7
+; GCN-IR-NEXT:    s_cbranch_vccz BB7_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[6:7], v3
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 0xffffffc4, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], 0, -1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[12:13], s[2:3], s10
+; GCN-IR-NEXT:    s_add_u32 s6, s6, 0xffffffc4
+; GCN-IR-NEXT:    s_addc_u32 s7, 0, -1
+; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
+; GCN-IR-NEXT:    s_mov_b32 s5, 0
 ; GCN-IR-NEXT:  BB7_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_sub_i32_e32 v2, vcc, 23, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, 0, v7, vcc
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, 24, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subbrev_u32_e64 v7, s[0:1], 0, v7, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s4, s9, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[12:13], s[12:13], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[8:9], 1
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[4:5]
+; GCN-IR-NEXT:    s_or_b64 s[8:9], s[10:11], s[8:9]
+; GCN-IR-NEXT:    s_sub_u32 s4, 23, s12
+; GCN-IR-NEXT:    s_subb_u32 s4, 0, s13
+; GCN-IR-NEXT:    s_ashr_i32 s10, s4, 31
+; GCN-IR-NEXT:    s_and_b32 s4, s10, 1
+; GCN-IR-NEXT:    s_and_b32 s10, s10, 24
+; GCN-IR-NEXT:    s_sub_u32 s12, s12, s10
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT:    s_subb_u32 s13, s13, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s7
+; GCN-IR-NEXT:    s_add_u32 s6, s6, 1
+; GCN-IR-NEXT:    s_addc_u32 s7, s7, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[10:11], s[4:5]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB7_3
+; GCN-IR-NEXT:  BB7_4: ; %Flow5
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[8:9], 1
+; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s5
 ; GCN-IR-NEXT:    s_branch BB7_6
-; GCN-IR-NEXT:  BB7_4:
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB7_7
 ; GCN-IR-NEXT:  BB7_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB7_6: ; %Flow5
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB7_7: ; %udiv-end
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s3
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[10:11]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[10:11]
+; GCN-IR-NEXT:  BB7_6: ; %udiv-end
 ; GCN-IR-NEXT:    v_mul_hi_u32 v2, v0, 24
 ; GCN-IR-NEXT:    v_mul_lo_u32 v1, v1, 24
 ; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, 24
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s7
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
+; GCN-IR-NEXT:    v_mov_b32_e32 v2, s3
+; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
-; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-IR-NEXT:    s_mov_b32 s6, -1
-; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-IR-NEXT:    s_mov_b32 s2, -1
+; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %result = urem i64 %x, 24
   store i64 %result, i64 addrspace(1)* %out
diff --git a/llvm/test/CodeGen/AMDGPU/vector-alloca-bitcast.ll b/llvm/test/CodeGen/AMDGPU/vector-alloca-bitcast.ll
index 28fb550c44af6..de326d50e166f 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-alloca-bitcast.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-alloca-bitcast.ll
@@ -13,13 +13,14 @@ target datalayout = "A5"
 ; GCN-ALLOCA-COUNT-4: buffer_store_dword
 ; GCN-ALLOCA:         buffer_load_dword
 
+; GCN_PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 2
 ; GCN-PROMOTE: v_cmp_eq_u32_e64 [[CC1:[^,]+]], s{{[0-9]+}}, 1
+; GCN-PROMOTE: s_cselect_b64 vcc, -1, 0
 ; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND1:v[0-9]+]], 0, 1, [[CC1]]
-; GCN-PROMOTE: v_cmp_ne_u32_e64 [[CC2:[^,]+]], s{{[0-9]+}}, 2
-; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND2:v[0-9]+]], 2, [[IND1]], [[CC2]]
-; GCN-PROMOTE: v_cmp_ne_u32_e64 [[CC3:[^,]+]], s{{[0-9]+}}, 3
-; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND3:v[0-9]+]], 3, [[IND2]], [[CC3]]
-
+; GCN_PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 3
+; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND2:v[0-9]+]], 2, [[IND1]], vcc
+; GCN-PROMOTE: s_cselect_b64 vcc, -1, 0
+; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND3:v[0-9]+]], 3, [[IND2]], vcc
 ; GCN-PROMOTE: ScratchSize: 0
 
 define amdgpu_kernel void @vector_read_alloca_bitcast(i32 addrspace(1)* %out, i32 %index) {
@@ -320,12 +321,14 @@ entry:
 ; GCN-ALLOCA-COUNT-4: buffer_store_dword
 ; GCN-ALLOCA:         buffer_load_dword
 
+; GCN_PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 2
 ; GCN-PROMOTE: v_cmp_eq_u32_e64 [[CC1:[^,]+]], s{{[0-9]+}}, 1
+; GCN-PROMOTE: s_cselect_b64 vcc, -1, 0
 ; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND1:v[0-9]+]], 0, 1, [[CC1]]
-; GCN-PROMOTE: v_cmp_ne_u32_e64 [[CC2:[^,]+]], s{{[0-9]+}}, 2
-; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND2:v[0-9]+]], 2, [[IND1]], [[CC2]]
-; GCN-PROMOTE: v_cmp_ne_u32_e64 [[CC3:[^,]+]], s{{[0-9]+}}, 3
-; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND3:v[0-9]+]], 3, [[IND2]], [[CC3]]
+; GCN_PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 3
+; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND2:v[0-9]+]], 2, [[IND1]], vcc
+; GCN-PROMOTE: s_cselect_b64 vcc, -1, 0
+; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND3:v[0-9]+]], 3, [[IND2]], vcc
 
 ; GCN-PROMOTE: ScratchSize: 0
 
diff --git a/llvm/test/CodeGen/AMDGPU/vector-alloca-limits.ll b/llvm/test/CodeGen/AMDGPU/vector-alloca-limits.ll
new file mode 100644
index 0000000000000..6235b9e3b16a4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/vector-alloca-limits.ll
@@ -0,0 +1,136 @@
+; RUN: opt -S -mtriple=amdgcn-- -amdgpu-promote-alloca -sroa -instcombine < %s | FileCheck -check-prefix=OPT %s
+; RUN: opt -S -mtriple=amdgcn-- -amdgpu-promote-alloca -sroa -instcombine -amdgpu-promote-alloca-to-vector-limit=32 < %s | FileCheck -check-prefix=LIMIT32 %s
+
+target datalayout = "A5"
+
+; OPT-LABEL: @alloca_8xi64_max1024(
+; OPT-NOT: alloca
+; OPT: <8 x i64>
+; LIMIT32: alloca
+; LIMIT32-NOT: <8 x i64>
+define amdgpu_kernel void @alloca_8xi64_max1024(i64 addrspace(1)* %out, i32 %index) #0 {
+entry:
+  %tmp = alloca [8 x i64], addrspace(5)
+  %x = getelementptr [8 x i64], [8 x i64] addrspace(5)* %tmp, i32 0, i32 0
+  store i64 0, i64 addrspace(5)* %x
+  %tmp1 = getelementptr [8 x i64], [8 x i64] addrspace(5)* %tmp, i32 0, i32 %index
+  %tmp2 = load i64, i64 addrspace(5)* %tmp1
+  store i64 %tmp2, i64 addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @alloca_9xi64_max1024(
+; OPT: alloca [9 x i64]
+; OPT-NOT: <9 x i64>
+; LIMIT32: alloca
+; LIMIT32-NOT: <9 x i64>
+define amdgpu_kernel void @alloca_9xi64_max1024(i64 addrspace(1)* %out, i32 %index) #0 {
+entry:
+  %tmp = alloca [9 x i64], addrspace(5)
+  %x = getelementptr [9 x i64], [9 x i64] addrspace(5)* %tmp, i32 0, i32 0
+  store i64 0, i64 addrspace(5)* %x
+  %tmp1 = getelementptr [9 x i64], [9 x i64] addrspace(5)* %tmp, i32 0, i32 %index
+  %tmp2 = load i64, i64 addrspace(5)* %tmp1
+  store i64 %tmp2, i64 addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @alloca_16xi64_max512(
+; OPT-NOT: alloca
+; OPT: <16 x i64>
+; LIMIT32: alloca
+; LIMIT32-NOT: <16 x i64>
+define amdgpu_kernel void @alloca_16xi64_max512(i64 addrspace(1)* %out, i32 %index) #1 {
+entry:
+  %tmp = alloca [16 x i64], addrspace(5)
+  %x = getelementptr [16 x i64], [16 x i64] addrspace(5)* %tmp, i32 0, i32 0
+  store i64 0, i64 addrspace(5)* %x
+  %tmp1 = getelementptr [16 x i64], [16 x i64] addrspace(5)* %tmp, i32 0, i32 %index
+  %tmp2 = load i64, i64 addrspace(5)* %tmp1
+  store i64 %tmp2, i64 addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @alloca_17xi64_max512(
+; OPT: alloca [17 x i64]
+; OPT-NOT: <17 x i64>
+; LIMIT32: alloca
+; LIMIT32-NOT: <17 x i64>
+define amdgpu_kernel void @alloca_17xi64_max512(i64 addrspace(1)* %out, i32 %index) #1 {
+entry:
+  %tmp = alloca [17 x i64], addrspace(5)
+  %x = getelementptr [17 x i64], [17 x i64] addrspace(5)* %tmp, i32 0, i32 0
+  store i64 0, i64 addrspace(5)* %x
+  %tmp1 = getelementptr [17 x i64], [17 x i64] addrspace(5)* %tmp, i32 0, i32 %index
+  %tmp2 = load i64, i64 addrspace(5)* %tmp1
+  store i64 %tmp2, i64 addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @alloca_9xi128_max512(
+; OPT: alloca [9 x i128]
+; OPT-NOT: <9 x i128>
+; LIMIT32: alloca
+; LIMIT32-NOT: <9 x i128>
+define amdgpu_kernel void @alloca_9xi128_max512(i128 addrspace(1)* %out, i32 %index) #1 {
+entry:
+  %tmp = alloca [9 x i128], addrspace(5)
+  %x = getelementptr [9 x i128], [9 x i128] addrspace(5)* %tmp, i32 0, i32 0
+  store i128 0, i128 addrspace(5)* %x
+  %tmp1 = getelementptr [9 x i128], [9 x i128] addrspace(5)* %tmp, i32 0, i32 %index
+  %tmp2 = load i128, i128 addrspace(5)* %tmp1
+  store i128 %tmp2, i128 addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @alloca_9xi128_max256(
+; OPT-NOT: alloca
+; OPT: <9 x i128>
+; LIMIT32: alloca
+; LIMIT32-NOT: <9 x i128>
+define amdgpu_kernel void @alloca_9xi128_max256(i128 addrspace(1)* %out, i32 %index) #2 {
+entry:
+  %tmp = alloca [9 x i128], addrspace(5)
+  %x = getelementptr [9 x i128], [9 x i128] addrspace(5)* %tmp, i32 0, i32 0
+  store i128 0, i128 addrspace(5)* %x
+  %tmp1 = getelementptr [9 x i128], [9 x i128] addrspace(5)* %tmp, i32 0, i32 %index
+  %tmp2 = load i128, i128 addrspace(5)* %tmp1
+  store i128 %tmp2, i128 addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @alloca_16xi128_max256(
+; OPT-NOT: alloca
+; OPT: <16 x i128>
+; LIMIT32: alloca
+; LIMIT32-NOT: <16 x i128>
+define amdgpu_kernel void @alloca_16xi128_max256(i128 addrspace(1)* %out, i32 %index) #2 {
+entry:
+  %tmp = alloca [16 x i128], addrspace(5)
+  %x = getelementptr [16 x i128], [16 x i128] addrspace(5)* %tmp, i32 0, i32 0
+  store i128 0, i128 addrspace(5)* %x
+  %tmp1 = getelementptr [16 x i128], [16 x i128] addrspace(5)* %tmp, i32 0, i32 %index
+  %tmp2 = load i128, i128 addrspace(5)* %tmp1
+  store i128 %tmp2, i128 addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @alloca_9xi256_max256(
+; OPT: alloca [9 x i256]
+; OPT-NOT: <9 x i256>
+; LIMIT32: alloca
+; LIMIT32-NOT: <9 x i256>
+define amdgpu_kernel void @alloca_9xi256_max256(i256 addrspace(1)* %out, i32 %index) #2 {
+entry:
+  %tmp = alloca [9 x i256], addrspace(5)
+  %x = getelementptr [9 x i256], [9 x i256] addrspace(5)* %tmp, i32 0, i32 0
+  store i256 0, i256 addrspace(5)* %x
+  %tmp1 = getelementptr [9 x i256], [9 x i256] addrspace(5)* %tmp, i32 0, i32 %index
+  %tmp2 = load i256, i256 addrspace(5)* %tmp1
+  store i256 %tmp2, i256 addrspace(1)* %out
+  ret void
+}
+
+attributes #0 = { "amdgpu-flat-work-group-size"="1,1024" }
+attributes #1 = { "amdgpu-flat-work-group-size"="1,512" }
+attributes #2 = { "amdgpu-flat-work-group-size"="1,256" }
diff --git a/llvm/test/CodeGen/AMDGPU/vselect.ll b/llvm/test/CodeGen/AMDGPU/vselect.ll
index 02ffd30be5fda..41f03ee0e37c8 100644
--- a/llvm/test/CodeGen/AMDGPU/vselect.ll
+++ b/llvm/test/CodeGen/AMDGPU/vselect.ll
@@ -7,10 +7,10 @@
 ; EG-DAG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW]}}, KC0[3].Z
 ; EG-DAG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW]}}, KC0[3].Y
 
-; SI: v_cmp_gt_i32_e32 vcc
-; SI: v_cndmask_b32_e32
-; SI: v_cmp_gt_i32_e32 vcc
-; SI: v_cndmask_b32_e32
+; SI: s_cmp_gt_i32
+; SI: s_cselect_b32
+; SI: s_cmp_gt_i32
+; SI: s_cselect_b32
 
 define amdgpu_kernel void @test_select_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in0, <2 x i32> addrspace(1)* %in1, <2 x i32> %val) {
 entry:
@@ -50,10 +50,10 @@ entry:
 ; EG-DAG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW]}}, KC0[3].Z
 ; EG-DAG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW]}}, KC0[3].Y
 
-; SI: v_cndmask_b32_e32
-; SI: v_cndmask_b32_e32
-; SI: v_cndmask_b32_e32
-; SI: v_cndmask_b32_e32
+; SI: s_cselect_b32
+; SI: s_cselect_b32
+; SI: s_cselect_b32
+; SI: s_cselect_b32
 
 define amdgpu_kernel void @test_select_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in0, <4 x i32> addrspace(1)* %in1, <4 x i32> %val) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-overflow.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-overflow.mir
index e06e3031c3d2a..f61056917c826 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-overflow.mir
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-overflow.mir
@@ -2,6 +2,14 @@
 # RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass si-insert-waitcnts -o - %s | FileCheck -check-prefixes=GFX9,GFX9_10 %s
 # RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -run-pass si-insert-waitcnts -o - %s | FileCheck -check-prefixes=GFX10,GFX9_10 %s
 
+--- |
+    define amdgpu_kernel void @max-counter-lgkmcnt() #0 { ret void }
+    define amdgpu_kernel void @max-counter-vmcnt() #0 { ret void }
+    define amdgpu_kernel void @max-counter-expcnt() #0 { ret void }
+
+    attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
+...
+
 # Check that we handle cases where a counter has overflowed.
 
 # Overflows lgkmcnt with gfx9 but not with gfx10.
diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll
index d9e1f0f009347..0b0b9a30f113f 100644
--- a/llvm/test/CodeGen/AMDGPU/wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave32.ll
@@ -345,9 +345,9 @@ bb:
 ; GFX1064: v_add_co_u32_e64 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX1064: v_add_co_u32_e64 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX1064: v_add_co_ci_u32_e32 v{{[0-9]+}}, vcc, 0, v{{[0-9]+}}, vcc{{$}}
-; GFX1064: v_sub_co_u32_e64 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX1064: v_subrev_co_ci_u32_e64 v{{[0-9]+}}, s[{{[0-9:]+}}], {{[vs][0-9]+}}, v{{[0-9]+}}, vcc{{$}}
-; GFX1064: v_sub_co_ci_u32_e32 v{{[0-9]+}}, vcc, {{[vs][0-9]+}}, v{{[0-9]+}}, vcc{{$}}
+; GFX1064: v_sub_co_u32_e64 v{{[0-9]+}}, s[{{[0-9:]+}}], s{{[0-9]+}}, v{{[0-9]+}}
+; GFX1064: v_subrev_co_ci_u32_e64 v{{[0-9]+}}, vcc, {{[vs][0-9]+}}, v{{[0-9]+}}, s[{{[0-9:]+}}]
+; GFX1064: v_sub_co_ci_u32_e64 v{{[0-9]+}}, s[{{[0-9:]+}}], {{[vs][0-9]+}}, v{{[0-9]+}}, s[{{[0-9:]+}}]
 define amdgpu_kernel void @test_udiv64(i64 addrspace(1)* %arg) #0 {
 bb:
   %tmp = getelementptr inbounds i64, i64 addrspace(1)* %arg, i64 1
diff --git a/llvm/test/CodeGen/AMDGPU/widen-smrd-loads.ll b/llvm/test/CodeGen/AMDGPU/widen-smrd-loads.ll
index b35180303e02b..a43c656b06071 100644
--- a/llvm/test/CodeGen/AMDGPU/widen-smrd-loads.ll
+++ b/llvm/test/CodeGen/AMDGPU/widen-smrd-loads.ll
@@ -6,10 +6,10 @@ define amdgpu_kernel void @widen_i16_constant_load(i16 addrspace(4)* %arg) {
 ; SI-LABEL: widen_i16_constant_load:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; SI-NEXT:    s_mov_b32 s5, 0
 ; SI-NEXT:    s_mov_b32 s4, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s5, s4
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -43,10 +43,10 @@ define amdgpu_kernel void @widen_i16_constant_load_zext_i32(i16 addrspace(4)* %a
 ; SI-LABEL: widen_i16_constant_load_zext_i32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; SI-NEXT:    s_mov_b32 s5, 0
 ; SI-NEXT:    s_mov_b32 s4, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s5, s4
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -83,10 +83,10 @@ define amdgpu_kernel void @widen_i16_constant_load_sext_i32(i16 addrspace(4)* %a
 ; SI-LABEL: widen_i16_constant_load_sext_i32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; SI-NEXT:    s_mov_b32 s5, 0
 ; SI-NEXT:    s_mov_b32 s4, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s5, s4
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -122,23 +122,25 @@ define amdgpu_kernel void @widen_i16_constant_load_sext_i32(i16 addrspace(4)* %a
 define amdgpu_kernel void @widen_i17_constant_load(i17 addrspace(4)* %arg) {
 ; SI-LABEL: widen_i17_constant_load:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; SI-NEXT:    s_mov_b32 s5, 0
-; SI-NEXT:    s_mov_b32 s4, 0
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s0, 0
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s1, s0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_add_i32 s0, s0, 34
-; SI-NEXT:    s_or_b32 s0, s0, 4
-; SI-NEXT:    v_mov_b32_e32 v0, s0
-; SI-NEXT:    s_bfe_u32 s0, s0, 0x10010
-; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; SI-NEXT:    s_load_dword s7, s[8:9], 0x0
 ; SI-NEXT:    s_mov_b32 s4, 2
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, s0
-; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_mov_b32 s5, s0
+; SI-NEXT:    s_mov_b32 s6, s2
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_add_i32 s7, s7, 34
+; SI-NEXT:    s_or_b32 s7, s7, 4
+; SI-NEXT:    s_bfe_u32 s8, s7, 0x10010
+; SI-NEXT:    v_mov_b32_e32 v0, s7
+; SI-NEXT:    s_mov_b32 s7, s3
+; SI-NEXT:    v_mov_b32_e32 v1, s8
+; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT:    buffer_store_byte v1, off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: widen_i17_constant_load:
@@ -174,10 +176,10 @@ define amdgpu_kernel void @widen_f16_constant_load(half addrspace(4)* %arg) {
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
-; SI-NEXT:    s_mov_b32 s1, 0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, s0
 ; SI-NEXT:    s_mov_b32 s0, 0
+; SI-NEXT:    s_mov_b32 s1, s0
 ; SI-NEXT:    v_add_f32_e32 v0, 4.0, v0
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
@@ -205,10 +207,10 @@ define amdgpu_kernel void @widen_v2i8_constant_load(<2 x i8> addrspace(4)* %arg)
 ; SI-LABEL: widen_v2i8_constant_load:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; SI-NEXT:    s_mov_b32 s5, 0
 ; SI-NEXT:    s_mov_b32 s4, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s5, s4
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -260,13 +262,14 @@ define amdgpu_kernel void @no_widen_i16_constant_divergent_load(i16 addrspace(4)
 ; SI-NEXT:    v_mov_b32_e32 v1, 0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[0:3], 0 addr64
-; SI-NEXT:    s_mov_b32 s1, 0
-; SI-NEXT:    s_mov_b32 s0, 0
-; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s4, s2
+; SI-NEXT:    s_mov_b32 s5, s2
+; SI-NEXT:    s_mov_b32 s7, s3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x3e7, v0
 ; SI-NEXT:    v_or_b32_e32 v0, 4, v0
-; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: no_widen_i16_constant_divergent_load:
@@ -299,10 +302,10 @@ define amdgpu_kernel void @widen_i1_constant_load(i1 addrspace(4)* %arg) {
 ; SI-LABEL: widen_i1_constant_load:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; SI-NEXT:    s_mov_b32 s5, 0
 ; SI-NEXT:    s_mov_b32 s4, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s5, s4
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -333,10 +336,10 @@ define amdgpu_kernel void @widen_i16_zextload_i64_constant_load(i16 addrspace(4)
 ; SI-LABEL: widen_i16_zextload_i64_constant_load:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; SI-NEXT:    s_mov_b32 s5, 0
 ; SI-NEXT:    s_mov_b32 s4, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s5, s4
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -373,10 +376,10 @@ define amdgpu_kernel void @widen_i1_zext_to_i64_constant_load(i1 addrspace(4)* %
 ; SI-LABEL: widen_i1_zext_to_i64_constant_load:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; SI-NEXT:    s_mov_b32 s5, 0
 ; SI-NEXT:    s_mov_b32 s4, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s5, s4
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -415,17 +418,16 @@ define amdgpu_kernel void @widen_i16_constant32_load(i16 addrspace(6)* %arg) {
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dword s0, s[0:1], 0x9
 ; SI-NEXT:    s_mov_b32 s1, 0
-; SI-NEXT:    s_mov_b32 s5, 0
-; SI-NEXT:    s_mov_b32 s4, 0
-; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
-; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_addk_i32 s0, 0x3e7
-; SI-NEXT:    s_or_b32 s0, s0, 4
-; SI-NEXT:    v_mov_b32_e32 v0, s0
-; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; SI-NEXT:    s_or_b32 s4, s0, 4
+; SI-NEXT:    s_mov_b32 s0, s1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: widen_i16_constant32_load:
@@ -453,10 +455,10 @@ define amdgpu_kernel void @widen_i16_global_invariant_load(i16 addrspace(1)* %ar
 ; SI-LABEL: widen_i16_global_invariant_load:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; SI-NEXT:    s_mov_b32 s5, 0
 ; SI-NEXT:    s_mov_b32 s4, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s5, s4
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/ARM/and-load-combine.ll b/llvm/test/CodeGen/ARM/and-load-combine.ll
index 1bc90ba1b8163..a281cff79ddca 100644
--- a/llvm/test/CodeGen/ARM/and-load-combine.ll
+++ b/llvm/test/CodeGen/ARM/and-load-combine.ll
@@ -1433,12 +1433,9 @@ define arm_aapcscc i64 @test23(i64* nocapture %p) {
 ;
 ; THUMB1-LABEL: test23:
 ; THUMB1:       @ %bb.0:
-; THUMB1-NEXT:    ldrb r1, [r0, #3]
-; THUMB1-NEXT:    ldrb r0, [r0, #4]
-; THUMB1-NEXT:    lsls r0, r0, #8
-; THUMB1-NEXT:    adds r1, r0, r1
-; THUMB1-NEXT:    lsls r0, r1, #24
-; THUMB1-NEXT:    lsrs r1, r1, #8
+; THUMB1-NEXT:    ldrb r1, [r0, #4]
+; THUMB1-NEXT:    ldrb r0, [r0, #3]
+; THUMB1-NEXT:    lsls r0, r0, #24
 ; THUMB1-NEXT:    bx lr
 ;
 ; THUMB2-LABEL: test23:
@@ -1498,13 +1495,15 @@ define arm_aapcscc i64 @test25(i64* nocapture %p) {
 ;
 ; THUMB1-LABEL: test25:
 ; THUMB1:       @ %bb.0:
-; THUMB1-NEXT:    ldrb r1, [r0, #5]
-; THUMB1-NEXT:    ldrb r0, [r0, #6]
-; THUMB1-NEXT:    lsls r0, r0, #8
-; THUMB1-NEXT:    adds r0, r0, r1
-; THUMB1-NEXT:    lsls r1, r0, #8
+; THUMB1-NEXT:    ldr r0, [r0, #4]
+; THUMB1-NEXT:    ldr r1, .LCPI37_0
+; THUMB1-NEXT:    ands r1, r0
 ; THUMB1-NEXT:    movs r0, #0
 ; THUMB1-NEXT:    bx lr
+; THUMB1-NEXT:    .p2align 2
+; THUMB1-NEXT:  @ %bb.1:
+; THUMB1-NEXT:  .LCPI37_0:
+; THUMB1-NEXT:    .long 16776960 @ 0xffff00
 ;
 ; THUMB2-LABEL: test25:
 ; THUMB2:       @ %bb.0:
diff --git a/llvm/test/CodeGen/ARM/arm-bf16-dotprod-intrinsics.ll b/llvm/test/CodeGen/ARM/arm-bf16-dotprod-intrinsics.ll
new file mode 100644
index 0000000000000..da6e1274e50ac
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/arm-bf16-dotprod-intrinsics.ll
@@ -0,0 +1,194 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple armv8.6a-arm-none-eabi -mattr=+neon,+bf16 -float-abi=hard -verify-machineinstrs < %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <2 x float> @test_vbfdot_f32(<2 x float> %r, <4 x bfloat> %a, <4 x bfloat> %b) {
+; CHECK-LABEL: test_vbfdot_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdot.bf16 d0, d1, d2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x bfloat> %a to <8 x i8>
+  %1 = bitcast <4 x bfloat> %b to <8 x i8>
+  %vbfdot1.i = call <2 x float> @llvm.arm.neon.bfdot.v2f32.v8i8(<2 x float> %r, <8 x i8> %0, <8 x i8> %1)
+  ret <2 x float> %vbfdot1.i
+}
+
+define <4 x float> @test_vbfdotq_f32(<4 x float> %r, <8 x bfloat> %a, <8 x bfloat> %b) {
+; CHECK-LABEL: test_vbfdotq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdot.bf16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <8 x bfloat> %a to <16 x i8>
+  %1 = bitcast <8 x bfloat> %b to <16 x i8>
+  %vbfdot1.i = call <4 x float> @llvm.arm.neon.bfdot.v4f32.v16i8(<4 x float> %r, <16 x i8> %0, <16 x i8> %1)
+  ret <4 x float> %vbfdot1.i
+}
+
+define <2 x float> @test_vbfdot_lane_f32(<2 x float> %r, <4 x bfloat> %a, <4 x bfloat> %b) {
+; CHECK-LABEL: test_vbfdot_lane_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdot.bf16 d0, d1, d2[0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x bfloat> %b to <2 x float>
+  %shuffle = shufflevector <2 x float> %0, <2 x float> undef, <2 x i32> zeroinitializer
+  %1 = bitcast <4 x bfloat> %a to <8 x i8>
+  %2 = bitcast <2 x float> %shuffle to <8 x i8>
+  %vbfdot1.i = call <2 x float> @llvm.arm.neon.bfdot.v2f32.v8i8(<2 x float> %r, <8 x i8> %1, <8 x i8> %2)
+  ret <2 x float> %vbfdot1.i
+}
+
+define <4 x float> @test_vbfdotq_laneq_f32(<4 x float> %r, <8 x bfloat> %a, <8 x bfloat> %b) {
+; CHECK-LABEL: test_vbfdotq_laneq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q8, d5[1]
+; CHECK-NEXT:    vdot.bf16 q0, q1, q8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <8 x bfloat> %b to <4 x float>
+  %shuffle = shufflevector <4 x float> %0, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %1 = bitcast <8 x bfloat> %a to <16 x i8>
+  %2 = bitcast <4 x float> %shuffle to <16 x i8>
+  %vbfdot1.i = call <4 x float> @llvm.arm.neon.bfdot.v4f32.v16i8(<4 x float> %r, <16 x i8> %1, <16 x i8> %2)
+  ret <4 x float> %vbfdot1.i
+}
+
+define <2 x float> @test_vbfdot_laneq_f32(<2 x float> %r, <4 x bfloat> %a, <8 x bfloat> %b) {
+; CHECK-LABEL: test_vbfdot_laneq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdot.bf16 d0, d1, d3[1]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <8 x bfloat> %b to <4 x float>
+  %shuffle = shufflevector <4 x float> %0, <4 x float> undef, <2 x i32> <i32 3, i32 3>
+  %1 = bitcast <4 x bfloat> %a to <8 x i8>
+  %2 = bitcast <2 x float> %shuffle to <8 x i8>
+  %vbfdot1.i = call <2 x float> @llvm.arm.neon.bfdot.v2f32.v8i8(<2 x float> %r, <8 x i8> %1, <8 x i8> %2)
+  ret <2 x float> %vbfdot1.i
+}
+
+define <4 x float> @test_vbfdotq_lane_f32(<4 x float> %r, <8 x bfloat> %a, <4 x bfloat> %b) {
+; CHECK-LABEL: test_vbfdotq_lane_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    @ kill: def $d4 killed $d4 def $q2
+; CHECK-NEXT:    vdot.bf16 q0, q1, d4[0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x bfloat> %b to <2 x float>
+  %shuffle = shufflevector <2 x float> %0, <2 x float> undef, <4 x i32> zeroinitializer
+  %1 = bitcast <8 x bfloat> %a to <16 x i8>
+  %2 = bitcast <4 x float> %shuffle to <16 x i8>
+  %vbfdot1.i = call <4 x float> @llvm.arm.neon.bfdot.v4f32.v16i8(<4 x float> %r, <16 x i8> %1, <16 x i8> %2)
+  ret <4 x float> %vbfdot1.i
+}
+
+define <4 x float> @test_vbfmmlaq_f32(<4 x float> %r, <8 x bfloat> %a, <8 x bfloat> %b) {
+; CHECK-LABEL: test_vbfmmlaq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmmla.bf16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <8 x bfloat> %a to <16 x i8>
+  %1 = bitcast <8 x bfloat> %b to <16 x i8>
+  %vbfmmla1.i = call <4 x float> @llvm.arm.neon.bfmmla.v4f32.v16i8(<4 x float> %r, <16 x i8> %0, <16 x i8> %1)
+  ret <4 x float> %vbfmmla1.i
+}
+
+define <4 x float> @test_vbfmlalbq_f32(<4 x float> %r, <8 x bfloat> %a, <8 x bfloat> %b) {
+; CHECK-LABEL: test_vbfmlalbq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vfmab.bf16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <8 x bfloat> %a to <16 x i8>
+  %1 = bitcast <8 x bfloat> %b to <16 x i8>
+  %vbfmlalb1.i = call <4 x float> @llvm.arm.neon.bfmlalb.v4f32.v16i8(<4 x float> %r, <16 x i8> %0, <16 x i8> %1)
+  ret <4 x float> %vbfmlalb1.i
+}
+
+define <4 x float> @test_vbfmlaltq_f32(<4 x float> %r, <8 x bfloat> %a, <8 x bfloat> %b) {
+; CHECK-LABEL: test_vbfmlaltq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vfmat.bf16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <8 x bfloat> %a to <16 x i8>
+  %1 = bitcast <8 x bfloat> %b to <16 x i8>
+  %vbfmlalt1.i = call <4 x float> @llvm.arm.neon.bfmlalt.v4f32.v16i8(<4 x float> %r, <16 x i8> %0, <16 x i8> %1)
+  ret <4 x float> %vbfmlalt1.i
+}
+
+define <4 x float> @test_vbfmlalbq_lane_f32(<4 x float> %r, <8 x bfloat> %a, <4 x bfloat> %b) {
+; CHECK-LABEL: test_vbfmlalbq_lane_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    @ kill: def $d4 killed $d4 def $q2
+; CHECK-NEXT:    vfmab.bf16 q0, q1, d4[0]
+; CHECK-NEXT:    bx lr
+entry:
+  %vecinit35 = shufflevector <4 x bfloat> %b, <4 x bfloat> undef, <8 x i32> zeroinitializer
+  %0 = bitcast <8 x bfloat> %a to <16 x i8>
+  %1 = bitcast <8 x bfloat> %vecinit35 to <16 x i8>
+  %vbfmlalb1.i = call <4 x float> @llvm.arm.neon.bfmlalb.v4f32.v16i8(<4 x float> %r, <16 x i8> %0, <16 x i8> %1)
+  ret <4 x float> %vbfmlalb1.i
+}
+
+define <4 x float> @test_vbfmlalbq_laneq_f32(<4 x float> %r, <8 x bfloat> %a, <8 x bfloat> %b) {
+; CHECK-LABEL: test_vbfmlalbq_laneq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vfmab.bf16 q0, q1, d4[3]
+; CHECK-NEXT:    bx lr
+entry:
+  %vecinit35 = shufflevector <8 x bfloat> %b, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %0 = bitcast <8 x bfloat> %a to <16 x i8>
+  %1 = bitcast <8 x bfloat> %vecinit35 to <16 x i8>
+  %vbfmlalb1.i = call <4 x float> @llvm.arm.neon.bfmlalb.v4f32.v16i8(<4 x float> %r, <16 x i8> %0, <16 x i8> %1)
+  ret <4 x float> %vbfmlalb1.i
+}
+
+define <4 x float> @test_vbfmlaltq_lane_f32(<4 x float> %r, <8 x bfloat> %a, <4 x bfloat> %b) {
+; CHECK-LABEL: test_vbfmlaltq_lane_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    @ kill: def $d4 killed $d4 def $q2
+; CHECK-NEXT:    vfmat.bf16 q0, q1, d4[0]
+; CHECK-NEXT:    bx lr
+entry:
+  %vecinit35 = shufflevector <4 x bfloat> %b, <4 x bfloat> undef, <8 x i32> zeroinitializer
+  %0 = bitcast <8 x bfloat> %a to <16 x i8>
+  %1 = bitcast <8 x bfloat> %vecinit35 to <16 x i8>
+  %vbfmlalt1.i = call <4 x float> @llvm.arm.neon.bfmlalt.v4f32.v16i8(<4 x float> %r, <16 x i8> %0, <16 x i8> %1)
+  ret <4 x float> %vbfmlalt1.i
+}
+
+define <4 x float> @test_vbfmlaltq_laneq_f32(<4 x float> %r, <8 x bfloat> %a, <8 x bfloat> %b) {
+; CHECK-LABEL: test_vbfmlaltq_laneq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vfmat.bf16 q0, q1, d4[3]
+; CHECK-NEXT:    bx lr
+entry:
+  %vecinit35 = shufflevector <8 x bfloat> %b, <8 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %0 = bitcast <8 x bfloat> %a to <16 x i8>
+  %1 = bitcast <8 x bfloat> %vecinit35 to <16 x i8>
+  %vbfmlalt1.i = call <4 x float> @llvm.arm.neon.bfmlalt.v4f32.v16i8(<4 x float> %r, <16 x i8> %0, <16 x i8> %1)
+  ret <4 x float> %vbfmlalt1.i
+}
+
+define <4 x float> @test_vbfmlaltq_laneq_f32_v2(<4 x float> %r, <8 x bfloat> %a, <8 x bfloat> %b) {
+; CHECK-LABEL: test_vbfmlaltq_laneq_f32_v2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q8, d5[2]
+; CHECK-NEXT:    vfmat.bf16 q0, q1, q8
+; CHECK-NEXT:    bx lr
+entry:
+  %vecinit35 = shufflevector <8 x bfloat> %b, <8 x bfloat> undef, <8 x i32> <i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6>
+  %0 = bitcast <8 x bfloat> %a to <16 x i8>
+  %1 = bitcast <8 x bfloat> %vecinit35 to <16 x i8>
+  %vbfmlalt1.i = call <4 x float> @llvm.arm.neon.bfmlalt.v4f32.v16i8(<4 x float> %r, <16 x i8> %0, <16 x i8> %1)
+  ret <4 x float> %vbfmlalt1.i
+}
+
+declare <2 x float> @llvm.arm.neon.bfdot.v2f32.v8i8(<2 x float>, <8 x i8>, <8 x i8>)
+declare <4 x float> @llvm.arm.neon.bfdot.v4f32.v16i8(<4 x float>, <16 x i8>, <16 x i8>)
+declare <4 x float> @llvm.arm.neon.bfmmla.v4f32.v16i8(<4 x float>, <16 x i8>, <16 x i8>)
+declare <4 x float> @llvm.arm.neon.bfmlalb.v4f32.v16i8(<4 x float>, <16 x i8>, <16 x i8>)
+declare <4 x float> @llvm.arm.neon.bfmlalt.v4f32.v16i8(<4 x float>, <16 x i8>, <16 x i8>)
diff --git a/llvm/test/CodeGen/ARM/arm-bf16-pcs.ll b/llvm/test/CodeGen/ARM/arm-bf16-pcs.ll
new file mode 100644
index 0000000000000..2e60b91cbc17d
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/arm-bf16-pcs.ll
@@ -0,0 +1,319 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple armv8.6a-arm-none-eabi -o - %s | FileCheck %s --check-prefix=BASE --check-prefix=BASE-ARM
+; RUN: llc -mtriple thumbv8.6a-arm-none-eabi -o - %s | FileCheck %s --check-prefix=BASE --check-prefix=BASE-THUMB
+; RUN: llc -mtriple armv8.6a-arm-none-eabi -mattr=+fullfp16 -o - %s | FileCheck %s --check-prefix=FULLFP16 --check-prefix=FULLFP16-ARM
+; RUN: llc -mtriple thumbv8.6a-arm-none-eabi -mattr=+fullfp16 -o - %s | FileCheck %s --check-prefix=FULLFP16 --check-prefix=FULLFP16-THUMB
+
+define bfloat @bf_load_soft(bfloat* %p) {
+; BASE-LABEL: bf_load_soft:
+; BASE:       @ %bb.0:
+; BASE-NEXT:    ldrh r0, [r0]
+; BASE-NEXT:    bx lr
+;
+; FULLFP16-LABEL: bf_load_soft:
+; FULLFP16:       @ %bb.0:
+; FULLFP16-NEXT:    vldr.16 s0, [r0]
+; FULLFP16-NEXT:    vmov r0, s0
+; FULLFP16-NEXT:    bx lr
+  %f = load bfloat, bfloat* %p, align 2
+  ret bfloat %f
+}
+
+define arm_aapcs_vfpcc bfloat @bf_load_hard(bfloat* %p) {
+; BASE-LABEL: bf_load_hard:
+; BASE:       @ %bb.0:
+; BASE-NEXT:    ldrh r0, [r0]
+; BASE-NEXT:    vmov s0, r0
+; BASE-NEXT:    bx lr
+;
+; FULLFP16-LABEL: bf_load_hard:
+; FULLFP16:       @ %bb.0:
+; FULLFP16-NEXT:    vldr.16 s0, [r0]
+; FULLFP16-NEXT:    bx lr
+  %f = load bfloat, bfloat* %p, align 2
+  ret bfloat %f
+}
+
+define void @bf_store_soft(bfloat* %p, bfloat %f) {
+; BASE-LABEL: bf_store_soft:
+; BASE:       @ %bb.0:
+; BASE-NEXT:    strh r1, [r0]
+; BASE-NEXT:    bx lr
+;
+; FULLFP16-LABEL: bf_store_soft:
+; FULLFP16:       @ %bb.0:
+; FULLFP16-NEXT:    vmov.f16 s0, r1
+; FULLFP16-NEXT:    vstr.16 s0, [r0]
+; FULLFP16-NEXT:    bx lr
+  store bfloat %f, bfloat* %p, align 2
+  ret void
+}
+
+define arm_aapcs_vfpcc void @bf_store_hard(bfloat* %p, bfloat %f) {
+; BASE-LABEL: bf_store_hard:
+; BASE:       @ %bb.0:
+; BASE-NEXT:    vmov r1, s0
+; BASE-NEXT:    strh r1, [r0]
+; BASE-NEXT:    bx lr
+;
+; FULLFP16-LABEL: bf_store_hard:
+; FULLFP16:       @ %bb.0:
+; FULLFP16-NEXT:    vstr.16 s0, [r0]
+; FULLFP16-NEXT:    bx lr
+  store bfloat %f, bfloat* %p, align 2
+  ret void
+}
+
+define i32 @bf_to_int_soft(bfloat %f) {
+; BASE-LABEL: bf_to_int_soft:
+; BASE:       @ %bb.0:
+; BASE-NEXT:    uxth r0, r0
+; BASE-NEXT:    bx lr
+;
+; FULLFP16-LABEL: bf_to_int_soft:
+; FULLFP16:       @ %bb.0:
+; FULLFP16-NEXT:    vmov.f16 s0, r0
+; FULLFP16-NEXT:    vmov.f16 r0, s0
+; FULLFP16-NEXT:    bx lr
+  %h = bitcast bfloat %f to i16
+  %w = zext i16 %h to i32
+  ret i32 %w
+}
+
+define arm_aapcs_vfpcc i32 @bf_to_int_hard(bfloat %f) {
+; BASE-LABEL: bf_to_int_hard:
+; BASE:       @ %bb.0:
+; BASE-NEXT:    vmov r0, s0
+; BASE-NEXT:    uxth r0, r0
+; BASE-NEXT:    bx lr
+;
+; FULLFP16-LABEL: bf_to_int_hard:
+; FULLFP16:       @ %bb.0:
+; FULLFP16-NEXT:    vmov.f16 r0, s0
+; FULLFP16-NEXT:    bx lr
+  %h = bitcast bfloat %f to i16
+  %w = zext i16 %h to i32
+  ret i32 %w
+}
+
+define bfloat @bf_from_int_soft(i32 %w) {
+; BASE-ARM-LABEL: bf_from_int_soft:
+; BASE-ARM:       @ %bb.0:
+; BASE-ARM-NEXT:    .pad #4
+; BASE-ARM-NEXT:    sub sp, sp, #4
+; BASE-ARM-NEXT:    strh r0, [sp, #2]
+; BASE-ARM-NEXT:    ldrh r0, [sp, #2]
+; BASE-ARM-NEXT:    add sp, sp, #4
+; BASE-ARM-NEXT:    bx lr
+;
+; BASE-THUMB-LABEL: bf_from_int_soft:
+; BASE-THUMB:       @ %bb.0:
+; BASE-THUMB-NEXT:    .pad #4
+; BASE-THUMB-NEXT:    sub sp, #4
+; BASE-THUMB-NEXT:    strh.w r0, [sp, #2]
+; BASE-THUMB-NEXT:    ldrh.w r0, [sp, #2]
+; BASE-THUMB-NEXT:    add sp, #4
+; BASE-THUMB-NEXT:    bx lr
+;
+; FULLFP16-LABEL: bf_from_int_soft:
+; FULLFP16:       @ %bb.0:
+; FULLFP16-NEXT:    vmov.f16 s0, r0
+; FULLFP16-NEXT:    vmov r0, s0
+; FULLFP16-NEXT:    bx lr
+  %h = trunc i32 %w to i16
+  %f = bitcast i16 %h to bfloat
+  ret bfloat %f
+}
+
+define arm_aapcs_vfpcc bfloat @bf_from_int_hard(i32 %w) {
+; BASE-ARM-LABEL: bf_from_int_hard:
+; BASE-ARM:       @ %bb.0:
+; BASE-ARM-NEXT:    .pad #4
+; BASE-ARM-NEXT:    sub sp, sp, #4
+; BASE-ARM-NEXT:    strh r0, [sp, #2]
+; BASE-ARM-NEXT:    ldrh r0, [sp, #2]
+; BASE-ARM-NEXT:    vmov s0, r0
+; BASE-ARM-NEXT:    add sp, sp, #4
+; BASE-ARM-NEXT:    bx lr
+;
+; BASE-THUMB-LABEL: bf_from_int_hard:
+; BASE-THUMB:       @ %bb.0:
+; BASE-THUMB-NEXT:    .pad #4
+; BASE-THUMB-NEXT:    sub sp, #4
+; BASE-THUMB-NEXT:    strh.w r0, [sp, #2]
+; BASE-THUMB-NEXT:    ldrh.w r0, [sp, #2]
+; BASE-THUMB-NEXT:    vmov s0, r0
+; BASE-THUMB-NEXT:    add sp, #4
+; BASE-THUMB-NEXT:    bx lr
+;
+; FULLFP16-LABEL: bf_from_int_hard:
+; FULLFP16:       @ %bb.0:
+; FULLFP16-NEXT:    vmov.f16 s0, r0
+; FULLFP16-NEXT:    bx lr
+  %h = trunc i32 %w to i16
+  %f = bitcast i16 %h to bfloat
+  ret bfloat %f
+}
+
+define bfloat @test_fncall_soft(bfloat %bf, bfloat (bfloat, bfloat)* %f) {
+; BASE-ARM-LABEL: test_fncall_soft:
+; BASE-ARM:       @ %bb.0:
+; BASE-ARM-NEXT:    .save {r4, r5, r11, lr}
+; BASE-ARM-NEXT:    push {r4, r5, r11, lr}
+; BASE-ARM-NEXT:    .pad #8
+; BASE-ARM-NEXT:    sub sp, sp, #8
+; BASE-ARM-NEXT:    uxth r5, r0
+; BASE-ARM-NEXT:    mov r4, r1
+; BASE-ARM-NEXT:    mov r0, r5
+; BASE-ARM-NEXT:    mov r1, r5
+; BASE-ARM-NEXT:    blx r4
+; BASE-ARM-NEXT:    strh r0, [sp, #6]
+; BASE-ARM-NEXT:    uxth r1, r0
+; BASE-ARM-NEXT:    mov r0, r5
+; BASE-ARM-NEXT:    blx r4
+; BASE-ARM-NEXT:    ldrh r0, [sp, #6]
+; BASE-ARM-NEXT:    add sp, sp, #8
+; BASE-ARM-NEXT:    pop {r4, r5, r11, pc}
+;
+; BASE-THUMB-LABEL: test_fncall_soft:
+; BASE-THUMB:       @ %bb.0:
+; BASE-THUMB-NEXT:    .save {r4, r5, r7, lr}
+; BASE-THUMB-NEXT:    push {r4, r5, r7, lr}
+; BASE-THUMB-NEXT:    .pad #8
+; BASE-THUMB-NEXT:    sub sp, #8
+; BASE-THUMB-NEXT:    uxth r5, r0
+; BASE-THUMB-NEXT:    mov r4, r1
+; BASE-THUMB-NEXT:    mov r0, r5
+; BASE-THUMB-NEXT:    mov r1, r5
+; BASE-THUMB-NEXT:    blx r4
+; BASE-THUMB-NEXT:    uxth r1, r0
+; BASE-THUMB-NEXT:    strh.w r0, [sp, #6]
+; BASE-THUMB-NEXT:    mov r0, r5
+; BASE-THUMB-NEXT:    blx r4
+; BASE-THUMB-NEXT:    ldrh.w r0, [sp, #6]
+; BASE-THUMB-NEXT:    add sp, #8
+; BASE-THUMB-NEXT:    pop {r4, r5, r7, pc}
+;
+; FULLFP16-ARM-LABEL: test_fncall_soft:
+; FULLFP16-ARM:       @ %bb.0:
+; FULLFP16-ARM-NEXT:    .save {r4, r5, r11, lr}
+; FULLFP16-ARM-NEXT:    push {r4, r5, r11, lr}
+; FULLFP16-ARM-NEXT:    .vsave {d8}
+; FULLFP16-ARM-NEXT:    vpush {d8}
+; FULLFP16-ARM-NEXT:    vmov.f16 s0, r0
+; FULLFP16-ARM-NEXT:    mov r4, r1
+; FULLFP16-ARM-NEXT:    vmov.f16 r5, s0
+; FULLFP16-ARM-NEXT:    mov r0, r5
+; FULLFP16-ARM-NEXT:    mov r1, r5
+; FULLFP16-ARM-NEXT:    blx r4
+; FULLFP16-ARM-NEXT:    vmov.f16 s16, r0
+; FULLFP16-ARM-NEXT:    mov r0, r5
+; FULLFP16-ARM-NEXT:    vmov.f16 r1, s16
+; FULLFP16-ARM-NEXT:    blx r4
+; FULLFP16-ARM-NEXT:    vmov r0, s16
+; FULLFP16-ARM-NEXT:    vpop {d8}
+; FULLFP16-ARM-NEXT:    pop {r4, r5, r11, pc}
+;
+; FULLFP16-THUMB-LABEL: test_fncall_soft:
+; FULLFP16-THUMB:       @ %bb.0:
+; FULLFP16-THUMB-NEXT:    .save {r4, r5, r7, lr}
+; FULLFP16-THUMB-NEXT:    push {r4, r5, r7, lr}
+; FULLFP16-THUMB-NEXT:    .vsave {d8}
+; FULLFP16-THUMB-NEXT:    vpush {d8}
+; FULLFP16-THUMB-NEXT:    vmov.f16 s0, r0
+; FULLFP16-THUMB-NEXT:    mov r4, r1
+; FULLFP16-THUMB-NEXT:    vmov.f16 r5, s0
+; FULLFP16-THUMB-NEXT:    mov r0, r5
+; FULLFP16-THUMB-NEXT:    mov r1, r5
+; FULLFP16-THUMB-NEXT:    blx r4
+; FULLFP16-THUMB-NEXT:    vmov.f16 s16, r0
+; FULLFP16-THUMB-NEXT:    mov r0, r5
+; FULLFP16-THUMB-NEXT:    vmov.f16 r1, s16
+; FULLFP16-THUMB-NEXT:    blx r4
+; FULLFP16-THUMB-NEXT:    vmov r0, s16
+; FULLFP16-THUMB-NEXT:    vpop {d8}
+; FULLFP16-THUMB-NEXT:    pop {r4, r5, r7, pc}
+  %call = tail call bfloat %f(bfloat %bf, bfloat %bf)
+  %call1 = tail call bfloat %f(bfloat %bf, bfloat %call)
+  ret bfloat %call
+}
+
+define arm_aapcs_vfpcc bfloat @test_fncall_hard(bfloat %bf, bfloat (bfloat, bfloat)* %f) {
+; BASE-ARM-LABEL: test_fncall_hard:
+; BASE-ARM:       @ %bb.0:
+; BASE-ARM-NEXT:    .save {r4, lr}
+; BASE-ARM-NEXT:    push {r4, lr}
+; BASE-ARM-NEXT:    .vsave {d8}
+; BASE-ARM-NEXT:    vpush {d8}
+; BASE-ARM-NEXT:    .pad #8
+; BASE-ARM-NEXT:    sub sp, sp, #8
+; BASE-ARM-NEXT:    mov r4, r0
+; BASE-ARM-NEXT:    vmov r0, s0
+; BASE-ARM-NEXT:    uxth r0, r0
+; BASE-ARM-NEXT:    vmov s16, r0
+; BASE-ARM-NEXT:    vmov.f32 s0, s16
+; BASE-ARM-NEXT:    vmov.f32 s1, s16
+; BASE-ARM-NEXT:    blx r4
+; BASE-ARM-NEXT:    vmov r0, s0
+; BASE-ARM-NEXT:    vmov.f32 s0, s16
+; BASE-ARM-NEXT:    strh r0, [sp, #6]
+; BASE-ARM-NEXT:    uxth r0, r0
+; BASE-ARM-NEXT:    vmov s1, r0
+; BASE-ARM-NEXT:    blx r4
+; BASE-ARM-NEXT:    ldrh r0, [sp, #6]
+; BASE-ARM-NEXT:    vmov s0, r0
+; BASE-ARM-NEXT:    add sp, sp, #8
+; BASE-ARM-NEXT:    vpop {d8}
+; BASE-ARM-NEXT:    pop {r4, pc}
+;
+; BASE-THUMB-LABEL: test_fncall_hard:
+; BASE-THUMB:       @ %bb.0:
+; BASE-THUMB-NEXT:    .save {r4, lr}
+; BASE-THUMB-NEXT:    push {r4, lr}
+; BASE-THUMB-NEXT:    .vsave {d8}
+; BASE-THUMB-NEXT:    vpush {d8}
+; BASE-THUMB-NEXT:    .pad #8
+; BASE-THUMB-NEXT:    sub sp, #8
+; BASE-THUMB-NEXT:    mov r4, r0
+; BASE-THUMB-NEXT:    vmov r0, s0
+; BASE-THUMB-NEXT:    uxth r0, r0
+; BASE-THUMB-NEXT:    vmov s16, r0
+; BASE-THUMB-NEXT:    vmov.f32 s0, s16
+; BASE-THUMB-NEXT:    vmov.f32 s1, s16
+; BASE-THUMB-NEXT:    blx r4
+; BASE-THUMB-NEXT:    vmov r0, s0
+; BASE-THUMB-NEXT:    vmov.f32 s0, s16
+; BASE-THUMB-NEXT:    strh.w r0, [sp, #6]
+; BASE-THUMB-NEXT:    uxth r0, r0
+; BASE-THUMB-NEXT:    vmov s1, r0
+; BASE-THUMB-NEXT:    blx r4
+; BASE-THUMB-NEXT:    ldrh.w r0, [sp, #6]
+; BASE-THUMB-NEXT:    vmov s0, r0
+; BASE-THUMB-NEXT:    add sp, #8
+; BASE-THUMB-NEXT:    vpop {d8}
+; BASE-THUMB-NEXT:    pop {r4, pc}
+;
+; FULLFP16-LABEL: test_fncall_hard:
+; FULLFP16:       @ %bb.0:
+; FULLFP16-NEXT:    .save {r4, lr}
+; FULLFP16-NEXT:    push {r4, lr}
+; FULLFP16-NEXT:    .vsave {d8, d9}
+; FULLFP16-NEXT:    vpush {d8, d9}
+; FULLFP16-NEXT:    mov r4, r0
+; FULLFP16-NEXT:    vmov.f16 r0, s0
+; FULLFP16-NEXT:    vmov s16, r0
+; FULLFP16-NEXT:    vmov.f32 s0, s16
+; FULLFP16-NEXT:    vmov.f32 s1, s16
+; FULLFP16-NEXT:    blx r4
+; FULLFP16-NEXT:    vmov.f16 r0, s0
+; FULLFP16-NEXT:    vmov.f32 s18, s0
+; FULLFP16-NEXT:    vmov.f32 s0, s16
+; FULLFP16-NEXT:    vmov s1, r0
+; FULLFP16-NEXT:    blx r4
+; FULLFP16-NEXT:    vmov.f32 s0, s18
+; FULLFP16-NEXT:    vpop {d8, d9}
+; FULLFP16-NEXT:    pop {r4, pc}
+  %call = tail call arm_aapcs_vfpcc bfloat %f(bfloat %bf, bfloat %bf)
+  %call1 = tail call arm_aapcs_vfpcc bfloat %f(bfloat %bf, bfloat %call)
+  ret bfloat %call
+}
diff --git a/llvm/test/CodeGen/ARM/arm-half-promote.ll b/llvm/test/CodeGen/ARM/arm-half-promote.ll
new file mode 100644
index 0000000000000..f3c9a9e081ba8
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/arm-half-promote.ll
@@ -0,0 +1,81 @@
+; RUN: llc < %s -mtriple=thumbv7s-apple-ios7.0.0 | FileCheck %s
+
+define arm_aapcs_vfpcc { <8 x half>, <8 x half> } @f1() {
+; CHECK-LABEL: _f1
+; CHECK: vpush   {d8}
+; CHECK-NEXT: vmov.f64        d8, #5.000000e-01
+; CHECK-NEXT: vmov.i32        d8, #0x0
+; CHECK-NEXT: vmov.i32        d0, #0x0
+; CHECK-NEXT: vmov.i32        d1, #0x0
+; CHECK-NEXT: vmov.i32        d2, #0x0
+; CHECK-NEXT: vmov.i32        d3, #0x0
+; CHECK-NEXT: vmov.i32        d4, #0x0
+; CHECK-NEXT: vmov.i32        d5, #0x0
+; CHECK-NEXT: vmov.i32        d6, #0x0
+; CHECK-NEXT: vmov.i32        d7, #0x0
+; CHECK-NEXT: vmov.f32        s1, s16
+; CHECK-NEXT: vmov.f32        s3, s16
+; CHECK-NEXT: vmov.f32        s5, s16
+; CHECK-NEXT: vmov.f32        s7, s16
+; CHECK-NEXT: vmov.f32        s9, s16
+; CHECK-NEXT: vmov.f32        s11, s16
+; CHECK-NEXT: vmov.f32        s13, s16
+; CHECK-NEXT: vmov.f32        s15, s16
+; CHECK-NEXT: vpop    {d8}
+; CHECK-NEXT: bx      lr
+  ret { <8 x half>, <8 x half> } zeroinitializer
+}
+
+define swiftcc { <8 x half>, <8 x half> } @f2() {
+; CHECK-LABEL: _f2
+; CHECK: vpush   {d8}
+; CHECK-NEXT: vmov.f64        d8, #5.000000e-01
+; CHECK-NEXT: vmov.i32        d8, #0x0
+; CHECK-NEXT: vmov.i32        d0, #0x0
+; CHECK-NEXT: vmov.i32        d1, #0x0
+; CHECK-NEXT: vmov.i32        d2, #0x0
+; CHECK-NEXT: vmov.i32        d3, #0x0
+; CHECK-NEXT: vmov.i32        d4, #0x0
+; CHECK-NEXT: vmov.i32        d5, #0x0
+; CHECK-NEXT: vmov.i32        d6, #0x0
+; CHECK-NEXT: vmov.i32        d7, #0x0
+; CHECK-NEXT: vmov.f32        s1, s16
+; CHECK-NEXT: vmov.f32        s3, s16
+; CHECK-NEXT: vmov.f32        s5, s16
+; CHECK-NEXT: vmov.f32        s7, s16
+; CHECK-NEXT: vmov.f32        s9, s16
+; CHECK-NEXT: vmov.f32        s11, s16
+; CHECK-NEXT: vmov.f32        s13, s16
+; CHECK-NEXT: vmov.f32        s15, s16
+; CHECK-NEXT: vpop    {d8}
+; CHECK-NEXT: bx      lr
+  ret { <8 x half>, <8 x half> } zeroinitializer
+}
+
+define fastcc { <8 x half>, <8 x half> } @f3() {
+; CHECK-LABEL: _f3
+; CHECK: vpush   {d8}
+; CHECK-NEXT: vmov.f64        d8, #5.000000e-01
+; CHECK-NEXT: vmov.i32        d8, #0x0
+; CHECK-NEXT: vmov.i32        d0, #0x0
+; CHECK-NEXT: vmov.i32        d1, #0x0
+; CHECK-NEXT: vmov.i32        d2, #0x0
+; CHECK-NEXT: vmov.i32        d3, #0x0
+; CHECK-NEXT: vmov.i32        d4, #0x0
+; CHECK-NEXT: vmov.i32        d5, #0x0
+; CHECK-NEXT: vmov.i32        d6, #0x0
+; CHECK-NEXT: vmov.i32        d7, #0x0
+; CHECK-NEXT: vmov.f32        s1, s16
+; CHECK-NEXT: vmov.f32        s3, s16
+; CHECK-NEXT: vmov.f32        s5, s16
+; CHECK-NEXT: vmov.f32        s7, s16
+; CHECK-NEXT: vmov.f32        s9, s16
+; CHECK-NEXT: vmov.f32        s11, s16
+; CHECK-NEXT: vmov.f32        s13, s16
+; CHECK-NEXT: vmov.f32        s15, s16
+; CHECK-NEXT: vpop    {d8}
+; CHECK-NEXT: bx      lr
+
+  ret { <8 x half>, <8 x half> } zeroinitializer
+}
+
diff --git a/llvm/test/CodeGen/ARM/bf16-convert-intrinsics.ll b/llvm/test/CodeGen/ARM/bf16-convert-intrinsics.ll
new file mode 100644
index 0000000000000..ea1966da77273
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/bf16-convert-intrinsics.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -verify-machineinstrs -mtriple=armv8.6a-arm-none-eabi -mattr=+neon,+bf16,+fullfp16 | FileCheck %s
+
+declare bfloat @llvm.arm.neon.vcvtbfp2bf(float)
+
+; Hard float ABI
+declare <4 x bfloat> @llvm.arm.neon.vcvtfp2bf.v4bf16(<4 x float>)
+
+define arm_aapcs_vfpcc <4 x bfloat> @test_vcvt_bf16_f32_hardfp(<4 x float> %a) {
+; CHECK-LABEL: test_vcvt_bf16_f32_hardfp:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcvt.bf16.f32 d0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %vcvtfp2bf1.i.i = call <4 x bfloat> @llvm.arm.neon.vcvtfp2bf.v4bf16(<4 x float> %a)
+  ret <4 x bfloat> %vcvtfp2bf1.i.i
+}
+
+define arm_aapcs_vfpcc bfloat @test_vcvth_bf16_f32_hardfp(float %a) {
+; CHECK-LABEL: test_vcvth_bf16_f32_hardfp:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcvtb.bf16.f32 s0, s0
+; CHECK-NEXT:    bx lr
+entry:
+  %vcvtbfp2bf.i = call bfloat @llvm.arm.neon.vcvtbfp2bf(float %a)
+  ret bfloat %vcvtbfp2bf.i
+}
+
+; Soft float ABI
+declare <4 x i16> @llvm.arm.neon.vcvtfp2bf.v4i16(<4 x float>)
+
+define <2 x i32> @test_vcvt_bf16_f32_softfp(<4 x float> %a) {
+; CHECK-LABEL: test_vcvt_bf16_f32_softfp:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov d17, r2, r3
+; CHECK-NEXT:    vmov d16, r0, r1
+; CHECK-NEXT:    vcvt.bf16.f32 d16, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    bx lr
+entry:
+  %vcvtfp2bf1.i.i = call <4 x i16> @llvm.arm.neon.vcvtfp2bf.v4i16(<4 x float> %a)
+  %.cast = bitcast <4 x i16> %vcvtfp2bf1.i.i to <2 x i32>
+  ret <2 x i32> %.cast
+}
+
+define bfloat @test_vcvth_bf16_f32_softfp(float %a) #1 {
+; CHECK-LABEL: test_vcvth_bf16_f32_softfp:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vcvtb.bf16.f32 s0, s0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    bx lr
+entry:
+  %vcvtbfp2bf.i = call bfloat @llvm.arm.neon.vcvtbfp2bf(float %a) #3
+  ret bfloat %vcvtbfp2bf.i
+}
diff --git a/llvm/test/CodeGen/ARM/cmov_fp16.ll b/llvm/test/CodeGen/ARM/cmov_fp16.ll
index 2c368d1a41bc6..805955d3e8306 100644
--- a/llvm/test/CodeGen/ARM/cmov_fp16.ll
+++ b/llvm/test/CodeGen/ARM/cmov_fp16.ll
@@ -1,14 +1,15 @@
-; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+fullfp16 %s -o - | FileCheck %s --check-prefixes CHECK-THUMB,CHECK
-; RUN: llc -mtriple=armv8.2a-arm-none-eabi -mattr=+fullfp16 %s -o - | FileCheck %s --check-prefixes CHECK-ARM,CHECK
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+fullfp16 %s -o - | FileCheck %s --check-prefixes CHECK,CHECK-THUMB
+; RUN: llc -mtriple=armv8.2a-arm-none-eabi -mattr=+fullfp16 %s -o - | FileCheck %s --check-prefixes CHECK,CHECK-ARM
 
 define i32 @test_ne(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-LABEL: test_ne:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov s2, r0
-; CHECK-NEXT:    cmp r2, r3
 ; CHECK-NEXT:    vmov s0, r1
-; CHECK-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-NEXT:    cmp r2, r3
+; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-NEXT:    vcvt.f16.u32 s2, s2
 ; CHECK-NEXT:    vseleq.f16 s0, s0, s2
 ; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    bx lr
@@ -25,11 +26,11 @@ entry:
 define i32 @test_eq(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-LABEL: test_eq:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov s2, r1
-; CHECK-NEXT:    cmp r2, r3
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-NEXT:    cmp r2, r3
+; CHECK-NEXT:    vmov s2, r1
 ; CHECK-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-NEXT:    vcvt.f16.u32 s2, s2
 ; CHECK-NEXT:    vseleq.f16 s0, s0, s2
 ; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    bx lr
@@ -46,11 +47,11 @@ entry:
 define i32 @test_gt(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-LABEL: test_gt:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov s2, r1
-; CHECK-NEXT:    cmp r2, r3
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-NEXT:    cmp r2, r3
+; CHECK-NEXT:    vmov s2, r1
 ; CHECK-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-NEXT:    vcvt.f16.u32 s2, s2
 ; CHECK-NEXT:    vselgt.f16 s0, s0, s2
 ; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    bx lr
@@ -67,11 +68,11 @@ entry:
 define i32 @test_ge(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-LABEL: test_ge:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov s2, r1
-; CHECK-NEXT:    cmp r2, r3
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-NEXT:    cmp r2, r3
+; CHECK-NEXT:    vmov s2, r1
 ; CHECK-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-NEXT:    vcvt.f16.u32 s2, s2
 ; CHECK-NEXT:    vselge.f16 s0, s0, s2
 ; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    bx lr
@@ -88,11 +89,11 @@ entry:
 define i32 @test_lt(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-LABEL: test_lt:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov s2, r0
-; CHECK-NEXT:    cmp r2, r3
 ; CHECK-NEXT:    vmov s0, r1
-; CHECK-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-NEXT:    cmp r2, r3
+; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-NEXT:    vcvt.f16.u32 s2, s2
 ; CHECK-NEXT:    vselge.f16 s0, s0, s2
 ; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    bx lr
@@ -109,11 +110,11 @@ entry:
 define i32 @test_le(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-LABEL: test_le:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov s2, r0
-; CHECK-NEXT:    cmp r2, r3
 ; CHECK-NEXT:    vmov s0, r1
-; CHECK-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-NEXT:    cmp r2, r3
+; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-NEXT:    vcvt.f16.u32 s2, s2
 ; CHECK-NEXT:    vselgt.f16 s0, s0, s2
 ; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    bx lr
@@ -130,11 +131,11 @@ entry:
 define i32 @test_hi(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-THUMB-LABEL: test_hi:
 ; CHECK-THUMB:       @ %bb.0: @ %entry
-; CHECK-THUMB-NEXT:    vmov s2, r0
-; CHECK-THUMB-NEXT:    cmp r2, r3
 ; CHECK-THUMB-NEXT:    vmov s0, r1
-; CHECK-THUMB-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-THUMB-NEXT:    cmp r2, r3
+; CHECK-THUMB-NEXT:    vmov s2, r0
 ; CHECK-THUMB-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-THUMB-NEXT:    vcvt.f16.u32 s2, s2
 ; CHECK-THUMB-NEXT:    it hi
 ; CHECK-THUMB-NEXT:    vmovhi.f32 s0, s2
 ; CHECK-THUMB-NEXT:    vmov.f16 r0, s0
@@ -142,11 +143,11 @@ define i32 @test_hi(i32 %x, i32 %y, i32 %a, i32 %b) {
 ;
 ; CHECK-ARM-LABEL: test_hi:
 ; CHECK-ARM:       @ %bb.0: @ %entry
-; CHECK-ARM-NEXT:    vmov s2, r0
-; CHECK-ARM-NEXT:    cmp r2, r3
 ; CHECK-ARM-NEXT:    vmov s0, r1
-; CHECK-ARM-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-ARM-NEXT:    cmp r2, r3
+; CHECK-ARM-NEXT:    vmov s2, r0
 ; CHECK-ARM-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-ARM-NEXT:    vcvt.f16.u32 s2, s2
 ; CHECK-ARM-NEXT:    vmovhi.f32 s0, s2
 ; CHECK-ARM-NEXT:    vmov.f16 r0, s0
 ; CHECK-ARM-NEXT:    bx lr
@@ -163,11 +164,11 @@ entry:
 define i32 @test_hs(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-THUMB-LABEL: test_hs:
 ; CHECK-THUMB:       @ %bb.0: @ %entry
-; CHECK-THUMB-NEXT:    vmov s2, r0
-; CHECK-THUMB-NEXT:    cmp r2, r3
 ; CHECK-THUMB-NEXT:    vmov s0, r1
-; CHECK-THUMB-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-THUMB-NEXT:    cmp r2, r3
+; CHECK-THUMB-NEXT:    vmov s2, r0
 ; CHECK-THUMB-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-THUMB-NEXT:    vcvt.f16.u32 s2, s2
 ; CHECK-THUMB-NEXT:    it hs
 ; CHECK-THUMB-NEXT:    vmovhs.f32 s0, s2
 ; CHECK-THUMB-NEXT:    vmov.f16 r0, s0
@@ -175,11 +176,11 @@ define i32 @test_hs(i32 %x, i32 %y, i32 %a, i32 %b) {
 ;
 ; CHECK-ARM-LABEL: test_hs:
 ; CHECK-ARM:       @ %bb.0: @ %entry
-; CHECK-ARM-NEXT:    vmov s2, r0
-; CHECK-ARM-NEXT:    cmp r2, r3
 ; CHECK-ARM-NEXT:    vmov s0, r1
-; CHECK-ARM-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-ARM-NEXT:    cmp r2, r3
+; CHECK-ARM-NEXT:    vmov s2, r0
 ; CHECK-ARM-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-ARM-NEXT:    vcvt.f16.u32 s2, s2
 ; CHECK-ARM-NEXT:    vmovhs.f32 s0, s2
 ; CHECK-ARM-NEXT:    vmov.f16 r0, s0
 ; CHECK-ARM-NEXT:    bx lr
@@ -196,11 +197,11 @@ entry:
 define i32 @test_lo(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-THUMB-LABEL: test_lo:
 ; CHECK-THUMB:       @ %bb.0: @ %entry
-; CHECK-THUMB-NEXT:    vmov s2, r0
-; CHECK-THUMB-NEXT:    cmp r2, r3
 ; CHECK-THUMB-NEXT:    vmov s0, r1
-; CHECK-THUMB-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-THUMB-NEXT:    cmp r2, r3
+; CHECK-THUMB-NEXT:    vmov s2, r0
 ; CHECK-THUMB-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-THUMB-NEXT:    vcvt.f16.u32 s2, s2
 ; CHECK-THUMB-NEXT:    it lo
 ; CHECK-THUMB-NEXT:    vmovlo.f32 s0, s2
 ; CHECK-THUMB-NEXT:    vmov.f16 r0, s0
@@ -208,11 +209,11 @@ define i32 @test_lo(i32 %x, i32 %y, i32 %a, i32 %b) {
 ;
 ; CHECK-ARM-LABEL: test_lo:
 ; CHECK-ARM:       @ %bb.0: @ %entry
-; CHECK-ARM-NEXT:    vmov s2, r0
-; CHECK-ARM-NEXT:    cmp r2, r3
 ; CHECK-ARM-NEXT:    vmov s0, r1
-; CHECK-ARM-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-ARM-NEXT:    cmp r2, r3
+; CHECK-ARM-NEXT:    vmov s2, r0
 ; CHECK-ARM-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-ARM-NEXT:    vcvt.f16.u32 s2, s2
 ; CHECK-ARM-NEXT:    vmovlo.f32 s0, s2
 ; CHECK-ARM-NEXT:    vmov.f16 r0, s0
 ; CHECK-ARM-NEXT:    bx lr
@@ -229,11 +230,11 @@ entry:
 define i32 @test_ls(i32 %x, i32 %y, i32 %a, i32 %b) {
 ; CHECK-THUMB-LABEL: test_ls:
 ; CHECK-THUMB:       @ %bb.0: @ %entry
-; CHECK-THUMB-NEXT:    vmov s2, r0
-; CHECK-THUMB-NEXT:    cmp r2, r3
 ; CHECK-THUMB-NEXT:    vmov s0, r1
-; CHECK-THUMB-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-THUMB-NEXT:    cmp r2, r3
+; CHECK-THUMB-NEXT:    vmov s2, r0
 ; CHECK-THUMB-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-THUMB-NEXT:    vcvt.f16.u32 s2, s2
 ; CHECK-THUMB-NEXT:    it ls
 ; CHECK-THUMB-NEXT:    vmovls.f32 s0, s2
 ; CHECK-THUMB-NEXT:    vmov.f16 r0, s0
@@ -241,11 +242,11 @@ define i32 @test_ls(i32 %x, i32 %y, i32 %a, i32 %b) {
 ;
 ; CHECK-ARM-LABEL: test_ls:
 ; CHECK-ARM:       @ %bb.0: @ %entry
-; CHECK-ARM-NEXT:    vmov s2, r0
-; CHECK-ARM-NEXT:    cmp r2, r3
 ; CHECK-ARM-NEXT:    vmov s0, r1
-; CHECK-ARM-NEXT:    vcvt.f16.u32 s2, s2
+; CHECK-ARM-NEXT:    cmp r2, r3
+; CHECK-ARM-NEXT:    vmov s2, r0
 ; CHECK-ARM-NEXT:    vcvt.f16.u32 s0, s0
+; CHECK-ARM-NEXT:    vcvt.f16.u32 s2, s2
 ; CHECK-ARM-NEXT:    vmovls.f32 s0, s2
 ; CHECK-ARM-NEXT:    vmov.f16 r0, s0
 ; CHECK-ARM-NEXT:    bx lr
diff --git a/llvm/test/CodeGen/ARM/cmse-vlldm-no-reorder.ll b/llvm/test/CodeGen/ARM/cmse-vlldm-no-reorder.ll
new file mode 100644
index 0000000000000..dc986c4b30a05
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/cmse-vlldm-no-reorder.ll
@@ -0,0 +1,21 @@
+; RUN: llc -mtriple=thumbv8m.main -mcpu=cortex-m33 --float-abi=hard %s -o - | \
+; RUN:   FileCheck %s
+
+@g = hidden local_unnamed_addr global float (...)* null, align 4
+@a = hidden local_unnamed_addr global float 0.000000e+00, align 4
+
+define hidden void @f() local_unnamed_addr #0 {
+entry:
+  %0 = load float ()*, float ()** bitcast (float (...)** @g to float ()**), align 4
+  %call = tail call nnan ninf nsz float %0() #1
+  store float %call, float* @a, align 4
+  ret void
+}
+
+; CHECK: blxns r{{[0-9]+}}
+; CHECK: vmov  r[[T:[0-9]+]], s0
+; CHECK: vlldm sp
+; CHECK: vmov  s0, r[[T]]
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind "cmse_nonsecure_call" }
diff --git a/llvm/test/CodeGen/ARM/cmse-vlldm-no-reorder.mir b/llvm/test/CodeGen/ARM/cmse-vlldm-no-reorder.mir
new file mode 100644
index 0000000000000..571ec0cec86cf
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/cmse-vlldm-no-reorder.mir
@@ -0,0 +1,112 @@
+# RUN: llc -mtriple=thumbv8m.main -mcpu=cortex-m33 --float-abi=hard --run-pass=arm-pseudo %s -o - | \
+# RUN: FileCheck %s
+--- |
+  ; ModuleID = 'cmse-vlldm-no-reorder.ll'
+  source_filename = "cmse-vlldm-no-reorder.ll"
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8m.main"
+  
+  @g = hidden local_unnamed_addr global float (...)* null, align 4
+  @a = hidden local_unnamed_addr global float 0.000000e+00, align 4
+  
+  ; Function Attrs: nounwind
+  define hidden void @f() local_unnamed_addr #0 {
+  entry:
+    %0 = load float ()*, float ()** bitcast (float (...)** @g to float ()**), align 4
+    %call = tail call nnan ninf nsz float %0() #2
+    store float %call, float* @a, align 4
+    ret void
+  }
+  
+  ; Function Attrs: nounwind
+  declare void @llvm.stackprotector(i8*, i8**) #1
+  
+  attributes #0 = { nounwind "target-cpu"="cortex-m33" }
+  attributes #1 = { nounwind }
+  attributes #2 = { nounwind "cmse_nonsecure_call" }
+
+...
+---
+name:            f
+alignment:       2
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:       []
+liveins:         []
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       8
+  offsetAdjustment: 0
+  maxAlignment:    4
+  adjustsStack:    true
+  hasCalls:        true
+  stackProtector:  ''
+  maxCallFrameSize: 0
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4, 
+      stack-id: default, callee-saved-register: '$lr', callee-saved-restored: false, 
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: -8, size: 4, alignment: 4, 
+      stack-id: default, callee-saved-register: '$r7', callee-saved-restored: true, 
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+callSites:       []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $r7, $lr
+  
+    $sp = frame-setup t2STMDB_UPD $sp, 14 /* CC::al */, $noreg, killed $r7, killed $lr
+    frame-setup CFI_INSTRUCTION def_cfa_offset 8
+    frame-setup CFI_INSTRUCTION offset $lr, -4
+    frame-setup CFI_INSTRUCTION offset $r7, -8
+    renamable $r0 = t2MOVi32imm @g
+    renamable $r0 = t2LDRi12 killed renamable $r0, 0, 14 /* CC::al */, $noreg :: (dereferenceable load 4 from `float ()** bitcast (float (...)** @g to float ()**)`)
+    tBLXNS_CALL killed renamable $r0, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $s0
+    renamable $r0 = t2MOVi32imm @a
+    VSTRS killed renamable $s0, killed renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4 into @a)
+    $sp = frame-destroy t2LDMIA_RET $sp, 14 /* CC::al */, $noreg, def $r7, def $pc
+
+...
+
+# CHECK-LABEL: bb.0.entry:
+# CHECK: $sp = t2STMDB_UPD $sp, 14 /* CC::al */, $noreg, $r4, $r5, $r6, undef $r7, $r8, $r9, $r10, $r11
+# CHECK-NEXT:  $r0 = t2BICri $r0, 1, 14 /* CC::al */, $noreg, $noreg
+# CHECK-NEXT:  $sp = tSUBspi $sp, 34, 14 /* CC::al */, $noreg
+# CHECK-NEXT:  VLSTM $sp, 14 /* CC::al */, $noreg, implicit undef $vpr, implicit undef $fpscr, implicit undef $fpscr_nzcv, implicit undef $q0, implicit undef $q1, implicit undef $q2, implicit undef $q3, implicit undef $q4, implicit undef $q5, implicit undef $q6, implicit undef $q7
+# CHECK-NEXT:  $r1 = tMOVr $r0, 14 /* CC::al */, $noreg
+# CHECK-NEXT:  $r2 = tMOVr $r0, 14 /* CC::al */, $noreg
+# CHECK-NEXT:  $r3 = tMOVr $r0, 14 /* CC::al */, $noreg
+# CHECK-NEXT:  $r4 = tMOVr $r0, 14 /* CC::al */, $noreg
+# CHECK-NEXT:  $r5 = tMOVr $r0, 14 /* CC::al */, $noreg
+# CHECK-NEXT:  $r6 = tMOVr $r0, 14 /* CC::al */, $noreg
+# CHECK-NEXT:  $r7 = tMOVr $r0, 14 /* CC::al */, $noreg
+# CHECK-NEXT:  $r8 = tMOVr $r0, 14 /* CC::al */, $noreg
+# CHECK-NEXT:  $r9 = tMOVr $r0, 14 /* CC::al */, $noreg
+# CHECK-NEXT:  $r10 = tMOVr $r0, 14 /* CC::al */, $noreg
+# CHECK-NEXT:  $r11 = tMOVr $r0, 14 /* CC::al */, $noreg
+# CHECK-NEXT:  $r12 = tMOVr $r0, 14 /* CC::al */, $noreg
+# CHECK-NEXT:  t2MSR_M 3072, $r0, 14 /* CC::al */, $noreg, implicit-def $cpsr
+# CHECK-NEXT:  tBLXNSr 14 /* CC::al */, $noreg, killed $r0, csr_aapcs, implicit-def $lr, implicit $sp, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $s0
+# CHECK-NEXT:  $r12 = VMOVRS $s0, 14 /* CC::al */, $noreg
+# CHECK-NEXT:  VLLDM $sp, 14 /* CC::al */, $noreg, implicit-def $q0, implicit-def $q1, implicit-def $q2, implicit-def $q3, implicit-def $q4, implicit-def $q5, implicit-def $q6, implicit-def $q7, implicit-def $vpr, implicit-def $fpscr, implicit-def $fpscr_nzcv
+# CHECK-NEXT:  $s0 = VMOVSR $r12, 14 /* CC::al */, $noreg
+# CHECK-NEXT:  $sp = tADDspi $sp, 34, 14 /* CC::al */, $noreg
+# CHECK-NEXT:  $sp = t2LDMIA_UPD $sp, 14 /* CC::al */, $noreg, def $r4, def $r5, def $r6, def $r7, def $r8, def $r9, def $r10, def $r11
+ 
\ No newline at end of file
diff --git a/llvm/test/CodeGen/ARM/dbg-tcreturn.ll b/llvm/test/CodeGen/ARM/dbg-tcreturn.ll
new file mode 100644
index 0000000000000..37ec4e3d92ee4
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/dbg-tcreturn.ll
@@ -0,0 +1,47 @@
+; XFAIL: *
+; RUN: llc %s -o - -stop-after=finalize-isel -verify-machineinstr | FileCheck %s
+
+target datalayout = "e-m:o-p:32:32-Fi8-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
+target triple = "thumbv7-apple-ios7.0.0"
+
+; CHECK-LABEL: name: test
+; CHECK:       body:
+; CHECK-NEXT:   bb.0.entry:
+; CHECK-NEXT:     liveins: $r0, $r1
+; CHECK:          %1:gpr = COPY $r1
+; CHECK-NEXT:     %0:gpr = COPY $r0
+; CHECK-NEXT:     $r0 = COPY %0
+; CHECK-NEXT:     $r1 = COPY %1
+; CHECK-NEXT:     TCRETURNdi &__divsi3, implicit $sp, implicit $r0, implicit $r1
+; CHECK-NEXT:     DBG_VALUE $noreg, $noreg, !13, !DIExpression(), debug-location !16
+
+define i32 @test(i32 %a1, i32 %a2) !dbg !5 {
+entry:
+  %res = sdiv i32 %a1, %a2
+  call void @llvm.dbg.value(metadata i32 %res, metadata !13, metadata !DIExpression()), !dbg !16
+  ret i32 %res
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_Swift, file: !1, producer: "Swift", isOptimized: true, runtimeVersion: 5, emissionKind: FullDebug)
+!1 = !DIFile(filename: "foo.swift", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"Swift Minor Version", i8 3}
+!5 = distinct !DISubprogram(name: "n0", linkageName: "n1", scope: !7, file: !6, line: 86, type: !8, scopeLine: 86, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!6 = !DIFile(filename: "bar.swift", directory: "")
+!7 = !DIModule(scope: null, name: "Swift")
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !12}
+!10 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Int", scope: !7, file: !11, size: 32, elements: !2, runtimeLang: DW_LANG_Swift, identifier: "$i1")
+!11 = !DIFile(filename: "f1.swift", directory: "")
+!12 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "n2", scope: !7, file: !6, size: 32, elements: !2, runtimeLang: DW_LANG_Swift, identifier: "n3")
+!13 = !DILocalVariable(name: "n4", scope: !14, file: !1, line: 89, type: !15)
+!14 = distinct !DILexicalBlock(scope: !5, file: !6, line: 86, column: 34)
+!15 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !10)
+!16 = !DILocation(line: 89, column: 9, scope: !14)
diff --git a/llvm/test/CodeGen/ARM/fp16-args.ll b/llvm/test/CodeGen/ARM/fp16-args.ll
index e858661d32fb8..7ed1e883eef19 100644
--- a/llvm/test/CodeGen/ARM/fp16-args.ll
+++ b/llvm/test/CodeGen/ARM/fp16-args.ll
@@ -11,10 +11,10 @@
 define half @foo(half %a, half %b) {
 ; SOFT-LABEL: foo:
 ; SOFT:       @ %bb.0: @ %entry
-; SOFT-NEXT:    vmov s2, r1
 ; SOFT-NEXT:    vmov s0, r0
-; SOFT-NEXT:    vcvtb.f32.f16 s2, s2
+; SOFT-NEXT:    vmov s2, r1
 ; SOFT-NEXT:    vcvtb.f32.f16 s0, s0
+; SOFT-NEXT:    vcvtb.f32.f16 s2, s2
 ; SOFT-NEXT:    vadd.f32 s0, s0, s2
 ; SOFT-NEXT:    vcvtb.f16.f32 s0, s0
 ; SOFT-NEXT:    vmov r0, s0
diff --git a/llvm/test/CodeGen/ARM/fp16-bitcast.ll b/llvm/test/CodeGen/ARM/fp16-bitcast.ll
index 4254f2ecef441..d26c2d96614a4 100644
--- a/llvm/test/CodeGen/ARM/fp16-bitcast.ll
+++ b/llvm/test/CodeGen/ARM/fp16-bitcast.ll
@@ -40,10 +40,10 @@ entry:
 define half @addf16(half %a, half %b) {
 ; CHECK-VFPV4-SOFT-LABEL: addf16:
 ; CHECK-VFPV4-SOFT:       @ %bb.0: @ %entry
-; CHECK-VFPV4-SOFT-NEXT:    vmov s2, r1
 ; CHECK-VFPV4-SOFT-NEXT:    vmov s0, r0
-; CHECK-VFPV4-SOFT-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-VFPV4-SOFT-NEXT:    vmov s2, r1
 ; CHECK-VFPV4-SOFT-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-VFPV4-SOFT-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-VFPV4-SOFT-NEXT:    vadd.f32 s0, s0, s2
 ; CHECK-VFPV4-SOFT-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-VFPV4-SOFT-NEXT:    vmov r0, s0
diff --git a/llvm/test/CodeGen/ARM/fp16-fullfp16.ll b/llvm/test/CodeGen/ARM/fp16-fullfp16.ll
index e912cee36246d..86a8caa76aa71 100644
--- a/llvm/test/CodeGen/ARM/fp16-fullfp16.ll
+++ b/llvm/test/CodeGen/ARM/fp16-fullfp16.ll
@@ -61,8 +61,8 @@ define arm_aapcs_vfpcc void @test_frem(half* %p, half* %q) {
 ; CHECK-LABEL: test_frem:
 ; CHECK:         .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    vldr.16 s2, [r1]
 ; CHECK-NEXT:    vldr.16 s0, [r0]
+; CHECK-NEXT:    vldr.16 s2, [r1]
 ; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s1, s2
@@ -285,8 +285,8 @@ define void @test_pow(half* %p, half* %q) {
 ; CHECK-LABEL: test_pow:
 ; CHECK:         .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    vldr.16 s2, [r1]
 ; CHECK-NEXT:    vldr.16 s0, [r0]
+; CHECK-NEXT:    vldr.16 s2, [r1]
 ; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s1, s2
diff --git a/llvm/test/CodeGen/ARM/fp16-insert-extract.ll b/llvm/test/CodeGen/ARM/fp16-insert-extract.ll
index 0079159722cb5..698df3aa40b32 100644
--- a/llvm/test/CodeGen/ARM/fp16-insert-extract.ll
+++ b/llvm/test/CodeGen/ARM/fp16-insert-extract.ll
@@ -5,15 +5,13 @@
 define float @test_vget_lane_f16_1(<4 x half> %a) nounwind {
 ; CHECKHARD-LABEL: test_vget_lane_f16_1:
 ; CHECKHARD:       @ %bb.0: @ %entry
-; CHECKHARD-NEXT:    vmovx.f16 s0, s0
-; CHECKHARD-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECKHARD-NEXT:    vcvtt.f32.f16 s0, s0
 ; CHECKHARD-NEXT:    bx lr
 ;
 ; CHECKSOFT-LABEL: test_vget_lane_f16_1:
 ; CHECKSOFT:       @ %bb.0: @ %entry
 ; CHECKSOFT-NEXT:    vmov d0, r0, r1
-; CHECKSOFT-NEXT:    vmovx.f16 s0, s0
-; CHECKSOFT-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECKSOFT-NEXT:    vcvtt.f32.f16 s0, s0
 ; CHECKSOFT-NEXT:    vmov r0, s0
 ; CHECKSOFT-NEXT:    bx lr
 entry:
@@ -61,15 +59,13 @@ entry:
 define float @test_vget_laneq_f16_7(<8 x half> %a) nounwind {
 ; CHECKHARD-LABEL: test_vget_laneq_f16_7:
 ; CHECKHARD:       @ %bb.0: @ %entry
-; CHECKHARD-NEXT:    vmovx.f16 s0, s3
-; CHECKHARD-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECKHARD-NEXT:    vcvtt.f32.f16 s0, s3
 ; CHECKHARD-NEXT:    bx lr
 ;
 ; CHECKSOFT-LABEL: test_vget_laneq_f16_7:
 ; CHECKSOFT:       @ %bb.0: @ %entry
 ; CHECKSOFT-NEXT:    vmov d1, r2, r3
-; CHECKSOFT-NEXT:    vmovx.f16 s0, s3
-; CHECKSOFT-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECKSOFT-NEXT:    vcvtt.f32.f16 s0, s3
 ; CHECKSOFT-NEXT:    vmov r0, s0
 ; CHECKSOFT-NEXT:    bx lr
 entry:
@@ -81,19 +77,15 @@ entry:
 define <4 x half> @test_vset_lane_f16(<4 x half> %a, float %fb) nounwind {
 ; CHECKHARD-LABEL: test_vset_lane_f16:
 ; CHECKHARD:       @ %bb.0: @ %entry
-; CHECKHARD-NEXT:    vcvtb.f16.f32 s2, s2
-; CHECKHARD-NEXT:    vmov r0, s2
-; CHECKHARD-NEXT:    vmov.16 d0[3], r0
+; CHECKHARD-NEXT:    vcvtt.f16.f32 s1, s2
 ; CHECKHARD-NEXT:    bx lr
 ;
 ; CHECKSOFT-LABEL: test_vset_lane_f16:
 ; CHECKSOFT:       @ %bb.0: @ %entry
-; CHECKSOFT-NEXT:    vmov s0, r2
-; CHECKSOFT-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECKSOFT-NEXT:    vmov d16, r0, r1
-; CHECKSOFT-NEXT:    vmov r2, s0
-; CHECKSOFT-NEXT:    vmov.16 d16[3], r2
-; CHECKSOFT-NEXT:    vmov r0, r1, d16
+; CHECKSOFT-NEXT:    vmov d0, r0, r1
+; CHECKSOFT-NEXT:    vmov s2, r2
+; CHECKSOFT-NEXT:    vcvtt.f16.f32 s1, s2
+; CHECKSOFT-NEXT:    vmov r0, r1, d0
 ; CHECKSOFT-NEXT:    bx lr
 entry:
   %b = fptrunc float %fb to half
@@ -104,21 +96,17 @@ entry:
 define <8 x half> @test_vset_laneq_f16_1(<8 x half> %a, float %fb) nounwind {
 ; CHECKHARD-LABEL: test_vset_laneq_f16_1:
 ; CHECKHARD:       @ %bb.0: @ %entry
-; CHECKHARD-NEXT:    vcvtb.f16.f32 s4, s4
-; CHECKHARD-NEXT:    vmov r0, s4
-; CHECKHARD-NEXT:    vmov.16 d0[1], r0
+; CHECKHARD-NEXT:    vcvtt.f16.f32 s0, s4
 ; CHECKHARD-NEXT:    bx lr
 ;
 ; CHECKSOFT-LABEL: test_vset_laneq_f16_1:
 ; CHECKSOFT:       @ %bb.0: @ %entry
-; CHECKSOFT-NEXT:    vldr s0, [sp]
-; CHECKSOFT-NEXT:    vmov d17, r2, r3
-; CHECKSOFT-NEXT:    vmov d16, r0, r1
-; CHECKSOFT-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECKSOFT-NEXT:    vmov r12, s0
-; CHECKSOFT-NEXT:    vmov.16 d16[1], r12
-; CHECKSOFT-NEXT:    vmov r2, r3, d17
-; CHECKSOFT-NEXT:    vmov r0, r1, d16
+; CHECKSOFT-NEXT:    vmov d1, r2, r3
+; CHECKSOFT-NEXT:    vldr s4, [sp]
+; CHECKSOFT-NEXT:    vmov d0, r0, r1
+; CHECKSOFT-NEXT:    vcvtt.f16.f32 s0, s4
+; CHECKSOFT-NEXT:    vmov r2, r3, d1
+; CHECKSOFT-NEXT:    vmov r0, r1, d0
 ; CHECKSOFT-NEXT:    bx lr
 entry:
   %b = fptrunc float %fb to half
@@ -129,21 +117,17 @@ entry:
 define <8 x half> @test_vset_laneq_f16_7(<8 x half> %a, float %fb) nounwind {
 ; CHECKHARD-LABEL: test_vset_laneq_f16_7:
 ; CHECKHARD:       @ %bb.0: @ %entry
-; CHECKHARD-NEXT:    vcvtb.f16.f32 s4, s4
-; CHECKHARD-NEXT:    vmov r0, s4
-; CHECKHARD-NEXT:    vmov.16 d1[3], r0
+; CHECKHARD-NEXT:    vcvtt.f16.f32 s3, s4
 ; CHECKHARD-NEXT:    bx lr
 ;
 ; CHECKSOFT-LABEL: test_vset_laneq_f16_7:
 ; CHECKSOFT:       @ %bb.0: @ %entry
-; CHECKSOFT-NEXT:    vldr s0, [sp]
-; CHECKSOFT-NEXT:    vmov d17, r2, r3
-; CHECKSOFT-NEXT:    vmov d16, r0, r1
-; CHECKSOFT-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECKSOFT-NEXT:    vmov r12, s0
-; CHECKSOFT-NEXT:    vmov.16 d17[3], r12
-; CHECKSOFT-NEXT:    vmov r0, r1, d16
-; CHECKSOFT-NEXT:    vmov r2, r3, d17
+; CHECKSOFT-NEXT:    vmov d1, r2, r3
+; CHECKSOFT-NEXT:    vldr s4, [sp]
+; CHECKSOFT-NEXT:    vmov d0, r0, r1
+; CHECKSOFT-NEXT:    vcvtt.f16.f32 s3, s4
+; CHECKSOFT-NEXT:    vmov r0, r1, d0
+; CHECKSOFT-NEXT:    vmov r2, r3, d1
 ; CHECKSOFT-NEXT:    bx lr
 entry:
   %b = fptrunc float %fb to half
diff --git a/llvm/test/CodeGen/ARM/fp16-instructions.ll b/llvm/test/CodeGen/ARM/fp16-instructions.ll
index 260dd12b3e234..5e3e4469fcc09 100644
--- a/llvm/test/CodeGen/ARM/fp16-instructions.ll
+++ b/llvm/test/CodeGen/ARM/fp16-instructions.ll
@@ -72,10 +72,10 @@ entry:
 ; CHECK-SOFTFP-VFP3:  vadd.f32
 ; CHECK-SOFTFP-VFP3:  bl  __aeabi_f2h
 
-; CHECK-SOFTFP-FP16:  vmov          [[S2:s[0-9]]], r1
-; CHECK-SOFTFP-FP16:  vmov          [[S0:s[0-9]]], r0
-; CHECK-SOFTFP-FP16:  vcvtb.f32.f16 [[S2]], [[S2]]
-; CHECK-SOFTFP-FP16:  vcvtb.f32.f16 [[S0]], [[S0]]
+; CHECK-SOFTFP-FP16-DAG:  vmov          [[S0:s[0-9]]], r0
+; CHECK-SOFTFP-FP16-DAG:  vmov          [[S2:s[0-9]]], r1
+; CHECK-SOFTFP-FP16-DAG:  vcvtb.f32.f16 [[S0]], [[S0]]
+; CHECK-SOFTFP-FP16-DAG:  vcvtb.f32.f16 [[S2]], [[S2]]
 ; CHECK-SOFTFP-FP16:  vadd.f32      [[S0]], [[S0]], [[S2]]
 ; CHECK-SOFTFP-FP16:  vcvtb.f16.f32 [[S0]], [[S0]]
 ; CHECK-SOFTFP-FP16:  vmov  r0, s0
@@ -355,10 +355,10 @@ entry:
 ; CHECK-SOFTFP-VFP3:  vdiv.f32
 ; CHECK-SOFTFP-VFP3:  bl  __aeabi_f2h
 
-; CHECK-SOFTFP-FP16:  vmov          [[S2:s[0-9]]], r1
-; CHECK-SOFTFP-FP16:  vmov          [[S0:s[0-9]]], r0
-; CHECK-SOFTFP-FP16:  vcvtb.f32.f16 [[S2]], [[S2]]
-; CHECK-SOFTFP-FP16:  vcvtb.f32.f16 [[S0]], [[S0]]
+; CHECK-SOFTFP-FP16-DAG:  vmov          [[S0:s[0-9]]], r0
+; CHECK-SOFTFP-FP16-DAG:  vmov          [[S2:s[0-9]]], r1
+; CHECK-SOFTFP-FP16-DAG:  vcvtb.f32.f16 [[S0]], [[S0]]
+; CHECK-SOFTFP-FP16-DAG:  vcvtb.f32.f16 [[S2]], [[S2]]
 ; CHECK-SOFTFP-FP16:  vdiv.f32      [[S0]], [[S0]], [[S2]]
 ; CHECK-SOFTFP-FP16:  vcvtb.f16.f32 [[S0]], [[S0]]
 ; CHECK-SOFTFP-FP16:  vmov  r0, s0
@@ -577,10 +577,10 @@ entry:
 ; CHECK-SOFTFP-VFP3:  vmul.f32
 ; CHECK-SOFTFP-VFP3:  bl  __aeabi_f2h
 
-; CHECK-SOFTFP-FP16:  vmov          [[S2:s[0-9]]], r1
-; CHECK-SOFTFP-FP16:  vmov          [[S0:s[0-9]]], r0
-; CHECK-SOFTFP-FP16:  vcvtb.f32.f16 [[S2]], [[S2]]
-; CHECK-SOFTFP-FP16:  vcvtb.f32.f16 [[S0]], [[S0]]
+; CHECK-SOFTFP-FP16-DAG:  vmov          [[S0:s[0-9]]], r0
+; CHECK-SOFTFP-FP16-DAG:  vmov          [[S2:s[0-9]]], r1
+; CHECK-SOFTFP-FP16-DAG:  vcvtb.f32.f16 [[S0]], [[S0]]
+; CHECK-SOFTFP-FP16-DAG:  vcvtb.f32.f16 [[S2]], [[S2]]
 ; CHECK-SOFTFP-FP16:  vmul.f32      [[S0]], [[S0]], [[S2]]
 ; CHECK-SOFTFP-FP16:  vcvtb.f16.f32 [[S0]], [[S0]]
 ; CHECK-SOFTFP-FP16:  vmov  r0, s0
@@ -973,10 +973,10 @@ entry:
 ; CHECK-SOFTFP-VFP3:  vsub.f32
 ; CHECK-SOFTFP-VFP3:  bl  __aeabi_f2h
 
-; CHECK-SOFTFP-FP16:  vmov          [[S2:s[0-9]]], r1
-; CHECK-SOFTFP-FP16:  vmov          [[S0:s[0-9]]], r0
-; CHECK-SOFTFP-FP16:  vcvtb.f32.f16 [[S2]], [[S2]]
-; CHECK-SOFTFP-FP16:  vcvtb.f32.f16 [[S0]], [[S0]]
+; CHECK-SOFTFP-FP16-DAG:  vmov          [[S0:s[0-9]]], r0
+; CHECK-SOFTFP-FP16-DAG:  vmov          [[S2:s[0-9]]], r1
+; CHECK-SOFTFP-FP16-DAG:  vcvtb.f32.f16 [[S0]], [[S0]]
+; CHECK-SOFTFP-FP16-DAG:  vcvtb.f32.f16 [[S2]], [[S2]]
 ; CHECK-SOFTFP-FP16:  vsub.f32      [[S0]], [[S0]], [[S2]]
 ; CHECK-SOFTFP-FP16:  vcvtb.f16.f32 [[S0]], [[S0]]
 ; CHECK-SOFTFP-FP16:  vmov  r0, s0
diff --git a/llvm/test/CodeGen/ARM/ifcvt-diamond-unanalyzable-common.mir b/llvm/test/CodeGen/ARM/ifcvt-diamond-unanalyzable-common.mir
index 227544961b2ab..a69e70e2a6af7 100644
--- a/llvm/test/CodeGen/ARM/ifcvt-diamond-unanalyzable-common.mir
+++ b/llvm/test/CodeGen/ARM/ifcvt-diamond-unanalyzable-common.mir
@@ -1,6 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc %s -o - -run-pass=if-converter -verify-machineinstrs | FileCheck %s
-# Make sure we correctly if-convert blocks containing an INLINEASM_BR.
+# Make sure we correctly if-convert blocks containing an unanalyzable branch sequence.
+# (In this case, multiple conditional branches)
 
 --- |
   target triple = "thumbv7-unknown-linux-gnueabi"
@@ -26,10 +27,12 @@ body:             |
   ; CHECK:   $r0 = t2MOVi 2, 1 /* CC::ne */, $cpsr, $noreg
   ; CHECK:   $r0 = t2MOVi 3, 0 /* CC::eq */, killed $cpsr, $noreg, implicit killed $r0
   ; CHECK:   tBL 14 /* CC::al */, $noreg, @fn2, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $r0, implicit killed $r1, implicit-def $sp, implicit-def dead $r0
-  ; CHECK:   INLINEASM_BR &"", 9 /* sideeffect mayload attdialect */, 13 /* imm */, 0, 13 /* imm */, blockaddress(@fn1, %ir-block.l_yes)
+  ; CHECK:   t2CMPri $sp, 34, 14 /* CC::al */, $noreg, implicit-def $cpsr
+  ; CHECK:   t2Bcc %bb.2, 1 /* CC::ne */, $cpsr
+  ; CHECK:   t2Bcc %bb.2, 2 /* CC::hs */, killed $cpsr
   ; CHECK:   t2B %bb.1, 14 /* CC::al */, $noreg
   ; CHECK: bb.1:
-  ; CHECK:   INLINEASM &"", 1 /* sideeffect attdialect */
+  ; CHECK:   INLINEASM &"", 1
   ; CHECK:   $sp = t2LDMIA_RET $sp, 14 /* CC::al */, $noreg, def $r4, def $pc
   ; CHECK: bb.2.l_yes (address-taken):
   ; CHECK:   $sp = t2LDMIA_RET $sp, 14 /* CC::al */, $noreg, def $r4, def $pc
@@ -47,7 +50,9 @@ body:             |
 
     $r0 = t2MOVi 3, 14, $noreg, $noreg
     tBL 14, $noreg, @fn2, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit $r0, implicit $r1, implicit-def $sp, implicit-def dead $r0
-    INLINEASM_BR &"", 9, 13, 0, 13, blockaddress(@fn1, %ir-block.l_yes)
+    t2CMPri $sp, 34, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.4, 1, $cpsr
+    t2Bcc %bb.4, 2, killed $cpsr
     t2B %bb.3, 14, $noreg
 
   bb.2:
@@ -56,7 +61,9 @@ body:             |
 
     $r0 = t2MOVi 2, 14, $noreg, $noreg
     tBL 14, $noreg, @fn2, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit $r0, implicit $r1, implicit-def $sp, implicit-def dead $r0
-    INLINEASM_BR &"", 9, 13, 0, 13, blockaddress(@fn1, %ir-block.l_yes)
+    t2CMPri $sp, 34, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.4, 1, $cpsr
+    t2Bcc %bb.4, 2, killed $cpsr
     t2B %bb.3, 14, $noreg
 
   bb.3:
diff --git a/llvm/test/CodeGen/ARM/ifcvt-size.mir b/llvm/test/CodeGen/ARM/ifcvt-size.mir
index 28391ba8079a7..9730b654156d4 100644
--- a/llvm/test/CodeGen/ARM/ifcvt-size.mir
+++ b/llvm/test/CodeGen/ARM/ifcvt-size.mir
@@ -525,32 +525,34 @@ tracksRegLiveness: true
 # CHECK-NEXT: INLINEASM_BR
 
 # DEBUG-LABEL: Ifcvt: function ({{[0-9]+}}) 'fn9'
-# DEBUG: MeetIfcvtSizeLimit(BranchBytes=2, CommonBytes=6, NumPredicatedInstructions=4, ExtraPredicateBytes=2)
+# DEBUG: MeetIfcvtSizeLimit(BranchBytes=2, CommonBytes=8, NumPredicatedInstructions=4, ExtraPredicateBytes=2)
 
 body:             |
   bb.0.entry:
     successors: %bb.1(0x30000000), %bb.3(0x50000000)
     liveins: $r0, $r1, $r2
 
-    tCMPi8 killed renamable $r2, 42, 14, $noreg, implicit-def $cpsr
+    tCMPi8 renamable $r2, 42, 14, $noreg, implicit-def $cpsr
     t2Bcc %bb.3, 1, killed $cpsr
 
   bb.1.if.then:
     successors:  %bb.5(0x7fffffff)
-    liveins: $r0
+    liveins: $r0, $r2
 
     renamable $r0 = tLDRi killed renamable $r0, 0, 14, $noreg
     INLINEASM_BR &"b ${0:l}", 1, 13, blockaddress(@fn9, %ir-block.lab1)
+    tBX_RET 14, $noreg, implicit $r2
 
   bb.3.if.else:
     successors: %bb.5(0x7fffffff)
-    liveins: $r1
+    liveins: $r1, $r2
 
     renamable $r0 = tLDRi killed renamable $r1, 0, 14, $noreg
     renamable $r0 = tLDRi killed renamable $r0, 0, 14, $noreg
     renamable $r0 = t2LDRSHi12 killed renamable $r0, 0, 14, $noreg
     INLINEASM_BR &"b ${0:l}", 1, 13, blockaddress(@fn9, %ir-block.lab1)
-
+    tBX_RET 14, $noreg, implicit $r2
+    
   bb.5.lab1 (address-taken):
     liveins: $r0
 
diff --git a/llvm/test/CodeGen/ARM/no-fpscr-liveness.ll b/llvm/test/CodeGen/ARM/no-fpscr-liveness.ll
index 6575ab6c55d55..4fdb961e8e54b 100644
--- a/llvm/test/CodeGen/ARM/no-fpscr-liveness.ll
+++ b/llvm/test/CodeGen/ARM/no-fpscr-liveness.ll
@@ -12,7 +12,9 @@ target triple = "thumbv7s-apple-ios"
 ; CHECK-LABEL: eggs:
 ; CHECK: sub sp, #8
 ; VMRS instruction comes before any other instruction writing FPSCR:
-; CHECK-NEXT: vmrs r0, fpscr
+; CHECK-NOT: vcmp
+; CHECK: vmrs {{r[0-9]}}, fpscr
+; CHECK; vcmp
 ; ...
 ; CHECK: add sp, #8
 ; CHECK: bx lr
diff --git a/llvm/test/CodeGen/ARM/t2-shrink-ldrpost.ll b/llvm/test/CodeGen/ARM/t2-shrink-ldrpost.ll
index e26f3848ddcdf..e7b1780c5998a 100644
--- a/llvm/test/CodeGen/ARM/t2-shrink-ldrpost.ll
+++ b/llvm/test/CodeGen/ARM/t2-shrink-ldrpost.ll
@@ -1,13 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i1:8:32-i8:8:32-i16:16:32-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
 target triple = "thumbv7m--linux-gnu"
 
-; CHECK-LABEL: f:
-; CHECK: ldm	r{{[0-9]}}!, {r[[x:[0-9]]]}
-; CHECK: add.w	r[[x]], r[[x]], #3
-; CHECK: stm	r{{[0-9]}}!, {r[[x]]}
+; NOTE: When optimising for minimum size, an LDM is expected to be generated
 define void @f(i32 %n, i32* nocapture %a, i32* nocapture readonly %b) optsize minsize {
+; CHECK-LABEL: f:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    cmp r0, #1
+; CHECK-NEXT:    blt .LBB0_2
+; CHECK-NEXT:  .LBB0_1: @ %.lr.ph
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldm r2!, {r3}
+; CHECK-NEXT:    adds r3, #3
+; CHECK-NEXT:    stm r1!, {r3}
+; CHECK-NEXT:    subs r0, #1
+; CHECK-NEXT:    bne .LBB0_1
+; CHECK-NEXT:  .LBB0_2: @ %._crit_edge
+; CHECK-NEXT:    bx lr
   %1 = icmp sgt i32 %n, 0
   br i1 %1, label %.lr.ph, label %._crit_edge
 
@@ -28,9 +39,22 @@ define void @f(i32 %n, i32* nocapture %a, i32* nocapture readonly %b) optsize mi
   ret void
 }
 
-; CHECK-LABEL: f_nominsize:
-; CHECK-NOT: ldm
+; NOTE: When not optimising for minimum size, an LDM is expected not to be generated
 define void @f_nominsize(i32 %n, i32* nocapture %a, i32* nocapture readonly %b) optsize {
+; CHECK-LABEL: f_nominsize:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    cmp r0, #1
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    bxlt lr
+; CHECK-NEXT:  .LBB1_1: @ %.lr.ph
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r3, [r2], #4
+; CHECK-NEXT:    subs r0, #1
+; CHECK-NEXT:    add.w r3, r3, #3
+; CHECK-NEXT:    str r3, [r1], #4
+; CHECK-NEXT:    bne .LBB1_1
+; CHECK-NEXT:  @ %bb.2: @ %._crit_edge
+; CHECK-NEXT:    bx lr
   %1 = icmp sgt i32 %n, 0
   br i1 %1, label %.lr.ph, label %._crit_edge
 
diff --git a/llvm/test/CodeGen/ARM/vcvt.ll b/llvm/test/CodeGen/ARM/vcvt.ll
index f16c8dc3a151f..8db0856fdb971 100644
--- a/llvm/test/CodeGen/ARM/vcvt.ll
+++ b/llvm/test/CodeGen/ARM/vcvt.ll
@@ -296,13 +296,13 @@ define <4 x i16> @fix_double_to_i16(<4 x double> %in) {
 ; CHECK-NEXT:    vcvt.s32.f64 s0, d18
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vadd.f64 d20, d16, d16
-; CHECK-NEXT:    vadd.f64 d19, d19, d19
 ; CHECK-NEXT:    vadd.f64 d16, d17, d17
 ; CHECK-NEXT:    vcvt.s32.f64 s2, d20
-; CHECK-NEXT:    vcvt.s32.f64 s4, d19
 ; CHECK-NEXT:    vcvt.s32.f64 s6, d16
 ; CHECK-NEXT:    vmov.32 d16[0], r0
 ; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vadd.f64 d19, d19, d19
+; CHECK-NEXT:    vcvt.s32.f64 s4, d19
 ; CHECK-NEXT:    vmov.32 d17[0], r0
 ; CHECK-NEXT:    vmov r0, s4
 ; CHECK-NEXT:    vmov.32 d16[1], r0
@@ -355,10 +355,8 @@ define i32 @multi_sint(double %c, i32* nocapture %p, i32* nocapture %q) {
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    vmov d16, r0, r1
 ; CHECK-NEXT:    vcvt.s32.f64 s0, d16
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vstr s0, [r2]
-; CHECK-NEXT:    vcvt.s32.f64 s0, d16
-; CHECK-NEXT:    vcvt.s32.f64 s2, d16
-; CHECK-NEXT:    vmov r0, s2
 ; CHECK-NEXT:    vstr s0, [r3]
 ; CHECK-NEXT:    mov pc, lr
   %conv = fptosi double %c to i32
@@ -372,10 +370,8 @@ define i32 @multi_uint(double %c, i32* nocapture %p, i32* nocapture %q) {
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    vmov d16, r0, r1
 ; CHECK-NEXT:    vcvt.u32.f64 s0, d16
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vstr s0, [r2]
-; CHECK-NEXT:    vcvt.u32.f64 s0, d16
-; CHECK-NEXT:    vcvt.u32.f64 s2, d16
-; CHECK-NEXT:    vmov r0, s2
 ; CHECK-NEXT:    vstr s0, [r3]
 ; CHECK-NEXT:    mov pc, lr
   %conv = fptoui double %c to i32
diff --git a/llvm/test/CodeGen/ARM/vlldm-vlstm-uops.mir b/llvm/test/CodeGen/ARM/vlldm-vlstm-uops.mir
index f3cab91301dae..63cd2a6c00308 100644
--- a/llvm/test/CodeGen/ARM/vlldm-vlstm-uops.mir
+++ b/llvm/test/CodeGen/ARM/vlldm-vlstm-uops.mir
@@ -62,7 +62,7 @@ body:             |
     $sp = tSUBspi $sp, 34, 14, $noreg
     VLSTM $sp, 14, $noreg
     tBLXNSr 14, $noreg, killed $r4, csr_aapcs, implicit-def $lr, implicit $sp, implicit-def dead $lr, implicit $sp, implicit-def $sp
-    VLLDM $sp, 14, $noreg
+    VLLDM $sp, 14, $noreg, implicit-def $q0, implicit-def $q1, implicit-def $q2, implicit-def $q3, implicit-def $q4, implicit-def $q5, implicit-def $q6, implicit-def $q7, implicit-def $vpr, implicit-def $fpscr, implicit-def $fpscr_nzcv
     $sp = tADDspi $sp, 34, 14, $noreg
     $sp = t2LDMIA_UPD $sp, 14, $noreg, def $r4, def $r5, def $r6, def $r7, def $r8, def $r9, def $r10, def $r11
     $sp = t2LDMIA_RET $sp, 14, $noreg, def $r4, def $pc
diff --git a/llvm/test/CodeGen/AVR/calling-conv/c/basic_aggr.ll b/llvm/test/CodeGen/AVR/calling-conv/c/basic_aggr.ll
new file mode 100644
index 0000000000000..0f6cf0ed73d08
--- /dev/null
+++ b/llvm/test/CodeGen/AVR/calling-conv/c/basic_aggr.ll
@@ -0,0 +1,84 @@
+; RUN: llc < %s -march=avr | FileCheck %s
+
+; CHECK-LABEL: ret_void_args_struct_i8_i32
+define void @ret_void_args_struct_i8_i32({ i8, i32 } %a) {
+start:
+  ; CHECK:      sts     4, r20
+  %0 = extractvalue { i8, i32 } %a, 0
+  store volatile i8 %0, i8* inttoptr (i64 4 to i8*)
+
+  ; CHECK-NEXT: sts     8, r24
+  ; CHECK-NEXT: sts     7, r23
+  ; CHECK-NEXT: sts     6, r22
+  ; CHECK-NEXT: sts     5, r21
+  %1 = extractvalue { i8, i32 } %a, 1
+  store volatile i32 %1, i32* inttoptr (i64 5 to i32*)
+  ret void
+}
+
+; CHECK-LABEL: ret_void_args_struct_i8_i8_i8_i8
+define void @ret_void_args_struct_i8_i8_i8_i8({ i8, i8, i8, i8 } %a) {
+start:
+  ; CHECK:      sts     4, r22
+  %0 = extractvalue { i8, i8, i8, i8 } %a, 0
+  store volatile i8 %0, i8* inttoptr (i64 4 to i8*)
+  ; CHECK-NEXT: sts     5, r23
+  %1 = extractvalue { i8, i8, i8, i8 } %a, 1
+  store volatile i8 %1, i8* inttoptr (i64 5 to i8*)
+  ; CHECK-NEXT: sts     6, r24
+  %2 = extractvalue { i8, i8, i8, i8 } %a, 2
+  store volatile i8 %2, i8* inttoptr (i64 6 to i8*)
+  ; CHECK-NEXT: sts     7, r25
+  %3 = extractvalue { i8, i8, i8, i8 } %a, 3
+  store volatile i8 %3, i8* inttoptr (i64 7 to i8*)
+  ret void
+}
+
+; CHECK-LABEL: ret_void_args_struct_i32_16_i8
+define void @ret_void_args_struct_i32_16_i8({ i32, i16, i8} %a) {
+start:
+  ; CHECK:      sts     7, r21
+  ; CHECK-NEXT: sts     6, r20
+  ; CHECK-NEXT: sts     5, r19
+  ; CHECK-NEXT: sts     4, r18
+  %0 = extractvalue { i32, i16, i8 } %a, 0
+  store volatile i32 %0, i32* inttoptr (i64 4 to i32*)
+
+  ; CHECK-NEXT: sts     5, r23
+  ; CHECK-NEXT: sts     4, r22
+  %1 = extractvalue { i32, i16, i8 } %a, 1
+  store volatile i16 %1, i16* inttoptr (i64 4 to i16*)
+
+  ; CHECK-NEXT: sts     4, r24
+  %2 = extractvalue { i32, i16, i8 } %a, 2
+  store volatile i8 %2, i8* inttoptr (i64 4 to i8*)
+  ret void
+}
+
+; CHECK-LABEL: ret_void_args_struct_i8_i32_struct_i32_i8
+define void @ret_void_args_struct_i8_i32_struct_i32_i8({ i8, i32 } %a, { i32, i8 } %b) {
+start:
+  ; CHECK:      sts     4, r20
+  %0 = extractvalue { i8, i32 } %a, 0
+  store volatile i8 %0, i8* inttoptr (i64 4 to i8*)
+
+  ; CHECK-NEXT: sts     8, r24
+  ; CHECK-NEXT: sts     7, r23
+  ; CHECK-NEXT: sts     6, r22
+  ; CHECK-NEXT: sts     5, r21
+  %1 = extractvalue { i8, i32 } %a, 1
+  store volatile i32 %1, i32* inttoptr (i64 5 to i32*)
+
+  ; CHECK-NEXT:      sts     9, r17
+  ; CHECK-NEXT:      sts     8, r16
+  ; CHECK-NEXT:      sts     7, r15
+  ; CHECK-NEXT:      sts     6, r14
+  %2 = extractvalue { i32, i8 } %b, 0
+  store volatile i32 %2, i32* inttoptr (i64 6 to i32*)
+
+  ; CHECK-NEXT: sts     7, r18
+  %3 = extractvalue { i32, i8 } %b, 1
+  store volatile i8 %3, i8* inttoptr (i64 7 to i8*)
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AVR/calling-conv/c/call.ll b/llvm/test/CodeGen/AVR/calling-conv/c/call.ll
new file mode 100644
index 0000000000000..e218ed37238a3
--- /dev/null
+++ b/llvm/test/CodeGen/AVR/calling-conv/c/call.ll
@@ -0,0 +1,89 @@
+; RUN: llc < %s -march=avr | FileCheck %s
+
+declare void @ret_void_args_i8(i8 %a)
+declare void @ret_void_args_i8_i32(i8 %a, i32 %b)
+declare void @ret_void_args_i8_i8_i8_i8(i8 %a, i8 %b, i8 %c, i8 %d)
+declare void @ret_void_args_i32_i16_i8(i32 %a, i16 %b, i8 %c)
+declare void @ret_void_args_i64(i64 %a)
+declare void @ret_void_args_i64_i64(i64 %a, i64 %b)
+declare void @ret_void_args_i64_i64_i16(i64 %a, i64 %b, i16 %c)
+
+; CHECK-LABEL: call_void_args_i8
+define void @call_void_args_i8() {
+    ; CHECK: ldi r24, 64
+    call void @ret_void_args_i8 (i8 64)
+    ret void
+}
+
+; CHECK-LABEL: call_void_args_i8_i32
+define void @call_void_args_i8_i32() {
+    ; CHECK: ldi r20, 4
+    ; CHECK-NEXT: ldi r21, 3
+    ; CHECK-NEXT: ldi r22, 2
+    ; CHECK-NEXT: ldi r23, 1
+    ; CHECK-NEXT: ldi r24, 64
+    call void @ret_void_args_i8_i32 (i8 64, i32 16909060)
+    ret void
+}
+
+; CHECK-LABEL: call_void_args_i8_i8_i8_i8
+define void @call_void_args_i8_i8_i8_i8() {
+    ; CHECK: ldi r24, 1
+    ; CHECK-NEXT: ldi r22, 2
+    ; CHECK-NEXT: ldi r20, 3
+    ; CHECK-NEXT: ldi r18, 4
+    call void @ret_void_args_i8_i8_i8_i8(i8 1, i8 2, i8 3, i8 4)
+    ret void
+}
+
+; CHECK-LABEL: call_void_args_i32_i16_i8
+define void @call_void_args_i32_i16_i8() {
+    ; CHECK: ldi r22, 4
+    ; CHECK-NEXT: ldi r23, 3
+    ; CHECK-NEXT: ldi r24, 2
+    ; CHECK-NEXT: ldi r25, 1
+    ; CHECK-NEXT: ldi r20, 1
+    ; CHECK-NEXT: ldi r21, 4
+    ; CHECK-NEXT: ldi r18, 64
+    call void @ret_void_args_i32_i16_i8(i32 16909060, i16 1025, i8 64)
+    ret void
+}
+
+; CHECK-LABEL: call_void_args_i64
+define void @call_void_args_i64() {
+    ; CHECK: ldi r18, 8
+    ; CHECK-NEXT: ldi r19, 7
+    ; CHECK-NEXT: ldi r20, 6
+    ; CHECK-NEXT: ldi r21, 5
+    ; CHECK-NEXT: ldi r22, 4
+    ; CHECK-NEXT: ldi r23, 3
+    ; CHECK-NEXT: ldi r24, 2
+    ; CHECK-NEXT: ldi r25, 1
+    call void @ret_void_args_i64(i64 72623859790382856)
+    ret void
+}
+
+; CHECK-LABEL: call_void_args_i64_i64
+define void @call_void_args_i64_i64() {
+    ; CHECK: ldi r18, 8
+    ; CHECK-NEXT: ldi r19, 7
+    ; CHECK-NEXT: ldi r20, 6
+    ; CHECK-NEXT: ldi r21, 5
+    ; CHECK-NEXT: ldi r22, 4
+    ; CHECK-NEXT: ldi r23, 3
+    ; CHECK-NEXT: ldi r24, 2
+    ; CHECK-NEXT: ldi r25, 1
+    ; the second arg is in r10:r17, but unordered
+    ; CHECK: r17,
+    ; CHECK: r10,
+    call void @ret_void_args_i64_i64(i64 72623859790382856, i64 651345242494996224)
+    ret void
+}
+
+; CHECK-LABEL: call_void_args_i64_i64_i16
+define void @call_void_args_i64_i64_i16() {
+    ; CHECK: r8,
+    ; CHECK: r9,
+    call void @ret_void_args_i64_i64_i16(i64 72623859790382856, i64 651345242494996224, i16 5655)
+    ret void
+}
diff --git a/llvm/test/CodeGen/AVR/calling-conv/c/call_aggr.ll b/llvm/test/CodeGen/AVR/calling-conv/c/call_aggr.ll
new file mode 100644
index 0000000000000..aeba3c8f9757a
--- /dev/null
+++ b/llvm/test/CodeGen/AVR/calling-conv/c/call_aggr.ll
@@ -0,0 +1,48 @@
+; RUN: llc < %s -march=avr | FileCheck %s
+
+declare void @ret_void_args_struct_i8_i32({ i8, i32 } %a)
+declare void @ret_void_args_struct_i8_i8_i8_i8({ i8, i8, i8, i8 } %a)
+declare void @ret_void_args_struct_i32_i16_i8({ i32, i16, i8} %a)
+declare void @ret_void_args_struct_i8_i32_struct_i32_i8({ i8, i32 } %a, { i32, i8 } %b)
+
+; CHECK-LABEL: call_void_args_struct_i8_i32
+define void @call_void_args_struct_i8_i32() {
+    ; CHECK: ldi r20, 64
+    ; CHECK-NEXT: r21,
+    ; CHECK-NEXT: r22,
+    ; CHECK-NEXT: r23,
+    ; CHECK-NEXT: r24,
+    call void @ret_void_args_struct_i8_i32({ i8, i32 } { i8 64, i32 16909060 })
+    ret void
+}
+
+; CHECK-LABEL: @call_void_args_struct_i8_i8_i8_i8
+define void @call_void_args_struct_i8_i8_i8_i8() {
+    ; CHECK: ldi r22, 1
+    ; CHECK-NEXT: ldi r23, 2
+    ; CHECK-NEXT: ldi r24, 3
+    ; CHECK-NEXT: ldi r25, 4
+    call void @ret_void_args_struct_i8_i8_i8_i8({ i8, i8, i8, i8 } { i8 1, i8 2, i8 3, i8 4 })
+    ret void
+}
+
+; CHECK-LABEL: @call_void_args_struct_i32_i16_i8
+define void @call_void_args_struct_i32_i16_i8() {
+    ; CHECK: ldi r18, 4
+    ; CHECK-NEXT: ldi r19, 3
+    ; CHECK-NEXT: ldi r20, 2
+    ; CHECK-NEXT: ldi r21, 1
+    ; CHECK-NEXT: ldi r22, 23
+    ; CHECK-NEXT: ldi r23, 22
+    ; CHECK-NEXT: ldi r24, 64
+    call void @ret_void_args_struct_i32_i16_i8({ i32, i16, i8 } { i32 16909060, i16 5655, i8 64 })
+    ret void
+}
+
+; CHECK-LABEL: @call_void_args_struct_i8_i32_struct_i32_i8
+define void @call_void_args_struct_i8_i32_struct_i32_i8() {
+    ; CHECK: ldi r20, 64
+    ; CHECK: ldi r18, 65
+    call void @ret_void_args_struct_i8_i32_struct_i32_i8({ i8, i32 } { i8 64, i32 16909060 }, { i32, i8 } { i32 287454020, i8 65 })
+    ret void
+}
diff --git a/llvm/test/CodeGen/AVR/calling-conv/c/return_aggr.ll b/llvm/test/CodeGen/AVR/calling-conv/c/return_aggr.ll
new file mode 100644
index 0000000000000..97668f6c05733
--- /dev/null
+++ b/llvm/test/CodeGen/AVR/calling-conv/c/return_aggr.ll
@@ -0,0 +1,31 @@
+; RUN: llc < %s -march=avr | FileCheck %s
+
+; CHECK-LABEL: ret_struct_i8_i16_i8
+define { i8, i16, i8 } @ret_struct_i8_i16_i8() {
+start:
+  ; for some reason the i16 is loaded to r24:r25
+  ; and then moved to r23:r24
+  ; CHECK: ldi r22, 64
+  ; CHECK-NEXT: r23,
+  ; CHECK-NEXT: r24,
+  ; CHECK-NEXT: r25, 11
+  %0 = insertvalue {i8, i16, i8} undef, i8 64, 0
+  %1 = insertvalue {i8, i16, i8} %0, i16 1024, 1
+  %2 = insertvalue {i8, i16, i8} %1, i8 11, 2
+  ret {i8, i16, i8} %2
+}
+
+; CHECK-LABEL: ret_struct_i32_i16
+define { i32, i16 } @ret_struct_i32_i16() {
+start:
+  ; CHECK: ldi r18, 4
+  ; CHECK-NEXT: ldi r19, 3
+  ; CHECK-NEXT: ldi r20, 2
+  ; CHECK-NEXT: ldi r21, 1
+  ; CHECK-NEXT: ldi r22, 0
+  ; CHECK-NEXT: ldi r23, 8
+  %0 = insertvalue { i32, i16 } undef, i32 16909060, 0
+  %1 = insertvalue { i32, i16 } %0, i16 2048, 1
+  ret { i32, i16} %1
+}
+
diff --git a/llvm/test/CodeGen/AVR/jmp-long.ll b/llvm/test/CodeGen/AVR/jmp-long.ll
new file mode 100644
index 0000000000000..9878fda1028a0
--- /dev/null
+++ b/llvm/test/CodeGen/AVR/jmp-long.ll
@@ -0,0 +1,1045 @@
+; RUN: llc -filetype=obj -march avr -mattr=jmpcall < %s | llvm-objdump -dr --mattr=jmpcall - | FileCheck %s
+
+; Test the fix in https://reviews.llvm.org/D78459.
+; Long branches (that use jmp instead of rjmp) were broken: the jump was to a
+; location 0x20000 away from where it should be. This test is a regression test
+; to make sure long branches are correct.
+
+declare void @bar() addrspace(1)
+
+define void @foo() addrspace(1) {
+  br label %1
+
+1:                                                ; preds = %0, %1
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  tail call addrspace(0) void asm sideeffect "nop", ""()
+  ; CHECK: call 0
+  ; CHECK:      R_AVR_CALL bar
+  ; CHECK: jmp  0
+  ; CHECK:      R_AVR_CALL .text
+  tail call addrspace(1) void @bar()
+  br label %1
+}
diff --git a/llvm/test/CodeGen/BPF/BTF/map-def-2.ll b/llvm/test/CodeGen/BPF/BTF/map-def-2.ll
new file mode 100644
index 0000000000000..bf3c4a7961fbf
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/BTF/map-def-2.ll
@@ -0,0 +1,90 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+;
+; Source code:
+;   struct key_type {
+;     int a1;
+;   };
+;   typedef struct map_type {
+;     struct key_type *key;
+;   } _map_type;
+;   typedef _map_type __map_type;
+;   __map_type __attribute__((section(".maps"))) hash_map;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t2.c
+
+%struct.map_type = type { %struct.key_type* }
+%struct.key_type = type { i32 }
+
+@hash_map = dso_local local_unnamed_addr global %struct.map_type zeroinitializer, section ".maps", align 8, !dbg !0
+
+; CHECK:             .long   0                               # BTF_KIND_PTR(id = 1)
+; CHECK-NEXT:        .long   33554432                        # 0x2000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   1                               # BTF_KIND_STRUCT(id = 2)
+; CHECK-NEXT:        .long   67108865                        # 0x4000001
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   10
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   0                               # 0x0
+; CHECK-NEXT:        .long   13                              # BTF_KIND_INT(id = 3)
+; CHECK-NEXT:        .long   16777216                        # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                        # 0x1000020
+; CHECK-NEXT:        .long   17                              # BTF_KIND_TYPEDEF(id = 4)
+; CHECK-NEXT:        .long   134217728                       # 0x8000000
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   28                              # BTF_KIND_TYPEDEF(id = 5)
+; CHECK-NEXT:        .long   134217728                       # 0x8000000
+; CHECK-NEXT:        .long   6
+; CHECK-NEXT:        .long   38                              # BTF_KIND_STRUCT(id = 6)
+; CHECK-NEXT:        .long   67108865                        # 0x4000001
+; CHECK-NEXT:        .long   8
+; CHECK-NEXT:        .long   47
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   0                               # 0x0
+; CHECK-NEXT:        .long   51                              # BTF_KIND_VAR(id = 7)
+; CHECK-NEXT:        .long   234881024                       # 0xe000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   60                              # BTF_KIND_DATASEC(id = 8)
+; CHECK-NEXT:        .long   251658241                       # 0xf000001
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   hash_map
+; CHECK-NEXT:        .long   8
+
+; CHECK:             .ascii  "key_type"                      # string offset=1
+; CHECK:             .ascii  "a1"                            # string offset=10
+; CHECK:             .ascii  "int"                           # string offset=13
+; CHECK:             .ascii  "__map_type"                    # string offset=17
+; CHECK:             .ascii  "_map_type"                     # string offset=28
+; CHECK:             .ascii  "map_type"                      # string offset=38
+; CHECK:             .ascii  "key"                           # string offset=47
+; CHECK:             .ascii  "hash_map"                      # string offset=51
+; CHECK:             .ascii  ".maps"                         # string offset=60
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!16, !17, !18}
+!llvm.ident = !{!19}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "hash_map", scope: !2, file: !3, line: 8, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 11.0.0 (https://github.com/llvm/llvm-project.git b8409c03ed90807f3d49c7d98dceea98cf461f7a)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, splitDebugInlining: false, nameTableKind: None)
+!3 = !DIFile(filename: "t2.c", directory: "/tmp/home/yhs/tmp1")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_typedef, name: "__map_type", file: !3, line: 7, baseType: !7)
+!7 = !DIDerivedType(tag: DW_TAG_typedef, name: "_map_type", file: !3, line: 6, baseType: !8)
+!8 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "map_type", file: !3, line: 4, size: 64, elements: !9)
+!9 = !{!10}
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "key", scope: !8, file: !3, line: 5, baseType: !11, size: 64)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64)
+!12 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "key_type", file: !3, line: 1, size: 32, elements: !13)
+!13 = !{!14}
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !12, file: !3, line: 2, baseType: !15, size: 32)
+!15 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!16 = !{i32 7, !"Dwarf Version", i32 4}
+!17 = !{i32 2, !"Debug Info Version", i32 3}
+!18 = !{i32 1, !"wchar_size", i32 4}
+!19 = !{!"clang version 11.0.0 (https://github.com/llvm/llvm-project.git b8409c03ed90807f3d49c7d98dceea98cf461f7a)"}
diff --git a/llvm/test/CodeGen/BPF/BTF/map-def-3.ll b/llvm/test/CodeGen/BPF/BTF/map-def-3.ll
new file mode 100644
index 0000000000000..e05470782ec26
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/BTF/map-def-3.ll
@@ -0,0 +1,65 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+;
+; Source code:
+;   struct key_type {
+;     int a1;
+;   };
+;   const struct key_type __attribute__((section(".maps"))) hash_map;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t3.c
+
+%struct.key_type = type { i32 }
+
+@hash_map = dso_local local_unnamed_addr constant %struct.key_type zeroinitializer, section ".maps", align 4, !dbg !0
+
+; CHECK:             .long   1                               # BTF_KIND_INT(id = 1)
+; CHECK-NEXT:        .long   16777216                        # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                        # 0x1000020
+; CHECK-NEXT:        .long   0                               # BTF_KIND_CONST(id = 2)
+; CHECK-NEXT:        .long   167772160                       # 0xa000000
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   5                               # BTF_KIND_STRUCT(id = 3)
+; CHECK-NEXT:        .long   67108865                        # 0x4000001
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   14
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   0                               # 0x0
+; CHECK-NEXT:        .long   17                              # BTF_KIND_VAR(id = 4)
+; CHECK-NEXT:        .long   234881024                       # 0xe000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   26                              # BTF_KIND_DATASEC(id = 5)
+; CHECK-NEXT:        .long   251658241                       # 0xf000001
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   hash_map
+; CHECK-NEXT:        .long   4
+
+; CHECK:             .ascii  "int"                           # string offset=1
+; CHECK:             .ascii  "key_type"                      # string offset=5
+; CHECK:             .ascii  "a1"                            # string offset=14
+; CHECK:             .ascii  "hash_map"                      # string offset=17
+; CHECK:             .ascii  ".maps"                         # string offset=26
+
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!11, !12, !13}
+!llvm.ident = !{!14}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "hash_map", scope: !2, file: !3, line: 4, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 11.0.0 (https://github.com/llvm/llvm-project.git 5bd074629f00d4798674b411cf00216f38016483)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, splitDebugInlining: false, nameTableKind: None)
+!3 = !DIFile(filename: "t3.c", directory: "/tmp/home/yhs/tmp1")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !7)
+!7 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "key_type", file: !3, line: 1, size: 32, elements: !8)
+!8 = !{!9}
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !7, file: !3, line: 2, baseType: !10, size: 32)
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{i32 7, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{!"clang version 11.0.0 (https://github.com/llvm/llvm-project.git 5bd074629f00d4798674b411cf00216f38016483)"}
diff --git a/llvm/test/CodeGen/BPF/BTF/map-def.ll b/llvm/test/CodeGen/BPF/BTF/map-def.ll
index cf777880efa17..e12cde3ef98ae 100644
--- a/llvm/test/CodeGen/BPF/BTF/map-def.ll
+++ b/llvm/test/CodeGen/BPF/BTF/map-def.ll
@@ -28,41 +28,41 @@
 ; CHECK-NEXT:        .long   168
 ; CHECK-NEXT:        .long   168
 ; CHECK-NEXT:        .long   65
-; CHECK-NEXT:        .long   1                       # BTF_KIND_STRUCT(id = 1)
-; CHECK-NEXT:        .long   67108866                # 0x4000002
-; CHECK-NEXT:        .long   16
-; CHECK-NEXT:        .long   10
-; CHECK-NEXT:        .long   2
-; CHECK-NEXT:        .long   0                       # 0x0
-; CHECK-NEXT:        .long   14
-; CHECK-NEXT:        .long   5
-; CHECK-NEXT:        .long   64                      # 0x40
-; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 2)
+; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 1)
 ; CHECK-NEXT:        .long   33554432                # 0x2000000
-; CHECK-NEXT:        .long   3
-; CHECK-NEXT:        .long   20                      # BTF_KIND_STRUCT(id = 3)
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   1                       # BTF_KIND_STRUCT(id = 2)
 ; CHECK-NEXT:        .long   67108866                # 0x4000002
 ; CHECK-NEXT:        .long   8
-; CHECK-NEXT:        .long   29
-; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   10
+; CHECK-NEXT:        .long   3
 ; CHECK-NEXT:        .long   0                       # 0x0
-; CHECK-NEXT:        .long   31
-; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   12
+; CHECK-NEXT:        .long   3
 ; CHECK-NEXT:        .long   32                      # 0x20
-; CHECK-NEXT:        .long   33                      # BTF_KIND_INT(id = 4)
+; CHECK-NEXT:        .long   14                      # BTF_KIND_INT(id = 3)
 ; CHECK-NEXT:        .long   16777216                # 0x1000000
 ; CHECK-NEXT:        .long   4
 ; CHECK-NEXT:        .long   16777248                # 0x1000020
-; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 5)
+; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 4)
 ; CHECK-NEXT:        .long   33554432                # 0x2000000
-; CHECK-NEXT:        .long   6
-; CHECK-NEXT:        .long   37                      # BTF_KIND_INT(id = 6)
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   18                      # BTF_KIND_INT(id = 5)
 ; CHECK-NEXT:        .long   16777216                # 0x1000000
 ; CHECK-NEXT:        .long   4
 ; CHECK-NEXT:        .long   32                      # 0x20
+; CHECK-NEXT:        .long   31                      # BTF_KIND_STRUCT(id = 6)
+; CHECK-NEXT:        .long   67108866                # 0x4000002
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   0                       # 0x0
+; CHECK-NEXT:        .long   44
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   64                      # 0x40
 ; CHECK-NEXT:        .long   50                      # BTF_KIND_VAR(id = 7)
 ; CHECK-NEXT:        .long   234881024               # 0xe000000
-; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   6
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   59                      # BTF_KIND_DATASEC(id = 8)
 ; CHECK-NEXT:        .long   251658241               # 0xf000001
@@ -71,21 +71,21 @@
 ; CHECK-NEXT:        .long   hash_map
 ; CHECK-NEXT:        .long   16
 ; CHECK-NEXT:        .byte   0                       # string offset=0
-; CHECK-NEXT:        .ascii  "map_type"              # string offset=1
+; CHECK-NEXT:        .ascii  "key_type"              # string offset=1
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .ascii  "key"                   # string offset=10
+; CHECK-NEXT:        .byte   97                      # string offset=10
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .ascii  "value"                 # string offset=14
+; CHECK-NEXT:        .byte   98                      # string offset=12
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .ascii  "key_type"              # string offset=20
+; CHECK-NEXT:        .ascii  "int"                   # string offset=14
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .byte   97                      # string offset=29
+; CHECK-NEXT:        .ascii  "unsigned int"          # string offset=18
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .byte   98                      # string offset=31
+; CHECK-NEXT:        .ascii  "map_type"              # string offset=31
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .ascii  "int"                   # string offset=33
+; CHECK-NEXT:        .ascii  "key"                   # string offset=40
 ; CHECK-NEXT:        .byte   0
-; CHECK-NEXT:        .ascii  "unsigned int"          # string offset=37
+; CHECK-NEXT:        .ascii  "value"                 # string offset=44
 ; CHECK-NEXT:        .byte   0
 ; CHECK-NEXT:        .ascii  "hash_map"              # string offset=50
 ; CHECK-NEXT:        .byte   0
diff --git a/llvm/test/CodeGen/BPF/BTF/struct-anon-2.ll b/llvm/test/CodeGen/BPF/BTF/struct-anon-2.ll
new file mode 100644
index 0000000000000..d8f4f2360eaf5
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/BTF/struct-anon-2.ll
@@ -0,0 +1,106 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+;
+; Source:
+;   struct s1 {
+;     struct { int A1; } a1;
+;     struct { long B1; } *b1;
+;   };
+;   int f1(struct s1 *s1) { return 0; }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+%struct.s1 = type { %struct.anon, %struct.anon.0* }
+%struct.anon = type { i32 }
+%struct.anon.0 = type { i64 }
+
+; Function Attrs: norecurse nounwind readnone
+define dso_local i32 @f1(%struct.s1* nocapture readnone %s1) local_unnamed_addr #0 !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.s1* %s1, metadata !25, metadata !DIExpression()), !dbg !26
+  ret i32 0, !dbg !27
+}
+
+; CHECK:             .long   0                       # BTF_KIND_PTR(id = 1)
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   1                       # BTF_KIND_STRUCT(id = 2)
+; CHECK-NEXT:        .long   67108866                # 0x4000002
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   0                       # 0x0
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   64                      # 0x40
+; CHECK-NEXT:        .long   0                       # BTF_KIND_STRUCT(id = 3)
+; CHECK-NEXT:        .long   67108865                # 0x4000001
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   10
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   0                       # 0x0
+; CHECK-NEXT:        .long   13                      # BTF_KIND_INT(id = 4)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 5)
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   6
+; CHECK-NEXT:        .long   0                       # BTF_KIND_STRUCT(id = 6)
+; CHECK-NEXT:        .long   67108865                # 0x4000001
+; CHECK-NEXT:        .long   8
+; CHECK-NEXT:        .long   17
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   0                       # 0x0
+; CHECK-NEXT:        .long   20                      # BTF_KIND_INT(id = 7)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   8
+; CHECK-NEXT:        .long   16777280                # 0x1000040
+
+; CHECK:             .ascii  "s1"                    # string offset=1
+; CHECK:             .ascii  "a1"                    # string offset=4
+; CHECK:             .ascii  "b1"                    # string offset=7
+; CHECK:             .ascii  "A1"                    # string offset=10
+; CHECK:             .ascii  "int"                   # string offset=13
+; CHECK:             .ascii  "B1"                    # string offset=17
+; CHECK:             .ascii  "long int"              # string offset=20
+
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0 (https://github.com/llvm/llvm-project.git ef36f5143d83897cc6f59ff918769d29ad5a0612)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "t.c", directory: "/tmp/home/yhs/btf/tmp")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 11.0.0 (https://github.com/llvm/llvm-project.git ef36f5143d83897cc6f59ff918769d29ad5a0612)"}
+!7 = distinct !DISubprogram(name: "f1", scope: !1, file: !1, line: 5, type: !8, scopeLine: 5, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !24)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !11}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64)
+!12 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 1, size: 128, elements: !13)
+!13 = !{!14, !18}
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !12, file: !1, line: 2, baseType: !15, size: 32)
+!15 = distinct !DICompositeType(tag: DW_TAG_structure_type, scope: !12, file: !1, line: 2, size: 32, elements: !16)
+!16 = !{!17}
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "A1", scope: !15, file: !1, line: 2, baseType: !10, size: 32)
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !12, file: !1, line: 3, baseType: !19, size: 64, offset: 64)
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !20, size: 64)
+!20 = distinct !DICompositeType(tag: DW_TAG_structure_type, scope: !12, file: !1, line: 3, size: 64, elements: !21)
+!21 = !{!22}
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "B1", scope: !20, file: !1, line: 3, baseType: !23, size: 64)
+!23 = !DIBasicType(name: "long int", size: 64, encoding: DW_ATE_signed)
+!24 = !{!25}
+!25 = !DILocalVariable(name: "s1", arg: 1, scope: !7, file: !1, line: 5, type: !11)
+!26 = !DILocation(line: 0, scope: !7)
+!27 = !DILocation(line: 5, column: 25, scope: !7)
diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-array-2.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-array-2.ll
new file mode 100644
index 0000000000000..8c8ee093f1ed4
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-array-2.ll
@@ -0,0 +1,82 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; RUN: llc -march=bpfeb -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK-ALU32 %s
+; Source:
+;   enum { FIELD_EXISTENCE = 2, };
+;   struct s1 { int a1; };
+;   int test() {
+;     struct s1 *v = 0;
+;     return __builtin_preserve_field_info(v[0], FIELD_EXISTENCE);
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm test.c
+
+%struct.s1 = type { i32 }
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @test() local_unnamed_addr #0 !dbg !17 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.s1* null, metadata !21, metadata !DIExpression()), !dbg !22
+  %0 = tail call %struct.s1* @llvm.preserve.array.access.index.p0s_struct.s1s.p0s_struct.s1s(%struct.s1* null, i32 0, i32 0), !dbg !23, !llvm.preserve.access.index !8
+  %1 = tail call i32 @llvm.bpf.preserve.field.info.p0s_struct.s1s(%struct.s1* %0, i64 2), !dbg !24
+  ret i32 %1, !dbg !25
+}
+
+; CHECK:             .long   16                      # BTF_KIND_STRUCT(id = 4)
+
+; CHECK:             .ascii  ".text"                 # string offset=10
+; CHECK:             .ascii  "s1"                    # string offset=16
+; CHECK:             .byte   48                      # string offset=22
+
+; CHECK:             .long   16                      # FieldReloc
+; CHECK-NEXT:        .long   10                      # Field reloc section string offset=10
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   22
+; CHECK-NEXT:        .long   2
+
+; Function Attrs: nounwind readnone
+declare %struct.s1* @llvm.preserve.array.access.index.p0s_struct.s1s.p0s_struct.s1s(%struct.s1*, i32 immarg, i32 immarg) #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bpf.preserve.field.info.p0s_struct.s1s(%struct.s1*, i64 immarg) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!13, !14, !15}
+!llvm.ident = !{!16}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0 (https://github.com/llvm/llvm-project.git 32791937d7aceb0a5e1eaabf1bb1a6dbe1639792)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !7, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !1, line: 1, baseType: !4, size: 32, elements: !5)
+!4 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!5 = !{!6}
+!6 = !DIEnumerator(name: "FIELD_EXISTENCE", value: 2, isUnsigned: true)
+!7 = !{!8}
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !9, size: 64)
+!9 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1", file: !1, line: 2, size: 32, elements: !10)
+!10 = !{!11}
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !9, file: !1, line: 2, baseType: !12, size: 32)
+!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!13 = !{i32 7, !"Dwarf Version", i32 4}
+!14 = !{i32 2, !"Debug Info Version", i32 3}
+!15 = !{i32 1, !"wchar_size", i32 4}
+!16 = !{!"clang version 11.0.0 (https://github.com/llvm/llvm-project.git 32791937d7aceb0a5e1eaabf1bb1a6dbe1639792)"}
+!17 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 3, type: !18, scopeLine: 3, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !20)
+!18 = !DISubroutineType(types: !19)
+!19 = !{!12}
+!20 = !{!21}
+!21 = !DILocalVariable(name: "v", scope: !17, file: !1, line: 4, type: !8)
+!22 = !DILocation(line: 0, scope: !17)
+!23 = !DILocation(line: 5, column: 40, scope: !17)
+!24 = !DILocation(line: 5, column: 10, scope: !17)
+!25 = !DILocation(line: 5, column: 3, scope: !17)
diff --git a/llvm/test/CodeGen/Hexagon/abi-padding-2.ll b/llvm/test/CodeGen/Hexagon/abi-padding-2.ll
new file mode 100644
index 0000000000000..4bda4aed9c8b8
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/abi-padding-2.ll
@@ -0,0 +1,45 @@
+; RUN: llc -march=hexagon -mcpu=hexagonv65 < %s | FileCheck %s
+; REQUIRES: hexagon
+
+; C file was:
+; struct S { char b; long long a; };
+; void foo(int x, struct S arg1);
+; void bar() {
+;   struct S s;
+;   s.b = 'a';
+;   foo(42, s);
+;}
+
+;
+; Test that while passing a 7-byte struct on the stack, the
+; size of the struct is 16 bytes including padding since its
+; largest member is of type long long. This was being handled
+; correctly but is a sanity check against any potential future
+; regressions.
+;
+
+; CHECK: memd(r{{[0-9]+}}+#8) =
+; CHECK: memd(r{{[0-9]+}}+#0) =
+
+%struct.S = type { i8, i64 }
+
+; Function Attrs: nounwind
+define dso_local void @bar() local_unnamed_addr #0 {
+entry:
+  %s = alloca %struct.S, align 8
+  %0 = getelementptr inbounds %struct.S, %struct.S* %s, i32 0, i32 0
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* nonnull %0) #3
+  store i8 97, i8* %0, align 8
+  tail call void @foo(i32 42, %struct.S* nonnull byval(%struct.S) align 8 %s) #3
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* nonnull %0) #3
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare dso_local void @foo(i32, %struct.S* byval(%struct.S) align 8) local_unnamed_addr #2
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
diff --git a/llvm/test/CodeGen/Hexagon/abi-padding.ll b/llvm/test/CodeGen/Hexagon/abi-padding.ll
new file mode 100644
index 0000000000000..ee2b990777b92
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/abi-padding.ll
@@ -0,0 +1,46 @@
+; RUN: llc -march=hexagon -mcpu=hexagonv65 < %s | FileCheck %s
+; REQUIRES: hexagon
+
+; C file was:
+; struct S { int a[3];};
+; void foo(int x, struct S arg1, struct S arg2);
+; void bar() {
+;  struct S s;
+;  s.a[0] = 9;
+;  foo(42, s, s);
+; }
+
+; Test that while passing a 12-byte struct on the stack, the
+; struct is aligned to 4 bytes since its largest member is of type int.
+; Previously, the struct was being aligned to 8 bytes
+
+; CHECK: memw(r{{[0-9]+}}+#12) = #9
+
+; Check that the flag hexagon-disable-args-min-alignment works and the struct
+; is aligned to 8 bytes.
+; RUN: llc -march=hexagon -mcpu=hexagonv65 -hexagon-disable-args-min-alignment < %s | FileCheck -check-prefix=HEXAGON_LEGACY %s
+
+; HEXAGON_LEGACY: memw(r{{[0-9]+}}+#16) = #9
+
+%struct.S = type { [3 x i32] }
+
+; Function Attrs: nounwind
+define dso_local void @bar() local_unnamed_addr #0 {
+entry:
+  %s = alloca %struct.S, align 4
+  %0 = bitcast %struct.S* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 12, i8* nonnull %0) #3
+  %arrayidx = getelementptr inbounds %struct.S, %struct.S* %s, i32 0, i32 0, i32 0
+  store i32 9, i32* %arrayidx, align 4
+  tail call void @foo(i32 42, %struct.S* nonnull byval(%struct.S) align 4 %s, %struct.S* nonnull byval(%struct.S) align 4 %s) #3
+  call void @llvm.lifetime.end.p0i8(i64 12, i8* nonnull %0) #3
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare dso_local void @foo(i32, %struct.S* byval(%struct.S) align 4, %struct.S* byval(%struct.S) align 4) local_unnamed_addr #2
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll
index db850950fd53d..b2cf9ec44dd45 100644
--- a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll
+++ b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll
@@ -1,5 +1,5 @@
 ; Check for recognizing the "memmove" idiom.
-; RUN: opt -basicaa -hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \
+; RUN: opt -basic-aa -hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \
 ; RUN:  | FileCheck %s
 ; CHECK: call void @llvm.memmove
 
diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll
index b9747a887a590..5a47fd7ba46bf 100644
--- a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll
+++ b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \
+; RUN: opt -basic-aa -hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \
 ; RUN:  | FileCheck %s
 
 define void @PR14241(i32* %s, i64 %size) #0 {
diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll
index 5916832919824..e9e6f61e224b7 100644
--- a/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll
+++ b/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -hexagon-loop-idiom -mtriple hexagon-unknown-elf < %s
+; RUN: opt -basic-aa -hexagon-loop-idiom -mtriple hexagon-unknown-elf < %s
 ; REQUIRES: asserts
 
 target triple = "hexagon"
diff --git a/llvm/test/CodeGen/MIR/AArch64/cfi.mir b/llvm/test/CodeGen/MIR/AArch64/cfi.mir
index 04380e07f3ee3..5768bcafe479e 100644
--- a/llvm/test/CodeGen/MIR/AArch64/cfi.mir
+++ b/llvm/test/CodeGen/MIR/AArch64/cfi.mir
@@ -42,7 +42,7 @@ body: |
     CFI_INSTRUCTION restore_state
     ; CHECK: CFI_INSTRUCTION restore_state
     CFI_INSTRUCTION escape 0x61, 0x62, 0x63
-    ; CHECK: CFI_INSTRUCTION escape 0x61, 0x62, 0x63
+    ; CHECK: CFI_INSTRUCTION escape 0x61, 0x62, 0x63{{$}}
     CFI_INSTRUCTION window_save
     ; CHECK: CFI_INSTRUCTION window_save
     CFI_INSTRUCTION negate_ra_sign_state
diff --git a/llvm/test/CodeGen/MIR/Generic/cfi-escape.mir b/llvm/test/CodeGen/MIR/Generic/cfi-escape.mir
new file mode 100644
index 0000000000000..61a814eb16946
--- /dev/null
+++ b/llvm/test/CodeGen/MIR/Generic/cfi-escape.mir
@@ -0,0 +1,11 @@
+# RUN: llc -run-pass none -o - %s | llc -x=mir -run-pass none -o - | FileCheck %s
+
+# Check that we don't print a trailing comma for CFI escape indices, and that
+# in general we can round trip them.
+
+name: func
+# CHECK-LABEL: name: func
+body: |
+  bb.0:
+    CFI_INSTRUCTION escape 0x61, 0x62, 0x63
+    ; CHECK: CFI_INSTRUCTION escape 0x61, 0x62, 0x63{{$}}
diff --git a/llvm/test/CodeGen/MSP430/asm-clobbers.ll b/llvm/test/CodeGen/MSP430/asm-clobbers.ll
index 0a0335057f1bc..bd6f88cc2436c 100644
--- a/llvm/test/CodeGen/MSP430/asm-clobbers.ll
+++ b/llvm/test/CodeGen/MSP430/asm-clobbers.ll
@@ -3,9 +3,52 @@
 target datalayout = "e-m:e-p:16:16-i32:16:32-a:16-n8:16"
 target triple = "msp430---elf"
 
-define void @test() {
+define void @test_no_clobber() {
 entry:
-; CHECK-LABEL: test:
+; CHECK-LABEL: test_no_clobber
+; CHECK-NOT: push
+  call void asm sideeffect "", ""()
+; CHECK-NOT: pop
+  ret void
+; CHECK: -- End function
+}
+
+define void @test_1() {
+entry:
+; CHECK-LABEL: test_1:
+; CHECK: push r8
+; CHECK: push r6
+; CHECK: push r4
+  call void asm sideeffect "", "~{r4},~{r6},~{r8}"()
+; CHECK: pop r4
+; CHECK: pop r6
+; CHECK: pop r8
+  ret void
+}
+
+define void @test_2() {
+entry:
+; CHECK-LABEL: test_2:
+; CHECK: push r9
+; CHECK: push r7
+; CHECK: push r5
+  call void asm sideeffect "", "~{r5},~{r7},~{r9}"()
+; CHECK: pop r5
+; CHECK: pop r7
+; CHECK: pop r9
+  ret void
+}
+
+; The r10 register is special because the sequence
+;   pop r10
+;   ret
+; can be replaced with
+;   jmp __mspabi_func_epilog_1
+; or other such function (depending on previous instructions).
+; Still, it is not replaced *yet*.
+define void @test_r10() {
+entry:
+; CHECK-LABEL: test_r10:
 ; CHECK: push r10
   call void asm sideeffect "", "~{r10}"()
 ; CHECK: pop r10
diff --git a/llvm/test/CodeGen/MSP430/cmp-return-type.ll b/llvm/test/CodeGen/MSP430/cmp-return-type.ll
new file mode 100644
index 0000000000000..6c3f31d509ccd
--- /dev/null
+++ b/llvm/test/CodeGen/MSP430/cmp-return-type.ll
@@ -0,0 +1,25 @@
+; RUN: llc < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16"
+target triple = "msp430"
+
+define i16 @f(float %a, float %b) optsize {
+  %cmp = fcmp une float %a, %b
+  %conv = zext i1 %cmp to i16
+; CHECK-LABEL: call #__mspabi_cmpf
+; CHECK-NOT:   r13
+; CHECK-LABEL: mov r2
+
+; This is quite fragile attempt to detect the return type:
+; Correct:
+;        call    #__mspabi_cmpf
+;        tst     r12
+;        mov     r2, r13
+; Incorrect:
+;        call    #__mspabi_cmpf
+;        bis     r12, r13        <-- checking (R12:R13)
+;        tst     r13
+;        mov     r2, r13
+
+  ret i16 %conv
+}
diff --git a/llvm/test/CodeGen/MSP430/inline-asm-register-names.ll b/llvm/test/CodeGen/MSP430/inline-asm-register-names.ll
new file mode 100644
index 0000000000000..5b8accdcc74e0
--- /dev/null
+++ b/llvm/test/CodeGen/MSP430/inline-asm-register-names.ll
@@ -0,0 +1,59 @@
+; RUN: llc < %s | FileCheck %s
+target datalayout = "e-p:16:8:8-i8:8:8-i16:8:8-i32:8:8"
+target triple = "msp430-generic-generic"
+
+; Test that correct register names are accepted *inside* inline asm listings.
+; Tested with PUSH instruction since it does not support memory operands.
+
+define void @accepted_rN() nounwind {
+; CHECK-LABEL: accepted_rN
+  call void asm sideeffect "push r0", ""() nounwind
+; CHECK: push r0
+  call void asm sideeffect "push r1", ""() nounwind
+; CHECK: push r1
+  call void asm sideeffect "push r2", ""() nounwind
+; CHECK: push r2
+  call void asm sideeffect "push r3", ""() nounwind
+; CHECK: push r3
+  call void asm sideeffect "push r4", ""() nounwind
+; CHECK: push r4
+  call void asm sideeffect "push r5", ""() nounwind
+; CHECK: push r5
+  call void asm sideeffect "push r6", ""() nounwind
+; CHECK: push r6
+  call void asm sideeffect "push r7", ""() nounwind
+; CHECK: push r7
+  call void asm sideeffect "push r8", ""() nounwind
+; CHECK: push r8
+  call void asm sideeffect "push r9", ""() nounwind
+; CHECK: push r9
+  call void asm sideeffect "push r10", ""() nounwind
+; CHECK: push r10
+  call void asm sideeffect "push r11", ""() nounwind
+; CHECK: push r11
+  call void asm sideeffect "push r12", ""() nounwind
+; CHECK: push r12
+  call void asm sideeffect "push r13", ""() nounwind
+; CHECK: push r13
+  call void asm sideeffect "push r14", ""() nounwind
+; CHECK: push r14
+  call void asm sideeffect "push r15", ""() nounwind
+; CHECK: push r15
+  ret void
+}
+
+define void @accepted_reg_aliases() nounwind {
+; CHECK-LABEL: accepted_reg_aliases
+; Ensure register aliases are renamed as expected
+call void asm sideeffect "push pc", ""() nounwind
+; CHECK: push r0
+call void asm sideeffect "push sp", ""() nounwind
+; CHECK: push r1
+call void asm sideeffect "push sr", ""() nounwind
+; CHECK: push r2
+call void asm sideeffect "push cg", ""() nounwind
+; CHECK: push r3
+call void asm sideeffect "push fp", ""() nounwind
+; CHECK: push r4
+        ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/module-inline-asm.ll b/llvm/test/CodeGen/NVPTX/module-inline-asm.ll
index cdbcf2013c000..a2ce6a7f87b38 100644
--- a/llvm/test/CodeGen/NVPTX/module-inline-asm.ll
+++ b/llvm/test/CodeGen/NVPTX/module-inline-asm.ll
@@ -2,9 +2,19 @@
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64"
 
+; module asm must come after PTX version/target directives.
+; CHECK-NOT: .global .b32 val;
+
+; CHECK-DAG: .version
+; CHECK-DAG: .target
+
 ; CHECK: .global .b32 val;
 module asm ".global .b32 val;"
 
+; module asm must happen before we emit other things.
+; CHECK-LABEL: .visible .func foo
 define void @foo() {
   ret void
 }
+; Make sure it does not show up anywhere else in the output.
+; CHECK-NOT: .global .b32 val;
diff --git a/llvm/test/CodeGen/PowerPC/aix-base-pointer.ll b/llvm/test/CodeGen/PowerPC/aix-base-pointer.ll
new file mode 100644
index 0000000000000..2566e31c025d1
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-base-pointer.ll
@@ -0,0 +1,42 @@
+; RUN: llc -mcpu=pwr7 -mattr=-altivec -verify-machineinstrs \
+; RUN:   -mtriple=powerpc-unknown-aix < %s  | FileCheck %s --check-prefix 32BIT
+
+; RUN: llc -mcpu=pwr7 -mattr=-altivec -verify-machineinstrs \
+; RUN:   -mtriple=powerpc64-unknown-aix < %s | FileCheck %s --check-prefix 64BIT
+
+; Use an overaligned buffer to force base-pointer usage. Test verifies:
+; - base pointer register (r30) is saved/defined/restored.
+; - stack frame is allocated with correct alignment.
+; - Address of %AlignedBuffer is calculated based off offset from the stack
+;   pointer.
+
+define void @caller() {
+  %AlignedBuffer = alloca [32 x i32], align 32
+  %Pointer = getelementptr inbounds [32 x i32], [32 x i32]* %AlignedBuffer, i64 0, i64 0
+  call void @callee(i32* %Pointer)
+  ret void
+}
+
+declare void @callee(i32*)
+
+; 32BIT-LABEL: .caller:
+; 32BIT:         stw 30, -8(1)
+; 32BIT:         mr 30, 1
+; 32BIT:         clrlwi  0, 1, 27
+; 32BIT:         subfic 0, 0, -224
+; 32BIT:         stwux 1, 1, 0
+; 32BIT:         addi 3, 1, 64
+; 32BIT:         bl .callee
+; 32BIT:         lwz 1, 0(1)
+; 32BIT:         lwz 30, -8(1)
+
+; 64BIT-LABEL: .caller:
+; 64BIT:         std 30, -16(1)
+; 64BIT:         mr 30, 1
+; 64BIT:         clrldi  0, 1, 59
+; 64BIT:         subfic 0, 0, -288
+; 64BIT:         stdux 1, 1, 0
+; 64BIT:         addi 3, 1, 128
+; 64BIT:         bl .callee
+; 64BIT:         ld 1, 0(1)
+; 64BIT:         ld 30, -16(1)
diff --git a/llvm/test/CodeGen/PowerPC/aix-lower-constant-pool-index.ll b/llvm/test/CodeGen/PowerPC/aix-lower-constant-pool-index.ll
index fb7f0ad257b93..1737097d368c9 100644
--- a/llvm/test/CodeGen/PowerPC/aix-lower-constant-pool-index.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-lower-constant-pool-index.ll
@@ -47,7 +47,7 @@ entry:
 
 ; 32SMALL-ASM:         .csect .rodata[RO],2
 ; 32SMALL-ASM:         .align  2
-; 32SMALL-ASM: .LCPI0_0:
+; 32SMALL-ASM: L..CPI0_0:
 ; 32SMALL-ASM:         .vbyte	4, 0x40b00000
 ; 32SMALL-ASM: .test_float:
 ; 32SMALL-ASM:         lwz [[REG1:[0-9]+]], L..C0(2)
@@ -56,7 +56,7 @@ entry:
 
 ; 32LARGE-ASM:         .csect .rodata[RO],2
 ; 32LARGE-ASM:         .align  2
-; 32LARGE-ASM: .LCPI0_0:
+; 32LARGE-ASM: L..CPI0_0:
 ; 32LARGE-ASM:         .vbyte	4, 0x40b00000
 ; 32LARGE-ASM: .test_float:
 ; 32LARGE-ASM:         addis [[REG1:[0-9]+]], L..C0@u(2)
@@ -66,7 +66,7 @@ entry:
 
 ; 64SMALL-ASM:         .csect .rodata[RO],2
 ; 64SMALL-ASM:         .align  2
-; 64SMALL-ASM: .LCPI0_0:
+; 64SMALL-ASM: L..CPI0_0:
 ; 64SMALL-ASM:         .vbyte	4, 0x40b00000
 ; 64SMALL-ASM: .test_float:
 ; 64SMALL-ASM:         ld [[REG1:[0-9]+]], L..C0(2)
@@ -75,7 +75,7 @@ entry:
 
 ; 64LARGE-ASM:         .csect .rodata[RO],2
 ; 64LARGE-ASM:         .align  2
-; 64LARGE-ASM: .LCPI0_0:
+; 64LARGE-ASM: L..CPI0_0:
 ; 64LARGE-ASM:         .vbyte	4, 0x40b00000
 ; 64LARGE-ASM: .test_float:
 ; 64LARGE-ASM:         addis [[REG1:[0-9]+]], L..C0@u(2)
@@ -84,4 +84,4 @@ entry:
 ; 64LARGE-ASM:         blr
 
 ; CHECK: .toc
-; CHECK: .tc .LCPI0_0[TC],.LCPI0_0
+; CHECK: .tc L..CPI0_0[TC],L..CPI0_0
diff --git a/llvm/test/CodeGen/PowerPC/aix-lower-jump-table.ll b/llvm/test/CodeGen/PowerPC/aix-lower-jump-table.ll
index 5479470d1dbbe..b4a09459d2afb 100644
--- a/llvm/test/CodeGen/PowerPC/aix-lower-jump-table.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-lower-jump-table.ll
@@ -98,11 +98,11 @@
 ; 32SMALL-ASM: 	    blr
 ; 32SMALL-ASM: 	    .csect .rodata[RO],2
 ; 32SMALL-ASM: 	    .align  2
-; 32SMALL-ASM: .LJTI0_0:
-; 32SMALL-ASM: 	    .vbyte	4, L..BB0_2-.LJTI0_0
-; 32SMALL-ASM: 	    .vbyte	4, L..BB0_3-.LJTI0_0
-; 32SMALL-ASM: 	    .vbyte	4, L..BB0_4-.LJTI0_0
-; 32SMALL-ASM: 	    .vbyte	4, L..BB0_5-.LJTI0_0
+; 32SMALL-ASM: L..JTI0_0:
+; 32SMALL-ASM: 	    .vbyte	4, L..BB0_2-L..JTI0_0
+; 32SMALL-ASM: 	    .vbyte	4, L..BB0_3-L..JTI0_0
+; 32SMALL-ASM: 	    .vbyte	4, L..BB0_4-L..JTI0_0
+; 32SMALL-ASM: 	    .vbyte	4, L..BB0_5-L..JTI0_0
 
 ; 32LARGE-ASM-LABEL: jump_table
 ; 32LARGE-ASM: .jump_table:
@@ -125,11 +125,11 @@
 ; 32LARGE-ASM:      blr
 ; 32LARGE-ASM:      .csect .rodata[RO],2
 ; 32LARGE-ASM:      .align  2
-; 32LARGE-ASM: .LJTI0_0:
-; 32LARGE-ASM:      .vbyte	4, L..BB0_2-.LJTI0_0
-; 32LARGE-ASM:      .vbyte	4, L..BB0_3-.LJTI0_0
-; 32LARGE-ASM:      .vbyte	4, L..BB0_4-.LJTI0_0
-; 32LARGE-ASM:      .vbyte	4, L..BB0_5-.LJTI0_0
+; 32LARGE-ASM: L..JTI0_0:
+; 32LARGE-ASM:      .vbyte	4, L..BB0_2-L..JTI0_0
+; 32LARGE-ASM:      .vbyte	4, L..BB0_3-L..JTI0_0
+; 32LARGE-ASM:      .vbyte	4, L..BB0_4-L..JTI0_0
+; 32LARGE-ASM:      .vbyte	4, L..BB0_5-L..JTI0_0
 
 ; 64SMALL-ASM-LABEL: jump_table
 ; 64SMALL-ASM: .jump_table:
@@ -151,11 +151,11 @@
 ; 64SMALL-ASM:      blr
 ; 64SMALL-ASM:      .csect .rodata[RO],2
 ; 64SMALL-ASM:      .align  2
-; 64SMALL-ASM: .LJTI0_0:
-; 64SMALL-ASM:      .vbyte	4, L..BB0_2-.LJTI0_0
-; 64SMALL-ASM:      .vbyte	4, L..BB0_3-.LJTI0_0
-; 64SMALL-ASM:      .vbyte	4, L..BB0_4-.LJTI0_0
-; 64SMALL-ASM:      .vbyte	4, L..BB0_5-.LJTI0_0
+; 64SMALL-ASM: L..JTI0_0:
+; 64SMALL-ASM:      .vbyte	4, L..BB0_2-L..JTI0_0
+; 64SMALL-ASM:      .vbyte	4, L..BB0_3-L..JTI0_0
+; 64SMALL-ASM:      .vbyte	4, L..BB0_4-L..JTI0_0
+; 64SMALL-ASM:      .vbyte	4, L..BB0_5-L..JTI0_0
 
 ; 64LARGE-ASM-LABEL: jump_table
 ; 64LARGE-ASM: .jump_table:
@@ -178,11 +178,11 @@
 ; 64LARGE-ASM:      blr
 ; 64LARGE-ASM:      .csect .rodata[RO],2
 ; 64LARGE-ASM:      .align  2
-; 64LARGE-ASM: .LJTI0_0:
-; 64LARGE-ASM:      .vbyte	4, L..BB0_2-.LJTI0_0
-; 64LARGE-ASM:      .vbyte	4, L..BB0_3-.LJTI0_0
-; 64LARGE-ASM:      .vbyte	4, L..BB0_4-.LJTI0_0
-; 64LARGE-ASM:      .vbyte	4, L..BB0_5-.LJTI0_0
+; 64LARGE-ASM: L..JTI0_0:
+; 64LARGE-ASM:      .vbyte	4, L..BB0_2-L..JTI0_0
+; 64LARGE-ASM:      .vbyte	4, L..BB0_3-L..JTI0_0
+; 64LARGE-ASM:      .vbyte	4, L..BB0_4-L..JTI0_0
+; 64LARGE-ASM:      .vbyte	4, L..BB0_5-L..JTI0_0
 
 ; CHECK: .toc
-; CHECK: .tc .LJTI0_0[TC],.LJTI0_0
+; CHECK: .tc L..JTI0_0[TC],L..JTI0_0
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-const.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-const.ll
index 09a9c4dc80f36..312ddbf73932e 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-const.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-const.ll
@@ -25,7 +25,7 @@ entry:
 ;CHECK:         .csect .rodata[RO],4
 
 ;CHECK-NEXT:         .align  4
-;CHECK-NEXT: .L__const.main.cnst32:
+;CHECK-NEXT: L..__const.main.cnst32:
 ;CHECK32-NEXT:         .vbyte	4, 1073741824
 ;CHECK32-NEXT:         .vbyte	4, 50
 ;CHECK64-NEXT:         .vbyte	8, 4611686018427387954
@@ -38,7 +38,7 @@ entry:
 ;CHECK-NEXT:           .space  4
 
 ;CHECK-NEXT:           .align  3
-;CHECK-NEXT: .L__const.main.cnst16:
+;CHECK-NEXT: L..__const.main.cnst16:
 ;CHECK32-NEXT:         .vbyte	4, 1073741824
 ;CHECK32-NEXT:         .vbyte	4, 22
 ;CHECK64-NEXT:         .vbyte	8, 4611686018427387926
@@ -46,12 +46,12 @@ entry:
 ;CHECK-NEXT:           .space  4
 
 ;CHECK-NEXT:         .align  3
-;CHECK-NEXT: .L__const.main.cnst8:
+;CHECK-NEXT: L..__const.main.cnst8:
 ;CHECK-NEXT:         .vbyte	4, 1073741832              # 0x40000008
 ;CHECK-NEXT:         .vbyte	4, 0                       # 0x0
 
 ;CHECK-NEXT:         .align  3
-;CHECK-NEXT: .L__const.main.cnst4:
+;CHECK-NEXT: L..__const.main.cnst4:
 ;CHECK-NEXT:         .vbyte	2, 16392                   # 0x4008
 ;CHECK-NEXT:         .byte   0                       # 0x0
 ;CHECK-NEXT:         .space  1
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-str.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-str.ll
index 80b695875b495..42ead4b9b4de7 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-str.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-str.ll
@@ -27,20 +27,20 @@ entry:
 
 ; CHECK:   .csect .rodata.str2.2[RO],2
 ; CHECK-NEXT:   .align  1
-; CHECK-NEXT: .Lmagic16:
+; CHECK-NEXT: L..magic16:
 ; CHECK-NEXT:   .vbyte	2, 264                     # 0x108
 ; CHECK-NEXT:   .vbyte	2, 272                     # 0x110
 ; CHECK-NEXT:   .vbyte	2, 213                     # 0xd5
 ; CHECK-NEXT:   .vbyte	2, 0                       # 0x0
 ; CHECK-NEXT:   .csect .rodata.str4.4[RO],2
 ; CHECK-NEXT:   .align  2
-; CHECK-NEXT: .Lmagic32:
+; CHECK-NEXT: L..magic32:
 ; CHECK-NEXT:   .vbyte	4, 464                     # 0x1d0
 ; CHECK-NEXT:   .vbyte	4, 472                     # 0x1d8
 ; CHECK-NEXT:   .vbyte	4, 413                     # 0x19d
 ; CHECK-NEXT:   .vbyte	4, 0                       # 0x0
 ; CHECK-NEXT:   .csect .rodata.str1.1[RO],2
-; CHECK-NEXT: .LstrA:
+; CHECK-NEXT: L..strA:
 ; CHECK-NEXT: .byte   104
 ; CHECK-NEXT: .byte   101
 ; CHECK-NEXT: .byte   108
@@ -55,7 +55,7 @@ entry:
 ; CHECK-NEXT: .byte   33
 ; CHECK-NEXT: .byte   10
 ; CHECK-NEXT: .byte   0
-; CHECK-NEXT: .L.str:
+; CHECK-NEXT: L...str:
 ; CHECK-NEXT: .byte   97
 ; CHECK-NEXT: .byte   98
 ; CHECK-NEXT: .byte   99
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-symbol-rename.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-symbol-rename.ll
new file mode 100644
index 0000000000000..ee3f9da68dadb
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-symbol-rename.ll
@@ -0,0 +1,161 @@
+;; This file tests how llc handles symbols containing invalid characters on an
+;; XCOFF platform.
+;; Since symbol name resolution is the same between 32-bit and 64-bit,
+;; tests for 64-bit mode are omitted.
+
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
+; RUN:     -mattr=-altivec < %s | \
+; RUN:   FileCheck --check-prefix=ASM %s
+
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
+; RUN:     -mattr=-altivec -filetype=obj -o %t.o < %s
+; RUN: llvm-objdump -D -r --symbol-description %t.o | \
+; RUN:   FileCheck --check-prefix=OBJ %s
+
+; This is f`o
+@"f\60o" = global i32 10, align 4
+
+; This is f"o"
+@"f\22o\22" = common global i32 0, align 4
+
+define internal i32 @f$o() {
+entry:
+  %call = call i32 bitcast (i32 (...)* @"f\40o" to i32 ()*)()
+  ret i32 %call
+}
+
+; This is f&o
+define i32 @"f\26o"() {
+entry:
+  %call = call i32 @f$o()
+  ret i32 %call
+}
+
+; This is f&_o
+define i32 (...)* @"f\26_o"() {
+entry:
+  ret i32 (...)* @"f\40o"
+}
+
+; This is f@o
+declare i32 @"f\40o"(...)
+
+; ASM:         .rename _Renamed..24f_o[DS],"f$o" # -- Begin function f$o
+; ASM-NEXT:    .lglobl _Renamed..24f_o[DS]
+; ASM-NEXT:    .rename ._Renamed..24f_o,".f$o"
+; ASM-NEXT:    .lglobl ._Renamed..24f_o
+; ASM-NEXT:    .align  4
+; ASM-NEXT:    .csect _Renamed..24f_o[DS],2
+; ASM-NEXT:    .vbyte  4, ._Renamed..24f_o     # @"f$o"
+; ASM-NEXT:    .vbyte  4, TOC[TC0]
+; ASM-NEXT:    .vbyte  4, 0
+; ASM-NEXT:    .csect .text[PR],2
+; ASM-NEXT:  ._Renamed..24f_o:
+; ASM:         bl ._Renamed..40f_o
+; ASM-NEXT:    nop
+; ASM:         .rename _Renamed..26f_o[DS],"f&o" # -- Begin function f&o
+; ASM-NEXT:    .globl  _Renamed..26f_o[DS]
+; ASM-NEXT:    .rename ._Renamed..26f_o,".f&o"
+; ASM-NEXT:    .globl  ._Renamed..26f_o
+; ASM-NEXT:    .align  4
+; ASM-NEXT:    .csect _Renamed..26f_o[DS],2
+; ASM-NEXT:    .vbyte  4, ._Renamed..26f_o     # @"f&o"
+; ASM-NEXT:    .vbyte  4, TOC[TC0]
+; ASM-NEXT:    .vbyte  4, 0
+; ASM-NEXT:    .csect .text[PR],2
+; ASM-NEXT:  ._Renamed..26f_o:
+; ASM:         bl ._Renamed..24f_o
+; ASM:         .rename _Renamed..265ff__o[DS],"f&_o" # -- Begin function f&_o
+; ASM-NEXT:    .globl  _Renamed..265ff__o[DS]
+; ASM-NEXT:    .rename ._Renamed..265ff__o,".f&_o"
+; ASM-NEXT:    .globl  ._Renamed..265ff__o
+; ASM-NEXT:    .align  4
+; ASM-NEXT:    .csect _Renamed..265ff__o[DS],2
+; ASM-NEXT:    .vbyte  4, ._Renamed..265ff__o  # @"f&_o"
+; ASM-NEXT:    .vbyte  4, TOC[TC0]
+; ASM-NEXT:    .vbyte  4, 0
+; ASM-NEXT:    .csect .text[PR],2
+; ASM-NEXT:  ._Renamed..265ff__o:
+; ASM:         .csect .data[RW],2
+; ASM-NEXT:    .rename _Renamed..60f_o,"f`o"
+; ASM-NEXT:    .globl  _Renamed..60f_o
+; ASM-NEXT:    .align  2
+; ASM-NEXT:  _Renamed..60f_o:
+; ASM-NEXT:    .vbyte  4, 10                   # 0xa
+; ASM-NEXT:    .rename _Renamed..2222f_o_[RW],"f""o"""
+; ASM-NEXT:    .comm _Renamed..2222f_o_[RW],4,2
+; ASM-NEXT:    .rename ._Renamed..40f_o,".f@o"
+; ASM-NEXT:    .extern ._Renamed..40f_o
+; ASM-NEXT:    .rename _Renamed..40f_o[DS],"f@o"
+; ASM-NEXT:    .extern _Renamed..40f_o[DS]
+; ASM-NEXT:    .toc
+; ASM-NEXT:  L..C0:
+; ASM-NEXT:    .rename _Renamed..40f_o[TC],"f@o"
+; ASM-NEXT:    .tc _Renamed..40f_o[TC],_Renamed..40f_o[DS]
+
+; OBJ:       Disassembly of section .text:
+; OBJ-EMPTY:
+; OBJ-NEXT:  00000000 (idx: 6) .f$o:
+; OBJ-NEXT:         0: 7c 08 02 a6   mflr 0
+; OBJ-NEXT:         4: 90 01 00 08   stw 0, 8(1)
+; OBJ-NEXT:         8: 94 21 ff c0   stwu 1, -64(1)
+; OBJ-NEXT:         c: 4b ff ff f5   bl 0x0
+; OBJ-NEXT:                          0000000c:  R_RBR        (idx: 0) .f@o[PR]
+; OBJ-NEXT:        10: 60 00 00 00   nop
+; OBJ-NEXT:        14: 38 21 00 40   addi 1, 1, 64
+; OBJ-NEXT:        18: 80 01 00 08   lwz 0, 8(1)
+; OBJ-NEXT:        1c: 7c 08 03 a6   mtlr 0
+; OBJ-NEXT:        20: 4e 80 00 20   blr
+; OBJ-NEXT:        24: 60 00 00 00   nop
+; OBJ-NEXT:        28: 60 00 00 00   nop
+; OBJ-NEXT:        2c: 60 00 00 00   nop
+; OBJ-EMPTY:
+; OBJ-NEXT:  00000030 (idx: 8) .f&o:
+; OBJ-NEXT:        30: 7c 08 02 a6   mflr 0
+; OBJ-NEXT:        34: 90 01 00 08   stw 0, 8(1)
+; OBJ-NEXT:        38: 94 21 ff c0   stwu 1, -64(1)
+; OBJ-NEXT:        3c: 4b ff ff c5   bl 0x0
+; OBJ-NEXT:        40: 38 21 00 40   addi 1, 1, 64
+; OBJ-NEXT:        44: 80 01 00 08   lwz 0, 8(1)
+; OBJ-NEXT:        48: 7c 08 03 a6   mtlr 0
+; OBJ-NEXT:        4c: 4e 80 00 20   blr
+; OBJ-EMPTY:
+; OBJ-NEXT:  00000050 (idx: 10) .f&_o:
+; OBJ-NEXT:        50: 80 62 00 00   lwz 3, 0(2)
+; OBJ-NEXT:                          00000052:  R_TOC        (idx: 24) f@o[TC]
+; OBJ-NEXT:        54: 4e 80 00 20   blr
+; OBJ-EMPTY:
+; OBJ-NEXT:  Disassembly of section .data:
+; OBJ-EMPTY:
+; OBJ-NEXT:  00000058 (idx: 14) f`o:
+; OBJ-NEXT:        58: 00 00 00 0a   <unknown>
+; OBJ-EMPTY:
+; OBJ-NEXT:  0000005c (idx: 16) f$o[DS]:
+; OBJ-NEXT:        5c: 00 00 00 00   <unknown>
+; OBJ-NEXT:                          0000005c:  R_POS        (idx: 6) .f$o
+; OBJ-NEXT:        60: 00 00 00 80   <unknown>
+; OBJ-NEXT:                          00000060:  R_POS        (idx: 22) TOC[TC0]
+; OBJ-NEXT:        64: 00 00 00 00   <unknown>
+; OBJ-EMPTY:
+; OBJ-NEXT:  00000068 (idx: 18) f&o[DS]:
+; OBJ-NEXT:        68: 00 00 00 30   <unknown>
+; OBJ-NEXT:                          00000068:  R_POS        (idx: 8) .f&o
+; OBJ-NEXT:        6c: 00 00 00 80   <unknown>
+; OBJ-NEXT:                          0000006c:  R_POS        (idx: 22) TOC[TC0]
+; OBJ-NEXT:        70: 00 00 00 00   <unknown>
+; OBJ-EMPTY:
+; OBJ-NEXT:  00000074 (idx: 20) f&_o[DS]:
+; OBJ-NEXT:        74: 00 00 00 50   <unknown>
+; OBJ-NEXT:                          00000074:  R_POS        (idx: 10) .f&_o
+; OBJ-NEXT:        78: 00 00 00 80   <unknown>
+; OBJ-NEXT:                          00000078:  R_POS        (idx: 22) TOC[TC0]
+; OBJ-NEXT:        7c: 00 00 00 00   <unknown>
+; OBJ-EMPTY:
+; OBJ-NEXT:  00000080 (idx: 24) f@o[TC]:
+; OBJ-NEXT:        80: 00 00 00 00   <unknown>
+; OBJ-NEXT:                          00000080:  R_POS        (idx: 2) f@o[DS]
+; OBJ-EMPTY:
+; OBJ-NEXT:  Disassembly of section .bss:
+; OBJ-EMPTY:
+; OBJ-NEXT:  00000084 (idx: 26) f"o"[RW]:
+; OBJ-NEXT:  ...
diff --git a/llvm/test/CodeGen/PowerPC/atomics-constant.ll b/llvm/test/CodeGen/PowerPC/atomics-constant.ll
index ac0b16b55f7c1..cc442898ab8f4 100644
--- a/llvm/test/CodeGen/PowerPC/atomics-constant.ll
+++ b/llvm/test/CodeGen/PowerPC/atomics-constant.ll
@@ -8,11 +8,10 @@ target triple = "powerpc64le-unknown-linux-gnu"
 define i64 @foo() {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, a@toc@ha
 ; CHECK-NEXT:    li 4, 0
-; CHECK-NEXT:    addi 3, 3, a@toc@l
+; CHECK-NEXT:    addis 3, 2, a@toc@ha
+; CHECK-NEXT:    ld 3, a@toc@l(3)
 ; CHECK-NEXT:    cmpd 7, 4, 4
-; CHECK-NEXT:    ld 3, 0(3)
 ; CHECK-NEXT:    li 3, 0
 ; CHECK-NEXT:    bne- 7, .+4
 ; CHECK-NEXT:    isync
diff --git a/llvm/test/CodeGen/PowerPC/builtins-ppc-p10permute.ll b/llvm/test/CodeGen/PowerPC/builtins-ppc-p10permute.ll
new file mode 100644
index 0000000000000..84bf4032aa34f
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/builtins-ppc-p10permute.ll
@@ -0,0 +1,255 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -ppc-asm-full-reg-names -mcpu=pwr10 \
+; RUN:   -ppc-vsr-nums-as-vr -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -ppc-asm-full-reg-names -mcpu=pwr10 \
+; RUN:   -ppc-vsr-nums-as-vr -mtriple=powerpc64-unknown-unknown < %s | FileCheck %s
+
+define <16 x i8> @testVSLDBI(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: testVSLDBI:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsldbi v2, v2, v3, 1
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <16 x i8> @llvm.ppc.altivec.vsldbi(<16 x i8> %a, <16 x i8> %b, i32 1)
+  ret <16 x i8> %0
+}
+declare <16 x i8> @llvm.ppc.altivec.vsldbi(<16 x i8>, <16 x i8>, i32 immarg)
+
+define <16 x i8> @testVSRDBI(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: testVSRDBI:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsrdbi v2, v2, v3, 1
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <16 x i8> @llvm.ppc.altivec.vsrdbi(<16 x i8> %a, <16 x i8> %b, i32 1)
+  ret <16 x i8> %0
+}
+declare <16 x i8> @llvm.ppc.altivec.vsrdbi(<16 x i8>, <16 x i8>, i32 immarg)
+
+define <16 x i8> @testXXPERMX(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: testXXPERMX:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxpermx v2, v2, v3, v4, 1
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <16 x i8> @llvm.ppc.vsx.xxpermx(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, i32 1)
+  ret <16 x i8> %0
+}
+declare <16 x i8> @llvm.ppc.vsx.xxpermx(<16 x i8>, <16 x i8>, <16 x i8>, i32 immarg)
+
+define <16 x i8> @testXXBLENDVB(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
+; CHECK-LABEL: testXXBLENDVB:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxblendvb v2, v2, v3, v4
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <16 x i8> @llvm.ppc.vsx.xxblendvb(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c)
+  ret <16 x i8> %0
+}
+declare <16 x i8> @llvm.ppc.vsx.xxblendvb(<16 x i8>, <16 x i8>, <16 x i8>)
+
+define <8 x i16> @testXXBLENDVH(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
+; CHECK-LABEL: testXXBLENDVH:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxblendvh v2, v2, v3, v4
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <8 x i16> @llvm.ppc.vsx.xxblendvh(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c)
+  ret <8 x i16> %0
+}
+declare <8 x i16> @llvm.ppc.vsx.xxblendvh(<8 x i16>, <8 x i16>, <8 x i16>)
+
+define <4 x i32> @testXXBLENDVW(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: testXXBLENDVW:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxblendvw v2, v2, v3, v4
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <4 x i32> @llvm.ppc.vsx.xxblendvw(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c)
+  ret <4 x i32> %0
+}
+declare <4 x i32> @llvm.ppc.vsx.xxblendvw(<4 x i32>, <4 x i32>, <4 x i32>)
+
+define <2 x i64> @testXXBLENDVD(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
+; CHECK-LABEL: testXXBLENDVD:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxblendvd v2, v2, v3, v4
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <2 x i64> @llvm.ppc.vsx.xxblendvd(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c)
+  ret <2 x i64> %0
+}
+declare <2 x i64> @llvm.ppc.vsx.xxblendvd(<2 x i64>, <2 x i64>, <2 x i64>)
+
+define <16 x i8> @testVINSBLX(<16 x i8> %a, i64 %b, i64 %c) {
+; CHECK-LABEL: testVINSBLX:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vinsblx v2, r5, r6
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <16 x i8> @llvm.ppc.altivec.vinsblx(<16 x i8> %a, i64 %b, i64 %c)
+  ret <16 x i8> %0
+}
+declare <16 x i8> @llvm.ppc.altivec.vinsblx(<16 x i8>, i64, i64)
+
+define <16 x i8> @testVINSBRX(<16 x i8> %a, i64 %b, i64 %c) {
+; CHECK-LABEL: testVINSBRX:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vinsbrx v2, r5, r6
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <16 x i8> @llvm.ppc.altivec.vinsbrx(<16 x i8> %a, i64 %b, i64 %c)
+  ret <16 x i8> %0
+}
+declare <16 x i8> @llvm.ppc.altivec.vinsbrx(<16 x i8>, i64, i64)
+
+define <8 x i16> @testVINSHLX(<8 x i16> %a, i64 %b, i64 %c) {
+; CHECK-LABEL: testVINSHLX:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vinshlx v2, r5, r6
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <8 x i16> @llvm.ppc.altivec.vinshlx(<8 x i16> %a, i64 %b, i64 %c)
+  ret <8 x i16> %0
+}
+declare <8 x i16> @llvm.ppc.altivec.vinshlx(<8 x i16>, i64, i64)
+
+define <8 x i16> @testVINSHRX(<8 x i16> %a, i64 %b, i64 %c) {
+; CHECK-LABEL: testVINSHRX:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vinshrx v2, r5, r6
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <8 x i16> @llvm.ppc.altivec.vinshrx(<8 x i16> %a, i64 %b, i64 %c)
+  ret <8 x i16> %0
+}
+declare <8 x i16> @llvm.ppc.altivec.vinshrx(<8 x i16>, i64, i64)
+
+define <4 x i32> @testVINSWLX(<4 x i32> %a, i64 %b, i64 %c) {
+; CHECK-LABEL: testVINSWLX:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vinswlx v2, r5, r6
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <4 x i32> @llvm.ppc.altivec.vinswlx(<4 x i32> %a, i64 %b, i64 %c)
+  ret <4 x i32> %0
+}
+declare <4 x i32> @llvm.ppc.altivec.vinswlx(<4 x i32>, i64, i64)
+
+define <4 x i32> @testVINSWRX(<4 x i32> %a, i64 %b, i64 %c) {
+; CHECK-LABEL: testVINSWRX:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vinswrx v2, r5, r6
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <4 x i32> @llvm.ppc.altivec.vinswrx(<4 x i32> %a, i64 %b, i64 %c)
+  ret <4 x i32> %0
+}
+declare <4 x i32> @llvm.ppc.altivec.vinswrx(<4 x i32>, i64, i64)
+
+define <2 x i64> @testVINSDLX(<2 x i64> %a, i64 %b, i64 %c) {
+; CHECK-LABEL: testVINSDLX:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vinsdlx v2, r5, r6
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <2 x i64> @llvm.ppc.altivec.vinsdlx(<2 x i64> %a, i64 %b, i64 %c)
+  ret <2 x i64> %0
+}
+declare <2 x i64> @llvm.ppc.altivec.vinsdlx(<2 x i64>, i64, i64)
+
+define <2 x i64> @testVINSDRX(<2 x i64> %a, i64 %b, i64 %c) {
+; CHECK-LABEL: testVINSDRX:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vinsdrx v2, r5, r6
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <2 x i64> @llvm.ppc.altivec.vinsdrx(<2 x i64> %a, i64 %b, i64 %c)
+  ret <2 x i64> %0
+}
+declare <2 x i64> @llvm.ppc.altivec.vinsdrx(<2 x i64>, i64, i64)
+
+define <16 x i8> @testVINSBVLX(<16 x i8> %a, i64 %b, <16 x i8> %c) {
+; CHECK-LABEL: testVINSBVLX:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vinsbvlx v2, r5, v3
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <16 x i8> @llvm.ppc.altivec.vinsbvlx(<16 x i8> %a, i64 %b, <16 x i8> %c)
+  ret <16 x i8> %0
+}
+declare <16 x i8> @llvm.ppc.altivec.vinsbvlx(<16 x i8>, i64, <16 x i8>)
+
+define <16 x i8> @testVINSBVRX(<16 x i8> %a, i64 %b, <16 x i8> %c) {
+; CHECK-LABEL: testVINSBVRX:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vinsbvrx v2, r5, v3
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <16 x i8> @llvm.ppc.altivec.vinsbvrx(<16 x i8> %a, i64 %b, <16 x i8> %c)
+  ret <16 x i8> %0
+}
+declare <16 x i8> @llvm.ppc.altivec.vinsbvrx(<16 x i8>, i64, <16 x i8>)
+
+define <8 x i16> @testVINSHVLX(<8 x i16> %a, i64 %b, <8 x i16> %c) {
+; CHECK-LABEL: testVINSHVLX:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vinshvlx v2, r5, v3
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <8 x i16> @llvm.ppc.altivec.vinshvlx(<8 x i16> %a, i64 %b, <8 x i16> %c)
+  ret <8 x i16> %0
+}
+declare <8 x i16> @llvm.ppc.altivec.vinshvlx(<8 x i16>, i64, <8 x i16>)
+
+define <8 x i16> @testVINSHVRX(<8 x i16> %a, i64 %b, <8 x i16> %c) {
+entry:
+  %0 = tail call <8 x i16> @llvm.ppc.altivec.vinshvrx(<8 x i16> %a, i64 %b, <8 x i16> %c)
+  ret <8 x i16> %0
+}
+declare <8 x i16> @llvm.ppc.altivec.vinshvrx(<8 x i16>, i64, <8 x i16>)
+
+define <4 x i32> @testVINSWVLX(<4 x i32> %a, i64 %b, <4 x i32> %c) {
+; CHECK-LABEL: testVINSWVLX:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vinswvlx v2, r5, v3
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <4 x i32> @llvm.ppc.altivec.vinswvlx(<4 x i32> %a, i64 %b, <4 x i32> %c)
+  ret <4 x i32> %0
+}
+declare <4 x i32> @llvm.ppc.altivec.vinswvlx(<4 x i32>, i64, <4 x i32>)
+
+define <4 x i32> @testVINSWVRX(<4 x i32> %a, i64 %b, <4 x i32> %c) {
+; CHECK-LABEL: testVINSWVRX:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vinswvrx v2, r5, v3
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <4 x i32> @llvm.ppc.altivec.vinswvrx(<4 x i32> %a, i64 %b, <4 x i32> %c)
+  ret <4 x i32> %0
+}
+declare <4 x i32> @llvm.ppc.altivec.vinswvrx(<4 x i32>, i64, <4 x i32>)
+
+define <4 x i32> @testVINSW(<4 x i32> %a, i64 %b) {
+; CHECK-LABEL: testVINSW:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vinsw v2, r5, 1
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <4 x i32> @llvm.ppc.altivec.vinsw(<4 x i32> %a, i64 %b, i32 1)
+  ret <4 x i32> %0
+}
+declare <4 x i32> @llvm.ppc.altivec.vinsw(<4 x i32>, i64, i32 immarg)
+
+define <2 x i64> @testVINSD(<2 x i64> %a, i64 %b) {
+; CHECK-LABEL: testVINSD:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vinsd v2, r5, 1
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <2 x i64> @llvm.ppc.altivec.vinsd(<2 x i64> %a, i64 %b, i32 1)
+  ret <2 x i64> %0
+}
+declare <2 x i64> @llvm.ppc.altivec.vinsd(<2 x i64>, i64, i32 immarg)
diff --git a/llvm/test/CodeGen/PowerPC/builtins-ppc-p9-f128.ll b/llvm/test/CodeGen/PowerPC/builtins-ppc-p9-f128.ll
index 366493ae76b26..bd8d6099c40f6 100644
--- a/llvm/test/CodeGen/PowerPC/builtins-ppc-p9-f128.ll
+++ b/llvm/test/CodeGen/PowerPC/builtins-ppc-p9-f128.ll
@@ -1,6 +1,5 @@
-; RUN: llc -verify-machineinstrs -mcpu=pwr9 -enable-ppc-quad-precision \
-; RUN:   -mtriple=powerpc64le-unknown-unknown -ppc-vsr-nums-as-vr \
-; RUN:   -ppc-asm-full-reg-names < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names < %s | FileCheck %s
 
 @A = common global fp128 0xL00000000000000000000000000000000, align 16
 @B = common global fp128 0xL00000000000000000000000000000000, align 16
diff --git a/llvm/test/CodeGen/PowerPC/canonical-merge-shuffles.ll b/llvm/test/CodeGen/PowerPC/canonical-merge-shuffles.ll
index 7fac0511e3c51..ada7c73cd9ed5 100644
--- a/llvm/test/CodeGen/PowerPC/canonical-merge-shuffles.ll
+++ b/llvm/test/CodeGen/PowerPC/canonical-merge-shuffles.ll
@@ -5,6 +5,9 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
 ; RUN:     -mcpu=pwr9 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
 ; RUN: FileCheck %s --check-prefix=CHECK-P9
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:     -mcpu=pwr8 -mattr=-vsx -ppc-asm-full-reg-names \
+; RUN:     -ppc-vsr-nums-as-vr < %s | FileCheck %s --check-prefix=CHECK-NOVSX
 
 define dso_local <16 x i8> @testmrghb(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr #0 {
 ; CHECK-P8-LABEL: testmrghb:
@@ -16,6 +19,11 @@ define dso_local <16 x i8> @testmrghb(<16 x i8> %a, <16 x i8> %b) local_unnamed_
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    vmrghb v2, v3, v2
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: testmrghb:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    vmrghb v2, v3, v2
+; CHECK-NOVSX-NEXT:    blr
 entry:
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
   ret <16 x i8> %shuffle
@@ -30,6 +38,14 @@ define dso_local <16 x i8> @testmrghb2(<16 x i8> %a, <16 x i8> %b) local_unnamed
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    vmrghb v2, v2, v3
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: testmrghb2:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    addis r3, r2, .LCPI1_0@toc@ha
+; CHECK-NOVSX-NEXT:    addi r3, r3, .LCPI1_0@toc@l
+; CHECK-NOVSX-NEXT:    lvx v4, 0, r3
+; CHECK-NOVSX-NEXT:    vperm v2, v3, v2, v4
+; CHECK-NOVSX-NEXT:    blr
 entry:
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 24, i32 8, i32 25, i32 9, i32 26, i32 10, i32 27, i32 11, i32 28, i32 12, i32 29, i32 13, i32 30, i32 14, i32 31, i32 15>
   ret <16 x i8> %shuffle
@@ -44,6 +60,11 @@ define dso_local <16 x i8> @testmrghh(<16 x i8> %a, <16 x i8> %b) local_unnamed_
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    vmrghh v2, v3, v2
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: testmrghh:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    vmrghh v2, v3, v2
+; CHECK-NOVSX-NEXT:    blr
 entry:
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 8, i32 9, i32 24, i32 25, i32 10, i32 11, i32 26, i32 27, i32 12, i32 13, i32 28, i32 29, i32 14, i32 15, i32 30, i32 31>
   ret <16 x i8> %shuffle
@@ -58,6 +79,14 @@ define dso_local <16 x i8> @testmrghh2(<16 x i8> %a, <16 x i8> %b) local_unnamed
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    vmrghh v2, v2, v3
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: testmrghh2:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    addis r3, r2, .LCPI3_0@toc@ha
+; CHECK-NOVSX-NEXT:    addi r3, r3, .LCPI3_0@toc@l
+; CHECK-NOVSX-NEXT:    lvx v4, 0, r3
+; CHECK-NOVSX-NEXT:    vperm v2, v3, v2, v4
+; CHECK-NOVSX-NEXT:    blr
 entry:
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 24, i32 25, i32 8, i32 9, i32 26, i32 27, i32 10, i32 11, i32 28, i32 29, i32 12, i32 13, i32 30, i32 31, i32 14, i32 15>
   ret <16 x i8> %shuffle
@@ -72,6 +101,11 @@ define dso_local <16 x i8> @testmrglb(<16 x i8> %a, <16 x i8> %b) local_unnamed_
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    vmrglb v2, v3, v2
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: testmrglb:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    vmrglb v2, v3, v2
+; CHECK-NOVSX-NEXT:    blr
 entry:
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
   ret <16 x i8> %shuffle
@@ -86,6 +120,14 @@ define dso_local <16 x i8> @testmrglb2(<16 x i8> %a, <16 x i8> %b) local_unnamed
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    vmrglb v2, v2, v3
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: testmrglb2:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    addis r3, r2, .LCPI5_0@toc@ha
+; CHECK-NOVSX-NEXT:    addi r3, r3, .LCPI5_0@toc@l
+; CHECK-NOVSX-NEXT:    lvx v4, 0, r3
+; CHECK-NOVSX-NEXT:    vperm v2, v3, v2, v4
+; CHECK-NOVSX-NEXT:    blr
 entry:
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 16, i32 0, i32 17, i32 1, i32 18, i32 2, i32 19, i32 3, i32 20, i32 4, i32 21, i32 5, i32 22, i32 6, i32 23, i32 7>
   ret <16 x i8> %shuffle
@@ -100,6 +142,11 @@ define dso_local <16 x i8> @testmrglh(<16 x i8> %a, <16 x i8> %b) local_unnamed_
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    vmrglh v2, v3, v2
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: testmrglh:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    vmrglh v2, v3, v2
+; CHECK-NOVSX-NEXT:    blr
 entry:
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 2, i32 3, i32 18, i32 19, i32 4, i32 5, i32 20, i32 21, i32 6, i32 7, i32 22, i32 23>
   ret <16 x i8> %shuffle
@@ -114,6 +161,14 @@ define dso_local <16 x i8> @testmrglh2(<16 x i8> %a, <16 x i8> %b) local_unnamed
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    vmrglh v2, v2, v3
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: testmrglh2:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    addis r3, r2, .LCPI7_0@toc@ha
+; CHECK-NOVSX-NEXT:    addi r3, r3, .LCPI7_0@toc@l
+; CHECK-NOVSX-NEXT:    lvx v4, 0, r3
+; CHECK-NOVSX-NEXT:    vperm v2, v3, v2, v4
+; CHECK-NOVSX-NEXT:    blr
 entry:
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 16, i32 17, i32 0, i32 1, i32 18, i32 19, i32 2, i32 3, i32 20, i32 21, i32 4, i32 5, i32 22, i32 23, i32 6, i32 7>
   ret <16 x i8> %shuffle
@@ -128,6 +183,11 @@ define dso_local <16 x i8> @testmrghw(<16 x i8> %a, <16 x i8> %b) local_unnamed_
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    vmrghw v2, v3, v2
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: testmrghw:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    vmrghw v2, v3, v2
+; CHECK-NOVSX-NEXT:    blr
 entry:
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 24, i32 25, i32 26, i32 27, i32 12, i32 13, i32 14, i32 15, i32 28, i32 29, i32 30, i32 31>
   ret <16 x i8> %shuffle
@@ -142,6 +202,14 @@ define dso_local <16 x i8> @testmrghw2(<16 x i8> %a, <16 x i8> %b) local_unnamed
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    vmrghw v2, v2, v3
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: testmrghw2:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    addis r3, r2, .LCPI9_0@toc@ha
+; CHECK-NOVSX-NEXT:    addi r3, r3, .LCPI9_0@toc@l
+; CHECK-NOVSX-NEXT:    lvx v4, 0, r3
+; CHECK-NOVSX-NEXT:    vperm v2, v3, v2, v4
+; CHECK-NOVSX-NEXT:    blr
 entry:
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 24, i32 25, i32 26, i32 27, i32 8, i32 9, i32 10, i32 11, i32 28, i32 29, i32 30, i32 31, i32 12, i32 13, i32 14, i32 15>
   ret <16 x i8> %shuffle
@@ -156,6 +224,11 @@ define dso_local <16 x i8> @testmrglw(<16 x i8> %a, <16 x i8> %b) local_unnamed_
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: testmrglw:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    vmrglw v2, v3, v2
+; CHECK-NOVSX-NEXT:    blr
 entry:
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23>
   ret <16 x i8> %shuffle
@@ -170,6 +243,14 @@ define dso_local <16 x i8> @testmrglw2(<16 x i8> %a, <16 x i8> %b) local_unnamed
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    vmrglw v2, v2, v3
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: testmrglw2:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    addis r3, r2, .LCPI11_0@toc@ha
+; CHECK-NOVSX-NEXT:    addi r3, r3, .LCPI11_0@toc@l
+; CHECK-NOVSX-NEXT:    lvx v4, 0, r3
+; CHECK-NOVSX-NEXT:    vperm v2, v3, v2, v4
+; CHECK-NOVSX-NEXT:    blr
 entry:
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 0, i32 1, i32 2, i32 3, i32 20, i32 21, i32 22, i32 23, i32 4, i32 5, i32 6, i32 7>
   ret <16 x i8> %shuffle
@@ -190,10 +271,153 @@ define dso_local <8 x i16> @testmrglb3(<8 x i8>* nocapture readonly %a) local_un
 ; CHECK-P9-NEXT:    xxlxor v3, v3, v3
 ; CHECK-P9-NEXT:    vmrghb v2, v3, v2
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: testmrglb3:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    vxor v2, v2, v2
+; CHECK-NOVSX-NEXT:    ld r3, 0(r3)
+; CHECK-NOVSX-NEXT:    addis r4, r2, .LCPI12_0@toc@ha
+; CHECK-NOVSX-NEXT:    addi r4, r4, .LCPI12_0@toc@l
+; CHECK-NOVSX-NEXT:    lvx v3, 0, r4
+; CHECK-NOVSX-NEXT:    std r3, -16(r1)
+; CHECK-NOVSX-NEXT:    addi r3, r1, -16
+; CHECK-NOVSX-NEXT:    lvx v4, 0, r3
+; CHECK-NOVSX-NEXT:    vperm v2, v4, v2, v3
+; CHECK-NOVSX-NEXT:    blr
 entry:
   %0 = load <8 x i8>, <8 x i8>* %a, align 8
   %1 = zext <8 x i8> %0 to <8 x i16>
   ret <8 x i16> %1
 }
 
+define dso_local void @no_crash_elt0_from_RHS(<2 x double>* noalias nocapture dereferenceable(16) %.vtx6) #0 {
+; CHECK-P8-LABEL: no_crash_elt0_from_RHS:
+; CHECK-P8:       # %bb.0: # %test_entry
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; CHECK-P8-NEXT:    std r0, 16(r1)
+; CHECK-P8-NEXT:    stdu r1, -48(r1)
+; CHECK-P8-NEXT:    mr r30, r3
+; CHECK-P8-NEXT:    bl dummy
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    xxlxor f0, f0, f0
+; CHECK-P8-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
+; CHECK-P8-NEXT:    xxswapd vs0, vs0
+; CHECK-P8-NEXT:    stxvd2x vs0, 0, r30
+;
+; CHECK-P9-LABEL: no_crash_elt0_from_RHS:
+; CHECK-P9:       # %bb.0: # %test_entry
+; CHECK-P9-NEXT:    mflr r0
+; CHECK-P9-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; CHECK-P9-NEXT:    std r0, 16(r1)
+; CHECK-P9-NEXT:    stdu r1, -48(r1)
+; CHECK-P9-NEXT:    mr r30, r3
+; CHECK-P9-NEXT:    bl dummy
+; CHECK-P9-NEXT:    nop
+; CHECK-P9-NEXT:    xxlxor f0, f0, f0
+; CHECK-P9-NEXT:    # kill: def $f1 killed $f1 def $vsl1
+; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
+; CHECK-P9-NEXT:    stxv vs0, 0(r30)
+;
+; CHECK-NOVSX-LABEL: no_crash_elt0_from_RHS:
+; CHECK-NOVSX:       # %bb.0: # %test_entry
+; CHECK-NOVSX-NEXT:    mflr r0
+; CHECK-NOVSX-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; CHECK-NOVSX-NEXT:    std r0, 16(r1)
+; CHECK-NOVSX-NEXT:    stdu r1, -48(r1)
+; CHECK-NOVSX-NEXT:    mr r30, r3
+; CHECK-NOVSX-NEXT:    bl dummy
+; CHECK-NOVSX-NEXT:    nop
+; CHECK-NOVSX-NEXT:    li r3, 0
+; CHECK-NOVSX-NEXT:    stfd f1, 8(r30)
+; CHECK-NOVSX-NEXT:    std r3, 0(r30)
+test_entry:
+  %_div_result = tail call double @dummy()
+  %oldret = insertvalue { double, double } undef, double %_div_result, 0
+  %0 = extractvalue { double, double } %oldret, 0
+  %.splatinsert = insertelement <2 x double> undef, double %0, i32 0
+  %.splat = shufflevector <2 x double> %.splatinsert, <2 x double> undef, <2 x i32> zeroinitializer
+  %1 = shufflevector <2 x double> zeroinitializer, <2 x double> %.splat, <2 x i32> <i32 0, i32 3>
+  store <2 x double> %1, <2 x double>* %.vtx6, align 16
+  unreachable
+}
+
+define dso_local <4 x i32> @replace_undefs_in_splat(<4 x i32> %a) local_unnamed_addr #0 {
+; CHECK-P8-LABEL: replace_undefs_in_splat:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI14_0@toc@ha
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI14_0@toc@l
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    vmrgow v2, v3, v2
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: replace_undefs_in_splat:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    addis r3, r2, .LCPI14_0@toc@ha
+; CHECK-P9-NEXT:    addi r3, r3, .LCPI14_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r3
+; CHECK-P9-NEXT:    vmrgow v2, v3, v2
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: replace_undefs_in_splat:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    addis r3, r2, .LCPI14_0@toc@ha
+; CHECK-NOVSX-NEXT:    addis r4, r2, .LCPI14_1@toc@ha
+; CHECK-NOVSX-NEXT:    addi r3, r3, .LCPI14_0@toc@l
+; CHECK-NOVSX-NEXT:    lvx v3, 0, r3
+; CHECK-NOVSX-NEXT:    addi r3, r4, .LCPI14_1@toc@l
+; CHECK-NOVSX-NEXT:    lvx v4, 0, r3
+; CHECK-NOVSX-NEXT:    vperm v2, v4, v2, v3
+; CHECK-NOVSX-NEXT:    blr
+entry:
+  %vecins1 = shufflevector <4 x i32> %a, <4 x i32> <i32 undef, i32 566, i32 undef, i32 566>, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x i32> %vecins1
+}
+
+define dso_local <16 x i8> @no_RAUW_in_combine_during_legalize(i32* nocapture readonly %ptr, i32 signext %offset) local_unnamed_addr #0 {
+; CHECK-P8-LABEL: no_RAUW_in_combine_during_legalize:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI15_0@toc@ha
+; CHECK-P8-NEXT:    sldi r4, r4, 2
+; CHECK-P8-NEXT:    xxlxor v4, v4, v4
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI15_0@toc@l
+; CHECK-P8-NEXT:    lxsiwzx v2, r3, r4
+; CHECK-P8-NEXT:    lvx v3, 0, r5
+; CHECK-P8-NEXT:    vperm v2, v4, v2, v3
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: no_RAUW_in_combine_during_legalize:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    sldi r4, r4, 2
+; CHECK-P9-NEXT:    lxsiwzx v2, r3, r4
+; CHECK-P9-NEXT:    addis r3, r2, .LCPI15_0@toc@ha
+; CHECK-P9-NEXT:    addi r3, r3, .LCPI15_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r3
+; CHECK-P9-NEXT:    xxlxor v4, v4, v4
+; CHECK-P9-NEXT:    vperm v2, v4, v2, v3
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: no_RAUW_in_combine_during_legalize:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    sldi r4, r4, 2
+; CHECK-NOVSX-NEXT:    vxor v2, v2, v2
+; CHECK-NOVSX-NEXT:    lwzx r3, r3, r4
+; CHECK-NOVSX-NEXT:    std r3, -16(r1)
+; CHECK-NOVSX-NEXT:    addi r3, r1, -16
+; CHECK-NOVSX-NEXT:    lvx v3, 0, r3
+; CHECK-NOVSX-NEXT:    vmrglb v2, v2, v3
+; CHECK-NOVSX-NEXT:    blr
+entry:
+  %idx.ext = sext i32 %offset to i64
+  %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 %idx.ext
+  %0 = load i32, i32* %add.ptr, align 4
+  %conv = zext i32 %0 to i64
+  %splat.splatinsert = insertelement <2 x i64> undef, i64 %conv, i32 0
+  %1 = bitcast <2 x i64> %splat.splatinsert to <16 x i8>
+  %shuffle = shufflevector <16 x i8> %1, <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
+  ret <16 x i8> %shuffle
+}
+
+declare double @dummy() local_unnamed_addr
 attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/constant-pool.ll b/llvm/test/CodeGen/PowerPC/constant-pool.ll
index 797fc74672a28..4355cfa6ba211 100644
--- a/llvm/test/CodeGen/PowerPC/constant-pool.ll
+++ b/llvm/test/CodeGen/PowerPC/constant-pool.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -enable-ppc-quad-precision -ppc-asm-full-reg-names \
-; RUN:   < %s | FileCheck %s
+; RUN:   -mcpu=future -ppc-asm-full-reg-names < %s | FileCheck %s
 
  define float @FloatConstantPool() {
 ; CHECK-LABEL: FloatConstantPool:
diff --git a/llvm/test/CodeGen/PowerPC/convert-ri-addi-to-ri.mir b/llvm/test/CodeGen/PowerPC/convert-ri-addi-to-ri.mir
new file mode 100644
index 0000000000000..80b813c16bfa7
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/convert-ri-addi-to-ri.mir
@@ -0,0 +1,67 @@
+# RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs \
+# RUN:   -run-pass ppc-mi-peepholes -ppc-convert-rr-to-ri %s -o - | FileCheck %s
+
+---
+name: foldNewDformStore
+# CHECK: name: foldNewDformStore
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x3
+
+    %0:g8rc_and_g8rc_nox0 = COPY $x3
+    %1:g8rc_and_g8rc_nox0 = ADDI8 %0:g8rc_and_g8rc_nox0, 144
+    %2:g8rc = LI8 0
+    ; CHECK: STD killed %2, 160, %0
+    STD killed %2:g8rc, 16, %1:g8rc_and_g8rc_nox0
+    BLR8 implicit $lr8, implicit $rm
+...
+---
+name: foldNewDformStoreAlignNotMatch
+# CHECK: name: foldNewDformStoreAlignNotMatch
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x3
+
+    %0:g8rc_and_g8rc_nox0 = COPY $x3
+    %1:g8rc_and_g8rc_nox0 = ADDI8 %0:g8rc_and_g8rc_nox0, 141
+    %2:g8rc = LI8 0
+    ; CHECK: STD killed %2, 16, %1
+    STD killed %2:g8rc, 16, %1:g8rc_and_g8rc_nox0
+    BLR8 implicit $lr8, implicit $rm
+...
+---
+name: foldNewDformStoreKilledFlag
+# CHECK: name: foldNewDformStoreKilledFlag
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x3
+
+    %0:g8rc_and_g8rc_nox0 = COPY $x3
+    %1:g8rc_and_g8rc_nox0 = ADDI8 %0:g8rc_and_g8rc_nox0, 144
+    %2:g8rc = LI8 0
+    ; CHECK: STD %1, 0, %0
+    STD %1:g8rc_and_g8rc_nox0, 0, killed %0:g8rc_and_g8rc_nox0
+    ; CHECK: STD killed %2, 160, killed %0
+    STD killed %2:g8rc, 16, %1:g8rc_and_g8rc_nox0
+    BLR8 implicit $lr8, implicit $rm
+...
+---
+name: foldNewDformPreferLIOperand
+# CHECK: name: foldNewDformPreferLIOperand
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+  liveins: $x3
+  %0:g8rc_and_g8rc_nox0 = COPY $x3
+  %1:g8rc = ADDI8 %0:g8rc_and_g8rc_nox0, 1
+  %2:g8rc = LI8 1
+  ; CHECK: SUBFIC8 killed %1, 1, implicit-def $carry
+  %3:g8rc = SUBFC8 killed %1:g8rc, %2:g8rc, implicit-def $carry
+  %4:g8rc = SUBFE8 %2:g8rc, %2:g8rc, implicit-def dead $carry, implicit $carry
+  %5:g8rc = NEG8 killed %4:g8rc
+  $x3 = COPY %5:g8rc
+  BLR8 implicit $lr8, implicit $rm, implicit $x3
+...
diff --git a/llvm/test/CodeGen/PowerPC/dcbt.ll b/llvm/test/CodeGen/PowerPC/dcbt.ll
new file mode 100644
index 0000000000000..2559a2cf2f070
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/dcbt.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu < %s \
+; RUN:     -verify-machineinstrs -ppc-asm-full-reg-names \
+; RUN:     -ppc-vsr-nums-as-vr | FileCheck %s
+
+define void @dcbt_with_hint_test1(i8* %a) {
+; CHECK-LABEL: dcbt_with_hint_test1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    dcbt 0, r3
+; CHECK-NEXT:    blr
+entry:
+  tail call void @llvm.ppc.dcbt.with.hint(i8* %a, i32 0)
+  ret void
+}
+
+define void @dcbt_with_hint_test2(i8* %a) {
+; CHECK-LABEL: dcbt_with_hint_test2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    dcbt 0, r3, 8
+; CHECK-NEXT:    blr
+entry:
+  tail call void @llvm.ppc.dcbt.with.hint(i8* %a, i32 8)
+  ret void
+}
+
+define void @dcbt_with_hint_test3(i8* %a) {
+; CHECK-LABEL: dcbt_with_hint_test3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    dcbt 0, r3, 15
+; CHECK-NEXT:    blr
+entry:
+  tail call void @llvm.ppc.dcbt.with.hint(i8* %a, i32 15)
+  ret void
+}
+
+define void @dcbtst_with_hint_test1(i8* %a) {
+; CHECK-LABEL: dcbtst_with_hint_test1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    dcbtst 0, r3
+; CHECK-NEXT:    blr
+entry:
+  tail call void @llvm.ppc.dcbtst.with.hint(i8* %a, i32 0)
+  ret void
+}
+
+define void @dcbtst_with_hint_test2(i8* %a) {
+; CHECK-LABEL: dcbtst_with_hint_test2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    dcbtst 0, r3, 8
+; CHECK-NEXT:    blr
+entry:
+  tail call void @llvm.ppc.dcbtst.with.hint(i8* %a, i32 8)
+  ret void
+}
+
+define void @dcbtst_with_hint_test3(i8* %a) {
+; CHECK-LABEL: dcbtst_with_hint_test3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    dcbtst 0, r3, 15
+; CHECK-NEXT:    blr
+entry:
+  tail call void @llvm.ppc.dcbtst.with.hint(i8* %a, i32 15)
+  ret void
+}
+
+declare void @llvm.ppc.dcbt.with.hint(i8*, i32)
+declare void @llvm.ppc.dcbtst.with.hint(i8*, i32)
diff --git a/llvm/test/CodeGen/PowerPC/f128-aggregates.ll b/llvm/test/CodeGen/PowerPC/f128-aggregates.ll
index 006ad745f6073..094d29e2f258d 100644
--- a/llvm/test/CodeGen/PowerPC/f128-aggregates.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-aggregates.ll
@@ -1,9 +1,8 @@
 ; RUN: llc -relocation-model=pic -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \
-; RUN:   -enable-ppc-quad-precision -verify-machineinstrs \
-; RUN:   -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names < %s | FileCheck %s
+; RUN:   -verify-machineinstrs -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names < %s \
+; RUN:   | FileCheck %s
 ; RUN: llc -relocation-model=pic -mcpu=pwr9 -mtriple=powerpc64-unknown-unknown \
-; RUN:   -enable-ppc-quad-precision -verify-machineinstrs \
-; RUN:   -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names < %s \
+; RUN:   -verify-machineinstrs -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names < %s \
 ; RUN:   | FileCheck -check-prefix=CHECK-BE %s
 
 ; Testing homogeneous aggregates.
diff --git a/llvm/test/CodeGen/PowerPC/f128-arith.ll b/llvm/test/CodeGen/PowerPC/f128-arith.ll
index a957e0e6bdaa9..40b123bb9276b 100644
--- a/llvm/test/CodeGen/PowerPC/f128-arith.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-arith.ll
@@ -1,5 +1,4 @@
-; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \
-; RUN:   -enable-ppc-quad-precision -verify-machineinstrs \
+; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown -verify-machineinstrs \
 ; RUN:   -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | FileCheck %s
 
 ; Function Attrs: norecurse nounwind
diff --git a/llvm/test/CodeGen/PowerPC/f128-bitcast.ll b/llvm/test/CodeGen/PowerPC/f128-bitcast.ll
index 68069e542ffd6..fca24f5fd5410 100644
--- a/llvm/test/CodeGen/PowerPC/f128-bitcast.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-bitcast.ll
@@ -1,10 +1,8 @@
-; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \
-; RUN:   -enable-ppc-quad-precision -verify-machineinstrs \
+; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown -verify-machineinstrs \
 ; RUN:   -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | FileCheck %s
-; RUN: llc -mcpu=pwr9 -mtriple=powerpc64-unknown-unknown \
-; RUN:   -enable-ppc-quad-precision -verify-machineinstrs \
-; RUN:   -ppc-asm-full-reg-names \
-; RUN:   -ppc-vsr-nums-as-vr < %s | FileCheck %s --check-prefix=CHECK-BE
+; RUN: llc -mcpu=pwr9 -mtriple=powerpc64-unknown-unknown -verify-machineinstrs \
+; RUN:   -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   FileCheck %s --check-prefix=CHECK-BE
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @getPart1(fp128 %in) local_unnamed_addr {
diff --git a/llvm/test/CodeGen/PowerPC/f128-compare.ll b/llvm/test/CodeGen/PowerPC/f128-compare.ll
index c876878f05fac..5376b3b3f1c5a 100644
--- a/llvm/test/CodeGen/PowerPC/f128-compare.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-compare.ll
@@ -1,5 +1,4 @@
-; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \
-; RUN:   -enable-ppc-quad-precision -verify-machineinstrs \
+; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown -verify-machineinstrs \
 ; RUN:   -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | FileCheck %s
 
 @a_qp = common global fp128 0xL00000000000000000000000000000000, align 16
diff --git a/llvm/test/CodeGen/PowerPC/f128-conv.ll b/llvm/test/CodeGen/PowerPC/f128-conv.ll
index 4c64341d63497..2cb3174925457 100644
--- a/llvm/test/CodeGen/PowerPC/f128-conv.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-conv.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -relocation-model=pic -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \
-; RUN:   -enable-ppc-quad-precision -ppc-vsr-nums-as-vr \
-; RUN:   -verify-machineinstrs -ppc-asm-full-reg-names < %s | FileCheck %s
+; RUN:   -ppc-vsr-nums-as-vr -verify-machineinstrs -ppc-asm-full-reg-names < %s \
+; RUN:   | FileCheck %s
 
 @mem = global [5 x i64] [i64 56, i64 63, i64 3, i64 5, i64 6], align 8
 @umem = global [5 x i64] [i64 560, i64 100, i64 34, i64 2, i64 5], align 8
diff --git a/llvm/test/CodeGen/PowerPC/f128-fma.ll b/llvm/test/CodeGen/PowerPC/f128-fma.ll
index 8f76520d32bd5..f63ae04699f49 100644
--- a/llvm/test/CodeGen/PowerPC/f128-fma.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-fma.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \
-; RUN:   -enable-ppc-quad-precision -ppc-vsr-nums-as-vr \
-; RUN:   -ppc-asm-full-reg-names < %s | FileCheck %s
+; RUN:   -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names < %s | FileCheck %s
 
 define void @qpFmadd(fp128* nocapture readonly %a, fp128* nocapture %b,
                    fp128* nocapture readonly %c, fp128* nocapture %res) {
diff --git a/llvm/test/CodeGen/PowerPC/f128-passByValue.ll b/llvm/test/CodeGen/PowerPC/f128-passByValue.ll
index cbccaea3bce18..8b2db6b035109 100644
--- a/llvm/test/CodeGen/PowerPC/f128-passByValue.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-passByValue.ll
@@ -1,5 +1,4 @@
-; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \
-; RUN:   -enable-ppc-quad-precision -ppc-vsr-nums-as-vr \
+; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown -ppc-vsr-nums-as-vr \
 ; RUN:   -verify-machineinstrs -ppc-asm-full-reg-names < %s | FileCheck %s
 
 ; Function Attrs: norecurse nounwind readnone
diff --git a/llvm/test/CodeGen/PowerPC/f128-rounding.ll b/llvm/test/CodeGen/PowerPC/f128-rounding.ll
index 063eb1456fd83..56f63be5734e8 100644
--- a/llvm/test/CodeGen/PowerPC/f128-rounding.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-rounding.ll
@@ -1,5 +1,4 @@
-; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \
-; RUN:   -enable-ppc-quad-precision -verify-machineinstrs \
+; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown -verify-machineinstrs \
 ; RUN:   -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names < %s | FileCheck %s
 
 
diff --git a/llvm/test/CodeGen/PowerPC/f128-truncateNconv.ll b/llvm/test/CodeGen/PowerPC/f128-truncateNconv.ll
index ebded9dcccbe5..10d56fb2c47a6 100644
--- a/llvm/test/CodeGen/PowerPC/f128-truncateNconv.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-truncateNconv.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -relocation-model=pic -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \
-; RUN:   -verify-machineinstrs -enable-ppc-quad-precision \
-; RUN:   -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names < %s | FileCheck %s
+; RUN:   -verify-machineinstrs -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names < %s \
+; RUN:   | FileCheck %s
 
 @f128Array = global [4 x fp128] [fp128 0xL00000000000000004004C00000000000,
                                  fp128 0xLF000000000000000400808AB851EB851,
diff --git a/llvm/test/CodeGen/PowerPC/f128-vecExtractNconv.ll b/llvm/test/CodeGen/PowerPC/f128-vecExtractNconv.ll
index bae676cd09cd3..8542b10722337 100644
--- a/llvm/test/CodeGen/PowerPC/f128-vecExtractNconv.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-vecExtractNconv.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown -ppc-vsr-nums-as-vr \
 ; RUN:   -relocation-model=pic -ppc-asm-full-reg-names -verify-machineinstrs \
-; RUN:   -enable-ppc-quad-precision < %s | FileCheck %s
+; RUN:   < %s | FileCheck %s
 ; RUN: llc -mcpu=pwr9 -mtriple=powerpc64-unknown-unknown -ppc-vsr-nums-as-vr \
 ; RUN:   -ppc-asm-full-reg-names -verify-machineinstrs \
-; RUN:   -enable-ppc-quad-precision < %s | FileCheck %s -check-prefix=CHECK-BE
+; RUN:   < %s | FileCheck %s -check-prefix=CHECK-BE
 
 ; Vector extract DWord and convert to quad precision.
 
diff --git a/llvm/test/CodeGen/PowerPC/float-load-store-pair.ll b/llvm/test/CodeGen/PowerPC/float-load-store-pair.ll
index a8ed39ee9ce0e..caeff71553343 100644
--- a/llvm/test/CodeGen/PowerPC/float-load-store-pair.ll
+++ b/llvm/test/CodeGen/PowerPC/float-load-store-pair.ll
@@ -52,24 +52,22 @@ define signext i32 @test() nounwind {
 ; CHECK-NEXT:    addis 3, 2, a10@toc@ha
 ; CHECK-NEXT:    lfd 10, a10@toc@l(3)
 ; CHECK-NEXT:    addis 3, 2, a11@toc@ha
-; CHECK-NEXT:    addis 6, 2, a17@toc@ha
+; CHECK-NEXT:    lfd 11, a11@toc@l(3)
+; CHECK-NEXT:    addis 3, 2, a12@toc@ha
 ; CHECK-NEXT:    addis 5, 2, a16@toc@ha
+; CHECK-NEXT:    addis 6, 2, a17@toc@ha
 ; CHECK-NEXT:    addi 6, 6, a17@toc@l
-; CHECK-NEXT:    addi 5, 5, a16@toc@l
 ; CHECK-NEXT:    lxvx 34, 0, 6
+; CHECK-NEXT:    lfd 12, a12@toc@l(3)
+; CHECK-NEXT:    addis 3, 2, a13@toc@ha
+; CHECK-NEXT:    addi 5, 5, a16@toc@l
 ; CHECK-NEXT:    addis 4, 2, a15@toc@ha
 ; CHECK-NEXT:    lxvx 0, 0, 5
 ; CHECK-NEXT:    ld 4, a15@toc@l(4)
-; CHECK-NEXT:    li 5, 168
-; CHECK-NEXT:    lfd 11, a11@toc@l(3)
-; CHECK-NEXT:    addis 3, 2, a12@toc@ha
-; CHECK-NEXT:    lfd 12, a12@toc@l(3)
-; CHECK-NEXT:    addis 3, 2, a13@toc@ha
+; CHECK-NEXT:    li 5, 152
 ; CHECK-NEXT:    lfd 13, a13@toc@l(3)
 ; CHECK-NEXT:    addis 3, 2, a14@toc@ha
 ; CHECK-NEXT:    ld 3, a14@toc@l(3)
-; CHECK-NEXT:    stxvx 34, 1, 5
-; CHECK-NEXT:    li 5, 152
 ; CHECK-NEXT:    stxvx 0, 1, 5
 ; CHECK-NEXT:    std 4, 144(1)
 ; CHECK-NEXT:    std 3, 136(1)
diff --git a/llvm/test/CodeGen/PowerPC/fp-strict-conv.ll b/llvm/test/CodeGen/PowerPC/fp-strict-conv.ll
new file mode 100644
index 0000000000000..ab806a19c158e
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/fp-strict-conv.ll
@@ -0,0 +1,274 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -ppc-asm-full-reg-names < %s -mcpu=e500 \
+; RUN:   -mtriple=powerpc-unknown-linux-gnu -mattr=spe | FileCheck %s \
+; RUN:   -check-prefix=SPE
+
+declare i32 @llvm.experimental.constrained.fptosi.i32.f64(double, metadata)
+declare i64 @llvm.experimental.constrained.fptosi.i64.f64(double, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.f64(double, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f64(double, metadata)
+
+declare i32 @llvm.experimental.constrained.fptosi.i32.f32(float, metadata)
+declare i64 @llvm.experimental.constrained.fptosi.i64.f32(float, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.f32(float, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f32(float, metadata)
+
+declare double @llvm.experimental.constrained.sitofp.f64.i32(i32, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i64(i64, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i32(i32, metadata, metadata)
+declare double @llvm.experimental.constrained.uitofp.f64.i64(i64, metadata, metadata)
+
+declare float @llvm.experimental.constrained.sitofp.f32.i64(i64, metadata, metadata)
+declare float @llvm.experimental.constrained.sitofp.f32.i32(i32, metadata, metadata)
+declare float @llvm.experimental.constrained.uitofp.f32.i32(i32, metadata, metadata)
+declare float @llvm.experimental.constrained.uitofp.f32.i64(i64, metadata, metadata)
+
+define i32 @d_to_i32(double %m) #0 {
+; SPE-LABEL: d_to_i32:
+; SPE:       # %bb.0: # %entry
+; SPE-NEXT:    evmergelo r3, r3, r4
+; SPE-NEXT:    efdctsiz r3, r3
+; SPE-NEXT:    blr
+entry:
+  %conv = call i32 @llvm.experimental.constrained.fptosi.i32.f64(double %m, metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+define i64 @d_to_i64(double %m) #0 {
+; SPE-LABEL: d_to_i64:
+; SPE:       # %bb.0: # %entry
+; SPE-NEXT:    mflr r0
+; SPE-NEXT:    stw r0, 4(r1)
+; SPE-NEXT:    stwu r1, -16(r1)
+; SPE-NEXT:    .cfi_def_cfa_offset 16
+; SPE-NEXT:    .cfi_offset lr, 4
+; SPE-NEXT:    evmergelo r4, r3, r4
+; SPE-NEXT:    evmergehi r3, r4, r4
+; SPE-NEXT:    # kill: def $r4 killed $r4 killed $s4
+; SPE-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; SPE-NEXT:    bl __fixdfdi
+; SPE-NEXT:    lwz r0, 20(r1)
+; SPE-NEXT:    addi r1, r1, 16
+; SPE-NEXT:    mtlr r0
+; SPE-NEXT:    blr
+entry:
+  %conv = call i64 @llvm.experimental.constrained.fptosi.i64.f64(double %m, metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+define i64 @d_to_u64(double %m) #0 {
+; SPE-LABEL: d_to_u64:
+; SPE:       # %bb.0: # %entry
+; SPE-NEXT:    mflr r0
+; SPE-NEXT:    stw r0, 4(r1)
+; SPE-NEXT:    stwu r1, -16(r1)
+; SPE-NEXT:    .cfi_def_cfa_offset 16
+; SPE-NEXT:    .cfi_offset lr, 4
+; SPE-NEXT:    evmergelo r4, r3, r4
+; SPE-NEXT:    evmergehi r3, r4, r4
+; SPE-NEXT:    # kill: def $r4 killed $r4 killed $s4
+; SPE-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; SPE-NEXT:    bl __fixunsdfdi
+; SPE-NEXT:    lwz r0, 20(r1)
+; SPE-NEXT:    addi r1, r1, 16
+; SPE-NEXT:    mtlr r0
+; SPE-NEXT:    blr
+entry:
+  %conv = call i64 @llvm.experimental.constrained.fptoui.i64.f64(double %m, metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+define zeroext i32 @d_to_u32(double %m) #0 {
+; SPE-LABEL: d_to_u32:
+; SPE:       # %bb.0: # %entry
+; SPE-NEXT:    evmergelo r3, r3, r4
+; SPE-NEXT:    efdctuiz r3, r3
+; SPE-NEXT:    blr
+entry:
+  %conv = call i32 @llvm.experimental.constrained.fptoui.i32.f64(double %m, metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+define signext i32 @f_to_i32(float %m) #0 {
+; SPE-LABEL: f_to_i32:
+; SPE:       # %bb.0: # %entry
+; SPE-NEXT:    efsctsiz r3, r3
+; SPE-NEXT:    blr
+entry:
+  %conv = call i32 @llvm.experimental.constrained.fptosi.i32.f32(float %m, metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+define i64 @f_to_i64(float %m) #0 {
+; SPE-LABEL: f_to_i64:
+; SPE:       # %bb.0: # %entry
+; SPE-NEXT:    mflr r0
+; SPE-NEXT:    stw r0, 4(r1)
+; SPE-NEXT:    stwu r1, -16(r1)
+; SPE-NEXT:    .cfi_def_cfa_offset 16
+; SPE-NEXT:    .cfi_offset lr, 4
+; SPE-NEXT:    bl __fixsfdi
+; SPE-NEXT:    lwz r0, 20(r1)
+; SPE-NEXT:    addi r1, r1, 16
+; SPE-NEXT:    mtlr r0
+; SPE-NEXT:    blr
+entry:
+  %conv = call i64 @llvm.experimental.constrained.fptosi.i64.f32(float %m, metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+define i64 @f_to_u64(float %m) #0 {
+; SPE-LABEL: f_to_u64:
+; SPE:       # %bb.0: # %entry
+; SPE-NEXT:    mflr r0
+; SPE-NEXT:    stw r0, 4(r1)
+; SPE-NEXT:    stwu r1, -16(r1)
+; SPE-NEXT:    .cfi_def_cfa_offset 16
+; SPE-NEXT:    .cfi_offset lr, 4
+; SPE-NEXT:    bl __fixunssfdi
+; SPE-NEXT:    lwz r0, 20(r1)
+; SPE-NEXT:    addi r1, r1, 16
+; SPE-NEXT:    mtlr r0
+; SPE-NEXT:    blr
+entry:
+  %conv = call i64 @llvm.experimental.constrained.fptoui.i64.f32(float %m, metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+define zeroext i32 @f_to_u32(float %m) #0 {
+; SPE-LABEL: f_to_u32:
+; SPE:       # %bb.0: # %entry
+; SPE-NEXT:    efsctuiz r3, r3
+; SPE-NEXT:    blr
+entry:
+  %conv = call i32 @llvm.experimental.constrained.fptoui.i32.f32(float %m, metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
+define double @i32_to_d(i32 signext %m) #0 {
+; SPE-LABEL: i32_to_d:
+; SPE:       # %bb.0: # %entry
+; SPE-NEXT:    efdcfsi r4, r3
+; SPE-NEXT:    evmergehi r3, r4, r4
+; SPE-NEXT:    # kill: def $r4 killed $r4 killed $s4
+; SPE-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; SPE-NEXT:    blr
+entry:
+  %conv = tail call double @llvm.experimental.constrained.sitofp.f64.i32(i32 %m, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret double %conv
+}
+
+define double @i64_to_d(i64 %m) #0 {
+; SPE-LABEL: i64_to_d:
+; SPE:       # %bb.0: # %entry
+; SPE-NEXT:    mflr r0
+; SPE-NEXT:    stw r0, 4(r1)
+; SPE-NEXT:    stwu r1, -16(r1)
+; SPE-NEXT:    .cfi_def_cfa_offset 16
+; SPE-NEXT:    .cfi_offset lr, 4
+; SPE-NEXT:    bl __floatdidf
+; SPE-NEXT:    evmergelo r4, r3, r4
+; SPE-NEXT:    evmergehi r3, r4, r4
+; SPE-NEXT:    lwz r0, 20(r1)
+; SPE-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; SPE-NEXT:    # kill: def $r4 killed $r4 killed $s4
+; SPE-NEXT:    addi r1, r1, 16
+; SPE-NEXT:    mtlr r0
+; SPE-NEXT:    blr
+entry:
+  %conv = tail call double @llvm.experimental.constrained.sitofp.f64.i64(i64 %m, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret double %conv
+}
+
+define double @u32_to_d(i32 zeroext %m) #0 {
+; SPE-LABEL: u32_to_d:
+; SPE:       # %bb.0: # %entry
+; SPE-NEXT:    efdcfui r4, r3
+; SPE-NEXT:    evmergehi r3, r4, r4
+; SPE-NEXT:    # kill: def $r4 killed $r4 killed $s4
+; SPE-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; SPE-NEXT:    blr
+entry:
+  %conv = tail call double @llvm.experimental.constrained.uitofp.f64.i32(i32 %m, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret double %conv
+}
+
+define double @u64_to_d(i64 %m) #0 {
+; SPE-LABEL: u64_to_d:
+; SPE:       # %bb.0: # %entry
+; SPE-NEXT:    mflr r0
+; SPE-NEXT:    stw r0, 4(r1)
+; SPE-NEXT:    stwu r1, -16(r1)
+; SPE-NEXT:    .cfi_def_cfa_offset 16
+; SPE-NEXT:    .cfi_offset lr, 4
+; SPE-NEXT:    bl __floatundidf
+; SPE-NEXT:    evmergelo r4, r3, r4
+; SPE-NEXT:    evmergehi r3, r4, r4
+; SPE-NEXT:    lwz r0, 20(r1)
+; SPE-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; SPE-NEXT:    # kill: def $r4 killed $r4 killed $s4
+; SPE-NEXT:    addi r1, r1, 16
+; SPE-NEXT:    mtlr r0
+; SPE-NEXT:    blr
+entry:
+  %conv = tail call double @llvm.experimental.constrained.uitofp.f64.i64(i64 %m, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret double %conv
+}
+
+define float @i32_to_f(i32 signext %m) #0 {
+; SPE-LABEL: i32_to_f:
+; SPE:       # %bb.0: # %entry
+; SPE-NEXT:    efscfsi r3, r3
+; SPE-NEXT:    blr
+entry:
+  %conv = tail call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %m, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret float %conv
+}
+
+define float @i64_to_f(i64 %m) #0 {
+; SPE-LABEL: i64_to_f:
+; SPE:       # %bb.0: # %entry
+; SPE-NEXT:    mflr r0
+; SPE-NEXT:    stw r0, 4(r1)
+; SPE-NEXT:    stwu r1, -16(r1)
+; SPE-NEXT:    .cfi_def_cfa_offset 16
+; SPE-NEXT:    .cfi_offset lr, 4
+; SPE-NEXT:    bl __floatdisf
+; SPE-NEXT:    lwz r0, 20(r1)
+; SPE-NEXT:    addi r1, r1, 16
+; SPE-NEXT:    mtlr r0
+; SPE-NEXT:    blr
+entry:
+  %conv = tail call float @llvm.experimental.constrained.sitofp.f32.i64(i64 %m, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret float %conv
+}
+
+define float @u32_to_f(i32 zeroext %m) #0 {
+; SPE-LABEL: u32_to_f:
+; SPE:       # %bb.0: # %entry
+; SPE-NEXT:    efscfui r3, r3
+; SPE-NEXT:    blr
+entry:
+  %conv = tail call float @llvm.experimental.constrained.uitofp.f32.i32(i32 %m, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret float %conv
+}
+
+define float @u64_to_f(i64 %m) #0 {
+; SPE-LABEL: u64_to_f:
+; SPE:       # %bb.0: # %entry
+; SPE-NEXT:    mflr r0
+; SPE-NEXT:    stw r0, 4(r1)
+; SPE-NEXT:    stwu r1, -16(r1)
+; SPE-NEXT:    .cfi_def_cfa_offset 16
+; SPE-NEXT:    .cfi_offset lr, 4
+; SPE-NEXT:    bl __floatundisf
+; SPE-NEXT:    lwz r0, 20(r1)
+; SPE-NEXT:    addi r1, r1, 16
+; SPE-NEXT:    mtlr r0
+; SPE-NEXT:    blr
+entry:
+  %conv = tail call float @llvm.experimental.constrained.uitofp.f32.i64(i64 %m, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret float %conv
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/PowerPC/fp-strict-f128.ll b/llvm/test/CodeGen/PowerPC/fp-strict-f128.ll
index 21ddb799141d0..cab949a6ebd2a 100644
--- a/llvm/test/CodeGen/PowerPC/fp-strict-f128.ll
+++ b/llvm/test/CodeGen/PowerPC/fp-strict-f128.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -verify-machineinstrs -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s -mtriple=powerpc64le-unknown-linux -mcpu=pwr9 -enable-ppc-quad-precision=true | FileCheck %s
+; RUN: llc -verify-machineinstrs -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   < %s -mtriple=powerpc64le-unknown-linux -mcpu=pwr9 | FileCheck %s
 
 declare fp128 @llvm.experimental.constrained.fadd.f128(fp128, fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.fsub.f128(fp128, fp128, metadata, metadata)
diff --git a/llvm/test/CodeGen/PowerPC/func-alias.ll b/llvm/test/CodeGen/PowerPC/func-alias.ll
new file mode 100644
index 0000000000000..7382722b97a7f
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/func-alias.ll
@@ -0,0 +1,46 @@
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:   -mcpu=pwr9 -ppc-asm-full-reg-names < %s | FileCheck %s --check-prefix=P9
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names < %s | FileCheck %s --check-prefix=P10
+
+@newname = dso_local alias i32 (...), bitcast (i32 ()* @oldname to i32 (...)*)
+
+; Function Attrs: noinline nounwind optnone
+define dso_local signext i32 @oldname() #0 {
+entry:
+  ret i32 55
+}
+
+; Function Attrs: noinline nounwind optnone
+define dso_local signext i32 @caller() #0 {
+; #P9-LABEL: caller
+; #P9:       bl newname
+; #P9-NOT:   nop
+; #P9:       blr
+; #P10-LABEL: caller
+; #P10:       bl newname@notoc
+; #P10-NOT:   nop
+; #P10:       blr
+entry:
+  %call = call signext i32 bitcast (i32 (...)* @newname to i32 ()*)()
+  ret i32 %call
+}
+
+; Function Attrs: noinline nounwind optnone -pcrelative-memops
+; This caller does not use PC Relative memops
+define dso_local signext i32 @caller_nopcrel() #1 {
+; #P9-LABEL: caller_nopcrel
+; #P9:       bl newname
+; #P9-NOT:   nop
+; #P9:       blr
+; #P10-LABEL: caller_nopcrel
+; #P10:       bl newname
+; #P10-NEXT:  nop
+; #P10:       blr
+entry:
+  %call = call signext i32 bitcast (i32 (...)* @newname to i32 ()*)()
+  ret i32 %call
+}
+
+attributes #0 = { noinline nounwind optnone }
+attributes #1 = { noinline nounwind optnone "target-features"="-pcrelative-memops" }
diff --git a/llvm/test/CodeGen/PowerPC/global-address-non-got-indirect-access.ll b/llvm/test/CodeGen/PowerPC/global-address-non-got-indirect-access.ll
index f29d9bd251ca4..5c49760a9f45a 100644
--- a/llvm/test/CodeGen/PowerPC/global-address-non-got-indirect-access.ll
+++ b/llvm/test/CodeGen/PowerPC/global-address-non-got-indirect-access.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -enable-ppc-quad-precision -ppc-asm-full-reg-names \
-; RUN:   -ppc-vsr-nums-as-vr < %s | FileCheck %s
+; RUN:   -mcpu=future -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s \
+; RUN:   | FileCheck %s
 
  @_ZL13StaticBoolVar = internal unnamed_addr global i8 0, align 1
 @_ZL19StaticSignedCharVar = internal unnamed_addr global i8 0, align 1
diff --git a/llvm/test/CodeGen/PowerPC/ifunc.ll b/llvm/test/CodeGen/PowerPC/ifunc.ll
index a964a2bac1c51..a58601f8f32f6 100644
--- a/llvm/test/CodeGen/PowerPC/ifunc.ll
+++ b/llvm/test/CodeGen/PowerPC/ifunc.ll
@@ -2,6 +2,10 @@
 ; RUN: llc %s -o - -mtriple=powerpc -relocation-model=pic | FileCheck --check-prefix=PLTREL %s
 ; RUN: llc %s -o - -mtriple=powerpc64 | FileCheck --check-prefix=REL %s
 ; RUN: llc %s -o - -mtriple=powerpc64 -relocation-model=pic | FileCheck --check-prefix=REL %s
+; RUN: llc %s -o - -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 \
+; RUN:   -verify-machineinstrs | FileCheck --check-prefix=LEP8 %s
+; RUN: llc %s -o - -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr10 \
+; RUN:   -verify-machineinstrs | FileCheck --check-prefix=LEP10 %s
 
 @ifunc1 = dso_local ifunc void(), i8*()* @resolver
 @ifunc2 = ifunc void(), i8*()* @resolver
@@ -9,10 +13,23 @@
 define i8* @resolver() { ret i8* null }
 
 define void @foo() #0 {
-  ; REL: bl ifunc1{{$}}
-  ; REL: bl ifunc2{{$}}
-  ; PLTREL: bl ifunc1@PLT+32768
-  ; PLTREL: bl ifunc2@PLT+32768
+  ; REL-LABEL:    foo
+  ; REL:          bl ifunc1{{$}}
+  ; REL:          bl ifunc2{{$}}
+  ; PLTREL-LABEL: foo
+  ; PLTREL:       bl ifunc1@PLT+32768
+  ; PLTREL:       bl ifunc2@PLT+32768
+  ; LEP8-LABEL:   foo
+  ; LEP8:         bl ifunc1
+  ; LEP8-NEXT:    nop
+  ; LEP8-NEXT:    bl ifunc2
+  ; LEP8-NEXT:    nop
+  ; LEP8:         blr
+  ; LEP10-LABEL:  foo
+  ; LEP10:        bl ifunc1@notoc
+  ; LEP10-NEXT:   bl ifunc2@notoc
+  ; LEP10-NOT:    nop
+  ; LEP10:        blr
   call void @ifunc1()
   call void @ifunc2()
   ret void
diff --git a/llvm/test/CodeGen/PowerPC/isync.ll b/llvm/test/CodeGen/PowerPC/isync.ll
new file mode 100644
index 0000000000000..77677434744d5
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/isync.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu < %s \
+; RUN:     -verify-machineinstrs -ppc-asm-full-reg-names \
+; RUN:     -ppc-vsr-nums-as-vr | FileCheck %s
+
+define void @isync_test() {
+; CHECK-LABEL: isync_test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    isync
+; CHECK-NEXT:    blr
+
+entry:
+  tail call void @llvm.ppc.isync()
+  ret void
+}
+
+declare void @llvm.ppc.isync()
diff --git a/llvm/test/CodeGen/PowerPC/legalize-vaarg.ll b/llvm/test/CodeGen/PowerPC/legalize-vaarg.ll
index 296dea2f1f219..d937acc09c64d 100644
--- a/llvm/test/CodeGen/PowerPC/legalize-vaarg.ll
+++ b/llvm/test/CodeGen/PowerPC/legalize-vaarg.ll
@@ -6,13 +6,6 @@ define <8 x i32> @test_large_vec_vaarg(i32 %n, ...) {
 ; BE-LABEL: test_large_vec_vaarg:
 ; BE:       # %bb.0:
 ; BE-NEXT:    ld 3, -8(1)
-; BE-NEXT:    std 4, 56(1)
-; BE-NEXT:    std 5, 64(1)
-; BE-NEXT:    std 6, 72(1)
-; BE-NEXT:    std 7, 80(1)
-; BE-NEXT:    std 8, 88(1)
-; BE-NEXT:    std 9, 96(1)
-; BE-NEXT:    std 10, 104(1)
 ; BE-NEXT:    addi 3, 3, 15
 ; BE-NEXT:    rldicr 3, 3, 0, 59
 ; BE-NEXT:    addi 4, 3, 16
@@ -28,15 +21,8 @@ define <8 x i32> @test_large_vec_vaarg(i32 %n, ...) {
 ; LE-LABEL: test_large_vec_vaarg:
 ; LE:       # %bb.0:
 ; LE-NEXT:    ld 3, -8(1)
-; LE-NEXT:    std 4, 40(1)
-; LE-NEXT:    std 5, 48(1)
-; LE-NEXT:    std 6, 56(1)
-; LE-NEXT:    std 7, 64(1)
 ; LE-NEXT:    addi 3, 3, 15
 ; LE-NEXT:    rldicr 3, 3, 0, 59
-; LE-NEXT:    std 8, 72(1)
-; LE-NEXT:    std 9, 80(1)
-; LE-NEXT:    std 10, 88(1)
 ; LE-NEXT:    addi 4, 3, 31
 ; LE-NEXT:    addi 5, 3, 16
 ; LE-NEXT:    rldicr 4, 4, 0, 59
diff --git a/llvm/test/CodeGen/PowerPC/mi-peephole.mir b/llvm/test/CodeGen/PowerPC/mi-peephole.mir
new file mode 100644
index 0000000000000..8bf72461d5453
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/mi-peephole.mir
@@ -0,0 +1,37 @@
+# RUN: llc -mtriple=powerpc64le--linux-gnu -run-pass ppc-mi-peepholes %s -o - \
+# RUN:   -verify-machineinstrs | FileCheck %s
+
+---
+name:            testRLDIC
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: g8rc }
+  - { id: 1, class: g8rc }
+  - { id: 2, class: g8rc }
+liveins:
+  - { reg: '$x3', virtual-reg: '%0' }
+  - { reg: '$x4', virtual-reg: '%1' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+body:             |
+  bb.0.entry:
+    liveins: $x3, $x4
+
+    %1:g8rc = COPY $x4
+    %0:g8rc = COPY $x3
+    %2:g8rc = RLDICL killed %1, 0, 32
+    %3:g8rc = RLDICR %2, 2, 61
+    $x3 = COPY %3
+    BLR8 implicit $lr8, implicit $rm, implicit $x3
+
+  ; CHECK-LABEL: testRLDIC
+  ; CHECK: bb.0.entry:
+  ; CHECK:   %1:g8rc = COPY $x4
+  ; CHECK:   %0:g8rc = COPY $x3
+  ; CHECK:   %3:g8rc = RLDIC %1, 2, 30
+  ; CHECK:   $x3 = COPY %3
+  ; CHECK:   BLR8 implicit $lr8, implicit $rm, implicit $x3
+...
diff --git a/llvm/test/CodeGen/PowerPC/p10-bit-manip-ops.ll b/llvm/test/CodeGen/PowerPC/p10-bit-manip-ops.ll
index fc2ebf89079ff..901b449bb5c39 100644
--- a/llvm/test/CodeGen/PowerPC/p10-bit-manip-ops.ll
+++ b/llvm/test/CodeGen/PowerPC/p10-bit-manip-ops.ll
@@ -9,6 +9,14 @@ declare <2 x i64> @llvm.ppc.altivec.vpdepd(<2 x i64>, <2 x i64>)
 declare <2 x i64> @llvm.ppc.altivec.vpextd(<2 x i64>, <2 x i64>)
 declare i64 @llvm.ppc.pdepd(i64, i64)
 declare i64 @llvm.ppc.pextd(i64, i64)
+declare <2 x i64> @llvm.ppc.altivec.vcfuged(<2 x i64>, <2 x i64>)
+declare i64 @llvm.ppc.cfuged(i64, i64)
+declare i64 @llvm.ppc.altivec.vgnb(<1 x i128>, i32)
+declare <2 x i64> @llvm.ppc.vsx.xxeval(<2 x i64>, <2 x i64>, <2 x i64>, i32)
+declare <2 x i64> @llvm.ppc.altivec.vclzdm(<2 x i64>, <2 x i64>)
+declare <2 x i64> @llvm.ppc.altivec.vctzdm(<2 x i64>, <2 x i64>)
+declare i64 @llvm.ppc.cntlzdm(i64, i64)
+declare i64 @llvm.ppc.cnttzdm(i64, i64)
 
 define <2 x i64> @test_vpdepd(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: test_vpdepd:
@@ -49,3 +57,103 @@ entry:
   %tmp = tail call i64 @llvm.ppc.pextd(i64 %a, i64 %b)
   ret i64 %tmp
 }
+
+define <2 x i64> @test_vcfuged(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vcfuged:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vcfuged v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %tmp = tail call <2 x i64> @llvm.ppc.altivec.vcfuged(<2 x i64> %a, <2 x i64> %b)
+  ret <2 x i64> %tmp
+}
+
+define i64 @test_cfuged(i64 %a, i64 %b) {
+; CHECK-LABEL: test_cfuged:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cfuged r3, r3, r4
+; CHECK-NEXT:    blr
+entry:
+  %tmp = tail call i64 @llvm.ppc.cfuged(i64 %a, i64 %b)
+  ret i64 %tmp
+}
+
+define i64 @test_vgnb_1(<1 x i128> %a) {
+; CHECK-LABEL: test_vgnb_1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vgnb r3, v2, 2
+; CHECK-NEXT:    blr
+entry:
+  %tmp = tail call i64 @llvm.ppc.altivec.vgnb(<1 x i128> %a, i32 2)
+  ret i64 %tmp
+}
+
+define i64 @test_vgnb_2(<1 x i128> %a) {
+; CHECK-LABEL: test_vgnb_2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vgnb r3, v2, 7
+; CHECK-NEXT:    blr
+entry:
+  %tmp = tail call i64 @llvm.ppc.altivec.vgnb(<1 x i128> %a, i32 7)
+  ret i64 %tmp
+}
+
+define i64 @test_vgnb_3(<1 x i128> %a) {
+; CHECK-LABEL: test_vgnb_3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vgnb r3, v2, 5
+; CHECK-NEXT:    blr
+entry:
+  %tmp = tail call i64 @llvm.ppc.altivec.vgnb(<1 x i128> %a, i32 5)
+  ret i64 %tmp
+}
+
+define <2 x i64> @test_xxeval(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
+; CHECK-LABEL: test_xxeval:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxeval v2, v2, v3, v4, 255
+; CHECK-NEXT:    blr
+entry:
+  %tmp = tail call <2 x i64> @llvm.ppc.vsx.xxeval(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c, i32 255)
+  ret <2 x i64> %tmp
+}
+
+define <2 x i64> @test_vclzdm(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vclzdm:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vclzdm v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %tmp = tail call <2 x i64> @llvm.ppc.altivec.vclzdm(<2 x i64> %a, <2 x i64> %b)
+  ret <2 x i64> %tmp
+}
+
+define <2 x i64> @test_vctzdm(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vctzdm:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vctzdm v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %tmp = tail call <2 x i64> @llvm.ppc.altivec.vctzdm(<2 x i64> %a, <2 x i64> %b)
+  ret <2 x i64> %tmp
+}
+
+define i64 @test_cntlzdm(i64 %a, i64 %b) {
+; CHECK-LABEL: test_cntlzdm:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cntlzdm r3, r3, r4
+; CHECK-NEXT:    blr
+entry:
+  %tmp = tail call i64 @llvm.ppc.cntlzdm(i64 %a, i64 %b)
+  ret i64 %tmp
+}
+
+define i64 @test_cnttzdm(i64 %a, i64 %b) {
+; CHECK-LABEL: test_cnttzdm:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cnttzdm r3, r3, r4
+; CHECK-NEXT:    blr
+entry:
+  %tmp = tail call i64 @llvm.ppc.cnttzdm(i64 %a, i64 %b)
+  ret i64 %tmp
+}
diff --git a/llvm/test/CodeGen/PowerPC/p10-splatImm-CPload-pcrel.ll b/llvm/test/CodeGen/PowerPC/p10-splatImm-CPload-pcrel.ll
new file mode 100644
index 0000000000000..2a49910214289
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/p10-splatImm-CPload-pcrel.ll
@@ -0,0 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
+; RUN:     -ppc-asm-full-reg-names -mcpu=pwr10 < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
+; RUN:     -ppc-asm-full-reg-names -mcpu=pwr10 < %s | FileCheck %s \
+; RUN:     --check-prefix=CHECK-NOPCREL
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
+; RUN:     -mattr=-pcrelative-memops -ppc-asm-full-reg-names -mcpu=pwr10 < %s | \
+; RUN:     FileCheck %s --check-prefix=CHECK-NOPCREL
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
+; RUN:     -ppc-asm-full-reg-names -target-abi=elfv2 -mcpu=pwr10 < %s | \
+; RUN:     FileCheck %s
+
+define dso_local <2 x double> @testDoubleToDoubleFail() local_unnamed_addr {
+; CHECK-LABEL: testDoubleToDoubleFail:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    plxv vs34, .LCPI0_0@PCREL(0), 1
+; CHECK-NEXT:    blr
+;
+; CHECK-NOPCREL-LABEL: testDoubleToDoubleFail:
+; CHECK-NOPCREL:       # %bb.0: # %entry
+; CHECK-NOPCREL-NEXT:    addis r3, r2, .LCPI0_0@toc@ha
+; CHECK-NOPCREL-NEXT:    addi r3, r3, .LCPI0_0@toc@l
+; CHECK-NOPCREL-NEXT:    lxvx vs34, 0, r3
+; CHECK-NOPCREL-NEXT:    blr
+
+entry:
+  ret <2 x double> <double 3.423300e+02, double 3.423300e+02>
+}
+
+define dso_local <2 x double> @testFloatDenormToDouble() local_unnamed_addr {
+; CHECK-LABEL: testFloatDenormToDouble:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    plxv vs34, .LCPI1_0@PCREL(0), 1
+; CHECK-NEXT:    blr
+;
+; CHECK-NOPCREL-LABEL: testFloatDenormToDouble:
+; CHECK-NOPCREL:       # %bb.0: # %entry
+; CHECK-NOPCREL-NEXT:    addis r3, r2, .LCPI1_0@toc@ha
+; CHECK-NOPCREL-NEXT:    addi r3, r3, .LCPI1_0@toc@l
+; CHECK-NOPCREL-NEXT:    lxvx vs34, 0, r3
+; CHECK-NOPCREL-NEXT:    blr
+
+entry:
+  ret <2 x double> <double 0x380B38FB80000000, double 0x380B38FB80000000>
+}
+
+define dso_local <2 x double> @testDoubleToDoubleNaNFail() local_unnamed_addr {
+; CHECK-LABEL: testDoubleToDoubleNaNFail:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    plxv vs34, .LCPI2_0@PCREL(0), 1
+; CHECK-NEXT:    blr
+;
+; CHECK-NOPCREL-LABEL: testDoubleToDoubleNaNFail:
+; CHECK-NOPCREL:       # %bb.0: # %entry
+; CHECK-NOPCREL-NEXT:    addis r3, r2, .LCPI2_0@toc@ha
+; CHECK-NOPCREL-NEXT:    addi r3, r3, .LCPI2_0@toc@l
+; CHECK-NOPCREL-NEXT:    lxvx vs34, 0, r3
+; CHECK-NOPCREL-NEXT:    blr
+
+entry:
+  ret <2 x double> <double 0xFFFFFFFFFFFFFFF0, double 0xFFFFFFFFFFFFFFF0>
+}
+
+define dso_local double @testDoubleNonRepresentableScalar() local_unnamed_addr {
+; CHECK-LABEL: testDoubleNonRepresentableScalar:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    plfd f1, .LCPI3_0@PCREL(0), 1
+; CHECK-NEXT:    blr
+;
+; CHECK-NOPCREL-LABEL: testDoubleNonRepresentableScalar:
+; CHECK-NOPCREL:       # %bb.0: # %entry
+; CHECK-NOPCREL-NEXT:    addis r3, r2, .LCPI3_0@toc@ha
+; CHECK-NOPCREL-NEXT:    lfd f1, .LCPI3_0@toc@l(r3)
+; CHECK-NOPCREL-NEXT:    blr
+
+entry:
+  ret double 3.423300e+02
+}
+
+define dso_local float @testFloatDenormScalar() local_unnamed_addr {
+; CHECK-LABEL: testFloatDenormScalar:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    plfs f1, .LCPI4_0@PCREL(0), 1
+; CHECK-NEXT:    blr
+;
+; CHECK-NOPCREL-LABEL: testFloatDenormScalar:
+; CHECK-NOPCREL:       # %bb.0: # %entry
+; CHECK-NOPCREL-NEXT:    addis r3, r2, .LCPI4_0@toc@ha
+; CHECK-NOPCREL-NEXT:    lfs f1, .LCPI4_0@toc@l(r3)
+; CHECK-NOPCREL-NEXT:    blr
+
+entry:
+  ret float 0x380B38FB80000000
+}
+
+define dso_local double @testFloatDenormToDoubleScalar() local_unnamed_addr {
+; CHECK-LABEL: testFloatDenormToDoubleScalar:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    plfs f1, .LCPI5_0@PCREL(0), 1
+; CHECK-NEXT:    blr
+;
+; CHECK-NOPCREL-LABEL: testFloatDenormToDoubleScalar:
+; CHECK-NOPCREL:       # %bb.0: # %entry
+; CHECK-NOPCREL-NEXT:    addis r3, r2, .LCPI5_0@toc@ha
+; CHECK-NOPCREL-NEXT:    lfs f1, .LCPI5_0@toc@l(r3)
+; CHECK-NOPCREL-NEXT:    blr
+
+entry:
+  ret double 0x380B38FB80000000
+}
diff --git a/llvm/test/CodeGen/PowerPC/p10-splatImm.ll b/llvm/test/CodeGen/PowerPC/p10-splatImm.ll
new file mode 100644
index 0000000000000..8bb83c22be586
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/p10-splatImm.ll
@@ -0,0 +1,306 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
+; RUN:     -ppc-asm-full-reg-names -mcpu=pwr10 < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
+; RUN:     -ppc-asm-full-reg-names -mcpu=pwr10 < %s | FileCheck %s
+
+define dso_local <4 x i32> @testZero() local_unnamed_addr {
+; CHECK-LABEL: testZero:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor vs34, vs34, vs34
+; CHECK-NEXT:    blr
+
+entry:
+  ret <4 x i32> zeroinitializer
+}
+
+define dso_local <4 x float> @testZeroF() local_unnamed_addr {
+; CHECK-LABEL: testZeroF:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor vs34, vs34, vs34
+; CHECK-NEXT:    blr
+
+entry:
+  ret <4 x float> zeroinitializer
+}
+
+define dso_local <4 x i32> @testAllOneS() local_unnamed_addr {
+; CHECK-LABEL: testAllOneS:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv vs34, vs34, vs34
+; CHECK-NEXT:    blr
+
+entry:
+  ret <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+}
+
+define dso_local <4 x i32> @test5Bit() local_unnamed_addr {
+; CHECK-LABEL: test5Bit:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vspltisw v2, 7
+; CHECK-NEXT:    blr
+
+entry:
+  ret <4 x i32> <i32 7, i32 7, i32 7, i32 7>
+}
+
+define dso_local <16 x i8> @test1ByteChar() local_unnamed_addr {
+; CHECK-LABEL: test1ByteChar:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib vs34, 7
+; CHECK-NEXT:    blr
+
+entry:
+  ret <16 x i8> <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
+}
+
+define dso_local <4 x i32> @test1ByteSplatInt() local_unnamed_addr {
+; Here the splat of 171 or 0xABABABAB can be done using a byte splat
+; of 0xAB using xxspltib while avoiding the use of xxspltiw.
+; CHECK-LABEL: test1ByteSplatInt:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib vs34, 171
+; CHECK-NEXT:    blr
+
+entry:
+  ret <4 x i32> <i32 -1414812757, i32 -1414812757, i32 -1414812757, i32 -1414812757>
+}
+
+define dso_local <4 x i32> @test5Bit2Ins() local_unnamed_addr {
+; Splats within the range [-32,31] can be done using two vsplti[bhw]
+; instructions, but we prefer the xxspltiw instruction to them.
+; CHECK-LABEL: test5Bit2Ins:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw vs34, 16
+; CHECK-NEXT:    blr
+
+entry:
+  ret <4 x i32> <i32 16, i32 16, i32 16, i32 16>
+}
+
+define dso_local <4 x float> @testFloatNegZero() local_unnamed_addr {
+; 0.0f is not the same as -0.0f. We try to splat -0.0f
+; CHECK-LABEL: testFloatNegZero:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw vs34, -2147483648
+; CHECK-NEXT:    blr
+
+entry:
+  ret <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>
+}
+
+define dso_local <4 x float> @testFloat() local_unnamed_addr {
+; CHECK-LABEL: testFloat:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw vs34, 1135323709
+; CHECK-NEXT:    blr
+
+entry:
+  ret <4 x float> <float 0x40757547A0000000, float 0x40757547A0000000, float 0x40757547A0000000, float 0x40757547A0000000>
+}
+
+define dso_local <4 x float> @testIntToFloat() local_unnamed_addr {
+; CHECK-LABEL: testIntToFloat:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw vs34, 1135312896
+; CHECK-NEXT:    blr
+
+entry:
+  ret <4 x float> <float 3.430000e+02, float 3.430000e+02, float 3.430000e+02, float 3.430000e+02>
+}
+
+define dso_local <4 x i32> @testUndefInt() local_unnamed_addr {
+; CHECK-LABEL: testUndefInt:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw vs34, 18
+; CHECK-NEXT:    blr
+
+entry:
+  ret <4 x i32> <i32 18, i32 undef, i32 undef, i32 18>
+}
+
+define dso_local <4 x float> @testUndefIntToFloat() local_unnamed_addr {
+; CHECK-LABEL: testUndefIntToFloat:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw vs34, 1135312896
+; CHECK-NEXT:    blr
+
+entry:
+  ret <4 x float> <float 3.430000e+02, float undef, float undef, float 3.430000e+02>
+}
+
+define dso_local <2 x i64> @testPseudo8Byte() local_unnamed_addr {
+; CHECK-LABEL: testPseudo8Byte:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw vs34, -1430532899
+; CHECK-NEXT:    blr
+
+entry:
+  ret <2 x i64> <i64 -6144092014192636707, i64 -6144092014192636707>
+}
+
+define dso_local <8 x i16> @test2Byte() local_unnamed_addr {
+; CHECK-LABEL: test2Byte:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw vs34, 1179666
+; CHECK-NEXT:    blr
+
+entry:
+  ret <8 x i16> <i16 18, i16 18, i16 18, i16 18, i16 18, i16 18, i16 18, i16 18>
+}
+
+define dso_local <8 x i16> @test2ByteUndef() local_unnamed_addr {
+; CHECK-LABEL: test2ByteUndef:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw vs34, 1179666
+; CHECK-NEXT:    blr
+
+entry:
+  ret <8 x i16> <i16 18, i16 undef, i16 18, i16 18, i16 18, i16 undef, i16 18, i16 18>
+}
+
+define dso_local <2 x double> @testFloatToDouble() local_unnamed_addr {
+; CHECK-LABEL: testFloatToDouble:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltidp vs34, 1135290941
+; CHECK-NEXT:    blr
+
+entry:
+  ret <2 x double> <double 0x40756547A0000000, double 0x40756547A0000000>
+}
+
+define dso_local <2 x double> @testDoubleLower4ByteZero() local_unnamed_addr {
+; The expanded double will have 0 in the last 32 bits. Imprecise handling of
+; return value of data structures like APInt, returned when calling getZextValue
+; , like saving the return value into an unsigned instead of uint64_t may cause
+; this test to fail.
+; CHECK-LABEL: testDoubleLower4ByteZero:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltidp vs34, 1093664768
+; CHECK-NEXT:    blr
+
+entry:
+  ret <2 x double> <double 1.100000e+01, double 1.100000e+01>
+}
+
+define dso_local <2 x double> @testDoubleToDoubleZero() local_unnamed_addr {
+; Should be using canonicalized form to splat zero and use shorter instructions
+; than xxspltidp.
+; CHECK-LABEL: testDoubleToDoubleZero:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor vs34, vs34, vs34
+; CHECK-NEXT:    blr
+
+entry:
+  ret <2 x double> zeroinitializer
+}
+
+define dso_local <2 x double> @testDoubleToDoubleNegZero() local_unnamed_addr {
+; CHECK-LABEL: testDoubleToDoubleNegZero:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltidp vs34, -2147483648
+; CHECK-NEXT:    blr
+
+entry:
+  ret <2 x double> <double -0.000000e+00, double -0.000000e+00>
+}
+
+define dso_local <2 x double> @testDoubleToDoubleNaN() local_unnamed_addr {
+; CHECK-LABEL: testDoubleToDoubleNaN:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltidp vs34, -16
+; CHECK-NEXT:    blr
+
+entry:
+  ret <2 x double> <double 0xFFFFFFFE00000000, double 0xFFFFFFFE00000000>
+}
+
+define dso_local <2 x double> @testDoubleToDoubleInfinity() local_unnamed_addr {
+; CHECK-LABEL: testDoubleToDoubleInfinity:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltidp vs34, 2139095040
+; CHECK-NEXT:    blr
+
+entry:
+  ret <2 x double> <double 0x7FF0000000000000, double 0x7FF0000000000000>
+}
+
+define dso_local <2 x double> @testFloatToDoubleNaN() local_unnamed_addr {
+; CHECK-LABEL: testFloatToDoubleNaN:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltidp vs34, -1
+; CHECK-NEXT:    blr
+
+entry:
+  ret <2 x double> <double 0xFFFFFFFFE0000000, double 0xFFFFFFFFE0000000>
+}
+
+define dso_local <2 x double> @testFloatToDoubleInfinity() local_unnamed_addr {
+; CHECK-LABEL: testFloatToDoubleInfinity:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltidp vs34, 2139095040
+; CHECK-NEXT:    blr
+
+entry:
+  ret <2 x double> <double 0x7FF0000000000000, double 0x7FF0000000000000>
+}
+
+define dso_local float @testFloatScalar() local_unnamed_addr {
+; CHECK-LABEL: testFloatScalar:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltidp vs1, 1135290941
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    blr
+
+entry:
+  ret float 0x40756547A0000000
+}
+
+define dso_local float @testFloatZeroScalar() local_unnamed_addr {
+; CHECK-LABEL: testFloatZeroScalar:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor f1, f1, f1
+; CHECK-NEXT:    blr
+
+entry:
+  ret float 0.000000e+00
+}
+
+define dso_local double @testDoubleRepresentableScalar() local_unnamed_addr {
+; CHECK-LABEL: testDoubleRepresentableScalar:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltidp vs1, 1135290941
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    blr
+
+entry:
+  ret double 0x40756547A0000000
+}
+
+define dso_local double @testDoubleZeroScalar() local_unnamed_addr {
+; CHECK-LABEL: testDoubleZeroScalar:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor f1, f1, f1
+; CHECK-NEXT:    blr
+
+entry:
+  ret double 0.000000e+00
+}
+
+define dso_local <4 x i32> @vec_splati() local_unnamed_addr {
+; CHECK-LABEL: vec_splati:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw vs34, -17
+; CHECK-NEXT:    blr
+entry:
+  ret <4 x i32> <i32 -17, i32 -17, i32 -17, i32 -17>
+}
+
+define dso_local <2 x double> @vec_splatid() local_unnamed_addr {
+; CHECK-LABEL: vec_splatid:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltidp vs34, 1065353216
+; CHECK-NEXT:    blr
+entry:
+  ret <2 x double> <double 1.000000e+00, double 1.000000e+00>
+}
diff --git a/llvm/test/CodeGen/PowerPC/p10-splatImm32.ll b/llvm/test/CodeGen/PowerPC/p10-splatImm32.ll
new file mode 100644
index 0000000000000..d610bd260fc95
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/p10-splatImm32.ll
@@ -0,0 +1,120 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
+; RUN:     -ppc-asm-full-reg-names -mcpu=pwr10 < %s | \
+; RUN:     FileCheck --check-prefix=CHECK-LE %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -O2 \
+; RUN:     -ppc-asm-full-reg-names -mcpu=pwr10 < %s | \
+; RUN:     FileCheck --check-prefix=CHECK-BE %s
+
+; Function Attrs: norecurse nounwind readnone
+define  <4 x i32> @test_xxsplti32dx_1(<4 x i32> %a) {
+; CHECK-LE-LABEL: test_xxsplti32dx_1:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    xxsplti32dx vs34, 0, 566
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_xxsplti32dx_1:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    xxsplti32dx vs34, 1, 566
+; CHECK-BE-NEXT:    blr
+entry:
+  %vecins1 = shufflevector <4 x i32> %a, <4 x i32> <i32 undef, i32 566, i32 undef, i32 566>, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x i32> %vecins1
+}
+
+; Function Attrs: norecurse nounwind readnone
+define  <4 x i32> @test_xxsplti32dx_2(<4 x i32> %a) {
+; CHECK-LE-LABEL: test_xxsplti32dx_2:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    xxsplti32dx vs34, 1, 33
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_xxsplti32dx_2:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    xxsplti32dx vs34, 0, 33
+; CHECK-BE-NEXT:    blr
+entry:
+  %vecins1 = shufflevector <4 x i32> <i32 33, i32 undef, i32 33, i32 undef>, <4 x i32> %a, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x i32> %vecins1
+}
+
+; Function Attrs: norecurse nounwind readnone
+define  <4 x i32> @test_xxsplti32dx_3(<4 x i32> %a) {
+; CHECK-LE-LABEL: test_xxsplti32dx_3:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    xxsplti32dx vs34, 0, 12
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_xxsplti32dx_3:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    xxsplti32dx vs34, 1, 12
+; CHECK-BE-NEXT:    blr
+entry:
+  %vecins1 = shufflevector <4 x i32> %a, <4 x i32> <i32 undef, i32 12, i32 undef, i32 12>, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x i32> %vecins1
+}
+
+; Function Attrs: norecurse nounwind readnone
+define  <4 x i32> @test_xxsplti32dx_4(<4 x i32> %a) {
+; CHECK-LE-LABEL: test_xxsplti32dx_4:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    xxsplti32dx vs34, 1, -683
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_xxsplti32dx_4:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    xxsplti32dx vs34, 0, -683
+; CHECK-BE-NEXT:    blr
+entry:
+  %vecins1 = shufflevector <4 x i32> <i32 -683, i32 undef, i32 -683, i32 undef>, <4 x i32> %a, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x i32> %vecins1
+}
+
+; Function Attrs: nounwind
+define  <4 x float> @test_xxsplti32dx_5(<4 x float> %vfa) {
+; CHECK-LE-LABEL: test_xxsplti32dx_5:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    xxsplti32dx vs34, 0, 1065353216
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_xxsplti32dx_5:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    xxsplti32dx vs34, 1, 1065353216
+; CHECK-BE-NEXT:    blr
+entry:
+  %vecins3.i = shufflevector <4 x float> %vfa, <4 x float> <float undef, float 1.000000e+00, float undef, float 1.000000e+00>, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x float> %vecins3.i
+}
+
+; Function Attrs: nounwind
+define  <4 x float> @test_xxsplti32dx_6(<4 x float> %vfa) {
+; CHECK-LE-LABEL: test_xxsplti32dx_6:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    xxsplti32dx vs34, 1, 1073741824
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_xxsplti32dx_6:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    xxsplti32dx vs34, 0, 1073741824
+; CHECK-BE-NEXT:    blr
+entry:
+  %vecins3.i = shufflevector <4 x float> <float 2.000000e+00, float undef, float 2.000000e+00, float undef>, <4 x float> %vfa, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x float> %vecins3.i
+}
+
+; Function Attrs: norecurse nounwind readnone
+; Test to illustrate when the splat is narrower than 32-bits.
+define dso_local <4 x i32> @test_xxsplti32dx_7(<4 x i32> %a) local_unnamed_addr #0 {
+; CHECK-LE-LABEL: test_xxsplti32dx_7:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    xxsplti32dx vs34, 1, -1414812757
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_xxsplti32dx_7:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    xxsplti32dx vs34, 0, -1414812757
+; CHECK-BE-NEXT:    blr
+entry:
+  %vecins1 = shufflevector <4 x i32> <i32 -1414812757, i32 undef, i32 -1414812757, i32 undef>, <4 x i32> %a, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x i32> %vecins1
+}
diff --git a/llvm/test/CodeGen/PowerPC/p10-vsx-pcv.ll b/llvm/test/CodeGen/PowerPC/p10-vsx-pcv.ll
new file mode 100644
index 0000000000000..ffab5dabf52cb
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/p10-vsx-pcv.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   FileCheck %s
+
+; These test cases aim to test the VSX PCV Generate Operations on Power10.
+
+declare <16 x i8> @llvm.ppc.vsx.xxgenpcvbm(<16 x i8>, i32)
+declare <8 x i16> @llvm.ppc.vsx.xxgenpcvhm(<8 x i16>, i32)
+declare <4 x i32> @llvm.ppc.vsx.xxgenpcvwm(<4 x i32>, i32)
+declare <2 x i64> @llvm.ppc.vsx.xxgenpcvdm(<2 x i64>, i32)
+
+define <16 x i8> @test_xxgenpcvbm(<16 x i8> %a) {
+; CHECK-LABEL: test_xxgenpcvbm:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxgenpcvbm v2, v2, 1
+; CHECK-NEXT:    blr
+entry:
+  %gen = tail call <16 x i8> @llvm.ppc.vsx.xxgenpcvbm(<16 x i8> %a, i32 1)
+  ret <16 x i8> %gen
+}
+
+define <8 x i16> @test_xxgenpcvhm(<8 x i16> %a) {
+; CHECK-LABEL: test_xxgenpcvhm:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxgenpcvhm v2, v2, 1
+; CHECK-NEXT:    blr
+entry:
+  %gen = tail call <8 x i16> @llvm.ppc.vsx.xxgenpcvhm(<8 x i16> %a, i32 1)
+  ret <8 x i16> %gen
+}
+
+define <4 x i32> @test_xxgenpcvwm(<4 x i32> %a) {
+; CHECK-LABEL: test_xxgenpcvwm:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxgenpcvwm v2, v2, 1
+; CHECK-NEXT:    blr
+entry:
+  %gen = tail call <4 x i32> @llvm.ppc.vsx.xxgenpcvwm(<4 x i32> %a, i32 1)
+  ret <4 x i32> %gen
+}
+
+define <2 x i64> @test_xxgenpcvdm(<2 x i64> %a) {
+; CHECK-LABEL: test_xxgenpcvdm:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxgenpcvdm v2, v2, 1
+; CHECK-NEXT:    blr
+entry:
+  %gen = tail call <2 x i64> @llvm.ppc.vsx.xxgenpcvdm(<2 x i64> %a, i32 1)
+  ret <2 x i64> %gen
+}
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-got-indirect.ll b/llvm/test/CodeGen/PowerPC/pcrel-got-indirect.ll
index 7f7659b356ee7..c838308f3aa65 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel-got-indirect.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel-got-indirect.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -enable-ppc-quad-precision -ppc-asm-full-reg-names \
-; RUN:   -ppc-vsr-nums-as-vr < %s | FileCheck %s
+; RUN:   -mcpu=future -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s \
+; RUN:   | FileCheck %s
 
 %struct.Struct = type { i8, i16, i32 }
 
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-local-caller-toc.ll b/llvm/test/CodeGen/PowerPC/pcrel-local-caller-toc.ll
new file mode 100644
index 0000000000000..2e248506c7b7b
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/pcrel-local-caller-toc.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names < %s | FileCheck %s
+
+; The purpose of this test is to check the call protocols for the situation
+; where the caller has PC Relative disabled, the callee has PC Relative
+; enabled and both functions are in the same file.
+; Note that the callee does not know if it clobbers the TOC because it
+; contains an external call to @externalFunc.
+
+@global = external local_unnamed_addr global i32, align 4
+
+define dso_local signext i32 @callee(i32 signext %a) local_unnamed_addr #0 {
+; CHECK-LABEL: callee:
+; CHECK:         .localentry callee, 1
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -48(r1)
+; CHECK-NEXT:    mr r30, r3
+; CHECK-NEXT:    bl externalFunc@notoc
+; CHECK-NEXT:    add r3, r3, r30
+; CHECK-NEXT:    extsw r3, r3
+; CHECK-NEXT:    addi r1, r1, 48
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+entry:
+  %call = tail call signext i32 @externalFunc(i32 signext %a) #3
+  %add = add nsw i32 %call, %a
+  ret i32 %add
+}
+
+declare signext i32 @externalFunc(i32 signext) local_unnamed_addr #1
+
+define dso_local void @caller(i32 signext %a) local_unnamed_addr #2 {
+; CHECK-LABEL: caller:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -48(r1)
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    ld r30, .LC0@toc@l(r4)
+; CHECK-NEXT:    lwz r4, 0(r30)
+; CHECK-NEXT:    add r3, r4, r3
+; CHECK-NEXT:    extsw r3, r3
+; CHECK-NEXT:    bl callee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mullw r3, r3, r3
+; CHECK-NEXT:    stw r3, 0(r30)
+; CHECK-NEXT:    addi r1, r1, 48
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+entry:
+  %0 = load i32, i32* @global, align 4
+  %add = add nsw i32 %0, %a
+  %call = tail call signext i32 @callee(i32 signext %add)
+  %mul = mul nsw i32 %call, %call
+  store i32 %mul, i32* @global, align 4
+  ret void
+}
+
+define dso_local signext i32 @tail_caller(i32 signext %a) local_unnamed_addr #2 {
+; CHECK-LABEL: tail_caller:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
+; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
+; CHECK-NEXT:    lwz r4, 0(r4)
+; CHECK-NEXT:    add r3, r4, r3
+; CHECK-NEXT:    extsw r3, r3
+; CHECK-NEXT:    bl callee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+entry:
+  %0 = load i32, i32* @global, align 4
+  %add = add nsw i32 %0, %a
+  %call = tail call signext i32 @callee(i32 signext %add)
+  ret i32 %call
+}
+
+
+; Left the target features in this test because it is important that caller has
+; -pcrelative-memops while callee has +pcrelative-memops
+attributes #0 = { nounwind "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+pcrelative-memops,+power8-vector,+power9-vector,+vsx,-htm,-qpx,-spe" }
+attributes #1 = { "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+pcrelative-memops,+power8-vector,+power9-vector,+vsx,-htm,-qpx,-spe" }
+attributes #2 = { nounwind "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+power9-vector,+vsx,-htm,-pcrelative-memops,-qpx,-spe" }
+attributes #3 = { nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/popcount.ll b/llvm/test/CodeGen/PowerPC/popcount.ll
new file mode 100644
index 0000000000000..fb20f1d3ee43b
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/popcount.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -O0 -mtriple=powerpc64le-unknown-unknown | FileCheck %s
+
+; Function Attrs: nobuiltin nounwind readonly
+define i8 @popcount128(i128* nocapture nonnull readonly %0) {
+; CHECK-LABEL: popcount128:
+; CHECK:       # %bb.0: # %Entry
+; CHECK-NEXT:    ld 4, 0(3)
+; CHECK-NEXT:    ld 3, 8(3)
+; CHECK-NEXT:    popcntd 3, 3
+; CHECK-NEXT:    popcntd 4, 4
+; CHECK-NEXT:    add 3, 4, 3
+; CHECK-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; CHECK-NEXT:    clrldi 3, 3, 56
+; CHECK-NEXT:    blr
+Entry:
+  %1 = load i128, i128* %0, align 16
+  %2 = tail call i128 @llvm.ctpop.i128(i128 %1)
+  %3 = trunc i128 %2 to i8
+  ret i8 %3
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare i128 @llvm.ctpop.i128(i128)
+
+; Function Attrs: nobuiltin nounwind readonly
+define i16 @popcount256(i256* nocapture nonnull readonly %0) {
+; CHECK-LABEL: popcount256:
+; CHECK:       # %bb.0: # %Entry
+; CHECK-NEXT:    ld 4, 0(3)
+; CHECK-NEXT:    ld 5, 8(3)
+; CHECK-NEXT:    ld 6, 16(3)
+; CHECK-NEXT:    ld 3, 24(3)
+; CHECK-NEXT:    popcntd 3, 3
+; CHECK-NEXT:    popcntd 6, 6
+; CHECK-NEXT:    add 3, 6, 3
+; CHECK-NEXT:    popcntd 5, 5
+; CHECK-NEXT:    popcntd 4, 4
+; CHECK-NEXT:    add 4, 4, 5
+; CHECK-NEXT:    add 3, 4, 3
+; CHECK-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; CHECK-NEXT:    clrldi 3, 3, 48
+; CHECK-NEXT:    blr
+Entry:
+  %1 = load i256, i256* %0, align 16
+  %2 = tail call i256 @llvm.ctpop.i256(i256 %1)
+  %3 = trunc i256 %2 to i16
+  ret i16 %3
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare i256 @llvm.ctpop.i256(i256)
+
+define <1 x i128> @popcount1x128(<1 x i128> %0) {
+; CHECK-LABEL: popcount1x128:
+; CHECK:       # %bb.0: # %Entry
+; CHECK-NEXT:    xxlor 0, 34, 34
+; CHECK-NEXT:    # kill: def $f0 killed $f0 killed $vsl0
+; CHECK-NEXT:    mffprd 3, 0
+; CHECK-NEXT:    popcntd 3, 3
+; CHECK-NEXT:    xxswapd 0, 34
+; CHECK-NEXT:    # kill: def $f0 killed $f0 killed $vsl0
+; CHECK-NEXT:    mffprd 4, 0
+; CHECK-NEXT:    popcntd 4, 4
+; CHECK-NEXT:    add 3, 4, 3
+; CHECK-NEXT:    mtfprd 0, 3
+; CHECK-NEXT:    # kill: def $vsl0 killed $f0
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:    mtfprd 1, 3
+; CHECK-NEXT:    # kill: def $vsl1 killed $f1
+; CHECK-NEXT:    xxmrghd 34, 1, 0
+; CHECK-NEXT:    blr
+Entry:
+  %1 = tail call <1 x i128> @llvm.ctpop.v1.i128(<1 x i128> %0)
+  ret <1 x i128> %1
+}
+
+declare <1 x i128> @llvm.ctpop.v1.i128(<1 x i128>)
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-P9-mod.ll b/llvm/test/CodeGen/PowerPC/ppc64-P9-mod.ll
index c59bec655f9b7..e99074e7f90f3 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-P9-mod.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-P9-mod.ll
@@ -1,6 +1,12 @@
 ; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr9 -verify-machineinstrs | FileCheck %s
 ; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr9 -verify-machineinstrs | FileCheck %s
 ; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-PWR8 -implicit-check-not mod[us][wd]
+; RUN: opt < %s -div-rem-pairs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr9 | \
+; RUN:   llc -verify-machineinstrs | FileCheck %s -check-prefix=CHECK-DRP
+; RUN: opt < %s -div-rem-pairs -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr9 | \
+; RUN:   llc -verify-machineinstrs | FileCheck %s -check-prefix=CHECK-DRP
+; RUN: opt < %s -div-rem-pairs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 | \
+; RUN:   llc -verify-machineinstrs | FileCheck %s -check-prefix=CHECK-PWR8 -implicit-check-not mod[us][wd]
 
 @mod_resultsw = local_unnamed_addr global i32 0, align 4
 @mod_resultud = local_unnamed_addr global i64 0, align 8
@@ -82,13 +88,16 @@ entry:
   store i32 %div, i32* @div_resultsw, align 4
   ret void
 ; CHECK-LABEL: modulo_div_sw
-; CHECK-NOT: modsw
-; CHECK: div
-; CHECK-NOT: modsw
-; CHECK: mull
-; CHECK-NOT: modsw
-; CHECK: sub
+; CHECK: modsw {{[0-9]+}}, 3, 4
 ; CHECK: blr
+; CHECK-DRP-LABEL: modulo_div_sw
+; CHECK-DRP-NOT: modsw
+; CHECK-DRP: div
+; CHECK-DRP-NOT: modsw
+; CHECK-DRP: mull
+; CHECK-DRP-NOT: modsw
+; CHECK-DRP: sub
+; CHECK-DRP: blr
 ; CHECK-PWR8-LABEL: modulo_div_sw
 ; CHECK-PWR8: div
 ; CHECK-PWR8: mull
@@ -123,13 +132,16 @@ entry:
   store i32 %div, i32* @div_resultuw, align 4
   ret void
 ; CHECK-LABEL: modulo_div_uw
-; CHECK-NOT: modsw
-; CHECK: div
-; CHECK-NOT: modsw
-; CHECK: mull
-; CHECK-NOT: modsw
-; CHECK: sub
+; CHECK: moduw {{[0-9]+}}, 3, 4
 ; CHECK: blr
+; CHECK-DRP-LABEL: modulo_div_uw
+; CHECK-DRP-NOT: moduw
+; CHECK-DRP: div
+; CHECK-DRP-NOT: moduw
+; CHECK-DRP: mull
+; CHECK-DRP-NOT: moduw
+; CHECK-DRP: sub
+; CHECK-DRP: blr
 ; CHECK-PWR8-LABEL: modulo_div_uw
 ; CHECK-PWR8: div
 ; CHECK-PWR8: mull
@@ -232,9 +244,6 @@ entry:
 }
 
 ; Function Attrs: norecurse nounwind
-; FIXME On power 9 this test will still produce modsw because the divide is in
-; a different block than the remainder. Due to the nature of the SDAG we cannot
-; see the div in the other block.
 define void @blocks_modulo_div_sw(i32 signext %a, i32 signext %b, i32 signext %c) local_unnamed_addr {
 entry:
   %div = sdiv i32 %a, %b
@@ -253,6 +262,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK: div
 ; CHECK: modsw {{[0-9]+}}, 3, 4
 ; CHECK: blr
+; CHECK-DRP-LABEL: blocks_modulo_div_sw
+; CHECK-DRP-NOT: modsw
+; CHECK-DRP: div
+; CHECK-DRP-NOT: modsw
+; CHECK-DRP: mull
+; CHECK-DRP-NOT: modsw
+; CHECK-DRP: sub
+; CHECK-DRP: blr
 ; CHECK-PWR8-LABEL: blocks_modulo_div_sw
 ; CHECK-PWR8: div
 ; CHECK-PWR8: mull
@@ -260,4 +277,3 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-PWR8: blr
 }
 
-
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-P9-setb.ll b/llvm/test/CodeGen/PowerPC/ppc64-P9-setb.ll
index feb20465c510c..4762b10af7dd4 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-P9-setb.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-P9-setb.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -verify-machineinstrs -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \
-; RUN:   -enable-ppc-quad-precision -ppc-asm-full-reg-names < %s | FileCheck %s
+; RUN:   -ppc-asm-full-reg-names < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mcpu=pwr8 -mtriple=powerpc64le-unknown-unknown \
 ; RUN:   -ppc-asm-full-reg-names < %s | FileCheck %s -check-prefix=CHECK-PWR8    \
 ; RUN:   -implicit-check-not "\<setb\>"
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-varargs.ll b/llvm/test/CodeGen/PowerPC/ppc64-varargs.ll
new file mode 100644
index 0000000000000..5aeedbf13e0b5
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/ppc64-varargs.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs -mtriple=powerpc64 \
+; RUN:   < %s | FileCheck --check-prefix=BE %s
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs -mtriple=powerpc64le \
+; RUN:   < %s | FileCheck --check-prefix=LE %s
+
+define i32 @f(...) nounwind {
+; BE-LABEL: f:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    li r3, 0
+; BE-NEXT:    blr
+;
+; LE-LABEL: f:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    li r3, 0
+; LE-NEXT:    blr
+entry:
+  ret i32 0
+}
+
+define i32 @f1(...) nounwind {
+; BE-LABEL: f1:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    mr r11, r3
+; BE-NEXT:    addi r12, r1, 48
+; BE-NEXT:    li r3, 0
+; BE-NEXT:    std r11, 48(r1)
+; BE-NEXT:    std r4, 56(r1)
+; BE-NEXT:    std r5, 64(r1)
+; BE-NEXT:    std r6, 72(r1)
+; BE-NEXT:    std r7, 80(r1)
+; BE-NEXT:    std r8, 88(r1)
+; BE-NEXT:    std r9, 96(r1)
+; BE-NEXT:    std r10, 104(r1)
+; BE-NEXT:    std r12, -8(r1)
+; BE-NEXT:    blr
+;
+; LE-LABEL: f1:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    std r3, 32(r1)
+; LE-NEXT:    std r4, 40(r1)
+; LE-NEXT:    addi r4, r1, 32
+; LE-NEXT:    li r3, 0
+; LE-NEXT:    std r5, 48(r1)
+; LE-NEXT:    std r6, 56(r1)
+; LE-NEXT:    std r7, 64(r1)
+; LE-NEXT:    std r8, 72(r1)
+; LE-NEXT:    std r9, 80(r1)
+; LE-NEXT:    std r10, 88(r1)
+; LE-NEXT:    std r4, -8(r1)
+; LE-NEXT:    blr
+entry:
+  %va = alloca i8*, align 8
+  %va.cast = bitcast i8** %va to i8*
+  call void @llvm.va_start(i8* %va.cast)
+  ret i32 0
+}
+
+declare void @llvm.va_start(i8*) nounwind
diff --git a/llvm/test/CodeGen/PowerPC/preemption.ll b/llvm/test/CodeGen/PowerPC/preemption.ll
index 9a0d81fbbd78b..fc38ba47d8dfd 100644
--- a/llvm/test/CodeGen/PowerPC/preemption.ll
+++ b/llvm/test/CodeGen/PowerPC/preemption.ll
@@ -1,8 +1,8 @@
-; RUN: llc -mtriple powerpc64le-unkown-gnu-linux  -relocation-model=pic \
+; RUN: llc -mtriple powerpc64le-unknown-gnu-linux  -relocation-model=pic \
 ; RUN: < %s |  FileCheck %s
-; RUN: llc -mtriple powerpc64le-unkown-gnu-linux -relocation-model=static \
+; RUN: llc -mtriple powerpc64le-unknown-gnu-linux -relocation-model=static \
 ; RUN: < %s |  FileCheck --check-prefix=STATIC %s
-; RUN: llc -mtriple powerpc64le-unkown-gnu-linux -relocation-model=pic \
+; RUN: llc -mtriple powerpc64le-unknown-gnu-linux -relocation-model=pic \
 ; RUN: < %s |  FileCheck %s
 
 ; globals
diff --git a/llvm/test/CodeGen/PowerPC/recipest.ll b/llvm/test/CodeGen/PowerPC/recipest.ll
index 4276d944af8f1..7ceddd95e5735 100644
--- a/llvm/test/CodeGen/PowerPC/recipest.ll
+++ b/llvm/test/CodeGen/PowerPC/recipest.ll
@@ -1176,14 +1176,7 @@ define fp128 @hoo5_fmf(fp128 %a) #1 {
 ;
 ; CHECK-P9-LABEL: hoo5_fmf:
 ; CHECK-P9:       # %bb.0:
-; CHECK-P9-NEXT:    mflr 0
-; CHECK-P9-NEXT:    std 0, 16(1)
-; CHECK-P9-NEXT:    stdu 1, -32(1)
-; CHECK-P9-NEXT:    bl sqrtl
-; CHECK-P9-NEXT:    nop
-; CHECK-P9-NEXT:    addi 1, 1, 32
-; CHECK-P9-NEXT:    ld 0, 16(1)
-; CHECK-P9-NEXT:    mtlr 0
+; CHECK-P9-NEXT:    xssqrtqp 2, 2
 ; CHECK-P9-NEXT:    blr
   %r = call reassoc ninf afn fp128 @llvm.sqrt.f128(fp128 %a)
   ret fp128 %r
@@ -1216,14 +1209,7 @@ define fp128 @hoo5_safe(fp128 %a) #1 {
 ;
 ; CHECK-P9-LABEL: hoo5_safe:
 ; CHECK-P9:       # %bb.0:
-; CHECK-P9-NEXT:    mflr 0
-; CHECK-P9-NEXT:    std 0, 16(1)
-; CHECK-P9-NEXT:    stdu 1, -32(1)
-; CHECK-P9-NEXT:    bl sqrtl
-; CHECK-P9-NEXT:    nop
-; CHECK-P9-NEXT:    addi 1, 1, 32
-; CHECK-P9-NEXT:    ld 0, 16(1)
-; CHECK-P9-NEXT:    mtlr 0
+; CHECK-P9-NEXT:    xssqrtqp 2, 2
 ; CHECK-P9-NEXT:    blr
   %r = call fp128 @llvm.sqrt.f128(fp128 %a)
   ret fp128 %r
diff --git a/llvm/test/CodeGen/PowerPC/rematerializable-instruction-machine-licm.ll b/llvm/test/CodeGen/PowerPC/rematerializable-instruction-machine-licm.ll
new file mode 100644
index 0000000000000..9ad5cb8864722
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/rematerializable-instruction-machine-licm.ll
@@ -0,0 +1,618 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+define zeroext i32 @test1(i64 %0, i64* %1) {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stdu 1, -720(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 720
+; CHECK-NEXT:    .cfi_offset r14, -144
+; CHECK-NEXT:    .cfi_offset r15, -136
+; CHECK-NEXT:    .cfi_offset r16, -128
+; CHECK-NEXT:    .cfi_offset r17, -120
+; CHECK-NEXT:    .cfi_offset r18, -112
+; CHECK-NEXT:    .cfi_offset r19, -104
+; CHECK-NEXT:    .cfi_offset r20, -96
+; CHECK-NEXT:    .cfi_offset r21, -88
+; CHECK-NEXT:    .cfi_offset r22, -80
+; CHECK-NEXT:    .cfi_offset r23, -72
+; CHECK-NEXT:    .cfi_offset r24, -64
+; CHECK-NEXT:    .cfi_offset r25, -56
+; CHECK-NEXT:    .cfi_offset r26, -48
+; CHECK-NEXT:    .cfi_offset r27, -40
+; CHECK-NEXT:    .cfi_offset r28, -32
+; CHECK-NEXT:    .cfi_offset r29, -24
+; CHECK-NEXT:    .cfi_offset r30, -16
+; CHECK-NEXT:    .cfi_offset r31, -8
+; CHECK-NEXT:    .cfi_offset r2, -152
+; CHECK-NEXT:    lis 5, 4
+; CHECK-NEXT:    ori 6, 5, 6292
+; CHECK-NEXT:    ori 5, 5, 6291
+; CHECK-NEXT:    sldi 6, 6, 32
+; CHECK-NEXT:    oris 7, 6, 13030
+; CHECK-NEXT:    oris 8, 6, 13066
+; CHECK-NEXT:    ori 7, 7, 3704
+; CHECK-NEXT:    oris 9, 6, 13054
+; CHECK-NEXT:    ori 8, 8, 44408
+; CHECK-NEXT:    ori 9, 9, 30840
+; CHECK-NEXT:    add 7, 4, 7
+; CHECK-NEXT:    oris 10, 6, 13042
+; CHECK-NEXT:    ori 10, 10, 17272
+; CHECK-NEXT:    std 7, 384(1) # 8-byte Folded Spill
+; CHECK-NEXT:    add 7, 4, 8
+; CHECK-NEXT:    oris 11, 6, 13078
+; CHECK-NEXT:    ori 11, 11, 57976
+; CHECK-NEXT:    std 7, 376(1) # 8-byte Folded Spill
+; CHECK-NEXT:    add 7, 4, 9
+; CHECK-NEXT:    oris 12, 6, 13115
+; CHECK-NEXT:    ori 12, 12, 33144
+; CHECK-NEXT:    std 7, 368(1) # 8-byte Folded Spill
+; CHECK-NEXT:    add 7, 4, 10
+; CHECK-NEXT:    oris 0, 6, 13103
+; CHECK-NEXT:    ori 0, 0, 19576
+; CHECK-NEXT:    std 7, 360(1) # 8-byte Folded Spill
+; CHECK-NEXT:    add 7, 4, 11
+; CHECK-NEXT:    std 30, 704(1) # 8-byte Folded Spill
+; CHECK-NEXT:    oris 30, 6, 13091
+; CHECK-NEXT:    ori 30, 30, 6008
+; CHECK-NEXT:    std 7, 352(1) # 8-byte Folded Spill
+; CHECK-NEXT:    add 7, 4, 12
+; CHECK-NEXT:    std 29, 696(1) # 8-byte Folded Spill
+; CHECK-NEXT:    oris 29, 6, 13127
+; CHECK-NEXT:    ori 29, 29, 46712
+; CHECK-NEXT:    sldi 5, 5, 32
+; CHECK-NEXT:    oris 5, 5, 29347
+; CHECK-NEXT:    ori 5, 5, 20088
+; CHECK-NEXT:    lis 8, 402
+; CHECK-NEXT:    lis 9, 451
+; CHECK-NEXT:    lis 10, 500
+; CHECK-NEXT:    lis 11, 549
+; CHECK-NEXT:    std 31, 712(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 2, 568(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 7, 344(1) # 8-byte Folded Spill
+; CHECK-NEXT:    add 7, 4, 0
+; CHECK-NEXT:    std 28, 688(1) # 8-byte Folded Spill
+; CHECK-NEXT:    oris 28, 6, 13164
+; CHECK-NEXT:    ori 28, 28, 21880
+; CHECK-NEXT:    std 7, 336(1) # 8-byte Folded Spill
+; CHECK-NEXT:    add 7, 4, 30
+; CHECK-NEXT:    std 27, 680(1) # 8-byte Folded Spill
+; CHECK-NEXT:    oris 27, 6, 13152
+; CHECK-NEXT:    ori 27, 27, 8312
+; CHECK-NEXT:    std 7, 328(1) # 8-byte Folded Spill
+; CHECK-NEXT:    add 7, 4, 29
+; CHECK-NEXT:    std 26, 672(1) # 8-byte Folded Spill
+; CHECK-NEXT:    oris 26, 6, 13139
+; CHECK-NEXT:    ori 26, 26, 60280
+; CHECK-NEXT:    std 7, 320(1) # 8-byte Folded Spill
+; CHECK-NEXT:    add 7, 4, 28
+; CHECK-NEXT:    std 25, 664(1) # 8-byte Folded Spill
+; CHECK-NEXT:    oris 25, 6, 13176
+; CHECK-NEXT:    ori 25, 25, 35448
+; CHECK-NEXT:    std 7, 312(1) # 8-byte Folded Spill
+; CHECK-NEXT:    add 7, 4, 27
+; CHECK-NEXT:    std 7, 304(1) # 8-byte Folded Spill
+; CHECK-NEXT:    add 7, 4, 26
+; CHECK-NEXT:    std 7, 296(1) # 8-byte Folded Spill
+; CHECK-NEXT:    add 7, 4, 25
+; CHECK-NEXT:    std 7, 288(1) # 8-byte Folded Spill
+; CHECK-NEXT:    oris 7, 6, 13213
+; CHECK-NEXT:    ori 7, 7, 10616
+; CHECK-NEXT:    add 7, 4, 7
+; CHECK-NEXT:    std 7, 280(1) # 8-byte Folded Spill
+; CHECK-NEXT:    oris 7, 6, 13200
+; CHECK-NEXT:    oris 6, 6, 13188
+; CHECK-NEXT:    ori 7, 7, 62584
+; CHECK-NEXT:    ori 6, 6, 49016
+; CHECK-NEXT:    add 7, 4, 7
+; CHECK-NEXT:    add 6, 4, 6
+; CHECK-NEXT:    add 4, 4, 5
+; CHECK-NEXT:    lis 5, 268
+; CHECK-NEXT:    std 4, 256(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 585
+; CHECK-NEXT:    ori 4, 4, 61440
+; CHECK-NEXT:    std 4, 560(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 48
+; CHECK-NEXT:    ori 4, 4, 54272
+; CHECK-NEXT:    std 4, 552(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 97
+; CHECK-NEXT:    ori 4, 4, 43008
+; CHECK-NEXT:    std 4, 544(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 146
+; CHECK-NEXT:    ori 4, 4, 31744
+; CHECK-NEXT:    std 4, 536(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 195
+; CHECK-NEXT:    ori 4, 4, 20480
+; CHECK-NEXT:    std 4, 528(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 244
+; CHECK-NEXT:    ori 4, 4, 9216
+; CHECK-NEXT:    std 4, 520(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 292
+; CHECK-NEXT:    ori 4, 4, 63488
+; CHECK-NEXT:    std 4, 512(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 341
+; CHECK-NEXT:    ori 4, 4, 52224
+; CHECK-NEXT:    std 4, 504(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 390
+; CHECK-NEXT:    ori 4, 4, 40960
+; CHECK-NEXT:    std 4, 496(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 439
+; CHECK-NEXT:    ori 4, 4, 29696
+; CHECK-NEXT:    std 4, 488(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 488
+; CHECK-NEXT:    ori 4, 4, 18432
+; CHECK-NEXT:    std 4, 480(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 537
+; CHECK-NEXT:    ori 4, 4, 7168
+; CHECK-NEXT:    std 4, 472(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 36
+; CHECK-NEXT:    ori 4, 4, 40704
+; CHECK-NEXT:    std 4, 464(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 85
+; CHECK-NEXT:    ori 4, 4, 29440
+; CHECK-NEXT:    std 4, 456(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 134
+; CHECK-NEXT:    ori 4, 4, 18176
+; CHECK-NEXT:    std 4, 448(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 183
+; CHECK-NEXT:    ori 4, 4, 6912
+; CHECK-NEXT:    std 4, 440(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 231
+; CHECK-NEXT:    ori 4, 4, 61184
+; CHECK-NEXT:    std 4, 432(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 280
+; CHECK-NEXT:    ori 4, 4, 49920
+; CHECK-NEXT:    std 4, 424(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 329
+; CHECK-NEXT:    ori 4, 4, 38656
+; CHECK-NEXT:    std 4, 416(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 378
+; CHECK-NEXT:    ori 4, 4, 27392
+; CHECK-NEXT:    std 4, 408(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 427
+; CHECK-NEXT:    ori 4, 4, 16128
+; CHECK-NEXT:    std 4, 400(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 476
+; CHECK-NEXT:    ori 4, 4, 4864
+; CHECK-NEXT:    std 4, 248(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 524
+; CHECK-NEXT:    ori 4, 4, 59136
+; CHECK-NEXT:    std 4, 240(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 573
+; CHECK-NEXT:    ori 4, 4, 47872
+; CHECK-NEXT:    std 4, 232(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 24
+; CHECK-NEXT:    ori 4, 4, 27136
+; CHECK-NEXT:    std 4, 224(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 73
+; CHECK-NEXT:    ori 4, 4, 15872
+; CHECK-NEXT:    std 4, 216(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 122
+; CHECK-NEXT:    ori 4, 4, 4608
+; CHECK-NEXT:    std 4, 208(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 170
+; CHECK-NEXT:    ori 4, 4, 58880
+; CHECK-NEXT:    std 4, 200(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 4, 219
+; CHECK-NEXT:    ori 4, 4, 47616
+; CHECK-NEXT:    std 4, 192(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 5, 36352
+; CHECK-NEXT:    lis 5, 317
+; CHECK-NEXT:    std 4, 184(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 5, 25088
+; CHECK-NEXT:    lis 5, 366
+; CHECK-NEXT:    std 4, 176(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 5, 13824
+; CHECK-NEXT:    lis 5, 415
+; CHECK-NEXT:    std 4, 168(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 5, 2560
+; CHECK-NEXT:    lis 5, 463
+; CHECK-NEXT:    std 4, 160(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 5, 56832
+; CHECK-NEXT:    lis 5, 512
+; CHECK-NEXT:    std 4, 152(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 5, 45568
+; CHECK-NEXT:    lis 5, 561
+; CHECK-NEXT:    std 4, 144(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 5, 34304
+; CHECK-NEXT:    lis 5, 12
+; CHECK-NEXT:    std 4, 136(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 5, 13568
+; CHECK-NEXT:    lis 5, 61
+; CHECK-NEXT:    std 4, 128(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 5, 2304
+; CHECK-NEXT:    lis 5, 109
+; CHECK-NEXT:    std 4, 120(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 5, 56576
+; CHECK-NEXT:    lis 5, 158
+; CHECK-NEXT:    std 4, 112(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 5, 45312
+; CHECK-NEXT:    lis 5, 207
+; CHECK-NEXT:    std 4, 104(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 5, 34048
+; CHECK-NEXT:    lis 5, 256
+; CHECK-NEXT:    std 6, 264(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 6, 305
+; CHECK-NEXT:    ld 30, 192(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 29, 184(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 28, 176(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 27, 168(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 26, 160(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 25, 152(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 0, 120(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 4, 96(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 5, 22784
+; CHECK-NEXT:    std 7, 272(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lis 7, 354
+; CHECK-NEXT:    std 4, 88(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 6, 11520
+; CHECK-NEXT:    ld 6, 240(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 4, 80(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 7, 256
+; CHECK-NEXT:    ld 7, 232(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 4, 72(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 8, 54528
+; CHECK-NEXT:    ld 8, 224(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 4, 64(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 9, 43264
+; CHECK-NEXT:    ld 9, 216(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 4, 56(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 10, 32000
+; CHECK-NEXT:    ld 10, 208(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 4, 48(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ori 4, 11, 20736
+; CHECK-NEXT:    ld 11, 200(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 4, 40(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 14, 576(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 15, 584(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 16, 592(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 17, 600(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 18, 608(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 19, 616(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 20, 624(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 21, 632(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 22, 640(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 23, 648(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 24, 656(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ld 5, 248(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 24, 144(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 23, 136(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 22, 112(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 21, 104(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 20, 96(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 19, 88(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 18, 80(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 17, 72(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 16, 64(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 15, 56(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 14, 48(1) # 8-byte Folded Reload
+; CHECK-NEXT:    li 4, 0
+; CHECK-NEXT:    ld 31, 40(1) # 8-byte Folded Reload
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  .LBB0_1: # =>This Loop Header: Depth=1
+; CHECK-NEXT:    # Child Loop BB0_2 Depth 2
+; CHECK-NEXT:    stw 4, 396(1) # 4-byte Folded Spill
+; CHECK-NEXT:    li 4, 83
+; CHECK-NEXT:    mtctr 4
+; CHECK-NEXT:    ld 12, 256(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 4, 128(1) # 8-byte Folded Reload
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB0_2: # Parent Loop BB0_1 Depth=1
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    ld 2, 560(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdux 3, 12, 2
+; CHECK-NEXT:    ld 2, 552(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 5
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 544(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 536(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 528(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 520(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 512(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 504(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 496(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 488(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 480(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 472(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 464(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 456(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 448(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 440(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 432(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 424(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 416(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 408(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    ld 2, 400(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stdx 3, 12, 2
+; CHECK-NEXT:    stdx 3, 12, 6
+; CHECK-NEXT:    stdx 3, 12, 7
+; CHECK-NEXT:    stdx 3, 12, 8
+; CHECK-NEXT:    stdx 3, 12, 9
+; CHECK-NEXT:    stdx 3, 12, 10
+; CHECK-NEXT:    stdx 3, 12, 11
+; CHECK-NEXT:    stdx 3, 12, 30
+; CHECK-NEXT:    stdx 3, 12, 29
+; CHECK-NEXT:    stdx 3, 12, 28
+; CHECK-NEXT:    stdx 3, 12, 27
+; CHECK-NEXT:    stdx 3, 12, 26
+; CHECK-NEXT:    stdx 3, 12, 25
+; CHECK-NEXT:    stdx 3, 12, 24
+; CHECK-NEXT:    stdx 3, 12, 23
+; CHECK-NEXT:    stdx 3, 12, 4
+; CHECK-NEXT:    stdx 3, 12, 0
+; CHECK-NEXT:    stdx 3, 12, 22
+; CHECK-NEXT:    stdx 3, 12, 21
+; CHECK-NEXT:    stdx 3, 12, 20
+; CHECK-NEXT:    stdx 3, 12, 19
+; CHECK-NEXT:    stdx 3, 12, 18
+; CHECK-NEXT:    stdx 3, 12, 17
+; CHECK-NEXT:    stdx 3, 12, 16
+; CHECK-NEXT:    stdx 3, 12, 15
+; CHECK-NEXT:    stdx 3, 12, 14
+; CHECK-NEXT:    stdx 3, 12, 31
+; CHECK-NEXT:    bdnz .LBB0_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    ld 12, 384(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 3, 0(12)
+; CHECK-NEXT:    ld 12, 376(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 3, 0(12)
+; CHECK-NEXT:    ld 12, 368(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 3, 0(12)
+; CHECK-NEXT:    ld 12, 360(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 3, 0(12)
+; CHECK-NEXT:    ld 12, 352(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 3, 0(12)
+; CHECK-NEXT:    ld 12, 344(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 3, 0(12)
+; CHECK-NEXT:    ld 12, 336(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 3, 0(12)
+; CHECK-NEXT:    ld 12, 328(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 3, 0(12)
+; CHECK-NEXT:    ld 12, 320(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 3, 0(12)
+; CHECK-NEXT:    ld 12, 312(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 3, 0(12)
+; CHECK-NEXT:    ld 12, 304(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 3, 0(12)
+; CHECK-NEXT:    ld 12, 296(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 3, 0(12)
+; CHECK-NEXT:    ld 12, 288(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 3, 0(12)
+; CHECK-NEXT:    ld 12, 280(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lwz 4, 396(1) # 4-byte Folded Reload
+; CHECK-NEXT:    addi 4, 4, 1
+; CHECK-NEXT:    std 3, 0(12)
+; CHECK-NEXT:    ld 12, 272(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 3, 0(12)
+; CHECK-NEXT:    xoris 12, 4, 6
+; CHECK-NEXT:    cmplwi 12, 6784
+; CHECK-NEXT:    ld 12, 264(1) # 8-byte Folded Reload
+; CHECK-NEXT:    std 3, 0(12)
+; CHECK-NEXT:    bne 0, .LBB0_1
+; CHECK-NEXT:  # %bb.4:
+; CHECK-NEXT:    ld 2, 568(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 31, 712(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 30, 704(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 29, 696(1) # 8-byte Folded Reload
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:    ld 28, 688(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 27, 680(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 26, 672(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 25, 664(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 24, 656(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 23, 648(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 22, 640(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 21, 632(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 20, 624(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 19, 616(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 18, 608(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 17, 600(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 16, 592(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 15, 584(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 14, 576(1) # 8-byte Folded Reload
+; CHECK-NEXT:    addi 1, 1, 720
+; CHECK-NEXT:    blr
+  %3 = getelementptr inbounds i64, i64* %1, i64 144115188075855
+  %4 = getelementptr i64, i64* %1, i64 144115586875855
+  %5 = getelementptr i64, i64* %1, i64 144115587175855
+  %6 = getelementptr i64, i64* %1, i64 144115587075855
+  %7 = getelementptr i64, i64* %1, i64 144115586975855
+  %8 = getelementptr i64, i64* %1, i64 144115587275855
+  %9 = getelementptr i64, i64* %1, i64 144115587575855
+  %10 = getelementptr i64, i64* %1, i64 144115587475855
+  %11 = getelementptr i64, i64* %1, i64 144115587375855
+  %12 = getelementptr i64, i64* %1, i64 144115587675855
+  %13 = getelementptr i64, i64* %1, i64 144115587975855
+  %14 = getelementptr i64, i64* %1, i64 144115587875855
+  %15 = getelementptr i64, i64* %1, i64 144115587775855
+  %16 = getelementptr i64, i64* %1, i64 144115588075855
+  %17 = getelementptr i64, i64* %1, i64 144115588375855
+  %18 = getelementptr i64, i64* %1, i64 144115588275855
+  %19 = getelementptr i64, i64* %1, i64 144115588175855
+  br label %20
+
+20:                                               ; preds = %2, %109
+  %21 = phi i32 [ 0, %2 ], [ %110, %109 ]
+  br label %22
+
+22:                                               ; preds = %22, %20
+  %23 = phi i64 [ 0, %20 ], [ %107, %22 ]
+  %24 = mul i64 %23, 400000
+  %25 = getelementptr i64, i64* %3, i64 %24
+  %26 = or i64 %23, 1
+  %27 = mul i64 %26, 400000
+  %28 = getelementptr i64, i64* %3, i64 %27
+  %29 = or i64 %23, 2
+  %30 = mul i64 %29, 400000
+  %31 = getelementptr i64, i64* %3, i64 %30
+  %32 = or i64 %23, 3
+  %33 = mul i64 %32, 400000
+  %34 = getelementptr i64, i64* %3, i64 %33
+  %35 = mul i64 %23, 400000
+  %36 = add i64 %35, 1600000
+  %37 = getelementptr i64, i64* %3, i64 %36
+  %38 = mul i64 %23, 400000
+  %39 = add i64 %38, 2000000
+  %40 = getelementptr i64, i64* %3, i64 %39
+  %41 = mul i64 %23, 400000
+  %42 = add i64 %41, 2400000
+  %43 = getelementptr i64, i64* %3, i64 %42
+  %44 = mul i64 %23, 400000
+  %45 = add i64 %44, 2800000
+  %46 = getelementptr i64, i64* %3, i64 %45
+  %47 = mul i64 %23, 400000
+  %48 = add i64 %47, 3200000
+  %49 = getelementptr i64, i64* %3, i64 %48
+  %50 = mul i64 %23, 400000
+  %51 = add i64 %50, 3600000
+  %52 = getelementptr i64, i64* %3, i64 %51
+  %53 = mul i64 %23, 400000
+  %54 = add i64 %53, 4000000
+  %55 = getelementptr i64, i64* %3, i64 %54
+  %56 = mul i64 %23, 400000
+  %57 = add i64 %56, 4400000
+  %58 = getelementptr i64, i64* %3, i64 %57
+  %59 = getelementptr inbounds i64, i64* %25, i64 400000
+  %60 = getelementptr inbounds i64, i64* %28, i64 400000
+  %61 = getelementptr inbounds i64, i64* %31, i64 400000
+  %62 = getelementptr inbounds i64, i64* %34, i64 400000
+  %63 = getelementptr inbounds i64, i64* %37, i64 400000
+  %64 = getelementptr inbounds i64, i64* %40, i64 400000
+  %65 = getelementptr inbounds i64, i64* %43, i64 400000
+  %66 = getelementptr inbounds i64, i64* %46, i64 400000
+  %67 = getelementptr inbounds i64, i64* %49, i64 400000
+  %68 = getelementptr inbounds i64, i64* %52, i64 400000
+  %69 = getelementptr inbounds i64, i64* %55, i64 400000
+  %70 = getelementptr inbounds i64, i64* %58, i64 400000
+  store i64 %0, i64* %59, align 8
+  store i64 %0, i64* %60, align 8
+  store i64 %0, i64* %61, align 8
+  store i64 %0, i64* %62, align 8
+  store i64 %0, i64* %63, align 8
+  store i64 %0, i64* %64, align 8
+  store i64 %0, i64* %65, align 8
+  store i64 %0, i64* %66, align 8
+  store i64 %0, i64* %67, align 8
+  store i64 %0, i64* %68, align 8
+  store i64 %0, i64* %69, align 8
+  store i64 %0, i64* %70, align 8
+  %71 = getelementptr inbounds i64, i64* %25, i64 700000
+  %72 = getelementptr inbounds i64, i64* %28, i64 700000
+  %73 = getelementptr inbounds i64, i64* %31, i64 700000
+  %74 = getelementptr inbounds i64, i64* %34, i64 700000
+  %75 = getelementptr inbounds i64, i64* %37, i64 700000
+  %76 = getelementptr inbounds i64, i64* %40, i64 700000
+  %77 = getelementptr inbounds i64, i64* %43, i64 700000
+  %78 = getelementptr inbounds i64, i64* %46, i64 700000
+  %79 = getelementptr inbounds i64, i64* %49, i64 700000
+  %80 = getelementptr inbounds i64, i64* %52, i64 700000
+  %81 = getelementptr inbounds i64, i64* %55, i64 700000
+  %82 = getelementptr inbounds i64, i64* %58, i64 700000
+  store i64 %0, i64* %71, align 8
+  store i64 %0, i64* %72, align 8
+  store i64 %0, i64* %73, align 8
+  store i64 %0, i64* %74, align 8
+  store i64 %0, i64* %75, align 8
+  store i64 %0, i64* %76, align 8
+  store i64 %0, i64* %77, align 8
+  store i64 %0, i64* %78, align 8
+  store i64 %0, i64* %79, align 8
+  store i64 %0, i64* %80, align 8
+  store i64 %0, i64* %81, align 8
+  store i64 %0, i64* %82, align 8
+  %83 = getelementptr inbounds i64, i64* %25, i64 600000
+  %84 = getelementptr inbounds i64, i64* %28, i64 600000
+  %85 = getelementptr inbounds i64, i64* %31, i64 600000
+  %86 = getelementptr inbounds i64, i64* %34, i64 600000
+  %87 = getelementptr inbounds i64, i64* %37, i64 600000
+  %88 = getelementptr inbounds i64, i64* %40, i64 600000
+  %89 = getelementptr inbounds i64, i64* %43, i64 600000
+  %90 = getelementptr inbounds i64, i64* %46, i64 600000
+  %91 = getelementptr inbounds i64, i64* %49, i64 600000
+  %92 = getelementptr inbounds i64, i64* %52, i64 600000
+  %93 = getelementptr inbounds i64, i64* %55, i64 600000
+  %94 = getelementptr inbounds i64, i64* %58, i64 600000
+  store i64 %0, i64* %83, align 8
+  store i64 %0, i64* %84, align 8
+  store i64 %0, i64* %85, align 8
+  store i64 %0, i64* %86, align 8
+  store i64 %0, i64* %87, align 8
+  store i64 %0, i64* %88, align 8
+  store i64 %0, i64* %89, align 8
+  store i64 %0, i64* %90, align 8
+  store i64 %0, i64* %91, align 8
+  store i64 %0, i64* %92, align 8
+  store i64 %0, i64* %93, align 8
+  store i64 %0, i64* %94, align 8
+  %95 = getelementptr inbounds i64, i64* %25, i64 500000
+  %96 = getelementptr inbounds i64, i64* %28, i64 500000
+  %97 = getelementptr inbounds i64, i64* %31, i64 500000
+  %98 = getelementptr inbounds i64, i64* %34, i64 500000
+  %99 = getelementptr inbounds i64, i64* %37, i64 500000
+  %100 = getelementptr inbounds i64, i64* %40, i64 500000
+  %101 = getelementptr inbounds i64, i64* %43, i64 500000
+  %102 = getelementptr inbounds i64, i64* %46, i64 500000
+  %103 = getelementptr inbounds i64, i64* %49, i64 500000
+  %104 = getelementptr inbounds i64, i64* %52, i64 500000
+  %105 = getelementptr inbounds i64, i64* %55, i64 500000
+  %106 = getelementptr inbounds i64, i64* %58, i64 500000
+  store i64 %0, i64* %95, align 8
+  store i64 %0, i64* %96, align 8
+  store i64 %0, i64* %97, align 8
+  store i64 %0, i64* %98, align 8
+  store i64 %0, i64* %99, align 8
+  store i64 %0, i64* %100, align 8
+  store i64 %0, i64* %101, align 8
+  store i64 %0, i64* %102, align 8
+  store i64 %0, i64* %103, align 8
+  store i64 %0, i64* %104, align 8
+  store i64 %0, i64* %105, align 8
+  store i64 %0, i64* %106, align 8
+  %107 = add i64 %23, 12
+  %108 = icmp eq i64 %107, 996
+  br i1 %108, label %109, label %22
+
+109:                                              ; preds = %22
+  store i64 %0, i64* %4, align 8
+  store i64 %0, i64* %5, align 8
+  store i64 %0, i64* %6, align 8
+  store i64 %0, i64* %7, align 8
+  store i64 %0, i64* %8, align 8
+  store i64 %0, i64* %9, align 8
+  store i64 %0, i64* %10, align 8
+  store i64 %0, i64* %11, align 8
+  store i64 %0, i64* %12, align 8
+  store i64 %0, i64* %13, align 8
+  store i64 %0, i64* %14, align 8
+  store i64 %0, i64* %15, align 8
+  store i64 %0, i64* %16, align 8
+  store i64 %0, i64* %17, align 8
+  store i64 %0, i64* %18, align 8
+  store i64 %0, i64* %19, align 8
+  %110 = add nuw nsw i32 %21, 1
+  %111 = icmp eq i32 %110, 400000
+  br i1 %111, label %112, label %20
+
+112:                                              ; preds = %109
+  ret i32 0
+}
diff --git a/llvm/test/CodeGen/PowerPC/sms-remark.ll b/llvm/test/CodeGen/PowerPC/sms-remark.ll
index 647b56fa7fcd3..e68eb5631df4f 100644
--- a/llvm/test/CodeGen/PowerPC/sms-remark.ll
+++ b/llvm/test/CodeGen/PowerPC/sms-remark.ll
@@ -1,14 +1,19 @@
 ; RUN: llc < %s -ppc-vsr-nums-as-vr -mtriple=powerpc64-unknown-linux-gnu \
 ; RUN:       -verify-machineinstrs -ppc-asm-full-reg-names -mcpu=pwr9 --ppc-enable-pipeliner \
 ; RUN:       -pass-remarks-analysis=pipeliner -pass-remarks=pipeliner -o /dev/null 2>&1 \
-; RUN:       | FileCheck %s
+; RUN:       | FileCheck %s --check-prefix=ENABLED
+; RUN: llc < %s -ppc-vsr-nums-as-vr -mtriple=powerpc64-unknown-linux-gnu \
+; RUN:       -verify-machineinstrs -ppc-asm-full-reg-names -mcpu=pwr8 --ppc-enable-pipeliner \
+; RUN:       -pass-remarks-analysis=pipeliner -pass-remarks=pipeliner -o /dev/null 2>&1 \
+; RUN:       | FileCheck %s --allow-empty --check-prefix=DISABLED
 
 @x = dso_local local_unnamed_addr global <{ i32, i32, i32, i32, [1020 x i32] }> <{ i32 1, i32 2, i32 3, i32 4, [1020 x i32] zeroinitializer }>, align 4
 @y = dso_local global [1024 x i32] zeroinitializer, align 4
 
 define dso_local i32* @foo() local_unnamed_addr {
-;CHECK: Schedule found with Initiation Interval
-;CHECK: Pipelined succesfully!
+;ENABLED: Schedule found with Initiation Interval
+;ENABLED: Pipelined succesfully!
+;DISABLED-NOT: remark
 entry:
   %.pre = load i32, i32* getelementptr inbounds ([1024 x i32], [1024 x i32]* @y, i64 0, i64 0), align 4
   br label %for.body
diff --git a/llvm/test/CodeGen/PowerPC/stack-clash-dynamic-alloca.ll b/llvm/test/CodeGen/PowerPC/stack-clash-dynamic-alloca.ll
new file mode 100644
index 0000000000000..6c136e9a541c4
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/stack-clash-dynamic-alloca.ll
@@ -0,0 +1,437 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs \
+; RUN:   -mtriple=powerpc64le-linux-gnu < %s | FileCheck \
+; RUN:   -check-prefix=CHECK-LE %s
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs \
+; RUN:   -mtriple=powerpc64le-linux-gnu -mcpu=pwr9 < %s | FileCheck \
+; RUN:   -check-prefix=CHECK-P9-LE %s
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs \
+; RUN:   -mtriple=powerpc64-linux-gnu < %s | FileCheck \
+; RUN:   -check-prefix=CHECK-BE %s
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs \
+; RUN:   -mtriple=powerpc-linux-gnu < %s | FileCheck \
+; RUN:   -check-prefix=CHECK-32 %s
+
+define i32 @foo(i32 %n) local_unnamed_addr #0 "stack-probe-size"="32768" nounwind {
+; CHECK-LE-LABEL: foo:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    std r31, -8(r1)
+; CHECK-LE-NEXT:    stdu r1, -48(r1)
+; CHECK-LE-NEXT:    rldic r3, r3, 2, 30
+; CHECK-LE-NEXT:    li r6, -32768
+; CHECK-LE-NEXT:    mr r31, r1
+; CHECK-LE-NEXT:    addi r3, r3, 15
+; CHECK-LE-NEXT:    addi r4, r31, 48
+; CHECK-LE-NEXT:    rldicl r3, r3, 60, 4
+; CHECK-LE-NEXT:    rldicl r3, r3, 4, 29
+; CHECK-LE-NEXT:    neg r5, r3
+; CHECK-LE-NEXT:    divd r7, r5, r6
+; CHECK-LE-NEXT:    add r3, r1, r5
+; CHECK-LE-NEXT:    mulld r6, r7, r6
+; CHECK-LE-NEXT:    sub r5, r5, r6
+; CHECK-LE-NEXT:    stdux r4, r1, r5
+; CHECK-LE-NEXT:    cmpd r1, r3
+; CHECK-LE-NEXT:    beq cr0, .LBB0_2
+; CHECK-LE-NEXT:  .LBB0_1:
+; CHECK-LE-NEXT:    stdu r4, -32768(r1)
+; CHECK-LE-NEXT:    cmpd r1, r3
+; CHECK-LE-NEXT:    bne cr0, .LBB0_1
+; CHECK-LE-NEXT:  .LBB0_2:
+; CHECK-LE-NEXT:    li r4, 1
+; CHECK-LE-NEXT:    addi r3, r1, 32
+; CHECK-LE-NEXT:    stw r4, 4792(r3)
+; CHECK-LE-NEXT:    lwz r3, 0(r3)
+; CHECK-LE-NEXT:    ld r1, 0(r1)
+; CHECK-LE-NEXT:    ld r31, -8(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-P9-LE-LABEL: foo:
+; CHECK-P9-LE:       # %bb.0:
+; CHECK-P9-LE-NEXT:    std r31, -8(r1)
+; CHECK-P9-LE-NEXT:    stdu r1, -48(r1)
+; CHECK-P9-LE-NEXT:    rldic r3, r3, 2, 30
+; CHECK-P9-LE-NEXT:    addi r3, r3, 15
+; CHECK-P9-LE-NEXT:    rldicl r3, r3, 60, 4
+; CHECK-P9-LE-NEXT:    rldicl r3, r3, 4, 29
+; CHECK-P9-LE-NEXT:    neg r5, r3
+; CHECK-P9-LE-NEXT:    li r6, -32768
+; CHECK-P9-LE-NEXT:    divd r7, r5, r6
+; CHECK-P9-LE-NEXT:    mulld r6, r7, r6
+; CHECK-P9-LE-NEXT:    mr r31, r1
+; CHECK-P9-LE-NEXT:    addi r4, r31, 48
+; CHECK-P9-LE-NEXT:    add r3, r1, r5
+; CHECK-P9-LE-NEXT:    sub r5, r5, r6
+; CHECK-P9-LE-NEXT:    stdux r4, r1, r5
+; CHECK-P9-LE-NEXT:    cmpd r1, r3
+; CHECK-P9-LE-NEXT:    beq cr0, .LBB0_2
+; CHECK-P9-LE-NEXT:  .LBB0_1:
+; CHECK-P9-LE-NEXT:    stdu r4, -32768(r1)
+; CHECK-P9-LE-NEXT:    cmpd r1, r3
+; CHECK-P9-LE-NEXT:    bne cr0, .LBB0_1
+; CHECK-P9-LE-NEXT:  .LBB0_2:
+; CHECK-P9-LE-NEXT:    addi r3, r1, 32
+; CHECK-P9-LE-NEXT:    li r4, 1
+; CHECK-P9-LE-NEXT:    stw r4, 4792(r3)
+; CHECK-P9-LE-NEXT:    lwz r3, 0(r3)
+; CHECK-P9-LE-NEXT:    ld r1, 0(r1)
+; CHECK-P9-LE-NEXT:    ld r31, -8(r1)
+; CHECK-P9-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: foo:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    std r31, -8(r1)
+; CHECK-BE-NEXT:    stdu r1, -64(r1)
+; CHECK-BE-NEXT:    rldic r3, r3, 2, 30
+; CHECK-BE-NEXT:    li r6, -32768
+; CHECK-BE-NEXT:    addi r3, r3, 15
+; CHECK-BE-NEXT:    rldicl r3, r3, 60, 4
+; CHECK-BE-NEXT:    mr r31, r1
+; CHECK-BE-NEXT:    rldicl r3, r3, 4, 29
+; CHECK-BE-NEXT:    addi r4, r31, 64
+; CHECK-BE-NEXT:    neg r5, r3
+; CHECK-BE-NEXT:    divd r7, r5, r6
+; CHECK-BE-NEXT:    add r3, r1, r5
+; CHECK-BE-NEXT:    mulld r6, r7, r6
+; CHECK-BE-NEXT:    sub r5, r5, r6
+; CHECK-BE-NEXT:    stdux r4, r1, r5
+; CHECK-BE-NEXT:    cmpd r1, r3
+; CHECK-BE-NEXT:    beq cr0, .LBB0_2
+; CHECK-BE-NEXT:  .LBB0_1:
+; CHECK-BE-NEXT:    stdu r4, -32768(r1)
+; CHECK-BE-NEXT:    cmpd r1, r3
+; CHECK-BE-NEXT:    bne cr0, .LBB0_1
+; CHECK-BE-NEXT:  .LBB0_2:
+; CHECK-BE-NEXT:    li r4, 1
+; CHECK-BE-NEXT:    addi r3, r1, 48
+; CHECK-BE-NEXT:    stw r4, 4792(r3)
+; CHECK-BE-NEXT:    lwz r3, 0(r3)
+; CHECK-BE-NEXT:    ld r1, 0(r1)
+; CHECK-BE-NEXT:    ld r31, -8(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: foo:
+; CHECK-32:       # %bb.0:
+; CHECK-32-NEXT:    stwu r1, -32(r1)
+; CHECK-32-NEXT:    slwi r3, r3, 2
+; CHECK-32-NEXT:    addi r3, r3, 15
+; CHECK-32-NEXT:    rlwinm r3, r3, 0, 0, 27
+; CHECK-32-NEXT:    neg r5, r3
+; CHECK-32-NEXT:    li r6, -32768
+; CHECK-32-NEXT:    divw r7, r5, r6
+; CHECK-32-NEXT:    stw r31, 28(r1)
+; CHECK-32-NEXT:    mr r31, r1
+; CHECK-32-NEXT:    addi r4, r31, 32
+; CHECK-32-NEXT:    add r3, r1, r5
+; CHECK-32-NEXT:    mullw r6, r7, r6
+; CHECK-32-NEXT:    sub r5, r5, r6
+; CHECK-32-NEXT:    stwux r4, r1, r5
+; CHECK-32-NEXT:    cmpw r1, r3
+; CHECK-32-NEXT:    beq cr0, .LBB0_2
+; CHECK-32-NEXT:  .LBB0_1:
+; CHECK-32-NEXT:    stwu r4, -32768(r1)
+; CHECK-32-NEXT:    cmpw r1, r3
+; CHECK-32-NEXT:    bne cr0, .LBB0_1
+; CHECK-32-NEXT:  .LBB0_2:
+; CHECK-32-NEXT:    li r4, 1
+; CHECK-32-NEXT:    addi r3, r1, 16
+; CHECK-32-NEXT:    stw r4, 4792(r3)
+; CHECK-32-NEXT:    lwz r3, 0(r3)
+; CHECK-32-NEXT:    lwz r31, 0(r1)
+; CHECK-32-NEXT:    lwz r0, -4(r31)
+; CHECK-32-NEXT:    mr r1, r31
+; CHECK-32-NEXT:    mr r31, r0
+; CHECK-32-NEXT:    blr
+  %a = alloca i32, i32 %n, align 16
+  %b = getelementptr inbounds i32, i32* %a, i64 1198
+  store volatile i32 1, i32* %b
+  %c = load volatile i32, i32* %a
+  ret i32 %c
+}
+
+define i32 @bar(i32 %n) local_unnamed_addr #0 nounwind {
+; CHECK-LE-LABEL: bar:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    std r31, -8(r1)
+; CHECK-LE-NEXT:    stdu r1, -48(r1)
+; CHECK-LE-NEXT:    rldic r4, r3, 2, 30
+; CHECK-LE-NEXT:    li r7, -4096
+; CHECK-LE-NEXT:    mr r31, r1
+; CHECK-LE-NEXT:    addi r4, r4, 15
+; CHECK-LE-NEXT:    addi r5, r31, 48
+; CHECK-LE-NEXT:    rldicl r4, r4, 60, 4
+; CHECK-LE-NEXT:    rldicl r4, r4, 4, 29
+; CHECK-LE-NEXT:    neg r6, r4
+; CHECK-LE-NEXT:    divd r8, r6, r7
+; CHECK-LE-NEXT:    add r4, r1, r6
+; CHECK-LE-NEXT:    mulld r7, r8, r7
+; CHECK-LE-NEXT:    sub r6, r6, r7
+; CHECK-LE-NEXT:    stdux r5, r1, r6
+; CHECK-LE-NEXT:    cmpd r1, r4
+; CHECK-LE-NEXT:    beq cr0, .LBB1_2
+; CHECK-LE-NEXT:  .LBB1_1:
+; CHECK-LE-NEXT:    stdu r5, -4096(r1)
+; CHECK-LE-NEXT:    cmpd r1, r4
+; CHECK-LE-NEXT:    bne cr0, .LBB1_1
+; CHECK-LE-NEXT:  .LBB1_2:
+; CHECK-LE-NEXT:    extsw r3, r3
+; CHECK-LE-NEXT:    li r5, 1
+; CHECK-LE-NEXT:    addi r4, r1, 32
+; CHECK-LE-NEXT:    sldi r3, r3, 2
+; CHECK-LE-NEXT:    add r3, r4, r3
+; CHECK-LE-NEXT:    stw r5, 4096(r3)
+; CHECK-LE-NEXT:    lwz r3, 0(r4)
+; CHECK-LE-NEXT:    ld r1, 0(r1)
+; CHECK-LE-NEXT:    ld r31, -8(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-P9-LE-LABEL: bar:
+; CHECK-P9-LE:       # %bb.0:
+; CHECK-P9-LE-NEXT:    std r31, -8(r1)
+; CHECK-P9-LE-NEXT:    stdu r1, -48(r1)
+; CHECK-P9-LE-NEXT:    rldic r4, r3, 2, 30
+; CHECK-P9-LE-NEXT:    addi r4, r4, 15
+; CHECK-P9-LE-NEXT:    rldicl r4, r4, 60, 4
+; CHECK-P9-LE-NEXT:    rldicl r4, r4, 4, 29
+; CHECK-P9-LE-NEXT:    neg r6, r4
+; CHECK-P9-LE-NEXT:    li r7, -4096
+; CHECK-P9-LE-NEXT:    divd r8, r6, r7
+; CHECK-P9-LE-NEXT:    mulld r7, r8, r7
+; CHECK-P9-LE-NEXT:    mr r31, r1
+; CHECK-P9-LE-NEXT:    addi r5, r31, 48
+; CHECK-P9-LE-NEXT:    add r4, r1, r6
+; CHECK-P9-LE-NEXT:    sub r6, r6, r7
+; CHECK-P9-LE-NEXT:    stdux r5, r1, r6
+; CHECK-P9-LE-NEXT:    cmpd r1, r4
+; CHECK-P9-LE-NEXT:    beq cr0, .LBB1_2
+; CHECK-P9-LE-NEXT:  .LBB1_1:
+; CHECK-P9-LE-NEXT:    stdu r5, -4096(r1)
+; CHECK-P9-LE-NEXT:    cmpd r1, r4
+; CHECK-P9-LE-NEXT:    bne cr0, .LBB1_1
+; CHECK-P9-LE-NEXT:  .LBB1_2:
+; CHECK-P9-LE-NEXT:    addi r4, r1, 32
+; CHECK-P9-LE-NEXT:    extswsli r3, r3, 2
+; CHECK-P9-LE-NEXT:    add r3, r4, r3
+; CHECK-P9-LE-NEXT:    li r5, 1
+; CHECK-P9-LE-NEXT:    stw r5, 4096(r3)
+; CHECK-P9-LE-NEXT:    lwz r3, 0(r4)
+; CHECK-P9-LE-NEXT:    ld r1, 0(r1)
+; CHECK-P9-LE-NEXT:    ld r31, -8(r1)
+; CHECK-P9-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: bar:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    std r31, -8(r1)
+; CHECK-BE-NEXT:    stdu r1, -64(r1)
+; CHECK-BE-NEXT:    rldic r4, r3, 2, 30
+; CHECK-BE-NEXT:    li r7, -4096
+; CHECK-BE-NEXT:    addi r4, r4, 15
+; CHECK-BE-NEXT:    rldicl r4, r4, 60, 4
+; CHECK-BE-NEXT:    mr r31, r1
+; CHECK-BE-NEXT:    rldicl r4, r4, 4, 29
+; CHECK-BE-NEXT:    addi r5, r31, 64
+; CHECK-BE-NEXT:    neg r6, r4
+; CHECK-BE-NEXT:    divd r8, r6, r7
+; CHECK-BE-NEXT:    add r4, r1, r6
+; CHECK-BE-NEXT:    mulld r7, r8, r7
+; CHECK-BE-NEXT:    sub r6, r6, r7
+; CHECK-BE-NEXT:    stdux r5, r1, r6
+; CHECK-BE-NEXT:    cmpd r1, r4
+; CHECK-BE-NEXT:    beq cr0, .LBB1_2
+; CHECK-BE-NEXT:  .LBB1_1:
+; CHECK-BE-NEXT:    stdu r5, -4096(r1)
+; CHECK-BE-NEXT:    cmpd r1, r4
+; CHECK-BE-NEXT:    bne cr0, .LBB1_1
+; CHECK-BE-NEXT:  .LBB1_2:
+; CHECK-BE-NEXT:    extsw r3, r3
+; CHECK-BE-NEXT:    addi r4, r1, 48
+; CHECK-BE-NEXT:    sldi r3, r3, 2
+; CHECK-BE-NEXT:    li r5, 1
+; CHECK-BE-NEXT:    add r3, r4, r3
+; CHECK-BE-NEXT:    stw r5, 4096(r3)
+; CHECK-BE-NEXT:    lwz r3, 0(r4)
+; CHECK-BE-NEXT:    ld r1, 0(r1)
+; CHECK-BE-NEXT:    ld r31, -8(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: bar:
+; CHECK-32:       # %bb.0:
+; CHECK-32-NEXT:    stwu r1, -32(r1)
+; CHECK-32-NEXT:    slwi r3, r3, 2
+; CHECK-32-NEXT:    addi r4, r3, 15
+; CHECK-32-NEXT:    rlwinm r4, r4, 0, 0, 27
+; CHECK-32-NEXT:    neg r6, r4
+; CHECK-32-NEXT:    li r7, -4096
+; CHECK-32-NEXT:    divw r8, r6, r7
+; CHECK-32-NEXT:    stw r31, 28(r1)
+; CHECK-32-NEXT:    mr r31, r1
+; CHECK-32-NEXT:    addi r5, r31, 32
+; CHECK-32-NEXT:    add r4, r1, r6
+; CHECK-32-NEXT:    mullw r7, r8, r7
+; CHECK-32-NEXT:    sub r6, r6, r7
+; CHECK-32-NEXT:    stwux r5, r1, r6
+; CHECK-32-NEXT:    cmpw r1, r4
+; CHECK-32-NEXT:    beq cr0, .LBB1_2
+; CHECK-32-NEXT:  .LBB1_1:
+; CHECK-32-NEXT:    stwu r5, -4096(r1)
+; CHECK-32-NEXT:    cmpw r1, r4
+; CHECK-32-NEXT:    bne cr0, .LBB1_1
+; CHECK-32-NEXT:  .LBB1_2:
+; CHECK-32-NEXT:    addi r4, r1, 16
+; CHECK-32-NEXT:    li r5, 1
+; CHECK-32-NEXT:    add r3, r4, r3
+; CHECK-32-NEXT:    stw r5, 4096(r3)
+; CHECK-32-NEXT:    lwz r3, 0(r4)
+; CHECK-32-NEXT:    lwz r31, 0(r1)
+; CHECK-32-NEXT:    lwz r0, -4(r31)
+; CHECK-32-NEXT:    mr r1, r31
+; CHECK-32-NEXT:    mr r31, r0
+; CHECK-32-NEXT:    blr
+  %a = alloca i32, i32 %n, align 16
+  %i = add i32 %n, 1024
+  %b = getelementptr inbounds i32, i32* %a, i32 %i
+  store volatile i32 1, i32* %b
+  %c = load volatile i32, i32* %a
+  ret i32 %c
+}
+
+define i32 @f(i32 %n) local_unnamed_addr #0 "stack-probe-size"="65536" nounwind {
+; CHECK-LE-LABEL: f:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    std r31, -8(r1)
+; CHECK-LE-NEXT:    stdu r1, -48(r1)
+; CHECK-LE-NEXT:    rldic r3, r3, 2, 30
+; CHECK-LE-NEXT:    lis r5, -1
+; CHECK-LE-NEXT:    mr r31, r1
+; CHECK-LE-NEXT:    addi r3, r3, 15
+; CHECK-LE-NEXT:    ori r5, r5, 0
+; CHECK-LE-NEXT:    addi r4, r31, 48
+; CHECK-LE-NEXT:    rldicl r3, r3, 60, 4
+; CHECK-LE-NEXT:    rldicl r3, r3, 4, 29
+; CHECK-LE-NEXT:    neg r6, r3
+; CHECK-LE-NEXT:    divd r7, r6, r5
+; CHECK-LE-NEXT:    add r3, r1, r6
+; CHECK-LE-NEXT:    mulld r7, r7, r5
+; CHECK-LE-NEXT:    sub r6, r6, r7
+; CHECK-LE-NEXT:    stdux r4, r1, r6
+; CHECK-LE-NEXT:    cmpd r1, r3
+; CHECK-LE-NEXT:    beq cr0, .LBB2_2
+; CHECK-LE-NEXT:  .LBB2_1:
+; CHECK-LE-NEXT:    stdux r4, r1, r5
+; CHECK-LE-NEXT:    cmpd r1, r3
+; CHECK-LE-NEXT:    bne cr0, .LBB2_1
+; CHECK-LE-NEXT:  .LBB2_2:
+; CHECK-LE-NEXT:    li r4, 1
+; CHECK-LE-NEXT:    addi r3, r1, 32
+; CHECK-LE-NEXT:    stw r4, 4792(r3)
+; CHECK-LE-NEXT:    lwz r3, 0(r3)
+; CHECK-LE-NEXT:    ld r1, 0(r1)
+; CHECK-LE-NEXT:    ld r31, -8(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-P9-LE-LABEL: f:
+; CHECK-P9-LE:       # %bb.0:
+; CHECK-P9-LE-NEXT:    std r31, -8(r1)
+; CHECK-P9-LE-NEXT:    stdu r1, -48(r1)
+; CHECK-P9-LE-NEXT:    rldic r3, r3, 2, 30
+; CHECK-P9-LE-NEXT:    addi r3, r3, 15
+; CHECK-P9-LE-NEXT:    rldicl r3, r3, 60, 4
+; CHECK-P9-LE-NEXT:    rldicl r3, r3, 4, 29
+; CHECK-P9-LE-NEXT:    lis r5, -1
+; CHECK-P9-LE-NEXT:    ori r5, r5, 0
+; CHECK-P9-LE-NEXT:    neg r6, r3
+; CHECK-P9-LE-NEXT:    divd r7, r6, r5
+; CHECK-P9-LE-NEXT:    mulld r7, r7, r5
+; CHECK-P9-LE-NEXT:    mr r31, r1
+; CHECK-P9-LE-NEXT:    addi r4, r31, 48
+; CHECK-P9-LE-NEXT:    add r3, r1, r6
+; CHECK-P9-LE-NEXT:    sub r6, r6, r7
+; CHECK-P9-LE-NEXT:    stdux r4, r1, r6
+; CHECK-P9-LE-NEXT:    cmpd r1, r3
+; CHECK-P9-LE-NEXT:    beq cr0, .LBB2_2
+; CHECK-P9-LE-NEXT:  .LBB2_1:
+; CHECK-P9-LE-NEXT:    stdux r4, r1, r5
+; CHECK-P9-LE-NEXT:    cmpd r1, r3
+; CHECK-P9-LE-NEXT:    bne cr0, .LBB2_1
+; CHECK-P9-LE-NEXT:  .LBB2_2:
+; CHECK-P9-LE-NEXT:    addi r3, r1, 32
+; CHECK-P9-LE-NEXT:    li r4, 1
+; CHECK-P9-LE-NEXT:    stw r4, 4792(r3)
+; CHECK-P9-LE-NEXT:    lwz r3, 0(r3)
+; CHECK-P9-LE-NEXT:    ld r1, 0(r1)
+; CHECK-P9-LE-NEXT:    ld r31, -8(r1)
+; CHECK-P9-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    std r31, -8(r1)
+; CHECK-BE-NEXT:    stdu r1, -64(r1)
+; CHECK-BE-NEXT:    rldic r3, r3, 2, 30
+; CHECK-BE-NEXT:    lis r5, -1
+; CHECK-BE-NEXT:    addi r3, r3, 15
+; CHECK-BE-NEXT:    rldicl r3, r3, 60, 4
+; CHECK-BE-NEXT:    ori r5, r5, 0
+; CHECK-BE-NEXT:    rldicl r3, r3, 4, 29
+; CHECK-BE-NEXT:    mr r31, r1
+; CHECK-BE-NEXT:    neg r6, r3
+; CHECK-BE-NEXT:    divd r7, r6, r5
+; CHECK-BE-NEXT:    addi r4, r31, 64
+; CHECK-BE-NEXT:    mulld r7, r7, r5
+; CHECK-BE-NEXT:    add r3, r1, r6
+; CHECK-BE-NEXT:    sub r6, r6, r7
+; CHECK-BE-NEXT:    stdux r4, r1, r6
+; CHECK-BE-NEXT:    cmpd r1, r3
+; CHECK-BE-NEXT:    beq cr0, .LBB2_2
+; CHECK-BE-NEXT:  .LBB2_1:
+; CHECK-BE-NEXT:    stdux r4, r1, r5
+; CHECK-BE-NEXT:    cmpd r1, r3
+; CHECK-BE-NEXT:    bne cr0, .LBB2_1
+; CHECK-BE-NEXT:  .LBB2_2:
+; CHECK-BE-NEXT:    li r4, 1
+; CHECK-BE-NEXT:    addi r3, r1, 48
+; CHECK-BE-NEXT:    stw r4, 4792(r3)
+; CHECK-BE-NEXT:    lwz r3, 0(r3)
+; CHECK-BE-NEXT:    ld r1, 0(r1)
+; CHECK-BE-NEXT:    ld r31, -8(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f:
+; CHECK-32:       # %bb.0:
+; CHECK-32-NEXT:    stwu r1, -32(r1)
+; CHECK-32-NEXT:    slwi r3, r3, 2
+; CHECK-32-NEXT:    addi r3, r3, 15
+; CHECK-32-NEXT:    rlwinm r3, r3, 0, 0, 27
+; CHECK-32-NEXT:    lis r5, -1
+; CHECK-32-NEXT:    neg r6, r3
+; CHECK-32-NEXT:    ori r5, r5, 0
+; CHECK-32-NEXT:    divw r7, r6, r5
+; CHECK-32-NEXT:    stw r31, 28(r1)
+; CHECK-32-NEXT:    mr r31, r1
+; CHECK-32-NEXT:    addi r4, r31, 32
+; CHECK-32-NEXT:    add r3, r1, r6
+; CHECK-32-NEXT:    mullw r7, r7, r5
+; CHECK-32-NEXT:    sub r6, r6, r7
+; CHECK-32-NEXT:    stwux r4, r1, r6
+; CHECK-32-NEXT:    cmpw r1, r3
+; CHECK-32-NEXT:    beq cr0, .LBB2_2
+; CHECK-32-NEXT:  .LBB2_1:
+; CHECK-32-NEXT:    stwux r4, r1, r5
+; CHECK-32-NEXT:    cmpw r1, r3
+; CHECK-32-NEXT:    bne cr0, .LBB2_1
+; CHECK-32-NEXT:  .LBB2_2:
+; CHECK-32-NEXT:    li r4, 1
+; CHECK-32-NEXT:    addi r3, r1, 16
+; CHECK-32-NEXT:    stw r4, 4792(r3)
+; CHECK-32-NEXT:    lwz r3, 0(r3)
+; CHECK-32-NEXT:    lwz r31, 0(r1)
+; CHECK-32-NEXT:    lwz r0, -4(r31)
+; CHECK-32-NEXT:    mr r1, r31
+; CHECK-32-NEXT:    mr r31, r0
+; CHECK-32-NEXT:    blr
+  %a = alloca i32, i32 %n, align 16
+  %b = getelementptr inbounds i32, i32* %a, i64 1198
+  store volatile i32 1, i32* %b
+  %c = load volatile i32, i32* %a
+  ret i32 %c
+}
+
+attributes #0 =  {"probe-stack"="inline-asm"}
diff --git a/llvm/test/CodeGen/PowerPC/stack-clash-prologue-nounwind.ll b/llvm/test/CodeGen/PowerPC/stack-clash-prologue-nounwind.ll
new file mode 100644
index 0000000000000..e595d8a732a5c
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/stack-clash-prologue-nounwind.ll
@@ -0,0 +1,474 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs \
+; RUN:   -mtriple=powerpc64le-linux-gnu < %s | FileCheck \
+; RUN:   -check-prefix=CHECK-LE %s
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs \
+; RUN:   -mtriple=powerpc64-linux-gnu < %s | FileCheck \
+; RUN:   -check-prefix=CHECK-BE %s
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs \
+; RUN:   -mtriple=powerpc-linux-gnu < %s | FileCheck \
+; RUN:   -check-prefix=CHECK-32 %s
+
+; Free probe
+define i8 @f0() #0 nounwind {
+; CHECK-LE-LABEL: f0:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    li r3, 3
+; CHECK-LE-NEXT:    stb r3, -64(r1)
+; CHECK-LE-NEXT:    lbz r3, -64(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f0:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    li r3, 3
+; CHECK-BE-NEXT:    stb r3, -64(r1)
+; CHECK-BE-NEXT:    lbz r3, -64(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f0:
+; CHECK-32:       # %bb.0: # %entry
+; CHECK-32-NEXT:    stwu r1, -80(r1)
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    stb r3, 16(r1)
+; CHECK-32-NEXT:    lbz r3, 16(r1)
+; CHECK-32-NEXT:    addi r1, r1, 80
+; CHECK-32-NEXT:    blr
+entry:
+  %a = alloca i8, i64 64
+  %b = getelementptr inbounds i8, i8* %a, i64 63
+  store volatile i8 3, i8* %a
+  %c = load volatile i8, i8* %a
+  ret i8 %c
+}
+
+define i8 @f1() #0 "stack-probe-size"="0" nounwind {
+; CHECK-LE-LABEL: f1:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    li r0, 259
+; CHECK-LE-NEXT:    mtctr r0
+; CHECK-LE-NEXT:  .LBB1_1: # %entry
+; CHECK-LE-NEXT:    #
+; CHECK-LE-NEXT:    stdu r12, -16(r1)
+; CHECK-LE-NEXT:    bdnz .LBB1_1
+; CHECK-LE-NEXT:  # %bb.2: # %entry
+; CHECK-LE-NEXT:    li r3, 3
+; CHECK-LE-NEXT:    stb r3, 48(r1)
+; CHECK-LE-NEXT:    lbz r3, 48(r1)
+; CHECK-LE-NEXT:    addi r1, r1, 4144
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f1:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    li r0, 260
+; CHECK-BE-NEXT:    mtctr r0
+; CHECK-BE-NEXT:  .LBB1_1: # %entry
+; CHECK-BE-NEXT:    #
+; CHECK-BE-NEXT:    stdu r12, -16(r1)
+; CHECK-BE-NEXT:    bdnz .LBB1_1
+; CHECK-BE-NEXT:  # %bb.2: # %entry
+; CHECK-BE-NEXT:    li r3, 3
+; CHECK-BE-NEXT:    stb r3, 64(r1)
+; CHECK-BE-NEXT:    lbz r3, 64(r1)
+; CHECK-BE-NEXT:    addi r1, r1, 4160
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f1:
+; CHECK-32:       # %bb.0: # %entry
+; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    li r0, 257
+; CHECK-32-NEXT:    mtctr r0
+; CHECK-32-NEXT:  .LBB1_1: # %entry
+; CHECK-32-NEXT:    #
+; CHECK-32-NEXT:    stwu r12, -16(r1)
+; CHECK-32-NEXT:    bdnz .LBB1_1
+; CHECK-32-NEXT:  # %bb.2: # %entry
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    sub r0, r1, r12
+; CHECK-32-NEXT:    stb r3, 16(r1)
+; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    lbz r3, 16(r1)
+; CHECK-32-NEXT:    addi r1, r1, 4112
+; CHECK-32-NEXT:    blr
+entry:
+  %a = alloca i8, i64 4096
+  %b = getelementptr inbounds i8, i8* %a, i64 63
+  store volatile i8 3, i8* %a
+  %c = load volatile i8, i8* %a
+  ret i8 %c
+}
+
+define i8 @f2() #0 nounwind {
+; CHECK-LE-LABEL: f2:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    stdu r12, -48(r1)
+; CHECK-LE-NEXT:    li r0, 16
+; CHECK-LE-NEXT:    mtctr r0
+; CHECK-LE-NEXT:  .LBB2_1: # %entry
+; CHECK-LE-NEXT:    #
+; CHECK-LE-NEXT:    stdu r12, -4096(r1)
+; CHECK-LE-NEXT:    bdnz .LBB2_1
+; CHECK-LE-NEXT:  # %bb.2: # %entry
+; CHECK-LE-NEXT:    li r3, 3
+; CHECK-LE-NEXT:    stb r3, 48(r1)
+; CHECK-LE-NEXT:    lbz r3, 48(r1)
+; CHECK-LE-NEXT:    ld r1, 0(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f2:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    stdu r12, -64(r1)
+; CHECK-BE-NEXT:    li r0, 16
+; CHECK-BE-NEXT:    mtctr r0
+; CHECK-BE-NEXT:  .LBB2_1: # %entry
+; CHECK-BE-NEXT:    #
+; CHECK-BE-NEXT:    stdu r12, -4096(r1)
+; CHECK-BE-NEXT:    bdnz .LBB2_1
+; CHECK-BE-NEXT:  # %bb.2: # %entry
+; CHECK-BE-NEXT:    li r3, 3
+; CHECK-BE-NEXT:    stb r3, 64(r1)
+; CHECK-BE-NEXT:    lbz r3, 64(r1)
+; CHECK-BE-NEXT:    ld r1, 0(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f2:
+; CHECK-32:       # %bb.0: # %entry
+; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    stwu r12, -16(r1)
+; CHECK-32-NEXT:    li r0, 16
+; CHECK-32-NEXT:    mtctr r0
+; CHECK-32-NEXT:  .LBB2_1: # %entry
+; CHECK-32-NEXT:    #
+; CHECK-32-NEXT:    stwu r12, -4096(r1)
+; CHECK-32-NEXT:    bdnz .LBB2_1
+; CHECK-32-NEXT:  # %bb.2: # %entry
+; CHECK-32-NEXT:    sub r0, r1, r12
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    stb r3, 16(r1)
+; CHECK-32-NEXT:    mr r0, r31
+; CHECK-32-NEXT:    lbz r3, 16(r1)
+; CHECK-32-NEXT:    lwz r31, 0(r1)
+; CHECK-32-NEXT:    mr r1, r31
+; CHECK-32-NEXT:    mr r31, r0
+; CHECK-32-NEXT:    blr
+entry:
+  %a = alloca i8, i64 65536
+  %b = getelementptr inbounds i8, i8* %a, i64 63
+  store volatile i8 3, i8* %a
+  %c = load volatile i8, i8* %a
+  ret i8 %c
+}
+
+define i8 @f3() #0 "stack-probe-size"="32768" nounwind {
+; CHECK-LE-LABEL: f3:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    stdu r12, -48(r1)
+; CHECK-LE-NEXT:    stdu r12, -32768(r1)
+; CHECK-LE-NEXT:    stdu r12, -32768(r1)
+; CHECK-LE-NEXT:    li r3, 3
+; CHECK-LE-NEXT:    stb r3, 48(r1)
+; CHECK-LE-NEXT:    lbz r3, 48(r1)
+; CHECK-LE-NEXT:    ld r1, 0(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f3:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    stdu r12, -64(r1)
+; CHECK-BE-NEXT:    stdu r12, -32768(r1)
+; CHECK-BE-NEXT:    stdu r12, -32768(r1)
+; CHECK-BE-NEXT:    li r3, 3
+; CHECK-BE-NEXT:    stb r3, 64(r1)
+; CHECK-BE-NEXT:    lbz r3, 64(r1)
+; CHECK-BE-NEXT:    ld r1, 0(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f3:
+; CHECK-32:       # %bb.0: # %entry
+; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    stwu r12, -16(r1)
+; CHECK-32-NEXT:    stwu r12, -32768(r1)
+; CHECK-32-NEXT:    stwu r12, -32768(r1)
+; CHECK-32-NEXT:    sub r0, r1, r12
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    stb r3, 16(r1)
+; CHECK-32-NEXT:    mr r0, r31
+; CHECK-32-NEXT:    lbz r3, 16(r1)
+; CHECK-32-NEXT:    lwz r31, 0(r1)
+; CHECK-32-NEXT:    mr r1, r31
+; CHECK-32-NEXT:    mr r31, r0
+; CHECK-32-NEXT:    blr
+entry:
+  %a = alloca i8, i64 65536
+  %b = getelementptr inbounds i8, i8* %a, i64 63
+  store volatile i8 3, i8* %a
+  %c = load volatile i8, i8* %a
+  ret i8 %c
+}
+
+; Same as f2, but without protection.
+define i8 @f4() nounwind {
+; CHECK-LE-LABEL: f4:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    lis r0, -2
+; CHECK-LE-NEXT:    ori r0, r0, 65488
+; CHECK-LE-NEXT:    stdux r1, r1, r0
+; CHECK-LE-NEXT:    li r3, 3
+; CHECK-LE-NEXT:    stb r3, 48(r1)
+; CHECK-LE-NEXT:    lbz r3, 48(r1)
+; CHECK-LE-NEXT:    ld r1, 0(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f4:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lis r0, -2
+; CHECK-BE-NEXT:    ori r0, r0, 65472
+; CHECK-BE-NEXT:    stdux r1, r1, r0
+; CHECK-BE-NEXT:    li r3, 3
+; CHECK-BE-NEXT:    stb r3, 64(r1)
+; CHECK-BE-NEXT:    lbz r3, 64(r1)
+; CHECK-BE-NEXT:    ld r1, 0(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f4:
+; CHECK-32:       # %bb.0: # %entry
+; CHECK-32-NEXT:    lis r0, -2
+; CHECK-32-NEXT:    ori r0, r0, 65520
+; CHECK-32-NEXT:    stwux r1, r1, r0
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    stb r3, 16(r1)
+; CHECK-32-NEXT:    mr r0, r31
+; CHECK-32-NEXT:    lbz r3, 16(r1)
+; CHECK-32-NEXT:    lwz r31, 0(r1)
+; CHECK-32-NEXT:    mr r1, r31
+; CHECK-32-NEXT:    mr r31, r0
+; CHECK-32-NEXT:    blr
+entry:
+  %a = alloca i8, i64 65536
+  %b = getelementptr inbounds i8, i8* %a, i64 63
+  store volatile i8 3, i8* %a
+  %c = load volatile i8, i8* %a
+  ret i8 %c
+}
+
+define i8 @f5() #0 "stack-probe-size"="65536" nounwind {
+; CHECK-LE-LABEL: f5:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    stdu r12, -48(r1)
+; CHECK-LE-NEXT:    li r0, 16
+; CHECK-LE-NEXT:    mtctr r0
+; CHECK-LE-NEXT:    lis r0, -1
+; CHECK-LE-NEXT:    nop
+; CHECK-LE-NEXT:  .LBB5_1: # %entry
+; CHECK-LE-NEXT:    #
+; CHECK-LE-NEXT:    stdux r12, r1, r0
+; CHECK-LE-NEXT:    bdnz .LBB5_1
+; CHECK-LE-NEXT:  # %bb.2: # %entry
+; CHECK-LE-NEXT:    li r3, 3
+; CHECK-LE-NEXT:    stb r3, 48(r1)
+; CHECK-LE-NEXT:    lbz r3, 48(r1)
+; CHECK-LE-NEXT:    ld r1, 0(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f5:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    stdu r12, -64(r1)
+; CHECK-BE-NEXT:    li r0, 16
+; CHECK-BE-NEXT:    mtctr r0
+; CHECK-BE-NEXT:    lis r0, -1
+; CHECK-BE-NEXT:    nop
+; CHECK-BE-NEXT:  .LBB5_1: # %entry
+; CHECK-BE-NEXT:    #
+; CHECK-BE-NEXT:    stdux r12, r1, r0
+; CHECK-BE-NEXT:    bdnz .LBB5_1
+; CHECK-BE-NEXT:  # %bb.2: # %entry
+; CHECK-BE-NEXT:    li r3, 3
+; CHECK-BE-NEXT:    stb r3, 64(r1)
+; CHECK-BE-NEXT:    lbz r3, 64(r1)
+; CHECK-BE-NEXT:    ld r1, 0(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f5:
+; CHECK-32:       # %bb.0: # %entry
+; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    stwu r12, -16(r1)
+; CHECK-32-NEXT:    li r0, 16
+; CHECK-32-NEXT:    mtctr r0
+; CHECK-32-NEXT:    lis r0, -1
+; CHECK-32-NEXT:    nop
+; CHECK-32-NEXT:  .LBB5_1: # %entry
+; CHECK-32-NEXT:    #
+; CHECK-32-NEXT:    stwux r12, r1, r0
+; CHECK-32-NEXT:    bdnz .LBB5_1
+; CHECK-32-NEXT:  # %bb.2: # %entry
+; CHECK-32-NEXT:    sub r0, r1, r12
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    stb r3, 16(r1)
+; CHECK-32-NEXT:    mr r0, r31
+; CHECK-32-NEXT:    lbz r3, 16(r1)
+; CHECK-32-NEXT:    lwz r31, 0(r1)
+; CHECK-32-NEXT:    mr r1, r31
+; CHECK-32-NEXT:    mr r31, r0
+; CHECK-32-NEXT:    blr
+entry:
+  %a = alloca i8, i64 1048576
+  %b = getelementptr inbounds i8, i8* %a, i64 63
+  store volatile i8 3, i8* %a
+  %c = load volatile i8, i8* %a
+  ret i8 %c
+}
+
+define i8 @f6() #0 nounwind {
+; CHECK-LE-LABEL: f6:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    stdu r12, -48(r1)
+; CHECK-LE-NEXT:    lis r0, 4
+; CHECK-LE-NEXT:    nop
+; CHECK-LE-NEXT:    mtctr r0
+; CHECK-LE-NEXT:  .LBB6_1: # %entry
+; CHECK-LE-NEXT:    #
+; CHECK-LE-NEXT:    stdu r12, -4096(r1)
+; CHECK-LE-NEXT:    bdnz .LBB6_1
+; CHECK-LE-NEXT:  # %bb.2: # %entry
+; CHECK-LE-NEXT:    li r3, 3
+; CHECK-LE-NEXT:    stb r3, 48(r1)
+; CHECK-LE-NEXT:    lbz r3, 48(r1)
+; CHECK-LE-NEXT:    ld r1, 0(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f6:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    stdu r12, -64(r1)
+; CHECK-BE-NEXT:    lis r0, 4
+; CHECK-BE-NEXT:    nop
+; CHECK-BE-NEXT:    mtctr r0
+; CHECK-BE-NEXT:  .LBB6_1: # %entry
+; CHECK-BE-NEXT:    #
+; CHECK-BE-NEXT:    stdu r12, -4096(r1)
+; CHECK-BE-NEXT:    bdnz .LBB6_1
+; CHECK-BE-NEXT:  # %bb.2: # %entry
+; CHECK-BE-NEXT:    li r3, 3
+; CHECK-BE-NEXT:    stb r3, 64(r1)
+; CHECK-BE-NEXT:    lbz r3, 64(r1)
+; CHECK-BE-NEXT:    ld r1, 0(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f6:
+; CHECK-32:       # %bb.0: # %entry
+; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    stwu r12, -16(r1)
+; CHECK-32-NEXT:    lis r0, 4
+; CHECK-32-NEXT:    nop
+; CHECK-32-NEXT:    mtctr r0
+; CHECK-32-NEXT:  .LBB6_1: # %entry
+; CHECK-32-NEXT:    #
+; CHECK-32-NEXT:    stwu r12, -4096(r1)
+; CHECK-32-NEXT:    bdnz .LBB6_1
+; CHECK-32-NEXT:  # %bb.2: # %entry
+; CHECK-32-NEXT:    sub r0, r1, r12
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    stb r3, 16(r1)
+; CHECK-32-NEXT:    mr r0, r31
+; CHECK-32-NEXT:    lbz r3, 16(r1)
+; CHECK-32-NEXT:    lwz r31, 0(r1)
+; CHECK-32-NEXT:    mr r1, r31
+; CHECK-32-NEXT:    mr r31, r0
+; CHECK-32-NEXT:    blr
+entry:
+  %a = alloca i8, i64 1073741824
+  %b = getelementptr inbounds i8, i8* %a, i64 63
+  store volatile i8 3, i8* %a
+  %c = load volatile i8, i8* %a
+  ret i8 %c
+}
+
+define i8 @f7() #0 "stack-probe-size"="65536" nounwind {
+; CHECK-LE-LABEL: f7:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    lis r0, -1
+; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    ori r0, r0, 13776
+; CHECK-LE-NEXT:    stdux r12, r1, r0
+; CHECK-LE-NEXT:    li r0, 15258
+; CHECK-LE-NEXT:    mtctr r0
+; CHECK-LE-NEXT:    lis r0, -1
+; CHECK-LE-NEXT:    nop
+; CHECK-LE-NEXT:  .LBB7_1: # %entry
+; CHECK-LE-NEXT:    #
+; CHECK-LE-NEXT:    stdux r12, r1, r0
+; CHECK-LE-NEXT:    bdnz .LBB7_1
+; CHECK-LE-NEXT:  # %bb.2: # %entry
+; CHECK-LE-NEXT:    li r3, 3
+; CHECK-LE-NEXT:    stb r3, 41(r1)
+; CHECK-LE-NEXT:    lbz r3, 41(r1)
+; CHECK-LE-NEXT:    ld r1, 0(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f7:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lis r0, -1
+; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    ori r0, r0, 13760
+; CHECK-BE-NEXT:    stdux r12, r1, r0
+; CHECK-BE-NEXT:    li r0, 15258
+; CHECK-BE-NEXT:    mtctr r0
+; CHECK-BE-NEXT:    lis r0, -1
+; CHECK-BE-NEXT:    nop
+; CHECK-BE-NEXT:  .LBB7_1: # %entry
+; CHECK-BE-NEXT:    #
+; CHECK-BE-NEXT:    stdux r12, r1, r0
+; CHECK-BE-NEXT:    bdnz .LBB7_1
+; CHECK-BE-NEXT:  # %bb.2: # %entry
+; CHECK-BE-NEXT:    li r3, 3
+; CHECK-BE-NEXT:    stb r3, 57(r1)
+; CHECK-BE-NEXT:    lbz r3, 57(r1)
+; CHECK-BE-NEXT:    ld r1, 0(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f7:
+; CHECK-32:       # %bb.0: # %entry
+; CHECK-32-NEXT:    lis r0, -1
+; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    ori r0, r0, 13808
+; CHECK-32-NEXT:    stwux r12, r1, r0
+; CHECK-32-NEXT:    li r0, 15258
+; CHECK-32-NEXT:    mtctr r0
+; CHECK-32-NEXT:    lis r0, -1
+; CHECK-32-NEXT:    nop
+; CHECK-32-NEXT:  .LBB7_1: # %entry
+; CHECK-32-NEXT:    #
+; CHECK-32-NEXT:    stwux r12, r1, r0
+; CHECK-32-NEXT:    bdnz .LBB7_1
+; CHECK-32-NEXT:  # %bb.2: # %entry
+; CHECK-32-NEXT:    sub r0, r1, r12
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    stb r3, 9(r1)
+; CHECK-32-NEXT:    mr r0, r31
+; CHECK-32-NEXT:    lbz r3, 9(r1)
+; CHECK-32-NEXT:    lwz r31, 0(r1)
+; CHECK-32-NEXT:    mr r1, r31
+; CHECK-32-NEXT:    mr r31, r0
+; CHECK-32-NEXT:    blr
+entry:
+  %a = alloca i8, i64 1000000007
+  %b = getelementptr inbounds i8, i8* %a, i64 101
+  store volatile i8 3, i8* %a
+  %c = load volatile i8, i8* %a
+  ret i8 %c
+}
+
+attributes #0 = { "probe-stack"="inline-asm" }
diff --git a/llvm/test/CodeGen/PowerPC/stack-clash-prologue.ll b/llvm/test/CodeGen/PowerPC/stack-clash-prologue.ll
new file mode 100644
index 0000000000000..eb8e05eef519f
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/stack-clash-prologue.ll
@@ -0,0 +1,495 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs \
+; RUN:   -mtriple=powerpc64le-linux-gnu < %s | FileCheck \
+; RUN:   -check-prefix=CHECK-LE %s
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs \
+; RUN:   -mtriple=powerpc64-linux-gnu < %s | FileCheck \
+; RUN:   -check-prefix=CHECK-BE %s
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs \
+; RUN:   -mtriple=powerpc-linux-gnu < %s | FileCheck \
+; RUN:   -check-prefix=CHECK-32 %s
+
+; Free probe
+define i8 @f0() #0 nounwind {
+; CHECK-LE-LABEL: f0:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    li r3, 3
+; CHECK-LE-NEXT:    stb r3, -64(r1)
+; CHECK-LE-NEXT:    lbz r3, -64(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f0:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    li r3, 3
+; CHECK-BE-NEXT:    stb r3, -64(r1)
+; CHECK-BE-NEXT:    lbz r3, -64(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f0:
+; CHECK-32:       # %bb.0: # %entry
+; CHECK-32-NEXT:    stwu r1, -80(r1)
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    stb r3, 16(r1)
+; CHECK-32-NEXT:    lbz r3, 16(r1)
+; CHECK-32-NEXT:    addi r1, r1, 80
+; CHECK-32-NEXT:    blr
+entry:
+  %a = alloca i8, i64 64
+  %b = getelementptr inbounds i8, i8* %a, i64 63
+  store volatile i8 3, i8* %a
+  %c = load volatile i8, i8* %a
+  ret i8 %c
+}
+
+define i8 @f1() #0 "stack-probe-size"="0" {
+; CHECK-LE-LABEL: f1:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    li r0, 259
+; CHECK-LE-NEXT:    mtctr r0
+; CHECK-LE-NEXT:  .LBB1_1: # %entry
+; CHECK-LE-NEXT:    #
+; CHECK-LE-NEXT:    stdu r12, -16(r1)
+; CHECK-LE-NEXT:    bdnz .LBB1_1
+; CHECK-LE-NEXT:  # %bb.2: # %entry
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 4144
+; CHECK-LE-NEXT:    li r3, 3
+; CHECK-LE-NEXT:    stb r3, 48(r1)
+; CHECK-LE-NEXT:    lbz r3, 48(r1)
+; CHECK-LE-NEXT:    addi r1, r1, 4144
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f1:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    li r0, 260
+; CHECK-BE-NEXT:    mtctr r0
+; CHECK-BE-NEXT:  .LBB1_1: # %entry
+; CHECK-BE-NEXT:    #
+; CHECK-BE-NEXT:    stdu r12, -16(r1)
+; CHECK-BE-NEXT:    bdnz .LBB1_1
+; CHECK-BE-NEXT:  # %bb.2: # %entry
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 4160
+; CHECK-BE-NEXT:    li r3, 3
+; CHECK-BE-NEXT:    stb r3, 64(r1)
+; CHECK-BE-NEXT:    lbz r3, 64(r1)
+; CHECK-BE-NEXT:    addi r1, r1, 4160
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f1:
+; CHECK-32:       # %bb.0: # %entry
+; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    li r0, 257
+; CHECK-32-NEXT:    mtctr r0
+; CHECK-32-NEXT:  .LBB1_1: # %entry
+; CHECK-32-NEXT:    #
+; CHECK-32-NEXT:    stwu r12, -16(r1)
+; CHECK-32-NEXT:    bdnz .LBB1_1
+; CHECK-32-NEXT:  # %bb.2: # %entry
+; CHECK-32-NEXT:    sub r0, r1, r12
+; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 4112
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    stb r3, 16(r1)
+; CHECK-32-NEXT:    lbz r3, 16(r1)
+; CHECK-32-NEXT:    addi r1, r1, 4112
+; CHECK-32-NEXT:    blr
+entry:
+  %a = alloca i8, i64 4096
+  %b = getelementptr inbounds i8, i8* %a, i64 63
+  store volatile i8 3, i8* %a
+  %c = load volatile i8, i8* %a
+  ret i8 %c
+}
+
+define i8 @f2() #0 {
+; CHECK-LE-LABEL: f2:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    stdu r12, -48(r1)
+; CHECK-LE-NEXT:    li r0, 16
+; CHECK-LE-NEXT:    mtctr r0
+; CHECK-LE-NEXT:  .LBB2_1: # %entry
+; CHECK-LE-NEXT:    #
+; CHECK-LE-NEXT:    stdu r12, -4096(r1)
+; CHECK-LE-NEXT:    bdnz .LBB2_1
+; CHECK-LE-NEXT:  # %bb.2: # %entry
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 65584
+; CHECK-LE-NEXT:    li r3, 3
+; CHECK-LE-NEXT:    stb r3, 48(r1)
+; CHECK-LE-NEXT:    lbz r3, 48(r1)
+; CHECK-LE-NEXT:    ld r1, 0(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f2:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    stdu r12, -64(r1)
+; CHECK-BE-NEXT:    li r0, 16
+; CHECK-BE-NEXT:    mtctr r0
+; CHECK-BE-NEXT:  .LBB2_1: # %entry
+; CHECK-BE-NEXT:    #
+; CHECK-BE-NEXT:    stdu r12, -4096(r1)
+; CHECK-BE-NEXT:    bdnz .LBB2_1
+; CHECK-BE-NEXT:  # %bb.2: # %entry
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 65600
+; CHECK-BE-NEXT:    li r3, 3
+; CHECK-BE-NEXT:    stb r3, 64(r1)
+; CHECK-BE-NEXT:    lbz r3, 64(r1)
+; CHECK-BE-NEXT:    ld r1, 0(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f2:
+; CHECK-32:       # %bb.0: # %entry
+; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    stwu r12, -16(r1)
+; CHECK-32-NEXT:    li r0, 16
+; CHECK-32-NEXT:    mtctr r0
+; CHECK-32-NEXT:  .LBB2_1: # %entry
+; CHECK-32-NEXT:    #
+; CHECK-32-NEXT:    stwu r12, -4096(r1)
+; CHECK-32-NEXT:    bdnz .LBB2_1
+; CHECK-32-NEXT:  # %bb.2: # %entry
+; CHECK-32-NEXT:    sub r0, r1, r12
+; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 65552
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    stb r3, 16(r1)
+; CHECK-32-NEXT:    mr r0, r31
+; CHECK-32-NEXT:    lbz r3, 16(r1)
+; CHECK-32-NEXT:    lwz r31, 0(r1)
+; CHECK-32-NEXT:    mr r1, r31
+; CHECK-32-NEXT:    mr r31, r0
+; CHECK-32-NEXT:    blr
+entry:
+  %a = alloca i8, i64 65536
+  %b = getelementptr inbounds i8, i8* %a, i64 63
+  store volatile i8 3, i8* %a
+  %c = load volatile i8, i8* %a
+  ret i8 %c
+}
+
+define i8 @f3() #0 "stack-probe-size"="32768" {
+; CHECK-LE-LABEL: f3:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    stdu r12, -48(r1)
+; CHECK-LE-NEXT:    stdu r12, -32768(r1)
+; CHECK-LE-NEXT:    stdu r12, -32768(r1)
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 65584
+; CHECK-LE-NEXT:    li r3, 3
+; CHECK-LE-NEXT:    stb r3, 48(r1)
+; CHECK-LE-NEXT:    lbz r3, 48(r1)
+; CHECK-LE-NEXT:    ld r1, 0(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f3:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    stdu r12, -64(r1)
+; CHECK-BE-NEXT:    stdu r12, -32768(r1)
+; CHECK-BE-NEXT:    stdu r12, -32768(r1)
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 65600
+; CHECK-BE-NEXT:    li r3, 3
+; CHECK-BE-NEXT:    stb r3, 64(r1)
+; CHECK-BE-NEXT:    lbz r3, 64(r1)
+; CHECK-BE-NEXT:    ld r1, 0(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f3:
+; CHECK-32:       # %bb.0: # %entry
+; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    stwu r12, -16(r1)
+; CHECK-32-NEXT:    stwu r12, -32768(r1)
+; CHECK-32-NEXT:    stwu r12, -32768(r1)
+; CHECK-32-NEXT:    sub r0, r1, r12
+; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 65552
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    stb r3, 16(r1)
+; CHECK-32-NEXT:    mr r0, r31
+; CHECK-32-NEXT:    lbz r3, 16(r1)
+; CHECK-32-NEXT:    lwz r31, 0(r1)
+; CHECK-32-NEXT:    mr r1, r31
+; CHECK-32-NEXT:    mr r31, r0
+; CHECK-32-NEXT:    blr
+entry:
+  %a = alloca i8, i64 65536
+  %b = getelementptr inbounds i8, i8* %a, i64 63
+  store volatile i8 3, i8* %a
+  %c = load volatile i8, i8* %a
+  ret i8 %c
+}
+
+; Same as f2, but without protection.
+define i8 @f4() {
+; CHECK-LE-LABEL: f4:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    lis r0, -2
+; CHECK-LE-NEXT:    ori r0, r0, 65488
+; CHECK-LE-NEXT:    stdux r1, r1, r0
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 65584
+; CHECK-LE-NEXT:    li r3, 3
+; CHECK-LE-NEXT:    stb r3, 48(r1)
+; CHECK-LE-NEXT:    lbz r3, 48(r1)
+; CHECK-LE-NEXT:    ld r1, 0(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f4:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lis r0, -2
+; CHECK-BE-NEXT:    ori r0, r0, 65472
+; CHECK-BE-NEXT:    stdux r1, r1, r0
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 65600
+; CHECK-BE-NEXT:    li r3, 3
+; CHECK-BE-NEXT:    stb r3, 64(r1)
+; CHECK-BE-NEXT:    lbz r3, 64(r1)
+; CHECK-BE-NEXT:    ld r1, 0(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f4:
+; CHECK-32:       # %bb.0: # %entry
+; CHECK-32-NEXT:    lis r0, -2
+; CHECK-32-NEXT:    ori r0, r0, 65520
+; CHECK-32-NEXT:    stwux r1, r1, r0
+; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 65552
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    stb r3, 16(r1)
+; CHECK-32-NEXT:    mr r0, r31
+; CHECK-32-NEXT:    lbz r3, 16(r1)
+; CHECK-32-NEXT:    lwz r31, 0(r1)
+; CHECK-32-NEXT:    mr r1, r31
+; CHECK-32-NEXT:    mr r31, r0
+; CHECK-32-NEXT:    blr
+entry:
+  %a = alloca i8, i64 65536
+  %b = getelementptr inbounds i8, i8* %a, i64 63
+  store volatile i8 3, i8* %a
+  %c = load volatile i8, i8* %a
+  ret i8 %c
+}
+
+define i8 @f5() #0 "stack-probe-size"="65536" {
+; CHECK-LE-LABEL: f5:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    stdu r12, -48(r1)
+; CHECK-LE-NEXT:    li r0, 16
+; CHECK-LE-NEXT:    mtctr r0
+; CHECK-LE-NEXT:    lis r0, -1
+; CHECK-LE-NEXT:    nop
+; CHECK-LE-NEXT:  .LBB5_1: # %entry
+; CHECK-LE-NEXT:    #
+; CHECK-LE-NEXT:    stdux r12, r1, r0
+; CHECK-LE-NEXT:    bdnz .LBB5_1
+; CHECK-LE-NEXT:  # %bb.2: # %entry
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 1048624
+; CHECK-LE-NEXT:    li r3, 3
+; CHECK-LE-NEXT:    stb r3, 48(r1)
+; CHECK-LE-NEXT:    lbz r3, 48(r1)
+; CHECK-LE-NEXT:    ld r1, 0(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f5:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    stdu r12, -64(r1)
+; CHECK-BE-NEXT:    li r0, 16
+; CHECK-BE-NEXT:    mtctr r0
+; CHECK-BE-NEXT:    lis r0, -1
+; CHECK-BE-NEXT:    nop
+; CHECK-BE-NEXT:  .LBB5_1: # %entry
+; CHECK-BE-NEXT:    #
+; CHECK-BE-NEXT:    stdux r12, r1, r0
+; CHECK-BE-NEXT:    bdnz .LBB5_1
+; CHECK-BE-NEXT:  # %bb.2: # %entry
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 1048640
+; CHECK-BE-NEXT:    li r3, 3
+; CHECK-BE-NEXT:    stb r3, 64(r1)
+; CHECK-BE-NEXT:    lbz r3, 64(r1)
+; CHECK-BE-NEXT:    ld r1, 0(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f5:
+; CHECK-32:       # %bb.0: # %entry
+; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    stwu r12, -16(r1)
+; CHECK-32-NEXT:    li r0, 16
+; CHECK-32-NEXT:    mtctr r0
+; CHECK-32-NEXT:    lis r0, -1
+; CHECK-32-NEXT:    nop
+; CHECK-32-NEXT:  .LBB5_1: # %entry
+; CHECK-32-NEXT:    #
+; CHECK-32-NEXT:    stwux r12, r1, r0
+; CHECK-32-NEXT:    bdnz .LBB5_1
+; CHECK-32-NEXT:  # %bb.2: # %entry
+; CHECK-32-NEXT:    sub r0, r1, r12
+; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 1048592
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    stb r3, 16(r1)
+; CHECK-32-NEXT:    mr r0, r31
+; CHECK-32-NEXT:    lbz r3, 16(r1)
+; CHECK-32-NEXT:    lwz r31, 0(r1)
+; CHECK-32-NEXT:    mr r1, r31
+; CHECK-32-NEXT:    mr r31, r0
+; CHECK-32-NEXT:    blr
+entry:
+  %a = alloca i8, i64 1048576
+  %b = getelementptr inbounds i8, i8* %a, i64 63
+  store volatile i8 3, i8* %a
+  %c = load volatile i8, i8* %a
+  ret i8 %c
+}
+
+define i8 @f6() #0 {
+; CHECK-LE-LABEL: f6:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    stdu r12, -48(r1)
+; CHECK-LE-NEXT:    lis r0, 4
+; CHECK-LE-NEXT:    nop
+; CHECK-LE-NEXT:    mtctr r0
+; CHECK-LE-NEXT:  .LBB6_1: # %entry
+; CHECK-LE-NEXT:    #
+; CHECK-LE-NEXT:    stdu r12, -4096(r1)
+; CHECK-LE-NEXT:    bdnz .LBB6_1
+; CHECK-LE-NEXT:  # %bb.2: # %entry
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 1073741872
+; CHECK-LE-NEXT:    li r3, 3
+; CHECK-LE-NEXT:    stb r3, 48(r1)
+; CHECK-LE-NEXT:    lbz r3, 48(r1)
+; CHECK-LE-NEXT:    ld r1, 0(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f6:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    stdu r12, -64(r1)
+; CHECK-BE-NEXT:    lis r0, 4
+; CHECK-BE-NEXT:    nop
+; CHECK-BE-NEXT:    mtctr r0
+; CHECK-BE-NEXT:  .LBB6_1: # %entry
+; CHECK-BE-NEXT:    #
+; CHECK-BE-NEXT:    stdu r12, -4096(r1)
+; CHECK-BE-NEXT:    bdnz .LBB6_1
+; CHECK-BE-NEXT:  # %bb.2: # %entry
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 1073741888
+; CHECK-BE-NEXT:    li r3, 3
+; CHECK-BE-NEXT:    stb r3, 64(r1)
+; CHECK-BE-NEXT:    lbz r3, 64(r1)
+; CHECK-BE-NEXT:    ld r1, 0(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f6:
+; CHECK-32:       # %bb.0: # %entry
+; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    stwu r12, -16(r1)
+; CHECK-32-NEXT:    lis r0, 4
+; CHECK-32-NEXT:    nop
+; CHECK-32-NEXT:    mtctr r0
+; CHECK-32-NEXT:  .LBB6_1: # %entry
+; CHECK-32-NEXT:    #
+; CHECK-32-NEXT:    stwu r12, -4096(r1)
+; CHECK-32-NEXT:    bdnz .LBB6_1
+; CHECK-32-NEXT:  # %bb.2: # %entry
+; CHECK-32-NEXT:    sub r0, r1, r12
+; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 1073741840
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    stb r3, 16(r1)
+; CHECK-32-NEXT:    mr r0, r31
+; CHECK-32-NEXT:    lbz r3, 16(r1)
+; CHECK-32-NEXT:    lwz r31, 0(r1)
+; CHECK-32-NEXT:    mr r1, r31
+; CHECK-32-NEXT:    mr r31, r0
+; CHECK-32-NEXT:    blr
+entry:
+  %a = alloca i8, i64 1073741824
+  %b = getelementptr inbounds i8, i8* %a, i64 63
+  store volatile i8 3, i8* %a
+  %c = load volatile i8, i8* %a
+  ret i8 %c
+}
+
+define i8 @f7() #0 "stack-probe-size"="65536" {
+; CHECK-LE-LABEL: f7:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    lis r0, -1
+; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    ori r0, r0, 13776
+; CHECK-LE-NEXT:    stdux r12, r1, r0
+; CHECK-LE-NEXT:    li r0, 15258
+; CHECK-LE-NEXT:    mtctr r0
+; CHECK-LE-NEXT:    lis r0, -1
+; CHECK-LE-NEXT:    nop
+; CHECK-LE-NEXT:  .LBB7_1: # %entry
+; CHECK-LE-NEXT:    #
+; CHECK-LE-NEXT:    stdux r12, r1, r0
+; CHECK-LE-NEXT:    bdnz .LBB7_1
+; CHECK-LE-NEXT:  # %bb.2: # %entry
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 1000000048
+; CHECK-LE-NEXT:    li r3, 3
+; CHECK-LE-NEXT:    stb r3, 41(r1)
+; CHECK-LE-NEXT:    lbz r3, 41(r1)
+; CHECK-LE-NEXT:    ld r1, 0(r1)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: f7:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lis r0, -1
+; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    ori r0, r0, 13760
+; CHECK-BE-NEXT:    stdux r12, r1, r0
+; CHECK-BE-NEXT:    li r0, 15258
+; CHECK-BE-NEXT:    mtctr r0
+; CHECK-BE-NEXT:    lis r0, -1
+; CHECK-BE-NEXT:    nop
+; CHECK-BE-NEXT:  .LBB7_1: # %entry
+; CHECK-BE-NEXT:    #
+; CHECK-BE-NEXT:    stdux r12, r1, r0
+; CHECK-BE-NEXT:    bdnz .LBB7_1
+; CHECK-BE-NEXT:  # %bb.2: # %entry
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 1000000064
+; CHECK-BE-NEXT:    li r3, 3
+; CHECK-BE-NEXT:    stb r3, 57(r1)
+; CHECK-BE-NEXT:    lbz r3, 57(r1)
+; CHECK-BE-NEXT:    ld r1, 0(r1)
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-32-LABEL: f7:
+; CHECK-32:       # %bb.0: # %entry
+; CHECK-32-NEXT:    lis r0, -1
+; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    ori r0, r0, 13808
+; CHECK-32-NEXT:    stwux r12, r1, r0
+; CHECK-32-NEXT:    li r0, 15258
+; CHECK-32-NEXT:    mtctr r0
+; CHECK-32-NEXT:    lis r0, -1
+; CHECK-32-NEXT:    nop
+; CHECK-32-NEXT:  .LBB7_1: # %entry
+; CHECK-32-NEXT:    #
+; CHECK-32-NEXT:    stwux r12, r1, r0
+; CHECK-32-NEXT:    bdnz .LBB7_1
+; CHECK-32-NEXT:  # %bb.2: # %entry
+; CHECK-32-NEXT:    sub r0, r1, r12
+; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 1000000016
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    stb r3, 9(r1)
+; CHECK-32-NEXT:    mr r0, r31
+; CHECK-32-NEXT:    lbz r3, 9(r1)
+; CHECK-32-NEXT:    lwz r31, 0(r1)
+; CHECK-32-NEXT:    mr r1, r31
+; CHECK-32-NEXT:    mr r31, r0
+; CHECK-32-NEXT:    blr
+entry:
+  %a = alloca i8, i64 1000000007
+  %b = getelementptr inbounds i8, i8* %a, i64 101
+  store volatile i8 3, i8* %a
+  %c = load volatile i8, i8* %a
+  ret i8 %c
+}
+
+attributes #0 = { "probe-stack"="inline-asm" }
diff --git a/llvm/test/CodeGen/PowerPC/test_func_desc.ll b/llvm/test/CodeGen/PowerPC/test_func_desc.ll
index 500be4b63d2da..4c79bafeb035f 100644
--- a/llvm/test/CodeGen/PowerPC/test_func_desc.ll
+++ b/llvm/test/CodeGen/PowerPC/test_func_desc.ll
@@ -56,6 +56,7 @@ entry:
 ; CHECK: bl .extern_foo
 ; CHECK: bl .static_foo
 
+; CHECK: .lglobl static_foo[DS]
 ; CHECK: .lglobl .static_foo
 ; 32BIT: .csect static_foo[DS],2
 ; 32BIT-NEXT: .vbyte	4, .static_foo
diff --git a/llvm/test/CodeGen/RISCV/add-imm.ll b/llvm/test/CodeGen/RISCV/add-imm.ll
new file mode 100644
index 0000000000000..2db13eb0ba839
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/add-imm.ll
@@ -0,0 +1,209 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+
+; These test how the immediate in an addition is materialized.
+
+define i32 @add_positive_low_bound_reject(i32 %a) nounwind {
+; RV32I-LABEL: add_positive_low_bound_reject:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a0, a0, 2047
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: add_positive_low_bound_reject:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a0, a0, 2047
+; RV64I-NEXT:    ret
+  %1 = add i32 %a, 2047
+  ret i32 %1
+}
+
+define i32 @add_positive_low_bound_accept(i32 %a) nounwind {
+; RV32I-LABEL: add_positive_low_bound_accept:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a0, a0, 1024
+; RV32I-NEXT:    addi a0, a0, 1024
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: add_positive_low_bound_accept:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a0, a0, 1024
+; RV64I-NEXT:    addi a0, a0, 1024
+; RV64I-NEXT:    ret
+  %1 = add i32 %a, 2048
+  ret i32 %1
+}
+
+define i32 @add_positive_high_bound_accept(i32 %a) nounwind {
+; RV32I-LABEL: add_positive_high_bound_accept:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a0, a0, 2047
+; RV32I-NEXT:    addi a0, a0, 2047
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: add_positive_high_bound_accept:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a0, a0, 2047
+; RV64I-NEXT:    addi a0, a0, 2047
+; RV64I-NEXT:    ret
+  %1 = add i32 %a, 4094
+  ret i32 %1
+}
+
+define i32 @add_positive_high_bound_reject(i32 %a) nounwind {
+; RV32I-LABEL: add_positive_high_bound_reject:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 1
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: add_positive_high_bound_reject:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 1
+; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ret
+  %1 = add i32 %a, 4095
+  ret i32 %1
+}
+
+define i32 @add_negative_high_bound_reject(i32 %a) nounwind {
+; RV32I-LABEL: add_negative_high_bound_reject:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a0, a0, -2048
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: add_negative_high_bound_reject:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a0, a0, -2048
+; RV64I-NEXT:    ret
+  %1 = add i32 %a, -2048
+  ret i32 %1
+}
+
+define i32 @add_negative_high_bound_accept(i32 %a) nounwind {
+; RV32I-LABEL: add_negative_high_bound_accept:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a0, a0, -1025
+; RV32I-NEXT:    addi a0, a0, -1024
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: add_negative_high_bound_accept:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a0, a0, -1025
+; RV64I-NEXT:    addi a0, a0, -1024
+; RV64I-NEXT:    ret
+  %1 = add i32 %a, -2049
+  ret i32 %1
+}
+
+define i32 @add_negative_low_bound_accept(i32 %a) nounwind {
+; RV32I-LABEL: add_negative_low_bound_accept:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a0, a0, -2048
+; RV32I-NEXT:    addi a0, a0, -2048
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: add_negative_low_bound_accept:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a0, a0, -2048
+; RV64I-NEXT:    addi a0, a0, -2048
+; RV64I-NEXT:    ret
+  %1 = add i32 %a, -4096
+  ret i32 %1
+}
+
+define i32 @add_negative_low_bound_reject(i32 %a) nounwind {
+; RV32I-LABEL: add_negative_low_bound_reject:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 1048575
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: add_negative_low_bound_reject:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 1048575
+; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ret
+  %1 = add i32 %a, -4097
+  ret i32 %1
+}
+
+define i32 @add32_accept(i32 %a) nounwind {
+; RV32I-LABEL: add32_accept:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a0, a0, 1500
+; RV32I-NEXT:    addi a0, a0, 1499
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: add32_accept:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a0, a0, 1500
+; RV64I-NEXT:    addi a0, a0, 1499
+; RV64I-NEXT:    ret
+  %1 = add i32 %a, 2999
+  ret i32 %1
+}
+
+define i64 @add64_accept(i64 %a) nounwind {
+; RV32I-LABEL: add64_accept:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a2, a0, 1500
+; RV32I-NEXT:    addi a2, a2, 1499
+; RV32I-NEXT:    sltu a0, a2, a0
+; RV32I-NEXT:    add a1, a1, a0
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: add64_accept:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a0, a0, 1500
+; RV64I-NEXT:    addi a0, a0, 1499
+; RV64I-NEXT:    ret
+  %1 = add i64 %a, 2999
+  ret i64 %1
+}
+
+@ga = global i32 0, align 4
+@gb = global i32 0, align 4
+define void @add32_reject() nounwind {
+; RV32I-LABEL: add32_reject:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a0, %hi(ga)
+; RV32I-NEXT:    lw a1, %lo(ga)(a0)
+; RV32I-NEXT:    lui a2, %hi(gb)
+; RV32I-NEXT:    lw a3, %lo(gb)(a2)
+; RV32I-NEXT:    lui a4, 1
+; RV32I-NEXT:    addi a4, a4, -1096
+; RV32I-NEXT:    add a1, a1, a4
+; RV32I-NEXT:    add a3, a3, a4
+; RV32I-NEXT:    sw a1, %lo(ga)(a0)
+; RV32I-NEXT:    sw a3, %lo(gb)(a2)
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: add32_reject:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a0, %hi(ga)
+; RV64I-NEXT:    lw a1, %lo(ga)(a0)
+; RV64I-NEXT:    lui a2, %hi(gb)
+; RV64I-NEXT:    lw a3, %lo(gb)(a2)
+; RV64I-NEXT:    lui a4, 1
+; RV64I-NEXT:    addiw a4, a4, -1096
+; RV64I-NEXT:    add a1, a1, a4
+; RV64I-NEXT:    add a3, a3, a4
+; RV64I-NEXT:    sw a1, %lo(ga)(a0)
+; RV64I-NEXT:    sw a3, %lo(gb)(a2)
+; RV64I-NEXT:    ret
+  %1 = load i32, i32* @ga, align 4
+  %2 = load i32, i32* @gb, align 4
+  %3 = add i32 %1, 3000
+  %4 = add i32 %2, 3000
+  store i32 %3, i32* @ga, align 4
+  store i32 %4, i32* @gb, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/addimm-mulimm.ll b/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
new file mode 100644
index 0000000000000..6a2b3c3f0e3e0
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
@@ -0,0 +1,95 @@
+;; Test that (mul (add x, c1), c2) can be transformed to
+;; (add (mul x, c2), c1*c2) if profitable.
+
+; RUN: llc -mtriple=riscv32 -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32IM %s
+; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IM %s
+
+define signext i32 @add_mul_trans_accept_1(i32 %x) {
+; RV32IM-LABEL: add_mul_trans_accept_1
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi a1, zero, 11
+; RV32IM-NEXT:    mul a0, a0, a1
+; RV32IM-NEXT:    addi a0, a0, 407
+; RV32IM-NEXT:    ret
+;
+; RV64IM-LABEL: add_mul_trans_accept_1
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    addi a1, zero, 11
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    addiw a0, a0, 407
+; RV64IM-NEXT:    ret
+  %tmp0 = add i32 %x, 37
+  %tmp1 = mul i32 %tmp0, 11
+  ret i32 %tmp1
+}
+
+define signext i32 @add_mul_trans_accept_2(i32 %x) {
+; RV32IM-LABEL: add_mul_trans_accept_2
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi a1, zero, 13
+; RV32IM-NEXT:    mul a0, a0, a1
+; RV32IM-NEXT:    lui a1, 28
+; RV32IM-NEXT:    addi a1, a1, 1701
+; RV32IM-NEXT:    add a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64IM-LABEL: add_mul_trans_accept_2
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    addi a1, zero, 13
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    lui a1, 28
+; RV64IM-NEXT:    addiw a1, a1, 1701
+; RV64IM-NEXT:    addw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %tmp0 = add i32 %x, 8953
+  %tmp1 = mul i32 %tmp0, 13
+  ret i32 %tmp1
+}
+
+define signext i32 @add_mul_trans_reject_1(i32 %x) {
+; RV32IM-LABEL: add_mul_trans_reject_1
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi a1, zero, 19
+; RV32IM-NEXT:    mul a0, a0, a1
+; RV32IM-NEXT:    lui a1, 9
+; RV32IM-NEXT:    addi a1, a1, 585
+; RV32IM-NEXT:    add a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64IM-LABEL: add_mul_trans_reject_1
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    addi a1, zero, 19
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    lui a1, 9
+; RV64IM-NEXT:    addiw a1, a1, 585
+; RV64IM-NEXT:    addw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %tmp0 = add i32 %x, 1971
+  %tmp1 = mul i32 %tmp0, 19
+  ret i32 %tmp1
+}
+
+define signext i32 @add_mul_trans_reject_2(i32 %x) {
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    lui a1, 792
+; RV32IM-NEXT:    addi a1, a1, -1709
+; RV32IM-NEXT:    mul a0, a0, a1
+; RV32IM-NEXT:    lui a1, 1014660
+; RV32IM-NEXT:    addi a1, a1, -1891
+; RV32IM-NEXT:    add a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lui a1, 792
+; RV64IM-NEXT:    addiw a1, a1, -1709
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    lui a1, 1014660
+; RV64IM-NEXT:    addiw a1, a1, -1891
+; RV64IM-NEXT:    addw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %tmp0 = add i32 %x, 1841231
+  %tmp1 = mul i32 %tmp0, 3242323
+  ret i32 %tmp1
+}
diff --git a/llvm/test/CodeGen/RISCV/branch-relaxation.ll b/llvm/test/CodeGen/RISCV/branch-relaxation.ll
index 56f0f27a06488..3d617bf0b26b4 100644
--- a/llvm/test/CodeGen/RISCV/branch-relaxation.ll
+++ b/llvm/test/CodeGen/RISCV/branch-relaxation.ll
@@ -11,7 +11,7 @@ define void @relax_bcc(i1 %a) nounwind {
 ; CHECK-NEXT:    j .LBB0_2
 ; CHECK-NEXT:  .LBB0_1: # %iftrue
 ; CHECK-NEXT:    #APP
-; CHECK-NEXT:    .space 4096
+; CHECK-NEXT:    .zero 4096
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:  .LBB0_2: # %tail
 ; CHECK-NEXT:    ret
@@ -38,7 +38,7 @@ define i32 @relax_jal(i1 %a) nounwind {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    #APP
-; CHECK-NEXT:    .space 1048576
+; CHECK-NEXT:    .zero 1048576
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    addi a0, zero, 1
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/callee-saved-fpr32s.ll b/llvm/test/CodeGen/RISCV/callee-saved-fpr32s.ll
index 56d3ff04d1635..2c5206d57c72d 100644
--- a/llvm/test/CodeGen/RISCV/callee-saved-fpr32s.ll
+++ b/llvm/test/CodeGen/RISCV/callee-saved-fpr32s.ll
@@ -1,15 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+f -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=ILP32-LP64
+; RUN:   | FileCheck %s -check-prefix=ILP32
 ; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=ILP32-LP64
+; RUN:   | FileCheck %s -check-prefix=LP64
 ; RUN: llc -mtriple=riscv32 -mattr=+f -target-abi ilp32f -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=ILP32F-LP64F
+; RUN:   | FileCheck %s -check-prefix=ILP32F
 ; RUN: llc -mtriple=riscv64 -mattr=+f -target-abi lp64f -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=ILP32F-LP64F
+; RUN:   | FileCheck %s -check-prefix=LP64F
 ; RUN: llc -mtriple=riscv32 -mattr=+d -target-abi ilp32d -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=ILP32D-LP64D
+; RUN:   | FileCheck %s -check-prefix=ILP32D
 ; RUN: llc -mtriple=riscv64 -mattr=+d -target-abi lp64d -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=ILP32D-LP64D
+; RUN:   | FileCheck %s -check-prefix=LP64D
 
 @var = global [32 x float] zeroinitializer
 
@@ -20,113 +21,529 @@
 ; something appropriate.
 
 define void @callee() nounwind {
-; ILP32-LP64-LABEL: callee:
-; ILP32-LP64:       # %bb.0:
-; ILP32-LP64-NEXT:    lui a0, %hi(var)
-; ILP32-LP64-NEXT:    flw ft0, %lo(var)(a0)
-; ILP32-LP64-NEXT:    addi a1, a0, %lo(var)
-; ILP32-LP64-NEXT:    flw ft1, 4(a1)
-; ILP32-LP64-NEXT:    flw ft2, 8(a1)
-; ILP32-LP64-NEXT:    flw ft3, 12(a1)
-; ILP32-LP64-NEXT:    flw ft4, 16(a1)
-; ILP32-LP64-NEXT:    flw ft5, 20(a1)
-; ILP32-LP64-NEXT:    flw ft6, 24(a1)
-; ILP32-LP64-NEXT:    flw ft7, 28(a1)
-; ILP32-LP64-NEXT:    flw fa0, 32(a1)
-; ILP32-LP64-NEXT:    flw fa1, 36(a1)
-; ILP32-LP64-NEXT:    flw fa2, 40(a1)
-; ILP32-LP64-NEXT:    flw fa3, 44(a1)
-; ILP32-LP64-NEXT:    flw fa4, 48(a1)
-; ILP32-LP64-NEXT:    flw fa5, 52(a1)
-; ILP32-LP64-NEXT:    flw fa6, 56(a1)
-; ILP32-LP64-NEXT:    flw fa7, 60(a1)
-; ILP32-LP64-NEXT:    flw ft8, 64(a1)
-; ILP32-LP64-NEXT:    flw ft9, 68(a1)
-; ILP32-LP64-NEXT:    flw ft10, 72(a1)
-; ILP32-LP64-NEXT:    flw ft11, 76(a1)
-; ILP32-LP64-NEXT:    flw fs0, 80(a1)
-; ILP32-LP64-NEXT:    flw fs1, 84(a1)
-; ILP32-LP64-NEXT:    flw fs2, 88(a1)
-; ILP32-LP64-NEXT:    flw fs3, 92(a1)
-; ILP32-LP64-NEXT:    flw fs4, 96(a1)
-; ILP32-LP64-NEXT:    flw fs5, 100(a1)
-; ILP32-LP64-NEXT:    flw fs6, 104(a1)
-; ILP32-LP64-NEXT:    flw fs7, 108(a1)
-; ILP32-LP64-NEXT:    flw fs8, 124(a1)
-; ILP32-LP64-NEXT:    flw fs9, 120(a1)
-; ILP32-LP64-NEXT:    flw fs10, 116(a1)
-; ILP32-LP64-NEXT:    flw fs11, 112(a1)
-; ILP32-LP64-NEXT:    fsw fs8, 124(a1)
-; ILP32-LP64-NEXT:    fsw fs9, 120(a1)
-; ILP32-LP64-NEXT:    fsw fs10, 116(a1)
-; ILP32-LP64-NEXT:    fsw fs11, 112(a1)
-; ILP32-LP64-NEXT:    fsw fs7, 108(a1)
-; ILP32-LP64-NEXT:    fsw fs6, 104(a1)
-; ILP32-LP64-NEXT:    fsw fs5, 100(a1)
-; ILP32-LP64-NEXT:    fsw fs4, 96(a1)
-; ILP32-LP64-NEXT:    fsw fs3, 92(a1)
-; ILP32-LP64-NEXT:    fsw fs2, 88(a1)
-; ILP32-LP64-NEXT:    fsw fs1, 84(a1)
-; ILP32-LP64-NEXT:    fsw fs0, 80(a1)
-; ILP32-LP64-NEXT:    fsw ft11, 76(a1)
-; ILP32-LP64-NEXT:    fsw ft10, 72(a1)
-; ILP32-LP64-NEXT:    fsw ft9, 68(a1)
-; ILP32-LP64-NEXT:    fsw ft8, 64(a1)
-; ILP32-LP64-NEXT:    fsw fa7, 60(a1)
-; ILP32-LP64-NEXT:    fsw fa6, 56(a1)
-; ILP32-LP64-NEXT:    fsw fa5, 52(a1)
-; ILP32-LP64-NEXT:    fsw fa4, 48(a1)
-; ILP32-LP64-NEXT:    fsw fa3, 44(a1)
-; ILP32-LP64-NEXT:    fsw fa2, 40(a1)
-; ILP32-LP64-NEXT:    fsw fa1, 36(a1)
-; ILP32-LP64-NEXT:    fsw fa0, 32(a1)
-; ILP32-LP64-NEXT:    fsw ft7, 28(a1)
-; ILP32-LP64-NEXT:    fsw ft6, 24(a1)
-; ILP32-LP64-NEXT:    fsw ft5, 20(a1)
-; ILP32-LP64-NEXT:    fsw ft4, 16(a1)
-; ILP32-LP64-NEXT:    fsw ft3, 12(a1)
-; ILP32-LP64-NEXT:    fsw ft2, 8(a1)
-; ILP32-LP64-NEXT:    fsw ft1, 4(a1)
-; ILP32-LP64-NEXT:    fsw ft0, %lo(var)(a0)
-; ILP32-LP64-NEXT:    ret
+; ILP32-LABEL: callee:
+; ILP32:       # %bb.0:
+; ILP32-NEXT:    lui a0, %hi(var)
+; ILP32-NEXT:    flw ft0, %lo(var)(a0)
+; ILP32-NEXT:    flw ft1, %lo(var+4)(a0)
+; ILP32-NEXT:    flw ft2, %lo(var+8)(a0)
+; ILP32-NEXT:    flw ft3, %lo(var+12)(a0)
+; ILP32-NEXT:    addi a1, a0, %lo(var)
+; ILP32-NEXT:    flw ft4, 16(a1)
+; ILP32-NEXT:    flw ft5, 20(a1)
+; ILP32-NEXT:    flw ft6, 24(a1)
+; ILP32-NEXT:    flw ft7, 28(a1)
+; ILP32-NEXT:    flw fa0, 32(a1)
+; ILP32-NEXT:    flw fa1, 36(a1)
+; ILP32-NEXT:    flw fa2, 40(a1)
+; ILP32-NEXT:    flw fa3, 44(a1)
+; ILP32-NEXT:    flw fa4, 48(a1)
+; ILP32-NEXT:    flw fa5, 52(a1)
+; ILP32-NEXT:    flw fa6, 56(a1)
+; ILP32-NEXT:    flw fa7, 60(a1)
+; ILP32-NEXT:    flw ft8, 64(a1)
+; ILP32-NEXT:    flw ft9, 68(a1)
+; ILP32-NEXT:    flw ft10, 72(a1)
+; ILP32-NEXT:    flw ft11, 76(a1)
+; ILP32-NEXT:    flw fs0, 80(a1)
+; ILP32-NEXT:    flw fs1, 84(a1)
+; ILP32-NEXT:    flw fs2, 88(a1)
+; ILP32-NEXT:    flw fs3, 92(a1)
+; ILP32-NEXT:    flw fs4, 96(a1)
+; ILP32-NEXT:    flw fs5, 100(a1)
+; ILP32-NEXT:    flw fs6, 104(a1)
+; ILP32-NEXT:    flw fs7, 108(a1)
+; ILP32-NEXT:    flw fs8, 124(a1)
+; ILP32-NEXT:    flw fs9, 120(a1)
+; ILP32-NEXT:    flw fs10, 116(a1)
+; ILP32-NEXT:    flw fs11, 112(a1)
+; ILP32-NEXT:    fsw fs8, 124(a1)
+; ILP32-NEXT:    fsw fs9, 120(a1)
+; ILP32-NEXT:    fsw fs10, 116(a1)
+; ILP32-NEXT:    fsw fs11, 112(a1)
+; ILP32-NEXT:    fsw fs7, 108(a1)
+; ILP32-NEXT:    fsw fs6, 104(a1)
+; ILP32-NEXT:    fsw fs5, 100(a1)
+; ILP32-NEXT:    fsw fs4, 96(a1)
+; ILP32-NEXT:    fsw fs3, 92(a1)
+; ILP32-NEXT:    fsw fs2, 88(a1)
+; ILP32-NEXT:    fsw fs1, 84(a1)
+; ILP32-NEXT:    fsw fs0, 80(a1)
+; ILP32-NEXT:    fsw ft11, 76(a1)
+; ILP32-NEXT:    fsw ft10, 72(a1)
+; ILP32-NEXT:    fsw ft9, 68(a1)
+; ILP32-NEXT:    fsw ft8, 64(a1)
+; ILP32-NEXT:    fsw fa7, 60(a1)
+; ILP32-NEXT:    fsw fa6, 56(a1)
+; ILP32-NEXT:    fsw fa5, 52(a1)
+; ILP32-NEXT:    fsw fa4, 48(a1)
+; ILP32-NEXT:    fsw fa3, 44(a1)
+; ILP32-NEXT:    fsw fa2, 40(a1)
+; ILP32-NEXT:    fsw fa1, 36(a1)
+; ILP32-NEXT:    fsw fa0, 32(a1)
+; ILP32-NEXT:    fsw ft7, 28(a1)
+; ILP32-NEXT:    fsw ft6, 24(a1)
+; ILP32-NEXT:    fsw ft5, 20(a1)
+; ILP32-NEXT:    fsw ft4, 16(a1)
+; ILP32-NEXT:    fsw ft3, %lo(var+12)(a0)
+; ILP32-NEXT:    fsw ft2, %lo(var+8)(a0)
+; ILP32-NEXT:    fsw ft1, %lo(var+4)(a0)
+; ILP32-NEXT:    fsw ft0, %lo(var)(a0)
+; ILP32-NEXT:    ret
 ;
-; ILP32F-LP64F-LABEL: callee:
-; ILP32F-LP64F:       # %bb.0:
-; ILP32F-LP64F-NEXT:    addi sp, sp, -48
-; ILP32F-LP64F-NEXT:    fsw fs0, 44(sp)
-; ILP32F-LP64F-NEXT:    fsw fs1, 40(sp)
-; ILP32F-LP64F-NEXT:    fsw fs2, 36(sp)
-; ILP32F-LP64F-NEXT:    fsw fs3, 32(sp)
-; ILP32F-LP64F-NEXT:    fsw fs4, 28(sp)
-; ILP32F-LP64F-NEXT:    fsw fs5, 24(sp)
-; ILP32F-LP64F-NEXT:    fsw fs6, 20(sp)
-; ILP32F-LP64F-NEXT:    fsw fs7, 16(sp)
-; ILP32F-LP64F-NEXT:    fsw fs8, 12(sp)
-; ILP32F-LP64F-NEXT:    fsw fs9, 8(sp)
-; ILP32F-LP64F-NEXT:    fsw fs10, 4(sp)
-; ILP32F-LP64F-NEXT:    fsw fs11, 0(sp)
-; ILP32F-LP64F-NEXT:    lui a0, %hi(var)
-; ILP32F-LP64F-NEXT:    flw ft0, %lo(var)(a0)
-; ILP32F-LP64F-NEXT:    addi a1, a0, %lo(var)
+; LP64-LABEL: callee:
+; LP64:       # %bb.0:
+; LP64-NEXT:    lui a0, %hi(var)
+; LP64-NEXT:    flw ft0, %lo(var)(a0)
+; LP64-NEXT:    flw ft1, %lo(var+4)(a0)
+; LP64-NEXT:    flw ft2, %lo(var+8)(a0)
+; LP64-NEXT:    flw ft3, %lo(var+12)(a0)
+; LP64-NEXT:    addi a1, a0, %lo(var)
+; LP64-NEXT:    flw ft4, 16(a1)
+; LP64-NEXT:    flw ft5, 20(a1)
+; LP64-NEXT:    flw ft6, 24(a1)
+; LP64-NEXT:    flw ft7, 28(a1)
+; LP64-NEXT:    flw fa0, 32(a1)
+; LP64-NEXT:    flw fa1, 36(a1)
+; LP64-NEXT:    flw fa2, 40(a1)
+; LP64-NEXT:    flw fa3, 44(a1)
+; LP64-NEXT:    flw fa4, 48(a1)
+; LP64-NEXT:    flw fa5, 52(a1)
+; LP64-NEXT:    flw fa6, 56(a1)
+; LP64-NEXT:    flw fa7, 60(a1)
+; LP64-NEXT:    flw ft8, 64(a1)
+; LP64-NEXT:    flw ft9, 68(a1)
+; LP64-NEXT:    flw ft10, 72(a1)
+; LP64-NEXT:    flw ft11, 76(a1)
+; LP64-NEXT:    flw fs0, 80(a1)
+; LP64-NEXT:    flw fs1, 84(a1)
+; LP64-NEXT:    flw fs2, 88(a1)
+; LP64-NEXT:    flw fs3, 92(a1)
+; LP64-NEXT:    flw fs4, 96(a1)
+; LP64-NEXT:    flw fs5, 100(a1)
+; LP64-NEXT:    flw fs6, 104(a1)
+; LP64-NEXT:    flw fs7, 108(a1)
+; LP64-NEXT:    flw fs8, 124(a1)
+; LP64-NEXT:    flw fs9, 120(a1)
+; LP64-NEXT:    flw fs10, 116(a1)
+; LP64-NEXT:    flw fs11, 112(a1)
+; LP64-NEXT:    fsw fs8, 124(a1)
+; LP64-NEXT:    fsw fs9, 120(a1)
+; LP64-NEXT:    fsw fs10, 116(a1)
+; LP64-NEXT:    fsw fs11, 112(a1)
+; LP64-NEXT:    fsw fs7, 108(a1)
+; LP64-NEXT:    fsw fs6, 104(a1)
+; LP64-NEXT:    fsw fs5, 100(a1)
+; LP64-NEXT:    fsw fs4, 96(a1)
+; LP64-NEXT:    fsw fs3, 92(a1)
+; LP64-NEXT:    fsw fs2, 88(a1)
+; LP64-NEXT:    fsw fs1, 84(a1)
+; LP64-NEXT:    fsw fs0, 80(a1)
+; LP64-NEXT:    fsw ft11, 76(a1)
+; LP64-NEXT:    fsw ft10, 72(a1)
+; LP64-NEXT:    fsw ft9, 68(a1)
+; LP64-NEXT:    fsw ft8, 64(a1)
+; LP64-NEXT:    fsw fa7, 60(a1)
+; LP64-NEXT:    fsw fa6, 56(a1)
+; LP64-NEXT:    fsw fa5, 52(a1)
+; LP64-NEXT:    fsw fa4, 48(a1)
+; LP64-NEXT:    fsw fa3, 44(a1)
+; LP64-NEXT:    fsw fa2, 40(a1)
+; LP64-NEXT:    fsw fa1, 36(a1)
+; LP64-NEXT:    fsw fa0, 32(a1)
+; LP64-NEXT:    fsw ft7, 28(a1)
+; LP64-NEXT:    fsw ft6, 24(a1)
+; LP64-NEXT:    fsw ft5, 20(a1)
+; LP64-NEXT:    fsw ft4, 16(a1)
+; LP64-NEXT:    fsw ft3, %lo(var+12)(a0)
+; LP64-NEXT:    fsw ft2, %lo(var+8)(a0)
+; LP64-NEXT:    fsw ft1, %lo(var+4)(a0)
+; LP64-NEXT:    fsw ft0, %lo(var)(a0)
+; LP64-NEXT:    ret
 ;
-; ILP32D-LP64D-LABEL: callee:
-; ILP32D-LP64D:       # %bb.0:
-; ILP32D-LP64D-NEXT:    addi sp, sp, -96
-; ILP32D-LP64D-NEXT:    fsd fs0, 88(sp)
-; ILP32D-LP64D-NEXT:    fsd fs1, 80(sp)
-; ILP32D-LP64D-NEXT:    fsd fs2, 72(sp)
-; ILP32D-LP64D-NEXT:    fsd fs3, 64(sp)
-; ILP32D-LP64D-NEXT:    fsd fs4, 56(sp)
-; ILP32D-LP64D-NEXT:    fsd fs5, 48(sp)
-; ILP32D-LP64D-NEXT:    fsd fs6, 40(sp)
-; ILP32D-LP64D-NEXT:    fsd fs7, 32(sp)
-; ILP32D-LP64D-NEXT:    fsd fs8, 24(sp)
-; ILP32D-LP64D-NEXT:    fsd fs9, 16(sp)
-; ILP32D-LP64D-NEXT:    fsd fs10, 8(sp)
-; ILP32D-LP64D-NEXT:    fsd fs11, 0(sp)
-; ILP32D-LP64D-NEXT:    lui a0, %hi(var)
-; ILP32D-LP64D-NEXT:    flw ft0, %lo(var)(a0)
-; ILP32D-LP64D-NEXT:    addi a1, a0, %lo(var)
+; ILP32F-LABEL: callee:
+; ILP32F:       # %bb.0:
+; ILP32F-NEXT:    addi sp, sp, -48
+; ILP32F-NEXT:    fsw fs0, 44(sp)
+; ILP32F-NEXT:    fsw fs1, 40(sp)
+; ILP32F-NEXT:    fsw fs2, 36(sp)
+; ILP32F-NEXT:    fsw fs3, 32(sp)
+; ILP32F-NEXT:    fsw fs4, 28(sp)
+; ILP32F-NEXT:    fsw fs5, 24(sp)
+; ILP32F-NEXT:    fsw fs6, 20(sp)
+; ILP32F-NEXT:    fsw fs7, 16(sp)
+; ILP32F-NEXT:    fsw fs8, 12(sp)
+; ILP32F-NEXT:    fsw fs9, 8(sp)
+; ILP32F-NEXT:    fsw fs10, 4(sp)
+; ILP32F-NEXT:    fsw fs11, 0(sp)
+; ILP32F-NEXT:    lui a0, %hi(var)
+; ILP32F-NEXT:    flw ft0, %lo(var)(a0)
+; ILP32F-NEXT:    flw ft1, %lo(var+4)(a0)
+; ILP32F-NEXT:    flw ft2, %lo(var+8)(a0)
+; ILP32F-NEXT:    flw ft3, %lo(var+12)(a0)
+; ILP32F-NEXT:    addi a1, a0, %lo(var)
+; ILP32F-NEXT:    flw ft4, 16(a1)
+; ILP32F-NEXT:    flw ft5, 20(a1)
+; ILP32F-NEXT:    flw ft6, 24(a1)
+; ILP32F-NEXT:    flw ft7, 28(a1)
+; ILP32F-NEXT:    flw fa0, 32(a1)
+; ILP32F-NEXT:    flw fa1, 36(a1)
+; ILP32F-NEXT:    flw fa2, 40(a1)
+; ILP32F-NEXT:    flw fa3, 44(a1)
+; ILP32F-NEXT:    flw fa4, 48(a1)
+; ILP32F-NEXT:    flw fa5, 52(a1)
+; ILP32F-NEXT:    flw fa6, 56(a1)
+; ILP32F-NEXT:    flw fa7, 60(a1)
+; ILP32F-NEXT:    flw ft8, 64(a1)
+; ILP32F-NEXT:    flw ft9, 68(a1)
+; ILP32F-NEXT:    flw ft10, 72(a1)
+; ILP32F-NEXT:    flw ft11, 76(a1)
+; ILP32F-NEXT:    flw fs0, 80(a1)
+; ILP32F-NEXT:    flw fs1, 84(a1)
+; ILP32F-NEXT:    flw fs2, 88(a1)
+; ILP32F-NEXT:    flw fs3, 92(a1)
+; ILP32F-NEXT:    flw fs4, 96(a1)
+; ILP32F-NEXT:    flw fs5, 100(a1)
+; ILP32F-NEXT:    flw fs6, 104(a1)
+; ILP32F-NEXT:    flw fs7, 108(a1)
+; ILP32F-NEXT:    flw fs8, 124(a1)
+; ILP32F-NEXT:    flw fs9, 120(a1)
+; ILP32F-NEXT:    flw fs10, 116(a1)
+; ILP32F-NEXT:    flw fs11, 112(a1)
+; ILP32F-NEXT:    fsw fs8, 124(a1)
+; ILP32F-NEXT:    fsw fs9, 120(a1)
+; ILP32F-NEXT:    fsw fs10, 116(a1)
+; ILP32F-NEXT:    fsw fs11, 112(a1)
+; ILP32F-NEXT:    fsw fs7, 108(a1)
+; ILP32F-NEXT:    fsw fs6, 104(a1)
+; ILP32F-NEXT:    fsw fs5, 100(a1)
+; ILP32F-NEXT:    fsw fs4, 96(a1)
+; ILP32F-NEXT:    fsw fs3, 92(a1)
+; ILP32F-NEXT:    fsw fs2, 88(a1)
+; ILP32F-NEXT:    fsw fs1, 84(a1)
+; ILP32F-NEXT:    fsw fs0, 80(a1)
+; ILP32F-NEXT:    fsw ft11, 76(a1)
+; ILP32F-NEXT:    fsw ft10, 72(a1)
+; ILP32F-NEXT:    fsw ft9, 68(a1)
+; ILP32F-NEXT:    fsw ft8, 64(a1)
+; ILP32F-NEXT:    fsw fa7, 60(a1)
+; ILP32F-NEXT:    fsw fa6, 56(a1)
+; ILP32F-NEXT:    fsw fa5, 52(a1)
+; ILP32F-NEXT:    fsw fa4, 48(a1)
+; ILP32F-NEXT:    fsw fa3, 44(a1)
+; ILP32F-NEXT:    fsw fa2, 40(a1)
+; ILP32F-NEXT:    fsw fa1, 36(a1)
+; ILP32F-NEXT:    fsw fa0, 32(a1)
+; ILP32F-NEXT:    fsw ft7, 28(a1)
+; ILP32F-NEXT:    fsw ft6, 24(a1)
+; ILP32F-NEXT:    fsw ft5, 20(a1)
+; ILP32F-NEXT:    fsw ft4, 16(a1)
+; ILP32F-NEXT:    fsw ft3, %lo(var+12)(a0)
+; ILP32F-NEXT:    fsw ft2, %lo(var+8)(a0)
+; ILP32F-NEXT:    fsw ft1, %lo(var+4)(a0)
+; ILP32F-NEXT:    fsw ft0, %lo(var)(a0)
+; ILP32F-NEXT:    flw fs11, 0(sp)
+; ILP32F-NEXT:    flw fs10, 4(sp)
+; ILP32F-NEXT:    flw fs9, 8(sp)
+; ILP32F-NEXT:    flw fs8, 12(sp)
+; ILP32F-NEXT:    flw fs7, 16(sp)
+; ILP32F-NEXT:    flw fs6, 20(sp)
+; ILP32F-NEXT:    flw fs5, 24(sp)
+; ILP32F-NEXT:    flw fs4, 28(sp)
+; ILP32F-NEXT:    flw fs3, 32(sp)
+; ILP32F-NEXT:    flw fs2, 36(sp)
+; ILP32F-NEXT:    flw fs1, 40(sp)
+; ILP32F-NEXT:    flw fs0, 44(sp)
+; ILP32F-NEXT:    addi sp, sp, 48
+; ILP32F-NEXT:    ret
+;
+; LP64F-LABEL: callee:
+; LP64F:       # %bb.0:
+; LP64F-NEXT:    addi sp, sp, -48
+; LP64F-NEXT:    fsw fs0, 44(sp)
+; LP64F-NEXT:    fsw fs1, 40(sp)
+; LP64F-NEXT:    fsw fs2, 36(sp)
+; LP64F-NEXT:    fsw fs3, 32(sp)
+; LP64F-NEXT:    fsw fs4, 28(sp)
+; LP64F-NEXT:    fsw fs5, 24(sp)
+; LP64F-NEXT:    fsw fs6, 20(sp)
+; LP64F-NEXT:    fsw fs7, 16(sp)
+; LP64F-NEXT:    fsw fs8, 12(sp)
+; LP64F-NEXT:    fsw fs9, 8(sp)
+; LP64F-NEXT:    fsw fs10, 4(sp)
+; LP64F-NEXT:    fsw fs11, 0(sp)
+; LP64F-NEXT:    lui a0, %hi(var)
+; LP64F-NEXT:    flw ft0, %lo(var)(a0)
+; LP64F-NEXT:    flw ft1, %lo(var+4)(a0)
+; LP64F-NEXT:    flw ft2, %lo(var+8)(a0)
+; LP64F-NEXT:    flw ft3, %lo(var+12)(a0)
+; LP64F-NEXT:    addi a1, a0, %lo(var)
+; LP64F-NEXT:    flw ft4, 16(a1)
+; LP64F-NEXT:    flw ft5, 20(a1)
+; LP64F-NEXT:    flw ft6, 24(a1)
+; LP64F-NEXT:    flw ft7, 28(a1)
+; LP64F-NEXT:    flw fa0, 32(a1)
+; LP64F-NEXT:    flw fa1, 36(a1)
+; LP64F-NEXT:    flw fa2, 40(a1)
+; LP64F-NEXT:    flw fa3, 44(a1)
+; LP64F-NEXT:    flw fa4, 48(a1)
+; LP64F-NEXT:    flw fa5, 52(a1)
+; LP64F-NEXT:    flw fa6, 56(a1)
+; LP64F-NEXT:    flw fa7, 60(a1)
+; LP64F-NEXT:    flw ft8, 64(a1)
+; LP64F-NEXT:    flw ft9, 68(a1)
+; LP64F-NEXT:    flw ft10, 72(a1)
+; LP64F-NEXT:    flw ft11, 76(a1)
+; LP64F-NEXT:    flw fs0, 80(a1)
+; LP64F-NEXT:    flw fs1, 84(a1)
+; LP64F-NEXT:    flw fs2, 88(a1)
+; LP64F-NEXT:    flw fs3, 92(a1)
+; LP64F-NEXT:    flw fs4, 96(a1)
+; LP64F-NEXT:    flw fs5, 100(a1)
+; LP64F-NEXT:    flw fs6, 104(a1)
+; LP64F-NEXT:    flw fs7, 108(a1)
+; LP64F-NEXT:    flw fs8, 124(a1)
+; LP64F-NEXT:    flw fs9, 120(a1)
+; LP64F-NEXT:    flw fs10, 116(a1)
+; LP64F-NEXT:    flw fs11, 112(a1)
+; LP64F-NEXT:    fsw fs8, 124(a1)
+; LP64F-NEXT:    fsw fs9, 120(a1)
+; LP64F-NEXT:    fsw fs10, 116(a1)
+; LP64F-NEXT:    fsw fs11, 112(a1)
+; LP64F-NEXT:    fsw fs7, 108(a1)
+; LP64F-NEXT:    fsw fs6, 104(a1)
+; LP64F-NEXT:    fsw fs5, 100(a1)
+; LP64F-NEXT:    fsw fs4, 96(a1)
+; LP64F-NEXT:    fsw fs3, 92(a1)
+; LP64F-NEXT:    fsw fs2, 88(a1)
+; LP64F-NEXT:    fsw fs1, 84(a1)
+; LP64F-NEXT:    fsw fs0, 80(a1)
+; LP64F-NEXT:    fsw ft11, 76(a1)
+; LP64F-NEXT:    fsw ft10, 72(a1)
+; LP64F-NEXT:    fsw ft9, 68(a1)
+; LP64F-NEXT:    fsw ft8, 64(a1)
+; LP64F-NEXT:    fsw fa7, 60(a1)
+; LP64F-NEXT:    fsw fa6, 56(a1)
+; LP64F-NEXT:    fsw fa5, 52(a1)
+; LP64F-NEXT:    fsw fa4, 48(a1)
+; LP64F-NEXT:    fsw fa3, 44(a1)
+; LP64F-NEXT:    fsw fa2, 40(a1)
+; LP64F-NEXT:    fsw fa1, 36(a1)
+; LP64F-NEXT:    fsw fa0, 32(a1)
+; LP64F-NEXT:    fsw ft7, 28(a1)
+; LP64F-NEXT:    fsw ft6, 24(a1)
+; LP64F-NEXT:    fsw ft5, 20(a1)
+; LP64F-NEXT:    fsw ft4, 16(a1)
+; LP64F-NEXT:    fsw ft3, %lo(var+12)(a0)
+; LP64F-NEXT:    fsw ft2, %lo(var+8)(a0)
+; LP64F-NEXT:    fsw ft1, %lo(var+4)(a0)
+; LP64F-NEXT:    fsw ft0, %lo(var)(a0)
+; LP64F-NEXT:    flw fs11, 0(sp)
+; LP64F-NEXT:    flw fs10, 4(sp)
+; LP64F-NEXT:    flw fs9, 8(sp)
+; LP64F-NEXT:    flw fs8, 12(sp)
+; LP64F-NEXT:    flw fs7, 16(sp)
+; LP64F-NEXT:    flw fs6, 20(sp)
+; LP64F-NEXT:    flw fs5, 24(sp)
+; LP64F-NEXT:    flw fs4, 28(sp)
+; LP64F-NEXT:    flw fs3, 32(sp)
+; LP64F-NEXT:    flw fs2, 36(sp)
+; LP64F-NEXT:    flw fs1, 40(sp)
+; LP64F-NEXT:    flw fs0, 44(sp)
+; LP64F-NEXT:    addi sp, sp, 48
+; LP64F-NEXT:    ret
+;
+; ILP32D-LABEL: callee:
+; ILP32D:       # %bb.0:
+; ILP32D-NEXT:    addi sp, sp, -96
+; ILP32D-NEXT:    fsd fs0, 88(sp)
+; ILP32D-NEXT:    fsd fs1, 80(sp)
+; ILP32D-NEXT:    fsd fs2, 72(sp)
+; ILP32D-NEXT:    fsd fs3, 64(sp)
+; ILP32D-NEXT:    fsd fs4, 56(sp)
+; ILP32D-NEXT:    fsd fs5, 48(sp)
+; ILP32D-NEXT:    fsd fs6, 40(sp)
+; ILP32D-NEXT:    fsd fs7, 32(sp)
+; ILP32D-NEXT:    fsd fs8, 24(sp)
+; ILP32D-NEXT:    fsd fs9, 16(sp)
+; ILP32D-NEXT:    fsd fs10, 8(sp)
+; ILP32D-NEXT:    fsd fs11, 0(sp)
+; ILP32D-NEXT:    lui a0, %hi(var)
+; ILP32D-NEXT:    flw ft0, %lo(var)(a0)
+; ILP32D-NEXT:    flw ft1, %lo(var+4)(a0)
+; ILP32D-NEXT:    flw ft2, %lo(var+8)(a0)
+; ILP32D-NEXT:    flw ft3, %lo(var+12)(a0)
+; ILP32D-NEXT:    addi a1, a0, %lo(var)
+; ILP32D-NEXT:    flw ft4, 16(a1)
+; ILP32D-NEXT:    flw ft5, 20(a1)
+; ILP32D-NEXT:    flw ft6, 24(a1)
+; ILP32D-NEXT:    flw ft7, 28(a1)
+; ILP32D-NEXT:    flw fa0, 32(a1)
+; ILP32D-NEXT:    flw fa1, 36(a1)
+; ILP32D-NEXT:    flw fa2, 40(a1)
+; ILP32D-NEXT:    flw fa3, 44(a1)
+; ILP32D-NEXT:    flw fa4, 48(a1)
+; ILP32D-NEXT:    flw fa5, 52(a1)
+; ILP32D-NEXT:    flw fa6, 56(a1)
+; ILP32D-NEXT:    flw fa7, 60(a1)
+; ILP32D-NEXT:    flw ft8, 64(a1)
+; ILP32D-NEXT:    flw ft9, 68(a1)
+; ILP32D-NEXT:    flw ft10, 72(a1)
+; ILP32D-NEXT:    flw ft11, 76(a1)
+; ILP32D-NEXT:    flw fs0, 80(a1)
+; ILP32D-NEXT:    flw fs1, 84(a1)
+; ILP32D-NEXT:    flw fs2, 88(a1)
+; ILP32D-NEXT:    flw fs3, 92(a1)
+; ILP32D-NEXT:    flw fs4, 96(a1)
+; ILP32D-NEXT:    flw fs5, 100(a1)
+; ILP32D-NEXT:    flw fs6, 104(a1)
+; ILP32D-NEXT:    flw fs7, 108(a1)
+; ILP32D-NEXT:    flw fs8, 124(a1)
+; ILP32D-NEXT:    flw fs9, 120(a1)
+; ILP32D-NEXT:    flw fs10, 116(a1)
+; ILP32D-NEXT:    flw fs11, 112(a1)
+; ILP32D-NEXT:    fsw fs8, 124(a1)
+; ILP32D-NEXT:    fsw fs9, 120(a1)
+; ILP32D-NEXT:    fsw fs10, 116(a1)
+; ILP32D-NEXT:    fsw fs11, 112(a1)
+; ILP32D-NEXT:    fsw fs7, 108(a1)
+; ILP32D-NEXT:    fsw fs6, 104(a1)
+; ILP32D-NEXT:    fsw fs5, 100(a1)
+; ILP32D-NEXT:    fsw fs4, 96(a1)
+; ILP32D-NEXT:    fsw fs3, 92(a1)
+; ILP32D-NEXT:    fsw fs2, 88(a1)
+; ILP32D-NEXT:    fsw fs1, 84(a1)
+; ILP32D-NEXT:    fsw fs0, 80(a1)
+; ILP32D-NEXT:    fsw ft11, 76(a1)
+; ILP32D-NEXT:    fsw ft10, 72(a1)
+; ILP32D-NEXT:    fsw ft9, 68(a1)
+; ILP32D-NEXT:    fsw ft8, 64(a1)
+; ILP32D-NEXT:    fsw fa7, 60(a1)
+; ILP32D-NEXT:    fsw fa6, 56(a1)
+; ILP32D-NEXT:    fsw fa5, 52(a1)
+; ILP32D-NEXT:    fsw fa4, 48(a1)
+; ILP32D-NEXT:    fsw fa3, 44(a1)
+; ILP32D-NEXT:    fsw fa2, 40(a1)
+; ILP32D-NEXT:    fsw fa1, 36(a1)
+; ILP32D-NEXT:    fsw fa0, 32(a1)
+; ILP32D-NEXT:    fsw ft7, 28(a1)
+; ILP32D-NEXT:    fsw ft6, 24(a1)
+; ILP32D-NEXT:    fsw ft5, 20(a1)
+; ILP32D-NEXT:    fsw ft4, 16(a1)
+; ILP32D-NEXT:    fsw ft3, %lo(var+12)(a0)
+; ILP32D-NEXT:    fsw ft2, %lo(var+8)(a0)
+; ILP32D-NEXT:    fsw ft1, %lo(var+4)(a0)
+; ILP32D-NEXT:    fsw ft0, %lo(var)(a0)
+; ILP32D-NEXT:    fld fs11, 0(sp)
+; ILP32D-NEXT:    fld fs10, 8(sp)
+; ILP32D-NEXT:    fld fs9, 16(sp)
+; ILP32D-NEXT:    fld fs8, 24(sp)
+; ILP32D-NEXT:    fld fs7, 32(sp)
+; ILP32D-NEXT:    fld fs6, 40(sp)
+; ILP32D-NEXT:    fld fs5, 48(sp)
+; ILP32D-NEXT:    fld fs4, 56(sp)
+; ILP32D-NEXT:    fld fs3, 64(sp)
+; ILP32D-NEXT:    fld fs2, 72(sp)
+; ILP32D-NEXT:    fld fs1, 80(sp)
+; ILP32D-NEXT:    fld fs0, 88(sp)
+; ILP32D-NEXT:    addi sp, sp, 96
+; ILP32D-NEXT:    ret
+;
+; LP64D-LABEL: callee:
+; LP64D:       # %bb.0:
+; LP64D-NEXT:    addi sp, sp, -96
+; LP64D-NEXT:    fsd fs0, 88(sp)
+; LP64D-NEXT:    fsd fs1, 80(sp)
+; LP64D-NEXT:    fsd fs2, 72(sp)
+; LP64D-NEXT:    fsd fs3, 64(sp)
+; LP64D-NEXT:    fsd fs4, 56(sp)
+; LP64D-NEXT:    fsd fs5, 48(sp)
+; LP64D-NEXT:    fsd fs6, 40(sp)
+; LP64D-NEXT:    fsd fs7, 32(sp)
+; LP64D-NEXT:    fsd fs8, 24(sp)
+; LP64D-NEXT:    fsd fs9, 16(sp)
+; LP64D-NEXT:    fsd fs10, 8(sp)
+; LP64D-NEXT:    fsd fs11, 0(sp)
+; LP64D-NEXT:    lui a0, %hi(var)
+; LP64D-NEXT:    flw ft0, %lo(var)(a0)
+; LP64D-NEXT:    flw ft1, %lo(var+4)(a0)
+; LP64D-NEXT:    flw ft2, %lo(var+8)(a0)
+; LP64D-NEXT:    flw ft3, %lo(var+12)(a0)
+; LP64D-NEXT:    addi a1, a0, %lo(var)
+; LP64D-NEXT:    flw ft4, 16(a1)
+; LP64D-NEXT:    flw ft5, 20(a1)
+; LP64D-NEXT:    flw ft6, 24(a1)
+; LP64D-NEXT:    flw ft7, 28(a1)
+; LP64D-NEXT:    flw fa0, 32(a1)
+; LP64D-NEXT:    flw fa1, 36(a1)
+; LP64D-NEXT:    flw fa2, 40(a1)
+; LP64D-NEXT:    flw fa3, 44(a1)
+; LP64D-NEXT:    flw fa4, 48(a1)
+; LP64D-NEXT:    flw fa5, 52(a1)
+; LP64D-NEXT:    flw fa6, 56(a1)
+; LP64D-NEXT:    flw fa7, 60(a1)
+; LP64D-NEXT:    flw ft8, 64(a1)
+; LP64D-NEXT:    flw ft9, 68(a1)
+; LP64D-NEXT:    flw ft10, 72(a1)
+; LP64D-NEXT:    flw ft11, 76(a1)
+; LP64D-NEXT:    flw fs0, 80(a1)
+; LP64D-NEXT:    flw fs1, 84(a1)
+; LP64D-NEXT:    flw fs2, 88(a1)
+; LP64D-NEXT:    flw fs3, 92(a1)
+; LP64D-NEXT:    flw fs4, 96(a1)
+; LP64D-NEXT:    flw fs5, 100(a1)
+; LP64D-NEXT:    flw fs6, 104(a1)
+; LP64D-NEXT:    flw fs7, 108(a1)
+; LP64D-NEXT:    flw fs8, 124(a1)
+; LP64D-NEXT:    flw fs9, 120(a1)
+; LP64D-NEXT:    flw fs10, 116(a1)
+; LP64D-NEXT:    flw fs11, 112(a1)
+; LP64D-NEXT:    fsw fs8, 124(a1)
+; LP64D-NEXT:    fsw fs9, 120(a1)
+; LP64D-NEXT:    fsw fs10, 116(a1)
+; LP64D-NEXT:    fsw fs11, 112(a1)
+; LP64D-NEXT:    fsw fs7, 108(a1)
+; LP64D-NEXT:    fsw fs6, 104(a1)
+; LP64D-NEXT:    fsw fs5, 100(a1)
+; LP64D-NEXT:    fsw fs4, 96(a1)
+; LP64D-NEXT:    fsw fs3, 92(a1)
+; LP64D-NEXT:    fsw fs2, 88(a1)
+; LP64D-NEXT:    fsw fs1, 84(a1)
+; LP64D-NEXT:    fsw fs0, 80(a1)
+; LP64D-NEXT:    fsw ft11, 76(a1)
+; LP64D-NEXT:    fsw ft10, 72(a1)
+; LP64D-NEXT:    fsw ft9, 68(a1)
+; LP64D-NEXT:    fsw ft8, 64(a1)
+; LP64D-NEXT:    fsw fa7, 60(a1)
+; LP64D-NEXT:    fsw fa6, 56(a1)
+; LP64D-NEXT:    fsw fa5, 52(a1)
+; LP64D-NEXT:    fsw fa4, 48(a1)
+; LP64D-NEXT:    fsw fa3, 44(a1)
+; LP64D-NEXT:    fsw fa2, 40(a1)
+; LP64D-NEXT:    fsw fa1, 36(a1)
+; LP64D-NEXT:    fsw fa0, 32(a1)
+; LP64D-NEXT:    fsw ft7, 28(a1)
+; LP64D-NEXT:    fsw ft6, 24(a1)
+; LP64D-NEXT:    fsw ft5, 20(a1)
+; LP64D-NEXT:    fsw ft4, 16(a1)
+; LP64D-NEXT:    fsw ft3, %lo(var+12)(a0)
+; LP64D-NEXT:    fsw ft2, %lo(var+8)(a0)
+; LP64D-NEXT:    fsw ft1, %lo(var+4)(a0)
+; LP64D-NEXT:    fsw ft0, %lo(var)(a0)
+; LP64D-NEXT:    fld fs11, 0(sp)
+; LP64D-NEXT:    fld fs10, 8(sp)
+; LP64D-NEXT:    fld fs9, 16(sp)
+; LP64D-NEXT:    fld fs8, 24(sp)
+; LP64D-NEXT:    fld fs7, 32(sp)
+; LP64D-NEXT:    fld fs6, 40(sp)
+; LP64D-NEXT:    fld fs5, 48(sp)
+; LP64D-NEXT:    fld fs4, 56(sp)
+; LP64D-NEXT:    fld fs3, 64(sp)
+; LP64D-NEXT:    fld fs2, 72(sp)
+; LP64D-NEXT:    fld fs1, 80(sp)
+; LP64D-NEXT:    fld fs0, 88(sp)
+; LP64D-NEXT:    addi sp, sp, 96
+; LP64D-NEXT:    ret
   %val = load [32 x float], [32 x float]* @var
   store volatile [32 x float] %val, [32 x float]* @var
   ret void
@@ -140,71 +557,863 @@ define void @callee() nounwind {
 ; fs0-fs11 are preserved across calls.
 
 define void @caller() nounwind {
-; ILP32-LP64-LABEL: caller:
-; ILP32-LP64-NOT:     ft{{[1-9][0-9]*}}
-; ILP32-LP64-NOT:     fs{{[0-9]+}}
-; ILP32-LP64-NOT:     fa{{[0-9]+}}
-; ILP32-LP64:         call callee
-; ILP32-LP64-NOT:     ft{{[1-9][0-9]*}}
-; ILP32-LP64-NOT:     fs{{[0-9]+}}
-; ILP32-LP64-NOT:     fa{{[0-9]+}}
-; ILP32-LP64:         ret
+; ILP32-LABEL: caller:
+; ILP32:       # %bb.0:
+; ILP32-NEXT:    addi sp, sp, -144
+; ILP32-NEXT:    sw ra, 140(sp)
+; ILP32-NEXT:    sw s0, 136(sp)
+; ILP32-NEXT:    sw s1, 132(sp)
+; ILP32-NEXT:    lui s0, %hi(var)
+; ILP32-NEXT:    flw ft0, %lo(var)(s0)
+; ILP32-NEXT:    fsw ft0, 128(sp)
+; ILP32-NEXT:    flw ft0, %lo(var+4)(s0)
+; ILP32-NEXT:    fsw ft0, 124(sp)
+; ILP32-NEXT:    flw ft0, %lo(var+8)(s0)
+; ILP32-NEXT:    fsw ft0, 120(sp)
+; ILP32-NEXT:    flw ft0, %lo(var+12)(s0)
+; ILP32-NEXT:    fsw ft0, 116(sp)
+; ILP32-NEXT:    addi s1, s0, %lo(var)
+; ILP32-NEXT:    flw ft0, 16(s1)
+; ILP32-NEXT:    fsw ft0, 112(sp)
+; ILP32-NEXT:    flw ft0, 20(s1)
+; ILP32-NEXT:    fsw ft0, 108(sp)
+; ILP32-NEXT:    flw ft0, 24(s1)
+; ILP32-NEXT:    fsw ft0, 104(sp)
+; ILP32-NEXT:    flw ft0, 28(s1)
+; ILP32-NEXT:    fsw ft0, 100(sp)
+; ILP32-NEXT:    flw ft0, 32(s1)
+; ILP32-NEXT:    fsw ft0, 96(sp)
+; ILP32-NEXT:    flw ft0, 36(s1)
+; ILP32-NEXT:    fsw ft0, 92(sp)
+; ILP32-NEXT:    flw ft0, 40(s1)
+; ILP32-NEXT:    fsw ft0, 88(sp)
+; ILP32-NEXT:    flw ft0, 44(s1)
+; ILP32-NEXT:    fsw ft0, 84(sp)
+; ILP32-NEXT:    flw ft0, 48(s1)
+; ILP32-NEXT:    fsw ft0, 80(sp)
+; ILP32-NEXT:    flw ft0, 52(s1)
+; ILP32-NEXT:    fsw ft0, 76(sp)
+; ILP32-NEXT:    flw ft0, 56(s1)
+; ILP32-NEXT:    fsw ft0, 72(sp)
+; ILP32-NEXT:    flw ft0, 60(s1)
+; ILP32-NEXT:    fsw ft0, 68(sp)
+; ILP32-NEXT:    flw ft0, 64(s1)
+; ILP32-NEXT:    fsw ft0, 64(sp)
+; ILP32-NEXT:    flw ft0, 68(s1)
+; ILP32-NEXT:    fsw ft0, 60(sp)
+; ILP32-NEXT:    flw ft0, 72(s1)
+; ILP32-NEXT:    fsw ft0, 56(sp)
+; ILP32-NEXT:    flw ft0, 76(s1)
+; ILP32-NEXT:    fsw ft0, 52(sp)
+; ILP32-NEXT:    flw ft0, 80(s1)
+; ILP32-NEXT:    fsw ft0, 48(sp)
+; ILP32-NEXT:    flw ft0, 84(s1)
+; ILP32-NEXT:    fsw ft0, 44(sp)
+; ILP32-NEXT:    flw ft0, 88(s1)
+; ILP32-NEXT:    fsw ft0, 40(sp)
+; ILP32-NEXT:    flw ft0, 92(s1)
+; ILP32-NEXT:    fsw ft0, 36(sp)
+; ILP32-NEXT:    flw ft0, 96(s1)
+; ILP32-NEXT:    fsw ft0, 32(sp)
+; ILP32-NEXT:    flw ft0, 100(s1)
+; ILP32-NEXT:    fsw ft0, 28(sp)
+; ILP32-NEXT:    flw ft0, 104(s1)
+; ILP32-NEXT:    fsw ft0, 24(sp)
+; ILP32-NEXT:    flw ft0, 108(s1)
+; ILP32-NEXT:    fsw ft0, 20(sp)
+; ILP32-NEXT:    flw ft0, 112(s1)
+; ILP32-NEXT:    fsw ft0, 16(sp)
+; ILP32-NEXT:    flw ft0, 116(s1)
+; ILP32-NEXT:    fsw ft0, 12(sp)
+; ILP32-NEXT:    flw ft0, 120(s1)
+; ILP32-NEXT:    fsw ft0, 8(sp)
+; ILP32-NEXT:    flw ft0, 124(s1)
+; ILP32-NEXT:    fsw ft0, 4(sp)
+; ILP32-NEXT:    call callee
+; ILP32-NEXT:    flw ft0, 4(sp)
+; ILP32-NEXT:    fsw ft0, 124(s1)
+; ILP32-NEXT:    flw ft0, 8(sp)
+; ILP32-NEXT:    fsw ft0, 120(s1)
+; ILP32-NEXT:    flw ft0, 12(sp)
+; ILP32-NEXT:    fsw ft0, 116(s1)
+; ILP32-NEXT:    flw ft0, 16(sp)
+; ILP32-NEXT:    fsw ft0, 112(s1)
+; ILP32-NEXT:    flw ft0, 20(sp)
+; ILP32-NEXT:    fsw ft0, 108(s1)
+; ILP32-NEXT:    flw ft0, 24(sp)
+; ILP32-NEXT:    fsw ft0, 104(s1)
+; ILP32-NEXT:    flw ft0, 28(sp)
+; ILP32-NEXT:    fsw ft0, 100(s1)
+; ILP32-NEXT:    flw ft0, 32(sp)
+; ILP32-NEXT:    fsw ft0, 96(s1)
+; ILP32-NEXT:    flw ft0, 36(sp)
+; ILP32-NEXT:    fsw ft0, 92(s1)
+; ILP32-NEXT:    flw ft0, 40(sp)
+; ILP32-NEXT:    fsw ft0, 88(s1)
+; ILP32-NEXT:    flw ft0, 44(sp)
+; ILP32-NEXT:    fsw ft0, 84(s1)
+; ILP32-NEXT:    flw ft0, 48(sp)
+; ILP32-NEXT:    fsw ft0, 80(s1)
+; ILP32-NEXT:    flw ft0, 52(sp)
+; ILP32-NEXT:    fsw ft0, 76(s1)
+; ILP32-NEXT:    flw ft0, 56(sp)
+; ILP32-NEXT:    fsw ft0, 72(s1)
+; ILP32-NEXT:    flw ft0, 60(sp)
+; ILP32-NEXT:    fsw ft0, 68(s1)
+; ILP32-NEXT:    flw ft0, 64(sp)
+; ILP32-NEXT:    fsw ft0, 64(s1)
+; ILP32-NEXT:    flw ft0, 68(sp)
+; ILP32-NEXT:    fsw ft0, 60(s1)
+; ILP32-NEXT:    flw ft0, 72(sp)
+; ILP32-NEXT:    fsw ft0, 56(s1)
+; ILP32-NEXT:    flw ft0, 76(sp)
+; ILP32-NEXT:    fsw ft0, 52(s1)
+; ILP32-NEXT:    flw ft0, 80(sp)
+; ILP32-NEXT:    fsw ft0, 48(s1)
+; ILP32-NEXT:    flw ft0, 84(sp)
+; ILP32-NEXT:    fsw ft0, 44(s1)
+; ILP32-NEXT:    flw ft0, 88(sp)
+; ILP32-NEXT:    fsw ft0, 40(s1)
+; ILP32-NEXT:    flw ft0, 92(sp)
+; ILP32-NEXT:    fsw ft0, 36(s1)
+; ILP32-NEXT:    flw ft0, 96(sp)
+; ILP32-NEXT:    fsw ft0, 32(s1)
+; ILP32-NEXT:    flw ft0, 100(sp)
+; ILP32-NEXT:    fsw ft0, 28(s1)
+; ILP32-NEXT:    flw ft0, 104(sp)
+; ILP32-NEXT:    fsw ft0, 24(s1)
+; ILP32-NEXT:    flw ft0, 108(sp)
+; ILP32-NEXT:    fsw ft0, 20(s1)
+; ILP32-NEXT:    flw ft0, 112(sp)
+; ILP32-NEXT:    fsw ft0, 16(s1)
+; ILP32-NEXT:    flw ft0, 116(sp)
+; ILP32-NEXT:    fsw ft0, %lo(var+12)(s0)
+; ILP32-NEXT:    flw ft0, 120(sp)
+; ILP32-NEXT:    fsw ft0, %lo(var+8)(s0)
+; ILP32-NEXT:    flw ft0, 124(sp)
+; ILP32-NEXT:    fsw ft0, %lo(var+4)(s0)
+; ILP32-NEXT:    flw ft0, 128(sp)
+; ILP32-NEXT:    fsw ft0, %lo(var)(s0)
+; ILP32-NEXT:    lw s1, 132(sp)
+; ILP32-NEXT:    lw s0, 136(sp)
+; ILP32-NEXT:    lw ra, 140(sp)
+; ILP32-NEXT:    addi sp, sp, 144
+; ILP32-NEXT:    ret
+;
+; LP64-LABEL: caller:
+; LP64:       # %bb.0:
+; LP64-NEXT:    addi sp, sp, -160
+; LP64-NEXT:    sd ra, 152(sp)
+; LP64-NEXT:    sd s0, 144(sp)
+; LP64-NEXT:    sd s1, 136(sp)
+; LP64-NEXT:    lui s0, %hi(var)
+; LP64-NEXT:    flw ft0, %lo(var)(s0)
+; LP64-NEXT:    fsw ft0, 132(sp)
+; LP64-NEXT:    flw ft0, %lo(var+4)(s0)
+; LP64-NEXT:    fsw ft0, 128(sp)
+; LP64-NEXT:    flw ft0, %lo(var+8)(s0)
+; LP64-NEXT:    fsw ft0, 124(sp)
+; LP64-NEXT:    flw ft0, %lo(var+12)(s0)
+; LP64-NEXT:    fsw ft0, 120(sp)
+; LP64-NEXT:    addi s1, s0, %lo(var)
+; LP64-NEXT:    flw ft0, 16(s1)
+; LP64-NEXT:    fsw ft0, 116(sp)
+; LP64-NEXT:    flw ft0, 20(s1)
+; LP64-NEXT:    fsw ft0, 112(sp)
+; LP64-NEXT:    flw ft0, 24(s1)
+; LP64-NEXT:    fsw ft0, 108(sp)
+; LP64-NEXT:    flw ft0, 28(s1)
+; LP64-NEXT:    fsw ft0, 104(sp)
+; LP64-NEXT:    flw ft0, 32(s1)
+; LP64-NEXT:    fsw ft0, 100(sp)
+; LP64-NEXT:    flw ft0, 36(s1)
+; LP64-NEXT:    fsw ft0, 96(sp)
+; LP64-NEXT:    flw ft0, 40(s1)
+; LP64-NEXT:    fsw ft0, 92(sp)
+; LP64-NEXT:    flw ft0, 44(s1)
+; LP64-NEXT:    fsw ft0, 88(sp)
+; LP64-NEXT:    flw ft0, 48(s1)
+; LP64-NEXT:    fsw ft0, 84(sp)
+; LP64-NEXT:    flw ft0, 52(s1)
+; LP64-NEXT:    fsw ft0, 80(sp)
+; LP64-NEXT:    flw ft0, 56(s1)
+; LP64-NEXT:    fsw ft0, 76(sp)
+; LP64-NEXT:    flw ft0, 60(s1)
+; LP64-NEXT:    fsw ft0, 72(sp)
+; LP64-NEXT:    flw ft0, 64(s1)
+; LP64-NEXT:    fsw ft0, 68(sp)
+; LP64-NEXT:    flw ft0, 68(s1)
+; LP64-NEXT:    fsw ft0, 64(sp)
+; LP64-NEXT:    flw ft0, 72(s1)
+; LP64-NEXT:    fsw ft0, 60(sp)
+; LP64-NEXT:    flw ft0, 76(s1)
+; LP64-NEXT:    fsw ft0, 56(sp)
+; LP64-NEXT:    flw ft0, 80(s1)
+; LP64-NEXT:    fsw ft0, 52(sp)
+; LP64-NEXT:    flw ft0, 84(s1)
+; LP64-NEXT:    fsw ft0, 48(sp)
+; LP64-NEXT:    flw ft0, 88(s1)
+; LP64-NEXT:    fsw ft0, 44(sp)
+; LP64-NEXT:    flw ft0, 92(s1)
+; LP64-NEXT:    fsw ft0, 40(sp)
+; LP64-NEXT:    flw ft0, 96(s1)
+; LP64-NEXT:    fsw ft0, 36(sp)
+; LP64-NEXT:    flw ft0, 100(s1)
+; LP64-NEXT:    fsw ft0, 32(sp)
+; LP64-NEXT:    flw ft0, 104(s1)
+; LP64-NEXT:    fsw ft0, 28(sp)
+; LP64-NEXT:    flw ft0, 108(s1)
+; LP64-NEXT:    fsw ft0, 24(sp)
+; LP64-NEXT:    flw ft0, 112(s1)
+; LP64-NEXT:    fsw ft0, 20(sp)
+; LP64-NEXT:    flw ft0, 116(s1)
+; LP64-NEXT:    fsw ft0, 16(sp)
+; LP64-NEXT:    flw ft0, 120(s1)
+; LP64-NEXT:    fsw ft0, 12(sp)
+; LP64-NEXT:    flw ft0, 124(s1)
+; LP64-NEXT:    fsw ft0, 8(sp)
+; LP64-NEXT:    call callee
+; LP64-NEXT:    flw ft0, 8(sp)
+; LP64-NEXT:    fsw ft0, 124(s1)
+; LP64-NEXT:    flw ft0, 12(sp)
+; LP64-NEXT:    fsw ft0, 120(s1)
+; LP64-NEXT:    flw ft0, 16(sp)
+; LP64-NEXT:    fsw ft0, 116(s1)
+; LP64-NEXT:    flw ft0, 20(sp)
+; LP64-NEXT:    fsw ft0, 112(s1)
+; LP64-NEXT:    flw ft0, 24(sp)
+; LP64-NEXT:    fsw ft0, 108(s1)
+; LP64-NEXT:    flw ft0, 28(sp)
+; LP64-NEXT:    fsw ft0, 104(s1)
+; LP64-NEXT:    flw ft0, 32(sp)
+; LP64-NEXT:    fsw ft0, 100(s1)
+; LP64-NEXT:    flw ft0, 36(sp)
+; LP64-NEXT:    fsw ft0, 96(s1)
+; LP64-NEXT:    flw ft0, 40(sp)
+; LP64-NEXT:    fsw ft0, 92(s1)
+; LP64-NEXT:    flw ft0, 44(sp)
+; LP64-NEXT:    fsw ft0, 88(s1)
+; LP64-NEXT:    flw ft0, 48(sp)
+; LP64-NEXT:    fsw ft0, 84(s1)
+; LP64-NEXT:    flw ft0, 52(sp)
+; LP64-NEXT:    fsw ft0, 80(s1)
+; LP64-NEXT:    flw ft0, 56(sp)
+; LP64-NEXT:    fsw ft0, 76(s1)
+; LP64-NEXT:    flw ft0, 60(sp)
+; LP64-NEXT:    fsw ft0, 72(s1)
+; LP64-NEXT:    flw ft0, 64(sp)
+; LP64-NEXT:    fsw ft0, 68(s1)
+; LP64-NEXT:    flw ft0, 68(sp)
+; LP64-NEXT:    fsw ft0, 64(s1)
+; LP64-NEXT:    flw ft0, 72(sp)
+; LP64-NEXT:    fsw ft0, 60(s1)
+; LP64-NEXT:    flw ft0, 76(sp)
+; LP64-NEXT:    fsw ft0, 56(s1)
+; LP64-NEXT:    flw ft0, 80(sp)
+; LP64-NEXT:    fsw ft0, 52(s1)
+; LP64-NEXT:    flw ft0, 84(sp)
+; LP64-NEXT:    fsw ft0, 48(s1)
+; LP64-NEXT:    flw ft0, 88(sp)
+; LP64-NEXT:    fsw ft0, 44(s1)
+; LP64-NEXT:    flw ft0, 92(sp)
+; LP64-NEXT:    fsw ft0, 40(s1)
+; LP64-NEXT:    flw ft0, 96(sp)
+; LP64-NEXT:    fsw ft0, 36(s1)
+; LP64-NEXT:    flw ft0, 100(sp)
+; LP64-NEXT:    fsw ft0, 32(s1)
+; LP64-NEXT:    flw ft0, 104(sp)
+; LP64-NEXT:    fsw ft0, 28(s1)
+; LP64-NEXT:    flw ft0, 108(sp)
+; LP64-NEXT:    fsw ft0, 24(s1)
+; LP64-NEXT:    flw ft0, 112(sp)
+; LP64-NEXT:    fsw ft0, 20(s1)
+; LP64-NEXT:    flw ft0, 116(sp)
+; LP64-NEXT:    fsw ft0, 16(s1)
+; LP64-NEXT:    flw ft0, 120(sp)
+; LP64-NEXT:    fsw ft0, %lo(var+12)(s0)
+; LP64-NEXT:    flw ft0, 124(sp)
+; LP64-NEXT:    fsw ft0, %lo(var+8)(s0)
+; LP64-NEXT:    flw ft0, 128(sp)
+; LP64-NEXT:    fsw ft0, %lo(var+4)(s0)
+; LP64-NEXT:    flw ft0, 132(sp)
+; LP64-NEXT:    fsw ft0, %lo(var)(s0)
+; LP64-NEXT:    ld s1, 136(sp)
+; LP64-NEXT:    ld s0, 144(sp)
+; LP64-NEXT:    ld ra, 152(sp)
+; LP64-NEXT:    addi sp, sp, 160
+; LP64-NEXT:    ret
+;
+; ILP32F-LABEL: caller:
+; ILP32F:       # %bb.0:
+; ILP32F-NEXT:    addi sp, sp, -144
+; ILP32F-NEXT:    sw ra, 140(sp)
+; ILP32F-NEXT:    sw s0, 136(sp)
+; ILP32F-NEXT:    sw s1, 132(sp)
+; ILP32F-NEXT:    fsw fs0, 128(sp)
+; ILP32F-NEXT:    fsw fs1, 124(sp)
+; ILP32F-NEXT:    fsw fs2, 120(sp)
+; ILP32F-NEXT:    fsw fs3, 116(sp)
+; ILP32F-NEXT:    fsw fs4, 112(sp)
+; ILP32F-NEXT:    fsw fs5, 108(sp)
+; ILP32F-NEXT:    fsw fs6, 104(sp)
+; ILP32F-NEXT:    fsw fs7, 100(sp)
+; ILP32F-NEXT:    fsw fs8, 96(sp)
+; ILP32F-NEXT:    fsw fs9, 92(sp)
+; ILP32F-NEXT:    fsw fs10, 88(sp)
+; ILP32F-NEXT:    fsw fs11, 84(sp)
+; ILP32F-NEXT:    lui s0, %hi(var)
+; ILP32F-NEXT:    flw ft0, %lo(var)(s0)
+; ILP32F-NEXT:    fsw ft0, 80(sp)
+; ILP32F-NEXT:    flw ft0, %lo(var+4)(s0)
+; ILP32F-NEXT:    fsw ft0, 76(sp)
+; ILP32F-NEXT:    flw ft0, %lo(var+8)(s0)
+; ILP32F-NEXT:    fsw ft0, 72(sp)
+; ILP32F-NEXT:    flw ft0, %lo(var+12)(s0)
+; ILP32F-NEXT:    fsw ft0, 68(sp)
+; ILP32F-NEXT:    addi s1, s0, %lo(var)
+; ILP32F-NEXT:    flw ft0, 16(s1)
+; ILP32F-NEXT:    fsw ft0, 64(sp)
+; ILP32F-NEXT:    flw ft0, 20(s1)
+; ILP32F-NEXT:    fsw ft0, 60(sp)
+; ILP32F-NEXT:    flw ft0, 24(s1)
+; ILP32F-NEXT:    fsw ft0, 56(sp)
+; ILP32F-NEXT:    flw ft0, 28(s1)
+; ILP32F-NEXT:    fsw ft0, 52(sp)
+; ILP32F-NEXT:    flw ft0, 32(s1)
+; ILP32F-NEXT:    fsw ft0, 48(sp)
+; ILP32F-NEXT:    flw ft0, 36(s1)
+; ILP32F-NEXT:    fsw ft0, 44(sp)
+; ILP32F-NEXT:    flw ft0, 40(s1)
+; ILP32F-NEXT:    fsw ft0, 40(sp)
+; ILP32F-NEXT:    flw ft0, 44(s1)
+; ILP32F-NEXT:    fsw ft0, 36(sp)
+; ILP32F-NEXT:    flw ft0, 48(s1)
+; ILP32F-NEXT:    fsw ft0, 32(sp)
+; ILP32F-NEXT:    flw ft0, 52(s1)
+; ILP32F-NEXT:    fsw ft0, 28(sp)
+; ILP32F-NEXT:    flw ft0, 56(s1)
+; ILP32F-NEXT:    fsw ft0, 24(sp)
+; ILP32F-NEXT:    flw ft0, 60(s1)
+; ILP32F-NEXT:    fsw ft0, 20(sp)
+; ILP32F-NEXT:    flw ft0, 64(s1)
+; ILP32F-NEXT:    fsw ft0, 16(sp)
+; ILP32F-NEXT:    flw ft0, 68(s1)
+; ILP32F-NEXT:    fsw ft0, 12(sp)
+; ILP32F-NEXT:    flw ft0, 72(s1)
+; ILP32F-NEXT:    fsw ft0, 8(sp)
+; ILP32F-NEXT:    flw ft0, 76(s1)
+; ILP32F-NEXT:    fsw ft0, 4(sp)
+; ILP32F-NEXT:    flw fs8, 80(s1)
+; ILP32F-NEXT:    flw fs9, 84(s1)
+; ILP32F-NEXT:    flw fs10, 88(s1)
+; ILP32F-NEXT:    flw fs11, 92(s1)
+; ILP32F-NEXT:    flw fs0, 96(s1)
+; ILP32F-NEXT:    flw fs1, 100(s1)
+; ILP32F-NEXT:    flw fs2, 104(s1)
+; ILP32F-NEXT:    flw fs3, 108(s1)
+; ILP32F-NEXT:    flw fs4, 112(s1)
+; ILP32F-NEXT:    flw fs5, 116(s1)
+; ILP32F-NEXT:    flw fs6, 120(s1)
+; ILP32F-NEXT:    flw fs7, 124(s1)
+; ILP32F-NEXT:    call callee
+; ILP32F-NEXT:    fsw fs7, 124(s1)
+; ILP32F-NEXT:    fsw fs6, 120(s1)
+; ILP32F-NEXT:    fsw fs5, 116(s1)
+; ILP32F-NEXT:    fsw fs4, 112(s1)
+; ILP32F-NEXT:    fsw fs3, 108(s1)
+; ILP32F-NEXT:    fsw fs2, 104(s1)
+; ILP32F-NEXT:    fsw fs1, 100(s1)
+; ILP32F-NEXT:    fsw fs0, 96(s1)
+; ILP32F-NEXT:    fsw fs11, 92(s1)
+; ILP32F-NEXT:    fsw fs10, 88(s1)
+; ILP32F-NEXT:    fsw fs9, 84(s1)
+; ILP32F-NEXT:    fsw fs8, 80(s1)
+; ILP32F-NEXT:    flw ft0, 4(sp)
+; ILP32F-NEXT:    fsw ft0, 76(s1)
+; ILP32F-NEXT:    flw ft0, 8(sp)
+; ILP32F-NEXT:    fsw ft0, 72(s1)
+; ILP32F-NEXT:    flw ft0, 12(sp)
+; ILP32F-NEXT:    fsw ft0, 68(s1)
+; ILP32F-NEXT:    flw ft0, 16(sp)
+; ILP32F-NEXT:    fsw ft0, 64(s1)
+; ILP32F-NEXT:    flw ft0, 20(sp)
+; ILP32F-NEXT:    fsw ft0, 60(s1)
+; ILP32F-NEXT:    flw ft0, 24(sp)
+; ILP32F-NEXT:    fsw ft0, 56(s1)
+; ILP32F-NEXT:    flw ft0, 28(sp)
+; ILP32F-NEXT:    fsw ft0, 52(s1)
+; ILP32F-NEXT:    flw ft0, 32(sp)
+; ILP32F-NEXT:    fsw ft0, 48(s1)
+; ILP32F-NEXT:    flw ft0, 36(sp)
+; ILP32F-NEXT:    fsw ft0, 44(s1)
+; ILP32F-NEXT:    flw ft0, 40(sp)
+; ILP32F-NEXT:    fsw ft0, 40(s1)
+; ILP32F-NEXT:    flw ft0, 44(sp)
+; ILP32F-NEXT:    fsw ft0, 36(s1)
+; ILP32F-NEXT:    flw ft0, 48(sp)
+; ILP32F-NEXT:    fsw ft0, 32(s1)
+; ILP32F-NEXT:    flw ft0, 52(sp)
+; ILP32F-NEXT:    fsw ft0, 28(s1)
+; ILP32F-NEXT:    flw ft0, 56(sp)
+; ILP32F-NEXT:    fsw ft0, 24(s1)
+; ILP32F-NEXT:    flw ft0, 60(sp)
+; ILP32F-NEXT:    fsw ft0, 20(s1)
+; ILP32F-NEXT:    flw ft0, 64(sp)
+; ILP32F-NEXT:    fsw ft0, 16(s1)
+; ILP32F-NEXT:    flw ft0, 68(sp)
+; ILP32F-NEXT:    fsw ft0, %lo(var+12)(s0)
+; ILP32F-NEXT:    flw ft0, 72(sp)
+; ILP32F-NEXT:    fsw ft0, %lo(var+8)(s0)
+; ILP32F-NEXT:    flw ft0, 76(sp)
+; ILP32F-NEXT:    fsw ft0, %lo(var+4)(s0)
+; ILP32F-NEXT:    flw ft0, 80(sp)
+; ILP32F-NEXT:    fsw ft0, %lo(var)(s0)
+; ILP32F-NEXT:    flw fs11, 84(sp)
+; ILP32F-NEXT:    flw fs10, 88(sp)
+; ILP32F-NEXT:    flw fs9, 92(sp)
+; ILP32F-NEXT:    flw fs8, 96(sp)
+; ILP32F-NEXT:    flw fs7, 100(sp)
+; ILP32F-NEXT:    flw fs6, 104(sp)
+; ILP32F-NEXT:    flw fs5, 108(sp)
+; ILP32F-NEXT:    flw fs4, 112(sp)
+; ILP32F-NEXT:    flw fs3, 116(sp)
+; ILP32F-NEXT:    flw fs2, 120(sp)
+; ILP32F-NEXT:    flw fs1, 124(sp)
+; ILP32F-NEXT:    flw fs0, 128(sp)
+; ILP32F-NEXT:    lw s1, 132(sp)
+; ILP32F-NEXT:    lw s0, 136(sp)
+; ILP32F-NEXT:    lw ra, 140(sp)
+; ILP32F-NEXT:    addi sp, sp, 144
+; ILP32F-NEXT:    ret
+;
+; LP64F-LABEL: caller:
+; LP64F:       # %bb.0:
+; LP64F-NEXT:    addi sp, sp, -160
+; LP64F-NEXT:    sd ra, 152(sp)
+; LP64F-NEXT:    sd s0, 144(sp)
+; LP64F-NEXT:    sd s1, 136(sp)
+; LP64F-NEXT:    fsw fs0, 132(sp)
+; LP64F-NEXT:    fsw fs1, 128(sp)
+; LP64F-NEXT:    fsw fs2, 124(sp)
+; LP64F-NEXT:    fsw fs3, 120(sp)
+; LP64F-NEXT:    fsw fs4, 116(sp)
+; LP64F-NEXT:    fsw fs5, 112(sp)
+; LP64F-NEXT:    fsw fs6, 108(sp)
+; LP64F-NEXT:    fsw fs7, 104(sp)
+; LP64F-NEXT:    fsw fs8, 100(sp)
+; LP64F-NEXT:    fsw fs9, 96(sp)
+; LP64F-NEXT:    fsw fs10, 92(sp)
+; LP64F-NEXT:    fsw fs11, 88(sp)
+; LP64F-NEXT:    lui s0, %hi(var)
+; LP64F-NEXT:    flw ft0, %lo(var)(s0)
+; LP64F-NEXT:    fsw ft0, 84(sp)
+; LP64F-NEXT:    flw ft0, %lo(var+4)(s0)
+; LP64F-NEXT:    fsw ft0, 80(sp)
+; LP64F-NEXT:    flw ft0, %lo(var+8)(s0)
+; LP64F-NEXT:    fsw ft0, 76(sp)
+; LP64F-NEXT:    flw ft0, %lo(var+12)(s0)
+; LP64F-NEXT:    fsw ft0, 72(sp)
+; LP64F-NEXT:    addi s1, s0, %lo(var)
+; LP64F-NEXT:    flw ft0, 16(s1)
+; LP64F-NEXT:    fsw ft0, 68(sp)
+; LP64F-NEXT:    flw ft0, 20(s1)
+; LP64F-NEXT:    fsw ft0, 64(sp)
+; LP64F-NEXT:    flw ft0, 24(s1)
+; LP64F-NEXT:    fsw ft0, 60(sp)
+; LP64F-NEXT:    flw ft0, 28(s1)
+; LP64F-NEXT:    fsw ft0, 56(sp)
+; LP64F-NEXT:    flw ft0, 32(s1)
+; LP64F-NEXT:    fsw ft0, 52(sp)
+; LP64F-NEXT:    flw ft0, 36(s1)
+; LP64F-NEXT:    fsw ft0, 48(sp)
+; LP64F-NEXT:    flw ft0, 40(s1)
+; LP64F-NEXT:    fsw ft0, 44(sp)
+; LP64F-NEXT:    flw ft0, 44(s1)
+; LP64F-NEXT:    fsw ft0, 40(sp)
+; LP64F-NEXT:    flw ft0, 48(s1)
+; LP64F-NEXT:    fsw ft0, 36(sp)
+; LP64F-NEXT:    flw ft0, 52(s1)
+; LP64F-NEXT:    fsw ft0, 32(sp)
+; LP64F-NEXT:    flw ft0, 56(s1)
+; LP64F-NEXT:    fsw ft0, 28(sp)
+; LP64F-NEXT:    flw ft0, 60(s1)
+; LP64F-NEXT:    fsw ft0, 24(sp)
+; LP64F-NEXT:    flw ft0, 64(s1)
+; LP64F-NEXT:    fsw ft0, 20(sp)
+; LP64F-NEXT:    flw ft0, 68(s1)
+; LP64F-NEXT:    fsw ft0, 16(sp)
+; LP64F-NEXT:    flw ft0, 72(s1)
+; LP64F-NEXT:    fsw ft0, 12(sp)
+; LP64F-NEXT:    flw ft0, 76(s1)
+; LP64F-NEXT:    fsw ft0, 8(sp)
+; LP64F-NEXT:    flw fs8, 80(s1)
+; LP64F-NEXT:    flw fs9, 84(s1)
+; LP64F-NEXT:    flw fs10, 88(s1)
+; LP64F-NEXT:    flw fs11, 92(s1)
+; LP64F-NEXT:    flw fs0, 96(s1)
+; LP64F-NEXT:    flw fs1, 100(s1)
+; LP64F-NEXT:    flw fs2, 104(s1)
+; LP64F-NEXT:    flw fs3, 108(s1)
+; LP64F-NEXT:    flw fs4, 112(s1)
+; LP64F-NEXT:    flw fs5, 116(s1)
+; LP64F-NEXT:    flw fs6, 120(s1)
+; LP64F-NEXT:    flw fs7, 124(s1)
+; LP64F-NEXT:    call callee
+; LP64F-NEXT:    fsw fs7, 124(s1)
+; LP64F-NEXT:    fsw fs6, 120(s1)
+; LP64F-NEXT:    fsw fs5, 116(s1)
+; LP64F-NEXT:    fsw fs4, 112(s1)
+; LP64F-NEXT:    fsw fs3, 108(s1)
+; LP64F-NEXT:    fsw fs2, 104(s1)
+; LP64F-NEXT:    fsw fs1, 100(s1)
+; LP64F-NEXT:    fsw fs0, 96(s1)
+; LP64F-NEXT:    fsw fs11, 92(s1)
+; LP64F-NEXT:    fsw fs10, 88(s1)
+; LP64F-NEXT:    fsw fs9, 84(s1)
+; LP64F-NEXT:    fsw fs8, 80(s1)
+; LP64F-NEXT:    flw ft0, 8(sp)
+; LP64F-NEXT:    fsw ft0, 76(s1)
+; LP64F-NEXT:    flw ft0, 12(sp)
+; LP64F-NEXT:    fsw ft0, 72(s1)
+; LP64F-NEXT:    flw ft0, 16(sp)
+; LP64F-NEXT:    fsw ft0, 68(s1)
+; LP64F-NEXT:    flw ft0, 20(sp)
+; LP64F-NEXT:    fsw ft0, 64(s1)
+; LP64F-NEXT:    flw ft0, 24(sp)
+; LP64F-NEXT:    fsw ft0, 60(s1)
+; LP64F-NEXT:    flw ft0, 28(sp)
+; LP64F-NEXT:    fsw ft0, 56(s1)
+; LP64F-NEXT:    flw ft0, 32(sp)
+; LP64F-NEXT:    fsw ft0, 52(s1)
+; LP64F-NEXT:    flw ft0, 36(sp)
+; LP64F-NEXT:    fsw ft0, 48(s1)
+; LP64F-NEXT:    flw ft0, 40(sp)
+; LP64F-NEXT:    fsw ft0, 44(s1)
+; LP64F-NEXT:    flw ft0, 44(sp)
+; LP64F-NEXT:    fsw ft0, 40(s1)
+; LP64F-NEXT:    flw ft0, 48(sp)
+; LP64F-NEXT:    fsw ft0, 36(s1)
+; LP64F-NEXT:    flw ft0, 52(sp)
+; LP64F-NEXT:    fsw ft0, 32(s1)
+; LP64F-NEXT:    flw ft0, 56(sp)
+; LP64F-NEXT:    fsw ft0, 28(s1)
+; LP64F-NEXT:    flw ft0, 60(sp)
+; LP64F-NEXT:    fsw ft0, 24(s1)
+; LP64F-NEXT:    flw ft0, 64(sp)
+; LP64F-NEXT:    fsw ft0, 20(s1)
+; LP64F-NEXT:    flw ft0, 68(sp)
+; LP64F-NEXT:    fsw ft0, 16(s1)
+; LP64F-NEXT:    flw ft0, 72(sp)
+; LP64F-NEXT:    fsw ft0, %lo(var+12)(s0)
+; LP64F-NEXT:    flw ft0, 76(sp)
+; LP64F-NEXT:    fsw ft0, %lo(var+8)(s0)
+; LP64F-NEXT:    flw ft0, 80(sp)
+; LP64F-NEXT:    fsw ft0, %lo(var+4)(s0)
+; LP64F-NEXT:    flw ft0, 84(sp)
+; LP64F-NEXT:    fsw ft0, %lo(var)(s0)
+; LP64F-NEXT:    flw fs11, 88(sp)
+; LP64F-NEXT:    flw fs10, 92(sp)
+; LP64F-NEXT:    flw fs9, 96(sp)
+; LP64F-NEXT:    flw fs8, 100(sp)
+; LP64F-NEXT:    flw fs7, 104(sp)
+; LP64F-NEXT:    flw fs6, 108(sp)
+; LP64F-NEXT:    flw fs5, 112(sp)
+; LP64F-NEXT:    flw fs4, 116(sp)
+; LP64F-NEXT:    flw fs3, 120(sp)
+; LP64F-NEXT:    flw fs2, 124(sp)
+; LP64F-NEXT:    flw fs1, 128(sp)
+; LP64F-NEXT:    flw fs0, 132(sp)
+; LP64F-NEXT:    ld s1, 136(sp)
+; LP64F-NEXT:    ld s0, 144(sp)
+; LP64F-NEXT:    ld ra, 152(sp)
+; LP64F-NEXT:    addi sp, sp, 160
+; LP64F-NEXT:    ret
 ;
-; ILP32F-LP64F-LABEL: caller:
-; ILP32F-LP64F:       flw fs8, 80(s1)
-; ILP32F-LP64F-NEXT:  flw fs9, 84(s1)
-; ILP32F-LP64F-NEXT:  flw fs10, 88(s1)
-; ILP32F-LP64F-NEXT:  flw fs11, 92(s1)
-; ILP32F-LP64F-NEXT:  flw fs0, 96(s1)
-; ILP32F-LP64F-NEXT:  flw fs1, 100(s1)
-; ILP32F-LP64F-NEXT:  flw fs2, 104(s1)
-; ILP32F-LP64F-NEXT:  flw fs3, 108(s1)
-; ILP32F-LP64F-NEXT:  flw fs4, 112(s1)
-; ILP32F-LP64F-NEXT:  flw fs5, 116(s1)
-; ILP32F-LP64F-NEXT:  flw fs6, 120(s1)
-; ILP32F-LP64F-NEXT:  flw fs7, 124(s1)
-; ILP32F-LP64F-NEXT:  call callee
-; ILP32F-LP64F-NEXT:  fsw fs7, 124(s1)
-; ILP32F-LP64F-NEXT:  fsw fs6, 120(s1)
-; ILP32F-LP64F-NEXT:  fsw fs5, 116(s1)
-; ILP32F-LP64F-NEXT:  fsw fs4, 112(s1)
-; ILP32F-LP64F-NEXT:  fsw fs3, 108(s1)
-; ILP32F-LP64F-NEXT:  fsw fs2, 104(s1)
-; ILP32F-LP64F-NEXT:  fsw fs1, 100(s1)
-; ILP32F-LP64F-NEXT:  fsw fs0, 96(s1)
-; ILP32F-LP64F-NEXT:  fsw fs11, 92(s1)
-; ILP32F-LP64F-NEXT:  fsw fs10, 88(s1)
-; ILP32F-LP64F-NEXT:  fsw fs9, 84(s1)
-; ILP32F-LP64F-NEXT:  fsw fs8, 80(s1)
-; ILP32F-LP64F-NEXT:  lw ft0, {{[0-9]+}}(sp)
+; ILP32D-LABEL: caller:
+; ILP32D:       # %bb.0:
+; ILP32D-NEXT:    addi sp, sp, -192
+; ILP32D-NEXT:    sw ra, 188(sp)
+; ILP32D-NEXT:    sw s0, 184(sp)
+; ILP32D-NEXT:    sw s1, 180(sp)
+; ILP32D-NEXT:    fsd fs0, 168(sp)
+; ILP32D-NEXT:    fsd fs1, 160(sp)
+; ILP32D-NEXT:    fsd fs2, 152(sp)
+; ILP32D-NEXT:    fsd fs3, 144(sp)
+; ILP32D-NEXT:    fsd fs4, 136(sp)
+; ILP32D-NEXT:    fsd fs5, 128(sp)
+; ILP32D-NEXT:    fsd fs6, 120(sp)
+; ILP32D-NEXT:    fsd fs7, 112(sp)
+; ILP32D-NEXT:    fsd fs8, 104(sp)
+; ILP32D-NEXT:    fsd fs9, 96(sp)
+; ILP32D-NEXT:    fsd fs10, 88(sp)
+; ILP32D-NEXT:    fsd fs11, 80(sp)
+; ILP32D-NEXT:    lui s0, %hi(var)
+; ILP32D-NEXT:    flw ft0, %lo(var)(s0)
+; ILP32D-NEXT:    fsw ft0, 76(sp)
+; ILP32D-NEXT:    flw ft0, %lo(var+4)(s0)
+; ILP32D-NEXT:    fsw ft0, 72(sp)
+; ILP32D-NEXT:    flw ft0, %lo(var+8)(s0)
+; ILP32D-NEXT:    fsw ft0, 68(sp)
+; ILP32D-NEXT:    flw ft0, %lo(var+12)(s0)
+; ILP32D-NEXT:    fsw ft0, 64(sp)
+; ILP32D-NEXT:    addi s1, s0, %lo(var)
+; ILP32D-NEXT:    flw ft0, 16(s1)
+; ILP32D-NEXT:    fsw ft0, 60(sp)
+; ILP32D-NEXT:    flw ft0, 20(s1)
+; ILP32D-NEXT:    fsw ft0, 56(sp)
+; ILP32D-NEXT:    flw ft0, 24(s1)
+; ILP32D-NEXT:    fsw ft0, 52(sp)
+; ILP32D-NEXT:    flw ft0, 28(s1)
+; ILP32D-NEXT:    fsw ft0, 48(sp)
+; ILP32D-NEXT:    flw ft0, 32(s1)
+; ILP32D-NEXT:    fsw ft0, 44(sp)
+; ILP32D-NEXT:    flw ft0, 36(s1)
+; ILP32D-NEXT:    fsw ft0, 40(sp)
+; ILP32D-NEXT:    flw ft0, 40(s1)
+; ILP32D-NEXT:    fsw ft0, 36(sp)
+; ILP32D-NEXT:    flw ft0, 44(s1)
+; ILP32D-NEXT:    fsw ft0, 32(sp)
+; ILP32D-NEXT:    flw ft0, 48(s1)
+; ILP32D-NEXT:    fsw ft0, 28(sp)
+; ILP32D-NEXT:    flw ft0, 52(s1)
+; ILP32D-NEXT:    fsw ft0, 24(sp)
+; ILP32D-NEXT:    flw ft0, 56(s1)
+; ILP32D-NEXT:    fsw ft0, 20(sp)
+; ILP32D-NEXT:    flw ft0, 60(s1)
+; ILP32D-NEXT:    fsw ft0, 16(sp)
+; ILP32D-NEXT:    flw ft0, 64(s1)
+; ILP32D-NEXT:    fsw ft0, 12(sp)
+; ILP32D-NEXT:    flw ft0, 68(s1)
+; ILP32D-NEXT:    fsw ft0, 8(sp)
+; ILP32D-NEXT:    flw ft0, 72(s1)
+; ILP32D-NEXT:    fsw ft0, 4(sp)
+; ILP32D-NEXT:    flw ft0, 76(s1)
+; ILP32D-NEXT:    fsw ft0, 0(sp)
+; ILP32D-NEXT:    flw fs8, 80(s1)
+; ILP32D-NEXT:    flw fs9, 84(s1)
+; ILP32D-NEXT:    flw fs10, 88(s1)
+; ILP32D-NEXT:    flw fs11, 92(s1)
+; ILP32D-NEXT:    flw fs0, 96(s1)
+; ILP32D-NEXT:    flw fs1, 100(s1)
+; ILP32D-NEXT:    flw fs2, 104(s1)
+; ILP32D-NEXT:    flw fs3, 108(s1)
+; ILP32D-NEXT:    flw fs4, 112(s1)
+; ILP32D-NEXT:    flw fs5, 116(s1)
+; ILP32D-NEXT:    flw fs6, 120(s1)
+; ILP32D-NEXT:    flw fs7, 124(s1)
+; ILP32D-NEXT:    call callee
+; ILP32D-NEXT:    fsw fs7, 124(s1)
+; ILP32D-NEXT:    fsw fs6, 120(s1)
+; ILP32D-NEXT:    fsw fs5, 116(s1)
+; ILP32D-NEXT:    fsw fs4, 112(s1)
+; ILP32D-NEXT:    fsw fs3, 108(s1)
+; ILP32D-NEXT:    fsw fs2, 104(s1)
+; ILP32D-NEXT:    fsw fs1, 100(s1)
+; ILP32D-NEXT:    fsw fs0, 96(s1)
+; ILP32D-NEXT:    fsw fs11, 92(s1)
+; ILP32D-NEXT:    fsw fs10, 88(s1)
+; ILP32D-NEXT:    fsw fs9, 84(s1)
+; ILP32D-NEXT:    fsw fs8, 80(s1)
+; ILP32D-NEXT:    flw ft0, 0(sp)
+; ILP32D-NEXT:    fsw ft0, 76(s1)
+; ILP32D-NEXT:    flw ft0, 4(sp)
+; ILP32D-NEXT:    fsw ft0, 72(s1)
+; ILP32D-NEXT:    flw ft0, 8(sp)
+; ILP32D-NEXT:    fsw ft0, 68(s1)
+; ILP32D-NEXT:    flw ft0, 12(sp)
+; ILP32D-NEXT:    fsw ft0, 64(s1)
+; ILP32D-NEXT:    flw ft0, 16(sp)
+; ILP32D-NEXT:    fsw ft0, 60(s1)
+; ILP32D-NEXT:    flw ft0, 20(sp)
+; ILP32D-NEXT:    fsw ft0, 56(s1)
+; ILP32D-NEXT:    flw ft0, 24(sp)
+; ILP32D-NEXT:    fsw ft0, 52(s1)
+; ILP32D-NEXT:    flw ft0, 28(sp)
+; ILP32D-NEXT:    fsw ft0, 48(s1)
+; ILP32D-NEXT:    flw ft0, 32(sp)
+; ILP32D-NEXT:    fsw ft0, 44(s1)
+; ILP32D-NEXT:    flw ft0, 36(sp)
+; ILP32D-NEXT:    fsw ft0, 40(s1)
+; ILP32D-NEXT:    flw ft0, 40(sp)
+; ILP32D-NEXT:    fsw ft0, 36(s1)
+; ILP32D-NEXT:    flw ft0, 44(sp)
+; ILP32D-NEXT:    fsw ft0, 32(s1)
+; ILP32D-NEXT:    flw ft0, 48(sp)
+; ILP32D-NEXT:    fsw ft0, 28(s1)
+; ILP32D-NEXT:    flw ft0, 52(sp)
+; ILP32D-NEXT:    fsw ft0, 24(s1)
+; ILP32D-NEXT:    flw ft0, 56(sp)
+; ILP32D-NEXT:    fsw ft0, 20(s1)
+; ILP32D-NEXT:    flw ft0, 60(sp)
+; ILP32D-NEXT:    fsw ft0, 16(s1)
+; ILP32D-NEXT:    flw ft0, 64(sp)
+; ILP32D-NEXT:    fsw ft0, %lo(var+12)(s0)
+; ILP32D-NEXT:    flw ft0, 68(sp)
+; ILP32D-NEXT:    fsw ft0, %lo(var+8)(s0)
+; ILP32D-NEXT:    flw ft0, 72(sp)
+; ILP32D-NEXT:    fsw ft0, %lo(var+4)(s0)
+; ILP32D-NEXT:    flw ft0, 76(sp)
+; ILP32D-NEXT:    fsw ft0, %lo(var)(s0)
+; ILP32D-NEXT:    fld fs11, 80(sp)
+; ILP32D-NEXT:    fld fs10, 88(sp)
+; ILP32D-NEXT:    fld fs9, 96(sp)
+; ILP32D-NEXT:    fld fs8, 104(sp)
+; ILP32D-NEXT:    fld fs7, 112(sp)
+; ILP32D-NEXT:    fld fs6, 120(sp)
+; ILP32D-NEXT:    fld fs5, 128(sp)
+; ILP32D-NEXT:    fld fs4, 136(sp)
+; ILP32D-NEXT:    fld fs3, 144(sp)
+; ILP32D-NEXT:    fld fs2, 152(sp)
+; ILP32D-NEXT:    fld fs1, 160(sp)
+; ILP32D-NEXT:    fld fs0, 168(sp)
+; ILP32D-NEXT:    lw s1, 180(sp)
+; ILP32D-NEXT:    lw s0, 184(sp)
+; ILP32D-NEXT:    lw ra, 188(sp)
+; ILP32D-NEXT:    addi sp, sp, 192
+; ILP32D-NEXT:    ret
 ;
-; ILP32D-LP64D-LABEL: caller:
-; ILP32D-LP64D:       flw fs8, 80(s1)
-; ILP32D-LP64D-NEXT:  flw fs9, 84(s1)
-; ILP32D-LP64D-NEXT:  flw fs10, 88(s1)
-; ILP32D-LP64D-NEXT:  flw fs11, 92(s1)
-; ILP32D-LP64D-NEXT:  flw fs0, 96(s1)
-; ILP32D-LP64D-NEXT:  flw fs1, 100(s1)
-; ILP32D-LP64D-NEXT:  flw fs2, 104(s1)
-; ILP32D-LP64D-NEXT:  flw fs3, 108(s1)
-; ILP32D-LP64D-NEXT:  flw fs4, 112(s1)
-; ILP32D-LP64D-NEXT:  flw fs5, 116(s1)
-; ILP32D-LP64D-NEXT:  flw fs6, 120(s1)
-; ILP32D-LP64D-NEXT:  flw fs7, 124(s1)
-; ILP32D-LP64D-NEXT:  call callee
-; ILP32D-LP64D-NEXT:  fsw fs7, 124(s1)
-; ILP32D-LP64D-NEXT:  fsw fs6, 120(s1)
-; ILP32D-LP64D-NEXT:  fsw fs5, 116(s1)
-; ILP32D-LP64D-NEXT:  fsw fs4, 112(s1)
-; ILP32D-LP64D-NEXT:  fsw fs3, 108(s1)
-; ILP32D-LP64D-NEXT:  fsw fs2, 104(s1)
-; ILP32D-LP64D-NEXT:  fsw fs1, 100(s1)
-; ILP32D-LP64D-NEXT:  fsw fs0, 96(s1)
-; ILP32D-LP64D-NEXT:  fsw fs11, 92(s1)
-; ILP32D-LP64D-NEXT:  fsw fs10, 88(s1)
-; ILP32D-LP64D-NEXT:  fsw fs9, 84(s1)
-; ILP32D-LP64D-NEXT:  fsw fs8, 80(s1)
-; ILP32D-LP64D-NEXT:  flw ft0, {{[0-9]+}}(sp)
+; LP64D-LABEL: caller:
+; LP64D:       # %bb.0:
+; LP64D-NEXT:    addi sp, sp, -208
+; LP64D-NEXT:    sd ra, 200(sp)
+; LP64D-NEXT:    sd s0, 192(sp)
+; LP64D-NEXT:    sd s1, 184(sp)
+; LP64D-NEXT:    fsd fs0, 176(sp)
+; LP64D-NEXT:    fsd fs1, 168(sp)
+; LP64D-NEXT:    fsd fs2, 160(sp)
+; LP64D-NEXT:    fsd fs3, 152(sp)
+; LP64D-NEXT:    fsd fs4, 144(sp)
+; LP64D-NEXT:    fsd fs5, 136(sp)
+; LP64D-NEXT:    fsd fs6, 128(sp)
+; LP64D-NEXT:    fsd fs7, 120(sp)
+; LP64D-NEXT:    fsd fs8, 112(sp)
+; LP64D-NEXT:    fsd fs9, 104(sp)
+; LP64D-NEXT:    fsd fs10, 96(sp)
+; LP64D-NEXT:    fsd fs11, 88(sp)
+; LP64D-NEXT:    lui s0, %hi(var)
+; LP64D-NEXT:    flw ft0, %lo(var)(s0)
+; LP64D-NEXT:    fsw ft0, 84(sp)
+; LP64D-NEXT:    flw ft0, %lo(var+4)(s0)
+; LP64D-NEXT:    fsw ft0, 80(sp)
+; LP64D-NEXT:    flw ft0, %lo(var+8)(s0)
+; LP64D-NEXT:    fsw ft0, 76(sp)
+; LP64D-NEXT:    flw ft0, %lo(var+12)(s0)
+; LP64D-NEXT:    fsw ft0, 72(sp)
+; LP64D-NEXT:    addi s1, s0, %lo(var)
+; LP64D-NEXT:    flw ft0, 16(s1)
+; LP64D-NEXT:    fsw ft0, 68(sp)
+; LP64D-NEXT:    flw ft0, 20(s1)
+; LP64D-NEXT:    fsw ft0, 64(sp)
+; LP64D-NEXT:    flw ft0, 24(s1)
+; LP64D-NEXT:    fsw ft0, 60(sp)
+; LP64D-NEXT:    flw ft0, 28(s1)
+; LP64D-NEXT:    fsw ft0, 56(sp)
+; LP64D-NEXT:    flw ft0, 32(s1)
+; LP64D-NEXT:    fsw ft0, 52(sp)
+; LP64D-NEXT:    flw ft0, 36(s1)
+; LP64D-NEXT:    fsw ft0, 48(sp)
+; LP64D-NEXT:    flw ft0, 40(s1)
+; LP64D-NEXT:    fsw ft0, 44(sp)
+; LP64D-NEXT:    flw ft0, 44(s1)
+; LP64D-NEXT:    fsw ft0, 40(sp)
+; LP64D-NEXT:    flw ft0, 48(s1)
+; LP64D-NEXT:    fsw ft0, 36(sp)
+; LP64D-NEXT:    flw ft0, 52(s1)
+; LP64D-NEXT:    fsw ft0, 32(sp)
+; LP64D-NEXT:    flw ft0, 56(s1)
+; LP64D-NEXT:    fsw ft0, 28(sp)
+; LP64D-NEXT:    flw ft0, 60(s1)
+; LP64D-NEXT:    fsw ft0, 24(sp)
+; LP64D-NEXT:    flw ft0, 64(s1)
+; LP64D-NEXT:    fsw ft0, 20(sp)
+; LP64D-NEXT:    flw ft0, 68(s1)
+; LP64D-NEXT:    fsw ft0, 16(sp)
+; LP64D-NEXT:    flw ft0, 72(s1)
+; LP64D-NEXT:    fsw ft0, 12(sp)
+; LP64D-NEXT:    flw ft0, 76(s1)
+; LP64D-NEXT:    fsw ft0, 8(sp)
+; LP64D-NEXT:    flw fs8, 80(s1)
+; LP64D-NEXT:    flw fs9, 84(s1)
+; LP64D-NEXT:    flw fs10, 88(s1)
+; LP64D-NEXT:    flw fs11, 92(s1)
+; LP64D-NEXT:    flw fs0, 96(s1)
+; LP64D-NEXT:    flw fs1, 100(s1)
+; LP64D-NEXT:    flw fs2, 104(s1)
+; LP64D-NEXT:    flw fs3, 108(s1)
+; LP64D-NEXT:    flw fs4, 112(s1)
+; LP64D-NEXT:    flw fs5, 116(s1)
+; LP64D-NEXT:    flw fs6, 120(s1)
+; LP64D-NEXT:    flw fs7, 124(s1)
+; LP64D-NEXT:    call callee
+; LP64D-NEXT:    fsw fs7, 124(s1)
+; LP64D-NEXT:    fsw fs6, 120(s1)
+; LP64D-NEXT:    fsw fs5, 116(s1)
+; LP64D-NEXT:    fsw fs4, 112(s1)
+; LP64D-NEXT:    fsw fs3, 108(s1)
+; LP64D-NEXT:    fsw fs2, 104(s1)
+; LP64D-NEXT:    fsw fs1, 100(s1)
+; LP64D-NEXT:    fsw fs0, 96(s1)
+; LP64D-NEXT:    fsw fs11, 92(s1)
+; LP64D-NEXT:    fsw fs10, 88(s1)
+; LP64D-NEXT:    fsw fs9, 84(s1)
+; LP64D-NEXT:    fsw fs8, 80(s1)
+; LP64D-NEXT:    flw ft0, 8(sp)
+; LP64D-NEXT:    fsw ft0, 76(s1)
+; LP64D-NEXT:    flw ft0, 12(sp)
+; LP64D-NEXT:    fsw ft0, 72(s1)
+; LP64D-NEXT:    flw ft0, 16(sp)
+; LP64D-NEXT:    fsw ft0, 68(s1)
+; LP64D-NEXT:    flw ft0, 20(sp)
+; LP64D-NEXT:    fsw ft0, 64(s1)
+; LP64D-NEXT:    flw ft0, 24(sp)
+; LP64D-NEXT:    fsw ft0, 60(s1)
+; LP64D-NEXT:    flw ft0, 28(sp)
+; LP64D-NEXT:    fsw ft0, 56(s1)
+; LP64D-NEXT:    flw ft0, 32(sp)
+; LP64D-NEXT:    fsw ft0, 52(s1)
+; LP64D-NEXT:    flw ft0, 36(sp)
+; LP64D-NEXT:    fsw ft0, 48(s1)
+; LP64D-NEXT:    flw ft0, 40(sp)
+; LP64D-NEXT:    fsw ft0, 44(s1)
+; LP64D-NEXT:    flw ft0, 44(sp)
+; LP64D-NEXT:    fsw ft0, 40(s1)
+; LP64D-NEXT:    flw ft0, 48(sp)
+; LP64D-NEXT:    fsw ft0, 36(s1)
+; LP64D-NEXT:    flw ft0, 52(sp)
+; LP64D-NEXT:    fsw ft0, 32(s1)
+; LP64D-NEXT:    flw ft0, 56(sp)
+; LP64D-NEXT:    fsw ft0, 28(s1)
+; LP64D-NEXT:    flw ft0, 60(sp)
+; LP64D-NEXT:    fsw ft0, 24(s1)
+; LP64D-NEXT:    flw ft0, 64(sp)
+; LP64D-NEXT:    fsw ft0, 20(s1)
+; LP64D-NEXT:    flw ft0, 68(sp)
+; LP64D-NEXT:    fsw ft0, 16(s1)
+; LP64D-NEXT:    flw ft0, 72(sp)
+; LP64D-NEXT:    fsw ft0, %lo(var+12)(s0)
+; LP64D-NEXT:    flw ft0, 76(sp)
+; LP64D-NEXT:    fsw ft0, %lo(var+8)(s0)
+; LP64D-NEXT:    flw ft0, 80(sp)
+; LP64D-NEXT:    fsw ft0, %lo(var+4)(s0)
+; LP64D-NEXT:    flw ft0, 84(sp)
+; LP64D-NEXT:    fsw ft0, %lo(var)(s0)
+; LP64D-NEXT:    fld fs11, 88(sp)
+; LP64D-NEXT:    fld fs10, 96(sp)
+; LP64D-NEXT:    fld fs9, 104(sp)
+; LP64D-NEXT:    fld fs8, 112(sp)
+; LP64D-NEXT:    fld fs7, 120(sp)
+; LP64D-NEXT:    fld fs6, 128(sp)
+; LP64D-NEXT:    fld fs5, 136(sp)
+; LP64D-NEXT:    fld fs4, 144(sp)
+; LP64D-NEXT:    fld fs3, 152(sp)
+; LP64D-NEXT:    fld fs2, 160(sp)
+; LP64D-NEXT:    fld fs1, 168(sp)
+; LP64D-NEXT:    fld fs0, 176(sp)
+; LP64D-NEXT:    ld s1, 184(sp)
+; LP64D-NEXT:    ld s0, 192(sp)
+; LP64D-NEXT:    ld ra, 200(sp)
+; LP64D-NEXT:    addi sp, sp, 208
+; LP64D-NEXT:    ret
   %val = load [32 x float], [32 x float]* @var
   call void @callee()
   store volatile [32 x float] %val, [32 x float]* @var
diff --git a/llvm/test/CodeGen/RISCV/callee-saved-fpr64s.ll b/llvm/test/CodeGen/RISCV/callee-saved-fpr64s.ll
index f95bc45736af0..d5c67fb462034 100644
--- a/llvm/test/CodeGen/RISCV/callee-saved-fpr64s.ll
+++ b/llvm/test/CodeGen/RISCV/callee-saved-fpr64s.ll
@@ -1,11 +1,12 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+d -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=ILP32-LP64
+; RUN:   | FileCheck %s -check-prefix=ILP32
 ; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=ILP32-LP64
+; RUN:   | FileCheck %s -check-prefix=LP64
 ; RUN: llc -mtriple=riscv32 -mattr=+d -target-abi ilp32d -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=ILP32D-LP64D
+; RUN:   | FileCheck %s -check-prefix=ILP32D
 ; RUN: llc -mtriple=riscv64 -mattr=+d -target-abi lp64d -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=ILP32D-LP64D
+; RUN:   | FileCheck %s -check-prefix=LP64D
 
 @var = global [32 x double] zeroinitializer
 
@@ -16,94 +17,337 @@
 ; something appropriate.
 
 define void @callee() nounwind {
-; ILP32-LP64-LABEL: callee:
-; ILP32-LP64:       # %bb.0:
-; ILP32-LP64-NEXT:    lui a0, %hi(var)
-; ILP32-LP64-NEXT:    fld ft0, %lo(var)(a0)
-; ILP32-LP64-NEXT:    addi a1, a0, %lo(var)
-; ILP32-LP64-NEXT:    fld ft1, 8(a1)
-; ILP32-LP64-NEXT:    fld ft2, 16(a1)
-; ILP32-LP64-NEXT:    fld ft3, 24(a1)
-; ILP32-LP64-NEXT:    fld ft4, 32(a1)
-; ILP32-LP64-NEXT:    fld ft5, 40(a1)
-; ILP32-LP64-NEXT:    fld ft6, 48(a1)
-; ILP32-LP64-NEXT:    fld ft7, 56(a1)
-; ILP32-LP64-NEXT:    fld fa0, 64(a1)
-; ILP32-LP64-NEXT:    fld fa1, 72(a1)
-; ILP32-LP64-NEXT:    fld fa2, 80(a1)
-; ILP32-LP64-NEXT:    fld fa3, 88(a1)
-; ILP32-LP64-NEXT:    fld fa4, 96(a1)
-; ILP32-LP64-NEXT:    fld fa5, 104(a1)
-; ILP32-LP64-NEXT:    fld fa6, 112(a1)
-; ILP32-LP64-NEXT:    fld fa7, 120(a1)
-; ILP32-LP64-NEXT:    fld ft8, 128(a1)
-; ILP32-LP64-NEXT:    fld ft9, 136(a1)
-; ILP32-LP64-NEXT:    fld ft10, 144(a1)
-; ILP32-LP64-NEXT:    fld ft11, 152(a1)
-; ILP32-LP64-NEXT:    fld fs0, 160(a1)
-; ILP32-LP64-NEXT:    fld fs1, 168(a1)
-; ILP32-LP64-NEXT:    fld fs2, 176(a1)
-; ILP32-LP64-NEXT:    fld fs3, 184(a1)
-; ILP32-LP64-NEXT:    fld fs4, 192(a1)
-; ILP32-LP64-NEXT:    fld fs5, 200(a1)
-; ILP32-LP64-NEXT:    fld fs6, 208(a1)
-; ILP32-LP64-NEXT:    fld fs7, 216(a1)
-; ILP32-LP64-NEXT:    fld fs8, 248(a1)
-; ILP32-LP64-NEXT:    fld fs9, 240(a1)
-; ILP32-LP64-NEXT:    fld fs10, 232(a1)
-; ILP32-LP64-NEXT:    fld fs11, 224(a1)
-; ILP32-LP64-NEXT:    fsd fs8, 248(a1)
-; ILP32-LP64-NEXT:    fsd fs9, 240(a1)
-; ILP32-LP64-NEXT:    fsd fs10, 232(a1)
-; ILP32-LP64-NEXT:    fsd fs11, 224(a1)
-; ILP32-LP64-NEXT:    fsd fs7, 216(a1)
-; ILP32-LP64-NEXT:    fsd fs6, 208(a1)
-; ILP32-LP64-NEXT:    fsd fs5, 200(a1)
-; ILP32-LP64-NEXT:    fsd fs4, 192(a1)
-; ILP32-LP64-NEXT:    fsd fs3, 184(a1)
-; ILP32-LP64-NEXT:    fsd fs2, 176(a1)
-; ILP32-LP64-NEXT:    fsd fs1, 168(a1)
-; ILP32-LP64-NEXT:    fsd fs0, 160(a1)
-; ILP32-LP64-NEXT:    fsd ft11, 152(a1)
-; ILP32-LP64-NEXT:    fsd ft10, 144(a1)
-; ILP32-LP64-NEXT:    fsd ft9, 136(a1)
-; ILP32-LP64-NEXT:    fsd ft8, 128(a1)
-; ILP32-LP64-NEXT:    fsd fa7, 120(a1)
-; ILP32-LP64-NEXT:    fsd fa6, 112(a1)
-; ILP32-LP64-NEXT:    fsd fa5, 104(a1)
-; ILP32-LP64-NEXT:    fsd fa4, 96(a1)
-; ILP32-LP64-NEXT:    fsd fa3, 88(a1)
-; ILP32-LP64-NEXT:    fsd fa2, 80(a1)
-; ILP32-LP64-NEXT:    fsd fa1, 72(a1)
-; ILP32-LP64-NEXT:    fsd fa0, 64(a1)
-; ILP32-LP64-NEXT:    fsd ft7, 56(a1)
-; ILP32-LP64-NEXT:    fsd ft6, 48(a1)
-; ILP32-LP64-NEXT:    fsd ft5, 40(a1)
-; ILP32-LP64-NEXT:    fsd ft4, 32(a1)
-; ILP32-LP64-NEXT:    fsd ft3, 24(a1)
-; ILP32-LP64-NEXT:    fsd ft2, 16(a1)
-; ILP32-LP64-NEXT:    fsd ft1, 8(a1)
-; ILP32-LP64-NEXT:    fsd ft0, %lo(var)(a0)
-; ILP32-LP64-NEXT:    ret
+; ILP32-LABEL: callee:
+; ILP32:       # %bb.0:
+; ILP32-NEXT:    lui a0, %hi(var)
+; ILP32-NEXT:    fld ft0, %lo(var)(a0)
+; ILP32-NEXT:    fld ft1, %lo(var+8)(a0)
+; ILP32-NEXT:    addi a1, a0, %lo(var)
+; ILP32-NEXT:    fld ft2, 16(a1)
+; ILP32-NEXT:    fld ft3, 24(a1)
+; ILP32-NEXT:    fld ft4, 32(a1)
+; ILP32-NEXT:    fld ft5, 40(a1)
+; ILP32-NEXT:    fld ft6, 48(a1)
+; ILP32-NEXT:    fld ft7, 56(a1)
+; ILP32-NEXT:    fld fa0, 64(a1)
+; ILP32-NEXT:    fld fa1, 72(a1)
+; ILP32-NEXT:    fld fa2, 80(a1)
+; ILP32-NEXT:    fld fa3, 88(a1)
+; ILP32-NEXT:    fld fa4, 96(a1)
+; ILP32-NEXT:    fld fa5, 104(a1)
+; ILP32-NEXT:    fld fa6, 112(a1)
+; ILP32-NEXT:    fld fa7, 120(a1)
+; ILP32-NEXT:    fld ft8, 128(a1)
+; ILP32-NEXT:    fld ft9, 136(a1)
+; ILP32-NEXT:    fld ft10, 144(a1)
+; ILP32-NEXT:    fld ft11, 152(a1)
+; ILP32-NEXT:    fld fs0, 160(a1)
+; ILP32-NEXT:    fld fs1, 168(a1)
+; ILP32-NEXT:    fld fs2, 176(a1)
+; ILP32-NEXT:    fld fs3, 184(a1)
+; ILP32-NEXT:    fld fs4, 192(a1)
+; ILP32-NEXT:    fld fs5, 200(a1)
+; ILP32-NEXT:    fld fs6, 208(a1)
+; ILP32-NEXT:    fld fs7, 216(a1)
+; ILP32-NEXT:    fld fs8, 248(a1)
+; ILP32-NEXT:    fld fs9, 240(a1)
+; ILP32-NEXT:    fld fs10, 232(a1)
+; ILP32-NEXT:    fld fs11, 224(a1)
+; ILP32-NEXT:    fsd fs8, 248(a1)
+; ILP32-NEXT:    fsd fs9, 240(a1)
+; ILP32-NEXT:    fsd fs10, 232(a1)
+; ILP32-NEXT:    fsd fs11, 224(a1)
+; ILP32-NEXT:    fsd fs7, 216(a1)
+; ILP32-NEXT:    fsd fs6, 208(a1)
+; ILP32-NEXT:    fsd fs5, 200(a1)
+; ILP32-NEXT:    fsd fs4, 192(a1)
+; ILP32-NEXT:    fsd fs3, 184(a1)
+; ILP32-NEXT:    fsd fs2, 176(a1)
+; ILP32-NEXT:    fsd fs1, 168(a1)
+; ILP32-NEXT:    fsd fs0, 160(a1)
+; ILP32-NEXT:    fsd ft11, 152(a1)
+; ILP32-NEXT:    fsd ft10, 144(a1)
+; ILP32-NEXT:    fsd ft9, 136(a1)
+; ILP32-NEXT:    fsd ft8, 128(a1)
+; ILP32-NEXT:    fsd fa7, 120(a1)
+; ILP32-NEXT:    fsd fa6, 112(a1)
+; ILP32-NEXT:    fsd fa5, 104(a1)
+; ILP32-NEXT:    fsd fa4, 96(a1)
+; ILP32-NEXT:    fsd fa3, 88(a1)
+; ILP32-NEXT:    fsd fa2, 80(a1)
+; ILP32-NEXT:    fsd fa1, 72(a1)
+; ILP32-NEXT:    fsd fa0, 64(a1)
+; ILP32-NEXT:    fsd ft7, 56(a1)
+; ILP32-NEXT:    fsd ft6, 48(a1)
+; ILP32-NEXT:    fsd ft5, 40(a1)
+; ILP32-NEXT:    fsd ft4, 32(a1)
+; ILP32-NEXT:    fsd ft3, 24(a1)
+; ILP32-NEXT:    fsd ft2, 16(a1)
+; ILP32-NEXT:    fsd ft1, %lo(var+8)(a0)
+; ILP32-NEXT:    fsd ft0, %lo(var)(a0)
+; ILP32-NEXT:    ret
 ;
-; ILP32D-LP64D-LABEL: callee:
-; ILP32D-LP64D:       # %bb.0:
-; ILP32D-LP64D-NEXT:    addi sp, sp, -96
-; ILP32D-LP64D-NEXT:    fsd fs0, 88(sp)
-; ILP32D-LP64D-NEXT:    fsd fs1, 80(sp)
-; ILP32D-LP64D-NEXT:    fsd fs2, 72(sp)
-; ILP32D-LP64D-NEXT:    fsd fs3, 64(sp)
-; ILP32D-LP64D-NEXT:    fsd fs4, 56(sp)
-; ILP32D-LP64D-NEXT:    fsd fs5, 48(sp)
-; ILP32D-LP64D-NEXT:    fsd fs6, 40(sp)
-; ILP32D-LP64D-NEXT:    fsd fs7, 32(sp)
-; ILP32D-LP64D-NEXT:    fsd fs8, 24(sp)
-; ILP32D-LP64D-NEXT:    fsd fs9, 16(sp)
-; ILP32D-LP64D-NEXT:    fsd fs10, 8(sp)
-; ILP32D-LP64D-NEXT:    fsd fs11, 0(sp)
-; ILP32D-LP64D-NEXT:    lui a0, %hi(var)
-; ILP32D-LP64D-NEXT:    fld ft0, %lo(var)(a0)
-; ILP32D-LP64D-NEXT:    addi a1, a0, %lo(var)
+; LP64-LABEL: callee:
+; LP64:       # %bb.0:
+; LP64-NEXT:    lui a0, %hi(var)
+; LP64-NEXT:    fld ft0, %lo(var)(a0)
+; LP64-NEXT:    fld ft1, %lo(var+8)(a0)
+; LP64-NEXT:    addi a1, a0, %lo(var)
+; LP64-NEXT:    fld ft2, 16(a1)
+; LP64-NEXT:    fld ft3, 24(a1)
+; LP64-NEXT:    fld ft4, 32(a1)
+; LP64-NEXT:    fld ft5, 40(a1)
+; LP64-NEXT:    fld ft6, 48(a1)
+; LP64-NEXT:    fld ft7, 56(a1)
+; LP64-NEXT:    fld fa0, 64(a1)
+; LP64-NEXT:    fld fa1, 72(a1)
+; LP64-NEXT:    fld fa2, 80(a1)
+; LP64-NEXT:    fld fa3, 88(a1)
+; LP64-NEXT:    fld fa4, 96(a1)
+; LP64-NEXT:    fld fa5, 104(a1)
+; LP64-NEXT:    fld fa6, 112(a1)
+; LP64-NEXT:    fld fa7, 120(a1)
+; LP64-NEXT:    fld ft8, 128(a1)
+; LP64-NEXT:    fld ft9, 136(a1)
+; LP64-NEXT:    fld ft10, 144(a1)
+; LP64-NEXT:    fld ft11, 152(a1)
+; LP64-NEXT:    fld fs0, 160(a1)
+; LP64-NEXT:    fld fs1, 168(a1)
+; LP64-NEXT:    fld fs2, 176(a1)
+; LP64-NEXT:    fld fs3, 184(a1)
+; LP64-NEXT:    fld fs4, 192(a1)
+; LP64-NEXT:    fld fs5, 200(a1)
+; LP64-NEXT:    fld fs6, 208(a1)
+; LP64-NEXT:    fld fs7, 216(a1)
+; LP64-NEXT:    fld fs8, 248(a1)
+; LP64-NEXT:    fld fs9, 240(a1)
+; LP64-NEXT:    fld fs10, 232(a1)
+; LP64-NEXT:    fld fs11, 224(a1)
+; LP64-NEXT:    fsd fs8, 248(a1)
+; LP64-NEXT:    fsd fs9, 240(a1)
+; LP64-NEXT:    fsd fs10, 232(a1)
+; LP64-NEXT:    fsd fs11, 224(a1)
+; LP64-NEXT:    fsd fs7, 216(a1)
+; LP64-NEXT:    fsd fs6, 208(a1)
+; LP64-NEXT:    fsd fs5, 200(a1)
+; LP64-NEXT:    fsd fs4, 192(a1)
+; LP64-NEXT:    fsd fs3, 184(a1)
+; LP64-NEXT:    fsd fs2, 176(a1)
+; LP64-NEXT:    fsd fs1, 168(a1)
+; LP64-NEXT:    fsd fs0, 160(a1)
+; LP64-NEXT:    fsd ft11, 152(a1)
+; LP64-NEXT:    fsd ft10, 144(a1)
+; LP64-NEXT:    fsd ft9, 136(a1)
+; LP64-NEXT:    fsd ft8, 128(a1)
+; LP64-NEXT:    fsd fa7, 120(a1)
+; LP64-NEXT:    fsd fa6, 112(a1)
+; LP64-NEXT:    fsd fa5, 104(a1)
+; LP64-NEXT:    fsd fa4, 96(a1)
+; LP64-NEXT:    fsd fa3, 88(a1)
+; LP64-NEXT:    fsd fa2, 80(a1)
+; LP64-NEXT:    fsd fa1, 72(a1)
+; LP64-NEXT:    fsd fa0, 64(a1)
+; LP64-NEXT:    fsd ft7, 56(a1)
+; LP64-NEXT:    fsd ft6, 48(a1)
+; LP64-NEXT:    fsd ft5, 40(a1)
+; LP64-NEXT:    fsd ft4, 32(a1)
+; LP64-NEXT:    fsd ft3, 24(a1)
+; LP64-NEXT:    fsd ft2, 16(a1)
+; LP64-NEXT:    fsd ft1, %lo(var+8)(a0)
+; LP64-NEXT:    fsd ft0, %lo(var)(a0)
+; LP64-NEXT:    ret
+;
+; ILP32D-LABEL: callee:
+; ILP32D:       # %bb.0:
+; ILP32D-NEXT:    addi sp, sp, -96
+; ILP32D-NEXT:    fsd fs0, 88(sp)
+; ILP32D-NEXT:    fsd fs1, 80(sp)
+; ILP32D-NEXT:    fsd fs2, 72(sp)
+; ILP32D-NEXT:    fsd fs3, 64(sp)
+; ILP32D-NEXT:    fsd fs4, 56(sp)
+; ILP32D-NEXT:    fsd fs5, 48(sp)
+; ILP32D-NEXT:    fsd fs6, 40(sp)
+; ILP32D-NEXT:    fsd fs7, 32(sp)
+; ILP32D-NEXT:    fsd fs8, 24(sp)
+; ILP32D-NEXT:    fsd fs9, 16(sp)
+; ILP32D-NEXT:    fsd fs10, 8(sp)
+; ILP32D-NEXT:    fsd fs11, 0(sp)
+; ILP32D-NEXT:    lui a0, %hi(var)
+; ILP32D-NEXT:    fld ft0, %lo(var)(a0)
+; ILP32D-NEXT:    fld ft1, %lo(var+8)(a0)
+; ILP32D-NEXT:    addi a1, a0, %lo(var)
+; ILP32D-NEXT:    fld ft2, 16(a1)
+; ILP32D-NEXT:    fld ft3, 24(a1)
+; ILP32D-NEXT:    fld ft4, 32(a1)
+; ILP32D-NEXT:    fld ft5, 40(a1)
+; ILP32D-NEXT:    fld ft6, 48(a1)
+; ILP32D-NEXT:    fld ft7, 56(a1)
+; ILP32D-NEXT:    fld fa0, 64(a1)
+; ILP32D-NEXT:    fld fa1, 72(a1)
+; ILP32D-NEXT:    fld fa2, 80(a1)
+; ILP32D-NEXT:    fld fa3, 88(a1)
+; ILP32D-NEXT:    fld fa4, 96(a1)
+; ILP32D-NEXT:    fld fa5, 104(a1)
+; ILP32D-NEXT:    fld fa6, 112(a1)
+; ILP32D-NEXT:    fld fa7, 120(a1)
+; ILP32D-NEXT:    fld ft8, 128(a1)
+; ILP32D-NEXT:    fld ft9, 136(a1)
+; ILP32D-NEXT:    fld ft10, 144(a1)
+; ILP32D-NEXT:    fld ft11, 152(a1)
+; ILP32D-NEXT:    fld fs0, 160(a1)
+; ILP32D-NEXT:    fld fs1, 168(a1)
+; ILP32D-NEXT:    fld fs2, 176(a1)
+; ILP32D-NEXT:    fld fs3, 184(a1)
+; ILP32D-NEXT:    fld fs4, 192(a1)
+; ILP32D-NEXT:    fld fs5, 200(a1)
+; ILP32D-NEXT:    fld fs6, 208(a1)
+; ILP32D-NEXT:    fld fs7, 216(a1)
+; ILP32D-NEXT:    fld fs8, 248(a1)
+; ILP32D-NEXT:    fld fs9, 240(a1)
+; ILP32D-NEXT:    fld fs10, 232(a1)
+; ILP32D-NEXT:    fld fs11, 224(a1)
+; ILP32D-NEXT:    fsd fs8, 248(a1)
+; ILP32D-NEXT:    fsd fs9, 240(a1)
+; ILP32D-NEXT:    fsd fs10, 232(a1)
+; ILP32D-NEXT:    fsd fs11, 224(a1)
+; ILP32D-NEXT:    fsd fs7, 216(a1)
+; ILP32D-NEXT:    fsd fs6, 208(a1)
+; ILP32D-NEXT:    fsd fs5, 200(a1)
+; ILP32D-NEXT:    fsd fs4, 192(a1)
+; ILP32D-NEXT:    fsd fs3, 184(a1)
+; ILP32D-NEXT:    fsd fs2, 176(a1)
+; ILP32D-NEXT:    fsd fs1, 168(a1)
+; ILP32D-NEXT:    fsd fs0, 160(a1)
+; ILP32D-NEXT:    fsd ft11, 152(a1)
+; ILP32D-NEXT:    fsd ft10, 144(a1)
+; ILP32D-NEXT:    fsd ft9, 136(a1)
+; ILP32D-NEXT:    fsd ft8, 128(a1)
+; ILP32D-NEXT:    fsd fa7, 120(a1)
+; ILP32D-NEXT:    fsd fa6, 112(a1)
+; ILP32D-NEXT:    fsd fa5, 104(a1)
+; ILP32D-NEXT:    fsd fa4, 96(a1)
+; ILP32D-NEXT:    fsd fa3, 88(a1)
+; ILP32D-NEXT:    fsd fa2, 80(a1)
+; ILP32D-NEXT:    fsd fa1, 72(a1)
+; ILP32D-NEXT:    fsd fa0, 64(a1)
+; ILP32D-NEXT:    fsd ft7, 56(a1)
+; ILP32D-NEXT:    fsd ft6, 48(a1)
+; ILP32D-NEXT:    fsd ft5, 40(a1)
+; ILP32D-NEXT:    fsd ft4, 32(a1)
+; ILP32D-NEXT:    fsd ft3, 24(a1)
+; ILP32D-NEXT:    fsd ft2, 16(a1)
+; ILP32D-NEXT:    fsd ft1, %lo(var+8)(a0)
+; ILP32D-NEXT:    fsd ft0, %lo(var)(a0)
+; ILP32D-NEXT:    fld fs11, 0(sp)
+; ILP32D-NEXT:    fld fs10, 8(sp)
+; ILP32D-NEXT:    fld fs9, 16(sp)
+; ILP32D-NEXT:    fld fs8, 24(sp)
+; ILP32D-NEXT:    fld fs7, 32(sp)
+; ILP32D-NEXT:    fld fs6, 40(sp)
+; ILP32D-NEXT:    fld fs5, 48(sp)
+; ILP32D-NEXT:    fld fs4, 56(sp)
+; ILP32D-NEXT:    fld fs3, 64(sp)
+; ILP32D-NEXT:    fld fs2, 72(sp)
+; ILP32D-NEXT:    fld fs1, 80(sp)
+; ILP32D-NEXT:    fld fs0, 88(sp)
+; ILP32D-NEXT:    addi sp, sp, 96
+; ILP32D-NEXT:    ret
+;
+; LP64D-LABEL: callee:
+; LP64D:       # %bb.0:
+; LP64D-NEXT:    addi sp, sp, -96
+; LP64D-NEXT:    fsd fs0, 88(sp)
+; LP64D-NEXT:    fsd fs1, 80(sp)
+; LP64D-NEXT:    fsd fs2, 72(sp)
+; LP64D-NEXT:    fsd fs3, 64(sp)
+; LP64D-NEXT:    fsd fs4, 56(sp)
+; LP64D-NEXT:    fsd fs5, 48(sp)
+; LP64D-NEXT:    fsd fs6, 40(sp)
+; LP64D-NEXT:    fsd fs7, 32(sp)
+; LP64D-NEXT:    fsd fs8, 24(sp)
+; LP64D-NEXT:    fsd fs9, 16(sp)
+; LP64D-NEXT:    fsd fs10, 8(sp)
+; LP64D-NEXT:    fsd fs11, 0(sp)
+; LP64D-NEXT:    lui a0, %hi(var)
+; LP64D-NEXT:    fld ft0, %lo(var)(a0)
+; LP64D-NEXT:    fld ft1, %lo(var+8)(a0)
+; LP64D-NEXT:    addi a1, a0, %lo(var)
+; LP64D-NEXT:    fld ft2, 16(a1)
+; LP64D-NEXT:    fld ft3, 24(a1)
+; LP64D-NEXT:    fld ft4, 32(a1)
+; LP64D-NEXT:    fld ft5, 40(a1)
+; LP64D-NEXT:    fld ft6, 48(a1)
+; LP64D-NEXT:    fld ft7, 56(a1)
+; LP64D-NEXT:    fld fa0, 64(a1)
+; LP64D-NEXT:    fld fa1, 72(a1)
+; LP64D-NEXT:    fld fa2, 80(a1)
+; LP64D-NEXT:    fld fa3, 88(a1)
+; LP64D-NEXT:    fld fa4, 96(a1)
+; LP64D-NEXT:    fld fa5, 104(a1)
+; LP64D-NEXT:    fld fa6, 112(a1)
+; LP64D-NEXT:    fld fa7, 120(a1)
+; LP64D-NEXT:    fld ft8, 128(a1)
+; LP64D-NEXT:    fld ft9, 136(a1)
+; LP64D-NEXT:    fld ft10, 144(a1)
+; LP64D-NEXT:    fld ft11, 152(a1)
+; LP64D-NEXT:    fld fs0, 160(a1)
+; LP64D-NEXT:    fld fs1, 168(a1)
+; LP64D-NEXT:    fld fs2, 176(a1)
+; LP64D-NEXT:    fld fs3, 184(a1)
+; LP64D-NEXT:    fld fs4, 192(a1)
+; LP64D-NEXT:    fld fs5, 200(a1)
+; LP64D-NEXT:    fld fs6, 208(a1)
+; LP64D-NEXT:    fld fs7, 216(a1)
+; LP64D-NEXT:    fld fs8, 248(a1)
+; LP64D-NEXT:    fld fs9, 240(a1)
+; LP64D-NEXT:    fld fs10, 232(a1)
+; LP64D-NEXT:    fld fs11, 224(a1)
+; LP64D-NEXT:    fsd fs8, 248(a1)
+; LP64D-NEXT:    fsd fs9, 240(a1)
+; LP64D-NEXT:    fsd fs10, 232(a1)
+; LP64D-NEXT:    fsd fs11, 224(a1)
+; LP64D-NEXT:    fsd fs7, 216(a1)
+; LP64D-NEXT:    fsd fs6, 208(a1)
+; LP64D-NEXT:    fsd fs5, 200(a1)
+; LP64D-NEXT:    fsd fs4, 192(a1)
+; LP64D-NEXT:    fsd fs3, 184(a1)
+; LP64D-NEXT:    fsd fs2, 176(a1)
+; LP64D-NEXT:    fsd fs1, 168(a1)
+; LP64D-NEXT:    fsd fs0, 160(a1)
+; LP64D-NEXT:    fsd ft11, 152(a1)
+; LP64D-NEXT:    fsd ft10, 144(a1)
+; LP64D-NEXT:    fsd ft9, 136(a1)
+; LP64D-NEXT:    fsd ft8, 128(a1)
+; LP64D-NEXT:    fsd fa7, 120(a1)
+; LP64D-NEXT:    fsd fa6, 112(a1)
+; LP64D-NEXT:    fsd fa5, 104(a1)
+; LP64D-NEXT:    fsd fa4, 96(a1)
+; LP64D-NEXT:    fsd fa3, 88(a1)
+; LP64D-NEXT:    fsd fa2, 80(a1)
+; LP64D-NEXT:    fsd fa1, 72(a1)
+; LP64D-NEXT:    fsd fa0, 64(a1)
+; LP64D-NEXT:    fsd ft7, 56(a1)
+; LP64D-NEXT:    fsd ft6, 48(a1)
+; LP64D-NEXT:    fsd ft5, 40(a1)
+; LP64D-NEXT:    fsd ft4, 32(a1)
+; LP64D-NEXT:    fsd ft3, 24(a1)
+; LP64D-NEXT:    fsd ft2, 16(a1)
+; LP64D-NEXT:    fsd ft1, %lo(var+8)(a0)
+; LP64D-NEXT:    fsd ft0, %lo(var)(a0)
+; LP64D-NEXT:    fld fs11, 0(sp)
+; LP64D-NEXT:    fld fs10, 8(sp)
+; LP64D-NEXT:    fld fs9, 16(sp)
+; LP64D-NEXT:    fld fs8, 24(sp)
+; LP64D-NEXT:    fld fs7, 32(sp)
+; LP64D-NEXT:    fld fs6, 40(sp)
+; LP64D-NEXT:    fld fs5, 48(sp)
+; LP64D-NEXT:    fld fs4, 56(sp)
+; LP64D-NEXT:    fld fs3, 64(sp)
+; LP64D-NEXT:    fld fs2, 72(sp)
+; LP64D-NEXT:    fld fs1, 80(sp)
+; LP64D-NEXT:    fld fs0, 88(sp)
+; LP64D-NEXT:    addi sp, sp, 96
+; LP64D-NEXT:    ret
   %val = load [32 x double], [32 x double]* @var
   store volatile [32 x double] %val, [32 x double]* @var
   ret void
@@ -117,43 +361,577 @@ define void @callee() nounwind {
 ; fs0-fs11 are preserved across calls.
 
 define void @caller() nounwind {
-; ILP32-LP64-LABEL: caller:
-; ILP32-LP64-NOT:     ft{{[1-9][0-9]*}}
-; ILP32-LP64-NOT:     fs{{[0-9]+}}
-; ILP32-LP64-NOT:     fa{{[0-9]+}}
-; ILP32-LP64:         call callee
-; ILP32-LP64-NOT:     ft{{[1-9][0-9]*}}
-; ILP32-LP64-NOT:     fs{{[0-9]+}}
-; ILP32-LP64-NOT:     fa{{[0-9]+}}
-; ILP32-LP64:         ret
+; ILP32-LABEL: caller:
+; ILP32:       # %bb.0:
+; ILP32-NEXT:    addi sp, sp, -272
+; ILP32-NEXT:    sw ra, 268(sp)
+; ILP32-NEXT:    sw s0, 264(sp)
+; ILP32-NEXT:    sw s1, 260(sp)
+; ILP32-NEXT:    lui s0, %hi(var)
+; ILP32-NEXT:    fld ft0, %lo(var)(s0)
+; ILP32-NEXT:    fsd ft0, 248(sp)
+; ILP32-NEXT:    fld ft0, %lo(var+8)(s0)
+; ILP32-NEXT:    fsd ft0, 240(sp)
+; ILP32-NEXT:    addi s1, s0, %lo(var)
+; ILP32-NEXT:    fld ft0, 16(s1)
+; ILP32-NEXT:    fsd ft0, 232(sp)
+; ILP32-NEXT:    fld ft0, 24(s1)
+; ILP32-NEXT:    fsd ft0, 224(sp)
+; ILP32-NEXT:    fld ft0, 32(s1)
+; ILP32-NEXT:    fsd ft0, 216(sp)
+; ILP32-NEXT:    fld ft0, 40(s1)
+; ILP32-NEXT:    fsd ft0, 208(sp)
+; ILP32-NEXT:    fld ft0, 48(s1)
+; ILP32-NEXT:    fsd ft0, 200(sp)
+; ILP32-NEXT:    fld ft0, 56(s1)
+; ILP32-NEXT:    fsd ft0, 192(sp)
+; ILP32-NEXT:    fld ft0, 64(s1)
+; ILP32-NEXT:    fsd ft0, 184(sp)
+; ILP32-NEXT:    fld ft0, 72(s1)
+; ILP32-NEXT:    fsd ft0, 176(sp)
+; ILP32-NEXT:    fld ft0, 80(s1)
+; ILP32-NEXT:    fsd ft0, 168(sp)
+; ILP32-NEXT:    fld ft0, 88(s1)
+; ILP32-NEXT:    fsd ft0, 160(sp)
+; ILP32-NEXT:    fld ft0, 96(s1)
+; ILP32-NEXT:    fsd ft0, 152(sp)
+; ILP32-NEXT:    fld ft0, 104(s1)
+; ILP32-NEXT:    fsd ft0, 144(sp)
+; ILP32-NEXT:    fld ft0, 112(s1)
+; ILP32-NEXT:    fsd ft0, 136(sp)
+; ILP32-NEXT:    fld ft0, 120(s1)
+; ILP32-NEXT:    fsd ft0, 128(sp)
+; ILP32-NEXT:    fld ft0, 128(s1)
+; ILP32-NEXT:    fsd ft0, 120(sp)
+; ILP32-NEXT:    fld ft0, 136(s1)
+; ILP32-NEXT:    fsd ft0, 112(sp)
+; ILP32-NEXT:    fld ft0, 144(s1)
+; ILP32-NEXT:    fsd ft0, 104(sp)
+; ILP32-NEXT:    fld ft0, 152(s1)
+; ILP32-NEXT:    fsd ft0, 96(sp)
+; ILP32-NEXT:    fld ft0, 160(s1)
+; ILP32-NEXT:    fsd ft0, 88(sp)
+; ILP32-NEXT:    fld ft0, 168(s1)
+; ILP32-NEXT:    fsd ft0, 80(sp)
+; ILP32-NEXT:    fld ft0, 176(s1)
+; ILP32-NEXT:    fsd ft0, 72(sp)
+; ILP32-NEXT:    fld ft0, 184(s1)
+; ILP32-NEXT:    fsd ft0, 64(sp)
+; ILP32-NEXT:    fld ft0, 192(s1)
+; ILP32-NEXT:    fsd ft0, 56(sp)
+; ILP32-NEXT:    fld ft0, 200(s1)
+; ILP32-NEXT:    fsd ft0, 48(sp)
+; ILP32-NEXT:    fld ft0, 208(s1)
+; ILP32-NEXT:    fsd ft0, 40(sp)
+; ILP32-NEXT:    fld ft0, 216(s1)
+; ILP32-NEXT:    fsd ft0, 32(sp)
+; ILP32-NEXT:    fld ft0, 224(s1)
+; ILP32-NEXT:    fsd ft0, 24(sp)
+; ILP32-NEXT:    fld ft0, 232(s1)
+; ILP32-NEXT:    fsd ft0, 16(sp)
+; ILP32-NEXT:    fld ft0, 240(s1)
+; ILP32-NEXT:    fsd ft0, 8(sp)
+; ILP32-NEXT:    fld ft0, 248(s1)
+; ILP32-NEXT:    fsd ft0, 0(sp)
+; ILP32-NEXT:    call callee
+; ILP32-NEXT:    fld ft0, 0(sp)
+; ILP32-NEXT:    fsd ft0, 248(s1)
+; ILP32-NEXT:    fld ft0, 8(sp)
+; ILP32-NEXT:    fsd ft0, 240(s1)
+; ILP32-NEXT:    fld ft0, 16(sp)
+; ILP32-NEXT:    fsd ft0, 232(s1)
+; ILP32-NEXT:    fld ft0, 24(sp)
+; ILP32-NEXT:    fsd ft0, 224(s1)
+; ILP32-NEXT:    fld ft0, 32(sp)
+; ILP32-NEXT:    fsd ft0, 216(s1)
+; ILP32-NEXT:    fld ft0, 40(sp)
+; ILP32-NEXT:    fsd ft0, 208(s1)
+; ILP32-NEXT:    fld ft0, 48(sp)
+; ILP32-NEXT:    fsd ft0, 200(s1)
+; ILP32-NEXT:    fld ft0, 56(sp)
+; ILP32-NEXT:    fsd ft0, 192(s1)
+; ILP32-NEXT:    fld ft0, 64(sp)
+; ILP32-NEXT:    fsd ft0, 184(s1)
+; ILP32-NEXT:    fld ft0, 72(sp)
+; ILP32-NEXT:    fsd ft0, 176(s1)
+; ILP32-NEXT:    fld ft0, 80(sp)
+; ILP32-NEXT:    fsd ft0, 168(s1)
+; ILP32-NEXT:    fld ft0, 88(sp)
+; ILP32-NEXT:    fsd ft0, 160(s1)
+; ILP32-NEXT:    fld ft0, 96(sp)
+; ILP32-NEXT:    fsd ft0, 152(s1)
+; ILP32-NEXT:    fld ft0, 104(sp)
+; ILP32-NEXT:    fsd ft0, 144(s1)
+; ILP32-NEXT:    fld ft0, 112(sp)
+; ILP32-NEXT:    fsd ft0, 136(s1)
+; ILP32-NEXT:    fld ft0, 120(sp)
+; ILP32-NEXT:    fsd ft0, 128(s1)
+; ILP32-NEXT:    fld ft0, 128(sp)
+; ILP32-NEXT:    fsd ft0, 120(s1)
+; ILP32-NEXT:    fld ft0, 136(sp)
+; ILP32-NEXT:    fsd ft0, 112(s1)
+; ILP32-NEXT:    fld ft0, 144(sp)
+; ILP32-NEXT:    fsd ft0, 104(s1)
+; ILP32-NEXT:    fld ft0, 152(sp)
+; ILP32-NEXT:    fsd ft0, 96(s1)
+; ILP32-NEXT:    fld ft0, 160(sp)
+; ILP32-NEXT:    fsd ft0, 88(s1)
+; ILP32-NEXT:    fld ft0, 168(sp)
+; ILP32-NEXT:    fsd ft0, 80(s1)
+; ILP32-NEXT:    fld ft0, 176(sp)
+; ILP32-NEXT:    fsd ft0, 72(s1)
+; ILP32-NEXT:    fld ft0, 184(sp)
+; ILP32-NEXT:    fsd ft0, 64(s1)
+; ILP32-NEXT:    fld ft0, 192(sp)
+; ILP32-NEXT:    fsd ft0, 56(s1)
+; ILP32-NEXT:    fld ft0, 200(sp)
+; ILP32-NEXT:    fsd ft0, 48(s1)
+; ILP32-NEXT:    fld ft0, 208(sp)
+; ILP32-NEXT:    fsd ft0, 40(s1)
+; ILP32-NEXT:    fld ft0, 216(sp)
+; ILP32-NEXT:    fsd ft0, 32(s1)
+; ILP32-NEXT:    fld ft0, 224(sp)
+; ILP32-NEXT:    fsd ft0, 24(s1)
+; ILP32-NEXT:    fld ft0, 232(sp)
+; ILP32-NEXT:    fsd ft0, 16(s1)
+; ILP32-NEXT:    fld ft0, 240(sp)
+; ILP32-NEXT:    fsd ft0, %lo(var+8)(s0)
+; ILP32-NEXT:    fld ft0, 248(sp)
+; ILP32-NEXT:    fsd ft0, %lo(var)(s0)
+; ILP32-NEXT:    lw s1, 260(sp)
+; ILP32-NEXT:    lw s0, 264(sp)
+; ILP32-NEXT:    lw ra, 268(sp)
+; ILP32-NEXT:    addi sp, sp, 272
+; ILP32-NEXT:    ret
+;
+; LP64-LABEL: caller:
+; LP64:       # %bb.0:
+; LP64-NEXT:    addi sp, sp, -288
+; LP64-NEXT:    sd ra, 280(sp)
+; LP64-NEXT:    sd s0, 272(sp)
+; LP64-NEXT:    sd s1, 264(sp)
+; LP64-NEXT:    lui s0, %hi(var)
+; LP64-NEXT:    fld ft0, %lo(var)(s0)
+; LP64-NEXT:    fsd ft0, 256(sp)
+; LP64-NEXT:    fld ft0, %lo(var+8)(s0)
+; LP64-NEXT:    fsd ft0, 248(sp)
+; LP64-NEXT:    addi s1, s0, %lo(var)
+; LP64-NEXT:    fld ft0, 16(s1)
+; LP64-NEXT:    fsd ft0, 240(sp)
+; LP64-NEXT:    fld ft0, 24(s1)
+; LP64-NEXT:    fsd ft0, 232(sp)
+; LP64-NEXT:    fld ft0, 32(s1)
+; LP64-NEXT:    fsd ft0, 224(sp)
+; LP64-NEXT:    fld ft0, 40(s1)
+; LP64-NEXT:    fsd ft0, 216(sp)
+; LP64-NEXT:    fld ft0, 48(s1)
+; LP64-NEXT:    fsd ft0, 208(sp)
+; LP64-NEXT:    fld ft0, 56(s1)
+; LP64-NEXT:    fsd ft0, 200(sp)
+; LP64-NEXT:    fld ft0, 64(s1)
+; LP64-NEXT:    fsd ft0, 192(sp)
+; LP64-NEXT:    fld ft0, 72(s1)
+; LP64-NEXT:    fsd ft0, 184(sp)
+; LP64-NEXT:    fld ft0, 80(s1)
+; LP64-NEXT:    fsd ft0, 176(sp)
+; LP64-NEXT:    fld ft0, 88(s1)
+; LP64-NEXT:    fsd ft0, 168(sp)
+; LP64-NEXT:    fld ft0, 96(s1)
+; LP64-NEXT:    fsd ft0, 160(sp)
+; LP64-NEXT:    fld ft0, 104(s1)
+; LP64-NEXT:    fsd ft0, 152(sp)
+; LP64-NEXT:    fld ft0, 112(s1)
+; LP64-NEXT:    fsd ft0, 144(sp)
+; LP64-NEXT:    fld ft0, 120(s1)
+; LP64-NEXT:    fsd ft0, 136(sp)
+; LP64-NEXT:    fld ft0, 128(s1)
+; LP64-NEXT:    fsd ft0, 128(sp)
+; LP64-NEXT:    fld ft0, 136(s1)
+; LP64-NEXT:    fsd ft0, 120(sp)
+; LP64-NEXT:    fld ft0, 144(s1)
+; LP64-NEXT:    fsd ft0, 112(sp)
+; LP64-NEXT:    fld ft0, 152(s1)
+; LP64-NEXT:    fsd ft0, 104(sp)
+; LP64-NEXT:    fld ft0, 160(s1)
+; LP64-NEXT:    fsd ft0, 96(sp)
+; LP64-NEXT:    fld ft0, 168(s1)
+; LP64-NEXT:    fsd ft0, 88(sp)
+; LP64-NEXT:    fld ft0, 176(s1)
+; LP64-NEXT:    fsd ft0, 80(sp)
+; LP64-NEXT:    fld ft0, 184(s1)
+; LP64-NEXT:    fsd ft0, 72(sp)
+; LP64-NEXT:    fld ft0, 192(s1)
+; LP64-NEXT:    fsd ft0, 64(sp)
+; LP64-NEXT:    fld ft0, 200(s1)
+; LP64-NEXT:    fsd ft0, 56(sp)
+; LP64-NEXT:    fld ft0, 208(s1)
+; LP64-NEXT:    fsd ft0, 48(sp)
+; LP64-NEXT:    fld ft0, 216(s1)
+; LP64-NEXT:    fsd ft0, 40(sp)
+; LP64-NEXT:    fld ft0, 224(s1)
+; LP64-NEXT:    fsd ft0, 32(sp)
+; LP64-NEXT:    fld ft0, 232(s1)
+; LP64-NEXT:    fsd ft0, 24(sp)
+; LP64-NEXT:    fld ft0, 240(s1)
+; LP64-NEXT:    fsd ft0, 16(sp)
+; LP64-NEXT:    fld ft0, 248(s1)
+; LP64-NEXT:    fsd ft0, 8(sp)
+; LP64-NEXT:    call callee
+; LP64-NEXT:    fld ft0, 8(sp)
+; LP64-NEXT:    fsd ft0, 248(s1)
+; LP64-NEXT:    fld ft0, 16(sp)
+; LP64-NEXT:    fsd ft0, 240(s1)
+; LP64-NEXT:    fld ft0, 24(sp)
+; LP64-NEXT:    fsd ft0, 232(s1)
+; LP64-NEXT:    fld ft0, 32(sp)
+; LP64-NEXT:    fsd ft0, 224(s1)
+; LP64-NEXT:    fld ft0, 40(sp)
+; LP64-NEXT:    fsd ft0, 216(s1)
+; LP64-NEXT:    fld ft0, 48(sp)
+; LP64-NEXT:    fsd ft0, 208(s1)
+; LP64-NEXT:    fld ft0, 56(sp)
+; LP64-NEXT:    fsd ft0, 200(s1)
+; LP64-NEXT:    fld ft0, 64(sp)
+; LP64-NEXT:    fsd ft0, 192(s1)
+; LP64-NEXT:    fld ft0, 72(sp)
+; LP64-NEXT:    fsd ft0, 184(s1)
+; LP64-NEXT:    fld ft0, 80(sp)
+; LP64-NEXT:    fsd ft0, 176(s1)
+; LP64-NEXT:    fld ft0, 88(sp)
+; LP64-NEXT:    fsd ft0, 168(s1)
+; LP64-NEXT:    fld ft0, 96(sp)
+; LP64-NEXT:    fsd ft0, 160(s1)
+; LP64-NEXT:    fld ft0, 104(sp)
+; LP64-NEXT:    fsd ft0, 152(s1)
+; LP64-NEXT:    fld ft0, 112(sp)
+; LP64-NEXT:    fsd ft0, 144(s1)
+; LP64-NEXT:    fld ft0, 120(sp)
+; LP64-NEXT:    fsd ft0, 136(s1)
+; LP64-NEXT:    fld ft0, 128(sp)
+; LP64-NEXT:    fsd ft0, 128(s1)
+; LP64-NEXT:    fld ft0, 136(sp)
+; LP64-NEXT:    fsd ft0, 120(s1)
+; LP64-NEXT:    fld ft0, 144(sp)
+; LP64-NEXT:    fsd ft0, 112(s1)
+; LP64-NEXT:    fld ft0, 152(sp)
+; LP64-NEXT:    fsd ft0, 104(s1)
+; LP64-NEXT:    fld ft0, 160(sp)
+; LP64-NEXT:    fsd ft0, 96(s1)
+; LP64-NEXT:    fld ft0, 168(sp)
+; LP64-NEXT:    fsd ft0, 88(s1)
+; LP64-NEXT:    fld ft0, 176(sp)
+; LP64-NEXT:    fsd ft0, 80(s1)
+; LP64-NEXT:    fld ft0, 184(sp)
+; LP64-NEXT:    fsd ft0, 72(s1)
+; LP64-NEXT:    fld ft0, 192(sp)
+; LP64-NEXT:    fsd ft0, 64(s1)
+; LP64-NEXT:    fld ft0, 200(sp)
+; LP64-NEXT:    fsd ft0, 56(s1)
+; LP64-NEXT:    fld ft0, 208(sp)
+; LP64-NEXT:    fsd ft0, 48(s1)
+; LP64-NEXT:    fld ft0, 216(sp)
+; LP64-NEXT:    fsd ft0, 40(s1)
+; LP64-NEXT:    fld ft0, 224(sp)
+; LP64-NEXT:    fsd ft0, 32(s1)
+; LP64-NEXT:    fld ft0, 232(sp)
+; LP64-NEXT:    fsd ft0, 24(s1)
+; LP64-NEXT:    fld ft0, 240(sp)
+; LP64-NEXT:    fsd ft0, 16(s1)
+; LP64-NEXT:    fld ft0, 248(sp)
+; LP64-NEXT:    fsd ft0, %lo(var+8)(s0)
+; LP64-NEXT:    fld ft0, 256(sp)
+; LP64-NEXT:    fsd ft0, %lo(var)(s0)
+; LP64-NEXT:    ld s1, 264(sp)
+; LP64-NEXT:    ld s0, 272(sp)
+; LP64-NEXT:    ld ra, 280(sp)
+; LP64-NEXT:    addi sp, sp, 288
+; LP64-NEXT:    ret
+;
+; ILP32D-LABEL: caller:
+; ILP32D:       # %bb.0:
+; ILP32D-NEXT:    addi sp, sp, -272
+; ILP32D-NEXT:    sw ra, 268(sp)
+; ILP32D-NEXT:    sw s0, 264(sp)
+; ILP32D-NEXT:    sw s1, 260(sp)
+; ILP32D-NEXT:    fsd fs0, 248(sp)
+; ILP32D-NEXT:    fsd fs1, 240(sp)
+; ILP32D-NEXT:    fsd fs2, 232(sp)
+; ILP32D-NEXT:    fsd fs3, 224(sp)
+; ILP32D-NEXT:    fsd fs4, 216(sp)
+; ILP32D-NEXT:    fsd fs5, 208(sp)
+; ILP32D-NEXT:    fsd fs6, 200(sp)
+; ILP32D-NEXT:    fsd fs7, 192(sp)
+; ILP32D-NEXT:    fsd fs8, 184(sp)
+; ILP32D-NEXT:    fsd fs9, 176(sp)
+; ILP32D-NEXT:    fsd fs10, 168(sp)
+; ILP32D-NEXT:    fsd fs11, 160(sp)
+; ILP32D-NEXT:    lui s0, %hi(var)
+; ILP32D-NEXT:    fld ft0, %lo(var)(s0)
+; ILP32D-NEXT:    fsd ft0, 152(sp)
+; ILP32D-NEXT:    fld ft0, %lo(var+8)(s0)
+; ILP32D-NEXT:    fsd ft0, 144(sp)
+; ILP32D-NEXT:    addi s1, s0, %lo(var)
+; ILP32D-NEXT:    fld ft0, 16(s1)
+; ILP32D-NEXT:    fsd ft0, 136(sp)
+; ILP32D-NEXT:    fld ft0, 24(s1)
+; ILP32D-NEXT:    fsd ft0, 128(sp)
+; ILP32D-NEXT:    fld ft0, 32(s1)
+; ILP32D-NEXT:    fsd ft0, 120(sp)
+; ILP32D-NEXT:    fld ft0, 40(s1)
+; ILP32D-NEXT:    fsd ft0, 112(sp)
+; ILP32D-NEXT:    fld ft0, 48(s1)
+; ILP32D-NEXT:    fsd ft0, 104(sp)
+; ILP32D-NEXT:    fld ft0, 56(s1)
+; ILP32D-NEXT:    fsd ft0, 96(sp)
+; ILP32D-NEXT:    fld ft0, 64(s1)
+; ILP32D-NEXT:    fsd ft0, 88(sp)
+; ILP32D-NEXT:    fld ft0, 72(s1)
+; ILP32D-NEXT:    fsd ft0, 80(sp)
+; ILP32D-NEXT:    fld ft0, 80(s1)
+; ILP32D-NEXT:    fsd ft0, 72(sp)
+; ILP32D-NEXT:    fld ft0, 88(s1)
+; ILP32D-NEXT:    fsd ft0, 64(sp)
+; ILP32D-NEXT:    fld ft0, 96(s1)
+; ILP32D-NEXT:    fsd ft0, 56(sp)
+; ILP32D-NEXT:    fld ft0, 104(s1)
+; ILP32D-NEXT:    fsd ft0, 48(sp)
+; ILP32D-NEXT:    fld ft0, 112(s1)
+; ILP32D-NEXT:    fsd ft0, 40(sp)
+; ILP32D-NEXT:    fld ft0, 120(s1)
+; ILP32D-NEXT:    fsd ft0, 32(sp)
+; ILP32D-NEXT:    fld ft0, 128(s1)
+; ILP32D-NEXT:    fsd ft0, 24(sp)
+; ILP32D-NEXT:    fld ft0, 136(s1)
+; ILP32D-NEXT:    fsd ft0, 16(sp)
+; ILP32D-NEXT:    fld ft0, 144(s1)
+; ILP32D-NEXT:    fsd ft0, 8(sp)
+; ILP32D-NEXT:    fld ft0, 152(s1)
+; ILP32D-NEXT:    fsd ft0, 0(sp)
+; ILP32D-NEXT:    fld fs8, 160(s1)
+; ILP32D-NEXT:    fld fs9, 168(s1)
+; ILP32D-NEXT:    fld fs10, 176(s1)
+; ILP32D-NEXT:    fld fs11, 184(s1)
+; ILP32D-NEXT:    fld fs0, 192(s1)
+; ILP32D-NEXT:    fld fs1, 200(s1)
+; ILP32D-NEXT:    fld fs2, 208(s1)
+; ILP32D-NEXT:    fld fs3, 216(s1)
+; ILP32D-NEXT:    fld fs4, 224(s1)
+; ILP32D-NEXT:    fld fs5, 232(s1)
+; ILP32D-NEXT:    fld fs6, 240(s1)
+; ILP32D-NEXT:    fld fs7, 248(s1)
+; ILP32D-NEXT:    call callee
+; ILP32D-NEXT:    fsd fs7, 248(s1)
+; ILP32D-NEXT:    fsd fs6, 240(s1)
+; ILP32D-NEXT:    fsd fs5, 232(s1)
+; ILP32D-NEXT:    fsd fs4, 224(s1)
+; ILP32D-NEXT:    fsd fs3, 216(s1)
+; ILP32D-NEXT:    fsd fs2, 208(s1)
+; ILP32D-NEXT:    fsd fs1, 200(s1)
+; ILP32D-NEXT:    fsd fs0, 192(s1)
+; ILP32D-NEXT:    fsd fs11, 184(s1)
+; ILP32D-NEXT:    fsd fs10, 176(s1)
+; ILP32D-NEXT:    fsd fs9, 168(s1)
+; ILP32D-NEXT:    fsd fs8, 160(s1)
+; ILP32D-NEXT:    fld ft0, 0(sp)
+; ILP32D-NEXT:    fsd ft0, 152(s1)
+; ILP32D-NEXT:    fld ft0, 8(sp)
+; ILP32D-NEXT:    fsd ft0, 144(s1)
+; ILP32D-NEXT:    fld ft0, 16(sp)
+; ILP32D-NEXT:    fsd ft0, 136(s1)
+; ILP32D-NEXT:    fld ft0, 24(sp)
+; ILP32D-NEXT:    fsd ft0, 128(s1)
+; ILP32D-NEXT:    fld ft0, 32(sp)
+; ILP32D-NEXT:    fsd ft0, 120(s1)
+; ILP32D-NEXT:    fld ft0, 40(sp)
+; ILP32D-NEXT:    fsd ft0, 112(s1)
+; ILP32D-NEXT:    fld ft0, 48(sp)
+; ILP32D-NEXT:    fsd ft0, 104(s1)
+; ILP32D-NEXT:    fld ft0, 56(sp)
+; ILP32D-NEXT:    fsd ft0, 96(s1)
+; ILP32D-NEXT:    fld ft0, 64(sp)
+; ILP32D-NEXT:    fsd ft0, 88(s1)
+; ILP32D-NEXT:    fld ft0, 72(sp)
+; ILP32D-NEXT:    fsd ft0, 80(s1)
+; ILP32D-NEXT:    fld ft0, 80(sp)
+; ILP32D-NEXT:    fsd ft0, 72(s1)
+; ILP32D-NEXT:    fld ft0, 88(sp)
+; ILP32D-NEXT:    fsd ft0, 64(s1)
+; ILP32D-NEXT:    fld ft0, 96(sp)
+; ILP32D-NEXT:    fsd ft0, 56(s1)
+; ILP32D-NEXT:    fld ft0, 104(sp)
+; ILP32D-NEXT:    fsd ft0, 48(s1)
+; ILP32D-NEXT:    fld ft0, 112(sp)
+; ILP32D-NEXT:    fsd ft0, 40(s1)
+; ILP32D-NEXT:    fld ft0, 120(sp)
+; ILP32D-NEXT:    fsd ft0, 32(s1)
+; ILP32D-NEXT:    fld ft0, 128(sp)
+; ILP32D-NEXT:    fsd ft0, 24(s1)
+; ILP32D-NEXT:    fld ft0, 136(sp)
+; ILP32D-NEXT:    fsd ft0, 16(s1)
+; ILP32D-NEXT:    fld ft0, 144(sp)
+; ILP32D-NEXT:    fsd ft0, %lo(var+8)(s0)
+; ILP32D-NEXT:    fld ft0, 152(sp)
+; ILP32D-NEXT:    fsd ft0, %lo(var)(s0)
+; ILP32D-NEXT:    fld fs11, 160(sp)
+; ILP32D-NEXT:    fld fs10, 168(sp)
+; ILP32D-NEXT:    fld fs9, 176(sp)
+; ILP32D-NEXT:    fld fs8, 184(sp)
+; ILP32D-NEXT:    fld fs7, 192(sp)
+; ILP32D-NEXT:    fld fs6, 200(sp)
+; ILP32D-NEXT:    fld fs5, 208(sp)
+; ILP32D-NEXT:    fld fs4, 216(sp)
+; ILP32D-NEXT:    fld fs3, 224(sp)
+; ILP32D-NEXT:    fld fs2, 232(sp)
+; ILP32D-NEXT:    fld fs1, 240(sp)
+; ILP32D-NEXT:    fld fs0, 248(sp)
+; ILP32D-NEXT:    lw s1, 260(sp)
+; ILP32D-NEXT:    lw s0, 264(sp)
+; ILP32D-NEXT:    lw ra, 268(sp)
+; ILP32D-NEXT:    addi sp, sp, 272
+; ILP32D-NEXT:    ret
 ;
-; ILP32F-LP64D-LABEL: caller:
-; ILP32D-LP64D: fld	fs8, 160(s1)
-; ILP32D-LP64D-NEXT: fld fs9, 168(s1)
-; ILP32D-LP64D-NEXT: fld fs10, 176(s1)
-; ILP32D-LP64D-NEXT: fld fs11, 184(s1)
-; ILP32D-LP64D-NEXT: fld fs0, 192(s1)
-; ILP32D-LP64D-NEXT: fld fs1, 200(s1)
-; ILP32D-LP64D-NEXT: fld fs2, 208(s1)
-; ILP32D-LP64D-NEXT: fld fs3, 216(s1)
-; ILP32D-LP64D-NEXT: fld fs4, 224(s1)
-; ILP32D-LP64D-NEXT: fld fs5, 232(s1)
-; ILP32D-LP64D-NEXT: fld fs6, 240(s1)
-; ILP32D-LP64D-NEXT: fld fs7, 248(s1)
-; ILP32D-LP64D-NEXT: call	callee
-; ILP32D-LP64D-NEXT: fsd fs7, 248(s1)
-; ILP32D-LP64D-NEXT: fsd fs6, 240(s1)
-; ILP32D-LP64D-NEXT: fsd fs5, 232(s1)
-; ILP32D-LP64D-NEXT: fsd fs4, 224(s1)
-; ILP32D-LP64D-NEXT: fsd fs3, 216(s1)
-; ILP32D-LP64D-NEXT: fsd fs2, 208(s1)
-; ILP32D-LP64D-NEXT: fsd fs1, 200(s1)
-; ILP32D-LP64D-NEXT: fsd fs0, 192(s1)
-; ILP32D-LP64D-NEXT: fsd fs11, 184(s1)
-; ILP32D-LP64D-NEXT: fsd fs10, 176(s1)
-; ILP32D-LP64D-NEXT: fsd fs9, 168(s1)
-; ILP32D-LP64D-NEXT: fsd fs8, 160(s1)
-; ILP32D-LP64D-NEXT: fld ft0, {{[0-9]+}}(sp)
+; LP64D-LABEL: caller:
+; LP64D:       # %bb.0:
+; LP64D-NEXT:    addi sp, sp, -288
+; LP64D-NEXT:    sd ra, 280(sp)
+; LP64D-NEXT:    sd s0, 272(sp)
+; LP64D-NEXT:    sd s1, 264(sp)
+; LP64D-NEXT:    fsd fs0, 256(sp)
+; LP64D-NEXT:    fsd fs1, 248(sp)
+; LP64D-NEXT:    fsd fs2, 240(sp)
+; LP64D-NEXT:    fsd fs3, 232(sp)
+; LP64D-NEXT:    fsd fs4, 224(sp)
+; LP64D-NEXT:    fsd fs5, 216(sp)
+; LP64D-NEXT:    fsd fs6, 208(sp)
+; LP64D-NEXT:    fsd fs7, 200(sp)
+; LP64D-NEXT:    fsd fs8, 192(sp)
+; LP64D-NEXT:    fsd fs9, 184(sp)
+; LP64D-NEXT:    fsd fs10, 176(sp)
+; LP64D-NEXT:    fsd fs11, 168(sp)
+; LP64D-NEXT:    lui s0, %hi(var)
+; LP64D-NEXT:    fld ft0, %lo(var)(s0)
+; LP64D-NEXT:    fsd ft0, 160(sp)
+; LP64D-NEXT:    fld ft0, %lo(var+8)(s0)
+; LP64D-NEXT:    fsd ft0, 152(sp)
+; LP64D-NEXT:    addi s1, s0, %lo(var)
+; LP64D-NEXT:    fld ft0, 16(s1)
+; LP64D-NEXT:    fsd ft0, 144(sp)
+; LP64D-NEXT:    fld ft0, 24(s1)
+; LP64D-NEXT:    fsd ft0, 136(sp)
+; LP64D-NEXT:    fld ft0, 32(s1)
+; LP64D-NEXT:    fsd ft0, 128(sp)
+; LP64D-NEXT:    fld ft0, 40(s1)
+; LP64D-NEXT:    fsd ft0, 120(sp)
+; LP64D-NEXT:    fld ft0, 48(s1)
+; LP64D-NEXT:    fsd ft0, 112(sp)
+; LP64D-NEXT:    fld ft0, 56(s1)
+; LP64D-NEXT:    fsd ft0, 104(sp)
+; LP64D-NEXT:    fld ft0, 64(s1)
+; LP64D-NEXT:    fsd ft0, 96(sp)
+; LP64D-NEXT:    fld ft0, 72(s1)
+; LP64D-NEXT:    fsd ft0, 88(sp)
+; LP64D-NEXT:    fld ft0, 80(s1)
+; LP64D-NEXT:    fsd ft0, 80(sp)
+; LP64D-NEXT:    fld ft0, 88(s1)
+; LP64D-NEXT:    fsd ft0, 72(sp)
+; LP64D-NEXT:    fld ft0, 96(s1)
+; LP64D-NEXT:    fsd ft0, 64(sp)
+; LP64D-NEXT:    fld ft0, 104(s1)
+; LP64D-NEXT:    fsd ft0, 56(sp)
+; LP64D-NEXT:    fld ft0, 112(s1)
+; LP64D-NEXT:    fsd ft0, 48(sp)
+; LP64D-NEXT:    fld ft0, 120(s1)
+; LP64D-NEXT:    fsd ft0, 40(sp)
+; LP64D-NEXT:    fld ft0, 128(s1)
+; LP64D-NEXT:    fsd ft0, 32(sp)
+; LP64D-NEXT:    fld ft0, 136(s1)
+; LP64D-NEXT:    fsd ft0, 24(sp)
+; LP64D-NEXT:    fld ft0, 144(s1)
+; LP64D-NEXT:    fsd ft0, 16(sp)
+; LP64D-NEXT:    fld ft0, 152(s1)
+; LP64D-NEXT:    fsd ft0, 8(sp)
+; LP64D-NEXT:    fld fs8, 160(s1)
+; LP64D-NEXT:    fld fs9, 168(s1)
+; LP64D-NEXT:    fld fs10, 176(s1)
+; LP64D-NEXT:    fld fs11, 184(s1)
+; LP64D-NEXT:    fld fs0, 192(s1)
+; LP64D-NEXT:    fld fs1, 200(s1)
+; LP64D-NEXT:    fld fs2, 208(s1)
+; LP64D-NEXT:    fld fs3, 216(s1)
+; LP64D-NEXT:    fld fs4, 224(s1)
+; LP64D-NEXT:    fld fs5, 232(s1)
+; LP64D-NEXT:    fld fs6, 240(s1)
+; LP64D-NEXT:    fld fs7, 248(s1)
+; LP64D-NEXT:    call callee
+; LP64D-NEXT:    fsd fs7, 248(s1)
+; LP64D-NEXT:    fsd fs6, 240(s1)
+; LP64D-NEXT:    fsd fs5, 232(s1)
+; LP64D-NEXT:    fsd fs4, 224(s1)
+; LP64D-NEXT:    fsd fs3, 216(s1)
+; LP64D-NEXT:    fsd fs2, 208(s1)
+; LP64D-NEXT:    fsd fs1, 200(s1)
+; LP64D-NEXT:    fsd fs0, 192(s1)
+; LP64D-NEXT:    fsd fs11, 184(s1)
+; LP64D-NEXT:    fsd fs10, 176(s1)
+; LP64D-NEXT:    fsd fs9, 168(s1)
+; LP64D-NEXT:    fsd fs8, 160(s1)
+; LP64D-NEXT:    fld ft0, 8(sp)
+; LP64D-NEXT:    fsd ft0, 152(s1)
+; LP64D-NEXT:    fld ft0, 16(sp)
+; LP64D-NEXT:    fsd ft0, 144(s1)
+; LP64D-NEXT:    fld ft0, 24(sp)
+; LP64D-NEXT:    fsd ft0, 136(s1)
+; LP64D-NEXT:    fld ft0, 32(sp)
+; LP64D-NEXT:    fsd ft0, 128(s1)
+; LP64D-NEXT:    fld ft0, 40(sp)
+; LP64D-NEXT:    fsd ft0, 120(s1)
+; LP64D-NEXT:    fld ft0, 48(sp)
+; LP64D-NEXT:    fsd ft0, 112(s1)
+; LP64D-NEXT:    fld ft0, 56(sp)
+; LP64D-NEXT:    fsd ft0, 104(s1)
+; LP64D-NEXT:    fld ft0, 64(sp)
+; LP64D-NEXT:    fsd ft0, 96(s1)
+; LP64D-NEXT:    fld ft0, 72(sp)
+; LP64D-NEXT:    fsd ft0, 88(s1)
+; LP64D-NEXT:    fld ft0, 80(sp)
+; LP64D-NEXT:    fsd ft0, 80(s1)
+; LP64D-NEXT:    fld ft0, 88(sp)
+; LP64D-NEXT:    fsd ft0, 72(s1)
+; LP64D-NEXT:    fld ft0, 96(sp)
+; LP64D-NEXT:    fsd ft0, 64(s1)
+; LP64D-NEXT:    fld ft0, 104(sp)
+; LP64D-NEXT:    fsd ft0, 56(s1)
+; LP64D-NEXT:    fld ft0, 112(sp)
+; LP64D-NEXT:    fsd ft0, 48(s1)
+; LP64D-NEXT:    fld ft0, 120(sp)
+; LP64D-NEXT:    fsd ft0, 40(s1)
+; LP64D-NEXT:    fld ft0, 128(sp)
+; LP64D-NEXT:    fsd ft0, 32(s1)
+; LP64D-NEXT:    fld ft0, 136(sp)
+; LP64D-NEXT:    fsd ft0, 24(s1)
+; LP64D-NEXT:    fld ft0, 144(sp)
+; LP64D-NEXT:    fsd ft0, 16(s1)
+; LP64D-NEXT:    fld ft0, 152(sp)
+; LP64D-NEXT:    fsd ft0, %lo(var+8)(s0)
+; LP64D-NEXT:    fld ft0, 160(sp)
+; LP64D-NEXT:    fsd ft0, %lo(var)(s0)
+; LP64D-NEXT:    fld fs11, 168(sp)
+; LP64D-NEXT:    fld fs10, 176(sp)
+; LP64D-NEXT:    fld fs9, 184(sp)
+; LP64D-NEXT:    fld fs8, 192(sp)
+; LP64D-NEXT:    fld fs7, 200(sp)
+; LP64D-NEXT:    fld fs6, 208(sp)
+; LP64D-NEXT:    fld fs5, 216(sp)
+; LP64D-NEXT:    fld fs4, 224(sp)
+; LP64D-NEXT:    fld fs3, 232(sp)
+; LP64D-NEXT:    fld fs2, 240(sp)
+; LP64D-NEXT:    fld fs1, 248(sp)
+; LP64D-NEXT:    fld fs0, 256(sp)
+; LP64D-NEXT:    ld s1, 264(sp)
+; LP64D-NEXT:    ld s0, 272(sp)
+; LP64D-NEXT:    ld ra, 280(sp)
+; LP64D-NEXT:    addi sp, sp, 288
+; LP64D-NEXT:    ret
   %val = load [32 x double], [32 x double]* @var
   call void @callee()
   store volatile [32 x double] %val, [32 x double]* @var
diff --git a/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll b/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
index eb3a4468bb9d1..99c07a35226c3 100644
--- a/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
+++ b/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s -check-prefix=RV32I
 ; RUN: llc -mtriple=riscv32 -mattr=+f -target-abi ilp32f -verify-machineinstrs < %s \
@@ -41,10 +42,99 @@ define void @callee() nounwind {
 ; RV32I-NEXT:    sw s9, 36(sp)
 ; RV32I-NEXT:    sw s10, 32(sp)
 ; RV32I-NEXT:    sw s11, 28(sp)
-; RV32I-NEXT:    lui a0, %hi(var)
-; RV32I-NEXT:    lw a1, %lo(var)(a0)
-; RV32I-NEXT:    sw a1, 24(sp)
-; RV32I-NEXT:    addi a2, a0, %lo(var)
+; RV32I-NEXT:    lui a7, %hi(var)
+; RV32I-NEXT:    lw a0, %lo(var)(a7)
+; RV32I-NEXT:    sw a0, 24(sp)
+; RV32I-NEXT:    lw a0, %lo(var+4)(a7)
+; RV32I-NEXT:    sw a0, 20(sp)
+; RV32I-NEXT:    lw a0, %lo(var+8)(a7)
+; RV32I-NEXT:    sw a0, 16(sp)
+; RV32I-NEXT:    lw a0, %lo(var+12)(a7)
+; RV32I-NEXT:    sw a0, 12(sp)
+; RV32I-NEXT:    addi a5, a7, %lo(var)
+; RV32I-NEXT:    lw a0, 16(a5)
+; RV32I-NEXT:    sw a0, 8(sp)
+; RV32I-NEXT:    lw a0, 20(a5)
+; RV32I-NEXT:    sw a0, 4(sp)
+; RV32I-NEXT:    lw t4, 24(a5)
+; RV32I-NEXT:    lw t5, 28(a5)
+; RV32I-NEXT:    lw t6, 32(a5)
+; RV32I-NEXT:    lw s2, 36(a5)
+; RV32I-NEXT:    lw s3, 40(a5)
+; RV32I-NEXT:    lw s4, 44(a5)
+; RV32I-NEXT:    lw s5, 48(a5)
+; RV32I-NEXT:    lw s6, 52(a5)
+; RV32I-NEXT:    lw s7, 56(a5)
+; RV32I-NEXT:    lw s8, 60(a5)
+; RV32I-NEXT:    lw s9, 64(a5)
+; RV32I-NEXT:    lw s10, 68(a5)
+; RV32I-NEXT:    lw s11, 72(a5)
+; RV32I-NEXT:    lw ra, 76(a5)
+; RV32I-NEXT:    lw s1, 80(a5)
+; RV32I-NEXT:    lw t3, 84(a5)
+; RV32I-NEXT:    lw t2, 88(a5)
+; RV32I-NEXT:    lw t1, 92(a5)
+; RV32I-NEXT:    lw t0, 96(a5)
+; RV32I-NEXT:    lw s0, 100(a5)
+; RV32I-NEXT:    lw a6, 104(a5)
+; RV32I-NEXT:    lw a4, 108(a5)
+; RV32I-NEXT:    lw a0, 124(a5)
+; RV32I-NEXT:    lw a1, 120(a5)
+; RV32I-NEXT:    lw a2, 116(a5)
+; RV32I-NEXT:    lw a3, 112(a5)
+; RV32I-NEXT:    sw a0, 124(a5)
+; RV32I-NEXT:    sw a1, 120(a5)
+; RV32I-NEXT:    sw a2, 116(a5)
+; RV32I-NEXT:    sw a3, 112(a5)
+; RV32I-NEXT:    sw a4, 108(a5)
+; RV32I-NEXT:    sw a6, 104(a5)
+; RV32I-NEXT:    sw s0, 100(a5)
+; RV32I-NEXT:    sw t0, 96(a5)
+; RV32I-NEXT:    sw t1, 92(a5)
+; RV32I-NEXT:    sw t2, 88(a5)
+; RV32I-NEXT:    sw t3, 84(a5)
+; RV32I-NEXT:    sw s1, 80(a5)
+; RV32I-NEXT:    sw ra, 76(a5)
+; RV32I-NEXT:    sw s11, 72(a5)
+; RV32I-NEXT:    sw s10, 68(a5)
+; RV32I-NEXT:    sw s9, 64(a5)
+; RV32I-NEXT:    sw s8, 60(a5)
+; RV32I-NEXT:    sw s7, 56(a5)
+; RV32I-NEXT:    sw s6, 52(a5)
+; RV32I-NEXT:    sw s5, 48(a5)
+; RV32I-NEXT:    sw s4, 44(a5)
+; RV32I-NEXT:    sw s3, 40(a5)
+; RV32I-NEXT:    sw s2, 36(a5)
+; RV32I-NEXT:    sw t6, 32(a5)
+; RV32I-NEXT:    sw t5, 28(a5)
+; RV32I-NEXT:    sw t4, 24(a5)
+; RV32I-NEXT:    lw a0, 4(sp)
+; RV32I-NEXT:    sw a0, 20(a5)
+; RV32I-NEXT:    lw a0, 8(sp)
+; RV32I-NEXT:    sw a0, 16(a5)
+; RV32I-NEXT:    lw a0, 12(sp)
+; RV32I-NEXT:    sw a0, %lo(var+12)(a7)
+; RV32I-NEXT:    lw a0, 16(sp)
+; RV32I-NEXT:    sw a0, %lo(var+8)(a7)
+; RV32I-NEXT:    lw a0, 20(sp)
+; RV32I-NEXT:    sw a0, %lo(var+4)(a7)
+; RV32I-NEXT:    lw a0, 24(sp)
+; RV32I-NEXT:    sw a0, %lo(var)(a7)
+; RV32I-NEXT:    lw s11, 28(sp)
+; RV32I-NEXT:    lw s10, 32(sp)
+; RV32I-NEXT:    lw s9, 36(sp)
+; RV32I-NEXT:    lw s8, 40(sp)
+; RV32I-NEXT:    lw s7, 44(sp)
+; RV32I-NEXT:    lw s6, 48(sp)
+; RV32I-NEXT:    lw s5, 52(sp)
+; RV32I-NEXT:    lw s4, 56(sp)
+; RV32I-NEXT:    lw s3, 60(sp)
+; RV32I-NEXT:    lw s2, 64(sp)
+; RV32I-NEXT:    lw s1, 68(sp)
+; RV32I-NEXT:    lw s0, 72(sp)
+; RV32I-NEXT:    lw ra, 76(sp)
+; RV32I-NEXT:    addi sp, sp, 80
+; RV32I-NEXT:    ret
 ;
 ; RV32I-WITH-FP-LABEL: callee:
 ; RV32I-WITH-FP:       # %bb.0:
@@ -63,31 +153,211 @@ define void @callee() nounwind {
 ; RV32I-WITH-FP-NEXT:    sw s10, 32(sp)
 ; RV32I-WITH-FP-NEXT:    sw s11, 28(sp)
 ; RV32I-WITH-FP-NEXT:    addi s0, sp, 80
-; RV32I-WITH-FP-NEXT:    lui a0, %hi(var)
-; RV32I-WITH-FP-NEXT:    lw a1, %lo(var)(a0)
-; RV32I-WITH-FP-NEXT:    sw a1, -56(s0)
-; RV32I-WITH-FP-NEXT:    addi a2, a0, %lo(var)
+; RV32I-WITH-FP-NEXT:    lui a7, %hi(var)
+; RV32I-WITH-FP-NEXT:    lw a0, %lo(var)(a7)
+; RV32I-WITH-FP-NEXT:    sw a0, -56(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, %lo(var+4)(a7)
+; RV32I-WITH-FP-NEXT:    sw a0, -60(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, %lo(var+8)(a7)
+; RV32I-WITH-FP-NEXT:    sw a0, -64(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, %lo(var+12)(a7)
+; RV32I-WITH-FP-NEXT:    sw a0, -68(s0)
+; RV32I-WITH-FP-NEXT:    addi a5, a7, %lo(var)
+; RV32I-WITH-FP-NEXT:    lw a0, 16(a5)
+; RV32I-WITH-FP-NEXT:    sw a0, -72(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 20(a5)
+; RV32I-WITH-FP-NEXT:    sw a0, -76(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 24(a5)
+; RV32I-WITH-FP-NEXT:    sw a0, -80(s0)
+; RV32I-WITH-FP-NEXT:    lw t5, 28(a5)
+; RV32I-WITH-FP-NEXT:    lw t6, 32(a5)
+; RV32I-WITH-FP-NEXT:    lw s2, 36(a5)
+; RV32I-WITH-FP-NEXT:    lw s3, 40(a5)
+; RV32I-WITH-FP-NEXT:    lw s4, 44(a5)
+; RV32I-WITH-FP-NEXT:    lw s5, 48(a5)
+; RV32I-WITH-FP-NEXT:    lw s6, 52(a5)
+; RV32I-WITH-FP-NEXT:    lw s7, 56(a5)
+; RV32I-WITH-FP-NEXT:    lw s8, 60(a5)
+; RV32I-WITH-FP-NEXT:    lw s9, 64(a5)
+; RV32I-WITH-FP-NEXT:    lw s10, 68(a5)
+; RV32I-WITH-FP-NEXT:    lw s11, 72(a5)
+; RV32I-WITH-FP-NEXT:    lw ra, 76(a5)
+; RV32I-WITH-FP-NEXT:    lw t4, 80(a5)
+; RV32I-WITH-FP-NEXT:    lw t3, 84(a5)
+; RV32I-WITH-FP-NEXT:    lw t2, 88(a5)
+; RV32I-WITH-FP-NEXT:    lw s1, 92(a5)
+; RV32I-WITH-FP-NEXT:    lw t1, 96(a5)
+; RV32I-WITH-FP-NEXT:    lw t0, 100(a5)
+; RV32I-WITH-FP-NEXT:    lw a6, 104(a5)
+; RV32I-WITH-FP-NEXT:    lw a4, 108(a5)
+; RV32I-WITH-FP-NEXT:    lw a0, 124(a5)
+; RV32I-WITH-FP-NEXT:    lw a1, 120(a5)
+; RV32I-WITH-FP-NEXT:    lw a2, 116(a5)
+; RV32I-WITH-FP-NEXT:    lw a3, 112(a5)
+; RV32I-WITH-FP-NEXT:    sw a0, 124(a5)
+; RV32I-WITH-FP-NEXT:    sw a1, 120(a5)
+; RV32I-WITH-FP-NEXT:    sw a2, 116(a5)
+; RV32I-WITH-FP-NEXT:    sw a3, 112(a5)
+; RV32I-WITH-FP-NEXT:    sw a4, 108(a5)
+; RV32I-WITH-FP-NEXT:    sw a6, 104(a5)
+; RV32I-WITH-FP-NEXT:    sw t0, 100(a5)
+; RV32I-WITH-FP-NEXT:    sw t1, 96(a5)
+; RV32I-WITH-FP-NEXT:    sw s1, 92(a5)
+; RV32I-WITH-FP-NEXT:    sw t2, 88(a5)
+; RV32I-WITH-FP-NEXT:    sw t3, 84(a5)
+; RV32I-WITH-FP-NEXT:    sw t4, 80(a5)
+; RV32I-WITH-FP-NEXT:    sw ra, 76(a5)
+; RV32I-WITH-FP-NEXT:    sw s11, 72(a5)
+; RV32I-WITH-FP-NEXT:    sw s10, 68(a5)
+; RV32I-WITH-FP-NEXT:    sw s9, 64(a5)
+; RV32I-WITH-FP-NEXT:    sw s8, 60(a5)
+; RV32I-WITH-FP-NEXT:    sw s7, 56(a5)
+; RV32I-WITH-FP-NEXT:    sw s6, 52(a5)
+; RV32I-WITH-FP-NEXT:    sw s5, 48(a5)
+; RV32I-WITH-FP-NEXT:    sw s4, 44(a5)
+; RV32I-WITH-FP-NEXT:    sw s3, 40(a5)
+; RV32I-WITH-FP-NEXT:    sw s2, 36(a5)
+; RV32I-WITH-FP-NEXT:    sw t6, 32(a5)
+; RV32I-WITH-FP-NEXT:    sw t5, 28(a5)
+; RV32I-WITH-FP-NEXT:    lw a0, -80(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 24(a5)
+; RV32I-WITH-FP-NEXT:    lw a0, -76(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 20(a5)
+; RV32I-WITH-FP-NEXT:    lw a0, -72(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 16(a5)
+; RV32I-WITH-FP-NEXT:    lw a0, -68(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, %lo(var+12)(a7)
+; RV32I-WITH-FP-NEXT:    lw a0, -64(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, %lo(var+8)(a7)
+; RV32I-WITH-FP-NEXT:    lw a0, -60(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, %lo(var+4)(a7)
+; RV32I-WITH-FP-NEXT:    lw a0, -56(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, %lo(var)(a7)
+; RV32I-WITH-FP-NEXT:    lw s11, 28(sp)
+; RV32I-WITH-FP-NEXT:    lw s10, 32(sp)
+; RV32I-WITH-FP-NEXT:    lw s9, 36(sp)
+; RV32I-WITH-FP-NEXT:    lw s8, 40(sp)
+; RV32I-WITH-FP-NEXT:    lw s7, 44(sp)
+; RV32I-WITH-FP-NEXT:    lw s6, 48(sp)
+; RV32I-WITH-FP-NEXT:    lw s5, 52(sp)
+; RV32I-WITH-FP-NEXT:    lw s4, 56(sp)
+; RV32I-WITH-FP-NEXT:    lw s3, 60(sp)
+; RV32I-WITH-FP-NEXT:    lw s2, 64(sp)
+; RV32I-WITH-FP-NEXT:    lw s1, 68(sp)
+; RV32I-WITH-FP-NEXT:    lw s0, 72(sp)
+; RV32I-WITH-FP-NEXT:    lw ra, 76(sp)
+; RV32I-WITH-FP-NEXT:    addi sp, sp, 80
+; RV32I-WITH-FP-NEXT:    ret
 ;
 ; RV64I-LABEL: callee:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -144
-; RV64I-NEXT:    sd ra, 136(sp)
-; RV64I-NEXT:    sd s0, 128(sp)
-; RV64I-NEXT:    sd s1, 120(sp)
-; RV64I-NEXT:    sd s2, 112(sp)
-; RV64I-NEXT:    sd s3, 104(sp)
-; RV64I-NEXT:    sd s4, 96(sp)
-; RV64I-NEXT:    sd s5, 88(sp)
-; RV64I-NEXT:    sd s6, 80(sp)
-; RV64I-NEXT:    sd s7, 72(sp)
-; RV64I-NEXT:    sd s8, 64(sp)
-; RV64I-NEXT:    sd s9, 56(sp)
-; RV64I-NEXT:    sd s10, 48(sp)
-; RV64I-NEXT:    sd s11, 40(sp)
-; RV64I-NEXT:    lui a0, %hi(var)
-; RV64I-NEXT:    lw a1, %lo(var)(a0)
-; RV64I-NEXT:    sd a1, 32(sp)
-; RV64I-NEXT:    addi a2, a0, %lo(var)
+; RV64I-NEXT:    addi sp, sp, -160
+; RV64I-NEXT:    sd ra, 152(sp)
+; RV64I-NEXT:    sd s0, 144(sp)
+; RV64I-NEXT:    sd s1, 136(sp)
+; RV64I-NEXT:    sd s2, 128(sp)
+; RV64I-NEXT:    sd s3, 120(sp)
+; RV64I-NEXT:    sd s4, 112(sp)
+; RV64I-NEXT:    sd s5, 104(sp)
+; RV64I-NEXT:    sd s6, 96(sp)
+; RV64I-NEXT:    sd s7, 88(sp)
+; RV64I-NEXT:    sd s8, 80(sp)
+; RV64I-NEXT:    sd s9, 72(sp)
+; RV64I-NEXT:    sd s10, 64(sp)
+; RV64I-NEXT:    sd s11, 56(sp)
+; RV64I-NEXT:    lui a7, %hi(var)
+; RV64I-NEXT:    lw a0, %lo(var)(a7)
+; RV64I-NEXT:    sd a0, 48(sp)
+; RV64I-NEXT:    lw a0, %lo(var+4)(a7)
+; RV64I-NEXT:    sd a0, 40(sp)
+; RV64I-NEXT:    lw a0, %lo(var+8)(a7)
+; RV64I-NEXT:    sd a0, 32(sp)
+; RV64I-NEXT:    lw a0, %lo(var+12)(a7)
+; RV64I-NEXT:    sd a0, 24(sp)
+; RV64I-NEXT:    addi a5, a7, %lo(var)
+; RV64I-NEXT:    lw a0, 16(a5)
+; RV64I-NEXT:    sd a0, 16(sp)
+; RV64I-NEXT:    lw a0, 20(a5)
+; RV64I-NEXT:    sd a0, 8(sp)
+; RV64I-NEXT:    lw t4, 24(a5)
+; RV64I-NEXT:    lw t5, 28(a5)
+; RV64I-NEXT:    lw t6, 32(a5)
+; RV64I-NEXT:    lw s2, 36(a5)
+; RV64I-NEXT:    lw s3, 40(a5)
+; RV64I-NEXT:    lw s4, 44(a5)
+; RV64I-NEXT:    lw s5, 48(a5)
+; RV64I-NEXT:    lw s6, 52(a5)
+; RV64I-NEXT:    lw s7, 56(a5)
+; RV64I-NEXT:    lw s8, 60(a5)
+; RV64I-NEXT:    lw s9, 64(a5)
+; RV64I-NEXT:    lw s10, 68(a5)
+; RV64I-NEXT:    lw s11, 72(a5)
+; RV64I-NEXT:    lw ra, 76(a5)
+; RV64I-NEXT:    lw s1, 80(a5)
+; RV64I-NEXT:    lw t3, 84(a5)
+; RV64I-NEXT:    lw t2, 88(a5)
+; RV64I-NEXT:    lw t1, 92(a5)
+; RV64I-NEXT:    lw t0, 96(a5)
+; RV64I-NEXT:    lw s0, 100(a5)
+; RV64I-NEXT:    lw a6, 104(a5)
+; RV64I-NEXT:    lw a4, 108(a5)
+; RV64I-NEXT:    lw a0, 124(a5)
+; RV64I-NEXT:    lw a1, 120(a5)
+; RV64I-NEXT:    lw a2, 116(a5)
+; RV64I-NEXT:    lw a3, 112(a5)
+; RV64I-NEXT:    sw a0, 124(a5)
+; RV64I-NEXT:    sw a1, 120(a5)
+; RV64I-NEXT:    sw a2, 116(a5)
+; RV64I-NEXT:    sw a3, 112(a5)
+; RV64I-NEXT:    sw a4, 108(a5)
+; RV64I-NEXT:    sw a6, 104(a5)
+; RV64I-NEXT:    sw s0, 100(a5)
+; RV64I-NEXT:    sw t0, 96(a5)
+; RV64I-NEXT:    sw t1, 92(a5)
+; RV64I-NEXT:    sw t2, 88(a5)
+; RV64I-NEXT:    sw t3, 84(a5)
+; RV64I-NEXT:    sw s1, 80(a5)
+; RV64I-NEXT:    sw ra, 76(a5)
+; RV64I-NEXT:    sw s11, 72(a5)
+; RV64I-NEXT:    sw s10, 68(a5)
+; RV64I-NEXT:    sw s9, 64(a5)
+; RV64I-NEXT:    sw s8, 60(a5)
+; RV64I-NEXT:    sw s7, 56(a5)
+; RV64I-NEXT:    sw s6, 52(a5)
+; RV64I-NEXT:    sw s5, 48(a5)
+; RV64I-NEXT:    sw s4, 44(a5)
+; RV64I-NEXT:    sw s3, 40(a5)
+; RV64I-NEXT:    sw s2, 36(a5)
+; RV64I-NEXT:    sw t6, 32(a5)
+; RV64I-NEXT:    sw t5, 28(a5)
+; RV64I-NEXT:    sw t4, 24(a5)
+; RV64I-NEXT:    ld a0, 8(sp)
+; RV64I-NEXT:    sw a0, 20(a5)
+; RV64I-NEXT:    ld a0, 16(sp)
+; RV64I-NEXT:    sw a0, 16(a5)
+; RV64I-NEXT:    ld a0, 24(sp)
+; RV64I-NEXT:    sw a0, %lo(var+12)(a7)
+; RV64I-NEXT:    ld a0, 32(sp)
+; RV64I-NEXT:    sw a0, %lo(var+8)(a7)
+; RV64I-NEXT:    ld a0, 40(sp)
+; RV64I-NEXT:    sw a0, %lo(var+4)(a7)
+; RV64I-NEXT:    ld a0, 48(sp)
+; RV64I-NEXT:    sw a0, %lo(var)(a7)
+; RV64I-NEXT:    ld s11, 56(sp)
+; RV64I-NEXT:    ld s10, 64(sp)
+; RV64I-NEXT:    ld s9, 72(sp)
+; RV64I-NEXT:    ld s8, 80(sp)
+; RV64I-NEXT:    ld s7, 88(sp)
+; RV64I-NEXT:    ld s6, 96(sp)
+; RV64I-NEXT:    ld s5, 104(sp)
+; RV64I-NEXT:    ld s4, 112(sp)
+; RV64I-NEXT:    ld s3, 120(sp)
+; RV64I-NEXT:    ld s2, 128(sp)
+; RV64I-NEXT:    ld s1, 136(sp)
+; RV64I-NEXT:    ld s0, 144(sp)
+; RV64I-NEXT:    ld ra, 152(sp)
+; RV64I-NEXT:    addi sp, sp, 160
+; RV64I-NEXT:    ret
 ;
 ; RV64I-WITH-FP-LABEL: callee:
 ; RV64I-WITH-FP:       # %bb.0:
@@ -106,10 +376,101 @@ define void @callee() nounwind {
 ; RV64I-WITH-FP-NEXT:    sd s10, 64(sp)
 ; RV64I-WITH-FP-NEXT:    sd s11, 56(sp)
 ; RV64I-WITH-FP-NEXT:    addi s0, sp, 160
-; RV64I-WITH-FP-NEXT:    lui a0, %hi(var)
-; RV64I-WITH-FP-NEXT:    lw a1, %lo(var)(a0)
-; RV64I-WITH-FP-NEXT:    sd a1, -112(s0)
-; RV64I-WITH-FP-NEXT:    addi a2, a0, %lo(var)
+; RV64I-WITH-FP-NEXT:    lui a7, %hi(var)
+; RV64I-WITH-FP-NEXT:    lw a0, %lo(var)(a7)
+; RV64I-WITH-FP-NEXT:    sd a0, -112(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, %lo(var+4)(a7)
+; RV64I-WITH-FP-NEXT:    sd a0, -120(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, %lo(var+8)(a7)
+; RV64I-WITH-FP-NEXT:    sd a0, -128(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, %lo(var+12)(a7)
+; RV64I-WITH-FP-NEXT:    sd a0, -136(s0)
+; RV64I-WITH-FP-NEXT:    addi a5, a7, %lo(var)
+; RV64I-WITH-FP-NEXT:    lw a0, 16(a5)
+; RV64I-WITH-FP-NEXT:    sd a0, -144(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 20(a5)
+; RV64I-WITH-FP-NEXT:    sd a0, -152(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 24(a5)
+; RV64I-WITH-FP-NEXT:    sd a0, -160(s0)
+; RV64I-WITH-FP-NEXT:    lw t5, 28(a5)
+; RV64I-WITH-FP-NEXT:    lw t6, 32(a5)
+; RV64I-WITH-FP-NEXT:    lw s2, 36(a5)
+; RV64I-WITH-FP-NEXT:    lw s3, 40(a5)
+; RV64I-WITH-FP-NEXT:    lw s4, 44(a5)
+; RV64I-WITH-FP-NEXT:    lw s5, 48(a5)
+; RV64I-WITH-FP-NEXT:    lw s6, 52(a5)
+; RV64I-WITH-FP-NEXT:    lw s7, 56(a5)
+; RV64I-WITH-FP-NEXT:    lw s8, 60(a5)
+; RV64I-WITH-FP-NEXT:    lw s9, 64(a5)
+; RV64I-WITH-FP-NEXT:    lw s10, 68(a5)
+; RV64I-WITH-FP-NEXT:    lw s11, 72(a5)
+; RV64I-WITH-FP-NEXT:    lw ra, 76(a5)
+; RV64I-WITH-FP-NEXT:    lw t4, 80(a5)
+; RV64I-WITH-FP-NEXT:    lw t3, 84(a5)
+; RV64I-WITH-FP-NEXT:    lw t2, 88(a5)
+; RV64I-WITH-FP-NEXT:    lw s1, 92(a5)
+; RV64I-WITH-FP-NEXT:    lw t1, 96(a5)
+; RV64I-WITH-FP-NEXT:    lw t0, 100(a5)
+; RV64I-WITH-FP-NEXT:    lw a6, 104(a5)
+; RV64I-WITH-FP-NEXT:    lw a4, 108(a5)
+; RV64I-WITH-FP-NEXT:    lw a0, 124(a5)
+; RV64I-WITH-FP-NEXT:    lw a1, 120(a5)
+; RV64I-WITH-FP-NEXT:    lw a2, 116(a5)
+; RV64I-WITH-FP-NEXT:    lw a3, 112(a5)
+; RV64I-WITH-FP-NEXT:    sw a0, 124(a5)
+; RV64I-WITH-FP-NEXT:    sw a1, 120(a5)
+; RV64I-WITH-FP-NEXT:    sw a2, 116(a5)
+; RV64I-WITH-FP-NEXT:    sw a3, 112(a5)
+; RV64I-WITH-FP-NEXT:    sw a4, 108(a5)
+; RV64I-WITH-FP-NEXT:    sw a6, 104(a5)
+; RV64I-WITH-FP-NEXT:    sw t0, 100(a5)
+; RV64I-WITH-FP-NEXT:    sw t1, 96(a5)
+; RV64I-WITH-FP-NEXT:    sw s1, 92(a5)
+; RV64I-WITH-FP-NEXT:    sw t2, 88(a5)
+; RV64I-WITH-FP-NEXT:    sw t3, 84(a5)
+; RV64I-WITH-FP-NEXT:    sw t4, 80(a5)
+; RV64I-WITH-FP-NEXT:    sw ra, 76(a5)
+; RV64I-WITH-FP-NEXT:    sw s11, 72(a5)
+; RV64I-WITH-FP-NEXT:    sw s10, 68(a5)
+; RV64I-WITH-FP-NEXT:    sw s9, 64(a5)
+; RV64I-WITH-FP-NEXT:    sw s8, 60(a5)
+; RV64I-WITH-FP-NEXT:    sw s7, 56(a5)
+; RV64I-WITH-FP-NEXT:    sw s6, 52(a5)
+; RV64I-WITH-FP-NEXT:    sw s5, 48(a5)
+; RV64I-WITH-FP-NEXT:    sw s4, 44(a5)
+; RV64I-WITH-FP-NEXT:    sw s3, 40(a5)
+; RV64I-WITH-FP-NEXT:    sw s2, 36(a5)
+; RV64I-WITH-FP-NEXT:    sw t6, 32(a5)
+; RV64I-WITH-FP-NEXT:    sw t5, 28(a5)
+; RV64I-WITH-FP-NEXT:    ld a0, -160(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 24(a5)
+; RV64I-WITH-FP-NEXT:    ld a0, -152(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 20(a5)
+; RV64I-WITH-FP-NEXT:    ld a0, -144(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 16(a5)
+; RV64I-WITH-FP-NEXT:    ld a0, -136(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, %lo(var+12)(a7)
+; RV64I-WITH-FP-NEXT:    ld a0, -128(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, %lo(var+8)(a7)
+; RV64I-WITH-FP-NEXT:    ld a0, -120(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, %lo(var+4)(a7)
+; RV64I-WITH-FP-NEXT:    ld a0, -112(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, %lo(var)(a7)
+; RV64I-WITH-FP-NEXT:    ld s11, 56(sp)
+; RV64I-WITH-FP-NEXT:    ld s10, 64(sp)
+; RV64I-WITH-FP-NEXT:    ld s9, 72(sp)
+; RV64I-WITH-FP-NEXT:    ld s8, 80(sp)
+; RV64I-WITH-FP-NEXT:    ld s7, 88(sp)
+; RV64I-WITH-FP-NEXT:    ld s6, 96(sp)
+; RV64I-WITH-FP-NEXT:    ld s5, 104(sp)
+; RV64I-WITH-FP-NEXT:    ld s4, 112(sp)
+; RV64I-WITH-FP-NEXT:    ld s3, 120(sp)
+; RV64I-WITH-FP-NEXT:    ld s2, 128(sp)
+; RV64I-WITH-FP-NEXT:    ld s1, 136(sp)
+; RV64I-WITH-FP-NEXT:    ld s0, 144(sp)
+; RV64I-WITH-FP-NEXT:    ld ra, 152(sp)
+; RV64I-WITH-FP-NEXT:    addi sp, sp, 160
+; RV64I-WITH-FP-NEXT:    ret
   %val = load [32 x i32], [32 x i32]* @var
   store volatile [32 x i32] %val, [32 x i32]* @var
   ret void
@@ -120,127 +481,583 @@ define void @callee() nounwind {
 
 define void @caller() nounwind {
 ; RV32I-LABEL: caller:
-; RV32I:         lui a0, %hi(var)
-; RV32I-NEXT:    lw a1, %lo(var)(a0)
-; RV32I-NEXT:    sw a1, 88(sp)
-; RV32I-NEXT:    addi s0, a0, %lo(var)
-
-; RV32I:         sw a0, 8(sp)
-; RV32I-NEXT:    lw s2, 84(s0)
-; RV32I-NEXT:    lw s3, 88(s0)
-; RV32I-NEXT:    lw s4, 92(s0)
-; RV32I-NEXT:    lw s5, 96(s0)
-; RV32I-NEXT:    lw s6, 100(s0)
-; RV32I-NEXT:    lw s7, 104(s0)
-; RV32I-NEXT:    lw s8, 108(s0)
-; RV32I-NEXT:    lw s9, 112(s0)
-; RV32I-NEXT:    lw s10, 116(s0)
-; RV32I-NEXT:    lw s11, 120(s0)
-; RV32I-NEXT:    lw s1, 124(s0)
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -144
+; RV32I-NEXT:    sw ra, 140(sp)
+; RV32I-NEXT:    sw s0, 136(sp)
+; RV32I-NEXT:    sw s1, 132(sp)
+; RV32I-NEXT:    sw s2, 128(sp)
+; RV32I-NEXT:    sw s3, 124(sp)
+; RV32I-NEXT:    sw s4, 120(sp)
+; RV32I-NEXT:    sw s5, 116(sp)
+; RV32I-NEXT:    sw s6, 112(sp)
+; RV32I-NEXT:    sw s7, 108(sp)
+; RV32I-NEXT:    sw s8, 104(sp)
+; RV32I-NEXT:    sw s9, 100(sp)
+; RV32I-NEXT:    sw s10, 96(sp)
+; RV32I-NEXT:    sw s11, 92(sp)
+; RV32I-NEXT:    lui s0, %hi(var)
+; RV32I-NEXT:    lw a0, %lo(var)(s0)
+; RV32I-NEXT:    sw a0, 88(sp)
+; RV32I-NEXT:    lw a0, %lo(var+4)(s0)
+; RV32I-NEXT:    sw a0, 84(sp)
+; RV32I-NEXT:    lw a0, %lo(var+8)(s0)
+; RV32I-NEXT:    sw a0, 80(sp)
+; RV32I-NEXT:    lw a0, %lo(var+12)(s0)
+; RV32I-NEXT:    sw a0, 76(sp)
+; RV32I-NEXT:    addi s1, s0, %lo(var)
+; RV32I-NEXT:    lw a0, 16(s1)
+; RV32I-NEXT:    sw a0, 72(sp)
+; RV32I-NEXT:    lw a0, 20(s1)
+; RV32I-NEXT:    sw a0, 68(sp)
+; RV32I-NEXT:    lw a0, 24(s1)
+; RV32I-NEXT:    sw a0, 64(sp)
+; RV32I-NEXT:    lw a0, 28(s1)
+; RV32I-NEXT:    sw a0, 60(sp)
+; RV32I-NEXT:    lw a0, 32(s1)
+; RV32I-NEXT:    sw a0, 56(sp)
+; RV32I-NEXT:    lw a0, 36(s1)
+; RV32I-NEXT:    sw a0, 52(sp)
+; RV32I-NEXT:    lw a0, 40(s1)
+; RV32I-NEXT:    sw a0, 48(sp)
+; RV32I-NEXT:    lw a0, 44(s1)
+; RV32I-NEXT:    sw a0, 44(sp)
+; RV32I-NEXT:    lw a0, 48(s1)
+; RV32I-NEXT:    sw a0, 40(sp)
+; RV32I-NEXT:    lw a0, 52(s1)
+; RV32I-NEXT:    sw a0, 36(sp)
+; RV32I-NEXT:    lw a0, 56(s1)
+; RV32I-NEXT:    sw a0, 32(sp)
+; RV32I-NEXT:    lw a0, 60(s1)
+; RV32I-NEXT:    sw a0, 28(sp)
+; RV32I-NEXT:    lw a0, 64(s1)
+; RV32I-NEXT:    sw a0, 24(sp)
+; RV32I-NEXT:    lw a0, 68(s1)
+; RV32I-NEXT:    sw a0, 20(sp)
+; RV32I-NEXT:    lw a0, 72(s1)
+; RV32I-NEXT:    sw a0, 16(sp)
+; RV32I-NEXT:    lw a0, 76(s1)
+; RV32I-NEXT:    sw a0, 12(sp)
+; RV32I-NEXT:    lw a0, 80(s1)
+; RV32I-NEXT:    sw a0, 8(sp)
+; RV32I-NEXT:    lw a0, 84(s1)
+; RV32I-NEXT:    sw a0, 4(sp)
+; RV32I-NEXT:    lw s4, 88(s1)
+; RV32I-NEXT:    lw s5, 92(s1)
+; RV32I-NEXT:    lw s6, 96(s1)
+; RV32I-NEXT:    lw s7, 100(s1)
+; RV32I-NEXT:    lw s8, 104(s1)
+; RV32I-NEXT:    lw s9, 108(s1)
+; RV32I-NEXT:    lw s10, 112(s1)
+; RV32I-NEXT:    lw s11, 116(s1)
+; RV32I-NEXT:    lw s2, 120(s1)
+; RV32I-NEXT:    lw s3, 124(s1)
 ; RV32I-NEXT:    call callee
-; RV32I-NEXT:    sw s1, 124(s0)
-; RV32I-NEXT:    sw s11, 120(s0)
-; RV32I-NEXT:    sw s10, 116(s0)
-; RV32I-NEXT:    sw s9, 112(s0)
-; RV32I-NEXT:    sw s8, 108(s0)
-; RV32I-NEXT:    sw s7, 104(s0)
-; RV32I-NEXT:    sw s6, 100(s0)
-; RV32I-NEXT:    sw s5, 96(s0)
-; RV32I-NEXT:    sw s4, 92(s0)
-; RV32I-NEXT:    sw s3, 88(s0)
-; RV32I-NEXT:    sw s2, 84(s0)
+; RV32I-NEXT:    sw s3, 124(s1)
+; RV32I-NEXT:    sw s2, 120(s1)
+; RV32I-NEXT:    sw s11, 116(s1)
+; RV32I-NEXT:    sw s10, 112(s1)
+; RV32I-NEXT:    sw s9, 108(s1)
+; RV32I-NEXT:    sw s8, 104(s1)
+; RV32I-NEXT:    sw s7, 100(s1)
+; RV32I-NEXT:    sw s6, 96(s1)
+; RV32I-NEXT:    sw s5, 92(s1)
+; RV32I-NEXT:    sw s4, 88(s1)
+; RV32I-NEXT:    lw a0, 4(sp)
+; RV32I-NEXT:    sw a0, 84(s1)
 ; RV32I-NEXT:    lw a0, 8(sp)
+; RV32I-NEXT:    sw a0, 80(s1)
+; RV32I-NEXT:    lw a0, 12(sp)
+; RV32I-NEXT:    sw a0, 76(s1)
+; RV32I-NEXT:    lw a0, 16(sp)
+; RV32I-NEXT:    sw a0, 72(s1)
+; RV32I-NEXT:    lw a0, 20(sp)
+; RV32I-NEXT:    sw a0, 68(s1)
+; RV32I-NEXT:    lw a0, 24(sp)
+; RV32I-NEXT:    sw a0, 64(s1)
+; RV32I-NEXT:    lw a0, 28(sp)
+; RV32I-NEXT:    sw a0, 60(s1)
+; RV32I-NEXT:    lw a0, 32(sp)
+; RV32I-NEXT:    sw a0, 56(s1)
+; RV32I-NEXT:    lw a0, 36(sp)
+; RV32I-NEXT:    sw a0, 52(s1)
+; RV32I-NEXT:    lw a0, 40(sp)
+; RV32I-NEXT:    sw a0, 48(s1)
+; RV32I-NEXT:    lw a0, 44(sp)
+; RV32I-NEXT:    sw a0, 44(s1)
+; RV32I-NEXT:    lw a0, 48(sp)
+; RV32I-NEXT:    sw a0, 40(s1)
+; RV32I-NEXT:    lw a0, 52(sp)
+; RV32I-NEXT:    sw a0, 36(s1)
+; RV32I-NEXT:    lw a0, 56(sp)
+; RV32I-NEXT:    sw a0, 32(s1)
+; RV32I-NEXT:    lw a0, 60(sp)
+; RV32I-NEXT:    sw a0, 28(s1)
+; RV32I-NEXT:    lw a0, 64(sp)
+; RV32I-NEXT:    sw a0, 24(s1)
+; RV32I-NEXT:    lw a0, 68(sp)
+; RV32I-NEXT:    sw a0, 20(s1)
+; RV32I-NEXT:    lw a0, 72(sp)
+; RV32I-NEXT:    sw a0, 16(s1)
+; RV32I-NEXT:    lw a0, 76(sp)
+; RV32I-NEXT:    sw a0, %lo(var+12)(s0)
+; RV32I-NEXT:    lw a0, 80(sp)
+; RV32I-NEXT:    sw a0, %lo(var+8)(s0)
+; RV32I-NEXT:    lw a0, 84(sp)
+; RV32I-NEXT:    sw a0, %lo(var+4)(s0)
+; RV32I-NEXT:    lw a0, 88(sp)
+; RV32I-NEXT:    sw a0, %lo(var)(s0)
+; RV32I-NEXT:    lw s11, 92(sp)
+; RV32I-NEXT:    lw s10, 96(sp)
+; RV32I-NEXT:    lw s9, 100(sp)
+; RV32I-NEXT:    lw s8, 104(sp)
+; RV32I-NEXT:    lw s7, 108(sp)
+; RV32I-NEXT:    lw s6, 112(sp)
+; RV32I-NEXT:    lw s5, 116(sp)
+; RV32I-NEXT:    lw s4, 120(sp)
+; RV32I-NEXT:    lw s3, 124(sp)
+; RV32I-NEXT:    lw s2, 128(sp)
+; RV32I-NEXT:    lw s1, 132(sp)
+; RV32I-NEXT:    lw s0, 136(sp)
+; RV32I-NEXT:    lw ra, 140(sp)
+; RV32I-NEXT:    addi sp, sp, 144
+; RV32I-NEXT:    ret
 ;
 ; RV32I-WITH-FP-LABEL: caller:
-; RV32I-WITH-FP:         addi s0, sp, 144
-; RV32I-WITH-FP-NEXT:    lui a0, %hi(var)
-; RV32I-WITH-FP-NEXT:    lw a1, %lo(var)(a0)
-; RV32I-WITH-FP-NEXT:    sw a1, -56(s0)
-; RV32I-WITH-FP-NEXT:    addi s1, a0, %lo(var)
-; RV32I-WITH-FP:         sw a0, -140(s0)
-; RV32I-WITH-FP-NEXT:    lw s5, 88(s1)
-; RV32I-WITH-FP-NEXT:    lw s6, 92(s1)
-; RV32I-WITH-FP-NEXT:    lw s7, 96(s1)
-; RV32I-WITH-FP-NEXT:    lw s8, 100(s1)
-; RV32I-WITH-FP-NEXT:    lw s9, 104(s1)
-; RV32I-WITH-FP-NEXT:    lw s10, 108(s1)
-; RV32I-WITH-FP-NEXT:    lw s11, 112(s1)
-; RV32I-WITH-FP-NEXT:    lw s2, 116(s1)
-; RV32I-WITH-FP-NEXT:    lw s3, 120(s1)
-; RV32I-WITH-FP-NEXT:    lw s4, 124(s1)
+; RV32I-WITH-FP:       # %bb.0:
+; RV32I-WITH-FP-NEXT:    addi sp, sp, -144
+; RV32I-WITH-FP-NEXT:    sw ra, 140(sp)
+; RV32I-WITH-FP-NEXT:    sw s0, 136(sp)
+; RV32I-WITH-FP-NEXT:    sw s1, 132(sp)
+; RV32I-WITH-FP-NEXT:    sw s2, 128(sp)
+; RV32I-WITH-FP-NEXT:    sw s3, 124(sp)
+; RV32I-WITH-FP-NEXT:    sw s4, 120(sp)
+; RV32I-WITH-FP-NEXT:    sw s5, 116(sp)
+; RV32I-WITH-FP-NEXT:    sw s6, 112(sp)
+; RV32I-WITH-FP-NEXT:    sw s7, 108(sp)
+; RV32I-WITH-FP-NEXT:    sw s8, 104(sp)
+; RV32I-WITH-FP-NEXT:    sw s9, 100(sp)
+; RV32I-WITH-FP-NEXT:    sw s10, 96(sp)
+; RV32I-WITH-FP-NEXT:    sw s11, 92(sp)
+; RV32I-WITH-FP-NEXT:    addi s0, sp, 144
+; RV32I-WITH-FP-NEXT:    lui s6, %hi(var)
+; RV32I-WITH-FP-NEXT:    lw a0, %lo(var)(s6)
+; RV32I-WITH-FP-NEXT:    sw a0, -56(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, %lo(var+4)(s6)
+; RV32I-WITH-FP-NEXT:    sw a0, -60(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, %lo(var+8)(s6)
+; RV32I-WITH-FP-NEXT:    sw a0, -64(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, %lo(var+12)(s6)
+; RV32I-WITH-FP-NEXT:    sw a0, -68(s0)
+; RV32I-WITH-FP-NEXT:    addi s1, s6, %lo(var)
+; RV32I-WITH-FP-NEXT:    lw a0, 16(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -72(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 20(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -76(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 24(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -80(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 28(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -84(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 32(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -88(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 36(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -92(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 40(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -96(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 44(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -100(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 48(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -104(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 52(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -108(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 56(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -112(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 60(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -116(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 64(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -120(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 68(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -124(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 72(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -128(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 76(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -132(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 80(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -136(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 84(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -140(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, 88(s1)
+; RV32I-WITH-FP-NEXT:    sw a0, -144(s0)
+; RV32I-WITH-FP-NEXT:    lw s8, 92(s1)
+; RV32I-WITH-FP-NEXT:    lw s9, 96(s1)
+; RV32I-WITH-FP-NEXT:    lw s10, 100(s1)
+; RV32I-WITH-FP-NEXT:    lw s11, 104(s1)
+; RV32I-WITH-FP-NEXT:    lw s2, 108(s1)
+; RV32I-WITH-FP-NEXT:    lw s3, 112(s1)
+; RV32I-WITH-FP-NEXT:    lw s4, 116(s1)
+; RV32I-WITH-FP-NEXT:    lw s5, 120(s1)
+; RV32I-WITH-FP-NEXT:    lw s7, 124(s1)
 ; RV32I-WITH-FP-NEXT:    call callee
-; RV32I-WITH-FP-NEXT:    sw s4, 124(s1)
-; RV32I-WITH-FP-NEXT:    sw s3, 120(s1)
-; RV32I-WITH-FP-NEXT:    sw s2, 116(s1)
-; RV32I-WITH-FP-NEXT:    sw s11, 112(s1)
-; RV32I-WITH-FP-NEXT:    sw s10, 108(s1)
-; RV32I-WITH-FP-NEXT:    sw s9, 104(s1)
-; RV32I-WITH-FP-NEXT:    sw s8, 100(s1)
-; RV32I-WITH-FP-NEXT:    sw s7, 96(s1)
-; RV32I-WITH-FP-NEXT:    sw s6, 92(s1)
-; RV32I-WITH-FP-NEXT:    sw s5, 88(s1)
+; RV32I-WITH-FP-NEXT:    sw s7, 124(s1)
+; RV32I-WITH-FP-NEXT:    sw s5, 120(s1)
+; RV32I-WITH-FP-NEXT:    sw s4, 116(s1)
+; RV32I-WITH-FP-NEXT:    sw s3, 112(s1)
+; RV32I-WITH-FP-NEXT:    sw s2, 108(s1)
+; RV32I-WITH-FP-NEXT:    sw s11, 104(s1)
+; RV32I-WITH-FP-NEXT:    sw s10, 100(s1)
+; RV32I-WITH-FP-NEXT:    sw s9, 96(s1)
+; RV32I-WITH-FP-NEXT:    sw s8, 92(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -144(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 88(s1)
 ; RV32I-WITH-FP-NEXT:    lw a0, -140(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 84(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -136(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 80(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -132(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 76(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -128(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 72(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -124(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 68(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -120(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 64(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -116(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 60(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -112(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 56(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -108(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 52(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -104(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 48(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -100(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 44(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -96(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 40(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -92(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 36(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -88(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 32(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -84(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 28(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -80(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 24(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -76(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 20(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -72(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, 16(s1)
+; RV32I-WITH-FP-NEXT:    lw a0, -68(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, %lo(var+12)(s6)
+; RV32I-WITH-FP-NEXT:    lw a0, -64(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, %lo(var+8)(s6)
+; RV32I-WITH-FP-NEXT:    lw a0, -60(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, %lo(var+4)(s6)
+; RV32I-WITH-FP-NEXT:    lw a0, -56(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, %lo(var)(s6)
+; RV32I-WITH-FP-NEXT:    lw s11, 92(sp)
+; RV32I-WITH-FP-NEXT:    lw s10, 96(sp)
+; RV32I-WITH-FP-NEXT:    lw s9, 100(sp)
+; RV32I-WITH-FP-NEXT:    lw s8, 104(sp)
+; RV32I-WITH-FP-NEXT:    lw s7, 108(sp)
+; RV32I-WITH-FP-NEXT:    lw s6, 112(sp)
+; RV32I-WITH-FP-NEXT:    lw s5, 116(sp)
+; RV32I-WITH-FP-NEXT:    lw s4, 120(sp)
+; RV32I-WITH-FP-NEXT:    lw s3, 124(sp)
+; RV32I-WITH-FP-NEXT:    lw s2, 128(sp)
+; RV32I-WITH-FP-NEXT:    lw s1, 132(sp)
+; RV32I-WITH-FP-NEXT:    lw s0, 136(sp)
+; RV32I-WITH-FP-NEXT:    lw ra, 140(sp)
+; RV32I-WITH-FP-NEXT:    addi sp, sp, 144
+; RV32I-WITH-FP-NEXT:    ret
 ;
 ; RV64I-LABEL: caller:
-; RV64I:         lui a0, %hi(var)
-; RV64I-NEXT:    lw a1, %lo(var)(a0)
-; RV64I-NEXT:    sd a1, 160(sp)
-; RV64I-NEXT:    addi s0, a0, %lo(var)
-; RV64I:         sd a0, 0(sp)
-; RV64I-NEXT:    lw s2, 84(s0)
-; RV64I-NEXT:    lw s3, 88(s0)
-; RV64I-NEXT:    lw s4, 92(s0)
-; RV64I-NEXT:    lw s5, 96(s0)
-; RV64I-NEXT:    lw s6, 100(s0)
-; RV64I-NEXT:    lw s7, 104(s0)
-; RV64I-NEXT:    lw s8, 108(s0)
-; RV64I-NEXT:    lw s9, 112(s0)
-; RV64I-NEXT:    lw s10, 116(s0)
-; RV64I-NEXT:    lw s11, 120(s0)
-; RV64I-NEXT:    lw s1, 124(s0)
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -288
+; RV64I-NEXT:    sd ra, 280(sp)
+; RV64I-NEXT:    sd s0, 272(sp)
+; RV64I-NEXT:    sd s1, 264(sp)
+; RV64I-NEXT:    sd s2, 256(sp)
+; RV64I-NEXT:    sd s3, 248(sp)
+; RV64I-NEXT:    sd s4, 240(sp)
+; RV64I-NEXT:    sd s5, 232(sp)
+; RV64I-NEXT:    sd s6, 224(sp)
+; RV64I-NEXT:    sd s7, 216(sp)
+; RV64I-NEXT:    sd s8, 208(sp)
+; RV64I-NEXT:    sd s9, 200(sp)
+; RV64I-NEXT:    sd s10, 192(sp)
+; RV64I-NEXT:    sd s11, 184(sp)
+; RV64I-NEXT:    lui s0, %hi(var)
+; RV64I-NEXT:    lw a0, %lo(var)(s0)
+; RV64I-NEXT:    sd a0, 176(sp)
+; RV64I-NEXT:    lw a0, %lo(var+4)(s0)
+; RV64I-NEXT:    sd a0, 168(sp)
+; RV64I-NEXT:    lw a0, %lo(var+8)(s0)
+; RV64I-NEXT:    sd a0, 160(sp)
+; RV64I-NEXT:    lw a0, %lo(var+12)(s0)
+; RV64I-NEXT:    sd a0, 152(sp)
+; RV64I-NEXT:    addi s1, s0, %lo(var)
+; RV64I-NEXT:    lw a0, 16(s1)
+; RV64I-NEXT:    sd a0, 144(sp)
+; RV64I-NEXT:    lw a0, 20(s1)
+; RV64I-NEXT:    sd a0, 136(sp)
+; RV64I-NEXT:    lw a0, 24(s1)
+; RV64I-NEXT:    sd a0, 128(sp)
+; RV64I-NEXT:    lw a0, 28(s1)
+; RV64I-NEXT:    sd a0, 120(sp)
+; RV64I-NEXT:    lw a0, 32(s1)
+; RV64I-NEXT:    sd a0, 112(sp)
+; RV64I-NEXT:    lw a0, 36(s1)
+; RV64I-NEXT:    sd a0, 104(sp)
+; RV64I-NEXT:    lw a0, 40(s1)
+; RV64I-NEXT:    sd a0, 96(sp)
+; RV64I-NEXT:    lw a0, 44(s1)
+; RV64I-NEXT:    sd a0, 88(sp)
+; RV64I-NEXT:    lw a0, 48(s1)
+; RV64I-NEXT:    sd a0, 80(sp)
+; RV64I-NEXT:    lw a0, 52(s1)
+; RV64I-NEXT:    sd a0, 72(sp)
+; RV64I-NEXT:    lw a0, 56(s1)
+; RV64I-NEXT:    sd a0, 64(sp)
+; RV64I-NEXT:    lw a0, 60(s1)
+; RV64I-NEXT:    sd a0, 56(sp)
+; RV64I-NEXT:    lw a0, 64(s1)
+; RV64I-NEXT:    sd a0, 48(sp)
+; RV64I-NEXT:    lw a0, 68(s1)
+; RV64I-NEXT:    sd a0, 40(sp)
+; RV64I-NEXT:    lw a0, 72(s1)
+; RV64I-NEXT:    sd a0, 32(sp)
+; RV64I-NEXT:    lw a0, 76(s1)
+; RV64I-NEXT:    sd a0, 24(sp)
+; RV64I-NEXT:    lw a0, 80(s1)
+; RV64I-NEXT:    sd a0, 16(sp)
+; RV64I-NEXT:    lw a0, 84(s1)
+; RV64I-NEXT:    sd a0, 8(sp)
+; RV64I-NEXT:    lw s4, 88(s1)
+; RV64I-NEXT:    lw s5, 92(s1)
+; RV64I-NEXT:    lw s6, 96(s1)
+; RV64I-NEXT:    lw s7, 100(s1)
+; RV64I-NEXT:    lw s8, 104(s1)
+; RV64I-NEXT:    lw s9, 108(s1)
+; RV64I-NEXT:    lw s10, 112(s1)
+; RV64I-NEXT:    lw s11, 116(s1)
+; RV64I-NEXT:    lw s2, 120(s1)
+; RV64I-NEXT:    lw s3, 124(s1)
 ; RV64I-NEXT:    call callee
-; RV64I-NEXT:    sw s1, 124(s0)
-; RV64I-NEXT:    sw s11, 120(s0)
-; RV64I-NEXT:    sw s10, 116(s0)
-; RV64I-NEXT:    sw s9, 112(s0)
-; RV64I-NEXT:    sw s8, 108(s0)
-; RV64I-NEXT:    sw s7, 104(s0)
-; RV64I-NEXT:    sw s6, 100(s0)
-; RV64I-NEXT:    sw s5, 96(s0)
-; RV64I-NEXT:    sw s4, 92(s0)
-; RV64I-NEXT:    sw s3, 88(s0)
-; RV64I-NEXT:    sw s2, 84(s0)
-; RV64I-NEXT:    ld a0, 0(sp)
+; RV64I-NEXT:    sw s3, 124(s1)
+; RV64I-NEXT:    sw s2, 120(s1)
+; RV64I-NEXT:    sw s11, 116(s1)
+; RV64I-NEXT:    sw s10, 112(s1)
+; RV64I-NEXT:    sw s9, 108(s1)
+; RV64I-NEXT:    sw s8, 104(s1)
+; RV64I-NEXT:    sw s7, 100(s1)
+; RV64I-NEXT:    sw s6, 96(s1)
+; RV64I-NEXT:    sw s5, 92(s1)
+; RV64I-NEXT:    sw s4, 88(s1)
+; RV64I-NEXT:    ld a0, 8(sp)
+; RV64I-NEXT:    sw a0, 84(s1)
+; RV64I-NEXT:    ld a0, 16(sp)
+; RV64I-NEXT:    sw a0, 80(s1)
+; RV64I-NEXT:    ld a0, 24(sp)
+; RV64I-NEXT:    sw a0, 76(s1)
+; RV64I-NEXT:    ld a0, 32(sp)
+; RV64I-NEXT:    sw a0, 72(s1)
+; RV64I-NEXT:    ld a0, 40(sp)
+; RV64I-NEXT:    sw a0, 68(s1)
+; RV64I-NEXT:    ld a0, 48(sp)
+; RV64I-NEXT:    sw a0, 64(s1)
+; RV64I-NEXT:    ld a0, 56(sp)
+; RV64I-NEXT:    sw a0, 60(s1)
+; RV64I-NEXT:    ld a0, 64(sp)
+; RV64I-NEXT:    sw a0, 56(s1)
+; RV64I-NEXT:    ld a0, 72(sp)
+; RV64I-NEXT:    sw a0, 52(s1)
+; RV64I-NEXT:    ld a0, 80(sp)
+; RV64I-NEXT:    sw a0, 48(s1)
+; RV64I-NEXT:    ld a0, 88(sp)
+; RV64I-NEXT:    sw a0, 44(s1)
+; RV64I-NEXT:    ld a0, 96(sp)
+; RV64I-NEXT:    sw a0, 40(s1)
+; RV64I-NEXT:    ld a0, 104(sp)
+; RV64I-NEXT:    sw a0, 36(s1)
+; RV64I-NEXT:    ld a0, 112(sp)
+; RV64I-NEXT:    sw a0, 32(s1)
+; RV64I-NEXT:    ld a0, 120(sp)
+; RV64I-NEXT:    sw a0, 28(s1)
+; RV64I-NEXT:    ld a0, 128(sp)
+; RV64I-NEXT:    sw a0, 24(s1)
+; RV64I-NEXT:    ld a0, 136(sp)
+; RV64I-NEXT:    sw a0, 20(s1)
+; RV64I-NEXT:    ld a0, 144(sp)
+; RV64I-NEXT:    sw a0, 16(s1)
+; RV64I-NEXT:    ld a0, 152(sp)
+; RV64I-NEXT:    sw a0, %lo(var+12)(s0)
+; RV64I-NEXT:    ld a0, 160(sp)
+; RV64I-NEXT:    sw a0, %lo(var+8)(s0)
+; RV64I-NEXT:    ld a0, 168(sp)
+; RV64I-NEXT:    sw a0, %lo(var+4)(s0)
+; RV64I-NEXT:    ld a0, 176(sp)
+; RV64I-NEXT:    sw a0, %lo(var)(s0)
+; RV64I-NEXT:    ld s11, 184(sp)
+; RV64I-NEXT:    ld s10, 192(sp)
+; RV64I-NEXT:    ld s9, 200(sp)
+; RV64I-NEXT:    ld s8, 208(sp)
+; RV64I-NEXT:    ld s7, 216(sp)
+; RV64I-NEXT:    ld s6, 224(sp)
+; RV64I-NEXT:    ld s5, 232(sp)
+; RV64I-NEXT:    ld s4, 240(sp)
+; RV64I-NEXT:    ld s3, 248(sp)
+; RV64I-NEXT:    ld s2, 256(sp)
+; RV64I-NEXT:    ld s1, 264(sp)
+; RV64I-NEXT:    ld s0, 272(sp)
+; RV64I-NEXT:    ld ra, 280(sp)
+; RV64I-NEXT:    addi sp, sp, 288
+; RV64I-NEXT:    ret
 ;
 ; RV64I-WITH-FP-LABEL: caller:
-; RV64I-WITH-FP:         addi s0, sp, 288
-; RV64I-WITH-FP-NEXT:    lui a0, %hi(var)
-; RV64I-WITH-FP-NEXT:    lw a1, %lo(var)(a0)
-; RV64I-WITH-FP-NEXT:    sd a1, -112(s0)
-; RV64I-WITH-FP-NEXT:    addi s1, a0, %lo(var)
-; RV64I-WITH-FP:         sd a0, -280(s0)
-; RV64I-WITH-FP-NEXT:    lw s5, 88(s1)
-; RV64I-WITH-FP-NEXT:    lw s6, 92(s1)
-; RV64I-WITH-FP-NEXT:    lw s7, 96(s1)
-; RV64I-WITH-FP-NEXT:    lw s8, 100(s1)
-; RV64I-WITH-FP-NEXT:    lw s9, 104(s1)
-; RV64I-WITH-FP-NEXT:    lw s10, 108(s1)
-; RV64I-WITH-FP-NEXT:    lw s11, 112(s1)
-; RV64I-WITH-FP-NEXT:    lw s2, 116(s1)
-; RV64I-WITH-FP-NEXT:    lw s3, 120(s1)
-; RV64I-WITH-FP-NEXT:    lw s4, 124(s1)
+; RV64I-WITH-FP:       # %bb.0:
+; RV64I-WITH-FP-NEXT:    addi sp, sp, -288
+; RV64I-WITH-FP-NEXT:    sd ra, 280(sp)
+; RV64I-WITH-FP-NEXT:    sd s0, 272(sp)
+; RV64I-WITH-FP-NEXT:    sd s1, 264(sp)
+; RV64I-WITH-FP-NEXT:    sd s2, 256(sp)
+; RV64I-WITH-FP-NEXT:    sd s3, 248(sp)
+; RV64I-WITH-FP-NEXT:    sd s4, 240(sp)
+; RV64I-WITH-FP-NEXT:    sd s5, 232(sp)
+; RV64I-WITH-FP-NEXT:    sd s6, 224(sp)
+; RV64I-WITH-FP-NEXT:    sd s7, 216(sp)
+; RV64I-WITH-FP-NEXT:    sd s8, 208(sp)
+; RV64I-WITH-FP-NEXT:    sd s9, 200(sp)
+; RV64I-WITH-FP-NEXT:    sd s10, 192(sp)
+; RV64I-WITH-FP-NEXT:    sd s11, 184(sp)
+; RV64I-WITH-FP-NEXT:    addi s0, sp, 288
+; RV64I-WITH-FP-NEXT:    lui s6, %hi(var)
+; RV64I-WITH-FP-NEXT:    lw a0, %lo(var)(s6)
+; RV64I-WITH-FP-NEXT:    sd a0, -112(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, %lo(var+4)(s6)
+; RV64I-WITH-FP-NEXT:    sd a0, -120(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, %lo(var+8)(s6)
+; RV64I-WITH-FP-NEXT:    sd a0, -128(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, %lo(var+12)(s6)
+; RV64I-WITH-FP-NEXT:    sd a0, -136(s0)
+; RV64I-WITH-FP-NEXT:    addi s1, s6, %lo(var)
+; RV64I-WITH-FP-NEXT:    lw a0, 16(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -144(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 20(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -152(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 24(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -160(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 28(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -168(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 32(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -176(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 36(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -184(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 40(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -192(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 44(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -200(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 48(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -208(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 52(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -216(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 56(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -224(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 60(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -232(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 64(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -240(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 68(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -248(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 72(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -256(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 76(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -264(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 80(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -272(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 84(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -280(s0)
+; RV64I-WITH-FP-NEXT:    lw a0, 88(s1)
+; RV64I-WITH-FP-NEXT:    sd a0, -288(s0)
+; RV64I-WITH-FP-NEXT:    lw s8, 92(s1)
+; RV64I-WITH-FP-NEXT:    lw s9, 96(s1)
+; RV64I-WITH-FP-NEXT:    lw s10, 100(s1)
+; RV64I-WITH-FP-NEXT:    lw s11, 104(s1)
+; RV64I-WITH-FP-NEXT:    lw s2, 108(s1)
+; RV64I-WITH-FP-NEXT:    lw s3, 112(s1)
+; RV64I-WITH-FP-NEXT:    lw s4, 116(s1)
+; RV64I-WITH-FP-NEXT:    lw s5, 120(s1)
+; RV64I-WITH-FP-NEXT:    lw s7, 124(s1)
 ; RV64I-WITH-FP-NEXT:    call callee
-; RV64I-WITH-FP-NEXT:    sw s4, 124(s1)
-; RV64I-WITH-FP-NEXT:    sw s3, 120(s1)
-; RV64I-WITH-FP-NEXT:    sw s2, 116(s1)
-; RV64I-WITH-FP-NEXT:    sw s11, 112(s1)
-; RV64I-WITH-FP-NEXT:    sw s10, 108(s1)
-; RV64I-WITH-FP-NEXT:    sw s9, 104(s1)
-; RV64I-WITH-FP-NEXT:    sw s8, 100(s1)
-; RV64I-WITH-FP-NEXT:    sw s7, 96(s1)
-; RV64I-WITH-FP-NEXT:    sw s6, 92(s1)
-; RV64I-WITH-FP-NEXT:    sw s5, 88(s1)
+; RV64I-WITH-FP-NEXT:    sw s7, 124(s1)
+; RV64I-WITH-FP-NEXT:    sw s5, 120(s1)
+; RV64I-WITH-FP-NEXT:    sw s4, 116(s1)
+; RV64I-WITH-FP-NEXT:    sw s3, 112(s1)
+; RV64I-WITH-FP-NEXT:    sw s2, 108(s1)
+; RV64I-WITH-FP-NEXT:    sw s11, 104(s1)
+; RV64I-WITH-FP-NEXT:    sw s10, 100(s1)
+; RV64I-WITH-FP-NEXT:    sw s9, 96(s1)
+; RV64I-WITH-FP-NEXT:    sw s8, 92(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -288(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 88(s1)
 ; RV64I-WITH-FP-NEXT:    ld a0, -280(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 84(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -272(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 80(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -264(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 76(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -256(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 72(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -248(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 68(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -240(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 64(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -232(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 60(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -224(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 56(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -216(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 52(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -208(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 48(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -200(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 44(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -192(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 40(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -184(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 36(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -176(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 32(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -168(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 28(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -160(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 24(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -152(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 20(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -144(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, 16(s1)
+; RV64I-WITH-FP-NEXT:    ld a0, -136(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, %lo(var+12)(s6)
+; RV64I-WITH-FP-NEXT:    ld a0, -128(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, %lo(var+8)(s6)
+; RV64I-WITH-FP-NEXT:    ld a0, -120(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, %lo(var+4)(s6)
+; RV64I-WITH-FP-NEXT:    ld a0, -112(s0)
+; RV64I-WITH-FP-NEXT:    sw a0, %lo(var)(s6)
+; RV64I-WITH-FP-NEXT:    ld s11, 184(sp)
+; RV64I-WITH-FP-NEXT:    ld s10, 192(sp)
+; RV64I-WITH-FP-NEXT:    ld s9, 200(sp)
+; RV64I-WITH-FP-NEXT:    ld s8, 208(sp)
+; RV64I-WITH-FP-NEXT:    ld s7, 216(sp)
+; RV64I-WITH-FP-NEXT:    ld s6, 224(sp)
+; RV64I-WITH-FP-NEXT:    ld s5, 232(sp)
+; RV64I-WITH-FP-NEXT:    ld s4, 240(sp)
+; RV64I-WITH-FP-NEXT:    ld s3, 248(sp)
+; RV64I-WITH-FP-NEXT:    ld s2, 256(sp)
+; RV64I-WITH-FP-NEXT:    ld s1, 264(sp)
+; RV64I-WITH-FP-NEXT:    ld s0, 272(sp)
+; RV64I-WITH-FP-NEXT:    ld ra, 280(sp)
+; RV64I-WITH-FP-NEXT:    addi sp, sp, 288
+; RV64I-WITH-FP-NEXT:    ret
+
   %val = load [32 x i32], [32 x i32]* @var
   call void @callee()
   store volatile [32 x i32] %val, [32 x i32]* @var
diff --git a/llvm/test/CodeGen/RISCV/codemodel-lowering.ll b/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
index 6c172a26f050b..84774feccf12c 100644
--- a/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
+++ b/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
@@ -16,9 +16,9 @@ define i32 @lower_global(i32 %a) nounwind {
 ;
 ; RV32I-MEDIUM-LABEL: lower_global:
 ; RV32I-MEDIUM:       # %bb.0:
-; RV32I-MEDIUM-NEXT:  .LBB0_1: # Label of block must be emitted
+; RV32I-MEDIUM-NEXT:  .Ltmp0:
 ; RV32I-MEDIUM-NEXT:    auipc a0, %pcrel_hi(G)
-; RV32I-MEDIUM-NEXT:    addi a0, a0, %pcrel_lo(.LBB0_1)
+; RV32I-MEDIUM-NEXT:    addi a0, a0, %pcrel_lo(.Ltmp0)
 ; RV32I-MEDIUM-NEXT:    lw a0, 0(a0)
 ; RV32I-MEDIUM-NEXT:    ret
   %1 = load volatile i32, i32* @G
@@ -39,9 +39,9 @@ define void @lower_blockaddress() nounwind {
 ;
 ; RV32I-MEDIUM-LABEL: lower_blockaddress:
 ; RV32I-MEDIUM:       # %bb.0:
-; RV32I-MEDIUM-NEXT:  .LBB1_1: # Label of block must be emitted
+; RV32I-MEDIUM-NEXT:  .Ltmp1:
 ; RV32I-MEDIUM-NEXT:    auipc a0, %pcrel_hi(addr)
-; RV32I-MEDIUM-NEXT:    addi a0, a0, %pcrel_lo(.LBB1_1)
+; RV32I-MEDIUM-NEXT:    addi a0, a0, %pcrel_lo(.Ltmp1)
 ; RV32I-MEDIUM-NEXT:    addi a1, zero, 1
 ; RV32I-MEDIUM-NEXT:    sw a1, 0(a0)
 ; RV32I-MEDIUM-NEXT:    ret
@@ -82,17 +82,16 @@ define signext i32 @lower_blockaddress_displ(i32 signext %w) nounwind {
 ; RV32I-MEDIUM:       # %bb.0: # %entry
 ; RV32I-MEDIUM-NEXT:    addi sp, sp, -16
 ; RV32I-MEDIUM-NEXT:    sw ra, 12(sp)
-; RV32I-MEDIUM-NEXT:  .LBB2_5: # %entry
-; RV32I-MEDIUM-NEXT:    # Label of block must be emitted
-; RV32I-MEDIUM-NEXT:    auipc a1, %pcrel_hi(.Ltmp0)
-; RV32I-MEDIUM-NEXT:    addi a1, a1, %pcrel_lo(.LBB2_5)
+; RV32I-MEDIUM-NEXT:  .Ltmp2:
+; RV32I-MEDIUM-NEXT:    auipc a1, %pcrel_hi(.Ltmp3)
+; RV32I-MEDIUM-NEXT:    addi a1, a1, %pcrel_lo(.Ltmp2)
 ; RV32I-MEDIUM-NEXT:    addi a2, zero, 101
 ; RV32I-MEDIUM-NEXT:    sw a1, 8(sp)
 ; RV32I-MEDIUM-NEXT:    blt a0, a2, .LBB2_3
 ; RV32I-MEDIUM-NEXT:  # %bb.1: # %if.then
 ; RV32I-MEDIUM-NEXT:    lw a0, 8(sp)
 ; RV32I-MEDIUM-NEXT:    jr a0
-; RV32I-MEDIUM-NEXT:  .Ltmp0: # Block address taken
+; RV32I-MEDIUM-NEXT:  .Ltmp3: # Block address taken
 ; RV32I-MEDIUM-NEXT:  .LBB2_2: # %return
 ; RV32I-MEDIUM-NEXT:    addi a0, zero, 4
 ; RV32I-MEDIUM-NEXT:    j .LBB2_4
@@ -140,9 +139,9 @@ define float @lower_constantpool(float %a) nounwind {
 ;
 ; RV32I-MEDIUM-LABEL: lower_constantpool:
 ; RV32I-MEDIUM:       # %bb.0:
-; RV32I-MEDIUM-NEXT:  .LBB3_1: # Label of block must be emitted
+; RV32I-MEDIUM-NEXT:  .Ltmp4:
 ; RV32I-MEDIUM-NEXT:    auipc a1, %pcrel_hi(.LCPI3_0)
-; RV32I-MEDIUM-NEXT:    addi a1, a1, %pcrel_lo(.LBB3_1)
+; RV32I-MEDIUM-NEXT:    addi a1, a1, %pcrel_lo(.Ltmp4)
 ; RV32I-MEDIUM-NEXT:    flw ft0, 0(a1)
 ; RV32I-MEDIUM-NEXT:    fmv.w.x ft1, a0
 ; RV32I-MEDIUM-NEXT:    fadd.s ft0, ft1, ft0
diff --git a/llvm/test/CodeGen/RISCV/double-previous-failure.ll b/llvm/test/CodeGen/RISCV/double-previous-failure.ll
index 1defcf8840369..7c40e96a9e1d1 100644
--- a/llvm/test/CodeGen/RISCV/double-previous-failure.ll
+++ b/llvm/test/CodeGen/RISCV/double-previous-failure.ll
@@ -25,19 +25,19 @@ define i32 @main() nounwind {
 ; RV32IFD-NEXT:    fld ft0, 0(sp)
 ; RV32IFD-NEXT:    lui a0, %hi(.LCPI1_0)
 ; RV32IFD-NEXT:    fld ft1, %lo(.LCPI1_0)(a0)
-; RV32IFD-NEXT:    flt.d a0, ft0, ft1
-; RV32IFD-NEXT:    bnez a0, .LBB1_3
-; RV32IFD-NEXT:  # %bb.1: # %entry
 ; RV32IFD-NEXT:    lui a0, %hi(.LCPI1_1)
-; RV32IFD-NEXT:    fld ft1, %lo(.LCPI1_1)(a0)
-; RV32IFD-NEXT:    flt.d a0, ft1, ft0
-; RV32IFD-NEXT:    xori a0, a0, 1
-; RV32IFD-NEXT:    beqz a0, .LBB1_3
-; RV32IFD-NEXT:  # %bb.2: # %if.end
+; RV32IFD-NEXT:    fld ft2, %lo(.LCPI1_1)(a0)
+; RV32IFD-NEXT:    flt.d a0, ft0, ft1
+; RV32IFD-NEXT:    not a0, a0
+; RV32IFD-NEXT:    flt.d a1, ft2, ft0
+; RV32IFD-NEXT:    xori a1, a1, 1
+; RV32IFD-NEXT:    and a0, a0, a1
+; RV32IFD-NEXT:    bnez a0, .LBB1_2
+; RV32IFD-NEXT:  # %bb.1: # %if.then
+; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:  .LBB1_2: # %if.end
 ; RV32IFD-NEXT:    mv a0, zero
 ; RV32IFD-NEXT:    call exit
-; RV32IFD-NEXT:  .LBB1_3: # %if.then
-; RV32IFD-NEXT:    call abort
 entry:
   %call = call double @test(double 2.000000e+00)
   %cmp = fcmp olt double %call, 2.400000e-01
diff --git a/llvm/test/CodeGen/RISCV/fold-addi-loadstore.ll b/llvm/test/CodeGen/RISCV/fold-addi-loadstore.ll
new file mode 100644
index 0000000000000..803778de1365d
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/fold-addi-loadstore.ll
@@ -0,0 +1,288 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+
+; We can often fold an ADDI into the offset of load/store instructions:
+;   (load (addi base, off1), off2) -> (load base, off1+off2)
+;   (store val, (addi base, off1), off2) -> (store val, base, off1+off2)
+; This is possible when the off1+off2 continues to fit the 12-bit immediate.
+; Check if we do the fold under various conditions. If off1 is (the low part of)
+; an address the fold's safety depends on the variable's alignment.
+
+@g_0 = global i64 0
+@g_1 = global i64 0, align 1
+@g_2 = global i64 0, align 2
+@g_4 = global i64 0, align 4
+@g_8 = global i64 0, align 8
+@g_16 = global i64 0, align 16
+
+define i64 @load_g_0() nounwind {
+; RV32I-LABEL: load_g_0:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a1, %hi(g_0)
+; RV32I-NEXT:    lw a0, %lo(g_0)(a1)
+; RV32I-NEXT:    lw a1, %lo(g_0+4)(a1)
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: load_g_0:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, %hi(g_0)
+; RV64I-NEXT:    ld a0, %lo(g_0)(a0)
+; RV64I-NEXT:    ret
+entry:
+  %0 = load i64, i64* @g_0
+  ret i64 %0
+}
+
+define i64 @load_g_1() nounwind {
+; RV32I-LABEL: load_g_1:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a1, %hi(g_1)
+; RV32I-NEXT:    lw a0, %lo(g_1)(a1)
+; RV32I-NEXT:    addi a1, a1, %lo(g_1)
+; RV32I-NEXT:    lw a1, 4(a1)
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: load_g_1:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, %hi(g_1)
+; RV64I-NEXT:    ld a0, %lo(g_1)(a0)
+; RV64I-NEXT:    ret
+entry:
+  %0 = load i64, i64* @g_1
+  ret i64 %0
+}
+
+define i64 @load_g_2() nounwind {
+; RV32I-LABEL: load_g_2:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a1, %hi(g_2)
+; RV32I-NEXT:    lw a0, %lo(g_2)(a1)
+; RV32I-NEXT:    addi a1, a1, %lo(g_2)
+; RV32I-NEXT:    lw a1, 4(a1)
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: load_g_2:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, %hi(g_2)
+; RV64I-NEXT:    ld a0, %lo(g_2)(a0)
+; RV64I-NEXT:    ret
+entry:
+  %0 = load i64, i64* @g_2
+  ret i64 %0
+}
+
+define i64 @load_g_4() nounwind {
+; RV32I-LABEL: load_g_4:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a1, %hi(g_4)
+; RV32I-NEXT:    lw a0, %lo(g_4)(a1)
+; RV32I-NEXT:    addi a1, a1, %lo(g_4)
+; RV32I-NEXT:    lw a1, 4(a1)
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: load_g_4:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, %hi(g_4)
+; RV64I-NEXT:    ld a0, %lo(g_4)(a0)
+; RV64I-NEXT:    ret
+entry:
+  %0 = load i64, i64* @g_4
+  ret i64 %0
+}
+
+define i64 @load_g_8() nounwind {
+; RV32I-LABEL: load_g_8:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a1, %hi(g_8)
+; RV32I-NEXT:    lw a0, %lo(g_8)(a1)
+; RV32I-NEXT:    lw a1, %lo(g_8+4)(a1)
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: load_g_8:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, %hi(g_8)
+; RV64I-NEXT:    ld a0, %lo(g_8)(a0)
+; RV64I-NEXT:    ret
+entry:
+  %0 = load i64, i64* @g_8
+  ret i64 %0
+}
+
+define i64 @load_g_16() nounwind {
+; RV32I-LABEL: load_g_16:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a1, %hi(g_16)
+; RV32I-NEXT:    lw a0, %lo(g_16)(a1)
+; RV32I-NEXT:    lw a1, %lo(g_16+4)(a1)
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: load_g_16:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, %hi(g_16)
+; RV64I-NEXT:    ld a0, %lo(g_16)(a0)
+; RV64I-NEXT:    ret
+entry:
+  %0 = load i64, i64* @g_16
+  ret i64 %0
+}
+
+define void @store_g_4() nounwind {
+; RV32I-LABEL: store_g_4:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, %hi(g_4)
+; RV32I-NEXT:    sw zero, %lo(g_4)(a0)
+; RV32I-NEXT:    addi a0, a0, %lo(g_4)
+; RV32I-NEXT:    sw zero, 4(a0)
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: store_g_4:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, %hi(g_4)
+; RV64I-NEXT:    sd zero, %lo(g_4)(a0)
+; RV64I-NEXT:    ret
+entry:
+   store i64 0, i64* @g_4
+   ret void
+}
+
+define void @store_g_8() nounwind {
+; RV32I-LABEL: store_g_8:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, %hi(g_8)
+; RV32I-NEXT:    sw zero, %lo(g_8+4)(a0)
+; RV32I-NEXT:    sw zero, %lo(g_8)(a0)
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: store_g_8:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, %hi(g_8)
+; RV64I-NEXT:    sd zero, %lo(g_8)(a0)
+; RV64I-NEXT:    ret
+entry:
+   store i64 0, i64* @g_8
+   ret void
+}
+
+; Check for folds in accesses to the second element of an i64 array.
+
+@ga_8 = dso_local local_unnamed_addr global [2 x i64] zeroinitializer, align 8
+@ga_16 = dso_local local_unnamed_addr global [2 x i64] zeroinitializer, align 16
+
+define i64 @load_ga_8() nounwind {
+; RV32I-LABEL: load_ga_8:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, %hi(ga_8)
+; RV32I-NEXT:    addi a1, a0, %lo(ga_8)
+; RV32I-NEXT:    lw a0, 8(a1)
+; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: load_ga_8:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, %hi(ga_8+8)
+; RV64I-NEXT:    ld a0, %lo(ga_8+8)(a0)
+; RV64I-NEXT:    ret
+entry:
+  %0 = load i64, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @ga_8, i32 0, i32 1)
+  ret i64 %0
+}
+
+define i64 @load_ga_16() nounwind {
+; RV32I-LABEL: load_ga_16:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a1, %hi(ga_16)
+; RV32I-NEXT:    lw a0, %lo(ga_16+8)(a1)
+; RV32I-NEXT:    lw a1, %lo(ga_16+12)(a1)
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: load_ga_16:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, %hi(ga_16)
+; RV64I-NEXT:    ld a0, %lo(ga_16+8)(a0)
+; RV64I-NEXT:    ret
+entry:
+  %0 = load i64, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @ga_16, i32 0, i32 1)
+  ret i64 %0
+}
+
+; Check for folds in accesses to thread-local variables.
+
+@tl_4 = thread_local global i64 0, align 4
+@tl_8 = thread_local global i64 0, align 8
+
+define i64 @load_tl_4() nounwind {
+; RV32I-LABEL: load_tl_4:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, %tprel_hi(tl_4)
+; RV32I-NEXT:    add a1, a0, tp, %tprel_add(tl_4)
+; RV32I-NEXT:    lw a0, %tprel_lo(tl_4)(a1)
+; RV32I-NEXT:    addi a1, a1, %tprel_lo(tl_4)
+; RV32I-NEXT:    lw a1, 4(a1)
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: load_tl_4:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, %tprel_hi(tl_4)
+; RV64I-NEXT:    add a0, a0, tp, %tprel_add(tl_4)
+; RV64I-NEXT:    ld a0, %tprel_lo(tl_4)(a0)
+; RV64I-NEXT:    ret
+entry:
+  %0 = load i64, i64* @tl_4
+  ret i64 %0
+}
+
+define i64 @load_tl_8() nounwind {
+; RV32I-LABEL: load_tl_8:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, %tprel_hi(tl_8)
+; RV32I-NEXT:    add a1, a0, tp, %tprel_add(tl_8)
+; RV32I-NEXT:    lw a0, %tprel_lo(tl_8)(a1)
+; RV32I-NEXT:    lw a1, %tprel_lo(tl_8+4)(a1)
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: load_tl_8:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    lui a0, %tprel_hi(tl_8)
+; RV64I-NEXT:    add a0, a0, tp, %tprel_add(tl_8)
+; RV64I-NEXT:    ld a0, %tprel_lo(tl_8)(a0)
+; RV64I-NEXT:    ret
+entry:
+  %0 = load i64, i64* @tl_8
+  ret i64 %0
+}
+
+define i64 @load_const_ok() nounwind {
+; RV32I-LABEL: load_const_ok:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lw a0, 2040(zero)
+; RV32I-NEXT:    lw a1, 2044(zero)
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: load_const_ok:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    ld a0, 2040(zero)
+; RV64I-NEXT:    ret
+entry:
+  %0 = load i64, i64* inttoptr (i32 2040 to i64*)
+  ret i64 %0
+}
+
+define i64 @load_cost_overflow() nounwind {
+; RV32I-LABEL: load_cost_overflow:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a0, 1
+; RV32I-NEXT:    lw a1, -2048(a0)
+; RV32I-NEXT:    lw a0, 2044(zero)
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: load_cost_overflow:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    ld a0, 2044(zero)
+; RV64I-NEXT:    ret
+entry:
+  %0 = load i64, i64* inttoptr (i64 2044 to i64*)
+  ret i64 %0
+}
diff --git a/llvm/test/CodeGen/RISCV/fp128.ll b/llvm/test/CodeGen/RISCV/fp128.ll
index 91b1702911af7..81a19d065ac55 100644
--- a/llvm/test/CodeGen/RISCV/fp128.ll
+++ b/llvm/test/CodeGen/RISCV/fp128.ll
@@ -14,27 +14,25 @@ define i32 @test_load_and_cmp() nounwind {
 ; RV32I-NEXT:    addi sp, sp, -48
 ; RV32I-NEXT:    sw ra, 44(sp)
 ; RV32I-NEXT:    lui a0, %hi(x)
-; RV32I-NEXT:    addi a1, a0, %lo(x)
-; RV32I-NEXT:    lw a6, 4(a1)
-; RV32I-NEXT:    lw a7, 8(a1)
-; RV32I-NEXT:    lw a1, 12(a1)
-; RV32I-NEXT:    lw a0, %lo(x)(a0)
+; RV32I-NEXT:    lw a6, %lo(x)(a0)
+; RV32I-NEXT:    lw a7, %lo(x+4)(a0)
+; RV32I-NEXT:    lw a3, %lo(x+8)(a0)
+; RV32I-NEXT:    lw a0, %lo(x+12)(a0)
 ; RV32I-NEXT:    lui a4, %hi(y)
-; RV32I-NEXT:    addi a5, a4, %lo(y)
-; RV32I-NEXT:    lw a2, 4(a5)
-; RV32I-NEXT:    lw a3, 8(a5)
-; RV32I-NEXT:    lw a5, 12(a5)
-; RV32I-NEXT:    lw a4, %lo(y)(a4)
-; RV32I-NEXT:    sw a4, 8(sp)
-; RV32I-NEXT:    sw a0, 24(sp)
-; RV32I-NEXT:    sw a5, 20(sp)
-; RV32I-NEXT:    sw a3, 16(sp)
+; RV32I-NEXT:    lw a5, %lo(y)(a4)
+; RV32I-NEXT:    lw a2, %lo(y+4)(a4)
+; RV32I-NEXT:    lw a1, %lo(y+8)(a4)
+; RV32I-NEXT:    lw a4, %lo(y+12)(a4)
+; RV32I-NEXT:    sw a4, 20(sp)
+; RV32I-NEXT:    sw a1, 16(sp)
 ; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    sw a1, 36(sp)
-; RV32I-NEXT:    sw a7, 32(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a0, 36(sp)
+; RV32I-NEXT:    sw a3, 32(sp)
+; RV32I-NEXT:    sw a7, 28(sp)
 ; RV32I-NEXT:    addi a0, sp, 24
 ; RV32I-NEXT:    addi a1, sp, 8
-; RV32I-NEXT:    sw a6, 28(sp)
+; RV32I-NEXT:    sw a6, 24(sp)
 ; RV32I-NEXT:    call __netf2
 ; RV32I-NEXT:    snez a0, a0
 ; RV32I-NEXT:    lw ra, 44(sp)
@@ -53,28 +51,26 @@ define i32 @test_add_and_fptosi() nounwind {
 ; RV32I-NEXT:    addi sp, sp, -80
 ; RV32I-NEXT:    sw ra, 76(sp)
 ; RV32I-NEXT:    lui a0, %hi(x)
-; RV32I-NEXT:    addi a1, a0, %lo(x)
-; RV32I-NEXT:    lw a6, 4(a1)
-; RV32I-NEXT:    lw a7, 8(a1)
-; RV32I-NEXT:    lw a1, 12(a1)
-; RV32I-NEXT:    lw a0, %lo(x)(a0)
+; RV32I-NEXT:    lw a6, %lo(x)(a0)
+; RV32I-NEXT:    lw a7, %lo(x+4)(a0)
+; RV32I-NEXT:    lw a2, %lo(x+8)(a0)
+; RV32I-NEXT:    lw a0, %lo(x+12)(a0)
 ; RV32I-NEXT:    lui a4, %hi(y)
-; RV32I-NEXT:    addi a5, a4, %lo(y)
-; RV32I-NEXT:    lw a3, 4(a5)
-; RV32I-NEXT:    lw a2, 8(a5)
-; RV32I-NEXT:    lw a5, 12(a5)
-; RV32I-NEXT:    lw a4, %lo(y)(a4)
-; RV32I-NEXT:    sw a4, 24(sp)
-; RV32I-NEXT:    sw a0, 40(sp)
-; RV32I-NEXT:    sw a5, 36(sp)
-; RV32I-NEXT:    sw a2, 32(sp)
+; RV32I-NEXT:    lw a5, %lo(y)(a4)
+; RV32I-NEXT:    lw a3, %lo(y+4)(a4)
+; RV32I-NEXT:    lw a1, %lo(y+8)(a4)
+; RV32I-NEXT:    lw a4, %lo(y+12)(a4)
+; RV32I-NEXT:    sw a4, 36(sp)
+; RV32I-NEXT:    sw a1, 32(sp)
 ; RV32I-NEXT:    sw a3, 28(sp)
-; RV32I-NEXT:    sw a1, 52(sp)
-; RV32I-NEXT:    sw a7, 48(sp)
+; RV32I-NEXT:    sw a5, 24(sp)
+; RV32I-NEXT:    sw a0, 52(sp)
+; RV32I-NEXT:    sw a2, 48(sp)
+; RV32I-NEXT:    sw a7, 44(sp)
 ; RV32I-NEXT:    addi a0, sp, 56
 ; RV32I-NEXT:    addi a1, sp, 40
 ; RV32I-NEXT:    addi a2, sp, 24
-; RV32I-NEXT:    sw a6, 44(sp)
+; RV32I-NEXT:    sw a6, 40(sp)
 ; RV32I-NEXT:    call __addtf3
 ; RV32I-NEXT:    lw a1, 56(sp)
 ; RV32I-NEXT:    lw a0, 60(sp)
diff --git a/llvm/test/CodeGen/RISCV/inline-asm-abi-names.ll b/llvm/test/CodeGen/RISCV/inline-asm-abi-names.ll
index 4d85e3ea006b8..f9ed4aed6ca32 100644
--- a/llvm/test/CodeGen/RISCV/inline-asm-abi-names.ll
+++ b/llvm/test/CodeGen/RISCV/inline-asm-abi-names.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -no-integrated-as < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I %s
-; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs -no-integrated-as < %s \
 ; RUN:   | FileCheck -check-prefix=RV64I %s
 
 ; These test that we can use both the architectural names (x*) and the ABI names
diff --git a/llvm/test/CodeGen/RISCV/inline-asm.ll b/llvm/test/CodeGen/RISCV/inline-asm.ll
index 43f951e352a68..de5d9a5f22a83 100644
--- a/llvm/test/CodeGen/RISCV/inline-asm.ll
+++ b/llvm/test/CodeGen/RISCV/inline-asm.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -no-integrated-as < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I %s
-; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs -no-integrated-as < %s \
 ; RUN:   | FileCheck -check-prefix=RV64I %s
 
 @gi = external global i32
diff --git a/llvm/test/CodeGen/RISCV/interrupt-attr-nocall.ll b/llvm/test/CodeGen/RISCV/interrupt-attr-nocall.ll
index c36a01ca3d984..025f92c3de968 100644
--- a/llvm/test/CodeGen/RISCV/interrupt-attr-nocall.ll
+++ b/llvm/test/CodeGen/RISCV/interrupt-attr-nocall.ll
@@ -383,16 +383,13 @@ define void @foo_double() nounwind #0 {
 ; CHECK-RV32-NEXT:    sw t6, 0(sp)
 ; CHECK-RV32-NEXT:    lui a1, %hi(h)
 ; CHECK-RV32-NEXT:    lw a0, %lo(h)(a1)
-; CHECK-RV32-NEXT:    addi a1, a1, %lo(h)
-; CHECK-RV32-NEXT:    lw a1, 4(a1)
+; CHECK-RV32-NEXT:    lw a1, %lo(h+4)(a1)
 ; CHECK-RV32-NEXT:    lui a3, %hi(i)
 ; CHECK-RV32-NEXT:    lw a2, %lo(i)(a3)
-; CHECK-RV32-NEXT:    addi a3, a3, %lo(i)
-; CHECK-RV32-NEXT:    lw a3, 4(a3)
+; CHECK-RV32-NEXT:    lw a3, %lo(i+4)(a3)
 ; CHECK-RV32-NEXT:    call __adddf3
 ; CHECK-RV32-NEXT:    lui a2, %hi(g)
-; CHECK-RV32-NEXT:    addi a3, a2, %lo(g)
-; CHECK-RV32-NEXT:    sw a1, 4(a3)
+; CHECK-RV32-NEXT:    sw a1, %lo(g+4)(a2)
 ; CHECK-RV32-NEXT:    sw a0, %lo(g)(a2)
 ; CHECK-RV32-NEXT:    lw t6, 0(sp)
 ; CHECK-RV32-NEXT:    lw t5, 4(sp)
@@ -466,16 +463,13 @@ define void @foo_double() nounwind #0 {
 ; CHECK-RV32IF-NEXT:    fsw fs11, 0(sp)
 ; CHECK-RV32IF-NEXT:    lui a1, %hi(h)
 ; CHECK-RV32IF-NEXT:    lw a0, %lo(h)(a1)
-; CHECK-RV32IF-NEXT:    addi a1, a1, %lo(h)
-; CHECK-RV32IF-NEXT:    lw a1, 4(a1)
+; CHECK-RV32IF-NEXT:    lw a1, %lo(h+4)(a1)
 ; CHECK-RV32IF-NEXT:    lui a3, %hi(i)
 ; CHECK-RV32IF-NEXT:    lw a2, %lo(i)(a3)
-; CHECK-RV32IF-NEXT:    addi a3, a3, %lo(i)
-; CHECK-RV32IF-NEXT:    lw a3, 4(a3)
+; CHECK-RV32IF-NEXT:    lw a3, %lo(i+4)(a3)
 ; CHECK-RV32IF-NEXT:    call __adddf3
 ; CHECK-RV32IF-NEXT:    lui a2, %hi(g)
-; CHECK-RV32IF-NEXT:    addi a3, a2, %lo(g)
-; CHECK-RV32IF-NEXT:    sw a1, 4(a3)
+; CHECK-RV32IF-NEXT:    sw a1, %lo(g+4)(a2)
 ; CHECK-RV32IF-NEXT:    sw a0, %lo(g)(a2)
 ; CHECK-RV32IF-NEXT:    flw fs11, 0(sp)
 ; CHECK-RV32IF-NEXT:    flw fs10, 4(sp)
@@ -580,16 +574,13 @@ define void @foo_fp_double() nounwind #1 {
 ; CHECK-RV32-NEXT:    addi s0, sp, 80
 ; CHECK-RV32-NEXT:    lui a1, %hi(h)
 ; CHECK-RV32-NEXT:    lw a0, %lo(h)(a1)
-; CHECK-RV32-NEXT:    addi a1, a1, %lo(h)
-; CHECK-RV32-NEXT:    lw a1, 4(a1)
+; CHECK-RV32-NEXT:    lw a1, %lo(h+4)(a1)
 ; CHECK-RV32-NEXT:    lui a3, %hi(i)
 ; CHECK-RV32-NEXT:    lw a2, %lo(i)(a3)
-; CHECK-RV32-NEXT:    addi a3, a3, %lo(i)
-; CHECK-RV32-NEXT:    lw a3, 4(a3)
+; CHECK-RV32-NEXT:    lw a3, %lo(i+4)(a3)
 ; CHECK-RV32-NEXT:    call __adddf3
 ; CHECK-RV32-NEXT:    lui a2, %hi(g)
-; CHECK-RV32-NEXT:    addi a3, a2, %lo(g)
-; CHECK-RV32-NEXT:    sw a1, 4(a3)
+; CHECK-RV32-NEXT:    sw a1, %lo(g+4)(a2)
 ; CHECK-RV32-NEXT:    sw a0, %lo(g)(a2)
 ; CHECK-RV32-NEXT:    lw t6, 12(sp)
 ; CHECK-RV32-NEXT:    lw t5, 16(sp)
@@ -666,16 +657,13 @@ define void @foo_fp_double() nounwind #1 {
 ; CHECK-RV32IF-NEXT:    addi s0, sp, 208
 ; CHECK-RV32IF-NEXT:    lui a1, %hi(h)
 ; CHECK-RV32IF-NEXT:    lw a0, %lo(h)(a1)
-; CHECK-RV32IF-NEXT:    addi a1, a1, %lo(h)
-; CHECK-RV32IF-NEXT:    lw a1, 4(a1)
+; CHECK-RV32IF-NEXT:    lw a1, %lo(h+4)(a1)
 ; CHECK-RV32IF-NEXT:    lui a3, %hi(i)
 ; CHECK-RV32IF-NEXT:    lw a2, %lo(i)(a3)
-; CHECK-RV32IF-NEXT:    addi a3, a3, %lo(i)
-; CHECK-RV32IF-NEXT:    lw a3, 4(a3)
+; CHECK-RV32IF-NEXT:    lw a3, %lo(i+4)(a3)
 ; CHECK-RV32IF-NEXT:    call __adddf3
 ; CHECK-RV32IF-NEXT:    lui a2, %hi(g)
-; CHECK-RV32IF-NEXT:    addi a3, a2, %lo(g)
-; CHECK-RV32IF-NEXT:    sw a1, 4(a3)
+; CHECK-RV32IF-NEXT:    sw a1, %lo(g+4)(a2)
 ; CHECK-RV32IF-NEXT:    sw a0, %lo(g)(a2)
 ; CHECK-RV32IF-NEXT:    flw fs11, 12(sp)
 ; CHECK-RV32IF-NEXT:    flw fs10, 16(sp)
diff --git a/llvm/test/CodeGen/RISCV/large-stack.ll b/llvm/test/CodeGen/RISCV/large-stack.ll
index 7acf0f4076e85..7cc6e83d7d85c 100644
--- a/llvm/test/CodeGen/RISCV/large-stack.ll
+++ b/llvm/test/CodeGen/RISCV/large-stack.ll
@@ -64,10 +64,12 @@ define void @test_emergency_spill_slot(i32 %a) {
 ; RV32I-FPELIM-NEXT:    add a1, a2, a1
 ; RV32I-FPELIM-NEXT:    #APP
 ; RV32I-FPELIM-NEXT:    nop
+; RV32I-FPELIM-EMPTY:
 ; RV32I-FPELIM-NEXT:    #NO_APP
 ; RV32I-FPELIM-NEXT:    sw a0, 0(a1)
 ; RV32I-FPELIM-NEXT:    #APP
 ; RV32I-FPELIM-NEXT:    nop
+; RV32I-FPELIM-EMPTY:
 ; RV32I-FPELIM-NEXT:    #NO_APP
 ; RV32I-FPELIM-NEXT:    lui a0, 97
 ; RV32I-FPELIM-NEXT:    addi a0, a0, 672
@@ -103,10 +105,12 @@ define void @test_emergency_spill_slot(i32 %a) {
 ; RV32I-WITHFP-NEXT:    add a1, a2, a1
 ; RV32I-WITHFP-NEXT:    #APP
 ; RV32I-WITHFP-NEXT:    nop
+; RV32I-WITHFP-EMPTY:
 ; RV32I-WITHFP-NEXT:    #NO_APP
 ; RV32I-WITHFP-NEXT:    sw a0, 0(a1)
 ; RV32I-WITHFP-NEXT:    #APP
 ; RV32I-WITHFP-NEXT:    nop
+; RV32I-WITHFP-EMPTY:
 ; RV32I-WITHFP-NEXT:    #NO_APP
 ; RV32I-WITHFP-NEXT:    lui a0, 97
 ; RV32I-WITHFP-NEXT:    addi a0, a0, 688
diff --git a/llvm/test/CodeGen/RISCV/mir-target-flags.ll b/llvm/test/CodeGen/RISCV/mir-target-flags.ll
index f41fb77dbb00c..b1bf935c4e3bf 100644
--- a/llvm/test/CodeGen/RISCV/mir-target-flags.ll
+++ b/llvm/test/CodeGen/RISCV/mir-target-flags.ll
@@ -27,11 +27,11 @@ define i32 @caller(i32 %a) nounwind {
 ; RV32-SMALL-NEXT: target-flags(riscv-hi) @g_i
 ; RV32-SMALL-NEXT: target-flags(riscv-lo) @g_i
 ; RV32-SMALL:      target-flags(riscv-tls-got-hi) @t_un
-; RV32-SMALL-NEXT: target-flags(riscv-pcrel-lo) %bb.1
+; RV32-SMALL-NEXT: target-flags(riscv-pcrel-lo)
 ; RV32-SMALL:      target-flags(riscv-tls-got-hi) @t_ld
-; RV32-SMALL-NEXT: target-flags(riscv-pcrel-lo) %bb.2
+; RV32-SMALL-NEXT: target-flags(riscv-pcrel-lo)
 ; RV32-SMALL:      target-flags(riscv-tls-got-hi) @t_ie
-; RV32-SMALL-NEXT: target-flags(riscv-pcrel-lo) %bb.3
+; RV32-SMALL-NEXT: target-flags(riscv-pcrel-lo)
 ; RV32-SMALL:      target-flags(riscv-tprel-hi) @t_le
 ; RV32-SMALL-NEXT: target-flags(riscv-tprel-add) @t_le
 ; RV32-SMALL-NEXT: target-flags(riscv-tprel-lo) @t_le
@@ -39,17 +39,17 @@ define i32 @caller(i32 %a) nounwind {
 ;
 ; RV32-MED-LABEL: name: caller
 ; RV32-MED:      target-flags(riscv-got-hi) @g_e
-; RV32-MED-NEXT: target-flags(riscv-pcrel-lo) %bb.1
+; RV32-MED-NEXT: target-flags(riscv-pcrel-lo)
 ; RV32-MED:      target-flags(riscv-pcrel-hi) @g_i
-; RV32-MED-NEXT: target-flags(riscv-pcrel-lo) %bb.2
+; RV32-MED-NEXT: target-flags(riscv-pcrel-lo)
 ; RV32-MED:      target-flags(riscv-tls-gd-hi) @t_un
-; RV32-MED-NEXT: target-flags(riscv-pcrel-lo) %bb.3
-; RV32-MED-NEXT: target-flags(riscv-plt) &__tls_get_addr
+; RV32-MED-NEXT: target-flags(riscv-pcrel-lo)
+; RV32-MED:      target-flags(riscv-plt) &__tls_get_addr
 ; RV32-MED:      target-flags(riscv-tls-gd-hi) @t_ld
-; RV32-MED-NEXT: target-flags(riscv-pcrel-lo) %bb.4
-; RV32-MED-NEXT: target-flags(riscv-plt) &__tls_get_addr
+; RV32-MED-NEXT: target-flags(riscv-pcrel-lo)
+; RV32-MED:      target-flags(riscv-plt) &__tls_get_addr
 ; RV32-MED:      target-flags(riscv-tls-got-hi) @t_ie
-; RV32-MED-NEXT: target-flags(riscv-pcrel-lo) %bb.5
+; RV32-MED-NEXT: target-flags(riscv-pcrel-lo)
 ; RV32-MED:      target-flags(riscv-tprel-hi) @t_le
 ; RV32-MED-NEXT: target-flags(riscv-tprel-add) @t_le
 ; RV32-MED-NEXT: target-flags(riscv-tprel-lo) @t_le
diff --git a/llvm/test/CodeGen/RISCV/mul.ll b/llvm/test/CodeGen/RISCV/mul.ll
index 2448580d39853..89c4bce122fd2 100644
--- a/llvm/test/CodeGen/RISCV/mul.ll
+++ b/llvm/test/CodeGen/RISCV/mul.ll
@@ -79,12 +79,8 @@ define signext i32 @mul(i32 %a, i32 %b) nounwind {
 define signext i32 @mul_constant(i32 %a) nounwind {
 ; RV32I-LABEL: mul_constant:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    addi a1, zero, 5
-; RV32I-NEXT:    call __mulsi3
-; RV32I-NEXT:    lw ra, 12(sp)
-; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    slli a1, a0, 2
+; RV32I-NEXT:    add a0, a1, a0
 ; RV32I-NEXT:    ret
 ;
 ; RV32IM-LABEL: mul_constant:
@@ -95,19 +91,14 @@ define signext i32 @mul_constant(i32 %a) nounwind {
 ;
 ; RV64I-LABEL: mul_constant:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    addi a1, zero, 5
-; RV64I-NEXT:    call __muldi3
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    ld ra, 8(sp)
-; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    slli a1, a0, 2
+; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    ret
 ;
 ; RV64IM-LABEL: mul_constant:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    addi a1, zero, 5
-; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    slli a1, a0, 2
+; RV64IM-NEXT:    addw a0, a1, a0
 ; RV64IM-NEXT:    ret
   %1 = mul i32 %a, 5
   ret i32 %1
@@ -177,13 +168,15 @@ define i64 @mul64(i64 %a, i64 %b) nounwind {
 define i64 @mul64_constant(i64 %a) nounwind {
 ; RV32I-LABEL: mul64_constant:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __muldi3
-; RV32I-NEXT:    lw ra, 12(sp)
-; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    slli a3, a0, 2
+; RV32I-NEXT:    add a2, a3, a0
+; RV32I-NEXT:    sltu a3, a2, a3
+; RV32I-NEXT:    srli a0, a0, 30
+; RV32I-NEXT:    slli a4, a1, 2
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    add a1, a0, a3
+; RV32I-NEXT:    mv a0, a2
 ; RV32I-NEXT:    ret
 ;
 ; RV32IM-LABEL: mul64_constant:
@@ -197,18 +190,14 @@ define i64 @mul64_constant(i64 %a) nounwind {
 ;
 ; RV64I-LABEL: mul64_constant:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    addi a1, zero, 5
-; RV64I-NEXT:    call __muldi3
-; RV64I-NEXT:    ld ra, 8(sp)
-; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    slli a1, a0, 2
+; RV64I-NEXT:    add a0, a1, a0
 ; RV64I-NEXT:    ret
 ;
 ; RV64IM-LABEL: mul64_constant:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    addi a1, zero, 5
-; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    slli a1, a0, 2
+; RV64IM-NEXT:    add a0, a1, a0
 ; RV64IM-NEXT:    ret
   %1 = mul i64 %a, 5
   ret i64 %1
@@ -301,3 +290,275 @@ define zeroext i32 @mulhu(i32 zeroext %a, i32 zeroext %b) nounwind {
   %5 = trunc i64 %4 to i32
   ret i32 %5
 }
+
+define i32 @muli32_p65(i32 %a) nounwind {
+; RV32I-LABEL: muli32_p65:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a0, 6
+; RV32I-NEXT:    add a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: muli32_p65:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi a1, zero, 65
+; RV32IM-NEXT:    mul a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: muli32_p65:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 6
+; RV64I-NEXT:    addw a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: muli32_p65:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    slli a1, a0, 6
+; RV64IM-NEXT:    addw a0, a1, a0
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, 65
+  ret i32 %1
+}
+
+define i32 @muli32_p63(i32 %a) nounwind {
+; RV32I-LABEL: muli32_p63:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a0, 6
+; RV32I-NEXT:    sub a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: muli32_p63:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi a1, zero, 63
+; RV32IM-NEXT:    mul a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: muli32_p63:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 6
+; RV64I-NEXT:    subw a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: muli32_p63:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    slli a1, a0, 6
+; RV64IM-NEXT:    subw a0, a1, a0
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, 63
+  ret i32 %1
+}
+
+define i64 @muli64_p65(i64 %a) nounwind {
+; RV32I-LABEL: muli64_p65:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a3, a0, 6
+; RV32I-NEXT:    add a2, a3, a0
+; RV32I-NEXT:    sltu a3, a2, a3
+; RV32I-NEXT:    srli a0, a0, 26
+; RV32I-NEXT:    slli a4, a1, 6
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    add a1, a0, a3
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: muli64_p65:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi a2, zero, 65
+; RV32IM-NEXT:    mul a1, a1, a2
+; RV32IM-NEXT:    mulhu a3, a0, a2
+; RV32IM-NEXT:    add a1, a3, a1
+; RV32IM-NEXT:    mul a0, a0, a2
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: muli64_p65:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 6
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: muli64_p65:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    slli a1, a0, 6
+; RV64IM-NEXT:    add a0, a1, a0
+; RV64IM-NEXT:    ret
+  %1 = mul i64 %a, 65
+  ret i64 %1
+}
+
+define i64 @muli64_p63(i64 %a) nounwind {
+; RV32I-LABEL: muli64_p63:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a2, a0, 6
+; RV32I-NEXT:    sltu a3, a2, a0
+; RV32I-NEXT:    srli a4, a0, 26
+; RV32I-NEXT:    slli a5, a1, 6
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    sub a1, a4, a1
+; RV32I-NEXT:    sub a1, a1, a3
+; RV32I-NEXT:    sub a0, a2, a0
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: muli64_p63:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi a2, zero, 63
+; RV32IM-NEXT:    mul a1, a1, a2
+; RV32IM-NEXT:    mulhu a3, a0, a2
+; RV32IM-NEXT:    add a1, a3, a1
+; RV32IM-NEXT:    mul a0, a0, a2
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: muli64_p63:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 6
+; RV64I-NEXT:    sub a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: muli64_p63:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    slli a1, a0, 6
+; RV64IM-NEXT:    sub a0, a1, a0
+; RV64IM-NEXT:    ret
+  %1 = mul i64 %a, 63
+  ret i64 %1
+}
+
+define i32 @muli32_m63(i32 %a) nounwind {
+; RV32I-LABEL: muli32_m63:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a0, 6
+; RV32I-NEXT:    sub a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: muli32_m63:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi a1, zero, -63
+; RV32IM-NEXT:    mul a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: muli32_m63:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 6
+; RV64I-NEXT:    subw a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: muli32_m63:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    slli a1, a0, 6
+; RV64IM-NEXT:    subw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, -63
+  ret i32 %1
+}
+
+define i32 @muli32_m65(i32 %a) nounwind {
+; RV32I-LABEL: muli32_m65:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a0, 6
+; RV32I-NEXT:    add a0, a1, a0
+; RV32I-NEXT:    neg a0, a0
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: muli32_m65:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi a1, zero, -65
+; RV32IM-NEXT:    mul a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: muli32_m65:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 6
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    negw a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: muli32_m65:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    slli a1, a0, 6
+; RV64IM-NEXT:    add a0, a1, a0
+; RV64IM-NEXT:    negw a0, a0
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, -65
+  ret i32 %1
+}
+
+define i64 @muli64_m63(i64 %a) nounwind {
+; RV32I-LABEL: muli64_m63:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a2, a0, 6
+; RV32I-NEXT:    sltu a3, a0, a2
+; RV32I-NEXT:    srli a4, a0, 26
+; RV32I-NEXT:    slli a5, a1, 6
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    sub a1, a1, a4
+; RV32I-NEXT:    sub a1, a1, a3
+; RV32I-NEXT:    sub a0, a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: muli64_m63:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi a2, zero, -63
+; RV32IM-NEXT:    mul a1, a1, a2
+; RV32IM-NEXT:    mulhu a3, a0, a2
+; RV32IM-NEXT:    sub a3, a3, a0
+; RV32IM-NEXT:    add a1, a3, a1
+; RV32IM-NEXT:    mul a0, a0, a2
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: muli64_m63:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 6
+; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: muli64_m63:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    slli a1, a0, 6
+; RV64IM-NEXT:    sub a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i64 %a, -63
+  ret i64 %1
+}
+
+define i64 @muli64_m65(i64 %a) nounwind {
+; RV32I-LABEL: muli64_m65:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a2, a0, 6
+; RV32I-NEXT:    add a3, a2, a0
+; RV32I-NEXT:    sltu a2, a3, a2
+; RV32I-NEXT:    srli a0, a0, 26
+; RV32I-NEXT:    slli a4, a1, 6
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    add a0, a0, a2
+; RV32I-NEXT:    snez a1, a3
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    neg a1, a0
+; RV32I-NEXT:    neg a0, a3
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: muli64_m65:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi a2, zero, -65
+; RV32IM-NEXT:    mul a1, a1, a2
+; RV32IM-NEXT:    mulhu a3, a0, a2
+; RV32IM-NEXT:    sub a3, a3, a0
+; RV32IM-NEXT:    add a1, a3, a1
+; RV32IM-NEXT:    mul a0, a0, a2
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: muli64_m65:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 6
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    neg a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: muli64_m65:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    slli a1, a0, 6
+; RV64IM-NEXT:    add a0, a1, a0
+; RV64IM-NEXT:    neg a0, a0
+; RV64IM-NEXT:    ret
+  %1 = mul i64 %a, -65
+  ret i64 %1
+}
diff --git a/llvm/test/CodeGen/RISCV/pic-models.ll b/llvm/test/CodeGen/RISCV/pic-models.ll
index 8d835ae99f406..46e9cee57d79d 100644
--- a/llvm/test/CodeGen/RISCV/pic-models.ll
+++ b/llvm/test/CodeGen/RISCV/pic-models.ll
@@ -26,10 +26,9 @@ define i32* @f1() nounwind {
 ;
 ; RV32-PIC-LABEL: f1:
 ; RV32-PIC:       # %bb.0: # %entry
-; RV32-PIC-NEXT:  .LBB0_1: # %entry
-; RV32-PIC-NEXT:    # Label of block must be emitted
+; RV32-PIC-NEXT:  .Ltmp0:
 ; RV32-PIC-NEXT:    auipc a0, %got_pcrel_hi(external_var)
-; RV32-PIC-NEXT:    lw a0, %pcrel_lo(.LBB0_1)(a0)
+; RV32-PIC-NEXT:    lw a0, %pcrel_lo(.Ltmp0)(a0)
 ; RV32-PIC-NEXT:    ret
 ;
 ; RV64-STATIC-LABEL: f1:
@@ -40,10 +39,9 @@ define i32* @f1() nounwind {
 ;
 ; RV64-PIC-LABEL: f1:
 ; RV64-PIC:       # %bb.0: # %entry
-; RV64-PIC-NEXT:  .LBB0_1: # %entry
-; RV64-PIC-NEXT:    # Label of block must be emitted
+; RV64-PIC-NEXT:  .Ltmp0:
 ; RV64-PIC-NEXT:    auipc a0, %got_pcrel_hi(external_var)
-; RV64-PIC-NEXT:    ld a0, %pcrel_lo(.LBB0_1)(a0)
+; RV64-PIC-NEXT:    ld a0, %pcrel_lo(.Ltmp0)(a0)
 ; RV64-PIC-NEXT:    ret
 entry:
   ret i32* @external_var
@@ -61,10 +59,9 @@ define i32* @f2() nounwind {
 ;
 ; RV32-PIC-LABEL: f2:
 ; RV32-PIC:       # %bb.0: # %entry
-; RV32-PIC-NEXT:  .LBB1_1: # %entry
-; RV32-PIC-NEXT:    # Label of block must be emitted
+; RV32-PIC-NEXT:  .Ltmp1:
 ; RV32-PIC-NEXT:    auipc a0, %pcrel_hi(internal_var)
-; RV32-PIC-NEXT:    addi a0, a0, %pcrel_lo(.LBB1_1)
+; RV32-PIC-NEXT:    addi a0, a0, %pcrel_lo(.Ltmp1)
 ; RV32-PIC-NEXT:    ret
 ;
 ; RV64-STATIC-LABEL: f2:
@@ -75,10 +72,9 @@ define i32* @f2() nounwind {
 ;
 ; RV64-PIC-LABEL: f2:
 ; RV64-PIC:       # %bb.0: # %entry
-; RV64-PIC-NEXT:  .LBB1_1: # %entry
-; RV64-PIC-NEXT:    # Label of block must be emitted
+; RV64-PIC-NEXT:  .Ltmp1:
 ; RV64-PIC-NEXT:    auipc a0, %pcrel_hi(internal_var)
-; RV64-PIC-NEXT:    addi a0, a0, %pcrel_lo(.LBB1_1)
+; RV64-PIC-NEXT:    addi a0, a0, %pcrel_lo(.Ltmp1)
 ; RV64-PIC-NEXT:    ret
 entry:
   ret i32* @internal_var
diff --git a/llvm/test/CodeGen/RISCV/select-and.ll b/llvm/test/CodeGen/RISCV/select-and.ll
new file mode 100644
index 0000000000000..b5319dab1439b
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/select-and.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+
+;; There are a few different ways to lower (select (and A, B), X, Y). This test
+;; ensures that we do so with as few branches as possible.
+
+define signext i32 @select_of_and(i1 zeroext %a, i1 zeroext %b, i32 signext %c, i32 signext %d) nounwind {
+; RV32I-LABEL: select_of_and:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    and a1, a0, a1
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    bnez a1, .LBB0_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    mv a0, a3
+; RV32I-NEXT:  .LBB0_2:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: select_of_and:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    and a1, a0, a1
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    bnez a1, .LBB0_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    mv a0, a3
+; RV64I-NEXT:  .LBB0_2:
+; RV64I-NEXT:    ret
+  %1 = and i1 %a, %b
+  %2 = select i1 %1, i32 %c, i32 %d
+  ret i32 %2
+}
+
+declare signext i32 @both() nounwind
+declare signext i32 @neither() nounwind
+
+define signext i32 @if_of_and(i1 zeroext %a, i1 zeroext %b) nounwind {
+; RV32I-LABEL: if_of_and:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    addi a1, zero, 1
+; RV32I-NEXT:    bne a0, a1, .LBB1_2
+; RV32I-NEXT:  # %bb.1: # %if.then
+; RV32I-NEXT:    call both
+; RV32I-NEXT:    j .LBB1_3
+; RV32I-NEXT:  .LBB1_2: # %if.else
+; RV32I-NEXT:    call neither
+; RV32I-NEXT:  .LBB1_3: # %if.end
+; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: if_of_and:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    addi a1, zero, 1
+; RV64I-NEXT:    bne a0, a1, .LBB1_2
+; RV64I-NEXT:  # %bb.1: # %if.then
+; RV64I-NEXT:    call both
+; RV64I-NEXT:    j .LBB1_3
+; RV64I-NEXT:  .LBB1_2: # %if.else
+; RV64I-NEXT:    call neither
+; RV64I-NEXT:  .LBB1_3: # %if.end
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = and i1 %a, %b
+  br i1 %1, label %if.then, label %if.else
+
+if.then:
+  %2 = tail call i32 @both()
+  br label %if.end
+
+if.else:
+  %3 = tail call i32 @neither()
+  br label %if.end
+
+if.end:
+  %4 = phi i32 [%2, %if.then], [%3, %if.else]
+  ret i32 %4
+}
diff --git a/llvm/test/CodeGen/RISCV/select-or.ll b/llvm/test/CodeGen/RISCV/select-or.ll
new file mode 100644
index 0000000000000..30b0d331d6f4e
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/select-or.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+
+;; There are a few different ways to lower (select (or A, B), X, Y). This test
+;; ensures that we do so with as few branches as possible.
+
+define signext i32 @select_of_or(i1 zeroext %a, i1 zeroext %b, i32 signext %c, i32 signext %d) nounwind {
+; RV32I-LABEL: select_of_or:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    or a1, a0, a1
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    bnez a1, .LBB0_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    mv a0, a3
+; RV32I-NEXT:  .LBB0_2:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: select_of_or:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    or a1, a0, a1
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    bnez a1, .LBB0_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    mv a0, a3
+; RV64I-NEXT:  .LBB0_2:
+; RV64I-NEXT:    ret
+  %1 = or i1 %a, %b
+  %2 = select i1 %1, i32 %c, i32 %d
+  ret i32 %2
+}
+
+declare signext i32 @either() nounwind
+declare signext i32 @neither() nounwind
+
+define signext i32 @if_of_or(i1 zeroext %a, i1 zeroext %b) nounwind {
+; RV32I-LABEL: if_of_or:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    addi a1, zero, 1
+; RV32I-NEXT:    bne a0, a1, .LBB1_2
+; RV32I-NEXT:  # %bb.1: # %if.then
+; RV32I-NEXT:    call either
+; RV32I-NEXT:    j .LBB1_3
+; RV32I-NEXT:  .LBB1_2: # %if.else
+; RV32I-NEXT:    call neither
+; RV32I-NEXT:  .LBB1_3: # %if.end
+; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: if_of_or:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    addi a1, zero, 1
+; RV64I-NEXT:    bne a0, a1, .LBB1_2
+; RV64I-NEXT:  # %bb.1: # %if.then
+; RV64I-NEXT:    call either
+; RV64I-NEXT:    j .LBB1_3
+; RV64I-NEXT:  .LBB1_2: # %if.else
+; RV64I-NEXT:    call neither
+; RV64I-NEXT:  .LBB1_3: # %if.end
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+  %1 = or i1 %a, %b
+  br i1 %1, label %if.then, label %if.else
+
+if.then:
+  %2 = tail call i32 @either()
+  br label %if.end
+
+if.else:
+  %3 = tail call i32 @neither()
+  br label %if.end
+
+if.end:
+  %4 = phi i32 [%2, %if.then], [%3, %if.else]
+  ret i32 %4
+}
diff --git a/llvm/test/CodeGen/RISCV/tls-models.ll b/llvm/test/CodeGen/RISCV/tls-models.ll
index 25a2f71beb317..27f63ff336740 100644
--- a/llvm/test/CodeGen/RISCV/tls-models.ll
+++ b/llvm/test/CodeGen/RISCV/tls-models.ll
@@ -23,10 +23,9 @@ define i32* @f1() nounwind {
 ; RV32-PIC:       # %bb.0: # %entry
 ; RV32-PIC-NEXT:    addi sp, sp, -16
 ; RV32-PIC-NEXT:    sw ra, 12(sp)
-; RV32-PIC-NEXT:  .LBB0_1: # %entry
-; RV32-PIC-NEXT:    # Label of block must be emitted
+; RV32-PIC-NEXT:  .Ltmp0:
 ; RV32-PIC-NEXT:    auipc a0, %tls_gd_pcrel_hi(unspecified)
-; RV32-PIC-NEXT:    addi a0, a0, %pcrel_lo(.LBB0_1)
+; RV32-PIC-NEXT:    addi a0, a0, %pcrel_lo(.Ltmp0)
 ; RV32-PIC-NEXT:    call __tls_get_addr@plt
 ; RV32-PIC-NEXT:    lw ra, 12(sp)
 ; RV32-PIC-NEXT:    addi sp, sp, 16
@@ -36,10 +35,9 @@ define i32* @f1() nounwind {
 ; RV64-PIC:       # %bb.0: # %entry
 ; RV64-PIC-NEXT:    addi sp, sp, -16
 ; RV64-PIC-NEXT:    sd ra, 8(sp)
-; RV64-PIC-NEXT:  .LBB0_1: # %entry
-; RV64-PIC-NEXT:    # Label of block must be emitted
+; RV64-PIC-NEXT:  .Ltmp0:
 ; RV64-PIC-NEXT:    auipc a0, %tls_gd_pcrel_hi(unspecified)
-; RV64-PIC-NEXT:    addi a0, a0, %pcrel_lo(.LBB0_1)
+; RV64-PIC-NEXT:    addi a0, a0, %pcrel_lo(.Ltmp0)
 ; RV64-PIC-NEXT:    call __tls_get_addr@plt
 ; RV64-PIC-NEXT:    ld ra, 8(sp)
 ; RV64-PIC-NEXT:    addi sp, sp, 16
@@ -47,19 +45,17 @@ define i32* @f1() nounwind {
 ;
 ; RV32-NOPIC-LABEL: f1:
 ; RV32-NOPIC:       # %bb.0: # %entry
-; RV32-NOPIC-NEXT:  .LBB0_1: # %entry
-; RV32-NOPIC-NEXT:    # Label of block must be emitted
+; RV32-NOPIC-NEXT:  .Ltmp0:
 ; RV32-NOPIC-NEXT:    auipc a0, %tls_ie_pcrel_hi(unspecified)
-; RV32-NOPIC-NEXT:    lw a0, %pcrel_lo(.LBB0_1)(a0)
+; RV32-NOPIC-NEXT:    lw a0, %pcrel_lo(.Ltmp0)(a0)
 ; RV32-NOPIC-NEXT:    add a0, a0, tp
 ; RV32-NOPIC-NEXT:    ret
 ;
 ; RV64-NOPIC-LABEL: f1:
 ; RV64-NOPIC:       # %bb.0: # %entry
-; RV64-NOPIC-NEXT:  .LBB0_1: # %entry
-; RV64-NOPIC-NEXT:    # Label of block must be emitted
+; RV64-NOPIC-NEXT:  .Ltmp0:
 ; RV64-NOPIC-NEXT:    auipc a0, %tls_ie_pcrel_hi(unspecified)
-; RV64-NOPIC-NEXT:    ld a0, %pcrel_lo(.LBB0_1)(a0)
+; RV64-NOPIC-NEXT:    ld a0, %pcrel_lo(.Ltmp0)(a0)
 ; RV64-NOPIC-NEXT:    add a0, a0, tp
 ; RV64-NOPIC-NEXT:    ret
 entry:
@@ -74,10 +70,9 @@ define i32* @f2() nounwind {
 ; RV32-PIC:       # %bb.0: # %entry
 ; RV32-PIC-NEXT:    addi sp, sp, -16
 ; RV32-PIC-NEXT:    sw ra, 12(sp)
-; RV32-PIC-NEXT:  .LBB1_1: # %entry
-; RV32-PIC-NEXT:    # Label of block must be emitted
+; RV32-PIC-NEXT:  .Ltmp1:
 ; RV32-PIC-NEXT:    auipc a0, %tls_gd_pcrel_hi(ld)
-; RV32-PIC-NEXT:    addi a0, a0, %pcrel_lo(.LBB1_1)
+; RV32-PIC-NEXT:    addi a0, a0, %pcrel_lo(.Ltmp1)
 ; RV32-PIC-NEXT:    call __tls_get_addr@plt
 ; RV32-PIC-NEXT:    lw ra, 12(sp)
 ; RV32-PIC-NEXT:    addi sp, sp, 16
@@ -87,10 +82,9 @@ define i32* @f2() nounwind {
 ; RV64-PIC:       # %bb.0: # %entry
 ; RV64-PIC-NEXT:    addi sp, sp, -16
 ; RV64-PIC-NEXT:    sd ra, 8(sp)
-; RV64-PIC-NEXT:  .LBB1_1: # %entry
-; RV64-PIC-NEXT:    # Label of block must be emitted
+; RV64-PIC-NEXT:  .Ltmp1:
 ; RV64-PIC-NEXT:    auipc a0, %tls_gd_pcrel_hi(ld)
-; RV64-PIC-NEXT:    addi a0, a0, %pcrel_lo(.LBB1_1)
+; RV64-PIC-NEXT:    addi a0, a0, %pcrel_lo(.Ltmp1)
 ; RV64-PIC-NEXT:    call __tls_get_addr@plt
 ; RV64-PIC-NEXT:    ld ra, 8(sp)
 ; RV64-PIC-NEXT:    addi sp, sp, 16
@@ -98,19 +92,17 @@ define i32* @f2() nounwind {
 ;
 ; RV32-NOPIC-LABEL: f2:
 ; RV32-NOPIC:       # %bb.0: # %entry
-; RV32-NOPIC-NEXT:  .LBB1_1: # %entry
-; RV32-NOPIC-NEXT:    # Label of block must be emitted
+; RV32-NOPIC-NEXT:  .Ltmp1:
 ; RV32-NOPIC-NEXT:    auipc a0, %tls_ie_pcrel_hi(ld)
-; RV32-NOPIC-NEXT:    lw a0, %pcrel_lo(.LBB1_1)(a0)
+; RV32-NOPIC-NEXT:    lw a0, %pcrel_lo(.Ltmp1)(a0)
 ; RV32-NOPIC-NEXT:    add a0, a0, tp
 ; RV32-NOPIC-NEXT:    ret
 ;
 ; RV64-NOPIC-LABEL: f2:
 ; RV64-NOPIC:       # %bb.0: # %entry
-; RV64-NOPIC-NEXT:  .LBB1_1: # %entry
-; RV64-NOPIC-NEXT:    # Label of block must be emitted
+; RV64-NOPIC-NEXT:  .Ltmp1:
 ; RV64-NOPIC-NEXT:    auipc a0, %tls_ie_pcrel_hi(ld)
-; RV64-NOPIC-NEXT:    ld a0, %pcrel_lo(.LBB1_1)(a0)
+; RV64-NOPIC-NEXT:    ld a0, %pcrel_lo(.Ltmp1)(a0)
 ; RV64-NOPIC-NEXT:    add a0, a0, tp
 ; RV64-NOPIC-NEXT:    ret
 entry:
@@ -123,37 +115,33 @@ entry:
 define i32* @f3() nounwind {
 ; RV32-PIC-LABEL: f3:
 ; RV32-PIC:       # %bb.0: # %entry
-; RV32-PIC-NEXT:  .LBB2_1: # %entry
-; RV32-PIC-NEXT:    # Label of block must be emitted
+; RV32-PIC-NEXT:  .Ltmp2:
 ; RV32-PIC-NEXT:    auipc a0, %tls_ie_pcrel_hi(ie)
-; RV32-PIC-NEXT:    lw a0, %pcrel_lo(.LBB2_1)(a0)
+; RV32-PIC-NEXT:    lw a0, %pcrel_lo(.Ltmp2)(a0)
 ; RV32-PIC-NEXT:    add a0, a0, tp
 ; RV32-PIC-NEXT:    ret
 ;
 ; RV64-PIC-LABEL: f3:
 ; RV64-PIC:       # %bb.0: # %entry
-; RV64-PIC-NEXT:  .LBB2_1: # %entry
-; RV64-PIC-NEXT:    # Label of block must be emitted
+; RV64-PIC-NEXT:  .Ltmp2:
 ; RV64-PIC-NEXT:    auipc a0, %tls_ie_pcrel_hi(ie)
-; RV64-PIC-NEXT:    ld a0, %pcrel_lo(.LBB2_1)(a0)
+; RV64-PIC-NEXT:    ld a0, %pcrel_lo(.Ltmp2)(a0)
 ; RV64-PIC-NEXT:    add a0, a0, tp
 ; RV64-PIC-NEXT:    ret
 ;
 ; RV32-NOPIC-LABEL: f3:
 ; RV32-NOPIC:       # %bb.0: # %entry
-; RV32-NOPIC-NEXT:  .LBB2_1: # %entry
-; RV32-NOPIC-NEXT:    # Label of block must be emitted
+; RV32-NOPIC-NEXT:  .Ltmp2:
 ; RV32-NOPIC-NEXT:    auipc a0, %tls_ie_pcrel_hi(ie)
-; RV32-NOPIC-NEXT:    lw a0, %pcrel_lo(.LBB2_1)(a0)
+; RV32-NOPIC-NEXT:    lw a0, %pcrel_lo(.Ltmp2)(a0)
 ; RV32-NOPIC-NEXT:    add a0, a0, tp
 ; RV32-NOPIC-NEXT:    ret
 ;
 ; RV64-NOPIC-LABEL: f3:
 ; RV64-NOPIC:       # %bb.0: # %entry
-; RV64-NOPIC-NEXT:  .LBB2_1: # %entry
-; RV64-NOPIC-NEXT:    # Label of block must be emitted
+; RV64-NOPIC-NEXT:  .Ltmp2:
 ; RV64-NOPIC-NEXT:    auipc a0, %tls_ie_pcrel_hi(ie)
-; RV64-NOPIC-NEXT:    ld a0, %pcrel_lo(.LBB2_1)(a0)
+; RV64-NOPIC-NEXT:    ld a0, %pcrel_lo(.Ltmp2)(a0)
 ; RV64-NOPIC-NEXT:    add a0, a0, tp
 ; RV64-NOPIC-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/wide-mem.ll b/llvm/test/CodeGen/RISCV/wide-mem.ll
index 02aae215fcec1..40a074bd87683 100644
--- a/llvm/test/CodeGen/RISCV/wide-mem.ll
+++ b/llvm/test/CodeGen/RISCV/wide-mem.ll
@@ -22,8 +22,7 @@ define i64 @load_i64_global() nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lui a1, %hi(val64)
 ; RV32I-NEXT:    lw a0, %lo(val64)(a1)
-; RV32I-NEXT:    addi a1, a1, %lo(val64)
-; RV32I-NEXT:    lw a1, 4(a1)
+; RV32I-NEXT:    lw a1, %lo(val64+4)(a1)
 ; RV32I-NEXT:    ret
   %1 = load i64, i64* @val64
   ret i64 %1
diff --git a/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll b/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll
index 7c4bbcdaa1e8f..7af31e3bdfba6 100644
--- a/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll
+++ b/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll
@@ -11,17 +11,16 @@ define i32 @test_zext_i8() nounwind {
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    lui a0, %hi(bytes)
 ; RV32I-NEXT:    lbu a1, %lo(bytes)(a0)
-; RV32I-NEXT:    addi a2, zero, 136
-; RV32I-NEXT:    bne a1, a2, .LBB0_3
-; RV32I-NEXT:  # %bb.1: # %entry
 ; RV32I-NEXT:    addi a0, a0, %lo(bytes)
 ; RV32I-NEXT:    lbu a0, 1(a0)
-; RV32I-NEXT:    addi a1, zero, 7
-; RV32I-NEXT:    bne a0, a1, .LBB0_3
-; RV32I-NEXT:  # %bb.2: # %if.end
+; RV32I-NEXT:    xori a1, a1, 136
+; RV32I-NEXT:    xori a0, a0, 7
+; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    bnez a0, .LBB0_2
+; RV32I-NEXT:  # %bb.1: # %if.end
 ; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB0_3: # %if.then
+; RV32I-NEXT:  .LBB0_2: # %if.then
 ; RV32I-NEXT:    addi a0, zero, 1
 ; RV32I-NEXT:    ret
 entry:
@@ -46,18 +45,18 @@ define i32 @test_zext_i16() nounwind {
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    lui a0, %hi(shorts)
 ; RV32I-NEXT:    lhu a1, %lo(shorts)(a0)
-; RV32I-NEXT:    lui a2, 16
-; RV32I-NEXT:    addi a2, a2, -120
-; RV32I-NEXT:    bne a1, a2, .LBB1_3
-; RV32I-NEXT:  # %bb.1: # %entry
 ; RV32I-NEXT:    addi a0, a0, %lo(shorts)
 ; RV32I-NEXT:    lhu a0, 2(a0)
-; RV32I-NEXT:    addi a1, zero, 7
-; RV32I-NEXT:    bne a0, a1, .LBB1_3
-; RV32I-NEXT:  # %bb.2: # %if.end
+; RV32I-NEXT:    lui a2, 16
+; RV32I-NEXT:    addi a2, a2, -120
+; RV32I-NEXT:    xor a1, a1, a2
+; RV32I-NEXT:    xori a0, a0, 7
+; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    bnez a0, .LBB1_2
+; RV32I-NEXT:  # %bb.1: # %if.end
 ; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB1_3: # %if.then
+; RV32I-NEXT:  .LBB1_2: # %if.then
 ; RV32I-NEXT:    addi a0, zero, 1
 ; RV32I-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/SystemZ/int-uadd-12.ll b/llvm/test/CodeGen/SystemZ/int-uadd-12.ll
index 4a5b858f8721a..63b37906caf57 100644
--- a/llvm/test/CodeGen/SystemZ/int-uadd-12.ll
+++ b/llvm/test/CodeGen/SystemZ/int-uadd-12.ll
@@ -2,7 +2,7 @@
 ; store into an alsi. This folding is suppose to not happen as it would
 ; introduce a loop in the DAG.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 -disable-basicaa -consthoist-gep | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 -disable-basic-aa -consthoist-gep | FileCheck %s
 
 @g_295 = external dso_local unnamed_addr global i32, align 4
 @g_672 = external dso_local unnamed_addr global i64, align 8
diff --git a/llvm/test/CodeGen/SystemZ/knownbits.ll b/llvm/test/CodeGen/SystemZ/knownbits.ll
index 08694d8e69973..9c40b307cf16d 100644
--- a/llvm/test/CodeGen/SystemZ/knownbits.ll
+++ b/llvm/test/CodeGen/SystemZ/knownbits.ll
@@ -4,18 +4,22 @@
 ; Test that DAGCombiner gets helped by computeKnownBitsForTargetNode().
 
 ; SystemZISD::REPLICATE
-define i32 @f0(<4 x i32> %a0) {
+define i32 @f0(<4 x i32> *%p0) {
 ; CHECK-LABEL: f0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vgbm %v0, 0
-; CHECK-NEXT:    vceqf %v0, %v24, %v0
+; CHECK-NEXT:    vl %v0, 0(%r2), 3
+; CHECK-NEXT:    vgbm %v1, 0
+; CHECK-NEXT:    vceqf %v0, %v0, %v1
 ; CHECK-NEXT:    vrepif %v1, 1
 ; CHECK-NEXT:    vnc %v0, %v1, %v0
+; CHECK-NEXT:    vst %v0, 0(%r2), 3
 ; CHECK-NEXT:    vlgvf %r2, %v0, 3
 ; CHECK-NEXT:    # kill: def $r2l killed $r2l killed $r2d
 ; CHECK-NEXT:    br %r14
+  %a0 = load <4 x i32>, <4 x i32>* %p0, align 8
   %cmp0 = icmp ne <4 x i32> %a0, zeroinitializer
   %zxt0 = zext <4 x i1> %cmp0 to <4 x i32>
+  store <4 x i32> %zxt0, <4 x i32>* %p0, align 8
   %ext0 = extractelement <4 x i32> %zxt0, i32 3
   br label %exit
 
@@ -31,36 +35,40 @@ exit:
 ; The DAG XOR has JOIN_DWORDS and REPLICATE operands. With KnownBits properly set
 ; for both these nodes, ICMP is used instead of TM during lowering because
 ; adjustForRedundantAnd() succeeds.
-define void @f1() {
+define void @f1(i64 %a0, i64 %a1) {
 ; CHECK-LABEL: f1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    clhhsi 0, 0
-; CHECK-NEXT:    lhi %r0, 0
-; CHECK-NEXT:    lochie %r0, 1
-; CHECK-NEXT:    lghi %r1, 1
+; CHECK-NEXT:    risbgn %r0, %r2, 63, 191, 0
+; CHECK-NEXT:    risbgn %r1, %r3, 63, 191, 0
 ; CHECK-NEXT:    vlvgp %v0, %r0, %r1
 ; CHECK-NEXT:    vrepig %v1, 1
 ; CHECK-NEXT:    vx %v0, %v0, %v1
-; CHECK-NEXT:    vlgvf %r0, %v0, 1
-; CHECK-NEXT:    cijlh %r0, 0, .LBB1_3
+; CHECK-NEXT:    vlgvg %r0, %v0, 0
+; CHECK-NEXT:    cgijlh %r0, 0, .LBB1_3
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    vlgvf %r0, %v0, 3
-; CHECK-NEXT:    cijlh %r0, 0, .LBB1_3
+; CHECK-NEXT:    vlgvg %r0, %v0, 1
+; CHECK-NEXT:    cgijlh %r0, 0, .LBB1_3
 ; CHECK-NEXT:  # %bb.2:
 ; CHECK-NEXT:  .LBB1_3:
-  %1 = load i16, i16* null, align 2
-  %2 = icmp eq i16 %1, 0
-  %3 = insertelement <2 x i1> undef, i1 %2, i32 0
-  %4 = insertelement <2 x i1> %3, i1 true, i32 1
-  %5 = xor <2 x i1> %4, <i1 true, i1 true>
-  %6 = extractelement <2 x i1> %5, i32 0
-  %7 = extractelement <2 x i1> %5, i32 1
-  %8 = or i1 %6, %7
-  br i1 %8, label %10, label %9
+  %1 = and i64 %a0, 1
+  %2 = and i64 %a1, 1
+  %3 = insertelement <2 x i64> undef, i64 %1, i32 0
+  %4 = insertelement <2 x i64> %3, i64 %2, i32 1
+  %5 = xor <2 x i64> %4, <i64 1, i64 1>
+  %6 = extractelement <2 x i64> %5, i32 0
+  %7 = and i64 %6, 1
+  %8 = icmp eq i64 %7, 0
+  br i1 %8, label %9, label %14
 
-; <label>:8:                                      ; preds = %0
+; <label>:9:                                      ; preds = %0
+  %10 = extractelement <2 x i64> %5, i32 1
+  %11 = and i64 %10, 1
+  %12 = icmp eq i64 %11, 0
+  br i1 %12, label %13, label %14
+
+; <label>:13:                                      ; preds = %0
   unreachable
 
-; <label>:9:                                      ; preds = %0
+; <label>:14:                                      ; preds = %0
   unreachable
 }
diff --git a/llvm/test/CodeGen/SystemZ/nomerge.ll b/llvm/test/CodeGen/SystemZ/nomerge.ll
new file mode 100644
index 0000000000000..6ccbf9a1bfc1e
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/nomerge.ll
@@ -0,0 +1,36 @@
+; RUN: llc < %s -mtriple=s390x-linux-gnu -o - | FileCheck %s
+
+define void @foo(i32 %i) {
+entry:
+  switch i32 %i, label %if.end3 [
+    i32 5, label %if.then
+    i32 7, label %if.then2
+  ]
+
+if.then:
+  tail call void @bar() #0
+  br label %if.end3
+
+if.then2:
+  tail call void @bar() #0
+  br label %if.end3
+
+if.end3:
+  tail call void @bar() #0
+  ret void
+}
+
+declare void @bar()
+
+attributes #0 = { nomerge }
+
+; CHECK-LABEL: foo:
+; CHECK: # %bb.0: # %entry
+; CHECK: # %bb.1: # %entry
+; CHECK: # %bb.2: # %if.then
+; CHECK-NEXT: brasl %r14, bar@PLT
+; CHECK: jg bar@PLT
+; CHECK: .LBB0_3: # %if.then2
+; CHECK: brasl %r14, bar@PLT
+; CHECK: .LBB0_4: # %if.end3
+; CHECK: jg bar@PLT
diff --git a/llvm/test/CodeGen/SystemZ/pr36164.ll b/llvm/test/CodeGen/SystemZ/pr36164.ll
index 2ed6fa901db08..1bde1254a1e08 100644
--- a/llvm/test/CodeGen/SystemZ/pr36164.ll
+++ b/llvm/test/CodeGen/SystemZ/pr36164.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc %s -o - -mtriple=s390x-linux-gnu -mcpu=z13 -disable-basicaa | FileCheck %s
+; RUN: llc %s -o - -mtriple=s390x-linux-gnu -mcpu=z13 -disable-basic-aa | FileCheck %s
 
 ; This test checks that we do not a reference to a deleted node.
 
diff --git a/llvm/test/CodeGen/SystemZ/vec-move-16.ll b/llvm/test/CodeGen/SystemZ/vec-move-16.ll
index cd25773968001..cc8237bcfa298 100644
--- a/llvm/test/CodeGen/SystemZ/vec-move-16.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-move-16.ll
@@ -40,9 +40,10 @@ define <4 x i32> @f4(<4 x i1> *%ptr) {
 ; Test a v4i8->v4i32 extension.
 define <4 x i32> @f5(<4 x i8> *%ptr) {
 ; CHECK-LABEL: f5:
+; CHECK: larl	%r1, .LCPI4_0
 ; CHECK: vlrepf [[REG1:%v[0-9]+]], 0(%r2)
-; CHECK: vuplhb [[REG2:%v[0-9]+]], [[REG1]]
-; CHECK: vuplhh %v24, [[REG2]]
+; CHECK: vl	%v1, 0(%r1), 3
+; CHECK: vperm	%v24, %v1, [[REG1]], %v1
 ; CHECK: br %r14
   %val = load <4 x i8>, <4 x i8> *%ptr
   %ret = zext <4 x i8> %val to <4 x i32>
@@ -71,10 +72,10 @@ define <2 x i64> @f7(<2 x i1> *%ptr) {
 ; Test a v2i8->v2i64 extension.
 define <2 x i64> @f8(<2 x i8> *%ptr) {
 ; CHECK-LABEL: f8:
-; CHECK: vlreph [[REG1:%v[0-9]+]], 0(%r2)
-; CHECK: vuplhb [[REG2:%v[0-9]+]], [[REG1]]
-; CHECK: vuplhh [[REG3:%v[0-9]+]], [[REG2]]
-; CHECK: vuplhf %v24, [[REG3]]
+; CHECK: larl	%r1, .LCPI7_0
+; CHECK: vlreph	[[REG1:%v[0-9]+]], 0(%r2)
+; CHECK: vl	%v1, 0(%r1), 3
+; CHECK: vperm	%v24, %v1, [[REG1]], %v1
 ; CHECK: br %r14
   %val = load <2 x i8>, <2 x i8> *%ptr
   %ret = zext <2 x i8> %val to <2 x i64>
@@ -84,9 +85,10 @@ define <2 x i64> @f8(<2 x i8> *%ptr) {
 ; Test a v2i16->v2i64 extension.
 define <2 x i64> @f9(<2 x i16> *%ptr) {
 ; CHECK-LABEL: f9:
-; CHECK: vlrepf [[REG1:%v[0-9]+]], 0(%r2)
-; CHECK: vuplhh [[REG2:%v[0-9]+]], [[REG1]]
-; CHECK: vuplhf %v24, [[REG2]]
+; CHECK: larl	%r1, .LCPI8_0
+; CHECK: vlrepf	[[REG1:%v[0-9]+]], 0(%r2)
+; CHECK: vl	%v1, 0(%r1), 3
+; CHECK: vperm	%v24, %v1, [[REG1]], %v1
 ; CHECK: br %r14
   %val = load <2 x i16>, <2 x i16> *%ptr
   %ret = zext <2 x i16> %val to <2 x i64>
diff --git a/llvm/test/CodeGen/SystemZ/vec-move-23.ll b/llvm/test/CodeGen/SystemZ/vec-move-23.ll
index 430e879bcc069..1e31c0b606ee0 100644
--- a/llvm/test/CodeGen/SystemZ/vec-move-23.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-move-23.ll
@@ -68,9 +68,9 @@ define void @fun3(<4 x i16> %Src, <4 x float>* %Dst) {
 
 define void @fun4(<2 x i8> %Src, <2 x double>* %Dst) {
 ; CHECK-LABEL: fun4:
-; CHECK:      vuplhb	%v0, %v24
-; CHECK-NEXT: vuplhh	%v0, %v0
-; CHECK-NEXT: vuplhf	%v0, %v0
+; CHECK:      larl	%r1, .LCPI4_0
+; CHECK-NEXT: vl	%v0, 0(%r1), 3
+; CHECK-NEXT: vperm	%v0, %v0, %v24, %v0
 ; CHECK-NEXT: vcdlgb	%v0, %v0, 0, 0
 ; CHECK-NEXT: vst	%v0, 0(%r2), 3
 ; CHECK-NEXT: br	%r14
@@ -81,8 +81,9 @@ define void @fun4(<2 x i8> %Src, <2 x double>* %Dst) {
 
 define void @fun5(<2 x i16> %Src, <2 x double>* %Dst) {
 ; CHECK-LABEL: fun5:
-; CHECK:      vuplhh	%v0, %v24
-; CHECK-NEXT: vuplhf	%v0, %v0
+; CHECK:      larl	%r1, .LCPI5_0
+; CHECK-NEXT: vl	%v0, 0(%r1), 3
+; CHECK-NEXT: vperm	%v0, %v0, %v24, %v0
 ; CHECK-NEXT: vcdlgb	%v0, %v0, 0, 0
 ; CHECK-NEXT: vst	%v0, 0(%r2), 3
 ; CHECK-NEXT: br	%r14
diff --git a/llvm/test/CodeGen/SystemZ/vec-move-24.ll b/llvm/test/CodeGen/SystemZ/vec-move-24.ll
new file mode 100644
index 0000000000000..4f95bb1315a92
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/vec-move-24.ll
@@ -0,0 +1,49 @@
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
+;
+; Test that vperm is not used if a single unpack is enough.
+
+define <4 x i32> @fun0(<4 x i32>* %Src) nounwind {
+; CHECK-LABEL: fun0:
+; CHECK-NOT: vperm
+  %tmp = load <4 x i32>, <4 x i32>* %Src
+  %tmp2 = shufflevector <4 x i32> zeroinitializer, <4 x i32> %tmp, <4 x i32> <i32 0, i32 4, i32 2, i32 5>
+  ret <4 x i32> %tmp2
+}
+
+define  void @fun1(i8 %Src, <32 x i8>* %Dst) nounwind {
+; CHECK-LABEL: fun1:
+; CHECK-NOT: vperm
+  %I0 = insertelement <16 x i8> undef, i8 %Src, i32 0
+  %I1 = insertelement <16 x i8> %I0, i8 %Src, i32 1
+  %I2 = insertelement <16 x i8> %I1, i8 %Src, i32 2
+  %I3 = insertelement <16 x i8> %I2, i8 %Src, i32 3
+  %I4 = insertelement <16 x i8> %I3, i8 %Src, i32 4
+  %I5 = insertelement <16 x i8> %I4, i8 %Src, i32 5
+  %I6 = insertelement <16 x i8> %I5, i8 %Src, i32 6
+  %I7 = insertelement <16 x i8> %I6, i8 %Src, i32 7
+  %I8 = insertelement <16 x i8> %I7, i8 %Src, i32 8
+  %I9 = insertelement <16 x i8> %I8, i8 %Src, i32 9
+  %I10 = insertelement <16 x i8> %I9, i8 %Src, i32 10
+  %I11 = insertelement <16 x i8> %I10, i8 %Src, i32 11
+  %I12 = insertelement <16 x i8> %I11, i8 %Src, i32 12
+  %I13 = insertelement <16 x i8> %I12, i8 %Src, i32 13
+  %I14 = insertelement <16 x i8> %I13, i8 %Src, i32 14
+  %I15 = insertelement <16 x i8> %I14, i8 %Src, i32 15
+
+  %tmp = shufflevector <16 x i8> zeroinitializer,
+                       <16 x i8> %I15,
+                       <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
+                                   i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
+                                   i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
+                                   i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %tmp9 = shufflevector <32 x i8> undef,
+                        <32 x i8> %tmp,
+                        <32 x i32> <i32 33, i32 32, i32 48, i32 49, i32 1, i32 17, i32 50, i32 51,
+                                    i32 2, i32 18, i32 52, i32 53, i32 3, i32 19, i32 54, i32 55,
+                                    i32 4, i32 20, i32 56, i32 57, i32 5, i32 21, i32 58, i32 59,
+                                    i32 6, i32 22, i32 60, i32 61, i32 7, i32 62, i32 55, i32 63>
+
+  store <32 x i8> %tmp9, <32 x i8>* %Dst
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/SystemZ/vec-zext.ll b/llvm/test/CodeGen/SystemZ/vec-zext.ll
index 831594d4020c4..b4c8f2307b0b7 100644
--- a/llvm/test/CodeGen/SystemZ/vec-zext.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-zext.ll
@@ -1,5 +1,5 @@
-; Test that vector zexts are done efficently with unpack instructions also in
-; case of fewer elements than allowed, e.g. <2 x i32>.
+; Test that vector zexts are done efficently also in case of fewer elements
+; than allowed, e.g. <2 x i32>.
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 
@@ -14,8 +14,9 @@ define <2 x i16> @fun1(<2 x i8> %val1) {
 
 define <2 x i32> @fun2(<2 x i8> %val1) {
 ; CHECK-LABEL: fun2:
-; CHECK:      	vuplhb	%v0, %v24
-; CHECK-NEXT: 	vuplhh	%v24, %v0
+; CHECK:        larl	%r1, .LCPI1_0
+; CHECK-NEXT:   vl	%v0, 0(%r1), 3
+; CHECK-NEXT:   vperm	%v24, %v0, %v24, %v0
 ; CHECK-NEXT: 	br	%r14
   %z = zext <2 x i8> %val1 to <2 x i32>
   ret <2 x i32> %z
@@ -23,9 +24,9 @@ define <2 x i32> @fun2(<2 x i8> %val1) {
 
 define <2 x i64> @fun3(<2 x i8> %val1) {
 ; CHECK-LABEL: fun3:
-; CHECK:      	vuplhb	%v0, %v24
-; CHECK-NEXT: 	vuplhh	%v0, %v0
-; CHECK-NEXT: 	vuplhf	%v24, %v0
+; CHECK: 	larl	%r1, .LCPI2_0
+; CHECK-NEXT: 	vl	%v0, 0(%r1), 3
+; CHECK-NEXT: 	vperm	%v24, %v0, %v24, %v0
 ; CHECK-NEXT: 	br	%r14
   %z = zext <2 x i8> %val1 to <2 x i64>
   ret <2 x i64> %z
@@ -41,8 +42,9 @@ define <2 x i32> @fun4(<2 x i16> %val1) {
 
 define <2 x i64> @fun5(<2 x i16> %val1) {
 ; CHECK-LABEL: fun5:
-; CHECK:      	vuplhh	%v0, %v24
-; CHECK-NEXT: 	vuplhf	%v24, %v0
+; CHECK: 	larl	%r1, .LCPI4_0
+; CHECK-NEXT: 	vl	%v0, 0(%r1), 3
+; CHECK-NEXT: 	vperm	%v24, %v0, %v24, %v0
 ; CHECK-NEXT: 	br	%r14
   %z = zext <2 x i16> %val1 to <2 x i64>
   ret <2 x i64> %z
@@ -66,8 +68,9 @@ define <4 x i16> @fun7(<4 x i8> %val1) {
 
 define <4 x i32> @fun8(<4 x i8> %val1) {
 ; CHECK-LABEL: fun8:
-; CHECK:      	vuplhb	%v0, %v24
-; CHECK-NEXT: 	vuplhh	%v24, %v0
+; CHECK: 	larl	%r1, .LCPI7_0
+; CHECK-NEXT: 	vl	%v0, 0(%r1), 3
+; CHECK-NEXT: 	vperm	%v24, %v0, %v24, %v0
 ; CHECK-NEXT: 	br	%r14
   %z = zext <4 x i8> %val1 to <4 x i32>
   ret <4 x i32> %z
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll
index 54ddf64683366..a00af0d6a9ec4 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll
@@ -49,7 +49,7 @@ vector.body:                                      ; preds = %vector.body, %vecto
   %tmp7 = bitcast i8* %tmp6 to <16 x i8>*
   tail call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %mul, <16 x i8>* %tmp7, i32 4, <16 x i1> %active.lane.mask)
   %index.next = add i32 %index, 16
-  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32(i32 %tmp14, i32 1)
   %tmp16 = icmp ne i32 %tmp15, 0
   br i1 %tmp16, label %vector.body, label %for.cond.cleanup
 
@@ -106,7 +106,7 @@ vector.body:                                      ; preds = %vector.body, %vecto
   %tmp7 = bitcast i16* %tmp6 to <8 x i16>*
   tail call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %mul, <8 x i16>* %tmp7, i32 4, <8 x i1> %active.lane.mask)
   %index.next = add i32 %index, 8
-  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32(i32 %tmp14, i32 1)
   %tmp16 = icmp ne i32 %tmp15, 0
   br i1 %tmp16, label %vector.body, label %for.cond.cleanup
 
@@ -160,7 +160,7 @@ vector.body:                                      ; preds = %vector.body, %vecto
   %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
   tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %mul, <4 x i32>* %tmp7, i32 4, <4 x i1> %active.lane.mask)
   %index.next = add i32 %index, 4
-  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32(i32 %tmp14, i32 1)
   %tmp16 = icmp ne i32 %tmp15, 0
   br i1 %tmp16, label %vector.body, label %for.cond.cleanup
 
@@ -221,7 +221,7 @@ vector.body:                                      ; preds = %vector.body, %vecto
   %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
   tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %combine, <4 x i32>* %tmp7, i32 4, <4 x i1> %active.lane.mask)
   %index.next = add i32 %index, 4
-  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32(i32 %tmp14, i32 1)
   %tmp16 = icmp ne i32 %tmp15, 0
   br i1 %tmp16, label %vector.body, label %for.cond.cleanup
 
@@ -277,7 +277,7 @@ vector.body:                                      ; preds = %vector.body, %vecto
   %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
   tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %active.lane.mask)
   %index.next = add i32 %index, 4
-  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32(i32 %tmp14, i32 1)
   %tmp16 = icmp ne i32 %tmp15, 0
   br i1 %tmp16, label %vector.body, label %for.cond.cleanup
 
@@ -336,7 +336,7 @@ vector.body:                                      ; preds = %vector.body, %vecto
   %tmp7 = bitcast i32* %tmp6 to <4 x i32>*
   tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %wrong)
   %index.next = add i32 %index, 4
-  %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1)
+  %tmp15 = call i32 @llvm.loop.decrement.reg.i32(i32 %tmp14, i32 1)
   %tmp16 = icmp ne i32 %tmp15, 0
   br i1 %tmp16, label %vector.body, label %for.cond.cleanup
 
@@ -344,6 +344,92 @@ for.cond.cleanup:                                 ; preds = %vector.body, %entry
   ret void
 }
 
+; TODO: Multiple intrinsics not yet supported.
+; This is currently rejected, because if the vector body is unrolled, the step
+; is not what we expect:
+;
+;   Step value 16 doesn't match vector width 4
+;
+; CHECK-LABEL: interleave4
+; CHECK: vector.body:
+; CHECK:  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
+; CHECK:  %active.lane.mask{{.*}} = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %v7, i32 %trip.count.minus.1)
+; CHECK:  %active.lane.mask{{.*}} = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %v8, i32 %trip.count.minus.1)
+; CHECK:  %active.lane.mask{{.*}} = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %v9, i32 %trip.count.minus.1)
+;
+define dso_local void @interleave4(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32 %N) local_unnamed_addr #0 {
+entry:
+  %cmp8 = icmp sgt i32 %N, 0
+  %v0 = add i32 %N, 15
+  %v1 = lshr i32 %v0, 4
+  %v2 = shl nuw i32 %v1, 4
+  %v3 = add i32 %v2, -16
+  %v4 = lshr i32 %v3, 4
+  %v5 = add nuw nsw i32 %v4, 1
+  br i1 %cmp8, label %vector.ph, label %for.cond.cleanup
+
+
+vector.ph:
+  %trip.count.minus.1 = add i32 %N, -1
+  %scevgep = getelementptr i32, i32* %A, i32 8
+  %scevgep30 = getelementptr i32, i32* %C, i32 8
+  %scevgep37 = getelementptr i32, i32* %B, i32 8
+  call void @llvm.set.loop.iterations.i32(i32 %v5)
+  br label %vector.body
+
+vector.body:
+  %lsr.iv38 = phi i32* [ %scevgep39, %vector.body ], [ %scevgep37, %vector.ph ]
+  %lsr.iv31 = phi i32* [ %scevgep32, %vector.body ], [ %scevgep30, %vector.ph ]
+  %lsr.iv = phi i32* [ %scevgep25, %vector.body ], [ %scevgep, %vector.ph ]
+  %index = phi i32 [ 0, %vector.ph ], [ %v14, %vector.body ]
+  %v6 = phi i32 [ %v5, %vector.ph ], [ %v15, %vector.body ]
+  %lsr.iv3840 = bitcast i32* %lsr.iv38 to <4 x i32>*
+  %lsr.iv3133 = bitcast i32* %lsr.iv31 to <4 x i32>*
+  %lsr.iv26 = bitcast i32* %lsr.iv to <4 x i32>*
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %v7 = add i32 %index, 4
+  %active.lane.mask15 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %v7, i32 %trip.count.minus.1)
+  %v8 = add i32 %v7, 4
+  %active.lane.mask16 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %v8, i32 %trip.count.minus.1)
+  %v9 = add i32 %v8, 4
+  %active.lane.mask17 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %v9, i32 %trip.count.minus.1)
+  %scevgep42 = getelementptr <4 x i32>, <4 x i32>* %lsr.iv3840, i32 -2
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %scevgep42, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
+  %scevgep43 = getelementptr <4 x i32>, <4 x i32>* %lsr.iv3840, i32 -1
+  %wide.masked.load18 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* nonnull %scevgep43, i32 4, <4 x i1> %active.lane.mask15, <4 x i32> undef)
+  %wide.masked.load19 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* nonnull %lsr.iv3840, i32 4, <4 x i1> %active.lane.mask16, <4 x i32> undef)
+  %scevgep41 = getelementptr <4 x i32>, <4 x i32>* %lsr.iv3840, i32 1
+  %wide.masked.load20 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* nonnull %scevgep41, i32 4, <4 x i1> %active.lane.mask17, <4 x i32> undef)
+  %scevgep34 = getelementptr <4 x i32>, <4 x i32>* %lsr.iv3133, i32 -2
+  %wide.masked.load21 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %scevgep34, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
+  %scevgep35 = getelementptr <4 x i32>, <4 x i32>* %lsr.iv3133, i32 -1
+  %wide.masked.load22 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* nonnull %scevgep35, i32 4, <4 x i1> %active.lane.mask15, <4 x i32> undef)
+  %wide.masked.load23 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* nonnull %lsr.iv3133, i32 4, <4 x i1> %active.lane.mask16, <4 x i32> undef)
+  %scevgep36 = getelementptr <4 x i32>, <4 x i32>* %lsr.iv3133, i32 1
+  %wide.masked.load24 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* nonnull %scevgep36, i32 4, <4 x i1> %active.lane.mask17, <4 x i32> undef)
+  %v10 = add nsw <4 x i32> %wide.masked.load21, %wide.masked.load
+  %v11 = add nsw <4 x i32> %wide.masked.load22, %wide.masked.load18
+  %v12 = add nsw <4 x i32> %wide.masked.load23, %wide.masked.load19
+  %v13 = add nsw <4 x i32> %wide.masked.load24, %wide.masked.load20
+  %scevgep27 = getelementptr <4 x i32>, <4 x i32>* %lsr.iv26, i32 -2
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v10, <4 x i32>* %scevgep27, i32 4, <4 x i1> %active.lane.mask)
+  %scevgep28 = getelementptr <4 x i32>, <4 x i32>* %lsr.iv26, i32 -1
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v11, <4 x i32>* %scevgep28, i32 4, <4 x i1> %active.lane.mask15)
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v12, <4 x i32>* %lsr.iv26, i32 4, <4 x i1> %active.lane.mask16)
+  %scevgep29 = getelementptr <4 x i32>, <4 x i32>* %lsr.iv26, i32 1
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v13, <4 x i32>* %scevgep29, i32 4, <4 x i1> %active.lane.mask17)
+  %scevgep25 = getelementptr i32, i32* %lsr.iv, i32 16
+  %scevgep32 = getelementptr i32, i32* %lsr.iv31, i32 16
+  %scevgep39 = getelementptr i32, i32* %lsr.iv38, i32 16
+  %v14 = add i32 %v9, 4
+  %v15 = call i32 @llvm.loop.decrement.reg.i32(i32 %v6, i32 1)
+  %v16 = icmp ne i32 %v15, 0
+  br i1 %v16, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+}
+
 declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32 immarg, <16 x i1>, <16 x i8>)
 declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32 immarg, <16 x i1>)
 declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>)
@@ -353,7 +439,7 @@ declare void @llvm.masked.store.v2i64.p0v2i64(<2 x i64>, <2 x i64>*, i32 immarg,
 declare <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>*, i32 immarg, <2 x i1>, <2 x i64>)
 declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>)
 declare void @llvm.set.loop.iterations.i32(i32)
-declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32)
+declare i32 @llvm.loop.decrement.reg.i32(i32, i32)
 declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)
 declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32)
 declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32, i32)
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
index 0380d5f20bc79..8c18159c24c56 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
@@ -344,18 +344,16 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(half* nocapture readonly %b, h
 ; CHECK-NEXT:    vmov.16 q6[3], r2
 ; CHECK-NEXT:  .LBB2_3: @ %else26
 ; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
-; CHECK-NEXT:    vmul.f16 q5, q6, q5
+; CHECK-NEXT:    vmul.f16 q0, q6, q5
 ; CHECK-NEXT:    adds r0, #8
-; CHECK-NEXT:    vmovx.f16 s2, s21
-; CHECK-NEXT:    vmovx.f16 s0, s20
-; CHECK-NEXT:    vcvtb.f32.f16 s27, s2
+; CHECK-NEXT:    vcvtt.f32.f16 s23, s1
 ; CHECK-NEXT:    adds r1, #8
-; CHECK-NEXT:    vcvtb.f32.f16 s26, s21
+; CHECK-NEXT:    vcvtb.f32.f16 s22, s1
 ; CHECK-NEXT:    adds r3, #4
-; CHECK-NEXT:    vcvtb.f32.f16 s25, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s21, s0
 ; CHECK-NEXT:    subs.w lr, lr, #1
-; CHECK-NEXT:    vcvtb.f32.f16 s24, s20
-; CHECK-NEXT:    vadd.f32 q5, q3, q6
+; CHECK-NEXT:    vcvtb.f32.f16 s20, s0
+; CHECK-NEXT:    vadd.f32 q5, q3, q5
 ; CHECK-NEXT:    bne .LBB2_4
 ; CHECK-NEXT:    b .LBB2_21
 ; CHECK-NEXT:  .LBB2_4: @ %vector.body
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
index b01c98cedeb7d..e69610f9df2a9 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
@@ -710,10 +710,10 @@ define arm_aapcs_vfpcc void @float_int_mul(float* nocapture readonly %a, i32* no
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r4, [r6], #4
 ; CHECK-NEXT:    add.w r12, r12, #1
-; CHECK-NEXT:    vmov s0, r4
-; CHECK-NEXT:    vcvt.f32.s32 s0, s0
 ; CHECK-NEXT:    vldr s2, [r5]
 ; CHECK-NEXT:    adds r5, #4
+; CHECK-NEXT:    vmov s0, r4
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
 ; CHECK-NEXT:    vmul.f32 s0, s2, s0
 ; CHECK-NEXT:    vstr s0, [r7]
 ; CHECK-NEXT:    adds r7, #4
@@ -739,19 +739,19 @@ define arm_aapcs_vfpcc void @float_int_mul(float* nocapture readonly %a, i32* no
 ; CHECK-NEXT:    vmul.f32 s0, s2, s0
 ; CHECK-NEXT:    vstr s0, [r6]
 ; CHECK-NEXT:    vldr s0, [r1, #-4]
-; CHECK-NEXT:    vcvt.f32.s32 s0, s0
 ; CHECK-NEXT:    vldr s2, [r7, #4]
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
 ; CHECK-NEXT:    vmul.f32 s0, s2, s0
 ; CHECK-NEXT:    vstr s0, [r6, #4]
 ; CHECK-NEXT:    vldr s0, [r1]
-; CHECK-NEXT:    vcvt.f32.s32 s0, s0
 ; CHECK-NEXT:    vldr s2, [r7, #8]
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
 ; CHECK-NEXT:    vmul.f32 s0, s2, s0
 ; CHECK-NEXT:    vstr s0, [r6, #8]
 ; CHECK-NEXT:    vldr s0, [r1, #4]
 ; CHECK-NEXT:    add.w r1, r1, #16
-; CHECK-NEXT:    vcvt.f32.s32 s0, s0
 ; CHECK-NEXT:    vldr s2, [r7, #12]
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
 ; CHECK-NEXT:    vmul.f32 s0, s2, s0
 ; CHECK-NEXT:    vstr s0, [r6, #12]
 ; CHECK-NEXT:    bne .LBB3_12
@@ -1011,13 +1011,11 @@ define arm_aapcs_vfpcc void @half_half_mul(half* nocapture readonly %a, half* no
 ; CHECK-NEXT:    vmov.32 q1[1], r10
 ; CHECK-NEXT:    adds r5, #8
 ; CHECK-NEXT:    vmul.f16 q0, q0, q1
-; CHECK-NEXT:    vmovx.f16 s6, s1
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s11, s6
-; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
-; CHECK-NEXT:    vcvtb.f32.f16 s9, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
-; CHECK-NEXT:    vstrb.8 q2, [r6], #16
+; CHECK-NEXT:    vcvtt.f32.f16 s7, s1
+; CHECK-NEXT:    vcvtb.f32.f16 s6, s1
+; CHECK-NEXT:    vcvtt.f32.f16 s5, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s4, s0
+; CHECK-NEXT:    vstrb.8 q1, [r6], #16
 ; CHECK-NEXT:    le lr, .LBB5_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    cmp r12, r3
@@ -1130,13 +1128,11 @@ define arm_aapcs_vfpcc void @half_half_add(half* nocapture readonly %a, half* no
 ; CHECK-NEXT:    vmov.32 q1[1], r10
 ; CHECK-NEXT:    adds r5, #8
 ; CHECK-NEXT:    vadd.f16 q0, q0, q1
-; CHECK-NEXT:    vmovx.f16 s6, s1
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s11, s6
-; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
-; CHECK-NEXT:    vcvtb.f32.f16 s9, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
-; CHECK-NEXT:    vstrb.8 q2, [r6], #16
+; CHECK-NEXT:    vcvtt.f32.f16 s7, s1
+; CHECK-NEXT:    vcvtb.f32.f16 s6, s1
+; CHECK-NEXT:    vcvtt.f32.f16 s5, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s4, s0
+; CHECK-NEXT:    vstrb.8 q1, [r6], #16
 ; CHECK-NEXT:    le lr, .LBB6_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    cmp r12, r3
@@ -1249,13 +1245,11 @@ define arm_aapcs_vfpcc void @half_half_sub(half* nocapture readonly %a, half* no
 ; CHECK-NEXT:    vmov.32 q1[1], r10
 ; CHECK-NEXT:    adds r5, #8
 ; CHECK-NEXT:    vsub.f16 q0, q0, q1
-; CHECK-NEXT:    vmovx.f16 s6, s1
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s11, s6
-; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
-; CHECK-NEXT:    vcvtb.f32.f16 s9, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
-; CHECK-NEXT:    vstrb.8 q2, [r6], #16
+; CHECK-NEXT:    vcvtt.f32.f16 s7, s1
+; CHECK-NEXT:    vcvtb.f32.f16 s6, s1
+; CHECK-NEXT:    vcvtt.f32.f16 s5, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s4, s0
+; CHECK-NEXT:    vstrb.8 q1, [r6], #16
 ; CHECK-NEXT:    le lr, .LBB7_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    cmp r12, r3
@@ -1373,13 +1367,11 @@ define arm_aapcs_vfpcc void @half_short_mul(half* nocapture readonly %a, i16* no
 ; CHECK-NEXT:    vmov.32 q1[0], r9
 ; CHECK-NEXT:    vmov.32 q1[1], r8
 ; CHECK-NEXT:    vmul.f16 q0, q1, q0
-; CHECK-NEXT:    vmovx.f16 s6, s1
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s11, s6
-; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
-; CHECK-NEXT:    vcvtb.f32.f16 s9, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
-; CHECK-NEXT:    vstrb.8 q2, [r6], #16
+; CHECK-NEXT:    vcvtt.f32.f16 s7, s1
+; CHECK-NEXT:    vcvtb.f32.f16 s6, s1
+; CHECK-NEXT:    vcvtt.f32.f16 s5, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s4, s0
+; CHECK-NEXT:    vstrb.8 q1, [r6], #16
 ; CHECK-NEXT:    le lr, .LBB8_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    cmp r12, r3
@@ -1498,18 +1490,18 @@ define arm_aapcs_vfpcc float @half_half_mac(half* nocapture readonly %a, half* n
 ; CHECK-NEXT:    vmul.f16 s2, s4, s2
 ; CHECK-NEXT:    vldr.16 s4, [r2, #4]
 ; CHECK-NEXT:    vldr.16 s10, [r4]
-; CHECK-NEXT:    adds r3, #8
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vmul.f16 s4, s6, s4
 ; CHECK-NEXT:    vldr.16 s6, [r2, #2]
-; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
+; CHECK-NEXT:    adds r3, #8
 ; CHECK-NEXT:    vmul.f16 s6, s8, s6
 ; CHECK-NEXT:    vldr.16 s8, [r2]
+; CHECK-NEXT:    vcvtb.f32.f16 s6, s6
+; CHECK-NEXT:    add.w r12, r12, #4
 ; CHECK-NEXT:    vmul.f16 s8, s10, s8
 ; CHECK-NEXT:    vcvtb.f32.f16 s8, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s6, s6
 ; CHECK-NEXT:    vadd.f32 s0, s0, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vadd.f32 s0, s0, s6
 ; CHECK-NEXT:    vadd.f32 s0, s0, s4
 ; CHECK-NEXT:    vadd.f32 s0, s0, s2
@@ -1655,18 +1647,18 @@ define arm_aapcs_vfpcc float @half_half_acc(half* nocapture readonly %a, half* n
 ; CHECK-NEXT:    vadd.f16 s2, s4, s2
 ; CHECK-NEXT:    vldr.16 s4, [r2, #4]
 ; CHECK-NEXT:    vldr.16 s10, [r4]
-; CHECK-NEXT:    adds r3, #8
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vadd.f16 s4, s6, s4
 ; CHECK-NEXT:    vldr.16 s6, [r2, #2]
-; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
+; CHECK-NEXT:    adds r3, #8
 ; CHECK-NEXT:    vadd.f16 s6, s8, s6
 ; CHECK-NEXT:    vldr.16 s8, [r2]
+; CHECK-NEXT:    vcvtb.f32.f16 s6, s6
+; CHECK-NEXT:    add.w r12, r12, #4
 ; CHECK-NEXT:    vadd.f16 s8, s10, s8
 ; CHECK-NEXT:    vcvtb.f32.f16 s8, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s6, s6
 ; CHECK-NEXT:    vadd.f32 s0, s0, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vadd.f32 s0, s0, s6
 ; CHECK-NEXT:    vadd.f32 s0, s0, s4
 ; CHECK-NEXT:    vadd.f32 s0, s0, s2
@@ -1806,32 +1798,32 @@ define arm_aapcs_vfpcc float @half_short_mac(half* nocapture readonly %a, i16* n
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrsh.w r4, [r3, #2]
 ; CHECK-NEXT:    vldr.16 s2, [r2, #2]
+; CHECK-NEXT:    ldrsh r5, [r3, #-2]
 ; CHECK-NEXT:    add.w r12, r12, #4
 ; CHECK-NEXT:    vmov s4, r4
-; CHECK-NEXT:    vcvt.f16.s32 s4, s4
 ; CHECK-NEXT:    ldrsh.w r4, [r3]
+; CHECK-NEXT:    vcvt.f16.s32 s4, s4
+; CHECK-NEXT:    vmov s8, r5
 ; CHECK-NEXT:    vmul.f16 s2, s2, s4
 ; CHECK-NEXT:    vldr.16 s4, [r2]
 ; CHECK-NEXT:    vmov s6, r4
-; CHECK-NEXT:    vcvt.f16.s32 s6, s6
-; CHECK-NEXT:    ldrsh r5, [r3, #-2]
 ; CHECK-NEXT:    ldrsh r4, [r3, #-4]
+; CHECK-NEXT:    vcvt.f16.s32 s6, s6
+; CHECK-NEXT:    vcvt.f16.s32 s8, s8
 ; CHECK-NEXT:    vmul.f16 s4, s4, s6
 ; CHECK-NEXT:    vldr.16 s6, [r2, #-2]
-; CHECK-NEXT:    adds r3, #8
-; CHECK-NEXT:    vmov s8, r5
-; CHECK-NEXT:    vcvt.f16.s32 s8, s8
 ; CHECK-NEXT:    vmov s10, r4
+; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
 ; CHECK-NEXT:    vmul.f16 s6, s6, s8
 ; CHECK-NEXT:    vldr.16 s8, [r2, #-4]
 ; CHECK-NEXT:    vcvt.f16.s32 s10, s10
-; CHECK-NEXT:    adds r2, #8
+; CHECK-NEXT:    vcvtb.f32.f16 s6, s6
 ; CHECK-NEXT:    vmul.f16 s8, s8, s10
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vcvtb.f32.f16 s8, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s6, s6
+; CHECK-NEXT:    adds r3, #8
 ; CHECK-NEXT:    vadd.f32 s0, s0, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    adds r2, #8
 ; CHECK-NEXT:    vadd.f32 s0, s0, s6
 ; CHECK-NEXT:    vadd.f32 s0, s0, s4
 ; CHECK-NEXT:    vadd.f32 s0, s0, s2
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/reductions.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/reductions.ll
new file mode 100644
index 0000000000000..12c3ca0525f21
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/reductions.ll
@@ -0,0 +1,608 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -disable-mve-tail-predication=false %s -o - | FileCheck %s
+
+define dso_local arm_aapcs_vfpcc zeroext i8 @one_loop_add_add_v16i8(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %N) local_unnamed_addr {
+; CHECK-LABEL: one_loop_add_add_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    ittt eq
+; CHECK-NEXT:    moveq r0, #0
+; CHECK-NEXT:    uxtbeq r0, r0
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    add.w r3, r2, #15
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    bic r3, r3, #15
+; CHECK-NEXT:    sub.w r12, r3, #16
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #4
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB0_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vctp.8 r2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.u8 q1, [r0], #16
+; CHECK-NEXT:    subs r2, #16
+; CHECK-NEXT:    vadd.i8 q1, q1, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.u8 q2, [r1], #16
+; CHECK-NEXT:    vadd.i8 q1, q1, q2
+; CHECK-NEXT:    le lr, .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vaddv.u8 r0, q0
+; CHECK-NEXT:    pop.w {r7, lr}
+; CHECK-NEXT:    uxtb r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp11 = icmp eq i32 %N, 0
+  br i1 %cmp11, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 15
+  %n.vec = and i32 %n.rnd.up, -16
+  %trip.count.minus.1 = add i32 %N, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <16 x i8> [ zeroinitializer, %vector.ph ], [ %i5, %vector.body ]
+  %i = getelementptr inbounds i8, i8* %a, i32 %index
+  %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %i1 = bitcast i8* %i to <16 x i8>*
+  %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %i1, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
+  %i2 = getelementptr inbounds i8, i8* %b, i32 %index
+  %i3 = bitcast i8* %i2 to <16 x i8>*
+  %wide.masked.load16 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %i3, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
+  %i4 = add <16 x i8> %wide.masked.load, %vec.phi
+  %i5 = add <16 x i8> %i4, %wide.masked.load16
+  %index.next = add i32 %index, 16
+  %i6 = icmp eq i32 %index.next, %n.vec
+  br i1 %i6, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %i7 = select <16 x i1> %active.lane.mask, <16 x i8> %i5, <16 x i8> %vec.phi
+  %i8 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> %i7)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i8 [ 0, %entry ], [ %i8, %middle.block ]
+  ret i8 %res.0.lcssa
+}
+
+define dso_local arm_aapcs_vfpcc signext i16 @one_loop_add_add_v8i16(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %N) local_unnamed_addr {
+; CHECK-LABEL: one_loop_add_add_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    ittt eq
+; CHECK-NEXT:    moveq r0, #0
+; CHECK-NEXT:    sxtheq r0, r0
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    adds r3, r2, #7
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    bic r3, r3, #7
+; CHECK-NEXT:    sub.w r12, r3, #8
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #3
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB1_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vctp.16 r2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.u16 q1, [r0], #8
+; CHECK-NEXT:    subs r2, #8
+; CHECK-NEXT:    vadd.i16 q1, q0, q1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrbt.u16 q2, [r1], #8
+; CHECK-NEXT:    vadd.i16 q1, q1, q2
+; CHECK-NEXT:    le lr, .LBB1_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vaddv.u16 r0, q0
+; CHECK-NEXT:    pop.w {r7, lr}
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp12 = icmp eq i32 %N, 0
+  br i1 %cmp12, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 7
+  %n.vec = and i32 %n.rnd.up, -8
+  %trip.count.minus.1 = add i32 %N, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <8 x i16> [ zeroinitializer, %vector.ph ], [ %i7, %vector.body ]
+  %i = getelementptr inbounds i8, i8* %a, i32 %index
+  %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %i1 = bitcast i8* %i to <8 x i8>*
+  %wide.masked.load = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %i1, i32 1, <8 x i1> %active.lane.mask, <8 x i8> undef)
+  %i2 = zext <8 x i8> %wide.masked.load to <8 x i16>
+  %i3 = getelementptr inbounds i8, i8* %b, i32 %index
+  %i4 = bitcast i8* %i3 to <8 x i8>*
+  %wide.masked.load17 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %i4, i32 1, <8 x i1> %active.lane.mask, <8 x i8> undef)
+  %i5 = zext <8 x i8> %wide.masked.load17 to <8 x i16>
+  %i6 = add <8 x i16> %vec.phi, %i2
+  %i7 = add <8 x i16> %i6, %i5
+  %index.next = add i32 %index, 8
+  %i8 = icmp eq i32 %index.next, %n.vec
+  br i1 %i8, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %i9 = select <8 x i1> %active.lane.mask, <8 x i16> %i7, <8 x i16> %vec.phi
+  %i10 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %i9)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i16 [ 0, %entry ], [ %i10, %middle.block ]
+  ret i16 %res.0.lcssa
+}
+
+define dso_local arm_aapcs_vfpcc zeroext i8 @one_loop_sub_add_v16i8(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %N) local_unnamed_addr {
+; CHECK-LABEL: one_loop_sub_add_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    ittt eq
+; CHECK-NEXT:    moveq r0, #0
+; CHECK-NEXT:    uxtbeq r0, r0
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    dlstp.8 lr, r2
+; CHECK-NEXT:  .LBB2_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
+; CHECK-NEXT:    vldrb.u8 q2, [r0], #16
+; CHECK-NEXT:    vsub.i8 q1, q2, q1
+; CHECK-NEXT:    vadd.i8 q0, q1, q0
+; CHECK-NEXT:    letp lr, .LBB2_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vaddv.u8 r0, q0
+; CHECK-NEXT:    pop.w {r7, lr}
+; CHECK-NEXT:    uxtb r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp11 = icmp eq i32 %N, 0
+  br i1 %cmp11, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 15
+  %n.vec = and i32 %n.rnd.up, -16
+  %trip.count.minus.1 = add i32 %N, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <16 x i8> [ zeroinitializer, %vector.ph ], [ %i5, %vector.body ]
+  %i = getelementptr inbounds i8, i8* %a, i32 %index
+  %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %i1 = bitcast i8* %i to <16 x i8>*
+  %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %i1, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
+  %i2 = getelementptr inbounds i8, i8* %b, i32 %index
+  %i3 = bitcast i8* %i2 to <16 x i8>*
+  %wide.masked.load16 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %i3, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
+  %i4 = sub <16 x i8> %wide.masked.load, %wide.masked.load16
+  %i5 = add <16 x i8> %i4, %vec.phi
+  %index.next = add i32 %index, 16
+  %i6 = icmp eq i32 %index.next, %n.vec
+  br i1 %i6, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %i7 = select <16 x i1> %active.lane.mask, <16 x i8> %i5, <16 x i8> %vec.phi
+  %i8 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> %i7)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i8 [ 0, %entry ], [ %i8, %middle.block ]
+  ret i8 %res.0.lcssa
+}
+
+define dso_local arm_aapcs_vfpcc signext i16 @one_loop_sub_add_v8i16(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %N) local_unnamed_addr {
+; CHECK-LABEL: one_loop_sub_add_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    ittt eq
+; CHECK-NEXT:    moveq r0, #0
+; CHECK-NEXT:    sxtheq r0, r0
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    dlstp.16 lr, r2
+; CHECK-NEXT:  .LBB3_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrb.u16 q1, [r0], #8
+; CHECK-NEXT:    vldrb.u16 q2, [r1], #8
+; CHECK-NEXT:    vsub.i16 q1, q2, q1
+; CHECK-NEXT:    vadd.i16 q0, q1, q0
+; CHECK-NEXT:    letp lr, .LBB3_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vaddv.u16 r0, q0
+; CHECK-NEXT:    pop.w {r7, lr}
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp12 = icmp eq i32 %N, 0
+  br i1 %cmp12, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 7
+  %n.vec = and i32 %n.rnd.up, -8
+  %trip.count.minus.1 = add i32 %N, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <8 x i16> [ zeroinitializer, %vector.ph ], [ %i7, %vector.body ]
+  %i = getelementptr inbounds i8, i8* %a, i32 %index
+  %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %i1 = bitcast i8* %i to <8 x i8>*
+  %wide.masked.load = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %i1, i32 1, <8 x i1> %active.lane.mask, <8 x i8> undef)
+  %i2 = zext <8 x i8> %wide.masked.load to <8 x i16>
+  %i3 = getelementptr inbounds i8, i8* %b, i32 %index
+  %i4 = bitcast i8* %i3 to <8 x i8>*
+  %wide.masked.load17 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %i4, i32 1, <8 x i1> %active.lane.mask, <8 x i8> undef)
+  %i5 = zext <8 x i8> %wide.masked.load17 to <8 x i16>
+  %i6 = sub <8 x i16> %i5, %i2
+  %i7 = add <8 x i16> %i6, %vec.phi
+  %index.next = add i32 %index, 8
+  %i8 = icmp eq i32 %index.next, %n.vec
+  br i1 %i8, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %i9 = select <8 x i1> %active.lane.mask, <8 x i16> %i7, <8 x i16> %vec.phi
+  %i10 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %i9)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i16 [ 0, %entry ], [ %i10, %middle.block ]
+  ret i16 %res.0.lcssa
+}
+
+define dso_local arm_aapcs_vfpcc zeroext i8 @one_loop_mul_add_v16i8(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %N) local_unnamed_addr {
+; CHECK-LABEL: one_loop_mul_add_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    ittt eq
+; CHECK-NEXT:    moveq r0, #0
+; CHECK-NEXT:    uxtbeq r0, r0
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    dlstp.8 lr, r2
+; CHECK-NEXT:  .LBB4_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrb.u8 q1, [r0], #16
+; CHECK-NEXT:    vldrb.u8 q2, [r1], #16
+; CHECK-NEXT:    vmul.i8 q1, q2, q1
+; CHECK-NEXT:    vadd.i8 q0, q1, q0
+; CHECK-NEXT:    letp lr, .LBB4_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vaddv.u8 r0, q0
+; CHECK-NEXT:    pop.w {r7, lr}
+; CHECK-NEXT:    uxtb r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 15
+  %n.vec = and i32 %n.rnd.up, -16
+  %trip.count.minus.1 = add i32 %N, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <16 x i8> [ zeroinitializer, %vector.ph ], [ %i5, %vector.body ]
+  %i = getelementptr inbounds i8, i8* %a, i32 %index
+  %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %i1 = bitcast i8* %i to <16 x i8>*
+  %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %i1, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
+  %i2 = getelementptr inbounds i8, i8* %b, i32 %index
+  %i3 = bitcast i8* %i2 to <16 x i8>*
+  %wide.masked.load15 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %i3, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
+  %i4 = mul <16 x i8> %wide.masked.load15, %wide.masked.load
+  %i5 = add <16 x i8> %i4, %vec.phi
+  %index.next = add i32 %index, 16
+  %i6 = icmp eq i32 %index.next, %n.vec
+  br i1 %i6, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %i7 = select <16 x i1> %active.lane.mask, <16 x i8> %i5, <16 x i8> %vec.phi
+  %i8 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> %i7)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i8 [ 0, %entry ], [ %i8, %middle.block ]
+  ret i8 %res.0.lcssa
+}
+
+define dso_local arm_aapcs_vfpcc signext i16 @one_loop_mul_add_v8i16(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %N) local_unnamed_addr {
+; CHECK-LABEL: one_loop_mul_add_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    ittt eq
+; CHECK-NEXT:    moveq r0, #0
+; CHECK-NEXT:    sxtheq r0, r0
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    dlstp.16 lr, r2
+; CHECK-NEXT:  .LBB5_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrb.u16 q1, [r0], #8
+; CHECK-NEXT:    vldrb.u16 q2, [r1], #8
+; CHECK-NEXT:    vmul.i16 q1, q2, q1
+; CHECK-NEXT:    vadd.i16 q0, q1, q0
+; CHECK-NEXT:    letp lr, .LBB5_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vaddv.u16 r0, q0
+; CHECK-NEXT:    pop.w {r7, lr}
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %cmp12 = icmp eq i32 %N, 0
+  br i1 %cmp12, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 7
+  %n.vec = and i32 %n.rnd.up, -8
+  %trip.count.minus.1 = add i32 %N, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <8 x i16> [ zeroinitializer, %vector.ph ], [ %i7, %vector.body ]
+  %i = getelementptr inbounds i8, i8* %a, i32 %index
+  %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %i1 = bitcast i8* %i to <8 x i8>*
+  %wide.masked.load = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %i1, i32 1, <8 x i1> %active.lane.mask, <8 x i8> undef)
+  %i2 = zext <8 x i8> %wide.masked.load to <8 x i16>
+  %i3 = getelementptr inbounds i8, i8* %b, i32 %index
+  %i4 = bitcast i8* %i3 to <8 x i8>*
+  %wide.masked.load17 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %i4, i32 1, <8 x i1> %active.lane.mask, <8 x i8> undef)
+  %i5 = zext <8 x i8> %wide.masked.load17 to <8 x i16>
+  %i6 = mul <8 x i16> %i5, %i2
+  %i7 = add <8 x i16> %i6, %vec.phi
+  %index.next = add i32 %index, 8
+  %i8 = icmp eq i32 %index.next, %n.vec
+  br i1 %i8, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %i9 = select <8 x i1> %active.lane.mask, <8 x i16> %i7, <8 x i16> %vec.phi
+  %i10 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %i9)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i16 [ 0, %entry ], [ %i10, %middle.block ]
+  ret i16 %res.0.lcssa
+}
+
+define dso_local arm_aapcs_vfpcc i32 @two_loops_mul_add_v4i32(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %N) local_unnamed_addr {
+; CHECK-LABEL: two_loops_mul_add_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    beq .LBB6_8
+; CHECK-NEXT:  @ %bb.1: @ %vector.ph
+; CHECK-NEXT:    adds r3, r2, #3
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    subs r6, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    add.w lr, r3, r6, lsr #2
+; CHECK-NEXT:    mov r3, r2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB6_2: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vctp.32 r3
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vldrbt.u32 q1, [r4], #4
+; CHECK-NEXT:    vldrbt.u32 q2, [r5], #4
+; CHECK-NEXT:    subs r3, #4
+; CHECK-NEXT:    vmul.i32 q1, q2, q1
+; CHECK-NEXT:    vadd.i32 q1, q1, q0
+; CHECK-NEXT:    le lr, .LBB6_2
+; CHECK-NEXT:  @ %bb.3: @ %middle.block
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vaddv.u32 r12, q0
+; CHECK-NEXT:    cbz r2, .LBB6_7
+; CHECK-NEXT:  @ %bb.4: @ %vector.ph47
+; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:    vdup.32 q0, r3
+; CHECK-NEXT:    vmov.32 q1[0], r12
+; CHECK-NEXT:  .LBB6_5: @ %vector.body46
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrb.u32 q0, [r0], #4
+; CHECK-NEXT:    vldrb.u32 q2, [r1], #4
+; CHECK-NEXT:    vmul.i32 q0, q2, q0
+; CHECK-NEXT:    vadd.i32 q1, q0, q1
+; CHECK-NEXT:    letp lr, .LBB6_5
+; CHECK-NEXT:  @ %bb.6: @ %middle.block44
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    vaddv.u32 r12, q0
+; CHECK-NEXT:  .LBB6_7: @ %for.cond.cleanup7
+; CHECK-NEXT:    mov r0, r12
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:  .LBB6_8:
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+entry:
+  %cmp35 = icmp eq i32 %N, 0
+  br i1 %cmp35, label %for.cond.cleanup7, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %i7, %vector.body ]
+  %i = getelementptr inbounds i8, i8* %a, i32 %index
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %i1 = bitcast i8* %i to <4 x i8>*
+  %wide.masked.load = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %i1, i32 1, <4 x i1> %active.lane.mask, <4 x i8> undef)
+  %i2 = zext <4 x i8> %wide.masked.load to <4 x i32>
+  %i3 = getelementptr inbounds i8, i8* %b, i32 %index
+  %i4 = bitcast i8* %i3 to <4 x i8>*
+  %wide.masked.load43 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %i4, i32 1, <4 x i1> %active.lane.mask, <4 x i8> undef)
+  %i5 = zext <4 x i8> %wide.masked.load43 to <4 x i32>
+  %i6 = mul nuw nsw <4 x i32> %i5, %i2
+  %i7 = add <4 x i32> %i6, %vec.phi
+  %index.next = add i32 %index, 4
+  %i8 = icmp eq i32 %index.next, %n.vec
+  br i1 %i8, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %i9 = select <4 x i1> %active.lane.mask, <4 x i32> %i7, <4 x i32> %vec.phi
+  %i10 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %i9)
+  br i1 %cmp35, label %for.cond.cleanup7, label %vector.ph47
+
+vector.ph47:                                      ; preds = %middle.block
+  %n.rnd.up48 = add i32 %N, 3
+  %n.vec50 = and i32 %n.rnd.up48, -4
+  %trip.count.minus.154 = add i32 %N, -1
+  %i11 = insertelement <4 x i32> <i32 undef, i32 0, i32 0, i32 0>, i32 %i10, i32 0
+  br label %vector.body46
+
+vector.body46:                                    ; preds = %vector.body46, %vector.ph47
+  %index51 = phi i32 [ 0, %vector.ph47 ], [ %index.next52, %vector.body46 ]
+  %vec.phi60 = phi <4 x i32> [ %i11, %vector.ph47 ], [ %i19, %vector.body46 ]
+  %i12 = getelementptr inbounds i8, i8* %a, i32 %index51
+  %active.lane.mask61 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index51, i32 %trip.count.minus.154)
+  %i13 = bitcast i8* %i12 to <4 x i8>*
+  %wide.masked.load62 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %i13, i32 1, <4 x i1> %active.lane.mask61, <4 x i8> undef)
+  %i14 = zext <4 x i8> %wide.masked.load62 to <4 x i32>
+  %i15 = getelementptr inbounds i8, i8* %b, i32 %index51
+  %i16 = bitcast i8* %i15 to <4 x i8>*
+  %wide.masked.load63 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %i16, i32 1, <4 x i1> %active.lane.mask61, <4 x i8> undef)
+  %i17 = zext <4 x i8> %wide.masked.load63 to <4 x i32>
+  %i18 = mul nuw nsw <4 x i32> %i17, %i14
+  %i19 = add <4 x i32> %i18, %vec.phi60
+  %index.next52 = add i32 %index51, 4
+  %i20 = icmp eq i32 %index.next52, %n.vec50
+  br i1 %i20, label %middle.block44, label %vector.body46
+
+middle.block44:                                   ; preds = %vector.body46
+  %i21 = select <4 x i1> %active.lane.mask61, <4 x i32> %i19, <4 x i32> %vec.phi60
+  %i22 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %i21)
+  br label %for.cond.cleanup7
+
+for.cond.cleanup7:                                ; preds = %middle.block44, %middle.block, %entry
+  %res.1.lcssa = phi i32 [ %i10, %middle.block ], [ 0, %entry ], [ %i22, %middle.block44 ]
+  ret i32 %res.1.lcssa
+}
+
+define dso_local arm_aapcs_vfpcc void @two_reductions_mul_add_v8i16(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %N) local_unnamed_addr {
+; CHECK-LABEL: two_reductions_mul_add_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    cbz r2, .LBB7_4
+; CHECK-NEXT:  @ %bb.1: @ %vector.ph
+; CHECK-NEXT:    adds r3, r2, #7
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    bic r3, r3, #7
+; CHECK-NEXT:    movs r4, #1
+; CHECK-NEXT:    subs r3, #8
+; CHECK-NEXT:    vmov q3, q1
+; CHECK-NEXT:    add.w lr, r4, r3, lsr #3
+; CHECK-NEXT:    mov r3, r0
+; CHECK-NEXT:    mov r4, r1
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB7_2: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vctp.16 r2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vldrbt.u16 q1, [r3], #8
+; CHECK-NEXT:    vldrbt.u16 q4, [r4], #8
+; CHECK-NEXT:    vmov q2, q3
+; CHECK-NEXT:    vsub.i16 q3, q4, q1
+; CHECK-NEXT:    vmul.i16 q1, q4, q1
+; CHECK-NEXT:    subs r2, #8
+; CHECK-NEXT:    vadd.i16 q3, q3, q2
+; CHECK-NEXT:    vadd.i16 q1, q1, q0
+; CHECK-NEXT:    le lr, .LBB7_2
+; CHECK-NEXT:  @ %bb.3: @ %middle.block
+; CHECK-NEXT:    vpsel q2, q3, q2
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vaddv.u16 r4, q2
+; CHECK-NEXT:    vaddv.u16 r2, q0
+; CHECK-NEXT:    b .LBB7_5
+; CHECK-NEXT:  .LBB7_4:
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    movs r4, #0
+; CHECK-NEXT:  .LBB7_5: @ %for.cond.cleanup
+; CHECK-NEXT:    strb r2, [r0]
+; CHECK-NEXT:    strb r4, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %cmp12 = icmp eq i32 %N, 0
+  br i1 %cmp12, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 7
+  %n.vec = and i32 %n.rnd.up, -8
+  %trip.count.minus.1 = add i32 %N, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <8 x i16> [ zeroinitializer, %vector.ph ], [ %i8, %vector.body ]
+  %vec.phi.1 = phi <8 x i16> [ zeroinitializer, %vector.ph ], [ %i9, %vector.body ]
+  %i = getelementptr inbounds i8, i8* %a, i32 %index
+  %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %i1 = bitcast i8* %i to <8 x i8>*
+  %wide.masked.load = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %i1, i32 1, <8 x i1> %active.lane.mask, <8 x i8> undef)
+  %i2 = zext <8 x i8> %wide.masked.load to <8 x i16>
+  %i3 = getelementptr inbounds i8, i8* %b, i32 %index
+  %i4 = bitcast i8* %i3 to <8 x i8>*
+  %wide.masked.load17 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %i4, i32 1, <8 x i1> %active.lane.mask, <8 x i8> undef)
+  %i5 = zext <8 x i8> %wide.masked.load17 to <8 x i16>
+  %i6 = mul <8 x i16> %i5, %i2
+  %i7 = sub <8 x i16> %i5, %i2
+  %i8 = add <8 x i16> %i6, %vec.phi
+  %i9 = add <8 x i16> %i7, %vec.phi.1
+  %index.next = add i32 %index, 8
+  %i10 = icmp eq i32 %index.next, %n.vec
+  br i1 %i10, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %i11 = select <8 x i1> %active.lane.mask, <8 x i16> %i8, <8 x i16> %vec.phi
+  %i12 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %i11)
+  %i13 = select <8 x i1> %active.lane.mask, <8 x i16> %i9, <8 x i16> %vec.phi.1
+  %i14 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %i13)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i16 [ 0, %entry ], [ %i12, %middle.block ]
+  %res.1.lcssa = phi i16 [ 0, %entry ], [ %i14, %middle.block ]
+  %trunc.res.0 = trunc i16 %res.0.lcssa to i8
+  store i8 %trunc.res.0, i8* %a
+  %trunc.res.1 = trunc i16 %res.1.lcssa to i8
+  store i8 %trunc.res.1, i8* %b
+  ret void
+}
+
+declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32, i32)
+declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32 immarg, <16 x i1>, <16 x i8>)
+declare i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8>)
+declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32)
+declare <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>*, i32 immarg, <8 x i1>, <8 x i8>)
+declare i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16>)
+declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)
+declare <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>*, i32 immarg, <4 x i1>, <4 x i8>)
+declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/sibling-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/sibling-loops.ll
index 245694942ffbd..69d370fc01a2f 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/sibling-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/sibling-loops.ll
@@ -29,16 +29,16 @@ define arm_aapcs_vfpcc void @test(i16* noalias nocapture readonly %off, i16* noa
 ; CHECK-NEXT:  @ %bb.4: @ %for.body15.us.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB0_2 Depth=1
 ; CHECK-NEXT:    movs r5, #0
+; CHECK-NEXT:    dls lr, r3
 ; CHECK-NEXT:  .LBB0_5: @ %for.body15.us
 ; CHECK-NEXT:    @ Parent Loop BB0_2 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    ldrh.w r6, [r0, r5, lsl #1]
-; CHECK-NEXT:    ldrh.w r7, [r1, r5, lsl #1]
-; CHECK-NEXT:    add r7, r6
-; CHECK-NEXT:    strh.w r7, [r2, r5, lsl #1]
+; CHECK-NEXT:    ldrh.w r7, [r0, r5, lsl #1]
+; CHECK-NEXT:    ldrh.w r6, [r1, r5, lsl #1]
+; CHECK-NEXT:    add r6, r7
+; CHECK-NEXT:    strh.w r6, [r2, r5, lsl #1]
 ; CHECK-NEXT:    adds r5, #1
-; CHECK-NEXT:    cmp r3, r5
-; CHECK-NEXT:    bne .LBB0_5
+; CHECK-NEXT:    le lr, .LBB0_5
 ; CHECK-NEXT:  @ %bb.6: @ %for.cond.cleanup14.us
 ; CHECK-NEXT:    @ in Loop: Header=BB0_2 Depth=1
 ; CHECK-NEXT:    add.w r8, r8, #1
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-const.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-const.ll
index dc9da0c9f7641..13d750310a56c 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-const.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-const.ll
@@ -266,16 +266,9 @@ for.cond.cleanup:
 }
 
 ; CHECK-LABEL: @overflow_BTC_plus_1(
-;
+; CHECK:       vector.body:
 ; CHECK-NOT:   @llvm.arm.mve.vctp32
-; CHECK-NOT:   @llvm.get.active.lane.mask
-;
-; CHECK: %lane.mask.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
-; CHECK: %lane.mask.splat = shufflevector <4 x i32> %lane.mask.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
-; CHECK: %lane.mask.induction = add <4 x i32> %lane.mask.splat, <i32 0, i32 1, i32 2, i32 3>
-; CHECK: %[[ICMP:.*]] = icmp ule <4 x i32> %lane.mask.induction, <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32({{.*}}, <4 x i1> %[[ICMP]], <4 x i32> undef)
-;
+; CHECK:       @llvm.get.active.lane.mask
 ; CHECK:       ret void
 ;
 define dso_local void @overflow_BTC_plus_1(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32* noalias nocapture readnone %D, i32 %N) local_unnamed_addr #0 {
@@ -316,8 +309,9 @@ for.cond.cleanup:
 }
 
 ; CHECK-LABEL: @overflow_in_sub(
+; CHECK:       vector.body:
 ; CHECK-NOT:   @llvm.arm.mve.vctp32
-; CHECK-NOT:   @llvm.get.active.lane.mask
+; CHECK:       @llvm.get.active.lane.mask
 ; CHECK:       ret void
 ;
 define dso_local void @overflow_in_sub(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32* noalias nocapture readnone %D, i32 %N) local_unnamed_addr #0 {
@@ -366,8 +360,9 @@ for.cond.cleanup:
 }
 
 ; CHECK-LABEL: @overflow_in_rounding_tripcount(
+; CHECK:       vector.body:
 ; CHECK-NOT:   @llvm.arm.mve.vctp32
-; CHECK-NOT:   @llvm.get.active.lane.mask
+; CHECK:       @llvm.get.active.lane.mask
 ; CHECK:       ret void
 ;
 define dso_local void @overflow_in_rounding_tripcount(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32* noalias nocapture readnone %D, i32 %N) local_unnamed_addr #0 {
@@ -413,15 +408,9 @@ for.cond.cleanup:
 
 
 ; CHECK-LABEL: @IV_not_an_induction(
-;
+; CHECK:       vector.body:
 ; CHECK-NOT:   @llvm.arm.mve.vctp32
-; CHECK-NOT:   @llvm.get.active.lane.mask
-;
-; CHECK:  %lane.mask.splatinsert = insertelement <4 x i32> undef, i32 %N, i32 0
-; CHECK:  %lane.mask.splat = shufflevector <4 x i32> %lane.mask.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
-; CHECK:  %lane.mask.induction = add <4 x i32> %lane.mask.splat, <i32 0, i32 1, i32 2, i32 3>
-; CHECK:  %[[ICMP:.*]] = icmp ule <4 x i32> %lane.mask.induction, <i32 32002, i32 32002, i32 32002, i32 32002>
-; CHECK: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32({{.*}}, <4 x i1> %[[ICMP]], <4 x i32> undef)
+; CHECK:       @llvm.get.active.lane.mask
 ; CHECK:       ret void
 ;
 define dso_local void @IV_not_an_induction(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32* noalias nocapture readnone %D, i32 %N) local_unnamed_addr #0 {
@@ -462,15 +451,9 @@ for.cond.cleanup:
 }
 
 ; CHECK-LABEL: @IV_wrong_step(
-;
+; CHECK:       vector.body:
 ; CHECK-NOT:   @llvm.arm.mve.vctp32
-; CHECK-NOT:   @llvm.get.active.lane.mask
-;
-; CHECK:  %lane.mask.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
-; CHECK:  %lane.mask.splat = shufflevector <4 x i32> %lane.mask.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
-; CHECK:  %lane.mask.induction = add <4 x i32> %lane.mask.splat, <i32 0, i32 1, i32 2, i32 3>
-; CHECK:  %[[ICMP:.*]] = icmp ule <4 x i32> %lane.mask.induction, <i32 32002, i32 32002, i32 32002, i32 32002>
-; CHECK: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32({{.*}}, <4 x i1> %[[ICMP]], <4 x i32> undef)
+; CHECK:       @llvm.get.active.lane.mask
 ; CHECK:       ret void
 ;
 define dso_local void @IV_wrong_step(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32* noalias nocapture readnone %D, i32 %N) local_unnamed_addr #0 {
@@ -514,15 +497,9 @@ for.cond.cleanup:
 }
 
 ; CHECK-LABEL: @IV_step_not_constant(
-;
+; CHECK:       vector.body:
 ; CHECK-NOT:   @llvm.arm.mve.vctp32
-; CHECK-NOT:   @llvm.get.active.lane.mask
-;
-; CHECK:  %lane.mask.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
-; CHECK:  %lane.mask.splat = shufflevector <4 x i32> %lane.mask.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
-; CHECK:  %lane.mask.induction = add <4 x i32> %lane.mask.splat, <i32 0, i32 1, i32 2, i32 3>
-; CHECK:  %[[ICMP:.*]] = icmp ule <4 x i32> %lane.mask.induction, <i32 32002, i32 32002, i32 32002, i32 32002>
-; CHECK: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32({{.*}}, <4 x i1> %[[ICMP]], <4 x i32> undef)
+; CHECK:       @llvm.get.active.lane.mask
 ; CHECK:       ret void
 ;
 define dso_local void @IV_step_not_constant(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32* noalias nocapture readnone %D, i32 %N) local_unnamed_addr #0 {
@@ -563,15 +540,9 @@ for.cond.cleanup:
 }
 
 ; CHECK-LABEL: @outerloop_phi(
-;
+; CHECK:       vector.body:
 ; CHECK-NOT:   @llvm.arm.mve.vctp32
-; CHECK-NOT:   @llvm.get.active.lane.mask
-; CHECK:  %lane.mask.splatinsert = insertelement <4 x i32> undef, i32 %j.025, i32 0
-; CHECK:  %lane.mask.splat = shufflevector <4 x i32> %lane.mask.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
-; CHECK:  %lane.mask.induction = add <4 x i32> %lane.mask.splat, <i32 0, i32 1, i32 2, i32 3>
-; CHECK:  %[[ICMP:.*]] = icmp ule <4 x i32> %lane.mask.induction, <i32 4096, i32 4096, i32 4096, i32 4096>
-; CHECK: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32({{.*}}, <4 x i1> %[[ICMP]], <4 x i32> undef)
-;
+; CHECK:       @llvm.get.active.lane.mask
 ; CHECK:       ret void
 ;
 define dso_local void @outerloop_phi(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32 %N) local_unnamed_addr #0 {
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-add-sat.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-add-sat.ll
new file mode 100644
index 0000000000000..d405657f4d17e
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-add-sat.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs -disable-mve-tail-predication=false -o - %s | FileCheck %s
+define arm_aapcs_vfpcc void @uadd_sat(i16* noalias nocapture readonly %pSrcA, i16* noalias nocapture readonly %pSrcB, i16* noalias nocapture %pDst, i32 %blockSize) {
+; CHECK-LABEL: uadd_sat:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.16 lr, r3
+; CHECK-NEXT:  .LBB0_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
+; CHECK-NEXT:    vldrh.u16 q1, [r0], #16
+; CHECK-NEXT:    vqadd.u16 q0, q1, q0
+; CHECK-NEXT:    vstrh.16 q0, [r2], #16
+; CHECK-NEXT:    letp lr, .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %while.end
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp7 = icmp eq i32 %blockSize, 0
+  br i1 %cmp7, label %while.end, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %blockSize, 7
+  %n.vec = and i32 %n.rnd.up, -8
+  %trip.count.minus.1 = add i32 %blockSize, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %next.gep = getelementptr i16, i16* %pSrcA, i32 %index
+  %next.gep20 = getelementptr i16, i16* %pDst, i32 %index
+  %next.gep21 = getelementptr i16, i16* %pSrcB, i32 %index
+  %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %0 = bitcast i16* %next.gep to <8 x i16>*
+  %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
+  %1 = bitcast i16* %next.gep21 to <8 x i16>*
+  %wide.masked.load24 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %1, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
+  %2 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %wide.masked.load, <8 x i16> %wide.masked.load24)
+  %3 = bitcast i16* %next.gep20 to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %2, <8 x i16>* %3, i32 2, <8 x i1> %active.lane.mask)
+  %index.next = add i32 %index, 8
+  %4 = icmp eq i32 %index.next, %n.vec
+  br i1 %4, label %while.end, label %vector.body
+
+while.end:                                        ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @sadd_sat(i16* noalias nocapture readonly %pSrcA, i16* noalias nocapture readonly %pSrcB, i16* noalias nocapture %pDst, i32 %blockSize) {
+; CHECK-LABEL: sadd_sat:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.16 lr, r3
+; CHECK-NEXT:  .LBB1_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
+; CHECK-NEXT:    vldrh.u16 q1, [r0], #16
+; CHECK-NEXT:    vqadd.s16 q0, q1, q0
+; CHECK-NEXT:    vstrh.16 q0, [r2], #16
+; CHECK-NEXT:    letp lr, .LBB1_1
+; CHECK-NEXT:  @ %bb.2: @ %while.end
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp7 = icmp eq i32 %blockSize, 0
+  br i1 %cmp7, label %while.end, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %blockSize, 7
+  %n.vec = and i32 %n.rnd.up, -8
+  %trip.count.minus.1 = add i32 %blockSize, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %next.gep = getelementptr i16, i16* %pSrcA, i32 %index
+  %next.gep20 = getelementptr i16, i16* %pDst, i32 %index
+  %next.gep21 = getelementptr i16, i16* %pSrcB, i32 %index
+  %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %0 = bitcast i16* %next.gep to <8 x i16>*
+  %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
+  %1 = bitcast i16* %next.gep21 to <8 x i16>*
+  %wide.masked.load24 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %1, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
+  %2 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %wide.masked.load, <8 x i16> %wide.masked.load24)
+  %3 = bitcast i16* %next.gep20 to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %2, <8 x i16>* %3, i32 2, <8 x i1> %active.lane.mask)
+  %index.next = add i32 %index, 8
+  %4 = icmp eq i32 %index.next, %n.vec
+  br i1 %4, label %while.end, label %vector.body
+
+while.end:                                        ; preds = %vector.body, %entry
+  ret void
+}
+
+declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32)
+
+declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>)
+
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>)
+
+declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>)
+
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32 immarg, <8 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-fabs.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-fabs.ll
new file mode 100644
index 0000000000000..d3247a3fd28e7
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-fabs.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -verify-machineinstrs -disable-mve-tail-predication=false -o - %s | FileCheck %s
+define arm_aapcs_vfpcc void @fabs(float* noalias nocapture readonly %pSrcA, float* noalias nocapture %pDst, i32 %blockSize) {
+; CHECK-LABEL: fabs:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:  .LBB0_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vabs.f32 q0, q0
+; CHECK-NEXT:    vstrw.32 q0, [r1], #16
+; CHECK-NEXT:    letp lr, .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %while.end
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp3 = icmp eq i32 %blockSize, 0
+  br i1 %cmp3, label %while.end, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %blockSize, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %blockSize, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %next.gep = getelementptr float, float* %pDst, i32 %index
+  %next.gep13 = getelementptr float, float* %pSrcA, i32 %index
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %0 = bitcast float* %next.gep13 to <4 x float>*
+  %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %active.lane.mask, <4 x float> undef)
+  %1 = call fast <4 x float> @llvm.fabs.v4f32(<4 x float> %wide.masked.load)
+  %2 = bitcast float* %next.gep to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %active.lane.mask)
+  %index.next = add i32 %index, 4
+  %3 = icmp eq i32 %index.next, %n.vec
+  br i1 %3, label %while.end, label %vector.body
+
+while.end:                                        ; preds = %vector.body, %entry
+  ret void
+}
+
+declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)
+
+declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32 immarg, <4 x i1>, <4 x float>)
+
+declare <4 x float> @llvm.fabs.v4f32(<4 x float>)
+
+declare void @llvm.masked.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32 immarg, <4 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-round.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-round.ll
new file mode 100644
index 0000000000000..962e9df3dc1e9
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-round.ll
@@ -0,0 +1,318 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -verify-machineinstrs -disable-mve-tail-predication=false -o - %s | FileCheck %s
+define arm_aapcs_vfpcc void @round(float* noalias nocapture readonly %pSrcA, float* noalias nocapture %pDst, i32 %n) #0 {
+; CHECK-LABEL: round:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:  .LBB0_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vrinta.f32 q0, q0
+; CHECK-NEXT:    vstrw.32 q0, [r1], #16
+; CHECK-NEXT:    letp lr, .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp5 = icmp eq i32 %n, 0
+  br i1 %cmp5, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %n, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %n, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %next.gep = getelementptr float, float* %pSrcA, i32 %index
+  %next.gep14 = getelementptr float, float* %pDst, i32 %index
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %0 = bitcast float* %next.gep to <4 x float>*
+  %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %active.lane.mask, <4 x float> undef)
+  %1 = call fast <4 x float> @llvm.round.v4f32(<4 x float> %wide.masked.load)
+  %2 = bitcast float* %next.gep14 to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %active.lane.mask)
+  %index.next = add i32 %index, 4
+  %3 = icmp eq i32 %index.next, %n.vec
+  br i1 %3, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @rint(float* noalias nocapture readonly %pSrcA, float* noalias nocapture %pDst, i32 %n) #0 {
+; CHECK-LABEL: rint:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:  .LBB1_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vrintx.f32 q0, q0
+; CHECK-NEXT:    vstrw.32 q0, [r1], #16
+; CHECK-NEXT:    letp lr, .LBB1_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp5 = icmp eq i32 %n, 0
+  br i1 %cmp5, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %n, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %n, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %next.gep = getelementptr float, float* %pSrcA, i32 %index
+  %next.gep14 = getelementptr float, float* %pDst, i32 %index
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %0 = bitcast float* %next.gep to <4 x float>*
+  %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %active.lane.mask, <4 x float> undef)
+  %1 = call fast <4 x float> @llvm.rint.v4f32(<4 x float> %wide.masked.load)
+  %2 = bitcast float* %next.gep14 to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %active.lane.mask)
+  %index.next = add i32 %index, 4
+  %3 = icmp eq i32 %index.next, %n.vec
+  br i1 %3, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @trunc(float* noalias nocapture readonly %pSrcA, float* noalias nocapture %pDst, i32 %n) #0 {
+; CHECK-LABEL: trunc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:  .LBB2_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vrintz.f32 q0, q0
+; CHECK-NEXT:    vstrw.32 q0, [r1], #16
+; CHECK-NEXT:    letp lr, .LBB2_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp5 = icmp eq i32 %n, 0
+  br i1 %cmp5, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %n, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %n, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %next.gep = getelementptr float, float* %pSrcA, i32 %index
+  %next.gep14 = getelementptr float, float* %pDst, i32 %index
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %0 = bitcast float* %next.gep to <4 x float>*
+  %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %active.lane.mask, <4 x float> undef)
+  %1 = call fast <4 x float> @llvm.trunc.v4f32(<4 x float> %wide.masked.load)
+  %2 = bitcast float* %next.gep14 to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %active.lane.mask)
+  %index.next = add i32 %index, 4
+  %3 = icmp eq i32 %index.next, %n.vec
+  br i1 %3, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @ceil(float* noalias nocapture readonly %pSrcA, float* noalias nocapture %pDst, i32 %n) #0 {
+; CHECK-LABEL: ceil:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:  .LBB3_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vrintp.f32 q0, q0
+; CHECK-NEXT:    vstrw.32 q0, [r1], #16
+; CHECK-NEXT:    letp lr, .LBB3_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp5 = icmp eq i32 %n, 0
+  br i1 %cmp5, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %n, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %n, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %next.gep = getelementptr float, float* %pSrcA, i32 %index
+  %next.gep14 = getelementptr float, float* %pDst, i32 %index
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %0 = bitcast float* %next.gep to <4 x float>*
+  %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %active.lane.mask, <4 x float> undef)
+  %1 = call fast <4 x float> @llvm.ceil.v4f32(<4 x float> %wide.masked.load)
+  %2 = bitcast float* %next.gep14 to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %active.lane.mask)
+  %index.next = add i32 %index, 4
+  %3 = icmp eq i32 %index.next, %n.vec
+  br i1 %3, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @floor(float* noalias nocapture readonly %pSrcA, float* noalias nocapture %pDst, i32 %n) #0 {
+; CHECK-LABEL: floor:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:  .LBB4_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vrintm.f32 q0, q0
+; CHECK-NEXT:    vstrw.32 q0, [r1], #16
+; CHECK-NEXT:    letp lr, .LBB4_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp5 = icmp eq i32 %n, 0
+  br i1 %cmp5, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %n, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %n, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %next.gep = getelementptr float, float* %pSrcA, i32 %index
+  %next.gep14 = getelementptr float, float* %pDst, i32 %index
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %0 = bitcast float* %next.gep to <4 x float>*
+  %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %active.lane.mask, <4 x float> undef)
+  %1 = call fast <4 x float> @llvm.floor.v4f32(<4 x float> %wide.masked.load)
+  %2 = bitcast float* %next.gep14 to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %active.lane.mask)
+  %index.next = add i32 %index, 4
+  %3 = icmp eq i32 %index.next, %n.vec
+  br i1 %3, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; nearbyint shouldn't be tail predicated because it's lowered into multiple instructions
+define arm_aapcs_vfpcc void @nearbyint(float* noalias nocapture readonly %pSrcA, float* noalias nocapture %pDst, i32 %n) #0 {
+; CHECK-LABEL: nearbyint:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    adds r3, r2, #3
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    adr r3, .LCPI5_0
+; CHECK-NEXT:    sub.w r12, r2, #1
+; CHECK-NEXT:    vldrw.u32 q0, [r3]
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    vdup.32 q1, r12
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB5_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vadd.i32 q2, q0, r2
+; CHECK-NEXT:    vdup.32 q3, r2
+; CHECK-NEXT:    vcmp.u32 hi, q3, q2
+; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vcmpt.u32 cs, q1, q2
+; CHECK-NEXT:    vldrwt.u32 q2, [r0], #16
+; CHECK-NEXT:    vrintr.f32 s15, s11
+; CHECK-NEXT:    vrintr.f32 s14, s10
+; CHECK-NEXT:    vrintr.f32 s13, s9
+; CHECK-NEXT:    vrintr.f32 s12, s8
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q3, [r1], #16
+; CHECK-NEXT:    le lr, .LBB5_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI5_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+entry:
+  %cmp5 = icmp eq i32 %n, 0
+  br i1 %cmp5, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %n, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %n, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %next.gep = getelementptr float, float* %pSrcA, i32 %index
+  %next.gep14 = getelementptr float, float* %pDst, i32 %index
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %0 = bitcast float* %next.gep to <4 x float>*
+  %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %active.lane.mask, <4 x float> undef)
+  %1 = call fast <4 x float> @llvm.nearbyint.v4f32(<4 x float> %wide.masked.load)
+  %2 = bitcast float* %next.gep14 to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %active.lane.mask)
+  %index.next = add i32 %index, 4
+  %3 = icmp eq i32 %index.next, %n.vec
+  br i1 %3, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32) #1
+
+declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32 immarg, <4 x i1>, <4 x float>) #2
+
+declare <4 x float> @llvm.trunc.v4f32(<4 x float>) #3
+
+declare <4 x float> @llvm.rint.v4f32(<4 x float>) #3
+
+declare <4 x float> @llvm.round.v4f32(<4 x float>) #3
+
+declare <4 x float> @llvm.ceil.v4f32(<4 x float>) #3
+
+declare <4 x float> @llvm.floor.v4f32(<4 x float>) #3
+
+declare <4 x float> @llvm.nearbyint.v4f32(<4 x float>) #1
+
+declare void @llvm.masked.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32 immarg, <4 x i1>) #4
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-sub-sat.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-sub-sat.ll
new file mode 100644
index 0000000000000..790311a54aa1d
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-sub-sat.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs -disable-mve-tail-predication=false -o - %s | FileCheck %s
+define arm_aapcs_vfpcc void @usub_sat(i16* noalias nocapture readonly %pSrcA, i16* noalias nocapture readonly %pSrcB, i16* noalias nocapture %pDst, i32 %blockSize) {
+; CHECK-LABEL: usub_sat:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.16 lr, r3
+; CHECK-NEXT:  .LBB0_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
+; CHECK-NEXT:    vldrh.u16 q1, [r0], #16
+; CHECK-NEXT:    vqsub.u16 q0, q1, q0
+; CHECK-NEXT:    vstrh.16 q0, [r2], #16
+; CHECK-NEXT:    letp lr, .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %while.end
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp7 = icmp eq i32 %blockSize, 0
+  br i1 %cmp7, label %while.end, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %blockSize, 7
+  %n.vec = and i32 %n.rnd.up, -8
+  %trip.count.minus.1 = add i32 %blockSize, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %next.gep = getelementptr i16, i16* %pSrcA, i32 %index
+  %next.gep20 = getelementptr i16, i16* %pDst, i32 %index
+  %next.gep21 = getelementptr i16, i16* %pSrcB, i32 %index
+  %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %0 = bitcast i16* %next.gep to <8 x i16>*
+  %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
+  %1 = bitcast i16* %next.gep21 to <8 x i16>*
+  %wide.masked.load24 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %1, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
+  %2 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %wide.masked.load, <8 x i16> %wide.masked.load24)
+  %3 = bitcast i16* %next.gep20 to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %2, <8 x i16>* %3, i32 2, <8 x i1> %active.lane.mask)
+  %index.next = add i32 %index, 8
+  %4 = icmp eq i32 %index.next, %n.vec
+  br i1 %4, label %while.end, label %vector.body
+
+while.end:                                        ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @ssub_sat(i16* noalias nocapture readonly %pSrcA, i16* noalias nocapture readonly %pSrcB, i16* noalias nocapture %pDst, i32 %blockSize) {
+; CHECK-LABEL: ssub_sat:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.16 lr, r3
+; CHECK-NEXT:  .LBB1_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
+; CHECK-NEXT:    vldrh.u16 q1, [r0], #16
+; CHECK-NEXT:    vqsub.s16 q0, q1, q0
+; CHECK-NEXT:    vstrh.16 q0, [r2], #16
+; CHECK-NEXT:    letp lr, .LBB1_1
+; CHECK-NEXT:  @ %bb.2: @ %while.end
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp7 = icmp eq i32 %blockSize, 0
+  br i1 %cmp7, label %while.end, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %blockSize, 7
+  %n.vec = and i32 %n.rnd.up, -8
+  %trip.count.minus.1 = add i32 %blockSize, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %next.gep = getelementptr i16, i16* %pSrcA, i32 %index
+  %next.gep20 = getelementptr i16, i16* %pDst, i32 %index
+  %next.gep21 = getelementptr i16, i16* %pSrcB, i32 %index
+  %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %0 = bitcast i16* %next.gep to <8 x i16>*
+  %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %0, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
+  %1 = bitcast i16* %next.gep21 to <8 x i16>*
+  %wide.masked.load24 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %1, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
+  %2 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %wide.masked.load, <8 x i16> %wide.masked.load24)
+  %3 = bitcast i16* %next.gep20 to <8 x i16>*
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %2, <8 x i16>* %3, i32 2, <8 x i1> %active.lane.mask)
+  %index.next = add i32 %index, 8
+  %4 = icmp eq i32 %index.next, %n.vec
+  br i1 %4, label %while.end, label %vector.body
+
+while.end:                                        ; preds = %vector.body, %entry
+  ret void
+}
+
+declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32)
+
+declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>)
+
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>)
+
+declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>)
+
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32 immarg, <8 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll
index 0b103ca54750b..5c753134744d6 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll
@@ -143,21 +143,10 @@ for.cond.cleanup:
 ;
 ; CHECK-LABEL: @reduction_not_guarded
 ;
+; CHECK:     vector.body:
 ; CHECK-NOT: @llvm.arm.mve.vctp
-; CHECK-NOT: @llvm.get.active.lane.mask.v8i1.i32
-;
-; CHECK: entry:
-; CHECK: %[[ELEMCOUNT:.*]] = add i32 %N, -1
-; CHECK: %broadcast.splatinsert1 = insertelement <8 x i32> undef, i32 %[[ELEMCOUNT]], i32 0
-; CHECK  %broadcast.splat2 = shufflevector <8 x i32> %broadcast.splatinsert1, <8 x i32> undef, <8 x i32> zeroinitializer
-;
-; CHECK: vector.body:
-; CHECK: %lane.mask.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0
-; CHECK: %lane.mask.splat = shufflevector <8 x i32> %lane.mask.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer
-; CHECK: %lane.mask.induction = add <8 x i32> %lane.mask.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK: %[[ICMP:.*]] = icmp ule <8 x i32> %lane.mask.induction, %broadcast.splat2
-; CHECK: call <8 x i16> @llvm.masked.load.v8i16.p0v8i16({{.*}}, <8 x i1> %[[ICMP]], <8 x i16> undef)
-; CHECK: ret
+; CHECK:     @llvm.get.active.lane.mask.v8i1.i32
+; CHECK:     ret
 ;
 define i16 @reduction_not_guarded(i16* nocapture readonly %A, i16 %B, i32 %N) local_unnamed_addr {
 entry:
@@ -213,20 +202,9 @@ middle.block:                                     ; preds = %vector.body
 ;
 ; CHECK-LABEL: @Correlation
 ;
-; CHECK: entry:
-; CHECK: for.body.lr.ph:          ; preds = %entry
-; CHECK: for.body:                ; preds = %for.end, %for.body.lr.ph
-; CHECK: vector.ph:               ; preds = %for.body
-; CHECK:   %trip.count.minus.1 = add i32 %8, -1
-; CHECK:   call void @llvm.set.loop.iterations.i32(i32 %7)
-; CHECK:   %insert.btc = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
-; CHECK:   %splat.btc = shufflevector <4 x i32> %insert.btc, <4 x i32> undef, <4 x i32> zeroinitializer
-; CHECK:   br label %vector.body
 ; CHECK: vector.body:
-; CHECK-NOT:   @llvm.arm.mve.vctp
-; CHECK:   %[[ICMP:.*]] = icmp ule <4 x i32> %lane.mask.induction, %splat.btc
-; CHECK:   call <4 x i16> @llvm.masked.load.v4i16.p0v4i16({{.*}}, <4 x i1> %[[ICMP]],{{.*}} 
-;
+; CHECK-NOT: @llvm.arm.mve.vctp
+; CHECK:     %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
 ;
 ; FORCE-LABEL: @Correlation
 ; FORCE: vector.ph:                                        ; preds = %for.body
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/varying-outer-2d-reduction.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/varying-outer-2d-reduction.ll
new file mode 100644
index 0000000000000..f1242db364851
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/varying-outer-2d-reduction.ll
@@ -0,0 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -disable-mve-tail-predication=false %s -o - | FileCheck %s
+
+define dso_local void @varying_outer_2d_reduction(i16* nocapture readonly %Input, i16* nocapture %Output, i16 signext %Size, i16 signext %N, i16 signext %Scale) local_unnamed_addr {
+; CHECK-LABEL: varying_outer_2d_reduction:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    cmp r3, #1
+; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    blt .LBB0_8
+; CHECK-NEXT:  @ %bb.1: @ %for.body.lr.ph
+; CHECK-NEXT:    ldr r0, [sp, #44]
+; CHECK-NEXT:    adr r7, .LCPI0_0
+; CHECK-NEXT:    ldr.w r10, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    add.w r9, r2, #3
+; CHECK-NEXT:    vldrw.u32 q0, [r7]
+; CHECK-NEXT:    mov.w r11, #0
+; CHECK-NEXT:    uxth r0, r0
+; CHECK-NEXT:    rsbs r5, r0, #0
+; CHECK-NEXT:    str.w r9, [sp] @ 4-byte Spill
+; CHECK-NEXT:    b .LBB0_4
+; CHECK-NEXT:  .LBB0_2: @ in Loop: Header=BB0_4 Depth=1
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:  .LBB0_3: @ %for.end
+; CHECK-NEXT:    @ in Loop: Header=BB0_4 Depth=1
+; CHECK-NEXT:    lsrs r0, r0, #16
+; CHECK-NEXT:    sub.w r9, r9, #1
+; CHECK-NEXT:    strh.w r0, [r1, r11, lsl #1]
+; CHECK-NEXT:    add.w r11, r11, #1
+; CHECK-NEXT:    add.w r10, r10, #2
+; CHECK-NEXT:    cmp r11, r3
+; CHECK-NEXT:    beq .LBB0_8
+; CHECK-NEXT:  .LBB0_4: @ %for.body
+; CHECK-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-NEXT:    @ Child Loop BB0_6 Depth 2
+; CHECK-NEXT:    cmp r2, r11
+; CHECK-NEXT:    ble .LBB0_2
+; CHECK-NEXT:  @ %bb.5: @ %vector.ph
+; CHECK-NEXT:    @ in Loop: Header=BB0_4 Depth=1
+; CHECK-NEXT:    bic r7, r9, #3
+; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    subs r7, #4
+; CHECK-NEXT:    sub.w r0, r2, r11
+; CHECK-NEXT:    vmov.i32 q2, #0x0
+; CHECK-NEXT:    add.w r8, r6, r7, lsr #2
+; CHECK-NEXT:    ldr r7, [sp] @ 4-byte Reload
+; CHECK-NEXT:    sub.w r4, r7, r11
+; CHECK-NEXT:    movs r7, #0
+; CHECK-NEXT:    bic r4, r4, #3
+; CHECK-NEXT:    subs r4, #4
+; CHECK-NEXT:    add.w r4, r6, r4, lsr #2
+; CHECK-NEXT:    subs r6, r0, #1
+; CHECK-NEXT:    dls lr, r4
+; CHECK-NEXT:    mov r4, r10
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:  .LBB0_6: @ %vector.body
+; CHECK-NEXT:    @ Parent Loop BB0_4 Depth=1
+; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    vmov q1, q2
+; CHECK-NEXT:    vadd.i32 q2, q0, r7
+; CHECK-NEXT:    vdup.32 q3, r7
+; CHECK-NEXT:    mov lr, r8
+; CHECK-NEXT:    vcmp.u32 hi, q3, q2
+; CHECK-NEXT:    vdup.32 q3, r6
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    sub.w r8, r8, #1
+; CHECK-NEXT:    vpsttt
+; CHECK-NEXT:    vcmpt.u32 cs, q3, q2
+; CHECK-NEXT:    vldrht.s32 q2, [r0], #8
+; CHECK-NEXT:    vldrht.s32 q3, [r4], #8
+; CHECK-NEXT:    adds r7, #4
+; CHECK-NEXT:    vmul.i32 q2, q3, q2
+; CHECK-NEXT:    vshl.s32 q2, r5
+; CHECK-NEXT:    vadd.i32 q2, q2, q1
+; CHECK-NEXT:    le lr, .LBB0_6
+; CHECK-NEXT:  @ %bb.7: @ %middle.block
+; CHECK-NEXT:    @ in Loop: Header=BB0_4 Depth=1
+; CHECK-NEXT:    vpsel q1, q2, q1
+; CHECK-NEXT:    vaddv.u32 r0, q1
+; CHECK-NEXT:    b .LBB0_3
+; CHECK-NEXT:  .LBB0_8: @ %for.end17
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.9:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+entry:
+  %conv = sext i16 %N to i32
+  %cmp36 = icmp sgt i16 %N, 0
+  br i1 %cmp36, label %for.body.lr.ph, label %for.end17
+
+for.body.lr.ph:                                   ; preds = %entry
+  %conv2 = sext i16 %Size to i32
+  %conv1032 = zext i16 %Scale to i32
+  %i = add i32 %conv2, 3
+  br label %for.body
+
+for.body:                                         ; preds = %for.end, %for.body.lr.ph
+  %lsr.iv51 = phi i32 [ %lsr.iv.next, %for.end ], [ %i, %for.body.lr.ph ]
+  %lsr.iv46 = phi i16* [ %scevgep47, %for.end ], [ %Input, %for.body.lr.ph ]
+  %i.037 = phi i32 [ 0, %for.body.lr.ph ], [ %inc16, %for.end ]
+  %i1 = mul nsw i32 %i.037, -1
+  %i2 = add i32 %i, %i1
+  %i3 = lshr i32 %i2, 2
+  %i4 = shl nuw i32 %i3, 2
+  %i5 = add i32 %i4, -4
+  %i6 = lshr i32 %i5, 2
+  %i7 = add nuw nsw i32 %i6, 1
+  %i8 = sub i32 %conv2, %i.037
+  %cmp433 = icmp slt i32 %i.037, %conv2
+  br i1 %cmp433, label %vector.ph, label %for.end
+
+vector.ph:                                        ; preds = %for.body
+  %trip.count.minus.1 = add i32 %i8, -1
+  call void @llvm.set.loop.iterations.i32(i32 %i7)
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %lsr.iv48 = phi i16* [ %scevgep49, %vector.body ], [ %lsr.iv46, %vector.ph ]
+  %lsr.iv = phi i16* [ %scevgep, %vector.body ], [ %Input, %vector.ph ]
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %i16, %vector.body ]
+  %i9 = phi i32 [ %i7, %vector.ph ], [ %i17, %vector.body ]
+  %lsr.iv4850 = bitcast i16* %lsr.iv48 to <4 x i16>*
+  %lsr.iv45 = bitcast i16* %lsr.iv to <4 x i16>*
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %wide.masked.load = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %lsr.iv45, i32 2, <4 x i1> %active.lane.mask, <4 x i16> undef)
+  %i10 = sext <4 x i16> %wide.masked.load to <4 x i32>
+  %wide.masked.load42 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %lsr.iv4850, i32 2, <4 x i1> %active.lane.mask, <4 x i16> undef)
+  %i11 = sext <4 x i16> %wide.masked.load42 to <4 x i32>
+  %i12 = mul nsw <4 x i32> %i11, %i10
+  %i13 = insertelement <4 x i32> undef, i32 %conv1032, i32 0
+  %i14 = shufflevector <4 x i32> %i13, <4 x i32> undef, <4 x i32> zeroinitializer
+  %i15 = ashr <4 x i32> %i12, %i14
+  %i16 = add <4 x i32> %i15, %vec.phi
+  %index.next = add i32 %index, 4
+  %scevgep = getelementptr i16, i16* %lsr.iv, i32 4
+  %scevgep49 = getelementptr i16, i16* %lsr.iv48, i32 4
+  %i17 = call i32 @llvm.loop.decrement.reg.i32(i32 %i9, i32 1)
+  %i18 = icmp ne i32 %i17, 0
+  br i1 %i18, label %vector.body, label %middle.block
+
+middle.block:                                     ; preds = %vector.body
+  %i19 = select <4 x i1> %active.lane.mask, <4 x i32> %i16, <4 x i32> %vec.phi
+  %i20 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %i19)
+  br label %for.end
+
+for.end:                                          ; preds = %middle.block, %for.body
+  %Sum.0.lcssa = phi i32 [ 0, %for.body ], [ %i20, %middle.block ]
+  %i21 = lshr i32 %Sum.0.lcssa, 16
+  %conv13 = trunc i32 %i21 to i16
+  %arrayidx14 = getelementptr inbounds i16, i16* %Output, i32 %i.037
+  store i16 %conv13, i16* %arrayidx14, align 2
+  %inc16 = add nuw nsw i32 %i.037, 1
+  %scevgep47 = getelementptr i16, i16* %lsr.iv46, i32 1
+  %lsr.iv.next = add i32 %lsr.iv51, -1
+  %exitcond39 = icmp eq i32 %inc16, %conv
+  br i1 %exitcond39, label %for.end17, label %for.body
+
+for.end17:                                        ; preds = %for.end, %entry
+  ret void
+}
+
+declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)
+declare <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>*, i32 immarg, <4 x i1>, <4 x i16>)
+declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
+declare i32 @llvm.loop.decrement.reg.i32(i32, i32)
+declare void @llvm.set.loop.iterations.i32(i32)
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-arith-codegen.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-arith-codegen.ll
index d2e9c584c257d..26a570ac4c29b 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-arith-codegen.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-arith-codegen.ll
@@ -9,28 +9,19 @@ define dso_local i32 @mul_reduce_add(i32* noalias nocapture readonly %a, i32* no
 ; CHECK-NEXT:    moveq r0, #0
 ; CHECK-NEXT:    bxeq lr
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    adds r3, r2, #3
-; CHECK-NEXT:    vmov.i32 q0, #0x0
-; CHECK-NEXT:    bic r3, r3, #3
-; CHECK-NEXT:    sub.w r12, r3, #4
-; CHECK-NEXT:    movs r3, #1
-; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    vmov.i32 q1, #0x0
 ; CHECK-NEXT:    movs r3, #0
-; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    dlstp.32 lr, r2
 ; CHECK-NEXT:  .LBB0_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vctp.32 r2
-; CHECK-NEXT:    vmov q1, q0
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vldrwt.u32 q0, [r0], #16
-; CHECK-NEXT:    vldrwt.u32 q2, [r1], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
 ; CHECK-NEXT:    adds r3, #4
 ; CHECK-NEXT:    vmul.i32 q0, q2, q0
-; CHECK-NEXT:    subs r2, #4
-; CHECK-NEXT:    vadd.i32 q0, q0, q1
-; CHECK-NEXT:    le lr, .LBB0_1
+; CHECK-NEXT:    vadd.i32 q1, q0, q1
+; CHECK-NEXT:    letp lr, .LBB0_1
 ; CHECK-NEXT:  @ %bb.2: @ %middle.block
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    vaddv.u32 r0, q0
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
@@ -85,26 +76,17 @@ define dso_local i32 @mul_reduce_add_const(i32* noalias nocapture readonly %a, i
 ; CHECK-NEXT:    moveq r0, #0
 ; CHECK-NEXT:    bxeq lr
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    adds r1, r2, #3
-; CHECK-NEXT:    movs r3, #1
-; CHECK-NEXT:    bic r1, r1, #3
-; CHECK-NEXT:    vmov.i32 q0, #0x0
-; CHECK-NEXT:    subs r1, #4
-; CHECK-NEXT:    add.w lr, r3, r1, lsr #2
+; CHECK-NEXT:    vmov.i32 q1, #0x0
 ; CHECK-NEXT:    movs r1, #0
-; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    dlstp.32 lr, r2
 ; CHECK-NEXT:  .LBB1_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vctp.32 r2
-; CHECK-NEXT:    vmov q1, q0
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrwt.u32 q0, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    adds r1, #4
-; CHECK-NEXT:    subs r2, #4
-; CHECK-NEXT:    vadd.i32 q0, q0, q1
-; CHECK-NEXT:    le lr, .LBB1_1
+; CHECK-NEXT:    vadd.i32 q1, q0, q1
+; CHECK-NEXT:    letp lr, .LBB1_1
 ; CHECK-NEXT:  @ %bb.2: @ %middle.block
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    vaddv.u32 r0, q0
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
@@ -155,26 +137,17 @@ define dso_local i32 @add_reduce_add_const(i32* noalias nocapture readonly %a, i
 ; CHECK-NEXT:    moveq r0, #0
 ; CHECK-NEXT:    bxeq lr
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    adds r1, r2, #3
-; CHECK-NEXT:    movs r3, #1
-; CHECK-NEXT:    bic r1, r1, #3
-; CHECK-NEXT:    vmov.i32 q0, #0x0
-; CHECK-NEXT:    subs r1, #4
-; CHECK-NEXT:    add.w lr, r3, r1, lsr #2
+; CHECK-NEXT:    vmov.i32 q1, #0x0
 ; CHECK-NEXT:    movs r1, #0
-; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    dlstp.32 lr, r2
 ; CHECK-NEXT:  .LBB2_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vctp.32 r2
-; CHECK-NEXT:    vmov q1, q0
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrwt.u32 q0, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    adds r1, #4
-; CHECK-NEXT:    subs r2, #4
-; CHECK-NEXT:    vadd.i32 q0, q0, q1
-; CHECK-NEXT:    le lr, .LBB2_1
+; CHECK-NEXT:    vadd.i32 q1, q0, q1
+; CHECK-NEXT:    letp lr, .LBB2_1
 ; CHECK-NEXT:  @ %bb.2: @ %middle.block
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    vaddv.u32 r0, q0
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-reduce-mve-tail.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-reduce-mve-tail.ll
index 1d766f378fc57..aaeae75e072f7 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-reduce-mve-tail.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-reduce-mve-tail.ll
@@ -15,8 +15,7 @@
 ; CHECK: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]],
 
 ; CHECK: middle.block:
-; CHECK: [[VCTP_CLONE:%[^ ]+]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[REMAT_ITER:%.*]])
-; CHECK: [[VPSEL:%[^ ]+]] = select <4 x i1> [[VCTP_CLONE]],
+; CHECK: [[VPSEL:%[^ ]+]] = select <4 x i1> [[VCTP]],
 ; CHECK: call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[VPSEL]])
 
 define i32 @vec_mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32 %N) {
@@ -64,16 +63,12 @@ vector.body:                                      ; preds = %vector.body, %vecto
   br i1 %11, label %vector.body, label %middle.block
 
 middle.block:                                     ; preds = %vector.body
-; TODO: check that the intrinsic is also emitted here by the loop vectoriser
-;  %12 = icmp ule <4 x i32> %induction, %broadcast.splat12
-  %12 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
-
-  %13 = select <4 x i1> %12, <4 x i32> %9, <4 x i32> %vec.phi
-  %14 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %13)
+  %12 = select <4 x i1> %7, <4 x i32> %9, <4 x i32> %vec.phi
+  %13 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %12)
   br label %for.cond.cleanup
 
 for.cond.cleanup:                                 ; preds = %middle.block, %entry
-  %res.0.lcssa = phi i32 [ 0, %entry ], [ %14, %middle.block ]
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %13, %middle.block ]
   ret i32 %res.0.lcssa
 }
 
diff --git a/llvm/test/CodeGen/Thumb2/active_lane_mask.ll b/llvm/test/CodeGen/Thumb2/active_lane_mask.ll
new file mode 100644
index 0000000000000..e9dfccd320dae
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/active_lane_mask.ll
@@ -0,0 +1,338 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve %s -o - | FileCheck %s
+
+define <4 x i32> @v4i32(i32 %index, i32 %BTC, <4 x i32> %V1, <4 x i32> %V2) {
+; CHECK-LABEL: v4i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    adr.w r12, .LCPI0_0
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vldrw.u32 q0, [r12]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    add r0, sp, #8
+; CHECK-NEXT:    vcmp.u32 hi, q1, q0
+; CHECK-NEXT:    vdup.32 q1, r1
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.u32 cs, q1, q0
+; CHECK-NEXT:    vmov d0, r2, r3
+; CHECK-NEXT:    vldr d1, [sp]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %BTC)
+  %select = select <4 x i1> %active.lane.mask, <4 x i32> %V1, <4 x i32> %V2
+  ret <4 x i32> %select
+}
+
+define <8 x i16> @v8i16(i32 %index, i32 %BTC, <8 x i16> %V1, <8 x i16> %V2) {
+; CHECK-LABEL: v8i16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    adr.w r12, .LCPI1_0
+; CHECK-NEXT:    vdup.32 q5, r1
+; CHECK-NEXT:    vldrw.u32 q0, [r12]
+; CHECK-NEXT:    vmov.i8 q1, #0x0
+; CHECK-NEXT:    vmov.i8 q2, #0xff
+; CHECK-NEXT:    vadd.i32 q3, q0, r0
+; CHECK-NEXT:    vcmp.u32 cs, q5, q3
+; CHECK-NEXT:    vpsel q4, q2, q1
+; CHECK-NEXT:    vmov r1, s16
+; CHECK-NEXT:    vmov.16 q0[0], r1
+; CHECK-NEXT:    vmov r1, s17
+; CHECK-NEXT:    vmov.16 q0[1], r1
+; CHECK-NEXT:    vmov r1, s18
+; CHECK-NEXT:    vmov.16 q0[2], r1
+; CHECK-NEXT:    vmov r1, s19
+; CHECK-NEXT:    vmov.16 q0[3], r1
+; CHECK-NEXT:    adr r1, .LCPI1_1
+; CHECK-NEXT:    vldrw.u32 q4, [r1]
+; CHECK-NEXT:    vadd.i32 q4, q4, r0
+; CHECK-NEXT:    vcmp.u32 cs, q5, q4
+; CHECK-NEXT:    vpsel q5, q2, q1
+; CHECK-NEXT:    vmov r1, s20
+; CHECK-NEXT:    vmov.16 q0[4], r1
+; CHECK-NEXT:    vmov r1, s21
+; CHECK-NEXT:    vmov.16 q0[5], r1
+; CHECK-NEXT:    vmov r1, s22
+; CHECK-NEXT:    vmov.16 q0[6], r1
+; CHECK-NEXT:    vmov r1, s23
+; CHECK-NEXT:    vdup.32 q5, r0
+; CHECK-NEXT:    vmov.16 q0[7], r1
+; CHECK-NEXT:    vcmp.u32 hi, q5, q3
+; CHECK-NEXT:    vpsel q6, q2, q1
+; CHECK-NEXT:    vcmp.u32 hi, q5, q4
+; CHECK-NEXT:    vmov r0, s24
+; CHECK-NEXT:    vpsel q1, q2, q1
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov r0, s25
+; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov r0, s26
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov r0, s27
+; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    add r0, sp, #56
+; CHECK-NEXT:    vcmp.i16 ne, q3, zr
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i16 ne, q0, zr
+; CHECK-NEXT:    vmov d0, r2, r3
+; CHECK-NEXT:    vldr d1, [sp, #48]
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI1_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+; CHECK-NEXT:  .LCPI1_1:
+; CHECK-NEXT:    .long 4 @ 0x4
+; CHECK-NEXT:    .long 5 @ 0x5
+; CHECK-NEXT:    .long 6 @ 0x6
+; CHECK-NEXT:    .long 7 @ 0x7
+  %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %BTC)
+  %select = select <8 x i1> %active.lane.mask, <8 x i16> %V1, <8 x i16> %V2
+  ret <8 x i16> %select
+}
+
+define <16 x i8> @v16i8(i32 %index, i32 %BTC, <16 x i8> %V1, <16 x i8> %V2) {
+; CHECK-LABEL: v16i8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    adr.w r12, .LCPI2_0
+; CHECK-NEXT:    vdup.32 q7, r1
+; CHECK-NEXT:    vldrw.u32 q0, [r12]
+; CHECK-NEXT:    vmov.i8 q5, #0x0
+; CHECK-NEXT:    vmov.i8 q4, #0xff
+; CHECK-NEXT:    vadd.i32 q1, q0, r0
+; CHECK-NEXT:    vcmp.u32 cs, q7, q1
+; CHECK-NEXT:    vpsel q0, q4, q5
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov.16 q2[0], r1
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    vmov.16 q2[1], r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov.16 q2[2], r1
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov.16 q2[3], r1
+; CHECK-NEXT:    adr r1, .LCPI2_1
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q3, q0, r0
+; CHECK-NEXT:    vcmp.u32 cs, q7, q3
+; CHECK-NEXT:    vpsel q0, q4, q5
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov.16 q2[4], r1
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    vmov.16 q2[5], r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov.16 q2[6], r1
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov.16 q2[7], r1
+; CHECK-NEXT:    vcmp.i16 ne, q2, zr
+; CHECK-NEXT:    vpsel q0, q4, q5
+; CHECK-NEXT:    vmov.u16 r1, q0[0]
+; CHECK-NEXT:    vmov.8 q2[0], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-NEXT:    vmov.8 q2[1], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    vmov.8 q2[2], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[3]
+; CHECK-NEXT:    vmov.8 q2[3], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[4]
+; CHECK-NEXT:    vmov.8 q2[4], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[5]
+; CHECK-NEXT:    vmov.8 q2[5], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
+; CHECK-NEXT:    vmov.8 q2[6], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    vmov.8 q2[7], r1
+; CHECK-NEXT:    adr r1, .LCPI2_2
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vcmp.u32 cs, q7, q0
+; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vpsel q6, q4, q5
+; CHECK-NEXT:    vmov r1, s24
+; CHECK-NEXT:    vmov.16 q0[0], r1
+; CHECK-NEXT:    vmov r1, s25
+; CHECK-NEXT:    vmov.16 q0[1], r1
+; CHECK-NEXT:    vmov r1, s26
+; CHECK-NEXT:    vmov.16 q0[2], r1
+; CHECK-NEXT:    vmov r1, s27
+; CHECK-NEXT:    vmov.16 q0[3], r1
+; CHECK-NEXT:    adr r1, .LCPI2_3
+; CHECK-NEXT:    vldrw.u32 q6, [r1]
+; CHECK-NEXT:    vadd.i32 q6, q6, r0
+; CHECK-NEXT:    vcmp.u32 cs, q7, q6
+; CHECK-NEXT:    vpsel q7, q4, q5
+; CHECK-NEXT:    vmov r1, s28
+; CHECK-NEXT:    vmov.16 q0[4], r1
+; CHECK-NEXT:    vmov r1, s29
+; CHECK-NEXT:    vmov.16 q0[5], r1
+; CHECK-NEXT:    vmov r1, s30
+; CHECK-NEXT:    vmov.16 q0[6], r1
+; CHECK-NEXT:    vmov r1, s31
+; CHECK-NEXT:    vmov.16 q0[7], r1
+; CHECK-NEXT:    vdup.32 q7, r0
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q4, q5
+; CHECK-NEXT:    vcmp.u32 hi, q7, q1
+; CHECK-NEXT:    vmov.u16 r1, q0[0]
+; CHECK-NEXT:    vpsel q1, q4, q5
+; CHECK-NEXT:    vmov.8 q2[8], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-NEXT:    vmov.8 q2[9], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    vmov.8 q2[10], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[3]
+; CHECK-NEXT:    vmov.8 q2[11], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[4]
+; CHECK-NEXT:    vmov.8 q2[12], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[5]
+; CHECK-NEXT:    vmov.8 q2[13], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.8 q2[14], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    vmov.16 q0[0], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vcmp.u32 hi, q7, q3
+; CHECK-NEXT:    vmov.16 q0[1], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vpsel q1, q4, q5
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.8 q2[15], r1
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q4, q5
+; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vmov.8 q3[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[1]
+; CHECK-NEXT:    vmov.8 q3[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-NEXT:    vmov.8 q3[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[3]
+; CHECK-NEXT:    vmov.8 q3[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[4]
+; CHECK-NEXT:    vmov.8 q3[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[5]
+; CHECK-NEXT:    vmov.8 q3[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
+; CHECK-NEXT:    vmov.8 q3[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[7]
+; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vmov.8 q3[7], r0
+; CHECK-NEXT:    vcmp.u32 hi, q7, q0
+; CHECK-NEXT:    vpsel q1, q4, q5
+; CHECK-NEXT:    vcmp.u32 hi, q7, q6
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q0[0], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.16 q0[1], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vpsel q1, q4, q5
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q4, q5
+; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vmov.8 q3[8], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[1]
+; CHECK-NEXT:    vmov.8 q3[9], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-NEXT:    vmov.8 q3[10], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[3]
+; CHECK-NEXT:    vmov.8 q3[11], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[4]
+; CHECK-NEXT:    vmov.8 q3[12], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[5]
+; CHECK-NEXT:    vmov.8 q3[13], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
+; CHECK-NEXT:    vmov.8 q3[14], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[7]
+; CHECK-NEXT:    vmov.8 q3[15], r0
+; CHECK-NEXT:    vmov d0, r2, r3
+; CHECK-NEXT:    add r0, sp, #88
+; CHECK-NEXT:    vcmp.i8 ne, q3, zr
+; CHECK-NEXT:    vldr d1, [sp, #80]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i8 ne, q2, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI2_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+; CHECK-NEXT:  .LCPI2_1:
+; CHECK-NEXT:    .long 4 @ 0x4
+; CHECK-NEXT:    .long 5 @ 0x5
+; CHECK-NEXT:    .long 6 @ 0x6
+; CHECK-NEXT:    .long 7 @ 0x7
+; CHECK-NEXT:  .LCPI2_2:
+; CHECK-NEXT:    .long 8 @ 0x8
+; CHECK-NEXT:    .long 9 @ 0x9
+; CHECK-NEXT:    .long 10 @ 0xa
+; CHECK-NEXT:    .long 11 @ 0xb
+; CHECK-NEXT:  .LCPI2_3:
+; CHECK-NEXT:    .long 12 @ 0xc
+; CHECK-NEXT:    .long 13 @ 0xd
+; CHECK-NEXT:    .long 14 @ 0xe
+; CHECK-NEXT:    .long 15 @ 0xf
+  %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %BTC)
+  %select = select <16 x i1> %active.lane.mask, <16 x i8> %V1, <16 x i8> %V2
+  ret <16 x i8> %select
+}
+
+declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)
+declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32)
+declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32, i32)
diff --git a/llvm/test/CodeGen/Thumb2/constant-hoisting.ll b/llvm/test/CodeGen/Thumb2/constant-hoisting.ll
index a106900dc3e99..5c8f934ce61d3 100644
--- a/llvm/test/CodeGen/Thumb2/constant-hoisting.ll
+++ b/llvm/test/CodeGen/Thumb2/constant-hoisting.ll
@@ -37,25 +37,26 @@ define i32 @test_values(i32 %a, i32 %b) minsize optsize {
 ; CHECK-V7M:         mov r2, r0
 ; CHECK-V7M-NEXT:    ldr r0, .LCPI0_0
 ; CHECK-V7M-NEXT:    cmp r2, #50
-; CHECK-V7M-NEXT:    beq .LBB0_3
+; CHECK-V7M-NEXT:    beq .LBB0_5
 ; CHECK-V7M-NEXT:    cmp r2, #1
-; CHECK-V7M-NEXT:    ittt eq
-; CHECK-V7M-NEXT:    addeq r0, r1
-; CHECK-V7M-NEXT:    addeq r0, #1
-; CHECK-V7M-NEXT:    bxeq lr
+; CHECK-V7M-NEXT:    beq .LBB0_7
 ; CHECK-V7M-NEXT:    cmp r2, #30
-; CHECK-V7M-NEXT:    ittt eq
-; CHECK-V7M-NEXT:    addeq r0, r1
-; CHECK-V7M-NEXT:    addeq r0, #2
-; CHECK-V7M-NEXT:    bxeq lr
-; CHECK-V7M-NEXT:    cbnz r2, .LBB0_4
-; CHECK-V7M-NEXT:  .LBB0_2:
+; CHECK-V7M-NEXT:    beq .LBB0_8
+; CHECK-V7M-NEXT:    cbnz r2, .LBB0_6
 ; CHECK-V7M-NEXT:    add r0, r1
 ; CHECK-V7M-NEXT:    bx lr
-; CHECK-V7M-NEXT:  .LBB0_3:
+; CHECK-V7M-NEXT:  .LBB0_5:
 ; CHECK-V7M-NEXT:    add r0, r1
 ; CHECK-V7M-NEXT:    adds r0, #4
-; CHECK-V7M-NEXT:  .LBB0_4:
+; CHECK-V7M-NEXT:  .LBB0_6:
+; CHECK-V7M-NEXT:    bx lr
+; CHECK-V7M-NEXT:  .LBB0_7:
+; CHECK-V7M-NEXT:    add r0, r1
+; CHECK-V7M-NEXT:    adds r0, #1
+; CHECK-V7M-NEXT:    bx lr
+; CHECK-V7M-NEXT:  .LBB0_8:
+; CHECK-V7M-NEXT:    add r0, r1
+; CHECK-V7M-NEXT:    adds r0, #2
 ; CHECK-V7M-NEXT:    bx lr
 ; CHECK-V7M-NEXT:    .p2align 2
 ; CHECK-V7M-NEXT:  .LCPI0_0:
diff --git a/llvm/test/CodeGen/Thumb2/mve-div-expand.ll b/llvm/test/CodeGen/Thumb2/mve-div-expand.ll
index d72495b215bce..ef804cb3bc542 100644
--- a/llvm/test/CodeGen/Thumb2/mve-div-expand.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-div-expand.ll
@@ -968,95 +968,72 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @frem_f16(<8 x half> %in1, <8 x half> %in2) {
 ; CHECK-LABEL: frem_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vmov q5, q1
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov q5, q0
+; CHECK-NEXT:    vmov q4, q1
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s20
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl fmodf
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmovx.f16 s2, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmovx.f16 s0, s20
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s20
+; CHECK-NEXT:    vmov s24, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s16
 ; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.16 q6[0], r4
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
+; CHECK-NEXT:    vcvtb.f16.f32 s24, s24
+; CHECK-NEXT:    vcvtt.f16.f32 s24, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s21
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vmov.16 q6[1], r0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s21
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmovx.f16 s2, s17
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vcvtb.f16.f32 s25, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s21
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s21
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q6[2], r0
-; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s17
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s25, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s22
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vmov.16 q6[3], r0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s22
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmovx.f16 s2, s18
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vcvtb.f16.f32 s26, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s22
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s22
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q6[4], r0
-; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s18
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s26, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s23
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vmov.16 q6[5], r0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s23
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmovx.f16 s2, s19
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vcvtb.f16.f32 s27, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s23
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s23
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q6[6], r0
-; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s19
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q6[7], r0
+; CHECK-NEXT:    vcvtt.f16.f32 s27, s0
 ; CHECK-NEXT:    vmov q0, q6
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %out = frem <8 x half> %in1, %in2
   ret <8 x half> %out
diff --git a/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll b/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
index 62d2f178561b3..42e8cc91ede8e 100644
--- a/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
@@ -1059,10 +1059,9 @@ define void @fir(%struct.arm_fir_instance_f32* nocapture readonly %S, half* noca
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
 ; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEXT:    .pad #28
-; CHECK-NEXT:    sub sp, #28
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
 ; CHECK-NEXT:    cmp r3, #8
-; CHECK-NEXT:    str r1, [sp, #24] @ 4-byte Spill
 ; CHECK-NEXT:    blo.w .LBB16_12
 ; CHECK-NEXT:  @ %bb.1: @ %if.then
 ; CHECK-NEXT:    movs r7, #0
@@ -1070,73 +1069,73 @@ define void @fir(%struct.arm_fir_instance_f32* nocapture readonly %S, half* noca
 ; CHECK-NEXT:    beq.w .LBB16_12
 ; CHECK-NEXT:  @ %bb.2: @ %while.body.lr.ph
 ; CHECK-NEXT:    ldrh r4, [r0]
-; CHECK-NEXT:    movs r1, #1
+; CHECK-NEXT:    lsr.w r10, r3, #2
 ; CHECK-NEXT:    ldrd r5, r12, [r0, #4]
-; CHECK-NEXT:    lsr.w r11, r3, #2
+; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    sub.w r0, r4, #8
-; CHECK-NEXT:    rsbs r3, r4, #0
+; CHECK-NEXT:    and r9, r0, #7
 ; CHECK-NEXT:    add.w r7, r0, r0, lsr #29
-; CHECK-NEXT:    and r0, r0, #7
 ; CHECK-NEXT:    asrs r6, r7, #3
 ; CHECK-NEXT:    cmp r6, #1
 ; CHECK-NEXT:    it gt
-; CHECK-NEXT:    asrgt r1, r7, #3
+; CHECK-NEXT:    asrgt r3, r7, #3
 ; CHECK-NEXT:    add.w r7, r5, r4, lsl #1
-; CHECK-NEXT:    str r1, [sp] @ 4-byte Spill
-; CHECK-NEXT:    subs r1, r7, #2
-; CHECK-NEXT:    str r3, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    add.w r3, r12, #16
-; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    adds r0, #1
-; CHECK-NEXT:    str r4, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
+; CHECK-NEXT:    subs r3, r7, #2
+; CHECK-NEXT:    str r3, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    rsbs r3, r4, #0
 ; CHECK-NEXT:    str r3, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    add.w r3, r12, #16
+; CHECK-NEXT:    str r4, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    b .LBB16_4
 ; CHECK-NEXT:  .LBB16_3: @ %while.end
 ; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    subs.w r11, r11, #1
+; CHECK-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    subs.w r10, r10, #1
+; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    vstrb.8 q0, [r2], #8
-; CHECK-NEXT:    add.w r0, r7, r0, lsl #1
+; CHECK-NEXT:    add.w r0, r5, r0, lsl #1
 ; CHECK-NEXT:    add.w r5, r0, #8
 ; CHECK-NEXT:    beq.w .LBB16_12
 ; CHECK-NEXT:  .LBB16_4: @ %while.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB16_6 Depth 2
 ; CHECK-NEXT:    @ Child Loop BB16_10 Depth 2
-; CHECK-NEXT:    ldr r0, [sp, #24] @ 4-byte Reload
+; CHECK-NEXT:    vldrw.u32 q0, [r1], #8
 ; CHECK-NEXT:    ldrh.w lr, [r12, #14]
-; CHECK-NEXT:    vldrw.u32 q0, [r0], #8
-; CHECK-NEXT:    ldrh.w r10, [r12, #12]
-; CHECK-NEXT:    ldrh.w r7, [r12, #10]
-; CHECK-NEXT:    ldrh.w r4, [r12, #8]
+; CHECK-NEXT:    ldrh.w r0, [r12, #12]
+; CHECK-NEXT:    str r1, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    ldrh.w r4, [r12, #10]
+; CHECK-NEXT:    ldrh.w r7, [r12, #8]
 ; CHECK-NEXT:    ldrh.w r3, [r12, #6]
 ; CHECK-NEXT:    ldrh.w r6, [r12, #4]
-; CHECK-NEXT:    ldrh.w r8, [r12, #2]
-; CHECK-NEXT:    ldrh.w r9, [r12]
+; CHECK-NEXT:    ldrh.w r11, [r12, #2]
+; CHECK-NEXT:    ldrh.w r8, [r12]
 ; CHECK-NEXT:    vstrb.8 q0, [r1], #8
 ; CHECK-NEXT:    vldrw.u32 q0, [r5]
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    adds r0, r5, #2
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    vmul.f16 q0, q0, r9
-; CHECK-NEXT:    adds r0, r5, #6
-; CHECK-NEXT:    vfma.f16 q0, q1, r8
+; CHECK-NEXT:    str r1, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    adds r1, r5, #2
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vmul.f16 q0, q0, r8
+; CHECK-NEXT:    adds r1, r5, #6
+; CHECK-NEXT:    vfma.f16 q0, q1, r11
 ; CHECK-NEXT:    vldrw.u32 q1, [r5, #4]
 ; CHECK-NEXT:    vfma.f16 q0, q1, r6
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    add.w r0, r5, #10
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    add.w r1, r5, #10
 ; CHECK-NEXT:    vfma.f16 q0, q1, r3
 ; CHECK-NEXT:    vldrw.u32 q1, [r5, #8]
-; CHECK-NEXT:    vfma.f16 q0, q1, r4
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    add.w r0, r5, #14
 ; CHECK-NEXT:    vfma.f16 q0, q1, r7
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vfma.f16 q0, q1, r4
 ; CHECK-NEXT:    vldrw.u32 q1, [r5, #12]
-; CHECK-NEXT:    add.w r7, r5, #16
-; CHECK-NEXT:    vfma.f16 q0, q1, r10
+; CHECK-NEXT:    vfma.f16 q0, q1, r0
+; CHECK-NEXT:    add.w r0, r5, #14
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    adds r5, #16
 ; CHECK-NEXT:    vfma.f16 q0, q1, lr
 ; CHECK-NEXT:    cmp r0, #16
 ; CHECK-NEXT:    blo .LBB16_7
@@ -1144,69 +1143,68 @@ define void @fir(%struct.arm_fir_instance_f32* nocapture readonly %S, half* noca
 ; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
 ; CHECK-NEXT:    ldr.w lr, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    dls lr, lr
-; CHECK-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:  .LBB16_6: @ %for.body
 ; CHECK-NEXT:    @ Parent Loop BB16_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    ldrh r0, [r6]
-; CHECK-NEXT:    vldrw.u32 q1, [r7]
-; CHECK-NEXT:    adds r3, r7, #2
+; CHECK-NEXT:    vldrw.u32 q1, [r5]
+; CHECK-NEXT:    adds r1, r5, #2
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
-; CHECK-NEXT:    vldrw.u32 q1, [r3]
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-NEXT:    ldrh r0, [r6, #2]
-; CHECK-NEXT:    adds r3, r7, #6
+; CHECK-NEXT:    adds r1, r5, #6
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
 ; CHECK-NEXT:    ldrh r0, [r6, #4]
-; CHECK-NEXT:    vldrw.u32 q1, [r7, #4]
+; CHECK-NEXT:    vldrw.u32 q1, [r5, #4]
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
-; CHECK-NEXT:    vldrw.u32 q1, [r3]
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-NEXT:    ldrh r0, [r6, #6]
-; CHECK-NEXT:    add.w r3, r7, #10
+; CHECK-NEXT:    add.w r1, r5, #10
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
 ; CHECK-NEXT:    ldrh r0, [r6, #8]
-; CHECK-NEXT:    vldrw.u32 q1, [r7, #8]
+; CHECK-NEXT:    vldrw.u32 q1, [r5, #8]
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
-; CHECK-NEXT:    vldrw.u32 q1, [r3]
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-NEXT:    ldrh r0, [r6, #10]
-; CHECK-NEXT:    ldrh r3, [r6, #14]
+; CHECK-NEXT:    ldrh r1, [r6, #14]
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
 ; CHECK-NEXT:    ldrh r0, [r6, #12]
-; CHECK-NEXT:    vldrw.u32 q1, [r7, #12]
+; CHECK-NEXT:    vldrw.u32 q1, [r5, #12]
 ; CHECK-NEXT:    adds r6, #16
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
-; CHECK-NEXT:    add.w r0, r7, #14
+; CHECK-NEXT:    add.w r0, r5, #14
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    adds r7, #16
-; CHECK-NEXT:    vfma.f16 q0, q1, r3
+; CHECK-NEXT:    adds r5, #16
+; CHECK-NEXT:    vfma.f16 q0, q1, r1
 ; CHECK-NEXT:    le lr, .LBB16_6
 ; CHECK-NEXT:    b .LBB16_8
 ; CHECK-NEXT:  .LBB16_7: @ in Loop: Header=BB16_4 Depth=1
-; CHECK-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:  .LBB16_8: @ %for.end
 ; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
-; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cmp.w r9, #0
 ; CHECK-NEXT:    beq.w .LBB16_3
-; CHECK-NEXT:  @ %bb.9: @ %while.body76.preheader
+; CHECK-NEXT:    b .LBB16_9
+; CHECK-NEXT:  .LBB16_9: @ %while.body76.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r5, r7
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov lr, r9
 ; CHECK-NEXT:  .LBB16_10: @ %while.body76
 ; CHECK-NEXT:    @ Parent Loop BB16_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    ldrh r3, [r6], #2
-; CHECK-NEXT:    vldrh.u16 q1, [r5], #2
-; CHECK-NEXT:    subs r0, #1
-; CHECK-NEXT:    vfma.f16 q0, q1, r3
-; CHECK-NEXT:    cmp r0, #1
-; CHECK-NEXT:    bgt .LBB16_10
-; CHECK-NEXT:  @ %bb.11: @ %while.end.loopexit
+; CHECK-NEXT:    ldrh r1, [r6], #2
+; CHECK-NEXT:    vldrh.u16 q1, [r0], #2
+; CHECK-NEXT:    subs.w lr, lr, #1
+; CHECK-NEXT:    vfma.f16 q0, q1, r1
+; CHECK-NEXT:    bne .LBB16_10
+; CHECK-NEXT:    b .LBB16_11
+; CHECK-NEXT:  .LBB16_11: @ %while.end.loopexit
 ; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
-; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    add.w r7, r7, r0, lsl #1
+; CHECK-NEXT:    add.w r5, r5, r9, lsl #1
 ; CHECK-NEXT:    b .LBB16_3
 ; CHECK-NEXT:  .LBB16_12: @ %if.end
-; CHECK-NEXT:    add sp, #28
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 entry:
   %pState1 = getelementptr inbounds %struct.arm_fir_instance_f32, %struct.arm_fir_instance_f32* %S, i32 0, i32 1
diff --git a/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll b/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
index caf9c7cdc59b8..bf6b57f223a46 100644
--- a/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
@@ -1040,121 +1040,121 @@ define void @fir(%struct.arm_fir_instance_f32* nocapture readonly %S, float* noc
 ; CHECK-NEXT:    cmp.w r7, r3, lsr #2
 ; CHECK-NEXT:    beq.w .LBB16_12
 ; CHECK-NEXT:  @ %bb.2: @ %while.body.lr.ph
-; CHECK-NEXT:    ldrh r5, [r0]
-; CHECK-NEXT:    lsr.w r9, r3, #2
-; CHECK-NEXT:    ldrd r8, r12, [r0, #4]
+; CHECK-NEXT:    ldrh r4, [r0]
+; CHECK-NEXT:    lsr.w r10, r3, #2
+; CHECK-NEXT:    ldrd r5, r12, [r0, #4]
 ; CHECK-NEXT:    movs r3, #1
-; CHECK-NEXT:    sub.w r0, r5, #8
-; CHECK-NEXT:    add.w r7, r0, r0, lsr #29
-; CHECK-NEXT:    and r0, r0, #7
-; CHECK-NEXT:    asrs r6, r7, #3
+; CHECK-NEXT:    sub.w r7, r4, #8
+; CHECK-NEXT:    add.w r0, r7, r7, lsr #29
+; CHECK-NEXT:    asrs r6, r0, #3
 ; CHECK-NEXT:    cmp r6, #1
 ; CHECK-NEXT:    it gt
-; CHECK-NEXT:    asrgt r3, r7, #3
-; CHECK-NEXT:    add.w r7, r8, r5, lsl #2
-; CHECK-NEXT:    sub.w r11, r7, #4
-; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
-; CHECK-NEXT:    rsbs r3, r5, #0
+; CHECK-NEXT:    asrgt r3, r0, #3
+; CHECK-NEXT:    add.w r0, r5, r4, lsl #2
+; CHECK-NEXT:    sub.w r9, r0, #4
+; CHECK-NEXT:    rsbs r0, r4, #0
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    and r3, r7, #7
+; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT:    add.w r0, r12, #32
+; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; CHECK-NEXT:    str r3, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    add.w r3, r12, #32
-; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    adds r0, #1
-; CHECK-NEXT:    str r5, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    str r3, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    b .LBB16_4
 ; CHECK-NEXT:  .LBB16_3: @ %while.end
 ; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    subs.w r9, r9, #1
-; CHECK-NEXT:    ldrd r11, r1, [sp, #24] @ 8-byte Folded Reload
+; CHECK-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    subs.w r10, r10, #1
 ; CHECK-NEXT:    vstrb.8 q0, [r2], #16
-; CHECK-NEXT:    add.w r0, r8, r0, lsl #2
-; CHECK-NEXT:    add.w r8, r0, #16
-; CHECK-NEXT:    beq .LBB16_12
+; CHECK-NEXT:    add.w r0, r5, r0, lsl #2
+; CHECK-NEXT:    add.w r5, r0, #16
+; CHECK-NEXT:    beq.w .LBB16_12
 ; CHECK-NEXT:  .LBB16_4: @ %while.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB16_6 Depth 2
 ; CHECK-NEXT:    @ Child Loop BB16_10 Depth 2
 ; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
-; CHECK-NEXT:    ldrd r0, r7, [r12]
-; CHECK-NEXT:    ldrd r4, r6, [r12, #8]
-; CHECK-NEXT:    ldrd r5, r3, [r12, #16]
-; CHECK-NEXT:    ldrd lr, r10, [r12, #24]
-; CHECK-NEXT:    vstrb.8 q0, [r11], #16
-; CHECK-NEXT:    vldrw.u32 q0, [r8], #32
-; CHECK-NEXT:    vldrw.u32 q1, [r8, #-28]
-; CHECK-NEXT:    vmul.f32 q0, q0, r0
-; CHECK-NEXT:    vldrw.u32 q6, [r8, #-24]
-; CHECK-NEXT:    vldrw.u32 q4, [r8, #-20]
-; CHECK-NEXT:    vfma.f32 q0, q1, r7
-; CHECK-NEXT:    vldrw.u32 q5, [r8, #-16]
-; CHECK-NEXT:    vfma.f32 q0, q6, r4
-; CHECK-NEXT:    vldrw.u32 q2, [r8, #-12]
+; CHECK-NEXT:    ldrd r7, r4, [r12]
+; CHECK-NEXT:    ldrd r0, r6, [r12, #8]
+; CHECK-NEXT:    ldrd r3, lr, [r12, #16]
+; CHECK-NEXT:    ldrd r11, r8, [r12, #24]
+; CHECK-NEXT:    vstrb.8 q0, [r9], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r5], #32
+; CHECK-NEXT:    vldrw.u32 q1, [r5, #-28]
+; CHECK-NEXT:    vmul.f32 q0, q0, r7
+; CHECK-NEXT:    vldrw.u32 q6, [r5, #-24]
+; CHECK-NEXT:    vldrw.u32 q4, [r5, #-20]
+; CHECK-NEXT:    vfma.f32 q0, q1, r4
+; CHECK-NEXT:    vldrw.u32 q5, [r5, #-16]
+; CHECK-NEXT:    vfma.f32 q0, q6, r0
+; CHECK-NEXT:    vldrw.u32 q2, [r5, #-12]
 ; CHECK-NEXT:    vfma.f32 q0, q4, r6
-; CHECK-NEXT:    vldrw.u32 q3, [r8, #-8]
-; CHECK-NEXT:    vfma.f32 q0, q5, r5
-; CHECK-NEXT:    vldrw.u32 q1, [r8, #-4]
-; CHECK-NEXT:    vfma.f32 q0, q2, r3
-; CHECK-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    vfma.f32 q0, q3, lr
-; CHECK-NEXT:    strd r11, r1, [sp, #24] @ 8-byte Folded Spill
-; CHECK-NEXT:    vfma.f32 q0, q1, r10
+; CHECK-NEXT:    vldrw.u32 q3, [r5, #-8]
+; CHECK-NEXT:    vfma.f32 q0, q5, r3
+; CHECK-NEXT:    vldrw.u32 q1, [r5, #-4]
+; CHECK-NEXT:    vfma.f32 q0, q2, lr
+; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    vfma.f32 q0, q3, r11
+; CHECK-NEXT:    strd r9, r1, [sp, #24] @ 8-byte Folded Spill
+; CHECK-NEXT:    vfma.f32 q0, q1, r8
 ; CHECK-NEXT:    cmp r0, #16
 ; CHECK-NEXT:    blo .LBB16_7
 ; CHECK-NEXT:  @ %bb.5: @ %for.body.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
-; CHECK-NEXT:    ldr.w lr, [sp] @ 4-byte Reload
+; CHECK-NEXT:    ldr.w lr, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:  .LBB16_6: @ %for.body
 ; CHECK-NEXT:    @ Parent Loop BB16_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    ldm.w r7, {r0, r3, r4, r5, r6, r10, r11}
-; CHECK-NEXT:    vldrw.u32 q1, [r8], #32
-; CHECK-NEXT:    vldrw.u32 q6, [r8, #-24]
-; CHECK-NEXT:    vldrw.u32 q4, [r8, #-20]
+; CHECK-NEXT:    ldm.w r7, {r0, r3, r4, r6}
+; CHECK-NEXT:    vldrw.u32 q1, [r5], #32
+; CHECK-NEXT:    add.w r11, r7, #16
+; CHECK-NEXT:    vldrw.u32 q6, [r5, #-24]
+; CHECK-NEXT:    vldrw.u32 q4, [r5, #-20]
 ; CHECK-NEXT:    vfma.f32 q0, q1, r0
-; CHECK-NEXT:    vldrw.u32 q1, [r8, #-28]
-; CHECK-NEXT:    vldrw.u32 q5, [r8, #-16]
-; CHECK-NEXT:    vldrw.u32 q2, [r8, #-12]
+; CHECK-NEXT:    vldrw.u32 q1, [r5, #-28]
+; CHECK-NEXT:    ldm.w r11, {r1, r8, r11}
+; CHECK-NEXT:    vldrw.u32 q5, [r5, #-16]
 ; CHECK-NEXT:    vfma.f32 q0, q1, r3
-; CHECK-NEXT:    vldrw.u32 q3, [r8, #-8]
+; CHECK-NEXT:    vldrw.u32 q2, [r5, #-12]
 ; CHECK-NEXT:    vfma.f32 q0, q6, r4
-; CHECK-NEXT:    ldr r1, [r7, #28]
-; CHECK-NEXT:    vfma.f32 q0, q4, r5
-; CHECK-NEXT:    vldrw.u32 q1, [r8, #-4]
-; CHECK-NEXT:    vfma.f32 q0, q5, r6
+; CHECK-NEXT:    vldrw.u32 q3, [r5, #-8]
+; CHECK-NEXT:    vfma.f32 q0, q4, r6
+; CHECK-NEXT:    ldr.w r9, [r7, #28]
+; CHECK-NEXT:    vfma.f32 q0, q5, r1
+; CHECK-NEXT:    vldrw.u32 q1, [r5, #-4]
+; CHECK-NEXT:    vfma.f32 q0, q2, r8
 ; CHECK-NEXT:    adds r7, #32
-; CHECK-NEXT:    vfma.f32 q0, q2, r10
 ; CHECK-NEXT:    vfma.f32 q0, q3, r11
-; CHECK-NEXT:    vfma.f32 q0, q1, r1
+; CHECK-NEXT:    vfma.f32 q0, q1, r9
 ; CHECK-NEXT:    le lr, .LBB16_6
 ; CHECK-NEXT:    b .LBB16_8
 ; CHECK-NEXT:  .LBB16_7: @ in Loop: Header=BB16_4 Depth=1
 ; CHECK-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:  .LBB16_8: @ %for.end
 ; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
-; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    ldrd r9, r1, [sp, #24] @ 8-byte Folded Reload
+; CHECK-NEXT:    ldr r3, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    cmp.w r3, #0
 ; CHECK-NEXT:    beq .LBB16_3
-; CHECK-NEXT:  @ %bb.9: @ %while.body76.preheader
+; CHECK-NEXT:    b .LBB16_9
+; CHECK-NEXT:  .LBB16_9: @ %while.body76.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r4, r8
+; CHECK-NEXT:    mov r4, r5
+; CHECK-NEXT:    mov lr, r3
 ; CHECK-NEXT:  .LBB16_10: @ %while.body76
 ; CHECK-NEXT:    @ Parent Loop BB16_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    ldr r1, [r7], #4
+; CHECK-NEXT:    ldr r0, [r7], #4
 ; CHECK-NEXT:    vldrw.u32 q1, [r4], #4
-; CHECK-NEXT:    subs r0, #1
-; CHECK-NEXT:    vfma.f32 q0, q1, r1
-; CHECK-NEXT:    cmp r0, #1
-; CHECK-NEXT:    bgt .LBB16_10
-; CHECK-NEXT:  @ %bb.11: @ %while.end.loopexit
+; CHECK-NEXT:    subs.w lr, lr, #1
+; CHECK-NEXT:    vfma.f32 q0, q1, r0
+; CHECK-NEXT:    bne .LBB16_10
+; CHECK-NEXT:    b .LBB16_11
+; CHECK-NEXT:  .LBB16_11: @ %while.end.loopexit
 ; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
-; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    add.w r8, r8, r0, lsl #2
+; CHECK-NEXT:    add.w r5, r5, r3, lsl #2
 ; CHECK-NEXT:    b .LBB16_3
 ; CHECK-NEXT:  .LBB16_12: @ %if.end
 ; CHECK-NEXT:    add sp, #32
@@ -1649,135 +1649,156 @@ define arm_aapcs_vfpcc void @arm_biquad_cascade_df1_f32(%struct.arm_biquad_casd_
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
-; CHECK-NEXT:    ldrd r6, r9, [r0]
+; CHECK-NEXT:    .pad #88
+; CHECK-NEXT:    sub sp, #88
+; CHECK-NEXT:    ldrd r12, r10, [r0]
+; CHECK-NEXT:    @ implicit-def: $s2
 ; CHECK-NEXT:    and r7, r3, #3
-; CHECK-NEXT:    ldr r0, [r0, #8]
-; CHECK-NEXT:    lsrs r3, r3, #2
-; CHECK-NEXT:    @ implicit-def: $r12
-; CHECK-NEXT:    str r7, [sp] @ 4-byte Spill
-; CHECK-NEXT:    str r3, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    str r2, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    ldr.w r11, [r0, #8]
+; CHECK-NEXT:    lsrs r0, r3, #2
+; CHECK-NEXT:    str r0, [sp, #60] @ 4-byte Spill
+; CHECK-NEXT:    str r7, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    str r2, [sp, #56] @ 4-byte Spill
 ; CHECK-NEXT:    b .LBB19_3
 ; CHECK-NEXT:  .LBB19_1: @ in Loop: Header=BB19_3 Depth=1
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    mov r7, r5
-; CHECK-NEXT:    mov r4, r11
-; CHECK-NEXT:    mov r8, r10
+; CHECK-NEXT:    vmov.f32 s14, s7
+; CHECK-NEXT:    ldr r2, [sp, #56] @ 4-byte Reload
+; CHECK-NEXT:    vmov.f32 s0, s10
+; CHECK-NEXT:    vmov.f32 s7, s6
 ; CHECK-NEXT:  .LBB19_2: @ %if.end69
 ; CHECK-NEXT:    @ in Loop: Header=BB19_3 Depth=1
-; CHECK-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    adds r0, #128
-; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    strd r7, r4, [r9]
-; CHECK-NEXT:    subs r6, #1
-; CHECK-NEXT:    strd r3, r8, [r9, #8]
-; CHECK-NEXT:    add.w r9, r9, #16
+; CHECK-NEXT:    vstr s8, [r10]
+; CHECK-NEXT:    subs.w r12, r12, #1
+; CHECK-NEXT:    vstr s0, [r10, #4]
+; CHECK-NEXT:    add.w r11, r11, #128
+; CHECK-NEXT:    vstr s14, [r10, #8]
 ; CHECK-NEXT:    mov r1, r2
+; CHECK-NEXT:    vstr s7, [r10, #12]
+; CHECK-NEXT:    add.w r10, r10, #16
 ; CHECK-NEXT:    beq.w .LBB19_13
 ; CHECK-NEXT:  .LBB19_3: @ %do.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB19_5 Depth 2
-; CHECK-NEXT:    str r6, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r6, r2
-; CHECK-NEXT:    ldrd r5, r11, [r9]
-; CHECK-NEXT:    ldrd r8, r10, [r9, #8]
-; CHECK-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    wls lr, r2, .LBB19_6
+; CHECK-NEXT:    vldr s7, [r10, #8]
+; CHECK-NEXT:    mov r5, r2
+; CHECK-NEXT:    ldr r0, [sp, #60] @ 4-byte Reload
+; CHECK-NEXT:    vldr s8, [r10]
+; CHECK-NEXT:    vldr s10, [r10, #4]
+; CHECK-NEXT:    vldr s6, [r10, #12]
+; CHECK-NEXT:    wls lr, r0, .LBB19_6
 ; CHECK-NEXT:  @ %bb.4: @ %while.body.lr.ph
 ; CHECK-NEXT:    @ in Loop: Header=BB19_3 Depth=1
-; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r4, r11
-; CHECK-NEXT:    ldr.w lr, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    ldrd r5, lr, [sp, #56] @ 8-byte Folded Reload
 ; CHECK-NEXT:  .LBB19_5: @ %while.body
 ; CHECK-NEXT:    @ Parent Loop BB19_3 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    ldr r5, [r1, #12]
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    vldrw.u32 q6, [r0, #16]
-; CHECK-NEXT:    ldm.w r1, {r2, r7, r11}
-; CHECK-NEXT:    vmul.f32 q1, q1, r5
-; CHECK-NEXT:    vldrw.u32 q7, [r0, #32]
-; CHECK-NEXT:    vfma.f32 q1, q6, r11
-; CHECK-NEXT:    vldrw.u32 q4, [r0, #48]
-; CHECK-NEXT:    vfma.f32 q1, q7, r7
-; CHECK-NEXT:    vldrw.u32 q5, [r0, #64]
-; CHECK-NEXT:    vfma.f32 q1, q4, r2
-; CHECK-NEXT:    vldrw.u32 q3, [r0, #80]
-; CHECK-NEXT:    vfma.f32 q1, q5, r3
-; CHECK-NEXT:    vldrw.u32 q2, [r0, #96]
-; CHECK-NEXT:    vfma.f32 q1, q3, r4
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #112]
-; CHECK-NEXT:    vfma.f32 q1, q2, r8
+; CHECK-NEXT:    vmov r4, s8
+; CHECK-NEXT:    vldr s8, [r1, #12]
+; CHECK-NEXT:    vldrw.u32 q0, [r11, #112]
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vldr s10, [r1, #8]
+; CHECK-NEXT:    vmov r7, s7
+; CHECK-NEXT:    vmov r9, s6
+; CHECK-NEXT:    vldrw.u32 q1, [r11]
+; CHECK-NEXT:    vstrw.32 q0, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT:    vmov r8, s8
+; CHECK-NEXT:    vldrw.u32 q0, [r11, #16]
+; CHECK-NEXT:    ldr r6, [r1, #4]
+; CHECK-NEXT:    vldrw.u32 q7, [r11, #32]
+; CHECK-NEXT:    vmul.f32 q1, q1, r8
+; CHECK-NEXT:    vmov r3, s10
+; CHECK-NEXT:    vldrw.u32 q3, [r11, #48]
+; CHECK-NEXT:    vfma.f32 q1, q0, r3
+; CHECK-NEXT:    ldr r3, [r1]
+; CHECK-NEXT:    vfma.f32 q1, q7, r6
+; CHECK-NEXT:    vldrw.u32 q6, [r11, #64]
+; CHECK-NEXT:    vfma.f32 q1, q3, r3
+; CHECK-NEXT:    vldrw.u32 q5, [r11, #80]
+; CHECK-NEXT:    vfma.f32 q1, q6, r4
+; CHECK-NEXT:    vldrw.u32 q4, [r11, #96]
+; CHECK-NEXT:    vfma.f32 q1, q5, r0
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    vfma.f32 q1, q4, r7
 ; CHECK-NEXT:    adds r1, #16
-; CHECK-NEXT:    vfma.f32 q1, q0, r10
-; CHECK-NEXT:    vmov r10, s6
-; CHECK-NEXT:    vstrb.8 q1, [r6], #16
-; CHECK-NEXT:    vmov r8, s7
-; CHECK-NEXT:    mov r4, r11
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    mov r12, r5
+; CHECK-NEXT:    vfma.f32 q1, q0, r9
+; CHECK-NEXT:    vmov.f32 s2, s8
+; CHECK-NEXT:    vstrb.8 q1, [r5], #16
 ; CHECK-NEXT:    le lr, .LBB19_5
 ; CHECK-NEXT:  .LBB19_6: @ %while.end
 ; CHECK-NEXT:    @ in Loop: Header=BB19_3 Depth=1
-; CHECK-NEXT:    ldr r2, [sp] @ 4-byte Reload
-; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    ldr r7, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    cmp r7, #0
 ; CHECK-NEXT:    beq .LBB19_1
 ; CHECK-NEXT:  @ %bb.7: @ %if.then
 ; CHECK-NEXT:    @ in Loop: Header=BB19_3 Depth=1
-; CHECK-NEXT:    ldrd lr, r4, [r1]
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    ldrd r7, r1, [r1, #8]
-; CHECK-NEXT:    vldrw.u32 q6, [r0, #16]
-; CHECK-NEXT:    vldrw.u32 q7, [r0, #32]
-; CHECK-NEXT:    vldrw.u32 q4, [r0, #48]
-; CHECK-NEXT:    vmul.f32 q0, q0, r1
-; CHECK-NEXT:    vldrw.u32 q5, [r0, #64]
-; CHECK-NEXT:    vfma.f32 q0, q6, r7
-; CHECK-NEXT:    vldrw.u32 q3, [r0, #80]
-; CHECK-NEXT:    vfma.f32 q0, q7, r4
-; CHECK-NEXT:    vldrw.u32 q2, [r0, #96]
-; CHECK-NEXT:    vfma.f32 q0, q4, lr
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #112]
-; CHECK-NEXT:    vfma.f32 q0, q5, r5
-; CHECK-NEXT:    cmp r2, #1
-; CHECK-NEXT:    vfma.f32 q0, q3, r11
-; CHECK-NEXT:    vfma.f32 q0, q2, r8
-; CHECK-NEXT:    vfma.f32 q0, q1, r10
-; CHECK-NEXT:    vmov r5, s0
+; CHECK-NEXT:    vldr s24, [r1]
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vldr s0, [r1, #4]
+; CHECK-NEXT:    vldrw.u32 q3, [r11]
+; CHECK-NEXT:    vldr s3, [r1, #12]
+; CHECK-NEXT:    vldrw.u32 q4, [r11, #32]
+; CHECK-NEXT:    vldr s1, [r1, #8]
+; CHECK-NEXT:    vmov r1, s10
+; CHECK-NEXT:    vldrw.u32 q2, [r11, #96]
+; CHECK-NEXT:    vmov r6, s3
+; CHECK-NEXT:    vmul.f32 q3, q3, r6
+; CHECK-NEXT:    vmov r6, s1
+; CHECK-NEXT:    vstrw.32 q2, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q2, [r11, #112]
+; CHECK-NEXT:    vldrw.u32 q5, [r11, #48]
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    vstrw.32 q2, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q2, [r11, #80]
+; CHECK-NEXT:    vldrw.u32 q7, [r11, #64]
+; CHECK-NEXT:    vmov r3, s24
+; CHECK-NEXT:    vstrw.32 q2, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q2, [r11, #16]
+; CHECK-NEXT:    vmov r2, s7
+; CHECK-NEXT:    cmp r7, #1
+; CHECK-NEXT:    vfma.f32 q3, q2, r6
+; CHECK-NEXT:    vldrw.u32 q2, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vfma.f32 q3, q4, r4
+; CHECK-NEXT:    vmov lr, s6
+; CHECK-NEXT:    vfma.f32 q3, q5, r3
+; CHECK-NEXT:    vfma.f32 q3, q7, r0
+; CHECK-NEXT:    vfma.f32 q3, q2, r1
+; CHECK-NEXT:    vldrw.u32 q2, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vfma.f32 q3, q2, r2
+; CHECK-NEXT:    vldrw.u32 q2, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    vfma.f32 q3, q2, lr
 ; CHECK-NEXT:    bne .LBB19_9
 ; CHECK-NEXT:  @ %bb.8: @ %if.then58
 ; CHECK-NEXT:    @ in Loop: Header=BB19_3 Depth=1
-; CHECK-NEXT:    str r5, [r6]
-; CHECK-NEXT:    mov r7, lr
-; CHECK-NEXT:    mov r4, r12
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    b .LBB19_12
+; CHECK-NEXT:    vstr s12, [r5]
+; CHECK-NEXT:    vmov.f32 s8, s24
+; CHECK-NEXT:    vmov.f32 s0, s2
+; CHECK-NEXT:    vmov.f32 s14, s12
+; CHECK-NEXT:    b .LBB19_11
 ; CHECK-NEXT:  .LBB19_9: @ %if.else
 ; CHECK-NEXT:    @ in Loop: Header=BB19_3 Depth=1
-; CHECK-NEXT:    vmov r8, s1
-; CHECK-NEXT:    cmp r2, #2
-; CHECK-NEXT:    vstr s1, [r6, #4]
-; CHECK-NEXT:    str r5, [r6]
-; CHECK-NEXT:    bne .LBB19_11
+; CHECK-NEXT:    cmp r7, #2
+; CHECK-NEXT:    vstmia r5, {s12, s13}
+; CHECK-NEXT:    bne .LBB19_12
 ; CHECK-NEXT:  @ %bb.10: @ in Loop: Header=BB19_3 Depth=1
-; CHECK-NEXT:    mov r7, r4
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    mov r4, lr
-; CHECK-NEXT:    mov r8, r5
-; CHECK-NEXT:    b .LBB19_12
-; CHECK-NEXT:  .LBB19_11: @ %if.else64
+; CHECK-NEXT:    vmov.f32 s8, s0
+; CHECK-NEXT:    vmov.f32 s14, s13
+; CHECK-NEXT:    vmov.f32 s0, s24
+; CHECK-NEXT:    vmov.f32 s7, s12
+; CHECK-NEXT:  .LBB19_11: @ %if.end69
 ; CHECK-NEXT:    @ in Loop: Header=BB19_3 Depth=1
-; CHECK-NEXT:    vmov r3, s2
-; CHECK-NEXT:    vstr s2, [r6, #8]
-; CHECK-NEXT:  .LBB19_12: @ %if.end69
+; CHECK-NEXT:    vmov.f32 s2, s3
+; CHECK-NEXT:    ldr r2, [sp, #56] @ 4-byte Reload
+; CHECK-NEXT:    b .LBB19_2
+; CHECK-NEXT:  .LBB19_12: @ %if.else64
 ; CHECK-NEXT:    @ in Loop: Header=BB19_3 Depth=1
-; CHECK-NEXT:    mov r12, r1
+; CHECK-NEXT:    vmov.f32 s7, s13
+; CHECK-NEXT:    ldr r2, [sp, #56] @ 4-byte Reload
+; CHECK-NEXT:    vmov.f32 s2, s3
+; CHECK-NEXT:    vstr s14, [r5, #8]
+; CHECK-NEXT:    vmov.f32 s8, s1
 ; CHECK-NEXT:    b .LBB19_2
 ; CHECK-NEXT:  .LBB19_13: @ %do.end
-; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    add sp, #88
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
diff --git a/llvm/test/CodeGen/Thumb2/mve-fmath.ll b/llvm/test/CodeGen/Thumb2/mve-fmath.ll
index d93f31c9e2171..1927fad046103 100644
--- a/llvm/test/CodeGen/Thumb2/mve-fmath.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-fmath.ll
@@ -112,74 +112,55 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @cos_float16_t(<8 x half> %src) {
 ; CHECK-LABEL: cos_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl cosf
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmovx.f16 s0, s16
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s16
+; CHECK-NEXT:    vmov s20, r0
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.16 q5[0], r4
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtb.f16.f32 s20, s20
+; CHECK-NEXT:    vcvtt.f16.f32 s20, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vmov.16 q5[1], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s17
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[2], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s21, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s17
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s21, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vmov.16 q5[3], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s18
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[4], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s22, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s18
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s22, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vmov.16 q5[5], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s19
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[6], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s23, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s19
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[7], r0
+; CHECK-NEXT:    vcvtt.f16.f32 s23, s0
 ; CHECK-NEXT:    vmov q0, q5
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = call fast <8 x half> @llvm.cos.v8f16(<8 x half> %src)
   ret <8 x half> %0
@@ -243,74 +224,55 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @sin_float16_t(<8 x half> %src) {
 ; CHECK-LABEL: sin_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl sinf
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmovx.f16 s0, s16
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s16
+; CHECK-NEXT:    vmov s20, r0
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.16 q5[0], r4
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtb.f16.f32 s20, s20
+; CHECK-NEXT:    vcvtt.f16.f32 s20, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vmov.16 q5[1], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s17
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[2], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s21, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s17
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s21, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vmov.16 q5[3], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s18
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[4], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s22, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s18
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s22, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vmov.16 q5[5], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s19
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[6], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s23, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s19
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[7], r0
+; CHECK-NEXT:    vcvtt.f16.f32 s23, s0
 ; CHECK-NEXT:    vmov q0, q5
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = call fast <8 x half> @llvm.sin.v8f16(<8 x half> %src)
   ret <8 x half> %0
@@ -374,74 +336,55 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @exp_float16_t(<8 x half> %src) {
 ; CHECK-LABEL: exp_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl expf
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmovx.f16 s0, s16
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s16
+; CHECK-NEXT:    vmov s20, r0
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.16 q5[0], r4
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtb.f16.f32 s20, s20
+; CHECK-NEXT:    vcvtt.f16.f32 s20, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vmov.16 q5[1], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s17
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[2], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s21, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s17
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s21, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vmov.16 q5[3], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s18
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[4], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s22, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s18
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s22, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vmov.16 q5[5], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s19
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[6], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s23, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s19
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[7], r0
+; CHECK-NEXT:    vcvtt.f16.f32 s23, s0
 ; CHECK-NEXT:    vmov q0, q5
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = call fast <8 x half> @llvm.exp.v8f16(<8 x half> %src)
   ret <8 x half> %0
@@ -505,74 +448,55 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @exp2_float16_t(<8 x half> %src) {
 ; CHECK-LABEL: exp2_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl exp2f
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmovx.f16 s0, s16
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s16
+; CHECK-NEXT:    vmov s20, r0
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.16 q5[0], r4
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtb.f16.f32 s20, s20
+; CHECK-NEXT:    vcvtt.f16.f32 s20, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vmov.16 q5[1], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s17
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[2], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s21, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s17
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s21, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vmov.16 q5[3], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s18
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[4], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s22, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s18
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s22, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vmov.16 q5[5], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s19
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[6], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s23, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s19
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[7], r0
+; CHECK-NEXT:    vcvtt.f16.f32 s23, s0
 ; CHECK-NEXT:    vmov q0, q5
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = call fast <8 x half> @llvm.exp2.v8f16(<8 x half> %src)
   ret <8 x half> %0
@@ -636,74 +560,55 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @log_float16_t(<8 x half> %src) {
 ; CHECK-LABEL: log_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl logf
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmovx.f16 s0, s16
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s16
+; CHECK-NEXT:    vmov s20, r0
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.16 q5[0], r4
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtb.f16.f32 s20, s20
+; CHECK-NEXT:    vcvtt.f16.f32 s20, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vmov.16 q5[1], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s17
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[2], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s21, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s17
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s21, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vmov.16 q5[3], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s18
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[4], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s22, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s18
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s22, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vmov.16 q5[5], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s19
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[6], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s23, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s19
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[7], r0
+; CHECK-NEXT:    vcvtt.f16.f32 s23, s0
 ; CHECK-NEXT:    vmov q0, q5
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = call fast <8 x half> @llvm.log.v8f16(<8 x half> %src)
   ret <8 x half> %0
@@ -767,74 +672,55 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @log2_float16_t(<8 x half> %src) {
 ; CHECK-LABEL: log2_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log2f
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmovx.f16 s0, s16
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s16
+; CHECK-NEXT:    vmov s20, r0
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.16 q5[0], r4
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtb.f16.f32 s20, s20
+; CHECK-NEXT:    vcvtt.f16.f32 s20, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vmov.16 q5[1], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s17
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[2], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s21, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s17
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s21, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vmov.16 q5[3], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s18
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[4], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s22, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s18
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s22, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vmov.16 q5[5], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s19
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[6], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s23, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s19
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[7], r0
+; CHECK-NEXT:    vcvtt.f16.f32 s23, s0
 ; CHECK-NEXT:    vmov q0, q5
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = call fast <8 x half> @llvm.log2.v8f16(<8 x half> %src)
   ret <8 x half> %0
@@ -898,74 +784,55 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @log10_float16_t(<8 x half> %src) {
 ; CHECK-LABEL: log10_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log10f
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmovx.f16 s0, s16
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s16
+; CHECK-NEXT:    vmov s20, r0
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.16 q5[0], r4
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtb.f16.f32 s20, s20
+; CHECK-NEXT:    vcvtt.f16.f32 s20, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vmov.16 q5[1], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s17
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[2], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s21, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s17
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s21, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vmov.16 q5[3], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s18
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[4], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s22, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s18
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s22, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vmov.16 q5[5], r0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s19
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q5[6], r0
+; CHECK-NEXT:    vcvtb.f16.f32 s23, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s19
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q5[7], r0
+; CHECK-NEXT:    vcvtt.f16.f32 s23, s0
 ; CHECK-NEXT:    vmov q0, q5
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = call fast <8 x half> @llvm.log10.v8f16(<8 x half> %src)
   ret <8 x half> %0
@@ -1035,95 +902,72 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @pow_float16_t(<8 x half> %src1, <8 x half> %src2) {
 ; CHECK-LABEL: pow_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vmov q5, q1
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
-; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov q5, q0
+; CHECK-NEXT:    vmov q4, q1
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s20
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl powf
-; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmovx.f16 s2, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vmovx.f16 s0, s20
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s20
+; CHECK-NEXT:    vmov s24, r0
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s16
 ; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmov.16 q6[0], r4
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
+; CHECK-NEXT:    vcvtb.f16.f32 s24, s24
+; CHECK-NEXT:    vcvtt.f16.f32 s24, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s21
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vmov.16 q6[1], r0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s21
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmovx.f16 s2, s17
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vcvtb.f16.f32 s25, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s21
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s21
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q6[2], r0
-; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s17
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s25, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s22
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vmov.16 q6[3], r0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s22
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmovx.f16 s2, s18
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vcvtb.f16.f32 s26, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s22
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s22
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q6[4], r0
-; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s18
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
+; CHECK-NEXT:    vcvtt.f16.f32 s26, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s23
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vmov.16 q6[5], r0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s23
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vmovx.f16 s2, s19
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vcvtb.f16.f32 s27, s0
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s23
 ; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmovx.f16 s0, s23
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vmov.16 q6[6], r0
-; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vcvtt.f32.f16 s0, s19
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q6[7], r0
+; CHECK-NEXT:    vcvtt.f16.f32 s27, s0
 ; CHECK-NEXT:    vmov q0, q6
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = call fast <8 x half> @llvm.pow.v8f16(<8 x half> %src1, <8 x half> %src2)
   ret <8 x half> %0
diff --git a/llvm/test/CodeGen/Thumb2/mve-fp16convertloops.ll b/llvm/test/CodeGen/Thumb2/mve-fp16convertloops.ll
index 4e102427cad6c..733b17bdb798e 100644
--- a/llvm/test/CodeGen/Thumb2/mve-fp16convertloops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-fp16convertloops.ll
@@ -14,23 +14,8 @@ define void @to_4(float* nocapture readonly %x, half* noalias nocapture %y) {
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
 ; CHECK-NEXT:    vmul.f32 q1, q1, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s4
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s5
-; CHECK-NEXT:    vmov r3, s8
-; CHECK-NEXT:    vmov.16 q2[0], r2
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s6
-; CHECK-NEXT:    vmov.16 q2[1], r3
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s7
-; CHECK-NEXT:    vmov.16 q2[2], r2
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vmov.16 q2[3], r2
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vmov r3, s9
-; CHECK-NEXT:    str r2, [r1]
-; CHECK-NEXT:    str r3, [r1, #4]
-; CHECK-NEXT:    adds r1, #8
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vstrh.32 q1, [r1], #8
 ; CHECK-NEXT:    le lr, .LBB0_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
@@ -73,35 +58,14 @@ define void @to_8(float* nocapture readonly %x, half* noalias nocapture %y) {
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB1_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vstrh.32 q1, [r1, #8]
 ; CHECK-NEXT:    vldrw.u32 q1, [r0], #32
-; CHECK-NEXT:    vmul.f32 q2, q1, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s8
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s9
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov.16 q1[0], r2
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s10
-; CHECK-NEXT:    vmov.16 q1[1], r3
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q1[2], r2
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vldrw.u32 q2, [r0, #-16]
-; CHECK-NEXT:    vmov.16 q1[3], r2
-; CHECK-NEXT:    vmul.f32 q2, q2, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s8
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s9
-; CHECK-NEXT:    vmov.16 q1[4], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s10
-; CHECK-NEXT:    vmov.16 q1[5], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q1[6], r2
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vmov.16 q1[7], r2
-; CHECK-NEXT:    vstrb.8 q1, [r1], #16
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vstrh.32 q1, [r1], #16
 ; CHECK-NEXT:    le lr, .LBB1_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
@@ -144,64 +108,22 @@ define void @to_16(float* nocapture readonly %x, half* noalias nocapture %y) {
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB2_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vstrh.32 q1, [r1, #24]
 ; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
-; CHECK-NEXT:    vmul.f32 q2, q1, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s8
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s9
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov.16 q1[0], r2
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s10
-; CHECK-NEXT:    vmov.16 q1[1], r3
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q1[2], r2
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vldrw.u32 q2, [r0, #48]
-; CHECK-NEXT:    vmov.16 q1[3], r2
-; CHECK-NEXT:    vmul.f32 q2, q2, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s8
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s9
-; CHECK-NEXT:    vmov.16 q1[4], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s10
-; CHECK-NEXT:    vmov.16 q1[5], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q1[6], r2
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vmov.16 q1[7], r2
-; CHECK-NEXT:    vstrh.16 q1, [r1, #16]
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vstrh.32 q1, [r1, #16]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vstrh.32 q1, [r1, #8]
 ; CHECK-NEXT:    vldrw.u32 q1, [r0], #64
-; CHECK-NEXT:    vmul.f32 q2, q1, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s9
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s8
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s10
-; CHECK-NEXT:    vmov.16 q1[0], r3
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q1[1], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vmov.16 q1[2], r2
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vldrw.u32 q2, [r0, #-48]
-; CHECK-NEXT:    vmov.16 q1[3], r2
-; CHECK-NEXT:    vmul.f32 q2, q2, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s8
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s9
-; CHECK-NEXT:    vmov.16 q1[4], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s10
-; CHECK-NEXT:    vmov.16 q1[5], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q1[6], r2
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vmov.16 q1[7], r2
-; CHECK-NEXT:    vstrh.16 q1, [r1], #32
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vstrh.32 q1, [r1], #32
 ; CHECK-NEXT:    le lr, .LBB2_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
@@ -244,18 +166,9 @@ define void @from_4(half* nocapture readonly %x, float* noalias nocapture %y) {
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB3_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr r2, [r0]
-; CHECK-NEXT:    ldr r3, [r0, #4]
-; CHECK-NEXT:    adds r0, #8
-; CHECK-NEXT:    vmov.32 q1[0], r2
-; CHECK-NEXT:    vmov.32 q1[1], r3
-; CHECK-NEXT:    vmovx.f16 s10, s5
-; CHECK-NEXT:    vmovx.f16 s8, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s15, s10
-; CHECK-NEXT:    vcvtb.f32.f16 s14, s5
-; CHECK-NEXT:    vcvtb.f32.f16 s13, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s12, s4
-; CHECK-NEXT:    vmul.f32 q1, q3, q0
+; CHECK-NEXT:    vldrh.u32 q1, [r0], #8
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
 ; CHECK-NEXT:    vstrb.8 q1, [r1], #16
 ; CHECK-NEXT:    le lr, .LBB3_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
@@ -293,34 +206,22 @@ define void @from_8(half* nocapture readonly %x, float* noalias nocapture %y) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    adr r2, .LCPI4_0
 ; CHECK-NEXT:    mov.w lr, #128
 ; CHECK-NEXT:    vldrw.u32 q0, [r2]
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB4_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u16 q1, [r0], #16
-; CHECK-NEXT:    vmovx.f16 s8, s5
-; CHECK-NEXT:    vmovx.f16 s13, s7
-; CHECK-NEXT:    vcvtb.f32.f16 s11, s8
-; CHECK-NEXT:    vmovx.f16 s14, s6
-; CHECK-NEXT:    vcvtb.f32.f16 s10, s5
-; CHECK-NEXT:    vcvtb.f32.f16 s19, s13
-; CHECK-NEXT:    vcvtb.f32.f16 s18, s7
-; CHECK-NEXT:    vmovx.f16 s12, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s17, s14
-; CHECK-NEXT:    vcvtb.f32.f16 s16, s6
-; CHECK-NEXT:    vcvtb.f32.f16 s9, s12
-; CHECK-NEXT:    vcvtb.f32.f16 s8, s4
-; CHECK-NEXT:    vmul.f32 q1, q4, q0
+; CHECK-NEXT:    vldrh.u32 q1, [r0, #8]
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
 ; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
-; CHECK-NEXT:    vmul.f32 q1, q2, q0
+; CHECK-NEXT:    vldrh.u32 q1, [r0], #16
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
 ; CHECK-NEXT:    vstrw.32 q1, [r1], #32
 ; CHECK-NEXT:    le lr, .LBB4_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
-; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.3:
@@ -355,51 +256,30 @@ define void @from_16(half* nocapture readonly %x, float* noalias nocapture %y) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    adr r2, .LCPI5_0
 ; CHECK-NEXT:    mov.w lr, #64
 ; CHECK-NEXT:    vldrw.u32 q0, [r2]
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB5_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u16 q1, [r0], #32
-; CHECK-NEXT:    vmovx.f16 s12, s5
-; CHECK-NEXT:    vldrh.u16 q2, [r0, #-16]
-; CHECK-NEXT:    vcvtb.f32.f16 s15, s12
-; CHECK-NEXT:    vmovx.f16 s18, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s14, s5
-; CHECK-NEXT:    vmovx.f16 s16, s7
-; CHECK-NEXT:    vcvtb.f32.f16 s13, s18
-; CHECK-NEXT:    vmovx.f16 s20, s9
-; CHECK-NEXT:    vcvtb.f32.f16 s12, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s19, s16
-; CHECK-NEXT:    vmovx.f16 s22, s6
-; CHECK-NEXT:    vcvtb.f32.f16 s18, s7
-; CHECK-NEXT:    vcvtb.f32.f16 s17, s22
-; CHECK-NEXT:    vcvtb.f32.f16 s23, s20
-; CHECK-NEXT:    vmovx.f16 s28, s11
-; CHECK-NEXT:    vcvtb.f32.f16 s22, s9
-; CHECK-NEXT:    vcvtb.f32.f16 s31, s28
-; CHECK-NEXT:    vmovx.f16 s26, s10
-; CHECK-NEXT:    vcvtb.f32.f16 s30, s11
-; CHECK-NEXT:    vmovx.f16 s24, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s29, s26
-; CHECK-NEXT:    vcvtb.f32.f16 s28, s10
-; CHECK-NEXT:    vcvtb.f32.f16 s21, s24
-; CHECK-NEXT:    vcvtb.f32.f16 s20, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s16, s6
-; CHECK-NEXT:    vmul.f32 q1, q7, q0
+; CHECK-NEXT:    vldrh.u32 q1, [r0, #24]
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
 ; CHECK-NEXT:    vstrw.32 q1, [r1, #48]
-; CHECK-NEXT:    vmul.f32 q1, q5, q0
+; CHECK-NEXT:    vldrh.u32 q1, [r0, #16]
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
 ; CHECK-NEXT:    vstrw.32 q1, [r1, #32]
-; CHECK-NEXT:    vmul.f32 q1, q4, q0
+; CHECK-NEXT:    vldrh.u32 q1, [r0, #8]
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
 ; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
-; CHECK-NEXT:    vmul.f32 q1, q3, q0
+; CHECK-NEXT:    vldrh.u32 q1, [r0], #32
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
 ; CHECK-NEXT:    vstrw.32 q1, [r1], #64
 ; CHECK-NEXT:    le lr, .LBB5_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
-; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.3:
@@ -440,35 +320,11 @@ define void @both_4(half* nocapture readonly %x, half* noalias nocapture %y) {
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB6_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr r2, [r0]
-; CHECK-NEXT:    ldr r3, [r0, #4]
-; CHECK-NEXT:    adds r0, #8
-; CHECK-NEXT:    vmov.32 q1[0], r2
-; CHECK-NEXT:    vmov.32 q1[1], r3
-; CHECK-NEXT:    vmovx.f16 s10, s5
-; CHECK-NEXT:    vmovx.f16 s8, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s15, s10
-; CHECK-NEXT:    vcvtb.f32.f16 s14, s5
-; CHECK-NEXT:    vcvtb.f32.f16 s13, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s12, s4
-; CHECK-NEXT:    vmul.f32 q1, q3, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s4
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s5
-; CHECK-NEXT:    vmov r3, s8
-; CHECK-NEXT:    vmov.16 q2[0], r2
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s6
-; CHECK-NEXT:    vmov.16 q2[1], r3
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s7
-; CHECK-NEXT:    vmov.16 q2[2], r2
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vmov.16 q2[3], r2
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vmov r3, s9
-; CHECK-NEXT:    str r2, [r1]
-; CHECK-NEXT:    str r3, [r1, #4]
-; CHECK-NEXT:    adds r1, #8
+; CHECK-NEXT:    vldrh.u32 q1, [r0], #8
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vstrh.32 q1, [r1], #8
 ; CHECK-NEXT:    le lr, .LBB6_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
@@ -506,57 +362,24 @@ define void @both_8(half* nocapture readonly %x, half* noalias nocapture %y) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    adr r2, .LCPI7_0
 ; CHECK-NEXT:    mov.w lr, #128
 ; CHECK-NEXT:    vldrw.u32 q0, [r2]
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB7_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u16 q2, [r0], #16
-; CHECK-NEXT:    vmovx.f16 s6, s9
-; CHECK-NEXT:    vmovx.f16 s4, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s15, s6
-; CHECK-NEXT:    vcvtb.f32.f16 s14, s9
-; CHECK-NEXT:    vcvtb.f32.f16 s13, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s12, s8
-; CHECK-NEXT:    vmul.f32 q3, q3, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s12
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s13
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov.16 q1[0], r2
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s14
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s15
-; CHECK-NEXT:    vmovx.f16 s14, s11
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vcvtb.f32.f16 s19, s14
-; CHECK-NEXT:    vmov.16 q1[1], r3
-; CHECK-NEXT:    vmov.16 q1[2], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vmovx.f16 s12, s10
-; CHECK-NEXT:    vcvtb.f32.f16 s18, s11
-; CHECK-NEXT:    vcvtb.f32.f16 s17, s12
-; CHECK-NEXT:    vmov.16 q1[3], r2
-; CHECK-NEXT:    vcvtb.f32.f16 s16, s10
-; CHECK-NEXT:    vmul.f32 q2, q4, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s8
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s9
-; CHECK-NEXT:    vmov.16 q1[4], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s10
-; CHECK-NEXT:    vmov.16 q1[5], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q1[6], r2
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vmov.16 q1[7], r2
-; CHECK-NEXT:    vstrb.8 q1, [r1], #16
+; CHECK-NEXT:    vldrh.u32 q1, [r0, #8]
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vstrh.32 q1, [r1, #8]
+; CHECK-NEXT:    vldrh.u32 q1, [r0], #16
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vstrh.32 q1, [r1], #16
 ; CHECK-NEXT:    le lr, .LBB7_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
-; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.3:
@@ -592,97 +415,34 @@ define void @both_16(half* nocapture readonly %x, half* noalias nocapture %y) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    adr r2, .LCPI8_0
 ; CHECK-NEXT:    mov.w lr, #64
 ; CHECK-NEXT:    vldrw.u32 q0, [r2]
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB8_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u16 q2, [r0, #16]
-; CHECK-NEXT:    vmovx.f16 s6, s9
-; CHECK-NEXT:    vmovx.f16 s4, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s15, s6
-; CHECK-NEXT:    vcvtb.f32.f16 s14, s9
-; CHECK-NEXT:    vcvtb.f32.f16 s13, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s12, s8
-; CHECK-NEXT:    vmul.f32 q3, q3, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s12
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s13
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov.16 q1[0], r2
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s14
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s15
-; CHECK-NEXT:    vmovx.f16 s14, s11
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vcvtb.f32.f16 s19, s14
-; CHECK-NEXT:    vmov.16 q1[1], r3
-; CHECK-NEXT:    vmov.16 q1[2], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vmovx.f16 s12, s10
-; CHECK-NEXT:    vcvtb.f32.f16 s18, s11
-; CHECK-NEXT:    vcvtb.f32.f16 s17, s12
-; CHECK-NEXT:    vmov.16 q1[3], r2
-; CHECK-NEXT:    vcvtb.f32.f16 s16, s10
-; CHECK-NEXT:    vmul.f32 q2, q4, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s8
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s9
-; CHECK-NEXT:    vmov.16 q1[4], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s10
-; CHECK-NEXT:    vmov.16 q1[5], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q1[6], r2
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vldrh.u16 q2, [r0], #32
-; CHECK-NEXT:    vmov.16 q1[7], r2
-; CHECK-NEXT:    vstrh.16 q1, [r1, #16]
-; CHECK-NEXT:    vmovx.f16 s6, s9
-; CHECK-NEXT:    vmovx.f16 s4, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s15, s6
-; CHECK-NEXT:    vcvtb.f32.f16 s14, s9
-; CHECK-NEXT:    vcvtb.f32.f16 s13, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s12, s8
-; CHECK-NEXT:    vmul.f32 q3, q3, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s12
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s13
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov.16 q1[0], r2
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s14
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s15
-; CHECK-NEXT:    vmovx.f16 s14, s11
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vcvtb.f32.f16 s19, s14
-; CHECK-NEXT:    vmov.16 q1[1], r3
-; CHECK-NEXT:    vmov.16 q1[2], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vmovx.f16 s12, s10
-; CHECK-NEXT:    vcvtb.f32.f16 s18, s11
-; CHECK-NEXT:    vcvtb.f32.f16 s17, s12
-; CHECK-NEXT:    vmov.16 q1[3], r2
-; CHECK-NEXT:    vcvtb.f32.f16 s16, s10
-; CHECK-NEXT:    vmul.f32 q2, q4, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s8
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s9
-; CHECK-NEXT:    vmov.16 q1[4], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s10
-; CHECK-NEXT:    vmov.16 q1[5], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q1[6], r2
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vmov.16 q1[7], r2
-; CHECK-NEXT:    vstrh.16 q1, [r1], #32
+; CHECK-NEXT:    vldrh.u32 q1, [r0, #24]
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vstrh.32 q1, [r1, #24]
+; CHECK-NEXT:    vldrh.u32 q1, [r0, #16]
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vstrh.32 q1, [r1, #16]
+; CHECK-NEXT:    vldrh.u32 q1, [r0, #8]
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vstrh.32 q1, [r1, #8]
+; CHECK-NEXT:    vldrh.u32 q1, [r0], #32
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vstrh.32 q1, [r1], #32
 ; CHECK-NEXT:    le lr, .LBB8_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
-; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.3:
@@ -718,57 +478,22 @@ define void @both_8_I(half* nocapture readonly %x, half* noalias nocapture %y) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    adr r2, .LCPI9_0
 ; CHECK-NEXT:    mov.w lr, #128
 ; CHECK-NEXT:    vldrw.u32 q0, [r2]
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB9_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u16 q2, [r0], #16
-; CHECK-NEXT:    vcvtb.f32.f16 s7, s11
-; CHECK-NEXT:    vmovx.f16 s13, s11
-; CHECK-NEXT:    vcvtb.f32.f16 s6, s10
-; CHECK-NEXT:    vmovx.f16 s14, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s5, s9
-; CHECK-NEXT:    vcvtb.f32.f16 s4, s8
-; CHECK-NEXT:    vmovx.f16 s8, s10
+; CHECK-NEXT:    vldrh.u16 q1, [r0], #16
+; CHECK-NEXT:    vcvtb.f32.f16 q2, q1
+; CHECK-NEXT:    vcvtt.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q2, q2, q0
 ; CHECK-NEXT:    vmul.f32 q1, q1, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s19, s13
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vmovx.f16 s12, s9
-; CHECK-NEXT:    vcvtb.f32.f16 s18, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s17, s12
-; CHECK-NEXT:    vcvtb.f32.f16 s16, s14
-; CHECK-NEXT:    vmul.f32 q2, q4, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s5
-; CHECK-NEXT:    vmov r3, s12
-; CHECK-NEXT:    vmov.16 q3[0], r2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vmov.16 q3[1], r3
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s9
-; CHECK-NEXT:    vmov.16 q3[2], r2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s6
-; CHECK-NEXT:    vmov.16 q3[3], r2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s10
-; CHECK-NEXT:    vmov.16 q3[4], r2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s7
-; CHECK-NEXT:    vmov.16 q3[5], r2
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s11
-; CHECK-NEXT:    vmov.16 q3[6], r2
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vmov.16 q3[7], r2
-; CHECK-NEXT:    vstrb.8 q3, [r1], #16
+; CHECK-NEXT:    vcvtb.f16.f32 q2, q2
+; CHECK-NEXT:    vcvtt.f16.f32 q2, q1
+; CHECK-NEXT:    vstrb.8 q2, [r1], #16
 ; CHECK-NEXT:    le lr, .LBB9_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
-; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.3:
@@ -809,97 +534,30 @@ define void @both_16_I(half* nocapture readonly %x, half* noalias nocapture %y)
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    adr r2, .LCPI10_0
 ; CHECK-NEXT:    mov.w lr, #128
 ; CHECK-NEXT:    vldrw.u32 q0, [r2]
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB10_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u16 q2, [r0]
-; CHECK-NEXT:    vcvtb.f32.f16 s7, s11
-; CHECK-NEXT:    vmovx.f16 s13, s11
-; CHECK-NEXT:    vcvtb.f32.f16 s6, s10
-; CHECK-NEXT:    vmovx.f16 s14, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s5, s9
-; CHECK-NEXT:    vcvtb.f32.f16 s4, s8
-; CHECK-NEXT:    vmovx.f16 s8, s10
+; CHECK-NEXT:    vldrh.u16 q1, [r0]
+; CHECK-NEXT:    vcvtb.f32.f16 q2, q1
+; CHECK-NEXT:    vcvtt.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q2, q2, q0
+; CHECK-NEXT:    vmul.f32 q1, q1, q0
+; CHECK-NEXT:    vcvtb.f16.f32 q2, q2
+; CHECK-NEXT:    vcvtt.f16.f32 q2, q1
+; CHECK-NEXT:    vldrh.u16 q1, [r0, #16]!
+; CHECK-NEXT:    vstrh.16 q2, [r1]
+; CHECK-NEXT:    vcvtb.f32.f16 q2, q1
+; CHECK-NEXT:    vcvtt.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q2, q2, q0
 ; CHECK-NEXT:    vmul.f32 q1, q1, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s19, s13
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vmovx.f16 s12, s9
-; CHECK-NEXT:    vcvtb.f32.f16 s18, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s17, s12
-; CHECK-NEXT:    vcvtb.f32.f16 s16, s14
-; CHECK-NEXT:    vmul.f32 q2, q4, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s5
-; CHECK-NEXT:    vmov r3, s12
-; CHECK-NEXT:    vmov.16 q3[0], r2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vmov.16 q3[1], r3
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s9
-; CHECK-NEXT:    vmov.16 q3[2], r2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s6
-; CHECK-NEXT:    vmov.16 q3[3], r2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s10
-; CHECK-NEXT:    vmov.16 q3[4], r2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s7
-; CHECK-NEXT:    vmov.16 q3[5], r2
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s11
-; CHECK-NEXT:    vmov.16 q3[6], r2
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vldrh.u16 q2, [r0, #16]!
-; CHECK-NEXT:    vmov.16 q3[7], r2
-; CHECK-NEXT:    vstrh.16 q3, [r1]
-; CHECK-NEXT:    vmovx.f16 s12, s11
-; CHECK-NEXT:    vmovx.f16 s14, s10
-; CHECK-NEXT:    vcvtb.f32.f16 s19, s12
-; CHECK-NEXT:    vmovx.f16 s4, s9
-; CHECK-NEXT:    vcvtb.f32.f16 s18, s14
-; CHECK-NEXT:    vmovx.f16 s6, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s17, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s16, s6
-; CHECK-NEXT:    vmul.f32 q1, q4, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s4
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f32.f16 s15, s11
-; CHECK-NEXT:    vcvtb.f32.f16 s14, s10
-; CHECK-NEXT:    vcvtb.f32.f16 s13, s9
-; CHECK-NEXT:    vcvtb.f32.f16 s12, s8
-; CHECK-NEXT:    vmul.f32 q3, q3, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s13
-; CHECK-NEXT:    vmov r3, s8
-; CHECK-NEXT:    vmov.16 q2[0], r3
-; CHECK-NEXT:    vmov.16 q2[1], r2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s5
-; CHECK-NEXT:    vmov.16 q2[2], r2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s14
-; CHECK-NEXT:    vmov.16 q2[3], r2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s6
-; CHECK-NEXT:    vmov.16 q2[4], r2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s15
-; CHECK-NEXT:    vmov.16 q2[5], r2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s7
-; CHECK-NEXT:    vmov.16 q2[6], r2
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vmov.16 q2[7], r2
+; CHECK-NEXT:    vcvtb.f16.f32 q2, q2
+; CHECK-NEXT:    vcvtt.f16.f32 q2, q1
 ; CHECK-NEXT:    vstrb.8 q2, [r1, #16]!
 ; CHECK-NEXT:    le lr, .LBB10_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
-; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.3:
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll b/llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll
index 365f5e8b3a6ab..fceda29f252a0 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll
@@ -594,57 +594,56 @@ define dso_local void @arm_mat_mult_q15(i16* noalias nocapture readonly %A, i16*
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
-; CHECK-NEXT:    strd r0, r2, [sp, #24] @ 8-byte Folded Spill
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    strd r0, r2, [sp, #16] @ 8-byte Folded Spill
 ; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    str r3, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r3
 ; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    ldrne.w lr, [sp, #120]
-; CHECK-NEXT:    cmpne.w lr, #0
+; CHECK-NEXT:    ldrne r0, [sp, #112]
+; CHECK-NEXT:    cmpne r0, #0
 ; CHECK-NEXT:    bne .LBB10_2
 ; CHECK-NEXT:  .LBB10_1: @ %for.cond.cleanup
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:  .LBB10_2: @ %for.cond1.preheader.us.preheader
-; CHECK-NEXT:    ldr.w r11, [sp, #124]
+; CHECK-NEXT:    ldr.w r11, [sp, #116]
 ; CHECK-NEXT:    mov r6, r1
 ; CHECK-NEXT:    movs r1, #1
-; CHECK-NEXT:    vdup.32 q4, lr
-; CHECK-NEXT:    bic r0, r11, #3
-; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    subs r0, #4
-; CHECK-NEXT:    lsl.w r4, lr, #1
 ; CHECK-NEXT:    mov.w r9, #0
-; CHECK-NEXT:    vshl.i32 q6, q4, #2
+; CHECK-NEXT:    bic r10, r11, #3
+; CHECK-NEXT:    sub.w r0, r10, #4
 ; CHECK-NEXT:    add.w r8, r1, r0, lsr #2
+; CHECK-NEXT:    ldr r1, [sp, #112]
 ; CHECK-NEXT:    lsl.w r0, r11, #1
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
 ; CHECK-NEXT:    adr r0, .LCPI10_0
+; CHECK-NEXT:    vdup.32 q4, r1
 ; CHECK-NEXT:    vldrw.u32 q5, [r0]
-; CHECK-NEXT:    ldr r0, [sp, #24] @ 4-byte Reload
+; CHECK-NEXT:    lsls r4, r1, #1
+; CHECK-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    vshl.i32 q6, q4, #2
 ; CHECK-NEXT:    movs r1, #0
-; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
 ; CHECK-NEXT:    b .LBB10_5
 ; CHECK-NEXT:  .LBB10_3: @ %for.cond5.preheader.us73.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB10_5 Depth=1
-; CHECK-NEXT:    ldr r0, [sp, #28] @ 4-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    mov r1, r4
 ; CHECK-NEXT:    add.w r0, r0, r12, lsl #1
 ; CHECK-NEXT:    bl __aeabi_memclr
-; CHECK-NEXT:    ldr.w lr, [sp, #120]
 ; CHECK-NEXT:  .LBB10_4: @ %for.cond1.for.cond.cleanup3_crit_edge.us
 ; CHECK-NEXT:    @ in Loop: Header=BB10_5 Depth=1
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    add r9, r11
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    add r1, r0
-; CHECK-NEXT:    str r1, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    add r1, r0
+; CHECK-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:    adds r1, #1
 ; CHECK-NEXT:    cmp r1, r0
 ; CHECK-NEXT:    beq .LBB10_1
@@ -653,21 +652,23 @@ define dso_local void @arm_mat_mult_q15(i16* noalias nocapture readonly %A, i16*
 ; CHECK-NEXT:    @ Child Loop BB10_8 Depth 2
 ; CHECK-NEXT:    @ Child Loop BB10_11 Depth 3
 ; CHECK-NEXT:    @ Child Loop BB10_14 Depth 3
-; CHECK-NEXT:    mul r12, r1, lr
+; CHECK-NEXT:    ldr r0, [sp, #112]
 ; CHECK-NEXT:    cmp.w r11, #0
-; CHECK-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    mul r12, r1, r0
 ; CHECK-NEXT:    beq .LBB10_3
 ; CHECK-NEXT:  @ %bb.6: @ %for.cond5.preheader.us.us.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB10_5 Depth=1
-; CHECK-NEXT:    mov.w r10, #0
+; CHECK-NEXT:    movs r1, #0
 ; CHECK-NEXT:    b .LBB10_8
 ; CHECK-NEXT:  .LBB10_7: @ %for.cond5.for.cond.cleanup7_crit_edge.us.us
 ; CHECK-NEXT:    @ in Loop: Header=BB10_8 Depth=2
-; CHECK-NEXT:    ldr r0, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    add.w r3, r10, r12
-; CHECK-NEXT:    add.w r10, r10, #1
-; CHECK-NEXT:    cmp r10, lr
+; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    add.w r3, r1, r12
+; CHECK-NEXT:    adds r1, #1
 ; CHECK-NEXT:    strh.w r2, [r0, r3, lsl #1]
+; CHECK-NEXT:    ldr r0, [sp, #112]
+; CHECK-NEXT:    cmp r1, r0
 ; CHECK-NEXT:    beq .LBB10_4
 ; CHECK-NEXT:  .LBB10_8: @ %for.cond5.preheader.us.us
 ; CHECK-NEXT:    @ Parent Loop BB10_5 Depth=1
@@ -677,15 +678,15 @@ define dso_local void @arm_mat_mult_q15(i16* noalias nocapture readonly %A, i16*
 ; CHECK-NEXT:    cmp.w r11, #3
 ; CHECK-NEXT:    bhi .LBB10_10
 ; CHECK-NEXT:  @ %bb.9: @ in Loop: Header=BB10_8 Depth=2
-; CHECK-NEXT:    movs r7, #0
+; CHECK-NEXT:    movs r5, #0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB10_13
 ; CHECK-NEXT:  .LBB10_10: @ %vector.ph
 ; CHECK-NEXT:    @ in Loop: Header=BB10_8 Depth=2
 ; CHECK-NEXT:    vmov q1, q4
-; CHECK-NEXT:    ldr r2, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
-; CHECK-NEXT:    vmlas.u32 q1, q5, r10
+; CHECK-NEXT:    vmlas.u32 q1, q5, r1
 ; CHECK-NEXT:    dls lr, r8
 ; CHECK-NEXT:  .LBB10_11: @ %vector.body
 ; CHECK-NEXT:    @ Parent Loop BB10_5 Depth=1
@@ -700,29 +701,29 @@ define dso_local void @arm_mat_mult_q15(i16* noalias nocapture readonly %A, i16*
 ; CHECK-NEXT:    le lr, .LBB10_11
 ; CHECK-NEXT:  @ %bb.12: @ %middle.block
 ; CHECK-NEXT:    @ in Loop: Header=BB10_8 Depth=2
-; CHECK-NEXT:    ldr r7, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    vaddv.u32 r2, q0
-; CHECK-NEXT:    ldr.w lr, [sp, #120]
-; CHECK-NEXT:    cmp r7, r11
+; CHECK-NEXT:    cmp r10, r11
+; CHECK-NEXT:    mov r5, r10
 ; CHECK-NEXT:    beq .LBB10_7
 ; CHECK-NEXT:  .LBB10_13: @ %for.body8.us.us.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB10_8 Depth=2
-; CHECK-NEXT:    mla r3, lr, r7, r10
-; CHECK-NEXT:    sub.w r5, r11, r7
-; CHECK-NEXT:    add r7, r9
-; CHECK-NEXT:    ldr r0, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    add.w r7, r0, r7, lsl #1
+; CHECK-NEXT:    ldr r0, [sp, #112]
+; CHECK-NEXT:    sub.w lr, r11, r5
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    mla r3, r0, r5, r1
+; CHECK-NEXT:    add r5, r9
+; CHECK-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    add.w r5, r0, r5, lsl #1
 ; CHECK-NEXT:    add.w r3, r6, r3, lsl #1
 ; CHECK-NEXT:  .LBB10_14: @ %for.body8.us.us
 ; CHECK-NEXT:    @ Parent Loop BB10_5 Depth=1
 ; CHECK-NEXT:    @ Parent Loop BB10_8 Depth=2
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=3
-; CHECK-NEXT:    ldrsh.w r1, [r3]
+; CHECK-NEXT:    ldrsh.w r0, [r3]
 ; CHECK-NEXT:    add r3, r4
-; CHECK-NEXT:    ldrsh r0, [r7], #2
-; CHECK-NEXT:    subs r5, #1
-; CHECK-NEXT:    smlabb r2, r1, r0, r2
-; CHECK-NEXT:    bne .LBB10_14
+; CHECK-NEXT:    ldrsh r7, [r5], #2
+; CHECK-NEXT:    smlabb r2, r0, r7, r2
+; CHECK-NEXT:    le lr, .LBB10_14
 ; CHECK-NEXT:    b .LBB10_7
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.15:
diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll b/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
index 066c0957f5263..a3f06e5d25377 100644
--- a/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
@@ -876,11 +876,11 @@ define void @foo_v4f32_v4f16(<4 x float> *%dest, <4 x i16> *%mask, <4 x half> *%
 ; CHECK-NEXT:    lsls.w r1, lr, #29
 ; CHECK-NEXT:    bmi .LBB18_7
 ; CHECK-NEXT:  .LBB18_4:
-; CHECK-NEXT:    vmov q2, q0
+; CHECK-NEXT:    vmov q1, q0
 ; CHECK-NEXT:    lsls.w r1, lr, #28
 ; CHECK-NEXT:    bmi .LBB18_8
 ; CHECK-NEXT:  .LBB18_5:
-; CHECK-NEXT:    vmov q1, q2
+; CHECK-NEXT:    vmov q2, q1
 ; CHECK-NEXT:    b .LBB18_9
 ; CHECK-NEXT:  .LBB18_6:
 ; CHECK-NEXT:    vmov q0, q1
@@ -889,37 +889,35 @@ define void @foo_v4f32_v4f16(<4 x float> *%dest, <4 x i16> *%mask, <4 x half> *%
 ; CHECK-NEXT:  .LBB18_7: @ %cond.load4
 ; CHECK-NEXT:    vmovx.f16 s4, s0
 ; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vldr.16 s8, [r2, #4]
 ; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vldr.16 s4, [r2, #4]
-; CHECK-NEXT:    vmov.16 q2[0], r1
+; CHECK-NEXT:    vmov.16 q1[0], r1
 ; CHECK-NEXT:    vmovx.f16 s0, s1
-; CHECK-NEXT:    vmov.16 q2[1], r3
-; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    vmov.16 q2[2], r1
+; CHECK-NEXT:    vmov.16 q1[1], r3
+; CHECK-NEXT:    vmov r1, s8
+; CHECK-NEXT:    vmov.16 q1[2], r1
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q2[3], r1
+; CHECK-NEXT:    vmov.16 q1[3], r1
 ; CHECK-NEXT:    lsls.w r1, lr, #28
 ; CHECK-NEXT:    bpl .LBB18_5
 ; CHECK-NEXT:  .LBB18_8: @ %cond.load7
-; CHECK-NEXT:    vmovx.f16 s0, s8
-; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmovx.f16 s0, s4
+; CHECK-NEXT:    vmov r3, s4
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q1[0], r3
+; CHECK-NEXT:    vmov.16 q2[0], r3
 ; CHECK-NEXT:    vldr.16 s0, [r2, #6]
-; CHECK-NEXT:    vmov.16 q1[1], r1
-; CHECK-NEXT:    vmov r1, s9
-; CHECK-NEXT:    vmov.16 q1[2], r1
+; CHECK-NEXT:    vmov.16 q2[1], r1
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vmov.16 q2[2], r1
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q1[3], r1
+; CHECK-NEXT:    vmov.16 q2[3], r1
 ; CHECK-NEXT:  .LBB18_9: @ %else8
 ; CHECK-NEXT:    vmrs r2, p0
-; CHECK-NEXT:    vmovx.f16 s0, s5
-; CHECK-NEXT:    vcvtb.f32.f16 s3, s0
-; CHECK-NEXT:    vmovx.f16 s8, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s5
 ; CHECK-NEXT:    movs r1, #0
-; CHECK-NEXT:    vcvtb.f32.f16 s1, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s4
+; CHECK-NEXT:    vcvtt.f32.f16 s3, s9
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s9
+; CHECK-NEXT:    vcvtt.f32.f16 s1, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s8
 ; CHECK-NEXT:    and r3, r2, #1
 ; CHECK-NEXT:    rsbs r3, r3, #0
 ; CHECK-NEXT:    bfi r1, r3, #0, #1
@@ -1004,11 +1002,11 @@ define void @foo_v4f32_v4f16_unaligned(<4 x float> *%dest, <4 x i16> *%mask, <4
 ; CHECK-NEXT:    lsls.w r1, lr, #29
 ; CHECK-NEXT:    bmi .LBB19_7
 ; CHECK-NEXT:  .LBB19_4:
-; CHECK-NEXT:    vmov q2, q0
+; CHECK-NEXT:    vmov q1, q0
 ; CHECK-NEXT:    lsls.w r1, lr, #28
 ; CHECK-NEXT:    bmi .LBB19_8
 ; CHECK-NEXT:  .LBB19_5:
-; CHECK-NEXT:    vmov q1, q2
+; CHECK-NEXT:    vmov q2, q1
 ; CHECK-NEXT:    b .LBB19_9
 ; CHECK-NEXT:  .LBB19_6:
 ; CHECK-NEXT:    vmov q0, q1
@@ -1017,37 +1015,35 @@ define void @foo_v4f32_v4f16_unaligned(<4 x float> *%dest, <4 x i16> *%mask, <4
 ; CHECK-NEXT:  .LBB19_7: @ %cond.load4
 ; CHECK-NEXT:    vmovx.f16 s4, s0
 ; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vldr.16 s8, [r2, #4]
 ; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vldr.16 s4, [r2, #4]
-; CHECK-NEXT:    vmov.16 q2[0], r1
+; CHECK-NEXT:    vmov.16 q1[0], r1
 ; CHECK-NEXT:    vmovx.f16 s0, s1
-; CHECK-NEXT:    vmov.16 q2[1], r3
-; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    vmov.16 q2[2], r1
+; CHECK-NEXT:    vmov.16 q1[1], r3
+; CHECK-NEXT:    vmov r1, s8
+; CHECK-NEXT:    vmov.16 q1[2], r1
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q2[3], r1
+; CHECK-NEXT:    vmov.16 q1[3], r1
 ; CHECK-NEXT:    lsls.w r1, lr, #28
 ; CHECK-NEXT:    bpl .LBB19_5
 ; CHECK-NEXT:  .LBB19_8: @ %cond.load7
-; CHECK-NEXT:    vmovx.f16 s0, s8
-; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmovx.f16 s0, s4
+; CHECK-NEXT:    vmov r3, s4
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q1[0], r3
+; CHECK-NEXT:    vmov.16 q2[0], r3
 ; CHECK-NEXT:    vldr.16 s0, [r2, #6]
-; CHECK-NEXT:    vmov.16 q1[1], r1
-; CHECK-NEXT:    vmov r1, s9
-; CHECK-NEXT:    vmov.16 q1[2], r1
+; CHECK-NEXT:    vmov.16 q2[1], r1
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vmov.16 q2[2], r1
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q1[3], r1
+; CHECK-NEXT:    vmov.16 q2[3], r1
 ; CHECK-NEXT:  .LBB19_9: @ %else8
 ; CHECK-NEXT:    vmrs r2, p0
-; CHECK-NEXT:    vmovx.f16 s0, s5
-; CHECK-NEXT:    vcvtb.f32.f16 s3, s0
-; CHECK-NEXT:    vmovx.f16 s8, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s5
 ; CHECK-NEXT:    movs r1, #0
-; CHECK-NEXT:    vcvtb.f32.f16 s1, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s4
+; CHECK-NEXT:    vcvtt.f32.f16 s3, s9
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s9
+; CHECK-NEXT:    vcvtt.f32.f16 s1, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s8
 ; CHECK-NEXT:    and r3, r2, #1
 ; CHECK-NEXT:    rsbs r3, r3, #0
 ; CHECK-NEXT:    bfi r1, r3, #0, #1
diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-store.ll b/llvm/test/CodeGen/Thumb2/mve-masked-store.ll
index 60366995453d8..3a905296fb48c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-masked-store.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-store.ll
@@ -1223,25 +1223,13 @@ define arm_aapcs_vfpcc void @masked_v4f16_align4(<4 x half> *%dest, <4 x float>
 ; CHECK-LE:       @ %bb.0: @ %entry
 ; CHECK-LE-NEXT:    .pad #4
 ; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcvtb.f16.f32 s4, s0
 ; CHECK-LE-NEXT:    vcmp.f32 s0, #0
-; CHECK-LE-NEXT:    vmov r1, s4
-; CHECK-LE-NEXT:    vcvtb.f16.f32 s4, s1
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q1[0], r1
-; CHECK-LE-NEXT:    vcvtb.f16.f32 s8, s2
-; CHECK-LE-NEXT:    vmov.16 q1[1], r2
-; CHECK-LE-NEXT:    vmov r1, s8
-; CHECK-LE-NEXT:    vcvtb.f16.f32 s8, s3
-; CHECK-LE-NEXT:    vmov.16 q1[2], r1
-; CHECK-LE-NEXT:    vmov r1, s8
-; CHECK-LE-NEXT:    vmov.16 q1[3], r1
+; CHECK-LE-NEXT:    movs r1, #0
 ; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-LE-NEXT:    mov.w r1, #0
-; CHECK-LE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-LE-NEXT:    it gt
 ; CHECK-LE-NEXT:    movgt r1, #1
 ; CHECK-LE-NEXT:    cmp r1, #0
+; CHECK-LE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-LE-NEXT:    cset r1, ne
 ; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-LE-NEXT:    and r1, r1, #1
@@ -1273,9 +1261,13 @@ define arm_aapcs_vfpcc void @masked_v4f16_align4(<4 x half> *%dest, <4 x float>
 ; CHECK-LE-NEXT:    cset r2, ne
 ; CHECK-LE-NEXT:    and r2, r2, #1
 ; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s4, s0
 ; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    vcvtt.f16.f32 s4, s1
 ; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s5, s2
+; CHECK-LE-NEXT:    vcvtt.f16.f32 s5, s3
 ; CHECK-LE-NEXT:    lsls r2, r1, #31
 ; CHECK-LE-NEXT:    bne .LBB25_5
 ; CHECK-LE-NEXT:  @ %bb.1: @ %else
@@ -1314,25 +1306,14 @@ define arm_aapcs_vfpcc void @masked_v4f16_align4(<4 x half> *%dest, <4 x float>
 ; CHECK-BE-NEXT:    .pad #4
 ; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    vcvtb.f16.f32 s0, s4
+; CHECK-BE-NEXT:    movs r1, #0
 ; CHECK-BE-NEXT:    vcmp.f32 s4, #0
-; CHECK-BE-NEXT:    vmov r1, s0
-; CHECK-BE-NEXT:    vcvtb.f16.f32 s0, s5
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q0[0], r1
-; CHECK-BE-NEXT:    vcvtb.f16.f32 s8, s6
-; CHECK-BE-NEXT:    vmov.16 q0[1], r2
-; CHECK-BE-NEXT:    vmov r1, s8
-; CHECK-BE-NEXT:    vcvtb.f16.f32 s8, s7
-; CHECK-BE-NEXT:    vmov.16 q0[2], r1
-; CHECK-BE-NEXT:    vmov r1, s8
-; CHECK-BE-NEXT:    vmov.16 q0[3], r1
+; CHECK-BE-NEXT:    movs r2, #0
 ; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-BE-NEXT:    mov.w r1, #0
-; CHECK-BE-NEXT:    vcmp.f32 s5, #0
 ; CHECK-BE-NEXT:    it gt
 ; CHECK-BE-NEXT:    movgt r1, #1
 ; CHECK-BE-NEXT:    cmp r1, #0
+; CHECK-BE-NEXT:    vcmp.f32 s5, #0
 ; CHECK-BE-NEXT:    cset r1, ne
 ; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-BE-NEXT:    and r1, r1, #1
@@ -1349,7 +1330,7 @@ define arm_aapcs_vfpcc void @masked_v4f16_align4(<4 x half> *%dest, <4 x float>
 ; CHECK-BE-NEXT:    and r3, r3, #1
 ; CHECK-BE-NEXT:    vcmp.f32 s7, #0
 ; CHECK-BE-NEXT:    rsb.w r3, r3, #0
-; CHECK-BE-NEXT:    mov.w r2, #0
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s0, s4
 ; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-BE-NEXT:    mov.w r3, #0
 ; CHECK-BE-NEXT:    it gt
@@ -1366,7 +1347,10 @@ define arm_aapcs_vfpcc void @masked_v4f16_align4(<4 x half> *%dest, <4 x float>
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    vcvtt.f16.f32 s0, s5
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s1, s6
+; CHECK-BE-NEXT:    vcvtt.f16.f32 s1, s7
 ; CHECK-BE-NEXT:    lsls r2, r1, #31
 ; CHECK-BE-NEXT:    bne .LBB25_5
 ; CHECK-BE-NEXT:  @ %bb.1: @ %else
@@ -1411,25 +1395,13 @@ define arm_aapcs_vfpcc void @masked_v4f16_align2(<4 x half> *%dest, <4 x float>
 ; CHECK-LE:       @ %bb.0: @ %entry
 ; CHECK-LE-NEXT:    .pad #4
 ; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcvtb.f16.f32 s4, s0
 ; CHECK-LE-NEXT:    vcmp.f32 s0, #0
-; CHECK-LE-NEXT:    vmov r1, s4
-; CHECK-LE-NEXT:    vcvtb.f16.f32 s4, s1
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q1[0], r1
-; CHECK-LE-NEXT:    vcvtb.f16.f32 s8, s2
-; CHECK-LE-NEXT:    vmov.16 q1[1], r2
-; CHECK-LE-NEXT:    vmov r1, s8
-; CHECK-LE-NEXT:    vcvtb.f16.f32 s8, s3
-; CHECK-LE-NEXT:    vmov.16 q1[2], r1
-; CHECK-LE-NEXT:    vmov r1, s8
-; CHECK-LE-NEXT:    vmov.16 q1[3], r1
+; CHECK-LE-NEXT:    movs r1, #0
 ; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-LE-NEXT:    mov.w r1, #0
-; CHECK-LE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-LE-NEXT:    it gt
 ; CHECK-LE-NEXT:    movgt r1, #1
 ; CHECK-LE-NEXT:    cmp r1, #0
+; CHECK-LE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-LE-NEXT:    cset r1, ne
 ; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-LE-NEXT:    and r1, r1, #1
@@ -1461,9 +1433,13 @@ define arm_aapcs_vfpcc void @masked_v4f16_align2(<4 x half> *%dest, <4 x float>
 ; CHECK-LE-NEXT:    cset r2, ne
 ; CHECK-LE-NEXT:    and r2, r2, #1
 ; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s4, s0
 ; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    vcvtt.f16.f32 s4, s1
 ; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s5, s2
+; CHECK-LE-NEXT:    vcvtt.f16.f32 s5, s3
 ; CHECK-LE-NEXT:    lsls r2, r1, #31
 ; CHECK-LE-NEXT:    bne .LBB26_5
 ; CHECK-LE-NEXT:  @ %bb.1: @ %else
@@ -1502,25 +1478,14 @@ define arm_aapcs_vfpcc void @masked_v4f16_align2(<4 x half> *%dest, <4 x float>
 ; CHECK-BE-NEXT:    .pad #4
 ; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    vcvtb.f16.f32 s0, s4
+; CHECK-BE-NEXT:    movs r1, #0
 ; CHECK-BE-NEXT:    vcmp.f32 s4, #0
-; CHECK-BE-NEXT:    vmov r1, s0
-; CHECK-BE-NEXT:    vcvtb.f16.f32 s0, s5
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q0[0], r1
-; CHECK-BE-NEXT:    vcvtb.f16.f32 s8, s6
-; CHECK-BE-NEXT:    vmov.16 q0[1], r2
-; CHECK-BE-NEXT:    vmov r1, s8
-; CHECK-BE-NEXT:    vcvtb.f16.f32 s8, s7
-; CHECK-BE-NEXT:    vmov.16 q0[2], r1
-; CHECK-BE-NEXT:    vmov r1, s8
-; CHECK-BE-NEXT:    vmov.16 q0[3], r1
+; CHECK-BE-NEXT:    movs r2, #0
 ; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-BE-NEXT:    mov.w r1, #0
-; CHECK-BE-NEXT:    vcmp.f32 s5, #0
 ; CHECK-BE-NEXT:    it gt
 ; CHECK-BE-NEXT:    movgt r1, #1
 ; CHECK-BE-NEXT:    cmp r1, #0
+; CHECK-BE-NEXT:    vcmp.f32 s5, #0
 ; CHECK-BE-NEXT:    cset r1, ne
 ; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-BE-NEXT:    and r1, r1, #1
@@ -1537,7 +1502,7 @@ define arm_aapcs_vfpcc void @masked_v4f16_align2(<4 x half> *%dest, <4 x float>
 ; CHECK-BE-NEXT:    and r3, r3, #1
 ; CHECK-BE-NEXT:    vcmp.f32 s7, #0
 ; CHECK-BE-NEXT:    rsb.w r3, r3, #0
-; CHECK-BE-NEXT:    mov.w r2, #0
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s0, s4
 ; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-BE-NEXT:    mov.w r3, #0
 ; CHECK-BE-NEXT:    it gt
@@ -1554,7 +1519,10 @@ define arm_aapcs_vfpcc void @masked_v4f16_align2(<4 x half> *%dest, <4 x float>
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    vcvtt.f16.f32 s0, s5
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s1, s6
+; CHECK-BE-NEXT:    vcvtt.f16.f32 s1, s7
 ; CHECK-BE-NEXT:    lsls r2, r1, #31
 ; CHECK-BE-NEXT:    bne .LBB26_5
 ; CHECK-BE-NEXT:  @ %bb.1: @ %else
@@ -1599,25 +1567,13 @@ define arm_aapcs_vfpcc void @masked_v4f16_align1(<4 x half> *%dest, <4 x float>
 ; CHECK-LE:       @ %bb.0: @ %entry
 ; CHECK-LE-NEXT:    .pad #20
 ; CHECK-LE-NEXT:    sub sp, #20
-; CHECK-LE-NEXT:    vcvtb.f16.f32 s4, s0
 ; CHECK-LE-NEXT:    vcmp.f32 s0, #0
-; CHECK-LE-NEXT:    vmov r1, s4
-; CHECK-LE-NEXT:    vcvtb.f16.f32 s4, s1
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q1[0], r1
-; CHECK-LE-NEXT:    vcvtb.f16.f32 s8, s2
-; CHECK-LE-NEXT:    vmov.16 q1[1], r2
-; CHECK-LE-NEXT:    vmov r1, s8
-; CHECK-LE-NEXT:    vcvtb.f16.f32 s8, s3
-; CHECK-LE-NEXT:    vmov.16 q1[2], r1
-; CHECK-LE-NEXT:    vmov r1, s8
-; CHECK-LE-NEXT:    vmov.16 q1[3], r1
+; CHECK-LE-NEXT:    movs r1, #0
 ; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-LE-NEXT:    mov.w r1, #0
-; CHECK-LE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-LE-NEXT:    it gt
 ; CHECK-LE-NEXT:    movgt r1, #1
 ; CHECK-LE-NEXT:    cmp r1, #0
+; CHECK-LE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-LE-NEXT:    cset r1, ne
 ; CHECK-LE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-LE-NEXT:    and r1, r1, #1
@@ -1649,9 +1605,13 @@ define arm_aapcs_vfpcc void @masked_v4f16_align1(<4 x half> *%dest, <4 x float>
 ; CHECK-LE-NEXT:    cset r2, ne
 ; CHECK-LE-NEXT:    and r2, r2, #1
 ; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s4, s0
 ; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    vcvtt.f16.f32 s4, s1
 ; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-LE-NEXT:    vcvtb.f16.f32 s5, s2
+; CHECK-LE-NEXT:    vcvtt.f16.f32 s5, s3
 ; CHECK-LE-NEXT:    lsls r2, r1, #31
 ; CHECK-LE-NEXT:    bne .LBB27_5
 ; CHECK-LE-NEXT:  @ %bb.1: @ %else
@@ -1698,25 +1658,14 @@ define arm_aapcs_vfpcc void @masked_v4f16_align1(<4 x half> *%dest, <4 x float>
 ; CHECK-BE-NEXT:    .pad #20
 ; CHECK-BE-NEXT:    sub sp, #20
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    vcvtb.f16.f32 s0, s4
+; CHECK-BE-NEXT:    movs r1, #0
 ; CHECK-BE-NEXT:    vcmp.f32 s4, #0
-; CHECK-BE-NEXT:    vmov r1, s0
-; CHECK-BE-NEXT:    vcvtb.f16.f32 s0, s5
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q0[0], r1
-; CHECK-BE-NEXT:    vcvtb.f16.f32 s8, s6
-; CHECK-BE-NEXT:    vmov.16 q0[1], r2
-; CHECK-BE-NEXT:    vmov r1, s8
-; CHECK-BE-NEXT:    vcvtb.f16.f32 s8, s7
-; CHECK-BE-NEXT:    vmov.16 q0[2], r1
-; CHECK-BE-NEXT:    vmov r1, s8
-; CHECK-BE-NEXT:    vmov.16 q0[3], r1
+; CHECK-BE-NEXT:    movs r2, #0
 ; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-BE-NEXT:    mov.w r1, #0
-; CHECK-BE-NEXT:    vcmp.f32 s5, #0
 ; CHECK-BE-NEXT:    it gt
 ; CHECK-BE-NEXT:    movgt r1, #1
 ; CHECK-BE-NEXT:    cmp r1, #0
+; CHECK-BE-NEXT:    vcmp.f32 s5, #0
 ; CHECK-BE-NEXT:    cset r1, ne
 ; CHECK-BE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-BE-NEXT:    and r1, r1, #1
@@ -1733,7 +1682,7 @@ define arm_aapcs_vfpcc void @masked_v4f16_align1(<4 x half> *%dest, <4 x float>
 ; CHECK-BE-NEXT:    and r3, r3, #1
 ; CHECK-BE-NEXT:    vcmp.f32 s7, #0
 ; CHECK-BE-NEXT:    rsb.w r3, r3, #0
-; CHECK-BE-NEXT:    mov.w r2, #0
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s0, s4
 ; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-BE-NEXT:    mov.w r3, #0
 ; CHECK-BE-NEXT:    it gt
@@ -1750,7 +1699,10 @@ define arm_aapcs_vfpcc void @masked_v4f16_align1(<4 x half> *%dest, <4 x float>
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    vcvtt.f16.f32 s0, s5
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-BE-NEXT:    vcvtb.f16.f32 s1, s6
+; CHECK-BE-NEXT:    vcvtt.f16.f32 s1, s7
 ; CHECK-BE-NEXT:    lsls r2, r1, #31
 ; CHECK-BE-NEXT:    bne .LBB27_5
 ; CHECK-BE-NEXT:  @ %bb.1: @ %else
diff --git a/llvm/test/CodeGen/Thumb2/mve-shuffleext.ll b/llvm/test/CodeGen/Thumb2/mve-shuffleext.ll
index ebfffad7af18d..c7165e71b5dd9 100644
--- a/llvm/test/CodeGen/Thumb2/mve-shuffleext.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-shuffleext.ll
@@ -196,11 +196,7 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @fpext_0246(<8 x half> %src) {
 ; CHECK-LABEL: fpext_0246:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcvtb.f32.f16 s7, s3
-; CHECK-NEXT:    vcvtb.f32.f16 s6, s2
-; CHECK-NEXT:    vcvtb.f32.f16 s5, s1
-; CHECK-NEXT:    vcvtb.f32.f16 s4, s0
-; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    vcvtb.f32.f16 q0, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %strided.vec = shufflevector <8 x half> %src, <8 x half> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -211,14 +207,7 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @fpext_1357(<8 x half> %src) {
 ; CHECK-LABEL: fpext_1357:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s8, s3
-; CHECK-NEXT:    vmovx.f16 s4, s1
-; CHECK-NEXT:    vmovx.f16 s6, s0
-; CHECK-NEXT:    vmovx.f16 s10, s2
-; CHECK-NEXT:    vcvtb.f32.f16 s3, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s10
-; CHECK-NEXT:    vcvtb.f32.f16 s1, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s6
+; CHECK-NEXT:    vcvtt.f32.f16 q0, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %strided.vec = shufflevector <8 x half> %src, <8 x half> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -229,16 +218,8 @@ entry:
 define arm_aapcs_vfpcc <8 x float> @fpext_02468101214(<16 x half> %src) {
 ; CHECK-LABEL: fpext_02468101214:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcvtb.f32.f16 s11, s3
-; CHECK-NEXT:    vcvtb.f32.f16 s10, s2
-; CHECK-NEXT:    vcvtb.f32.f16 s9, s1
-; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s15, s7
-; CHECK-NEXT:    vcvtb.f32.f16 s14, s6
-; CHECK-NEXT:    vmov q0, q2
-; CHECK-NEXT:    vcvtb.f32.f16 s13, s5
-; CHECK-NEXT:    vcvtb.f32.f16 s12, s4
-; CHECK-NEXT:    vmov q1, q3
+; CHECK-NEXT:    vcvtb.f32.f16 q0, q0
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %strided.vec = shufflevector <16 x half> %src, <16 x half> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -249,22 +230,8 @@ entry:
 define arm_aapcs_vfpcc <8 x float> @fpext_13579111315(<16 x half> %src) {
 ; CHECK-LABEL: fpext_13579111315:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s14, s6
-; CHECK-NEXT:    vmovx.f16 s8, s5
-; CHECK-NEXT:    vmovx.f16 s5, s3
-; CHECK-NEXT:    vmovx.f16 s10, s4
-; CHECK-NEXT:    vmovx.f16 s12, s7
-; CHECK-NEXT:    vmovx.f16 s4, s1
-; CHECK-NEXT:    vmovx.f16 s6, s0
-; CHECK-NEXT:    vmovx.f16 s7, s2
-; CHECK-NEXT:    vcvtb.f32.f16 s3, s5
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s7
-; CHECK-NEXT:    vcvtb.f32.f16 s1, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s6
-; CHECK-NEXT:    vcvtb.f32.f16 s7, s12
-; CHECK-NEXT:    vcvtb.f32.f16 s6, s14
-; CHECK-NEXT:    vcvtb.f32.f16 s5, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s4, s10
+; CHECK-NEXT:    vcvtt.f32.f16 q0, q0
+; CHECK-NEXT:    vcvtt.f32.f16 q1, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %strided.vec = shufflevector <16 x half> %src, <16 x half> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcvt.ll b/llvm/test/CodeGen/Thumb2/mve-vcvt.ll
index 0cfe5cc80c1f6..ddc36f3be583f 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcvt.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcvt.ll
@@ -213,31 +213,31 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @foo_int16_half(<8 x half> %src) {
 ; CHECK-MVE-LABEL: foo_int16_half:
 ; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s0
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s0, s0
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s14, s14
+; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmovx.f16 s4, s3
 ; CHECK-MVE-NEXT:    vmovx.f16 s6, s2
 ; CHECK-MVE-NEXT:    vmovx.f16 s10, s1
-; CHECK-MVE-NEXT:    vmovx.f16 s14, s0
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s4
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s6, s6
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s8, s3
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s10, s10
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s12, s2
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s14, s14
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s5, s1
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s0, s0
-; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q0[0], r0
 ; CHECK-MVE-NEXT:    vmov r0, s14
 ; CHECK-MVE-NEXT:    vmov.16 q0[1], r0
 ; CHECK-MVE-NEXT:    vmov r0, s5
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s10, s10
 ; CHECK-MVE-NEXT:    vmov.16 q0[2], r0
 ; CHECK-MVE-NEXT:    vmov r0, s10
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s6, s6
 ; CHECK-MVE-NEXT:    vmov.16 q0[3], r0
 ; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    vmov.16 q0[4], r0
 ; CHECK-MVE-NEXT:    vmov r0, s6
 ; CHECK-MVE-NEXT:    vmov.16 q0[5], r0
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s4
 ; CHECK-MVE-NEXT:    vmov.16 q0[6], r0
 ; CHECK-MVE-NEXT:    vmov r0, s4
 ; CHECK-MVE-NEXT:    vmov.16 q0[7], r0
@@ -255,31 +255,31 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @foo_uint16_half(<8 x half> %src) {
 ; CHECK-MVE-LABEL: foo_uint16_half:
 ; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vmovx.f16 s14, s0
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s0, s0
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s14, s14
+; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmovx.f16 s4, s3
 ; CHECK-MVE-NEXT:    vmovx.f16 s6, s2
 ; CHECK-MVE-NEXT:    vmovx.f16 s10, s1
-; CHECK-MVE-NEXT:    vmovx.f16 s14, s0
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s4
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s6, s6
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s8, s3
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s10, s10
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s12, s2
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s14, s14
 ; CHECK-MVE-NEXT:    vcvt.s32.f16 s5, s1
-; CHECK-MVE-NEXT:    vcvt.s32.f16 s0, s0
-; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q0[0], r0
 ; CHECK-MVE-NEXT:    vmov r0, s14
 ; CHECK-MVE-NEXT:    vmov.16 q0[1], r0
 ; CHECK-MVE-NEXT:    vmov r0, s5
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s10, s10
 ; CHECK-MVE-NEXT:    vmov.16 q0[2], r0
 ; CHECK-MVE-NEXT:    vmov r0, s10
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s6, s6
 ; CHECK-MVE-NEXT:    vmov.16 q0[3], r0
 ; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    vmov.16 q0[4], r0
 ; CHECK-MVE-NEXT:    vmov r0, s6
 ; CHECK-MVE-NEXT:    vmov.16 q0[5], r0
 ; CHECK-MVE-NEXT:    vmov r0, s8
+; CHECK-MVE-NEXT:    vcvt.s32.f16 s4, s4
 ; CHECK-MVE-NEXT:    vmov.16 q0[6], r0
 ; CHECK-MVE-NEXT:    vmov r0, s4
 ; CHECK-MVE-NEXT:    vmov.16 q0[7], r0
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcvt16.ll b/llvm/test/CodeGen/Thumb2/mve-vcvt16.ll
index 06c21a2115cd1..c5421feb6c9d7 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcvt16.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcvt16.ll
@@ -4,11 +4,9 @@
 define arm_aapcs_vfpcc <4 x float> @fpext_4(<4 x half> %src1) {
 ; CHECK-LABEL: fpext_4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s4, s1
-; CHECK-NEXT:    vmovx.f16 s8, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s7, s4
+; CHECK-NEXT:    vcvtt.f32.f16 s7, s1
 ; CHECK-NEXT:    vcvtb.f32.f16 s6, s1
-; CHECK-NEXT:    vcvtb.f32.f16 s5, s8
+; CHECK-NEXT:    vcvtt.f32.f16 s5, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s4, s0
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
@@ -20,19 +18,15 @@ entry:
 define arm_aapcs_vfpcc <8 x float> @fpext_8(<8 x half> %src1) {
 ; CHECK-LABEL: fpext_8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s8, s1
-; CHECK-NEXT:    vmovx.f16 s6, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s11, s8
-; CHECK-NEXT:    vmovx.f16 s4, s3
-; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
-; CHECK-NEXT:    vmovx.f16 s12, s2
-; CHECK-NEXT:    vcvtb.f32.f16 s9, s6
-; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s7, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s6, s3
-; CHECK-NEXT:    vcvtb.f32.f16 s5, s12
-; CHECK-NEXT:    vcvtb.f32.f16 s4, s2
-; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmov q2, q0
+; CHECK-NEXT:    vcvtt.f32.f16 s3, s9
+; CHECK-NEXT:    vcvtt.f32.f16 s7, s11
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s9
+; CHECK-NEXT:    vcvtb.f32.f16 s6, s11
+; CHECK-NEXT:    vcvtt.f32.f16 s1, s8
+; CHECK-NEXT:    vcvtt.f32.f16 s5, s10
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s4, s10
 ; CHECK-NEXT:    bx lr
 entry:
   %out = fpext <8 x half> %src1 to <8 x float>
@@ -44,17 +38,9 @@ define arm_aapcs_vfpcc <4 x half> @fptrunc_4(<4 x float> %src1) {
 ; CHECK-LABEL: fptrunc_4:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vcvtb.f16.f32 s4, s0
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s1
-; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    vmov.16 q1[0], r0
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s2
-; CHECK-NEXT:    vmov.16 q1[1], r1
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s3
-; CHECK-NEXT:    vmov.16 q1[2], r0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vcvtt.f16.f32 s4, s1
+; CHECK-NEXT:    vcvtb.f16.f32 s5, s2
+; CHECK-NEXT:    vcvtt.f16.f32 s5, s3
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -67,29 +53,13 @@ define arm_aapcs_vfpcc <8 x half> @fptrunc_8(<8 x float> %src1) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmov q2, q0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s8
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s9
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s10
-; CHECK-NEXT:    vmov.16 q0[1], r1
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q0[2], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s4
-; CHECK-NEXT:    vmov.16 q0[3], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s5
-; CHECK-NEXT:    vmov.16 q0[4], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s6
-; CHECK-NEXT:    vmov.16 q0[5], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s7
-; CHECK-NEXT:    vmov.16 q0[6], r0
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vcvtt.f16.f32 s0, s9
+; CHECK-NEXT:    vcvtb.f16.f32 s1, s10
+; CHECK-NEXT:    vcvtt.f16.f32 s1, s11
+; CHECK-NEXT:    vcvtb.f16.f32 s2, s4
+; CHECK-NEXT:    vcvtt.f16.f32 s2, s5
+; CHECK-NEXT:    vcvtb.f16.f32 s3, s6
+; CHECK-NEXT:    vcvtt.f16.f32 s3, s7
 ; CHECK-NEXT:    bx lr
 entry:
   %out = fptrunc <8 x float> %src1 to <8 x half>
@@ -100,31 +70,8 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @shuffle_trunc1(<4 x float> %src1, <4 x float> %src2) {
 ; CHECK-LABEL: shuffle_trunc1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov q2, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s8
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s4
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s9
-; CHECK-NEXT:    vmov.16 q0[1], r1
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s5
-; CHECK-NEXT:    vmov.16 q0[2], r0
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s10
-; CHECK-NEXT:    vmov.16 q0[3], r0
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s6
-; CHECK-NEXT:    vmov.16 q0[4], r0
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q0[5], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s7
-; CHECK-NEXT:    vmov.16 q0[6], r0
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vcvtb.f16.f32 q0, q0
+; CHECK-NEXT:    vcvtt.f16.f32 q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %strided.vec = shufflevector <4 x float> %src1, <4 x float> %src2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
@@ -135,31 +82,9 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @shuffle_trunc2(<4 x float> %src1, <4 x float> %src2) {
 ; CHECK-LABEL: shuffle_trunc2:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s4
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s0
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s5
-; CHECK-NEXT:    vmov.16 q2[1], r1
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s1
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s6
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s2
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s7
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s3
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vcvtt.f16.f32 q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %strided.vec = shufflevector <4 x float> %src1, <4 x float> %src2, <8 x i32> <i32 4, i32 0, i32 5, i32 1, i32 6, i32 2, i32 7, i32 3>
@@ -170,59 +95,10 @@ entry:
 define arm_aapcs_vfpcc <16 x half> @shuffle_trunc3(<8 x float> %src1, <8 x float> %src2) {
 ; CHECK-LABEL: shuffle_trunc3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9, d10}
-; CHECK-NEXT:    vpush {d8, d9, d10}
-; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s16
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s8
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s17
-; CHECK-NEXT:    vmov.16 q0[1], r1
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s9
-; CHECK-NEXT:    vmov.16 q0[2], r0
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s18
-; CHECK-NEXT:    vmov.16 q0[3], r0
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s10
-; CHECK-NEXT:    vmov.16 q0[4], r0
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s19
-; CHECK-NEXT:    vmov.16 q0[5], r0
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q0[6], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s4
-; CHECK-NEXT:    vmov.16 q0[7], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s12
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s5
-; CHECK-NEXT:    vmov.16 q2[1], r1
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s13
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s6
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s14
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s7
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s15
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov q1, q2
-; CHECK-NEXT:    vpop {d8, d9, d10}
+; CHECK-NEXT:    vcvtb.f16.f32 q0, q0
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vcvtt.f16.f32 q0, q2
+; CHECK-NEXT:    vcvtt.f16.f32 q1, q3
 ; CHECK-NEXT:    bx lr
 entry:
   %strided.vec = shufflevector <8 x float> %src1, <8 x float> %src2, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
@@ -233,59 +109,12 @@ entry:
 define arm_aapcs_vfpcc <16 x half> @shuffle_trunc4(<8 x float> %src1, <8 x float> %src2) {
 ; CHECK-LABEL: shuffle_trunc4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9, d10}
-; CHECK-NEXT:    vpush {d8, d9, d10}
-; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s8
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s16
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s9
-; CHECK-NEXT:    vmov.16 q0[1], r1
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s17
-; CHECK-NEXT:    vmov.16 q0[2], r0
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s10
-; CHECK-NEXT:    vmov.16 q0[3], r0
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s18
-; CHECK-NEXT:    vmov.16 q0[4], r0
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q0[5], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s19
-; CHECK-NEXT:    vmov.16 q0[6], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s12
-; CHECK-NEXT:    vmov.16 q0[7], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s4
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s13
-; CHECK-NEXT:    vmov.16 q2[1], r1
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s5
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s14
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s6
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s15
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s7
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov q1, q2
-; CHECK-NEXT:    vpop {d8, d9, d10}
+; CHECK-NEXT:    vcvtb.f16.f32 q2, q2
+; CHECK-NEXT:    vcvtb.f16.f32 q3, q3
+; CHECK-NEXT:    vcvtt.f16.f32 q2, q0
+; CHECK-NEXT:    vcvtt.f16.f32 q3, q1
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmov q1, q3
 ; CHECK-NEXT:    bx lr
 entry:
   %strided.vec = shufflevector <8 x float> %src1, <8 x float> %src2, <16 x i32> <i32 8, i32 0, i32 9, i32 1, i32 10, i32 2, i32 11, i32 3, i32 12, i32 4, i32 13, i32 5, i32 14, i32 6, i32 15, i32 7>
@@ -296,31 +125,8 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @shuffle_trunc5(<4 x float> %src1, <4 x float> %src2) {
 ; CHECK-LABEL: shuffle_trunc5:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov q2, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s8
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s4
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s9
-; CHECK-NEXT:    vmov.16 q0[1], r1
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s5
-; CHECK-NEXT:    vmov.16 q0[2], r0
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s10
-; CHECK-NEXT:    vmov.16 q0[3], r0
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s6
-; CHECK-NEXT:    vmov.16 q0[4], r0
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q0[5], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s7
-; CHECK-NEXT:    vmov.16 q0[6], r0
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vcvtb.f16.f32 q0, q0
+; CHECK-NEXT:    vcvtt.f16.f32 q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %out1 = fptrunc <4 x float> %src1 to <4 x half>
@@ -332,31 +138,9 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @shuffle_trunc6(<4 x float> %src1, <4 x float> %src2) {
 ; CHECK-LABEL: shuffle_trunc6:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s4
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s0
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s5
-; CHECK-NEXT:    vmov.16 q2[1], r1
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s1
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s6
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s2
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s7
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s3
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vcvtt.f16.f32 q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %out1 = fptrunc <4 x float> %src1 to <4 x half>
@@ -368,59 +152,10 @@ entry:
 define arm_aapcs_vfpcc <16 x half> @shuffle_trunc7(<8 x float> %src1, <8 x float> %src2) {
 ; CHECK-LABEL: shuffle_trunc7:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9, d10}
-; CHECK-NEXT:    vpush {d8, d9, d10}
-; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s16
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s8
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s17
-; CHECK-NEXT:    vmov.16 q0[1], r1
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s9
-; CHECK-NEXT:    vmov.16 q0[2], r0
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s18
-; CHECK-NEXT:    vmov.16 q0[3], r0
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s10
-; CHECK-NEXT:    vmov.16 q0[4], r0
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s19
-; CHECK-NEXT:    vmov.16 q0[5], r0
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q0[6], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s4
-; CHECK-NEXT:    vmov.16 q0[7], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s12
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s5
-; CHECK-NEXT:    vmov.16 q2[1], r1
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s13
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s6
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s14
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s7
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s15
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov q1, q2
-; CHECK-NEXT:    vpop {d8, d9, d10}
+; CHECK-NEXT:    vcvtb.f16.f32 q0, q0
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vcvtt.f16.f32 q0, q2
+; CHECK-NEXT:    vcvtt.f16.f32 q1, q3
 ; CHECK-NEXT:    bx lr
 entry:
   %out1 = fptrunc <8 x float> %src1 to <8 x half>
@@ -432,59 +167,12 @@ entry:
 define arm_aapcs_vfpcc <16 x half> @shuffle_trunc8(<8 x float> %src1, <8 x float> %src2) {
 ; CHECK-LABEL: shuffle_trunc8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9, d10}
-; CHECK-NEXT:    vpush {d8, d9, d10}
-; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s8
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s16
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s9
-; CHECK-NEXT:    vmov.16 q0[1], r1
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s17
-; CHECK-NEXT:    vmov.16 q0[2], r0
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s10
-; CHECK-NEXT:    vmov.16 q0[3], r0
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s18
-; CHECK-NEXT:    vmov.16 q0[4], r0
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q0[5], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s19
-; CHECK-NEXT:    vmov.16 q0[6], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s12
-; CHECK-NEXT:    vmov.16 q0[7], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s4
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s13
-; CHECK-NEXT:    vmov.16 q2[1], r1
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s5
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s14
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s6
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s15
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s7
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov q1, q2
-; CHECK-NEXT:    vpop {d8, d9, d10}
+; CHECK-NEXT:    vcvtb.f16.f32 q2, q2
+; CHECK-NEXT:    vcvtb.f16.f32 q3, q3
+; CHECK-NEXT:    vcvtt.f16.f32 q2, q0
+; CHECK-NEXT:    vcvtt.f16.f32 q3, q1
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmov q1, q3
 ; CHECK-NEXT:    bx lr
 entry:
   %out1 = fptrunc <8 x float> %src1 to <8 x half>
@@ -499,15 +187,8 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @load_ext_4(<4 x half>* %src) {
 ; CHECK-LABEL: load_ext_4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    ldrd r1, r0, [r0]
-; CHECK-NEXT:    vmov.32 q1[0], r1
-; CHECK-NEXT:    vmov.32 q1[1], r0
-; CHECK-NEXT:    vmovx.f16 s0, s5
-; CHECK-NEXT:    vmovx.f16 s8, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s3, s0
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s5
-; CHECK-NEXT:    vcvtb.f32.f16 s1, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s4
+; CHECK-NEXT:    vldrh.u32 q0, [r0]
+; CHECK-NEXT:    vcvtb.f32.f16 q0, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %wide.load = load <4 x half>, <4 x half>* %src, align 4
@@ -518,19 +199,10 @@ entry:
 define arm_aapcs_vfpcc <8 x float> @load_ext_8(<8 x half>* %src) {
 ; CHECK-LABEL: load_ext_8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q2, [r0]
-; CHECK-NEXT:    vmovx.f16 s0, s9
-; CHECK-NEXT:    vmovx.f16 s6, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s3, s0
-; CHECK-NEXT:    vmovx.f16 s4, s11
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s9
-; CHECK-NEXT:    vmovx.f16 s12, s10
-; CHECK-NEXT:    vcvtb.f32.f16 s1, s6
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s7, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s6, s11
-; CHECK-NEXT:    vcvtb.f32.f16 s5, s12
-; CHECK-NEXT:    vcvtb.f32.f16 s4, s10
+; CHECK-NEXT:    vldrh.u32 q0, [r0]
+; CHECK-NEXT:    vldrh.u32 q1, [r0, #8]
+; CHECK-NEXT:    vcvtb.f32.f16 q0, q0
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %wide.load = load <8 x half>, <8 x half>* %src, align 4
@@ -541,35 +213,14 @@ entry:
 define arm_aapcs_vfpcc <16 x float> @load_ext_16(<16 x half>* %src) {
 ; CHECK-LABEL: load_ext_16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9, d10}
-; CHECK-NEXT:    vpush {d8, d9, d10}
-; CHECK-NEXT:    vldrw.u32 q2, [r0]
-; CHECK-NEXT:    vldrw.u32 q4, [r0, #16]
-; CHECK-NEXT:    vmovx.f16 s0, s9
-; CHECK-NEXT:    vmovx.f16 s6, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s3, s0
-; CHECK-NEXT:    vmovx.f16 s4, s11
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s9
-; CHECK-NEXT:    vmovx.f16 s15, s10
-; CHECK-NEXT:    vcvtb.f32.f16 s1, s6
-; CHECK-NEXT:    vmovx.f16 s13, s17
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s7, s4
-; CHECK-NEXT:    vcvtb.f32.f16 s6, s11
-; CHECK-NEXT:    vmovx.f16 s14, s16
-; CHECK-NEXT:    vcvtb.f32.f16 s5, s15
-; CHECK-NEXT:    vmovx.f16 s12, s19
-; CHECK-NEXT:    vcvtb.f32.f16 s4, s10
-; CHECK-NEXT:    vcvtb.f32.f16 s11, s13
-; CHECK-NEXT:    vcvtb.f32.f16 s10, s17
-; CHECK-NEXT:    vmovx.f16 s20, s18
-; CHECK-NEXT:    vcvtb.f32.f16 s9, s14
-; CHECK-NEXT:    vcvtb.f32.f16 s8, s16
-; CHECK-NEXT:    vcvtb.f32.f16 s15, s12
-; CHECK-NEXT:    vcvtb.f32.f16 s14, s19
-; CHECK-NEXT:    vcvtb.f32.f16 s13, s20
-; CHECK-NEXT:    vcvtb.f32.f16 s12, s18
-; CHECK-NEXT:    vpop {d8, d9, d10}
+; CHECK-NEXT:    vldrh.u32 q0, [r0]
+; CHECK-NEXT:    vldrh.u32 q1, [r0, #8]
+; CHECK-NEXT:    vldrh.u32 q2, [r0, #16]
+; CHECK-NEXT:    vldrh.u32 q3, [r0, #24]
+; CHECK-NEXT:    vcvtb.f32.f16 q0, q0
+; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vcvtb.f32.f16 q2, q2
+; CHECK-NEXT:    vcvtb.f32.f16 q3, q3
 ; CHECK-NEXT:    bx lr
 entry:
   %wide.load = load <16 x half>, <16 x half>* %src, align 4
@@ -580,11 +231,8 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @load_shuffleext_8(<8 x half>* %src) {
 ; CHECK-LABEL: load_shuffleext_8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    vcvtb.f32.f16 s3, s7
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s6
-; CHECK-NEXT:    vcvtb.f32.f16 s1, s5
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s4
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vcvtb.f32.f16 q0, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %wide.load = load <8 x half>, <8 x half>* %src, align 4
@@ -596,23 +244,16 @@ entry:
 define arm_aapcs_vfpcc <8 x float> @load_shuffleext_16(<16 x half>* %src) {
 ; CHECK-LABEL: load_shuffleext_16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8}
-; CHECK-NEXT:    vpush {d8}
 ; CHECK-NEXT:    vld20.16 {q2, q3}, [r0]
 ; CHECK-NEXT:    vld21.16 {q2, q3}, [r0]
-; CHECK-NEXT:    vmovx.f16 s0, s9
-; CHECK-NEXT:    vmovx.f16 s6, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s3, s0
-; CHECK-NEXT:    vmovx.f16 s4, s11
+; CHECK-NEXT:    vcvtt.f32.f16 s3, s9
+; CHECK-NEXT:    vcvtt.f32.f16 s7, s11
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s9
-; CHECK-NEXT:    vmovx.f16 s16, s10
-; CHECK-NEXT:    vcvtb.f32.f16 s1, s6
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s7, s4
 ; CHECK-NEXT:    vcvtb.f32.f16 s6, s11
-; CHECK-NEXT:    vcvtb.f32.f16 s5, s16
+; CHECK-NEXT:    vcvtt.f32.f16 s1, s8
+; CHECK-NEXT:    vcvtt.f32.f16 s5, s10
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s8
 ; CHECK-NEXT:    vcvtb.f32.f16 s4, s10
-; CHECK-NEXT:    vpop {d8}
 ; CHECK-NEXT:    bx lr
 entry:
   %wide.load = load <16 x half>, <16 x half>* %src, align 4
@@ -627,21 +268,8 @@ entry:
 define arm_aapcs_vfpcc void @store_trunc_4(<4 x half>* %src, <4 x float> %val) {
 ; CHECK-LABEL: store_trunc_4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s0
-; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s1
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vmov.16 q1[0], r1
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s2
-; CHECK-NEXT:    vmov.16 q1[1], r2
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s3
-; CHECK-NEXT:    vmov.16 q1[2], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q1[3], r1
-; CHECK-NEXT:    vmov r2, s5
-; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    strd r1, r2, [r0]
+; CHECK-NEXT:    vcvtb.f16.f32 q0, q0
+; CHECK-NEXT:    vstrh.32 q0, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %e = fptrunc <4 x float> %val to <4 x half>
@@ -652,31 +280,10 @@ entry:
 define arm_aapcs_vfpcc void @store_trunc_8(<8 x half>* %src, <8 x float> %val) {
 ; CHECK-LABEL: store_trunc_8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s0
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s1
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vmov.16 q2[0], r1
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s2
-; CHECK-NEXT:    vmov.16 q2[1], r2
-; CHECK-NEXT:    vmov r1, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s3
-; CHECK-NEXT:    vmov.16 q2[2], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s4
-; CHECK-NEXT:    vmov.16 q2[3], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s5
-; CHECK-NEXT:    vmov.16 q2[4], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s6
-; CHECK-NEXT:    vmov.16 q2[5], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s7
-; CHECK-NEXT:    vmov.16 q2[6], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q2[7], r1
-; CHECK-NEXT:    vstrw.32 q2, [r0]
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vcvtb.f16.f32 q0, q0
+; CHECK-NEXT:    vstrh.32 q1, [r0, #8]
+; CHECK-NEXT:    vstrh.32 q0, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %e = fptrunc <8 x float> %val to <8 x half>
@@ -687,59 +294,14 @@ entry:
 define arm_aapcs_vfpcc void @store_trunc_16(<16 x half>* %src, <16 x float> %val) {
 ; CHECK-LABEL: store_trunc_16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9, d10}
-; CHECK-NEXT:    vpush {d8, d9, d10}
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s8
-; CHECK-NEXT:    vmov r1, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s9
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vmov.16 q4[0], r1
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s10
-; CHECK-NEXT:    vmov.16 q4[1], r2
-; CHECK-NEXT:    vmov r1, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s11
-; CHECK-NEXT:    vmov.16 q4[2], r1
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s12
-; CHECK-NEXT:    vmov.16 q4[3], r1
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s13
-; CHECK-NEXT:    vmov.16 q4[4], r1
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s14
-; CHECK-NEXT:    vmov.16 q4[5], r1
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s15
-; CHECK-NEXT:    vmov.16 q4[6], r1
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vmov.16 q4[7], r1
-; CHECK-NEXT:    vstrw.32 q4, [r0, #16]
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s0
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s1
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vmov.16 q2[0], r1
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s2
-; CHECK-NEXT:    vmov.16 q2[1], r2
-; CHECK-NEXT:    vmov r1, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s3
-; CHECK-NEXT:    vmov.16 q2[2], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s4
-; CHECK-NEXT:    vmov.16 q2[3], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s5
-; CHECK-NEXT:    vmov.16 q2[4], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s6
-; CHECK-NEXT:    vmov.16 q2[5], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s7
-; CHECK-NEXT:    vmov.16 q2[6], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q2[7], r1
-; CHECK-NEXT:    vstrw.32 q2, [r0]
-; CHECK-NEXT:    vpop {d8, d9, d10}
+; CHECK-NEXT:    vcvtb.f16.f32 q3, q3
+; CHECK-NEXT:    vcvtb.f16.f32 q2, q2
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vcvtb.f16.f32 q0, q0
+; CHECK-NEXT:    vstrh.32 q3, [r0, #24]
+; CHECK-NEXT:    vstrh.32 q2, [r0, #16]
+; CHECK-NEXT:    vstrh.32 q1, [r0, #8]
+; CHECK-NEXT:    vstrh.32 q0, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %e = fptrunc <16 x float> %val to <16 x half>
@@ -750,31 +312,9 @@ entry:
 define arm_aapcs_vfpcc void @store_shuffletrunc_8(<8 x half>* %src, <4 x float> %val1, <4 x float> %val2) {
 ; CHECK-LABEL: store_shuffletrunc_8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s0
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vcvtb.f16.f32 s8, s4
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vmov.16 q2[0], r1
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s1
-; CHECK-NEXT:    vmov.16 q2[1], r2
-; CHECK-NEXT:    vmov r1, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s5
-; CHECK-NEXT:    vmov.16 q2[2], r1
-; CHECK-NEXT:    vmov r1, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s2
-; CHECK-NEXT:    vmov.16 q2[3], r1
-; CHECK-NEXT:    vmov r1, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s6
-; CHECK-NEXT:    vmov.16 q2[4], r1
-; CHECK-NEXT:    vmov r1, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s3
-; CHECK-NEXT:    vmov.16 q2[5], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s7
-; CHECK-NEXT:    vmov.16 q2[6], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q2[7], r1
-; CHECK-NEXT:    vstrw.32 q2, [r0]
+; CHECK-NEXT:    vcvtb.f16.f32 q0, q0
+; CHECK-NEXT:    vcvtt.f16.f32 q0, q1
+; CHECK-NEXT:    vstrw.32 q0, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %strided.vec = shufflevector <4 x float> %val1, <4 x float> %val2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
@@ -786,59 +326,12 @@ entry:
 define arm_aapcs_vfpcc void @store_shuffletrunc_16(<16 x half>* %src, <8 x float> %val1, <8 x float> %val2) {
 ; CHECK-LABEL: store_shuffletrunc_16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9, d10}
-; CHECK-NEXT:    vpush {d8, d9, d10}
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s4
-; CHECK-NEXT:    vmov r1, s16
-; CHECK-NEXT:    vcvtb.f16.f32 s16, s12
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vmov.16 q4[0], r1
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s5
-; CHECK-NEXT:    vmov.16 q4[1], r2
-; CHECK-NEXT:    vmov r1, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s13
-; CHECK-NEXT:    vmov.16 q4[2], r1
-; CHECK-NEXT:    vmov r1, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s6
-; CHECK-NEXT:    vmov.16 q4[3], r1
-; CHECK-NEXT:    vmov r1, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s20, s14
-; CHECK-NEXT:    vmov.16 q4[4], r1
-; CHECK-NEXT:    vmov r1, s20
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s7
-; CHECK-NEXT:    vmov.16 q4[5], r1
-; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s15
-; CHECK-NEXT:    vmov.16 q4[6], r1
-; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    vmov.16 q4[7], r1
-; CHECK-NEXT:    vstrw.32 q4, [r0, #16]
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s8
-; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s4, s0
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s1
-; CHECK-NEXT:    vmov.16 q1[0], r2
-; CHECK-NEXT:    vmov.16 q1[1], r1
-; CHECK-NEXT:    vmov r1, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s9
-; CHECK-NEXT:    vmov.16 q1[2], r1
-; CHECK-NEXT:    vmov r1, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s2
-; CHECK-NEXT:    vmov.16 q1[3], r1
-; CHECK-NEXT:    vmov r1, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s12, s10
-; CHECK-NEXT:    vmov.16 q1[4], r1
-; CHECK-NEXT:    vmov r1, s12
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s3
-; CHECK-NEXT:    vmov.16 q1[5], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vcvtb.f16.f32 s0, s11
-; CHECK-NEXT:    vmov.16 q1[6], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q1[7], r1
-; CHECK-NEXT:    vstrw.32 q1, [r0]
-; CHECK-NEXT:    vpop {d8, d9, d10}
+; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
+; CHECK-NEXT:    vcvtb.f16.f32 q0, q0
+; CHECK-NEXT:    vcvtt.f16.f32 q1, q3
+; CHECK-NEXT:    vcvtt.f16.f32 q0, q2
+; CHECK-NEXT:    vstrw.32 q1, [r0, #16]
+; CHECK-NEXT:    vstrw.32 q0, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %strided.vec = shufflevector <8 x float> %val1, <8 x float> %val2, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-bit.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-bit.ll
index 650db38d00891..fc06181978b70 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-bit.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-bit.ll
@@ -16,12 +16,12 @@ entry:
 define arm_aapcs_vfpcc i32 @and_v4i32(<4 x i32> %x) {
 ; CHECK-LABEL: and_v4i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    ands r0, r1
+; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    ands r1, r2
 ; CHECK-NEXT:    ands r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -33,12 +33,12 @@ define arm_aapcs_vfpcc i32 @and_v8i32(<8 x i32> %x) {
 ; CHECK-LABEL: and_v8i32:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vand q0, q0, q1
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    ands r0, r1
+; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    ands r1, r2
 ; CHECK-NEXT:    ands r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -49,12 +49,12 @@ entry:
 define arm_aapcs_vfpcc i16 @and_v4i16(<4 x i16> %x) {
 ; CHECK-LABEL: and_v4i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    ands r0, r1
+; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    ands r1, r2
 ; CHECK-NEXT:    ands r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -65,20 +65,14 @@ entry:
 define arm_aapcs_vfpcc i16 @and_v8i16(<8 x i16> %x) {
 ; CHECK-LABEL: and_v8i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[2]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[3]
-; CHECK-NEXT:    ands r0, r1
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
 ; CHECK-NEXT:    vmov.u16 r1, q0[4]
 ; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[6]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    vmov.u16 r2, q0[0]
+; CHECK-NEXT:    ands r1, r2
 ; CHECK-NEXT:    ands r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -90,20 +84,14 @@ define arm_aapcs_vfpcc i16 @and_v16i16(<16 x i16> %x) {
 ; CHECK-LABEL: and_v16i16:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vand q0, q0, q1
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[2]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[3]
-; CHECK-NEXT:    ands r0, r1
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
 ; CHECK-NEXT:    vmov.u16 r1, q0[4]
 ; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[6]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    vmov.u16 r2, q0[0]
+; CHECK-NEXT:    ands r1, r2
 ; CHECK-NEXT:    ands r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -114,20 +102,14 @@ entry:
 define arm_aapcs_vfpcc i8 @and_v8i8(<8 x i8> %x) {
 ; CHECK-LABEL: and_v8i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[2]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[3]
-; CHECK-NEXT:    ands r0, r1
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
 ; CHECK-NEXT:    vmov.u16 r1, q0[4]
 ; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[6]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    vmov.u16 r2, q0[0]
+; CHECK-NEXT:    ands r1, r2
 ; CHECK-NEXT:    ands r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -138,36 +120,16 @@ entry:
 define arm_aapcs_vfpcc i8 @and_v16i8(<16 x i8> %x) {
 ; CHECK-LABEL: and_v16i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u8 r0, q0[1]
-; CHECK-NEXT:    vmov.u8 r1, q0[0]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[2]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[3]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[4]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[5]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[6]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[7]
-; CHECK-NEXT:    ands r0, r1
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vrev32.8 q1, q0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
 ; CHECK-NEXT:    vmov.u8 r1, q0[8]
 ; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[9]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[10]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[11]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[12]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[13]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[14]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[15]
+; CHECK-NEXT:    vmov.u8 r1, q0[4]
+; CHECK-NEXT:    vmov.u8 r2, q0[0]
+; CHECK-NEXT:    ands r1, r2
 ; CHECK-NEXT:    ands r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -179,36 +141,16 @@ define arm_aapcs_vfpcc i8 @and_v32i8(<32 x i8> %x) {
 ; CHECK-LABEL: and_v32i8:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vand q0, q0, q1
-; CHECK-NEXT:    vmov.u8 r0, q0[1]
-; CHECK-NEXT:    vmov.u8 r1, q0[0]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[2]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[3]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[4]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[5]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[6]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[7]
-; CHECK-NEXT:    ands r0, r1
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vrev32.8 q1, q0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
 ; CHECK-NEXT:    vmov.u8 r1, q0[8]
 ; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[9]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[10]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[11]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[12]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[13]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[14]
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[15]
+; CHECK-NEXT:    vmov.u8 r1, q0[4]
+; CHECK-NEXT:    vmov.u8 r2, q0[0]
+; CHECK-NEXT:    ands r1, r2
 ; CHECK-NEXT:    ands r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -273,12 +215,12 @@ entry:
 define arm_aapcs_vfpcc i32 @and_v4i32_acc(<4 x i32> %x, i32 %y) {
 ; CHECK-LABEL: and_v4i32_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r1, s1
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    ands r1, r2
+; CHECK-NEXT:    vmov r1, s3
 ; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
 ; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    ands r2, r3
 ; CHECK-NEXT:    ands r1, r2
 ; CHECK-NEXT:    ands r0, r1
 ; CHECK-NEXT:    bx lr
@@ -292,12 +234,12 @@ define arm_aapcs_vfpcc i32 @and_v8i32_acc(<8 x i32> %x, i32 %y) {
 ; CHECK-LABEL: and_v8i32_acc:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vand q0, q0, q1
-; CHECK-NEXT:    vmov r1, s1
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    ands r1, r2
+; CHECK-NEXT:    vmov r1, s3
 ; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
 ; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    ands r2, r3
 ; CHECK-NEXT:    ands r1, r2
 ; CHECK-NEXT:    ands r0, r1
 ; CHECK-NEXT:    bx lr
@@ -310,12 +252,12 @@ entry:
 define arm_aapcs_vfpcc i16 @and_v4i16_acc(<4 x i16> %x, i16 %y) {
 ; CHECK-LABEL: and_v4i16_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r1, s1
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    ands r1, r2
+; CHECK-NEXT:    vmov r1, s3
 ; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
 ; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    ands r2, r3
 ; CHECK-NEXT:    ands r1, r2
 ; CHECK-NEXT:    ands r0, r1
 ; CHECK-NEXT:    bx lr
@@ -328,20 +270,14 @@ entry:
 define arm_aapcs_vfpcc i16 @and_v8i16_acc(<8 x i16> %x, i16 %y) {
 ; CHECK-LABEL: and_v8i16_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov.u16 r2, q0[0]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[2]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[3]
-; CHECK-NEXT:    ands r1, r2
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
 ; CHECK-NEXT:    vmov.u16 r2, q0[4]
 ; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[5]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[6]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.u16 r3, q0[0]
+; CHECK-NEXT:    ands r2, r3
 ; CHECK-NEXT:    ands r1, r2
 ; CHECK-NEXT:    ands r0, r1
 ; CHECK-NEXT:    bx lr
@@ -355,20 +291,14 @@ define arm_aapcs_vfpcc i16 @and_v16i16_acc(<16 x i16> %x, i16 %y) {
 ; CHECK-LABEL: and_v16i16_acc:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vand q0, q0, q1
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov.u16 r2, q0[0]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[2]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[3]
-; CHECK-NEXT:    ands r1, r2
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
 ; CHECK-NEXT:    vmov.u16 r2, q0[4]
 ; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[5]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[6]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.u16 r3, q0[0]
+; CHECK-NEXT:    ands r2, r3
 ; CHECK-NEXT:    ands r1, r2
 ; CHECK-NEXT:    ands r0, r1
 ; CHECK-NEXT:    bx lr
@@ -381,20 +311,14 @@ entry:
 define arm_aapcs_vfpcc i8 @and_v8i8_acc(<8 x i8> %x, i8 %y) {
 ; CHECK-LABEL: and_v8i8_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov.u16 r2, q0[0]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[2]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[3]
-; CHECK-NEXT:    ands r1, r2
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
 ; CHECK-NEXT:    vmov.u16 r2, q0[4]
 ; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[5]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[6]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.u16 r3, q0[0]
+; CHECK-NEXT:    ands r2, r3
 ; CHECK-NEXT:    ands r1, r2
 ; CHECK-NEXT:    ands r0, r1
 ; CHECK-NEXT:    bx lr
@@ -407,36 +331,16 @@ entry:
 define arm_aapcs_vfpcc i8 @and_v16i8_acc(<16 x i8> %x, i8 %y) {
 ; CHECK-LABEL: and_v16i8_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u8 r1, q0[1]
-; CHECK-NEXT:    vmov.u8 r2, q0[0]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[2]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[3]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[4]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[5]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[6]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[7]
-; CHECK-NEXT:    ands r1, r2
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vrev32.8 q1, q0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vmov.u8 r1, q0[12]
 ; CHECK-NEXT:    vmov.u8 r2, q0[8]
 ; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[9]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[10]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[11]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[12]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[13]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[14]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[15]
+; CHECK-NEXT:    vmov.u8 r2, q0[4]
+; CHECK-NEXT:    vmov.u8 r3, q0[0]
+; CHECK-NEXT:    ands r2, r3
 ; CHECK-NEXT:    ands r1, r2
 ; CHECK-NEXT:    ands r0, r1
 ; CHECK-NEXT:    bx lr
@@ -450,36 +354,16 @@ define arm_aapcs_vfpcc i8 @and_v32i8_acc(<32 x i8> %x, i8 %y) {
 ; CHECK-LABEL: and_v32i8_acc:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vand q0, q0, q1
-; CHECK-NEXT:    vmov.u8 r1, q0[1]
-; CHECK-NEXT:    vmov.u8 r2, q0[0]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[2]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[3]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[4]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[5]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[6]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[7]
-; CHECK-NEXT:    ands r1, r2
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vrev32.8 q1, q0
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vmov.u8 r1, q0[12]
 ; CHECK-NEXT:    vmov.u8 r2, q0[8]
 ; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[9]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[10]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[11]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[12]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[13]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[14]
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[15]
+; CHECK-NEXT:    vmov.u8 r2, q0[4]
+; CHECK-NEXT:    vmov.u8 r3, q0[0]
+; CHECK-NEXT:    ands r2, r3
 ; CHECK-NEXT:    ands r1, r2
 ; CHECK-NEXT:    ands r0, r1
 ; CHECK-NEXT:    bx lr
@@ -553,12 +437,12 @@ entry:
 define arm_aapcs_vfpcc i32 @or_v4i32(<4 x i32> %x) {
 ; CHECK-LABEL: or_v4i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    orrs r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -570,12 +454,12 @@ define arm_aapcs_vfpcc i32 @or_v8i32(<8 x i32> %x) {
 ; CHECK-LABEL: or_v8i32:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vorr q0, q0, q1
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    orrs r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -586,12 +470,12 @@ entry:
 define arm_aapcs_vfpcc i16 @or_v4i16(<4 x i16> %x) {
 ; CHECK-LABEL: or_v4i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    orrs r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -602,20 +486,14 @@ entry:
 define arm_aapcs_vfpcc i16 @or_v8i16(<8 x i16> %x) {
 ; CHECK-LABEL: or_v8i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[2]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[3]
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
 ; CHECK-NEXT:    vmov.u16 r1, q0[4]
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[6]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    vmov.u16 r2, q0[0]
+; CHECK-NEXT:    orrs r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -627,20 +505,14 @@ define arm_aapcs_vfpcc i16 @or_v16i16(<16 x i16> %x) {
 ; CHECK-LABEL: or_v16i16:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vorr q0, q0, q1
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[2]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[3]
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
 ; CHECK-NEXT:    vmov.u16 r1, q0[4]
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[6]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    vmov.u16 r2, q0[0]
+; CHECK-NEXT:    orrs r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -651,20 +523,14 @@ entry:
 define arm_aapcs_vfpcc i8 @or_v8i8(<8 x i8> %x) {
 ; CHECK-LABEL: or_v8i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[2]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[3]
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
 ; CHECK-NEXT:    vmov.u16 r1, q0[4]
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[6]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    vmov.u16 r2, q0[0]
+; CHECK-NEXT:    orrs r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -675,36 +541,16 @@ entry:
 define arm_aapcs_vfpcc i8 @or_v16i8(<16 x i8> %x) {
 ; CHECK-LABEL: or_v16i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u8 r0, q0[1]
-; CHECK-NEXT:    vmov.u8 r1, q0[0]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[2]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[3]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[4]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[5]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[6]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[7]
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    vrev32.8 q1, q0
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
 ; CHECK-NEXT:    vmov.u8 r1, q0[8]
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[9]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[10]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[11]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[12]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[13]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[14]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[15]
+; CHECK-NEXT:    vmov.u8 r1, q0[4]
+; CHECK-NEXT:    vmov.u8 r2, q0[0]
+; CHECK-NEXT:    orrs r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -716,36 +562,16 @@ define arm_aapcs_vfpcc i8 @or_v32i8(<32 x i8> %x) {
 ; CHECK-LABEL: or_v32i8:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vorr q0, q0, q1
-; CHECK-NEXT:    vmov.u8 r0, q0[1]
-; CHECK-NEXT:    vmov.u8 r1, q0[0]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[2]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[3]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[4]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[5]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[6]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[7]
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    vrev32.8 q1, q0
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
 ; CHECK-NEXT:    vmov.u8 r1, q0[8]
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[9]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[10]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[11]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[12]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[13]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[14]
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[15]
+; CHECK-NEXT:    vmov.u8 r1, q0[4]
+; CHECK-NEXT:    vmov.u8 r2, q0[0]
+; CHECK-NEXT:    orrs r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -810,12 +636,12 @@ entry:
 define arm_aapcs_vfpcc i32 @or_v4i32_acc(<4 x i32> %x, i32 %y) {
 ; CHECK-LABEL: or_v4i32_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r1, s1
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    orrs r1, r2
+; CHECK-NEXT:    vmov r1, s3
 ; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
 ; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    orrs r2, r3
 ; CHECK-NEXT:    orrs r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    bx lr
@@ -829,12 +655,12 @@ define arm_aapcs_vfpcc i32 @or_v8i32_acc(<8 x i32> %x, i32 %y) {
 ; CHECK-LABEL: or_v8i32_acc:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vorr q0, q0, q1
-; CHECK-NEXT:    vmov r1, s1
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    orrs r1, r2
+; CHECK-NEXT:    vmov r1, s3
 ; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
 ; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    orrs r2, r3
 ; CHECK-NEXT:    orrs r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    bx lr
@@ -847,12 +673,12 @@ entry:
 define arm_aapcs_vfpcc i16 @or_v4i16_acc(<4 x i16> %x, i16 %y) {
 ; CHECK-LABEL: or_v4i16_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r1, s1
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    orrs r1, r2
+; CHECK-NEXT:    vmov r1, s3
 ; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
 ; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    orrs r2, r3
 ; CHECK-NEXT:    orrs r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    bx lr
@@ -865,20 +691,14 @@ entry:
 define arm_aapcs_vfpcc i16 @or_v8i16_acc(<8 x i16> %x, i16 %y) {
 ; CHECK-LABEL: or_v8i16_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov.u16 r2, q0[0]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[2]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[3]
-; CHECK-NEXT:    orrs r1, r2
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
 ; CHECK-NEXT:    vmov.u16 r2, q0[4]
 ; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[5]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[6]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.u16 r3, q0[0]
+; CHECK-NEXT:    orrs r2, r3
 ; CHECK-NEXT:    orrs r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    bx lr
@@ -892,20 +712,14 @@ define arm_aapcs_vfpcc i16 @or_v16i16_acc(<16 x i16> %x, i16 %y) {
 ; CHECK-LABEL: or_v16i16_acc:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vorr q0, q0, q1
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov.u16 r2, q0[0]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[2]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[3]
-; CHECK-NEXT:    orrs r1, r2
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
 ; CHECK-NEXT:    vmov.u16 r2, q0[4]
 ; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[5]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[6]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.u16 r3, q0[0]
+; CHECK-NEXT:    orrs r2, r3
 ; CHECK-NEXT:    orrs r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    bx lr
@@ -918,20 +732,14 @@ entry:
 define arm_aapcs_vfpcc i8 @or_v8i8_acc(<8 x i8> %x, i8 %y) {
 ; CHECK-LABEL: or_v8i8_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov.u16 r2, q0[0]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[2]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[3]
-; CHECK-NEXT:    orrs r1, r2
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
 ; CHECK-NEXT:    vmov.u16 r2, q0[4]
 ; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[5]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[6]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.u16 r3, q0[0]
+; CHECK-NEXT:    orrs r2, r3
 ; CHECK-NEXT:    orrs r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    bx lr
@@ -944,36 +752,16 @@ entry:
 define arm_aapcs_vfpcc i8 @or_v16i8_acc(<16 x i8> %x, i8 %y) {
 ; CHECK-LABEL: or_v16i8_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u8 r1, q0[1]
-; CHECK-NEXT:    vmov.u8 r2, q0[0]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[2]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[3]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[4]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[5]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[6]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[7]
-; CHECK-NEXT:    orrs r1, r2
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    vrev32.8 q1, q0
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    vmov.u8 r1, q0[12]
 ; CHECK-NEXT:    vmov.u8 r2, q0[8]
 ; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[9]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[10]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[11]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[12]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[13]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[14]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[15]
+; CHECK-NEXT:    vmov.u8 r2, q0[4]
+; CHECK-NEXT:    vmov.u8 r3, q0[0]
+; CHECK-NEXT:    orrs r2, r3
 ; CHECK-NEXT:    orrs r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    bx lr
@@ -987,36 +775,16 @@ define arm_aapcs_vfpcc i8 @or_v32i8_acc(<32 x i8> %x, i8 %y) {
 ; CHECK-LABEL: or_v32i8_acc:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vorr q0, q0, q1
-; CHECK-NEXT:    vmov.u8 r1, q0[1]
-; CHECK-NEXT:    vmov.u8 r2, q0[0]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[2]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[3]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[4]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[5]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[6]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[7]
-; CHECK-NEXT:    orrs r1, r2
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    vrev32.8 q1, q0
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    vmov.u8 r1, q0[12]
 ; CHECK-NEXT:    vmov.u8 r2, q0[8]
 ; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[9]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[10]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[11]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[12]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[13]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[14]
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[15]
+; CHECK-NEXT:    vmov.u8 r2, q0[4]
+; CHECK-NEXT:    vmov.u8 r3, q0[0]
+; CHECK-NEXT:    orrs r2, r3
 ; CHECK-NEXT:    orrs r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    bx lr
@@ -1090,12 +858,12 @@ entry:
 define arm_aapcs_vfpcc i32 @xor_v4i32(<4 x i32> %x) {
 ; CHECK-LABEL: xor_v4i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -1107,12 +875,12 @@ define arm_aapcs_vfpcc i32 @xor_v8i32(<8 x i32> %x) {
 ; CHECK-LABEL: xor_v8i32:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    veor q0, q0, q1
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -1123,12 +891,12 @@ entry:
 define arm_aapcs_vfpcc i16 @xor_v4i16(<4 x i16> %x) {
 ; CHECK-LABEL: xor_v4i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -1139,20 +907,14 @@ entry:
 define arm_aapcs_vfpcc i16 @xor_v8i16(<8 x i16> %x) {
 ; CHECK-LABEL: xor_v8i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[2]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[3]
-; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    veor q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
 ; CHECK-NEXT:    vmov.u16 r1, q0[4]
 ; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[6]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    vmov.u16 r2, q0[0]
+; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -1164,20 +926,14 @@ define arm_aapcs_vfpcc i16 @xor_v16i16(<16 x i16> %x) {
 ; CHECK-LABEL: xor_v16i16:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    veor q0, q0, q1
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[2]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[3]
-; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    veor q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
 ; CHECK-NEXT:    vmov.u16 r1, q0[4]
 ; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[6]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    vmov.u16 r2, q0[0]
+; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -1188,20 +944,14 @@ entry:
 define arm_aapcs_vfpcc i8 @xor_v8i8(<8 x i8> %x) {
 ; CHECK-LABEL: xor_v8i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[2]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[3]
-; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    veor q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
 ; CHECK-NEXT:    vmov.u16 r1, q0[4]
 ; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[6]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    vmov.u16 r2, q0[0]
+; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -1212,36 +962,16 @@ entry:
 define arm_aapcs_vfpcc i8 @xor_v16i8(<16 x i8> %x) {
 ; CHECK-LABEL: xor_v16i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u8 r0, q0[1]
-; CHECK-NEXT:    vmov.u8 r1, q0[0]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[2]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[3]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[4]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[5]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[6]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[7]
-; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    veor q0, q0, q1
+; CHECK-NEXT:    vrev32.8 q1, q0
+; CHECK-NEXT:    veor q0, q0, q1
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
 ; CHECK-NEXT:    vmov.u8 r1, q0[8]
 ; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[9]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[10]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[11]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[12]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[13]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[14]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[15]
+; CHECK-NEXT:    vmov.u8 r1, q0[4]
+; CHECK-NEXT:    vmov.u8 r2, q0[0]
+; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -1253,36 +983,16 @@ define arm_aapcs_vfpcc i8 @xor_v32i8(<32 x i8> %x) {
 ; CHECK-LABEL: xor_v32i8:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    veor q0, q0, q1
-; CHECK-NEXT:    vmov.u8 r0, q0[1]
-; CHECK-NEXT:    vmov.u8 r1, q0[0]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[2]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[3]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[4]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[5]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[6]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[7]
-; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    veor q0, q0, q1
+; CHECK-NEXT:    vrev32.8 q1, q0
+; CHECK-NEXT:    veor q0, q0, q1
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
 ; CHECK-NEXT:    vmov.u8 r1, q0[8]
 ; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[9]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[10]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[11]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[12]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[13]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[14]
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q0[15]
+; CHECK-NEXT:    vmov.u8 r1, q0[4]
+; CHECK-NEXT:    vmov.u8 r2, q0[0]
+; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -1347,12 +1057,12 @@ entry:
 define arm_aapcs_vfpcc i32 @xor_v4i32_acc(<4 x i32> %x, i32 %y) {
 ; CHECK-LABEL: xor_v4i32_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r1, s1
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vmov r1, s3
 ; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
 ; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    eors r2, r3
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    bx lr
@@ -1366,12 +1076,12 @@ define arm_aapcs_vfpcc i32 @xor_v8i32_acc(<8 x i32> %x, i32 %y) {
 ; CHECK-LABEL: xor_v8i32_acc:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    veor q0, q0, q1
-; CHECK-NEXT:    vmov r1, s1
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vmov r1, s3
 ; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
 ; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    eors r2, r3
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    bx lr
@@ -1384,12 +1094,12 @@ entry:
 define arm_aapcs_vfpcc i16 @xor_v4i16_acc(<4 x i16> %x, i16 %y) {
 ; CHECK-LABEL: xor_v4i16_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r1, s1
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vmov r1, s3
 ; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
 ; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    eors r2, r3
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    bx lr
@@ -1402,20 +1112,14 @@ entry:
 define arm_aapcs_vfpcc i16 @xor_v8i16_acc(<8 x i16> %x, i16 %y) {
 ; CHECK-LABEL: xor_v8i16_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov.u16 r2, q0[0]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[2]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[3]
-; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    veor q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
 ; CHECK-NEXT:    vmov.u16 r2, q0[4]
 ; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[5]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[6]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.u16 r3, q0[0]
+; CHECK-NEXT:    eors r2, r3
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    bx lr
@@ -1429,20 +1133,14 @@ define arm_aapcs_vfpcc i16 @xor_v16i16_acc(<16 x i16> %x, i16 %y) {
 ; CHECK-LABEL: xor_v16i16_acc:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    veor q0, q0, q1
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov.u16 r2, q0[0]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[2]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[3]
-; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    veor q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
 ; CHECK-NEXT:    vmov.u16 r2, q0[4]
 ; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[5]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[6]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.u16 r3, q0[0]
+; CHECK-NEXT:    eors r2, r3
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    bx lr
@@ -1455,20 +1153,14 @@ entry:
 define arm_aapcs_vfpcc i8 @xor_v8i8_acc(<8 x i8> %x, i8 %y) {
 ; CHECK-LABEL: xor_v8i8_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov.u16 r2, q0[0]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[2]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[3]
-; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    veor q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
 ; CHECK-NEXT:    vmov.u16 r2, q0[4]
 ; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[5]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[6]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.u16 r3, q0[0]
+; CHECK-NEXT:    eors r2, r3
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    bx lr
@@ -1481,36 +1173,16 @@ entry:
 define arm_aapcs_vfpcc i8 @xor_v16i8_acc(<16 x i8> %x, i8 %y) {
 ; CHECK-LABEL: xor_v16i8_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u8 r1, q0[1]
-; CHECK-NEXT:    vmov.u8 r2, q0[0]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[2]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[3]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[4]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[5]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[6]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[7]
-; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    veor q0, q0, q1
+; CHECK-NEXT:    vrev32.8 q1, q0
+; CHECK-NEXT:    veor q0, q0, q1
+; CHECK-NEXT:    vmov.u8 r1, q0[12]
 ; CHECK-NEXT:    vmov.u8 r2, q0[8]
 ; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[9]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[10]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[11]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[12]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[13]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[14]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[15]
+; CHECK-NEXT:    vmov.u8 r2, q0[4]
+; CHECK-NEXT:    vmov.u8 r3, q0[0]
+; CHECK-NEXT:    eors r2, r3
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    bx lr
@@ -1524,36 +1196,16 @@ define arm_aapcs_vfpcc i8 @xor_v32i8_acc(<32 x i8> %x, i8 %y) {
 ; CHECK-LABEL: xor_v32i8_acc:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    veor q0, q0, q1
-; CHECK-NEXT:    vmov.u8 r1, q0[1]
-; CHECK-NEXT:    vmov.u8 r2, q0[0]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[2]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[3]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[4]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[5]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[6]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[7]
-; CHECK-NEXT:    eors r1, r2
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    veor q0, q0, q1
+; CHECK-NEXT:    vrev32.8 q1, q0
+; CHECK-NEXT:    veor q0, q0, q1
+; CHECK-NEXT:    vmov.u8 r1, q0[12]
 ; CHECK-NEXT:    vmov.u8 r2, q0[8]
 ; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[9]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[10]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[11]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[12]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[13]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[14]
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    vmov.u8 r2, q0[15]
+; CHECK-NEXT:    vmov.u8 r2, q0[4]
+; CHECK-NEXT:    vmov.u8 r3, q0[0]
+; CHECK-NEXT:    eors r2, r3
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    bx lr
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-fadd.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-fadd.ll
index d8d40e5b5186c..a1f25e0f33342 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-fadd.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-fadd.ll
@@ -3,31 +3,51 @@
 ; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve,+fullfp16,+fp64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NOFP
 
 define arm_aapcs_vfpcc float @fadd_v2f32(<2 x float> %x, float %y) {
-; CHECK-LABEL: fadd_v2f32:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vadd.f32 s0, s0, s1
-; CHECK-NEXT:    vldr s2, .LCPI0_0
-; CHECK-NEXT:    vadd.f32 s0, s0, s2
-; CHECK-NEXT:    vadd.f32 s0, s0, s2
-; CHECK-NEXT:    vadd.f32 s0, s4, s0
-; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 2
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI0_0:
-; CHECK-NEXT:    .long 0x00000000 @ float 0
+; CHECK-FP-LABEL: fadd_v2f32:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vadd.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vldr s2, .LCPI0_0
+; CHECK-FP-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-FP-NEXT:    vadd.f32 s0, s4, s0
+; CHECK-FP-NEXT:    bx lr
+; CHECK-FP-NEXT:    .p2align 2
+; CHECK-FP-NEXT:  @ %bb.1:
+; CHECK-FP-NEXT:  .LCPI0_0:
+; CHECK-FP-NEXT:    .long 0x00000000 @ float 0
+;
+; CHECK-NOFP-LABEL: fadd_v2f32:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vadd.f32 s0, s0, s1
+; CHECK-NOFP-NEXT:    vldr s2, .LCPI0_0
+; CHECK-NOFP-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-NOFP-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-NOFP-NEXT:    vadd.f32 s0, s4, s0
+; CHECK-NOFP-NEXT:    bx lr
+; CHECK-NOFP-NEXT:    .p2align 2
+; CHECK-NOFP-NEXT:  @ %bb.1:
+; CHECK-NOFP-NEXT:  .LCPI0_0:
+; CHECK-NOFP-NEXT:    .long 0x00000000 @ float 0
 entry:
   %z = call fast float @llvm.experimental.vector.reduce.v2.fadd.f32.v2f32(float %y, <2 x float> %x)
   ret float %z
 }
 
 define arm_aapcs_vfpcc float @fadd_v4f32(<4 x float> %x, float %y) {
-; CHECK-LABEL: fadd_v4f32:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vadd.f32 s6, s0, s1
-; CHECK-NEXT:    vadd.f32 s6, s6, s2
-; CHECK-NEXT:    vadd.f32 s0, s6, s3
-; CHECK-NEXT:    vadd.f32 s0, s4, s0
-; CHECK-NEXT:    bx lr
+; CHECK-FP-LABEL: fadd_v4f32:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vadd.f32 s6, s2, s3
+; CHECK-FP-NEXT:    vadd.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vadd.f32 s0, s0, s6
+; CHECK-FP-NEXT:    vadd.f32 s0, s4, s0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fadd_v4f32:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vadd.f32 s6, s0, s1
+; CHECK-NOFP-NEXT:    vadd.f32 s6, s6, s2
+; CHECK-NOFP-NEXT:    vadd.f32 s0, s6, s3
+; CHECK-NOFP-NEXT:    vadd.f32 s0, s4, s0
+; CHECK-NOFP-NEXT:    bx lr
 entry:
   %z = call fast float @llvm.experimental.vector.reduce.v2.fadd.f32.v4f32(float %y, <4 x float> %x)
   ret float %z
@@ -37,9 +57,9 @@ define arm_aapcs_vfpcc float @fadd_v8f32(<8 x float> %x, float %y) {
 ; CHECK-FP-LABEL: fadd_v8f32:
 ; CHECK-FP:       @ %bb.0: @ %entry
 ; CHECK-FP-NEXT:    vadd.f32 q0, q0, q1
-; CHECK-FP-NEXT:    vadd.f32 s4, s0, s1
-; CHECK-FP-NEXT:    vadd.f32 s4, s4, s2
-; CHECK-FP-NEXT:    vadd.f32 s0, s4, s3
+; CHECK-FP-NEXT:    vadd.f32 s4, s2, s3
+; CHECK-FP-NEXT:    vadd.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vadd.f32 s0, s0, s4
 ; CHECK-FP-NEXT:    vadd.f32 s0, s8, s0
 ; CHECK-FP-NEXT:    bx lr
 ;
@@ -59,27 +79,75 @@ entry:
   ret float %z
 }
 
+define arm_aapcs_vfpcc void @fadd_v2f16(<2 x half> %x, half* %yy) {
+; CHECK-FP-LABEL: fadd_v2f16:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmovx.f16 s4, s0
+; CHECK-FP-NEXT:    vadd.f16 s0, s0, s4
+; CHECK-FP-NEXT:    vldr.16 s2, [r0]
+; CHECK-FP-NEXT:    vadd.f16 s0, s2, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fadd_v2f16:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s0
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vldr.16 s2, .LCPI3_0
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+; CHECK-NOFP-NEXT:    .p2align 1
+; CHECK-NOFP-NEXT:  @ %bb.1:
+; CHECK-NOFP-NEXT:  .LCPI3_0:
+; CHECK-NOFP-NEXT:    .short 0x0000 @ half 0
+entry:
+  %y = load half, half* %yy
+  %z = call fast half @llvm.experimental.vector.reduce.v2.fadd.f16.v2f16(half %y, <2 x half> %x)
+  store half %z, half* %yy
+  ret void
+}
+
 define arm_aapcs_vfpcc void @fadd_v4f16(<4 x half> %x, half* %yy) {
-; CHECK-LABEL: fadd_v4f16:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vadd.f16 s4, s0, s4
-; CHECK-NEXT:    vmovx.f16 s0, s1
-; CHECK-NEXT:    vadd.f16 s4, s4, s1
-; CHECK-NEXT:    vldr.16 s2, .LCPI3_0
-; CHECK-NEXT:    vadd.f16 s0, s4, s0
-; CHECK-NEXT:    vadd.f16 s0, s0, s2
-; CHECK-NEXT:    vadd.f16 s0, s0, s2
-; CHECK-NEXT:    vadd.f16 s0, s0, s2
-; CHECK-NEXT:    vadd.f16 s0, s0, s2
-; CHECK-NEXT:    vldr.16 s2, [r0]
-; CHECK-NEXT:    vadd.f16 s0, s2, s0
-; CHECK-NEXT:    vstr.16 s0, [r0]
-; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 1
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI3_0:
-; CHECK-NEXT:    .short 0x0000 @ half 0
+; CHECK-FP-LABEL: fadd_v4f16:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmovx.f16 s4, s1
+; CHECK-FP-NEXT:    vmovx.f16 s6, s0
+; CHECK-FP-NEXT:    vadd.f16 s0, s0, s6
+; CHECK-FP-NEXT:    vadd.f16 s4, s1, s4
+; CHECK-FP-NEXT:    vldr.16 s2, [r0]
+; CHECK-FP-NEXT:    vadd.f16 s0, s0, s4
+; CHECK-FP-NEXT:    vadd.f16 s0, s2, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fadd_v4f16:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s0
+; CHECK-NOFP-NEXT:    vadd.f16 s4, s0, s4
+; CHECK-NOFP-NEXT:    vmovx.f16 s0, s1
+; CHECK-NOFP-NEXT:    vadd.f16 s4, s4, s1
+; CHECK-NOFP-NEXT:    vldr.16 s2, .LCPI4_0
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s4, s0
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+; CHECK-NOFP-NEXT:    .p2align 1
+; CHECK-NOFP-NEXT:  @ %bb.1:
+; CHECK-NOFP-NEXT:  .LCPI4_0:
+; CHECK-NOFP-NEXT:    .short 0x0000 @ half 0
 entry:
   %y = load half, half* %yy
   %z = call fast half @llvm.experimental.vector.reduce.v2.fadd.f16.v4f16(half %y, <4 x half> %x)
@@ -88,23 +156,35 @@ entry:
 }
 
 define arm_aapcs_vfpcc void @fadd_v8f16(<8 x half> %x, half* %yy) {
-; CHECK-LABEL: fadd_v8f16:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vmovx.f16 s6, s1
-; CHECK-NEXT:    vadd.f16 s4, s0, s4
-; CHECK-NEXT:    vmovx.f16 s0, s3
-; CHECK-NEXT:    vadd.f16 s4, s4, s1
-; CHECK-NEXT:    vadd.f16 s4, s4, s6
-; CHECK-NEXT:    vmovx.f16 s6, s2
-; CHECK-NEXT:    vadd.f16 s4, s4, s2
-; CHECK-NEXT:    vldr.16 s2, [r0]
-; CHECK-NEXT:    vadd.f16 s4, s4, s6
-; CHECK-NEXT:    vadd.f16 s4, s4, s3
-; CHECK-NEXT:    vadd.f16 s0, s4, s0
-; CHECK-NEXT:    vadd.f16 s0, s2, s0
-; CHECK-NEXT:    vstr.16 s0, [r0]
-; CHECK-NEXT:    bx lr
+; CHECK-FP-LABEL: fadd_v8f16:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vrev32.16 q1, q0
+; CHECK-FP-NEXT:    vadd.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vadd.f16 s4, s2, s3
+; CHECK-FP-NEXT:    vadd.f16 s0, s0, s1
+; CHECK-FP-NEXT:    vldr.16 s2, [r0]
+; CHECK-FP-NEXT:    vadd.f16 s0, s0, s4
+; CHECK-FP-NEXT:    vadd.f16 s0, s2, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fadd_v8f16:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s0
+; CHECK-NOFP-NEXT:    vmovx.f16 s6, s1
+; CHECK-NOFP-NEXT:    vadd.f16 s4, s0, s4
+; CHECK-NOFP-NEXT:    vmovx.f16 s0, s3
+; CHECK-NOFP-NEXT:    vadd.f16 s4, s4, s1
+; CHECK-NOFP-NEXT:    vadd.f16 s4, s4, s6
+; CHECK-NOFP-NEXT:    vmovx.f16 s6, s2
+; CHECK-NOFP-NEXT:    vadd.f16 s4, s4, s2
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vadd.f16 s4, s4, s6
+; CHECK-NOFP-NEXT:    vadd.f16 s4, s4, s3
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s4, s0
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
 entry:
   %y = load half, half* %yy
   %z = call fast half @llvm.experimental.vector.reduce.v2.fadd.f16.v8f16(half %y, <8 x half> %x)
@@ -116,18 +196,12 @@ define arm_aapcs_vfpcc void @fadd_v16f16(<16 x half> %x, half* %yy) {
 ; CHECK-FP-LABEL: fadd_v16f16:
 ; CHECK-FP:       @ %bb.0: @ %entry
 ; CHECK-FP-NEXT:    vadd.f16 q0, q0, q1
-; CHECK-FP-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP-NEXT:    vmovx.f16 s6, s1
-; CHECK-FP-NEXT:    vadd.f16 s4, s0, s4
-; CHECK-FP-NEXT:    vmovx.f16 s0, s3
-; CHECK-FP-NEXT:    vadd.f16 s4, s4, s1
-; CHECK-FP-NEXT:    vadd.f16 s4, s4, s6
-; CHECK-FP-NEXT:    vmovx.f16 s6, s2
-; CHECK-FP-NEXT:    vadd.f16 s4, s4, s2
+; CHECK-FP-NEXT:    vrev32.16 q1, q0
+; CHECK-FP-NEXT:    vadd.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vadd.f16 s4, s2, s3
+; CHECK-FP-NEXT:    vadd.f16 s0, s0, s1
 ; CHECK-FP-NEXT:    vldr.16 s2, [r0]
-; CHECK-FP-NEXT:    vadd.f16 s4, s4, s6
-; CHECK-FP-NEXT:    vadd.f16 s4, s4, s3
-; CHECK-FP-NEXT:    vadd.f16 s0, s4, s0
+; CHECK-FP-NEXT:    vadd.f16 s0, s0, s4
 ; CHECK-FP-NEXT:    vadd.f16 s0, s2, s0
 ; CHECK-FP-NEXT:    vstr.16 s0, [r0]
 ; CHECK-FP-NEXT:    bx lr
@@ -365,5 +439,6 @@ declare float @llvm.experimental.vector.reduce.v2.fadd.f32.v2f32(float, <2 x flo
 declare float @llvm.experimental.vector.reduce.v2.fadd.f32.v4f32(float, <4 x float>)
 declare float @llvm.experimental.vector.reduce.v2.fadd.f32.v8f32(float, <8 x float>)
 declare half @llvm.experimental.vector.reduce.v2.fadd.f16.v16f16(half, <16 x half>)
+declare half @llvm.experimental.vector.reduce.v2.fadd.f16.v2f16(half, <2 x half>)
 declare half @llvm.experimental.vector.reduce.v2.fadd.f16.v4f16(half, <4 x half>)
 declare half @llvm.experimental.vector.reduce.v2.fadd.f16.v8f16(half, <8 x half>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-fminmax.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-fminmax.ll
index 72636ad037dbc..6936b7ea3ad1f 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-fminmax.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-fminmax.ll
@@ -4,29 +4,47 @@
 
 ; FIXME minnum nonan X, +Inf -> X   ?
 define arm_aapcs_vfpcc float @fmin_v2f32(<2 x float> %x) {
-; CHECK-LABEL: fmin_v2f32:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldr s4, .LCPI0_0
-; CHECK-NEXT:    vminnm.f32 s0, s0, s1
-; CHECK-NEXT:    vminnm.f32 s0, s0, s4
-; CHECK-NEXT:    vminnm.f32 s0, s0, s4
-; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 2
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI0_0:
-; CHECK-NEXT:    .long 0x7f800000 @ float +Inf
+; CHECK-FP-LABEL: fmin_v2f32:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vldr s4, .LCPI0_0
+; CHECK-FP-NEXT:    vminnm.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vminnm.f32 s0, s0, s4
+; CHECK-FP-NEXT:    bx lr
+; CHECK-FP-NEXT:    .p2align 2
+; CHECK-FP-NEXT:  @ %bb.1:
+; CHECK-FP-NEXT:  .LCPI0_0:
+; CHECK-FP-NEXT:    .long 0x7f800000 @ float +Inf
+;
+; CHECK-NOFP-LABEL: fmin_v2f32:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vldr s4, .LCPI0_0
+; CHECK-NOFP-NEXT:    vminnm.f32 s0, s0, s1
+; CHECK-NOFP-NEXT:    vminnm.f32 s0, s0, s4
+; CHECK-NOFP-NEXT:    vminnm.f32 s0, s0, s4
+; CHECK-NOFP-NEXT:    bx lr
+; CHECK-NOFP-NEXT:    .p2align 2
+; CHECK-NOFP-NEXT:  @ %bb.1:
+; CHECK-NOFP-NEXT:  .LCPI0_0:
+; CHECK-NOFP-NEXT:    .long 0x7f800000 @ float +Inf
 entry:
   %z = call fast float @llvm.experimental.vector.reduce.fmin.v2f32(<2 x float> %x)
   ret float %z
 }
 
 define arm_aapcs_vfpcc float @fmin_v4f32(<4 x float> %x) {
-; CHECK-LABEL: fmin_v4f32:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vminnm.f32 s4, s0, s1
-; CHECK-NEXT:    vminnm.f32 s4, s4, s2
-; CHECK-NEXT:    vminnm.f32 s0, s4, s3
-; CHECK-NEXT:    bx lr
+; CHECK-FP-LABEL: fmin_v4f32:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vminnm.f32 s4, s2, s3
+; CHECK-FP-NEXT:    vminnm.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vminnm.f32 s0, s0, s4
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v4f32:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vminnm.f32 s4, s0, s1
+; CHECK-NOFP-NEXT:    vminnm.f32 s4, s4, s2
+; CHECK-NOFP-NEXT:    vminnm.f32 s0, s4, s3
+; CHECK-NOFP-NEXT:    bx lr
 entry:
   %z = call fast float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float> %x)
   ret float %z
@@ -37,9 +55,9 @@ define arm_aapcs_vfpcc float @fmin_v8f32(<8 x float> %x) {
 ; CHECK-FP-LABEL: fmin_v8f32:
 ; CHECK-FP:       @ %bb.0: @ %entry
 ; CHECK-FP-NEXT:    vminnm.f32 q0, q0, q1
-; CHECK-FP-NEXT:    vminnm.f32 s4, s0, s1
-; CHECK-FP-NEXT:    vminnm.f32 s4, s4, s2
-; CHECK-FP-NEXT:    vminnm.f32 s0, s4, s3
+; CHECK-FP-NEXT:    vminnm.f32 s4, s2, s3
+; CHECK-FP-NEXT:    vminnm.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vminnm.f32 s0, s0, s4
 ; CHECK-FP-NEXT:    bx lr
 ;
 ; CHECK-NOFP-LABEL: fmin_v8f32:
@@ -66,43 +84,61 @@ entry:
 }
 
 define arm_aapcs_vfpcc half @fmin_v4f16(<4 x half> %x) {
-; CHECK-LABEL: fmin_v4f16:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vminnm.f16 s4, s0, s4
-; CHECK-NEXT:    vmovx.f16 s0, s1
-; CHECK-NEXT:    vminnm.f16 s4, s4, s1
-; CHECK-NEXT:    vldr.16 s2, .LCPI3_0
-; CHECK-NEXT:    vminnm.f16 s0, s4, s0
-; CHECK-NEXT:    vminnm.f16 s0, s0, s2
-; CHECK-NEXT:    vminnm.f16 s0, s0, s2
-; CHECK-NEXT:    vminnm.f16 s0, s0, s2
-; CHECK-NEXT:    vminnm.f16 s0, s0, s2
-; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 1
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI3_0:
-; CHECK-NEXT:    .short 0x7c00 @ half +Inf
+; CHECK-FP-LABEL: fmin_v4f16:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmovx.f16 s4, s1
+; CHECK-FP-NEXT:    vmovx.f16 s6, s0
+; CHECK-FP-NEXT:    vminnm.f16 s4, s1, s4
+; CHECK-FP-NEXT:    vminnm.f16 s0, s0, s6
+; CHECK-FP-NEXT:    vminnm.f16 s0, s0, s4
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v4f16:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s0
+; CHECK-NOFP-NEXT:    vminnm.f16 s4, s0, s4
+; CHECK-NOFP-NEXT:    vmovx.f16 s0, s1
+; CHECK-NOFP-NEXT:    vminnm.f16 s4, s4, s1
+; CHECK-NOFP-NEXT:    vldr.16 s2, .LCPI3_0
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s4, s0
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    bx lr
+; CHECK-NOFP-NEXT:    .p2align 1
+; CHECK-NOFP-NEXT:  @ %bb.1:
+; CHECK-NOFP-NEXT:  .LCPI3_0:
+; CHECK-NOFP-NEXT:    .short 0x7c00 @ half +Inf
 entry:
   %z = call fast half @llvm.experimental.vector.reduce.fmin.v4f16(<4 x half> %x)
   ret half %z
 }
 
 define arm_aapcs_vfpcc half @fmin_v8f16(<8 x half> %x) {
-; CHECK-LABEL: fmin_v8f16:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vmovx.f16 s6, s1
-; CHECK-NEXT:    vminnm.f16 s4, s0, s4
-; CHECK-NEXT:    vmovx.f16 s0, s3
-; CHECK-NEXT:    vminnm.f16 s4, s4, s1
-; CHECK-NEXT:    vminnm.f16 s4, s4, s6
-; CHECK-NEXT:    vmovx.f16 s6, s2
-; CHECK-NEXT:    vminnm.f16 s4, s4, s2
-; CHECK-NEXT:    vminnm.f16 s4, s4, s6
-; CHECK-NEXT:    vminnm.f16 s4, s4, s3
-; CHECK-NEXT:    vminnm.f16 s0, s4, s0
-; CHECK-NEXT:    bx lr
+; CHECK-FP-LABEL: fmin_v8f16:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vrev32.16 q1, q0
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vminnm.f16 s4, s2, s3
+; CHECK-FP-NEXT:    vminnm.f16 s0, s0, s1
+; CHECK-FP-NEXT:    vminnm.f16 s0, s0, s4
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v8f16:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s0
+; CHECK-NOFP-NEXT:    vmovx.f16 s6, s1
+; CHECK-NOFP-NEXT:    vminnm.f16 s4, s0, s4
+; CHECK-NOFP-NEXT:    vmovx.f16 s0, s3
+; CHECK-NOFP-NEXT:    vminnm.f16 s4, s4, s1
+; CHECK-NOFP-NEXT:    vminnm.f16 s4, s4, s6
+; CHECK-NOFP-NEXT:    vmovx.f16 s6, s2
+; CHECK-NOFP-NEXT:    vminnm.f16 s4, s4, s2
+; CHECK-NOFP-NEXT:    vminnm.f16 s4, s4, s6
+; CHECK-NOFP-NEXT:    vminnm.f16 s4, s4, s3
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s4, s0
+; CHECK-NOFP-NEXT:    bx lr
 entry:
   %z = call fast half @llvm.experimental.vector.reduce.fmin.v8f16(<8 x half> %x)
   ret half %z
@@ -112,17 +148,11 @@ define arm_aapcs_vfpcc half @fmin_v16f16(<16 x half> %x) {
 ; CHECK-FP-LABEL: fmin_v16f16:
 ; CHECK-FP:       @ %bb.0: @ %entry
 ; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
-; CHECK-FP-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP-NEXT:    vmovx.f16 s6, s1
-; CHECK-FP-NEXT:    vminnm.f16 s4, s0, s4
-; CHECK-FP-NEXT:    vmovx.f16 s0, s3
-; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s1
-; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s6
-; CHECK-FP-NEXT:    vmovx.f16 s6, s2
-; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s2
-; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s6
-; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s3
-; CHECK-FP-NEXT:    vminnm.f16 s0, s4, s0
+; CHECK-FP-NEXT:    vrev32.16 q1, q0
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vminnm.f16 s4, s2, s3
+; CHECK-FP-NEXT:    vminnm.f16 s0, s0, s1
+; CHECK-FP-NEXT:    vminnm.f16 s0, s0, s4
 ; CHECK-FP-NEXT:    bx lr
 ;
 ; CHECK-NOFP-LABEL: fmin_v16f16:
@@ -502,18 +532,30 @@ entry:
 }
 
 define arm_aapcs_vfpcc float @fmin_v2f32_acc(<2 x float> %x, float %y) {
-; CHECK-LABEL: fmin_v2f32_acc:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldr s6, .LCPI18_0
-; CHECK-NEXT:    vminnm.f32 s0, s0, s1
-; CHECK-NEXT:    vminnm.f32 s0, s0, s6
-; CHECK-NEXT:    vminnm.f32 s0, s0, s6
-; CHECK-NEXT:    vminnm.f32 s0, s4, s0
-; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 2
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI18_0:
-; CHECK-NEXT:    .long 0x7f800000 @ float +Inf
+; CHECK-FP-LABEL: fmin_v2f32_acc:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vldr s6, .LCPI18_0
+; CHECK-FP-NEXT:    vminnm.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vminnm.f32 s0, s0, s6
+; CHECK-FP-NEXT:    vminnm.f32 s0, s4, s0
+; CHECK-FP-NEXT:    bx lr
+; CHECK-FP-NEXT:    .p2align 2
+; CHECK-FP-NEXT:  @ %bb.1:
+; CHECK-FP-NEXT:  .LCPI18_0:
+; CHECK-FP-NEXT:    .long 0x7f800000 @ float +Inf
+;
+; CHECK-NOFP-LABEL: fmin_v2f32_acc:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vldr s6, .LCPI18_0
+; CHECK-NOFP-NEXT:    vminnm.f32 s0, s0, s1
+; CHECK-NOFP-NEXT:    vminnm.f32 s0, s0, s6
+; CHECK-NOFP-NEXT:    vminnm.f32 s0, s0, s6
+; CHECK-NOFP-NEXT:    vminnm.f32 s0, s4, s0
+; CHECK-NOFP-NEXT:    bx lr
+; CHECK-NOFP-NEXT:    .p2align 2
+; CHECK-NOFP-NEXT:  @ %bb.1:
+; CHECK-NOFP-NEXT:  .LCPI18_0:
+; CHECK-NOFP-NEXT:    .long 0x7f800000 @ float +Inf
 entry:
   %z = call fast float @llvm.experimental.vector.reduce.fmin.v2f32(<2 x float> %x)
   %c = fcmp fast olt float %y, %z
@@ -522,13 +564,21 @@ entry:
 }
 
 define arm_aapcs_vfpcc float @fmin_v4f32_acc(<4 x float> %x, float %y) {
-; CHECK-LABEL: fmin_v4f32_acc:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vminnm.f32 s6, s0, s1
-; CHECK-NEXT:    vminnm.f32 s6, s6, s2
-; CHECK-NEXT:    vminnm.f32 s0, s6, s3
-; CHECK-NEXT:    vminnm.f32 s0, s4, s0
-; CHECK-NEXT:    bx lr
+; CHECK-FP-LABEL: fmin_v4f32_acc:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vminnm.f32 s6, s2, s3
+; CHECK-FP-NEXT:    vminnm.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vminnm.f32 s0, s0, s6
+; CHECK-FP-NEXT:    vminnm.f32 s0, s4, s0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v4f32_acc:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vminnm.f32 s6, s0, s1
+; CHECK-NOFP-NEXT:    vminnm.f32 s6, s6, s2
+; CHECK-NOFP-NEXT:    vminnm.f32 s0, s6, s3
+; CHECK-NOFP-NEXT:    vminnm.f32 s0, s4, s0
+; CHECK-NOFP-NEXT:    bx lr
 entry:
   %z = call fast float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float> %x)
   %c = fcmp fast olt float %y, %z
@@ -540,9 +590,9 @@ define arm_aapcs_vfpcc float @fmin_v8f32_acc(<8 x float> %x, float %y) {
 ; CHECK-FP-LABEL: fmin_v8f32_acc:
 ; CHECK-FP:       @ %bb.0: @ %entry
 ; CHECK-FP-NEXT:    vminnm.f32 q0, q0, q1
-; CHECK-FP-NEXT:    vminnm.f32 s4, s0, s1
-; CHECK-FP-NEXT:    vminnm.f32 s4, s4, s2
-; CHECK-FP-NEXT:    vminnm.f32 s0, s4, s3
+; CHECK-FP-NEXT:    vminnm.f32 s4, s2, s3
+; CHECK-FP-NEXT:    vminnm.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vminnm.f32 s0, s0, s4
 ; CHECK-FP-NEXT:    vminnm.f32 s0, s8, s0
 ; CHECK-FP-NEXT:    bx lr
 ;
@@ -573,26 +623,38 @@ entry:
 }
 
 define arm_aapcs_vfpcc void @fmin_v4f16_acc(<4 x half> %x, half* %yy) {
-; CHECK-LABEL: fmin_v4f16_acc:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vminnm.f16 s4, s0, s4
-; CHECK-NEXT:    vmovx.f16 s0, s1
-; CHECK-NEXT:    vminnm.f16 s4, s4, s1
-; CHECK-NEXT:    vldr.16 s2, .LCPI21_0
-; CHECK-NEXT:    vminnm.f16 s0, s4, s0
-; CHECK-NEXT:    vminnm.f16 s0, s0, s2
-; CHECK-NEXT:    vminnm.f16 s0, s0, s2
-; CHECK-NEXT:    vminnm.f16 s0, s0, s2
-; CHECK-NEXT:    vminnm.f16 s0, s0, s2
-; CHECK-NEXT:    vldr.16 s2, [r0]
-; CHECK-NEXT:    vminnm.f16 s0, s2, s0
-; CHECK-NEXT:    vstr.16 s0, [r0]
-; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 1
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI21_0:
-; CHECK-NEXT:    .short 0x7c00 @ half +Inf
+; CHECK-FP-LABEL: fmin_v4f16_acc:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmovx.f16 s4, s1
+; CHECK-FP-NEXT:    vmovx.f16 s6, s0
+; CHECK-FP-NEXT:    vminnm.f16 s0, s0, s6
+; CHECK-FP-NEXT:    vminnm.f16 s4, s1, s4
+; CHECK-FP-NEXT:    vldr.16 s2, [r0]
+; CHECK-FP-NEXT:    vminnm.f16 s0, s0, s4
+; CHECK-FP-NEXT:    vminnm.f16 s0, s2, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v4f16_acc:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s0
+; CHECK-NOFP-NEXT:    vminnm.f16 s4, s0, s4
+; CHECK-NOFP-NEXT:    vmovx.f16 s0, s1
+; CHECK-NOFP-NEXT:    vminnm.f16 s4, s4, s1
+; CHECK-NOFP-NEXT:    vldr.16 s2, .LCPI21_0
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s4, s0
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+; CHECK-NOFP-NEXT:    .p2align 1
+; CHECK-NOFP-NEXT:  @ %bb.1:
+; CHECK-NOFP-NEXT:  .LCPI21_0:
+; CHECK-NOFP-NEXT:    .short 0x7c00 @ half +Inf
 entry:
   %y = load half, half* %yy
   %z = call fast half @llvm.experimental.vector.reduce.fmin.v4f16(<4 x half> %x)
@@ -602,24 +664,74 @@ entry:
   ret void
 }
 
+define arm_aapcs_vfpcc void @fmin_v2f16_acc(<2 x half> %x, half* %yy) {
+; CHECK-FP-LABEL: fmin_v2f16_acc:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmovx.f16 s4, s0
+; CHECK-FP-NEXT:    vminnm.f16 s0, s0, s4
+; CHECK-FP-NEXT:    vldr.16 s2, [r0]
+; CHECK-FP-NEXT:    vminnm.f16 s0, s2, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v2f16_acc:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s0
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vldr.16 s2, .LCPI22_0
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+; CHECK-NOFP-NEXT:    .p2align 1
+; CHECK-NOFP-NEXT:  @ %bb.1:
+; CHECK-NOFP-NEXT:  .LCPI22_0:
+; CHECK-NOFP-NEXT:    .short 0x7c00 @ half +Inf
+entry:
+  %y = load half, half* %yy
+  %z = call fast half @llvm.experimental.vector.reduce.fmin.v2f16(<2 x half> %x)
+  %c = fcmp fast olt half %y, %z
+  %r = select i1 %c, half %y, half %z
+  store half %r, half* %yy
+  ret void
+}
+
 define arm_aapcs_vfpcc void @fmin_v8f16_acc(<8 x half> %x, half* %yy) {
-; CHECK-LABEL: fmin_v8f16_acc:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vmovx.f16 s6, s1
-; CHECK-NEXT:    vminnm.f16 s4, s0, s4
-; CHECK-NEXT:    vmovx.f16 s0, s3
-; CHECK-NEXT:    vminnm.f16 s4, s4, s1
-; CHECK-NEXT:    vminnm.f16 s4, s4, s6
-; CHECK-NEXT:    vmovx.f16 s6, s2
-; CHECK-NEXT:    vminnm.f16 s4, s4, s2
-; CHECK-NEXT:    vldr.16 s2, [r0]
-; CHECK-NEXT:    vminnm.f16 s4, s4, s6
-; CHECK-NEXT:    vminnm.f16 s4, s4, s3
-; CHECK-NEXT:    vminnm.f16 s0, s4, s0
-; CHECK-NEXT:    vminnm.f16 s0, s2, s0
-; CHECK-NEXT:    vstr.16 s0, [r0]
-; CHECK-NEXT:    bx lr
+; CHECK-FP-LABEL: fmin_v8f16_acc:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vrev32.16 q1, q0
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vminnm.f16 s4, s2, s3
+; CHECK-FP-NEXT:    vminnm.f16 s0, s0, s1
+; CHECK-FP-NEXT:    vldr.16 s2, [r0]
+; CHECK-FP-NEXT:    vminnm.f16 s0, s0, s4
+; CHECK-FP-NEXT:    vminnm.f16 s0, s2, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmin_v8f16_acc:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s0
+; CHECK-NOFP-NEXT:    vmovx.f16 s6, s1
+; CHECK-NOFP-NEXT:    vminnm.f16 s4, s0, s4
+; CHECK-NOFP-NEXT:    vmovx.f16 s0, s3
+; CHECK-NOFP-NEXT:    vminnm.f16 s4, s4, s1
+; CHECK-NOFP-NEXT:    vminnm.f16 s4, s4, s6
+; CHECK-NOFP-NEXT:    vmovx.f16 s6, s2
+; CHECK-NOFP-NEXT:    vminnm.f16 s4, s4, s2
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vminnm.f16 s4, s4, s6
+; CHECK-NOFP-NEXT:    vminnm.f16 s4, s4, s3
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s4, s0
+; CHECK-NOFP-NEXT:    vminnm.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
 entry:
   %y = load half, half* %yy
   %z = call fast half @llvm.experimental.vector.reduce.fmin.v8f16(<8 x half> %x)
@@ -633,18 +745,12 @@ define arm_aapcs_vfpcc void @fmin_v16f16_acc(<16 x half> %x, half* %yy) {
 ; CHECK-FP-LABEL: fmin_v16f16_acc:
 ; CHECK-FP:       @ %bb.0: @ %entry
 ; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
-; CHECK-FP-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP-NEXT:    vmovx.f16 s6, s1
-; CHECK-FP-NEXT:    vminnm.f16 s4, s0, s4
-; CHECK-FP-NEXT:    vmovx.f16 s0, s3
-; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s1
-; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s6
-; CHECK-FP-NEXT:    vmovx.f16 s6, s2
-; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s2
+; CHECK-FP-NEXT:    vrev32.16 q1, q0
+; CHECK-FP-NEXT:    vminnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vminnm.f16 s4, s2, s3
+; CHECK-FP-NEXT:    vminnm.f16 s0, s0, s1
 ; CHECK-FP-NEXT:    vldr.16 s2, [r0]
-; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s6
-; CHECK-FP-NEXT:    vminnm.f16 s4, s4, s3
-; CHECK-FP-NEXT:    vminnm.f16 s0, s4, s0
+; CHECK-FP-NEXT:    vminnm.f16 s0, s0, s4
 ; CHECK-FP-NEXT:    vminnm.f16 s0, s2, s0
 ; CHECK-FP-NEXT:    vstr.16 s0, [r0]
 ; CHECK-FP-NEXT:    bx lr
@@ -1115,29 +1221,47 @@ entry:
 }
 
 define arm_aapcs_vfpcc float @fmax_v2f32(<2 x float> %x) {
-; CHECK-LABEL: fmax_v2f32:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldr s4, .LCPI36_0
-; CHECK-NEXT:    vmaxnm.f32 s0, s0, s1
-; CHECK-NEXT:    vmaxnm.f32 s0, s0, s4
-; CHECK-NEXT:    vmaxnm.f32 s0, s0, s4
-; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 2
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI36_0:
-; CHECK-NEXT:    .long 0xff800000 @ float -Inf
+; CHECK-FP-LABEL: fmax_v2f32:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vldr s4, .LCPI37_0
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s0, s4
+; CHECK-FP-NEXT:    bx lr
+; CHECK-FP-NEXT:    .p2align 2
+; CHECK-FP-NEXT:  @ %bb.1:
+; CHECK-FP-NEXT:  .LCPI37_0:
+; CHECK-FP-NEXT:    .long 0xff800000 @ float -Inf
+;
+; CHECK-NOFP-LABEL: fmax_v2f32:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vldr s4, .LCPI37_0
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s0, s0, s1
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s0, s0, s4
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s0, s0, s4
+; CHECK-NOFP-NEXT:    bx lr
+; CHECK-NOFP-NEXT:    .p2align 2
+; CHECK-NOFP-NEXT:  @ %bb.1:
+; CHECK-NOFP-NEXT:  .LCPI37_0:
+; CHECK-NOFP-NEXT:    .long 0xff800000 @ float -Inf
 entry:
   %z = call fast float @llvm.experimental.vector.reduce.fmax.v2f32(<2 x float> %x)
   ret float %z
 }
 
 define arm_aapcs_vfpcc float @fmax_v4f32(<4 x float> %x) {
-; CHECK-LABEL: fmax_v4f32:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmaxnm.f32 s4, s0, s1
-; CHECK-NEXT:    vmaxnm.f32 s4, s4, s2
-; CHECK-NEXT:    vmaxnm.f32 s0, s4, s3
-; CHECK-NEXT:    bx lr
+; CHECK-FP-LABEL: fmax_v4f32:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmaxnm.f32 s4, s2, s3
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s0, s4
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v4f32:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s4, s0, s1
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s4, s4, s2
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s0, s4, s3
+; CHECK-NOFP-NEXT:    bx lr
 entry:
   %z = call fast float @llvm.experimental.vector.reduce.fmax.v4f32(<4 x float> %x)
   ret float %z
@@ -1147,9 +1271,9 @@ define arm_aapcs_vfpcc float @fmax_v8f32(<8 x float> %x) {
 ; CHECK-FP-LABEL: fmax_v8f32:
 ; CHECK-FP:       @ %bb.0: @ %entry
 ; CHECK-FP-NEXT:    vmaxnm.f32 q0, q0, q1
-; CHECK-FP-NEXT:    vmaxnm.f32 s4, s0, s1
-; CHECK-FP-NEXT:    vmaxnm.f32 s4, s4, s2
-; CHECK-FP-NEXT:    vmaxnm.f32 s0, s4, s3
+; CHECK-FP-NEXT:    vmaxnm.f32 s4, s2, s3
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s0, s4
 ; CHECK-FP-NEXT:    bx lr
 ;
 ; CHECK-NOFP-LABEL: fmax_v8f32:
@@ -1176,43 +1300,61 @@ entry:
 }
 
 define arm_aapcs_vfpcc half @fmax_v4f16(<4 x half> %x) {
-; CHECK-LABEL: fmax_v4f16:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vmaxnm.f16 s4, s0, s4
-; CHECK-NEXT:    vmovx.f16 s0, s1
-; CHECK-NEXT:    vmaxnm.f16 s4, s4, s1
-; CHECK-NEXT:    vldr.16 s2, .LCPI39_0
-; CHECK-NEXT:    vmaxnm.f16 s0, s4, s0
-; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
-; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
-; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
-; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
-; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 1
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI39_0:
-; CHECK-NEXT:    .short 0xfc00 @ half -Inf
+; CHECK-FP-LABEL: fmax_v4f16:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmovx.f16 s4, s1
+; CHECK-FP-NEXT:    vmovx.f16 s6, s0
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s1, s4
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s0, s6
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s0, s4
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v4f16:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s0
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s4, s0, s4
+; CHECK-NOFP-NEXT:    vmovx.f16 s0, s1
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s4, s4, s1
+; CHECK-NOFP-NEXT:    vldr.16 s2, .LCPI40_0
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s4, s0
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    bx lr
+; CHECK-NOFP-NEXT:    .p2align 1
+; CHECK-NOFP-NEXT:  @ %bb.1:
+; CHECK-NOFP-NEXT:  .LCPI40_0:
+; CHECK-NOFP-NEXT:    .short 0xfc00 @ half -Inf
 entry:
   %z = call fast half @llvm.experimental.vector.reduce.fmax.v4f16(<4 x half> %x)
   ret half %z
 }
 
 define arm_aapcs_vfpcc half @fmax_v8f16(<8 x half> %x) {
-; CHECK-LABEL: fmax_v8f16:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vmovx.f16 s6, s1
-; CHECK-NEXT:    vmaxnm.f16 s4, s0, s4
-; CHECK-NEXT:    vmovx.f16 s0, s3
-; CHECK-NEXT:    vmaxnm.f16 s4, s4, s1
-; CHECK-NEXT:    vmaxnm.f16 s4, s4, s6
-; CHECK-NEXT:    vmovx.f16 s6, s2
-; CHECK-NEXT:    vmaxnm.f16 s4, s4, s2
-; CHECK-NEXT:    vmaxnm.f16 s4, s4, s6
-; CHECK-NEXT:    vmaxnm.f16 s4, s4, s3
-; CHECK-NEXT:    vmaxnm.f16 s0, s4, s0
-; CHECK-NEXT:    bx lr
+; CHECK-FP-LABEL: fmax_v8f16:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vrev32.16 q1, q0
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s2, s3
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s0, s1
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s0, s4
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v8f16:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s0
+; CHECK-NOFP-NEXT:    vmovx.f16 s6, s1
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s4, s0, s4
+; CHECK-NOFP-NEXT:    vmovx.f16 s0, s3
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s4, s4, s1
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s4, s4, s6
+; CHECK-NOFP-NEXT:    vmovx.f16 s6, s2
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s4, s4, s2
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s4, s4, s6
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s4, s4, s3
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s4, s0
+; CHECK-NOFP-NEXT:    bx lr
 entry:
   %z = call fast half @llvm.experimental.vector.reduce.fmax.v8f16(<8 x half> %x)
   ret half %z
@@ -1222,17 +1364,11 @@ define arm_aapcs_vfpcc half @fmax_v16f16(<16 x half> %x) {
 ; CHECK-FP-LABEL: fmax_v16f16:
 ; CHECK-FP:       @ %bb.0: @ %entry
 ; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
-; CHECK-FP-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP-NEXT:    vmovx.f16 s6, s1
-; CHECK-FP-NEXT:    vmaxnm.f16 s4, s0, s4
-; CHECK-FP-NEXT:    vmovx.f16 s0, s3
-; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s1
-; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s6
-; CHECK-FP-NEXT:    vmovx.f16 s6, s2
-; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s2
-; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s6
-; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s3
-; CHECK-FP-NEXT:    vmaxnm.f16 s0, s4, s0
+; CHECK-FP-NEXT:    vrev32.16 q1, q0
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s2, s3
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s0, s1
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s0, s4
 ; CHECK-FP-NEXT:    bx lr
 ;
 ; CHECK-NOFP-LABEL: fmax_v16f16:
@@ -1610,18 +1746,30 @@ entry:
 }
 
 define arm_aapcs_vfpcc float @fmax_v2f32_acc(<2 x float> %x, float %y) {
-; CHECK-LABEL: fmax_v2f32_acc:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldr s6, .LCPI54_0
-; CHECK-NEXT:    vmaxnm.f32 s0, s0, s1
-; CHECK-NEXT:    vmaxnm.f32 s0, s0, s6
-; CHECK-NEXT:    vmaxnm.f32 s0, s0, s6
-; CHECK-NEXT:    vmaxnm.f32 s0, s4, s0
-; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 2
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI54_0:
-; CHECK-NEXT:    .long 0xff800000 @ float -Inf
+; CHECK-FP-LABEL: fmax_v2f32_acc:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vldr s6, .LCPI55_0
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s0, s6
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s4, s0
+; CHECK-FP-NEXT:    bx lr
+; CHECK-FP-NEXT:    .p2align 2
+; CHECK-FP-NEXT:  @ %bb.1:
+; CHECK-FP-NEXT:  .LCPI55_0:
+; CHECK-FP-NEXT:    .long 0xff800000 @ float -Inf
+;
+; CHECK-NOFP-LABEL: fmax_v2f32_acc:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vldr s6, .LCPI55_0
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s0, s0, s1
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s0, s0, s6
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s0, s0, s6
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s0, s4, s0
+; CHECK-NOFP-NEXT:    bx lr
+; CHECK-NOFP-NEXT:    .p2align 2
+; CHECK-NOFP-NEXT:  @ %bb.1:
+; CHECK-NOFP-NEXT:  .LCPI55_0:
+; CHECK-NOFP-NEXT:    .long 0xff800000 @ float -Inf
 entry:
   %z = call fast float @llvm.experimental.vector.reduce.fmax.v2f32(<2 x float> %x)
   %c = fcmp fast ogt float %y, %z
@@ -1630,13 +1778,21 @@ entry:
 }
 
 define arm_aapcs_vfpcc float @fmax_v4f32_acc(<4 x float> %x, float %y) {
-; CHECK-LABEL: fmax_v4f32_acc:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmaxnm.f32 s6, s0, s1
-; CHECK-NEXT:    vmaxnm.f32 s6, s6, s2
-; CHECK-NEXT:    vmaxnm.f32 s0, s6, s3
-; CHECK-NEXT:    vmaxnm.f32 s0, s4, s0
-; CHECK-NEXT:    bx lr
+; CHECK-FP-LABEL: fmax_v4f32_acc:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmaxnm.f32 s6, s2, s3
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s0, s6
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s4, s0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v4f32_acc:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s6, s0, s1
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s6, s6, s2
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s0, s6, s3
+; CHECK-NOFP-NEXT:    vmaxnm.f32 s0, s4, s0
+; CHECK-NOFP-NEXT:    bx lr
 entry:
   %z = call fast float @llvm.experimental.vector.reduce.fmax.v4f32(<4 x float> %x)
   %c = fcmp fast ogt float %y, %z
@@ -1648,9 +1804,9 @@ define arm_aapcs_vfpcc float @fmax_v8f32_acc(<8 x float> %x, float %y) {
 ; CHECK-FP-LABEL: fmax_v8f32_acc:
 ; CHECK-FP:       @ %bb.0: @ %entry
 ; CHECK-FP-NEXT:    vmaxnm.f32 q0, q0, q1
-; CHECK-FP-NEXT:    vmaxnm.f32 s4, s0, s1
-; CHECK-FP-NEXT:    vmaxnm.f32 s4, s4, s2
-; CHECK-FP-NEXT:    vmaxnm.f32 s0, s4, s3
+; CHECK-FP-NEXT:    vmaxnm.f32 s4, s2, s3
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vmaxnm.f32 s0, s0, s4
 ; CHECK-FP-NEXT:    vmaxnm.f32 s0, s8, s0
 ; CHECK-FP-NEXT:    bx lr
 ;
@@ -1680,27 +1836,77 @@ entry:
   ret float %r
 }
 
+define arm_aapcs_vfpcc void @fmax_v2f16_acc(<2 x half> %x, half* %yy) {
+; CHECK-FP-LABEL: fmax_v2f16_acc:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmovx.f16 s4, s0
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s0, s4
+; CHECK-FP-NEXT:    vldr.16 s2, [r0]
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s2, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v2f16_acc:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s0
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s0, s4
+; CHECK-NOFP-NEXT:    vldr.16 s2, .LCPI58_0
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+; CHECK-NOFP-NEXT:    .p2align 1
+; CHECK-NOFP-NEXT:  @ %bb.1:
+; CHECK-NOFP-NEXT:  .LCPI58_0:
+; CHECK-NOFP-NEXT:    .short 0xfc00 @ half -Inf
+entry:
+  %y = load half, half* %yy
+  %z = call fast half @llvm.experimental.vector.reduce.fmax.v2f16(<2 x half> %x)
+  %c = fcmp fast ogt half %y, %z
+  %r = select i1 %c, half %y, half %z
+  store half %r, half* %yy
+  ret void
+}
+
 define arm_aapcs_vfpcc void @fmax_v4f16_acc(<4 x half> %x, half* %yy) {
-; CHECK-LABEL: fmax_v4f16_acc:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vmaxnm.f16 s4, s0, s4
-; CHECK-NEXT:    vmovx.f16 s0, s1
-; CHECK-NEXT:    vmaxnm.f16 s4, s4, s1
-; CHECK-NEXT:    vldr.16 s2, .LCPI57_0
-; CHECK-NEXT:    vmaxnm.f16 s0, s4, s0
-; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
-; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
-; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
-; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
-; CHECK-NEXT:    vldr.16 s2, [r0]
-; CHECK-NEXT:    vmaxnm.f16 s0, s2, s0
-; CHECK-NEXT:    vstr.16 s0, [r0]
-; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 1
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI57_0:
-; CHECK-NEXT:    .short 0xfc00 @ half -Inf
+; CHECK-FP-LABEL: fmax_v4f16_acc:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmovx.f16 s4, s1
+; CHECK-FP-NEXT:    vmovx.f16 s6, s0
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s0, s6
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s1, s4
+; CHECK-FP-NEXT:    vldr.16 s2, [r0]
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s0, s4
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s2, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v4f16_acc:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s0
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s4, s0, s4
+; CHECK-NOFP-NEXT:    vmovx.f16 s0, s1
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s4, s4, s1
+; CHECK-NOFP-NEXT:    vldr.16 s2, .LCPI59_0
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s4, s0
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
+; CHECK-NOFP-NEXT:    .p2align 1
+; CHECK-NOFP-NEXT:  @ %bb.1:
+; CHECK-NOFP-NEXT:  .LCPI59_0:
+; CHECK-NOFP-NEXT:    .short 0xfc00 @ half -Inf
 entry:
   %y = load half, half* %yy
   %z = call fast half @llvm.experimental.vector.reduce.fmax.v4f16(<4 x half> %x)
@@ -1711,23 +1917,35 @@ entry:
 }
 
 define arm_aapcs_vfpcc void @fmax_v8f16_acc(<8 x half> %x, half* %yy) {
-; CHECK-LABEL: fmax_v8f16_acc:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vmovx.f16 s6, s1
-; CHECK-NEXT:    vmaxnm.f16 s4, s0, s4
-; CHECK-NEXT:    vmovx.f16 s0, s3
-; CHECK-NEXT:    vmaxnm.f16 s4, s4, s1
-; CHECK-NEXT:    vmaxnm.f16 s4, s4, s6
-; CHECK-NEXT:    vmovx.f16 s6, s2
-; CHECK-NEXT:    vmaxnm.f16 s4, s4, s2
-; CHECK-NEXT:    vldr.16 s2, [r0]
-; CHECK-NEXT:    vmaxnm.f16 s4, s4, s6
-; CHECK-NEXT:    vmaxnm.f16 s4, s4, s3
-; CHECK-NEXT:    vmaxnm.f16 s0, s4, s0
-; CHECK-NEXT:    vmaxnm.f16 s0, s2, s0
-; CHECK-NEXT:    vstr.16 s0, [r0]
-; CHECK-NEXT:    bx lr
+; CHECK-FP-LABEL: fmax_v8f16_acc:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vrev32.16 q1, q0
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s2, s3
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s0, s1
+; CHECK-FP-NEXT:    vldr.16 s2, [r0]
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s0, s4
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s2, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmax_v8f16_acc:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s0
+; CHECK-NOFP-NEXT:    vmovx.f16 s6, s1
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s4, s0, s4
+; CHECK-NOFP-NEXT:    vmovx.f16 s0, s3
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s4, s4, s1
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s4, s4, s6
+; CHECK-NOFP-NEXT:    vmovx.f16 s6, s2
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s4, s4, s2
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s4, s4, s6
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s4, s4, s3
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s4, s0
+; CHECK-NOFP-NEXT:    vmaxnm.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
 entry:
   %y = load half, half* %yy
   %z = call fast half @llvm.experimental.vector.reduce.fmax.v8f16(<8 x half> %x)
@@ -1741,18 +1959,12 @@ define arm_aapcs_vfpcc void @fmax_v16f16_acc(<16 x half> %x, half* %yy) {
 ; CHECK-FP-LABEL: fmax_v16f16_acc:
 ; CHECK-FP:       @ %bb.0: @ %entry
 ; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
-; CHECK-FP-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP-NEXT:    vmovx.f16 s6, s1
-; CHECK-FP-NEXT:    vmaxnm.f16 s4, s0, s4
-; CHECK-FP-NEXT:    vmovx.f16 s0, s3
-; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s1
-; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s6
-; CHECK-FP-NEXT:    vmovx.f16 s6, s2
-; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s2
+; CHECK-FP-NEXT:    vrev32.16 q1, q0
+; CHECK-FP-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmaxnm.f16 s4, s2, s3
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s0, s1
 ; CHECK-FP-NEXT:    vldr.16 s2, [r0]
-; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s6
-; CHECK-FP-NEXT:    vmaxnm.f16 s4, s4, s3
-; CHECK-FP-NEXT:    vmaxnm.f16 s0, s4, s0
+; CHECK-FP-NEXT:    vmaxnm.f16 s0, s0, s4
 ; CHECK-FP-NEXT:    vmaxnm.f16 s0, s2, s0
 ; CHECK-FP-NEXT:    vstr.16 s0, [r0]
 ; CHECK-FP-NEXT:    bx lr
@@ -2235,8 +2447,10 @@ declare float @llvm.experimental.vector.reduce.fmin.v2f32(<2 x float>)
 declare float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float>)
 declare float @llvm.experimental.vector.reduce.fmin.v8f32(<8 x float>)
 declare half @llvm.experimental.vector.reduce.fmax.v16f16(<16 x half>)
+declare half @llvm.experimental.vector.reduce.fmax.v2f16(<2 x half>)
 declare half @llvm.experimental.vector.reduce.fmax.v4f16(<4 x half>)
 declare half @llvm.experimental.vector.reduce.fmax.v8f16(<8 x half>)
 declare half @llvm.experimental.vector.reduce.fmin.v16f16(<16 x half>)
+declare half @llvm.experimental.vector.reduce.fmin.v2f16(<2 x half>)
 declare half @llvm.experimental.vector.reduce.fmin.v4f16(<4 x half>)
 declare half @llvm.experimental.vector.reduce.fmin.v8f16(<8 x half>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-fmul.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-fmul.ll
index e4ac03e9b3c81..89d1546932a4e 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-fmul.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-fmul.ll
@@ -14,13 +14,21 @@ entry:
 }
 
 define arm_aapcs_vfpcc float @fmul_v4f32(<4 x float> %x, float %y) {
-; CHECK-LABEL: fmul_v4f32:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmul.f32 s6, s0, s1
-; CHECK-NEXT:    vmul.f32 s6, s6, s2
-; CHECK-NEXT:    vmul.f32 s0, s6, s3
-; CHECK-NEXT:    vmul.f32 s0, s4, s0
-; CHECK-NEXT:    bx lr
+; CHECK-FP-LABEL: fmul_v4f32:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmul.f32 s6, s2, s3
+; CHECK-FP-NEXT:    vmul.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vmul.f32 s0, s0, s6
+; CHECK-FP-NEXT:    vmul.f32 s0, s4, s0
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmul_v4f32:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmul.f32 s6, s0, s1
+; CHECK-NOFP-NEXT:    vmul.f32 s6, s6, s2
+; CHECK-NOFP-NEXT:    vmul.f32 s0, s6, s3
+; CHECK-NOFP-NEXT:    vmul.f32 s0, s4, s0
+; CHECK-NOFP-NEXT:    bx lr
 entry:
   %z = call fast float @llvm.experimental.vector.reduce.v2.fmul.f32.v4f32(float %y, <4 x float> %x)
   ret float %z
@@ -30,9 +38,9 @@ define arm_aapcs_vfpcc float @fmul_v8f32(<8 x float> %x, float %y) {
 ; CHECK-FP-LABEL: fmul_v8f32:
 ; CHECK-FP:       @ %bb.0: @ %entry
 ; CHECK-FP-NEXT:    vmul.f32 q0, q0, q1
-; CHECK-FP-NEXT:    vmul.f32 s4, s0, s1
-; CHECK-FP-NEXT:    vmul.f32 s4, s4, s2
-; CHECK-FP-NEXT:    vmul.f32 s0, s4, s3
+; CHECK-FP-NEXT:    vmul.f32 s4, s2, s3
+; CHECK-FP-NEXT:    vmul.f32 s0, s0, s1
+; CHECK-FP-NEXT:    vmul.f32 s0, s0, s4
 ; CHECK-FP-NEXT:    vmul.f32 s0, s8, s0
 ; CHECK-FP-NEXT:    bx lr
 ;
@@ -52,18 +60,46 @@ entry:
   ret float %z
 }
 
-define arm_aapcs_vfpcc void @fmul_v4f16(<4 x half> %x, half* %yy) {
-; CHECK-LABEL: fmul_v4f16:
+define arm_aapcs_vfpcc void @fmul_v2f16(<2 x half> %x, half* %yy) {
+; CHECK-LABEL: fmul_v2f16:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vmul.f16 s4, s0, s4
-; CHECK-NEXT:    vmovx.f16 s0, s1
-; CHECK-NEXT:    vmul.f16 s4, s4, s1
+; CHECK-NEXT:    vmul.f16 s0, s0, s4
 ; CHECK-NEXT:    vldr.16 s2, [r0]
-; CHECK-NEXT:    vmul.f16 s0, s4, s0
 ; CHECK-NEXT:    vmul.f16 s0, s2, s0
 ; CHECK-NEXT:    vstr.16 s0, [r0]
 ; CHECK-NEXT:    bx lr
+entry:
+  %y = load half, half* %yy
+  %z = call fast half @llvm.experimental.vector.reduce.v2.fmul.f16.v2f16(half %y, <2 x half> %x)
+  store half %z, half* %yy
+  ret void
+}
+
+define arm_aapcs_vfpcc void @fmul_v4f16(<4 x half> %x, half* %yy) {
+; CHECK-FP-LABEL: fmul_v4f16:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vmovx.f16 s4, s1
+; CHECK-FP-NEXT:    vmovx.f16 s6, s0
+; CHECK-FP-NEXT:    vmul.f16 s0, s0, s6
+; CHECK-FP-NEXT:    vmul.f16 s4, s1, s4
+; CHECK-FP-NEXT:    vldr.16 s2, [r0]
+; CHECK-FP-NEXT:    vmul.f16 s0, s0, s4
+; CHECK-FP-NEXT:    vmul.f16 s0, s2, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmul_v4f16:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s0
+; CHECK-NOFP-NEXT:    vmul.f16 s4, s0, s4
+; CHECK-NOFP-NEXT:    vmovx.f16 s0, s1
+; CHECK-NOFP-NEXT:    vmul.f16 s4, s4, s1
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vmul.f16 s0, s4, s0
+; CHECK-NOFP-NEXT:    vmul.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
 entry:
   %y = load half, half* %yy
   %z = call fast half @llvm.experimental.vector.reduce.v2.fmul.f16.v4f16(half %y, <4 x half> %x)
@@ -72,23 +108,35 @@ entry:
 }
 
 define arm_aapcs_vfpcc void @fmul_v8f16(<8 x half> %x, half* %yy) {
-; CHECK-LABEL: fmul_v8f16:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vmovx.f16 s6, s1
-; CHECK-NEXT:    vmul.f16 s4, s0, s4
-; CHECK-NEXT:    vmovx.f16 s0, s3
-; CHECK-NEXT:    vmul.f16 s4, s4, s1
-; CHECK-NEXT:    vmul.f16 s4, s4, s6
-; CHECK-NEXT:    vmovx.f16 s6, s2
-; CHECK-NEXT:    vmul.f16 s4, s4, s2
-; CHECK-NEXT:    vldr.16 s2, [r0]
-; CHECK-NEXT:    vmul.f16 s4, s4, s6
-; CHECK-NEXT:    vmul.f16 s4, s4, s3
-; CHECK-NEXT:    vmul.f16 s0, s4, s0
-; CHECK-NEXT:    vmul.f16 s0, s2, s0
-; CHECK-NEXT:    vstr.16 s0, [r0]
-; CHECK-NEXT:    bx lr
+; CHECK-FP-LABEL: fmul_v8f16:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    vrev32.16 q1, q0
+; CHECK-FP-NEXT:    vmul.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmul.f16 s4, s2, s3
+; CHECK-FP-NEXT:    vmul.f16 s0, s0, s1
+; CHECK-FP-NEXT:    vldr.16 s2, [r0]
+; CHECK-FP-NEXT:    vmul.f16 s0, s0, s4
+; CHECK-FP-NEXT:    vmul.f16 s0, s2, s0
+; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-FP-NEXT:    bx lr
+;
+; CHECK-NOFP-LABEL: fmul_v8f16:
+; CHECK-NOFP:       @ %bb.0: @ %entry
+; CHECK-NOFP-NEXT:    vmovx.f16 s4, s0
+; CHECK-NOFP-NEXT:    vmovx.f16 s6, s1
+; CHECK-NOFP-NEXT:    vmul.f16 s4, s0, s4
+; CHECK-NOFP-NEXT:    vmovx.f16 s0, s3
+; CHECK-NOFP-NEXT:    vmul.f16 s4, s4, s1
+; CHECK-NOFP-NEXT:    vmul.f16 s4, s4, s6
+; CHECK-NOFP-NEXT:    vmovx.f16 s6, s2
+; CHECK-NOFP-NEXT:    vmul.f16 s4, s4, s2
+; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vmul.f16 s4, s4, s6
+; CHECK-NOFP-NEXT:    vmul.f16 s4, s4, s3
+; CHECK-NOFP-NEXT:    vmul.f16 s0, s4, s0
+; CHECK-NOFP-NEXT:    vmul.f16 s0, s2, s0
+; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NOFP-NEXT:    bx lr
 entry:
   %y = load half, half* %yy
   %z = call fast half @llvm.experimental.vector.reduce.v2.fmul.f16.v8f16(half %y, <8 x half> %x)
@@ -100,18 +148,12 @@ define arm_aapcs_vfpcc void @fmul_v16f16(<16 x half> %x, half* %yy) {
 ; CHECK-FP-LABEL: fmul_v16f16:
 ; CHECK-FP:       @ %bb.0: @ %entry
 ; CHECK-FP-NEXT:    vmul.f16 q0, q0, q1
-; CHECK-FP-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP-NEXT:    vmovx.f16 s6, s1
-; CHECK-FP-NEXT:    vmul.f16 s4, s0, s4
-; CHECK-FP-NEXT:    vmovx.f16 s0, s3
-; CHECK-FP-NEXT:    vmul.f16 s4, s4, s1
-; CHECK-FP-NEXT:    vmul.f16 s4, s4, s6
-; CHECK-FP-NEXT:    vmovx.f16 s6, s2
-; CHECK-FP-NEXT:    vmul.f16 s4, s4, s2
+; CHECK-FP-NEXT:    vrev32.16 q1, q0
+; CHECK-FP-NEXT:    vmul.f16 q0, q0, q1
+; CHECK-FP-NEXT:    vmul.f16 s4, s2, s3
+; CHECK-FP-NEXT:    vmul.f16 s0, s0, s1
 ; CHECK-FP-NEXT:    vldr.16 s2, [r0]
-; CHECK-FP-NEXT:    vmul.f16 s4, s4, s6
-; CHECK-FP-NEXT:    vmul.f16 s4, s4, s3
-; CHECK-FP-NEXT:    vmul.f16 s0, s4, s0
+; CHECK-FP-NEXT:    vmul.f16 s0, s0, s4
 ; CHECK-FP-NEXT:    vmul.f16 s0, s2, s0
 ; CHECK-FP-NEXT:    vstr.16 s0, [r0]
 ; CHECK-FP-NEXT:    bx lr
@@ -227,6 +269,22 @@ entry:
   ret float %z
 }
 
+define arm_aapcs_vfpcc void @fmul_v2f16_nofast(<2 x half> %x, half* %yy) {
+; CHECK-LABEL: fmul_v2f16_nofast:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr.16 s4, [r0]
+; CHECK-NEXT:    vmul.f16 s4, s4, s0
+; CHECK-NEXT:    vmovx.f16 s0, s0
+; CHECK-NEXT:    vmul.f16 s0, s4, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %y = load half, half* %yy
+  %z = call half @llvm.experimental.vector.reduce.v2.fmul.f16.v2f16(half %y, <2 x half> %x)
+  store half %z, half* %yy
+  ret void
+}
+
 define arm_aapcs_vfpcc void @fmul_v4f16_nofast(<4 x half> %x, half* %yy) {
 ; CHECK-LABEL: fmul_v4f16_nofast:
 ; CHECK:       @ %bb.0: @ %entry
@@ -349,5 +407,6 @@ declare float @llvm.experimental.vector.reduce.v2.fmul.f32.v2f32(float, <2 x flo
 declare float @llvm.experimental.vector.reduce.v2.fmul.f32.v4f32(float, <4 x float>)
 declare float @llvm.experimental.vector.reduce.v2.fmul.f32.v8f32(float, <8 x float>)
 declare half @llvm.experimental.vector.reduce.v2.fmul.f16.v16f16(half, <16 x half>)
+declare half @llvm.experimental.vector.reduce.v2.fmul.f16.v2f16(half, <2 x half>)
 declare half @llvm.experimental.vector.reduce.v2.fmul.f16.v4f16(half, <4 x half>)
 declare half @llvm.experimental.vector.reduce.v2.fmul.f16.v8f16(half, <8 x half>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-loops.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-loops.ll
index c817f73ff817d..29e441e3e90cf 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-loops.ll
@@ -122,14 +122,14 @@ define i32 @mul_i32(i32* nocapture readonly %x, i32 %n) {
 ; CHECK-NEXT:    vmul.i32 q0, q1, q0
 ; CHECK-NEXT:    le lr, .LBB1_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
-; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r2, s3
 ; CHECK-NEXT:    cmp r12, r1
-; CHECK-NEXT:    vmov r3, s0
-; CHECK-NEXT:    mul r2, r3, r2
 ; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    mul lr, r3, r2
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    mul r2, r3, r2
-; CHECK-NEXT:    vmov r3, s3
-; CHECK-NEXT:    mul r2, r3, r2
+; CHECK-NEXT:    mul r2, r2, lr
 ; CHECK-NEXT:    beq .LBB1_8
 ; CHECK-NEXT:  .LBB1_6: @ %for.body.preheader1
 ; CHECK-NEXT:    sub.w lr, r1, r12
@@ -222,13 +222,13 @@ define i32 @and_i32(i32* nocapture readonly %x, i32 %n) {
 ; CHECK-NEXT:    vand q0, q1, q0
 ; CHECK-NEXT:    le lr, .LBB2_5
 ; CHECK-NEXT:  @ %bb.6: @ %middle.block
-; CHECK-NEXT:    vmov r12, s1
+; CHECK-NEXT:    vmov r12, s3
 ; CHECK-NEXT:    cmp r3, r1
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    and.w r12, r12, r2
 ; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov lr, s1
 ; CHECK-NEXT:    and.w r12, r12, r2
-; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    and.w r2, r2, lr
 ; CHECK-NEXT:    and.w r2, r2, r12
 ; CHECK-NEXT:    beq .LBB2_9
 ; CHECK-NEXT:  .LBB2_7: @ %for.body.preheader1
@@ -322,13 +322,13 @@ define i32 @or_i32(i32* nocapture readonly %x, i32 %n) {
 ; CHECK-NEXT:    vorr q0, q1, q0
 ; CHECK-NEXT:    le lr, .LBB3_5
 ; CHECK-NEXT:  @ %bb.6: @ %middle.block
-; CHECK-NEXT:    vmov r12, s1
+; CHECK-NEXT:    vmov r12, s3
 ; CHECK-NEXT:    cmp r3, r1
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    orr.w r12, r12, r2
 ; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov lr, s1
 ; CHECK-NEXT:    orr.w r12, r12, r2
-; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    orr.w r2, r2, lr
 ; CHECK-NEXT:    orr.w r2, r2, r12
 ; CHECK-NEXT:    beq .LBB3_9
 ; CHECK-NEXT:  .LBB3_7: @ %for.body.preheader1
@@ -422,13 +422,13 @@ define i32 @xor_i32(i32* nocapture readonly %x, i32 %n) {
 ; CHECK-NEXT:    veor q0, q1, q0
 ; CHECK-NEXT:    le lr, .LBB4_5
 ; CHECK-NEXT:  @ %bb.6: @ %middle.block
-; CHECK-NEXT:    vmov r12, s1
+; CHECK-NEXT:    vmov r12, s3
 ; CHECK-NEXT:    cmp r3, r1
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    eor.w r12, r12, r2
 ; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov lr, s1
 ; CHECK-NEXT:    eor.w r12, r12, r2
-; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    eor.w r2, r2, lr
 ; CHECK-NEXT:    eor.w r2, r2, r12
 ; CHECK-NEXT:    beq .LBB4_9
 ; CHECK-NEXT:  .LBB4_7: @ %for.body.preheader1
@@ -522,10 +522,10 @@ define float @fadd_f32(float* nocapture readonly %x, i32 %n) {
 ; CHECK-NEXT:    vadd.f32 q0, q1, q0
 ; CHECK-NEXT:    le lr, .LBB5_5
 ; CHECK-NEXT:  @ %bb.6: @ %middle.block
-; CHECK-NEXT:    vadd.f32 s4, s0, s1
+; CHECK-NEXT:    vadd.f32 s4, s2, s3
 ; CHECK-NEXT:    cmp r2, r1
-; CHECK-NEXT:    vadd.f32 s4, s4, s2
-; CHECK-NEXT:    vadd.f32 s0, s4, s3
+; CHECK-NEXT:    vadd.f32 s0, s0, s1
+; CHECK-NEXT:    vadd.f32 s0, s0, s4
 ; CHECK-NEXT:    beq .LBB5_9
 ; CHECK-NEXT:  .LBB5_7: @ %for.body.preheader1
 ; CHECK-NEXT:    sub.w lr, r1, r2
@@ -623,10 +623,10 @@ define float @fmul_f32(float* nocapture readonly %x, i32 %n) {
 ; CHECK-NEXT:    vmul.f32 q0, q1, q0
 ; CHECK-NEXT:    le lr, .LBB6_5
 ; CHECK-NEXT:  @ %bb.6: @ %middle.block
-; CHECK-NEXT:    vmul.f32 s4, s0, s1
+; CHECK-NEXT:    vmul.f32 s4, s2, s3
 ; CHECK-NEXT:    cmp r2, r1
-; CHECK-NEXT:    vmul.f32 s4, s4, s2
-; CHECK-NEXT:    vmul.f32 s0, s4, s3
+; CHECK-NEXT:    vmul.f32 s0, s0, s1
+; CHECK-NEXT:    vmul.f32 s0, s0, s4
 ; CHECK-NEXT:    beq .LBB6_9
 ; CHECK-NEXT:  .LBB6_7: @ %for.body.preheader1
 ; CHECK-NEXT:    sub.w lr, r1, r2
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-mul.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-mul.ll
index 15ae58de9357d..7510169fbbd5c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-mul.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-mul.ll
@@ -16,12 +16,12 @@ entry:
 define arm_aapcs_vfpcc i32 @mul_v4i32(<4 x i32> %x) {
 ; CHECK-LABEL: mul_v4i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    muls r0, r1, r0
+; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    muls r1, r2, r1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
 entry:
@@ -33,12 +33,12 @@ define arm_aapcs_vfpcc i32 @mul_v8i32(<8 x i32> %x) {
 ; CHECK-LABEL: mul_v8i32:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmul.i32 q0, q0, q1
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    muls r0, r1, r0
+; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    muls r1, r2, r1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
 entry:
@@ -49,12 +49,12 @@ entry:
 define arm_aapcs_vfpcc i16 @mul_v4i16(<4 x i16> %x) {
 ; CHECK-LABEL: mul_v4i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    muls r0, r1, r0
+; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    muls r1, r2, r1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
 entry:
@@ -65,20 +65,14 @@ entry:
 define arm_aapcs_vfpcc i16 @mul_v8i16(<8 x i16> %x) {
 ; CHECK-LABEL: mul_v8i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q0[2]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q0[3]
-; CHECK-NEXT:    muls r0, r1, r0
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
 ; CHECK-NEXT:    vmov.u16 r1, q0[4]
 ; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q0[6]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    vmov.u16 r2, q0[0]
+; CHECK-NEXT:    muls r1, r2, r1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
 entry:
@@ -90,20 +84,14 @@ define arm_aapcs_vfpcc i16 @mul_v16i16(<16 x i16> %x) {
 ; CHECK-LABEL: mul_v16i16:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmul.i16 q0, q0, q1
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q0[2]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q0[3]
-; CHECK-NEXT:    muls r0, r1, r0
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
 ; CHECK-NEXT:    vmov.u16 r1, q0[4]
 ; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q0[6]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    vmov.u16 r2, q0[0]
+; CHECK-NEXT:    muls r1, r2, r1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
 entry:
@@ -114,20 +102,14 @@ entry:
 define arm_aapcs_vfpcc i8 @mul_v8i8(<8 x i8> %x) {
 ; CHECK-LABEL: mul_v8i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q0[2]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q0[3]
-; CHECK-NEXT:    muls r0, r1, r0
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
 ; CHECK-NEXT:    vmov.u16 r1, q0[4]
 ; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q0[6]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    vmov.u16 r2, q0[0]
+; CHECK-NEXT:    muls r1, r2, r1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
 entry:
@@ -138,36 +120,16 @@ entry:
 define arm_aapcs_vfpcc i8 @mul_v16i8(<16 x i8> %x) {
 ; CHECK-LABEL: mul_v16i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u8 r0, q0[1]
-; CHECK-NEXT:    vmov.u8 r1, q0[0]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[2]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[3]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[4]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[5]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[6]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[7]
-; CHECK-NEXT:    muls r0, r1, r0
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    vmul.i8 q0, q0, q1
+; CHECK-NEXT:    vrev32.8 q1, q0
+; CHECK-NEXT:    vmul.i8 q0, q0, q1
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
 ; CHECK-NEXT:    vmov.u8 r1, q0[8]
 ; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[9]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[10]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[11]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[12]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[13]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[14]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[15]
+; CHECK-NEXT:    vmov.u8 r1, q0[4]
+; CHECK-NEXT:    vmov.u8 r2, q0[0]
+; CHECK-NEXT:    muls r1, r2, r1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
 entry:
@@ -179,36 +141,16 @@ define arm_aapcs_vfpcc i8 @mul_v32i8(<32 x i8> %x) {
 ; CHECK-LABEL: mul_v32i8:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmul.i8 q0, q0, q1
-; CHECK-NEXT:    vmov.u8 r0, q0[1]
-; CHECK-NEXT:    vmov.u8 r1, q0[0]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[2]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[3]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[4]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[5]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[6]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[7]
-; CHECK-NEXT:    muls r0, r1, r0
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    vmul.i8 q0, q0, q1
+; CHECK-NEXT:    vrev32.8 q1, q0
+; CHECK-NEXT:    vmul.i8 q0, q0, q1
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
 ; CHECK-NEXT:    vmov.u8 r1, q0[8]
 ; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[9]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[10]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[11]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[12]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[13]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[14]
-; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[15]
+; CHECK-NEXT:    vmov.u8 r1, q0[4]
+; CHECK-NEXT:    vmov.u8 r2, q0[0]
+; CHECK-NEXT:    muls r1, r2, r1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
 entry:
@@ -286,12 +228,12 @@ entry:
 define arm_aapcs_vfpcc i32 @mul_v4i32_acc(<4 x i32> %x, i32 %y) {
 ; CHECK-LABEL: mul_v4i32_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r1, s1
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    muls r1, r2, r1
+; CHECK-NEXT:    vmov r1, s3
 ; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
 ; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    muls r2, r3, r2
 ; CHECK-NEXT:    muls r1, r2, r1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
@@ -305,12 +247,12 @@ define arm_aapcs_vfpcc i32 @mul_v8i32_acc(<8 x i32> %x, i32 %y) {
 ; CHECK-LABEL: mul_v8i32_acc:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmul.i32 q0, q0, q1
-; CHECK-NEXT:    vmov r1, s1
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    muls r1, r2, r1
+; CHECK-NEXT:    vmov r1, s3
 ; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
 ; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    muls r2, r3, r2
 ; CHECK-NEXT:    muls r1, r2, r1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
@@ -323,12 +265,12 @@ entry:
 define arm_aapcs_vfpcc i16 @mul_v4i16_acc(<4 x i16> %x, i16 %y) {
 ; CHECK-LABEL: mul_v4i16_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r1, s1
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    muls r1, r2, r1
+; CHECK-NEXT:    vmov r1, s3
 ; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
 ; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    muls r2, r3, r2
 ; CHECK-NEXT:    muls r1, r2, r1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
@@ -341,20 +283,14 @@ entry:
 define arm_aapcs_vfpcc i16 @mul_v8i16_acc(<8 x i16> %x, i16 %y) {
 ; CHECK-LABEL: mul_v8i16_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov.u16 r2, q0[0]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u16 r2, q0[2]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u16 r2, q0[3]
-; CHECK-NEXT:    muls r1, r2, r1
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
 ; CHECK-NEXT:    vmov.u16 r2, q0[4]
 ; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u16 r2, q0[5]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u16 r2, q0[6]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.u16 r3, q0[0]
+; CHECK-NEXT:    muls r2, r3, r2
 ; CHECK-NEXT:    muls r1, r2, r1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
@@ -368,20 +304,14 @@ define arm_aapcs_vfpcc i16 @mul_v16i16_acc(<16 x i16> %x, i16 %y) {
 ; CHECK-LABEL: mul_v16i16_acc:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmul.i16 q0, q0, q1
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov.u16 r2, q0[0]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u16 r2, q0[2]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u16 r2, q0[3]
-; CHECK-NEXT:    muls r1, r2, r1
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
 ; CHECK-NEXT:    vmov.u16 r2, q0[4]
 ; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u16 r2, q0[5]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u16 r2, q0[6]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.u16 r3, q0[0]
+; CHECK-NEXT:    muls r2, r3, r2
 ; CHECK-NEXT:    muls r1, r2, r1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
@@ -394,20 +324,14 @@ entry:
 define arm_aapcs_vfpcc i8 @mul_v8i8_acc(<8 x i8> %x, i8 %y) {
 ; CHECK-LABEL: mul_v8i8_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov.u16 r2, q0[0]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u16 r2, q0[2]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u16 r2, q0[3]
-; CHECK-NEXT:    muls r1, r2, r1
+; CHECK-NEXT:    vrev32.16 q1, q0
+; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
 ; CHECK-NEXT:    vmov.u16 r2, q0[4]
 ; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u16 r2, q0[5]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u16 r2, q0[6]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.u16 r3, q0[0]
+; CHECK-NEXT:    muls r2, r3, r2
 ; CHECK-NEXT:    muls r1, r2, r1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
@@ -420,36 +344,16 @@ entry:
 define arm_aapcs_vfpcc i8 @mul_v16i8_acc(<16 x i8> %x, i8 %y) {
 ; CHECK-LABEL: mul_v16i8_acc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u8 r1, q0[1]
-; CHECK-NEXT:    vmov.u8 r2, q0[0]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[2]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[3]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[4]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[5]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[6]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[7]
-; CHECK-NEXT:    muls r1, r2, r1
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    vmul.i8 q0, q0, q1
+; CHECK-NEXT:    vrev32.8 q1, q0
+; CHECK-NEXT:    vmul.i8 q0, q0, q1
+; CHECK-NEXT:    vmov.u8 r1, q0[12]
 ; CHECK-NEXT:    vmov.u8 r2, q0[8]
 ; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[9]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[10]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[11]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[12]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[13]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[14]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[15]
+; CHECK-NEXT:    vmov.u8 r2, q0[4]
+; CHECK-NEXT:    vmov.u8 r3, q0[0]
+; CHECK-NEXT:    muls r2, r3, r2
 ; CHECK-NEXT:    muls r1, r2, r1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
@@ -463,36 +367,16 @@ define arm_aapcs_vfpcc i8 @mul_v32i8_acc(<32 x i8> %x, i8 %y) {
 ; CHECK-LABEL: mul_v32i8_acc:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmul.i8 q0, q0, q1
-; CHECK-NEXT:    vmov.u8 r1, q0[1]
-; CHECK-NEXT:    vmov.u8 r2, q0[0]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[2]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[3]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[4]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[5]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[6]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[7]
-; CHECK-NEXT:    muls r1, r2, r1
+; CHECK-NEXT:    vrev16.8 q1, q0
+; CHECK-NEXT:    vmul.i8 q0, q0, q1
+; CHECK-NEXT:    vrev32.8 q1, q0
+; CHECK-NEXT:    vmul.i8 q0, q0, q1
+; CHECK-NEXT:    vmov.u8 r1, q0[12]
 ; CHECK-NEXT:    vmov.u8 r2, q0[8]
 ; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[9]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[10]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[11]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[12]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[13]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[14]
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    vmov.u8 r2, q0[15]
+; CHECK-NEXT:    vmov.u8 r2, q0[4]
+; CHECK-NEXT:    vmov.u8 r3, q0[0]
+; CHECK-NEXT:    muls r2, r3, r2
 ; CHECK-NEXT:    muls r1, r2, r1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
diff --git a/llvm/test/CodeGen/Thumb2/mve-widen-narrow.ll b/llvm/test/CodeGen/Thumb2/mve-widen-narrow.ll
index 5191cac94f4a0..b6245fb94a0ed 100644
--- a/llvm/test/CodeGen/Thumb2/mve-widen-narrow.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-widen-narrow.ll
@@ -134,14 +134,14 @@ entry:
 define void @foo_int32_int8_double(<16 x i32>* %dest, <16 x i8>* readonly %src, i32 %n) {
 ; CHECK-LABEL: foo_int32_int8_double:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.s32 q0, [r1, #4]
-; CHECK-NEXT:    vldrb.s32 q1, [r1]
-; CHECK-NEXT:    vldrb.s32 q2, [r1, #12]
-; CHECK-NEXT:    vldrb.s32 q3, [r1, #8]
-; CHECK-NEXT:    vstrw.32 q1, [r0]
-; CHECK-NEXT:    vstrw.32 q0, [r0, #16]
-; CHECK-NEXT:    vstrw.32 q3, [r0, #32]
-; CHECK-NEXT:    vstrw.32 q2, [r0, #48]
+; CHECK-NEXT:    vldrb.s32 q0, [r1]
+; CHECK-NEXT:    vldrb.s32 q1, [r1, #4]
+; CHECK-NEXT:    vldrb.s32 q2, [r1, #8]
+; CHECK-NEXT:    vldrb.s32 q3, [r1, #12]
+; CHECK-NEXT:    vstrw.32 q1, [r0, #16]
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q3, [r0, #48]
+; CHECK-NEXT:    vstrw.32 q2, [r0, #32]
 ; CHECK-NEXT:    bx lr
 entry:
   %wide.load = load <16 x i8>, <16 x i8>* %src, align 1
@@ -224,14 +224,14 @@ entry:
 define void @foo_uint32_uint8_double(<16 x i32>* %dest, <16 x i8>* readonly %src, i32 %n) {
 ; CHECK-LABEL: foo_uint32_uint8_double:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u32 q0, [r1, #4]
-; CHECK-NEXT:    vldrb.u32 q1, [r1]
-; CHECK-NEXT:    vldrb.u32 q2, [r1, #12]
-; CHECK-NEXT:    vldrb.u32 q3, [r1, #8]
-; CHECK-NEXT:    vstrw.32 q1, [r0]
-; CHECK-NEXT:    vstrw.32 q0, [r0, #16]
-; CHECK-NEXT:    vstrw.32 q3, [r0, #32]
-; CHECK-NEXT:    vstrw.32 q2, [r0, #48]
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vldrb.u32 q1, [r1, #4]
+; CHECK-NEXT:    vldrb.u32 q2, [r1, #8]
+; CHECK-NEXT:    vldrb.u32 q3, [r1, #12]
+; CHECK-NEXT:    vstrw.32 q1, [r0, #16]
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q3, [r0, #48]
+; CHECK-NEXT:    vstrw.32 q2, [r0, #32]
 ; CHECK-NEXT:    bx lr
 entry:
   %wide.load = load <16 x i8>, <16 x i8>* %src, align 1
@@ -347,12 +347,12 @@ define <16 x i16>* @foo_uint32_uint16_quad_offset(<16 x i32>* %dest, <16 x i16>*
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrh.s32 q0, [r1, #32]!
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
-; CHECK-NEXT:    vldrh.s32 q2, [r1, #24]
-; CHECK-NEXT:    vldrh.s32 q3, [r1, #16]
+; CHECK-NEXT:    vldrh.s32 q2, [r1, #16]
+; CHECK-NEXT:    vldrh.s32 q3, [r1, #24]
 ; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    vstrw.32 q2, [r0, #48]
+; CHECK-NEXT:    vstrw.32 q2, [r0, #32]
 ; CHECK-NEXT:    vstrw.32 q1, [r0, #16]
-; CHECK-NEXT:    vstrw.32 q3, [r0, #32]
+; CHECK-NEXT:    vstrw.32 q3, [r0, #48]
 ; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/VE/load_off.ll b/llvm/test/CodeGen/VE/load_off.ll
new file mode 100644
index 0000000000000..cc3da7a3a1cd5
--- /dev/null
+++ b/llvm/test/CodeGen/VE/load_off.ll
@@ -0,0 +1,204 @@
+; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
+
+@bufi8 = global [3 x i8] zeroinitializer, align 1
+@bufi16 = global [3 x i16] zeroinitializer, align 2
+@bufi32 = global [3 x i32] zeroinitializer, align 4
+@bufi64 = global [3 x i64] zeroinitializer, align 8
+@bufi128 = global [3 x i128] zeroinitializer, align 16
+@buff32 = global [3 x float] zeroinitializer, align 4
+@buff64 = global [3 x double] zeroinitializer, align 8
+@buff128 = global [3 x fp128] zeroinitializer, align 16
+
+; Function Attrs: noinline nounwind optnone
+define signext i8 @loadi8s() {
+; CHECK-LABEL: loadi8s:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s0, bufi8+2@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s0, bufi8+2@hi(, %s0)
+; CHECK-NEXT:    ld1b.sx %s0, (, %s0)
+; CHECK-NEXT:    or %s11, 0, %s9
+entry:
+  %0 = load i8, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @bufi8, i64 0, i64 2), align 1
+  ret i8 %0
+}
+
+; Function Attrs: noinline nounwind optnone
+define signext i16 @loadi16s() {
+; CHECK-LABEL: loadi16s:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s0, bufi16+4@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s0, bufi16+4@hi(, %s0)
+; CHECK-NEXT:    ld2b.sx %s0, (, %s0)
+; CHECK-NEXT:    or %s11, 0, %s9
+entry:
+  %0 = load i16, i16* getelementptr inbounds ([3 x i16], [3 x i16]* @bufi16, i64 0, i64 2), align 2
+  ret i16 %0
+}
+
+; Function Attrs: noinline nounwind optnone
+define signext i32 @loadi32s() {
+; CHECK-LABEL: loadi32s:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s0, bufi32+8@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s0, bufi32+8@hi(, %s0)
+; CHECK-NEXT:    ldl.sx %s0, (, %s0)
+; CHECK-NEXT:    or %s11, 0, %s9
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([3 x i32], [3 x i32]* @bufi32, i64 0, i64 2), align 4
+  ret i32 %0
+}
+
+; Function Attrs: noinline nounwind optnone
+define i64 @loadi64s() {
+; CHECK-LABEL: loadi64s:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s0, bufi64+16@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s0, bufi64+16@hi(, %s0)
+; CHECK-NEXT:    ld %s0, (, %s0)
+; CHECK-NEXT:    or %s11, 0, %s9
+entry:
+  %0 = load i64, i64* getelementptr inbounds ([3 x i64], [3 x i64]* @bufi64, i64 0, i64 2), align 8
+  ret i64 %0
+}
+
+; Function Attrs: noinline nounwind optnone
+define i128 @loadi128s() {
+; CHECK-LABEL: loadi128s:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s0, bufi128+32@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s0, bufi128+32@hi(, %s0)
+; CHECK-NEXT:    ld %s0, (, %s0)
+; CHECK-NEXT:    lea %s1, bufi128+40@lo
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    lea.sl %s1, bufi128+40@hi(, %s1)
+; CHECK-NEXT:    ld %s1, (, %s1)
+; CHECK-NEXT:    or %s11, 0, %s9
+entry:
+  %0 = load i128, i128* getelementptr inbounds ([3 x i128], [3 x i128]* @bufi128, i64 0, i64 2), align 16
+  ret i128 %0
+}
+
+; Function Attrs: noinline nounwind optnone
+define zeroext i8 @loadi8z() {
+; CHECK-LABEL: loadi8z:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s0, bufi8+2@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s0, bufi8+2@hi(, %s0)
+; CHECK-NEXT:    ld1b.zx %s0, (, %s0)
+; CHECK-NEXT:    or %s11, 0, %s9
+entry:
+  %0 = load i8, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @bufi8, i64 0, i64 2), align 1
+  ret i8 %0
+}
+
+; Function Attrs: noinline nounwind optnone
+define zeroext i16 @loadi16z() {
+; CHECK-LABEL: loadi16z:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s0, bufi16+4@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s0, bufi16+4@hi(, %s0)
+; CHECK-NEXT:    ld2b.zx %s0, (, %s0)
+; CHECK-NEXT:    or %s11, 0, %s9
+entry:
+  %0 = load i16, i16* getelementptr inbounds ([3 x i16], [3 x i16]* @bufi16, i64 0, i64 2), align 2
+  ret i16 %0
+}
+
+; Function Attrs: noinline nounwind optnone
+define zeroext i32 @loadi32z() {
+; CHECK-LABEL: loadi32z:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s0, bufi32+8@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s0, bufi32+8@hi(, %s0)
+; CHECK-NEXT:    ldl.sx %s0, (, %s0)
+; CHECK-NEXT:    or %s11, 0, %s9
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([3 x i32], [3 x i32]* @bufi32, i64 0, i64 2), align 4
+  ret i32 %0
+}
+
+; Function Attrs: noinline nounwind optnone
+define i64 @loadi64z() {
+; CHECK-LABEL: loadi64z:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s0, bufi64+16@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s0, bufi64+16@hi(, %s0)
+; CHECK-NEXT:    ld %s0, (, %s0)
+; CHECK-NEXT:    or %s11, 0, %s9
+entry:
+  %0 = load i64, i64* getelementptr inbounds ([3 x i64], [3 x i64]* @bufi64, i64 0, i64 2), align 8
+  ret i64 %0
+}
+
+; Function Attrs: noinline nounwind optnone
+define i128 @loadi128z() {
+; CHECK-LABEL: loadi128z:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s0, bufi128+32@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s0, bufi128+32@hi(, %s0)
+; CHECK-NEXT:    ld %s0, (, %s0)
+; CHECK-NEXT:    lea %s1, bufi128+40@lo
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    lea.sl %s1, bufi128+40@hi(, %s1)
+; CHECK-NEXT:    ld %s1, (, %s1)
+; CHECK-NEXT:    or %s11, 0, %s9
+entry:
+  %0 = load i128, i128* getelementptr inbounds ([3 x i128], [3 x i128]* @bufi128, i64 0, i64 2), align 16
+  ret i128 %0
+}
+
+; Function Attrs: noinline nounwind optnone
+define float @loadf32() {
+; CHECK-LABEL: loadf32:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s0, buff32+8@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s0, buff32+8@hi(, %s0)
+; CHECK-NEXT:    ldu %s0, (, %s0)
+; CHECK-NEXT:    or %s11, 0, %s9
+entry:
+  %0 = load float, float* getelementptr inbounds ([3 x float], [3 x float]* @buff32, i64 0, i64 2), align 4
+  ret float %0
+}
+
+; Function Attrs: noinline nounwind optnone
+define double @loadf64() {
+; CHECK-LABEL: loadf64:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s0, buff64+16@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s0, buff64+16@hi(, %s0)
+; CHECK-NEXT:    ld %s0, (, %s0)
+; CHECK-NEXT:    or %s11, 0, %s9
+entry:
+  %0 = load double, double* getelementptr inbounds ([3 x double], [3 x double]* @buff64, i64 0, i64 2), align 8
+  ret double %0
+}
+
+; Function Attrs: noinline nounwind optnone
+define fp128 @loadf128() {
+; CHECK-LABEL: loadf128:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s0, buff128+32@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s0, buff128+32@hi(, %s0)
+; CHECK-NEXT:    ld %s0, (, %s0)
+; CHECK-NEXT:    lea %s1, buff128+40@lo
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    lea.sl %s1, buff128+40@hi(, %s1)
+; CHECK-NEXT:    ld %s1, (, %s1)
+; CHECK-NEXT:    or %s11, 0, %s9
+entry:
+  %0 = load fp128, fp128* getelementptr inbounds ([3 x fp128], [3 x fp128]* @buff128, i64 0, i64 2), align 16
+  ret fp128 %0
+}
diff --git a/llvm/test/CodeGen/WebAssembly/bulk-memory-intrinsics.ll b/llvm/test/CodeGen/WebAssembly/bulk-memory-intrinsics.ll
deleted file mode 100644
index dfb74b78f64fd..0000000000000
--- a/llvm/test/CodeGen/WebAssembly/bulk-memory-intrinsics.ll
+++ /dev/null
@@ -1,28 +0,0 @@
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+bulk-memory | FileCheck %s
-
-; Test that bulk memory intrinsics lower correctly
-
-target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
-target triple = "wasm32-unknown-unknown"
-
-; CHECK-LABEL: memory_init:
-; CHECK-NEXT: .functype  memory_init (i32, i32, i32) -> ()
-; CHECK-NEXT: memory.init 3, 0, $0, $1, $2
-; CHECK-NEXT: return
-declare void @llvm.wasm.memory.init(i32, i32, i8*, i32, i32)
-define void @memory_init(i8* %dest, i32 %offset, i32 %size) {
-  call void @llvm.wasm.memory.init(
-    i32 3, i32 0, i8* %dest, i32 %offset, i32 %size
-  )
-  ret void
-}
-
-; CHECK-LABEL: data_drop:
-; CHECK-NEXT: .functype data_drop () -> ()
-; CHECK-NEXT: data.drop 3
-; CHECK-NEXT: return
-declare void @llvm.wasm.data.drop(i32)
-define void @data_drop() {
-  call void @llvm.wasm.data.drop(i32 3)
-  ret void
-}
diff --git a/llvm/test/CodeGen/WebAssembly/bulk-memory64.ll b/llvm/test/CodeGen/WebAssembly/bulk-memory64.ll
new file mode 100644
index 0000000000000..6c450f5b53982
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/bulk-memory64.ll
@@ -0,0 +1,210 @@
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+bulk-memory | FileCheck %s --check-prefixes CHECK,BULK-MEM
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=-bulk-memory | FileCheck %s --check-prefixes CHECK,NO-BULK-MEM
+
+; Test that basic bulk memory codegen works correctly
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm64-unknown-unknown"
+
+declare void @llvm.memcpy.p0i8.p0i8.i8(i8*, i8*, i8, i1)
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8*, i64, i1)
+declare void @llvm.memcpy.p0i32.p0i32.i64(i32*, i32*, i64, i1)
+
+declare void @llvm.memmove.p0i8.p0i8.i8(i8*, i8*, i8, i1)
+declare void @llvm.memmove.p0i8.p0i8.i64(i8*, i8*, i64, i1)
+declare void @llvm.memmove.p0i32.p0i32.i64(i32*, i32*, i64, i1)
+
+declare void @llvm.memset.p0i8.i8(i8*, i8, i8, i1)
+declare void @llvm.memset.p0i8.i64(i8*, i8, i64, i1)
+declare void @llvm.memset.p0i32.i64(i32*, i8, i64, i1)
+
+; CHECK-LABEL: memcpy_i8:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memcpy_i8 (i64, i64, i32) -> ()
+; BULK-MEM-NEXT: i64.extend_i32_u $push0=, $2
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop0
+; BULK-MEM-NEXT: return
+define void @memcpy_i8(i8* %dest, i8* %src, i8 zeroext %len) {
+  call void @llvm.memcpy.p0i8.p0i8.i8(i8* %dest, i8* %src, i8 %len, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memmove_i8:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memmove_i8 (i64, i64, i32) -> ()
+; BULK-MEM-NEXT: i64.extend_i32_u $push0=, $2
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop0
+; BULK-MEM-NEXT: return
+define void @memmove_i8(i8* %dest, i8* %src, i8 zeroext %len) {
+  call void @llvm.memmove.p0i8.p0i8.i8(i8* %dest, i8* %src, i8 %len, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memset_i8:
+; NO-BULK-MEM-NOT: memory.fill
+; BULK-MEM-NEXT: .functype memset_i8 (i64, i32, i32) -> ()
+; BULK-MEM-NEXT: i64.extend_i32_u $push0=, $2
+; BULK-MEM-NEXT: memory.fill 0, $0, $1, $pop0
+; BULK-MEM-NEXT: return
+define void @memset_i8(i8* %dest, i8 %val, i8 zeroext %len) {
+  call void @llvm.memset.p0i8.i8(i8* %dest, i8 %val, i8 %len, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memcpy_i32:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memcpy_i32 (i64, i64, i64) -> ()
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2
+; BULK-MEM-NEXT: return
+define void @memcpy_i32(i32* %dest, i32* %src, i64 %len) {
+  call void @llvm.memcpy.p0i32.p0i32.i64(i32* %dest, i32* %src, i64 %len, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memmove_i32:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memmove_i32 (i64, i64, i64) -> ()
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2
+; BULK-MEM-NEXT: return
+define void @memmove_i32(i32* %dest, i32* %src, i64 %len) {
+  call void @llvm.memmove.p0i32.p0i32.i64(i32* %dest, i32* %src, i64 %len, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memset_i32:
+; NO-BULK-MEM-NOT: memory.fill
+; BULK-MEM-NEXT: .functype memset_i32 (i64, i32, i64) -> ()
+; BULK-MEM-NEXT: memory.fill 0, $0, $1, $2
+; BULK-MEM-NEXT: return
+define void @memset_i32(i32* %dest, i8 %val, i64 %len) {
+  call void @llvm.memset.p0i32.i64(i32* %dest, i8 %val, i64 %len, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memcpy_1:
+; CHECK-NEXT: .functype memcpy_1 (i64, i64) -> ()
+; CHECK-NEXT: i32.load8_u $push[[L0:[0-9]+]]=, 0($1)
+; CHECK-NEXT: i32.store8 0($0), $pop[[L0]]
+; CHECK-NEXT: return
+define void @memcpy_1(i8* %dest, i8* %src) {
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dest, i8* %src, i64 1, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memmove_1:
+; CHECK-NEXT: .functype memmove_1 (i64, i64) -> ()
+; CHECK-NEXT: i32.load8_u $push[[L0:[0-9]+]]=, 0($1)
+; CHECK-NEXT: i32.store8 0($0), $pop[[L0]]
+; CHECK-NEXT: return
+define void @memmove_1(i8* %dest, i8* %src) {
+  call void @llvm.memmove.p0i8.p0i8.i64(i8* %dest, i8* %src, i64 1, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memset_1:
+; NO-BULK-MEM-NOT: memory.fill
+; BULK-MEM-NEXT: .functype memset_1 (i64, i32) -> ()
+; BULK-MEM-NEXT: i32.store8 0($0), $1
+; BULK-MEM-NEXT: return
+define void @memset_1(i8* %dest, i8 %val) {
+  call void @llvm.memset.p0i8.i64(i8* %dest, i8 %val, i64 1, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memcpy_1024:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memcpy_1024 (i64, i64) -> ()
+; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 1024
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L0]]
+; BULK-MEM-NEXT: return
+define void @memcpy_1024(i8* %dest, i8* %src) {
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dest, i8* %src, i64 1024, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memmove_1024:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memmove_1024 (i64, i64) -> ()
+; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 1024
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L0]]
+; BULK-MEM-NEXT: return
+define void @memmove_1024(i8* %dest, i8* %src) {
+  call void @llvm.memmove.p0i8.p0i8.i64(i8* %dest, i8* %src, i64 1024, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memset_1024:
+; NO-BULK-MEM-NOT: memory.fill
+; BULK-MEM-NEXT: .functype memset_1024 (i64, i32) -> ()
+; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 1024
+; BULK-MEM-NEXT: memory.fill 0, $0, $1, $pop[[L0]]
+; BULK-MEM-NEXT: return
+define void @memset_1024(i8* %dest, i8 %val) {
+  call void @llvm.memset.p0i8.i64(i8* %dest, i8 %val, i64 1024, i1 0)
+  ret void
+}
+
+; The following tests check that frame index elimination works for
+; bulk memory instructions. The stack pointer is bumped by 112 instead
+; of 100 because the stack pointer in WebAssembly is currently always
+; 16-byte aligned, even in leaf functions, although it is not written
+; back to the global in this case.
+
+; TODO: Change TransientStackAlignment to 1 to avoid this extra
+; arithmetic. This will require forcing the use of StackAlignment in
+; PrologEpilogEmitter.cpp when
+; WebAssemblyFrameLowering::needsSPWriteback would be true.
+
+; CHECK-LABEL: memcpy_alloca_src:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memcpy_alloca_src (i64) -> ()
+; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer
+; BULK-MEM-NEXT: i64.const $push[[L1:[0-9]+]]=, 112
+; BULK-MEM-NEXT: i64.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
+; BULK-MEM-NEXT: i64.const $push[[L3:[0-9]+]]=, 12
+; BULK-MEM-NEXT: i64.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]
+; BULK-MEM-NEXT: i64.const $push[[L5:[0-9]+]]=, 100
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $pop[[L4]], $pop[[L5]]
+; BULK-MEM-NEXT: return
+define void @memcpy_alloca_src(i8* %dst) {
+  %a = alloca [100 x i8]
+  %p = bitcast [100 x i8]* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %p, i64 100, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: memcpy_alloca_dst:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memcpy_alloca_dst (i64) -> ()
+; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer
+; BULK-MEM-NEXT: i64.const $push[[L1:[0-9]+]]=, 112
+; BULK-MEM-NEXT: i64.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
+; BULK-MEM-NEXT: i64.const $push[[L3:[0-9]+]]=, 12
+; BULK-MEM-NEXT: i64.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]
+; BULK-MEM-NEXT: i64.const $push[[L5:[0-9]+]]=, 100
+; BULK-MEM-NEXT: memory.copy 0, 0, $pop[[L4]], $0, $pop[[L5]]
+; BULK-MEM-NEXT: return
+define void @memcpy_alloca_dst(i8* %src) {
+  %a = alloca [100 x i8]
+  %p = bitcast [100 x i8]* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %p, i8* %src, i64 100, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: memset_alloca:
+; NO-BULK-MEM-NOT: memory.fill
+; BULK-MEM-NEXT: .functype memset_alloca (i32) -> ()
+; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer
+; BULK-MEM-NEXT: i64.const $push[[L1:[0-9]+]]=, 112
+; BULK-MEM-NEXT: i64.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
+; BULK-MEM-NEXT: i64.const $push[[L3:[0-9]+]]=, 12
+; BULK-MEM-NEXT: i64.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]
+; BULK-MEM-NEXT: i64.const $push[[L5:[0-9]+]]=, 100
+; BULK-MEM-NEXT: memory.fill 0, $pop[[L4]], $0, $pop[[L5]]
+; BULK-MEM-NEXT: return
+define void @memset_alloca(i8 %val) {
+  %a = alloca [100 x i8]
+  %p = bitcast [100 x i8]* %a to i8*
+  call void @llvm.memset.p0i8.i64(i8* %p, i8 %val, i64 100, i1 false)
+  ret void
+}
diff --git a/llvm/test/CodeGen/WebAssembly/memory-addr64.ll b/llvm/test/CodeGen/WebAssembly/memory-addr64.ll
new file mode 100644
index 0000000000000..7268d166783ad
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/memory-addr64.ll
@@ -0,0 +1,27 @@
+; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s
+
+; Test that basic memory operations assemble as expected with 64-bit addresses.
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm64-unknown-unknown"
+
+declare i64 @llvm.wasm.memory.size.i64(i32) nounwind readonly
+declare i64 @llvm.wasm.memory.grow.i64(i32, i64) nounwind
+
+; CHECK-LABEL: memory_size:
+; CHECK-NEXT: .functype memory_size () -> (i64){{$}}
+; CHECK-NEXT: memory.size $push0=, 0{{$}}
+; CHECK-NEXT: return $pop0{{$}}
+define i64 @memory_size() {
+  %a = call i64 @llvm.wasm.memory.size.i64(i32 0)
+  ret i64 %a
+}
+
+; CHECK-LABEL: memory_grow:
+; CHECK-NEXT: .functype memory_grow (i64) -> (i64){{$}}
+; CHECK: memory.grow $push0=, 0, $0{{$}}
+; CHECK-NEXT: return $pop0{{$}}
+define i64 @memory_grow(i64 %n) {
+  %a = call i64 @llvm.wasm.memory.grow.i64(i32 0, i64 %n)
+  ret i64 %a
+}
diff --git a/llvm/test/CodeGen/WebAssembly/simd-arith.ll b/llvm/test/CodeGen/WebAssembly/simd-arith.ll
index c56566991a8c8..fca4710b582f6 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-arith.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-arith.ll
@@ -1278,9 +1278,8 @@ define <4 x float> @abs_v4f32(<4 x float> %x) {
 ; CHECK-LABEL: min_unordered_v4f32:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .functype min_unordered_v4f32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f32.const $push[[L0:[0-9]+]]=, 0x1.4p2
-; SIMD128-NEXT: f32x4.splat $push[[L1:[0-9]+]]=, $pop[[L0]]
-; SIMD128-NEXT: f32x4.min $push[[R:[0-9]+]]=, $0, $pop[[L1]]{{$}}
+; SIMD128-NEXT: v128.const $push[[L0:[0-9]+]]=, 0x1.4p2, 0x1.4p2, 0x1.4p2, 0x1.4p2{{$}}
+; SIMD128-NEXT: f32x4.min $push[[R:[0-9]+]]=, $0, $pop[[L0]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @min_unordered_v4f32(<4 x float> %x) {
   %cmps = fcmp ule <4 x float> %x, <float 5., float 5., float 5., float 5.>
@@ -1292,9 +1291,8 @@ define <4 x float> @min_unordered_v4f32(<4 x float> %x) {
 ; CHECK-LABEL: max_unordered_v4f32:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .functype max_unordered_v4f32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f32.const $push[[L0:[0-9]+]]=, 0x1.4p2
-; SIMD128-NEXT: f32x4.splat $push[[L1:[0-9]+]]=, $pop[[L0]]
-; SIMD128-NEXT: f32x4.max $push[[R:[0-9]+]]=, $0, $pop[[L1]]{{$}}
+; SIMD128-NEXT: v128.const $push[[L0:[0-9]+]]=, 0x1.4p2, 0x1.4p2, 0x1.4p2, 0x1.4p2
+; SIMD128-NEXT: f32x4.max $push[[R:[0-9]+]]=, $0, $pop[[L0]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @max_unordered_v4f32(<4 x float> %x) {
   %cmps = fcmp uge <4 x float> %x, <float 5., float 5., float 5., float 5.>
@@ -1306,9 +1304,8 @@ define <4 x float> @max_unordered_v4f32(<4 x float> %x) {
 ; CHECK-LABEL: min_ordered_v4f32:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .functype min_ordered_v4f32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f32.const $push[[L0:[0-9]+]]=, 0x1.4p2
-; SIMD128-NEXT: f32x4.splat $push[[L1:[0-9]+]]=, $pop[[L0]]
-; SIMD128-NEXT: f32x4.min $push[[R:[0-9]+]]=, $0, $pop[[L1]]{{$}}
+; SIMD128-NEXT: v128.const $push[[L0:[0-9]+]]=, 0x1.4p2, 0x1.4p2, 0x1.4p2, 0x1.4p2{{$}}
+; SIMD128-NEXT: f32x4.min $push[[R:[0-9]+]]=, $0, $pop[[L0]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @min_ordered_v4f32(<4 x float> %x) {
   %cmps = fcmp ole <4 x float> <float 5., float 5., float 5., float 5.>, %x
@@ -1320,9 +1317,8 @@ define <4 x float> @min_ordered_v4f32(<4 x float> %x) {
 ; CHECK-LABEL: max_ordered_v4f32:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .functype max_ordered_v4f32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f32.const $push[[L0:[0-9]+]]=, 0x1.4p2
-; SIMD128-NEXT: f32x4.splat $push[[L1:[0-9]+]]=, $pop[[L0]]
-; SIMD128-NEXT: f32x4.max $push[[R:[0-9]+]]=, $0, $pop[[L1]]{{$}}
+; SIMD128-NEXT: v128.const $push[[L0:[0-9]+]]=, 0x1.4p2, 0x1.4p2, 0x1.4p2, 0x1.4p2{{$}}
+; SIMD128-NEXT: f32x4.max $push[[R:[0-9]+]]=, $0, $pop[[L0]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @max_ordered_v4f32(<4 x float> %x) {
   %cmps = fcmp oge <4 x float> <float 5., float 5., float 5., float 5.>, %x
@@ -1378,8 +1374,7 @@ define <4 x float> @maxnum_intrinsic_v4f32(<4 x float> %x, <4 x float> %y) {
 ; CHECK-LABEL: min_const_intrinsic_v4f32:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .functype min_const_intrinsic_v4f32 () -> (v128){{$}}
-; SIMD128-NEXT: f32.const $push[[L:[0-9]+]]=, 0x1.4p2{{$}}
-; SIMD128-NEXT: f32x4.splat $push[[R:[0-9]+]]=, $pop[[L]]{{$}}
+; SIMD128-NEXT: v128.const $push[[R:[0-9]+]]=, 0x1.4p2, 0x1.4p2, 0x1.4p2, 0x1.4p2{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @min_const_intrinsic_v4f32() {
   %a = call <4 x float> @llvm.minimum.v4f32(
@@ -1392,8 +1387,7 @@ define <4 x float> @min_const_intrinsic_v4f32() {
 ; CHECK-LABEL: max_const_intrinsic_v4f32:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .functype max_const_intrinsic_v4f32 () -> (v128){{$}}
-; SIMD128-NEXT: f32.const $push[[L:[0-9]+]]=, 0x1.5p5{{$}}
-; SIMD128-NEXT: f32x4.splat $push[[R:[0-9]+]]=, $pop[[L]]{{$}}
+; SIMD128-NEXT: v128.const $push[[R:[0-9]+]]=, 0x1.5p5, 0x1.5p5, 0x1.5p5, 0x1.5p5{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @max_const_intrinsic_v4f32() {
   %a = call <4 x float> @llvm.maximum.v4f32(
@@ -1482,9 +1476,8 @@ define <2 x double> @abs_v2f64(<2 x double> %x) {
 ; CHECK-LABEL: min_unordered_v2f64:
 ; NO-SIMD128-NOT: f64x2
 ; SIMD128-NEXT: .functype min_unordered_v2f64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f64.const $push[[L0:[0-9]+]]=, 0x1.4p2
-; SIMD128-NEXT: f64x2.splat $push[[L1:[0-9]+]]=, $pop[[L0]]
-; SIMD128-NEXT: f64x2.min $push[[R:[0-9]+]]=, $0, $pop[[L1]]{{$}}
+; SIMD128-NEXT: v128.const $push[[L0:[0-9]+]]=, 0x1.4p2, 0x1.4p2{{$}}
+; SIMD128-NEXT: f64x2.min $push[[R:[0-9]+]]=, $0, $pop[[L0]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @min_unordered_v2f64(<2 x double> %x) {
   %cmps = fcmp ule <2 x double> %x, <double 5., double 5.>
@@ -1496,9 +1489,8 @@ define <2 x double> @min_unordered_v2f64(<2 x double> %x) {
 ; CHECK-LABEL: max_unordered_v2f64:
 ; NO-SIMD128-NOT: f64x2
 ; SIMD128-NEXT: .functype max_unordered_v2f64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f64.const $push[[L0:[0-9]+]]=, 0x1.4p2
-; SIMD128-NEXT: f64x2.splat $push[[L1:[0-9]+]]=, $pop[[L0]]
-; SIMD128-NEXT: f64x2.max $push[[R:[0-9]+]]=, $0, $pop[[L1]]{{$}}
+; SIMD128-NEXT: v128.const $push[[L0:[0-9]+]]=, 0x1.4p2, 0x1.4p2{{$}}
+; SIMD128-NEXT: f64x2.max $push[[R:[0-9]+]]=, $0, $pop[[L0]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @max_unordered_v2f64(<2 x double> %x) {
   %cmps = fcmp uge <2 x double> %x, <double 5., double 5.>
@@ -1510,9 +1502,8 @@ define <2 x double> @max_unordered_v2f64(<2 x double> %x) {
 ; CHECK-LABEL: min_ordered_v2f64:
 ; NO-SIMD128-NOT: f64x2
 ; SIMD128-NEXT: .functype min_ordered_v2f64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f64.const $push[[L0:[0-9]+]]=, 0x1.4p2
-; SIMD128-NEXT: f64x2.splat $push[[L1:[0-9]+]]=, $pop[[L0]]
-; SIMD128-NEXT: f64x2.min $push[[R:[0-9]+]]=, $0, $pop[[L1]]{{$}}
+; SIMD128-NEXT: v128.const $push[[L0:[0-9]+]]=, 0x1.4p2, 0x1.4p2{{$}}
+; SIMD128-NEXT: f64x2.min $push[[R:[0-9]+]]=, $0, $pop[[L0]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @min_ordered_v2f64(<2 x double> %x) {
   %cmps = fcmp ole <2 x double> <double 5., double 5.>, %x
@@ -1524,9 +1515,8 @@ define <2 x double> @min_ordered_v2f64(<2 x double> %x) {
 ; CHECK-LABEL: max_ordered_v2f64:
 ; NO-SIMD128-NOT: f64x2
 ; SIMD128-NEXT: .functype max_ordered_v2f64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f64.const $push[[L0:[0-9]+]]=, 0x1.4p2
-; SIMD128-NEXT: f64x2.splat $push[[L1:[0-9]+]]=, $pop[[L0]]
-; SIMD128-NEXT: f64x2.max $push[[R:[0-9]+]]=, $0, $pop[[L1]]{{$}}
+; SIMD128-NEXT: v128.const $push[[L0:[0-9]+]]=, 0x1.4p2, 0x1.4p2{{$}}
+; SIMD128-NEXT: f64x2.max $push[[R:[0-9]+]]=, $0, $pop[[L0]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @max_ordered_v2f64(<2 x double> %x) {
   %cmps = fcmp oge <2 x double> <double 5., double 5.>, %x
@@ -1560,8 +1550,7 @@ define <2 x double> @max_intrinsic_v2f64(<2 x double> %x, <2 x double> %y) {
 ; CHECK-LABEL: min_const_intrinsic_v2f64:
 ; NO-SIMD128-NOT: f64x2
 ; SIMD128-NEXT: .functype min_const_intrinsic_v2f64 () -> (v128){{$}}
-; SIMD128-NEXT: f64.const $push[[L:[0-9]+]]=, 0x1.4p2{{$}}
-; SIMD128-NEXT: f64x2.splat $push[[R:[0-9]+]]=, $pop[[L]]{{$}}
+; SIMD128-NEXT: v128.const $push[[R:[0-9]+]]=, 0x1.4p2, 0x1.4p2{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @min_const_intrinsic_v2f64() {
   %a = call <2 x double> @llvm.minimum.v2f64(
@@ -1574,8 +1563,7 @@ define <2 x double> @min_const_intrinsic_v2f64() {
 ; CHECK-LABEL: max_const_intrinsic_v2f64:
 ; NO-SIMD128-NOT: f64x2
 ; SIMD128-NEXT: .functype max_const_intrinsic_v2f64 () -> (v128){{$}}
-; SIMD128-NEXT: f64.const $push[[L:[0-9]+]]=, 0x1.5p5{{$}}
-; SIMD128-NEXT: f64x2.splat $push[[R:[0-9]+]]=, $pop[[L]]{{$}}
+; SIMD128-NEXT: v128.const $push[[R:[0-9]+]]=, 0x1.5p5, 0x1.5p5{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @max_const_intrinsic_v2f64() {
   %a = call <2 x double> @llvm.maximum.v2f64(
diff --git a/llvm/test/CodeGen/WebAssembly/simd-shift-complex-splats.ll b/llvm/test/CodeGen/WebAssembly/simd-shift-complex-splats.ll
new file mode 100644
index 0000000000000..2473f0b27b7e8
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/simd-shift-complex-splats.ll
@@ -0,0 +1,103 @@
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s
+
+; Test that SIMD shifts can be lowered correctly even with shift
+; values that are more complex than plain splats.
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+;; TODO: Optimize this further by scalarizing the add
+
+; CHECK-LABEL: shl_add:
+; CHECK-NEXT: .functype shl_add (v128, i32, i32) -> (v128)
+; CHECK-NEXT: i8x16.splat $push1=, $1
+; CHECK-NEXT: i8x16.splat $push0=, $2
+; CHECK-NEXT: i8x16.add $push2=, $pop1, $pop0
+; CHECK-NEXT: i8x16.extract_lane_u $push3=, $pop2, 0
+; CHECK-NEXT: i8x16.shl $push4=, $0, $pop3
+; CHECK-NEXT: return $pop4
+define <16 x i8> @shl_add(<16 x i8> %v, i8 %a, i8 %b) {
+  %t1 = insertelement <16 x i8> undef, i8 %a, i32 0
+  %va = shufflevector <16 x i8> %t1, <16 x i8> undef, <16 x i32> zeroinitializer
+  %t2 = insertelement <16 x i8> undef, i8 %b, i32 0
+  %vb = shufflevector <16 x i8> %t2, <16 x i8> undef, <16 x i32> zeroinitializer
+  %shift = add <16 x i8> %va, %vb
+  %r = shl <16 x i8> %v, %shift
+  ret <16 x i8> %r
+}
+
+; CHECK-LABEL: shl_abs:
+; CHECK-NEXT: .functype shl_abs (v128, i32) -> (v128)
+; CHECK-NEXT: i8x16.extract_lane_u $push8=, $0, 0
+; CHECK-NEXT: i8x16.splat $push0=, $1
+; CHECK-NEXT: i8x16.abs $push98=, $pop0
+; CHECK-NEXT: local.tee $push97=, $2=, $pop98
+; CHECK-NEXT: i8x16.extract_lane_u $push6=, $pop97, 0
+; CHECK-NEXT: i32.const $push2=, 7
+; CHECK-NEXT: i32.and $push7=, $pop6, $pop2
+; CHECK-NEXT: i32.shl $push9=, $pop8, $pop7
+; CHECK-NEXT: i8x16.splat $push10=, $pop9
+; CHECK-NEXT: i8x16.extract_lane_u $push4=, $0, 1
+; CHECK-NEXT: i8x16.extract_lane_u $push1=, $2, 1
+; CHECK-NEXT: i32.const $push96=, 7
+; CHECK-NEXT: i32.and $push3=, $pop1, $pop96
+; CHECK-NEXT: i32.shl $push5=, $pop4, $pop3
+; CHECK-NEXT: i8x16.replace_lane $push11=, $pop10, 1, $pop5
+; ...
+; CHECK:      i8x16.extract_lane_u $push79=, $0, 15
+; CHECK-NEXT: i8x16.extract_lane_u $push77=, $2, 15
+; CHECK-NEXT: i32.const $push82=, 7
+; CHECK-NEXT: i32.and $push78=, $pop77, $pop82
+; CHECK-NEXT: i32.shl $push80=, $pop79, $pop78
+; CHECK-NEXT: i8x16.replace_lane $push81=, $pop76, 15, $pop80
+; CHECK-NEXT: return $pop81
+define <16 x i8> @shl_abs(<16 x i8> %v, i8 %a) {
+  %t1 = insertelement <16 x i8> undef, i8 %a, i32 0
+  %va = shufflevector <16 x i8> %t1, <16 x i8> undef, <16 x i32> zeroinitializer
+  %nva = sub <16 x i8> zeroinitializer, %va
+  %c = icmp sgt <16 x i8> %va, zeroinitializer
+  %shift = select <16 x i1> %c, <16 x i8> %va, <16 x i8> %nva
+  %r = shl <16 x i8> %v, %shift
+  ret <16 x i8> %r
+}
+
+; CHECK-LABEL: shl_abs_add:
+; CHECK-NEXT: .functype shl_abs_add (v128, i32, i32) -> (v128)
+; CHECK-NEXT: i8x16.extract_lane_u $push11=, $0, 0
+; CHECK-NEXT: i8x16.splat $push1=, $1
+; CHECK-NEXT: i8x16.splat $push0=, $2
+; CHECK-NEXT: i8x16.add $push2=, $pop1, $pop0
+; CHECK-NEXT: v8x16.shuffle $push3=, $pop2, $0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK-NEXT: i8x16.abs $push101=, $pop3
+; CHECK-NEXT: local.tee $push100=, $3=, $pop101
+; CHECK-NEXT: i8x16.extract_lane_u $push9=, $pop100, 0
+; CHECK-NEXT: i32.const $push5=, 7
+; CHECK-NEXT: i32.and $push10=, $pop9, $pop5
+; CHECK-NEXT: i32.shl $push12=, $pop11, $pop10
+; CHECK-NEXT: i8x16.splat $push13=, $pop12
+; CHECK-NEXT: i8x16.extract_lane_u $push7=, $0, 1
+; CHECK-NEXT: i8x16.extract_lane_u $push4=, $3, 1
+; CHECK-NEXT: i32.const $push99=, 7
+; CHECK-NEXT: i32.and $push6=, $pop4, $pop99
+; CHECK-NEXT: i32.shl $push8=, $pop7, $pop6
+; CHECK-NEXT: i8x16.replace_lane $push14=, $pop13, 1, $pop8
+; ...
+; CHECK:      i8x16.extract_lane_u $push82=, $0, 15
+; CHECK-NEXT: i8x16.extract_lane_u $push80=, $3, 15
+; CHECK-NEXT: i32.const $push85=, 7
+; CHECK-NEXT: i32.and $push81=, $pop80, $pop85
+; CHECK-NEXT: i32.shl $push83=, $pop82, $pop81
+; CHECK-NEXT: i8x16.replace_lane $push84=, $pop79, 15, $pop83
+; CHECK-NEXT: return $pop84
+define <16 x i8> @shl_abs_add(<16 x i8> %v, i8 %a, i8 %b) {
+  %t1 = insertelement <16 x i8> undef, i8 %a, i32 0
+  %va = shufflevector <16 x i8> %t1, <16 x i8> undef, <16 x i32> zeroinitializer
+  %t2 = insertelement <16 x i8> undef, i8 %b, i32 0
+  %vb = shufflevector <16 x i8> %t2, <16 x i8> undef, <16 x i32> zeroinitializer
+  %vadd = add <16 x i8> %va, %vb
+  %nvadd = sub <16 x i8> zeroinitializer, %vadd
+  %c = icmp sgt <16 x i8> %vadd, zeroinitializer
+  %shift = select <16 x i1> %c, <16 x i8> %vadd, <16 x i8> %nvadd
+  %r = shl <16 x i8> %v, %shift
+  ret <16 x i8> %r
+}
diff --git a/llvm/test/CodeGen/WebAssembly/simd-shift-unroll.ll b/llvm/test/CodeGen/WebAssembly/simd-shift-unroll.ll
deleted file mode 100644
index 2a5422cb01105..0000000000000
--- a/llvm/test/CodeGen/WebAssembly/simd-shift-unroll.ll
+++ /dev/null
@@ -1,128 +0,0 @@
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 | FileCheck %s --check-prefixes CHECK,SIMD128,SIMD128-SLOW
-
-;; Test that the custom shift unrolling works correctly in cases that
-;; cause assertion failures due to illegal types when using
-;; DAG.UnrollVectorOp. Regression test for PR45178.
-
-target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
-target triple = "wasm32-unknown-unknown"
-
-; CHECK-LABEL: shl_v16i8:
-; CHECK-NEXT: .functype       shl_v16i8 (v128) -> (v128)
-; CHECK-NEXT: i8x16.extract_lane_u    $push0=, $0, 0
-; CHECK-NEXT: i32.const       $push1=, 3
-; CHECK-NEXT: i32.shl         $push2=, $pop0, $pop1
-; CHECK-NEXT: i8x16.splat     $push3=, $pop2
-; CHECK-NEXT: i8x16.extract_lane_u    $push4=, $0, 1
-; CHECK-NEXT: i8x16.replace_lane      $push5=, $pop3, 1, $pop4
-; ...
-; CHECK:      i8x16.extract_lane_u    $push32=, $0, 15
-; CHECK-NEXT: i8x16.replace_lane      $push33=, $pop31, 15, $pop32
-; CHECK-NEXT: v8x16.shuffle   $push34=, $pop33, $0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK-NEXT: return  $pop34
-define <16 x i8> @shl_v16i8(<16 x i8> %in) {
-  %out = shl <16 x i8> %in,
-    <i8 3, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0,
-     i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
-  %ret = shufflevector <16 x i8> %out, <16 x i8> undef, <16 x i32> zeroinitializer
-  ret <16 x i8> %ret
-}
-
-; CHECK-LABEL: shr_s_v16i8:
-; CHECK-NEXT: functype       shr_s_v16i8 (v128) -> (v128)
-; CHECK-NEXT: i8x16.extract_lane_s    $push0=, $0, 0
-; CHECK-NEXT: i32.const       $push1=, 3
-; CHECK-NEXT: i32.shr_s       $push2=, $pop0, $pop1
-; CHECK-NEXT: i8x16.splat     $push3=, $pop2
-; CHECK-NEXT: i8x16.extract_lane_s    $push4=, $0, 1
-; CHECK-NEXT: i8x16.replace_lane      $push5=, $pop3, 1, $pop4
-; ...
-; CHECK:      i8x16.extract_lane_s    $push32=, $0, 15
-; CHECK-NEXT: i8x16.replace_lane      $push33=, $pop31, 15, $pop32
-; CHECK-NEXT: v8x16.shuffle   $push34=, $pop33, $0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK-NEXT: return  $pop34
-define <16 x i8> @shr_s_v16i8(<16 x i8> %in) {
-  %out = ashr <16 x i8> %in,
-    <i8 3, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0,
-     i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
-  %ret = shufflevector <16 x i8> %out, <16 x i8> undef, <16 x i32> zeroinitializer
-  ret <16 x i8> %ret
-}
-
-; CHECK-LABEL: shr_u_v16i8:
-; CHECK-NEXT: functype       shr_u_v16i8 (v128) -> (v128)
-; CHECK-NEXT: i8x16.extract_lane_u    $push0=, $0, 0
-; CHECK-NEXT: i32.const       $push1=, 3
-; CHECK-NEXT: i32.shr_u       $push2=, $pop0, $pop1
-; CHECK-NEXT: i8x16.splat     $push3=, $pop2
-; CHECK-NEXT: i8x16.extract_lane_u    $push4=, $0, 1
-; CHECK-NEXT: i8x16.replace_lane      $push5=, $pop3, 1, $pop4
-; ...
-; CHECK:      i8x16.extract_lane_u    $push32=, $0, 15
-; CHECK-NEXT: i8x16.replace_lane      $push33=, $pop31, 15, $pop32
-; CHECK-NEXT: v8x16.shuffle   $push34=, $pop33, $0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK-NEXT: return  $pop34
-define <16 x i8> @shr_u_v16i8(<16 x i8> %in) {
-  %out = lshr <16 x i8> %in,
-    <i8 3, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0,
-     i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
-  %ret = shufflevector <16 x i8> %out, <16 x i8> undef, <16 x i32> zeroinitializer
-  ret <16 x i8> %ret
-}
-
-; CHECK-LABEL: shl_v8i16:
-; CHECK-NEXT: functype       shl_v8i16 (v128) -> (v128)
-; CHECK-NEXT: i16x8.extract_lane_u    $push0=, $0, 0
-; CHECK-NEXT: i32.const       $push1=, 9
-; CHECK-NEXT: i32.shl         $push2=, $pop0, $pop1
-; CHECK-NEXT: i16x8.splat     $push3=, $pop2
-; CHECK-NEXT: i16x8.extract_lane_u    $push4=, $0, 1
-; CHECK-NEXT: i16x8.replace_lane      $push5=, $pop3, 1, $pop4
-; ...
-; CHECK:      i16x8.extract_lane_u    $push16=, $0, 7
-; CHECK-NEXT: i16x8.replace_lane      $push17=, $pop15, 7, $pop16
-; CHECK-NEXT: v8x16.shuffle   $push18=, $pop17, $0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK-NEXT: return  $pop18
-define <8 x i16> @shl_v8i16(<8 x i16> %in) {
-  %out = shl <8 x i16> %in, <i16 9, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
-  %ret = shufflevector <8 x i16> %out, <8 x i16> undef, <8 x i32> zeroinitializer
-  ret <8 x i16> %ret
-}
-
-; CHECK-LABEL: shr_s_v8i16:
-; CHECK-NEXT: functype       shr_s_v8i16 (v128) -> (v128)
-; CHECK-NEXT: i16x8.extract_lane_s    $push0=, $0, 0
-; CHECK-NEXT: i32.const       $push1=, 9
-; CHECK-NEXT: i32.shr_s       $push2=, $pop0, $pop1
-; CHECK-NEXT: i16x8.splat     $push3=, $pop2
-; CHECK-NEXT: i16x8.extract_lane_s    $push4=, $0, 1
-; CHECK-NEXT: i16x8.replace_lane      $push5=, $pop3, 1, $pop4
-; ...
-; CHECK:      i16x8.extract_lane_s    $push16=, $0, 7
-; CHECK-NEXT: i16x8.replace_lane      $push17=, $pop15, 7, $pop16
-; CHECK-NEXT: v8x16.shuffle   $push18=, $pop17, $0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK-NEXT: return  $pop18
-define <8 x i16> @shr_s_v8i16(<8 x i16> %in) {
-  %out = ashr <8 x i16> %in, <i16 9, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
-  %ret = shufflevector <8 x i16> %out, <8 x i16> undef, <8 x i32> zeroinitializer
-  ret <8 x i16> %ret
-}
-
-; CHECK-LABEL: shr_u_v8i16:
-; CHECK-NEXT: functype       shr_u_v8i16 (v128) -> (v128)
-; CHECK-NEXT: i16x8.extract_lane_u    $push0=, $0, 0
-; CHECK-NEXT: i32.const       $push1=, 9
-; CHECK-NEXT: i32.shr_u       $push2=, $pop0, $pop1
-; CHECK-NEXT: i16x8.splat     $push3=, $pop2
-; CHECK-NEXT: i16x8.extract_lane_u    $push4=, $0, 1
-; CHECK-NEXT: i16x8.replace_lane      $push5=, $pop3, 1, $pop4
-; ...
-; CHECK:      i16x8.extract_lane_u    $push16=, $0, 7
-; CHECK-NEXT: i16x8.replace_lane      $push17=, $pop15, 7, $pop16
-; CHECK-NEXT: v8x16.shuffle   $push18=, $pop17, $0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
-; CHECK-NEXT: return  $pop18
-define <8 x i16> @shr_u_v8i16(<8 x i16> %in) {
-  %out = lshr <8 x i16> %in, <i16 9, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
-  %ret = shufflevector <8 x i16> %out, <8 x i16> undef, <8 x i32> zeroinitializer
-  ret <8 x i16> %ret
-}
diff --git a/llvm/test/CodeGen/WebAssembly/simd-shuffle-bitcast.ll b/llvm/test/CodeGen/WebAssembly/simd-shuffle-bitcast.ll
index 7990608f6f535..b3e2db98861d5 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-shuffle-bitcast.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-shuffle-bitcast.ll
@@ -5,7 +5,7 @@
 ; DAG combine, this ends up doing both a splat and a shuffle.
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
-target triple = "wasm32-unknown-unkown"
+target triple = "wasm32-unknown-unknown"
 
 ; CHECK-LABEL: f32x4_splat:
 ; CHECK-NEXT: .functype f32x4_splat (f32) -> (v128){{$}}
diff --git a/llvm/test/CodeGen/WebAssembly/simd.ll b/llvm/test/CodeGen/WebAssembly/simd.ll
index 2934d2c9beaca..25e647f07230a 100644
--- a/llvm/test/CodeGen/WebAssembly/simd.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd.ll
@@ -36,7 +36,7 @@ define <16 x i8> @splat_v16i8(i8 %x) {
 }
 
 ; CHECK-LABEL: const_splat_v16i8:
-; SIMD128: i8x16.splat
+; SIMD128: v128.const $push0=, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42{{$}}
 define <16 x i8> @const_splat_v16i8() {
   ret <16 x i8> <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42,
                  i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
@@ -299,7 +299,7 @@ define <8 x i16> @splat_v8i16(i16 %x) {
 }
 
 ; CHECK-LABEL: const_splat_v8i16:
-; SIMD128: i16x8.splat
+; SIMD128: v128.const $push0=, 42, 42, 42, 42, 42, 42, 42, 42{{$}}
 define <8 x i16> @const_splat_v8i16() {
   ret <8 x i16> <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>
 }
@@ -547,7 +547,7 @@ define <4 x i32> @splat_v4i32(i32 %x) {
 }
 
 ; CHECK-LABEL: const_splat_v4i32:
-; SIMD128: i32x4.splat
+; SIMD128: v128.const $push0=, 42, 42, 42, 42{{$}}
 define <4 x i32> @const_splat_v4i32() {
   ret <4 x i32> <i32 42, i32 42, i32 42, i32 42>
 }
@@ -698,7 +698,7 @@ define <2 x i64> @splat_v2i64(i64 %x) {
 }
 
 ; CHECK-LABEL: const_splat_v2i64:
-; SIMD128: i64x2.splat
+; SIMD128: v128.const $push0=, 42, 42{{$}}
 define <2 x i64> @const_splat_v2i64() {
   ret <2 x i64> <i64 42, i64 42>
 }
@@ -847,7 +847,7 @@ define <4 x float> @splat_v4f32(float %x) {
 }
 
 ; CHECK-LABEL: const_splat_v4f32
-; SIMD128: f32x4.splat
+; SIMD128: v128.const $push0=, 0x1.5p5, 0x1.5p5, 0x1.5p5, 0x1.5p5{{$}}
 define <4 x float> @const_splat_v4f32() {
   ret <4 x float> <float 42., float 42., float 42., float 42.>
 }
@@ -998,7 +998,7 @@ define <2 x double> @splat_v2f64(double %x) {
 }
 
 ; CHECK-LABEL: const_splat_v2f64:
-; SIMD128: f64x2.splat
+; SIMD128: v128.const $push0=, 0x1.5p5, 0x1.5p5{{$}}
 define <2 x double> @const_splat_v2f64() {
   ret <2 x double> <double 42., double 42.>
 }
diff --git a/llvm/test/CodeGen/WebAssembly/stack-alignment.ll b/llvm/test/CodeGen/WebAssembly/stack-alignment.ll
index 770dfea78a267..229843ed41902 100644
--- a/llvm/test/CodeGen/WebAssembly/stack-alignment.ll
+++ b/llvm/test/CodeGen/WebAssembly/stack-alignment.ll
@@ -1,22 +1,20 @@
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers | FileCheck %s
-
-target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
-target triple = "wasm32-unknown-unknown"
+; RUN: llc < %s --mtriple=wasm32-unknown-unknown -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers | FileCheck -DPTR=32 %s
+; RUN: llc < %s --mtriple=wasm64-unknown-unknown -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers | FileCheck -DPTR=64 %s
 
 declare void @somefunc(i32*)
 
 ; CHECK-LABEL: underalign:
 ; CHECK:      global.get $push[[L1:.+]]=, __stack_pointer{{$}}
-; CHECK-NEXT: i32.const $push[[L2:.+]]=, 16
-; CHECK-NEXT: i32.sub   $push[[L10:.+]]=, $pop[[L1]], $pop[[L2]]
+; CHECK-NEXT: i[[PTR]].const $push[[L2:.+]]=, 16
+; CHECK-NEXT: i[[PTR]].sub $push[[L10:.+]]=, $pop[[L1]], $pop[[L2]]
 ; CHECK-NEXT: local.tee $push{{.+}}=, [[SP:.+]], $pop[[L10]]
 
 ; CHECK:      local.get $push[[L3:.+]]=, [[SP]]{{$}}
-; CHECK:      i32.add   $push[[underaligned:.+]]=, $pop[[L3]], $pop{{.+}}
-; CHECK-NEXT: call      somefunc, $pop[[underaligned]]
+; CHECK:      i[[PTR]].add $push[[underaligned:.+]]=, $pop[[L3]], $pop{{.+}}
+; CHECK-NEXT: call somefunc, $pop[[underaligned]]
 
 ; CHECK:      local.get $push[[M4:.+]]=, [[SP]]{{$}}
-; CHECK:      i32.add   $push[[L5:.+]]=, $pop[[M4]], $pop{{.+}}
+; CHECK:      i[[PTR]].add $push[[L5:.+]]=, $pop[[M4]], $pop{{.+}}
 ; CHECK-NEXT: global.set __stack_pointer, $pop[[L5]]
 define void @underalign() {
 entry:
@@ -27,17 +25,17 @@ entry:
 
 ; CHECK-LABEL: overalign:
 ; CHECK:      global.get $push[[L10:.+]]=, __stack_pointer{{$}}
-; CHECK-NEXT: local.tee  $push[[L9:.+]]=, [[BP:.+]], $pop[[L10]]
-; CHECK-NEXT: i32.const  $push[[L2:.+]]=, 32
-; CHECK-NEXT: i32.sub    $push[[L8:.+]]=, $pop[[L9]], $pop[[L2]]
-; CHECK-NEXT: i32.const  $push[[L3:.+]]=, -32
-; CHECK-NEXT: i32.and    $push[[L7:.+]]=, $pop[[L8]], $pop[[L3]]
-; CHECK-NEXT: local.tee  $push{{.+}}=, [[SP:.+]], $pop[[L7]]
+; CHECK-NEXT: local.tee $push[[L9:.+]]=, [[BP:.+]], $pop[[L10]]
+; CHECK-NEXT: i[[PTR]].const $push[[L2:.+]]=, 32
+; CHECK-NEXT: i[[PTR]].sub $push[[L8:.+]]=, $pop[[L9]], $pop[[L2]]
+; CHECK-NEXT: i[[PTR]].const $push[[L3:.+]]=, -32
+; CHECK-NEXT: i[[PTR]].and $push[[L7:.+]]=, $pop[[L8]], $pop[[L3]]
+; CHECK-NEXT: local.tee $push{{.+}}=, [[SP:.+]], $pop[[L7]]
 
-; CHECK:      local.get  $push[[M5:.+]]=, [[SP]]{{$}}
-; CHECK:      call       somefunc, $pop[[M5]]{{$}}
+; CHECK:      local.get $push[[M5:.+]]=, [[SP]]{{$}}
+; CHECK:      call somefunc, $pop[[M5]]{{$}}
 
-; CHECK:      local.get  $push[[M6:.+]]=, [[BP]]{{$}}
+; CHECK:      local.get $push[[M6:.+]]=, [[BP]]{{$}}
 ; CHECK-NEXT: global.set __stack_pointer, $pop[[M6]]
 define void @overalign() {
 entry:
@@ -48,19 +46,19 @@ entry:
 
 ; CHECK-LABEL: over_and_normal_align:
 ; CHECK:      global.get $push[[L14:.+]]=, __stack_pointer{{$}}
-; CHECK-NEXT: local.tee  $push[[L13:.+]]=, [[BP:.+]], $pop[[L14]]
-; CHECK:      i32.sub    $push[[L12:.+]]=, $pop[[L13]], $pop{{.+}}
-; CHECK:      i32.and    $push[[L11:.+]]=, $pop[[L12]], $pop{{.+}}
-; CHECK-NEXT: local.tee  $push{{.+}}=, [[SP:.+]], $pop[[L11]]
+; CHECK-NEXT: local.tee $push[[L13:.+]]=, [[BP:.+]], $pop[[L14]]
+; CHECK:      i[[PTR]].sub $push[[L12:.+]]=, $pop[[L13]], $pop{{.+}}
+; CHECK:      i[[PTR]].and $push[[L11:.+]]=, $pop[[L12]], $pop{{.+}}
+; CHECK-NEXT: local.tee $push{{.+}}=, [[SP:.+]], $pop[[L11]]
 
 ; CHECK:      local.get  $push[[M6:.+]]=, [[SP]]{{$}}
-; CHECK:      i32.add    $push[[L6:.+]]=, $pop[[M6]], $pop{{.+}}
-; CHECK-NEXT: call       somefunc, $pop[[L6]]
-; CHECK:      local.get  $push[[M7:.+]]=, [[SP]]{{$}}
-; CHECK:      i32.add    $push[[L8:.+]]=, $pop[[M7]], $pop{{.+}}
-; CHECK-NEXT: call       somefunc, $pop[[L8]]
+; CHECK:      i[[PTR]].add $push[[L6:.+]]=, $pop[[M6]], $pop{{.+}}
+; CHECK-NEXT: call somefunc, $pop[[L6]]
+; CHECK:      local.get $push[[M7:.+]]=, [[SP]]{{$}}
+; CHECK:      i[[PTR]].add $push[[L8:.+]]=, $pop[[M7]], $pop{{.+}}
+; CHECK-NEXT: call somefunc, $pop[[L8]]
 
-; CHECK:      local.get  $push[[L6:.+]]=, [[BP]]{{$}}
+; CHECK:      local.get $push[[L6:.+]]=, [[BP]]{{$}}
 ; CHECK-NEXT: global.set __stack_pointer, $pop[[L6]]
 define void @over_and_normal_align() {
 entry:
@@ -73,14 +71,14 @@ entry:
 
 ; CHECK-LABEL: dynamic_overalign:
 ; CHECK:      global.get $push[[L18:.+]]=, __stack_pointer{{$}}
-; CHECK-NEXT: local.tee  $push[[L17:.+]]=, [[SP:.+]], $pop[[L18]]
-; CHECK-NEXT: local.set  [[BP:.+]], $pop[[L17]]
-; CHECK:      local.tee  $push{{.+}}=, [[SP_2:.+]], $pop{{.+}}
+; CHECK-NEXT: local.tee $push[[L17:.+]]=, [[SP:.+]], $pop[[L18]]
+; CHECK-NEXT: local.set [[BP:.+]], $pop[[L17]]
+; CHECK:      local.tee $push{{.+}}=, [[SP_2:.+]], $pop{{.+}}
 
-; CHECK:      local.get  $push[[M8:.+]]=, [[SP_2]]{{$}}
-; CHECK:      call       somefunc, $pop[[M8]]
+; CHECK:      local.get $push[[M8:.+]]=, [[SP_2]]{{$}}
+; CHECK:      call somefunc, $pop[[M8]]
 
-; CHECK:      local.get  $push[[M9:.+]]=, [[BP]]{{$}}
+; CHECK:      local.get $push[[M9:.+]]=, [[BP]]{{$}}
 ; CHECK-NEXT: global.set __stack_pointer, $pop[[M9]]
 define void @dynamic_overalign(i32 %num) {
 entry:
@@ -91,18 +89,18 @@ entry:
 
 ; CHECK-LABEL: overalign_and_dynamic:
 ; CHECK:      global.get $push[[L21:.+]]=, __stack_pointer{{$}}
-; CHECK-NEXT: local.tee  $push[[L20:.+]]=, [[BP:.+]], $pop[[L21]]
-; CHECK:      i32.sub    $push[[L19:.+]]=, $pop[[L20]], $pop{{.+}}
-; CHECK:      i32.and    $push[[L18:.+]]=, $pop[[L19]], $pop{{.+}}
-; CHECK:      local.tee  $push{{.+}}=, [[FP:.+]], $pop[[L18]]
-; CHECK:      local.get  $push[[M10:.+]]=, [[FP]]{{$}}
-; CHECK:      i32.sub    $push[[L16:.+]]=, $pop[[M10]], $pop{{.+}}
-; CHECK-NEXT: local.tee  $push{{.+}}=, [[SP:.+]], $pop[[L16]]
-
-; CHECK:      local.get  $push[[over:.+]]=, [[FP]]
-; CHECK-NEXT: call       somefunc, $pop[[over]]
-; CHECK:      local.get  $push[[another:.+]]=, [[SP]]
-; CHECK-NEXT: call       somefunc, $pop[[another]]
+; CHECK-NEXT: local.tee $push[[L20:.+]]=, [[BP:.+]], $pop[[L21]]
+; CHECK:      i[[PTR]].sub $push[[L19:.+]]=, $pop[[L20]], $pop{{.+}}
+; CHECK:      i[[PTR]].and $push[[L18:.+]]=, $pop[[L19]], $pop{{.+}}
+; CHECK:      local.tee $push{{.+}}=, [[FP:.+]], $pop[[L18]]
+; CHECK:      local.get $push[[M10:.+]]=, [[FP]]{{$}}
+; CHECK:      i[[PTR]].sub $push[[L16:.+]]=, $pop[[M10]], $pop{{.+}}
+; CHECK-NEXT: local.tee $push{{.+}}=, [[SP:.+]], $pop[[L16]]
+
+; CHECK:      local.get $push[[over:.+]]=, [[FP]]
+; CHECK-NEXT: call somefunc, $pop[[over]]
+; CHECK:      local.get $push[[another:.+]]=, [[SP]]
+; CHECK-NEXT: call somefunc, $pop[[another]]
 
 ; CHECK:      local.get  $push[[M11:.+]]=, [[BP]]{{$}}
 ; CHECK-NEXT: global.set __stack_pointer, $pop[[M11]]
@@ -117,23 +115,23 @@ entry:
 
 ; CHECK-LABEL: overalign_static_and_dynamic:
 ; CHECK:      global.get $push[[L26:.+]]=, __stack_pointer{{$}}
-; CHECK-NEXT: local.tee  $push[[L25:.+]]=, [[BP:.+]], $pop[[L26]]
-; CHECK:      i32.sub    $push[[L24:.+]]=, $pop[[L25]], $pop{{.+}}
-; CHECK:      i32.and    $push[[L23:.+]]=, $pop[[L24]], $pop{{.+}}
-; CHECK:      local.tee  $push{{.+}}=, [[FP:.+]], $pop[[L23]]
-; CHECK:      local.get  $push[[M12:.+]]=, [[FP]]{{$}}
-; CHECK:      i32.sub    $push[[L21:.+]]=, $pop[[M12]], $pop{{.+}}
-; CHECK-NEXT: local.tee  $push{{.+}}=, [[SP:.+]], $pop[[L21]]
-
-; CHECK:      local.get  $push[[L19:.+]]=, [[FP]]
-; CHECK:      local.tee  $push[[L18:.+]]=, [[FP_2:.+]], $pop[[L19]]
-; CHECK:      i32.add    $push[[over:.+]]=, $pop[[L18]], $pop{{.+}}
-; CHECK-NEXT: call       somefunc, $pop[[over]]
-; CHECK:      local.get  $push[[M12:.+]]=, [[SP]]
-; CHECK:      call       somefunc, $pop[[M12]]
-; CHECK:      local.get  $push[[M13:.+]]=, [[FP_2]]
-; CHECK:      i32.add    $push[[static:.+]]=, $pop[[M13]], $pop{{.+}}
-; CHECK-NEXT: call       somefunc, $pop[[static]]
+; CHECK-NEXT: local.tee $push[[L25:.+]]=, [[BP:.+]], $pop[[L26]]
+; CHECK:      i[[PTR]].sub $push[[L24:.+]]=, $pop[[L25]], $pop{{.+}}
+; CHECK:      i[[PTR]].and $push[[L23:.+]]=, $pop[[L24]], $pop{{.+}}
+; CHECK:      local.tee $push{{.+}}=, [[FP:.+]], $pop[[L23]]
+; CHECK:      local.get $push[[M12:.+]]=, [[FP]]{{$}}
+; CHECK:      i[[PTR]].sub $push[[L21:.+]]=, $pop[[M12]], $pop{{.+}}
+; CHECK-NEXT: local.tee $push{{.+}}=, [[SP:.+]], $pop[[L21]]
+
+; CHECK:      local.get $push[[L19:.+]]=, [[FP]]
+; CHECK:      local.tee $push[[L18:.+]]=, [[FP_2:.+]], $pop[[L19]]
+; CHECK:      i[[PTR]].add $push[[over:.+]]=, $pop[[L18]], $pop{{.+}}
+; CHECK-NEXT: call somefunc, $pop[[over]]
+; CHECK:      local.get $push[[M12:.+]]=, [[SP]]
+; CHECK:      call somefunc, $pop[[M12]]
+; CHECK:      local.get $push[[M13:.+]]=, [[FP_2]]
+; CHECK:      i[[PTR]].add $push[[static:.+]]=, $pop[[M13]], $pop{{.+}}
+; CHECK-NEXT: call somefunc, $pop[[static]]
 
 ; CHECK:      local.get  $push[[M14:.+]]=, [[BP]]{{$}}
 ; CHECK-NEXT: global.set __stack_pointer, $pop[[M14]]
diff --git a/llvm/test/CodeGen/WebAssembly/stack-protector.ll b/llvm/test/CodeGen/WebAssembly/stack-protector.ll
index 2b6521b437708..eda24bcb366b6 100644
--- a/llvm/test/CodeGen/WebAssembly/stack-protector.ll
+++ b/llvm/test/CodeGen/WebAssembly/stack-protector.ll
@@ -1,13 +1,13 @@
 ; RUN: llc -verify-machineinstrs -mtriple=wasm32-unknown-unknown < %s | FileCheck -check-prefix=WASM32 %s
 
-; WASM32: i32.load        28
-; WASM32-NEXT: i32.ne
-; WASM32-NEXT: br_if           0
-
-; WASM32: __stack_chk_fail
-
 @"\01LC" = internal constant [11 x i8] c"buf == %s\0A\00"		; <[11 x i8]*> [#uses=1]
 
+; WASM32-LABEL: test
+; WASM32:      i32.load        28
+; WASM32:      br_if           0
+; WASM32:      call __stack_chk_fail
+; WASM32-NEXT: unreachable
+
 define void @test(i8* %a) nounwind ssp {
 entry:
 	%a_addr = alloca i8*		; <i8**> [#uses=2]
@@ -25,6 +25,27 @@ return:		; preds = %entry
 	ret void
 }
 
+; WASM32-LABEL: test_return_i32
+; WASM32:      call __stack_chk_fail
+; WASM32-NEXT: unreachable
+
+define i32 @test_return_i32(i8* %a) nounwind ssp {
+entry:
+  %a_addr = alloca i8*    ; <i8**> [#uses=2]
+  %buf = alloca [8 x i8]    ; <[8 x i8]*> [#uses=2]
+  %"alloca point" = bitcast i32 0 to i32    ; <i32> [#uses=0]
+  store i8* %a, i8** %a_addr
+  %buf1 = bitcast [8 x i8]* %buf to i8*    ; <i8*> [#uses=1]
+  %0 = load i8*, i8** %a_addr, align 4    ; <i8*> [#uses=1]
+  %1 = call i8* @strcpy(i8* %buf1, i8* %0) nounwind    ; <i8*> [#uses=0]
+  %buf2 = bitcast [8 x i8]* %buf to i8*    ; <i8*> [#uses=1]
+  %2 = call i32 (i8*, ...) @printf(i8* getelementptr ([11 x i8], [11 x i8]* @"\01LC", i32 0, i32 0), i8* %buf2) nounwind    ; <i32> [#uses=0]
+  br label %return
+
+return:    ; preds = %entry
+  ret i32 0
+}
+
 declare i8* @strcpy(i8*, i8*) nounwind
 
 declare i32 @printf(i8*, ...) nounwind
diff --git a/llvm/test/CodeGen/WebAssembly/switch.ll b/llvm/test/CodeGen/WebAssembly/switch.ll
index 3a9da703e7898..fbb59ddd10610 100644
--- a/llvm/test/CodeGen/WebAssembly/switch.ll
+++ b/llvm/test/CodeGen/WebAssembly/switch.ll
@@ -95,26 +95,30 @@ sw.epilog:                                        ; preds = %entry, %sw.bb.5, %s
 
 ; CHECK-LABEL: bar64:
 ; CHECK: block   {{$}}
+; CHECK: i64.const
+; CHECK: i64.gt_u
+; CHECK: br_if 0
 ; CHECK: block   {{$}}
 ; CHECK: block   {{$}}
 ; CHECK: block   {{$}}
 ; CHECK: block   {{$}}
 ; CHECK: block   {{$}}
 ; CHECK: block   {{$}}
-; CHECK: br_table {{[^,]+}}, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 4, 5, 6{{$}}
-; CHECK: .LBB{{[0-9]+}}_1:
-; CHECK:   call foo0{{$}}
+; CHECK: i32.wrap_i64
+; CHECK: br_table {{[^,]+}}, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 4, 5, 0{{$}}
 ; CHECK: .LBB{{[0-9]+}}_2:
-; CHECK:   call foo1{{$}}
+; CHECK:   call foo0{{$}}
 ; CHECK: .LBB{{[0-9]+}}_3:
-; CHECK:   call foo2{{$}}
+; CHECK:   call foo1{{$}}
 ; CHECK: .LBB{{[0-9]+}}_4:
-; CHECK:   call foo3{{$}}
+; CHECK:   call foo2{{$}}
 ; CHECK: .LBB{{[0-9]+}}_5:
-; CHECK:   call foo4{{$}}
+; CHECK:   call foo3{{$}}
 ; CHECK: .LBB{{[0-9]+}}_6:
-; CHECK:   call foo5{{$}}
+; CHECK:   call foo4{{$}}
 ; CHECK: .LBB{{[0-9]+}}_7:
+; CHECK:   call foo5{{$}}
+; CHECK: .LBB{{[0-9]+}}_8:
 ; CHECK:   return{{$}}
 define void @bar64(i64 %n) {
 entry:
@@ -172,3 +176,43 @@ sw.bb.5:                                          ; preds = %entry
 sw.epilog:                                        ; preds = %entry, %sw.bb.5, %sw.bb.4, %sw.bb.3, %sw.bb.2, %sw.bb.1, %sw.bb
   ret void
 }
+
+; CHECK-LABEL: truncated:
+; CHECK:   block
+; CHECK:   block
+; CHECK:   block
+; CHECK:   i32.wrap_i64
+; CHECK:   br_table {{[^,]+}}, 0, 1, 2{{$}}
+; CHECK: .LBB{{[0-9]+}}_1
+; CHECK:   end_block
+; CHECK:   call foo0{{$}}
+; CHECK:   return{{$}}
+; CHECK: .LBB{{[0-9]+}}_2
+; CHECK:   end_block
+; CHECK:   call foo1{{$}}
+; CHECK:   return{{$}}
+; CHECK: .LBB{{[0-9]+}}_3
+; CHECK:   end_block
+; CHECK:   call foo2{{$}}
+; CHECK:   return{{$}}
+; CHECK:   end_function
+define void @truncated(i64 %n) {
+entry:
+  %m = trunc i64 %n to i32
+  switch i32 %m, label %default [
+    i32 0, label %bb1
+    i32 1, label %bb2
+  ]
+
+bb1:
+  tail call void @foo0()
+  ret void
+
+bb2:
+  tail call void @foo1()
+  ret void
+
+default:
+  tail call void @foo2()
+  ret void
+}
diff --git a/llvm/test/CodeGen/WebAssembly/userstack.ll b/llvm/test/CodeGen/WebAssembly/userstack.ll
index dd73c2ed9a47a..dec202ea6af9a 100644
--- a/llvm/test/CodeGen/WebAssembly/userstack.ll
+++ b/llvm/test/CodeGen/WebAssembly/userstack.ll
@@ -1,18 +1,16 @@
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers | FileCheck %s
-
-target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
-target triple = "wasm32-unknown-unknown"
+; RUN: llc < %s --mtriple=wasm32-unknown-unknown -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers | FileCheck -DPTR=32 %s
+; RUN: llc < %s --mtriple=wasm64-unknown-unknown -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers | FileCheck -DPTR=64 %s
 
 declare void @ext_func(i64* %ptr)
 declare void @ext_func_i32(i32* %ptr)
 
 ; CHECK-LABEL: alloca32:
 ; Check that there is an extra local for the stack pointer.
-; CHECK: .local i32{{$}}
+; CHECK: .local i[[PTR]]{{$}}
 define void @alloca32() noredzone {
  ; CHECK-NEXT: global.get $push[[L2:.+]]=, __stack_pointer{{$}}
- ; CHECK-NEXT: i32.const $push[[L3:.+]]=, 16
- ; CHECK-NEXT: i32.sub $push[[L9:.+]]=, $pop[[L2]], $pop[[L3]]
+ ; CHECK-NEXT: i[[PTR]].const $push[[L3:.+]]=, 16
+ ; CHECK-NEXT: i[[PTR]].sub $push[[L9:.+]]=, $pop[[L2]], $pop[[L3]]
  ; CHECK-NEXT: local.tee $push[[L8:.+]]=, [[SP:.+]], $pop[[L9]]{{$}}
  ; CHECK-NEXT: global.set __stack_pointer, $pop[[L8]]{{$}}
  %retval = alloca i32
@@ -21,18 +19,18 @@ define void @alloca32() noredzone {
  ; CHECK: i32.store 12($pop[[L4]]), $pop[[L0]]
  store i32 0, i32* %retval
  ; CHECK: local.get $push[[L6:.+]]=, [[SP]]{{$}}
- ; CHECK-NEXT: i32.const $push[[L5:.+]]=, 16
- ; CHECK-NEXT: i32.add $push[[L7:.+]]=, $pop[[L6]], $pop[[L5]]
+ ; CHECK-NEXT: i[[PTR]].const $push[[L5:.+]]=, 16
+ ; CHECK-NEXT: i[[PTR]].add $push[[L7:.+]]=, $pop[[L6]], $pop[[L5]]
  ; CHECK-NEXT: global.set __stack_pointer, $pop[[L7]]
  ret void
 }
 
 ; CHECK-LABEL: alloca3264:
-; CHECK: .local i32{{$}}
+; CHECK: .local i[[PTR]]{{$}}
 define void @alloca3264() {
  ; CHECK: global.get $push[[L3:.+]]=, __stack_pointer{{$}}
- ; CHECK-NEXT: i32.const $push[[L4:.+]]=, 16
- ; CHECK-NEXT: i32.sub $push[[L6:.+]]=, $pop[[L3]], $pop[[L4]]
+ ; CHECK-NEXT: i[[PTR]].const $push[[L4:.+]]=, 16
+ ; CHECK-NEXT: i[[PTR]].sub $push[[L6:.+]]=, $pop[[L3]], $pop[[L4]]
  ; CHECK-NEXT: local.tee $push[[L5:.+]]=, [[SP:.+]], $pop[[L6]]
  %r1 = alloca i32
  %r2 = alloca double
@@ -48,17 +46,17 @@ define void @alloca3264() {
 }
 
 ; CHECK-LABEL: allocarray:
-; CHECK: .local i32{{$}}
+; CHECK: .local i[[PTR]]{{$}}
 define void @allocarray() {
  ; CHECK-NEXT: global.get $push[[L4:.+]]=, __stack_pointer{{$}}
- ; CHECK-NEXT: i32.const $push[[L5:.+]]=, 144{{$}}
- ; CHECK-NEXT: i32.sub $push[[L12:.+]]=, $pop[[L4]], $pop[[L5]]
+ ; CHECK-NEXT: i[[PTR]].const $push[[L5:.+]]=, 144{{$}}
+ ; CHECK-NEXT: i[[PTR]].sub $push[[L12:.+]]=, $pop[[L4]], $pop[[L5]]
  ; CHECK-NEXT: local.tee $push[[L11:.+]]=, 0, $pop[[L12]]
  ; CHECK-NEXT: global.set __stack_pointer, $pop[[L11]]
  %r = alloca [33 x i32]
 
- ; CHECK:      i32.const $push{{.+}}=, 24
- ; CHECK-NEXT: i32.add $push[[L3:.+]]=, $pop{{.+}}, $pop{{.+}}
+ ; CHECK:      i[[PTR]].const $push{{.+}}=, 24
+ ; CHECK-NEXT: i[[PTR]].add $push[[L3:.+]]=, $pop{{.+}}, $pop{{.+}}
  ; CHECK-NEXT: i32.const $push[[L1:.+]]=, 1{{$}}
  ; CHECK-NEXT: i32.store 0($pop[[L3]]), $pop[[L1]]{{$}}
  ; CHECK-NEXT: local.get $push[[L4:.+]]=, 0{{$}}
@@ -70,16 +68,16 @@ define void @allocarray() {
  store i32 1, i32* %p2
 
  ; CHECK-NEXT: local.get $push[[L2:.+]]=, [[SP]]{{$}}
- ; CHECK-NEXT: i32.const $push[[L7:.+]]=, 144
- ; CHECK-NEXT: i32.add $push[[L8:.+]]=, $pop[[L2]], $pop[[L7]]
+ ; CHECK-NEXT: i[[PTR]].const $push[[L7:.+]]=, 144
+ ; CHECK-NEXT: i[[PTR]].add $push[[L8:.+]]=, $pop[[L2]], $pop[[L7]]
  ; CHECK-NEXT: global.set __stack_pointer, $pop[[L8]]
  ret void
 }
 
 ; CHECK-LABEL: non_mem_use
 define void @non_mem_use(i8** %addr) {
- ; CHECK: i32.const $push[[L2:.+]]=, 48
- ; CHECK-NEXT: i32.sub $push[[L12:.+]]=, {{.+}}, $pop[[L2]]
+ ; CHECK: i[[PTR]].const $push[[L2:.+]]=, 48
+ ; CHECK-NEXT: i[[PTR]].sub $push[[L12:.+]]=, {{.+}}, $pop[[L2]]
  ; CHECK-NEXT: local.tee $push[[L11:.+]]=, [[SP:.+]], $pop[[L12]]
  ; CHECK-NEXT: global.set __stack_pointer, $pop[[L11]]
  %buf = alloca [27 x i8], align 16
@@ -87,8 +85,8 @@ define void @non_mem_use(i8** %addr) {
  %r2 = alloca i64
  ; %r is at SP+8
  ; CHECK: local.get $push[[L3:.+]]=, [[SP]]
- ; CHECK: i32.const $push[[OFF:.+]]=, 8
- ; CHECK-NEXT: i32.add $push[[ARG1:.+]]=, $pop[[L3]], $pop[[OFF]]
+ ; CHECK: i[[PTR]].const $push[[OFF:.+]]=, 8
+ ; CHECK-NEXT: i[[PTR]].add $push[[ARG1:.+]]=, $pop[[L3]], $pop[[OFF]]
  ; CHECK-NEXT: call ext_func, $pop[[ARG1]]
  call void @ext_func(i64* %r)
  ; %r2 is at SP+0, no add needed
@@ -98,20 +96,20 @@ define void @non_mem_use(i8** %addr) {
  ; Use as a value, but in a store
  ; %buf is at SP+16
  ; CHECK: local.get $push[[L5:.+]]=, [[SP]]
- ; CHECK: i32.const $push[[OFF:.+]]=, 16
- ; CHECK-NEXT: i32.add $push[[VAL:.+]]=, $pop[[L5]], $pop[[OFF]]
- ; CHECK-NEXT: i32.store 0($pop{{.+}}), $pop[[VAL]]
+ ; CHECK: i[[PTR]].const $push[[OFF:.+]]=, 16
+ ; CHECK-NEXT: i[[PTR]].add $push[[VAL:.+]]=, $pop[[L5]], $pop[[OFF]]
+ ; CHECK-NEXT: i[[PTR]].store 0($pop{{.+}}), $pop[[VAL]]
  %gep = getelementptr inbounds [27 x i8], [27 x i8]* %buf, i32 0, i32 0
  store i8* %gep, i8** %addr
  ret void
 }
 
 ; CHECK-LABEL: allocarray_inbounds:
-; CHECK: .local i32{{$}}
+; CHECK: .local i[[PTR]]{{$}}
 define void @allocarray_inbounds() {
  ; CHECK: global.get $push[[L3:.+]]=, __stack_pointer{{$}}
- ; CHECK-NEXT: i32.const $push[[L4:.+]]=, 32{{$}}
- ; CHECK-NEXT: i32.sub $push[[L11:.+]]=, $pop[[L3]], $pop[[L4]]
+ ; CHECK-NEXT: i[[PTR]].const $push[[L4:.+]]=, 32{{$}}
+ ; CHECK-NEXT: i[[PTR]].sub $push[[L11:.+]]=, $pop[[L3]], $pop[[L4]]
  ; CHECK-NEXT: local.tee $push[[L10:.+]]=, [[SP:.+]], $pop[[L11]]
  ; CHECK-NEXT: global.set __stack_pointer, $pop[[L10]]{{$}}
  %r = alloca [5 x i32]
@@ -125,8 +123,8 @@ define void @allocarray_inbounds() {
  store i32 1, i32* %p2
  call void @ext_func(i64* null);
  ; CHECK: call ext_func
- ; CHECK: i32.const $push[[L5:.+]]=, 32{{$}}
- ; CHECK-NEXT: i32.add $push[[L7:.+]]=, ${{.+}}, $pop[[L5]]
+ ; CHECK: i[[PTR]].const $push[[L5:.+]]=, 32{{$}}
+ ; CHECK-NEXT: i[[PTR]].add $push[[L7:.+]]=, ${{.+}}, $pop[[L5]]
  ; CHECK-NEXT: global.set __stack_pointer, $pop[[L7]]
  ret void
 }
@@ -136,7 +134,7 @@ define void @dynamic_alloca(i32 %alloc) {
  ; CHECK: global.get $push[[L13:.+]]=, __stack_pointer{{$}}
  ; CHECK-NEXT: local.tee $push[[L12:.+]]=, [[SP:.+]], $pop[[L13]]{{$}}
  ; Target independent codegen bumps the stack pointer.
- ; CHECK: i32.sub
+ ; CHECK: i[[PTR]].sub
  ; Check that SP is written back to memory after decrement
  ; CHECK: global.set __stack_pointer,
  %r = alloca i32, i32 %alloc
@@ -152,12 +150,12 @@ define void @dynamic_alloca_redzone(i32 %alloc) {
  ; CHECK: global.get $push[[L13:.+]]=, __stack_pointer{{$}}
  ; CHECK-NEXT: local.tee $push[[L12:.+]]=, [[SP:.+]], $pop[[L13]]{{$}}
  ; Target independent codegen bumps the stack pointer
- ; CHECK: i32.sub
+ ; CHECK: i[[PTR]].sub
  %r = alloca i32, i32 %alloc
- ; CHECK-NEXT: local.tee       $push[[L8:.+]]=, {{.+}}, $pop
- ; CHECK: local.get $push[[L7:.+]]=, 0{{$}}
- ; CHECK-NEXT: i32.const       $push[[L6:.+]]=, 0{{$}}
- ; CHECK-NEXT: i32.store       0($pop[[L7]]), $pop[[L6]]{{$}}
+ ; CHECK-NEXT: local.tee $push[[L8:.+]]=, [[SP2:.+]], $pop
+ ; CHECK: local.get $push[[L7:.+]]=, [[SP2]]{{$}}
+ ; CHECK-NEXT: i32.const $push[[L6:.+]]=, 0{{$}}
+ ; CHECK-NEXT: i32.store 0($pop[[L7]]), $pop[[L6]]{{$}}
  store i32 0, i32* %r
  ; CHECK-NEXT: return
  ret void
@@ -167,8 +165,8 @@ define void @dynamic_alloca_redzone(i32 %alloc) {
 define void @dynamic_static_alloca(i32 %alloc) noredzone {
  ; Decrement SP in the prolog by the static amount and writeback to memory.
  ; CHECK: global.get $push[[L11:.+]]=, __stack_pointer{{$}}
- ; CHECK-NEXT: i32.const $push[[L12:.+]]=, 16
- ; CHECK-NEXT: i32.sub $push[[L23:.+]]=, $pop[[L11]], $pop[[L12]]
+ ; CHECK-NEXT: i[[PTR]].const $push[[L12:.+]]=, 16
+ ; CHECK-NEXT: i[[PTR]].sub $push[[L23:.+]]=, $pop[[L11]], $pop[[L12]]
  ; CHECK-NEXT: local.tee $push[[L22:.+]]=, [[SP:.+]], $pop[[L23]]
  ; CHECK-NEXT: global.set __stack_pointer, $pop[[L22]]
 
@@ -181,7 +179,7 @@ define void @dynamic_static_alloca(i32 %alloc) noredzone {
  store volatile i32 101, i32* %static
 
  ; Decrement SP in the body by the dynamic amount.
- ; CHECK: i32.sub
+ ; CHECK: i[[PTR]].sub
  ; CHECK: local.tee $push[[L16:.+]]=, [[dynamic_local:.+]], $pop{{.+}}
  ; CHECK: local.tee $push[[L15:.+]]=, [[other:.+]], $pop[[L16]]{{$}}
  ; CHECK: global.set __stack_pointer, $pop[[L15]]{{$}}
@@ -201,7 +199,7 @@ define void @dynamic_static_alloca(i32 %alloc) noredzone {
  store volatile i32 103, i32* %dynamic
 
  ; Decrement SP in the body by the dynamic amount.
- ; CHECK: i32.sub
+ ; CHECK: i[[PTR]].sub
  ; CHECK: local.tee $push{{.+}}=, [[dynamic2_local:.+]], $pop{{.+}}
  %dynamic.2 = alloca i32, i32 %alloc
 
@@ -224,8 +222,8 @@ define void @dynamic_static_alloca(i32 %alloc) noredzone {
 
  ; Writeback to memory.
  ; CHECK: local.get $push[[L24:.+]]=, [[FP]]{{$}}
- ; CHECK: i32.const $push[[L18:.+]]=, 16
- ; CHECK-NEXT: i32.add $push[[L19:.+]]=, $pop[[L24]], $pop[[L18]]
+ ; CHECK: i[[PTR]].const $push[[L18:.+]]=, 16
+ ; CHECK-NEXT: i[[PTR]].add $push[[L19:.+]]=, $pop[[L24]], $pop[[L18]]
  ; CHECK-NEXT: global.set __stack_pointer, $pop[[L19]]
  ret void
 }
@@ -273,11 +271,11 @@ define void @dynamic_alloca_nouse(i32 %alloc) noredzone {
 ; CHECK-LABEL: copytoreg_fi:
 define void @copytoreg_fi(i1 %cond, i32* %b) {
 entry:
- ; CHECK: i32.const $push[[L1:.+]]=, 16
- ; CHECK-NEXT: i32.sub $push[[L3:.+]]=, {{.+}}, $pop[[L1]]
+ ; CHECK: i[[PTR]].const $push[[L1:.+]]=, 16
+ ; CHECK-NEXT: i[[PTR]].sub $push[[L3:.+]]=, {{.+}}, $pop[[L1]]
  %addr = alloca i32
- ; CHECK: i32.const $push[[OFF:.+]]=, 12
- ; CHECK-NEXT: i32.add $push[[ADDR:.+]]=, $pop[[L3]], $pop[[OFF]]
+ ; CHECK: i[[PTR]].const $push[[OFF:.+]]=, 12
+ ; CHECK-NEXT: i[[PTR]].add $push[[ADDR:.+]]=, $pop[[L3]], $pop[[OFF]]
  ; CHECK-NEXT: local.set [[COPY:.+]], $pop[[ADDR]]
  br label %body
 body:
@@ -309,7 +307,7 @@ define void @frameaddress_0() {
 ; Test __builtin_frame_address(1).
 
 ; CHECK-LABEL: frameaddress_1:
-; CHECK:      i32.const $push0=, 0{{$}}
+; CHECK:      i[[PTR]].const $push0=, 0{{$}}
 ; CHECK-NEXT: call use_i8_star, $pop0{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @frameaddress_1() {
@@ -330,6 +328,6 @@ define void @inline_asm() {
   ret void
 }
 
-; CHECK: .globaltype	__stack_pointer, i32{{$}}
+; CHECK: .globaltype	__stack_pointer, i[[PTR]]{{$}}
 
 ; TODO: test over-aligned alloca
diff --git a/llvm/test/CodeGen/X86/AMX/amx-bf16-intrinsics.ll b/llvm/test/CodeGen/X86/AMX/amx-bf16-intrinsics.ll
new file mode 100644
index 0000000000000..a415d9c152422
--- /dev/null
+++ b/llvm/test/CodeGen/X86/AMX/amx-bf16-intrinsics.ll
@@ -0,0 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-tile -mattr=+amx-bf16 -verify-machineinstrs | FileCheck %s
+
+define void @test_amx() {
+; CHECK-LABEL: test_amx:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    tdpbf16ps %tmm7, %tmm4, %tmm3
+; CHECK-NEXT:    retq
+  call void @llvm.x86.tdpbf16ps(i8 3, i8 4, i8 7)
+  ret void
+}
+
+declare void @llvm.x86.tdpbf16ps(i8 %tile0, i8 %tile1, i8 %tile2)
diff --git a/llvm/test/CodeGen/X86/AMX/amx-int8-intrinsics.ll b/llvm/test/CodeGen/X86/AMX/amx-int8-intrinsics.ll
new file mode 100644
index 0000000000000..49e69aeab5109
--- /dev/null
+++ b/llvm/test/CodeGen/X86/AMX/amx-int8-intrinsics.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-int8 -verify-machineinstrs | FileCheck %s
+
+define void @test_amx() {
+; CHECK-LABEL: test_amx:
+; CHECK:       # %bb.0:
+  call void @llvm.x86.tdpbssd(i8 3, i8 4, i8 7)
+; CHECK-NEXT:    tdpbssd %tmm7, %tmm4, %tmm3
+
+  call void @llvm.x86.tdpbsud(i8 3, i8 4, i8 7)
+; CHECK-NEXT:    tdpbsud %tmm7, %tmm4, %tmm3
+
+  call void @llvm.x86.tdpbusd(i8 3, i8 0, i8 7)
+; CHECK-NEXT:    tdpbusd %tmm7, %tmm0, %tmm3
+
+  call void @llvm.x86.tdpbuud(i8 3, i8 4, i8 1)
+; CHECK-NEXT:    tdpbuud %tmm1, %tmm4, %tmm3
+  ret void
+}
+
+declare void @llvm.x86.tdpbssd(i8 %tile0, i8 %tile1, i8 %tile2)
+declare void @llvm.x86.tdpbsud(i8 %tile0, i8 %tile1, i8 %tile2)
+declare void @llvm.x86.tdpbusd(i8 %tile0, i8 %tile1, i8 %tile2)
+declare void @llvm.x86.tdpbuud(i8 %tile0, i8 %tile1, i8 %tile2)
diff --git a/llvm/test/CodeGen/X86/AMX/amx-tile-intrinsics.ll b/llvm/test/CodeGen/X86/AMX/amx-tile-intrinsics.ll
new file mode 100644
index 0000000000000..6b8e040abb9ab
--- /dev/null
+++ b/llvm/test/CodeGen/X86/AMX/amx-tile-intrinsics.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-tile -verify-machineinstrs | FileCheck %s
+
+define void @test_amx(i8* %pointer, i8* %base, i64 %stride) {
+; CHECK-LABEL: test_amx:
+; CHECK:       # %bb.0:
+  call void @llvm.x86.ldtilecfg(i8* %pointer)
+; CHECK-NEXT:    ldtilecfg (%rdi)
+
+  call void @llvm.x86.sttilecfg(i8* %pointer)
+; CHECK-NEXT:    sttilecfg (%rdi)
+
+  call void @llvm.x86.tilerelease()
+; CHECK-NEXT:    tilerelease
+
+  call void @llvm.x86.tilezero(i8 3)
+; CHECK-NEXT:    tilezero %tmm3
+
+  call void @llvm.x86.tileloadd64(i8 3, i8* %base, i64 %stride)
+; CHECK-NEXT:    tileloadd (%rsi,%rdx), %tmm3
+
+  call void @llvm.x86.tileloaddt164(i8 3, i8* %base, i64 %stride)
+; CHECK-NEXT:    tileloaddt1 (%rsi,%rdx), %tmm3
+
+  call void @llvm.x86.tilestored64(i8 3, i8* %base, i64 %stride)
+; CHECK-NEXT:    tilestored %tmm3, (%rsi,%rdx)
+  ret void
+}
+
+declare void @llvm.x86.tileloadd64(i8 %tile, i8* %base, i64 %stride)
+declare void @llvm.x86.tileloaddt164(i8 %tile, i8* %base, i64 %stride)
+declare void @llvm.x86.tilestored64(i8 %tile, i8* %base, i64 %stride)
+declare void @llvm.x86.ldtilecfg(i8* %pointer)
+declare void @llvm.x86.sttilecfg(i8* %pointer)
+declare void @llvm.x86.tilerelease()
+declare void @llvm.x86.tilezero(i8 %tile)
diff --git a/llvm/test/CodeGen/X86/O0-pipeline.ll b/llvm/test/CodeGen/X86/O0-pipeline.ll
index a1cd828abeab4..528b3c39c8794 100644
--- a/llvm/test/CodeGen/X86/O0-pipeline.ll
+++ b/llvm/test/CodeGen/X86/O0-pipeline.ll
@@ -46,7 +46,6 @@
 ; CHECK-NEXT:       Fast Register Allocator
 ; CHECK-NEXT:       Bundle Machine CFG Edges
 ; CHECK-NEXT:       X86 FP Stackifier
-; CHECK-NEXT:       X86 Load Value Injection (LVI) Load Hardening (Unoptimized)
 ; CHECK-NEXT:       Fixup Statepoint Caller Saved
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
diff --git a/llvm/test/CodeGen/X86/atomicf128.ll b/llvm/test/CodeGen/X86/atomicf128.ll
index 21115d4f9904a..cbec96c1c88f0 100644
--- a/llvm/test/CodeGen/X86/atomicf128.ll
+++ b/llvm/test/CodeGen/X86/atomicf128.ll
@@ -11,15 +11,20 @@ define void @atomic_fetch_swapf128(fp128 %x) nounwind {
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    movq _fsc128@{{.*}}(%rip), %rsi
-; CHECK-NEXT:    movq (%rsi), %rax
-; CHECK-NEXT:    movq 8(%rsi), %rdx
+; CHECK-NEXT:    movaps (%rsi), %xmm1
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB0_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq -{{[0-9]+}}(%rsp), %rbx
 ; CHECK-NEXT:    movq -{{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT:    movq -{{[0-9]+}}(%rsp), %rdx
 ; CHECK-NEXT:    lock cmpxchg16b (%rsi)
+; CHECK-NEXT:    movq %rdx, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps -{{[0-9]+}}(%rsp), %xmm1
 ; CHECK-NEXT:    jne LBB0_1
 ; CHECK-NEXT:  ## %bb.2: ## %atomicrmw.end
 ; CHECK-NEXT:    popq %rbx
diff --git a/llvm/test/CodeGen/X86/avg.ll b/llvm/test/CodeGen/X86/avg.ll
index c6c2230a1d77d..d2638a1681e85 100644
--- a/llvm/test/CodeGen/X86/avg.ll
+++ b/llvm/test/CodeGen/X86/avg.ll
@@ -233,9 +233,8 @@ define void @avg_v24i8(<24 x i8>* %a, <24 x i8>* %b) nounwind {
 ; AVX2-NEXT:    vpsrld $1, %ymm2, %ymm2
 ; AVX2-NEXT:    vpsrld $1, %ymm1, %ymm1
 ; AVX2-NEXT:    vpsrld $1, %ymm0, %ymm0
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm1[2,3],ymm0[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
@@ -597,25 +596,22 @@ define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind {
 ; AVX2-NEXT:    vpsubd %ymm6, %ymm5, %ymm5
 ; AVX2-NEXT:    vpsrld $1, %ymm5, %ymm5
 ; AVX2-NEXT:    vpsrld $1, %ymm4, %ymm4
+; AVX2-NEXT:    vpackusdw %ymm4, %ymm5, %ymm4
 ; AVX2-NEXT:    vpsrld $1, %ymm3, %ymm3
 ; AVX2-NEXT:    vpsrld $1, %ymm2, %ymm2
+; AVX2-NEXT:    vpackusdw %ymm2, %ymm3, %ymm2
 ; AVX2-NEXT:    vpsrld $1, %ymm1, %ymm1
 ; AVX2-NEXT:    vpsrld $1, %ymm0, %ymm0
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm6 = ymm1[2,3],ymm0[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX2-NEXT:    vpackusdw %ymm6, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm6 = ymm3[2,3],ymm2[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX2-NEXT:    vpackusdw %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
 ; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm2
 ; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm2[2,3],ymm0[2,3]
 ; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm5[2,3],ymm4[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm3
-; AVX2-NEXT:    vpackusdw %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm4[0,2,1,3]
 ; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/avx-vperm2x128.ll b/llvm/test/CodeGen/X86/avx-vperm2x128.ll
index 27edb3155a391..26a5cd328d5c7 100644
--- a/llvm/test/CodeGen/X86/avx-vperm2x128.ll
+++ b/llvm/test/CodeGen/X86/avx-vperm2x128.ll
@@ -394,6 +394,14 @@ define <4 x double> @shuffle_v4f64_zz23_optsize(<4 x double> %a) optsize {
   %s = shufflevector <4 x double> %a, <4 x double> <double 0.0, double 0.0, double undef, double undef>, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
   ret <4 x double> %s
 }
+define <4 x double> @shuffle_v4f64_zz23_pgso(<4 x double> %a) !prof !14 {
+; ALL-LABEL: shuffle_v4f64_zz23_pgso:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vperm2f128 {{.*#+}} ymm0 = zero,zero,ymm0[2,3]
+; ALL-NEXT:    retq
+  %s = shufflevector <4 x double> %a, <4 x double> <double 0.0, double 0.0, double undef, double undef>, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  ret <4 x double> %s
+}
 
 define <4 x double> @shuffle_v4f64_zz45(<4 x double> %a) {
 ; ALL-LABEL: shuffle_v4f64_zz45:
@@ -429,6 +437,14 @@ define <4 x double> @shuffle_v4f64_zz67_optsize(<4 x double> %a) optsize {
   %s = shufflevector <4 x double> <double 0.0, double 0.0, double undef, double undef>, <4 x double> %a, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   ret <4 x double> %s
 }
+define <4 x double> @shuffle_v4f64_zz67_pgso(<4 x double> %a) !prof !14 {
+; ALL-LABEL: shuffle_v4f64_zz67_pgso:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vperm2f128 {{.*#+}} ymm0 = zero,zero,ymm0[2,3]
+; ALL-NEXT:    retq
+  %s = shufflevector <4 x double> <double 0.0, double 0.0, double undef, double undef>, <4 x double> %a, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  ret <4 x double> %s
+}
 
 define <4 x double> @shuffle_v4f64_01zz(<4 x double> %a) {
 ; ALL-LABEL: shuffle_v4f64_01zz:
@@ -603,9 +619,8 @@ entry:
 define <4 x i64> @ld0_hi0_lo1_4i64(<4 x i64> * %pa, <4 x i64> %b) nounwind uwtable readnone ssp {
 ; AVX1-LABEL: ld0_hi0_lo1_4i64:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = mem[2,3],ymm0[0,1]
-; AVX1-NEXT:    vpaddq {{.*}}(%rip), %xmm0, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT:    vpaddq {{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vpaddq {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -647,12 +662,10 @@ entry:
 define <8 x i32> @ld0_hi0_lo1_8i32(<8 x i32> * %pa, <8 x i32> %b) nounwind uwtable readnone ssp {
 ; AVX1-LABEL: ld0_hi0_lo1_8i32:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = mem[2,3],ymm0[0,1]
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [1,2,3,4]
-; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,2,3,4]
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddd 16(%rdi), %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ld0_hi0_lo1_8i32:
@@ -670,9 +683,9 @@ entry:
 define <8 x i32> @ld1_hi0_hi1_8i32(<8 x i32> %a, <8 x i32> * %pb) nounwind uwtable readnone ssp {
 ; AVX1-LABEL: ld1_hi0_hi1_8i32:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,2,3,4]
 ; AVX1-NEXT:    vpaddd 16(%rdi), %xmm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -688,3 +701,20 @@ entry:
   %res = add <8 x i32> %shuffle, <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
   ret <8 x i32> %res
 }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/avx2-phaddsub.ll b/llvm/test/CodeGen/X86/avx2-phaddsub.ll
index 23e2fd69cf07e..cbe98904c17f2 100644
--- a/llvm/test/CodeGen/X86/avx2-phaddsub.ll
+++ b/llvm/test/CodeGen/X86/avx2-phaddsub.ll
@@ -1,19 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686--   -mattr=+avx2           | FileCheck %s --check-prefixes=X32,X32-SLOW
-; RUN: llc < %s -mtriple=i686--   -mattr=+avx2,fast-hops | FileCheck %s --check-prefixes=X32,X32-FAST
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2           | FileCheck %s --check-prefixes=X64,X64-SLOW
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,fast-hops | FileCheck %s --check-prefixes=X64,X64-FAST
+; RUN: llc < %s -mtriple=i686--   -mattr=+avx2           | FileCheck %s --check-prefixes=CHECK,X86,SLOW,X86-SLOW
+; RUN: llc < %s -mtriple=i686--   -mattr=+avx2,fast-hops | FileCheck %s --check-prefixes=CHECK,X86,FAST,X86-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2           | FileCheck %s --check-prefixes=CHECK,X64,SLOW,X64-SLOW
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,fast-hops | FileCheck %s --check-prefixes=CHECK,X64,FAST,X64-FAST
 
 define <16 x i16> @phaddw1(<16 x i16> %x, <16 x i16> %y) {
-; X32-LABEL: phaddw1:
-; X32:       # %bb.0:
-; X32-NEXT:    vphaddw %ymm1, %ymm0, %ymm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: phaddw1:
-; X64:       # %bb.0:
-; X64-NEXT:    vphaddw %ymm1, %ymm0, %ymm0
-; X64-NEXT:    retq
+; CHECK-LABEL: phaddw1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vphaddw %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %a = shufflevector <16 x i16> %x, <16 x i16> %y, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
   %b = shufflevector <16 x i16> %x, <16 x i16> %y, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
   %r = add <16 x i16> %a, %b
@@ -21,15 +16,10 @@ define <16 x i16> @phaddw1(<16 x i16> %x, <16 x i16> %y) {
 }
 
 define <16 x i16> @phaddw2(<16 x i16> %x, <16 x i16> %y) {
-; X32-LABEL: phaddw2:
-; X32:       # %bb.0:
-; X32-NEXT:    vphaddw %ymm1, %ymm0, %ymm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: phaddw2:
-; X64:       # %bb.0:
-; X64-NEXT:    vphaddw %ymm1, %ymm0, %ymm0
-; X64-NEXT:    retq
+; CHECK-LABEL: phaddw2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vphaddw %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %a = shufflevector <16 x i16> %x, <16 x i16> %y, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
   %b = shufflevector <16 x i16> %y, <16 x i16> %x, <16 x i32> <i32 16, i32 18, i32 20, i32 22, i32 0, i32 2, i32 4, i32 6, i32 24, i32 26, i32 28, i32 30, i32 8, i32 10, i32 12, i32 14>
   %r = add <16 x i16> %a, %b
@@ -37,15 +27,10 @@ define <16 x i16> @phaddw2(<16 x i16> %x, <16 x i16> %y) {
 }
 
 define <8 x i32> @phaddd1(<8 x i32> %x, <8 x i32> %y) {
-; X32-LABEL: phaddd1:
-; X32:       # %bb.0:
-; X32-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: phaddd1:
-; X64:       # %bb.0:
-; X64-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
-; X64-NEXT:    retq
+; CHECK-LABEL: phaddd1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %a = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
   %b = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
   %r = add <8 x i32> %a, %b
@@ -53,15 +38,10 @@ define <8 x i32> @phaddd1(<8 x i32> %x, <8 x i32> %y) {
 }
 
 define <8 x i32> @phaddd2(<8 x i32> %x, <8 x i32> %y) {
-; X32-LABEL: phaddd2:
-; X32:       # %bb.0:
-; X32-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: phaddd2:
-; X64:       # %bb.0:
-; X64-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
-; X64-NEXT:    retq
+; CHECK-LABEL: phaddd2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %a = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> <i32 1, i32 2, i32 9, i32 10, i32 5, i32 6, i32 13, i32 14>
   %b = shufflevector <8 x i32> %y, <8 x i32> %x, <8 x i32> <i32 8, i32 11, i32 0, i32 3, i32 12, i32 15, i32 4, i32 7>
   %r = add <8 x i32> %a, %b
@@ -69,15 +49,10 @@ define <8 x i32> @phaddd2(<8 x i32> %x, <8 x i32> %y) {
 }
 
 define <8 x i32> @phaddd3(<8 x i32> %x) {
-; X32-LABEL: phaddd3:
-; X32:       # %bb.0:
-; X32-NEXT:    vphaddd %ymm0, %ymm0, %ymm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: phaddd3:
-; X64:       # %bb.0:
-; X64-NEXT:    vphaddd %ymm0, %ymm0, %ymm0
-; X64-NEXT:    retq
+; CHECK-LABEL: phaddd3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vphaddd %ymm0, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %a = shufflevector <8 x i32> %x, <8 x i32> undef, <8 x i32> <i32 undef, i32 2, i32 8, i32 10, i32 4, i32 6, i32 undef, i32 14>
   %b = shufflevector <8 x i32> %x, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 9, i32 undef, i32 5, i32 7, i32 13, i32 15>
   %r = add <8 x i32> %a, %b
@@ -85,15 +60,10 @@ define <8 x i32> @phaddd3(<8 x i32> %x) {
 }
 
 define <16 x i16> @phsubw1(<16 x i16> %x, <16 x i16> %y) {
-; X32-LABEL: phsubw1:
-; X32:       # %bb.0:
-; X32-NEXT:    vphsubw %ymm1, %ymm0, %ymm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: phsubw1:
-; X64:       # %bb.0:
-; X64-NEXT:    vphsubw %ymm1, %ymm0, %ymm0
-; X64-NEXT:    retq
+; CHECK-LABEL: phsubw1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vphsubw %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %a = shufflevector <16 x i16> %x, <16 x i16> %y, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
   %b = shufflevector <16 x i16> %x, <16 x i16> %y, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
   %r = sub <16 x i16> %a, %b
@@ -101,15 +71,10 @@ define <16 x i16> @phsubw1(<16 x i16> %x, <16 x i16> %y) {
 }
 
 define <8 x i32> @phsubd1(<8 x i32> %x, <8 x i32> %y) {
-; X32-LABEL: phsubd1:
-; X32:       # %bb.0:
-; X32-NEXT:    vphsubd %ymm1, %ymm0, %ymm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: phsubd1:
-; X64:       # %bb.0:
-; X64-NEXT:    vphsubd %ymm1, %ymm0, %ymm0
-; X64-NEXT:    retq
+; CHECK-LABEL: phsubd1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vphsubd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %a = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
   %b = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
   %r = sub <8 x i32> %a, %b
@@ -117,15 +82,10 @@ define <8 x i32> @phsubd1(<8 x i32> %x, <8 x i32> %y) {
 }
 
 define <8 x i32> @phsubd2(<8 x i32> %x, <8 x i32> %y) {
-; X32-LABEL: phsubd2:
-; X32:       # %bb.0:
-; X32-NEXT:    vphsubd %ymm1, %ymm0, %ymm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: phsubd2:
-; X64:       # %bb.0:
-; X64-NEXT:    vphsubd %ymm1, %ymm0, %ymm0
-; X64-NEXT:    retq
+; CHECK-LABEL: phsubd2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vphsubd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %a = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> <i32 0, i32 undef, i32 8, i32 undef, i32 4, i32 6, i32 12, i32 14>
   %b = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> <i32 1, i32 undef, i32 9, i32 11, i32 5, i32 7, i32 undef, i32 15>
   %r = sub <8 x i32> %a, %b
diff --git a/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll b/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
index aa07a4db8d597..c3d5f23d0fd22 100644
--- a/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
+++ b/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
@@ -4633,3 +4633,52 @@ bb1:                                              ; preds = %bb1, %bb
 bb10:                                             ; preds = %bb1
   ret void
 }
+
+; The or/and pattern here should be turned into vpternlog. The multiply is
+; there to increase the use count of the loads so they can't fold. We want to
+; unfold the broadcast and pull it out of the loop.
+define void @bcast_unfold_vpternlog_v16i32(i32* %arg, i32* %arg1) {
+; CHECK-LABEL: bcast_unfold_vpternlog_v16i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB131_1: # %bb2
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu64 4096(%rdi,%rax), %zmm1
+; CHECK-NEXT:    vmovdqu64 4096(%rsi,%rax), %zmm2
+; CHECK-NEXT:    vpmulld %zmm2, %zmm1, %zmm3
+; CHECK-NEXT:    vpternlogd $216, %zmm0, %zmm1, %zmm2
+; CHECK-NEXT:    vpmulld %zmm3, %zmm2, %zmm1
+; CHECK-NEXT:    vmovdqu64 %zmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $64, %rax
+; CHECK-NEXT:    jne .LBB131_1
+; CHECK-NEXT:  # %bb.2: # %bb20
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp18, %bb2 ]
+  %tmp3 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp4 = bitcast i32* %tmp3 to <16 x i32>*
+  %tmp5 = load <16 x i32>, <16 x i32>* %tmp4, align 4
+  %tmp6 = getelementptr inbounds i32, i32* %arg1, i64 %tmp
+  %tmp10 = bitcast i32* %tmp6 to <16 x i32>*
+  %tmp11 = load <16 x i32>, <16 x i32>* %tmp10, align 4
+  %tmp12 = and <16 x i32> %tmp5, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
+  %tmp13 = and <16 x i32> %tmp11, <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
+  %tmp14 = or <16 x i32> %tmp12, %tmp13
+  %tmp15 = mul <16 x i32> %tmp14, %tmp5
+  %tmp16 = mul <16 x i32> %tmp15, %tmp11
+  %tmp17 = bitcast i32* %tmp3 to <16 x i32>*
+  store <16 x i32> %tmp16, <16 x i32>* %tmp17, align 4
+  %tmp18 = add i64 %tmp, 16
+  %tmp19 = icmp eq i64 %tmp18, 1024
+  br i1 %tmp19, label %bb20, label %bb2
+
+bb20:                                             ; preds = %bb2
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/X86/avx512-cvt.ll b/llvm/test/CodeGen/X86/avx512-cvt.ll
index 5362e2b262a44..e92528b9bbcc8 100644
--- a/llvm/test/CodeGen/X86/avx512-cvt.ll
+++ b/llvm/test/CodeGen/X86/avx512-cvt.ll
@@ -321,8 +321,8 @@ define <4 x float> @ulto4f32(<4 x i64> %a) {
 define <8 x double> @ulto8f64(<8 x i64> %a) {
 ; NODQ-LABEL: ulto8f64:
 ; NODQ:       # %bb.0:
-; NODQ-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm1
-; NODQ-NEXT:    vporq {{.*}}(%rip){1to8}, %zmm1, %zmm1
+; NODQ-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
+; NODQ-NEXT:    vpternlogq $248, {{.*}}(%rip){1to8}, %zmm0, %zmm1
 ; NODQ-NEXT:    vpsrlq $32, %zmm0, %zmm0
 ; NODQ-NEXT:    vporq {{.*}}(%rip){1to8}, %zmm0, %zmm0
 ; NODQ-NEXT:    vsubpd {{.*}}(%rip){1to8}, %zmm0, %zmm0
@@ -346,21 +346,20 @@ define <16 x double> @ulto16f64(<16 x i64> %a) {
 ; NODQ-LABEL: ulto16f64:
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vpbroadcastq {{.*#+}} zmm2 = [4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295]
-; NODQ-NEXT:    vpandq %zmm2, %zmm0, %zmm3
-; NODQ-NEXT:    vpbroadcastq {{.*#+}} zmm4 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
-; NODQ-NEXT:    vporq %zmm4, %zmm3, %zmm3
+; NODQ-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
+; NODQ-NEXT:    vmovdqa64 %zmm3, %zmm4
+; NODQ-NEXT:    vpternlogq $248, %zmm2, %zmm0, %zmm4
 ; NODQ-NEXT:    vpsrlq $32, %zmm0, %zmm0
 ; NODQ-NEXT:    vpbroadcastq {{.*#+}} zmm5 = [4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072]
 ; NODQ-NEXT:    vporq %zmm5, %zmm0, %zmm0
 ; NODQ-NEXT:    vbroadcastsd {{.*#+}} zmm6 = [1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25]
 ; NODQ-NEXT:    vsubpd %zmm6, %zmm0, %zmm0
-; NODQ-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
-; NODQ-NEXT:    vpandq %zmm2, %zmm1, %zmm2
-; NODQ-NEXT:    vporq %zmm4, %zmm2, %zmm2
+; NODQ-NEXT:    vaddpd %zmm0, %zmm4, %zmm0
+; NODQ-NEXT:    vpternlogq $248, %zmm2, %zmm1, %zmm3
 ; NODQ-NEXT:    vpsrlq $32, %zmm1, %zmm1
 ; NODQ-NEXT:    vporq %zmm5, %zmm1, %zmm1
 ; NODQ-NEXT:    vsubpd %zmm6, %zmm1, %zmm1
-; NODQ-NEXT:    vaddpd %zmm1, %zmm2, %zmm1
+; NODQ-NEXT:    vaddpd %zmm1, %zmm3, %zmm1
 ; NODQ-NEXT:    retq
 ;
 ; VLDQ-LABEL: ulto16f64:
diff --git a/llvm/test/CodeGen/X86/avx512-gfni-intrinsics.ll b/llvm/test/CodeGen/X86/avx512-gfni-intrinsics.ll
index 241a6ea9995e7..65fcc7091a7a0 100644
--- a/llvm/test/CodeGen/X86/avx512-gfni-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512-gfni-intrinsics.ll
@@ -7,21 +7,21 @@ define <16 x i8> @test_vgf2p8affineinvqb_128(<16 x i8> %src1, <16 x i8> %src2, <
 ; X86-LABEL: test_vgf2p8affineinvqb_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vgf2p8affineinvqb $3, %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xf9,0xcf,0xd9,0x03]
-; X86-NEXT:    vgf2p8affineinvqb $4, %xmm1, %xmm0, %xmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0xcf,0xe1,0x04]
+; X86-NEXT:    vgf2p8affineinvqb $3, %xmm1, %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xf9,0xcf,0xe1,0x03]
+; X86-NEXT:    vgf2p8affineinvqb $4, %xmm1, %xmm0, %xmm3 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0xcf,0xd9,0x04]
 ; X86-NEXT:    vgf2p8affineinvqb $5, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0xcf,0xd1,0x05]
-; X86-NEXT:    vpxor %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xc3]
-; X86-NEXT:    vpxor %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xc0]
+; X86-NEXT:    vpternlogq $150, %xmm2, %xmm4, %xmm3 # encoding: [0x62,0xf3,0xdd,0x08,0x25,0xda,0x96]
+; X86-NEXT:    vmovdqa %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_vgf2p8affineinvqb_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vgf2p8affineinvqb $3, %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xf9,0xcf,0xd9,0x03]
-; X64-NEXT:    vgf2p8affineinvqb $4, %xmm1, %xmm0, %xmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0xcf,0xe1,0x04]
+; X64-NEXT:    vgf2p8affineinvqb $3, %xmm1, %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xf9,0xcf,0xe1,0x03]
+; X64-NEXT:    vgf2p8affineinvqb $4, %xmm1, %xmm0, %xmm3 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0xcf,0xd9,0x04]
 ; X64-NEXT:    vgf2p8affineinvqb $5, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0xcf,0xd1,0x05]
-; X64-NEXT:    vpxor %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xc3]
-; X64-NEXT:    vpxor %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xc0]
+; X64-NEXT:    vpternlogq $150, %xmm2, %xmm4, %xmm3 # encoding: [0x62,0xf3,0xdd,0x08,0x25,0xda,0x96]
+; X64-NEXT:    vmovdqa %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i16 %mask to <16 x i1>
   %2 = call <16 x i8> @llvm.x86.vgf2p8affineinvqb.128(<16 x i8> %src1, <16 x i8> %src2, i8 3)
@@ -39,21 +39,21 @@ define <32 x i8> @test_vgf2p8affineinvqb_256(<32 x i8> %src1, <32 x i8> %src2, <
 ; X86-LABEL: test_vgf2p8affineinvqb_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vgf2p8affineinvqb $3, %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xfd,0xcf,0xd9,0x03]
-; X86-NEXT:    vgf2p8affineinvqb $4, %ymm1, %ymm0, %ymm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xa9,0xcf,0xe1,0x04]
+; X86-NEXT:    vgf2p8affineinvqb $3, %ymm1, %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xfd,0xcf,0xe1,0x03]
+; X86-NEXT:    vgf2p8affineinvqb $4, %ymm1, %ymm0, %ymm3 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xa9,0xcf,0xd9,0x04]
 ; X86-NEXT:    vgf2p8affineinvqb $5, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0xcf,0xd1,0x05]
-; X86-NEXT:    vpxor %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xef,0xc3]
-; X86-NEXT:    vpxor %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xc0]
+; X86-NEXT:    vpternlogq $150, %ymm2, %ymm4, %ymm3 # encoding: [0x62,0xf3,0xdd,0x28,0x25,0xda,0x96]
+; X86-NEXT:    vmovdqa %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_vgf2p8affineinvqb_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vgf2p8affineinvqb $3, %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xfd,0xcf,0xd9,0x03]
-; X64-NEXT:    vgf2p8affineinvqb $4, %ymm1, %ymm0, %ymm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xa9,0xcf,0xe1,0x04]
+; X64-NEXT:    vgf2p8affineinvqb $3, %ymm1, %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xfd,0xcf,0xe1,0x03]
+; X64-NEXT:    vgf2p8affineinvqb $4, %ymm1, %ymm0, %ymm3 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xa9,0xcf,0xd9,0x04]
 ; X64-NEXT:    vgf2p8affineinvqb $5, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0xcf,0xd1,0x05]
-; X64-NEXT:    vpxor %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xef,0xc3]
-; X64-NEXT:    vpxor %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xc0]
+; X64-NEXT:    vpternlogq $150, %ymm2, %ymm4, %ymm3 # encoding: [0x62,0xf3,0xdd,0x28,0x25,0xda,0x96]
+; X64-NEXT:    vmovdqa %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i32 %mask to <32 x i1>
   %2 = call <32 x i8> @llvm.x86.vgf2p8affineinvqb.256(<32 x i8> %src1, <32 x i8> %src2, i8 3)
@@ -71,21 +71,21 @@ define <64 x i8> @test_vgf2p8affineinvqb_512(<64 x i8> %src1, <64 x i8> %src2, <
 ; X86-LABEL: test_vgf2p8affineinvqb_512:
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vgf2p8affineinvqb $3, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0xcf,0xd9,0x03]
-; X86-NEXT:    vgf2p8affineinvqb $4, %zmm1, %zmm0, %zmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xc9,0xcf,0xe1,0x04]
+; X86-NEXT:    vgf2p8affineinvqb $3, %zmm1, %zmm0, %zmm4 # encoding: [0x62,0xf3,0xfd,0x48,0xcf,0xe1,0x03]
+; X86-NEXT:    vgf2p8affineinvqb $4, %zmm1, %zmm0, %zmm3 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xc9,0xcf,0xd9,0x04]
 ; X86-NEXT:    vgf2p8affineinvqb $5, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0xcf,0xd1,0x05]
-; X86-NEXT:    vpxorq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xef,0xc3]
-; X86-NEXT:    vpxorq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xef,0xc0]
+; X86-NEXT:    vpternlogq $150, %zmm2, %zmm4, %zmm3 # encoding: [0x62,0xf3,0xdd,0x48,0x25,0xda,0x96]
+; X86-NEXT:    vmovdqa64 %zmm3, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_vgf2p8affineinvqb_512:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
-; X64-NEXT:    vgf2p8affineinvqb $3, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0xcf,0xd9,0x03]
-; X64-NEXT:    vgf2p8affineinvqb $4, %zmm1, %zmm0, %zmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xc9,0xcf,0xe1,0x04]
+; X64-NEXT:    vgf2p8affineinvqb $3, %zmm1, %zmm0, %zmm4 # encoding: [0x62,0xf3,0xfd,0x48,0xcf,0xe1,0x03]
+; X64-NEXT:    vgf2p8affineinvqb $4, %zmm1, %zmm0, %zmm3 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xc9,0xcf,0xd9,0x04]
 ; X64-NEXT:    vgf2p8affineinvqb $5, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0xcf,0xd1,0x05]
-; X64-NEXT:    vpxorq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xef,0xc3]
-; X64-NEXT:    vpxorq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xef,0xc0]
+; X64-NEXT:    vpternlogq $150, %zmm2, %zmm4, %zmm3 # encoding: [0x62,0xf3,0xdd,0x48,0x25,0xda,0x96]
+; X64-NEXT:    vmovdqa64 %zmm3, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i64 %mask to <64 x i1>
   %2 = call <64 x i8> @llvm.x86.vgf2p8affineinvqb.512(<64 x i8> %src1, <64 x i8> %src2, i8 3)
@@ -103,21 +103,21 @@ define <16 x i8> @test_vgf2p8affineqb_128(<16 x i8> %src1, <16 x i8> %src2, <16
 ; X86-LABEL: test_vgf2p8affineqb_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vgf2p8affineqb $3, %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xf9,0xce,0xd9,0x03]
-; X86-NEXT:    vgf2p8affineqb $4, %xmm1, %xmm0, %xmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0xce,0xe1,0x04]
+; X86-NEXT:    vgf2p8affineqb $3, %xmm1, %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xf9,0xce,0xe1,0x03]
+; X86-NEXT:    vgf2p8affineqb $4, %xmm1, %xmm0, %xmm3 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0xce,0xd9,0x04]
 ; X86-NEXT:    vgf2p8affineqb $5, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0xce,0xd1,0x05]
-; X86-NEXT:    vpxor %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xc3]
-; X86-NEXT:    vpxor %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xc0]
+; X86-NEXT:    vpternlogq $150, %xmm2, %xmm4, %xmm3 # encoding: [0x62,0xf3,0xdd,0x08,0x25,0xda,0x96]
+; X86-NEXT:    vmovdqa %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_vgf2p8affineqb_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vgf2p8affineqb $3, %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xf9,0xce,0xd9,0x03]
-; X64-NEXT:    vgf2p8affineqb $4, %xmm1, %xmm0, %xmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0xce,0xe1,0x04]
+; X64-NEXT:    vgf2p8affineqb $3, %xmm1, %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xf9,0xce,0xe1,0x03]
+; X64-NEXT:    vgf2p8affineqb $4, %xmm1, %xmm0, %xmm3 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0xce,0xd9,0x04]
 ; X64-NEXT:    vgf2p8affineqb $5, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0xce,0xd1,0x05]
-; X64-NEXT:    vpxor %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xc3]
-; X64-NEXT:    vpxor %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xc0]
+; X64-NEXT:    vpternlogq $150, %xmm2, %xmm4, %xmm3 # encoding: [0x62,0xf3,0xdd,0x08,0x25,0xda,0x96]
+; X64-NEXT:    vmovdqa %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i16 %mask to <16 x i1>
   %2 = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %src1, <16 x i8> %src2, i8 3)
@@ -135,21 +135,21 @@ define <32 x i8> @test_vgf2p8affineqb_256(<32 x i8> %src1, <32 x i8> %src2, <32
 ; X86-LABEL: test_vgf2p8affineqb_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vgf2p8affineqb $3, %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xfd,0xce,0xd9,0x03]
-; X86-NEXT:    vgf2p8affineqb $4, %ymm1, %ymm0, %ymm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xa9,0xce,0xe1,0x04]
+; X86-NEXT:    vgf2p8affineqb $3, %ymm1, %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xfd,0xce,0xe1,0x03]
+; X86-NEXT:    vgf2p8affineqb $4, %ymm1, %ymm0, %ymm3 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xa9,0xce,0xd9,0x04]
 ; X86-NEXT:    vgf2p8affineqb $5, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0xce,0xd1,0x05]
-; X86-NEXT:    vpxor %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xef,0xc3]
-; X86-NEXT:    vpxor %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xc0]
+; X86-NEXT:    vpternlogq $150, %ymm2, %ymm4, %ymm3 # encoding: [0x62,0xf3,0xdd,0x28,0x25,0xda,0x96]
+; X86-NEXT:    vmovdqa %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_vgf2p8affineqb_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vgf2p8affineqb $3, %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xfd,0xce,0xd9,0x03]
-; X64-NEXT:    vgf2p8affineqb $4, %ymm1, %ymm0, %ymm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xa9,0xce,0xe1,0x04]
+; X64-NEXT:    vgf2p8affineqb $3, %ymm1, %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xfd,0xce,0xe1,0x03]
+; X64-NEXT:    vgf2p8affineqb $4, %ymm1, %ymm0, %ymm3 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xa9,0xce,0xd9,0x04]
 ; X64-NEXT:    vgf2p8affineqb $5, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0xce,0xd1,0x05]
-; X64-NEXT:    vpxor %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xef,0xc3]
-; X64-NEXT:    vpxor %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xc0]
+; X64-NEXT:    vpternlogq $150, %ymm2, %ymm4, %ymm3 # encoding: [0x62,0xf3,0xdd,0x28,0x25,0xda,0x96]
+; X64-NEXT:    vmovdqa %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i32 %mask to <32 x i1>
   %2 = call <32 x i8> @llvm.x86.vgf2p8affineqb.256(<32 x i8> %src1, <32 x i8> %src2, i8 3)
@@ -167,21 +167,21 @@ define <64 x i8> @test_vgf2p8affineqb_512(<64 x i8> %src1, <64 x i8> %src2, <64
 ; X86-LABEL: test_vgf2p8affineqb_512:
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vgf2p8affineqb $3, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0xce,0xd9,0x03]
-; X86-NEXT:    vgf2p8affineqb $4, %zmm1, %zmm0, %zmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xc9,0xce,0xe1,0x04]
+; X86-NEXT:    vgf2p8affineqb $3, %zmm1, %zmm0, %zmm4 # encoding: [0x62,0xf3,0xfd,0x48,0xce,0xe1,0x03]
+; X86-NEXT:    vgf2p8affineqb $4, %zmm1, %zmm0, %zmm3 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xc9,0xce,0xd9,0x04]
 ; X86-NEXT:    vgf2p8affineqb $5, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0xce,0xd1,0x05]
-; X86-NEXT:    vpxorq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xef,0xc3]
-; X86-NEXT:    vpxorq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xef,0xc0]
+; X86-NEXT:    vpternlogq $150, %zmm2, %zmm4, %zmm3 # encoding: [0x62,0xf3,0xdd,0x48,0x25,0xda,0x96]
+; X86-NEXT:    vmovdqa64 %zmm3, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_vgf2p8affineqb_512:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
-; X64-NEXT:    vgf2p8affineqb $3, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0xce,0xd9,0x03]
-; X64-NEXT:    vgf2p8affineqb $4, %zmm1, %zmm0, %zmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xc9,0xce,0xe1,0x04]
+; X64-NEXT:    vgf2p8affineqb $3, %zmm1, %zmm0, %zmm4 # encoding: [0x62,0xf3,0xfd,0x48,0xce,0xe1,0x03]
+; X64-NEXT:    vgf2p8affineqb $4, %zmm1, %zmm0, %zmm3 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xc9,0xce,0xd9,0x04]
 ; X64-NEXT:    vgf2p8affineqb $5, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0xce,0xd1,0x05]
-; X64-NEXT:    vpxorq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xef,0xc3]
-; X64-NEXT:    vpxorq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xef,0xc0]
+; X64-NEXT:    vpternlogq $150, %zmm2, %zmm4, %zmm3 # encoding: [0x62,0xf3,0xdd,0x48,0x25,0xda,0x96]
+; X64-NEXT:    vmovdqa64 %zmm3, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i64 %mask to <64 x i1>
   %2 = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %src1, <64 x i8> %src2, i8 3)
diff --git a/llvm/test/CodeGen/X86/avx512-hadd-hsub.ll b/llvm/test/CodeGen/X86/avx512-hadd-hsub.ll
index 283f950b28bec..1fd3e15c3e019 100644
--- a/llvm/test/CodeGen/X86/avx512-hadd-hsub.ll
+++ b/llvm/test/CodeGen/X86/avx512-hadd-hsub.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-;RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl | FileCheck %s --check-prefix=KNL
-;RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s --check-prefix=SKX
+;RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl | FileCheck %s --check-prefixes=CHECK,KNL
+;RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s --check-prefixes=CHECK,SKX
 
 define i32 @hadd_16(<16 x i32> %x225) {
 ; KNL-LABEL: hadd_16:
@@ -107,15 +107,10 @@ define float @fhsub_16(<16 x float> %x225) {
 }
 
 define <16 x i32> @hadd_16_3(<16 x i32> %x225, <16 x i32> %x227) {
-; KNL-LABEL: hadd_16_3:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: hadd_16_3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
-; SKX-NEXT:    retq
+; CHECK-LABEL: hadd_16_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %x226 = shufflevector <16 x i32> %x225, <16 x i32> %x227, <16 x i32> <i32 0, i32 2, i32 16, i32 18
 , i32 4, i32 6, i32 20, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   %x228 = shufflevector <16 x i32> %x225, <16 x i32> %x227, <16 x i32> <i32 1, i32 3, i32 17, i32 19
@@ -126,15 +121,10 @@ define <16 x i32> @hadd_16_3(<16 x i32> %x225, <16 x i32> %x227) {
 }
 
 define <16 x float> @fhadd_16_3(<16 x float> %x225, <16 x float> %x227) {
-; KNL-LABEL: fhadd_16_3:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vhaddps %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: fhadd_16_3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vhaddps %ymm1, %ymm0, %ymm0
-; SKX-NEXT:    retq
+; CHECK-LABEL: fhadd_16_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vhaddps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %x226 = shufflevector <16 x float> %x225, <16 x float> %x227, <16 x i32> <i32 0, i32 2, i32 16, i32 18
 , i32 4, i32 6, i32 20, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   %x228 = shufflevector <16 x float> %x225, <16 x float> %x227, <16 x i32> <i32 1, i32 3, i32 17, i32 19
@@ -144,15 +134,10 @@ define <16 x float> @fhadd_16_3(<16 x float> %x225, <16 x float> %x227) {
 }
 
 define <8 x double> @fhadd_16_4(<8 x double> %x225, <8 x double> %x227) {
-; KNL-LABEL: fhadd_16_4:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: fhadd_16_4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0
-; SKX-NEXT:    retq
+; CHECK-LABEL: fhadd_16_4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %x226 = shufflevector <8 x double> %x225, <8 x double> %x227, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 undef, i32 undef, i32 undef, i32 undef>
   %x228 = shufflevector <8 x double> %x225, <8 x double> %x227, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 undef ,i32 undef, i32 undef, i32 undef>
   %x229 = fadd <8 x double> %x226, %x228
@@ -160,15 +145,10 @@ define <8 x double> @fhadd_16_4(<8 x double> %x225, <8 x double> %x227) {
 }
 
 define <4 x double> @fadd_noundef_low(<8 x double> %x225, <8 x double> %x227) {
-; KNL-LABEL: fadd_noundef_low:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: fadd_noundef_low:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0
-; SKX-NEXT:    retq
+; CHECK-LABEL: fadd_noundef_low:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %x226 = shufflevector <8 x double> %x225, <8 x double> %x227, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
   %x228 = shufflevector <8 x double> %x225, <8 x double> %x227, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5 ,i32 13, i32 7, i32 15>
   %x229 = fadd <8 x double> %x226, %x228
@@ -177,23 +157,14 @@ define <4 x double> @fadd_noundef_low(<8 x double> %x225, <8 x double> %x227) {
 }
 
 define <4 x double> @fadd_noundef_high(<8 x double> %x225, <8 x double> %x227) {
-; KNL-LABEL: fadd_noundef_high:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vunpcklpd {{.*#+}} zmm2 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
-; KNL-NEXT:    vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7]
-; KNL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; KNL-NEXT:    vextractf64x4 $1, %zmm2, %ymm1
-; KNL-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: fadd_noundef_high:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm2 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7]
-; SKX-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; SKX-NEXT:    vextractf64x4 $1, %zmm2, %ymm1
-; SKX-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
-; SKX-NEXT:    retq
+; CHECK-LABEL: fadd_noundef_high:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vunpcklpd {{.*#+}} zmm2 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
+; CHECK-NEXT:    vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7]
+; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; CHECK-NEXT:    vextractf64x4 $1, %zmm2, %ymm1
+; CHECK-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
   %x226 = shufflevector <8 x double> %x225, <8 x double> %x227, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
   %x228 = shufflevector <8 x double> %x225, <8 x double> %x227, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5 ,i32 13, i32 7, i32 15>
   %x229 = fadd <8 x double> %x226, %x228
@@ -203,15 +174,10 @@ define <4 x double> @fadd_noundef_high(<8 x double> %x225, <8 x double> %x227) {
 
 
 define <8 x i32> @hadd_16_3_sv(<16 x i32> %x225, <16 x i32> %x227) {
-; KNL-LABEL: hadd_16_3_sv:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: hadd_16_3_sv:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
-; SKX-NEXT:    retq
+; CHECK-LABEL: hadd_16_3_sv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
   %x226 = shufflevector <16 x i32> %x225, <16 x i32> %x227, <16 x i32> <i32 0, i32 2, i32 16, i32 18
 , i32 4, i32 6, i32 20, i32 22, i32 8, i32 10, i32 24, i32 26, i32 12, i32 14, i32 28, i32 30>
   %x228 = shufflevector <16 x i32> %x225, <16 x i32> %x227, <16 x i32> <i32 1, i32 3, i32 17, i32 19
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
index 295b5271ed0b4..f115f9a6ef382 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
@@ -7740,7 +7740,7 @@ define i64 @test_mm512_reduce_max_epi64(<8 x i64> %__W) {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7,0,1,2,3]
 ; X86-NEXT:    vpmaxsq %zmm0, %zmm1, %zmm0
-; X86-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[2,3,0,1,6,7,4,5]
+; X86-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
 ; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X86-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
@@ -7753,7 +7753,7 @@ define i64 @test_mm512_reduce_max_epi64(<8 x i64> %__W) {
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7,0,1,2,3]
 ; X64-NEXT:    vpmaxsq %zmm0, %zmm1, %zmm0
-; X64-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[2,3,0,1,6,7,4,5]
+; X64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
 ; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X64-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
@@ -7779,7 +7779,7 @@ define i64 @test_mm512_reduce_max_epu64(<8 x i64> %__W) {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7,0,1,2,3]
 ; X86-NEXT:    vpmaxuq %zmm0, %zmm1, %zmm0
-; X86-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[2,3,0,1,6,7,4,5]
+; X86-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
 ; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X86-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
@@ -7792,7 +7792,7 @@ define i64 @test_mm512_reduce_max_epu64(<8 x i64> %__W) {
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7,0,1,2,3]
 ; X64-NEXT:    vpmaxuq %zmm0, %zmm1, %zmm0
-; X64-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[2,3,0,1,6,7,4,5]
+; X64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
 ; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X64-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
@@ -7865,7 +7865,7 @@ define i64 @test_mm512_reduce_min_epi64(<8 x i64> %__W) {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7,0,1,2,3]
 ; X86-NEXT:    vpminsq %zmm0, %zmm1, %zmm0
-; X86-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[2,3,0,1,6,7,4,5]
+; X86-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
 ; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X86-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
@@ -7878,7 +7878,7 @@ define i64 @test_mm512_reduce_min_epi64(<8 x i64> %__W) {
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7,0,1,2,3]
 ; X64-NEXT:    vpminsq %zmm0, %zmm1, %zmm0
-; X64-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[2,3,0,1,6,7,4,5]
+; X64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
 ; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X64-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
@@ -7904,7 +7904,7 @@ define i64 @test_mm512_reduce_min_epu64(<8 x i64> %__W) {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7,0,1,2,3]
 ; X86-NEXT:    vpminuq %zmm0, %zmm1, %zmm0
-; X86-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[2,3,0,1,6,7,4,5]
+; X86-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
 ; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X86-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
@@ -7917,7 +7917,7 @@ define i64 @test_mm512_reduce_min_epu64(<8 x i64> %__W) {
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7,0,1,2,3]
 ; X64-NEXT:    vpminuq %zmm0, %zmm1, %zmm0
-; X64-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[2,3,0,1,6,7,4,5]
+; X64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
 ; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X64-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
@@ -7994,7 +7994,7 @@ define i64 @test_mm512_mask_reduce_max_epi64(i8 zeroext %__M, <8 x i64> %__W) {
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
 ; X86-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[4,5,6,7,0,1,2,3]
 ; X86-NEXT:    vpmaxsq %zmm0, %zmm1, %zmm0
-; X86-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[2,3,0,1,6,7,4,5]
+; X86-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
 ; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X86-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
@@ -8010,7 +8010,7 @@ define i64 @test_mm512_mask_reduce_max_epi64(i8 zeroext %__M, <8 x i64> %__W) {
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
 ; X64-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[4,5,6,7,0,1,2,3]
 ; X64-NEXT:    vpmaxsq %zmm0, %zmm1, %zmm0
-; X64-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[2,3,0,1,6,7,4,5]
+; X64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
 ; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X64-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
@@ -8041,7 +8041,7 @@ define i64 @test_mm512_mask_reduce_max_epu64(i8 zeroext %__M, <8 x i64> %__W) {
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7,0,1,2,3]
 ; X86-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
-; X86-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[2,3,0,1,6,7,4,5]
+; X86-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
 ; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X86-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
@@ -8056,7 +8056,7 @@ define i64 @test_mm512_mask_reduce_max_epu64(i8 zeroext %__M, <8 x i64> %__W) {
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm0 {%k1} {z}
 ; X64-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7,0,1,2,3]
 ; X64-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
-; X64-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[2,3,0,1,6,7,4,5]
+; X64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
 ; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X64-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
@@ -8144,7 +8144,7 @@ define i64 @test_mm512_mask_reduce_min_epi64(i8 zeroext %__M, <8 x i64> %__W) {
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
 ; X86-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[4,5,6,7,0,1,2,3]
 ; X86-NEXT:    vpminsq %zmm0, %zmm1, %zmm0
-; X86-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[2,3,0,1,6,7,4,5]
+; X86-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
 ; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X86-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
@@ -8160,7 +8160,7 @@ define i64 @test_mm512_mask_reduce_min_epi64(i8 zeroext %__M, <8 x i64> %__W) {
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
 ; X64-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[4,5,6,7,0,1,2,3]
 ; X64-NEXT:    vpminsq %zmm0, %zmm1, %zmm0
-; X64-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[2,3,0,1,6,7,4,5]
+; X64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
 ; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X64-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
@@ -8192,7 +8192,7 @@ define i64 @test_mm512_mask_reduce_min_epu64(i8 zeroext %__M, <8 x i64> %__W) {
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
 ; X86-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[4,5,6,7,0,1,2,3]
 ; X86-NEXT:    vpminuq %zmm0, %zmm1, %zmm0
-; X86-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[2,3,0,1,6,7,4,5]
+; X86-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
 ; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X86-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
@@ -8208,7 +8208,7 @@ define i64 @test_mm512_mask_reduce_min_epu64(i8 zeroext %__M, <8 x i64> %__W) {
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
 ; X64-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[4,5,6,7,0,1,2,3]
 ; X64-NEXT:    vpminuq %zmm0, %zmm1, %zmm0
-; X64-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[2,3,0,1,6,7,4,5]
+; X64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
 ; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X64-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
diff --git a/llvm/test/CodeGen/X86/avx512-logic.ll b/llvm/test/CodeGen/X86/avx512-logic.ll
index c7aa636bd8f01..88a3b5aea9bd4 100644
--- a/llvm/test/CodeGen/X86/avx512-logic.ll
+++ b/llvm/test/CodeGen/X86/avx512-logic.ll
@@ -853,3 +853,69 @@ entry:
   ret <16 x float> %4
 }
 
+define <16 x i32> @ternlog_and_andn(<16 x i32> %x, <16 x i32> %y, <16 x i32> %z) {
+; ALL-LABEL: ternlog_and_andn:
+; ALL:       ## %bb.0:
+; ALL-NEXT:    vpternlogd $8, %zmm1, %zmm2, %zmm0
+; ALL-NEXT:    retq
+  %a = xor <16 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %b = and <16 x i32> %y, %a
+  %c = and <16 x i32> %b, %z
+  ret <16 x i32> %c
+}
+
+define <16 x i32> @ternlog_or_andn(<16 x i32> %x, <16 x i32> %y, <16 x i32> %z) {
+; ALL-LABEL: ternlog_or_andn:
+; ALL:       ## %bb.0:
+; ALL-NEXT:    vpternlogd $206, %zmm1, %zmm2, %zmm0
+; ALL-NEXT:    retq
+  %a = xor <16 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %b = and <16 x i32> %y, %a
+  %c = or <16 x i32> %b, %z
+  ret <16 x i32> %c
+}
+
+define <16 x i32> @ternlog_xor_andn(<16 x i32> %x, <16 x i32> %y, <16 x i32> %z) {
+; ALL-LABEL: ternlog_xor_andn:
+; ALL:       ## %bb.0:
+; ALL-NEXT:    vpternlogd $198, %zmm1, %zmm2, %zmm0
+; ALL-NEXT:    retq
+  %a = xor <16 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %b = and <16 x i32> %y, %a
+  %c = xor <16 x i32> %b, %z
+  ret <16 x i32> %c
+}
+
+define <16 x i32> @ternlog_or_and_mask(<16 x i32> %x, <16 x i32> %y) {
+; KNL-LABEL: ternlog_or_and_mask:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
+; KNL-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_or_and_mask:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vandps {{.*}}(%rip), %zmm0, %zmm0
+; SKX-NEXT:    vorps %zmm1, %zmm0, %zmm0
+; SKX-NEXT:    retq
+  %a = and <16 x i32> %x, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %b = or <16 x i32> %a, %y
+  ret <16 x i32> %b
+}
+
+define <8 x i64> @ternlog_xor_and_mask(<8 x i64> %x, <8 x i64> %y) {
+; KNL-LABEL: ternlog_xor_and_mask:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpandd {{.*}}(%rip), %zmm0, %zmm0
+; KNL-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_xor_and_mask:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vandps {{.*}}(%rip), %zmm0, %zmm0
+; SKX-NEXT:    vxorps %zmm1, %zmm0, %zmm0
+; SKX-NEXT:    retq
+  %a = and <8 x i64> %x, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
+  %b = xor <8 x i64> %a, %y
+  ret <8 x i64> %b
+}
diff --git a/llvm/test/CodeGen/X86/avx512-mask-op.ll b/llvm/test/CodeGen/X86/avx512-mask-op.ll
index 4b3c52d16f01b..e67b81581396d 100644
--- a/llvm/test/CodeGen/X86/avx512-mask-op.ll
+++ b/llvm/test/CodeGen/X86/avx512-mask-op.ll
@@ -1867,10 +1867,7 @@ define void @store_i8_i1(i8 %x, i1 *%y) {
 define <32 x i16> @test_build_vec_v32i1(<32 x i16> %x) {
 ; KNL-LABEL: test_build_vec_v32i1:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm1
-; KNL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; KNL-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; KNL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; KNL-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test_build_vec_v32i1:
@@ -1885,10 +1882,7 @@ define <32 x i16> @test_build_vec_v32i1(<32 x i16> %x) {
 ;
 ; AVX512DQ-LABEL: test_build_vec_v32i1:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm1
-; AVX512DQ-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; AVX512DQ-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: test_build_vec_v32i1:
@@ -1902,10 +1896,7 @@ define <32 x i16> @test_build_vec_v32i1(<32 x i16> %x) {
 define <32 x i16> @test_build_vec_v32i1_optsize(<32 x i16> %x) optsize {
 ; KNL-LABEL: test_build_vec_v32i1_optsize:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm1
-; KNL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; KNL-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; KNL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; KNL-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test_build_vec_v32i1_optsize:
@@ -1924,10 +1915,7 @@ define <32 x i16> @test_build_vec_v32i1_optsize(<32 x i16> %x) optsize {
 ;
 ; AVX512DQ-LABEL: test_build_vec_v32i1_optsize:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm1
-; AVX512DQ-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; AVX512DQ-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: test_build_vec_v32i1_optsize:
@@ -1943,10 +1931,7 @@ define <32 x i16> @test_build_vec_v32i1_optsize(<32 x i16> %x) optsize {
 define <32 x i16> @test_build_vec_v32i1_pgso(<32 x i16> %x) !prof !14 {
 ; KNL-LABEL: test_build_vec_v32i1_pgso:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm1
-; KNL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; KNL-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; KNL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; KNL-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test_build_vec_v32i1_pgso:
@@ -1965,10 +1950,7 @@ define <32 x i16> @test_build_vec_v32i1_pgso(<32 x i16> %x) !prof !14 {
 ;
 ; AVX512DQ-LABEL: test_build_vec_v32i1_pgso:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm1
-; AVX512DQ-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; AVX512DQ-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: test_build_vec_v32i1_pgso:
@@ -1984,33 +1966,27 @@ define <32 x i16> @test_build_vec_v32i1_pgso(<32 x i16> %x) !prof !14 {
 define <64 x i8> @test_build_vec_v64i1(<64 x i8> %x) {
 ; KNL-LABEL: test_build_vec_v64i1:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm1
-; KNL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; KNL-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; KNL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; KNL-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test_build_vec_v64i1:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zmm0[2],zero,zero,zero,zmm0[6],zero,zmm0[8],zero,zmm0[10],zero,zmm0[12],zero,zero,zmm0[15],zero,zero,zmm0[18],zero,zmm0[20],zero,zmm0[22],zero,zmm0[24],zero,zero,zmm0[27],zero,zero,zmm0[30],zero,zmm0[32],zero,zmm0[34],zero,zero,zero,zmm0[38],zero,zmm0[40],zero,zero,zmm0[43,44],zero,zmm0[46],zero,zmm0[48],zero,zmm0[50],zero,zero,zero,zmm0[54],zero,zmm0[56],zero,zero,zmm0[59,60],zero,zmm0[62],zero
+; SKX-NEXT:    vandps {{.*}}(%rip), %zmm0, %zmm0
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_build_vec_v64i1:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zmm0[2],zero,zero,zero,zmm0[6],zero,zmm0[8],zero,zmm0[10],zero,zmm0[12],zero,zero,zmm0[15],zero,zero,zmm0[18],zero,zmm0[20],zero,zmm0[22],zero,zmm0[24],zero,zero,zmm0[27],zero,zero,zmm0[30],zero,zmm0[32],zero,zmm0[34],zero,zero,zero,zmm0[38],zero,zmm0[40],zero,zero,zmm0[43,44],zero,zmm0[46],zero,zmm0[48],zero,zmm0[50],zero,zero,zero,zmm0[54],zero,zmm0[56],zero,zero,zmm0[59,60],zero,zmm0[62],zero
+; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: test_build_vec_v64i1:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm1
-; AVX512DQ-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; AVX512DQ-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-NEXT:    vandps {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: test_build_vec_v64i1:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zmm0[2],zero,zero,zero,zmm0[6],zero,zmm0[8],zero,zmm0[10],zero,zmm0[12],zero,zero,zmm0[15],zero,zero,zmm0[18],zero,zmm0[20],zero,zmm0[22],zero,zmm0[24],zero,zero,zmm0[27],zero,zero,zmm0[30],zero,zmm0[32],zero,zmm0[34],zero,zero,zero,zmm0[38],zero,zmm0[40],zero,zero,zmm0[43,44],zero,zmm0[46],zero,zmm0[48],zero,zmm0[50],zero,zero,zero,zmm0[54],zero,zmm0[56],zero,zero,zmm0[59,60],zero,zmm0[62],zero
+; X86-NEXT:    vandps LCPI43_0, %zmm0, %zmm0
 ; X86-NEXT:    retl
   %ret = select <64 x i1> <i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false>, <64 x i8> %x, <64 x i8> zeroinitializer
   ret <64 x i8> %ret
@@ -4864,12 +4840,11 @@ define void @ktest_6(<32 x i16> %w, <32 x i16> %x, <32 x i16> %y, <32 x i16> %z)
 ; KNL-NEXT:    vpcmpeqw %ymm5, %ymm2, %ymm2
 ; KNL-NEXT:    vpcmpeqw %ymm5, %ymm3, %ymm3
 ; KNL-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; KNL-NEXT:    vporq %zmm2, %zmm1, %zmm1
-; KNL-NEXT:    vpandq %zmm1, %zmm0, %zmm0
-; KNL-NEXT:    vpmovsxwd %ymm0, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; KNL-NEXT:    vpternlogq $200, %zmm1, %zmm0, %zmm2
+; KNL-NEXT:    vpmovsxwd %ymm2, %zmm0
+; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
+; KNL-NEXT:    vextracti64x4 $1, %zmm2, %ymm0
 ; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; KNL-NEXT:    kmovw %k0, %ecx
@@ -4952,12 +4927,11 @@ define void @ktest_6(<32 x i16> %w, <32 x i16> %x, <32 x i16> %y, <32 x i16> %z)
 ; AVX512DQ-NEXT:    vpcmpeqw %ymm5, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpcmpeqw %ymm5, %ymm3, %ymm3
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512DQ-NEXT:    vporq %zmm2, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vpandq %zmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm1
-; AVX512DQ-NEXT:    vpmovd2m %zmm1, %k0
+; AVX512DQ-NEXT:    vpternlogq $200, %zmm1, %zmm0, %zmm2
+; AVX512DQ-NEXT:    vpmovsxwd %ymm2, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
 ; AVX512DQ-NEXT:    kmovw %k0, %eax
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm2, %ymm0
 ; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
 ; AVX512DQ-NEXT:    kmovw %k0, %ecx
@@ -5037,10 +5011,9 @@ define void @ktest_7(<64 x i8> %w, <64 x i8> %x, <64 x i8> %y, <64 x i8> %z) {
 ; KNL-NEXT:    vpcmpeqb %ymm5, %ymm2, %ymm2
 ; KNL-NEXT:    vpcmpeqb %ymm5, %ymm3, %ymm3
 ; KNL-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; KNL-NEXT:    vporq %zmm2, %zmm1, %zmm1
-; KNL-NEXT:    vpandq %zmm1, %zmm0, %zmm0
-; KNL-NEXT:    vpmovmskb %ymm0, %eax
-; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
+; KNL-NEXT:    vpternlogq $200, %zmm1, %zmm0, %zmm2
+; KNL-NEXT:    vpmovmskb %ymm2, %eax
+; KNL-NEXT:    vextracti64x4 $1, %zmm2, %ymm0
 ; KNL-NEXT:    vpmovmskb %ymm0, %ecx
 ; KNL-NEXT:    shlq $32, %rcx
 ; KNL-NEXT:    orq %rax, %rcx
@@ -5121,10 +5094,9 @@ define void @ktest_7(<64 x i8> %w, <64 x i8> %x, <64 x i8> %y, <64 x i8> %z) {
 ; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm3, %ymm3
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512DQ-NEXT:    vporq %zmm2, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vpandq %zmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovmskb %ymm0, %eax
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
+; AVX512DQ-NEXT:    vpternlogq $200, %zmm1, %zmm0, %zmm2
+; AVX512DQ-NEXT:    vpmovmskb %ymm2, %eax
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm2, %ymm0
 ; AVX512DQ-NEXT:    vpmovmskb %ymm0, %ecx
 ; AVX512DQ-NEXT:    shlq $32, %rcx
 ; AVX512DQ-NEXT:    orq %rax, %rcx
diff --git a/llvm/test/CodeGen/X86/avx512-movmsk.ll b/llvm/test/CodeGen/X86/avx512-movmsk.ll
new file mode 100644
index 0000000000000..1f7fc15fdee6a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512-movmsk.ll
@@ -0,0 +1,167 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx512vl | FileCheck %s --check-prefixes=AVX512VL
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx512vl,avx512dq,avx512bw | FileCheck %s --check-prefixes=AVX512VLDQBW
+
+; This test makes sure we don't use movmsk instructions when masked compares
+; would be better. The use of the getmant intrinsic introduces a convertion
+; scalar to vXi1 late after movmsk has been formed. Requiring it to be reversed.
+
+declare <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double>, i32, <2 x double>, i8)
+declare <4 x double> @llvm.x86.avx512.mask.getmant.pd.256(<4 x double>, i32, <4 x double>, i8)
+declare <8 x double> @llvm.x86.avx512.mask.getmant.pd.512(<8 x double>, i32, <8 x double>, i8, i32)
+declare <16 x float> @llvm.x86.avx512.mask.getmant.ps.512(<16 x float>, i32, <16 x float>, i16, i32)
+
+define <2 x double> @movmsk2(<2 x double> %x0, <2 x double> %x2, <2 x i64> %mask) {
+; AVX512VL-LABEL: movmsk2:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512VL-NEXT:    vpcmpgtq %xmm2, %xmm3, %k1
+; AVX512VL-NEXT:    vgetmantpd $11, %xmm0, %xmm1 {%k1}
+; AVX512VL-NEXT:    vmovapd %xmm1, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512VLDQBW-LABEL: movmsk2:
+; AVX512VLDQBW:       ## %bb.0:
+; AVX512VLDQBW-NEXT:    vpmovq2m %xmm2, %k1
+; AVX512VLDQBW-NEXT:    vgetmantpd $11, %xmm0, %xmm1 {%k1}
+; AVX512VLDQBW-NEXT:    vmovapd %xmm1, %xmm0
+; AVX512VLDQBW-NEXT:    retq
+  %a = icmp slt <2 x i64> %mask, zeroinitializer
+  %b = bitcast <2 x i1> %a to i2
+  %c = zext i2 %b to i8
+  %res = call <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double> %x0, i32 11, <2 x double> %x2, i8 %c)
+  ret <2 x double> %res
+}
+
+define <4 x double> @movmsk4(<4 x double> %x0, <4 x double> %x2, <4 x i32> %mask) {
+; AVX512VL-LABEL: movmsk4:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512VL-NEXT:    vpcmpgtd %xmm2, %xmm3, %k1
+; AVX512VL-NEXT:    vgetmantpd $11, %ymm0, %ymm1 {%k1}
+; AVX512VL-NEXT:    vmovapd %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512VLDQBW-LABEL: movmsk4:
+; AVX512VLDQBW:       ## %bb.0:
+; AVX512VLDQBW-NEXT:    vpmovd2m %xmm2, %k1
+; AVX512VLDQBW-NEXT:    vgetmantpd $11, %ymm0, %ymm1 {%k1}
+; AVX512VLDQBW-NEXT:    vmovapd %ymm1, %ymm0
+; AVX512VLDQBW-NEXT:    retq
+  %a = icmp slt <4 x i32> %mask, zeroinitializer
+  %b = bitcast <4 x i1> %a to i4
+  %c = zext i4 %b to i8
+  %res = call <4 x double> @llvm.x86.avx512.mask.getmant.pd.256(<4 x double> %x0, i32 11, <4 x double> %x2, i8 %c)
+  ret <4 x double> %res
+}
+
+define <8 x double> @movmsk8(<8 x double> %x0, <8 x double> %x2, <8 x i32> %mask) {
+; AVX512VL-LABEL: movmsk8:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512VL-NEXT:    vpcmpgtd %ymm2, %ymm3, %k1
+; AVX512VL-NEXT:    vgetmantpd $11, %zmm0, %zmm1 {%k1}
+; AVX512VL-NEXT:    vmovapd %zmm1, %zmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512VLDQBW-LABEL: movmsk8:
+; AVX512VLDQBW:       ## %bb.0:
+; AVX512VLDQBW-NEXT:    vpmovd2m %ymm2, %k1
+; AVX512VLDQBW-NEXT:    vgetmantpd $11, %zmm0, %zmm1 {%k1}
+; AVX512VLDQBW-NEXT:    vmovapd %zmm1, %zmm0
+; AVX512VLDQBW-NEXT:    retq
+  %a = icmp slt <8 x i32> %mask, zeroinitializer
+  %b = bitcast <8 x i1> %a to i8
+  %res = call <8 x double> @llvm.x86.avx512.mask.getmant.pd.512(<8 x double> %x0, i32 11, <8 x double> %x2, i8 %b, i32 4)
+  ret <8 x double> %res
+}
+
+define <16 x float> @movmsk16(<16 x float> %x0, <16 x float> %x2, <16 x i8> %mask) {
+; AVX512VL-LABEL: movmsk16:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vpmovmskb %xmm2, %eax
+; AVX512VL-NEXT:    kmovw %eax, %k1
+; AVX512VL-NEXT:    vgetmantps $11, %zmm0, %zmm1 {%k1}
+; AVX512VL-NEXT:    vmovaps %zmm1, %zmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512VLDQBW-LABEL: movmsk16:
+; AVX512VLDQBW:       ## %bb.0:
+; AVX512VLDQBW-NEXT:    vpmovb2m %xmm2, %k1
+; AVX512VLDQBW-NEXT:    vgetmantps $11, %zmm0, %zmm1 {%k1}
+; AVX512VLDQBW-NEXT:    vmovaps %zmm1, %zmm0
+; AVX512VLDQBW-NEXT:    retq
+  %a = icmp slt <16 x i8> %mask, zeroinitializer
+  %b = bitcast <16 x i1> %a to i16
+  %res = call <16 x float> @llvm.x86.avx512.mask.getmant.ps.512(<16 x float> %x0, i32 11, <16 x float> %x2, i16 %b, i32 4)
+  ret <16 x float> %res
+}
+
+; Similar to above but with fp types bitcasted to int for the slt.
+define <2 x double> @movmsk2_fp(<2 x double> %x0, <2 x double> %x2, <2 x double> %mask) {
+; AVX512VL-LABEL: movmsk2_fp:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512VL-NEXT:    vpcmpgtq %xmm2, %xmm3, %k1
+; AVX512VL-NEXT:    vgetmantpd $11, %xmm0, %xmm1 {%k1}
+; AVX512VL-NEXT:    vmovapd %xmm1, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512VLDQBW-LABEL: movmsk2_fp:
+; AVX512VLDQBW:       ## %bb.0:
+; AVX512VLDQBW-NEXT:    vpmovq2m %xmm2, %k1
+; AVX512VLDQBW-NEXT:    vgetmantpd $11, %xmm0, %xmm1 {%k1}
+; AVX512VLDQBW-NEXT:    vmovapd %xmm1, %xmm0
+; AVX512VLDQBW-NEXT:    retq
+  %q = bitcast <2 x double> %mask to <2 x i64>
+  %a = icmp slt <2 x i64> %q, zeroinitializer
+  %b = bitcast <2 x i1> %a to i2
+  %c = zext i2 %b to i8
+  %res = call <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double> %x0, i32 11, <2 x double> %x2, i8 %c)
+  ret <2 x double> %res
+}
+
+define <4 x double> @movmsk4_fp(<4 x double> %x0, <4 x double> %x2, <4 x float> %mask) {
+; AVX512VL-LABEL: movmsk4_fp:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512VL-NEXT:    vpcmpgtd %xmm2, %xmm3, %k1
+; AVX512VL-NEXT:    vgetmantpd $11, %ymm0, %ymm1 {%k1}
+; AVX512VL-NEXT:    vmovapd %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512VLDQBW-LABEL: movmsk4_fp:
+; AVX512VLDQBW:       ## %bb.0:
+; AVX512VLDQBW-NEXT:    vpmovd2m %xmm2, %k1
+; AVX512VLDQBW-NEXT:    vgetmantpd $11, %ymm0, %ymm1 {%k1}
+; AVX512VLDQBW-NEXT:    vmovapd %ymm1, %ymm0
+; AVX512VLDQBW-NEXT:    retq
+  %q = bitcast <4 x float> %mask to <4 x i32>
+  %a = icmp slt <4 x i32> %q, zeroinitializer
+  %b = bitcast <4 x i1> %a to i4
+  %c = zext i4 %b to i8
+  %res = call <4 x double> @llvm.x86.avx512.mask.getmant.pd.256(<4 x double> %x0, i32 11, <4 x double> %x2, i8 %c)
+  ret <4 x double> %res
+}
+
+define <8 x double> @movmsk8_fp(<8 x double> %x0, <8 x double> %x2, <8 x float> %mask) {
+; AVX512VL-LABEL: movmsk8_fp:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512VL-NEXT:    vpcmpgtd %ymm2, %ymm3, %k1
+; AVX512VL-NEXT:    vgetmantpd $11, %zmm0, %zmm1 {%k1}
+; AVX512VL-NEXT:    vmovapd %zmm1, %zmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512VLDQBW-LABEL: movmsk8_fp:
+; AVX512VLDQBW:       ## %bb.0:
+; AVX512VLDQBW-NEXT:    vpmovd2m %ymm2, %k1
+; AVX512VLDQBW-NEXT:    vgetmantpd $11, %zmm0, %zmm1 {%k1}
+; AVX512VLDQBW-NEXT:    vmovapd %zmm1, %zmm0
+; AVX512VLDQBW-NEXT:    retq
+  %q = bitcast <8 x float> %mask to <8 x i32>
+  %a = icmp slt <8 x i32> %q, zeroinitializer
+  %b = bitcast <8 x i1> %a to i8
+  %res = call <8 x double> @llvm.x86.avx512.mask.getmant.pd.512(<8 x double> %x0, i32 11, <8 x double> %x2, i8 %b, i32 4)
+  ret <8 x double> %res
+}
diff --git a/llvm/test/CodeGen/X86/avx512vl-logic.ll b/llvm/test/CodeGen/X86/avx512vl-logic.ll
index 52b135c7c293c..26d905ebeae77 100644
--- a/llvm/test/CodeGen/X86/avx512vl-logic.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-logic.ll
@@ -954,3 +954,80 @@ entry:
   %4 = select <4 x i1> %extract.i, <4 x float> %2, <4 x float> zeroinitializer
   ret <4 x float> %4
 }
+
+define <4 x i32> @ternlog_and_andn(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
+; CHECK-LABEL: ternlog_and_andn:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpternlogd $8, %xmm1, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %a = xor <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %b = and <4 x i32> %y, %a
+  %c = and <4 x i32> %b, %z
+  ret <4 x i32> %c
+}
+
+define <4 x i32> @ternlog_or_andn(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
+; CHECK-LABEL: ternlog_or_andn:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpternlogd $206, %xmm1, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %a = xor <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %b = and <4 x i32> %y, %a
+  %c = or <4 x i32> %b, %z
+  ret <4 x i32> %c
+}
+
+define <4 x i32> @ternlog_xor_andn(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
+; CHECK-LABEL: ternlog_xor_andn:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpternlogd $198, %xmm1, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %a = xor <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %b = and <4 x i32> %y, %a
+  %c = xor <4 x i32> %b, %z
+  ret <4 x i32> %c
+}
+
+define <4 x i32> @ternlog_or_and_mask(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: ternlog_or_and_mask:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %a = and <4 x i32> %x, <i32 255, i32 255, i32 255, i32 255>
+  %b = or <4 x i32> %a, %y
+  ret <4 x i32> %b
+}
+
+define <8 x i32> @ternlog_or_and_mask_ymm(<8 x i32> %x, <8 x i32> %y) {
+; CHECK-LABEL: ternlog_or_and_mask_ymm:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
+  %a = and <8 x i32> %x, <i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216>
+  %b = or <8 x i32> %a, %y
+  ret <8 x i32> %b
+}
+
+define <2 x i64> @ternlog_xor_and_mask(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: ternlog_xor_and_mask:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT:    vxorps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %a = and <2 x i64> %x, <i64 1099511627775, i64 1099511627775>
+  %b = xor <2 x i64> %a, %y
+  ret <2 x i64> %b
+}
+
+define <4 x i64> @ternlog_xor_and_mask_ymm(<4 x i64> %x, <4 x i64> %y) {
+; CHECK-LABEL: ternlog_xor_and_mask_ymm:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
+  %a = and <4 x i64> %x, <i64 72057594037927935, i64 72057594037927935, i64 72057594037927935, i64 72057594037927935>
+  %b = xor <4 x i64> %a, %y
+  ret <4 x i64> %b
+}
diff --git a/llvm/test/CodeGen/X86/bitcast-and-setcc-512.ll b/llvm/test/CodeGen/X86/bitcast-and-setcc-512.ll
index 2424e1c3a74d8..bf251635ffb0d 100644
--- a/llvm/test/CodeGen/X86/bitcast-and-setcc-512.ll
+++ b/llvm/test/CodeGen/X86/bitcast-and-setcc-512.ll
@@ -251,15 +251,12 @@ define i32 @v32i16(<32 x i16> %a, <32 x i16> %b, <32 x i16> %c, <32 x i16> %d) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpcmpgtw %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    vpcmpgtw %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpacksswb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpcmpgtw %ymm7, %ymm5, %ymm1
 ; AVX2-NEXT:    vpcmpgtw %ymm6, %ymm4, %ymm2
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm2[2,3],ymm1[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpacksswb %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpacksswb %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %ymm0, %eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
index cb2af233ead6b..f3805163b691a 100644
--- a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
+++ b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
@@ -204,9 +204,8 @@ define <4 x i64> @ext_i4_4i64(i4 %a0) {
 ;
 ; AVX1-LABEL: ext_i4_4i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    # kill: def $edi killed $edi def $rdi
-; AVX1-NEXT:    vmovq %rdi, %xmm0
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; AVX1-NEXT:    vmovd %edi, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
@@ -432,9 +431,8 @@ define <8 x i64> @ext_i8_8i64(i8 %a0) {
 ;
 ; AVX1-LABEL: ext_i8_8i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    # kill: def $edi killed $edi def $rdi
-; AVX1-NEXT:    vmovq %rdi, %xmm0
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; AVX1-NEXT:    vmovd %edi, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
diff --git a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
index 0f76c35d9ff99..20135e8ca29b3 100644
--- a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
+++ b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
@@ -260,9 +260,8 @@ define <4 x i64> @ext_i4_4i64(i4 %a0) {
 ;
 ; AVX1-LABEL: ext_i4_4i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    # kill: def $edi killed $edi def $rdi
-; AVX1-NEXT:    vmovq %rdi, %xmm0
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; AVX1-NEXT:    vmovd %edi, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
@@ -555,9 +554,8 @@ define <8 x i64> @ext_i8_8i64(i8 %a0) {
 ;
 ; AVX1-LABEL: ext_i8_8i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    # kill: def $edi killed $edi def $rdi
-; AVX1-NEXT:    vmovq %rdi, %xmm0
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; AVX1-NEXT:    vmovd %edi, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm0
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
diff --git a/llvm/test/CodeGen/X86/bitcast-setcc-512.ll b/llvm/test/CodeGen/X86/bitcast-setcc-512.ll
index efe04317919c1..2a6673651f5b2 100644
--- a/llvm/test/CodeGen/X86/bitcast-setcc-512.ll
+++ b/llvm/test/CodeGen/X86/bitcast-setcc-512.ll
@@ -114,9 +114,9 @@ define i16 @v16i32(<16 x i32> %a, <16 x i32> %b) {
 ; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
@@ -175,9 +175,9 @@ define i16 @v16f32(<16 x float> %a, <16 x float> %b) {
 ; AVX2-NEXT:    vcmpltps %ymm1, %ymm3, %ymm1
 ; AVX2-NEXT:    vcmpltps %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
@@ -573,9 +573,9 @@ define void @bitcast_16i32_store(i16* %p, <16 x i32> %a0) {
 ; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX2-NEXT:    movw %ax, (%rdi)
 ; AVX2-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/bitcast-vector-bool.ll b/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
index fd7594c3d2450..faccb8158d322 100644
--- a/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
+++ b/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE2-SSSE3,SSE2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE2-SSSE3,SSSE3
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX12,AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX12,AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX12,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX12,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX,AVX512
 
 ;
 ; 128-bit vectors
@@ -55,25 +55,15 @@ define i2 @bitcast_v4i32_to_v2i2(<4 x i32> %a0) nounwind {
 ; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-SSSE3-NEXT:    retq
 ;
-; AVX12-LABEL: bitcast_v4i32_to_v2i2:
-; AVX12:       # %bb.0:
-; AVX12-NEXT:    vmovmskps %xmm0, %eax
-; AVX12-NEXT:    movl %eax, %ecx
-; AVX12-NEXT:    shrb $2, %cl
-; AVX12-NEXT:    andb $3, %al
-; AVX12-NEXT:    addb %cl, %al
-; AVX12-NEXT:    # kill: def $al killed $al killed $eax
-; AVX12-NEXT:    retq
-;
-; AVX512-LABEL: bitcast_v4i32_to_v2i2:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovmskps %xmm0, %eax
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    shrb $2, %cl
-; AVX512-NEXT:    andb $3, %al
-; AVX512-NEXT:    addb %cl, %al
-; AVX512-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512-NEXT:    retq
+; AVX-LABEL: bitcast_v4i32_to_v2i2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovmskps %xmm0, %eax
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    shrb $2, %cl
+; AVX-NEXT:    andb $3, %al
+; AVX-NEXT:    addb %cl, %al
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    retq
   %1 = icmp slt <4 x i32> %a0, zeroinitializer
   %2 = bitcast <4 x i1> %1 to <2 x i2>
   %3 = extractelement <2 x i2> %2, i32 0
@@ -133,23 +123,14 @@ define i8 @bitcast_v16i8_to_v2i8(<16 x i8> %a0) nounwind {
 ; SSE2-SSSE3-NEXT:    addb -{{[0-9]+}}(%rsp), %al
 ; SSE2-SSSE3-NEXT:    retq
 ;
-; AVX12-LABEL: bitcast_v16i8_to_v2i8:
-; AVX12:       # %bb.0:
-; AVX12-NEXT:    vpmovmskb %xmm0, %ecx
-; AVX12-NEXT:    movl %ecx, %eax
-; AVX12-NEXT:    shrl $8, %eax
-; AVX12-NEXT:    addb %cl, %al
-; AVX12-NEXT:    # kill: def $al killed $al killed $eax
-; AVX12-NEXT:    retq
-;
-; AVX512-LABEL: bitcast_v16i8_to_v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovmskb %xmm0, %ecx
-; AVX512-NEXT:    movl %ecx, %eax
-; AVX512-NEXT:    shrl $8, %eax
-; AVX512-NEXT:    addb %cl, %al
-; AVX512-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512-NEXT:    retq
+; AVX-LABEL: bitcast_v16i8_to_v2i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovmskb %xmm0, %ecx
+; AVX-NEXT:    movl %ecx, %eax
+; AVX-NEXT:    shrl $8, %eax
+; AVX-NEXT:    addb %cl, %al
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    retq
   %1 = icmp slt <16 x i8> %a0, zeroinitializer
   %2 = bitcast <16 x i1> %1 to <2 x i8>
   %3 = extractelement <2 x i8> %2, i32 0
@@ -174,27 +155,16 @@ define i2 @bitcast_v4i64_to_v2i2(<4 x i64> %a0) nounwind {
 ; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-SSSE3-NEXT:    retq
 ;
-; AVX12-LABEL: bitcast_v4i64_to_v2i2:
-; AVX12:       # %bb.0:
-; AVX12-NEXT:    vmovmskpd %ymm0, %eax
-; AVX12-NEXT:    movl %eax, %ecx
-; AVX12-NEXT:    shrb $2, %cl
-; AVX12-NEXT:    andb $3, %al
-; AVX12-NEXT:    addb %cl, %al
-; AVX12-NEXT:    # kill: def $al killed $al killed $eax
-; AVX12-NEXT:    vzeroupper
-; AVX12-NEXT:    retq
-;
-; AVX512-LABEL: bitcast_v4i64_to_v2i2:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovmskpd %ymm0, %eax
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    shrb $2, %cl
-; AVX512-NEXT:    andb $3, %al
-; AVX512-NEXT:    addb %cl, %al
-; AVX512-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX-LABEL: bitcast_v4i64_to_v2i2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovmskpd %ymm0, %eax
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    shrb $2, %cl
+; AVX-NEXT:    andb $3, %al
+; AVX-NEXT:    addb %cl, %al
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
   %1 = icmp slt <4 x i64> %a0, zeroinitializer
   %2 = bitcast <4 x i1> %1 to <2 x i2>
   %3 = extractelement <2 x i2> %2, i32 0
@@ -216,27 +186,16 @@ define i4 @bitcast_v8i32_to_v2i4(<8 x i32> %a0) nounwind {
 ; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-SSSE3-NEXT:    retq
 ;
-; AVX12-LABEL: bitcast_v8i32_to_v2i4:
-; AVX12:       # %bb.0:
-; AVX12-NEXT:    vmovmskps %ymm0, %eax
-; AVX12-NEXT:    movl %eax, %ecx
-; AVX12-NEXT:    shrb $4, %cl
-; AVX12-NEXT:    andb $15, %al
-; AVX12-NEXT:    addb %cl, %al
-; AVX12-NEXT:    # kill: def $al killed $al killed $eax
-; AVX12-NEXT:    vzeroupper
-; AVX12-NEXT:    retq
-;
-; AVX512-LABEL: bitcast_v8i32_to_v2i4:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovmskps %ymm0, %eax
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    shrb $4, %cl
-; AVX512-NEXT:    andb $15, %al
-; AVX512-NEXT:    addb %cl, %al
-; AVX512-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX-LABEL: bitcast_v8i32_to_v2i4:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovmskps %ymm0, %eax
+; AVX-NEXT:    movl %eax, %ecx
+; AVX-NEXT:    shrb $4, %cl
+; AVX-NEXT:    andb $15, %al
+; AVX-NEXT:    addb %cl, %al
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
   %1 = icmp slt <8 x i32> %a0, zeroinitializer
   %2 = bitcast <8 x i1> %1 to <2 x i4>
   %3 = extractelement <2 x i4> %2, i32 0
@@ -450,9 +409,9 @@ define i8 @bitcast_v16i32_to_v2i8(<16 x i32> %a0) nounwind {
 ; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %xmm0, %ecx
 ; AVX2-NEXT:    movl %ecx, %eax
 ; AVX2-NEXT:    shrl $8, %eax
diff --git a/llvm/test/CodeGen/X86/broadcastm-lowering.ll b/llvm/test/CodeGen/X86/broadcastm-lowering.ll
index f995c77f76ea6..2c84a09fcebf4 100644
--- a/llvm/test/CodeGen/X86/broadcastm-lowering.ll
+++ b/llvm/test/CodeGen/X86/broadcastm-lowering.ll
@@ -10,9 +10,9 @@ define <2 x i64> @test_mm_epi64(<8 x i16> %a, <8 x i16> %b) {
 ; AVX512CD-NEXT:    vpmovsxwq %xmm0, %zmm0
 ; AVX512CD-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; AVX512CD-NEXT:    kmovw %k0, %eax
-; AVX512CD-NEXT:    movzbl %al, %ecx
-; AVX512CD-NEXT:    vmovd %ecx, %xmm0
-; AVX512CD-NEXT:    vpinsrb $8, %eax, %xmm0, %xmm0
+; AVX512CD-NEXT:    movzbl %al, %eax
+; AVX512CD-NEXT:    vmovq %rax, %xmm0
+; AVX512CD-NEXT:    vpbroadcastq %xmm0, %xmm0
 ; AVX512CD-NEXT:    vzeroupper
 ; AVX512CD-NEXT:    retq
 ;
@@ -157,12 +157,8 @@ define <8 x i32> @test_mm256_epi32(<16 x i16> %a, <16 x i16> %b) {
 ; AVX512CD-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; AVX512CD-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512CD-NEXT:    kmovw %k0, %eax
-; AVX512CD-NEXT:    kmovw %k0, %ecx
 ; AVX512CD-NEXT:    vmovd %eax, %xmm0
-; AVX512CD-NEXT:    vpinsrw $2, %ecx, %xmm0, %xmm0
-; AVX512CD-NEXT:    vpinsrw $4, %ecx, %xmm0, %xmm0
-; AVX512CD-NEXT:    vpinsrw $6, %ecx, %xmm0, %xmm0
-; AVX512CD-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512CD-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; AVX512CD-NEXT:    retq
 ;
 ; AVX512VLCDBW-LABEL: test_mm256_epi32:
diff --git a/llvm/test/CodeGen/X86/bug80500.ll b/llvm/test/CodeGen/X86/bug80500.ll
new file mode 100644
index 0000000000000..94893365a32ef
--- /dev/null
+++ b/llvm/test/CodeGen/X86/bug80500.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686 -mcpu=skylake | FileCheck %s
+
+; Fix for a typo introduced by D80500
+
+define i32 @load_fold_udiv1(i32* %p) {
+; CHECK-LABEL: load_fold_udiv1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl $-2004318071, %edx # imm = 0x88888889
+; CHECK-NEXT:    mulxl (%eax), %eax, %eax
+; CHECK-NEXT:    shrl $3, %eax
+; CHECK-NEXT:    retl
+  %v = load i32, i32* %p, align 4
+  %ret = udiv i32 %v, 15
+  ret i32 %ret
+}
diff --git a/llvm/test/CodeGen/X86/buildvec-insertvec.ll b/llvm/test/CodeGen/X86/buildvec-insertvec.ll
index 9fb78491b6085..e428ae8d59194 100644
--- a/llvm/test/CodeGen/X86/buildvec-insertvec.ll
+++ b/llvm/test/CodeGen/X86/buildvec-insertvec.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
 
 define void @foo(<3 x float> %in, <4 x i8>* nocapture %out) nounwind {
 ; SSE2-LABEL: foo:
@@ -26,6 +28,15 @@ define void @foo(<3 x float> %in, <4 x i8>* nocapture %out) nounwind {
 ; SSE41-NEXT:    pinsrb $3, %eax, %xmm0
 ; SSE41-NEXT:    movd %xmm0, (%rdi)
 ; SSE41-NEXT:    retq
+;
+; AVX-LABEL: foo:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    movl $255, %eax
+; AVX-NEXT:    vpinsrb $3, %eax, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, (%rdi)
+; AVX-NEXT:    retq
   %t0 = fptoui <3 x float> %in to <3 x i8>
   %t1 = shufflevector <3 x i8> %t0, <3 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
   %t2 = insertelement <4 x i8> %t1, i8 -1, i32 3
@@ -52,6 +63,11 @@ define <4 x float> @test_negative_zero_1(<4 x float> %A) {
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2],zero
 ; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_negative_zero_1:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2],zero
+; AVX-NEXT:    retq
 entry:
   %0 = extractelement <4 x float> %A, i32 0
   %1 = insertelement <4 x float> undef, float %0, i32 0
@@ -74,6 +90,11 @@ define <2 x double> @test_negative_zero_2(<2 x double> %A) {
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]
 ; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_negative_zero_2:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]
+; AVX-NEXT:    retq
 entry:
   %0 = extractelement <2 x double> %A, i32 0
   %1 = insertelement <2 x double> undef, double %0, i32 0
@@ -95,6 +116,13 @@ define <4 x float> @test_buildvector_v4f32_register(float %f0, float %f1, float
 ; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
 ; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
 ; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_buildvector_v4f32_register:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
+; AVX-NEXT:    retq
   %ins0 = insertelement <4 x float> undef, float %f0, i32 0
   %ins1 = insertelement <4 x float> %ins0, float %f1, i32 1
   %ins2 = insertelement <4 x float> %ins1, float %f2, i32 2
@@ -121,6 +149,14 @@ define <4 x float> @test_buildvector_v4f32_load(float* %p0, float* %p1, float* %
 ; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
 ; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
 ; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_buildvector_v4f32_load:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
+; AVX-NEXT:    retq
   %f0 = load float, float* %p0, align 4
   %f1 = load float, float* %p1, align 4
   %f2 = load float, float* %p2, align 4
@@ -147,6 +183,13 @@ define <4 x float> @test_buildvector_v4f32_partial_load(float %f0, float %f1, fl
 ; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
 ; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
 ; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_buildvector_v4f32_partial_load:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
+; AVX-NEXT:    retq
   %f3 = load float, float* %p3, align 4
   %ins0 = insertelement <4 x float> undef, float %f0, i32 0
   %ins1 = insertelement <4 x float> %ins0, float %f1, i32 1
@@ -174,6 +217,14 @@ define <4 x i32> @test_buildvector_v4i32_register(i32 %a0, i32 %a1, i32 %a2, i32
 ; SSE41-NEXT:    pinsrd $2, %edx, %xmm0
 ; SSE41-NEXT:    pinsrd $3, %ecx, %xmm0
 ; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_buildvector_v4i32_register:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %edi, %xmm0
+; AVX-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %ins0 = insertelement <4 x i32> undef, i32 %a0, i32 0
   %ins1 = insertelement <4 x i32> %ins0, i32 %a1, i32 1
   %ins2 = insertelement <4 x i32> %ins1, i32 %a2, i32 2
@@ -195,6 +246,12 @@ define <4 x i32> @test_buildvector_v4i32_partial(i32 %a0, i32 %a3) {
 ; SSE41-NEXT:    movd %edi, %xmm0
 ; SSE41-NEXT:    pinsrd $3, %esi, %xmm0
 ; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_buildvector_v4i32_partial:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %edi, %xmm0
+; AVX-NEXT:    vpinsrd $3, %esi, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %ins0 = insertelement <4 x i32> undef, i32   %a0, i32 0
   %ins1 = insertelement <4 x i32> %ins0, i32 undef, i32 1
   %ins2 = insertelement <4 x i32> %ins1, i32 undef, i32 2
@@ -203,14 +260,23 @@ define <4 x i32> @test_buildvector_v4i32_partial(i32 %a0, i32 %a3) {
 }
 
 define <4 x i32> @test_buildvector_v4i32_register_zero(i32 %a0, i32 %a2, i32 %a3) {
-; CHECK-LABEL: test_buildvector_v4i32_register_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movd %edx, %xmm0
-; CHECK-NEXT:    movd %esi, %xmm1
-; CHECK-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; CHECK-NEXT:    movd %edi, %xmm0
-; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; CHECK-NEXT:    retq
+; SSE-LABEL: test_buildvector_v4i32_register_zero:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movd %edx, %xmm0
+; SSE-NEXT:    movd %esi, %xmm1
+; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT:    movd %edi, %xmm0
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_buildvector_v4i32_register_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %edx, %xmm0
+; AVX-NEXT:    vmovd %esi, %xmm1
+; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vmovd %edi, %xmm1
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-NEXT:    retq
   %ins0 = insertelement <4 x i32> undef, i32 %a0, i32 0
   %ins1 = insertelement <4 x i32> %ins0, i32   0, i32 1
   %ins2 = insertelement <4 x i32> %ins1, i32 %a2, i32 2
@@ -219,14 +285,23 @@ define <4 x i32> @test_buildvector_v4i32_register_zero(i32 %a0, i32 %a2, i32 %a3
 }
 
 define <4 x i32> @test_buildvector_v4i32_register_zero_2(i32 %a1, i32 %a2, i32 %a3) {
-; CHECK-LABEL: test_buildvector_v4i32_register_zero_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movd %edx, %xmm0
-; CHECK-NEXT:    movd %esi, %xmm1
-; CHECK-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; CHECK-NEXT:    movd %edi, %xmm0
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,0],xmm1[0,1]
-; CHECK-NEXT:    retq
+; SSE-LABEL: test_buildvector_v4i32_register_zero_2:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movd %edx, %xmm0
+; SSE-NEXT:    movd %esi, %xmm1
+; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT:    movd %edi, %xmm0
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,0],xmm1[0,1]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_buildvector_v4i32_register_zero_2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %edx, %xmm0
+; AVX-NEXT:    vmovd %esi, %xmm1
+; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vmovd %edi, %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[1,0],xmm0[0,1]
+; AVX-NEXT:    retq
   %ins0 = insertelement <4 x i32> undef, i32   0, i32 0
   %ins1 = insertelement <4 x i32> %ins0, i32 %a1, i32 1
   %ins2 = insertelement <4 x i32> %ins1, i32 %a2, i32 2
@@ -265,6 +340,18 @@ define <8 x i16> @test_buildvector_v8i16_register(i16 %a0, i16 %a1, i16 %a2, i16
 ; SSE41-NEXT:    pinsrw $6, {{[0-9]+}}(%rsp), %xmm0
 ; SSE41-NEXT:    pinsrw $7, {{[0-9]+}}(%rsp), %xmm0
 ; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_buildvector_v8i16_register:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %edi, %xmm0
+; AVX-NEXT:    vpinsrw $1, %esi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $2, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $3, %ecx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $4, %r8d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $5, %r9d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-NEXT:    retq
   %ins0 = insertelement <8 x i16> undef, i16 %a0, i32 0
   %ins1 = insertelement <8 x i16> %ins0, i16 %a1, i32 1
   %ins2 = insertelement <8 x i16> %ins1, i16 %a2, i32 2
@@ -277,14 +364,23 @@ define <8 x i16> @test_buildvector_v8i16_register(i16 %a0, i16 %a1, i16 %a2, i16
 }
 
 define <8 x i16> @test_buildvector_v8i16_partial(i16 %a1, i16 %a3, i16 %a4, i16 %a5) {
-; CHECK-LABEL: test_buildvector_v8i16_partial:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pxor %xmm0, %xmm0
-; CHECK-NEXT:    pinsrw $1, %edi, %xmm0
-; CHECK-NEXT:    pinsrw $3, %esi, %xmm0
-; CHECK-NEXT:    pinsrw $4, %edx, %xmm0
-; CHECK-NEXT:    pinsrw $5, %ecx, %xmm0
-; CHECK-NEXT:    retq
+; SSE-LABEL: test_buildvector_v8i16_partial:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm0, %xmm0
+; SSE-NEXT:    pinsrw $1, %edi, %xmm0
+; SSE-NEXT:    pinsrw $3, %esi, %xmm0
+; SSE-NEXT:    pinsrw $4, %edx, %xmm0
+; SSE-NEXT:    pinsrw $5, %ecx, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_buildvector_v8i16_partial:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $1, %edi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $3, %esi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $4, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $5, %ecx, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %ins0 = insertelement <8 x i16> undef, i16 undef, i32 0
   %ins1 = insertelement <8 x i16> %ins0, i16   %a1, i32 1
   %ins2 = insertelement <8 x i16> %ins1, i16 undef, i32 2
@@ -297,14 +393,23 @@ define <8 x i16> @test_buildvector_v8i16_partial(i16 %a1, i16 %a3, i16 %a4, i16
 }
 
 define <8 x i16> @test_buildvector_v8i16_register_zero(i16 %a0, i16 %a3, i16 %a4, i16 %a5) {
-; CHECK-LABEL: test_buildvector_v8i16_register_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movzwl %di, %eax
-; CHECK-NEXT:    movd %eax, %xmm0
-; CHECK-NEXT:    pinsrw $3, %esi, %xmm0
-; CHECK-NEXT:    pinsrw $4, %edx, %xmm0
-; CHECK-NEXT:    pinsrw $5, %ecx, %xmm0
-; CHECK-NEXT:    retq
+; SSE-LABEL: test_buildvector_v8i16_register_zero:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movzwl %di, %eax
+; SSE-NEXT:    movd %eax, %xmm0
+; SSE-NEXT:    pinsrw $3, %esi, %xmm0
+; SSE-NEXT:    pinsrw $4, %edx, %xmm0
+; SSE-NEXT:    pinsrw $5, %ecx, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_buildvector_v8i16_register_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    movzwl %di, %eax
+; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    vpinsrw $3, %esi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $4, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $5, %ecx, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %ins0 = insertelement <8 x i16> undef, i16   %a0, i32 0
   %ins1 = insertelement <8 x i16> %ins0, i16     0, i32 1
   %ins2 = insertelement <8 x i16> %ins1, i16     0, i32 2
@@ -317,14 +422,23 @@ define <8 x i16> @test_buildvector_v8i16_register_zero(i16 %a0, i16 %a3, i16 %a4
 }
 
 define <8 x i16> @test_buildvector_v8i16_register_zero_2(i16 %a1, i16 %a3, i16 %a4, i16 %a5) {
-; CHECK-LABEL: test_buildvector_v8i16_register_zero_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pxor %xmm0, %xmm0
-; CHECK-NEXT:    pinsrw $1, %edi, %xmm0
-; CHECK-NEXT:    pinsrw $3, %esi, %xmm0
-; CHECK-NEXT:    pinsrw $4, %edx, %xmm0
-; CHECK-NEXT:    pinsrw $5, %ecx, %xmm0
-; CHECK-NEXT:    retq
+; SSE-LABEL: test_buildvector_v8i16_register_zero_2:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm0, %xmm0
+; SSE-NEXT:    pinsrw $1, %edi, %xmm0
+; SSE-NEXT:    pinsrw $3, %esi, %xmm0
+; SSE-NEXT:    pinsrw $4, %edx, %xmm0
+; SSE-NEXT:    pinsrw $5, %ecx, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_buildvector_v8i16_register_zero_2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $1, %edi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $3, %esi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $4, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $5, %ecx, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %ins0 = insertelement <8 x i16> undef, i16     0, i32 0
   %ins1 = insertelement <8 x i16> %ins0, i16   %a1, i32 1
   %ins2 = insertelement <8 x i16> %ins1, i16     0, i32 2
@@ -391,6 +505,26 @@ define <16 x i8> @test_buildvector_v16i8_register(i8 %a0, i8 %a1, i8 %a2, i8 %a3
 ; SSE41-NEXT:    pinsrb $14, {{[0-9]+}}(%rsp), %xmm0
 ; SSE41-NEXT:    pinsrb $15, {{[0-9]+}}(%rsp), %xmm0
 ; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_buildvector_v16i8_register:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %edi, %xmm0
+; AVX-NEXT:    vpinsrb $1, %esi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $3, %ecx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $4, %r8d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $5, %r9d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-NEXT:    retq
   %ins0  = insertelement <16 x i8> undef,  i8 %a0,  i32 0
   %ins1  = insertelement <16 x i8> %ins0,  i8 %a1,  i32 1
   %ins2  = insertelement <16 x i8> %ins1,  i8 %a2,  i32 2
@@ -434,6 +568,17 @@ define <16 x i8> @test_buildvector_v16i8_partial(i8 %a2, i8 %a6, i8 %a8, i8 %a11
 ; SSE41-NEXT:    pinsrb $12, %r8d, %xmm0
 ; SSE41-NEXT:    pinsrb $15, %r9d, %xmm0
 ; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_buildvector_v16i8_partial:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $2, %edi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $6, %esi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $8, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $11, %ecx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $12, %r8d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $15, %r9d, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %ins0  = insertelement <16 x i8> undef,  i8 undef, i32 0
   %ins1  = insertelement <16 x i8> %ins0,  i8 undef, i32 1
   %ins2  = insertelement <16 x i8> %ins1,  i8   %a2, i32 2
@@ -484,6 +629,18 @@ define <16 x i8> @test_buildvector_v16i8_register_zero(i8 %a0, i8 %a4, i8 %a6, i
 ; SSE41-NEXT:    pinsrb $12, %r9d, %xmm0
 ; SSE41-NEXT:    pinsrb $15, {{[0-9]+}}(%rsp), %xmm0
 ; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_buildvector_v16i8_register_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    movzbl %dil, %eax
+; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    vpinsrb $4, %esi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $6, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $8, %ecx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $11, %r8d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $12, %r9d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-NEXT:    retq
   %ins0  = insertelement <16 x i8> undef,  i8   %a0, i32 0
   %ins1  = insertelement <16 x i8> %ins0,  i8     0, i32 1
   %ins2  = insertelement <16 x i8> %ins1,  i8     0, i32 2
@@ -535,6 +692,18 @@ define <16 x i8> @test_buildvector_v16i8_register_zero_2(i8 %a2, i8 %a3, i8 %a6,
 ; SSE41-NEXT:    pinsrb $12, %r9d, %xmm0
 ; SSE41-NEXT:    pinsrb $15, {{[0-9]+}}(%rsp), %xmm0
 ; SSE41-NEXT:    retq
+;
+; AVX-LABEL: test_buildvector_v16i8_register_zero_2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $2, %edi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $3, %esi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $6, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $8, %ecx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $11, %r8d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $12, %r9d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-NEXT:    retq
   %ins0  = insertelement <16 x i8> undef,  i8     0, i32 0
   %ins1  = insertelement <16 x i8> %ins0,  i8     0, i32 1
   %ins2  = insertelement <16 x i8> %ins1,  i8   %a2, i32 2
@@ -554,6 +723,52 @@ define <16 x i8> @test_buildvector_v16i8_register_zero_2(i8 %a2, i8 %a3, i8 %a6,
   ret <16 x i8> %ins15
 }
 
+; PR46461 - Don't let reduceBuildVecExtToExtBuildVec break splat(zero_extend) patterns,
+; resulting in the BUILD_VECTOR lowering to individual insertions into a zero vector.
+
+define void @PR46461(i16 %x, <16 x i32>* %y) {
+; SSE-LABEL: PR46461:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movzwl %di, %eax
+; SSE-NEXT:    shrl %eax
+; SSE-NEXT:    movd %eax, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE-NEXT:    movdqa %xmm0, 48(%rsi)
+; SSE-NEXT:    movdqa %xmm0, 32(%rsi)
+; SSE-NEXT:    movdqa %xmm0, 16(%rsi)
+; SSE-NEXT:    movdqa %xmm0, (%rsi)
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: PR46461:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    movzwl %di, %eax
+; AVX1-NEXT:    shrl %eax
+; AVX1-NEXT:    vmovd %eax, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    vmovaps %ymm0, 32(%rsi)
+; AVX1-NEXT:    vmovaps %ymm0, (%rsi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: PR46461:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    movzwl %di, %eax
+; AVX2-NEXT:    shrl %eax
+; AVX2-NEXT:    vmovd %eax, %xmm0
+; AVX2-NEXT:    vpbroadcastd %xmm0, %ymm0
+; AVX2-NEXT:    vmovdqa %ymm0, 32(%rsi)
+; AVX2-NEXT:    vmovdqa %ymm0, (%rsi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %z = lshr i16 %x, 1
+  %a = zext i16 %z to i32
+  %b = insertelement <16 x i32> undef, i32 %a, i32 0
+  %c = shufflevector <16 x i32> %b, <16 x i32> undef, <16 x i32> zeroinitializer
+  store <16 x i32> %c, <16 x i32>* %y
+  ret void
+}
+
 ; OSS-Fuzz #5688
 ; https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=5688
 define <4 x i32> @ossfuzz5688(i32 %a0) {
@@ -568,3 +783,51 @@ define <4 x i32> @ossfuzz5688(i32 %a0) {
   store i32 %4, i32* undef
   ret <4 x i32> %5
 }
+
+; If we do not define all bytes that are extracted, this is a miscompile.
+
+define i32 @PR46586(i8* %p, <4 x i32> %v) {
+; SSE2-LABEL: PR46586:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movzbl 3(%rdi), %eax
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pinsrw $6, %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    xorl %edx, %edx
+; SSE2-NEXT:    divl %ecx
+; SSE2-NEXT:    movl %edx, %eax
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: PR46586:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; SSE41-NEXT:    extractps $3, %xmm0, %ecx
+; SSE41-NEXT:    pextrd $3, %xmm1, %eax
+; SSE41-NEXT:    xorl %edx, %edx
+; SSE41-NEXT:    divl %ecx
+; SSE41-NEXT:    movl %edx, %eax
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: PR46586:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; AVX-NEXT:    vextractps $3, %xmm0, %ecx
+; AVX-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX-NEXT:    xorl %edx, %edx
+; AVX-NEXT:    divl %ecx
+; AVX-NEXT:    movl %edx, %eax
+; AVX-NEXT:    retq
+  %p0 = getelementptr inbounds i8, i8* %p, i64 0
+  %p3 = getelementptr inbounds i8, i8* %p, i64 3
+  %t25 = load i8, i8* %p0
+  %t28 = load i8, i8* %p3
+  %t29 = insertelement <4 x i8> undef, i8 %t25, i32 0
+  %t32 = insertelement <4 x i8> %t29, i8 %t28, i32 3
+  %t33 = zext <4 x i8> %t32 to <4 x i32>
+  %t34 = urem <4 x i32> %t33, %v
+  %t35 = extractelement <4 x i32> %t34, i32 3
+  ret i32 %t35
+}
diff --git a/llvm/test/CodeGen/X86/byval-callee-cleanup.ll b/llvm/test/CodeGen/X86/byval-callee-cleanup.ll
index 8e059d4334468..73e355edf42b1 100644
--- a/llvm/test/CodeGen/X86/byval-callee-cleanup.ll
+++ b/llvm/test/CodeGen/X86/byval-callee-cleanup.ll
@@ -8,19 +8,19 @@
 
 %struct.Six = type { [6 x i8] }
 
-define x86_stdcallcc void @f(%struct.Six* byval %a) {
+define x86_stdcallcc void @f(%struct.Six* byval(%struct.Six) %a) {
   ret void
 }
 ; CHECK-LABEL: _f@8:
 ; CHECK: retl $8
 
-define x86_thiscallcc void @g(i8* %this, %struct.Six* byval %a) {
+define x86_thiscallcc void @g(i8* %this, %struct.Six* byval(%struct.Six) %a) {
   ret void
 }
 ; CHECK-LABEL: _g:
 ; CHECK: retl $8
 
-define x86_fastcallcc void @h(i32 inreg %x, i32 inreg %y, %struct.Six* byval %a) {
+define x86_fastcallcc void @h(i32 inreg %x, i32 inreg %y, %struct.Six* byval(%struct.Six) %a) {
   ret void
 }
 ; CHECK-LABEL: @h@16:
diff --git a/llvm/test/CodeGen/X86/callbr-asm-blockplacement.ll b/llvm/test/CodeGen/X86/callbr-asm-blockplacement.ll
index d0af12fa20e64..fb658e32d7d2e 100644
--- a/llvm/test/CodeGen/X86/callbr-asm-blockplacement.ll
+++ b/llvm/test/CodeGen/X86/callbr-asm-blockplacement.ll
@@ -48,7 +48,7 @@ define i32 @foo(i32 %arg, i32 (i8*)* %arg3) nounwind {
 ; CHECK-NEXT:    movabsq $-2305847407260205056, %rbx # imm = 0xDFFFFC0000000000
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .LBB0_4: # %bb17
+; CHECK-NEXT:  # %bb.4: # %bb17
 ; CHECK-NEXT:    callq widget
 ; CHECK-NEXT:  .Ltmp0: # Block address taken
 ; CHECK-NEXT:  .LBB0_5: # %bb18
diff --git a/llvm/test/CodeGen/X86/callbr-asm-branch-folding.ll b/llvm/test/CodeGen/X86/callbr-asm-branch-folding.ll
index 4842e282e3a6c..c978ac3ced233 100644
--- a/llvm/test/CodeGen/X86/callbr-asm-branch-folding.ll
+++ b/llvm/test/CodeGen/X86/callbr-asm-branch-folding.ll
@@ -58,7 +58,7 @@ define void @n(i32* %o, i32 %p, i32 %u) nounwind {
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    jmp .LBB0_10
 ; CHECK-NEXT:  .Ltmp0: # Block address taken
-; CHECK-NEXT:  .LBB0_8: # %if.then20.critedge
+; CHECK-NEXT:  # %bb.8: # %if.then20.critedge
 ; CHECK-NEXT:    movl {{.*}}(%rip), %edi
 ; CHECK-NEXT:    movslq %eax, %rcx
 ; CHECK-NEXT:    movl $1, %esi
diff --git a/llvm/test/CodeGen/X86/callbr-asm-instr-scheduling.ll b/llvm/test/CodeGen/X86/callbr-asm-instr-scheduling.ll
new file mode 100644
index 0000000000000..4074991bab8de
--- /dev/null
+++ b/llvm/test/CodeGen/X86/callbr-asm-instr-scheduling.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -verify-machineinstrs -mcpu=znver2 -O2 -frame-pointer=none < %s | FileCheck %s
+
+; Make sure that instructions aren't scheduled after the "callbr". In the
+; example below, we don't want the "shrxq" through "leaq" instructions to be
+; moved after the "callbr".
+
+%struct.cpuinfo_x86 = type { i8, i8, i8, i8, i32, [3 x i32], i8, i8, i8, i8, i32, i32, %union.anon.83, [16 x i8], [64 x i8], i32, i32, i32, i32, i32, i32, i64, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i32, i8, i8 }
+%union.anon.83 = type { i64, [72 x i8] }
+%struct.pgd_t = type { i64 }
+%struct.p4d_t = type { i64 }
+%struct.pud_t = type { i64 }
+
+@boot_cpu_data = external dso_local global %struct.cpuinfo_x86, align 8
+@page_offset_base = external dso_local local_unnamed_addr global i64, align 8
+@pgdir_shift = external dso_local local_unnamed_addr global i32, align 4
+@__force_order = external dso_local global i64, align 8
+@ptrs_per_p4d = external dso_local local_unnamed_addr global i32, align 4
+
+define i64 @early_ioremap_pmd(i64 %addr) {
+; CHECK-LABEL: early_ioremap_pmd:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    movq %cr3, %rax
+; CHECK-EMPTY:
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movabsq $9223372036854771712, %rdx # imm = 0x7FFFFFFFFFFFF000
+; CHECK-NEXT:    andq %rax, %rdx
+; CHECK-NEXT:    movb {{.*}}(%rip), %al
+; CHECK-NEXT:    movq {{.*}}(%rip), %rcx
+; CHECK-NEXT:    shrxq %rax, %rdi, %rax
+; CHECK-NEXT:    addq %rcx, %rdx
+; CHECK-NEXT:    andl $511, %eax # imm = 0x1FF
+; CHECK-NEXT:    leaq (%rdx,%rax,8), %rax
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:  .Ltmp2:
+; CHECK-NEXT:    jmp .Ltmp3
+; CHECK-NEXT:  .Ltmp4:
+; CHECK-NEXT:    .zero (-(((.Ltmp5-.Ltmp6)-(.Ltmp4-.Ltmp2))>0))*((.Ltmp5-.Ltmp6)-(.Ltmp4-.Ltmp2)),144
+; CHECK-NEXT:  .Ltmp7:
+entry:
+  %0 = tail call i64 asm sideeffect "mov %cr3,$0\0A\09", "=r,=*m,~{dirflag},~{fpsr},~{flags}"(i64* nonnull @__force_order)
+  %and.i = and i64 %0, 9223372036854771712
+  %1 = load i64, i64* @page_offset_base, align 8
+  %add = add i64 %and.i, %1
+  %2 = inttoptr i64 %add to %struct.pgd_t*
+  %3 = load i32, i32* @pgdir_shift, align 4
+  %sh_prom = zext i32 %3 to i64
+  %shr = lshr i64 %addr, %sh_prom
+  %and = and i64 %shr, 511
+  %arrayidx = getelementptr %struct.pgd_t, %struct.pgd_t* %2, i64 %and
+  callbr void asm sideeffect "1: jmp 6f\0A2:\0A.skip -(((5f-4f) - (2b-1b)) > 0) * ((5f-4f) - (2b-1b)),0x90\0A3:\0A.section .altinstructions,\22a\22\0A .long 1b - .\0A .long 4f - .\0A .word ${1:P}\0A .byte 3b - 1b\0A .byte 5f - 4f\0A .byte 3b - 2b\0A.previous\0A.section .altinstr_replacement,\22ax\22\0A4: jmp ${5:l}\0A5:\0A.previous\0A.section .altinstructions,\22a\22\0A .long 1b - .\0A .long 0\0A .word ${0:P}\0A .byte 3b - 1b\0A .byte 0\0A .byte 0\0A.previous\0A.section .altinstr_aux,\22ax\22\0A6:\0A testb $2,$3\0A jnz ${4:l}\0A jmp ${5:l}\0A.previous\0A", "i,i,i,*m,X,X,~{dirflag},~{fpsr},~{flags}"(i16 528, i32 117, i32 1, i8* getelementptr inbounds (%struct.cpuinfo_x86, %struct.cpuinfo_x86* @boot_cpu_data, i64 0, i32 12, i32 1, i64 58), i8* blockaddress(@early_ioremap_pmd, %if.end.i), i8* blockaddress(@early_ioremap_pmd, %if.then.i))
+          to label %_static_cpu_has.exit.thread.i [label %if.end.i, label %if.then.i]
+
+_static_cpu_has.exit.thread.i:                    ; preds = %entry
+  br label %if.end.i
+
+if.then.i:                                        ; preds = %entry
+  %4 = bitcast %struct.pgd_t* %arrayidx to %struct.p4d_t*
+  br label %p4d_offset.exit
+
+if.end.i:                                         ; preds = %_static_cpu_has.exit.thread.i, %entry
+  %coerce.dive.i = getelementptr inbounds %struct.pgd_t, %struct.pgd_t* %arrayidx, i64 0, i32 0
+  %5 = load i64, i64* %coerce.dive.i, align 8
+  %6 = inttoptr i64 %5 to %struct.p4d_t*
+  %7 = load i32, i32* @ptrs_per_p4d, align 4
+  %sub.i.i = add i32 %7, 33554431
+  %8 = and i32 %sub.i.i, 33554431
+  %and.i1.i = zext i32 %8 to i64
+  %add.ptr.i = getelementptr %struct.p4d_t, %struct.p4d_t* %6, i64 %and.i1.i
+  br label %p4d_offset.exit
+
+p4d_offset.exit:                                  ; preds = %if.end.i, %if.then.i
+  %retval.0.i = phi %struct.p4d_t* [ %add.ptr.i, %if.end.i ], [ %4, %if.then.i ]
+  %coerce.dive.i12 = getelementptr inbounds %struct.p4d_t, %struct.p4d_t* %retval.0.i, i64 0, i32 0
+  %9 = load i64, i64* %coerce.dive.i12, align 8
+  %and.i.i13 = and i64 %9, 4503599627366400
+  %add.i.i14 = add i64 %and.i.i13, %1
+  %10 = inttoptr i64 %add.i.i14 to %struct.pud_t*
+  %coerce.dive.i16 = getelementptr %struct.pud_t, %struct.pud_t* %10, i64 511, i32 0
+  %11 = load i64, i64* %coerce.dive.i16, align 8
+  %tobool.i.i.i = icmp slt i64 %11, 0
+  %..i.i.i = select i1 %tobool.i.i.i, i64 4503598553628672, i64 4503599627366400
+  ret i64 %..i.i.i
+}
diff --git a/llvm/test/CodeGen/X86/callbr-asm-label-addr.ll b/llvm/test/CodeGen/X86/callbr-asm-label-addr.ll
index dc0e83634658a..70d32359a60c6 100644
--- a/llvm/test/CodeGen/X86/callbr-asm-label-addr.ll
+++ b/llvm/test/CodeGen/X86/callbr-asm-label-addr.ll
@@ -4,10 +4,10 @@ define i32 @test1(i32 %x) {
 ; CHECK-LABEL: test1:
 ; CHECK:         .quad .Ltmp0
 ; CHECK-NEXT:    .quad .Ltmp1
-; CHECK-LABEL: .Ltmp1:
-; CHECK-LABEL: .LBB0_1: # %bar
+; CHECK: .Ltmp1:
+; CHECK-NEXT: # %bb.1: # %bar
 ; CHECK-NEXT:    callq foo
-; CHECK-LABEL: .Ltmp0:
+; CHECK-NEXT: .Ltmp0:
 ; CHECK-NEXT:  # %bb.2: # %baz
 entry:
   callbr void asm sideeffect ".quad ${0:l}\0A\09.quad ${1:l}", "i,X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@test1, %baz), i8* blockaddress(@test1, %bar))
diff --git a/llvm/test/CodeGen/X86/callbr-asm-outputs-pred-succ.ll b/llvm/test/CodeGen/X86/callbr-asm-outputs-pred-succ.ll
index b35aa98e8d83e..56b376ac2f1bb 100644
--- a/llvm/test/CodeGen/X86/callbr-asm-outputs-pred-succ.ll
+++ b/llvm/test/CodeGen/X86/callbr-asm-outputs-pred-succ.ll
@@ -4,28 +4,16 @@
 ; RUN: llc -stop-after=finalize-isel -print-after=finalize-isel -mtriple=i686-- < %s 2>&1 | FileCheck %s
 
 ; The block containting the INLINEASM_BR should have a fallthrough and its
-; indirect targets as its successors. The fallthrough is a block we synthesized
-; in InstrEmitter::EmitMachineNode. Fallthrough should have 100% branch weight,
+; indirect targets as its successors. Fallthrough should have 100% branch weight,
 ; while the indirect targets have 0%.
 ; CHECK: bb.0 (%ir-block.2):
-; CHECK-NEXT: successors: %bb.4(0x00000000), %bb.6(0x80000000); %bb.4(0.00%), %bb.6(100.00%)
+; CHECK-NEXT: successors: %bb.1(0x80000000), %bb.4(0x00000000); %bb.1(100.00%), %bb.4(0.00%)
 
-; The fallthrough block is predaccessed by the block containing INLINEASM_BR,
-; and succeeded by the INLINEASM_BR's original fallthrough block pre-splitting.
-; CHECK: bb.6 (%ir-block.2):
-; CHECK-NEXT: predecessors: %bb.0
-; CHECK-NEXT: successors: %bb.1(0x80000000); %bb.1(100.00%)
-
-; Another block containing a second INLINEASM_BR. Check it has two successors,
-; and the the probability for fallthrough is 100%. Predecessor check irrelevant.
+; The fallthrough is a block containing a second INLINEASM_BR. Check it has two successors,
+; and the the probability for fallthrough is 100%.
 ; CHECK: bb.1 (%ir-block.4):
-; CHECK: successors: %bb.2(0x00000000), %bb.7(0x80000000); %bb.2(0.00%), %bb.7(100.00%)
-
-; Check the synthesized fallthrough block for the second INLINEASM_BR is
-; preceded correctly, and has the original successor pre-splitting.
-; CHECK: bb.7 (%ir-block.4):
-; CHECK-NEXT: predecessors: %bb.1
-; CHECK-NEXT: successors: %bb.3(0x80000000); %bb.3(100.00%)
+; CHECK-NEXT: predecessors: %bb.0
+; CHECK-NEXT: successors: %bb.3(0x80000000), %bb.2(0x00000000); %bb.3(100.00%), %bb.2(0.00%)
 
 ; Check the second INLINEASM_BR target block is preceded by the block with the
 ; second INLINEASM_BR.
diff --git a/llvm/test/CodeGen/X86/callbr-asm-outputs.ll b/llvm/test/CodeGen/X86/callbr-asm-outputs.ll
index 8718363f3fd71..a4447bc15f112 100644
--- a/llvm/test/CodeGen/X86/callbr-asm-outputs.ll
+++ b/llvm/test/CodeGen/X86/callbr-asm-outputs.ll
@@ -12,7 +12,7 @@ define i32 @test1(i32 %x) {
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    jmp .Ltmp0
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .LBB0_1: # %normal
+; CHECK-NEXT:  # %bb.1: # %normal
 ; CHECK-NEXT:    retl
 ; CHECK-NEXT:  .Ltmp0: # Block address taken
 ; CHECK-NEXT:  .LBB0_2: # %abnormal
@@ -43,36 +43,35 @@ define i32 @test2(i32 %out1, i32 %out2) {
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; CHECK-NEXT:    movl $-1, %eax
 ; CHECK-NEXT:    cmpl %edi, %esi
-; CHECK-NEXT:    jge .LBB1_3
+; CHECK-NEXT:    jge .LBB1_2
 ; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    testl %esi, %esi
 ; CHECK-NEXT:    testl %edi, %esi
 ; CHECK-NEXT:    jne .Ltmp1
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .LBB1_2: # %if.then
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    addl %esi, %eax
+; CHECK-NEXT:    jmp .LBB1_3
+; CHECK-NEXT:  .LBB1_2: # %if.else
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    testl %esi, %edi
+; CHECK-NEXT:    testl %esi, %edi
+; CHECK-NEXT:    jne .Ltmp2
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:  .LBB1_3:
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    addl %edi, %eax
 ; CHECK-NEXT:  .Ltmp2: # Block address taken
-; CHECK-NEXT:  .LBB1_6: # %return
+; CHECK-NEXT:  .LBB1_5: # %return
 ; CHECK-NEXT:    popl %esi
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    popl %edi
 ; CHECK-NEXT:    .cfi_def_cfa_offset 4
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:  .LBB1_3: # %if.else
-; CHECK-NEXT:    .cfi_def_cfa_offset 12
-; CHECK-NEXT:    #APP
-; CHECK-NEXT:    testl %esi, %edi
-; CHECK-NEXT:    testl %esi, %edi
-; CHECK-NEXT:    jne .Ltmp2
-; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .LBB1_4: # %if.else
-; CHECK-NEXT:    jmp .LBB1_2
 ; CHECK-NEXT:  .Ltmp1: # Block address taken
-; CHECK-NEXT:  .LBB1_5: # %label_true
+; CHECK-NEXT:  .LBB1_4: # %label_true
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
 ; CHECK-NEXT:    movl $-2, %eax
-; CHECK-NEXT:    jmp .LBB1_6
+; CHECK-NEXT:    jmp .LBB1_5
 entry:
   %cmp = icmp slt i32 %out1, %out2
   br i1 %cmp, label %if.then, label %if.else
@@ -116,7 +115,7 @@ define i32 @test3(i1 %cmp) {
 ; CHECK-NEXT:    .short %esi
 ; CHECK-NEXT:    .short %edi
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .LBB2_2: # %true
+; CHECK-NEXT:  # %bb.2:
 ; CHECK-NEXT:    movl %edi, %eax
 ; CHECK-NEXT:    jmp .LBB2_5
 ; CHECK-NEXT:  .LBB2_3: # %false
@@ -124,7 +123,7 @@ define i32 @test3(i1 %cmp) {
 ; CHECK-NEXT:    .short %eax
 ; CHECK-NEXT:    .short %edx
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .LBB2_4: # %false
+; CHECK-NEXT:  # %bb.4:
 ; CHECK-NEXT:    movl %edx, %eax
 ; CHECK-NEXT:  .LBB2_5: # %asm.fallthrough
 ; CHECK-NEXT:    popl %esi
@@ -166,13 +165,13 @@ define i32 @test4(i32 %out1, i32 %out2) {
 ; CHECK-NEXT:    testl %edx, %ecx
 ; CHECK-NEXT:    jne .Ltmp4
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .LBB3_1: # %asm.fallthrough
+; CHECK-NEXT:  # %bb.1: # %asm.fallthrough
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    testl %ecx, %edx
 ; CHECK-NEXT:    testl %ecx, %edx
 ; CHECK-NEXT:    jne .Ltmp5
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .LBB3_2: # %asm.fallthrough
+; CHECK-NEXT:  # %bb.2: # %asm.fallthrough2
 ; CHECK-NEXT:    addl %edx, %ecx
 ; CHECK-NEXT:    movl %ecx, %eax
 ; CHECK-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/callbr-asm.ll b/llvm/test/CodeGen/X86/callbr-asm.ll
index df7bf3a02be61..1df69b6d8dbdc 100644
--- a/llvm/test/CodeGen/X86/callbr-asm.ll
+++ b/llvm/test/CodeGen/X86/callbr-asm.ll
@@ -14,7 +14,7 @@ define i32 @test1(i32 %a) {
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    jmp .Ltmp0
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .LBB0_1: # %normal
+; CHECK-NEXT:  # %bb.1: # %normal
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    retl
 ; CHECK-NEXT:  .Ltmp0: # Block address taken
@@ -89,7 +89,7 @@ define i32 @test3(i32 %a) {
 ; CHECK-NEXT:    jmp .Ltmp2
 ; CHECK-NEXT:    jmp .Ltmp3
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .LBB2_5: # %normal0
+; CHECK-NEXT:  # %bb.5: # %normal0
 ; CHECK-NEXT:    # in Loop: Header=BB2_4 Depth=4
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    jmp .Ltmp1
@@ -97,7 +97,7 @@ define i32 @test3(i32 %a) {
 ; CHECK-NEXT:    jmp .Ltmp3
 ; CHECK-NEXT:    jmp .Ltmp4
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .LBB2_6: # %normal1
+; CHECK-NEXT:  # %bb.6: # %normal1
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    retl
 entry:
@@ -135,11 +135,11 @@ define void @test4() {
 ; CHECK-LABEL: test4:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    #APP
-; CHECK-NEXT:    ja .Ltmp5{{$}}
+; CHECK-NEXT:    ja .Ltmp5
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .LBB3_1: # %asm.fallthrough
+; CHECK-NEXT:  # %bb.1: # %asm.fallthrough
 ; CHECK-NEXT:    #APP
-; CHECK-NEXT:    ja .Ltmp5{{$}}
+; CHECK-NEXT:    ja .Ltmp5
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:  .Ltmp5: # Block address taken
 ; CHECK-NEXT:  .LBB3_3: # %quux
diff --git a/llvm/test/CodeGen/X86/codegen-prepare-collapse.ll b/llvm/test/CodeGen/X86/codegen-prepare-collapse.ll
index 18e3ef7afbd14..69843af898019 100644
--- a/llvm/test/CodeGen/X86/codegen-prepare-collapse.ll
+++ b/llvm/test/CodeGen/X86/codegen-prepare-collapse.ll
@@ -1,4 +1,4 @@
-; RUN: llc -fast-isel=true -O1 -mtriple=x86_64-unkown-linux-gnu -start-before=codegenprepare -stop-after=codegenprepare -o - < %s | FileCheck %s
+; RUN: llc -fast-isel=true -O1 -mtriple=x86_64-unknown-linux-gnu -start-before=codegenprepare -stop-after=codegenprepare -o - < %s | FileCheck %s
 
 ; CHECK-LABEL: @foo
 define void @foo() {
diff --git a/llvm/test/CodeGen/X86/combine-sdiv.ll b/llvm/test/CodeGen/X86/combine-sdiv.ll
index 618b0a8d26067..a221e42ade42d 100644
--- a/llvm/test/CodeGen/X86/combine-sdiv.ll
+++ b/llvm/test/CodeGen/X86/combine-sdiv.ll
@@ -933,26 +933,27 @@ define <32 x i16> @combine_vec_sdiv_by_pow2b_v32i16(<32 x i16> %x) {
 ;
 ; AVX512F-LABEL: combine_vec_sdiv_by_pow2b_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT:    vpsraw $15, %ymm1, %ymm2
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,4,2,16,8,32,64,2,0,4,2,16,8,32,64,2]
-; AVX512F-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpmulhuw %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddw %ymm2, %ymm1, %ymm2
-; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,2,1,4,3,5,6,1,0,2,1,4,3,5,6,1]
-; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpsravd %zmm4, %zmm2, %zmm2
-; AVX512F-NEXT:    vpmovdw %zmm2, %ymm2
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3,4,5,6,7],ymm1[8],ymm2[9,10,11,12,13,14,15]
-; AVX512F-NEXT:    vpsraw $15, %ymm0, %ymm2
-; AVX512F-NEXT:    vpmulhuw %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddw %ymm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vpsraw $15, %ymm0, %ymm1
+; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,4,2,16,8,32,64,2,0,4,2,16,8,32,64,2]
+; AVX512F-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddw %ymm1, %ymm0, %ymm1
+; AVX512F-NEXT:    vpmovsxwd %ymm1, %zmm1
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [0,2,1,4,3,5,6,1,0,2,1,4,3,5,6,1]
+; AVX512F-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpsravd %zmm3, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX512F-NEXT:    vpsraw $15, %ymm4, %ymm5
+; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm5, %ymm2
+; AVX512F-NEXT:    vpaddw %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm2
-; AVX512F-NEXT:    vpsravd %zmm4, %zmm2, %zmm2
+; AVX512F-NEXT:    vpsravd %zmm3, %zmm2, %zmm2
 ; AVX512F-NEXT:    vpmovdw %zmm2, %ymm2
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3,4,5,6,7],ymm0[8],ymm2[9,10,11,12,13,14,15]
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [0,65535,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,65535]
+; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpternlogq $216, %zmm2, %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: combine_vec_sdiv_by_pow2b_v32i16:
diff --git a/llvm/test/CodeGen/X86/dagcombine-cse.ll b/llvm/test/CodeGen/X86/dagcombine-cse.ll
index a532d87170d80..9ca88d2efa5e4 100644
--- a/llvm/test/CodeGen/X86/dagcombine-cse.ll
+++ b/llvm/test/CodeGen/X86/dagcombine-cse.ll
@@ -1,16 +1,16 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i386-apple-darwin -mattr=+sse2 | FileCheck %s --check-prefix=X32
+; RUN: llc < %s -mtriple=i386-apple-darwin -mattr=+sse2 | FileCheck %s --check-prefix=X86
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+sse2 | FileCheck %s --check-prefix=X64
 
 define i32 @t(i8* %ref_frame_ptr, i32 %ref_frame_stride, i32 %idxX, i32 %idxY) nounwind  {
-; X32-LABEL: t:
-; X32:       ## %bb.0: ## %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    imull {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    addl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl (%eax,%ecx), %eax
-; X32-NEXT:    retl
+; X86-LABEL: t:
+; X86:       ## %bb.0: ## %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%eax,%ecx), %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: t:
 ; X64:       ## %bb.0: ## %entry
@@ -18,8 +18,6 @@ define i32 @t(i8* %ref_frame_ptr, i32 %ref_frame_stride, i32 %idxX, i32 %idxY) n
 ; X64-NEXT:    addl %edx, %esi
 ; X64-NEXT:    movslq %esi, %rax
 ; X64-NEXT:    movl (%rdi,%rax), %eax
-; X64-NEXT:    movq %rax, %xmm0
-; X64-NEXT:    movd %xmm0, %eax
 ; X64-NEXT:    retq
 entry:
 	%tmp7 = mul i32 %idxY, %ref_frame_stride		; <i32> [#uses=2]
diff --git a/llvm/test/CodeGen/X86/extractelement-load.ll b/llvm/test/CodeGen/X86/extractelement-load.ll
index 332fea81adff1..94628c70d989e 100644
--- a/llvm/test/CodeGen/X86/extractelement-load.ll
+++ b/llvm/test/CodeGen/X86/extractelement-load.ll
@@ -1,7 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=X32-SSE2
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=X64,X64-SSSE3
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefixes=X64,X64-AVX
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx  | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX2
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
@@ -265,3 +266,37 @@ entry:
   %cond = select i1 %cmp, float 1.000000e+00, float %vecext
   ret float %cond
 }
+
+; Test for bad extractions from a VBROADCAST_LOAD of the <2 x i16> non-uniform constant bitcast as <4 x i32>.
+define void @subextract_broadcast_load_constant(<2 x i16>* nocapture %0, i16* nocapture %1, i16* nocapture %2) {
+; X32-SSE2-LABEL: subextract_broadcast_load_constant:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-SSE2-NEXT:    movl $-1583308898, (%edx) # imm = 0xA1A09F9E
+; X32-SSE2-NEXT:    movw $-24674, (%ecx) # imm = 0x9F9E
+; X32-SSE2-NEXT:    movw $-24160, (%eax) # imm = 0xA1A0
+; X32-SSE2-NEXT:    retl
+;
+; X64-LABEL: subextract_broadcast_load_constant:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $-1583308898, (%rdi) # imm = 0xA1A09F9E
+; X64-NEXT:    movw $-24674, (%rsi) # imm = 0x9F9E
+; X64-NEXT:    movw $-24160, (%rdx) # imm = 0xA1A0
+; X64-NEXT:    retq
+  %4 = bitcast <2 x i16>* %0 to i8*
+  store i8 -98, i8* %4, align 1
+  %5 = getelementptr inbounds i8, i8* %4, i64 1
+  store i8 -97, i8* %5, align 1
+  %6 = getelementptr inbounds i8, i8* %4, i64 2
+  store i8 -96, i8* %6, align 1
+  %7 = getelementptr inbounds i8, i8* %4, i64 3
+  store i8 -95, i8* %7, align 1
+  %8 = load <2 x i16>, <2 x i16>* %0, align 4
+  %9 = extractelement <2 x i16> %8, i32 0
+  store i16 %9, i16* %1, align 2
+  %10 = extractelement <2 x i16> %8, i32 1
+  store i16 %10, i16* %2, align 2
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/fast-isel-bitcast-crash.ll b/llvm/test/CodeGen/X86/fast-isel-bitcast-crash.ll
new file mode 100644
index 0000000000000..c46b5cca558d3
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fast-isel-bitcast-crash.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -fast-isel -O1 | FileCheck %s
+
+; This used to crash due to the bitcast in the entry block reusing the vreg
+; from its input. This resulted in known bits being calculated on the v2i64
+; type. But the second basic block tried to use them with a v8i16 type. This
+; was fixed by emitting a reg-reg copy for the bitcast so the vreg type will
+; be seen the same in both basic blocks.
+
+; We need the entry block to fall out of fast isel after selecting the bitcast.
+; The shuffle vector guarantees that. The zext gives us a useful known bits
+; value. We also need the second basic block to fall out of fast isel which the
+; intrinsic guarantees.
+
+define <8 x i16> @bitcast_crash(i32 %arg, <8 x i16> %x, i1 %c) {
+; CHECK-LABEL: bitcast_crash:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    movq %rax, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
+; CHECK-NEXT:    testb $1, %sil
+; CHECK-NEXT:    je .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %bb1
+; CHECK-NEXT:    psraw %xmm1, %xmm0
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB0_2: # %bb2
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    retq
+bb:
+  %tmp = zext i32 %arg to i64
+  %tmp1 = insertelement <2 x i64> undef, i64 %tmp, i32 0
+  %tmp2 = shufflevector <2 x i64> %tmp1, <2 x i64> undef, <2 x i32> zeroinitializer
+  %tmp5 = bitcast <2 x i64> %tmp2 to <8 x i16>
+  br i1 %c, label %bb1, label %bb2
+
+bb1:                                              ; preds = %bb8, %bb6
+  %tmp9 = call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %x, <8 x i16> %tmp5)
+  ret <8 x i16> %tmp9
+
+bb2:
+  ret <8 x i16> %tmp5
+}
+
+declare <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16>, <8 x i16>)
diff --git a/llvm/test/CodeGen/X86/fast-isel-gc-intrinsics.ll b/llvm/test/CodeGen/X86/fast-isel-gc-intrinsics.ll
index 9943bed1e65c7..e14099a9cf0b7 100644
--- a/llvm/test/CodeGen/X86/fast-isel-gc-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/fast-isel-gc-intrinsics.ll
@@ -7,8 +7,8 @@ target triple = "x86_64-pc-linux-gnu"
 ; gcrelocate call should not be an LLVM Machine Block by itself.
 define i8 addrspace(1)* @test_gcrelocate(i8 addrspace(1)* %v) gc "statepoint-example" {
 entry:
-  %tok = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0, i8 addrspace(1)* %v)
-  %vnew = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok,  i32 7, i32 7)
+  %tok = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i8 addrspace(1)* %v)]
+  %vnew = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok,  i32 0, i32 0)
   ret i8 addrspace(1)* %vnew
 }
 
@@ -26,9 +26,9 @@ exit:
 ; block.
 define i1 @test_gcresult_gcrelocate(i8 addrspace(1)* %v) gc "statepoint-example" {
 entry:
-  %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i8 addrspace(1)* %v)
+  %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i8 addrspace(1)* %v)]
   %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token)
-  %vnew = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %safepoint_token,  i32 7, i32 7)
+  %vnew = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %safepoint_token,  i32 0, i32 0)
   br label %exit
 exit:
   ret i1 %call1
@@ -42,8 +42,8 @@ entry:
 
 func_call:
  call void @dummy()
- %tok = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0, i8 addrspace(1)* %v)
- %vnew = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok,  i32 7, i32 7)
+ %tok = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i8 addrspace(1)* %v)]
+ %vnew = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok,  i32 0, i32 0)
  ret i8 addrspace(1)* %vnew
 
 exit:
diff --git a/llvm/test/CodeGen/X86/fma_patterns.ll b/llvm/test/CodeGen/X86/fma_patterns.ll
index a11ddf1718301..43b1f4a79aff8 100644
--- a/llvm/test/CodeGen/X86/fma_patterns.ll
+++ b/llvm/test/CodeGen/X86/fma_patterns.ll
@@ -1821,25 +1821,57 @@ define double @fadd_fma_fmul_1(double %a, double %b, double %c, double %d, doubl
   ret double %a2
 }
 
-; Minimum FMF, commute final add operands, change type.
+; Minimum FMF - the 1st fadd is contracted because that combines
+; fmul+fadd as specified by the order of operations; the 2nd fadd
+; requires reassociation to fuse with c*d.
 
-define float @fadd_fma_fmul_2(float %a, float %b, float %c, float %d, float %n0) nounwind {
-; FMA-LABEL: fadd_fma_fmul_2:
+define float @fadd_fma_fmul_fmf(float %a, float %b, float %c, float %d, float %n0) nounwind {
+; FMA-LABEL: fadd_fma_fmul_fmf:
 ; FMA:       # %bb.0:
 ; FMA-NEXT:    vfmadd213ss {{.*#+}} xmm2 = (xmm3 * xmm2) + xmm4
 ; FMA-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
 ; FMA-NEXT:    retq
 ;
-; FMA4-LABEL: fadd_fma_fmul_2:
+; FMA4-LABEL: fadd_fma_fmul_fmf:
 ; FMA4:       # %bb.0:
 ; FMA4-NEXT:    vfmaddss {{.*#+}} xmm2 = (xmm2 * xmm3) + xmm4
 ; FMA4-NEXT:    vfmaddss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm2
 ; FMA4-NEXT:    retq
 ;
-; AVX512-LABEL: fadd_fma_fmul_2:
+; AVX512-LABEL: fadd_fma_fmul_fmf:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vfmadd213ss {{.*#+}} xmm2 = (xmm3 * xmm2) + xmm4
 ; AVX512-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
+; AVX512-NEXT:    retq
+  %m1 = fmul float %a, %b
+  %m2 = fmul float %c, %d
+  %a1 = fadd contract float %m1, %m2
+  %a2 = fadd reassoc float %n0, %a1
+  ret float %a2
+}
+
+; Not minimum FMF.
+
+define float @fadd_fma_fmul_2(float %a, float %b, float %c, float %d, float %n0) nounwind {
+; FMA-LABEL: fadd_fma_fmul_2:
+; FMA:       # %bb.0:
+; FMA-NEXT:    vmulss %xmm3, %xmm2, %xmm2
+; FMA-NEXT:    vfmadd231ss {{.*#+}} xmm2 = (xmm1 * xmm0) + xmm2
+; FMA-NEXT:    vaddss %xmm2, %xmm4, %xmm0
+; FMA-NEXT:    retq
+;
+; FMA4-LABEL: fadd_fma_fmul_2:
+; FMA4:       # %bb.0:
+; FMA4-NEXT:    vmulss %xmm3, %xmm2, %xmm2
+; FMA4-NEXT:    vfmaddss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm2
+; FMA4-NEXT:    vaddss %xmm0, %xmm4, %xmm0
+; FMA4-NEXT:    retq
+;
+; AVX512-LABEL: fadd_fma_fmul_2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmulss %xmm3, %xmm2, %xmm2
+; AVX512-NEXT:    vfmadd231ss {{.*#+}} xmm2 = (xmm1 * xmm0) + xmm2
+; AVX512-NEXT:    vaddss %xmm2, %xmm4, %xmm0
 ; AVX512-NEXT:    retq
   %m1 = fmul float %a, %b
   %m2 = fmul float %c, %d
diff --git a/llvm/test/CodeGen/X86/fp-round.ll b/llvm/test/CodeGen/X86/fp-round.ll
index 4defa872fc0a6..f67ddb65be570 100644
--- a/llvm/test/CodeGen/X86/fp-round.ll
+++ b/llvm/test/CodeGen/X86/fp-round.ll
@@ -455,8 +455,8 @@ define <16 x float> @round_v16f32(<16 x float> %x) {
 ;
 ; AVX512-LABEL: round_v16f32:
 ; AVX512:       ## %bb.0:
-; AVX512-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm1
-; AVX512-NEXT:    vpord {{.*}}(%rip){1to16}, %zmm1, %zmm1
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
+; AVX512-NEXT:    vpternlogd $248, {{.*}}(%rip){1to16}, %zmm0, %zmm1
 ; AVX512-NEXT:    vaddps %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vrndscaleps $11, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
@@ -554,8 +554,8 @@ define <8 x double> @round_v8f64(<8 x double> %x) {
 ;
 ; AVX512-LABEL: round_v8f64:
 ; AVX512:       ## %bb.0:
-; AVX512-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm1
-; AVX512-NEXT:    vporq {{.*}}(%rip){1to8}, %zmm1, %zmm1
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
+; AVX512-NEXT:    vpternlogq $248, {{.*}}(%rip){1to8}, %zmm0, %zmm1
 ; AVX512-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vrndscalepd $11, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-add.ll b/llvm/test/CodeGen/X86/horizontal-reduce-add.ll
new file mode 100644
index 0000000000000..64d8de9aead78
--- /dev/null
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-add.ll
@@ -0,0 +1,408 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse2            | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3           | FileCheck %s --check-prefixes=SSE,SSSE3,SSSE3-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3,fast-hops | FileCheck %s --check-prefixes=SSE,SSSE3,SSSE3-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx             | FileCheck %s --check-prefixes=AVX,AVX1,AVX1-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,fast-hops   | FileCheck %s --check-prefixes=AVX,AVX1,AVX1-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2            | FileCheck %s --check-prefixes=AVX,AVX2
+
+; PR37890 - subvector reduction followed by shuffle reduction
+
+define i32 @PR37890_v4i32(<4 x i32> %a)  {
+; SSE2-LABEL: PR37890_v4i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    retq
+;
+; SSSE3-SLOW-LABEL: PR37890_v4i32:
+; SSSE3-SLOW:       # %bb.0:
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
+; SSSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSSE3-SLOW-NEXT:    retq
+;
+; SSSE3-FAST-LABEL: PR37890_v4i32:
+; SSSE3-FAST:       # %bb.0:
+; SSSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSSE3-FAST-NEXT:    retq
+;
+; AVX1-SLOW-LABEL: PR37890_v4i32:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: PR37890_v4i32:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-LABEL: PR37890_v4i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    retq
+  %hi0 = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %lo0 = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %sum0 = add <2 x i32> %lo0, %hi0
+  %hi1 = shufflevector <2 x i32> %sum0, <2 x i32> undef, <2 x i32> <i32 1, i32 undef>
+  %sum1 = add <2 x i32> %sum0, %hi1
+  %e = extractelement <2 x i32> %sum1, i32 0
+  ret i32 %e
+}
+
+define i16 @PR37890_v8i16(<8 x i16> %a)  {
+; SSE2-LABEL: PR37890_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    paddw %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    paddw %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm1
+; SSE2-NEXT:    paddw %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    retq
+;
+; SSSE3-SLOW-LABEL: PR37890_v8i16:
+; SSSE3-SLOW:       # %bb.0:
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSSE3-SLOW-NEXT:    paddw %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSSE3-SLOW-NEXT:    paddw %xmm1, %xmm0
+; SSSE3-SLOW-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    psrld $16, %xmm1
+; SSSE3-SLOW-NEXT:    paddw %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    movd %xmm1, %eax
+; SSSE3-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSSE3-SLOW-NEXT:    retq
+;
+; SSSE3-FAST-LABEL: PR37890_v8i16:
+; SSSE3-FAST:       # %bb.0:
+; SSSE3-FAST-NEXT:    phaddw %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    phaddw %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    phaddw %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSSE3-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSSE3-FAST-NEXT:    retq
+;
+; AVX1-SLOW-LABEL: PR37890_v8i16:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: PR37890_v8i16:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX1-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-LABEL: PR37890_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT:    retq
+  %hi0 = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %lo0 = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %sum0 = add <4 x i16> %lo0, %hi0
+  %hi1 = shufflevector <4 x i16> %sum0, <4 x i16> undef, <2 x i32> <i32 2, i32 3>
+  %lo1 = shufflevector <4 x i16> %sum0, <4 x i16> undef, <2 x i32> <i32 0, i32 1>
+  %sum1 = add <2 x i16> %lo1, %hi1
+  %hi2 = shufflevector <2 x i16> %sum1, <2 x i16> undef, <2 x i32> <i32 1, i32 undef>
+  %sum2 = add <2 x i16> %sum1, %hi2
+  %e = extractelement <2 x i16> %sum2, i32 0
+  ret i16 %e
+}
+
+define i32 @PR37890_v8i32(<8 x i32> %a)  {
+; SSE2-LABEL: PR37890_v8i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    retq
+;
+; SSSE3-SLOW-LABEL: PR37890_v8i32:
+; SSSE3-SLOW:       # %bb.0:
+; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
+; SSSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSSE3-SLOW-NEXT:    retq
+;
+; SSSE3-FAST-LABEL: PR37890_v8i32:
+; SSSE3-FAST:       # %bb.0:
+; SSSE3-FAST-NEXT:    paddd %xmm1, %xmm0
+; SSSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSSE3-FAST-NEXT:    retq
+;
+; AVX1-SLOW-LABEL: PR37890_v8i32:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT:    vzeroupper
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: PR37890_v8i32:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX1-FAST-NEXT:    vzeroupper
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-LABEL: PR37890_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %hi0 = shufflevector <8 x i32> %a, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %lo0 = shufflevector <8 x i32> %a, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %sum0 = add <4 x i32> %lo0, %hi0
+  %hi1 = shufflevector <4 x i32> %sum0, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %lo1 = shufflevector <4 x i32> %sum0, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %sum1 = add <2 x i32> %lo1, %hi1
+  %hi2 = shufflevector <2 x i32> %sum1, <2 x i32> undef, <2 x i32> <i32 1, i32 undef>
+  %sum2 = add <2 x i32> %sum1, %hi2
+  %e = extractelement <2 x i32> %sum2, i32 0
+  ret i32 %e
+}
+
+define i16 @PR37890_v16i16(<16 x i16> %a)  {
+; SSE2-LABEL: PR37890_v16i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    paddw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    paddw %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    paddw %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm1
+; SSE2-NEXT:    paddw %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT:    retq
+;
+; SSSE3-SLOW-LABEL: PR37890_v16i16:
+; SSSE3-SLOW:       # %bb.0:
+; SSSE3-SLOW-NEXT:    paddw %xmm1, %xmm0
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSSE3-SLOW-NEXT:    paddw %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSSE3-SLOW-NEXT:    paddw %xmm1, %xmm0
+; SSSE3-SLOW-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    psrld $16, %xmm1
+; SSSE3-SLOW-NEXT:    paddw %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    movd %xmm1, %eax
+; SSSE3-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSSE3-SLOW-NEXT:    retq
+;
+; SSSE3-FAST-LABEL: PR37890_v16i16:
+; SSSE3-FAST:       # %bb.0:
+; SSSE3-FAST-NEXT:    paddw %xmm1, %xmm0
+; SSSE3-FAST-NEXT:    phaddw %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    phaddw %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    phaddw %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSSE3-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSSE3-FAST-NEXT:    retq
+;
+; AVX1-SLOW-LABEL: PR37890_v16i16:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX1-SLOW-NEXT:    vzeroupper
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: PR37890_v16i16:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-FAST-NEXT:    vphaddw %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX1-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX1-FAST-NEXT:    vzeroupper
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-LABEL: PR37890_v16i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %hi0 = shufflevector <16 x i16> %a, <16 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %lo0 = shufflevector <16 x i16> %a, <16 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %sum0 = add <8 x i16> %lo0, %hi0
+  %hi1 = shufflevector <8 x i16> %sum0, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %lo1 = shufflevector <8 x i16> %sum0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %sum1 = add <4 x i16> %lo1, %hi1
+  %hi2 = shufflevector <4 x i16> %sum1, <4 x i16> undef, <2 x i32> <i32 2, i32 3>
+  %lo2 = shufflevector <4 x i16> %sum1, <4 x i16> undef, <2 x i32> <i32 0, i32 1>
+  %sum2 = add <2 x i16> %lo2, %hi2
+  %hi3 = shufflevector <2 x i16> %sum2, <2 x i16> undef, <2 x i32> <i32 1, i32 undef>
+  %sum3 = add <2 x i16> %sum2, %hi3
+  %e = extractelement <2 x i16> %sum3, i32 0
+  ret i16 %e
+}
+
+define i32 @PR37890_v16i32(<16 x i32> %a)  {
+; SSE2-LABEL: PR37890_v16i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    paddd %xmm3, %xmm1
+; SSE2-NEXT:    paddd %xmm2, %xmm1
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    retq
+;
+; SSSE3-SLOW-LABEL: PR37890_v16i32:
+; SSSE3-SLOW:       # %bb.0:
+; SSSE3-SLOW-NEXT:    paddd %xmm3, %xmm1
+; SSSE3-SLOW-NEXT:    paddd %xmm2, %xmm1
+; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    movd %xmm1, %eax
+; SSSE3-SLOW-NEXT:    retq
+;
+; SSSE3-FAST-LABEL: PR37890_v16i32:
+; SSSE3-FAST:       # %bb.0:
+; SSSE3-FAST-NEXT:    paddd %xmm3, %xmm1
+; SSSE3-FAST-NEXT:    paddd %xmm2, %xmm1
+; SSSE3-FAST-NEXT:    paddd %xmm0, %xmm1
+; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSSE3-FAST-NEXT:    paddd %xmm1, %xmm0
+; SSSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSSE3-FAST-NEXT:    retq
+;
+; AVX1-SLOW-LABEL: PR37890_v16i32:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT:    vzeroupper
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: PR37890_v16i32:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm2
+; AVX1-FAST-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vphaddd %xmm2, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX1-FAST-NEXT:    vzeroupper
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-LABEL: PR37890_v16i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %hi0 = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %lo0 = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %sum0 = add <8 x i32> %lo0, %hi0
+  %hi1 = shufflevector <8 x i32> %sum0, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %lo1 = shufflevector <8 x i32> %sum0, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %sum1 = add <4 x i32> %lo1, %hi1
+  %hi2 = shufflevector <4 x i32> %sum1, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %lo2 = shufflevector <4 x i32> %sum1, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %sum2 = add <2 x i32> %lo2, %hi2
+  %hi3 = shufflevector <2 x i32> %sum2, <2 x i32> undef, <2 x i32> <i32 1, i32 undef>
+  %sum3 = add <2 x i32> %sum2, %hi3
+  %e = extractelement <2 x i32> %sum3, i32 0
+  ret i32 %e
+}
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-fadd.ll b/llvm/test/CodeGen/X86/horizontal-reduce-fadd.ll
new file mode 100644
index 0000000000000..ff635b94de4df
--- /dev/null
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-fadd.ll
@@ -0,0 +1,352 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse2            | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3           | FileCheck %s --check-prefixes=SSE,SSSE3,SSSE3-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3,fast-hops | FileCheck %s --check-prefixes=SSE,SSSE3,SSSE3-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx             | FileCheck %s --check-prefixes=AVX,AVX1,AVX1-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,fast-hops   | FileCheck %s --check-prefixes=AVX,AVX1,AVX1-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2            | FileCheck %s --check-prefixes=AVX,AVX2
+
+; PR37890 - subvector reduction followed by shuffle reduction
+
+define float @PR37890_v4f32(<4 x float> %a)  {
+; SSE2-LABEL: PR37890_v4f32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps %xmm0, %xmm1
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE2-NEXT:    addps %xmm0, %xmm1
+; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    addss %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-SLOW-LABEL: PR37890_v4f32:
+; SSSE3-SLOW:       # %bb.0:
+; SSSE3-SLOW-NEXT:    movaps %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSSE3-SLOW-NEXT:    addps %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSSE3-SLOW-NEXT:    addss %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    movaps %xmm1, %xmm0
+; SSSE3-SLOW-NEXT:    retq
+;
+; SSSE3-FAST-LABEL: PR37890_v4f32:
+; SSSE3-FAST:       # %bb.0:
+; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    retq
+;
+; AVX1-SLOW-LABEL: PR37890_v4f32:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX1-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: PR37890_v4f32:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-LABEL: PR37890_v4f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX2-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %hi0 = shufflevector <4 x float> %a, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  %lo0 = shufflevector <4 x float> %a, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  %sum0 = fadd fast <2 x float> %lo0, %hi0
+  %hi1 = shufflevector <2 x float> %sum0, <2 x float> undef, <2 x i32> <i32 1, i32 undef>
+  %sum1 = fadd fast <2 x float> %sum0, %hi1
+  %e = extractelement <2 x float> %sum1, i32 0
+  ret float %e
+}
+
+define double @PR37890_v4f64(<4 x double> %a)  {
+; SSE2-LABEL: PR37890_v4f64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    addpd %xmm1, %xmm0
+; SSE2-NEXT:    movapd %xmm0, %xmm1
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE2-NEXT:    addsd %xmm0, %xmm1
+; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-SLOW-LABEL: PR37890_v4f64:
+; SSSE3-SLOW:       # %bb.0:
+; SSSE3-SLOW-NEXT:    addpd %xmm1, %xmm0
+; SSSE3-SLOW-NEXT:    movapd %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSSE3-SLOW-NEXT:    addsd %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    movapd %xmm1, %xmm0
+; SSSE3-SLOW-NEXT:    retq
+;
+; SSSE3-FAST-LABEL: PR37890_v4f64:
+; SSSE3-FAST:       # %bb.0:
+; SSSE3-FAST-NEXT:    addpd %xmm1, %xmm0
+; SSSE3-FAST-NEXT:    haddpd %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    retq
+;
+; AVX1-SLOW-LABEL: PR37890_v4f64:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-SLOW-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX1-SLOW-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vzeroupper
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: PR37890_v4f64:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-FAST-NEXT:    vhaddpd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vzeroupper
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-LABEL: PR37890_v4f64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX2-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %hi0 = shufflevector <4 x double> %a, <4 x double> undef, <2 x i32> <i32 2, i32 3>
+  %lo0 = shufflevector <4 x double> %a, <4 x double> undef, <2 x i32> <i32 0, i32 1>
+  %sum0 = fadd fast <2 x double> %lo0, %hi0
+  %hi1 = shufflevector <2 x double> %sum0, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+  %sum1 = fadd fast <2 x double> %sum0, %hi1
+  %e = extractelement <2 x double> %sum1, i32 0
+  ret double %e
+}
+
+define float @PR37890_v8f32(<8 x float> %a)  {
+; SSE2-LABEL: PR37890_v8f32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    addps %xmm1, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm1
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE2-NEXT:    addps %xmm0, %xmm1
+; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    addss %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-SLOW-LABEL: PR37890_v8f32:
+; SSSE3-SLOW:       # %bb.0:
+; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm0
+; SSSE3-SLOW-NEXT:    movaps %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSSE3-SLOW-NEXT:    addps %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSSE3-SLOW-NEXT:    addss %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    movaps %xmm1, %xmm0
+; SSSE3-SLOW-NEXT:    retq
+;
+; SSSE3-FAST-LABEL: PR37890_v8f32:
+; SSSE3-FAST:       # %bb.0:
+; SSSE3-FAST-NEXT:    addps %xmm1, %xmm0
+; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    retq
+;
+; AVX1-SLOW-LABEL: PR37890_v8f32:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX1-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vzeroupper
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: PR37890_v8f32:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vzeroupper
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-LABEL: PR37890_v8f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX2-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %hi0 = shufflevector <8 x float> %a, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %lo0 = shufflevector <8 x float> %a, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %sum0 = fadd fast <4 x float> %lo0, %hi0
+  %hi1 = shufflevector <4 x float> %sum0, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  %lo1 = shufflevector <4 x float> %sum0, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  %sum1 = fadd fast <2 x float> %lo1, %hi1
+  %hi2 = shufflevector <2 x float> %sum1, <2 x float> undef, <2 x i32> <i32 1, i32 undef>
+  %sum2 = fadd fast <2 x float> %sum1, %hi2
+  %e = extractelement <2 x float> %sum2, i32 0
+  ret float %e
+}
+
+define double @PR37890_v8f64(<8 x double> %a)  {
+; SSE2-LABEL: PR37890_v8f64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    addpd %xmm3, %xmm1
+; SSE2-NEXT:    addpd %xmm2, %xmm1
+; SSE2-NEXT:    addpd %xmm0, %xmm1
+; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; SSE2-NEXT:    addsd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-SLOW-LABEL: PR37890_v8f64:
+; SSSE3-SLOW:       # %bb.0:
+; SSSE3-SLOW-NEXT:    addpd %xmm3, %xmm1
+; SSSE3-SLOW-NEXT:    addpd %xmm2, %xmm1
+; SSSE3-SLOW-NEXT:    addpd %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    movapd %xmm1, %xmm0
+; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; SSSE3-SLOW-NEXT:    addsd %xmm1, %xmm0
+; SSSE3-SLOW-NEXT:    retq
+;
+; SSSE3-FAST-LABEL: PR37890_v8f64:
+; SSSE3-FAST:       # %bb.0:
+; SSSE3-FAST-NEXT:    addpd %xmm3, %xmm1
+; SSSE3-FAST-NEXT:    addpd %xmm2, %xmm1
+; SSSE3-FAST-NEXT:    addpd %xmm0, %xmm1
+; SSSE3-FAST-NEXT:    haddpd %xmm1, %xmm1
+; SSSE3-FAST-NEXT:    movapd %xmm1, %xmm0
+; SSSE3-FAST-NEXT:    retq
+;
+; AVX1-SLOW-LABEL: PR37890_v8f64:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX1-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-SLOW-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX1-SLOW-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vzeroupper
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: PR37890_v8f64:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX1-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-FAST-NEXT:    vhaddpd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vzeroupper
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-LABEL: PR37890_v8f64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX2-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %hi0 = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %lo0 = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %sum0 = fadd fast <4 x double> %lo0, %hi0
+  %hi1 = shufflevector <4 x double> %sum0, <4 x double> undef, <2 x i32> <i32 2, i32 3>
+  %lo1 = shufflevector <4 x double> %sum0, <4 x double> undef, <2 x i32> <i32 0, i32 1>
+  %sum1 = fadd fast <2 x double> %lo1, %hi1
+  %hi2 = shufflevector <2 x double> %sum1, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+  %sum2 = fadd fast <2 x double> %sum1, %hi2
+  %e = extractelement <2 x double> %sum2, i32 0
+  ret double %e
+}
+
+define float @PR37890_v16f32(<16 x float> %a)  {
+; SSE2-LABEL: PR37890_v16f32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    addps %xmm3, %xmm1
+; SSE2-NEXT:    addps %xmm2, %xmm1
+; SSE2-NEXT:    addps %xmm0, %xmm1
+; SSE2-NEXT:    movaps %xmm1, %xmm2
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
+; SSE2-NEXT:    addps %xmm1, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm2[2,3]
+; SSE2-NEXT:    addss %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-SLOW-LABEL: PR37890_v16f32:
+; SSSE3-SLOW:       # %bb.0:
+; SSSE3-SLOW-NEXT:    addps %xmm3, %xmm1
+; SSSE3-SLOW-NEXT:    addps %xmm2, %xmm1
+; SSSE3-SLOW-NEXT:    addps %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    movaps %xmm1, %xmm2
+; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
+; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm2
+; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSSE3-SLOW-NEXT:    addss %xmm2, %xmm0
+; SSSE3-SLOW-NEXT:    retq
+;
+; SSSE3-FAST-LABEL: PR37890_v16f32:
+; SSSE3-FAST:       # %bb.0:
+; SSSE3-FAST-NEXT:    addps %xmm3, %xmm1
+; SSSE3-FAST-NEXT:    addps %xmm2, %xmm1
+; SSSE3-FAST-NEXT:    addps %xmm0, %xmm1
+; SSSE3-FAST-NEXT:    movaps %xmm1, %xmm0
+; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; SSSE3-FAST-NEXT:    addps %xmm1, %xmm0
+; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    retq
+;
+; AVX1-SLOW-LABEL: PR37890_v16f32:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX1-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX1-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vzeroupper
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: PR37890_v16f32:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX1-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vzeroupper
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-LABEL: PR37890_v16f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX2-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+  %hi0 = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %lo0 = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %sum0 = fadd fast <8 x float> %lo0, %hi0
+  %hi1 = shufflevector <8 x float> %sum0, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %lo1 = shufflevector <8 x float> %sum0, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %sum1 = fadd fast <4 x float> %lo1, %hi1
+  %hi2 = shufflevector <4 x float> %sum1, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  %lo2 = shufflevector <4 x float> %sum1, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  %sum2 = fadd fast <2 x float> %lo2, %hi2
+  %hi3 = shufflevector <2 x float> %sum2, <2 x float> undef, <2 x i32> <i32 1, i32 undef>
+  %sum3 = fadd fast <2 x float> %sum2, %hi3
+  %e = extractelement <2 x float> %sum3, i32 0
+  ret float %e
+}
diff --git a/llvm/test/CodeGen/X86/ipra-reg-usage.ll b/llvm/test/CodeGen/X86/ipra-reg-usage.ll
index 7d05600404903..e9039da29aa25 100644
--- a/llvm/test/CodeGen/X86/ipra-reg-usage.ll
+++ b/llvm/test/CodeGen/X86/ipra-reg-usage.ll
@@ -3,7 +3,7 @@
 target triple = "x86_64-unknown-unknown"
 declare void @bar1()
 define preserve_allcc void @foo()#0 {
-; CHECK: foo Clobbered Registers: $cs $df $ds $eflags $eip $eiz $es $fpcw $fpsw $fs $gs $hip $ip $mxcsr $rip $riz $ss $ssp $bnd0 $bnd1 $bnd2 $bnd3 $cr0 $cr1 $cr2 $cr3 $cr4 $cr5 $cr6 $cr7 $cr8 $cr9 $cr10 $cr11 $cr12 $cr13 $cr14 $cr15 $dr0 $dr1 $dr2 $dr3 $dr4 $dr5 $dr6 $dr7 $dr8 $dr9 $dr10 $dr11 $dr12 $dr13 $dr14 $dr15 $fp0 $fp1 $fp2 $fp3 $fp4 $fp5 $fp6 $fp7 $k0 $k1 $k2 $k3 $k4 $k5 $k6 $k7 $mm0 $mm1 $mm2 $mm3 $mm4 $mm5 $mm6 $mm7 $r11 $st0 $st1 $st2 $st3 $st4 $st5 $st6 $st7 $xmm16 $xmm17 $xmm18 $xmm19 $xmm20 $xmm21 $xmm22 $xmm23 $xmm24 $xmm25 $xmm26 $xmm27 $xmm28 $xmm29 $xmm30 $xmm31 $ymm0 $ymm1 $ymm2 $ymm3 $ymm4 $ymm5 $ymm6 $ymm7 $ymm8 $ymm9 $ymm10 $ymm11 $ymm12 $ymm13 $ymm14 $ymm15 $ymm16 $ymm17 $ymm18 $ymm19 $ymm20 $ymm21 $ymm22 $ymm23 $ymm24 $ymm25 $ymm26 $ymm27 $ymm28 $ymm29 $ymm30 $ymm31 $zmm0 $zmm1 $zmm2 $zmm3 $zmm4 $zmm5 $zmm6 $zmm7 $zmm8 $zmm9 $zmm10 $zmm11 $zmm12 $zmm13 $zmm14 $zmm15 $zmm16 $zmm17 $zmm18 $zmm19 $zmm20 $zmm21 $zmm22 $zmm23 $zmm24 $zmm25 $zmm26 $zmm27 $zmm28 $zmm29 $zmm30 $zmm31 $r11b $r11bh $r11d $r11w $r11wh
+; CHECK: foo Clobbered Registers: $cs $df $ds $eflags $eip $eiz $es $fpcw $fpsw $fs $gs $hip $ip $mxcsr $rip $riz $ss $ssp $bnd0 $bnd1 $bnd2 $bnd3 $cr0 $cr1 $cr2 $cr3 $cr4 $cr5 $cr6 $cr7 $cr8 $cr9 $cr10 $cr11 $cr12 $cr13 $cr14 $cr15 $dr0 $dr1 $dr2 $dr3 $dr4 $dr5 $dr6 $dr7 $dr8 $dr9 $dr10 $dr11 $dr12 $dr13 $dr14 $dr15 $fp0 $fp1 $fp2 $fp3 $fp4 $fp5 $fp6 $fp7 $k0 $k1 $k2 $k3 $k4 $k5 $k6 $k7 $mm0 $mm1 $mm2 $mm3 $mm4 $mm5 $mm6 $mm7 $r11 $st0 $st1 $st2 $st3 $st4 $st5 $st6 $st7 $tmm0 $tmm1 $tmm2 $tmm3 $tmm4 $tmm5 $tmm6 $tmm7 $xmm16 $xmm17 $xmm18 $xmm19 $xmm20 $xmm21 $xmm22 $xmm23 $xmm24 $xmm25 $xmm26 $xmm27 $xmm28 $xmm29 $xmm30 $xmm31 $ymm0 $ymm1 $ymm2 $ymm3 $ymm4 $ymm5 $ymm6 $ymm7 $ymm8 $ymm9 $ymm10 $ymm11 $ymm12 $ymm13 $ymm14 $ymm15 $ymm16 $ymm17 $ymm18 $ymm19 $ymm20 $ymm21 $ymm22 $ymm23 $ymm24 $ymm25 $ymm26 $ymm27 $ymm28 $ymm29 $ymm30 $ymm31 $zmm0 $zmm1 $zmm2 $zmm3 $zmm4 $zmm5 $zmm6 $zmm7 $zmm8 $zmm9 $zmm10 $zmm11 $zmm12 $zmm13 $zmm14 $zmm15 $zmm16 $zmm17 $zmm18 $zmm19 $zmm20 $zmm21 $zmm22 $zmm23 $zmm24 $zmm25 $zmm26 $zmm27 $zmm28 $zmm29 $zmm30 $zmm31 $r11b $r11bh $r11d $r11w $r11wh
   call void @bar1()
   call void @bar2()
   ret void
diff --git a/llvm/test/CodeGen/X86/known-signbits-vector.ll b/llvm/test/CodeGen/X86/known-signbits-vector.ll
index 8eb242d7c1fb0..b18b8079fd236 100644
--- a/llvm/test/CodeGen/X86/known-signbits-vector.ll
+++ b/llvm/test/CodeGen/X86/known-signbits-vector.ll
@@ -256,9 +256,8 @@ define <4 x double> @signbits_sext_shuffle_sitofp(<4 x i32> %a0, <4 x i64> %a1)
 ; X86-NEXT:    vpmovsxdq %xmm0, %xmm0
 ; X86-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; X86-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
-; X86-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
 ; X86-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X86-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; X86-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[0,2],xmm0[0,2]
 ; X86-NEXT:    vcvtdq2pd %xmm0, %ymm0
 ; X86-NEXT:    retl
 ;
@@ -269,9 +268,8 @@ define <4 x double> @signbits_sext_shuffle_sitofp(<4 x i32> %a0, <4 x i64> %a1)
 ; X64-AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; X64-AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
-; X64-AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
 ; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X64-AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; X64-AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[0,2],xmm0[0,2]
 ; X64-AVX1-NEXT:    vcvtdq2pd %xmm0, %ymm0
 ; X64-AVX1-NEXT:    retq
 ;
@@ -389,7 +387,6 @@ define float @signbits_ashr_sextvecinreg_bitops_extract_sitofp(<2 x i64> %a0, <4
 ; X86-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,0,8,0]
 ; X86-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; X86-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
-; X86-NEXT:    vpmovsxdq %xmm1, %xmm1
 ; X86-NEXT:    vpand %xmm1, %xmm0, %xmm2
 ; X86-NEXT:    vpor %xmm1, %xmm2, %xmm1
 ; X86-NEXT:    vpxor %xmm0, %xmm1, %xmm0
@@ -407,7 +404,6 @@ define float @signbits_ashr_sextvecinreg_bitops_extract_sitofp(<2 x i64> %a0, <4
 ; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,8]
 ; X64-AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpmovsxdq %xmm1, %xmm1
 ; X64-AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm2
 ; X64-AVX1-NEXT:    vpor %xmm1, %xmm2, %xmm1
 ; X64-AVX1-NEXT:    vpxor %xmm0, %xmm1, %xmm0
@@ -420,7 +416,6 @@ define float @signbits_ashr_sextvecinreg_bitops_extract_sitofp(<2 x i64> %a0, <4
 ; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,8]
 ; X64-AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpmovsxdq %xmm1, %xmm1
 ; X64-AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm2
 ; X64-AVX2-NEXT:    vpor %xmm1, %xmm2, %xmm1
 ; X64-AVX2-NEXT:    vpxor %xmm0, %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/lvi-hardening-loads.ll b/llvm/test/CodeGen/X86/lvi-hardening-loads.ll
index e40911aa1f0c2..ff8276f6f1c22 100644
--- a/llvm/test/CodeGen/X86/lvi-hardening-loads.ll
+++ b/llvm/test/CodeGen/X86/lvi-hardening-loads.ll
@@ -26,8 +26,11 @@ entry:
 ; X64-NEXT:      jmp .LBB0_1
 
 ; X64-NOOPT: # %bb.0: # %entry
+; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movl %esi, -{{[0-9]+}}(%rsp)
+; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movl $0, -{{[0-9]+}}(%rsp)
 ; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movl $0, -{{[0-9]+}}(%rsp)
@@ -48,6 +51,7 @@ for.cond:                                         ; preds = %for.inc, %entry
 
 ; X64-NOOPT: .LBB0_1: # %for.cond
 ; X64-NOOPT-NEXT:      # =>This Inner Loop Header: Depth=1
+; X64-NOOPT-NEXT:  lfence
 ; X64-NOOPT-NEXT:      movl -{{[0-9]+}}(%rsp), %eax
 ; X64-NOOPT-NEXT:  lfence
 ; X64-NOOPT-NEXT:      cmpl -{{[0-9]+}}(%rsp), %eax
@@ -73,12 +77,13 @@ for.body:                                         ; preds = %for.cond
 
 ; X64-NOOPT: # %bb.2: # %for.body
 ; X64-NOOPT-NEXT: # in Loop: Header=BB0_1 Depth=1
-; X64-NOOPT-NEXT:      movl -{{[0-9]+}}(%rsp), %eax
 ; X64-NOOPT-NEXT:  lfence
+; X64-NOOPT-NEXT:      movl -{{[0-9]+}}(%rsp), %eax
 ; X64-NOOPT-NEXT:      cltd
 ; X64-NOOPT-NEXT:      movl $2, %ecx
 ; X64-NOOPT-NEXT:      idivl %ecx
 ; X64-NOOPT-NEXT:      cmpl $0, %edx
+; X64-NOOPT-NEXT:  lfence
 ; X64-NOOPT-NEXT:      jne .LBB0_4
 
 if.then:                                          ; preds = %for.body
@@ -105,6 +110,7 @@ if.then:                                          ; preds = %for.body
 
 ; X64-NOOPT: # %bb.3: # %if.then
 ; X64-NOOPT-NEXT: # in Loop: Header=BB0_1 Depth=1
+; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movq -{{[0-9]+}}(%rsp), %rax
 ; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movslq -{{[0-9]+}}(%rsp), %rcx
@@ -126,10 +132,12 @@ for.inc:                                          ; preds = %if.end
 
 ; X64-NOOPT: .LBB0_5: # %for.inc
 ; X64-NOOPT-NEXT: # in Loop: Header=BB0_1 Depth=1
-; X64-NOOPT-NEXT:      movl -{{[0-9]+}}(%rsp), %eax
 ; X64-NOOPT-NEXT:      lfence
+; X64-NOOPT-NEXT:      movl -{{[0-9]+}}(%rsp), %eax
 ; X64-NOOPT-NEXT:      addl $1, %eax
+; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movl %eax, -{{[0-9]+}}(%rsp)
+; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      jmp .LBB0_1
 
 for.end:                                          ; preds = %for.cond
diff --git a/llvm/test/CodeGen/X86/machine-combiner-int-vec.ll b/llvm/test/CodeGen/X86/machine-combiner-int-vec.ll
index e4420be56b464..4e07b4abde4bd 100644
--- a/llvm/test/CodeGen/X86/machine-combiner-int-vec.ll
+++ b/llvm/test/CodeGen/X86/machine-combiner-int-vec.ll
@@ -13,12 +13,18 @@ define <4 x i32> @reassociate_and_v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: reassociate_and_v4i32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpand %xmm3, %xmm2, %xmm1
-; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    retq
+; AVX2-LABEL: reassociate_and_v4i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpand %xmm3, %xmm2, %xmm1
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: reassociate_and_v4i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpternlogd $128, %xmm2, %xmm3, %xmm0
+; AVX512-NEXT:    retq
 
   %t0 = add <4 x i32> %x0, %x1
   %t1 = and <4 x i32> %x2, %t0
@@ -34,12 +40,18 @@ define <4 x i32> @reassociate_or_v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %
 ; SSE-NEXT:    por %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: reassociate_or_v4i32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpor %xmm3, %xmm2, %xmm1
-; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    retq
+; AVX2-LABEL: reassociate_or_v4i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpor %xmm3, %xmm2, %xmm1
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: reassociate_or_v4i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpternlogd $254, %xmm2, %xmm3, %xmm0
+; AVX512-NEXT:    retq
 
   %t0 = add <4 x i32> %x0, %x1
   %t1 = or <4 x i32> %x2, %t0
@@ -55,12 +67,18 @@ define <4 x i32> @reassociate_xor_v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>
 ; SSE-NEXT:    pxor %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: reassociate_xor_v4i32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm1
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    retq
+; AVX2-LABEL: reassociate_xor_v4i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm1
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: reassociate_xor_v4i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpternlogd $150, %xmm2, %xmm3, %xmm0
+; AVX512-NEXT:    retq
 
   %t0 = add <4 x i32> %x0, %x1
   %t1 = xor <4 x i32> %x2, %t0
@@ -81,12 +99,18 @@ define <8 x i32> @reassociate_and_v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>
 ; SSE-NEXT:    pand %xmm5, %xmm1
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: reassociate_and_v8i32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
-; AVX-NEXT:    vpand %ymm3, %ymm2, %ymm1
-; AVX-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX-NEXT:    retq
+; AVX2-LABEL: reassociate_and_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm3, %ymm2, %ymm1
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: reassociate_and_v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpternlogd $128, %ymm2, %ymm3, %ymm0
+; AVX512-NEXT:    retq
 
   %t0 = add <8 x i32> %x0, %x1
   %t1 = and <8 x i32> %x2, %t0
@@ -105,12 +129,18 @@ define <8 x i32> @reassociate_or_v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %
 ; SSE-NEXT:    por %xmm5, %xmm1
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: reassociate_or_v8i32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
-; AVX-NEXT:    vpor %ymm3, %ymm2, %ymm1
-; AVX-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX-NEXT:    retq
+; AVX2-LABEL: reassociate_or_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %ymm3, %ymm2, %ymm1
+; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: reassociate_or_v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpternlogd $254, %ymm2, %ymm3, %ymm0
+; AVX512-NEXT:    retq
 
   %t0 = add <8 x i32> %x0, %x1
   %t1 = or <8 x i32> %x2, %t0
@@ -129,12 +159,18 @@ define <8 x i32> @reassociate_xor_v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>
 ; SSE-NEXT:    pxor %xmm5, %xmm1
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: reassociate_xor_v8i32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
-; AVX-NEXT:    vpxor %ymm3, %ymm2, %ymm1
-; AVX-NEXT:    vpxor %ymm1, %ymm0, %ymm0
-; AVX-NEXT:    retq
+; AVX2-LABEL: reassociate_xor_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm1
+; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: reassociate_xor_v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpternlogd $150, %ymm2, %ymm3, %ymm0
+; AVX512-NEXT:    retq
 
   %t0 = add <8 x i32> %x0, %x1
   %t1 = xor <8 x i32> %x2, %t0
@@ -175,8 +211,7 @@ define <16 x i32> @reassociate_and_v16i32(<16 x i32> %x0, <16 x i32> %x1, <16 x
 ; AVX512-LABEL: reassociate_and_v16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpandd %zmm3, %zmm2, %zmm1
-; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpternlogd $128, %zmm2, %zmm3, %zmm0
 ; AVX512-NEXT:    retq
 
   %t0 = add <16 x i32> %x0, %x1
@@ -215,8 +250,7 @@ define <16 x i32> @reassociate_or_v16i32(<16 x i32> %x0, <16 x i32> %x1, <16 x i
 ; AVX512-LABEL: reassociate_or_v16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpord %zmm3, %zmm2, %zmm1
-; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpternlogd $254, %zmm2, %zmm3, %zmm0
 ; AVX512-NEXT:    retq
 
   %t0 = add <16 x i32> %x0, %x1
@@ -255,8 +289,7 @@ define <16 x i32> @reassociate_xor_v16i32(<16 x i32> %x0, <16 x i32> %x1, <16 x
 ; AVX512-LABEL: reassociate_xor_v16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpxord %zmm3, %zmm2, %zmm1
-; AVX512-NEXT:    vpxord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpternlogd $150, %zmm2, %zmm3, %zmm0
 ; AVX512-NEXT:    retq
 
   %t0 = add <16 x i32> %x0, %x1
diff --git a/llvm/test/CodeGen/X86/machine-sink-and-implicit-null-checks.ll b/llvm/test/CodeGen/X86/machine-sink-and-implicit-null-checks.ll
index 16ee6ebbbcdb2..f14954782b8c0 100644
--- a/llvm/test/CodeGen/X86/machine-sink-and-implicit-null-checks.ll
+++ b/llvm/test/CodeGen/X86/machine-sink-and-implicit-null-checks.ll
@@ -38,8 +38,8 @@ declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32, i32) no
 define i1 @g(i8 addrspace(1)* %p0, i8* %p1) gc "statepoint-example" {
  entry:
   %c0 = icmp eq i8 addrspace(1)* %p0, null
-  %tok = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0, i8 addrspace(1)* %p0)
-  %p0.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok, i32 7, i32 7) ; (%p0, %p0)
+  %tok = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i8 addrspace(1)* %p0)]
+  %p0.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok, i32 0, i32 0) ; (%p0, %p0)
   br i1 %c0, label %throw0, label %continue0, !make.implicit !0
 
  continue0:
diff --git a/llvm/test/CodeGen/X86/masked_expandload.ll b/llvm/test/CodeGen/X86/masked_expandload.ll
index a66f5a3c2aa9a..7d9e841c624f8 100644
--- a/llvm/test/CodeGen/X86/masked_expandload.ll
+++ b/llvm/test/CodeGen/X86/masked_expandload.ll
@@ -873,9 +873,9 @@ define <16 x double> @expandload_v16f64_v16i32(double* %base, <16 x double> %src
 ; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm5, %ymm5
 ; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm4, %ymm4
 ; AVX2-NEXT:    vpackssdw %ymm5, %ymm4, %ymm4
-; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm4, %xmm5
 ; AVX2-NEXT:    vpacksswb %xmm5, %xmm4, %xmm4
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %xmm4, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    jne LBB3_1
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc.ll b/llvm/test/CodeGen/X86/masked_store_trunc.ll
index 51e0c25a45e98..341a349911180 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc.ll
@@ -2377,9 +2377,9 @@ define void @truncstore_v16i32_v16i16(<16 x i32> %x, <16 x i16>* %p, <16 x i32>
 ; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm2, %ymm2
 ; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    jne .LBB9_1
@@ -3058,9 +3058,9 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 ; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm2, %ymm2
 ; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    jne .LBB10_1
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll b/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
index 51a9d923c247c..c120684c42610 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
@@ -782,9 +782,9 @@ define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask
 ; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm0, %ymm5
 ; AVX2-NEXT:    vblendvpd %ymm5, %ymm0, %ymm4, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm1
 ; AVX2-NEXT:    vmovmskps %ymm1, %eax
 ; AVX2-NEXT:    notl %eax
@@ -3290,9 +3290,9 @@ define void @truncstore_v16i32_v16i16(<16 x i32> %x, <16 x i16>* %p, <16 x i32>
 ; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm2, %ymm2
 ; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    jne .LBB9_1
@@ -3952,18 +3952,18 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm3, %ymm1
 ; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
 ; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm2, %ymm2
 ; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    jne .LBB10_1
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll b/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
index f90d001ab4171..0160733732d70 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
@@ -655,9 +655,9 @@ define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask
 ; AVX2-NEXT:    vpcmpgtq %ymm5, %ymm7, %ymm5
 ; AVX2-NEXT:    vblendvpd %ymm5, %ymm0, %ymm4, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm1
 ; AVX2-NEXT:    vmovmskps %ymm1, %eax
 ; AVX2-NEXT:    notl %eax
@@ -2958,9 +2958,9 @@ define void @truncstore_v16i32_v16i16(<16 x i32> %x, <16 x i16>* %p, <16 x i32>
 ; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm2, %ymm2
 ; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    jne .LBB9_1
@@ -3658,18 +3658,18 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 ; AVX2-NEXT:    vpminud %ymm5, %ymm1, %ymm1
 ; AVX2-NEXT:    vpminud %ymm5, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm3, %ymm1
 ; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
 ; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm2, %ymm2
 ; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    jne .LBB10_1
diff --git a/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll b/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll
index 9c2768792e000..dcb5806c51fb1 100644
--- a/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll
+++ b/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll
@@ -2261,12 +2261,12 @@ define <8 x i16> @vec128_i16_unsigned_reg_reg(<8 x i16> %a1, <8 x i16> %a2) noun
 ; AVX512VL-FALLBACK:       # %bb.0:
 ; AVX512VL-FALLBACK-NEXT:    vpminuw %xmm1, %xmm0, %xmm2
 ; AVX512VL-FALLBACK-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm3
-; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %xmm3, %xmm3, %xmm3
-; AVX512VL-FALLBACK-NEXT:    vpor {{.*}}(%rip), %xmm3, %xmm3
+; AVX512VL-FALLBACK-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX512VL-FALLBACK-NEXT:    vpternlogq $190, {{.*}}(%rip), %xmm3, %xmm4
 ; AVX512VL-FALLBACK-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm1
 ; AVX512VL-FALLBACK-NEXT:    vpsubw %xmm2, %xmm1, %xmm1
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %xmm1, %xmm1
-; AVX512VL-FALLBACK-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
+; AVX512VL-FALLBACK-NEXT:    vpmullw %xmm4, %xmm1, %xmm1
 ; AVX512VL-FALLBACK-NEXT:    vpaddw %xmm0, %xmm1, %xmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
 ;
@@ -3121,14 +3121,14 @@ define <16 x i8> @vec128_i8_unsigned_reg_reg(<16 x i8> %a1, <16 x i8> %a2) nounw
 ; AVX512VL-FALLBACK:       # %bb.0:
 ; AVX512VL-FALLBACK-NEXT:    vpminub %xmm1, %xmm0, %xmm2
 ; AVX512VL-FALLBACK-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm3
-; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %xmm3, %xmm3, %xmm3
-; AVX512VL-FALLBACK-NEXT:    vpor {{.*}}(%rip), %xmm3, %xmm3
+; AVX512VL-FALLBACK-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX512VL-FALLBACK-NEXT:    vpternlogq $190, {{.*}}(%rip), %xmm3, %xmm4
 ; AVX512VL-FALLBACK-NEXT:    vpmaxub %xmm1, %xmm0, %xmm1
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %xmm2, %xmm1, %xmm1
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %xmm1, %xmm1
 ; AVX512VL-FALLBACK-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; AVX512VL-FALLBACK-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512VL-FALLBACK-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
+; AVX512VL-FALLBACK-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
 ; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512VL-FALLBACK-NEXT:    vpmovdb %zmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/midpoint-int-vec-256.ll b/llvm/test/CodeGen/X86/midpoint-int-vec-256.ll
index 0710e953d6d43..f7f1c8bac87b4 100644
--- a/llvm/test/CodeGen/X86/midpoint-int-vec-256.ll
+++ b/llvm/test/CodeGen/X86/midpoint-int-vec-256.ll
@@ -1855,12 +1855,12 @@ define <16 x i16> @vec256_i16_unsigned_reg_reg(<16 x i16> %a1, <16 x i16> %a2) n
 ; AVX512VL-FALLBACK:       # %bb.0:
 ; AVX512VL-FALLBACK-NEXT:    vpminuw %ymm1, %ymm0, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %ymm3, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpor {{.*}}(%rip), %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX512VL-FALLBACK-NEXT:    vpternlogq $190, {{.*}}(%rip), %ymm3, %ymm4
 ; AVX512VL-FALLBACK-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm2, %ymm1, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm1, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm0, %ymm1, %ymm0
 ; AVX512VL-FALLBACK-NEXT:    retq
 ;
@@ -2789,21 +2789,21 @@ define <32 x i8> @vec256_i8_unsigned_reg_reg(<32 x i8> %a1, <32 x i8> %a2) nounw
 ; AVX512VL-FALLBACK:       # %bb.0:
 ; AVX512VL-FALLBACK-NEXT:    vpminub %ymm1, %ymm0, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %ymm3, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpor {{.*}}(%rip), %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX512VL-FALLBACK-NEXT:    vpternlogq $190, {{.*}}(%rip), %ymm3, %ymm4
 ; AVX512VL-FALLBACK-NEXT:    vpmaxub %ymm1, %ymm0, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm2, %ymm1, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm1, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm3, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
 ; AVX512VL-FALLBACK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/midpoint-int-vec-512.ll b/llvm/test/CodeGen/X86/midpoint-int-vec-512.ll
index cb9951ccb4947..3cd81a0c9707f 100644
--- a/llvm/test/CodeGen/X86/midpoint-int-vec-512.ll
+++ b/llvm/test/CodeGen/X86/midpoint-int-vec-512.ll
@@ -418,10 +418,9 @@ define <32 x i16> @vec512_i16_unsigned_reg_reg(<32 x i16> %a1, <32 x i16> %a2) n
 ; AVX512VL-FALLBACK-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; AVX512VL-FALLBACK-NEXT:    vpminuw %ymm2, %ymm3, %ymm4
 ; AVX512VL-FALLBACK-NEXT:    vpcmpeqw %ymm4, %ymm3, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %ymm5, %ymm5, %ymm5
 ; AVX512VL-FALLBACK-NEXT:    vpminuw %ymm1, %ymm0, %ymm6
 ; AVX512VL-FALLBACK-NEXT:    vpcmpeqw %ymm6, %ymm0, %ymm7
-; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %ymm7, %ymm7, %ymm7
+; AVX512VL-FALLBACK-NEXT:    vpcmpeqd %ymm8, %ymm8, %ymm8
 ; AVX512VL-FALLBACK-NEXT:    vpmaxuw %ymm2, %ymm3, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm6, %ymm1, %ymm1
@@ -429,10 +428,10 @@ define <32 x i16> @vec512_i16_unsigned_reg_reg(<32 x i16> %a1, <32 x i16> %a2) n
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm1, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512VL-FALLBACK-NEXT:    vpor %ymm4, %ymm7, %ymm6
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm6, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpor %ymm4, %ymm5, %ymm4
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpternlogq $222, %ymm8, %ymm4, %ymm7
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm7, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpternlogq $222, %ymm8, %ymm4, %ymm5
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm3, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm0, %ymm1, %ymm0
 ; AVX512VL-FALLBACK-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
@@ -877,45 +876,44 @@ define <64 x i8> @vec512_i8_unsigned_reg_reg(<64 x i8> %a1, <64 x i8> %a2) nounw
 ;
 ; AVX512VL-FALLBACK-LABEL: vec512_i8_unsigned_reg_reg:
 ; AVX512VL-FALLBACK:       # %bb.0:
-; AVX512VL-FALLBACK-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
 ; AVX512VL-FALLBACK-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpminub %ymm3, %ymm2, %ymm4
-; AVX512VL-FALLBACK-NEXT:    vpcmpeqb %ymm4, %ymm2, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %ymm5, %ymm5, %ymm5
+; AVX512VL-FALLBACK-NEXT:    vpminub %ymm4, %ymm2, %ymm5
+; AVX512VL-FALLBACK-NEXT:    vpcmpeqb %ymm5, %ymm2, %ymm3
 ; AVX512VL-FALLBACK-NEXT:    vpminub %ymm1, %ymm0, %ymm6
 ; AVX512VL-FALLBACK-NEXT:    vpcmpeqb %ymm6, %ymm0, %ymm7
-; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %ymm7, %ymm7, %ymm7
-; AVX512VL-FALLBACK-NEXT:    vpmaxub %ymm3, %ymm2, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpcmpeqd %ymm8, %ymm8, %ymm8
+; AVX512VL-FALLBACK-NEXT:    vpmaxub %ymm4, %ymm2, %ymm4
 ; AVX512VL-FALLBACK-NEXT:    vpmaxub %ymm1, %ymm0, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm6, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm4, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm5, %ymm4, %ymm4
+; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm4, %ymm4
+; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm4, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm5, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512VL-FALLBACK-NEXT:    vpor %ymm6, %ymm7, %ymm7
-; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm4, %ymm8, %ymm4
-; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm8 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm4, %ymm8, %ymm4
+; AVX512VL-FALLBACK-NEXT:    vpternlogq $222, %ymm8, %ymm6, %ymm7
+; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm9, %ymm5
+; AVX512VL-FALLBACK-NEXT:    vmovdqa {{.*#+}} ymm9 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm5, %ymm9, %ymm5
 ; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm7, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm1, %ymm8, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512VL-FALLBACK-NEXT:    vpor %ymm6, %ymm5, %ymm5
-; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm6, %ymm4, %ymm4
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm4, %ymm8, %ymm4
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm1, %ymm9, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm5, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512VL-FALLBACK-NEXT:    vpternlogq $222, %ymm8, %ymm6, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm6, %ymm5, %ymm5
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm5, %ymm9, %ymm5
+; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512VL-FALLBACK-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm5, %ymm3, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpand %ymm3, %ymm8, %ymm3
-; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm4, %ymm3, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpmullw %ymm3, %ymm4, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpand %ymm3, %ymm9, %ymm3
+; AVX512VL-FALLBACK-NEXT:    vpackuswb %ymm5, %ymm3, %ymm3
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm2, %ymm3, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
 ; AVX512VL-FALLBACK-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index ca4af0c75eda1..2980a1665db14 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -1639,3 +1639,162 @@ define void @cmp_v8i64_zext(<8 x i64>* %xptr, <8 x i64>* %yptr, <8 x i64>* %zptr
   store <8 x i64> %ext, <8 x i64>* %zptr
   ret void
 }
+
+define <16 x i8> @var_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind "min-legal-vector-width"="256" {
+; CHECK-LABEL: var_rotate_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; CHECK-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
+; CHECK-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; CHECK-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; CHECK-NEXT:    vpsllvw %ymm1, %ymm0, %ymm1
+; CHECK-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
+; CHECK-NEXT:    vpsrlvw %ymm2, %ymm0, %ymm0
+; CHECK-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    vpmovwb %ymm0, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %b8 = sub <16 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %b
+  %shl = shl <16 x i8> %a, %b
+  %lshr = lshr <16 x i8> %a, %b8
+  %or = or <16 x i8> %shl, %lshr
+  ret <16 x i8> %or
+}
+
+define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind "min-legal-vector-width"="256" {
+; CHECK-LABEL: var_rotate_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllw $4, %ymm0, %ymm2
+; CHECK-NEXT:    vpsrlw $4, %ymm0, %ymm3
+; CHECK-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
+; CHECK-NEXT:    vpsllw $5, %ymm1, %ymm1
+; CHECK-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; CHECK-NEXT:    vpsllw $2, %ymm0, %ymm2
+; CHECK-NEXT:    vpsrlw $6, %ymm0, %ymm3
+; CHECK-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
+; CHECK-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; CHECK-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; CHECK-NEXT:    vpsrlw $7, %ymm0, %ymm2
+; CHECK-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
+; CHECK-NEXT:    vpternlogq $248, {{.*}}(%rip), %ymm2, %ymm3
+; CHECK-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; CHECK-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; CHECK-NEXT:    retq
+  %b8 = sub <32 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %b
+  %shl = shl <32 x i8> %a, %b
+  %lshr = lshr <32 x i8> %a, %b8
+  %or = or <32 x i8> %shl, %lshr
+  ret <32 x i8> %or
+}
+
+define <32 x i8> @splatvar_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind "min-legal-vector-width"="256" {
+; CHECK-LABEL: splatvar_rotate_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpbroadcastb %xmm1, %xmm1
+; CHECK-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vpsllw %xmm2, %ymm0, %ymm3
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; CHECK-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; CHECK-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; CHECK-NEXT:    vpsllw %xmm2, %xmm4, %xmm2
+; CHECK-NEXT:    vpbroadcastb %xmm2, %ymm2
+; CHECK-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vpsrlw %xmm1, %ymm0, %ymm5
+; CHECK-NEXT:    vpand %ymm2, %ymm3, %ymm2
+; CHECK-NEXT:    vpsrlw %xmm1, %xmm4, %xmm0
+; CHECK-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; CHECK-NEXT:    vpbroadcastb %xmm0, %ymm0
+; CHECK-NEXT:    vpternlogq $236, %ymm5, %ymm2, %ymm0
+; CHECK-NEXT:    retq
+  %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer
+  %splat8 = sub <32 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %splat
+  %shl = shl <32 x i8> %a, %splat
+  %lshr = lshr <32 x i8> %a, %splat8
+  %or = or <32 x i8> %shl, %lshr
+  ret <32 x i8> %or
+}
+
+define <32 x i8> @constant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {
+; CHECK-AVX512-LABEL: constant_rotate_v32i8:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    vpsllw $4, %ymm0, %ymm1
+; CHECK-AVX512-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; CHECK-AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
+; CHECK-AVX512-NEXT:    # ymm2 = mem[0,1,0,1]
+; CHECK-AVX512-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm1
+; CHECK-AVX512-NEXT:    vpsllw $2, %ymm1, %ymm3
+; CHECK-AVX512-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; CHECK-AVX512-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
+; CHECK-AVX512-NEXT:    vpblendvb %ymm2, %ymm3, %ymm1, %ymm1
+; CHECK-AVX512-NEXT:    vpaddb %ymm1, %ymm1, %ymm3
+; CHECK-AVX512-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
+; CHECK-AVX512-NEXT:    vpblendvb %ymm2, %ymm3, %ymm1, %ymm1
+; CHECK-AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; CHECK-AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
+; CHECK-AVX512-NEXT:    vpsllvw {{.*}}(%rip), %ymm3, %ymm3
+; CHECK-AVX512-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; CHECK-AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
+; CHECK-AVX512-NEXT:    vpsllvw {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-AVX512-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; CHECK-AVX512-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
+; CHECK-AVX512-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; CHECK-AVX512-NEXT:    retq
+;
+; CHECK-VBMI-LABEL: constant_rotate_v32i8:
+; CHECK-VBMI:       # %bb.0:
+; CHECK-VBMI-NEXT:    vpsllw $4, %ymm0, %ymm1
+; CHECK-VBMI-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; CHECK-VBMI-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
+; CHECK-VBMI-NEXT:    # ymm2 = mem[0,1,0,1]
+; CHECK-VBMI-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm1
+; CHECK-VBMI-NEXT:    vpsllw $2, %ymm1, %ymm3
+; CHECK-VBMI-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; CHECK-VBMI-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
+; CHECK-VBMI-NEXT:    vpblendvb %ymm2, %ymm3, %ymm1, %ymm1
+; CHECK-VBMI-NEXT:    vpaddb %ymm1, %ymm1, %ymm3
+; CHECK-VBMI-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
+; CHECK-VBMI-NEXT:    vpblendvb %ymm2, %ymm3, %ymm1, %ymm1
+; CHECK-VBMI-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; CHECK-VBMI-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
+; CHECK-VBMI-NEXT:    vpsllvw {{.*}}(%rip), %ymm3, %ymm3
+; CHECK-VBMI-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
+; CHECK-VBMI-NEXT:    vpsllvw {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-VBMI-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,3,5,7,9,11,13,15,33,35,37,39,41,43,45,47,17,19,21,23,25,27,29,31,49,51,53,55,57,59,61,63]
+; CHECK-VBMI-NEXT:    vpermi2b %ymm3, %ymm0, %ymm2
+; CHECK-VBMI-NEXT:    vpor %ymm2, %ymm1, %ymm0
+; CHECK-VBMI-NEXT:    retq
+  %shl = shl <32 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>
+  %lshr = lshr <32 x i8> %a, <i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>
+  %or = or <32 x i8> %shl, %lshr
+  ret <32 x i8> %or
+}
+
+define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {
+; CHECK-LABEL: splatconstant_rotate_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllw $4, %ymm0, %ymm1
+; CHECK-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; CHECK-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %shl = shl <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
+  %lshr = lshr <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
+  %or = or <32 x i8> %shl, %lshr
+  ret <32 x i8> %or
+}
+
+define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {
+; CHECK-LABEL: splatconstant_rotate_mask_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllw $4, %ymm0, %ymm1
+; CHECK-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; CHECK-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
+; CHECK-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    retq
+  %shl = shl <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
+  %lshr = lshr <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
+  %rmask = and <32 x i8> %lshr, <i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55>
+  %lmask = and <32 x i8> %shl, <i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33>
+  %or = or <32 x i8> %lmask, %rmask
+  ret <32 x i8> %or
+}
diff --git a/llvm/test/CodeGen/X86/mixed-ptr-sizes-i686.ll b/llvm/test/CodeGen/X86/mixed-ptr-sizes-i686.ll
index 8b92376c966e4..14a233ed7fd4e 100644
--- a/llvm/test/CodeGen/X86/mixed-ptr-sizes-i686.ll
+++ b/llvm/test/CodeGen/X86/mixed-ptr-sizes-i686.ll
@@ -236,3 +236,111 @@ entry:
   tail call void @use_foo(%struct.Foo* %f)
   ret void
 }
+
+define i32 @test_load_sptr32(i32 addrspace(270)* %i) {
+; CHECK-LABEL: test_load_sptr32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl (%eax), %eax
+; CHECK-NEXT:    retl
+; CHECK-O0-LABEL: test_load_sptr32:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-O0-NEXT:    movl (%eax), %eax
+; CHECK-O0-NEXT:    retl
+entry:
+  %0 = load i32, i32 addrspace(270)* %i, align 4
+  ret i32 %0
+}
+
+define i32 @test_load_uptr32(i32 addrspace(271)* %i) {
+; CHECK-LABEL: test_load_uptr32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl (%eax), %eax
+; CHECK-NEXT:    retl
+; CHECK-O0-LABEL: test_load_uptr32:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-O0-NEXT:    movl (%eax), %eax
+; CHECK-O0-NEXT:    retl
+entry:
+  %0 = load i32, i32 addrspace(271)* %i, align 4
+  ret i32 %0
+}
+
+define i32 @test_load_ptr64(i32 addrspace(272)* %i) {
+; CHECK-LABEL: test_load_ptr64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl (%eax), %eax
+; CHECK-NEXT:    retl
+; CHECK-O0-LABEL: test_load_ptr64:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    pushl %eax
+; CHECK-O0-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-O0-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-O0-NEXT:    movl (%ecx), %ecx
+; CHECK-O0-NEXT:    movl %eax, (%esp)
+; CHECK-O0-NEXT:    movl %ecx, %eax
+; CHECK-O0-NEXT:    popl %ecx
+; CHECK-O0-NEXT:    retl
+entry:
+  %0 = load i32, i32 addrspace(272)* %i, align 8
+  ret i32 %0
+}
+
+define void @test_store_sptr32(i32 addrspace(270)* %s, i32 %i) {
+; CHECK-LABEL: test_store_sptr32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl %eax, (%ecx)
+; CHECK-NEXT:    retl
+; CHECK-O0-LABEL: test_store_sptr32:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-O0-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-O0-NEXT:    movl %eax, (%ecx)
+; CHECK-O0-NEXT:    retl
+entry:
+  store i32 %i, i32 addrspace(270)* %s, align 4
+  ret void
+}
+
+define void @test_store_uptr32(i32 addrspace(271)* %s, i32 %i) {
+; CHECK-LABEL: test_store_uptr32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl %eax, (%ecx)
+; CHECK-NEXT:    retl
+; CHECK-O0-LABEL: test_store_uptr32:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-O0-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-O0-NEXT:    movl %eax, (%ecx)
+; CHECK-O0-NEXT:    retl
+entry:
+  store i32 %i, i32 addrspace(271)* %s, align 4
+  ret void
+}
+
+define void @test_store_ptr64(i32 addrspace(272)* %s, i32 %i) {
+; CHECK-LABEL: test_store_ptr64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl %eax, (%ecx)
+; CHECK-NEXT:    retl
+; CHECK-O0-LABEL: test_store_ptr64:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-O0-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-O0-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-O0-NEXT:    movl %edx, (%ecx)
+; CHECK-O0-NEXT:    retl
+entry:
+  store i32 %i, i32 addrspace(272)* %s, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/mixed-ptr-sizes.ll b/llvm/test/CodeGen/X86/mixed-ptr-sizes.ll
index 7a470fb213173..ac55e1a1fc653 100644
--- a/llvm/test/CodeGen/X86/mixed-ptr-sizes.ll
+++ b/llvm/test/CodeGen/X86/mixed-ptr-sizes.ll
@@ -125,6 +125,23 @@ entry:
 
 ; Test that null can be passed as a 32-bit pointer.
 define dso_local void @test_null_arg(%struct.Foo* %f) {
+; CHECK-LABEL: test_null_arg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK:         xorl %edx, %edx
+; CHECK-NEXT:    callq test_noop1
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addq $40, %rsp
+; CHECK-NEXT:    retq
+;
+; CHECK-O0-LABEL: test_null_arg:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    subq $40, %rsp
+; CHECK-O0:         xorl %edx, %edx
+; CHECK-O0-NEXT:    callq test_noop1
+; CHECK-O0-NEXT:    nop
+; CHECK-O0-NEXT:    addq $40, %rsp
+; CHECK-O0-NEXT:    retq
 entry:
   call void @test_noop1(%struct.Foo* %f, i32 addrspace(270)* null)
   ret void
@@ -170,3 +187,97 @@ entry:
   tail call void @use_foo(%struct.Foo* %f)
   ret void
 }
+
+define i32 @test_load_sptr32(i32 addrspace(270)* %i) {
+; CHECK-LABEL: test_load_sptr32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movslq  %ecx, %rax
+; CHECK-NEXT:    movl (%rax), %eax
+; CHECK-NEXT:    retq
+; CHECK-O0-LABEL: test_load_sptr32:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    movslq  %ecx, %rax
+; CHECK-O0-NEXT:    movl (%rax), %eax
+; CHECK-O0-NEXT:    retq
+entry:
+  %0 = load i32, i32 addrspace(270)* %i, align 4
+  ret i32 %0
+}
+
+define i32 @test_load_uptr32(i32 addrspace(271)* %i) {
+; CHECK-LABEL: test_load_uptr32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl %ecx, %eax
+; CHECK-NEXT:    movl (%rax), %eax
+; CHECK-NEXT:    retq
+; CHECK-O0-LABEL: test_load_uptr32:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    movl %ecx, %eax
+; CHECK-O0-NEXT:    # kill: def $rax killed $eax
+; CHECK-O0-NEXT:    movl (%rax), %eax
+; CHECK-O0-NEXT:    retq
+entry:
+  %0 = load i32, i32 addrspace(271)* %i, align 4
+  ret i32 %0
+}
+
+define i32 @test_load_ptr64(i32 addrspace(272)* %i) {
+; CHECK-LABEL: test_load_ptr64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl (%rcx), %eax
+; CHECK-NEXT:    retq
+; CHECK-O0-LABEL: test_load_ptr64:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    movl (%rcx), %eax
+; CHECK-O0-NEXT:    retq
+entry:
+  %0 = load i32, i32 addrspace(272)* %i, align 8
+  ret i32 %0
+}
+
+define void @test_store_sptr32(i32 addrspace(270)* %s, i32 %i) {
+; CHECK-LABEL: test_store_sptr32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movslq %ecx, %rax
+; CHECK-NEXT:    movl %edx, (%rax)
+; CHECK-NEXT:    retq
+; CHECK-O0-LABEL: test_store_sptr32:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    movslq %ecx, %rax
+; CHECK-O0-NEXT:    movl %edx, (%rax)
+; CHECK-O0-NEXT:    retq
+entry:
+  store i32 %i, i32 addrspace(270)* %s, align 4
+  ret void
+}
+
+define void @test_store_uptr32(i32 addrspace(271)* %s, i32 %i) {
+; CHECK-LABEL: test_store_uptr32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl %ecx, %eax
+; CHECK-NEXT:    movl %edx, (%rax)
+; CHECK-NEXT:    retq
+; CHECK-O0-LABEL: test_store_uptr32:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    movl %ecx, %eax
+; CHECK-O0-NEXT:    # kill: def $rax killed $eax
+; CHECK-O0-NEXT:    movl %edx, (%rax)
+; CHECK-O0-NEXT:    retq
+entry:
+  store i32 %i, i32 addrspace(271)* %s, align 4
+  ret void
+}
+
+define void @test_store_ptr64(i32 addrspace(272)* %s, i32 %i) {
+; CHECK-LABEL: test_store_ptr64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl %edx, (%rcx)
+; CHECK-NEXT:    retq
+; CHECK-O0-LABEL: test_store_ptr64:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    movl %edx, (%rcx)
+; CHECK-O0-NEXT:    retq
+entry:
+  store i32 %i, i32 addrspace(272)* %s, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/mmx-intrinsics.ll b/llvm/test/CodeGen/X86/mmx-intrinsics.ll
index 48d4ad0490f76..abc8fea58014a 100644
--- a/llvm/test/CodeGen/X86/mmx-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/mmx-intrinsics.ll
@@ -311,6 +311,19 @@ entry:
   ret i64 %4
 }
 
+define i64 @test72_2(<1 x i64> %a) nounwind readnone optsize ssp {
+; ALL-LABEL: @test72_2
+; ALL-NOT: psraw
+entry:
+  %0 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %0 to x86_mmx
+  %1 = tail call x86_mmx @llvm.x86.mmx.psrai.w(x86_mmx %mmx_var.i, i32 0) nounwind
+  %2 = bitcast x86_mmx %1 to <4 x i16>
+  %3 = bitcast <4 x i16> %2 to <1 x i64>
+  %4 = extractelement <1 x i64> %3, i32 0
+  ret i64 %4
+}
+
 declare x86_mmx @llvm.x86.mmx.psrli.q(x86_mmx, i32) nounwind readnone
 
 define i64 @test71(<1 x i64> %a) nounwind readnone optsize ssp {
@@ -339,6 +352,19 @@ entry:
   ret i64 %4
 }
 
+define i64 @test70_2(<1 x i64> %a) nounwind readnone optsize ssp {
+; ALL-LABEL: @test70_2
+; ALL-NOT: psrld
+entry:
+  %0 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %0 to x86_mmx
+  %1 = tail call x86_mmx @llvm.x86.mmx.psrli.d(x86_mmx %mmx_var.i, i32 0) nounwind
+  %2 = bitcast x86_mmx %1 to <2 x i32>
+  %3 = bitcast <2 x i32> %2 to <1 x i64>
+  %4 = extractelement <1 x i64> %3, i32 0
+  ret i64 %4
+}
+
 declare x86_mmx @llvm.x86.mmx.psrli.w(x86_mmx, i32) nounwind readnone
 
 define i64 @test69(<1 x i64> %a) nounwind readnone optsize ssp {
@@ -397,6 +423,19 @@ entry:
   ret i64 %4
 }
 
+define i64 @test66_2(<1 x i64> %a) nounwind readnone optsize ssp {
+; ALL-LABEL: @test66_2
+; ALL-NOT: psllw
+entry:
+  %0 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %0 to x86_mmx
+  %1 = tail call x86_mmx @llvm.x86.mmx.pslli.w(x86_mmx %mmx_var.i, i32 0) nounwind
+  %2 = bitcast x86_mmx %1 to <4 x i16>
+  %3 = bitcast <4 x i16> %2 to <1 x i64>
+  %4 = extractelement <1 x i64> %3, i32 0
+  ret i64 %4
+}
+
 declare x86_mmx @llvm.x86.mmx.psra.d(x86_mmx, x86_mmx) nounwind readnone
 
 define i64 @test65(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp {
diff --git a/llvm/test/CodeGen/X86/movmsk-cmp.ll b/llvm/test/CodeGen/X86/movmsk-cmp.ll
index 288069bedd941..08706502f4d23 100644
--- a/llvm/test/CodeGen/X86/movmsk-cmp.ll
+++ b/llvm/test/CodeGen/X86/movmsk-cmp.ll
@@ -706,7 +706,6 @@ define i1 @allones_v16i32_sign(<16 x i32> %arg) {
 ; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
@@ -767,9 +766,8 @@ define i1 @allzeros_v16i32_sign(<16 x i32> %arg) {
 ; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; AVX2-NEXT:    testl $-1431655766, %eax # imm = 0xAAAAAAAA
+; AVX2-NEXT:    testl %eax, %eax
 ; AVX2-NEXT:    sete %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -2149,7 +2147,6 @@ define i1 @allones_v16i32_and1(<16 x i32> %arg) {
 ; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
@@ -2211,9 +2208,8 @@ define i1 @allzeros_v16i32_and1(<16 x i32> %arg) {
 ; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; AVX2-NEXT:    testl $-1431655766, %eax # imm = 0xAAAAAAAA
+; AVX2-NEXT:    testl %eax, %eax
 ; AVX2-NEXT:    sete %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -3522,7 +3518,6 @@ define i1 @allones_v16i32_and4(<16 x i32> %arg) {
 ; AVX2-NEXT:    vpslld $29, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
@@ -3584,9 +3579,8 @@ define i1 @allzeros_v16i32_and4(<16 x i32> %arg) {
 ; AVX2-NEXT:    vpslld $29, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; AVX2-NEXT:    testl $-1431655766, %eax # imm = 0xAAAAAAAA
+; AVX2-NEXT:    testl %eax, %eax
 ; AVX2-NEXT:    sete %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/O3-pipeline.ll b/llvm/test/CodeGen/X86/opt-pipeline.ll
similarity index 97%
rename from llvm/test/CodeGen/X86/O3-pipeline.ll
rename to llvm/test/CodeGen/X86/opt-pipeline.ll
index c91b8143e09c7..2aa88abd2db8c 100644
--- a/llvm/test/CodeGen/X86/O3-pipeline.ll
+++ b/llvm/test/CodeGen/X86/opt-pipeline.ll
@@ -1,5 +1,9 @@
 ; When EXPENSIVE_CHECKS are enabled, the machine verifier appears between each
 ; pass. Ignore it with 'grep -v'.
+; RUN: llc -mtriple=x86_64-- -O1 -debug-pass=Structure < %s -o /dev/null 2>&1 \
+; RUN:   | grep -v 'Verify generated machine code' | FileCheck %s
+; RUN: llc -mtriple=x86_64-- -O2 -debug-pass=Structure < %s -o /dev/null 2>&1 \
+; RUN:   | grep -v 'Verify generated machine code' | FileCheck %s
 ; RUN: llc -mtriple=x86_64-- -O3 -debug-pass=Structure < %s -o /dev/null 2>&1 \
 ; RUN:   | grep -v 'Verify generated machine code' | FileCheck %s
 
diff --git a/llvm/test/CodeGen/X86/optimize-max-0.ll b/llvm/test/CodeGen/X86/optimize-max-0.ll
index 5fb2371cb8e20..e7f885625b764 100644
--- a/llvm/test/CodeGen/X86/optimize-max-0.ll
+++ b/llvm/test/CodeGen/X86/optimize-max-0.ll
@@ -15,30 +15,31 @@ define void @foo(i8* %r, i32 %s, i32 %w, i32 %x, i8* %j, i32 %d) nounwind {
 ; CHECK-NEXT:    pushl %edi
 ; CHECK-NEXT:    pushl %esi
 ; CHECK-NEXT:    subl $28, %esp
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebx
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; CHECK-NEXT:    movl %ebx, %eax
-; CHECK-NEXT:    imull %edi, %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl %edi, %ebx
+; CHECK-NEXT:    imull %ebp, %ebx
 ; CHECK-NEXT:    cmpl $1, {{[0-9]+}}(%esp)
-; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    movl %ebx, (%esp) ## 4-byte Spill
 ; CHECK-NEXT:    je LBB0_19
 ; CHECK-NEXT:  ## %bb.1: ## %bb10.preheader
-; CHECK-NEXT:    movl %eax, %ebp
-; CHECK-NEXT:    sarl $31, %ebp
-; CHECK-NEXT:    shrl $30, %ebp
-; CHECK-NEXT:    addl %eax, %ebp
-; CHECK-NEXT:    sarl $2, %ebp
-; CHECK-NEXT:    testl %ebx, %ebx
+; CHECK-NEXT:    movl %ebx, %ecx
+; CHECK-NEXT:    sarl $31, %ecx
+; CHECK-NEXT:    shrl $30, %ecx
+; CHECK-NEXT:    addl %ebx, %ecx
+; CHECK-NEXT:    sarl $2, %ecx
+; CHECK-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    testl %edi, %edi
 ; CHECK-NEXT:    jle LBB0_12
 ; CHECK-NEXT:  ## %bb.2: ## %bb.nph9
-; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    testl %ebp, %ebp
 ; CHECK-NEXT:    jle LBB0_12
 ; CHECK-NEXT:  ## %bb.3: ## %bb.nph9.split
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    incl %eax
 ; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK-NEXT:    xorl %esi, %esi
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB0_4: ## %bb6
@@ -46,35 +47,33 @@ define void @foo(i8* %r, i32 %s, i32 %w, i32 %x, i8* %j, i32 %d) nounwind {
 ; CHECK-NEXT:    movzbl (%eax,%esi,2), %ebx
 ; CHECK-NEXT:    movb %bl, (%edx,%esi)
 ; CHECK-NEXT:    incl %esi
-; CHECK-NEXT:    cmpl %edi, %esi
+; CHECK-NEXT:    cmpl %ebp, %esi
 ; CHECK-NEXT:    jl LBB0_4
 ; CHECK-NEXT:  ## %bb.5: ## %bb9
 ; CHECK-NEXT:    ## in Loop: Header=BB0_4 Depth=1
 ; CHECK-NEXT:    incl %ecx
 ; CHECK-NEXT:    addl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT:    addl %edi, %edx
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; CHECK-NEXT:    cmpl %ebx, %ecx
+; CHECK-NEXT:    addl %ebp, %edx
+; CHECK-NEXT:    cmpl %edi, %ecx
 ; CHECK-NEXT:    je LBB0_12
 ; CHECK-NEXT:  ## %bb.6: ## %bb7.preheader
 ; CHECK-NEXT:    ## in Loop: Header=BB0_4 Depth=1
 ; CHECK-NEXT:    xorl %esi, %esi
 ; CHECK-NEXT:    jmp LBB0_4
 ; CHECK-NEXT:  LBB0_12: ## %bb18.loopexit
-; CHECK-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
-; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax ## 4-byte Reload
-; CHECK-NEXT:    addl %ebp, %eax
+; CHECK-NEXT:    movl (%esp), %eax ## 4-byte Reload
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx ## 4-byte Reload
+; CHECK-NEXT:    addl %ecx, %eax
 ; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
-; CHECK-NEXT:    cmpl $1, %ebx
+; CHECK-NEXT:    cmpl $1, %edi
 ; CHECK-NEXT:    jle LBB0_13
 ; CHECK-NEXT:  ## %bb.7: ## %bb.nph5
-; CHECK-NEXT:    cmpl $2, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    cmpl $2, %ebp
 ; CHECK-NEXT:    jl LBB0_13
 ; CHECK-NEXT:  ## %bb.8: ## %bb.nph5.split
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT:    movl %eax, %edx
+; CHECK-NEXT:    movl %ebp, %edx
 ; CHECK-NEXT:    shrl $31, %edx
-; CHECK-NEXT:    addl %eax, %edx
+; CHECK-NEXT:    addl %ebp, %edx
 ; CHECK-NEXT:    sarl %edx
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl %eax, %ecx
@@ -88,22 +87,18 @@ define void @foo(i8* %r, i32 %s, i32 %w, i32 %x, i8* %j, i32 %d) nounwind {
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; CHECK-NEXT:    leal 2(%esi), %esi
 ; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
-; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi ## 4-byte Reload
+; CHECK-NEXT:    movl (%esp), %esi ## 4-byte Reload
 ; CHECK-NEXT:    addl %esi, %ecx
-; CHECK-NEXT:    xorl %ebx, %ebx
-; CHECK-NEXT:    xorl %ebp, %ebp
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    xorl %edi, %edi
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB0_9: ## %bb13
 ; CHECK-NEXT:    ## =>This Loop Header: Depth=1
 ; CHECK-NEXT:    ## Child Loop BB0_10 Depth 2
-; CHECK-NEXT:    movl %ebp, %esi
-; CHECK-NEXT:    shrl $31, %esi
-; CHECK-NEXT:    addl %ebp, %esi
-; CHECK-NEXT:    andl $-2, %esi
-; CHECK-NEXT:    movl %ebp, %edi
-; CHECK-NEXT:    subl %esi, %edi
-; CHECK-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
-; CHECK-NEXT:    addl %ebx, %edi
+; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    andl $1, %edi
+; CHECK-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
+; CHECK-NEXT:    addl %esi, %edi
 ; CHECK-NEXT:    imull {{[0-9]+}}(%esp), %edi
 ; CHECK-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi ## 4-byte Folded Reload
 ; CHECK-NEXT:    xorl %esi, %esi
@@ -120,67 +115,67 @@ define void @foo(i8* %r, i32 %s, i32 %w, i32 %x, i8* %j, i32 %d) nounwind {
 ; CHECK-NEXT:    jl LBB0_10
 ; CHECK-NEXT:  ## %bb.11: ## %bb17
 ; CHECK-NEXT:    ## in Loop: Header=BB0_9 Depth=1
-; CHECK-NEXT:    incl %ebp
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi ## 4-byte Reload
+; CHECK-NEXT:    incl %edi
 ; CHECK-NEXT:    addl %edx, %eax
-; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx ## 4-byte Reload
-; CHECK-NEXT:    addl $2, %ebx
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi ## 4-byte Reload
+; CHECK-NEXT:    addl $2, %esi
 ; CHECK-NEXT:    addl %edx, %ecx
-; CHECK-NEXT:    cmpl {{[-0-9]+}}(%e{{[sb]}}p), %ebp ## 4-byte Folded Reload
+; CHECK-NEXT:    cmpl {{[-0-9]+}}(%e{{[sb]}}p), %edi ## 4-byte Folded Reload
 ; CHECK-NEXT:    jl LBB0_9
 ; CHECK-NEXT:  LBB0_13: ## %bb20
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK-NEXT:    cmpl $1, %edx
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    cmpl $1, %eax
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; CHECK-NEXT:    je LBB0_19
 ; CHECK-NEXT:  ## %bb.14: ## %bb20
-; CHECK-NEXT:    cmpl $3, %edx
+; CHECK-NEXT:    cmpl $3, %eax
 ; CHECK-NEXT:    jne LBB0_24
 ; CHECK-NEXT:  ## %bb.15: ## %bb22
 ; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi ## 4-byte Reload
 ; CHECK-NEXT:    addl %esi, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Folded Spill
-; CHECK-NEXT:    testl %ebx, %ebx
+; CHECK-NEXT:    testl %edi, %edi
 ; CHECK-NEXT:    jle LBB0_18
 ; CHECK-NEXT:  ## %bb.16: ## %bb.nph
-; CHECK-NEXT:    leal 15(%ebx), %eax
+; CHECK-NEXT:    leal 15(%edi), %eax
 ; CHECK-NEXT:    andl $-16, %eax
 ; CHECK-NEXT:    imull {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT:    addl %eax, %ebp
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT:    addl $15, %eax
+; CHECK-NEXT:    addl %eax, %ecx
+; CHECK-NEXT:    leal 15(%ebp), %eax
 ; CHECK-NEXT:    andl $-16, %eax
 ; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
 ; CHECK-NEXT:    addl %esi, %esi
-; CHECK-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %esi ## 4-byte Folded Reload
+; CHECK-NEXT:    addl (%esp), %esi ## 4-byte Folded Reload
 ; CHECK-NEXT:    addl {{[0-9]+}}(%esp), %esi
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB0_17: ## %bb23
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    subl $4, %esp
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; CHECK-NEXT:    pushl %edi
 ; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    pushl %ecx
 ; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    movl %ecx, %ebx
 ; CHECK-NEXT:    calll _memcpy
+; CHECK-NEXT:    movl %ebx, %ecx
 ; CHECK-NEXT:    addl $16, %esp
-; CHECK-NEXT:    addl %edi, %esi
-; CHECK-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp ## 4-byte Folded Reload
-; CHECK-NEXT:    decl %ebx
+; CHECK-NEXT:    addl %ebp, %esi
+; CHECK-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx ## 4-byte Folded Reload
+; CHECK-NEXT:    decl %edi
 ; CHECK-NEXT:    jne LBB0_17
 ; CHECK-NEXT:  LBB0_18: ## %bb26
-; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx ## 4-byte Reload
-; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi ## 4-byte Reload
-; CHECK-NEXT:    addl %ecx, %esi
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK-NEXT:    addl %esi, %edx
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax ## 4-byte Reload
+; CHECK-NEXT:    movl (%esp), %edx ## 4-byte Reload
+; CHECK-NEXT:    addl %edx, %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    addl %eax, %ecx
 ; CHECK-NEXT:    jmp LBB0_23
 ; CHECK-NEXT:  LBB0_19: ## %bb29
-; CHECK-NEXT:    testl %ebx, %ebx
+; CHECK-NEXT:    testl %edi, %edi
 ; CHECK-NEXT:    jle LBB0_22
 ; CHECK-NEXT:  ## %bb.20: ## %bb.nph11
 ; CHECK-NEXT:    movl %edi, %esi
-; CHECK-NEXT:    leal 15(%edi), %eax
+; CHECK-NEXT:    leal 15(%ebp), %eax
 ; CHECK-NEXT:    andl $-16, %eax
 ; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
@@ -188,28 +183,30 @@ define void @foo(i8* %r, i32 %s, i32 %w, i32 %x, i8* %j, i32 %d) nounwind {
 ; CHECK-NEXT:  LBB0_21: ## %bb30
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    subl $4, %esp
-; CHECK-NEXT:    pushl %esi
 ; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    pushl %ecx
 ; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    movl %ecx, %ebx
 ; CHECK-NEXT:    calll _memcpy
+; CHECK-NEXT:    movl %ebx, %ecx
 ; CHECK-NEXT:    addl $16, %esp
-; CHECK-NEXT:    addl %esi, %edi
-; CHECK-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp ## 4-byte Folded Reload
-; CHECK-NEXT:    decl %ebx
+; CHECK-NEXT:    addl %ebp, %edi
+; CHECK-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx ## 4-byte Folded Reload
+; CHECK-NEXT:    decl %esi
 ; CHECK-NEXT:    jne LBB0_21
 ; CHECK-NEXT:  LBB0_22: ## %bb33
-; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx ## 4-byte Reload
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK-NEXT:    addl %ecx, %edx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl (%esp), %edx ## 4-byte Reload
+; CHECK-NEXT:    addl %edx, %ecx
 ; CHECK-NEXT:  LBB0_23: ## %bb33
-; CHECK-NEXT:    movl %ecx, %eax
+; CHECK-NEXT:    movl %edx, %eax
 ; CHECK-NEXT:    shrl $31, %eax
-; CHECK-NEXT:    addl %ecx, %eax
+; CHECK-NEXT:    addl %edx, %eax
 ; CHECK-NEXT:    sarl %eax
 ; CHECK-NEXT:    subl $4, %esp
 ; CHECK-NEXT:    pushl %eax
 ; CHECK-NEXT:    pushl $128
-; CHECK-NEXT:    pushl %edx
+; CHECK-NEXT:    pushl %ecx
 ; CHECK-NEXT:    calll _memset
 ; CHECK-NEXT:    addl $44, %esp
 ; CHECK-NEXT:  LBB0_25: ## %return
diff --git a/llvm/test/CodeGen/X86/packss.ll b/llvm/test/CodeGen/X86/packss.ll
index 9a4025ab75e4b..16349ae2c7f9b 100644
--- a/llvm/test/CodeGen/X86/packss.ll
+++ b/llvm/test/CodeGen/X86/packss.ll
@@ -356,18 +356,12 @@ define <32 x i8> @packsswb_icmp_zero_trunc_256(<16 x i16> %a0) {
 ;
 ; AVX1-LABEL: packsswb_icmp_zero_trunc_256:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = zero,zero,ymm0[0,1]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    ret{{[l|q]}}
 ;
@@ -375,10 +369,8 @@ define <32 x i8> @packsswb_icmp_zero_trunc_256(<16 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm0 = zero,zero,ymm0[0,1]
-; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpacksswb %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,3]
 ; AVX2-NEXT:    ret{{[l|q]}}
   %1 = icmp eq <16 x i16> %a0, zeroinitializer
   %2 = sext <16 x i1> %1 to <16 x i16>
diff --git a/llvm/test/CodeGen/X86/phaddsub-extract.ll b/llvm/test/CodeGen/X86/phaddsub-extract.ll
index b7af19b7b1e4e..dd258c5f424a4 100644
--- a/llvm/test/CodeGen/X86/phaddsub-extract.ll
+++ b/llvm/test/CodeGen/X86/phaddsub-extract.ll
@@ -2094,6 +2094,28 @@ define i32 @hadd32_4_optsize(<4 x i32> %x225) optsize {
   ret i32 %x230
 }
 
+define i32 @hadd32_4_pgso(<4 x i32> %x225) !prof !14 {
+; SSE3-LABEL: hadd32_4_pgso:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    phaddd %xmm0, %xmm0
+; SSE3-NEXT:    phaddd %xmm0, %xmm0
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: hadd32_4_pgso:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    retq
+  %x226 = shufflevector <4 x i32> %x225, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %x227 = add <4 x i32> %x225, %x226
+  %x228 = shufflevector <4 x i32> %x227, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %x229 = add <4 x i32> %x227, %x228
+  %x230 = extractelement <4 x i32> %x229, i32 0
+  ret i32 %x230
+}
+
 define i32 @hadd32_8_optsize(<8 x i32> %x225) optsize {
 ; SSE3-LABEL: hadd32_8_optsize:
 ; SSE3:       # %bb.0:
@@ -2141,3 +2163,20 @@ define i32 @hadd32_16_optsize(<16 x i32> %x225) optsize {
   %x230 = extractelement <16 x i32> %x229, i32 0
   ret i32 %x230
 }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/CodeGen/X86/pr33010.ll b/llvm/test/CodeGen/X86/pr33010.ll
index 5b311e5c612df..0d8a4a494753c 100644
--- a/llvm/test/CodeGen/X86/pr33010.ll
+++ b/llvm/test/CodeGen/X86/pr33010.ll
@@ -32,9 +32,9 @@ define i32 addrspace(1)* @test(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i1 %w
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
 entry:
-  %tok = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @f, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %a, i32 addrspace(1)* %b)
-  %a.r = tail call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok, i32 7, i32 7) ; (%a, %a)
-  %b.r = tail call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok, i32 8, i32 8) ; (%b, %b)
+  %tok = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @f, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %a, i32 addrspace(1)* %b)]
+  %a.r = tail call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok, i32 0, i32 0) ; (%a, %a)
+  %b.r = tail call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok, i32 1, i32 1) ; (%b, %b)
   %cond.v = select i1 %which, i8 addrspace(1)* %a.r, i8 addrspace(1)* %b.r
   %cond = bitcast i8 addrspace(1)* %cond.v to i32 addrspace(1)*
   ret i32 addrspace(1)* %cond
diff --git a/llvm/test/CodeGen/X86/pr45443.ll b/llvm/test/CodeGen/X86/pr45443.ll
index 081451a2758cb..1e40ab94e9ca4 100644
--- a/llvm/test/CodeGen/X86/pr45443.ll
+++ b/llvm/test/CodeGen/X86/pr45443.ll
@@ -3,29 +3,10 @@
 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,X64
 
 define <16 x float> @PR45443() {
-; X86-LABEL: PR45443:
-; X86:       # %bb.0: # %bb
-; X86-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080]
-; X86-NEXT:    vfmadd231ps {{.*#+}} zmm0 = (zmm0 * mem) + zmm0
-; X86-NEXT:    vpcmpltud {{\.LCPI.*}}{1to16}, %zmm1, %k1
-; X86-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [16777215,16777215,16777215,16777215,16777215,16777215,16777215,16777215]
-; X86-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; X86-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
-; X86-NEXT:    vptestmd %zmm1, %zmm1, %k1 {%k1}
-; X86-NEXT:    vbroadcastss {{\.LCPI.*}}, %zmm0 {%k1}
-; X86-NEXT:    retl
-;
-; X64-LABEL: PR45443:
-; X64:       # %bb.0: # %bb
-; X64-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080,2181038080]
-; X64-NEXT:    vfmadd231ps {{.*#+}} zmm0 = (zmm0 * mem) + zmm0
-; X64-NEXT:    vpcmpltud {{.*}}(%rip){1to16}, %zmm1, %k1
-; X64-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [16777215,16777215,16777215,16777215,16777215,16777215,16777215,16777215]
-; X64-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; X64-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
-; X64-NEXT:    vptestmd %zmm1, %zmm1, %k1 {%k1}
-; X64-NEXT:    vbroadcastss {{.*}}(%rip), %zmm0 {%k1}
-; X64-NEXT:    retq
+; CHECK-LABEL: PR45443:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    vfmadd231ps {{.*#+}} zmm0 = (zmm0 * mem) + zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
 bb:
   %tmp = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> <i32 1090519040, i32 1090519040, i32 1090519040, i32 1090519040, i32 1090519040, i32 1090519040, i32 1090519040, i32 1090519040, i32 1090519040, i32 1090519040, i32 1090519040, i32 1090519040, i32 1090519040, i32 1090519040, i32 1090519040, i32 1090519040>, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>)
   %tmp4 = tail call fast <16 x float> @llvm.fma.v16f32(<16 x float> undef, <16 x float> <float 0x3FE6300000000000, float 0x3FE6300000000000, float 0x3FE6300000000000, float 0x3FE6300000000000, float 0x3FE6300000000000, float 0x3FE6300000000000, float 0x3FE6300000000000, float 0x3FE6300000000000, float 0x3FE6300000000000, float 0x3FE6300000000000, float 0x3FE6300000000000, float 0x3FE6300000000000, float 0x3FE6300000000000, float 0x3FE6300000000000, float 0x3FE6300000000000, float 0x3FE6300000000000>, <16 x float> undef)
diff --git a/llvm/test/CodeGen/X86/pr46527.ll b/llvm/test/CodeGen/X86/pr46527.ll
new file mode 100644
index 0000000000000..1d3f16f8c1aeb
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr46527.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+;RUN: llc < %s -mtriple=i686-unknown -mattr=sse2 -relocation-model=pic | FileCheck %s
+
+define void @f(<16 x i8>* %out, <16 x i8> %in, i1 %flag) {
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    calll .L0$pb
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 4
+; CHECK-NEXT:  .L0$pb:
+; CHECK-NEXT:    popl %eax
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -4
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp0-.L0$pb), %eax
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; CHECK-NEXT:    notb %dl
+; CHECK-NEXT:    andb $1, %dl
+; CHECK-NEXT:    movzbl %dl, %edx
+; CHECK-NEXT:    movd %edx, %xmm1
+; CHECK-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; CHECK-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; CHECK-NEXT:    paddb %xmm1, %xmm1
+; CHECK-NEXT:    pxor %xmm0, %xmm1
+; CHECK-NEXT:    pxor {{\.LCPI.*}}@GOTOFF(%eax), %xmm1
+; CHECK-NEXT:    movdqa %xmm1, (%ecx)
+; CHECK-NEXT:    retl
+entry:
+  %0 = select i1 %flag, i8 0, i8 2
+  %1 = insertelement <16 x i8> undef, i8 %0, i32 0
+  %2 = shufflevector <16 x i8> %1, <16 x i8> undef, <16 x i32> zeroinitializer
+  %3 = xor <16 x i8> %2, %in
+  %4 = xor <16 x i8> %3, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  store <16 x i8> %4, <16 x i8>* %out, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/pr46532.ll b/llvm/test/CodeGen/X86/pr46532.ll
new file mode 100644
index 0000000000000..3e0638b24f35a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr46532.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -mcpu=haswell | FileCheck %s
+
+define void @WhileWithLoopInvariantOperation.21() {
+; CHECK-LABEL: WhileWithLoopInvariantOperation.21:
+; CHECK:       # %bb.0: # %while.1.body.preheader
+; CHECK-NEXT:    movq (%rax), %rax
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vmovaps %xmm0, 32(%rax)
+; CHECK-NEXT:    vmovaps {{.*#+}} ymm0 = [4294967295,4294967295,0,0,0,0,0,0]
+; CHECK-NEXT:    vmaskmovps %ymm0, %ymm0, (%rax)
+while.1.body.preheader:
+  %0 = load i8*, i8** undef, align 8, !invariant.load !0, !dereferenceable !1, !align !2
+  %1 = getelementptr inbounds i8, i8* %0, i64 32
+  tail call void @llvm.memset.p0i8.i64(i8* nonnull align 16 dereferenceable(16) %1, i8 0, i64 16, i1 false)
+  %2 = tail call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> <i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>, <8 x float> <float undef, float undef, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>)
+  %3 = tail call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* undef, i32 4, <16 x i1> <i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>, <16 x float> <float undef, float undef, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>)
+  tail call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> undef, <16 x float>* nonnull undef, i32 4, <16 x i1> <i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>)
+  unreachable
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg)
+
+declare <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>*, i32 immarg, <8 x i1>, <8 x float>)
+
+declare <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>*, i32 immarg, <16 x i1>, <16 x float>)
+
+declare void @llvm.masked.store.v16f32.p0v16f32(<16 x float>, <16 x float>*, i32 immarg, <16 x i1>)
+
+!0 = !{}
+!1 = !{i64 65}
+!2 = !{i64 16}
+
diff --git a/llvm/test/CodeGen/X86/pr46585.ll b/llvm/test/CodeGen/X86/pr46585.ll
new file mode 100644
index 0000000000000..7bea63176d1bb
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr46585.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-macosx | FileCheck %s
+
+@global = external local_unnamed_addr global i8
+@global.1 = external local_unnamed_addr global i64
+
+define void @spam() local_unnamed_addr {
+; CHECK-LABEL: spam:
+; CHECK:       ## %bb.0: ## %bb
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    je LBB0_2
+; CHECK-NEXT:  ## %bb.1: ## %bb9
+; CHECK-NEXT:    movq _global.1@{{.*}}(%rip), %rax
+; CHECK-NEXT:    movq $1, (%rax)
+; CHECK-NEXT:  LBB0_2: ## %bb8
+; CHECK-NEXT:    retq
+bb:
+  %tmp = load i8, i8* @global, align 1
+  %tmp1 = and i8 %tmp, 1
+  %tmp2 = insertelement <8 x i8> undef, i8 %tmp1, i32 0
+  %tmp3 = ashr <8 x i8> %tmp2, <i8 7, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+  %tmp4 = shufflevector <8 x i8> %tmp3, <8 x i8> undef, <8 x i32> zeroinitializer
+  %tmp5 = icmp slt <8 x i8> %tmp4, zeroinitializer
+  %tmp6 = bitcast <8 x i1> %tmp5 to i8
+  %tmp7 = icmp eq i8 %tmp6, 0
+  br i1 %tmp7, label %bb8, label %bb9
+
+bb8:                                              ; preds = %bb9, %bb
+  ret void
+
+bb9:                                              ; preds = %bb
+  store i64 1, i64* @global.1, align 8
+  br label %bb8
+}
diff --git a/llvm/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll b/llvm/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll
index 464723172dad2..0706254f4e5c0 100644
--- a/llvm/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll
+++ b/llvm/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll
@@ -148,18 +148,17 @@ define <32 x i1> @shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0
 ; AVX256VL-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
 ; AVX256VL-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
 ; AVX256VL-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3],ymm2[4,5],ymm1[6],ymm2[7]
-; AVX256VL-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[6,7,12,13,u,u,8,9,6,7,14,15,14,15,0,1,22,23,28,29,18,19,26,27,22,23,u,u,30,31,16,17]
+; AVX256VL-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[6,7,12,13],zero,zero,ymm1[8,9,6,7,14,15,14,15,0,1,22,23,28,29,18,19,26,27,22,23],zero,zero,ymm1[30,31,16,17]
 ; AVX256VL-NEXT:    vmovdqa32 %ymm0, %ymm2 {%k1} {z}
 ; AVX256VL-NEXT:    vpmovdw %ymm2, %xmm2
 ; AVX256VL-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
 ; AVX256VL-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
 ; AVX256VL-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,1,2,1]
-; AVX256VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,255,255,255,255]
-; AVX256VL-NEXT:    vpblendvb %ymm3, %ymm1, %ymm2, %ymm1
-; AVX256VL-NEXT:    vpmovsxwd %xmm1, %ymm2
-; AVX256VL-NEXT:    vpslld $31, %ymm2, %ymm2
-; AVX256VL-NEXT:    vptestmd %ymm2, %ymm2, %k1
-; AVX256VL-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX256VL-NEXT:    vpternlogq $220, {{.*}}(%rip), %ymm1, %ymm2
+; AVX256VL-NEXT:    vpmovsxwd %xmm2, %ymm1
+; AVX256VL-NEXT:    vpslld $31, %ymm1, %ymm1
+; AVX256VL-NEXT:    vptestmd %ymm1, %ymm1, %k1
+; AVX256VL-NEXT:    vextracti128 $1, %ymm2, %xmm1
 ; AVX256VL-NEXT:    vpmovsxwd %xmm1, %ymm1
 ; AVX256VL-NEXT:    vpslld $31, %ymm1, %ymm1
 ; AVX256VL-NEXT:    vptestmd %ymm1, %ymm1, %k0
diff --git a/llvm/test/CodeGen/X86/prefetch.ll b/llvm/test/CodeGen/X86/prefetch.ll
index 839948174a43e..f00788145f4e0 100644
--- a/llvm/test/CodeGen/X86/prefetch.ll
+++ b/llvm/test/CodeGen/X86/prefetch.ll
@@ -10,7 +10,7 @@
 ; RUN: llc < %s -mtriple=i686-- -mattr=-sse,+prefetchwt1 | FileCheck %s -check-prefix=PREFETCHWT1
 ; RUN: llc < %s -mtriple=i686-- -mattr=-sse,+3dnow,+prefetchwt1 | FileCheck %s -check-prefix=PREFETCHWT1
 ; RUN: llc < %s -mtriple=i686-- -mattr=+3dnow | FileCheck %s -check-prefix=3DNOW
-; RUN: llc < %s -mtriple=i686-- -mattr=+3dnow,+prfchw | FileCheck %s -check-prefix=PRFCHW3DNOW
+; RUN: llc < %s -mtriple=i686-- -mattr=+3dnow,+prfchw | FileCheck %s -check-prefix=3DNOW
 
 ; Rules:
 ; 3dnow by itself get you just the single prefetch instruction with no hints
@@ -68,24 +68,11 @@ define void @t(i8* %ptr) nounwind  {
 ; 3DNOW-NEXT:    prefetch (%eax)
 ; 3DNOW-NEXT:    prefetch (%eax)
 ; 3DNOW-NEXT:    prefetch (%eax)
-; 3DNOW-NEXT:    prefetch (%eax)
-; 3DNOW-NEXT:    prefetch (%eax)
-; 3DNOW-NEXT:    prefetch (%eax)
-; 3DNOW-NEXT:    prefetch (%eax)
+; 3DNOW-NEXT:    prefetchw (%eax)
+; 3DNOW-NEXT:    prefetchw (%eax)
+; 3DNOW-NEXT:    prefetchw (%eax)
+; 3DNOW-NEXT:    prefetchw (%eax)
 ; 3DNOW-NEXT:    retl
-;
-; PRFCHW3DNOW-LABEL: t:
-; PRFCHW3DNOW:       # %bb.0: # %entry
-; PRFCHW3DNOW-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; PRFCHW3DNOW-NEXT:    prefetch (%eax)
-; PRFCHW3DNOW-NEXT:    prefetch (%eax)
-; PRFCHW3DNOW-NEXT:    prefetch (%eax)
-; PRFCHW3DNOW-NEXT:    prefetch (%eax)
-; PRFCHW3DNOW-NEXT:    prefetchw (%eax)
-; PRFCHW3DNOW-NEXT:    prefetchw (%eax)
-; PRFCHW3DNOW-NEXT:    prefetchw (%eax)
-; PRFCHW3DNOW-NEXT:    prefetchw (%eax)
-; PRFCHW3DNOW-NEXT:    retl
 entry:
 	tail call void @llvm.prefetch( i8* %ptr, i32 0, i32 1, i32 1 )
 	tail call void @llvm.prefetch( i8* %ptr, i32 0, i32 2, i32 1 )
diff --git a/llvm/test/CodeGen/X86/promote-cmp.ll b/llvm/test/CodeGen/X86/promote-cmp.ll
index ce3c7d1cb8acb..f23900e8b8f8c 100644
--- a/llvm/test/CodeGen/X86/promote-cmp.ll
+++ b/llvm/test/CodeGen/X86/promote-cmp.ll
@@ -30,19 +30,20 @@ define <4 x i64> @PR45808(<4 x i64> %0, <4 x i64> %1) {
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
 ; SSE2-NEXT:    por %xmm4, %xmm5
 ; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,2],xmm6[0,2]
-; SSE2-NEXT:    movaps {{.*#+}} xmm4 = <1,1,u,u>
-; SSE2-NEXT:    xorps %xmm5, %xmm4
-; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE2-NEXT:    movaps %xmm5, %xmm6
+; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[2,1],xmm5[3,3]
+; SSE2-NEXT:    psllq $63, %xmm6
+; SSE2-NEXT:    psrad $31, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pandn %xmm3, %xmm6
+; SSE2-NEXT:    por %xmm6, %xmm1
+; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1,1,3]
+; SSE2-NEXT:    xorps %xmm4, %xmm5
 ; SSE2-NEXT:    psllq $63, %xmm5
 ; SSE2-NEXT:    psrad $31, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pandn %xmm3, %xmm5
-; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,1,1,3]
-; SSE2-NEXT:    psllq $63, %xmm3
-; SSE2-NEXT:    psrad $31, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm3, %xmm0
 ; SSE2-NEXT:    pandn %xmm2, %xmm3
 ; SSE2-NEXT:    por %xmm3, %xmm0
@@ -56,10 +57,11 @@ define <4 x i64> @PR45808(<4 x i64> %0, <4 x i64> %1) {
 ; SSE4-NEXT:    movdqa %xmm4, %xmm5
 ; SSE4-NEXT:    pcmpgtq %xmm2, %xmm5
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
-; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm5
+; SSE4-NEXT:    pcmpeqd %xmm6, %xmm6
+; SSE4-NEXT:    pxor %xmm5, %xmm6
 ; SSE4-NEXT:    psllq $63, %xmm0
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
-; SSE4-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm5[0],zero,xmm5[1],zero
+; SSE4-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm6[0],zero,xmm6[1],zero
 ; SSE4-NEXT:    psllq $63, %xmm0
 ; SSE4-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
 ; SSE4-NEXT:    movapd %xmm2, %xmm0
@@ -72,9 +74,8 @@ define <4 x i64> @PR45808(<4 x i64> %0, <4 x i64> %1) {
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX1-NEXT:    vxorpd {{.*}}(%rip), %ymm2, %ymm2
 ; AVX1-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -82,7 +83,6 @@ define <4 x i64> @PR45808(<4 x i64> %0, <4 x i64> %1) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm2
 ; AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm2, %ymm2
-; AVX2-NEXT:    vpsllq $63, %ymm2, %ymm2
 ; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
   %3 = icmp sgt <4 x i64> %0, %1
diff --git a/llvm/test/CodeGen/X86/psubus.ll b/llvm/test/CodeGen/X86/psubus.ll
index 6e01c606745fc..28ab2e1968dbe 100644
--- a/llvm/test/CodeGen/X86/psubus.ll
+++ b/llvm/test/CodeGen/X86/psubus.ll
@@ -1732,9 +1732,9 @@ define <8 x i16> @psubus_8i64_max(<8 x i16> %x, <8 x i64> %y) nounwind {
 ; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm5, %ymm3
 ; AVX2-NEXT:    vblendvpd %ymm3, %ymm1, %ymm6, %ymm1
 ; AVX2-NEXT:    vpackusdw %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
 ; AVX2-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/sadd_sat_vec.ll b/llvm/test/CodeGen/X86/sadd_sat_vec.ll
index 3096d4fb94729..f5fbd3915c125 100644
--- a/llvm/test/CodeGen/X86/sadd_sat_vec.ll
+++ b/llvm/test/CodeGen/X86/sadd_sat_vec.ll
@@ -514,20 +514,64 @@ define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 ; SSE-NEXT:    psubb %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: v16i4:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpsllw $4, %xmm1, %xmm1
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: v16i4:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v16i4:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllw $4, %xmm1, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: v16i4:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsllw $4, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512F-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: v16i4:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsllw $4, %xmm1, %xmm1
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512BW-NEXT:    vpternlogq $108, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512BW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
   %z = call <16 x i4> @llvm.sadd.sat.v16i4(<16 x i4> %x, <16 x i4> %y)
   ret <16 x i4> %z
 }
diff --git a/llvm/test/CodeGen/X86/setcc-logic.ll b/llvm/test/CodeGen/X86/setcc-logic.ll
index 46db9271d31b0..59e0efc18c87f 100644
--- a/llvm/test/CodeGen/X86/setcc-logic.ll
+++ b/llvm/test/CodeGen/X86/setcc-logic.ll
@@ -315,6 +315,33 @@ return:
   ret i32 192
 }
 
+; PR44565 - https://bugs.llvm.org/show_bug.cgi?id=44565
+
+define i32 @vec_extract_branch(<2 x double> %x)  {
+; CHECK-LABEL: vec_extract_branch:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorpd %xmm1, %xmm1
+; CHECK-NEXT:    cmpltpd %xmm0, %xmm1
+; CHECK-NEXT:    movmskpd %xmm1, %eax
+; CHECK-NEXT:    cmpb $3, %al
+; CHECK-NEXT:    jne .LBB16_2
+; CHECK-NEXT:  # %bb.1: # %true
+; CHECK-NEXT:    movl $42, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB16_2: # %false
+; CHECK-NEXT:    movl $88, %eax
+; CHECK-NEXT:    retq
+  %t1 = fcmp ogt <2 x double> %x, zeroinitializer
+  %t2 = extractelement <2 x i1> %t1, i32 0
+  %t3 = extractelement <2 x i1> %t1, i32 1
+  %t4 = and i1 %t2, %t3
+  br i1 %t4, label %true, label %false
+true:
+  ret i32 42
+false:
+  ret i32 88
+}
+
 define <4 x i1> @all_bits_clear_vec(<4 x i32> %P, <4 x i32> %Q) nounwind {
 ; CHECK-LABEL: all_bits_clear_vec:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/setcc-lowering.ll b/llvm/test/CodeGen/X86/setcc-lowering.ll
index aade54b0b02ed..34f3ad697ad52 100644
--- a/llvm/test/CodeGen/X86/setcc-lowering.ll
+++ b/llvm/test/CodeGen/X86/setcc-lowering.ll
@@ -16,8 +16,6 @@ define <8 x i16> @pr25080(<8 x i32> %a) {
 ; AVX-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpor {{.*}}(%rip), %xmm0, %xmm0
-; AVX-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX-NEXT:    vpsraw $15, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/shadow-stack.ll b/llvm/test/CodeGen/X86/shadow-stack.ll
index bb5461062dedd..7316ad80cda7e 100644
--- a/llvm/test/CodeGen/X86/shadow-stack.ll
+++ b/llvm/test/CodeGen/X86/shadow-stack.ll
@@ -38,7 +38,7 @@ define i32 @bar(i32 %i) local_unnamed_addr {
 ; X86_64-NEXT:    .cfi_offset %rbp, -16
 ; X86_64-NEXT:    movq _buf@{{.*}}(%rip), %rax
 ; X86_64-NEXT:    movq (%rax), %rax
-; X86_64-NEXT:    xorq %rdx, %rdx
+; X86_64-NEXT:    xorl %edx, %edx
 ; X86_64-NEXT:    rdsspq %rdx
 ; X86_64-NEXT:    testq %rdx, %rdx
 ; X86_64-NEXT:    je LBB0_5
diff --git a/llvm/test/CodeGen/X86/shrink-const.ll b/llvm/test/CodeGen/X86/shrink-const.ll
new file mode 100644
index 0000000000000..8502960f6e0d4
--- /dev/null
+++ b/llvm/test/CodeGen/X86/shrink-const.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX
+
+; If targetShrinkDemandedConstant extends xor/or constants ensure it extends from the msb of the active bits
+define <4 x i32> @sext_vector_constants(<4 x i32> %a0) {
+; SSE-LABEL: sext_vector_constants:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psrld $9, %xmm0
+; SSE-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE-NEXT:    pslld $26, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sext_vector_constants:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrld $9, %xmm0, %xmm0
+; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpslld $26, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = lshr <4 x i32> %a0, <i32 9, i32 9, i32 9, i32 9>
+  %2 = xor <4 x i32> %1, <i32 314523200, i32 -2085372448, i32 144496960, i32 1532773600>
+  %3 = shl <4 x i32> %2, <i32 26, i32 26, i32 26, i32 26>
+  ret <4 x i32> %3
+}
diff --git a/llvm/test/CodeGen/X86/shrinkwrap-callbr.ll b/llvm/test/CodeGen/X86/shrinkwrap-callbr.ll
new file mode 100644
index 0000000000000..ced7fdbf262d5
--- /dev/null
+++ b/llvm/test/CodeGen/X86/shrinkwrap-callbr.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -enable-shrink-wrap=true | FileCheck %s
+
+;; Ensure that shrink-wrapping understands that INLINEASM_BR may exit
+;; the block before the end, and you cannot simply place stack
+;; adjustment at the end of that block.
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i32 @fn()
+
+; Function Attrs: uwtable
+define i32 @test1(i32 %v) {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB0_3
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    callq fn
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    # jump to .Ltmp0
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:  # %bb.2: # %return
+; CHECK-NEXT:    movl $4, %eax
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB0_3: # %ret0
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .Ltmp0: # Block address taken
+; CHECK-NEXT:  .LBB0_4: # %two
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    jmp fn # TAILCALL
+entry:
+  %tobool = icmp eq i32 %v, 0
+  br i1 %tobool, label %ret0, label %if.end
+
+ret0:
+  ret i32 0
+
+if.end:
+  %call = tail call i32 @fn()
+  callbr void asm sideeffect "# jump to $0", "X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@test1, %two))
+          to label %return [label %two]
+
+two:
+  %call1 = tail call i32 @fn()
+  br label %return
+
+return:
+  %retval.1 = phi i32 [ %call1, %two ], [ 4, %if.end ]
+  ret i32 %retval.1
+}
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
index d8f1e54b959cc..6b49f22f21f1f 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
@@ -1345,10 +1345,11 @@ define <16 x i8> @negative(<32 x i8> %v, <32 x i8> %w) nounwind {
 ;
 ; AVX512VL-LABEL: negative:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
-; AVX512VL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
+; AVX512VL-NEXT:    vpshufb {{.*#+}} ymm0 = zero,ymm0[2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
+; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512VL-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX512VL-NEXT:    vpternlogq $206, %ymm1, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpermq {{.*#+}} ymm0 = ymm2[0,3,2,3]
 ; AVX512VL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/speculative-execution-side-effect-suppression.ll b/llvm/test/CodeGen/X86/speculative-execution-side-effect-suppression.ll
index acbdc9e9387e3..fdd56382448a4 100644
--- a/llvm/test/CodeGen/X86/speculative-execution-side-effect-suppression.ll
+++ b/llvm/test/CodeGen/X86/speculative-execution-side-effect-suppression.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -x86-seses-enable %s -o - | FileCheck %s
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -x86-seses-enable -x86-seses-one-lfence-per-bb %s -o - | FileCheck %s --check-prefix=X86-ONE-LFENCE
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -x86-seses-enable -x86-seses-omit-branch-lfences %s -o - | FileCheck %s --check-prefix=X86-OMIT-BR
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -x86-seses-enable -x86-seses-only-lfence-non-const %s -o - | FileCheck %s --check-prefix=X86-NON-CONST
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -x86-seses-enable-without-lvi-cfi %s -o - | FileCheck %s
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -x86-seses-enable-without-lvi-cfi -x86-seses-one-lfence-per-bb %s -o - | FileCheck %s --check-prefix=X86-ONE-LFENCE
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -x86-seses-enable-without-lvi-cfi -x86-seses-omit-branch-lfences %s -o - | FileCheck %s --check-prefix=X86-OMIT-BR
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -x86-seses-enable-without-lvi-cfi -x86-seses-only-lfence-non-const %s -o - | FileCheck %s --check-prefix=X86-NON-CONST
 
 define void @_Z4buzzv() {
 ; CHECK-LABEL: _Z4buzzv:
diff --git a/llvm/test/CodeGen/X86/sqrt-fastmath.ll b/llvm/test/CodeGen/X86/sqrt-fastmath.ll
index b2593bc43578b..29d21f6927bd0 100644
--- a/llvm/test/CodeGen/X86/sqrt-fastmath.ll
+++ b/llvm/test/CodeGen/X86/sqrt-fastmath.ll
@@ -10,7 +10,11 @@ declare float @llvm.sqrt.f32(float)
 declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)
 declare <8 x float> @llvm.sqrt.v8f32(<8 x float>)
 declare <16 x float> @llvm.sqrt.v16f32(<16 x float>)
+declare double @llvm.sqrt.f64(double)
 
+declare float @llvm.fabs.f32(float)
+declare <4 x float> @llvm.fabs.v4f32(<4 x float>)
+declare double @llvm.fabs.f64(double)
 
 define double @finite_f64_no_estimate(double %d) #0 {
 ; SSE-LABEL: finite_f64_no_estimate:
@@ -614,6 +618,182 @@ define <16 x float> @v16f32_estimate(<16 x float> %x) #1 {
   ret <16 x float> %div
 }
 
+; x / (fabs(y) * sqrt(z)) --> x * rsqrt(y*y*z)
+
+define float @div_sqrt_fabs_f32(float %x, float %y, float %z) {
+; SSE-LABEL: div_sqrt_fabs_f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    mulss %xmm1, %xmm1
+; SSE-NEXT:    mulss %xmm2, %xmm1
+; SSE-NEXT:    xorps %xmm2, %xmm2
+; SSE-NEXT:    rsqrtss %xmm1, %xmm2
+; SSE-NEXT:    mulss %xmm2, %xmm1
+; SSE-NEXT:    mulss %xmm2, %xmm1
+; SSE-NEXT:    addss {{.*}}(%rip), %xmm1
+; SSE-NEXT:    mulss {{.*}}(%rip), %xmm2
+; SSE-NEXT:    mulss %xmm0, %xmm2
+; SSE-NEXT:    mulss %xmm1, %xmm2
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: div_sqrt_fabs_f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmulss %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vmulss %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vrsqrtss %xmm1, %xmm1, %xmm2
+; AVX1-NEXT:    vmulss %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vmulss %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vaddss {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vmulss {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vmulss %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: div_sqrt_fabs_f32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmulss %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vrsqrtss %xmm1, %xmm1, %xmm2
+; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vfmadd213ss {{.*#+}} xmm1 = (xmm2 * xmm1) + mem
+; AVX512-NEXT:    vmulss {{.*}}(%rip), %xmm2, %xmm2
+; AVX512-NEXT:    vmulss %xmm0, %xmm2, %xmm0
+; AVX512-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %s = call fast float @llvm.sqrt.f32(float %z)
+  %a = call fast float @llvm.fabs.f32(float %y)
+  %m = fmul fast float %s, %a
+  %d = fdiv fast float %x, %m
+  ret float %d
+}
+
+; x / (fabs(y) * sqrt(z)) --> x * rsqrt(y*y*z)
+
+define <4 x float> @div_sqrt_fabs_v4f32(<4 x float> %x, <4 x float> %y, <4 x float> %z) {
+; SSE-LABEL: div_sqrt_fabs_v4f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    mulps %xmm1, %xmm1
+; SSE-NEXT:    mulps %xmm2, %xmm1
+; SSE-NEXT:    rsqrtps %xmm1, %xmm2
+; SSE-NEXT:    mulps %xmm2, %xmm1
+; SSE-NEXT:    mulps %xmm2, %xmm1
+; SSE-NEXT:    addps {{.*}}(%rip), %xmm1
+; SSE-NEXT:    mulps {{.*}}(%rip), %xmm2
+; SSE-NEXT:    mulps %xmm1, %xmm2
+; SSE-NEXT:    mulps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: div_sqrt_fabs_v4f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmulps %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vmulps %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vrsqrtps %xmm1, %xmm2
+; AVX1-NEXT:    vmulps %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vmulps %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vaddps {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vmulps {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vmulps %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: div_sqrt_fabs_v4f32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmulps %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vmulps %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vrsqrtps %xmm1, %xmm2
+; AVX512-NEXT:    vmulps %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm3 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]
+; AVX512-NEXT:    vfmadd231ps {{.*#+}} xmm3 = (xmm2 * xmm1) + xmm3
+; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; AVX512-NEXT:    vmulps %xmm1, %xmm2, %xmm1
+; AVX512-NEXT:    vmulps %xmm3, %xmm1, %xmm1
+; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+  %s = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %z)
+  %a = call <4 x float> @llvm.fabs.v4f32(<4 x float> %y)
+  %m = fmul reassoc <4 x float> %a, %s
+  %d = fdiv reassoc arcp <4 x float> %x, %m
+  ret <4 x float> %d
+}
+
+; This has 'arcp' but does not have 'reassoc' FMF.
+; We allow converting the sqrt to an estimate, but
+; do not pull the divisor into the estimate.
+; x / (fabs(y) * sqrt(z)) --> x * rsqrt(z) / fabs(y)
+
+define <4 x float> @div_sqrt_fabs_v4f32_fmf(<4 x float> %x, <4 x float> %y, <4 x float> %z) {
+; SSE-LABEL: div_sqrt_fabs_v4f32_fmf:
+; SSE:       # %bb.0:
+; SSE-NEXT:    rsqrtps %xmm2, %xmm3
+; SSE-NEXT:    mulps %xmm3, %xmm2
+; SSE-NEXT:    mulps %xmm3, %xmm2
+; SSE-NEXT:    addps {{.*}}(%rip), %xmm2
+; SSE-NEXT:    mulps {{.*}}(%rip), %xmm3
+; SSE-NEXT:    mulps %xmm2, %xmm3
+; SSE-NEXT:    andps {{.*}}(%rip), %xmm1
+; SSE-NEXT:    divps %xmm1, %xmm3
+; SSE-NEXT:    mulps %xmm3, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: div_sqrt_fabs_v4f32_fmf:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vrsqrtps %xmm2, %xmm3
+; AVX1-NEXT:    vmulps %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vmulps %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vaddps {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vmulps {{.*}}(%rip), %xmm3, %xmm3
+; AVX1-NEXT:    vmulps %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vandps {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vdivps %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: div_sqrt_fabs_v4f32_fmf:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrsqrtps %xmm2, %xmm3
+; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm4 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; AVX512-NEXT:    vmulps %xmm4, %xmm3, %xmm4
+; AVX512-NEXT:    vmulps %xmm3, %xmm2, %xmm2
+; AVX512-NEXT:    vmulps %xmm3, %xmm2, %xmm2
+; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm3 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]
+; AVX512-NEXT:    vaddps %xmm3, %xmm2, %xmm2
+; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN]
+; AVX512-NEXT:    vmulps %xmm2, %xmm4, %xmm2
+; AVX512-NEXT:    vandps %xmm3, %xmm1, %xmm1
+; AVX512-NEXT:    vdivps %xmm1, %xmm2, %xmm1
+; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+  %s = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %z)
+  %a = call <4 x float> @llvm.fabs.v4f32(<4 x float> %y)
+  %m = fmul <4 x float> %a, %s
+  %d = fdiv arcp <4 x float> %x, %m
+  ret <4 x float> %d
+}
+
+; No estimates for f64, so do not convert fabs into an fmul.
+
+define double @div_sqrt_fabs_f64(double %x, double %y, double %z) {
+; SSE-LABEL: div_sqrt_fabs_f64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    sqrtsd %xmm2, %xmm2
+; SSE-NEXT:    andpd {{.*}}(%rip), %xmm1
+; SSE-NEXT:    mulsd %xmm2, %xmm1
+; SSE-NEXT:    divsd %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: div_sqrt_fabs_f64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vsqrtsd %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vandpd {{.*}}(%rip), %xmm1, %xmm1
+; AVX-NEXT:    vmulsd %xmm1, %xmm2, %xmm1
+; AVX-NEXT:    vdivsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %s = call fast double @llvm.sqrt.f64(double %z)
+  %a = call fast double @llvm.fabs.f64(double %y)
+  %m = fmul fast double %s, %a
+  %d = fdiv fast double %x, %m
+  ret double %d
+}
 
 attributes #0 = { "unsafe-fp-math"="true" "reciprocal-estimates"="!sqrtf,!vec-sqrtf,!divf,!vec-divf" }
 attributes #1 = { "unsafe-fp-math"="true" "reciprocal-estimates"="sqrt,vec-sqrt" }
diff --git a/llvm/test/CodeGen/X86/ssub_sat_vec.ll b/llvm/test/CodeGen/X86/ssub_sat_vec.ll
index 59a18bfa85f6f..4126656fd8058 100644
--- a/llvm/test/CodeGen/X86/ssub_sat_vec.ll
+++ b/llvm/test/CodeGen/X86/ssub_sat_vec.ll
@@ -510,20 +510,64 @@ define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 ; SSE-NEXT:    psubb %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: v16i4:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpsllw $4, %xmm1, %xmm1
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: v16i4:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v16i4:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllw $4, %xmm1, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: v16i4:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsllw $4, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512F-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: v16i4:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsllw $4, %xmm1, %xmm1
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512BW-NEXT:    vpternlogq $108, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512BW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
   %z = call <16 x i4> @llvm.ssub.sat.v16i4(<16 x i4> %x, <16 x i4> %y)
   ret <16 x i4> %z
 }
diff --git a/llvm/test/CodeGen/X86/stack-clash-dynamic-alloca.ll b/llvm/test/CodeGen/X86/stack-clash-dynamic-alloca.ll
index 928b7a9cba024..bc46785640831 100644
--- a/llvm/test/CodeGen/X86/stack-clash-dynamic-alloca.ll
+++ b/llvm/test/CodeGen/X86/stack-clash-dynamic-alloca.ll
@@ -23,12 +23,12 @@ attributes #0 =  {"probe-stack"="inline-asm"}
 ; CHECK-X86-64-NEXT:  	leaq 15(,%rcx,4), %rcx
 ; CHECK-X86-64-NEXT:  	andq	$-16, %rcx
 ; CHECK-X86-64-NEXT:  	subq	%rcx, %rax
-; CHECK-X86-64-NEXT:  	cmpq	%rax, %rsp
+; CHECK-X86-64-NEXT:  	cmpq	%rsp, %rax
 ; CHECK-X86-64-NEXT:  	jl	.LBB0_3
 ; CHECK-X86-64-NEXT:  .LBB0_2: # =>This Inner Loop Header: Depth=1
 ; CHECK-X86-64-NEXT:  	movq	$0, (%rsp)
 ; CHECK-X86-64-NEXT:  	subq	$4096, %rsp # imm = 0x1000
-; CHECK-X86-64-NEXT:  	cmpq	%rax, %rsp
+; CHECK-X86-64-NEXT:  	cmpq	%rsp, %rax
 ; CHECK-X86-64-NEXT:  	jge	.LBB0_2
 ; CHECK-X86-64-NEXT:  .LBB0_3:
 ; CHECK-X86-64-NEXT:  	movq	%rax, %rsp
@@ -53,12 +53,12 @@ attributes #0 =  {"probe-stack"="inline-asm"}
 ; CHECK-X86-32-NEXT:    leal    15(,%ecx,4), %ecx
 ; CHECK-X86-32-NEXT:    andl    $-16, %ecx
 ; CHECK-X86-32-NEXT:    subl    %ecx, %eax
-; CHECK-X86-32-NEXT:    cmpl    %eax, %esp
+; CHECK-X86-32-NEXT:    cmpl    %esp, %eax
 ; CHECK-X86-32-NEXT:    jl  .LBB0_3
 ; CHECK-X86-32-NEXT:  .LBB0_2: # =>This Inner Loop Header: Depth=1
 ; CHECK-X86-32-NEXT:    movl    $0, (%esp)
 ; CHECK-X86-32-NEXT:    subl    $4096, %esp # imm = 0x1000
-; CHECK-X86-32-NEXT:    cmpl    %eax, %esp
+; CHECK-X86-32-NEXT:    cmpl    %esp, %eax
 ; CHECK-X86-32-NEXT:    jge .LBB0_2
 ; CHECK-X86-32-NEXT:  .LBB0_3:
 ; CHECK-X86-32-NEXT:    movl    %eax, %esp
diff --git a/llvm/test/CodeGen/X86/stack-clash-large.ll b/llvm/test/CodeGen/X86/stack-clash-large.ll
index ccc52f213e377..dd53cd8f69646 100644
--- a/llvm/test/CodeGen/X86/stack-clash-large.ll
+++ b/llvm/test/CodeGen/X86/stack-clash-large.ll
@@ -22,9 +22,9 @@ attributes #0 =  {"probe-stack"="inline-asm"}
 ; CHECK-X86-64-NEXT:	subq	$4096, %rsp # imm = 0x1000
 ; CHECK-X86-64-NEXT:	movq	$0, (%rsp)
 ; CHECK-X86-64-NEXT:	cmpq	%r11, %rsp
-; CHECK-X86-64-NEXT:	jl	.LBB0_1
+; CHECK-X86-64-NEXT:	jne	.LBB0_1
 ; CHECK-X86-64-NEXT:# %bb.2:
-; CHECK-X86-64-NEXT:	movq    %r11, %rsp
+; CHECK-X86-64-NEXT:	subq    $2248, %rsp
 ; CHECK-X86-64-NEXT:	.cfi_def_cfa_offset 71888
 ; CHECK-X86-64-NEXT:	movl	$1, 264(%rsp)
 ; CHECK-X86-64-NEXT:	movl	$1, 28664(%rsp)
@@ -41,9 +41,9 @@ attributes #0 =  {"probe-stack"="inline-asm"}
 ; CHECK-X86-32-NEXT:    subl    $4096, %esp # imm = 0x1000
 ; CHECK-X86-32-NEXT:    movl    $0, (%esp)
 ; CHECK-X86-32-NEXT:    cmpl    %r11d, %esp
-; CHECK-X86-32-NEXT:    jl  .LBB0_1
+; CHECK-X86-32-NEXT:    jne  .LBB0_1
 ; CHECK-X86-32-NEXT:# %bb.2:
-; CHECK-X86-32-NEXT:    movl    %r11d, %esp
+; CHECK-X86-32-NEXT:    subl    $2380, %esp
 ; CHECK-X86-32-NEXT:    .cfi_def_cfa_offset 72016
 ; CHECK-X86-32-NEXT:    movl    $1, 392(%esp)
 ; CHECK-X86-32-NEXT:    movl    $1, 28792(%esp)
diff --git a/llvm/test/CodeGen/X86/stackmap-large-location-size.ll b/llvm/test/CodeGen/X86/stackmap-large-location-size.ll
index 6c90ddaedcc95..1f618e8b72624 100644
--- a/llvm/test/CodeGen/X86/stackmap-large-location-size.ll
+++ b/llvm/test/CodeGen/X86/stackmap-large-location-size.ll
@@ -161,8 +161,8 @@ define <400 x i8 addrspace(1)*> @f_3(<400 x i8 addrspace(1)*> %obj) gc "statepoi
 ; CHECK-NEXT: .long	0
 ; Padding
 ; CHECK-NEXT: .p2align	3
-  %tok = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 4242, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, <400 x i8 addrspace(1)*> %obj)
-  %obj.r = call coldcc <400 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v400p1i8(token %tok, i32 7, i32 7)
+  %tok = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 4242, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) ["gc-live"(<400 x i8 addrspace(1)*> %obj)]
+  %obj.r = call coldcc <400 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v400p1i8(token %tok, i32 0, i32 0)
   ret <400 x i8 addrspace(1)*> %obj.r
 }
 
diff --git a/llvm/test/CodeGen/X86/statepoint-duplicates-export.ll b/llvm/test/CodeGen/X86/statepoint-duplicates-export.ll
index f61604b169f2e..baf9d6fda9a86 100644
--- a/llvm/test/CodeGen/X86/statepoint-duplicates-export.ll
+++ b/llvm/test/CodeGen/X86/statepoint-duplicates-export.ll
@@ -25,15 +25,15 @@ define i1 @test() gc "statepoint-example" {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
 entry:
-  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* null, i32 addrspace(1)* null)
-  %base = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 7, i32 7)
-  %derived = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 7, i32 8)
-  %safepoint_token2 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %base, i32 addrspace(1)* %derived)
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* null, i32 addrspace(1)* null)]
+  %base = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
+  %derived = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 1)
+  %safepoint_token2 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %base, i32 addrspace(1)* %derived)]
   br label %next
 
 next:
-  %base_reloc = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2,  i32 7, i32 7)
-  %derived_reloc = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2,  i32 7, i32 8)
+  %base_reloc = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2,  i32 0, i32 0)
+  %derived_reloc = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2,  i32 0, i32 1)
   %cmp1 = icmp eq i32 addrspace(1)* %base_reloc, null
   %cmp2 = icmp eq i32 addrspace(1)* %derived_reloc, null
   %cmp = and i1 %cmp1, %cmp2
@@ -57,15 +57,15 @@ define i1 @test2(i32 addrspace(1)* %arg) gc "statepoint-example" {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
 entry:
-  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %arg, i32 addrspace(1)* %arg)
-  %base = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 7, i32 7)
-  %derived = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 7, i32 8)
-  %safepoint_token2 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %base, i32 addrspace(1)* %derived)
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %arg, i32 addrspace(1)* %arg)]
+  %base = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
+  %derived = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 1)
+  %safepoint_token2 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %base, i32 addrspace(1)* %derived)]
   br label %next
 
 next:
-  %base_reloc = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2,  i32 7, i32 7)
-  %derived_reloc = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2,  i32 7, i32 8)
+  %base_reloc = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2,  i32 0, i32 0)
+  %derived_reloc = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2,  i32 0, i32 1)
   %cmp1 = icmp eq i32 addrspace(1)* %base_reloc, null
   %cmp2 = icmp eq i32 addrspace(1)* %derived_reloc, null
   %cmp = and i1 %cmp1, %cmp2
diff --git a/llvm/test/CodeGen/X86/statepoint-forward.ll b/llvm/test/CodeGen/X86/statepoint-forward.ll
index bee4b5ac884e9..b7f164cbdffd5 100644
--- a/llvm/test/CodeGen/X86/statepoint-forward.ll
+++ b/llvm/test/CodeGen/X86/statepoint-forward.ll
@@ -25,8 +25,8 @@ entry:
   %before = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* %p
   %cmp1 = call i1 @f(i32 addrspace(1)* %before)
   call void @llvm.assume(i1 %cmp1)
-  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* addrspace(1)* %p)
-  %pnew = call i32 addrspace(1)* addrspace(1)* @llvm.experimental.gc.relocate.p1p1i32(token %safepoint_token,  i32 7, i32 7)
+  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* addrspace(1)* %p)]
+  %pnew = call i32 addrspace(1)* addrspace(1)* @llvm.experimental.gc.relocate.p1p1i32(token %safepoint_token,  i32 0, i32 0)
   %after = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* %pnew
   %cmp2 = call i1 @f(i32 addrspace(1)* %after)
   ret i1 %cmp2
@@ -44,8 +44,8 @@ entry:
   %cmp1 = call i1 @f(i32 addrspace(1)* %v)
   call void @llvm.assume(i1 %cmp1)
   store i32 addrspace(1)* %v, i32 addrspace(1)* addrspace(1)* %p
-  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* addrspace(1)* %p)
-  %pnew = call i32 addrspace(1)* addrspace(1)* @llvm.experimental.gc.relocate.p1p1i32(token %safepoint_token,  i32 7, i32 7)
+  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* addrspace(1)* %p)]
+  %pnew = call i32 addrspace(1)* addrspace(1)* @llvm.experimental.gc.relocate.p1p1i32(token %safepoint_token,  i32 0, i32 0)
   %after = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* %pnew
   %cmp2 = call i1 @f(i32 addrspace(1)* %after)
   ret i1 %cmp2
diff --git a/llvm/test/CodeGen/X86/statepoint-gctransition-call-lowering.ll b/llvm/test/CodeGen/X86/statepoint-gctransition-call-lowering.ll
index b1b266c5699f2..0743f4c4e9c1f 100644
--- a/llvm/test/CodeGen/X86/statepoint-gctransition-call-lowering.ll
+++ b/llvm/test/CodeGen/X86/statepoint-gctransition-call-lowering.ll
@@ -72,8 +72,8 @@ define i1 @test_relocate(i32 addrspace(1)* %a) gc "statepoint-example" {
 ; CHECK-NEXT: .cfi_def_cfa_offset 8
 ; CHECK-NEXT: retq
 entry:
-  %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 1, i32 0, i32 0, i32 addrspace(1)* %a)
-  %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 7)
+  %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 1, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a)]
+  %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 0, i32 0)
   %call2 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token)
   ret i1 %call2
 }
diff --git a/llvm/test/CodeGen/X86/statepoint-no-extra-const.ll b/llvm/test/CodeGen/X86/statepoint-no-extra-const.ll
index a4291020196b4..38cfd812a615d 100644
--- a/llvm/test/CodeGen/X86/statepoint-no-extra-const.ll
+++ b/llvm/test/CodeGen/X86/statepoint-no-extra-const.ll
@@ -14,8 +14,8 @@ define i8 addrspace(1)* @no_extra_const(i8 addrspace(1)* %obj) gc "statepoint-ex
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
 entry:
-  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 4, void ()* null, i32 0, i32 0, i32 0, i32 0, i8 addrspace(1)* %obj)
-  %obj.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %safepoint_token, i32 7, i32 7) ; (%obj, %obj)
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 4, void ()* null, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i8 addrspace(1)* %obj)]
+  %obj.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %safepoint_token, i32 0, i32 0) ; (%obj, %obj)
   ret i8 addrspace(1)* %obj.relocated
 }
 
diff --git a/llvm/test/CodeGen/X86/statepoint-no-realign-stack.ll b/llvm/test/CodeGen/X86/statepoint-no-realign-stack.ll
index 40e721bfa80f3..e030be0555717 100644
--- a/llvm/test/CodeGen/X86/statepoint-no-realign-stack.ll
+++ b/llvm/test/CodeGen/X86/statepoint-no-realign-stack.ll
@@ -76,8 +76,8 @@ define <4 x i8 addrspace(1)*> @spillfill_can_realign(<4 x i8 addrspace(1)*> %obj
 ; CHECK-NEXT:    .cfi_def_cfa %rsp, 8
 ; CHECK-NEXT:    retq
 entry:
-  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, <4 x i8 addrspace(1)*> %obj)
-  %obj.relocated = call coldcc <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8(token %safepoint_token, i32 7, i32 7) ; (%obj, %obj)
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) ["gc-live" (<4 x i8 addrspace(1)*> %obj)]
+  %obj.relocated = call coldcc <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8(token %safepoint_token, i32 0, i32 0) ; (%obj, %obj)
   ret <4 x i8 addrspace(1)*> %obj.relocated
 }
 
@@ -95,8 +95,8 @@ define <4 x i8 addrspace(1)*> @spillfill_no_realign(<4 x i8 addrspace(1)*> %obj)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
 entry:
-  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, <4 x i8 addrspace(1)*> %obj)
-  %obj.relocated = call coldcc <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8(token %safepoint_token, i32 7, i32 7) ; (%obj, %obj)
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) ["gc-live" (<4 x i8 addrspace(1)*> %obj)]
+  %obj.relocated = call coldcc <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8(token %safepoint_token, i32 0, i32 0) ; (%obj, %obj)
   ret <4 x i8 addrspace(1)*> %obj.relocated
 }
 
diff --git a/llvm/test/CodeGen/X86/statepoint-regs.ll b/llvm/test/CodeGen/X86/statepoint-regs.ll
index af237cdb3345f..b7a74ee1b36b2 100644
--- a/llvm/test/CodeGen/X86/statepoint-regs.ll
+++ b/llvm/test/CodeGen/X86/statepoint-regs.ll
@@ -115,10 +115,10 @@ define  i32 addrspace(1)* @test5(i32 %a, i32 addrspace(1)* %p) gc "statepoint-ex
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    retq
 entry:
-  %token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %p, i32 addrspace(1)* %p) ["deopt"(i32 %a)]
-  %p2 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %token,  i32 8, i32 8)
-  %token2 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %p2, i32 addrspace(1)* %p2) ["deopt"(i32 %a)]
-  %p3 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %token2,  i32 8, i32 8)
+  %token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %p, i32 addrspace(1)* %p), "deopt"(i32 %a)]
+  %p2 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %token,  i32 1, i32 1)
+  %token2 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %p2, i32 addrspace(1)* %p2), "deopt"(i32 %a)]
+  %p3 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %token2,  i32 1, i32 1)
   ret i32 addrspace(1)* %p3
 }
 
@@ -683,12 +683,12 @@ define i32 addrspace(1)*  @test_fpconst_deopt(i32 addrspace(1)* %in) gc "statepo
 ; CHECK-NEXT:    movq (%rsp), %rax
 ; CHECK-NEXT:    popq %rcx
 ; CHECK-NEXT:    retq
-    %statepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2, i32 5, void ()* nonnull @bar, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %in) ["deopt" (
+    %statepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2, i32 5, void ()* nonnull @bar, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %in), "deopt" (
     float 0x40421A1CA0000000, float 0x40459A1CA0000000, float 0x40401A1CA0000000, float 0x40479A1CA0000000, float 0x403C343940000000,
     float 0x403E343940000000, float 0x40469A1CA0000000, float 0x40489A1CA0000000, float 0x404A9A1CA0000000, float 0x40499A1CA0000000,
     float 0xC05FCD2F20000000, float 0xC05C0D2F20000000, float 0xC060269780000000, float 0xC05B8D2F20000000, float 0xC060669780000000,
     float 0xC05B0D2F20000000, float 0xC060A69780000000, float 0xC05A8D2F20000000, float 0xC060E69780000000, float 0x40439A1CA0000000)]
-    %out = call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %statepoint_token, i32 7, i32 7)
+    %out = call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %statepoint_token, i32 0, i32 0)
     ret i32 addrspace(1)* %out
 }
 
diff --git a/llvm/test/CodeGen/X86/statepoint-stack-usage.ll b/llvm/test/CodeGen/X86/statepoint-stack-usage.ll
index 1841bd03d6d3c..4e355b2372ec9 100644
--- a/llvm/test/CodeGen/X86/statepoint-stack-usage.ll
+++ b/llvm/test/CodeGen/X86/statepoint-stack-usage.ll
@@ -16,17 +16,17 @@ define i32 @back_to_back_calls(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 a
 ; CHECK-DAG: movq	%rsi, (%rsp)
 ; There should be no more than three moves
 ; CHECK-NOT: movq
-  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) ["deopt" (i32 0, i32 -1, i32 0, i32 0, i32 0)]
-  %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 7)
-  %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 8)
-  %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 9)
+  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c), "deopt" (i32 0, i32 -1, i32 0, i32 0, i32 0)]
+  %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 0, i32 0)
+  %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 0, i32 1)
+  %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 0, i32 2)
 ; CHECK: callq
 ; This is the key check.  There should NOT be any memory moves here
 ; CHECK-NOT: movq
-  %safepoint_token2 = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %c1, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1) ["deopt" (i32 0, i32 -1, i32 0, i32 0, i32 0)]
-  %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 7, i32 9)
-  %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 7, i32 8)
-  %c2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 7, i32 7)
+  %safepoint_token2 = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %c1, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1), "deopt" (i32 0, i32 -1, i32 0, i32 0, i32 0)]
+  %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 0, i32 2)
+  %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 0, i32 1)
+  %c2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 0, i32 0)
 ; CHECK: callq
   ret i32 1
 }
@@ -39,17 +39,17 @@ define i32 @reserve_first(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrsp
 ; CHECK-DAG: movq	%rdi, 16(%rsp)
 ; CHECK-DAG: movq	%rdx, 8(%rsp)
 ; CHECK-DAG: movq	%rsi, (%rsp)
-  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) ["deopt" (i32 0, i32 -1, i32 0, i32 0, i32 0)]
-  %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 7)
-  %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 8)
-  %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 9)
+  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c), "deopt" (i32 0, i32 -1, i32 0, i32 0, i32 0)]
+  %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 0, i32 0)
+  %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 0, i32 1)
+  %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 0, i32 2)
 ; CHECK: callq
 ; This is the key check.  There should NOT be any memory moves here
 ; CHECK-NOT: movq
-  %safepoint_token2 = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %c1, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1) ["deopt" (i32 addrspace(1)* %a1, i32 0, i32 addrspace(1)* %c1, i32 0, i32 0)]
-  %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 7, i32 9)
-  %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 7, i32 8)
-  %c2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 7, i32 7)
+  %safepoint_token2 = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %c1, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1), "deopt" (i32 addrspace(1)* %a1, i32 0, i32 addrspace(1)* %c1, i32 0, i32 0)]
+  %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 0, i32 2)
+  %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 0, i32 1)
+  %c2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 0, i32 0)
 ; CHECK: callq
   ret i32 1
 }
@@ -93,25 +93,25 @@ entry:
   ; CHECK-DAG: movq	%rdx, 8(%rsp)
   ; CHECK-DAG: movq	%rsi, (%rsp)
   ; CHECK: callq
-  %safepoint_token = invoke token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) ["deopt" (i32 0, i32 -1, i32 0, i32 0, i32 0)]
+  %safepoint_token = invoke token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c), "deopt" (i32 0, i32 -1, i32 0, i32 0, i32 0)]
                    to label %normal_return unwind label %exceptional_return
 
 normal_return:
-  %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 7)
-  %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 8)
-  %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 9)
+  %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 0, i32 0)
+  %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 0, i32 1)
+  %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 0, i32 2)
   ; Should work even through bitcasts
   %c1.casted = bitcast i32 addrspace(1)* %c1 to i8 addrspace(1)*
   ; This is the key check.  There should NOT be any memory moves here
   ; CHECK-NOT: movq
   ; CHECK: callq
-  %safepoint_token2 = invoke token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 0, i8 addrspace(1)* %c1.casted, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1) ["deopt" (i32 0, i32 -1, i32 0, i32 0, i32 0)]
+  %safepoint_token2 = invoke token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i8 addrspace(1)* %c1.casted, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1), "deopt" (i32 0, i32 -1, i32 0, i32 0, i32 0)]
                     to label %normal_return2 unwind label %exceptional_return2
 
 normal_return2:
-  %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 7, i32 9)
-  %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 7, i32 8)
-  %c2 = tail call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %safepoint_token2, i32 7, i32 7)
+  %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 0, i32 2)
+  %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 0, i32 1)
+  %c2 = tail call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %safepoint_token2, i32 0, i32 0)
   ret i32 1
 
 exceptional_return:
diff --git a/llvm/test/CodeGen/X86/statepoint-stackmap-format.ll b/llvm/test/CodeGen/X86/statepoint-stackmap-format.ll
index 71f7663cafd6c..896aedfaeb273 100644
--- a/llvm/test/CodeGen/X86/statepoint-stackmap-format.ll
+++ b/llvm/test/CodeGen/X86/statepoint-stackmap-format.ll
@@ -25,11 +25,11 @@ entry:
   %metadata1 = alloca i32 addrspace(1)*, i32 2, align 8
   store i32 addrspace(1)* null, i32 addrspace(1)** %metadata1
   %ptr_derived = getelementptr i32, i32 addrspace(1)* %ptr_base, i32 %arg
-  %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %ptr_base, i32 addrspace(1)* %ptr_derived, i32 addrspace(1)* null) ["deopt" (i32 addrspace(1)* %ptr_base, i32 addrspace(1)* null)]
+  %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %ptr_base, i32 addrspace(1)* %ptr_derived, i32 addrspace(1)* null), "deopt" (i32 addrspace(1)* %ptr_base, i32 addrspace(1)* null)]
   %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token)
-  %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 7)
-  %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 8)
-  %c = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 9, i32 9)
+  %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 0, i32 0)
+  %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 0, i32 1)
+  %c = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 2, i32 2)
 ; 
   ret i1 %call1
 }
@@ -53,11 +53,11 @@ define i1 @test_derived_arg(i32 addrspace(1)* %ptr_base,
 entry:
   %metadata1 = alloca i32 addrspace(1)*, i32 2, align 8
   store i32 addrspace(1)* null, i32 addrspace(1)** %metadata1
-  %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %ptr_base, i32 addrspace(1)* %ptr_derived, i32 addrspace(1)* null) ["deopt" (i32 addrspace(1)* %ptr_base, i32 addrspace(1)* null)]
+  %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %ptr_base, i32 addrspace(1)* %ptr_derived, i32 addrspace(1)* null), "deopt" (i32 addrspace(1)* %ptr_base, i32 addrspace(1)* null)]
   %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token)
-  %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 7)
-  %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 8)
-  %c = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 9, i32 9)
+  %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 0, i32 0)
+  %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 0, i32 1)
+  %c = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 2, i32 2)
 ; 
   ret i1 %call1
 }
@@ -90,8 +90,8 @@ define i32 @test_spadj(i32 addrspace(1)* %p) gc "statepoint-example" {
   ; CHECK: callq many_arg
   ; CHECK: addq $16, %rsp
   ; CHECK: movq (%rsp)
-  %statepoint_token = call token (i64, i32, void (i64, i64, i64, i64, i64, i64, i64, i64)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidi64i64i64i64i64i64i64i64f(i64 0, i32 0, void (i64, i64, i64, i64, i64, i64, i64, i64)* @many_arg, i32 8, i32 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i32 0, i32 0, i32 addrspace(1)* %p)
-  %p.relocated = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %statepoint_token, i32 15, i32 15) ; (%p, %p)
+  %statepoint_token = call token (i64, i32, void (i64, i64, i64, i64, i64, i64, i64, i64)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidi64i64i64i64i64i64i64i64f(i64 0, i32 0, void (i64, i64, i64, i64, i64, i64, i64, i64)* @many_arg, i32 8, i32 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %p)]
+  %p.relocated = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %statepoint_token, i32 0, i32 0) ; (%p, %p)
   %ld = load i32, i32 addrspace(1)* %p.relocated
   ret i32 %ld
 }
diff --git a/llvm/test/CodeGen/X86/statepoint-stackmap-size.ll b/llvm/test/CodeGen/X86/statepoint-stackmap-size.ll
index 55830f2c3113b..cb0ed3443139b 100644
--- a/llvm/test/CodeGen/X86/statepoint-stackmap-size.ll
+++ b/llvm/test/CodeGen/X86/statepoint-stackmap-size.ll
@@ -11,9 +11,9 @@ declare void @func()
 
 define i1 @test1(i32 addrspace(1)* %arg) gc "statepoint-example" {
 entry:
-  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %arg)
-  %reloc1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 7, i32 7)
-  %reloc2 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 7, i32 7)
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %arg)]
+  %reloc1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
+  %reloc2 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
   %cmp1 = icmp eq i32 addrspace(1)* %reloc1, null
   %cmp2 = icmp eq i32 addrspace(1)* %reloc2, null
   %cmp = and i1 %cmp1, %cmp2
diff --git a/llvm/test/CodeGen/X86/statepoint-uniqueing.ll b/llvm/test/CodeGen/X86/statepoint-uniqueing.ll
index 2b0e3764c756b..02f69f4f7cfff 100644
--- a/llvm/test/CodeGen/X86/statepoint-uniqueing.ll
+++ b/llvm/test/CodeGen/X86/statepoint-uniqueing.ll
@@ -27,9 +27,9 @@ define void @test_gcrelocate_uniqueing(i32 addrspace(1)* %ptr) gc "statepoint-ex
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
   %tok = tail call token (i64, i32, void ()*, i32, i32, ...)
-      @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @f, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %ptr, i32 addrspace(1)* %ptr) ["deopt" (i32 addrspace(1)* %ptr, i32 undef)]
-  %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 7, i32 7)
-  %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 8, i32 8)
+      @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @f, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %ptr, i32 addrspace(1)* %ptr), "deopt" (i32 addrspace(1)* %ptr, i32 undef)]
+  %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 0, i32 0)
+  %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 1, i32 1)
   call void (...) @use(i32 addrspace(1)* %a, i32 addrspace(1)* %b)
   ret void
 }
@@ -52,9 +52,9 @@ define void @test_gcptr_uniqueing(i32 addrspace(1)* %ptr) gc "statepoint-example
 ; CHECK-NEXT:    retq
   %ptr2 = bitcast i32 addrspace(1)* %ptr to i8 addrspace(1)*
   %tok = tail call token (i64, i32, void ()*, i32, i32, ...)
-      @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @f, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %ptr, i8 addrspace(1)* %ptr2) ["deopt" (i32 addrspace(1)* %ptr, i32 undef)]
-  %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 7, i32 7)
-  %b = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok, i32 8, i32 8)
+      @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @f, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %ptr, i8 addrspace(1)* %ptr2), "deopt" (i32 addrspace(1)* %ptr, i32 undef)]
+  %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 0, i32 0)
+  %b = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok, i32 1, i32 1)
   call void (...) @use(i32 addrspace(1)* %a, i8 addrspace(1)* %b)
   ret void
 }
@@ -73,7 +73,7 @@ define void @test_deopt_use(i32 addrspace(1)* %ptr) gc "statepoint-example" {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
   tail call token (i64, i32, void ()*, i32, i32, ...)
-      @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @f, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %ptr) ["deopt" (i32 addrspace(1)* %ptr, i32 undef)]
+      @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @f, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %ptr), "deopt" (i32 addrspace(1)* %ptr, i32 undef)]
   ret void
 }
 
@@ -89,6 +89,6 @@ define void @test_dse(i32 addrspace(1)* %ptr) gc "statepoint-example" {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
   tail call token (i64, i32, void ()*, i32, i32, ...)
-      @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @f, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %ptr)
+      @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @f, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %ptr)]
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/statepoint-vector-bad-spill.ll b/llvm/test/CodeGen/X86/statepoint-vector-bad-spill.ll
index 7c55491bb1bea..070ed541713e8 100644
--- a/llvm/test/CodeGen/X86/statepoint-vector-bad-spill.ll
+++ b/llvm/test/CodeGen/X86/statepoint-vector-bad-spill.ll
@@ -8,14 +8,14 @@ define <2 x i8 addrspace(1)*> @test0(i8 addrspace(1)* %el, <2 x i8 addrspace(1)*
 ; CHECK-LABEL: test0:
 
 entry:
-  %tok0 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, i8 addrspace(1)* %el)
-  %el.relocated = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok0, i32 7, i32 7)
+  %tok0 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i8 addrspace(1)* %el)]
+  %el.relocated = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok0, i32 0, i32 0)
 
   %obj.pre = load <2 x i8 addrspace(1)*>, <2 x i8 addrspace(1)*>* %vec_ptr
   %obj = insertelement <2 x i8 addrspace(1)*> %obj.pre, i8 addrspace(1)* %el.relocated, i32 0  ; No real objective here, except to use %el
 
-  %tok1 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, <2 x i8 addrspace(1)*> %obj)
-  %obj.relocated = call <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %tok1, i32 7, i32 7)
+  %tok1 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) ["gc-live" (<2 x i8 addrspace(1)*> %obj)]
+  %obj.relocated = call <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %tok1, i32 0, i32 0)
   ret <2 x i8 addrspace(1)*> %obj.relocated
 }
 
@@ -23,12 +23,12 @@ define i8 addrspace(1)* @test1(<2 x i8 addrspace(1)*> %obj) gc "statepoint-examp
 ; CHECK-LABEL: test1:
 
 entry:
-  %tok1 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, <2 x i8 addrspace(1)*> %obj)
-  %obj.relocated = call <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %tok1, i32 7, i32 7)
+  %tok1 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) ["gc-live" (<2 x i8 addrspace(1)*> %obj)]
+  %obj.relocated = call <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %tok1, i32 0, i32 0)
 
   %el = extractelement <2 x i8 addrspace(1)*> %obj.relocated, i32 0
-  %tok0 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, i8 addrspace(1)* %el)
-  %el.relocated = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok0, i32 7, i32 7)
+  %tok0 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i8 addrspace(1)* %el)]
+  %el.relocated = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok0, i32 0, i32 0)
   ret i8 addrspace(1)* %el.relocated
 }
 
diff --git a/llvm/test/CodeGen/X86/statepoint-vector.ll b/llvm/test/CodeGen/X86/statepoint-vector.ll
index 36532765192e8..a7d7be8ed0699 100644
--- a/llvm/test/CodeGen/X86/statepoint-vector.ll
+++ b/llvm/test/CodeGen/X86/statepoint-vector.ll
@@ -18,8 +18,8 @@ define <2 x i8 addrspace(1)*> @test(<2 x i8 addrspace(1)*> %obj) gc "statepoint-
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
 entry:
-  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, <2 x i8 addrspace(1)*> %obj)
-  %obj.relocated = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %safepoint_token, i32 7, i32 7) ; (%obj, %obj)
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) ["gc-live" (<2 x i8 addrspace(1)*> %obj)]
+  %obj.relocated = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %safepoint_token, i32 0, i32 0) ; (%obj, %obj)
   ret <2 x i8 addrspace(1)*> %obj.relocated
 }
 
@@ -42,8 +42,8 @@ define <2 x i8 addrspace(1)*> @test2(<2 x i8 addrspace(1)*> %obj, i64 %offset) g
 ; CHECK-NEXT:    retq
 entry:
   %derived = getelementptr i8, <2 x i8 addrspace(1)*> %obj, i64 %offset
-  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, <2 x i8 addrspace(1)*> %obj, <2 x i8 addrspace(1)*> %derived)
-  %derived.relocated = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %safepoint_token, i32 7, i32 8) ; (%obj, %derived)
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) ["gc-live" (<2 x i8 addrspace(1)*> %obj, <2 x i8 addrspace(1)*> %derived)]
+  %derived.relocated = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %safepoint_token, i32 0, i32 1) ; (%obj, %derived)
   ret <2 x i8 addrspace(1)*> %derived.relocated
 }
 
@@ -83,10 +83,10 @@ untaken:                                          ; preds = %entry
 merge:                                            ; preds = %untaken, %taken
   %obj.base = phi <2 x i64 addrspace(1)*> [ %obja, %taken ], [ %objb, %untaken ]
   %obj = phi <2 x i64 addrspace(1)*> [ %obja, %taken ], [ %objb, %untaken ]
-  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, <2 x i64 addrspace(1)*> %obj, <2 x i64 addrspace(1)*> %obj.base)
-  %obj.relocated = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %safepoint_token, i32 8, i32 7) ; (%obj.base, %obj)
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) ["gc-live" (<2 x i64 addrspace(1)*> %obj, <2 x i64 addrspace(1)*> %obj.base)]
+  %obj.relocated = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %safepoint_token, i32 1, i32 0) ; (%obj.base, %obj)
   %obj.relocated.casted = bitcast <2 x i8 addrspace(1)*> %obj.relocated to <2 x i64 addrspace(1)*>
-  %obj.base.relocated = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %safepoint_token, i32 8, i32 8) ; (%obj.base, %obj.base)
+  %obj.base.relocated = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %safepoint_token, i32 1, i32 1) ; (%obj.base, %obj.base)
   %obj.base.relocated.casted = bitcast <2 x i8 addrspace(1)*> %obj.base.relocated to <2 x i64 addrspace(1)*>
   ret <2 x i64 addrspace(1)*> %obj.relocated.casted
 }
@@ -107,26 +107,31 @@ define <2 x i8 addrspace(1)*> @test4() gc "statepoint-example" {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
 entry:
-  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, <2 x i8 addrspace(1)*> zeroinitializer)
-  %obj.relocated = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %safepoint_token, i32 7, i32 7) ; (%obj, %obj)
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) ["gc-live" (<2 x i8 addrspace(1)*> zeroinitializer)]
+  %obj.relocated = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %safepoint_token, i32 0, i32 0)
   ret <2 x i8 addrspace(1)*> %obj.relocated
 }
 
-; Check that we can lower a constant typed as i128 correctly.  Note that the
-; actual value is representable in 64 bits.  We don't have a representation
-; of larger than 64 bit constant in the StackMap format.
+; Check that we can lower a constant typed as i128 correctly.  We don't have
+; a representation of larger than 64 bit constant in the StackMap format. At
+; the moment, this simply means spilling them, but there's a potential
+; optimization for values representable as sext(Con64).  
 define void @test5() gc "statepoint-example" {
 ; CHECK-LABEL: test5:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    movups %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq $-1, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq $-1, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    callq do_safepoint
 ; CHECK-NEXT:  .Ltmp4:
-; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    addq $40, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
 entry:
-  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) ["deopt" (i128 0)]
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) ["deopt" (i128 0, i128 -1)]
   ret void
 }
 
diff --git a/llvm/test/CodeGen/X86/stores-merging.ll b/llvm/test/CodeGen/X86/stores-merging.ll
index 6420ac7dc3ed5..768684067f32f 100644
--- a/llvm/test/CodeGen/X86/stores-merging.ll
+++ b/llvm/test/CodeGen/X86/stores-merging.ll
@@ -242,3 +242,200 @@ define void @pr43446_1(i8* %a) {
   store i1 true, i1* %b, align 1
   ret void
 }
+
+define void @rotate16_in_place(i8* %p) {
+; CHECK-LABEL: rotate16_in_place:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movb (%rdi), %al
+; CHECK-NEXT:    movb 1(%rdi), %cl
+; CHECK-NEXT:    movb %cl, (%rdi)
+; CHECK-NEXT:    movb %al, 1(%rdi)
+; CHECK-NEXT:    retq
+  %p0 = getelementptr i8, i8* %p, i64 0
+  %p1 = getelementptr i8, i8* %p, i64 1
+  %i0 = load i8, i8* %p0, align 1
+  %i1 = load i8, i8* %p1, align 1
+  store i8 %i1, i8* %p0, align 1
+  store i8 %i0, i8* %p1, align 1
+  ret void
+}
+
+define void @rotate16(i8* %p, i8* %q) {
+; CHECK-LABEL: rotate16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movb (%rdi), %al
+; CHECK-NEXT:    movb 1(%rdi), %cl
+; CHECK-NEXT:    movb %cl, (%rsi)
+; CHECK-NEXT:    movb %al, 1(%rsi)
+; CHECK-NEXT:    retq
+  %p0 = getelementptr i8, i8* %p, i64 0
+  %p1 = getelementptr i8, i8* %p, i64 1
+  %q0 = getelementptr i8, i8* %q, i64 0
+  %q1 = getelementptr i8, i8* %q, i64 1
+  %i0 = load i8, i8* %p0, align 1
+  %i1 = load i8, i8* %p1, align 1
+  store i8 %i1, i8* %q0, align 1
+  store i8 %i0, i8* %q1, align 1
+  ret void
+}
+
+define void @rotate32_in_place(i16* %p) {
+; CHECK-LABEL: rotate32_in_place:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movzwl (%rdi), %eax
+; CHECK-NEXT:    movzwl 2(%rdi), %ecx
+; CHECK-NEXT:    movw %cx, (%rdi)
+; CHECK-NEXT:    movw %ax, 2(%rdi)
+; CHECK-NEXT:    retq
+  %p0 = getelementptr i16, i16* %p, i64 0
+  %p1 = getelementptr i16, i16* %p, i64 1
+  %i0 = load i16, i16* %p0, align 2
+  %i1 = load i16, i16* %p1, align 2
+  store i16 %i1, i16* %p0, align 2
+  store i16 %i0, i16* %p1, align 2
+  ret void
+}
+
+define void @rotate32(i16* %p) {
+; CHECK-LABEL: rotate32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movzwl (%rdi), %eax
+; CHECK-NEXT:    movzwl 2(%rdi), %ecx
+; CHECK-NEXT:    movw %cx, 84(%rdi)
+; CHECK-NEXT:    movw %ax, 86(%rdi)
+; CHECK-NEXT:    retq
+  %p0 = getelementptr i16, i16* %p, i64 0
+  %p1 = getelementptr i16, i16* %p, i64 1
+  %p42 = getelementptr i16, i16* %p, i64 42
+  %p43 = getelementptr i16, i16* %p, i64 43
+  %i0 = load i16, i16* %p0, align 2
+  %i1 = load i16, i16* %p1, align 2
+  store i16 %i1, i16* %p42, align 2
+  store i16 %i0, i16* %p43, align 2
+  ret void
+}
+
+define void @rotate64_in_place(i32* %p) {
+; CHECK-LABEL: rotate64_in_place:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl (%rdi), %eax
+; CHECK-NEXT:    movl 4(%rdi), %ecx
+; CHECK-NEXT:    movl %ecx, (%rdi)
+; CHECK-NEXT:    movl %eax, 4(%rdi)
+; CHECK-NEXT:    retq
+  %p0 = getelementptr i32, i32* %p, i64 0
+  %p1 = getelementptr i32, i32* %p, i64 1
+  %i0 = load i32, i32* %p0, align 4
+  %i1 = load i32, i32* %p1, align 4
+  store i32 %i1, i32* %p0, align 4
+  store i32 %i0, i32* %p1, align 4
+  ret void
+}
+
+define void @rotate64(i32* %p) {
+; CHECK-LABEL: rotate64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl (%rdi), %eax
+; CHECK-NEXT:    movl 4(%rdi), %ecx
+; CHECK-NEXT:    movl %ecx, 8(%rdi)
+; CHECK-NEXT:    movl %eax, 12(%rdi)
+; CHECK-NEXT:    retq
+  %p0 = getelementptr i32, i32* %p, i64 0
+  %p1 = getelementptr i32, i32* %p, i64 1
+  %p2 = getelementptr i32, i32* %p, i64 2
+  %p3 = getelementptr i32, i32* %p, i64 3
+  %i0 = load i32, i32* %p0, align 4
+  %i1 = load i32, i32* %p1, align 4
+  store i32 %i1, i32* %p2, align 4
+  store i32 %i0, i32* %p3, align 4
+  ret void
+}
+
+define void @rotate64_iterate(i16* %p) {
+; CHECK-LABEL: rotate64_iterate:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl (%rdi), %eax
+; CHECK-NEXT:    movl 4(%rdi), %ecx
+; CHECK-NEXT:    movl %ecx, 84(%rdi)
+; CHECK-NEXT:    movl %eax, 88(%rdi)
+; CHECK-NEXT:    retq
+  %p0 = getelementptr i16, i16* %p, i64 0
+  %p1 = getelementptr i16, i16* %p, i64 1
+  %p2 = getelementptr i16, i16* %p, i64 2
+  %p3 = getelementptr i16, i16* %p, i64 3
+  %p42 = getelementptr i16, i16* %p, i64 42
+  %p43 = getelementptr i16, i16* %p, i64 43
+  %p44 = getelementptr i16, i16* %p, i64 44
+  %p45 = getelementptr i16, i16* %p, i64 45
+  %i0 = load i16, i16* %p0, align 2
+  %i1 = load i16, i16* %p1, align 2
+  %i2 = load i16, i16* %p2, align 2
+  %i3 = load i16, i16* %p3, align 2
+  store i16 %i2, i16* %p42, align 2
+  store i16 %i3, i16* %p43, align 2
+  store i16 %i0, i16* %p44, align 2
+  store i16 %i1, i16* %p45, align 2
+  ret void
+}
+
+define void @rotate32_consecutive(i16* %p) {
+; CHECK-LABEL: rotate32_consecutive:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movzwl (%rdi), %eax
+; CHECK-NEXT:    movzwl 2(%rdi), %ecx
+; CHECK-NEXT:    movzwl 4(%rdi), %edx
+; CHECK-NEXT:    movzwl 6(%rdi), %esi
+; CHECK-NEXT:    movw %cx, 84(%rdi)
+; CHECK-NEXT:    movw %ax, 86(%rdi)
+; CHECK-NEXT:    movw %si, 88(%rdi)
+; CHECK-NEXT:    movw %dx, 90(%rdi)
+; CHECK-NEXT:    retq
+  %p0 = getelementptr i16, i16* %p, i64 0
+  %p1 = getelementptr i16, i16* %p, i64 1
+  %p2 = getelementptr i16, i16* %p, i64 2
+  %p3 = getelementptr i16, i16* %p, i64 3
+  %p42 = getelementptr i16, i16* %p, i64 42
+  %p43 = getelementptr i16, i16* %p, i64 43
+  %p44 = getelementptr i16, i16* %p, i64 44
+  %p45 = getelementptr i16, i16* %p, i64 45
+  %i0 = load i16, i16* %p0, align 2
+  %i1 = load i16, i16* %p1, align 2
+  %i2 = load i16, i16* %p2, align 2
+  %i3 = load i16, i16* %p3, align 2
+  store i16 %i1, i16* %p42, align 2
+  store i16 %i0, i16* %p43, align 2
+  store i16 %i3, i16* %p44, align 2
+  store i16 %i2, i16* %p45, align 2
+  ret void
+}
+
+define void @rotate32_twice(i16* %p) {
+; CHECK-LABEL: rotate32_twice:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movzwl (%rdi), %eax
+; CHECK-NEXT:    movzwl 2(%rdi), %ecx
+; CHECK-NEXT:    movzwl 4(%rdi), %edx
+; CHECK-NEXT:    movzwl 6(%rdi), %esi
+; CHECK-NEXT:    movw %cx, 84(%rdi)
+; CHECK-NEXT:    movw %ax, 86(%rdi)
+; CHECK-NEXT:    movw %si, 108(%rdi)
+; CHECK-NEXT:    movw %dx, 110(%rdi)
+; CHECK-NEXT:    retq
+  %p0 = getelementptr i16, i16* %p, i64 0
+  %p1 = getelementptr i16, i16* %p, i64 1
+  %p2 = getelementptr i16, i16* %p, i64 2
+  %p3 = getelementptr i16, i16* %p, i64 3
+  %p42 = getelementptr i16, i16* %p, i64 42
+  %p43 = getelementptr i16, i16* %p, i64 43
+  %p54 = getelementptr i16, i16* %p, i64 54
+  %p55 = getelementptr i16, i16* %p, i64 55
+  %i0 = load i16, i16* %p0, align 2
+  %i1 = load i16, i16* %p1, align 2
+  %i2 = load i16, i16* %p2, align 2
+  %i3 = load i16, i16* %p3, align 2
+  store i16 %i1, i16* %p42, align 2
+  store i16 %i0, i16* %p43, align 2
+  store i16 %i3, i16* %p54, align 2
+  store i16 %i2, i16* %p55, align 2
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/unreachableblockelim.ll b/llvm/test/CodeGen/X86/unreachableblockelim.ll
index adaedb5e8d780..4aafa21f4ea5e 100644
--- a/llvm/test/CodeGen/X86/unreachableblockelim.ll
+++ b/llvm/test/CodeGen/X86/unreachableblockelim.ll
@@ -1,5 +1,4 @@
 ; RUN: opt -S < %s -unreachableblockelim | FileCheck %s
-; RUN: opt -S < %s -passes=unreachableblockelim | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/CodeGen/X86/var-permute-256.ll b/llvm/test/CodeGen/X86/var-permute-256.ll
index 4c04219b3883c..7590add145a03 100644
--- a/llvm/test/CodeGen/X86/var-permute-256.ll
+++ b/llvm/test/CodeGen/X86/var-permute-256.ll
@@ -1104,21 +1104,16 @@ entry:
 define <4 x i32> @var_shuffle_v4i32_from_v8i32(<8 x i32> %v, <4 x i32> %indices) unnamed_addr nounwind {
 ; XOP-LABEL: var_shuffle_v4i32_from_v8i32:
 ; XOP:       # %bb.0: # %entry
-; XOP-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
-; XOP-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3]
-; XOP-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; XOP-NEXT:    vpermil2ps $0, %ymm1, %ymm2, %ymm0, %ymm0
-; XOP-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; XOP-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; XOP-NEXT:    vpermil2ps $0, %xmm1, %xmm2, %xmm0, %xmm0
 ; XOP-NEXT:    vzeroupper
 ; XOP-NEXT:    retq
 ;
 ; AVX1-LABEL: var_shuffle_v4i32_from_v8i32:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3]
-; AVX1-NEXT:    vpermilps %ymm1, %ymm2, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpermilps %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vpermilps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpcmpgtd {{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/vec-strict-inttofp-512.ll b/llvm/test/CodeGen/X86/vec-strict-inttofp-512.ll
index be1d15ba68b39..328f3c15fc48b 100644
--- a/llvm/test/CodeGen/X86/vec-strict-inttofp-512.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-inttofp-512.ll
@@ -362,8 +362,8 @@ define <8 x double> @sitofp_v8i64_v8f64(<8 x i64> %x) #0 {
 define <8 x double> @uitofp_v8i64_v8f64(<8 x i64> %x) #0 {
 ; NODQ-32-LABEL: uitofp_v8i64_v8f64:
 ; NODQ-32:       # %bb.0:
-; NODQ-32-NEXT:    vpandq {{\.LCPI.*}}, %zmm0, %zmm1
-; NODQ-32-NEXT:    vporq {{\.LCPI.*}}, %zmm1, %zmm1
+; NODQ-32-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1127219200,0,1127219200,0,1127219200,0,1127219200,0,1127219200,0,1127219200,0,1127219200,0,1127219200]
+; NODQ-32-NEXT:    vpternlogq $248, {{\.LCPI.*}}, %zmm0, %zmm1
 ; NODQ-32-NEXT:    vpsrlq $32, %zmm0, %zmm0
 ; NODQ-32-NEXT:    vporq {{\.LCPI.*}}, %zmm0, %zmm0
 ; NODQ-32-NEXT:    vsubpd {{\.LCPI.*}}{1to8}, %zmm0, %zmm0
@@ -372,8 +372,8 @@ define <8 x double> @uitofp_v8i64_v8f64(<8 x i64> %x) #0 {
 ;
 ; NODQ-64-LABEL: uitofp_v8i64_v8f64:
 ; NODQ-64:       # %bb.0:
-; NODQ-64-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm1
-; NODQ-64-NEXT:    vporq {{.*}}(%rip){1to8}, %zmm1, %zmm1
+; NODQ-64-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
+; NODQ-64-NEXT:    vpternlogq $248, {{.*}}(%rip){1to8}, %zmm0, %zmm1
 ; NODQ-64-NEXT:    vpsrlq $32, %zmm0, %zmm0
 ; NODQ-64-NEXT:    vporq {{.*}}(%rip){1to8}, %zmm0, %zmm0
 ; NODQ-64-NEXT:    vsubpd {{.*}}(%rip){1to8}, %zmm0, %zmm0
diff --git a/llvm/test/CodeGen/X86/vector-compare-combines.ll b/llvm/test/CodeGen/X86/vector-compare-combines.ll
index 722de10094891..6160b6e9f33e8 100644
--- a/llvm/test/CodeGen/X86/vector-compare-combines.ll
+++ b/llvm/test/CodeGen/X86/vector-compare-combines.ll
@@ -42,4 +42,3 @@ define <4 x i32> @PR27924_cmpgt(<4 x i32> %a, <4 x i32> %b) {
   %ret = sext <4 x i1> %untruth to <4 x i32>
   ret <4 x i32> %ret
 }
-
diff --git a/llvm/test/CodeGen/X86/vector-compare-results.ll b/llvm/test/CodeGen/X86/vector-compare-results.ll
index d0eccaa01703d..568dff8e13585 100644
--- a/llvm/test/CodeGen/X86/vector-compare-results.ll
+++ b/llvm/test/CodeGen/X86/vector-compare-results.ll
@@ -410,9 +410,9 @@ define <8 x i1> @test_cmp_v8f64(<8 x double> %a0, <8 x double> %a1) nounwind {
 ; AVX2-NEXT:    vcmpltpd %ymm1, %ymm3, %ymm1
 ; AVX2-NEXT:    vcmpltpd %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -475,9 +475,9 @@ define <16 x i1> @test_cmp_v16f32(<16 x float> %a0, <16 x float> %a1) nounwind {
 ; AVX2-NEXT:    vcmpltps %ymm1, %ymm3, %ymm1
 ; AVX2-NEXT:    vcmpltps %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -589,9 +589,9 @@ define <8 x i1> @test_cmp_v8i64(<8 x i64> %a0, <8 x i64> %a1) nounwind {
 ; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -657,9 +657,9 @@ define <16 x i1> @test_cmp_v16i32(<16 x i32> %a0, <16 x i32> %a1) nounwind {
 ; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -946,9 +946,9 @@ define <16 x i1> @test_cmp_v16f64(<16 x double> %a0, <16 x double> %a1) nounwind
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpackssdw %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -1243,9 +1243,9 @@ define <16 x i1> @test_cmp_v16i64(<16 x i64> %a0, <16 x i64> %a1) nounwind {
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpackssdw %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -1453,15 +1453,13 @@ define <64 x i1> @test_cmp_v64i16(<64 x i16> %a0, <64 x i16> %a1) nounwind {
 ; AVX2-NEXT:    movq %rdi, %rax
 ; AVX2-NEXT:    vpcmpgtw %ymm5, %ymm1, %ymm1
 ; AVX2-NEXT:    vpcmpgtw %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm4 = ymm0[2,3],ymm1[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpacksswb %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %ymm0, %ecx
 ; AVX2-NEXT:    vpcmpgtw %ymm7, %ymm3, %ymm0
 ; AVX2-NEXT:    vpcmpgtw %ymm6, %ymm2, %ymm1
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm1[2,3],ymm0[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX2-NEXT:    vpacksswb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpacksswb %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %ymm0, %edx
 ; AVX2-NEXT:    shlq $32, %rdx
 ; AVX2-NEXT:    orq %rcx, %rdx
diff --git a/llvm/test/CodeGen/X86/vector-fshl-128.ll b/llvm/test/CodeGen/X86/vector-fshl-128.ll
index 7cd454f9dc965..d8442048f65ec 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-128.ll
@@ -2905,8 +2905,7 @@ define <16 x i8> @constant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %zmm2, %zmm2
 ; AVX512VL-NEXT:    vpord %zmm1, %zmm2, %zmm1
 ; AVX512VL-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
-; AVX512VL-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip){1to2}, %xmm1, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-fshl-256.ll b/llvm/test/CodeGen/X86/vector-fshl-256.ll
index 860c2d576c728..12feea7658988 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-256.ll
@@ -1513,12 +1513,11 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512VL-NEXT:    vpsrlw %xmm4, %xmm5, %xmm4
 ; AVX512VL-NEXT:    vpsrlw $8, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpbroadcastb %xmm4, %ymm4
-; AVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm2
-; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    vpternlogq $236, %ymm1, %ymm3, %ymm4
+; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm1
+; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm0, %ymm4, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i8:
@@ -2377,8 +2376,7 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip){1to4}, %ymm1, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v32i8:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-512.ll b/llvm/test/CodeGen/X86/vector-fshl-512.ll
index 20cfba41a1c46..6e0cb76398df9 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-512.ll
@@ -893,12 +893,11 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512BW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
 ; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT:    vpsrlw %xmm4, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpsrlw %xmm4, %xmm5, %xmm4
-; AVX512BW-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX512BW-NEXT:    vpbroadcastb %xmm4, %zmm4
-; AVX512BW-NEXT:    vpandq %zmm4, %zmm1, %zmm1
-; AVX512BW-NEXT:    vporq %zmm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vpsrlw %xmm4, %zmm1, %zmm6
+; AVX512BW-NEXT:    vpsrlw %xmm4, %xmm5, %xmm1
+; AVX512BW-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpbroadcastb %xmm1, %zmm1
+; AVX512BW-NEXT:    vpternlogq $236, %zmm6, %zmm3, %zmm1
 ; AVX512BW-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512BW-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
@@ -917,12 +916,11 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512VBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
 ; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VBMI2-NEXT:    vpsrlw %xmm4, %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpsrlw %xmm4, %xmm5, %xmm4
-; AVX512VBMI2-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX512VBMI2-NEXT:    vpbroadcastb %xmm4, %zmm4
-; AVX512VBMI2-NEXT:    vpandq %zmm4, %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vporq %zmm1, %zmm3, %zmm1
+; AVX512VBMI2-NEXT:    vpsrlw %xmm4, %zmm1, %zmm6
+; AVX512VBMI2-NEXT:    vpsrlw %xmm4, %xmm5, %xmm1
+; AVX512VBMI2-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpternlogq $236, %zmm6, %zmm3, %zmm1
 ; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512VBMI2-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512VBMI2-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
@@ -941,12 +939,11 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512VLBW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
 ; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLBW-NEXT:    vpsrlw %xmm4, %zmm1, %zmm1
-; AVX512VLBW-NEXT:    vpsrlw %xmm4, %xmm5, %xmm4
-; AVX512VLBW-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm4, %zmm4
-; AVX512VLBW-NEXT:    vpandq %zmm4, %zmm1, %zmm1
-; AVX512VLBW-NEXT:    vporq %zmm1, %zmm3, %zmm1
+; AVX512VLBW-NEXT:    vpsrlw %xmm4, %zmm1, %zmm6
+; AVX512VLBW-NEXT:    vpsrlw %xmm4, %xmm5, %xmm1
+; AVX512VLBW-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpbroadcastb %xmm1, %zmm1
+; AVX512VLBW-NEXT:    vpternlogq $236, %zmm6, %zmm3, %zmm1
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512VLBW-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
@@ -965,12 +962,11 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512VLVBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
 ; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLVBMI2-NEXT:    vpsrlw %xmm4, %zmm1, %zmm1
-; AVX512VLVBMI2-NEXT:    vpsrlw %xmm4, %xmm5, %xmm4
-; AVX512VLVBMI2-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm4, %zmm4
-; AVX512VLVBMI2-NEXT:    vpandq %zmm4, %zmm1, %zmm1
-; AVX512VLVBMI2-NEXT:    vporq %zmm1, %zmm3, %zmm1
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm4, %zmm1, %zmm6
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm4, %xmm5, %xmm1
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpternlogq $236, %zmm6, %zmm3, %zmm1
 ; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512VLVBMI2-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
@@ -1075,17 +1071,16 @@ define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) nounwin
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = <u,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768>
 ; AVX512F-NEXT:    vpmulhuw %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
-; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm5
-; AVX512F-NEXT:    vpor %ymm2, %ymm5, %ymm2
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm2[1,2,3,4,5,6,7]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512F-NEXT:    vpmulhuw %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpmullw %ymm3, %ymm0, %ymm3
-; AVX512F-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512F-NEXT:    vporq %zmm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpternlogq $216, %zmm2, %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v32i16:
@@ -1093,17 +1088,16 @@ define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) nounwin
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = <u,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768>
 ; AVX512VL-NEXT:    vpmulhuw %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
-; AVX512VL-NEXT:    vpmullw %ymm3, %ymm4, %ymm5
-; AVX512VL-NEXT:    vpor %ymm2, %ymm5, %ymm2
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm2[1,2,3,4,5,6,7]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512VL-NEXT:    vpmulhuw %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpmullw %ymm3, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512VL-NEXT:    vporq %zmm1, %zmm2, %zmm1
+; AVX512VL-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512VL-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512VL-NEXT:    vpternlogq $216, %zmm2, %zmm1, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v32i16:
@@ -1150,49 +1144,47 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512F-NEXT:    vpand %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
 ; AVX512F-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpblendvb %ymm5, %ymm3, %ymm2, %ymm3
-; AVX512F-NEXT:    vpsllw $2, %ymm3, %ymm6
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512F-NEXT:    vpand %ymm7, %ymm6, %ymm6
-; AVX512F-NEXT:    vpaddb %ymm5, %ymm5, %ymm8
-; AVX512F-NEXT:    vpblendvb %ymm8, %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm6
-; AVX512F-NEXT:    vpaddb %ymm8, %ymm8, %ymm9
-; AVX512F-NEXT:    vpblendvb %ymm9, %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
-; AVX512F-NEXT:    vpxor %xmm10, %xmm10, %xmm10
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm11 = ymm6[8],ymm10[8],ymm6[9],ymm10[9],ymm6[10],ymm10[10],ymm6[11],ymm10[11],ymm6[12],ymm10[12],ymm6[13],ymm10[13],ymm6[14],ymm10[14],ymm6[15],ymm10[15],ymm6[24],ymm10[24],ymm6[25],ymm10[25],ymm6[26],ymm10[26],ymm6[27],ymm10[27],ymm6[28],ymm10[28],ymm6[29],ymm10[29],ymm6[30],ymm10[30],ymm6[31],ymm10[31]
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
-; AVX512F-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpmullw %ymm12, %ymm11, %ymm11
-; AVX512F-NEXT:    vpsrlw $8, %ymm11, %ymm11
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm6[0],ymm10[0],ymm6[1],ymm10[1],ymm6[2],ymm10[2],ymm6[3],ymm10[3],ymm6[4],ymm10[4],ymm6[5],ymm10[5],ymm6[6],ymm10[6],ymm6[7],ymm10[7],ymm6[16],ymm10[16],ymm6[17],ymm10[17],ymm6[18],ymm10[18],ymm6[19],ymm10[19],ymm6[20],ymm10[20],ymm6[21],ymm10[21],ymm6[22],ymm10[22],ymm6[23],ymm10[23]
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
-; AVX512F-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpmullw %ymm6, %ymm13, %ymm6
-; AVX512F-NEXT:    vpsrlw $8, %ymm6, %ymm6
-; AVX512F-NEXT:    vpackuswb %ymm11, %ymm6, %ymm6
-; AVX512F-NEXT:    vpor %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [18446744073709551360,18446744073709551360,18446744073709551360,18446744073709551360]
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpblendvb %ymm5, %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsllw $2, %ymm2, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm5, %ymm5, %ymm7
+; AVX512F-NEXT:    vpblendvb %ymm7, %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm7, %ymm7, %ymm8
+; AVX512F-NEXT:    vpblendvb %ymm8, %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm3
 ; AVX512F-NEXT:    vpand %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpblendvb %ymm5, %ymm3, %ymm0, %ymm3
 ; AVX512F-NEXT:    vpsllw $2, %ymm3, %ymm4
-; AVX512F-NEXT:    vpand %ymm7, %ymm4, %ymm4
-; AVX512F-NEXT:    vpblendvb %ymm8, %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpand %ymm6, %ymm4, %ymm4
+; AVX512F-NEXT:    vpblendvb %ymm7, %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm4
-; AVX512F-NEXT:    vpblendvb %ymm9, %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm10[8],ymm1[9],ymm10[9],ymm1[10],ymm10[10],ymm1[11],ymm10[11],ymm1[12],ymm10[12],ymm1[13],ymm10[13],ymm1[14],ymm10[14],ymm1[15],ymm10[15],ymm1[24],ymm10[24],ymm1[25],ymm10[25],ymm1[26],ymm10[26],ymm1[27],ymm10[27],ymm1[28],ymm10[28],ymm1[29],ymm10[29],ymm1[30],ymm10[30],ymm1[31],ymm10[31]
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm12, %ymm4
-; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm10[0],ymm1[1],ymm10[1],ymm1[2],ymm10[2],ymm1[3],ymm10[3],ymm1[4],ymm10[4],ymm1[5],ymm10[5],ymm1[6],ymm10[6],ymm1[7],ymm10[7],ymm1[16],ymm10[16],ymm1[17],ymm10[17],ymm1[18],ymm10[18],ymm1[19],ymm10[19],ymm1[20],ymm10[20],ymm1[21],ymm10[21],ymm1[22],ymm10[22],ymm1[23],ymm10[23]
-; AVX512F-NEXT:    vpmullw %ymm1, %ymm13, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm8, %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15],ymm3[24],ymm4[24],ymm3[25],ymm4[25],ymm3[26],ymm4[26],ymm3[27],ymm4[27],ymm3[28],ymm4[28],ymm3[29],ymm4[29],ymm3[30],ymm4[30],ymm3[31],ymm4[31]
+; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
+; AVX512F-NEXT:    # ymm6 = mem[0,1,0,1]
+; AVX512F-NEXT:    vpmullw %ymm6, %ymm5, %ymm5
+; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[16],ymm4[16],ymm3[17],ymm4[17],ymm3[18],ymm4[18],ymm3[19],ymm4[19],ymm3[20],ymm4[20],ymm3[21],ymm4[21],ymm3[22],ymm4[22],ymm3[23],ymm4[23]
+; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
+; AVX512F-NEXT:    # ymm7 = mem[0,1,0,1]
+; AVX512F-NEXT:    vpmullw %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX512F-NEXT:    vpackuswb %ymm5, %ymm3, %ymm3
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8],ymm4[8],ymm1[9],ymm4[9],ymm1[10],ymm4[10],ymm1[11],ymm4[11],ymm1[12],ymm4[12],ymm1[13],ymm4[13],ymm1[14],ymm4[14],ymm1[15],ymm4[15],ymm1[24],ymm4[24],ymm1[25],ymm4[25],ymm1[26],ymm4[26],ymm1[27],ymm4[27],ymm1[28],ymm4[28],ymm1[29],ymm4[29],ymm1[30],ymm4[30],ymm1[31],ymm4[31]
+; AVX512F-NEXT:    vpmullw %ymm6, %ymm5, %ymm5
+; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[2],ymm4[2],ymm1[3],ymm4[3],ymm1[4],ymm4[4],ymm1[5],ymm4[5],ymm1[6],ymm4[6],ymm1[7],ymm4[7],ymm1[16],ymm4[16],ymm1[17],ymm4[17],ymm1[18],ymm4[18],ymm1[19],ymm4[19],ymm1[20],ymm4[20],ymm1[21],ymm4[21],ymm1[22],ymm4[22],ymm1[23],ymm4[23]
+; AVX512F-NEXT:    vpmullw %ymm7, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512F-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512F-NEXT:    vpblendvb %ymm6, %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpackuswb %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512F-NEXT:    vporq %zmm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vpternlogq $216, {{.*}}(%rip){1to8}, %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v64i8:
@@ -1203,49 +1195,47 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
 ; AVX512VL-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm3, %ymm2, %ymm3
-; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm6
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512VL-NEXT:    vpand %ymm7, %ymm6, %ymm6
-; AVX512VL-NEXT:    vpaddb %ymm5, %ymm5, %ymm8
-; AVX512VL-NEXT:    vpblendvb %ymm8, %ymm6, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm6
-; AVX512VL-NEXT:    vpaddb %ymm8, %ymm8, %ymm9
-; AVX512VL-NEXT:    vpblendvb %ymm9, %ymm6, %ymm3, %ymm3
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
-; AVX512VL-NEXT:    vpxor %xmm10, %xmm10, %xmm10
-; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm11 = ymm6[8],ymm10[8],ymm6[9],ymm10[9],ymm6[10],ymm10[10],ymm6[11],ymm10[11],ymm6[12],ymm10[12],ymm6[13],ymm10[13],ymm6[14],ymm10[14],ymm6[15],ymm10[15],ymm6[24],ymm10[24],ymm6[25],ymm10[25],ymm6[26],ymm10[26],ymm6[27],ymm10[27],ymm6[28],ymm10[28],ymm6[29],ymm10[29],ymm6[30],ymm10[30],ymm6[31],ymm10[31]
-; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
-; AVX512VL-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpmullw %ymm12, %ymm11, %ymm11
-; AVX512VL-NEXT:    vpsrlw $8, %ymm11, %ymm11
-; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm6[0],ymm10[0],ymm6[1],ymm10[1],ymm6[2],ymm10[2],ymm6[3],ymm10[3],ymm6[4],ymm10[4],ymm6[5],ymm10[5],ymm6[6],ymm10[6],ymm6[7],ymm10[7],ymm6[16],ymm10[16],ymm6[17],ymm10[17],ymm6[18],ymm10[18],ymm6[19],ymm10[19],ymm6[20],ymm10[20],ymm6[21],ymm10[21],ymm6[22],ymm10[22],ymm6[23],ymm10[23]
-; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
-; AVX512VL-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpmullw %ymm6, %ymm13, %ymm6
-; AVX512VL-NEXT:    vpsrlw $8, %ymm6, %ymm6
-; AVX512VL-NEXT:    vpackuswb %ymm11, %ymm6, %ymm6
-; AVX512VL-NEXT:    vpor %ymm6, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [18446744073709551360,18446744073709551360,18446744073709551360,18446744073709551360]
-; AVX512VL-NEXT:    vpblendvb %ymm6, %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsllw $2, %ymm2, %ymm3
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpaddb %ymm5, %ymm5, %ymm7
+; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm3
+; AVX512VL-NEXT:    vpaddb %ymm7, %ymm7, %ymm8
+; AVX512VL-NEXT:    vpblendvb %ymm8, %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm3
 ; AVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm3, %ymm0, %ymm3
 ; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm4
-; AVX512VL-NEXT:    vpand %ymm7, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpblendvb %ymm8, %ymm4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpand %ymm6, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm4, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm4
-; AVX512VL-NEXT:    vpblendvb %ymm9, %ymm4, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm10[8],ymm1[9],ymm10[9],ymm1[10],ymm10[10],ymm1[11],ymm10[11],ymm1[12],ymm10[12],ymm1[13],ymm10[13],ymm1[14],ymm10[14],ymm1[15],ymm10[15],ymm1[24],ymm10[24],ymm1[25],ymm10[25],ymm1[26],ymm10[26],ymm1[27],ymm10[27],ymm1[28],ymm10[28],ymm1[29],ymm10[29],ymm1[30],ymm10[30],ymm1[31],ymm10[31]
-; AVX512VL-NEXT:    vpmullw %ymm4, %ymm12, %ymm4
-; AVX512VL-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm10[0],ymm1[1],ymm10[1],ymm1[2],ymm10[2],ymm1[3],ymm10[3],ymm1[4],ymm10[4],ymm1[5],ymm10[5],ymm1[6],ymm10[6],ymm1[7],ymm10[7],ymm1[16],ymm10[16],ymm1[17],ymm10[17],ymm1[18],ymm10[18],ymm1[19],ymm10[19],ymm1[20],ymm10[20],ymm1[21],ymm10[21],ymm1[22],ymm10[22],ymm1[23],ymm10[23]
-; AVX512VL-NEXT:    vpmullw %ymm1, %ymm13, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm8, %ymm4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15],ymm3[24],ymm4[24],ymm3[25],ymm4[25],ymm3[26],ymm4[26],ymm3[27],ymm4[27],ymm3[28],ymm4[28],ymm3[29],ymm4[29],ymm3[30],ymm4[30],ymm3[31],ymm4[31]
+; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
+; AVX512VL-NEXT:    # ymm6 = mem[0,1,0,1]
+; AVX512VL-NEXT:    vpmullw %ymm6, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpsrlw $8, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[16],ymm4[16],ymm3[17],ymm4[17],ymm3[18],ymm4[18],ymm3[19],ymm4[19],ymm3[20],ymm4[20],ymm3[21],ymm4[21],ymm3[22],ymm4[22],ymm3[23],ymm4[23]
+; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
+; AVX512VL-NEXT:    # ymm7 = mem[0,1,0,1]
+; AVX512VL-NEXT:    vpmullw %ymm7, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpackuswb %ymm5, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8],ymm4[8],ymm1[9],ymm4[9],ymm1[10],ymm4[10],ymm1[11],ymm4[11],ymm1[12],ymm4[12],ymm1[13],ymm4[13],ymm1[14],ymm4[14],ymm1[15],ymm4[15],ymm1[24],ymm4[24],ymm1[25],ymm4[25],ymm1[26],ymm4[26],ymm1[27],ymm4[27],ymm1[28],ymm4[28],ymm1[29],ymm4[29],ymm1[30],ymm4[30],ymm1[31],ymm4[31]
+; AVX512VL-NEXT:    vpmullw %ymm6, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpsrlw $8, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[2],ymm4[2],ymm1[3],ymm4[3],ymm1[4],ymm4[4],ymm1[5],ymm4[5],ymm1[6],ymm4[6],ymm1[7],ymm4[7],ymm1[16],ymm4[16],ymm1[17],ymm4[17],ymm1[18],ymm4[18],ymm1[19],ymm4[19],ymm1[20],ymm4[20],ymm1[21],ymm4[21],ymm1[22],ymm4[22],ymm1[23],ymm4[23]
+; AVX512VL-NEXT:    vpmullw %ymm7, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm6, %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpackuswb %ymm5, %ymm1, %ymm1
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512VL-NEXT:    vporq %zmm1, %zmm2, %zmm1
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip){1to8}, %zmm1, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v64i8:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
index e8fb824076f2f..655b6e4c25048 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
@@ -446,12 +446,11 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpternlogq $248, {{.*}}(%rip), %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_funnnel_v32i8:
@@ -833,12 +832,11 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512VL-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm4, %xmm1
-; AVX512VL-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm4, %xmm0
+; AVX512VL-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpbroadcastb %xmm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $236, %ymm3, %ymm2, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i8:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
index 8d11e104bf8d4..49f229bf1d67b 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
@@ -550,12 +550,11 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind
 ; AVX512BW-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
 ; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpsrlw %xmm1, %xmm5, %xmm1
-; AVX512BW-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %zmm1
-; AVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpsrlw %xmm1, %xmm5, %xmm0
+; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpbroadcastb %xmm0, %zmm0
+; AVX512BW-NEXT:    vpternlogq $236, %zmm2, %zmm3, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v64i8:
@@ -573,12 +572,11 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind
 ; AVX512VLBW-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
 ; AVX512VLBW-NEXT:    vpand %xmm2, %xmm1, %xmm1
 ; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLBW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpsrlw %xmm1, %xmm5, %xmm1
-; AVX512VLBW-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm1, %zmm1
-; AVX512VLBW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512VLBW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm2
+; AVX512VLBW-NEXT:    vpsrlw %xmm1, %xmm5, %xmm0
+; AVX512VLBW-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpbroadcastb %xmm0, %zmm0
+; AVX512VLBW-NEXT:    vpternlogq $236, %zmm2, %zmm3, %zmm0
 ; AVX512VLBW-NEXT:    retq
   %splat = shufflevector <64 x i8> %amt, <64 x i8> undef, <64 x i32> zeroinitializer
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> %splat)
diff --git a/llvm/test/CodeGen/X86/vector-fshr-128.ll b/llvm/test/CodeGen/X86/vector-fshr-128.ll
index 35ba0e8125856..b7cc39a32d718 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-128.ll
@@ -2652,8 +2652,7 @@ define <16 x i8> @constant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpord %zmm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
-; AVX512VL-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX512VL-NEXT:    vpternlogq $228, {{.*}}(%rip){1to2}, %xmm1, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll
index e40e3cdfbd65a..bbeaed5cc725c 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll
@@ -1502,25 +1502,24 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512VL-LABEL: splatvar_funnnel_v32i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VL-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX512VL-NEXT:    vpsllw %xmm3, %xmm4, %xmm3
+; AVX512VL-NEXT:    vpbroadcastb %xmm3, %ymm3
+; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm3, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX512VL-NEXT:    vpsrlw %xmm3, %xmm5, %xmm3
+; AVX512VL-NEXT:    vpsrlw %xmm3, %ymm1, %ymm5
+; AVX512VL-NEXT:    vpsrlw %xmm3, %xmm4, %xmm3
 ; AVX512VL-NEXT:    vpsrlw $8, %xmm3, %xmm3
 ; AVX512VL-NEXT:    vpbroadcastb %xmm3, %ymm3
-; AVX512VL-NEXT:    vpand %ymm3, %ymm4, %ymm3
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VL-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsllw %xmm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsllw %xmm4, %xmm5, %xmm4
-; AVX512VL-NEXT:    vpbroadcastb %xmm4, %ymm4
-; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm2
-; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $236, %ymm5, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm0
+; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm0, %ymm1, %ymm3, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i8:
@@ -2085,8 +2084,7 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpackuswb %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    vpternlogq $228, {{.*}}(%rip){1to4}, %ymm1, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v32i8:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-512.ll b/llvm/test/CodeGen/X86/vector-fshr-512.ll
index fd59d40cefc88..c89782bc359cc 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-512.ll
@@ -872,21 +872,20 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512BW-LABEL: splatvar_funnnel_v64i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT:    vpsrlw %xmm3, %zmm1, %zmm4
-; AVX512BW-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX512BW-NEXT:    vpsrlw %xmm3, %xmm5, %xmm3
-; AVX512BW-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512BW-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsllw %xmm3, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX512BW-NEXT:    vpsllw %xmm3, %xmm4, %xmm3
 ; AVX512BW-NEXT:    vpbroadcastb %xmm3, %zmm3
-; AVX512BW-NEXT:    vpandq %zmm3, %zmm4, %zmm3
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512BW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT:    vpsllw %xmm4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpsllw %xmm4, %xmm5, %xmm4
-; AVX512BW-NEXT:    vpbroadcastb %xmm4, %zmm4
-; AVX512BW-NEXT:    vpandq %zmm4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vporq %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm3
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsrlw %xmm0, %zmm1, %zmm5
+; AVX512BW-NEXT:    vpsrlw %xmm0, %xmm4, %xmm0
+; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpbroadcastb %xmm0, %zmm0
+; AVX512BW-NEXT:    vpternlogq $236, %zmm5, %zmm3, %zmm0
 ; AVX512BW-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512BW-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512BW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
@@ -895,21 +894,20 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v64i8:
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VBMI2-NEXT:    vpsrlw %xmm3, %zmm1, %zmm4
-; AVX512VBMI2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX512VBMI2-NEXT:    vpsrlw %xmm3, %xmm5, %xmm3
-; AVX512VBMI2-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm3, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpsllw %xmm3, %xmm4, %xmm3
 ; AVX512VBMI2-NEXT:    vpbroadcastb %xmm3, %zmm3
-; AVX512VBMI2-NEXT:    vpandq %zmm3, %zmm4, %zmm3
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VBMI2-NEXT:    vpsllw %xmm4, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpsllw %xmm4, %xmm5, %xmm4
-; AVX512VBMI2-NEXT:    vpbroadcastb %xmm4, %zmm4
-; AVX512VBMI2-NEXT:    vpandq %zmm4, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vporq %zmm3, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpandq %zmm3, %zmm0, %zmm3
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm0, %zmm1, %zmm5
+; AVX512VBMI2-NEXT:    vpsrlw %xmm0, %xmm4, %xmm0
+; AVX512VBMI2-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpternlogq $236, %zmm5, %zmm3, %zmm0
 ; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512VBMI2-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
@@ -918,21 +916,20 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512VLBW-LABEL: splatvar_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLBW-NEXT:    vpsrlw %xmm3, %zmm1, %zmm4
-; AVX512VLBW-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX512VLBW-NEXT:    vpsrlw %xmm3, %xmm5, %xmm3
-; AVX512VLBW-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLBW-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpsllw %xmm3, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX512VLBW-NEXT:    vpsllw %xmm3, %xmm4, %xmm3
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm3, %zmm3
-; AVX512VLBW-NEXT:    vpandq %zmm3, %zmm4, %zmm3
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLBW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLBW-NEXT:    vpsllw %xmm4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpsllw %xmm4, %xmm5, %xmm4
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm4, %zmm4
-; AVX512VLBW-NEXT:    vpandq %zmm4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vporq %zmm3, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpandq %zmm3, %zmm0, %zmm3
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpsrlw %xmm0, %zmm1, %zmm5
+; AVX512VLBW-NEXT:    vpsrlw %xmm0, %xmm4, %xmm0
+; AVX512VLBW-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpbroadcastb %xmm0, %zmm0
+; AVX512VLBW-NEXT:    vpternlogq $236, %zmm5, %zmm3, %zmm0
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512VLBW-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
@@ -941,21 +938,20 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512VLVBMI2-LABEL: splatvar_funnnel_v64i8:
 ; AVX512VLVBMI2:       # %bb.0:
 ; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLVBMI2-NEXT:    vpsrlw %xmm3, %zmm1, %zmm4
-; AVX512VLVBMI2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX512VLVBMI2-NEXT:    vpsrlw %xmm3, %xmm5, %xmm3
-; AVX512VLVBMI2-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm3, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm3, %xmm4, %xmm3
 ; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm3, %zmm3
-; AVX512VLVBMI2-NEXT:    vpandq %zmm3, %zmm4, %zmm3
-; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLVBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLVBMI2-NEXT:    vpsllw %xmm4, %zmm0, %zmm0
-; AVX512VLVBMI2-NEXT:    vpsllw %xmm4, %xmm5, %xmm4
-; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm4, %zmm4
-; AVX512VLVBMI2-NEXT:    vpandq %zmm4, %zmm0, %zmm0
-; AVX512VLVBMI2-NEXT:    vporq %zmm3, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpandq %zmm3, %zmm0, %zmm3
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm0, %zmm1, %zmm5
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm0, %xmm4, %xmm0
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $236, %zmm5, %zmm3, %zmm0
 ; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512VLVBMI2-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
@@ -1062,36 +1058,34 @@ define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) nounwin
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = <u,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2>
-; AVX512F-NEXT:    vpmulhuw %ymm3, %ymm2, %ymm4
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
-; AVX512F-NEXT:    vpmullw %ymm3, %ymm5, %ymm5
-; AVX512F-NEXT:    vpor %ymm4, %ymm5, %ymm4
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1,2,3,4,5,6,7]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
+; AVX512F-NEXT:    vpmulhuw %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpmulhuw %ymm3, %ymm1, %ymm4
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm4, %zmm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1,2,3,4,5,6,7]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
+; AVX512F-NEXT:    vporq %zmm2, %zmm0, %zmm2
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpternlogq $202, %zmm1, %zmm2, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = <u,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2>
-; AVX512VL-NEXT:    vpmulhuw %ymm3, %ymm2, %ymm4
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
-; AVX512VL-NEXT:    vpmullw %ymm3, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpor %ymm4, %ymm5, %ymm4
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1,2,3,4,5,6,7]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
+; AVX512VL-NEXT:    vpmulhuw %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpmulhuw %ymm3, %ymm1, %ymm4
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm4, %zmm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX512VL-NEXT:    vpmullw %ymm3, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1,2,3,4,5,6,7]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
+; AVX512VL-NEXT:    vporq %zmm2, %zmm0, %zmm2
+; AVX512VL-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512VL-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
+; AVX512VL-NEXT:    vpternlogq $202, %zmm1, %zmm2, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v32i16:
@@ -1147,22 +1141,6 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm3
 ; AVX512F-NEXT:    vpaddb %ymm7, %ymm7, %ymm8
 ; AVX512F-NEXT:    vpblendvb %ymm8, %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
-; AVX512F-NEXT:    vpxor %xmm9, %xmm9, %xmm9
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm3[8],ymm9[8],ymm3[9],ymm9[9],ymm3[10],ymm9[10],ymm3[11],ymm9[11],ymm3[12],ymm9[12],ymm3[13],ymm9[13],ymm3[14],ymm9[14],ymm3[15],ymm9[15],ymm3[24],ymm9[24],ymm3[25],ymm9[25],ymm3[26],ymm9[26],ymm3[27],ymm9[27],ymm3[28],ymm9[28],ymm3[29],ymm9[29],ymm3[30],ymm9[30],ymm3[31],ymm9[31]
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
-; AVX512F-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpmullw %ymm11, %ymm10, %ymm10
-; AVX512F-NEXT:    vpsrlw $8, %ymm10, %ymm10
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm12 = ymm3[0],ymm9[0],ymm3[1],ymm9[1],ymm3[2],ymm9[2],ymm3[3],ymm9[3],ymm3[4],ymm9[4],ymm3[5],ymm9[5],ymm3[6],ymm9[6],ymm3[7],ymm9[7],ymm3[16],ymm9[16],ymm3[17],ymm9[17],ymm3[18],ymm9[18],ymm3[19],ymm9[19],ymm3[20],ymm9[20],ymm3[21],ymm9[21],ymm3[22],ymm9[22],ymm3[23],ymm9[23]
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
-; AVX512F-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpmullw %ymm13, %ymm12, %ymm12
-; AVX512F-NEXT:    vpsrlw $8, %ymm12, %ymm12
-; AVX512F-NEXT:    vpackuswb %ymm10, %ymm12, %ymm10
-; AVX512F-NEXT:    vpor %ymm2, %ymm10, %ymm2
-; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [18446744073709551360,18446744073709551360,18446744073709551360,18446744073709551360]
-; AVX512F-NEXT:    vpblendvb %ymm10, %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm3
 ; AVX512F-NEXT:    vpand %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpblendvb %ymm5, %ymm3, %ymm0, %ymm0
@@ -1171,16 +1149,30 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512F-NEXT:    vpblendvb %ymm7, %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
 ; AVX512F-NEXT:    vpblendvb %ymm8, %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm9[8],ymm1[9],ymm9[9],ymm1[10],ymm9[10],ymm1[11],ymm9[11],ymm1[12],ymm9[12],ymm1[13],ymm9[13],ymm1[14],ymm9[14],ymm1[15],ymm9[15],ymm1[24],ymm9[24],ymm1[25],ymm9[25],ymm1[26],ymm9[26],ymm1[27],ymm9[27],ymm1[28],ymm9[28],ymm1[29],ymm9[29],ymm1[30],ymm9[30],ymm1[31],ymm9[31]
-; AVX512F-NEXT:    vpmullw %ymm3, %ymm11, %ymm3
-; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm1[0],ymm9[0],ymm1[1],ymm9[1],ymm1[2],ymm9[2],ymm1[3],ymm9[3],ymm1[4],ymm9[4],ymm1[5],ymm9[5],ymm1[6],ymm9[6],ymm1[7],ymm9[7],ymm1[16],ymm9[16],ymm1[17],ymm9[17],ymm1[18],ymm9[18],ymm1[19],ymm9[19],ymm1[20],ymm9[20],ymm1[21],ymm9[21],ymm1[22],ymm9[22],ymm1[23],ymm9[23]
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm13, %ymm4
-; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpblendvb %ymm10, %ymm0, %ymm1, %ymm0
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31]
+; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
+; AVX512F-NEXT:    # ymm5 = mem[0,1,0,1]
+; AVX512F-NEXT:    vpmullw %ymm5, %ymm4, %ymm4
+; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[16],ymm3[16],ymm2[17],ymm3[17],ymm2[18],ymm3[18],ymm2[19],ymm3[19],ymm2[20],ymm3[20],ymm2[21],ymm3[21],ymm2[22],ymm3[22],ymm2[23],ymm3[23]
+; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
+; AVX512F-NEXT:    # ymm6 = mem[0,1,0,1]
+; AVX512F-NEXT:    vpmullw %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512F-NEXT:    vpackuswb %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15],ymm1[24],ymm3[24],ymm1[25],ymm3[25],ymm1[26],ymm3[26],ymm1[27],ymm3[27],ymm1[28],ymm3[28],ymm1[29],ymm3[29],ymm1[30],ymm3[30],ymm1[31],ymm3[31]
+; AVX512F-NEXT:    vpmullw %ymm5, %ymm4, %ymm4
+; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[4],ymm3[4],ymm1[5],ymm3[5],ymm1[6],ymm3[6],ymm1[7],ymm3[7],ymm1[16],ymm3[16],ymm1[17],ymm3[17],ymm1[18],ymm3[18],ymm1[19],ymm3[19],ymm1[20],ymm3[20],ymm1[21],ymm3[21],ymm1[22],ymm3[22],ymm1[23],ymm3[23]
+; AVX512F-NEXT:    vpmullw %ymm6, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX512F-NEXT:    vpackuswb %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512F-NEXT:    vporq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpternlogq $228, {{.*}}(%rip){1to8}, %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v64i8:
@@ -1200,22 +1192,6 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm7, %ymm7, %ymm8
 ; AVX512VL-NEXT:    vpblendvb %ymm8, %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
-; AVX512VL-NEXT:    vpxor %xmm9, %xmm9, %xmm9
-; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm3[8],ymm9[8],ymm3[9],ymm9[9],ymm3[10],ymm9[10],ymm3[11],ymm9[11],ymm3[12],ymm9[12],ymm3[13],ymm9[13],ymm3[14],ymm9[14],ymm3[15],ymm9[15],ymm3[24],ymm9[24],ymm3[25],ymm9[25],ymm3[26],ymm9[26],ymm3[27],ymm9[27],ymm3[28],ymm9[28],ymm3[29],ymm9[29],ymm3[30],ymm9[30],ymm3[31],ymm9[31]
-; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
-; AVX512VL-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpmullw %ymm11, %ymm10, %ymm10
-; AVX512VL-NEXT:    vpsrlw $8, %ymm10, %ymm10
-; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm12 = ymm3[0],ymm9[0],ymm3[1],ymm9[1],ymm3[2],ymm9[2],ymm3[3],ymm9[3],ymm3[4],ymm9[4],ymm3[5],ymm9[5],ymm3[6],ymm9[6],ymm3[7],ymm9[7],ymm3[16],ymm9[16],ymm3[17],ymm9[17],ymm3[18],ymm9[18],ymm3[19],ymm9[19],ymm3[20],ymm9[20],ymm3[21],ymm9[21],ymm3[22],ymm9[22],ymm3[23],ymm9[23]
-; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
-; AVX512VL-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpmullw %ymm13, %ymm12, %ymm12
-; AVX512VL-NEXT:    vpsrlw $8, %ymm12, %ymm12
-; AVX512VL-NEXT:    vpackuswb %ymm10, %ymm12, %ymm10
-; AVX512VL-NEXT:    vpor %ymm2, %ymm10, %ymm2
-; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [18446744073709551360,18446744073709551360,18446744073709551360,18446744073709551360]
-; AVX512VL-NEXT:    vpblendvb %ymm10, %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm3
 ; AVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm3, %ymm0, %ymm0
@@ -1224,16 +1200,30 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512VL-NEXT:    vpblendvb %ymm7, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
 ; AVX512VL-NEXT:    vpblendvb %ymm8, %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm9[8],ymm1[9],ymm9[9],ymm1[10],ymm9[10],ymm1[11],ymm9[11],ymm1[12],ymm9[12],ymm1[13],ymm9[13],ymm1[14],ymm9[14],ymm1[15],ymm9[15],ymm1[24],ymm9[24],ymm1[25],ymm9[25],ymm1[26],ymm9[26],ymm1[27],ymm9[27],ymm1[28],ymm9[28],ymm1[29],ymm9[29],ymm1[30],ymm9[30],ymm1[31],ymm9[31]
-; AVX512VL-NEXT:    vpmullw %ymm3, %ymm11, %ymm3
-; AVX512VL-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm1[0],ymm9[0],ymm1[1],ymm9[1],ymm1[2],ymm9[2],ymm1[3],ymm9[3],ymm1[4],ymm9[4],ymm1[5],ymm9[5],ymm1[6],ymm9[6],ymm1[7],ymm9[7],ymm1[16],ymm9[16],ymm1[17],ymm9[17],ymm1[18],ymm9[18],ymm1[19],ymm9[19],ymm1[20],ymm9[20],ymm1[21],ymm9[21],ymm1[22],ymm9[22],ymm1[23],ymm9[23]
-; AVX512VL-NEXT:    vpmullw %ymm4, %ymm13, %ymm4
-; AVX512VL-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpackuswb %ymm3, %ymm4, %ymm3
-; AVX512VL-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpblendvb %ymm10, %ymm0, %ymm1, %ymm0
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31]
+; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
+; AVX512VL-NEXT:    # ymm5 = mem[0,1,0,1]
+; AVX512VL-NEXT:    vpmullw %ymm5, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpsrlw $8, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[16],ymm3[16],ymm2[17],ymm3[17],ymm2[18],ymm3[18],ymm2[19],ymm3[19],ymm2[20],ymm3[20],ymm2[21],ymm3[21],ymm2[22],ymm3[22],ymm2[23],ymm3[23]
+; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
+; AVX512VL-NEXT:    # ymm6 = mem[0,1,0,1]
+; AVX512VL-NEXT:    vpmullw %ymm6, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpackuswb %ymm4, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15],ymm1[24],ymm3[24],ymm1[25],ymm3[25],ymm1[26],ymm3[26],ymm1[27],ymm3[27],ymm1[28],ymm3[28],ymm1[29],ymm3[29],ymm1[30],ymm3[30],ymm1[31],ymm3[31]
+; AVX512VL-NEXT:    vpmullw %ymm5, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpsrlw $8, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[4],ymm3[4],ymm1[5],ymm3[5],ymm1[6],ymm3[6],ymm1[7],ymm3[7],ymm1[16],ymm3[16],ymm1[17],ymm3[17],ymm1[18],ymm3[18],ymm1[19],ymm3[19],ymm1[20],ymm3[20],ymm1[21],ymm3[21],ymm1[22],ymm3[22],ymm1[23],ymm3[23]
+; AVX512VL-NEXT:    vpmullw %ymm6, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpackuswb %ymm4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512VL-NEXT:    vporq %zmm2, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpternlogq $228, {{.*}}(%rip){1to8}, %zmm1, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v64i8:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
index 930795283a24e..61c45a118e47a 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
@@ -487,12 +487,11 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpternlogq $248, {{.*}}(%rip), %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_funnnel_v32i8:
@@ -908,12 +907,11 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512VL-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm4, %xmm1
-; AVX512VL-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm4, %xmm0
+; AVX512VL-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpbroadcastb %xmm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $236, %ymm3, %ymm2, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i8:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
index 79ffaaf53859f..d642e513c49bf 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
@@ -534,47 +534,45 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind
 ; AVX512BW-LABEL: splatvar_funnnel_v64i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT:    vpsrlw %xmm3, %zmm0, %zmm4
+; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512BW-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsllw %xmm2, %zmm0, %zmm4
 ; AVX512BW-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX512BW-NEXT:    vpsrlw %xmm3, %xmm5, %xmm3
-; AVX512BW-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpbroadcastb %xmm3, %zmm3
-; AVX512BW-NEXT:    vpandq %zmm3, %zmm4, %zmm3
-; AVX512BW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512BW-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllw %xmm2, %xmm5, %xmm2
+; AVX512BW-NEXT:    vpbroadcastb %xmm2, %zmm2
+; AVX512BW-NEXT:    vpandq %zmm2, %zmm4, %zmm2
+; AVX512BW-NEXT:    vpand %xmm3, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT:    vpsllw %xmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpsllw %xmm1, %xmm5, %xmm1
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %zmm1
-; AVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vporq %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm3
+; AVX512BW-NEXT:    vpsrlw %xmm1, %xmm5, %xmm0
+; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpbroadcastb %xmm0, %zmm0
+; AVX512BW-NEXT:    vpternlogq $236, %zmm3, %zmm2, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm1, %xmm1
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VLBW-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLBW-NEXT:    vpsrlw %xmm3, %zmm0, %zmm4
+; AVX512VLBW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLBW-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLBW-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpsllw %xmm2, %zmm0, %zmm4
 ; AVX512VLBW-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX512VLBW-NEXT:    vpsrlw %xmm3, %xmm5, %xmm3
-; AVX512VLBW-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm3, %zmm3
-; AVX512VLBW-NEXT:    vpandq %zmm3, %zmm4, %zmm3
-; AVX512VLBW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VLBW-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; AVX512VLBW-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpsllw %xmm2, %xmm5, %xmm2
+; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %zmm2
+; AVX512VLBW-NEXT:    vpandq %zmm2, %zmm4, %zmm2
+; AVX512VLBW-NEXT:    vpand %xmm3, %xmm1, %xmm1
 ; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLBW-NEXT:    vpsllw %xmm1, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpsllw %xmm1, %xmm5, %xmm1
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm1, %zmm1
-; AVX512VLBW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vporq %zmm3, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm3
+; AVX512VLBW-NEXT:    vpsrlw %xmm1, %xmm5, %xmm0
+; AVX512VLBW-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpbroadcastb %xmm0, %zmm0
+; AVX512VLBW-NEXT:    vpternlogq $236, %zmm3, %zmm2, %zmm0
 ; AVX512VLBW-NEXT:    retq
   %splat = shufflevector <64 x i8> %amt, <64 x i8> undef, <64 x i32> zeroinitializer
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> %splat)
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
index b48a67c129f0d..c3fa89445c2f0 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
@@ -185,9 +185,8 @@ define <64 x i8> @test_div7_64i8(<64 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm2, %zmm1
 ; AVX512BW-NEXT:    vpaddb %zmm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vpsrlw $2, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512BW-NEXT:    vpxorq %zmm2, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpternlogq $108, {{.*}}(%rip), %zmm2, %zmm1
 ; AVX512BW-NEXT:    vpsrlw $7, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpaddb %zmm0, %zmm1, %zmm0
@@ -539,9 +538,8 @@ define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm2, %zmm1
 ; AVX512BW-NEXT:    vpaddb %zmm0, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpsrlw $2, %zmm1, %zmm2
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm2, %zmm2
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512BW-NEXT:    vpxorq %zmm3, %zmm2, %zmm2
+; AVX512BW-NEXT:    vpternlogq $108, {{.*}}(%rip), %zmm3, %zmm2
 ; AVX512BW-NEXT:    vpsrlw $7, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpaddb %zmm1, %zmm2, %zmm1
diff --git a/llvm/test/CodeGen/X86/vector-pack-128.ll b/llvm/test/CodeGen/X86/vector-pack-128.ll
new file mode 100644
index 0000000000000..9b0bbac0199de
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-pack-128.ll
@@ -0,0 +1,326 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2   | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE,SSE4
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx    | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2   | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512f  | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BW
+
+; trunc(concat(x,y)) -> pack
+
+define <8 x i16> @trunc_concat_packssdw_128(<4 x i32> %a0, <4 x i32> %a1) nounwind {
+; SSE-LABEL: trunc_concat_packssdw_128:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psrad $17, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE-NEXT:    packssdw %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: trunc_concat_packssdw_128:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsrad $17, %xmm0, %xmm0
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_concat_packssdw_128:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrad $17, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [15,15,15,15]
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_concat_packssdw_128:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrad $17, %xmm0, %xmm0
+; AVX512-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm1, %xmm1
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %1 = ashr <4 x i32> %a0, <i32 17, i32 17, i32 17, i32 17>
+  %2 = and  <4 x i32> %a1, <i32 15, i32 15, i32 15, i32 15>
+  %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %4 = trunc <8 x i32> %3 to <8 x i16>
+  ret <8 x i16> %4
+}
+
+define <8 x i16> @trunc_concat_packusdw_128(<4 x i32> %a0, <4 x i32> %a1) nounwind {
+; SSE2-LABEL: trunc_concat_packusdw_128:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psrld $17, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: trunc_concat_packusdw_128:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    psrld $17, %xmm0
+; SSE4-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE4-NEXT:    packusdw %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: trunc_concat_packusdw_128:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsrld $17, %xmm0, %xmm0
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_concat_packusdw_128:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrld $17, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [15,15,15,15]
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_concat_packusdw_128:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrld $17, %xmm0, %xmm0
+; AVX512-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm1, %xmm1
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %1 = lshr <4 x i32> %a0, <i32 17, i32 17, i32 17, i32 17>
+  %2 = and  <4 x i32> %a1, <i32 15, i32 15, i32 15, i32 15>
+  %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %4 = trunc <8 x i32> %3 to <8 x i16>
+  ret <8 x i16> %4
+}
+
+define <16 x i8> @trunc_concat_packsswb_128(<8 x i16> %a0, <8 x i16> %a1) nounwind {
+; SSE-LABEL: trunc_concat_packsswb_128:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psraw $15, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE-NEXT:    packsswb %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: trunc_concat_packsswb_128:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsraw $15, %xmm0, %xmm0
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_concat_packsswb_128:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsraw $15, %xmm0, %xmm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_concat_packsswb_128:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsraw $15, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_concat_packsswb_128:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsraw $15, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %1 = ashr <8 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %2 = and  <8 x i16> %a1, <i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1>
+  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %4 = trunc <16 x i16> %3 to <16 x i8>
+  ret <16 x i8> %4
+}
+
+define <16 x i8> @trunc_concat_packuswb_128(<8 x i16> %a0, <8 x i16> %a1) nounwind {
+; SSE-LABEL: trunc_concat_packuswb_128:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psrlw $15, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE-NEXT:    packuswb %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: trunc_concat_packuswb_128:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsrlw $15, %xmm0, %xmm0
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_concat_packuswb_128:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrlw $15, %xmm0, %xmm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_concat_packuswb_128:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $15, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_concat_packuswb_128:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $15, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %1 = lshr <8 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %2 = and  <8 x i16> %a1, <i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1>
+  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %4 = trunc <16 x i16> %3 to <16 x i8>
+  ret <16 x i8> %4
+}
+
+; concat(trunc(x),trunc(y)) -> pack
+
+define <8 x i16> @concat_trunc_packssdw_128(<4 x i32> %a0, <4 x i32> %a1) nounwind {
+; SSE2-LABEL: concat_trunc_packssdw_128:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psrad $17, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    packssdw %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: concat_trunc_packssdw_128:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    psrad $17, %xmm0
+; SSE4-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE4-NEXT:    packssdw %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: concat_trunc_packssdw_128:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsrad $17, %xmm0, %xmm0
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: concat_trunc_packssdw_128:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrad $17, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [15,15,15,15]
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: concat_trunc_packssdw_128:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrad $17, %xmm0, %xmm0
+; AVX512-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm1, %xmm1
+; AVX512-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+  %1 = ashr <4 x i32> %a0, <i32 17, i32 17, i32 17, i32 17>
+  %2 = and  <4 x i32> %a1, <i32 15, i32 15, i32 15, i32 15>
+  %3 = trunc <4 x i32> %1 to <4 x i16>
+  %4 = trunc <4 x i32> %2 to <4 x i16>
+  %5 = shufflevector <4 x i16> %3, <4 x i16> %4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %5
+}
+
+define <8 x i16> @concat_trunc_packusdw_128(<4 x i32> %a0, <4 x i32> %a1) nounwind {
+; SSE2-LABEL: concat_trunc_packusdw_128:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psrld $17, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    packssdw %xmm0, %xmm0
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: concat_trunc_packusdw_128:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    psrld $17, %xmm0
+; SSE4-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE4-NEXT:    packusdw %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: concat_trunc_packusdw_128:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsrld $17, %xmm0, %xmm0
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: concat_trunc_packusdw_128:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrld $17, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [15,15,15,15]
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: concat_trunc_packusdw_128:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrld $17, %xmm0, %xmm0
+; AVX512-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm1, %xmm1
+; AVX512-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+  %1 = lshr <4 x i32> %a0, <i32 17, i32 17, i32 17, i32 17>
+  %2 = and  <4 x i32> %a1, <i32 15, i32 15, i32 15, i32 15>
+  %3 = trunc <4 x i32> %1 to <4 x i16>
+  %4 = trunc <4 x i32> %2 to <4 x i16>
+  %5 = shufflevector <4 x i16> %3, <4 x i16> %4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i16> %5
+}
+
+define <16 x i8> @concat_trunc_packsswb_128(<8 x i16> %a0, <8 x i16> %a1) nounwind {
+; SSE-LABEL: concat_trunc_packsswb_128:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psraw $15, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE-NEXT:    packsswb %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: concat_trunc_packsswb_128:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsraw $15, %xmm0, %xmm0
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = ashr <8 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %2 = and  <8 x i16> %a1, <i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1>
+  %3 = trunc <8 x i16> %1 to <8 x i8>
+  %4 = trunc <8 x i16> %2 to <8 x i8>
+  %5 = shufflevector <8 x i8> %3, <8 x i8> %4, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %5
+}
+
+define <16 x i8> @concat_trunc_packuswb_128(<8 x i16> %a0, <8 x i16> %a1) nounwind {
+; SSE-LABEL: concat_trunc_packuswb_128:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psrlw $15, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE-NEXT:    packuswb %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: concat_trunc_packuswb_128:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrlw $15, %xmm0, %xmm0
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = lshr <8 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %2 = and  <8 x i16> %a1, <i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1>
+  %3 = trunc <8 x i16> %1 to <8 x i8>
+  %4 = trunc <8 x i16> %2 to <8 x i8>
+  %5 = shufflevector <8 x i8> %3, <8 x i8> %4, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %5
+}
diff --git a/llvm/test/CodeGen/X86/vector-pack-256.ll b/llvm/test/CodeGen/X86/vector-pack-256.ll
new file mode 100644
index 0000000000000..6d1016ba1d737
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-pack-256.ll
@@ -0,0 +1,392 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512f  | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BW
+
+; trunc(concat(x,y)) -> pack
+
+define <16 x i16> @trunc_concat_packssdw_256(<8 x i32> %a0, <8 x i32> %a1) nounwind {
+; AVX1-LABEL: trunc_concat_packssdw_256:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsrad $17, %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsrad $17, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrad $23, %xmm1, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vpsrad $23, %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_concat_packssdw_256:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrad $17, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrad $23, %ymm1, %ymm1
+; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_concat_packssdw_256:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrad $17, %ymm0, %ymm0
+; AVX512-NEXT:    vpsrad $23, %ymm1, %ymm1
+; AVX512-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512-NEXT:    retq
+  %1 = ashr <8 x i32> %a0, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+  %2 = ashr <8 x i32> %a1, <i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23>
+  %3 = shufflevector <8 x i32> %1, <8 x i32> %2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
+  %4 = trunc <16 x i32> %3 to <16 x i16>
+  ret <16 x i16> %4
+}
+
+define <16 x i16> @trunc_concat_packusdw_256(<8 x i32> %a0, <8 x i32> %a1) nounwind {
+; AVX1-LABEL: trunc_concat_packusdw_256:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsrld $17, %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsrld $17, %xmm0, %xmm0
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_concat_packusdw_256:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrld $17, %ymm0, %ymm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_concat_packusdw_256:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrld $17, %ymm0, %ymm0
+; AVX512-NEXT:    vpandd {{.*}}(%rip){1to8}, %ymm1, %ymm1
+; AVX512-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512-NEXT:    retq
+  %1 = lshr <8 x i32> %a0, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+  %2 = and  <8 x i32> %a1, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+  %3 = shufflevector <8 x i32> %1, <8 x i32> %2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
+  %4 = trunc <16 x i32> %3 to <16 x i16>
+  ret <16 x i16> %4
+}
+
+define <32 x i8> @trunc_concat_packsswb_256(<16 x i16> %a0, <16 x i16> %a1) nounwind {
+; AVX1-LABEL: trunc_concat_packsswb_256:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsraw $15, %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsraw $15, %xmm0, %xmm0
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpacksswb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_concat_packsswb_256:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsraw $15, %ymm0, %ymm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_concat_packsswb_256:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsraw $15, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_concat_packsswb_256:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsraw $15, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    retq
+  %1 = ashr <16 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %2 = and  <16 x i16> %a1, <i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1>
+  %3 = shufflevector <16 x i16> %1, <16 x i16> %2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %4 = trunc <32 x i16> %3 to <32 x i8>
+  ret <32 x i8> %4
+}
+
+define <32 x i8> @trunc_concat_packuswb_256(<16 x i16> %a0, <16 x i16> %a1) nounwind {
+; AVX1-LABEL: trunc_concat_packuswb_256:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsrlw $15, %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsrlw $15, %xmm0, %xmm0
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_concat_packuswb_256:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrlw $15, %ymm0, %ymm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: trunc_concat_packuswb_256:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $15, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_concat_packuswb_256:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $15, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    retq
+  %1 = lshr <16 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %2 = and  <16 x i16> %a1, <i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1>
+  %3 = shufflevector <16 x i16> %1, <16 x i16> %2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %4 = trunc <32 x i16> %3 to <32 x i8>
+  ret <32 x i8> %4
+}
+
+; concat(trunc(x),trunc(y)) -> pack
+
+
+define <16 x i16> @concat_trunc_packssdw_256(<8 x i32> %a0, <8 x i32> %a1) nounwind {
+; AVX1-LABEL: concat_trunc_packssdw_256:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsrad $17, %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsrad $17, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrad $23, %xmm1, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vpsrad $23, %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: concat_trunc_packssdw_256:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrad $17, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrad $23, %ymm1, %ymm1
+; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: concat_trunc_packssdw_256:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrad $17, %ymm0, %ymm0
+; AVX512-NEXT:    vpsrad $23, %ymm1, %ymm1
+; AVX512-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512-NEXT:    vpmovdw %ymm1, %xmm1
+; AVX512-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm0[1],xmm1[1]
+; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+  %1 = ashr <8 x i32> %a0, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+  %2 = ashr <8 x i32> %a1, <i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23>
+  %3 = trunc <8 x i32> %1 to <8 x i16>
+  %4 = trunc <8 x i32> %2 to <8 x i16>
+  %5 = shufflevector <8 x i16> %3, <8 x i16> %4, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i16> %5
+}
+
+define <16 x i16> @concat_trunc_packusdw_256(<8 x i32> %a0, <8 x i32> %a1) nounwind {
+; AVX1-LABEL: concat_trunc_packusdw_256:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsrld $17, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrld $17, %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX1-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm0[1],xmm1[1]
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: concat_trunc_packusdw_256:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrld $17, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm0[1],xmm1[1]
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: concat_trunc_packusdw_256:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrld $17, %ymm0, %ymm0
+; AVX512-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512-NEXT:    vpmovdw %ymm1, %xmm1
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm0[1],xmm1[1]
+; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+  %1 = lshr <8 x i32> %a0, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+  %2 = and  <8 x i32> %a1, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+  %3 = trunc <8 x i32> %1 to <8 x i16>
+  %4 = trunc <8 x i32> %2 to <8 x i16>
+  %5 = shufflevector <8 x i16> %3, <8 x i16> %4, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i16> %5
+}
+
+define <32 x i8> @concat_trunc_packsswb_256(<16 x i16> %a0, <16 x i16> %a1) nounwind {
+; AVX1-LABEL: concat_trunc_packsswb_256:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsraw $15, %xmm2, %xmm2
+; AVX1-NEXT:    vpsraw $15, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm0[1],xmm1[1]
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: concat_trunc_packsswb_256:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsraw $15, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpacksswb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm0[1],xmm1[1]
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: concat_trunc_packsswb_256:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsraw $15, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm0[1],xmm1[1]
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: concat_trunc_packsswb_256:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsraw $15, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BW-NEXT:    vpmovwb %ymm1, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm0[1],xmm1[1]
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+  %1 = ashr <16 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %2 = and  <16 x i16> %a1, <i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1>
+  %3 = trunc <16 x i16> %1 to <16 x i8>
+  %4 = trunc <16 x i16> %2 to <16 x i8>
+  %5 = shufflevector <16 x i8> %3, <16 x i8> %4, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  ret <32 x i8> %5
+}
+
+define <32 x i8> @concat_trunc_packuswb_256(<16 x i16> %a0, <16 x i16> %a1) nounwind {
+; AVX1-LABEL: concat_trunc_packuswb_256:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsrlw $15, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $15, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm0[1],xmm1[1]
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: concat_trunc_packuswb_256:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrlw $15, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm0[1],xmm1[1]
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: concat_trunc_packuswb_256:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $15, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm0[1],xmm1[1]
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: concat_trunc_packuswb_256:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $15, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BW-NEXT:    vpmovwb %ymm1, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm0[1],xmm1[1]
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+  %1 = lshr <16 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %2 = and  <16 x i16> %a1, <i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1>
+  %3 = trunc <16 x i16> %1 to <16 x i8>
+  %4 = trunc <16 x i16> %2 to <16 x i8>
+  %5 = shufflevector <16 x i8> %3, <16 x i8> %4, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  ret <32 x i8> %5
+}
diff --git a/llvm/test/CodeGen/X86/vector-pack-512.ll b/llvm/test/CodeGen/X86/vector-pack-512.ll
new file mode 100644
index 0000000000000..0612e08e830a4
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-pack-512.ll
@@ -0,0 +1,271 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512f  | FileCheck %s --check-prefixes=AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
+
+; trunc(concat(x,y)) -> pack
+
+define <32 x i16> @trunc_concat_packssdw_512(<16 x i32> %a0, <16 x i32> %a1) nounwind {
+; AVX512-LABEL: trunc_concat_packssdw_512:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrad $17, %zmm0, %zmm0
+; AVX512-NEXT:    vpsrad $23, %zmm1, %zmm1
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [4,5,12,13,6,7,14,15]
+; AVX512-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,8,9,2,3,10,11]
+; AVX512-NEXT:    vpermi2q %zmm1, %zmm0, %zmm3
+; AVX512-NEXT:    vpmovdw %zmm3, %ymm0
+; AVX512-NEXT:    vpmovdw %zmm2, %ymm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = ashr <16 x i32> %a0, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+  %2 = ashr <16 x i32> %a1, <i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23>
+  %3 = shufflevector <16 x i32> %1, <16 x i32> %2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 24, i32 25, i32 26, i32 27, i32 12, i32 13, i32 14, i32 15, i32 28, i32 29, i32 30, i32 31>
+  %4 = trunc <32 x i32> %3 to <32 x i16>
+  ret <32 x i16> %4
+}
+
+define <32 x i16> @trunc_concat_packusdw_512(<16 x i32> %a0, <16 x i32> %a1) nounwind {
+; AVX512-LABEL: trunc_concat_packusdw_512:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrld $17, %zmm0, %zmm0
+; AVX512-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm1, %zmm1
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [4,5,12,13,6,7,14,15]
+; AVX512-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,8,9,2,3,10,11]
+; AVX512-NEXT:    vpermi2q %zmm1, %zmm0, %zmm3
+; AVX512-NEXT:    vpmovdw %zmm3, %ymm0
+; AVX512-NEXT:    vpmovdw %zmm2, %ymm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = lshr <16 x i32> %a0, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+  %2 = and  <16 x i32> %a1, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+  %3 = shufflevector <16 x i32> %1, <16 x i32> %2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 24, i32 25, i32 26, i32 27, i32 12, i32 13, i32 14, i32 15, i32 28, i32 29, i32 30, i32 31>
+  %4 = trunc <32 x i32> %3 to <32 x i16>
+  ret <32 x i16> %4
+}
+
+define <64 x i8> @trunc_concat_packsswb_512(<32 x i16> %a0, <32 x i16> %a1) nounwind {
+; AVX512F-LABEL: trunc_concat_packsswb_512:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsraw $15, %ymm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
+; AVX512F-NEXT:    vpsraw $15, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
+; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,1,8,9,2,3,10,11]
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [4,5,12,13,6,7,14,15]
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm0, %zmm3
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm3, %ymm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm2
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_concat_packsswb_512:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsraw $15, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [4,5,12,13,6,7,14,15]
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,8,9,2,3,10,11]
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm3
+; AVX512BW-NEXT:    vpmovwb %zmm3, %ymm0
+; AVX512BW-NEXT:    vpmovwb %zmm2, %ymm1
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+  %1 = ashr <32 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %2 = and  <32 x i16> %a1, <i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1>
+  %3 = shufflevector <32 x i16> %1, <32 x i16> %2, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+  %4 = trunc <64 x i16> %3 to <64 x i8>
+  ret <64 x i8> %4
+}
+
+define <64 x i8> @trunc_concat_packuswb_512(<32 x i16> %a0, <32 x i16> %a1) nounwind {
+; AVX512F-LABEL: trunc_concat_packuswb_512:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $15, %ymm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $15, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
+; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,1,8,9,2,3,10,11]
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [4,5,12,13,6,7,14,15]
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm0, %zmm3
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm3, %ymm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm2
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc_concat_packuswb_512:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $15, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [4,5,12,13,6,7,14,15]
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,8,9,2,3,10,11]
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm3
+; AVX512BW-NEXT:    vpmovwb %zmm3, %ymm0
+; AVX512BW-NEXT:    vpmovwb %zmm2, %ymm1
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+  %1 = lshr <32 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %2 = and  <32 x i16> %a1, <i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1>
+  %3 = shufflevector <32 x i16> %1, <32 x i16> %2, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+  %4 = trunc <64 x i16> %3 to <64 x i8>
+  ret <64 x i8> %4
+}
+
+; concat(trunc(x),trunc(y)) -> pack
+
+define <32 x i16> @concat_trunc_packssdw_512(<16 x i32> %a0, <16 x i32> %a1) nounwind {
+; AVX512-LABEL: concat_trunc_packssdw_512:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrad $17, %zmm0, %zmm0
+; AVX512-NEXT:    vpsrad $23, %zmm1, %zmm1
+; AVX512-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,6,3,7]
+; AVX512-NEXT:    vpermi2q %ymm1, %ymm0, %ymm2
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,4,1,5]
+; AVX512-NEXT:    vpermi2q %ymm1, %ymm0, %ymm3
+; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm0
+; AVX512-NEXT:    retq
+  %1 = ashr <16 x i32> %a0, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+  %2 = ashr <16 x i32> %a1, <i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23>
+  %3 = trunc <16 x i32> %1 to <16 x i16>
+  %4 = trunc <16 x i32> %2 to <16 x i16>
+  %5 = shufflevector <16 x i16> %3, <16 x i16> %4, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 24, i32 25, i32 26, i32 27, i32 12, i32 13, i32 14, i32 15, i32 28, i32 29, i32 30, i32 31>
+  ret <32 x i16> %5
+}
+
+define <32 x i16> @concat_trunc_packusdw_512(<16 x i32> %a0, <16 x i32> %a1) nounwind {
+; AVX512-LABEL: concat_trunc_packusdw_512:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrld $17, %zmm0, %zmm0
+; AVX512-NEXT:    vpsrld $23, %zmm1, %zmm1
+; AVX512-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,6,3,7]
+; AVX512-NEXT:    vpermi2q %ymm1, %ymm0, %ymm2
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,4,1,5]
+; AVX512-NEXT:    vpermi2q %ymm1, %ymm0, %ymm3
+; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm0
+; AVX512-NEXT:    retq
+  %1 = lshr <16 x i32> %a0, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+  %2 = lshr <16 x i32> %a1, <i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23>
+  %3 = trunc <16 x i32> %1 to <16 x i16>
+  %4 = trunc <16 x i32> %2 to <16 x i16>
+  %5 = shufflevector <16 x i16> %3, <16 x i16> %4, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 24, i32 25, i32 26, i32 27, i32 12, i32 13, i32 14, i32 15, i32 28, i32 29, i32 30, i32 31>
+  ret <32 x i16> %5
+}
+
+define <64 x i8> @concat_trunc_packsswb_512(<32 x i16> %a0, <32 x i16> %a1) nounwind {
+; AVX512F-LABEL: concat_trunc_packsswb_512:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpsraw $15, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsraw $15, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,6,3,7]
+; AVX512F-NEXT:    vpermi2q %ymm1, %ymm0, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,4,1,5]
+; AVX512F-NEXT:    vpermi2q %ymm1, %ymm0, %ymm3
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: concat_trunc_packsswb_512:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsraw $15, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,6,3,7]
+; AVX512BW-NEXT:    vpermi2q %ymm1, %ymm0, %ymm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,4,1,5]
+; AVX512BW-NEXT:    vpermi2q %ymm1, %ymm0, %ymm3
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm0
+; AVX512BW-NEXT:    retq
+  %1 = ashr <32 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %2 = and  <32 x i16> %a1, <i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1>
+  %3 = trunc <32 x i16> %1 to <32 x i8>
+  %4 = trunc <32 x i16> %2 to <32 x i8>
+  %5 = shufflevector <32 x i8> %3, <32 x i8> %4, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+  ret <64 x i8> %5
+}
+
+define <64 x i8> @concat_trunc_packuswb_512(<32 x i16> %a0, <32 x i16> %a1) nounwind {
+; AVX512F-LABEL: concat_trunc_packuswb_512:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpsrlw $15, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw $15, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,6,3,7]
+; AVX512F-NEXT:    vpermi2q %ymm1, %ymm0, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,4,1,5]
+; AVX512F-NEXT:    vpermi2q %ymm1, %ymm0, %ymm3
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: concat_trunc_packuswb_512:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $15, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,6,3,7]
+; AVX512BW-NEXT:    vpermi2q %ymm1, %ymm0, %ymm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,4,1,5]
+; AVX512BW-NEXT:    vpermi2q %ymm1, %ymm0, %ymm3
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm0
+; AVX512BW-NEXT:    retq
+  %1 = lshr <32 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %2 = and  <32 x i16> %a1, <i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1>
+  %3 = trunc <32 x i16> %1 to <32 x i8>
+  %4 = trunc <32 x i16> %2 to <32 x i8>
+  %5 = shufflevector <32 x i8> %3, <32 x i8> %4, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+  ret <64 x i8> %5
+}
diff --git a/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
index b79689b54a10d..98125765e27db 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
@@ -1506,7 +1506,6 @@ define i1 @icmp_v16i32_v16i1(<16 x i32>) {
 ; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
diff --git a/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
index be936221fb52b..3491839b6d137 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
@@ -1497,9 +1497,8 @@ define i1 @icmp_v16i32_v16i1(<16 x i32>) {
 ; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %ymm0, %eax
-; AVX2-NEXT:    testl $-1431655766, %eax # imm = 0xAAAAAAAA
+; AVX2-NEXT:    testl %eax, %eax
 ; AVX2-NEXT:    setne %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll b/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll
index 194c7402b0bfa..8223e6bd23804 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll
@@ -816,6 +816,233 @@ define i1 @test_v128i8(<128 x i8> %a0) {
   ret i1 %2
 }
 
+;
+; Compare Truncated/Masked OR Reductions
+;
+
+define i1 @trunc_v2i64(<2 x i64> %a0) {
+; SSE2-LABEL: trunc_v2i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    testw %ax, %ax
+; SSE2-NEXT:    sete %al
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: trunc_v2i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    ptest {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    sete %al
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: trunc_v2i64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vptest {{.*}}(%rip), %xmm0
+; AVX-NEXT:    sete %al
+; AVX-NEXT:    retq
+  %1 = call i64 @llvm.experimental.vector.reduce.or.v2i64(<2 x i64> %a0)
+  %2 = trunc i64 %1 to i16
+  %3 = icmp eq i16 %2, 0
+  ret i1 %3
+}
+
+define i1 @mask_v8i32(<8 x i32> %a0) {
+; SSE2-LABEL: mask_v8i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
+; SSE2-NEXT:    pmovmskb %xmm1, %eax
+; SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; SSE2-NEXT:    sete %al
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: mask_v8i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    ptest {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    sete %al
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: mask_v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vptest {{.*}}(%rip), %ymm0
+; AVX1-NEXT:    sete %al
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: mask_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9223372039002259456,9223372039002259456,9223372039002259456,9223372039002259456]
+; AVX2-NEXT:    vptest %ymm1, %ymm0
+; AVX2-NEXT:    sete %al
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: mask_v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9223372039002259456,9223372039002259456,9223372039002259456,9223372039002259456]
+; AVX512-NEXT:    vptest %ymm1, %ymm0
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %1 = call i32 @llvm.experimental.vector.reduce.or.v8i32(<8 x i32> %a0)
+  %2 = and i32 %1, 2147483648
+  %3 = icmp eq i32 %2, 0
+  ret i1 %3
+}
+
+define i1 @trunc_v16i16(<16 x i16> %a0) {
+; SSE2-LABEL: trunc_v16i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
+; SSE2-NEXT:    pmovmskb %xmm1, %eax
+; SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; SSE2-NEXT:    setne %al
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: trunc_v16i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    ptest {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    setne %al
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: trunc_v16i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vptest {{.*}}(%rip), %ymm0
+; AVX1-NEXT:    setne %al
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_v16i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [71777214294589695,71777214294589695,71777214294589695,71777214294589695]
+; AVX2-NEXT:    vptest %ymm1, %ymm0
+; AVX2-NEXT:    setne %al
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: trunc_v16i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [71777214294589695,71777214294589695,71777214294589695,71777214294589695]
+; AVX512-NEXT:    vptest %ymm1, %ymm0
+; AVX512-NEXT:    setne %al
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %1 = call i16 @llvm.experimental.vector.reduce.or.v16i16(<16 x i16> %a0)
+  %2 = trunc i16 %1 to i8
+  %3 = icmp ne i8 %2, 0
+  ret i1 %3
+}
+
+define i1 @mask_v128i8(<128 x i8> %a0) {
+; SSE2-LABEL: mask_v128i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    por %xmm7, %xmm3
+; SSE2-NEXT:    por %xmm5, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpeqb %xmm2, %xmm0
+; SSE2-NEXT:    pmovmskb %xmm0, %eax
+; SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; SSE2-NEXT:    sete %al
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: mask_v128i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    por %xmm7, %xmm3
+; SSE41-NEXT:    por %xmm5, %xmm3
+; SSE41-NEXT:    por %xmm1, %xmm3
+; SSE41-NEXT:    por %xmm6, %xmm2
+; SSE41-NEXT:    por %xmm4, %xmm2
+; SSE41-NEXT:    por %xmm3, %xmm2
+; SSE41-NEXT:    por %xmm0, %xmm2
+; SSE41-NEXT:    ptest {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    sete %al
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: mask_v128i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vorps %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vorps %ymm1, %ymm2, %ymm1
+; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vptest {{.*}}(%rip), %ymm0
+; AVX1-NEXT:    sete %al
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: mask_v128i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpor %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [72340172838076673,72340172838076673,72340172838076673,72340172838076673]
+; AVX2-NEXT:    vptest %ymm1, %ymm0
+; AVX2-NEXT:    sete %al
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: mask_v128i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [72340172838076673,72340172838076673,72340172838076673,72340172838076673]
+; AVX512-NEXT:    vptest %ymm1, %ymm0
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %1 = call i8 @llvm.experimental.vector.reduce.or.v128i8(<128 x i8> %a0)
+  %2 = and i8 %1, 1
+  %3 = icmp eq i8 %2, 0
+  ret i1 %3
+}
+
+%struct.Box = type { i32, i32, i32, i32 }
+define zeroext i1 @PR44781(%struct.Box* %0) {
+; SSE2-LABEL: PR44781:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqu (%rdi), %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
+; SSE2-NEXT:    pmovmskb %xmm1, %eax
+; SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; SSE2-NEXT:    sete %al
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: PR44781:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqu (%rdi), %xmm0
+; SSE41-NEXT:    ptest {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    sete %al
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: PR44781:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; AVX-NEXT:    vptest {{.*}}(%rip), %xmm0
+; AVX-NEXT:    sete %al
+; AVX-NEXT:    retq
+  %2 = bitcast %struct.Box* %0 to <4 x i32>*
+  %3 = load <4 x i32>, <4 x i32>* %2, align 4
+  %4 = call i32 @llvm.experimental.vector.reduce.or.v4i32(<4 x i32> %3)
+  %5 = and i32 %4, 15
+  %6 = icmp eq i32 %5, 0
+  ret i1 %6
+}
+
 declare i64 @llvm.experimental.vector.reduce.or.v2i64(<2 x i64>)
 declare i64 @llvm.experimental.vector.reduce.or.v4i64(<4 x i64>)
 declare i64 @llvm.experimental.vector.reduce.or.v8i64(<8 x i64>)
diff --git a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
index ed4748211ed64..627faa02d2b7d 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
@@ -1710,9 +1710,9 @@ define i1 @icmp_v16i32_v16i1(<16 x i32>) {
 ; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX2-NEXT:    movl %eax, %ecx
 ; AVX2-NEXT:    shrl $8, %ecx
diff --git a/llvm/test/CodeGen/X86/vector-rotate-128.ll b/llvm/test/CodeGen/X86/vector-rotate-128.ll
index 48b5777ecd2d2..67dd15ee87abe 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-128.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-128.ll
@@ -1968,13 +1968,35 @@ define <8 x i16> @splatconstant_rotate_mask_v8i16(<8 x i16> %a) nounwind {
 ; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_rotate_mask_v8i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $11, %xmm0, %xmm1
-; AVX512-NEXT:    vpsllw $5, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_rotate_mask_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $11, %xmm0, %xmm1
+; AVX512F-NEXT:    vpsllw $5, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_rotate_mask_v8i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $5, %xmm0, %xmm1
+; AVX512VL-NEXT:    vpsrlw $11, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpternlogq $168, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_rotate_mask_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $11, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpsllw $5, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_rotate_mask_v8i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $5, %xmm0, %xmm1
+; AVX512VLBW-NEXT:    vpsrlw $11, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpternlogq $168, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_rotate_mask_v8i16:
 ; XOP:       # %bb.0:
@@ -2020,14 +2042,39 @@ define <16 x i8> @splatconstant_rotate_mask_v16i8(<16 x i8> %a) nounwind {
 ; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_rotate_mask_v16i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllw $4, %xmm0, %xmm1
-; AVX512-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_rotate_mask_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512F-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_rotate_mask_v16i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VL-NEXT:    vpsrlw $4, %xmm0, %xmm2
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VL-NEXT:    vpternlogq $248, {{.*}}(%rip), %xmm2, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_rotate_mask_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_rotate_mask_v16i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VLBW-NEXT:    vpsrlw $4, %xmm0, %xmm2
+; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VLBW-NEXT:    vpternlogq $248, {{.*}}(%rip), %xmm2, %xmm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_rotate_mask_v16i8:
 ; XOP:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-rotate-256.ll b/llvm/test/CodeGen/X86/vector-rotate-256.ll
index ad92c1c67111e..31fe575024753 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-256.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-256.ll
@@ -442,12 +442,11 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpternlogq $248, {{.*}}(%rip), %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_rotate_v32i8:
@@ -823,12 +822,11 @@ define <32 x i8> @splatvar_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512VL-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm4, %xmm1
-; AVX512VL-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm4, %xmm0
+; AVX512VL-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpbroadcastb %xmm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $236, %ymm3, %ymm2, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_rotate_v32i8:
@@ -1707,13 +1705,35 @@ define <16 x i16> @splatconstant_rotate_mask_v16i16(<16 x i16> %a) nounwind {
 ; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_rotate_mask_v16i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $11, %ymm0, %ymm1
-; AVX512-NEXT:    vpsllw $5, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_rotate_mask_v16i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $11, %ymm0, %ymm1
+; AVX512F-NEXT:    vpsllw $5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_rotate_mask_v16i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $5, %ymm0, %ymm1
+; AVX512VL-NEXT:    vpsrlw $11, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $168, {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_rotate_mask_v16i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $11, %ymm0, %ymm1
+; AVX512BW-NEXT:    vpsllw $5, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_rotate_mask_v16i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $5, %ymm0, %ymm1
+; AVX512VLBW-NEXT:    vpsrlw $11, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpternlogq $168, {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_rotate_mask_v16i16:
 ; XOPAVX1:       # %bb.0:
@@ -1799,10 +1819,9 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind {
 ; AVX512VLBW-LABEL: splatconstant_rotate_mask_v32i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %ymm0, %ymm1
-; AVX512VLBW-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512VLBW-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512VLBW-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VLBW-NEXT:    vpternlogq $248, {{.*}}(%rip), %ymm2, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_rotate_mask_v32i8:
diff --git a/llvm/test/CodeGen/X86/vector-rotate-512.ll b/llvm/test/CodeGen/X86/vector-rotate-512.ll
index 2acabc352481f..05d989ebaa307 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-512.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-512.ll
@@ -488,12 +488,11 @@ define <64 x i8> @splatvar_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512BW-NEXT:    vpsllw %xmm2, %xmm4, %xmm2
 ; AVX512BW-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512BW-NEXT:    vpandq %zmm2, %zmm3, %zmm2
-; AVX512BW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpsrlw %xmm1, %xmm4, %xmm1
-; AVX512BW-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %zmm1
-; AVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vporq %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm3
+; AVX512BW-NEXT:    vpsrlw %xmm1, %xmm4, %xmm0
+; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpbroadcastb %xmm0, %zmm0
+; AVX512BW-NEXT:    vpternlogq $236, %zmm3, %zmm2, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_rotate_v64i8:
@@ -507,12 +506,11 @@ define <64 x i8> @splatvar_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512VLBW-NEXT:    vpsllw %xmm2, %xmm4, %xmm2
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512VLBW-NEXT:    vpandq %zmm2, %zmm3, %zmm2
-; AVX512VLBW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpsrlw %xmm1, %xmm4, %xmm1
-; AVX512VLBW-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm1, %zmm1
-; AVX512VLBW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vporq %zmm0, %zmm2, %zmm0
+; AVX512VLBW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm3
+; AVX512VLBW-NEXT:    vpsrlw %xmm1, %xmm4, %xmm0
+; AVX512VLBW-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpbroadcastb %xmm0, %zmm0
+; AVX512VLBW-NEXT:    vpternlogq $236, %zmm3, %zmm2, %zmm0
 ; AVX512VLBW-NEXT:    retq
   %splat = shufflevector <64 x i8> %b, <64 x i8> undef, <64 x i32> zeroinitializer
   %splat8 = sub <64 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %splat
@@ -919,10 +917,9 @@ define <32 x i16> @splatconstant_rotate_mask_v32i16(<32 x i16> %a) nounwind {
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
 ; AVX512F-NEXT:    vpsrlw $11, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $11, %ymm2, %ymm2
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512F-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm2
+; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm0
+; AVX512F-NEXT:    vpternlogq $248, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_rotate_mask_v32i16:
@@ -933,28 +930,25 @@ define <32 x i16> @splatconstant_rotate_mask_v32i16(<32 x i16> %a) nounwind {
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
 ; AVX512VL-NEXT:    vpsrlw $11, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $11, %ymm2, %ymm2
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VL-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VL-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm2
+; AVX512VL-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VL-NEXT:    vpternlogq $248, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_rotate_mask_v32i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $5, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpsrlw $11, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512BW-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpsrlw $11, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm0
+; AVX512BW-NEXT:    vpternlogq $248, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_rotate_mask_v32i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $5, %zmm0, %zmm1
-; AVX512VLBW-NEXT:    vpsrlw $11, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VLBW-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512VLBW-NEXT:    vpsrlw $11, %zmm0, %zmm2
+; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VLBW-NEXT:    vpternlogq $248, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VLBW-NEXT:    retq
   %shl = shl <32 x i16> %a, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
   %lshr = lshr <32 x i16> %a, <i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11>
@@ -979,10 +973,9 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind {
 ; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm2, %zmm1
-; AVX512F-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
+; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm2, %zmm0
+; AVX512F-NEXT:    vpternlogq $248, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_rotate_mask_v64i8:
@@ -999,28 +992,25 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VL-NEXT:    vpandq {{.*}}(%rip), %zmm2, %zmm1
-; AVX512VL-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
+; AVX512VL-NEXT:    vpandq {{.*}}(%rip), %zmm2, %zmm0
+; AVX512VL-NEXT:    vpternlogq $248, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_rotate_mask_v64i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512BW-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm0
+; AVX512BW-NEXT:    vpternlogq $248, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_rotate_mask_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm1
-; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VLBW-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm2
+; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VLBW-NEXT:    vpternlogq $248, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VLBW-NEXT:    retq
   %shl = shl <64 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
   %lshr = lshr <64 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
index dc0d0a1168b71..5764d19f4c7f7 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
@@ -1443,9 +1443,8 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind {
 ; AVX512VL-LABEL: splatconstant_shift_v16i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrlw $3, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpternlogq $108, {{.*}}(%rip), %xmm1, %xmm0
 ; AVX512VL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
index 708ab48479b03..358f9b8cc4dec 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
@@ -944,15 +944,14 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; AVX512DQVL:       # %bb.0:
 ; AVX512DQVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQVL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpbroadcastb %xmm2, %ymm2
-; AVX512DQVL-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} ymm2 = [32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896]
-; AVX512DQVL-NEXT:    vpsrlw %xmm1, %ymm2, %ymm1
-; AVX512DQVL-NEXT:    vpxor %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
+; AVX512DQVL-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
+; AVX512DQVL-NEXT:    vpsrlw %xmm1, %xmm3, %xmm1
+; AVX512DQVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %ymm1
+; AVX512DQVL-NEXT:    vpternlogq $108, %ymm0, %ymm2, %ymm1
+; AVX512DQVL-NEXT:    vpsubb %ymm2, %ymm1, %ymm0
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v32i8:
@@ -1630,9 +1629,8 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
 ; AVX512VL-LABEL: splatconstant_shift_v32i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrlw $3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $108, {{.*}}(%rip), %ymm1, %ymm0
 ; AVX512VL-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
index a71d257bfe552..0acdbac821dae 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
@@ -212,15 +212,14 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpbroadcastb %xmm2, %zmm2
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896]
-; AVX512BW-NEXT:    vpsrlw %xmm1, %zmm2, %zmm1
-; AVX512BW-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpsubb %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw %xmm1, %zmm2, %zmm2
+; AVX512BW-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
+; AVX512BW-NEXT:    vpsrlw %xmm1, %xmm3, %xmm1
+; AVX512BW-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpbroadcastb %xmm1, %zmm1
+; AVX512BW-NEXT:    vpternlogq $108, %zmm0, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpsubb %zmm2, %zmm1, %zmm0
 ; AVX512BW-NEXT:    retq
   %splat = shufflevector <64 x i8> %b, <64 x i8> undef, <64 x i32> zeroinitializer
   %shift = ashr <64 x i8> %a, %splat
@@ -375,9 +374,8 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) nounwind {
 ; AVX512BW-LABEL: splatconstant_shift_v64i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsrlw $3, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512BW-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpternlogq $108, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512BW-NEXT:    vpsubb %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
   %shift = ashr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
index 0e4016e835949..11d118bf31c3d 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
@@ -2354,9 +2354,8 @@ define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind {
 ; AVX512VL-LABEL: splatconstant_shift_v8i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrlw $3, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpternlogq $108, {{.*}}(%rip), %xmm1, %xmm0
 ; AVX512VL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
@@ -2408,9 +2407,8 @@ define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind {
 ; AVX512VL-LABEL: splatconstant_shift_v4i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrlw $3, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpternlogq $108, {{.*}}(%rip), %xmm1, %xmm0
 ; AVX512VL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
@@ -2462,9 +2460,8 @@ define <2 x i8> @splatconstant_shift_v2i8(<2 x i8> %a) nounwind {
 ; AVX512VL-LABEL: splatconstant_shift_v2i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrlw $3, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpternlogq $108, {{.*}}(%rip), %xmm1, %xmm0
 ; AVX512VL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
index 1de61039fcb97..04064f1c1970d 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
@@ -3570,7 +3570,7 @@ define <8 x i16> @insert_dup_mem_v8i16_sext_i16_i64(i16* %ptr) {
 ; SSE-LABEL: insert_dup_mem_v8i16_sext_i16_i64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movzwl (%rdi), %eax
-; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    movd %eax, %xmm0
 ; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    retq
@@ -3578,7 +3578,7 @@ define <8 x i16> @insert_dup_mem_v8i16_sext_i16_i64(i16* %ptr) {
 ; AVX1-LABEL: insert_dup_mem_v8i16_sext_i16_i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    movzwl (%rdi), %eax
-; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vmovd %eax, %xmm0
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
@@ -3591,7 +3591,7 @@ define <8 x i16> @insert_dup_mem_v8i16_sext_i16_i64(i16* %ptr) {
 ; XOPAVX1-LABEL: insert_dup_mem_v8i16_sext_i16_i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    movzwl (%rdi), %eax
-; XOPAVX1-NEXT:    vmovq %rax, %xmm0
+; XOPAVX1-NEXT:    vmovd %eax, %xmm0
 ; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
index 6bbb89559b89b..635d94b9e7239 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
@@ -7683,7 +7683,7 @@ define <16 x i16> @insert_dup_mem_v16i16_sext_i16_i64(i16* %ptr) {
 ; AVX1-LABEL: insert_dup_mem_v16i16_sext_i16_i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    movzwl (%rdi), %eax
-; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vmovd %eax, %xmm0
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
@@ -7697,7 +7697,7 @@ define <16 x i16> @insert_dup_mem_v16i16_sext_i16_i64(i16* %ptr) {
 ; XOPAVX1-LABEL: insert_dup_mem_v16i16_sext_i16_i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    movzwl (%rdi), %eax
-; XOPAVX1-NEXT:    vmovq %rax, %xmm0
+; XOPAVX1-NEXT:    vmovd %eax, %xmm0
 ; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
index 3b14959035eb4..bcd3a7b851500 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
@@ -260,6 +260,27 @@ define <16 x float> @shuffle_v16f32_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08
   ret <16 x float> %c
 }
 
+; PR46249
+define <16 x i32> @shuffle_v16i32_0b_0a_09_08_0f_0e_0d_0c_03_02_01_00_07_06_05_04(<16 x i32> %a) {
+; ALL-LABEL: shuffle_v16i32_0b_0a_09_08_0f_0e_0d_0c_03_02_01_00_07_06_05_04:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vpshufd {{.*#+}} zmm0 = zmm0[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
+; ALL-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[4,5,6,7,0,1,2,3]
+; ALL-NEXT:    retq
+  %1 = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32> <i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+  ret <16 x i32> %1
+}
+
+define <16 x float> @shuffle_v16f32_0b_0a_09_08_0f_0e_0d_0c_03_02_01_00_07_06_05_04(<16 x float> %a) {
+; ALL-LABEL: shuffle_v16f32_0b_0a_09_08_0f_0e_0d_0c_03_02_01_00_07_06_05_04:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
+; ALL-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[4,5,6,7,0,1,2,3]
+; ALL-NEXT:    retq
+  %1 = shufflevector <16 x float> %a, <16 x float> undef, <16 x i32> <i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12, i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+  ret <16 x float> %1
+}
+
 define <16 x float> @shuffle_v16f32_load_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_18(<16 x float> %a, <16 x float>* %b)  {
 ; ALL-LABEL: shuffle_v16f32_load_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_18:
 ; ALL:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll
index 59c4769017820..b2d4dc76a10e0 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll
@@ -201,8 +201,8 @@ define <32 x i16> @shuffle_v32i16_03_00_01_02_07_04_05_06_11_08_09_10_15_12_13_1
 define <32 x i16> @shuffle_v32i16_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz(<32 x i16> %a) {
 ; KNL-LABEL: shuffle_v32i16_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vmovaps {{.*#+}} xmm1 = [65535,0,0,0]
-; KNL-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; KNL-NEXT:    vmovdqa {{.*#+}} xmm1 = [65535,0,0,0]
+; KNL-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: shuffle_v32i16_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
index 2cf8ab11ce278..3199cc0fa9b9b 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
@@ -109,24 +109,26 @@ define <64 x i8> @shuffle_v64i8_79_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_
 define <64 x i8> @shuffle_v64i8_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz(<64 x i8> %a) {
 ; AVX512F-LABEL: shuffle_v64i8_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovaps {{.*#+}} xmm1 = [255,0,0,0]
-; AVX512F-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,0,0,0]
+; AVX512F-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[0],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,0,0,0]
+; AVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: shuffle_v64i8_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vmovaps {{.*#+}} xmm1 = [255,0,0,0]
-; AVX512DQ-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vandps %zmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz:
 ; AVX512VBMI:       # %bb.0:
-; AVX512VBMI-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[0],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,0,0,0]
+; AVX512VBMI-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512VBMI-NEXT:    retq
   %shuffle = shufflevector <64 x i8> %a, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
   ret <64 x i8> %shuffle
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll b/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
index 1891fb5a46839..cb2dd3ef7e86d 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
@@ -313,11 +313,8 @@ define <64 x i8> @test_mm512_mask_blend_epi8(<64 x i8> %A, <64 x i8> %W){
 ; KNL-LABEL: test_mm512_mask_blend_epi8:
 ; KNL:       # %bb.0: # %entry
 ; KNL-NEXT:    vpbroadcastw {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
-; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
-; KNL-NEXT:    vpblendvb %ymm2, %ymm3, %ymm4, %ymm3
-; KNL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
+; KNL-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm2
+; KNL-NEXT:    vpternlogq $216, %zmm2, %zmm1, %zmm0
 ; KNL-NEXT:    ret{{[l|q]}}
 ;
 ; SKX32-LABEL: test_mm512_mask_blend_epi8:
@@ -340,14 +337,15 @@ define <32 x i16> @test_mm512_mask_blend_epi16(<32 x i16> %A, <32 x i16> %W){
 ; SKX-NEXT:    vpblendmw %zmm0, %zmm1, %zmm0 {%k1}
 ; SKX-NEXT:    ret{{[l|q]}}
 ;
-; KNL-LABEL: test_mm512_mask_blend_epi16:
-; KNL:       # %bb.0: # %entry
-; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
-; KNL-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7],ymm3[8],ymm2[9],ymm3[10],ymm2[11],ymm3[12],ymm2[13],ymm3[14],ymm2[15]
-; KNL-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7],ymm1[8],ymm0[9],ymm1[10],ymm0[11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
-; KNL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; KNL-NEXT:    ret{{[l|q]}}
+; KNL64-LABEL: test_mm512_mask_blend_epi16:
+; KNL64:       # %bb.0: # %entry
+; KNL64-NEXT:    vpternlogd $216, {{.*}}(%rip){1to16}, %zmm1, %zmm0
+; KNL64-NEXT:    retq
+;
+; KNL32-LABEL: test_mm512_mask_blend_epi16:
+; KNL32:       # %bb.0: # %entry
+; KNL32-NEXT:    vpternlogd $216, {{\.LCPI.*}}{1to16}, %zmm1, %zmm0
+; KNL32-NEXT:    retl
 entry:
   %0 = shufflevector <32 x i16> %A, <32 x i16> %W, <32 x i32>  <i32 32, i32 1, i32 34, i32 3, i32 36, i32 5, i32 38, i32 7, i32 40, i32 9, i32 42, i32 11, i32 44, i32 13, i32 46, i32 15, i32 48, i32 17, i32 50, i32 19, i32 52, i32 21, i32 54, i32 23, i32 56, i32 25, i32 58, i32 27, i32 60, i32 29, i32 62, i32 31>
   ret <32 x i16> %0
@@ -528,6 +526,71 @@ define <16 x float> @test_masked_permps_v16f32(<16 x float>* %vp, <16 x float> %
   ret <16 x float> %res
 }
 
+define void @test_demandedelts_pshufb_v32i8_v16i8(<2 x i32>* %src, <8 x i32>* %dst) {
+; SKX64-LABEL: test_demandedelts_pshufb_v32i8_v16i8:
+; SKX64:       # %bb.0:
+; SKX64-NEXT:    vmovdqa 32(%rdi), %xmm0
+; SKX64-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[12,13,14,15,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
+; SKX64-NEXT:    vmovdqa %ymm0, 672(%rsi)
+; SKX64-NEXT:    vmovdqa 208(%rdi), %xmm0
+; SKX64-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,6,7,0,1,2,3],zero,zero,zero,zero,zero,zero,zero,zero
+; SKX64-NEXT:    vmovdqa %ymm0, 832(%rsi)
+; SKX64-NEXT:    vzeroupper
+; SKX64-NEXT:    retq
+;
+; KNL64-LABEL: test_demandedelts_pshufb_v32i8_v16i8:
+; KNL64:       # %bb.0:
+; KNL64-NEXT:    vmovdqa 32(%rdi), %xmm0
+; KNL64-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[12,13,14,15,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
+; KNL64-NEXT:    vmovdqa %ymm0, 672(%rsi)
+; KNL64-NEXT:    vmovdqa 208(%rdi), %xmm0
+; KNL64-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,6,7,0,1,2,3],zero,zero,zero,zero,zero,zero,zero,zero
+; KNL64-NEXT:    vmovdqa %ymm0, 832(%rsi)
+; KNL64-NEXT:    retq
+;
+; SKX32-LABEL: test_demandedelts_pshufb_v32i8_v16i8:
+; SKX32:       # %bb.0:
+; SKX32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SKX32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; SKX32-NEXT:    vmovdqa 32(%ecx), %xmm0
+; SKX32-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[12,13,14,15,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
+; SKX32-NEXT:    vmovdqa %ymm0, 672(%eax)
+; SKX32-NEXT:    vmovdqa 208(%ecx), %xmm0
+; SKX32-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,6,7,0,1,2,3],zero,zero,zero,zero,zero,zero,zero,zero
+; SKX32-NEXT:    vmovdqa %ymm0, 832(%eax)
+; SKX32-NEXT:    vzeroupper
+; SKX32-NEXT:    retl
+;
+; KNL32-LABEL: test_demandedelts_pshufb_v32i8_v16i8:
+; KNL32:       # %bb.0:
+; KNL32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL32-NEXT:    vmovdqa 32(%eax), %xmm0
+; KNL32-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[12,13,14,15,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
+; KNL32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; KNL32-NEXT:    vmovdqa %ymm0, 672(%ecx)
+; KNL32-NEXT:    vmovdqa 208(%eax), %xmm0
+; KNL32-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,6,7,0,1,2,3],zero,zero,zero,zero,zero,zero,zero,zero
+; KNL32-NEXT:    vmovdqa %ymm0, 832(%ecx)
+; KNL32-NEXT:    retl
+  %t64 = bitcast <2 x i32>* %src to <16 x i32>*
+  %t87 = load <16 x i32>, <16 x i32>* %t64, align 64
+  %t88 = extractelement <16 x i32> %t87, i64 11
+  %t89 = insertelement <8 x i32> <i32 undef, i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, i32 %t88, i64 0
+  %t90 = insertelement <8 x i32> %t89, i32 %t88, i64 1
+  %ptridx49.i = getelementptr inbounds <8 x i32>, <8 x i32>* %dst, i64 21
+  store <8 x i32> %t90, <8 x i32>* %ptridx49.i, align 32
+  %ptridx56.i = getelementptr inbounds <2 x i32>, <2 x i32>* %src, i64 24
+  %t00 = bitcast <2 x i32>* %ptridx56.i to <16 x i32>*
+  %t09 = load <16 x i32>, <16 x i32>* %t00, align 64
+  %t10 = extractelement <16 x i32> %t09, i64 5
+  %t11 = insertelement <8 x i32> <i32 undef, i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, i32 %t10, i64 0
+  %t12 = extractelement <16 x i32> %t09, i64 4
+  %t13 = insertelement <8 x i32> %t11, i32 %t12, i64 1
+  %ptridx64.i = getelementptr inbounds <8 x i32>, <8 x i32>* %dst, i64 26
+  store <8 x i32> %t13, <8 x i32>* %ptridx64.i, align 32
+  ret void
+}
+
 %union1= type { <16 x float> }
 @src1 = external dso_local local_unnamed_addr global %union1, align 64
 
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-v48.ll b/llvm/test/CodeGen/X86/vector-shuffle-v48.ll
index 5cd35af8cc303..e854923258257 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-v48.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-v48.ll
@@ -52,17 +52,16 @@ define <32 x i8> @foo(<48 x i8>* %x0) {
 ;
 ; AVX512F-LABEL: foo:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqu 32(%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqu (%rdi), %ymm1
+; AVX512F-NEXT:    vmovdqu (%rdi), %ymm0
+; AVX512F-NEXT:    vmovdqu 32(%rdi), %xmm1
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u,1,2,4,5,7,8,10,11,13,14]
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512F-NEXT:    vmovdqu 16(%rdi), %xmm2
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,u,u,u,u,0,2,3,5,6]
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,1,3,4,6,7,9,10,12,13,15,u,u,u,u,u,24,25,27,28,30,31,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = <255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,u,u,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpblendvb %ymm3, %ymm1, %ymm2, %ymm1
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,1,2,4,5,7,8,10,11,13,14]
-; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[0,2,3,5,6],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,3,4,6,7,9,10,12,13,15],zero,zero,zero,zero,zero,ymm0[24,25,27,28,30,31,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: foo:
diff --git a/llvm/test/CodeGen/X86/vector-trunc-packus.ll b/llvm/test/CodeGen/X86/vector-trunc-packus.ll
index 9d6e582ff5ad9..fe1645059decf 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-packus.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-packus.ll
@@ -2234,9 +2234,9 @@ define <8 x i16> @trunc_packus_v8i64_v8i16(<8 x i64>* %p0) "min-legal-vector-wid
 ; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm2
 ; AVX2-NEXT:    vpand %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -5027,9 +5027,9 @@ define <16 x i8> @trunc_packus_v16i64_v16i8(<16 x i64>* %p0) "min-legal-vector-w
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -5457,9 +5457,9 @@ define <16 x i8> @trunc_packus_v16i32_v16i8(<16 x i32>* %p0) "min-legal-vector-w
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX2-NEXT:    vpackssdw 32(%rdi), %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -5513,9 +5513,9 @@ define void @trunc_packus_v16i32_v16i8_store(<16 x i32>* %p0, <16 x i8>* %p1) "m
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX2-NEXT:    vpackssdw 32(%rdi), %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vmovdqa %xmm0, (%rsi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-trunc-ssat.ll b/llvm/test/CodeGen/X86/vector-trunc-ssat.ll
index 0d9f9869eb0a6..442d903fe039a 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-ssat.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-ssat.ll
@@ -2216,9 +2216,9 @@ define <8 x i16> @trunc_ssat_v8i64_v8i16(<8 x i64>* %p0) "min-legal-vector-width
 ; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm3
 ; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -4829,9 +4829,9 @@ define <16 x i8> @trunc_ssat_v16i64_v16i8(<16 x i64>* %p0) "min-legal-vector-wid
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -5232,9 +5232,9 @@ define <16 x i8> @trunc_ssat_v16i32_v16i8(<16 x i32>* %p0) "min-legal-vector-wid
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX2-NEXT:    vpackssdw 32(%rdi), %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -5288,9 +5288,9 @@ define void @trunc_ssat_v16i32_v16i8_store(<16 x i32>* %p0, <16 x i8>* %p1) "min
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX2-NEXT:    vpackssdw 32(%rdi), %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vmovdqa %xmm0, (%rsi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-trunc-usat.ll b/llvm/test/CodeGen/X86/vector-trunc-usat.ll
index a6e3e9f13d6c4..2e8f3bb6a22d6 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-usat.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-usat.ll
@@ -1541,9 +1541,9 @@ define <8 x i16> @trunc_usat_v8i64_v8i16(<8 x i64>* %p0) {
 ; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm5, %ymm3
 ; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -3537,9 +3537,9 @@ define <16 x i8> @trunc_usat_v16i64_v16i8(<16 x i64>* %p0) {
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -4086,9 +4086,9 @@ define <16 x i8> @trunc_usat_v16i32_v16i8(<16 x i32>* %p0) {
 ; AVX2-NEXT:    vpminud 32(%rdi), %ymm0, %ymm1
 ; AVX2-NEXT:    vpminud (%rdi), %ymm0, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -4230,9 +4230,9 @@ define void @trunc_usat_v16i32_v16i8_store(<16 x i32>* %p0, <16 x i8>* %p1) {
 ; AVX2-NEXT:    vpminud 32(%rdi), %ymm0, %ymm1
 ; AVX2-NEXT:    vpminud (%rdi), %ymm0, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vmovdqa %xmm0, (%rsi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-trunc.ll b/llvm/test/CodeGen/X86/vector-trunc.ll
index b0a93a8102e1a..1d596f5db3aeb 100644
--- a/llvm/test/CodeGen/X86/vector-trunc.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc.ll
@@ -456,7 +456,7 @@ define <8 x i16> @trunc8i32_8i16_lshr(<8 x i32> %a) {
 ;
 ; SSSE3-LABEL: trunc8i32_8i16_lshr:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2,3,6,7,10,11,14,15,10,11,14,15,14,15,255,255]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2,3,6,7,10,11,14,15,10,11,14,15,14,15,128,128]
 ; SSSE3-NEXT:    pshufb %xmm2, %xmm1
 ; SSSE3-NEXT:    pshufb %xmm2, %xmm0
 ; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
@@ -927,9 +927,9 @@ define void @trunc16i32_16i8_ashr(<16 x i32> %a) {
 ; AVX2-NEXT:    vpsrad $24, %ymm1, %ymm1
 ; AVX2-NEXT:    vpsrad $24, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vmovdqu %xmm0, (%rax)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1004,9 +1004,9 @@ define void @trunc16i32_16i8_lshr(<16 x i32> %a) {
 ; AVX2-NEXT:    vpsrld $24, %ymm1, %ymm1
 ; AVX2-NEXT:    vpsrld $24, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vmovdqu %xmm0, (%rax)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vselect-packss.ll b/llvm/test/CodeGen/X86/vselect-packss.ll
index 34b336c8820f4..157245c0eee12 100644
--- a/llvm/test/CodeGen/X86/vselect-packss.ll
+++ b/llvm/test/CodeGen/X86/vselect-packss.ll
@@ -52,14 +52,23 @@ define <16 x i8> @vselect_packss_v16i16(<16 x i16> %a0, <16 x i16> %a1, <16 x i8
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512NOBW-LABEL: vselect_packss_v16i16:
-; AVX512NOBW:       # %bb.0:
-; AVX512NOBW-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
-; AVX512NOBW-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512NOBW-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512NOBW-NEXT:    vpblendvb %xmm0, %xmm2, %xmm3, %xmm0
-; AVX512NOBW-NEXT:    vzeroupper
-; AVX512NOBW-NEXT:    retq
+; AVX512F-LABEL: vselect_packss_v16i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vpblendvb %xmm0, %xmm2, %xmm3, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: vselect_packss_v16i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512VL-NEXT:    vpternlogq $202, %xmm3, %xmm2, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
 ;
 ; AVX512BWNOVL-LABEL: vselect_packss_v16i16:
 ; AVX512BWNOVL:       # %bb.0:
@@ -73,7 +82,7 @@ define <16 x i8> @vselect_packss_v16i16(<16 x i16> %a0, <16 x i16> %a1, <16 x i8
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpcmpeqw %ymm1, %ymm0, %k0
 ; AVX512BWVL-NEXT:    vpmovm2b %k0, %xmm0
-; AVX512BWVL-NEXT:    vpblendvb %xmm0, %xmm2, %xmm3, %xmm0
+; AVX512BWVL-NEXT:    vpternlogq $202, %xmm3, %xmm2, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
   %1 = icmp eq <16 x i16> %a0, %a1
@@ -137,21 +146,30 @@ define <16 x i8> @vselect_packss_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i8
 ; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpblendvb %xmm0, %xmm4, %xmm5, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512NOBW-LABEL: vselect_packss_v16i32:
-; AVX512NOBW:       # %bb.0:
-; AVX512NOBW-NEXT:    vpcmpeqd %zmm1, %zmm0, %k1
-; AVX512NOBW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512NOBW-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512NOBW-NEXT:    vpblendvb %xmm0, %xmm2, %xmm3, %xmm0
-; AVX512NOBW-NEXT:    vzeroupper
-; AVX512NOBW-NEXT:    retq
+; AVX512F-LABEL: vselect_packss_v16i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpcmpeqd %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vpblendvb %xmm0, %xmm2, %xmm3, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: vselect_packss_v16i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpcmpeqd %zmm1, %zmm0, %k1
+; AVX512VL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512VL-NEXT:    vpternlogq $202, %xmm3, %xmm2, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
 ;
 ; AVX512BWNOVL-LABEL: vselect_packss_v16i32:
 ; AVX512BWNOVL:       # %bb.0:
@@ -165,7 +183,7 @@ define <16 x i8> @vselect_packss_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i8
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
 ; AVX512BWVL-NEXT:    vpmovm2b %k0, %xmm0
-; AVX512BWVL-NEXT:    vpblendvb %xmm0, %xmm2, %xmm3, %xmm0
+; AVX512BWVL-NEXT:    vpternlogq $202, %xmm3, %xmm2, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
   %1 = icmp eq <16 x i32> %a0, %a1
@@ -283,25 +301,36 @@ define <16 x i8> @vselect_packss_v16i64(<16 x i64> %a0, <16 x i64> %a1, <16 x i8
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpackssdw %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
 ; AVX2-NEXT:    vpandn {{[0-9]+}}(%rsp), %xmm0, %xmm0
 ; AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512NOBW-LABEL: vselect_packss_v16i64:
-; AVX512NOBW:       # %bb.0:
-; AVX512NOBW-NEXT:    vpcmpeqq %zmm2, %zmm0, %k0
-; AVX512NOBW-NEXT:    vpcmpeqq %zmm3, %zmm1, %k1
-; AVX512NOBW-NEXT:    kunpckbw %k0, %k1, %k1
-; AVX512NOBW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512NOBW-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512NOBW-NEXT:    vpblendvb %xmm0, %xmm4, %xmm5, %xmm0
-; AVX512NOBW-NEXT:    vzeroupper
-; AVX512NOBW-NEXT:    retq
+; AVX512F-LABEL: vselect_packss_v16i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpcmpeqq %zmm2, %zmm0, %k0
+; AVX512F-NEXT:    vpcmpeqq %zmm3, %zmm1, %k1
+; AVX512F-NEXT:    kunpckbw %k0, %k1, %k1
+; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vpblendvb %xmm0, %xmm4, %xmm5, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: vselect_packss_v16i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpcmpeqq %zmm2, %zmm0, %k0
+; AVX512VL-NEXT:    vpcmpeqq %zmm3, %zmm1, %k1
+; AVX512VL-NEXT:    kunpckbw %k0, %k1, %k1
+; AVX512VL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512VL-NEXT:    vpternlogq $202, %xmm5, %xmm4, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
 ;
 ; AVX512BWNOVL-LABEL: vselect_packss_v16i64:
 ; AVX512BWNOVL:       # %bb.0:
@@ -319,7 +348,7 @@ define <16 x i8> @vselect_packss_v16i64(<16 x i64> %a0, <16 x i64> %a1, <16 x i8
 ; AVX512BWVL-NEXT:    vpcmpeqq %zmm3, %zmm1, %k1
 ; AVX512BWVL-NEXT:    kunpckbw %k0, %k1, %k0
 ; AVX512BWVL-NEXT:    vpmovm2b %k0, %xmm0
-; AVX512BWVL-NEXT:    vpblendvb %xmm0, %xmm4, %xmm5, %xmm0
+; AVX512BWVL-NEXT:    vpternlogq $202, %xmm5, %xmm4, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
   %1 = icmp eq <16 x i64> %a0, %a1
@@ -375,14 +404,41 @@ define <16 x i8> @vselect_packss(<16 x i16> %a0, <16 x i16> %a1, <16 x i8> %a2,
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: vselect_packss:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpblendvb %xmm0, %xmm2, %xmm3, %xmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: vselect_packss:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpblendvb %xmm0, %xmm2, %xmm3, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: vselect_packss:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512VL-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpternlogq $202, %xmm3, %xmm2, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BWNOVL-LABEL: vselect_packss:
+; AVX512BWNOVL:       # %bb.0:
+; AVX512BWNOVL-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
+; AVX512BWNOVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512BWNOVL-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX512BWNOVL-NEXT:    vpblendvb %xmm0, %xmm2, %xmm3, %xmm0
+; AVX512BWNOVL-NEXT:    vzeroupper
+; AVX512BWNOVL-NEXT:    retq
+;
+; AVX512BWVL-LABEL: vselect_packss:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512BWVL-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpternlogq $202, %xmm3, %xmm2, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
   %1 = icmp eq <16 x i16> %a0, %a1
   %2 = sext <16 x i1> %1 to <16 x i16>
   %3 = shufflevector <16 x i16> %2, <16 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
diff --git a/llvm/test/CodeGen/X86/vselect-pcmp.ll b/llvm/test/CodeGen/X86/vselect-pcmp.ll
index 4db1b18c97ec3..4c56c654defae 100644
--- a/llvm/test/CodeGen/X86/vselect-pcmp.ll
+++ b/llvm/test/CodeGen/X86/vselect-pcmp.ll
@@ -1,8 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx       | FileCheck %s --check-prefix=AVX --check-prefix=AVX12F --check-prefix=AVX12 --check-prefix=AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2      | FileCheck %s --check-prefix=AVX --check-prefix=AVX12F --check-prefix=AVX12 --check-prefix=AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f   | FileCheck %s --check-prefix=AVX --check-prefix=AVX12F --check-prefix=AVX512 --check-prefix=AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl  | FileCheck %s --check-prefix=AVX                       --check-prefix=AVX512 --check-prefix=AVX512VL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx       | FileCheck %s --check-prefixes=CHECK,AVX,AVX12F,AVX12,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2      | FileCheck %s --check-prefixes=CHECK,AVX,AVX12F,AVX12,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f   | FileCheck %s --check-prefixes=CHECK,AVX,AVX12F,AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl  | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512VL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=xop       | FileCheck %s --check-prefixes=CHECK,XOP
 
 ; The condition vector for BLENDV* only cares about the sign bit of each element.
 ; So in these tests, if we generate BLENDV*, we should be able to remove the redundant cmp op.
@@ -10,10 +11,10 @@
 ; Test 128-bit vectors for all legal element types.
 
 define <16 x i8> @signbit_sel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %mask) {
-; AVX-LABEL: signbit_sel_v16i8:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    retq
+; CHECK-LABEL: signbit_sel_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %tr = icmp slt <16 x i8> %mask, zeroinitializer
   %z = select <16 x i1> %tr, <16 x i8> %x, <16 x i8> %y
   ret <16 x i8> %z
@@ -28,6 +29,13 @@ define <8 x i16> @signbit_sel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %mask)
 ; AVX-NEXT:    vpcmpgtw %xmm2, %xmm3, %xmm2
 ; AVX-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
+;
+; XOP-LABEL: signbit_sel_v8i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; XOP-NEXT:    vpcomltw %xmm3, %xmm2, %xmm2
+; XOP-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    retq
   %tr = icmp slt <8 x i16> %mask, zeroinitializer
   %z = select <8 x i1> %tr, <8 x i16> %x, <8 x i16> %y
   ret <8 x i16> %z
@@ -57,6 +65,11 @@ define <4 x i32> @signbit_sel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %mask)
 ; AVX512VL-NEXT:    vpcmpgtd %xmm2, %xmm3, %k1
 ; AVX512VL-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: signbit_sel_v4i32:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    retq
   %tr = icmp slt <4 x i32> %mask, zeroinitializer
   %z = select <4 x i1> %tr, <4 x i32> %x, <4 x i32> %y
   ret <4 x i32> %z
@@ -86,6 +99,11 @@ define <2 x i64> @signbit_sel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %mask)
 ; AVX512VL-NEXT:    vpcmpgtq %xmm2, %xmm3, %k1
 ; AVX512VL-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
 ; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: signbit_sel_v2i64:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    retq
   %tr = icmp slt <2 x i64> %mask, zeroinitializer
   %z = select <2 x i1> %tr, <2 x i64> %x, <2 x i64> %y
   ret <2 x i64> %z
@@ -115,6 +133,11 @@ define <4 x float> @signbit_sel_v4f32(<4 x float> %x, <4 x float> %y, <4 x i32>
 ; AVX512VL-NEXT:    vpcmpgtd %xmm2, %xmm3, %k1
 ; AVX512VL-NEXT:    vblendmps %xmm0, %xmm1, %xmm0 {%k1}
 ; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: signbit_sel_v4f32:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    retq
   %tr = icmp slt <4 x i32> %mask, zeroinitializer
   %z = select <4 x i1> %tr, <4 x float> %x, <4 x float> %y
   ret <4 x float> %z
@@ -144,6 +167,11 @@ define <2 x double> @signbit_sel_v2f64(<2 x double> %x, <2 x double> %y, <2 x i6
 ; AVX512VL-NEXT:    vpcmpgtq %xmm2, %xmm3, %k1
 ; AVX512VL-NEXT:    vblendmpd %xmm0, %xmm1, %xmm0 {%k1}
 ; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: signbit_sel_v2f64:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    retq
   %tr = icmp slt <2 x i64> %mask, zeroinitializer
   %z = select <2 x i1> %tr, <2 x double> %x, <2 x double> %y
   ret <2 x double> %z
@@ -173,6 +201,16 @@ define <32 x i8> @signbit_sel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %mask)
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    retq
+;
+; XOP-LABEL: signbit_sel_v32i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; XOP-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; XOP-NEXT:    vpcomltb %xmm4, %xmm3, %xmm3
+; XOP-NEXT:    vpcomltb %xmm4, %xmm2, %xmm2
+; XOP-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; XOP-NEXT:    vpcmov %ymm2, %ymm1, %ymm0, %ymm0
+; XOP-NEXT:    retq
   %tr = icmp slt <32 x i8> %mask, zeroinitializer
   %z = select <32 x i1> %tr, <32 x i8> %x, <32 x i8> %y
   ret <32 x i8> %z
@@ -206,6 +244,16 @@ define <16 x i16> @signbit_sel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i16> %
 ; AVX512-NEXT:    vpcmpgtw %ymm2, %ymm3, %ymm2
 ; AVX512-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    retq
+;
+; XOP-LABEL: signbit_sel_v16i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; XOP-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; XOP-NEXT:    vpcomltw %xmm4, %xmm3, %xmm3
+; XOP-NEXT:    vpcomltw %xmm4, %xmm2, %xmm2
+; XOP-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; XOP-NEXT:    vpcmov %ymm2, %ymm1, %ymm0, %ymm0
+; XOP-NEXT:    retq
   %tr = icmp slt <16 x i16> %mask, zeroinitializer
   %z = select <16 x i1> %tr, <16 x i16> %x, <16 x i16> %y
   ret <16 x i16> %z
@@ -234,6 +282,11 @@ define <8 x i32> @signbit_sel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %mask)
 ; AVX512VL-NEXT:    vpcmpgtd %ymm2, %ymm3, %k1
 ; AVX512VL-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: signbit_sel_v8i32:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; XOP-NEXT:    retq
   %tr = icmp slt <8 x i32> %mask, zeroinitializer
   %z = select <8 x i1> %tr, <8 x i32> %x, <8 x i32> %y
   ret <8 x i32> %z
@@ -262,6 +315,11 @@ define <4 x i64> @signbit_sel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %mask)
 ; AVX512VL-NEXT:    vpcmpgtq %ymm2, %ymm3, %k1
 ; AVX512VL-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: signbit_sel_v4i64:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; XOP-NEXT:    retq
   %tr = icmp slt <4 x i64> %mask, zeroinitializer
   %z = select <4 x i1> %tr, <4 x i64> %x, <4 x i64> %y
   ret <4 x i64> %z
@@ -290,6 +348,11 @@ define <4 x double> @signbit_sel_v4f64(<4 x double> %x, <4 x double> %y, <4 x i6
 ; AVX512VL-NEXT:    vpcmpgtq %ymm2, %ymm3, %k1
 ; AVX512VL-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: signbit_sel_v4f64:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; XOP-NEXT:    retq
   %tr = icmp slt <4 x i64> %mask, zeroinitializer
   %z = select <4 x i1> %tr, <4 x double> %x, <4 x double> %y
   ret <4 x double> %z
@@ -330,6 +393,15 @@ define <4 x double> @signbit_sel_v4f64_small_mask(<4 x double> %x, <4 x double>
 ; AVX512VL-NEXT:    vpcmpgtd %xmm2, %xmm3, %k1
 ; AVX512VL-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: signbit_sel_v4f64_small_mask:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpmovsxdq %xmm2, %xmm3
+; XOP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; XOP-NEXT:    vpmovsxdq %xmm2, %xmm2
+; XOP-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; XOP-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; XOP-NEXT:    retq
   %tr = icmp slt <4 x i32> %mask, zeroinitializer
   %z = select <4 x i1> %tr, <4 x double> %x, <4 x double> %y
   ret <4 x double> %z
@@ -350,6 +422,12 @@ define <8 x double> @signbit_sel_v8f64(<8 x double> %x, <8 x double> %y, <8 x i6
 ; AVX512-NEXT:    vpcmpgtq %zmm2, %zmm3, %k1
 ; AVX512-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1}
 ; AVX512-NEXT:    retq
+;
+; XOP-LABEL: signbit_sel_v8f64:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vblendvpd %ymm4, %ymm0, %ymm2, %ymm0
+; XOP-NEXT:    vblendvpd %ymm5, %ymm1, %ymm3, %ymm1
+; XOP-NEXT:    retq
   %tr = icmp slt <8 x i64> %mask, zeroinitializer
   %z = select <8 x i1> %tr, <8 x double> %x, <8 x double> %y
   ret <8 x double> %z
@@ -384,9 +462,850 @@ define <4 x float> @signbit_sel_v4f32_fcmp(<4 x float> %x, <4 x float> %y, <4 x
 ; AVX512VL-NEXT:    vcmpltps %xmm2, %xmm0, %k1
 ; AVX512VL-NEXT:    vblendmps %xmm0, %xmm1, %xmm0 {%k1}
 ; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: signbit_sel_v4f32_fcmp:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; XOP-NEXT:    vcmpltps %xmm2, %xmm0, %xmm2
+; XOP-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    retq
   %cmp = fcmp olt <4 x float> %x, zeroinitializer
   %sel = select <4 x i1> %cmp, <4 x float> %x, <4 x float> %y
   ret <4 x float> %sel
 }
 
+define <4 x i64> @blend_splat1_mask_cond_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %z) {
+; AVX1-LABEL: blend_splat1_mask_cond_v4i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsllq $63, %xmm0, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsllq $63, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm2, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: blend_splat1_mask_cond_v4i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllq $63, %ymm0, %ymm0
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm2, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: blend_splat1_mask_cond_v4i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vptestnmq {{.*}}(%rip){1to8}, %zmm0, %k1
+; AVX512F-NEXT:    vpblendmq %zmm1, %zmm2, %zmm0 {%k1}
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: blend_splat1_mask_cond_v4i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestnmq {{.*}}(%rip){1to4}, %ymm0, %k1
+; AVX512VL-NEXT:    vpblendmq %ymm1, %ymm2, %ymm0 {%k1}
+; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: blend_splat1_mask_cond_v4i64:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; XOP-NEXT:    vpsllq $63, %xmm3, %xmm3
+; XOP-NEXT:    vmovdqa {{.*#+}} xmm4 = [18446744073709551553,18446744073709551553]
+; XOP-NEXT:    vpshaq %xmm4, %xmm3, %xmm3
+; XOP-NEXT:    vpsllq $63, %xmm0, %xmm0
+; XOP-NEXT:    vpshaq %xmm4, %xmm0, %xmm0
+; XOP-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; XOP-NEXT:    vblendvpd %ymm0, %ymm2, %ymm1, %ymm0
+; XOP-NEXT:    retq
+  %a = and <4 x i64> %x, <i64 1, i64 1, i64 1, i64 1>
+  %c = icmp eq <4 x i64> %a, zeroinitializer
+  %r = select <4 x i1> %c, <4 x i64> %y, <4 x i64> %z
+  ret <4 x i64> %r
+}
+
+define <4 x i32> @blend_splat1_mask_cond_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
+; AVX12-LABEL: blend_splat1_mask_cond_v4i32:
+; AVX12:       # %bb.0:
+; AVX12-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX12-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
+; AVX12-NEXT:    retq
+;
+; AVX512F-LABEL: blend_splat1_mask_cond_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vptestnmd {{.*}}(%rip){1to16}, %zmm0, %k1
+; AVX512F-NEXT:    vpblendmd %zmm1, %zmm2, %zmm0 {%k1}
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: blend_splat1_mask_cond_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestnmd {{.*}}(%rip){1to4}, %xmm0, %k1
+; AVX512VL-NEXT:    vpblendmd %xmm1, %xmm2, %xmm0 {%k1}
+; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: blend_splat1_mask_cond_v4i32:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; XOP-NEXT:    vpcomneqd %xmm3, %xmm0, %xmm0
+; XOP-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
+; XOP-NEXT:    retq
+  %a = and <4 x i32> %x, <i32 1, i32 1, i32 1, i32 1>
+  %c = icmp eq <4 x i32> %a, zeroinitializer
+  %r = select <4 x i1> %c, <4 x i32> %y, <4 x i32> %z
+  ret <4 x i32> %r
+}
+
+define <16 x i16> @blend_splat1_mask_cond_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i16> %z) {
+; AVX1-LABEL: blend_splat1_mask_cond_v16i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsllw $15, %xmm0, %xmm3
+; AVX1-NEXT:    vpsraw $15, %xmm3, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsllw $15, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $15, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT:    vandnps %ymm1, %ymm0, %ymm1
+; AVX1-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: blend_splat1_mask_cond_v16i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllw $15, %ymm0, %ymm0
+; AVX2-NEXT:    vpsraw $15, %ymm0, %ymm0
+; AVX2-NEXT:    vpblendvb %ymm0, %ymm2, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: blend_splat1_mask_cond_v16i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512-NEXT:    vpcmpeqw %ymm3, %ymm0, %ymm0
+; AVX512-NEXT:    vpblendvb %ymm0, %ymm1, %ymm2, %ymm0
+; AVX512-NEXT:    retq
+;
+; XOP-LABEL: blend_splat1_mask_cond_v16i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpsllw $15, %xmm0, %xmm3
+; XOP-NEXT:    vpsraw $15, %xmm3, %xmm3
+; XOP-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; XOP-NEXT:    vpsllw $15, %xmm0, %xmm0
+; XOP-NEXT:    vpsraw $15, %xmm0, %xmm0
+; XOP-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; XOP-NEXT:    vpcmov %ymm0, %ymm1, %ymm2, %ymm0
+; XOP-NEXT:    retq
+  %a = and <16 x i16> %x, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %c = icmp eq <16 x i16> %a, zeroinitializer
+  %r = select <16 x i1> %c, <16 x i16> %y, <16 x i16> %z
+  ret <16 x i16> %r
+}
+
+define <16 x i8> @blend_splat1_mask_cond_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %z) {
+; AVX12-LABEL: blend_splat1_mask_cond_v16i8:
+; AVX12:       # %bb.0:
+; AVX12-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX12-NEXT:    vpblendvb %xmm0, %xmm2, %xmm1, %xmm0
+; AVX12-NEXT:    retq
+;
+; AVX512-LABEL: blend_splat1_mask_cond_v16i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
+; AVX512-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm0
+; AVX512-NEXT:    retq
+;
+; XOP-LABEL: blend_splat1_mask_cond_v16i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; XOP-NEXT:    vpcomneqb %xmm3, %xmm0, %xmm0
+; XOP-NEXT:    vpblendvb %xmm0, %xmm2, %xmm1, %xmm0
+; XOP-NEXT:    retq
+  %a = and <16 x i8> %x, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  %c = icmp eq <16 x i8> %a, zeroinitializer
+  %r = select <16 x i1> %c, <16 x i8> %y, <16 x i8> %z
+  ret <16 x i8> %r
+}
+
+define <2 x i64> @blend_splatmax_mask_cond_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %z) {
+; AVX12-LABEL: blend_splatmax_mask_cond_v2i64:
+; AVX12:       # %bb.0:
+; AVX12-NEXT:    vblendvpd %xmm0, %xmm2, %xmm1, %xmm0
+; AVX12-NEXT:    retq
+;
+; AVX512F-LABEL: blend_splatmax_mask_cond_v2i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
+; AVX512F-NEXT:    vptestnmq %zmm3, %zmm0, %k1
+; AVX512F-NEXT:    vpblendmq %zmm1, %zmm2, %zmm0 {%k1}
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: blend_splatmax_mask_cond_v2i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestnmq {{.*}}(%rip), %xmm0, %k1
+; AVX512VL-NEXT:    vpblendmq %xmm1, %xmm2, %xmm0 {%k1}
+; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: blend_splatmax_mask_cond_v2i64:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; XOP-NEXT:    vpcomneqq %xmm3, %xmm0, %xmm0
+; XOP-NEXT:    vblendvpd %xmm0, %xmm2, %xmm1, %xmm0
+; XOP-NEXT:    retq
+  %a = and <2 x i64> %x, <i64 9223372036854775808, i64 9223372036854775808>
+  %c = icmp eq <2 x i64> %a, zeroinitializer
+  %r = select <2 x i1> %c, <2 x i64> %y, <2 x i64> %z
+  ret <2 x i64> %r
+}
+
+define <8 x i32> @blend_splatmax_mask_cond_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %z) {
+; AVX12-LABEL: blend_splatmax_mask_cond_v8i32:
+; AVX12:       # %bb.0:
+; AVX12-NEXT:    vblendvps %ymm0, %ymm2, %ymm1, %ymm0
+; AVX12-NEXT:    retq
+;
+; AVX512F-LABEL: blend_splatmax_mask_cond_v8i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vptestnmd {{.*}}(%rip){1to16}, %zmm0, %k1
+; AVX512F-NEXT:    vpblendmd %zmm1, %zmm2, %zmm0 {%k1}
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: blend_splatmax_mask_cond_v8i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestnmd {{.*}}(%rip){1to8}, %ymm0, %k1
+; AVX512VL-NEXT:    vpblendmd %ymm1, %ymm2, %ymm0 {%k1}
+; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: blend_splatmax_mask_cond_v8i32:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vblendvps %ymm0, %ymm2, %ymm1, %ymm0
+; XOP-NEXT:    retq
+  %a = and <8 x i32> %x, <i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648>
+  %c = icmp eq <8 x i32> %a, zeroinitializer
+  %r = select <8 x i1> %c, <8 x i32> %y, <8 x i32> %z
+  ret <8 x i32> %r
+}
+
+define <8 x i16> @blend_splatmax_mask_cond_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %z) {
+; AVX12-LABEL: blend_splatmax_mask_cond_v8i16:
+; AVX12:       # %bb.0:
+; AVX12-NEXT:    vpsraw $15, %xmm0, %xmm0
+; AVX12-NEXT:    vpblendvb %xmm0, %xmm2, %xmm1, %xmm0
+; AVX12-NEXT:    retq
+;
+; AVX512-LABEL: blend_splatmax_mask_cond_v8i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512-NEXT:    vpcmpeqw %xmm3, %xmm0, %xmm0
+; AVX512-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm0
+; AVX512-NEXT:    retq
+;
+; XOP-LABEL: blend_splatmax_mask_cond_v8i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; XOP-NEXT:    vpcomneqw %xmm3, %xmm0, %xmm0
+; XOP-NEXT:    vpblendvb %xmm0, %xmm2, %xmm1, %xmm0
+; XOP-NEXT:    retq
+  %a = and <8 x i16> %x, <i16 32768, i16 32768, i16 32768, i16 32768, i16 32768, i16 32768, i16 32768, i16 32768>
+  %c = icmp eq <8 x i16> %a, zeroinitializer
+  %r = select <8 x i1> %c, <8 x i16> %y, <8 x i16> %z
+  ret <8 x i16> %r
+}
+
+define <32 x i8> @blend_splatmax_mask_cond_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %z) {
+; AVX1-LABEL: blend_splatmax_mask_cond_v32i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpgtb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    vandnps %ymm1, %ymm0, %ymm1
+; AVX1-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: blend_splatmax_mask_cond_v32i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpblendvb %ymm0, %ymm2, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: blend_splatmax_mask_cond_v32i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm0
+; AVX512-NEXT:    vpblendvb %ymm0, %ymm1, %ymm2, %ymm0
+; AVX512-NEXT:    retq
+;
+; XOP-LABEL: blend_splatmax_mask_cond_v32i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; XOP-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; XOP-NEXT:    vpcmpgtb %xmm3, %xmm4, %xmm3
+; XOP-NEXT:    vpcmpgtb %xmm0, %xmm4, %xmm0
+; XOP-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; XOP-NEXT:    vpcmov %ymm0, %ymm1, %ymm2, %ymm0
+; XOP-NEXT:    retq
+  %a = and <32 x i8> %x, <i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128>
+  %c = icmp eq <32 x i8> %a, zeroinitializer
+  %r = select <32 x i1> %c, <32 x i8> %y, <32 x i8> %z
+  ret <32 x i8> %r
+}
+
+define <4 x i64> @blend_splat_mask_cond_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %z) {
+; AVX1-LABEL: blend_splat_mask_cond_v4i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsllq $62, %xmm0, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsllq $62, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm2, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: blend_splat_mask_cond_v4i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllq $62, %ymm0, %ymm0
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm2, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: blend_splat_mask_cond_v4i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vptestnmq {{.*}}(%rip){1to8}, %zmm0, %k1
+; AVX512F-NEXT:    vpblendmq %zmm1, %zmm2, %zmm0 {%k1}
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: blend_splat_mask_cond_v4i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestnmq {{.*}}(%rip){1to4}, %ymm0, %k1
+; AVX512VL-NEXT:    vpblendmq %ymm1, %ymm2, %ymm0 {%k1}
+; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: blend_splat_mask_cond_v4i64:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; XOP-NEXT:    vpsllq $62, %xmm3, %xmm3
+; XOP-NEXT:    vmovdqa {{.*#+}} xmm4 = [18446744073709551553,18446744073709551553]
+; XOP-NEXT:    vpshaq %xmm4, %xmm3, %xmm3
+; XOP-NEXT:    vpsllq $62, %xmm0, %xmm0
+; XOP-NEXT:    vpshaq %xmm4, %xmm0, %xmm0
+; XOP-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; XOP-NEXT:    vblendvpd %ymm0, %ymm2, %ymm1, %ymm0
+; XOP-NEXT:    retq
+  %a = and <4 x i64> %x, <i64 2, i64 2, i64 2, i64 2>
+  %c = icmp eq <4 x i64> %a, zeroinitializer
+  %r = select <4 x i1> %c, <4 x i64> %y, <4 x i64> %z
+  ret <4 x i64> %r
+}
+
+define <4 x i32> @blend_splat_mask_cond_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
+; AVX12-LABEL: blend_splat_mask_cond_v4i32:
+; AVX12:       # %bb.0:
+; AVX12-NEXT:    vpslld $15, %xmm0, %xmm0
+; AVX12-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
+; AVX12-NEXT:    retq
+;
+; AVX512F-LABEL: blend_splat_mask_cond_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vptestnmd {{.*}}(%rip){1to16}, %zmm0, %k1
+; AVX512F-NEXT:    vpblendmd %zmm1, %zmm2, %zmm0 {%k1}
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: blend_splat_mask_cond_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestnmd {{.*}}(%rip){1to4}, %xmm0, %k1
+; AVX512VL-NEXT:    vpblendmd %xmm1, %xmm2, %xmm0 {%k1}
+; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: blend_splat_mask_cond_v4i32:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; XOP-NEXT:    vpcomneqd %xmm3, %xmm0, %xmm0
+; XOP-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
+; XOP-NEXT:    retq
+  %a = and <4 x i32> %x, <i32 65536, i32 65536, i32 65536, i32 65536>
+  %c = icmp eq <4 x i32> %a, zeroinitializer
+  %r = select <4 x i1> %c, <4 x i32> %y, <4 x i32> %z
+  ret <4 x i32> %r
+}
+
+define <16 x i16> @blend_splat_mask_cond_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i16> %z) {
+; AVX1-LABEL: blend_splat_mask_cond_v16i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsllw $5, %xmm0, %xmm3
+; AVX1-NEXT:    vpsraw $15, %xmm3, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsllw $5, %xmm0, %xmm0
+; AVX1-NEXT:    vpsraw $15, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT:    vandnps %ymm1, %ymm0, %ymm1
+; AVX1-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: blend_splat_mask_cond_v16i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllw $5, %ymm0, %ymm0
+; AVX2-NEXT:    vpsraw $15, %ymm0, %ymm0
+; AVX2-NEXT:    vpblendvb %ymm0, %ymm2, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: blend_splat_mask_cond_v16i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512-NEXT:    vpcmpeqw %ymm3, %ymm0, %ymm0
+; AVX512-NEXT:    vpblendvb %ymm0, %ymm1, %ymm2, %ymm0
+; AVX512-NEXT:    retq
+;
+; XOP-LABEL: blend_splat_mask_cond_v16i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpsllw $5, %xmm0, %xmm3
+; XOP-NEXT:    vpsraw $15, %xmm3, %xmm3
+; XOP-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; XOP-NEXT:    vpsllw $5, %xmm0, %xmm0
+; XOP-NEXT:    vpsraw $15, %xmm0, %xmm0
+; XOP-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; XOP-NEXT:    vpcmov %ymm0, %ymm1, %ymm2, %ymm0
+; XOP-NEXT:    retq
+  %a = and <16 x i16> %x, <i16 1024, i16 1024, i16 1024, i16 1024, i16 1024, i16 1024, i16 1024, i16 1024, i16 1024, i16 1024, i16 1024, i16 1024, i16 1024, i16 1024, i16 1024, i16 1024>
+  %c = icmp eq <16 x i16> %a, zeroinitializer
+  %r = select <16 x i1> %c, <16 x i16> %y, <16 x i16> %z
+  ret <16 x i16> %r
+}
+
+define <16 x i8> @blend_splat_mask_cond_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %z) {
+; AVX12-LABEL: blend_splat_mask_cond_v16i8:
+; AVX12:       # %bb.0:
+; AVX12-NEXT:    vpsllw $5, %xmm0, %xmm0
+; AVX12-NEXT:    vpblendvb %xmm0, %xmm2, %xmm1, %xmm0
+; AVX12-NEXT:    retq
+;
+; AVX512-LABEL: blend_splat_mask_cond_v16i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
+; AVX512-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm0
+; AVX512-NEXT:    retq
+;
+; XOP-LABEL: blend_splat_mask_cond_v16i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; XOP-NEXT:    vpcomneqb %xmm3, %xmm0, %xmm0
+; XOP-NEXT:    vpblendvb %xmm0, %xmm2, %xmm1, %xmm0
+; XOP-NEXT:    retq
+  %a = and <16 x i8> %x, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
+  %c = icmp eq <16 x i8> %a, zeroinitializer
+  %r = select <16 x i1> %c, <16 x i8> %y, <16 x i8> %z
+  ret <16 x i8> %r
+}
+
+define <2 x i64> @blend_mask_cond_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %z) {
+; AVX1-LABEL: blend_mask_cond_v2i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vblendvpd %xmm0, %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: blend_mask_cond_v2i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vblendvpd %xmm0, %xmm2, %xmm1, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: blend_mask_cond_v2i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,4]
+; AVX512F-NEXT:    vptestnmq %zmm3, %zmm0, %k1
+; AVX512F-NEXT:    vpblendmq %zmm1, %zmm2, %zmm0 {%k1}
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: blend_mask_cond_v2i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestnmq {{.*}}(%rip), %xmm0, %k1
+; AVX512VL-NEXT:    vpblendmq %xmm1, %xmm2, %xmm0 {%k1}
+; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: blend_mask_cond_v2i64:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshlq {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vblendvpd %xmm0, %xmm2, %xmm1, %xmm0
+; XOP-NEXT:    retq
+  %a = and <2 x i64> %x, <i64 1, i64 4>
+  %c = icmp eq <2 x i64> %a, zeroinitializer
+  %r = select <2 x i1> %c, <2 x i64> %y, <2 x i64> %z
+  ret <2 x i64> %r
+}
+
+define <4 x i32> @blend_mask_cond_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
+; AVX1-LABEL: blend_mask_cond_v4i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: blend_mask_cond_v4i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: blend_mask_cond_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [65536,512,2,1]
+; AVX512F-NEXT:    vptestnmd %zmm3, %zmm0, %k1
+; AVX512F-NEXT:    vpblendmd %zmm1, %zmm2, %zmm0 {%k1}
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: blend_mask_cond_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestnmd {{.*}}(%rip), %xmm0, %k1
+; AVX512VL-NEXT:    vpblendmd %xmm1, %xmm2, %xmm0 {%k1}
+; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: blend_mask_cond_v4i32:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshld {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
+; XOP-NEXT:    retq
+  %a = and <4 x i32> %x, <i32 65536, i32 512, i32 2, i32 1>
+  %c = icmp eq <4 x i32> %a, zeroinitializer
+  %r = select <4 x i1> %c, <4 x i32> %y, <4 x i32> %z
+  ret <4 x i32> %r
+}
+
+define <8 x i16> @blend_mask_cond_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %z) {
+; AVX-LABEL: blend_mask_cond_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX-NEXT:    vpcmpeqw %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm0
+; AVX-NEXT:    retq
+;
+; XOP-LABEL: blend_mask_cond_v8i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; XOP-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpcomltw %xmm3, %xmm0, %xmm0
+; XOP-NEXT:    vpblendvb %xmm0, %xmm2, %xmm1, %xmm0
+; XOP-NEXT:    retq
+  %a = and <8 x i16> %x, <i16 1, i16 2, i16 8, i16 4, i16 8, i16 1024, i16 2, i16 4096>
+  %c = icmp eq <8 x i16> %a, zeroinitializer
+  %r = select <8 x i1> %c, <8 x i16> %y, <8 x i16> %z
+  ret <8 x i16> %r
+}
+
+define <16 x i8> @blend_mask_cond_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %z) {
+; AVX-LABEL: blend_mask_cond_v16i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm0
+; AVX-NEXT:    retq
+;
+; XOP-LABEL: blend_mask_cond_v16i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpblendvb %xmm0, %xmm2, %xmm1, %xmm0
+; XOP-NEXT:    retq
+  %a = and <16 x i8> %x, <i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 4, i8 4, i8 4, i8 4, i8 2, i8 2, i8 2, i8 2>
+  %c = icmp eq <16 x i8> %a, zeroinitializer
+  %r = select <16 x i1> %c, <16 x i8> %y, <16 x i8> %z
+  ret <16 x i8> %r
+}
+
+define <4 x i64> @blend_mask_cond_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %z) {
+; AVX1-LABEL: blend_mask_cond_v4i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqq %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqq %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm1, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: blend_mask_cond_v4i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm2, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: blend_mask_cond_v4i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [2,4,32768,1]
+; AVX512F-NEXT:    vptestnmq %zmm3, %zmm0, %k1
+; AVX512F-NEXT:    vpblendmq %zmm1, %zmm2, %zmm0 {%k1}
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: blend_mask_cond_v4i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestnmq {{.*}}(%rip), %ymm0, %k1
+; AVX512VL-NEXT:    vpblendmq %ymm1, %ymm2, %ymm0 {%k1}
+; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: blend_mask_cond_v4i64:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshlq {{.*}}(%rip), %xmm0, %xmm3
+; XOP-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; XOP-NEXT:    vpshlq {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; XOP-NEXT:    vblendvpd %ymm0, %ymm2, %ymm1, %ymm0
+; XOP-NEXT:    retq
+  %a = and <4 x i64> %x, <i64 2, i64 4, i64 32768, i64 1>
+  %c = icmp eq <4 x i64> %a, zeroinitializer
+  %r = select <4 x i1> %c, <4 x i64> %y, <4 x i64> %z
+  ret <4 x i64> %r
+}
+
+define <8 x i32> @blend_mask_cond_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %z) {
+; AVX1-LABEL: blend_mask_cond_v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT:    vblendvps %ymm0, %ymm2, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: blend_mask_cond_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vblendvps %ymm0, %ymm2, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: blend_mask_cond_v8i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [1,2,8,4,8,1024,2,4096]
+; AVX512F-NEXT:    vptestnmd %zmm3, %zmm0, %k1
+; AVX512F-NEXT:    vpblendmd %zmm1, %zmm2, %zmm0 {%k1}
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: blend_mask_cond_v8i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestnmd {{.*}}(%rip), %ymm0, %k1
+; AVX512VL-NEXT:    vpblendmd %ymm1, %ymm2, %ymm0 {%k1}
+; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: blend_mask_cond_v8i32:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshld {{.*}}(%rip), %xmm0, %xmm3
+; XOP-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; XOP-NEXT:    vpshld {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; XOP-NEXT:    vblendvps %ymm0, %ymm2, %ymm1, %ymm0
+; XOP-NEXT:    retq
+  %a = and <8 x i32> %x, <i32 1, i32 2, i32 8, i32 4, i32 8, i32 1024, i32 2, i32 4096>
+  %c = icmp eq <8 x i32> %a, zeroinitializer
+  %r = select <8 x i1> %c, <8 x i32> %y, <8 x i32> %z
+  ret <8 x i32> %r
+}
+
+define <16 x i16> @blend_mask_cond_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i16> %z) {
+; AVX1-LABEL: blend_mask_cond_v16i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    vandnps %ymm2, %ymm0, %ymm2
+; AVX1-NEXT:    vandps %ymm0, %ymm1, %ymm0
+; AVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: blend_mask_cond_v16i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpeqw %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpblendvb %ymm0, %ymm1, %ymm2, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: blend_mask_cond_v16i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512-NEXT:    vpcmpeqw %ymm3, %ymm0, %ymm0
+; AVX512-NEXT:    vpblendvb %ymm0, %ymm1, %ymm2, %ymm0
+; AVX512-NEXT:    retq
+;
+; XOP-LABEL: blend_mask_cond_v16i16:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; XOP-NEXT:    vpshlw {{.*}}(%rip), %xmm3, %xmm3
+; XOP-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; XOP-NEXT:    vpcomltw %xmm4, %xmm3, %xmm3
+; XOP-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpcomltw %xmm4, %xmm0, %xmm0
+; XOP-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; XOP-NEXT:    vpcmov %ymm0, %ymm1, %ymm2, %ymm0
+; XOP-NEXT:    retq
+  %a = and <16 x i16> %x, <i16 1, i16 2, i16 8, i16 4, i16 8, i16 2, i16 2, i16 2, i16 2, i16 8, i16 8, i16 64, i16 64, i16 1024, i16 4096, i16 1024>
+  %c = icmp eq <16 x i16> %a, zeroinitializer
+  %r = select <16 x i1> %c, <16 x i16> %y, <16 x i16> %z
+  ret <16 x i16> %r
+}
+
+define <32 x i8> @blend_mask_cond_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %z) {
+; AVX1-LABEL: blend_mask_cond_v32i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    vandnps %ymm2, %ymm0, %ymm2
+; AVX1-NEXT:    vandps %ymm0, %ymm1, %ymm0
+; AVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: blend_mask_cond_v32i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpblendvb %ymm0, %ymm1, %ymm2, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: blend_mask_cond_v32i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm0
+; AVX512-NEXT:    vpblendvb %ymm0, %ymm1, %ymm2, %ymm0
+; AVX512-NEXT:    retq
+;
+; XOP-LABEL: blend_mask_cond_v32i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm3, %xmm3
+; XOP-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; XOP-NEXT:    vpcomltb %xmm4, %xmm3, %xmm3
+; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
+; XOP-NEXT:    vpcomltb %xmm4, %xmm0, %xmm0
+; XOP-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; XOP-NEXT:    vpcmov %ymm0, %ymm1, %ymm2, %ymm0
+; XOP-NEXT:    retq
+  %a = and <32 x i8> %x, <i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 4, i8 4, i8 4, i8 4, i8 2, i8 2, i8 2, i8 2, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 4, i8 4, i8 4, i8 4, i8 128, i8 4, i8 2, i8 16>
+  %c = icmp eq <32 x i8> %a, zeroinitializer
+  %r = select <32 x i1> %c, <32 x i8> %y, <32 x i8> %z
+  ret <32 x i8> %r
+}
+
+define void @PR46531(i32* %x, i32* %y, i32* %z) {
+; AVX12-LABEL: PR46531:
+; AVX12:       # %bb.0:
+; AVX12-NEXT:    vmovdqu (%rsi), %xmm0
+; AVX12-NEXT:    vmovdqu (%rdx), %xmm1
+; AVX12-NEXT:    vpor %xmm0, %xmm1, %xmm2
+; AVX12-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; AVX12-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX12-NEXT:    vblendvps %xmm1, %xmm0, %xmm2, %xmm0
+; AVX12-NEXT:    vmovups %xmm0, (%rdi)
+; AVX12-NEXT:    retq
+;
+; AVX512F-LABEL: PR46531:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovdqu (%rsi), %xmm0
+; AVX512F-NEXT:    vmovdqu (%rdx), %xmm1
+; AVX512F-NEXT:    vpor %xmm0, %xmm1, %xmm2
+; AVX512F-NEXT:    vptestnmd {{.*}}(%rip){1to16}, %zmm1, %k1
+; AVX512F-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT:    vmovdqa32 %zmm2, %zmm0 {%k1}
+; AVX512F-NEXT:    vmovdqu %xmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: PR46531:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqu (%rsi), %xmm0
+; AVX512VL-NEXT:    vmovdqu (%rdx), %xmm1
+; AVX512VL-NEXT:    vptestnmd {{.*}}(%rip){1to4}, %xmm1, %k1
+; AVX512VL-NEXT:    vpxor %xmm0, %xmm1, %xmm2
+; AVX512VL-NEXT:    vpord %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512VL-NEXT:    vmovdqu %xmm2, (%rdi)
+; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: PR46531:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vmovdqu (%rsi), %xmm0
+; XOP-NEXT:    vmovdqu (%rdx), %xmm1
+; XOP-NEXT:    vpor %xmm0, %xmm1, %xmm2
+; XOP-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm3
+; XOP-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; XOP-NEXT:    vpcomneqd %xmm4, %xmm3, %xmm3
+; XOP-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    vblendvps %xmm3, %xmm0, %xmm2, %xmm0
+; XOP-NEXT:    vmovups %xmm0, (%rdi)
+; XOP-NEXT:    retq
+  %vy = bitcast i32* %y to <4 x i32>*
+  %a = load <4 x i32>, <4 x i32>* %vy, align 4
+  %vz = bitcast i32* %z to <4 x i32>*
+  %b = load <4 x i32>, <4 x i32>* %vz, align 4
+  %or = or <4 x i32> %b, %a
+  %and = and <4 x i32> %b, <i32 1, i32 1, i32 1, i32 1>
+  %cmp = icmp eq <4 x i32> %and, zeroinitializer
+  %xor = xor <4 x i32> %b, %a
+  %sel = select <4 x i1> %cmp, <4 x i32> %or, <4 x i32> %xor
+  %vx = bitcast i32* %x to <4 x i32>*
+  store <4 x i32> %sel, <4 x i32>* %vx, align 4
+  ret void
+}
+
 attributes #0 = { "no-nans-fp-math"="true" }
diff --git a/llvm/test/CodeGen/X86/win64-byval.ll b/llvm/test/CodeGen/X86/win64-byval.ll
index 2fefa736e64e7..af2af4e6e9b9e 100644
--- a/llvm/test/CodeGen/X86/win64-byval.ll
+++ b/llvm/test/CodeGen/X86/win64-byval.ll
@@ -32,3 +32,31 @@ define void @baz({ float, double }* byval %arg)
     call void @foo({ float, double }* byval %arg)
     ret void
 }
+
+declare void @foo2({ float, double }* byval, { float, double }* byval, { float, double }* byval, { float, double }* byval, { float, double }* byval, i64 %f)
+@data = external constant { float, double }
+
+define void @test() {
+; CHECK-LABEL: @test
+; CHECK:      movq    (%rax), %rcx
+; CHECK-NEXT: movq    8(%rax), %rax
+; CHECK-NEXT: movq    %rax, 120(%rsp)
+; CHECK-NEXT: movq    %rcx, 112(%rsp)
+; CHECK-NEXT: movq    %rcx, 96(%rsp)
+; CHECK-NEXT: movq    %rax, 104(%rsp)
+; CHECK-NEXT: movq    %rcx, 80(%rsp)
+; CHECK-NEXT: movq    %rax, 88(%rsp)
+; CHECK-NEXT: movq    %rcx, 64(%rsp)
+; CHECK-NEXT: movq    %rax, 72(%rsp)
+; CHECK-NEXT: movq    %rax, 56(%rsp)
+; CHECK-NEXT: movq    %rcx, 48(%rsp)
+; CHECK-NEXT: leaq    48(%rsp), %rax
+; CHECK-NEXT: movq    %rax, 32(%rsp)
+; CHECK-NEXT: movq    $10, 40(%rsp)
+; CHECK-NEXT: leaq    112(%rsp), %rcx
+; CHECK-NEXT: leaq    96(%rsp), %rdx
+; CHECK-NEXT: leaq    80(%rsp), %r8
+; CHECK-NEXT: leaq    64(%rsp), %r9
+  call void @foo2({ float, double }* byval @G, { float, double }* byval @G, { float, double }* byval @G, { float, double }* byval @G, { float, double }* byval @G, i64 10)
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/x86-interleaved-access.ll b/llvm/test/CodeGen/X86/x86-interleaved-access.ll
index c323db1deeac1..b80775ac7d57c 100644
--- a/llvm/test/CodeGen/X86/x86-interleaved-access.ll
+++ b/llvm/test/CodeGen/X86/x86-interleaved-access.ll
@@ -584,279 +584,265 @@ define <16 x i1> @interleaved_load_vf16_i8_stride4(<64 x i8>* %ptr) {
 define <32 x i1> @interleaved_load_vf32_i8_stride4(<128 x i8>* %ptr) {
 ; AVX1-LABEL: interleaved_load_vf32_i8_stride4:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm0 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vmovdqa 112(%rdi), %xmm11
-; AVX1-NEXT:    vpshufb %xmm0, %xmm11, %xmm1
-; AVX1-NEXT:    vmovdqa 96(%rdi), %xmm12
-; AVX1-NEXT:    vpshufb %xmm0, %xmm12, %xmm3
-; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vmovdqa 80(%rdi), %xmm14
-; AVX1-NEXT:    vpshufb %xmm2, %xmm14, %xmm4
-; AVX1-NEXT:    vmovdqa 64(%rdi), %xmm6
-; AVX1-NEXT:    vpshufb %xmm2, %xmm6, %xmm5
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm10
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm11
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm12
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm13
+; AVX1-NEXT:    vpshufb %xmm6, %xmm13, %xmm4
+; AVX1-NEXT:    vpshufb %xmm6, %xmm12, %xmm5
 ; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm4[0,1,2,3],xmm1[4,5,6,7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm8
-; AVX1-NEXT:    vmovdqa (%rdi), %xmm13
-; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm15
-; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm7
-; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm5
-; AVX1-NEXT:    vpshufb %xmm0, %xmm5, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm0 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm0, %xmm11, %xmm5
+; AVX1-NEXT:    vpshufb %xmm0, %xmm10, %xmm7
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm8 = xmm5[0,1,2,3],xmm4[4,5,6,7]
+; AVX1-NEXT:    vmovdqa 112(%rdi), %xmm14
+; AVX1-NEXT:    vpshufb %xmm6, %xmm14, %xmm7
+; AVX1-NEXT:    vmovdqa 96(%rdi), %xmm5
+; AVX1-NEXT:    vpshufb %xmm6, %xmm5, %xmm6
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; AVX1-NEXT:    vmovdqa 80(%rdi), %xmm6
+; AVX1-NEXT:    vpshufb %xmm0, %xmm6, %xmm2
+; AVX1-NEXT:    vmovdqa 64(%rdi), %xmm7
 ; AVX1-NEXT:    vpshufb %xmm0, %xmm7, %xmm0
-; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX1-NEXT:    vpshufb %xmm2, %xmm15, %xmm1
-; AVX1-NEXT:    vpshufb %xmm2, %xmm13, %xmm2
-; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm8 = ymm0[0,1,2,3],ymm8[4,5,6,7]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm0 = <u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm0, %xmm11, %xmm1
-; AVX1-NEXT:    vpshufb %xmm0, %xmm12, %xmm2
-; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm9 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = <u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm1, %xmm13, %xmm2
+; AVX1-NEXT:    vpshufb %xmm1, %xmm12, %xmm0
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm2, %xmm14, %xmm3
-; AVX1-NEXT:    vpshufb %xmm2, %xmm6, %xmm4
+; AVX1-NEXT:    vpshufb %xmm2, %xmm11, %xmm3
+; AVX1-NEXT:    vpshufb %xmm2, %xmm10, %xmm4
 ; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX1-NEXT:    vpshufb %xmm0, %xmm5, %xmm3
-; AVX1-NEXT:    vpshufb %xmm0, %xmm7, %xmm0
-; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; AVX1-NEXT:    vpshufb %xmm2, %xmm15, %xmm3
-; AVX1-NEXT:    vpshufb %xmm2, %xmm13, %xmm2
-; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm9 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpcmpeqb %xmm0, %xmm8, %xmm0
+; AVX1-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1-NEXT:    vpshufb %xmm1, %xmm14, %xmm0
+; AVX1-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX1-NEXT:    vpshufb %xmm2, %xmm6, %xmm1
+; AVX1-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpcmpeqb %xmm0, %xmm9, %xmm9
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm0 = <u,u,u,u,2,6,10,14,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm0, %xmm11, %xmm1
+; AVX1-NEXT:    vpshufb %xmm0, %xmm13, %xmm1
 ; AVX1-NEXT:    vpshufb %xmm0, %xmm12, %xmm2
 ; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm2, %xmm14, %xmm3
-; AVX1-NEXT:    vpshufb %xmm2, %xmm6, %xmm4
+; AVX1-NEXT:    vpshufb %xmm2, %xmm11, %xmm3
+; AVX1-NEXT:    vpshufb %xmm2, %xmm10, %xmm4
 ; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX1-NEXT:    vpshufb %xmm0, %xmm5, %xmm3
-; AVX1-NEXT:    vpshufb %xmm0, %xmm7, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm8 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; AVX1-NEXT:    vpshufb %xmm0, %xmm14, %xmm3
+; AVX1-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
 ; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; AVX1-NEXT:    vpshufb %xmm2, %xmm15, %xmm3
-; AVX1-NEXT:    vpshufb %xmm2, %xmm13, %xmm2
+; AVX1-NEXT:    vpshufb %xmm2, %xmm6, %xmm3
+; AVX1-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
 ; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm10 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm0 = <u,u,u,u,3,7,11,15,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm0, %xmm11, %xmm1
-; AVX1-NEXT:    vpshufb %xmm0, %xmm12, %xmm2
-; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm2, %xmm14, %xmm3
-; AVX1-NEXT:    vpshufb %xmm2, %xmm6, %xmm4
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm15 = xmm2[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <u,u,u,u,3,7,11,15,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm2, %xmm13, %xmm3
+; AVX1-NEXT:    vpshufb %xmm2, %xmm12, %xmm4
 ; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX1-NEXT:    vpshufb %xmm0, %xmm5, %xmm3
-; AVX1-NEXT:    vpshufb %xmm0, %xmm7, %xmm0
-; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; AVX1-NEXT:    vpshufb %xmm2, %xmm15, %xmm3
-; AVX1-NEXT:    vpshufb %xmm2, %xmm13, %xmm2
-; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX1-NEXT:    vextractf128 $1, %ymm9, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm8, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm9, %xmm8, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm10, %xmm3
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm0, %xmm10, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vxorps %ymm0, %ymm1, %ymm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX1-NEXT:    vpshufb %xmm4, %xmm11, %xmm0
+; AVX1-NEXT:    vpshufb %xmm4, %xmm10, %xmm1
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
+; AVX1-NEXT:    vpcmpeqb %xmm0, %xmm8, %xmm0
+; AVX1-NEXT:    vpshufb %xmm2, %xmm14, %xmm1
+; AVX1-NEXT:    vpshufb %xmm2, %xmm5, %xmm2
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; AVX1-NEXT:    vpshufb %xmm4, %xmm6, %xmm2
+; AVX1-NEXT:    vpshufb %xmm4, %xmm7, %xmm3
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm15, %xmm1
+; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm2, %ymm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm2, %ymm0
 ; AVX1-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: interleaved_load_vf32_i8_stride4:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa (%rdi), %xmm9
-; AVX2-NEXT:    vmovdqa 16(%rdi), %xmm11
-; AVX2-NEXT:    vmovdqa 32(%rdi), %xmm12
-; AVX2-NEXT:    vmovdqa 48(%rdi), %xmm13
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm6 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm6, %xmm13, %xmm4
-; AVX2-NEXT:    vpshufb %xmm6, %xmm12, %xmm5
-; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm0 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm0, %xmm11, %xmm5
-; AVX2-NEXT:    vpshufb %xmm0, %xmm9, %xmm7
-; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm8 = xmm5[0,1],xmm4[2,3]
-; AVX2-NEXT:    vmovdqa 112(%rdi), %xmm14
-; AVX2-NEXT:    vpshufb %xmm6, %xmm14, %xmm7
-; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = mem[2,3,0,1]
-; AVX2-NEXT:    vextracti128 $1, %ymm5, %xmm5
-; AVX2-NEXT:    vpshufb %xmm6, %xmm5, %xmm6
-; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; AVX2-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm10
-; AVX2-NEXT:    vmovdqa 80(%rdi), %xmm6
-; AVX2-NEXT:    vpshufb %xmm0, %xmm6, %xmm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm7 = mem[2,3,0,1]
-; AVX2-NEXT:    vextracti128 $1, %ymm7, %xmm7
-; AVX2-NEXT:    vpshufb %xmm0, %xmm7, %xmm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm0 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vmovdqa 112(%rdi), %xmm9
+; AVX2-NEXT:    vpshufb %xmm0, %xmm9, %xmm1
+; AVX2-NEXT:    vmovdqa 96(%rdi), %xmm10
+; AVX2-NEXT:    vpshufb %xmm0, %xmm10, %xmm3
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vmovdqa 80(%rdi), %xmm12
+; AVX2-NEXT:    vpshufb %xmm2, %xmm12, %xmm4
+; AVX2-NEXT:    vmovdqa 64(%rdi), %xmm5
+; AVX2-NEXT:    vpshufb %xmm2, %xmm5, %xmm6
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
+; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm8 = ymm4[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-NEXT:    vmovdqa (%rdi), %xmm11
+; AVX2-NEXT:    vmovdqa 16(%rdi), %xmm13
+; AVX2-NEXT:    vmovdqa 32(%rdi), %xmm6
+; AVX2-NEXT:    vmovdqa 48(%rdi), %xmm7
+; AVX2-NEXT:    vpshufb %xmm0, %xmm7, %xmm1
+; AVX2-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
 ; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm10[6,7]
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NEXT:    vpshufb %xmm2, %xmm13, %xmm1
+; AVX2-NEXT:    vpshufb %xmm2, %xmm11, %xmm2
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm8 = ymm0[0,1,2,3],ymm8[4,5,6,7]
 ; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = <u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm1, %xmm13, %xmm0
-; AVX2-NEXT:    vpshufb %xmm1, %xmm12, %xmm2
-; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; AVX2-NEXT:    vpshufb %xmm1, %xmm9, %xmm2
+; AVX2-NEXT:    vpshufb %xmm1, %xmm10, %xmm0
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm2, %xmm11, %xmm3
-; AVX2-NEXT:    vpshufb %xmm2, %xmm9, %xmm4
+; AVX2-NEXT:    vpshufb %xmm2, %xmm12, %xmm3
+; AVX2-NEXT:    vpshufb %xmm2, %xmm5, %xmm4
 ; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
-; AVX2-NEXT:    vpshufb %xmm1, %xmm14, %xmm3
-; AVX2-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-NEXT:    vpshufb %xmm1, %xmm7, %xmm3
+; AVX2-NEXT:    vpshufb %xmm1, %xmm6, %xmm1
 ; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX2-NEXT:    vpshufb %xmm2, %xmm6, %xmm3
-; AVX2-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX2-NEXT:    vpshufb %xmm2, %xmm13, %xmm3
+; AVX2-NEXT:    vpshufb %xmm2, %xmm11, %xmm2
 ; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-NEXT:    vpcmpeqb %ymm0, %ymm8, %ymm8
 ; AVX2-NEXT:    vmovdqa {{.*#+}} xmm0 = <u,u,u,u,2,6,10,14,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm0, %xmm13, %xmm1
-; AVX2-NEXT:    vpshufb %xmm0, %xmm12, %xmm2
+; AVX2-NEXT:    vpshufb %xmm0, %xmm9, %xmm1
+; AVX2-NEXT:    vpshufb %xmm0, %xmm10, %xmm2
 ; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm2, %xmm11, %xmm3
-; AVX2-NEXT:    vpshufb %xmm2, %xmm9, %xmm4
+; AVX2-NEXT:    vpshufb %xmm2, %xmm12, %xmm3
+; AVX2-NEXT:    vpshufb %xmm2, %xmm5, %xmm4
 ; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX2-NEXT:    vpshufb %xmm0, %xmm14, %xmm3
-; AVX2-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-NEXT:    vpshufb %xmm0, %xmm7, %xmm3
+; AVX2-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
 ; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX2-NEXT:    vpshufb %xmm2, %xmm6, %xmm3
-; AVX2-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX2-NEXT:    vpshufb %xmm2, %xmm13, %xmm3
+; AVX2-NEXT:    vpshufb %xmm2, %xmm11, %xmm2
 ; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm10 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = <u,u,u,u,3,7,11,15,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm1, %xmm13, %xmm2
-; AVX2-NEXT:    vpshufb %xmm1, %xmm12, %xmm3
-; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm3, %xmm11, %xmm4
-; AVX2-NEXT:    vpshufb %xmm3, %xmm9, %xmm0
-; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
-; AVX2-NEXT:    vpshufb %xmm1, %xmm14, %xmm2
-; AVX2-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
-; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX2-NEXT:    vpshufb %xmm3, %xmm6, %xmm2
-; AVX2-NEXT:    vpshufb %xmm3, %xmm7, %xmm3
+; AVX2-NEXT:    vpshufb %xmm1, %xmm9, %xmm2
+; AVX2-NEXT:    vpshufb %xmm1, %xmm10, %xmm3
 ; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
 ; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-NEXT:    vpcmpeqb %ymm0, %ymm10, %ymm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-NEXT:    vpshufb %xmm3, %xmm12, %xmm4
+; AVX2-NEXT:    vpshufb %xmm3, %xmm5, %xmm5
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-NEXT:    vpshufb %xmm1, %xmm7, %xmm4
+; AVX2-NEXT:    vpshufb %xmm1, %xmm6, %xmm1
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
+; AVX2-NEXT:    vpshufb %xmm3, %xmm13, %xmm4
+; AVX2-NEXT:    vpshufb %xmm3, %xmm11, %xmm3
+; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %ymm0, %ymm8, %ymm0
 ; AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: interleaved_load_vf32_i8_stride4:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm10
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm2, %xmm10, %xmm3
-; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = mem[2,3,0,1]
-; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm11
-; AVX512-NEXT:    vpshufb %xmm2, %xmm11, %xmm2
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm4
-; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm12
+; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm11
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm0, %xmm11, %xmm3
+; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm13
+; AVX512-NEXT:    vpshufb %xmm0, %xmm13, %xmm0
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm14
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm5 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm5, %xmm12, %xmm6
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = mem[2,3,0,1]
-; AVX512-NEXT:    vextracti128 $1, %ymm3, %xmm13
-; AVX512-NEXT:    vpshufb %xmm5, %xmm13, %xmm5
+; AVX512-NEXT:    vpshufb %xmm5, %xmm14, %xmm6
+; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm4
+; AVX512-NEXT:    vpshufb %xmm5, %xmm4, %xmm5
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3,4,5],ymm0[6,7]
 ; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm5
 ; AVX512-NEXT:    vpmovdb %zmm5, %xmm5
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm5[0,1,2,3],ymm4[4,5,6,7]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = <u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm14
-; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm6
-; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm7
-; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm4
-; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm1
-; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm2
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm2, %xmm6, %xmm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm14, %xmm5
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm10, %xmm3
-; AVX512-NEXT:    vpshufb %xmm0, %xmm11, %xmm0
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm5[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm5 = <u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm5, %xmm11, %xmm0
+; AVX512-NEXT:    vpshufb %xmm5, %xmm13, %xmm6
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm6[0],xmm0[0],xmm6[1],xmm0[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vpshufb %xmm2, %xmm12, %xmm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm13, %xmm2
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm1, %xmm14, %xmm6
+; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm7
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm6[0,1,2,3,4,5],ymm0[6,7]
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm10
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm12
+; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm7
+; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm0
+; AVX512-NEXT:    vpshufb %xmm5, %xmm0, %xmm6
+; AVX512-NEXT:    vpshufb %xmm5, %xmm7, %xmm5
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm12, %xmm6
+; AVX512-NEXT:    vpshufb %xmm1, %xmm10, %xmm1
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm5[2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm1[0,1,2,3],ymm8[4,5,6,7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = <u,u,u,u,2,6,10,14,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm1, %xmm11, %xmm5
+; AVX512-NEXT:    vpshufb %xmm1, %xmm13, %xmm6
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm6 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm6, %xmm14, %xmm2
+; AVX512-NEXT:    vpshufb %xmm6, %xmm4, %xmm3
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = <u,u,u,u,2,6,10,14,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm1
-; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm2
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm2, %xmm6, %xmm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm14, %xmm5
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm0, %xmm3
+; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm1
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX512-NEXT:    vpshufb %xmm6, %xmm12, %xmm3
+; AVX512-NEXT:    vpshufb %xmm6, %xmm10, %xmm5
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm10, %xmm3
-; AVX512-NEXT:    vpshufb %xmm0, %xmm11, %xmm0
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vpshufb %xmm2, %xmm12, %xmm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm13, %xmm2
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = <u,u,u,u,3,7,11,15,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm2
-; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm3
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm3, %xmm6, %xmm4
-; AVX512-NEXT:    vpshufb %xmm3, %xmm14, %xmm5
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm2 = xmm4[0,1],xmm2[2,3]
-; AVX512-NEXT:    vpshufb %xmm1, %xmm10, %xmm4
-; AVX512-NEXT:    vpshufb %xmm1, %xmm11, %xmm1
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT:    vpshufb %xmm3, %xmm12, %xmm4
-; AVX512-NEXT:    vpshufb %xmm3, %xmm13, %xmm3
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <u,u,u,u,3,7,11,15,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm2, %xmm11, %xmm3
+; AVX512-NEXT:    vpshufb %xmm2, %xmm13, %xmm5
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm5 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm5, %xmm14, %xmm6
+; AVX512-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; AVX512-NEXT:    vpshufb %xmm5, %xmm12, %xmm2
+; AVX512-NEXT:    vpshufb %xmm5, %xmm10, %xmm4
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512-NEXT:    vpcmpeqb %zmm8, %zmm9, %k0
-; AVX512-NEXT:    vpcmpeqb %zmm1, %zmm0, %k1
+; AVX512-NEXT:    vpcmpeqb %zmm0, %zmm1, %k1
 ; AVX512-NEXT:    kxnord %k1, %k0, %k0
 ; AVX512-NEXT:    vpmovm2b %k0, %zmm0
 ; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
diff --git a/llvm/test/CodeGen/X86/zero-initialized-in-bss.ll b/llvm/test/CodeGen/X86/zero-initialized-in-bss.ll
new file mode 100644
index 0000000000000..2feee54856e2c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/zero-initialized-in-bss.ll
@@ -0,0 +1,16 @@
+; RUN: llc < %s -mtriple=x86_64 | FileCheck %s --check-prefix=BSS
+
+; BSS:      .bss
+; BSS-NEXT: .globl a
+; BSS:      .section .tbss,"awT",@nobits
+; BSS-NEXT: .globl b
+
+; RUN: llc < %s -mtriple=x86_64 -nozero-initialized-in-bss | FileCheck %s --check-prefix=DATA
+
+; DATA:      .data
+; DATA-NEXT: .globl a
+; DATA:      .section .tdata,"awT",@progbits
+; DATA-NEXT: .globl b
+
+@a = global i32 0
+@b = thread_local global i32 0
diff --git a/llvm/test/DebugInfo/Generic/licm-hoist-debug-loc.ll b/llvm/test/DebugInfo/Generic/licm-hoist-debug-loc.ll
index 04429d81e9518..0a13e9604f0c8 100644
--- a/llvm/test/DebugInfo/Generic/licm-hoist-debug-loc.ll
+++ b/llvm/test/DebugInfo/Generic/licm-hoist-debug-loc.ll
@@ -5,7 +5,7 @@
 ; Generated with
 ; clang -O0 -S -emit-llvm test.cpp -g -gline-tables-only -o t.ll
 ; opt -S -sroa -adce -simplifycfg -reassociate -domtree -loops \
-;     -loop-simplify -lcssa -basicaa -aa -scalar-evolution -loop-rotate t.ll > test.ll
+;     -loop-simplify -lcssa -basic-aa -aa -scalar-evolution -loop-rotate t.ll > test.ll
 ;
 ; void bar(int *);
 ; void foo(int k, int p)
diff --git a/llvm/test/DebugInfo/Inputs/dwarfdump-16bit-addr.o b/llvm/test/DebugInfo/Inputs/dwarfdump-16bit-addr.o
new file mode 100644
index 0000000000000..9777911b726a4
Binary files /dev/null and b/llvm/test/DebugInfo/Inputs/dwarfdump-16bit-addr.o differ
diff --git a/llvm/test/DebugInfo/MIR/Mips/dbg-call-site-copy-sub-reg.mir b/llvm/test/DebugInfo/MIR/Mips/dbg-call-site-copy-sub-reg.mir
new file mode 100644
index 0000000000000..9827b2803be3f
--- /dev/null
+++ b/llvm/test/DebugInfo/MIR/Mips/dbg-call-site-copy-sub-reg.mir
@@ -0,0 +1,109 @@
+# RUN: llc -mtriple mips64-linux -emit-call-site-info -start-after=livedebugvalues -filetype=obj %s -o -| llvm-dwarfdump -| FileCheck %s
+
+## Test case where a parameter value could be described as a sub-register.
+## Built from source:
+## extern int foo(char*,int,int);
+## int f1(int a, char* str){
+##   return foo(str, a, 0);
+## }
+## fastcc attribute of foo() is added manually to the .ll to match the real case
+
+#CHECK: DW_TAG_GNU_call_site
+#CHECK-NEXT: DW_AT_abstract_origin	{{.*}} "foo"
+
+#CHECK: DW_TAG_GNU_call_site_parameter
+#CHECK-NEXT: DW_AT_location	(DW_OP_reg6 A2_64)
+#CHECK-NEXT: DW_AT_GNU_call_site_value	(DW_OP_lit0)
+#CHECK-EMPTY:
+#CHECK-NEXT: DW_TAG_GNU_call_site_parameter
+#CHECK-NEXT:   DW_AT_location	(DW_OP_reg4 A0_64)
+#CHECK-NEXT:   DW_AT_GNU_call_site_value	(DW_OP_GNU_entry_value(DW_OP_reg5 A1_64))
+#CHECK-EMPTY:
+## The parameter, which value is loaded by COPY instr using sub-register, should be omitted.
+#CHECK-NOT: DW_TAG_GNU_call_site_parameter
+#CHECK-NOT:   DW_AT_location	(DW_OP_reg5 A1_64)
+#CHECK-NOT:   DW_AT_GNU_call_site_value	(DW_OP_GNU_entry_value(DW_OP_reg4 A0_64))
+--- |
+  ; ModuleID = 'm.ll'
+  source_filename = "m.c"
+  target datalayout = "E-m:e-i8:8:32-i16:16:32-i64:64-n32:64-S128"
+  target triple = "mips64-unknown-linux"
+  
+  ; Function Attrs: nounwind
+  define signext i32 @f1(i32 signext %a, i8* %str) local_unnamed_addr !dbg !8 {
+  entry:
+    call void @llvm.dbg.value(metadata i32 %a, metadata !16, metadata !DIExpression()), !dbg !18
+    call void @llvm.dbg.value(metadata i8* %str, metadata !17, metadata !DIExpression()), !dbg !18
+    %call = tail call fastcc signext i32 @foo(i8* %str, i32 signext %a, i32 signext 0), !dbg !18
+    ret i32 %call, !dbg !18
+  }
+  
+  declare !dbg !21 fastcc signext i32 @foo(i8*, i32 signext, i32 signext) local_unnamed_addr
+  
+  ; Function Attrs: nounwind readnone speculatable willreturn
+  declare void @llvm.dbg.value(metadata, metadata, metadata)
+  
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!3, !4, !5, !6}
+  !llvm.ident = !{!7}
+  
+  !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+  !1 = !DIFile(filename: "<stdin>", directory: "/dir")
+  !2 = !{}
+  !3 = !{i32 7, !"Dwarf Version", i32 4}
+  !4 = !{i32 2, !"Debug Info Version", i32 3}
+  !5 = !{i32 1, !"wchar_size", i32 4}
+  !6 = !{i32 7, !"PIC Level", i32 1}
+  !7 = !{!"clang version 11.0.0"}
+  !8 = distinct !DISubprogram(name: "f1", scope: !9, file: !9, line: 4, type: !10, scopeLine: 4, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !15)
+  !9 = !DIFile(filename: "m.c", directory: "/dir")
+  !10 = !DISubroutineType(types: !11)
+  !11 = !{!12, !12, !13}
+  !12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+  !13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !14, size: 64)
+  !14 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+  !15 = !{!16, !17}
+  !16 = !DILocalVariable(name: "a", arg: 1, scope: !8, file: !9, line: 4, type: !12)
+  !17 = !DILocalVariable(name: "str", arg: 2, scope: !8, file: !9, line: 4, type: !13)
+  !18 = !DILocation(line: 0, scope: !8)
+  !21 = !DISubprogram(name: "foo", scope: !9, file: !9, line: 2, type: !22, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+  !22 = !DISubroutineType(types: !23)
+  !23 = !{!12, !13, !12, !12}
+
+...
+---
+name:            f1
+alignment:       8
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -8, size: 8, alignment: 8, 
+      stack-id: default, callee-saved-register: '$ra_64', callee-saved-restored: true, 
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+callSites:
+  - { bb: 0, offset: 12, fwdArgRegs: 
+      - { arg: 0, reg: '$a0_64' }
+      - { arg: 1, reg: '$a1' }
+      - { arg: 2, reg: '$a2' } }
+body:             |
+  bb.0.entry:
+    DBG_VALUE $a0_64, $noreg, !16, !DIExpression(), debug-location !18
+    DBG_VALUE $a1_64, $noreg, !17, !DIExpression(), debug-location !18
+    $sp_64 = DADDiu $sp_64, -16
+    CFI_INSTRUCTION def_cfa_offset 16
+    SD killed $ra_64, $sp_64, 8 :: (store 8 into %stack.0)
+    CFI_INSTRUCTION offset $ra_64, -8
+    $at_64 = OR64 $a0_64, $zero_64
+    DBG_VALUE $a1_64, $noreg, !17, !DIExpression(), debug-location !18
+    DBG_VALUE $at_64, $noreg, !16, !DIExpression(), debug-location !18
+    $a0_64 = OR64 killed $a1_64, $zero_64, debug-location !18
+    DBG_VALUE $a0_64, $noreg, !17, !DIExpression(), debug-location !18
+    $a1 = OR $at, $zero, implicit killed $at_64, debug-location !18
+    JAL @foo, csr_n64, implicit-def dead $ra, implicit $a0_64, implicit $a1, implicit killed $a2, implicit-def $sp, implicit-def $v0, debug-location !18 {
+      $a2 = ADDiu $zero, 0, debug-location !18
+    }
+    renamable $v0_64 = SLL64_32 killed renamable $v0, debug-location !18
+    $ra_64 = LD $sp_64, 8, debug-location !18 :: (load 8 from %stack.0)
+    PseudoReturn64 undef $ra_64, implicit $v0_64, debug-location !18 {
+      $sp_64 = DADDiu $sp_64, 16
+    }
+
+...
diff --git a/llvm/test/DebugInfo/MIR/X86/singlelocation-cutoffs.mir b/llvm/test/DebugInfo/MIR/X86/singlelocation-cutoffs.mir
new file mode 100644
index 0000000000000..6ad64d9d74bbb
--- /dev/null
+++ b/llvm/test/DebugInfo/MIR/X86/singlelocation-cutoffs.mir
@@ -0,0 +1,65 @@
+# Test cutoffs for single-location variable analysis.
+# Disable validThroughout if the input size exceeds the specified limit
+
+# RUN: llc %s -o - -start-after=livedebugvalues -mtriple=x86_64-unknown-unknown \
+# RUN:   --singlevarlocation-input-bb-limit=0 -filetype=obj\
+# RUN:   | llvm-dwarfdump -v -\
+# RUN:   | FileCheck %s -check-prefix=LIMITED
+
+# RUN: llc %s -o - -start-after=livedebugvalues -mtriple=x86_64-unknown-unknown \
+# RUN: --singlevarlocation-input-bb-limit=20 -filetype=obj | llvm-dwarfdump -v -\
+# RUN:   | FileCheck %s -check-prefix=UNLIMITED
+
+# LIMITED: DW_AT_location [DW_FORM_sec_offset]
+
+# UNLIMITED: DW_AT_location [DW_FORM_exprloc]
+
+--- |
+  target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+  declare i32 @use(i32)
+
+  define i32 @foo(i32 %x) !dbg !6 {
+  entry:
+    ret i32 1, !dbg !15
+  }
+
+  declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.debugify = !{!3, !4}
+  !llvm.module.flags = !{!5}
+
+  !0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+  !1 = !DIFile(filename: "/tmp/t.ll", directory: "/")
+  !2 = !{}
+  !3 = !{i32 4}
+  !4 = !{i32 2}
+  !5 = !{i32 2, !"Debug Info Version", i32 3}
+  !6 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: null, file: !1, line: 1, type: !7, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8)
+  !7 = !DISubroutineType(types: !2)
+  !8 = !{!9, !11}
+  !9 = !DILocalVariable(name: "1", scope: !6, file: !1, line: 1, type: !10)
+  !10 = !DIBasicType(name: "ty32", size: 32, encoding: DW_ATE_unsigned)
+  !11 = !DILocalVariable(name: "2", scope: !6, file: !1, line: 3, type: !10)
+  !12 = !DILocation(line: 1, column: 1, scope: !6)
+  !13 = !DILocation(line: 2, column: 1, scope: !6)
+  !14 = !DILocation(line: 3, column: 1, scope: !6)
+  !15 = !DILocation(line: 4, column: 1, scope: !6)
+
+...
+---
+name:            foo
+liveins:
+  - { reg: '$edi', virtual-reg: '' }
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -12, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.0.entry:
+    liveins: $edi
+    DBG_VALUE renamable $edi, $noreg, !11, !DIExpression(), debug-location !14
+    RETQ debug-location !14
+
+...
diff --git a/llvm/test/DebugInfo/MSP430/dwarf-basics.ll b/llvm/test/DebugInfo/MSP430/dwarf-basics.ll
new file mode 100644
index 0000000000000..c446247eacfff
--- /dev/null
+++ b/llvm/test/DebugInfo/MSP430/dwarf-basics.ll
@@ -0,0 +1,136 @@
+; RUN: llc --filetype=obj -o %t < %s
+; RUN: llvm-dwarfdump --debug-info %t | FileCheck %s
+; RUN: llvm-dwarfdump --verify %t
+
+; This file was based on output of
+;
+;   clang -target msp430 -S -emit-llvm -gdwarf-3 -Os dwarf-basics.c
+;
+; for the following dwarf-basics.c
+;
+;   struct X {
+;     void *a;
+;   };
+;
+;   int f(long y, struct X *p)
+;   {
+;     return 42;
+;   }
+;
+
+; CHECK: file format elf32-msp430
+
+; CHECK: .debug_info contents:
+; CHECK: Compile Unit: length = 0x{{.*}}, format = DWARF32, version = 0x0003, abbr_offset = 0x0000, addr_size = 0x02 (next unit at 0x{{.*}})
+
+; CHECK: DW_TAG_compile_unit
+; CHECK:   DW_AT_producer    ("clang version 11.0.0 (git@...)")
+; CHECK:   DW_AT_language    (DW_LANG_C99)
+; CHECK:   DW_AT_name        ("dwarf-basics.c")
+; CHECK:   DW_AT_stmt_list   (0x{{.*}})
+; CHECK:   DW_AT_comp_dir    ("/tmp")
+; CHECK:   DW_AT_low_pc      (0x{{.*}})
+; CHECK:   DW_AT_high_pc     (0x{{.*}})
+
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_low_pc    (0x{{.*}})
+; CHECK:     DW_AT_high_pc   (0x{{.*}})
+; CHECK:     DW_AT_frame_base        (DW_OP_reg1 SPB)
+; CHECK:     DW_AT_name      ("f")
+; CHECK:     DW_AT_decl_file ("/tmp{{[/\\]}}dwarf-basics.c")
+; CHECK:     DW_AT_decl_line (5)
+; CHECK:     DW_AT_prototyped        (0x01)
+; CHECK:     DW_AT_type      (0x{{.*}} "int")
+; CHECK:     DW_AT_external  (0x01)
+
+; CHECK:       DW_TAG_formal_parameter
+; CHECK:         DW_AT_location        (0x{{.*}}:
+; CHECK:            [0x0000, 0x0004): DW_OP_reg12 R12B)
+; CHECK:         DW_AT_name    ("y")
+; CHECK:         DW_AT_decl_file       ("/tmp{{[/\\]}}dwarf-basics.c")
+; CHECK:         DW_AT_decl_line       (5)
+; CHECK:         DW_AT_type    (0x{{.*}} "long int")
+
+; CHECK:       DW_TAG_formal_parameter
+; CHECK:         DW_AT_location        (DW_OP_reg14 R14B)
+; CHECK:         DW_AT_name    ("p")
+; CHECK:         DW_AT_decl_file       ("/tmp{{[/\\]}}dwarf-basics.c")
+; CHECK:         DW_AT_decl_line       (5)
+; CHECK:         DW_AT_type    (0x{{.*}} "X*")
+
+; CHECK:       NULL
+
+; CHECK:     DW_TAG_base_type
+; CHECK:       DW_AT_name      ("int")
+; CHECK:       DW_AT_encoding  (DW_ATE_signed)
+; CHECK:       DW_AT_byte_size (0x02)
+
+; CHECK:     DW_TAG_base_type
+; CHECK:       DW_AT_name      ("long int")
+; CHECK:       DW_AT_encoding  (DW_ATE_signed)
+; CHECK:       DW_AT_byte_size (0x04)
+
+; CHECK:     DW_TAG_pointer_type
+; CHECK:       DW_AT_type      (0x{{.*}} "X")
+
+; CHECK:     DW_TAG_structure_type
+; CHECK:       DW_AT_name      ("X")
+; CHECK:       DW_AT_byte_size (0x02)
+; CHECK:       DW_AT_decl_file ("/tmp{{[/\\]}}dwarf-basics.c")
+; CHECK:       DW_AT_decl_line (1)
+
+; CHECK:       DW_TAG_member
+; CHECK:         DW_AT_name    ("a")
+; CHECK:         DW_AT_type    (0x{{.*}} "*")
+; CHECK:         DW_AT_decl_file       ("/tmp{{[/\\]}}dwarf-basics.c")
+; CHECK:         DW_AT_decl_line       (2)
+; CHECK:         DW_AT_data_member_location    (0x00)
+
+; CHECK:       NULL
+
+; CHECK:     DW_TAG_pointer_type
+
+; CHECK:     NULL
+
+
+source_filename = "dwarf-basics.c"
+target datalayout = "e-m:e-p:16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16"
+target triple = "msp430"
+
+%struct.X = type { i8* }
+
+define i16 @f(i32 %y, %struct.X* %p) !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %y, metadata !18, metadata !DIExpression()), !dbg !20
+  call void @llvm.dbg.value(metadata %struct.X* %p, metadata !19, metadata !DIExpression()), !dbg !20
+  ret i16 42, !dbg !21
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0 (git@...)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "dwarf-basics.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 3}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 2}
+!6 = !{!"clang version 11.0.0 (git@...)"}
+!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 5, type: !8, scopeLine: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !17)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !11, !12}
+!10 = !DIBasicType(name: "int", size: 16, encoding: DW_ATE_signed)
+!11 = !DIBasicType(name: "long int", size: 32, encoding: DW_ATE_signed)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 16)
+!13 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "X", file: !1, line: 1, size: 16, elements: !14)
+!14 = !{!15}
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !13, file: !1, line: 2, baseType: !16, size: 16)
+!16 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 16)
+!17 = !{!18, !19}
+!18 = !DILocalVariable(name: "y", arg: 1, scope: !7, file: !1, line: 5, type: !11)
+!19 = !DILocalVariable(name: "p", arg: 2, scope: !7, file: !1, line: 5, type: !12)
+!20 = !DILocation(line: 0, scope: !7)
+!21 = !DILocation(line: 7, column: 3, scope: !7)
diff --git a/llvm/test/DebugInfo/NVPTX/packed_bitfields.ll b/llvm/test/DebugInfo/NVPTX/packed_bitfields.ll
new file mode 100644
index 0000000000000..36433064fa924
--- /dev/null
+++ b/llvm/test/DebugInfo/NVPTX/packed_bitfields.ll
@@ -0,0 +1,42 @@
+; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda | FileCheck %s
+
+; Produced at -O0 from:
+; struct {
+;   char : 3;
+;   char a : 6;
+; } __attribute__((__packed__)) b;
+
+; Note that DWARF 2 counts bit offsets backwards from the high end of
+; the storage unit to the high end of the bit field.
+
+; CHECK: .section .debug_info
+; CHECK:      .b8 3    // Abbrev {{.*}} DW_TAG_structure_type
+; CHECK:      .b8 3    // DW_AT_decl_line
+; CHECK-NEXT: .b8 1    // DW_AT_byte_size
+; CHECK-NEXT: .b8 6    // DW_AT_bit_size
+; CHECK-NEXT: .b64 -1  // DW_AT_bit_offset
+; CHECK-NEXT: .b8 2    // DW_AT_data_member_location
+
+%struct.anon = type { i16 }
+
+@b = global %struct.anon zeroinitializer, align 1, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!10, !11, !12, !13}
+!llvm.ident = !{!14}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "b", scope: !2, file: !3, line: 4, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 11.0.0 ", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None, sysroot: "/")
+!3 = !DIFile(filename: "repro.c", directory: "/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, file: !3, line: 1, size: 16, elements: !7)
+!7 = !{!8}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !6, file: !3, line: 3, baseType: !9, size: 6, offset: 3, flags: DIFlagBitField, extraData: i64 0)
+!9 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!10 = !{i32 7, !"Dwarf Version", i32 2}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{i32 1, !"wchar_size", i32 4}
+!13 = !{i32 7, !"PIC Level", i32 2}
+!14 = !{!"clang version 11.0.0 "}
diff --git a/llvm/test/DebugInfo/Sparc/entry-value-complex-reg-expr.ll b/llvm/test/DebugInfo/Sparc/entry-value-complex-reg-expr.ll
index 0af5619b75d78..096df49110c5c 100644
--- a/llvm/test/DebugInfo/Sparc/entry-value-complex-reg-expr.ll
+++ b/llvm/test/DebugInfo/Sparc/entry-value-complex-reg-expr.ll
@@ -1,11 +1,14 @@
 ; RUN: llc -debug-entry-values -filetype=asm -o - %s | FileCheck %s
 
-; Verify that the entry value covers both of the DW_OP_regx pieces. Previously
-; the size operand of the entry value would be hardcoded to one.
+; The q0 register does not have a DWARF register number, and is instead emitted
+; as a composite location description with two sub-registers. Previously we
+; emitted a single DW_OP_entry_value wrapping that whole composite location
+; description, but that is not valid DWARF; DW_OP_entry_value operations can
+; only hold DWARF expressions and register location descriptions.
 ;
-; XXX: Is this really what should be emitted, or should we instead emit one
-; entry value operation per DW_OP_regx? GDB can currently not understand
-; entry values containing complex expressions like this.
+; In the future we may want to emit a composite location description where each
+; DW_OP_regx operation is wrapped in an entry value operation, but for now
+; just verify that no invalid DWARF is emitted.
 
 target datalayout = "E-m:e-i64:64-n32:64-S128"
 target triple = "sparc64"
@@ -20,8 +23,11 @@ target triple = "sparc64"
 ;   return 123;
 ; }
 
-; CHECK:      .byte   243       ! DW_OP_GNU_entry_value
-; CHECK-NEXT: .byte   8         ! 8
+; Verify that we got an entry value in the DIExpression...
+; CHECK: DEBUG_VALUE: foo:p <- [DW_OP_LLVM_entry_value 1] $q0
+
+; ... but that no entry value location was emitted:
+; CHECK:      .half   8         ! Loc expr size
 ; CHECK-NEXT: .byte   144       ! sub-register DW_OP_regx
 ; CHECK-NEXT: .byte   72        ! 72
 ; CHECK-NEXT: .byte   147       ! DW_OP_piece
@@ -30,7 +36,8 @@ target triple = "sparc64"
 ; CHECK-NEXT: .byte   73        ! 73
 ; CHECK-NEXT: .byte   147       ! DW_OP_piece
 ; CHECK-NEXT: .byte   8         ! 8
-; CHECK-NEXT: .byte   159       ! DW_OP_stack_value
+; CHECK-NEXT: .xword  0
+; CHECK-NEXT: .xword  0
 
 @global = common global fp128 0xL00000000000000000000000000000000, align 16, !dbg !0
 
diff --git a/llvm/test/DebugInfo/X86/basicblock-sections_1.ll b/llvm/test/DebugInfo/X86/basicblock-sections_1.ll
new file mode 100644
index 0000000000000..f3bedd977693f
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/basicblock-sections_1.ll
@@ -0,0 +1,97 @@
+; RUN: llc -O0 %s -mtriple=x86_64-* -filetype=obj -o %t && llvm-dwarfdump  -debug-info -v %t | FileCheck --check-prefix=NO-SECTIONS %s
+; RUN: llc -O0 %s --basicblock-sections=all --unique-bb-section-names -mtriple=x86_64-* -filetype=obj -o %t && llvm-dwarfdump  -debug-info -v %t | FileCheck --check-prefix=BB-SECTIONS %s
+; RUN: llc -O0 %s --basicblock-sections=all --unique-bb-section-names -mtriple=x86_64-* -filetype=obj -split-dwarf-file=%t.dwo -o %t && llvm-dwarfdump  -debug-info -v %t | FileCheck --check-prefix=BB-SECTIONS %s
+; RUN: llc -O0 %s --basicblock-sections=all -mtriple=x86_64-* -filetype=asm -o - | FileCheck --check-prefix=BB-SECTIONS-ASM %s
+
+; From:
+; int foo(int a) {
+;   if (a > 20)
+;     return 2;
+;   else
+;     return 0;
+; }
+
+; NO-SECTIONS: DW_AT_low_pc [DW_FORM_addr] (0x0000000000000000 ".text")
+; NO-SECTIONS: DW_AT_high_pc [DW_FORM_data4] ({{.*}})
+; BB-SECTIONS: DW_AT_low_pc [DW_FORM_addr] (0x0000000000000000)
+; BB-SECTIONS-NEXT: DW_AT_ranges [DW_FORM_sec_offset]
+; BB-SECTIONS-NEXT: [{{.*}}) ".text._Z3fooi.1"
+; BB-SECTIONS-NEXT: [{{.*}}) ".text._Z3fooi.2"
+; BB-SECTIONS-NEXT: [{{.*}}) ".text._Z3fooi.3"
+; BB-SECTIONS-NEXT: [{{.*}}) ".text"
+; BB-SECTIONS-ASM: _Z3fooi:
+; BB-SECTIONS-ASM: .Ltmp{{[0-9]+}}:
+; BB-SECTIONS-ASM-NEXT: .loc 1 2 9 prologue_end
+; BB-SECTIONS-ASM: .Ltmp{{[0-9]+}}:
+; BB-SECTIONS-ASM-NEXT: .loc 1 2 7 is_stmt
+; BB-SECTIONS-ASM: _Z3fooi.1:
+; BB-SECTIONS-ASM: .Ltmp{{[0-9]+}}:
+; BB-SECTIONS-ASM: .size	_Z3fooi.1, .Ltmp{{[0-9]+}}-_Z3fooi.1
+; BB-SECTIONS-ASM: _Z3fooi.2:
+; BB-SECTIONS-ASM: .Ltmp{{[0-9]+}}:
+; BB-SECTIONS-ASM-NEXT: .Ltmp{{[0-9]+}}:
+; BB-SECTIONS-ASM: .size	_Z3fooi.2, .Ltmp{{[0-9]+}}-_Z3fooi.2
+; BB-SECTIONS-ASM: _Z3fooi.3:
+; BB-SECTIONS-ASM: .Ltmp{{[0-9]+}}:
+; BB-SECTIONS-ASM-NEXT: .Ltmp{{[0-9]+}}:
+; BB-SECTIONS-ASM: .size	_Z3fooi.3, .Ltmp{{[0-9]+}}-_Z3fooi.3
+; BB-SECTIONS-ASM: .Lfunc_end0:
+; BB-SECTIONS-ASM: .Ldebug_ranges0:
+; BB-SECTIONS-ASM-NEXT:	.quad	_Z3fooi.1
+; BB-SECTIONS-ASM-NEXT:	.quad	.Ltmp{{[0-9]+}}
+; BB-SECTIONS-ASM-NEXT:	.quad	_Z3fooi.2
+; BB-SECTIONS-ASM-NEXT:	.quad	.Ltmp{{[0-9]+}}
+; BB-SECTIONS-ASM-NEXT:	.quad	_Z3fooi.3
+; BB-SECTIONS-ASM-NEXT:	.quad	.Ltmp{{[0-9]+}}
+; BB-SECTIONS-ASM-NEXT:	.quad	.Lfunc_begin0
+; BB-SECTIONS-ASM-NEXT:	.quad	.Lfunc_end0
+; BB-SECTIONS-ASM-NEXT:	.quad	0
+; BB-SECTIONS-ASM-NEXT:	.quad	0
+
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local i32 @_Z3fooi(i32 %0) !dbg !7 {
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  store i32 %0, i32* %3, align 4
+  call void @llvm.dbg.declare(metadata i32* %3, metadata !11, metadata !DIExpression()), !dbg !12
+  %4 = load i32, i32* %3, align 4, !dbg !13
+  %5 = icmp sgt i32 %4, 20, !dbg !15
+  br i1 %5, label %6, label %7, !dbg !16
+
+6:                                                ; preds = %1
+  store i32 2, i32* %2, align 4, !dbg !17
+  br label %8, !dbg !17
+
+7:                                                ; preds = %1
+  store i32 0, i32* %2, align 4, !dbg !18
+  br label %8, !dbg !18
+
+8:                                                ; preds = %7, %6
+  %9 = load i32, i32* %2, align 4, !dbg !19
+  ret i32 %9, !dbg !19
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 10.0.0 (git@github.com:google/llvm-propeller.git f9421ebf4b3d8b64678bf6c49d1607fdce3f50c5)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "debuginfo.cc", directory: "/")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!7 = distinct !DISubprogram(name: "foo", linkageName: "_Z3fooi", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 1, type: !10)
+!12 = !DILocation(line: 1, column: 13, scope: !7)
+!13 = !DILocation(line: 2, column: 7, scope: !14)
+!14 = distinct !DILexicalBlock(scope: !7, file: !1, line: 2, column: 7)
+!15 = !DILocation(line: 2, column: 9, scope: !14)
+!16 = !DILocation(line: 2, column: 7, scope: !7)
+!17 = !DILocation(line: 3, column: 5, scope: !14)
+!18 = !DILocation(line: 5, column: 5, scope: !14)
+!19 = !DILocation(line: 6, column: 1, scope: !7)
diff --git a/llvm/test/DebugInfo/X86/packed_bitfields.ll b/llvm/test/DebugInfo/X86/packed_bitfields.ll
new file mode 100644
index 0000000000000..c36df72439ebb
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/packed_bitfields.ll
@@ -0,0 +1,50 @@
+; RUN: llc -dwarf-version=2 -mtriple x86_64-apple-macosx -O0 -filetype=obj -o %t_2_le.o %s
+; RUN: llvm-dwarfdump -v -debug-info %t_2_le.o | FileCheck %s
+; RUN: llc -dwarf-version=4 -debugger-tune=gdb -mtriple x86_64-apple-macosx -O0 -filetype=obj -o %t_4_le.o %s
+; RUN: llvm-dwarfdump -v -debug-info %t_4_le.o | FileCheck %s
+
+; Produced at -O0 from:
+; struct {
+;   char : 3;
+;   char a : 6;
+; } __attribute__((__packed__)) b;
+
+; Note that DWARF 2 counts bit offsets backwards from the high end of
+; the storage unit to the high end of the bit field.
+
+; CHECK: DW_TAG_member
+; CHECK-NEXT: DW_AT_name{{.*}}"a"
+; CHECK-NOT: DW_TAG_member
+; CHECK:      DW_AT_byte_size  {{.*}} (0x01)
+; CHECK-NEXT: DW_AT_bit_size   {{.*}} (0x06)
+; CHECK-NEXT: DW_AT_bit_offset {{.*}} (0xffffffffffffffff)
+; CHECK-NEXT: DW_AT_data_member_location {{.*}} ({{.*}}0x0{{0*}})
+
+; ModuleID = 'repro.c'
+source_filename = "repro.c"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.4.0"
+
+%struct.anon = type { i16 }
+
+@b = global %struct.anon zeroinitializer, align 1, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!10, !11, !12, !13}
+!llvm.ident = !{!14}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "b", scope: !2, file: !3, line: 4, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 11.0.0 ", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None, sysroot: "/")
+!3 = !DIFile(filename: "repro.c", directory: "/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, file: !3, line: 1, size: 16, elements: !7)
+!7 = !{!8}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !6, file: !3, line: 3, baseType: !9, size: 6, offset: 3, flags: DIFlagBitField, extraData: i64 0)
+!9 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!10 = !{i32 7, !"Dwarf Version", i32 2}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{i32 1, !"wchar_size", i32 4}
+!13 = !{i32 7, !"PIC Level", i32 2}
+!14 = !{!"clang version 11.0.0 "}
diff --git a/llvm/test/DebugInfo/X86/sections-as-references-cu-offset.ll b/llvm/test/DebugInfo/X86/sections-as-references-cu-offset.ll
new file mode 100644
index 0000000000000..6349698f48838
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/sections-as-references-cu-offset.ll
@@ -0,0 +1,27 @@
+; Test that references to CU are 4 bytes long when the target is 64-bit
+; and -dwarf-sections-as-references=Enable is specified.
+
+; RUN: llc -filetype=asm -mtriple=x86_64-linux-gnu -dwarf-sections-as-references=Enable < %s \
+; RUN:   | FileCheck %s
+
+; CHECK: .section .debug_pubnames
+; CHECK: .long .debug_info      # Offset of Compilation Unit Info
+; CHECK: .section .debug_pubtypes
+; CHECK: .long .debug_info      # Offset of Compilation Unit Info
+
+%struct.foo = type { i8 }
+
+@f = global %struct.foo zeroinitializer, align 1, !dbg !0
+
+!llvm.dbg.cu = !{!5}
+!llvm.module.flags = !{!8}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = !DIGlobalVariable(name: "f", scope: null, file: !2, line: 2, type: !3, isLocal: false, isDefinition: true)
+!2 = !DIFile(filename: "tu1.cpp", directory: "/dir")
+!3 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", file: !2, line: 1, size: 8, align: 8, elements: !4, identifier: "_ZTS3foo")
+!4 = !{}
+!5 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !2, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !6, globals: !7, imports: !4)
+!6 = !{!3}
+!7 = !{!0}
+!8 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/DebugInfo/debugify-report-missing-locs-only.ll b/llvm/test/DebugInfo/debugify-report-missing-locs-only.ll
index 43682768c5825..abf392f531d55 100644
--- a/llvm/test/DebugInfo/debugify-report-missing-locs-only.ll
+++ b/llvm/test/DebugInfo/debugify-report-missing-locs-only.ll
@@ -1,6 +1,6 @@
-; RUN: opt -check-debugify < %s -S -o - 2>&1 | FileCheck %s -implicit-check-not "ERROR: Instruction with empty DebugLoc in function bar"
+; RUN: opt -check-debugify < %s -S -o - 2>&1 | FileCheck %s -implicit-check-not "WARNING: Instruction with empty DebugLoc in function bar"
 
-; CHECK: ERROR: Instruction with empty DebugLoc in function foo --   ret void
+; CHECK: WARNING: Instruction with empty DebugLoc in function foo --   ret void
 define void @foo() !dbg !6 {
   ret void
 }
diff --git a/llvm/test/DebugInfo/debugify.ll b/llvm/test/DebugInfo/debugify.ll
index 89f62e2dbaae4..e821ce06c2c73 100644
--- a/llvm/test/DebugInfo/debugify.ll
+++ b/llvm/test/DebugInfo/debugify.ll
@@ -14,10 +14,10 @@
 ; RUN:   FileCheck %s -implicit-check-not="CheckModuleDebugify: FAIL"
 
 ; RUN: opt -debugify -strip -check-debugify -S -o - < %s 2>&1 | \
-; RUN:   FileCheck %s -check-prefix=CHECK-FAIL
+; RUN:   FileCheck %s -check-prefix=CHECK-WARN
 
 ; RUN: opt -enable-debugify -strip -S -o - < %s 2>&1 | \
-; RUN:   FileCheck %s -check-prefix=CHECK-FAIL
+; RUN:   FileCheck %s -check-prefix=CHECK-WARN
 
 ; RUN: opt -enable-debugify -S -o - < %s 2>&1 | FileCheck %s -check-prefix=PASS
 
@@ -88,15 +88,15 @@ define i32 @boom() {
 ; CHECK-REPEAT: ModuleDebugify: Skipping module with debug info
 
 ; --- Failure case
-; CHECK-FAIL: ERROR: Instruction with empty DebugLoc in function foo --   ret void
-; CHECK-FAIL: ERROR: Instruction with empty DebugLoc in function bar --   call void @foo()
-; CHECK-FAIL: ERROR: Instruction with empty DebugLoc in function bar --   {{.*}} add i32 0, 1
-; CHECK-FAIL: ERROR: Instruction with empty DebugLoc in function bar --   ret i32 0
-; CHECK-FAIL: WARNING: Missing line 1
-; CHECK-FAIL: WARNING: Missing line 2
-; CHECK-FAIL: WARNING: Missing line 3
-; CHECK-FAIL: WARNING: Missing line 4
-; CHECK-FAIL: WARNING: Missing variable 1
-; CHECK-FAIL: CheckModuleDebugify: FAIL
+; CHECK-WARN: WARNING: Instruction with empty DebugLoc in function foo --   ret void
+; CHECK-WARN: WARNING: Instruction with empty DebugLoc in function bar --   call void @foo()
+; CHECK-WARN: WARNING: Instruction with empty DebugLoc in function bar --   {{.*}} add i32 0, 1
+; CHECK-WARN: WARNING: Instruction with empty DebugLoc in function bar --   ret i32 0
+; CHECK-WARN: WARNING: Missing line 1
+; CHECK-WARN: WARNING: Missing line 2
+; CHECK-WARN: WARNING: Missing line 3
+; CHECK-WARN: WARNING: Missing line 4
+; CHECK-WARN: WARNING: Missing variable 1
+; CHECK-WARN: CheckModuleDebugify: PASS
 
 ; PASS: CheckModuleDebugify: PASS
diff --git a/llvm/test/DebugInfo/dwarfdump-16bit-addr.test b/llvm/test/DebugInfo/dwarfdump-16bit-addr.test
new file mode 100644
index 0000000000000..9c4dbca5215d1
--- /dev/null
+++ b/llvm/test/DebugInfo/dwarfdump-16bit-addr.test
@@ -0,0 +1,34 @@
+; Test that llvm-dwarfdump can handle DWARF32 v3 with addr_size = 0x02
+; Input object file was produced from source containing just "int x;" with
+;   clang -target avr -gdwarf-3 -c dwarfdump-16bit-addr.c
+
+; REQUIRES: avr-registered-target
+; RUN: llvm-dwarfdump --debug-info %S/Inputs/dwarfdump-16bit-addr.o | FileCheck %s
+; RUN: llvm-dwarfdump --verify %S/Inputs/dwarfdump-16bit-addr.o
+
+; CHECK: Inputs/dwarfdump-16bit-addr.o:  file format elf32-avr
+
+; CHECK: .debug_info contents:
+; CHECK: 0x00000000: Compile Unit: length = 0x00000032, format = DWARF32, version = 0x0003, abbr_offset = 0x0000, addr_size = 0x02 (next unit at 0x00000036)
+
+; CHECK: 0x0000000b: DW_TAG_compile_unit
+; CHECK:               DW_AT_producer    ("clang version 10.0.0-4ubuntu1 ")
+; CHECK:               DW_AT_language    (DW_LANG_C99)
+; CHECK:               DW_AT_name        ("dwarfdump-16bit-addr.c")
+; CHECK:               DW_AT_stmt_list   (0x00000000)
+; CHECK:               DW_AT_comp_dir    ("/tmp")
+
+; CHECK: 0x0000001e:   DW_TAG_variable
+; CHECK:                 DW_AT_name      ("x")
+; CHECK:                 DW_AT_type      (0x0000002e "int")
+; CHECK:                 DW_AT_external  (0x01)
+; CHECK:                 DW_AT_decl_file ("/tmp{{[/\\]}}dwarfdump-16bit-addr.c")
+; CHECK:                 DW_AT_decl_line (1)
+; CHECK:                 DW_AT_location  (DW_OP_addr 0x0)
+
+; CHECK: 0x0000002e:   DW_TAG_base_type
+; CHECK:                 DW_AT_name      ("int")
+; CHECK:                 DW_AT_encoding  (DW_ATE_signed)
+; CHECK:                 DW_AT_byte_size (0x02)
+
+; CHECK: 0x00000035:   NULL
diff --git a/llvm/test/Examples/Kaleidoscope/Chapter4.test b/llvm/test/Examples/Kaleidoscope/Chapter4.test
index 7cc65216a0e73..cf8f743073279 100644
--- a/llvm/test/Examples/Kaleidoscope/Chapter4.test
+++ b/llvm/test/Examples/Kaleidoscope/Chapter4.test
@@ -1,5 +1,5 @@
 # RUN: Kaleidoscope-Ch4 < %s 2>&1 | FileCheck %s
-# REQUIRES: default_target
+# REQUIRES: default_triple
 
 # Test basic definition, binding, and execution.
 def foo(x) x + 1;
diff --git a/llvm/test/Examples/Kaleidoscope/Chapter5.test b/llvm/test/Examples/Kaleidoscope/Chapter5.test
index b22633874fd0d..f9509197f0668 100644
--- a/llvm/test/Examples/Kaleidoscope/Chapter5.test
+++ b/llvm/test/Examples/Kaleidoscope/Chapter5.test
@@ -1,5 +1,5 @@
 # RUN: Kaleidoscope-Ch5 < %s 2>&1 | FileCheck %s
-# REQUIRES: default_target
+# REQUIRES: default_triple
 
 # Test 'if' expression.
 def foo(x) if x < 10 then 0 else 1;
diff --git a/llvm/test/Examples/Kaleidoscope/Chapter6.test b/llvm/test/Examples/Kaleidoscope/Chapter6.test
index 57d8eae9f29c7..1126e0df1f018 100644
--- a/llvm/test/Examples/Kaleidoscope/Chapter6.test
+++ b/llvm/test/Examples/Kaleidoscope/Chapter6.test
@@ -1,5 +1,5 @@
 # RUN: Kaleidoscope-Ch6 < %s 2>&1 | FileCheck %s
-# REQUIRES: default_target
+# REQUIRES: default_triple
 
 # Test unary operator definition.
 def unary-(x) 0 - x;
diff --git a/llvm/test/Examples/Kaleidoscope/Chapter7.test b/llvm/test/Examples/Kaleidoscope/Chapter7.test
index 75d20ae756af9..4c77ebc42169d 100644
--- a/llvm/test/Examples/Kaleidoscope/Chapter7.test
+++ b/llvm/test/Examples/Kaleidoscope/Chapter7.test
@@ -1,5 +1,5 @@
 # RUN: Kaleidoscope-Ch7 < %s 2>&1 | FileCheck %s
-# REQUIRES: default_target
+# REQUIRES: default_triple
 
 # Sequence operator and iterative fibonacci function to test user defined vars.
 def binary : 1 (x y) y;
diff --git a/llvm/test/ExecutionEngine/RuntimeDyld/X86/ELF_x86-64_none.yaml b/llvm/test/ExecutionEngine/RuntimeDyld/X86/ELF_x86-64_none.yaml
index ed4b243fae131..09e14c832a209 100644
--- a/llvm/test/ExecutionEngine/RuntimeDyld/X86/ELF_x86-64_none.yaml
+++ b/llvm/test/ExecutionEngine/RuntimeDyld/X86/ELF_x86-64_none.yaml
@@ -22,8 +22,8 @@ Sections:
       - Offset:          0x0000000000000000
         Type:            R_X86_64_NONE
 Symbols:
-  Global:
-    - Name: _main
-      Section: .text
-      Value: 0
-      Size: 4
+  - Name: _main
+    Section: .text
+    Binding: STB_GLOBAL
+    Value: 0
+    Size: 4
diff --git a/llvm/test/Feature/OperandBundles/basic-aa-argmemonly.ll b/llvm/test/Feature/OperandBundles/basic-aa-argmemonly.ll
index aa94458860602..61f514324aa1d 100644
--- a/llvm/test/Feature/OperandBundles/basic-aa-argmemonly.ll
+++ b/llvm/test/Feature/OperandBundles/basic-aa-argmemonly.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -gvn < %s | FileCheck %s
+; RUN: opt -S -basic-aa -gvn < %s | FileCheck %s
 
 declare void @argmemonly_function(i32 *) argmemonly
 
diff --git a/llvm/test/FileCheck/dump-input-annotations.txt b/llvm/test/FileCheck/dump-input-annotations.txt
index f4f0d3ca60225..785ee23d03dd5 100644
--- a/llvm/test/FileCheck/dump-input-annotations.txt
+++ b/llvm/test/FileCheck/dump-input-annotations.txt
@@ -24,7 +24,7 @@
 ; ALIGN:{{.*}}error:{{.*}}
 ; ALIGN:{{.*}}possible intended match here{{.*}}
 
-; ALIGN:Full input was:
+; ALIGN:Input was:
 ; ALIGN-NEXT:<<<<<<
 ; ALIGN-NEXT:           1: hello world
 ; ALIGN-NEXT:check:1       ^~~~~
diff --git a/llvm/test/FileCheck/dump-input-context.txt b/llvm/test/FileCheck/dump-input-context.txt
new file mode 100644
index 0000000000000..2e4382ec12edf
--- /dev/null
+++ b/llvm/test/FileCheck/dump-input-context.txt
@@ -0,0 +1,293 @@
+;--------------------------------------------------
+; Input file, check file, and directives for checking the size of the context.
+;
+; These are designed to be used with -dump-input=fail -vv.
+;
+; In the resulting input dump, there are three potential ellipses:
+;
+; - S: At the start of the input.
+; - M: Between two input lines included by the filter.
+; - E: At the end of the input.
+;
+; They are all present at -dump-input-context=4.  One becomes useless each time
+; -dump-input-context is incremented beyond that because then that ellipsis
+; becomes equal to or larger than the input lines it elides.
+;--------------------------------------------------
+
+; RUN: echo foo8       >  %t.in
+; RUN: echo foo7       >> %t.in
+; RUN: echo foo6       >> %t.in
+; RUN: echo foo5       >> %t.in
+; RUN: echo foo4       >> %t.in
+; RUN: echo foo3       >> %t.in
+; RUN: echo foo2       >> %t.in
+; RUN: echo foo1       >> %t.in
+; RUN: echo lab1 hello >> %t.in
+; RUN: echo foo1       >> %t.in
+; RUN: echo foo2       >> %t.in
+; RUN: echo foo3       >> %t.in
+; RUN: echo foo4       >> %t.in
+; RUN: echo foo5       >> %t.in
+; RUN: echo foo6       >> %t.in
+; RUN: echo foo7       >> %t.in
+; RUN: echo foo7       >> %t.in
+; RUN: echo foo6       >> %t.in
+; RUN: echo foo5       >> %t.in
+; RUN: echo foo4       >> %t.in
+; RUN: echo foo3       >> %t.in
+; RUN: echo foo2       >> %t.in
+; RUN: echo foo1       >> %t.in
+; RUN: echo lab2 world >> %t.in
+; RUN: echo foo1       >> %t.in
+; RUN: echo foo2       >> %t.in
+; RUN: echo foo3       >> %t.in
+; RUN: echo foo4       >> %t.in
+; RUN: echo foo5       >> %t.in
+; RUN: echo foo6       >> %t.in
+; RUN: echo foo7       >> %t.in
+; RUN: echo foo8       >> %t.in
+; RUN: echo foo9       >> %t.in
+; RUN: echo foo0       >> %t.in
+
+; RUN: echo 'CHECK-LABEL: lab1'  >  %t.chk
+; RUN: echo ' CHECK-NEXT: hello' >> %t.chk
+; RUN: echo 'CHECK-LABEL: lab2'  >> %t.chk
+; RUN: echo ' CHECK-NEXT: world' >> %t.chk
+
+;      C0: <<<<<<
+; CS-NEXT:            .
+; CS-NEXT:            .
+; CS-NEXT:            .
+; C5-NEXT:            1: foo8
+; C5-NEXT:            2: foo7
+; C5-NEXT:            3: foo6
+; C5-NEXT:            4: foo5
+; C4-NEXT:            5: foo4
+; C3-NEXT:            6: foo3
+; C2-NEXT:            7: foo2
+; C1-NEXT:            8: foo1
+; C0-NEXT:            9: lab1 hello
+; C0-NEXT: label:1'0     ^~~~
+; C0-NEXT: label:1'1     ^~~~
+; C0-NEXT: next:2             !~~~~ error: match on wrong line
+; C1-NEXT:           10: foo1
+; C2-NEXT:           11: foo2
+; C3-NEXT:           12: foo3
+; C4-NEXT:           13: foo4
+; C5-NEXT:           14: foo5
+; C6-NEXT:           15: foo6
+; C6-NEXT:           16: foo7
+; CM-NEXT:            .
+; CM-NEXT:            .
+; CM-NEXT:            .
+; C6-NEXT:           17: foo7
+; C6-NEXT:           18: foo6
+; C5-NEXT:           19: foo5
+; C4-NEXT:           20: foo4
+; C3-NEXT:           21: foo3
+; C2-NEXT:           22: foo2
+; C1-NEXT:           23: foo1
+; C0-NEXT:           24: lab2 world
+; C0-NEXT: label:3       ^~~~
+; C0-NEXT: next:4             !~~~~ error: match on wrong line
+; C1-NEXT:           25: foo1
+; C2-NEXT:           26: foo2
+; C3-NEXT:           27: foo3
+; C4-NEXT:           28: foo4
+; C5-NEXT:           29: foo5
+; C6-NEXT:           30: foo6
+; C7-NEXT:           31: foo7
+; C7-NEXT:           32: foo8
+; C7-NEXT:           33: foo9
+; C7-NEXT:           34: foo0
+; CE-NEXT:            .
+; CE-NEXT:            .
+; CE-NEXT:            .
+; C0-NEXT: >>>>>>
+
+; Now build an alternate set of checks where input lines that might be elided by
+; ellipses have annotations.
+
+; RUN: cp %t.in %t.wide.in
+; RUN: echo 'CHECK-LABEL: lab1'    >  %t.wide.chk
+; RUN: echo '      CHECK: hello'   >> %t.wide.chk
+; RUN: echo '      CHECK: goodbye' >> %t.wide.chk
+; RUN: echo 'CHECK-LABEL: lab2'    >> %t.wide.chk
+; RUN: echo ' CHECK-NEXT: world'   >> %t.wide.chk
+
+;      W5: <<<<<<
+;      W5:            9: lab1 hello
+; W5-NEXT: label:1'0     ^~~~
+; W5-NEXT: label:1'1     ^~~~
+; W5-NEXT: check:2            ^~~~~
+; W5-NEXT:           10: foo1
+; W5-NEXT: check:3       X~~~ error: no match found
+; W5-NEXT:           11: foo2
+; W5-NEXT: check:3       ~~~~
+; W5-NEXT:           12: foo3
+; W5-NEXT: check:3       ~~~~
+; W5-NEXT:           13: foo4
+; W5-NEXT: check:3       ~~~~
+; W5-NEXT:           14: foo5
+; W5-NEXT: check:3       ~~~~
+; W5-NEXT:           15: foo6
+; W5-NEXT: check:3       ~~~~
+; W6-NEXT:           16: foo7
+; W6-NEXT: check:3       ~~~~
+; WM-NEXT:            .
+; WM-NEXT:            .
+; WM-NEXT:            .
+; W6-NEXT:           17: foo7
+; W6-NEXT: check:3       ~~~~
+; W6-NEXT:           18: foo6
+; W6-NEXT: check:3       ~~~~
+; W5-NEXT:           19: foo5
+; W5-NEXT: check:3       ~~~~
+; W5-NEXT:           20: foo4
+; W5-NEXT: check:3       ~~~~
+; W5-NEXT:           21: foo3
+; W5-NEXT: check:3       ~~~~
+; W5-NEXT:           22: foo2
+; W5-NEXT: check:3       ~~~~
+; W5-NEXT:           23: foo1
+; W5-NEXT: check:3       ~~~~
+; W5-NEXT:           24: lab2 world
+; W5-NEXT: label:4       ^~~~
+; W5-NEXT: check:3       ~~~~
+; W5-NEXT: next:5             !~~~~ error: match on wrong line
+
+;--------------------------------------------------
+; Check -dump-input-context=<bad value>.
+;--------------------------------------------------
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-context=-1 \
+; RUN: | FileCheck %s -match-full-lines -check-prefix=BADVAL -DVAL=-1
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-context=foobar \
+; RUN: | FileCheck %s -match-full-lines -check-prefix=BADVAL -DVAL=foobar
+
+BADVAL: {{F|f}}ile{{C|c}}heck{{.*}}: for the --dump-input-context option: '[[VAL]]' value invalid for uint argument!
+
+;--------------------------------------------------
+; Check -dump-input-context explicit values.
+;--------------------------------------------------
+
+; 0 is an important boundary case.
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=fail -vv  %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-context=0 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=C0,CS,CM,CE
+
+; 1 is an important boundary case.
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-context=1 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=C0,C1,CS,CM,CE
+
+; 4 is the boundary case at which all ellipses are present in our test.
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-context=4 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=C0,C1,C2,C3,C4,CS,CM,CE
+
+; 5 is the boundary case at which the start ellipsis is useless.
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-context=5 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=C0,C1,C2,C3,C4,C5,CM,CE
+
+; 6 is the boundary case at which the middle ellipsis is useless.
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-context=6 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=C0,C1,C2,C3,C4,C5,C6,CE
+
+; 7 is the boundary case at which the end ellipsis is useless.
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-context=7 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=C0,C1,C2,C3,C4,C5,C6,C7
+
+; Make sure all is fine when -dump-input-context is far larger than the input.
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-context=200 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=C0,C1,C2,C3,C4,C5,C6,C7
+
+;--------------------------------------------------
+; Check that -dump-input-context default is 5.
+;--------------------------------------------------
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=C0,C1,C2,C3,C4,C5,CM,CE
+
+;--------------------------------------------------
+; Check multiple -dump-input-context options.
+;
+; This might occur when a test author specifies -dump-input-context on a
+; specific FileCheck call while a test runner specifies -dump-input-context in
+; FILECHECK_OPTS, but check the behavior generally.
+;
+; The largest value wins because it provides the most information.
+;--------------------------------------------------
+
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+; Check duplicate.
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-context=1 -dump-input-context=1 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=C0,C1,CS,CM,CE
+
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+; Check precedence.
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-context=0 -dump-input-context=1 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=C0,C1,CS,CM,CE
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-context=1 -dump-input-context=0 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=C0,C1,CS,CM,CE
+
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+; Check that FILECHECK_OPTS isn't handled differently.
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+
+; RUN: %ProtectFileCheckOutput FILECHECK_OPTS=-dump-input-context=0 \
+; RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-context=1 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=C0,C1,CS,CM,CE
+
+; RUN: %ProtectFileCheckOutput FILECHECK_OPTS=-dump-input-context=1 \
+; RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-context=0 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=C0,C1,CS,CM,CE
+
+;--------------------------------------------------
+; Check how annotations on input lines that might be elided by ellipses affect
+; whether they are actually elided.
+;--------------------------------------------------
+
+; At -dump-input-context=5, the ellipsis is useful but only when annotations on
+; elided input lines are considered.
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=fail -vv %t.wide.chk < %t.wide.in 2>&1 \
+; RUN:               -dump-input-context=5 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=W5,WM
+
+; At -dump-input-context=6, the ellipsis is not useful even when annotations on
+; elided input lines are considered.
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=fail -vv %t.wide.chk < %t.wide.in 2>&1 \
+; RUN:               -dump-input-context=6 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=W5,W6
diff --git a/llvm/test/FileCheck/dump-input-enable.txt b/llvm/test/FileCheck/dump-input-enable.txt
index 48a6eef417154..932701b0019bd 100644
--- a/llvm/test/FileCheck/dump-input-enable.txt
+++ b/llvm/test/FileCheck/dump-input-enable.txt
@@ -144,7 +144,7 @@ BADVAL: {{F|f}}ile{{C|c}}heck{{.*}}: for the --dump-input option: Cannot find op
 ;--------------------------------------------------
 ; Check multiple -dump-input options.
 ;
-; This ocurrs most commonly when a test author specifies -dump-input on a
+; This occurs most commonly when a test author specifies -dump-input on a
 ; specific FileCheck call while a test runner specifies -dump-input in
 ; FILECHECK_OPTS, but check the behavior generally.
 ;
@@ -224,7 +224,7 @@ BADVAL: {{F|f}}ile{{C|c}}heck{{.*}}: for the --dump-input option: Cannot find op
 
 ; HELP-NOT: {{.}}
 ; HELP: The following description was requested by -dump-input=help
-; HELP: try{{.*}}-color
+; HELP: - colors {{.*}}
 ; HELP-NOT: {{.}}
 
 ; Trace is sometimes suppressed.
@@ -236,7 +236,7 @@ BADVAL: {{F|f}}ile{{C|c}}heck{{.*}}: for the --dump-input option: Cannot find op
 
 ; NODUMP-NOT: <<<<<<
 
-; DUMP-OK:      Full input was:
+; DUMP-OK:      Input was:
 ; DUMP-OK-NEXT: <<<<<<
 ; DUMP-OK-NEXT:          1: hello
 ; DUMP-OK-NEXT: check:1     ^~~~~
@@ -244,7 +244,7 @@ BADVAL: {{F|f}}ile{{C|c}}heck{{.*}}: for the --dump-input option: Cannot find op
 ; DUMP-OK-NEXT: next:2      ^~~~~
 ; DUMP-OK-NEXT: >>>>>>
 
-; DUMP-ERR:        Full input was:
+; DUMP-ERR:        Input was:
 ; DUMP-ERR-NEXT:   <<<<<<
 ; DUMP-ERR-NEXT:            1: hello
 ; DUMP-ERR-V-NEXT: check:1     ^~~~~
diff --git a/llvm/test/FileCheck/dump-input-filter.txt b/llvm/test/FileCheck/dump-input-filter.txt
new file mode 100644
index 0000000000000..29f43ff1ce80f
--- /dev/null
+++ b/llvm/test/FileCheck/dump-input-filter.txt
@@ -0,0 +1,425 @@
+; To keep this test maintainable, avoid depending on -dump-input-context's
+; default value, which is checked in dump-input-context.txt instead.
+
+;--------------------------------------------------
+; Create the input file and the check file.
+;--------------------------------------------------
+
+; line 1
+; RUN: echo start >  %t.in
+; RUN: echo foo0  >> %t.in
+; RUN: echo foo1  >> %t.in
+; RUN: echo foo2  >> %t.in
+; RUN: echo foo3  >> %t.in
+; RUN: echo foo4  >> %t.in
+; RUN: echo foo5  >> %t.in
+; RUN: echo foo6  >> %t.in
+; RUN: echo foo7  >> %t.in
+; RUN: echo foo8  >> %t.in
+; RUN: echo foo9  >> %t.in
+; line 12
+; RUN: echo hello >> %t.in
+; RUN: echo foo0  >> %t.in
+; RUN: echo foo1  >> %t.in
+; RUN: echo foo2  >> %t.in
+; RUN: echo foo3  >> %t.in
+; RUN: echo foo4  >> %t.in
+; RUN: echo foo5  >> %t.in
+; RUN: echo foo6  >> %t.in
+; RUN: echo foo7  >> %t.in
+; RUN: echo foo8  >> %t.in
+; RUN: echo foo9  >> %t.in
+; line 23
+; RUN: echo word  >> %t.in
+; RUN: echo foo0  >> %t.in
+; RUN: echo foo1  >> %t.in
+; RUN: echo foo2  >> %t.in
+; RUN: echo foo3  >> %t.in
+; RUN: echo foo4  >> %t.in
+; RUN: echo foo5  >> %t.in
+; RUN: echo foo6  >> %t.in
+; RUN: echo foo7  >> %t.in
+; RUN: echo foo8  >> %t.in
+; RUN: echo foo9  >> %t.in
+; line 34
+; RUN: echo end   >> %t.in
+
+; RUN: echo 'CHECK: start' >  %t.chk
+; RUN: echo 'CHECK: hello' >> %t.chk
+; RUN: echo 'CHECK: world' >> %t.chk
+; RUN: echo 'CHECK: end'   >> %t.chk
+
+;--------------------------------------------------
+; Directives for checking the dump.
+;--------------------------------------------------
+
+;      ALL: <<<<<<
+; ALL-NEXT:            1: start
+; ALL-NEXT: check:1       ^~~~~
+; ALL-NEXT:            2: foo0
+; ALL-NEXT:            3: foo1
+; ALL-NEXT:            4: foo2
+; ALL-NEXT:            5: foo3
+; ALL-NEXT:            6: foo4
+; ALL-NEXT:            7: foo5
+; ALL-NEXT:            8: foo6
+; ALL-NEXT:            9: foo7
+; ALL-NEXT:           10: foo8
+; ALL-NEXT:           11: foo9
+; ALL-NEXT:           12: hello
+; ALL-NEXT: check:2       ^~~~~
+; ALL-NEXT:           13: foo0
+; ALL-NEXT: check:3'0     X~~~ error: no match found
+; ALL-NEXT:           14: foo1
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           15: foo2
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           16: foo3
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           17: foo4
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           18: foo5
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           19: foo6
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           20: foo7
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           21: foo8
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           22: foo9
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           23: word
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT: check:3'1     ?     possible intended match
+; ALL-NEXT:           24: foo0
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           25: foo1
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           26: foo2
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           27: foo3
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           28: foo4
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           29: foo5
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           30: foo6
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           31: foo7
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           32: foo8
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           33: foo9
+; ALL-NEXT: check:3'0     ~~~~
+; ALL-NEXT:           34: end
+; ALL-NEXT: check:3'0     ~~~
+; ALL-NEXT: >>>>>>
+
+;      ANNOTATION-FULL: <<<<<<
+; ANNOTATION-FULL-NEXT:            1: start
+; ANNOTATION-FULL-NEXT: check:1       ^~~~~
+; ANNOTATION-FULL-NEXT:            2: foo0
+; ANNOTATION-FULL-NEXT:            3: foo1
+; ANNOTATION-FULL-NEXT:            .
+; ANNOTATION-FULL-NEXT:            .
+; ANNOTATION-FULL-NEXT:            .
+; ANNOTATION-FULL-NEXT:           10: foo8
+; ANNOTATION-FULL-NEXT:           11: foo9
+; ANNOTATION-FULL-NEXT:           12: hello
+; ANNOTATION-FULL-NEXT: check:2       ^~~~~
+; ANNOTATION-FULL-NEXT:           13: foo0
+; ANNOTATION-FULL-NEXT: check:3'0     X~~~ error: no match found
+; ANNOTATION-FULL-NEXT:           14: foo1
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           15: foo2
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           16: foo3
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           17: foo4
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           18: foo5
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           19: foo6
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           20: foo7
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           21: foo8
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           22: foo9
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           23: word
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT: check:3'1     ?     possible intended match
+; ANNOTATION-FULL-NEXT:           24: foo0
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           25: foo1
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           26: foo2
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           27: foo3
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           28: foo4
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           29: foo5
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           30: foo6
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           31: foo7
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           32: foo8
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           33: foo9
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~~
+; ANNOTATION-FULL-NEXT:           34: end
+; ANNOTATION-FULL-NEXT: check:3'0     ~~~
+; ANNOTATION-FULL-NEXT: >>>>>>
+
+;      ANNOTATION: <<<<<<
+; ANNOTATION-NEXT:            1: start
+; ANNOTATION-NEXT: check:1       ^~~~~
+; ANNOTATION-NEXT:            2: foo0
+; ANNOTATION-NEXT:            3: foo1
+; ANNOTATION-NEXT:            .
+; ANNOTATION-NEXT:            .
+; ANNOTATION-NEXT:            .
+; ANNOTATION-NEXT:           10: foo8
+; ANNOTATION-NEXT:           11: foo9
+; ANNOTATION-NEXT:           12: hello
+; ANNOTATION-NEXT: check:2       ^~~~~
+; ANNOTATION-NEXT:           13: foo0
+; ANNOTATION-NEXT: check:3'0     X~~~ error: no match found
+; ANNOTATION-NEXT:           14: foo1
+; ANNOTATION-NEXT: check:3'0     ~~~~
+; ANNOTATION-NEXT:           15: foo2
+; ANNOTATION-NEXT: check:3'0     ~~~~
+; ANNOTATION-NEXT:            .
+; ANNOTATION-NEXT:            .
+; ANNOTATION-NEXT:            .
+; ANNOTATION-NEXT:           21: foo8
+; ANNOTATION-NEXT: check:3'0     ~~~~
+; ANNOTATION-NEXT:           22: foo9
+; ANNOTATION-NEXT: check:3'0     ~~~~
+; ANNOTATION-NEXT:           23: word
+; ANNOTATION-NEXT: check:3'0     ~~~~
+; ANNOTATION-NEXT: check:3'1     ?     possible intended match
+; ANNOTATION-NEXT:           24: foo0
+; ANNOTATION-NEXT: check:3'0     ~~~~
+; ANNOTATION-NEXT:           25: foo1
+; ANNOTATION-NEXT: check:3'0     ~~~~
+; ANNOTATION-NEXT:            .
+; ANNOTATION-NEXT:            .
+; ANNOTATION-NEXT:            .
+; ANNOTATION-NEXT: >>>>>>
+
+;      ERROR: <<<<<<
+; ERROR-NEXT:            .
+; ERROR-NEXT:            .
+; ERROR-NEXT:            .
+; ERROR-NEXT:           11: foo9
+; ERROR-NEXT:           12: hello
+; ERROR-NEXT: check:2       ^~~~~
+; ERROR-NEXT:           13: foo0
+; ERROR-NEXT: check:3'0     X~~~ error: no match found
+; ERROR-NEXT:           14: foo1
+; ERROR-NEXT: check:3'0     ~~~~
+; ERROR-NEXT:           15: foo2
+; ERROR-NEXT: check:3'0     ~~~~
+; ERROR-NEXT:            .
+; ERROR-NEXT:            .
+; ERROR-NEXT:            .
+; ERROR-NEXT:           21: foo8
+; ERROR-NEXT: check:3'0     ~~~~
+; ERROR-NEXT:           22: foo9
+; ERROR-NEXT: check:3'0     ~~~~
+; ERROR-NEXT:           23: word
+; ERROR-NEXT: check:3'0     ~~~~
+; ERROR-NEXT: check:3'1     ?     possible intended match
+; ERROR-NEXT:           24: foo0
+; ERROR-NEXT: check:3'0     ~~~~
+; ERROR-NEXT:           25: foo1
+; ERROR-NEXT: check:3'0     ~~~~
+; ERROR-NEXT:            .
+; ERROR-NEXT:            .
+; ERROR-NEXT:            .
+; ERROR-NEXT: >>>>>>
+
+;--------------------------------------------------
+; Check -dump-input-filter=<bad value>.
+;--------------------------------------------------
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input-context=2 -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-filter=foobar \
+; RUN: | FileCheck %s -match-full-lines -check-prefix=BADVAL
+
+BADVAL: {{F|f}}ile{{C|c}}heck{{.*}}: for the --dump-input-filter option: Cannot find option named 'foobar'!
+
+;--------------------------------------------------
+; Check -dump-input-filter explicit values.
+;--------------------------------------------------
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input-context=2 -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-filter=all \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=ALL
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input-context=2 -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-filter=annotation-full \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=ANNOTATION-FULL
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input-context=2 -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-filter=annotation \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=ANNOTATION
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input-context=2 -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-filter=error \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=ERROR
+
+;--------------------------------------------------
+; Check -dump-input-filter defaults.
+;--------------------------------------------------
+
+; no -dump-input => -dump-input-filter=error
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input-context=2 -vv %t.chk < %t.in 2>&1 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=ERROR
+
+; -dump-input=fail => -dump-input-filter=error
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input-context=2 -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input=fail \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=ERROR
+
+; -dump-input=always => -dump-input-filter=all
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input-context=2 -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input=always \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=ALL
+
+;--------------------------------------------------
+; Check multiple -dump-input-filter options.
+;
+; This might occur when a test author specifies -dump-input-filter on a specific
+; FileCheck call while a test runner specifies -dump-input-filter in
+; FILECHECK_OPTS, but check the behavior generally.
+;
+; The value providing the most information wins.
+;--------------------------------------------------
+
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+; Check duplicate.
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+
+; all, all => all
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input-context=2 -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-filter=all -dump-input-filter=all \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=ALL
+
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+; Check precedence.
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+
+; all, annotation-full => all
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input-context=2 -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-filter=all -dump-input-filter=annotation-full \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=ALL
+
+; annotation-full, annotation => annotation-full
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input-context=2 -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-filter=annotation-full \
+; RUN:               -dump-input-filter=annotation \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=ANNOTATION-FULL
+
+; annotation, error => annotation
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input-context=2 -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-filter=annotation -dump-input-filter=error \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=ANNOTATION
+
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+; Check that order doesn't matter.
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+
+; error, annotation => annotation
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input-context=2 -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-filter=error -dump-input-filter=annotation \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=ANNOTATION
+
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+; Check that FILECHECK_OPTS isn't handled differently.
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+
+; annotation, error => annotation
+; RUN: %ProtectFileCheckOutput FILECHECK_OPTS=-dump-input-filter=annotation \
+; RUN: not FileCheck -dump-input-context=2 -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-filter=error \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=ANNOTATION
+
+; error, annotation => annotation
+; RUN: %ProtectFileCheckOutput FILECHECK_OPTS=-dump-input-filter=error \
+; RUN: not FileCheck -dump-input-context=2 -vv %t.chk < %t.in 2>&1 \
+; RUN:               -dump-input-filter=annotation \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=ANNOTATION
+
+;--------------------------------------------------
+; Check the case where all input lines are filtered out.
+;--------------------------------------------------
+
+; RUN: echo 'CHECK: hello' > %t.good.chk
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: FileCheck -dump-input=always -dump-input-filter=error -vv %t.good.chk \
+; RUN:           < %t.in 2>&1 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=EMPTY
+
+;      EMPTY: <<<<<<
+; EMPTY-NEXT:          .
+; EMPTY-NEXT:          .
+; EMPTY-NEXT:          .
+; EMPTY-NEXT: >>>>>>
+
+;--------------------------------------------------
+; Check that other kinds of errors are included by -dump-input-filter=error.
+;
+; "error: no match found" and "possible intended match" are checked above.
+;--------------------------------------------------
+
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+; error: no match expected.
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+
+; RUN: echo 'foo'            > %t.not-err.in
+; RUN: echo 'CHECK-NOT: foo' > %t.not-err.chk
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input-context=0 -dump-input-filter=error \
+; RUN:               %t.not-err.chk < %t.not-err.in 2>&1 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=NOT-ERR
+
+;      NOT-ERR:        1: foo
+; NOT-ERR-NEXT: not:1     !~~ error: no match expected
+
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+; error: match on wrong line.
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+
+; RUN: echo 'foo'             >  %t.next-err.in
+; RUN: echo 'foo'             >> %t.next-err.in
+; RUN: echo 'bar'             >> %t.next-err.in
+; RUN: echo 'CHECK: foo'      >  %t.next-err.chk
+; RUN: echo 'CHECK-NEXT: bar' >> %t.next-err.chk
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input-context=0 -dump-input-filter=error \
+; RUN:               %t.next-err.chk < %t.next-err.in 2>&1 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=NEXT-ERR
+
+;      NEXT-ERR:         3: bar
+; NEXT-ERR-NEXT: next:2     !~~ error: match on wrong line
\ No newline at end of file
diff --git a/llvm/test/FileCheck/verbose.txt b/llvm/test/FileCheck/verbose.txt
index f852702a9b1f8..d0532d02f9fc6 100644
--- a/llvm/test/FileCheck/verbose.txt
+++ b/llvm/test/FileCheck/verbose.txt
@@ -1,9 +1,32 @@
-; RUN: %ProtectFileCheckOutput FileCheck -dump-input=never -input-file %s %s 2>&1 \
-; RUN: | FileCheck -check-prefix QUIET --allow-empty %s
-; RUN: %ProtectFileCheckOutput FileCheck -dump-input=never -v -input-file %s %s 2>&1 \
-; RUN: | FileCheck --strict-whitespace -check-prefix V %s
-; RUN: %ProtectFileCheckOutput FileCheck -dump-input=never -vv -input-file %s %s 2>&1 \
-; RUN: | FileCheck --strict-whitespace -check-prefixes V,VV %s
+; RUN: %ProtectFileCheckOutput \
+; RUN: FileCheck --dump-input=never --input-file %s %s 2>&1 | \
+; RUN:   FileCheck --check-prefix QUIET --allow-empty %s
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: FileCheck --dump-input=never -v --input-file %s %s 2>&1 | \
+; RUN:   FileCheck --strict-whitespace --check-prefix V %s
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: FileCheck --dump-input=never -vv --input-file %s %s 2>&1 | \
+; RUN:   FileCheck --strict-whitespace --check-prefixes V,VV %s
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: FileCheck --dump-input=never -v -v --input-file %s %s 2>&1 | \
+; RUN:   FileCheck --strict-whitespace --check-prefix V %s
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: FileCheck --dump-input=never -vv -vv --input-file %s %s 2>&1 | \
+; RUN:   FileCheck --strict-whitespace --check-prefixes V,VV %s
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: FileCheck --dump-input=never -v -vv --input-file %s %s 2>&1 | \
+; RUN:   FileCheck --strict-whitespace --check-prefixes V,VV %s
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: FileCheck --dump-input=never -vv -v --input-file %s %s 2>&1 | \
+; RUN:   FileCheck --strict-whitespace --check-prefixes V,VV %s
+
+; END.
 
 foo
 bar
diff --git a/llvm/test/Instrumentation/AddressSanitizer/asan-vs-gvn.ll b/llvm/test/Instrumentation/AddressSanitizer/asan-vs-gvn.ll
index 1d0897f2d0ca4..f9cfa7af19f62 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/asan-vs-gvn.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/asan-vs-gvn.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -asan -asan-module -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -asan -asan-module -S | FileCheck %s
 ; ASAN conflicts with load widening iff the widened load accesses data out of bounds
 ; (while the original unwidened loads do not).
 ; https://github.com/google/sanitizers/issues/20#issuecomment-136381262
diff --git a/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll b/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll
index 681035d188686..7a265b0ac3e2b 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/msan_basic.ll
@@ -95,7 +95,6 @@ declare void @foo(...)
 ; CHECK: call void @__msan_warning_with_origin_noreturn(i32
 ; CHECK-ORIGINS-SAME %[[ORIGIN]])
 ; CHECK-CONT: 
-; CHECK-NEXT: call void asm sideeffect
 ; CHECK-NEXT: unreachable
 ; CHECK: br i1 %tobool
 ; CHECK: ret void
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll b/llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll
index a743642d5e2ab..f870b6eaba36a 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll
@@ -16,7 +16,7 @@
 ; and add sanitize_address to @_ZN1A1fEv
 
 ; Test that __sanitizer_cov_trace_pc_guard call has !dbg pointing to the opening { of A::f().
-; CHECK: call void @__sanitizer_cov_trace_pc_guard(i32*{{.*}}), !dbg [[A:!.*]]
+; CHECK: call void @__sanitizer_cov_trace_pc_guard(i32*{{.*}}) #{{.*}}, !dbg [[A:!.*]]
 ; CHECK: [[A]] = !DILocation(line: 6, scope: !{{.*}})
 
 
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coverage.ll b/llvm/test/Instrumentation/SanitizerCoverage/coverage.ll
index 4947af8f2c9f3..bb0bed06a47ef 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coverage.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coverage.ll
@@ -85,7 +85,6 @@ entry:
 
 ; CHECK_TRACE_PC-LABEL: define void @foo
 ; CHECK_TRACE_PC: call void @__sanitizer_cov_trace_pc
-; CHECK_TRACE_PC: call void asm sideeffect "", ""()
 ; CHECK_TRACE_PC: ret void
 
 ; CHECK_TRACE_PC-LABEL: define void @CallViaVptr
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll b/llvm/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll
index 14462677e36ad..11ec286ce1ee0 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll
@@ -18,8 +18,8 @@ target triple = "x86_64-unknown-linux-gnu"
 ; Check that __sanitizer_cov call has !dgb pointing to the beginning
 ; of appropriate basic blocks.
 ; CHECK-LABEL:_Z3fooPi
-; CHECK: call void @__sanitizer_cov{{.*}}(i32*{{.*}}), !dbg [[A:!.*]]
-; CHECK: call void @__sanitizer_cov{{.*}}(i32*{{.*}}), !dbg [[B:!.*]]
+; CHECK: call void @__sanitizer_cov{{.*}}(i32*{{.*}}) #{{.*}}, !dbg [[A:!.*]]
+; CHECK: call void @__sanitizer_cov{{.*}}(i32*{{.*}}) #{{.*}}, !dbg [[B:!.*]]
 ; CHECK: ret void
 ; CHECK: [[A]] = !DILocation(line: 1, scope: !{{.*}})
 ; CHECK: [[B]] = !DILocation(line: 3, column: 5, scope: !{{.*}})
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-comdat.ll b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-comdat.ll
index d68eca0fadb92..a278dbcfdcab4 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-comdat.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-comdat.ll
@@ -31,7 +31,6 @@ entry:
 
 ; CHECK_TRACE_PC_GUARD-LABEL: define void @foo
 ; CHECK_TRACE_PC_GUARD: call void @__sanitizer_cov_trace_pc
-; CHECK_TRACE_PC_GUARD: call void asm sideeffect "", ""()
 ; CHECK_TRACE_PC_GUARD: ret void
 
 ; CHECK_TRACE_PC_GUARD-LABEL: define void @CallViaVptr
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-nocomdat.ll b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-nocomdat.ll
index 95cb2b96b2271..1cfbfebdf2689 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-nocomdat.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-nocomdat.ll
@@ -31,7 +31,6 @@ entry:
 
 ; CHECK_TRACE_PC_GUARD-LABEL: define void @foo
 ; CHECK_TRACE_PC_GUARD: call void @__sanitizer_cov_trace_pc
-; CHECK_TRACE_PC_GUARD: call void asm sideeffect "", ""()
 ; CHECK_TRACE_PC_GUARD: ret void
 
 ; CHECK_TRACE_PC_GUARD-LABEL: define void @CallViaVptr
diff --git a/llvm/test/Instrumentation/ThreadSanitizer/tsan-vs-gvn.ll b/llvm/test/Instrumentation/ThreadSanitizer/tsan-vs-gvn.ll
index df7608dd449cd..a0a9370b6291b 100644
--- a/llvm/test/Instrumentation/ThreadSanitizer/tsan-vs-gvn.ll
+++ b/llvm/test/Instrumentation/ThreadSanitizer/tsan-vs-gvn.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -tsan -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -tsan -S | FileCheck %s
 ; TSAN conflicts with load widening. Make sure the load widening is off with -tsan.
 
 ; 32-bit little endian target.
diff --git a/llvm/test/Instrumentation/cgprofile.ll b/llvm/test/Instrumentation/cgprofile.ll
index 1edf3b6ec5189..70a1f81aa53ec 100644
--- a/llvm/test/Instrumentation/cgprofile.ll
+++ b/llvm/test/Instrumentation/cgprofile.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -passes cg-profile -S | FileCheck %s
+; RUN: opt < %s -cg-profile -S | FileCheck %s
 
 declare void @b()
 
diff --git a/llvm/test/MC/AArch64/armv8.2a-dotprod.s b/llvm/test/MC/AArch64/armv8.2a-dotprod.s
index b8aafbe885700..3b9f416a63fb6 100644
--- a/llvm/test/MC/AArch64/armv8.2a-dotprod.s
+++ b/llvm/test/MC/AArch64/armv8.2a-dotprod.s
@@ -4,6 +4,9 @@
 // RUN: llvm-mc -triple aarch64 -mcpu=cortex-a65ae -show-encoding < %s| FileCheck %s --check-prefix=CHECK-DOTPROD
 // RUN: llvm-mc -triple aarch64 -mcpu=cortex-a75 -show-encoding < %s | FileCheck %s --check-prefix=CHECK-DOTPROD
 // RUN: llvm-mc -triple aarch64 -mcpu=cortex-a76 -show-encoding < %s | FileCheck %s --check-prefix=CHECK-DOTPROD
+// RUN: llvm-mc -triple aarch64 -mcpu=cortex-a77 -show-encoding < %s     | FileCheck %s --check-prefix=CHECK-DOTPROD
+// RUN: llvm-mc -triple aarch64 -mcpu=cortex-a78 -show-encoding < %s     | FileCheck %s --check-prefix=CHECK-DOTPROD
+// RUN: llvm-mc -triple aarch64 -mcpu=cortex-x1 -show-encoding < %s     | FileCheck %s --check-prefix=CHECK-DOTPROD
 // RUN: llvm-mc -triple aarch64 -mcpu=neoverse-e1 -show-encoding < %s| FileCheck %s --check-prefix=CHECK-DOTPROD
 // RUN: llvm-mc -triple aarch64 -mcpu=neoverse-n1 -show-encoding < %s| FileCheck %s --check-prefix=CHECK-DOTPROD
 // RUN: llvm-mc -triple aarch64 -mcpu=tsv110 -show-encoding < %s | FileCheck %s --check-prefix=CHECK-DOTPROD
@@ -16,6 +19,12 @@
 // RUN: FileCheck --check-prefix=CHECK-NO-DOTPROD < %t %s
 // RUN: not llvm-mc -triple aarch64 -mcpu=neoverse-e1 -mattr=-dotprod -show-encoding < %s 2> %t
 // RUN: FileCheck --check-prefix=CHECK-NO-DOTPROD < %t %s
+// RUN: not llvm-mc -triple aarch64 -mcpu=cortex-a77 -mattr=-dotprod -show-encoding < %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-NO-DOTPROD < %t %s
+// RUN: not llvm-mc -triple aarch64 -mcpu=cortex-a78 -mattr=-dotprod -show-encoding < %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-NO-DOTPROD < %t %s
+// RUN: not llvm-mc -triple aarch64 -mcpu=cortex-x1 -mattr=-dotprod -show-encoding < %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-NO-DOTPROD < %t %s
 // RUN: not llvm-mc -triple aarch64 -mcpu=neoverse-n1 -mattr=-dotprod -show-encoding < %s 2> %t
 // RUN: FileCheck --check-prefix=CHECK-NO-DOTPROD < %t %s
 
diff --git a/llvm/test/MC/AArch64/mapping-within-section.s b/llvm/test/MC/AArch64/mapping-within-section.s
index c7c3bbe08bed7..07a2d3cc0fed9 100644
--- a/llvm/test/MC/AArch64/mapping-within-section.s
+++ b/llvm/test/MC/AArch64/mapping-within-section.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple=aarch64-none-linux-gnu -filetype=obj %s | llvm-nm - | FileCheck %s
+// RUN: llvm-mc -triple=aarch64-none-linux-gnu -filetype=obj %s | llvm-nm --special-syms - | FileCheck %s
 
     .text
 // $x at 0x0000
diff --git a/llvm/test/MC/AMDGPU/hsa-metadata-kernel-args.s b/llvm/test/MC/AMDGPU/hsa-metadata-kernel-args.s
index f0c2908224409..390eba266d895 100644
--- a/llvm/test/MC/AMDGPU/hsa-metadata-kernel-args.s
+++ b/llvm/test/MC/AMDGPU/hsa-metadata-kernel-args.s
@@ -2,6 +2,9 @@
 // RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx800 -mattr=-code-object-v3 -show-encoding %s | FileCheck --check-prefix=CHECK --check-prefix=GFX800 %s
 // RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-code-object-v3 -show-encoding %s | FileCheck --check-prefix=CHECK --check-prefix=GFX900 %s
 
+// The legacy ValueType field should be parsed without error, but not
+// re-emitted.
+
 // CHECK:  .amd_amdgpu_hsa_metadata
 // CHECK:    Version: [ 1, 0 ]
 // CHECK:    Printf:
@@ -17,24 +20,19 @@
 // CHECK:            Size:          1
 // CHECK:            Align:         1
 // CHECK:            ValueKind:     ByValue
-// CHECK:            ValueType:     I8
 // CHECK:            AccQual:       Default
 // CHECK:          - Size:          8
 // CHECK:            Align:         8
 // CHECK:            ValueKind:     HiddenGlobalOffsetX
-// CHECK:            ValueType:     I64
 // CHECK:          - Size:          8
 // CHECK:            Align:         8
 // CHECK:            ValueKind:     HiddenGlobalOffsetY
-// CHECK:            ValueType:     I64
 // CHECK:          - Size:          8
 // CHECK:            Align:         8
 // CHECK:            ValueKind:     HiddenGlobalOffsetZ
-// CHECK:            ValueType:     I64
 // CHECK:          - Size:          8
 // CHECK:            Align:         8
 // CHECK:            ValueKind:     HiddenPrintfBuffer
-// CHECK:            ValueType:     I8
 // CHECK:            AddrSpaceQual: Global
 // CHECK:  .end_amd_amdgpu_hsa_metadata
 .amd_amdgpu_hsa_metadata
diff --git a/llvm/test/MC/ARM/armv8.2a-dotprod-a32.s b/llvm/test/MC/ARM/armv8.2a-dotprod-a32.s
index 366f85c5cb44e..8ca2a97c602dc 100644
--- a/llvm/test/MC/ARM/armv8.2a-dotprod-a32.s
+++ b/llvm/test/MC/ARM/armv8.2a-dotprod-a32.s
@@ -3,9 +3,18 @@
 // RUN: llvm-mc -triple arm -mcpu=cortex-a75 -show-encoding < %s | FileCheck %s  --check-prefix=CHECK
 // RUN: llvm-mc -triple arm -mcpu=cortex-a76 -show-encoding < %s | FileCheck %s  --check-prefix=CHECK
 // RUN: llvm-mc -triple arm -mcpu=neoverse-n1 -show-encoding < %s | FileCheck %s --check-prefix=CHECK
+// RUN: llvm-mc -triple arm -mcpu=cortex-a77 -show-encoding < %s | FileCheck %s --check-prefix=CHECK
+// RUN: llvm-mc -triple arm -mcpu=cortex-a78 -show-encoding < %s | FileCheck %s --check-prefix=CHECK
+// RUN: llvm-mc -triple arm -mcpu=cortex-x1 -show-encoding < %s | FileCheck %s --check-prefix=CHECK
 
 // RUN: not llvm-mc -triple arm -mattr=-dotprod -show-encoding < %s 2> %t
 // RUN: FileCheck --check-prefix=CHECK-NO-DOTPROD < %t %s
+// RUN: not llvm-mc -triple arm -mcpu=cortex-a77 -mattr=-dotprod -show-encoding < %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-NO-DOTPROD < %t %s
+// RUN: not llvm-mc -triple arm -mcpu=cortex-a78 -mattr=-dotprod -show-encoding < %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-NO-DOTPROD < %t %s
+// RUN: not llvm-mc -triple arm -mcpu=cortex-x1 -mattr=-dotprod -show-encoding < %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-NO-DOTPROD < %t %s
 // RUN: not llvm-mc -triple arm -show-encoding < %s 2> %t
 // RUN: FileCheck --check-prefix=CHECK-NO-DOTPROD < %t %s
 // RUN: not llvm-mc -triple arm -mattr=+v8.1a -show-encoding < %s 2> %t
diff --git a/llvm/test/MC/ARM/armv8.2a-dotprod-t32.s b/llvm/test/MC/ARM/armv8.2a-dotprod-t32.s
index 3964fddaedd4b..8570a7be3150e 100644
--- a/llvm/test/MC/ARM/armv8.2a-dotprod-t32.s
+++ b/llvm/test/MC/ARM/armv8.2a-dotprod-t32.s
@@ -2,6 +2,9 @@
 // RUN: llvm-mc -triple thumb -mcpu=cortex-a55 -show-encoding < %s | FileCheck %s  --check-prefix=CHECK
 // RUN: llvm-mc -triple thumb -mcpu=cortex-a75 -show-encoding < %s | FileCheck %s  --check-prefix=CHECK
 // RUN: llvm-mc -triple thumb -mcpu=cortex-a76 -show-encoding < %s | FileCheck %s  --check-prefix=CHECK
+// RUN: llvm-mc -triple thumb -mcpu=cortex-a77 -show-encoding < %s | FileCheck %s --check-prefix=CHECK
+// RUN: llvm-mc -triple thumb -mcpu=cortex-a78 -show-encoding < %s | FileCheck %s --check-prefix=CHECK
+// RUN: llvm-mc -triple thumb -mcpu=cortex-x1 -show-encoding < %s | FileCheck %s --check-prefix=CHECK
 // RUN: llvm-mc -triple thumb -mcpu=neoverse-n1 -show-encoding < %s | FileCheck %s --check-prefix=CHECK
 
 // RUN: not llvm-mc -triple thumb -mattr=-dotprod -show-encoding < %s 2> %t
diff --git a/llvm/test/MC/ARM/coprocessors.s b/llvm/test/MC/ARM/coprocessors.s
index d2ef581578847..5ab29ccd3be07 100644
--- a/llvm/test/MC/ARM/coprocessors.s
+++ b/llvm/test/MC/ARM/coprocessors.s
@@ -1,13 +1,11 @@
-@ RUN: not llvm-mc -triple=armv7 < %s 2> %t | FileCheck --check-prefix=ACCEPT-01234567CD --check-prefix=ACCEPT-89 --check-prefix=ACCEPT-EF %s
-@ RUN: FileCheck --check-prefix=REJECT-AB < %t %s
-@ RUN: not llvm-mc -triple=thumbv7 < %s 2> %t | FileCheck --check-prefix=ACCEPT-01234567CD --check-prefix=ACCEPT-89 --check-prefix=ACCEPT-EF %s
-@ RUN: FileCheck --check-prefix=REJECT-AB < %t %s
+@ RUN: llvm-mc -triple=armv7 < %s 2> %t | FileCheck --check-prefix=ACCEPT-01234567CD --check-prefix=ACCEPT-89 --check-prefix=ACCEPT-AB --check-prefix=ACCEPT-EF %s
+@ RUN: llvm-mc -triple=thumbv7 < %s 2> %t | FileCheck --check-prefix=ACCEPT-01234567CD --check-prefix=ACCEPT-89 --check-prefix=ACCEPT-AB --check-prefix=ACCEPT-EF %s
 @ RUN: not llvm-mc -triple=armv8 < %s 2> %t | FileCheck --check-prefix=ACCEPT-EF %s
 @ RUN: FileCheck --check-prefix=REJECT-01234567CD --check-prefix=REJECT-89 --check-prefix=REJECT-AB < %t %s
 @ RUN: not llvm-mc -triple=thumbv8 < %s 2> %t | FileCheck --check-prefix=ACCEPT-EF %s
 @ RUN: FileCheck --check-prefix=REJECT-01234567CD --check-prefix=REJECT-89 --check-prefix=REJECT-AB < %t %s
-@ RUN: not llvm-mc -triple=thumbv8.1m.main < %s 2> %t | FileCheck --check-prefix=ACCEPT-01234567CD %s
-@ RUN: FileCheck --check-prefix=REJECT-89 --check-prefix=REJECT-AB --check-prefix=REJECT-EF < %t %s
+@ RUN: not llvm-mc -triple=thumbv8.1m.main < %s 2> %t | FileCheck --check-prefix=ACCEPT-01234567CD --check-prefix=ACCEPT-AB %s
+@ RUN: FileCheck --check-prefix=REJECT-89 --check-prefix=REJECT-EF < %t %s
 
 mrc   p0, #1, r2, c3, c4, #5
 @ ACCEPT-01234567CD: mrc   p0, #1, r2, c3, c4, #5
diff --git a/llvm/test/MC/ARM/diagnostics.s b/llvm/test/MC/ARM/diagnostics.s
index 1e84a802f29d6..e6d80ea7a6280 100644
--- a/llvm/test/MC/ARM/diagnostics.s
+++ b/llvm/test/MC/ARM/diagnostics.s
@@ -173,8 +173,8 @@
         @ p10 and p11 are reserved for NEON
         mcr p10, #2, r5, c1, c1, #4
         mcrr p11, #8, r5, r4, c1
-@ CHECK-ERRORS: error: invalid operand for instruction
-@ CHECK-ERRORS: error: invalid operand for instruction
+@ CHECK-WARN: warning: since v7, cp10 and cp11 are reserved for advanced SIMD or floating point instructions
+@ CHECK-WARN: warning: since v7, cp10 and cp11 are reserved for advanced SIMD or floating point instructions
 
         @ Out of range immediate for MOV
         movw r9, 0x10000
diff --git a/llvm/test/MC/ARM/lsl-zero.s b/llvm/test/MC/ARM/lsl-zero.s
index 5d097115448f8..6e64e00123624 100644
--- a/llvm/test/MC/ARM/lsl-zero.s
+++ b/llvm/test/MC/ARM/lsl-zero.s
@@ -1,6 +1,6 @@
-// RUN: llvm-mc -triple=thumbv7 -show-encoding < %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-NONARM --check-prefix=CHECK-THUMBV7 %s
-// RUN: llvm-mc -triple=thumbv8 -show-encoding < %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-NONARM --check-prefix=CHECK-THUMBV8 %s
-// RUN: llvm-mc -triple=armv7 -show-encoding < %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-ARM %s
+// RUN: llvm-mc -triple=thumbv7 -show-encoding < %s 2>/dev/null | FileCheck --check-prefix=CHECK --check-prefix=CHECK-NONARM --check-prefix=CHECK-THUMBV7 %s
+// RUN: llvm-mc -triple=thumbv8 -show-encoding < %s 2>/dev/null | FileCheck --check-prefix=CHECK --check-prefix=CHECK-NONARM --check-prefix=CHECK-THUMBV8 %s
+// RUN: llvm-mc -triple=armv7 -show-encoding < %s 2>/dev/null | FileCheck --check-prefix=CHECK --check-prefix=CHECK-ARM %s
 
         // lsl #0 is actually mov, so here we check that it behaves the same as
         // mov with regards to the permitted registers and how it behaves in an
diff --git a/llvm/test/MC/AVR/hex-immediates.s b/llvm/test/MC/AVR/hex-immediates.s
new file mode 100644
index 0000000000000..ca4c8b9f33551
--- /dev/null
+++ b/llvm/test/MC/AVR/hex-immediates.s
@@ -0,0 +1,7 @@
+; RUN: llvm-mc -filetype=obj -triple=avr %s -o %t
+; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=DEC
+; RUN: llvm-objdump -d --print-imm-hex %t | FileCheck %s --check-prefix=HEX
+
+; DEC: ldi r24, 66
+; HEX: ldi r24, 0x42
+  ldi r24, 0x42
diff --git a/llvm/test/MC/AVR/inst-adiw.s b/llvm/test/MC/AVR/inst-adiw.s
index 4de394d91f9d7..067bde97faf9b 100644
--- a/llvm/test/MC/AVR/inst-adiw.s
+++ b/llvm/test/MC/AVR/inst-adiw.s
@@ -1,5 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=addsubiw -show-encoding < %s | FileCheck %s
-; RUNx: llvm-mc -filetype=obj -triple avr -mattr=addsubiw < %s | llvm-objdump -d --mattr=addsubiw - | FileCheck --check-prefix=CHECK-INST %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=addsubiw < %s | llvm-objdump -dr --mattr=addsubiw - | FileCheck --check-prefix=CHECK-INST %s
 
 
 foo:
@@ -10,8 +10,11 @@ foo:
   adiw r28,  17
   adiw r28,  0
 
-  adiw r30,  63
   adiw r30,  3
+  adiw r30,  63
+  adiw r24,  63
+  adiw r24,  0
+  adiw r30,  0
 
   adiw r24, SYMBOL
 
@@ -21,8 +24,11 @@ foo:
 ; CHECK: adiw r28,  17               ; encoding: [0x61,0x96]
 ; CHECK: adiw r28,  0                ; encoding: [0x20,0x96]
 
-; CHECK: adiw r30,  63               ; encoding: [0xff,0x96]
 ; CHECK: adiw r30,  3                ; encoding: [0x33,0x96]
+; CHECK: adiw r30,  63               ; encoding: [0xff,0x96]
+; CHECK: adiw r24,  63               ; encoding: [0xcf,0x96]
+; CHECK: adiw r24,  0                ; encoding: [0x00,0x96]
+; CHECK: adiw r30,  0                ; encoding: [0x30,0x96]
 
 ; CHECK: adiw r24, SYMBOL            ; encoding: [0b00AAAAAA,0x96]
                                      ;   fixup A - offset: 0, value: SYMBOL, kind: fixup_6_adiw
@@ -33,7 +39,11 @@ foo:
 ; CHECK-INST: adiw r28,  17
 ; CHECK-INST: adiw r28,  0
 
-; CHECK-INST: adiw r30,  63
 ; CHECK-INST: adiw r30,  3
+; CHECK-INST: adiw r30,  63
+; CHECK-INST: adiw r24,  63
+; CHECK-INST: adiw r24,  0
+; CHECK-INST: adiw r30,  0
 
 ; CHECK-INST: adiw r24, 0
+; CHECK-INST:      R_AVR_6_ADIW SYMBOL
diff --git a/llvm/test/MC/AVR/inst-asr.s b/llvm/test/MC/AVR/inst-asr.s
index 1ddd2af004cdc..1b59d027dc2bc 100644
--- a/llvm/test/MC/AVR/inst-asr.s
+++ b/llvm/test/MC/AVR/inst-asr.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr < %s | llvm-objdump -d - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -12,3 +13,8 @@ foo:
 ; CHECK: asr r25                ; encoding: [0x95,0x95]
 ; CHECK: asr r5                 ; encoding: [0x55,0x94]
 ; CHECK: asr r0                 ; encoding: [0x05,0x94]
+
+; CHECK-INST: asr r31
+; CHECK-INST: asr r25
+; CHECK-INST: asr r5
+; CHECK-INST: asr r0
diff --git a/llvm/test/MC/AVR/inst-com.s b/llvm/test/MC/AVR/inst-com.s
index f89373464df18..43e3b0a96ccd7 100644
--- a/llvm/test/MC/AVR/inst-com.s
+++ b/llvm/test/MC/AVR/inst-com.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr < %s | llvm-objdump -d - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -12,3 +13,8 @@ foo:
 ; CHECK: com r17                  ; encoding: [0x10,0x95]
 ; CHECK: com r4                   ; encoding: [0x40,0x94]
 ; CHECK: com r0                   ; encoding: [0x00,0x94]
+
+; CHECK-INST: com r30
+; CHECK-INST: com r17
+; CHECK-INST: com r4
+; CHECK-INST: com r0
diff --git a/llvm/test/MC/AVR/inst-dec.s b/llvm/test/MC/AVR/inst-dec.s
index bc689c1cb6bb2..c419f7f97ee46 100644
--- a/llvm/test/MC/AVR/inst-dec.s
+++ b/llvm/test/MC/AVR/inst-dec.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr < %s | llvm-objdump -d - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -12,3 +13,8 @@ foo:
 ; CHECK: dec r3                   ; encoding: [0x3a,0x94]
 ; CHECK: dec r24                  ; encoding: [0x8a,0x95]
 ; CHECK: dec r20                  ; encoding: [0x4a,0x95]
+
+; CHECK-INST: dec r26
+; CHECK-INST: dec r3
+; CHECK-INST: dec r24
+; CHECK-INST: dec r20
diff --git a/llvm/test/MC/AVR/inst-elpm.s b/llvm/test/MC/AVR/inst-elpm.s
index 4f58cdbd7c16c..ac67fdb22403e 100644
--- a/llvm/test/MC/AVR/inst-elpm.s
+++ b/llvm/test/MC/AVR/inst-elpm.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=elpm,elpmx -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=elpm,elpmx < %s | llvm-objdump -d --mattr=elpm,elpmx - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -18,3 +19,11 @@ foo:
 
 ; CHECK: elpm r8, Z+           ; encoding: [0x87,0x90]
 ; CHECK: elpm r0, Z+           ; encoding: [0x07,0x90]
+
+; CHECK-INST: elpm
+
+; CHECK-INST: elpm r3,  Z
+; CHECK-INST: elpm r23, Z
+
+; CHECK-INST: elpm r8, Z+
+; CHECK-INST: elpm r0, Z+
diff --git a/llvm/test/MC/AVR/inst-fmul.s b/llvm/test/MC/AVR/inst-fmul.s
index 4cdf27b360848..45443c99434b7 100644
--- a/llvm/test/MC/AVR/inst-fmul.s
+++ b/llvm/test/MC/AVR/inst-fmul.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=mul -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=mul < %s | llvm-objdump -d --mattr=mul - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -7,8 +8,19 @@ foo:
   fmul r19, r17
   fmul r21, r23
   fmul r23, r23
+  fmul r16, r16
+  fmul r16, r23
 
 ; CHECK: fmul r22, r16                   ; encoding: [0x68,0x03]
 ; CHECK: fmul r19, r17                   ; encoding: [0x39,0x03]
 ; CHECK: fmul r21, r23                   ; encoding: [0x5f,0x03]
 ; CHECK: fmul r23, r23                   ; encoding: [0x7f,0x03]
+; CHECK: fmul r16, r16                   ; encoding: [0x08,0x03]
+; CHECK: fmul r16, r23                   ; encoding: [0x0f,0x03]
+
+; CHECK-INST: fmul r22, r16
+; CHECK-INST: fmul r19, r17
+; CHECK-INST: fmul r21, r23
+; CHECK-INST: fmul r23, r23
+; CHECK-INST: fmul r16, r16
+; CHECK-INST: fmul r16, r23
diff --git a/llvm/test/MC/AVR/inst-fmuls.s b/llvm/test/MC/AVR/inst-fmuls.s
index a8eb6aee1939c..58a3d06722d06 100644
--- a/llvm/test/MC/AVR/inst-fmuls.s
+++ b/llvm/test/MC/AVR/inst-fmuls.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=mul -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=mul < %s | llvm-objdump -d --mattr=mul - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -12,3 +13,8 @@ foo:
 ; CHECK: fmuls r19, r17                   ; encoding: [0xb1,0x03]
 ; CHECK: fmuls r21, r23                   ; encoding: [0xd7,0x03]
 ; CHECK: fmuls r23, r23                   ; encoding: [0xf7,0x03]
+
+; CHECK-INST: fmuls r22, r16
+; CHECK-INST: fmuls r19, r17
+; CHECK-INST: fmuls r21, r23
+; CHECK-INST: fmuls r23, r23
diff --git a/llvm/test/MC/AVR/inst-fmulsu.s b/llvm/test/MC/AVR/inst-fmulsu.s
index cd9e562eebf3d..bc9548421625e 100644
--- a/llvm/test/MC/AVR/inst-fmulsu.s
+++ b/llvm/test/MC/AVR/inst-fmulsu.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=mul -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=mul < %s | llvm-objdump -d --mattr=mul - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -12,3 +13,8 @@ foo:
 ; CHECK: fmulsu r19, r17                   ; encoding: [0xb9,0x03]
 ; CHECK: fmulsu r21, r23                   ; encoding: [0xdf,0x03]
 ; CHECK: fmulsu r23, r23                   ; encoding: [0xff,0x03]
+
+; CHECK-INST: fmulsu r22, r16
+; CHECK-INST: fmulsu r19, r17
+; CHECK-INST: fmulsu r21, r23
+; CHECK-INST: fmulsu r23, r23
diff --git a/llvm/test/MC/AVR/inst-inc.s b/llvm/test/MC/AVR/inst-inc.s
index 7300d0d2ec884..d5d1b1aec07c3 100644
--- a/llvm/test/MC/AVR/inst-inc.s
+++ b/llvm/test/MC/AVR/inst-inc.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr < %s | llvm-objdump -d - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -7,8 +8,19 @@ foo:
   inc r29
   inc r6
   inc r20
+  inc r0
+  inc r31
 
 ; CHECK: inc r12                  ; encoding: [0xc3,0x94]
 ; CHECK: inc r29                  ; encoding: [0xd3,0x95]
 ; CHECK: inc r6                   ; encoding: [0x63,0x94]
 ; CHECK: inc r20                  ; encoding: [0x43,0x95]
+; CHECK: inc r0                   ; encoding: [0x03,0x94]
+; CHECK: inc r31                  ; encoding: [0xf3,0x95]
+
+; CHECK-INST: inc r12
+; CHECK-INST: inc r29
+; CHECK-INST: inc r6
+; CHECK-INST: inc r20
+; CHECK-INST: inc r0
+; CHECK-INST: inc r31
diff --git a/llvm/test/MC/AVR/inst-lac.s b/llvm/test/MC/AVR/inst-lac.s
index da17572a587aa..e20998c74c134 100644
--- a/llvm/test/MC/AVR/inst-lac.s
+++ b/llvm/test/MC/AVR/inst-lac.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=rmw -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=rmw < %s | llvm-objdump -d --mattr=rmw - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -12,3 +13,8 @@ foo:
 ; CHECK: lac Z, r0                   ; encoding: [0x06,0x92]
 ; CHECK: lac Z, r31                  ; encoding: [0xf6,0x93]
 ; CHECK: lac Z, r3                   ; encoding: [0x36,0x92]
+
+; CHECK-INST: lac Z, r13
+; CHECK-INST: lac Z, r0
+; CHECK-INST: lac Z, r31
+; CHECK-INST: lac Z, r3
diff --git a/llvm/test/MC/AVR/inst-las.s b/llvm/test/MC/AVR/inst-las.s
index 3ee52f4b3c9e7..d8ff8d510df51 100644
--- a/llvm/test/MC/AVR/inst-las.s
+++ b/llvm/test/MC/AVR/inst-las.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=rmw -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=rmw < %s | llvm-objdump -d --mattr=rmw - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -12,3 +13,8 @@ foo:
 ; CHECK: las Z, r0                   ; encoding: [0x05,0x92]
 ; CHECK: las Z, r31                  ; encoding: [0xf5,0x93]
 ; CHECK: las Z, r3                   ; encoding: [0x35,0x92]
+
+; CHECK-INST: las Z, r13
+; CHECK-INST: las Z, r0
+; CHECK-INST: las Z, r31
+; CHECK-INST: las Z, r3
diff --git a/llvm/test/MC/AVR/inst-lat.s b/llvm/test/MC/AVR/inst-lat.s
index e10a2169c947c..ae001c0691403 100644
--- a/llvm/test/MC/AVR/inst-lat.s
+++ b/llvm/test/MC/AVR/inst-lat.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=rmw -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=rmw < %s | llvm-objdump -d --mattr=rmw - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -12,3 +13,8 @@ foo:
 ; CHECK: lat Z, r0                   ; encoding: [0x07,0x92]
 ; CHECK: lat Z, r31                  ; encoding: [0xf7,0x93]
 ; CHECK: lat Z, r3                   ; encoding: [0x37,0x92]
+
+; CHECK-INST: lat Z, r13
+; CHECK-INST: lat Z, r0
+; CHECK-INST: lat Z, r31
+; CHECK-INST: lat Z, r3
diff --git a/llvm/test/MC/AVR/inst-lpm.s b/llvm/test/MC/AVR/inst-lpm.s
index a0bdda756bad3..7ddacd4b0647a 100644
--- a/llvm/test/MC/AVR/inst-lpm.s
+++ b/llvm/test/MC/AVR/inst-lpm.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=lpm,lpmx -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=lpm,lpmx < %s | llvm-objdump -d --mattr=lpm,lpmx - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -10,6 +11,7 @@ foo:
 
   lpm r8, Z+
   lpm r0, Z+
+  lpm r31, Z+
 
 ; CHECK: lpm                  ; encoding: [0xc8,0x95]
 
@@ -18,3 +20,13 @@ foo:
 
 ; CHECK: lpm r8, Z+           ; encoding: [0x85,0x90]
 ; CHECK: lpm r0, Z+           ; encoding: [0x05,0x90]
+; CHECK: lpm r31, Z+          ; encoding: [0xf5,0x91]
+
+; CHECK-INST: lpm
+
+; CHECK-INST: lpm r3,  Z
+; CHECK-INST: lpm r23, Z
+
+; CHECK-INST: lpm r8, Z+
+; CHECK-INST: lpm r0, Z+
+; CHECK-INST: lpm r31, Z+
diff --git a/llvm/test/MC/AVR/inst-lsr.s b/llvm/test/MC/AVR/inst-lsr.s
index 3145eb80ee454..59b0174beede6 100644
--- a/llvm/test/MC/AVR/inst-lsr.s
+++ b/llvm/test/MC/AVR/inst-lsr.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr < %s | llvm-objdump -d - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -12,3 +13,8 @@ foo:
 ; CHECK: lsr r25                ; encoding: [0x96,0x95]
 ; CHECK: lsr r5                 ; encoding: [0x56,0x94]
 ; CHECK: lsr r0                 ; encoding: [0x06,0x94]
+
+; CHECK-INST: lsr r31
+; CHECK-INST: lsr r25
+; CHECK-INST: lsr r5
+; CHECK-INST: lsr r0
diff --git a/llvm/test/MC/AVR/inst-movw.s b/llvm/test/MC/AVR/inst-movw.s
index 6d63bca35d92f..b93b530e9a701 100644
--- a/llvm/test/MC/AVR/inst-movw.s
+++ b/llvm/test/MC/AVR/inst-movw.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=movw -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=movw < %s | llvm-objdump -d --mattr=movw - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -7,8 +8,25 @@ foo:
   movw r12, r16
   movw r20, r22
   movw r8,  r12
+  movw r0,  r0
+  movw r0,  r30
+  movw r30, r30
+  movw r30, r0
 
 ; CHECK: movw r10, r8            ; encoding: [0x54,0x01]
 ; CHECK: movw r12, r16           ; encoding: [0x68,0x01]
 ; CHECK: movw r20, r22           ; encoding: [0xab,0x01]
 ; CHECK: movw r8,  r12           ; encoding: [0x46,0x01]
+; CHECK: movw r0,  r0            ; encoding: [0x00,0x01]
+; CHECK: movw r0,  r30           ; encoding: [0x0f,0x01]
+; CHECK: movw r30, r30           ; encoding: [0xff,0x01]
+; CHECK: movw r30, r0            ; encoding: [0xf0,0x01]
+
+; CHECK-INST: movw r10, r8
+; CHECK-INST: movw r12, r16
+; CHECK-INST: movw r20, r22
+; CHECK-INST: movw r8,  r12
+; CHECK-INST: movw r0,  r0
+; CHECK-INST: movw r0,  r30
+; CHECK-INST: movw r30, r30
+; CHECK-INST: movw r30, r0
diff --git a/llvm/test/MC/AVR/inst-muls.s b/llvm/test/MC/AVR/inst-muls.s
index 55d20d71d1ad9..9ae3d15d8fa1e 100644
--- a/llvm/test/MC/AVR/inst-muls.s
+++ b/llvm/test/MC/AVR/inst-muls.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=mul -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=mul < %s | llvm-objdump -d --mattr=mul - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -7,8 +8,19 @@ foo:
   muls r19, r17
   muls r28, r31
   muls r31, r31
+  muls r16, r16
+  muls r16, r31
 
 ; CHECK: muls r22, r16                   ; encoding: [0x60,0x02]
 ; CHECK: muls r19, r17                   ; encoding: [0x31,0x02]
 ; CHECK: muls r28, r31                   ; encoding: [0xcf,0x02]
 ; CHECK: muls r31, r31                   ; encoding: [0xff,0x02]
+; CHECK: muls r16, r16                   ; encoding: [0x00,0x02]
+; CHECK: muls r16, r31                   ; encoding: [0x0f,0x02]
+
+; CHECK-INST: muls r22, r16
+; CHECK-INST: muls r19, r17
+; CHECK-INST: muls r28, r31
+; CHECK-INST: muls r31, r31
+; CHECK-INST: muls r16, r16
+; CHECK-INST: muls r16, r31
diff --git a/llvm/test/MC/AVR/inst-mulsu.s b/llvm/test/MC/AVR/inst-mulsu.s
index 93654e787bf01..a74322dbe137e 100644
--- a/llvm/test/MC/AVR/inst-mulsu.s
+++ b/llvm/test/MC/AVR/inst-mulsu.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=mul -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=mul < %s | llvm-objdump -d --mattr=mul - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -12,3 +13,8 @@ foo:
 ; CHECK: mulsu r19, r17                   ; encoding: [0x31,0x03]
 ; CHECK: mulsu r21, r23                   ; encoding: [0x57,0x03]
 ; CHECK: mulsu r23, r23                   ; encoding: [0x77,0x03]
+
+; CHECK-INST: mulsu r22, r16
+; CHECK-INST: mulsu r19, r17
+; CHECK-INST: mulsu r21, r23
+; CHECK-INST: mulsu r23, r23
diff --git a/llvm/test/MC/AVR/inst-neg.s b/llvm/test/MC/AVR/inst-neg.s
index c602cdd5a1c36..7f36a1aa39798 100644
--- a/llvm/test/MC/AVR/inst-neg.s
+++ b/llvm/test/MC/AVR/inst-neg.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr < %s | llvm-objdump -d - | FileCheck -check-prefix=CHECK-INST %s
 
 foo:
   neg r15
@@ -10,3 +11,8 @@ foo:
 ; CHECK: neg r1                   ; encoding: [0x11,0x94]
 ; CHECK: neg r22                  ; encoding: [0x61,0x95]
 ; CHECK: neg r31                  ; encoding: [0xf1,0x95]
+
+; CHECK-INST: neg r15
+; CHECK-INST: neg r1
+; CHECK-INST: neg r22
+; CHECK-INST: neg r31
diff --git a/llvm/test/MC/AVR/inst-pop.s b/llvm/test/MC/AVR/inst-pop.s
index 754d3a9c230b1..2e02fedc806da 100644
--- a/llvm/test/MC/AVR/inst-pop.s
+++ b/llvm/test/MC/AVR/inst-pop.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=sram -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=sram < %s | llvm-objdump -d --mattr=sram - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -12,3 +13,8 @@ foo:
 ; CHECK: pop r25                ; encoding: [0x9f,0x91]
 ; CHECK: pop r5                 ; encoding: [0x5f,0x90]
 ; CHECK: pop r0                 ; encoding: [0x0f,0x90]
+
+; CHECK-INST: pop r31
+; CHECK-INST: pop r25
+; CHECK-INST: pop r5
+; CHECK-INST: pop r0
diff --git a/llvm/test/MC/AVR/inst-push.s b/llvm/test/MC/AVR/inst-push.s
index ca8497ab54e80..d5601b84e3cd1 100644
--- a/llvm/test/MC/AVR/inst-push.s
+++ b/llvm/test/MC/AVR/inst-push.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=sram -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=sram < %s | llvm-objdump -d --mattr=sram - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -12,3 +13,8 @@ foo:
 ; CHECK: push r25                ; encoding: [0x9f,0x93]
 ; CHECK: push r5                 ; encoding: [0x5f,0x92]
 ; CHECK: push r0                 ; encoding: [0x0f,0x92]
+
+; CHECK-INST: push r31
+; CHECK-INST: push r25
+; CHECK-INST: push r5
+; CHECK-INST: push r0
diff --git a/llvm/test/MC/AVR/inst-ror.s b/llvm/test/MC/AVR/inst-ror.s
index 0d1e4d6c5a5bc..ca7a90f81f971 100644
--- a/llvm/test/MC/AVR/inst-ror.s
+++ b/llvm/test/MC/AVR/inst-ror.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr < %s | llvm-objdump -d - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -12,3 +13,8 @@ foo:
 ; CHECK: ror r25                ; encoding: [0x97,0x95]
 ; CHECK: ror r5                 ; encoding: [0x57,0x94]
 ; CHECK: ror r0                 ; encoding: [0x07,0x94]
+
+; CHECK-INST: ror r31
+; CHECK-INST: ror r25
+; CHECK-INST: ror r5
+; CHECK-INST: ror r0
diff --git a/llvm/test/MC/AVR/inst-sbiw.s b/llvm/test/MC/AVR/inst-sbiw.s
index cb0242841dff2..109d1fc74ce60 100644
--- a/llvm/test/MC/AVR/inst-sbiw.s
+++ b/llvm/test/MC/AVR/inst-sbiw.s
@@ -1,5 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=addsubiw -show-encoding < %s | FileCheck %s
-; RUNx: llvm-mc -filetype=obj -triple avr -mattr=addsubiw < %s | llvm-objdump -d --mattr=addsubiw - | FileCheck --check-prefix=CHECK-INST %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=addsubiw < %s | llvm-objdump -dr --mattr=addsubiw - | FileCheck --check-prefix=CHECK-INST %s
 
 
 foo:
@@ -34,15 +34,16 @@ foo:
                                        ;   fixup A - offset: 0, value: SYMBOL-1, kind: fixup_6_adiw
 
 ; CHECK-INST: sbiw r26, 54
-; CHECK-INST: sbiw X,   63
+; CHECK-INST: sbiw r26, 63
 
-; CHECK-INST: sbiw 28,  52
+; CHECK-INST: sbiw r28, 52
 ; CHECK-INST: sbiw r28, 0
 
 ; CHECK-INST: sbiw r30, 63
-; CHECK-INST: sbiw Z,   47
+; CHECK-INST: sbiw r30, 47
 
 ; CHECK-INST: sbiw r24, 1
 ; CHECK-INST: sbiw r24, 2
 
-; CHECK-INST: sbiw r24, SYMBOL-1
+; CHECK-INST: sbiw r24, 0
+; CHECK-INST:      R_AVR_6_ADIW SYMBOL-0x1
diff --git a/llvm/test/MC/AVR/inst-spm.s b/llvm/test/MC/AVR/inst-spm.s
index dbcb98a251391..3e1b8dd5774a4 100644
--- a/llvm/test/MC/AVR/inst-spm.s
+++ b/llvm/test/MC/AVR/inst-spm.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=spm,spmx -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=spm,spmx < %s | llvm-objdump -d --mattr=spm,spmx - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -8,3 +9,6 @@ foo:
 
 ; CHECK: spm                  ; encoding: [0xe8,0x95]
 ; CHECK: spm Z+               ; encoding: [0xf8,0x95]
+
+; CHECK-INST: spm
+; CHECK-INST: spm Z+
diff --git a/llvm/test/MC/AVR/inst-swap.s b/llvm/test/MC/AVR/inst-swap.s
index 4a5f1ea35b532..4a265639f70ac 100644
--- a/llvm/test/MC/AVR/inst-swap.s
+++ b/llvm/test/MC/AVR/inst-swap.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr < %s | llvm-objdump -d - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -12,3 +13,8 @@ foo:
 ; CHECK: swap r25                ; encoding: [0x92,0x95]
 ; CHECK: swap r5                 ; encoding: [0x52,0x94]
 ; CHECK: swap r0                 ; encoding: [0x02,0x94]
+
+; CHECK-INST: swap r31
+; CHECK-INST: swap r25
+; CHECK-INST: swap r5
+; CHECK-INST: swap r0
diff --git a/llvm/test/MC/AVR/inst-xch.s b/llvm/test/MC/AVR/inst-xch.s
index 05e512b8a2e56..36d30e2bc52bf 100644
--- a/llvm/test/MC/AVR/inst-xch.s
+++ b/llvm/test/MC/AVR/inst-xch.s
@@ -1,4 +1,5 @@
 ; RUN: llvm-mc -triple avr -mattr=rmw -show-encoding < %s | FileCheck %s
+; RUN: llvm-mc -filetype=obj -triple avr -mattr=rmw < %s | llvm-objdump -d --mattr=rmw - | FileCheck -check-prefix=CHECK-INST %s
 
 
 foo:
@@ -12,3 +13,8 @@ foo:
 ; CHECK: xch Z, r0                   ; encoding: [0x04,0x92]
 ; CHECK: xch Z, r31                  ; encoding: [0xf4,0x93]
 ; CHECK: xch Z, r3                   ; encoding: [0x34,0x92]
+
+; CHECK-INST: xch Z, r13
+; CHECK-INST: xch Z, r0
+; CHECK-INST: xch Z, r31
+; CHECK-INST: xch Z, r3
diff --git a/llvm/test/MC/AsmParser/directive_cgprofile.s b/llvm/test/MC/AsmParser/directive_cgprofile.s
index 1db93dcbb0336..b7bb82ed270d0 100644
--- a/llvm/test/MC/AsmParser/directive_cgprofile.s
+++ b/llvm/test/MC/AsmParser/directive_cgprofile.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple i386-unknown-unknown %s | FileCheck %s
-
+# RUN: llvm-mc -triple x86_64-pc-win32 %s | FileCheck %s
 	.cg_profile a, b, 32
 	.cg_profile freq, a, 11
 	.cg_profile freq, b, 20
diff --git a/llvm/test/MC/AsmParser/layout-interdependency.s b/llvm/test/MC/AsmParser/layout-interdependency.s
new file mode 100644
index 0000000000000..6e275e00d9ec7
--- /dev/null
+++ b/llvm/test/MC/AsmParser/layout-interdependency.s
@@ -0,0 +1,10 @@
+# RUN: not llvm-mc --filetype=obj %s -o /dev/null 2>&1 | FileCheck %s
+
+fct_end:
+
+# CHECK: layout-interdependency.s:[[#@LINE+1]]:7: error: expected assembly-time absolute expression
+.fill (data_start - fct_end), 1, 42
+# CHECK: layout-interdependency.s:[[#@LINE+1]]:7: error: expected assembly-time absolute expression
+.fill (fct_end - data_start), 1, 42
+
+data_start:
diff --git a/llvm/test/MC/COFF/cgprofile.s b/llvm/test/MC/COFF/cgprofile.s
new file mode 100644
index 0000000000000..a0c47a69c0692
--- /dev/null
+++ b/llvm/test/MC/COFF/cgprofile.s
@@ -0,0 +1,119 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s -o %t
+# RUN: llvm-readobj -S --symbols --sd --cg-profile %t | FileCheck %s
+
+  .section .test,"w"
+a:
+
+  .cg_profile a, b, 32
+  .cg_profile freq, a, 11
+  .cg_profile late, late2, 20
+  .cg_profile .L.local, b, 42
+
+	.globl late
+late:
+late2: .word 0
+late3:
+.L.local:
+
+# CHECK:      Name: .llvm.call-graph-profile
+# CHECK-NEXT: VirtualSize:
+# CHECK-NEXT: VirtualAddress:
+# CHECK-NEXT: RawDataSize: 48
+# CHECK-NEXT: PointerToRawData:
+# CHECK-NEXT: PointerToRelocations:
+# CHECK-NEXT: PointerToLineNumbers:
+# CHECK-NEXT: RelocationCount:
+# CHECK-NEXT: LineNumberCount:
+# CHECK-NEXT: Characteristics [ (0x100800)
+# CHECK-NEXT:   IMAGE_SCN_ALIGN_1BYTES (0x100000)
+# CHECK-NEXT:   IMAGE_SCN_LNK_REMOVE (0x800)
+# CHECK-NEXT: ]
+# CHECK-NEXT: SectionData (
+# CHECK-NEXT:   0000: 0A000000 0E000000 20000000 00000000
+# CHECK-NEXT:   0010: 11000000 0A000000 0B000000 00000000
+# CHECK-NEXT:   0020: 0B000000 0C000000 14000000 00000000
+# CHECK-NEXT: )
+
+# CHECK: Symbols [
+# CHECK:      Name: a
+# CHECK-NEXT: Value:
+# CHECK-NEXT: Section: .test
+# CHECK-NEXT: BaseType:
+# CHECK-NEXT: ComplexType:
+# CHECK-NEXT: StorageClass: Static
+# CHECK-NEXT: AuxSymbolCount:
+# CHECK:      Name: late
+# CHECK-NEXT: Value:
+# CHECK-NEXT: Section: .test
+# CHECK-NEXT: BaseType:
+# CHECK-NEXT: ComplexType:
+# CHECK-NEXT: StorageClass: External
+# CHECK-NEXT: AuxSymbolCount:
+# CHECK:      Name: late2
+# CHECK-NEXT: Value:
+# CHECK-NEXT: Section: .test
+# CHECK-NEXT: BaseType:
+# CHECK-NEXT: ComplexType:
+# CHECK-NEXT: StorageClass: Static
+# CHECK-NEXT: AuxSymbolCount:
+# CHECK:      Name: late3
+# CHECK-NEXT: Value:
+# CHECK-NEXT: Section: .test
+# CHECK-NEXT: BaseType:
+# CHECK-NEXT: ComplexType:
+# CHECK-NEXT: StorageClass: Static
+# CHECK-NEXT: AuxSymbolCount:
+# CHECK:      Name: b
+# CHECK-NEXT: Value:
+# CHECK-NEXT: Section: IMAGE_SYM_UNDEFINED
+# CHECK-NEXT: BaseType:
+# CHECK-NEXT: ComplexType:
+# CHECK-NEXT: StorageClass: WeakExternal
+# CHECK-NEXT: AuxSymbolCount: 1
+# CHECK-NEXT: AuxWeakExternal {
+# CHECK-NEXT:   Linked: .weak.b.default.late
+# CHECK-NEXT:   Search: Alias
+# CHECK-NEXT: }
+# CHECK:      Name: .weak.b.default.late
+# CHECK-NEXT: Value:
+# CHECK-NEXT: Section: IMAGE_SYM_ABSOLUTE
+# CHECK-NEXT: BaseType:
+# CHECK-NEXT: ComplexType:
+# CHECK-NEXT: StorageClass: External
+# CHECK-NEXT: AuxSymbolCount: 0
+# CHECK:      Name: freq
+# CHECK-NEXT: Value:
+# CHECK-NEXT: Section: IMAGE_SYM_UNDEFINED
+# CHECK-NEXT: BaseType:
+# CHECK-NEXT: ComplexType:
+# CHECK-NEXT: StorageClass: WeakExternal
+# CHECK-NEXT: AuxSymbolCount: 1
+# CHECK-NEXT: AuxWeakExternal {
+# CHECK-NEXT:   Linked: .weak.freq.default.late
+# CHECK-NEXT:   Search: Alias
+# CHECK-NEXT: }
+# CHECK:      Name: .weak.freq.default.late
+# CHECK-NEXT: Value:
+# CHECK-NEXT: Section: IMAGE_SYM_ABSOLUTE
+# CHECK-NEXT: BaseType:
+# CHECK-NEXT: ComplexType:
+# CHECK-NEXT: StorageClass: External
+# CHECK-NEXT: AuxSymbolCount: 0
+
+# CHECK:      CGProfile [
+# CHECK-NEXT:   CGProfileEntry {
+# CHECK-NEXT:     From: a
+# CHECK-NEXT:     To: b
+# CHECK-NEXT:     Weight: 32
+# CHECK-NEXT:   }
+# CHECK-NEXT:   CGProfileEntry {
+# CHECK-NEXT:     From: freq
+# CHECK-NEXT:     To: a
+# CHECK-NEXT:     Weight: 11
+# CHECK-NEXT:   }
+# CHECK-NEXT:   CGProfileEntry {
+# CHECK-NEXT:     From: late
+# CHECK-NEXT:     To: late2
+# CHECK-NEXT:     Weight: 20
+# CHECK-NEXT:   }
+# CHECK-NEXT: ]
diff --git a/llvm/test/MC/COFF/section.s b/llvm/test/MC/COFF/section.s
index b7b5422babdb4..5116d179f16d9 100644
--- a/llvm/test/MC/COFF/section.s
+++ b/llvm/test/MC/COFF/section.s
@@ -190,5 +190,11 @@
 // CHECK-NEXT:     ]
 // CHECK:        }
 
+/// The section name can be quoted.
+.section "@#$-{","n"
+// CHECK:        Section {
+// CHECK-NEXT:     Number:
+// CHECK-NEXT:     Name: @#$-{
 
+// CHECK-NOT:    Section {
 // CHECK:      ]
diff --git a/llvm/test/MC/Disassembler/AArch64/armv8.3a-rcpc.txt b/llvm/test/MC/Disassembler/AArch64/armv8.3a-rcpc.txt
index 3bd935b3f794c..8b0aac5269990 100644
--- a/llvm/test/MC/Disassembler/AArch64/armv8.3a-rcpc.txt
+++ b/llvm/test/MC/Disassembler/AArch64/armv8.3a-rcpc.txt
@@ -4,6 +4,9 @@
 # RUN: llvm-mc -triple aarch64-none-linux-gnu -mcpu=cortex-a65 --disassemble < %s | FileCheck %s
 # RUN: llvm-mc -triple aarch64-none-linux-gnu -mcpu=cortex-a65ae --disassemble < %s | FileCheck %s
 # RUN: llvm-mc -triple aarch64-none-linux-gnu -mcpu=cortex-a75 --disassemble < %s | FileCheck %s
+# RUN: llvm-mc -triple aarch64-none-linux-gnu -mcpu=cortex-a77 --disassemble < %s | FileCheck %s
+# RUN: llvm-mc -triple aarch64-none-linux-gnu -mcpu=cortex-a78 --disassemble < %s | FileCheck %s
+# RUN: llvm-mc -triple aarch64-none-linux-gnu -mcpu=cortex-x1 --disassemble < %s | FileCheck %s
 # RUN: llvm-mc -triple aarch64-none-linux-gnu -mcpu=neoverse-e1 --disassemble < %s | FileCheck %s
 # RUN: llvm-mc -triple aarch64-none-linux-gnu -mcpu=neoverse-n1 --disassemble < %s | FileCheck %s
 
diff --git a/llvm/test/MC/Disassembler/AArch64/armv8.3a-signed-pointer.txt b/llvm/test/MC/Disassembler/AArch64/armv8.3a-signed-pointer.txt
index 7215d086c693c..110c6681cd7f3 100644
--- a/llvm/test/MC/Disassembler/AArch64/armv8.3a-signed-pointer.txt
+++ b/llvm/test/MC/Disassembler/AArch64/armv8.3a-signed-pointer.txt
@@ -1,4 +1,13 @@
-# RUN: llvm-mc -triple aarch64-none-linux-gnu -mattr=+v8.3a --disassemble < %s | FileCheck %s
+# RUN: llvm-mc -triple aarch64-none-linux-gnu -mattr=+v8.3a --disassemble < %s 2>&1 | FileCheck %s
+
+# CHECK: warning: potentially undefined instruction encoding
+# CHECK-NEXT: [0x00,0x0c,0x20,0xf8]
+# CHECK: warning: potentially undefined instruction encoding
+# CHECK-NEXT: [0x00,0x0c,0xa0,0xf8]
+# CHECK: ldraa x0, [x0, #0]!
+# CHECK: ldrab x0, [x0, #0]!
+[0x00,0x0c,0x20,0xf8]
+[0x00,0x0c,0xa0,0xf8]
 
 # CHECK: paciasp
 # CHECK: autiasp
diff --git a/llvm/test/MC/Disassembler/PowerPC/p10insts.txt b/llvm/test/MC/Disassembler/PowerPC/p10insts.txt
index bd83c5daca1f0..e2bfee65eedae 100644
--- a/llvm/test/MC/Disassembler/PowerPC/p10insts.txt
+++ b/llvm/test/MC/Disassembler/PowerPC/p10insts.txt
@@ -1,20 +1,4 @@
-# RUN: llvm-mc --disassemble %s -triple powerpc64-unknown-linux-gnu \
-# RUN:   -mcpu=pwr10 | FileCheck %s
-
-# CHECK: vpdepd 1, 2, 0
-0x10 0x22 0x05 0xcd
-
-# CHECK: vpextd 1, 2, 0
-0x10 0x22 0x05 0x8d
-
-# CHECK: pdepd 1, 2, 4
-0x7c 0x41 0x21 0x38
-
-# CHECK: pextd 1, 2, 4
-0x7c 0x41 0x21 0x78
-
-# CHECK: vclrlb 1, 4, 3
-0x10 0x24 0x19 0x8d
-
-# CHECK: vclrrb 1, 4, 3
-0x10 0x24 0x19 0xcd
+# RUN: llvm-mc --disassemble %s -triple powerpc64-unknown-linux-gnu -mcpu=pwr10
+##############################################################################
+# CONTENT MOVED TO ppc64-encoding-ISA31.txt, please do NOT add to this file
+##############################################################################
diff --git a/llvm/test/MC/Disassembler/PowerPC/future-invalid.txt b/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31-invalid.txt
similarity index 98%
rename from llvm/test/MC/Disassembler/PowerPC/future-invalid.txt
rename to llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31-invalid.txt
index af458f8ddfce6..3b2e4e2a0161e 100644
--- a/llvm/test/MC/Disassembler/PowerPC/future-invalid.txt
+++ b/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31-invalid.txt
@@ -1,4 +1,4 @@
-# RUN: llvm-mc --disassemble %s -mcpu=future -triple \
+# RUN: llvm-mc --disassemble %s -mcpu=pwr10 -triple \
 # RUN:   powerpc64-unknown-linux-gnu < %s 2>&1 | FileCheck %s
 
 # paddi 1, 2, 8589934591, 1. However, RA is not zero with R=1
diff --git a/llvm/test/MC/Disassembler/PowerPC/futureinsts.txt b/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31.txt
similarity index 52%
rename from llvm/test/MC/Disassembler/PowerPC/futureinsts.txt
rename to llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31.txt
index 26ca8cc843aca..f8d310fa7e147 100644
--- a/llvm/test/MC/Disassembler/PowerPC/futureinsts.txt
+++ b/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31.txt
@@ -1,5 +1,5 @@
 # RUN: llvm-mc --disassemble %s -triple powerpc64-unknown-linux-gnu \
-# RUN:   -mcpu=future | FileCheck %s
+# RUN:   -mcpu=pwr10 | FileCheck %s
 
 # CHECK: paddi 1, 2, 8589934591, 0
 0x06 0x01 0xff 0xff 0x38 0x22 0xff 0xff
@@ -133,3 +133,172 @@
 # CHECK: pld 1, 8589934591(0), 1
 0x04 0x11 0xff 0xff 0xe4 0x20 0xff 0xff
 
+# CHECK: vpdepd 1, 2, 0
+0x10 0x22 0x05 0xcd
+
+# CHECK: vpextd 1, 2, 0
+0x10 0x22 0x05 0x8d
+
+# CHECK: pdepd 1, 2, 4
+0x7c 0x41 0x21 0x38
+
+# CHECK: pextd 1, 2, 4
+0x7c 0x41 0x21 0x78
+
+# CHECK: vcfuged 1, 2, 4
+0x10 0x22 0x25 0x4d
+
+# CHECK: cfuged 1, 2, 4
+0x7c 0x41 0x21 0xb8
+
+# CHECK: vgnb 1, 2, 2
+0x10 0x22 0x14 0xcc
+
+# CHECK: xxeval 32, 1, 2, 3, 2
+0x05 0x00 0x00 0x02 0x88 0x01 0x10 0xd1
+
+# CHECK: vclzdm 1, 2, 3
+0x10 0x22 0x1f 0x84
+
+# CHECK: vctzdm 1, 2, 3
+0x10 0x22 0x1f 0xc4
+
+# CHECK: cntlzdm 1, 3, 2
+0x7c 0x61 0x10 0x76
+
+# CHECK: cnttzdm 1, 3, 2
+0x7c 0x61 0x14 0x76
+
+# CHECK xxgenpcvbm 0, 1, 2
+0xf0 0x02 0x0f 0x28
+
+# CHECK xxgenpcvhm 0, 1, 2
+0xf0 0x02 0x0f 0x2a
+
+# CHECK xxgenpcvwm 0, 1, 2
+0xf0 0x02 0x0f 0x68
+
+# CHECK xxgenpcvdm 0, 1, 2
+0xf0 0x02 0x0f 0x6a
+
+# CHECK: vclrlb 1, 4, 3
+0x10 0x24 0x19 0x8d
+
+# CHECK: vclrrb 1, 4, 3
+0x10 0x24 0x19 0xcd
+
+# Boundary conditions of 8RR_DForm_IMM32_XT6's immediates
+# CHECK: xxspltiw 63, 4294901760
+0x05 0x00 0xff 0xff 0x83 0xe7 0x00 0x00
+
+# CHECK: xxspltiw 63, 65535
+0x05 0x00 0x00 0x00 0x83 0xe7 0xff 0xff
+
+# CHECK: xxspltiw 63, 4294967295
+0x05 0x00 0xff 0xff 0x83 0xe7 0xff 0xff
+
+# CHECK: xxspltidp 63, 4294967295
+0x05 0x00 0xff 0xff 0x83 0xe5 0xff 0xff
+
+# Boundary conditions of 8RR_DForm_IMM32_XT6_IX's immediates
+# CHECK: xxsplti32dx 63, 1, 4294901760
+0x05 0x00 0xff 0xff 0x83 0xe3 0x00 0x00
+
+# CHECK: xxsplti32dx 63, 1, 65535
+0x05 0x00 0x00 0x00 0x83 0xe3 0xff 0xff
+
+# CHECK: xxsplti32dx 63, 1, 4294967295
+0x05 0x00 0xff 0xff 0x83 0xe3 0xff 0xff
+
+# CHECK: xxpermx 6, 63, 21, 34, 2
+0x05 0x00 0x00 0x02 0x88 0xdf 0xa8 0x8c
+
+# CHECK: xxblendvb 6, 63, 21, 34
+0x05 0x00 0x00 0x00 0x84 0xdf 0xa8 0x8c
+
+# CHECK: xxblendvh 6, 63, 21, 34
+0x05 0x00 0x00 0x00 0x84 0xdf 0xa8 0x9c
+
+# CHECK: xxblendvw 6, 63, 21, 34
+0x05 0x00 0x00 0x00 0x84 0xdf 0xa8 0xac
+
+# CHECK: xxblendvd 6, 63, 21, 34
+0x05 0x00 0x00 0x00 0x84 0xdf 0xa8 0xbc
+
+# CHECK: vsldbi 2, 3, 4, 5
+0x10 0x43 0x21 0x56
+
+# CHECK: vsrdbi 2, 3, 4, 5
+0x10 0x43 0x23 0x56
+
+# CHECK: vinsw 2, 3, 12
+0x10 0x4c 0x18 0xcf
+
+# CHECK: vinsd 2, 3, 12
+0x10 0x4c 0x19 0xcf
+
+# CHECK: vinsbvlx 1, 3, 5
+0x10 0x23 0x28 0x0f
+
+# CHECK: vinsbvrx 1, 3, 5
+0x10 0x23 0x29 0x0f
+
+# CHECK: vinshvlx 1, 3, 5
+0x10 0x23 0x28 0x4f
+
+# CHECK: vinshvrx 1, 3, 5
+0x10 0x23 0x29 0x4f
+
+# CHECK: vinswvlx 1, 3, 5
+0x10 0x23 0x28 0x8f
+
+# CHECK: vinswvrx 1, 3, 5
+0x10 0x23 0x29 0x8f
+
+# CHECK: vinsblx 1, 2, 3
+0x10 0x22 0x1a 0x0f
+
+# CHECK: vinsbrx 1, 2, 3
+0x10 0x22 0x1b 0x0f
+
+# CHECK: vinshlx 1, 2, 3
+0x10 0x22 0x1a 0x4f
+
+# CHECK: vinshrx 1, 2, 3
+0x10 0x22 0x1b 0x4f
+
+# CHECK: vinswlx 1, 2, 3
+0x10 0x22 0x1a 0x8f
+
+# CHECK: vinswrx 1, 2, 3
+0x10 0x22 0x1b 0x8f
+
+# CHECK: vinsdlx 1, 2, 3
+0x10 0x22 0x1a 0xcf
+
+# CHECK: vinsdrx 1, 2, 3
+0x10 0x22 0x1b 0xcf
+
+# CHECK: lxvrbx 32, 1, 2
+0x7c 0x01 0x10 0x1b
+
+# CHECK: lxvrhx 33, 1, 2
+0x7c 0x21 0x10 0x5b
+
+# CHECK: lxvrdx 34, 1, 2
+0x7c 0x41 0x10 0xdb
+
+# CHECK: lxvrwx 35, 1, 2
+0x7c 0x61 0x10 0x9b
+
+# CHECK: stxvrbx 32, 3, 1
+0x7c 0x03 0x09 0x1b
+
+# CHECK: stxvrhx 33, 3, 1
+0x7c 0x23 0x09 0x5b
+
+# CHECK: stxvrwx 34, 3, 1
+0x7c 0x43 0x09 0x9b
+
+# CHECK: stxvrdx 35, 3, 1
+0x7c 0x63 0x09 0xdb
diff --git a/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-bf16-att.txt b/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-bf16-att.txt
new file mode 100644
index 0000000000000..e6725de1dc17f
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-bf16-att.txt
@@ -0,0 +1,25 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64-apple-darwin9 | FileCheck %s
+
+# CHECK: tdpbf16ps %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x5a,0x5c,0xf5
+
+# CHECK: tdpbf16ps %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x72,0x5c,0xda
+
+# CHECK: tdpbf16ps %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x5a,0x5c,0xf5
+
+# CHECK: tdpbf16ps %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x72,0x5c,0xda
+
+# CHECK: tdpbf16ps %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x5a,0x5c,0xf5
+
+# CHECK: tdpbf16ps %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x72,0x5c,0xda
+
+# CHECK: tdpbf16ps %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x5a,0x5c,0xf5
+
+# CHECK: tdpbf16ps %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x72,0x5c,0xda
diff --git a/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-bf16-intel.txt b/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-bf16-intel.txt
new file mode 100644
index 0000000000000..8aa22cc0010a0
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-bf16-intel.txt
@@ -0,0 +1,25 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s
+
+# CHECK: tdpbf16ps tmm6, tmm5, tmm4
+0xc4,0xe2,0x5a,0x5c,0xf5
+
+# CHECK: tdpbf16ps tmm3, tmm2, tmm1
+0xc4,0xe2,0x72,0x5c,0xda
+
+# CHECK: tdpbf16ps tmm6, tmm5, tmm4
+0xc4,0xe2,0x5a,0x5c,0xf5
+
+# CHECK: tdpbf16ps tmm3, tmm2, tmm1
+0xc4,0xe2,0x72,0x5c,0xda
+
+# CHECK: tdpbf16ps tmm6, tmm5, tmm4
+0xc4,0xe2,0x5a,0x5c,0xf5
+
+# CHECK: tdpbf16ps tmm3, tmm2, tmm1
+0xc4,0xe2,0x72,0x5c,0xda
+
+# CHECK: tdpbf16ps tmm6, tmm5, tmm4
+0xc4,0xe2,0x5a,0x5c,0xf5
+
+# CHECK: tdpbf16ps tmm3, tmm2, tmm1
+0xc4,0xe2,0x72,0x5c,0xda
diff --git a/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-error.txt b/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-error.txt
new file mode 100644
index 0000000000000..b5a1bb8db2af9
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-error.txt
@@ -0,0 +1,4 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64 2>&1 | FileCheck %s
+
+# CHECK: invalid instruction encoding
+0xc4,0xe2,0x1a,0x5c,0xf5
diff --git a/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-int8-att.txt b/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-int8-att.txt
new file mode 100644
index 0000000000000..a6b55313c601a
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-int8-att.txt
@@ -0,0 +1,97 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64-apple-darwin9 | FileCheck %s
+
+# CHECK: tdpbssd %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x5b,0x5e,0xf5
+
+# CHECK: tdpbssd %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x73,0x5e,0xda
+
+# CHECK: tdpbsud %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x5a,0x5e,0xf5
+
+# CHECK: tdpbsud %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x72,0x5e,0xda
+
+# CHECK: tdpbusd %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x59,0x5e,0xf5
+
+# CHECK: tdpbusd %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x71,0x5e,0xda
+
+# CHECK: tdpbuud %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x58,0x5e,0xf5
+
+# CHECK: tdpbuud %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x70,0x5e,0xda
+
+# CHECK: tdpbssd %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x5b,0x5e,0xf5
+
+# CHECK: tdpbssd %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x73,0x5e,0xda
+
+# CHECK: tdpbsud %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x5a,0x5e,0xf5
+
+# CHECK: tdpbsud %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x72,0x5e,0xda
+
+# CHECK: tdpbusd %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x59,0x5e,0xf5
+
+# CHECK: tdpbusd %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x71,0x5e,0xda
+
+# CHECK: tdpbuud %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x58,0x5e,0xf5
+
+# CHECK: tdpbuud %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x70,0x5e,0xda
+
+# CHECK: tdpbssd %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x5b,0x5e,0xf5
+
+# CHECK: tdpbssd %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x73,0x5e,0xda
+
+# CHECK: tdpbsud %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x5a,0x5e,0xf5
+
+# CHECK: tdpbsud %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x72,0x5e,0xda
+
+# CHECK: tdpbusd %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x59,0x5e,0xf5
+
+# CHECK: tdpbusd %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x71,0x5e,0xda
+
+# CHECK: tdpbuud %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x58,0x5e,0xf5
+
+# CHECK: tdpbuud %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x70,0x5e,0xda
+
+# CHECK: tdpbssd %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x5b,0x5e,0xf5
+
+# CHECK: tdpbssd %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x73,0x5e,0xda
+
+# CHECK: tdpbsud %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x5a,0x5e,0xf5
+
+# CHECK: tdpbsud %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x72,0x5e,0xda
+
+# CHECK: tdpbusd %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x59,0x5e,0xf5
+
+# CHECK: tdpbusd %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x71,0x5e,0xda
+
+# CHECK: tdpbuud %tmm4, %tmm5, %tmm6
+0xc4,0xe2,0x58,0x5e,0xf5
+
+# CHECK: tdpbuud %tmm1, %tmm2, %tmm3
+0xc4,0xe2,0x70,0x5e,0xda
diff --git a/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-int8-intel.txt b/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-int8-intel.txt
new file mode 100644
index 0000000000000..8ef714b1ac8da
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-int8-intel.txt
@@ -0,0 +1,97 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s
+
+# CHECK: tdpbssd tmm6, tmm5, tmm4
+0xc4,0xe2,0x5b,0x5e,0xf5
+
+# CHECK: tdpbssd tmm3, tmm2, tmm1
+0xc4,0xe2,0x73,0x5e,0xda
+
+# CHECK: tdpbsud tmm6, tmm5, tmm4
+0xc4,0xe2,0x5a,0x5e,0xf5
+
+# CHECK: tdpbsud tmm3, tmm2, tmm1
+0xc4,0xe2,0x72,0x5e,0xda
+
+# CHECK: tdpbusd tmm6, tmm5, tmm4
+0xc4,0xe2,0x59,0x5e,0xf5
+
+# CHECK: tdpbusd tmm3, tmm2, tmm1
+0xc4,0xe2,0x71,0x5e,0xda
+
+# CHECK: tdpbuud tmm6, tmm5, tmm4
+0xc4,0xe2,0x58,0x5e,0xf5
+
+# CHECK: tdpbuud tmm3, tmm2, tmm1
+0xc4,0xe2,0x70,0x5e,0xda
+
+# CHECK: tdpbssd tmm6, tmm5, tmm4
+0xc4,0xe2,0x5b,0x5e,0xf5
+
+# CHECK: tdpbssd tmm3, tmm2, tmm1
+0xc4,0xe2,0x73,0x5e,0xda
+
+# CHECK: tdpbsud tmm6, tmm5, tmm4
+0xc4,0xe2,0x5a,0x5e,0xf5
+
+# CHECK: tdpbsud tmm3, tmm2, tmm1
+0xc4,0xe2,0x72,0x5e,0xda
+
+# CHECK: tdpbusd tmm6, tmm5, tmm4
+0xc4,0xe2,0x59,0x5e,0xf5
+
+# CHECK: tdpbusd tmm3, tmm2, tmm1
+0xc4,0xe2,0x71,0x5e,0xda
+
+# CHECK: tdpbuud tmm6, tmm5, tmm4
+0xc4,0xe2,0x58,0x5e,0xf5
+
+# CHECK: tdpbuud tmm3, tmm2, tmm1
+0xc4,0xe2,0x70,0x5e,0xda
+
+# CHECK: tdpbssd tmm6, tmm5, tmm4
+0xc4,0xe2,0x5b,0x5e,0xf5
+
+# CHECK: tdpbssd tmm3, tmm2, tmm1
+0xc4,0xe2,0x73,0x5e,0xda
+
+# CHECK: tdpbsud tmm6, tmm5, tmm4
+0xc4,0xe2,0x5a,0x5e,0xf5
+
+# CHECK: tdpbsud tmm3, tmm2, tmm1
+0xc4,0xe2,0x72,0x5e,0xda
+
+# CHECK: tdpbusd tmm6, tmm5, tmm4
+0xc4,0xe2,0x59,0x5e,0xf5
+
+# CHECK: tdpbusd tmm3, tmm2, tmm1
+0xc4,0xe2,0x71,0x5e,0xda
+
+# CHECK: tdpbuud tmm6, tmm5, tmm4
+0xc4,0xe2,0x58,0x5e,0xf5
+
+# CHECK: tdpbuud tmm3, tmm2, tmm1
+0xc4,0xe2,0x70,0x5e,0xda
+
+# CHECK: tdpbssd tmm6, tmm5, tmm4
+0xc4,0xe2,0x5b,0x5e,0xf5
+
+# CHECK: tdpbssd tmm3, tmm2, tmm1
+0xc4,0xe2,0x73,0x5e,0xda
+
+# CHECK: tdpbsud tmm6, tmm5, tmm4
+0xc4,0xe2,0x5a,0x5e,0xf5
+
+# CHECK: tdpbsud tmm3, tmm2, tmm1
+0xc4,0xe2,0x72,0x5e,0xda
+
+# CHECK: tdpbusd tmm6, tmm5, tmm4
+0xc4,0xe2,0x59,0x5e,0xf5
+
+# CHECK: tdpbusd tmm3, tmm2, tmm1
+0xc4,0xe2,0x71,0x5e,0xda
+
+# CHECK: tdpbuud tmm6, tmm5, tmm4
+0xc4,0xe2,0x58,0x5e,0xf5
+
+# CHECK: tdpbuud tmm3, tmm2, tmm1
+0xc4,0xe2,0x70,0x5e,0xda
diff --git a/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-tile-att.txt b/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-tile-att.txt
new file mode 100644
index 0000000000000..2bb5c12fd1b73
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-tile-att.txt
@@ -0,0 +1,145 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64-apple-darwin9 | FileCheck %s
+
+# CHECK: tilerelease
+0xc4,0xe2,0x78,0x49,0xc0
+
+# CHECK: tilezero %tmm6
+0xc4,0xe2,0x7b,0x49,0xf0
+
+# CHECK: tilezero %tmm3
+0xc4,0xe2,0x7b,0x49,0xd8
+
+# CHECK: tilezero %tmm6
+0xc4,0xe2,0x7b,0x49,0xf0
+
+# CHECK: tilezero %tmm3
+0xc4,0xe2,0x7b,0x49,0xd8
+
+# CHECK: ldtilecfg  268435456(%rbp,%r14,8)
+0xc4,0xa2,0x78,0x49,0x84,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: ldtilecfg  291(%r8,%rax,4)
+0xc4,0xc2,0x78,0x49,0x84,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: ldtilecfg  (%rip)
+0xc4,0xe2,0x78,0x49,0x05,0x00,0x00,0x00,0x00
+
+# CHECK: ldtilecfg  -2048(,%rbp,2)
+0xc4,0xe2,0x78,0x49,0x04,0x6d,0x00,0xf8,0xff,0xff
+
+# CHECK: sttilecfg  268435456(%rbp,%r14,8)
+0xc4,0xa2,0x79,0x49,0x84,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: sttilecfg  291(%r8,%rax,4)
+0xc4,0xc2,0x79,0x49,0x84,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: sttilecfg  (%rip)
+0xc4,0xe2,0x79,0x49,0x05,0x00,0x00,0x00,0x00
+
+# CHECK: sttilecfg  -2048(,%rbp,2)
+0xc4,0xe2,0x79,0x49,0x04,0x6d,0x00,0xf8,0xff,0xff
+
+# CHECK: tileloadd 268435456(%rbp,%r14,8), %tmm6
+0xc4,0xa2,0x7b,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: tileloadd 291(%r8,%rax,4), %tmm3
+0xc4,0xc2,0x7b,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: tileloadd -32(,%rbp,2), %tmm3
+0xc4,0xe2,0x7b,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff
+
+# CHECK: tileloadd 64(%rbx), %tmm4
+0xc4,0xe2,0x7b,0x4b,0x64,0x23,0x40
+
+# CHECK: tileloaddt1 268435456(%rbp,%r14,8), %tmm6
+0xc4,0xa2,0x79,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: tileloaddt1 291(%r8,%rax,4), %tmm3
+0xc4,0xc2,0x79,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: tileloaddt1 -32(,%rbp,2), %tmm3
+0xc4,0xe2,0x79,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff
+
+# CHECK: tileloaddt1 16(%rbp), %tmm6
+0xc4,0xe2,0x79,0x4b,0x74,0x25,0x10
+
+# CHECK: tilerelease
+0xc4,0xe2,0x78,0x49,0xc0
+
+# CHECK: tilestored %tmm6, 268435456(%rbp,%r14,8)
+0xc4,0xa2,0x7a,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: tilestored %tmm3, 291(%r8,%rax,4)
+0xc4,0xc2,0x7a,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: tilestored %tmm3, -32(,%rbp,2)
+0xc4,0xe2,0x7a,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff
+
+# CHECK: tilezero %tmm6
+0xc4,0xe2,0x7b,0x49,0xf0
+
+# CHECK: tilezero %tmm3
+0xc4,0xe2,0x7b,0x49,0xd8
+
+# CHECK: ldtilecfg  268435456(%rbp,%r14,8)
+0xc4,0xa2,0x78,0x49,0x84,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: ldtilecfg  291(%r8,%rax,4)
+0xc4,0xc2,0x78,0x49,0x84,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: ldtilecfg  (%rip)
+0xc4,0xe2,0x78,0x49,0x05,0x00,0x00,0x00,0x00
+
+# CHECK: ldtilecfg  -2048(,%rbp,2)
+0xc4,0xe2,0x78,0x49,0x04,0x6d,0x00,0xf8,0xff,0xff
+
+# CHECK: sttilecfg  268435456(%rbp,%r14,8)
+0xc4,0xa2,0x79,0x49,0x84,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: sttilecfg  291(%r8,%rax,4)
+0xc4,0xc2,0x79,0x49,0x84,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: sttilecfg  (%rip)
+0xc4,0xe2,0x79,0x49,0x05,0x00,0x00,0x00,0x00
+
+# CHECK: sttilecfg  -2048(,%rbp,2)
+0xc4,0xe2,0x79,0x49,0x04,0x6d,0x00,0xf8,0xff,0xff
+
+# CHECK: tileloadd 268435456(%rbp,%r14,8), %tmm6
+0xc4,0xa2,0x7b,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: tileloadd 291(%r8,%rax,4), %tmm3
+0xc4,0xc2,0x7b,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: tileloadd -32(,%rbp,2), %tmm3
+0xc4,0xe2,0x7b,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff
+
+# CHECK: tileloaddt1 268435456(%rbp,%r14,8), %tmm6
+0xc4,0xa2,0x79,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: tileloaddt1 291(%r8,%rax,4), %tmm3
+0xc4,0xc2,0x79,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: tileloaddt1 -32(,%rbp,2), %tmm3
+0xc4,0xe2,0x79,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff
+
+# CHECK: tilerelease
+0xc4,0xe2,0x78,0x49,0xc0
+
+# CHECK: tilestored %tmm6, 268435456(%rbp,%r14,8)
+0xc4,0xa2,0x7a,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: tilestored %tmm3, 291(%r8,%rax,4)
+0xc4,0xc2,0x7a,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: tilestored %tmm3, -32(,%rbp,2)
+0xc4,0xe2,0x7a,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff
+
+# CHECK: tilestored %tmm3, (%r8)
+0xc4,0xc2,0x7a,0x4b,0x1c,0x20
+
+# CHECK: tilezero %tmm6
+0xc4,0xe2,0x7b,0x49,0xf0
+
+# CHECK: tilezero %tmm3
+0xc4,0xe2,0x7b,0x49,0xd8
diff --git a/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-tile-intel.txt b/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-tile-intel.txt
new file mode 100644
index 0000000000000..2893f3d5a09ea
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/AMX/x86-64-amx-tile-intel.txt
@@ -0,0 +1,148 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s
+
+# CHECK: tilerelease
+0xc4,0xe2,0x78,0x49,0xc0
+
+# CHECK: tilezero tmm6
+0xc4,0xe2,0x7b,0x49,0xf0
+
+# CHECK: tilezero tmm3
+0xc4,0xe2,0x7b,0x49,0xd8
+
+# CHECK: tilerelease
+0xc4,0xe2,0x78,0x49,0xc0
+
+# CHECK: tilezero tmm6
+0xc4,0xe2,0x7b,0x49,0xf0
+
+# CHECK: tilezero tmm3
+0xc4,0xe2,0x7b,0x49,0xd8
+
+# CHECK: ldtilecfg [rbp + 8*r14 + 268435456]
+0xc4,0xa2,0x78,0x49,0x84,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: ldtilecfg [r8 + 4*rax + 291]
+0xc4,0xc2,0x78,0x49,0x84,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: ldtilecfg [rip]
+0xc4,0xe2,0x78,0x49,0x05,0x00,0x00,0x00,0x00
+
+# CHECK: ldtilecfg [2*rbp - 2048]
+0xc4,0xe2,0x78,0x49,0x04,0x6d,0x00,0xf8,0xff,0xff
+
+# CHECK: sttilecfg [rbp + 8*r14 + 268435456]
+0xc4,0xa2,0x79,0x49,0x84,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: sttilecfg [r8 + 4*rax + 291]
+0xc4,0xc2,0x79,0x49,0x84,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: sttilecfg [rip]
+0xc4,0xe2,0x79,0x49,0x05,0x00,0x00,0x00,0x00
+
+# CHECK: sttilecfg [2*rbp - 2048]
+0xc4,0xe2,0x79,0x49,0x04,0x6d,0x00,0xf8,0xff,0xff
+
+# CHECK: tileloadd tmm6, [rbp + 8*r14 + 268435456]
+0xc4,0xa2,0x7b,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: tileloadd tmm3, [r8 + 4*rax + 291]
+0xc4,0xc2,0x7b,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: tileloadd tmm3, [2*rbp - 32]
+0xc4,0xe2,0x7b,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff
+
+# CHECK: tileloadd tmm4, [rbx + 64]
+0xc4,0xe2,0x7b,0x4b,0x64,0x23,0x40
+
+# CHECK: tileloaddt1 tmm6, [rbp + 8*r14 + 268435456]
+0xc4,0xa2,0x79,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: tileloaddt1 tmm3, [r8 + 4*rax + 291]
+0xc4,0xc2,0x79,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: tileloaddt1 tmm3, [2*rbp - 32]
+0xc4,0xe2,0x79,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff
+
+# CHECK: tileloaddt1     tmm6, [rbp + 16]
+0xc4,0xe2,0x79,0x4b,0x74,0x25,0x10
+
+# CHECK: tilerelease
+0xc4,0xe2,0x78,0x49,0xc0
+
+# CHECK: tilestored [rbp + 8*r14 + 268435456], tmm6
+0xc4,0xa2,0x7a,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: tilestored [r8 + 4*rax + 291], tmm3
+0xc4,0xc2,0x7a,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: tilestored [2*rbp - 32], tmm3
+0xc4,0xe2,0x7a,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff
+
+# CHECK: tilestored [r8], tmm3
+0xc4,0xc2,0x7a,0x4b,0x1c,0x20
+
+# CHECK: tilezero tmm6
+0xc4,0xe2,0x7b,0x49,0xf0
+
+# CHECK: tilezero tmm3
+0xc4,0xe2,0x7b,0x49,0xd8
+
+# CHECK: ldtilecfg [rbp + 8*r14 + 268435456]
+0xc4,0xa2,0x78,0x49,0x84,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: ldtilecfg [r8 + 4*rax + 291]
+0xc4,0xc2,0x78,0x49,0x84,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: ldtilecfg [rip]
+0xc4,0xe2,0x78,0x49,0x05,0x00,0x00,0x00,0x00
+
+# CHECK: ldtilecfg [2*rbp - 2048]
+0xc4,0xe2,0x78,0x49,0x04,0x6d,0x00,0xf8,0xff,0xff
+
+# CHECK: sttilecfg [rbp + 8*r14 + 268435456]
+0xc4,0xa2,0x79,0x49,0x84,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: sttilecfg [r8 + 4*rax + 291]
+0xc4,0xc2,0x79,0x49,0x84,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: sttilecfg [rip]
+0xc4,0xe2,0x79,0x49,0x05,0x00,0x00,0x00,0x00
+
+# CHECK: sttilecfg [2*rbp - 2048]
+0xc4,0xe2,0x79,0x49,0x04,0x6d,0x00,0xf8,0xff,0xff
+
+# CHECK: tileloadd tmm6, [rbp + 8*r14 + 268435456]
+0xc4,0xa2,0x7b,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: tileloadd tmm3, [r8 + 4*rax + 291]
+0xc4,0xc2,0x7b,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: tileloadd tmm3, [2*rbp - 32]
+0xc4,0xe2,0x7b,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff
+
+# CHECK: tileloaddt1 tmm6, [rbp + 8*r14 + 268435456]
+0xc4,0xa2,0x79,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: tileloaddt1 tmm3, [r8 + 4*rax + 291]
+0xc4,0xc2,0x79,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: tileloaddt1 tmm3, [2*rbp - 32]
+0xc4,0xe2,0x79,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff
+
+# CHECK: tilerelease
+0xc4,0xe2,0x78,0x49,0xc0
+
+# CHECK: tilestored [rbp + 8*r14 + 268435456], tmm6
+0xc4,0xa2,0x7a,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# CHECK: tilestored [r8 + 4*rax + 291], tmm3
+0xc4,0xc2,0x7a,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00
+
+# CHECK: tilestored [2*rbp - 32], tmm3
+0xc4,0xe2,0x7a,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff
+
+# CHECK: tilezero tmm6
+0xc4,0xe2,0x7b,0x49,0xf0
+
+# CHECK: tilezero tmm3
+0xc4,0xe2,0x7b,0x49,0xd8
diff --git a/llvm/test/MC/MachO/AArch64/arm-darwin-version-min-load-command.s b/llvm/test/MC/MachO/AArch64/arm-darwin-version-min-load-command.s
new file mode 100644
index 0000000000000..0af61fbf4d7ca
--- /dev/null
+++ b/llvm/test/MC/MachO/AArch64/arm-darwin-version-min-load-command.s
@@ -0,0 +1,73 @@
+// RUN: llvm-mc -triple arm64-apple-macos10.10.2 %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-MACOS-ARM64
+// RUN: llvm-mc -triple arm64-apple-macos11 %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-MACOS-ARM64
+// RUN: llvm-mc -triple arm64-apple-macos11.1 %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-MACOS-ARM64_1
+// RUN: llvm-mc -triple arm64-apple-ios13.0-macabi %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-MACCATALYST-ARM64
+// RUN: llvm-mc -triple arm64-apple-ios14.1-macabi %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-MACCATALYST-ARM64_1
+
+// RUN: llvm-mc -triple arm64-apple-ios10-simulator %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-IOSSIM2
+// RUN: llvm-mc -triple arm64-apple-ios13-simulator %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-IOSSIM2
+// RUN: llvm-mc -triple arm64-apple-ios14-simulator %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-IOSSIM2
+// RUN: llvm-mc -triple arm64-apple-ios14.1-simulator %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-IOSSIM3
+// RUN: llvm-mc -triple arm64-apple-tvos10-simulator %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-TVOSSIM2
+// RUN: llvm-mc -triple arm64-apple-watchos3-simulator %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-WATCHOSSIM2
+
+// CHECK-BUILD-IOSSIM2:           cmd LC_BUILD_VERSION
+// CHECK-BUILD-IOSSIM2-NEXT:  cmdsize 24
+// CHECK-BUILD-IOSSIM2-NEXT: platform iossim
+// CHECK-BUILD-IOSSIM2-NEXT:      sdk n/a
+// CHECK-BUILD-IOSSIM2-NEXT:    minos 14.0
+// CHECK-BUILD-IOSSIM2-NEXT:   ntools 0
+// CHECK-BUILD-IOSSIM2-NOT: LC_VERSION_MIN
+
+// CHECK-BUILD-IOSSIM3:           cmd LC_BUILD_VERSION
+// CHECK-BUILD-IOSSIM3-NEXT:  cmdsize 24
+// CHECK-BUILD-IOSSIM3-NEXT: platform iossim
+// CHECK-BUILD-IOSSIM3-NEXT:      sdk n/a
+// CHECK-BUILD-IOSSIM3-NEXT:    minos 14.1
+// CHECK-BUILD-IOSSIM3-NEXT:   ntools 0
+// CHECK-BUILD-IOSSIM3-NOT: LC_VERSION_MIN
+
+// CHECK-BUILD-TVOSSIM2:           cmd LC_BUILD_VERSION
+// CHECK-BUILD-TVOSSIM2-NEXT:  cmdsize 24
+// CHECK-BUILD-TVOSSIM2-NEXT: platform tvossim
+// CHECK-BUILD-TVOSSIM2-NEXT:      sdk n/a
+// CHECK-BUILD-TVOSSIM2-NEXT:    minos 14.0
+// CHECK-BUILD-TVOSSIM2-NEXT:   ntools 0
+// CHECK-BUILD-TVOSSIM2-NOT: LC_VERSION_MIN
+
+// CHECK-BUILD-WATCHOSSIM2:           cmd LC_BUILD_VERSION
+// CHECK-BUILD-WATCHOSSIM2-NEXT:  cmdsize 24
+// CHECK-BUILD-WATCHOSSIM2-NEXT: platform watchossim
+// CHECK-BUILD-WATCHOSSIM2-NEXT:      sdk n/a
+// CHECK-BUILD-WATCHOSSIM2-NEXT:    minos 7.0
+// CHECK-BUILD-WATCHOSSIM2-NEXT:   ntools 0
+// CHECK-BUILD-WATCHOSSIM2-NOT: LC_VERSION_MIN
+
+// CHECK-BUILD-MACOS-ARM64:           cmd LC_BUILD_VERSION
+// CHECK-BUILD-MACOS-ARM64-NEXT:  cmdsize 24
+// CHECK-BUILD-MACOS-ARM64-NEXT: platform macos
+// CHECK-BUILD-MACOS-ARM64-NEXT:      sdk n/a
+// CHECK-BUILD-MACOS-ARM64-NEXT:    minos 11.0
+// CHECK-BUILD-MACOS-ARM64-NEXT:   ntools 0
+// CHECK-BUILD-MACOS-ARM64-NOT: LC_VERSION_MIN
+
+// CHECK-BUILD-MACOS-ARM64_1:           cmd LC_BUILD_VERSION
+// CHECK-BUILD-MACOS-ARM64_1-NEXT:  cmdsize 24
+// CHECK-BUILD-MACOS-ARM64_1-NEXT: platform macos
+// CHECK-BUILD-MACOS-ARM64_1-NEXT:      sdk n/a
+// CHECK-BUILD-MACOS-ARM64_1-NEXT:    minos 11.1
+// CHECK-BUILD-MACOS-ARM64_1-NEXT:   ntools 0
+
+// CHECK-MACCATALYST-ARM64:           cmd LC_BUILD_VERSION
+// CHECK-MACCATALYST-ARM64-NEXT:  cmdsize 24
+// CHECK-MACCATALYST-ARM64-NEXT: platform macCatalyst
+// CHECK-MACCATALYST-ARM64-NEXT:      sdk n/a
+// CHECK-MACCATALYST-ARM64-NEXT:    minos 14.0
+// CHECK-MACCATALYST-ARM64-NEXT:   ntools 0
+
+// CHECK-MACCATALYST-ARM64_1:           cmd LC_BUILD_VERSION
+// CHECK-MACCATALYST-ARM64_1-NEXT:  cmdsize 24
+// CHECK-MACCATALYST-ARM64_1-NEXT: platform macCatalyst
+// CHECK-MACCATALYST-ARM64_1-NEXT:      sdk n/a
+// CHECK-MACCATALYST-ARM64_1-NEXT:    minos 14.1
+// CHECK-MACCATALYST-ARM64_1-NEXT:   ntools 0
diff --git a/llvm/test/MC/MachO/darwin-sdk-version.ll b/llvm/test/MC/MachO/darwin-sdk-version.ll
index bb3db56337020..df67b69001ccc 100644
--- a/llvm/test/MC/MachO/darwin-sdk-version.ll
+++ b/llvm/test/MC/MachO/darwin-sdk-version.ll
@@ -1,7 +1,7 @@
 ; RUN: llc %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s
 ; RUN: llc %s -filetype=asm -o - | FileCheck --check-prefix=ASM %s
 
-target triple = "x86_64-apple-macos10.14";
+target triple = "x86_64-apple-macos10.13";
 !llvm.module.flags = !{!0};
 !0 = !{i32 2, !"SDK Version", [3 x i32] [ i32 10, i32 14, i32 2 ] };
 
@@ -12,7 +12,7 @@ entry:
 
 ; CHECK:           cmd LC_VERSION_MIN_MACOSX
 ; CHECK-NEXT:  cmdsize 16
-; CHECK-NEXT:  version 10.14
+; CHECK-NEXT:  version 10.13
 ; CHECK-NEXT:      sdk 10.14.2
 
-; ASM: .macosx_version_min 10, 14 sdk_version 10, 14, 2
+; ASM: .macosx_version_min 10, 13 sdk_version 10, 14, 2
diff --git a/llvm/test/MC/MachO/darwin-version-min-load-command.s b/llvm/test/MC/MachO/darwin-version-min-load-command.s
index ad43bd7235668..6e1b4c20a6700 100644
--- a/llvm/test/MC/MachO/darwin-version-min-load-command.s
+++ b/llvm/test/MC/MachO/darwin-version-min-load-command.s
@@ -3,6 +3,14 @@
 // RUN: llvm-mc -triple x86_64-apple-darwin %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-DARWIN
 // RUN: llvm-mc -triple x86_64-apple-ios13.0-macabi %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-MACCATALYST
 
+// RUN: llvm-mc -triple x86_64-apple-macos10.14 %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-MACOS
+// RUN: llvm-mc -triple x86_64-apple-ios12 %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-IOS
+// RUN: llvm-mc -triple x86_64-apple-tvos12 %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-TVOS
+// RUN: llvm-mc -triple x86_64-apple-watchos5 %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-WATCHOS
+// RUN: llvm-mc -triple x86_64-apple-ios12-simulator %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-IOSSIM
+// RUN: llvm-mc -triple x86_64-apple-tvos12-simulator %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-TVOSSIM
+// RUN: llvm-mc -triple x86_64-apple-watchos5-simulator %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-WATCHOSSIM
+
 // Test version-min load command should be inferred from triple and should always be generated on Darwin
 // CHECK: Load command
 // CHECK:       cmd LC_VERSION_MIN_MACOSX
@@ -41,3 +49,60 @@
 // CHECK-MACCATALYST-NEXT:      sdk n/a
 // CHECK-MACCATALYST-NEXT:    minos 13.0
 // CHECK-MACCATALYST-NEXT:   ntools 0
+
+// CHECK-BUILD-MACOS:           cmd LC_BUILD_VERSION
+// CHECK-BUILD-MACOS-NEXT:  cmdsize 24
+// CHECK-BUILD-MACOS-NEXT: platform macos
+// CHECK-BUILD-MACOS-NEXT:      sdk n/a
+// CHECK-BUILD-MACOS-NEXT:    minos 10.14
+// CHECK-BUILD-MACOS-NEXT:   ntools 0
+// CHECK-BUILD-MACOS-NOT: LC_VERSION_MIN
+
+// CHECK-BUILD-IOS:           cmd LC_BUILD_VERSION
+// CHECK-BUILD-IOS-NEXT:  cmdsize 24
+// CHECK-BUILD-IOS-NEXT: platform ios
+// CHECK-BUILD-IOS-NEXT:      sdk n/a
+// CHECK-BUILD-IOS-NEXT:    minos 12.0
+// CHECK-BUILD-IOS-NEXT:   ntools 0
+// CHECK-BUILD-IOS-NOT: LC_VERSION_MIN
+
+// CHECK-BUILD-TVOS:           cmd LC_BUILD_VERSION
+// CHECK-BUILD-TVOS-NEXT:  cmdsize 24
+// CHECK-BUILD-TVOS-NEXT: platform tvos
+// CHECK-BUILD-TVOS-NEXT:      sdk n/a
+// CHECK-BUILD-TVOS-NEXT:    minos 12.0
+// CHECK-BUILD-TVOS-NEXT:   ntools 0
+// CHECK-BUILD-TVOS-NOT: LC_VERSION_MIN
+
+// CHECK-BUILD-WATCHOS:           cmd LC_BUILD_VERSION
+// CHECK-BUILD-WATCHOS-NEXT:  cmdsize 24
+// CHECK-BUILD-WATCHOS-NEXT: platform watchos
+// CHECK-BUILD-WATCHOS-NEXT:      sdk n/a
+// CHECK-BUILD-WATCHOS-NEXT:    minos 5.0
+// CHECK-BUILD-WATCHOS-NEXT:   ntools 0
+// CHECK-BUILD-WATCHOS-NOT: LC_VERSION_MIN
+
+// CHECK-BUILD-IOSSIM:           cmd LC_BUILD_VERSION
+// CHECK-BUILD-IOSSIM-NEXT:  cmdsize 24
+// CHECK-BUILD-IOSSIM-NEXT: platform iossim
+// CHECK-BUILD-IOSSIM-NEXT:      sdk n/a
+// CHECK-BUILD-IOSSIM-NEXT:    minos 12.0
+// CHECK-BUILD-IOSSIM-NEXT:   ntools 0
+// CHECK-BUILD-IOSSIM-NOT: LC_VERSION_MIN
+
+// CHECK-BUILD-TVOSSIM:           cmd LC_BUILD_VERSION
+// CHECK-BUILD-TVOSSIM-NEXT:  cmdsize 24
+// CHECK-BUILD-TVOSSIM-NEXT: platform tvossim
+// CHECK-BUILD-TVOSSIM-NEXT:      sdk n/a
+// CHECK-BUILD-TVOSSIM-NEXT:    minos 12.0
+// CHECK-BUILD-TVOSSIM-NEXT:   ntools 0
+// CHECK-BUILD-TVOSSIM-NOT: LC_VERSION_MIN
+
+// CHECK-BUILD-WATCHOSSIM:           cmd LC_BUILD_VERSION
+// CHECK-BUILD-WATCHOSSIM-NEXT:  cmdsize 24
+// CHECK-BUILD-WATCHOSSIM-NEXT: platform watchossim
+// CHECK-BUILD-WATCHOSSIM-NEXT:      sdk n/a
+// CHECK-BUILD-WATCHOSSIM-NEXT:    minos 5.0
+// CHECK-BUILD-WATCHOSSIM-NEXT:   ntools 0
+// CHECK-BUILD-WATCHOSSIM-NOT: LC_VERSION_MIN
+
diff --git a/llvm/test/MC/PowerPC/p10.s b/llvm/test/MC/PowerPC/p10.s
index 29a11c8756eb7..b99d28e861581 100644
--- a/llvm/test/MC/PowerPC/p10.s
+++ b/llvm/test/MC/PowerPC/p10.s
@@ -1,23 +1,4 @@
-# RUN: llvm-mc -triple powerpc64-unknown-linux-gnu --show-encoding %s | \
-# RUN:   FileCheck -check-prefix=CHECK-BE %s
-# RUN: llvm-mc -triple powerpc64le-unknown-linux-gnu --show-encoding %s | \
-# RUN:   FileCheck -check-prefix=CHECK-LE %s
-
-# CHECK-BE: vpdepd 1, 2, 0                        # encoding: [0x10,0x22,0x05,0xcd]
-# CHECK-LE: vpdepd 1, 2, 0                        # encoding: [0xcd,0x05,0x22,0x10]
-            vpdepd 1, 2, 0
-# CHECK-BE: vpextd 1, 2, 0                        # encoding: [0x10,0x22,0x05,0x8d]
-# CHECK-LE: vpextd 1, 2, 0                        # encoding: [0x8d,0x05,0x22,0x10]
-            vpextd 1, 2, 0
-# CHECK-BE: pdepd 1, 2, 4                         # encoding: [0x7c,0x41,0x21,0x38]
-# CHECK-LE: pdepd 1, 2, 4                         # encoding: [0x38,0x21,0x41,0x7c]
-            pdepd 1, 2, 4
-# CHECK-BE: pextd 1, 2, 4                         # encoding: [0x7c,0x41,0x21,0x78]
-# CHECK-LE: pextd 1, 2, 4                         # encoding: [0x78,0x21,0x41,0x7c]
-            pextd 1, 2, 4
-# CHECK-BE: vclrlb 1, 4, 3                        # encoding: [0x10,0x24,0x19,0x8d]
-# CHECK-LE: vclrlb 1, 4, 3                        # encoding: [0x8d,0x19,0x24,0x10]
-            vclrlb 1, 4, 3
-# CHECK-BE: vclrrb 1, 4, 3                        # encoding: [0x10,0x24,0x19,0xcd]
-# CHECK-LE: vclrrb 1, 4, 3                        # encoding: [0xcd,0x19,0x24,0x10]
-            vclrrb 1, 4, 3
+# RUN: llvm-mc -triple powerpc64-unknown-linux-gnu --show-encoding %s
+##############################################################################
+# CONTENT MOVED TO ppc64-encoding-ISA31.s, please do NOT add to this file
+##############################################################################
diff --git a/llvm/test/MC/PowerPC/future-errors.s b/llvm/test/MC/PowerPC/ppc64-encoding-ISA31-errors.s
similarity index 100%
rename from llvm/test/MC/PowerPC/future-errors.s
rename to llvm/test/MC/PowerPC/ppc64-encoding-ISA31-errors.s
diff --git a/llvm/test/MC/PowerPC/future-reloc.s b/llvm/test/MC/PowerPC/ppc64-encoding-ISA31-pcrel-reloc.s
similarity index 100%
rename from llvm/test/MC/PowerPC/future-reloc.s
rename to llvm/test/MC/PowerPC/ppc64-encoding-ISA31-pcrel-reloc.s
diff --git a/llvm/test/MC/PowerPC/future.s b/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s
similarity index 53%
rename from llvm/test/MC/PowerPC/future.s
rename to llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s
index dd5c549e7a4bf..5ed6b14d38aea 100644
--- a/llvm/test/MC/PowerPC/future.s
+++ b/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s
@@ -3,7 +3,6 @@
 # RUN: llvm-mc -triple powerpc64le-unknown-linux-gnu --show-encoding %s | \
 # RUN:   FileCheck -check-prefix=CHECK-LE %s
 
-
 # CHECK-BE: plxv 63, 8589934591(0), 1       # encoding: [0x04,0x11,0xff,0xff
 # CHECK-BE-SAME:                                         0xcf,0xe0,0xff,0xff]
 # CHECK-LE: plxv 63, 8589934591(0), 1       # encoding: [0xff,0xff,0x11,0x04
@@ -224,4 +223,209 @@
 # CHECK-LE: pld 1, 8589934591(0), 1               # encoding: [0xff,0xff,0x11,0x04,
 # CHECK-LE-SAME:                                               0xff,0xff,0x20,0xe4]
             pld 1, 8589934591(0), 1
-
+# CHECK-BE: vpdepd 1, 2, 0                        # encoding: [0x10,0x22,0x05,0xcd]
+# CHECK-LE: vpdepd 1, 2, 0                        # encoding: [0xcd,0x05,0x22,0x10]
+            vpdepd 1, 2, 0
+# CHECK-BE: vpextd 1, 2, 0                        # encoding: [0x10,0x22,0x05,0x8d]
+# CHECK-LE: vpextd 1, 2, 0                        # encoding: [0x8d,0x05,0x22,0x10]
+            vpextd 1, 2, 0
+# CHECK-BE: pdepd 1, 2, 4                         # encoding: [0x7c,0x41,0x21,0x38]
+# CHECK-LE: pdepd 1, 2, 4                         # encoding: [0x38,0x21,0x41,0x7c]
+            pdepd 1, 2, 4
+# CHECK-BE: pextd 1, 2, 4                         # encoding: [0x7c,0x41,0x21,0x78]
+# CHECK-LE: pextd 1, 2, 4                         # encoding: [0x78,0x21,0x41,0x7c]
+            pextd 1, 2, 4
+# CHECK-BE: vcfuged 1, 2, 4                       # encoding: [0x10,0x22,0x25,0x4d]
+# CHECK-LE: vcfuged 1, 2, 4                       # encoding: [0x4d,0x25,0x22,0x10]
+            vcfuged 1, 2, 4
+# CHECK-BE: cfuged 1, 2, 4                        # encoding: [0x7c,0x41,0x21,0xb8]
+# CHECK-LE: cfuged 1, 2, 4                        # encoding: [0xb8,0x21,0x41,0x7c]
+            cfuged 1, 2, 4
+# CHECK-BE: vgnb 1, 2, 2                          # encoding: [0x10,0x22,0x14,0xcc]
+# CHECK-LE: vgnb 1, 2, 2                          # encoding: [0xcc,0x14,0x22,0x10]
+            vgnb 1, 2, 2
+# CHECK-BE: xxeval 32, 1, 2, 3, 2                 # encoding: [0x05,0x00,0x00,0x02,
+# CHECK-BE-SAME:                                               0x88,0x01,0x10,0xd1]
+# CHECK-LE: xxeval 32, 1, 2, 3, 2                 # encoding: [0x02,0x00,0x00,0x05,
+# CHECK-LE-SAME:                                               0xd1,0x10,0x01,0x88]
+            xxeval 32, 1, 2, 3, 2
+# CHECK-BE: vclzdm 1, 2, 3                        # encoding: [0x10,0x22,0x1f,0x84]
+# CHECK-LE: vclzdm 1, 2, 3                        # encoding: [0x84,0x1f,0x22,0x10]
+            vclzdm 1, 2, 3
+# CHECK-BE: vctzdm 1, 2, 3                        # encoding: [0x10,0x22,0x1f,0xc4]
+# CHECK-LE: vctzdm 1, 2, 3                        # encoding: [0xc4,0x1f,0x22,0x10]
+            vctzdm 1, 2, 3
+# CHECK-BE: cntlzdm 1, 3, 2                       # encoding: [0x7c,0x61,0x10,0x76]
+# CHECK-LE: cntlzdm 1, 3, 2                       # encoding: [0x76,0x10,0x61,0x7c]
+            cntlzdm 1, 3, 2
+# CHECK-BE: cnttzdm 1, 3, 2                       # encoding: [0x7c,0x61,0x14,0x76]
+# CHECK-LE: cnttzdm 1, 3, 2                       # encoding: [0x76,0x14,0x61,0x7c]
+            cnttzdm 1, 3, 2
+# CHECK-BE: xxgenpcvbm 0, 1, 2                    # encoding: [0xf0,0x02,0x0f,0x28]
+# CHECK-LE: xxgenpcvbm 0, 1, 2                    # encoding: [0x28,0x0f,0x02,0xf0]
+            xxgenpcvbm 0, 1, 2
+# CHECK-BE: xxgenpcvhm 0, 1, 2                    # encoding: [0xf0,0x02,0x0f,0x2a]
+# CHECK-LE: xxgenpcvhm 0, 1, 2                    # encoding: [0x2a,0x0f,0x02,0xf0]
+            xxgenpcvhm 0, 1, 2
+# CHECK-BE: xxgenpcvwm 0, 1, 2                    # encoding: [0xf0,0x02,0x0f,0x68]
+# CHECK-LE: xxgenpcvwm 0, 1, 2                    # encoding: [0x68,0x0f,0x02,0xf0]
+            xxgenpcvwm 0, 1, 2
+# CHECK-BE: xxgenpcvdm 0, 1, 2                    # encoding: [0xf0,0x02,0x0f,0x6a]
+# CHECK-LE: xxgenpcvdm 0, 1, 2                    # encoding: [0x6a,0x0f,0x02,0xf0]
+            xxgenpcvdm 0, 1, 2
+# CHECK-BE: vclrlb 1, 4, 3                        # encoding: [0x10,0x24,0x19,0x8d]
+# CHECK-LE: vclrlb 1, 4, 3                        # encoding: [0x8d,0x19,0x24,0x10]
+            vclrlb 1, 4, 3
+# CHECK-BE: vclrrb 1, 4, 3                        # encoding: [0x10,0x24,0x19,0xcd]
+# CHECK-LE: vclrrb 1, 4, 3                        # encoding: [0xcd,0x19,0x24,0x10]
+            vclrrb 1, 4, 3
+# Boundary conditions of 8RR_DForm_IMM32_XT6's immediates
+# CHECK-BE: xxspltiw 63, 4294901760               # encoding: [0x05,0x00,0xff,0xff,
+# CHECK-BE-SAME:                                               0x83,0xe7,0x00,0x00]
+# CHECK-LE: xxspltiw 63, 4294901760               # encoding: [0xff,0xff,0x00,0x05,
+# CHECK-LE-SAME:                                               0x00,0x00,0xe7,0x83]
+            xxspltiw 63, 4294901760
+# CHECK-BE: xxspltiw 63, 65535                    # encoding: [0x05,0x00,0x00,0x00,
+# CHECK-BE-SAME:                                               0x83,0xe7,0xff,0xff]
+# CHECK-LE: xxspltiw 63, 65535                    # encoding: [0x00,0x00,0x00,0x05,
+# CHECK-LE-SAME:                                               0xff,0xff,0xe7,0x83]
+            xxspltiw 63, 65535
+# CHECK-BE: xxspltiw 63, 4294967295               # encoding: [0x05,0x00,0xff,0xff,
+# CHECK-BE-SAME:                                               0x83,0xe7,0xff,0xff]
+# CHECK-LE: xxspltiw 63, 4294967295               # encoding: [0xff,0xff,0x00,0x05,
+# CHECK-LE-SAME:                                               0xff,0xff,0xe7,0x83]
+            xxspltiw 63, 4294967295
+# CHECK-BE: xxspltiw 63, -1                       # encoding: [0x05,0x00,0xff,0xff,
+# CHECK-BE-SAME:                                               0x83,0xe7,0xff,0xff]
+# CHECK-LE: xxspltiw 63, -1                       # encoding: [0xff,0xff,0x00,0x05,
+# CHECK-LE-SAME:                                               0xff,0xff,0xe7,0x83]
+            xxspltiw 63, -1
+# CHECK-BE: xxspltidp 63, 4294967295              # encoding: [0x05,0x00,0xff,0xff,
+# CHECK-BE-SAME:                                               0x83,0xe5,0xff,0xff]
+# CHECK-LE: xxspltidp 63, 4294967295              # encoding: [0xff,0xff,0x00,0x05,
+# CHECK-LE-SAME:                                               0xff,0xff,0xe5,0x83]
+            xxspltidp 63, 4294967295
+# Boundary conditions of 8RR_DForm_IMM32_XT6_IX's immediates
+# CHECK-BE: xxsplti32dx 63, 1, 4294901760         # encoding: [0x05,0x00,0xff,0xff,
+# CHECK-BE-SAME:                                               0x83,0xe3,0x00,0x00]
+# CHECK-LE: xxsplti32dx 63, 1, 4294901760         # encoding: [0xff,0xff,0x00,0x05,
+# CHECK-LE-SAME:                                               0x00,0x00,0xe3,0x83]
+            xxsplti32dx 63, 1, 4294901760
+# CHECK-BE: xxsplti32dx 63, 1, 65535              # encoding: [0x05,0x00,0x00,0x00,
+# CHECK-BE-SAME:                                               0x83,0xe3,0xff,0xff]
+# CHECK-LE: xxsplti32dx 63, 1, 65535              # encoding: [0x00,0x00,0x00,0x05,
+# CHECK-LE-SAME:                                               0xff,0xff,0xe3,0x83]
+            xxsplti32dx 63, 1, 65535
+# CHECK-BE: xxsplti32dx 63, 1, 4294967295         # encoding: [0x05,0x00,0xff,0xff,
+# CHECK-BE-SAME:                                               0x83,0xe3,0xff,0xff]
+# CHECK-LE: xxsplti32dx 63, 1, 4294967295         # encoding: [0xff,0xff,0x00,0x05,
+# CHECK-LE-SAME:                                               0xff,0xff,0xe3,0x83]
+            xxsplti32dx 63, 1, 4294967295
+# CHECK-BE: xxsplti32dx 63, 1, -1                 # encoding: [0x05,0x00,0xff,0xff,
+# CHECK-BE-SAME:                                               0x83,0xe3,0xff,0xff]
+# CHECK-LE: xxsplti32dx 63, 1, -1                 # encoding: [0xff,0xff,0x00,0x05,
+# CHECK-LE-SAME:                                               0xff,0xff,0xe3,0x83]
+            xxsplti32dx 63, 1, -1
+# CHECK-BE: xxpermx 6, 63, 21, 34, 2              # encoding: [0x05,0x00,0x00,0x02,
+# CHECK-BE-SAME:                                               0x88,0xdf,0xa8,0x8c]
+# CHECK-LE: xxpermx 6, 63, 21, 34, 2              # encoding: [0x02,0x00,0x00,0x05,
+# CHECK-LE-SAME:                                               0x8c,0xa8,0xdf,0x88]
+            xxpermx 6, 63, 21, 34, 2
+# CHECK-BE: xxblendvb 6, 63, 21, 34               # encoding: [0x05,0x00,0x00,0x00,
+# CHECK-BE-SAME:                                               0x84,0xdf,0xa8,0x8c]
+# CHECK-LE: xxblendvb 6, 63, 21, 34               # encoding: [0x00,0x00,0x00,0x05,
+# CHECK-LE-SAME:                                               0x8c,0xa8,0xdf,0x84]
+            xxblendvb 6, 63, 21, 34
+# CHECK-BE: xxblendvh 6, 63, 21, 34               # encoding: [0x05,0x00,0x00,0x00,
+# CHECK-BE-SAME:                                               0x84,0xdf,0xa8,0x9c]
+# CHECK-LE: xxblendvh 6, 63, 21, 34               # encoding: [0x00,0x00,0x00,0x05,
+# CHECK-LE-SAME:                                               0x9c,0xa8,0xdf,0x84]
+            xxblendvh 6, 63, 21, 34
+# CHECK-BE: xxblendvw 6, 63, 21, 34               # encoding: [0x05,0x00,0x00,0x00,
+# CHECK-BE-SAME:                                               0x84,0xdf,0xa8,0xac]
+# CHECK-LE: xxblendvw 6, 63, 21, 34               # encoding: [0x00,0x00,0x00,0x05,
+# CHECK-LE-SAME:                                               0xac,0xa8,0xdf,0x84]
+            xxblendvw 6, 63, 21, 34
+# CHECK-BE: xxblendvd 6, 63, 21, 34               # encoding: [0x05,0x00,0x00,0x00,
+# CHECK-BE-SAME:                                               0x84,0xdf,0xa8,0xbc]
+# CHECK-LE: xxblendvd 6, 63, 21, 34               # encoding: [0x00,0x00,0x00,0x05,
+# CHECK-LE-SAME:                                               0xbc,0xa8,0xdf,0x84]
+            xxblendvd 6, 63, 21, 34
+# CHECK-BE: vsldbi 2, 3, 4, 5                     # encoding: [0x10,0x43,0x21,0x56]
+# CHECK-LE: vsldbi 2, 3, 4, 5                     # encoding: [0x56,0x21,0x43,0x10]
+            vsldbi 2, 3, 4, 5
+# CHECK-BE: vsrdbi 2, 3, 4, 5                     # encoding: [0x10,0x43,0x23,0x56]
+# CHECK-LE: vsrdbi 2, 3, 4, 5                     # encoding: [0x56,0x23,0x43,0x10]
+            vsrdbi 2, 3, 4, 5
+# CHECK-BE: vinsw 2, 3, 12                        # encoding: [0x10,0x4c,0x18,0xcf]
+# CHECK-LE: vinsw 2, 3, 12                        # encoding: [0xcf,0x18,0x4c,0x10]
+            vinsw 2, 3, 12
+# CHECK-BE: vinsd 2, 3, 12                        # encoding: [0x10,0x4c,0x19,0xcf]
+# CHECK-LE: vinsd 2, 3, 12                        # encoding: [0xcf,0x19,0x4c,0x10]
+            vinsd 2, 3, 12
+# CHECK-BE: vinsbvlx 1, 3, 5                      # encoding: [0x10,0x23,0x28,0x0f]
+# CHECK-LE: vinsbvlx 1, 3, 5                      # encoding: [0x0f,0x28,0x23,0x10]
+            vinsbvlx 1, 3, 5
+# CHECK-BE: vinsbvrx 1, 3, 5                      # encoding: [0x10,0x23,0x29,0x0f]
+# CHECK-LE: vinsbvrx 1, 3, 5                      # encoding: [0x0f,0x29,0x23,0x10]
+            vinsbvrx 1, 3, 5
+# CHECK-BE: vinshvlx 1, 3, 5                      # encoding: [0x10,0x23,0x28,0x4f]
+# CHECK-LE: vinshvlx 1, 3, 5                      # encoding: [0x4f,0x28,0x23,0x10]
+            vinshvlx 1, 3, 5
+# CHECK-BE: vinshvrx 1, 3, 5                      # encoding: [0x10,0x23,0x29,0x4f]
+# CHECK-LE: vinshvrx 1, 3, 5                      # encoding: [0x4f,0x29,0x23,0x10]
+            vinshvrx 1, 3, 5
+# CHECK-BE: vinswvlx 1, 3, 5                      # encoding: [0x10,0x23,0x28,0x8f]
+# CHECK-LE: vinswvlx 1, 3, 5                      # encoding: [0x8f,0x28,0x23,0x10]
+            vinswvlx 1, 3, 5
+# CHECK-BE: vinswvrx 1, 3, 5                      # encoding: [0x10,0x23,0x29,0x8f]
+# CHECK-LE: vinswvrx 1, 3, 5                      # encoding: [0x8f,0x29,0x23,0x10]
+            vinswvrx 1, 3, 5
+# CHECK-BE: vinsblx 1, 2, 3                       # encoding: [0x10,0x22,0x1a,0x0f]
+# CHECK-LE: vinsblx 1, 2, 3                       # encoding: [0x0f,0x1a,0x22,0x10]
+            vinsblx 1, 2, 3
+# CHECK-BE: vinsbrx 1, 2, 3                       # encoding: [0x10,0x22,0x1b,0x0f]
+# CHECK-LE: vinsbrx 1, 2, 3                       # encoding: [0x0f,0x1b,0x22,0x10]
+            vinsbrx 1, 2, 3
+# CHECK-BE: vinshlx 1, 2, 3                       # encoding: [0x10,0x22,0x1a,0x4f]
+# CHECK-LE: vinshlx 1, 2, 3                       # encoding: [0x4f,0x1a,0x22,0x10]
+            vinshlx 1, 2, 3
+# CHECK-BE: vinshrx 1, 2, 3                       # encoding: [0x10,0x22,0x1b,0x4f]
+# CHECK-LE: vinshrx 1, 2, 3                       # encoding: [0x4f,0x1b,0x22,0x10]
+            vinshrx 1, 2, 3
+# CHECK-BE: vinswlx 1, 2, 3                       # encoding: [0x10,0x22,0x1a,0x8f]
+# CHECK-LE: vinswlx 1, 2, 3                       # encoding: [0x8f,0x1a,0x22,0x10]
+            vinswlx 1, 2, 3
+# CHECK-BE: vinswrx 1, 2, 3                       # encoding: [0x10,0x22,0x1b,0x8f]
+# CHECK-LE: vinswrx 1, 2, 3                       # encoding: [0x8f,0x1b,0x22,0x10]
+            vinswrx 1, 2, 3
+# CHECK-BE: vinsdlx 1, 2, 3                       # encoding: [0x10,0x22,0x1a,0xcf]
+# CHECK-LE: vinsdlx 1, 2, 3                       # encoding: [0xcf,0x1a,0x22,0x10]
+            vinsdlx 1, 2, 3
+# CHECK-BE: vinsdrx 1, 2, 3                       # encoding: [0x10,0x22,0x1b,0xcf]
+# CHECK-LE: vinsdrx 1, 2, 3                       # encoding: [0xcf,0x1b,0x22,0x10]
+            vinsdrx 1, 2, 3
+# CHECK-BE: lxvrbx 32, 1, 2                       # encoding: [0x7c,0x01,0x10,0x1b]
+# CHECK-LE: lxvrbx 32, 1, 2                       # encoding: [0x1b,0x10,0x01,0x7c]
+            lxvrbx 32, 1, 2
+# CHECK-BE: lxvrhx 33, 1, 2                       # encoding: [0x7c,0x21,0x10,0x5b]
+# CHECK-LE: lxvrhx 33, 1, 2                       # encoding: [0x5b,0x10,0x21,0x7c]
+            lxvrhx 33, 1, 2
+# CHECK-BE: lxvrdx 34, 1, 2                       # encoding: [0x7c,0x41,0x10,0xdb]
+# CHECK-LE: lxvrdx 34, 1, 2                       # encoding: [0xdb,0x10,0x41,0x7c]
+            lxvrdx 34, 1, 2
+# CHECK-BE: lxvrwx 35, 1, 2                       # encoding: [0x7c,0x61,0x10,0x9b]
+# CHECK-LE: lxvrwx 35, 1, 2                       # encoding: [0x9b,0x10,0x61,0x7c]
+            lxvrwx 35, 1, 2
+# CHECK-BE: stxvrbx 32, 3, 1                      # encoding: [0x7c,0x03,0x09,0x1b]
+# CHECK-LE: stxvrbx 32, 3, 1                      # encoding: [0x1b,0x09,0x03,0x7c]
+            stxvrbx 32, 3, 1
+# CHECK-BE: stxvrhx 33, 3, 1                      # encoding: [0x7c,0x23,0x09,0x5b]
+# CHECK-LE: stxvrhx 33, 3, 1                      # encoding: [0x5b,0x09,0x23,0x7c]
+            stxvrhx 33, 3, 1
+# CHECK-BE: stxvrwx 34, 3, 1                      # encoding: [0x7c,0x43,0x09,0x9b]
+# CHECK-LE: stxvrwx 34, 3, 1                      # encoding: [0x9b,0x09,0x43,0x7c]
+            stxvrwx 34, 3, 1
+# CHECK-BE: stxvrdx 35, 3, 1                      # encoding: [0x7c,0x63,0x09,0xdb]
+# CHECK-LE: stxvrdx 35, 3, 1                      # encoding: [0xdb,0x09,0x63,0x7c]
+            stxvrdx 35, 3, 1
diff --git a/llvm/test/MC/RISCV/machine-csr-names.s b/llvm/test/MC/RISCV/machine-csr-names.s
index 6467b04930649..93ecd7e173ffd 100644
--- a/llvm/test/MC/RISCV/machine-csr-names.s
+++ b/llvm/test/MC/RISCV/machine-csr-names.s
@@ -849,6 +849,20 @@ csrrs t2, 0xB1F, zero
 ######################################
 # Machine Counter Setup
 ######################################
+# mcountinhibit
+# name
+# CHECK-INST: csrrs t1, mcountinhibit, zero
+# CHECK-ENC:  encoding: [0x73,0x23,0x00,0x32]
+# CHECK-INST-ALIAS: csrr t1, mcountinhibit
+# uimm12
+# CHECK-INST: csrrs t2, mcountinhibit, zero
+# CHECK-ENC:  encoding: [0xf3,0x23,0x00,0x32]
+# CHECK-INST-ALIAS: csrr t2, mcountinhibit
+# name
+csrrs t1, mcountinhibit, zero
+# uimm12
+csrrs t2, 0x320, zero
+
 # mhpmevent3
 # name
 # CHECK-INST: csrrs t1, mhpmevent3, zero
diff --git a/llvm/test/MC/RISCV/rv32c-invalid.s b/llvm/test/MC/RISCV/rv32c-invalid.s
index 29cf0ac239fa4..53b62c289e75e 100644
--- a/llvm/test/MC/RISCV/rv32c-invalid.s
+++ b/llvm/test/MC/RISCV/rv32c-invalid.s
@@ -1,4 +1,4 @@
-# RUN: not llvm-mc -triple=riscv32 -mattr=+c -mattr=-rvc-hints < %s 2>&1 \
+# RUN: not llvm-mc -triple=riscv32 -mattr=+c -mattr=+no-rvc-hints < %s 2>&1 \
 # RUN:     | FileCheck %s
 
 ## GPRC
diff --git a/llvm/test/MC/RISCV/rvv/add.s b/llvm/test/MC/RISCV/rvv/add.s
new file mode 100644
index 0000000000000..f588d90a9c579
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/add.s
@@ -0,0 +1,339 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vadd.vv v8, v4, v20, v0.t
+# CHECK-INST: vadd.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x00]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 00 <unknown>
+
+vadd.vv v8, v4, v20
+# CHECK-INST: vadd.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x02]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 02 <unknown>
+
+vadd.vx v8, v4, a0, v0.t
+# CHECK-INST: vadd.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x00]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 00 <unknown>
+
+vadd.vx v8, v4, a0
+# CHECK-INST: vadd.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x02]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 02 <unknown>
+
+vadd.vi v8, v4, 15, v0.t
+# CHECK-INST: vadd.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x00]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 00 <unknown>
+
+vadd.vi v8, v4, 15
+# CHECK-INST: vadd.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x02]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 02 <unknown>
+
+vwaddu.vv v8, v4, v20, v0.t
+# CHECK-INST: vwaddu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xc0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a c0 <unknown>
+
+vwaddu.vv v8, v4, v20
+# CHECK-INST: vwaddu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xc2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a c2 <unknown>
+
+vwaddu.vx v8, v4, a0, v0.t
+# CHECK-INST: vwaddu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xc0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 c0 <unknown>
+
+vwaddu.vx v8, v4, a0
+# CHECK-INST: vwaddu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0xc2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 c2 <unknown>
+
+vwadd.vv v8, v4, v20, v0.t
+# CHECK-INST: vwadd.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xc4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a c4 <unknown>
+
+vwadd.vv v8, v4, v20
+# CHECK-INST: vwadd.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xc6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a c6 <unknown>
+
+vwadd.vx v8, v4, a0, v0.t
+# CHECK-INST: vwadd.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xc4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 c4 <unknown>
+
+vwadd.vx v8, v4, a0
+# CHECK-INST: vwadd.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0xc6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 c6 <unknown>
+
+vwaddu.wv v8, v4, v20, v0.t
+# CHECK-INST: vwaddu.wv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xd0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a d0 <unknown>
+
+vwaddu.wv v8, v4, v20
+# CHECK-INST: vwaddu.wv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xd2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a d2 <unknown>
+
+vwaddu.wx v8, v4, a0, v0.t
+# CHECK-INST: vwaddu.wx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xd0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 d0 <unknown>
+
+vwaddu.wx v8, v4, a0
+# CHECK-INST: vwaddu.wx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0xd2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 d2 <unknown>
+
+vwadd.wv v8, v4, v20, v0.t
+# CHECK-INST: vwadd.wv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xd4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a d4 <unknown>
+
+vwadd.wv v8, v4, v20
+# CHECK-INST: vwadd.wv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xd6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a d6 <unknown>
+
+vwadd.wx v8, v4, a0, v0.t
+# CHECK-INST: vwadd.wx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xd4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 d4 <unknown>
+
+vwadd.wx v8, v4, a0
+# CHECK-INST: vwadd.wx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0xd6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 d6 <unknown>
+
+vadc.vvm v8, v4, v20, v0
+# CHECK-INST: vadc.vvm v8, v4, v20, v0
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x40]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 40 <unknown>
+
+vadc.vxm v8, v4, a0, v0
+# CHECK-INST: vadc.vxm v8, v4, a0, v0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x40]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 40 <unknown>
+
+vadc.vim v8, v4, 15, v0
+# CHECK-INST: vadc.vim v8, v4, 15, v0
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x40]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 40 <unknown>
+
+vmadc.vvm v8, v4, v20, v0
+# CHECK-INST: vmadc.vvm v8, v4, v20, v0
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x44]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 44 <unknown>
+
+vmadc.vxm v8, v4, a0, v0
+# CHECK-INST: vmadc.vxm v8, v4, a0, v0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x44]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 44 <unknown>
+
+vmadc.vim v8, v4, 15, v0
+# CHECK-INST: vmadc.vim v8, v4, 15, v0
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x44]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 44 <unknown>
+
+vmadc.vv v8, v4, v20
+# CHECK-INST: vmadc.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x46]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 46 <unknown>
+
+vmadc.vx v8, v4, a0
+# CHECK-INST: vmadc.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x46]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 46 <unknown>
+
+vmadc.vi v8, v4, 15
+# CHECK-INST: vmadc.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x46]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 46 <unknown>
+
+vsaddu.vv v8, v4, v20, v0.t
+# CHECK-INST: vsaddu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x80]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 80 <unknown>
+
+vsaddu.vv v8, v4, v20
+# CHECK-INST: vsaddu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x82]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 82 <unknown>
+
+vsaddu.vx v8, v4, a0, v0.t
+# CHECK-INST: vsaddu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x80]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 80 <unknown>
+
+vsaddu.vx v8, v4, a0
+# CHECK-INST: vsaddu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x82]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 82 <unknown>
+
+vsaddu.vi v8, v4, 15, v0.t
+# CHECK-INST: vsaddu.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x80]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 80 <unknown>
+
+vsaddu.vi v8, v4, 15
+# CHECK-INST: vsaddu.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x82]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 82 <unknown>
+
+vsadd.vv v8, v4, v20, v0.t
+# CHECK-INST: vsadd.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x84]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 84 <unknown>
+
+vsadd.vv v8, v4, v20
+# CHECK-INST: vsadd.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x86]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 86 <unknown>
+
+vsadd.vx v8, v4, a0, v0.t
+# CHECK-INST: vsadd.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x84]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 84 <unknown>
+
+vsadd.vx v8, v4, a0
+# CHECK-INST: vsadd.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x86]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 86 <unknown>
+
+vsadd.vi v8, v4, 15, v0.t
+# CHECK-INST: vsadd.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x84]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 84 <unknown>
+
+vsadd.vi v8, v4, 15
+# CHECK-INST: vsadd.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x86]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 86 <unknown>
+
+vaadd.vv v8, v4, v20, v0.t
+# CHECK-INST: vaadd.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x24]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 24 <unknown>
+
+vaadd.vv v8, v4, v20
+# CHECK-INST: vaadd.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x26]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 26 <unknown>
+
+vaadd.vx v8, v4, a0, v0.t
+# CHECK-INST: vaadd.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0x24]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 24 <unknown>
+
+vaadd.vx v8, v4, a0
+# CHECK-INST: vaadd.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0x26]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 26 <unknown>
+
+vaaddu.vv v8, v4, v20, v0.t
+# CHECK-INST: vaaddu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x20]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 20 <unknown>
+
+vaaddu.vv v8, v4, v20
+# CHECK-INST: vaaddu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x22]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 22 <unknown>
+
+vaaddu.vx v8, v4, a0, v0.t
+# CHECK-INST: vaaddu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0x20]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 20 <unknown>
+
+vaaddu.vx v8, v4, a0
+# CHECK-INST: vaaddu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0x22]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 22 <unknown>
+
+vwcvt.x.x.v v8, v4, v0.t
+# CHECK-INST: vwcvt.x.x.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x40,0xc4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 40 c4 <unknown>
+
+vwcvt.x.x.v v8, v4
+# CHECK-INST: vwadd.vx v8, v4, zero
+# CHECK-ENCODING: [0x57,0x64,0x40,0xc6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 40 c6 <unknown>
+
+vwcvtu.x.x.v v8, v4, v0.t
+# CHECK-INST: vwcvtu.x.x.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x40,0xc0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 40 c0 <unknown>
+
+vwcvtu.x.x.v v8, v4
+# CHECK-INST: vwaddu.vx v8, v4, zero
+# CHECK-ENCODING: [0x57,0x64,0x40,0xc2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 40 c2 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/and.s b/llvm/test/MC/RISCV/rvv/and.s
new file mode 100644
index 0000000000000..29021df3c292b
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/and.s
@@ -0,0 +1,45 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vand.vv v8, v4, v20, v0.t
+# CHECK-INST: vand.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x24]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 24 <unknown>
+
+vand.vv v8, v4, v20
+# CHECK-INST: vand.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x26]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 26 <unknown>
+
+vand.vx v8, v4, a0, v0.t
+# CHECK-INST: vand.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x24]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 24 <unknown>
+
+vand.vx v8, v4, a0
+# CHECK-INST: vand.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x26]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 26 <unknown>
+
+vand.vi v8, v4, 15, v0.t
+# CHECK-INST: vand.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x24]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 24 <unknown>
+
+vand.vi v8, v4, 15
+# CHECK-INST: vand.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x26]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 26 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/clip.s b/llvm/test/MC/RISCV/rvv/clip.s
new file mode 100644
index 0000000000000..c079409478d24
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/clip.s
@@ -0,0 +1,81 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vnclipu.wv v8, v4, v20, v0.t
+# CHECK-INST: vnclipu.wv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xb8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a b8 <unknown>
+
+vnclipu.wv v8, v4, v20
+# CHECK-INST: vnclipu.wv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xba]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a ba <unknown>
+
+vnclipu.wx v8, v4, a0, v0.t
+# CHECK-INST: vnclipu.wx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0xb8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 b8 <unknown>
+
+vnclipu.wx v8, v4, a0
+# CHECK-INST: vnclipu.wx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0xba]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 ba <unknown>
+
+vnclipu.wi v8, v4, 31, v0.t
+# CHECK-INST: vnclipu.wi v8, v4, 31, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0xb8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f b8 <unknown>
+
+vnclipu.wi v8, v4, 31
+# CHECK-INST: vnclipu.wi v8, v4, 31
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0xba]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f ba <unknown>
+
+vnclip.wv v8, v4, v20, v0.t
+# CHECK-INST: vnclip.wv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xbc]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a bc <unknown>
+
+vnclip.wv v8, v4, v20
+# CHECK-INST: vnclip.wv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xbe]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a be <unknown>
+
+vnclip.wx v8, v4, a0, v0.t
+# CHECK-INST: vnclip.wx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0xbc]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 bc <unknown>
+
+vnclip.wx v8, v4, a0
+# CHECK-INST: vnclip.wx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0xbe]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 be <unknown>
+
+vnclip.wi v8, v4, 31, v0.t
+# CHECK-INST: vnclip.wi v8, v4, 31, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0xbc]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f bc <unknown>
+
+vnclip.wi v8, v4, 31
+# CHECK-INST: vnclip.wi v8, v4, 31
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0xbe]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f be <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/compare.s b/llvm/test/MC/RISCV/rvv/compare.s
new file mode 100644
index 0000000000000..7bf452f771a7e
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/compare.s
@@ -0,0 +1,345 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vmseq.vv v8, v4, v20, v0.t
+# CHECK-INST: vmseq.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x60]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 60 <unknown>
+
+vmseq.vv v8, v4, v20
+# CHECK-INST: vmseq.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x62]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 62 <unknown>
+
+vmseq.vx v8, v4, a0, v0.t
+# CHECK-INST: vmseq.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x60]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 60 <unknown>
+
+vmseq.vx v8, v4, a0
+# CHECK-INST: vmseq.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x62]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 62 <unknown>
+
+vmseq.vi v8, v4, 15, v0.t
+# CHECK-INST: vmseq.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x60]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 60 <unknown>
+
+vmseq.vi v8, v4, 15
+# CHECK-INST: vmseq.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x62]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 62 <unknown>
+
+vmsne.vv v8, v4, v20, v0.t
+# CHECK-INST: vmsne.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x64]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 64 <unknown>
+
+vmsne.vv v8, v4, v20
+# CHECK-INST: vmsne.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x66]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 66 <unknown>
+
+vmsne.vx v8, v4, a0, v0.t
+# CHECK-INST: vmsne.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x64]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 64 <unknown>
+
+vmsne.vx v8, v4, a0
+# CHECK-INST: vmsne.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x66]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 66 <unknown>
+
+vmsne.vi v8, v4, 15, v0.t
+# CHECK-INST: vmsne.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x64]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 64 <unknown>
+
+vmsne.vi v8, v4, 15
+# CHECK-INST: vmsne.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x66]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 66 <unknown>
+
+vmsltu.vv v8, v4, v20, v0.t
+# CHECK-INST: vmsltu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x68]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 68 <unknown>
+
+vmsltu.vv v8, v4, v20
+# CHECK-INST: vmsltu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x6a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 6a <unknown>
+
+vmsltu.vx v8, v4, a0, v0.t
+# CHECK-INST: vmsltu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x68]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 68 <unknown>
+
+vmsltu.vx v8, v4, a0
+# CHECK-INST: vmsltu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x6a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 6a <unknown>
+
+vmslt.vv v8, v4, v20, v0.t
+# CHECK-INST: vmslt.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x6c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 6c <unknown>
+
+vmslt.vv v8, v4, v20
+# CHECK-INST: vmslt.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x6e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 6e <unknown>
+
+vmslt.vx v8, v4, a0, v0.t
+# CHECK-INST: vmslt.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x6c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 6c <unknown>
+
+vmslt.vx v8, v4, a0
+# CHECK-INST: vmslt.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x6e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 6e <unknown>
+
+vmsleu.vv v8, v4, v20, v0.t
+# CHECK-INST: vmsleu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x70]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 70 <unknown>
+
+vmsleu.vv v8, v4, v20
+# CHECK-INST: vmsleu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x72]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 72 <unknown>
+
+vmsleu.vx v8, v4, a0, v0.t
+# CHECK-INST: vmsleu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x70]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 70 <unknown>
+
+vmsleu.vx v8, v4, a0
+# CHECK-INST: vmsleu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x72]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 72 <unknown>
+
+vmsleu.vi v8, v4, 15, v0.t
+# CHECK-INST: vmsleu.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x70]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 70 <unknown>
+
+vmsleu.vi v8, v4, 15
+# CHECK-INST: vmsleu.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x72]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 72 <unknown>
+
+vmsle.vv v8, v4, v20, v0.t
+# CHECK-INST: vmsle.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x74]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 74 <unknown>
+
+vmsle.vv v8, v4, v20
+# CHECK-INST: vmsle.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x76]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 76 <unknown>
+
+vmsle.vx v8, v4, a0, v0.t
+# CHECK-INST: vmsle.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x74]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 74 <unknown>
+
+vmsle.vx v8, v4, a0
+# CHECK-INST: vmsle.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x76]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 76 <unknown>
+
+vmsle.vi v8, v4, 15, v0.t
+# CHECK-INST: vmsle.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x74]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 74 <unknown>
+
+vmsle.vi v8, v4, 15
+# CHECK-INST: vmsle.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x76]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 76 <unknown>
+
+vmsgtu.vx v8, v4, a0, v0.t
+# CHECK-INST: vmsgtu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x78]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 78 <unknown>
+
+vmsgtu.vx v8, v4, a0
+# CHECK-INST: vmsgtu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x7a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 7a <unknown>
+
+vmsgtu.vi v8, v4, 15, v0.t
+# CHECK-INST: vmsgtu.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x78]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 78 <unknown>
+
+vmsgtu.vi v8, v4, 15
+# CHECK-INST: vmsgtu.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x7a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 7a <unknown>
+
+vmsgt.vx v8, v4, a0, v0.t
+# CHECK-INST: vmsgt.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x7c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 7c <unknown>
+
+vmsgt.vx v8, v4, a0
+# CHECK-INST: vmsgt.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x7e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 7e <unknown>
+
+vmsgt.vi v8, v4, 15, v0.t
+# CHECK-INST: vmsgt.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x7c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 7c <unknown>
+
+vmsgt.vi v8, v4, 15
+# CHECK-INST: vmsgt.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x7e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 7e <unknown>
+
+vmsgtu.vv v8, v20, v4, v0.t
+# CHECK-INST: vmsltu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x68]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 68 <unknown>
+
+vmsgtu.vv v8, v20, v4
+# CHECK-INST: vmsltu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x6a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 6a <unknown>
+
+vmsgt.vv v8, v20, v4, v0.t
+# CHECK-INST: vmslt.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x6c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 6c <unknown>
+
+vmsgt.vv v8, v20, v4
+# CHECK-INST: vmslt.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x6e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 6e <unknown>
+
+vmsgeu.vv v8, v20, v4, v0.t
+# CHECK-INST: vmsleu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x70]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 70 <unknown>
+
+vmsgeu.vv v8, v20, v4
+# CHECK-INST: vmsleu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x72]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 72 <unknown>
+
+vmsge.vv v8, v20, v4, v0.t
+# CHECK-INST: vmsle.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x74]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 74 <unknown>
+
+vmsge.vv v8, v20, v4
+# CHECK-INST: vmsle.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x76]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 76 <unknown>
+
+vmsltu.vi v8, v4, 16, v0.t
+# CHECK-INST: vmsleu.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x70]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 70 <unknown>
+
+vmsltu.vi v8, v4, 16
+# CHECK-INST: vmsleu.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x72]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 72 <unknown>
+
+vmslt.vi v8, v4, 16, v0.t
+# CHECK-INST: vmsle.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x74]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 74 <unknown>
+
+vmslt.vi v8, v4, 16
+# CHECK-INST: vmsle.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x76]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 76 <unknown>
+
+vmsgeu.vi v8, v4, 16, v0.t
+# CHECK-INST: vmsgtu.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x78]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 78 <unknown>
+
+vmsgeu.vi v8, v4, 16
+# CHECK-INST: vmsgtu.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x7a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 7a <unknown>
+
+vmsge.vi v8, v4, 16, v0.t
+# CHECK-INST: vmsgt.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x7c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 7c <unknown>
+
+vmsge.vi v8, v4, 16
+# CHECK-INST: vmsgt.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x7e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 7e <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/convert.s b/llvm/test/MC/RISCV/rvv/convert.s
new file mode 100644
index 0000000000000..e9f8860e97b59
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/convert.s
@@ -0,0 +1,189 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vfcvt.xu.f.v v8, v4, v0.t
+# CHECK-INST: vfcvt.xu.f.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x40,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 40 88 <unknown>
+
+vfcvt.xu.f.v v8, v4
+# CHECK-INST: vfcvt.xu.f.v v8, v4
+# CHECK-ENCODING: [0x57,0x14,0x40,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 40 8a <unknown>
+
+vfcvt.x.f.v v8, v4, v0.t
+# CHECK-INST: vfcvt.x.f.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x94,0x40,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 40 88 <unknown>
+
+vfcvt.x.f.v v8, v4
+# CHECK-INST: vfcvt.x.f.v v8, v4
+# CHECK-ENCODING: [0x57,0x94,0x40,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 40 8a <unknown>
+
+vfcvt.f.xu.v v8, v4, v0.t
+# CHECK-INST: vfcvt.f.xu.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x41,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 41 88 <unknown>
+
+vfcvt.f.xu.v v8, v4
+# CHECK-INST: vfcvt.f.xu.v v8, v4
+# CHECK-ENCODING: [0x57,0x14,0x41,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 41 8a <unknown>
+
+vfcvt.f.x.v v8, v4, v0.t
+# CHECK-INST: vfcvt.f.x.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x94,0x41,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 41 88 <unknown>
+
+vfcvt.f.x.v v8, v4
+# CHECK-INST: vfcvt.f.x.v v8, v4
+# CHECK-ENCODING: [0x57,0x94,0x41,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 41 8a <unknown>
+
+vfwcvt.xu.f.v v8, v4, v0.t
+# CHECK-INST: vfwcvt.xu.f.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x44,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 44 88 <unknown>
+
+vfwcvt.xu.f.v v8, v4
+# CHECK-INST: vfwcvt.xu.f.v v8, v4
+# CHECK-ENCODING: [0x57,0x14,0x44,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 44 8a <unknown>
+
+vfwcvt.x.f.v v8, v4, v0.t
+# CHECK-INST: vfwcvt.x.f.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x94,0x44,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 44 88 <unknown>
+
+vfwcvt.x.f.v v8, v4
+# CHECK-INST: vfwcvt.x.f.v v8, v4
+# CHECK-ENCODING: [0x57,0x94,0x44,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 44 8a <unknown>
+
+vfwcvt.f.xu.v v8, v4, v0.t
+# CHECK-INST: vfwcvt.f.xu.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x45,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 45 88 <unknown>
+
+vfwcvt.f.xu.v v8, v4
+# CHECK-INST: vfwcvt.f.xu.v v8, v4
+# CHECK-ENCODING: [0x57,0x14,0x45,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 45 8a <unknown>
+
+vfwcvt.f.x.v v8, v4, v0.t
+# CHECK-INST: vfwcvt.f.x.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x94,0x45,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 45 88 <unknown>
+
+vfwcvt.f.x.v v8, v4
+# CHECK-INST: vfwcvt.f.x.v v8, v4
+# CHECK-ENCODING: [0x57,0x94,0x45,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 45 8a <unknown>
+
+vfwcvt.f.f.v v8, v4, v0.t
+# CHECK-INST: vfwcvt.f.f.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x46,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 46 88 <unknown>
+
+vfwcvt.f.f.v v8, v4
+# CHECK-INST: vfwcvt.f.f.v v8, v4
+# CHECK-ENCODING: [0x57,0x14,0x46,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 46 8a <unknown>
+
+vfncvt.xu.f.w v8, v4, v0.t
+# CHECK-INST: vfncvt.xu.f.w v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x48,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 48 88 <unknown>
+
+vfncvt.xu.f.w v8, v4
+# CHECK-INST: vfncvt.xu.f.w v8, v4
+# CHECK-ENCODING: [0x57,0x14,0x48,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 48 8a <unknown>
+
+vfncvt.x.f.w v8, v4, v0.t
+# CHECK-INST: vfncvt.x.f.w v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x94,0x48,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 48 88 <unknown>
+
+vfncvt.x.f.w v8, v4
+# CHECK-INST: vfncvt.x.f.w v8, v4
+# CHECK-ENCODING: [0x57,0x94,0x48,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 48 8a <unknown>
+
+vfncvt.f.xu.w v8, v4, v0.t
+# CHECK-INST: vfncvt.f.xu.w v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x49,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 49 88 <unknown>
+
+vfncvt.f.xu.w v8, v4
+# CHECK-INST: vfncvt.f.xu.w v8, v4
+# CHECK-ENCODING: [0x57,0x14,0x49,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 49 8a <unknown>
+
+vfncvt.f.x.w v8, v4, v0.t
+# CHECK-INST: vfncvt.f.x.w v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x94,0x49,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 49 88 <unknown>
+
+vfncvt.f.x.w v8, v4
+# CHECK-INST: vfncvt.f.x.w v8, v4
+# CHECK-ENCODING: [0x57,0x94,0x49,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 49 8a <unknown>
+
+vfncvt.f.f.w v8, v4, v0.t
+# CHECK-INST: vfncvt.f.f.w v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 88 <unknown>
+
+vfncvt.f.f.w v8, v4
+# CHECK-INST: vfncvt.f.f.w v8, v4
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 8a <unknown>
+
+vfncvt.rod.f.f.w v8, v4, v0.t
+# CHECK-INST: vfncvt.rod.f.f.w v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x94,0x4a,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 4a 88 <unknown>
+
+vfncvt.rod.f.f.w v8, v4
+# CHECK-INST: vfncvt.rod.f.f.w v8, v4
+# CHECK-ENCODING: [0x57,0x94,0x4a,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 4a 8a <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/div.s b/llvm/test/MC/RISCV/rvv/div.s
new file mode 100644
index 0000000000000..e91e643fd8b59
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/div.s
@@ -0,0 +1,105 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vdivu.vv v8, v4, v20, v0.t
+# CHECK-INST: vdivu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x80]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 80 <unknown>
+
+vdivu.vv v8, v4, v20
+# CHECK-INST: vdivu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x82]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 82 <unknown>
+
+vdivu.vx v8, v4, a0, v0.t
+# CHECK-INST: vdivu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0x80]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 80 <unknown>
+
+vdivu.vx v8, v4, a0
+# CHECK-INST: vdivu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0x82]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 82 <unknown>
+
+vdiv.vv v8, v4, v20, v0.t
+# CHECK-INST: vdiv.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x84]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 84 <unknown>
+
+vdiv.vv v8, v4, v20
+# CHECK-INST: vdiv.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x86]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 86 <unknown>
+
+vdiv.vx v8, v4, a0, v0.t
+# CHECK-INST: vdiv.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0x84]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 84 <unknown>
+
+vdiv.vx v8, v4, a0
+# CHECK-INST: vdiv.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0x86]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 86 <unknown>
+
+vremu.vv v8, v4, v20, v0.t
+# CHECK-INST: vremu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 88 <unknown>
+
+vremu.vv v8, v4, v20
+# CHECK-INST: vremu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 8a <unknown>
+
+vremu.vx v8, v4, a0, v0.t
+# CHECK-INST: vremu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 88 <unknown>
+
+vremu.vx v8, v4, a0
+# CHECK-INST: vremu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 8a <unknown>
+
+vrem.vv v8, v4, v20, v0.t
+# CHECK-INST: vrem.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x8c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 8c <unknown>
+
+vrem.vv v8, v4, v20
+# CHECK-INST: vrem.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x8e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 8e <unknown>
+
+vrem.vx v8, v4, a0, v0.t
+# CHECK-INST: vrem.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0x8c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 8c <unknown>
+
+vrem.vx v8, v4, a0
+# CHECK-INST: vrem.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0x8e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 8e <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/fadd.s b/llvm/test/MC/RISCV/rvv/fadd.s
new file mode 100644
index 0000000000000..1c648c5a7634d
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/fadd.s
@@ -0,0 +1,81 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vfadd.vv v8, v4, v20, v0.t
+# CHECK-INST: vfadd.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x00]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 00 <unknown>
+
+vfadd.vv v8, v4, v20
+# CHECK-INST: vfadd.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x02]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 02 <unknown>
+
+vfadd.vf v8, v4, fa0, v0.t
+# CHECK-INST: vfadd.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x00]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 00 <unknown>
+
+vfadd.vf v8, v4, fa0
+# CHECK-INST: vfadd.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x02]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 02 <unknown>
+
+vfwadd.vv v8, v4, v20, v0.t
+# CHECK-INST: vfwadd.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xc0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a c0 <unknown>
+
+vfwadd.vv v8, v4, v20
+# CHECK-INST: vfwadd.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xc2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a c2 <unknown>
+
+vfwadd.vf v8, v4, fa0, v0.t
+# CHECK-INST: vfwadd.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xc0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 c0 <unknown>
+
+vfwadd.vf v8, v4, fa0
+# CHECK-INST: vfwadd.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0xc2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 c2 <unknown>
+
+vfwadd.wv v8, v4, v20, v0.t
+# CHECK-INST: vfwadd.wv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xd0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a d0 <unknown>
+
+vfwadd.wv v8, v4, v20
+# CHECK-INST: vfwadd.wv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xd2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a d2 <unknown>
+
+vfwadd.wf v8, v4, fa0, v0.t
+# CHECK-INST: vfwadd.wf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xd0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 d0 <unknown>
+
+vfwadd.wf v8, v4, fa0
+# CHECK-INST: vfwadd.wf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0xd2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 d2 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/fcompare.s b/llvm/test/MC/RISCV/rvv/fcompare.s
new file mode 100644
index 0000000000000..9ad55dc54a07e
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/fcompare.s
@@ -0,0 +1,153 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vmfeq.vv v8, v4, v20, v0.t
+# CHECK-INST: vmfeq.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x60]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 60 <unknown>
+
+vmfeq.vv v8, v4, v20
+# CHECK-INST: vmfeq.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x62]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 62 <unknown>
+
+vmfeq.vf v8, v4, fa0, v0.t
+# CHECK-INST: vmfeq.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x60]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 60 <unknown>
+
+vmfeq.vf v8, v4, fa0
+# CHECK-INST: vmfeq.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x62]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 62 <unknown>
+
+vmfne.vv v8, v4, v20, v0.t
+# CHECK-INST: vmfne.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x70]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 70 <unknown>
+
+vmfne.vv v8, v4, v20
+# CHECK-INST: vmfne.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x72]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 72 <unknown>
+
+vmfne.vf v8, v4, fa0, v0.t
+# CHECK-INST: vmfne.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x70]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 70 <unknown>
+
+vmfne.vf v8, v4, fa0
+# CHECK-INST: vmfne.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x72]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 72 <unknown>
+
+vmflt.vv v8, v4, v20, v0.t
+# CHECK-INST: vmflt.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x6c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 6c <unknown>
+
+vmflt.vv v8, v4, v20
+# CHECK-INST: vmflt.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x6e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 6e <unknown>
+
+vmflt.vf v8, v4, fa0, v0.t
+# CHECK-INST: vmflt.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x6c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 6c <unknown>
+
+vmflt.vf v8, v4, fa0
+# CHECK-INST: vmflt.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x6e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 6e <unknown>
+
+vmfle.vv v8, v4, v20, v0.t
+# CHECK-INST: vmfle.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x64]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 64 <unknown>
+
+vmfle.vv v8, v4, v20
+# CHECK-INST: vmfle.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x66]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 66 <unknown>
+
+vmfle.vf v8, v4, fa0, v0.t
+# CHECK-INST: vmfle.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x64]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 64 <unknown>
+
+vmfle.vf v8, v4, fa0
+# CHECK-INST: vmfle.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x66]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 66 <unknown>
+
+vmfgt.vf v8, v4, fa0, v0.t
+# CHECK-INST: vmfgt.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x74]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 74 <unknown>
+
+vmfgt.vf v8, v4, fa0
+# CHECK-INST: vmfgt.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x76]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 76 <unknown>
+
+vmfge.vf v8, v4, fa0, v0.t
+# CHECK-INST: vmfge.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x7c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 7c <unknown>
+
+vmfge.vf v8, v4, fa0
+# CHECK-INST: vmfge.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x7e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 7e <unknown>
+
+vmfgt.vv v8, v20, v4, v0.t
+# CHECK-INST: vmflt.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x6c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 6c <unknown>
+
+vmfgt.vv v8, v20, v4
+# CHECK-INST: vmflt.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x6e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 6e <unknown>
+
+vmfge.vv v8, v20, v4, v0.t
+# CHECK-INST: vmfle.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x64]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 64 <unknown>
+
+vmfge.vv v8, v20, v4
+# CHECK-INST: vmfle.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x66]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 66 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/fdiv.s b/llvm/test/MC/RISCV/rvv/fdiv.s
new file mode 100644
index 0000000000000..28c067ec4fcdd
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/fdiv.s
@@ -0,0 +1,45 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vfdiv.vv v8, v4, v20, v0.t
+# CHECK-INST: vfdiv.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x80]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 80 <unknown>
+
+vfdiv.vv v8, v4, v20
+# CHECK-INST: vfdiv.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x82]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 82 <unknown>
+
+vfdiv.vf v8, v4, fa0, v0.t
+# CHECK-INST: vfdiv.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x80]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 80 <unknown>
+
+vfdiv.vf v8, v4, fa0
+# CHECK-INST: vfdiv.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x82]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 82 <unknown>
+
+vfrdiv.vf v8, v4, fa0, v0.t
+# CHECK-INST: vfrdiv.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x84]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 84 <unknown>
+
+vfrdiv.vf v8, v4, fa0
+# CHECK-INST: vfrdiv.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x86]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 86 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/fmacc.s b/llvm/test/MC/RISCV/rvv/fmacc.s
new file mode 100644
index 0000000000000..4dc39e4d57b30
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/fmacc.s
@@ -0,0 +1,297 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vfmacc.vv v8, v20, v4, v0.t
+# CHECK-INST: vfmacc.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xb0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a b0 <unknown>
+
+vfmacc.vv v8, v20, v4
+# CHECK-INST: vfmacc.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xb2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a b2 <unknown>
+
+vfmacc.vf v8, fa0, v4, v0.t
+# CHECK-INST: vfmacc.vf v8, fa0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xb0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 b0 <unknown>
+
+vfmacc.vf v8, fa0, v4
+# CHECK-INST: vfmacc.vf v8, fa0, v4
+# CHECK-ENCODING: [0x57,0x54,0x45,0xb2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 b2 <unknown>
+
+vfnmacc.vv v8, v20, v4, v0.t
+# CHECK-INST: vfnmacc.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xb4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a b4 <unknown>
+
+vfnmacc.vv v8, v20, v4
+# CHECK-INST: vfnmacc.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xb6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a b6 <unknown>
+
+vfnmacc.vf v8, fa0, v4, v0.t
+# CHECK-INST: vfnmacc.vf v8, fa0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xb4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 b4 <unknown>
+
+vfnmacc.vf v8, fa0, v4
+# CHECK-INST: vfnmacc.vf v8, fa0, v4
+# CHECK-ENCODING: [0x57,0x54,0x45,0xb6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 b6 <unknown>
+
+vfmsac.vv v8, v20, v4, v0.t
+# CHECK-INST: vfmsac.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xb8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a b8 <unknown>
+
+vfmsac.vv v8, v20, v4
+# CHECK-INST: vfmsac.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xba]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a ba <unknown>
+
+vfmsac.vf v8, fa0, v4, v0.t
+# CHECK-INST: vfmsac.vf v8, fa0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xb8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 b8 <unknown>
+
+vfmsac.vf v8, fa0, v4
+# CHECK-INST: vfmsac.vf v8, fa0, v4
+# CHECK-ENCODING: [0x57,0x54,0x45,0xba]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 ba <unknown>
+
+vfnmsac.vv v8, v20, v4, v0.t
+# CHECK-INST: vfnmsac.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xbc]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a bc <unknown>
+
+vfnmsac.vv v8, v20, v4
+# CHECK-INST: vfnmsac.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xbe]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a be <unknown>
+
+vfnmsac.vf v8, fa0, v4, v0.t
+# CHECK-INST: vfnmsac.vf v8, fa0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xbc]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 bc <unknown>
+
+vfnmsac.vf v8, fa0, v4
+# CHECK-INST: vfnmsac.vf v8, fa0, v4
+# CHECK-ENCODING: [0x57,0x54,0x45,0xbe]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 be <unknown>
+
+vfmadd.vv v8, v20, v4, v0.t
+# CHECK-INST: vfmadd.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xa0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a a0 <unknown>
+
+vfmadd.vv v8, v20, v4
+# CHECK-INST: vfmadd.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xa2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a a2 <unknown>
+
+vfmadd.vf v8, fa0, v4, v0.t
+# CHECK-INST: vfmadd.vf v8, fa0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xa0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 a0 <unknown>
+
+vfmadd.vf v8, fa0, v4
+# CHECK-INST: vfmadd.vf v8, fa0, v4
+# CHECK-ENCODING: [0x57,0x54,0x45,0xa2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 a2 <unknown>
+
+vfnmadd.vv v8, v20, v4, v0.t
+# CHECK-INST: vfnmadd.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xa4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a a4 <unknown>
+
+vfnmadd.vv v8, v20, v4
+# CHECK-INST: vfnmadd.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xa6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a a6 <unknown>
+
+vfnmadd.vf v8, fa0, v4, v0.t
+# CHECK-INST: vfnmadd.vf v8, fa0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xa4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 a4 <unknown>
+
+vfnmadd.vf v8, fa0, v4
+# CHECK-INST: vfnmadd.vf v8, fa0, v4
+# CHECK-ENCODING: [0x57,0x54,0x45,0xa6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 a6 <unknown>
+
+vfmsub.vv v8, v20, v4, v0.t
+# CHECK-INST: vfmsub.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xa8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a a8 <unknown>
+
+vfmsub.vv v8, v20, v4
+# CHECK-INST: vfmsub.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xaa]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a aa <unknown>
+
+vfmsub.vf v8, fa0, v4, v0.t
+# CHECK-INST: vfmsub.vf v8, fa0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xa8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 a8 <unknown>
+
+vfmsub.vf v8, fa0, v4
+# CHECK-INST: vfmsub.vf v8, fa0, v4
+# CHECK-ENCODING: [0x57,0x54,0x45,0xaa]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 aa <unknown>
+
+vfnmsub.vv v8, v20, v4, v0.t
+# CHECK-INST: vfnmsub.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xac]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a ac <unknown>
+
+vfnmsub.vv v8, v20, v4
+# CHECK-INST: vfnmsub.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xae]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a ae <unknown>
+
+vfnmsub.vf v8, fa0, v4, v0.t
+# CHECK-INST: vfnmsub.vf v8, fa0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xac]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 ac <unknown>
+
+vfnmsub.vf v8, fa0, v4
+# CHECK-INST: vfnmsub.vf v8, fa0, v4
+# CHECK-ENCODING: [0x57,0x54,0x45,0xae]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 ae <unknown>
+
+vfwmacc.vv v8, v20, v4, v0.t
+# CHECK-INST: vfwmacc.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xf0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a f0 <unknown>
+
+vfwmacc.vv v8, v20, v4
+# CHECK-INST: vfwmacc.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xf2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a f2 <unknown>
+
+vfwmacc.vf v8, fa0, v4, v0.t
+# CHECK-INST: vfwmacc.vf v8, fa0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xf0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 f0 <unknown>
+
+vfwmacc.vf v8, fa0, v4
+# CHECK-INST: vfwmacc.vf v8, fa0, v4
+# CHECK-ENCODING: [0x57,0x54,0x45,0xf2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 f2 <unknown>
+
+vfwnmacc.vv v8, v20, v4, v0.t
+# CHECK-INST: vfwnmacc.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xf4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a f4 <unknown>
+
+vfwnmacc.vv v8, v20, v4
+# CHECK-INST: vfwnmacc.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xf6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a f6 <unknown>
+
+vfwnmacc.vf v8, fa0, v4, v0.t
+# CHECK-INST: vfwnmacc.vf v8, fa0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xf4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 f4 <unknown>
+
+vfwnmacc.vf v8, fa0, v4
+# CHECK-INST: vfwnmacc.vf v8, fa0, v4
+# CHECK-ENCODING: [0x57,0x54,0x45,0xf6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 f6 <unknown>
+
+vfwmsac.vv v8, v20, v4, v0.t
+# CHECK-INST: vfwmsac.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xf8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a f8 <unknown>
+
+vfwmsac.vv v8, v20, v4
+# CHECK-INST: vfwmsac.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xfa]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a fa <unknown>
+
+vfwmsac.vf v8, fa0, v4, v0.t
+# CHECK-INST: vfwmsac.vf v8, fa0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xf8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 f8 <unknown>
+
+vfwmsac.vf v8, fa0, v4
+# CHECK-INST: vfwmsac.vf v8, fa0, v4
+# CHECK-ENCODING: [0x57,0x54,0x45,0xfa]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 fa <unknown>
+
+vfwnmsac.vv v8, v20, v4, v0.t
+# CHECK-INST: vfwnmsac.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xfc]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a fc <unknown>
+
+vfwnmsac.vv v8, v20, v4
+# CHECK-INST: vfwnmsac.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xfe]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a fe <unknown>
+
+vfwnmsac.vf v8, fa0, v4, v0.t
+# CHECK-INST: vfwnmsac.vf v8, fa0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xfc]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 fc <unknown>
+
+vfwnmsac.vf v8, fa0, v4
+# CHECK-INST: vfwnmsac.vf v8, fa0, v4
+# CHECK-ENCODING: [0x57,0x54,0x45,0xfe]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 fe <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/fminmax.s b/llvm/test/MC/RISCV/rvv/fminmax.s
new file mode 100644
index 0000000000000..8ce3db16b3eaf
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/fminmax.s
@@ -0,0 +1,57 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vfmin.vv v8, v4, v20, v0.t
+# CHECK-INST: vfmin.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x10]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 10 <unknown>
+
+vfmin.vv v8, v4, v20
+# CHECK-INST: vfmin.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x12]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 12 <unknown>
+
+vfmin.vf v8, v4, fa0, v0.t
+# CHECK-INST: vfmin.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x10]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 10 <unknown>
+
+vfmin.vf v8, v4, fa0
+# CHECK-INST: vfmin.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x12]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 12 <unknown>
+
+vfmax.vv v8, v4, v20, v0.t
+# CHECK-INST: vfmax.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x18]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 18 <unknown>
+
+vfmax.vv v8, v4, v20
+# CHECK-INST: vfmax.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x1a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 1a <unknown>
+
+vfmax.vf v8, v4, fa0, v0.t
+# CHECK-INST: vfmax.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x18]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 18 <unknown>
+
+vfmax.vf v8, v4, fa0
+# CHECK-INST: vfmax.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x1a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 1a <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/fmul.s b/llvm/test/MC/RISCV/rvv/fmul.s
new file mode 100644
index 0000000000000..82f3bd3a4bf66
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/fmul.s
@@ -0,0 +1,57 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vfmul.vv v8, v4, v20, v0.t
+# CHECK-INST: vfmul.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x90]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 90 <unknown>
+
+vfmul.vv v8, v4, v20
+# CHECK-INST: vfmul.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x92]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 92 <unknown>
+
+vfmul.vf v8, v4, fa0, v0.t
+# CHECK-INST: vfmul.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x90]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 90 <unknown>
+
+vfmul.vf v8, v4, fa0
+# CHECK-INST: vfmul.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x92]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 92 <unknown>
+
+vfwmul.vv v8, v4, v20, v0.t
+# CHECK-INST: vfwmul.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xe0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a e0 <unknown>
+
+vfwmul.vv v8, v4, v20
+# CHECK-INST: vfwmul.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xe2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a e2 <unknown>
+
+vfwmul.vf v8, v4, fa0, v0.t
+# CHECK-INST: vfwmul.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xe0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 e0 <unknown>
+
+vfwmul.vf v8, v4, fa0
+# CHECK-INST: vfwmul.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0xe2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 e2 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/fmv.s b/llvm/test/MC/RISCV/rvv/fmv.s
new file mode 100644
index 0000000000000..74e3f0268bda2
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/fmv.s
@@ -0,0 +1,27 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vfmv.v.f v8, fa0
+# CHECK-INST: vfmv.v.f v8, fa0
+# CHECK-ENCODING: [0x57,0x54,0x05,0x5e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 05 5e <unknown>
+
+vfmv.f.s fa0, v4
+# CHECK-INST: vfmv.f.s fa0, v4
+# CHECK-ENCODING: [0x57,0x15,0x40,0x42]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 15 40 42 <unknown>
+
+vfmv.s.f v8, fa0
+# CHECK-INST: vfmv.s.f v8, fa0
+# CHECK-ENCODING: [0x57,0x54,0x05,0x42]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 05 42 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/fothers.s b/llvm/test/MC/RISCV/rvv/fothers.s
new file mode 100644
index 0000000000000..768800def4248
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/fothers.s
@@ -0,0 +1,39 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vfsqrt.v v8, v4, v0.t
+# CHECK-INST: vfsqrt.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x40,0x8c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 40 8c <unknown>
+
+vfsqrt.v v8, v4
+# CHECK-INST: vfsqrt.v v8, v4
+# CHECK-ENCODING: [0x57,0x14,0x40,0x8e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 40 8e <unknown>
+
+vfclass.v v8, v4, v0.t
+# CHECK-INST: vfclass.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x48,0x8c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 48 8c <unknown>
+
+vfclass.v v8, v4
+# CHECK-INST: vfclass.v v8, v4
+# CHECK-ENCODING: [0x57,0x14,0x48,0x8e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 48 8e <unknown>
+
+vfmerge.vfm v8, v4, fa0, v0
+# CHECK-INST: vfmerge.vfm v8, v4, fa0, v0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x5c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 5c <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/freduction.s b/llvm/test/MC/RISCV/rvv/freduction.s
new file mode 100644
index 0000000000000..2131dadff8ca8
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/freduction.s
@@ -0,0 +1,81 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vfredosum.vs v8, v4, v20, v0.t
+# CHECK-INST: vfredosum.vs v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 0c <unknown>
+
+vfredosum.vs v8, v4, v20
+# CHECK-INST: vfredosum.vs v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x0e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 0e <unknown>
+
+vfredsum.vs v8, v4, v20, v0.t
+# CHECK-INST: vfredsum.vs v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x04]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 04 <unknown>
+
+vfredsum.vs v8, v4, v20
+# CHECK-INST: vfredsum.vs v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x06]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 06 <unknown>
+
+vfredmax.vs v8, v4, v20, v0.t
+# CHECK-INST: vfredmax.vs v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x1c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 1c <unknown>
+
+vfredmax.vs v8, v4, v20
+# CHECK-INST: vfredmax.vs v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x1e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 1e <unknown>
+
+vfredmin.vs v8, v4, v20, v0.t
+# CHECK-INST: vfredmin.vs v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x14]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 14 <unknown>
+
+vfredmin.vs v8, v4, v20
+# CHECK-INST: vfredmin.vs v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x16]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 16 <unknown>
+
+vfwredosum.vs v8, v4, v20, v0.t
+# CHECK-INST: vfwredosum.vs v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xcc]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a cc <unknown>
+
+vfwredosum.vs v8, v4, v20
+# CHECK-INST: vfwredosum.vs v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xce]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a ce <unknown>
+
+vfwredsum.vs v8, v4, v20, v0.t
+# CHECK-INST: vfwredsum.vs v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xc4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a c4 <unknown>
+
+vfwredsum.vs v8, v4, v20
+# CHECK-INST: vfwredsum.vs v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xc6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a c6 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/fsub.s b/llvm/test/MC/RISCV/rvv/fsub.s
new file mode 100644
index 0000000000000..753da9a202b15
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/fsub.s
@@ -0,0 +1,93 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vfsub.vv v8, v4, v20, v0.t
+# CHECK-INST: vfsub.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x08]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 08 <unknown>
+
+vfsub.vv v8, v4, v20
+# CHECK-INST: vfsub.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x0a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 0a <unknown>
+
+vfsub.vf v8, v4, fa0, v0.t
+# CHECK-INST: vfsub.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x08]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 08 <unknown>
+
+vfsub.vf v8, v4, fa0
+# CHECK-INST: vfsub.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x0a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 0a <unknown>
+
+vfrsub.vf v8, v4, fa0, v0.t
+# CHECK-INST: vfrsub.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x9c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 9c <unknown>
+
+vfrsub.vf v8, v4, fa0
+# CHECK-INST: vfrsub.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x9e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 9e <unknown>
+
+vfwsub.vv v8, v4, v20, v0.t
+# CHECK-INST: vfwsub.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xc8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a c8 <unknown>
+
+vfwsub.vv v8, v4, v20
+# CHECK-INST: vfwsub.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xca]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a ca <unknown>
+
+vfwsub.vf v8, v4, fa0, v0.t
+# CHECK-INST: vfwsub.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xc8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 c8 <unknown>
+
+vfwsub.vf v8, v4, fa0
+# CHECK-INST: vfwsub.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0xca]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 ca <unknown>
+
+vfwsub.wv v8, v4, v20, v0.t
+# CHECK-INST: vfwsub.wv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xd8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a d8 <unknown>
+
+vfwsub.wv v8, v4, v20
+# CHECK-INST: vfwsub.wv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0xda]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a da <unknown>
+
+vfwsub.wf v8, v4, fa0, v0.t
+# CHECK-INST: vfwsub.wf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0xd8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 d8 <unknown>
+
+vfwsub.wf v8, v4, fa0
+# CHECK-INST: vfwsub.wf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0xda]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 da <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/invalid.s b/llvm/test/MC/RISCV/rvv/invalid.s
new file mode 100644
index 0000000000000..814093eba5059
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/invalid.s
@@ -0,0 +1,780 @@
+# RUN: not llvm-mc -triple=riscv64 --mattr=+experimental-v --mattr=+f %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+
+vsetvli a2, a0, e31
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8]
+
+vsetvli a2, a0, e32,m3
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8]
+
+vsetvli a2, a0, m1,e32
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8]
+
+vsetvli a2, a0, e32,m16
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8]
+
+vsetvli a2, a0, e2048,m8
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8]
+
+vsetvli a2, a0, e1,m8
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8]
+
+vadd.vv v1, v3, v2, v4.t
+# CHECK-ERROR: operand must be v0.t
+
+vadd.vv v1, v3, v2, v0
+# CHECK-ERROR: expected '.t' suffix
+
+vmslt.vi v1, v2, -16
+# CHECK-ERROR: immediate must be in the range [-15, 16]
+
+vmslt.vi v1, v2, 17
+# CHECK-ERROR: immediate must be in the range [-15, 16]
+
+viota.m v0, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: viota.m v0, v2, v0.t
+
+viota.m v2, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: viota.m v2, v2
+
+vfwcvt.xu.f.v v0, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwcvt.xu.f.v v0, v2, v0.t
+
+vfwcvt.xu.f.v v2, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwcvt.xu.f.v v2, v2
+
+vfwcvt.xu.f.v v2, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwcvt.xu.f.v v2, v3
+
+vfwcvt.x.f.v v0, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwcvt.x.f.v v0, v2, v0.t
+
+vfwcvt.x.f.v v2, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwcvt.x.f.v v2, v2
+
+vfwcvt.x.f.v v2, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwcvt.x.f.v v2, v3
+
+vfwcvt.f.xu.v v0, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwcvt.f.xu.v v0, v2, v0.t
+
+vfwcvt.f.xu.v v2, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwcvt.f.xu.v v2, v2
+
+vfwcvt.f.xu.v v2, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwcvt.f.xu.v v2, v3
+
+vfwcvt.f.x.v v0, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwcvt.f.x.v v0, v2, v0.t
+
+vfwcvt.f.x.v v2, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwcvt.f.x.v v2, v2
+
+vfwcvt.f.x.v v2, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwcvt.f.x.v v2, v3
+
+vfwcvt.f.f.v v0, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwcvt.f.f.v v0, v2, v0.t
+
+vfwcvt.f.f.v v2, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwcvt.f.f.v v2, v2
+
+vfwcvt.f.f.v v2, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwcvt.f.f.v v2, v3
+
+vslideup.vx v0, v2, a0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vslideup.vx v0, v2, a0, v0.t
+
+vslideup.vx v2, v2, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vslideup.vx v2, v2, a0
+
+vslideup.vi v0, v2, 31, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vslideup.vi v0, v2, 31, v0.t
+
+vslideup.vi v2, v2, 31
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vslideup.vi v2, v2, 31
+
+vslide1up.vx v0, v2, a0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vslide1up.vx v0, v2, a0, v0.t
+
+vslide1up.vx v2, v2, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vslide1up.vx v2, v2, a0
+
+vnsrl.wv v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vnsrl.wv v2, v2, v4
+
+vnsrl.wv v3, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vnsrl.wv v3, v2, v4
+
+vnsrl.wx v2, v2, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vnsrl.wx v2, v2, a0
+
+vnsrl.wi v2, v2, 31
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vnsrl.wi v2, v2, 31
+
+vnsra.wv v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vnsra.wv v2, v2, v4
+
+vnsra.wv v3, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vnsra.wv v3, v2, v4
+
+vnsra.wx v2, v2, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vnsra.wx v2, v2, a0
+
+vnsra.wi v2, v2, 31
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vnsra.wi v2, v2, 31
+
+vnclipu.wv v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vnclipu.wv v2, v2, v4
+
+vnclipu.wv v3, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vnclipu.wv v3, v2, v4
+
+vnclipu.wx v2, v2, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vnclipu.wx v2, v2, a0
+
+vnclipu.wi v2, v2, 31
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vnclipu.wi v2, v2, 31
+
+vnclip.wv v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vnclip.wv v2, v2, v4
+
+vnclip.wv v3, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vnclip.wv v3, v2, v4
+
+vnclip.wx v2, v2, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vnclip.wx v2, v2, a0
+
+vnclip.wi v2, v2, 31
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vnclip.wi v2, v2, 31
+
+vfncvt.xu.f.w v2, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfncvt.xu.f.w v2, v2
+
+vfncvt.xu.f.w v3, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfncvt.xu.f.w v3, v2
+
+vfncvt.x.f.w v2, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfncvt.x.f.w v2, v2
+
+vfncvt.x.f.w v3, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfncvt.x.f.w v3, v2
+
+vfncvt.f.xu.w v2, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfncvt.f.xu.w v2, v2
+
+vfncvt.f.xu.w v3, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfncvt.f.xu.w v3, v2
+
+vfncvt.f.x.w v2, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfncvt.f.x.w v2, v2
+
+vfncvt.f.x.w v3, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfncvt.f.x.w v3, v2
+
+vfncvt.f.f.w v2, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfncvt.f.f.w v2, v2
+
+vfncvt.f.f.w v3, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfncvt.f.f.w v3, v2
+
+vfncvt.rod.f.f.w v2, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfncvt.rod.f.f.w v2, v2
+
+vfncvt.rod.f.f.w v3, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfncvt.rod.f.f.w v3, v2
+
+vrgather.vv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vrgather.vv v0, v2, v4, v0.t
+
+vrgather.vv v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vrgather.vv v2, v2, v4
+
+vrgather.vx v0, v2, a0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vrgather.vx v0, v2, a0, v0.t
+
+vrgather.vx v2, v2, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vrgather.vx v2, v2, a0
+
+vrgather.vi v0, v2, 31, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vrgather.vi v0, v2, 31, v0.t
+
+vrgather.vi v2, v2, 31
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vrgather.vi v2, v2, 31
+
+vwaddu.vv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwaddu.vv v0, v2, v4, v0.t
+
+vwaddu.vv v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwaddu.vv v2, v2, v4
+
+vwaddu.vv v2, v3, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwaddu.vv v2, v3, v4
+
+vwsubu.vv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwsubu.vv v0, v2, v4, v0.t
+
+vwsubu.vv v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwsubu.vv v2, v2, v4
+
+vwsubu.vv v2, v3, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwsubu.vv v2, v3, v4
+
+vwadd.vv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwadd.vv v0, v2, v4, v0.t
+
+vwadd.vv v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwadd.vv v2, v2, v4
+
+vwadd.vv v2, v3, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwadd.vv v2, v3, v4
+
+vwsub.vv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwsub.vv v0, v2, v4, v0.t
+
+vwsub.vv v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwsub.vv v2, v2, v4
+
+vwsub.vv v2, v3, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwsub.vv v2, v3, v4
+
+vwmul.vv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwmul.vv v0, v2, v4, v0.t
+
+vwmul.vv v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmul.vv v2, v2, v4
+
+vwmul.vv v2, v3, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmul.vv v2, v3, v4
+
+vwmulu.vv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwmulu.vv v0, v2, v4, v0.t
+
+vwmulu.vv v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmulu.vv v2, v2, v4
+
+vwmulu.vv v2, v3, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmulu.vv v2, v3, v4
+
+vwmulsu.vv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwmulsu.vv v0, v2, v4, v0.t
+
+vwmulsu.vv v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmulsu.vv v2, v2, v4
+
+vwmulsu.vv v2, v3, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmulsu.vv v2, v3, v4
+
+vwmaccu.vv v0, v4, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwmaccu.vv v0, v4, v2, v0.t
+
+vwmaccu.vv v2, v4, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmaccu.vv v2, v4, v2
+
+vwmaccu.vv v2, v4, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmaccu.vv v2, v4, v3
+
+vwmacc.vv v0, v4, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwmacc.vv v0, v4, v2, v0.t
+
+vwmacc.vv v2, v4, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmacc.vv v2, v4, v2
+
+vwmacc.vv v2, v4, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmacc.vv v2, v4, v3
+
+vwmaccsu.vv v0, v4, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwmaccsu.vv v0, v4, v2, v0.t
+
+vwmaccsu.vv v2, v4, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmaccsu.vv v2, v4, v2
+
+vwmaccsu.vv v2, v4, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmaccsu.vv v2, v4, v3
+
+vfwadd.vv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwadd.vv v0, v2, v4, v0.t
+
+vfwadd.vv v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwadd.vv v2, v2, v4
+
+vfwadd.vv v2, v3, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwadd.vv v2, v3, v4
+
+vfwsub.vv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwsub.vv v0, v2, v4, v0.t
+
+vfwsub.vv v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwsub.vv v2, v2, v4
+
+vfwsub.vv v2, v3, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwsub.vv v2, v3, v4
+
+vfwmul.vv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwmul.vv v0, v2, v4, v0.t
+
+vfwmul.vv v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwmul.vv v2, v2, v4
+
+vfwmul.vv v2, v3, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwmul.vv v2, v3, v4
+
+vfwmacc.vv v0, v4, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwmacc.vv v0, v4, v2, v0.t
+
+vfwmacc.vv v2, v4, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwmacc.vv v2, v4, v2
+
+vfwmacc.vv v2, v4, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwmacc.vv v2, v4, v3
+
+vfwnmacc.vv v0, v4, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwnmacc.vv v0, v4, v2, v0.t
+
+vfwnmacc.vv v2, v4, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwnmacc.vv v2, v4, v2
+
+vfwnmacc.vv v2, v4, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwnmacc.vv v2, v4, v3
+
+vfwmsac.vv v0, v4, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwmsac.vv v0, v4, v2, v0.t
+
+vfwmsac.vv v2, v4, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwmsac.vv v2, v4, v2
+
+vfwmsac.vv v2, v4, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwmsac.vv v2, v4, v3
+
+vfwnmsac.vv v0, v4, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwnmsac.vv v0, v4, v2, v0.t
+
+vfwnmsac.vv v2, v4, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwnmsac.vv v2, v4, v2
+
+vfwnmsac.vv v2, v4, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwnmsac.vv v2, v4, v3
+
+vwaddu.vx v0, v2, a0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwaddu.vx v0, v2, a0, v0.t
+
+vwaddu.vx v2, v2, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwaddu.vx v2, v2, a0
+
+vwaddu.vx v2, v3, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwaddu.vx v2, v3, a0
+
+vwsubu.vx v0, v2, a0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwsubu.vx v0, v2, a0, v0.t
+
+vwsubu.vx v2, v2, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwsubu.vx v2, v2, a0
+
+vwsubu.vx v2, v3, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwsubu.vx v2, v3, a0
+
+vwadd.vx v0, v2, a0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwadd.vx v0, v2, a0, v0.t
+
+vwadd.vx v2, v2, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwadd.vx v2, v2, a0
+
+vwadd.vx v2, v3, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwadd.vx v2, v3, a0
+
+vwsub.vx v0, v2, a0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwsub.vx v0, v2, a0, v0.t
+
+vwsub.vx v2, v2, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwsub.vx v2, v2, a0
+
+vwsub.vx v2, v3, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwsub.vx v2, v3, a0
+
+vwmul.vx v0, v2, a0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwmul.vx v0, v2, a0, v0.t
+
+vwmul.vx v2, v2, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmul.vx v2, v2, a0
+
+vwmul.vx v2, v3, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmul.vx v2, v3, a0
+
+vwmulu.vx v0, v2, a0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwmulu.vx v0, v2, a0, v0.t
+
+vwmulu.vx v2, v2, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmulu.vx v2, v2, a0
+
+vwmulu.vx v2, v3, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmulu.vx v2, v3, a0
+
+vwmulsu.vx v0, v2, a0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwmulsu.vx v0, v2, a0, v0.t
+
+vwmulsu.vx v2, v2, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmulsu.vx v2, v2, a0
+
+vwmulsu.vx v2, v3, a0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmulsu.vx v2, v3, a0
+
+vwmaccu.vx v0, a0, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwmaccu.vx v0, a0, v2, v0.t
+
+vwmaccu.vx v2, a0, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmaccu.vx v2, a0, v2
+
+vwmaccu.vx v2, a0, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmaccu.vx v2, a0, v3
+
+vwmacc.vx v0, a0, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwmacc.vx v0, a0, v2, v0.t
+
+vwmacc.vx v2, a0, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmacc.vx v2, a0, v2
+
+vwmacc.vx v2, a0, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmacc.vx v2, a0, v3
+
+vwmaccsu.vx v0, a0, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwmaccsu.vx v0, a0, v2, v0.t
+
+vwmaccsu.vx v2, a0, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmaccsu.vx v2, a0, v2
+
+vwmaccsu.vx v2, a0, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmaccsu.vx v2, a0, v3
+
+vwmaccus.vx v0, a0, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwmaccus.vx v0, a0, v2, v0.t
+
+vwmaccus.vx v2, a0, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmaccus.vx v2, a0, v2
+
+vwmaccus.vx v2, a0, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwmaccus.vx v2, a0, v3
+
+vfwadd.vf v0, v2, fa0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwadd.vf v0, v2, fa0, v0.t
+
+vfwadd.vf v2, v2, fa0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwadd.vf v2, v2, fa0
+
+vfwadd.vf v2, v3, fa0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwadd.vf v2, v3, fa0
+
+vfwsub.vf v0, v2, fa0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwsub.vf v0, v2, fa0, v0.t
+
+vfwsub.vf v2, v2, fa0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwsub.vf v2, v2, fa0
+
+vfwsub.vf v2, v3, fa0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwsub.vf v2, v3, fa0
+
+vfwmul.vf v0, v2, fa0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwmul.vf v0, v2, fa0, v0.t
+
+vfwmul.vf v2, v2, fa0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwmul.vf v2, v2, fa0
+
+vfwmul.vf v2, v3, fa0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwmul.vf v2, v3, fa0
+
+vfwmacc.vf v0, fa0, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwmacc.vf v0, fa0, v2, v0.t
+
+vfwmacc.vf v2, fa0, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwmacc.vf v2, fa0, v2
+
+vfwmacc.vf v2, fa0, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwmacc.vf v2, fa0, v3
+
+vfwnmacc.vf v0, fa0, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwnmacc.vf v0, fa0, v2, v0.t
+
+vfwnmacc.vf v2, fa0, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwnmacc.vf v2, fa0, v2
+
+vfwnmacc.vf v2, fa0, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwnmacc.vf v2, fa0, v3
+
+vfwmsac.vf v0, fa0, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwmsac.vf v0, fa0, v2, v0.t
+
+vfwmsac.vf v2, fa0, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwmsac.vf v2, fa0, v2
+
+vfwmsac.vf v2, fa0, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwmsac.vf v2, fa0, v3
+
+vfwnmsac.vf v0, fa0, v2, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwnmsac.vf v0, fa0, v2, v0.t
+
+vfwnmsac.vf v2, fa0, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwnmsac.vf v2, fa0, v2
+
+vfwnmsac.vf v2, fa0, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwnmsac.vf v2, fa0, v3
+
+vcompress.vm v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vcompress.vm v2, v2, v4
+
+vwaddu.wv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwaddu.wv v0, v2, v4, v0.t
+
+vwaddu.wv v2, v4, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwaddu.wv v2, v4, v2
+
+vwaddu.wv v2, v4, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwaddu.wv v2, v4, v3
+
+vwsubu.wv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwsubu.wv v0, v2, v4, v0.t
+
+vwsubu.wv v2, v4, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwsubu.wv v2, v4, v2
+
+vwsubu.wv v2, v4, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwsubu.wv v2, v4, v3
+
+vwadd.wv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwadd.wv v0, v2, v4, v0.t
+
+vwadd.wv v2, v4, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwadd.wv v2, v4, v2
+
+vwadd.wv v2, v4, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwadd.wv v2, v4, v3
+
+vwsub.wv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwsub.wv v0, v2, v4, v0.t
+
+vwsub.wv v2, v4, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwsub.wv v2, v4, v2
+
+vwsub.wv v2, v4, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwsub.wv v2, v4, v3
+
+vfwadd.wv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwadd.wv v0, v2, v4, v0.t
+
+vfwadd.wv v2, v4, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwadd.wv v2, v4, v2
+
+vfwadd.wv v2, v4, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwadd.wv v2, v4, v3
+
+vfwsub.wv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwsub.wv v0, v2, v4, v0.t
+
+vfwsub.wv v2, v4, v2
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwsub.wv v2, v4, v2
+
+vfwsub.wv v2, v4, v3
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vfwsub.wv v2, v4, v3
+
+vwaddu.wx v0, v2, a0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwaddu.wx v0, v2, a0, v0.t
+
+vwsubu.wx v0, v2, a0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwsubu.wx v0, v2, a0, v0.t
+
+vwadd.wx v0, v2, a0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwadd.wx v0, v2, a0, v0.t
+
+vwsub.wx v0, v2, a0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vwsub.wx v0, v2, a0, v0.t
+
+vfwadd.wf v0, v2, fa0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwadd.wf v0, v2, fa0, v0.t
+
+vfwsub.wf v0, v2, fa0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vfwsub.wf v0, v2, fa0, v0.t
diff --git a/llvm/test/MC/RISCV/rvv/load.s b/llvm/test/MC/RISCV/rvv/load.s
new file mode 100644
index 0000000000000..c8284b561f4af
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/load.s
@@ -0,0 +1,339 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vlb.v v8, (a0), v0.t
+# CHECK-INST: vlb.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x04,0x05,0x10]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 05 10 <unknown>
+
+vlb.v v8, (a0)
+# CHECK-INST: vlb.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x04,0x05,0x12]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 05 12 <unknown>
+
+vlh.v v8, (a0), v0.t
+# CHECK-INST: vlh.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x54,0x05,0x10]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 05 10 <unknown>
+
+vlh.v v8, (a0)
+# CHECK-INST: vlh.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x54,0x05,0x12]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 05 12 <unknown>
+
+vlw.v v8, (a0), v0.t
+# CHECK-INST: vlw.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x64,0x05,0x10]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 05 10 <unknown>
+
+vlw.v v8, (a0)
+# CHECK-INST: vlw.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x64,0x05,0x12]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 05 12 <unknown>
+
+vlbu.v v8, (a0), v0.t
+# CHECK-INST: vlbu.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x04,0x05,0x00]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 05 00 <unknown>
+
+vlbu.v v8, (a0)
+# CHECK-INST: vlbu.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x04,0x05,0x02]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 05 02 <unknown>
+
+vlhu.v v8, (a0), v0.t
+# CHECK-INST: vlhu.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x54,0x05,0x00]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 05 00 <unknown>
+
+vlhu.v v8, (a0)
+# CHECK-INST: vlhu.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x54,0x05,0x02]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 05 02 <unknown>
+
+vlwu.v v8, (a0), v0.t
+# CHECK-INST: vlwu.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x64,0x05,0x00]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 05 00 <unknown>
+
+vlwu.v v8, (a0)
+# CHECK-INST: vlwu.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x64,0x05,0x02]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 05 02 <unknown>
+
+vlbff.v v8, (a0), v0.t
+# CHECK-INST: vlbff.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x04,0x05,0x11]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 05 11 <unknown>
+
+vlbff.v v8, (a0)
+# CHECK-INST: vlbff.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x04,0x05,0x13]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 05 13 <unknown>
+
+vlhff.v v8, (a0), v0.t
+# CHECK-INST: vlhff.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x54,0x05,0x11]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 05 11 <unknown>
+
+vlhff.v v8, (a0)
+# CHECK-INST: vlhff.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x54,0x05,0x13]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 05 13 <unknown>
+
+vlwff.v v8, (a0), v0.t
+# CHECK-INST: vlwff.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x64,0x05,0x11]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 05 11 <unknown>
+
+vlwff.v v8, (a0)
+# CHECK-INST: vlwff.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x64,0x05,0x13]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 05 13 <unknown>
+
+vlbuff.v v8, (a0), v0.t
+# CHECK-INST: vlbuff.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x04,0x05,0x01]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 05 01 <unknown>
+
+vlbuff.v v8, (a0)
+# CHECK-INST: vlbuff.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x04,0x05,0x03]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 05 03 <unknown>
+
+vlhuff.v v8, (a0), v0.t
+# CHECK-INST: vlhuff.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x54,0x05,0x01]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 05 01 <unknown>
+
+vlhuff.v v8, (a0)
+# CHECK-INST: vlhuff.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x54,0x05,0x03]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 05 03 <unknown>
+
+vlwuff.v v8, (a0), v0.t
+# CHECK-INST: vlwuff.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x64,0x05,0x01]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 05 01 <unknown>
+
+vlwuff.v v8, (a0)
+# CHECK-INST: vlwuff.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x64,0x05,0x03]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 05 03 <unknown>
+
+vleff.v v8, (a0), v0.t
+# CHECK-INST: vleff.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x74,0x05,0x01]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 74 05 01 <unknown>
+
+vleff.v v8, (a0)
+# CHECK-INST: vleff.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x74,0x05,0x03]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 74 05 03 <unknown>
+
+vlsb.v v8, (a0), a1, v0.t
+# CHECK-INST: vlsb.v v8, (a0), a1, v0.t
+# CHECK-ENCODING: [0x07,0x04,0xb5,0x18]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 b5 18 <unknown>
+
+vlsb.v v8, (a0), a1
+# CHECK-INST: vlsb.v v8, (a0), a1
+# CHECK-ENCODING: [0x07,0x04,0xb5,0x1a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 b5 1a <unknown>
+
+vlsh.v v8, (a0), a1, v0.t
+# CHECK-INST: vlsh.v v8, (a0), a1, v0.t
+# CHECK-ENCODING: [0x07,0x54,0xb5,0x18]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 b5 18 <unknown>
+
+vlsh.v v8, (a0), a1
+# CHECK-INST: vlsh.v v8, (a0), a1
+# CHECK-ENCODING: [0x07,0x54,0xb5,0x1a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 b5 1a <unknown>
+
+vlsw.v v8, (a0), a1, v0.t
+# CHECK-INST: vlsw.v v8, (a0), a1, v0.t
+# CHECK-ENCODING: [0x07,0x64,0xb5,0x18]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 b5 18 <unknown>
+
+vlsw.v v8, (a0), a1
+# CHECK-INST: vlsw.v v8, (a0), a1
+# CHECK-ENCODING: [0x07,0x64,0xb5,0x1a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 b5 1a <unknown>
+
+vlsbu.v v8, (a0), a1, v0.t
+# CHECK-INST: vlsbu.v v8, (a0), a1, v0.t
+# CHECK-ENCODING: [0x07,0x04,0xb5,0x08]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 b5 08 <unknown>
+
+vlsbu.v v8, (a0), a1
+# CHECK-INST: vlsbu.v v8, (a0), a1
+# CHECK-ENCODING: [0x07,0x04,0xb5,0x0a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 b5 0a <unknown>
+
+vlshu.v v8, (a0), a1, v0.t
+# CHECK-INST: vlshu.v v8, (a0), a1, v0.t
+# CHECK-ENCODING: [0x07,0x54,0xb5,0x08]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 b5 08 <unknown>
+
+vlshu.v v8, (a0), a1
+# CHECK-INST: vlshu.v v8, (a0), a1
+# CHECK-ENCODING: [0x07,0x54,0xb5,0x0a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 b5 0a <unknown>
+
+vlswu.v v8, (a0), a1, v0.t
+# CHECK-INST: vlswu.v v8, (a0), a1, v0.t
+# CHECK-ENCODING: [0x07,0x64,0xb5,0x08]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 b5 08 <unknown>
+
+vlswu.v v8, (a0), a1
+# CHECK-INST: vlswu.v v8, (a0), a1
+# CHECK-ENCODING: [0x07,0x64,0xb5,0x0a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 b5 0a <unknown>
+
+vlse.v v8, (a0), a1, v0.t
+# CHECK-INST: vlse.v v8, (a0), a1, v0.t
+# CHECK-ENCODING: [0x07,0x74,0xb5,0x08]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 74 b5 08 <unknown>
+
+vlse.v v8, (a0), a1
+# CHECK-INST: vlse.v v8, (a0), a1
+# CHECK-ENCODING: [0x07,0x74,0xb5,0x0a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 74 b5 0a <unknown>
+
+vlxb.v v8, (a0), v4, v0.t
+# CHECK-INST: vlxb.v v8, (a0), v4, v0.t
+# CHECK-ENCODING: [0x07,0x04,0x45,0x1c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 45 1c <unknown>
+
+vlxb.v v8, (a0), v4
+# CHECK-INST: vlxb.v v8, (a0), v4
+# CHECK-ENCODING: [0x07,0x04,0x45,0x1e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 45 1e <unknown>
+
+vlxh.v v8, (a0), v4, v0.t
+# CHECK-INST: vlxh.v v8, (a0), v4, v0.t
+# CHECK-ENCODING: [0x07,0x54,0x45,0x1c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 45 1c <unknown>
+
+vlxh.v v8, (a0), v4
+# CHECK-INST: vlxh.v v8, (a0), v4
+# CHECK-ENCODING: [0x07,0x54,0x45,0x1e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 45 1e <unknown>
+
+vlxw.v v8, (a0), v4, v0.t
+# CHECK-INST: vlxw.v v8, (a0), v4, v0.t
+# CHECK-ENCODING: [0x07,0x64,0x45,0x1c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 45 1c <unknown>
+
+vlxw.v v8, (a0), v4
+# CHECK-INST: vlxw.v v8, (a0), v4
+# CHECK-ENCODING: [0x07,0x64,0x45,0x1e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 45 1e <unknown>
+
+vlxbu.v v8, (a0), v4, v0.t
+# CHECK-INST: vlxbu.v v8, (a0), v4, v0.t
+# CHECK-ENCODING: [0x07,0x04,0x45,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 45 0c <unknown>
+
+vlxbu.v v8, (a0), v4
+# CHECK-INST: vlxbu.v v8, (a0), v4
+# CHECK-ENCODING: [0x07,0x04,0x45,0x0e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 45 0e <unknown>
+
+vlxhu.v v8, (a0), v4, v0.t
+# CHECK-INST: vlxhu.v v8, (a0), v4, v0.t
+# CHECK-ENCODING: [0x07,0x54,0x45,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 45 0c <unknown>
+
+vlxhu.v v8, (a0), v4
+# CHECK-INST: vlxhu.v v8, (a0), v4
+# CHECK-ENCODING: [0x07,0x54,0x45,0x0e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 45 0e <unknown>
+
+vlxwu.v v8, (a0), v4, v0.t
+# CHECK-INST: vlxwu.v v8, (a0), v4, v0.t
+# CHECK-ENCODING: [0x07,0x64,0x45,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 45 0c <unknown>
+
+vlxwu.v v8, (a0), v4
+# CHECK-INST: vlxwu.v v8, (a0), v4
+# CHECK-ENCODING: [0x07,0x64,0x45,0x0e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 45 0e <unknown>
+
+vlxe.v v8, (a0), v4, v0.t
+# CHECK-INST: vlxe.v v8, (a0), v4, v0.t
+# CHECK-ENCODING: [0x07,0x74,0x45,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 74 45 0c <unknown>
+
+vlxe.v v8, (a0), v4
+# CHECK-INST: vlxe.v v8, (a0), v4
+# CHECK-ENCODING: [0x07,0x74,0x45,0x0e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 74 45 0e <unknown>
+
+vl1r.v v8, (a0)
+# CHECK-INST: vl1r.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x74,0x85,0x02]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 74 85 02 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/macc.s b/llvm/test/MC/RISCV/rvv/macc.s
new file mode 100644
index 0000000000000..1092ff5df44d8
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/macc.s
@@ -0,0 +1,189 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vmacc.vv v8, v20, v4, v0.t
+# CHECK-INST: vmacc.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xb4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a b4 <unknown>
+
+vmacc.vv v8, v20, v4
+# CHECK-INST: vmacc.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xb6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a b6 <unknown>
+
+vmacc.vx v8, a0, v4, v0.t
+# CHECK-INST: vmacc.vx v8, a0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xb4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 b4 <unknown>
+
+vmacc.vx v8, a0, v4
+# CHECK-INST: vmacc.vx v8, a0, v4
+# CHECK-ENCODING: [0x57,0x64,0x45,0xb6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 b6 <unknown>
+
+vnmsac.vv v8, v20, v4, v0.t
+# CHECK-INST: vnmsac.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xbc]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a bc <unknown>
+
+vnmsac.vv v8, v20, v4
+# CHECK-INST: vnmsac.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xbe]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a be <unknown>
+
+vnmsac.vx v8, a0, v4, v0.t
+# CHECK-INST: vnmsac.vx v8, a0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xbc]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 bc <unknown>
+
+vnmsac.vx v8, a0, v4
+# CHECK-INST: vnmsac.vx v8, a0, v4
+# CHECK-ENCODING: [0x57,0x64,0x45,0xbe]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 be <unknown>
+
+vmadd.vv v8, v20, v4, v0.t
+# CHECK-INST: vmadd.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xa4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a a4 <unknown>
+
+vmadd.vv v8, v20, v4
+# CHECK-INST: vmadd.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xa6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a a6 <unknown>
+
+vmadd.vx v8, a0, v4, v0.t
+# CHECK-INST: vmadd.vx v8, a0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xa4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 a4 <unknown>
+
+vmadd.vx v8, a0, v4
+# CHECK-INST: vmadd.vx v8, a0, v4
+# CHECK-ENCODING: [0x57,0x64,0x45,0xa6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 a6 <unknown>
+
+vnmsub.vv v8, v20, v4, v0.t
+# CHECK-INST: vnmsub.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xac]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a ac <unknown>
+
+vnmsub.vv v8, v20, v4
+# CHECK-INST: vnmsub.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xae]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a ae <unknown>
+
+vnmsub.vx v8, a0, v4, v0.t
+# CHECK-INST: vnmsub.vx v8, a0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xac]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 ac <unknown>
+
+vnmsub.vx v8, a0, v4
+# CHECK-INST: vnmsub.vx v8, a0, v4
+# CHECK-ENCODING: [0x57,0x64,0x45,0xae]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 ae <unknown>
+
+vwmaccu.vv v8, v20, v4, v0.t
+# CHECK-INST: vwmaccu.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xf0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a f0 <unknown>
+
+vwmaccu.vv v8, v20, v4
+# CHECK-INST: vwmaccu.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xf2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a f2 <unknown>
+
+vwmaccu.vx v8, a0, v4, v0.t
+# CHECK-INST: vwmaccu.vx v8, a0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xf0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 f0 <unknown>
+
+vwmaccu.vx v8, a0, v4
+# CHECK-INST: vwmaccu.vx v8, a0, v4
+# CHECK-ENCODING: [0x57,0x64,0x45,0xf2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 f2 <unknown>
+
+vwmacc.vv v8, v20, v4, v0.t
+# CHECK-INST: vwmacc.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xf4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a f4 <unknown>
+
+vwmacc.vv v8, v20, v4
+# CHECK-INST: vwmacc.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xf6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a f6 <unknown>
+
+vwmacc.vx v8, a0, v4, v0.t
+# CHECK-INST: vwmacc.vx v8, a0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xf4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 f4 <unknown>
+
+vwmacc.vx v8, a0, v4
+# CHECK-INST: vwmacc.vx v8, a0, v4
+# CHECK-ENCODING: [0x57,0x64,0x45,0xf6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 f6 <unknown>
+
+vwmaccsu.vv v8, v20, v4, v0.t
+# CHECK-INST: vwmaccsu.vv v8, v20, v4, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xfc]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a fc <unknown>
+
+vwmaccsu.vv v8, v20, v4
+# CHECK-INST: vwmaccsu.vv v8, v20, v4
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xfe]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a fe <unknown>
+
+vwmaccsu.vx v8, a0, v4, v0.t
+# CHECK-INST: vwmaccsu.vx v8, a0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xfc]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 fc <unknown>
+
+vwmaccsu.vx v8, a0, v4
+# CHECK-INST: vwmaccsu.vx v8, a0, v4
+# CHECK-ENCODING: [0x57,0x64,0x45,0xfe]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 fe <unknown>
+
+vwmaccus.vx v8, a0, v4, v0.t
+# CHECK-INST: vwmaccus.vx v8, a0, v4, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xf8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 f8 <unknown>
+
+vwmaccus.vx v8, a0, v4
+# CHECK-INST: vwmaccus.vx v8, a0, v4
+# CHECK-ENCODING: [0x57,0x64,0x45,0xfa]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 fa <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/mask.s b/llvm/test/MC/RISCV/rvv/mask.s
new file mode 100644
index 0000000000000..d2157bc215bf3
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/mask.s
@@ -0,0 +1,165 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vmand.mm v8, v4, v20
+# CHECK-INST: vmand.mm v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x66]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 66 <unknown>
+
+vmnand.mm v8, v4, v20
+# CHECK-INST: vmnand.mm v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x76]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 76 <unknown>
+
+vmandnot.mm v8, v4, v20
+# CHECK-INST: vmandnot.mm v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x62]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 62 <unknown>
+
+vmxor.mm v8, v4, v20
+# CHECK-INST: vmxor.mm v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x6e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 6e <unknown>
+
+vmor.mm v8, v4, v20
+# CHECK-INST: vmor.mm v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x6a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 6a <unknown>
+
+vmnor.mm v8, v4, v20
+# CHECK-INST: vmnor.mm v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x7a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 7a <unknown>
+
+vmornot.mm v8, v4, v20
+# CHECK-INST: vmornot.mm v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x72]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 72 <unknown>
+
+vmxnor.mm v8, v4, v20
+# CHECK-INST: vmxnor.mm v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x7e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 7e <unknown>
+
+vpopc.m a2, v4, v0.t
+# CHECK-INST: vpopc.m a2, v4, v0.t
+# CHECK-ENCODING: [0x57,0x26,0x48,0x40]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 26 48 40 <unknown>
+
+vpopc.m a2, v4
+# CHECK-INST: vpopc.m a2, v4
+# CHECK-ENCODING: [0x57,0x26,0x48,0x42]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 26 48 42 <unknown>
+
+vfirst.m a2, v4, v0.t
+# CHECK-INST: vfirst.m a2, v4, v0.t
+# CHECK-ENCODING: [0x57,0xa6,0x48,0x40]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 a6 48 40 <unknown>
+
+vfirst.m a2, v4
+# CHECK-INST: vfirst.m a2, v4
+# CHECK-ENCODING: [0x57,0xa6,0x48,0x42]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 a6 48 42 <unknown>
+
+vmsbf.m v8, v4, v0.t
+# CHECK-INST: vmsbf.m v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0xa4,0x40,0x50]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 a4 40 50 <unknown>
+
+vmsbf.m v8, v4
+# CHECK-INST: vmsbf.m v8, v4
+# CHECK-ENCODING: [0x57,0xa4,0x40,0x52]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 a4 40 52 <unknown>
+
+vmsif.m v8, v4, v0.t
+# CHECK-INST: vmsif.m v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0xa4,0x41,0x50]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 a4 41 50 <unknown>
+
+vmsif.m v8, v4
+# CHECK-INST: vmsif.m v8, v4
+# CHECK-ENCODING: [0x57,0xa4,0x41,0x52]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 a4 41 52 <unknown>
+
+vmsof.m v8, v4, v0.t
+# CHECK-INST: vmsof.m v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x41,0x50]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 41 50 <unknown>
+
+vmsof.m v8, v4
+# CHECK-INST: vmsof.m v8, v4
+# CHECK-ENCODING: [0x57,0x24,0x41,0x52]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 41 52 <unknown>
+
+viota.m v8, v4, v0.t
+# CHECK-INST: viota.m v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x48,0x50]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 48 50 <unknown>
+
+viota.m v8, v4
+# CHECK-INST: viota.m v8, v4
+# CHECK-ENCODING: [0x57,0x24,0x48,0x52]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 48 52 <unknown>
+
+vid.v v8, v0.t
+# CHECK-INST: vid.v v8, v0.t
+# CHECK-ENCODING: [0x57,0xa4,0x08,0x50]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 a4 08 50 <unknown>
+
+vid.v v8
+# CHECK-INST: vid.v v8
+# CHECK-ENCODING: [0x57,0xa4,0x08,0x52]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 a4 08 52 <unknown>
+
+vmcpy.m v8, v4
+# CHECK-INST: vmcpy.m v8, v4
+# CHECK-ENCODING: [0x57,0x24,0x42,0x66]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 42 66 <unknown>
+
+vmclr.m v8
+# CHECK-INST: vmclr.m v8
+# CHECK-ENCODING: [0x57,0x24,0x84,0x6e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 84 6e <unknown>
+
+vmset.m v8
+# CHECK-INST: vmset.m v8
+# CHECK-ENCODING: [0x57,0x24,0x84,0x7e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 84 7e <unknown>
+
+vmnot.m v8, v4
+# CHECK-INST: vmnot.m v8, v4
+# CHECK-ENCODING: [0x57,0x24,0x42,0x76]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 42 76 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/minmax.s b/llvm/test/MC/RISCV/rvv/minmax.s
new file mode 100644
index 0000000000000..7759baf350487
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/minmax.s
@@ -0,0 +1,105 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vminu.vv v8, v4, v20, v0.t
+# CHECK-INST: vminu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x10]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 10 <unknown>
+
+vminu.vv v8, v4, v20
+# CHECK-INST: vminu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x12]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 12 <unknown>
+
+vminu.vx v8, v4, a0, v0.t
+# CHECK-INST: vminu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x10]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 10 <unknown>
+
+vminu.vx v8, v4, a0
+# CHECK-INST: vminu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x12]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 12 <unknown>
+
+vmin.vv v8, v4, v20, v0.t
+# CHECK-INST: vmin.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x14]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 14 <unknown>
+
+vmin.vv v8, v4, v20
+# CHECK-INST: vmin.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x16]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 16 <unknown>
+
+vmin.vx v8, v4, a0, v0.t
+# CHECK-INST: vmin.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x14]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 14 <unknown>
+
+vmin.vx v8, v4, a0
+# CHECK-INST: vmin.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x16]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 16 <unknown>
+
+vmaxu.vv v8, v4, v20, v0.t
+# CHECK-INST: vmaxu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x18]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 18 <unknown>
+
+vmaxu.vv v8, v4, v20
+# CHECK-INST: vmaxu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x1a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 1a <unknown>
+
+vmaxu.vx v8, v4, a0, v0.t
+# CHECK-INST: vmaxu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x18]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 18 <unknown>
+
+vmaxu.vx v8, v4, a0
+# CHECK-INST: vmaxu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x1a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 1a <unknown>
+
+vmax.vv v8, v4, v20, v0.t
+# CHECK-INST: vmax.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x1c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 1c <unknown>
+
+vmax.vv v8, v4, v20
+# CHECK-INST: vmax.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x1e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 1e <unknown>
+
+vmax.vx v8, v4, a0, v0.t
+# CHECK-INST: vmax.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x1c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 1c <unknown>
+
+vmax.vx v8, v4, a0
+# CHECK-INST: vmax.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x1e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 1e <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/mul.s b/llvm/test/MC/RISCV/rvv/mul.s
new file mode 100644
index 0000000000000..36e3c32b2c8ec
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/mul.s
@@ -0,0 +1,201 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vmul.vv v8, v4, v20, v0.t
+# CHECK-INST: vmul.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x94]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 94 <unknown>
+
+vmul.vv v8, v4, v20
+# CHECK-INST: vmul.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x96]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 96 <unknown>
+
+vmul.vx v8, v4, a0, v0.t
+# CHECK-INST: vmul.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0x94]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 94 <unknown>
+
+vmul.vx v8, v4, a0
+# CHECK-INST: vmul.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0x96]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 96 <unknown>
+
+vmulh.vv v8, v4, v20, v0.t
+# CHECK-INST: vmulh.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x9c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 9c <unknown>
+
+vmulh.vv v8, v4, v20
+# CHECK-INST: vmulh.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x9e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 9e <unknown>
+
+vmulh.vx v8, v4, a0, v0.t
+# CHECK-INST: vmulh.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0x9c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 9c <unknown>
+
+vmulh.vx v8, v4, a0
+# CHECK-INST: vmulh.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0x9e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 9e <unknown>
+
+vmulhu.vv v8, v4, v20, v0.t
+# CHECK-INST: vmulhu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x90]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 90 <unknown>
+
+vmulhu.vv v8, v4, v20
+# CHECK-INST: vmulhu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x92]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 92 <unknown>
+
+vmulhu.vx v8, v4, a0, v0.t
+# CHECK-INST: vmulhu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0x90]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 90 <unknown>
+
+vmulhu.vx v8, v4, a0
+# CHECK-INST: vmulhu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0x92]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 92 <unknown>
+
+vmulhsu.vv v8, v4, v20, v0.t
+# CHECK-INST: vmulhsu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x98]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 98 <unknown>
+
+vmulhsu.vv v8, v4, v20
+# CHECK-INST: vmulhsu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x9a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 9a <unknown>
+
+vmulhsu.vx v8, v4, a0, v0.t
+# CHECK-INST: vmulhsu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0x98]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 98 <unknown>
+
+vmulhsu.vx v8, v4, a0
+# CHECK-INST: vmulhsu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0x9a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 9a <unknown>
+
+vwmul.vv v8, v4, v20, v0.t
+# CHECK-INST: vwmul.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xec]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a ec <unknown>
+
+vwmul.vv v8, v4, v20
+# CHECK-INST: vwmul.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xee]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a ee <unknown>
+
+vwmul.vx v8, v4, a0, v0.t
+# CHECK-INST: vwmul.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xec]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 ec <unknown>
+
+vwmul.vx v8, v4, a0
+# CHECK-INST: vwmul.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0xee]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 ee <unknown>
+
+vwmulu.vv v8, v4, v20, v0.t
+# CHECK-INST: vwmulu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xe0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a e0 <unknown>
+
+vwmulu.vv v8, v4, v20
+# CHECK-INST: vwmulu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xe2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a e2 <unknown>
+
+vwmulu.vx v8, v4, a0, v0.t
+# CHECK-INST: vwmulu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xe0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 e0 <unknown>
+
+vwmulu.vx v8, v4, a0
+# CHECK-INST: vwmulu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0xe2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 e2 <unknown>
+
+vwmulsu.vv v8, v4, v20, v0.t
+# CHECK-INST: vwmulsu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xe8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a e8 <unknown>
+
+vwmulsu.vv v8, v4, v20
+# CHECK-INST: vwmulsu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xea]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a ea <unknown>
+
+vwmulsu.vx v8, v4, a0, v0.t
+# CHECK-INST: vwmulsu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xe8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 e8 <unknown>
+
+vwmulsu.vx v8, v4, a0
+# CHECK-INST: vwmulsu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0xea]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 ea <unknown>
+
+vsmul.vv v8, v4, v20, v0.t
+# CHECK-INST: vsmul.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x9c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 9c <unknown>
+
+vsmul.vv v8, v4, v20
+# CHECK-INST: vsmul.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x9e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 9e <unknown>
+
+vsmul.vx v8, v4, a0, v0.t
+# CHECK-INST: vsmul.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x9c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 9c <unknown>
+
+vsmul.vx v8, v4, a0
+# CHECK-INST: vsmul.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x9e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 9e <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/mv.s b/llvm/test/MC/RISCV/rvv/mv.s
new file mode 100644
index 0000000000000..3bfcce5426d73
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/mv.s
@@ -0,0 +1,63 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vmv.v.v v8, v20
+# CHECK-INST: vmv.v.v v8, v20
+# CHECK-ENCODING: [0x57,0x04,0x0a,0x5e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 0a 5e <unknown>
+
+vmv.v.x v8, a0
+# CHECK-INST: vmv.v.x v8, a0
+# CHECK-ENCODING: [0x57,0x44,0x05,0x5e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 05 5e <unknown>
+
+vmv.v.i v8, 15
+# CHECK-INST: vmv.v.i v8, 15
+# CHECK-ENCODING: [0x57,0xb4,0x07,0x5e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 07 5e <unknown>
+
+vmv.x.s a2, v4
+# CHECK-INST: vmv.x.s a2, v4
+# CHECK-ENCODING: [0x57,0x26,0x40,0x42]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 26 40 42 <unknown>
+
+vmv.s.x v8, a0
+# CHECK-INST: vmv.s.x v8, a0
+# CHECK-ENCODING: [0x57,0x64,0x05,0x42]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 05 42 <unknown>
+
+vmv1r.v v8, v4
+# CHECK-INST: vmv1r.v v8, v4
+# CHECK-ENCODING: [0x57,0x34,0x40,0x9e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 34 40 9e <unknown>
+
+vmv2r.v v8, v4
+# CHECK-INST: vmv2r.v v8, v4
+# CHECK-ENCODING: [0x57,0xb4,0x40,0x9e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 40 9e <unknown>
+
+vmv4r.v v8, v4
+# CHECK-INST: vmv4r.v v8, v4
+# CHECK-ENCODING: [0x57,0xb4,0x41,0x9e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 41 9e <unknown>
+
+vmv8r.v v8, v24
+# CHECK-INST: vmv8r.v v8, v24
+# CHECK-ENCODING: [0x57,0xb4,0x83,0x9f]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 83 9f <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/or.s b/llvm/test/MC/RISCV/rvv/or.s
new file mode 100644
index 0000000000000..2201712aed523
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/or.s
@@ -0,0 +1,45 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vor.vv v8, v4, v20, v0.t
+# CHECK-INST: vor.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x28]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 28 <unknown>
+
+vor.vv v8, v4, v20
+# CHECK-INST: vor.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x2a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 2a <unknown>
+
+vor.vx v8, v4, a0, v0.t
+# CHECK-INST: vor.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x28]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 28 <unknown>
+
+vor.vx v8, v4, a0
+# CHECK-INST: vor.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x2a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 2a <unknown>
+
+vor.vi v8, v4, 15, v0.t
+# CHECK-INST: vor.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x28]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 28 <unknown>
+
+vor.vi v8, v4, 15
+# CHECK-INST: vor.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x2a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 2a <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/others.s b/llvm/test/MC/RISCV/rvv/others.s
new file mode 100644
index 0000000000000..727230090051e
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/others.s
@@ -0,0 +1,141 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vmerge.vvm v8, v4, v20, v0
+# CHECK-INST: vmerge.vvm v8, v4, v20, v0
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x5c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 5c <unknown>
+
+vmerge.vxm v8, v4, a0, v0
+# CHECK-INST: vmerge.vxm v8, v4, a0, v0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x5c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 5c <unknown>
+
+vmerge.vim v8, v4, 15, v0
+# CHECK-INST: vmerge.vim v8, v4, 15, v0
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x5c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 5c <unknown>
+
+vslideup.vx v8, v4, a0, v0.t
+# CHECK-INST: vslideup.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x38]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 38 <unknown>
+
+vslideup.vx v8, v4, a0
+# CHECK-INST: vslideup.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x3a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 3a <unknown>
+
+vslideup.vi v8, v4, 31, v0.t
+# CHECK-INST: vslideup.vi v8, v4, 31, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0x38]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f 38 <unknown>
+
+vslideup.vi v8, v4, 31
+# CHECK-INST: vslideup.vi v8, v4, 31
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0x3a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f 3a <unknown>
+
+vslidedown.vx v8, v4, a0, v0.t
+# CHECK-INST: vslidedown.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x3c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 3c <unknown>
+
+vslidedown.vx v8, v4, a0
+# CHECK-INST: vslidedown.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x3e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 3e <unknown>
+
+vslidedown.vi v8, v4, 31, v0.t
+# CHECK-INST: vslidedown.vi v8, v4, 31, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0x3c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f 3c <unknown>
+
+vslidedown.vi v8, v4, 31
+# CHECK-INST: vslidedown.vi v8, v4, 31
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0x3e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f 3e <unknown>
+
+vslide1up.vx v8, v4, a0, v0.t
+# CHECK-INST: vslide1up.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0x38]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 38 <unknown>
+
+vslide1up.vx v8, v4, a0
+# CHECK-INST: vslide1up.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0x3a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 3a <unknown>
+
+vslide1down.vx v8, v4, a0, v0.t
+# CHECK-INST: vslide1down.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0x3c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 3c <unknown>
+
+vslide1down.vx v8, v4, a0
+# CHECK-INST: vslide1down.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0x3e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 3e <unknown>
+
+vrgather.vv v8, v4, v20, v0.t
+# CHECK-INST: vrgather.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x30]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 30 <unknown>
+
+vrgather.vv v8, v4, v20
+# CHECK-INST: vrgather.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x32]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 32 <unknown>
+
+vrgather.vx v8, v4, a0, v0.t
+# CHECK-INST: vrgather.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x30]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 30 <unknown>
+
+vrgather.vx v8, v4, a0
+# CHECK-INST: vrgather.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x32]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 32 <unknown>
+
+vrgather.vi v8, v4, 31, v0.t
+# CHECK-INST: vrgather.vi v8, v4, 31, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0x30]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f 30 <unknown>
+
+vrgather.vi v8, v4, 31
+# CHECK-INST: vrgather.vi v8, v4, 31
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0x32]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f 32 <unknown>
+
+vcompress.vm v8, v4, v20
+# CHECK-INST: vcompress.vm v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x5e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 5e <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/reduction.s b/llvm/test/MC/RISCV/rvv/reduction.s
new file mode 100644
index 0000000000000..dc5adec0a5668
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/reduction.s
@@ -0,0 +1,129 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vredsum.vs v8, v4, v20, v0.t
+# CHECK-INST: vredsum.vs v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x00]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 00 <unknown>
+
+vredsum.vs v8, v4, v20
+# CHECK-INST: vredsum.vs v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x02]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 02 <unknown>
+
+vredmaxu.vs v8, v4, v20, v0.t
+# CHECK-INST: vredmaxu.vs v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x18]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 18 <unknown>
+
+vredmaxu.vs v8, v4, v20
+# CHECK-INST: vredmaxu.vs v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x1a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 1a <unknown>
+
+vredmax.vs v8, v4, v20, v0.t
+# CHECK-INST: vredmax.vs v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x1c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 1c <unknown>
+
+vredmax.vs v8, v4, v20
+# CHECK-INST: vredmax.vs v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x1e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 1e <unknown>
+
+vredminu.vs v8, v4, v20, v0.t
+# CHECK-INST: vredminu.vs v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x10]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 10 <unknown>
+
+vredminu.vs v8, v4, v20
+# CHECK-INST: vredminu.vs v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x12]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 12 <unknown>
+
+vredmin.vs v8, v4, v20, v0.t
+# CHECK-INST: vredmin.vs v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x14]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 14 <unknown>
+
+vredmin.vs v8, v4, v20
+# CHECK-INST: vredmin.vs v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x16]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 16 <unknown>
+
+vredand.vs v8, v4, v20, v0.t
+# CHECK-INST: vredand.vs v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x04]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 04 <unknown>
+
+vredand.vs v8, v4, v20
+# CHECK-INST: vredand.vs v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x06]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 06 <unknown>
+
+vredor.vs v8, v4, v20, v0.t
+# CHECK-INST: vredor.vs v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x08]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 08 <unknown>
+
+vredor.vs v8, v4, v20
+# CHECK-INST: vredor.vs v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x0a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 0a <unknown>
+
+vredxor.vs v8, v4, v20, v0.t
+# CHECK-INST: vredxor.vs v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 0c <unknown>
+
+vredxor.vs v8, v4, v20
+# CHECK-INST: vredxor.vs v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x0e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 0e <unknown>
+
+vwredsumu.vs v8, v4, v20, v0.t
+# CHECK-INST: vwredsumu.vs v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xc0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a c0 <unknown>
+
+vwredsumu.vs v8, v4, v20
+# CHECK-INST: vwredsumu.vs v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xc2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a c2 <unknown>
+
+vwredsum.vs v8, v4, v20, v0.t
+# CHECK-INST: vwredsum.vs v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xc4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a c4 <unknown>
+
+vwredsum.vs v8, v4, v20
+# CHECK-INST: vwredsum.vs v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xc6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a c6 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/shift.s b/llvm/test/MC/RISCV/rvv/shift.s
new file mode 100644
index 0000000000000..27d40a57a84b5
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/shift.s
@@ -0,0 +1,261 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vsll.vv v8, v4, v20, v0.t
+# CHECK-INST: vsll.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x94]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 94 <unknown>
+
+vsll.vv v8, v4, v20
+# CHECK-INST: vsll.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x96]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 96 <unknown>
+
+vsll.vx v8, v4, a0, v0.t
+# CHECK-INST: vsll.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x94]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 94 <unknown>
+
+vsll.vx v8, v4, a0
+# CHECK-INST: vsll.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x96]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 96 <unknown>
+
+vsll.vi v8, v4, 31, v0.t
+# CHECK-INST: vsll.vi v8, v4, 31, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0x94]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f 94 <unknown>
+
+vsll.vi v8, v4, 31
+# CHECK-INST: vsll.vi v8, v4, 31
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0x96]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f 96 <unknown>
+
+vsrl.vv v8, v4, v20, v0.t
+# CHECK-INST: vsrl.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xa0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a a0 <unknown>
+
+vsrl.vv v8, v4, v20
+# CHECK-INST: vsrl.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xa2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a a2 <unknown>
+
+vsrl.vx v8, v4, a0, v0.t
+# CHECK-INST: vsrl.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0xa0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 a0 <unknown>
+
+vsrl.vx v8, v4, a0
+# CHECK-INST: vsrl.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0xa2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 a2 <unknown>
+
+vsrl.vi v8, v4, 31, v0.t
+# CHECK-INST: vsrl.vi v8, v4, 31, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0xa0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f a0 <unknown>
+
+vsrl.vi v8, v4, 31
+# CHECK-INST: vsrl.vi v8, v4, 31
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0xa2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f a2 <unknown>
+
+vsra.vv v8, v4, v20, v0.t
+# CHECK-INST: vsra.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xa4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a a4 <unknown>
+
+vsra.vv v8, v4, v20
+# CHECK-INST: vsra.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xa6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a a6 <unknown>
+
+vsra.vx v8, v4, a0, v0.t
+# CHECK-INST: vsra.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0xa4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 a4 <unknown>
+
+vsra.vx v8, v4, a0
+# CHECK-INST: vsra.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0xa6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 a6 <unknown>
+
+vsra.vi v8, v4, 31, v0.t
+# CHECK-INST: vsra.vi v8, v4, 31, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0xa4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f a4 <unknown>
+
+vsra.vi v8, v4, 31
+# CHECK-INST: vsra.vi v8, v4, 31
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0xa6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f a6 <unknown>
+
+vnsrl.wv v8, v4, v20, v0.t
+# CHECK-INST: vnsrl.wv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xb0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a b0 <unknown>
+
+vnsrl.wv v8, v4, v20
+# CHECK-INST: vnsrl.wv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xb2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a b2 <unknown>
+
+vnsrl.wx v8, v4, a0, v0.t
+# CHECK-INST: vnsrl.wx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0xb0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 b0 <unknown>
+
+vnsrl.wx v8, v4, a0
+# CHECK-INST: vnsrl.wx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0xb2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 b2 <unknown>
+
+vnsrl.wi v8, v4, 31, v0.t
+# CHECK-INST: vnsrl.wi v8, v4, 31, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0xb0]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f b0 <unknown>
+
+vnsrl.wi v8, v4, 31
+# CHECK-INST: vnsrl.wi v8, v4, 31
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0xb2]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f b2 <unknown>
+
+vnsra.wv v8, v4, v20, v0.t
+# CHECK-INST: vnsra.wv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xb4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a b4 <unknown>
+
+vnsra.wv v8, v4, v20
+# CHECK-INST: vnsra.wv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xb6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a b6 <unknown>
+
+vnsra.wx v8, v4, a0, v0.t
+# CHECK-INST: vnsra.wx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0xb4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 b4 <unknown>
+
+vnsra.wx v8, v4, a0
+# CHECK-INST: vnsra.wx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0xb6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 b6 <unknown>
+
+vnsra.wi v8, v4, 31, v0.t
+# CHECK-INST: vnsra.wi v8, v4, 31, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0xb4]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f b4 <unknown>
+
+vnsra.wi v8, v4, 31
+# CHECK-INST: vnsra.wi v8, v4, 31
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0xb6]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f b6 <unknown>
+
+vssrl.vv v8, v4, v20, v0.t
+# CHECK-INST: vssrl.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xa8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a a8 <unknown>
+
+vssrl.vv v8, v4, v20
+# CHECK-INST: vssrl.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xaa]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a aa <unknown>
+
+vssrl.vx v8, v4, a0, v0.t
+# CHECK-INST: vssrl.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0xa8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 a8 <unknown>
+
+vssrl.vx v8, v4, a0
+# CHECK-INST: vssrl.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0xaa]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 aa <unknown>
+
+vssrl.vi v8, v4, 31, v0.t
+# CHECK-INST: vssrl.vi v8, v4, 31, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0xa8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f a8 <unknown>
+
+vssrl.vi v8, v4, 31
+# CHECK-INST: vssrl.vi v8, v4, 31
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0xaa]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f aa <unknown>
+
+vssra.vv v8, v4, v20, v0.t
+# CHECK-INST: vssra.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xac]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a ac <unknown>
+
+vssra.vv v8, v4, v20
+# CHECK-INST: vssra.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0xae]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a ae <unknown>
+
+vssra.vx v8, v4, a0, v0.t
+# CHECK-INST: vssra.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0xac]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 ac <unknown>
+
+vssra.vx v8, v4, a0
+# CHECK-INST: vssra.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0xae]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 ae <unknown>
+
+vssra.vi v8, v4, 31, v0.t
+# CHECK-INST: vssra.vi v8, v4, 31, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0xac]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f ac <unknown>
+
+vssra.vi v8, v4, 31
+# CHECK-INST: vssra.vi v8, v4, 31
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0xae]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f ae <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/sign-injection.s b/llvm/test/MC/RISCV/rvv/sign-injection.s
new file mode 100644
index 0000000000000..0831f923be3fe
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/sign-injection.s
@@ -0,0 +1,81 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vfsgnj.vv v8, v4, v20, v0.t
+# CHECK-INST: vfsgnj.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x20]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 20 <unknown>
+
+vfsgnj.vv v8, v4, v20
+# CHECK-INST: vfsgnj.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x22]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 22 <unknown>
+
+vfsgnj.vf v8, v4, fa0, v0.t
+# CHECK-INST: vfsgnj.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x20]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 20 <unknown>
+
+vfsgnj.vf v8, v4, fa0
+# CHECK-INST: vfsgnj.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x22]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 22 <unknown>
+
+vfsgnjn.vv v8, v4, v20, v0.t
+# CHECK-INST: vfsgnjn.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x24]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 24 <unknown>
+
+vfsgnjn.vv v8, v4, v20
+# CHECK-INST: vfsgnjn.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x26]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 26 <unknown>
+
+vfsgnjn.vf v8, v4, fa0, v0.t
+# CHECK-INST: vfsgnjn.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x24]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 24 <unknown>
+
+vfsgnjn.vf v8, v4, fa0
+# CHECK-INST: vfsgnjn.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x26]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 26 <unknown>
+
+vfsgnjx.vv v8, v4, v20, v0.t
+# CHECK-INST: vfsgnjx.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x28]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 28 <unknown>
+
+vfsgnjx.vv v8, v4, v20
+# CHECK-INST: vfsgnjx.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x2a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4a 2a <unknown>
+
+vfsgnjx.vf v8, v4, fa0, v0.t
+# CHECK-INST: vfsgnjx.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x28]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 28 <unknown>
+
+vfsgnjx.vf v8, v4, fa0
+# CHECK-INST: vfsgnjx.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x2a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 2a <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/snippet.s b/llvm/test/MC/RISCV/rvv/snippet.s
new file mode 100644
index 0000000000000..fd38631c4ba7e
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/snippet.s
@@ -0,0 +1,32 @@
+## A snippet from https://github.com/riscv/riscv-v-spec.
+
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v < %s \
+# RUN: | llvm-objdump -d --mattr=+experimental-v - \
+# RUN: | FileCheck %s --check-prefix=CHECK-INST
+
+loop:
+    vsetvli a3, a0, e16,m4  # vtype = 16-bit integer vectors
+# CHECK-INST: d7 76 65 00    vsetvli a3, a0, e16,m4
+    vlh.v v4, (a1)          # Get 16b vector
+# CHECK-INST: 07 d2 05 12    vlh.v   v4, (a1)
+    slli t1, a3, 1          # Multiply length by two bytes/element
+# CHECK-INST: 13 93 16 00    slli    t1, a3, 1
+    add a1, a1, t1          # Bump pointer
+# CHECK-INST: b3 85 65 00    add     a1, a1, t1
+    vwmul.vx v8, v4, x10    # 32b in <v8--v15>
+# CHECK-INST: 57 64 45 ee    vwmul.vx        v8, v4, a0
+
+    vsetvli x0, a0, e32,m8  # Operate on 32b values
+# CHECK-INST: 57 70 b5 00    vsetvli zero, a0, e32,m8
+    vsrl.vi v8, v8, 3
+# CHECK-INST: 57 b4 81 a2    vsrl.vi v8, v8, 3
+    vsw.v v8, (a2)          # Store vector of 32b
+# CHECK-INST: 27 64 06 02    vsw.v   v8, (a2)
+    slli t1, a3, 2          # Multiply length by four bytes/element
+# CHECK-INST: 13 93 26 00    slli    t1, a3, 2
+    add a2, a2, t1          # Bump pointer
+# CHECK-INST: 33 06 66 00    add     a2, a2, t1
+    sub a0, a0, a3          # Decrement count
+# CHECK-INST: 33 05 d5 40    sub     a0, a0, a3
+    bnez a0, loop           # Any more?
+# CHECK-INST: e3 1a 05 fc    bnez    a0, -44
diff --git a/llvm/test/MC/RISCV/rvv/store.s b/llvm/test/MC/RISCV/rvv/store.s
new file mode 100644
index 0000000000000..9d25b593ef824
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/store.s
@@ -0,0 +1,207 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vsb.v v24, (a0), v0.t
+# CHECK-INST: vsb.v v24, (a0), v0.t
+# CHECK-ENCODING: [0x27,0x0c,0x05,0x00]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 0c 05 00 <unknown>
+
+vsb.v v24, (a0)
+# CHECK-INST: vsb.v v24, (a0)
+# CHECK-ENCODING: [0x27,0x0c,0x05,0x02]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 0c 05 02 <unknown>
+
+vsh.v v24, (a0), v0.t
+# CHECK-INST: vsh.v v24, (a0), v0.t
+# CHECK-ENCODING: [0x27,0x5c,0x05,0x00]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 5c 05 00 <unknown>
+
+vsh.v v24, (a0)
+# CHECK-INST: vsh.v v24, (a0)
+# CHECK-ENCODING: [0x27,0x5c,0x05,0x02]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 5c 05 02 <unknown>
+
+vsw.v v24, (a0), v0.t
+# CHECK-INST: vsw.v v24, (a0), v0.t
+# CHECK-ENCODING: [0x27,0x6c,0x05,0x00]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 6c 05 00 <unknown>
+
+vsw.v v24, (a0)
+# CHECK-INST: vsw.v v24, (a0)
+# CHECK-ENCODING: [0x27,0x6c,0x05,0x02]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 6c 05 02 <unknown>
+
+vse.v v24, (a0), v0.t
+# CHECK-INST: vse.v v24, (a0), v0.t
+# CHECK-ENCODING: [0x27,0x7c,0x05,0x00]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 7c 05 00 <unknown>
+
+vse.v v24, (a0)
+# CHECK-INST: vse.v v24, (a0)
+# CHECK-ENCODING: [0x27,0x7c,0x05,0x02]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 7c 05 02 <unknown>
+
+vssb.v v24, (a0), a1, v0.t
+# CHECK-INST: vssb.v v24, (a0), a1, v0.t
+# CHECK-ENCODING: [0x27,0x0c,0xb5,0x08]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 0c b5 08 <unknown>
+
+vssb.v v24, (a0), a1
+# CHECK-INST: vssb.v v24, (a0), a1
+# CHECK-ENCODING: [0x27,0x0c,0xb5,0x0a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 0c b5 0a <unknown>
+
+vssh.v v24, (a0), a1, v0.t
+# CHECK-INST: vssh.v v24, (a0), a1, v0.t
+# CHECK-ENCODING: [0x27,0x5c,0xb5,0x08]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 5c b5 08 <unknown>
+
+vssh.v v24, (a0), a1
+# CHECK-INST: vssh.v v24, (a0), a1
+# CHECK-ENCODING: [0x27,0x5c,0xb5,0x0a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 5c b5 0a <unknown>
+
+vssw.v v24, (a0), a1, v0.t
+# CHECK-INST: vssw.v v24, (a0), a1, v0.t
+# CHECK-ENCODING: [0x27,0x6c,0xb5,0x08]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 6c b5 08 <unknown>
+
+vssw.v v24, (a0), a1
+# CHECK-INST: vssw.v v24, (a0), a1
+# CHECK-ENCODING: [0x27,0x6c,0xb5,0x0a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 6c b5 0a <unknown>
+
+vsse.v v24, (a0), a1, v0.t
+# CHECK-INST: vsse.v v24, (a0), a1, v0.t
+# CHECK-ENCODING: [0x27,0x7c,0xb5,0x08]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 7c b5 08 <unknown>
+
+vsse.v v24, (a0), a1
+# CHECK-INST: vsse.v v24, (a0), a1
+# CHECK-ENCODING: [0x27,0x7c,0xb5,0x0a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 7c b5 0a <unknown>
+
+vsxb.v v24, (a0), v4, v0.t
+# CHECK-INST: vsxb.v v24, (a0), v4, v0.t
+# CHECK-ENCODING: [0x27,0x0c,0x45,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 0c 45 0c <unknown>
+
+vsxb.v v24, (a0), v4
+# CHECK-INST: vsxb.v v24, (a0), v4
+# CHECK-ENCODING: [0x27,0x0c,0x45,0x0e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 0c 45 0e <unknown>
+
+vsxh.v v24, (a0), v4, v0.t
+# CHECK-INST: vsxh.v v24, (a0), v4, v0.t
+# CHECK-ENCODING: [0x27,0x5c,0x45,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 5c 45 0c <unknown>
+
+vsxh.v v24, (a0), v4
+# CHECK-INST: vsxh.v v24, (a0), v4
+# CHECK-ENCODING: [0x27,0x5c,0x45,0x0e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 5c 45 0e <unknown>
+
+vsxw.v v24, (a0), v4, v0.t
+# CHECK-INST: vsxw.v v24, (a0), v4, v0.t
+# CHECK-ENCODING: [0x27,0x6c,0x45,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 6c 45 0c <unknown>
+
+vsxw.v v24, (a0), v4
+# CHECK-INST: vsxw.v v24, (a0), v4
+# CHECK-ENCODING: [0x27,0x6c,0x45,0x0e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 6c 45 0e <unknown>
+
+vsxe.v v24, (a0), v4, v0.t
+# CHECK-INST: vsxe.v v24, (a0), v4, v0.t
+# CHECK-ENCODING: [0x27,0x7c,0x45,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 7c 45 0c <unknown>
+
+vsxe.v v24, (a0), v4
+# CHECK-INST: vsxe.v v24, (a0), v4
+# CHECK-ENCODING: [0x27,0x7c,0x45,0x0e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 7c 45 0e <unknown>
+
+vsuxb.v v24, (a0), v4, v0.t
+# CHECK-INST: vsuxb.v v24, (a0), v4, v0.t
+# CHECK-ENCODING: [0x27,0x0c,0x45,0x1c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 0c 45 1c <unknown>
+
+vsuxb.v v24, (a0), v4
+# CHECK-INST: vsuxb.v v24, (a0), v4
+# CHECK-ENCODING: [0x27,0x0c,0x45,0x1e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 0c 45 1e <unknown>
+
+vsuxh.v v24, (a0), v4, v0.t
+# CHECK-INST: vsuxh.v v24, (a0), v4, v0.t
+# CHECK-ENCODING: [0x27,0x5c,0x45,0x1c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 5c 45 1c <unknown>
+
+vsuxh.v v24, (a0), v4
+# CHECK-INST: vsuxh.v v24, (a0), v4
+# CHECK-ENCODING: [0x27,0x5c,0x45,0x1e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 5c 45 1e <unknown>
+
+vsuxw.v v24, (a0), v4, v0.t
+# CHECK-INST: vsuxw.v v24, (a0), v4, v0.t
+# CHECK-ENCODING: [0x27,0x6c,0x45,0x1c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 6c 45 1c <unknown>
+
+vsuxw.v v24, (a0), v4
+# CHECK-INST: vsuxw.v v24, (a0), v4
+# CHECK-ENCODING: [0x27,0x6c,0x45,0x1e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 6c 45 1e <unknown>
+
+vsuxe.v v24, (a0), v4, v0.t
+# CHECK-INST: vsuxe.v v24, (a0), v4, v0.t
+# CHECK-ENCODING: [0x27,0x7c,0x45,0x1c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 7c 45 1c <unknown>
+
+vsuxe.v v24, (a0), v4
+# CHECK-INST: vsuxe.v v24, (a0), v4
+# CHECK-ENCODING: [0x27,0x7c,0x45,0x1e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 7c 45 1e <unknown>
+
+vs1r.v v24, (a0)
+# CHECK-INST: vs1r.v v24, (a0)
+# CHECK-ENCODING: [0x27,0x7c,0x85,0x02]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 7c 85 02 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/sub.s b/llvm/test/MC/RISCV/rvv/sub.s
new file mode 100644
index 0000000000000..99b0c7a1e60f4
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/sub.s
@@ -0,0 +1,285 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vsub.vv v8, v4, v20, v0.t
+# CHECK-INST: vsub.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x08]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 08 <unknown>
+
+vsub.vv v8, v4, v20
+# CHECK-INST: vsub.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x0a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 0a <unknown>
+
+vsub.vx v8, v4, a0, v0.t
+# CHECK-INST: vsub.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x08]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 08 <unknown>
+
+vsub.vx v8, v4, a0
+# CHECK-INST: vsub.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x0a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 0a <unknown>
+
+vrsub.vx v8, v4, a0, v0.t
+# CHECK-INST: vrsub.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 0c <unknown>
+
+vrsub.vx v8, v4, a0
+# CHECK-INST: vrsub.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x0e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 0e <unknown>
+
+vrsub.vi v8, v4, 15, v0.t
+# CHECK-INST: vrsub.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 0c <unknown>
+
+vrsub.vi v8, v4, 15
+# CHECK-INST: vrsub.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x0e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 0e <unknown>
+
+vwsubu.vv v8, v4, v20, v0.t
+# CHECK-INST: vwsubu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xc8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a c8 <unknown>
+
+vwsubu.vv v8, v4, v20
+# CHECK-INST: vwsubu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xca]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a ca <unknown>
+
+vwsubu.vx v8, v4, a0, v0.t
+# CHECK-INST: vwsubu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xc8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 c8 <unknown>
+
+vwsubu.vx v8, v4, a0
+# CHECK-INST: vwsubu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0xca]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 ca <unknown>
+
+vwsub.vv v8, v4, v20, v0.t
+# CHECK-INST: vwsub.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xcc]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a cc <unknown>
+
+vwsub.vv v8, v4, v20
+# CHECK-INST: vwsub.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xce]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a ce <unknown>
+
+vwsub.vx v8, v4, a0, v0.t
+# CHECK-INST: vwsub.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xcc]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 cc <unknown>
+
+vwsub.vx v8, v4, a0
+# CHECK-INST: vwsub.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0xce]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 ce <unknown>
+
+vwsubu.wv v8, v4, v20, v0.t
+# CHECK-INST: vwsubu.wv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xd8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a d8 <unknown>
+
+vwsubu.wv v8, v4, v20
+# CHECK-INST: vwsubu.wv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xda]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a da <unknown>
+
+vwsubu.wx v8, v4, a0, v0.t
+# CHECK-INST: vwsubu.wx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xd8]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 d8 <unknown>
+
+vwsubu.wx v8, v4, a0
+# CHECK-INST: vwsubu.wx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0xda]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 da <unknown>
+
+vwsub.wv v8, v4, v20, v0.t
+# CHECK-INST: vwsub.wv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xdc]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a dc <unknown>
+
+vwsub.wv v8, v4, v20
+# CHECK-INST: vwsub.wv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0xde]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a de <unknown>
+
+vwsub.wx v8, v4, a0, v0.t
+# CHECK-INST: vwsub.wx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0xdc]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 dc <unknown>
+
+vwsub.wx v8, v4, a0
+# CHECK-INST: vwsub.wx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0xde]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 de <unknown>
+
+vsbc.vvm v8, v4, v20, v0
+# CHECK-INST: vsbc.vvm v8, v4, v20, v0
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x48]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 48 <unknown>
+
+vsbc.vxm v8, v4, a0, v0
+# CHECK-INST: vsbc.vxm v8, v4, a0, v0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x48]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 48 <unknown>
+
+vmsbc.vvm v8, v4, v20, v0
+# CHECK-INST: vmsbc.vvm v8, v4, v20, v0
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x4c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 4c <unknown>
+
+vmsbc.vxm v8, v4, a0, v0
+# CHECK-INST: vmsbc.vxm v8, v4, a0, v0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x4c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 4c <unknown>
+
+vmsbc.vv v8, v4, v20
+# CHECK-INST: vmsbc.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x4e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 4e <unknown>
+
+vmsbc.vx v8, v4, a0
+# CHECK-INST: vmsbc.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x4e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 4e <unknown>
+
+vssubu.vv v8, v4, v20, v0.t
+# CHECK-INST: vssubu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 88 <unknown>
+
+vssubu.vv v8, v4, v20
+# CHECK-INST: vssubu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 8a <unknown>
+
+vssubu.vx v8, v4, a0, v0.t
+# CHECK-INST: vssubu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x88]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 88 <unknown>
+
+vssubu.vx v8, v4, a0
+# CHECK-INST: vssubu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x8a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 8a <unknown>
+
+vssub.vv v8, v4, v20, v0.t
+# CHECK-INST: vssub.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x8c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 8c <unknown>
+
+vssub.vv v8, v4, v20
+# CHECK-INST: vssub.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x8e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 8e <unknown>
+
+vssub.vx v8, v4, a0, v0.t
+# CHECK-INST: vssub.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x8c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 8c <unknown>
+
+vssub.vx v8, v4, a0
+# CHECK-INST: vssub.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x8e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 8e <unknown>
+
+vasub.vv v8, v4, v20, v0.t
+# CHECK-INST: vasub.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x2c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 2c <unknown>
+
+vasub.vv v8, v4, v20
+# CHECK-INST: vasub.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x2e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 2e <unknown>
+
+vasub.vx v8, v4, a0, v0.t
+# CHECK-INST: vasub.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0x2c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 2c <unknown>
+
+vasub.vx v8, v4, a0
+# CHECK-INST: vasub.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0x2e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 2e <unknown>
+
+vasubu.vv v8, v4, v20, v0.t
+# CHECK-INST: vasubu.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x28]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 28 <unknown>
+
+vasubu.vv v8, v4, v20
+# CHECK-INST: vasubu.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x24,0x4a,0x2a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 4a 2a <unknown>
+
+vasubu.vx v8, v4, a0, v0.t
+# CHECK-INST: vasubu.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x64,0x45,0x28]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 28 <unknown>
+
+vasubu.vx v8, v4, a0
+# CHECK-INST: vasubu.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x64,0x45,0x2a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 64 45 2a <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/vsetvl.s b/llvm/test/MC/RISCV/rvv/vsetvl.s
new file mode 100644
index 0000000000000..784533445f0c6
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/vsetvl.s
@@ -0,0 +1,21 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vsetvli a2, a0, e32,m4
+# CHECK-INST: vsetvli a2, a0, e32,m4
+# CHECK-ENCODING: [0x57,0x76,0xa5,0x00]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 76 a5 00 <unknown>
+
+vsetvl a2, a0, a1
+# CHECK-INST: vsetvl a2, a0, a1
+# CHECK-ENCODING: [0x57,0x76,0xb5,0x80]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 76 b5 80 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/xor.s b/llvm/test/MC/RISCV/rvv/xor.s
new file mode 100644
index 0000000000000..f9c432c8e7dc2
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/xor.s
@@ -0,0 +1,57 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vxor.vv v8, v4, v20, v0.t
+# CHECK-INST: vxor.vv v8, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x2c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 2c <unknown>
+
+vxor.vv v8, v4, v20
+# CHECK-INST: vxor.vv v8, v4, v20
+# CHECK-ENCODING: [0x57,0x04,0x4a,0x2e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 04 4a 2e <unknown>
+
+vxor.vx v8, v4, a0, v0.t
+# CHECK-INST: vxor.vx v8, v4, a0, v0.t
+# CHECK-ENCODING: [0x57,0x44,0x45,0x2c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 2c <unknown>
+
+vxor.vx v8, v4, a0
+# CHECK-INST: vxor.vx v8, v4, a0
+# CHECK-ENCODING: [0x57,0x44,0x45,0x2e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 44 45 2e <unknown>
+
+vxor.vi v8, v4, 15, v0.t
+# CHECK-INST: vxor.vi v8, v4, 15, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x2c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 2c <unknown>
+
+vxor.vi v8, v4, 15
+# CHECK-INST: vxor.vi v8, v4, 15
+# CHECK-ENCODING: [0x57,0xb4,0x47,0x2e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 47 2e <unknown>
+
+vnot.v v8, v4, v0.t
+# CHECK-INST: vnot.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0x2c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f 2c <unknown>
+
+vnot.v v8, v4
+# CHECK-INST: vxor.vi v8, v4, -1
+# CHECK-ENCODING: [0x57,0xb4,0x4f,0x2e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 b4 4f 2e <unknown>
diff --git a/llvm/test/MC/SystemZ/insn-bad.s b/llvm/test/MC/SystemZ/insn-bad.s
index c330fa496a25a..59174067c4b28 100644
--- a/llvm/test/MC/SystemZ/insn-bad.s
+++ b/llvm/test/MC/SystemZ/insn-bad.s
@@ -1343,7 +1343,7 @@
 #CHECK: clc	0, 0
 #CHECK: error: missing length in address
 #CHECK: clc	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: clc	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: clc	0(0,%r1), 0(%r1)
@@ -2573,7 +2573,7 @@
 #CHECK: ed	0, 0
 #CHECK: error: missing length in address
 #CHECK: ed	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: ed	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: ed	0(0,%r1), 0(%r1)
@@ -2611,7 +2611,7 @@
 #CHECK: edmk	0, 0
 #CHECK: error: missing length in address
 #CHECK: edmk	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: edmk	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: edmk	0(0,%r1), 0(%r1)
@@ -4373,7 +4373,7 @@
 #CHECK: mvc	0, 0
 #CHECK: error: missing length in address
 #CHECK: mvc	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: mvc	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: mvc	0(0,%r1), 0(%r1)
@@ -4428,7 +4428,7 @@
 #CHECK: mvcin	0, 0
 #CHECK: error: missing length in address
 #CHECK: mvcin	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: mvcin	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: mvcin	0(0,%r1), 0(%r1)
@@ -4462,7 +4462,7 @@
 	mvcin	0(1,%r2), 0(%r1,%r2)
 	mvcin	0(-), 0
 
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: mvck	0(%r1,%r1), 0(2,%r1), %r3
 #CHECK: error: invalid operand
 #CHECK: mvck	-1(%r1,%r1), 0(%r1), %r3
@@ -4474,7 +4474,7 @@
 #CHECK: mvck	0(%r1,%r1), 4096(%r1), %r3
 #CHECK: error: invalid use of indexed addressing
 #CHECK: mvck	0(%r1,%r2), 0(%r1,%r2), %r3
-#CHECK: error: unknown token in expression
+#CHECK: error: unexpected token in address
 #CHECK: mvck	0(-), 0, %r3
 
 	mvck	0(%r1,%r1), 0(2,%r1), %r3
@@ -4538,7 +4538,7 @@
         mvcos	0(%r1), -1(%r15), %r2
         mvcos	0(%r1), 4096(%r15), %r2
 
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: mvcp	0(%r1,%r1), 0(2,%r1), %r3
 #CHECK: error: invalid operand
 #CHECK: mvcp	-1(%r1,%r1), 0(%r1), %r3
@@ -4550,7 +4550,7 @@
 #CHECK: mvcp	0(%r1,%r1), 4096(%r1), %r3
 #CHECK: error: invalid use of indexed addressing
 #CHECK: mvcp	0(%r1,%r2), 0(%r1,%r2), %r3
-#CHECK: error: unknown token in expression
+#CHECK: error: unexpected token in address
 #CHECK: mvcp	0(-), 0, %r3
 
 	mvcp	0(%r1,%r1), 0(2,%r1), %r3
@@ -4561,7 +4561,7 @@
 	mvcp	0(%r1,%r2), 0(%r1,%r2), %r3
 	mvcp	0(-), 0, %r3
 
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: mvcs	0(%r1,%r1), 0(2,%r1), %r3
 #CHECK: error: invalid operand
 #CHECK: mvcs	-1(%r1,%r1), 0(%r1), %r3
@@ -4573,7 +4573,7 @@
 #CHECK: mvcs	0(%r1,%r1), 4096(%r1), %r3
 #CHECK: error: invalid use of indexed addressing
 #CHECK: mvcs	0(%r1,%r2), 0(%r1,%r2), %r3
-#CHECK: error: unknown token in expression
+#CHECK: error: unexpected token in address
 #CHECK: mvcs	0(-), 0, %r3
 
 	mvcs	0(%r1,%r1), 0(2,%r1), %r3
@@ -4690,7 +4690,7 @@
 #CHECK: mvn	0, 0
 #CHECK: error: missing length in address
 #CHECK: mvn	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: mvn	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: mvn	0(0,%r1), 0(%r1)
@@ -4775,7 +4775,7 @@
 #CHECK: mvz	0, 0
 #CHECK: error: missing length in address
 #CHECK: mvz	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: mvz	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: mvz	0(0,%r1), 0(%r1)
@@ -4917,7 +4917,7 @@
 #CHECK: nc	0, 0
 #CHECK: error: missing length in address
 #CHECK: nc	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: nc	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: nc	0(0,%r1), 0(%r1)
@@ -5071,7 +5071,7 @@
 #CHECK: oc	0, 0
 #CHECK: error: missing length in address
 #CHECK: oc	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: oc	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: oc	0(0,%r1), 0(%r1)
@@ -5319,7 +5319,7 @@
 #CHECK: pka	0, 0
 #CHECK: error: missing length in address
 #CHECK: pka	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: pka	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: pka	0(%r1), 0(0,%r1)
@@ -5357,7 +5357,7 @@
 #CHECK: pku	0, 0
 #CHECK: error: missing length in address
 #CHECK: pku	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: pku	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: pku	0(%r1), 0(0,%r1)
@@ -6232,7 +6232,7 @@
 #CHECK: srp	0, 0, 0
 #CHECK: error: missing length in address
 #CHECK: srp	0(%r1), 0(%r1), 0
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: srp	0(1,%r1), 0(2,%r1), 0
 #CHECK: error: invalid operand
 #CHECK: srp	0(0,%r1), 0(%r1), 0
@@ -7118,7 +7118,7 @@
 #CHECK: tr	0, 0
 #CHECK: error: missing length in address
 #CHECK: tr	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: tr	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: tr	0(0,%r1), 0(%r1)
@@ -7216,7 +7216,7 @@
 #CHECK: trt	0, 0
 #CHECK: error: missing length in address
 #CHECK: trt	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: trt	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: trt	0(0,%r1), 0(%r1)
@@ -7276,7 +7276,7 @@
 #CHECK: trtr	0, 0
 #CHECK: error: missing length in address
 #CHECK: trtr	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: trtr	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: trtr	0(0,%r1), 0(%r1)
@@ -7405,7 +7405,7 @@
 #CHECK: unpka	0, 0
 #CHECK: error: missing length in address
 #CHECK: unpka	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: unpka	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: unpka	0(0,%r1), 0(%r1)
@@ -7443,7 +7443,7 @@
 #CHECK: unpku	0, 0
 #CHECK: error: missing length in address
 #CHECK: unpku	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: unpku	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: unpku	0(0,%r1), 0(%r1)
@@ -7489,7 +7489,7 @@
 #CHECK: xc	0, 0
 #CHECK: error: missing length in address
 #CHECK: xc	0(%r1), 0(%r1)
-#CHECK: error: invalid use of length addressing
+#CHECK: error: invalid use of indexed addressing
 #CHECK: xc	0(1,%r1), 0(2,%r1)
 #CHECK: error: invalid operand
 #CHECK: xc	0(0,%r1), 0(%r1)
diff --git a/llvm/test/MC/SystemZ/insn-good-z13.s b/llvm/test/MC/SystemZ/insn-good-z13.s
index df99a1f489518..4e3ba50ad7f9e 100644
--- a/llvm/test/MC/SystemZ/insn-good-z13.s
+++ b/llvm/test/MC/SystemZ/insn-good-z13.s
@@ -2879,6 +2879,20 @@
 	vgbm	%v31, 0
 	vgbm	%v17, 0x1234
 
+#CHECK: vgef    %v0, 0(%v0), 0          # encoding: [0xe7,0x00,0x00,0x00,0x00,0x13]
+#CHECK: vgef    %v0, 0(%v0,%r1), 0      # encoding: [0xe7,0x00,0x10,0x00,0x00,0x13]
+#CHECK: vgef    %v0, 0(%v0,%r1), 3      # encoding: [0xe7,0x00,0x10,0x00,0x30,0x13]
+#CHECK: vgef    %v0, 0(%v0,%r15), 0     # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x13]
+#CHECK: vgef    %v0, 0(%v15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x13]
+#CHECK: vgef    %v0, 0(%v31,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x04,0x13]
+#CHECK: vgef    %v0, 4095(%v0,%r1), 0   # encoding: [0xe7,0x00,0x1f,0xff,0x00,0x13]
+#CHECK: vgef    %v15, 0(%v0,%r1), 0     # encoding: [0xe7,0xf0,0x10,0x00,0x00,0x13]
+#CHECK: vgef    %v31, 0(%v0,%r1), 0     # encoding: [0xe7,0xf0,0x10,0x00,0x08,0x13]
+#CHECK: vgef    %v10, 1000(%v19,%r7), 1 # encoding: [0xe7,0xa3,0x73,0xe8,0x14,0x13]
+#CHECK: vgef    %v0, 0(%v0,%r1), 0      # encoding: [0xe7,0x00,0x10,0x00,0x00,0x13]
+#CHECK: vgef    %v0, 0(%v0,%r1), 3      # encoding: [0xe7,0x00,0x10,0x00,0x30,0x13]
+#CHECK: vgef    %v0, 0(%v0,%r15), 0     # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x13]
+#CHECK: vgef    %v0, 0(%v15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x13]
 #CHECK: vgef    %v0, 0(%v0), 0          # encoding: [0xe7,0x00,0x00,0x00,0x00,0x13]
 #CHECK: vgef    %v0, 0(%v0,%r1), 0      # encoding: [0xe7,0x00,0x10,0x00,0x00,0x13]
 #CHECK: vgef    %v0, 0(%v0,%r1), 3      # encoding: [0xe7,0x00,0x10,0x00,0x30,0x13]
@@ -2900,7 +2914,35 @@
 	vgef	%v15, 0(%v0,%r1), 0
 	vgef	%v31, 0(%v0,%r1), 0
 	vgef	%v10, 1000(%v19,%r7), 1
+	vgef	%v0, 0(0,%r1), 0
+	vgef	%v0, 0(%v0,1), 3
+	vgef	%v0, 0(0,%r15), 0
+	vgef	%v0, 0(%v15,1), 0
+	vgef	0, 0(0), 0
+	vgef	0, 0(0,1), 0
+	vgef	0, 0(0,1), 3
+	vgef	0, 0(0,15), 0
+	vgef	0, 0(15,1), 0
+	vgef	0, 0(31,1), 0
+	vgef	0, 4095(0, 1), 0
+	vgef	15, 0(0,1), 0
+	vgef	31, 0(0,1), 0
+	vgef	10, 1000(19,7), 1
 
+#CHECK: vgeg    %v0, 0(%v0), 0          # encoding: [0xe7,0x00,0x00,0x00,0x00,0x12]
+#CHECK: vgeg    %v0, 0(%v0,%r1), 0      # encoding: [0xe7,0x00,0x10,0x00,0x00,0x12]
+#CHECK: vgeg    %v0, 0(%v0,%r1), 1      # encoding: [0xe7,0x00,0x10,0x00,0x10,0x12]
+#CHECK: vgeg    %v0, 0(%v0,%r15), 0     # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x12]
+#CHECK: vgeg    %v0, 0(%v15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x12]
+#CHECK: vgeg    %v0, 0(%v31,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x04,0x12]
+#CHECK: vgeg    %v0, 4095(%v0,%r1), 0   # encoding: [0xe7,0x00,0x1f,0xff,0x00,0x12]
+#CHECK: vgeg    %v15, 0(%v0,%r1), 0     # encoding: [0xe7,0xf0,0x10,0x00,0x00,0x12]
+#CHECK: vgeg    %v31, 0(%v0,%r1), 0     # encoding: [0xe7,0xf0,0x10,0x00,0x08,0x12]
+#CHECK: vgeg    %v10, 1000(%v19,%r7), 1 # encoding: [0xe7,0xa3,0x73,0xe8,0x14,0x12]
+#CHECK: vgeg    %v0, 0(%v0,%r1), 0      # encoding: [0xe7,0x00,0x10,0x00,0x00,0x12]
+#CHECK: vgeg    %v0, 0(%v0,%r1), 1      # encoding: [0xe7,0x00,0x10,0x00,0x10,0x12]
+#CHECK: vgeg    %v0, 0(%v0,%r15), 0     # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x12]
+#CHECK: vgeg    %v0, 0(%v15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x12]
 #CHECK: vgeg    %v0, 0(%v0), 0          # encoding: [0xe7,0x00,0x00,0x00,0x00,0x12]
 #CHECK: vgeg    %v0, 0(%v0,%r1), 0      # encoding: [0xe7,0x00,0x10,0x00,0x00,0x12]
 #CHECK: vgeg    %v0, 0(%v0,%r1), 1      # encoding: [0xe7,0x00,0x10,0x00,0x10,0x12]
@@ -2922,6 +2964,21 @@
 	vgeg	%v15, 0(%v0,%r1), 0
 	vgeg	%v31, 0(%v0,%r1), 0
 	vgeg	%v10, 1000(%v19,%r7), 1
+	vgeg	%v0, 0(0,%r1), 0
+	vgeg	%v0, 0(%v0,1), 1
+	vgeg	%v0, 0(0,%r15), 0
+	vgeg	%v0, 0(%v15,1), 0
+	vgeg	0, 0(0), 0
+	vgeg	0, 0(0,1), 0
+	vgeg	0, 0(0,1), 1
+	vgeg	0, 0(0,15), 0
+	vgeg	0, 0(15,1), 0
+	vgeg	0, 0(31,1), 0
+	vgeg	0, 4095(0,1), 0
+	vgeg	15, 0(0,1), 0
+	vgeg	31, 0(0,1), 0
+	vgeg	10, 1000(19,7), 1
+
 
 #CHECK: vgfm    %v0, %v0, %v0, 0        # encoding: [0xe7,0x00,0x00,0x00,0x00,0xb4]
 #CHECK: vgfm    %v0, %v0, %v0, 15       # encoding: [0xe7,0x00,0x00,0x00,0xf0,0xb4]
@@ -3229,6 +3286,16 @@
 	vl	%v31, 0
 	vl	%v18, 0x567(%r3,%r4), 3
 
+#CHECK: vlbb    %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x07]
+#CHECK: vlbb    %v0, 0, 15              # encoding: [0xe7,0x00,0x00,0x00,0xf0,0x07]
+#CHECK: vlbb    %v0, 4095, 0            # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x07]
+#CHECK: vlbb    %v0, 0(%r15), 0         # encoding: [0xe7,0x00,0xf0,0x00,0x00,0x07]
+#CHECK: vlbb    %v0, 0(%r15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x07]
+#CHECK: vlbb    %v15, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x00,0x07]
+#CHECK: vlbb    %v31, 0, 0              # encoding: [0xe7,0xf0,0x00,0x00,0x08,0x07]
+#CHECK: vlbb    %v18, 1383(%r3,%r4), 8  # encoding: [0xe7,0x23,0x45,0x67,0x88,0x07]
+#CHECK: vlbb    %v0, 0(%r15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x07]
+#CHECK: vlbb    %v0, 0(%r15,%r1), 0     # encoding: [0xe7,0x0f,0x10,0x00,0x00,0x07]
 #CHECK: vlbb    %v0, 0, 0               # encoding: [0xe7,0x00,0x00,0x00,0x00,0x07]
 #CHECK: vlbb    %v0, 0, 15              # encoding: [0xe7,0x00,0x00,0x00,0xf0,0x07]
 #CHECK: vlbb    %v0, 4095, 0            # encoding: [0xe7,0x00,0x0f,0xff,0x00,0x07]
@@ -3246,6 +3313,17 @@
 	vlbb	%v15, 0, 0
 	vlbb	%v31, 0, 0
 	vlbb	%v18, 1383(%r3,%r4), 8
+	vlbb	%v18, 1383(%r3, 4), 8
+	vlbb	%v0, 0(15,%r1), 0
+	vlbb	%v0, 0(%r15,1), 0
+	vlbb	0, 0, 0
+	vlbb	0, 0, 15
+	vlbb	0, 4095, 0
+	vlbb	0, 0(15), 0
+	vlbb	0, 0(15,1), 0
+	vlbb	15, 0, 0
+	vlbb	31, 0, 0
+	vlbb	18, 1383(3,4), 8
 
 #CHECK: vlc     %v0, %v0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x00,0xde]
 #CHECK: vlc     %v0, %v0, 15            # encoding: [0xe7,0x00,0x00,0x00,0xf0,0xde]
@@ -3922,6 +4000,7 @@
 #CHECK: vlvgf   %v0, %r0, 0             # encoding: [0xe7,0x00,0x00,0x00,0x20,0x22]
 #CHECK: vlvgf   %v0, %r0, 4095          # encoding: [0xe7,0x00,0x0f,0xff,0x20,0x22]
 #CHECK: vlvgf   %v0, %r0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x22]
+#CHECK: vlvgf   %v0, %r0, 0(%r15)       # encoding: [0xe7,0x00,0xf0,0x00,0x20,0x22]
 #CHECK: vlvgf   %v0, %r15, 0            # encoding: [0xe7,0x0f,0x00,0x00,0x20,0x22]
 #CHECK: vlvgf   %v15, %r0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x20,0x22]
 #CHECK: vlvgf   %v31, %r0, 0            # encoding: [0xe7,0xf0,0x00,0x00,0x28,0x22]
@@ -3930,6 +4009,7 @@
 	vlvgf	%v0, %r0, 0
 	vlvgf	%v0, %r0, 4095
 	vlvgf	%v0, %r0, 0(%r15)
+	vlvgf	%v0, %r0, 0(15)
 	vlvgf	%v0, %r15, 0
 	vlvgf	%v15, %r0, 0
 	vlvgf	%v31, %r0, 0
diff --git a/llvm/test/MC/SystemZ/insn-good-z14.s b/llvm/test/MC/SystemZ/insn-good-z14.s
index 1fcdcb4ccab08..ec12283ecbae8 100644
--- a/llvm/test/MC/SystemZ/insn-good-z14.s
+++ b/llvm/test/MC/SystemZ/insn-good-z14.s
@@ -4,6 +4,16 @@
 # RUN: llvm-mc -triple s390x-linux-gnu -mcpu=arch12 -show-encoding %s \
 # RUN:   | FileCheck %s
 
+#CHECK: agh	%r0, -524288            # encoding: [0xe3,0x00,0x00,0x00,0x80,0x38]
+#CHECK: agh	%r0, -1                 # encoding: [0xe3,0x00,0x0f,0xff,0xff,0x38]
+#CHECK: agh	%r0, 0                  # encoding: [0xe3,0x00,0x00,0x00,0x00,0x38]
+#CHECK: agh	%r0, 1                  # encoding: [0xe3,0x00,0x00,0x01,0x00,0x38]
+#CHECK: agh	%r0, 524287             # encoding: [0xe3,0x00,0x0f,0xff,0x7f,0x38]
+#CHECK: agh	%r0, 0(%r1)             # encoding: [0xe3,0x00,0x10,0x00,0x00,0x38]
+#CHECK: agh	%r0, 0(%r15)            # encoding: [0xe3,0x00,0xf0,0x00,0x00,0x38]
+#CHECK: agh	%r0, 524287(%r1,%r15)   # encoding: [0xe3,0x01,0xff,0xff,0x7f,0x38]
+#CHECK: agh	%r0, 524287(%r15,%r1)   # encoding: [0xe3,0x0f,0x1f,0xff,0x7f,0x38]
+#CHECK: agh	%r15, 0                 # encoding: [0xe3,0xf0,0x00,0x00,0x00,0x38]
 #CHECK: agh	%r0, -524288            # encoding: [0xe3,0x00,0x00,0x00,0x80,0x38]
 #CHECK: agh	%r0, -1                 # encoding: [0xe3,0x00,0x0f,0xff,0xff,0x38]
 #CHECK: agh	%r0, 0                  # encoding: [0xe3,0x00,0x00,0x00,0x00,0x38]
@@ -25,6 +35,16 @@
 	agh	%r0, 524287(%r1,%r15)
 	agh	%r0, 524287(%r15,%r1)
 	agh	%r15, 0
+	agh	0, -524288
+	agh	0, -1
+	agh	0, 0
+	agh	0, 1
+	agh	0, 524287
+	agh	0, 0(1)
+	agh	0, 0(15)
+	agh	0, 524287(1,15)
+	agh	0, 524287(15,1)
+	agh	15, 0
 
 #CHECK: bi	-524288                 # encoding: [0xe3,0xf0,0x00,0x00,0x80,0x47]
 #CHECK: bi	-1                      # encoding: [0xe3,0xf0,0x0f,0xff,0xff,0x47]
@@ -1200,6 +1220,13 @@
 	vlip	%v31, 0, 0
 	vlip	%v17, 0x1234, 7
 
+#CHECK: vllezlf	%v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x60,0x04]
+#CHECK: vllezlf	%v0, 4095               # encoding: [0xe7,0x00,0x0f,0xff,0x60,0x04]
+#CHECK: vllezlf	%v0, 0(%r15)            # encoding: [0xe7,0x00,0xf0,0x00,0x60,0x04]
+#CHECK: vllezlf	%v0, 0(%r15,%r1)        # encoding: [0xe7,0x0f,0x10,0x00,0x60,0x04]
+#CHECK: vllezlf	%v15, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x60,0x04]
+#CHECK: vllezlf	%v31, 0                 # encoding: [0xe7,0xf0,0x00,0x00,0x68,0x04]
+#CHECK: vllezlf	%v18, 1383(%r3,%r4)     # encoding: [0xe7,0x23,0x45,0x67,0x68,0x04]
 #CHECK: vllezlf	%v0, 0                  # encoding: [0xe7,0x00,0x00,0x00,0x60,0x04]
 #CHECK: vllezlf	%v0, 4095               # encoding: [0xe7,0x00,0x0f,0xff,0x60,0x04]
 #CHECK: vllezlf	%v0, 0(%r15)            # encoding: [0xe7,0x00,0xf0,0x00,0x60,0x04]
@@ -1215,7 +1242,21 @@
 	vllezlf	%v15, 0
 	vllezlf	%v31, 0
 	vllezlf	%v18, 0x567(%r3,%r4)
+	vllezlf	0, 0
+	vllezlf	0, 4095
+	vllezlf	0, 0(15)
+	vllezlf	0, 0(15,1)
+	vllezlf	15, 0
+	vllezlf	31, 0
+	vllezlf	18, 0x567(3,4)
 
+#CHECK: vlrl	%v0, 0, 0               # encoding: [0xe6,0x00,0x00,0x00,0x00,0x35]
+#CHECK: vlrl	%v0, 4095, 0            # encoding: [0xe6,0x00,0x0f,0xff,0x00,0x35]
+#CHECK: vlrl	%v0, 0(%r15), 0         # encoding: [0xe6,0x00,0xf0,0x00,0x00,0x35]
+#CHECK: vlrl	%v0, 0, 255             # encoding: [0xe6,0xff,0x00,0x00,0x00,0x35]
+#CHECK: vlrl	%v15, 0, 0              # encoding: [0xe6,0x00,0x00,0x00,0xf0,0x35]
+#CHECK: vlrl	%v31, 0, 0              # encoding: [0xe6,0x00,0x00,0x00,0xf1,0x35]
+#CHECK: vlrl	%v18, 1383(%r4), 3      # encoding: [0xe6,0x03,0x45,0x67,0x21,0x35]
 #CHECK: vlrl	%v0, 0, 0               # encoding: [0xe6,0x00,0x00,0x00,0x00,0x35]
 #CHECK: vlrl	%v0, 4095, 0            # encoding: [0xe6,0x00,0x0f,0xff,0x00,0x35]
 #CHECK: vlrl	%v0, 0(%r15), 0         # encoding: [0xe6,0x00,0xf0,0x00,0x00,0x35]
@@ -1231,6 +1272,13 @@
 	vlrl	%v15, 0, 0
 	vlrl	%v31, 0, 0
 	vlrl	%v18, 1383(%r4), 3
+	vlrl	0, 0, 0
+	vlrl	0, 4095, 0
+	vlrl	0, 0(15), 0
+	vlrl	0, 0, 255
+	vlrl	15, 0, 0
+	vlrl	31, 0, 0
+	vlrl	18, 1383(4), 3
 
 #CHECK: vlrlr	%v0, %r0, 0             # encoding: [0xe6,0x00,0x00,0x00,0x00,0x37]
 #CHECK: vlrlr	%v0, %r0, 4095          # encoding: [0xe6,0x00,0x0f,0xff,0x00,0x37]
diff --git a/llvm/test/MC/SystemZ/insn-good-z15.s b/llvm/test/MC/SystemZ/insn-good-z15.s
index 1eb3b743cc61d..36476161ea46d 100644
--- a/llvm/test/MC/SystemZ/insn-good-z15.s
+++ b/llvm/test/MC/SystemZ/insn-good-z15.s
@@ -24,6 +24,13 @@
 	kdsa	%r15, %r2
 	kdsa	%r7, %r10
 
+#CHECK: vllebrzg %v0, 0                 # encoding: [0xe6,0x00,0x00,0x00,0x30,0x04]
+#CHECK: vllebrzg %v0, 4095              # encoding: [0xe6,0x00,0x0f,0xff,0x30,0x04]
+#CHECK: vllebrzg %v0, 0(%r15)           # encoding: [0xe6,0x00,0xf0,0x00,0x30,0x04]
+#CHECK: vllebrzg %v0, 0(%r15,%r1)       # encoding: [0xe6,0x0f,0x10,0x00,0x30,0x04]
+#CHECK: vllebrzg %v15, 0                # encoding: [0xe6,0xf0,0x00,0x00,0x30,0x04]
+#CHECK: vllebrzg %v31, 0                # encoding: [0xe6,0xf0,0x00,0x00,0x38,0x04]
+#CHECK: vllebrzg %v18, 1383(%r3,%r4)    # encoding: [0xe6,0x23,0x45,0x67,0x38,0x04]
 #CHECK: vllebrzg %v0, 0                 # encoding: [0xe6,0x00,0x00,0x00,0x30,0x04]
 #CHECK: vllebrzg %v0, 4095              # encoding: [0xe6,0x00,0x0f,0xff,0x30,0x04]
 #CHECK: vllebrzg %v0, 0(%r15)           # encoding: [0xe6,0x00,0xf0,0x00,0x30,0x04]
@@ -39,6 +46,14 @@
 	ldrv	%f15, 0
 	ldrv	%v31, 0
 	ldrv	%v18, 0x567(%r3,%r4)
+	ldrv	0, 0
+	ldrv	0, 4095
+	ldrv	0, 0(15)
+	ldrv	0, 0(15,1)
+	ldrv	15, 0
+	ldrv	31, 0
+	ldrv	18, 0x567(3,4)
+
 
 #CHECK: vllebrze %v0, 0                 # encoding: [0xe6,0x00,0x00,0x00,0x60,0x04]
 #CHECK: vllebrze %v0, 4095              # encoding: [0xe6,0x00,0x0f,0xff,0x60,0x04]
diff --git a/llvm/test/MC/SystemZ/insn-good.s b/llvm/test/MC/SystemZ/insn-good.s
index 3c5b34abe0a9d..07f721bfa5e49 100644
--- a/llvm/test/MC/SystemZ/insn-good.s
+++ b/llvm/test/MC/SystemZ/insn-good.s
@@ -1,6 +1,13 @@
 # For z10 and above.
 # RUN: llvm-mc -triple s390x-linux-gnu -show-encoding %s | FileCheck %s
 
+#CHECK: a	%r0, 0                  # encoding: [0x5a,0x00,0x00,0x00]
+#CHECK: a	%r0, 4095               # encoding: [0x5a,0x00,0x0f,0xff]
+#CHECK: a	%r0, 0(%r1)             # encoding: [0x5a,0x00,0x10,0x00]
+#CHECK: a	%r0, 0(%r15)            # encoding: [0x5a,0x00,0xf0,0x00]
+#CHECK: a	%r0, 4095(%r1,%r15)     # encoding: [0x5a,0x01,0xff,0xff]
+#CHECK: a	%r0, 4095(%r15,%r1)     # encoding: [0x5a,0x0f,0x1f,0xff]
+#CHECK: a	%r15, 0                 # encoding: [0x5a,0xf0,0x00,0x00]
 #CHECK: a	%r0, 0                  # encoding: [0x5a,0x00,0x00,0x00]
 #CHECK: a	%r0, 4095               # encoding: [0x5a,0x00,0x0f,0xff]
 #CHECK: a	%r0, 0(%r1)             # encoding: [0x5a,0x00,0x10,0x00]
@@ -16,6 +23,14 @@
 	a	%r0, 4095(%r1,%r15)
 	a	%r0, 4095(%r15,%r1)
 	a	%r15, 0
+	a	0, 0
+	a	0, 4095
+	a	0, 0(1)
+	a	0, 0(15)
+	a	0, 4095(1,15)
+	a	0, 4095(15,1)
+	a	15, 0
+
 
 #CHECK: ad	%f0, 0                  # encoding: [0x6a,0x00,0x00,0x00]
 #CHECK: ad	%f0, 4095               # encoding: [0x6a,0x00,0x0f,0xff]
@@ -319,6 +334,13 @@
 	ahy	%r0, 524287(%r15,%r1)
 	ahy	%r15, 0
 
+#CHECK: al	%r0, 0                  # encoding: [0x5e,0x00,0x00,0x00]
+#CHECK: al	%r0, 4095               # encoding: [0x5e,0x00,0x0f,0xff]
+#CHECK: al	%r0, 0(%r1)             # encoding: [0x5e,0x00,0x10,0x00]
+#CHECK: al	%r0, 0(%r15)            # encoding: [0x5e,0x00,0xf0,0x00]
+#CHECK: al	%r0, 4095(%r1,%r15)     # encoding: [0x5e,0x01,0xff,0xff]
+#CHECK: al	%r0, 4095(%r15,%r1)     # encoding: [0x5e,0x0f,0x1f,0xff]
+#CHECK: al	%r15, 0                 # encoding: [0x5e,0xf0,0x00,0x00]
 #CHECK: al	%r0, 0                  # encoding: [0x5e,0x00,0x00,0x00]
 #CHECK: al	%r0, 4095               # encoding: [0x5e,0x00,0x0f,0xff]
 #CHECK: al	%r0, 0(%r1)             # encoding: [0x5e,0x00,0x10,0x00]
@@ -334,6 +356,13 @@
 	al	%r0, 4095(%r1,%r15)
 	al	%r0, 4095(%r15,%r1)
 	al	%r15, 0
+	al	0, 0
+	al	0, 4095
+	al	0, 0(1)
+	al	0, 0(15)
+	al	0, 4095(1,15)
+	al	0, 4095(15,1)
+	al	15, 0
 
 #CHECK: alc	%r0, -524288            # encoding: [0xe3,0x00,0x00,0x00,0x80,0x98]
 #CHECK: alc	%r0, -1                 # encoding: [0xe3,0x00,0x0f,0xff,0xff,0x98]
@@ -407,6 +436,16 @@
 	alfi	%r0, (1 << 32) - 1
 	alfi	%r15, 0
 
+#CHECK: alg	%r0, -524288            # encoding: [0xe3,0x00,0x00,0x00,0x80,0x0a]
+#CHECK: alg	%r0, -1                 # encoding: [0xe3,0x00,0x0f,0xff,0xff,0x0a]
+#CHECK: alg	%r0, 0                  # encoding: [0xe3,0x00,0x00,0x00,0x00,0x0a]
+#CHECK: alg	%r0, 1                  # encoding: [0xe3,0x00,0x00,0x01,0x00,0x0a]
+#CHECK: alg	%r0, 524287             # encoding: [0xe3,0x00,0x0f,0xff,0x7f,0x0a]
+#CHECK: alg	%r0, 0(%r1)             # encoding: [0xe3,0x00,0x10,0x00,0x00,0x0a]
+#CHECK: alg	%r0, 0(%r15)            # encoding: [0xe3,0x00,0xf0,0x00,0x00,0x0a]
+#CHECK: alg	%r0, 524287(%r1,%r15)   # encoding: [0xe3,0x01,0xff,0xff,0x7f,0x0a]
+#CHECK: alg	%r0, 524287(%r15,%r1)   # encoding: [0xe3,0x0f,0x1f,0xff,0x7f,0x0a]
+#CHECK: alg	%r15, 0                 # encoding: [0xe3,0xf0,0x00,0x00,0x00,0x0a]
 #CHECK: alg	%r0, -524288            # encoding: [0xe3,0x00,0x00,0x00,0x80,0x0a]
 #CHECK: alg	%r0, -1                 # encoding: [0xe3,0x00,0x0f,0xff,0xff,0x0a]
 #CHECK: alg	%r0, 0                  # encoding: [0xe3,0x00,0x00,0x00,0x00,0x0a]
@@ -428,6 +467,16 @@
 	alg	%r0, 524287(%r1,%r15)
 	alg	%r0, 524287(%r15,%r1)
 	alg	%r15, 0
+	alg	0, -524288
+	alg	0, -1
+	alg	0, 0
+	alg	0, 1
+	alg	0, 524287
+	alg	0, 0(1)
+	alg	0, 0(15)
+	alg	0, 524287(1,15)
+	alg	0, 524287(15,1)
+	alg	15, 0
 
 #CHECK: algf	%r0, -524288            # encoding: [0xe3,0x00,0x00,0x00,0x80,0x1a]
 #CHECK: algf	%r0, -1                 # encoding: [0xe3,0x00,0x0f,0xff,0xff,0x1a]
@@ -479,6 +528,19 @@
 	algr	%r15,%r0
 	algr	%r7,%r8
 
+#CHECK: algsi	-524288, 0              # encoding: [0xeb,0x00,0x00,0x00,0x80,0x7e]
+#CHECK: algsi	-1, 0                   # encoding: [0xeb,0x00,0x0f,0xff,0xff,0x7e]
+#CHECK: algsi	0, 0                    # encoding: [0xeb,0x00,0x00,0x00,0x00,0x7e]
+#CHECK: algsi	1, 0                    # encoding: [0xeb,0x00,0x00,0x01,0x00,0x7e]
+#CHECK: algsi	524287, 0               # encoding: [0xeb,0x00,0x0f,0xff,0x7f,0x7e]
+#CHECK: algsi	0, -128                 # encoding: [0xeb,0x80,0x00,0x00,0x00,0x7e]
+#CHECK: algsi	0, -1                   # encoding: [0xeb,0xff,0x00,0x00,0x00,0x7e]
+#CHECK: algsi	0, 1                    # encoding: [0xeb,0x01,0x00,0x00,0x00,0x7e]
+#CHECK: algsi	0, 127                  # encoding: [0xeb,0x7f,0x00,0x00,0x00,0x7e]
+#CHECK: algsi	0(%r1), 42              # encoding: [0xeb,0x2a,0x10,0x00,0x00,0x7e]
+#CHECK: algsi	0(%r15), 42             # encoding: [0xeb,0x2a,0xf0,0x00,0x00,0x7e]
+#CHECK: algsi	524287(%r1), 42         # encoding: [0xeb,0x2a,0x1f,0xff,0x7f,0x7e]
+#CHECK: algsi	524287(%r15), 42        # encoding: [0xeb,0x2a,0xff,0xff,0x7f,0x7e]
 #CHECK: algsi	-524288, 0              # encoding: [0xeb,0x00,0x00,0x00,0x80,0x7e]
 #CHECK: algsi	-1, 0                   # encoding: [0xeb,0x00,0x0f,0xff,0xff,0x7e]
 #CHECK: algsi	0, 0                    # encoding: [0xeb,0x00,0x00,0x00,0x00,0x7e]
@@ -506,6 +568,19 @@
 	algsi	0(%r15), 42
 	algsi	524287(%r1), 42
 	algsi	524287(%r15), 42
+	algsi	-524288, 0
+	algsi	-1, 0
+	algsi	0, 0
+	algsi	1, 0
+	algsi	524287, 0
+	algsi	0, -128
+	algsi	0, -1
+	algsi	0, 1
+	algsi	0, 127
+	algsi	0(1), 42
+	algsi	0(15), 42
+	algsi	524287(1), 42
+	algsi	524287(15), 42
 
 #CHECK: alr	%r0, %r0                # encoding: [0x1e,0x00]
 #CHECK: alr	%r0, %r15               # encoding: [0x1e,0x0f]
@@ -596,6 +671,20 @@
 	ap	0(16,%r15), 0(1)
 	ap	0(1), 0(16,%r1)
 	ap	0(1), 0(16,%r15)
+	ap	0(1), 0(1)
+	ap	0(1), 0(1,1)
+	ap	0(1), 0(1,15)
+	ap	0(1), 4095(1)
+	ap	0(1), 4095(1,1)
+	ap	0(1), 4095(1,15)
+	ap	0(1,1), 0(1)
+	ap	0(1,15), 0(1)
+	ap	4095(1,1), 0(1)
+	ap	4095(1,15), 0(1)
+	ap	0(16,1), 0(1)
+	ap	0(16,15), 0(1)
+	ap	0(1), 0(16,1)
+	ap	0(1), 0(16,15)
 
 #CHECK: ar	%r0, %r0                # encoding: [0x1a,0x00]
 #CHECK: ar	%r0, %r15               # encoding: [0x1a,0x0f]
diff --git a/llvm/test/MC/SystemZ/regs-good.s b/llvm/test/MC/SystemZ/regs-good.s
index c6157b5e2c855..b4c1edd1b591e 100644
--- a/llvm/test/MC/SystemZ/regs-good.s
+++ b/llvm/test/MC/SystemZ/regs-good.s
@@ -152,6 +152,29 @@
 	lctl	%c14,%c15,0
 	lctl	0,15,0
 
+#CHECK: st	%r0, 0                  # encoding: [0x50,0x00,0x00,0x00]
+#CHECK: st	%r0, 4095               # encoding: [0x50,0x00,0x0f,0xff]
+#CHECK: st	%r0, 0(%r1)             # encoding: [0x50,0x00,0x10,0x00]
+#CHECK: st	%r0, 0(%r15)            # encoding: [0x50,0x00,0xf0,0x00]
+#CHECK: st	%r0, 4095(%r1,%r15)     # encoding: [0x50,0x01,0xff,0xff]
+#CHECK: st	%r0, 4095(%r15,%r1)     # encoding: [0x50,0x0f,0x1f,0xff]
+#CHECK: st	%r15, 0                 # encoding: [0x50,0xf0,0x00,0x00]
+#CHECK: st	%r0, 0(%r1)             # encoding: [0x50,0x00,0x10,0x00]
+#CHECK: st	%r0, 0(%r15)            # encoding: [0x50,0x00,0xf0,0x00]
+#CHECK: st	%r0, 4095(%r1,%r15)     # encoding: [0x50,0x01,0xff,0xff]
+#CHECK: st	%r0, 4095(%r15,%r1)     # encoding: [0x50,0x0f,0x1f,0xff]
+
+	st	%r0, 0
+	st	%r0, 4095
+	st	%r0, 0(%r1)
+	st	%r0, 0(%r15)
+	st	%r0, 4095(%r1,%r15)
+	st	%r0, 4095(%r15,%r1)
+	st	%r15, 0
+	st	0, 0(1)
+	st	0, 0(15)
+	st	0, 4095(1,15)
+	st	0, 4095(15,1)
 
 #CHECK: .cfi_offset %r0, 0
 #CHECK: .cfi_offset %r1, 8
diff --git a/llvm/test/MC/SystemZ/tokens.s b/llvm/test/MC/SystemZ/tokens.s
index 177378615ed42..bf8c4e9067856 100644
--- a/llvm/test/MC/SystemZ/tokens.s
+++ b/llvm/test/MC/SystemZ/tokens.s
@@ -57,6 +57,14 @@
 #CHECK: foo	%, 200
 #CHECK: error: unknown token in expression
 #CHECK: foo	{, 200
+#CHECK: error: invalid instruction
+#CHECK: foo	100(15), 300
+#CHECK: error: register expected
+#CHECK: foo	100(15,), 300
+#CHECK: error: invalid instruction
+#CHECK: foo	100(15,%r1), 300
+#CHECK: error: invalid instruction
+#CHECK: foo	100(%v20,10), 300
 
 	foo	100, 200
 	foo	100(, 200
@@ -86,3 +94,7 @@
 	foo	%c, 200
 	foo	%, 200
 	foo	{, 200
+	foo	100(15), 300
+	foo	100(15,), 300
+	foo	100(15,%r1), 300
+	foo	100(%v20,10), 300
diff --git a/llvm/test/MC/VE/sym-br.s b/llvm/test/MC/VE/sym-br.s
new file mode 100644
index 0000000000000..87800b518b063
--- /dev/null
+++ b/llvm/test/MC/VE/sym-br.s
@@ -0,0 +1,10 @@
+# RUN: llvm-mc -triple=ve %s -o - | FileCheck %s
+# RUN: llvm-mc -triple=ve -filetype=obj %s -o - | llvm-objdump -r - | FileCheck %s --check-prefix=CHECK-OBJ
+
+        b.l.t tgt(, %s1)
+        b.l.t tgt+24(, %s1)
+# CHECK: b.l.t tgt(, %s1)
+# CHECK-NEXT: b.l.t tgt+24(, %s1)
+
+# CHECK-OBJ: 0 R_VE_REFLONG tgt
+# CHECK-OBJ-NEXT: 8 R_VE_REFLONG tgt+0x18
diff --git a/llvm/test/MC/VE/symbols.s b/llvm/test/MC/VE/symbols.s
index 55702f89258cc..1f1d9a341af6d 100644
--- a/llvm/test/MC/VE/symbols.s
+++ b/llvm/test/MC/VE/symbols.s
@@ -5,11 +5,19 @@
         lea %s1, var@lo
         and %s1, %s1, (32)0
         lea.sl %s1, var@hi(, %s1)
+        lea %s1, var+8@lo
+        and %s1, %s1, (32)0
+        lea.sl %s1, var+8@hi(, %s1)
 # CHECK: lea %s0, var
 # CHECK-NEXT: lea %s1, var@lo
 # CHECK-NEXT: and %s1, %s1, (32)0
 # CHECK-NEXT: lea.sl %s1, var@hi(, %s1)
+# CHECK-NEXT: lea %s1, var+8@lo
+# CHECK-NEXT: and %s1, %s1, (32)0
+# CHECK-NEXT: lea.sl %s1, var+8@hi(, %s1)
 
 # CHECK-OBJ: 0 R_VE_REFLONG var
 # CHECK-OBJ-NEXT: 8 R_VE_LO32 var
 # CHECK-OBJ-NEXT: 18 R_VE_HI32 var
+# CHECK-OBJ-NEXT: 20 R_VE_LO32 var+0x8
+# CHECK-OBJ-NEXT: 30 R_VE_HI32 var+0x8
diff --git a/llvm/test/MC/WebAssembly/atomics-encodings.s b/llvm/test/MC/WebAssembly/atomics-encodings.s
index 119de8f87df4c..56dd90cfe274d 100644
--- a/llvm/test/MC/WebAssembly/atomics-encodings.s
+++ b/llvm/test/MC/WebAssembly/atomics-encodings.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -show-encoding -triple=wasm32-unkown-unknown -mattr=+atomics < %s | FileCheck %s
+# RUN: llvm-mc -show-encoding -triple=wasm32-unknown-unknown -mattr=+atomics < %s | FileCheck %s
 
 main:
   .functype main () -> ()
diff --git a/llvm/test/MC/WebAssembly/bulk-memory-encodings.s b/llvm/test/MC/WebAssembly/bulk-memory-encodings.s
index 7047d6428af13..0863527c8b367 100644
--- a/llvm/test/MC/WebAssembly/bulk-memory-encodings.s
+++ b/llvm/test/MC/WebAssembly/bulk-memory-encodings.s
@@ -1,4 +1,5 @@
-# RUN: llvm-mc -show-encoding -triple=wasm32-unkown-unknown -mattr=+bulk-memory < %s | FileCheck %s
+# RUN: llvm-mc -show-encoding -triple=wasm32-unknown-unknown -mattr=+bulk-memory < %s | FileCheck %s
+# RUN: llvm-mc -show-encoding -triple=wasm64-unknown-unknown -mattr=+bulk-memory < %s | FileCheck %s
 
 main:
     .functype main () -> ()
diff --git a/llvm/test/MC/WebAssembly/externref.s b/llvm/test/MC/WebAssembly/externref.s
new file mode 100644
index 0000000000000..f660e4627911a
--- /dev/null
+++ b/llvm/test/MC/WebAssembly/externref.s
@@ -0,0 +1,36 @@
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -filetype=obj < %s | obj2yaml | FileCheck %s
+
+# Tests use of externref type, including storing it in a global
+
+.globaltype my_global, externref
+
+.globl read_externref
+read_externref:
+  .functype read_externref () -> (externref)
+  global.get my_global
+  end_function
+
+.globl write_externref
+write_externref:
+  .functype write_externref (externref) -> ()
+  local.get 0
+  global.set my_global
+  end_function
+
+.globl call_with_ref
+call_with_ref:
+  .functype call_with_ref () -> ()
+  call read_externref
+  call write_externref
+  end_function
+
+my_global:
+
+#      CHECK:  - Type:            GLOBAL
+# CHECK-NEXT:    Globals:
+# CHECK-NEXT:      - Index:           0
+# CHECK-NEXT:        Type:            EXTERNREF
+# CHECK-NEXT:        Mutable:         true
+# CHECK-NEXT:        InitExpr:
+# CHECK-NEXT:          Opcode:          REF_NULL
+# CHECK-NEXT:          Type:            EXTERNREF
diff --git a/llvm/test/MC/WebAssembly/simd-encodings.s b/llvm/test/MC/WebAssembly/simd-encodings.s
index 2bb77c98ea183..5818588380a1f 100644
--- a/llvm/test/MC/WebAssembly/simd-encodings.s
+++ b/llvm/test/MC/WebAssembly/simd-encodings.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -show-encoding -triple=wasm32-unkown-unknown -mattr=+unimplemented-simd128 < %s | FileCheck %s
+# RUN: llvm-mc -show-encoding -triple=wasm32-unknown-unknown -mattr=+unimplemented-simd128 < %s | FileCheck %s
 
 main:
     .functype main () -> ()
diff --git a/llvm/test/MC/WebAssembly/stack-ptr.ll b/llvm/test/MC/WebAssembly/stack-ptr.ll
index 147f04ffec158..b92c773c085fa 100644
--- a/llvm/test/MC/WebAssembly/stack-ptr.ll
+++ b/llvm/test/MC/WebAssembly/stack-ptr.ll
@@ -1,6 +1,5 @@
-; RUN: llc -filetype=obj %s -o - | obj2yaml | FileCheck %s
-
-target triple = "wasm32-unknown-unknown"
+; RUN: llc --mtriple=wasm32-unknown-unknown -filetype=obj %s -o - | obj2yaml | FileCheck --check-prefixes CHECK,CHK32 %s
+; RUN: llc --mtriple=wasm64-unknown-unknown -filetype=obj %s -o - | obj2yaml | FileCheck --check-prefixes CHECK,CHK64 %s
 
 ; Function that uses explict stack, and should generate a reference to
 ; __stack_pointer, along with the corresponding reloction entry.
@@ -15,7 +14,8 @@ entry:
 ; CHECK:       - Module:          env
 ; CHECK:         Field:           __stack_pointer
 ; CHECK:         Kind:            GLOBAL
-; CHECK:         GlobalType:      I32
+; CHK32:         GlobalType:      I32
+; CHK64:         GlobalType:      I64
 ; CHECK:         GlobalMutable:   true
 ; CHECK:   - Type:            CODE
 ; CHECK:     Relocations:
diff --git a/llvm/test/MC/WebAssembly/tail-call-encodings.s b/llvm/test/MC/WebAssembly/tail-call-encodings.s
index 69839adcfbfa5..405e6337f5fa9 100644
--- a/llvm/test/MC/WebAssembly/tail-call-encodings.s
+++ b/llvm/test/MC/WebAssembly/tail-call-encodings.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -show-encoding -triple=wasm32-unkown-unknown -mattr=+tail-call < %s | FileCheck %s
+# RUN: llvm-mc -show-encoding -triple=wasm32-unknown-unknown -mattr=+tail-call < %s | FileCheck %s
 
 bar1:
     .functype bar1 () -> ()
diff --git a/llvm/test/MC/WebAssembly/wasm64.s b/llvm/test/MC/WebAssembly/wasm64.s
index e31bfcf123874..b89718816a9f8 100644
--- a/llvm/test/MC/WebAssembly/wasm64.s
+++ b/llvm/test/MC/WebAssembly/wasm64.s
@@ -51,6 +51,11 @@ test:
     i64.const   0
     f32.store   .L.str    # relocatable offset!
 
+    ### 64-bit SP
+
+    global.get  __stack_pointer
+    drop
+
     end_function
 
     .section    .rodata..L.str,"",@
@@ -62,7 +67,7 @@ test:
     .size       .L.str, 24
 
     .globaltype myglob64, i64
-
+    .globaltype __stack_pointer, i64
 
 
 # CHECK:              .functype       test (i64) -> ()
@@ -142,6 +147,7 @@ test:
 # BIN-NEXT:         Field:           __linear_memory
 # BIN-NEXT:         Kind:            MEMORY
 # BIN-NEXT:         Memory:
+# BIN-NEXT:           Flags:           [ IS_64 ]
 # BIN-NEXT:           Initial:         0x00000001
 # BIN-NEXT:       - Module:          env
 # BIN-NEXT:         Field:           __indirect_function_table
@@ -155,6 +161,11 @@ test:
 # BIN-NEXT:         Kind:            GLOBAL
 # BIN-NEXT:         GlobalType:      I64
 # BIN-NEXT:         GlobalMutable:   true
+# BIN-NEXT:       - Module:          env
+# BIN-NEXT:         Field:           __stack_pointer
+# BIN-NEXT:         Kind:            GLOBAL
+# BIN-NEXT:         GlobalType:      I64
+# BIN-NEXT:         GlobalMutable:   true
 # BIN-NEXT:   - Type:            FUNCTION
 # BIN-NEXT:     FunctionTypes:   [ 0 ]
 # BIN-NEXT:   - Type:            DATACOUNT
@@ -179,12 +190,15 @@ test:
 # BIN-NEXT:       - Type:            R_WASM_MEMORY_ADDR_LEB64
 # BIN-NEXT:         Index:           1
 # BIN-NEXT:         Offset:          0x00000078
+# BIN-NEXT:       - Type: R_WASM_GLOBAL_INDEX_LEB
+# BIN-NEXT:         Index: 3
+# BIN-NEXT:         Offset: 0x00000083
 # BIN-NEXT:     Functions:
 # BIN-NEXT:       - Index:           0
 # BIN-NEXT:         Locals:
 # BIN-NEXT:           - Type:            I64
 # BIN-NEXT:             Count:           1
-# BIN-NEXT:         Body:            42002A02001A20002A02001A42808080808080808080002A02001A2380808080002A02001A42002A02808080808080808080001A4300000000420038020043000000002000380200430000000042808080808080808080003802004300000000238080808000380200430000000042003802808080808080808080000B
+# BIN-NEXT:         Body:            42002A02001A20002A02001A42808080808080808080002A02001A2380808080002A02001A42002A02808080808080808080001A4300000000420038020043000000002000380200430000000042808080808080808080003802004300000000238080808000380200430000000042003802808080808080808080002381808080001A0B
 # BIN-NEXT:   - Type:            DATA
 # BIN-NEXT:     Relocations:
 # BIN-NEXT:       - Type:            R_WASM_MEMORY_ADDR_I64
@@ -217,6 +231,11 @@ test:
 # BIN-NEXT:         Name:            myglob64
 # BIN-NEXT:         Flags:           [ UNDEFINED ]
 # BIN-NEXT:         Global:          0
+# BIN-NEXT:       - Index:           3
+# BIN-NEXT:         Kind:            GLOBAL
+# BIN-NEXT:         Name:            __stack_pointer
+# BIN-NEXT:         Flags:           [ UNDEFINED ]
+# BIN-NEXT:         Global:          1
 # BIN-NEXT:     SegmentInfo:
 # BIN-NEXT:       - Index:           0
 # BIN-NEXT:         Name:            .rodata..L.str
diff --git a/llvm/test/MC/X86/AMX/x86-64-amx-bf16-att.s b/llvm/test/MC/X86/AMX/x86-64-amx-bf16-att.s
new file mode 100644
index 0000000000000..5a8e759d63098
--- /dev/null
+++ b/llvm/test/MC/X86/AMX/x86-64-amx-bf16-att.s
@@ -0,0 +1,34 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown -show-encoding %s | FileCheck %s
+// some AMX instruction must use SIB.
+
+// CHECK: tdpbf16ps %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x5a,0x5c,0xf5]
+          tdpbf16ps %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbf16ps %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x72,0x5c,0xda]
+          tdpbf16ps %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbf16ps %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x5a,0x5c,0xf5]
+          tdpbf16ps %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbf16ps %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x72,0x5c,0xda]
+          tdpbf16ps %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbf16ps %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x5a,0x5c,0xf5]
+          tdpbf16ps %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbf16ps %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x72,0x5c,0xda]
+          tdpbf16ps %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbf16ps %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x5a,0x5c,0xf5]
+          tdpbf16ps %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbf16ps %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x72,0x5c,0xda]
+          tdpbf16ps %tmm1, %tmm2, %tmm3
diff --git a/llvm/test/MC/X86/AMX/x86-64-amx-bf16-intel.s b/llvm/test/MC/X86/AMX/x86-64-amx-bf16-intel.s
new file mode 100644
index 0000000000000..b3b1e275ed3bf
--- /dev/null
+++ b/llvm/test/MC/X86/AMX/x86-64-amx-bf16-intel.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s
+
+// CHECK: tdpbf16ps tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x5a,0x5c,0xf5]
+          tdpbf16ps tmm6, tmm5, tmm4
+
+// CHECK: tdpbf16ps tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x72,0x5c,0xda]
+          tdpbf16ps tmm3, tmm2, tmm1
+
+// CHECK: tdpbf16ps tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x5a,0x5c,0xf5]
+          tdpbf16ps tmm6, tmm5, tmm4
+
+// CHECK: tdpbf16ps tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x72,0x5c,0xda]
+          tdpbf16ps tmm3, tmm2, tmm1
+
+// CHECK: tdpbf16ps tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x5a,0x5c,0xf5]
+          tdpbf16ps tmm6, tmm5, tmm4
+
+// CHECK: tdpbf16ps tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x72,0x5c,0xda]
+          tdpbf16ps tmm3, tmm2, tmm1
+
+// CHECK: tdpbf16ps tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x5a,0x5c,0xf5]
+          tdpbf16ps tmm6, tmm5, tmm4
+
+// CHECK: tdpbf16ps tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x72,0x5c,0xda]
+          tdpbf16ps tmm3, tmm2, tmm1
diff --git a/llvm/test/MC/X86/AMX/x86-64-amx-error.s b/llvm/test/MC/X86/AMX/x86-64-amx-error.s
new file mode 100644
index 0000000000000..e92fe9ee9a83e
--- /dev/null
+++ b/llvm/test/MC/X86/AMX/x86-64-amx-error.s
@@ -0,0 +1,10 @@
+// RUN: not llvm-mc -triple x86_64-unknown-unknown %s -o /dev/null 2>&1 | FileCheck %s
+
+// CHECK: invalid operand for instruction
+tileloadd (%rip), %tmm0
+
+// CHECK: invalid operand for instruction
+tileloaddt1 1(%rip), %tmm1
+
+// CHECK: invalid operand for instruction
+tilestored %tmm2, (%rip)
diff --git a/llvm/test/MC/X86/AMX/x86-64-amx-int8-att.s b/llvm/test/MC/X86/AMX/x86-64-amx-int8-att.s
new file mode 100644
index 0000000000000..c9e693c144de0
--- /dev/null
+++ b/llvm/test/MC/X86/AMX/x86-64-amx-int8-att.s
@@ -0,0 +1,130 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown -show-encoding %s | FileCheck %s
+// some AMX instruction must use SIB.
+
+// CHECK: tdpbssd %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x5b,0x5e,0xf5]
+          tdpbssd %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbssd %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x73,0x5e,0xda]
+          tdpbssd %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbsud %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x5a,0x5e,0xf5]
+          tdpbsud %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbsud %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x72,0x5e,0xda]
+          tdpbsud %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbusd %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x59,0x5e,0xf5]
+          tdpbusd %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbusd %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x71,0x5e,0xda]
+          tdpbusd %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbuud %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x58,0x5e,0xf5]
+          tdpbuud %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbuud %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x70,0x5e,0xda]
+          tdpbuud %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbssd %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x5b,0x5e,0xf5]
+          tdpbssd %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbssd %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x73,0x5e,0xda]
+          tdpbssd %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbsud %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x5a,0x5e,0xf5]
+          tdpbsud %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbsud %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x72,0x5e,0xda]
+          tdpbsud %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbusd %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x59,0x5e,0xf5]
+          tdpbusd %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbusd %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x71,0x5e,0xda]
+          tdpbusd %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbuud %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x58,0x5e,0xf5]
+          tdpbuud %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbuud %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x70,0x5e,0xda]
+          tdpbuud %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbssd %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x5b,0x5e,0xf5]
+          tdpbssd %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbssd %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x73,0x5e,0xda]
+          tdpbssd %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbsud %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x5a,0x5e,0xf5]
+          tdpbsud %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbsud %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x72,0x5e,0xda]
+          tdpbsud %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbusd %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x59,0x5e,0xf5]
+          tdpbusd %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbusd %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x71,0x5e,0xda]
+          tdpbusd %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbuud %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x58,0x5e,0xf5]
+          tdpbuud %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbuud %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x70,0x5e,0xda]
+          tdpbuud %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbssd %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x5b,0x5e,0xf5]
+          tdpbssd %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbssd %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x73,0x5e,0xda]
+          tdpbssd %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbsud %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x5a,0x5e,0xf5]
+          tdpbsud %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbsud %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x72,0x5e,0xda]
+          tdpbsud %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbusd %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x59,0x5e,0xf5]
+          tdpbusd %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbusd %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x71,0x5e,0xda]
+          tdpbusd %tmm1, %tmm2, %tmm3
+
+// CHECK: tdpbuud %tmm4, %tmm5, %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x58,0x5e,0xf5]
+          tdpbuud %tmm4, %tmm5, %tmm6
+
+// CHECK: tdpbuud %tmm1, %tmm2, %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x70,0x5e,0xda]
+          tdpbuud %tmm1, %tmm2, %tmm3
diff --git a/llvm/test/MC/X86/AMX/x86-64-amx-int8-intel.s b/llvm/test/MC/X86/AMX/x86-64-amx-int8-intel.s
new file mode 100644
index 0000000000000..6b28b80c66608
--- /dev/null
+++ b/llvm/test/MC/X86/AMX/x86-64-amx-int8-intel.s
@@ -0,0 +1,129 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s
+
+// CHECK: tdpbssd tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x5b,0x5e,0xf5]
+          tdpbssd tmm6, tmm5, tmm4
+
+// CHECK: tdpbssd tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x73,0x5e,0xda]
+          tdpbssd tmm3, tmm2, tmm1
+
+// CHECK: tdpbsud tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x5a,0x5e,0xf5]
+          tdpbsud tmm6, tmm5, tmm4
+
+// CHECK: tdpbsud tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x72,0x5e,0xda]
+          tdpbsud tmm3, tmm2, tmm1
+
+// CHECK: tdpbusd tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x59,0x5e,0xf5]
+          tdpbusd tmm6, tmm5, tmm4
+
+// CHECK: tdpbusd tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x5e,0xda]
+          tdpbusd tmm3, tmm2, tmm1
+
+// CHECK: tdpbuud tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x58,0x5e,0xf5]
+          tdpbuud tmm6, tmm5, tmm4
+
+// CHECK: tdpbuud tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x70,0x5e,0xda]
+          tdpbuud tmm3, tmm2, tmm1
+
+// CHECK: tdpbssd tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x5b,0x5e,0xf5]
+          tdpbssd tmm6, tmm5, tmm4
+
+// CHECK: tdpbssd tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x73,0x5e,0xda]
+          tdpbssd tmm3, tmm2, tmm1
+
+// CHECK: tdpbsud tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x5a,0x5e,0xf5]
+          tdpbsud tmm6, tmm5, tmm4
+
+// CHECK: tdpbsud tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x72,0x5e,0xda]
+          tdpbsud tmm3, tmm2, tmm1
+
+// CHECK: tdpbusd tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x59,0x5e,0xf5]
+          tdpbusd tmm6, tmm5, tmm4
+
+// CHECK: tdpbusd tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x5e,0xda]
+          tdpbusd tmm3, tmm2, tmm1
+
+// CHECK: tdpbuud tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x58,0x5e,0xf5]
+          tdpbuud tmm6, tmm5, tmm4
+
+// CHECK: tdpbuud tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x70,0x5e,0xda]
+          tdpbuud tmm3, tmm2, tmm1
+
+// CHECK: tdpbssd tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x5b,0x5e,0xf5]
+          tdpbssd tmm6, tmm5, tmm4
+
+// CHECK: tdpbssd tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x73,0x5e,0xda]
+          tdpbssd tmm3, tmm2, tmm1
+
+// CHECK: tdpbsud tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x5a,0x5e,0xf5]
+          tdpbsud tmm6, tmm5, tmm4
+
+// CHECK: tdpbsud tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x72,0x5e,0xda]
+          tdpbsud tmm3, tmm2, tmm1
+
+// CHECK: tdpbusd tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x59,0x5e,0xf5]
+          tdpbusd tmm6, tmm5, tmm4
+
+// CHECK: tdpbusd tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x5e,0xda]
+          tdpbusd tmm3, tmm2, tmm1
+
+// CHECK: tdpbuud tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x58,0x5e,0xf5]
+          tdpbuud tmm6, tmm5, tmm4
+
+// CHECK: tdpbuud tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x70,0x5e,0xda]
+          tdpbuud tmm3, tmm2, tmm1
+
+// CHECK: tdpbssd tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x5b,0x5e,0xf5]
+          tdpbssd tmm6, tmm5, tmm4
+
+// CHECK: tdpbssd tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x73,0x5e,0xda]
+          tdpbssd tmm3, tmm2, tmm1
+
+// CHECK: tdpbsud tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x5a,0x5e,0xf5]
+          tdpbsud tmm6, tmm5, tmm4
+
+// CHECK: tdpbsud tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x72,0x5e,0xda]
+          tdpbsud tmm3, tmm2, tmm1
+
+// CHECK: tdpbusd tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x59,0x5e,0xf5]
+          tdpbusd tmm6, tmm5, tmm4
+
+// CHECK: tdpbusd tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x5e,0xda]
+          tdpbusd tmm3, tmm2, tmm1
+
+// CHECK: tdpbuud tmm6, tmm5, tmm4
+// CHECK: encoding: [0xc4,0xe2,0x58,0x5e,0xf5]
+          tdpbuud tmm6, tmm5, tmm4
+
+// CHECK: tdpbuud tmm3, tmm2, tmm1
+// CHECK: encoding: [0xc4,0xe2,0x70,0x5e,0xda]
+          tdpbuud tmm3, tmm2, tmm1
diff --git a/llvm/test/MC/X86/AMX/x86-64-amx-tile-att.s b/llvm/test/MC/X86/AMX/x86-64-amx-tile-att.s
new file mode 100644
index 0000000000000..9762c821f971f
--- /dev/null
+++ b/llvm/test/MC/X86/AMX/x86-64-amx-tile-att.s
@@ -0,0 +1,198 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown -show-encoding %s | FileCheck %s
+// some AMX instruction must use SIB.
+
+// CHECK: tilerelease
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0xc0]
+          tilerelease
+
+// CHECK: tilezero %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xf0]
+          tilezero %tmm6
+
+// CHECK: tilezero %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xd8]
+          tilezero %tmm3
+
+// CHECK: tilerelease
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0xc0]
+          tilerelease
+
+// CHECK: tilezero %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xf0]
+          tilezero %tmm6
+
+// CHECK: tilezero %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xd8]
+          tilezero %tmm3
+
+// CHECK: ldtilecfg  268435456(%rbp,%r14,8)
+// CHECK: encoding: [0xc4,0xa2,0x78,0x49,0x84,0xf5,0x00,0x00,0x00,0x10]
+          ldtilecfg  268435456(%rbp,%r14,8)
+
+// CHECK: ldtilecfg  291(%r8,%rax,4)
+// CHECK: encoding: [0xc4,0xc2,0x78,0x49,0x84,0x80,0x23,0x01,0x00,0x00]
+          ldtilecfg  291(%r8,%rax,4)
+
+// CHECK: ldtilecfg  (%rip)
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0x05,0x00,0x00,0x00,0x00]
+          ldtilecfg  (%rip)
+
+// CHECK: ldtilecfg  -2048(,%rbp,2)
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0x04,0x6d,0x00,0xf8,0xff,0xff]
+          ldtilecfg  -2048(,%rbp,2)
+
+// CHECK: sttilecfg  268435456(%rbp,%r14,8)
+// CHECK: encoding: [0xc4,0xa2,0x79,0x49,0x84,0xf5,0x00,0x00,0x00,0x10]
+          sttilecfg  268435456(%rbp,%r14,8)
+
+// CHECK: sttilecfg  291(%r8,%rax,4)
+// CHECK: encoding: [0xc4,0xc2,0x79,0x49,0x84,0x80,0x23,0x01,0x00,0x00]
+          sttilecfg  291(%r8,%rax,4)
+
+// CHECK: sttilecfg  (%rip)
+// CHECK: encoding: [0xc4,0xe2,0x79,0x49,0x05,0x00,0x00,0x00,0x00]
+          sttilecfg  (%rip)
+
+// CHECK: sttilecfg  -2048(,%rbp,2)
+// CHECK: encoding: [0xc4,0xe2,0x79,0x49,0x04,0x6d,0x00,0xf8,0xff,0xff]
+          sttilecfg  -2048(,%rbp,2)
+
+// CHECK: tileloadd 268435456(%rbp,%r14,8), %tmm6
+// CHECK: encoding: [0xc4,0xa2,0x7b,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          tileloadd 268435456(%rbp,%r14,8), %tmm6
+
+// CHECK: tileloadd 291(%r8,%rax,4), %tmm3
+// CHECK: encoding: [0xc4,0xc2,0x7b,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00]
+          tileloadd 291(%r8,%rax,4), %tmm3
+
+// CHECK: tileloadd 64(%rbx), %tmm4
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x4b,0x64,0x23,0x40]
+          tileloadd 64(%rbx), %tmm4
+
+// CHECK: tileloadd -32(,%rbp,2), %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff]
+          tileloadd -32(,%rbp,2), %tmm3
+
+// CHECK: tileloaddt1 268435456(%rbp,%r14,8), %tmm6
+// CHECK: encoding: [0xc4,0xa2,0x79,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          tileloaddt1 268435456(%rbp,%r14,8), %tmm6
+
+// CHECK: tileloaddt1 291(%r8,%rax,4), %tmm3
+// CHECK: encoding: [0xc4,0xc2,0x79,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00]
+          tileloaddt1 291(%r8,%rax,4), %tmm3
+
+// CHECK: tileloaddt1 -32(,%rbp,2), %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x79,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff]
+          tileloaddt1 -32(,%rbp,2), %tmm3
+
+// CHECK: tileloaddt1 16(%rbp), %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x79,0x4b,0x74,0x25,0x10]
+          tileloaddt1 16(%rbp), %tmm6
+
+// CHECK: tilerelease
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0xc0]
+          tilerelease
+
+// CHECK: tilestored %tmm6, 268435456(%rbp,%r14,8)
+// CHECK: encoding: [0xc4,0xa2,0x7a,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          tilestored %tmm6, 268435456(%rbp,%r14,8)
+
+// CHECK: tilestored %tmm3, 291(%r8,%rax,4)
+// CHECK: encoding: [0xc4,0xc2,0x7a,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00]
+          tilestored %tmm3, 291(%r8,%rax,4)
+
+// CHECK: tilestored %tmm3, -32(,%rbp,2)
+// CHECK: encoding: [0xc4,0xe2,0x7a,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff]
+          tilestored %tmm3, -32(,%rbp,2)
+
+// CHECK: tilestored %tmm3, (%r8)
+// CHECK: encoding: [0xc4,0xc2,0x7a,0x4b,0x1c,0x20]
+          tilestored %tmm3, (%r8)
+
+// CHECK: tilezero %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xf0]
+          tilezero %tmm6
+
+// CHECK: tilezero %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xd8]
+          tilezero %tmm3
+
+// CHECK: ldtilecfg  268435456(%rbp,%r14,8)
+// CHECK: encoding: [0xc4,0xa2,0x78,0x49,0x84,0xf5,0x00,0x00,0x00,0x10]
+          ldtilecfg  268435456(%rbp,%r14,8)
+
+// CHECK: ldtilecfg  291(%r8,%rax,4)
+// CHECK: encoding: [0xc4,0xc2,0x78,0x49,0x84,0x80,0x23,0x01,0x00,0x00]
+          ldtilecfg  291(%r8,%rax,4)
+
+// CHECK: ldtilecfg  (%rip)
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0x05,0x00,0x00,0x00,0x00]
+          ldtilecfg  (%rip)
+
+// CHECK: ldtilecfg  -2048(,%rbp,2)
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0x04,0x6d,0x00,0xf8,0xff,0xff]
+          ldtilecfg  -2048(,%rbp,2)
+
+// CHECK: sttilecfg  268435456(%rbp,%r14,8)
+// CHECK: encoding: [0xc4,0xa2,0x79,0x49,0x84,0xf5,0x00,0x00,0x00,0x10]
+          sttilecfg  268435456(%rbp,%r14,8)
+
+// CHECK: sttilecfg  291(%r8,%rax,4)
+// CHECK: encoding: [0xc4,0xc2,0x79,0x49,0x84,0x80,0x23,0x01,0x00,0x00]
+          sttilecfg  291(%r8,%rax,4)
+
+// CHECK: sttilecfg  (%rip)
+// CHECK: encoding: [0xc4,0xe2,0x79,0x49,0x05,0x00,0x00,0x00,0x00]
+          sttilecfg  (%rip)
+
+// CHECK: sttilecfg  -2048(,%rbp,2)
+// CHECK: encoding: [0xc4,0xe2,0x79,0x49,0x04,0x6d,0x00,0xf8,0xff,0xff]
+          sttilecfg  -2048(,%rbp,2)
+
+// CHECK: tileloadd 268435456(%rbp,%r14,8), %tmm6
+// CHECK: encoding: [0xc4,0xa2,0x7b,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          tileloadd 268435456(%rbp,%r14,8), %tmm6
+
+// CHECK: tileloadd 291(%r8,%rax,4), %tmm3
+// CHECK: encoding: [0xc4,0xc2,0x7b,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00]
+          tileloadd 291(%r8,%rax,4), %tmm3
+
+// CHECK: tileloadd -32(,%rbp,2), %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff]
+          tileloadd -32(,%rbp,2), %tmm3
+
+// CHECK: tileloaddt1 268435456(%rbp,%r14,8), %tmm6
+// CHECK: encoding: [0xc4,0xa2,0x79,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          tileloaddt1 268435456(%rbp,%r14,8), %tmm6
+
+// CHECK: tileloaddt1 291(%r8,%rax,4), %tmm3
+// CHECK: encoding: [0xc4,0xc2,0x79,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00]
+          tileloaddt1 291(%r8,%rax,4), %tmm3
+
+// CHECK: tileloaddt1 -32(,%rbp,2), %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x79,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff]
+          tileloaddt1 -32(,%rbp,2), %tmm3
+
+// CHECK: tilerelease
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0xc0]
+          tilerelease
+
+// CHECK: tilestored %tmm6, 268435456(%rbp,%r14,8)
+// CHECK: encoding: [0xc4,0xa2,0x7a,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          tilestored %tmm6, 268435456(%rbp,%r14,8)
+
+// CHECK: tilestored %tmm3, 291(%r8,%rax,4)
+// CHECK: encoding: [0xc4,0xc2,0x7a,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00]
+          tilestored %tmm3, 291(%r8,%rax,4)
+
+// CHECK: tilestored %tmm3, -32(,%rbp,2)
+// CHECK: encoding: [0xc4,0xe2,0x7a,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff]
+          tilestored %tmm3, -32(,%rbp,2)
+
+// CHECK: tilezero %tmm6
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xf0]
+          tilezero %tmm6
+
+// CHECK: tilezero %tmm3
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xd8]
+          tilezero %tmm3
diff --git a/llvm/test/MC/X86/AMX/x86-64-amx-tile-intel.s b/llvm/test/MC/X86/AMX/x86-64-amx-tile-intel.s
new file mode 100644
index 0000000000000..7fad214dac6a3
--- /dev/null
+++ b/llvm/test/MC/X86/AMX/x86-64-amx-tile-intel.s
@@ -0,0 +1,197 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s
+
+// CHECK: tilerelease
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0xc0]
+          tilerelease
+
+// CHECK: tilezero tmm6
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xf0]
+          tilezero tmm6
+
+// CHECK: tilezero tmm3
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xd8]
+          tilezero tmm3
+
+// CHECK: tilerelease
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0xc0]
+          tilerelease
+
+// CHECK: tilezero tmm6
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xf0]
+          tilezero tmm6
+
+// CHECK: tilezero tmm3
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xd8]
+          tilezero tmm3
+
+// CHECK: ldtilecfg [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0xc4,0xa2,0x78,0x49,0x84,0xf5,0x00,0x00,0x00,0x10]
+          ldtilecfg [rbp + 8*r14 + 268435456]
+
+// CHECK: ldtilecfg [r8 + 4*rax + 291]
+// CHECK: encoding: [0xc4,0xc2,0x78,0x49,0x84,0x80,0x23,0x01,0x00,0x00]
+          ldtilecfg [r8 + 4*rax + 291]
+
+// CHECK: ldtilecfg [rip]
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0x05,0x00,0x00,0x00,0x00]
+          ldtilecfg [rip]
+
+// CHECK: ldtilecfg [2*rbp - 2048]
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0x04,0x6d,0x00,0xf8,0xff,0xff]
+          ldtilecfg [2*rbp - 2048]
+
+// CHECK: sttilecfg [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0xc4,0xa2,0x79,0x49,0x84,0xf5,0x00,0x00,0x00,0x10]
+          sttilecfg [rbp + 8*r14 + 268435456]
+
+// CHECK: sttilecfg [r8 + 4*rax + 291]
+// CHECK: encoding: [0xc4,0xc2,0x79,0x49,0x84,0x80,0x23,0x01,0x00,0x00]
+          sttilecfg [r8 + 4*rax + 291]
+
+// CHECK: sttilecfg [rip]
+// CHECK: encoding: [0xc4,0xe2,0x79,0x49,0x05,0x00,0x00,0x00,0x00]
+          sttilecfg [rip]
+
+// CHECK: sttilecfg [2*rbp - 2048]
+// CHECK: encoding: [0xc4,0xe2,0x79,0x49,0x04,0x6d,0x00,0xf8,0xff,0xff]
+          sttilecfg [2*rbp - 2048]
+
+// CHECK: tileloadd tmm6, [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0xc4,0xa2,0x7b,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          tileloadd tmm6, [rbp + 8*r14 + 268435456]
+
+// CHECK: tileloadd tmm3, [r8 + 4*rax + 291]
+// CHECK: encoding: [0xc4,0xc2,0x7b,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00]
+          tileloadd tmm3, [r8 + 4*rax + 291]
+
+// CHECK: tileloadd tmm3, [2*rbp - 32]
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff]
+          tileloadd tmm3, [2*rbp - 32]
+
+// CHECK: tileloadd tmm4, [rbx + 64]
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x4b,0x64,0x23,0x40]
+          tileloadd tmm4, [rbx + 64]
+
+// CHECK: tileloaddt1 tmm6, [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0xc4,0xa2,0x79,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          tileloaddt1 tmm6, [rbp + 8*r14 + 268435456]
+
+// CHECK: tileloaddt1 tmm3, [r8 + 4*rax + 291]
+// CHECK: encoding: [0xc4,0xc2,0x79,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00]
+          tileloaddt1 tmm3, [r8 + 4*rax + 291]
+
+// CHECK: tileloaddt1 tmm3, [2*rbp - 32]
+// CHECK: encoding: [0xc4,0xe2,0x79,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff]
+          tileloaddt1 tmm3, [2*rbp - 32]
+
+// CHECK: tileloaddt1     tmm6, [rbp + 16]
+// CHECK: encoding: [0xc4,0xe2,0x79,0x4b,0x74,0x25,0x10]
+          tileloaddt1     tmm6, [rbp + 16]
+
+// CHECK: tilerelease
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0xc0]
+          tilerelease
+
+// CHECK: tilestored [rbp + 8*r14 + 268435456], tmm6
+// CHECK: encoding: [0xc4,0xa2,0x7a,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          tilestored [rbp + 8*r14 + 268435456], tmm6
+
+// CHECK: tilestored [r8 + 4*rax + 291], tmm3
+// CHECK: encoding: [0xc4,0xc2,0x7a,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00]
+          tilestored [r8 + 4*rax + 291], tmm3
+
+// CHECK: tilestored [2*rbp - 32], tmm3
+// CHECK: encoding: [0xc4,0xe2,0x7a,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff]
+          tilestored [2*rbp - 32], tmm3
+
+// CHECK: tilestored [r8], tmm3
+// CHECK: encoding: [0xc4,0xc2,0x7a,0x4b,0x1c,0x20]
+          tilestored [r8], tmm3
+
+// CHECK: tilezero tmm6
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xf0]
+          tilezero tmm6
+
+// CHECK: tilezero tmm3
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xd8]
+          tilezero tmm3
+
+// CHECK: ldtilecfg [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0xc4,0xa2,0x78,0x49,0x84,0xf5,0x00,0x00,0x00,0x10]
+          ldtilecfg [rbp + 8*r14 + 268435456]
+
+// CHECK: ldtilecfg [r8 + 4*rax + 291]
+// CHECK: encoding: [0xc4,0xc2,0x78,0x49,0x84,0x80,0x23,0x01,0x00,0x00]
+          ldtilecfg [r8 + 4*rax + 291]
+
+// CHECK: ldtilecfg [rip]
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0x05,0x00,0x00,0x00,0x00]
+          ldtilecfg [rip]
+
+// CHECK: ldtilecfg [2*rbp - 2048]
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0x04,0x6d,0x00,0xf8,0xff,0xff]
+          ldtilecfg [2*rbp - 2048]
+
+// CHECK: sttilecfg [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0xc4,0xa2,0x79,0x49,0x84,0xf5,0x00,0x00,0x00,0x10]
+          sttilecfg [rbp + 8*r14 + 268435456]
+
+// CHECK: sttilecfg [r8 + 4*rax + 291]
+// CHECK: encoding: [0xc4,0xc2,0x79,0x49,0x84,0x80,0x23,0x01,0x00,0x00]
+          sttilecfg [r8 + 4*rax + 291]
+
+// CHECK: sttilecfg [rip]
+// CHECK: encoding: [0xc4,0xe2,0x79,0x49,0x05,0x00,0x00,0x00,0x00]
+          sttilecfg [rip]
+
+// CHECK: sttilecfg [2*rbp - 2048]
+// CHECK: encoding: [0xc4,0xe2,0x79,0x49,0x04,0x6d,0x00,0xf8,0xff,0xff]
+          sttilecfg [2*rbp - 2048]
+
+// CHECK: tileloadd tmm6, [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0xc4,0xa2,0x7b,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          tileloadd tmm6, [rbp + 8*r14 + 268435456]
+
+// CHECK: tileloadd tmm3, [r8 + 4*rax + 291]
+// CHECK: encoding: [0xc4,0xc2,0x7b,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00]
+          tileloadd tmm3, [r8 + 4*rax + 291]
+
+// CHECK: tileloadd tmm3, [2*rbp - 32]
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff]
+          tileloadd tmm3, [2*rbp - 32]
+
+// CHECK: tileloaddt1 tmm6, [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0xc4,0xa2,0x79,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          tileloaddt1 tmm6, [rbp + 8*r14 + 268435456]
+
+// CHECK: tileloaddt1 tmm3, [r8 + 4*rax + 291]
+// CHECK: encoding: [0xc4,0xc2,0x79,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00]
+          tileloaddt1 tmm3, [r8 + 4*rax + 291]
+
+// CHECK: tileloaddt1 tmm3, [2*rbp - 32]
+// CHECK: encoding: [0xc4,0xe2,0x79,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff]
+          tileloaddt1 tmm3, [2*rbp - 32]
+
+// CHECK: tilerelease
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0xc0]
+          tilerelease
+
+// CHECK: tilestored [rbp + 8*r14 + 268435456], tmm6
+// CHECK: encoding: [0xc4,0xa2,0x7a,0x4b,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          tilestored [rbp + 8*r14 + 268435456], tmm6
+
+// CHECK: tilestored [r8 + 4*rax + 291], tmm3
+// CHECK: encoding: [0xc4,0xc2,0x7a,0x4b,0x9c,0x80,0x23,0x01,0x00,0x00]
+          tilestored [r8 + 4*rax + 291], tmm3
+
+// CHECK: tilestored [2*rbp - 32], tmm3
+// CHECK: encoding: [0xc4,0xe2,0x7a,0x4b,0x1c,0x6d,0xe0,0xff,0xff,0xff]
+          tilestored [2*rbp - 32], tmm3
+
+// CHECK: tilezero tmm6
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xf0]
+          tilezero tmm6
+
+// CHECK: tilezero tmm3
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xd8]
+          tilezero tmm3
diff --git a/llvm/test/MC/X86/align-branch-bundle.s b/llvm/test/MC/X86/align-branch-bundle.s
index 43dabca9e4779..a621599436838 100644
--- a/llvm/test/MC/X86/align-branch-bundle.s
+++ b/llvm/test/MC/X86/align-branch-bundle.s
@@ -6,7 +6,6 @@
 # CHECK-NEXT:       1:       testq    $2, %rdx
 # CHECK-NEXT:       8:       jne
 # CHECK-NEXT:       e:       nop
-# CHECK-NEXT:       f:       nop
 # CHECK-NEXT:      10:       jle
 
     .text
diff --git a/llvm/test/MC/X86/align-branch-pad-max-prefix.s b/llvm/test/MC/X86/align-branch-pad-max-prefix.s
index 6cce49d55d471..9576562667acb 100644
--- a/llvm/test/MC/X86/align-branch-pad-max-prefix.s
+++ b/llvm/test/MC/X86/align-branch-pad-max-prefix.s
@@ -10,9 +10,7 @@
   # following nops, doing so would make the jmp misaligned.
 # CHECK:      18:          jmp
   jmp bar
-# CHECK:      1d:          nop
-# CHECK:      1e:          nop
-# CHECK:      1f:          nop
+# CHECK:      1d:          nopl (%rax)
 # CHECK:      20:          int3
   .p2align 5
   int3
diff --git a/llvm/test/MachineVerifier/generic-vreg-undef-use.mir b/llvm/test/MachineVerifier/generic-vreg-undef-use.mir
new file mode 100644
index 0000000000000..d35ca6183188d
--- /dev/null
+++ b/llvm/test/MachineVerifier/generic-vreg-undef-use.mir
@@ -0,0 +1,25 @@
+# RUN: not --crash llc -march=aarch64 -o /dev/null -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: aarch64-registered-target
+
+# Undef uses are illegal for generic vregs.
+
+---
+name:            test_undef_use
+liveins:
+body:             |
+  bb.0:
+    %0:_(s32) = G_CONSTANT i32 0
+
+    ; Test generic instruction
+    ; CHECK: *** Bad machine code: Generic virtual register use cannot be undef ***
+    G_STORE %0, undef %1:_(p0) :: (store 4)
+
+    ; Make sure this fails on a post-isel generic instruction.
+    ; CHECK: *** Bad machine code: Generic virtual register use cannot be undef ***
+    $x0 = COPY undef %2:_(s64)
+
+    ; Make sure this fails with a target instruction
+    ; CHECK: *** Bad machine code: Generic virtual register use cannot be undef ***
+    RET_ReallyLR implicit $x0, implicit undef %3:_(s32)
+...
+
diff --git a/llvm/test/MachineVerifier/test_g_bitcast.mir b/llvm/test/MachineVerifier/test_g_bitcast.mir
index a399c859404fc..24ee95ba4b63e 100644
--- a/llvm/test/MachineVerifier/test_g_bitcast.mir
+++ b/llvm/test/MachineVerifier/test_g_bitcast.mir
@@ -34,4 +34,6 @@ body:             |
     %10:_(p1) = G_IMPLICIT_DEF
     %11:_(p3) = G_BITCAST %8
 
+    ; CHECK: Bad machine code: bitcast must change the type
+    %12:_(s64) = G_BITCAST %0
 ...
diff --git a/llvm/test/Object/invalid.test b/llvm/test/Object/invalid.test
index a7363abcace08..3a7ddab8d043a 100644
--- a/llvm/test/Object/invalid.test
+++ b/llvm/test/Object/invalid.test
@@ -115,9 +115,9 @@ Symbols:
 ## Check that llvm-readobj reports an error if .symtab has an invalid sh_entsize.
 
 # RUN: yaml2obj %s --docnum=6 -o %t6
-# RUN: not llvm-readobj --symbols %t6 2>&1 | FileCheck -DFILE=%t6 --check-prefix=INVALID-SYM-SIZE %s
+# RUN: llvm-readobj --symbols %t6 2>&1 | FileCheck -DFILE=%t6 --check-prefix=INVALID-SYM-SIZE %s
 
-# INVALID-SYM-SIZE: error: '[[FILE]]': section [index 1] has an invalid sh_entsize: 32
+# INVALID-SYM-SIZE: warning: '[[FILE]]': unable to read symbols from the SHT_SYMTAB section: section [index 1] has an invalid sh_entsize: 32
 
 --- !ELF
 FileHeader:
@@ -152,13 +152,13 @@ Sections:
 DynamicSymbols:
   - Name: foo
 
-## Check that llvm-readobj reports an error if .symtab has an invalid sh_link value,
+## Check that llvm-readobj reports a warning if .symtab has an invalid sh_link value,
 ## which is greater than number of sections.
 
 # RUN: yaml2obj %s --docnum=8 -o %t8
-# RUN: not llvm-readobj --symbols %t8 2>&1 | FileCheck -DFILE=%t8 --check-prefix=INVALID-SYMTAB-LINK %s
+# RUN: llvm-readobj --symbols %t8 2>&1 | FileCheck -DFILE=%t8 --check-prefix=INVALID-SYMTAB-LINK %s
 
-# INVALID-SYMTAB-LINK: error: '[[FILE]]': invalid section index: 255
+# INVALID-SYMTAB-LINK: warning: '[[FILE]]': unable to get the string table for the SHT_SYMTAB section: invalid section index: 255
 
 --- !ELF
 FileHeader:
@@ -187,13 +187,13 @@ FileHeader:
   Machine:   EM_X86_64
   SHEntSize: 1
 
-## Check that llvm-readobj reports an error if .symtab has sh_size
+## Check that llvm-readobj reports a warning if .symtab has sh_size
 ## that is not a multiple of sh_entsize.
 
 # RUN: yaml2obj %s --docnum=10 -o %t10
-# RUN: not llvm-readobj --symbols %t10 2>&1 | FileCheck -DFILE=%t10 --check-prefix=INVALID-SYMTAB-SIZE %s
+# RUN: llvm-readobj --symbols %t10 2>&1 | FileCheck -DFILE=%t10 --check-prefix=INVALID-SYMTAB-SIZE %s
 
-# INVALID-SYMTAB-SIZE: error: '[[FILE]]': section [index 1] has an invalid sh_size (1) which is not a multiple of its sh_entsize (24)
+# INVALID-SYMTAB-SIZE: warning: '[[FILE]]': unable to read symbols from the SHT_SYMTAB section: section [index 1] has an invalid sh_size (1) which is not a multiple of its sh_entsize (24)
 
 --- !ELF
 FileHeader:
@@ -276,13 +276,13 @@ Sections:
     Type:     SHT_RELA
     ShOffset: 0x10000
 
-## Check that llvm-objdump reports an error when we try to print symbols and
+## Check that llvm-objdump reports a warning when we try to print symbols and
 ## .shstrtab has a broken sh_offset so large that sh_offset + sh_size overflows the platform address size type.
 
 # RUN: yaml2obj %s --docnum=14 -o %t14
-# RUN: not llvm-readobj --symbols %t14 2>&1 | FileCheck -DFILE=%t14 --check-prefix=INVALID-SECTION-SIZE2 %s
+# RUN: llvm-readobj --symbols %t14 2>&1 | FileCheck -DFILE=%t14 --check-prefix=INVALID-SECTION-SIZE2 %s
 
-# INVALID-SECTION-SIZE2: error: '[[FILE]]': section [index 1] has a sh_offset (0xffffffff) + sh_size (0x1b) that cannot be represented
+# INVALID-SECTION-SIZE2: warning: '[[FILE]]': unable to get the name of the SHT_SYMTAB section: section [index 1] has a sh_offset (0xffffffff) + sh_size (0x1b) that cannot be represented
 
 --- !ELF
 FileHeader:
@@ -364,13 +364,13 @@ Sections:
     Type:     SHT_PROGBITS
     ShOffset: 0xFFFF0000
 
-## Check that llvm-readobj reports an error if symbol name
+## Check that llvm-readobj reports a warning if symbol name
 ## offset goes past the end of the symbol string table.
 
 # RUN: yaml2obj %s --docnum=18 -o %t18
-# RUN: not llvm-readobj --symbols %t18 2>&1 | FileCheck -DFILE=%t18 --check-prefix=INVALID-SYM-NAME %s
+# RUN: llvm-readobj --symbols %t18 2>&1 | FileCheck -DFILE=%t18 --check-prefix=INVALID-SYM-NAME %s
 
-# INVALID-SYM-NAME: error: '[[FILE]]': st_name (0x1) is past the end of the string table of size 0x1
+# INVALID-SYM-NAME: warning: '[[FILE]]': st_name (0x1) is past the end of the string table of size 0x1
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/Object/lit.local.cfg b/llvm/test/Object/lit.local.cfg
deleted file mode 100644
index ec8ad451d2da0..0000000000000
--- a/llvm/test/Object/lit.local.cfg
+++ /dev/null
@@ -1 +0,0 @@
-config.suffixes = ['.test', '.ll', '.s', '.yaml']
diff --git a/llvm/test/ObjectYAML/MachO/DWARF-debug_info.yaml b/llvm/test/ObjectYAML/MachO/DWARF-debug_info.yaml
index af997b010b2d2..98af99b86e0a5 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF-debug_info.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF-debug_info.yaml
@@ -1,4 +1,10 @@
-# RUN: yaml2obj %s | obj2yaml | FileCheck %s
+## This file contains test cases for generating a DAWRFv4 .debug_info
+## section in object files from the DWARF entry of Mach-O YAML inputs.
+
+## a) Test that yaml2obj emits a DWARF32 .debug_info sections and obj2yaml
+## converts it back.
+
+# RUN: yaml2obj --docnum=1 %s | obj2yaml | FileCheck %s --check-prefix=DWARF32
 
 --- !mach-o
 FileHeader:      
@@ -384,8 +390,7 @@ DWARF:
         - Address:         0x0000000100000F50
           Length:          52
   debug_info:      
-    - Length:          
-        TotalLength:     117
+    - Length:          117
       Version:         4
       AbbrOffset:      0
       AddrSize:        8
@@ -506,69 +511,166 @@ DWARF:
 ...
 ...
 
+#      DWARF32: DWARF:
+#      DWARF32:   debug_info:
+# DWARF32-NEXT:     - Length:     117
+# DWARF32-NEXT:       Version:    4
+# DWARF32-NEXT:       AbbrOffset: 0
+# DWARF32-NEXT:       AddrSize:   8
+# DWARF32-NEXT:       Entries:
+# DWARF32-NEXT:         - AbbrCode: 0x00000001
+# DWARF32-NEXT:           Values:
+# DWARF32-NEXT:             - Value: 0x0000000000000001
+# DWARF32-NEXT:             - Value: 0x000000000000000C
+# DWARF32-NEXT:             - Value: 0x0000000000000038
+# DWARF32-NEXT:             - Value: 0x0000000000000000
+# DWARF32-NEXT:             - Value: 0x0000000000000046
+# DWARF32-NEXT:             - Value: 0x0000000100000F50
+# DWARF32-NEXT:             - Value: 0x0000000000000034
+# DWARF32-NEXT:         - AbbrCode: 0x00000002
+# DWARF32-NEXT:           Values:
+# DWARF32-NEXT:             - Value: 0x0000000100000F50
+# DWARF32-NEXT:             - Value: 0x0000000000000034
+# DWARF32-NEXT:             - Value: 0x0000000000000001
+# DWARF32-NEXT:               BlockData: [ 0x56 ]
+# DWARF32-NEXT:             - Value: 0x0000000000000076
+# DWARF32-NEXT:             - Value: 0x0000000000000001
+# DWARF32-NEXT:             - Value: 0x0000000000000003
+# DWARF32-NEXT:             - Value: 0x0000000000000001
+# DWARF32-NEXT:             - Value: 0x0000000000000060
+# DWARF32-NEXT:             - Value: 0x0000000000000001
+# DWARF32-NEXT:         - AbbrCode:        0x00000003
+# DWARF32-NEXT:           Values:
+# DWARF32-NEXT:             - Value: 0x0000000000000002
+# DWARF32-NEXT:               BlockData: [ 0x91, 0x78 ]
+# DWARF32-NEXT:             - Value: 0x000000000000007B
+# DWARF32-NEXT:             - Value: 0x0000000000000001
+# DWARF32-NEXT:             - Value: 0x0000000000000003
+# DWARF32-NEXT:             - Value: 0x0000000000000060
+# DWARF32-NEXT:         - AbbrCode: 0x00000003
+# DWARF32-NEXT:           Values:
+# DWARF32-NEXT:             - Value: 0x0000000000000002
+# DWARF32-NEXT:               BlockData: [ 0x91, 0x70 ]
+# DWARF32-NEXT:             - Value: 0x0000000000000080
+# DWARF32-NEXT:             - Value: 0x0000000000000001
+# DWARF32-NEXT:             - Value: 0x0000000000000003
+# DWARF32-NEXT:             - Value: 0x0000000000000067
+# DWARF32-NEXT:         - AbbrCode: 0x00000000
+# DWARF32-NEXT:           Values:
+# DWARF32-NEXT:         - AbbrCode: 0x00000004
+# DWARF32-NEXT:           Values:
+# DWARF32-NEXT:             - Value: 0x0000000000000085
+# DWARF32-NEXT:             - Value: 0x0000000000000005
+# DWARF32-NEXT:             - Value: 0x0000000000000004
+# DWARF32-NEXT:         - AbbrCode: 0x00000005
+# DWARF32-NEXT:           Values:
+# DWARF32-NEXT:             - Value: 0x000000000000006C
+# DWARF32-NEXT:         - AbbrCode: 0x00000005
+# DWARF32-NEXT:           Values:
+# DWARF32-NEXT:             - Value: 0x0000000000000071
+# DWARF32-NEXT:         - AbbrCode: 0x00000004
+# DWARF32-NEXT:           Values:
+# DWARF32-NEXT:             - Value: 0x0000000000000089
+# DWARF32-NEXT:             - Value: 0x0000000000000006
+# DWARF32-NEXT:             - Value: 0x0000000000000001
+# DWARF32-NEXT:         - AbbrCode: 0x00000000
+# DWARF32-NEXT:           Values:
+
+## b) Test that yaml2obj emits a correct unit header and obj2yaml is able to convert it back.
+
+## Test DWARF32 unit header.
+
+# RUN: yaml2obj --docnum=2 -DFORMAT=DWARF32 %s | \
+# RUN:   obj2yaml | FileCheck %s --check-prefix=DWARF32-YAML
+
+#      DWARF32-YAML: debug_info:
+# DWARF32-YAML-NEXT:   - Length:     12
+# DWARF32-YAML-NEXT:     Version:    4
+# DWARF32-YAML-NEXT:     AbbrOffset: 0
+# DWARF32-YAML-NEXT:     AddrSize:   8
+# DWARF32-YAML-NEXT:     Entries:
+# DWARF32-YAML-NEXT:       - AbbrCode: 0x00000001
+# DWARF32-YAML-NEXT:         Values:
+# DWARF32-YAML-NEXT:           - Value: 0x0000000000001234
+
+--- !mach-o
+FileHeader:
+  magic:      0xFEEDFACF
+  cputype:    0x01000007
+  cpusubtype: 0x00000003
+  filetype:   0x0000000A
+  ncmds:      1
+  sizeofcmds: 232
+  flags:      0x00000000
+  reserved:   0x00000000
+LoadCommands:
+  - cmd:      LC_SEGMENT_64
+    cmdsize:  232
+    segname:  __DWARF
+    vmaddr:   0x00
+    vmsize:   0x00
+    fileoff:  0x00
+    filesize: 0x00
+    maxprot:  0
+    initprot: 0
+    nsects:   2
+    flags:    0
+    Sections:
+      - sectname:  __debug_abbrev
+        segname:   __DWARF
+        addr:      0x00
+        size:      12
+        offset:    528
+        align:     0
+        reloff:    0x00000000
+        nreloc:    0
+        flags:     0x00000000
+        reserved1: 0x00000000
+        reserved2: 0x00000000
+        reserved3: 0x00000000
+      - sectname:  __debug_info
+        segname:   __DWARF
+        addr:      0x00
+        size:      0xffff
+        offset:    1070
+        align:     0
+        reloff:    0x00000000
+        nreloc:    0
+        flags:     0x00000000
+        reserved1: 0x00000000
+        reserved2: 0x00000000
+        reserved3: 0x00000000
+DWARF:
+  debug_abbrev:    
+    - Code:     1
+      Tag:      DW_TAG_compile_unit
+      Children: DW_CHILDREN_no
+      Attributes:
+        - Attribute: DW_AT_low_pc
+          Form:      DW_FORM_addr
+  debug_info:
+    - Format:     [[FORMAT]]
+      Length:     12
+      Version:    4
+      AbbrOffset: 0
+      AddrSize:   8
+      Entries:
+        - AbbrCode: 1
+          Values:
+            - Value: 0x1234
+
+## Test DWARF64 unit header.
+
+# RUN: yaml2obj --docnum=2 -DFORMAT=DWARF64 %s | \
+# RUN:   obj2yaml | FileCheck %s --check-prefix=DWARF64-YAML
 
-#CHECK: DWARF:           
-#CHECK:   debug_info:      
-#CHECK:     - Length:          
-#CHECK:         TotalLength:     117
-#CHECK:       Version:         4
-#CHECK:       AbbrOffset:      0
-#CHECK:       AddrSize:        8
-#CHECK:       Entries:         
-#CHECK:         - AbbrCode:        0x00000001
-#CHECK:           Values:          
-#CHECK:             - Value:           0x0000000000000001
-#CHECK:             - Value:           0x000000000000000C
-#CHECK:             - Value:           0x0000000000000038
-#CHECK:             - Value:           0x0000000000000000
-#CHECK:             - Value:           0x0000000000000046
-#CHECK:             - Value:           0x0000000100000F50
-#CHECK:             - Value:           0x0000000000000034
-#CHECK:         - AbbrCode:        0x00000002
-#CHECK:           Values:          
-#CHECK:             - Value:           0x0000000100000F50
-#CHECK:             - Value:           0x0000000000000034
-#CHECK:             - Value:           0x0000000000000001
-#CHECK:               BlockData:       [ 0x56 ]
-#CHECK:             - Value:           0x0000000000000076
-#CHECK:             - Value:           0x0000000000000001
-#CHECK:             - Value:           0x0000000000000003
-#CHECK:             - Value:           0x0000000000000001
-#CHECK:             - Value:           0x0000000000000060
-#CHECK:             - Value:           0x0000000000000001
-#CHECK:         - AbbrCode:        0x00000003
-#CHECK:           Values:          
-#CHECK:             - Value:           0x0000000000000002
-#CHECK:               BlockData:       [ 0x91, 0x78 ]
-#CHECK:             - Value:           0x000000000000007B
-#CHECK:             - Value:           0x0000000000000001
-#CHECK:             - Value:           0x0000000000000003
-#CHECK:             - Value:           0x0000000000000060
-#CHECK:         - AbbrCode:        0x00000003
-#CHECK:           Values:          
-#CHECK:             - Value:           0x0000000000000002
-#CHECK:               BlockData:       [ 0x91, 0x70 ]
-#CHECK:             - Value:           0x0000000000000080
-#CHECK:             - Value:           0x0000000000000001
-#CHECK:             - Value:           0x0000000000000003
-#CHECK:             - Value:           0x0000000000000067
-#CHECK:         - AbbrCode:        0x00000000
-#CHECK:           Values:          
-#CHECK:         - AbbrCode:        0x00000004
-#CHECK:           Values:          
-#CHECK:             - Value:           0x0000000000000085
-#CHECK:             - Value:           0x0000000000000005
-#CHECK:             - Value:           0x0000000000000004
-#CHECK:         - AbbrCode:        0x00000005
-#CHECK:           Values:          
-#CHECK:             - Value:           0x000000000000006C
-#CHECK:         - AbbrCode:        0x00000005
-#CHECK:           Values:          
-#CHECK:             - Value:           0x0000000000000071
-#CHECK:         - AbbrCode:        0x00000004
-#CHECK:           Values:          
-#CHECK:             - Value:           0x0000000000000089
-#CHECK:             - Value:           0x0000000000000006
-#CHECK:             - Value:           0x0000000000000001
-#CHECK:         - AbbrCode:        0x00000000
-#CHECK:           Values:          
+#      DWARF64-YAML: debug_info:
+# DWARF64-YAML-NEXT:   - Format:     DWARF64
+# DWARF64-YAML-NEXT:     Length:     12
+# DWARF64-YAML-NEXT:     Version:    4
+# DWARF64-YAML-NEXT:     AbbrOffset: 0
+# DWARF64-YAML-NEXT:     AddrSize:   8
+# DWARF64-YAML-NEXT:     Entries:
+# DWARF64-YAML-NEXT:       - AbbrCode: 0x00000001
+# DWARF64-YAML-NEXT:         Values:
+# DWARF64-YAML-NEXT:           - Value: 0x0000000000001234
diff --git a/llvm/test/ObjectYAML/MachO/DWARF-debug_line.yaml b/llvm/test/ObjectYAML/MachO/DWARF-debug_line.yaml
index d24fe7c8a4b86..6575e95ed2126 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF-debug_line.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF-debug_line.yaml
@@ -423,8 +423,7 @@ DWARF:
       - DieOffset:       0x00000071
         Name:            char
   debug_info:      
-    - Length:          
-        TotalLength:     117
+    - Length:          117
       Version:         4
       AbbrOffset:      0
       AddrSize:        8
diff --git a/llvm/test/ObjectYAML/MachO/DWARF-debug_ranges.yaml b/llvm/test/ObjectYAML/MachO/DWARF-debug_ranges.yaml
index fe972a0fb40bd..df1ee738b15a9 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF-debug_ranges.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF-debug_ranges.yaml
@@ -213,8 +213,7 @@ DWARF:
         - LowOffset:       0x0000000000000000
           HighOffset:      0x0000000000001000
   debug_info:
-    - Length:
-        TotalLength:     82
+    - Length:          82
       Version:         4
       AbbrOffset:      0
       AddrSize:        8
diff --git a/llvm/test/ObjectYAML/MachO/DWARF2-AddrSize8-FormValues.yaml b/llvm/test/ObjectYAML/MachO/DWARF2-AddrSize8-FormValues.yaml
index 8e30d49532711..142ed0ac02eae 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF2-AddrSize8-FormValues.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF2-AddrSize8-FormValues.yaml
@@ -204,7 +204,7 @@ LoadCommands:
       - sectname:        __debug_abbrev
         segname:         __DWARF
         addr:            0x000000010000312C
-        size:            84
+        size:            86
         offset:          0x00002167
         align:           0
         reloff:          0x00000000
@@ -217,7 +217,7 @@ LoadCommands:
         segname:         __DWARF
         addr:            0x0000000100003178
         size:            83
-        offset:          0x000021BB
+        offset:          0x000021BC
         align:           0
         reloff:          0x00000000
         nreloc:          0
@@ -334,8 +334,7 @@ DWARF:
         - Attribute:       0x201A
           Form:            DW_FORM_addr
   debug_info:      
-    - Length:          
-        TotalLength:     168
+    - Length:          168
       Version:         2
       AbbrOffset:      0
       AddrSize:        8
@@ -416,8 +415,7 @@ DWARF:
 ...
 
 #CHECK:   debug_info:      
-#CHECK:     - Length:          
-#CHECK:         TotalLength:     168
+#CHECK:     - Length:          168
 #CHECK:       Version:         2
 #CHECK:       AbbrOffset:      0
 #CHECK:       AddrSize:        8
diff --git a/llvm/test/ObjectYAML/MachO/DWARF5-debug_info.yaml b/llvm/test/ObjectYAML/MachO/DWARF5-debug_info.yaml
index 9e8865f3c08ef..1ed156a63599b 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF5-debug_info.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF5-debug_info.yaml
@@ -1,4 +1,10 @@
-# RUN: yaml2obj %s | obj2yaml | FileCheck %s
+## This file contains test cases for generating a DWARFv5 .debug_info
+## section in object files from the DWARF entry of Mach-O YAML inputs.
+
+## a) Test that yaml2obj emits a DWARF32 .debug_info sections and obj2yaml
+## converts it back.
+
+# RUN: yaml2obj --docnum=1 %s | obj2yaml | FileCheck %s --check-prefix=DWARF32
 
 --- !mach-o
 FileHeader:      
@@ -384,8 +390,7 @@ DWARF:
         - Address:         0x0000000100000F50
           Length:          52
   debug_info:      
-    - Length:          
-        TotalLength:     118
+    - Length:          118
       Version:         5
       UnitType:        DW_UT_compile
       AbbrOffset:      0
@@ -506,70 +511,170 @@ DWARF:
           Data:            11
 ...
 
+#      DWARF32: DWARF:
+#      DWARF32:   debug_info:
+# DWARF32-NEXT:     - Length:     118
+# DWARF32-NEXT:       Version:    5
+# DWARF32-NEXT:       UnitType:   DW_UT_compile
+# DWARF32-NEXT:       AbbrOffset: 0
+# DWARF32-NEXT:       AddrSize:   8
+# DWARF32-NEXT:       Entries:
+# DWARF32-NEXT:         - AbbrCode: 0x00000001
+# DWARF32-NEXT:           Values:
+# DWARF32-NEXT:             - Value: 0x0000000000000001
+# DWARF32-NEXT:             - Value: 0x000000000000000C
+# DWARF32-NEXT:             - Value: 0x0000000000000038
+# DWARF32-NEXT:             - Value: 0x0000000000000000
+# DWARF32-NEXT:             - Value: 0x0000000000000046
+# DWARF32-NEXT:             - Value: 0x0000000100000F50
+# DWARF32-NEXT:             - Value: 0x0000000000000034
+# DWARF32-NEXT:         - AbbrCode: 0x00000002
+# DWARF32-NEXT:           Values:
+# DWARF32-NEXT:             - Value: 0x0000000100000F50
+# DWARF32-NEXT:             - Value: 0x0000000000000034
+# DWARF32-NEXT:             - Value: 0x0000000000000001
+# DWARF32-NEXT:               BlockData: [ 0x56 ]
+# DWARF32-NEXT:             - Value: 0x0000000000000076
+# DWARF32-NEXT:             - Value: 0x0000000000000001
+# DWARF32-NEXT:             - Value: 0x0000000000000003
+# DWARF32-NEXT:             - Value: 0x0000000000000001
+# DWARF32-NEXT:             - Value: 0x0000000000000060
+# DWARF32-NEXT:             - Value: 0x0000000000000001
+# DWARF32-NEXT:         - AbbrCode: 0x00000003
+# DWARF32-NEXT:           Values:
+# DWARF32-NEXT:             - Value: 0x0000000000000002
+# DWARF32-NEXT:               BlockData: [ 0x91, 0x78 ]
+# DWARF32-NEXT:             - Value: 0x000000000000007B
+# DWARF32-NEXT:             - Value: 0x0000000000000001
+# DWARF32-NEXT:             - Value: 0x0000000000000003
+# DWARF32-NEXT:             - Value: 0x0000000000000060
+# DWARF32-NEXT:         - AbbrCode:        0x00000003
+# DWARF32-NEXT:           Values:          
+# DWARF32-NEXT:             - Value:           0x0000000000000002
+# DWARF32-NEXT:               BlockData:       [ 0x91, 0x70 ]
+# DWARF32-NEXT:             - Value:           0x0000000000000080
+# DWARF32-NEXT:             - Value:           0x0000000000000001
+# DWARF32-NEXT:             - Value:           0x0000000000000003
+# DWARF32-NEXT:             - Value:           0x0000000000000067
+# DWARF32-NEXT:         - AbbrCode:        0x00000000
+# DWARF32-NEXT:           Values:          
+# DWARF32-NEXT:         - AbbrCode:        0x00000004
+# DWARF32-NEXT:           Values:          
+# DWARF32-NEXT:             - Value:           0x0000000000000085
+# DWARF32-NEXT:             - Value:           0x0000000000000005
+# DWARF32-NEXT:             - Value:           0x0000000000000004
+# DWARF32-NEXT:         - AbbrCode:        0x00000005
+# DWARF32-NEXT:           Values:          
+# DWARF32-NEXT:             - Value:           0x000000000000006C
+# DWARF32-NEXT:         - AbbrCode:        0x00000005
+# DWARF32-NEXT:           Values:          
+# DWARF32-NEXT:             - Value:           0x0000000000000071
+# DWARF32-NEXT:         - AbbrCode:        0x00000004
+# DWARF32-NEXT:           Values:          
+# DWARF32-NEXT:             - Value:           0x0000000000000089
+# DWARF32-NEXT:             - Value:           0x0000000000000006
+# DWARF32-NEXT:             - Value:           0x0000000000000001
+# DWARF32-NEXT:         - AbbrCode:        0x00000000
+# DWARF32-NEXT:           Values:          
+
+## b) Test that yaml2obj emits a correct unit header and obj2yaml is able to convert it back.
+
+## Test DWARF32 unit header.
+
+# RUN: yaml2obj --docnum=2 -DFORMAT=DWARF32 -DLENGTH=12 %s | \
+# RUN:   obj2yaml | FileCheck %s --check-prefix=DWARF32-YAML
+
+#      DWARF32-YAML: debug_info:
+# DWARF32-YAML-NEXT:   - Length:     12
+# DWARF32-YAML-NEXT:     Version:    5
+# DWARF32-YAML-NEXT:     UnitType:   DW_UT_compile
+# DWARF32-YAML-NEXT:     AbbrOffset: 0x0000000000000000
+# DWARF32-YAML-NEXT:     AddrSize:   8
+# DWARF32-YAML-NEXT:     Entries:
+# DWARF32-YAML-NEXT:       - AbbrCode: 0x00000001
+# DWARF32-YAML-NEXT:         Values:
+# DWARF32-YAML-NEXT:           - Value: 0x0000000000001234
+
+--- !mach-o
+FileHeader:
+  magic:      0xFEEDFACF
+  cputype:    0x01000007
+  cpusubtype: 0x00000003
+  filetype:   0x0000000A
+  ncmds:      1
+  sizeofcmds: 232
+  flags:      0x00000000
+  reserved:   0x00000000
+LoadCommands:
+  - cmd:      LC_SEGMENT_64
+    cmdsize:  232
+    segname:  __DWARF
+    vmaddr:   0x00
+    vmsize:   0x00
+    fileoff:  0x00
+    filesize: 0x00
+    maxprot:  0
+    initprot: 0
+    nsects:   2
+    flags:    0
+    Sections:
+      - sectname:  __debug_abbrev
+        segname:   __DWARF
+        addr:      0x00
+        size:      12
+        offset:    528
+        align:     0
+        reloff:    0x00000000
+        nreloc:    0
+        flags:     0x00000000
+        reserved1: 0x00000000
+        reserved2: 0x00000000
+        reserved3: 0x00000000
+      - sectname:  __debug_info
+        segname:   __DWARF
+        addr:      0x00
+        size:      0xffff
+        offset:    1070
+        align:     0
+        reloff:    0x00000000
+        nreloc:    0
+        flags:     0x00000000
+        reserved1: 0x00000000
+        reserved2: 0x00000000
+        reserved3: 0x00000000
+DWARF:
+  debug_abbrev:    
+    - Code:     1
+      Tag:      DW_TAG_compile_unit
+      Children: DW_CHILDREN_no
+      Attributes:
+        - Attribute: DW_AT_low_pc
+          Form:      DW_FORM_addr
+  debug_info:
+    - Format:     [[FORMAT]]
+      Length:     [[LENGTH]]
+      Version:    5
+      UnitType:   DW_UT_compile
+      AbbrOffset: 0
+      AddrSize:   8
+      Entries:
+        - AbbrCode: 1
+          Values:
+            - Value: 0x1234
+
+## Test DWARF64 unit header.
+
+# RUN: yaml2obj --docnum=2 -DFORMAT=DWARF64 -DLENGTH=20 %s | \
+# RUN:   obj2yaml | FileCheck %s --check-prefix=DWARF64-YAML
 
-#CHECK: DWARF:           
-#CHECK:   debug_info:      
-#CHECK:     - Length:          
-#CHECK:         TotalLength:     118
-#CHECK:       Version:         5
-#CHECK:       UnitType:        DW_UT_compile
-#CHECK:       AbbrOffset:      0
-#CHECK:       AddrSize:        8
-#CHECK:       Entries:         
-#CHECK:         - AbbrCode:        0x00000001
-#CHECK:           Values:          
-#CHECK:             - Value:           0x0000000000000001
-#CHECK:             - Value:           0x000000000000000C
-#CHECK:             - Value:           0x0000000000000038
-#CHECK:             - Value:           0x0000000000000000
-#CHECK:             - Value:           0x0000000000000046
-#CHECK:             - Value:           0x0000000100000F50
-#CHECK:             - Value:           0x0000000000000034
-#CHECK:         - AbbrCode:        0x00000002
-#CHECK:           Values:          
-#CHECK:             - Value:           0x0000000100000F50
-#CHECK:             - Value:           0x0000000000000034
-#CHECK:             - Value:           0x0000000000000001
-#CHECK:               BlockData:       [ 0x56 ]
-#CHECK:             - Value:           0x0000000000000076
-#CHECK:             - Value:           0x0000000000000001
-#CHECK:             - Value:           0x0000000000000003
-#CHECK:             - Value:           0x0000000000000001
-#CHECK:             - Value:           0x0000000000000060
-#CHECK:             - Value:           0x0000000000000001
-#CHECK:         - AbbrCode:        0x00000003
-#CHECK:           Values:          
-#CHECK:             - Value:           0x0000000000000002
-#CHECK:               BlockData:       [ 0x91, 0x78 ]
-#CHECK:             - Value:           0x000000000000007B
-#CHECK:             - Value:           0x0000000000000001
-#CHECK:             - Value:           0x0000000000000003
-#CHECK:             - Value:           0x0000000000000060
-#CHECK:         - AbbrCode:        0x00000003
-#CHECK:           Values:          
-#CHECK:             - Value:           0x0000000000000002
-#CHECK:               BlockData:       [ 0x91, 0x70 ]
-#CHECK:             - Value:           0x0000000000000080
-#CHECK:             - Value:           0x0000000000000001
-#CHECK:             - Value:           0x0000000000000003
-#CHECK:             - Value:           0x0000000000000067
-#CHECK:         - AbbrCode:        0x00000000
-#CHECK:           Values:          
-#CHECK:         - AbbrCode:        0x00000004
-#CHECK:           Values:          
-#CHECK:             - Value:           0x0000000000000085
-#CHECK:             - Value:           0x0000000000000005
-#CHECK:             - Value:           0x0000000000000004
-#CHECK:         - AbbrCode:        0x00000005
-#CHECK:           Values:          
-#CHECK:             - Value:           0x000000000000006C
-#CHECK:         - AbbrCode:        0x00000005
-#CHECK:           Values:          
-#CHECK:             - Value:           0x0000000000000071
-#CHECK:         - AbbrCode:        0x00000004
-#CHECK:           Values:          
-#CHECK:             - Value:           0x0000000000000089
-#CHECK:             - Value:           0x0000000000000006
-#CHECK:             - Value:           0x0000000000000001
-#CHECK:         - AbbrCode:        0x00000000
-#CHECK:           Values:          
+#      DWARF64-YAML: debug_info:
+# DWARF64-YAML-NEXT:   - Format:     DWARF64
+# DWARF64-YAML-NEXT:     Length:     20
+# DWARF64-YAML-NEXT:     Version:    5
+# DWARF64-YAML-NEXT:     UnitType:   DW_UT_compile
+# DWARF64-YAML-NEXT:     AbbrOffset: 0x0000000000000000
+# DWARF64-YAML-NEXT:     AddrSize:   8
+# DWARF64-YAML-NEXT:     Entries:
+# DWARF64-YAML-NEXT:       - AbbrCode: 0x00000001
+# DWARF64-YAML-NEXT:         Values:
+# DWARF64-YAML-NEXT:           - Value: 0x0000000000001234
diff --git a/llvm/test/ObjectYAML/lit.local.cfg b/llvm/test/ObjectYAML/lit.local.cfg
deleted file mode 100644
index 8169b9f95e118..0000000000000
--- a/llvm/test/ObjectYAML/lit.local.cfg
+++ /dev/null
@@ -1 +0,0 @@
-config.suffixes = ['.yaml']
diff --git a/llvm/test/Other/lint.ll b/llvm/test/Other/lint.ll
index db8d53a14f5a6..45c8bd55fa014 100644
--- a/llvm/test/Other/lint.ll
+++ b/llvm/test/Other/lint.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -lint -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -lint -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64"
 
 declare fastcc void @bar()
diff --git a/llvm/test/Other/loop-pass-printer.ll b/llvm/test/Other/loop-pass-printer.ll
index ba752264586aa..aab4dc91573a5 100644
--- a/llvm/test/Other/loop-pass-printer.ll
+++ b/llvm/test/Other/loop-pass-printer.ll
@@ -11,13 +11,13 @@
 ; RUN: 	   -loop-unroll -print-after=loop-unroll -filter-print-funcs=bar \
 ; RUN:	   | FileCheck %s -check-prefix=BAR -check-prefix=BAR-OLD
 ; RUN: opt < %s 2>&1 -disable-output \
-; RUN: 	   -passes='require<opt-remark-emit>,loop(unroll-full)' -print-after-all -filter-print-funcs=bar \
+; RUN: 	   -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -print-after-all -filter-print-funcs=bar \
 ; RUN:	   | FileCheck %s -check-prefix=BAR
 ; RUN: opt < %s 2>&1 -disable-output \
 ; RUN: 	   -loop-unroll -print-after=loop-unroll -filter-print-funcs=foo -print-module-scope \
 ; RUN:	   | FileCheck %s -check-prefix=FOO-MODULE -check-prefix=FOO-MODULE-OLD
 ; RUN: opt < %s 2>&1 -disable-output \
-; RUN: 	   -passes='require<opt-remark-emit>,loop(unroll-full)' -print-after-all -filter-print-funcs=foo -print-module-scope \
+; RUN: 	   -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -print-after-all -filter-print-funcs=foo -print-module-scope \
 ; RUN:	   | FileCheck %s -check-prefix=FOO-MODULE
 
 ; DEL:	    IR Dump Before {{Delete dead loops|LoopDeletionPass}}
diff --git a/llvm/test/Other/new-pm-cgprofile.ll b/llvm/test/Other/new-pm-cgprofile.ll
deleted file mode 100644
index c7fe31ab570f3..0000000000000
--- a/llvm/test/Other/new-pm-cgprofile.ll
+++ /dev/null
@@ -1,11 +0,0 @@
-; RUN: opt -debug-pass-manager -passes='default<O2>' %s 2>&1 |FileCheck %s --check-prefixes=DEFAULT
-; RUN: opt -debug-pass-manager -passes='default<O2>' -enable-npm-call-graph-profile=0 %s 2>&1 |FileCheck %s --check-prefixes=OFF
-; RUN: opt -debug-pass-manager -passes='default<O2>' -enable-npm-call-graph-profile=1 %s 2>&1 |FileCheck %s --check-prefixes=ON
-;
-; DEFAULT: Running pass: CGProfilePass
-; OFF-NOT: Running pass: CGProfilePass
-; ON: Running pass: CGProfilePass
-
-define void @foo() {
-  ret void
-}
diff --git a/llvm/test/Other/opt-O2-pipeline.ll b/llvm/test/Other/opt-O2-pipeline.ll
index ca72ec1f75678..56f85d0fb9a8c 100644
--- a/llvm/test/Other/opt-O2-pipeline.ll
+++ b/llvm/test/Other/opt-O2-pipeline.ll
@@ -280,6 +280,12 @@
 ; CHECK-NEXT:     Strip Unused Function Prototypes
 ; CHECK-NEXT:     Dead Global Elimination
 ; CHECK-NEXT:     Merge Duplicate Global Constants
+; CHECK-NEXT:     Call Graph Profile
+; CHECK-NEXT:       FunctionPass Manager
+; CHECK-NEXT:         Dominator Tree Construction
+; CHECK-NEXT:         Natural Loop Information
+; CHECK-NEXT:         Lazy Branch Probability Analysis
+; CHECK-NEXT:         Lazy Block Frequency Analysis
 ; CHECK-NEXT:     FunctionPass Manager
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Natural Loop Information
diff --git a/llvm/test/Other/opt-O3-pipeline.ll b/llvm/test/Other/opt-O3-pipeline.ll
index f629bfc3444b9..942f7d9dfead5 100644
--- a/llvm/test/Other/opt-O3-pipeline.ll
+++ b/llvm/test/Other/opt-O3-pipeline.ll
@@ -285,6 +285,12 @@
 ; CHECK-NEXT:     Strip Unused Function Prototypes
 ; CHECK-NEXT:     Dead Global Elimination
 ; CHECK-NEXT:     Merge Duplicate Global Constants
+; CHECK-NEXT:     Call Graph Profile
+; CHECK-NEXT:       FunctionPass Manager
+; CHECK-NEXT:         Dominator Tree Construction
+; CHECK-NEXT:         Natural Loop Information
+; CHECK-NEXT:         Lazy Branch Probability Analysis
+; CHECK-NEXT:         Lazy Block Frequency Analysis
 ; CHECK-NEXT:     FunctionPass Manager
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Natural Loop Information
diff --git a/llvm/test/Other/opt-Os-pipeline.ll b/llvm/test/Other/opt-Os-pipeline.ll
index dde9fbeb9950c..d975cc48b629c 100644
--- a/llvm/test/Other/opt-Os-pipeline.ll
+++ b/llvm/test/Other/opt-Os-pipeline.ll
@@ -266,6 +266,12 @@
 ; CHECK-NEXT:     Strip Unused Function Prototypes
 ; CHECK-NEXT:     Dead Global Elimination
 ; CHECK-NEXT:     Merge Duplicate Global Constants
+; CHECK-NEXT:     Call Graph Profile
+; CHECK-NEXT:       FunctionPass Manager
+; CHECK-NEXT:         Dominator Tree Construction
+; CHECK-NEXT:         Natural Loop Information
+; CHECK-NEXT:         Lazy Branch Probability Analysis
+; CHECK-NEXT:         Lazy Block Frequency Analysis
 ; CHECK-NEXT:     FunctionPass Manager
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Natural Loop Information
diff --git a/llvm/test/Other/opt-hot-cold-split.ll b/llvm/test/Other/opt-hot-cold-split.ll
index 971fe130b11c9..f43f3a3d893ce 100644
--- a/llvm/test/Other/opt-hot-cold-split.ll
+++ b/llvm/test/Other/opt-hot-cold-split.ll
@@ -1,8 +1,8 @@
 ; RUN: opt -mtriple=x86_64-- -Os -hot-cold-split=true -debug-pass=Structure < %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=DEFAULT-Os
-; RUN: opt -mtriple=x86_64-- -Os -hot-cold-split=true -passes='lto-pre-link<Os>' -debug-pass-manager < %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=LTO-PRELINK-Os
-; RUN: opt -mtriple=x86_64-- -Os -hot-cold-split=true -passes='thinlto-pre-link<Os>' -debug-pass-manager < %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=THINLTO-PRELINK-Os
-; RUN: opt -mtriple=x86_64-- -Os -hot-cold-split=true -passes='lto<Os>' -debug-pass-manager < %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=LTO-POSTLINK-Os
-; RUN: opt -mtriple=x86_64-- -Os -hot-cold-split=true -passes='thinlto<Os>' -debug-pass-manager < %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=THINLTO-POSTLINK-Os
+; RUN: opt -mtriple=x86_64-- -hot-cold-split=true -passes='lto-pre-link<Os>' -debug-pass-manager < %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=LTO-PRELINK-Os
+; RUN: opt -mtriple=x86_64-- -hot-cold-split=true -passes='thinlto-pre-link<Os>' -debug-pass-manager < %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=THINLTO-PRELINK-Os
+; RUN: opt -mtriple=x86_64-- -hot-cold-split=true -passes='lto<Os>' -debug-pass-manager < %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=LTO-POSTLINK-Os
+; RUN: opt -mtriple=x86_64-- -hot-cold-split=true -passes='thinlto<Os>' -debug-pass-manager < %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=THINLTO-POSTLINK-Os
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Other/opt-npm.ll b/llvm/test/Other/opt-npm.ll
new file mode 100644
index 0000000000000..0b1e651b41f73
--- /dev/null
+++ b/llvm/test/Other/opt-npm.ll
@@ -0,0 +1,6 @@
+; RUN: opt -dce -enable-new-pm -disable-output -debug-pass-manager %s 2>&1 | FileCheck %s
+
+; CHECK: DCEPass
+define void @foo() {
+    ret void
+}
diff --git a/llvm/test/Other/opt-old-new-pm-passes.ll b/llvm/test/Other/opt-old-new-pm-passes.ll
new file mode 100644
index 0000000000000..09bc7c4016048
--- /dev/null
+++ b/llvm/test/Other/opt-old-new-pm-passes.ll
@@ -0,0 +1,2 @@
+; RUN: not opt -dce --passes=inline %s 2>&1 | FileCheck %s
+; CHECK: Cannot specify passes via both -foo-pass and --passes=foo-pass
diff --git a/llvm/test/Reduce/Inputs/remove-multiple-use-of-args-in-same-instruction.py b/llvm/test/Reduce/Inputs/remove-multiple-use-of-args-in-same-instruction.py
new file mode 100644
index 0000000000000..93fa5c0bc29e7
--- /dev/null
+++ b/llvm/test/Reduce/Inputs/remove-multiple-use-of-args-in-same-instruction.py
@@ -0,0 +1,13 @@
+import sys
+
+FunctionCallPresent = False
+
+input = open(sys.argv[1], "r")
+for line in input:
+  if "call void @use" in line:
+    FunctionCallPresent = True
+
+if FunctionCallPresent:
+  sys.exit(0) # Interesting!
+
+sys.exit(1)
diff --git a/llvm/test/Reduce/Inputs/remove-multiple-use-of-global-vars-in-same-instruction.py b/llvm/test/Reduce/Inputs/remove-multiple-use-of-global-vars-in-same-instruction.py
new file mode 100644
index 0000000000000..93fa5c0bc29e7
--- /dev/null
+++ b/llvm/test/Reduce/Inputs/remove-multiple-use-of-global-vars-in-same-instruction.py
@@ -0,0 +1,13 @@
+import sys
+
+FunctionCallPresent = False
+
+input = open(sys.argv[1], "r")
+for line in input:
+  if "call void @use" in line:
+    FunctionCallPresent = True
+
+if FunctionCallPresent:
+  sys.exit(0) # Interesting!
+
+sys.exit(1)
diff --git a/llvm/test/Reduce/remove-args.ll b/llvm/test/Reduce/remove-args.ll
index 8d6130262bf50..161a6fd3731bb 100644
--- a/llvm/test/Reduce/remove-args.ll
+++ b/llvm/test/Reduce/remove-args.ll
@@ -1,6 +1,5 @@
 ; Test that llvm-reduce can remove uninteresting function arguments from function definitions as well as their calls.
 ;
-; RUN: rm -rf %t
 ; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-args.py %s -o %t
 ; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
 
diff --git a/llvm/test/Reduce/remove-attributes-from-intrinsic-like-functions.ll b/llvm/test/Reduce/remove-attributes-from-intrinsic-like-functions.ll
new file mode 100644
index 0000000000000..60df12e94feb2
--- /dev/null
+++ b/llvm/test/Reduce/remove-attributes-from-intrinsic-like-functions.ll
@@ -0,0 +1,40 @@
+; Just because a function is named like an intrinsic does not mean we should skip it's attributes.
+;
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
+
+; CHECK-ALL: declare i32 @llvm.not.really.an.intrinsic(i32, i32) #0
+declare i32 @llvm.not.really.an.intrinsic(i32, i32) #0
+
+define i32 @t(i32 %a) {
+; CHECK-ALL-LABEL: @t(
+
+; CHECK-INTERESTINGNESS: %r =
+; CHECK-INTERESTINGNESS-SAME: call
+; CHECK-INTERESTINGNESS-SAME: "arg0"
+; CHECK-INTERESTINGNESS-SAME: i32 @llvm.not.really.an.intrinsic(i32
+; CHECK-INTERESTINGNESS-SAME: "arg3"
+; CHECK-INTERESTINGNESS-SAME: %a
+; CHECK-INTERESTINGNESS-SAME: i32
+; CHECK-INTERESTINGNESS-SAME: %a
+; CHECK-INTERESTINGNESS-SAME: #1
+
+; CHECK-FINAL: %r = call "arg0" i32 @llvm.not.really.an.intrinsic(i32 "arg3" %a, i32 %a) #1
+; CHECK-ALL: ret i32 %r
+
+  %r = call "arg0" "arg1" i32 @llvm.not.really.an.intrinsic(i32 "arg2" "arg3" %a, i32 %a) "arg4" "arg5"
+  ret i32 %r
+}
+
+; CHECK-INTERESTINGNESS: attributes #0 = {
+; CHECK-INTERESTINGNESS-SAME: "arg6"
+
+; CHECK-INTERESTINGNESS: attributes #1 = {
+; CHECK-INTERESTINGNESS-SAME: "arg4"
+
+; CHECK-FINAL: attributes #0 = { "arg6" }
+; CHECK-FINAL: attributes #1 = { "arg4" }
+
+; CHECK-ALL-NOT: attributes #
+
+attributes #0 = { "arg6" "arg7" }
diff --git a/llvm/test/Reduce/remove-attributes-from-intrinsics.ll b/llvm/test/Reduce/remove-attributes-from-intrinsics.ll
new file mode 100644
index 0000000000000..7a8a8f0eb1146
--- /dev/null
+++ b/llvm/test/Reduce/remove-attributes-from-intrinsics.ll
@@ -0,0 +1,38 @@
+; We can't actually put attributes on intrinsic declarations, only on call sites.
+;
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
+
+define i32 @t(i32 %a) {
+; CHECK-ALL-LABEL: @t(
+
+; CHECK-INTERESTINGNESS: %r =
+; CHECK-INTERESTINGNESS-SAME: call
+; CHECK-INTERESTINGNESS-SAME: "arg0"
+; CHECK-INTERESTINGNESS-SAME: i32 @llvm.uadd.sat.i32(i32
+; CHECK-INTERESTINGNESS-SAME: "arg3"
+; CHECK-INTERESTINGNESS-SAME: %a
+; CHECK-INTERESTINGNESS-SAME: i32
+; CHECK-INTERESTINGNESS-SAME: %a
+; CHECK-INTERESTINGNESS-SAME: #1
+
+; CHECK-FINAL: %r = call "arg0" i32 @llvm.uadd.sat.i32(i32 "arg3" %a, i32 %a) #1
+; CHECK-ALL: ret i32 %r
+
+  %r = call "arg0" "arg1" i32 @llvm.uadd.sat.i32(i32 "arg2" "arg3" %a, i32 %a) "arg4" "arg5"
+  ret i32 %r
+}
+
+; CHECK-ALL: declare i32 @llvm.uadd.sat.i32(i32, i32) #0
+declare i32 @llvm.uadd.sat.i32(i32, i32) #0
+
+; CHECK-ALL: attributes #0 = { nounwind readnone speculatable willreturn }
+
+; CHECK-INTERESTINGNESS: attributes #1 = {
+; CHECK-INTERESTINGNESS-SAME: "arg4"
+
+; CHECK-FINAL: attributes #1 = { "arg4" }
+
+; CHECK-ALL-NOT: attributes #
+
+attributes #0 = { "arg6" "arg7" }
diff --git a/llvm/test/Reduce/remove-call-site-attributes.ll b/llvm/test/Reduce/remove-call-site-attributes.ll
new file mode 100644
index 0000000000000..e8f50355812ab
--- /dev/null
+++ b/llvm/test/Reduce/remove-call-site-attributes.ll
@@ -0,0 +1,38 @@
+; Test that llvm-reduce can remove uninteresting operand bundles from calls.
+;
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
+
+; CHECK-ALL: declare i32 @f1(i32, i32)
+declare i32 @f1(i32, i32)
+
+; CHECK-FINAL-LABEL: define i32 @interesting(i32 %arg0, i32 %arg1) {
+; CHECK-FINAL-NEXT:  entry:
+; CHECK-FINAL-NEXT:    %r = call "attr0" i32 @f1(i32 "attr4" %arg0, i32 %arg1) #0
+; CHECK-FINAL-NEXT:    ret i32 %r
+; CHECK-FINAL-NEXT:  }
+define i32 @interesting(i32 %arg0, i32 %arg1) {
+entry:
+; CHECK-INTERESTINGNESS-LABEL: @interesting(
+
+; CHECK-INTERESTINGNESS: %r = call
+; CHECK-INTERESTINGNESS-SAME: "attr0"
+; CHECK-INTERESTINGNESS-SAME: i32 @f1(
+; CHECK-INTERESTINGNESS-SAME: i32
+; CHECK-INTERESTINGNESS-SAME: "attr4"
+; CHECK-INTERESTINGNESS-SAME: %arg0
+; CHECK-INTERESTINGNESS-SAME: i32
+; CHECK-INTERESTINGNESS-SAME: %arg1
+; CHECK-INTERESTINGNESS-SAME: #0
+; CHECK-INTERESTINGNESS: ret i32 %r
+
+  %r = call "attr0" "attr1" "attr2" i32 @f1(i32 "attr3" "attr4" "attr5" %arg0, i32 "attr6" "attr7" "attr8" %arg1) #0
+  ret i32 %r
+}
+
+; CHECK-INTERESTINGNESS: attributes #0 = {
+; CHECK-INTERESTINGNESS-SAME: "attr10"
+
+; CHECK-FINAL:  attributes #0 = { "attr10" }
+
+attributes #0 = { "attr9" "attr10" "attr11" }
diff --git a/llvm/test/Reduce/remove-funcs.ll b/llvm/test/Reduce/remove-funcs.ll
index 8e9b2579a974b..59ffd849193db 100644
--- a/llvm/test/Reduce/remove-funcs.ll
+++ b/llvm/test/Reduce/remove-funcs.ll
@@ -1,7 +1,6 @@
 ; Test that llvm-reduce can remove uninteresting functions as well as
 ; their InstCalls.
 ;
-; RUN: rm -rf %t
 ; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-funcs.py %s -o %t
 ; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
 
diff --git a/llvm/test/Reduce/remove-function-attributes.ll b/llvm/test/Reduce/remove-function-attributes.ll
new file mode 100644
index 0000000000000..52bbda36f332a
--- /dev/null
+++ b/llvm/test/Reduce/remove-function-attributes.ll
@@ -0,0 +1,23 @@
+; Test that llvm-reduce can remove uninteresting attributes.
+;
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
+
+; CHECK-INTERESTINGNESS: declare
+; CHECK-INTERESTINGNESS-SAME: "attr0"
+; CHECK-INTERESTINGNESS-SAME: void @f0
+; CHECK-INTERESTINGNESS-SAME: i32
+; CHECK-INTERESTINGNESS-SAME: i32
+; CHECK-INTERESTINGNESS-SAME: "attr6"
+; CHECK-INTERESTINGNESS-SAME: #0
+
+; CHECK-FINAL: declare "attr0" void @f0(i32, i32 "attr6") #0
+
+declare "attr0" "attr1" "attr2" void @f0(i32 "attr3" "attr4" "attr5", i32 "attr6" "attr7" "attr8") #0
+
+; CHECK-INTERESTINGNESS: attributes #0 = {
+; CHECK-INTERESTINGNESS-SAME: "attr10"
+
+; CHECK-FINAL:  attributes #0 = { "attr10" }
+
+attributes #0 = { "attr9" "attr10" "attr11" }
diff --git a/llvm/test/Reduce/remove-global-variable-attributes.ll b/llvm/test/Reduce/remove-global-variable-attributes.ll
new file mode 100644
index 0000000000000..bec3afd960e90
--- /dev/null
+++ b/llvm/test/Reduce/remove-global-variable-attributes.ll
@@ -0,0 +1,27 @@
+; Test that llvm-reduce can remove uninteresting attributes.
+;
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
+
+; CHECK-ALL: @gv0 = global i32 0 #0
+; CHECK-ALL-NEXT: @gv1 = global i32 0 #1
+; CHECK-ALL-NEXT: @gv2 = global i32 0
+@gv0 = global i32 0 #0
+@gv1 = global i32 0 #1
+@gv2 = global i32 0 #2
+
+; CHECK-INTERESTINGNESS: attributes #0 = {
+; CHECK-INTERESTINGNESS-SAME: "attr0"
+; CHECK-INTERESTINGNESS-SAME: "attr2"
+
+; CHECK-INTERESTINGNESS-NEXT: attributes #1 = {
+; CHECK-INTERESTINGNESS-SAME: "attr4"
+
+; CHECK-FINAL:  attributes #0 = { "attr0" "attr2" }
+; CHECK-FINAL-NEXT:  attributes #1 = { "attr4" }
+
+; CHECK-FINAL-NOT:  attributes #2
+
+attributes #0 = { "attr0" "attr1" "attr2"}
+attributes #1 = { "attr3" "attr4" "attr5"}
+attributes #2 = { "attr6" "attr7" "attr8"}
diff --git a/llvm/test/Reduce/remove-global-vars.ll b/llvm/test/Reduce/remove-global-vars.ll
index 921083fc93b0f..4fca4a1e6973f 100644
--- a/llvm/test/Reduce/remove-global-vars.ll
+++ b/llvm/test/Reduce/remove-global-vars.ll
@@ -1,7 +1,6 @@
 ; Test that llvm-reduce can remove uninteresting Global Variables as well as
 ; their direct uses (which in turn are replaced with 'undef').
 ;
-; RUN: rm -rf %t
 ; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-global-vars.py %s -o %t
 ; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
 
diff --git a/llvm/test/Reduce/remove-metadata.ll b/llvm/test/Reduce/remove-metadata.ll
index da7d5a2f16b48..51a50ca20a985 100644
--- a/llvm/test/Reduce/remove-metadata.ll
+++ b/llvm/test/Reduce/remove-metadata.ll
@@ -1,7 +1,6 @@
 ; Test that llvm-reduce can remove uninteresting metadata from an IR file.
 ; The Metadata pass erases named & unnamed metadata nodes.
 ;
-; RUN: rm -rf %t
 ; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-metadata.py %s -o %t
 ; RUN: cat %t | FileCheck -implicit-check-not=! %s
 
diff --git a/llvm/test/Reduce/remove-multiple-use-of-args-in-same-instruction.ll b/llvm/test/Reduce/remove-multiple-use-of-args-in-same-instruction.ll
new file mode 100644
index 0000000000000..21a638f1e6bce
--- /dev/null
+++ b/llvm/test/Reduce/remove-multiple-use-of-args-in-same-instruction.ll
@@ -0,0 +1,16 @@
+; Test that llvm-reduce can remove uninteresting function arguments from function definitions as well as their calls.
+;
+; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-multiple-use-of-args-in-same-instruction.py %s -o %t
+; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
+
+declare void @use(i32, i32, i32)
+
+; CHECK-LABEL: @interesting(i32 %uninteresting1, i32 %uninteresting2, i32 %uninteresting3
+define void @interesting(i32 %uninteresting1, i32 %uninteresting2, i32 %uninteresting3) {
+entry:
+  ; CHECK: call void @use(i32 %uninteresting1, i32 %uninteresting2, i32 %uninteresting3)
+  call void @use(i32 %uninteresting1, i32 %uninteresting2, i32 %uninteresting3)
+  call void @use(i32 %uninteresting1, i32 %uninteresting2, i32 %uninteresting3)
+  call void @use(i32 %uninteresting1, i32 %uninteresting2, i32 %uninteresting3)
+  ret void
+}
diff --git a/llvm/test/Reduce/remove-multiple-use-of-global-vars-in-same-instruction.ll b/llvm/test/Reduce/remove-multiple-use-of-global-vars-in-same-instruction.ll
new file mode 100644
index 0000000000000..4400bc818e554
--- /dev/null
+++ b/llvm/test/Reduce/remove-multiple-use-of-global-vars-in-same-instruction.ll
@@ -0,0 +1,23 @@
+; Test that llvm-reduce can remove uninteresting function arguments from function definitions as well as their calls.
+;
+; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-multiple-use-of-global-vars-in-same-instruction.py %s -o %t
+; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
+
+; CHECK: @uninteresting1 = global
+; CHECK: @uninteresting2 = global
+; CHECK: @uninteresting3 = global
+@uninteresting1 = global i32 0, align 4
+@uninteresting2 = global i32 0, align 4
+@uninteresting3 = global i32 0, align 4
+
+declare void @use(i32*, i32*, i32*)
+
+; CHECK-LABEL: @interesting()
+define void @interesting() {
+entry:
+  ; CHECK: call void @use(i32* @uninteresting1, i32* @uninteresting2, i32* @uninteresting3)
+  call void @use(i32* @uninteresting1, i32* @uninteresting2, i32* @uninteresting3)
+  call void @use(i32* @uninteresting1, i32* @uninteresting2, i32* @uninteresting3)
+  call void @use(i32* @uninteresting1, i32* @uninteresting2, i32* @uninteresting3)
+  ret void
+}
diff --git a/llvm/test/Reduce/remove-operand-bundles.ll b/llvm/test/Reduce/remove-operand-bundles.ll
new file mode 100644
index 0000000000000..b0f3af6dbc85a
--- /dev/null
+++ b/llvm/test/Reduce/remove-operand-bundles.ll
@@ -0,0 +1,40 @@
+; Test that llvm-reduce can remove uninteresting operand bundles from calls.
+;
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
+
+; CHECK-ALL: declare void @f1()
+; CHECK-ALL: declare void @f2()
+; CHECK-ALL: declare void @f3()
+declare void @f1()
+declare void @f2()
+declare void @f3()
+
+; CHECK-FINAL-LABEL: define void @interesting(i32 %arg0, i32 %arg1, i32 %arg2) {
+; CHECK-FINAL-NEXT:  entry:
+; CHECK-FINAL-NEXT:    call void @f1() [ "bundle0"(), "align"(i32 %arg0), "whatever0"() ]
+; CHECK-FINAL-NEXT:    call void @f2()
+; CHECK-FINAL-NEXT:    call void @f3() [ "align"(i32 %arg2) ]
+; CHECK-FINAL-NEXT:    ret void
+; CHECK-FINAL-NEXT:  }
+define void @interesting(i32 %arg0, i32 %arg1, i32 %arg2) {
+entry:
+; CHECK-INTERESTINGNESS-LABEL: @interesting(
+
+; CHECK-INTERESTINGNESS: call void @f1()
+; CHECK-INTERESTINGNESS: "bundle0"()
+; CHECK-INTERESTINGNESS: "align"(i32 %arg0)
+; CHECK-INTERESTINGNESS: "whatever0"()
+
+; CHECK-INTERESTINGNESS: call void @f2()
+
+; CHECK-INTERESTINGNESS: call void @f3()
+; CHECK-INTERESTINGNESS: "align"(i32 %arg2)
+
+; CHECK-INTERESTINGNESS: ret
+
+  call void @f1() [ "bundle0"(),        "align"(i32 %arg0), "whatever0"() ]
+  call void @f2() [ "align"(i32 %arg1), "whatever1"(),      "bundle1"() ]
+  call void @f3() [ "whatever2"(),      "bundle2"(),        "align"(i32 %arg2) ]
+  ret void
+}
diff --git a/llvm/test/TableGen/directive1.td b/llvm/test/TableGen/directive1.td
new file mode 100644
index 0000000000000..b293196d4d556
--- /dev/null
+++ b/llvm/test/TableGen/directive1.td
@@ -0,0 +1,183 @@
+// RUN: llvm-tblgen -gen-directive-decl -I %p/../../include %s | FileCheck -match-full-lines %s
+// RUN: llvm-tblgen -gen-directive-impl -I %p/../../include %s | FileCheck -match-full-lines %s -check-prefix=IMPL
+// RUN: llvm-tblgen -gen-directive-gen -I %p/../../include %s | FileCheck -match-full-lines %s -check-prefix=GEN
+
+include "llvm/Frontend/Directive/DirectiveBase.td"
+
+def TestDirectiveLanguage : DirectiveLanguage {
+  let name = "Tdl";
+
+  let cppNamespace = "tdl";
+  let directivePrefix = "TDLD_";
+  let clausePrefix = "TDLC_";
+  let makeEnumAvailableInNamespace = 1;
+  let enableBitmaskEnumInNamespace = 1;
+}
+
+def TDLC_ClauseA : Clause<"clausea"> {}
+def TDLC_ClauseB : Clause<"clauseb"> {
+  let isDefault = 1;
+}
+
+def TDL_DirA : Directive<"dira"> {
+  let allowedClauses = [
+    VersionedClause<TDLC_ClauseA>,
+    VersionedClause<TDLC_ClauseB>
+  ];
+  let isDefault = 1;
+}
+
+// CHECK:       #ifndef LLVM_Tdl_INC
+// CHECK-NEXT:  #define LLVM_Tdl_INC
+// CHECK-EMPTY:
+// CHECK-NEXT:  #include "llvm/ADT/BitmaskEnum.h"
+// CHECK-EMPTY:
+// CHECK-NEXT:  namespace llvm {
+// CHECK-NEXT:  class StringRef;
+// CHECK-NEXT:  namespace tdl {
+// CHECK-EMPTY:
+// CHECK-NEXT:  LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
+// CHECK-EMPTY:
+// CHECK-NEXT:  enum class Directive {
+// CHECK-NEXT:    TDLD_dira,
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  static constexpr std::size_t Directive_enumSize = 1;
+// CHECK-EMPTY:
+// CHECK-NEXT:  constexpr auto TDLD_dira = llvm::tdl::Directive::TDLD_dira;
+// CHECK-EMPTY:
+// CHECK-NEXT:  enum class Clause {
+// CHECK-NEXT:    TDLC_clausea,
+// CHECK-NEXT:    TDLC_clauseb,
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  static constexpr std::size_t Clause_enumSize = 2;
+// CHECK-EMPTY:
+// CHECK-NEXT:  constexpr auto TDLC_clausea = llvm::tdl::Clause::TDLC_clausea;
+// CHECK-NEXT:  constexpr auto TDLC_clauseb = llvm::tdl::Clause::TDLC_clauseb;
+// CHECK-EMPTY:
+// CHECK-NEXT:  // Enumeration helper functions
+// CHECK-NEXT:  Directive getTdlDirectiveKind(llvm::StringRef Str);
+// CHECK-EMPTY:
+// CHECK-NEXT:  llvm::StringRef getTdlDirectiveName(Directive D);
+// CHECK-EMPTY:
+// CHECK-NEXT:  Clause getTdlClauseKind(llvm::StringRef Str);
+// CHECK-EMPTY:
+// CHECK-NEXT:  llvm::StringRef getTdlClauseName(Clause C);
+// CHECK-EMPTY:
+// CHECK-NEXT:  /// Return true if \p C is a valid clause for \p D in version \p Version.
+// CHECK-NEXT:  bool isAllowedClauseForDirective(Directive D, Clause C, unsigned Version);
+// CHECK-EMPTY:
+// CHECK-NEXT:  } // namespace tdl
+// CHECK-NEXT:  } // namespace llvm
+// CHECK-NEXT:  #endif // LLVM_Tdl_INC
+
+
+// IMPL:       #include "llvm/ADT/StringRef.h"
+// IMPL-NEXT:  #include "llvm/ADT/StringSwitch.h"
+// IMPL-EMPTY:
+// IMPL-NEXT:  using namespace llvm;
+// IMPL-NEXT:  using namespace tdl;
+// IMPL-EMPTY:
+// IMPL-NEXT:  Directive llvm::tdl::getTdlDirectiveKind(llvm::StringRef Str) {
+// IMPL-NEXT:    return llvm::StringSwitch<Directive>(Str)
+// IMPL-NEXT:      .Case("dira",TDLD_dira)
+// IMPL-NEXT:      .Default(TDLD_dira);
+// IMPL-NEXT:  }
+// IMPL-EMPTY:
+// IMPL-NEXT:  llvm::StringRef llvm::tdl::getTdlDirectiveName(Directive Kind) {
+// IMPL-NEXT:    switch (Kind) {
+// IMPL-NEXT:      case TDLD_dira:
+// IMPL-NEXT:        return "dira";
+// IMPL-NEXT:    }
+// IMPL-NEXT:    llvm_unreachable("Invalid Tdl Directive kind");
+// IMPL-NEXT:  }
+// IMPL-EMPTY:
+// IMPL-NEXT:  Clause llvm::tdl::getTdlClauseKind(llvm::StringRef Str) {
+// IMPL-NEXT:    return llvm::StringSwitch<Clause>(Str)
+// IMPL-NEXT:      .Case("clausea",TDLC_clausea)
+// IMPL-NEXT:      .Case("clauseb",TDLC_clauseb)
+// IMPL-NEXT:      .Default(TDLC_clauseb);
+// IMPL-NEXT:  }
+// IMPL-EMPTY:
+// IMPL-NEXT:  llvm::StringRef llvm::tdl::getTdlClauseName(Clause Kind) {
+// IMPL-NEXT:    switch (Kind) {
+// IMPL-NEXT:      case TDLC_clausea:
+// IMPL-NEXT:        return "clausea";
+// IMPL-NEXT:      case TDLC_clauseb:
+// IMPL-NEXT:        return "clauseb";
+// IMPL-NEXT:    }
+// IMPL-NEXT:    llvm_unreachable("Invalid Tdl Clause kind");
+// IMPL-NEXT:  }
+// IMPL-EMPTY:
+// IMPL-NEXT:  bool llvm::tdl::isAllowedClauseForDirective(Directive D, Clause C, unsigned Version) {
+// IMPL-NEXT:    assert(unsigned(D) <= llvm::tdl::Directive_enumSize);
+// IMPL-NEXT:    assert(unsigned(C) <= llvm::tdl::Clause_enumSize);
+// IMPL-NEXT:    switch (D) {
+// IMPL-NEXT:      case TDLD_dira:
+// IMPL-NEXT:        switch (C) {
+// IMPL-NEXT:          case TDLC_clausea:
+// IMPL-NEXT:            return 1 <= Version && 2147483647 >= Version;
+// IMPL-NEXT:          case TDLC_clauseb:
+// IMPL-NEXT:            return 1 <= Version && 2147483647 >= Version;
+// IMPL-NEXT:          default:
+// IMPL-NEXT:            return false;
+// IMPL-NEXT:        }
+// IMPL-NEXT:        break;
+// IMPL-NEXT:    }
+// IMPL-NEXT:    llvm_unreachable("Invalid Tdl Directive kind");
+// IMPL-NEXT:  }
+// IMPL-EMPTY:
+
+
+
+// GEN:       #ifdef GEN_FLANG_DIRECTIVE_CLAUSE_SETS
+// GEN-NEXT:  #undef GEN_FLANG_DIRECTIVE_CLAUSE_SETS
+// GEN-EMPTY:
+// GEN-NEXT:  namespace llvm {
+// GEN-NEXT:  namespace tdl {
+// GEN-EMPTY:
+// GEN-NEXT:    // Sets for dira
+// GEN-EMPTY:
+// GEN-NEXT:    static  allowedClauses_TDLD_dira {
+// GEN-NEXT:      llvm::tdl::Clause::TDLC_clausea,
+// GEN-NEXT:      llvm::tdl::Clause::TDLC_clauseb,
+// GEN-NEXT:    };
+// GEN-EMPTY:
+// GEN-NEXT:    static  allowedOnceClauses_TDLD_dira {
+// GEN-NEXT:    };
+// GEN-EMPTY:
+// GEN-NEXT:    static  allowedExclusiveClauses_TDLD_dira {
+// GEN-NEXT:    };
+// GEN-EMPTY:
+// GEN-NEXT:    static  requiredClauses_TDLD_dira {
+// GEN-NEXT:    };
+// GEN-NEXT:  } // namespace tdl
+// GEN-NEXT:  } // namespace llvm
+// GEN-EMPTY:
+// GEN-NEXT:  #endif // GEN_FLANG_DIRECTIVE_CLAUSE_SETS
+// GEN-EMPTY:
+// GEN-NEXT:  #ifdef GEN_FLANG_DIRECTIVE_CLAUSE_MAP
+// GEN-NEXT:  #undef GEN_FLANG_DIRECTIVE_CLAUSE_MAP
+// GEN-EMPTY:
+// GEN-NEXT:  struct TdlDirectiveClauses {
+// GEN-NEXT:    const  allowed;
+// GEN-NEXT:    const  allowedOnce;
+// GEN-NEXT:    const  allowedExclusive;
+// GEN-NEXT:    const  requiredOneOf;
+// GEN-NEXT:  };
+// GEN-EMPTY:
+// GEN-NEXT:  std::unordered_map<llvm::tdl::Directive, TdlDirectiveClauses>
+// GEN-NEXT:      directiveClausesTable = {
+// GEN-NEXT:    {llvm::tdl::Directive::TDLD_dira,
+// GEN-NEXT:      {
+// GEN-NEXT:        llvm::tdl::allowedClauses_TDLD_dira,
+// GEN-NEXT:        llvm::tdl::allowedOnceClauses_TDLD_dira,
+// GEN-NEXT:        llvm::tdl::allowedExclusiveClauses_TDLD_dira,
+// GEN-NEXT:        llvm::tdl::requiredClauses_TDLD_dira,
+// GEN-NEXT:      }
+// GEN-NEXT:    },
+// GEN-NEXT:  };
+// GEN-EMPTY:
+// GEN-NEXT:  #endif // GEN_FLANG_DIRECTIVE_CLAUSE_MAP
+
diff --git a/llvm/test/TableGen/directive2.td b/llvm/test/TableGen/directive2.td
new file mode 100644
index 0000000000000..517c79d457988
--- /dev/null
+++ b/llvm/test/TableGen/directive2.td
@@ -0,0 +1,173 @@
+// RUN: llvm-tblgen -gen-directive-decl -I %p/../../include %s | FileCheck -match-full-lines %s
+// RUN: llvm-tblgen -gen-directive-impl -I %p/../../include %s | FileCheck -match-full-lines %s -check-prefix=IMPL
+// RUN: llvm-tblgen -gen-directive-gen -I %p/../../include %s | FileCheck -match-full-lines %s -check-prefix=GEN
+
+include "llvm/Frontend/Directive/DirectiveBase.td"
+
+def TestDirectiveLanguage : DirectiveLanguage {
+  let name = "Tdl";
+
+  let cppNamespace = "tdl";
+  let directivePrefix = "TDLD_";
+  let clausePrefix = "TDLC_";
+  let includeHeader = "tdl.h.inc";
+}
+
+def TDLC_ClauseA : Clause<"clausea"> {
+  let isImplicit = 1;
+}
+def TDLC_ClauseB : Clause<"clauseb"> {
+  let isDefault = 1;
+}
+
+def TDL_DirA : Directive<"dira"> {
+  let allowedClauses = [
+    VersionedClause<TDLC_ClauseA, 2, 4>,
+    VersionedClause<TDLC_ClauseB, 2>
+  ];
+  let isDefault = 1;
+}
+
+// CHECK:       #ifndef LLVM_Tdl_INC
+// CHECK-NEXT:  #define LLVM_Tdl_INC
+// CHECK-EMPTY:
+// CHECK-NEXT:  namespace llvm {
+// CHECK-NEXT:  class StringRef;
+// CHECK-NEXT:  namespace tdl {
+// CHECK-EMPTY:
+// CHECK-NEXT:  enum class Directive {
+// CHECK-NEXT:    TDLD_dira,
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  static constexpr std::size_t Directive_enumSize = 1;
+// CHECK-EMPTY:
+// CHECK-NEXT:  enum class Clause {
+// CHECK-NEXT:    TDLC_clausea,
+// CHECK-NEXT:    TDLC_clauseb,
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT:  static constexpr std::size_t Clause_enumSize = 2;
+// CHECK-EMPTY:
+// CHECK-NEXT:  // Enumeration helper functions
+// CHECK-NEXT:  Directive getTdlDirectiveKind(llvm::StringRef Str);
+// CHECK-EMPTY:
+// CHECK-NEXT:  llvm::StringRef getTdlDirectiveName(Directive D);
+// CHECK-EMPTY:
+// CHECK-NEXT:  Clause getTdlClauseKind(llvm::StringRef Str);
+// CHECK-EMPTY:
+// CHECK-NEXT:  llvm::StringRef getTdlClauseName(Clause C);
+// CHECK-EMPTY:
+// CHECK-NEXT:  /// Return true if \p C is a valid clause for \p D in version \p Version.
+// CHECK-NEXT:  bool isAllowedClauseForDirective(Directive D, Clause C, unsigned Version);
+// CHECK-EMPTY:
+// CHECK-NEXT:  } // namespace tdl
+// CHECK-NEXT:  } // namespace llvm
+// CHECK-NEXT:  #endif // LLVM_Tdl_INC
+
+// IMPL:       #include "tdl.h.inc"
+// IMPL-EMPTY:
+// IMPL-NEXT:  #include "llvm/ADT/StringRef.h"
+// IMPL-NEXT:  #include "llvm/ADT/StringSwitch.h"
+// IMPL-EMPTY:
+// IMPL-NEXT:  using namespace llvm;
+// IMPL-NEXT:  using namespace tdl;
+// IMPL-EMPTY:
+// IMPL-NEXT:  Directive llvm::tdl::getTdlDirectiveKind(llvm::StringRef Str) {
+// IMPL-NEXT:    return llvm::StringSwitch<Directive>(Str)
+// IMPL-NEXT:      .Case("dira",TDLD_dira)
+// IMPL-NEXT:      .Default(TDLD_dira);
+// IMPL-NEXT:  }
+// IMPL-EMPTY:
+// IMPL-NEXT:  llvm::StringRef llvm::tdl::getTdlDirectiveName(Directive Kind) {
+// IMPL-NEXT:    switch (Kind) {
+// IMPL-NEXT:      case TDLD_dira:
+// IMPL-NEXT:        return "dira";
+// IMPL-NEXT:    }
+// IMPL-NEXT:    llvm_unreachable("Invalid Tdl Directive kind");
+// IMPL-NEXT:  }
+// IMPL-EMPTY:
+// IMPL-NEXT:  Clause llvm::tdl::getTdlClauseKind(llvm::StringRef Str) {
+// IMPL-NEXT:    return llvm::StringSwitch<Clause>(Str)
+// IMPL-NEXT:      .Case("clausea",TDLC_clauseb)
+// IMPL-NEXT:      .Case("clauseb",TDLC_clauseb)
+// IMPL-NEXT:      .Default(TDLC_clauseb);
+// IMPL-NEXT:  }
+// IMPL-EMPTY:
+// IMPL-NEXT:  llvm::StringRef llvm::tdl::getTdlClauseName(Clause Kind) {
+// IMPL-NEXT:    switch (Kind) {
+// IMPL-NEXT:      case TDLC_clausea:
+// IMPL-NEXT:        return "clausea";
+// IMPL-NEXT:      case TDLC_clauseb:
+// IMPL-NEXT:        return "clauseb";
+// IMPL-NEXT:    }
+// IMPL-NEXT:    llvm_unreachable("Invalid Tdl Clause kind");
+// IMPL-NEXT:  }
+// IMPL-EMPTY:
+// IMPL-NEXT:  bool llvm::tdl::isAllowedClauseForDirective(Directive D, Clause C, unsigned Version) {
+// IMPL-NEXT:    assert(unsigned(D) <= llvm::tdl::Directive_enumSize);
+// IMPL-NEXT:    assert(unsigned(C) <= llvm::tdl::Clause_enumSize);
+// IMPL-NEXT:    switch (D) {
+// IMPL-NEXT:      case TDLD_dira:
+// IMPL-NEXT:        switch (C) {
+// IMPL-NEXT:          case TDLC_clausea:
+// IMPL-NEXT:            return 2 <= Version && 4 >= Version;
+// IMPL-NEXT:          case TDLC_clauseb:
+// IMPL-NEXT:            return 2 <= Version && 2147483647 >= Version;
+// IMPL-NEXT:          default:
+// IMPL-NEXT:            return false;
+// IMPL-NEXT:        }
+// IMPL-NEXT:        break;
+// IMPL-NEXT:    }
+// IMPL-NEXT:    llvm_unreachable("Invalid Tdl Directive kind");
+// IMPL-NEXT:  }
+
+
+// GEN:      #ifdef GEN_FLANG_DIRECTIVE_CLAUSE_SETS
+// GEN-NEXT: #undef GEN_FLANG_DIRECTIVE_CLAUSE_SETS
+// GEN-EMPTY:
+// GEN-NEXT: namespace llvm {
+// GEN-NEXT: namespace tdl {
+// GEN-EMPTY:
+// GEN-NEXT:   // Sets for dira
+// GEN-EMPTY:
+// GEN-NEXT:   static  allowedClauses_TDLD_dira {
+// GEN-NEXT:     llvm::tdl::Clause::TDLC_clausea,
+// GEN-NEXT:     llvm::tdl::Clause::TDLC_clauseb,
+// GEN-NEXT:   };
+// GEN-EMPTY:
+// GEN-NEXT:   static  allowedOnceClauses_TDLD_dira {
+// GEN-NEXT:   };
+// GEN-EMPTY:
+// GEN-NEXT:   static  allowedExclusiveClauses_TDLD_dira {
+// GEN-NEXT:   };
+// GEN-EMPTY:
+// GEN-NEXT:   static  requiredClauses_TDLD_dira {
+// GEN-NEXT:   };
+// GEN-NEXT: } // namespace tdl
+// GEN-NEXT: } // namespace llvm
+// GEN-EMPTY:
+// GEN-NEXT: #endif // GEN_FLANG_DIRECTIVE_CLAUSE_SETS
+// GEN-EMPTY:
+// GEN-NEXT: #ifdef GEN_FLANG_DIRECTIVE_CLAUSE_MAP
+// GEN-NEXT: #undef GEN_FLANG_DIRECTIVE_CLAUSE_MAP
+// GEN-EMPTY:
+// GEN-NEXT: struct TdlDirectiveClauses {
+// GEN-NEXT:   const  allowed;
+// GEN-NEXT:   const  allowedOnce;
+// GEN-NEXT:   const  allowedExclusive;
+// GEN-NEXT:   const  requiredOneOf;
+// GEN-NEXT: };
+// GEN-EMPTY:
+// GEN-NEXT: std::unordered_map<llvm::tdl::Directive, TdlDirectiveClauses>
+// GEN-NEXT:     directiveClausesTable = {
+// GEN-NEXT:   {llvm::tdl::Directive::TDLD_dira,
+// GEN-NEXT:     {
+// GEN-NEXT:       llvm::tdl::allowedClauses_TDLD_dira,
+// GEN-NEXT:       llvm::tdl::allowedOnceClauses_TDLD_dira,
+// GEN-NEXT:       llvm::tdl::allowedExclusiveClauses_TDLD_dira,
+// GEN-NEXT:       llvm::tdl::requiredClauses_TDLD_dira,
+// GEN-NEXT:     }
+// GEN-NEXT:   },
+// GEN-NEXT: };
+// GEN-EMPTY:
+// GEN-NEXT: #endif // GEN_FLANG_DIRECTIVE_CLAUSE_MAP
diff --git a/llvm/test/ThinLTO/X86/cfi-devirt.ll b/llvm/test/ThinLTO/X86/cfi-devirt.ll
index 311bed084deeb..61e0c166812c8 100644
--- a/llvm/test/ThinLTO/X86/cfi-devirt.ll
+++ b/llvm/test/ThinLTO/X86/cfi-devirt.ll
@@ -110,10 +110,10 @@ cont2:
 
   ; Check that traps are conditional. Invalid TYPE_ID can cause
   ; unconditional traps.
-  ; CHECK-IR: br i1 {{.*}}, label %trap
+  ; CHECK-IR: br i1 {{.*}}, label %trap, label %cont2
 
   ; We still have to call it as virtual.
-  ; CHECK-IR: %call3 = tail call i32 %8
+  ; CHECK-IR: %call3 = tail call i32 %7
   %call3 = tail call i32 %8(%struct.A* nonnull %obj, i32 %call)
   ret i32 %call3
 }
diff --git a/llvm/test/ThinLTO/X86/devirt_multiple_type_test.ll b/llvm/test/ThinLTO/X86/devirt_multiple_type_test.ll
new file mode 100644
index 0000000000000..d5524b44286ea
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/devirt_multiple_type_test.ll
@@ -0,0 +1,80 @@
+; Test to ensure that devirtualization will succeed when there is an earlier
+; type test also corresponding to the same vtable (when indicated by invariant
+; load metadata), that provides a more refined type. This could happen in
+; after inlining into a caller passing a derived type.
+
+; RUN: opt -module-summary %s -o %t.o
+; RUN: llvm-lto2 run -o %t.out %t.o \
+; RUN:	 -pass-remarks=wholeprogramdevirt \
+; RUN:	 -r %t.o,_ZN1A3fooEv,px \
+; RUN:	 -r %t.o,_ZN1B3fooEv,px \
+; RUN:	 -r %t.o,_Z6callerP1B,px \
+; RUN:	 -r %t.o,_ZTV1A,px \
+; RUN:	 -r %t.o,_ZTV1B,px \
+; RUN:	 -save-temps 2>&1 | FileCheck %s
+
+; CHECK-COUNT-2: single-impl: devirtualized a call to _ZN1B3fooEv
+
+; RUN: llvm-dis %t.out.1.4.opt.bc -o - | FileCheck %s --check-prefix=IR
+; IR-NOT: tail call void %
+
+; ModuleID = 'devirt_multiple_type_test.o'
+source_filename = "devirt_multiple_type_test.cc"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%class.A = type { i32 (...)** }
+%class.B = type { %class.A }
+
+@_ZTV1A = hidden unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* undef, i8* bitcast (void (%class.A*)* @_ZN1A3fooEv to i8*)] }, align 8, !type !0, !vcall_visibility !2
+@_ZTV1B = hidden unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* undef, i8* bitcast (void (%class.B*)* @_ZN1B3fooEv to i8*)] }, align 8, !type !0, !type !3, !vcall_visibility !2
+
+declare void @_ZN1A3fooEv(%class.A* nocapture %this)
+
+define hidden void @_ZN1B3fooEv(%class.B* nocapture %this) {
+entry:
+  ret void
+}
+
+; Function Attrs: nounwind readnone willreturn
+declare i1 @llvm.type.test(i8*, metadata)
+
+; Function Attrs: nounwind willreturn
+declare void @llvm.assume(i1)
+
+; Function Attrs: uwtable
+define hidden void @_Z6callerP1B(%class.B* %b) local_unnamed_addr {
+entry:
+  %0 = bitcast %class.B* %b to void (%class.B*)***
+  %vtable = load void (%class.B*)**, void (%class.B*)*** %0, align 8, !tbaa !12, !invariant.group !15
+  %1 = bitcast void (%class.B*)** %vtable to i8*
+  %2 = tail call i1 @llvm.type.test(i8* %1, metadata !"_ZTS1B")
+  tail call void @llvm.assume(i1 %2)
+  %3 = load void (%class.B*)*, void (%class.B*)** %vtable, align 8, !invariant.load !15
+  tail call void %3(%class.B* %b)
+  %4 = getelementptr %class.B, %class.B* %b, i64 0, i32 0
+  %5 = bitcast void (%class.B*)** %vtable to i8*
+  %6 = tail call i1 @llvm.type.test(i8* nonnull %5, metadata !"_ZTS1A")
+  tail call void @llvm.assume(i1 %6)
+  %7 = bitcast void (%class.B*)* %3 to void (%class.A*)*
+  tail call void %7(%class.A* %4)
+  ret void
+}
+
+!llvm.module.flags = !{!5, !6, !8, !9, !10}
+!llvm.ident = !{!11}
+
+!0 = !{i64 16, !"_ZTS1A"}
+!2 = !{i64 1}
+!3 = !{i64 16, !"_ZTS1B"}
+!5 = !{i32 1, !"StrictVTablePointers", i32 1}
+!6 = !{i32 3, !"StrictVTablePointersRequirement", !7}
+!7 = !{!"StrictVTablePointers", i32 1}
+!8 = !{i32 1, !"wchar_size", i32 4}
+!9 = !{i32 1, !"Virtual Function Elim", i32 0}
+!10 = !{i32 1, !"EnableSplitLTOUnit", i32 0}
+!11 = !{!"clang version 11.0.0 (https://github.com/llvm/llvm-project.git 85247c1e898f88d65154b9a437b4bd83fcad8d52)"}
+!12 = !{!13, !13, i64 0}
+!13 = !{!"vtable pointer", !14, i64 0}
+!14 = !{!"Simple C++ TBAA"}
+!15 = !{}
diff --git a/llvm/test/Transforms/AlignmentFromAssumptions/simple32.ll b/llvm/test/Transforms/AlignmentFromAssumptions/simple32.ll
index 5aabe95184154..3f0819e3641b3 100644
--- a/llvm/test/Transforms/AlignmentFromAssumptions/simple32.ll
+++ b/llvm/test/Transforms/AlignmentFromAssumptions/simple32.ll
@@ -1,8 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64"
 ; RUN: opt < %s -alignment-from-assumptions -S | FileCheck %s
 ; RUN: opt < %s -passes=alignment-from-assumptions -S | FileCheck %s
 
 define i32 @foo(i32* nocapture %a) nounwind uwtable readonly {
+; CHECK-LABEL: define {{[^@]+}}@foo
+; CHECK-SAME: (i32* nocapture [[A:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A]], align 32
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
 entry:
   %ptrint = ptrtoint i32* %a to i64
   %maskedptr = and i64 %ptrint, 31
@@ -11,12 +22,21 @@ entry:
   %0 = load i32, i32* %a, align 4
   ret i32 %0
 
-; CHECK-LABEL: @foo
-; CHECK: load i32, i32* {{[^,]+}}, align 32
-; CHECK: ret i32
 }
 
 define i32 @foo2(i32* nocapture %a) nounwind uwtable readonly {
+; CHECK-LABEL: define {{[^@]+}}@foo2
+; CHECK-SAME: (i32* nocapture [[A:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64
+; CHECK-NEXT:    [[OFFSETPTR:%.*]] = add i64 [[PTRINT]], 24
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[OFFSETPTR]], 31
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 2
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 16
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
 entry:
   %ptrint = ptrtoint i32* %a to i64
   %offsetptr = add i64 %ptrint, 24
@@ -27,12 +47,21 @@ entry:
   %0 = load i32, i32* %arrayidx, align 4
   ret i32 %0
 
-; CHECK-LABEL: @foo2
-; CHECK: load i32, i32* {{[^,]+}}, align 16
-; CHECK: ret i32
 }
 
 define i32 @foo2a(i32* nocapture %a) nounwind uwtable readonly {
+; CHECK-LABEL: define {{[^@]+}}@foo2a
+; CHECK-SAME: (i32* nocapture [[A:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64
+; CHECK-NEXT:    [[OFFSETPTR:%.*]] = add i64 [[PTRINT]], 28
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[OFFSETPTR]], 31
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 -1
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 32
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
 entry:
   %ptrint = ptrtoint i32* %a to i64
   %offsetptr = add i64 %ptrint, 28
@@ -43,12 +72,19 @@ entry:
   %0 = load i32, i32* %arrayidx, align 4
   ret i32 %0
 
-; CHECK-LABEL: @foo2a
-; CHECK: load i32, i32* {{[^,]+}}, align 32
-; CHECK: ret i32
 }
 
 define i32 @goo(i32* nocapture %a) nounwind uwtable readonly {
+; CHECK-LABEL: define {{[^@]+}}@goo
+; CHECK-SAME: (i32* nocapture [[A:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A]], align 32
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
 entry:
   %ptrint = ptrtoint i32* %a to i64
   %maskedptr = and i64 %ptrint, 31
@@ -57,12 +93,31 @@ entry:
   %0 = load i32, i32* %a, align 4
   ret i32 %0
 
-; CHECK-LABEL: @goo
-; CHECK: load i32, i32* {{[^,]+}}, align 32
-; CHECK: ret i32
 }
 
 define i32 @hoo(i32* nocapture %a) nounwind uwtable readonly {
+; CHECK-LABEL: define {{[^@]+}}@hoo
+; CHECK-SAME: (i32* nocapture [[A:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[R_06:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 32
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[TMP0]], [[R_06]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP1]], 2048
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
+;
 entry:
   %ptrint = ptrtoint i32* %a to i64
   %maskedptr = and i64 %ptrint, 31
@@ -85,12 +140,31 @@ for.end:                                          ; preds = %for.body
   %add.lcssa = phi i32 [ %add, %for.body ]
   ret i32 %add.lcssa
 
-; CHECK-LABEL: @hoo
-; CHECK: load i32, i32* %arrayidx, align 32
-; CHECK: ret i32 %add.lcssa
 }
 
 define i32 @joo(i32* nocapture %a) nounwind uwtable readonly {
+; CHECK-LABEL: define {{[^@]+}}@joo
+; CHECK-SAME: (i32* nocapture [[A:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 4, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[R_06:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 16
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[TMP0]], [[R_06]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP1]], 2048
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
+;
 entry:
   %ptrint = ptrtoint i32* %a to i64
   %maskedptr = and i64 %ptrint, 31
@@ -113,12 +187,31 @@ for.end:                                          ; preds = %for.body
   %add.lcssa = phi i32 [ %add, %for.body ]
   ret i32 %add.lcssa
 
-; CHECK-LABEL: @joo
-; CHECK: load i32, i32* %arrayidx, align 16
-; CHECK: ret i32 %add.lcssa
 }
 
 define i32 @koo(i32* nocapture %a) nounwind uwtable readonly {
+; CHECK-LABEL: define {{[^@]+}}@koo
+; CHECK-SAME: (i32* nocapture [[A:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[R_06:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 16
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[TMP0]], [[R_06]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP1]], 2048
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
+;
 entry:
   %ptrint = ptrtoint i32* %a to i64
   %maskedptr = and i64 %ptrint, 31
@@ -141,12 +234,31 @@ for.end:                                          ; preds = %for.body
   %add.lcssa = phi i32 [ %add, %for.body ]
   ret i32 %add.lcssa
 
-; CHECK-LABEL: @koo
-; CHECK: load i32, i32* %arrayidx, align 16
-; CHECK: ret i32 %add.lcssa
 }
 
 define i32 @koo2(i32* nocapture %a) nounwind uwtable readonly {
+; CHECK-LABEL: define {{[^@]+}}@koo2
+; CHECK-SAME: (i32* nocapture [[A:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ -4, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[R_06:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 16
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[TMP0]], [[R_06]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP1]], 2048
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
+;
 entry:
   %ptrint = ptrtoint i32* %a to i64
   %maskedptr = and i64 %ptrint, 31
@@ -169,12 +281,20 @@ for.end:                                          ; preds = %for.body
   %add.lcssa = phi i32 [ %add, %for.body ]
   ret i32 %add.lcssa
 
-; CHECK-LABEL: @koo2
-; CHECK: load i32, i32* %arrayidx, align 16
-; CHECK: ret i32 %add.lcssa
 }
 
 define i32 @moo(i32* nocapture %a) nounwind uwtable {
+; CHECK-LABEL: define {{[^@]+}}@moo
+; CHECK-SAME: (i32* nocapture [[A:%.*]]) #1
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[A]] to i8*
+; CHECK-NEXT:    tail call void @llvm.memset.p0i8.i64(i8* align 32 [[TMP0]], i8 0, i64 64, i1 false)
+; CHECK-NEXT:    ret i32 undef
+;
 entry:
   %ptrint = ptrtoint i32* %a to i64
   %maskedptr = and i64 %ptrint, 31
@@ -184,12 +304,25 @@ entry:
   tail call void @llvm.memset.p0i8.i64(i8* align 4 %0, i8 0, i64 64, i1 false)
   ret i32 undef
 
-; CHECK-LABEL: @moo
-; CHECK: @llvm.memset.p0i8.i64(i8* align 32 %0, i8 0, i64 64, i1 false)
-; CHECK: ret i32 undef
 }
 
 define i32 @moo2(i32* nocapture %a, i32* nocapture %b) nounwind uwtable {
+; CHECK-LABEL: define {{[^@]+}}@moo2
+; CHECK-SAME: (i32* nocapture [[A:%.*]], i32* nocapture [[B:%.*]]) #1
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    [[PTRINT1:%.*]] = ptrtoint i32* [[B]] to i64
+; CHECK-NEXT:    [[MASKEDPTR3:%.*]] = and i64 [[PTRINT1]], 127
+; CHECK-NEXT:    [[MASKCOND4:%.*]] = icmp eq i64 [[MASKEDPTR3]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND4]])
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[A]] to i8*
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[B]] to i8*
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 32 [[TMP0]], i8* align 128 [[TMP1]], i64 64, i1 false)
+; CHECK-NEXT:    ret i32 undef
+;
 entry:
   %ptrint = ptrtoint i32* %a to i64
   %maskedptr = and i64 %ptrint, 31
@@ -204,9 +337,6 @@ entry:
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %0, i8* align 4 %1, i64 64, i1 false)
   ret i32 undef
 
-; CHECK-LABEL: @moo2
-; CHECK: @llvm.memcpy.p0i8.p0i8.i64(i8* align 32 %0, i8* align 128 %1, i64 64, i1 false)
-; CHECK: ret i32 undef
 }
 
 declare void @llvm.assume(i1) nounwind
diff --git a/llvm/test/Transforms/ArgumentPromotion/basictest.ll b/llvm/test/Transforms/ArgumentPromotion/basictest.ll
index 13cbd77812cc8..43f27b9208ee7 100644
--- a/llvm/test/Transforms/ArgumentPromotion/basictest.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/basictest.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
-; RUN: opt < %s -basicaa -argpromotion -mem2reg -S | FileCheck %s
+; RUN: opt < %s -basic-aa -argpromotion -mem2reg -S | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 define internal i32 @test(i32* %X, i32* %Y) {
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
index a963c3a31c376..b943af621940a 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
@@ -14,8 +14,8 @@ define internal i32 @deref(i32* %x) nounwind {
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@deref
 ; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]])
 ; IS__TUNIT_NPM-NEXT:  entry:
-; IS__TUNIT_NPM-NEXT:    [[X_PRIV:%.*]] = alloca i32
-; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[X_PRIV]]
+; IS__TUNIT_NPM-NEXT:    [[X_PRIV:%.*]] = alloca i32, align 4
+; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[X_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[TMP2:%.*]] = load i32, i32* [[X_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[TMP2]]
 ;
@@ -34,7 +34,7 @@ define i32 @f(i32 %x) {
 ; NOT_TUNIT_NPM-LABEL: define {{[^@]+}}@f
 ; NOT_TUNIT_NPM-SAME: (i32 [[X:%.*]])
 ; NOT_TUNIT_NPM-NEXT:  entry:
-; NOT_TUNIT_NPM-NEXT:    [[X_ADDR:%.*]] = alloca i32
+; NOT_TUNIT_NPM-NEXT:    [[X_ADDR:%.*]] = alloca i32, align 4
 ; NOT_TUNIT_NPM-NEXT:    store i32 [[X]], i32* [[X_ADDR]], align 4
 ; NOT_TUNIT_NPM-NEXT:    [[TMP1:%.*]] = call i32 @deref(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[X_ADDR]])
 ; NOT_TUNIT_NPM-NEXT:    ret i32 [[TMP1]]
@@ -42,7 +42,7 @@ define i32 @f(i32 %x) {
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@f
 ; IS__TUNIT_NPM-SAME: (i32 [[X:%.*]])
 ; IS__TUNIT_NPM-NEXT:  entry:
-; IS__TUNIT_NPM-NEXT:    [[X_ADDR:%.*]] = alloca i32
+; IS__TUNIT_NPM-NEXT:    [[X_ADDR:%.*]] = alloca i32, align 4
 ; IS__TUNIT_NPM-NEXT:    store i32 [[X]], i32* [[X_ADDR]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[TMP0:%.*]] = load i32, i32* [[X_ADDR]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[TMP1:%.*]] = call i32 @deref(i32 [[TMP0]])
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll
index f0aeb8d15add5..49342ae4f04c0 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll
@@ -100,8 +100,8 @@ define internal fastcc void @promote_avx2(<4 x i64>* %arg, <4 x i64>* readonly %
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@promote_avx2
 ; IS__TUNIT_NPM-SAME: (<4 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(32) [[ARG:%.*]], <4 x i64> [[TMP0:%.*]])
 ; IS__TUNIT_NPM-NEXT:  bb:
-; IS__TUNIT_NPM-NEXT:    [[ARG1_PRIV:%.*]] = alloca <4 x i64>
-; IS__TUNIT_NPM-NEXT:    store <4 x i64> [[TMP0]], <4 x i64>* [[ARG1_PRIV]]
+; IS__TUNIT_NPM-NEXT:    [[ARG1_PRIV:%.*]] = alloca <4 x i64>, align 32
+; IS__TUNIT_NPM-NEXT:    store <4 x i64> [[TMP0]], <4 x i64>* [[ARG1_PRIV]], align 32
 ; IS__TUNIT_NPM-NEXT:    [[TMP:%.*]] = load <4 x i64>, <4 x i64>* [[ARG1_PRIV]], align 32
 ; IS__TUNIT_NPM-NEXT:    store <4 x i64> [[TMP]], <4 x i64>* [[ARG]], align 32
 ; IS__TUNIT_NPM-NEXT:    ret void
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
index e3a33c6121e67..361dfa0cd5923 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
@@ -21,8 +21,8 @@ define internal fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal5
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512
 ; IS__TUNIT_NPM-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
 ; IS__TUNIT_NPM-NEXT:  bb:
-; IS__TUNIT_NPM-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>
-; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]]
+; IS__TUNIT_NPM-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>, align 64
+; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]], align 64
 ; IS__TUNIT_NPM-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 64
 ; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 64
 ; IS__TUNIT_NPM-NEXT:    ret void
@@ -108,8 +108,8 @@ define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal5
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256
 ; IS__TUNIT_NPM-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
 ; IS__TUNIT_NPM-NEXT:  bb:
-; IS__TUNIT_NPM-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>
-; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]]
+; IS__TUNIT_NPM-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>, align 64
+; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]], align 64
 ; IS__TUNIT_NPM-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 64
 ; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 64
 ; IS__TUNIT_NPM-NEXT:    ret void
@@ -195,8 +195,8 @@ define internal fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal5
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256
 ; IS__TUNIT_NPM-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
 ; IS__TUNIT_NPM-NEXT:  bb:
-; IS__TUNIT_NPM-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>
-; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]]
+; IS__TUNIT_NPM-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>, align 64
+; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]], align 64
 ; IS__TUNIT_NPM-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 64
 ; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 64
 ; IS__TUNIT_NPM-NEXT:    ret void
@@ -282,8 +282,8 @@ define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal5
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512
 ; IS__TUNIT_NPM-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
 ; IS__TUNIT_NPM-NEXT:  bb:
-; IS__TUNIT_NPM-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>
-; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]]
+; IS__TUNIT_NPM-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>, align 64
+; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]], align 64
 ; IS__TUNIT_NPM-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 64
 ; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 64
 ; IS__TUNIT_NPM-NEXT:    ret void
@@ -537,8 +537,8 @@ define internal fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_p
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256
 ; IS__TUNIT_NPM-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
 ; IS__TUNIT_NPM-NEXT:  bb:
-; IS__TUNIT_NPM-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>
-; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]]
+; IS__TUNIT_NPM-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>, align 64
+; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]], align 64
 ; IS__TUNIT_NPM-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 64
 ; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 64
 ; IS__TUNIT_NPM-NEXT:    ret void
@@ -624,8 +624,8 @@ define internal fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_p
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256
 ; IS__TUNIT_NPM-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
 ; IS__TUNIT_NPM-NEXT:  bb:
-; IS__TUNIT_NPM-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>
-; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]]
+; IS__TUNIT_NPM-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>, align 64
+; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]], align 64
 ; IS__TUNIT_NPM-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 64
 ; IS__TUNIT_NPM-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 64
 ; IS__TUNIT_NPM-NEXT:    ret void
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll
index c869ba50874bf..0367a5e42ec16 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll
@@ -33,8 +33,8 @@ define internal void @g(i32* %a) {
 ;
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@g
 ; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]])
-; IS__TUNIT_NPM-NEXT:    [[A_PRIV:%.*]] = alloca i32
-; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[A_PRIV]]
+; IS__TUNIT_NPM-NEXT:    [[A_PRIV:%.*]] = alloca i32, align 4
+; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[A_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[AA:%.*]] = load i32, i32* [[A_PRIV]], align 1
 ; IS__TUNIT_NPM-NEXT:    call void @z(i32 [[AA]])
 ; IS__TUNIT_NPM-NEXT:    ret void
@@ -70,10 +70,10 @@ define internal i32 @test(i32* %X, i64* %Y) {
 ;
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@test
 ; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]], i64 [[TMP1:%.*]])
-; IS__TUNIT_NPM-NEXT:    [[Y_PRIV:%.*]] = alloca i64
-; IS__TUNIT_NPM-NEXT:    store i64 [[TMP1]], i64* [[Y_PRIV]]
-; IS__TUNIT_NPM-NEXT:    [[X_PRIV:%.*]] = alloca i32
-; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[X_PRIV]]
+; IS__TUNIT_NPM-NEXT:    [[Y_PRIV:%.*]] = alloca i64, align 8
+; IS__TUNIT_NPM-NEXT:    store i64 [[TMP1]], i64* [[Y_PRIV]], align 4
+; IS__TUNIT_NPM-NEXT:    [[X_PRIV:%.*]] = alloca i32, align 4
+; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[X_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[A:%.*]] = load i32, i32* [[X_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[B:%.*]] = load i64, i64* [[Y_PRIV]], align 8
 ; IS__TUNIT_NPM-NEXT:    [[C:%.*]] = add i32 [[A]], 1
@@ -113,16 +113,16 @@ Return2:
 define internal i32 @caller(i32* %A) {
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@caller
 ; IS__TUNIT_OPM-SAME: (i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A:%.*]])
-; IS__TUNIT_OPM-NEXT:    [[B:%.*]] = alloca i64
+; IS__TUNIT_OPM-NEXT:    [[B:%.*]] = alloca i64, align 8
 ; IS__TUNIT_OPM-NEXT:    store i64 1, i64* [[B]], align 8
 ; IS__TUNIT_OPM-NEXT:    [[C:%.*]] = call i32 @test(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A]], i64* noalias nocapture nofree nonnull readonly align 8 dereferenceable(8) [[B]])
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[C]]
 ;
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@caller
 ; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]])
-; IS__TUNIT_NPM-NEXT:    [[A_PRIV:%.*]] = alloca i32
-; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[A_PRIV]]
-; IS__TUNIT_NPM-NEXT:    [[B:%.*]] = alloca i64
+; IS__TUNIT_NPM-NEXT:    [[A_PRIV:%.*]] = alloca i32, align 4
+; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[A_PRIV]], align 4
+; IS__TUNIT_NPM-NEXT:    [[B:%.*]] = alloca i64, align 8
 ; IS__TUNIT_NPM-NEXT:    store i64 1, i64* [[B]], align 8
 ; IS__TUNIT_NPM-NEXT:    [[TMP2:%.*]] = load i32, i32* [[A_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[TMP3:%.*]] = load i64, i64* [[B]], align 8
@@ -131,7 +131,7 @@ define internal i32 @caller(i32* %A) {
 ;
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@caller
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A:%.*]])
-; IS__CGSCC____-NEXT:    [[B:%.*]] = alloca i64
+; IS__CGSCC____-NEXT:    [[B:%.*]] = alloca i64, align 8
 ; IS__CGSCC____-NEXT:    store i64 1, i64* [[B]], align 8
 ; IS__CGSCC____-NEXT:    [[C:%.*]] = call i32 @test(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A]], i64* noalias nocapture nofree nonnull readonly align 8 dereferenceable(8) [[B]])
 ; IS__CGSCC____-NEXT:    ret i32 [[C]]
@@ -144,13 +144,13 @@ define internal i32 @caller(i32* %A) {
 
 define i32 @callercaller() {
 ; NOT_TUNIT_NPM-LABEL: define {{[^@]+}}@callercaller()
-; NOT_TUNIT_NPM-NEXT:    [[B:%.*]] = alloca i32
+; NOT_TUNIT_NPM-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; NOT_TUNIT_NPM-NEXT:    store i32 2, i32* [[B]], align 4
 ; NOT_TUNIT_NPM-NEXT:    [[X:%.*]] = call i32 @caller(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]])
 ; NOT_TUNIT_NPM-NEXT:    ret i32 [[X]]
 ;
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@callercaller()
-; IS__TUNIT_NPM-NEXT:    [[B:%.*]] = alloca i32
+; IS__TUNIT_NPM-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; IS__TUNIT_NPM-NEXT:    store i32 2, i32* [[B]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[B]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[X:%.*]] = call i32 @caller(i32 [[TMP1]])
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
index 3877097e127ee..3519c96a731a4 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
@@ -15,10 +15,10 @@ define internal i32 @test(i32* %X, i32* %Y) {
 ;
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@test
 ; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]])
-; IS__TUNIT_NPM-NEXT:    [[Y_PRIV:%.*]] = alloca i32
-; IS__TUNIT_NPM-NEXT:    store i32 [[TMP1]], i32* [[Y_PRIV]]
-; IS__TUNIT_NPM-NEXT:    [[X_PRIV:%.*]] = alloca i32
-; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[X_PRIV]]
+; IS__TUNIT_NPM-NEXT:    [[Y_PRIV:%.*]] = alloca i32, align 4
+; IS__TUNIT_NPM-NEXT:    store i32 [[TMP1]], i32* [[Y_PRIV]], align 4
+; IS__TUNIT_NPM-NEXT:    [[X_PRIV:%.*]] = alloca i32, align 4
+; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[X_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[A:%.*]] = load i32, i32* [[X_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[B:%.*]] = load i32, i32* [[Y_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[C:%.*]] = add i32 [[A]], [[B]]
@@ -40,16 +40,16 @@ define internal i32 @test(i32* %X, i32* %Y) {
 define internal i32 @caller(i32* %B) {
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@caller
 ; IS__TUNIT_OPM-SAME: (i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B:%.*]])
-; IS__TUNIT_OPM-NEXT:    [[A:%.*]] = alloca i32
+; IS__TUNIT_OPM-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; IS__TUNIT_OPM-NEXT:    store i32 1, i32* [[A]], align 4
 ; IS__TUNIT_OPM-NEXT:    [[C:%.*]] = call i32 @test(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A]], i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]])
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[C]]
 ;
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@caller
 ; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]])
-; IS__TUNIT_NPM-NEXT:    [[B_PRIV:%.*]] = alloca i32
-; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[B_PRIV]]
-; IS__TUNIT_NPM-NEXT:    [[A:%.*]] = alloca i32
+; IS__TUNIT_NPM-NEXT:    [[B_PRIV:%.*]] = alloca i32, align 4
+; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[B_PRIV]], align 4
+; IS__TUNIT_NPM-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; IS__TUNIT_NPM-NEXT:    store i32 1, i32* [[A]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[TMP2:%.*]] = load i32, i32* [[A]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[TMP3:%.*]] = load i32, i32* [[B_PRIV]], align 4
@@ -58,7 +58,7 @@ define internal i32 @caller(i32* %B) {
 ;
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@caller
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B:%.*]])
-; IS__CGSCC____-NEXT:    [[A:%.*]] = alloca i32
+; IS__CGSCC____-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; IS__CGSCC____-NEXT:    store i32 1, i32* [[A]], align 4
 ; IS__CGSCC____-NEXT:    [[C:%.*]] = call i32 @test(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A]], i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]])
 ; IS__CGSCC____-NEXT:    ret i32 [[C]]
@@ -71,13 +71,13 @@ define internal i32 @caller(i32* %B) {
 
 define i32 @callercaller() {
 ; NOT_TUNIT_NPM-LABEL: define {{[^@]+}}@callercaller()
-; NOT_TUNIT_NPM-NEXT:    [[B:%.*]] = alloca i32
+; NOT_TUNIT_NPM-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; NOT_TUNIT_NPM-NEXT:    store i32 2, i32* [[B]], align 4
 ; NOT_TUNIT_NPM-NEXT:    [[X:%.*]] = call i32 @caller(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]])
 ; NOT_TUNIT_NPM-NEXT:    ret i32 [[X]]
 ;
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@callercaller()
-; IS__TUNIT_NPM-NEXT:    [[B:%.*]] = alloca i32
+; IS__TUNIT_NPM-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; IS__TUNIT_NPM-NEXT:    store i32 2, i32* [[B]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[B]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[X:%.*]] = call i32 @caller(i32 [[TMP1]])
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll
index 194188cdf8ae0..a74151b777a5a 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll
@@ -94,15 +94,15 @@ define i32 @main() nounwind  {
 ; IS__TUNIT_NPM-NEXT:    store i32 1, i32* [[TMP1]], align 8
 ; IS__TUNIT_NPM-NEXT:    [[TMP4:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
 ; IS__TUNIT_NPM-NEXT:    store i64 2, i64* [[TMP4]], align 4
-; IS__TUNIT_NPM-NEXT:    [[S_CAST1:%.*]] = bitcast %struct.ss* [[S]] to i32*
-; IS__TUNIT_NPM-NEXT:    [[TMP0:%.*]] = load i32, i32* [[S_CAST1]], align 8
-; IS__TUNIT_NPM-NEXT:    [[S_0_12:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
-; IS__TUNIT_NPM-NEXT:    [[TMP1:%.*]] = load i64, i64* [[S_0_12]], align 8
-; IS__TUNIT_NPM-NEXT:    [[C0:%.*]] = call i32 @f(i32 [[TMP0]], i64 [[TMP1]])
 ; IS__TUNIT_NPM-NEXT:    [[S_CAST:%.*]] = bitcast %struct.ss* [[S]] to i32*
-; IS__TUNIT_NPM-NEXT:    [[TMP2:%.*]] = load i32, i32* [[S_CAST]], align 32
+; IS__TUNIT_NPM-NEXT:    [[TMP0:%.*]] = load i32, i32* [[S_CAST]], align 8
 ; IS__TUNIT_NPM-NEXT:    [[S_0_1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
-; IS__TUNIT_NPM-NEXT:    [[TMP3:%.*]] = load i64, i64* [[S_0_1]], align 32
+; IS__TUNIT_NPM-NEXT:    [[TMP1:%.*]] = load i64, i64* [[S_0_1]], align 8
+; IS__TUNIT_NPM-NEXT:    [[C0:%.*]] = call i32 @f(i32 [[TMP0]], i64 [[TMP1]])
+; IS__TUNIT_NPM-NEXT:    [[S_CAST1:%.*]] = bitcast %struct.ss* [[S]] to i32*
+; IS__TUNIT_NPM-NEXT:    [[TMP2:%.*]] = load i32, i32* [[S_CAST1]], align 32
+; IS__TUNIT_NPM-NEXT:    [[S_0_12:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; IS__TUNIT_NPM-NEXT:    [[TMP3:%.*]] = load i64, i64* [[S_0_12]], align 32
 ; IS__TUNIT_NPM-NEXT:    [[C1:%.*]] = call i32 @g(i32 [[TMP2]], i64 [[TMP3]])
 ; IS__TUNIT_NPM-NEXT:    [[A:%.*]] = add i32 [[C0]], [[C1]]
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[A]]
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll
index 61b1c19e49a06..eaeebf14ae0ad 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll
@@ -18,8 +18,8 @@ define internal i32 @callee(i1 %C, i32* %P) {
 ;
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@callee
 ; IS__TUNIT_NPM-SAME: (i1 [[C:%.*]], i32 [[TMP0:%.*]])
-; IS__TUNIT_NPM-NEXT:    [[P_PRIV:%.*]] = alloca i32
-; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[P_PRIV]]
+; IS__TUNIT_NPM-NEXT:    [[P_PRIV:%.*]] = alloca i32, align 4
+; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[P_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    br label [[F:%.*]]
 ; IS__TUNIT_NPM:       T:
 ; IS__TUNIT_NPM-NEXT:    unreachable
@@ -48,20 +48,20 @@ F:              ; preds = %0
 
 define i32 @foo() {
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@foo()
-; IS__TUNIT_OPM-NEXT:    [[A:%.*]] = alloca i32
+; IS__TUNIT_OPM-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; IS__TUNIT_OPM-NEXT:    store i32 17, i32* [[A]], align 4
 ; IS__TUNIT_OPM-NEXT:    [[X:%.*]] = call i32 @callee(i1 false, i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A]])
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[X]]
 ;
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@foo()
-; IS__TUNIT_NPM-NEXT:    [[A:%.*]] = alloca i32
+; IS__TUNIT_NPM-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; IS__TUNIT_NPM-NEXT:    store i32 17, i32* [[A]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[A]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[X:%.*]] = call i32 @callee(i1 false, i32 [[TMP1]])
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[X]]
 ;
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@foo()
-; IS__CGSCC____-NEXT:    [[A:%.*]] = alloca i32
+; IS__CGSCC____-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; IS__CGSCC____-NEXT:    store i32 17, i32* [[A]], align 4
 ; IS__CGSCC____-NEXT:    [[X:%.*]] = call i32 @callee(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A]])
 ; IS__CGSCC____-NEXT:    ret i32 [[X]]
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll
index 66e3ad1cc6d38..5a03026c2dc2c 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll
@@ -75,8 +75,8 @@ define i32 @test_inf_promote_caller(i32 %arg) {
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test_inf_promote_caller
 ; IS__CGSCC____-SAME: (i32 [[ARG:%.*]])
 ; IS__CGSCC____-NEXT:  bb:
-; IS__CGSCC____-NEXT:    [[TMP:%.*]] = alloca [[S:%.*]]
-; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = alloca [[S]]
+; IS__CGSCC____-NEXT:    [[TMP:%.*]] = alloca [[S:%.*]], align 8
+; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = alloca [[S]], align 8
 ; IS__CGSCC____-NEXT:    unreachable
 ;
 bb:
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
index 070df749a15de..4da401f1db296 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
@@ -41,13 +41,13 @@ dead:
 
 define internal i32 @caller(i32* %B) {
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@caller()
-; IS__CGSCC_OPM-NEXT:    [[A:%.*]] = alloca i32
+; IS__CGSCC_OPM-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; IS__CGSCC_OPM-NEXT:    store i32 1, i32* [[A]], align 4
 ; IS__CGSCC_OPM-NEXT:    [[C:%.*]] = call i32 @test(i32* noalias nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[A]])
 ; IS__CGSCC_OPM-NEXT:    ret i32 0
 ;
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@caller()
-; IS__CGSCC_NPM-NEXT:    [[A:%.*]] = alloca i32
+; IS__CGSCC_NPM-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; IS__CGSCC_NPM-NEXT:    store i32 1, i32* [[A]], align 4
 ; IS__CGSCC_NPM-NEXT:    [[C:%.*]] = call i32 @test(i32* noalias nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[A]])
 ; IS__CGSCC_NPM-NEXT:    ret i32 undef
@@ -60,7 +60,7 @@ define internal i32 @caller(i32* %B) {
 
 define i32 @callercaller() {
 ; CHECK-LABEL: define {{[^@]+}}@callercaller()
-; CHECK-NEXT:    [[B:%.*]] = alloca i32
+; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store i32 2, i32* [[B]], align 4
 ; CHECK-NEXT:    ret i32 0
 ;
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll
index 4491ceb2d2773..9d8d5e5decc39 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll
@@ -51,21 +51,21 @@ dead:
 define internal i32 @caller(i32* %B) {
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@caller
 ; IS__TUNIT____-SAME: (i32* noalias nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[B:%.*]])
-; IS__TUNIT____-NEXT:    [[A:%.*]] = alloca i32
+; IS__TUNIT____-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; IS__TUNIT____-NEXT:    store i32 1, i32* [[A]], align 4
 ; IS__TUNIT____-NEXT:    [[C:%.*]] = call i32 @test(i32* noalias nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[B]])
 ; IS__TUNIT____-NEXT:    ret i32 0
 ;
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@caller
 ; IS__CGSCC_OPM-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[B:%.*]])
-; IS__CGSCC_OPM-NEXT:    [[A:%.*]] = alloca i32
+; IS__CGSCC_OPM-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; IS__CGSCC_OPM-NEXT:    store i32 1, i32* [[A]], align 4
 ; IS__CGSCC_OPM-NEXT:    [[C:%.*]] = call i32 @test(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[B]])
 ; IS__CGSCC_OPM-NEXT:    ret i32 0
 ;
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@caller
 ; IS__CGSCC_NPM-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[B:%.*]])
-; IS__CGSCC_NPM-NEXT:    [[A:%.*]] = alloca i32
+; IS__CGSCC_NPM-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; IS__CGSCC_NPM-NEXT:    store i32 1, i32* [[A]], align 4
 ; IS__CGSCC_NPM-NEXT:    [[C:%.*]] = call i32 @test(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[B]])
 ; IS__CGSCC_NPM-NEXT:    ret i32 undef
@@ -78,7 +78,7 @@ define internal i32 @caller(i32* %B) {
 
 define i32 @callercaller() {
 ; CHECK-LABEL: define {{[^@]+}}@callercaller()
-; CHECK-NEXT:    [[B:%.*]] = alloca i32
+; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store i32 2, i32* [[B]], align 4
 ; CHECK-NEXT:    [[X:%.*]] = call i32 @caller(i32* noalias nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[B]])
 ; CHECK-NEXT:    ret i32 0
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
index e33891d27c312..a372028311a63 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
@@ -15,7 +15,7 @@
 
 define void @foo() {
 ; CHECK-LABEL: define {{[^@]+}}@foo()
-; CHECK-NEXT:    [[TMP:%.*]] = alloca void (i16*)*
+; CHECK-NEXT:    [[TMP:%.*]] = alloca void (i16*)*, align 8
 ; CHECK-NEXT:    store void (i16*)* @bar, void (i16*)** [[TMP]], align 8
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll
index 87bd530d96476..cacc6b95d2638 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll
@@ -9,13 +9,13 @@ target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:1
 
 define void @caller() #0 {
 ; NOT_TUNIT_NPM-LABEL: define {{[^@]+}}@caller()
-; NOT_TUNIT_NPM-NEXT:    [[X:%.*]] = alloca i32
+; NOT_TUNIT_NPM-NEXT:    [[X:%.*]] = alloca i32, align 4
 ; NOT_TUNIT_NPM-NEXT:    store i32 42, i32* [[X]], align 4
 ; NOT_TUNIT_NPM-NEXT:    call void @promote_i32_ptr(i32* noalias nocapture nonnull readonly align 4 dereferenceable(4) [[X]]), !prof !0
 ; NOT_TUNIT_NPM-NEXT:    ret void
 ;
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@caller()
-; IS__TUNIT_NPM-NEXT:    [[X:%.*]] = alloca i32
+; IS__TUNIT_NPM-NEXT:    [[X:%.*]] = alloca i32, align 4
 ; IS__TUNIT_NPM-NEXT:    store i32 42, i32* [[X]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[X]], align 4
 ; IS__TUNIT_NPM-NEXT:    call void @promote_i32_ptr(i32 [[TMP1]]), !prof !0
@@ -36,8 +36,8 @@ define internal void @promote_i32_ptr(i32* %xp) {
 ;
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@promote_i32_ptr
 ; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]])
-; IS__TUNIT_NPM-NEXT:    [[XP_PRIV:%.*]] = alloca i32
-; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[XP_PRIV]]
+; IS__TUNIT_NPM-NEXT:    [[XP_PRIV:%.*]] = alloca i32, align 4
+; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[XP_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[X:%.*]] = load i32, i32* [[XP_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    call void @use_i32(i32 [[X]])
 ; IS__TUNIT_NPM-NEXT:    ret void
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
index 12408c1a37928..a84ab1f4ef4b2 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
@@ -40,14 +40,14 @@ define internal void @add({i32, i32}* %this, i32* sret %r) {
 
 define void @f() {
 ; IS________OPM-LABEL: define {{[^@]+}}@f()
-; IS________OPM-NEXT:    [[R:%.*]] = alloca i32
-; IS________OPM-NEXT:    [[PAIR:%.*]] = alloca { i32, i32 }
+; IS________OPM-NEXT:    [[R:%.*]] = alloca i32, align 4
+; IS________OPM-NEXT:    [[PAIR:%.*]] = alloca { i32, i32 }, align 8
 ; IS________OPM-NEXT:    call void @add({ i32, i32 }* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[PAIR]], i32* nocapture nofree nonnull sret writeonly align 4 dereferenceable(4) [[R]])
 ; IS________OPM-NEXT:    ret void
 ;
 ; IS________NPM-LABEL: define {{[^@]+}}@f()
-; IS________NPM-NEXT:    [[R:%.*]] = alloca i32
-; IS________NPM-NEXT:    [[PAIR:%.*]] = alloca { i32, i32 }
+; IS________NPM-NEXT:    [[R:%.*]] = alloca i32, align 4
+; IS________NPM-NEXT:    [[PAIR:%.*]] = alloca { i32, i32 }, align 8
 ; IS________NPM-NEXT:    call void @add({ i32, i32 }* noalias nocapture nofree nonnull readonly align 8 dereferenceable(8) [[PAIR]], i32* noalias nocapture nofree nonnull sret writeonly align 4 dereferenceable(4) [[R]])
 ; IS________NPM-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll b/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll
index a285ffc03f27d..aa1374a7c3c51 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll
@@ -19,7 +19,7 @@ define internal void @foo(i32 %x) nounwind readnone {
 ; IS__CGSCC____-SAME: (i32 [[X:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
 ; IS__CGSCC____-NEXT:    [[B:%.*]] = alloca i32, align 4
-; IS__CGSCC____-NEXT:    store volatile i32 -1, i32* [[B]]
+; IS__CGSCC____-NEXT:    store volatile i32 -1, i32* [[B]], align 4
 ; IS__CGSCC____-NEXT:    ret void
 ;
 entry:
@@ -41,9 +41,9 @@ define internal void @bar(i32* nocapture %pc) nounwind readonly {
 ; IS__CGSCC_OPM:       indirectgoto:
 ; IS__CGSCC_OPM-NEXT:    [[INDVAR]] = phi i32 [ [[INDVAR_NEXT]], [[LAB0:%.*]] ], [ 0, [[ENTRY:%.*]] ]
 ; IS__CGSCC_OPM-NEXT:    [[PC_ADDR_0:%.*]] = getelementptr i32, i32* [[PC]], i32 [[INDVAR]]
-; IS__CGSCC_OPM-NEXT:    [[TMP1_PN:%.*]] = load i32, i32* [[PC_ADDR_0]]
+; IS__CGSCC_OPM-NEXT:    [[TMP1_PN:%.*]] = load i32, i32* [[PC_ADDR_0]], align 4
 ; IS__CGSCC_OPM-NEXT:    [[INDIRECT_GOTO_DEST_IN:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* @bar.l, i32 0, i32 [[TMP1_PN]]
-; IS__CGSCC_OPM-NEXT:    [[INDIRECT_GOTO_DEST:%.*]] = load i8*, i8** [[INDIRECT_GOTO_DEST_IN]]
+; IS__CGSCC_OPM-NEXT:    [[INDIRECT_GOTO_DEST:%.*]] = load i8*, i8** [[INDIRECT_GOTO_DEST_IN]], align 8
 ; IS__CGSCC_OPM-NEXT:    indirectbr i8* [[INDIRECT_GOTO_DEST]], [label [[LAB0]], label %end]
 ;
 entry:
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll b/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
index 738b19744e67a..4b19fbebbcb1e 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
@@ -58,7 +58,7 @@ define dso_local void @foo(i32 %N) {
 ; IS__CGSCC_OPM-NEXT:    store i32 [[N]], i32* [[N_ADDR]], align 4
 ; IS__CGSCC_OPM-NEXT:    store float 3.000000e+00, float* [[P]], align 4
 ; IS__CGSCC_OPM-NEXT:    store i32 7, i32* [[N_ADDR]], align 4
-; IS__CGSCC_OPM-NEXT:    call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* nonnull align 8 dereferenceable(24) @1, i32 3, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i32*, float*, i64)* @.omp_outlined. to void (i32*, i32*, ...)*), i32* nonnull align 4 dereferenceable(4) [[N_ADDR]], float* nonnull align 4 dereferenceable(4) [[P]], i64 4617315517961601024)
+; IS__CGSCC_OPM-NEXT:    call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* nonnull align 8 dereferenceable(24) @1, i32 3, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i32*, float*, i64)* @.omp_outlined. to void (i32*, i32*, ...)*), i32* nocapture nonnull readonly align 4 dereferenceable(4) [[N_ADDR]], float* nocapture nonnull readonly align 4 dereferenceable(4) [[P]], i64 4617315517961601024)
 ; IS__CGSCC_OPM-NEXT:    ret void
 ;
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@foo
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll b/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll
index f45b79f8e726a..b28b38f896877 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll
@@ -42,27 +42,16 @@ define dso_local i32 @main() {
 ; IS__TUNIT____-NEXT:    [[CALL3:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias nocapture align 536870912 null, i8* (i8*)* nonnull @buz, i8* noalias nofree nonnull readnone align 8 dereferenceable(1) "no-capture-maybe-returned" [[ALLOC2]])
 ; IS__TUNIT____-NEXT:    ret i32 0
 ;
-; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@main()
-; IS__CGSCC_OPM-NEXT:  entry:
-; IS__CGSCC_OPM-NEXT:    [[ALLOC1:%.*]] = alloca i8, align 8
-; IS__CGSCC_OPM-NEXT:    [[ALLOC2:%.*]] = alloca i8, align 8
-; IS__CGSCC_OPM-NEXT:    [[THREAD:%.*]] = alloca i64, align 8
-; IS__CGSCC_OPM-NEXT:    [[CALL:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias nocapture align 536870912 null, i8* (i8*)* nonnull @foo, i8* noalias nocapture align 536870912 null)
-; IS__CGSCC_OPM-NEXT:    [[CALL1:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias nocapture align 536870912 null, i8* (i8*)* nonnull @bar, i8* nonnull align 8 dereferenceable(8) bitcast (i8** @GlobalVPtr to i8*))
-; IS__CGSCC_OPM-NEXT:    [[CALL2:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias nocapture align 536870912 null, i8* (i8*)* nonnull @baz, i8* nocapture nonnull align 8 dereferenceable(1) [[ALLOC1]])
-; IS__CGSCC_OPM-NEXT:    [[CALL3:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias nocapture align 536870912 null, i8* (i8*)* nonnull @buz, i8* nonnull align 8 dereferenceable(1) [[ALLOC2]])
-; IS__CGSCC_OPM-NEXT:    ret i32 0
-;
-; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@main()
-; IS__CGSCC_NPM-NEXT:  entry:
-; IS__CGSCC_NPM-NEXT:    [[ALLOC1:%.*]] = alloca i8, align 8
-; IS__CGSCC_NPM-NEXT:    [[ALLOC2:%.*]] = alloca i8, align 8
-; IS__CGSCC_NPM-NEXT:    [[THREAD:%.*]] = alloca i64, align 8
-; IS__CGSCC_NPM-NEXT:    [[CALL:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias nocapture align 536870912 null, i8* (i8*)* nonnull @foo, i8* noalias nocapture nofree readnone align 536870912 null)
-; IS__CGSCC_NPM-NEXT:    [[CALL1:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias nocapture align 536870912 null, i8* (i8*)* nonnull @bar, i8* noalias nofree nonnull readnone align 8 dereferenceable(8) bitcast (i8** @GlobalVPtr to i8*))
-; IS__CGSCC_NPM-NEXT:    [[CALL2:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias nocapture align 536870912 null, i8* (i8*)* nonnull @baz, i8* noalias nocapture nofree nonnull readnone align 8 dereferenceable(1) [[ALLOC1]])
-; IS__CGSCC_NPM-NEXT:    [[CALL3:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias nocapture align 536870912 null, i8* (i8*)* nonnull @buz, i8* noalias nofree nonnull readnone align 8 dereferenceable(1) [[ALLOC2]])
-; IS__CGSCC_NPM-NEXT:    ret i32 0
+; IS__CGSCC____-LABEL: define {{[^@]+}}@main()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[ALLOC1:%.*]] = alloca i8, align 8
+; IS__CGSCC____-NEXT:    [[ALLOC2:%.*]] = alloca i8, align 8
+; IS__CGSCC____-NEXT:    [[THREAD:%.*]] = alloca i64, align 8
+; IS__CGSCC____-NEXT:    [[CALL:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias nocapture align 536870912 null, i8* (i8*)* nonnull @foo, i8* noalias nocapture nofree readnone align 536870912 null)
+; IS__CGSCC____-NEXT:    [[CALL1:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias nocapture align 536870912 null, i8* (i8*)* nonnull @bar, i8* noalias nofree nonnull readnone align 8 dereferenceable(8) bitcast (i8** @GlobalVPtr to i8*))
+; IS__CGSCC____-NEXT:    [[CALL2:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias nocapture align 536870912 null, i8* (i8*)* nonnull @baz, i8* noalias nocapture nofree nonnull readnone align 8 dereferenceable(1) [[ALLOC1]])
+; IS__CGSCC____-NEXT:    [[CALL3:%.*]] = call i32 @pthread_create(i64* nonnull align 8 dereferenceable(8) [[THREAD]], %union.pthread_attr_t* noalias nocapture align 536870912 null, i8* (i8*)* nonnull @buz, i8* noalias nofree nonnull readnone align 8 dereferenceable(1) [[ALLOC2]])
+; IS__CGSCC____-NEXT:    ret i32 0
 ;
 entry:
   %alloc1 = alloca i8, align 8
@@ -93,15 +82,10 @@ define internal i8* @bar(i8* %arg) {
 ; IS__TUNIT____-NEXT:  entry:
 ; IS__TUNIT____-NEXT:    ret i8* bitcast (i8** @GlobalVPtr to i8*)
 ;
-; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@bar
-; IS__CGSCC_OPM-SAME: (i8* nofree nonnull readnone returned align 8 dereferenceable(8) "no-capture-maybe-returned" [[ARG:%.*]])
-; IS__CGSCC_OPM-NEXT:  entry:
-; IS__CGSCC_OPM-NEXT:    ret i8* bitcast (i8** @GlobalVPtr to i8*)
-;
-; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@bar
-; IS__CGSCC_NPM-SAME: (i8* nofree readnone returned "no-capture-maybe-returned" [[ARG:%.*]])
-; IS__CGSCC_NPM-NEXT:  entry:
-; IS__CGSCC_NPM-NEXT:    ret i8* bitcast (i8** @GlobalVPtr to i8*)
+; IS__CGSCC____-LABEL: define {{[^@]+}}@bar
+; IS__CGSCC____-SAME: (i8* nofree readnone returned "no-capture-maybe-returned" [[ARG:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret i8* bitcast (i8** @GlobalVPtr to i8*)
 ;
 entry:
   ret i8* %arg
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll b/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll
index e514f43967768..040be04ec3eee 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll
@@ -65,7 +65,7 @@ define internal { i32, i32 } @foo(i32 %A, i32 %B) {
 define void @caller(i1 %C) personality i32 (...)* @__gxx_personality_v0 {
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@caller
 ; IS__TUNIT____-SAME: (i1 [[C:%.*]]) #2 personality i32 (...)* @__gxx_personality_v0
-; IS__TUNIT____-NEXT:    [[Q:%.*]] = alloca i32
+; IS__TUNIT____-NEXT:    [[Q:%.*]] = alloca i32, align 4
 ; IS__TUNIT____-NEXT:    [[W:%.*]] = call align 4 i32* @incdec(i1 [[C]], i32* noalias nofree nonnull align 4 dereferenceable(4) "no-capture-maybe-returned" [[Q]])
 ; IS__TUNIT____-NEXT:    [[S1:%.*]] = call { i32, i32 } @foo(i32 1, i32 2)
 ; IS__TUNIT____-NEXT:    [[X1:%.*]] = extractvalue { i32, i32 } [[S1]], 0
@@ -83,7 +83,7 @@ define void @caller(i1 %C) personality i32 (...)* @__gxx_personality_v0 {
 ;
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@caller
 ; IS__CGSCC____-SAME: (i1 [[C:%.*]]) #1 personality i32 (...)* @__gxx_personality_v0
-; IS__CGSCC____-NEXT:    [[Q:%.*]] = alloca i32
+; IS__CGSCC____-NEXT:    [[Q:%.*]] = alloca i32, align 4
 ; IS__CGSCC____-NEXT:    [[W:%.*]] = call align 4 i32* @incdec(i1 [[C]], i32* noalias nofree nonnull align 4 dereferenceable(4) [[Q]])
 ; IS__CGSCC____-NEXT:    [[S1:%.*]] = call { i32, i32 } @foo(i32 1, i32 2)
 ; IS__CGSCC____-NEXT:    [[X1:%.*]] = extractvalue { i32, i32 } [[S1]], 0
diff --git a/llvm/test/Transforms/Attributor/allow_list.ll b/llvm/test/Transforms/Attributor/allow_list.ll
new file mode 100644
index 0000000000000..7670090cb03b7
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/allow_list.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S -passes=attributor --attributor-seed-allow-list asd < %s | FileCheck %s --check-prefixes=CHECK_DISABLED
+; RUN: opt -S -passes=attributor --attributor-seed-allow-list AAValueSimplify < %s | FileCheck %s --check-prefixes=CHECK_ENABLED
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: nounwind uwtable
+define internal i32 @range_test(i32 %a) #0 {
+; CHECK_DISABLED-LABEL: define {{[^@]+}}@range_test
+; CHECK_DISABLED-SAME: (i32 [[A:%.*]])
+; CHECK_DISABLED-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[A]], 100
+; CHECK_DISABLED-NEXT:    [[TMP2:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK_DISABLED-NEXT:    ret i32 [[TMP2]]
+;
+  %1 = icmp sgt i32 %a, 100
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+; Function Attrs: nounwind uwtable
+define i32 @range_use() #0 {
+; CHECK_DISABLED-LABEL: define {{[^@]+}}@range_use()
+; CHECK_DISABLED-NEXT:    [[TMP1:%.*]] = call i32 @range_test(i32 123)
+; CHECK_DISABLED-NEXT:    ret i32 [[TMP1]]
+;
+; CHECK_ENABLED-LABEL: define {{[^@]+}}@range_use()
+; CHECK_ENABLED-NEXT:    ret i32 1
+;
+  %1 = call i32 @range_test(i32 123)
+  ret i32 %1
+}
+
+attributes #0 = { nounwind uwtable noinline }
\ No newline at end of file
diff --git a/llvm/test/Transforms/Attributor/callbacks.ll b/llvm/test/Transforms/Attributor/callbacks.ll
index 849b83552da89..249b9fd0276e8 100644
--- a/llvm/test/Transforms/Attributor/callbacks.ll
+++ b/llvm/test/Transforms/Attributor/callbacks.ll
@@ -49,7 +49,7 @@ define void @t0_caller(i32* %a) {
 ; IS__CGSCC_OPM-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B]] to i8*
 ; IS__CGSCC_OPM-NEXT:    store i32 42, i32* [[B]], align 32
 ; IS__CGSCC_OPM-NEXT:    store i32* [[B]], i32** [[C]], align 64
-; IS__CGSCC_OPM-NEXT:    call void (i32*, i32*, void (i32*, i32*, ...)*, ...) @t0_callback_broker(i32* noalias nocapture align 536870912 null, i32* nonnull align 128 dereferenceable(4) [[PTR]], void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i32*, i64, i32**)* @t0_callback_callee to void (i32*, i32*, ...)*), i32* align 256 [[A]], i64 99, i32** nonnull align 64 dereferenceable(8) [[C]])
+; IS__CGSCC_OPM-NEXT:    call void (i32*, i32*, void (i32*, i32*, ...)*, ...) @t0_callback_broker(i32* noalias nocapture align 536870912 null, i32* nonnull align 128 dereferenceable(4) [[PTR]], void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i32*, i64, i32**)* @t0_callback_callee to void (i32*, i32*, ...)*), i32* align 256 [[A]], i64 99, i32** nocapture nonnull readonly align 64 dereferenceable(8) [[C]])
 ; IS__CGSCC_OPM-NEXT:    ret void
 ;
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@t0_caller
@@ -140,7 +140,7 @@ define void @t1_caller(i32* noalias %a) {
 ; IS__TUNIT_NPM-NEXT:    ret void
 ;
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@t1_caller
-; IS__CGSCC_OPM-SAME: (i32* noalias align 256 [[A:%.*]])
+; IS__CGSCC_OPM-SAME: (i32* noalias nocapture align 256 [[A:%.*]])
 ; IS__CGSCC_OPM-NEXT:  entry:
 ; IS__CGSCC_OPM-NEXT:    [[B:%.*]] = alloca i32, align 32
 ; IS__CGSCC_OPM-NEXT:    [[C:%.*]] = alloca i32*, align 64
@@ -148,7 +148,7 @@ define void @t1_caller(i32* noalias %a) {
 ; IS__CGSCC_OPM-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B]] to i8*
 ; IS__CGSCC_OPM-NEXT:    store i32 42, i32* [[B]], align 32
 ; IS__CGSCC_OPM-NEXT:    store i32* [[B]], i32** [[C]], align 64
-; IS__CGSCC_OPM-NEXT:    call void (i32*, i32*, void (i32*, i32*, ...)*, ...) @t1_callback_broker(i32* noalias nocapture align 536870912 null, i32* nocapture nonnull align 128 dereferenceable(4) [[PTR]], void (i32*, i32*, ...)* nocapture bitcast (void (i32*, i32*, i32*, i64, i32**)* @t1_callback_callee to void (i32*, i32*, ...)*), i32* align 256 [[A]], i64 99, i32** nonnull align 64 dereferenceable(8) [[C]])
+; IS__CGSCC_OPM-NEXT:    call void (i32*, i32*, void (i32*, i32*, ...)*, ...) @t1_callback_broker(i32* noalias nocapture align 536870912 null, i32* nocapture nonnull align 128 dereferenceable(4) [[PTR]], void (i32*, i32*, ...)* nocapture bitcast (void (i32*, i32*, i32*, i64, i32**)* @t1_callback_callee to void (i32*, i32*, ...)*), i32* nocapture align 256 [[A]], i64 99, i32** nocapture nonnull readonly align 64 dereferenceable(8) [[C]])
 ; IS__CGSCC_OPM-NEXT:    ret void
 ;
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@t1_caller
@@ -238,7 +238,7 @@ define void @t2_caller(i32* noalias %a) {
 ; IS__TUNIT_NPM-NEXT:    ret void
 ;
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@t2_caller
-; IS__CGSCC_OPM-SAME: (i32* noalias align 256 [[A:%.*]])
+; IS__CGSCC_OPM-SAME: (i32* noalias nocapture align 256 [[A:%.*]])
 ; IS__CGSCC_OPM-NEXT:  entry:
 ; IS__CGSCC_OPM-NEXT:    [[B:%.*]] = alloca i32, align 32
 ; IS__CGSCC_OPM-NEXT:    [[C:%.*]] = alloca i32*, align 64
@@ -246,7 +246,7 @@ define void @t2_caller(i32* noalias %a) {
 ; IS__CGSCC_OPM-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B]] to i8*
 ; IS__CGSCC_OPM-NEXT:    store i32 42, i32* [[B]], align 32
 ; IS__CGSCC_OPM-NEXT:    store i32* [[B]], i32** [[C]], align 64
-; IS__CGSCC_OPM-NEXT:    call void (i32*, i32*, void (i32*, i32*, ...)*, ...) @t2_callback_broker(i32* noalias nocapture align 536870912 null, i32* nocapture nonnull align 128 dereferenceable(4) [[PTR]], void (i32*, i32*, ...)* nocapture bitcast (void (i32*, i32*, i32*, i64, i32**)* @t2_callback_callee to void (i32*, i32*, ...)*), i32* align 256 [[A]], i64 99, i32** nonnull align 64 dereferenceable(8) [[C]])
+; IS__CGSCC_OPM-NEXT:    call void (i32*, i32*, void (i32*, i32*, ...)*, ...) @t2_callback_broker(i32* noalias nocapture align 536870912 null, i32* nocapture nonnull align 128 dereferenceable(4) [[PTR]], void (i32*, i32*, ...)* nocapture bitcast (void (i32*, i32*, i32*, i64, i32**)* @t2_callback_callee to void (i32*, i32*, ...)*), i32* nocapture align 256 [[A]], i64 99, i32** nocapture nonnull readonly align 64 dereferenceable(8) [[C]])
 ; IS__CGSCC_OPM-NEXT:    ret void
 ;
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@t2_caller
@@ -340,7 +340,7 @@ define void @t3_caller(i32* noalias %a) {
 ; IS__TUNIT_NPM-NEXT:    ret void
 ;
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@t3_caller
-; IS__CGSCC_OPM-SAME: (i32* noalias align 256 [[A:%.*]])
+; IS__CGSCC_OPM-SAME: (i32* noalias nocapture align 256 [[A:%.*]])
 ; IS__CGSCC_OPM-NEXT:  entry:
 ; IS__CGSCC_OPM-NEXT:    [[B:%.*]] = alloca i32, align 32
 ; IS__CGSCC_OPM-NEXT:    [[C:%.*]] = alloca i32*, align 64
@@ -348,8 +348,8 @@ define void @t3_caller(i32* noalias %a) {
 ; IS__CGSCC_OPM-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B]] to i8*
 ; IS__CGSCC_OPM-NEXT:    store i32 42, i32* [[B]], align 32
 ; IS__CGSCC_OPM-NEXT:    store i32* [[B]], i32** [[C]], align 64
-; IS__CGSCC_OPM-NEXT:    call void (i32*, i32*, void (i32*, i32*, ...)*, ...) @t3_callback_broker(i32* noalias nocapture align 536870912 null, i32* nocapture nonnull align 128 dereferenceable(4) [[PTR]], void (i32*, i32*, ...)* nocapture bitcast (void (i32*, i32*, i32*, i64, i32**)* @t3_callback_callee to void (i32*, i32*, ...)*), i32* align 256 [[A]], i64 99, i32** nonnull align 64 dereferenceable(8) [[C]])
-; IS__CGSCC_OPM-NEXT:    call void (i32*, i32*, void (i32*, i32*, ...)*, ...) @t3_callback_broker(i32* noalias nocapture align 536870912 null, i32* nocapture nonnull align 128 dereferenceable(4) [[PTR]], void (i32*, i32*, ...)* nocapture bitcast (void (i32*, i32*, i32*, i64, i32**)* @t3_callback_callee to void (i32*, i32*, ...)*), i32* align 256 [[A]], i64 99, i32** nonnull align 64 dereferenceable(8) [[C]])
+; IS__CGSCC_OPM-NEXT:    call void (i32*, i32*, void (i32*, i32*, ...)*, ...) @t3_callback_broker(i32* noalias nocapture align 536870912 null, i32* nocapture nonnull align 128 dereferenceable(4) [[PTR]], void (i32*, i32*, ...)* nocapture bitcast (void (i32*, i32*, i32*, i64, i32**)* @t3_callback_callee to void (i32*, i32*, ...)*), i32* nocapture align 256 [[A]], i64 99, i32** nocapture nonnull readonly align 64 dereferenceable(8) [[C]])
+; IS__CGSCC_OPM-NEXT:    call void (i32*, i32*, void (i32*, i32*, ...)*, ...) @t3_callback_broker(i32* noalias nocapture align 536870912 null, i32* nocapture nonnull align 128 dereferenceable(4) [[PTR]], void (i32*, i32*, ...)* nocapture bitcast (void (i32*, i32*, i32*, i64, i32**)* @t3_callback_callee to void (i32*, i32*, ...)*), i32* nocapture align 256 [[A]], i64 99, i32** nocapture nonnull readonly align 64 dereferenceable(8) [[C]])
 ; IS__CGSCC_OPM-NEXT:    ret void
 ;
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@t3_caller
diff --git a/llvm/test/Transforms/Attributor/heap_to_stack.ll b/llvm/test/Transforms/Attributor/heap_to_stack.ll
index 140a5ebac0194..909cda54fc868 100644
--- a/llvm/test/Transforms/Attributor/heap_to_stack.ll
+++ b/llvm/test/Transforms/Attributor/heap_to_stack.ll
@@ -82,7 +82,7 @@ define void @test3() {
 ; IS________OPM-NEXT:    ret void
 ;
 ; IS________NPM-LABEL: define {{[^@]+}}@test3()
-; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 4
+; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 4, align 1
 ; IS________NPM-NEXT:    tail call void @no_sync_func(i8* noalias nocapture nofree [[TMP1]])
 ; IS________NPM-NEXT:    ret void
 ;
@@ -102,7 +102,7 @@ define void @test3a(i8* %p) {
 ;
 ; IS________NPM-LABEL: define {{[^@]+}}@test3a
 ; IS________NPM-SAME: (i8* nocapture [[P:%.*]])
-; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 4
+; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 4, align 1
 ; IS________NPM-NEXT:    tail call void @nofree_arg_only(i8* noalias nocapture nofree [[TMP1]], i8* nocapture [[P]])
 ; IS________NPM-NEXT:    ret void
 ;
@@ -157,7 +157,7 @@ define void @test0() {
 ; IS________OPM-NEXT:    ret void
 ;
 ; IS________NPM-LABEL: define {{[^@]+}}@test0()
-; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 8
+; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 8, align 1
 ; IS________NPM-NEXT:    [[CALLOC_BC:%.*]] = bitcast i8* [[TMP1]] to i8*
 ; IS________NPM-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[CALLOC_BC]], i8 0, i64 8, i1 false)
 ; IS________NPM-NEXT:    tail call void @no_sync_func(i8* noalias nocapture nofree [[TMP1]])
@@ -177,7 +177,7 @@ define void @test4() {
 ; IS________OPM-NEXT:    ret void
 ;
 ; IS________NPM-LABEL: define {{[^@]+}}@test4()
-; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 4
+; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 4, align 1
 ; IS________NPM-NEXT:    tail call void @nofree_func(i8* noalias nocapture nofree [[TMP1]])
 ; IS________NPM-NEXT:    ret void
 ;
@@ -207,7 +207,7 @@ define void @test5(i32, i8* %p) {
 ;
 ; IS________NPM-LABEL: define {{[^@]+}}@test5
 ; IS________NPM-SAME: (i32 [[TMP0:%.*]], i8* nocapture [[P:%.*]])
-; IS________NPM-NEXT:    [[TMP2:%.*]] = alloca i8, i64 4
+; IS________NPM-NEXT:    [[TMP2:%.*]] = alloca i8, i64 4, align 1
 ; IS________NPM-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP0]], 0
 ; IS________NPM-NEXT:    br i1 [[TMP3]], label [[TMP5:%.*]], label [[TMP4:%.*]]
 ; IS________NPM:       4:
@@ -256,7 +256,7 @@ define void @test6(i32) {
 ;
 ; IS________NPM-LABEL: define {{[^@]+}}@test6
 ; IS________NPM-SAME: (i32 [[TMP0:%.*]])
-; IS________NPM-NEXT:    [[TMP2:%.*]] = alloca i8, i64 4
+; IS________NPM-NEXT:    [[TMP2:%.*]] = alloca i8, i64 4, align 1
 ; IS________NPM-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP0]], 0
 ; IS________NPM-NEXT:    br i1 [[TMP3]], label [[TMP5:%.*]], label [[TMP4:%.*]]
 ; IS________NPM:       4:
@@ -293,7 +293,7 @@ define void @test7() {
 ; IS________OPM-NEXT:    unreachable
 ;
 ; IS________NPM-LABEL: define {{[^@]+}}@test7()
-; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 4
+; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 4, align 1
 ; IS________NPM-NEXT:    [[TMP2:%.*]] = tail call i32 @no_return_call()
 ; IS________NPM-NEXT:    unreachable
 ;
@@ -359,7 +359,7 @@ define i32 @test10() {
 ; IS________OPM-NEXT:    ret i32 [[TMP3]]
 ;
 ; IS________NPM-LABEL: define {{[^@]+}}@test10()
-; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 4
+; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 4, align 1
 ; IS________NPM-NEXT:    tail call void @no_sync_func(i8* noalias nocapture nofree [[TMP1]])
 ; IS________NPM-NEXT:    [[TMP2:%.*]] = bitcast i8* [[TMP1]] to i32*
 ; IS________NPM-NEXT:    store i32 10, i32* [[TMP2]], align 4
@@ -387,7 +387,7 @@ define i32 @test_lifetime() {
 ; IS________OPM-NEXT:    ret i32 [[TMP3]]
 ;
 ; IS________NPM-LABEL: define {{[^@]+}}@test_lifetime()
-; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 4
+; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 4, align 1
 ; IS________NPM-NEXT:    tail call void @no_sync_func(i8* noalias nocapture nofree [[TMP1]])
 ; IS________NPM-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* noalias nocapture nonnull align 4 dereferenceable(4) [[TMP1]])
 ; IS________NPM-NEXT:    [[TMP2:%.*]] = bitcast i8* [[TMP1]] to i32*
@@ -455,7 +455,7 @@ define i32 @irreducible_cfg(i32 %0) {
 ;
 ; IS________NPM-LABEL: define {{[^@]+}}@irreducible_cfg
 ; IS________NPM-SAME: (i32 [[TMP0:%.*]])
-; IS________NPM-NEXT:    [[TMP2:%.*]] = alloca i8, i64 4
+; IS________NPM-NEXT:    [[TMP2:%.*]] = alloca i8, i64 4, align 1
 ; IS________NPM-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP2]] to i32*
 ; IS________NPM-NEXT:    store i32 10, i32* [[TMP3]], align 4
 ; IS________NPM-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP0]], 1
@@ -555,7 +555,7 @@ define i32 @malloc_in_loop(i32 %0) {
 ; IS________NPM-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP6]], 0
 ; IS________NPM-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP11:%.*]]
 ; IS________NPM:       8:
-; IS________NPM-NEXT:    [[TMP9:%.*]] = alloca i8, i64 4
+; IS________NPM-NEXT:    [[TMP9:%.*]] = alloca i8, i64 4, align 1
 ; IS________NPM-NEXT:    [[TMP10:%.*]] = bitcast i8* [[TMP9]] to i32*
 ; IS________NPM-NEXT:    store i32 1, i32* [[TMP10]], align 8
 ; IS________NPM-NEXT:    br label [[TMP4]]
@@ -680,7 +680,7 @@ define void @test16a(i8 %v, i8** %P) {
 ;
 ; IS________NPM-LABEL: define {{[^@]+}}@test16a
 ; IS________NPM-SAME: (i8 [[V:%.*]], i8** nocapture nofree readnone [[P:%.*]])
-; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 4
+; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 4, align 1
 ; IS________NPM-NEXT:    store i8 [[V]], i8* [[TMP1]], align 1
 ; IS________NPM-NEXT:    tail call void @no_sync_func(i8* noalias nocapture nofree nonnull dereferenceable(1) [[TMP1]])
 ; IS________NPM-NEXT:    ret void
diff --git a/llvm/test/Transforms/Attributor/internal-noalias.ll b/llvm/test/Transforms/Attributor/internal-noalias.ll
index 2605e46cb243f..182cadf48a0b0 100644
--- a/llvm/test/Transforms/Attributor/internal-noalias.ll
+++ b/llvm/test/Transforms/Attributor/internal-noalias.ll
@@ -123,10 +123,10 @@ define internal i32 @noalias_args_argmem_ro(i32* %A, i32* %B) #1 {
 ;
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@noalias_args_argmem_ro
 ; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]])
-; IS__TUNIT_NPM-NEXT:    [[B_PRIV:%.*]] = alloca i32
-; IS__TUNIT_NPM-NEXT:    store i32 [[TMP1]], i32* [[B_PRIV]]
-; IS__TUNIT_NPM-NEXT:    [[A_PRIV:%.*]] = alloca i32
-; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[A_PRIV]]
+; IS__TUNIT_NPM-NEXT:    [[B_PRIV:%.*]] = alloca i32, align 4
+; IS__TUNIT_NPM-NEXT:    store i32 [[TMP1]], i32* [[B_PRIV]], align 4
+; IS__TUNIT_NPM-NEXT:    [[A_PRIV:%.*]] = alloca i32, align 4
+; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[A_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[T0:%.*]] = load i32, i32* [[A_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[T1:%.*]] = load i32, i32* [[B_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[ADD:%.*]] = add nsw i32 [[T0]], [[T1]]
diff --git a/llvm/test/Transforms/Attributor/liveness.ll b/llvm/test/Transforms/Attributor/liveness.ll
index bab313465e51a..1ce3ff007092b 100644
--- a/llvm/test/Transforms/Attributor/liveness.ll
+++ b/llvm/test/Transforms/Attributor/liveness.ll
@@ -1778,7 +1778,7 @@ indirectgoto:                                     ; preds = %lab0, %entry
 define i32 @main() {
 ; CHECK-LABEL: define {{[^@]+}}@main()
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[F:%.*]] = alloca i32
+; CHECK-NEXT:    [[F:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    br label [[FOR_COND_0:%.*]]
 ; CHECK:       for.cond.0:
 ; CHECK-NEXT:    [[G_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY_0:%.*]] ]
@@ -1856,8 +1856,8 @@ define i32 @h(i32 %i) {
 define void @bad_gep() {
 ; CHECK-LABEL: define {{[^@]+}}@bad_gep()
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[N:%.*]] = alloca i8
-; CHECK-NEXT:    [[M:%.*]] = alloca i8
+; CHECK-NEXT:    [[N:%.*]] = alloca i8, align 1
+; CHECK-NEXT:    [[M:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 1, i8* noalias nocapture nonnull dereferenceable(1) [[N]])
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       while.body:
diff --git a/llvm/test/Transforms/Attributor/memory_locations.ll b/llvm/test/Transforms/Attributor/memory_locations.ll
index 3e6facb445776..42dc2e51be3a6 100644
--- a/llvm/test/Transforms/Attributor/memory_locations.ll
+++ b/llvm/test/Transforms/Attributor/memory_locations.ll
@@ -325,7 +325,7 @@ define void @callerA2(i8* %arg) {
 ; CHECK: Function Attrs: readnone
 define void @callerB1() {
 ; CHECK-LABEL: define {{[^@]+}}@callerB1()
-; CHECK-NEXT:    [[STACK:%.*]] = alloca i8
+; CHECK-NEXT:    [[STACK:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @argmem_only(i8* nonnull dereferenceable(1) [[STACK]])
 ; CHECK-NEXT:    ret void
 ;
@@ -336,7 +336,7 @@ define void @callerB1() {
 ; CHECK: Function Attrs: inaccessiblememonly
 define void @callerB2() {
 ; CHECK-LABEL: define {{[^@]+}}@callerB2()
-; CHECK-NEXT:    [[STACK:%.*]] = alloca i8
+; CHECK-NEXT:    [[STACK:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @inaccesible_argmem_only_decl(i8* nonnull dereferenceable(1) [[STACK]])
 ; CHECK-NEXT:    ret void
 ;
@@ -476,7 +476,7 @@ define void @writeonly_global_via_arg_internal() {
 define i8 @recursive_not_readnone(i8* %ptr, i1 %c) {
 ; CHECK-LABEL: define {{[^@]+}}@recursive_not_readnone
 ; CHECK-SAME: (i8* nocapture nofree writeonly [[PTR:%.*]], i1 [[C:%.*]])
-; CHECK-NEXT:    [[ALLOC:%.*]] = alloca i8
+; CHECK-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; CHECK:       t:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone(i8* noalias nocapture nofree nonnull writeonly dereferenceable(1) [[ALLOC]], i1 false)
@@ -504,7 +504,7 @@ f:
 define internal i8 @recursive_not_readnone_internal(i8* %ptr, i1 %c) {
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@recursive_not_readnone_internal
 ; IS__TUNIT____-SAME: (i8* noalias nocapture nofree nonnull writeonly dereferenceable(1) [[PTR:%.*]], i1 [[C:%.*]])
-; IS__TUNIT____-NEXT:    [[ALLOC:%.*]] = alloca i8
+; IS__TUNIT____-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
 ; IS__TUNIT____-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; IS__TUNIT____:       t:
 ; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone_internal(i8* noalias nocapture nofree nonnull writeonly dereferenceable(1) [[ALLOC]], i1 false)
@@ -516,7 +516,7 @@ define internal i8 @recursive_not_readnone_internal(i8* %ptr, i1 %c) {
 ;
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@recursive_not_readnone_internal
 ; IS__CGSCC____-SAME: (i8* nocapture nofree nonnull writeonly dereferenceable(1) [[PTR:%.*]], i1 [[C:%.*]])
-; IS__CGSCC____-NEXT:    [[ALLOC:%.*]] = alloca i8
+; IS__CGSCC____-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
 ; IS__CGSCC____-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; IS__CGSCC____:       t:
 ; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone_internal(i8* noalias nocapture nofree nonnull writeonly dereferenceable(1) [[ALLOC]], i1 false)
@@ -542,7 +542,7 @@ f:
 define i8 @readnone_caller(i1 %c) {
 ; CHECK-LABEL: define {{[^@]+}}@readnone_caller
 ; CHECK-SAME: (i1 [[C:%.*]])
-; CHECK-NEXT:    [[A:%.*]] = alloca i8
+; CHECK-NEXT:    [[A:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[R:%.*]] = call i8 @recursive_not_readnone_internal(i8* noalias nocapture nofree nonnull writeonly dereferenceable(1) [[A]], i1 [[C]])
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
@@ -558,7 +558,7 @@ define i8 @readnone_caller(i1 %c) {
 define internal i8 @recursive_not_readnone_internal2(i8* %ptr, i1 %c) {
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@recursive_not_readnone_internal2
 ; IS__TUNIT____-SAME: (i8* noalias nocapture nofree nonnull writeonly [[PTR:%.*]], i1 [[C:%.*]])
-; IS__TUNIT____-NEXT:    [[ALLOC:%.*]] = alloca i8
+; IS__TUNIT____-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
 ; IS__TUNIT____-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; IS__TUNIT____:       t:
 ; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone_internal2(i8* noalias nocapture nofree nonnull writeonly dereferenceable(1) [[ALLOC]], i1 false)
@@ -570,7 +570,7 @@ define internal i8 @recursive_not_readnone_internal2(i8* %ptr, i1 %c) {
 ;
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@recursive_not_readnone_internal2
 ; IS__CGSCC____-SAME: (i8* nocapture nofree nonnull writeonly [[PTR:%.*]], i1 [[C:%.*]])
-; IS__CGSCC____-NEXT:    [[ALLOC:%.*]] = alloca i8
+; IS__CGSCC____-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
 ; IS__CGSCC____-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
 ; IS__CGSCC____:       t:
 ; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = call i8 @recursive_not_readnone_internal2(i8* noalias nocapture nofree nonnull writeonly dereferenceable(1) [[ALLOC]], i1 false)
diff --git a/llvm/test/Transforms/Attributor/misc_crash.ll b/llvm/test/Transforms/Attributor/misc_crash.ll
index d4cfe681c5d0d..f8e10fa445454 100644
--- a/llvm/test/Transforms/Attributor/misc_crash.ll
+++ b/llvm/test/Transforms/Attributor/misc_crash.ll
@@ -87,7 +87,7 @@ define void @func4() {
 
 define internal void @func5(i32 %0) {
 ; CHECK-LABEL: define {{[^@]+}}@func5()
-; CHECK-NEXT:    [[TMP:%.*]] = alloca i8*
+; CHECK-NEXT:    [[TMP:%.*]] = alloca i8*, align 8
 ; CHECK-NEXT:    br label [[BLOCK:%.*]]
 ; CHECK:       block:
 ; CHECK-NEXT:    store i8* blockaddress(@func5, [[BLOCK]]), i8** [[TMP]], align 8
diff --git a/llvm/test/Transforms/Attributor/range.ll b/llvm/test/Transforms/Attributor/range.ll
index 03338b4ce4999..f105bb3fad0e1 100644
--- a/llvm/test/Transforms/Attributor/range.ll
+++ b/llvm/test/Transforms/Attributor/range.ll
@@ -1063,6 +1063,71 @@ end:
 }
 
 
+define i32 @func(i1 %c) {
+; CHECK-LABEL: define {{[^@]+}}@func
+; CHECK-SAME: (i1 [[C:%.*]])
+; CHECK-NEXT:    [[RET:%.*]] = select i1 [[C]], i32 0, i32 1
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %ret = select i1 %c, i32 0, i32 1
+  ret i32 %ret
+}
+
+define i32 @simplify_callsite_argument(i1 %d) {
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@simplify_callsite_argument
+; IS__TUNIT_OPM-SAME: (i1 [[D:%.*]])
+; IS__TUNIT_OPM-NEXT:    [[C:%.*]] = select i1 [[D]], i1 true, i1 false
+; IS__TUNIT_OPM-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
+; IS__TUNIT_OPM:       t:
+; IS__TUNIT_OPM-NEXT:    [[RET1:%.*]] = call i32 @func(i1 [[C]]) #2, !range !3
+; IS__TUNIT_OPM-NEXT:    ret i32 [[RET1]]
+; IS__TUNIT_OPM:       f:
+; IS__TUNIT_OPM-NEXT:    [[RET2:%.*]] = call i32 @func(i1 false) #2, !range !3
+; IS__TUNIT_OPM-NEXT:    ret i32 [[RET2]]
+;
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@simplify_callsite_argument
+; IS__TUNIT_NPM-SAME: (i1 [[D:%.*]])
+; IS__TUNIT_NPM-NEXT:    [[C:%.*]] = select i1 [[D]], i1 true, i1 false
+; IS__TUNIT_NPM-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
+; IS__TUNIT_NPM:       t:
+; IS__TUNIT_NPM-NEXT:    [[RET1:%.*]] = call i32 @func(i1 true) #1, !range !4
+; IS__TUNIT_NPM-NEXT:    ret i32 [[RET1]]
+; IS__TUNIT_NPM:       f:
+; IS__TUNIT_NPM-NEXT:    [[RET2:%.*]] = call i32 @func(i1 false) #1, !range !4
+; IS__TUNIT_NPM-NEXT:    ret i32 [[RET2]]
+;
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@simplify_callsite_argument
+; IS__CGSCC_OPM-SAME: (i1 [[D:%.*]])
+; IS__CGSCC_OPM-NEXT:    [[C:%.*]] = select i1 [[D]], i1 true, i1 false
+; IS__CGSCC_OPM-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
+; IS__CGSCC_OPM:       t:
+; IS__CGSCC_OPM-NEXT:    [[RET1:%.*]] = call i32 @func(i1 [[C]])
+; IS__CGSCC_OPM-NEXT:    ret i32 [[RET1]]
+; IS__CGSCC_OPM:       f:
+; IS__CGSCC_OPM-NEXT:    [[RET2:%.*]] = call i32 @func(i1 false)
+; IS__CGSCC_OPM-NEXT:    ret i32 [[RET2]]
+;
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@simplify_callsite_argument
+; IS__CGSCC_NPM-SAME: (i1 [[D:%.*]])
+; IS__CGSCC_NPM-NEXT:    [[C:%.*]] = select i1 [[D]], i1 true, i1 false
+; IS__CGSCC_NPM-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
+; IS__CGSCC_NPM:       t:
+; IS__CGSCC_NPM-NEXT:    [[RET1:%.*]] = call i32 @func(i1 true)
+; IS__CGSCC_NPM-NEXT:    ret i32 [[RET1]]
+; IS__CGSCC_NPM:       f:
+; IS__CGSCC_NPM-NEXT:    [[RET2:%.*]] = call i32 @func(i1 false)
+; IS__CGSCC_NPM-NEXT:    ret i32 [[RET2]]
+;
+  %c = select i1 %d, i1 true, i1 false
+  br i1 %c, label %t, label %f
+t:
+  %ret1 = call i32 @func(i1 %c)
+  ret i32 %ret1
+f:
+  %ret2 = call i32 @func(i1 false)
+  ret i32 %ret2
+}
+
 !0 = !{i32 0, i32 10}
 !1 = !{i32 10, i32 100}
 
diff --git a/llvm/test/Transforms/Attributor/readattrs.ll b/llvm/test/Transforms/Attributor/readattrs.ll
index 1a7301e3c1b62..a78ffb0653b69 100644
--- a/llvm/test/Transforms/Attributor/readattrs.ll
+++ b/llvm/test/Transforms/Attributor/readattrs.ll
@@ -195,7 +195,7 @@ declare void @escape_readonly_ptr(i8** %addr, i8* readonly %ptr)
 define void @unsound_readnone(i8* %ignored, i8* %escaped_then_written) {
 ; CHECK-LABEL: define {{[^@]+}}@unsound_readnone
 ; CHECK-SAME: (i8* nocapture nofree readnone [[IGNORED:%.*]], i8* [[ESCAPED_THEN_WRITTEN:%.*]])
-; CHECK-NEXT:    [[ADDR:%.*]] = alloca i8*
+; CHECK-NEXT:    [[ADDR:%.*]] = alloca i8*, align 8
 ; CHECK-NEXT:    call void @escape_readnone_ptr(i8** nonnull align 8 dereferenceable(8) [[ADDR]], i8* noalias readnone [[ESCAPED_THEN_WRITTEN]])
 ; CHECK-NEXT:    [[ADDR_LD:%.*]] = load i8*, i8** [[ADDR]], align 8
 ; CHECK-NEXT:    store i8 0, i8* [[ADDR_LD]], align 1
@@ -211,7 +211,7 @@ define void @unsound_readnone(i8* %ignored, i8* %escaped_then_written) {
 define void @unsound_readonly(i8* %ignored, i8* %escaped_then_written) {
 ; CHECK-LABEL: define {{[^@]+}}@unsound_readonly
 ; CHECK-SAME: (i8* nocapture nofree readnone [[IGNORED:%.*]], i8* [[ESCAPED_THEN_WRITTEN:%.*]])
-; CHECK-NEXT:    [[ADDR:%.*]] = alloca i8*
+; CHECK-NEXT:    [[ADDR:%.*]] = alloca i8*, align 8
 ; CHECK-NEXT:    call void @escape_readonly_ptr(i8** nonnull align 8 dereferenceable(8) [[ADDR]], i8* readonly [[ESCAPED_THEN_WRITTEN]])
 ; CHECK-NEXT:    [[ADDR_LD:%.*]] = load i8*, i8** [[ADDR]], align 8
 ; CHECK-NEXT:    store i8 0, i8* [[ADDR_LD]], align 1
diff --git a/llvm/test/Transforms/Attributor/undefined_behavior.ll b/llvm/test/Transforms/Attributor/undefined_behavior.ll
index 8a2185f5adc7a..b49c26e8a2f7a 100644
--- a/llvm/test/Transforms/Attributor/undefined_behavior.ll
+++ b/llvm/test/Transforms/Attributor/undefined_behavior.ll
@@ -340,7 +340,7 @@ e:
 ; FIXME: Currently it doesn't propagate the undef.
 define i32 @cond_br_on_undef_uninit() {
 ; CHECK-LABEL: define {{[^@]+}}@cond_br_on_undef_uninit()
-; CHECK-NEXT:    [[ALLOC:%.*]] = alloca i1
+; CHECK-NEXT:    [[ALLOC:%.*]] = alloca i1, align 1
 ; CHECK-NEXT:    [[COND:%.*]] = load i1, i1* [[ALLOC]], align 1
 ; CHECK-NEXT:    br i1 [[COND]], label [[T:%.*]], label [[E:%.*]]
 ; CHECK:       t:
diff --git a/llvm/test/Transforms/Attributor/value-simplify.ll b/llvm/test/Transforms/Attributor/value-simplify.ll
index ac0fcaef16936..febfaba5b19a1 100644
--- a/llvm/test/Transforms/Attributor/value-simplify.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify.ll
@@ -290,10 +290,25 @@ define internal i32* @test_preallocated(i32* preallocated(i32) %a) {
   ret i32* %a
 }
 define i32* @complicated_args_preallocated() {
-; CHECK-LABEL: define {{[^@]+}}@complicated_args_preallocated()
-; CHECK-NEXT:    [[C:%.*]] = call token @llvm.call.preallocated.setup(i32 1)
-; CHECK-NEXT:    [[CALL:%.*]] = call i32* @test_preallocated(i32* noalias nocapture nofree writeonly preallocated(i32) align 536870912 null)
-; CHECK-NEXT:    ret i32* [[CALL]]
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@complicated_args_preallocated()
+; IS__TUNIT_OPM-NEXT:    [[C:%.*]] = call token @llvm.call.preallocated.setup(i32 1)
+; IS__TUNIT_OPM-NEXT:    [[CALL:%.*]] = call i32* @test_preallocated(i32* noalias nocapture nofree writeonly preallocated(i32) align 536870912 null) #5 [ "preallocated"(token [[C]]) ]
+; IS__TUNIT_OPM-NEXT:    ret i32* [[CALL]]
+;
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@complicated_args_preallocated()
+; IS__TUNIT_NPM-NEXT:    [[C:%.*]] = call token @llvm.call.preallocated.setup(i32 1)
+; IS__TUNIT_NPM-NEXT:    [[CALL:%.*]] = call i32* @test_preallocated(i32* noalias nocapture nofree writeonly preallocated(i32) align 536870912 null) #4 [ "preallocated"(token [[C]]) ]
+; IS__TUNIT_NPM-NEXT:    ret i32* [[CALL]]
+;
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@complicated_args_preallocated()
+; IS__CGSCC_OPM-NEXT:    [[C:%.*]] = call token @llvm.call.preallocated.setup(i32 1)
+; IS__CGSCC_OPM-NEXT:    [[CALL:%.*]] = call i32* @test_preallocated(i32* noalias nocapture nofree writeonly preallocated(i32) align 536870912 null) #6 [ "preallocated"(token [[C]]) ]
+; IS__CGSCC_OPM-NEXT:    ret i32* [[CALL]]
+;
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@complicated_args_preallocated()
+; IS__CGSCC_NPM-NEXT:    [[C:%.*]] = call token @llvm.call.preallocated.setup(i32 1)
+; IS__CGSCC_NPM-NEXT:    [[CALL:%.*]] = call i32* @test_preallocated(i32* noalias nocapture nofree writeonly preallocated(i32) align 536870912 null) #5 [ "preallocated"(token [[C]]) ]
+; IS__CGSCC_NPM-NEXT:    ret i32* [[CALL]]
 ;
   %c = call token @llvm.call.preallocated.setup(i32 1)
   %call = call i32* @test_preallocated(i32* preallocated(i32) null) ["preallocated"(token %c)]
diff --git a/llvm/test/Transforms/BDCE/sext_multi_uses.ll b/llvm/test/Transforms/BDCE/sext_multi_uses.ll
new file mode 100644
index 0000000000000..fa8549e0dec3b
--- /dev/null
+++ b/llvm/test/Transforms/BDCE/sext_multi_uses.ll
@@ -0,0 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -bdce < %s | FileCheck %s
+define i32 @ZEXT_0(i16 %a) {
+; CHECK-LABEL: @ZEXT_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXT1:%.*]] = zext i16 [[A:%.*]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[EXT1]], 65280
+; CHECK-NEXT:    [[LSR:%.*]] = lshr i32 [[EXT1]], 8
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[LSR]], 255
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[AND]], [[AND2]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+entry:
+  %ext = sext i16 %a to i32
+  %and = and i32 %ext, 65280
+  %lsr = lshr i32 %ext, 8
+  %and2 = and i32 %lsr, 255
+  %or = or i32 %and, %and2
+  ret i32 %or
+}
+
+define i32 @ZEXT_1(i16 %a) {
+; CHECK-LABEL: @ZEXT_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXT1:%.*]] = zext i16 [[A:%.*]] to i32
+; CHECK-NEXT:    [[LSR:%.*]] = lshr i32 [[EXT1]], 8
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[LSR]], 255
+; CHECK-NEXT:    [[AND:%.*]] = or i32 [[EXT1]], -65536
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[AND]], [[AND2]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+entry:
+  %ext = sext i16 %a to i32
+  %lsr = lshr i32 %ext, 8
+  %and2 = and i32 %lsr, 255
+  %and = or i32 %ext, 4294901760
+  %or = or i32 %and, %and2
+  ret i32 %or
+}
+
+define i16 @NOT_ZEXT_0(i16 %a) {
+; CHECK-LABEL: @NOT_ZEXT_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXT:%.*]] = sext i16 [[A:%.*]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[EXT]], 65280
+; CHECK-NEXT:    [[LSR:%.*]] = lshr i32 [[EXT]], 9
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[LSR]], 255
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[AND]], [[AND2]]
+; CHECK-NEXT:    [[RET:%.*]] = trunc i32 [[OR]] to i16
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+entry:
+  %ext = sext i16 %a to i32
+  %and = and i32 %ext, 65280
+  %lsr = lshr i32 %ext, 9
+  %and2 = and i32 %lsr, 255
+  %or = or i32 %and, %and2
+  %ret = trunc i32 %or to i16
+  ret i16 %ret
+}
+
+define i32 @NOT_ZEXT_1(i16 %a) {
+; CHECK-LABEL: @NOT_ZEXT_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXT:%.*]] = sext i16 [[A:%.*]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[EXT]], 85280
+; CHECK-NEXT:    [[LSR:%.*]] = lshr i32 [[EXT]], 8
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[LSR]], 255
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[AND]], [[AND2]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+entry:
+  %ext = sext i16 %a to i32
+  %and = and i32 %ext, 85280
+  %lsr = lshr i32 %ext, 8
+  %and2 = and i32 %lsr, 255
+  %or = or i32 %and, %and2
+  ret i32 %or
+}
+
+define i32 @NOT_ZEXT_2(i16 %a) {
+; CHECK-LABEL: @NOT_ZEXT_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXT:%.*]] = sext i16 [[A:%.*]] to i32
+; CHECK-NEXT:    [[LSR:%.*]] = lshr i32 [[EXT]], 8
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[LSR]], 255
+; CHECK-NEXT:    [[AND:%.*]] = xor i32 [[EXT]], -65536
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[AND]], [[AND2]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+entry:
+  %ext = sext i16 %a to i32
+  %lsr = lshr i32 %ext, 8
+  %and2 = and i32 %lsr, 255
+  %and = xor i32 %ext, 4294901760
+  %or = or i32 %and, %and2
+  ret i32 %or
+}
+
+define i16 @clear_assumptions(i8 %x, i16 %y) {
+; CHECK-LABEL: @clear_assumptions(
+; CHECK-NEXT:    [[EXT1:%.*]] = zext i8 [[X:%.*]] to i16
+; CHECK-NEXT:    [[ADD:%.*]] = add i16 [[EXT1]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i16 [[ADD]], 255
+; CHECK-NEXT:    ret i16 [[AND]]
+;
+  %ext = sext i8 %x to i16
+  %add = add nsw i16 %ext, %y
+  %and = and i16 %add, 255
+  ret i16 %and
+}
diff --git a/llvm/test/Transforms/Coroutines/coro-split-02.ll b/llvm/test/Transforms/Coroutines/coro-split-02.ll
index 993374291f415..6ab0015d1fb19 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-02.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-02.ll
@@ -31,6 +31,8 @@ await.ready:
   %val = load i32, i32* %Result.i19
   %cast = bitcast i32* %testval to i8*
   call void @llvm.lifetime.start.p0i8(i64 4, i8* %cast)
+  %test = load i32, i32* %testval
+  call void @print(i32 %test)
   call void @llvm.lifetime.end.p0i8(i64 4, i8*  %cast)
   call void @print(i32 %val)
   br label %exit
@@ -47,6 +49,8 @@ exit:
 ; CHECK-NEXT:    %val = load i32, i32* %Result
 ; CHECK-NEXT:    %cast = bitcast i32* %testval to i8*
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* %cast)
+; CHECK-NEXT:    %test = load i32, i32* %testval
+; CHECK-NEXT:    call void @print(i32 %test)
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 4, i8* %cast)
 ; CHECK-NEXT:    call void @print(i32 %val)
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-01.ll b/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-01.ll
new file mode 100644
index 0000000000000..9f9c1661138cd
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-01.ll
@@ -0,0 +1,71 @@
+; Tests that coro-split will optimize the lifetime.start maker of each local variable,
+; sink them to the places after the suspend block.
+; RUN: opt < %s -coro-split -S | FileCheck %s
+; RUN: opt < %s -passes=coro-split -S | FileCheck %s
+
+%"struct.std::coroutine_handle" = type { i8* }
+%"struct.std::coroutine_handle.0" = type { %"struct.std::coroutine_handle" }
+%"struct.lean_future<int>::Awaiter" = type { i32, %"struct.std::coroutine_handle.0" }
+
+declare i8* @malloc(i64)
+declare void @print(i32)
+
+define void @a() "coroutine.presplit"="1" {
+entry:
+  %ref.tmp7 = alloca %"struct.lean_future<int>::Awaiter", align 8
+  %testval = alloca i32
+  %cast = bitcast i32* %testval to i8*
+  ; lifetime of %testval starts here, but not used until await.ready.
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %cast)
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %alloc = call i8* @malloc(i64 16) #3
+  %vFrame = call noalias nonnull i8* @llvm.coro.begin(token %id, i8* %alloc)
+
+  %save = call token @llvm.coro.save(i8* null)
+  %Result.i19 = getelementptr inbounds %"struct.lean_future<int>::Awaiter", %"struct.lean_future<int>::Awaiter"* %ref.tmp7, i64 0, i32 0
+  %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
+  switch i8 %suspend, label %exit [
+    i8 0, label %await.ready
+    i8 1, label %exit
+  ]
+await.ready:
+  %StrayCoroSave = call token @llvm.coro.save(i8* null)
+  %val = load i32, i32* %Result.i19
+  %test = load i32, i32* %testval
+  call void @print(i32 %test)
+  call void @llvm.lifetime.end.p0i8(i64 4, i8*  %cast)
+  call void @print(i32 %val)
+  br label %exit
+exit:
+  call i1 @llvm.coro.end(i8* null, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: @a.resume(
+; CHECK:         %testval = alloca i32, align 4
+; CHECK-NEXT:    %0 = bitcast i32* %testval to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* %0)
+; CHECK-NEXT:    getelementptr inbounds %a.Frame
+; CHECK-NEXT:    getelementptr inbounds %"struct.lean_future<int>::Awaiter"
+; CHECK-NEXT:    %val = load i32, i32* %Result
+; CHECK-NEXT:    %test = load i32, i32* %testval
+; CHECK-NEXT:    call void @print(i32 %test)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 4, i8* %0)
+; CHECK-NEXT:    call void @print(i32 %val)
+; CHECK-NEXT:    ret void
+
+declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*)
+declare i1 @llvm.coro.alloc(token) #3
+declare noalias nonnull i8* @"\01??2@YAPEAX_K@Z"(i64) local_unnamed_addr
+declare i64 @llvm.coro.size.i64() #5
+declare i8* @llvm.coro.begin(token, i8* writeonly) #3
+declare void @"\01?puts@@YAXZZ"(...)
+declare token @llvm.coro.save(i8*) #3
+declare i8* @llvm.coro.frame() #5
+declare i8 @llvm.coro.suspend(token, i1) #3
+declare void @"\01??3@YAXPEAX@Z"(i8*) local_unnamed_addr #10
+declare i8* @llvm.coro.free(token, i8* nocapture readonly) #2
+declare i1 @llvm.coro.end(i8*, i1) #3
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #4
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #4
+
diff --git a/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-02.ll b/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-02.ll
new file mode 100644
index 0000000000000..6695c22a1f09c
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-02.ll
@@ -0,0 +1,80 @@
+; Tests that coro-split will optimize the lifetime.start maker of each local variable,
+; sink them to the places after the suspend block.
+; RUN: opt < %s -coro-split -S | FileCheck %s
+; RUN: opt < %s -passes=coro-split -S | FileCheck %s
+
+%"struct.std::coroutine_handle" = type { i8* }
+%"struct.std::coroutine_handle.0" = type { %"struct.std::coroutine_handle" }
+%"struct.lean_future<int>::Awaiter" = type { i32, %"struct.std::coroutine_handle.0" }
+
+declare i1 @getcond()
+declare i8* @malloc(i64)
+declare void @print(i32)
+
+define void @a() "coroutine.presplit"="1" {
+entry:
+  %ref.tmp7 = alloca %"struct.lean_future<int>::Awaiter", align 8
+  %testval = alloca i32
+  %cast = bitcast i32* %testval to i8*
+  ; lifetime of %testval starts here, but not used until await.ready.
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %cast)
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %alloc = call i8* @malloc(i64 16) #3
+  %vFrame = call noalias nonnull i8* @llvm.coro.begin(token %id, i8* %alloc)
+  %testcond = call i1 @getcond()
+  br i1 %testcond, label %if.suspend, label %else.direct
+
+if.suspend:
+  %save = call token @llvm.coro.save(i8* null)
+  %Result.i19 = getelementptr inbounds %"struct.lean_future<int>::Awaiter", %"struct.lean_future<int>::Awaiter"* %ref.tmp7, i64 0, i32 0
+  %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
+  switch i8 %suspend, label %exit [
+    i8 0, label %await.ready
+    i8 1, label %exit
+  ]
+
+else.direct:
+  br label %after.await
+
+await.ready:
+  %StrayCoroSave = call token @llvm.coro.save(i8* null)
+  %val = load i32, i32* %Result.i19
+  %test = load i32, i32* %testval
+  call void @print(i32 %test)
+  call void @print(i32 %val)
+  br label %after.await
+
+after.await:
+  %test1 = load i32, i32* %testval
+  call void @print(i32 %test1)
+  call void @llvm.lifetime.end.p0i8(i64 4, i8*  %cast)
+  br label %exit
+
+exit:
+  call i1 @llvm.coro.end(i8* null, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: @a.resume(
+; CHECK:    %[[VAL:testval.+]] = getelementptr inbounds %a.Frame
+; CHECK-NOT:     %testval = alloca i32, align 4
+; CHECK-NOT:     %[[CAST:.+]] = bitcast i32* %testval to i8*
+; CHECK-NOT:     call void @llvm.lifetime.start.p0i8(i64 4, i8* %[[CAST]])
+; CHECK:         %test = load i32, i32* %[[VAL]]
+; CHECK-NOT:     %test = load i32, i32* %testval
+
+declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*)
+declare i1 @llvm.coro.alloc(token) #3
+declare noalias nonnull i8* @"\01??2@YAPEAX_K@Z"(i64) local_unnamed_addr
+declare i64 @llvm.coro.size.i64() #5
+declare i8* @llvm.coro.begin(token, i8* writeonly) #3
+declare void @"\01?puts@@YAXZZ"(...)
+declare token @llvm.coro.save(i8*) #3
+declare i8* @llvm.coro.frame() #5
+declare i8 @llvm.coro.suspend(token, i1) #3
+declare void @"\01??3@YAXPEAX@Z"(i8*) local_unnamed_addr #10
+declare i8* @llvm.coro.free(token, i8* nocapture readonly) #2
+declare i1 @llvm.coro.end(i8*, i1) #3
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #4
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #4
+
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll b/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
index fa2e6ff9af661..0eb28bdc5b444 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
@@ -396,7 +396,7 @@ define i1 @test13(i8 %x, i64* %p) {
 ; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[X:%.*]] to i64
 ; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i64 [[ZEXT]], 128
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[ADD]], 384
-; CHECK-NEXT:    store i64 [[ADD]], i64* [[P:%.*]]
+; CHECK-NEXT:    store i64 [[ADD]], i64* [[P:%.*]], align 8
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %zext = zext i8 %x to i64
@@ -652,4 +652,341 @@ exit:
 
 declare i1 @get_bool()
 
+define void @test_icmp_or_ult(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_or_ult(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[OR]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    ret void
+; CHECK:       if.false:
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp uge i32 [[A]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP4]])
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp uge i32 [[B]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP5]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %or = or i32 %a, %b
+  %cmp = icmp ult i32 %or, 42
+  br i1 %cmp, label %if.true, label %if.false
+
+if.true:
+  %cmp2 = icmp ult i32 %a, 42
+  call void @check1(i1 %cmp2)
+  %cmp3 = icmp ult i32 %b, 42
+  call void @check1(i1 %cmp3)
+  ret void
+
+if.false:
+  %cmp4 = icmp uge i32 %a, 42
+  call void @check1(i1 %cmp4)
+  %cmp5 = icmp uge i32 %b, 42
+  call void @check1(i1 %cmp5)
+  ret void
+}
+
+define void @test_icmp_or_ule(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_or_ule(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[OR]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    ret void
+; CHECK:       if.false:
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp ugt i32 [[A]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP4]])
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp ugt i32 [[B]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP5]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %or = or i32 %a, %b
+  %cmp = icmp ule i32 %or, 42
+  br i1 %cmp, label %if.true, label %if.false
+
+if.true:
+  %cmp2 = icmp ule i32 %a, 42
+  call void @check1(i1 %cmp2)
+  %cmp3 = icmp ule i32 %b, 42
+  call void @check1(i1 %cmp3)
+  ret void
+
+if.false:
+  %cmp4 = icmp ugt i32 %a, 42
+  call void @check1(i1 %cmp4)
+  %cmp5 = icmp ugt i32 %b, 42
+  call void @check1(i1 %cmp5)
+  ret void
+}
+
+define void @test_icmp_or_ugt(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_or_ugt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[OR]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[A]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP2]])
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ugt i32 [[B]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP3]])
+; CHECK-NEXT:    ret void
+; CHECK:       if.false:
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %or = or i32 %a, %b
+  %cmp = icmp ugt i32 %or, 42
+  br i1 %cmp, label %if.true, label %if.false
+
+if.true:
+  %cmp2 = icmp ugt i32 %a, 42
+  call void @check1(i1 %cmp2)
+  %cmp3 = icmp ugt i32 %b, 42
+  call void @check1(i1 %cmp3)
+  ret void
+
+if.false:
+  %cmp4 = icmp ule i32 %a, 42
+  call void @check1(i1 %cmp4)
+  %cmp5 = icmp ule i32 %b, 42
+  call void @check1(i1 %cmp5)
+  ret void
+}
+
+define void @test_icmp_or_uge(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_or_uge(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[OR]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i32 [[A]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP2]])
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp uge i32 [[B]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP3]])
+; CHECK-NEXT:    ret void
+; CHECK:       if.false:
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %or = or i32 %a, %b
+  %cmp = icmp uge i32 %or, 42
+  br i1 %cmp, label %if.true, label %if.false
+
+if.true:
+  %cmp2 = icmp uge i32 %a, 42
+  call void @check1(i1 %cmp2)
+  %cmp3 = icmp uge i32 %b, 42
+  call void @check1(i1 %cmp3)
+  ret void
+
+if.false:
+  %cmp4 = icmp ult i32 %a, 42
+  call void @check1(i1 %cmp4)
+  %cmp5 = icmp ult i32 %b, 42
+  call void @check1(i1 %cmp5)
+  ret void
+}
+
+define void @test_icmp_or_slt(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_or_slt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[OR]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[A]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP2]])
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[B]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP3]])
+; CHECK-NEXT:    ret void
+; CHECK:       if.false:
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp sge i32 [[A]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP4]])
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp sge i32 [[B]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP5]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %or = or i32 %a, %b
+  %cmp = icmp slt i32 %or, 42
+  br i1 %cmp, label %if.true, label %if.false
+
+if.true:
+  %cmp2 = icmp slt i32 %a, 42
+  call void @check1(i1 %cmp2)
+  %cmp3 = icmp slt i32 %b, 42
+  call void @check1(i1 %cmp3)
+  ret void
+
+if.false:
+  %cmp4 = icmp sge i32 %a, 42
+  call void @check1(i1 %cmp4)
+  %cmp5 = icmp sge i32 %b, 42
+  call void @check1(i1 %cmp5)
+  ret void
+}
+
+define void @test_icmp_and_ugt(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_and_ugt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[AND]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    ret void
+; CHECK:       if.false:
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp ule i32 [[A]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP4]])
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp ule i32 [[B]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP5]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %and = and i32 %a, %b
+  %cmp = icmp ugt i32 %and, 42
+  br i1 %cmp, label %if.true, label %if.false
+
+if.true:
+  %cmp2 = icmp ugt i32 %a, 42
+  call void @check1(i1 %cmp2)
+  %cmp3 = icmp ugt i32 %b, 42
+  call void @check1(i1 %cmp3)
+  ret void
+
+if.false:
+  %cmp4 = icmp ule i32 %a, 42
+  call void @check1(i1 %cmp4)
+  %cmp5 = icmp ule i32 %b, 42
+  call void @check1(i1 %cmp5)
+  ret void
+}
+
+define void @test_icmp_and_uge(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_and_uge(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[AND]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    ret void
+; CHECK:       if.false:
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp ult i32 [[A]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP4]])
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp ult i32 [[B]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP5]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %and = and i32 %a, %b
+  %cmp = icmp uge i32 %and, 42
+  br i1 %cmp, label %if.true, label %if.false
+
+if.true:
+  %cmp2 = icmp uge i32 %a, 42
+  call void @check1(i1 %cmp2)
+  %cmp3 = icmp uge i32 %b, 42
+  call void @check1(i1 %cmp3)
+  ret void
+
+if.false:
+  %cmp4 = icmp ult i32 %a, 42
+  call void @check1(i1 %cmp4)
+  %cmp5 = icmp ult i32 %b, 42
+  call void @check1(i1 %cmp5)
+  ret void
+}
+
+define void @test_icmp_and_ult(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_and_ult(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[AND]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[A]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP2]])
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult i32 [[B]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP3]])
+; CHECK-NEXT:    ret void
+; CHECK:       if.false:
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %and = and i32 %a, %b
+  %cmp = icmp ult i32 %and, 42
+  br i1 %cmp, label %if.true, label %if.false
+
+if.true:
+  %cmp2 = icmp ult i32 %a, 42
+  call void @check1(i1 %cmp2)
+  %cmp3 = icmp ult i32 %b, 42
+  call void @check1(i1 %cmp3)
+  ret void
+
+if.false:
+  %cmp4 = icmp uge i32 %a, 42
+  call void @check1(i1 %cmp4)
+  %cmp5 = icmp uge i32 %b, 42
+  call void @check1(i1 %cmp5)
+  ret void
+}
+
+define void @test_icmp_and_sgt(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_and_sgt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[AND]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[A]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP2]])
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp sgt i32 [[B]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP3]])
+; CHECK-NEXT:    ret void
+; CHECK:       if.false:
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp sle i32 [[A]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP4]])
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp sle i32 [[B]], 42
+; CHECK-NEXT:    call void @check1(i1 [[CMP5]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %and = and i32 %a, %b
+  %cmp = icmp sgt i32 %and, 42
+  br i1 %cmp, label %if.true, label %if.false
+
+if.true:
+  %cmp2 = icmp sgt i32 %a, 42
+  call void @check1(i1 %cmp2)
+  %cmp3 = icmp sgt i32 %b, 42
+  call void @check1(i1 %cmp3)
+  ret void
+
+if.false:
+  %cmp4 = icmp sle i32 %a, 42
+  call void @check1(i1 %cmp4)
+  %cmp5 = icmp sle i32 %b, 42
+  call void @check1(i1 %cmp5)
+  ret void
+}
+
 attributes #4 = { noreturn }
diff --git a/llvm/test/Transforms/DeadStoreElimination/2011-03-25-DSEMiscompile.ll b/llvm/test/Transforms/DeadStoreElimination/2011-03-25-DSEMiscompile.ll
index 7746ccee52182..25c2d5ffe7f56 100644
--- a/llvm/test/Transforms/DeadStoreElimination/2011-03-25-DSEMiscompile.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/2011-03-25-DSEMiscompile.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -S | FileCheck %s
 ; PR9561
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
 target triple = "i386-apple-darwin9.8"
diff --git a/llvm/test/Transforms/DeadStoreElimination/2016-07-17-UseAfterFree.ll b/llvm/test/Transforms/DeadStoreElimination/2016-07-17-UseAfterFree.ll
index 90629a263f7ac..3501b43600168 100644
--- a/llvm/test/Transforms/DeadStoreElimination/2016-07-17-UseAfterFree.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/2016-07-17-UseAfterFree.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -dse -S -enable-dse-partial-overwrite-tracking | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -S -enable-dse-partial-overwrite-tracking | FileCheck %s
 ; PR28588
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/DeadStoreElimination/DeleteThrowableInst.ll b/llvm/test/Transforms/DeadStoreElimination/DeleteThrowableInst.ll
index f392b2e946a8c..5bb29529c2665 100644
--- a/llvm/test/Transforms/DeadStoreElimination/DeleteThrowableInst.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/DeleteThrowableInst.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -S | FileCheck %s
 
 declare i8* @_Znwj(i32) local_unnamed_addr
 declare void @foo() readnone
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/2011-03-25-DSEMiscompile.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/2011-03-25-DSEMiscompile.ll
index f21e6c35fa36a..c90da22026727 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/2011-03-25-DSEMiscompile.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/2011-03-25-DSEMiscompile.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 ; PR9561
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
 target triple = "i386-apple-darwin9.8"
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/2011-09-06-EndOfFunction.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/2011-09-06-EndOfFunction.ll
index 0d84e9cf9d51b..b9a0ea76d7fbb 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/2011-09-06-EndOfFunction.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/2011-09-06-EndOfFunction.ll
@@ -1,4 +1,3 @@
-; XFAIL: *
 ; RUN: opt -dse -enable-dse-memoryssa -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/2016-07-17-UseAfterFree.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/2016-07-17-UseAfterFree.ll
index 96580e0869617..85a749f81d50b 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/2016-07-17-UseAfterFree.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/2016-07-17-UseAfterFree.ll
@@ -1,5 +1,4 @@
-; XFAIL: *
-; RUN: opt < %s -basicaa -dse-enable-dse-memoryssa  -S -enable-dse-partial-overwrite-tracking | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa  -S -enable-dse-partial-overwrite-tracking | FileCheck %s
 ; PR28588
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/OverwriteStoreBegin.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/OverwriteStoreBegin.ll
index a7278d56e04cf..93e8860bdaf31 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/OverwriteStoreBegin.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/OverwriteStoreBegin.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 define void @write4to7(i32* nocapture %p) {
 ; CHECK-LABEL: @write4to7(
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/OverwriteStoreEnd.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/OverwriteStoreEnd.ll
index d8791aa5e930b..1cdeade120a69 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/OverwriteStoreEnd.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/OverwriteStoreEnd.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 %struct.vec2 = type { <4 x i32>, <4 x i32> }
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/PartialStore.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/PartialStore.ll
index 2760afb89059f..4f99ec09d2a03 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/PartialStore.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/PartialStore.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -enable-dse-partial-store-merging=false -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -enable-dse-partial-store-merging=false -S | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 ; Ensure that the dead store is deleted in this case.  It is wholely
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/atomic-todo.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/atomic-todo.ll
index d27ac812fc209..04361e63e6d08 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/atomic-todo.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/atomic-todo.ll
@@ -1,5 +1,5 @@
 ; XFAIL: *
-; RUN: opt -basicaa -dse -enable-dse-memoryssa -S < %s | FileCheck %s
+; RUN: opt -basic-aa -dse -enable-dse-memoryssa -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-macosx10.7.0"
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/atomic.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/atomic.ll
index 73f6dd3e188b6..ba4bc2ed5c4c3 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/atomic.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/atomic.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -dse -enable-dse-memoryssa -S < %s | FileCheck %s
+; RUN: opt -basic-aa -dse -enable-dse-memoryssa -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-macosx10.7.0"
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/calloc-store.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/calloc-store.ll
index 397c5947ae5c9..d8fc8136f0d7e 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/calloc-store.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/calloc-store.ll
@@ -1,6 +1,4 @@
-; XFAIL: *
-
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 declare noalias i8* @calloc(i64, i64)
 
@@ -65,3 +63,98 @@ define i8* @test7(i8 %arg) {
 ; CHECK: store i8 %arg, i8* %1, align 4
   ret i8* %1
 }
+
+define i8* @test8() {
+; CHECK-LABEL: test8
+; CHECK-NOT: store
+  %p = tail call noalias i8* @calloc(i64 1, i64 4)
+  store i8 0, i8* %p, align 1
+  %p.1 = getelementptr i8, i8* %p, i32 1
+  store i8 0, i8* %p.1, align 1
+  %p.3 = getelementptr i8, i8* %p, i32 3
+  store i8 0, i8* %p.3, align 1
+  %p.2 = getelementptr i8, i8* %p, i32 2
+  store i8 0, i8* %p.2, align 1
+  ret i8* %p
+}
+
+define i8* @test9() {
+; CHECK-LABEL: test9
+; CHECK-NEXT:    %p = tail call noalias i8* @calloc(i64 1, i64 4)
+; CHECK-NEXT:    store i8 5, i8* %p, align 1
+; CHECK-NEXT:    ret i8* %p
+
+  %p = tail call noalias i8* @calloc(i64 1, i64 4)
+  store i8 5, i8* %p, align 1
+  %p.1 = getelementptr i8, i8* %p, i32 1
+  store i8 0, i8* %p.1, align 1
+  %p.3 = getelementptr i8, i8* %p, i32 3
+  store i8 0, i8* %p.3, align 1
+  %p.2 = getelementptr i8, i8* %p, i32 2
+  store i8 0, i8* %p.2, align 1
+  ret i8* %p
+}
+
+define i8* @test10() {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[P:%.*]] = tail call noalias i8* @calloc(i64 1, i64 4)
+; CHECK-NEXT:    [[P_3:%.*]] = getelementptr i8, i8* [[P]], i32 3
+; CHECK-NEXT:    store i8 5, i8* [[P_3]], align 1
+; CHECK-NEXT:    ret i8* [[P]]
+;
+
+  %p = tail call noalias i8* @calloc(i64 1, i64 4)
+  store i8 0, i8* %p, align 1
+  %p.1 = getelementptr i8, i8* %p, i32 1
+  store i8 0, i8* %p.1, align 1
+  %p.3 = getelementptr i8, i8* %p, i32 3
+  store i8 5, i8* %p.3, align 1
+  %p.2 = getelementptr i8, i8* %p, i32 2
+  store i8 0, i8* %p.2, align 1
+  ret i8* %p
+}
+
+; TODO: we could also eliminate the last store i8 0, i8* %p.3.2, but currently
+; don't because those are eliminated before eliminating killed stores.
+define i8* @test11() {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[P:%.*]] = tail call noalias i8* @calloc(i64 1, i64 4)
+; CHECK-NEXT:    [[P_3_2:%.*]] = getelementptr i8, i8* [[P]], i32 3
+; CHECK-NEXT:    store i8 0, i8* [[P_3_2]], align 1
+; CHECK-NEXT:    ret i8* [[P]]
+;
+
+  %p = tail call noalias i8* @calloc(i64 1, i64 4)
+  store i8 0, i8* %p, align 1
+  %p.1 = getelementptr i8, i8* %p, i32 1
+  store i8 0, i8* %p.1, align 1
+  %p.3 = getelementptr i8, i8* %p, i32 3
+  store i8 5, i8* %p.3, align 1
+  %p.2 = getelementptr i8, i8* %p, i32 2
+  store i8 0, i8* %p.2, align 1
+  %p.3.2 = getelementptr i8, i8* %p, i32 3
+  store i8 0, i8* %p.3.2, align 1
+  ret i8* %p
+}
+
+define i8* @test12() {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[P:%.*]] = tail call noalias i8* @calloc(i64 1, i64 4)
+; CHECK-NEXT:    [[P_3:%.*]] = getelementptr i8, i8* [[P]], i32 3
+; CHECK-NEXT:    store i8 5, i8* [[P_3]], align 1
+; CHECK-NEXT:    call void @use(i8* [[P]])
+; CHECK-NEXT:    [[P_3_2:%.*]] = getelementptr i8, i8* [[P]], i32 3
+; CHECK-NEXT:    store i8 0, i8* [[P_3_2]], align 1
+; CHECK-NEXT:    ret i8* [[P]]
+;
+
+  %p = tail call noalias i8* @calloc(i64 1, i64 4)
+  %p.3 = getelementptr i8, i8* %p, i32 3
+  store i8 5, i8* %p.3, align 1
+  call void @use(i8* %p)
+  %p.3.2 = getelementptr i8, i8* %p, i32 3
+  store i8 0, i8* %p.3.2, align 1
+  ret i8* %p
+}
+
+declare void @use(i8*) readonly
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/combined-partial-overwrites.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/combined-partial-overwrites.ll
index e30fa2e7777c0..6cd77a59df6b1 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/combined-partial-overwrites.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/combined-partial-overwrites.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -dse -enable-dse-partial-store-merging=false < %s | FileCheck %s
+; RUN: opt -S -dse -enable-dse-memoryssa -enable-dse-partial-store-merging=false < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-bgq-linux"
 
@@ -252,6 +252,7 @@ define signext i8 @test5(i32 *%ptr) {
 ; CHECK-NEXT:    [[BPTR:%.*]] = bitcast i32* [[PTR:%.*]] to i8*
 ; CHECK-NEXT:    [[BPTR1:%.*]] = getelementptr inbounds i8, i8* [[BPTR]], i64 1
 ; CHECK-NEXT:    [[BPTR2:%.*]] = getelementptr inbounds i8, i8* [[BPTR]], i64 2
+; CHECK-NEXT:    [[BPTR3:%.*]] = getelementptr inbounds i8, i8* [[BPTR]], i64 3
 ; CHECK-NEXT:    [[WPTR:%.*]] = bitcast i8* [[BPTR]] to i16*
 ; CHECK-NEXT:    [[WPTR1:%.*]] = bitcast i8* [[BPTR1]] to i16*
 ; CHECK-NEXT:    [[WPTR2:%.*]] = bitcast i8* [[BPTR2]] to i16*
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/const-pointers.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/const-pointers.ll
index 8a08a623bf265..839fdfcf2d2cd 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/const-pointers.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/const-pointers.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -dse -enable-dse-memoryssa -S < %s | FileCheck %s
+; RUN: opt -basic-aa -dse -enable-dse-memoryssa -S < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 %t = type { i32 }
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/crash.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/crash.ll
index 987bd038a7505..c3860f1fe6421 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/crash.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/crash.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i386-apple-darwin10.0"
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/cs-cs-aliasing.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/cs-cs-aliasing.ll
index dae6a6f21c71b..7ae6c450bb560 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/cs-cs-aliasing.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/cs-cs-aliasing.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -dse -enable-dse-memoryssa -S < %s | FileCheck %s
+; RUN: opt -basic-aa -dse -enable-dse-memoryssa -S < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/debug-counter.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/debug-counter.ll
index 4378ffc240767..9def782900899 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/debug-counter.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/debug-counter.ll
@@ -3,16 +3,16 @@
 ; REQUIRES: asserts
 
 ; Eliminates store to %R in the entry block.
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -debug-counter=dse-memoryssa-skip=0,dse-memoryssa-count=1 -S | FileCheck --check-prefix=SKIP0-COUNT1 %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -debug-counter=dse-memoryssa-skip=0,dse-memoryssa-count=1 -S | FileCheck --check-prefix=SKIP0-COUNT1 %s
 
 ; Eliminates store to %P in the entry block.
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -debug-counter=dse-memoryssa-skip=1,dse-memoryssa-count=1 -S | FileCheck --check-prefix=SKIP1-COUNT1 %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -debug-counter=dse-memoryssa-skip=1,dse-memoryssa-count=1 -S | FileCheck --check-prefix=SKIP1-COUNT1 %s
 
 ; Eliminates both stores in the entry block.
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -debug-counter=dse-memoryssa-skip=0,dse-memoryssa-count=2 -S | FileCheck --check-prefix=SKIP0-COUNT2 %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -debug-counter=dse-memoryssa-skip=0,dse-memoryssa-count=2 -S | FileCheck --check-prefix=SKIP0-COUNT2 %s
 
 ; Eliminates no stores.
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -debug-counter=dse-memoryssa-skip=2,dse-memoryssa-count=1 -S | FileCheck --check-prefix=SKIP2-COUNT1 %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -debug-counter=dse-memoryssa-skip=2,dse-memoryssa-count=1 -S | FileCheck --check-prefix=SKIP2-COUNT1 %s
 
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/debuginfo.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/debuginfo.ll
index 7351edf0633e1..f4e7e1fd148c5 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/debuginfo.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/debuginfo.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -debugify -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -debugify -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/fence-todo.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/fence-todo.ll
index 4ee295821e227..cdd12ef302736 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/fence-todo.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/fence-todo.ll
@@ -1,6 +1,6 @@
 ; XFAIL: *
 
-; RUN: opt -S -basicaa -dse -enable-dse-memoryssa < %s | FileCheck %s
+; RUN: opt -S -basic-aa -dse -enable-dse-memoryssa < %s | FileCheck %s
 
 ; We DSE stack alloc'ed and byval locations, in the presence of fences.
 ; Fence does not make an otherwise thread local store visible.
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/fence.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/fence.ll
index b22fc9c92004d..fc72f1d96ddaf 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/fence.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/fence.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -dse -enable-dse-memoryssa < %s | FileCheck %s
+; RUN: opt -S -basic-aa -dse -enable-dse-memoryssa < %s | FileCheck %s
 
 ; We conservative choose to prevent dead store elimination
 ; across release or stronger fences.  It's not required 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/free.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/free.ll
index a6e47d7bb107c..13cfb7002cf1e 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/free.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/free.ll
@@ -1,6 +1,4 @@
-; XFAIL: *
-
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/invariant.start.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/invariant.start.ll
index 179c6a284d808..82e168b45f754 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/invariant.start.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/invariant.start.ll
@@ -1,5 +1,5 @@
 ; Test to make sure llvm.invariant.start calls are not treated as clobbers.
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 declare {}* @llvm.invariant.start.p0i8(i64, i8* nocapture) nounwind readonly
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/launder.invariant.group.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/launder.invariant.group.ll
index 028f3f9697bf7..46f3c261f7bc0 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/launder.invariant.group.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/launder.invariant.group.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 ; CHECK-LABEL: void @skipBarrier(i8* %ptr)
 define void @skipBarrier(i8* %ptr) {
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/libcalls.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/libcalls.ll
index df6bc74ec4e74..94faca15c1b64 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/libcalls.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/libcalls.ll
@@ -1,5 +1,6 @@
-; XFAIL: *
-; RUN: opt -S -basicaa -dse -enable-dse-memoryssa < %s | FileCheck %s
+; RUN: opt -S -basic-aa -dse -enable-dse-memoryssa < %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
 
 declare i8* @strcpy(i8* %dest, i8* %src) nounwind
 define void @test1(i8* %src) {
@@ -12,13 +13,13 @@ define void @test1(i8* %src) {
   ret void
 }
 
-declare i8* @strncpy(i8* %dest, i8* %src, i32 %n) nounwind
+declare i8* @strncpy(i8* %dest, i8* %src, i64 %n) nounwind
 define void @test2(i8* %src) {
 ; CHECK-LABEL: @test2(
   %B = alloca [16 x i8]
   %dest = getelementptr inbounds [16 x i8], [16 x i8]* %B, i64 0, i64 0
 ; CHECK-NOT: @strncpy
-  %call = call i8* @strncpy(i8* %dest, i8* %src, i32 12)
+  %call = call i8* @strncpy(i8* %dest, i8* %src, i64 12)
 ; CHECK: ret void
   ret void
 }
@@ -34,13 +35,13 @@ define void @test3(i8* %src) {
   ret void
 }
 
-declare i8* @strncat(i8* %dest, i8* %src, i32 %n) nounwind
+declare i8* @strncat(i8* %dest, i8* %src, i64 %n) nounwind
 define void @test4(i8* %src) {
 ; CHECK-LABEL: @test4(
   %B = alloca [16 x i8]
   %dest = getelementptr inbounds [16 x i8], [16 x i8]* %B, i64 0, i64 0
 ; CHECK-NOT: @strncat
-  %call = call i8* @strncat(i8* %dest, i8* %src, i32 12)
+  %call = call i8* @strncat(i8* %dest, i8* %src, i64 12)
 ; CHECK: ret void
   ret void
 }
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/lifetime.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/lifetime.ll
index e4d4722677a79..29ff7726c4eee 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/lifetime.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/lifetime.ll
@@ -1,6 +1,4 @@
-; XFAIL: *
-
-; RUN: opt -S -basicaa -dse -enable-dse-memoryssa < %s | FileCheck %s
+; RUN: opt -S -basic-aa -dse -enable-dse-memoryssa < %s | FileCheck %s
 
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
@@ -35,5 +33,3 @@ define void @test2(i32* %P) {
 ; CHECK: lifetime.end
   ret void
 }
-
-
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/mda-with-dbg-values.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/mda-with-dbg-values.ll
index 06bcc346a4351..937f10d3502c7 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/mda-with-dbg-values.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/mda-with-dbg-values.ll
@@ -69,8 +69,9 @@ attributes #2 = { argmemonly nounwind }
 !20 = !DILocation(line: 9, column: 5, scope: !14)
 !21 = !DILocation(line: 10, column: 1, scope: !14)
 
-; Check that the store is removed and that the memcpy is still there
+; Check that the both the store and memcpy are removed because they both access
+; an alloca that is not read.
 ; CHECK-LABEL: foo
 ; CHECK-NOT:   store i8
-; CHECK:       call void @llvm.memcpy
+; CHECK-NOT:   call void @llvm.memcpy
 ; CHECK:       ret void
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/memcpy-complete-overwrite.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/memcpy-complete-overwrite.ll
index 931597265e414..70c0265813634 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/memcpy-complete-overwrite.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/memcpy-complete-overwrite.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 
 ; XFAIL: *
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=dse -enable-dse-memoryssa -S | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/memcpy-lifetimes.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/memcpy-lifetimes.ll
index 6344088326ffe..62b3eff0bdd67 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/memcpy-lifetimes.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/memcpy-lifetimes.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=dse -S | FileCheck %s
 
 target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/memintrinsics.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/memintrinsics.ll
index c22ad19198b45..81ba0a6764a66 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/memintrinsics.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/memintrinsics.ll
@@ -1,5 +1,4 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; XFAIL: *
 ; RUN: opt -S -dse -enable-dse-memoryssa < %s | FileCheck %s
 
 declare void @llvm.memcpy.p0i8.p0i8.i8(i8* nocapture, i8* nocapture, i8, i1) nounwind
@@ -36,6 +35,7 @@ define void @test2() {
 
 define void @test3() {
 ; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[B:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    ret void
 ;
   %A = alloca i8
@@ -85,6 +85,7 @@ define void @test5() {
 
 define void @test6() {
 ; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[B:%.*]] = alloca i16, i16 1024, align 2
 ; CHECK-NEXT:    ret void
 ;
   %A = alloca i16, i16 1024, align 2
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/memoryssa-scan-limit.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/memoryssa-scan-limit.ll
index 43a9f0c92f1b4..c2b2eba7880ac 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/memoryssa-scan-limit.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/memoryssa-scan-limit.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck --check-prefix=NO-LIMIT %s
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -dse-memoryssa-scanlimit=0 -S | FileCheck --check-prefix=LIMIT-0 %s
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -dse-memoryssa-scanlimit=3 -S | FileCheck --check-prefix=LIMIT-3 %s
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -dse-memoryssa-scanlimit=4 -S | FileCheck --check-prefix=LIMIT-4 %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck --check-prefix=NO-LIMIT %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -dse-memoryssa-scanlimit=0 -S | FileCheck --check-prefix=LIMIT-0 %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -dse-memoryssa-scanlimit=3 -S | FileCheck --check-prefix=LIMIT-3 %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -dse-memoryssa-scanlimit=4 -S | FileCheck --check-prefix=LIMIT-4 %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/memset-and-memcpy.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/memset-and-memcpy.ll
index 6473518130a3e..d54fe130d05f6 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/memset-and-memcpy.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/memset-and-memcpy.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=dse -enable-dse-memoryssa -S | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/memset-missing-debugloc.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/memset-missing-debugloc.ll
index 80db7f5d6b6eb..c28f0cc901247 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/memset-missing-debugloc.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/memset-missing-debugloc.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the getelementptr generated when the dse pass determines that
 ; a memset can be shortened has the debugloc carried over from the memset.
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-captures.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-captures.ll
index c1f7392341963..e28713929e9ab 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-captures.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-captures.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
@@ -207,15 +207,13 @@ exit:
   call void @capture(i8* %m)
   ret i8* %m
 }
-; TODO: Remove store in exit.
+
 ; Stores to stack objects can be eliminated if they are not captured inside the function.
 define void @test_alloca_nocapture_1() {
 ; CHECK-LABEL: @test_alloca_nocapture_1(
-; CHECK-NEXT:    [[M:%.*]] = alloca i8
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    store i8 1, i8* [[M]]
 ; CHECK-NEXT:    ret void
 ;
   %m = alloca i8
@@ -228,7 +226,6 @@ exit:
   ret void
 }
 
-; TODO: Remove store in exit.
 ; Cannot remove first store i8 0, i8* %m, as the call to @capture captures the object.
 define void @test_alloca_capture_1() {
 ; CHECK-LABEL: @test_alloca_capture_1(
@@ -237,7 +234,6 @@ define void @test_alloca_capture_1() {
 ; CHECK-NEXT:    call void @capture(i8* [[M]])
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    store i8 1, i8* [[M]]
 ; CHECK-NEXT:    ret void
 ;
   %m = alloca i8
@@ -250,7 +246,6 @@ exit:
   ret void
 }
 
-; TODO: Remove store at exit.
 ; We can remove the last store to %m, even though it escapes because the alloca
 ; becomes invalid after the function returns.
 define void @test_alloca_capture_2(%S1* %E) {
@@ -260,7 +255,6 @@ define void @test_alloca_capture_2(%S1* %E) {
 ; CHECK:       exit:
 ; CHECK-NEXT:    [[F_PTR:%.*]] = getelementptr [[S1:%.*]], %S1* [[E:%.*]], i32 0, i32 0
 ; CHECK-NEXT:    store i8* [[M]], i8** [[F_PTR]]
-; CHECK-NEXT:    store i8 1, i8* [[M]]
 ; CHECK-NEXT:    ret void
 ;
   %m = alloca i8
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-exceptions.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-exceptions.ll
index 942a46bad6c62..8357ef9302006 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-exceptions.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-exceptions.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 declare void @f()
@@ -29,7 +29,6 @@ define void @test12(i32* %p) personality i32 (...)* @__CxxFrameHandler3 {
 ; CHECK-NEXT:    [[C1:%.*]] = cleanuppad within none []
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
-; CHECK-NEXT:    store i32 40, i32* [[SV]]
 ; CHECK-NEXT:    ret void
 ;
 block1:
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-loops.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-loops.ll
index 008344f7cd442..b213edbaf09e6 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-loops.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-loops.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1) nounwind
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-malloc-free.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-malloc-free.ll
index 20b2d281570a7..04cdae285d814 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-malloc-free.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-malloc-free.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 declare void @unknown_func()
@@ -17,13 +17,12 @@ declare void @free(i8* nocapture) #2
 define void @test16(i32* noalias %P) {
 ; CHECK-LABEL: @test16(
 ; CHECK-NEXT:    [[P2:%.*]] = bitcast i32* [[P:%.*]] to i8*
-; CHECK-NEXT:    store i32 1, i32* [[P]]
 ; CHECK-NEXT:    br i1 true, label [[BB1:%.*]], label [[BB3:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    store i32 1, i32* [[P]]
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    call void @free(i8* [[P2]])
+; CHECK-NEXT:    store i32 1, i32* [[P]]
 ; CHECK-NEXT:    ret void
 ;
   %P2 = bitcast i32* %P to i8*
@@ -34,6 +33,7 @@ bb1:
   br label %bb3
 bb3:
   call void @free(i8* %P2)
+  store i32 1, i32* %P
   ret void
 }
 
@@ -41,11 +41,9 @@ bb3:
 define void @test17(i32* noalias %P) {
 ; CHECK-LABEL: @test17(
 ; CHECK-NEXT:    [[P2:%.*]] = bitcast i32* [[P:%.*]] to i8*
-; CHECK-NEXT:    store i32 1, i32* [[P]]
 ; CHECK-NEXT:    br i1 true, label [[BB1:%.*]], label [[BB3:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    call void @unknown_func()
-; CHECK-NEXT:    store i32 1, i32* [[P]]
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    call void @free(i8* [[P2]])
@@ -63,6 +61,30 @@ bb3:
   ret void
 }
 
+define void @test17_read_after_free(i32* noalias %P) {
+; CHECK-LABEL: @test17_read_after_free(
+; CHECK-NEXT:    [[P2:%.*]] = bitcast i32* [[P:%.*]] to i8*
+; CHECK-NEXT:    br i1 true, label [[BB1:%.*]], label [[BB3:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    call void @free(i8* [[P2]])
+; CHECK-NEXT:    [[LV:%.*]] = load i8, i8* [[P2]]
+; CHECK-NEXT:    ret void
+;
+  %P2 = bitcast i32* %P to i8*
+  store i32 1, i32* %P
+  br i1 true, label %bb1, label %bb3
+bb1:
+  store i32 1, i32* %P
+  br label %bb3
+bb3:
+  call void @free(i8* %P2)
+  %lv = load i8, i8* %P2
+  ret void
+}
+
+
 define void @test6(i32* noalias %P) {
 ; CHECK-LABEL: @test6(
 ; CHECK-NEXT:    br i1 true, label [[BB1:%.*]], label [[BB2:%.*]]
@@ -246,7 +268,6 @@ define void @test27() {
 ; CHECK-NEXT:    br i1 true, label [[BB2:%.*]], label [[BB3:%.*]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[M:%.*]] = call noalias i8* @malloc(i64 10)
-; CHECK-NEXT:    store i8 1, i8* [[M]]
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[R:%.*]] = phi i8* [ null, [[BB1:%.*]] ], [ [[M]], [[BB2]] ]
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-memintrinsics.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-memintrinsics.ll
index bb7c556612717..d7945e888f4d0 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-memintrinsics.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-memintrinsics.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 declare void @unknown_func()
@@ -123,18 +123,10 @@ bb3:
 define void @alloca_1(i1 %c) {
 ; CHECK-LABEL: @alloca_1(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[P_ALLOCA:%.*]] = alloca [32 x i32]
-; CHECK-NEXT:    [[P:%.*]] = bitcast [32 x i32]* [[P_ALLOCA]] to i32*
-; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 1
-; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[ARRAYIDX0]] to i8*
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 4
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i1 false)
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 1
-; CHECK-NEXT:    store i32 1, i32* [[ARRAYIDX1]], align 4
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    ret void
@@ -160,20 +152,10 @@ bb3:
 define void @alloca_2(i1 %c) {
 ; CHECK-LABEL: @alloca_2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[P_ALLOCA:%.*]] = alloca [32 x i32]
-; CHECK-NEXT:    [[P:%.*]] = bitcast [32 x i32]* [[P_ALLOCA]] to i32*
-; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 1
-; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[ARRAYIDX0]] to i8*
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 4
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i1 false)
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 1
-; CHECK-NEXT:    store i32 1, i32* [[ARRAYIDX1]], align 4
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 1
-; CHECK-NEXT:    store i32 1, i32* [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    ret void
@@ -204,20 +186,12 @@ bb3:
 define void @alloca_3(i1 %c) {
 ; CHECK-LABEL: @alloca_3(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[P_ALLOCA:%.*]] = alloca [32 x i32]
-; CHECK-NEXT:    [[P:%.*]] = bitcast [32 x i32]* [[P_ALLOCA]] to i32*
-; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 1
-; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[ARRAYIDX0]] to i8*
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 4
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i1 false)
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 1
-; CHECK-NEXT:    store i32 1, i32* [[ARRAYIDX1]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-memoryphis.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-memoryphis.ll
index 4ea5f08df41aa..0ace57e690fe1 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-memoryphis.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-memoryphis.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-multipath-throwing.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-multipath-throwing.ll
index ca47661cbb840..944586253bedb 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-multipath-throwing.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-multipath-throwing.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-multipath.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-multipath.ll
index 7cbad2a9777ed..28f81015c799a 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-multipath.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-multipath.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
@@ -589,14 +589,11 @@ define void @alloca_5(i1 %c) {
 ; CHECK-LABEL: @alloca_5(
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[TMP:%.*]] = alloca [[STRUCT_BLAM_4:%.*]], align 8
-; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr inbounds [[STRUCT_BLAM_4]], %struct.blam.4* [[TMP]], i64 0, i32 0, i32 1
-; CHECK-NEXT:    [[TMP37:%.*]] = bitcast i64** [[TMP36]] to i8*
 ; CHECK-NEXT:    [[TMP38:%.*]] = getelementptr inbounds [[STRUCT_BLAM_4]], %struct.blam.4* [[TMP]], i64 0, i32 0, i32 3
 ; CHECK-NEXT:    [[TMP39:%.*]] = bitcast i64* [[TMP38]] to i64*
 ; CHECK-NEXT:    store i64 0, i64* [[TMP39]], align 4
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB46:%.*]], label [[BB47:%.*]]
 ; CHECK:       bb46:
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 8 dereferenceable(20) [[TMP37]], i8 0, i64 26, i1 false)
 ; CHECK-NEXT:    ret void
 ; CHECK:       bb47:
 ; CHECK-NEXT:    [[TMP48:%.*]] = getelementptr inbounds [[STRUCT_BLAM_4]], %struct.blam.4* [[TMP]], i64 0, i32 0, i32 2
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-partial.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-partial.ll
index f3e019be78823..b2a5c04f31fd4 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-partial.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-partial.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-simple.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-simple.ll
index 718e55c19ffed..aa09235e76986 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-simple.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-simple.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
@@ -174,10 +174,8 @@ bb3:
 
 define void @test11() {
 ; CHECK-LABEL: @test11(
-; CHECK-NEXT:    [[P:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    br i1 true, label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    store i32 0, i32* [[P]], align 4
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-throwing.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-throwing.ll
index 5fd4f9015b2d6..93a9a2d999e1a 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-throwing.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-throwing.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; XFAIL: *
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 declare void @unknown_func()
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/no-targetdata.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/no-targetdata.ll
index cf58b47a107f0..7e6a4cdf3a7ce 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/no-targetdata.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/no-targetdata.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -dse -enable-dse-memoryssa -S < %s | FileCheck %s
+; RUN: opt -basic-aa -dse -enable-dse-memoryssa -S < %s | FileCheck %s
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/noop-stores.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/noop-stores.ll
index 11eaf911392fa..6a9c4b80b3ddf 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/noop-stores.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/noop-stores.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=dse -enable-dse-memoryssa -S | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/operand-bundles.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/operand-bundles.ll
index 3a59286c53f2c..5940f2bf052bf 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/operand-bundles.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/operand-bundles.ll
@@ -1,5 +1,4 @@
-; XFAIL: *
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 declare noalias i8* @malloc(i64) "malloc-like"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/overlap.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/overlap.ll
index 4fe92abcbc401..e3e6b8f583a92 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/overlap.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/overlap.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=dse -enable-dse-memoryssa -S | FileCheck %s
 
 declare void @use(i64*)
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/pr11390.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/pr11390.ll
index ff08a218d610d..c58fc18d2a9d6 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/pr11390.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/pr11390.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -dse -enable-dse-memoryssa -S < %s | FileCheck %s
+; RUN: opt -basic-aa -dse -enable-dse-memoryssa -S < %s | FileCheck %s
 ; PR11390
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/simple-todo.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/simple-todo.ll
index 82228fa590c90..f4274a9e87f30 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/simple-todo.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/simple-todo.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; XFAIL: *
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=dse -enable-dse-memoryssa -S | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
@@ -10,37 +10,6 @@ declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) n
 declare void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32) nounwind
 declare void @llvm.init.trampoline(i8*, i8*, i8*)
 
-; Test for byval handling.
-%struct.x = type { i32, i32, i32, i32 }
-define void @test9(%struct.x* byval  %a) nounwind  {
-; CHECK-LABEL: @test9(
-; CHECK-NEXT:    ret void
-;
-  %tmp2 = getelementptr %struct.x, %struct.x* %a, i32 0, i32 0
-  store i32 1, i32* %tmp2, align 4
-  ret void
-}
-
-; Test for inalloca handling.
-define void @test9_2(%struct.x* inalloca  %a) nounwind  {
-; CHECK-LABEL: @test9_2(
-; CHECK-NEXT:    ret void
-;
-  %tmp2 = getelementptr %struct.x, %struct.x* %a, i32 0, i32 0
-  store i32 1, i32* %tmp2, align 4
-  ret void
-}
-
-; Test for preallocated handling.
-define void @test9_3(%struct.x* preallocated(%struct.x)  %a) nounwind  {
-; CHECK-LABEL: @test9_3(
-; CHECK-NEXT:    ret void
-;
-  %tmp2 = getelementptr %struct.x, %struct.x* %a, i32 0, i32 0
-  store i32 1, i32* %tmp2, align 4
-  ret void
-}
-
 ; DSE should delete the dead trampoline.
 declare void @test11f()
 define void @test11() {
@@ -55,47 +24,6 @@ define void @test11() {
 
 
 declare noalias i8* @malloc(i32)
-declare noalias i8* @calloc(i32, i32)
-
-define void @test14(i32* %Q) {
-; CHECK-LABEL: @test14(
-; CHECK-NEXT:    ret void
-;
-  %P = alloca i32
-  %DEAD = load i32, i32* %Q
-  store i32 %DEAD, i32* %P
-  ret void
-
-}
-
-define void @test20() {
-; CHECK-LABEL: @test20(
-; CHECK-NEXT:    ret void
-;
-  %m = call i8* @malloc(i32 24)
-  store i8 0, i8* %m
-  ret void
-}
-
-define void @test21() {
-; CHECK-LABEL: @test21(
-; CHECK-NEXT:    ret void
-;
-  %m = call i8* @calloc(i32 9, i32 7)
-  store i8 0, i8* %m
-  ret void
-}
-
-define void @test22(i1 %i, i32 %k, i32 %m) nounwind {
-; CHECK-LABEL: @test22(
-; CHECK-NEXT:    ret void
-;
-  %k.addr = alloca i32
-  %m.addr = alloca i32
-  %k.addr.m.addr = select i1 %i, i32* %k.addr, i32* %m.addr
-  store i32 0, i32* %k.addr.m.addr, align 4
-  ret void
-}
 
 declare void @unknown_func()
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/simple.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/simple.ll
index 6eff9b8cd9a1d..ef785f10ffafc 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/simple.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/simple.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=dse -enable-dse-memoryssa -S | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
@@ -136,6 +136,37 @@ define i32 @test8() {
 
 }
 
+; Test for byval handling.
+%struct.x = type { i32, i32, i32, i32 }
+define void @test9(%struct.x* byval  %a) nounwind  {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    ret void
+;
+  %tmp2 = getelementptr %struct.x, %struct.x* %a, i32 0, i32 0
+  store i32 1, i32* %tmp2, align 4
+  ret void
+}
+
+; Test for inalloca handling.
+define void @test9_2(%struct.x* inalloca  %a) nounwind  {
+; CHECK-LABEL: @test9_2(
+; CHECK-NEXT:    ret void
+;
+  %tmp2 = getelementptr %struct.x, %struct.x* %a, i32 0, i32 0
+  store i32 1, i32* %tmp2, align 4
+  ret void
+}
+
+; Test for preallocated handling.
+define void @test9_3(%struct.x* preallocated(%struct.x)  %a) nounwind  {
+; CHECK-LABEL: @test9_3(
+; CHECK-NEXT:    ret void
+;
+  %tmp2 = getelementptr %struct.x, %struct.x* %a, i32 0, i32 0
+  store i32 1, i32* %tmp2, align 4
+  ret void
+}
+
 ; va_arg has fuzzy dependence, the store shouldn't be zapped.
 define double @test10(i8* %X) {
 ; CHECK-LABEL: @test10(
@@ -194,6 +225,17 @@ define i32 addrspace(1)* @test13_addrspacecast() {
 declare noalias i8* @malloc(i32)
 declare noalias i8* @calloc(i32, i32)
 
+define void @test14(i32* %Q) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    ret void
+;
+  %P = alloca i32
+  %DEAD = load i32, i32* %Q
+  store i32 %DEAD, i32* %P
+  ret void
+
+}
+
 ; The store here is not dead because the byval call reads it.
 declare void @test19f({i32}* byval align 4 %P)
 
@@ -213,6 +255,35 @@ bb:
 
 }
 
+define void @test20() {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    ret void
+;
+  %m = call i8* @malloc(i32 24)
+  store i8 0, i8* %m
+  ret void
+}
+
+define void @test21() {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    ret void
+;
+  %m = call i8* @calloc(i32 9, i32 7)
+  store i8 0, i8* %m
+  ret void
+}
+
+define void @test22(i1 %i, i32 %k, i32 %m) nounwind {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    ret void
+;
+  %k.addr = alloca i32
+  %m.addr = alloca i32
+  %k.addr.m.addr = select i1 %i, i32* %k.addr, i32* %m.addr
+  store i32 0, i32* %k.addr.m.addr, align 4
+  ret void
+}
+
 ; PR13547
 declare noalias i8* @strdup(i8* nocapture) nounwind
 define noalias i8* @test23() nounwind uwtable ssp {
@@ -554,7 +625,6 @@ define void @test41(i32* noalias %P) {
 ; CHECK-NEXT:    [[P2:%.*]] = bitcast i32* [[P:%.*]] to i8*
 ; CHECK-NEXT:    store i32 1, i32* [[P]], align 4
 ; CHECK-NEXT:    call void @unknown_func()
-; CHECK-NEXT:    store i32 2, i32* [[P]], align 4
 ; CHECK-NEXT:    call void @free(i8* [[P2]])
 ; CHECK-NEXT:    ret void
 ;
@@ -610,3 +680,54 @@ entry:
   store atomic i8 3, i8* %P2 unordered, align 4
   ret void
 }
+
+; Some tests where volatile may block removing a store.
+
+; Here we can remove the first non-volatile store. We cannot remove the
+; volatile store.
+define void @test44_volatile(i32* %P) {
+; CHECK-LABEL: @test44_volatile(
+; CHECK-NEXT:    store volatile i32 2, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    store i32 3, i32* [[P]], align 4
+; CHECK-NEXT:    ret void
+;
+  store i32 1, i32* %P, align 4
+  store volatile i32 2, i32* %P, align 4
+  store i32 3, i32* %P, align 4
+  ret void
+}
+
+define void @test45_volatile(i32* %P) {
+; CHECK-LABEL: @test45_volatile(
+; CHECK-NEXT:    store volatile i32 2, i32* [[P]], align 4
+; CHECK-NEXT:    store volatile i32 3, i32* [[P]], align 4
+; CHECK-NEXT:    ret void
+;
+  store i32 1, i32* %P, align 4
+  store volatile i32 2, i32* %P, align 4
+  store volatile i32 3, i32* %P, align 4
+  ret void
+}
+
+define void @test46_volatile(i32* %P) {
+; CHECK-LABEL: @test46_volatile(
+; CHECK-NEXT:    store volatile i32 2, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    store volatile i32 3, i32* [[P]], align 4
+; CHECK-NEXT:    ret void
+;
+  store volatile i32 2, i32* %P, align 4
+  store i32 1, i32* %P, align 4
+  store volatile i32 3, i32* %P, align 4
+  ret void
+}
+
+define void @test47_volatile(i32* %P) {
+; CHECK-LABEL: @test47_volatile(
+; CHECK-NEXT:    store volatile i32 2, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    store volatile i32 3, i32* [[P]], align 4
+; CHECK-NEXT:    ret void
+;
+  store volatile i32 2, i32* %P, align 4
+  store volatile i32 3, i32* %P, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/stats.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/stats.ll
index 3cc2e893d5fcd..bd4f6f0e58668 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/stats.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/stats.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -stats -S 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-memoryssa -stats -S 2>&1 | FileCheck %s
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/OverwriteStoreBegin.ll b/llvm/test/Transforms/DeadStoreElimination/OverwriteStoreBegin.ll
index ba0d46ad062b3..b5d9c40cbdbc3 100644
--- a/llvm/test/Transforms/DeadStoreElimination/OverwriteStoreBegin.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/OverwriteStoreBegin.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -S | FileCheck %s
 
 define void @write4to7(i32* nocapture %p) {
 ; CHECK-LABEL: @write4to7(
diff --git a/llvm/test/Transforms/DeadStoreElimination/OverwriteStoreEnd.ll b/llvm/test/Transforms/DeadStoreElimination/OverwriteStoreEnd.ll
index f3934263e3436..b6ae657d17e5e 100644
--- a/llvm/test/Transforms/DeadStoreElimination/OverwriteStoreEnd.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/OverwriteStoreEnd.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 %struct.vec2 = type { <4 x i32>, <4 x i32> }
diff --git a/llvm/test/Transforms/DeadStoreElimination/PartialStore.ll b/llvm/test/Transforms/DeadStoreElimination/PartialStore.ll
index 2a07fae001353..1dd894e6658cc 100644
--- a/llvm/test/Transforms/DeadStoreElimination/PartialStore.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/PartialStore.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -dse -enable-dse-partial-store-merging=false -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -enable-dse-partial-store-merging=false -S | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 ; Ensure that the dead store is deleted in this case.  It is wholely
diff --git a/llvm/test/Transforms/DeadStoreElimination/atomic.ll b/llvm/test/Transforms/DeadStoreElimination/atomic.ll
index 79beee803e4ed..29850b7ac27f8 100644
--- a/llvm/test/Transforms/DeadStoreElimination/atomic.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/atomic.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -dse -S < %s | FileCheck %s
+; RUN: opt -basic-aa -dse -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-macosx10.7.0"
diff --git a/llvm/test/Transforms/DeadStoreElimination/calloc-store.ll b/llvm/test/Transforms/DeadStoreElimination/calloc-store.ll
index daba613320654..5bd384b033fbc 100644
--- a/llvm/test/Transforms/DeadStoreElimination/calloc-store.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/calloc-store.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -S | FileCheck %s
 
 declare noalias i8* @calloc(i64, i64)
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/const-pointers.ll b/llvm/test/Transforms/DeadStoreElimination/const-pointers.ll
index e4403edbed206..a2218b725cd3b 100644
--- a/llvm/test/Transforms/DeadStoreElimination/const-pointers.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/const-pointers.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -dse -S < %s | FileCheck %s
+; RUN: opt -basic-aa -dse -S < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 %t = type { i32 }
diff --git a/llvm/test/Transforms/DeadStoreElimination/crash.ll b/llvm/test/Transforms/DeadStoreElimination/crash.ll
index 92765697429e3..ccee7fb8ba58b 100644
--- a/llvm/test/Transforms/DeadStoreElimination/crash.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/crash.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -dse -S
+; RUN: opt < %s -basic-aa -dse -S
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i386-apple-darwin10.0"
diff --git a/llvm/test/Transforms/DeadStoreElimination/cs-cs-aliasing.ll b/llvm/test/Transforms/DeadStoreElimination/cs-cs-aliasing.ll
index a225a9023ecf4..b403e3382234d 100644
--- a/llvm/test/Transforms/DeadStoreElimination/cs-cs-aliasing.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/cs-cs-aliasing.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -dse -S < %s | FileCheck %s
+; RUN: opt -basic-aa -dse -S < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/debuginfo.ll b/llvm/test/Transforms/DeadStoreElimination/debuginfo.ll
index b14c779288aa1..b927965dc4054 100644
--- a/llvm/test/Transforms/DeadStoreElimination/debuginfo.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/debuginfo.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -debugify -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -debugify -basic-aa -dse -S | FileCheck %s
 
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/fence.ll b/llvm/test/Transforms/DeadStoreElimination/fence.ll
index 667f942877502..0ef29b0bd7222 100644
--- a/llvm/test/Transforms/DeadStoreElimination/fence.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/fence.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -dse < %s | FileCheck %s
+; RUN: opt -S -basic-aa -dse < %s | FileCheck %s
 
 ; We conservative choose to prevent dead store elimination
 ; across release or stronger fences.  It's not required 
diff --git a/llvm/test/Transforms/DeadStoreElimination/free.ll b/llvm/test/Transforms/DeadStoreElimination/free.ll
index 5cfe7263c0b67..66ccc7b4f47b5 100644
--- a/llvm/test/Transforms/DeadStoreElimination/free.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/free.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/invariant.start.ll b/llvm/test/Transforms/DeadStoreElimination/invariant.start.ll
index cac37136a5625..27400cd4ed16c 100644
--- a/llvm/test/Transforms/DeadStoreElimination/invariant.start.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/invariant.start.ll
@@ -1,5 +1,5 @@
 ; Test to make sure llvm.invariant.start calls are not treated as clobbers.
-; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -S | FileCheck %s
 
 declare {}* @llvm.invariant.start.p0i8(i64, i8* nocapture) nounwind readonly
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/launder.invariant.group.ll b/llvm/test/Transforms/DeadStoreElimination/launder.invariant.group.ll
index 725ab6e5e3468..dbbc9451edd09 100644
--- a/llvm/test/Transforms/DeadStoreElimination/launder.invariant.group.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/launder.invariant.group.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -S | FileCheck %s
 
 ; CHECK-LABEL: void @skipBarrier(i8* %ptr)
 define void @skipBarrier(i8* %ptr) {
diff --git a/llvm/test/Transforms/DeadStoreElimination/libcalls.ll b/llvm/test/Transforms/DeadStoreElimination/libcalls.ll
index eac7724608cf8..a18021d0c5c65 100644
--- a/llvm/test/Transforms/DeadStoreElimination/libcalls.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/libcalls.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -dse < %s | FileCheck %s
+; RUN: opt -S -basic-aa -dse < %s | FileCheck %s
 
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/libcalls2.ll b/llvm/test/Transforms/DeadStoreElimination/libcalls2.ll
index 0e8d9eade9ee8..df3011314e9eb 100644
--- a/llvm/test/Transforms/DeadStoreElimination/libcalls2.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/libcalls2.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -dse < %s | FileCheck %s
+; RUN: opt -S -basic-aa -dse < %s | FileCheck %s
 
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/lifetime.ll b/llvm/test/Transforms/DeadStoreElimination/lifetime.ll
index c0cc2109c202d..e9999ff0740f3 100644
--- a/llvm/test/Transforms/DeadStoreElimination/lifetime.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/lifetime.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -dse < %s | FileCheck %s
+; RUN: opt -S -basic-aa -dse < %s | FileCheck %s
 
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/no-targetdata.ll b/llvm/test/Transforms/DeadStoreElimination/no-targetdata.ll
index b66b75e14d869..aec3076678787 100644
--- a/llvm/test/Transforms/DeadStoreElimination/no-targetdata.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/no-targetdata.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -dse -S < %s | FileCheck %s
+; RUN: opt -basic-aa -dse -S < %s | FileCheck %s
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/operand-bundles.ll b/llvm/test/Transforms/DeadStoreElimination/operand-bundles.ll
index 784b2e8e55f79..f3df74be031b7 100644
--- a/llvm/test/Transforms/DeadStoreElimination/operand-bundles.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/operand-bundles.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -S | FileCheck %s
 
 declare noalias i8* @malloc(i64) "malloc-like"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/pr11390.ll b/llvm/test/Transforms/DeadStoreElimination/pr11390.ll
index 6105a2e334601..56ca604eff98b 100644
--- a/llvm/test/Transforms/DeadStoreElimination/pr11390.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/pr11390.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -dse -S < %s | FileCheck %s
+; RUN: opt -basic-aa -dse -S < %s | FileCheck %s
 ; PR11390
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/DeadStoreElimination/simple.ll b/llvm/test/Transforms/DeadStoreElimination/simple.ll
index 9c381fc5d82df..871190c584120 100644
--- a/llvm/test/Transforms/DeadStoreElimination/simple.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/simple.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -dse -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=dse -S | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
diff --git a/llvm/test/Transforms/EarlyCSE/AArch64/intrinsics.ll b/llvm/test/Transforms/EarlyCSE/AArch64/intrinsics.ll
index bec6a7539aed1..698d5d1e61c3e 100644
--- a/llvm/test/Transforms/EarlyCSE/AArch64/intrinsics.ll
+++ b/llvm/test/Transforms/EarlyCSE/AArch64/intrinsics.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -S -mtriple=aarch64-none-linux-gnu -mattr=+neon -early-cse | FileCheck %s
-; RUN: opt < %s -S -mtriple=aarch64-none-linux-gnu -mattr=+neon -basicaa -early-cse-memssa | FileCheck %s
+; RUN: opt < %s -S -mtriple=aarch64-none-linux-gnu -mattr=+neon -basic-aa -early-cse-memssa | FileCheck %s
 ; RUN: opt < %s -S -mtriple=aarch64-none-linux-gnu -mattr=+neon -passes=early-cse | FileCheck %s
 ; RUN: opt < %s -S -mtriple=aarch64-none-linux-gnu -mattr=+neon -aa-pipeline=basic-aa -passes=early-cse-memssa | FileCheck %s
 
diff --git a/llvm/test/Transforms/EarlyCSE/AArch64/ldstN.ll b/llvm/test/Transforms/EarlyCSE/AArch64/ldstN.ll
index e3c6c8d9417fa..1a11fa17dd42d 100644
--- a/llvm/test/Transforms/EarlyCSE/AArch64/ldstN.ll
+++ b/llvm/test/Transforms/EarlyCSE/AArch64/ldstN.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S -early-cse < %s | FileCheck %s
-; RUN: opt -S -basicaa -early-cse-memssa < %s | FileCheck %s
+; RUN: opt -S -basic-aa -early-cse-memssa < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
 
diff --git a/llvm/test/Transforms/EarlyCSE/PowerPC/read-reg.ll b/llvm/test/Transforms/EarlyCSE/PowerPC/read-reg.ll
index 25f5f8019e137..95dd9d515cd93 100644
--- a/llvm/test/Transforms/EarlyCSE/PowerPC/read-reg.ll
+++ b/llvm/test/Transforms/EarlyCSE/PowerPC/read-reg.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S -early-cse < %s | FileCheck %s
-; RUN: opt -S -basicaa -early-cse-memssa < %s | FileCheck %s
+; RUN: opt -S -basic-aa -early-cse-memssa < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/EarlyCSE/and_or.ll b/llvm/test/Transforms/EarlyCSE/and_or.ll
index f9b88fea40238..28161ddfb3c6c 100644
--- a/llvm/test/Transforms/EarlyCSE/and_or.ll
+++ b/llvm/test/Transforms/EarlyCSE/and_or.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -early-cse -S < %s | FileCheck %s
-; RUN: opt -basicaa -early-cse-memssa -S < %s | FileCheck %s
+; RUN: opt -basic-aa -early-cse-memssa -S < %s | FileCheck %s
 
 define i32 @test_01(i32 %a, i32 %b) {
 ; CHECK-LABEL: @test_01(
diff --git a/llvm/test/Transforms/EarlyCSE/atomics.ll b/llvm/test/Transforms/EarlyCSE/atomics.ll
index 7881be7bd9ec6..4284265d0aec0 100644
--- a/llvm/test/Transforms/EarlyCSE/atomics.ll
+++ b/llvm/test/Transforms/EarlyCSE/atomics.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -S -early-cse | FileCheck %s
-; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s
+; RUN: opt < %s -S -basic-aa -early-cse-memssa | FileCheck %s
 
 ; CHECK-LABEL: @test12(
 define i32 @test12(i1 %B, i32* %P1, i32* %P2) {
diff --git a/llvm/test/Transforms/EarlyCSE/basic.ll b/llvm/test/Transforms/EarlyCSE/basic.ll
index 25d2dc3fd9b9e..f69a8168602ac 100644
--- a/llvm/test/Transforms/EarlyCSE/basic.ll
+++ b/llvm/test/Transforms/EarlyCSE/basic.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -S -early-cse | FileCheck %s
-; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s
+; RUN: opt < %s -S -basic-aa -early-cse-memssa | FileCheck %s
 ; RUN: opt < %s -S -passes=early-cse | FileCheck %s
 
 declare void @llvm.assume(i1) nounwind
diff --git a/llvm/test/Transforms/EarlyCSE/commute.ll b/llvm/test/Transforms/EarlyCSE/commute.ll
index 6fdb0af0d7e47..b569393dbdaf0 100644
--- a/llvm/test/Transforms/EarlyCSE/commute.ll
+++ b/llvm/test/Transforms/EarlyCSE/commute.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -S -early-cse -earlycse-debug-hash | FileCheck %s
-; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s
+; RUN: opt < %s -S -basic-aa -early-cse-memssa | FileCheck %s
 
 define void @test1(float %A, float %B, float* %PA, float* %PB) {
 ; CHECK-LABEL: @test1(
diff --git a/llvm/test/Transforms/EarlyCSE/conditional.ll b/llvm/test/Transforms/EarlyCSE/conditional.ll
index d2bab4c9aca4a..ff0acac5a357d 100644
--- a/llvm/test/Transforms/EarlyCSE/conditional.ll
+++ b/llvm/test/Transforms/EarlyCSE/conditional.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -early-cse -S < %s | FileCheck %s
-; RUN: opt -basicaa -early-cse-memssa -S < %s | FileCheck %s
+; RUN: opt -basic-aa -early-cse-memssa -S < %s | FileCheck %s
 
 ; Can we CSE a known condition to a constant?
 define i1 @test(i8* %p) {
diff --git a/llvm/test/Transforms/EarlyCSE/edge.ll b/llvm/test/Transforms/EarlyCSE/edge.ll
index dd8c1b7671f6e..88bd05d4ec348 100644
--- a/llvm/test/Transforms/EarlyCSE/edge.ll
+++ b/llvm/test/Transforms/EarlyCSE/edge.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -early-cse -S < %s | FileCheck %s
-; RUN: opt -basicaa -early-cse-memssa -S < %s | FileCheck %s
+; RUN: opt -basic-aa -early-cse-memssa -S < %s | FileCheck %s
 ; Same as GVN/edge.ll, but updated to reflect EarlyCSE's less powerful
 ; implementation.  EarlyCSE currently doesn't exploit equality comparisons
 ; against constants.
diff --git a/llvm/test/Transforms/EarlyCSE/fence.ll b/llvm/test/Transforms/EarlyCSE/fence.ll
index 5eefe82cd4980..0f53edc332bd3 100644
--- a/llvm/test/Transforms/EarlyCSE/fence.ll
+++ b/llvm/test/Transforms/EarlyCSE/fence.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S -early-cse < %s | FileCheck %s
-; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s
+; RUN: opt < %s -S -basic-aa -early-cse-memssa | FileCheck %s
 ; NOTE: This file is testing the current implementation.  Some of
 ; the transforms used as negative tests below would be legal, but 
 ; only if reached through a chain of logic which EarlyCSE is incapable
diff --git a/llvm/test/Transforms/EarlyCSE/flags.ll b/llvm/test/Transforms/EarlyCSE/flags.ll
index 41ad20cf57c95..c3e74df624282 100644
--- a/llvm/test/Transforms/EarlyCSE/flags.ll
+++ b/llvm/test/Transforms/EarlyCSE/flags.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -early-cse -S < %s | FileCheck %s
-; RUN: opt -basicaa -early-cse-memssa -S < %s | FileCheck %s
+; RUN: opt -basic-aa -early-cse-memssa -S < %s | FileCheck %s
 
 declare void @use(i1)
 
diff --git a/llvm/test/Transforms/EarlyCSE/floatingpoint.ll b/llvm/test/Transforms/EarlyCSE/floatingpoint.ll
index 998f86dff7590..a4c56bf0059ae 100644
--- a/llvm/test/Transforms/EarlyCSE/floatingpoint.ll
+++ b/llvm/test/Transforms/EarlyCSE/floatingpoint.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -S -early-cse | FileCheck %s
-; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s
+; RUN: opt < %s -S -basic-aa -early-cse-memssa | FileCheck %s
 
 ; Ensure we don't simplify away additions vectors of +0.0's (same as scalars).
 define <4 x float> @fV( <4 x float> %a) {
diff --git a/llvm/test/Transforms/EarlyCSE/guards.ll b/llvm/test/Transforms/EarlyCSE/guards.ll
index a64f9612957bc..55ec46b186c6d 100644
--- a/llvm/test/Transforms/EarlyCSE/guards.ll
+++ b/llvm/test/Transforms/EarlyCSE/guards.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -early-cse < %s | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
-; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
-; RUN: opt < %s -S -basicaa -early-cse-memssa --enable-knowledge-retention | FileCheck %s --check-prefixes=CHECK,USE_ASSUME
+; RUN: opt < %s -S -basic-aa -early-cse-memssa | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
+; RUN: opt < %s -S -basic-aa -early-cse-memssa --enable-knowledge-retention | FileCheck %s --check-prefixes=CHECK,USE_ASSUME
 
 declare void @llvm.experimental.guard(i1,...)
 
diff --git a/llvm/test/Transforms/EarlyCSE/instsimplify-dom.ll b/llvm/test/Transforms/EarlyCSE/instsimplify-dom.ll
index f41ce27a87e70..e9ba93a1852cb 100644
--- a/llvm/test/Transforms/EarlyCSE/instsimplify-dom.ll
+++ b/llvm/test/Transforms/EarlyCSE/instsimplify-dom.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -early-cse -S < %s | FileCheck %s
-; RUN: opt -basicaa -early-cse-memssa -S < %s | FileCheck %s
+; RUN: opt -basic-aa -early-cse-memssa -S < %s | FileCheck %s
 ; PR12231
 
 declare i32 @f()
diff --git a/llvm/test/Transforms/EarlyCSE/invariant-loads.ll b/llvm/test/Transforms/EarlyCSE/invariant-loads.ll
index 12e3048e45ee2..c5246009b42b0 100644
--- a/llvm/test/Transforms/EarlyCSE/invariant-loads.ll
+++ b/llvm/test/Transforms/EarlyCSE/invariant-loads.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -early-cse < %s | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
-; RUN: opt -S -basicaa -early-cse-memssa < %s | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
-; RUN: opt -S -basicaa -early-cse-memssa --enable-knowledge-retention < %s | FileCheck %s --check-prefixes=CHECK,USE_ASSUME
+; RUN: opt -S -basic-aa -early-cse-memssa < %s | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
+; RUN: opt -S -basic-aa -early-cse-memssa --enable-knowledge-retention < %s | FileCheck %s --check-prefixes=CHECK,USE_ASSUME
 
 declare void @clobber_and_use(i32)
 
diff --git a/llvm/test/Transforms/EarlyCSE/memoryssa.ll b/llvm/test/Transforms/EarlyCSE/memoryssa.ll
index 41e8d488ab4d4..5d33ed49bb1b4 100644
--- a/llvm/test/Transforms/EarlyCSE/memoryssa.ll
+++ b/llvm/test/Transforms/EarlyCSE/memoryssa.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -S -early-cse | FileCheck %s --check-prefix=CHECK-NOMEMSSA
-; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s
+; RUN: opt < %s -S -basic-aa -early-cse-memssa | FileCheck %s
 ; RUN: opt < %s -S -passes='early-cse' | FileCheck %s --check-prefix=CHECK-NOMEMSSA
 ; RUN: opt < %s -S -aa-pipeline=basic-aa -passes='early-cse-memssa' | FileCheck %s
 
diff --git a/llvm/test/Transforms/FixIrreducible/unreachable.ll b/llvm/test/Transforms/FixIrreducible/unreachable.ll
new file mode 100644
index 0000000000000..71cd81e01953e
--- /dev/null
+++ b/llvm/test/Transforms/FixIrreducible/unreachable.ll
@@ -0,0 +1,24 @@
+; RUN: opt %s -fix-irreducible -S -o - | FileCheck %s
+
+; CHECK-LABEL: @unreachable(
+; CHECK: entry:
+; CHECK-NOT: irr.guard:
+define void @unreachable(i32 %n) {
+entry:
+  br label %loop.body
+
+loop.body:
+  br label %inner.block
+
+unreachable.block:
+  br label %inner.block
+
+inner.block:
+  br i1 undef, label %loop.exit, label %loop.latch
+
+loop.latch:
+  br label %loop.body
+
+loop.exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadNone.ll b/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadNone.ll
index d747fe727fae5..b3035a67abaca 100644
--- a/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadNone.ll
+++ b/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadNone.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -functionattrs -S | FileCheck %s
+; RUN: opt < %s -basic-aa -functionattrs -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=function-attrs -S | FileCheck %s
 
 @x = global i32 0
diff --git a/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadOnly.ll b/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadOnly.ll
index 35cb5342d8df2..1df26459e89f4 100644
--- a/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadOnly.ll
+++ b/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadOnly.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -functionattrs -S | FileCheck %s
+; RUN: opt < %s -basic-aa -functionattrs -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=function-attrs -S | FileCheck %s
 
 ; CHECK: define i32 @f() #0
diff --git a/llvm/test/Transforms/FunctionAttrs/2008-12-29-Constant.ll b/llvm/test/Transforms/FunctionAttrs/2008-12-29-Constant.ll
index ee1a8caa20edf..69ff6cc0a6eb0 100644
--- a/llvm/test/Transforms/FunctionAttrs/2008-12-29-Constant.ll
+++ b/llvm/test/Transforms/FunctionAttrs/2008-12-29-Constant.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -functionattrs -S | FileCheck %s
+; RUN: opt < %s -basic-aa -functionattrs -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=function-attrs -S | FileCheck %s
 
 @s = external constant i8		; <i8*> [#uses=1]
diff --git a/llvm/test/Transforms/FunctionAttrs/atomic.ll b/llvm/test/Transforms/FunctionAttrs/atomic.ll
index 35de0ce800dee..a0a08890cb65a 100644
--- a/llvm/test/Transforms/FunctionAttrs/atomic.ll
+++ b/llvm/test/Transforms/FunctionAttrs/atomic.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -functionattrs -S < %s | FileCheck %s
+; RUN: opt -basic-aa -functionattrs -S < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes=function-attrs -S < %s | FileCheck %s
 
 ; Atomic load/store to local doesn't affect whether a function is
diff --git a/llvm/test/Transforms/FunctionAttrs/norecurse.ll b/llvm/test/Transforms/FunctionAttrs/norecurse.ll
index b780bb7b7189b..ad03694c32850 100644
--- a/llvm/test/Transforms/FunctionAttrs/norecurse.ll
+++ b/llvm/test/Transforms/FunctionAttrs/norecurse.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -functionattrs -rpo-functionattrs -S | FileCheck %s 
+; RUN: opt < %s -basic-aa -functionattrs -rpo-functionattrs -S | FileCheck %s 
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes='cgscc(function-attrs),rpo-functionattrs' -S | FileCheck %s
 
 ; CHECK: Function Attrs
diff --git a/llvm/test/Transforms/GVN/2007-07-25-InfiniteLoop.ll b/llvm/test/Transforms/GVN/2007-07-25-InfiniteLoop.ll
index 98fb6b3c382ec..5be4328467fa5 100644
--- a/llvm/test/Transforms/GVN/2007-07-25-InfiniteLoop.ll
+++ b/llvm/test/Transforms/GVN/2007-07-25-InfiniteLoop.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 	%struct.INT2 = type { i32, i32 }
 @blkshifts = external global %struct.INT2*		; <%struct.INT2**> [#uses=2]
diff --git a/llvm/test/Transforms/GVN/2007-07-26-InterlockingLoops.ll b/llvm/test/Transforms/GVN/2007-07-26-InterlockingLoops.ll
index 98e0024adf5b2..f1b53abc02a11 100644
--- a/llvm/test/Transforms/GVN/2007-07-26-InterlockingLoops.ll
+++ b/llvm/test/Transforms/GVN/2007-07-26-InterlockingLoops.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 @last = external global [65 x i32*]
 
diff --git a/llvm/test/Transforms/GVN/2007-07-31-NoDomInherit.ll b/llvm/test/Transforms/GVN/2007-07-31-NoDomInherit.ll
index ebd6ea247ef53..dc0bb049f964f 100644
--- a/llvm/test/Transforms/GVN/2007-07-31-NoDomInherit.ll
+++ b/llvm/test/Transforms/GVN/2007-07-31-NoDomInherit.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 	%struct.anon = type { i32 (i32, i32, i32)*, i32, i32, [3 x i32], i8*, i8*, i8* }
 @debug = external constant i32		; <i32*> [#uses=0]
diff --git a/llvm/test/Transforms/GVN/2007-07-31-RedundantPhi.ll b/llvm/test/Transforms/GVN/2007-07-31-RedundantPhi.ll
index 11ec736d2dffd..83d3fd57033dc 100644
--- a/llvm/test/Transforms/GVN/2007-07-31-RedundantPhi.ll
+++ b/llvm/test/Transforms/GVN/2007-07-31-RedundantPhi.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 @img_width = external global i16		; <i16*> [#uses=2]
 
diff --git a/llvm/test/Transforms/GVN/2008-07-02-Unreachable.ll b/llvm/test/Transforms/GVN/2008-07-02-Unreachable.ll
index 0cd80e8da0c7e..ef3a12e8739b5 100644
--- a/llvm/test/Transforms/GVN/2008-07-02-Unreachable.ll
+++ b/llvm/test/Transforms/GVN/2008-07-02-Unreachable.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 ; PR2503
 
 @g_3 = external global i8		; <i8*> [#uses=2]
diff --git a/llvm/test/Transforms/GVN/2010-03-31-RedundantPHIs.ll b/llvm/test/Transforms/GVN/2010-03-31-RedundantPHIs.ll
index 9d9ad54240750..c93dcee5a37b3 100644
--- a/llvm/test/Transforms/GVN/2010-03-31-RedundantPHIs.ll
+++ b/llvm/test/Transforms/GVN/2010-03-31-RedundantPHIs.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 ; CHECK-NOT: load
 ; CHECK-NOT: phi
diff --git a/llvm/test/Transforms/GVN/2010-11-13-Simplify.ll b/llvm/test/Transforms/GVN/2010-11-13-Simplify.ll
index 9d0becc4d0eac..367d740a6d856 100644
--- a/llvm/test/Transforms/GVN/2010-11-13-Simplify.ll
+++ b/llvm/test/Transforms/GVN/2010-11-13-Simplify.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 declare i32 @foo(i32) readnone
 
diff --git a/llvm/test/Transforms/GVN/2011-09-07-TypeIdFor.ll b/llvm/test/Transforms/GVN/2011-09-07-TypeIdFor.ll
index d6b69d3eb4e86..7bf84e96c04dc 100644
--- a/llvm/test/Transforms/GVN/2011-09-07-TypeIdFor.ll
+++ b/llvm/test/Transforms/GVN/2011-09-07-TypeIdFor.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 %struct.__fundamental_type_info_pseudo = type { %struct.__type_info_pseudo }
 %struct.__type_info_pseudo = type { i8*, i8* }
 
diff --git a/llvm/test/Transforms/GVN/2016-08-30-MaskedScatterGather.ll b/llvm/test/Transforms/GVN/2016-08-30-MaskedScatterGather.ll
index 5b10a1bfc79e6..e6193f22c4277 100644
--- a/llvm/test/Transforms/GVN/2016-08-30-MaskedScatterGather.ll
+++ b/llvm/test/Transforms/GVN/2016-08-30-MaskedScatterGather.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 declare void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> , <2 x i32*> , i32 , <2 x i1> )
 declare <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*>, i32, <2 x i1>, <2 x i32>)
diff --git a/llvm/test/Transforms/GVN/MemdepMiscompile.ll b/llvm/test/Transforms/GVN/MemdepMiscompile.ll
index 0652304b3d7dc..241313ec19419 100644
--- a/llvm/test/Transforms/GVN/MemdepMiscompile.ll
+++ b/llvm/test/Transforms/GVN/MemdepMiscompile.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-macosx10.7.0"
 
diff --git a/llvm/test/Transforms/GVN/PRE/2011-06-01-NonLocalMemdepMiscompile.ll b/llvm/test/Transforms/GVN/PRE/2011-06-01-NonLocalMemdepMiscompile.ll
index 6592c69b027fc..4a6ea1cfee96f 100644
--- a/llvm/test/Transforms/GVN/PRE/2011-06-01-NonLocalMemdepMiscompile.ll
+++ b/llvm/test/Transforms/GVN/PRE/2011-06-01-NonLocalMemdepMiscompile.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 ; This test is checking that (a) this doesn't crash, and (b) we don't
 ; conclude the value of %tmp17 is available in bb1.bb15_crit_edge.
 ; rdar://9429882
diff --git a/llvm/test/Transforms/GVN/PRE/atomic.ll b/llvm/test/Transforms/GVN/PRE/atomic.ll
index 3479bc9a0e33a..f825bdeb2a618 100644
--- a/llvm/test/Transforms/GVN/PRE/atomic.ll
+++ b/llvm/test/Transforms/GVN/PRE/atomic.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -gvn -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-macosx10.7.0"
diff --git a/llvm/test/Transforms/GVN/PRE/invariant-load.ll b/llvm/test/Transforms/GVN/PRE/invariant-load.ll
index 715e79e87e099..4d6deb88f4993 100644
--- a/llvm/test/Transforms/GVN/PRE/invariant-load.ll
+++ b/llvm/test/Transforms/GVN/PRE/invariant-load.ll
@@ -1,5 +1,5 @@
 ; Test if the !invariant.load metadata is maintained by GVN.
-; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -gvn -S < %s | FileCheck %s
 
 define i32 @test1(i32* nocapture %p, i8* nocapture %q) {
 ; CHECK-LABEL: test1
diff --git a/llvm/test/Transforms/GVN/PRE/load-pre-licm.ll b/llvm/test/Transforms/GVN/PRE/load-pre-licm.ll
index ff1d94cb0ba56..6d16587ed1528 100644
--- a/llvm/test/Transforms/GVN/PRE/load-pre-licm.ll
+++ b/llvm/test/Transforms/GVN/PRE/load-pre-licm.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -gvn < %s | FileCheck %s
+; RUN: opt -S -basic-aa -gvn < %s | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
 target triple = "i386-apple-darwin11.0.0"
 
diff --git a/llvm/test/Transforms/GVN/PRE/load-pre-nonlocal.ll b/llvm/test/Transforms/GVN/PRE/load-pre-nonlocal.ll
index 8aada26e1291a..a09a21c5f5d68 100644
--- a/llvm/test/Transforms/GVN/PRE/load-pre-nonlocal.ll
+++ b/llvm/test/Transforms/GVN/PRE/load-pre-nonlocal.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -o - -basicaa -domtree -gvn %s | FileCheck %s
+; RUN: opt -S -o - -basic-aa -domtree -gvn %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/GVN/PRE/lpre-call-wrap-2.ll b/llvm/test/Transforms/GVN/PRE/lpre-call-wrap-2.ll
index 5dc779ef40d1b..fb32b42e5306d 100644
--- a/llvm/test/Transforms/GVN/PRE/lpre-call-wrap-2.ll
+++ b/llvm/test/Transforms/GVN/PRE/lpre-call-wrap-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -gvn -enable-load-pre < %s | FileCheck %s
+; RUN: opt -S -basic-aa -gvn -enable-load-pre < %s | FileCheck %s
 ;
 ; The partially redundant load in bb1 should be hoisted to "bb".  This comes
 ; from this C code (GCC PR 23455):
diff --git a/llvm/test/Transforms/GVN/PRE/phi-translate.ll b/llvm/test/Transforms/GVN/PRE/phi-translate.ll
index 861cb9f58523b..f80e002eab0ad 100644
--- a/llvm/test/Transforms/GVN/PRE/phi-translate.ll
+++ b/llvm/test/Transforms/GVN/PRE/phi-translate.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -gvn -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64"
 
diff --git a/llvm/test/Transforms/GVN/PRE/pre-gep-load.ll b/llvm/test/Transforms/GVN/PRE/pre-gep-load.ll
index b5f5fd01bdcf6..cfea28eff03c8 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-gep-load.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-gep-load.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -gvn -enable-load-pre -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -enable-load-pre -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=gvn -enable-load-pre -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes="gvn<load-pre>" -enable-load-pre=false -S | FileCheck %s
 
diff --git a/llvm/test/Transforms/GVN/PRE/pre-load-guards.ll b/llvm/test/Transforms/GVN/PRE/pre-load-guards.ll
index eeb3ef649d945..cad3090f04822 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-load-guards.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-load-guards.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -enable-load-pre -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -enable-load-pre -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 declare void @llvm.experimental.guard(i1, ...)
diff --git a/llvm/test/Transforms/GVN/PRE/pre-load-in-loop.ll b/llvm/test/Transforms/GVN/PRE/pre-load-in-loop.ll
index 1c7c04f5afd2c..776a95ff031c2 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-load-in-loop.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-load-in-loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -enable-load-in-loop-pre=false -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -enable-load-in-loop-pre=false -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ;void test1(int N, double *G) {
diff --git a/llvm/test/Transforms/GVN/PRE/pre-load.ll b/llvm/test/Transforms/GVN/PRE/pre-load.ll
index 45a93109e6954..831086d0f68a4 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-load.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-load.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -enable-load-pre -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -enable-load-pre -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes="gvn<load-pre>" -enable-load-pre=false -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/llvm/test/Transforms/GVN/PRE/preserve-tbaa.ll b/llvm/test/Transforms/GVN/PRE/preserve-tbaa.ll
index 2ec8e88184f56..32834dd876e9c 100644
--- a/llvm/test/Transforms/GVN/PRE/preserve-tbaa.ll
+++ b/llvm/test/Transforms/GVN/PRE/preserve-tbaa.ll
@@ -1,4 +1,4 @@
-; RUN: opt -tbaa -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -tbaa -basic-aa -gvn -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64"
 
diff --git a/llvm/test/Transforms/GVN/PRE/rle-addrspace-cast.ll b/llvm/test/Transforms/GVN/PRE/rle-addrspace-cast.ll
index d8de5b360ba18..4fc446599f396 100644
--- a/llvm/test/Transforms/GVN/PRE/rle-addrspace-cast.ll
+++ b/llvm/test/Transforms/GVN/PRE/rle-addrspace-cast.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -data-layout="e-p:32:32:32-p1:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-n8:16:32" -basicaa -gvn -S -die | FileCheck %s
+; RUN: opt < %s -data-layout="e-p:32:32:32-p1:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-n8:16:32" -basic-aa -gvn -S -die | FileCheck %s
 
 define i8 @coerce_offset0_addrspacecast(i32 %V, i32* %P) {
   store i32 %V, i32* %P
diff --git a/llvm/test/Transforms/GVN/PRE/rle-semidominated.ll b/llvm/test/Transforms/GVN/PRE/rle-semidominated.ll
index f9704d3efbffa..46d4fa3313d31 100644
--- a/llvm/test/Transforms/GVN/PRE/rle-semidominated.ll
+++ b/llvm/test/Transforms/GVN/PRE/rle-semidominated.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 define i32 @main(i32* %p, i32 %x, i32 %y) {
 block1:
diff --git a/llvm/test/Transforms/GVN/PRE/rle.ll b/llvm/test/Transforms/GVN/PRE/rle.ll
index a461be6f4f8fd..4d4e2cbf121d2 100644
--- a/llvm/test/Transforms/GVN/PRE/rle.ll
+++ b/llvm/test/Transforms/GVN/PRE/rle.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -data-layout="e-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-n8:16:32" -basicaa -gvn -S -die | FileCheck %s
-; RUN: opt < %s -data-layout="E-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-n32"      -basicaa -gvn -S -die | FileCheck %s
+; RUN: opt < %s -data-layout="e-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-n8:16:32" -basic-aa -gvn -S -die | FileCheck %s
+; RUN: opt < %s -data-layout="E-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-n32"      -basic-aa -gvn -S -die | FileCheck %s
 
 ;; Trivial RLE test.
 define i32 @test0(i32 %V, i32* %P) {
diff --git a/llvm/test/Transforms/GVN/PRE/volatile.ll b/llvm/test/Transforms/GVN/PRE/volatile.ll
index 74b296a027347..ef17996463184 100644
--- a/llvm/test/Transforms/GVN/PRE/volatile.ll
+++ b/llvm/test/Transforms/GVN/PRE/volatile.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Tests that check our handling of volatile instructions encountered
 ; when scanning for dependencies
-; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -gvn -S < %s | FileCheck %s
 
 ; Check that we can bypass a volatile load when searching
 ; for dependencies of a non-volatile load
diff --git a/llvm/test/Transforms/GVN/basic-undef-test.ll b/llvm/test/Transforms/GVN/basic-undef-test.ll
index 4f94095678069..d0a43b6fce1ba 100644
--- a/llvm/test/Transforms/GVN/basic-undef-test.ll
+++ b/llvm/test/Transforms/GVN/basic-undef-test.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -gvn -S < %s | FileCheck %s
 ; ModuleID = 'test3.ll'
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/GVN/calloc-load-removal.ll b/llvm/test/Transforms/GVN/calloc-load-removal.ll
index a51f71fd77a18..a80c308dc88c6 100644
--- a/llvm/test/Transforms/GVN/calloc-load-removal.ll
+++ b/llvm/test/Transforms/GVN/calloc-load-removal.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S -basicaa -gvn < %s | FileCheck %s
-; RUN: opt -S -basicaa -gvn -disable-simplify-libcalls < %s | FileCheck %s -check-prefix=CHECK_NO_LIBCALLS
+; RUN: opt -S -basic-aa -gvn < %s | FileCheck %s
+; RUN: opt -S -basic-aa -gvn -disable-simplify-libcalls < %s | FileCheck %s -check-prefix=CHECK_NO_LIBCALLS
 ; Check that loads from calloc are recognized as being zero.
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/GVN/calls-nonlocal.ll b/llvm/test/Transforms/GVN/calls-nonlocal.ll
index d3c03d6c78d75..81057dd6cc99e 100644
--- a/llvm/test/Transforms/GVN/calls-nonlocal.ll
+++ b/llvm/test/Transforms/GVN/calls-nonlocal.ll
@@ -1,5 +1,5 @@
 ; Two occurrences of strlen should be zapped.
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i386-apple-darwin9"
 
diff --git a/llvm/test/Transforms/GVN/calls-readonly.ll b/llvm/test/Transforms/GVN/calls-readonly.ll
index 35b69d49ac718..3bdb0b34f55dd 100644
--- a/llvm/test/Transforms/GVN/calls-readonly.ll
+++ b/llvm/test/Transforms/GVN/calls-readonly.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 ; Should delete the second call to strlen even though the intervening strchr call exists.
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
diff --git a/llvm/test/Transforms/GVN/cond_br.ll b/llvm/test/Transforms/GVN/cond_br.ll
index aeb1a6e6bf70d..7428bec33ada4 100644
--- a/llvm/test/Transforms/GVN/cond_br.ll
+++ b/llvm/test/Transforms/GVN/cond_br.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -gvn -S < %s | FileCheck %s
 @y = external global i32
 @z = external global i32
 
diff --git a/llvm/test/Transforms/GVN/cond_br2.ll b/llvm/test/Transforms/GVN/cond_br2.ll
index 557ae536396e0..faccf75754c28 100644
--- a/llvm/test/Transforms/GVN/cond_br2.ll
+++ b/llvm/test/Transforms/GVN/cond_br2.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -gvn -S < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 %"class.llvm::SmallVector" = type { %"class.llvm::SmallVectorImpl", [1 x %"union.llvm::SmallVectorBase::U"] }
diff --git a/llvm/test/Transforms/GVN/condprop.ll b/llvm/test/Transforms/GVN/condprop.ll
index 949fdd0c0c92a..1c07ad8441ee5 100644
--- a/llvm/test/Transforms/GVN/condprop.ll
+++ b/llvm/test/Transforms/GVN/condprop.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 @a = external global i32		; <i32*> [#uses=7]
 
diff --git a/llvm/test/Transforms/GVN/crash-no-aa.ll b/llvm/test/Transforms/GVN/crash-no-aa.ll
index 0d09ecedc6acd..461b506080993 100644
--- a/llvm/test/Transforms/GVN/crash-no-aa.ll
+++ b/llvm/test/Transforms/GVN/crash-no-aa.ll
@@ -1,4 +1,4 @@
-; RUN: opt -disable-basicaa -gvn -S < %s
+; RUN: opt -disable-basic-aa -gvn -S < %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-freebsd8.0"
diff --git a/llvm/test/Transforms/GVN/fence.ll b/llvm/test/Transforms/GVN/fence.ll
index f39fb951865d9..8d618e0d4ed98 100644
--- a/llvm/test/Transforms/GVN/fence.ll
+++ b/llvm/test/Transforms/GVN/fence.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -gvn < %s | FileCheck %s
+; RUN: opt -S -basic-aa -gvn < %s | FileCheck %s
 
 @a = external constant i32
 ; We can value forward across the fence since we can (semantically) 
diff --git a/llvm/test/Transforms/GVN/funclet.ll b/llvm/test/Transforms/GVN/funclet.ll
index 2669256f0bdcb..a6453356dfbbd 100644
--- a/llvm/test/Transforms/GVN/funclet.ll
+++ b/llvm/test/Transforms/GVN/funclet.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -gvn -S < %s | FileCheck %s
 target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
 target triple = "i686-pc-windows-msvc"
 
diff --git a/llvm/test/Transforms/GVN/lifetime-simple.ll b/llvm/test/Transforms/GVN/lifetime-simple.ll
index 8da3e4cbd30f0..374dfb86a00b9 100644
--- a/llvm/test/Transforms/GVN/lifetime-simple.ll
+++ b/llvm/test/Transforms/GVN/lifetime-simple.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i386-apple-darwin7"
diff --git a/llvm/test/Transforms/GVN/load-constant-mem.ll b/llvm/test/Transforms/GVN/load-constant-mem.ll
index a61c83d33047a..fb57f6ec71b3d 100644
--- a/llvm/test/Transforms/GVN/load-constant-mem.ll
+++ b/llvm/test/Transforms/GVN/load-constant-mem.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -instcombine -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -instcombine -S | FileCheck %s
 ; PR4189
 @G = external constant [4 x i32]
 
diff --git a/llvm/test/Transforms/GVN/malloc-load-removal.ll b/llvm/test/Transforms/GVN/malloc-load-removal.ll
index 84f4746344ca9..bbb6f5ca7851d 100644
--- a/llvm/test/Transforms/GVN/malloc-load-removal.ll
+++ b/llvm/test/Transforms/GVN/malloc-load-removal.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S -basicaa -gvn < %s | FileCheck %s
-; RUN: opt -S -basicaa -gvn -disable-simplify-libcalls < %s | FileCheck %s -check-prefix=CHECK_NO_LIBCALLS
+; RUN: opt -S -basic-aa -gvn < %s | FileCheck %s
+; RUN: opt -S -basic-aa -gvn -disable-simplify-libcalls < %s | FileCheck %s -check-prefix=CHECK_NO_LIBCALLS
 ; PR13694
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/GVN/noalias.ll b/llvm/test/Transforms/GVN/noalias.ll
index b307e102090ba..69c21f110b5ea 100644
--- a/llvm/test/Transforms/GVN/noalias.ll
+++ b/llvm/test/Transforms/GVN/noalias.ll
@@ -1,4 +1,4 @@
-; RUN: opt -scoped-noalias -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -scoped-noalias -basic-aa -gvn -S < %s | FileCheck %s
 
 define i32 @test1(i32* %p, i32* %q) {
 ; CHECK-LABEL: @test1(i32* %p, i32* %q)
diff --git a/llvm/test/Transforms/GVN/non-local-offset.ll b/llvm/test/Transforms/GVN/non-local-offset.ll
index 2373ef5888b91..bdc5f53d0a602 100644
--- a/llvm/test/Transforms/GVN/non-local-offset.ll
+++ b/llvm/test/Transforms/GVN/non-local-offset.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -gvn -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64"
 
diff --git a/llvm/test/Transforms/GVN/nonescaping-malloc.ll b/llvm/test/Transforms/GVN/nonescaping-malloc.ll
index 639fc686e193b..fd61835e14a40 100644
--- a/llvm/test/Transforms/GVN/nonescaping-malloc.ll
+++ b/llvm/test/Transforms/GVN/nonescaping-malloc.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -basicaa -gvn -stats -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -stats -disable-output 2>&1 | FileCheck %s
 ; rdar://7363102
 
 ; CHECK: Number of loads deleted
diff --git a/llvm/test/Transforms/GVN/null-aliases-nothing.ll b/llvm/test/Transforms/GVN/null-aliases-nothing.ll
index 0b7c5eb5bf955..1352c32af4e06 100644
--- a/llvm/test/Transforms/GVN/null-aliases-nothing.ll
+++ b/llvm/test/Transforms/GVN/null-aliases-nothing.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 %t = type { i32 }
 declare void @test1f(i8*)
diff --git a/llvm/test/Transforms/GVN/phi-translate-partial-alias.ll b/llvm/test/Transforms/GVN/phi-translate-partial-alias.ll
index f1cf53e45dec6..8e1a635528e3c 100644
--- a/llvm/test/Transforms/GVN/phi-translate-partial-alias.ll
+++ b/llvm/test/Transforms/GVN/phi-translate-partial-alias.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -gvn -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-f128:128:128-n8:16:32:64"
 
diff --git a/llvm/test/Transforms/GVN/pr10820.ll b/llvm/test/Transforms/GVN/pr10820.ll
index c6a9a931df020..4f68dde3a5e2f 100644
--- a/llvm/test/Transforms/GVN/pr10820.ll
+++ b/llvm/test/Transforms/GVN/pr10820.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 target datalayout =
 "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-f128:128:128-n8:16:32:64"
diff --git a/llvm/test/Transforms/GVN/pr14166.ll b/llvm/test/Transforms/GVN/pr14166.ll
index c652360010075..452604b060eb1 100644
--- a/llvm/test/Transforms/GVN/pr14166.ll
+++ b/llvm/test/Transforms/GVN/pr14166.ll
@@ -1,4 +1,4 @@
-; RUN: opt -disable-basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -disable-basic-aa -gvn -S < %s | FileCheck %s
 target datalayout = "e-p:32:32:32"
 target triple = "i386-pc-linux-gnu"
 define <2 x i32> @test1() {
diff --git a/llvm/test/Transforms/GVN/pr17852.ll b/llvm/test/Transforms/GVN/pr17852.ll
index 9a8a709ba6a51..3d41a357f0052 100644
--- a/llvm/test/Transforms/GVN/pr17852.ll
+++ b/llvm/test/Transforms/GVN/pr17852.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn
+; RUN: opt < %s -basic-aa -gvn
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 %struct.S0 = type { [2 x i8], [2 x i8], [4 x i8], [2 x i8], i32, i32, i32, i32 }
 define void @fn1(%struct.S0* byval align 8 %p1) {
diff --git a/llvm/test/Transforms/GVN/pr24397.ll b/llvm/test/Transforms/GVN/pr24397.ll
index db43964e2e4ce..d99e868893e34 100644
--- a/llvm/test/Transforms/GVN/pr24397.ll
+++ b/llvm/test/Transforms/GVN/pr24397.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -gvn -disable-output < %s
+; RUN: opt -basic-aa -gvn -disable-output < %s
 
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/GVN/pre-new-inst.ll b/llvm/test/Transforms/GVN/pre-new-inst.ll
index 238b8a687cccc..ee450484b8986 100644
--- a/llvm/test/Transforms/GVN/pre-new-inst.ll
+++ b/llvm/test/Transforms/GVN/pre-new-inst.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -gvn -S %s | FileCheck %s
+; RUN: opt -basic-aa -gvn -S %s | FileCheck %s
 
 %MyStruct = type { i32, i32 }
 define i8 @foo(i64 %in, i8* %arr) {
diff --git a/llvm/test/Transforms/GVN/range.ll b/llvm/test/Transforms/GVN/range.ll
index a2f2e67dae509..34ae3b13dc6a2 100644
--- a/llvm/test/Transforms/GVN/range.ll
+++ b/llvm/test/Transforms/GVN/range.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -gvn -S < %s | FileCheck %s
 
 define i32 @test1(i32* %p) {
 ; CHECK-LABEL: @test1(i32* %p)
diff --git a/llvm/test/Transforms/GVN/rle-must-alias.ll b/llvm/test/Transforms/GVN/rle-must-alias.ll
index e5fafd9d6b311..d19157af4e734 100644
--- a/llvm/test/Transforms/GVN/rle-must-alias.ll
+++ b/llvm/test/Transforms/GVN/rle-must-alias.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 ; GVN should eliminate the fully redundant %9 GEP which 
 ; allows DEAD to be removed.  This is PR3198.
diff --git a/llvm/test/Transforms/GVN/rle-nonlocal.ll b/llvm/test/Transforms/GVN/rle-nonlocal.ll
index 7975462313c03..3247f13a4e500 100644
--- a/llvm/test/Transforms/GVN/rle-nonlocal.ll
+++ b/llvm/test/Transforms/GVN/rle-nonlocal.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 
 define i32 @main(i32** %p, i32 %x, i32 %y) {
 block1:
diff --git a/llvm/test/Transforms/GVN/tbaa.ll b/llvm/test/Transforms/GVN/tbaa.ll
index 5cb4e03599700..e0bd28786ac6f 100644
--- a/llvm/test/Transforms/GVN/tbaa.ll
+++ b/llvm/test/Transforms/GVN/tbaa.ll
@@ -1,4 +1,4 @@
-; RUN: opt -tbaa -basicaa -gvn -S < %s | FileCheck %s
+; RUN: opt -tbaa -basic-aa -gvn -S < %s | FileCheck %s
 
 define i32 @test1(i8* %p, i8* %q) {
 ; CHECK-LABEL: @test1(i8* %p, i8* %q)
diff --git a/llvm/test/Transforms/GVN/vscale.ll b/llvm/test/Transforms/GVN/vscale.ll
index d3ed848e3c2c0..a84df6146ebff 100644
--- a/llvm/test/Transforms/GVN/vscale.ll
+++ b/llvm/test/Transforms/GVN/vscale.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S < %s -basicaa -gvn -dce | FileCheck %s
+; RUN: opt -S < %s -basic-aa -gvn -dce | FileCheck %s
 
 ; Analyze Load from clobbering Load.
 
diff --git a/llvm/test/Transforms/GlobalOpt/fastcc.ll b/llvm/test/Transforms/GlobalOpt/fastcc.ll
index 9c9076d0155b4..edd0688ea92b7 100644
--- a/llvm/test/Transforms/GlobalOpt/fastcc.ll
+++ b/llvm/test/Transforms/GlobalOpt/fastcc.ll
@@ -35,6 +35,17 @@ define internal i32 @inalloca(i32* inalloca %p) {
   ret i32 %rv
 }
 
+define i32 @inalloca2_caller(i32* inalloca %p) {
+  %rv = musttail call i32 @inalloca2(i32* inalloca %p)
+  ret i32 %rv
+}
+define internal i32 @inalloca2(i32* inalloca %p) {
+; Because of the musttail caller, this inalloca cannot be dropped.
+; CHECK-LABEL: define internal i32 @inalloca2(i32* inalloca %p)
+  %rv = load i32, i32* %p
+  ret i32 %rv
+}
+
 define internal i32 @preallocated(i32* preallocated(i32) %p) {
 ; CHECK-LABEL: define internal fastcc i32 @preallocated(i32* %p)
   %rv = load i32, i32* %p
diff --git a/llvm/test/Transforms/IRCE/bug-loop-varying-upper-limit.ll b/llvm/test/Transforms/IRCE/bug-loop-varying-upper-limit.ll
index 2842bced1cfcb..46a83de97e432 100644
--- a/llvm/test/Transforms/IRCE/bug-loop-varying-upper-limit.ll
+++ b/llvm/test/Transforms/IRCE/bug-loop-varying-upper-limit.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -irce-print-changed-loops -S -verify-loop-info -irce -verify < %s 2>&1 | FileCheck %s
-; RUN: opt -irce-print-changed-loops -S -verify-loop-info -passes='require<branch-prob>,irce' -verify < %s 2>&1 | FileCheck %s
+; RUN: opt -irce-print-changed-loops -S -verify-loop-info -passes='require<branch-prob>,irce,verify' < %s 2>&1 | FileCheck %s
 
 ; CHECK-NOT: constrained loop
 
diff --git a/llvm/test/Transforms/IRCE/non-loop-invariant-rhs-instr.ll b/llvm/test/Transforms/IRCE/non-loop-invariant-rhs-instr.ll
index 5d6e8830d377b..3612270eaa9af 100644
--- a/llvm/test/Transforms/IRCE/non-loop-invariant-rhs-instr.ll
+++ b/llvm/test/Transforms/IRCE/non-loop-invariant-rhs-instr.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -irce -verify-loop-info -verify < %s 2>&1 | FileCheck %s
-; RUN: opt -S -passes=irce -verify-loop-info -verify < %s 2>&1 | FileCheck %s
+; RUN: opt -S -verify-loop-info -passes=irce,verify < %s 2>&1 | FileCheck %s
 
 define i32 @test_01(i32 %A, i64 %Len, i32 *%array) {
 ; CHECK-LABEL: @test_01(
diff --git a/llvm/test/Transforms/IndVarSimplify/X86/pr45360.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr45360.ll
index c4562d811d095..a100f7b4e8222 100644
--- a/llvm/test/Transforms/IndVarSimplify/X86/pr45360.ll
+++ b/llvm/test/Transforms/IndVarSimplify/X86/pr45360.ll
@@ -26,7 +26,6 @@ define dso_local i32 @main() {
 ; CHECK:       bb19:
 ; CHECK-NEXT:    [[I8_LCSSA9:%.*]] = phi i32 [ [[D_PROMOTED8]], [[BB:%.*]] ], [ [[I8:%.*]], [[BB27:%.*]] ]
 ; CHECK-NEXT:    [[I8]] = and i32 [[I8_LCSSA9]], [[I6]]
-; CHECK-NEXT:    [[TMP0:%.*]] = urem i32 [[I24]], [[I8]]
 ; CHECK-NEXT:    [[I21:%.*]] = icmp eq i32 [[I8]], 0
 ; CHECK-NEXT:    br i1 [[I21]], label [[BB27_THREAD:%.*]], label [[BB27]]
 ; CHECK:       bb27.thread:
@@ -35,8 +34,9 @@ define dso_local i32 @main() {
 ; CHECK-NEXT:    store i32 0, i32* @c, align 4
 ; CHECK-NEXT:    br label [[BB32:%.*]]
 ; CHECK:       bb27:
-; CHECK-NEXT:    store i32 [[TMP0]], i32* @e, align 4
-; CHECK-NEXT:    [[I30:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    [[I26:%.*]] = urem i32 [[I24]], [[I8]]
+; CHECK-NEXT:    store i32 [[I26]], i32* @e, align 4
+; CHECK-NEXT:    [[I30:%.*]] = icmp eq i32 [[I26]], 0
 ; CHECK-NEXT:    br i1 [[I30]], label [[BB32_LOOPEXIT:%.*]], label [[BB19]]
 ; CHECK:       bb32.loopexit:
 ; CHECK-NEXT:    store i32 -1, i32* @f, align 4
diff --git a/llvm/test/Transforms/IndVarSimplify/elim-extend.ll b/llvm/test/Transforms/IndVarSimplify/elim-extend.ll
index 14c9c99903ce4..033f6aa1d5e90 100644
--- a/llvm/test/Transforms/IndVarSimplify/elim-extend.ll
+++ b/llvm/test/Transforms/IndVarSimplify/elim-extend.ll
@@ -16,12 +16,12 @@ define void @postincConstIV(i8* %base, i32 %limit) nounwind {
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[PREADR:%.*]] = getelementptr i8, i8* [[BASE:%.*]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    store i8 0, i8* [[PREADR]]
+; CHECK-NEXT:    store i8 0, i8* [[PREADR]], align 1
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[POSTADR:%.*]] = getelementptr i8, i8* [[BASE]], i64 [[INDVARS_IV_NEXT]]
-; CHECK-NEXT:    store i8 0, i8* [[POSTADR]]
+; CHECK-NEXT:    store i8 0, i8* [[POSTADR]], align 1
 ; CHECK-NEXT:    [[POSTADRNSW:%.*]] = getelementptr inbounds i8, i8* [[BASE]], i64 [[INDVARS_IV_NEXT]]
-; CHECK-NEXT:    store i8 0, i8* [[POSTADRNSW]]
+; CHECK-NEXT:    store i8 0, i8* [[POSTADRNSW]], align 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
@@ -68,12 +68,12 @@ define void @postincVarIV(i8* %base, i32 %init, i32 %limit) nounwind {
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], [[LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[PREADR:%.*]] = getelementptr i8, i8* [[BASE:%.*]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    store i8 0, i8* [[PREADR]]
+; CHECK-NEXT:    store i8 0, i8* [[PREADR]], align 1
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[POSTADR:%.*]] = getelementptr i8, i8* [[BASE]], i64 [[INDVARS_IV_NEXT]]
-; CHECK-NEXT:    store i8 0, i8* [[POSTADR]]
+; CHECK-NEXT:    store i8 0, i8* [[POSTADR]], align 1
 ; CHECK-NEXT:    [[POSTADRNSW:%.*]] = getelementptr i8, i8* [[BASE]], i64 [[INDVARS_IV_NEXT]]
-; CHECK-NEXT:    store i8 0, i8* [[POSTADRNSW]]
+; CHECK-NEXT:    store i8 0, i8* [[POSTADRNSW]], align 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
@@ -125,7 +125,7 @@ define void @nestedIV(i8* %address, i32 %limit) nounwind {
 ; CHECK-NEXT:    [[INNERCOUNT:%.*]] = phi i32 [ [[INNERCOUNT_MERGE:%.*]], [[OUTERMERGE]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nsw i64 [[INDVARS_IV1]], -1
 ; CHECK-NEXT:    [[ADR1:%.*]] = getelementptr i8, i8* [[ADDRESS:%.*]], i64 [[TMP2]]
-; CHECK-NEXT:    store i8 0, i8* [[ADR1]]
+; CHECK-NEXT:    store i8 0, i8* [[ADR1]], align 1
 ; CHECK-NEXT:    br label [[INNERPREHEADER:%.*]]
 ; CHECK:       innerpreheader:
 ; CHECK-NEXT:    [[INNERPRECMP:%.*]] = icmp sgt i32 [[LIMITDEC]], [[INNERCOUNT]]
@@ -137,9 +137,9 @@ define void @nestedIV(i8* %address, i32 %limit) nounwind {
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[TMP3]], [[INNERLOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[INNERLOOP]] ]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[ADR2:%.*]] = getelementptr i8, i8* [[ADDRESS]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    store i8 0, i8* [[ADR2]]
+; CHECK-NEXT:    store i8 0, i8* [[ADR2]], align 1
 ; CHECK-NEXT:    [[ADR3:%.*]] = getelementptr i8, i8* [[ADDRESS]], i64 [[INDVARS_IV_NEXT]]
-; CHECK-NEXT:    store i8 0, i8* [[ADR3]]
+; CHECK-NEXT:    store i8 0, i8* [[ADR3]], align 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[TMP0]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[INNERLOOP]], label [[INNEREXIT:%.*]]
 ; CHECK:       innerexit:
@@ -149,10 +149,10 @@ define void @nestedIV(i8* %address, i32 %limit) nounwind {
 ; CHECK:       outermerge:
 ; CHECK-NEXT:    [[INNERCOUNT_MERGE]] = phi i32 [ [[TMP4]], [[INNEREXIT]] ], [ [[INNERCOUNT]], [[INNERPREHEADER]] ]
 ; CHECK-NEXT:    [[ADR4:%.*]] = getelementptr i8, i8* [[ADDRESS]], i64 [[INDVARS_IV1]]
-; CHECK-NEXT:    store i8 0, i8* [[ADR4]]
+; CHECK-NEXT:    store i8 0, i8* [[ADR4]], align 1
 ; CHECK-NEXT:    [[OFS5:%.*]] = sext i32 [[INNERCOUNT_MERGE]] to i64
 ; CHECK-NEXT:    [[ADR5:%.*]] = getelementptr i8, i8* [[ADDRESS]], i64 [[OFS5]]
-; CHECK-NEXT:    store i8 0, i8* [[ADR5]]
+; CHECK-NEXT:    store i8 0, i8* [[ADR5]], align 1
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT2]] = add nuw nsw i64 [[INDVARS_IV1]], 1
 ; CHECK-NEXT:    [[EXITCOND4:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT2]], [[WIDE_TRIP_COUNT]]
 ; CHECK-NEXT:    br i1 [[EXITCOND4]], label [[OUTERLOOP]], label [[RETURN:%.*]]
diff --git a/llvm/test/Transforms/IndVarSimplify/widen-i32-i8ptr.ll b/llvm/test/Transforms/IndVarSimplify/widen-i32-i8ptr.ll
new file mode 100644
index 0000000000000..80191d4e5b771
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/widen-i32-i8ptr.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n32:64"
+
+define dso_local void @Widen_i32_i8ptr() local_unnamed_addr {
+; CHECK-LABEL: @Widen_i32_i8ptr(
+; CHECK: phi i8*
+; CHECK: phi i32
+entry:
+  %ptrids = alloca [15 x i8*], align 8
+  %arraydecay2032 = getelementptr inbounds [15 x i8*], [15 x i8*]* %ptrids, i64 0, i64 0
+  store i8** %arraydecay2032, i8*** inttoptr (i64 8 to i8***), align 8
+  br label %for.cond2106
+
+for.cond2106:                                     ; preds = %for.cond2106, %entry
+  %gid.0 = phi i8* [ null, %entry ], [ %incdec.ptr, %for.cond2106 ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc2117, %for.cond2106 ]
+  %incdec.ptr = getelementptr inbounds i8, i8* %gid.0, i64 1
+  %idxprom2114 = zext i32 %i.0 to i64
+  %arrayidx2115 = getelementptr inbounds [15 x i8*], [15 x i8*]* %ptrids, i64 0, i64 %idxprom2114
+  store i8* %gid.0, i8** %arrayidx2115, align 8
+  %inc2117 = add nuw nsw i32 %i.0, 1
+  br label %for.cond2106
+}
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/noop-ptrint-pair.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/noop-ptrint-pair.ll
new file mode 100644
index 0000000000000..24cab4f7bf6ea
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/noop-ptrint-pair.ll
@@ -0,0 +1,101 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -o - -infer-address-spaces %s | FileCheck -check-prefixes=COMMON,AMDGCN %s
+; RUN: opt -S -o - -infer-address-spaces -assume-default-is-flat-addrspace %s | FileCheck -check-prefixes=COMMON,NOTTI %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7"
+
+; COMMON-LABEL: @noop_ptrint_pair(
+; AMDGCN-NEXT: store i32 0, i32 addrspace(1)* %{{.*}}
+; AMDGCN-NEXT: ret void
+; NOTTI-NEXT: %1 = ptrtoint i32 addrspace(1)* %x.coerce to i64
+; NOTTI-NEXT: %2 = inttoptr i64 %1 to i32*
+; NOTTI-NEXT: store i32 0, i32* %2
+; NOTTI-NEXT: ret void
+define void @noop_ptrint_pair(i32 addrspace(1)* %x.coerce) {
+  %1 = ptrtoint i32 addrspace(1)* %x.coerce to i64
+  %2 = inttoptr i64 %1 to i32*
+  store i32 0, i32* %2
+  ret void
+}
+
+; COMMON-LABEL: @non_noop_ptrint_pair(
+; AMDGCN-NEXT: ptrtoint i32 addrspace(3)* %{{.*}} to i64
+; AMDGCN-NEXT: inttoptr i64 %{{.*}} to i32*
+; AMDGCN-NEXT: store i32 0, i32* %{{.*}}
+; AMDGCN-NEXT: ret void
+; NOTTI-NEXT: ptrtoint i32 addrspace(3)* %{{.*}} to i64
+; NOTTI-NEXT: inttoptr i64 %{{.*}} to i32*
+; NOTTI-NEXT: store i32 0, i32* %{{.*}}
+; NOTTI-NEXT: ret void
+define void @non_noop_ptrint_pair(i32 addrspace(3)* %x.coerce) {
+  %1 = ptrtoint i32 addrspace(3)* %x.coerce to i64
+  %2 = inttoptr i64 %1 to i32*
+  store i32 0, i32* %2
+  ret void
+}
+
+; COMMON-LABEL: @non_noop_ptrint_pair2(
+; AMDGCN-NEXT: ptrtoint i32 addrspace(1)* %{{.*}} to i32
+; AMDGCN-NEXT: inttoptr i32 %{{.*}} to i32*
+; AMDGCN-NEXT: store i32 0, i32* %{{.*}}
+; AMDGCN-NEXT: ret void
+; NOTTI-NEXT: ptrtoint i32 addrspace(1)* %{{.*}} to i32
+; NOTTI-NEXT: inttoptr i32 %{{.*}} to i32*
+; NOTTI-NEXT: store i32 0, i32* %{{.*}}
+; NOTTI-NEXT: ret void
+define void @non_noop_ptrint_pair2(i32 addrspace(1)* %x.coerce) {
+  %1 = ptrtoint i32 addrspace(1)* %x.coerce to i32
+  %2 = inttoptr i32 %1 to i32*
+  store i32 0, i32* %2
+  ret void
+}
+
+@g = addrspace(1) global i32 0, align 4
+@l = addrspace(3) global i32 0, align 4
+
+; COMMON-LABEL: @noop_ptrint_pair_ce(
+; AMDGCN-NEXT: store i32 0, i32 addrspace(1)* @g
+; AMDGCN-NEXT: ret void
+; NOTTI-NEXT: store i32 0, i32* inttoptr (i64 ptrtoint (i32 addrspace(1)* @g to i64) to i32*)
+; NOTTI-NEXT: ret void
+define void @noop_ptrint_pair_ce() {
+  store i32 0, i32* inttoptr (i64 ptrtoint (i32 addrspace(1)* @g to i64) to i32*)
+  ret void
+}
+
+; COMMON-LABEL: @noop_ptrint_pair_ce2(
+; AMDGCN-NEXT: ret i32* addrspacecast (i32 addrspace(1)* @g to i32*)
+; NOTTI-NEXT: ret i32* inttoptr (i64 ptrtoint (i32 addrspace(1)* @g to i64) to i32*)
+define i32* @noop_ptrint_pair_ce2() {
+  ret i32* inttoptr (i64 ptrtoint (i32 addrspace(1)* @g to i64) to i32*)
+}
+
+; COMMON-LABEL: @non_noop_ptrint_pair_ce(
+; AMDGCN-NEXT: store i32 0, i32* inttoptr (i64 ptrtoint (i32 addrspace(3)* @l to i64) to i32*)
+; AMDGCN-NEXT: ret void
+; NOTTI-NEXT: store i32 0, i32* inttoptr (i64 ptrtoint (i32 addrspace(3)* @l to i64) to i32*)
+; NOTTI-NEXT: ret void
+define void @non_noop_ptrint_pair_ce() {
+  store i32 0, i32* inttoptr (i64 ptrtoint (i32 addrspace(3)* @l to i64) to i32*)
+  ret void
+}
+
+; COMMON-LABEL: @non_noop_ptrint_pair_ce2(
+; AMDGCN-NEXT: ret i32* inttoptr (i64 ptrtoint (i32 addrspace(3)* @l to i64) to i32*)
+; NOTTI-NEXT: ret i32* inttoptr (i64 ptrtoint (i32 addrspace(3)* @l to i64) to i32*)
+define i32* @non_noop_ptrint_pair_ce2() {
+  ret i32* inttoptr (i64 ptrtoint (i32 addrspace(3)* @l to i64) to i32*)
+}
+
+; COMMON-LABEL: @non_noop_ptrint_pair_ce3(
+; AMDGCN-NEXT: ret i32* inttoptr (i32 ptrtoint (i32 addrspace(1)* @g to i32) to i32*)
+; NOTTI-NEXT: ret i32* inttoptr (i32 ptrtoint (i32 addrspace(1)* @g to i32) to i32*)
+define i32* @non_noop_ptrint_pair_ce3() {
+  ret i32* inttoptr (i32 ptrtoint (i32 addrspace(1)* @g to i32) to i32*)
+}
+
+; COMMON-LABEL: @non_noop_ptrint_pair_ce4(
+; AMDGCN-NEXT: ret i32* inttoptr (i128 ptrtoint (i32 addrspace(3)* @l to i128) to i32*)
+; NOTTI-NEXT: ret i32* inttoptr (i128 ptrtoint (i32 addrspace(3)* @l to i128) to i32*)
+define i32* @non_noop_ptrint_pair_ce4() {
+  ret i32* inttoptr (i128 ptrtoint (i32 addrspace(3)* @l to i128) to i32*)
+}
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/select.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/select.ll
index 1fa4bdc1964e8..3acd21c739585 100644
--- a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/select.ll
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/select.ll
@@ -221,7 +221,7 @@ define amdgpu_kernel void @store_select_group_global_mismatch_inttoptr_flat_null
 }
 
 ; CHECK-LABEL: @store_select_group_global_mismatch_undef_undef_constexpr(
-; CHECK: store i32 7, i32 addrspace(3)* null
+; CHECK: store i32 7, i32* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32* addrspacecast (i32 addrspace(3)* null to i32*), i32* undef), align 4
 define amdgpu_kernel void @store_select_group_global_mismatch_undef_undef_constexpr() #0 {
   store i32 7, i32* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32* addrspacecast (i32 addrspace(3)* null to i32*), i32* addrspacecast (i32 addrspace(1)* undef to i32*)), align 4
   ret void
diff --git a/llvm/test/Transforms/Inline/ML/Inputs/test-module.ll b/llvm/test/Transforms/Inline/ML/Inputs/test-module.ll
new file mode 100644
index 0000000000000..b8279e5db6a02
--- /dev/null
+++ b/llvm/test/Transforms/Inline/ML/Inputs/test-module.ll
@@ -0,0 +1,64 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+declare void @external_fct(i32)
+
+define dso_local i32 @top() {
+  %a = call i32 @multiplier(i32 5)
+  %b = call i32 @adder(i32 10)
+  %ret = add nsw i32 %a, %b
+  call void @external_fct(i32 %ret)
+  ret i32 %ret
+}
+
+define internal dso_local i32 @adder(i32) {
+  %2 = alloca i32, align 4
+  store i32 %0, i32* %2, align 4
+  %3 = load i32, i32* %2, align 4
+  %4 = call i32 @multiplier(i32 %3)
+  %5 = load i32, i32* %2, align 4
+  %6 = call i32 @switcher(i32 1)
+  %7 = add nsw i32 %4, %6
+  ret i32 %7
+}
+
+define internal i32 @multiplier(i32) {
+  %2 = alloca i32, align 4
+  store i32 %0, i32* %2, align 4
+  %3 = load i32, i32* %2, align 4
+  %4 = load i32, i32* %2, align 4
+  %5 = mul nsw i32 %3, %4
+  ret i32 %5
+}
+
+define i32 @switcher(i32) {
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  store i32 %0, i32* %3, align 4
+  %4 = load i32, i32* %3, align 4
+  switch i32 %4, label %11 [
+    i32 1, label %5
+    i32 2, label %6
+  ]
+
+; <label>:5:                                      ; preds = %1
+  store i32 2, i32* %2, align 4
+  br label %12
+
+; <label>:6:                                      ; preds = %1
+  %7 = load i32, i32* %3, align 4
+  %8 = load i32, i32* %3, align 4
+  %9 = call i32 @multiplier(i32 %8)
+  %10 = add nsw i32 %7, %9
+  store i32 %10, i32* %2, align 4
+  br label %12
+
+; <label>:11:                                     ; preds = %1
+  %adder.result = call i32 @adder(i32 2)
+  store i32 %adder.result, i32* %2, align 4
+  br label %12
+
+; <label>:12:                                     ; preds = %11, %6, %5
+  %13 = load i32, i32* %2, align 4
+  ret i32 %13
+}
\ No newline at end of file
diff --git a/llvm/test/Transforms/Inline/ML/bounds-checks.ll b/llvm/test/Transforms/Inline/ML/bounds-checks.ll
new file mode 100644
index 0000000000000..b98e23acb4270
--- /dev/null
+++ b/llvm/test/Transforms/Inline/ML/bounds-checks.ll
@@ -0,0 +1,41 @@
+; Test behavior when inlining policy grows size out of control.
+; In all cases, the end result is the same: mandatory inlinings must happen.
+; However, when we discover we 'trip' over the artificially-low size increase 
+; factor, we don't inline anymore.
+; REQUIRES: have_tf_aot
+; RUN: opt -passes=scc-oz-module-inliner -enable-ml-inliner=release -ml-advisor-size-increase-threshold=10.0 -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=NOBOUNDS
+; RUN: opt -passes=scc-oz-module-inliner -enable-ml-inliner=release -ml-advisor-size-increase-threshold=1.0 -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=BOUNDS
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+declare i64 @f1()
+
+define i64 @f2() #0 {
+  %r = call i64 @f1()
+  %r2 = add i64 13, %r
+  ret i64 %r2
+}
+
+define i64 @some_function() {
+  %r = call i64 @f1()
+  %r2 = add i64 13, %r
+  ret i64 %r2
+}
+
+define i64 @top() {
+  %r = call i64 @f2()
+  %r2 = call i64 @some_function()
+  %r3 = add i64 %r, %r2
+  ret i64 %r3
+}
+
+attributes #0 = { alwaysinline }
+
+; CHECK-LABEL: @top
+; f2 must always be inlined, so we won't find a call to it in @top()
+; CHECK-NOT: call i64 @f2
+; @some-function isn't mandatory, and when we set the increase threshold too low,
+; it won't be inlined.
+; NOBOUNDS-NOT: @some_function
+; BOUNDS: call i64 @some_function
\ No newline at end of file
diff --git a/llvm/test/Transforms/Inline/ML/ml-test-release-mode.ll b/llvm/test/Transforms/Inline/ML/ml-test-release-mode.ll
new file mode 100644
index 0000000000000..1ac0efd3cd0d4
--- /dev/null
+++ b/llvm/test/Transforms/Inline/ML/ml-test-release-mode.ll
@@ -0,0 +1,14 @@
+; The default inliner doesn't elide @adder, it believes it's too costly to inline 
+; adder into switcher. The ML inliner carries out that inlining, resulting in
+; a smaller result (part of it is that adder gets elided).
+;
+; This test uses Inputs/test-module.ll, as it will share it with a similar test
+; for the 'development' mode.
+;
+; REQUIRES: have_tf_aot
+; RUN: opt -passes=scc-oz-module-inliner -enable-ml-inliner=release -S < %S/Inputs/test-module.ll 2>&1 | FileCheck %s --check-prefix=CHECK
+; RUN: opt -passes=scc-oz-module-inliner -enable-ml-inliner=default -S < %S/Inputs/test-module.ll 2>&1 | FileCheck %s --check-prefix=DEFAULT
+
+; CHECK-NOT: @adder
+; DEFAULT-LABEL:        @adder
+; DEFAULT-NEXT:         %2 = mul
\ No newline at end of file
diff --git a/llvm/test/Transforms/Inline/align.ll b/llvm/test/Transforms/Inline/align.ll
index c91fe8046174e..ede6c3fa7bcf4 100644
--- a/llvm/test/Transforms/Inline/align.ll
+++ b/llvm/test/Transforms/Inline/align.ll
@@ -1,8 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
 ; RUN: opt -inline -preserve-alignment-assumptions-during-inlining -S < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @hello(float* align 128 nocapture %a, float* nocapture readonly %c) #0 {
+; CHECK-LABEL: define {{[^@]+}}@hello
+; CHECK-SAME: (float* nocapture align 128 [[A:%.*]], float* nocapture readonly [[C:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[C]], align 4
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[A]], i64 5
+; CHECK-NEXT:    store float [[TMP0]], float* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %0 = load float, float* %c, align 4
   %arrayidx = getelementptr inbounds float, float* %a, i64 5
@@ -11,6 +20,21 @@ entry:
 }
 
 define void @foo(float* nocapture %a, float* nocapture readonly %c) #0 {
+; CHECK-LABEL: define {{[^@]+}}@foo
+; CHECK-SAME: (float* nocapture [[A:%.*]], float* nocapture readonly [[C:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint float* [[A]] to i64
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 127
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[C]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_I:%.*]] = getelementptr inbounds float, float* [[A]], i64 5
+; CHECK-NEXT:    store float [[TMP0]], float* [[ARRAYIDX_I]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[C]], align 4
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[A]], i64 7
+; CHECK-NEXT:    store float [[TMP1]], float* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   tail call void @hello(float* %a, float* %c)
   %0 = load float, float* %c, align 4
@@ -19,22 +43,18 @@ entry:
   ret void
 }
 
-; CHECK: define void @foo(float* nocapture %a, float* nocapture readonly %c) #0 {
-; CHECK: entry:
-; CHECK:   %ptrint = ptrtoint float* %a to i64
-; CHECK:   %maskedptr = and i64 %ptrint, 127
-; CHECK:   %maskcond = icmp eq i64 %maskedptr, 0
-; CHECK:   call void @llvm.assume(i1 %maskcond)
-; CHECK:   %0 = load float, float* %c, align 4
-; CHECK:   %arrayidx.i = getelementptr inbounds float, float* %a, i64 5
-; CHECK:   store float %0, float* %arrayidx.i, align 4
-; CHECK:   %1 = load float, float* %c, align 4
-; CHECK:   %arrayidx = getelementptr inbounds float, float* %a, i64 7
-; CHECK:   store float %1, float* %arrayidx, align 4
-; CHECK:   ret void
-; CHECK: }
-
 define void @fooa(float* nocapture align 128 %a, float* nocapture readonly %c) #0 {
+; CHECK-LABEL: define {{[^@]+}}@fooa
+; CHECK-SAME: (float* nocapture align 128 [[A:%.*]], float* nocapture readonly [[C:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[C]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_I:%.*]] = getelementptr inbounds float, float* [[A]], i64 5
+; CHECK-NEXT:    store float [[TMP0]], float* [[ARRAYIDX_I]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[C]], align 4
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[A]], i64 7
+; CHECK-NEXT:    store float [[TMP1]], float* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   tail call void @hello(float* %a, float* %c)
   %0 = load float, float* %c, align 4
@@ -43,18 +63,17 @@ entry:
   ret void
 }
 
-; CHECK: define void @fooa(float* nocapture align 128 %a, float* nocapture readonly %c) #0 {
-; CHECK: entry:
-; CHECK:   %0 = load float, float* %c, align 4
-; CHECK:   %arrayidx.i = getelementptr inbounds float, float* %a, i64 5
-; CHECK:   store float %0, float* %arrayidx.i, align 4
-; CHECK:   %1 = load float, float* %c, align 4
-; CHECK:   %arrayidx = getelementptr inbounds float, float* %a, i64 7
-; CHECK:   store float %1, float* %arrayidx, align 4
-; CHECK:   ret void
-; CHECK: }
-
 define void @hello2(float* align 128 nocapture %a, float* align 128 nocapture %b, float* nocapture readonly %c) #0 {
+; CHECK-LABEL: define {{[^@]+}}@hello2
+; CHECK-SAME: (float* nocapture align 128 [[A:%.*]], float* nocapture align 128 [[B:%.*]], float* nocapture readonly [[C:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[C]], align 4
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[A]], i64 5
+; CHECK-NEXT:    store float [[TMP0]], float* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, float* [[B]], i64 8
+; CHECK-NEXT:    store float [[TMP0]], float* [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %0 = load float, float* %c, align 4
   %arrayidx = getelementptr inbounds float, float* %a, i64 5
@@ -65,6 +84,27 @@ entry:
 }
 
 define void @foo2(float* nocapture %a, float* nocapture %b, float* nocapture readonly %c) #0 {
+; CHECK-LABEL: define {{[^@]+}}@foo2
+; CHECK-SAME: (float* nocapture [[A:%.*]], float* nocapture [[B:%.*]], float* nocapture readonly [[C:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint float* [[A]] to i64
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 127
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    [[PTRINT1:%.*]] = ptrtoint float* [[B]] to i64
+; CHECK-NEXT:    [[MASKEDPTR2:%.*]] = and i64 [[PTRINT1]], 127
+; CHECK-NEXT:    [[MASKCOND3:%.*]] = icmp eq i64 [[MASKEDPTR2]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND3]])
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[C]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_I:%.*]] = getelementptr inbounds float, float* [[A]], i64 5
+; CHECK-NEXT:    store float [[TMP0]], float* [[ARRAYIDX_I]], align 4
+; CHECK-NEXT:    [[ARRAYIDX1_I:%.*]] = getelementptr inbounds float, float* [[B]], i64 8
+; CHECK-NEXT:    store float [[TMP0]], float* [[ARRAYIDX1_I]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[C]], align 4
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[A]], i64 7
+; CHECK-NEXT:    store float [[TMP1]], float* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   tail call void @hello2(float* %a, float* %b, float* %c)
   %0 = load float, float* %c, align 4
@@ -73,26 +113,5 @@ entry:
   ret void
 }
 
-; CHECK: define void @foo2(float* nocapture %a, float* nocapture %b, float* nocapture readonly %c) #0 {
-; CHECK: entry:
-; CHECK:   %ptrint = ptrtoint float* %a to i64
-; CHECK:   %maskedptr = and i64 %ptrint, 127
-; CHECK:   %maskcond = icmp eq i64 %maskedptr, 0
-; CHECK:   call void @llvm.assume(i1 %maskcond)
-; CHECK:   %ptrint1 = ptrtoint float* %b to i64
-; CHECK:   %maskedptr2 = and i64 %ptrint1, 127
-; CHECK:   %maskcond3 = icmp eq i64 %maskedptr2, 0
-; CHECK:   call void @llvm.assume(i1 %maskcond3)
-; CHECK:   %0 = load float, float* %c, align 4
-; CHECK:   %arrayidx.i = getelementptr inbounds float, float* %a, i64 5
-; CHECK:   store float %0, float* %arrayidx.i, align 4
-; CHECK:   %arrayidx1.i = getelementptr inbounds float, float* %b, i64 8
-; CHECK:   store float %0, float* %arrayidx1.i, align 4
-; CHECK:   %1 = load float, float* %c, align 4
-; CHECK:   %arrayidx = getelementptr inbounds float, float* %a, i64 7
-; CHECK:   store float %1, float* %arrayidx, align 4
-; CHECK:   ret void
-; CHECK: }
-
 attributes #0 = { nounwind uwtable }
 
diff --git a/llvm/test/Transforms/Inline/always-inline.ll b/llvm/test/Transforms/Inline/always-inline.ll
index 791eb94779b70..81cbb73fbf59d 100644
--- a/llvm/test/Transforms/Inline/always-inline.ll
+++ b/llvm/test/Transforms/Inline/always-inline.ll
@@ -212,7 +212,7 @@ entry:
 ; CHECK: ret void
 }
 
-; The 'inner10' and 'outer10' functions test a frustrating consquence of the
+; The 'inner10' and 'outer10' functions test a frustrating consequence of the
 ; current 'alwaysinline' semantic model. Because such functions are allowed to
 ; be external functions, it may be necessary to both inline all of their uses
 ; and leave them in the final output. These tests can be removed if and when
diff --git a/llvm/test/Transforms/Inline/attributes.ll b/llvm/test/Transforms/Inline/attributes.ll
index 5f27ac5551292..07d1b36e485b2 100644
--- a/llvm/test/Transforms/Inline/attributes.ll
+++ b/llvm/test/Transforms/Inline/attributes.ll
@@ -440,9 +440,177 @@ define i32 @test_null-pointer-is-valid2(i32 %i) null_pointer_is_valid {
 ; CHECK-NEXT: ret i32
 }
 
+define i32 @no-infs-fp-math_callee0(i32 %i) "no-infs-fp-math"="false" {
+  ret i32 %i
+; CHECK: @no-infs-fp-math_callee0(i32 %i) [[NO_INFS_FPMATH_FALSE:#[0-9]+]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @no-infs-fp-math_callee1(i32 %i) "no-infs-fp-math"="true" {
+  ret i32 %i
+; CHECK: @no-infs-fp-math_callee1(i32 %i) [[NO_INFS_FPMATH_TRUE:#[0-9]+]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-infs-fp-math0(i32 %i) "no-infs-fp-math"="false" {
+  %1 = call i32 @no-infs-fp-math_callee0(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-infs-fp-math0(i32 %i) [[NO_INFS_FPMATH_FALSE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-infs-fp-math1(i32 %i) "no-infs-fp-math"="false" {
+  %1 = call i32 @no-infs-fp-math_callee1(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-infs-fp-math1(i32 %i) [[NO_INFS_FPMATH_FALSE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-infs-fp-math2(i32 %i) "no-infs-fp-math"="true" {
+  %1 = call i32 @no-infs-fp-math_callee0(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-infs-fp-math2(i32 %i) [[NO_INFS_FPMATH_FALSE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-infs-fp-math3(i32 %i) "no-infs-fp-math"="true" {
+  %1 = call i32 @no-infs-fp-math_callee1(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-infs-fp-math3(i32 %i) [[NO_INFS_FPMATH_TRUE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @no-nans-fp-math_callee0(i32 %i) "no-nans-fp-math"="false" {
+  ret i32 %i
+; CHECK: @no-nans-fp-math_callee0(i32 %i) [[NO_NANS_FPMATH_FALSE:#[0-9]+]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @no-nans-fp-math_callee1(i32 %i) "no-nans-fp-math"="true" {
+  ret i32 %i
+; CHECK: @no-nans-fp-math_callee1(i32 %i) [[NO_NANS_FPMATH_TRUE:#[0-9]+]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-nans-fp-math0(i32 %i) "no-nans-fp-math"="false" {
+  %1 = call i32 @no-nans-fp-math_callee0(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-nans-fp-math0(i32 %i) [[NO_NANS_FPMATH_FALSE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-nans-fp-math1(i32 %i) "no-nans-fp-math"="false" {
+  %1 = call i32 @no-nans-fp-math_callee1(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-nans-fp-math1(i32 %i) [[NO_NANS_FPMATH_FALSE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-nans-fp-math2(i32 %i) "no-nans-fp-math"="true" {
+  %1 = call i32 @no-nans-fp-math_callee0(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-nans-fp-math2(i32 %i) [[NO_NANS_FPMATH_FALSE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-nans-fp-math3(i32 %i) "no-nans-fp-math"="true" {
+  %1 = call i32 @no-nans-fp-math_callee1(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-nans-fp-math3(i32 %i) [[NO_NANS_FPMATH_TRUE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @no-signed-zeros-fp-math_callee0(i32 %i) "no-signed-zeros-fp-math"="false" {
+  ret i32 %i
+; CHECK: @no-signed-zeros-fp-math_callee0(i32 %i) [[NO_SIGNED_ZEROS_FPMATH_FALSE:#[0-9]+]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @no-signed-zeros-fp-math_callee1(i32 %i) "no-signed-zeros-fp-math"="true" {
+  ret i32 %i
+; CHECK: @no-signed-zeros-fp-math_callee1(i32 %i) [[NO_SIGNED_ZEROS_FPMATH_TRUE:#[0-9]+]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-signed-zeros-fp-math0(i32 %i) "no-signed-zeros-fp-math"="false" {
+  %1 = call i32 @no-signed-zeros-fp-math_callee0(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-signed-zeros-fp-math0(i32 %i) [[NO_SIGNED_ZEROS_FPMATH_FALSE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-signed-zeros-fp-math1(i32 %i) "no-signed-zeros-fp-math"="false" {
+  %1 = call i32 @no-signed-zeros-fp-math_callee1(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-signed-zeros-fp-math1(i32 %i) [[NO_SIGNED_ZEROS_FPMATH_FALSE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-signed-zeros-fp-math2(i32 %i) "no-signed-zeros-fp-math"="true" {
+  %1 = call i32 @no-signed-zeros-fp-math_callee0(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-signed-zeros-fp-math2(i32 %i) [[NO_SIGNED_ZEROS_FPMATH_FALSE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-signed-zeros-fp-math3(i32 %i) "no-signed-zeros-fp-math"="true" {
+  %1 = call i32 @no-signed-zeros-fp-math_callee1(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-signed-zeros-fp-math3(i32 %i) [[NO_SIGNED_ZEROS_FPMATH_TRUE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @unsafe-fp-math_callee0(i32 %i) "unsafe-fp-math"="false" {
+  ret i32 %i
+; CHECK: @unsafe-fp-math_callee0(i32 %i) [[UNSAFE_FPMATH_FALSE:#[0-9]+]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @unsafe-fp-math_callee1(i32 %i) "unsafe-fp-math"="true" {
+  ret i32 %i
+; CHECK: @unsafe-fp-math_callee1(i32 %i) [[UNSAFE_FPMATH_TRUE:#[0-9]+]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_unsafe-fp-math0(i32 %i) "unsafe-fp-math"="false" {
+  %1 = call i32 @unsafe-fp-math_callee0(i32 %i)
+  ret i32 %1
+; CHECK: @test_unsafe-fp-math0(i32 %i) [[UNSAFE_FPMATH_FALSE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_unsafe-fp-math1(i32 %i) "unsafe-fp-math"="false" {
+  %1 = call i32 @unsafe-fp-math_callee1(i32 %i)
+  ret i32 %1
+; CHECK: @test_unsafe-fp-math1(i32 %i) [[UNSAFE_FPMATH_FALSE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_unsafe-fp-math2(i32 %i) "unsafe-fp-math"="true" {
+  %1 = call i32 @unsafe-fp-math_callee0(i32 %i)
+  ret i32 %1
+; CHECK: @test_unsafe-fp-math2(i32 %i) [[UNSAFE_FPMATH_FALSE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_unsafe-fp-math3(i32 %i) "unsafe-fp-math"="true" {
+  %1 = call i32 @unsafe-fp-math_callee1(i32 %i)
+  ret i32 %1
+; CHECK: @test_unsafe-fp-math3(i32 %i) [[UNSAFE_FPMATH_TRUE]] {
+; CHECK-NEXT: ret i32
+}
+
 ; CHECK: attributes [[SLH]] = { speculative_load_hardening }
 ; CHECK: attributes [[FPMAD_FALSE]] = { "less-precise-fpmad"="false" }
 ; CHECK: attributes [[FPMAD_TRUE]] = { "less-precise-fpmad"="true" }
 ; CHECK: attributes [[NOIMPLICITFLOAT]] = { noimplicitfloat }
 ; CHECK: attributes [[NOUSEJUMPTABLES]] = { "no-jump-tables"="true" }
 ; CHECK: attributes [[NULLPOINTERISVALID]] = { null_pointer_is_valid }
+; CHECK: attributes [[NO_INFS_FPMATH_FALSE]] = { "no-infs-fp-math"="false" }
+; CHECK: attributes [[NO_INFS_FPMATH_TRUE]] = { "no-infs-fp-math"="true" }
+; CHECK: attributes [[NO_NANS_FPMATH_FALSE]] = { "no-nans-fp-math"="false" }
+; CHECK: attributes [[NO_NANS_FPMATH_TRUE]] = { "no-nans-fp-math"="true" }
+; CHECK: attributes [[NO_SIGNED_ZEROS_FPMATH_FALSE]] = { "no-signed-zeros-fp-math"="false" }
+; CHECK: attributes [[NO_SIGNED_ZEROS_FPMATH_TRUE]] = { "no-signed-zeros-fp-math"="true" }
+; CHECK: attributes [[UNSAFE_FPMATH_FALSE]] = { "unsafe-fp-math"="false" }
+; CHECK: attributes [[UNSAFE_FPMATH_TRUE]] = { "unsafe-fp-math"="true" }
diff --git a/llvm/test/Transforms/Inline/byval-align.ll b/llvm/test/Transforms/Inline/byval-align.ll
new file mode 100644
index 0000000000000..47887606e901f
--- /dev/null
+++ b/llvm/test/Transforms/Inline/byval-align.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
+; RUN: opt -inline -preserve-alignment-assumptions-during-inlining -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Test behavior of inserted alignment assumptions with byval. No
+; assume should be inserted.
+define void @byval_callee(float* byval(float) align 128 nocapture %a, float* %b) #0 {
+; CHECK-LABEL: define {{[^@]+}}@byval_callee
+; CHECK-SAME: (float* nocapture byval(float) align 128 [[A:%.*]], float* [[B:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOAD:%.*]] = load float, float* [[A]], align 4
+; CHECK-NEXT:    [[B_IDX:%.*]] = getelementptr inbounds float, float* [[B]], i64 8
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[LOAD]], 2.000000e+00
+; CHECK-NEXT:    store float [[ADD]], float* [[B_IDX]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %load = load float, float* %a, align 4
+  %b.idx = getelementptr inbounds float, float* %b, i64 8
+  %add = fadd float %load, 2.0
+  store float %add, float* %b.idx, align 4
+  ret void
+}
+
+define void @byval_caller(float* nocapture align 64 %a, float* %b) #0 {
+; CHECK-LABEL: define {{[^@]+}}@byval_caller
+; CHECK-SAME: (float* nocapture align 64 [[A:%.*]], float* [[B:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A1:%.*]] = alloca float, align 128
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[A1]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[A1]] to i8*
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[A]] to i8*
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 [[TMP1]], i8* align 1 [[TMP2]], i64 4, i1 false)
+; CHECK-NEXT:    [[LOAD_I:%.*]] = load float, float* [[A1]], align 4
+; CHECK-NEXT:    [[B_IDX_I:%.*]] = getelementptr inbounds float, float* [[B]], i64 8
+; CHECK-NEXT:    [[ADD_I:%.*]] = fadd float [[LOAD_I]], 2.000000e+00
+; CHECK-NEXT:    store float [[ADD_I]], float* [[B_IDX_I]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[A1]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP3]])
+; CHECK-NEXT:    [[CALLER_LOAD:%.*]] = load float, float* [[B]], align 4
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[A]], i64 7
+; CHECK-NEXT:    store float [[CALLER_LOAD]], float* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @byval_callee(float* byval(float) align 128 %a, float* %b)
+  %caller.load = load float, float* %b, align 4
+  %arrayidx = getelementptr inbounds float, float* %a, i64 7
+  store float %caller.load, float* %arrayidx, align 4
+  ret void
+}
+
+attributes #0 = { nounwind uwtable }
diff --git a/llvm/test/Transforms/Inline/byval-tail-call.ll b/llvm/test/Transforms/Inline/byval-tail-call.ll
index be495f1bcd379..1a3cdc558e6b9 100644
--- a/llvm/test/Transforms/Inline/byval-tail-call.ll
+++ b/llvm/test/Transforms/Inline/byval-tail-call.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -tailcallelim -inline -instcombine -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -tailcallelim -inline -instcombine -dse -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes='function(tailcallelim),cgscc(inline,function(instcombine,dse))' -S | FileCheck %s
 ; PR7272
 
diff --git a/llvm/test/Transforms/Inline/debuginline-cost-delta.ll b/llvm/test/Transforms/Inline/debuginline-cost-delta.ll
index 2ef604d05bc35..2e6a8f4520797 100644
--- a/llvm/test/Transforms/Inline/debuginline-cost-delta.ll
+++ b/llvm/test/Transforms/Inline/debuginline-cost-delta.ll
@@ -1,32 +1,16 @@
-; Require asserts for -debug-only
-; REQUIRES: asserts
-
-; RUN: opt < %s -inline -debug-only=inline-cost -disable-output -print-instruction-comments 2>&1 | FileCheck %s
+; RUN: opt < %s -passes="print<inline-cost>" 2>&1 | FileCheck %s
 
 ; CHECK:       Analyzing call of callee1... (caller:foo)
-; CHECK: define i32 @callee1(i32 %x) {
-; CHECK: ; cost before = {{.*}}, cost after = {{.*}}, threshold before = {{.*}}, threshold after = {{.*}}, cost delta = 5
-; CHECK:   %x1 = add i32 %x, 1
-; CHECK: ; cost before = {{.*}}, cost after = {{.*}}, threshold before = {{.*}}, threshold after = {{.*}}, cost delta = 5
-; CHECK:   %x2 = add i32 %x1, 1
-; CHECK: ; cost before = {{.*}}, cost after = {{.*}}, threshold before = {{.*}}, threshold after = {{.*}}, cost delta = 5
-; CHECK:   %x3 = add i32 %x2, 1
-; CHECK: ; cost before = {{.*}}, cost after = {{.*}}, threshold before = {{.*}}, threshold after = {{.*}}, cost delta = 0
-; CHECK:   ret i32 %x3
-; CHECK: }
-; CHECK:      NumConstantArgs: 0
-; CHECK:      NumConstantOffsetPtrArgs: 0
-; CHECK:      NumAllocaArgs: 0
-; CHECK:      NumConstantPtrCmps: 0
-; CHECK:      NumConstantPtrDiffs: 0
-; CHECK:      NumInstructionsSimplified: 1
-; CHECK:      NumInstructions: 4
-; CHECK:      SROACostSavings: 0
-; CHECK:      SROACostSavingsLost: 0
-; CHECK:      LoadEliminationCost: 0
-; CHECK:      ContainsNoDuplicateCall: 0
-; CHECK:      Cost: {{.*}}
-; CHECK:      Threshold: {{.*}}
+; CHECK-NEXT: define i32 @callee1(i32 %x) {
+; CHECK-NEXT: cost before = {{.*}}, cost after = {{.*}}, threshold before = {{.*}}, threshold after = {{.*}}, cost delta = {{.*}}
+; CHECK-NEXT:   %x1 = add i32 %x, 1
+; CHECK-NEXT: cost before = {{.*}}, cost after = {{.*}}, threshold before = {{.*}}, threshold after = {{.*}}, cost delta = {{.*}}
+; CHECK-NEXT:   %x2 = add i32 %x1, 1
+; CHECK-NEXT: cost before = {{.*}}, cost after = {{.*}}, threshold before = {{.*}}, threshold after = {{.*}}, cost delta = {{.*}}
+; CHECK-NEXT:   %x3 = add i32 %x2, 1
+; CHECK-NEXT: cost before = {{.*}}, cost after = {{.*}}, threshold before = {{.*}}, threshold after = {{.*}}, cost delta = {{.*}}
+; CHECK-NEXT:   ret i32 %x3
+; CHECK-NEXT: }
 
 define i32 @foo(i32 %y) {
   %x = call i32 @callee1(i32 %y)
diff --git a/llvm/test/Transforms/Inline/devirtualize-3.ll b/llvm/test/Transforms/Inline/devirtualize-3.ll
index 2a0a6d7f65ac6..e7d1073c69b02 100644
--- a/llvm/test/Transforms/Inline/devirtualize-3.ll
+++ b/llvm/test/Transforms/Inline/devirtualize-3.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -inline -S -sroa -gvn -instcombine < %s | FileCheck %s
+; RUN: opt -basic-aa -inline -S -sroa -gvn -instcombine < %s | FileCheck %s
 ; PR5009
 
 ; CHECK: define i32 @main() 
diff --git a/llvm/test/Transforms/Inline/dynamic-alloca-simplified-large.ll b/llvm/test/Transforms/Inline/dynamic-alloca-simplified-large.ll
new file mode 100644
index 0000000000000..11c38bfb1303a
--- /dev/null
+++ b/llvm/test/Transforms/Inline/dynamic-alloca-simplified-large.ll
@@ -0,0 +1,175 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -inline < %s -S -o - | FileCheck %s
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.15.0"
+
+define void @caller1(i8 *%p1, i1 %b) {
+; CHECK-LABEL: @caller1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i1 [[B:%.*]], true
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT:%.*]], label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    call void @callee(i8* [[P1:%.*]], i32 0, i32 -1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond = icmp eq i1 %b, true
+  br i1 %cond, label %exit, label %split
+
+split:
+  ; This path may be generated from CS splitting and never taken at runtime.
+  call void @callee(i8* %p1, i32 0, i32 -1)
+  br label %exit
+
+exit:
+  ret void
+}
+
+define  void @callee(i8* %p1, i32 %l1, i32 %l2) {
+entry:
+  %ext = zext i32 %l2 to i64
+  %vla = alloca float, i64 %ext, align 16
+  call void @extern_call(float* nonnull %vla) #3
+  ret void
+}
+
+
+define void @caller2_below_threshold(i8 *%p1, i1 %b) {
+; CHECK-LABEL: @caller2_below_threshold(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VLA_I:%.*]] = alloca float, i64 15000, align 16
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i1 [[B:%.*]], true
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT:%.*]], label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    [[SAVEDSTACK:%.*]] = call i8* @llvm.stacksave()
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[VLA_I]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 60000, i8* [[TMP0]])
+; CHECK-NEXT:    call void @extern_call(float* nonnull [[VLA_I]]) #2
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[VLA_I]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 60000, i8* [[TMP1]])
+; CHECK-NEXT:    call void @llvm.stackrestore(i8* [[SAVEDSTACK]])
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond = icmp eq i1 %b, true
+  br i1 %cond, label %exit, label %split
+
+split:
+  call void @callee(i8* %p1, i32 0, i32 15000)
+  br label %exit
+
+exit:
+  ret void
+}
+
+define  void @callee2_not_in_entry(i8* %p1, i32 %l1, i32 %l2) {
+entry:
+  %ext = zext i32 %l2 to i64
+  %c = icmp eq i32 %l1, 42
+  br i1 %c, label %bb2, label %bb3
+bb2:
+  %vla = alloca float, i64 %ext, align 16
+  call void @extern_call(float* nonnull %vla) #3
+  ret void
+bb3:
+  ret void
+}
+
+define void @caller3_alloca_not_in_entry(i8 *%p1, i1 %b) {
+; CHECK-LABEL: @caller3_alloca_not_in_entry(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i1 [[B:%.*]], true
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT:%.*]], label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond = icmp eq i1 %b, true
+  br i1 %cond, label %exit, label %split
+
+split:
+  call void @callee2_not_in_entry(i8* %p1, i32 0, i32 -1)
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @caller4_over_threshold(i8 *%p1, i1 %b, i32 %len) {
+; CHECK-LABEL: @caller4_over_threshold(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i1 [[B:%.*]], true
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT:%.*]], label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    call void @callee(i8* [[P1:%.*]], i32 0, i32 16500)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond = icmp eq i1 %b, true
+  br i1 %cond, label %exit, label %split
+
+split:
+  call void @callee(i8* %p1, i32 0, i32 16500)
+  br label %exit
+
+exit:
+  ret void
+}
+
+declare noalias i8* @malloc(i64)
+define i8* @stack_allocate(i32 %size) #2 {
+entry:
+  %cmp = icmp ult i32 %size, 100
+  %conv = zext i32 %size to i64
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %0 = alloca i8, i64 %conv, align 8
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %call = tail call i8* @malloc(i64 %conv) #3
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %retval.0 = phi i8* [ %0, %if.then ], [ %call, %if.end ]
+  ret i8* %retval.0
+}
+
+define i8* @test_stack_allocate_always(i32 %size) {
+; CHECK-LABEL: @test_stack_allocate_always(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SAVEDSTACK:%.*]] = call i8* @llvm.stacksave()
+; CHECK-NEXT:    [[CMP_I:%.*]] = icmp ult i32 [[SIZE:%.*]], 100
+; CHECK-NEXT:    [[CONV_I:%.*]] = zext i32 [[SIZE]] to i64
+; CHECK-NEXT:    br i1 [[CMP_I]], label [[IF_THEN_I:%.*]], label [[IF_END_I:%.*]]
+; CHECK:       if.then.i:
+; CHECK-NEXT:    [[TMP0:%.*]] = alloca i8, i64 [[CONV_I]], align 8
+; CHECK-NEXT:    br label [[STACK_ALLOCATE_EXIT:%.*]]
+; CHECK:       if.end.i:
+; CHECK-NEXT:    [[CALL_I:%.*]] = tail call i8* @malloc(i64 [[CONV_I]]) #2
+; CHECK-NEXT:    br label [[STACK_ALLOCATE_EXIT]]
+; CHECK:       stack_allocate.exit:
+; CHECK-NEXT:    [[RETVAL_0_I:%.*]] = phi i8* [ [[TMP0]], [[IF_THEN_I]] ], [ [[CALL_I]], [[IF_END_I]] ]
+; CHECK-NEXT:    call void @llvm.stackrestore(i8* [[SAVEDSTACK]])
+; CHECK-NEXT:    ret i8* [[RETVAL_0_I]]
+;
+entry:
+  %call = tail call i8* @stack_allocate(i32 %size)
+  ret i8* %call
+}
+
+declare void @extern_call(float*)
+
+attributes #1 = { argmemonly nounwind willreturn writeonly }
+attributes #2 = { alwaysinline }
+attributes #3 = { nounwind }
+
diff --git a/llvm/test/Transforms/Inline/gep_from_constant.ll b/llvm/test/Transforms/Inline/gep_from_constant.ll
new file mode 100644
index 0000000000000..eafb4710690b6
--- /dev/null
+++ b/llvm/test/Transforms/Inline/gep_from_constant.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -passes="print<inline-cost>" 2>&1 | FileCheck %s
+
+; CHECK-LABEL: @foo
+; CHECK: cost before = {{.*}}, cost after = {{.*}}, threshold before = {{.*}}, threshold after = {{.*}}, cost delta = {{.*}}, simplified to i8 addrspace(1)** getelementptr (i8 addrspace(1)*, i8 addrspace(1)** inttoptr (i64 754974720 to i8 addrspace(1)**), i64 5)
+
+define i8 addrspace(1)** @foo(i64 %0) {
+  %2 = inttoptr i64 754974720 to i8 addrspace(1)**
+  %3 = getelementptr i8 addrspace(1)*, i8 addrspace(1)** %2, i64 %0
+  ret i8 addrspace(1)** %3
+}
+
+define i8 addrspace(1)** @main() {
+  %1 = call i8 addrspace(1)** @foo(i64 5)
+  ret i8 addrspace(1)** %1
+}
diff --git a/llvm/test/Transforms/Inline/gvn-inline-iteration.ll b/llvm/test/Transforms/Inline/gvn-inline-iteration.ll
index b87c0609ea2be..e0f59af15cae0 100644
--- a/llvm/test/Transforms/Inline/gvn-inline-iteration.ll
+++ b/llvm/test/Transforms/Inline/gvn-inline-iteration.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -inline -gvn -S -max-cg-scc-iterations=1 < %s | FileCheck %s
+; RUN: opt -basic-aa -inline -gvn -S -max-cg-scc-iterations=1 < %s | FileCheck %s
 ; rdar://6295824 and PR6724
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/llvm/test/Transforms/Inline/inline-cost-annotation-pass.ll b/llvm/test/Transforms/Inline/inline-cost-annotation-pass.ll
new file mode 100644
index 0000000000000..73b8a0bb0c0b3
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-cost-annotation-pass.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -passes="print<inline-cost>" 2>&1 | FileCheck %s
+
+; CHECK:       Analyzing call of foo... (caller:main)
+; CHECK: define i8 addrspace(1)** @foo() {
+; CHECK:  cost before = {{.*}}, cost after = {{.*}}, threshold before = {{.*}}, threshold after = {{.*}}, cost delta = {{.*}}
+; CHECK:  %1 = inttoptr i64 754974720 to i8 addrspace(1)**
+; CHECK:  cost before = {{.*}}, cost after = {{.*}}, threshold before = {{.*}}, threshold after = {{.*}}, cost delta = {{.*}}
+; CHECK:  ret i8 addrspace(1)** %1
+; CHECK: }
+; CHECK:       NumConstantArgs: {{.*}}
+; CHECK:       NumConstantOffsetPtrArgs: {{.*}}
+; CHECK:       NumAllocaArgs: {{.*}}
+; CHECK:       NumConstantPtrCmps: {{.*}}
+; CHECK:       NumConstantPtrDiffs: {{.*}}
+; CHECK:       NumInstructionsSimplified: {{.*}}
+; CHECK:       NumInstructions: {{.*}}
+; CHECK:       SROACostSavings: {{.*}}
+; CHECK:       SROACostSavingsLost: {{.*}}
+; CHECK:       LoadEliminationCost: {{.*}}
+; CHECK:       ContainsNoDuplicateCall: {{.*}}
+; CHECK:       Cost: {{.*}}
+; CHECK:       Threshold: {{.*}}
+
+define i8 addrspace(1)** @foo() {
+  %1 = inttoptr i64 754974720 to i8 addrspace(1)**
+  ret i8 addrspace(1)** %1
+}
+
+define i8 addrspace(1)** @main() {
+  %1 = call i8 addrspace(1)** @foo()
+  ret i8 addrspace(1)** %1
+}
diff --git a/llvm/test/Transforms/Inline/inline-optnone.ll b/llvm/test/Transforms/Inline/inline-optnone.ll
index 9b99c4558ea05..475ef7692ce73 100644
--- a/llvm/test/Transforms/Inline/inline-optnone.ll
+++ b/llvm/test/Transforms/Inline/inline-optnone.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s --passes=inline -S | FileCheck %s
 
 ; Test that functions with attribute optnone are not inlined.
 ; Also test that only functions with attribute alwaysinline are
diff --git a/llvm/test/Transforms/Inline/inlining-advisor-default.ll b/llvm/test/Transforms/Inline/inlining-advisor-default.ll
index 808ddd97dc6b6..7ee3e176bbeab 100644
--- a/llvm/test/Transforms/Inline/inlining-advisor-default.ll
+++ b/llvm/test/Transforms/Inline/inlining-advisor-default.ll
@@ -1,6 +1,6 @@
 ; Check that, in the absence of dependencies, we emit an error message when
 ; trying to use ML-driven inlining.
-;
+; REQUIRES: !have_tf_aot
 ; RUN: not opt -passes=scc-oz-module-inliner -enable-ml-inliner=development -S < %s 2>&1 | FileCheck %s
 ; RUN: not opt -passes=scc-oz-module-inliner -enable-ml-inliner=release -S < %s 2>&1 | FileCheck %s
 
diff --git a/llvm/test/Transforms/Inline/last-call-bonus.ll b/llvm/test/Transforms/Inline/last-call-bonus.ll
index 7de67861cf8ae..d1c466b1d765d 100644
--- a/llvm/test/Transforms/Inline/last-call-bonus.ll
+++ b/llvm/test/Transforms/Inline/last-call-bonus.ll
@@ -10,7 +10,7 @@
 ; preprocess the test.
 
 ; RUN: opt < %s -loop-unroll -inline -unroll-threshold=15000 -inline-threshold=250 -S | FileCheck %s
-; RUN: opt < %s -passes='function(require<opt-remark-emit>,unroll),require<profile-summary>,cgscc(inline)' -unroll-threshold=15000 -inline-threshold=250 -S | FileCheck %s
+; RUN: opt < %s -passes='function(require<opt-remark-emit>,loop-unroll),require<profile-summary>,cgscc(inline)' -unroll-threshold=15000 -inline-threshold=250 -S | FileCheck %s
 ; CHECK-LABEL: define internal i32 @bar()
 
 define internal i32 @baz() {
diff --git a/llvm/test/Transforms/Inline/last-call-no-bonus.ll b/llvm/test/Transforms/Inline/last-call-no-bonus.ll
index 14fe23734220a..7d0fae3984b26 100644
--- a/llvm/test/Transforms/Inline/last-call-no-bonus.ll
+++ b/llvm/test/Transforms/Inline/last-call-no-bonus.ll
@@ -2,7 +2,7 @@
 ; to the internal functions are cold, thereby preventing the last call to
 ; static bonus from being applied.
 
-; RUN: opt < %s -passes='function(require<opt-remark-emit>,unroll),require<profile-summary>,cgscc(inline)' -unroll-threshold=15000 -inline-threshold=250 -S | FileCheck %s
+; RUN: opt < %s -passes='function(require<opt-remark-emit>,loop-unroll),require<profile-summary>,cgscc(inline)' -unroll-threshold=15000 -inline-threshold=250 -S | FileCheck %s
 
 ; CHECK-LABEL: define internal i32 @baz
 define internal i32 @baz() {
diff --git a/llvm/test/Transforms/Inline/noalias-calls.ll b/llvm/test/Transforms/Inline/noalias-calls.ll
index 965f5ce923664..d93a6acbe4bd1 100644
--- a/llvm/test/Transforms/Inline/noalias-calls.ll
+++ b/llvm/test/Transforms/Inline/noalias-calls.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -inline -enable-noalias-to-md-conversion -S < %s | FileCheck %s
+; RUN: opt -basic-aa -inline -enable-noalias-to-md-conversion -S < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/Inline/print-instructions-deltas-unfinished.ll b/llvm/test/Transforms/Inline/print-instructions-deltas-unfinished.ll
deleted file mode 100644
index 70e223c951e3c..0000000000000
--- a/llvm/test/Transforms/Inline/print-instructions-deltas-unfinished.ll
+++ /dev/null
@@ -1,22 +0,0 @@
-; Require asserts for -debug-only
-; REQUIRES: asserts
-
-; This test ensures that the hadling of instructions which were not analyzed by
-; '-print-instruction-deltas' flag due to the early exit was done correctly.
-
-; RUN: opt < %s -inline -debug-only=inline-cost -disable-output -print-instruction-comments -inline-threshold=0 2>&1 | FileCheck %s
-
-; CHECK: No analysis for the instruction
-; CHECK:   ret void
-
-declare void @callee1()
-
-define void @bar() {
-  call void @callee1()
-  ret void
-}
-
-define void @foo() {
-  call void @bar()
-  ret void
-}
diff --git a/llvm/test/Transforms/Inline/simplified_to.ll b/llvm/test/Transforms/Inline/simplified_to.ll
new file mode 100644
index 0000000000000..92c84b586800c
--- /dev/null
+++ b/llvm/test/Transforms/Inline/simplified_to.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -passes="print<inline-cost>" 2>&1 | FileCheck %s
+
+; CHECK-LABEL: @test()
+; CHECK: cost before = {{.*}}, cost after = {{.*}}, threshold before = {{.*}}, threshold after = {{.*}}, cost delta = {{.*}}, simplified to i1 false
+; CHECK:   %1 = icmp eq i32 4, 5
+
+define i32 @test() {
+  %1 = icmp eq i32 4, 5
+  ret i32 0
+}
+
+define void @main() {
+  %1 = call i32 @test()
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-01-21-MulTrunc.ll b/llvm/test/Transforms/InstCombine/2008-01-21-MulTrunc.ll
index 87c2b75d2499a..89e4a3c1aaed6 100644
--- a/llvm/test/Transforms/InstCombine/2008-01-21-MulTrunc.ll
+++ b/llvm/test/Transforms/InstCombine/2008-01-21-MulTrunc.ll
@@ -1,17 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 
 define i16 @test1(i16 %a) {
-        %tmp = zext i16 %a to i32               ; <i32> [#uses=2]
-        %tmp21 = lshr i32 %tmp, 8               ; <i32> [#uses=1]
-; CHECK: %tmp21 = lshr i16 %a, 8
-        %tmp5 = mul i32 %tmp, 5         ; <i32> [#uses=1]
-; CHECK: %tmp5 = mul i16 %a, 5
-        %tmp.upgrd.32 = or i32 %tmp21, %tmp5            ; <i32> [#uses=1]
-; CHECK: %tmp.upgrd.32 = or i16 %tmp21, %tmp5
-        %tmp.upgrd.3 = trunc i32 %tmp.upgrd.32 to i16           ; <i16> [#uses=1]
-        ret i16 %tmp.upgrd.3
-; CHECK: ret i16 %tmp.upgrd.32
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[C:%.*]] = lshr i16 [[A:%.*]], 8
+; CHECK-NEXT:    [[D:%.*]] = mul i16 [[A]], 5
+; CHECK-NEXT:    [[E:%.*]] = or i16 [[C]], [[D]]
+; CHECK-NEXT:    ret i16 [[E]]
+;
+  %b = zext i16 %a to i32    ; <i32> [#uses=2]
+  %c = lshr i32 %b, 8        ; <i32> [#uses=1]
+  %d = mul i32 %b, 5         ; <i32> [#uses=1]
+  %e = or i32 %c, %d         ; <i32> [#uses=1]
+  %f = trunc i32 %e to i16   ; <i16> [#uses=1]
+  ret i16 %f
 }
 
+define <2 x i16> @test1_vec(<2 x i16> %a) {
+; CHECK-LABEL: @test1_vec(
+; CHECK-NEXT:    [[C:%.*]] = lshr <2 x i16> [[A:%.*]], <i16 8, i16 8>
+; CHECK-NEXT:    [[D:%.*]] = mul <2 x i16> [[A]], <i16 5, i16 5>
+; CHECK-NEXT:    [[E:%.*]] = or <2 x i16> [[C]], [[D]]
+; CHECK-NEXT:    ret <2 x i16> [[E]]
+;
+  %b = zext <2 x i16> %a to <2 x i32>
+  %c = lshr <2 x i32> %b, <i32 8, i32 8>
+  %d = mul <2 x i32> %b, <i32 5, i32 5>
+  %e = or <2 x i32> %c, %d
+  %f = trunc <2 x i32> %e to <2 x i16>
+  ret <2 x i16> %f
+}
+
+define <2 x i16> @test1_vec_nonuniform(<2 x i16> %a) {
+; CHECK-LABEL: @test1_vec_nonuniform(
+; CHECK-NEXT:    [[C:%.*]] = lshr <2 x i16> [[A:%.*]], <i16 8, i16 9>
+; CHECK-NEXT:    [[D:%.*]] = mul <2 x i16> [[A]], <i16 5, i16 6>
+; CHECK-NEXT:    [[E:%.*]] = or <2 x i16> [[C]], [[D]]
+; CHECK-NEXT:    ret <2 x i16> [[E]]
+;
+  %b = zext <2 x i16> %a to <2 x i32>
+  %c = lshr <2 x i32> %b, <i32 8, i32 9>
+  %d = mul <2 x i32> %b, <i32 5, i32 6>
+  %e = or <2 x i32> %c, %d
+  %f = trunc <2 x i32> %e to <2 x i16>
+  ret <2 x i16> %f
+}
+
+define <2 x i16> @test1_vec_undef(<2 x i16> %a) {
+; CHECK-LABEL: @test1_vec_undef(
+; CHECK-NEXT:    [[B:%.*]] = zext <2 x i16> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[C:%.*]] = lshr <2 x i32> [[B]], <i32 8, i32 undef>
+; CHECK-NEXT:    [[D:%.*]] = mul <2 x i32> [[B]], <i32 5, i32 undef>
+; CHECK-NEXT:    [[E:%.*]] = or <2 x i32> [[C]], [[D]]
+; CHECK-NEXT:    [[F:%.*]] = trunc <2 x i32> [[E]] to <2 x i16>
+; CHECK-NEXT:    ret <2 x i16> [[F]]
+;
+  %b = zext <2 x i16> %a to <2 x i32>
+  %c = lshr <2 x i32> %b, <i32 8, i32 undef>
+  %d = mul <2 x i32> %b, <i32 5, i32 undef>
+  %e = or <2 x i32> %c, %d
+  %f = trunc <2 x i32> %e to <2 x i16>
+  ret <2 x i16> %f
+}
diff --git a/llvm/test/Transforms/InstCombine/apint-cast.ll b/llvm/test/Transforms/InstCombine/apint-cast.ll
index 85e7a4fca93eb..0be2134b9e01d 100644
--- a/llvm/test/Transforms/InstCombine/apint-cast.ll
+++ b/llvm/test/Transforms/InstCombine/apint-cast.ll
@@ -1,30 +1,36 @@
-; Tests to make sure elimination of casts is working correctly
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 
+; Tests to make sure elimination of casts is working correctly
+
 define i17 @test1(i17 %a) {
-        %tmp = zext i17 %a to i37               ; <i37> [#uses=2]
-        %tmp21 = lshr i37 %tmp, 8               ; <i37> [#uses=1]
-; CHECK: %tmp21 = lshr i17 %a, 8
-        %tmp5 = shl i37 %tmp, 8         ; <i37> [#uses=1]
-; CHECK: %tmp5 = shl i17 %a, 8
-        %tmp.upgrd.32 = or i37 %tmp21, %tmp5            ; <i37> [#uses=1]
-; CHECK: %tmp.upgrd.32 = or i17 %tmp21, %tmp5
-        %tmp.upgrd.3 = trunc i37 %tmp.upgrd.32 to i17   ; <i17> [#uses=1]
-        ret i17 %tmp.upgrd.3
-; CHECK: ret i17 %tmp.upgrd.32
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[C:%.*]] = lshr i17 [[A:%.*]], 8
+; CHECK-NEXT:    [[D:%.*]] = shl i17 [[A]], 8
+; CHECK-NEXT:    [[E:%.*]] = or i17 [[C]], [[D]]
+; CHECK-NEXT:    ret i17 [[E]]
+;
+  %b = zext i17 %a to i37    ; <i37> [#uses=2]
+  %c = lshr i37 %b, 8        ; <i37> [#uses=1]
+  %d = shl i37 %b, 8         ; <i37> [#uses=1]
+  %e = or i37 %c, %d         ; <i37> [#uses=1]
+  %f = trunc i37 %e to i17   ; <i17> [#uses=1]
+  ret i17 %f
 }
 
 define i167 @test2(i167 %a) {
-        %tmp = zext i167 %a to i577               ; <i577> [#uses=2]
-        %tmp21 = lshr i577 %tmp, 9               ; <i577> [#uses=1]
-; CHECK: %tmp21 = lshr i167 %a, 9
-        %tmp5 = shl i577 %tmp, 8         ; <i577> [#uses=1]
-; CHECK: %tmp5 = shl i167 %a, 8
-        %tmp.upgrd.32 = or i577 %tmp21, %tmp5            ; <i577> [#uses=1]
-; CHECK: %tmp.upgrd.32 = or i167 %tmp21, %tmp5
-        %tmp.upgrd.3 = trunc i577 %tmp.upgrd.32 to i167  ; <i167> [#uses=1]
-        ret i167 %tmp.upgrd.3
-; CHECK: ret i167 %tmp.upgrd.32
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[C:%.*]] = lshr i167 [[A:%.*]], 9
+; CHECK-NEXT:    [[D:%.*]] = shl i167 [[A]], 8
+; CHECK-NEXT:    [[E:%.*]] = or i167 [[C]], [[D]]
+; CHECK-NEXT:    ret i167 [[E]]
+;
+  %b = zext i167 %a to i577   ; <i577> [#uses=2]
+  %c = lshr i577 %b, 9        ; <i577> [#uses=1]
+  %d = shl i577 %b, 8         ; <i577> [#uses=1]
+  %e = or i577 %c, %d         ; <i577> [#uses=1]
+  %f = trunc i577 %e to i167  ; <i167> [#uses=1]
+  ret i167 %f
 }
diff --git a/llvm/test/Transforms/InstCombine/apint-shift.ll b/llvm/test/Transforms/InstCombine/apint-shift.ll
index 8f7e36ca6330f..32da6045456a1 100644
--- a/llvm/test/Transforms/InstCombine/apint-shift.ll
+++ b/llvm/test/Transforms/InstCombine/apint-shift.ll
@@ -307,8 +307,8 @@ define i53 @test15a(i1 %X) {
 
 define i1 @test16(i84 %X) {
 ; CHECK-LABEL: @test16(
-; CHECK-NEXT:    [[AND:%.*]] = and i84 [[X:%.*]], 16
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i84 [[AND]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i84 [[X:%.*]], 16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i84 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shr = ashr i84 %X, 4
@@ -329,6 +329,30 @@ define <2 x i1> @test16vec(<2 x i84> %X) {
   ret <2 x i1> %cmp
 }
 
+define <2 x i1> @test16vec_nonuniform(<2 x i84> %X) {
+; CHECK-LABEL: @test16vec_nonuniform(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i84> [[X:%.*]], <i84 16, i84 4>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i84> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shr = ashr <2 x i84> %X, <i84 4, i84 2>
+  %and = and <2 x i84> %shr, <i84 1, i84 1>
+  %cmp = icmp ne <2 x i84> %and, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @test16vec_undef(<2 x i84> %X) {
+; CHECK-LABEL: @test16vec_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i84> [[X:%.*]], <i84 16, i84 undef>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i84> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shr = ashr <2 x i84> %X, <i84 4, i84 undef>
+  %and = and <2 x i84> %shr, <i84 1, i84 1>
+  %cmp = icmp ne <2 x i84> %and, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
 define i1 @test17(i106 %A) {
 ; CHECK-LABEL: @test17(
 ; CHECK-NEXT:    [[B_MASK:%.*]] = and i106 [[A:%.*]], -8
@@ -502,18 +526,18 @@ define <2 x i43> @lshr_shl_eq_amt_multi_use_splat_vec(<2 x i43> %A) {
   ret <2 x i43> %D
 }
 
-define i37 @test25(i37 %tmp.2, i37 %AA) {
+define i37 @test25(i37 %AA, i37 %BB) {
 ; CHECK-LABEL: @test25(
-; CHECK-NEXT:    [[TMP_3:%.*]] = and i37 [[TMP_2:%.*]], -131072
-; CHECK-NEXT:    [[X2:%.*]] = add i37 [[TMP_3]], [[AA:%.*]]
-; CHECK-NEXT:    [[TMP_6:%.*]] = and i37 [[X2]], -131072
-; CHECK-NEXT:    ret i37 [[TMP_6]]
+; CHECK-NEXT:    [[D:%.*]] = and i37 [[AA:%.*]], -131072
+; CHECK-NEXT:    [[C2:%.*]] = add i37 [[D]], [[BB:%.*]]
+; CHECK-NEXT:    [[F:%.*]] = and i37 [[C2]], -131072
+; CHECK-NEXT:    ret i37 [[F]]
 ;
-  %x = lshr i37 %AA, 17
-  %tmp.3 = lshr i37 %tmp.2, 17
-  %tmp.5 = add i37 %tmp.3, %x
-  %tmp.6 = shl i37 %tmp.5, 17
-  ret i37 %tmp.6
+  %C = lshr i37 %BB, 17
+  %D = lshr i37 %AA, 17
+  %E = add i37 %D, %C
+  %F = shl i37 %E, 17
+  ret i37 %F
 }
 
 define i40 @test26(i40 %A) {
@@ -534,10 +558,10 @@ define i177 @ossfuzz_9880(i177 %X) {
 ; CHECK-NEXT:    [[A:%.*]] = alloca i177, align 8
 ; CHECK-NEXT:    [[L1:%.*]] = load i177, i177* [[A]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i177 [[L1]], -1
-; CHECK-NEXT:    [[TMP2:%.*]] = sext i1 [[TMP1]] to i177
-; CHECK-NEXT:    [[B14:%.*]] = add i177 [[L1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i177 [[B14]], -1
-; CHECK-NEXT:    [[B1:%.*]] = zext i1 [[TMP3]] to i177
+; CHECK-NEXT:    [[B5_NEG:%.*]] = sext i1 [[TMP1]] to i177
+; CHECK-NEXT:    [[B14:%.*]] = add i177 [[L1]], [[B5_NEG]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i177 [[B14]], -1
+; CHECK-NEXT:    [[B1:%.*]] = zext i1 [[TMP2]] to i177
 ; CHECK-NEXT:    ret i177 [[B1]]
 ;
   %A = alloca i177
diff --git a/llvm/test/Transforms/InstCombine/bitcast.ll b/llvm/test/Transforms/InstCombine/bitcast.ll
index 0f0cbdb364af5..c4ee52f27a8c2 100644
--- a/llvm/test/Transforms/InstCombine/bitcast.ll
+++ b/llvm/test/Transforms/InstCombine/bitcast.ll
@@ -561,3 +561,9 @@ define void @constant_fold_vector_to_half() {
   store volatile half bitcast (<4 x i4> <i4 0, i4 0, i4 0, i4 4> to half), half* undef
   ret void
 }
+
+; Ensure that we do not crash when looking at such a weird bitcast.
+define i8* @bitcast_from_single_element_pointer_vector_to_pointer(<1 x i8*> %ptrvec) {
+  %ptr = bitcast <1 x i8*> %ptrvec to i8*
+  ret i8* %ptr
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-selects-icmp-condition-bittest.ll b/llvm/test/Transforms/InstCombine/canonicalize-selects-icmp-condition-bittest.ll
new file mode 100644
index 0000000000000..57390b376b507
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-selects-icmp-condition-bittest.ll
@@ -0,0 +1,142 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @use8(i8)
+declare void @use1(i1)
+
+; Basic case - all good.
+define i8 @p0(i8 %x, i8 %v0, i8 %v1) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[T0:%.*]] = and i8 [[X:%.*]], 1
+; CHECK-NEXT:    [[T1_NOT:%.*]] = icmp eq i8 [[T0]], 0
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T1_NOT]], i8 [[V1:%.*]], i8 [[V0:%.*]], !prof !0
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %t0 = and i8 %x, 1
+  %t1 = icmp eq i8 %t0, 1
+  %r = select i1 %t1, i8 %v0, i8 %v1, !prof !0
+  ret i8 %r
+}
+define i8 @p1(i8 %x, i8 %v0, i8 %v1) {
+; CHECK-LABEL: @p1(
+; CHECK-NEXT:    [[T0:%.*]] = and i8 [[X:%.*]], 1
+; CHECK-NEXT:    [[T1_NOT:%.*]] = icmp eq i8 [[T0]], 0
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T1_NOT]], i8 [[V1:%.*]], i8 [[V0:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %t0 = and i8 %x, 1
+  %t1 = icmp ne i8 %t0, 0
+  %r = select i1 %t1, i8 %v0, i8 %v1
+  ret i8 %r
+}
+
+; Can't invert all users of original condition
+define i8 @n2(i8 %x, i8 %v0, i8 %v1) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[T0:%.*]] = and i8 [[X:%.*]], 1
+; CHECK-NEXT:    [[T1:%.*]] = icmp ne i8 [[T0]], 0
+; CHECK-NEXT:    call void @use1(i1 [[T1]])
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T1]], i8 [[V0:%.*]], i8 [[V1:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %t0 = and i8 %x, 1
+  %t1 = icmp eq i8 %t0, 1
+  call void @use1(i1 %t1) ; condition has un-invertable use
+  %r = select i1 %t1, i8 %v0, i8 %v1
+  ret i8 %r
+}
+
+; Extra use can be adjusted. While there, test multi-bb case.
+define i8 @t3(i8 %x, i8 %v0, i8 %v1, i8 %v2, i8 %v3, i8* %out, i1 %c) {
+; CHECK-LABEL: @t3(
+; CHECK-NEXT:  bb0:
+; CHECK-NEXT:    [[T0:%.*]] = and i8 [[X:%.*]], 1
+; CHECK-NEXT:    [[T1_NOT:%.*]] = icmp eq i8 [[T0]], 0
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[R0:%.*]] = select i1 [[T1_NOT]], i8 [[V1:%.*]], i8 [[V0:%.*]]
+; CHECK-NEXT:    store i8 [[R0]], i8* [[OUT:%.*]], align 1
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[T1_NOT]], i8 [[V3:%.*]], i8 [[V2:%.*]]
+; CHECK-NEXT:    ret i8 [[R1]]
+;
+bb0:
+  %t0 = and i8 %x, 1
+  %t1 = icmp eq i8 %t0, 1
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  %r0 = select i1 %t1, i8 %v0, i8 %v1
+  store i8 %r0, i8* %out
+  br label %bb2
+bb2:
+  %r1 = select i1 %t1, i8 %v2, i8 %v3
+  ret i8 %r1
+}
+define i8 @t4(i8 %x, i8 %v0, i8 %v1, i8 %v2, i8 %v3, i8* %out) {
+; CHECK-LABEL: @t4(
+; CHECK-NEXT:    [[T0:%.*]] = and i8 [[X:%.*]], 1
+; CHECK-NEXT:    [[T1_NOT:%.*]] = icmp eq i8 [[T0]], 0
+; CHECK-NEXT:    [[R0:%.*]] = select i1 [[T1_NOT]], i8 [[V1:%.*]], i8 [[V0:%.*]]
+; CHECK-NEXT:    store i8 [[R0]], i8* [[OUT:%.*]], align 1
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[T1_NOT]], i8 [[V3:%.*]], i8 [[V2:%.*]]
+; CHECK-NEXT:    ret i8 [[R1]]
+;
+  %t0 = and i8 %x, 1
+  %t1 = icmp ne i8 %t0, 0
+  %r0 = select i1 %t1, i8 %v0, i8 %v1
+  store i8 %r0, i8* %out
+  %r1 = select i1 %t1, i8 %v2, i8 %v3
+  ret i8 %r1
+}
+
+; Weird comparisons
+define i8 @n5(i8 %x, i8 %v0, i8 %v1) {
+; CHECK-LABEL: @n5(
+; CHECK-NEXT:    ret i8 [[V1:%.*]]
+;
+  %t0 = and i8 %x, 1
+  %t1 = icmp eq i8 %t0, 2 ; checking some other bit
+  %r = select i1 %t1, i8 %v0, i8 %v1
+  ret i8 %r
+}
+define i8 @n6(i8 %x, i8 %v0, i8 %v1) {
+; CHECK-LABEL: @n6(
+; CHECK-NEXT:    ret i8 [[V1:%.*]]
+;
+  %t0 = and i8 %x, 1
+  %t1 = icmp eq i8 %t0, 3 ; checking some other bit
+  %r = select i1 %t1, i8 %v0, i8 %v1
+  ret i8 %r
+}
+define i8 @n7(i8 %x, i8 %v0, i8 %v1) {
+; CHECK-LABEL: @n7(
+; CHECK-NEXT:    [[T0:%.*]] = and i8 [[X:%.*]], 1
+; CHECK-NEXT:    [[T1_NOT_NOT:%.*]] = icmp eq i8 [[T0]], 0
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T1_NOT_NOT]], i8 [[V0:%.*]], i8 [[V1:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %t0 = and i8 %x, 1
+  %t1 = icmp ne i8 %t0, 1 ; not checking that it's zero
+  %r = select i1 %t1, i8 %v0, i8 %v1
+  ret i8 %r
+}
+
+; Potentially have more than a single bit set
+define i8 @n8(i8 %x, i8 %v0, i8 %v1) {
+; CHECK-LABEL: @n8(
+; CHECK-NEXT:    [[T0:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[T1:%.*]] = icmp eq i8 [[T0]], 1
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T1]], i8 [[V0:%.*]], i8 [[V1:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %t0 = and i8 %x, 3 ; Not a single bit
+  %t1 = icmp eq i8 %t0, 1
+  %r = select i1 %t1, i8 %v0, i8 %v1
+  ret i8 %r
+}
+
+!0  = !{!"branch_weights", i32 0,  i32 100}
+
+; Ensure that the branch metadata is reversed to match the reversals above.
+; CHECK: !0 = {{.*}} i32 100, i32 0}
diff --git a/llvm/test/Transforms/InstCombine/cast.ll b/llvm/test/Transforms/InstCombine/cast.ll
index a68d81acdde9e..18b411103122c 100644
--- a/llvm/test/Transforms/InstCombine/cast.ll
+++ b/llvm/test/Transforms/InstCombine/cast.ll
@@ -500,6 +500,38 @@ define <2 x i16> @test40vec(<2 x i16> %a) {
   ret <2 x i16> %r
 }
 
+define <2 x i16> @test40vec_nonuniform(<2 x i16> %a) {
+; ALL-LABEL: @test40vec_nonuniform(
+; ALL-NEXT:    [[T21:%.*]] = lshr <2 x i16> [[A:%.*]], <i16 9, i16 10>
+; ALL-NEXT:    [[T5:%.*]] = shl <2 x i16> [[A]], <i16 8, i16 9>
+; ALL-NEXT:    [[T32:%.*]] = or <2 x i16> [[T21]], [[T5]]
+; ALL-NEXT:    ret <2 x i16> [[T32]]
+;
+  %t = zext <2 x i16> %a to <2 x i32>
+  %t21 = lshr <2 x i32> %t, <i32 9, i32 10>
+  %t5 = shl <2 x i32> %t, <i32 8, i32 9>
+  %t32 = or <2 x i32> %t21, %t5
+  %r = trunc <2 x i32> %t32 to <2 x i16>
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @test40vec_undef(<2 x i16> %a) {
+; ALL-LABEL: @test40vec_undef(
+; ALL-NEXT:    [[T:%.*]] = zext <2 x i16> [[A:%.*]] to <2 x i32>
+; ALL-NEXT:    [[T21:%.*]] = lshr <2 x i32> [[T]], <i32 9, i32 undef>
+; ALL-NEXT:    [[T5:%.*]] = shl <2 x i32> [[T]], <i32 8, i32 undef>
+; ALL-NEXT:    [[T32:%.*]] = or <2 x i32> [[T21]], [[T5]]
+; ALL-NEXT:    [[R:%.*]] = trunc <2 x i32> [[T32]] to <2 x i16>
+; ALL-NEXT:    ret <2 x i16> [[R]]
+;
+  %t = zext <2 x i16> %a to <2 x i32>
+  %t21 = lshr <2 x i32> %t, <i32 9, i32 undef>
+  %t5 = shl <2 x i32> %t, <i32 8, i32 undef>
+  %t32 = or <2 x i32> %t21, %t5
+  %r = trunc <2 x i32> %t32 to <2 x i16>
+  ret <2 x i16> %r
+}
+
 ; PR1263
 define i32* @test41(i32* %t1) {
 ; ALL-LABEL: @test41(
diff --git a/llvm/test/Transforms/InstCombine/fdiv.ll b/llvm/test/Transforms/InstCombine/fdiv.ll
index deceb724007ed..7b1a556439b7c 100644
--- a/llvm/test/Transforms/InstCombine/fdiv.ll
+++ b/llvm/test/Transforms/InstCombine/fdiv.ll
@@ -598,9 +598,8 @@ define float @fabs_same_op_extra_use(float %x) {
 
 define float @fabs_fabs(float %x, float %y) {
 ; CHECK-LABEL: @fabs_fabs(
-; CHECK-NEXT:    [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
-; CHECK-NEXT:    [[Y_FABS:%.*]] = call float @llvm.fabs.f32(float [[Y:%.*]])
-; CHECK-NEXT:    [[R:%.*]] = fdiv float [[X_FABS]], [[Y_FABS]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = call float @llvm.fabs.f32(float [[TMP1]])
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %x.fabs = call float @llvm.fabs.f32(float %x)
@@ -613,8 +612,8 @@ define float @fabs_fabs_extra_use1(float %x, float %y) {
 ; CHECK-LABEL: @fabs_fabs_extra_use1(
 ; CHECK-NEXT:    [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
 ; CHECK-NEXT:    call void @use_f32(float [[X_FABS]])
-; CHECK-NEXT:    [[Y_FABS:%.*]] = call float @llvm.fabs.f32(float [[Y:%.*]])
-; CHECK-NEXT:    [[R:%.*]] = fdiv ninf float [[X_FABS]], [[Y_FABS]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv ninf float [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = call ninf float @llvm.fabs.f32(float [[TMP1]])
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %x.fabs = call float @llvm.fabs.f32(float %x)
@@ -626,10 +625,10 @@ define float @fabs_fabs_extra_use1(float %x, float %y) {
 
 define float @fabs_fabs_extra_use2(float %x, float %y) {
 ; CHECK-LABEL: @fabs_fabs_extra_use2(
-; CHECK-NEXT:    [[X_FABS:%.*]] = call fast float @llvm.fabs.f32(float [[X:%.*]])
 ; CHECK-NEXT:    [[Y_FABS:%.*]] = call fast float @llvm.fabs.f32(float [[Y:%.*]])
 ; CHECK-NEXT:    call void @use_f32(float [[Y_FABS]])
-; CHECK-NEXT:    [[R:%.*]] = fdiv reassoc ninf float [[X_FABS]], [[Y_FABS]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv reassoc ninf float [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = call reassoc ninf float @llvm.fabs.f32(float [[TMP1]])
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %x.fabs = call fast float @llvm.fabs.f32(float %x)
@@ -639,6 +638,8 @@ define float @fabs_fabs_extra_use2(float %x, float %y) {
   ret float %r
 }
 
+; negative test - don't create an extra instruction
+
 define float @fabs_fabs_extra_use3(float %x, float %y) {
 ; CHECK-LABEL: @fabs_fabs_extra_use3(
 ; CHECK-NEXT:    [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
diff --git a/llvm/test/Transforms/InstCombine/fma.ll b/llvm/test/Transforms/InstCombine/fma.ll
index f123f5d56b807..6d9f8ea694c69 100644
--- a/llvm/test/Transforms/InstCombine/fma.ll
+++ b/llvm/test/Transforms/InstCombine/fma.ll
@@ -3,6 +3,7 @@
 
 declare float @llvm.fma.f32(float, float, float) #1
 declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>) #1
+declare <8 x half> @llvm.fma.v8f16(<8 x half>, <8 x half>, <8 x half>) #1
 declare float @llvm.fmuladd.f32(float, float, float) #1
 declare float @llvm.fabs.f32(float) #1
 
@@ -182,7 +183,7 @@ define float @fmuladd_unary_fneg_x_unary_fneg_y(float %x, float %y, float %z) {
 
 define float @fmuladd_fneg_x_fneg_y_fast(float %x, float %y, float %z) {
 ; CHECK-LABEL: @fmuladd_fneg_x_fneg_y_fast(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    [[FMULADD:%.*]] = fadd fast float [[TMP1]], [[Z:%.*]]
 ; CHECK-NEXT:    ret float [[FMULADD]]
 ;
@@ -194,7 +195,7 @@ define float @fmuladd_fneg_x_fneg_y_fast(float %x, float %y, float %z) {
 
 define float @fmuladd_unary_fneg_x_unary_fneg_y_fast(float %x, float %y, float %z) {
 ; CHECK-LABEL: @fmuladd_unary_fneg_x_unary_fneg_y_fast(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    [[FMULADD:%.*]] = fadd fast float [[TMP1]], [[Z:%.*]]
 ; CHECK-NEXT:    ret float [[FMULADD]]
 ;
@@ -360,6 +361,87 @@ define float @fma_1_1_z(float %z) {
   ret float %fma
 }
 
+define float @fma_x_y_0(float %x, float %y) {
+; CHECK-LABEL: @fma_x_y_0(
+; CHECK-NEXT:    [[FMA:%.*]] = call float @llvm.fma.f32(float [[X:%.*]], float [[Y:%.*]], float 0.000000e+00)
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %fma = call float @llvm.fma.f32(float %x, float %y, float 0.0)
+  ret float %fma
+}
+
+define float @fma_x_y_0_nsz(float %x, float %y) {
+; CHECK-LABEL: @fma_x_y_0_nsz(
+; CHECK-NEXT:    [[FMA:%.*]] = fmul nsz float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %fma = call nsz float @llvm.fma.f32(float %x, float %y, float 0.0)
+  ret float %fma
+}
+
+define <8 x half> @fma_x_y_0_v(<8 x half> %x, <8 x half> %y) {
+; CHECK-LABEL: @fma_x_y_0_v(
+; CHECK-NEXT:    [[FMA:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[X:%.*]], <8 x half> [[Y:%.*]], <8 x half> zeroinitializer)
+; CHECK-NEXT:    ret <8 x half> [[FMA]]
+;
+  %fma = call <8 x half> @llvm.fma.v8f16(<8 x half> %x, <8 x half> %y, <8 x half> zeroinitializer)
+  ret <8 x half> %fma
+}
+
+define <8 x half> @fma_x_y_0_nsz_v(<8 x half> %x, <8 x half> %y) {
+; CHECK-LABEL: @fma_x_y_0_nsz_v(
+; CHECK-NEXT:    [[FMA:%.*]] = fmul nsz <8 x half> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <8 x half> [[FMA]]
+;
+  %fma = call nsz <8 x half> @llvm.fma.v8f16(<8 x half> %x, <8 x half> %y, <8 x half> zeroinitializer)
+  ret <8 x half> %fma
+}
+
+define float @fmuladd_x_y_0(float %x, float %y) {
+; CHECK-LABEL: @fmuladd_x_y_0(
+; CHECK-NEXT:    [[FMA:%.*]] = call float @llvm.fmuladd.f32(float [[X:%.*]], float [[Y:%.*]], float 0.000000e+00)
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %fma = call float @llvm.fmuladd.f32(float %x, float %y, float 0.0)
+  ret float %fma
+}
+
+define float @fmuladd_x_y_0_nsz(float %x, float %y) {
+; CHECK-LABEL: @fmuladd_x_y_0_nsz(
+; CHECK-NEXT:    [[FMA:%.*]] = fmul nsz float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %fma = call nsz float @llvm.fmuladd.f32(float %x, float %y, float 0.0)
+  ret float %fma
+}
+
+define float @fma_x_y_m0(float %x, float %y) {
+; CHECK-LABEL: @fma_x_y_m0(
+; CHECK-NEXT:    [[FMA:%.*]] = fmul float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %fma = call float @llvm.fma.f32(float %x, float %y, float -0.0)
+  ret float %fma
+}
+
+define <8 x half> @fma_x_y_m0_v(<8 x half> %x, <8 x half> %y) {
+; CHECK-LABEL: @fma_x_y_m0_v(
+; CHECK-NEXT:    [[FMA:%.*]] = fmul <8 x half> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <8 x half> [[FMA]]
+;
+  %fma = call <8 x half> @llvm.fma.v8f16(<8 x half> %x, <8 x half> %y, <8 x half> <half -0.0, half -0.0, half -0.0, half -0.0, half -0.0, half -0.0, half -0.0, half -0.0>)
+  ret <8 x half> %fma
+}
+
+define float @fmuladd_x_y_m0(float %x, float %y) {
+; CHECK-LABEL: @fmuladd_x_y_m0(
+; CHECK-NEXT:    [[FMA:%.*]] = fmul float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %fma = call float @llvm.fmuladd.f32(float %x, float %y, float -0.0)
+  ret float %fma
+}
+
 define float @fmuladd_x_1_z_fast(float %x, float %z) {
 ; CHECK-LABEL: @fmuladd_x_1_z_fast(
 ; CHECK-NEXT:    [[FMULADD:%.*]] = fadd fast float [[X:%.*]], [[Z:%.*]]
diff --git a/llvm/test/Transforms/InstCombine/fmul.ll b/llvm/test/Transforms/InstCombine/fmul.ll
index 0559a7907de05..8e168f252978e 100644
--- a/llvm/test/Transforms/InstCombine/fmul.ll
+++ b/llvm/test/Transforms/InstCombine/fmul.ll
@@ -482,9 +482,8 @@ define float @fabs_squared_fast(float %x) {
 
 define float @fabs_fabs(float %x, float %y) {
 ; CHECK-LABEL: @fabs_fabs(
-; CHECK-NEXT:    [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
-; CHECK-NEXT:    [[Y_FABS:%.*]] = call float @llvm.fabs.f32(float [[Y:%.*]])
-; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[X_FABS]], [[Y_FABS]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = call float @llvm.fabs.f32(float [[TMP1]])
 ; CHECK-NEXT:    ret float [[MUL]]
 ;
   %x.fabs = call float @llvm.fabs.f32(float %x)
@@ -497,8 +496,8 @@ define float @fabs_fabs_extra_use1(float %x, float %y) {
 ; CHECK-LABEL: @fabs_fabs_extra_use1(
 ; CHECK-NEXT:    [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
 ; CHECK-NEXT:    call void @use_f32(float [[X_FABS]])
-; CHECK-NEXT:    [[Y_FABS:%.*]] = call float @llvm.fabs.f32(float [[Y:%.*]])
-; CHECK-NEXT:    [[MUL:%.*]] = fmul ninf float [[X_FABS]], [[Y_FABS]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul ninf float [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = call ninf float @llvm.fabs.f32(float [[TMP1]])
 ; CHECK-NEXT:    ret float [[MUL]]
 ;
   %x.fabs = call float @llvm.fabs.f32(float %x)
@@ -510,10 +509,10 @@ define float @fabs_fabs_extra_use1(float %x, float %y) {
 
 define float @fabs_fabs_extra_use2(float %x, float %y) {
 ; CHECK-LABEL: @fabs_fabs_extra_use2(
-; CHECK-NEXT:    [[X_FABS:%.*]] = call fast float @llvm.fabs.f32(float [[X:%.*]])
 ; CHECK-NEXT:    [[Y_FABS:%.*]] = call fast float @llvm.fabs.f32(float [[Y:%.*]])
 ; CHECK-NEXT:    call void @use_f32(float [[Y_FABS]])
-; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc ninf float [[X_FABS]], [[Y_FABS]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc ninf float [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[MUL:%.*]] = call reassoc ninf float @llvm.fabs.f32(float [[TMP1]])
 ; CHECK-NEXT:    ret float [[MUL]]
 ;
   %x.fabs = call fast float @llvm.fabs.f32(float %x)
@@ -523,6 +522,8 @@ define float @fabs_fabs_extra_use2(float %x, float %y) {
   ret float %mul
 }
 
+; negative test - don't create an extra instruction
+
 define float @fabs_fabs_extra_use3(float %x, float %y) {
 ; CHECK-LABEL: @fabs_fabs_extra_use3(
 ; CHECK-NEXT:    [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
diff --git a/llvm/test/Transforms/InstCombine/icmp-mul-zext.ll b/llvm/test/Transforms/InstCombine/icmp-mul-zext.ll
index 6b8cfd4f4ae28..9b4d9947d1619 100644
--- a/llvm/test/Transforms/InstCombine/icmp-mul-zext.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-mul-zext.ll
@@ -13,13 +13,13 @@ define i32 @sterix(i32, i8, i64) {
 ; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[MUL]], [[SH_PROM]]
 ; CHECK-NEXT:    [[CONV2:%.*]] = zext i32 [[SHR]] to i64
 ; CHECK-NEXT:    [[MUL3:%.*]] = mul nuw nsw i64 [[CONV]], [[CONV2]]
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[MUL3]], 4294967295
-; CHECK-NEXT:    br i1 [[TMP3]], label [[LOR_END:%.*]], label [[LOR_RHS:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[MUL3]], 4294967296
+; CHECK-NEXT:    br i1 [[TMP3]], label [[LOR_RHS:%.*]], label [[LOR_END:%.*]]
 ; CHECK:       lor.rhs:
 ; CHECK-NEXT:    [[AND:%.*]] = and i64 [[MUL3]], [[TMP2]]
 ; CHECK-NEXT:    [[CONV4:%.*]] = trunc i64 [[AND]] to i32
-; CHECK-NEXT:    [[TOBOOL7:%.*]] = icmp eq i32 [[CONV4]], 0
-; CHECK-NEXT:    [[PHITMP:%.*]] = zext i1 [[TOBOOL7]] to i32
+; CHECK-NEXT:    [[TOBOOL7_NOT:%.*]] = icmp eq i32 [[CONV4]], 0
+; CHECK-NEXT:    [[PHITMP:%.*]] = zext i1 [[TOBOOL7_NOT]] to i32
 ; CHECK-NEXT:    br label [[LOR_END]]
 ; CHECK:       lor.end:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[PHITMP]], [[LOR_RHS]] ]
@@ -120,3 +120,37 @@ ret1:
 ret2:
   ret i16 %trunc_unnecessary              ; crash visiting this use after corrupting iterator
 }
+
+; This miscompiled because of broken pattern matching.
+
+define i1 @PR46561(i1 %a, i1 %x, i1 %y, i8 %z) {
+; CHECK-LABEL: @PR46561(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[A:%.*]], label [[COND_TRUE:%.*]], label [[END:%.*]]
+; CHECK:       cond.true:
+; CHECK-NEXT:    [[MULBOOL:%.*]] = and i1 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[Z:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 0
+; CHECK-NEXT:    [[PHI_CMP:%.*]] = xor i1 [[TMP1]], [[MULBOOL]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[P:%.*]] = phi i1 [ [[PHI_CMP]], [[COND_TRUE]] ], [ false, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i1 [[P]]
+;
+entry:
+  br i1 %a, label %cond.true, label %end
+
+cond.true:
+  %tz = trunc i8 %z to i1
+  %zx = zext i1 %x to i32
+  %zy = zext i1 %y to i32
+  %zz = zext i1 %tz to i32
+  %mul = mul i32 %zx, %zy
+  %cond = xor i32 %mul, %zz
+  br label %end
+
+end:
+  %p = phi i32 [ %cond, %cond.true ], [ -1, %entry ]
+  %r = icmp eq i32 %p, 0
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp.ll b/llvm/test/Transforms/InstCombine/icmp.ll
index 67d254a3fe524..1f848680a4eee 100644
--- a/llvm/test/Transforms/InstCombine/icmp.ll
+++ b/llvm/test/Transforms/InstCombine/icmp.ll
@@ -2686,6 +2686,32 @@ define <2 x i1> @icmp_and_or_lshr_cst_vec(<2 x i32> %x) {
   ret <2 x i1> %ret
 }
 
+define <2 x i1> @icmp_and_or_lshr_cst_vec_nonuniform(<2 x i32> %x) {
+; CHECK-LABEL: @icmp_and_or_lshr_cst_vec_nonuniform(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 3, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %shf = lshr <2 x i32> %x, <i32 1, i32 2>
+  %or = or <2 x i32> %shf, %x
+  %and = and <2 x i32> %or, <i32 1, i32 1>
+  %ret = icmp ne <2 x i32> %and, zeroinitializer
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @icmp_and_or_lshr_cst_vec_undef(<2 x i32> %x) {
+; CHECK-LABEL: @icmp_and_or_lshr_cst_vec_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 3, i32 -1>
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %shf = lshr <2 x i32> %x, <i32 1, i32 undef>
+  %or = or <2 x i32> %shf, %x
+  %and = and <2 x i32> %or, <i32 1, i32 1>
+  %ret = icmp ne <2 x i32> %and, zeroinitializer
+  ret <2 x i1> %ret
+}
+
 define <2 x i1> @icmp_and_or_lshr_cst_vec_commute(<2 x i32> %xp) {
 ; CHECK-LABEL: @icmp_and_or_lshr_cst_vec_commute(
 ; CHECK-NEXT:    [[X:%.*]] = srem <2 x i32> [[XP:%.*]], <i32 42, i32 42>
@@ -2701,6 +2727,36 @@ define <2 x i1> @icmp_and_or_lshr_cst_vec_commute(<2 x i32> %xp) {
   ret <2 x i1> %ret
 }
 
+define <2 x i1> @icmp_and_or_lshr_cst_vec_nonuniform_commute(<2 x i32> %xp) {
+; CHECK-LABEL: @icmp_and_or_lshr_cst_vec_nonuniform_commute(
+; CHECK-NEXT:    [[X:%.*]] = srem <2 x i32> [[XP:%.*]], <i32 42, i32 42>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], <i32 3, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %x = srem <2 x i32> %xp, <i32 42, i32 -42> ; prevent complexity-based canonicalization
+  %shf = lshr <2 x i32> %x, <i32 1, i32 2>
+  %or = or <2 x i32> %x, %shf
+  %and = and <2 x i32> %or, <i32 1, i32 1>
+  %ret = icmp ne <2 x i32> %and, zeroinitializer
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @icmp_and_or_lshr_cst_vec_undef_commute(<2 x i32> %xp) {
+; CHECK-LABEL: @icmp_and_or_lshr_cst_vec_undef_commute(
+; CHECK-NEXT:    [[X:%.*]] = srem <2 x i32> [[XP:%.*]], <i32 42, i32 42>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], <i32 3, i32 -1>
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %x = srem <2 x i32> %xp, <i32 42, i32 -42> ; prevent complexity-based canonicalization
+  %shf = lshr <2 x i32> %x, <i32 1, i32 undef>
+  %or = or <2 x i32> %x, %shf
+  %and = and <2 x i32> %or, <i32 1, i32 1>
+  %ret = icmp ne <2 x i32> %and, zeroinitializer
+  ret <2 x i1> %ret
+}
+
 define i1 @shl_ap1_zero_ap2_non_zero_2(i32 %a) {
 ; CHECK-LABEL: @shl_ap1_zero_ap2_non_zero_2(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A:%.*]], 29
@@ -2721,6 +2777,17 @@ define <2 x i1> @shl_ap1_zero_ap2_non_zero_2_vec(<2 x i32> %a) {
   ret <2 x i1> %cmp
 }
 
+define <2 x i1> @shl_ap1_zero_ap2_non_zero_2_vec_nonuniform(<2 x i32> %a) {
+; CHECK-LABEL: @shl_ap1_zero_ap2_non_zero_2_vec_nonuniform(
+; CHECK-NEXT:    [[SHL:%.*]] = shl <2 x i32> <i32 4, i32 5>, [[A:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[SHL]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl <2 x i32> <i32 4, i32 5>, %a
+  %cmp = icmp eq <2 x i32> %shl, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
 define i1 @shl_ap1_zero_ap2_non_zero_4(i32 %a) {
 ; CHECK-LABEL: @shl_ap1_zero_ap2_non_zero_4(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A:%.*]], 30
diff --git a/llvm/test/Transforms/InstCombine/load_combine_aa.ll b/llvm/test/Transforms/InstCombine/load_combine_aa.ll
index b84b81ddd5d96..5b1a609e2a70d 100644
--- a/llvm/test/Transforms/InstCombine/load_combine_aa.ll
+++ b/llvm/test/Transforms/InstCombine/load_combine_aa.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -instcombine -S < %s | FileCheck %s
+; RUN: opt -basic-aa -instcombine -S < %s | FileCheck %s
 
 ; CHECK-LABEL: @test_load_combine_aa(
 ; CHECK: %[[V:.*]] = load i32, i32* %0
diff --git a/llvm/test/Transforms/InstCombine/logical-select.ll b/llvm/test/Transforms/InstCombine/logical-select.ll
index e14e2bf964d9b..2f532be03fbfc 100644
--- a/llvm/test/Transforms/InstCombine/logical-select.ll
+++ b/llvm/test/Transforms/InstCombine/logical-select.ll
@@ -19,8 +19,8 @@ define i32 @foo(i32 %a, i32 %b, i32 %c, i32 %d) {
 
 define i32 @bar(i32 %a, i32 %b, i32 %c, i32 %d) {
 ; CHECK-LABEL: @bar(
-; CHECK-NEXT:    [[E:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[E]], i32 [[C:%.*]], i32 [[D:%.*]]
+; CHECK-NEXT:    [[E_NOT:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[E_NOT]], i32 [[C:%.*]], i32 [[D:%.*]]
 ; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
   %e = icmp slt i32 %a, %b
@@ -69,8 +69,8 @@ define i32 @fold_inverted_icmp_preds(i32 %a, i32 %b, i32 %c, i32 %d) {
 ; CHECK-LABEL: @fold_inverted_icmp_preds(
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[CMP1]], i32 [[C:%.*]], i32 0
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[A]], [[B]]
-; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[CMP2]], i32 0, i32 [[D:%.*]]
+; CHECK-NEXT:    [[CMP2_NOT:%.*]] = icmp slt i32 [[A]], [[B]]
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[CMP2_NOT]], i32 0, i32 [[D:%.*]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SEL1]], [[SEL2]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
@@ -88,8 +88,8 @@ define i32 @fold_inverted_icmp_preds_reverse(i32 %a, i32 %b, i32 %c, i32 %d) {
 ; CHECK-LABEL: @fold_inverted_icmp_preds_reverse(
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[CMP1]], i32 0, i32 [[C:%.*]]
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[A]], [[B]]
-; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[CMP2]], i32 [[D:%.*]], i32 0
+; CHECK-NEXT:    [[CMP2_NOT:%.*]] = icmp slt i32 [[A]], [[B]]
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[CMP2_NOT]], i32 [[D:%.*]], i32 0
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SEL1]], [[SEL2]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
@@ -124,8 +124,8 @@ define i32 @fold_inverted_fcmp_preds(float %a, float %b, i32 %c, i32 %d) {
 
 define <2 x i32> @fold_inverted_icmp_vector_preds(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x i32> %d) {
 ; CHECK-LABEL: @fold_inverted_icmp_vector_preds(
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq <2 x i32> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL1:%.*]] = select <2 x i1> [[CMP1]], <2 x i32> zeroinitializer, <2 x i32> [[C:%.*]]
+; CHECK-NEXT:    [[CMP1_NOT:%.*]] = icmp eq <2 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[SEL1:%.*]] = select <2 x i1> [[CMP1_NOT]], <2 x i32> zeroinitializer, <2 x i32> [[C:%.*]]
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq <2 x i32> [[A]], [[B]]
 ; CHECK-NEXT:    [[SEL2:%.*]] = select <2 x i1> [[CMP2]], <2 x i32> [[D:%.*]], <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> [[SEL1]], [[SEL2]]
@@ -535,9 +535,9 @@ define <4 x i32> @vec_sel_xor_multi_use(<4 x i32> %a, <4 x i32> %b, <4 x i1> %c)
 
 define i32 @allSignBits(i32 %cond, i32 %tval, i32 %fval) {
 ; CHECK-LABEL: @allSignBits(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[COND:%.*]], -1
-; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[FVAL:%.*]], i32 [[TVAL:%.*]]
-; CHECK-NEXT:    ret i32 [[TMP2]]
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp slt i32 [[COND:%.*]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTNOT]], i32 [[TVAL:%.*]], i32 [[FVAL:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
   %bitmask = ashr i32 %cond, 31
   %not_bitmask = xor i32 %bitmask, -1
@@ -549,9 +549,9 @@ define i32 @allSignBits(i32 %cond, i32 %tval, i32 %fval) {
 
 define <4 x i8> @allSignBits_vec(<4 x i8> %cond, <4 x i8> %tval, <4 x i8> %fval) {
 ; CHECK-LABEL: @allSignBits_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i8> [[COND:%.*]], <i8 -1, i8 -1, i8 -1, i8 -1>
-; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i8> [[FVAL:%.*]], <4 x i8> [[TVAL:%.*]]
-; CHECK-NEXT:    ret <4 x i8> [[TMP2]]
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp sgt <4 x i8> [[COND:%.*]], <i8 -1, i8 -1, i8 -1, i8 -1>
+; CHECK-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[DOTNOT]], <4 x i8> [[FVAL:%.*]], <4 x i8> [[TVAL:%.*]]
+; CHECK-NEXT:    ret <4 x i8> [[TMP1]]
 ;
   %bitmask = ashr <4 x i8> %cond, <i8 7, i8 7, i8 7, i8 7>
   %not_bitmask = xor <4 x i8> %bitmask, <i8 -1, i8 -1, i8 -1, i8 -1>
diff --git a/llvm/test/Transforms/InstCombine/malloc-free-delete.ll b/llvm/test/Transforms/InstCombine/malloc-free-delete.ll
index 77a60e5d0b0a7..fad542893c91b 100644
--- a/llvm/test/Transforms/InstCombine/malloc-free-delete.ll
+++ b/llvm/test/Transforms/InstCombine/malloc-free-delete.ll
@@ -276,6 +276,8 @@ declare void @_ZdaPvjSt11align_val_t(i8*, i32, i32) nobuiltin
 ; delete[](void*, unsigned long, align_val_t)
 declare void @_ZdaPvmSt11align_val_t(i8*, i64, i64) nobuiltin
 
+declare void @llvm.assume(i1)
+
 define void @test8() {
 ; CHECK-LABEL: @test8(
 ; CHECK-NEXT:    ret void
@@ -317,6 +319,12 @@ define void @test8() {
   call void @_ZdaPvmSt11align_val_t(i8* %naa2, i64 32, i64 8) builtin
   %naja2 = call i8* @_ZnajSt11align_val_t(i32 32, i32 8) builtin
   call void @_ZdaPvjSt11align_val_t(i8* %naja2, i32 32, i32 8) builtin
+
+  ; Check that the alignment assume does not prevent the removal.
+  %nwa3 = call i8* @_ZnwmSt11align_val_t(i64 32, i64 16) builtin
+  call void @llvm.assume(i1 true) [ "align"(i8* %nwa3, i64 16) ]
+  call void @_ZdlPvmSt11align_val_t(i8* %nwa3, i64 32, i64 16) builtin
+
   ret void
 }
 
@@ -343,7 +351,7 @@ define void @test10()  {
 
 define void @test11() {
 ; CHECK-LABEL: @test11(
-; CHECK-NEXT:    [[CALL:%.*]] = call dereferenceable(8) i8* @_Znwm(i64 8) #6
+; CHECK-NEXT:    [[CALL:%.*]] = call dereferenceable(8) i8* @_Znwm(i64 8) #7
 ; CHECK-NEXT:    call void @_ZdlPv(i8* nonnull [[CALL]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/InstCombine/merging-multiple-stores-into-successor.ll b/llvm/test/Transforms/InstCombine/merging-multiple-stores-into-successor.ll
new file mode 100644
index 0000000000000..5be341fa62280
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/merging-multiple-stores-into-successor.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -instcombine-infinite-loop-threshold=2 -S | FileCheck %s
+
+@var_7 = external global i8, align 1
+@var_1 = external global i32, align 4
+@var_0 = external global i16, align 2
+@var_5 = external global i64, align 8
+@arr_2 = external global [0 x i32], align 4
+@arr_4 = external global [0 x i16], align 2
+@arr_3 = external global [8 x i32], align 16
+
+define void @_Z4testv() {
+; CHECK-LABEL: @_Z4testv(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[I:%.*]] = load i8, i8* @var_7, align 1
+; CHECK-NEXT:    [[I1:%.*]] = icmp eq i8 [[I]], -1
+; CHECK-NEXT:    [[I4:%.*]] = load i16, i16* @var_0, align 2
+; CHECK-NEXT:    [[I8:%.*]] = sext i16 [[I4]] to i32
+; CHECK-NEXT:    br i1 [[I1]], label [[BB10:%.*]], label [[BB9:%.*]]
+; CHECK:       bb9:
+; CHECK-NEXT:    br label [[BB12:%.*]]
+; CHECK:       bb10:
+; CHECK-NEXT:    [[I2:%.*]] = load i32, i32* @var_1, align 4
+; CHECK-NEXT:    [[I3:%.*]] = icmp eq i32 [[I2]], 0
+; CHECK-NEXT:    [[I6:%.*]] = load i64, i64* @var_5, align 8
+; CHECK-NEXT:    [[I5:%.*]] = sext i16 [[I4]] to i64
+; CHECK-NEXT:    [[I7:%.*]] = select i1 [[I3]], i64 [[I6]], i64 [[I5]]
+; CHECK-NEXT:    [[I11:%.*]] = trunc i64 [[I7]] to i32
+; CHECK-NEXT:    br label [[BB12]]
+; CHECK:       bb12:
+; CHECK-NEXT:    [[STOREMERGE1:%.*]] = phi i32 [ [[I11]], [[BB10]] ], [ 1, [[BB9]] ]
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ [[I11]], [[BB10]] ], [ 1, [[BB9]] ]
+; CHECK-NEXT:    store i32 [[STOREMERGE1]], i32* getelementptr inbounds ([0 x i32], [0 x i32]* @arr_2, i64 0, i64 0), align 4
+; CHECK-NEXT:    store i16 [[I4]], i16* getelementptr inbounds ([0 x i16], [0 x i16]* @arr_4, i64 0, i64 0), align 2
+; CHECK-NEXT:    store i32 [[I8]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @arr_3, i64 0, i64 0), align 16
+; CHECK-NEXT:    store i32 [[STOREMERGE]], i32* getelementptr inbounds ([0 x i32], [0 x i32]* @arr_2, i64 0, i64 1), align 4
+; CHECK-NEXT:    store i16 [[I4]], i16* getelementptr inbounds ([0 x i16], [0 x i16]* @arr_4, i64 0, i64 1), align 2
+; CHECK-NEXT:    store i32 [[I8]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @arr_3, i64 0, i64 1), align 4
+; CHECK-NEXT:    ret void
+;
+bb:
+  %i = load i8, i8* @var_7, align 1
+  %i1 = icmp eq i8 %i, -1
+  %i2 = load i32, i32* @var_1, align 4
+  %i3 = icmp eq i32 %i2, 0
+  %i4 = load i16, i16* @var_0, align 2
+  %i5 = sext i16 %i4 to i64
+  %i6 = load i64, i64* @var_5, align 8
+  %i7 = select i1 %i3, i64 %i6, i64 %i5
+  %i8 = sext i16 %i4 to i32
+  br i1 %i1, label %bb10, label %bb9
+
+bb9:                                              ; preds = %bb
+  store i32 1, i32* getelementptr inbounds ([0 x i32], [0 x i32]* @arr_2, i64 0, i64 0), align 4
+  store i16 %i4, i16* getelementptr inbounds ([0 x i16], [0 x i16]* @arr_4, i64 0, i64 0), align 2
+  store i32 %i8, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @arr_3, i64 0, i64 0), align 4
+  store i32 1, i32* getelementptr inbounds ([0 x i32], [0 x i32]* @arr_2, i64 0, i64 1), align 4
+  store i16 %i4, i16* getelementptr inbounds ([0 x i16], [0 x i16]* @arr_4, i64 0, i64 1), align 2
+  store i32 %i8, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @arr_3, i64 0, i64 1), align 4
+  br label %bb12
+
+bb10:                                             ; preds = %bb
+  %i11 = trunc i64 %i7 to i32
+  store i32 %i11, i32* getelementptr inbounds ([0 x i32], [0 x i32]* @arr_2, i64 0, i64 0), align 4
+  store i16 %i4, i16* getelementptr inbounds ([0 x i16], [0 x i16]* @arr_4, i64 0, i64 0), align 2
+  store i32 %i8, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @arr_3, i64 0, i64 0), align 4
+  store i32 %i11, i32* getelementptr inbounds ([0 x i32], [0 x i32]* @arr_2, i64 0, i64 1), align 4
+  store i16 %i4, i16* getelementptr inbounds ([0 x i16], [0 x i16]* @arr_4, i64 0, i64 1), align 2
+  store i32 %i8, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @arr_3, i64 0, i64 1), align 4
+  br label %bb12
+
+bb12:                                             ; preds = %bb10, %bb9
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/mul.ll b/llvm/test/Transforms/InstCombine/mul.ll
index 0abefdccc7a57..059b18d30b907 100644
--- a/llvm/test/Transforms/InstCombine/mul.ll
+++ b/llvm/test/Transforms/InstCombine/mul.ll
@@ -31,11 +31,11 @@ define i8 @combine_shl(i8 %A) {
 
 define i32 @neg(i32 %i) {
 ; CHECK-LABEL: @neg(
-; CHECK-NEXT:    [[TMP:%.*]] = sub i32 0, [[I:%.*]]
-; CHECK-NEXT:    ret i32 [[TMP]]
+; CHECK-NEXT:    [[T:%.*]] = sub i32 0, [[I:%.*]]
+; CHECK-NEXT:    ret i32 [[T]]
 ;
-  %tmp = mul i32 %i, -1
-  ret i32 %tmp
+  %t = mul i32 %i, -1
+  ret i32 %t
 }
 
 ; Use the sign-bit as a mask:
@@ -127,6 +127,183 @@ define <2 x i32> @mul_bool_vec_commute(<2 x i32> %x, <2 x i1> %y) {
   ret <2 x i32> %m
 }
 
+define <3 x i7> @mul_bools(<3 x i1> %x, <3 x i1> %y) {
+; CHECK-LABEL: @mul_bools(
+; CHECK-NEXT:    [[MULBOOL:%.*]] = and <3 x i1> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = zext <3 x i1> [[MULBOOL]] to <3 x i7>
+; CHECK-NEXT:    ret <3 x i7> [[R]]
+;
+  %zx = zext <3 x i1> %x to <3 x i7>
+  %zy = zext <3 x i1> %y to <3 x i7>
+  %r = mul <3 x i7> %zx, %zy
+  ret <3 x i7> %r
+}
+
+define i32 @mul_bools_use1(i1 %x, i1 %y) {
+; CHECK-LABEL: @mul_bools_use1(
+; CHECK-NEXT:    [[ZY:%.*]] = zext i1 [[Y:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[ZY]])
+; CHECK-NEXT:    [[MULBOOL:%.*]] = and i1 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[MULBOOL]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %zx = zext i1 %x to i32
+  %zy = zext i1 %y to i32
+  call void @use32(i32 %zy)
+  %r = mul i32 %zx, %zy
+  ret i32 %r
+}
+
+define i32 @mul_bools_use2(i1 %x, i1 %y) {
+; CHECK-LABEL: @mul_bools_use2(
+; CHECK-NEXT:    [[ZY:%.*]] = zext i1 [[Y:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[ZY]])
+; CHECK-NEXT:    [[MULBOOL:%.*]] = and i1 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[MULBOOL]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %zx = zext i1 %x to i32
+  %zy = zext i1 %y to i32
+  call void @use32(i32 %zy)
+  %r = mul i32 %zy, %zx
+  ret i32 %r
+}
+
+define i32 @mul_bools_use3(i1 %x, i1 %y) {
+; CHECK-LABEL: @mul_bools_use3(
+; CHECK-NEXT:    [[ZX:%.*]] = zext i1 [[X:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[ZX]])
+; CHECK-NEXT:    [[ZY:%.*]] = zext i1 [[Y:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[ZY]])
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[X]], i32 [[ZY]], i32 0
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %zx = zext i1 %x to i32
+  call void @use32(i32 %zx)
+  %zy = zext i1 %y to i32
+  call void @use32(i32 %zy)
+  %r = mul i32 %zx, %zy
+  ret i32 %r
+}
+
+define <3 x i32> @mul_bools_sext(<3 x i1> %x, <3 x i1> %y) {
+; CHECK-LABEL: @mul_bools_sext(
+; CHECK-NEXT:    [[MULBOOL:%.*]] = and <3 x i1> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = zext <3 x i1> [[MULBOOL]] to <3 x i32>
+; CHECK-NEXT:    ret <3 x i32> [[R]]
+;
+  %sx = sext <3 x i1> %x to <3 x i32>
+  %sy = sext <3 x i1> %y to <3 x i32>
+  %r = mul <3 x i32> %sx, %sy
+  ret <3 x i32> %r
+}
+
+define i32 @mul_bools_sext_use1(i1 %x, i1 %y) {
+; CHECK-LABEL: @mul_bools_sext_use1(
+; CHECK-NEXT:    [[SY:%.*]] = sext i1 [[Y:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[SY]])
+; CHECK-NEXT:    [[MULBOOL:%.*]] = and i1 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[MULBOOL]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sx = sext i1 %x to i32
+  %sy = sext i1 %y to i32
+  call void @use32(i32 %sy)
+  %r = mul i32 %sx, %sy
+  ret i32 %r
+}
+
+define i32 @mul_bools_sext_use2(i1 %x, i1 %y) {
+; CHECK-LABEL: @mul_bools_sext_use2(
+; CHECK-NEXT:    [[SY:%.*]] = sext i1 [[Y:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[SY]])
+; CHECK-NEXT:    [[MULBOOL:%.*]] = and i1 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[MULBOOL]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sx = sext i1 %x to i32
+  %sy = sext i1 %y to i32
+  call void @use32(i32 %sy)
+  %r = mul i32 %sy, %sx
+  ret i32 %r
+}
+
+define i32 @mul_bools_sext_use3(i1 %x, i1 %y) {
+; CHECK-LABEL: @mul_bools_sext_use3(
+; CHECK-NEXT:    [[SX:%.*]] = sext i1 [[X:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[SX]])
+; CHECK-NEXT:    [[SY:%.*]] = sext i1 [[Y:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[SY]])
+; CHECK-NEXT:    [[R:%.*]] = mul nsw i32 [[SY]], [[SX]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sx = sext i1 %x to i32
+  call void @use32(i32 %sx)
+  %sy = sext i1 %y to i32
+  call void @use32(i32 %sy)
+  %r = mul i32 %sy, %sx
+  ret i32 %r
+}
+
+define <3 x i32> @mul_bools_mixed_ext(<3 x i1> %x, <3 x i1> %y) {
+; CHECK-LABEL: @mul_bools_mixed_ext(
+; CHECK-NEXT:    [[MULBOOL:%.*]] = and <3 x i1> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sext <3 x i1> [[MULBOOL]] to <3 x i32>
+; CHECK-NEXT:    ret <3 x i32> [[R]]
+;
+  %zx = zext <3 x i1> %x to <3 x i32>
+  %sy = sext <3 x i1> %y to <3 x i32>
+  %r = mul <3 x i32> %zx, %sy
+  ret <3 x i32> %r
+}
+
+define i32 @mul_bools_mixed_ext_use1(i1 %x, i1 %y) {
+; CHECK-LABEL: @mul_bools_mixed_ext_use1(
+; CHECK-NEXT:    [[ZY:%.*]] = zext i1 [[Y:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[ZY]])
+; CHECK-NEXT:    [[MULBOOL:%.*]] = and i1 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = sext i1 [[MULBOOL]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sx = sext i1 %x to i32
+  %zy = zext i1 %y to i32
+  call void @use32(i32 %zy)
+  %r = mul i32 %sx, %zy
+  ret i32 %r
+}
+
+define i32 @mul_bools_mixed_ext_use2(i1 %x, i1 %y) {
+; CHECK-LABEL: @mul_bools_mixed_ext_use2(
+; CHECK-NEXT:    [[SY:%.*]] = sext i1 [[Y:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[SY]])
+; CHECK-NEXT:    [[MULBOOL:%.*]] = and i1 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sext i1 [[MULBOOL]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %zx = zext i1 %x to i32
+  %sy = sext i1 %y to i32
+  call void @use32(i32 %sy)
+  %r = mul i32 %sy, %zx
+  ret i32 %r
+}
+
+define i32 @mul_bools_mixed_ext_use3(i1 %x, i1 %y) {
+; CHECK-LABEL: @mul_bools_mixed_ext_use3(
+; CHECK-NEXT:    [[SX:%.*]] = sext i1 [[X:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[SX]])
+; CHECK-NEXT:    [[ZY:%.*]] = zext i1 [[Y:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[ZY]])
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[Y]], i32 [[SX]], i32 0
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sx = sext i1 %x to i32
+  call void @use32(i32 %sx)
+  %zy = zext i1 %y to i32
+  call void @use32(i32 %zy)
+  %r = mul i32 %zy, %sx
+  ret i32 %r
+}
+
 ; (A >>u 31) * B --> (A >>s 31) & B
 
 define i32 @signbit_mul(i32 %a, i32 %b) {
diff --git a/llvm/test/Transforms/InstCombine/phi.ll b/llvm/test/Transforms/InstCombine/phi.ll
index c417737fdf23d..d2e028534d87f 100644
--- a/llvm/test/Transforms/InstCombine/phi.ll
+++ b/llvm/test/Transforms/InstCombine/phi.ll
@@ -519,9 +519,9 @@ Exit:           ; preds = %Loop
         %E = add i32 %B, 19
         ret i32 %E
 ; CHECK-LABEL: @test23(
-; CHECK: %phitmp = add i32 %A, 19
+; CHECK: %phi.bo = add i32 %A, 19
 ; CHECK: Loop:
-; CHECK-NEXT: %B = phi i32 [ %phitmp, %BB0 ], [ 61, %Loop ]
+; CHECK-NEXT: %B = phi i32 [ %phi.bo, %BB0 ], [ 61, %Loop ]
 ; CHECK: Exit:
 ; CHECK-NEXT: ret i32 %B
 }
diff --git a/llvm/test/Transforms/InstCombine/pr19420.ll b/llvm/test/Transforms/InstCombine/pr19420.ll
index 015f35eaaa53d..0a292865c9fc9 100644
--- a/llvm/test/Transforms/InstCombine/pr19420.ll
+++ b/llvm/test/Transforms/InstCombine/pr19420.ll
@@ -1,8 +1,9 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -instcombine < %s | FileCheck %s
 
 define <4 x i32> @test_FoldShiftByConstant_CreateSHL(<4 x i32> %in) {
 ; CHECK-LABEL: @test_FoldShiftByConstant_CreateSHL(
-; CHECK-NEXT:    [[VSHL_N:%.*]] = mul <4 x i32> %in, <i32 0, i32 -32, i32 0, i32 -32>
+; CHECK-NEXT:    [[VSHL_N:%.*]] = mul <4 x i32> [[IN:%.*]], <i32 0, i32 -32, i32 0, i32 -32>
 ; CHECK-NEXT:    ret <4 x i32> [[VSHL_N]]
 ;
   %mul.i = mul <4 x i32> %in, <i32 0, i32 -1, i32 0, i32 -1>
@@ -12,7 +13,7 @@ define <4 x i32> @test_FoldShiftByConstant_CreateSHL(<4 x i32> %in) {
 
 define <8 x i16> @test_FoldShiftByConstant_CreateSHL2(<8 x i16> %in) {
 ; CHECK-LABEL: @test_FoldShiftByConstant_CreateSHL2(
-; CHECK-NEXT:    [[VSHL_N:%.*]] = mul <8 x i16> %in, <i16 0, i16 -32, i16 0, i16 -32, i16 0, i16 -32, i16 0, i16 -32>
+; CHECK-NEXT:    [[VSHL_N:%.*]] = mul <8 x i16> [[IN:%.*]], <i16 0, i16 -32, i16 0, i16 -32, i16 0, i16 -32, i16 0, i16 -32>
 ; CHECK-NEXT:    ret <8 x i16> [[VSHL_N]]
 ;
   %mul.i = mul <8 x i16> %in, <i16 0, i16 -1, i16 0, i16 -1, i16 0, i16 -1, i16 0, i16 -1>
@@ -22,8 +23,8 @@ define <8 x i16> @test_FoldShiftByConstant_CreateSHL2(<8 x i16> %in) {
 
 define <16 x i8> @test_FoldShiftByConstant_CreateAnd(<16 x i8> %in0) {
 ; CHECK-LABEL: @test_FoldShiftByConstant_CreateAnd(
-; CHECK-NEXT:    [[VSRA_N2:%.*]] = mul <16 x i8> %in0, <i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33>
-; CHECK-NEXT:    [[VSHL_N:%.*]] = and <16 x i8> [[VSRA_N2]], <i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32>
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <16 x i8> [[IN0:%.*]], <i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33>
+; CHECK-NEXT:    [[VSHL_N:%.*]] = and <16 x i8> [[TMP1]], <i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32>
 ; CHECK-NEXT:    ret <16 x i8> [[VSHL_N]]
 ;
   %vsra_n = ashr <16 x i8> %in0, <i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5>
@@ -34,8 +35,8 @@ define <16 x i8> @test_FoldShiftByConstant_CreateAnd(<16 x i8> %in0) {
 
 define i32 @bar(i32 %x, i32 %y) {
 ; CHECK-LABEL: @bar(
-; CHECK-NEXT:    [[B1:%.*]] = shl i32 %y, 4
-; CHECK-NEXT:    [[A2:%.*]] = add i32 [[B1]], %x
+; CHECK-NEXT:    [[B1:%.*]] = shl i32 [[Y:%.*]], 4
+; CHECK-NEXT:    [[A2:%.*]] = add i32 [[B1]], [[X:%.*]]
 ; CHECK-NEXT:    [[C:%.*]] = and i32 [[A2]], -16
 ; CHECK-NEXT:    ret i32 [[C]]
 ;
@@ -47,8 +48,8 @@ define i32 @bar(i32 %x, i32 %y) {
 
 define <2 x i32> @bar_v2i32(<2 x i32> %x, <2 x i32> %y) {
 ; CHECK-LABEL: @bar_v2i32(
-; CHECK-NEXT:    [[B1:%.*]] = shl <2 x i32> %y, <i32 5, i32 5>
-; CHECK-NEXT:    [[A2:%.*]] = add <2 x i32> [[B1]], %x
+; CHECK-NEXT:    [[B1:%.*]] = shl <2 x i32> [[Y:%.*]], <i32 5, i32 5>
+; CHECK-NEXT:    [[A2:%.*]] = add <2 x i32> [[B1]], [[X:%.*]]
 ; CHECK-NEXT:    [[C:%.*]] = and <2 x i32> [[A2]], <i32 -32, i32 -32>
 ; CHECK-NEXT:    ret <2 x i32> [[C]]
 ;
@@ -60,8 +61,8 @@ define <2 x i32> @bar_v2i32(<2 x i32> %x, <2 x i32> %y) {
 
 define i32 @foo(i32 %x, i32 %y) {
 ; CHECK-LABEL: @foo(
-; CHECK-NEXT:    [[C1:%.*]] = shl i32 %y, 4
-; CHECK-NEXT:    [[X_MASK:%.*]] = and i32 %x, 128
+; CHECK-NEXT:    [[C1:%.*]] = shl i32 [[Y:%.*]], 4
+; CHECK-NEXT:    [[X_MASK:%.*]] = and i32 [[X:%.*]], 128
 ; CHECK-NEXT:    [[D:%.*]] = add i32 [[X_MASK]], [[C1]]
 ; CHECK-NEXT:    ret i32 [[D]]
 ;
@@ -74,9 +75,9 @@ define i32 @foo(i32 %x, i32 %y) {
 
 define <2 x i32> @foo_v2i32(<2 x i32> %x, <2 x i32> %y) {
 ; CHECK-LABEL: @foo_v2i32(
-; CHECK-NEXT:    [[A:%.*]] = lshr <2 x i32> %x, <i32 4, i32 4>
+; CHECK-NEXT:    [[A:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 4, i32 4>
 ; CHECK-NEXT:    [[B:%.*]] = and <2 x i32> [[A]], <i32 8, i32 8>
-; CHECK-NEXT:    [[C:%.*]] = add <2 x i32> [[B]], %y
+; CHECK-NEXT:    [[C:%.*]] = add <2 x i32> [[B]], [[Y:%.*]]
 ; CHECK-NEXT:    [[D:%.*]] = shl <2 x i32> [[C]], <i32 4, i32 4>
 ; CHECK-NEXT:    ret <2 x i32> [[D]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/select-bitext.ll b/llvm/test/Transforms/InstCombine/select-bitext.ll
index d44be27357354..1e156e248d748 100644
--- a/llvm/test/Transforms/InstCombine/select-bitext.ll
+++ b/llvm/test/Transforms/InstCombine/select-bitext.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
+declare void @use32(i32)
+
 ; Widen a select of constants to eliminate an extend.
 
 define i16 @sel_sext_constants(i1 %cmp) {
@@ -451,6 +453,36 @@ define i32 @test_bits_zext(i8 %a, i1 %cc) {
   ret i32 %r
 }
 
+define i32 @sel_sext_const_uses(i8 %a, i8 %x) {
+; CHECK-LABEL: @sel_sext_const_uses(
+; CHECK-NEXT:    [[COND:%.*]] = icmp ugt i8 [[X:%.*]], 15
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[A_EXT]])
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[COND]], i32 [[A_EXT]], i32 127
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cond = icmp ugt i8 %x, 15
+  %a.ext = sext i8 %a to i32
+  call void @use32(i32 %a.ext)
+  %r = select i1 %cond, i32 %a.ext, i32 127
+  ret i32 %r
+}
+
+define i32 @sel_zext_const_uses(i8 %a, i8 %x) {
+; CHECK-LABEL: @sel_zext_const_uses(
+; CHECK-NEXT:    [[COND:%.*]] = icmp sgt i8 [[X:%.*]], 15
+; CHECK-NEXT:    [[A_EXT:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[A_EXT]])
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[COND]], i32 255, i32 [[A_EXT]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cond = icmp sgt i8 %x, 15
+  %a.ext = zext i8 %a to i32
+  call void @use32(i32 %a.ext)
+  %r = select i1 %cond, i32 255, i32 %a.ext
+  ret i32 %r
+}
+
 define i32 @test_op_op(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @test_op_op(
 ; CHECK-NEXT:    [[CCA:%.*]] = icmp sgt i32 [[A:%.*]], 0
diff --git a/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll b/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll
index 44d284c6590a3..c534acd70cdb6 100644
--- a/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll
+++ b/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll
@@ -224,8 +224,7 @@ define i4 @PR45762(i3 %x4) {
 ; CHECK-NEXT:    [[T7:%.*]] = zext i3 [[T4]] to i4
 ; CHECK-NEXT:    [[ONE_HOT_16:%.*]] = shl i4 1, [[T7]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i3 [[X4]], 0
-; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP1]], i3 0, i3 [[T4]]
-; CHECK-NEXT:    [[UMUL_23:%.*]] = zext i3 [[NARROW]] to i4
+; CHECK-NEXT:    [[UMUL_23:%.*]] = select i1 [[TMP1]], i4 0, i4 [[T7]]
 ; CHECK-NEXT:    [[SEL_71:%.*]] = shl i4 [[ONE_HOT_16]], [[UMUL_23]]
 ; CHECK-NEXT:    ret i4 [[SEL_71]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/select-with-bitwise-ops.ll b/llvm/test/Transforms/InstCombine/select-with-bitwise-ops.ll
index d46227d74db37..6eccbecd1d4dc 100644
--- a/llvm/test/Transforms/InstCombine/select-with-bitwise-ops.ll
+++ b/llvm/test/Transforms/InstCombine/select-with-bitwise-ops.ll
@@ -153,9 +153,9 @@ define <2 x i32> @select_icmp_ne_0_and_4096_or_4096_vec(<2 x i32> %x, <2 x i32>
 define i32 @select_icmp_ne_0_and_4096_xor_4096(i32 %x, i32 %y) {
 ; CHECK-LABEL: @select_icmp_ne_0_and_4096_xor_4096(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 4096
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[XOR]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[XOR]], i32 [[Y]]
 ; CHECK-NEXT:    ret i32 [[SELECT]]
 ;
   %and = and i32 %x, 4096
@@ -168,9 +168,9 @@ define i32 @select_icmp_ne_0_and_4096_xor_4096(i32 %x, i32 %y) {
 define i32 @select_icmp_ne_0_and_4096_and_not_4096(i32 %x, i32 %y) {
 ; CHECK-LABEL: @select_icmp_ne_0_and_4096_and_not_4096(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -4097
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[AND2]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[AND2]], i32 [[Y]]
 ; CHECK-NEXT:    ret i32 [[SELECT]]
 ;
   %and = and i32 %x, 4096
@@ -311,9 +311,9 @@ define i32 @select_icmp_ne_0_and_4096_or_32(i32 %x, i32 %y) {
 define i32 @select_icmp_ne_0_and_4096_xor_32(i32 %x, i32 %y) {
 ; CHECK-LABEL: @select_icmp_ne_0_and_4096_xor_32(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 32
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[XOR]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[XOR]], i32 [[Y]]
 ; CHECK-NEXT:    ret i32 [[SELECT]]
 ;
   %and = and i32 %x, 4096
@@ -326,9 +326,9 @@ define i32 @select_icmp_ne_0_and_4096_xor_32(i32 %x, i32 %y) {
 define i32 @select_icmp_ne_0_and_4096_and_not_32(i32 %x, i32 %y) {
 ; CHECK-LABEL: @select_icmp_ne_0_and_4096_and_not_32(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -33
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[AND2]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[AND2]], i32 [[Y]]
 ; CHECK-NEXT:    ret i32 [[SELECT]]
 ;
   %and = and i32 %x, 4096
@@ -371,9 +371,9 @@ define <2 x i32> @select_icmp_ne_0_and_32_or_4096_vec(<2 x i32> %x, <2 x i32> %y
 define i32 @select_icmp_ne_0_and_32_xor_4096(i32 %x, i32 %y) {
 ; CHECK-LABEL: @select_icmp_ne_0_and_32_xor_4096(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 32
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 4096
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[XOR]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[XOR]], i32 [[Y]]
 ; CHECK-NEXT:    ret i32 [[SELECT]]
 ;
   %and = and i32 %x, 32
@@ -386,9 +386,9 @@ define i32 @select_icmp_ne_0_and_32_xor_4096(i32 %x, i32 %y) {
 define i32 @select_icmp_ne_0_and_32_and_not_4096(i32 %x, i32 %y) {
 ; CHECK-LABEL: @select_icmp_ne_0_and_32_and_not_4096(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 32
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -4097
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[AND2]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[AND2]], i32 [[Y]]
 ; CHECK-NEXT:    ret i32 [[SELECT]]
 ;
   %and = and i32 %x, 32
@@ -401,9 +401,9 @@ define i32 @select_icmp_ne_0_and_32_and_not_4096(i32 %x, i32 %y) {
 define i8 @select_icmp_ne_0_and_1073741824_or_8(i32 %x, i8 %y) {
 ; CHECK-LABEL: @select_icmp_ne_0_and_1073741824_or_8(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1073741824
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[OR:%.*]] = or i8 [[Y:%.*]], 8
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i8 [[OR]], i8 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i8 [[OR]], i8 [[Y]]
 ; CHECK-NEXT:    ret i8 [[SELECT]]
 ;
   %and = and i32 %x, 1073741824
@@ -416,9 +416,9 @@ define i8 @select_icmp_ne_0_and_1073741824_or_8(i32 %x, i8 %y) {
 define i8 @select_icmp_ne_0_and_1073741824_xor_8(i32 %x, i8 %y) {
 ; CHECK-LABEL: @select_icmp_ne_0_and_1073741824_xor_8(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1073741824
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i8 [[Y:%.*]], 8
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i8 [[XOR]], i8 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i8 [[XOR]], i8 [[Y]]
 ; CHECK-NEXT:    ret i8 [[SELECT]]
 ;
   %and = and i32 %x, 1073741824
@@ -431,9 +431,9 @@ define i8 @select_icmp_ne_0_and_1073741824_xor_8(i32 %x, i8 %y) {
 define i8 @select_icmp_ne_0_and_1073741824_and_not_8(i32 %x, i8 %y) {
 ; CHECK-LABEL: @select_icmp_ne_0_and_1073741824_and_not_8(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1073741824
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[AND2:%.*]] = and i8 [[Y:%.*]], -9
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i8 [[AND2]], i8 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i8 [[AND2]], i8 [[Y]]
 ; CHECK-NEXT:    ret i8 [[SELECT]]
 ;
   %and = and i32 %x, 1073741824
@@ -446,9 +446,9 @@ define i8 @select_icmp_ne_0_and_1073741824_and_not_8(i32 %x, i8 %y) {
 define i32 @select_icmp_ne_0_and_8_or_1073741824(i8 %x, i32 %y) {
 ; CHECK-LABEL: @select_icmp_ne_0_and_8_or_1073741824(
 ; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X:%.*]], 8
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i8 [[AND]], 0
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 1073741824
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[OR]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[OR]], i32 [[Y]]
 ; CHECK-NEXT:    ret i32 [[SELECT]]
 ;
   %and = and i8 %x, 8
@@ -461,9 +461,9 @@ define i32 @select_icmp_ne_0_and_8_or_1073741824(i8 %x, i32 %y) {
 define i32 @select_icmp_ne_0_and_8_xor_1073741824(i8 %x, i32 %y) {
 ; CHECK-LABEL: @select_icmp_ne_0_and_8_xor_1073741824(
 ; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X:%.*]], 8
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i8 [[AND]], 0
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 1073741824
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[XOR]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[XOR]], i32 [[Y]]
 ; CHECK-NEXT:    ret i32 [[SELECT]]
 ;
   %and = and i8 %x, 8
@@ -476,9 +476,9 @@ define i32 @select_icmp_ne_0_and_8_xor_1073741824(i8 %x, i32 %y) {
 define i32 @select_icmp_ne_0_and_8_and_not_1073741824(i8 %x, i32 %y) {
 ; CHECK-LABEL: @select_icmp_ne_0_and_8_and_not_1073741824(
 ; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X:%.*]], 8
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i8 [[AND]], 0
 ; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -1073741825
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[AND2]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[AND2]], i32 [[Y]]
 ; CHECK-NEXT:    ret i32 [[SELECT]]
 ;
   %and = and i8 %x, 8
@@ -731,9 +731,9 @@ define <2 x i32> @test69vec(<2 x i32> %x, <2 x i32> %y) {
 define i32 @test69_xor(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test69_xor(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i8
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[TMP1]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp sgt i8 [[TMP1]], -1
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 2
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[XOR]], i32 [[Y]]
 ; CHECK-NEXT:    ret i32 [[SELECT]]
 ;
   %and = and i32 %x, 128
@@ -746,9 +746,9 @@ define i32 @test69_xor(i32 %x, i32 %y) {
 define i32 @test69_and(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test69_and(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i8
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[TMP1]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp sgt i8 [[TMP1]], -1
 ; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], 2
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[AND2]], i32 [[Y]]
 ; CHECK-NEXT:    ret i32 [[SELECT]]
 ;
   %and = and i32 %x, 128
@@ -892,9 +892,9 @@ define i32 @no_shift_xor_multiuse_or(i32 %x, i32 %y) {
 define i32 @no_shift_xor_multiuse_xor(i32 %x, i32 %y) {
 ; CHECK-LABEL: @no_shift_xor_multiuse_xor(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 4096
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[XOR]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[XOR]], i32 [[Y]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[XOR]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
@@ -909,9 +909,9 @@ define i32 @no_shift_xor_multiuse_xor(i32 %x, i32 %y) {
 define i32 @no_shift_xor_multiuse_and(i32 %x, i32 %y) {
 ; CHECK-LABEL: @no_shift_xor_multiuse_and(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -4097
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[AND2]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[AND2]], i32 [[Y]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[AND2]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
@@ -926,9 +926,9 @@ define i32 @no_shift_xor_multiuse_and(i32 %x, i32 %y) {
 define i32 @shift_xor_multiuse_or(i32 %x, i32 %y) {
 ; CHECK-LABEL: @shift_xor_multiuse_or(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 2048
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[OR]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[OR]], i32 [[Y]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[OR]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
@@ -943,9 +943,9 @@ define i32 @shift_xor_multiuse_or(i32 %x, i32 %y) {
 define i32 @shift_xor_multiuse_xor(i32 %x, i32 %y) {
 ; CHECK-LABEL: @shift_xor_multiuse_xor(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 2048
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[XOR]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[XOR]], i32 [[Y]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[XOR]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
@@ -960,9 +960,9 @@ define i32 @shift_xor_multiuse_xor(i32 %x, i32 %y) {
 define i32 @shift_xor_multiuse_and(i32 %x, i32 %y) {
 ; CHECK-LABEL: @shift_xor_multiuse_and(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -2049
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[AND2]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[AND2]], i32 [[Y]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[AND2]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
@@ -1090,10 +1090,10 @@ define i32 @no_shift_no_xor_multiuse_cmp_with_and(i32 %x, i32 %y, i32 %z, i32 %w
 define i32 @no_shift_xor_multiuse_cmp(i32 %x, i32 %y, i32 %z, i32 %w) {
 ; CHECK-LABEL: @no_shift_xor_multiuse_cmp(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[AND]], 4096
 ; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], [[Y:%.*]]
-; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[W:%.*]], i32 [[Z:%.*]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP_NOT]], i32 [[W:%.*]], i32 [[Z:%.*]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[TMP2]], [[SELECT2]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
@@ -1109,10 +1109,10 @@ define i32 @no_shift_xor_multiuse_cmp(i32 %x, i32 %y, i32 %z, i32 %w) {
 define i32 @no_shift_xor_multiuse_cmp_with_xor(i32 %x, i32 %y, i32 %z, i32 %w) {
 ; CHECK-LABEL: @no_shift_xor_multiuse_cmp_with_xor(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 4096
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
-; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[XOR]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP_NOT]], i32 [[W:%.*]], i32 [[Z:%.*]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
@@ -1128,10 +1128,10 @@ define i32 @no_shift_xor_multiuse_cmp_with_xor(i32 %x, i32 %y, i32 %z, i32 %w) {
 define i32 @no_shift_xor_multiuse_cmp_with_and(i32 %x, i32 %y, i32 %z, i32 %w) {
 ; CHECK-LABEL: @no_shift_xor_multiuse_cmp_with_and(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -4097
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
-; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[AND2]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP_NOT]], i32 [[W:%.*]], i32 [[Z:%.*]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
@@ -1147,10 +1147,10 @@ define i32 @no_shift_xor_multiuse_cmp_with_and(i32 %x, i32 %y, i32 %z, i32 %w) {
 define i32 @shift_xor_multiuse_cmp(i32 %x, i32 %y, i32 %z, i32 %w) {
 ; CHECK-LABEL: @shift_xor_multiuse_cmp(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 2048
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[OR]]
-; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[OR]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP_NOT]], i32 [[W:%.*]], i32 [[Z:%.*]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
@@ -1166,10 +1166,10 @@ define i32 @shift_xor_multiuse_cmp(i32 %x, i32 %y, i32 %z, i32 %w) {
 define i32 @shift_xor_multiuse_cmp_with_xor(i32 %x, i32 %y, i32 %z, i32 %w) {
 ; CHECK-LABEL: @shift_xor_multiuse_cmp_with_xor(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 2048
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
-; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[XOR]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP_NOT]], i32 [[W:%.*]], i32 [[Z:%.*]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
@@ -1185,10 +1185,10 @@ define i32 @shift_xor_multiuse_cmp_with_xor(i32 %x, i32 %y, i32 %z, i32 %w) {
 define i32 @shift_xor_multiuse_cmp_with_and(i32 %x, i32 %y, i32 %z, i32 %w) {
 ; CHECK-LABEL: @shift_xor_multiuse_cmp_with_and(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -2049
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
-; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[AND2]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP_NOT]], i32 [[W:%.*]], i32 [[Z:%.*]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
@@ -1330,10 +1330,10 @@ define i32 @no_shift_no_xor_multiuse_cmp_and(i32 %x, i32 %y, i32 %z, i32 %w) {
 define i32 @no_shift_xor_multiuse_cmp_or(i32 %x, i32 %y, i32 %z, i32 %w) {
 ; CHECK-LABEL: @no_shift_xor_multiuse_cmp_or(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 4096
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[OR]]
-; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[OR]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP_NOT]], i32 [[W:%.*]], i32 [[Z:%.*]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
 ; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[OR]]
 ; CHECK-NEXT:    ret i32 [[RES2]]
@@ -1351,10 +1351,10 @@ define i32 @no_shift_xor_multiuse_cmp_or(i32 %x, i32 %y, i32 %z, i32 %w) {
 define i32 @no_shift_xor_multiuse_cmp_xor(i32 %x, i32 %y, i32 %z, i32 %w) {
 ; CHECK-LABEL: @no_shift_xor_multiuse_cmp_xor(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 4096
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
-; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[XOR]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP_NOT]], i32 [[W:%.*]], i32 [[Z:%.*]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
 ; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[XOR]]
 ; CHECK-NEXT:    ret i32 [[RES2]]
@@ -1372,10 +1372,10 @@ define i32 @no_shift_xor_multiuse_cmp_xor(i32 %x, i32 %y, i32 %z, i32 %w) {
 define i32 @no_shift_xor_multiuse_cmp_and(i32 %x, i32 %y, i32 %z, i32 %w) {
 ; CHECK-LABEL: @no_shift_xor_multiuse_cmp_and(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -4097
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
-; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[AND2]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP_NOT]], i32 [[W:%.*]], i32 [[Z:%.*]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
 ; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[AND2]]
 ; CHECK-NEXT:    ret i32 [[RES2]]
@@ -1393,10 +1393,10 @@ define i32 @no_shift_xor_multiuse_cmp_and(i32 %x, i32 %y, i32 %z, i32 %w) {
 define i32 @shift_xor_multiuse_cmp_or(i32 %x, i32 %y, i32 %z, i32 %w) {
 ; CHECK-LABEL: @shift_xor_multiuse_cmp_or(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 2048
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[OR]]
-; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[OR]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP_NOT]], i32 [[W:%.*]], i32 [[Z:%.*]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
 ; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[OR]]
 ; CHECK-NEXT:    ret i32 [[RES2]]
@@ -1414,10 +1414,10 @@ define i32 @shift_xor_multiuse_cmp_or(i32 %x, i32 %y, i32 %z, i32 %w) {
 define i32 @shift_xor_multiuse_cmp_xor(i32 %x, i32 %y, i32 %z, i32 %w) {
 ; CHECK-LABEL: @shift_xor_multiuse_cmp_xor(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 2048
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
-; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[XOR]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP_NOT]], i32 [[W:%.*]], i32 [[Z:%.*]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
 ; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[XOR]]
 ; CHECK-NEXT:    ret i32 [[RES2]]
@@ -1435,10 +1435,10 @@ define i32 @shift_xor_multiuse_cmp_xor(i32 %x, i32 %y, i32 %z, i32 %w) {
 define i32 @shift_xor_multiuse_cmp_and(i32 %x, i32 %y, i32 %z, i32 %w) {
 ; CHECK-LABEL: @shift_xor_multiuse_cmp_and(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], 2048
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
-; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP_NOT]], i32 [[AND2]], i32 [[Y]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP_NOT]], i32 [[W:%.*]], i32 [[Z:%.*]]
 ; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
 ; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[AND2]]
 ; CHECK-NEXT:    ret i32 [[RES2]]
diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll
index da336e24002d8..08e547a6ea0ad 100644
--- a/llvm/test/Transforms/InstCombine/select.ll
+++ b/llvm/test/Transforms/InstCombine/select.ll
@@ -1255,8 +1255,8 @@ define i128 @test86(i1 %flag) {
 
 define i32 @test_select_select0(i32 %a, i32 %r0, i32 %r1, i32 %v1, i32 %v2) {
 ; CHECK-LABEL: @test_select_select0(
-; CHECK-NEXT:    [[C0:%.*]] = icmp slt i32 [[A:%.*]], [[V1:%.*]]
-; CHECK-NEXT:    [[S0:%.*]] = select i1 [[C0]], i32 [[R1:%.*]], i32 [[R0:%.*]]
+; CHECK-NEXT:    [[C0_NOT:%.*]] = icmp slt i32 [[A:%.*]], [[V1:%.*]]
+; CHECK-NEXT:    [[S0:%.*]] = select i1 [[C0_NOT]], i32 [[R1:%.*]], i32 [[R0:%.*]]
 ; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[A]], [[V2:%.*]]
 ; CHECK-NEXT:    [[S1:%.*]] = select i1 [[C1]], i32 [[S0]], i32 [[R1]]
 ; CHECK-NEXT:    ret i32 [[S1]]
@@ -1270,8 +1270,8 @@ define i32 @test_select_select0(i32 %a, i32 %r0, i32 %r1, i32 %v1, i32 %v2) {
 
 define i32 @test_select_select1(i32 %a, i32 %r0, i32 %r1, i32 %v1, i32 %v2) {
 ; CHECK-LABEL: @test_select_select1(
-; CHECK-NEXT:    [[C0:%.*]] = icmp slt i32 [[A:%.*]], [[V1:%.*]]
-; CHECK-NEXT:    [[S0:%.*]] = select i1 [[C0]], i32 [[R1:%.*]], i32 [[R0:%.*]]
+; CHECK-NEXT:    [[C0_NOT:%.*]] = icmp slt i32 [[A:%.*]], [[V1:%.*]]
+; CHECK-NEXT:    [[S0:%.*]] = select i1 [[C0_NOT]], i32 [[R1:%.*]], i32 [[R0:%.*]]
 ; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[A]], [[V2:%.*]]
 ; CHECK-NEXT:    [[S1:%.*]] = select i1 [[C1]], i32 [[R0]], i32 [[S0]]
 ; CHECK-NEXT:    ret i32 [[S1]]
@@ -1680,7 +1680,7 @@ define i32 @select_dominating_cond(i1 %cond, i32 %x, i32 %y) {
 ; CHECK:       if.false:
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND]], i32 [[X:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[Y:%.*]], [[IF_FALSE]] ], [ [[X:%.*]], [[IF_TRUE]] ]
 ; CHECK-NEXT:    ret i32 [[S]]
 ;
 entry:
@@ -1697,6 +1697,33 @@ merge:
   ret i32 %s
 }
 
+define i32 @select_dominating_inverted(i1 %cond, i32 %x, i32 %y) {
+; CHECK-LABEL: @select_dominating_inverted(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_FALSE:%.*]], label [[IF_TRUE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[X:%.*]], [[IF_FALSE]] ], [ [[Y:%.*]], [[IF_TRUE]] ]
+; CHECK-NEXT:    ret i32 [[S]]
+;
+entry:
+  %inverted = xor i1 %cond, 1
+  br i1 %inverted, label %if.true, label %if.false
+
+if.true:
+  br label %merge
+
+if.false:
+  br label %merge
+
+merge:
+  %s = select i1 %cond, i32 %x, i32 %y
+  ret i32 %s
+}
+
 ; More complex CFG: the block with select has multiple predecessors.
 define i32 @select_dominating_cond_multiple_preds(i1 %cond, i1 %cond2, i1 %cond3, i32 %x, i32 %y) {
 ; CHECK-LABEL: @select_dominating_cond_multiple_preds(
@@ -1713,7 +1740,7 @@ define i32 @select_dominating_cond_multiple_preds(i1 %cond, i1 %cond2, i1 %cond3
 ; CHECK:       if.false.1:
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND]], i32 [[X:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[Y:%.*]], [[IF_FALSE_1]] ], [ [[X:%.*]], [[IF_TRUE_2]] ], [ [[X]], [[IF_TRUE_1]] ]
 ; CHECK-NEXT:    ret i32 [[S]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret i32 0
@@ -1760,7 +1787,7 @@ define i32 @select_dominating_cond_inverted_multiple_preds(i1 %cond, i1 %cond2,
 ; CHECK:       if.false.1:
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND]], i32 [[X:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[X:%.*]], [[IF_FALSE_1]] ], [ [[Y:%.*]], [[IF_TRUE_2]] ], [ [[Y]], [[IF_TRUE_1]] ]
 ; CHECK-NEXT:    ret i32 [[S]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret i32 0
@@ -1792,6 +1819,57 @@ exit:
   ret i32 0
 }
 
+; More complex CFG for inverted case: the block with select has multiple predecessors that can duplicate.
+define i32 @select_dominating_cond_inverted_multiple_duplicating_preds(i1 %cond, i32 %cond2, i1 %cond3, i32 %x, i32 %y) {
+; CHECK-LABEL: @select_dominating_cond_inverted_multiple_duplicating_preds(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_FALSE:%.*]], label [[IF_TRUE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    switch i32 [[COND2:%.*]], label [[SWITCH_CASE_1:%.*]] [
+; CHECK-NEXT:    i32 1, label [[MERGE:%.*]]
+; CHECK-NEXT:    i32 2, label [[MERGE]]
+; CHECK-NEXT:    i32 3, label [[MERGE]]
+; CHECK-NEXT:    ]
+; CHECK:       switch.case.1:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br i1 [[COND3:%.*]], label [[IF_FALSE_1:%.*]], label [[EXIT:%.*]]
+; CHECK:       if.false.1:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[X:%.*]], [[IF_FALSE_1]] ], [ [[Y:%.*]], [[SWITCH_CASE_1]] ], [ [[Y]], [[IF_TRUE]] ], [ [[Y]], [[IF_TRUE]] ], [ [[Y]], [[IF_TRUE]] ]
+; CHECK-NEXT:    ret i32 [[S]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %inverted = xor i1 %cond, 1
+  br i1 %inverted, label %if.true, label %if.false
+
+if.true:
+  switch i32 %cond2, label %switch.case.1 [
+  i32 1, label %merge
+  i32 2, label %merge
+  i32 3, label %merge
+  ]
+
+switch.case.1:
+  br label %merge
+
+if.false:
+  br i1 %cond3, label %if.false.1, label %exit
+
+if.false.1:
+  br label %merge
+
+merge:
+  %s = select i1 %cond, i32 %x, i32 %y
+  ret i32 %s
+
+exit:
+  ret i32 0
+}
+
 ; Negative test: currently we take condition from IDom, but might be willing to expand it in the future.
 define i32 @select_not_imm_dominating_cond_neg(i1 %cond, i32 %x, i32 %y) {
 ; CHECK-LABEL: @select_not_imm_dominating_cond_neg(
@@ -1847,9 +1925,9 @@ define i32 @select_dominance_chain(i1 %cond, i32 %x, i32 %y) {
 ; CHECK:       if.false.3:
 ; CHECK-NEXT:    br label [[MERGE_3]]
 ; CHECK:       merge.3:
-; CHECK-NEXT:    [[S_2:%.*]] = select i1 [[COND]], i32 [[X:%.*]], i32 [[Y:%.*]]
-; CHECK-NEXT:    [[S_1:%.*]] = select i1 [[COND]], i32 [[X]], i32 [[Y]]
-; CHECK-NEXT:    [[S_3:%.*]] = select i1 [[COND]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[S_3:%.*]] = phi i32 [ [[Y:%.*]], [[IF_FALSE_3]] ], [ [[X:%.*]], [[IF_TRUE_3]] ]
+; CHECK-NEXT:    [[S_2:%.*]] = phi i32 [ [[Y]], [[IF_FALSE_3]] ], [ [[X]], [[IF_TRUE_3]] ]
+; CHECK-NEXT:    [[S_1:%.*]] = phi i32 [ [[Y]], [[IF_FALSE_3]] ], [ [[X]], [[IF_TRUE_3]] ]
 ; CHECK-NEXT:    [[SUM_1:%.*]] = add i32 [[S_1]], [[S_2]]
 ; CHECK-NEXT:    [[SUM_2:%.*]] = add i32 [[SUM_1]], [[S_3]]
 ; CHECK-NEXT:    ret i32 [[SUM_2]]
@@ -1922,7 +2000,6 @@ merge:
   ret i32 %s
 }
 
-; TODO: Replace with phi[x, z].
 define i32 @select_phi_same_condition(i1 %cond, i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @select_phi_same_condition(
 ; CHECK-NEXT:  entry:
@@ -1932,8 +2009,7 @@ define i32 @select_phi_same_condition(i1 %cond, i32 %x, i32 %y, i32 %z) {
 ; CHECK:       if.false:
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ 0, [[IF_TRUE]] ], [ [[Z:%.*]], [[IF_FALSE]] ]
-; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND]], i32 [[X:%.*]], i32 [[PHI]]
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[Z:%.*]], [[IF_FALSE]] ], [ [[X:%.*]], [[IF_TRUE]] ]
 ; CHECK-NEXT:    ret i32 [[S]]
 ;
 entry:
@@ -1984,3 +2060,419 @@ merge:
   %s = select i1 %cond, i32 %a, i32 %phi
   ret i32 %s
 }
+
+declare i32 @__gxx_personality_v0(...)
+declare i1 @foo()
+
+define i32 @test_invoke_neg(i32 %x, i32 %y) nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: @test_invoke_neg(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND:%.*]] = invoke i1 @foo()
+; CHECK-NEXT:    to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]]
+; CHECK:       invoke.cont:
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], i32 [[X:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK:       lpad:
+; CHECK-NEXT:    [[LP:%.*]] = landingpad { i1, i32 }
+; CHECK-NEXT:    filter [0 x i1] zeroinitializer
+; CHECK-NEXT:    unreachable
+;
+entry:
+  %cond = invoke i1 @foo()
+  to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  %sel = select i1 %cond, i32 %x, i32 %y
+  ret i32 %sel
+
+lpad:
+  %lp = landingpad { i1, i32 }
+  filter [0 x i1] zeroinitializer
+  unreachable
+}
+
+declare i32 @bar()
+
+define i32 @test_invoke_2_neg(i1 %cond, i32 %x, i32 %y) nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: @test_invoke_2_neg(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    [[RESULT:%.*]] = invoke i32 @bar()
+; CHECK-NEXT:    to label [[MERGE]] unwind label [[LPAD:%.*]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ 0, [[IF_TRUE]] ], [ [[RESULT]], [[IF_FALSE]] ]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], i32 1, i32 [[PHI]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK:       lpad:
+; CHECK-NEXT:    [[LP:%.*]] = landingpad { i1, i32 }
+; CHECK-NEXT:    filter [0 x i1] zeroinitializer
+; CHECK-NEXT:    unreachable
+;
+entry:
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  br label %merge
+
+if.false:
+  %result = invoke i32 @bar()
+  to label %merge unwind label %lpad
+
+merge:
+  %phi = phi i32 [ 0, %if.true ], [ %result, %if.false ]
+  %sel = select i1 %cond, i32 1, i32 %phi
+  ret i32 %sel
+
+lpad:
+  %lp = landingpad { i1, i32 }
+  filter [0 x i1] zeroinitializer
+  unreachable
+}
+
+define i32 @select_phi_same_condition_switch(i1 %cond, i32 %x, i32 %y) {
+; CHECK-LABEL: @select_phi_same_condition_switch(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    switch i32 [[X:%.*]], label [[EXIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[MERGE:%.*]]
+; CHECK-NEXT:    i32 2, label [[MERGE]]
+; CHECK-NEXT:    ]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[Y:%.*]], [[IF_FALSE]] ], [ [[X]], [[IF_TRUE]] ], [ [[X]], [[IF_TRUE]] ]
+; CHECK-NEXT:    ret i32 [[S]]
+;
+entry:
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  switch i32 %x, label %exit [
+  i32 1, label %merge
+  i32 2, label %merge
+  ]
+
+exit:
+  ret i32 0
+
+if.false:
+  br label %merge
+
+merge:
+  %phi = phi i32 [0, %if.true], [0, %if.true], [%y, %if.false]
+  %s = select i1 %cond, i32 %x, i32 %phi
+  ret i32 %s
+}
+
+define i32 @transit_different_values_through_phi(i1 %cond, i1 %cond2) {
+; CHECK-LABEL: @transit_different_values_through_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[IF_TRUE_1:%.*]], label [[IF_TRUE_2:%.*]]
+; CHECK:       if.true.1:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.true.2:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ 3, [[IF_FALSE]] ], [ 2, [[IF_TRUE_2]] ], [ 1, [[IF_TRUE_1]] ]
+; CHECK-NEXT:    ret i32 [[S]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  br i1 %cond2, label %if.true.1, label %if.true.2
+
+if.true.1:
+  br label %merge
+
+if.true.2:
+  br label %merge
+
+if.false:
+  br label %merge
+
+merge:
+  %p = phi i32 [ 1, %if.true.1 ], [ 2, %if.true.2 ], [ 4, %if.false ]
+  %s = select i1 %cond, i32 %p, i32 3
+  ret i32 %s
+
+exit:
+  ret i32 0
+}
+
+define i32 @select_phi_degenerate(i1 %cond, i1 %cond2) {
+; CHECK-LABEL: @select_phi_degenerate(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[LOOP:%.*]], label [[EXIT:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[SELECT:%.*]] = phi i32 [ [[IV_INC:%.*]], [[LOOP]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[IV_INC]] = add i32 [[SELECT]], 1
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[LOOP]], label [[EXIT2:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       exit2:
+; CHECK-NEXT:    ret i32 [[IV_INC]]
+;
+entry:
+  br i1 %cond, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %select = select i1 %cond, i32 %iv, i32 -1
+  %iv.inc = add i32 %select, 1
+  br i1 %cond2, label %loop, label %exit2
+
+exit:
+  ret i32 0
+
+exit2:
+  ret i32 %iv.inc
+}
+
+define i32 @test_select_into_phi_not_idom(i1 %cond, i32 %A, i32 %B)  {
+; CHECK-LABEL: @test_select_into_phi_not_idom(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+entry:
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  br label %merge
+
+if.false:
+  br label %merge
+
+merge:
+  %phi = phi i32 [%A, %if.true], [%B, %if.false]
+  br label %exit
+
+exit:
+  %sel = select i1 %cond, i32 %phi, i32 %A
+  ret i32 %sel
+}
+
+define i32 @test_select_into_phi_not_idom_2(i1 %cond, i32 %A, i32 %B)  {
+; CHECK-LABEL: @test_select_into_phi_not_idom_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 [[B:%.*]]
+;
+entry:
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  br label %merge
+
+if.false:
+  br label %merge
+
+merge:
+  %phi = phi i32 [%A, %if.true], [%B, %if.false]
+  br label %exit
+
+exit:
+  %sel = select i1 %cond, i32 %B, i32 %phi
+  ret i32 %sel
+}
+
+define i32 @test_select_into_phi_not_idom_inverted(i1 %cond, i32 %A, i32 %B)  {
+; CHECK-LABEL: @test_select_into_phi_not_idom_inverted(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_FALSE:%.*]], label [[IF_TRUE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[SEL:%.*]] = phi i32 [ [[B:%.*]], [[IF_FALSE]] ], [ [[A:%.*]], [[IF_TRUE]] ]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+entry:
+  %inverted = xor i1 %cond, 1
+  br i1 %inverted, label %if.true, label %if.false
+
+if.true:
+  br label %merge
+
+if.false:
+  br label %merge
+
+merge:
+  %phi = phi i32 [%A, %if.true], [%B, %if.false]
+  br label %exit
+
+exit:
+  %sel = select i1 %cond, i32 %phi, i32 %A
+  ret i32 %sel
+}
+
+define i32 @test_select_into_phi_not_idom_inverted_2(i1 %cond, i32 %A, i32 %B)  {
+; CHECK-LABEL: @test_select_into_phi_not_idom_inverted_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_FALSE:%.*]], label [[IF_TRUE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[SEL:%.*]] = phi i32 [ [[B:%.*]], [[IF_FALSE]] ], [ [[A:%.*]], [[IF_TRUE]] ]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+entry:
+  %inverted = xor i1 %cond, 1
+  br i1 %inverted, label %if.true, label %if.false
+
+if.true:
+  br label %merge
+
+if.false:
+  br label %merge
+
+merge:
+  %phi = phi i32 [%A, %if.true], [%B, %if.false]
+  br label %exit
+
+exit:
+  %sel = select i1 %cond, i32 %B, i32 %phi
+  ret i32 %sel
+}
+
+define i32 @test_select_into_phi_not_idom_no_dom_input_1(i1 %cond, i32 %A, i32 %B, i32 *%p)  {
+; CHECK-LABEL: @test_select_into_phi_not_idom_no_dom_input_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    [[C:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[SEL:%.*]] = phi i32 [ [[A:%.*]], [[IF_FALSE]] ], [ [[C]], [[IF_TRUE]] ]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+entry:
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  %C = load i32, i32* %p
+  br label %merge
+
+if.false:
+  br label %merge
+
+merge:
+  %phi = phi i32 [%C, %if.true], [%B, %if.false]
+  br label %exit
+
+exit:
+  %sel = select i1 %cond, i32 %phi, i32 %A
+  ret i32 %sel
+}
+
+define i32 @test_select_into_phi_not_idom_no_dom_input_2(i1 %cond, i32 %A, i32 %B, i32 *%p)  {
+; CHECK-LABEL: @test_select_into_phi_not_idom_no_dom_input_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    [[C:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[SEL:%.*]] = phi i32 [ [[C]], [[IF_FALSE]] ], [ [[B:%.*]], [[IF_TRUE]] ]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+entry:
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  br label %merge
+
+if.false:
+  %C = load i32, i32* %p
+  br label %merge
+
+merge:
+  %phi = phi i32 [%A, %if.true], [%C, %if.false]
+  br label %exit
+
+exit:
+  %sel = select i1 %cond, i32 %B, i32 %phi
+  ret i32 %sel
+}
+
+; Negative tests to ensure we don't remove selects with undef true/false values.
+; See https://bugs.llvm.org/show_bug.cgi?id=31633
+; https://lists.llvm.org/pipermail/llvm-dev/2016-October/106182.html
+; https://reviews.llvm.org/D83360
+define i32 @false_undef(i1 %cond, i32 %x) {
+; CHECK-LABEL: @false_undef(
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND:%.*]], i32 [[X:%.*]], i32 undef
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %s = select i1 %cond, i32 %x, i32 undef
+  ret i32 %s
+}
+
+define i32 @true_undef(i1 %cond, i32 %x) {
+; CHECK-LABEL: @true_undef(
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND:%.*]], i32 undef, i32 [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %s = select i1 %cond, i32 undef, i32 %x
+  ret i32 %s
+}
+
+define <2 x i32> @false_undef_vec(i1 %cond, <2 x i32> %x) {
+; CHECK-LABEL: @false_undef_vec(
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND:%.*]], <2 x i32> [[X:%.*]], <2 x i32> undef
+; CHECK-NEXT:    ret <2 x i32> [[S]]
+;
+  %s = select i1 %cond, <2 x i32> %x, <2 x i32> undef
+  ret <2 x i32> %s
+}
+
+define <2 x i32> @true_undef_vec(i1 %cond, <2 x i32> %x) {
+; CHECK-LABEL: @true_undef_vec(
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND:%.*]], <2 x i32> undef, <2 x i32> [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[S]]
+;
+  %s = select i1 %cond, <2 x i32> undef, <2 x i32> %x
+  ret <2 x i32> %s
+}
diff --git a/llvm/test/Transforms/InstCombine/sext.ll b/llvm/test/Transforms/InstCombine/sext.ll
index faf33719927dd..a85c2ece15b08 100644
--- a/llvm/test/Transforms/InstCombine/sext.ll
+++ b/llvm/test/Transforms/InstCombine/sext.ll
@@ -9,9 +9,9 @@ declare i32 @llvm.cttz.i32(i32, i1)
 
 define i64 @test1(i32 %x) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[T:%.*]] = call i32 @llvm.ctpop.i32(i32 %x)
-; CHECK-NEXT:    [[S1:%.*]] = zext i32 [[T]] to i64
-; CHECK-NEXT:    ret i64 [[S1]]
+; CHECK-NEXT:    [[T:%.*]] = call i32 @llvm.ctpop.i32(i32 [[X:%.*]]), !range !0
+; CHECK-NEXT:    [[S:%.*]] = zext i32 [[T]] to i64
+; CHECK-NEXT:    ret i64 [[S]]
 ;
   %t = call i32 @llvm.ctpop.i32(i32 %x)
   %s = sext i32 %t to i64
@@ -20,9 +20,9 @@ define i64 @test1(i32 %x) {
 
 define i64 @test2(i32 %x) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[T:%.*]] = call i32 @llvm.ctlz.i32(i32 %x, i1 true)
-; CHECK-NEXT:    [[S1:%.*]] = zext i32 [[T]] to i64
-; CHECK-NEXT:    ret i64 [[S1]]
+; CHECK-NEXT:    [[T:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X:%.*]], i1 true), !range !0
+; CHECK-NEXT:    [[S:%.*]] = zext i32 [[T]] to i64
+; CHECK-NEXT:    ret i64 [[S]]
 ;
   %t = call i32 @llvm.ctlz.i32(i32 %x, i1 true)
   %s = sext i32 %t to i64
@@ -31,9 +31,9 @@ define i64 @test2(i32 %x) {
 
 define i64 @test3(i32 %x) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    [[T:%.*]] = call i32 @llvm.cttz.i32(i32 %x, i1 true)
-; CHECK-NEXT:    [[S1:%.*]] = zext i32 [[T]] to i64
-; CHECK-NEXT:    ret i64 [[S1]]
+; CHECK-NEXT:    [[T:%.*]] = call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 true), !range !0
+; CHECK-NEXT:    [[S:%.*]] = zext i32 [[T]] to i64
+; CHECK-NEXT:    ret i64 [[S]]
 ;
   %t = call i32 @llvm.cttz.i32(i32 %x, i1 true)
   %s = sext i32 %t to i64
@@ -42,9 +42,9 @@ define i64 @test3(i32 %x) {
 
 define i64 @test4(i32 %x) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[T:%.*]] = udiv i32 %x, 3
-; CHECK-NEXT:    [[S1:%.*]] = zext i32 [[T]] to i64
-; CHECK-NEXT:    ret i64 [[S1]]
+; CHECK-NEXT:    [[T:%.*]] = udiv i32 [[X:%.*]], 3
+; CHECK-NEXT:    [[S:%.*]] = zext i32 [[T]] to i64
+; CHECK-NEXT:    ret i64 [[S]]
 ;
   %t = udiv i32 %x, 3
   %s = sext i32 %t to i64
@@ -53,9 +53,9 @@ define i64 @test4(i32 %x) {
 
 define i64 @test5(i32 %x) {
 ; CHECK-LABEL: @test5(
-; CHECK-NEXT:    [[T:%.*]] = urem i32 %x, 30000
-; CHECK-NEXT:    [[S1:%.*]] = zext i32 [[T]] to i64
-; CHECK-NEXT:    ret i64 [[S1]]
+; CHECK-NEXT:    [[T:%.*]] = urem i32 [[X:%.*]], 30000
+; CHECK-NEXT:    [[S:%.*]] = zext i32 [[T]] to i64
+; CHECK-NEXT:    ret i64 [[S]]
 ;
   %t = urem i32 %x, 30000
   %s = sext i32 %t to i64
@@ -64,10 +64,10 @@ define i64 @test5(i32 %x) {
 
 define i64 @test6(i32 %x) {
 ; CHECK-LABEL: @test6(
-; CHECK-NEXT:    [[U:%.*]] = lshr i32 %x, 3
+; CHECK-NEXT:    [[U:%.*]] = lshr i32 [[X:%.*]], 3
 ; CHECK-NEXT:    [[T:%.*]] = mul nuw nsw i32 [[U]], 3
-; CHECK-NEXT:    [[S1:%.*]] = zext i32 [[T]] to i64
-; CHECK-NEXT:    ret i64 [[S1]]
+; CHECK-NEXT:    [[S:%.*]] = zext i32 [[T]] to i64
+; CHECK-NEXT:    ret i64 [[S]]
 ;
   %u = lshr i32 %x, 3
   %t = mul i32 %u, 3
@@ -77,10 +77,10 @@ define i64 @test6(i32 %x) {
 
 define i64 @test7(i32 %x) {
 ; CHECK-LABEL: @test7(
-; CHECK-NEXT:    [[T:%.*]] = and i32 %x, 511
+; CHECK-NEXT:    [[T:%.*]] = and i32 [[X:%.*]], 511
 ; CHECK-NEXT:    [[U:%.*]] = sub nuw nsw i32 20000, [[T]]
-; CHECK-NEXT:    [[S1:%.*]] = zext i32 [[U]] to i64
-; CHECK-NEXT:    ret i64 [[S1]]
+; CHECK-NEXT:    [[S:%.*]] = zext i32 [[U]] to i64
+; CHECK-NEXT:    ret i64 [[S]]
 ;
   %t = and i32 %x, 511
   %u = sub i32 20000, %t
@@ -90,8 +90,8 @@ define i64 @test7(i32 %x) {
 
 define i32 @test8(i8 %a, i32 %f, i1 %p, i32* %z) {
 ; CHECK-LABEL: @test8(
-; CHECK-NEXT:    [[D:%.*]] = lshr i32 %f, 24
-; CHECK-NEXT:    [[N:%.*]] = select i1 %p, i32 [[D]], i32 0
+; CHECK-NEXT:    [[D:%.*]] = lshr i32 [[F:%.*]], 24
+; CHECK-NEXT:    [[N:%.*]] = select i1 [[P:%.*]], i32 [[D]], i32 0
 ; CHECK-NEXT:    ret i32 [[N]]
 ;
   %d = lshr i32 %f, 24
@@ -127,22 +127,61 @@ F:
 ; PR2638
 define i32 @test10(i32 %i) {
 ; CHECK-LABEL: @test10(
-; CHECK-NEXT:    [[B1:%.*]] = shl i32 %i, 30
-; CHECK-NEXT:    [[B:%.*]] = ashr exact i32 [[B1]], 30
-; CHECK-NEXT:    ret i32 [[B]]
+; CHECK-NEXT:    [[D1:%.*]] = shl i32 [[I:%.*]], 30
+; CHECK-NEXT:    [[D:%.*]] = ashr exact i32 [[D1]], 30
+; CHECK-NEXT:    ret i32 [[D]]
 ;
-  %tmp12 = trunc i32 %i to i8
-  %tmp16 = shl i8 %tmp12, 6
-  %a = ashr i8 %tmp16, 6
-  %b = sext i8 %a to i32
-  ret i32 %b
+  %A = trunc i32 %i to i8
+  %B = shl i8 %A, 6
+  %C = ashr i8 %B, 6
+  %D = sext i8 %C to i32
+  ret i32 %D
+}
+
+define <2 x i32> @test10_vec(<2 x i32> %i) {
+; CHECK-LABEL: @test10_vec(
+; CHECK-NEXT:    [[D1:%.*]] = shl <2 x i32> [[I:%.*]], <i32 30, i32 30>
+; CHECK-NEXT:    [[D:%.*]] = ashr exact <2 x i32> [[D1]], <i32 30, i32 30>
+; CHECK-NEXT:    ret <2 x i32> [[D]]
+;
+  %A = trunc <2 x i32> %i to <2 x i8>
+  %B = shl <2 x i8> %A, <i8 6, i8 6>
+  %C = ashr <2 x i8> %B, <i8 6, i8 6>
+  %D = sext <2 x i8> %C to <2 x i32>
+  ret <2 x i32> %D
+}
+
+define <2 x i32> @test10_vec_nonuniform(<2 x i32> %i) {
+; CHECK-LABEL: @test10_vec_nonuniform(
+; CHECK-NEXT:    [[D1:%.*]] = shl <2 x i32> [[I:%.*]], <i32 30, i32 27>
+; CHECK-NEXT:    [[D:%.*]] = ashr <2 x i32> [[D1]], <i32 30, i32 27>
+; CHECK-NEXT:    ret <2 x i32> [[D]]
+;
+  %A = trunc <2 x i32> %i to <2 x i8>
+  %B = shl <2 x i8> %A, <i8 6, i8 3>
+  %C = ashr <2 x i8> %B, <i8 6, i8 3>
+  %D = sext <2 x i8> %C to <2 x i32>
+  ret <2 x i32> %D
+}
+
+define <2 x i32> @test10_vec_undef(<2 x i32> %i) {
+; CHECK-LABEL: @test10_vec_undef(
+; CHECK-NEXT:    [[D1:%.*]] = shl <2 x i32> [[I:%.*]], <i32 30, i32 0>
+; CHECK-NEXT:    [[D:%.*]] = ashr <2 x i32> [[D1]], <i32 30, i32 0>
+; CHECK-NEXT:    ret <2 x i32> [[D]]
+;
+  %A = trunc <2 x i32> %i to <2 x i8>
+  %B = shl <2 x i8> %A, <i8 6, i8 undef>
+  %C = ashr <2 x i8> %B, <i8 6, i8 undef>
+  %D = sext <2 x i8> %C to <2 x i32>
+  ret <2 x i32> %D
 }
 
 define void @test11(<2 x i16> %srcA, <2 x i16> %srcB, <2 x i16>* %dst) {
 ; CHECK-LABEL: @test11(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i16> %srcB, %srcA
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i16> [[SRCB:%.*]], [[SRCA:%.*]]
 ; CHECK-NEXT:    [[SEXT:%.*]] = sext <2 x i1> [[CMP]] to <2 x i16>
-; CHECK-NEXT:    store <2 x i16> [[SEXT]], <2 x i16>* %dst, align 4
+; CHECK-NEXT:    store <2 x i16> [[SEXT]], <2 x i16>* [[DST:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %cmp = icmp eq <2 x i16> %srcB, %srcA
@@ -154,7 +193,7 @@ define void @test11(<2 x i16> %srcA, <2 x i16> %srcB, <2 x i16>* %dst) {
 
 define i64 @test12(i32 %x) {
 ; CHECK-LABEL: @test12(
-; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 %x, 1
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[X:%.*]], 1
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[SHR]]
 ; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[SUB]] to i64
 ; CHECK-NEXT:    ret i64 [[CONV]]
@@ -167,7 +206,7 @@ define i64 @test12(i32 %x) {
 
 define i32 @test13(i32 %x) {
 ; CHECK-LABEL: @test13(
-; CHECK-NEXT:    [[AND:%.*]] = lshr i32 %x, 3
+; CHECK-NEXT:    [[AND:%.*]] = lshr i32 [[X:%.*]], 3
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[AND]], 1
 ; CHECK-NEXT:    [[SEXT:%.*]] = add nsw i32 [[TMP1]], -1
 ; CHECK-NEXT:    ret i32 [[SEXT]]
@@ -180,7 +219,7 @@ define i32 @test13(i32 %x) {
 
 define i32 @test14(i16 %x) {
 ; CHECK-LABEL: @test14(
-; CHECK-NEXT:    [[AND:%.*]] = lshr i16 %x, 4
+; CHECK-NEXT:    [[AND:%.*]] = lshr i16 [[X:%.*]], 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[AND]], 1
 ; CHECK-NEXT:    [[SEXT:%.*]] = add nsw i16 [[TMP1]], -1
 ; CHECK-NEXT:    [[EXT:%.*]] = sext i16 [[SEXT]] to i32
@@ -194,8 +233,8 @@ define i32 @test14(i16 %x) {
 
 define i32 @test15(i32 %x) {
 ; CHECK-LABEL: @test15(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 %x, 27
-; CHECK-NEXT:    [[SEXT:%.*]] = ashr i32 [[TMP1]], 31
+; CHECK-NEXT:    [[AND:%.*]] = shl i32 [[X:%.*]], 27
+; CHECK-NEXT:    [[SEXT:%.*]] = ashr i32 [[AND]], 31
 ; CHECK-NEXT:    ret i32 [[SEXT]]
 ;
   %and = and i32 %x, 16
@@ -206,8 +245,8 @@ define i32 @test15(i32 %x) {
 
 define i32 @test16(i16 %x) {
 ; CHECK-LABEL: @test16(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i16 %x, 12
-; CHECK-NEXT:    [[SEXT:%.*]] = ashr i16 [[TMP1]], 15
+; CHECK-NEXT:    [[AND:%.*]] = shl i16 [[X:%.*]], 12
+; CHECK-NEXT:    [[SEXT:%.*]] = ashr i16 [[AND]], 15
 ; CHECK-NEXT:    [[EXT:%.*]] = sext i16 [[SEXT]] to i32
 ; CHECK-NEXT:    ret i32 [[EXT]]
 ;
@@ -219,8 +258,8 @@ define i32 @test16(i16 %x) {
 
 define i32 @test17(i1 %x) {
 ; CHECK-LABEL: @test17(
-; CHECK-NEXT:    [[C2:%.*]] = zext i1 %x to i32
-; CHECK-NEXT:    ret i32 [[C2]]
+; CHECK-NEXT:    [[C1_NEG:%.*]] = zext i1 [[X:%.*]] to i32
+; CHECK-NEXT:    ret i32 [[C1_NEG]]
 ;
   %c1 = sext i1 %x to i32
   %c2 = sub i32 0, %c1
@@ -229,10 +268,10 @@ define i32 @test17(i1 %x) {
 
 define i32 @test18(i16 %x) {
 ; CHECK-LABEL: @test18(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i16 %x, 0
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP1]], i16 %x, i16 0
-; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[SEL]] to i32
-; CHECK-NEXT:    ret i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i16 [[X:%.*]], 0
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP1]], i16 [[X]], i16 0
+; CHECK-NEXT:    [[EXT:%.*]] = zext i16 [[SEL]] to i32
+; CHECK-NEXT:    ret i32 [[EXT]]
 ;
   %cmp = icmp slt i16 %x, 0
   %sel = select i1 %cmp, i16 0, i16 %x
diff --git a/llvm/test/Transforms/InstCombine/shift.ll b/llvm/test/Transforms/InstCombine/shift.ll
index 822f915d5668c..714a08d247a24 100644
--- a/llvm/test/Transforms/InstCombine/shift.ll
+++ b/llvm/test/Transforms/InstCombine/shift.ll
@@ -213,14 +213,14 @@ define i32 @test15a(i1 %C) {
 
 define i1 @test16(i32 %X) {
 ; CHECK-LABEL: @test16(
-; CHECK-NEXT:    [[TMP_6:%.*]] = and i32 [[X:%.*]], 16
-; CHECK-NEXT:    [[TMP_7:%.*]] = icmp ne i32 [[TMP_6]], 0
-; CHECK-NEXT:    ret i1 [[TMP_7]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 16
+; CHECK-NEXT:    [[I_7:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[I_7]]
 ;
-  %tmp.3 = ashr i32 %X, 4
-  %tmp.6 = and i32 %tmp.3, 1
-  %tmp.7 = icmp ne i32 %tmp.6, 0
-  ret i1 %tmp.7
+  %i.3 = ashr i32 %X, 4
+  %i.6 = and i32 %i.3, 1
+  %i.7 = icmp ne i32 %i.6, 0
+  ret i1 %i.7
 }
 
 define i1 @test17(i32 %A) {
@@ -352,32 +352,32 @@ define i8 @test24(i8 %X) {
   ret i8 %Q
 }
 
-define i32 @test25(i32 %tmp.2, i32 %AA) {
+define i32 @test25(i32 %i.2, i32 %AA) {
 ; CHECK-LABEL: @test25(
-; CHECK-NEXT:    [[TMP_3:%.*]] = and i32 [[TMP_2:%.*]], -131072
-; CHECK-NEXT:    [[X2:%.*]] = add i32 [[TMP_3]], [[AA:%.*]]
-; CHECK-NEXT:    [[TMP_6:%.*]] = and i32 [[X2]], -131072
-; CHECK-NEXT:    ret i32 [[TMP_6]]
+; CHECK-NEXT:    [[I_3:%.*]] = and i32 [[I_2:%.*]], -131072
+; CHECK-NEXT:    [[X2:%.*]] = add i32 [[I_3]], [[AA:%.*]]
+; CHECK-NEXT:    [[I_6:%.*]] = and i32 [[X2]], -131072
+; CHECK-NEXT:    ret i32 [[I_6]]
 ;
   %x = lshr i32 %AA, 17
-  %tmp.3 = lshr i32 %tmp.2, 17
-  %tmp.5 = add i32 %tmp.3, %x
-  %tmp.6 = shl i32 %tmp.5, 17
-  ret i32 %tmp.6
+  %i.3 = lshr i32 %i.2, 17
+  %i.5 = add i32 %i.3, %x
+  %i.6 = shl i32 %i.5, 17
+  ret i32 %i.6
 }
 
-define <2 x i32> @test25_vector(<2 x i32> %tmp.2, <2 x i32> %AA) {
+define <2 x i32> @test25_vector(<2 x i32> %i.2, <2 x i32> %AA) {
 ; CHECK-LABEL: @test25_vector(
-; CHECK-NEXT:    [[TMP_3:%.*]] = and <2 x i32> [[TMP_2:%.*]], <i32 -131072, i32 -131072>
-; CHECK-NEXT:    [[X2:%.*]] = add <2 x i32> [[TMP_3]], [[AA:%.*]]
-; CHECK-NEXT:    [[TMP_6:%.*]] = and <2 x i32> [[X2]], <i32 -131072, i32 -131072>
-; CHECK-NEXT:    ret <2 x i32> [[TMP_6]]
+; CHECK-NEXT:    [[I_3:%.*]] = and <2 x i32> [[I_2:%.*]], <i32 -131072, i32 -131072>
+; CHECK-NEXT:    [[X2:%.*]] = add <2 x i32> [[I_3]], [[AA:%.*]]
+; CHECK-NEXT:    [[I_6:%.*]] = and <2 x i32> [[X2]], <i32 -131072, i32 -131072>
+; CHECK-NEXT:    ret <2 x i32> [[I_6]]
 ;
   %x = lshr <2 x i32> %AA, <i32 17, i32 17>
-  %tmp.3 = lshr <2 x i32> %tmp.2, <i32 17, i32 17>
-  %tmp.5 = add <2 x i32> %tmp.3, %x
-  %tmp.6 = shl <2 x i32> %tmp.5, <i32 17, i32 17>
-  ret <2 x i32> %tmp.6
+  %i.3 = lshr <2 x i32> %i.2, <i32 17, i32 17>
+  %i.5 = add <2 x i32> %i.3, %x
+  %i.6 = shl <2 x i32> %i.5, <i32 17, i32 17>
+  ret <2 x i32> %i.6
 }
 
 ;; handle casts between shifts.
@@ -427,40 +427,40 @@ define <2 x i1> @test28vec(<2 x i8> %x) {
 define i8 @test28a(i8 %x, i8 %y) {
 ; CHECK-LABEL: @test28a(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X:%.*]], 7
-; CHECK-NEXT:    [[COND1:%.*]] = icmp slt i8 [[X]], 0
-; CHECK-NEXT:    br i1 [[COND1]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK-NEXT:    [[I1:%.*]] = lshr i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[COND1_NOT:%.*]] = icmp sgt i8 [[X]], -1
+; CHECK-NEXT:    br i1 [[COND1_NOT]], label [[BB2:%.*]], label [[BB1:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    ret i8 [[TMP1]]
+; CHECK-NEXT:    ret i8 [[I1]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    [[TMP2:%.*]] = add i8 [[TMP1]], [[Y:%.*]]
-; CHECK-NEXT:    ret i8 [[TMP2]]
+; CHECK-NEXT:    [[I2:%.*]] = add i8 [[I1]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[I2]]
 ;
 entry:
 ; This shouldn't be transformed.
-  %tmp1 = lshr i8 %x, 7
-  %cond1 = icmp ne i8 %tmp1, 0
+  %i1 = lshr i8 %x, 7
+  %cond1 = icmp ne i8 %i1, 0
   br i1 %cond1, label %bb1, label %bb2
 bb1:
-  ret i8 %tmp1
+  ret i8 %i1
 bb2:
-  %tmp2 = add i8 %tmp1, %y
-  ret i8 %tmp2
+  %i2 = add i8 %i1, %y
+  ret i8 %i2
 }
 
 
 define i32 @test29(i64 %d18) {
 ; CHECK-LABEL: @test29(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP916:%.*]] = lshr i64 [[D18:%.*]], 63
-; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP916]] to i32
-; CHECK-NEXT:    ret i32 [[TMP10]]
+; CHECK-NEXT:    [[I916:%.*]] = lshr i64 [[D18:%.*]], 63
+; CHECK-NEXT:    [[I10:%.*]] = trunc i64 [[I916]] to i32
+; CHECK-NEXT:    ret i32 [[I10]]
 ;
 entry:
-  %tmp916 = lshr i64 %d18, 32
-  %tmp917 = trunc i64 %tmp916 to i32
-  %tmp10 = lshr i32 %tmp917, 31
-  ret i32 %tmp10
+  %i916 = lshr i64 %d18, 32
+  %i917 = trunc i64 %i916 to i32
+  %i10 = lshr i32 %i917, 31
+  ret i32 %i10
 }
 
 
@@ -502,83 +502,83 @@ define i32 @test32(i32 %A, i32 %B, i32 %C) {
 
 define i1 @test33(i32 %X) {
 ; CHECK-LABEL: @test33(
-; CHECK-NEXT:    [[TMP1_MASK:%.*]] = and i32 [[X:%.*]], 16777216
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1_MASK]], 0
-; CHECK-NEXT:    ret i1 [[TMP2]]
+; CHECK-NEXT:    [[I1_MASK:%.*]] = and i32 [[X:%.*]], 16777216
+; CHECK-NEXT:    [[I2:%.*]] = icmp ne i32 [[I1_MASK]], 0
+; CHECK-NEXT:    ret i1 [[I2]]
 ;
-  %tmp1 = shl i32 %X, 7
-  %tmp2 = icmp slt i32 %tmp1, 0
-  ret i1 %tmp2
+  %i1 = shl i32 %X, 7
+  %i2 = icmp slt i32 %i1, 0
+  ret i1 %i2
 }
 
 define <2 x i1> @test33vec(<2 x i32> %X) {
 ; CHECK-LABEL: @test33vec(
-; CHECK-NEXT:    [[TMP1_MASK:%.*]] = and <2 x i32> [[X:%.*]], <i32 16777216, i32 16777216>
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i32> [[TMP1_MASK]], zeroinitializer
-; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
+; CHECK-NEXT:    [[I1_MASK:%.*]] = and <2 x i32> [[X:%.*]], <i32 16777216, i32 16777216>
+; CHECK-NEXT:    [[I2:%.*]] = icmp ne <2 x i32> [[I1_MASK]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[I2]]
 ;
-  %tmp1 = shl <2 x i32> %X, <i32 7, i32 7>
-  %tmp2 = icmp slt <2 x i32> %tmp1, zeroinitializer
-  ret <2 x i1> %tmp2
+  %i1 = shl <2 x i32> %X, <i32 7, i32 7>
+  %i2 = icmp slt <2 x i32> %i1, zeroinitializer
+  ret <2 x i1> %i2
 }
 
 define i1 @test34(i32 %X) {
 ; CHECK-LABEL: @test34(
 ; CHECK-NEXT:    ret i1 false
 ;
-  %tmp1 = lshr i32 %X, 7
-  %tmp2 = icmp slt i32 %tmp1, 0
-  ret i1 %tmp2
+  %i1 = lshr i32 %X, 7
+  %i2 = icmp slt i32 %i1, 0
+  ret i1 %i2
 }
 
 define i1 @test35(i32 %X) {
 ; CHECK-LABEL: @test35(
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X:%.*]], 0
-; CHECK-NEXT:    ret i1 [[TMP2]]
+; CHECK-NEXT:    [[I2:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[I2]]
 ;
-  %tmp1 = ashr i32 %X, 7
-  %tmp2 = icmp slt i32 %tmp1, 0
-  ret i1 %tmp2
+  %i1 = ashr i32 %X, 7
+  %i2 = icmp slt i32 %i1, 0
+  ret i1 %i2
 }
 
 define <2 x i1> @test35vec(<2 x i32> %X) {
 ; CHECK-LABEL: @test35vec(
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <2 x i32> [[X:%.*]], zeroinitializer
-; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
+; CHECK-NEXT:    [[I2:%.*]] = icmp slt <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[I2]]
 ;
-  %tmp1 = ashr <2 x i32> %X, <i32 7, i32 7>
-  %tmp2 = icmp slt <2 x i32> %tmp1, zeroinitializer
-  ret <2 x i1> %tmp2
+  %i1 = ashr <2 x i32> %X, <i32 7, i32 7>
+  %i2 = icmp slt <2 x i32> %i1, zeroinitializer
+  ret <2 x i1> %i2
 }
 
 define i128 @test36(i128 %A, i128 %B) {
 ; CHECK-LABEL: @test36(
-; CHECK-NEXT:    [[TMP231:%.*]] = or i128 [[B:%.*]], [[A:%.*]]
-; CHECK-NEXT:    [[INS:%.*]] = and i128 [[TMP231]], 18446744073709551615
+; CHECK-NEXT:    [[I231:%.*]] = or i128 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[INS:%.*]] = and i128 [[I231]], 18446744073709551615
 ; CHECK-NEXT:    ret i128 [[INS]]
 ;
-  %tmp27 = shl i128 %A, 64
-  %tmp23 = shl i128 %B, 64
-  %ins = or i128 %tmp23, %tmp27
-  %tmp45 = lshr i128 %ins, 64
-  ret i128 %tmp45
+  %i27 = shl i128 %A, 64
+  %i23 = shl i128 %B, 64
+  %ins = or i128 %i23, %i27
+  %i45 = lshr i128 %ins, 64
+  ret i128 %i45
 }
 
 define i64 @test37(i128 %A, i32 %B) {
 ; CHECK-LABEL: @test37(
-; CHECK-NEXT:    [[TMP22:%.*]] = zext i32 [[B:%.*]] to i128
-; CHECK-NEXT:    [[TMP23:%.*]] = shl nuw nsw i128 [[TMP22]], 32
-; CHECK-NEXT:    [[INS:%.*]] = or i128 [[TMP23]], [[A:%.*]]
-; CHECK-NEXT:    [[TMP46:%.*]] = trunc i128 [[INS]] to i64
-; CHECK-NEXT:    ret i64 [[TMP46]]
+; CHECK-NEXT:    [[I22:%.*]] = zext i32 [[B:%.*]] to i128
+; CHECK-NEXT:    [[I23:%.*]] = shl nuw nsw i128 [[I22]], 32
+; CHECK-NEXT:    [[INS:%.*]] = or i128 [[I23]], [[A:%.*]]
+; CHECK-NEXT:    [[I46:%.*]] = trunc i128 [[INS]] to i64
+; CHECK-NEXT:    ret i64 [[I46]]
 ;
-  %tmp27 = shl i128 %A, 64
-  %tmp22 = zext i32 %B to i128
-  %tmp23 = shl i128 %tmp22, 96
-  %ins = or i128 %tmp23, %tmp27
-  %tmp45 = lshr i128 %ins, 64
-  %tmp46 = trunc i128 %tmp45 to i64
-  ret i64 %tmp46
+  %i27 = shl i128 %A, 64
+  %i22 = zext i32 %B to i128
+  %i23 = shl i128 %i22, 96
+  %ins = or i128 %i23, %i27
+  %i45 = lshr i128 %ins, 64
+  %i46 = trunc i128 %i45 to i64
+  ret i64 %i46
 }
 
 define <2 x i32> @shl_nuw_nsw_splat_vec(<2 x i8> %x) {
@@ -607,28 +607,28 @@ define i32 @test38(i32 %x) nounwind readnone {
 define i8 @test39(i32 %a0) {
 ; CHECK-LABEL: @test39(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[A0:%.*]] to i8
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i8 [[TMP4]], 5
-; CHECK-NEXT:    [[TMP49:%.*]] = shl i8 [[TMP4]], 6
-; CHECK-NEXT:    [[TMP50:%.*]] = and i8 [[TMP49]], 64
-; CHECK-NEXT:    [[TMP51:%.*]] = xor i8 [[TMP50]], [[TMP5]]
-; CHECK-NEXT:    [[TMP0:%.*]] = shl i8 [[TMP4]], 2
-; CHECK-NEXT:    [[TMP54:%.*]] = and i8 [[TMP0]], 16
-; CHECK-NEXT:    [[TMP551:%.*]] = or i8 [[TMP54]], [[TMP51]]
-; CHECK-NEXT:    ret i8 [[TMP551]]
+; CHECK-NEXT:    [[I4:%.*]] = trunc i32 [[A0:%.*]] to i8
+; CHECK-NEXT:    [[I5:%.*]] = shl i8 [[I4]], 5
+; CHECK-NEXT:    [[I49:%.*]] = shl i8 [[I4]], 6
+; CHECK-NEXT:    [[I50:%.*]] = and i8 [[I49]], 64
+; CHECK-NEXT:    [[I51:%.*]] = xor i8 [[I50]], [[I5]]
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i8 [[I4]], 2
+; CHECK-NEXT:    [[I54:%.*]] = and i8 [[TMP0]], 16
+; CHECK-NEXT:    [[I551:%.*]] = or i8 [[I54]], [[I51]]
+; CHECK-NEXT:    ret i8 [[I551]]
 ;
 entry:
-  %tmp4 = trunc i32 %a0 to i8
-  %tmp5 = shl i8 %tmp4, 5
-  %tmp48 = and i8 %tmp5, 32
-  %tmp49 = lshr i8 %tmp48, 5
-  %tmp50 = mul i8 %tmp49, 64
-  %tmp51 = xor i8 %tmp50, %tmp5
-  %tmp52 = and i8 %tmp51, -128
-  %tmp53 = lshr i8 %tmp52, 7
-  %tmp54 = mul i8 %tmp53, 16
-  %tmp55 = xor i8 %tmp54, %tmp51
-  ret i8 %tmp55
+  %i4 = trunc i32 %a0 to i8
+  %i5 = shl i8 %i4, 5
+  %i48 = and i8 %i5, 32
+  %i49 = lshr i8 %i48, 5
+  %i50 = mul i8 %i49, 64
+  %i51 = xor i8 %i50, %i5
+  %i52 = and i8 %i51, -128
+  %i53 = lshr i8 %i52, 7
+  %i54 = mul i8 %i53, 16
+  %i55 = xor i8 %i54, %i51
+  ret i8 %i55
 }
 
 ; PR9809
@@ -1120,19 +1120,19 @@ bb2:                                              ; preds = %bb7, %bb1
   br i1 undef, label %bb3, label %bb7
 
 bb3:                                              ; preds = %bb2
-  %tmp = lshr i128 %arg, 36893488147419103232
-  %tmp4 = shl i128 %tmp, 0
-  %tmp5 = or i128 %tmp4, undef
-  %tmp6 = trunc i128 %tmp5 to i16
+  %i = lshr i128 %arg, 36893488147419103232
+  %i4 = shl i128 %i, 0
+  %i5 = or i128 %i4, undef
+  %i6 = trunc i128 %i5 to i16
   br label %bb8
 
 bb7:                                              ; preds = %bb2
   br i1 undef, label %bb8, label %bb2
 
 bb8:                                              ; preds = %bb7, %bb3
-  %tmp9 = phi i16 [ %tmp6, %bb3 ], [ undef, %bb7 ]
-  %tmp10 = icmp eq i16 %tmp9, 0
-  br i1 %tmp10, label %bb11, label %bb12
+  %i9 = phi i16 [ %i6, %bb3 ], [ undef, %bb7 ]
+  %i10 = icmp eq i16 %i9, 0
+  br i1 %i10, label %bb11, label %bb12
 
 bb11:                                             ; preds = %bb8
   br i1 undef, label %bb1, label %bb12
@@ -1656,15 +1656,15 @@ define void @ashr_out_of_range(i177* %A) {
 ; CHECK-LABEL: @ashr_out_of_range(
 ; CHECK-NEXT:    [[L:%.*]] = load i177, i177* [[A:%.*]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i177 [[L]], -1
-; CHECK-NEXT:    [[B2:%.*]] = select i1 [[TMP1]], i64 -1, i64 -2
-; CHECK-NEXT:    [[G11:%.*]] = getelementptr i177, i177* [[A]], i64 [[B2]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i64 -1, i64 -2
+; CHECK-NEXT:    [[G11:%.*]] = getelementptr i177, i177* [[A]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[L7:%.*]] = load i177, i177* [[G11]], align 4
 ; CHECK-NEXT:    [[B36:%.*]] = select i1 [[TMP1]], i177 0, i177 [[L7]]
 ; CHECK-NEXT:    [[C17:%.*]] = icmp sgt i177 [[B36]], [[L7]]
-; CHECK-NEXT:    [[TMP2:%.*]] = sext i1 [[C17]] to i64
-; CHECK-NEXT:    [[G62:%.*]] = getelementptr i177, i177* [[G11]], i64 [[TMP2]]
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i177 [[L7]], -1
-; CHECK-NEXT:    [[B28:%.*]] = select i1 [[TMP3]], i177 0, i177 [[L7]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sext i1 [[C17]] to i64
+; CHECK-NEXT:    [[G62:%.*]] = getelementptr i177, i177* [[G11]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i177 [[L7]], -1
+; CHECK-NEXT:    [[B28:%.*]] = select i1 [[TMP4]], i177 0, i177 [[L7]]
 ; CHECK-NEXT:    store i177 [[B28]], i177* [[G62]], align 4
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/InstCombine/sink_to_unreachable_dbg.ll b/llvm/test/Transforms/InstCombine/sink_to_unreachable_dbg.ll
new file mode 100644
index 0000000000000..e642276224b84
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sink_to_unreachable_dbg.ll
@@ -0,0 +1,46 @@
+; RUN: opt -debugify -debugify-level=locations -instcombine -S < %s | FileCheck %s
+
+; CHECK-LABEL: @test1(
+; CHECK: [[phi:%.*]] = phi i32
+; CHECK-NEXT: [[add:%.*]] = add i32 {{.*}}, 1{{$}}
+; CHECK-NEXT: add i32 [[phi]], [[add]], !dbg
+define i32 @test1(i32 %0, i1 %1) {
+  %3 = add i32 %0, 1
+  br i1 %1, label %4, label %5
+
+4:                                                ; preds = %2
+  br label %6
+
+5:                                                ; preds = %2
+  br label %6
+
+6:                                                ; preds = %5, %4
+  %7 = phi i32 [ 0, %4 ], [ 1, %5 ]
+  %8 = add i32 %7, %3
+  ret i32 %8
+}
+
+; Function Attrs: nounwind readnone
+declare i32 @external(i32) #0
+
+; CHECK-LABEL: @test2(
+; CHECK: [[phi:%.*]] = phi i32
+; CHECK-NEXT: [[add:%.*]] = call i32 @external(i32 {{.*}}), !dbg
+; CHECK-NEXT: add i32 [[phi]], [[add]], !dbg
+define i32 @test2(i32 %0, i1 %1) {
+  %3 = call i32 @external(i32 %0)
+  br i1 %1, label %4, label %5
+
+4:                                                ; preds = %2
+  br label %6
+
+5:                                                ; preds = %2
+  br label %6
+
+6:                                                ; preds = %5, %4
+  %7 = phi i32 [ 0, %4 ], [ 1, %5 ]
+  %8 = add i32 %7, %3
+  ret i32 %8
+}
+
+attributes #0 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/trunc-binop-ext.ll b/llvm/test/Transforms/InstCombine/trunc-binop-ext.ll
index 40d58f31458eb..570153163ab93 100644
--- a/llvm/test/Transforms/InstCombine/trunc-binop-ext.ll
+++ b/llvm/test/Transforms/InstCombine/trunc-binop-ext.ll
@@ -315,3 +315,88 @@ define <2 x i16> @narrow_zext_mul_commute(<2 x i16> %x16, <2 x i32> %y32) {
   ret <2 x i16> %r
 }
 
+; Test cases for PR43580
+define i8 @narrow_zext_ashr_keep_trunc(i8 %i1, i8 %i2) {
+; CHECK-LABEL: @narrow_zext_ashr_keep_trunc(
+; CHECK-NEXT:    [[I1_EXT:%.*]] = sext i8 [[I1:%.*]] to i16
+; CHECK-NEXT:    [[I2_EXT:%.*]] = sext i8 [[I2:%.*]] to i16
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i16 [[I1_EXT]], [[I2_EXT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i16 [[SUB]], 1
+; CHECK-NEXT:    [[T:%.*]] = trunc i16 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[T]]
+;
+  %i1.ext = sext i8 %i1 to i32
+  %i2.ext = sext i8 %i2 to i32
+  %sub = add nsw i32 %i1.ext, %i2.ext
+  %shift = ashr i32 %sub, 1
+  %t = trunc i32 %shift to i8
+  ret i8 %t
+}
+
+define i8 @narrow_zext_ashr_keep_trunc2(i9 %i1, i9 %i2) {
+; CHECK-LABEL: @narrow_zext_ashr_keep_trunc2(
+; CHECK-NEXT:    [[I1_EXT1:%.*]] = zext i9 [[I1:%.*]] to i16
+; CHECK-NEXT:    [[I2_EXT2:%.*]] = zext i9 [[I2:%.*]] to i16
+; CHECK-NEXT:    [[SUB:%.*]] = add nuw nsw i16 [[I1_EXT1]], [[I2_EXT2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i16 [[SUB]], 1
+; CHECK-NEXT:    [[T:%.*]] = trunc i16 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[T]]
+;
+  %i1.ext = sext i9 %i1 to i64
+  %i2.ext = sext i9 %i2 to i64
+  %sub = add nsw i64 %i1.ext, %i2.ext
+  %shift = ashr i64 %sub, 1
+  %t = trunc i64 %shift to i8
+  ret i8 %t
+}
+
+define i7 @narrow_zext_ashr_keep_trunc3(i8 %i1, i8 %i2) {
+; CHECK-LABEL: @narrow_zext_ashr_keep_trunc3(
+; CHECK-NEXT:    [[I1_EXT1:%.*]] = zext i8 [[I1:%.*]] to i14
+; CHECK-NEXT:    [[I2_EXT2:%.*]] = zext i8 [[I2:%.*]] to i14
+; CHECK-NEXT:    [[SUB:%.*]] = add nuw nsw i14 [[I1_EXT1]], [[I2_EXT2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i14 [[SUB]], 1
+; CHECK-NEXT:    [[T:%.*]] = trunc i14 [[TMP1]] to i7
+; CHECK-NEXT:    ret i7 [[T]]
+;
+  %i1.ext = sext i8 %i1 to i64
+  %i2.ext = sext i8 %i2 to i64
+  %sub = add nsw i64 %i1.ext, %i2.ext
+  %shift = ashr i64 %sub, 1
+  %t = trunc i64 %shift to i7
+  ret i7 %t
+}
+
+define <8 x i8> @narrow_zext_ashr_keep_trunc_vector(<8 x i8> %i1, <8 x i8> %i2) {
+; CHECK-LABEL: @narrow_zext_ashr_keep_trunc_vector(
+; CHECK-NEXT:    [[I1_EXT:%.*]] = sext <8 x i8> [[I1:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[I2_EXT:%.*]] = sext <8 x i8> [[I2:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw <8 x i32> [[I1_EXT]], [[I2_EXT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> [[SUB]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[T:%.*]] = trunc <8 x i32> [[TMP1]] to <8 x i8>
+; CHECK-NEXT:    ret <8 x i8> [[T]]
+;
+  %i1.ext = sext <8 x i8> %i1 to <8 x i32>
+  %i2.ext = sext <8 x i8> %i2 to <8 x i32>
+  %sub = add nsw <8 x i32> %i1.ext, %i2.ext
+  %shift = ashr <8 x i32> %sub, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %t = trunc <8 x i32> %shift to <8 x i8>
+  ret <8 x i8> %t
+}
+
+define i8 @dont_narrow_zext_ashr_keep_trunc(i8 %i1, i8 %i2) {
+; CHECK-LABEL: @dont_narrow_zext_ashr_keep_trunc(
+; CHECK-NEXT:    [[I1_EXT:%.*]] = sext i8 [[I1:%.*]] to i16
+; CHECK-NEXT:    [[I2_EXT:%.*]] = sext i8 [[I2:%.*]] to i16
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i16 [[I1_EXT]], [[I2_EXT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i16 [[SUB]], 1
+; CHECK-NEXT:    [[T:%.*]] = trunc i16 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[T]]
+;
+  %i1.ext = sext i8 %i1 to i16
+  %i2.ext = sext i8 %i2 to i16
+  %sub = add nsw i16 %i1.ext, %i2.ext
+  %shift = ashr i16 %sub, 1
+  %t = trunc i16 %shift to i8
+  ret i8 %t
+}
diff --git a/llvm/test/Transforms/InstCombine/trunc.ll b/llvm/test/Transforms/InstCombine/trunc.ll
index d9ad3753e91a1..d8a615cc4c9a3 100644
--- a/llvm/test/Transforms/InstCombine/trunc.ll
+++ b/llvm/test/Transforms/InstCombine/trunc.ll
@@ -5,6 +5,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 ; Instcombine should be able to eliminate all of these ext casts.
 
 declare void @use(i32)
+declare void @use_vec(<2 x i32>)
 
 define i64 @test1(i64 %a) {
 ; CHECK-LABEL: @test1(
@@ -20,6 +21,48 @@ define i64 @test1(i64 %a) {
   ret i64 %d
 }
 
+define <2 x i64> @test1_vec(<2 x i64> %a) {
+; CHECK-LABEL: @test1_vec(
+; CHECK-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[D:%.*]] = and <2 x i64> [[A]], <i64 15, i64 15>
+; CHECK-NEXT:    call void @use_vec(<2 x i32> [[B]])
+; CHECK-NEXT:    ret <2 x i64> [[D]]
+;
+  %b = trunc <2 x i64> %a to <2 x i32>
+  %c = and <2 x i32> %b, <i32 15, i32 15>
+  %d = zext <2 x i32> %c to <2 x i64>
+  call void @use_vec(<2 x i32> %b)
+  ret <2 x i64> %d
+}
+
+define <2 x i64> @test1_vec_nonuniform(<2 x i64> %a) {
+; CHECK-LABEL: @test1_vec_nonuniform(
+; CHECK-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[D:%.*]] = and <2 x i64> [[A]], <i64 15, i64 7>
+; CHECK-NEXT:    call void @use_vec(<2 x i32> [[B]])
+; CHECK-NEXT:    ret <2 x i64> [[D]]
+;
+  %b = trunc <2 x i64> %a to <2 x i32>
+  %c = and <2 x i32> %b, <i32 15, i32 7>
+  %d = zext <2 x i32> %c to <2 x i64>
+  call void @use_vec(<2 x i32> %b)
+  ret <2 x i64> %d
+}
+
+define <2 x i64> @test1_vec_undef(<2 x i64> %a) {
+; CHECK-LABEL: @test1_vec_undef(
+; CHECK-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[D:%.*]] = and <2 x i64> [[A]], <i64 15, i64 0>
+; CHECK-NEXT:    call void @use_vec(<2 x i32> [[B]])
+; CHECK-NEXT:    ret <2 x i64> [[D]]
+;
+  %b = trunc <2 x i64> %a to <2 x i32>
+  %c = and <2 x i32> %b, <i32 15, i32 undef>
+  %d = zext <2 x i32> %c to <2 x i64>
+  call void @use_vec(<2 x i32> %b)
+  ret <2 x i64> %d
+}
+
 define i64 @test2(i64 %a) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:    [[B:%.*]] = trunc i64 [[A:%.*]] to i32
@@ -36,6 +79,54 @@ define i64 @test2(i64 %a) {
   ret i64 %d
 }
 
+define <2 x i64> @test2_vec(<2 x i64> %a) {
+; CHECK-LABEL: @test2_vec(
+; CHECK-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[D1:%.*]] = shl <2 x i64> [[A]], <i64 36, i64 36>
+; CHECK-NEXT:    [[D:%.*]] = ashr exact <2 x i64> [[D1]], <i64 36, i64 36>
+; CHECK-NEXT:    call void @use_vec(<2 x i32> [[B]])
+; CHECK-NEXT:    ret <2 x i64> [[D]]
+;
+  %b = trunc <2 x i64> %a to <2 x i32>
+  %c = shl <2 x i32> %b, <i32 4, i32 4>
+  %q = ashr <2 x i32> %c, <i32 4, i32 4>
+  %d = sext <2 x i32> %q to <2 x i64>
+  call void @use_vec(<2 x i32> %b)
+  ret <2 x i64> %d
+}
+
+define <2 x i64> @test2_vec_nonuniform(<2 x i64> %a) {
+; CHECK-LABEL: @test2_vec_nonuniform(
+; CHECK-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[D1:%.*]] = shl <2 x i64> [[A]], <i64 36, i64 37>
+; CHECK-NEXT:    [[D:%.*]] = ashr <2 x i64> [[D1]], <i64 36, i64 37>
+; CHECK-NEXT:    call void @use_vec(<2 x i32> [[B]])
+; CHECK-NEXT:    ret <2 x i64> [[D]]
+;
+  %b = trunc <2 x i64> %a to <2 x i32>
+  %c = shl <2 x i32> %b, <i32 4, i32 5>
+  %q = ashr <2 x i32> %c, <i32 4, i32 5>
+  %d = sext <2 x i32> %q to <2 x i64>
+  call void @use_vec(<2 x i32> %b)
+  ret <2 x i64> %d
+}
+
+define <2 x i64> @test2_vec_undef(<2 x i64> %a) {
+; CHECK-LABEL: @test2_vec_undef(
+; CHECK-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[D1:%.*]] = shl <2 x i64> [[A]], <i64 36, i64 0>
+; CHECK-NEXT:    [[D:%.*]] = ashr <2 x i64> [[D1]], <i64 36, i64 0>
+; CHECK-NEXT:    call void @use_vec(<2 x i32> [[B]])
+; CHECK-NEXT:    ret <2 x i64> [[D]]
+;
+  %b = trunc <2 x i64> %a to <2 x i32>
+  %c = shl <2 x i32> %b, <i32 4, i32 undef>
+  %q = ashr <2 x i32> %c, <i32 4, i32 undef>
+  %d = sext <2 x i32> %q to <2 x i64>
+  call void @use_vec(<2 x i32> %b)
+  ret <2 x i64> %d
+}
+
 define i64 @test3(i64 %a) {
 ; CHECK-LABEL: @test3(
 ; CHECK-NEXT:    [[B:%.*]] = trunc i64 [[A:%.*]] to i32
@@ -163,18 +254,67 @@ define i92 @test7(i64 %A) {
 
 define i64 @test8(i32 %A, i32 %B) {
 ; CHECK-LABEL: @test8(
-; CHECK-NEXT:    [[TMP38:%.*]] = zext i32 [[A:%.*]] to i64
-; CHECK-NEXT:    [[TMP32:%.*]] = zext i32 [[B:%.*]] to i64
-; CHECK-NEXT:    [[TMP33:%.*]] = shl nuw i64 [[TMP32]], 32
-; CHECK-NEXT:    [[INS35:%.*]] = or i64 [[TMP33]], [[TMP38]]
-; CHECK-NEXT:    ret i64 [[INS35]]
-;
-  %tmp38 = zext i32 %A to i128
-  %tmp32 = zext i32 %B to i128
-  %tmp33 = shl i128 %tmp32, 32
-  %ins35 = or i128 %tmp33, %tmp38
-  %tmp42 = trunc i128 %ins35 to i64
-  ret i64 %tmp42
+; CHECK-NEXT:    [[C:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[D:%.*]] = zext i32 [[B:%.*]] to i64
+; CHECK-NEXT:    [[E:%.*]] = shl nuw i64 [[D]], 32
+; CHECK-NEXT:    [[F:%.*]] = or i64 [[E]], [[C]]
+; CHECK-NEXT:    ret i64 [[F]]
+;
+  %C = zext i32 %A to i128
+  %D = zext i32 %B to i128
+  %E = shl i128 %D, 32
+  %F = or i128 %E, %C
+  %G = trunc i128 %F to i64
+  ret i64 %G
+}
+
+define <2 x i64> @test8_vec(<2 x i32> %A, <2 x i32> %B) {
+; CHECK-LABEL: @test8_vec(
+; CHECK-NEXT:    [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[D:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[E:%.*]] = shl nuw <2 x i64> [[D]], <i64 32, i64 32>
+; CHECK-NEXT:    [[F:%.*]] = or <2 x i64> [[E]], [[C]]
+; CHECK-NEXT:    ret <2 x i64> [[F]]
+;
+  %C = zext <2 x i32> %A to <2 x i128>
+  %D = zext <2 x i32> %B to <2 x i128>
+  %E = shl <2 x i128> %D, <i128 32, i128 32>
+  %F = or <2 x i128> %E, %C
+  %G = trunc <2 x i128> %F to <2 x i64>
+  ret <2 x i64> %G
+}
+
+define <2 x i64> @test8_vec_nonuniform(<2 x i32> %A, <2 x i32> %B) {
+; CHECK-LABEL: @test8_vec_nonuniform(
+; CHECK-NEXT:    [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[D:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[E:%.*]] = shl <2 x i64> [[D]], <i64 32, i64 48>
+; CHECK-NEXT:    [[F:%.*]] = or <2 x i64> [[E]], [[C]]
+; CHECK-NEXT:    ret <2 x i64> [[F]]
+;
+  %C = zext <2 x i32> %A to <2 x i128>
+  %D = zext <2 x i32> %B to <2 x i128>
+  %E = shl <2 x i128> %D, <i128 32, i128 48>
+  %F = or <2 x i128> %E, %C
+  %G = trunc <2 x i128> %F to <2 x i64>
+  ret <2 x i64> %G
+}
+
+define <2 x i64> @test8_vec_undef(<2 x i32> %A, <2 x i32> %B) {
+; CHECK-LABEL: @test8_vec_undef(
+; CHECK-NEXT:    [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i128>
+; CHECK-NEXT:    [[D:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i128>
+; CHECK-NEXT:    [[E:%.*]] = shl <2 x i128> [[D]], <i128 32, i128 undef>
+; CHECK-NEXT:    [[F:%.*]] = or <2 x i128> [[E]], [[C]]
+; CHECK-NEXT:    [[G:%.*]] = trunc <2 x i128> [[F]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[G]]
+;
+  %C = zext <2 x i32> %A to <2 x i128>
+  %D = zext <2 x i32> %B to <2 x i128>
+  %E = shl <2 x i128> %D, <i128 32, i128 undef>
+  %F = or <2 x i128> %E, %C
+  %G = trunc <2 x i128> %F to <2 x i64>
+  ret <2 x i64> %G
 }
 
 define i8 @test9(i32 %X) {
@@ -200,6 +340,201 @@ define i8 @test10(i32 %X) {
   ret i8 %Z
 }
 
+define i64 @test11(i32 %A, i32 %B) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[C:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[B:%.*]], 31
+; CHECK-NEXT:    [[E:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[F:%.*]] = shl i64 [[C]], [[E]]
+; CHECK-NEXT:    ret i64 [[F]]
+;
+  %C = zext i32 %A to i128
+  %D = zext i32 %B to i128
+  %E = and i128 %D, 31
+  %F = shl i128 %C, %E
+  %G = trunc i128 %F to i64
+  ret i64 %G
+}
+
+define <2 x i64> @test11_vec(<2 x i32> %A, <2 x i32> %B) {
+; CHECK-LABEL: @test11_vec(
+; CHECK-NEXT:    [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[B:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    [[E:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[F:%.*]] = shl <2 x i64> [[C]], [[E]]
+; CHECK-NEXT:    ret <2 x i64> [[F]]
+;
+  %C = zext <2 x i32> %A to <2 x i128>
+  %D = zext <2 x i32> %B to <2 x i128>
+  %E = and <2 x i128> %D, <i128 31, i128 31>
+  %F = shl <2 x i128> %C, %E
+  %G = trunc <2 x i128> %F to <2 x i64>
+  ret <2 x i64> %G
+}
+
+define <2 x i64> @test11_vec_nonuniform(<2 x i32> %A, <2 x i32> %B) {
+; CHECK-LABEL: @test11_vec_nonuniform(
+; CHECK-NEXT:    [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[B:%.*]], <i32 31, i32 15>
+; CHECK-NEXT:    [[E:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[F:%.*]] = shl <2 x i64> [[C]], [[E]]
+; CHECK-NEXT:    ret <2 x i64> [[F]]
+;
+  %C = zext <2 x i32> %A to <2 x i128>
+  %D = zext <2 x i32> %B to <2 x i128>
+  %E = and <2 x i128> %D, <i128 31, i128 15>
+  %F = shl <2 x i128> %C, %E
+  %G = trunc <2 x i128> %F to <2 x i64>
+  ret <2 x i64> %G
+}
+
+define <2 x i64> @test11_vec_undef(<2 x i32> %A, <2 x i32> %B) {
+; CHECK-LABEL: @test11_vec_undef(
+; CHECK-NEXT:    [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i128>
+; CHECK-NEXT:    [[D:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i128>
+; CHECK-NEXT:    [[E:%.*]] = and <2 x i128> [[D]], <i128 31, i128 undef>
+; CHECK-NEXT:    [[F:%.*]] = shl <2 x i128> [[C]], [[E]]
+; CHECK-NEXT:    [[G:%.*]] = trunc <2 x i128> [[F]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[G]]
+;
+  %C = zext <2 x i32> %A to <2 x i128>
+  %D = zext <2 x i32> %B to <2 x i128>
+  %E = and <2 x i128> %D, <i128 31, i128 undef>
+  %F = shl <2 x i128> %C, %E
+  %G = trunc <2 x i128> %F to <2 x i64>
+  ret <2 x i64> %G
+}
+
+define i64 @test12(i32 %A, i32 %B) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[C:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[B:%.*]], 31
+; CHECK-NEXT:    [[E:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[F:%.*]] = lshr i64 [[C]], [[E]]
+; CHECK-NEXT:    ret i64 [[F]]
+;
+  %C = zext i32 %A to i128
+  %D = zext i32 %B to i128
+  %E = and i128 %D, 31
+  %F = lshr i128 %C, %E
+  %G = trunc i128 %F to i64
+  ret i64 %G
+}
+
+define <2 x i64> @test12_vec(<2 x i32> %A, <2 x i32> %B) {
+; CHECK-LABEL: @test12_vec(
+; CHECK-NEXT:    [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[B:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    [[E:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[F:%.*]] = lshr <2 x i64> [[C]], [[E]]
+; CHECK-NEXT:    ret <2 x i64> [[F]]
+;
+  %C = zext <2 x i32> %A to <2 x i128>
+  %D = zext <2 x i32> %B to <2 x i128>
+  %E = and <2 x i128> %D, <i128 31, i128 31>
+  %F = lshr <2 x i128> %C, %E
+  %G = trunc <2 x i128> %F to <2 x i64>
+  ret <2 x i64> %G
+}
+
+define <2 x i64> @test12_vec_nonuniform(<2 x i32> %A, <2 x i32> %B) {
+; CHECK-LABEL: @test12_vec_nonuniform(
+; CHECK-NEXT:    [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[B:%.*]], <i32 31, i32 15>
+; CHECK-NEXT:    [[E:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[F:%.*]] = lshr <2 x i64> [[C]], [[E]]
+; CHECK-NEXT:    ret <2 x i64> [[F]]
+;
+  %C = zext <2 x i32> %A to <2 x i128>
+  %D = zext <2 x i32> %B to <2 x i128>
+  %E = and <2 x i128> %D, <i128 31, i128 15>
+  %F = lshr <2 x i128> %C, %E
+  %G = trunc <2 x i128> %F to <2 x i64>
+  ret <2 x i64> %G
+}
+
+define <2 x i64> @test12_vec_undef(<2 x i32> %A, <2 x i32> %B) {
+; CHECK-LABEL: @test12_vec_undef(
+; CHECK-NEXT:    [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i128>
+; CHECK-NEXT:    [[D:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i128>
+; CHECK-NEXT:    [[E:%.*]] = and <2 x i128> [[D]], <i128 31, i128 undef>
+; CHECK-NEXT:    [[F:%.*]] = lshr <2 x i128> [[C]], [[E]]
+; CHECK-NEXT:    [[G:%.*]] = trunc <2 x i128> [[F]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[G]]
+;
+  %C = zext <2 x i32> %A to <2 x i128>
+  %D = zext <2 x i32> %B to <2 x i128>
+  %E = and <2 x i128> %D, <i128 31, i128 undef>
+  %F = lshr <2 x i128> %C, %E
+  %G = trunc <2 x i128> %F to <2 x i64>
+  ret <2 x i64> %G
+}
+
+define i64 @test13(i32 %A, i32 %B) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[C:%.*]] = sext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[B:%.*]], 31
+; CHECK-NEXT:    [[E:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[F:%.*]] = ashr i64 [[C]], [[E]]
+; CHECK-NEXT:    ret i64 [[F]]
+;
+  %C = sext i32 %A to i128
+  %D = zext i32 %B to i128
+  %E = and i128 %D, 31
+  %F = ashr i128 %C, %E
+  %G = trunc i128 %F to i64
+  ret i64 %G
+}
+
+define <2 x i64> @test13_vec(<2 x i32> %A, <2 x i32> %B) {
+; CHECK-LABEL: @test13_vec(
+; CHECK-NEXT:    [[C:%.*]] = sext <2 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[B:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    [[E:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[F:%.*]] = ashr <2 x i64> [[C]], [[E]]
+; CHECK-NEXT:    ret <2 x i64> [[F]]
+;
+  %C = sext <2 x i32> %A to <2 x i128>
+  %D = zext <2 x i32> %B to <2 x i128>
+  %E = and <2 x i128> %D, <i128 31, i128 31>
+  %F = ashr <2 x i128> %C, %E
+  %G = trunc <2 x i128> %F to <2 x i64>
+  ret <2 x i64> %G
+}
+
+define <2 x i64> @test13_vec_nonuniform(<2 x i32> %A, <2 x i32> %B) {
+; CHECK-LABEL: @test13_vec_nonuniform(
+; CHECK-NEXT:    [[C:%.*]] = sext <2 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[B:%.*]], <i32 31, i32 15>
+; CHECK-NEXT:    [[E:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[F:%.*]] = ashr <2 x i64> [[C]], [[E]]
+; CHECK-NEXT:    ret <2 x i64> [[F]]
+;
+  %C = sext <2 x i32> %A to <2 x i128>
+  %D = zext <2 x i32> %B to <2 x i128>
+  %E = and <2 x i128> %D, <i128 31, i128 15>
+  %F = ashr <2 x i128> %C, %E
+  %G = trunc <2 x i128> %F to <2 x i64>
+  ret <2 x i64> %G
+}
+
+define <2 x i64> @test13_vec_undef(<2 x i32> %A, <2 x i32> %B) {
+; CHECK-LABEL: @test13_vec_undef(
+; CHECK-NEXT:    [[C:%.*]] = sext <2 x i32> [[A:%.*]] to <2 x i128>
+; CHECK-NEXT:    [[D:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i128>
+; CHECK-NEXT:    [[E:%.*]] = and <2 x i128> [[D]], <i128 31, i128 undef>
+; CHECK-NEXT:    [[F:%.*]] = ashr <2 x i128> [[C]], [[E]]
+; CHECK-NEXT:    [[G:%.*]] = trunc <2 x i128> [[F]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[G]]
+;
+  %C = sext <2 x i32> %A to <2 x i128>
+  %D = zext <2 x i32> %B to <2 x i128>
+  %E = and <2 x i128> %D, <i128 31, i128 undef>
+  %F = ashr <2 x i128> %C, %E
+  %G = trunc <2 x i128> %F to <2 x i64>
+  ret <2 x i64> %G
+}
+
 ; PR25543
 ; https://llvm.org/bugs/show_bug.cgi?id=25543
 ; This is an extractelement.
@@ -409,79 +744,79 @@ define void @trunc_shl_31_i32_i64_multi_use(i64 %val, i32 addrspace(1)* %ptr0, i
 
 define i32 @trunc_shl_lshr_infloop(i64 %arg) {
 ; CHECK-LABEL: @trunc_shl_lshr_infloop(
-; CHECK-NEXT:    [[TMP0:%.*]] = lshr i64 [[ARG:%.*]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
-; CHECK-NEXT:    ret i32 [[TMP2]]
+; CHECK-NEXT:    [[A:%.*]] = lshr i64 [[ARG:%.*]], 1
+; CHECK-NEXT:    [[B:%.*]] = shl i64 [[A]], 2
+; CHECK-NEXT:    [[C:%.*]] = trunc i64 [[B]] to i32
+; CHECK-NEXT:    ret i32 [[C]]
 ;
-  %tmp0 = lshr i64 %arg, 1
-  %tmp1 = shl i64 %tmp0, 2
-  %tmp2 = trunc i64 %tmp1 to i32
-  ret i32 %tmp2
+  %A = lshr i64 %arg, 1
+  %B = shl i64 %A, 2
+  %C = trunc i64 %B to i32
+  ret i32 %C
 }
 
 define i32 @trunc_shl_ashr_infloop(i64 %arg) {
 ; CHECK-LABEL: @trunc_shl_ashr_infloop(
-; CHECK-NEXT:    [[TMP0:%.*]] = ashr i64 [[ARG:%.*]], 3
-; CHECK-NEXT:    [[TMP1:%.*]] = shl nsw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
-; CHECK-NEXT:    ret i32 [[TMP2]]
+; CHECK-NEXT:    [[A:%.*]] = ashr i64 [[ARG:%.*]], 3
+; CHECK-NEXT:    [[B:%.*]] = shl nsw i64 [[A]], 2
+; CHECK-NEXT:    [[C:%.*]] = trunc i64 [[B]] to i32
+; CHECK-NEXT:    ret i32 [[C]]
 ;
-  %tmp0 = ashr i64 %arg, 3
-  %tmp1 = shl i64 %tmp0, 2
-  %tmp2 = trunc i64 %tmp1 to i32
-  ret i32 %tmp2
+  %A = ashr i64 %arg, 3
+  %B = shl i64 %A, 2
+  %C = trunc i64 %B to i32
+  ret i32 %C
 }
 
 define i32 @trunc_shl_shl_infloop(i64 %arg) {
 ; CHECK-LABEL: @trunc_shl_shl_infloop(
 ; CHECK-NEXT:    [[ARG_TR:%.*]] = trunc i64 [[ARG:%.*]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[ARG_TR]], 3
-; CHECK-NEXT:    ret i32 [[TMP2]]
+; CHECK-NEXT:    [[C:%.*]] = shl i32 [[ARG_TR]], 3
+; CHECK-NEXT:    ret i32 [[C]]
 ;
-  %tmp0 = shl i64 %arg, 1
-  %tmp1 = shl i64 %tmp0, 2
-  %tmp2 = trunc i64 %tmp1 to i32
-  ret i32 %tmp2
+  %A = shl i64 %arg, 1
+  %B = shl i64 %A, 2
+  %C = trunc i64 %B to i32
+  ret i32 %C
 }
 
 define i32 @trunc_shl_lshr_var(i64 %arg, i64 %val) {
 ; CHECK-LABEL: @trunc_shl_lshr_var(
-; CHECK-NEXT:    [[TMP0:%.*]] = lshr i64 [[ARG:%.*]], [[VAL:%.*]]
-; CHECK-NEXT:    [[TMP0_TR:%.*]] = trunc i64 [[TMP0]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP0_TR]], 2
-; CHECK-NEXT:    ret i32 [[TMP2]]
+; CHECK-NEXT:    [[A:%.*]] = lshr i64 [[ARG:%.*]], [[VAL:%.*]]
+; CHECK-NEXT:    [[A_TR:%.*]] = trunc i64 [[A]] to i32
+; CHECK-NEXT:    [[C:%.*]] = shl i32 [[A_TR]], 2
+; CHECK-NEXT:    ret i32 [[C]]
 ;
-  %tmp0 = lshr i64 %arg, %val
-  %tmp1 = shl i64 %tmp0, 2
-  %tmp2 = trunc i64 %tmp1 to i32
-  ret i32 %tmp2
+  %A = lshr i64 %arg, %val
+  %B = shl i64 %A, 2
+  %C = trunc i64 %B to i32
+  ret i32 %C
 }
 
 define i32 @trunc_shl_ashr_var(i64 %arg, i64 %val) {
 ; CHECK-LABEL: @trunc_shl_ashr_var(
-; CHECK-NEXT:    [[TMP0:%.*]] = ashr i64 [[ARG:%.*]], [[VAL:%.*]]
-; CHECK-NEXT:    [[TMP0_TR:%.*]] = trunc i64 [[TMP0]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP0_TR]], 2
-; CHECK-NEXT:    ret i32 [[TMP2]]
+; CHECK-NEXT:    [[A:%.*]] = ashr i64 [[ARG:%.*]], [[VAL:%.*]]
+; CHECK-NEXT:    [[A_TR:%.*]] = trunc i64 [[A]] to i32
+; CHECK-NEXT:    [[C:%.*]] = shl i32 [[A_TR]], 2
+; CHECK-NEXT:    ret i32 [[C]]
 ;
-  %tmp0 = ashr i64 %arg, %val
-  %tmp1 = shl i64 %tmp0, 2
-  %tmp2 = trunc i64 %tmp1 to i32
-  ret i32 %tmp2
+  %A = ashr i64 %arg, %val
+  %B = shl i64 %A, 2
+  %C = trunc i64 %B to i32
+  ret i32 %C
 }
 
 define i32 @trunc_shl_shl_var(i64 %arg, i64 %val) {
 ; CHECK-LABEL: @trunc_shl_shl_var(
-; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[ARG:%.*]], [[VAL:%.*]]
-; CHECK-NEXT:    [[TMP0_TR:%.*]] = trunc i64 [[TMP0]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP0_TR]], 2
-; CHECK-NEXT:    ret i32 [[TMP2]]
-;
-  %tmp0 = shl i64 %arg, %val
-  %tmp1 = shl i64 %tmp0, 2
-  %tmp2 = trunc i64 %tmp1 to i32
-  ret i32 %tmp2
+; CHECK-NEXT:    [[A:%.*]] = shl i64 [[ARG:%.*]], [[VAL:%.*]]
+; CHECK-NEXT:    [[A_TR:%.*]] = trunc i64 [[A]] to i32
+; CHECK-NEXT:    [[C:%.*]] = shl i32 [[A_TR]], 2
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = shl i64 %arg, %val
+  %B = shl i64 %A, 2
+  %C = trunc i64 %B to i32
+  ret i32 %C
 }
 
 define <8 x i16> @trunc_shl_v8i15_v8i32_15(<8 x i32> %a) {
diff --git a/llvm/test/Transforms/InstCombine/vector-reductions.ll b/llvm/test/Transforms/InstCombine/vector-reductions.ll
index dfac8da97b359..f11307468457c 100644
--- a/llvm/test/Transforms/InstCombine/vector-reductions.ll
+++ b/llvm/test/Transforms/InstCombine/vector-reductions.ll
@@ -5,6 +5,10 @@ declare float @llvm.experimental.vector.reduce.v2.fadd.f32.v4f32(float, <4 x flo
 declare float @llvm.experimental.vector.reduce.v2.fadd.f32.v8f32(float, <8 x float>)
 declare void @use_f32(float)
 
+declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
+declare i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32>)
+declare void @use_i32(i32)
+
 define float @diff_of_sums_v4f32(float %a0, <4 x float> %v0, float %a1, <4 x float> %v1) {
 ; CHECK-LABEL: @diff_of_sums_v4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fsub reassoc nsz <4 x float> [[V0:%.*]], [[V1:%.*]]
@@ -81,3 +85,64 @@ define float @diff_of_sums_type_mismatch(float %a0, <4 x float> %v0, float %a1,
   %r = fsub fast float %r0, %r1
   ret float %r
 }
+
+define i32 @diff_of_sums_v4i32(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @diff_of_sums_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <4 x i32> [[V0:%.*]], [[V1:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %r0 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %v0)
+  %r1 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %v1)
+  %r = sub i32 %r0, %r1
+  ret i32 %r
+}
+
+; negative test - extra uses could create extra instructions
+
+define i32 @diff_of_sums_v4i32_extra_use1(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @diff_of_sums_v4i32_extra_use1(
+; CHECK-NEXT:    [[R0:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[V0:%.*]])
+; CHECK-NEXT:    call void @use_i32(i32 [[R0]])
+; CHECK-NEXT:    [[R1:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[V1:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = sub i32 [[R0]], [[R1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %r0 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %v0)
+  call void @use_i32(i32 %r0)
+  %r1 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %v1)
+  %r = sub i32 %r0, %r1
+  ret i32 %r
+}
+
+; negative test - extra uses could create extra instructions
+
+define i32 @diff_of_sums_v4i32_extra_use2(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @diff_of_sums_v4i32_extra_use2(
+; CHECK-NEXT:    [[R0:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[V0:%.*]])
+; CHECK-NEXT:    [[R1:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[V1:%.*]])
+; CHECK-NEXT:    call void @use_i32(i32 [[R1]])
+; CHECK-NEXT:    [[R:%.*]] = sub i32 [[R0]], [[R1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %r0 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %v0)
+  %r1 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %v1)
+  call void @use_i32(i32 %r1)
+  %r = sub i32 %r0, %r1
+  ret i32 %r
+}
+
+; negative test - can't reassociate different vector types
+
+define i32 @diff_of_sums_type_mismatch2(<8 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @diff_of_sums_type_mismatch2(
+; CHECK-NEXT:    [[R0:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> [[V0:%.*]])
+; CHECK-NEXT:    [[R1:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[V1:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = sub i32 [[R0]], [[R1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %r0 = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> %v0)
+  %r1 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %v1)
+  %r = sub i32 %r0, %r1
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/vector_gep1.ll b/llvm/test/Transforms/InstCombine/vector_gep1.ll
index 8e5bcf963ea1c..4eb449edb3480 100644
--- a/llvm/test/Transforms/InstCombine/vector_gep1.ll
+++ b/llvm/test/Transforms/InstCombine/vector_gep1.ll
@@ -62,3 +62,13 @@ define <2 x i32*> @test7(<2 x {i32, i32}*> %a) {
   ret <2 x i32*> %w
 }
 
+define <vscale x 2 x i1> @test8() {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    ret <vscale x 2 x i1> icmp ult (<vscale x 2 x i64> zext (<vscale x 2 x i32> shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> undef, i32 1, i32 0), <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer) to <vscale x 2 x i64>), <vscale x 2 x i64> zeroinitializer)
+;
+  %ins = insertelement <vscale x 2 x i32> undef, i32 1, i32 0
+  %b = shufflevector <vscale x 2 x i32> %ins, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
+  %c = inttoptr <vscale x 2 x i32> %b to <vscale x 2 x i8*>
+  %d = icmp ult <vscale x 2 x i8*> %c, zeroinitializer
+  ret <vscale x 2 x i1> %d
+}
diff --git a/llvm/test/Transforms/InstMerge/exceptions.ll b/llvm/test/Transforms/InstMerge/exceptions.ll
index 54c39960f011d..d4e2b7e3ea563 100644
--- a/llvm/test/Transforms/InstMerge/exceptions.ll
+++ b/llvm/test/Transforms/InstMerge/exceptions.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s
+; RUN: opt -basic-aa -memdep -mldst-motion -S < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<memdep>',mldst-motion \
 ; RUN:   -S < %s | FileCheck %s
 
diff --git a/llvm/test/Transforms/InstMerge/st_sink_barrier_call.ll b/llvm/test/Transforms/InstMerge/st_sink_barrier_call.ll
index cdcc34624d5c1..155a77ea69c8f 100644
--- a/llvm/test/Transforms/InstMerge/st_sink_barrier_call.ll
+++ b/llvm/test/Transforms/InstMerge/st_sink_barrier_call.ll
@@ -1,6 +1,6 @@
 ; Test to make sure that a function call that needs to be a barrier to sinking stores is indeed a barrier.
 ; Stores sunks into the footer.
-; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s
+; RUN: opt -basic-aa -memdep -mldst-motion -S < %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
 %struct.node = type { i32, %struct.node*, %struct.node*, %struct.node*, i32, i32, i32, i32 }
diff --git a/llvm/test/Transforms/InstMerge/st_sink_no_barrier_call.ll b/llvm/test/Transforms/InstMerge/st_sink_no_barrier_call.ll
index c2da0f3d0ecf7..d625c2410481f 100644
--- a/llvm/test/Transforms/InstMerge/st_sink_no_barrier_call.ll
+++ b/llvm/test/Transforms/InstMerge/st_sink_no_barrier_call.ll
@@ -1,6 +1,6 @@
 ; Test to make sure that stores in a diamond get merged with a non barrier function call after the store instruction
 ; Stores sunks into the footer.
-; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s
+; RUN: opt -basic-aa -memdep -mldst-motion -S < %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
 %struct.node = type { i32, %struct.node*, %struct.node*, %struct.node*, i32, i32, i32, i32 }
diff --git a/llvm/test/Transforms/InstMerge/st_sink_no_barrier_load.ll b/llvm/test/Transforms/InstMerge/st_sink_no_barrier_load.ll
index b7236e4c6a06b..889942f86a31c 100644
--- a/llvm/test/Transforms/InstMerge/st_sink_no_barrier_load.ll
+++ b/llvm/test/Transforms/InstMerge/st_sink_no_barrier_load.ll
@@ -1,6 +1,6 @@
 ; Test to make sure that stores in a diamond get merged with a non barrier load after the store instruction
 ; Stores sunks into the footer.
-; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s
+; RUN: opt -basic-aa -memdep -mldst-motion -S < %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
 %struct.node = type { i32, %struct.node*, %struct.node*, %struct.node*, i32, i32, i32, i32 }
diff --git a/llvm/test/Transforms/InstMerge/st_sink_no_barrier_store.ll b/llvm/test/Transforms/InstMerge/st_sink_no_barrier_store.ll
index e13f28aa5e17a..f10a64ca4a40f 100644
--- a/llvm/test/Transforms/InstMerge/st_sink_no_barrier_store.ll
+++ b/llvm/test/Transforms/InstMerge/st_sink_no_barrier_store.ll
@@ -1,6 +1,6 @@
 ; Test to make sure that stores in a diamond get merged with a non barrier store after the store instruction to be sunk
 ; Stores sunks into the footer.
-; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s
+; RUN: opt -basic-aa -memdep -mldst-motion -S < %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
 %struct.node = type { i32, %struct.node*, %struct.node*, %struct.node*, i32, i32, i32, i32 }
diff --git a/llvm/test/Transforms/InstMerge/st_sink_split_bb.ll b/llvm/test/Transforms/InstMerge/st_sink_split_bb.ll
index ee4d5647ddc3e..dfa22f6e03d74 100644
--- a/llvm/test/Transforms/InstMerge/st_sink_split_bb.ll
+++ b/llvm/test/Transforms/InstMerge/st_sink_split_bb.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test to make sure that a new block is inserted if we
 ; have more than 2 predecessors for the block we're going to sink to.
-; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s --check-prefix=CHECK-NO
+; RUN: opt -basic-aa -memdep -mldst-motion -S < %s | FileCheck %s --check-prefix=CHECK-NO
 ; RUN: opt -debug-pass-manager -aa-pipeline=basic-aa -passes='require<memdep>,mldst-motion' -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO,CHECK-INV-NO
 ; RUN: opt -debug-pass-manager -aa-pipeline=basic-aa -passes='require<memdep>,mldst-motion<split-footer-bb>' -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-YES,CHECK-INV-YES
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
diff --git a/llvm/test/Transforms/InstMerge/st_sink_two_stores.ll b/llvm/test/Transforms/InstMerge/st_sink_two_stores.ll
index 5b5582f438c85..82a12e3497f05 100644
--- a/llvm/test/Transforms/InstMerge/st_sink_two_stores.ll
+++ b/llvm/test/Transforms/InstMerge/st_sink_two_stores.ll
@@ -1,6 +1,6 @@
 ; Test to make sure that stores in a diamond get merged
 ; Stores sunks into the footer.
-; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s
+; RUN: opt -basic-aa -memdep -mldst-motion -S < %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
 %struct.node = type { i32, %struct.node*, %struct.node*, %struct.node*, i32, i32, i32, i32 }
diff --git a/llvm/test/Transforms/InstMerge/st_sink_with_barrier.ll b/llvm/test/Transforms/InstMerge/st_sink_with_barrier.ll
index a05ae88e38473..03276c708682c 100644
--- a/llvm/test/Transforms/InstMerge/st_sink_with_barrier.ll
+++ b/llvm/test/Transforms/InstMerge/st_sink_with_barrier.ll
@@ -1,5 +1,5 @@
 ; Test to make sure that load from the same address as a store and appears after the store prevents the store from being sunk
-; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s
+; RUN: opt -basic-aa -memdep -mldst-motion -S < %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
 %struct.node = type { i32, %struct.node*, %struct.node*, %struct.node*, i32, i32, i32, i32 }
diff --git a/llvm/test/Transforms/InstSimplify/assume_icmp.ll b/llvm/test/Transforms/InstSimplify/assume_icmp.ll
new file mode 100644
index 0000000000000..0a4e8e89d194a
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/assume_icmp.ll
@@ -0,0 +1,121 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+declare void @llvm.assume(i1)
+declare void @use(i1)
+
+define void @basic_ugt(i32 %x, i32 %y) {
+; CHECK-LABEL: @basic_ugt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    ret void
+;
+  %cmp1 = icmp ugt i32 %x, %y
+  call void @llvm.assume(i1 %cmp1)
+
+  %cmp2 = icmp ugt i32 %x, %y
+  call void @use(i1 %cmp2)
+  %cmp3 = icmp uge i32 %x, %y
+  call void @use(i1 %cmp3)
+  %cmp4 = icmp ult i32 %x, %y
+  call void @use(i1 %cmp4)
+  %cmp5 = icmp ule i32 %x, %y
+  call void @use(i1 %cmp5)
+
+  %cmp6 = icmp ugt i32 %y, %x
+  call void @use(i1 %cmp6)
+  %cmp7 = icmp uge i32 %y, %x
+  call void @use(i1 %cmp7)
+  %cmp8 = icmp ult i32 %y, %x
+  call void @use(i1 %cmp8)
+  %cmp9 = icmp ule i32 %y, %x
+  call void @use(i1 %cmp9)
+
+  ret void
+}
+
+define void @basic_uge(i32 %x, i32 %y) {
+; CHECK-LABEL: @basic_uge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[X]], [[Y]]
+; CHECK-NEXT:    call void @use(i1 [[CMP2]])
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp ule i32 [[X]], [[Y]]
+; CHECK-NEXT:    call void @use(i1 [[CMP5]])
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    [[CMP7:%.*]] = icmp uge i32 [[Y]], [[X]]
+; CHECK-NEXT:    call void @use(i1 [[CMP7]])
+; CHECK-NEXT:    [[CMP8:%.*]] = icmp ult i32 [[Y]], [[X]]
+; CHECK-NEXT:    call void @use(i1 [[CMP8]])
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    ret void
+;
+  %cmp1 = icmp uge i32 %x, %y
+  call void @llvm.assume(i1 %cmp1)
+
+  %cmp2 = icmp ugt i32 %x, %y
+  call void @use(i1 %cmp2)
+  %cmp3 = icmp uge i32 %x, %y
+  call void @use(i1 %cmp3)
+  %cmp4 = icmp ult i32 %x, %y
+  call void @use(i1 %cmp4)
+  %cmp5 = icmp ule i32 %x, %y
+  call void @use(i1 %cmp5)
+
+  %cmp6 = icmp ugt i32 %y, %x
+  call void @use(i1 %cmp6)
+  %cmp7 = icmp uge i32 %y, %x
+  call void @use(i1 %cmp7)
+  %cmp8 = icmp ult i32 %y, %x
+  call void @use(i1 %cmp8)
+  %cmp9 = icmp ule i32 %y, %x
+  call void @use(i1 %cmp9)
+
+  ret void
+}
+
+; This does not simplify in InstSimplify, because AssumptionCache tracker
+; does not track values through "and". The "and" assume will be broken
+; down into two separate assume calls by InstCombine.
+define void @and(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @and(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[Z:%.*]], [[Y]]
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[AND]])
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ugt i32 [[X]], [[Y]]
+; CHECK-NEXT:    call void @use(i1 [[CMP3]])
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp uge i32 [[X]], [[Y]]
+; CHECK-NEXT:    call void @use(i1 [[CMP4]])
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp ugt i32 [[Z]], [[Y]]
+; CHECK-NEXT:    call void @use(i1 [[CMP5]])
+; CHECK-NEXT:    [[CMP6:%.*]] = icmp uge i32 [[Z]], [[Y]]
+; CHECK-NEXT:    call void @use(i1 [[CMP6]])
+; CHECK-NEXT:    ret void
+;
+  %cmp1 = icmp ugt i32 %x, %y
+  %cmp2 = icmp ugt i32 %z, %y
+  %and = and i1 %cmp1, %cmp2
+  call void @llvm.assume(i1 %and)
+
+  %cmp3 = icmp ugt i32 %x, %y
+  call void @use(i1 %cmp3)
+  %cmp4 = icmp uge i32 %x, %y
+  call void @use(i1 %cmp4)
+  %cmp5 = icmp ugt i32 %z, %y
+  call void @use(i1 %cmp5)
+  %cmp6 = icmp uge i32 %z, %y
+  call void @use(i1 %cmp6)
+
+  ret void
+}
diff --git a/llvm/test/Transforms/InstSimplify/cmp_ext.ll b/llvm/test/Transforms/InstSimplify/cmp_ext.ll
new file mode 100644
index 0000000000000..63d3652fefe56
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/cmp_ext.ll
@@ -0,0 +1,232 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i1 @zext_uge_sext(i32 %x) {
+; CHECK-LABEL: @zext_uge_sext(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i64 [[ZEXT]], [[SEXT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %x to i64
+  %cmp = icmp uge i64 %zext, %sext
+  ret i1 %cmp
+}
+
+define i1 @zext_ugt_sext(i32 %x) {
+; CHECK-LABEL: @zext_ugt_sext(
+; CHECK-NEXT:    ret i1 false
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %x to i64
+  %cmp = icmp ugt i64 %zext, %sext
+  ret i1 %cmp
+}
+
+define i1 @zext_ult_sext(i32 %x) {
+; CHECK-LABEL: @zext_ult_sext(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[ZEXT]], [[SEXT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %x to i64
+  %cmp = icmp ult i64 %zext, %sext
+  ret i1 %cmp
+}
+
+define i1 @zext_ule_sext(i32 %x) {
+; CHECK-LABEL: @zext_ule_sext(
+; CHECK-NEXT:    ret i1 true
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %x to i64
+  %cmp = icmp ule i64 %zext, %sext
+  ret i1 %cmp
+}
+
+define i1 @zext_sge_sext(i32 %x) {
+; CHECK-LABEL: @zext_sge_sext(
+; CHECK-NEXT:    ret i1 true
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %x to i64
+  %cmp = icmp sge i64 %zext, %sext
+  ret i1 %cmp
+}
+
+define i1 @zext_sgt_sext(i32 %x) {
+; CHECK-LABEL: @zext_sgt_sext(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[ZEXT]], [[SEXT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %x to i64
+  %cmp = icmp sgt i64 %zext, %sext
+  ret i1 %cmp
+}
+
+define i1 @zext_slt_sext(i32 %x) {
+; CHECK-LABEL: @zext_slt_sext(
+; CHECK-NEXT:    ret i1 false
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %x to i64
+  %cmp = icmp slt i64 %zext, %sext
+  ret i1 %cmp
+}
+
+define i1 @zext_sle_sext(i32 %x) {
+; CHECK-LABEL: @zext_sle_sext(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[ZEXT]], [[SEXT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %x to i64
+  %cmp = icmp sle i64 %zext, %sext
+  ret i1 %cmp
+}
+
+define i1 @sext_uge_zext(i32 %x) {
+; CHECK-LABEL: @sext_uge_zext(
+; CHECK-NEXT:    ret i1 true
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %x to i64
+  %cmp = icmp uge i64 %sext, %zext
+  ret i1 %cmp
+}
+
+define i1 @sext_ugt_zext(i32 %x) {
+; CHECK-LABEL: @sext_ugt_zext(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[SEXT]], [[ZEXT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %x to i64
+  %cmp = icmp ugt i64 %sext, %zext
+  ret i1 %cmp
+}
+
+define i1 @sext_ult_zext(i32 %x) {
+; CHECK-LABEL: @sext_ult_zext(
+; CHECK-NEXT:    ret i1 false
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %x to i64
+  %cmp = icmp ult i64 %sext, %zext
+  ret i1 %cmp
+}
+
+define i1 @sext_ule_zext(i32 %x) {
+; CHECK-LABEL: @sext_ule_zext(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[SEXT]], [[ZEXT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %x to i64
+  %cmp = icmp ule i64 %sext, %zext
+  ret i1 %cmp
+}
+
+define i1 @sext_sge_zext(i32 %x) {
+; CHECK-LABEL: @sext_sge_zext(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i64 [[SEXT]], [[ZEXT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %x to i64
+  %cmp = icmp sge i64 %sext, %zext
+  ret i1 %cmp
+}
+
+define i1 @sext_sgt_zext(i32 %x) {
+; CHECK-LABEL: @sext_sgt_zext(
+; CHECK-NEXT:    ret i1 false
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %x to i64
+  %cmp = icmp sgt i64 %sext, %zext
+  ret i1 %cmp
+}
+
+define i1 @sext_slt_zext(i32 %x) {
+; CHECK-LABEL: @sext_slt_zext(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[SEXT]], [[ZEXT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %x to i64
+  %cmp = icmp slt i64 %sext, %zext
+  ret i1 %cmp
+}
+
+define i1 @sext_sle_zext(i32 %x) {
+; CHECK-LABEL: @sext_sle_zext(
+; CHECK-NEXT:    ret i1 true
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %x to i64
+  %cmp = icmp sle i64 %sext, %zext
+  ret i1 %cmp
+}
+
+define <4 x i1> @zext_ugt_sext_vec(<4 x i32> %x) {
+; CHECK-LABEL: @zext_ugt_sext_vec(
+; CHECK-NEXT:    ret <4 x i1> zeroinitializer
+;
+  %sext = sext <4 x i32> %x to <4 x i64>
+  %zext = zext <4 x i32> %x to <4 x i64>
+  %cmp = icmp ugt <4 x i64> %zext, %sext
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @sext_ult_zext_vec(<4 x i32> %x) {
+; CHECK-LABEL: @sext_ult_zext_vec(
+; CHECK-NEXT:    ret <4 x i1> zeroinitializer
+;
+  %sext = sext <4 x i32> %x to <4 x i64>
+  %zext = zext <4 x i32> %x to <4 x i64>
+  %cmp = icmp ult <4 x i64> %sext, %zext
+  ret <4 x i1> %cmp
+}
+
+define i1 @zext_ugt_sext_different_operand(i32 %x, i32 %y) {
+; CHECK-LABEL: @zext_ugt_sext_different_operand(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[ZEXT]], [[SEXT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %y to i64
+  %cmp = icmp ugt i64 %zext, %sext
+  ret i1 %cmp
+}
+
+define i1 @sext_ult_zext_different_operand(i32 %x, i32 %y) {
+; CHECK-LABEL: @sext_ult_zext_different_operand(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[SEXT]], [[ZEXT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sext = sext i32 %x to i64
+  %zext = zext i32 %y to i64
+  %cmp = icmp ult i64 %sext, %zext
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstSimplify/floating-point-arithmetic.ll b/llvm/test/Transforms/InstSimplify/floating-point-arithmetic.ll
index ee097ffab0945..653b730fac36b 100644
--- a/llvm/test/Transforms/InstSimplify/floating-point-arithmetic.ll
+++ b/llvm/test/Transforms/InstSimplify/floating-point-arithmetic.ll
@@ -1333,3 +1333,53 @@ define float @fsub_fadd_common_op_wrong_commute_commute(float %x, float %y) {
   %r = fadd reassoc nsz float %s, %y
   ret float %r
 }
+
+define float @maxnum_with_poszero_op(float %a) {
+; CHECK-LABEL: @maxnum_with_poszero_op(
+; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.maxnum.f32(float [[A:%.*]], float 0.000000e+00)
+; CHECK-NEXT:    ret float [[MAX]]
+;
+  %max = call float @llvm.maxnum.f32(float %a, float 0.0)
+  %fabs = call float @llvm.fabs.f32(float %max)
+  ret float %fabs
+}
+
+define float @maxnum_with_poszero_op_commute(float %a) {
+; CHECK-LABEL: @maxnum_with_poszero_op_commute(
+; CHECK-NEXT:    [[SQRT:%.*]] = call float @llvm.sqrt.f32(float [[A:%.*]])
+; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.maxnum.f32(float 0.000000e+00, float [[SQRT]])
+; CHECK-NEXT:    ret float [[MAX]]
+;
+  %sqrt = call float @llvm.sqrt.f32(float %a)
+  %max = call float @llvm.maxnum.f32(float 0.0, float %sqrt)
+  %fabs = call float @llvm.fabs.f32(float %max)
+  ret float %fabs
+}
+
+define float @maxnum_with_negzero_op(float %a) {
+; CHECK-LABEL: @maxnum_with_negzero_op(
+; CHECK-NEXT:    [[NNAN:%.*]] = call nnan float @llvm.sqrt.f32(float [[A:%.*]])
+; CHECK-NEXT:    [[FABSA:%.*]] = call float @llvm.fabs.f32(float [[NNAN]])
+; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.maxnum.f32(float -0.000000e+00, float [[FABSA]])
+; CHECK-NEXT:    ret float [[MAX]]
+;
+  %nnan = call nnan float @llvm.sqrt.f32(float %a)
+  %fabsa = call float @llvm.fabs.f32(float %nnan)
+  %max = call float @llvm.maxnum.f32(float -0.0, float %fabsa)
+  %fabs = call float @llvm.fabs.f32(float %max)
+  ret float %fabs
+}
+
+define float @maxnum_with_negzero_op_commute(float %a) {
+; CHECK-LABEL: @maxnum_with_negzero_op_commute(
+; CHECK-NEXT:    [[NNAN:%.*]] = call nnan float @llvm.sqrt.f32(float [[A:%.*]])
+; CHECK-NEXT:    [[FABSA:%.*]] = call float @llvm.fabs.f32(float [[NNAN]])
+; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.maxnum.f32(float [[FABSA]], float -0.000000e+00)
+; CHECK-NEXT:    ret float [[MAX]]
+;
+  %nnan = call nnan float @llvm.sqrt.f32(float %a)
+  %fabsa = call float @llvm.fabs.f32(float %nnan)
+  %max = call float @llvm.maxnum.f32(float %fabsa, float -0.0)
+  %fabs = call float @llvm.fabs.f32(float %max)
+  ret float %fabs
+}
diff --git a/llvm/test/Transforms/InstSimplify/gep.ll b/llvm/test/Transforms/InstSimplify/gep.ll
index c4fa8df717cc2..5044b50f7c9d3 100644
--- a/llvm/test/Transforms/InstSimplify/gep.ll
+++ b/llvm/test/Transforms/InstSimplify/gep.ll
@@ -152,7 +152,7 @@ define <4 x i32*> @vector_idx_vector() {
 %struct = type { double, float }
 define <4 x float*> @vector_idx_mix_scalar_vector() {
 ; CHECK-LABEL: @vector_idx_mix_scalar_vector(
-; CHECK-NEXT:    ret <4 x float*> getelementptr (%struct, <4 x %struct*> zeroinitializer, <4 x i64> zeroinitializer, <4 x i32> <i32 1, i32 1, i32 1, i32 1>)
+; CHECK-NEXT:    ret <4 x float*> getelementptr (%struct, <4 x %struct*> zeroinitializer, <4 x i64> zeroinitializer, i32 1)
 ;
   %gep = getelementptr %struct, <4 x %struct*> zeroinitializer, i32 0, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
   ret <4 x float*> %gep
@@ -168,4 +168,20 @@ define <vscale x 4 x i32*> @scalable_idx_scalar() {
   ret <vscale x 4 x i32*> %gep
 }
 
+define <vscale x 4 x float*> @scalable_vector_idx_mix_scalar_vector() {
+; CHECK-LABEL: @scalable_vector_idx_mix_scalar_vector(
+; CHECK-NEXT:    ret <vscale x 4 x float*> getelementptr (%struct, <vscale x 4 x %struct*> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 1)
+;
+  %gep = getelementptr %struct, <vscale x 4 x %struct*> zeroinitializer, i32 0, i32 1
+  ret <vscale x 4 x float*> %gep
+}
+
+define <vscale x 2 x i64*> @ptr_idx_mix_scalar_scalable_vector() {
+; CHECK-LABEL: @ptr_idx_mix_scalar_scalable_vector(
+; CHECK-NEXT:    ret <vscale x 2 x i64*> zeroinitializer
+;
+  %v = getelementptr [2 x i64], [2 x i64]* null, i64 0, <vscale x 2 x i64> zeroinitializer
+  ret <vscale x 2 x i64*> %v
+}
+
 ; Check ConstantExpr::getGetElementPtr() using ElementCount for size queries - end.
diff --git a/llvm/test/Transforms/InstSimplify/icmp.ll b/llvm/test/Transforms/InstSimplify/icmp.ll
new file mode 100644
index 0000000000000..c326994178814
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/icmp.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-p1:16:16:16-p2:32:32:32-p3:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+declare void @usei8ptr(i8* %ptr)
+
+; Ensure that we do not crash when looking at such a weird bitcast.
+define i1 @bitcast_from_single_element_pointer_vector_to_pointer(<1 x i8*> %ptr1vec, i8* %ptr2) {
+; CHECK-LABEL: @bitcast_from_single_element_pointer_vector_to_pointer(
+  %ptr1 = bitcast <1 x i8*> %ptr1vec to i8*
+  call void @usei8ptr(i8* %ptr1)
+  %cmp = icmp eq i8* %ptr1, %ptr2
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstSimplify/select.ll b/llvm/test/Transforms/InstSimplify/select.ll
index 139f5e3c3c232..353f2e6a6753d 100644
--- a/llvm/test/Transforms/InstSimplify/select.ll
+++ b/llvm/test/Transforms/InstSimplify/select.ll
@@ -750,3 +750,181 @@ define i1 @y_might_be_poison(float %x, float %y) {
   %c3 = select i1 %c1, i1 %c2, i1 false
   ret i1 %c3
 }
+
+; Negative tests to ensure we don't remove selects with undef true/false values.
+; See https://bugs.llvm.org/show_bug.cgi?id=31633
+; https://lists.llvm.org/pipermail/llvm-dev/2016-October/106182.html
+; https://reviews.llvm.org/D83360
+define i32 @false_undef(i1 %cond, i32 %x) {
+; CHECK-LABEL: @false_undef(
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND:%.*]], i32 [[X:%.*]], i32 undef
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %s = select i1 %cond, i32 %x, i32 undef
+  ret i32 %s
+}
+
+define i32 @true_undef(i1 %cond, i32 %x) {
+; CHECK-LABEL: @true_undef(
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND:%.*]], i32 undef, i32 [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %s = select i1 %cond, i32 undef, i32 %x
+  ret i32 %s
+}
+
+define <2 x i32> @false_undef_vec(i1 %cond, <2 x i32> %x) {
+; CHECK-LABEL: @false_undef_vec(
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND:%.*]], <2 x i32> [[X:%.*]], <2 x i32> undef
+; CHECK-NEXT:    ret <2 x i32> [[S]]
+;
+  %s = select i1 %cond, <2 x i32> %x, <2 x i32> undef
+  ret <2 x i32> %s
+}
+
+define <2 x i32> @true_undef_vec(i1 %cond, <2 x i32> %x) {
+; CHECK-LABEL: @true_undef_vec(
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND:%.*]], <2 x i32> undef, <2 x i32> [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[S]]
+;
+  %s = select i1 %cond, <2 x i32> undef, <2 x i32> %x
+  ret <2 x i32> %s
+}
+
+; These can be folded because the other value is guaranteed not to be poison.
+define i32 @false_undef_true_constant(i1 %cond) {
+; CHECK-LABEL: @false_undef_true_constant(
+; CHECK-NEXT:    ret i32 10
+;
+  %s = select i1 %cond, i32 10, i32 undef
+  ret i32 %s
+}
+
+define i32 @true_undef_false_constant(i1 %cond) {
+; CHECK-LABEL: @true_undef_false_constant(
+; CHECK-NEXT:    ret i32 20
+;
+  %s = select i1 %cond, i32 undef, i32 20
+  ret i32 %s
+}
+
+define <2 x i32> @false_undef_true_constant_vec(i1 %cond) {
+; CHECK-LABEL: @false_undef_true_constant_vec(
+; CHECK-NEXT:    ret <2 x i32> <i32 42, i32 -42>
+;
+  %s = select i1 %cond, <2 x i32> <i32 42, i32 -42>, <2 x i32> undef
+  ret <2 x i32> %s
+}
+
+define <2 x i32> @true_undef_false_constant_vec(i1 %cond) {
+; CHECK-LABEL: @true_undef_false_constant_vec(
+; CHECK-NEXT:    ret <2 x i32> <i32 -42, i32 42>
+;
+  %s = select i1 %cond, <2 x i32> undef, <2 x i32> <i32 -42, i32 42>
+  ret <2 x i32> %s
+}
+
+; If one input is undef and the other is freeze, we can fold it to the freeze.
+define i32 @false_undef_true_freeze(i1 %cond, i32 %x) {
+; CHECK-LABEL: @false_undef_true_freeze(
+; CHECK-NEXT:    [[XF:%.*]] = freeze i32 [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[XF]]
+;
+  %xf = freeze i32 %x
+  %s = select i1 %cond, i32 %xf, i32 undef
+  ret i32 %s
+}
+
+define i32 @false_undef_false_freeze(i1 %cond, i32 %x) {
+; CHECK-LABEL: @false_undef_false_freeze(
+; CHECK-NEXT:    [[XF:%.*]] = freeze i32 [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[XF]]
+;
+  %xf = freeze i32 %x
+  %s = select i1 %cond, i32 undef, i32 %xf
+  ret i32 %s
+}
+
+@g = external global i32, align 1
+
+; Make sure we don't fold partial undef vectors when constexprs are involved.
+; We would need to prove the constexpr doesn't result in poison which we aren't
+; equiped to do yet.
+define <2 x i32> @false_undef_true_constextpr_vec(i1 %cond) {
+; CHECK-LABEL: @false_undef_true_constextpr_vec(
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND:%.*]], <2 x i32> <i32 undef, i32 ptrtoint (i32* @g to i32)>, <2 x i32> <i32 ptrtoint (i32* @g to i32), i32 undef>
+; CHECK-NEXT:    ret <2 x i32> [[S]]
+;
+  %s = select i1 %cond, <2 x i32> <i32 undef, i32 ptrtoint (i32* @g to i32)>, <2 x i32> <i32 ptrtoint (i32* @g to i32), i32 undef>
+  ret <2 x i32> %s
+}
+
+define i32 @all_constant_true_undef() {
+; CHECK-LABEL: @all_constant_true_undef(
+; CHECK-NEXT:    ret i32 1
+;
+  %s = select i1 ptrtoint (i32 ()* @all_constant_true_undef to i1), i32 undef, i32 1
+  ret i32 %s
+}
+
+define float @all_constant_false_undef() {
+; CHECK-LABEL: @all_constant_false_undef(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %s = select i1 ptrtoint (float ()* @all_constant_false_undef to i1), float undef, float 1.0
+  ret float %s
+}
+
+define <2 x i32> @all_constant_true_undef_vec() {
+; CHECK-LABEL: @all_constant_true_undef_vec(
+; CHECK-NEXT:    ret <2 x i32> <i32 1, i32 -1>
+;
+  %s = select i1 ptrtoint (<2 x i32> ()* @all_constant_true_undef_vec to i1), <2 x i32> undef, <2 x i32> <i32 1, i32 -1>
+  ret <2 x i32> %s
+}
+
+define <2 x float> @all_constant_false_undef_vec() {
+; CHECK-LABEL: @all_constant_false_undef_vec(
+; CHECK-NEXT:    ret <2 x float> <float 1.000000e+00, float -1.000000e+00>
+;
+  %s = select i1 ptrtoint (<2 x float> ()* @all_constant_false_undef_vec to i1), <2 x float> undef, <2 x float> <float 1.0, float -1.0>
+  ret <2 x float> %s
+}
+
+; Negative tests. Don't fold if the non-undef operand is a constexpr.
+define i32 @all_constant_false_undef_true_constexpr() {
+; CHECK-LABEL: @all_constant_false_undef_true_constexpr(
+; CHECK-NEXT:    [[S:%.*]] = select i1 ptrtoint (i32 ()* @all_constant_false_undef_true_constexpr to i1), i32 ptrtoint (i32 ()* @all_constant_false_undef_true_constexpr to i32), i32 undef
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %s = select i1 ptrtoint (i32 ()* @all_constant_false_undef_true_constexpr to i1), i32 ptrtoint (i32 ()* @all_constant_false_undef_true_constexpr to i32), i32 undef
+  ret i32 %s
+}
+
+define i32 @all_constant_true_undef_false_constexpr() {
+; CHECK-LABEL: @all_constant_true_undef_false_constexpr(
+; CHECK-NEXT:    [[S:%.*]] = select i1 ptrtoint (i32 ()* @all_constant_true_undef_false_constexpr to i1), i32 undef, i32 ptrtoint (i32 ()* @all_constant_true_undef_false_constexpr to i32)
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %s = select i1 ptrtoint (i32 ()* @all_constant_true_undef_false_constexpr to i1), i32 undef, i32 ptrtoint (i32 ()* @all_constant_true_undef_false_constexpr to i32)
+  ret i32 %s
+}
+
+; Negative tests. Don't fold if the non-undef operand is a vector containing a constexpr.
+define <2 x i32> @all_constant_false_undef_true_constexpr_vec() {
+; CHECK-LABEL: @all_constant_false_undef_true_constexpr_vec(
+; CHECK-NEXT:    [[S:%.*]] = select i1 ptrtoint (<2 x i32> ()* @all_constant_false_undef_true_constexpr_vec to i1), <2 x i32> <i32 ptrtoint (<2 x i32> ()* @all_constant_false_undef_true_constexpr_vec to i32), i32 -1>, <2 x i32> undef
+; CHECK-NEXT:    ret <2 x i32> [[S]]
+;
+  %s = select i1 ptrtoint (<2 x i32> ()* @all_constant_false_undef_true_constexpr_vec to i1), <2 x i32> <i32 ptrtoint (<2 x i32> ()* @all_constant_false_undef_true_constexpr_vec to i32), i32 -1>, <2 x i32> undef
+  ret <2 x i32> %s
+}
+
+define <2 x i32> @all_constant_true_undef_false_constexpr_vec() {
+; CHECK-LABEL: @all_constant_true_undef_false_constexpr_vec(
+; CHECK-NEXT:    [[S:%.*]] = select i1 ptrtoint (<2 x i32> ()* @all_constant_true_undef_false_constexpr_vec to i1), <2 x i32> undef, <2 x i32> <i32 -1, i32 ptrtoint (<2 x i32> ()* @all_constant_true_undef_false_constexpr_vec to i32)>
+; CHECK-NEXT:    ret <2 x i32> [[S]]
+;
+  %s = select i1 ptrtoint (<2 x i32> ()* @all_constant_true_undef_false_constexpr_vec to i1), <2 x i32> undef, <2 x i32><i32 -1, i32 ptrtoint (<2 x i32> ()* @all_constant_true_undef_false_constexpr_vec to i32)>
+  ret <2 x i32> %s
+}
diff --git a/llvm/test/Transforms/InstSimplify/vscale.ll b/llvm/test/Transforms/InstSimplify/vscale.ll
index 669c824685e8e..d396f02891969 100644
--- a/llvm/test/Transforms/InstSimplify/vscale.ll
+++ b/llvm/test/Transforms/InstSimplify/vscale.ll
@@ -95,6 +95,15 @@ define i32 @insert_extract_element_same_vec_idx_2(<vscale x 4 x i32> %a) {
   ret i32 %r
 }
 
+; more complicated expressions
+
+define <vscale x 2 x i1> @cmp_le_smax_always_true(<vscale x 2 x i64> %x) {
+; CHECK-LABEL: @cmp_le_smax_always_true(
+; CHECK-NEXT:    ret <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> undef, i1 true, i32 0), <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer)
+   %cmp = icmp sle <vscale x 2 x i64> %x, shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> undef, i64 9223372036854775807, i32 0), <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer)
+   ret <vscale x 2 x i1> %cmp
+}
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Memory Access and Addressing Operations
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
diff --git a/llvm/test/Transforms/LICM/2003-02-26-LoopExitNotDominated.ll b/llvm/test/Transforms/LICM/2003-02-26-LoopExitNotDominated.ll
index 1b50a6c87af77..ffb738cb9def5 100644
--- a/llvm/test/Transforms/LICM/2003-02-26-LoopExitNotDominated.ll
+++ b/llvm/test/Transforms/LICM/2003-02-26-LoopExitNotDominated.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -licm -disable-output
+; RUN: opt < %s -basic-aa -licm -disable-output
 
 ;%MoveArray = external global [64 x ulong]
 
diff --git a/llvm/test/Transforms/LICM/2008-07-22-LoadGlobalConstant.ll b/llvm/test/Transforms/LICM/2008-07-22-LoadGlobalConstant.ll
index a715af032a965..c7b5f65dde8a2 100644
--- a/llvm/test/Transforms/LICM/2008-07-22-LoadGlobalConstant.ll
+++ b/llvm/test/Transforms/LICM/2008-07-22-LoadGlobalConstant.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -licm -S | FileCheck %s
+; RUN: opt < %s -basic-aa -licm -S | FileCheck %s
 
 @a = external constant float*
 
diff --git a/llvm/test/Transforms/LICM/2011-04-06-HoistMissedASTUpdate.ll b/llvm/test/Transforms/LICM/2011-04-06-HoistMissedASTUpdate.ll
index b462885e5fbc0..fb0fbbdc69462 100644
--- a/llvm/test/Transforms/LICM/2011-04-06-HoistMissedASTUpdate.ll
+++ b/llvm/test/Transforms/LICM/2011-04-06-HoistMissedASTUpdate.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -licm -S | FileCheck %s
+; RUN: opt < %s -basic-aa -licm -S | FileCheck %s
 ; PR9630
 
 @g_39 = external global i16, align 2
diff --git a/llvm/test/Transforms/LICM/PR21582.ll b/llvm/test/Transforms/LICM/PR21582.ll
index 5664f2e2b6e04..2fbbc66e57414 100644
--- a/llvm/test/Transforms/LICM/PR21582.ll
+++ b/llvm/test/Transforms/LICM/PR21582.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -licm -S | FileCheck %s
+; RUN: opt < %s -basic-aa -licm -S | FileCheck %s
 @b = external global i32, align 4
 @fn3.i = external global i32, align 4
 
diff --git a/llvm/test/Transforms/LICM/argmemonly-call.ll b/llvm/test/Transforms/LICM/argmemonly-call.ll
index 5050253859b5c..b726dc1c7610d 100644
--- a/llvm/test/Transforms/LICM/argmemonly-call.ll
+++ b/llvm/test/Transforms/LICM/argmemonly-call.ll
@@ -1,6 +1,6 @@
-; RUN: opt -S -basicaa -licm -licm-n2-threshold=0 -enable-mssa-loop-dependency=false %s | FileCheck %s
-; RUN: opt -S -basicaa -licm -licm-n2-threshold=0 -enable-mssa-loop-dependency=true -verify-memoryssa %s | FileCheck %s --check-prefix=ALIAS-N2
-; RUN: opt -licm -basicaa -licm-n2-threshold=200 < %s -S | FileCheck %s --check-prefix=ALIAS-N2
+; RUN: opt -S -basic-aa -licm -licm-n2-threshold=0 -enable-mssa-loop-dependency=false %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm -licm-n2-threshold=0 -enable-mssa-loop-dependency=true -verify-memoryssa %s | FileCheck %s --check-prefix=ALIAS-N2
+; RUN: opt -licm -basic-aa -licm-n2-threshold=200 < %s -S | FileCheck %s --check-prefix=ALIAS-N2
 
 ; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=0 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=0 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop-mssa(licm)' < %s -S | FileCheck %s --check-prefix=ALIAS-N2
diff --git a/llvm/test/Transforms/LICM/assume.ll b/llvm/test/Transforms/LICM/assume.ll
index 30faf7dee9d3c..8c3abb5f8b2f0 100644
--- a/llvm/test/Transforms/LICM/assume.ll
+++ b/llvm/test/Transforms/LICM/assume.ll
@@ -1,4 +1,4 @@
-; RUN: opt -licm -basicaa < %s -S | FileCheck %s
+; RUN: opt -licm -basic-aa < %s -S | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
 
 define void @f_0(i1 %p) nounwind ssp {
diff --git a/llvm/test/Transforms/LICM/atomics.ll b/llvm/test/Transforms/LICM/atomics.ll
index 214078e6fd6f0..3ef6b1194a7bb 100644
--- a/llvm/test/Transforms/LICM/atomics.ll
+++ b/llvm/test/Transforms/LICM/atomics.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -S -basicaa -licm -enable-mssa-loop-dependency=false | FileCheck -check-prefixes=CHECK,AST %s
-; RUN: opt < %s -S -basicaa -licm -enable-mssa-loop-dependency=true  | FileCheck -check-prefixes=CHECK,MSSA %s
+; RUN: opt < %s -S -basic-aa -licm -enable-mssa-loop-dependency=false | FileCheck -check-prefixes=CHECK,AST %s
+; RUN: opt < %s -S -basic-aa -licm -enable-mssa-loop-dependency=true  | FileCheck -check-prefixes=CHECK,MSSA %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck -check-prefixes=CHECK,AST %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop-mssa(licm)' < %s -S | FileCheck -check-prefixes=CHECK,MSSA %s
 
diff --git a/llvm/test/Transforms/LICM/call-hoisting.ll b/llvm/test/Transforms/LICM/call-hoisting.ll
index 9e29af3039a20..170f4ac535bec 100644
--- a/llvm/test/Transforms/LICM/call-hoisting.ll
+++ b/llvm/test/Transforms/LICM/call-hoisting.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -licm %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
 
 declare i32 @load(i32* %p) argmemonly readonly nounwind
diff --git a/llvm/test/Transforms/LICM/constexpr.ll b/llvm/test/Transforms/LICM/constexpr.ll
index 488821ac8fd40..02a37feca243c 100644
--- a/llvm/test/Transforms/LICM/constexpr.ll
+++ b/llvm/test/Transforms/LICM/constexpr.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -S -basicaa -licm | FileCheck %s
+; RUN: opt < %s -S -basic-aa -licm | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
 ; This fixes PR22460
 
diff --git a/llvm/test/Transforms/LICM/debug-value.ll b/llvm/test/Transforms/LICM/debug-value.ll
index 64bb9c65a8fb9..db566cc022e71 100644
--- a/llvm/test/Transforms/LICM/debug-value.ll
+++ b/llvm/test/Transforms/LICM/debug-value.ll
@@ -1,4 +1,4 @@
-; RUN: opt -licm -basicaa < %s -S | FileCheck %s
+; RUN: opt -licm -basic-aa < %s -S | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
 
 define void @dgefa() nounwind ssp {
diff --git a/llvm/test/Transforms/LICM/explicit_guards.ll b/llvm/test/Transforms/LICM/explicit_guards.ll
index a684ebe9d88aa..d51a1e24db400 100644
--- a/llvm/test/Transforms/LICM/explicit_guards.ll
+++ b/llvm/test/Transforms/LICM/explicit_guards.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -make-guards-explicit -basicaa -licm < %s        | FileCheck %s
+; RUN: opt -S -make-guards-explicit -basic-aa -licm < %s        | FileCheck %s
 ; RUN: opt -S -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,make-guards-explicit,loop(licm)' < %s | FileCheck %s
 
 declare void @llvm.experimental.guard(i1,...)
diff --git a/llvm/test/Transforms/LICM/fence.ll b/llvm/test/Transforms/LICM/fence.ll
index f1dfe9e3511ca..7df9e21039b87 100644
--- a/llvm/test/Transforms/LICM/fence.ll
+++ b/llvm/test/Transforms/LICM/fence.ll
@@ -1,4 +1,4 @@
-; RUN: opt -licm -basicaa < %s -S | FileCheck %s
+; RUN: opt -licm -basic-aa < %s -S | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
 
 define void @test1(i64 %n) {
diff --git a/llvm/test/Transforms/LICM/guards.ll b/llvm/test/Transforms/LICM/guards.ll
index e17e5ff859aa6..e4dad88e49982 100644
--- a/llvm/test/Transforms/LICM/guards.ll
+++ b/llvm/test/Transforms/LICM/guards.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; REQUIRES: asserts
-; RUN: opt -licm -basicaa -enable-mssa-loop-dependency=false -ipt-expensive-asserts=true < %s -S | FileCheck %s
-; RUN: opt -licm -basicaa -enable-mssa-loop-dependency=true -ipt-expensive-asserts=true < %s -S | FileCheck %s --check-prefixes=CHECK,MSSA
+; RUN: opt -licm -basic-aa -enable-mssa-loop-dependency=false -ipt-expensive-asserts=true < %s -S | FileCheck %s
+; RUN: opt -licm -basic-aa -enable-mssa-loop-dependency=true -ipt-expensive-asserts=true < %s -S | FileCheck %s --check-prefixes=CHECK,MSSA
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -ipt-expensive-asserts=true < %s -S | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop-mssa(licm)' -ipt-expensive-asserts=true < %s -S | FileCheck %s --check-prefixes=CHECK,MSSA
 
diff --git a/llvm/test/Transforms/LICM/hoist-bitcast-load.ll b/llvm/test/Transforms/LICM/hoist-bitcast-load.ll
index fcd6d08f2b47e..1406ae5cba455 100644
--- a/llvm/test/Transforms/LICM/hoist-bitcast-load.ll
+++ b/llvm/test/Transforms/LICM/hoist-bitcast-load.ll
@@ -1,6 +1,6 @@
-; RUN: opt -S -basicaa -licm < %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(simplify-cfg,licm)' -S < %s | FileCheck %s
-; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LICM/hoist-deref-load.ll b/llvm/test/Transforms/LICM/hoist-deref-load.ll
index dd4d8821955d9..2f1ae0518f3ef 100644
--- a/llvm/test/Transforms/LICM/hoist-deref-load.ll
+++ b/llvm/test/Transforms/LICM/hoist-deref-load.ll
@@ -1,6 +1,6 @@
-; RUN: opt -S -basicaa -licm < %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(simplify-cfg,licm)' -S < %s | FileCheck %s
-; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop-mssa(simplify-cfg,licm)' -verify-memoryssa -S < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LICM/hoist-invariant-load.ll b/llvm/test/Transforms/LICM/hoist-invariant-load.ll
index 909652b8f46c9..f711d8017e98a 100644
--- a/llvm/test/Transforms/LICM/hoist-invariant-load.ll
+++ b/llvm/test/Transforms/LICM/hoist-invariant-load.ll
@@ -1,6 +1,6 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -licm -disable-basicaa -stats -S 2>&1 | grep "1 licm"
-; RUN: opt < %s -licm -enable-mssa-loop-dependency=true -verify-memoryssa -disable-basicaa -stats -S 2>&1 | grep "1 licm"
+; RUN: opt < %s -licm -disable-basic-aa -stats -S 2>&1 | grep "1 licm"
+; RUN: opt < %s -licm -enable-mssa-loop-dependency=true -verify-memoryssa -disable-basic-aa -stats -S 2>&1 | grep "1 licm"
 
 @"\01L_OBJC_METH_VAR_NAME_" = internal global [4 x i8] c"foo\00", section "__TEXT,__objc_methname,cstring_literals", align 1
 @"\01L_OBJC_SELECTOR_REFERENCES_" = internal global i8* getelementptr inbounds ([4 x i8], [4 x i8]* @"\01L_OBJC_METH_VAR_NAME_", i32 0, i32 0), section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
diff --git a/llvm/test/Transforms/LICM/hoist-mustexec.ll b/llvm/test/Transforms/LICM/hoist-mustexec.ll
index 59184eb542cfe..25ccaa0736203 100644
--- a/llvm/test/Transforms/LICM/hoist-mustexec.ll
+++ b/llvm/test/Transforms/LICM/hoist-mustexec.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -S -basicaa -licm -ipt-expensive-asserts=true < %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm -ipt-expensive-asserts=true < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' -ipt-expensive-asserts=true -S %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LICM/hoist-nounwind.ll b/llvm/test/Transforms/LICM/hoist-nounwind.ll
index a582173a156a9..e74bf59bc36d1 100644
--- a/llvm/test/Transforms/LICM/hoist-nounwind.ll
+++ b/llvm/test/Transforms/LICM/hoist-nounwind.ll
@@ -1,6 +1,6 @@
-; RUN: opt -S -basicaa -licm < %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
-; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/LICM/infinite_loops.ll b/llvm/test/Transforms/LICM/infinite_loops.ll
index 6189bdb1a9744..3f27d3563c184 100644
--- a/llvm/test/Transforms/LICM/infinite_loops.ll
+++ b/llvm/test/Transforms/LICM/infinite_loops.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -basicaa -licm < %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
 
 ; Make sure we don't hoist the unsafe division to some executable block.
diff --git a/llvm/test/Transforms/LICM/invariant.start.ll b/llvm/test/Transforms/LICM/invariant.start.ll
index d5d44757c40b2..73e23f0effb3c 100644
--- a/llvm/test/Transforms/LICM/invariant.start.ll
+++ b/llvm/test/Transforms/LICM/invariant.start.ll
@@ -1,5 +1,5 @@
-; RUN: opt -licm -basicaa -licm-n2-threshold=0 < %s -S | FileCheck %s
-; RUN: opt -licm -basicaa -licm-n2-threshold=200 < %s -S | FileCheck %s --check-prefix=ALIAS-N2
+; RUN: opt -licm -basic-aa -licm-n2-threshold=0 < %s -S | FileCheck %s
+; RUN: opt -licm -basic-aa -licm-n2-threshold=200 < %s -S | FileCheck %s --check-prefix=ALIAS-N2
 ; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=0 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=200 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s --check-prefix=ALIAS-N2
 
diff --git a/llvm/test/Transforms/LICM/lcssa-ssa-promoter.ll b/llvm/test/Transforms/LICM/lcssa-ssa-promoter.ll
index 0644a627f7186..39e70748fb5d5 100644
--- a/llvm/test/Transforms/LICM/lcssa-ssa-promoter.ll
+++ b/llvm/test/Transforms/LICM/lcssa-ssa-promoter.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -licm < %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s| FileCheck %s
 ;
 ; Manually validate LCSSA form is preserved even after SSAUpdater is used to
diff --git a/llvm/test/Transforms/LICM/pr26843.ll b/llvm/test/Transforms/LICM/pr26843.ll
index e72821d7fdbe9..0301f2793e334 100644
--- a/llvm/test/Transforms/LICM/pr26843.ll
+++ b/llvm/test/Transforms/LICM/pr26843.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -licm < %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm < %s | FileCheck %s
 
 target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
 target triple = "i686-pc-windows-msvc18.0.0"
diff --git a/llvm/test/Transforms/LICM/pr27262.ll b/llvm/test/Transforms/LICM/pr27262.ll
index 0245ec2ca8bf7..9ba42d5dd7986 100644
--- a/llvm/test/Transforms/LICM/pr27262.ll
+++ b/llvm/test/Transforms/LICM/pr27262.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -licm < %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm < %s | FileCheck %s
 
 target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
 target triple = "i686-pc-windows-msvc18.0.0"
diff --git a/llvm/test/Transforms/LICM/pr38513.ll b/llvm/test/Transforms/LICM/pr38513.ll
index ebc27510602a2..dea268d88cd73 100644
--- a/llvm/test/Transforms/LICM/pr38513.ll
+++ b/llvm/test/Transforms/LICM/pr38513.ll
@@ -1,4 +1,4 @@
-; RUN: opt -enable-mssa-loop-dependency=false -disable-basicaa -alias-set-saturation-threshold=2 -licm -S < %s | FileCheck %s
+; RUN: opt -enable-mssa-loop-dependency=false -disable-basic-aa -alias-set-saturation-threshold=2 -licm -S < %s | FileCheck %s
 ; REQUIRES: asserts
 
 ; CHECK-LABEL: @f1()
diff --git a/llvm/test/Transforms/LICM/pragma-licm-disable.ll b/llvm/test/Transforms/LICM/pragma-licm-disable.ll
index cba109943e323..16c0125ebe82b 100644
--- a/llvm/test/Transforms/LICM/pragma-licm-disable.ll
+++ b/llvm/test/Transforms/LICM/pragma-licm-disable.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -S -basicaa -licm | FileCheck %s
+; RUN: opt < %s -S -basic-aa -licm | FileCheck %s
 
 ; Check that the LICM pass does not operate on a loop which has the
 ; llvm.licm.disable metadata.
diff --git a/llvm/test/Transforms/LICM/promote-order.ll b/llvm/test/Transforms/LICM/promote-order.ll
index e187404f3912f..1a46f95386a83 100644
--- a/llvm/test/Transforms/LICM/promote-order.ll
+++ b/llvm/test/Transforms/LICM/promote-order.ll
@@ -1,5 +1,5 @@
-; RUN: opt -tbaa -basicaa -licm -enable-mssa-loop-dependency=false -S < %s | FileCheck %s --check-prefixes=CHECK,AST
-; RUN: opt -tbaa -basicaa -licm -enable-mssa-loop-dependency=true -S < %s | FileCheck %s --check-prefixes=CHECK,MSSA
+; RUN: opt -tbaa -basic-aa -licm -enable-mssa-loop-dependency=false -S < %s | FileCheck %s --check-prefixes=CHECK,AST
+; RUN: opt -tbaa -basic-aa -licm -enable-mssa-loop-dependency=true -S < %s | FileCheck %s --check-prefixes=CHECK,MSSA
 ; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s  --check-prefixes=CHECK,AST
 ; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop-mssa(licm)' -S %s | FileCheck %s --check-prefixes=CHECK,MSSA
 
diff --git a/llvm/test/Transforms/LICM/promote-tls.ll b/llvm/test/Transforms/LICM/promote-tls.ll
index db4d4c3120122..ccd73733f62b4 100644
--- a/llvm/test/Transforms/LICM/promote-tls.ll
+++ b/llvm/test/Transforms/LICM/promote-tls.ll
@@ -1,4 +1,4 @@
-; RUN: opt -tbaa -basicaa -licm -S < %s | FileCheck %s
+; RUN: opt -tbaa -basic-aa -licm -S < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
 
 ; If we can prove a local is thread local, we can insert stores during
diff --git a/llvm/test/Transforms/LICM/read-only-calls.ll b/llvm/test/Transforms/LICM/read-only-calls.ll
index 19e9a118cfe16..91a26446244ca 100644
--- a/llvm/test/Transforms/LICM/read-only-calls.ll
+++ b/llvm/test/Transforms/LICM/read-only-calls.ll
@@ -1,6 +1,6 @@
-; RUN: opt -S -basicaa -licm -licm-n2-threshold=0 -enable-mssa-loop-dependency=false %s | FileCheck %s
-; RUN: opt -S -basicaa -licm -licm-n2-threshold=0 -enable-mssa-loop-dependency=true %s | FileCheck %s --check-prefix=ALIAS-N2
-; RUN: opt -licm -basicaa -licm-n2-threshold=200 < %s -S | FileCheck %s --check-prefix=ALIAS-N2
+; RUN: opt -S -basic-aa -licm -licm-n2-threshold=0 -enable-mssa-loop-dependency=false %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm -licm-n2-threshold=0 -enable-mssa-loop-dependency=true %s | FileCheck %s --check-prefix=ALIAS-N2
+; RUN: opt -licm -basic-aa -licm-n2-threshold=200 < %s -S | FileCheck %s --check-prefix=ALIAS-N2
 
 ; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=0 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=0 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop-mssa(licm)' < %s -S | FileCheck %s --check-prefix=ALIAS-N2
diff --git a/llvm/test/Transforms/LICM/scalar-promote-memmodel.ll b/llvm/test/Transforms/LICM/scalar-promote-memmodel.ll
index c09c2b361e021..92724cf97330a 100644
--- a/llvm/test/Transforms/LICM/scalar-promote-memmodel.ll
+++ b/llvm/test/Transforms/LICM/scalar-promote-memmodel.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -licm -S | FileCheck %s
+; RUN: opt < %s -basic-aa -licm -S | FileCheck %s
 ; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
 
 ; Make sure we don't hoist a conditionally-executed store out of the loop;
diff --git a/llvm/test/Transforms/LICM/scalar-promote-unwind.ll b/llvm/test/Transforms/LICM/scalar-promote-unwind.ll
index 5de6ae9c27770..6bcea9953aa3b 100644
--- a/llvm/test/Transforms/LICM/scalar-promote-unwind.ll
+++ b/llvm/test/Transforms/LICM/scalar-promote-unwind.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -licm -S | FileCheck %s
+; RUN: opt < %s -basic-aa -licm -S | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LICM/scalar-promote.ll b/llvm/test/Transforms/LICM/scalar-promote.ll
index b719c1e6cb46f..c0fc153e8722a 100644
--- a/llvm/test/Transforms/LICM/scalar-promote.ll
+++ b/llvm/test/Transforms/LICM/scalar-promote.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -tbaa -licm -S | FileCheck %s
+; RUN: opt < %s -basic-aa -tbaa -licm -S | FileCheck %s
 ; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
diff --git a/llvm/test/Transforms/LICM/sink-promote.ll b/llvm/test/Transforms/LICM/sink-promote.ll
index 45024c46da958..ac05af5a6400d 100644
--- a/llvm/test/Transforms/LICM/sink-promote.ll
+++ b/llvm/test/Transforms/LICM/sink-promote.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -licm -S | FileCheck %s
+; RUN: opt < %s -basic-aa -licm -S | FileCheck %s
 
 ; Test moved from sinking.ll, as it tests sinking of a store who alone touches
 ; a memory location in a loop.
diff --git a/llvm/test/Transforms/LICM/sinking.ll b/llvm/test/Transforms/LICM/sinking.ll
index cc30494a518e4..a3a7487a1b234 100644
--- a/llvm/test/Transforms/LICM/sinking.ll
+++ b/llvm/test/Transforms/LICM/sinking.ll
@@ -1,6 +1,6 @@
-; RUN: opt < %s -basicaa -licm -S | FileCheck %s
-; RUN: opt < %s -debugify -basicaa -licm -S | FileCheck %s -check-prefix=DEBUGIFY
-; RUN: opt < %s -basicaa -licm -S -enable-mssa-loop-dependency=true -verify-memoryssa | FileCheck %s
+; RUN: opt < %s -basic-aa -licm -S | FileCheck %s
+; RUN: opt < %s -debugify -basic-aa -licm -S | FileCheck %s -check-prefix=DEBUGIFY
+; RUN: opt < %s -basic-aa -licm -S -enable-mssa-loop-dependency=true -verify-memoryssa | FileCheck %s
 
 
 declare i32 @strlen(i8*) readonly nounwind
diff --git a/llvm/test/Transforms/LICM/store-hoisting.ll b/llvm/test/Transforms/LICM/store-hoisting.ll
index c1df6efdcb7ac..b66098aef97ec 100644
--- a/llvm/test/Transforms/LICM/store-hoisting.ll
+++ b/llvm/test/Transforms/LICM/store-hoisting.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=false %s | FileCheck -check-prefixes=CHECK,AST %s
-; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true %s | FileCheck  -check-prefixes=CHECK,MSSA %s
+; RUN: opt -S -basic-aa -licm -enable-mssa-loop-dependency=false %s | FileCheck -check-prefixes=CHECK,AST %s
+; RUN: opt -S -basic-aa -licm -enable-mssa-loop-dependency=true %s | FileCheck  -check-prefixes=CHECK,MSSA %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck -check-prefixes=CHECK,AST %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop-mssa(licm)' < %s -S | FileCheck -check-prefixes=CHECK,MSSA %s
 
diff --git a/llvm/test/Transforms/LICM/strlen.ll b/llvm/test/Transforms/LICM/strlen.ll
index 82b3374184db4..230385e42ed4a 100644
--- a/llvm/test/Transforms/LICM/strlen.ll
+++ b/llvm/test/Transforms/LICM/strlen.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -inferattrs -basicaa -licm < %s | FileCheck %s
+; RUN: opt -S -inferattrs -basic-aa -licm < %s | FileCheck %s
 
 define void @test(i64* noalias %loc, i8* noalias %a) {
 ; CHECK-LABEL: @test
diff --git a/llvm/test/Transforms/LICM/unrolled-deeply-nested.ll b/llvm/test/Transforms/LICM/unrolled-deeply-nested.ll
index fad3e8f0c0e31..84e0ee9a02718 100644
--- a/llvm/test/Transforms/LICM/unrolled-deeply-nested.ll
+++ b/llvm/test/Transforms/LICM/unrolled-deeply-nested.ll
@@ -4,7 +4,7 @@
 ; loop.
 ;
 ; RUN: opt -S -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' < %s | FileCheck %s
-; RUN: opt -S -basicaa -licm < %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm < %s | FileCheck %s
 
 define i32 @test(i32* %a, i64 %n.0, i64 %n.0.0, i64 %n.0.0.0, i64 %n.0.0.0.0) nounwind uwtable readonly {
 ; CHECK-LABEL: define i32 @test
diff --git a/llvm/test/Transforms/LICM/volatile-alias.ll b/llvm/test/Transforms/LICM/volatile-alias.ll
index 605720a3b84db..be310e398c000 100644
--- a/llvm/test/Transforms/LICM/volatile-alias.ll
+++ b/llvm/test/Transforms/LICM/volatile-alias.ll
@@ -1,6 +1,6 @@
-; RUN: opt -basicaa -sroa -loop-rotate -licm -S < %s | FileCheck %s
-; RUN: opt -basicaa -sroa -loop-rotate %s | opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S | FileCheck %s
-; RUN: opt -basicaa -sroa -loop-rotate -licm -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
+; RUN: opt -basic-aa -sroa -loop-rotate -licm -S < %s | FileCheck %s
+; RUN: opt -basic-aa -sroa -loop-rotate %s | opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S | FileCheck %s
+; RUN: opt -basic-aa -sroa -loop-rotate -licm -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
 ; The objects *p and *q are aliased to each other, but even though *q is
 ; volatile, *p can be considered invariant in the loop. Check if it is moved
 ; out of the loop.
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll
index d2834be18b0b5..8a0641ffe0d52 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll
@@ -1,5 +1,5 @@
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -scoped-noalias -load-store-vectorizer -S -o - %s | FileCheck -check-prefix=SCOPE -check-prefix=ALL %s
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck -check-prefix=NOSCOPE -check-prefix=ALL %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basic-aa -scoped-noalias -load-store-vectorizer -S -o - %s | FileCheck -check-prefix=SCOPE -check-prefix=ALL %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basic-aa -load-store-vectorizer -S -o - %s | FileCheck -check-prefix=NOSCOPE -check-prefix=ALL %s
 
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/complex-index.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/complex-index.ll
index cd1c7fdc521b5..ac3da6a9b164f 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/complex-index.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/complex-index.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basic-aa -load-store-vectorizer -S -o - %s | FileCheck %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
 
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/extended-index.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/extended-index.ll
index b8e95a6793e8c..9f3618f4f9d55 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/extended-index.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/extended-index.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basic-aa -load-store-vectorizer -S -o - %s | FileCheck %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
 
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/insertion-point.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/insertion-point.ll
index 35836f80456d4..1b4acd1698e3c 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/insertion-point.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/insertion-point.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basic-aa -load-store-vectorizer -S -o - %s | FileCheck %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
 
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/interleaved-mayalias-store.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/interleaved-mayalias-store.ll
index 81ebb712e335d..e24dbf7c6356a 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/interleaved-mayalias-store.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/interleaved-mayalias-store.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basic-aa -load-store-vectorizer -S -o - %s | FileCheck %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
 
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll
index 15c47716aafc2..b432b7e618747 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basic-aa -load-store-vectorizer -S -o - %s | FileCheck %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
 
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
index bcf2265f31009..6e401b1fac876 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basic-aa -load-store-vectorizer -S -o - %s | FileCheck %s
 
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/multiple_tails.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/multiple_tails.ll
index ffd651b2c65b8..410212e81e0fa 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/multiple_tails.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/multiple_tails.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basic-aa -load-store-vectorizer -S -o - %s | FileCheck %s
 
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll
index 9290749bb8985..8a6dafe0750bd 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basic-aa -load-store-vectorizer -S -o - %s | FileCheck %s
 
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll
index e29f3dfa537fd..01606a2f2a5e4 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll
@@ -1,7 +1,6 @@
 ; RUN: opt -codegenprepare -load-store-vectorizer %s -S -o - | FileCheck %s
 ; RUN: opt                 -load-store-vectorizer %s -S -o - | FileCheck %s
-; RUN: opt -codegenprepare -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' %s -S -o - | FileCheck %s
-; RUN: opt                 -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' %s -S -o - | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' %s -S -o - | FileCheck %s
 
 target triple = "x86_64--"
 
diff --git a/llvm/test/Transforms/LoopDeletion/basic-remark.ll b/llvm/test/Transforms/LoopDeletion/basic-remark.ll
new file mode 100644
index 0000000000000..c56ae9dcb03e8
--- /dev/null
+++ b/llvm/test/Transforms/LoopDeletion/basic-remark.ll
@@ -0,0 +1,37 @@
+; RUN: opt -loop-deletion %s -o /dev/null --pass-remarks-output=%t --pass-remarks-filter=loop-delete
+; RUN: cat %t | FileCheck %s
+
+; Check that we use the right debug location: the loop header.
+; CHECK:      --- !Passed
+; CHECK-NEXT: Pass:            loop-delete
+; CHECK-NEXT: Name:            Invariant
+; CHECK-NEXT: DebugLoc:        { File: loop.c, Line: 2, Column: 3 }
+; CHECK-NEXT: Function:        main
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          Loop deleted because it is invariant
+; CHECK-NEXT: ...
+define i32 @main() local_unnamed_addr #0 {
+entry:
+  br label %for.cond, !dbg !9
+
+for.cond:
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.cond ]
+  %cmp = icmp ult i32 %i.0, 1000
+  %inc = add nuw nsw i32 %i.0, 1
+  br i1 %cmp, label %for.cond, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret i32 0
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2, nameTableKind: None, sysroot: "/")
+!1 = !DIFile(filename: "loop.c", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = distinct !DISubprogram(name: "main", scope: !7, file: !7, line: 1, type: !8, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!7 = !DIFile(filename: "loop.c", directory: "")
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 2, column: 3, scope: !6)
diff --git a/llvm/test/Transforms/LoopDeletion/unreachable-loops.ll b/llvm/test/Transforms/LoopDeletion/unreachable-loops.ll
index 5bdaeb11232a5..a74ddf99285e2 100644
--- a/llvm/test/Transforms/LoopDeletion/unreachable-loops.ll
+++ b/llvm/test/Transforms/LoopDeletion/unreachable-loops.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -loop-deletion -verify-dom-info -S | FileCheck %s
+; RUN: opt < %s -loop-deletion -verify-dom-info --pass-remarks-output=%t --pass-remarks-filter=loop-delete -S | FileCheck %s
+; RUN: cat %t | FileCheck %s --check-prefix=REMARKS
 
 ; Checking that we can delete loops that are never executed.
 ; We do not change the constant conditional branch statement (where the not-taken target
@@ -10,6 +11,8 @@ define void @test1(i64 %n, i64 %m) nounwind {
 ; CHECK-LABEL: entry:
 ; CHECK-NEXT: br i1 true, label %return, label %bb.preheader
 ; CHECK-NOT: bb:
+; REMARKS-LABEL: Function: test1
+; REMARKS: Loop deleted because it never executes
 entry:
   br i1 true, label %return, label %bb
 
@@ -63,6 +66,8 @@ define i64 @test3(i64 %n, i64 %m, i64 %maybe_zero) nounwind {
 ; CHECK-LABEL: return:
 ; CHECK-NEXT: %x.lcssa = phi i64 [ 20, %entry ], [ %x.lcssa.ph, %return.loopexit ]
 ; CHECK-NEXT: ret i64 %x.lcssa
+; REMARKS-LABEL: Function: test3
+; REMARKS: Loop deleted because it never executes
 entry:
   br i1 false, label %bb, label %return
 
@@ -121,6 +126,8 @@ define void @test5(i64 %n, i64 %m, i1 %cond) nounwind {
 ; CHECK-LABEL: looppred2:
 ; CHECK-NEXT: br i1 true, label %return, label %bb.preheader
 ; CHECK-NOT: bb:
+; REMARKS-LABEL: Function: test5
+; REMARKS: Loop deleted because it never executes
 entry:
   br i1 %cond, label %looppred1, label %looppred2
 
@@ -179,6 +186,8 @@ define i64 @test7(i64 %n) {
 ; CHECK-LABEL: L1Latch:
 ; CHECK-NEXT: %y = phi i64 [ %y.next, %L1 ], [ %y.L2.lcssa, %L1Latch.loopexit ]
 ; CHECK: br i1 %cond2, label %exit, label %L1
+; REMARKS-LABEL: Function: test7
+; REMARKS: Loop deleted because it never executes
 entry: 
   br label %L1
 
@@ -213,6 +222,8 @@ define void @test8(i64 %n) {
 ; CHECK-NEXT: br label %exit
 ; CHECK-LABEL: exit:
 ; CHECK-NEXT: ret void
+; REMARKS-LABEL: Function: test8
+; REMARKS: Loop deleted because it never executes
 entry: 
   br label %L1
 
@@ -246,6 +257,8 @@ define void @test9(i64 %n) {
 ; CHECK-NEXT: br label %L3
 ; CHECK-LABEL: L3:
 ; CHECK: br i1 %cond2, label %L3, label %L1.loopexit
+; REMARKS-LABEL: Function: test9
+; REMARKS: Loop deleted because it never executes
 entry: 
   br label %L1
 
@@ -311,6 +324,12 @@ define void @test11(i64 %n) {
 ; CHECK-NEXT: br label %exit
 ; CHECK-LABEL: exit:
 ; CHECK-NEXT: ret void
+; REMARKS-LABEL: Function: test11
+; REMARKS: Loop deleted because it is invariant
+; REMARKS-LABEL: Function: test11
+; REMARKS: Loop deleted because it never executes
+; REMARKS-LABEL: Function: test11
+; REMARKS: Loop deleted because it is invariant
 entry: 
   br label %L1
 
@@ -345,6 +364,8 @@ define i64 @test12(i64 %n){
 ; CHECK-LABEL: exit:
 ; CHECK-NEXT:    %y.phi = phi i64 [ undef, %L1.preheader ]
 ; CHECK-NEXT:    ret i64 %y.phi
+; REMARKS-LABEL: Function: test12
+; REMARKS: Loop deleted because it never executes
 
 entry:
   br i1 true, label %exit1, label %L1
@@ -380,6 +401,8 @@ define i64 @test13(i64 %n) {
 ; CHECK-LABEL: exit:
 ; CHECK-NEXT:    %y.phi = phi i64 [ undef, %L1.preheader ]
 ; CHECK-NEXT:    ret i64 %y.phi
+; REMARKS-LABEL: Function: test13
+; REMARKS: Loop deleted because it never executes
 
 entry:
   br i1 true, label %exit1, label %L1
diff --git a/llvm/test/Transforms/LoopDistribute/basic-with-memchecks.ll b/llvm/test/Transforms/LoopDistribute/basic-with-memchecks.ll
index 51a4fe8f60e61..5f72b8d49b51b 100644
--- a/llvm/test/Transforms/LoopDistribute/basic-with-memchecks.ll
+++ b/llvm/test/Transforms/LoopDistribute/basic-with-memchecks.ll
@@ -1,11 +1,11 @@
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S \
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S \
 ; RUN:   < %s | FileCheck %s
 
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -loop-vectorize -force-vector-width=4 \
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -loop-vectorize -force-vector-width=4 \
 ; RUN:   -verify-loop-info -verify-dom-info -S < %s | \
 ; RUN:   FileCheck --check-prefix=VECTORIZE %s
 
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info \
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info \
 ; RUN:   -loop-accesses -analyze < %s | FileCheck %s --check-prefix=ANALYSIS
 
 ; The memcheck version of basic.ll.  We should distribute and vectorize the
diff --git a/llvm/test/Transforms/LoopDistribute/basic.ll b/llvm/test/Transforms/LoopDistribute/basic.ll
index 0d7b6f21824cd..d696c0cf3a45c 100644
--- a/llvm/test/Transforms/LoopDistribute/basic.ll
+++ b/llvm/test/Transforms/LoopDistribute/basic.ll
@@ -1,10 +1,10 @@
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S \
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S \
 ; RUN:   < %s | FileCheck %s
 
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info \
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info \
 ; RUN:   -loop-accesses -analyze < %s | FileCheck %s --check-prefix=ANALYSIS
 
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -loop-vectorize -force-vector-width=4 -S \
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -loop-vectorize -force-vector-width=4 -S \
 ; RUN:   < %s | FileCheck %s --check-prefix=VECTORIZE
 
 ; We should distribute this loop into a safe (2nd statement) and unsafe loop
diff --git a/llvm/test/Transforms/LoopDistribute/bounds-expansion-bug.ll b/llvm/test/Transforms/LoopDistribute/bounds-expansion-bug.ll
index 673b2942bad39..c6226181085fc 100644
--- a/llvm/test/Transforms/LoopDistribute/bounds-expansion-bug.ll
+++ b/llvm/test/Transforms/LoopDistribute/bounds-expansion-bug.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -S < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -S < %s | FileCheck %s
 
 ; When emitting the memchecks for:
 ;
diff --git a/llvm/test/Transforms/LoopDistribute/convergent-no-cross-partition-checks.ll b/llvm/test/Transforms/LoopDistribute/convergent-no-cross-partition-checks.ll
index 1ea5b71aeac04..e825d1f7211ec 100644
--- a/llvm/test/Transforms/LoopDistribute/convergent-no-cross-partition-checks.ll
+++ b/llvm/test/Transforms/LoopDistribute/convergent-no-cross-partition-checks.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute \
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute \
 ; RUN:   -verify-loop-info -verify-dom-info -S < %s | FileCheck %s
 
 ; Derived from crash-in-memcheck-generation.ll
diff --git a/llvm/test/Transforms/LoopDistribute/crash-in-memcheck-generation.ll b/llvm/test/Transforms/LoopDistribute/crash-in-memcheck-generation.ll
index 4ac3ad173ab7f..f804b72e4afb6 100644
--- a/llvm/test/Transforms/LoopDistribute/crash-in-memcheck-generation.ll
+++ b/llvm/test/Transforms/LoopDistribute/crash-in-memcheck-generation.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -loop-vectorize -force-vector-width=4 \
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -loop-vectorize -force-vector-width=4 \
 ; RUN:   -verify-loop-info -verify-dom-info -S < %s | FileCheck %s
 
 ; If only A and B can alias here, we don't need memchecks to distribute since
diff --git a/llvm/test/Transforms/LoopDistribute/followup.ll b/llvm/test/Transforms/LoopDistribute/followup.ll
index a70a6d4fb9efa..35ae5e73b57d9 100644
--- a/llvm/test/Transforms/LoopDistribute/followup.ll
+++ b/llvm/test/Transforms/LoopDistribute/followup.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-distribute -S < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-distribute -S < %s | FileCheck %s
 ;
 ; Check that followup loop-attributes are applied to the loops after
 ; loop distribution.
diff --git a/llvm/test/Transforms/LoopDistribute/metadata.ll b/llvm/test/Transforms/LoopDistribute/metadata.ll
index 6c99340f7d0ba..e906b48077c78 100644
--- a/llvm/test/Transforms/LoopDistribute/metadata.ll
+++ b/llvm/test/Transforms/LoopDistribute/metadata.ll
@@ -1,5 +1,5 @@
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute=0 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=EXPLICIT --check-prefix=DEFAULT_OFF
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute=1 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=EXPLICIT --check-prefix=DEFAULT_ON
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute=0 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=EXPLICIT --check-prefix=DEFAULT_OFF
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute=1 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=EXPLICIT --check-prefix=DEFAULT_ON
 
 ; Same loop as in basic.ll.  Check that distribution is enabled/disabled
 ; properly according to -enable-loop-distribute=0/1 and the
diff --git a/llvm/test/Transforms/LoopDistribute/no-if-convert.ll b/llvm/test/Transforms/LoopDistribute/no-if-convert.ll
index 81901972f379e..a458b3cca6be6 100644
--- a/llvm/test/Transforms/LoopDistribute/no-if-convert.ll
+++ b/llvm/test/Transforms/LoopDistribute/no-if-convert.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S < %s \
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S < %s \
 ; RUN:   | FileCheck %s
 
 ; We should distribute this loop along === but not along ---.  The last
diff --git a/llvm/test/Transforms/LoopDistribute/pr28443.ll b/llvm/test/Transforms/LoopDistribute/pr28443.ll
index 1048c1a4c5933..ecb3ff813338f 100644
--- a/llvm/test/Transforms/LoopDistribute/pr28443.ll
+++ b/llvm/test/Transforms/LoopDistribute/pr28443.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S \
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S \
 ; RUN:   < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll b/llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll
index 8ad19cb47075b..caa39623b0082 100644
--- a/llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll
+++ b/llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -S -enable-mem-access-versioning=0 < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -S -enable-mem-access-versioning=0 < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopDistribute/symbolic-stride.ll b/llvm/test/Transforms/LoopDistribute/symbolic-stride.ll
index e760af3586732..6992c88799f0b 100644
--- a/llvm/test/Transforms/LoopDistribute/symbolic-stride.ll
+++ b/llvm/test/Transforms/LoopDistribute/symbolic-stride.ll
@@ -1,7 +1,7 @@
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -S < %s | \
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -S < %s | \
 ; RUN:     FileCheck %s --check-prefix=ALL --check-prefix=STRIDE_SPEC
 
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -S -enable-mem-access-versioning=0 < %s | \
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -S -enable-mem-access-versioning=0 < %s | \
 ; RUN:     FileCheck %s --check-prefix=ALL --check-prefix=NO_STRIDE_SPEC
 
 ; If we don't speculate stride for 1 we can't distribute along the line
diff --git a/llvm/test/Transforms/LoopDistribute/unknown-bounds-for-memchecks.ll b/llvm/test/Transforms/LoopDistribute/unknown-bounds-for-memchecks.ll
index d8d91da8a66ce..c4f67a98b8793 100644
--- a/llvm/test/Transforms/LoopDistribute/unknown-bounds-for-memchecks.ll
+++ b/llvm/test/Transforms/LoopDistribute/unknown-bounds-for-memchecks.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -S < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -S < %s | FileCheck %s
 
 ; If we can't find the bounds for one of the arrays in order to generate the
 ; memchecks (e.g., C[i * i] below), loop shold not get distributed.
diff --git a/llvm/test/Transforms/LoopIdiom/X86/unordered-atomic-memcpy.ll b/llvm/test/Transforms/LoopIdiom/X86/unordered-atomic-memcpy.ll
index 04a13d3467300..c810b782bec1c 100644
--- a/llvm/test/Transforms/LoopIdiom/X86/unordered-atomic-memcpy.ll
+++ b/llvm/test/Transforms/LoopIdiom/X86/unordered-atomic-memcpy.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+; RUN: opt -basic-aa -loop-idiom < %s -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/LoopIdiom/basic-address-space.ll b/llvm/test/Transforms/LoopIdiom/basic-address-space.ll
index c709b9acd580b..936f314e36828 100644
--- a/llvm/test/Transforms/LoopIdiom/basic-address-space.ll
+++ b/llvm/test/Transforms/LoopIdiom/basic-address-space.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+; RUN: opt -basic-aa -loop-idiom < %s -S | FileCheck %s
 
 target datalayout = "e-p:32:32:32-p1:64:64:64-p2:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32-n8:16:32:64"
 target triple = "x86_64-apple-darwin10.0.0"
diff --git a/llvm/test/Transforms/LoopIdiom/basic.ll b/llvm/test/Transforms/LoopIdiom/basic.ll
index 89d8dab0d6886..5a305a61ad908 100644
--- a/llvm/test/Transforms/LoopIdiom/basic.ll
+++ b/llvm/test/Transforms/LoopIdiom/basic.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+; RUN: opt -basic-aa -loop-idiom < %s -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ; For @test11_pattern
diff --git a/llvm/test/Transforms/LoopIdiom/crash.ll b/llvm/test/Transforms/LoopIdiom/crash.ll
index c964aacd00c2d..6022745b4374e 100644
--- a/llvm/test/Transforms/LoopIdiom/crash.ll
+++ b/llvm/test/Transforms/LoopIdiom/crash.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-idiom -S < %s
+; RUN: opt -basic-aa -loop-idiom -S < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
diff --git a/llvm/test/Transforms/LoopIdiom/lir-heurs-multi-block-loop.ll b/llvm/test/Transforms/LoopIdiom/lir-heurs-multi-block-loop.ll
index e1e07a06d6908..ed4e0c11c88b0 100644
--- a/llvm/test/Transforms/LoopIdiom/lir-heurs-multi-block-loop.ll
+++ b/llvm/test/Transforms/LoopIdiom/lir-heurs-multi-block-loop.ll
@@ -1,4 +1,4 @@
-;  RUN: opt -basicaa -loop-idiom -use-lir-code-size-heurs=true < %s -S | FileCheck %s
+;  RUN: opt -basic-aa -loop-idiom -use-lir-code-size-heurs=true < %s -S | FileCheck %s
 
 ; When compiling for codesize we avoid idiom recognition for a
 ; multi-block loop unless it is one of
diff --git a/llvm/test/Transforms/LoopIdiom/memcpy-debugify-remarks.ll b/llvm/test/Transforms/LoopIdiom/memcpy-debugify-remarks.ll
index f66af1b5e3c7f..3578540cc4d2d 100644
--- a/llvm/test/Transforms/LoopIdiom/memcpy-debugify-remarks.ll
+++ b/llvm/test/Transforms/LoopIdiom/memcpy-debugify-remarks.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -basicaa -debugify -loop-idiom -pass-remarks=loop-idiom -pass-remarks-analysis=loop-idiom -verify -verify-each -verify-dom-info -verify-loop-info  < %s -S 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -debugify -loop-idiom -pass-remarks=loop-idiom -pass-remarks-analysis=loop-idiom -verify -verify-each -verify-dom-info -verify-loop-info  < %s -S 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopIdiom/memset-debugify-remarks.ll b/llvm/test/Transforms/LoopIdiom/memset-debugify-remarks.ll
index cfd1075d4d7b4..06e17fecec6da 100644
--- a/llvm/test/Transforms/LoopIdiom/memset-debugify-remarks.ll
+++ b/llvm/test/Transforms/LoopIdiom/memset-debugify-remarks.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -basicaa -debugify -loop-idiom -pass-remarks=loop-idiom -pass-remarks-analysis=loop-idiom -verify -verify-each -verify-dom-info -verify-loop-info  < %s -S 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -debugify -loop-idiom -pass-remarks=loop-idiom -pass-remarks-analysis=loop-idiom -verify -verify-each -verify-dom-info -verify-loop-info  < %s -S 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopIdiom/non-integral-pointers.ll b/llvm/test/Transforms/LoopIdiom/non-integral-pointers.ll
index 7646d5ac72d1f..6846e88253940 100644
--- a/llvm/test/Transforms/LoopIdiom/non-integral-pointers.ll
+++ b/llvm/test/Transforms/LoopIdiom/non-integral-pointers.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -loop-idiom < %s | FileCheck %s
+; RUN: opt -S -basic-aa -loop-idiom < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:4"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopIdiom/struct-custom-dl.ll b/llvm/test/Transforms/LoopIdiom/struct-custom-dl.ll
index f2330b636f428..a295e45be5092 100644
--- a/llvm/test/Transforms/LoopIdiom/struct-custom-dl.ll
+++ b/llvm/test/Transforms/LoopIdiom/struct-custom-dl.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+; RUN: opt -basic-aa -loop-idiom < %s -S | FileCheck %s
 target datalayout = "e-p:40:64:64:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 %struct.foo = type { i32, i32 }
diff --git a/llvm/test/Transforms/LoopIdiom/struct.ll b/llvm/test/Transforms/LoopIdiom/struct.ll
index 2828024952e2a..5f7e1d96a855f 100644
--- a/llvm/test/Transforms/LoopIdiom/struct.ll
+++ b/llvm/test/Transforms/LoopIdiom/struct.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+; RUN: opt -basic-aa -loop-idiom < %s -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 target triple = "x86_64-apple-darwin10.0.0"
diff --git a/llvm/test/Transforms/LoopIdiom/struct_pattern.ll b/llvm/test/Transforms/LoopIdiom/struct_pattern.ll
index d7809b746b15f..91cda98d50c60 100644
--- a/llvm/test/Transforms/LoopIdiom/struct_pattern.ll
+++ b/llvm/test/Transforms/LoopIdiom/struct_pattern.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+; RUN: opt -basic-aa -loop-idiom < %s -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ; CHECK: @.memset_pattern = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
diff --git a/llvm/test/Transforms/LoopIdiom/unordered-atomic-memcpy-noarch.ll b/llvm/test/Transforms/LoopIdiom/unordered-atomic-memcpy-noarch.ll
index 341a7a0baebf0..65127c95a5113 100644
--- a/llvm/test/Transforms/LoopIdiom/unordered-atomic-memcpy-noarch.ll
+++ b/llvm/test/Transforms/LoopIdiom/unordered-atomic-memcpy-noarch.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+; RUN: opt -basic-aa -loop-idiom < %s -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ;; memcpy.atomic formation (atomic load & store) -- element size 2
diff --git a/llvm/test/Transforms/LoopIdiom/unroll-custom-dl.ll b/llvm/test/Transforms/LoopIdiom/unroll-custom-dl.ll
index fb09fd952ad49..94f52f6e0f2ca 100644
--- a/llvm/test/Transforms/LoopIdiom/unroll-custom-dl.ll
+++ b/llvm/test/Transforms/LoopIdiom/unroll-custom-dl.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+; RUN: opt -basic-aa -loop-idiom < %s -S | FileCheck %s
 target datalayout = "e-p:64:64:64:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ; CHECK: @.memset_pattern = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
diff --git a/llvm/test/Transforms/LoopIdiom/unroll.ll b/llvm/test/Transforms/LoopIdiom/unroll.ll
index 5981c3e4492f3..e9fe719ca1b5f 100644
--- a/llvm/test/Transforms/LoopIdiom/unroll.ll
+++ b/llvm/test/Transforms/LoopIdiom/unroll.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+; RUN: opt -basic-aa -loop-idiom < %s -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ; CHECK: @.memset_pattern = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
diff --git a/llvm/test/Transforms/LoopInterchange/call-instructions.ll b/llvm/test/Transforms/LoopInterchange/call-instructions.ll
index 31676e034bfb8..d945fe6d720c9 100644
--- a/llvm/test/Transforms/LoopInterchange/call-instructions.ll
+++ b/llvm/test/Transforms/LoopInterchange/call-instructions.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -basicaa -loop-interchange -pass-remarks-missed='loop-interchange' -pass-remarks-output=%t -S \
+; RUN: opt < %s -basic-aa -loop-interchange -pass-remarks-missed='loop-interchange' -pass-remarks-output=%t -S \
 ; RUN:     -verify-dom-info -verify-loop-info -stats 2>&1 | FileCheck -check-prefix=STATS %s
 ; RUN: FileCheck --input-file=%t %s
 
diff --git a/llvm/test/Transforms/LoopInterchange/currentLimitation.ll b/llvm/test/Transforms/LoopInterchange/currentLimitation.ll
index 4f20bdc1e2e68..768dd3bb2e23d 100644
--- a/llvm/test/Transforms/LoopInterchange/currentLimitation.ll
+++ b/llvm/test/Transforms/LoopInterchange/currentLimitation.ll
@@ -1,8 +1,8 @@
-; RUN: opt < %s -basicaa -loop-interchange -pass-remarks-missed='loop-interchange' \
+; RUN: opt < %s -basic-aa -loop-interchange -pass-remarks-missed='loop-interchange' \
 ; RUN:   -pass-remarks-output=%t -verify-loop-info -verify-dom-info -S | FileCheck -check-prefix=IR %s
 ; RUN: FileCheck --input-file=%t %s
 
-; RUN: opt < %s -basicaa -loop-interchange -pass-remarks-missed='loop-interchange' \
+; RUN: opt < %s -basic-aa -loop-interchange -pass-remarks-missed='loop-interchange' \
 ; RUN:   -da-disable-delinearization-checks -pass-remarks-output=%t             \
 ; RUN:   -verify-loop-info -verify-dom-info -S | FileCheck -check-prefix=IR %s
 ; RUN: FileCheck --check-prefix=DELIN --input-file=%t %s
diff --git a/llvm/test/Transforms/LoopInterchange/debuginfo.ll b/llvm/test/Transforms/LoopInterchange/debuginfo.ll
index c35fda0c9bbee..e2187b9328142 100644
--- a/llvm/test/Transforms/LoopInterchange/debuginfo.ll
+++ b/llvm/test/Transforms/LoopInterchange/debuginfo.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -loop-interchange -pass-remarks='loop-interchange' -pass-remarks-output=%t -S \
+; RUN: opt < %s -basic-aa -loop-interchange -pass-remarks='loop-interchange' -pass-remarks-output=%t -S \
 ; RUN:     -verify-dom-info -verify-loop-info | FileCheck %s
 ; RUN: FileCheck -check-prefix=REMARK --input-file=%t %s
 
diff --git a/llvm/test/Transforms/LoopInterchange/inner-only-reductions.ll b/llvm/test/Transforms/LoopInterchange/inner-only-reductions.ll
index 7c77b10dd8e2f..d0eac6d65e826 100644
--- a/llvm/test/Transforms/LoopInterchange/inner-only-reductions.ll
+++ b/llvm/test/Transforms/LoopInterchange/inner-only-reductions.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -loop-interchange -pass-remarks-missed='loop-interchange' -pass-remarks-output=%t -S \
+; RUN: opt < %s -basic-aa -loop-interchange -pass-remarks-missed='loop-interchange' -pass-remarks-output=%t -S \
 ; RUN:     -verify-dom-info -verify-loop-info -verify-loop-lcssa 2>&1 | FileCheck -check-prefix=IR %s
 ; RUN: FileCheck --input-file=%t %s
 
diff --git a/llvm/test/Transforms/LoopInterchange/interchange-flow-dep-outer.ll b/llvm/test/Transforms/LoopInterchange/interchange-flow-dep-outer.ll
index 02a30f4c1f7db..096eda0da1bc7 100644
--- a/llvm/test/Transforms/LoopInterchange/interchange-flow-dep-outer.ll
+++ b/llvm/test/Transforms/LoopInterchange/interchange-flow-dep-outer.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -basicaa -loop-interchange -verify-dom-info -verify-loop-info \
+; RUN: opt < %s -basic-aa -loop-interchange -verify-dom-info -verify-loop-info \
 ; RUN:     -S -debug 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopInterchange/interchange-insts-between-indvar.ll b/llvm/test/Transforms/LoopInterchange/interchange-insts-between-indvar.ll
index 6653f0779e980..fa2f112e71198 100644
--- a/llvm/test/Transforms/LoopInterchange/interchange-insts-between-indvar.ll
+++ b/llvm/test/Transforms/LoopInterchange/interchange-insts-between-indvar.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -loop-interchange -verify-dom-info -verify-loop-info \
+; RUN: opt < %s -basic-aa -loop-interchange -verify-dom-info -verify-loop-info \
 ; RUN:     -S -pass-remarks=loop-interchange 2>&1 | FileCheck %s
 
 @A10 = local_unnamed_addr global [3 x [3 x i32]] zeroinitializer, align 16
diff --git a/llvm/test/Transforms/LoopInterchange/interchangeable.ll b/llvm/test/Transforms/LoopInterchange/interchangeable.ll
index 3dc6bdc43fb2b..9bfa8333374a8 100644
--- a/llvm/test/Transforms/LoopInterchange/interchangeable.ll
+++ b/llvm/test/Transforms/LoopInterchange/interchangeable.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -loop-interchange -verify-dom-info -verify-loop-info -verify-scev -verify-loop-lcssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -loop-interchange -verify-dom-info -verify-loop-info -verify-scev -verify-loop-lcssa -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopInterchange/lcssa-preheader.ll b/llvm/test/Transforms/LoopInterchange/lcssa-preheader.ll
index 9e6a06489b1ec..3d2b532d65aa4 100644
--- a/llvm/test/Transforms/LoopInterchange/lcssa-preheader.ll
+++ b/llvm/test/Transforms/LoopInterchange/lcssa-preheader.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -loop-interchange -pass-remarks-missed='loop-interchange' -verify-loop-lcssa -S | FileCheck %s
+; RUN: opt < %s -basic-aa -loop-interchange -pass-remarks-missed='loop-interchange' -verify-loop-lcssa -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopInterchange/lcssa.ll b/llvm/test/Transforms/LoopInterchange/lcssa.ll
index edc9800ecd76f..d7b6fdc148dbe 100644
--- a/llvm/test/Transforms/LoopInterchange/lcssa.ll
+++ b/llvm/test/Transforms/LoopInterchange/lcssa.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -loop-interchange -pass-remarks-missed='loop-interchange' -verify-loop-lcssa -pass-remarks-output=%t -S
+; RUN: opt < %s -basic-aa -loop-interchange -pass-remarks-missed='loop-interchange' -verify-loop-lcssa -pass-remarks-output=%t -S
 ; RUN: FileCheck --input-file %t --check-prefix REMARK %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopInterchange/loop-interchange-optimization-remarks.ll b/llvm/test/Transforms/LoopInterchange/loop-interchange-optimization-remarks.ll
index 8aaf7c7e97c45..fb66c86a34741 100644
--- a/llvm/test/Transforms/LoopInterchange/loop-interchange-optimization-remarks.ll
+++ b/llvm/test/Transforms/LoopInterchange/loop-interchange-optimization-remarks.ll
@@ -1,11 +1,11 @@
 ; Test optimization remarks generated by the LoopInterchange pass.
 ;
-; RUN: opt < %s -basicaa -loop-interchange -verify-dom-info -verify-loop-info \
+; RUN: opt < %s -basic-aa -loop-interchange -verify-dom-info -verify-loop-info \
 ; RUN:     -pass-remarks-output=%t -pass-remarks-missed='loop-interchange' \
 ; RUN:     -pass-remarks='loop-interchange' -S
 ; RUN: cat %t |  FileCheck %s
 
-; RUN: opt < %s -basicaa -loop-interchange -verify-dom-info -verify-loop-info \
+; RUN: opt < %s -basic-aa -loop-interchange -verify-dom-info -verify-loop-info \
 ; RUN:     -pass-remarks-output=%t -pass-remarks-missed='loop-interchange' \
 ; RUN:     -pass-remarks='loop-interchange' -S -da-disable-delinearization-checks
 ; RUN: cat %t |  FileCheck --check-prefix=DELIN %s
diff --git a/llvm/test/Transforms/LoopInterchange/not-interchanged-dependencies-1.ll b/llvm/test/Transforms/LoopInterchange/not-interchanged-dependencies-1.ll
index 11010039f0877..3bf1e214cfefb 100644
--- a/llvm/test/Transforms/LoopInterchange/not-interchanged-dependencies-1.ll
+++ b/llvm/test/Transforms/LoopInterchange/not-interchanged-dependencies-1.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -basicaa -loop-interchange -verify-dom-info -verify-loop-info \
+; RUN: opt < %s -basic-aa -loop-interchange -verify-dom-info -verify-loop-info \
 ; RUN:     -S -debug 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopInterchange/not-interchanged-loop-nest-3.ll b/llvm/test/Transforms/LoopInterchange/not-interchanged-loop-nest-3.ll
index 7dfbe119537cf..2c169f6e56a2a 100644
--- a/llvm/test/Transforms/LoopInterchange/not-interchanged-loop-nest-3.ll
+++ b/llvm/test/Transforms/LoopInterchange/not-interchanged-loop-nest-3.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -basicaa -loop-interchange -verify-dom-info -verify-loop-info \
+; RUN: opt < %s -basic-aa -loop-interchange -verify-dom-info -verify-loop-info \
 ; RUN:     -S -debug 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopInterchange/not-interchanged-tightly-nested.ll b/llvm/test/Transforms/LoopInterchange/not-interchanged-tightly-nested.ll
index 13f6e38914c36..167d4bfdc8bb5 100644
--- a/llvm/test/Transforms/LoopInterchange/not-interchanged-tightly-nested.ll
+++ b/llvm/test/Transforms/LoopInterchange/not-interchanged-tightly-nested.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -basicaa -loop-interchange -verify-dom-info -verify-loop-info \
+; RUN: opt < %s -basic-aa -loop-interchange -verify-dom-info -verify-loop-info \
 ; RUN:     -S -debug 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopInterchange/outer-only-reductions.ll b/llvm/test/Transforms/LoopInterchange/outer-only-reductions.ll
index c58e7d3d27224..7e9db5a4d4e59 100644
--- a/llvm/test/Transforms/LoopInterchange/outer-only-reductions.ll
+++ b/llvm/test/Transforms/LoopInterchange/outer-only-reductions.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -loop-interchange -pass-remarks-missed='loop-interchange' -pass-remarks-output=%t -S \
+; RUN: opt < %s -basic-aa -loop-interchange -pass-remarks-missed='loop-interchange' -pass-remarks-output=%t -S \
 ; RUN:     -verify-dom-info -verify-loop-info -verify-loop-lcssa 2>&1 | FileCheck -check-prefix=IR %s
 ; RUN: FileCheck --input-file=%t %s
 
diff --git a/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll b/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll
index deff8ccc4f280..24c5184a4156c 100644
--- a/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll
+++ b/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -loop-interchange -pass-remarks-missed='loop-interchange' -pass-remarks-output=%t -S \
+; RUN: opt < %s -basic-aa -loop-interchange -pass-remarks-missed='loop-interchange' -pass-remarks-output=%t -S \
 ; RUN:     -verify-dom-info -verify-loop-info -verify-loop-lcssa -stats 2>&1 | FileCheck %s
 ; RUN: FileCheck --input-file=%t --check-prefix=REMARKS %s
 
diff --git a/llvm/test/Transforms/LoopLoadElim/multiple-stores-same-block.ll b/llvm/test/Transforms/LoopLoadElim/multiple-stores-same-block.ll
index b0c0f3dee86ea..b3c9b7720ed55 100644
--- a/llvm/test/Transforms/LoopLoadElim/multiple-stores-same-block.ll
+++ b/llvm/test/Transforms/LoopLoadElim/multiple-stores-same-block.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-load-elim -S < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-load-elim -S < %s | FileCheck %s
 
 ; In this case the later store forward to the load:
 ;
diff --git a/llvm/test/Transforms/LoopLoadElim/opt-size.ll b/llvm/test/Transforms/LoopLoadElim/opt-size.ll
index f9d82fec2a22f..d9fc6bdc9dd20 100644
--- a/llvm/test/Transforms/LoopLoadElim/opt-size.ll
+++ b/llvm/test/Transforms/LoopLoadElim/opt-size.ll
@@ -1,6 +1,6 @@
-; RUN: opt -basicaa -loop-load-elim -S < %s | FileCheck %s
-; RUN: opt -basicaa -loop-load-elim -pgso -S < %s | FileCheck %s -check-prefix=PGSO
-; RUN: opt -basicaa -loop-load-elim -pgso=false -S < %s | FileCheck %s -check-prefix=NPGSO
+; RUN: opt -basic-aa -loop-load-elim -S < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-load-elim -pgso -S < %s | FileCheck %s -check-prefix=PGSO
+; RUN: opt -basic-aa -loop-load-elim -pgso=false -S < %s | FileCheck %s -check-prefix=NPGSO
 
 ; When optimizing for size don't eliminate in this loop because the loop would
 ; have to be versioned first because A and C may alias.
diff --git a/llvm/test/Transforms/LoopLoadElim/unknown-dep.ll b/llvm/test/Transforms/LoopLoadElim/unknown-dep.ll
index d2df718ca4c38..58d3308f1474f 100644
--- a/llvm/test/Transforms/LoopLoadElim/unknown-dep.ll
+++ b/llvm/test/Transforms/LoopLoadElim/unknown-dep.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-load-elim -S < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-load-elim -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopPredication/invariant_load.ll b/llvm/test/Transforms/LoopPredication/invariant_load.ll
index ffdc38274efbc..1002d97081851 100644
--- a/llvm/test/Transforms/LoopPredication/invariant_load.ll
+++ b/llvm/test/Transforms/LoopPredication/invariant_load.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -basicaa -loop-predication < %s 2>&1 | FileCheck %s
+; RUN: opt -S -basic-aa -loop-predication < %s 2>&1 | FileCheck %s
 ; RUN: opt -S -aa-pipeline=basic-aa -passes='require<aa>,require<scalar-evolution>,loop(loop-predication)' < %s 2>&1 | FileCheck %s
 
 declare void @llvm.experimental.guard(i1, ...)
diff --git a/llvm/test/Transforms/LoopSimplify/ashr-crash.ll b/llvm/test/Transforms/LoopSimplify/ashr-crash.ll
index b5cc1449cec26..00f65684542c6 100644
--- a/llvm/test/Transforms/LoopSimplify/ashr-crash.ll
+++ b/llvm/test/Transforms/LoopSimplify/ashr-crash.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-rotate -licm -instcombine -indvars -loop-unroll -S %s | FileCheck %s
+; RUN: opt -basic-aa -loop-rotate -licm -instcombine -indvars -loop-unroll -S %s | FileCheck %s
 ;
 ; PR18361: ScalarEvolution::getAddRecExpr():
 ;          Assertion `isLoopInvariant(Operands[i],...
diff --git a/llvm/test/Transforms/LoopStrengthReduce/ivchain.ll b/llvm/test/Transforms/LoopStrengthReduce/ivchain.ll
index 52faddea94a3d..875ea2c3882bf 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/ivchain.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/ivchain.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -loop-reduce -S | FileCheck %s
-; RUN: opt -passes='require<scalar-evolution>,require<targetir>,loop(strength-reduce)' < %s -S | FileCheck %s
+; RUN: opt -passes='require<scalar-evolution>,require<targetir>,loop(loop-reduce)' < %s -S | FileCheck %s
 ;
 ; PR11782: bad cast to AddRecExpr.
 ; A sign extend feeds an IVUser and cannot be hoisted into the AddRec.
diff --git a/llvm/test/Transforms/LoopUnroll/FullUnroll.ll b/llvm/test/Transforms/LoopUnroll/FullUnroll.ll
index 01936e487682b..5dee20be33253 100644
--- a/llvm/test/Transforms/LoopUnroll/FullUnroll.ll
+++ b/llvm/test/Transforms/LoopUnroll/FullUnroll.ll
@@ -1,4 +1,4 @@
-; RUN: opt -passes='default<O1>' -disable-verify --mtriple x86_64-pc-linux-gnu -new-pm-disable-loop-unrolling=true \
+; RUN: opt -passes='default<O1>' -disable-verify --mtriple x86_64-pc-linux-gnu -disable-loop-unrolling=true \
 ; RUN: -S -o - %s | FileCheck %s
 
 ; This checks that the loop full unroller will fire in the new pass manager
diff --git a/llvm/test/Transforms/LoopUnroll/X86/pr46430.ll b/llvm/test/Transforms/LoopUnroll/X86/pr46430.ll
new file mode 100644
index 0000000000000..7698689ad8edf
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/X86/pr46430.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -loop-unroll -costmodel-reduxcost -S -o - %s | FileCheck %s
+
+define void @g() {
+; CHECK-LABEL: @g(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[F_EXIT:%.*]]
+; CHECK:       f.exit:
+; CHECK-NEXT:    [[RDX_SHUF9_I:%.*]] = shufflevector <4 x i16> <i16 1, i16 0, i16 undef, i16 undef>, <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX10_I:%.*]] = xor <4 x i16> <i16 1, i16 0, i16 undef, i16 undef>, [[RDX_SHUF9_I]]
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x i16> [[BIN_RDX10_I]], i32 0
+; CHECK-NEXT:    br label [[F_EXIT]]
+;
+entry:
+  br label %f.exit
+
+f.exit:                                           ; preds = %f.exit, %entry
+  %rdx.shuf9.i = shufflevector <4 x i16> <i16 1, i16 0, i16 undef, i16 undef>, <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %bin.rdx10.i = xor <4 x i16> <i16 1, i16 0, i16 undef, i16 undef>, %rdx.shuf9.i
+  %0 = extractelement <4 x i16> %bin.rdx10.i, i32 0
+  br label %f.exit
+}
+
diff --git a/llvm/test/Transforms/LoopUnroll/basic.ll b/llvm/test/Transforms/LoopUnroll/basic.ll
index 1ceac9354820d..a6fa027cc0538 100644
--- a/llvm/test/Transforms/LoopUnroll/basic.ll
+++ b/llvm/test/Transforms/LoopUnroll/basic.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -loop-unroll -S | FileCheck %s
-; RUN: opt < %s -passes='require<opt-remark-emit>,unroll' -S | FileCheck %s
+; RUN: opt < %s -passes='require<opt-remark-emit>,loop-unroll' -S | FileCheck %s
 
 
 ; This should not unroll since the address of the loop header is taken.
diff --git a/llvm/test/Transforms/LoopUnroll/disable-full-unroll-by-opt.ll b/llvm/test/Transforms/LoopUnroll/disable-full-unroll-by-opt.ll
index c89948ad52cb7..66ff59703249a 100644
--- a/llvm/test/Transforms/LoopUnroll/disable-full-unroll-by-opt.ll
+++ b/llvm/test/Transforms/LoopUnroll/disable-full-unroll-by-opt.ll
@@ -1,19 +1,19 @@
 ; Default behavior
-; RUN: opt < %s -passes='unroll' -S | FileCheck %s -check-prefixes=ENABLE,COMMON
+; RUN: opt < %s -passes='loop-unroll' -S | FileCheck %s -check-prefixes=ENABLE,COMMON
 
 ; Pass option
-; RUN: opt < %s -passes='unroll<full-unroll-max=0>'  -S | FileCheck %s -check-prefixes=DISABLE,COMMON
-; RUN: opt < %s -passes='unroll<full-unroll-max=30>' -S | FileCheck %s -check-prefixes=DISABLE,COMMON
-; RUN: opt < %s -passes='unroll<full-unroll-max=36>' -S | FileCheck %s -check-prefixes=ENABLE,COMMON
+; RUN: opt < %s -passes='loop-unroll<full-unroll-max=0>'  -S | FileCheck %s -check-prefixes=DISABLE,COMMON
+; RUN: opt < %s -passes='loop-unroll<full-unroll-max=30>' -S | FileCheck %s -check-prefixes=DISABLE,COMMON
+; RUN: opt < %s -passes='loop-unroll<full-unroll-max=36>' -S | FileCheck %s -check-prefixes=ENABLE,COMMON
 
 ; cl::opt option
-; RUN: opt < %s -passes='unroll' -unroll-full-max-count=0 -S | FileCheck %s -check-prefixes=DISABLE,COMMON
-; RUN: opt < %s -passes='unroll' -unroll-full-max-count=30 -S | FileCheck %s -check-prefixes=DISABLE,COMMON
-; RUN: opt < %s -passes='unroll' -unroll-full-max-count=36 -S | FileCheck %s -check-prefixes=ENABLE,COMMON
+; RUN: opt < %s -passes='loop-unroll' -unroll-full-max-count=0 -S | FileCheck %s -check-prefixes=DISABLE,COMMON
+; RUN: opt < %s -passes='loop-unroll' -unroll-full-max-count=30 -S | FileCheck %s -check-prefixes=DISABLE,COMMON
+; RUN: opt < %s -passes='loop-unroll' -unroll-full-max-count=36 -S | FileCheck %s -check-prefixes=ENABLE,COMMON
 
 ; Pass option has a priority over cl::opt
-; RUN: opt < %s -passes='unroll<full-unroll-max=30>' -unroll-full-max-count=36 -S | FileCheck %s -check-prefixes=DISABLE,COMMON
-; RUN: opt < %s -passes='unroll<full-unroll-max=36>' -unroll-full-max-count=30 -S | FileCheck %s -check-prefixes=ENABLE,COMMON
+; RUN: opt < %s -passes='loop-unroll<full-unroll-max=30>' -unroll-full-max-count=36 -S | FileCheck %s -check-prefixes=DISABLE,COMMON
+; RUN: opt < %s -passes='loop-unroll<full-unroll-max=36>' -unroll-full-max-count=30 -S | FileCheck %s -check-prefixes=ENABLE,COMMON
 
 define void @test() {
 ; COMMON-LABEL: @test(
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-bad-cost.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-bad-cost.ll
index 9bbd21accc8ef..d8759fbb12cc2 100644
--- a/llvm/test/Transforms/LoopUnroll/full-unroll-bad-cost.ll
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-bad-cost.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S -loop-unroll < %s | FileCheck %s
-; RUN: opt < %s -passes='require<opt-remark-emit>,loop(unroll-full)' -S | FileCheck %s
+; RUN: opt < %s -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -S | FileCheck %s
 
 ; LLVM should not try to fully unroll this loop.
 
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-crashers.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-crashers.ll
index d83e56635e8cf..e495b9b44cde4 100644
--- a/llvm/test/Transforms/LoopUnroll/full-unroll-crashers.ll
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-crashers.ll
@@ -1,6 +1,6 @@
 ; Check that we don't crash on corner cases.
 ; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=1 -unroll-max-percent-threshold-boost=200 -o /dev/null
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=1 -unroll-max-percent-threshold-boost=200 -o /dev/null
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=1 -unroll-max-percent-threshold-boost=200 -o /dev/null
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 @known_constant = internal unnamed_addr constant [10 x i32] [i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1], align 16
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-2.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-2.ll
index a143056affe77..4345b9fed1d07 100644
--- a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-2.ll
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-2.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 @unknown_global = internal unnamed_addr global [9 x i32] [i32 0, i32 -1, i32 0, i32 -1, i32 5, i32 -1, i32 0, i32 -1, i32 0], align 16
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-cmp.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-cmp.ll
index e65f794286ee9..f76ad5aaf1eab 100644
--- a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-cmp.ll
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-cmp.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 @known_constant = internal unnamed_addr constant [10 x i32] [i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1], align 16
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-dce.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-dce.ll
index 57de3041f5713..de94b62dfc7f8 100644
--- a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-dce.ll
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-dce.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=12 -unroll-max-percent-threshold-boost=400 | FileCheck %s
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=12 -unroll-max-percent-threshold-boost=400 | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=12 -unroll-max-percent-threshold-boost=400 | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 @known_constant = internal unnamed_addr constant [10 x i32] [i32 0, i32 0, i32 0, i32 0, i32 1, i32 0, i32 0, i32 0, i32 0, i32 0], align 16
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-geps.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-geps.ll
index 238869d120ba8..38ad3a089b2ce 100644
--- a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-geps.ll
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-geps.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 ; When examining gep-instructions we shouldn't consider them simplified if the
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-phi-prop.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-phi-prop.ll
index aa517cb1589da..e358d18ceb9ba 100644
--- a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-phi-prop.ll
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-phi-prop.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 define i64 @propagate_loop_phis() {
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics.ll
index 27ecb65ad61a4..7da9a99703a20 100644
--- a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics.ll
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics.ll
@@ -21,14 +21,14 @@
 ; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=20 -unroll-max-percent-threshold-boost=200 | FileCheck %s -check-prefix=TEST2
 ; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=20 -unroll-max-percent-threshold-boost=100 | FileCheck %s -check-prefix=TEST3
 
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=10 -unroll-max-percent-threshold-boost=100 | FileCheck %s -check-prefix=TEST1
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=20 -unroll-max-percent-threshold-boost=200 | FileCheck %s -check-prefix=TEST2
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=20 -unroll-max-percent-threshold-boost=100 | FileCheck %s -check-prefix=TEST3
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=10 -unroll-max-percent-threshold-boost=100 | FileCheck %s -check-prefix=TEST1
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=20 -unroll-max-percent-threshold-boost=200 | FileCheck %s -check-prefix=TEST2
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=20 -unroll-max-percent-threshold-boost=100 | FileCheck %s -check-prefix=TEST3
 
 ; Check that these work when the unroller has partial unrolling enabled too.
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=10 -unroll-max-percent-threshold-boost=100 | FileCheck %s -check-prefix=TEST1
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=20 -unroll-max-percent-threshold-boost=200 | FileCheck %s -check-prefix=TEST2
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=20 -unroll-max-percent-threshold-boost=100 | FileCheck %s -check-prefix=TEST3
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=10 -unroll-max-percent-threshold-boost=100 | FileCheck %s -check-prefix=TEST1
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=20 -unroll-max-percent-threshold-boost=200 | FileCheck %s -check-prefix=TEST2
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=20 -unroll-max-percent-threshold-boost=100 | FileCheck %s -check-prefix=TEST3
 
 ; If the absolute threshold is too low, we should not unroll:
 ; TEST1: %array_const_idx = getelementptr inbounds [9 x i32], [9 x i32]* @known_constant, i64 0, i64 %iv
@@ -41,7 +41,7 @@
 
 ; And check that we don't crash when we're not allowed to do any analysis.
 ; RUN: opt < %s -loop-unroll -unroll-max-iteration-count-to-analyze=0 -disable-output
-; RUN: opt < %s -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=0 -disable-output
+; RUN: opt < %s -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -unroll-max-iteration-count-to-analyze=0 -disable-output
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 @known_constant = internal unnamed_addr constant [9 x i32] [i32 0, i32 -1, i32 0, i32 -1, i32 5, i32 -1, i32 0, i32 -1, i32 0], align 16
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-keep-first-exit.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-keep-first-exit.ll
index 682d1b35c1fae..caa59da17490c 100644
--- a/llvm/test/Transforms/LoopUnroll/full-unroll-keep-first-exit.ll
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-keep-first-exit.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S -loop-unroll < %s | FileCheck %s
-; RUN: opt -S -passes='require<opt-remark-emit>,loop(unroll-full)' < %s | FileCheck %s
+; RUN: opt -S -passes='require<opt-remark-emit>,loop(loop-unroll-full)' < %s | FileCheck %s
 
 ; Unroll twice, with first loop exit kept
 ; CHECK-LABEL: @s32_max1
diff --git a/llvm/test/Transforms/LoopUnroll/nonlatchcondbr.ll b/llvm/test/Transforms/LoopUnroll/nonlatchcondbr.ll
index 3b6e909ce7445..b433ebb4d6420 100644
--- a/llvm/test/Transforms/LoopUnroll/nonlatchcondbr.ll
+++ b/llvm/test/Transforms/LoopUnroll/nonlatchcondbr.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -loop-unroll -unroll-runtime -unroll-count=4 -S | FileCheck %s
-; RUN: opt < %s -passes='require<opt-remark-emit>,unroll' -unroll-runtime -unroll-count=4 -S | FileCheck %s
+; RUN: opt < %s -passes='require<opt-remark-emit>,loop-unroll' -unroll-runtime -unroll-count=4 -S | FileCheck %s
 
 ; Check that loop unroll pass correctly handle loops with
 ; single exiting block not the loop header or latch.
diff --git a/llvm/test/Transforms/LoopUnroll/partial-unroll-const-bounds.ll b/llvm/test/Transforms/LoopUnroll/partial-unroll-const-bounds.ll
index 3ccf4329a0ab1..df5430ab2791d 100644
--- a/llvm/test/Transforms/LoopUnroll/partial-unroll-const-bounds.ll
+++ b/llvm/test/Transforms/LoopUnroll/partial-unroll-const-bounds.ll
@@ -1,8 +1,8 @@
 ; RUN: opt < %s -S -unroll-partial-threshold=20 -unroll-threshold=20 -loop-unroll -unroll-allow-partial -unroll-runtime -unroll-allow-remainder -unroll-max-percent-threshold-boost=100 | FileCheck %s
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-partial-threshold=20 -unroll-threshold=20 -unroll-allow-partial -unroll-runtime -unroll-allow-remainder -unroll-max-percent-threshold-boost=100 | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll' -unroll-partial-threshold=20 -unroll-threshold=20 -unroll-allow-partial -unroll-runtime -unroll-allow-remainder -unroll-max-percent-threshold-boost=100 | FileCheck %s
 ;
 ; Also check that the simple unroller doesn't allow the partial unrolling.
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-partial-threshold=20 -unroll-threshold=20 -unroll-allow-partial -unroll-runtime -unroll-allow-remainder -unroll-max-percent-threshold-boost=100 | FileCheck %s --check-prefix=CHECK-NO-UNROLL
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -unroll-partial-threshold=20 -unroll-threshold=20 -unroll-allow-partial -unroll-runtime -unroll-allow-remainder -unroll-max-percent-threshold-boost=100 | FileCheck %s --check-prefix=CHECK-NO-UNROLL
 
 ; The Loop TripCount is 9. However unroll factors 3 or 9 exceed given threshold.
 ; The test checks that we choose a smaller, power-of-two, unroll count and do not give up on unrolling.
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-inner.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-inner.ll
index a83f3bfb93586..7b088b31a54c4 100644
--- a/llvm/test/Transforms/LoopUnroll/peel-loop-inner.ll
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-inner.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll<peeling;no-runtime>,simplify-cfg,instcombine' -unroll-force-peel-count=3 -verify-dom-info | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll<peeling;no-runtime>,simplify-cfg,instcombine' -unroll-force-peel-count=3 -verify-dom-info | FileCheck %s
 
 define void @basic(i32 %K, i32 %N) {
 ; CHECK-LABEL: @basic(
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt-idom-2.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt-idom-2.ll
index 006f2bbb4c81f..29134c0311aad 100644
--- a/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt-idom-2.ll
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt-idom-2.ll
@@ -1,6 +1,6 @@
 ; REQUIRES: asserts
 ; RUN: opt < %s -S -debug-only=loop-unroll -loop-unroll -unroll-runtime -unroll-peel-multi-deopt-exit 2>&1 | FileCheck %s
-; RUN: opt < %s -S -debug-only=loop-unroll -unroll-peel-multi-deopt-exit -passes='require<profile-summary>,function(require<opt-remark-emit>,unroll)' 2>&1 | FileCheck %s
+; RUN: opt < %s -S -debug-only=loop-unroll -unroll-peel-multi-deopt-exit -passes='require<profile-summary>,function(require<opt-remark-emit>,loop-unroll)' 2>&1 | FileCheck %s
 
 ; Regression test for setting the correct idom for exit blocks.
 
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt-idom.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt-idom.ll
index 7b0dcac3ed4d5..184a07151dfa5 100644
--- a/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt-idom.ll
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt-idom.ll
@@ -1,6 +1,6 @@
 ; REQUIRES: asserts
 ; RUN: opt < %s -S -debug-only=loop-unroll -loop-unroll -unroll-runtime -unroll-peel-multi-deopt-exit 2>&1 | FileCheck %s
-; RUN: opt < %s -S -debug-only=loop-unroll -unroll-peel-multi-deopt-exit -passes='require<profile-summary>,function(require<opt-remark-emit>,unroll)' 2>&1 | FileCheck %s
+; RUN: opt < %s -S -debug-only=loop-unroll -unroll-peel-multi-deopt-exit -passes='require<profile-summary>,function(require<opt-remark-emit>,loop-unroll)' 2>&1 | FileCheck %s
 
 ; Regression test for setting the correct idom for exit blocks.
 
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt.ll
index 9c059ce23603f..205a2832bb7ab 100644
--- a/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt.ll
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-pgo-deopt.ll
@@ -1,7 +1,7 @@
 ; REQUIRES: asserts
 ; RUN: opt < %s -S -debug-only=loop-unroll -loop-unroll -unroll-runtime -unroll-peel-multi-deopt-exit 2>&1 | FileCheck %s
-; RUN: opt < %s -S -debug-only=loop-unroll -unroll-peel-multi-deopt-exit -passes='require<profile-summary>,function(require<opt-remark-emit>,unroll)' 2>&1 | FileCheck %s
-; RUN: opt < %s -S -debug-only=loop-unroll -unroll-peel-multi-deopt-exit -passes='require<profile-summary>,function(require<opt-remark-emit>,unroll<no-profile-peeling>)' 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PEEL
+; RUN: opt < %s -S -debug-only=loop-unroll -unroll-peel-multi-deopt-exit -passes='require<profile-summary>,function(require<opt-remark-emit>,loop-unroll)' 2>&1 | FileCheck %s
+; RUN: opt < %s -S -debug-only=loop-unroll -unroll-peel-multi-deopt-exit -passes='require<profile-summary>,function(require<opt-remark-emit>,loop-unroll<no-profile-peeling>)' 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PEEL
 
 ; Make sure we use the profile information correctly to peel-off 3 iterations
 ; from the loop, and update the branch weights for the peeled loop properly.
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-pgo.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-pgo.ll
index c3f6698a3fbd5..38cd6e049e9f3 100644
--- a/llvm/test/Transforms/LoopUnroll/peel-loop-pgo.ll
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-pgo.ll
@@ -1,8 +1,8 @@
 ; RUN: opt < %s -S -debug-only=loop-unroll -loop-unroll 2>&1 | FileCheck %s
-; RUN: opt < %s -S -debug-only=loop-unroll -passes='require<profile-summary>,function(require<opt-remark-emit>,unroll)' 2>&1 | FileCheck %s
+; RUN: opt < %s -S -debug-only=loop-unroll -passes='require<profile-summary>,function(require<opt-remark-emit>,loop-unroll)' 2>&1 | FileCheck %s
 ; Confirm that peeling is disabled if the number of counts required to reach
 ; the hot percentile is above the threshold.
-; RUN: opt < %s -S -profile-summary-huge-working-set-size-threshold=9 -debug-only=loop-unroll -passes='require<profile-summary>,function(require<opt-remark-emit>,unroll)' 2>&1 | FileCheck %s --check-prefix=NOPEEL
+; RUN: opt < %s -S -profile-summary-huge-working-set-size-threshold=9 -debug-only=loop-unroll -passes='require<profile-summary>,function(require<opt-remark-emit>,loop-unroll)' 2>&1 | FileCheck %s --check-prefix=NOPEEL
 ; REQUIRES: asserts
 
 ; Make sure we use the profile information correctly to peel-off 3 iterations
@@ -14,7 +14,7 @@
 ; CHECK-NOT: PEELING
 
 ; Confirm that no peeling occurs when we are performing full unrolling.
-; RUN: opt < %s -S -debug-only=loop-unroll -passes='require<opt-remark-emit>,loop(unroll-full)' 2>&1 | FileCheck %s --check-prefix=NOPEEL
+; RUN: opt < %s -S -debug-only=loop-unroll -passes='require<opt-remark-emit>,loop(loop-unroll-full)' 2>&1 | FileCheck %s --check-prefix=NOPEEL
 ; NOPEEL-NOT: PEELING
 
 ; CHECK-LABEL: @basic
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop.ll b/llvm/test/Transforms/LoopUnroll/peel-loop.ll
index eb3d29cb49436..d9dce0914418e 100644
--- a/llvm/test/Transforms/LoopUnroll/peel-loop.ll
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -S -loop-unroll -unroll-force-peel-count=3 -verify-dom-info -simplifycfg -instcombine | FileCheck %s
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll,simplify-cfg,instcombine' -unroll-force-peel-count=3 -verify-dom-info | FileCheck %s
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll<peeling;no-runtime>,simplify-cfg,instcombine' -unroll-force-peel-count=3 -verify-dom-info | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll,simplify-cfg,instcombine' -unroll-force-peel-count=3 -verify-dom-info | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll<peeling;no-runtime>,simplify-cfg,instcombine' -unroll-force-peel-count=3 -verify-dom-info | FileCheck %s
 
 ; Basic loop peeling - check that we can peel-off the first 3 loop iterations
 ; when explicitly requested.
diff --git a/llvm/test/Transforms/LoopUnroll/revisit.ll b/llvm/test/Transforms/LoopUnroll/revisit.ll
index 6420396e9ff79..8c4f04d10854a 100644
--- a/llvm/test/Transforms/LoopUnroll/revisit.ll
+++ b/llvm/test/Transforms/LoopUnroll/revisit.ll
@@ -4,12 +4,12 @@
 ; current two cases.
 ;
 ; RUN: opt < %s -disable-output -debug-pass-manager 2>&1 \
-; RUN:     -passes='require<opt-remark-emit>,loop(unroll-full)' \
+; RUN:     -passes='require<opt-remark-emit>,loop(loop-unroll-full)' \
 ; RUN:     | FileCheck %s
 ;
 ; Also run in a special mode that visits children.
 ; RUN: opt < %s -disable-output -debug-pass-manager -unroll-revisit-child-loops 2>&1 \
-; RUN:     -passes='require<opt-remark-emit>,loop(unroll-full)' \
+; RUN:     -passes='require<opt-remark-emit>,loop(loop-unroll-full)' \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK,CHECK-CHILDREN
 
 ; Basic test is fully unrolled and we revisit the post-unroll new sibling
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll
index f905355f1a8ab..7be217b06aaed 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll
@@ -1,8 +1,8 @@
 ; RUN: opt < %s -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=true -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -S | FileCheck %s -check-prefix=EPILOG-NO-IC
 ; RUN: opt < %s -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=true -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -instcombine -S | FileCheck %s -check-prefix=EPILOG
-; RUN: opt < %s -loop-unroll -unroll-runtime -unroll-count=2 -unroll-runtime-epilog=true -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -instcombine
+; RUN: opt < %s -loop-unroll -unroll-runtime -unroll-count=2 -unroll-runtime-epilog=true -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -instcombine -S
 ; RUN: opt < %s -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=false -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -instcombine -S | FileCheck %s -check-prefix=PROLOG
-; RUN: opt < %s -loop-unroll -unroll-runtime -unroll-runtime-epilog=false -unroll-count=2 -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -instcombine
+; RUN: opt < %s -loop-unroll -unroll-runtime -unroll-runtime-epilog=false -unroll-count=2 -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -instcombine -S
 
 ; REQUIRES: asserts
 
@@ -23,12 +23,12 @@ define void @test1(i64 %trip, i1 %cond) {
 ; EPILOG-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; EPILOG:  loop_latch.epil:
 ; EPILOG-NEXT:     %epil.iter.sub = add i64 %epil.iter, -1
-; EPILOG-NEXT:     %epil.iter.cmp = icmp eq i64 %epil.iter.sub, 0
-; EPILOG-NEXT:     br i1 %epil.iter.cmp, label %exit2.loopexit.epilog-lcssa, label %loop_header.epil
+; EPILOG-NEXT:     %epil.iter.cmp.not = icmp eq i64 %epil.iter.sub, 0
+; EPILOG-NEXT:     br i1 %epil.iter.cmp.not, label %exit2.loopexit.epilog-lcssa, label %loop_header.epil
 ; EPILOG:  loop_latch.7:
 ; EPILOG-NEXT:     %niter.nsub.7 = add i64 %niter, -8
-; EPILOG-NEXT:     %niter.ncmp.7 = icmp eq i64 %niter.nsub.7, 0
-; EPILOG-NEXT:     br i1 %niter.ncmp.7, label %exit2.loopexit.unr-lcssa.loopexit, label %loop_header
+; EPILOG-NEXT:     %niter.ncmp.7.not = icmp eq i64 %niter.nsub.7, 0
+; EPILOG-NEXT:     br i1 %niter.ncmp.7.not, label %exit2.loopexit.unr-lcssa.loopexit, label %loop_header
 
 ; PROLOG: test1(
 ; PROLOG-NEXT:  entry:
@@ -43,12 +43,12 @@ define void @test1(i64 %trip, i1 %cond) {
 ; PROLOG:       loop_latch.prol:
 ; PROLOG-NEXT:    %iv_next.prol = add i64 %iv.prol, 1
 ; PROLOG-NEXT:    %prol.iter.sub = add i64 %prol.iter, -1
-; PROLOG-NEXT:    %prol.iter.cmp = icmp eq i64 %prol.iter.sub, 0
-; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %loop_header.prol.loopexit.unr-lcssa, label %loop_header.prol
+; PROLOG-NEXT:    %prol.iter.cmp.not = icmp eq i64 %prol.iter.sub, 0
+; PROLOG-NEXT:    br i1 %prol.iter.cmp.not, label %loop_header.prol.loopexit.unr-lcssa, label %loop_header.prol
 ; PROLOG:  loop_latch.7:
 ; PROLOG-NEXT:     %iv_next.7 = add i64 %iv, 8
-; PROLOG-NEXT:     %cmp.7 = icmp eq i64 %iv_next.7, %trip
-; PROLOG-NEXT:     br i1 %cmp.7, label %exit2.loopexit.unr-lcssa, label %loop_header
+; PROLOG-NEXT:     %cmp.7.not = icmp eq i64 %iv_next.7, %trip
+; PROLOG-NEXT:     br i1 %cmp.7.not, label %exit2.loopexit.unr-lcssa, label %loop_header
 entry:
   br label %loop_header
 
@@ -157,8 +157,8 @@ define void @test3(i64 %trip, i64 %add) {
 ; EPILOG:  loop_latch.7:
 ; EPILOG-NEXT:     %sum.next.7 = add i64 %sum.next.6, %add
 ; EPILOG-NEXT:     %niter.nsub.7 = add i64 %niter, -8
-; EPILOG-NEXT:     %niter.ncmp.7 = icmp eq i64 %niter.nsub.7, 0
-; EPILOG-NEXT:     br i1 %niter.ncmp.7, label %exit2.loopexit.unr-lcssa.loopexit, label %loop_header
+; EPILOG-NEXT:     %niter.ncmp.7.not = icmp eq i64 %niter.nsub.7, 0
+; EPILOG-NEXT:     br i1 %niter.ncmp.7.not, label %exit2.loopexit.unr-lcssa.loopexit, label %loop_header
 
 ; PROLOG:  test3(
 ; PROLOG-NEXT:  entry:
@@ -174,8 +174,8 @@ define void @test3(i64 %trip, i64 %add) {
 ; PROLOG:  loop_latch.7:
 ; PROLOG-NEXT:     %iv_next.7 = add nuw nsw i64 %iv, 8
 ; PROLOG-NEXT:     %sum.next.7 = add i64 %sum.next.6, %add
-; PROLOG-NEXT:     %cmp.7 = icmp eq i64 %iv_next.7, %trip
-; PROLOG-NEXT:     br i1 %cmp.7, label %exit2.loopexit.unr-lcssa, label %loop_header
+; PROLOG-NEXT:     %cmp.7.not = icmp eq i64 %iv_next.7, %trip
+; PROLOG-NEXT:     br i1 %cmp.7.not, label %exit2.loopexit.unr-lcssa, label %loop_header
 entry:
   br label %loop_header
 
@@ -375,15 +375,15 @@ define void @unique_exit(i32 %arg) {
 ; PROLOG-NOT: .unr
 ; PROLOG-NOT: .prol
 entry:
-  %tmp = icmp sgt i32 undef, %arg
-  br i1 %tmp, label %preheader, label %returnblock
+  %i = icmp sgt i32 undef, %arg
+  br i1 %i, label %preheader, label %returnblock
 
 preheader:                                 ; preds = %entry
   br label %header
 
 header:                                           ; preds = %preheader, %latch
-  %tmp4 = phi i32 [ %inc, %latch ], [ %arg, %preheader ]
-  %inc = add nsw i32 %tmp4, 1
+  %i4 = phi i32 [ %inc, %latch ], [ %arg, %preheader ]
+  %inc = add nsw i32 %i4, 1
   br i1 true, label %latchExit, label %latch
 
 latch:                                            ; preds = %header
@@ -391,11 +391,11 @@ latch:                                            ; preds = %header
   br i1 %cmp, label %header, label %latchExit
 
 latchExit:                                ; preds = %header, %latch
-  %tmp2.ph = phi i32 [ %tmp4, %header ], [ -1, %latch ]
+  %i2.ph = phi i32 [ %i4, %header ], [ -1, %latch ]
   br label %returnblock
 
 returnblock:                                         ; preds = %latchExit, %entry
-  %tmp2 = phi i32 [ -1, %entry ], [ %tmp2.ph, %latchExit ]
+  %i2 = phi i32 [ -1, %entry ], [ %i2.ph, %latchExit ]
   ret void
 }
 
@@ -531,22 +531,22 @@ define i32 @test7(i32 %arg, i32 %arg1, i32 %arg2) {
 ; EPILOG-NO-IC: loopexit1:
 ; EPILOG-NO-IC-NEXT:   %sext3 = phi i32 [ %sext3.ph, %loopexit1.loopexit ], [ %sext3.ph2, %loopexit1.loopexit1 ]
 bb:
-  %tmp = icmp slt i32 undef, 2
+  %i = icmp slt i32 undef, 2
   %sext = sext i32 undef to i64
   %shft = ashr exact i32 %arg, 16
-  br i1 %tmp, label %loopexit2, label %preheader
+  br i1 %i, label %loopexit2, label %preheader
 
 preheader:                                              ; preds = %bb2
   br label %header
 
 header:                                              ; preds = %latch, %preheader
-  %tmp6 = phi i64 [ 1, %preheader ], [ %add, %latch ]
+  %i6 = phi i64 [ 1, %preheader ], [ %add, %latch ]
   br i1 false, label %loopexit1, label %latch
 
 latch:                                              ; preds = %header
-  %add = add nuw nsw i64 %tmp6, 1
-  %tmp9 = icmp slt i64 %add, %sext
-  br i1 %tmp9, label %header, label %latchexit
+  %add = add nuw nsw i64 %i6, 1
+  %i9 = icmp slt i64 %add, %sext
+  br i1 %i9, label %header, label %latchexit
 
 latchexit:                                             ; preds = %latch
   unreachable
@@ -575,29 +575,29 @@ define void @test8() {
 ; PROLOG: test8(
 ; PROLOG: outerloop:
 ; PROLOG-NEXT: phi i64 [ 3, %bb ], [ 0, %outerloop.loopexit ]
-; PROLOG:      %lcmp.mod = icmp eq i64
-; PROLOG-NEXT: br i1 %lcmp.mod, label %innerH.prol.loopexit, label %innerH.prol.preheader
+; PROLOG:      %lcmp.mod.not = icmp eq i64
+; PROLOG-NEXT: br i1 %lcmp.mod.not, label %innerH.prol.loopexit, label %innerH.prol.preheader
 ; PROLOG: latch.6:
 ; PROLOG-NEXT: br i1 false, label %outerloop.loopexit.loopexit, label %latch.7
 ; PROLOG: latch.7:
-; PROLOG-NEXT: %tmp4.7 = add nuw nsw i64 %tmp3, 8
-; PROLOG-NEXT: %tmp6.7 = icmp ult i64 %tmp3, 92
-; PROLOG-NEXT: br i1 %tmp6.7, label %innerH, label %exit.unr-lcssa
+; PROLOG-NEXT: %i4.7 = add nuw nsw i64 %i3, 8
+; PROLOG-NEXT: %i6.7 = icmp ult i64 %i3, 92
+; PROLOG-NEXT: br i1 %i6.7, label %innerH, label %exit.unr-lcssa
 bb:
   br label %outerloop
 
 outerloop:                                              ; preds = %innerH, %bb
-  %tmp = phi i64 [ 3, %bb ], [ 0, %innerH ]
+  %i = phi i64 [ 3, %bb ], [ 0, %innerH ]
   br label %innerH
 
 innerH:                                              ; preds = %latch, %outerloop
-  %tmp3 = phi i64 [ %tmp4, %latch ], [ %tmp, %outerloop ]
-  %tmp4 = add nuw nsw i64 %tmp3, 1
+  %i3 = phi i64 [ %i4, %latch ], [ %i, %outerloop ]
+  %i4 = add nuw nsw i64 %i3, 1
   br i1 false, label %outerloop, label %latch
 
 latch:                                              ; preds = %innerH
-  %tmp6 = icmp ult i64 %tmp4, 100
-  br i1 %tmp6, label %innerH, label %exit
+  %i6 = icmp ult i64 %i4, 100
+  br i1 %i6, label %innerH, label %exit
 
 exit:                                              ; preds = %latch
   ret void
@@ -613,7 +613,7 @@ define i8 addrspace(1)* @test9(i8* nocapture readonly %arg, i32 %n) {
 ; PROLOG-NEXT: %phi.prol = phi i64 [ 0, %header.prol.preheader ], [ %iv.next.prol, %latch.prol ]
 ; PROLOG: latch.prol:
 ; PROLOG-NOT: trip
-; PROLOG:     br i1 %prol.iter.cmp, label %header.prol.loopexit.unr-lcssa, label %header.prol
+; PROLOG:     br i1 %prol.iter.cmp.not, label %header.prol.loopexit.unr-lcssa, label %header.prol
 bb:
   br label %outerloopHdr
 
@@ -623,21 +623,21 @@ outerloopHdr:                                              ; preds = %outerLatch
   br i1 %outercnd, label %preheader, label %outerLatch
 
 preheader:                                              ; preds = %outerloopHdr
-  %tmp4 = zext i32 0 to i64
+  %i4 = zext i32 0 to i64
   br label %header
 
 header:                                              ; preds = %latch, %preheader
-  %phi = phi i64 [ %tmp4, %preheader ], [ %iv.next, %latch ]
-  %tmp7 = trunc i64 %phi to i32
+  %phi = phi i64 [ %i4, %preheader ], [ %iv.next, %latch ]
+  %i7 = trunc i64 %phi to i32
   br i1 true, label %latch, label %innerexit
 
 innerexit:                                              ; preds = %header
-  %tmp9 = call i8 addrspace(1)* @foo(i32 %trip)
-  ret i8 addrspace(1)* %tmp9
+  %i9 = call i8 addrspace(1)* @foo(i32 %trip)
+  ret i8 addrspace(1)* %i9
 
 latch:                                             ; preds = %header
-  %tmp11 = add nsw i32 %tmp7, 1
-  %innercnd = icmp slt i32 %tmp11, %trip
+  %i11 = add nsw i32 %i7, 1
+  %innercnd = icmp slt i32 %i11, %trip
   %iv.next = add nuw nsw i64 %phi, 1
   br i1 %innercnd, label %header, label %outerLatch
 
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop.ll
index 19072855d2592..98316905e1fef 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-loop.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop.ll
@@ -1,16 +1,16 @@
 ; RUN: opt < %s -S -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=true  | FileCheck %s -check-prefixes=EPILOG,COMMON
 ; RUN: opt < %s -S -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefixes=PROLOG,COMMON
 ;
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-runtime=true -unroll-runtime-epilog=true  | FileCheck %s -check-prefixes=EPILOG,COMMON
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefixes=PROLOG,COMMON
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll' -unroll-runtime=true -unroll-runtime-epilog=true  | FileCheck %s -check-prefixes=EPILOG,COMMON
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll' -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefixes=PROLOG,COMMON
 ;
 ; Restricted versions of unroll (unroll<peeling;noruntime>, unroll-full) should not be doing runtime unrolling
 ; even if it is globally enabled through -unroll-runtime option
 ;
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll<peeling;no-runtime>' -unroll-runtime=true -unroll-runtime-epilog=true  | FileCheck %s -check-prefixes=NOEPILOG,COMMON
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll<peeling;no-runtime>' -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefixes=NOPROLOG,COMMON
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-runtime=true -unroll-runtime-epilog=true  | FileCheck %s -check-prefixes=NOEPILOG,COMMON
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefixes=NOPROLOG,COMMON
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll<peeling;no-runtime>' -unroll-runtime=true -unroll-runtime-epilog=true  | FileCheck %s -check-prefixes=NOEPILOG,COMMON
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll<peeling;no-runtime>' -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefixes=NOPROLOG,COMMON
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -unroll-runtime=true -unroll-runtime-epilog=true  | FileCheck %s -check-prefixes=NOEPILOG,COMMON
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefixes=NOPROLOG,COMMON
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop1.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop1.ll
index 7e4866ced4baf..b56da618d5d67 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-loop1.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop1.ll
@@ -1,8 +1,8 @@
 ; RUN: opt < %s -S -loop-unroll -unroll-runtime -unroll-count=2 -unroll-runtime-epilog=true | FileCheck %s -check-prefix=EPILOG
 ; RUN: opt < %s -S -loop-unroll -unroll-runtime -unroll-count=2 -unroll-runtime-epilog=false | FileCheck %s -check-prefix=PROLOG
 
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-runtime -unroll-count=2 -unroll-runtime-epilog=true | FileCheck %s -check-prefix=EPILOG
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-runtime -unroll-count=2 -unroll-runtime-epilog=false | FileCheck %s -check-prefix=PROLOG
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll' -unroll-runtime -unroll-count=2 -unroll-runtime-epilog=true | FileCheck %s -check-prefix=EPILOG
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll' -unroll-runtime -unroll-count=2 -unroll-runtime-epilog=false | FileCheck %s -check-prefix=PROLOG
 
 ; This tests that setting the unroll count works
 
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop2.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop2.ll
index 5a35d528a785e..45c47e230cc14 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-loop2.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop2.ll
@@ -1,8 +1,8 @@
 ; RUN: opt < %s -S -loop-unroll -unroll-threshold=25 -unroll-partial-threshold=25 -unroll-runtime -unroll-runtime-epilog=true  -unroll-count=8 | FileCheck %s  -check-prefix=EPILOG
 ; RUN: opt < %s -S -loop-unroll -unroll-threshold=25 -unroll-partial-threshold=25 -unroll-runtime -unroll-runtime-epilog=false | FileCheck %s -check-prefix=PROLOG
 
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-threshold=25 -unroll-partial-threshold=25 -unroll-runtime -unroll-runtime-epilog=true  -unroll-count=8 | FileCheck %s  -check-prefix=EPILOG
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-threshold=25 -unroll-partial-threshold=25 -unroll-runtime -unroll-runtime-epilog=false | FileCheck %s -check-prefix=PROLOG
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll' -unroll-threshold=25 -unroll-partial-threshold=25 -unroll-runtime -unroll-runtime-epilog=true  -unroll-count=8 | FileCheck %s  -check-prefix=EPILOG
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll' -unroll-threshold=25 -unroll-partial-threshold=25 -unroll-runtime -unroll-runtime-epilog=false | FileCheck %s -check-prefix=PROLOG
 
 ; Choose a smaller, power-of-two, unroll count if the loop is too large.
 ; This test makes sure we're not unrolling 'odd' counts
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop3.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop3.ll
index 26790bcb95e1a..f3f6a437a55ac 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-loop3.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop3.ll
@@ -1,6 +1,6 @@
 ; REQUIRES: asserts
 ; RUN: opt < %s -disable-output -stats -loop-unroll -unroll-runtime -unroll-partial-threshold=200 -unroll-threshold=400 -info-output-file - | FileCheck %s --check-prefix=STATS
-; RUN: opt < %s -disable-output -stats -passes='require<opt-remark-emit>,unroll' -unroll-runtime -unroll-partial-threshold=200 -unroll-threshold=400 -info-output-file - | FileCheck %s --check-prefix=STATS
+; RUN: opt < %s -disable-output -stats -passes='require<opt-remark-emit>,loop-unroll' -unroll-runtime -unroll-partial-threshold=200 -unroll-threshold=400 -info-output-file - | FileCheck %s --check-prefix=STATS
 
 ; Test that nested loops can be unrolled.  We need to increase threshold to do it
 
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop5.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop5.ll
index ebb9a85600bc5..a6415d654a98d 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-loop5.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop5.ll
@@ -1,8 +1,8 @@
 ; RUN: opt < %s -S -loop-unroll -unroll-runtime=true -unroll-count=16 | FileCheck --check-prefix=UNROLL-16 %s
 ; RUN: opt < %s -S -loop-unroll -unroll-runtime=true -unroll-count=4 | FileCheck --check-prefix=UNROLL-4 %s
 
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-runtime=true -unroll-count=16 | FileCheck --check-prefix=UNROLL-16 %s
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-runtime=true -unroll-count=4 | FileCheck --check-prefix=UNROLL-4 %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll' -unroll-runtime=true -unroll-count=16 | FileCheck --check-prefix=UNROLL-16 %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll' -unroll-runtime=true -unroll-count=4 | FileCheck --check-prefix=UNROLL-4 %s
 
 ; Given that the trip-count of this loop is a 3-bit value, we cannot
 ; safely unroll it with a count of anything more than 8.
diff --git a/llvm/test/Transforms/LoopUnroll/unloop.ll b/llvm/test/Transforms/LoopUnroll/unloop.ll
index 36b79818ff397..f8c21e6c54c0f 100644
--- a/llvm/test/Transforms/LoopUnroll/unloop.ll
+++ b/llvm/test/Transforms/LoopUnroll/unloop.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -S -loop-unroll -verify-loop-info | FileCheck %s
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll,verify<loops>' | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop-unroll,verify<loops>' | FileCheck %s
 ;
 ; Unit tests for LoopInfo::markAsRemoved.
 
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-loop-invalidation.ll b/llvm/test/Transforms/LoopUnroll/unroll-loop-invalidation.ll
index ea79d7164dcec..ec377ffb2235b 100644
--- a/llvm/test/Transforms/LoopUnroll/unroll-loop-invalidation.ll
+++ b/llvm/test/Transforms/LoopUnroll/unroll-loop-invalidation.ll
@@ -3,7 +3,7 @@
 ; analysis is cached, then unroll the loop (deleting it) and make sure that the
 ; next function doesn't get a cache "hit" for this stale analysis result.
 ;
-; RUN: opt -S -passes='loop(require<access-info>),unroll,loop(print-access-info)' -debug-pass-manager < %s 2>&1 | FileCheck %s
+; RUN: opt -S -passes='loop(require<access-info>),loop-unroll,loop(print-access-info)' -debug-pass-manager < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Starting llvm::Function pass manager run.
 ; CHECK: Running pass: FunctionToLoopPassAdaptor
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-opt-attribute.ll b/llvm/test/Transforms/LoopUnroll/unroll-opt-attribute.ll
index 2f0eb756fb64b..e219349fb093e 100644
--- a/llvm/test/Transforms/LoopUnroll/unroll-opt-attribute.ll
+++ b/llvm/test/Transforms/LoopUnroll/unroll-opt-attribute.ll
@@ -1,176 +1,176 @@
-; RUN: opt < %s -S -loop-unroll -unroll-count=4 | FileCheck -check-prefix=CHECK_COUNT4 %s
-; RUN: opt < %s -S -loop-unroll | FileCheck -check-prefix=CHECK_NOCOUNT %s
-; RUN: opt < %s -S -passes='require<profile-summary>,function(unroll)' -pgso | FileCheck -check-prefix=PGSO %s
-; RUN: opt < %s -S -passes='require<profile-summary>,function(unroll)' -pgso=false | FileCheck -check-prefix=NPGSO %s
-
-
-;///////////////////// TEST 1 //////////////////////////////
-
-; This test shows that the loop is unrolled according to the specified
-; unroll factor.
-
-define void @Test1() nounwind {
-entry:
-  br label %loop
-
-loop:
-  %iv = phi i32 [ 0, %entry ], [ %inc, %loop ]
-  %inc = add i32 %iv, 1
-  %exitcnd = icmp uge i32 %inc, 1024
-  br i1 %exitcnd, label %exit, label %loop
-
-exit:
-  ret void
-}
-
-; CHECK_COUNT4-LABEL: @Test1
-; CHECK_COUNT4:      phi
-; CHECK_COUNT4-NEXT: add
-; CHECK_COUNT4-NEXT: add
-; CHECK_COUNT4-NEXT: add
-; CHECK_COUNT4-NEXT: add
-; CHECK_COUNT4-NEXT: icmp
-
-
-;///////////////////// TEST 2 //////////////////////////////
-
-; This test shows that with optnone attribute, the loop is not unrolled
-; even if an unroll factor was specified.
-
-define void @Test2() nounwind optnone noinline {
-entry:
-  br label %loop
-
-loop:
-  %iv = phi i32 [ 0, %entry ], [ %inc, %loop ]
-  %inc = add i32 %iv, 1
-  %exitcnd = icmp uge i32 %inc, 1024
-  br i1 %exitcnd, label %exit, label %loop
-
-exit:
-  ret void
-}
-
-; CHECK_COUNT4-LABEL: @Test2
-; CHECK_COUNT4:      phi
-; CHECK_COUNT4-NEXT: add
-; CHECK_COUNT4-NEXT: icmp
-
-
-;///////////////////// TEST 3 //////////////////////////////
-
-; This test shows that this loop is fully unrolled by default.
-
-@tab = common global [24 x i32] zeroinitializer, align 4
-
-define i32 @Test3() {
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %for.body, %entry
-  %i.05 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
-  %arrayidx = getelementptr inbounds [24 x i32], [24 x i32]* @tab, i32 0, i32 %i.05
-  store i32 %i.05, i32* %arrayidx, align 4
-  %inc = add nuw nsw i32 %i.05, 1
-  %exitcond = icmp eq i32 %inc, 24
-  br i1 %exitcond, label %for.end, label %for.body
-
-for.end:                                          ; preds = %for.body
-  ret i32 42
-}
-
-; CHECK_NOCOUNT-LABEL: @Test3
-; CHECK_NOCOUNT:      store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: store
-; CHECK_NOCOUNT-NEXT: ret
-
-
-;///////////////////// TEST 4 //////////////////////////////
-
-; This test shows that with optsize attribute, this loop is not unrolled.
-
-define i32 @Test4() optsize {
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %for.body, %entry
-  %i.05 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
-  %arrayidx = getelementptr inbounds [24 x i32], [24 x i32]* @tab, i32 0, i32 %i.05
-  store i32 %i.05, i32* %arrayidx, align 4
-  %inc = add nuw nsw i32 %i.05, 1
-  %exitcond = icmp eq i32 %inc, 24
-  br i1 %exitcond, label %for.end, label %for.body
-
-for.end:                                          ; preds = %for.body
-  ret i32 42
-}
-
-; CHECK_NOCOUNT-LABEL: @Test4
-; CHECK_NOCOUNT:      phi
-; CHECK_NOCOUNT:      icmp
-
-;///////////////////// TEST 5 //////////////////////////////
-
-; This test shows that with PGO, this loop is cold and not unrolled.
-
-define i32 @Test5() !prof !14 {
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %for.body, %entry
-  %i.05 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
-  %arrayidx = getelementptr inbounds [24 x i32], [24 x i32]* @tab, i32 0, i32 %i.05
-  store i32 %i.05, i32* %arrayidx, align 4
-  %inc = add nuw nsw i32 %i.05, 1
-  %exitcond = icmp eq i32 %inc, 24
-  br i1 %exitcond, label %for.end, label %for.body
-
-for.end:                                          ; preds = %for.body
-  ret i32 42
-}
-
-; PGSO-LABEL: @Test5
-; PGSO:      phi
-; PGSO:      icmp
-; NPGSO-LABEL: @Test5
-; NPGSO-NOT:      phi
-; NPGSO-NOT:      icmp
-
-!llvm.module.flags = !{!0}
-!0 = !{i32 1, !"ProfileSummary", !1}
-!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
-!2 = !{!"ProfileFormat", !"InstrProf"}
-!3 = !{!"TotalCount", i64 10000}
-!4 = !{!"MaxCount", i64 10}
-!5 = !{!"MaxInternalCount", i64 1}
-!6 = !{!"MaxFunctionCount", i64 1000}
-!7 = !{!"NumCounts", i64 3}
-!8 = !{!"NumFunctions", i64 3}
-!9 = !{!"DetailedSummary", !10}
-!10 = !{!11, !12, !13}
-!11 = !{i32 10000, i64 100, i32 1}
-!12 = !{i32 999000, i64 100, i32 1}
-!13 = !{i32 999999, i64 1, i32 2}
-!14 = !{!"function_entry_count", i64 0}
+; RUN: opt < %s -S -loop-unroll -unroll-count=4 | FileCheck -check-prefix=CHECK_COUNT4 %s
+; RUN: opt < %s -S -loop-unroll | FileCheck -check-prefix=CHECK_NOCOUNT %s
+; RUN: opt < %s -S -passes='require<profile-summary>,function(loop-unroll)' -pgso | FileCheck -check-prefix=PGSO %s
+; RUN: opt < %s -S -passes='require<profile-summary>,function(loop-unroll)' -pgso=false | FileCheck -check-prefix=NPGSO %s
+
+
+;///////////////////// TEST 1 //////////////////////////////
+
+; This test shows that the loop is unrolled according to the specified
+; unroll factor.
+
+define void @Test1() nounwind {
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  %inc = add i32 %iv, 1
+  %exitcnd = icmp uge i32 %inc, 1024
+  br i1 %exitcnd, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; CHECK_COUNT4-LABEL: @Test1
+; CHECK_COUNT4:      phi
+; CHECK_COUNT4-NEXT: add
+; CHECK_COUNT4-NEXT: add
+; CHECK_COUNT4-NEXT: add
+; CHECK_COUNT4-NEXT: add
+; CHECK_COUNT4-NEXT: icmp
+
+
+;///////////////////// TEST 2 //////////////////////////////
+
+; This test shows that with optnone attribute, the loop is not unrolled
+; even if an unroll factor was specified.
+
+define void @Test2() nounwind optnone noinline {
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  %inc = add i32 %iv, 1
+  %exitcnd = icmp uge i32 %inc, 1024
+  br i1 %exitcnd, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; CHECK_COUNT4-LABEL: @Test2
+; CHECK_COUNT4:      phi
+; CHECK_COUNT4-NEXT: add
+; CHECK_COUNT4-NEXT: icmp
+
+
+;///////////////////// TEST 3 //////////////////////////////
+
+; This test shows that this loop is fully unrolled by default.
+
+@tab = common global [24 x i32] zeroinitializer, align 4
+
+define i32 @Test3() {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.05 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [24 x i32], [24 x i32]* @tab, i32 0, i32 %i.05
+  store i32 %i.05, i32* %arrayidx, align 4
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 24
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 42
+}
+
+; CHECK_NOCOUNT-LABEL: @Test3
+; CHECK_NOCOUNT:      store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: ret
+
+
+;///////////////////// TEST 4 //////////////////////////////
+
+; This test shows that with optsize attribute, this loop is not unrolled.
+
+define i32 @Test4() optsize {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.05 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [24 x i32], [24 x i32]* @tab, i32 0, i32 %i.05
+  store i32 %i.05, i32* %arrayidx, align 4
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 24
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 42
+}
+
+; CHECK_NOCOUNT-LABEL: @Test4
+; CHECK_NOCOUNT:      phi
+; CHECK_NOCOUNT:      icmp
+
+;///////////////////// TEST 5 //////////////////////////////
+
+; This test shows that with PGO, this loop is cold and not unrolled.
+
+define i32 @Test5() !prof !14 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.05 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [24 x i32], [24 x i32]* @tab, i32 0, i32 %i.05
+  store i32 %i.05, i32* %arrayidx, align 4
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 24
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 42
+}
+
+; PGSO-LABEL: @Test5
+; PGSO:      phi
+; PGSO:      icmp
+; NPGSO-LABEL: @Test5
+; NPGSO-NOT:      phi
+; NPGSO-NOT:      icmp
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/Transforms/LoopUnroll/update-loop-info-in-subloops.ll b/llvm/test/Transforms/LoopUnroll/update-loop-info-in-subloops.ll
index 4718e0500a605..c6a83b4d62aa3 100644
--- a/llvm/test/Transforms/LoopUnroll/update-loop-info-in-subloops.ll
+++ b/llvm/test/Transforms/LoopUnroll/update-loop-info-in-subloops.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S < %s -loop-unroll -block-freq | FileCheck %s
-; RUN: opt -S < %s -passes='require<opt-remark-emit>,unroll,require<block-freq>' | FileCheck %s
+; RUN: opt -S < %s -passes='require<opt-remark-emit>,loop-unroll,require<block-freq>' | FileCheck %s
 ; Crasher from PR20987.
 
 ; CHECK: define void @update_loop_info_in_subloops
diff --git a/llvm/test/Transforms/LoopUnroll/wrong_assert_in_peeling.ll b/llvm/test/Transforms/LoopUnroll/wrong_assert_in_peeling.ll
index af0c6aeb7dbde..c618b290ed26d 100644
--- a/llvm/test/Transforms/LoopUnroll/wrong_assert_in_peeling.ll
+++ b/llvm/test/Transforms/LoopUnroll/wrong_assert_in_peeling.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S < %s -loop-unroll | FileCheck %s
-; RUN: opt -S < %s -passes=unroll | FileCheck %s
+; RUN: opt -S < %s -passes=loop-unroll | FileCheck %s
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128-ni:1-p2:32:8:8:32-ni:2"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/dependencies.ll b/llvm/test/Transforms/LoopUnrollAndJam/dependencies.ll
index e1ab0ea6cd684..ec3ec1e874e11 100644
--- a/llvm/test/Transforms/LoopUnrollAndJam/dependencies.ll
+++ b/llvm/test/Transforms/LoopUnrollAndJam/dependencies.ll
@@ -1,5 +1,5 @@
-; RUN: opt -basicaa -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 < %s -S | FileCheck %s
-; RUN: opt -aa-pipeline=basic-aa -passes='unroll-and-jam' -allow-unroll-and-jam -unroll-and-jam-count=4 < %s -S | FileCheck %s
+; RUN: opt -basic-aa -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='loop-unroll-and-jam' -allow-unroll-and-jam -unroll-and-jam-count=4 < %s -S | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/dependencies_multidims.ll b/llvm/test/Transforms/LoopUnrollAndJam/dependencies_multidims.ll
index cb422dd4730c1..f8f33e975981c 100644
--- a/llvm/test/Transforms/LoopUnrollAndJam/dependencies_multidims.ll
+++ b/llvm/test/Transforms/LoopUnrollAndJam/dependencies_multidims.ll
@@ -1,5 +1,5 @@
-; RUN: opt -da-disable-delinearization-checks -basicaa -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 < %s -S | FileCheck %s
-; RUN: opt -da-disable-delinearization-checks -aa-pipeline=basic-aa -passes='unroll-and-jam' -allow-unroll-and-jam -unroll-and-jam-count=4 < %s -S | FileCheck %s
+; RUN: opt -da-disable-delinearization-checks -basic-aa -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 < %s -S | FileCheck %s
+; RUN: opt -da-disable-delinearization-checks -aa-pipeline=basic-aa -passes='loop-unroll-and-jam' -allow-unroll-and-jam -unroll-and-jam-count=4 < %s -S | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/disable.ll b/llvm/test/Transforms/LoopUnrollAndJam/disable.ll
index 6e879896c55fc..11906441a5d99 100644
--- a/llvm/test/Transforms/LoopUnrollAndJam/disable.ll
+++ b/llvm/test/Transforms/LoopUnrollAndJam/disable.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 -pass-remarks=loop-unroll-and-jam < %s -S 2>&1 | FileCheck %s
-; RUN: opt -passes='unroll-and-jam' -allow-unroll-and-jam -unroll-and-jam-count=4 -pass-remarks=loop-unroll-and-jam < %s -S 2>&1 | FileCheck %s
+; RUN: opt -passes='loop-unroll-and-jam' -allow-unroll-and-jam -unroll-and-jam-count=4 -pass-remarks=loop-unroll-and-jam < %s -S 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/followup.ll b/llvm/test/Transforms/LoopUnrollAndJam/followup.ll
index 1c0975b1ce888..ca1c218f7c0f7 100644
--- a/llvm/test/Transforms/LoopUnrollAndJam/followup.ll
+++ b/llvm/test/Transforms/LoopUnrollAndJam/followup.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -tbaa -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 -unroll-remainder < %s -S | FileCheck %s
+; RUN: opt -basic-aa -tbaa -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 -unroll-remainder < %s -S | FileCheck %s
 ;
 ; Check that followup attributes are set in the new loops.
 ;
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/innerloop.ll b/llvm/test/Transforms/LoopUnrollAndJam/innerloop.ll
index d84260650c67d..79c32c90174eb 100644
--- a/llvm/test/Transforms/LoopUnrollAndJam/innerloop.ll
+++ b/llvm/test/Transforms/LoopUnrollAndJam/innerloop.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -loop-unroll-and-jam -allow-unroll-and-jam -verify-loop-info < %s -S | FileCheck %s
-; RUN: opt -passes='unroll-and-jam,verify<loops>' -allow-unroll-and-jam < %s -S | FileCheck %s
+; RUN: opt -passes='loop-unroll-and-jam,verify<loops>' -allow-unroll-and-jam < %s -S | FileCheck %s
 
 ; Check that the newly created loops to not fail to be added to LI
 ; This test deliberately disables UnJ on the middle loop, performing it instead on the
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll b/llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll
index f2c12702f9e89..91843dbd71c02 100644
--- a/llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll
+++ b/llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -loop-unroll-and-jam -allow-unroll-and-jam -unroll-runtime -unroll-partial-threshold=60 < %s -S | FileCheck %s
-; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='unroll-and-jam' -allow-unroll-and-jam -unroll-runtime -unroll-partial-threshold=60 < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='loop-unroll-and-jam' -allow-unroll-and-jam -unroll-runtime -unroll-partial-threshold=60 < %s -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll b/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll
index 7580b50f8703b..82f7bdf8ffce2 100644
--- a/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll
+++ b/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -basicaa -tbaa -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 -unroll-remainder < %s -S | FileCheck %s
-; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='unroll-and-jam' -allow-unroll-and-jam -unroll-and-jam-count=4 -unroll-remainder < %s -S | FileCheck %s
+; RUN: opt -basic-aa -tbaa -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 -unroll-remainder < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='loop-unroll-and-jam' -allow-unroll-and-jam -unroll-and-jam-count=4 -unroll-remainder < %s -S | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Transforms/LoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll b/llvm/test/Transforms/LoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
index 2c1847a545b2b..67742fcab46ae 100644
--- a/llvm/test/Transforms/LoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
+++ b/llvm/test/Transforms/LoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -instcombine -inline -functionattrs -licm -loop-unswitch -gvn -verify
+; RUN: opt < %s -basic-aa -instcombine -inline -functionattrs -licm -loop-unswitch -gvn -verify
 ; PR12573
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.7.0"
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll b/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll
index 3a7bbbf795aab..eb12803a3447c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll
@@ -7,12 +7,12 @@ target triple = "aarch64--linux-gnu"
 
 ; This test checks that we correctly compute the scalarized operands for a
 ; user-specified vectorization factor when interleaving is disabled. We use the
-; "optsize" attribute to disable all interleaving calculations.  A cost of 5
+; "optsize" attribute to disable all interleaving calculations.  A cost of 4
 ; for %tmp4 indicates that we would scalarize it's operand (%tmp3), giving
 ; %tmp4 a lower scalarization overhead.
 ;
 ; COST-LABEL:  predicated_udiv_scalarized_operand
-; COST:        LV: Found an estimated cost of 5 for VF 2 For instruction: %tmp4 = udiv i64 %tmp2, %tmp3
+; COST:        LV: Found an estimated cost of 4 for VF 2 For instruction: %tmp4 = udiv i64 %tmp2, %tmp3
 ;
 ; CHECK-LABEL: @predicated_udiv_scalarized_operand(
 ; CHECK:       vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll b/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll
index 877d8a844305b..80d2e282176ad 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll
@@ -8,7 +8,7 @@
 ; Check scalar cost for extractvalue. The constant and loop invariant operands are free,
 ; leaving cost 3 for scalarizing the result + 2 for executing the op with VF 2.
 
-; CM: LV: Scalar loop costs: 9.
+; CM: LV: Scalar loop costs: 7.
 ; CM: LV: Found an estimated cost of 5 for VF 2 For instruction:   %a = extractvalue { i64, i64 } %sv, 0
 ; CM-NEXT: LV: Found an estimated cost of 5 for VF 2 For instruction:   %b = extractvalue { i64, i64 } %sv, 1
 
@@ -57,7 +57,7 @@ exit:
 ; Similar to the test case above, but checks getVectorCallCost as well.
 declare float @pow(float, float) readnone nounwind
 
-; CM: LV: Scalar loop costs: 18.
+; CM: LV: Scalar loop costs: 16.
 ; CM: LV: Found an estimated cost of 5 for VF 2 For instruction:   %a = extractvalue { float, float } %sv, 0
 ; CM-NEXT: LV: Found an estimated cost of 5 for VF 2 For instruction:   %b = extractvalue { float, float } %sv, 1
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll b/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
index 1149afe7b9f4a..619cb4f4ab7a3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S < %s -basicaa -loop-vectorize -force-vector-interleave=1 2>&1 | FileCheck %s
+; RUN: opt -S < %s -basic-aa -loop-vectorize -force-vector-interleave=1 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64"
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
index 57a3dfdbc848f..b0ebb4edf2ad8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
@@ -19,7 +19,7 @@ target triple = "aarch64--linux-gnu"
 ;   (udiv(2) + extractelement(6) + insertelement(3)) / 2 = 5
 ;
 ; CHECK: Scalarizing and predicating: %tmp4 = udiv i32 %tmp2, %tmp3
-; CHECK: Found an estimated cost of 6 for VF 2 For instruction: %tmp4 = udiv i32 %tmp2, %tmp3
+; CHECK: Found an estimated cost of 5 for VF 2 For instruction: %tmp4 = udiv i32 %tmp2, %tmp3
 ;
 define i32 @predicated_udiv(i32* %a, i32* %b, i1 %c, i64 %n) {
 entry:
@@ -101,7 +101,7 @@ for.end:
 ; CHECK: Scalarizing: %tmp3 = add nsw i32 %tmp2, %x
 ; CHECK: Scalarizing and predicating: %tmp4 = udiv i32 %tmp2, %tmp3
 ; CHECK: Found an estimated cost of 2 for VF 2 For instruction: %tmp3 = add nsw i32 %tmp2, %x
-; CHECK: Found an estimated cost of 5 for VF 2 For instruction: %tmp4 = udiv i32 %tmp2, %tmp3
+; CHECK: Found an estimated cost of 4 for VF 2 For instruction: %tmp4 = udiv i32 %tmp2, %tmp3
 ;
 define i32 @predicated_udiv_scalarized_operand(i32* %a, i1 %c, i32 %x, i64 %n) {
 entry:
@@ -198,8 +198,8 @@ for.end:
 ; CHECK:     Scalarizing: %tmp5 = sub i32 %tmp4, %x
 ; CHECK:     Scalarizing and predicating: store i32 %tmp5, i32* %tmp0, align 4
 ; CHECK:     Found an estimated cost of 1 for VF 2 For instruction: %tmp2 = add i32 %tmp1, %x
-; CHECK:     Found an estimated cost of 6 for VF 2 For instruction: %tmp3 = sdiv i32 %tmp1, %tmp2
-; CHECK:     Found an estimated cost of 6 for VF 2 For instruction: %tmp4 = udiv i32 %tmp3, %tmp2
+; CHECK:     Found an estimated cost of 5 for VF 2 For instruction: %tmp3 = sdiv i32 %tmp1, %tmp2
+; CHECK:     Found an estimated cost of 5 for VF 2 For instruction: %tmp4 = udiv i32 %tmp3, %tmp2
 ; CHECK:     Found an estimated cost of 2 for VF 2 For instruction: %tmp5 = sub i32 %tmp4, %x
 ; CHECK:     Found an estimated cost of 2 for VF 2 For instruction: store i32 %tmp5, i32* %tmp0, align 4
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll b/llvm/test/Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll
index 0c2502df6f0a4..ff3e03c7bad42 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll
@@ -562,8 +562,6 @@ for.body:
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
 }
 
-; TODO: this fpext could be allowed, but we don't lower it very efficiently yet,
-; so reject this for now.
 define void @fpext_allowed(float* noalias nocapture %A, half* noalias nocapture readonly %B, float* noalias nocapture readonly %C) #0 {
 ; CHECK-LABEL:        fpext_allowed(
 ; PREFER-FOLDING:     vector.body:
@@ -591,8 +589,6 @@ for.body:
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
 }
 
-; TODO: this fptrunc could be allowed, but we don't lower it very efficiently yet,
-; so reject this for now.
 define void @fptrunc_allowed(half* noalias nocapture %A, float* noalias nocapture readonly %B, float* noalias nocapture readonly %C) #0 {
 ; CHECK-LABEL:        fptrunc_allowed(
 ; PREFER-FOLDING:     vector.body:
@@ -622,10 +618,10 @@ for.body:
 
 define void @fptrunc_not_allowed(float* noalias nocapture %A, float* noalias nocapture readonly %B, float* noalias nocapture readonly %C, half* noalias nocapture %D) #0 {
 ; CHECK-LABEL:        fptrunc_not_allowed(
-; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING-NOT: vector.body:
 ; PREFER-FOLDING-NOT: llvm.masked.load
 ; PREFER-FOLDING-NOT: llvm.masked.store
-; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %vector.body
+; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %for.body
 entry:
   br label %for.body
 
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll
index 3491e08bbaa2c..9fdfb6f90e7bf 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -loop-vectorize < %s | FileCheck %s
+; RUN: opt -S -basic-aa -loop-vectorize < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/stride-vectorization.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/stride-vectorization.ll
index f6f2609e8fbe0..42eefcb53787f 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/stride-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/stride-vectorization.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -loop-vectorize < %s | FileCheck %s
+; RUN: opt -S -basic-aa -loop-vectorize < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/max-mstore.ll b/llvm/test/Transforms/LoopVectorize/X86/max-mstore.ll
index a9ac04d456068..3195c6bac495f 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/max-mstore.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/max-mstore.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-vectorize -force-vector-interleave=1 -S -mcpu=core-avx2
+; RUN: opt -basic-aa -loop-vectorize -force-vector-interleave=1 -S -mcpu=core-avx2
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/optsize.ll b/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
index ad72f90d08522..0bbb62379c2c0 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
@@ -218,38 +218,35 @@ for.end:                                          ; preds = %for.body
 attributes #1 = { minsize }
 
 
-; We can't vectorize this one because we version for stride==1; even having TC
-; a multiple of VF.
+; We can vectorize this one by refraining from versioning for stride==1.
 define void @scev4stride1(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %k) #2 {
 ; CHECK-LABEL: @scev4stride1(
 ; CHECK-NEXT:  for.body.preheader:
-; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK:       for.body:
-; CHECK-NEXT:    [[I_07:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ]
-; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[I_07]], [[K:%.*]]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i32 [[MUL]]
-; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i32 [[I_07]]
-; CHECK-NEXT:    store i32 [[TMP0]], i32* [[ARRAYIDX1]], align 4
-; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_07]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 256
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <64 x i32> undef, i32 [[K:%.*]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <64 x i32> [[BROADCAST_SPLATINSERT]], <64 x i32> undef, <64 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 256, 256
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
 ; CHECK:       for.end.loopexit:
 ; CHECK-NEXT:    ret void
 ;
 ; AUTOVF-LABEL: @scev4stride1(
 ; AUTOVF-NEXT:  for.body.preheader:
-; AUTOVF-NEXT:    br label [[FOR_BODY:%.*]]
-; AUTOVF:       for.body:
-; AUTOVF-NEXT:    [[I_07:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ]
-; AUTOVF-NEXT:    [[MUL:%.*]] = mul nsw i32 [[I_07]], [[K:%.*]]
-; AUTOVF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i32 [[MUL]]
-; AUTOVF-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
-; AUTOVF-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i32 [[I_07]]
-; AUTOVF-NEXT:    store i32 [[TMP0]], i32* [[ARRAYIDX1]], align 4
-; AUTOVF-NEXT:    [[INC]] = add nuw nsw i32 [[I_07]], 1
-; AUTOVF-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 256
-; AUTOVF-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; AUTOVF-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; AUTOVF:       vector.ph:
+; AUTOVF-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> undef, i32 [[K:%.*]], i32 0
+; AUTOVF-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> undef, <8 x i32> zeroinitializer
+; AUTOVF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; AUTOVF:       vector.body:
+; AUTOVF:       middle.block:
+; AUTOVF-NEXT:    [[CMP_N:%.*]] = icmp eq i32 256, 256
+; AUTOVF-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; AUTOVF:       scalar.ph:
 ; AUTOVF:       for.end.loopexit:
 ; AUTOVF-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll b/llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll
index 6393002d50716..4d5ca19a71168 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -loop-vectorize -mcpu=prescott -disable-basicaa < %s | FileCheck %s
+; RUN: opt -S -loop-vectorize -mcpu=prescott -disable-basic-aa < %s | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S128"
 target triple = "i386-apple-darwin"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
index 23fcf8e8baf7b..af5820021275e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -loop-vectorize-with-block-frequency -dce -instcombine -S | FileCheck %s
+; RUN: opt < %s -basic-aa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -loop-vectorize-with-block-frequency -dce -instcombine -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll b/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll
index 967dd3f2c26b7..8852513cc3db8 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -loop-vectorize -S | FileCheck %s --check-prefixes=CHECK,DEFAULT
-; RUN: opt < %s -loop-vectorize -prefer-predicate-over-epilog -S | FileCheck %s --check-prefixes=CHECK,PREDFLAG
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+; RUN: opt < %s -loop-vectorize -prefer-predicate-over-epilog -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -74,141 +74,56 @@ for.body:
   br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !6
 }
 
+; Marking function as optsize turns tail folding on, as if explicit tail folding
+; flag was enabled.
 define dso_local void @tail_folding_disabled(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) local_unnamed_addr #0 {
-; DEFAULT-LABEL: @tail_folding_disabled(
-; DEFAULT-NEXT:  entry:
-; DEFAULT-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; DEFAULT:       vector.ph:
-; DEFAULT-NEXT:    br label [[VECTOR_BODY:%.*]]
-; DEFAULT:       vector.body:
-; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; DEFAULT-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; DEFAULT-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 8
-; DEFAULT-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 16
-; DEFAULT-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 24
-; DEFAULT-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP0]]
-; DEFAULT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP1]]
-; DEFAULT-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP2]]
-; DEFAULT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP3]]
-; DEFAULT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0
-; DEFAULT-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <8 x i32>*
-; DEFAULT-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, <8 x i32>* [[TMP9]], align 4
-; DEFAULT-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 8
-; DEFAULT-NEXT:    [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <8 x i32>*
-; DEFAULT-NEXT:    [[WIDE_LOAD1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP11]], align 4
-; DEFAULT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 16
-; DEFAULT-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <8 x i32>*
-; DEFAULT-NEXT:    [[WIDE_LOAD2:%.*]] = load <8 x i32>, <8 x i32>* [[TMP13]], align 4
-; DEFAULT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 24
-; DEFAULT-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <8 x i32>*
-; DEFAULT-NEXT:    [[WIDE_LOAD3:%.*]] = load <8 x i32>, <8 x i32>* [[TMP15]], align 4
-; DEFAULT-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 [[TMP0]]
-; DEFAULT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[TMP1]]
-; DEFAULT-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[TMP2]]
-; DEFAULT-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[TMP3]]
-; DEFAULT-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TMP16]], i32 0
-; DEFAULT-NEXT:    [[TMP21:%.*]] = bitcast i32* [[TMP20]] to <8 x i32>*
-; DEFAULT-NEXT:    [[WIDE_LOAD4:%.*]] = load <8 x i32>, <8 x i32>* [[TMP21]], align 4
-; DEFAULT-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[TMP16]], i32 8
-; DEFAULT-NEXT:    [[TMP23:%.*]] = bitcast i32* [[TMP22]] to <8 x i32>*
-; DEFAULT-NEXT:    [[WIDE_LOAD5:%.*]] = load <8 x i32>, <8 x i32>* [[TMP23]], align 4
-; DEFAULT-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[TMP16]], i32 16
-; DEFAULT-NEXT:    [[TMP25:%.*]] = bitcast i32* [[TMP24]] to <8 x i32>*
-; DEFAULT-NEXT:    [[WIDE_LOAD6:%.*]] = load <8 x i32>, <8 x i32>* [[TMP25]], align 4
-; DEFAULT-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[TMP16]], i32 24
-; DEFAULT-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <8 x i32>*
-; DEFAULT-NEXT:    [[WIDE_LOAD7:%.*]] = load <8 x i32>, <8 x i32>* [[TMP27]], align 4
-; DEFAULT-NEXT:    [[TMP28:%.*]] = add nsw <8 x i32> [[WIDE_LOAD4]], [[WIDE_LOAD]]
-; DEFAULT-NEXT:    [[TMP29:%.*]] = add nsw <8 x i32> [[WIDE_LOAD5]], [[WIDE_LOAD1]]
-; DEFAULT-NEXT:    [[TMP30:%.*]] = add nsw <8 x i32> [[WIDE_LOAD6]], [[WIDE_LOAD2]]
-; DEFAULT-NEXT:    [[TMP31:%.*]] = add nsw <8 x i32> [[WIDE_LOAD7]], [[WIDE_LOAD3]]
-; DEFAULT-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP0]]
-; DEFAULT-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP1]]
-; DEFAULT-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP2]]
-; DEFAULT-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP3]]
-; DEFAULT-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[TMP32]], i32 0
-; DEFAULT-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <8 x i32>*
-; DEFAULT-NEXT:    store <8 x i32> [[TMP28]], <8 x i32>* [[TMP37]], align 4
-; DEFAULT-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[TMP32]], i32 8
-; DEFAULT-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <8 x i32>*
-; DEFAULT-NEXT:    store <8 x i32> [[TMP29]], <8 x i32>* [[TMP39]], align 4
-; DEFAULT-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i32, i32* [[TMP32]], i32 16
-; DEFAULT-NEXT:    [[TMP41:%.*]] = bitcast i32* [[TMP40]] to <8 x i32>*
-; DEFAULT-NEXT:    store <8 x i32> [[TMP30]], <8 x i32>* [[TMP41]], align 4
-; DEFAULT-NEXT:    [[TMP42:%.*]] = getelementptr inbounds i32, i32* [[TMP32]], i32 24
-; DEFAULT-NEXT:    [[TMP43:%.*]] = bitcast i32* [[TMP42]] to <8 x i32>*
-; DEFAULT-NEXT:    store <8 x i32> [[TMP31]], <8 x i32>* [[TMP43]], align 4
-; DEFAULT-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
-; DEFAULT-NEXT:    [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], 416
-; DEFAULT-NEXT:    br i1 [[TMP44]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !4
-; DEFAULT:       middle.block:
-; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 430, 416
-; DEFAULT-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
-; DEFAULT:       scalar.ph:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 416, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; DEFAULT-NEXT:    br label [[FOR_BODY:%.*]]
-; DEFAULT:       for.cond.cleanup:
-; DEFAULT-NEXT:    ret void
-; DEFAULT:       for.body:
-; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; DEFAULT-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
-; DEFAULT-NEXT:    [[TMP45:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
-; DEFAULT-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]]
-; DEFAULT-NEXT:    [[TMP46:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
-; DEFAULT-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP46]], [[TMP45]]
-; DEFAULT-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
-; DEFAULT-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX4]], align 4
-; DEFAULT-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; DEFAULT-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 430
-; DEFAULT-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop !5
-;
-; PREDFLAG-LABEL: @tail_folding_disabled(
-; PREDFLAG-NEXT:  entry:
-; PREDFLAG-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; PREDFLAG:       vector.ph:
-; PREDFLAG-NEXT:    br label [[VECTOR_BODY:%.*]]
-; PREDFLAG:       vector.body:
-; PREDFLAG-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; PREDFLAG-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> undef, i64 [[INDEX]], i32 0
-; PREDFLAG-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> undef, <8 x i32> zeroinitializer
-; PREDFLAG-NEXT:    [[INDUCTION:%.*]] = add <8 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
-; PREDFLAG-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; PREDFLAG-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP0]]
-; PREDFLAG-NEXT:    [[TMP2:%.*]] = icmp ule <8 x i64> [[INDUCTION]], <i64 429, i64 429, i64 429, i64 429, i64 429, i64 429, i64 429, i64 429>
-; PREDFLAG-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i32 0
-; PREDFLAG-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <8 x i32>*
-; PREDFLAG-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP4]], i32 4, <8 x i1> [[TMP2]], <8 x i32> undef)
-; PREDFLAG-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 [[TMP0]]
-; PREDFLAG-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i32 0
-; PREDFLAG-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>*
-; PREDFLAG-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP7]], i32 4, <8 x i1> [[TMP2]], <8 x i32> undef)
-; PREDFLAG-NEXT:    [[TMP8:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD1]], [[WIDE_MASKED_LOAD]]
-; PREDFLAG-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP0]]
-; PREDFLAG-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0
-; PREDFLAG-NEXT:    [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <8 x i32>*
-; PREDFLAG-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[TMP8]], <8 x i32>* [[TMP11]], i32 4, <8 x i1> [[TMP2]])
-; PREDFLAG-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
-; PREDFLAG-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 432
-; PREDFLAG-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !4
-; PREDFLAG:       middle.block:
-; PREDFLAG-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
-; PREDFLAG:       scalar.ph:
-; PREDFLAG-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 432, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; PREDFLAG-NEXT:    br label [[FOR_BODY:%.*]]
-; PREDFLAG:       for.cond.cleanup:
-; PREDFLAG-NEXT:    ret void
-; PREDFLAG:       for.body:
-; PREDFLAG-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; PREDFLAG-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
-; PREDFLAG-NEXT:    [[TMP13:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
-; PREDFLAG-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]]
-; PREDFLAG-NEXT:    [[TMP14:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
-; PREDFLAG-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP14]], [[TMP13]]
-; PREDFLAG-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
-; PREDFLAG-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX4]], align 4
-; PREDFLAG-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; PREDFLAG-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 430
-; PREDFLAG-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop !5
+; CHECK-LABEL: @tail_folding_disabled(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> undef, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <8 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule <8 x i64> [[INDUCTION]], <i64 429, i64 429, i64 429, i64 429, i64 429, i64 429, i64 429, i64 429>
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <8 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP4]], i32 4, <8 x i1> [[TMP2]], <8 x i32> undef)
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP7]], i32 4, <8 x i1> [[TMP2]], <8 x i32> undef)
+; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD1]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <8 x i32>*
+; CHECK-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[TMP8]], <8 x i32>* [[TMP11]], i32 4, <8 x i1> [[TMP2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 432
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !4
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 432, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP14]], [[TMP13]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 430
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop !5
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/uniform_load.ll b/llvm/test/Transforms/LoopVectorize/X86/uniform_load.ll
index e71292265c2a0..1dbfa6dcd8841 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/uniform_load.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/uniform_load.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-vectorize -S -mcpu=core-avx2 < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-vectorize -S -mcpu=core-avx2 < %s | FileCheck %s
 
 ;float inc = 0.5;
 ;void foo(float *A, unsigned N) {
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vector_ptr_load_store.ll b/llvm/test/Transforms/LoopVectorize/X86/vector_ptr_load_store.ll
index cca829b9457e0..acb5d6839f5ee 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vector_ptr_load_store.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vector_ptr_load_store.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-vectorize -mcpu=corei7-avx -debug -S < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa -loop-vectorize -mcpu=corei7-avx -debug -S < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorize-only-for-real.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorize-only-for-real.ll
index d1473552c98af..10b82d1469636 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vectorize-only-for-real.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorize-only-for-real.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -loop-vectorize < %s | FileCheck %s
+; RUN: opt -S -basic-aa -loop-vectorize < %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
diff --git a/llvm/test/Transforms/LoopVectorize/calloc.ll b/llvm/test/Transforms/LoopVectorize/calloc.ll
index 14fae33589b4a..4543fa237281f 100644
--- a/llvm/test/Transforms/LoopVectorize/calloc.ll
+++ b/llvm/test/Transforms/LoopVectorize/calloc.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s  -basicaa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+; RUN: opt < %s  -basic-aa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopVectorize/gcc-examples.ll b/llvm/test/Transforms/LoopVectorize/gcc-examples.ll
index 96a266dd5ba5f..4bf3787dcb407 100644
--- a/llvm/test/Transforms/LoopVectorize/gcc-examples.ll
+++ b/llvm/test/Transforms/LoopVectorize/gcc-examples.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s  -basicaa -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -dce -instcombine -S | FileCheck %s
-; RUN: opt < %s  -basicaa -loop-vectorize -force-vector-width=4 -force-vector-interleave=4 -dce -instcombine -S | FileCheck %s -check-prefix=UNROLL
+; RUN: opt < %s  -basic-aa -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -dce -instcombine -S | FileCheck %s
+; RUN: opt < %s  -basic-aa -loop-vectorize -force-vector-width=4 -force-vector-interleave=4 -dce -instcombine -S | FileCheck %s -check-prefix=UNROLL
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopVectorize/multiple-address-spaces.ll b/llvm/test/Transforms/LoopVectorize/multiple-address-spaces.ll
index 84e533ee71e5c..c083feab8b7c2 100644
--- a/llvm/test/Transforms/LoopVectorize/multiple-address-spaces.ll
+++ b/llvm/test/Transforms/LoopVectorize/multiple-address-spaces.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s  -basicaa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+; RUN: opt < %s  -basic-aa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
 
 ; From a simple program with two address spaces:
 ; char Y[4*10000] __attribute__((address_space(1)));
diff --git a/llvm/test/Transforms/LoopVectorize/noalias-md-licm.ll b/llvm/test/Transforms/LoopVectorize/noalias-md-licm.ll
index 233d530dc1029..d54a9aefe73a1 100644
--- a/llvm/test/Transforms/LoopVectorize/noalias-md-licm.ll
+++ b/llvm/test/Transforms/LoopVectorize/noalias-md-licm.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -scoped-noalias -loop-vectorize -licm -force-vector-width=2 \
+; RUN: opt -basic-aa -scoped-noalias -loop-vectorize -licm -force-vector-width=2 \
 ; RUN:     -force-vector-interleave=1 -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/noalias-md.ll b/llvm/test/Transforms/LoopVectorize/noalias-md.ll
index 787ea88f9457b..b141998163aef 100644
--- a/llvm/test/Transforms/LoopVectorize/noalias-md.ll
+++ b/llvm/test/Transforms/LoopVectorize/noalias-md.ll
@@ -1,7 +1,7 @@
-; RUN: opt -basicaa -loop-vectorize -force-vector-width=2 \
+; RUN: opt -basic-aa -loop-vectorize -force-vector-width=2 \
 ; RUN:     -force-vector-interleave=1 -S < %s \
 ; RUN:     | FileCheck %s -check-prefix=BOTH -check-prefix=LV
-; RUN: opt -basicaa -scoped-noalias -loop-vectorize -dse -force-vector-width=2 \
+; RUN: opt -basic-aa -scoped-noalias -loop-vectorize -dse -force-vector-width=2 \
 ; RUN:     -force-vector-interleave=1 -S < %s \
 ; RUN:     | FileCheck %s -check-prefix=BOTH -check-prefix=DSE
 
diff --git a/llvm/test/Transforms/LoopVectorize/optsize.ll b/llvm/test/Transforms/LoopVectorize/optsize.ll
index f0ca35b411c53..0e88f362746fb 100644
--- a/llvm/test/Transforms/LoopVectorize/optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/optsize.ll
@@ -154,6 +154,99 @@ exit:
   ret i32 %for
 }
 
+; PR46228: Vectorize w/o versioning for unit stride under optsize and enabled
+; vectorization.
+
+; NOTE: Some assertions have been autogenerated by utils/update_test_checks.py
+define void @stride1(i16* noalias %B, i32 %BStride) optsize {
+; CHECK-LABEL: @stride1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> undef, i32 [[BSTRIDE:%.*]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE2]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul nsw <2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <2 x i32> [[VEC_IND]], <i32 1024, i32 1024>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; CHECK:       pred.store.if:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i16, i16* [[B:%.*]], i32 [[TMP3]]
+; CHECK-NEXT:    store i16 42, i16* [[TMP4]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; CHECK:       pred.store.continue:
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1
+; CHECK-NEXT:    br i1 [[TMP5]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
+; CHECK:       pred.store.if1:
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i16, i16* [[B]], i32 [[TMP6]]
+; CHECK-NEXT:    store i16 42, i16* [[TMP7]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE2]]
+; CHECK:       pred.store.continue2:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], <i32 2, i32 2>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1026
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !19
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+; PGSO-LABEL: @stride1(
+; PGSO-NEXT:  entry:
+; PGSO-NEXT:    br i1 false, label %scalar.ph, label %vector.ph
+;
+; NPGSO-LABEL: @stride1(
+; NPGSO-NEXT:  entry:
+; NPGSO-NEXT:    br i1 false, label %scalar.ph, label %vector.ph
+
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]
+  %mulB = mul nsw i32 %iv, %BStride
+  %gepOfB = getelementptr inbounds i16, i16* %B, i32 %mulB
+  store i16 42, i16* %gepOfB, align 4
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exitcond = icmp eq i32 %iv.next, 1025
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !15
+
+for.end:
+  ret void
+}
+
+; PR46652: Check that the need for stride==1 check prevents vectorizing a loop
+; having tiny trip count, when compiling w/o -Os/-Oz.
+; CHECK-LABEL: @pr46652
+; CHECK-NOT: vector.scevcheck
+; CHECK-NOT: vector.body
+; CHECK-LABEL: for.body
+
+@g = external global [1 x i16], align 1
+
+define void @pr46652(i16 %stride) {
+entry:
+  br label %for.body
+
+for.body:                                        ; preds = %for.body, %entry
+  %l1.02 = phi i16 [ 1, %entry ], [ %inc9, %for.body ]
+  %mul = mul nsw i16 %l1.02, %stride
+  %arrayidx6 = getelementptr inbounds [1 x i16], [1 x i16]* @g, i16 0, i16 %mul
+  %0 = load i16, i16* %arrayidx6, align 1
+  %inc9 = add nuw nsw i16 %l1.02, 1
+  %exitcond.not = icmp eq i16 %inc9, 16
+  br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:                                        ; preds = %for.body
+  ret void
+}
+
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"ProfileSummary", !1}
 !1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
@@ -170,3 +263,5 @@ exit:
 !12 = !{i32 999000, i64 100, i32 1}
 !13 = !{i32 999999, i64 1, i32 2}
 !14 = !{!"function_entry_count", i64 0}
+!15 = distinct !{!15, !16}
+!16 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/pr39417-optsize-scevchecks.ll b/llvm/test/Transforms/LoopVectorize/pr39417-optsize-scevchecks.ll
index 6032fb18a387c..d07e35687932a 100644
--- a/llvm/test/Transforms/LoopVectorize/pr39417-optsize-scevchecks.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr39417-optsize-scevchecks.ll
@@ -26,13 +26,57 @@ bb68:
   ret void
 }
 
-; Check that the need for stride==1 check prevents vectorizing a loop under opt
-; for size.
-; CHECK-LABEL: @scev4stride1
-; CHECK-NOT: vector.scevcheck
-; CHECK-NOT: vector.body:
-; CHECK-LABEL: for.body:
+; Check that a loop under opt-for-size is vectorized, w/o checking for
+; stride==1.
+; NOTE: Some assertions have been autogenerated by utils/update_test_checks.py
 define void @scev4stride1(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %k) #0 {
+; CHECK-LABEL: @scev4stride1(
+; CHECK-NEXT:  for.body.preheader:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> undef, i32 [[K:%.*]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i32> [[TMP4]], i32 3
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP15:%.*]] = load i32, i32* [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP16:%.*]] = load i32, i32* [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> undef, i32 [[TMP13]], i32 0
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP14]], i32 1
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP15]], i32 2
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP16]], i32 3
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[TMP21]], i32 0
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast i32* [[TMP22]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP23]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 1024, 1024
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK:       for.body:
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    ret void
+;
 for.body.preheader:
   br label %for.body
 
diff --git a/llvm/test/Transforms/LoopVectorize/pr46525-expander-insertpoint.ll b/llvm/test/Transforms/LoopVectorize/pr46525-expander-insertpoint.ll
new file mode 100644
index 0000000000000..98f394e2dc3d7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr46525-expander-insertpoint.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-vectorize -force-vector-width=2 -S -prefer-predicate-over-epilog %s | FileCheck %s
+
+
+; Test case for PR46525. There are two candidates to pick for
+; `udiv i64 %y, %add` when expanding SCEV expressions. Make sure we pick %div,
+; which dominates the vector loop.
+
+define void @test(i16 %x, i64 %y, i32* %ptr) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV19:%.*]] = sext i16 [[X:%.*]] to i64
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[CONV19]], 492802768830814067
+; CHECK-NEXT:    br label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[Y:%.*]], [[ADD]]
+; CHECK-NEXT:    [[INC:%.*]] = add i64 [[DIV]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = add nuw nsw i64 [[DIV]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], [[INC]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], [[INC]]
+; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP2]], 1
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> undef, i64 [[TRIP_COUNT_MINUS_1]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], [[INC]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> undef, i64 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> undef, i64 [[INC]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i64> [[DOTSPLATINSERT]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = mul <2 x i64> <i64 0, i64 1>, [[DOTSPLAT]]
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <2 x i64> [[BROADCAST_SPLAT2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 0, [[INC]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], [[TMP4]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x i64> undef, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT3]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[VEC_IV:%.*]] = add <2 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule <2 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i1> [[TMP6]], i32 0
+; CHECK-NEXT:    br i1 [[TMP7]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; CHECK:       pred.store.if:
+; CHECK-NEXT:    store i32 0, i32* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; CHECK:       pred.store.continue:
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[TMP6]], i32 1
+; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
+; CHECK:       pred.store.if5:
+; CHECK-NEXT:    store i32 0, i32* [[PTR]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE6]]
+; CHECK:       pred.store.continue6:
+; CHECK-NEXT:    [[OFFSET_IDX7:%.*]] = mul i64 [[INDEX]], [[INC]]
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[OFFSET_IDX7]] to i8
+; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[INC]] to i8
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <2 x i8> undef, i8 [[TMP9]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT9:%.*]] = shufflevector <2 x i8> [[BROADCAST_SPLATINSERT8]], <2 x i8> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLATINSERT10:%.*]] = insertelement <2 x i8> undef, i8 [[TMP10]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT11:%.*]] = shufflevector <2 x i8> [[DOTSPLATINSERT10]], <2 x i8> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = mul <2 x i8> <i8 0, i8 1>, [[DOTSPLAT11]]
+; CHECK-NEXT:    [[INDUCTION12:%.*]] = add <2 x i8> [[BROADCAST_SPLAT9]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i8 0, [[TMP10]]
+; CHECK-NEXT:    [[TMP13:%.*]] = add i8 [[TMP9]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+;
+entry:
+  %conv19 = sext i16 %x to i64
+  %add = add i64 %conv19, 492802768830814067
+  br label %loop.preheader
+
+loop.preheader:
+  %div = udiv i64 %y, %add
+  %inc = add i64 %div, 1
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %loop.preheader ]
+  store i32 0, i32* %ptr, align 4
+  %v2 = trunc i64 %iv to i8
+  %v3 = add i8 %v2, 1
+  %cmp15 = icmp slt i8 %v3, 5
+  %iv.next = add i64 %iv, %inc
+  br i1 %cmp15, label %loop, label %loop.exit
+
+loop.exit:
+  %div.1 = udiv i64 %y, %add
+  %v1 = add i64 %div.1, 1
+  br label %loop.2
+
+loop.2:
+  %iv.1 = phi i64 [ %iv.next.1, %loop.2 ], [ 0, %loop.exit ]
+  %iv.next.1 = add i64 %iv.1, %v1
+  call void @use(i64 %iv.next.1)
+  %ec = icmp ult i64 %iv.next.1, 200
+  br i1 %ec, label %loop.2, label %loop.2.exit
+
+loop.2.exit:
+  %c = call i1 @cond()
+  br i1 %c, label %loop.preheader, label %exit
+
+exit:
+  ret void
+}
+
+declare void @use(i64)
+declare i1 @cond()
diff --git a/llvm/test/Transforms/LoopVectorize/ptr_loops.ll b/llvm/test/Transforms/LoopVectorize/ptr_loops.ll
index c374b006147a7..c8c928b086349 100644
--- a/llvm/test/Transforms/LoopVectorize/ptr_loops.ll
+++ b/llvm/test/Transforms/LoopVectorize/ptr_loops.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s  -basicaa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S -enable-if-conversion | FileCheck %s
+; RUN: opt < %s  -basic-aa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S -enable-if-conversion | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check-address-space.ll b/llvm/test/Transforms/LoopVectorize/runtime-check-address-space.ll
index b97cd43e3bfca..8f09dde0812d9 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-check-address-space.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check-address-space.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -march=r600 -mcpu=cayman -basicaa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine < %s | FileCheck %s
+; RUN: opt -S -march=r600 -mcpu=cayman -basic-aa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine < %s | FileCheck %s
 
 ; Check vectorization that would ordinarily require a runtime bounds
 ; check on the pointers when mixing address spaces. For now we cannot
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check.ll b/llvm/test/Transforms/LoopVectorize/runtime-check.ll
index ecabef22f7255..37f9c0ce0de40 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-check.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
-; RUN: opt < %s -loop-vectorize -disable-basicaa -S -pass-remarks-analysis='loop-vectorize' 2>&1 | FileCheck %s -check-prefix=FORCED_OPTSIZE
+; RUN: opt < %s -loop-vectorize -disable-basic-aa -S -pass-remarks-analysis='loop-vectorize' 2>&1 | FileCheck %s -check-prefix=FORCED_OPTSIZE
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
@@ -162,9 +162,9 @@ loopexit:
 
 define dso_local void @forced_optsize(i64* noalias nocapture readonly %x_p, i64* noalias nocapture readonly %y_p, i64* noalias nocapture %z_p) minsize optsize {
 ;
-; FORCED_OPTSIZE: remark: <unknown>:0:0: Code-size may be reduced by not forcing vectorization, or by source-code modifications eliminating the need for runtime checks (e.g., adding 'restrict').
+; FORCED_OPTSIZE: remark: <unknown>:0:0: loop not vectorized: runtime pointer checks needed. Enable vectorization of this loop with '#pragma clang loop vectorize(enable)' when compiling with -Os/-Oz
 ; FORCED_OPTSIZE-LABEL: @forced_optsize(
-; FORCED_OPTSIZE:       vector.body:
+; FORCED_OPTSIZE-NOT:       vector.body:
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/store-shuffle-bug.ll b/llvm/test/Transforms/LoopVectorize/store-shuffle-bug.ll
index 62b148e2c79c5..414a026fcf9b3 100644
--- a/llvm/test/Transforms/LoopVectorize/store-shuffle-bug.ll
+++ b/llvm/test/Transforms/LoopVectorize/store-shuffle-bug.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine < %s | FileCheck %s
+; RUN: opt -S -basic-aa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopVectorize/tbaa-nodep.ll b/llvm/test/Transforms/LoopVectorize/tbaa-nodep.ll
index d02f9285450eb..af89f6e5f0605 100644
--- a/llvm/test/Transforms/LoopVectorize/tbaa-nodep.ll
+++ b/llvm/test/Transforms/LoopVectorize/tbaa-nodep.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s  -tbaa -basicaa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -simplifycfg -S | FileCheck %s
-; RUN: opt < %s  -basicaa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -simplifycfg -S | FileCheck %s --check-prefix=CHECK-NOTBAA
+; RUN: opt < %s  -tbaa -basic-aa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -simplifycfg -S | FileCheck %s
+; RUN: opt < %s  -basic-aa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -simplifycfg -S | FileCheck %s --check-prefix=CHECK-NOTBAA
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; Function Attrs: nounwind uwtable
diff --git a/llvm/test/Transforms/LoopVectorize/version-mem-access.ll b/llvm/test/Transforms/LoopVectorize/version-mem-access.ll
index 774b6f268599e..fdb64659461c5 100644
--- a/llvm/test/Transforms/LoopVectorize/version-mem-access.ll
+++ b/llvm/test/Transforms/LoopVectorize/version-mem-access.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-vectorize -enable-mem-access-versioning -force-vector-width=2 -force-vector-interleave=1 < %s -S | FileCheck %s
+; RUN: opt -basic-aa -loop-vectorize -enable-mem-access-versioning -force-vector-width=2 -force-vector-interleave=1 < %s -S | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopVersioning/basic.ll b/llvm/test/Transforms/LoopVersioning/basic.ll
index f59caecadae7e..bcd5e8c875c69 100644
--- a/llvm/test/Transforms/LoopVersioning/basic.ll
+++ b/llvm/test/Transforms/LoopVersioning/basic.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-versioning -S < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-versioning -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopVersioning/convergent.ll b/llvm/test/Transforms/LoopVersioning/convergent.ll
index 48c2725a7656a..5751752059672 100644
--- a/llvm/test/Transforms/LoopVersioning/convergent.ll
+++ b/llvm/test/Transforms/LoopVersioning/convergent.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-versioning -S < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-versioning -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopVersioning/lcssa.ll b/llvm/test/Transforms/LoopVersioning/lcssa.ll
index 01178a5931e4e..f507b43ea2b51 100644
--- a/llvm/test/Transforms/LoopVersioning/lcssa.ll
+++ b/llvm/test/Transforms/LoopVersioning/lcssa.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-versioning -S < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-versioning -S < %s | FileCheck %s
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @fill(i8** %ls1.20, i8** %ls2.21, i8* %cse3.22) {
diff --git a/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll b/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll
index c53dc858c5cef..89e898a90c178 100644
--- a/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll
+++ b/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -loop-simplify -scoped-noalias \
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -loop-simplify -scoped-noalias \
 ; RUN:     -loop-versioning -S < %s | FileCheck %s
 
 ; Test the metadata generated when versioning an already versioned loop.  Here
diff --git a/llvm/test/Transforms/LoopVersioning/noalias.ll b/llvm/test/Transforms/LoopVersioning/noalias.ll
index c2539726db4d4..dac875fa60e75 100644
--- a/llvm/test/Transforms/LoopVersioning/noalias.ll
+++ b/llvm/test/Transforms/LoopVersioning/noalias.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -loop-versioning -S < %s | FileCheck %s
+; RUN: opt -basic-aa -loop-versioning -S < %s | FileCheck %s
 
 ; A very simple case.  After versioning the %loadA and %loadB can't alias with
 ; the store.
diff --git a/llvm/test/Transforms/LowerConstantIntrinsics/objectsize_basic.ll b/llvm/test/Transforms/LowerConstantIntrinsics/objectsize_basic.ll
index 94aba44fcc4fa..ee5d792d53299 100644
--- a/llvm/test/Transforms/LowerConstantIntrinsics/objectsize_basic.ll
+++ b/llvm/test/Transforms/LowerConstantIntrinsics/objectsize_basic.ll
@@ -81,3 +81,11 @@ entry:
                                                i1 false, i1 false)
   ret i64 %0
 }
+
+; CHECK-LABEL: @test_objectsize_byval_arg(
+; CHECK: ret i64 42
+define i64 @test_objectsize_byval_arg([42 x i8]* byval([42 x i8]) %ptr) {
+  %cast = bitcast [42 x i8]* %ptr to i8*
+  %size = tail call i64 @llvm.objectsize.i64(i8* %cast, i1 true, i1 false, i1 false)
+  ret i64 %size
+}
diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll b/llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll
new file mode 100644
index 0000000000000..8972ee01c12d2
--- /dev/null
+++ b/llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll
@@ -0,0 +1,295 @@
+; RUN: opt -lower-expect -strip-dead-prototypes -S -o - < %s | FileCheck %s
+; RUN: opt -S -passes='function(lower-expect),strip-dead-prototypes' < %s | FileCheck %s
+
+; CHECK-LABEL: @test1(
+define i32 @test1(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %cmp = icmp sgt i32 %tmp, 1
+  %conv = zext i1 %cmp to i32
+  %conv1 = sext i32 %conv to i64
+  %expval = call i64 @llvm.expect.with.probability.i64(i64 %conv1, i64 1, double 8.000000e-01)
+  %tobool = icmp ne i64 %expval, 0
+; CHECK: !prof !0, !misexpect !1
+; CHECK-NOT: @llvm.expect.with.probability
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i32 (...) @f()
+  store i32 %call, i32* %retval
+  br label %return
+
+if.end:                                           ; preds = %entry
+  store i32 1, i32* %retval
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+declare i64 @llvm.expect.with.probability.i64(i64, i64, double) nounwind readnone
+
+declare i32 @f(...)
+
+; CHECK-LABEL: @test2(
+define i32 @test2(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %conv = sext i32 %tmp to i64
+  %expval = call i64 @llvm.expect.with.probability.i64(i64 %conv, i64 1, double 8.000000e-01)
+  %tobool = icmp ne i64 %expval, 0
+; CHECK: !prof !0, !misexpect !1
+; CHECK-NOT: @llvm.expect.with.probability
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i32 (...) @f()
+  store i32 %call, i32* %retval
+  br label %return
+
+if.end:                                           ; preds = %entry
+  store i32 1, i32* %retval
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test3(
+define i32 @test3(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %tobool = icmp ne i32 %tmp, 0
+  %lnot = xor i1 %tobool, true
+  %lnot.ext = zext i1 %lnot to i32
+  %conv = sext i32 %lnot.ext to i64
+  %expval = call i64 @llvm.expect.with.probability.i64(i64 %conv, i64 1, double 8.000000e-01)
+  %tobool1 = icmp ne i64 %expval, 0
+; CHECK: !prof !0, !misexpect !1
+; CHECK-NOT: @llvm.expect.with.probability
+  br i1 %tobool1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i32 (...) @f()
+  store i32 %call, i32* %retval
+  br label %return
+
+if.end:                                           ; preds = %entry
+  store i32 1, i32* %retval
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test4(
+define i32 @test4(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %tobool = icmp ne i32 %tmp, 0
+  %lnot = xor i1 %tobool, true
+  %lnot1 = xor i1 %lnot, true
+  %lnot.ext = zext i1 %lnot1 to i32
+  %conv = sext i32 %lnot.ext to i64
+  %expval = call i64 @llvm.expect.with.probability.i64(i64 %conv, i64 1, double 8.000000e-01)
+  %tobool2 = icmp ne i64 %expval, 0
+; CHECK: !prof !0, !misexpect !1
+; CHECK-NOT: @llvm.expect.with.probability
+  br i1 %tobool2, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i32 (...) @f()
+  store i32 %call, i32* %retval
+  br label %return
+
+if.end:                                           ; preds = %entry
+  store i32 1, i32* %retval
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test5(
+define i32 @test5(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %cmp = icmp slt i32 %tmp, 0
+  %conv = zext i1 %cmp to i32
+  %conv1 = sext i32 %conv to i64
+  %expval = call i64 @llvm.expect.with.probability.i64(i64 %conv1, i64 0, double 8.000000e-01)
+  %tobool = icmp ne i64 %expval, 0
+; CHECK: !prof !2, !misexpect !3
+; CHECK-NOT: @llvm.expect.with.probability
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i32 (...) @f()
+  store i32 %call, i32* %retval
+  br label %return
+
+if.end:                                           ; preds = %entry
+  store i32 1, i32* %retval
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test6(
+define i32 @test6(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %conv = sext i32 %tmp to i64
+  %expval = call i64 @llvm.expect.with.probability.i64(i64 %conv, i64 2, double 8.000000e-01)
+; CHECK: !prof !4, !misexpect !5
+; CHECK-NOT: @llvm.expect.with.probability
+  switch i64 %expval, label %sw.epilog [
+    i64 1, label %sw.bb
+    i64 2, label %sw.bb
+  ]
+
+sw.bb:                                            ; preds = %entry, %entry
+  store i32 0, i32* %retval
+  br label %return
+
+sw.epilog:                                        ; preds = %entry
+  store i32 1, i32* %retval
+  br label %return
+
+return:                                           ; preds = %sw.epilog, %sw.bb
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test7(
+define i32 @test7(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %conv = sext i32 %tmp to i64
+  %expval = call i64 @llvm.expect.with.probability.i64(i64 %conv, i64 1, double 8.000000e-01)
+; CHECK: !prof !6, !misexpect !7
+; CHECK-NOT: @llvm.expect.with.probability
+  switch i64 %expval, label %sw.epilog [
+    i64 2, label %sw.bb
+    i64 3, label %sw.bb
+  ]
+
+sw.bb:                                            ; preds = %entry, %entry
+  %tmp1 = load i32, i32* %x.addr, align 4
+  store i32 %tmp1, i32* %retval
+  br label %return
+
+sw.epilog:                                        ; preds = %entry
+  store i32 0, i32* %retval
+  br label %return
+
+return:                                           ; preds = %sw.epilog, %sw.bb
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test8(
+define i32 @test8(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %cmp = icmp sgt i32 %tmp, 1
+  %conv = zext i1 %cmp to i32
+  %expval = call i32 @llvm.expect.with.probability.i32(i32 %conv, i32 1, double 8.000000e-01)
+  %tobool = icmp ne i32 %expval, 0
+; CHECK: !prof !0, !misexpect !1
+; CHECK-NOT: @llvm.expect.with.probability
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i32 (...) @f()
+  store i32 %call, i32* %retval
+  br label %return
+
+if.end:                                           ; preds = %entry
+  store i32 1, i32* %retval
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+declare i32 @llvm.expect.with.probability.i32(i32, i32, double) nounwind readnone
+
+; CHECK-LABEL: @test9(
+define i32 @test9(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %cmp = icmp sgt i32 %tmp, 1
+  %expval = call i1 @llvm.expect.with.probability.i1(i1 %cmp, i1 1, double 8.000000e-01)
+; CHECK: !prof !0, !misexpect !1
+; CHECK-NOT: @llvm.expect.with.probability
+  br i1 %expval, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i32 (...) @f()
+  store i32 %call, i32* %retval
+  br label %return
+
+if.end:                                           ; preds = %entry
+  store i32 1, i32* %retval
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test10(
+define i32 @test10(i64 %t6) {
+  %t7 = call i64 @llvm.expect.with.probability.i64(i64 %t6, i64 0, double 8.000000e-01)
+  %t8 = icmp ne i64 %t7, 0
+  %t9 = select i1 %t8, i32 1, i32 2
+; CHECK: select{{.*}}, !prof !2, !misexpect !3
+  ret i32 %t9
+}
+
+
+declare i1 @llvm.expect.with.probability.i1(i1, i1, double) nounwind readnone
+
+; CHECK: !0 = !{!"branch_weights", i32 1717986918, i32 429496731}
+; CHECK: !1 = !{!"misexpect", i64 0, i64 1717986918, i64 429496731}
+; CHECK: !2 = !{!"branch_weights", i32 429496731, i32 1717986918}
+; CHECK: !3 = !{!"misexpect", i64 1, i64 1717986918, i64 429496731}
+; CHECK: !4 = !{!"branch_weights", i32 214748366, i32 214748366, i32 1717986918}
+; CHECK: !5 = !{!"misexpect", i64 2, i64 1717986918, i64 214748366}
+; CHECK: !6 = !{!"branch_weights", i32 1717986918, i32 214748366, i32 214748366}
+; CHECK: !7 = !{!"misexpect", i64 0, i64 1717986918, i64 214748366}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/remarks-shared-subtrees.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/remarks-shared-subtrees.ll
index e92733f9a81a2..2846889bf2393 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/remarks-shared-subtrees.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/remarks-shared-subtrees.ll
@@ -18,8 +18,7 @@
 ; YAML-NEXT:    - String:          ' loads, '
 ; YAML-NEXT:    - NumComputeOps:   '0'
 ; YAML-NEXT:    - String:          ' compute ops'
-; YAML-NEXT:    - String:          ',
-; YAML-NEXT:  additionally '
+; YAML-NEXT:    - String:          ",\nadditionally "
 ; YAML-NEXT:    - NumStores:       '0'
 ; YAML-NEXT:    - String:          ' stores, '
 ; YAML-NEXT:    - NumLoads:        '4'
@@ -47,8 +46,7 @@
 ; YAML-NEXT:    - String:          ' loads, '
 ; YAML-NEXT:    - NumComputeOps:   '120'
 ; YAML-NEXT:    - String:          ' compute ops'
-; YAML-NEXT:    - String:          ',
-; YAML-NEXT:  additionally '
+; YAML-NEXT:    - String:          ",\nadditionally "
 ; YAML-NEXT:    - NumStores:       '0'
 ; YAML-NEXT:    - String:          ' stores, '
 ; YAML-NEXT:    - NumLoads:        '4'
diff --git a/llvm/test/Transforms/MemCpyOpt/2008-02-24-MultipleUseofSRet.ll b/llvm/test/Transforms/MemCpyOpt/2008-02-24-MultipleUseofSRet.ll
index 7ff149f6f7796..237b8fec4f645 100644
--- a/llvm/test/Transforms/MemCpyOpt/2008-02-24-MultipleUseofSRet.ll
+++ b/llvm/test/Transforms/MemCpyOpt/2008-02-24-MultipleUseofSRet.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -memcpyopt -dse -S | grep "call.*initialize" | not grep memtmp
+; RUN: opt < %s -basic-aa -memcpyopt -dse -S | grep "call.*initialize" | not grep memtmp
 ; PR2077
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
diff --git a/llvm/test/Transforms/MemCpyOpt/2008-03-13-ReturnSlotBitcast.ll b/llvm/test/Transforms/MemCpyOpt/2008-03-13-ReturnSlotBitcast.ll
index 26c221d678676..a0f34b9baa6df 100644
--- a/llvm/test/Transforms/MemCpyOpt/2008-03-13-ReturnSlotBitcast.ll
+++ b/llvm/test/Transforms/MemCpyOpt/2008-03-13-ReturnSlotBitcast.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -memcpyopt -S | not grep "call.*memcpy."
+; RUN: opt < %s -basic-aa -memcpyopt -S | not grep "call.*memcpy."
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64"
 
 %a = type { i32 }
diff --git a/llvm/test/Transforms/MemCpyOpt/2011-06-02-CallSlotOverwritten.ll b/llvm/test/Transforms/MemCpyOpt/2011-06-02-CallSlotOverwritten.ll
index 8e4a0230d7fdf..8ba8df4d8b395 100644
--- a/llvm/test/Transforms/MemCpyOpt/2011-06-02-CallSlotOverwritten.ll
+++ b/llvm/test/Transforms/MemCpyOpt/2011-06-02-CallSlotOverwritten.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -memcpyopt -S | FileCheck %s
+; RUN: opt < %s -basic-aa -memcpyopt -S | FileCheck %s
 ; PR10067
 ; Make sure the call+copy isn't optimized in such a way that
 ; %ret ends up with the wrong value.
diff --git a/llvm/test/Transforms/MemCpyOpt/align.ll b/llvm/test/Transforms/MemCpyOpt/align.ll
index 738928bac623c..2e683bfa91f5b 100644
--- a/llvm/test/Transforms/MemCpyOpt/align.ll
+++ b/llvm/test/Transforms/MemCpyOpt/align.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -S -basicaa -memcpyopt | FileCheck %s
+; RUN: opt < %s -S -basic-aa -memcpyopt | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64"
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
diff --git a/llvm/test/Transforms/MemCpyOpt/atomic.ll b/llvm/test/Transforms/MemCpyOpt/atomic.ll
index f8fc39faa582b..65f6c925e205b 100644
--- a/llvm/test/Transforms/MemCpyOpt/atomic.ll
+++ b/llvm/test/Transforms/MemCpyOpt/atomic.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -memcpyopt -S < %s | FileCheck %s
+; RUN: opt -basic-aa -memcpyopt -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-macosx10.7.0"
diff --git a/llvm/test/Transforms/MemCpyOpt/callslot_aa.ll b/llvm/test/Transforms/MemCpyOpt/callslot_aa.ll
index d840b726a9aa0..1d45cbe9e5cb7 100644
--- a/llvm/test/Transforms/MemCpyOpt/callslot_aa.ll
+++ b/llvm/test/Transforms/MemCpyOpt/callslot_aa.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -S -basicaa -memcpyopt | FileCheck %s
+; RUN: opt < %s -S -basic-aa -memcpyopt | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 %T = type { i64, i64 }
diff --git a/llvm/test/Transforms/MemCpyOpt/callslot_deref.ll b/llvm/test/Transforms/MemCpyOpt/callslot_deref.ll
index a1ba2bae79e71..ad578be711cd0 100644
--- a/llvm/test/Transforms/MemCpyOpt/callslot_deref.ll
+++ b/llvm/test/Transforms/MemCpyOpt/callslot_deref.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -S -basicaa -memcpyopt | FileCheck %s
+; RUN: opt < %s -S -basic-aa -memcpyopt | FileCheck %s
 target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) unnamed_addr nounwind
diff --git a/llvm/test/Transforms/MemCpyOpt/capturing-func.ll b/llvm/test/Transforms/MemCpyOpt/capturing-func.ll
index 2671a9aad818e..0ea889a664979 100644
--- a/llvm/test/Transforms/MemCpyOpt/capturing-func.ll
+++ b/llvm/test/Transforms/MemCpyOpt/capturing-func.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -memcpyopt -S | FileCheck %s
+; RUN: opt < %s -basic-aa -memcpyopt -S | FileCheck %s
 
 target datalayout = "e"
 
diff --git a/llvm/test/Transforms/MemCpyOpt/crash.ll b/llvm/test/Transforms/MemCpyOpt/crash.ll
index 464a261e4e406..1fd4d0deae6df 100644
--- a/llvm/test/Transforms/MemCpyOpt/crash.ll
+++ b/llvm/test/Transforms/MemCpyOpt/crash.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -memcpyopt -disable-output
+; RUN: opt < %s -basic-aa -memcpyopt -disable-output
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64"
 target triple = "armv7-eabi"
diff --git a/llvm/test/Transforms/MemCpyOpt/invariant.start.ll b/llvm/test/Transforms/MemCpyOpt/invariant.start.ll
index 4842114b58467..b7e3160c7da73 100644
--- a/llvm/test/Transforms/MemCpyOpt/invariant.start.ll
+++ b/llvm/test/Transforms/MemCpyOpt/invariant.start.ll
@@ -1,5 +1,5 @@
 ; MemCpy optimizations should take place even in presence of invariant.start
-; RUN: opt < %s -basicaa -memcpyopt -dse -S | FileCheck %s
+; RUN: opt < %s -basic-aa -memcpyopt -dse -S | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 
diff --git a/llvm/test/Transforms/MemCpyOpt/load-store-to-memcpy.ll b/llvm/test/Transforms/MemCpyOpt/load-store-to-memcpy.ll
index 7eca765c8a025..81043d4f46ddf 100644
--- a/llvm/test/Transforms/MemCpyOpt/load-store-to-memcpy.ll
+++ b/llvm/test/Transforms/MemCpyOpt/load-store-to-memcpy.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -basicaa -scoped-noalias -memcpyopt -S %s | FileCheck %s
+; RUN: opt -basic-aa -scoped-noalias -memcpyopt -S %s | FileCheck %s
 
 %T = type { i8, i32 }
 
diff --git a/llvm/test/Transforms/MemCpyOpt/loadstore-sret.ll b/llvm/test/Transforms/MemCpyOpt/loadstore-sret.ll
index 4c6136cf62593..0f8a70a5511dc 100644
--- a/llvm/test/Transforms/MemCpyOpt/loadstore-sret.ll
+++ b/llvm/test/Transforms/MemCpyOpt/loadstore-sret.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S < %s -basicaa -memcpyopt | FileCheck %s
+; RUN: opt -S < %s -basic-aa -memcpyopt | FileCheck %s
 ; <rdar://problem/8536696>
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll b/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
index ee2f6adb431f7..6affd8c794936 100644
--- a/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -basicaa -memcpyopt -instcombine -S < %s | FileCheck %s
+; RUN: opt -basic-aa -memcpyopt -instcombine -S < %s | FileCheck %s
 
 target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy-undef.ll b/llvm/test/Transforms/MemCpyOpt/memcpy-undef.ll
index 9cf4f91e6aa26..5cdd1a27258c5 100644
--- a/llvm/test/Transforms/MemCpyOpt/memcpy-undef.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy-undef.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -memcpyopt -S | FileCheck %s
+; RUN: opt < %s -basic-aa -memcpyopt -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy.ll b/llvm/test/Transforms/MemCpyOpt/memcpy.ll
index 23b13760889aa..1741da030c2ed 100644
--- a/llvm/test/Transforms/MemCpyOpt/memcpy.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -memcpyopt -dse -S | FileCheck -enable-var-scope %s
+; RUN: opt < %s -basic-aa -memcpyopt -dse -S | FileCheck -enable-var-scope %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i686-apple-darwin9"
diff --git a/llvm/test/Transforms/MemCpyOpt/memmove.ll b/llvm/test/Transforms/MemCpyOpt/memmove.ll
index 91f2851ab7802..d152cfb63f2b7 100644
--- a/llvm/test/Transforms/MemCpyOpt/memmove.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memmove.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -memcpyopt -S | FileCheck %s
+; RUN: opt < %s -basic-aa -memcpyopt -S | FileCheck %s
 ; These memmoves should get optimized to memcpys.
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
diff --git a/llvm/test/Transforms/MemCpyOpt/memset-memcpy-redundant-memset.ll b/llvm/test/Transforms/MemCpyOpt/memset-memcpy-redundant-memset.ll
index a3ca96c3ab52d..651ac3194a157 100644
--- a/llvm/test/Transforms/MemCpyOpt/memset-memcpy-redundant-memset.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memset-memcpy-redundant-memset.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -memcpyopt -S %s | FileCheck %s
+; RUN: opt -basic-aa -memcpyopt -S %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/MemCpyOpt/sret.ll b/llvm/test/Transforms/MemCpyOpt/sret.ll
index a99b52d5d6e3c..af625127f56ba 100644
--- a/llvm/test/Transforms/MemCpyOpt/sret.ll
+++ b/llvm/test/Transforms/MemCpyOpt/sret.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -memcpyopt -S | not grep "call.*memcpy"
+; RUN: opt < %s -basic-aa -memcpyopt -S | not grep "call.*memcpy"
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i686-apple-darwin9"
diff --git a/llvm/test/Transforms/NewGVN/2007-07-25-InfiniteLoop.ll b/llvm/test/Transforms/NewGVN/2007-07-25-InfiniteLoop.ll
index fcbfb4c216b55..5becacfeed89c 100644
--- a/llvm/test/Transforms/NewGVN/2007-07-25-InfiniteLoop.ll
+++ b/llvm/test/Transforms/NewGVN/2007-07-25-InfiniteLoop.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 
 	%struct.INT2 = type { i32, i32 }
 @blkshifts = external global %struct.INT2*		; <%struct.INT2**> [#uses=2]
diff --git a/llvm/test/Transforms/NewGVN/2007-07-26-InterlockingLoops.ll b/llvm/test/Transforms/NewGVN/2007-07-26-InterlockingLoops.ll
index 1eb90ad2a1f11..d1471f4266eed 100644
--- a/llvm/test/Transforms/NewGVN/2007-07-26-InterlockingLoops.ll
+++ b/llvm/test/Transforms/NewGVN/2007-07-26-InterlockingLoops.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 
 @last = external global [65 x i32*]
 
diff --git a/llvm/test/Transforms/NewGVN/2007-07-31-NoDomInherit.ll b/llvm/test/Transforms/NewGVN/2007-07-31-NoDomInherit.ll
index 0fd7588b8ad9e..88da79d3dedf3 100644
--- a/llvm/test/Transforms/NewGVN/2007-07-31-NoDomInherit.ll
+++ b/llvm/test/Transforms/NewGVN/2007-07-31-NoDomInherit.ll
@@ -1,5 +1,5 @@
 ; XFAIL: *
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 
 	%struct.anon = type { i32 (i32, i32, i32)*, i32, i32, [3 x i32], i8*, i8*, i8* }
 @debug = external constant i32		; <i32*> [#uses=0]
diff --git a/llvm/test/Transforms/NewGVN/2007-07-31-RedundantPhi.ll b/llvm/test/Transforms/NewGVN/2007-07-31-RedundantPhi.ll
index 3b59bad1824c8..ccd29df8a364b 100644
--- a/llvm/test/Transforms/NewGVN/2007-07-31-RedundantPhi.ll
+++ b/llvm/test/Transforms/NewGVN/2007-07-31-RedundantPhi.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 
 @img_width = external global i16		; <i16*> [#uses=2]
 
diff --git a/llvm/test/Transforms/NewGVN/2008-07-02-Unreachable.ll b/llvm/test/Transforms/NewGVN/2008-07-02-Unreachable.ll
index 797cf574f0131..59b0e782b0723 100644
--- a/llvm/test/Transforms/NewGVN/2008-07-02-Unreachable.ll
+++ b/llvm/test/Transforms/NewGVN/2008-07-02-Unreachable.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 ; PR2503
 
 @g_3 = external global i8		; <i8*> [#uses=2]
diff --git a/llvm/test/Transforms/NewGVN/2010-03-31-RedundantPHIs.ll b/llvm/test/Transforms/NewGVN/2010-03-31-RedundantPHIs.ll
index 0ff199182f1a2..54a23de6d41f9 100644
--- a/llvm/test/Transforms/NewGVN/2010-03-31-RedundantPHIs.ll
+++ b/llvm/test/Transforms/NewGVN/2010-03-31-RedundantPHIs.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 
 ; CHECK-NOT: load
 ; CHECK-NOT: phi
diff --git a/llvm/test/Transforms/NewGVN/2010-11-13-Simplify.ll b/llvm/test/Transforms/NewGVN/2010-11-13-Simplify.ll
index 635c4b801b4cf..127afa87f9b68 100644
--- a/llvm/test/Transforms/NewGVN/2010-11-13-Simplify.ll
+++ b/llvm/test/Transforms/NewGVN/2010-11-13-Simplify.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 
 declare i32 @foo(i32) readnone
 
diff --git a/llvm/test/Transforms/NewGVN/2011-09-07-TypeIdFor.ll b/llvm/test/Transforms/NewGVN/2011-09-07-TypeIdFor.ll
index 719ce6d5f3895..18f91c66a0c6c 100644
--- a/llvm/test/Transforms/NewGVN/2011-09-07-TypeIdFor.ll
+++ b/llvm/test/Transforms/NewGVN/2011-09-07-TypeIdFor.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 %struct.__fundamental_type_info_pseudo = type { %struct.__type_info_pseudo }
 %struct.__type_info_pseudo = type { i8*, i8* }
 
diff --git a/llvm/test/Transforms/NewGVN/2016-08-30-MaskedScatterGather.ll b/llvm/test/Transforms/NewGVN/2016-08-30-MaskedScatterGather.ll
index b3087c1577ca4..76d5a4362c13a 100644
--- a/llvm/test/Transforms/NewGVN/2016-08-30-MaskedScatterGather.ll
+++ b/llvm/test/Transforms/NewGVN/2016-08-30-MaskedScatterGather.ll
@@ -1,5 +1,5 @@
 ; XFAIL: *
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 
 declare void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> , <2 x i32*> , i32 , <2 x i1> )
 declare <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*>, i32, <2 x i1>, <2 x i32>)
diff --git a/llvm/test/Transforms/NewGVN/MemdepMiscompile.ll b/llvm/test/Transforms/NewGVN/MemdepMiscompile.ll
index 559882cdc4c7e..47c4fd70fbace 100644
--- a/llvm/test/Transforms/NewGVN/MemdepMiscompile.ll
+++ b/llvm/test/Transforms/NewGVN/MemdepMiscompile.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-macosx10.7.0"
 
diff --git a/llvm/test/Transforms/NewGVN/assumes.ll b/llvm/test/Transforms/NewGVN/assumes.ll
index 065cc0fb62e08..ea20b38bff6af 100644
--- a/llvm/test/Transforms/NewGVN/assumes.ll
+++ b/llvm/test/Transforms/NewGVN/assumes.ll
@@ -1,16 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -newgvn -S | FileCheck %s
 
-; CHECK-LABEL: @test1
-; CHECK: ret i32 %arg
 define i32 @test1(i32 %arg) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[ARG:%.*]], 5
+; CHECK-NEXT:    call void @llvm.assume(i1 true)
+; CHECK-NEXT:    ret i32 [[ARG]]
+;
   %cmp = icmp sge i32 %arg, 5
   call void @llvm.assume(i1 %cmp)
   ret i32 %arg
 }
 
-; CHECK-LABEL: @test2
-; CHECK: ret i32 %arg
 define i32 @test2(i32 %arg, i1 %b) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 1, [[TMP0:%.*]] ], [ 2, [[BB]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[ARG:%.*]], [[A]]
+; CHECK-NEXT:    call void @llvm.assume(i1 true)
+; CHECK-NEXT:    br i1 [[B:%.*]], label [[BB]], label [[END:%.*]]
+; CHECK:       end:
+; CHECK-NEXT:    ret i32 [[ARG]]
+;
   br label %bb
 
 bb:
diff --git a/llvm/test/Transforms/NewGVN/basic-cyclic-opt.ll b/llvm/test/Transforms/NewGVN/basic-cyclic-opt.ll
index a712a69546273..1a717a55c5bbc 100644
--- a/llvm/test/Transforms/NewGVN/basic-cyclic-opt.ll
+++ b/llvm/test/Transforms/NewGVN/basic-cyclic-opt.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 ;; Function Attrs: nounwind ssp uwtable
diff --git a/llvm/test/Transforms/NewGVN/basic-undef-test.ll b/llvm/test/Transforms/NewGVN/basic-undef-test.ll
index 681e77b2629ee..e6e0beaafad8b 100644
--- a/llvm/test/Transforms/NewGVN/basic-undef-test.ll
+++ b/llvm/test/Transforms/NewGVN/basic-undef-test.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -newgvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -newgvn -S < %s | FileCheck %s
 ; ModuleID = 'test3.ll'
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/NewGVN/calloc-load-removal.ll b/llvm/test/Transforms/NewGVN/calloc-load-removal.ll
index cdeb971a23e2f..e86df4fb41ada 100644
--- a/llvm/test/Transforms/NewGVN/calloc-load-removal.ll
+++ b/llvm/test/Transforms/NewGVN/calloc-load-removal.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S -basicaa -newgvn < %s | FileCheck %s
-; RUN: opt -S -basicaa -newgvn -disable-simplify-libcalls < %s | FileCheck %s -check-prefix=CHECK_NO_LIBCALLS
+; RUN: opt -S -basic-aa -newgvn < %s | FileCheck %s
+; RUN: opt -S -basic-aa -newgvn -disable-simplify-libcalls < %s | FileCheck %s -check-prefix=CHECK_NO_LIBCALLS
 ; Check that loads from calloc are recognized as being zero.
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/NewGVN/calls-nonlocal.ll b/llvm/test/Transforms/NewGVN/calls-nonlocal.ll
index 6e918050d5910..a8e191822e4ab 100644
--- a/llvm/test/Transforms/NewGVN/calls-nonlocal.ll
+++ b/llvm/test/Transforms/NewGVN/calls-nonlocal.ll
@@ -2,7 +2,7 @@
 ;; NewGVN zaps the strlens, but currently takes two iterations to evaluate the conditions, because
 ;; we prune predicateinfo, and the icmps only become equivalent after the strlens are zapped
 ; Two occurrences of strlen should be zapped.
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i386-apple-darwin9"
 
diff --git a/llvm/test/Transforms/NewGVN/calls-readonly.ll b/llvm/test/Transforms/NewGVN/calls-readonly.ll
index 8bc3bf2b48cc1..8150eb1299745 100644
--- a/llvm/test/Transforms/NewGVN/calls-readonly.ll
+++ b/llvm/test/Transforms/NewGVN/calls-readonly.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 ; Should delete the second call to strlen even though the intervening strchr call exists.
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
diff --git a/llvm/test/Transforms/NewGVN/completeness.ll b/llvm/test/Transforms/NewGVN/completeness.ll
index dd273c73b86a2..3d9147e11afa2 100644
--- a/llvm/test/Transforms/NewGVN/completeness.ll
+++ b/llvm/test/Transforms/NewGVN/completeness.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -enable-phi-of-ops=true -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -enable-phi-of-ops=true -S | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 define i32 @test1(i32, i8**) {
diff --git a/llvm/test/Transforms/NewGVN/cond_br.ll b/llvm/test/Transforms/NewGVN/cond_br.ll
index a2584f3aa295d..632af4b3dc891 100644
--- a/llvm/test/Transforms/NewGVN/cond_br.ll
+++ b/llvm/test/Transforms/NewGVN/cond_br.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -newgvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -newgvn -S < %s | FileCheck %s
 @y = external global i32
 @z = external global i32
 
diff --git a/llvm/test/Transforms/NewGVN/cond_br2.ll b/llvm/test/Transforms/NewGVN/cond_br2.ll
index cb537565a9c74..1f08bb950556f 100644
--- a/llvm/test/Transforms/NewGVN/cond_br2.ll
+++ b/llvm/test/Transforms/NewGVN/cond_br2.ll
@@ -1,5 +1,5 @@
 ; XFAIL: *
-; RUN: opt -basicaa -newgvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -newgvn -S < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 %"class.llvm::SmallVector" = type { %"class.llvm::SmallVectorImpl", [1 x %"union.llvm::SmallVectorBase::U"] }
diff --git a/llvm/test/Transforms/NewGVN/condprop-xfail.ll b/llvm/test/Transforms/NewGVN/condprop-xfail.ll
index 5c049617f8751..8224cf9ef8aa8 100644
--- a/llvm/test/Transforms/NewGVN/condprop-xfail.ll
+++ b/llvm/test/Transforms/NewGVN/condprop-xfail.ll
@@ -1,5 +1,5 @@
 ; XFAIL: *
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 
 @a = external global i32		; <i32*> [#uses=7]
 
diff --git a/llvm/test/Transforms/NewGVN/condprop.ll b/llvm/test/Transforms/NewGVN/condprop.ll
index 6eb9bb6b26194..faef4b171e283 100644
--- a/llvm/test/Transforms/NewGVN/condprop.ll
+++ b/llvm/test/Transforms/NewGVN/condprop.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 
 
 declare void @foo(i1)
diff --git a/llvm/test/Transforms/NewGVN/crash-no-aa.ll b/llvm/test/Transforms/NewGVN/crash-no-aa.ll
index d511422e7979a..739fbb683b3d3 100644
--- a/llvm/test/Transforms/NewGVN/crash-no-aa.ll
+++ b/llvm/test/Transforms/NewGVN/crash-no-aa.ll
@@ -1,4 +1,4 @@
-; RUN: opt -disable-basicaa -newgvn -S < %s
+; RUN: opt -disable-basic-aa -newgvn -S < %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-freebsd8.0"
diff --git a/llvm/test/Transforms/NewGVN/cyclic-phi-handling.ll b/llvm/test/Transforms/NewGVN/cyclic-phi-handling.ll
index 283c78548995d..18a9d2eb889c4 100644
--- a/llvm/test/Transforms/NewGVN/cyclic-phi-handling.ll
+++ b/llvm/test/Transforms/NewGVN/cyclic-phi-handling.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 define void @foo(i32 %arg, i32 %arg1, i32 (i32, i32)* %arg2) {
diff --git a/llvm/test/Transforms/NewGVN/deadstore.ll b/llvm/test/Transforms/NewGVN/deadstore.ll
index 778f42b77581a..808b90b1416d8 100644
--- a/llvm/test/Transforms/NewGVN/deadstore.ll
+++ b/llvm/test/Transforms/NewGVN/deadstore.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 
 ;; Most of these are borrowed from transforms/DSE/simple.ll
 ;; NewGVN should be able to eliminate any stores of the same value that are actually redundnat.
diff --git a/llvm/test/Transforms/NewGVN/equivalent-phi.ll b/llvm/test/Transforms/NewGVN/equivalent-phi.ll
index 34d916b9cd33d..e0ea4cfb7f8c7 100644
--- a/llvm/test/Transforms/NewGVN/equivalent-phi.ll
+++ b/llvm/test/Transforms/NewGVN/equivalent-phi.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 @global = common global [1024 x i32] zeroinitializer, align 16
diff --git a/llvm/test/Transforms/NewGVN/fence.ll b/llvm/test/Transforms/NewGVN/fence.ll
index 190fd7344922b..f69f91334b83d 100644
--- a/llvm/test/Transforms/NewGVN/fence.ll
+++ b/llvm/test/Transforms/NewGVN/fence.ll
@@ -1,5 +1,5 @@
 ; XFAIL: *
-; RUN: opt -S -basicaa -newgvn < %s | FileCheck %s
+; RUN: opt -S -basic-aa -newgvn < %s | FileCheck %s
 
 @a = external constant i32
 ; We can value forward across the fence since we can (semantically) 
diff --git a/llvm/test/Transforms/NewGVN/funclet.ll b/llvm/test/Transforms/NewGVN/funclet.ll
index 44cb3ae0c360b..f995373ffb357 100644
--- a/llvm/test/Transforms/NewGVN/funclet.ll
+++ b/llvm/test/Transforms/NewGVN/funclet.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -newgvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -newgvn -S < %s | FileCheck %s
 target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
 target triple = "i686-pc-windows-msvc"
 
diff --git a/llvm/test/Transforms/NewGVN/lifetime-simple.ll b/llvm/test/Transforms/NewGVN/lifetime-simple.ll
index 382c7da2b3fba..201c3b96e67ce 100644
--- a/llvm/test/Transforms/NewGVN/lifetime-simple.ll
+++ b/llvm/test/Transforms/NewGVN/lifetime-simple.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i386-apple-darwin7"
diff --git a/llvm/test/Transforms/NewGVN/load-constant-mem.ll b/llvm/test/Transforms/NewGVN/load-constant-mem.ll
index 4c1624e09f600..f2818594b14c3 100644
--- a/llvm/test/Transforms/NewGVN/load-constant-mem.ll
+++ b/llvm/test/Transforms/NewGVN/load-constant-mem.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 ; PR4189
 @G = external constant [4 x i32]
 
diff --git a/llvm/test/Transforms/NewGVN/loadforward.ll b/llvm/test/Transforms/NewGVN/loadforward.ll
index b4cbcc6b0f4db..b348004708170 100644
--- a/llvm/test/Transforms/NewGVN/loadforward.ll
+++ b/llvm/test/Transforms/NewGVN/loadforward.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 %rec11 = type { i16, i16, i16 }
diff --git a/llvm/test/Transforms/NewGVN/malloc-load-removal.ll b/llvm/test/Transforms/NewGVN/malloc-load-removal.ll
index c62bac950df25..ca2f019d0686b 100644
--- a/llvm/test/Transforms/NewGVN/malloc-load-removal.ll
+++ b/llvm/test/Transforms/NewGVN/malloc-load-removal.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S -basicaa -newgvn < %s | FileCheck %s
-; RUN: opt -S -basicaa -newgvn -disable-simplify-libcalls < %s | FileCheck %s -check-prefix=CHECK_NO_LIBCALLS
+; RUN: opt -S -basic-aa -newgvn < %s | FileCheck %s
+; RUN: opt -S -basic-aa -newgvn -disable-simplify-libcalls < %s | FileCheck %s -check-prefix=CHECK_NO_LIBCALLS
 ; PR13694
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/NewGVN/memory-handling.ll b/llvm/test/Transforms/NewGVN/memory-handling.ll
index 79197a3dba262..a34cf66989076 100644
--- a/llvm/test/Transforms/NewGVN/memory-handling.ll
+++ b/llvm/test/Transforms/NewGVN/memory-handling.ll
@@ -1,6 +1,6 @@
 ;; This test is really dependent on propagating a lot of memory info around, but in the end, not
 ;; screwing up a single add.
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 %struct.Letter = type { i32, i32, i32, i32 }
diff --git a/llvm/test/Transforms/NewGVN/noalias.ll b/llvm/test/Transforms/NewGVN/noalias.ll
index a9b2ab40272b8..c5f23bfad89a5 100644
--- a/llvm/test/Transforms/NewGVN/noalias.ll
+++ b/llvm/test/Transforms/NewGVN/noalias.ll
@@ -1,4 +1,4 @@
-; RUN: opt -scoped-noalias -basicaa -newgvn -S < %s | FileCheck %s
+; RUN: opt -scoped-noalias -basic-aa -newgvn -S < %s | FileCheck %s
 
 define i32 @test1(i32* %p, i32* %q) {
 ; CHECK-LABEL: @test1(i32* %p, i32* %q)
diff --git a/llvm/test/Transforms/NewGVN/non-local-offset.ll b/llvm/test/Transforms/NewGVN/non-local-offset.ll
index 4a934127e7cc1..f38d106a10507 100644
--- a/llvm/test/Transforms/NewGVN/non-local-offset.ll
+++ b/llvm/test/Transforms/NewGVN/non-local-offset.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -newgvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -newgvn -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64"
 
diff --git a/llvm/test/Transforms/NewGVN/nonescaping-malloc.ll b/llvm/test/Transforms/NewGVN/nonescaping-malloc.ll
index 678211cf15e56..39b5d3658d286 100644
--- a/llvm/test/Transforms/NewGVN/nonescaping-malloc.ll
+++ b/llvm/test/Transforms/NewGVN/nonescaping-malloc.ll
@@ -1,6 +1,6 @@
 ; XFAIL: *
 ; REQUIRES: asserts
-; RUN: opt < %s -basicaa -newgvn -stats -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -stats -disable-output 2>&1 | FileCheck %s
 ; rdar://7363102
 
 ; CHECK: Number of loads deleted
diff --git a/llvm/test/Transforms/NewGVN/null-aliases-nothing.ll b/llvm/test/Transforms/NewGVN/null-aliases-nothing.ll
index 801832bf041d1..95f48c5fc838d 100644
--- a/llvm/test/Transforms/NewGVN/null-aliases-nothing.ll
+++ b/llvm/test/Transforms/NewGVN/null-aliases-nothing.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 
 %t = type { i32 }
 declare void @test1f(i8*)
diff --git a/llvm/test/Transforms/NewGVN/phi-translate-partial-alias.ll b/llvm/test/Transforms/NewGVN/phi-translate-partial-alias.ll
index cd4a9a2ff01bf..aea09e70f02da 100644
--- a/llvm/test/Transforms/NewGVN/phi-translate-partial-alias.ll
+++ b/llvm/test/Transforms/NewGVN/phi-translate-partial-alias.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -newgvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -newgvn -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-f128:128:128-n8:16:32:64"
 
diff --git a/llvm/test/Transforms/NewGVN/pr10820.ll b/llvm/test/Transforms/NewGVN/pr10820.ll
index dbb1376874db6..bd676f855c99b 100644
--- a/llvm/test/Transforms/NewGVN/pr10820.ll
+++ b/llvm/test/Transforms/NewGVN/pr10820.ll
@@ -1,5 +1,5 @@
 ; XFAIL: *
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 ; NewGVN fails this due to missing load coercion
 target datalayout =
 "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-f128:128:128-n8:16:32:64"
diff --git a/llvm/test/Transforms/NewGVN/pr14166.ll b/llvm/test/Transforms/NewGVN/pr14166.ll
index c526c50bc75d5..0ddd99cb8f327 100644
--- a/llvm/test/Transforms/NewGVN/pr14166.ll
+++ b/llvm/test/Transforms/NewGVN/pr14166.ll
@@ -1,5 +1,5 @@
 ; XFAIL: *
-; RUN: opt -disable-basicaa -newgvn -S < %s | FileCheck %s
+; RUN: opt -disable-basic-aa -newgvn -S < %s | FileCheck %s
 ; NewGVN fails this due to missing load coercion
 target datalayout = "e-p:32:32:32"
 target triple = "i386-pc-linux-gnu"
diff --git a/llvm/test/Transforms/NewGVN/pr17852.ll b/llvm/test/Transforms/NewGVN/pr17852.ll
index 31eb7c9ceac98..a7907bd2f6d45 100644
--- a/llvm/test/Transforms/NewGVN/pr17852.ll
+++ b/llvm/test/Transforms/NewGVN/pr17852.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -newgvn
+; RUN: opt < %s -basic-aa -newgvn
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 %struct.S0 = type { [2 x i8], [2 x i8], [4 x i8], [2 x i8], i32, i32, i32, i32 }
 define void @fn1(%struct.S0* byval align 8 %p1) {
diff --git a/llvm/test/Transforms/NewGVN/pr24397.ll b/llvm/test/Transforms/NewGVN/pr24397.ll
index 247eaa2e1bc1d..32d164c939395 100644
--- a/llvm/test/Transforms/NewGVN/pr24397.ll
+++ b/llvm/test/Transforms/NewGVN/pr24397.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -newgvn -disable-output < %s
+; RUN: opt -basic-aa -newgvn -disable-output < %s
 
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/NewGVN/pr31472.ll b/llvm/test/Transforms/NewGVN/pr31472.ll
index db3ed29453cde..0ec20e6708850 100644
--- a/llvm/test/Transforms/NewGVN/pr31472.ll
+++ b/llvm/test/Transforms/NewGVN/pr31472.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.12.0"
 
diff --git a/llvm/test/Transforms/NewGVN/pr31483.ll b/llvm/test/Transforms/NewGVN/pr31483.ll
index 640d2a61789f9..999ff74ad5881 100644
--- a/llvm/test/Transforms/NewGVN/pr31483.ll
+++ b/llvm/test/Transforms/NewGVN/pr31483.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
 
 @global = external hidden unnamed_addr constant [11 x i8], align 1
diff --git a/llvm/test/Transforms/NewGVN/pr31491.ll b/llvm/test/Transforms/NewGVN/pr31491.ll
index 6f190cef2ae28..3409f4d5b173a 100644
--- a/llvm/test/Transforms/NewGVN/pr31491.ll
+++ b/llvm/test/Transforms/NewGVN/pr31491.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 ;; Test that we do not infinite loop on this testcase, and that we do not try
 ;; to replace the phi node argument with the result of the phi node.
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/NewGVN/pr31501.ll b/llvm/test/Transforms/NewGVN/pr31501.ll
index fad10110cc10c..67f14026604c6 100644
--- a/llvm/test/Transforms/NewGVN/pr31501.ll
+++ b/llvm/test/Transforms/NewGVN/pr31501.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 %struct.foo = type { %struct.wombat.28*, %struct.zot, %struct.wombat.28* }
diff --git a/llvm/test/Transforms/NewGVN/pr31573.ll b/llvm/test/Transforms/NewGVN/pr31573.ll
index 0450b4b1299be..726579efea9f8 100644
--- a/llvm/test/Transforms/NewGVN/pr31573.ll
+++ b/llvm/test/Transforms/NewGVN/pr31573.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/NewGVN/pr31594.ll b/llvm/test/Transforms/NewGVN/pr31594.ll
index 7228487ab8924..72144f8f44a0e 100644
--- a/llvm/test/Transforms/NewGVN/pr31594.ll
+++ b/llvm/test/Transforms/NewGVN/pr31594.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/NewGVN/pr31613.ll b/llvm/test/Transforms/NewGVN/pr31613.ll
index 8cf9ae889d484..d545c1a93514a 100644
--- a/llvm/test/Transforms/NewGVN/pr31613.ll
+++ b/llvm/test/Transforms/NewGVN/pr31613.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -enable-store-refinement -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -enable-store-refinement -S | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 ;; Both of these tests are tests of phi nodes that end up all equivalent to each other
diff --git a/llvm/test/Transforms/NewGVN/pr31682.ll b/llvm/test/Transforms/NewGVN/pr31682.ll
index 96103fad15c25..a4b612e01956a 100644
--- a/llvm/test/Transforms/NewGVN/pr31682.ll
+++ b/llvm/test/Transforms/NewGVN/pr31682.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 %struct.foo = type { i32, i32, [2 x [4 x [6 x [6 x i16]]]] }
diff --git a/llvm/test/Transforms/NewGVN/pr33196.ll b/llvm/test/Transforms/NewGVN/pr33196.ll
index 140965ae00519..e117537c93ec1 100644
--- a/llvm/test/Transforms/NewGVN/pr33196.ll
+++ b/llvm/test/Transforms/NewGVN/pr33196.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -newgvn %s | FileCheck %s
+; RUN: opt -S -basic-aa -newgvn %s | FileCheck %s
 
 ; CHECK: define i32 @main() {
 ; CHECK-NEXT: entry:
diff --git a/llvm/test/Transforms/NewGVN/pre-new-inst.ll b/llvm/test/Transforms/NewGVN/pre-new-inst.ll
index 2b6fc2f2c05ff..d6c2a746bd7e5 100644
--- a/llvm/test/Transforms/NewGVN/pre-new-inst.ll
+++ b/llvm/test/Transforms/NewGVN/pre-new-inst.ll
@@ -1,5 +1,5 @@
 ; XFAIL: *
-; RUN: opt -basicaa -newgvn -S %s | FileCheck %s
+; RUN: opt -basic-aa -newgvn -S %s | FileCheck %s
 
 %MyStruct = type { i32, i32 }
 define i8 @foo(i64 %in, i8* %arr) {
diff --git a/llvm/test/Transforms/NewGVN/predicates.ll b/llvm/test/Transforms/NewGVN/predicates.ll
index 61b35c5e5c67c..0bcd655783067 100644
--- a/llvm/test/Transforms/NewGVN/predicates.ll
+++ b/llvm/test/Transforms/NewGVN/predicates.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -basicaa -newgvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -newgvn -S < %s | FileCheck %s
 
 ; Function Attrs: noinline norecurse nounwind readonly ssp uwtable
 define i32 @mp_unsgn_cmp(i32 %n, i32* nocapture readonly %in1, i32* nocapture readonly %in2) local_unnamed_addr {
diff --git a/llvm/test/Transforms/NewGVN/range.ll b/llvm/test/Transforms/NewGVN/range.ll
index 76d2d11c4c832..76daf7c7d13bf 100644
--- a/llvm/test/Transforms/NewGVN/range.ll
+++ b/llvm/test/Transforms/NewGVN/range.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -newgvn -S < %s | FileCheck %s
+; RUN: opt -basic-aa -newgvn -S < %s | FileCheck %s
 
 define i32 @test1(i32* %p) {
 ; CHECK-LABEL: @test1(i32* %p)
diff --git a/llvm/test/Transforms/NewGVN/refine-stores.ll b/llvm/test/Transforms/NewGVN/refine-stores.ll
index e49e324c6e67a..c11065f651877 100644
--- a/llvm/test/Transforms/NewGVN/refine-stores.ll
+++ b/llvm/test/Transforms/NewGVN/refine-stores.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 ;; Now that we do store refinement, we have to verify that we add fake uses
 ;; when we skip existing stores.
 ;; We also are testing that various variations that cause stores to move classes
diff --git a/llvm/test/Transforms/NewGVN/rle-must-alias.ll b/llvm/test/Transforms/NewGVN/rle-must-alias.ll
index 7e4941754dd98..e8c487a3b1613 100644
--- a/llvm/test/Transforms/NewGVN/rle-must-alias.ll
+++ b/llvm/test/Transforms/NewGVN/rle-must-alias.ll
@@ -1,5 +1,5 @@
 ; XFAIL: *
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 
 ; GVN should eliminate the fully redundant %9 GEP which 
 ; allows DEAD to be removed.  This is PR3198.
diff --git a/llvm/test/Transforms/NewGVN/rle-nonlocal.ll b/llvm/test/Transforms/NewGVN/rle-nonlocal.ll
index d318cd5240d81..4f4fc930f96b1 100644
--- a/llvm/test/Transforms/NewGVN/rle-nonlocal.ll
+++ b/llvm/test/Transforms/NewGVN/rle-nonlocal.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
 
 define i32 @main(i32** %p, i32 %x, i32 %y) {
 ; CHECK-LABEL: @main(
diff --git a/llvm/test/Transforms/NewGVN/rle.ll b/llvm/test/Transforms/NewGVN/rle.ll
index f6e96ebd1b15a..6a2becc88344a 100644
--- a/llvm/test/Transforms/NewGVN/rle.ll
+++ b/llvm/test/Transforms/NewGVN/rle.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -data-layout="e-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-n8:16:32" -basicaa -newgvn -S -die | FileCheck %s
-; RUN: opt < %s -data-layout="E-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-n32"      -basicaa -newgvn -S -die | FileCheck %s
+; RUN: opt < %s -data-layout="e-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-n8:16:32" -basic-aa -newgvn -S -die | FileCheck %s
+; RUN: opt < %s -data-layout="E-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-n32"      -basic-aa -newgvn -S -die | FileCheck %s
 ; memset -> i16 forwarding.
 define signext i16 @memset_to_i16_local(i16* %A) nounwind ssp {
 entry:
diff --git a/llvm/test/Transforms/NewGVN/tbaa.ll b/llvm/test/Transforms/NewGVN/tbaa.ll
index d48ededac03aa..b461e195781be 100644
--- a/llvm/test/Transforms/NewGVN/tbaa.ll
+++ b/llvm/test/Transforms/NewGVN/tbaa.ll
@@ -1,4 +1,4 @@
-; RUN: opt -tbaa -basicaa -newgvn -S < %s | FileCheck %s
+; RUN: opt -tbaa -basic-aa -newgvn -S < %s | FileCheck %s
 
 define i32 @test1(i8* %p, i8* %q) {
 ; CHECK-LABEL: @test1(i8* %p, i8* %q)
diff --git a/llvm/test/Transforms/ObjCARC/basic.ll b/llvm/test/Transforms/ObjCARC/basic.ll
index 5c4f1d143faac..cc1120b74b674 100644
--- a/llvm/test/Transforms/ObjCARC/basic.ll
+++ b/llvm/test/Transforms/ObjCARC/basic.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -objc-arc -S < %s | FileCheck %s
+; RUN: opt -basic-aa -objc-arc -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64"
 
diff --git a/llvm/test/Transforms/ObjCARC/gvn.ll b/llvm/test/Transforms/ObjCARC/gvn.ll
index f2977d0c513e8..fb9a71f14727b 100644
--- a/llvm/test/Transforms/ObjCARC/gvn.ll
+++ b/llvm/test/Transforms/ObjCARC/gvn.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -objc-arc-aa -gvn < %s | FileCheck %s
+; RUN: opt -S -basic-aa -objc-arc-aa -gvn < %s | FileCheck %s
 
 @x = common global i8* null, align 8
 
diff --git a/llvm/test/Transforms/ObjCARC/inlined-autorelease-return-value.ll b/llvm/test/Transforms/ObjCARC/inlined-autorelease-return-value.ll
index 84d33193ece68..2b153914c38bc 100644
--- a/llvm/test/Transforms/ObjCARC/inlined-autorelease-return-value.ll
+++ b/llvm/test/Transforms/ObjCARC/inlined-autorelease-return-value.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -objc-arc -S < %s | FileCheck %s
+; RUN: opt -basic-aa -objc-arc -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64"
 
diff --git a/llvm/test/Transforms/ObjCARC/intrinsic-use.ll b/llvm/test/Transforms/ObjCARC/intrinsic-use.ll
index 8a4ac52d2a6db..6f3815113d698 100644
--- a/llvm/test/Transforms/ObjCARC/intrinsic-use.ll
+++ b/llvm/test/Transforms/ObjCARC/intrinsic-use.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa -objc-arc -S < %s | FileCheck %s
+; RUN: opt -basic-aa -objc-arc -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64"
 
diff --git a/llvm/test/Transforms/ObjCARC/provenance.ll b/llvm/test/Transforms/ObjCARC/provenance.ll
index 2587c11d01edf..340c66a53cad2 100644
--- a/llvm/test/Transforms/ObjCARC/provenance.ll
+++ b/llvm/test/Transforms/ObjCARC/provenance.ll
@@ -1,4 +1,4 @@
-; RUN: opt -disable-output -disable-basicaa -pa-eval %s 2>&1 | FileCheck %s
+; RUN: opt -disable-output -disable-basic-aa -pa-eval %s 2>&1 | FileCheck %s
 
 @"\01l_objc_msgSend_fixup_" = global i8 0
 @g1 = global i8 0, section "__OBJC,__message_refs,literal_pointers,no_dead_strip"
diff --git a/llvm/test/Transforms/ObjCARC/retain-block-side-effects.ll b/llvm/test/Transforms/ObjCARC/retain-block-side-effects.ll
index a980ffd63a34c..1ef59c7c89f96 100644
--- a/llvm/test/Transforms/ObjCARC/retain-block-side-effects.ll
+++ b/llvm/test/Transforms/ObjCARC/retain-block-side-effects.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -objc-arc-aa -basicaa -gvn < %s | FileCheck %s
+; RUN: opt -S -objc-arc-aa -basic-aa -gvn < %s | FileCheck %s
 ; rdar://10050579
 
 ; objc_retainBlock stores into %repeater so the load from after the
diff --git a/llvm/test/Transforms/ObjCARC/rle-s2l.ll b/llvm/test/Transforms/ObjCARC/rle-s2l.ll
index 5bf63f20aa08c..f3b2db4ff4b79 100644
--- a/llvm/test/Transforms/ObjCARC/rle-s2l.ll
+++ b/llvm/test/Transforms/ObjCARC/rle-s2l.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -objc-arc < %s | FileCheck %s
+; RUN: opt -S -basic-aa -objc-arc < %s | FileCheck %s
 
 declare i8* @llvm.objc.loadWeak(i8**)
 declare i8* @llvm.objc.loadWeakRetained(i8**)
diff --git a/llvm/test/Transforms/ObjCARC/weak-copies.ll b/llvm/test/Transforms/ObjCARC/weak-copies.ll
index 440f3fb2bb3f7..906c2fc9fc2c2 100644
--- a/llvm/test/Transforms/ObjCARC/weak-copies.ll
+++ b/llvm/test/Transforms/ObjCARC/weak-copies.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -objc-arc < %s | FileCheck %s
+; RUN: opt -S -basic-aa -objc-arc < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-darwin11.0.0"
diff --git a/llvm/test/Transforms/ObjCARC/weak-dce.ll b/llvm/test/Transforms/ObjCARC/weak-dce.ll
index e499ac13cf909..b0ebbc52d79e9 100644
--- a/llvm/test/Transforms/ObjCARC/weak-dce.ll
+++ b/llvm/test/Transforms/ObjCARC/weak-dce.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -objc-arc < %s | FileCheck %s
+; RUN: opt -S -basic-aa -objc-arc < %s | FileCheck %s
 ; rdar://11434915
 
 ; Delete the weak calls and replace them with just the net retain.
diff --git a/llvm/test/Transforms/OpenMP/dead_use.ll b/llvm/test/Transforms/OpenMP/dead_use.ll
new file mode 100644
index 0000000000000..4aca7935b10c1
--- /dev/null
+++ b/llvm/test/Transforms/OpenMP/dead_use.ll
@@ -0,0 +1,73 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
+; RUN: opt -S -openmpopt < %s | FileCheck %s
+; RUN: opt -S -passes=openmpopt < %s | FileCheck %s
+%struct.ident_t = type { i32, i32, i32, i32, i8* }
+
+@.str = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00", align 1
+@0 = private unnamed_addr global %struct.ident_t { i32 0, i32 2, i32 0, i32 0, i8* getelementptr inbounds ([23 x i8], [23 x i8]* @.str, i32 0, i32 0) }, align 8
+
+; Function Attrs: nounwind uwtable
+define dso_local i32 @b() #0 {
+; CHECK-LABEL: define {{[^@]+}}@b() #0
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @a()
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]], align 4
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = alloca i32, align 4
+  %2 = call i32 @a()
+  %3 = load i32, i32* %1, align 4
+  ret i32 %3
+}
+
+; Function Attrs: nounwind uwtable
+define internal i32 @a() #0 {
+; CHECK-LABEL: define {{[^@]+}}@a() #0
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @b()
+; CHECK-NEXT:    call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* @0, i32 0, void (i32*, i32*, ...)* bitcast (void (i32*, i32*)* @.omp_outlined. to void (i32*, i32*, ...)*))
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]], align 4
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = alloca i32, align 4
+  %2 = call i32 @b()
+  call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* @0, i32 0, void (i32*, i32*, ...)* bitcast (void (i32*, i32*)* @.omp_outlined. to void (i32*, i32*, ...)*))
+  %3 = load i32, i32* %1, align 4
+  ret i32 %3
+}
+
+; Function Attrs: norecurse nounwind uwtable
+define internal void @.omp_outlined.(i32* noalias %0, i32* noalias %1) #1 {
+; CHECK-LABEL: define {{[^@]+}}@.omp_outlined.
+; CHECK-SAME: (i32* noalias [[TMP0:%.*]], i32* noalias [[TMP1:%.*]]) #1
+; CHECK-NEXT:    [[TMP3:%.*]] = alloca i32*, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = alloca i32*, align 8
+; CHECK-NEXT:    store i32* [[TMP0]], i32** [[TMP3]], align 8, !tbaa !2
+; CHECK-NEXT:    store i32* [[TMP1]], i32** [[TMP4]], align 8, !tbaa !2
+; CHECK-NEXT:    ret void
+;
+  %3 = alloca i32*, align 8
+  %4 = alloca i32*, align 8
+  store i32* %0, i32** %3, align 8, !tbaa !2
+  store i32* %1, i32** %4, align 8, !tbaa !2
+  ret void
+}
+
+; Function Attrs: nounwind
+declare !callback !6 void @__kmpc_fork_call(%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) #2
+
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { norecurse nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"Debian clang version 11.0.0-++20200709100646+c92a8c0a0f6-1~exp1~20200709201313.3348"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"any pointer", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = !{!7}
+!7 = !{i64 2, i64 -1, i64 -1, i1 true}
diff --git a/llvm/test/Transforms/OpenMP/gpu_kernel_detection_remarks.ll b/llvm/test/Transforms/OpenMP/gpu_kernel_detection_remarks.ll
new file mode 100644
index 0000000000000..ccdf0b981dc2c
--- /dev/null
+++ b/llvm/test/Transforms/OpenMP/gpu_kernel_detection_remarks.ll
@@ -0,0 +1,27 @@
+; RUN: opt -passes=openmpopt -pass-remarks=openmp-opt -openmp-print-gpu-kernels -disable-output < %s 2>&1 | FileCheck %s --implicit-check-not=non_kernel
+; RUN: opt        -openmpopt -pass-remarks=openmp-opt -openmp-print-gpu-kernels -disable-output < %s 2>&1 | FileCheck %s --implicit-check-not=non_kernel
+
+; CHECK-DAG: remark: <unknown>:0:0: OpenMP GPU kernel kernel1
+; CHECK-DAG: remark: <unknown>:0:0: OpenMP GPU kernel kernel2
+
+define void @kernel1() {
+  ret void
+}
+
+define void @kernel2() {
+  ret void
+}
+
+define void @non_kernel() {
+  ret void
+}
+
+; Needed to trigger the openmp-opt pass
+declare dso_local void @__kmpc_kernel_prepare_parallel(i8*)
+
+!nvvm.annotations = !{!2, !0, !1, !3, !1, !2}
+
+!0 = !{void ()* @kernel1, !"kernel", i32 1}
+!1 = !{void ()* @non_kernel, !"non_kernel", i32 1}
+!2 = !{null, !"align", i32 1}
+!3 = !{void ()* @kernel2, !"kernel", i32 1}
diff --git a/llvm/test/Transforms/OpenMP/gpu_state_machine_function_ptr_replacement.ll b/llvm/test/Transforms/OpenMP/gpu_state_machine_function_ptr_replacement.ll
new file mode 100644
index 0000000000000..0a8d7a9d231ae
--- /dev/null
+++ b/llvm/test/Transforms/OpenMP/gpu_state_machine_function_ptr_replacement.ll
@@ -0,0 +1,153 @@
+; RUN: opt -S -passes=openmpopt -pass-remarks=openmp-opt -openmp-print-gpu-kernels < %s | FileCheck %s
+; RUN: opt -S        -openmpopt -pass-remarks=openmp-opt -openmp-print-gpu-kernels < %s | FileCheck %s
+
+; C input used for this test:
+
+; void bar(void) {
+;     #pragma omp parallel
+;     { }
+; }
+; void foo(void) {
+;   #pragma omp target teams
+;   {
+;     #pragma omp parallel
+;     {}
+;     bar();
+;     #pragma omp parallel
+;     {}
+;   }
+; }
+
+; Verify we replace the function pointer uses for the first and last outlined
+; region (1 and 3) but not for the middle one (2) because it could be called from
+; another kernel.
+
+; CHECK-DAG: @__omp_outlined__1_wrapper.ID = private constant i8 undef
+; CHECK-DAG: @__omp_outlined__3_wrapper.ID = private constant i8 undef
+
+; CHECK-DAG:   icmp eq i8* %5, @__omp_outlined__1_wrapper.ID
+; CHECK-DAG:   icmp eq i8* %7, @__omp_outlined__3_wrapper.ID
+
+; CHECK-DAG:   call void @__kmpc_kernel_prepare_parallel(i8* @__omp_outlined__1_wrapper.ID)
+; CHECK-DAG:   call void @__kmpc_kernel_prepare_parallel(i8* bitcast (void ()* @__omp_outlined__2_wrapper to i8*))
+; CHECK-DAG:   call void @__kmpc_kernel_prepare_parallel(i8* @__omp_outlined__3_wrapper.ID)
+
+
+%struct.ident_t = type { i32, i32, i32, i32, i8* }
+
+define internal void @__omp_offloading_35_a1e179_foo_l7_worker() {
+entry:
+  %work_fn = alloca i8*, align 8
+  %exec_status = alloca i8, align 1
+  store i8* null, i8** %work_fn, align 8
+  store i8 0, i8* %exec_status, align 1
+  br label %.await.work
+
+.await.work:                                      ; preds = %.barrier.parallel, %entry
+  call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
+  %0 = call i1 @__kmpc_kernel_parallel(i8** %work_fn)
+  %1 = zext i1 %0 to i8
+  store i8 %1, i8* %exec_status, align 1
+  %2 = load i8*, i8** %work_fn, align 8
+  %should_terminate = icmp eq i8* %2, null
+  br i1 %should_terminate, label %.exit, label %.select.workers
+
+.select.workers:                                  ; preds = %.await.work
+  %3 = load i8, i8* %exec_status, align 1
+  %is_active = icmp ne i8 %3, 0
+  br i1 %is_active, label %.execute.parallel, label %.barrier.parallel
+
+.execute.parallel:                                ; preds = %.select.workers
+  %4 = call i32 @__kmpc_global_thread_num(%struct.ident_t* null)
+  %5 = load i8*, i8** %work_fn, align 8
+  %work_match = icmp eq i8* %5, bitcast (void ()* @__omp_outlined__1_wrapper to i8*)
+  br i1 %work_match, label %.execute.fn, label %.check.next
+
+.execute.fn:                                      ; preds = %.execute.parallel
+  call void @__omp_outlined__1_wrapper()
+  br label %.terminate.parallel
+
+.check.next:                                      ; preds = %.execute.parallel
+  %6 = load i8*, i8** %work_fn, align 8
+  %work_match1 = icmp eq i8* %6, bitcast (void ()* @__omp_outlined__2_wrapper to i8*)
+  br i1 %work_match1, label %.execute.fn2, label %.check.next3
+
+.execute.fn2:                                     ; preds = %.check.next
+  call void @__omp_outlined__2_wrapper()
+  br label %.terminate.parallel
+
+.check.next3:                                     ; preds = %.check.next
+  %7 = load i8*, i8** %work_fn, align 8
+  %work_match4 = icmp eq i8* %7, bitcast (void ()* @__omp_outlined__3_wrapper to i8*)
+  br i1 %work_match4, label %.execute.fn5, label %.check.next6
+
+.execute.fn5:                                     ; preds = %.check.next3
+  call void @__omp_outlined__3_wrapper()
+  br label %.terminate.parallel
+
+.check.next6:                                     ; preds = %.check.next3
+  %8 = bitcast i8* %2 to void ()*
+  call void %8()
+  br label %.terminate.parallel
+
+.terminate.parallel:                              ; preds = %.check.next6, %.execute.fn5, %.execute.fn2, %.execute.fn
+  call void @__kmpc_kernel_end_parallel()
+  br label %.barrier.parallel
+
+.barrier.parallel:                                ; preds = %.terminate.parallel, %.select.workers
+  call void @__kmpc_barrier_simple_spmd(%struct.ident_t* null, i32 0)
+  br label %.await.work
+
+.exit:                                            ; preds = %.await.work
+  ret void
+}
+
+define weak void @__omp_offloading_35_a1e179_foo_l7() {
+  call void @__omp_offloading_35_a1e179_foo_l7_worker()
+  call void @__omp_outlined__()
+  ret void
+}
+
+define internal void @__omp_outlined__() {
+  call void @__kmpc_kernel_prepare_parallel(i8* bitcast (void ()* @__omp_outlined__1_wrapper to i8*))
+  call void @bar()
+  call void @__kmpc_kernel_prepare_parallel(i8* bitcast (void ()* @__omp_outlined__3_wrapper to i8*))
+  ret void
+}
+
+define internal void @__omp_outlined__1() {
+  ret void
+}
+
+define internal void @__omp_outlined__1_wrapper() {
+  call void @__omp_outlined__1()
+  ret void
+}
+
+define hidden void @bar() {
+  call void @__kmpc_kernel_prepare_parallel(i8* bitcast (void ()* @__omp_outlined__2_wrapper to i8*))
+  ret void
+}
+
+define internal void @__omp_outlined__2_wrapper() {
+  ret void
+}
+
+define internal void @__omp_outlined__3_wrapper() {
+  ret void
+}
+
+declare void @__kmpc_kernel_prepare_parallel(i8* %WorkFn)
+
+declare zeroext i1 @__kmpc_kernel_parallel(i8** nocapture %WorkFn)
+
+declare void @__kmpc_kernel_end_parallel()
+
+declare void @__kmpc_barrier_simple_spmd(%struct.ident_t* nocapture readnone %loc_ref, i32 %tid)
+
+declare i32 @__kmpc_global_thread_num(%struct.ident_t* nocapture readnone)
+
+
+!nvvm.annotations = !{!0}
+
+!0 = !{void ()* @__omp_offloading_35_a1e179_foo_l7, !"kernel", i32 1}
diff --git a/llvm/test/Transforms/OpenMP/icv_remarks.ll b/llvm/test/Transforms/OpenMP/icv_remarks.ll
new file mode 100644
index 0000000000000..19fcaac63c482
--- /dev/null
+++ b/llvm/test/Transforms/OpenMP/icv_remarks.ll
@@ -0,0 +1,131 @@
+; RUN: opt -passes=openmpopt -pass-remarks=openmp-opt -openmp-print-icv-values -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt -openmpopt -pass-remarks=openmp-opt -openmp-print-icv-values -disable-output < %s 2>&1 | FileCheck %s
+
+; ModuleID = 'icv_remarks.c'
+source_filename = "icv_remarks.c"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.ident_t = type { i32, i32, i32, i32, i8* }
+
+@.str = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00", align 1
+@0 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 0, i8* getelementptr inbounds ([23 x i8], [23 x i8]* @.str, i32 0, i32 0) }, align 8
+@1 = private unnamed_addr constant [26 x i8] c";icv_remarks.c;foo;18;1;;\00", align 1
+
+; CHECK-DAG: remark: icv_remarks.c:12:0: OpenMP ICV nthreads Value: IMPLEMENTATION_DEFINED
+; CHECK-DAG: remark: icv_remarks.c:12:0: OpenMP ICV active_levels Value: 0
+; CHECK-DAG: remark: icv_remarks.c:12:0: OpenMP ICV cancel Value: 0
+define dso_local void @foo(i32 %a) local_unnamed_addr #0 !dbg !17 {
+entry:
+  %.kmpc_loc.addr = alloca %struct.ident_t, align 8
+  %0 = bitcast %struct.ident_t* %.kmpc_loc.addr to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 8 dereferenceable(24) %0, i8* nonnull align 8 dereferenceable(24) bitcast (%struct.ident_t* @0 to i8*), i64 16, i1 false)
+  call void @llvm.dbg.value(metadata i32 %a, metadata !19, metadata !DIExpression()), !dbg !21
+  tail call void @omp_set_num_threads(i32 %a) #1, !dbg !22
+  %call = tail call i32 @omp_get_max_threads() #1, !dbg !23
+  call void @llvm.dbg.value(metadata i32 %call, metadata !20, metadata !DIExpression()), !dbg !21
+  tail call void @use(i32 %call) #1, !dbg !24
+  %1 = getelementptr inbounds %struct.ident_t, %struct.ident_t* %.kmpc_loc.addr, i64 0, i32 4, !dbg !25
+  store i8* getelementptr inbounds ([26 x i8], [26 x i8]* @1, i64 0, i64 0), i8** %1, align 8, !dbg !25, !tbaa !26
+  call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* nonnull %.kmpc_loc.addr, i32 0, void (i32*, i32*, ...)* bitcast (void (i32*, i32*)* @.omp_outlined. to void (i32*, i32*, ...)*)) #1, !dbg !25
+  ret void, !dbg !32
+}
+
+declare !dbg !4 dso_local void @omp_set_num_threads(i32) local_unnamed_addr #1
+
+declare !dbg !9 dso_local i32 @omp_get_max_threads() local_unnamed_addr #1
+
+declare !dbg !12 dso_local void @use(i32) local_unnamed_addr #2
+
+; CHECK-DAG: remark: icv_remarks.c:18:0: OpenMP ICV nthreads Value: IMPLEMENTATION_DEFINED
+; CHECK-DAG: remark: icv_remarks.c:18:0: OpenMP ICV active_levels Value: 0
+; CHECK-DAG: remark: icv_remarks.c:18:0: OpenMP ICV cancel Value: 0
+define internal void @.omp_outlined.(i32* noalias nocapture readnone %.global_tid., i32* noalias nocapture readnone %.bound_tid.) #3 !dbg !33 {
+entry:
+  call void @llvm.dbg.value(metadata i32* %.global_tid., metadata !41, metadata !DIExpression()), !dbg !43
+  call void @llvm.dbg.value(metadata i32* %.bound_tid., metadata !42, metadata !DIExpression()), !dbg !43
+  call void @llvm.dbg.value(metadata i32* undef, metadata !44, metadata !DIExpression()) #1, !dbg !50
+  call void @llvm.dbg.value(metadata i32* undef, metadata !47, metadata !DIExpression()) #1, !dbg !50
+  tail call void @omp_set_num_threads(i32 10) #1, !dbg !52
+  %call.i = tail call i32 @omp_get_max_threads() #1, !dbg !53
+  call void @llvm.dbg.value(metadata i32 %call.i, metadata !48, metadata !DIExpression()) #1, !dbg !54
+  tail call void @use(i32 %call.i) #1, !dbg !55
+  ret void, !dbg !56
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64, i1 immarg) #4
+
+declare !callback !57 dso_local void @__kmpc_fork_call(%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) local_unnamed_addr #1
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #5
+
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { norecurse nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { argmemonly nounwind willreturn }
+attributes #5 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!13, !14, !15}
+!llvm.ident = !{!16}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0 (https://github.com/llvm/llvm-project.git 73cea83a6f5ab521edf3cccfc603534776d691ec)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "icv_remarks.c", directory: "/tmp")
+!2 = !{}
+!3 = !{!4, !9, !12}
+!4 = !DISubprogram(name: "omp_set_num_threads", scope: !5, file: !5, line: 57, type: !6, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!5 = !DIFile(filename: "/usr/local/lib/clang/11.0.0/include/omp.h", directory: "")
+!6 = !DISubroutineType(types: !7)
+!7 = !{null, !8}
+!8 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!9 = !DISubprogram(name: "omp_get_max_threads", scope: !5, file: !5, line: 67, type: !10, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!10 = !DISubroutineType(types: !11)
+!11 = !{!8}
+!12 = !DISubprogram(name: "use", scope: !1, file: !1, line: 10, type: !6, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!13 = !{i32 7, !"Dwarf Version", i32 4}
+!14 = !{i32 2, !"Debug Info Version", i32 3}
+!15 = !{i32 1, !"wchar_size", i32 4}
+!16 = !{!"clang version 11.0.0 (https://github.com/llvm/llvm-project.git 73cea83a6f5ab521edf3cccfc603534776d691ec)"}
+!17 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 12, type: !6, scopeLine: 12, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !18)
+!18 = !{!19, !20}
+!19 = !DILocalVariable(name: "a", arg: 1, scope: !17, file: !1, line: 12, type: !8)
+!20 = !DILocalVariable(name: "num", scope: !17, file: !1, line: 15, type: !8)
+!21 = !DILocation(line: 0, scope: !17)
+!22 = !DILocation(line: 13, column: 3, scope: !17)
+!23 = !DILocation(line: 15, column: 13, scope: !17)
+!24 = !DILocation(line: 17, column: 3, scope: !17)
+!25 = !DILocation(line: 18, column: 1, scope: !17)
+!26 = !{!27, !31, i64 16}
+!27 = !{!"ident_t", !28, i64 0, !28, i64 4, !28, i64 8, !28, i64 12, !31, i64 16}
+!28 = !{!"int", !29, i64 0}
+!29 = !{!"omnipotent char", !30, i64 0}
+!30 = !{!"Simple C/C++ TBAA"}
+!31 = !{!"any pointer", !29, i64 0}
+!32 = !DILocation(line: 24, column: 1, scope: !17)
+!33 = distinct !DISubprogram(name: ".omp_outlined.", scope: !1, file: !1, line: 18, type: !34, scopeLine: 18, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !40)
+!34 = !DISubroutineType(types: !35)
+!35 = !{null, !36, !36}
+!36 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !37)
+!37 = !DIDerivedType(tag: DW_TAG_restrict_type, baseType: !38)
+!38 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !39, size: 64)
+!39 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !8)
+!40 = !{!41, !42}
+!41 = !DILocalVariable(name: ".global_tid.", arg: 1, scope: !33, type: !36, flags: DIFlagArtificial)
+!42 = !DILocalVariable(name: ".bound_tid.", arg: 2, scope: !33, type: !36, flags: DIFlagArtificial)
+!43 = !DILocation(line: 0, scope: !33)
+!44 = !DILocalVariable(name: ".global_tid.", arg: 1, scope: !45, type: !36, flags: DIFlagArtificial)
+!45 = distinct !DISubprogram(name: ".omp_outlined._debug__", scope: !1, file: !1, line: 19, type: !34, scopeLine: 19, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !46)
+!46 = !{!44, !47, !48}
+!47 = !DILocalVariable(name: ".bound_tid.", arg: 2, scope: !45, type: !36, flags: DIFlagArtificial)
+!48 = !DILocalVariable(name: "num1", scope: !49, file: !1, line: 21, type: !8)
+!49 = distinct !DILexicalBlock(scope: !45, file: !1, line: 19, column: 3)
+!50 = !DILocation(line: 0, scope: !45, inlinedAt: !51)
+!51 = distinct !DILocation(line: 18, column: 1, scope: !33)
+!52 = !DILocation(line: 20, column: 5, scope: !49, inlinedAt: !51)
+!53 = !DILocation(line: 21, column: 16, scope: !49, inlinedAt: !51)
+!54 = !DILocation(line: 0, scope: !49, inlinedAt: !51)
+!55 = !DILocation(line: 22, column: 5, scope: !49, inlinedAt: !51)
+!56 = !DILocation(line: 18, column: 1, scope: !33)
+!57 = !{!58}
+!58 = !{i64 2, i64 -1, i64 -1, i1 true}
diff --git a/llvm/test/Transforms/OpenMP/icv_tracking.ll b/llvm/test/Transforms/OpenMP/icv_tracking.ll
index e3704338a7a94..c2b5d40ce97af 100644
--- a/llvm/test/Transforms/OpenMP/icv_tracking.ll
+++ b/llvm/test/Transforms/OpenMP/icv_tracking.ll
@@ -11,16 +11,12 @@ define dso_local i32 @foo(i32 %0, i32 %1) {
 ; CHECK-LABEL: define {{[^@]+}}@foo
 ; CHECK-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]])
 ; CHECK-NEXT:    tail call void @omp_set_num_threads(i32 [[TMP0]])
-; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @omp_get_max_threads()
 ; CHECK-NEXT:    tail call void @omp_set_num_threads(i32 [[TMP1]])
-; CHECK-NEXT:    [[TMP4:%.*]] = tail call i32 @omp_get_max_threads()
-; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @omp_get_max_threads()
-; CHECK-NEXT:    [[TMP6:%.*]] = tail call i32 @omp_get_max_threads()
-; CHECK-NEXT:    tail call void @use(i32 [[TMP4]])
-; CHECK-NEXT:    tail call void @use(i32 [[TMP5]])
+; CHECK-NEXT:    tail call void @use(i32 [[TMP1]])
+; CHECK-NEXT:    tail call void @use(i32 [[TMP1]])
 ; CHECK-NEXT:    tail call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* nonnull @0, i32 0, void (i32*, i32*, ...)* bitcast (void (i32*, i32*)* @.omp_outlined. to void (i32*, i32*, ...)*))
-; CHECK-NEXT:    [[TMP7:%.*]] = tail call i32 @omp_get_max_threads()
-; CHECK-NEXT:    tail call void @use(i32 [[TMP7]])
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @omp_get_max_threads()
+; CHECK-NEXT:    tail call void @use(i32 [[TMP3]])
 ; CHECK-NEXT:    ret i32 0
 ;
   tail call void @omp_set_num_threads(i32 %0)
@@ -51,15 +47,13 @@ define internal void @.omp_outlined.(i32* %0, i32* %1) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = tail call i32 @omp_get_max_threads()
 ; CHECK-NEXT:    tail call void @use(i32 [[TMP4]])
 ; CHECK-NEXT:    tail call void @omp_set_num_threads(i32 10)
-; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @omp_get_max_threads()
-; CHECK-NEXT:    tail call void @use(i32 [[TMP5]])
+; CHECK-NEXT:    tail call void @use(i32 10)
 ; CHECK-NEXT:    ret void
 ;
 ; FIXME: this value should be tracked and the rest of the getters deduplicated and replaced with it.
   %3 = tail call i32 @omp_get_max_threads()
   %4 = tail call i32 @omp_get_max_threads()
   tail call void @use(i32 %4)
-; FIXME: this value ( min(%3, 10) ) should be tracked and the rest of the getters deduplicated and replaced with it.
   tail call void @omp_set_num_threads(i32 10)
   %5 = tail call i32 @omp_get_max_threads()
   tail call void @use(i32 %5)
@@ -74,10 +68,9 @@ define dso_local i32 @bar(i32 %0, i32 %1) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 [[TMP0]], i32 [[TMP1]]
 ; CHECK-NEXT:    tail call void @omp_set_num_threads(i32 [[TMP4]])
-; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @omp_get_max_threads()
 ; CHECK-NEXT:    tail call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* nonnull @0, i32 0, void (i32*, i32*, ...)* bitcast (void (i32*, i32*)* @.omp_outlined..1 to void (i32*, i32*, ...)*))
-; CHECK-NEXT:    [[TMP6:%.*]] = tail call i32 @omp_get_max_threads()
-; CHECK-NEXT:    tail call void @use(i32 [[TMP6]])
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @omp_get_max_threads()
+; CHECK-NEXT:    tail call void @use(i32 [[TMP5]])
 ; CHECK-NEXT:    ret i32 0
 ;
   %3 = icmp sgt i32 %0, %1
@@ -97,10 +90,9 @@ define internal void @.omp_outlined..1(i32* %0, i32*  %1) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @omp_get_max_threads()
 ; CHECK-NEXT:    tail call void @use(i32 [[TMP3]])
 ; CHECK-NEXT:    tail call void @omp_set_num_threads(i32 10)
+; CHECK-NEXT:    tail call void @use(i32 10)
 ; CHECK-NEXT:    [[TMP4:%.*]] = tail call i32 @omp_get_max_threads()
 ; CHECK-NEXT:    tail call void @use(i32 [[TMP4]])
-; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @omp_get_max_threads()
-; CHECK-NEXT:    tail call void @use(i32 [[TMP5]])
 ; CHECK-NEXT:    ret void
 ;
   %3 = tail call i32 @omp_get_max_threads()
diff --git a/llvm/test/Transforms/PGOProfile/chr.ll b/llvm/test/Transforms/PGOProfile/chr.ll
index 9a6477aeb1018..e2161d617022f 100644
--- a/llvm/test/Transforms/PGOProfile/chr.ll
+++ b/llvm/test/Transforms/PGOProfile/chr.ll
@@ -36,15 +36,15 @@ define void @test_chr_1(i32* %i) !prof !14 {
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       entry.split.nonchr:
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
-; CHECK-NEXT:    br i1 [[TMP4]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
 ; CHECK:       bb0.nonchr:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB1_NONCHR]]
 ; CHECK:       bb1.nonchr:
-; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
-; CHECK-NEXT:    br i1 [[TMP6]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
 ; CHECK:       bb2.nonchr:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB3]]
@@ -109,22 +109,22 @@ define void @test_chr_1_1(i32* %i) !prof !14 {
 ; CHECK-NEXT:    br label [[BB5:%.*]]
 ; CHECK:       entry.split.nonchr:
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
-; CHECK-NEXT:    br i1 [[TMP4]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
 ; CHECK:       bb0.nonchr:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB1_NONCHR]]
 ; CHECK:       bb1.nonchr:
-; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
-; CHECK-NEXT:    br i1 [[TMP6]], label [[BB2_NONCHR:%.*]], label [[BB3_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB2_NONCHR:%.*]], label [[BB3_NONCHR:%.*]], !prof !16
 ; CHECK:       bb2.nonchr:
 ; CHECK-NEXT:    call void @bar()
 ; CHECK-NEXT:    br label [[BB3_NONCHR]]
 ; CHECK:       bb3.nonchr:
-; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[TMP8]], label [[BB5]], label [[BB4_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TMP7]], label [[BB5]], label [[BB4_NONCHR:%.*]], !prof !16
 ; CHECK:       bb4.nonchr:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB5]]
@@ -195,16 +195,16 @@ define void @test_chr_2(i32* %i) !prof !14 {
 ; CHECK-NEXT:    br label [[BB4:%.*]]
 ; CHECK:       entry.split.nonchr:
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP0]], 255
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
-; CHECK-NEXT:    br i1 [[TMP4]], label [[BB4]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[BB4]], label [[BB0_NONCHR:%.*]], !prof !16
 ; CHECK:       bb0.nonchr:
-; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
-; CHECK-NEXT:    br i1 [[TMP6]], label [[BB2_NONCHR:%.*]], label [[BB1_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB2_NONCHR:%.*]], label [[BB1_NONCHR:%.*]], !prof !16
 ; CHECK:       bb2.nonchr:
-; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[TMP8]], label [[BB4]], label [[BB3_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TMP7]], label [[BB4]], label [[BB3_NONCHR:%.*]], !prof !16
 ; CHECK:       bb3.nonchr:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB4]]
@@ -288,38 +288,38 @@ define void @test_chr_3(i32* %i) !prof !14 {
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       entry.split.nonchr:
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
-; CHECK-NEXT:    br i1 [[TMP4]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
 ; CHECK:       bb0.nonchr:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB1_NONCHR]]
 ; CHECK:       bb1.nonchr:
-; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
-; CHECK-NEXT:    br i1 [[TMP6]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
 ; CHECK:       bb2.nonchr:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* [[I]], align 4
-; CHECK-NEXT:    [[TMP8:%.*]] = and i32 [[TMP7]], 12
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 12
-; CHECK-NEXT:    br i1 [[TMP9]], label [[BB4:%.*]], label [[BB3_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* [[I]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP6]], 12
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 12
+; CHECK-NEXT:    br i1 [[TMP8]], label [[BB4:%.*]], label [[BB3_SPLIT_NONCHR:%.*]], !prof !15
 ; CHECK:       bb4:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB7:%.*]]
 ; CHECK:       bb3.split.nonchr:
-; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP7]], 4
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 0
-; CHECK-NEXT:    br i1 [[TMP11]], label [[BB5_NONCHR:%.*]], label [[BB4_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[TMP9:%.*]] = and i32 [[TMP6]], 4
+; CHECK-NEXT:    [[DOTNOT1:%.*]] = icmp eq i32 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[DOTNOT1]], label [[BB5_NONCHR:%.*]], label [[BB4_NONCHR:%.*]], !prof !16
 ; CHECK:       bb4.nonchr:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB5_NONCHR]]
 ; CHECK:       bb5.nonchr:
-; CHECK-NEXT:    [[TMP12:%.*]] = and i32 [[TMP7]], 8
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP12]], 0
-; CHECK-NEXT:    br i1 [[TMP13]], label [[BB7]], label [[BB6_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP6]], 8
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[TMP11]], label [[BB7]], label [[BB6_NONCHR:%.*]], !prof !16
 ; CHECK:       bb6.nonchr:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB7]]
@@ -396,12 +396,12 @@ define i32 @test_chr_4(i32* %i, i32 %sum0) !prof !14 {
 ; CHECK:       entry.split.nonchr:
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[SUM0]], 42
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
-; CHECK-NEXT:    [[SUM1_NONCHR:%.*]] = select i1 [[TMP6]], i32 [[SUM0]], i32 [[TMP4]], !prof !16
-; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 0
-; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[SUM1_NONCHR]], 43
-; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP8]], i32 [[SUM1_NONCHR]], i32 [[TMP9]], !prof !16
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    [[SUM1_NONCHR:%.*]] = select i1 [[DOTNOT]], i32 [[SUM0]], i32 [[TMP4]], !prof !16
+; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = add i32 [[SUM1_NONCHR]], 43
+; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP7]], i32 [[SUM1_NONCHR]], i32 [[TMP8]], !prof !16
 ; CHECK-NEXT:    ret i32 [[SUM2_NONCHR]]
 ;
 entry:
@@ -455,24 +455,24 @@ define i32 @test_chr_5(i32* %i, i32 %sum0) !prof !14 {
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       entry.split.nonchr:
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 255
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
-; CHECK-NEXT:    br i1 [[TMP6]], label [[BB3]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[BB3]], label [[BB0_NONCHR:%.*]], !prof !16
 ; CHECK:       bb0.nonchr:
-; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 0
-; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[SUM0]], 42
-; CHECK-NEXT:    [[SUM1_NONCHR:%.*]] = select i1 [[TMP8]], i32 [[SUM0]], i32 [[TMP9]], !prof !16
-; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[SUM1_NONCHR]], 43
-; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP11]], i32 [[SUM1_NONCHR]], i32 [[TMP12]], !prof !16
-; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], 0
-; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP0]], 8
-; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 0
-; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP16]], i32 44, i32 88
+; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = add i32 [[SUM0]], 42
+; CHECK-NEXT:    [[SUM1_NONCHR:%.*]] = select i1 [[TMP7]], i32 [[SUM0]], i32 [[TMP8]], !prof !16
+; CHECK-NEXT:    [[TMP9:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = add i32 [[SUM1_NONCHR]], 43
+; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP10]], i32 [[SUM1_NONCHR]], i32 [[TMP11]], !prof !16
+; CHECK-NEXT:    [[TMP12:%.*]] = and i32 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = and i32 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[TMP14]], 0
+; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP15]], i32 44, i32 88
 ; CHECK-NEXT:    [[SUM4_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], [[SUM4_NONCHR_V]]
-; CHECK-NEXT:    [[SUM5_NONCHR:%.*]] = select i1 [[TMP14]], i32 [[SUM2_NONCHR]], i32 [[SUM4_NONCHR]], !prof !16
+; CHECK-NEXT:    [[SUM5_NONCHR:%.*]] = select i1 [[TMP13]], i32 [[SUM2_NONCHR]], i32 [[SUM4_NONCHR]], !prof !16
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP4]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM5_NONCHR]], [[BB0_NONCHR]] ]
@@ -555,24 +555,24 @@ define i32 @test_chr_5_1(i32* %i, i32 %sum0) !prof !14 {
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       entry.split.nonchr:
 ; CHECK-NEXT:    [[TMP8:%.*]] = and i32 [[TMP0]], 255
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
-; CHECK-NEXT:    br i1 [[TMP9]], label [[BB3]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[BB3]], label [[BB0_NONCHR:%.*]], !prof !16
 ; CHECK:       bb0.nonchr:
-; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[SUM0]], 42
-; CHECK-NEXT:    [[SUM1_NONCHR:%.*]] = select i1 [[TMP11]], i32 [[SUM0]], i32 [[TMP12]], !prof !16
-; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], 0
-; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[SUM1_NONCHR]], 43
-; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP14]], i32 [[SUM1_NONCHR]], i32 [[TMP15]], !prof !16
-; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[SUM0]], 4
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i32 [[TMP16]], 0
-; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP0]], 8
-; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[TMP18]], 0
-; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP19]], i32 44, i32 88
+; CHECK-NEXT:    [[TMP9:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = add i32 [[SUM0]], 42
+; CHECK-NEXT:    [[SUM1_NONCHR:%.*]] = select i1 [[TMP10]], i32 [[SUM0]], i32 [[TMP11]], !prof !16
+; CHECK-NEXT:    [[TMP12:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = add i32 [[SUM1_NONCHR]], 43
+; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP13]], i32 [[SUM1_NONCHR]], i32 [[TMP14]], !prof !16
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[SUM0]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP17:%.*]] = and i32 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i32 [[TMP17]], 0
+; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP18]], i32 44, i32 88
 ; CHECK-NEXT:    [[SUM4_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], [[SUM4_NONCHR_V]]
-; CHECK-NEXT:    [[SUM5_NONCHR:%.*]] = select i1 [[TMP17]], i32 [[SUM2_NONCHR]], i32 [[SUM4_NONCHR]], !prof !16
+; CHECK-NEXT:    [[SUM5_NONCHR:%.*]] = select i1 [[TMP16]], i32 [[SUM2_NONCHR]], i32 [[SUM4_NONCHR]], !prof !16
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP7]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM5_NONCHR]], [[BB0_NONCHR]] ]
@@ -656,8 +656,8 @@ define i32 @test_chr_6(i32* %i, i32* %j, i32 %sum0) !prof !14 {
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       entry.split.nonchr:
 ; CHECK-NEXT:    [[V1:%.*]] = and i32 [[I0]], 255
-; CHECK-NEXT:    [[V2:%.*]] = icmp eq i32 [[V1]], 0
-; CHECK-NEXT:    br i1 [[V2]], label [[BB3]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[V2_NOT:%.*]] = icmp eq i32 [[V1]], 0
+; CHECK-NEXT:    br i1 [[V2_NOT]], label [[BB3]], label [[BB0_NONCHR:%.*]], !prof !16
 ; CHECK:       bb0.nonchr:
 ; CHECK-NEXT:    [[V3_NONCHR:%.*]] = and i32 [[I0]], 2
 ; CHECK-NEXT:    [[V4_NONCHR:%.*]] = icmp eq i32 [[V3_NONCHR]], 0
@@ -807,8 +807,8 @@ define i32 @test_chr_7_1(i32* %i, i32* %j, i32 %sum0) !prof !14 {
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       entry.split.nonchr:
 ; CHECK-NEXT:    [[V9:%.*]] = and i32 [[J0]], 4
-; CHECK-NEXT:    [[V10:%.*]] = icmp eq i32 [[V9]], 0
-; CHECK-NEXT:    br i1 [[V10]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[V10_NOT:%.*]] = icmp eq i32 [[V9]], 0
+; CHECK-NEXT:    br i1 [[V10_NOT]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
 ; CHECK:       bb0.nonchr:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB1_NONCHR]]
@@ -948,22 +948,22 @@ define i32 @test_chr_9(i32* %i, i32* %j) !prof !14 {
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       entry.split.nonchr:
 ; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP4]], 0
-; CHECK-NEXT:    br i1 [[TMP5]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
 ; CHECK:       bb0.nonchr:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB1_NONCHR]]
 ; CHECK:       bb1.nonchr:
-; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[TMP7]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
 ; CHECK:       bb2.nonchr:
-; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* [[J]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* [[J]], align 4
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[TMP9:%.*]] = phi i32 [ [[TMP3]], [[BB0]] ], [ [[TMP0]], [[BB1_NONCHR]] ], [ [[TMP8]], [[BB2_NONCHR]] ]
-; CHECK-NEXT:    ret i32 [[TMP9]]
+; CHECK-NEXT:    [[TMP8:%.*]] = phi i32 [ [[TMP3]], [[BB0]] ], [ [[TMP0]], [[BB1_NONCHR]] ], [ [[TMP7]], [[BB2_NONCHR]] ]
+; CHECK-NEXT:    ret i32 [[TMP8]]
 ;
 entry:
   %0 = load i32, i32* %i
@@ -1029,25 +1029,25 @@ define i32 @test_chr_10(i32* %i, i32* %j) !prof !14 {
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       entry.split.nonchr:
 ; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP4]], 0
-; CHECK-NEXT:    br i1 [[TMP5]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
 ; CHECK:       bb0.nonchr:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB1_NONCHR]]
 ; CHECK:       bb1.nonchr:
-; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* [[J]], align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[TMP8]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* [[J]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TMP7]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
 ; CHECK:       bb2.nonchr:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[TMP9:%.*]] = phi i32 [ [[TMP3]], [[BB0]] ], [ [[TMP6]], [[BB2_NONCHR]] ], [ [[TMP6]], [[BB1_NONCHR]] ]
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 42
-; CHECK-NEXT:    [[TMP11:%.*]] = add i32 [[TMP9]], -99
-; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[TMP10]], [[TMP11]]
-; CHECK-NEXT:    ret i32 [[TMP12]]
+; CHECK-NEXT:    [[TMP8:%.*]] = phi i32 [ [[TMP3]], [[BB0]] ], [ [[TMP5]], [[BB2_NONCHR]] ], [ [[TMP5]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i32 [[TMP8]], 42
+; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[TMP8]], -99
+; CHECK-NEXT:    [[TMP11:%.*]] = add i32 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    ret i32 [[TMP11]]
 ;
 entry:
   %0 = load i32, i32* %i
@@ -1476,24 +1476,24 @@ define i32 @test_chr_16(i32* %i) !prof !14 {
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       entry.split.nonchr:
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
-; CHECK-NEXT:    br i1 [[TMP4]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
 ; CHECK:       bb0.nonchr:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB1_NONCHR]]
 ; CHECK:       bb1.nonchr:
 ; CHECK-NEXT:    [[V40_NONCHR:%.*]] = add i32 [[TMP0]], 44
-; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
-; CHECK-NEXT:    br i1 [[TMP6]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
 ; CHECK:       bb2.nonchr:
 ; CHECK-NEXT:    [[V41_NONCHR:%.*]] = add i32 [[TMP0]], 99
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ [[V40]], [[BB0]] ], [ [[V40_NONCHR]], [[BB2_NONCHR]] ], [ [[V40_NONCHR]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = phi i32 [ [[V40]], [[BB0]] ], [ [[V40_NONCHR]], [[BB2_NONCHR]] ], [ [[V40_NONCHR]], [[BB1_NONCHR]] ]
 ; CHECK-NEXT:    [[V42:%.*]] = phi i32 [ [[V41]], [[BB0]] ], [ [[V41_NONCHR]], [[BB2_NONCHR]] ], [ [[V40_NONCHR]], [[BB1_NONCHR]] ]
-; CHECK-NEXT:    [[V43:%.*]] = add i32 [[V42]], [[TMP7]]
+; CHECK-NEXT:    [[V43:%.*]] = add i32 [[V42]], [[TMP6]]
 ; CHECK-NEXT:    ret i32 [[V43]]
 ;
 entry:
@@ -1661,8 +1661,8 @@ define i32 @test_chr_18(i32* %i, i32 %sum0) !prof !14 {
 ; CHECK-NEXT:    br i1 [[CMP4_NONCHR]], label [[BB2]], label [[BB1_NONCHR:%.*]], !prof !16
 ; CHECK:       bb1.nonchr:
 ; CHECK-NEXT:    [[A1:%.*]] = and i32 [[LI]], 1
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A1]], 0
-; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[CMP1]], i32 [[SUM0]], i32 [[SUM1]], !prof !16
+; CHECK-NEXT:    [[CMP1_NOT:%.*]] = icmp eq i32 [[A1]], 0
+; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[CMP1_NOT]], i32 [[SUM0]], i32 [[SUM1]], !prof !16
 ; CHECK-NEXT:    [[SUM3_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], 44
 ; CHECK-NEXT:    br label [[BB2]]
 ; CHECK:       bb2:
@@ -1744,18 +1744,18 @@ define i32 @test_chr_19(i32* %i, i32 %sum0) !prof !14 {
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       entry.split.nonchr:
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 255
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
-; CHECK-NEXT:    br i1 [[TMP6]], label [[BB3]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[BB3]], label [[BB0_NONCHR:%.*]], !prof !16
 ; CHECK:       bb0.nonchr:
-; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 0
-; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[SUM0]], 85
-; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP8]], i32 [[SUM0]], i32 [[TMP9]], !prof !16
-; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP0]], 8
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 0
-; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP11]], i32 44, i32 88
+; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = add i32 [[SUM0]], 85
+; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP7]], i32 [[SUM0]], i32 [[TMP8]], !prof !16
+; CHECK-NEXT:    [[TMP9:%.*]] = and i32 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP9]], 0
+; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP10]], i32 44, i32 88
 ; CHECK-NEXT:    [[SUM4_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], [[SUM4_NONCHR_V]]
-; CHECK-NEXT:    [[SUM5_NONCHR:%.*]] = select i1 [[TMP8]], i32 [[SUM2_NONCHR]], i32 [[SUM4_NONCHR]], !prof !16
+; CHECK-NEXT:    [[SUM5_NONCHR:%.*]] = select i1 [[TMP7]], i32 [[SUM2_NONCHR]], i32 [[SUM4_NONCHR]], !prof !16
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP4]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM5_NONCHR]], [[BB0_NONCHR]] ]
@@ -1842,8 +1842,8 @@ define i32 @test_chr_20(i32* %i, i32 %sum0, i1 %j) !prof !14 {
 ; CHECK:       entry.split.nonchr:
 ; CHECK-NEXT:    [[V8:%.*]] = add i32 [[SUM0]], 43
 ; CHECK-NEXT:    [[V3:%.*]] = and i32 [[I0]], 2
-; CHECK-NEXT:    [[V4:%.*]] = icmp eq i32 [[V3]], 0
-; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[V4]], i32 [[SUM0]], i32 [[V8]], !prof !16
+; CHECK-NEXT:    [[V4_NOT:%.*]] = icmp eq i32 [[V3]], 0
+; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[V4_NOT]], i32 [[SUM0]], i32 [[V8]], !prof !16
 ; CHECK-NEXT:    [[V6_NONCHR:%.*]] = and i32 [[I0]], 4
 ; CHECK-NEXT:    [[V5_NONCHR:%.*]] = icmp eq i32 [[V6_NONCHR]], 0
 ; CHECK-NEXT:    [[V9_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], 44
@@ -2328,7 +2328,7 @@ define i64 @test_chr_23(i64 %v0) !prof !14 {
 ; CHECK-LABEL: @test_chr_23(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[V0:%.*]], 50
-; CHECK-NEXT:    [[V10:%.*]] = icmp ne i64 [[TMP0]], -50
+; CHECK-NEXT:    [[V10_NOT:%.*]] = icmp eq i64 [[TMP0]], -50
 ; CHECK-NEXT:    ret i64 99
 ;
 entry:
diff --git a/llvm/test/Transforms/PGOProfile/instr_entry_bb.ll b/llvm/test/Transforms/PGOProfile/instr_entry_bb.ll
new file mode 100644
index 0000000000000..4ce701a767ccf
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/instr_entry_bb.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -pgo-instr-gen -pgo-instrument-entry -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen -pgo-instrument-entry -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -pgo-instr-gen -pgo-instrument-entry -instrprof -atomic-first-counter -S | FileCheck %s --check-prefix=GENA
+; RUN: opt < %s -passes=pgo-instr-gen,instrprof -pgo-instrument-entry -atomic-first-counter -S | FileCheck %s --check-prefix=GENA
+
+; RUN: llvm-profdata merge %S/Inputs/branch2.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instrument-entry -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instrument-entry -S | FileCheck %s --check-prefix=USE
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; GEN: $__llvm_profile_raw_version = comdat any
+; GEN: @__llvm_profile_raw_version = constant i64 {{[0-9]+}}, comdat
+; GEN: @__profn_test_br_2 = private constant [9 x i8] c"test_br_2"
+
+define i32 @test_br_2(i32 %i) {
+entry:
+; GEN: entry:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 29667547796, i32 2, i32 0)
+; GENA: entry:
+; GENA: %{{[0-9+]}} = atomicrmw add i64* getelementptr inbounds ([2 x i64], [2 x i64]* @__profc_test_br_2, i64 0, i64 0), i64 1 monotonic
+; USE: br i1 %cmp, label %if.then, label %if.else
+; USE-SAME: !prof ![[BW_ENTRY:[0-9]+]]
+; USE: ![[BW_ENTRY]] = !{!"branch_weights", i32 0, i32 1}
+  %cmp = icmp sgt i32 %i, 0
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+; GEN: if.then:
+; GEN-NOT: llvm.instrprof.increment
+  %add = add nsw i32 %i, 2
+  br label %if.end
+
+if.else:
+; GEN: if.else:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 29667547796, i32 2, i32 1)
+; GENA: if.else:
+; GENA:  %pgocount = load i64, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @__profc_test_br_2, i64 0, i64 1), align 8
+; GENA:  [[V:%[0-9]*]] = add i64 %pgocount, 1
+; GENA:  store i64 [[V]], i64* getelementptr inbounds ([2 x i64], [2 x i64]* @__profc_test_br_2, i64 0, i64 1), align 8
+  %sub = sub nsw i32 %i, 2
+  br label %if.end
+
+if.end:
+; GEN: if.end:
+; GEN-NOT: llvm.instrprof.increment
+  %retv = phi i32 [ %add, %if.then ], [ %sub, %if.else ]
+  ret i32 %retv
+; GEN: ret
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/shuffle.ll b/llvm/test/Transforms/PhaseOrdering/X86/shuffle.ll
index 1b2ee9fa3ade9..b9eb5671b8d13 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/shuffle.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/shuffle.ll
@@ -164,8 +164,8 @@ define <4 x i32> @shuffle_8_add_32_masks_are_eq_and_can_be_converted_up(<16 x i8
 define <8 x i16> @shuffle_32_bitcast_16_shuffle_16_can_be_converted_up(<4 x i32> %v1) {
 ; CHECK-LABEL: @shuffle_32_bitcast_16_shuffle_16_can_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <8 x i16>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
 ; CHECK-NEXT:    ret <8 x i16> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
@@ -180,8 +180,8 @@ define <8 x i16> @shuffle_32_bitcast_16_shuffle_16_can_be_converted_up(<4 x i32>
 define <8 x i16> @shuffle_32_bitcast_16_shuffle_16_can_not_be_converted_up(<4 x i32> %v1) {
 ; CHECK-LABEL: @shuffle_32_bitcast_16_shuffle_16_can_not_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <8 x i16>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> undef, <8 x i32> <i32 5, i32 4, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> undef, <8 x i32> <i32 5, i32 4, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <8 x i16> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
@@ -196,8 +196,8 @@ define <8 x i16> @shuffle_32_bitcast_16_shuffle_16_can_not_be_converted_up(<4 x
 define <16 x i8> @shuffle_32_bitcast_8_shuffle_8_can_be_converted_up(<4 x i32> %v1) {
 ; CHECK-LABEL: @shuffle_32_bitcast_8_shuffle_8_can_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[TMP2]], <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[BC1]], <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <16 x i8> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
@@ -212,8 +212,8 @@ define <16 x i8> @shuffle_32_bitcast_8_shuffle_8_can_be_converted_up(<4 x i32> %
 define <16 x i8> @shuffle_32_bitcast_8_shuffle_8_can_not_be_converted_up(<4 x i32> %v1) {
 ; CHECK-LABEL: @shuffle_32_bitcast_8_shuffle_8_can_not_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[TMP2]], <16 x i8> undef, <16 x i32> <i32 5, i32 4, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[BC1]], <16 x i8> undef, <16 x i32> <i32 5, i32 4, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    ret <16 x i8> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
@@ -228,8 +228,8 @@ define <16 x i8> @shuffle_32_bitcast_8_shuffle_8_can_not_be_converted_up(<4 x i3
 define <4 x i32> @shuffle_8_bitcast_32_shuffle_32_can_be_converted_up(<16 x i8> %v1) {
 ; CHECK-LABEL: @shuffle_8_bitcast_32_shuffle_32_can_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[V1:%.*]] to <4 x i32>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
 ; CHECK-NEXT:    ret <4 x i32> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
@@ -244,8 +244,8 @@ define <4 x i32> @shuffle_8_bitcast_32_shuffle_32_can_be_converted_up(<16 x i8>
 define <4 x i32> @shuffle_16_bitcast_32_shuffle_32_can_be_converted_up(<8 x i16> %v1) {
 ; CHECK-LABEL: @shuffle_16_bitcast_32_shuffle_32_can_be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[V1:%.*]] to <4 x i32>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
 ; CHECK-NEXT:    ret <4 x i32> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
@@ -292,8 +292,8 @@ define <4 x i32> @shuffle_16_bitcast_32_shuffle_32_can_not_be_converted_up(<8 x
 define <8 x i16> @shuffle_8_bitcast_16_shuffle_16_can__be_converted_up(<16 x i8> %v1) {
 ; CHECK-LABEL: @shuffle_8_bitcast_16_shuffle_16_can__be_converted_up(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[V1:%.*]] to <8 x i16>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
-; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    [[BC1:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
 ; CHECK-NEXT:    ret <8 x i16> [[SHUFFLE2]]
 ;
   %shuffle1 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll
index b4bfb4a0c3d73..cf01ead15b0e5 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll
@@ -24,12 +24,12 @@ define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
 
 define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
 ; CHECK-LABEL: @ext_ext_partial_add_reduction_v4i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[X]]
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP4]], i64 0
-; CHECK-NEXT:    ret i32 [[TMP5]]
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[X]]
+; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[SHIFT1]]
+; CHECK-NEXT:    [[X210:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    ret i32 [[X210]]
 ;
   %x0 = extractelement <4 x i32> %x, i32 0
   %x1 = extractelement <4 x i32> %x, i32 1
@@ -41,14 +41,14 @@ define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
 
 define i32 @ext_ext_partial_add_reduction_and_extra_add_v4i32(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @ext_ext_partial_add_reduction_and_extra_add_v4i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP3]], [[Y]]
-; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP4]], [[TMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP5]], [[TMP2]]
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP6]], i32 0
-; CHECK-NEXT:    ret i32 [[TMP7]]
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[Y:%.*]]
+; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[SHIFT1]]
+; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[TMP2]], [[SHIFT2]]
+; CHECK-NEXT:    [[X2Y210:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; CHECK-NEXT:    ret i32 [[X2Y210]]
 ;
   %y0 = extractelement <4 x i32> %y, i32 0
   %y1 = extractelement <4 x i32> %y, i32 1
@@ -132,10 +132,9 @@ define i32 @TestVectorsEqual_alt(i32* noalias %Vec0, i32* noalias %Vec1, i32 %To
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[VEC1:%.*]] to <4 x i32>*
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP1]])
-; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
-; CHECK-NEXT:    [[ADD_3:%.*]] = sub i32 [[TMP4]], [[TMP5]]
-; CHECK-NEXT:    [[CMP3:%.*]] = icmp ule i32 [[ADD_3]], [[TOLERANCE:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ule i32 [[TMP5]], [[TOLERANCE:%.*]]
 ; CHECK-NEXT:    [[COND:%.*]] = zext i1 [[CMP3]] to i32
 ; CHECK-NEXT:    ret i32 [[COND]]
 ;
@@ -279,3 +278,67 @@ for.end:
   %cond = select i1 %cmp3, i32 1, i32 0
   ret i32 %cond
 }
+
+; PR43745 - https://bugs.llvm.org/show_bug.cgi?id=43745
+
+define i1 @cmp_lt_gt(double %a, double %b, double %c) {
+; CHECK-LABEL: @cmp_lt_gt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[FNEG:%.*]] = fneg double [[B:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[A:%.*]], 2.000000e+00
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[FNEG]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[C]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x double> undef, double [[MUL]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = fdiv <2 x double> [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fcmp olt <2 x double> [[TMP7]], <double 0x3EB0C6F7A0B5ED8D, double 0x3EB0C6F7A0B5ED8D>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <2 x i1> [[TMP8]], <2 x i1> undef, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP9:%.*]] = and <2 x i1> [[TMP8]], [[SHIFT]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = extractelement <2 x i1> [[TMP9]], i64 0
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[CLEANUP:%.*]], label [[LOR_LHS_FALSE:%.*]]
+; CHECK:       lor.lhs.false:
+; CHECK-NEXT:    [[TMP10:%.*]] = fcmp ule <2 x double> [[TMP7]], <double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <2 x i1> [[TMP10]], <2 x i1> undef, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP11:%.*]] = or <2 x i1> [[TMP10]], [[SHIFT2]]
+; CHECK-NEXT:    [[NOT_OR_COND1:%.*]] = extractelement <2 x i1> [[TMP11]], i32 0
+; CHECK-NEXT:    ret i1 [[NOT_OR_COND1]]
+; CHECK:       cleanup:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %fneg = fneg double %b
+  %add = fadd double %fneg, %c
+  %mul = fmul double 2.0, %a
+  %div = fdiv double %add, %mul
+  %fneg1 = fneg double %b
+  %sub = fsub double %fneg1, %c
+  %mul2 = fmul double 2.0, %a
+  %div3 = fdiv double %sub, %mul2
+  %cmp = fcmp olt double %div, 0x3EB0C6F7A0B5ED8D
+  br i1 %cmp, label %land.lhs.true, label %lor.lhs.false
+
+land.lhs.true:
+  %cmp4 = fcmp olt double %div3, 0x3EB0C6F7A0B5ED8D
+  br i1 %cmp4, label %if.then, label %lor.lhs.false
+
+lor.lhs.false:
+  %cmp5 = fcmp ogt double %div, 1.0
+  br i1 %cmp5, label %land.lhs.true6, label %if.end
+
+land.lhs.true6:
+  %cmp7 = fcmp ogt double %div3, 1.0
+  br i1 %cmp7, label %if.then, label %if.end
+
+if.then:
+  br label %cleanup
+
+if.end:
+  br label %cleanup
+
+cleanup:
+  %retval.0 = phi i1 [ false, %if.then ], [ true, %if.end ]
+  ret i1 %retval.0
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/inlining-alignment-assumptions.ll b/llvm/test/Transforms/PhaseOrdering/inlining-alignment-assumptions.ll
index 5a9d4442d9e33..61287e35005ff 100644
--- a/llvm/test/Transforms/PhaseOrdering/inlining-alignment-assumptions.ll
+++ b/llvm/test/Transforms/PhaseOrdering/inlining-alignment-assumptions.ll
@@ -38,10 +38,10 @@ define void @caller1(i1 %c, i64* align 1 %ptr) {
 ; ASSUMPTIONS-OFF-NEXT:    ret void
 ;
 ; ASSUMPTIONS-ON-LABEL: @caller1(
-; ASSUMPTIONS-ON-NEXT:    br i1 [[C:%.*]], label [[TRUE1:%.*]], label [[FALSE1:%.*]]
-; ASSUMPTIONS-ON:       true1:
-; ASSUMPTIONS-ON-NEXT:    [[C_PR:%.*]] = phi i1 [ false, [[FALSE1]] ], [ true, [[TMP0:%.*]] ]
-; ASSUMPTIONS-ON-NEXT:    [[PTRINT:%.*]] = ptrtoint i64* [[PTR:%.*]] to i64
+; ASSUMPTIONS-ON-NEXT:    br i1 [[C:%.*]], label [[TRUE2_CRITEDGE:%.*]], label [[FALSE1:%.*]]
+; ASSUMPTIONS-ON:       false1:
+; ASSUMPTIONS-ON-NEXT:    store volatile i64 1, i64* [[PTR:%.*]], align 8
+; ASSUMPTIONS-ON-NEXT:    [[PTRINT:%.*]] = ptrtoint i64* [[PTR]] to i64
 ; ASSUMPTIONS-ON-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 7
 ; ASSUMPTIONS-ON-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
 ; ASSUMPTIONS-ON-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
@@ -51,16 +51,21 @@ define void @caller1(i1 %c, i64* align 1 %ptr) {
 ; ASSUMPTIONS-ON-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
 ; ASSUMPTIONS-ON-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
 ; ASSUMPTIONS-ON-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
-; ASSUMPTIONS-ON-NEXT:    br i1 [[C_PR]], label [[TRUE2:%.*]], label [[FALSE2:%.*]]
-; ASSUMPTIONS-ON:       false1:
-; ASSUMPTIONS-ON-NEXT:    store volatile i64 1, i64* [[PTR]], align 4
-; ASSUMPTIONS-ON-NEXT:    br label [[TRUE1]]
-; ASSUMPTIONS-ON:       true2:
-; ASSUMPTIONS-ON-NEXT:    store volatile i64 2, i64* [[PTR]], align 8
-; ASSUMPTIONS-ON-NEXT:    ret void
-; ASSUMPTIONS-ON:       false2:
 ; ASSUMPTIONS-ON-NEXT:    store volatile i64 3, i64* [[PTR]], align 8
 ; ASSUMPTIONS-ON-NEXT:    ret void
+; ASSUMPTIONS-ON:       true2.critedge:
+; ASSUMPTIONS-ON-NEXT:    [[PTRINT_C:%.*]] = ptrtoint i64* [[PTR]] to i64
+; ASSUMPTIONS-ON-NEXT:    [[MASKEDPTR_C:%.*]] = and i64 [[PTRINT_C]], 7
+; ASSUMPTIONS-ON-NEXT:    [[MASKCOND_C:%.*]] = icmp eq i64 [[MASKEDPTR_C]], 0
+; ASSUMPTIONS-ON-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND_C]])
+; ASSUMPTIONS-ON-NEXT:    store volatile i64 0, i64* [[PTR]], align 8
+; ASSUMPTIONS-ON-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
+; ASSUMPTIONS-ON-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
+; ASSUMPTIONS-ON-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
+; ASSUMPTIONS-ON-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
+; ASSUMPTIONS-ON-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
+; ASSUMPTIONS-ON-NEXT:    store volatile i64 2, i64* [[PTR]], align 8
+; ASSUMPTIONS-ON-NEXT:    ret void
 ;
   br i1 %c, label %true1, label %false1
 
@@ -86,7 +91,7 @@ false2:
   ret void
 }
 
-; This test illustrates that alignment assumptions may prevent SROA.
+; This test checks that alignment assumptions do not prevent SROA.
 ; See PR45763.
 
 define internal void @callee2(i64* noalias sret align 8 %arg) {
diff --git a/llvm/test/Transforms/PhaseOrdering/rotate.ll b/llvm/test/Transforms/PhaseOrdering/rotate.ll
index e10a46cb8300e..4661de3688e60 100644
--- a/llvm/test/Transforms/PhaseOrdering/rotate.ll
+++ b/llvm/test/Transforms/PhaseOrdering/rotate.ll
@@ -5,20 +5,12 @@
 ; This should become a single funnel shift through a combination
 ; of aggressive-instcombine, simplifycfg, and instcombine.
 ; https://bugs.llvm.org/show_bug.cgi?id=34924
-; These are equivalent, but the value name with the new-pm shows a bug -
-; this code should not have been converted to a speculative select with
-; an intermediate transform.
 
 define i32 @rotl(i32 %a, i32 %b) {
-; OLDPM-LABEL: @rotl(
-; OLDPM-NEXT:  entry:
-; OLDPM-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.fshl.i32(i32 [[A:%.*]], i32 [[A]], i32 [[B:%.*]])
-; OLDPM-NEXT:    ret i32 [[TMP0]]
-;
-; NEWPM-LABEL: @rotl(
-; NEWPM-NEXT:  entry:
-; NEWPM-NEXT:    [[SPEC_SELECT:%.*]] = tail call i32 @llvm.fshl.i32(i32 [[A:%.*]], i32 [[A]], i32 [[B:%.*]])
-; NEWPM-NEXT:    ret i32 [[SPEC_SELECT]]
+; ANY-LABEL: @rotl(
+; ANY-NEXT:  entry:
+; ANY-NEXT:    [[COND:%.*]] = tail call i32 @llvm.fshl.i32(i32 [[A:%.*]], i32 [[A]], i32 [[B:%.*]])
+; ANY-NEXT:    ret i32 [[COND]]
 ;
 entry:
   %cmp = icmp eq i32 %b, 0
@@ -35,4 +27,3 @@ end:
   %cond = phi i32 [ %or, %rotbb ], [ %a, %entry ]
   ret i32 %cond
 }
-
diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/load-relative.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/load-relative.ll
index 43cb0cc5a1d7b..56bedd9b101fd 100644
--- a/llvm/test/Transforms/PreISelIntrinsicLowering/load-relative.ll
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/load-relative.ll
@@ -1,5 +1,4 @@
 ; RUN: opt -pre-isel-intrinsic-lowering -S -o - %s | FileCheck %s
-; RUN: opt -passes='pre-isel-intrinsic-lowering' -S -o - %s | FileCheck %s
 
 ; CHECK: define i8* @foo32(i8* [[P:%.*]], i32 [[O:%.*]])
 define i8* @foo32(i8* %p, i32 %o) {
diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/objc-arc.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/objc-arc.ll
index 5ff620a71b87d..aff01910cd9b3 100644
--- a/llvm/test/Transforms/PreISelIntrinsicLowering/objc-arc.ll
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/objc-arc.ll
@@ -1,5 +1,4 @@
 ; RUN: opt -pre-isel-intrinsic-lowering -S -o - %s | FileCheck %s
-; RUN: opt -passes='pre-isel-intrinsic-lowering' -S -o - %s | FileCheck %s
 
 ; Make sure calls to the objc intrinsics are translated to calls in to the
 ; runtime
diff --git a/llvm/test/Transforms/Reassociate/pr12245.ll b/llvm/test/Transforms/Reassociate/pr12245.ll
index 0e7152e2a177d..898222edf3af6 100644
--- a/llvm/test/Transforms/Reassociate/pr12245.ll
+++ b/llvm/test/Transforms/Reassociate/pr12245.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -inline -instcombine -reassociate -dse -disable-output
+; RUN: opt < %s -basic-aa -inline -instcombine -reassociate -dse -disable-output
 ; PR12245
 
 @a = common global i32 0, align 4
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/relocate-invoke-result.ll b/llvm/test/Transforms/RewriteStatepointsForGC/relocate-invoke-result.ll
index d198b2727780d..b823b6c2a5baf 100644
--- a/llvm/test/Transforms/RewriteStatepointsForGC/relocate-invoke-result.ll
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/relocate-invoke-result.ll
@@ -1,6 +1,6 @@
 
 ;; RUN: opt -rewrite-statepoints-for-gc -verify -S < %s | FileCheck %s
-;; RUN: opt -passes=rewrite-statepoints-for-gc -verify -S < %s | FileCheck %s
+;; RUN: opt -passes=rewrite-statepoints-for-gc,verify -S < %s | FileCheck %s
 ;; This test is to verify that RewriteStatepointsForGC correctly relocates values
 ;; defined by invoke instruction results. 
 
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/rewrite-invoke.ll b/llvm/test/Transforms/RewriteStatepointsForGC/rewrite-invoke.ll
index f096f30ba06a7..5db1c46027962 100644
--- a/llvm/test/Transforms/RewriteStatepointsForGC/rewrite-invoke.ll
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/rewrite-invoke.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -rewrite-statepoints-for-gc -verify -S < %s | FileCheck %s
-; RUN: opt -passes=rewrite-statepoints-for-gc -verify -S < %s | FileCheck %s
+; RUN: opt -passes=rewrite-statepoints-for-gc,verify -S < %s | FileCheck %s
 
 declare i8 addrspace(1)* @gc_call()
 
diff --git a/llvm/test/Transforms/IPConstantProp/2009-09-24-byval-ptr.ll b/llvm/test/Transforms/SCCP/2009-09-24-byval-ptr.ll
similarity index 100%
rename from llvm/test/Transforms/IPConstantProp/2009-09-24-byval-ptr.ll
rename to llvm/test/Transforms/SCCP/2009-09-24-byval-ptr.ll
diff --git a/llvm/test/Transforms/IPConstantProp/PR16052.ll b/llvm/test/Transforms/SCCP/PR16052.ll
similarity index 100%
rename from llvm/test/Transforms/IPConstantProp/PR16052.ll
rename to llvm/test/Transforms/SCCP/PR16052.ll
diff --git a/llvm/test/Transforms/IPConstantProp/PR26044.ll b/llvm/test/Transforms/SCCP/PR26044.ll
similarity index 100%
rename from llvm/test/Transforms/IPConstantProp/PR26044.ll
rename to llvm/test/Transforms/SCCP/PR26044.ll
diff --git a/llvm/test/Transforms/SCCP/assume.ll b/llvm/test/Transforms/SCCP/assume.ll
new file mode 100644
index 0000000000000..764c1737c2874
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/assume.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -ipsccp -S | FileCheck %s
+
+declare void @use(i1)
+declare void @llvm.assume(i1)
+
+define void @basic(i32 %v) {
+; CHECK-LABEL: @basic(
+; CHECK-NEXT:    [[A1:%.*]] = icmp ult i32 [[V:%.*]], 10
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A1]])
+; CHECK-NEXT:    [[A2:%.*]] = icmp ugt i32 [[V]], 5
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A2]])
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[V]], 9
+; CHECK-NEXT:    call void @use(i1 [[C2]])
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    [[C4:%.*]] = icmp ugt i32 [[V]], 8
+; CHECK-NEXT:    call void @use(i1 [[C4]])
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[C6:%.*]] = icmp ugt i32 [[V]], 6
+; CHECK-NEXT:    call void @use(i1 [[C6]])
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    [[C8:%.*]] = icmp ult i32 [[V]], 7
+; CHECK-NEXT:    call void @use(i1 [[C8]])
+; CHECK-NEXT:    ret void
+;
+  %a1 = icmp ult i32 %v, 10
+  call void @llvm.assume(i1 %a1)
+  %a2 = icmp ugt i32 %v, 5
+  call void @llvm.assume(i1 %a2)
+  %c1 = icmp ult i32 %v, 10
+  call void @use(i1 %c1)
+  %c2 = icmp ult i32 %v, 9
+  call void @use(i1 %c2)
+  %c3 = icmp ugt i32 %v, 9
+  call void @use(i1 %c3)
+  %c4 = icmp ugt i32 %v, 8
+  call void @use(i1 %c4)
+  %c5 = icmp ugt i32 %v, 5
+  call void @use(i1 %c5)
+  %c6 = icmp ugt i32 %v, 6
+  call void @use(i1 %c6)
+  %c7 = icmp ult i32 %v, 6
+  call void @use(i1 %c7)
+  %c8 = icmp ult i32 %v, 7
+  call void @use(i1 %c8)
+  ret void
+}
diff --git a/llvm/test/Transforms/SCCP/conditions-ranges.ll b/llvm/test/Transforms/SCCP/conditions-ranges.ll
index ea857a1008546..612a38f008fc5 100644
--- a/llvm/test/Transforms/SCCP/conditions-ranges.ll
+++ b/llvm/test/Transforms/SCCP/conditions-ranges.ll
@@ -814,14 +814,11 @@ define void @f16_conditions_and(i32 %a, i32 %b) {
 ; CHECK-NEXT:    [[BC:%.*]] = and i1 [[LT]], [[GT]]
 ; CHECK-NEXT:    br i1 [[BC]], label [[TRUE:%.*]], label [[FALSE:%.*]]
 ; CHECK:       true:
-; CHECK-NEXT:    [[F_1:%.*]] = icmp eq i32 [[A]], 0
-; CHECK-NEXT:    call void @use(i1 [[F_1]])
-; CHECK-NEXT:    [[F_2:%.*]] = icmp eq i32 [[A]], 20
-; CHECK-NEXT:    call void @use(i1 [[F_2]])
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 false)
 ; CHECK-NEXT:    call void @use(i1 false)
 ; CHECK-NEXT:    call void @use(i1 true)
-; CHECK-NEXT:    [[T_2:%.*]] = icmp ne i32 [[A]], 20
-; CHECK-NEXT:    call void @use(i1 [[T_2]])
+; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp eq i32 [[A]], 21
 ; CHECK-NEXT:    call void @use(i1 [[C_1]])
 ; CHECK-NEXT:    [[C_2:%.*]] = icmp ugt i32 [[A]], 21
@@ -899,10 +896,8 @@ define void @f17_conditions_or(i32 %a, i32 %b) {
 ; CHECK:       false:
 ; CHECK-NEXT:    call void @use(i1 false)
 ; CHECK-NEXT:    call void @use(i1 false)
-; CHECK-NEXT:    [[F_3:%.*]] = icmp ugt i32 [[A]], 100
-; CHECK-NEXT:    call void @use(i1 [[F_3]])
-; CHECK-NEXT:    [[T_1:%.*]] = icmp ult i32 [[A]], 100
-; CHECK-NEXT:    call void @use(i1 [[T_1]])
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp eq i32 [[A]], 21
 ; CHECK-NEXT:    call void @use(i1 [[C_1]])
diff --git a/llvm/test/Transforms/IPConstantProp/dangling-block-address.ll b/llvm/test/Transforms/SCCP/dangling-block-address.ll
similarity index 100%
rename from llvm/test/Transforms/IPConstantProp/dangling-block-address.ll
rename to llvm/test/Transforms/SCCP/dangling-block-address.ll
diff --git a/llvm/test/Transforms/IPConstantProp/fp-bc-icmp-const-fold.ll b/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll
similarity index 100%
rename from llvm/test/Transforms/IPConstantProp/fp-bc-icmp-const-fold.ll
rename to llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll
diff --git a/llvm/test/Transforms/IPConstantProp/global.ll b/llvm/test/Transforms/SCCP/global.ll
similarity index 100%
rename from llvm/test/Transforms/IPConstantProp/global.ll
rename to llvm/test/Transforms/SCCP/global.ll
diff --git a/llvm/test/Transforms/SCCP/metadata.ll b/llvm/test/Transforms/SCCP/metadata.ll
new file mode 100644
index 0000000000000..844e2103ae318
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/metadata.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -ipsccp -S | FileCheck %s
+
+declare void @use(i1)
+declare i32 @get_i32()
+
+define void @load_range(i32* %p) {
+; CHECK-LABEL: @load_range(
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4, !range !0
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[V]], 9
+; CHECK-NEXT:    call void @use(i1 [[C2]])
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    [[C4:%.*]] = icmp ugt i32 [[V]], 8
+; CHECK-NEXT:    call void @use(i1 [[C4]])
+; CHECK-NEXT:    ret void
+;
+  %v = load i32, i32* %p, !range !{i32 0, i32 10}
+  %c1 = icmp ult i32 %v, 10
+  call void @use(i1 %c1)
+  %c2 = icmp ult i32 %v, 9
+  call void @use(i1 %c2)
+  %c3 = icmp ugt i32 %v, 9
+  call void @use(i1 %c3)
+  %c4 = icmp ugt i32 %v, 8
+  call void @use(i1 %c4)
+  ret void
+}
+
+define i32 @load_range_single(i32* %p) {
+; CHECK-LABEL: @load_range_single(
+; CHECK-NEXT:    ret i32 0
+;
+  %v = load i32, i32* %p, !range !{i32 0, i32 1}
+  ret i32 %v
+}
+
+define i32 @load_range_single_volatile(i32* %p) {
+; CHECK-LABEL: @load_range_single_volatile(
+; CHECK-NEXT:    [[V:%.*]] = load volatile i32, i32* [[P:%.*]], align 4, !range !1
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %v = load volatile i32, i32* %p, !range !{i32 0, i32 1}
+  ret i32 %v
+}
+
+define void @load_nonnull(i32** %p, i32** %p2) {
+; CHECK-LABEL: @load_nonnull(
+; CHECK-NEXT:    [[V:%.*]] = load i32*, i32** [[P:%.*]], align 8, !nonnull !2
+; CHECK-NEXT:    [[V2:%.*]] = load i32*, i32** [[P2:%.*]], align 8, !nonnull !2
+; CHECK-NEXT:    [[C1:%.*]] = icmp ne i32* [[V]], null
+; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    [[C2:%.*]] = icmp eq i32* [[V]], null
+; CHECK-NEXT:    call void @use(i1 [[C2]])
+; CHECK-NEXT:    [[C3:%.*]] = icmp ne i32* null, [[V]]
+; CHECK-NEXT:    call void @use(i1 [[C3]])
+; CHECK-NEXT:    [[C4:%.*]] = icmp eq i32* null, [[V]]
+; CHECK-NEXT:    call void @use(i1 [[C4]])
+; CHECK-NEXT:    [[C5:%.*]] = icmp eq i32* [[V]], [[V2]]
+; CHECK-NEXT:    call void @use(i1 [[C5]])
+; CHECK-NEXT:    [[C6:%.*]] = icmp ne i32* [[V]], [[V2]]
+; CHECK-NEXT:    call void @use(i1 [[C6]])
+; CHECK-NEXT:    ret void
+;
+  %v = load i32*, i32** %p, !nonnull !{}
+  %v2 = load i32*, i32** %p2, !nonnull !{}
+  %c1 = icmp ne i32* %v, null
+  call void @use(i1 %c1)
+  %c2 = icmp eq i32* %v, null
+  call void @use(i1 %c2)
+  %c3 = icmp ne i32* null, %v
+  call void @use(i1 %c3)
+  %c4 = icmp eq i32* null, %v
+  call void @use(i1 %c4)
+  ; There is no particular relationship between two nonnull values.
+  %c5 = icmp eq i32* %v, %v2
+  call void @use(i1 %c5)
+  %c6 = icmp ne i32* %v, %v2
+  call void @use(i1 %c6)
+  ret void
+}
+
+define void @call_range(i32* %p) {
+; CHECK-LABEL: @call_range(
+; CHECK-NEXT:    [[V:%.*]] = call i32 @get_i32(), !range !0
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[V]], 9
+; CHECK-NEXT:    call void @use(i1 [[C2]])
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    [[C4:%.*]] = icmp ugt i32 [[V]], 8
+; CHECK-NEXT:    call void @use(i1 [[C4]])
+; CHECK-NEXT:    ret void
+;
+  %v = call i32 @get_i32(), !range !{i32 0, i32 10}
+  %c1 = icmp ult i32 %v, 10
+  call void @use(i1 %c1)
+  %c2 = icmp ult i32 %v, 9
+  call void @use(i1 %c2)
+  %c3 = icmp ugt i32 %v, 9
+  call void @use(i1 %c3)
+  %c4 = icmp ugt i32 %v, 8
+  call void @use(i1 %c4)
+  ret void
+}
+
+define internal i1 @ip_cmp_range(i32 %v) {
+; CHECK-LABEL: @ip_cmp_range(
+; CHECK-NEXT:    ret i1 undef
+;
+  %c = icmp ult i32 %v, 10
+  ret i1 %c
+}
+
+define i1 @ip_load_range(i32* %p) {
+; CHECK-LABEL: @ip_load_range(
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4, !range !0
+; CHECK-NEXT:    [[C:%.*]] = call i1 @ip_cmp_range(i32 [[V]])
+; CHECK-NEXT:    ret i1 true
+;
+  %v = load i32, i32* %p, !range !{i32 0, i32 10}
+  %c = call i1 @ip_cmp_range(i32 %v)
+  ret i1 %c
+}
diff --git a/llvm/test/Transforms/IPConstantProp/musttail-call.ll b/llvm/test/Transforms/SCCP/musttail-call.ll
similarity index 100%
rename from llvm/test/Transforms/IPConstantProp/musttail-call.ll
rename to llvm/test/Transforms/SCCP/musttail-call.ll
diff --git a/llvm/test/Transforms/SCCP/predicateinfo-cond.ll b/llvm/test/Transforms/SCCP/predicateinfo-cond.ll
new file mode 100644
index 0000000000000..d8528918babed
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/predicateinfo-cond.ll
@@ -0,0 +1,110 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -ipsccp < %s | FileCheck %s
+
+; Test that information about the true/false value of conditions themselves
+; is also used, not information implied by comparisions.
+
+define i32 @switch(i32 %x) {
+; CHECK-LABEL: @switch(
+; CHECK-NEXT:    switch i32 [[X:%.*]], label [[CASE_DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[CASE_0:%.*]]
+; CHECK-NEXT:    i32 2, label [[CASE_2:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       case.0:
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X]], 1
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       case.2:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[X]], 1
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       case.default:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[ADD]], [[CASE_0]] ], [ [[SUB]], [[CASE_2]] ], [ 1, [[CASE_DEFAULT]] ]
+; CHECK-NEXT:    ret i32 [[PHI]]
+;
+  switch i32 %x, label %case.default [
+  i32 0, label %case.0
+  i32 2, label %case.2
+  ]
+
+case.0:
+  %add = add i32 %x, 1
+  br label %end
+
+case.2:
+  %sub = sub i32 %x, 1
+  br label %end
+
+case.default:
+  br label %end
+
+end:
+  %phi = phi i32 [ %add, %case.0 ], [ %sub, %case.2 ], [ 1, %case.default]
+  ret i32 %phi
+}
+
+define i1 @assume(i32 %x) {
+; CHECK-LABEL: @assume(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = icmp sge i32 %x, 0
+  call void @llvm.assume(i1 %cmp)
+  ret i1 %cmp
+}
+
+define i32 @branch(i32 %x) {
+; CHECK-LABEL: @branch(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN1:%.*]], label [[IF_THEN2:%.*]]
+; CHECK:       if.then1:
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF2_THEN1:%.*]], label [[IF2_THEN2:%.*]]
+; CHECK:       if2.then1:
+; CHECK-NEXT:    br label [[IF2_END:%.*]]
+; CHECK:       if2.then2:
+; CHECK-NEXT:    br label [[IF2_END]]
+; CHECK:       if2.end:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ 0, [[IF2_THEN1]] ], [ 1, [[IF2_THEN2]] ]
+; CHECK-NEXT:    ret i32 [[PHI]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF3_THEN1:%.*]], label [[IF3_THEN2:%.*]]
+; CHECK:       if3.then1:
+; CHECK-NEXT:    br label [[IF3_END:%.*]]
+; CHECK:       if3.then2:
+; CHECK-NEXT:    br label [[IF3_END]]
+; CHECK:       if3.end:
+; CHECK-NEXT:    [[PHI2:%.*]] = phi i32 [ 0, [[IF3_THEN1]] ], [ 1, [[IF3_THEN2]] ]
+; CHECK-NEXT:    ret i32 [[PHI2]]
+;
+  %cmp = icmp sge i32 %x, 0
+  br i1 %cmp, label %if.then1, label %if.then2
+
+if.then1:
+  br i1 %cmp, label %if2.then1, label %if2.then2
+
+if2.then1:
+  br label %if2.end
+
+if2.then2:
+  br label %if2.end
+
+if2.end:
+  %phi = phi i32 [ 0, %if2.then1 ], [ 1, %if2.then2 ]
+  ret i32 %phi
+
+if.then2:
+  br i1 %cmp, label %if3.then1, label %if3.then2
+
+if3.then1:
+  br label %if3.end
+
+if3.then2:
+  br label %if3.end
+
+if3.end:
+  %phi2 = phi i32 [ 0, %if3.then1 ], [ 1, %if3.then2 ]
+  ret i32 %phi2
+}
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/IPConstantProp/remove-call-inst.ll b/llvm/test/Transforms/SCCP/remove-call-inst.ll
similarity index 100%
rename from llvm/test/Transforms/IPConstantProp/remove-call-inst.ll
rename to llvm/test/Transforms/SCCP/remove-call-inst.ll
diff --git a/llvm/test/Transforms/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll b/llvm/test/Transforms/SCCP/solve-after-each-resolving-undefs-for-function.ll
similarity index 100%
rename from llvm/test/Transforms/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
rename to llvm/test/Transforms/SCCP/solve-after-each-resolving-undefs-for-function.ll
diff --git a/llvm/test/Transforms/IPConstantProp/user-with-multiple-uses.ll b/llvm/test/Transforms/SCCP/user-with-multiple-uses.ll
similarity index 100%
rename from llvm/test/Transforms/IPConstantProp/user-with-multiple-uses.ll
rename to llvm/test/Transforms/SCCP/user-with-multiple-uses.ll
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
index 8278d060756f7..16cdcd2272d50 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
@@ -36,7 +36,7 @@ target triple = "aarch64--linux-gnu"
 ; YAML-NEXT: Function:        getelementptr_4x32
 ; YAML-NEXT: Args:
 ; YAML-NEXT:   - String:          'SLP vectorized with cost '
-; YAML-NEXT:   - Cost:            '6'
+; YAML-NEXT:   - Cost:            '16'
 ; YAML-NEXT:   - String:          ' and with tree size '
 ; YAML-NEXT:   - TreeSize:        '3'
 
@@ -46,50 +46,49 @@ define i32 @getelementptr_4x32(i32* nocapture readonly %g, i32 %n, i32 %x, i32 %
 ; CHECK-NEXT:    [[CMP31:%.*]] = icmp sgt i32 [[N:%.*]], 0
 ; CHECK-NEXT:    br i1 [[CMP31]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       for.body.preheader:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 0, i32 undef>, i32 [[X:%.*]], i32 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> undef, i32 [[Y:%.*]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Z:%.*]], i32 1
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>, i32 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[Y:%.*]], i32 2
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[Z:%.*]], i32 3
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP22:%.*]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP21:%.*]], i32 1
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[TMP4:%.*]] = phi <2 x i32> [ zeroinitializer, [[FOR_BODY_PREHEADER]] ], [ [[TMP22]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = phi <2 x i32> [ zeroinitializer, [[FOR_BODY_PREHEADER]] ], [ [[TMP21]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[TMP4]], i32 0
 ; CHECK-NEXT:    [[T4:%.*]] = shl nsw i32 [[TMP5]], 1
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> undef, i32 [[T4]], i32 0
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <2 x i32> [[TMP7]], [[TMP0]]
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i32> [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> undef, i32 [[T4]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <4 x i32> [[TMP7]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP8]], i32 0
 ; CHECK-NEXT:    [[TMP10:%.*]] = sext i32 [[TMP9]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[G:%.*]], i64 [[TMP10]]
 ; CHECK-NEXT:    [[T6:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[TMP4]], i32 1
 ; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[T6]], [[TMP11]]
-; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i32> [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[TMP8]], i32 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = sext i32 [[TMP12]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP13]]
 ; CHECK-NEXT:    [[T8:%.*]] = load i32, i32* [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    [[ADD6:%.*]] = add nsw i32 [[ADD1]], [[T8]]
-; CHECK-NEXT:    [[TMP14:%.*]] = add nsw <2 x i32> [[TMP7]], [[TMP2]]
-; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i32> [[TMP14]], i32 0
-; CHECK-NEXT:    [[TMP16:%.*]] = sext i32 [[TMP15]] to i64
-; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP16]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i32> [[TMP8]], i32 2
+; CHECK-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP15]]
 ; CHECK-NEXT:    [[T10:%.*]] = load i32, i32* [[ARRAYIDX10]], align 4
 ; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[ADD6]], [[T10]]
-; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i32> [[TMP14]], i32 1
-; CHECK-NEXT:    [[TMP18:%.*]] = sext i32 [[TMP17]] to i64
-; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP18]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i32> [[TMP8]], i32 3
+; CHECK-NEXT:    [[TMP17:%.*]] = sext i32 [[TMP16]] to i64
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP17]]
 ; CHECK-NEXT:    [[T12:%.*]] = load i32, i32* [[ARRAYIDX15]], align 4
-; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <2 x i32> undef, i32 [[TMP5]], i32 0
-; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[ADD11]], i32 1
-; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <2 x i32> <i32 1, i32 undef>, i32 [[T12]], i32 1
-; CHECK-NEXT:    [[TMP22]] = add nsw <2 x i32> [[TMP20]], [[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x i32> [[TMP22]], i32 0
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP23]], [[N]]
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <2 x i32> undef, i32 [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <2 x i32> [[TMP18]], i32 [[ADD11]], i32 1
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <2 x i32> <i32 1, i32 undef>, i32 [[T12]], i32 1
+; CHECK-NEXT:    [[TMP21]] = add nsw <2 x i32> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x i32> [[TMP21]], i32 0
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP22]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]
 ;
 entry:
@@ -235,3 +234,161 @@ for.body:
   %exitcond = icmp eq i32 %indvars.iv.next , %n
   br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
 }
+
+@global = internal global { i32* } zeroinitializer, align 8
+
+; Make sure we vectorize to maximize the load with when loading i16 and
+; extending it for compute operations.
+define void @test_i16_extend(i16* %p.1, i16* %p.2, i32 %idx.i32) {
+; CHECK-LABEL: @test_i16_extend(
+; CHECK-NEXT:    [[P_0:%.*]] = load i32*, i32** getelementptr inbounds ({ i32* }, { i32* }* @global, i64 0, i32 0), align 8
+; CHECK-NEXT:    [[IDX_0:%.*]] = zext i32 [[IDX_I32:%.*]] to i64
+; CHECK-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i16, i16* [[P_1:%.*]], i64 [[IDX_0]]
+; CHECK-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i16, i16* [[P_2:%.*]], i64 [[IDX_0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[TMP53]] to <8 x i16>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16* [[TMP56]] to <8 x i16>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* [[TMP4]], align 2
+; CHECK-NEXT:    [[TMP6:%.*]] = zext <8 x i16> [[TMP5]] to <8 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = sub nsw <8 x i32> [[TMP3]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr inbounds i32, i32* [[P_0]], i64 [[TMP9]]
+; CHECK-NEXT:    [[L_1:%.*]] = load i32, i32* [[TMP60]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[P_0]], i64 [[TMP11]]
+; CHECK-NEXT:    [[L_2:%.*]] = load i32, i32* [[TMP71]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i32> [[TMP7]], i32 2
+; CHECK-NEXT:    [[TMP13:%.*]] = sext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[TMP82:%.*]] = getelementptr inbounds i32, i32* [[P_0]], i64 [[TMP13]]
+; CHECK-NEXT:    [[L_3:%.*]] = load i32, i32* [[TMP82]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP7]], i32 3
+; CHECK-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[TMP93:%.*]] = getelementptr inbounds i32, i32* [[P_0]], i64 [[TMP15]]
+; CHECK-NEXT:    [[L_4:%.*]] = load i32, i32* [[TMP93]], align 4
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[TMP7]], i32 4
+; CHECK-NEXT:    [[TMP17:%.*]] = sext i32 [[TMP16]] to i64
+; CHECK-NEXT:    [[TMP104:%.*]] = getelementptr inbounds i32, i32* [[P_0]], i64 [[TMP17]]
+; CHECK-NEXT:    [[L_5:%.*]] = load i32, i32* [[TMP104]], align 4
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i32> [[TMP7]], i32 5
+; CHECK-NEXT:    [[TMP19:%.*]] = sext i32 [[TMP18]] to i64
+; CHECK-NEXT:    [[TMP115:%.*]] = getelementptr inbounds i32, i32* [[P_0]], i64 [[TMP19]]
+; CHECK-NEXT:    [[L_6:%.*]] = load i32, i32* [[TMP115]], align 4
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP7]], i32 6
+; CHECK-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
+; CHECK-NEXT:    [[TMP126:%.*]] = getelementptr inbounds i32, i32* [[P_0]], i64 [[TMP21]]
+; CHECK-NEXT:    [[L_7:%.*]] = load i32, i32* [[TMP126]], align 4
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP7]], i32 7
+; CHECK-NEXT:    [[TMP23:%.*]] = sext i32 [[TMP22]] to i64
+; CHECK-NEXT:    [[TMP137:%.*]] = getelementptr inbounds i32, i32* [[P_0]], i64 [[TMP23]]
+; CHECK-NEXT:    [[L_8:%.*]] = load i32, i32* [[TMP137]], align 4
+; CHECK-NEXT:    call void @use(i32 [[L_1]], i32 [[L_2]], i32 [[L_3]], i32 [[L_4]], i32 [[L_5]], i32 [[L_6]], i32 [[L_7]], i32 [[L_8]])
+; CHECK-NEXT:    ret void
+;
+  %g = getelementptr inbounds { i32*}, { i32 *}* @global, i64 0, i32 0
+  %p.0 = load i32*, i32** %g, align 8
+
+  %idx.0 = zext i32 %idx.i32 to i64
+  %idx.1 = add nsw i64 %idx.0, 1
+  %idx.2 = add nsw i64 %idx.0, 2
+  %idx.3 = add nsw i64 %idx.0, 3
+  %idx.4 = add nsw i64 %idx.0, 4
+  %idx.5 = add nsw i64 %idx.0, 5
+  %idx.6 = add nsw i64 %idx.0, 6
+  %idx.7 = add nsw i64 %idx.0, 7
+
+  %tmp53 = getelementptr inbounds i16, i16* %p.1, i64 %idx.0
+  %op1.l = load i16, i16* %tmp53, align 2
+  %op1.ext = zext i16 %op1.l to i64
+  %tmp56 = getelementptr inbounds i16, i16* %p.2, i64 %idx.0
+  %op2.l = load i16, i16* %tmp56, align 2
+  %op2.ext = zext i16 %op2.l to i64
+  %sub.1 = sub nsw i64 %op1.ext, %op2.ext
+
+  %tmp60 = getelementptr inbounds i32, i32* %p.0, i64 %sub.1
+  %l.1 = load i32, i32* %tmp60, align 4
+
+  %tmp64 = getelementptr inbounds i16, i16* %p.1, i64 %idx.1
+  %tmp65 = load i16, i16* %tmp64, align 2
+  %tmp66 = zext i16 %tmp65 to i64
+  %tmp67 = getelementptr inbounds i16, i16* %p.2, i64 %idx.1
+  %tmp68 = load i16, i16* %tmp67, align 2
+  %tmp69 = zext i16 %tmp68 to i64
+  %sub.2 = sub nsw i64 %tmp66, %tmp69
+
+  %tmp71 = getelementptr inbounds i32, i32* %p.0, i64 %sub.2
+  %l.2 = load i32, i32* %tmp71, align 4
+
+  %tmp75 = getelementptr inbounds i16, i16* %p.1, i64 %idx.2
+  %tmp76 = load i16, i16* %tmp75, align 2
+  %tmp77 = zext i16 %tmp76 to i64
+  %tmp78 = getelementptr inbounds i16, i16* %p.2, i64 %idx.2
+  %tmp79 = load i16, i16* %tmp78, align 2
+  %tmp80 = zext i16 %tmp79 to i64
+  %sub.3 = sub nsw i64 %tmp77, %tmp80
+
+  %tmp82 = getelementptr inbounds i32, i32* %p.0, i64 %sub.3
+  %l.3 = load i32, i32* %tmp82, align 4
+
+  %tmp86 = getelementptr inbounds i16, i16* %p.1, i64 %idx.3
+  %tmp87 = load i16, i16* %tmp86, align 2
+  %tmp88 = zext i16 %tmp87 to i64
+  %tmp89 = getelementptr inbounds i16, i16* %p.2, i64 %idx.3
+  %tmp90 = load i16, i16* %tmp89, align 2
+  %tmp91 = zext i16 %tmp90 to i64
+  %sub.4 = sub nsw i64 %tmp88, %tmp91
+
+  %tmp93 = getelementptr inbounds i32, i32* %p.0, i64 %sub.4
+  %l.4 = load i32, i32* %tmp93, align 4
+
+  %tmp97 = getelementptr inbounds i16, i16* %p.1, i64 %idx.4
+  %tmp98 = load i16, i16* %tmp97, align 2
+  %tmp99 = zext i16 %tmp98 to i64
+  %tmp100 = getelementptr inbounds i16, i16* %p.2, i64 %idx.4
+  %tmp101 = load i16, i16* %tmp100, align 2
+  %tmp102 = zext i16 %tmp101 to i64
+  %sub.5 = sub nsw i64 %tmp99, %tmp102
+
+  %tmp104 = getelementptr inbounds i32, i32* %p.0, i64 %sub.5
+  %l.5 = load i32, i32* %tmp104, align 4
+
+  %tmp108 = getelementptr inbounds i16, i16* %p.1, i64 %idx.5
+  %tmp109 = load i16, i16* %tmp108, align 2
+  %tmp110 = zext i16 %tmp109 to i64
+  %tmp111 = getelementptr inbounds i16, i16* %p.2, i64 %idx.5
+  %tmp112 = load i16, i16* %tmp111, align 2
+  %tmp113 = zext i16 %tmp112 to i64
+  %sub.6 = sub nsw i64 %tmp110, %tmp113
+
+  %tmp115 = getelementptr inbounds i32, i32* %p.0, i64 %sub.6
+  %l.6 = load i32, i32* %tmp115, align 4
+
+  %tmp119 = getelementptr inbounds i16, i16* %p.1, i64 %idx.6
+  %tmp120 = load i16, i16* %tmp119, align 2
+  %tmp121 = zext i16 %tmp120 to i64
+  %tmp122 = getelementptr inbounds i16, i16* %p.2, i64 %idx.6
+  %tmp123 = load i16, i16* %tmp122, align 2
+  %tmp124 = zext i16 %tmp123 to i64
+  %sub.7 = sub nsw i64 %tmp121, %tmp124
+
+  %tmp126 = getelementptr inbounds i32, i32* %p.0, i64 %sub.7
+  %l.7 = load i32, i32* %tmp126, align 4
+
+  %tmp130 = getelementptr inbounds i16, i16* %p.1, i64 %idx.7
+  %tmp131 = load i16, i16* %tmp130, align 2
+  %tmp132 = zext i16 %tmp131 to i64
+  %tmp133 = getelementptr inbounds i16, i16* %p.2, i64 %idx.7
+  %tmp134 = load i16, i16* %tmp133, align 2
+  %tmp135 = zext i16 %tmp134 to i64
+  %sub.8 = sub nsw i64 %tmp132, %tmp135
+
+  %tmp137 = getelementptr inbounds i32, i32* %p.0, i64 %sub.8
+  %l.8 = load i32, i32* %tmp137, align 4
+
+  call void @use(i32 %l.1, i32 %l.2, i32 %l.3, i32 %l.4, i32 %l.5, i32 %l.6, i32 %l.7, i32 %l.8)
+  ret void
+}
+
+declare void @use(i32, i32, i32, i32, i32, i32, i32, i32)
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/load-store-q.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/load-store-q.ll
index 6ff1118d3cc67..b8dabed51e6ab 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/load-store-q.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/load-store-q.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -basicaa -slp-vectorizer < %s | FileCheck %s
+; RUN: opt -S -basic-aa -slp-vectorizer < %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 target triple = "arm64-apple-ios5.0.0"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/matmul.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/matmul.ll
index fdd12c1e807e0..52aa77e01d2b3 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/matmul.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/matmul.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=aarch64-unknown-unknown -mcpu=cortex-a53 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=aarch64-unknown-unknown -mcpu=cortex-a53 | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/nontemporal.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/nontemporal.ll
index 98c033262a080..e93447603711c 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/nontemporal.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/nontemporal.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -basicaa -slp-vectorizer -dce < %s | FileCheck %s
+; RUN: opt -S -basic-aa -slp-vectorizer -dce < %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 target triple = "arm64-apple-ios5.0.0"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/sdiv-pow2.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/sdiv-pow2.ll
index 8311f20514927..dfd322013399f 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/sdiv-pow2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/sdiv-pow2.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu -mcpu=cortex-a57 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu -mcpu=cortex-a57 | FileCheck %s
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-order.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-order.ll
new file mode 100644
index 0000000000000..9286b7ce9a690
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-order.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios13.0.0"
+
+declare i1 @cond()
+declare i32* @get_ptr()
+
+define void @test(i64* %ptr, i64* noalias %res) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[CALL_I_I:%.*]] = call i32* @get_ptr()
+; CHECK-NEXT:    [[L_0_0:%.*]] = load i32, i32* [[CALL_I_I]], align 2
+; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr i32, i32* [[CALL_I_I]], i32 2
+; CHECK-NEXT:    [[L_1_0:%.*]] = load i32, i32* [[GEP_1]], align 2
+; CHECK-NEXT:    [[EXT_0_0:%.*]] = zext i32 [[L_0_0]] to i64
+; CHECK-NEXT:    [[EXT_1_0:%.*]] = zext i32 [[L_1_0]] to i64
+; CHECK-NEXT:    [[SUB_1:%.*]] = sub nsw i64 [[EXT_0_0]], [[EXT_1_0]]
+; CHECK-NEXT:    [[GEP_2:%.*]] = getelementptr i32, i32* [[CALL_I_I]], i32 1
+; CHECK-NEXT:    [[L_0_1:%.*]] = load i32, i32* [[GEP_2]], align 2
+; CHECK-NEXT:    [[GEP_3:%.*]] = getelementptr i32, i32* [[CALL_I_I]], i32 3
+; CHECK-NEXT:    [[L_1_1:%.*]] = load i32, i32* [[GEP_3]], align 2
+; CHECK-NEXT:    [[EXT_0_1:%.*]] = zext i32 [[L_0_1]] to i64
+; CHECK-NEXT:    [[EXT_1_1:%.*]] = zext i32 [[L_1_1]] to i64
+; CHECK-NEXT:    [[SUB_2:%.*]] = sub nsw i64 [[EXT_0_1]], [[EXT_1_1]]
+; CHECK-NEXT:    store i64 [[SUB_1]], i64* [[RES:%.*]], align 8
+; CHECK-NEXT:    [[RES_1:%.*]] = getelementptr i64, i64* [[RES]], i64 1
+; CHECK-NEXT:    store i64 [[SUB_2]], i64* [[RES_1]], align 8
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label [[FOR_BODY]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:                                  ; preds = %for.body, %entry
+  %call.i.i = call i32* @get_ptr()
+  %l.0.0 = load i32, i32* %call.i.i, align 2
+  %gep.1 = getelementptr i32, i32* %call.i.i, i32 2
+  %l.1.0 = load i32, i32* %gep.1, align 2
+  %ext.0.0 = zext i32 %l.0.0 to i64
+  %ext.1.0 = zext i32 %l.1.0 to i64
+  %sub.1 = sub nsw i64 %ext.0.0, %ext.1.0
+
+  %gep.2 = getelementptr i32, i32* %call.i.i, i32 1
+  %l.0.1 = load i32, i32* %gep.2, align 2
+  %gep.3 = getelementptr i32, i32* %call.i.i, i32 3
+  %l.1.1 = load i32, i32* %gep.3, align 2
+  %ext.0.1 = zext i32 %l.0.1 to i64
+  %ext.1.1 = zext i32 %l.1.1 to i64
+  %sub.2 = sub nsw i64 %ext.0.1, %ext.1.1
+
+  store i64 %sub.1, i64* %res
+  %res.1 = getelementptr i64, i64* %res, i64 1
+  store i64 %sub.2, i64* %res.1
+
+  %c = call i1 @cond()
+  br i1 %c, label %for.body, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/ARM/memory.ll b/llvm/test/Transforms/SLPVectorizer/ARM/memory.ll
index 70e87033207a4..651cdcb53fbb1 100644
--- a/llvm/test/Transforms/SLPVectorizer/ARM/memory.ll
+++ b/llvm/test/Transforms/SLPVectorizer/ARM/memory.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=thumbv7-apple-ios3.0.0 -mcpu=swift | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=thumbv7-apple-ios3.0.0 -mcpu=swift | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/ARM/sroa.ll b/llvm/test/Transforms/SLPVectorizer/ARM/sroa.ll
index c43e8f10a59b8..cd17cd83bfa4f 100644
--- a/llvm/test/Transforms/SLPVectorizer/ARM/sroa.ll
+++ b/llvm/test/Transforms/SLPVectorizer/ARM/sroa.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mcpu=swift -mtriple=thumbv7-apple-ios -basicaa -slp-vectorizer < %s | FileCheck %s
+; RUN: opt -S -mcpu=swift -mtriple=thumbv7-apple-ios -basic-aa -slp-vectorizer < %s | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/NVPTX/v2f16.ll b/llvm/test/Transforms/SLPVectorizer/NVPTX/v2f16.ll
index 6c474705abcfa..c45a900ccc555 100644
--- a/llvm/test/Transforms/SLPVectorizer/NVPTX/v2f16.ll
+++ b/llvm/test/Transforms/SLPVectorizer/NVPTX/v2f16.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=nvptx64-nvidia-cuda -mcpu=sm_70 | FileCheck %s
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=nvptx64-nvidia-cuda -mcpu=sm_40 | FileCheck %s -check-prefix=NOVECTOR
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=nvptx64-nvidia-cuda -mcpu=sm_70 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=nvptx64-nvidia-cuda -mcpu=sm_40 | FileCheck %s -check-prefix=NOVECTOR
 
 define void @fusion(i8* noalias nocapture align 256 dereferenceable(19267584) %arg, i8* noalias nocapture readonly align 256 dereferenceable(19267584) %arg1, i32 %arg2, i32 %arg3) local_unnamed_addr #0 {
 ; CHECK-LABEL: @fusion(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/addsub.ll b/llvm/test/Transforms/SLPVectorizer/X86/addsub.ll
index 510bc9e5b20f6..4cca20ee5dd58 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/addsub.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/addsub.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/align.ll b/llvm/test/Transforms/SLPVectorizer/X86/align.ll
index c27890ca16917..a4ddfa1989d9a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/align.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/align.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
index 4c72225abb210..3c595c8022a29 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
 
 define <8 x float> @ceil_floor(<8 x float> %a) {
 ; CHECK-LABEL: @ceil_floor(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
index f94fbdef4d88e..2db27fd75da3b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
 
 define <8 x float> @sitofp_uitofp(<8 x i32> %a) {
 ; SSE-LABEL: @sitofp_uitofp(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
index e1cad3e14014f..88414d4f1a5ec 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
 
 define <8 x float> @fadd_fsub_v8f32(<8 x float> %a, <8 x float> %b) {
 ; CHECK-LABEL: @fadd_fsub_v8f32(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
index 0324f35330c18..ccb19a4874b3c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
 
 define <8 x i32> @add_sub_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; CHECK-LABEL: @add_sub_v8i32(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-saddo.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-saddo.ll
index b310c15c2b0d6..683c1a9fc6450 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-saddo.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-saddo.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-ssat.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-ssat.ll
index 4ec98a89c100d..2b5c6ea9131a2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-ssat.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-ssat.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-uaddo.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-uaddo.ll
index b057121a2753a..cf328fbc57bc8 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-uaddo.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-uaddo.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll
index 65104d2a50717..8ae6a6b22de95 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-add.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-add.ll
index 087b27b522977..015a5353e9178 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-add.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-add.ll
@@ -1,13 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=-prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=+prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=-prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=+prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=-prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=+prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=-prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=+prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fix.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fix.ll
index af0b42f7ae1e0..612145c63b8da 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-fix.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fix.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
index 9de83db6ab734..abdb09246f1b7 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=-prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=+prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=-prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=+prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=-prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=+prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=-prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=+prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
 
 ;
 ; 128-bit Vectors
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-mul-smulo.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-mul-smulo.ll
index 6aa7010bf2b05..9ed3a3b5fc2a9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-mul-smulo.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-mul-smulo.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-mul-umulo.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-mul-umulo.ll
index d9f9fae3e74ab..d8716bde9910e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-mul-umulo.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-mul-umulo.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-mul.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-mul.ll
index 8dc59d3e84843..b1f10b41bee39 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-mul.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-mul.ll
@@ -1,13 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=-prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=+prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=-prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=+prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=-prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=+prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=-prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=+prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssat.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssat.ll
index dd9d4f62f5b7b..c309355d77a85 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssat.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssat.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssubo.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssubo.ll
index f21ba1f96342d..65bb608228af0 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssubo.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssubo.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll
index 5080d05e32e89..6558d8e4d467b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usubo.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usubo.ll
index 5edbcd93fcc06..bd0303b2ebb6f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usubo.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usubo.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub.ll
index eb78e39b02924..c31f463208fab 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub.ll
@@ -1,13 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=-prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=+prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=-prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=+prefer-128-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=-prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=+prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=-prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=+prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/atomics.ll b/llvm/test/Transforms/SLPVectorizer/X86/atomics.ll
index c7f05491a1859..bbfa3914ed104 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/atomics.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/atomics.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S |FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S |FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 @x = global [4 x i32] zeroinitializer, align 16
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bad_types.ll b/llvm/test/Transforms/SLPVectorizer/X86/bad_types.ll
index d229acd17894a..b9fd032b5662f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bad_types.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bad_types.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/barriercall.ll b/llvm/test/Transforms/SLPVectorizer/X86/barriercall.ll
index 2ea29ed95c77f..485a66940bc6f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/barriercall.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/barriercall.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bitreverse.ll b/llvm/test/Transforms/SLPVectorizer/X86/bitreverse.ll
index 58b173202759e..ce8e7be29736e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bitreverse.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bitreverse.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=XOP
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver4 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=XOP
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=XOP
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver4 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=XOP
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bswap.ll b/llvm/test/Transforms/SLPVectorizer/X86/bswap.ll
index 79ce24f59119f..bdb384f6fa9c5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bswap.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bswap.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/call.ll b/llvm/test/Transforms/SLPVectorizer/X86/call.ll
index 0933897527058..dd9d14c78faa1 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/call.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/call.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=-999 -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -slp-threshold=-999 -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cast.ll b/llvm/test/Transforms/SLPVectorizer/X86/cast.ll
index 5d023008e60f0..4d476791ff91d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cast.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cast.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 -basicaa -slp-vectorizer -dce -S | FileCheck %s --check-prefixes=CHECK,SSE42
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -basicaa -slp-vectorizer -dce -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 -basic-aa -slp-vectorizer -dce -S | FileCheck %s --check-prefixes=CHECK,SSE42
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -basic-aa -slp-vectorizer -dce -S | FileCheck %s --check-prefixes=CHECK,AVX
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cmp_sel.ll b/llvm/test/Transforms/SLPVectorizer/X86/cmp_sel.ll
index 331c8e0646639..668ee74df3a60 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cmp_sel.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cmp_sel.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 ; int foo(double * restrict A, double * restrict B, double G) {
 ;   A[0] = (B[10] ? G : 1);
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/compare-reduce.ll b/llvm/test/Transforms/SLPVectorizer/X86/compare-reduce.ll
index c16ac5385598a..3ac8c04774a4c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/compare-reduce.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/compare-reduce.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.7.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll b/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
index 42c95805d2c5c..e4000b52c4a96 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.9.0"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/continue_vectorizing.ll b/llvm/test/Transforms/SLPVectorizer/X86/continue_vectorizing.ll
index 060cb054b7f00..75248e1d7fa5c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/continue_vectorizing.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/continue_vectorizing.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_7zip.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_7zip.ll
index e7bff49c09f03..59bc984118fab 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_7zip.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_7zip.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_binaryop.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_binaryop.ll
index eec53739b6796..ba315a179ce2b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_binaryop.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_binaryop.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-darwin13.3.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet.ll
index a1a3f50765a41..bbce681cdfb7f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll
index 0ee8943f58a3d..abf241a810541 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_cmpop.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_cmpop.ll
index 550b8314d606f..fb18d292475c4 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_cmpop.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_cmpop.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s -check-prefixes=CHECK,SSE
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mattr=+avx | FileCheck %s -check-prefixes=CHECK,AVX
+; RUN: opt < %s -basic-aa -slp-vectorizer -S | FileCheck %s -check-prefixes=CHECK,SSE
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mattr=+avx | FileCheck %s -check-prefixes=CHECK,AVX
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.10.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_dequeue.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_dequeue.ll
index b5f736af62445..fc1533ff09864 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_dequeue.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_dequeue.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_flop7.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_flop7.ll
index 57dd780ec4a9e..d095316bbdae8 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_flop7.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_flop7.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_gep.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_gep.ll
index 8fdc72396dd44..346f81e8b1ae3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_gep.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_gep.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod.ll
index 53f61fe68ea61..161c1077a9e71 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_mandeltext.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_mandeltext.ll
index 8f57a820197e4..75e901afbaf71 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_mandeltext.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_mandeltext.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_netbsd_decompress.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_netbsd_decompress.ll
index a52ec6ba9de00..de17d44f39a04 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_netbsd_decompress.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_netbsd_decompress.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling.ll
index 7cb25685493b7..8e93552ff3663 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-darwin13.3.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_sim4b1.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_sim4b1.ll
index 9a07cfc8431b3..b4bc79a5247d2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_sim4b1.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_sim4b1.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
index b5dad94c65bcf..5548dd2b895c9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_vectorizeTree.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_vectorizeTree.ll
index 24f5bad039894..d646180f36d98 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_vectorizeTree.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_vectorizeTree.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -basicaa -slp-vectorizer -mtriple=x86_64-apple-macosx10.9.0 -mcpu=corei7-avx -S < %s | FileCheck %s
+; RUN: opt -basic-aa -slp-vectorizer -mtriple=x86_64-apple-macosx10.9.0 -mcpu=corei7-avx -S < %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.9.0"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cross_block_slp.ll b/llvm/test/Transforms/SLPVectorizer/X86/cross_block_slp.ll
index a3d98e39ce1b7..407cfbc2090e0 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cross_block_slp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cross_block_slp.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cse.ll b/llvm/test/Transforms/SLPVectorizer/X86/cse.ll
index 60a75421c6836..ca6d6ccce767c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cse.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cse.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
 target triple = "i386-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll b/llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll
index 562d5d916d26b..fef682d4b9bda 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE42
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ctpop.ll b/llvm/test/Transforms/SLPVectorizer/X86/ctpop.ll
index 8782e6768c05d..8dfcfbf285da1 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/ctpop.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ctpop.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -mattr=+sse2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE2
-; RUN: opt < %s -mtriple=x86_64-unknown -mattr=+sse4.2,+popcnt -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE42
-; RUN: opt < %s -mtriple=x86_64-unknown -mattr=+avx,+popcnt -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mattr=+avx2,+popcnt -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mattr=+sse2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mattr=+sse4.2,+popcnt -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown -mattr=+avx,+popcnt -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mattr=+avx2,+popcnt -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cttz.ll b/llvm/test/Transforms/SLPVectorizer/X86/cttz.ll
index 90c7c297d0be8..eb37546f07a05 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cttz.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cttz.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE42
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cycle_dup.ll b/llvm/test/Transforms/SLPVectorizer/X86/cycle_dup.ll
index 2ba0a15fed22a..36aec7fcbf48a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cycle_dup.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cycle_dup.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.9.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/debug_info.ll b/llvm/test/Transforms/SLPVectorizer/X86/debug_info.ll
index de7c1206ee66e..25ff6b6ba702a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/debug_info.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/debug_info.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.7.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/diamond.ll b/llvm/test/Transforms/SLPVectorizer/X86/diamond.ll
index 3cba8ea00f7cd..24fdb2a3a8004 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/diamond.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/diamond.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/external_user.ll b/llvm/test/Transforms/SLPVectorizer/X86/external_user.ll
index 1e47f7a51fd10..9f14ff88cc816 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/external_user.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/external_user.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract.ll b/llvm/test/Transforms/SLPVectorizer/X86/extract.ll
index 9a741cbb4cfdc..5ee85a8d9a868 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extract.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extract.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract_in_tree_user.ll b/llvm/test/Transforms/SLPVectorizer/X86/extract_in_tree_user.ll
index e7197f8b113fb..1f48bc73a132e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extract_in_tree_user.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extract_in_tree_user.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=i386-apple-macosx10.9.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=i386-apple-macosx10.9.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extractcost.ll b/llvm/test/Transforms/SLPVectorizer/X86/extractcost.ll
index 834f5a089b508..d8c5f49a03ec8 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extractcost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extractcost.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fabs.ll b/llvm/test/Transforms/SLPVectorizer/X86/fabs.ll
index 101fd50e3e57c..19898df286711 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fabs.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fabs.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fcopysign.ll b/llvm/test/Transforms/SLPVectorizer/X86/fcopysign.ll
index b304fed3dc998..c8948e85a4608 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fcopysign.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fcopysign.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/flag.ll b/llvm/test/Transforms/SLPVectorizer/X86/flag.ll
index e695c02ef1042..3ebd34870d3ca 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/flag.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/flag.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=1000 -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -slp-threshold=1000 -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fma.ll b/llvm/test/Transforms/SLPVectorizer/X86/fma.ll
index 95b6a71e5455c..5c9f93d278533 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fma.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fma.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=NO-FMA
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=NO-FMA
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=FMA --check-prefix=FMA256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=FMA --check-prefix=FMA256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=FMA --check-prefix=FMA512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=FMA --check-prefix=FMA256
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=NO-FMA
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=NO-FMA
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=FMA --check-prefix=FMA256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=FMA --check-prefix=FMA256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=FMA --check-prefix=FMA512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=FMA --check-prefix=FMA256
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fmaxnum.ll b/llvm/test/Transforms/SLPVectorizer/X86/fmaxnum.ll
index 50710290f0da9..e03f3f808a4ff 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fmaxnum.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fmaxnum.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
 
 @srcA64 = common global [8 x double] zeroinitializer, align 64
 @srcB64 = common global [8 x double] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fminnum.ll b/llvm/test/Transforms/SLPVectorizer/X86/fminnum.ll
index 4b9cd95ae436a..b830d826c2e96 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fminnum.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fminnum.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
 
 @srcA64 = common global [8 x double] zeroinitializer, align 64
 @srcB64 = common global [8 x double] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fptosi.ll b/llvm/test/Transforms/SLPVectorizer/X86/fptosi.ll
index cd8fd1a57a062..088c77eab96c2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fptosi.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fptosi.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256DQ
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256DQ
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fptoui.ll b/llvm/test/Transforms/SLPVectorizer/X86/fptoui.ll
index 216cd7a686934..a23494e9898b6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fptoui.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fptoui.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256DQ
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256DQ
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fround.ll b/llvm/test/Transforms/SLPVectorizer/X86/fround.ll
index 071632b215647..8db65a7f20f17 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fround.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fround.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE41
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE41
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gep.ll b/llvm/test/Transforms/SLPVectorizer/X86/gep.ll
index a3cce8b0c5ee0..f04fff807379d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/gep.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gep.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S |FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S |FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=slp-vectorizer -S |FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-unknown"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll b/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
index de686fe0e18cf..963939289323c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
 
 ;
 ; 128-bit vectors
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hoist.ll b/llvm/test/Transforms/SLPVectorizer/X86/hoist.ll
index 5abf85f319b98..8120dc835f07b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/hoist.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hoist.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
 target triple = "i386-apple-macosx10.9.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll b/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll
index c71e0da973369..99d9a99f163a6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
 
 ;
 ; 128-bit vectors
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/implicitfloat.ll b/llvm/test/Transforms/SLPVectorizer/X86/implicitfloat.ll
index 2fbb8f0477f23..9c92bf00a2cf6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/implicitfloat.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/implicitfloat.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/in-tree-user.ll b/llvm/test/Transforms/SLPVectorizer/X86/in-tree-user.ll
index 7e0cfb77c6b7d..19deb20f65c3c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/in-tree-user.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/in-tree-user.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.7.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/insertvalue.ll b/llvm/test/Transforms/SLPVectorizer/X86/insertvalue.ll
index 1af11609fe6f7..dd126c61dd7db 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/insertvalue.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/insertvalue.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7-avx | FileCheck %s
 
 define void @julia_2xdouble([2 x double]* sret, [2 x double]*, [2 x double]*, [2 x double]*) {
 ; CHECK-LABEL: @julia_2xdouble(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/intrinsic.ll b/llvm/test/Transforms/SLPVectorizer/X86/intrinsic.ll
index fbce7551b00b2..ae60c165d7c04 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/intrinsic.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/intrinsic.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=-999 -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -slp-threshold=-999 -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/long_chains.ll b/llvm/test/Transforms/SLPVectorizer/X86/long_chains.ll
index 52c1c6bee1216..33b30adb7bdb5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/long_chains.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/long_chains.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/loopinvariant.ll b/llvm/test/Transforms/SLPVectorizer/X86/loopinvariant.ll
index 020b50d546324..d7a9210265c79 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/loopinvariant.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/loopinvariant.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/metadata.ll b/llvm/test/Transforms/SLPVectorizer/X86/metadata.ll
index fdfd032f3ce42..7956ab3411e64 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/metadata.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/metadata.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/multi_block.ll b/llvm/test/Transforms/SLPVectorizer/X86/multi_block.ll
index f785926cc4120..b0cc02b649f04 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/multi_block.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/multi_block.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.7.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/multi_user.ll b/llvm/test/Transforms/SLPVectorizer/X86/multi_user.ll
index 9268adf9481c2..54195b9708ef2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/multi_user.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/multi_user.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.7.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll b/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll
index b7321aaf4582c..4f72ddb35c100 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 ;int foo(char * restrict A, float * restrict B, float T) {
 ;  A[0] = (T * B[10] + 4.0);
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll b/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
index 726f04aafb116..65f26abc87b41 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=-100 -instcombine -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -slp-threshold=-100 -instcombine -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ordering.ll b/llvm/test/Transforms/SLPVectorizer/X86/ordering.ll
index bc6ef182c98d0..f1b2283fff49d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/ordering.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ordering.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi.ll
index fe604e2652d27..cb7e774c06455 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/phi.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=-100 -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -slp-threshold=-100 -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
 target triple = "i386-apple-macosx10.9.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi3.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi3.ll
index 18653d6cf6d48..a0ad821fb0814 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/phi3.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi3.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi_overalignedtype.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi_overalignedtype.ll
index f708341877af5..b051e2900c5f9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/phi_overalignedtype.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi_overalignedtype.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=-100 -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -slp-threshold=-100 -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 ; We purposely over-align f64 to 128bit here.
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:128:128-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/powof2div.ll b/llvm/test/Transforms/SLPVectorizer/X86/powof2div.ll
index d7f80cf9ff54f..e6fb9cae52ecf 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/powof2div.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/powof2div.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx  | FileCheck %s --check-prefixes=CHECK,AVX1
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx  | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX2
 
 define void @powof2div_uniform(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32* noalias nocapture readonly %c){
 ; CHECK-LABEL: @powof2div_uniform(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr16628.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr16628.ll
index 9f195649c6330..63d88fcb01f4c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr16628.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr16628.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.9.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr19657.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr19657.ll
index 39db608fc9b41..30e6c03f68a49 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr19657.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr19657.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mcpu=corei7-avx | FileCheck %s --check-prefixes=ANY,AVX
-; RUN: opt < %s -basicaa -slp-vectorizer -slp-max-reg-size=128 -S -mcpu=corei7-avx | FileCheck %s --check-prefixes=ANY,MAX128
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mcpu=corei7-avx | FileCheck %s --check-prefixes=ANY,AVX
+; RUN: opt < %s -basic-aa -slp-vectorizer -slp-max-reg-size=128 -S -mcpu=corei7-avx | FileCheck %s --check-prefixes=ANY,MAX128
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr23510.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr23510.ll
index 420fdde3631bb..391fd0bcd11d6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr23510.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr23510.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; PR23510
-; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll b/llvm/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll
index 722d6f8f809ef..e3a5e3b6399d0 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S | FileCheck %s
 
 ; Check propagation of optional IR flags (PR20802). For a flag to
 ; propagate from scalar instructions to their vector replacement,
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction.ll
index 8cbc3591f9192..30ceee8f9e81c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
 target triple = "i386-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll
index b5f4335492746..531ae0006cca0 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
 target triple = "i386-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/remark_unsupported.ll b/llvm/test/Transforms/SLPVectorizer/X86/remark_unsupported.ll
index a134aec00bbbc..afc08aa4fb277 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/remark_unsupported.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/remark_unsupported.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=x86_64-pc-linux-gnu -mcpu=generic -slp-vectorizer -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=x86_64-pc-linux-gnu -mcpu=generic -slp-vectorizer --slp-max-reg-size=256 -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: FileCheck --input-file=%t --check-prefix=YAML %s
 
 ; This type is not supported by SLP
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder_phi.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder_phi.ll
index afee26f0d4076..e4b35b6f06f9e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reorder_phi.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder_phi.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer  -S -mtriple=x86_64-unknown -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer  -S -mtriple=x86_64-unknown -mcpu=corei7-avx | FileCheck %s
 
 %struct.complex = type { float, float }
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/return.ll b/llvm/test/Transforms/SLPVectorizer/X86/return.ll
index 445dcba20d5bc..4f7448f371028 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/return.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/return.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S | FileCheck %s
 target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
 target triple = "x86_64--linux-gnu"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll b/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll
index c7e419b5f562c..ce6b6be5fbc86 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
 target triple = "i386-apple-macosx10.9.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/saxpy.ll b/llvm/test/Transforms/SLPVectorizer/X86/saxpy.ll
index 7e9109a4ef00a..5d99181d76607 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/saxpy.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/saxpy.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/schedule_budget.ll b/llvm/test/Transforms/SLPVectorizer/X86/schedule_budget.ll
index 27d84b105720a..26e7e19157a73 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/schedule_budget.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/schedule_budget.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S  -slp-schedule-budget=16 -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S  -slp-schedule-budget=16 -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.9.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/scheduling.ll b/llvm/test/Transforms/SLPVectorizer/X86/scheduling.ll
index 9265724978d9e..d3bf7f1dd4989 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/scheduling.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/scheduling.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 define i32 @foo(i32* nocapture readonly %diff) #0 {
 ; CHECK-LABEL: @foo(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sext.ll b/llvm/test/Transforms/SLPVectorizer/X86/sext.ll
index 69459788fbf22..a5c35cc7e15ae 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/sext.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sext.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+avx512bw -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+avx512bw -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512BW
 
 ;
 ; vXi8
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/shift-ashr.ll b/llvm/test/Transforms/SLPVectorizer/X86/shift-ashr.ll
index 9ced739e511e1..190bbd670d72c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/shift-ashr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/shift-ashr.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver4 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=XOP
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver4 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=XOP
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/shift-lshr.ll b/llvm/test/Transforms/SLPVectorizer/X86/shift-lshr.ll
index 3164d4bd3be4c..1d680c221ff51 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/shift-lshr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/shift-lshr.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver4 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=XOP
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver4 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=XOP
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/shift-shl.ll b/llvm/test/Transforms/SLPVectorizer/X86/shift-shl.ll
index 82f9733853595..4a612f19c3063 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/shift-shl.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/shift-shl.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver4 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=XOP
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver4 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=XOP
 
 @a64 = common global [8 x i64] zeroinitializer, align 64
 @b64 = common global [8 x i64] zeroinitializer, align 64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/simple-loop.ll b/llvm/test/Transforms/SLPVectorizer/X86/simple-loop.ll
index 59b94cad17e43..72b1ff2609d64 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/simple-loop.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/simple-loop.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/simplebb.ll b/llvm/test/Transforms/SLPVectorizer/X86/simplebb.ll
index 4ddc9cbe6fb92..80a60dd4a3da1 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/simplebb.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/simplebb.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sitofp.ll b/llvm/test/Transforms/SLPVectorizer/X86/sitofp.ll
index 435836a7c200f..9d3c66d3d7b91 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/sitofp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sitofp.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256DQ
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256DQ
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sqrt.ll b/llvm/test/Transforms/SLPVectorizer/X86/sqrt.ll
index a07da8a0fbc4a..c8baf37637b4a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/sqrt.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sqrt.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mattr=avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mattr=avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mattr=avx512vl,avx512dq,avx512bw -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mattr=avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mattr=avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mattr=avx512vl,avx512dq,avx512bw -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll b/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
index 19f06805ff50e..791bea08b9b7a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 define void @tiny_tree_fully_vectorizable(double* noalias nocapture %dst, double* noalias nocapture readonly %src, i64 %count) #0 {
 ; CHECK-LABEL: @tiny_tree_fully_vectorizable(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/uitofp.ll b/llvm/test/Transforms/SLPVectorizer/X86/uitofp.ll
index 97b4cbefd0850..e07743d27834d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/uitofp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/uitofp.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256DQ
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256DQ
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/unreachable.ll b/llvm/test/Transforms/SLPVectorizer/X86/unreachable.ll
index cc1f1fca8cac6..255d27fae9c5a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/unreachable.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/unreachable.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 ; Check if the SLPVectorizer does not crash when handling
 ; unreachable blocks with unscheduleable instructions.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/zext.ll b/llvm/test/Transforms/SLPVectorizer/X86/zext.ll
index 45e5d70345750..35a9391f713fb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/zext.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/zext.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+avx512bw -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+avx512bw -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512BW
 
 ;
 ; vXi8
diff --git a/llvm/test/Transforms/SLPVectorizer/XCore/no-vector-registers.ll b/llvm/test/Transforms/SLPVectorizer/XCore/no-vector-registers.ll
index d84a4993ec304..2030657321210 100644
--- a/llvm/test/Transforms/SLPVectorizer/XCore/no-vector-registers.ll
+++ b/llvm/test/Transforms/SLPVectorizer/XCore/no-vector-registers.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=xcore  | FileCheck %s
+; RUN: opt < %s -basic-aa -slp-vectorizer -dce -S -mtriple=xcore  | FileCheck %s
 
 target datalayout = "e-p:32:32:32-a0:0:32-n32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f16:16:32-f32:32:32-f64:32:32"
 target triple = "xcore"
diff --git a/llvm/test/Transforms/SLPVectorizer/slp-max-phi-size.ll b/llvm/test/Transforms/SLPVectorizer/slp-max-phi-size.ll
new file mode 100644
index 0000000000000..466e83d0260af
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/slp-max-phi-size.ll
@@ -0,0 +1,607 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -slp-max-reg-size=32 < %s | FileCheck -check-prefix=MAX32 %s
+; RUN: opt -slp-vectorizer -S -slp-max-reg-size=256 < %s | FileCheck -check-prefix=MAX256 %s
+; RUN: opt -slp-vectorizer -S -slp-max-reg-size=1024 < %s | FileCheck -check-prefix=MAX1024 %s
+
+define void @phi_float32(half %hval, float %fval) {
+; MAX32-LABEL: @phi_float32(
+; MAX32-NEXT:  bb:
+; MAX32-NEXT:    br label [[BB1:%.*]]
+; MAX32:       bb1:
+; MAX32-NEXT:    [[I:%.*]] = fpext half [[HVAL:%.*]] to float
+; MAX32-NEXT:    [[I1:%.*]] = fmul float [[I]], [[FVAL:%.*]]
+; MAX32-NEXT:    [[I2:%.*]] = fadd float 0.000000e+00, [[I1]]
+; MAX32-NEXT:    [[I3:%.*]] = fpext half [[HVAL]] to float
+; MAX32-NEXT:    [[I4:%.*]] = fmul float [[I3]], [[FVAL]]
+; MAX32-NEXT:    [[I5:%.*]] = fadd float 0.000000e+00, [[I4]]
+; MAX32-NEXT:    [[I6:%.*]] = fpext half [[HVAL]] to float
+; MAX32-NEXT:    [[I7:%.*]] = fmul float [[I6]], [[FVAL]]
+; MAX32-NEXT:    [[I8:%.*]] = fadd float 0.000000e+00, [[I7]]
+; MAX32-NEXT:    [[I9:%.*]] = fpext half [[HVAL]] to float
+; MAX32-NEXT:    [[I10:%.*]] = fmul float [[I9]], [[FVAL]]
+; MAX32-NEXT:    [[I11:%.*]] = fadd float 0.000000e+00, [[I10]]
+; MAX32-NEXT:    [[I12:%.*]] = fmul float [[I]], [[FVAL]]
+; MAX32-NEXT:    [[I13:%.*]] = fadd float 0.000000e+00, [[I12]]
+; MAX32-NEXT:    [[I14:%.*]] = fmul float [[I3]], [[FVAL]]
+; MAX32-NEXT:    [[I15:%.*]] = fadd float 0.000000e+00, [[I14]]
+; MAX32-NEXT:    [[I16:%.*]] = fmul float [[I6]], [[FVAL]]
+; MAX32-NEXT:    [[I17:%.*]] = fadd float 0.000000e+00, [[I16]]
+; MAX32-NEXT:    [[I18:%.*]] = fmul float [[I9]], [[FVAL]]
+; MAX32-NEXT:    [[I19:%.*]] = fadd float 0.000000e+00, [[I18]]
+; MAX32-NEXT:    [[I20:%.*]] = fmul float [[I]], [[FVAL]]
+; MAX32-NEXT:    [[I21:%.*]] = fadd float 0.000000e+00, [[I20]]
+; MAX32-NEXT:    [[I22:%.*]] = fmul float [[I3]], [[FVAL]]
+; MAX32-NEXT:    [[I23:%.*]] = fadd float 0.000000e+00, [[I22]]
+; MAX32-NEXT:    [[I24:%.*]] = fmul float [[I6]], [[FVAL]]
+; MAX32-NEXT:    [[I25:%.*]] = fadd float 0.000000e+00, [[I24]]
+; MAX32-NEXT:    [[I26:%.*]] = fmul float [[I9]], [[FVAL]]
+; MAX32-NEXT:    [[I27:%.*]] = fadd float 0.000000e+00, [[I26]]
+; MAX32-NEXT:    [[I28:%.*]] = fmul float [[I]], [[FVAL]]
+; MAX32-NEXT:    [[I29:%.*]] = fadd float 0.000000e+00, [[I28]]
+; MAX32-NEXT:    [[I30:%.*]] = fmul float [[I3]], [[FVAL]]
+; MAX32-NEXT:    [[I31:%.*]] = fadd float 0.000000e+00, [[I30]]
+; MAX32-NEXT:    [[I32:%.*]] = fmul float [[I6]], [[FVAL]]
+; MAX32-NEXT:    [[I33:%.*]] = fadd float 0.000000e+00, [[I32]]
+; MAX32-NEXT:    [[I34:%.*]] = fmul float [[I9]], [[FVAL]]
+; MAX32-NEXT:    [[I35:%.*]] = fadd float 0.000000e+00, [[I34]]
+; MAX32-NEXT:    [[I36:%.*]] = fmul float [[I]], [[FVAL]]
+; MAX32-NEXT:    [[I37:%.*]] = fadd float 0.000000e+00, [[I36]]
+; MAX32-NEXT:    [[I38:%.*]] = fmul float [[I3]], [[FVAL]]
+; MAX32-NEXT:    [[I39:%.*]] = fadd float 0.000000e+00, [[I38]]
+; MAX32-NEXT:    [[I40:%.*]] = fmul float [[I6]], [[FVAL]]
+; MAX32-NEXT:    [[I41:%.*]] = fadd float 0.000000e+00, [[I40]]
+; MAX32-NEXT:    [[I42:%.*]] = fmul float [[I9]], [[FVAL]]
+; MAX32-NEXT:    [[I43:%.*]] = fadd float 0.000000e+00, [[I42]]
+; MAX32-NEXT:    [[I44:%.*]] = fmul float [[I]], [[FVAL]]
+; MAX32-NEXT:    [[I45:%.*]] = fadd float 0.000000e+00, [[I44]]
+; MAX32-NEXT:    [[I46:%.*]] = fmul float [[I3]], [[FVAL]]
+; MAX32-NEXT:    [[I47:%.*]] = fadd float 0.000000e+00, [[I46]]
+; MAX32-NEXT:    [[I48:%.*]] = fmul float [[I6]], [[FVAL]]
+; MAX32-NEXT:    [[I49:%.*]] = fadd float 0.000000e+00, [[I48]]
+; MAX32-NEXT:    [[I50:%.*]] = fmul float [[I9]], [[FVAL]]
+; MAX32-NEXT:    [[I51:%.*]] = fadd float 0.000000e+00, [[I50]]
+; MAX32-NEXT:    [[I52:%.*]] = fmul float [[I]], [[FVAL]]
+; MAX32-NEXT:    [[I53:%.*]] = fadd float 0.000000e+00, [[I52]]
+; MAX32-NEXT:    [[I54:%.*]] = fmul float [[I3]], [[FVAL]]
+; MAX32-NEXT:    [[I55:%.*]] = fadd float 0.000000e+00, [[I54]]
+; MAX32-NEXT:    [[I56:%.*]] = fmul float [[I6]], [[FVAL]]
+; MAX32-NEXT:    [[I57:%.*]] = fadd float 0.000000e+00, [[I56]]
+; MAX32-NEXT:    [[I58:%.*]] = fmul float [[I9]], [[FVAL]]
+; MAX32-NEXT:    [[I59:%.*]] = fadd float 0.000000e+00, [[I58]]
+; MAX32-NEXT:    [[I60:%.*]] = fmul float [[I]], [[FVAL]]
+; MAX32-NEXT:    [[I61:%.*]] = fadd float 0.000000e+00, [[I60]]
+; MAX32-NEXT:    [[I62:%.*]] = fmul float [[I3]], [[FVAL]]
+; MAX32-NEXT:    [[I63:%.*]] = fadd float 0.000000e+00, [[I62]]
+; MAX32-NEXT:    [[I64:%.*]] = fmul float [[I6]], [[FVAL]]
+; MAX32-NEXT:    [[I65:%.*]] = fadd float 0.000000e+00, [[I64]]
+; MAX32-NEXT:    [[I66:%.*]] = fmul float [[I9]], [[FVAL]]
+; MAX32-NEXT:    [[I67:%.*]] = fadd float 0.000000e+00, [[I66]]
+; MAX32-NEXT:    switch i32 undef, label [[BB5:%.*]] [
+; MAX32-NEXT:    i32 0, label [[BB2:%.*]]
+; MAX32-NEXT:    i32 1, label [[BB3:%.*]]
+; MAX32-NEXT:    i32 2, label [[BB4:%.*]]
+; MAX32-NEXT:    ]
+; MAX32:       bb3:
+; MAX32-NEXT:    br label [[BB2]]
+; MAX32:       bb4:
+; MAX32-NEXT:    br label [[BB2]]
+; MAX32:       bb5:
+; MAX32-NEXT:    br label [[BB2]]
+; MAX32:       bb2:
+; MAX32-NEXT:    [[PHI1:%.*]] = phi float [ [[I19]], [[BB3]] ], [ [[I19]], [[BB4]] ], [ [[FVAL]], [[BB5]] ], [ [[I19]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI2:%.*]] = phi float [ [[I17]], [[BB3]] ], [ [[FVAL]], [[BB4]] ], [ [[I17]], [[BB5]] ], [ [[I17]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI3:%.*]] = phi float [ [[I15]], [[BB3]] ], [ [[FVAL]], [[BB4]] ], [ [[FVAL]], [[BB5]] ], [ [[FVAL]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI4:%.*]] = phi float [ [[I13]], [[BB3]] ], [ [[I13]], [[BB4]] ], [ [[I13]], [[BB5]] ], [ [[FVAL]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI5:%.*]] = phi float [ [[I11]], [[BB3]] ], [ [[I11]], [[BB4]] ], [ [[FVAL]], [[BB5]] ], [ [[I11]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI6:%.*]] = phi float [ [[I8]], [[BB3]] ], [ [[FVAL]], [[BB4]] ], [ [[I8]], [[BB5]] ], [ [[I8]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI7:%.*]] = phi float [ [[I5]], [[BB3]] ], [ [[FVAL]], [[BB4]] ], [ [[FVAL]], [[BB5]] ], [ [[FVAL]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI8:%.*]] = phi float [ [[I2]], [[BB3]] ], [ [[I2]], [[BB4]] ], [ [[I2]], [[BB5]] ], [ [[FVAL]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI9:%.*]] = phi float [ [[I21]], [[BB3]] ], [ [[I21]], [[BB4]] ], [ [[I21]], [[BB5]] ], [ [[FVAL]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI10:%.*]] = phi float [ [[I23]], [[BB3]] ], [ [[FVAL]], [[BB4]] ], [ [[FVAL]], [[BB5]] ], [ [[FVAL]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI11:%.*]] = phi float [ [[I25]], [[BB3]] ], [ [[FVAL]], [[BB4]] ], [ [[I25]], [[BB5]] ], [ [[I25]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI12:%.*]] = phi float [ [[I27]], [[BB3]] ], [ [[I27]], [[BB4]] ], [ [[FVAL]], [[BB5]] ], [ [[I27]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI13:%.*]] = phi float [ [[I29]], [[BB3]] ], [ [[I29]], [[BB4]] ], [ [[I29]], [[BB5]] ], [ [[FVAL]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI14:%.*]] = phi float [ [[I31]], [[BB3]] ], [ [[FVAL]], [[BB4]] ], [ [[FVAL]], [[BB5]] ], [ [[FVAL]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI15:%.*]] = phi float [ [[I33]], [[BB3]] ], [ [[FVAL]], [[BB4]] ], [ [[I33]], [[BB5]] ], [ [[I33]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI16:%.*]] = phi float [ [[I35]], [[BB3]] ], [ [[I35]], [[BB4]] ], [ [[FVAL]], [[BB5]] ], [ [[I35]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI17:%.*]] = phi float [ [[I37]], [[BB3]] ], [ [[I37]], [[BB4]] ], [ [[I37]], [[BB5]] ], [ [[FVAL]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI18:%.*]] = phi float [ [[I39]], [[BB3]] ], [ [[FVAL]], [[BB4]] ], [ [[FVAL]], [[BB5]] ], [ [[FVAL]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI19:%.*]] = phi float [ [[I41]], [[BB3]] ], [ [[FVAL]], [[BB4]] ], [ [[I41]], [[BB5]] ], [ [[I41]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI20:%.*]] = phi float [ [[I43]], [[BB3]] ], [ [[I43]], [[BB4]] ], [ [[FVAL]], [[BB5]] ], [ [[I43]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI21:%.*]] = phi float [ [[I45]], [[BB3]] ], [ [[I45]], [[BB4]] ], [ [[I45]], [[BB5]] ], [ [[FVAL]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI22:%.*]] = phi float [ [[I47]], [[BB3]] ], [ [[FVAL]], [[BB4]] ], [ [[FVAL]], [[BB5]] ], [ [[FVAL]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI23:%.*]] = phi float [ [[I49]], [[BB3]] ], [ [[FVAL]], [[BB4]] ], [ [[I49]], [[BB5]] ], [ [[I49]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI24:%.*]] = phi float [ [[I51]], [[BB3]] ], [ [[I51]], [[BB4]] ], [ [[FVAL]], [[BB5]] ], [ [[I51]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI25:%.*]] = phi float [ [[I53]], [[BB3]] ], [ [[I53]], [[BB4]] ], [ [[I53]], [[BB5]] ], [ [[FVAL]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI26:%.*]] = phi float [ [[I55]], [[BB3]] ], [ [[FVAL]], [[BB4]] ], [ [[FVAL]], [[BB5]] ], [ [[FVAL]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI27:%.*]] = phi float [ [[I57]], [[BB3]] ], [ [[FVAL]], [[BB4]] ], [ [[I57]], [[BB5]] ], [ [[I57]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI28:%.*]] = phi float [ [[I59]], [[BB3]] ], [ [[I59]], [[BB4]] ], [ [[FVAL]], [[BB5]] ], [ [[I59]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI29:%.*]] = phi float [ [[I61]], [[BB3]] ], [ [[I61]], [[BB4]] ], [ [[I61]], [[BB5]] ], [ [[FVAL]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI30:%.*]] = phi float [ [[I63]], [[BB3]] ], [ [[FVAL]], [[BB4]] ], [ [[FVAL]], [[BB5]] ], [ [[FVAL]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI31:%.*]] = phi float [ [[I65]], [[BB3]] ], [ [[FVAL]], [[BB4]] ], [ [[I65]], [[BB5]] ], [ [[I65]], [[BB1]] ]
+; MAX32-NEXT:    [[PHI32:%.*]] = phi float [ [[I67]], [[BB3]] ], [ [[I67]], [[BB4]] ], [ [[FVAL]], [[BB5]] ], [ [[I67]], [[BB1]] ]
+; MAX32-NEXT:    ret void
+;
+; MAX256-LABEL: @phi_float32(
+; MAX256-NEXT:  bb:
+; MAX256-NEXT:    br label [[BB1:%.*]]
+; MAX256:       bb1:
+; MAX256-NEXT:    [[TMP0:%.*]] = insertelement <4 x half> undef, half [[HVAL:%.*]], i32 0
+; MAX256-NEXT:    [[TMP1:%.*]] = insertelement <4 x half> [[TMP0]], half [[HVAL]], i32 1
+; MAX256-NEXT:    [[TMP2:%.*]] = insertelement <4 x half> [[TMP1]], half [[HVAL]], i32 2
+; MAX256-NEXT:    [[TMP3:%.*]] = insertelement <4 x half> [[TMP2]], half [[HVAL]], i32 3
+; MAX256-NEXT:    [[TMP4:%.*]] = fpext <4 x half> [[TMP3]] to <4 x float>
+; MAX256-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; MAX256-NEXT:    [[TMP5:%.*]] = insertelement <8 x float> undef, float [[FVAL:%.*]], i32 0
+; MAX256-NEXT:    [[TMP6:%.*]] = insertelement <8 x float> [[TMP5]], float [[FVAL]], i32 1
+; MAX256-NEXT:    [[TMP7:%.*]] = insertelement <8 x float> [[TMP6]], float [[FVAL]], i32 2
+; MAX256-NEXT:    [[TMP8:%.*]] = insertelement <8 x float> [[TMP7]], float [[FVAL]], i32 3
+; MAX256-NEXT:    [[TMP9:%.*]] = insertelement <8 x float> [[TMP8]], float [[FVAL]], i32 4
+; MAX256-NEXT:    [[TMP10:%.*]] = insertelement <8 x float> [[TMP9]], float [[FVAL]], i32 5
+; MAX256-NEXT:    [[TMP11:%.*]] = insertelement <8 x float> [[TMP10]], float [[FVAL]], i32 6
+; MAX256-NEXT:    [[TMP12:%.*]] = insertelement <8 x float> [[TMP11]], float [[FVAL]], i32 7
+; MAX256-NEXT:    [[TMP13:%.*]] = fmul <8 x float> [[SHUFFLE]], [[TMP12]]
+; MAX256-NEXT:    [[TMP14:%.*]] = fadd <8 x float> zeroinitializer, [[TMP13]]
+; MAX256-NEXT:    [[TMP15:%.*]] = extractelement <8 x float> [[SHUFFLE]], i32 3
+; MAX256-NEXT:    [[TMP16:%.*]] = extractelement <8 x float> [[SHUFFLE]], i32 2
+; MAX256-NEXT:    [[TMP17:%.*]] = extractelement <8 x float> [[SHUFFLE]], i32 1
+; MAX256-NEXT:    [[TMP18:%.*]] = extractelement <8 x float> [[SHUFFLE]], i32 0
+; MAX256-NEXT:    [[TMP19:%.*]] = insertelement <8 x float> undef, float [[TMP15]], i32 0
+; MAX256-NEXT:    [[TMP20:%.*]] = insertelement <8 x float> [[TMP19]], float [[TMP16]], i32 1
+; MAX256-NEXT:    [[TMP21:%.*]] = insertelement <8 x float> [[TMP20]], float [[TMP17]], i32 2
+; MAX256-NEXT:    [[TMP22:%.*]] = insertelement <8 x float> [[TMP21]], float [[TMP18]], i32 3
+; MAX256-NEXT:    [[TMP23:%.*]] = insertelement <8 x float> [[TMP22]], float [[TMP15]], i32 4
+; MAX256-NEXT:    [[TMP24:%.*]] = insertelement <8 x float> [[TMP23]], float [[TMP16]], i32 5
+; MAX256-NEXT:    [[TMP25:%.*]] = insertelement <8 x float> [[TMP24]], float [[TMP17]], i32 6
+; MAX256-NEXT:    [[TMP26:%.*]] = insertelement <8 x float> [[TMP25]], float [[TMP18]], i32 7
+; MAX256-NEXT:    [[TMP27:%.*]] = fmul <8 x float> [[TMP26]], [[TMP12]]
+; MAX256-NEXT:    [[TMP28:%.*]] = fadd <8 x float> zeroinitializer, [[TMP27]]
+; MAX256-NEXT:    [[TMP29:%.*]] = fmul <8 x float> [[TMP26]], [[TMP12]]
+; MAX256-NEXT:    [[TMP30:%.*]] = fadd <8 x float> zeroinitializer, [[TMP29]]
+; MAX256-NEXT:    [[TMP31:%.*]] = fmul <8 x float> [[TMP26]], [[TMP12]]
+; MAX256-NEXT:    [[TMP32:%.*]] = fadd <8 x float> zeroinitializer, [[TMP31]]
+; MAX256-NEXT:    [[TMP33:%.*]] = extractelement <8 x float> [[TMP14]], i32 0
+; MAX256-NEXT:    [[TMP34:%.*]] = insertelement <8 x float> undef, float [[TMP33]], i32 0
+; MAX256-NEXT:    [[TMP35:%.*]] = extractelement <8 x float> [[TMP14]], i32 1
+; MAX256-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP35]], i32 1
+; MAX256-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[FVAL]], i32 2
+; MAX256-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[FVAL]], i32 3
+; MAX256-NEXT:    [[TMP39:%.*]] = extractelement <8 x float> [[TMP14]], i32 4
+; MAX256-NEXT:    [[TMP40:%.*]] = insertelement <8 x float> [[TMP38]], float [[TMP39]], i32 4
+; MAX256-NEXT:    [[TMP41:%.*]] = extractelement <8 x float> [[TMP14]], i32 5
+; MAX256-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP41]], i32 5
+; MAX256-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[FVAL]], i32 6
+; MAX256-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[FVAL]], i32 7
+; MAX256-NEXT:    [[TMP45:%.*]] = extractelement <8 x float> [[TMP28]], i32 2
+; MAX256-NEXT:    [[TMP46:%.*]] = insertelement <8 x float> [[TMP6]], float [[TMP45]], i32 2
+; MAX256-NEXT:    [[TMP47:%.*]] = extractelement <8 x float> [[TMP28]], i32 3
+; MAX256-NEXT:    [[TMP48:%.*]] = insertelement <8 x float> [[TMP46]], float [[TMP47]], i32 3
+; MAX256-NEXT:    [[TMP49:%.*]] = insertelement <8 x float> [[TMP48]], float [[FVAL]], i32 4
+; MAX256-NEXT:    [[TMP50:%.*]] = insertelement <8 x float> [[TMP49]], float [[FVAL]], i32 5
+; MAX256-NEXT:    [[TMP51:%.*]] = extractelement <8 x float> [[TMP28]], i32 6
+; MAX256-NEXT:    [[TMP52:%.*]] = insertelement <8 x float> [[TMP50]], float [[TMP51]], i32 6
+; MAX256-NEXT:    [[TMP53:%.*]] = extractelement <8 x float> [[TMP28]], i32 7
+; MAX256-NEXT:    [[TMP54:%.*]] = insertelement <8 x float> [[TMP52]], float [[TMP53]], i32 7
+; MAX256-NEXT:    [[TMP55:%.*]] = extractelement <8 x float> [[TMP30]], i32 2
+; MAX256-NEXT:    [[TMP56:%.*]] = insertelement <8 x float> [[TMP6]], float [[TMP55]], i32 2
+; MAX256-NEXT:    [[TMP57:%.*]] = extractelement <8 x float> [[TMP30]], i32 3
+; MAX256-NEXT:    [[TMP58:%.*]] = insertelement <8 x float> [[TMP56]], float [[TMP57]], i32 3
+; MAX256-NEXT:    [[TMP59:%.*]] = insertelement <8 x float> [[TMP58]], float [[FVAL]], i32 4
+; MAX256-NEXT:    [[TMP60:%.*]] = insertelement <8 x float> [[TMP59]], float [[FVAL]], i32 5
+; MAX256-NEXT:    [[TMP61:%.*]] = extractelement <8 x float> [[TMP30]], i32 6
+; MAX256-NEXT:    [[TMP62:%.*]] = insertelement <8 x float> [[TMP60]], float [[TMP61]], i32 6
+; MAX256-NEXT:    [[TMP63:%.*]] = extractelement <8 x float> [[TMP30]], i32 7
+; MAX256-NEXT:    [[TMP64:%.*]] = insertelement <8 x float> [[TMP62]], float [[TMP63]], i32 7
+; MAX256-NEXT:    [[TMP65:%.*]] = extractelement <8 x float> [[TMP32]], i32 2
+; MAX256-NEXT:    [[TMP66:%.*]] = insertelement <8 x float> [[TMP6]], float [[TMP65]], i32 2
+; MAX256-NEXT:    [[TMP67:%.*]] = extractelement <8 x float> [[TMP32]], i32 3
+; MAX256-NEXT:    [[TMP68:%.*]] = insertelement <8 x float> [[TMP66]], float [[TMP67]], i32 3
+; MAX256-NEXT:    [[TMP69:%.*]] = insertelement <8 x float> [[TMP68]], float [[FVAL]], i32 4
+; MAX256-NEXT:    [[TMP70:%.*]] = insertelement <8 x float> [[TMP69]], float [[FVAL]], i32 5
+; MAX256-NEXT:    [[TMP71:%.*]] = extractelement <8 x float> [[TMP32]], i32 6
+; MAX256-NEXT:    [[TMP72:%.*]] = insertelement <8 x float> [[TMP70]], float [[TMP71]], i32 6
+; MAX256-NEXT:    [[TMP73:%.*]] = extractelement <8 x float> [[TMP32]], i32 7
+; MAX256-NEXT:    [[TMP74:%.*]] = insertelement <8 x float> [[TMP72]], float [[TMP73]], i32 7
+; MAX256-NEXT:    switch i32 undef, label [[BB5:%.*]] [
+; MAX256-NEXT:    i32 0, label [[BB2:%.*]]
+; MAX256-NEXT:    i32 1, label [[BB3:%.*]]
+; MAX256-NEXT:    i32 2, label [[BB4:%.*]]
+; MAX256-NEXT:    ]
+; MAX256:       bb3:
+; MAX256-NEXT:    br label [[BB2]]
+; MAX256:       bb4:
+; MAX256-NEXT:    [[TMP75:%.*]] = insertelement <8 x float> [[TMP34]], float [[FVAL]], i32 1
+; MAX256-NEXT:    [[TMP76:%.*]] = insertelement <8 x float> [[TMP75]], float [[FVAL]], i32 2
+; MAX256-NEXT:    [[TMP77:%.*]] = extractelement <8 x float> [[TMP14]], i32 3
+; MAX256-NEXT:    [[TMP78:%.*]] = insertelement <8 x float> [[TMP76]], float [[TMP77]], i32 3
+; MAX256-NEXT:    [[TMP79:%.*]] = insertelement <8 x float> [[TMP78]], float [[TMP39]], i32 4
+; MAX256-NEXT:    [[TMP80:%.*]] = insertelement <8 x float> [[TMP79]], float [[FVAL]], i32 5
+; MAX256-NEXT:    [[TMP81:%.*]] = insertelement <8 x float> [[TMP80]], float [[FVAL]], i32 6
+; MAX256-NEXT:    [[TMP82:%.*]] = extractelement <8 x float> [[TMP14]], i32 7
+; MAX256-NEXT:    [[TMP83:%.*]] = insertelement <8 x float> [[TMP81]], float [[TMP82]], i32 7
+; MAX256-NEXT:    [[TMP84:%.*]] = extractelement <8 x float> [[TMP28]], i32 0
+; MAX256-NEXT:    [[TMP85:%.*]] = insertelement <8 x float> undef, float [[TMP84]], i32 0
+; MAX256-NEXT:    [[TMP86:%.*]] = insertelement <8 x float> [[TMP85]], float [[FVAL]], i32 1
+; MAX256-NEXT:    [[TMP87:%.*]] = insertelement <8 x float> [[TMP86]], float [[FVAL]], i32 2
+; MAX256-NEXT:    [[TMP88:%.*]] = insertelement <8 x float> [[TMP87]], float [[TMP47]], i32 3
+; MAX256-NEXT:    [[TMP89:%.*]] = extractelement <8 x float> [[TMP28]], i32 4
+; MAX256-NEXT:    [[TMP90:%.*]] = insertelement <8 x float> [[TMP88]], float [[TMP89]], i32 4
+; MAX256-NEXT:    [[TMP91:%.*]] = insertelement <8 x float> [[TMP90]], float [[FVAL]], i32 5
+; MAX256-NEXT:    [[TMP92:%.*]] = insertelement <8 x float> [[TMP91]], float [[FVAL]], i32 6
+; MAX256-NEXT:    [[TMP93:%.*]] = insertelement <8 x float> [[TMP92]], float [[TMP53]], i32 7
+; MAX256-NEXT:    [[TMP94:%.*]] = extractelement <8 x float> [[TMP30]], i32 0
+; MAX256-NEXT:    [[TMP95:%.*]] = insertelement <8 x float> undef, float [[TMP94]], i32 0
+; MAX256-NEXT:    [[TMP96:%.*]] = insertelement <8 x float> [[TMP95]], float [[FVAL]], i32 1
+; MAX256-NEXT:    [[TMP97:%.*]] = insertelement <8 x float> [[TMP96]], float [[FVAL]], i32 2
+; MAX256-NEXT:    [[TMP98:%.*]] = insertelement <8 x float> [[TMP97]], float [[TMP57]], i32 3
+; MAX256-NEXT:    [[TMP99:%.*]] = extractelement <8 x float> [[TMP30]], i32 4
+; MAX256-NEXT:    [[TMP100:%.*]] = insertelement <8 x float> [[TMP98]], float [[TMP99]], i32 4
+; MAX256-NEXT:    [[TMP101:%.*]] = insertelement <8 x float> [[TMP100]], float [[FVAL]], i32 5
+; MAX256-NEXT:    [[TMP102:%.*]] = insertelement <8 x float> [[TMP101]], float [[FVAL]], i32 6
+; MAX256-NEXT:    [[TMP103:%.*]] = insertelement <8 x float> [[TMP102]], float [[TMP63]], i32 7
+; MAX256-NEXT:    [[TMP104:%.*]] = extractelement <8 x float> [[TMP32]], i32 0
+; MAX256-NEXT:    [[TMP105:%.*]] = insertelement <8 x float> undef, float [[TMP104]], i32 0
+; MAX256-NEXT:    [[TMP106:%.*]] = insertelement <8 x float> [[TMP105]], float [[FVAL]], i32 1
+; MAX256-NEXT:    [[TMP107:%.*]] = insertelement <8 x float> [[TMP106]], float [[FVAL]], i32 2
+; MAX256-NEXT:    [[TMP108:%.*]] = insertelement <8 x float> [[TMP107]], float [[TMP67]], i32 3
+; MAX256-NEXT:    [[TMP109:%.*]] = extractelement <8 x float> [[TMP32]], i32 4
+; MAX256-NEXT:    [[TMP110:%.*]] = insertelement <8 x float> [[TMP108]], float [[TMP109]], i32 4
+; MAX256-NEXT:    [[TMP111:%.*]] = insertelement <8 x float> [[TMP110]], float [[FVAL]], i32 5
+; MAX256-NEXT:    [[TMP112:%.*]] = insertelement <8 x float> [[TMP111]], float [[FVAL]], i32 6
+; MAX256-NEXT:    [[TMP113:%.*]] = insertelement <8 x float> [[TMP112]], float [[TMP73]], i32 7
+; MAX256-NEXT:    br label [[BB2]]
+; MAX256:       bb5:
+; MAX256-NEXT:    [[TMP114:%.*]] = insertelement <8 x float> [[TMP5]], float [[TMP35]], i32 1
+; MAX256-NEXT:    [[TMP115:%.*]] = insertelement <8 x float> [[TMP114]], float [[FVAL]], i32 2
+; MAX256-NEXT:    [[TMP116:%.*]] = extractelement <8 x float> [[TMP14]], i32 3
+; MAX256-NEXT:    [[TMP117:%.*]] = insertelement <8 x float> [[TMP115]], float [[TMP116]], i32 3
+; MAX256-NEXT:    [[TMP118:%.*]] = insertelement <8 x float> [[TMP117]], float [[FVAL]], i32 4
+; MAX256-NEXT:    [[TMP119:%.*]] = insertelement <8 x float> [[TMP118]], float [[TMP41]], i32 5
+; MAX256-NEXT:    [[TMP120:%.*]] = insertelement <8 x float> [[TMP119]], float [[FVAL]], i32 6
+; MAX256-NEXT:    [[TMP121:%.*]] = extractelement <8 x float> [[TMP14]], i32 7
+; MAX256-NEXT:    [[TMP122:%.*]] = insertelement <8 x float> [[TMP120]], float [[TMP121]], i32 7
+; MAX256-NEXT:    [[TMP123:%.*]] = extractelement <8 x float> [[TMP28]], i32 0
+; MAX256-NEXT:    [[TMP124:%.*]] = insertelement <8 x float> undef, float [[TMP123]], i32 0
+; MAX256-NEXT:    [[TMP125:%.*]] = insertelement <8 x float> [[TMP124]], float [[FVAL]], i32 1
+; MAX256-NEXT:    [[TMP126:%.*]] = insertelement <8 x float> [[TMP125]], float [[TMP45]], i32 2
+; MAX256-NEXT:    [[TMP127:%.*]] = insertelement <8 x float> [[TMP126]], float [[FVAL]], i32 3
+; MAX256-NEXT:    [[TMP128:%.*]] = extractelement <8 x float> [[TMP28]], i32 4
+; MAX256-NEXT:    [[TMP129:%.*]] = insertelement <8 x float> [[TMP127]], float [[TMP128]], i32 4
+; MAX256-NEXT:    [[TMP130:%.*]] = insertelement <8 x float> [[TMP129]], float [[FVAL]], i32 5
+; MAX256-NEXT:    [[TMP131:%.*]] = insertelement <8 x float> [[TMP130]], float [[TMP51]], i32 6
+; MAX256-NEXT:    [[TMP132:%.*]] = insertelement <8 x float> [[TMP131]], float [[FVAL]], i32 7
+; MAX256-NEXT:    [[TMP133:%.*]] = extractelement <8 x float> [[TMP30]], i32 0
+; MAX256-NEXT:    [[TMP134:%.*]] = insertelement <8 x float> undef, float [[TMP133]], i32 0
+; MAX256-NEXT:    [[TMP135:%.*]] = insertelement <8 x float> [[TMP134]], float [[FVAL]], i32 1
+; MAX256-NEXT:    [[TMP136:%.*]] = insertelement <8 x float> [[TMP135]], float [[TMP55]], i32 2
+; MAX256-NEXT:    [[TMP137:%.*]] = insertelement <8 x float> [[TMP136]], float [[FVAL]], i32 3
+; MAX256-NEXT:    [[TMP138:%.*]] = extractelement <8 x float> [[TMP30]], i32 4
+; MAX256-NEXT:    [[TMP139:%.*]] = insertelement <8 x float> [[TMP137]], float [[TMP138]], i32 4
+; MAX256-NEXT:    [[TMP140:%.*]] = insertelement <8 x float> [[TMP139]], float [[FVAL]], i32 5
+; MAX256-NEXT:    [[TMP141:%.*]] = insertelement <8 x float> [[TMP140]], float [[TMP61]], i32 6
+; MAX256-NEXT:    [[TMP142:%.*]] = insertelement <8 x float> [[TMP141]], float [[FVAL]], i32 7
+; MAX256-NEXT:    [[TMP143:%.*]] = extractelement <8 x float> [[TMP32]], i32 0
+; MAX256-NEXT:    [[TMP144:%.*]] = insertelement <8 x float> undef, float [[TMP143]], i32 0
+; MAX256-NEXT:    [[TMP145:%.*]] = insertelement <8 x float> [[TMP144]], float [[FVAL]], i32 1
+; MAX256-NEXT:    [[TMP146:%.*]] = insertelement <8 x float> [[TMP145]], float [[TMP65]], i32 2
+; MAX256-NEXT:    [[TMP147:%.*]] = insertelement <8 x float> [[TMP146]], float [[FVAL]], i32 3
+; MAX256-NEXT:    [[TMP148:%.*]] = extractelement <8 x float> [[TMP32]], i32 4
+; MAX256-NEXT:    [[TMP149:%.*]] = insertelement <8 x float> [[TMP147]], float [[TMP148]], i32 4
+; MAX256-NEXT:    [[TMP150:%.*]] = insertelement <8 x float> [[TMP149]], float [[FVAL]], i32 5
+; MAX256-NEXT:    [[TMP151:%.*]] = insertelement <8 x float> [[TMP150]], float [[TMP71]], i32 6
+; MAX256-NEXT:    [[TMP152:%.*]] = insertelement <8 x float> [[TMP151]], float [[FVAL]], i32 7
+; MAX256-NEXT:    br label [[BB2]]
+; MAX256:       bb2:
+; MAX256-NEXT:    [[TMP153:%.*]] = phi <8 x float> [ [[TMP14]], [[BB3]] ], [ [[TMP83]], [[BB4]] ], [ [[TMP122]], [[BB5]] ], [ [[TMP44]], [[BB1]] ]
+; MAX256-NEXT:    [[TMP154:%.*]] = phi <8 x float> [ [[TMP28]], [[BB3]] ], [ [[TMP93]], [[BB4]] ], [ [[TMP132]], [[BB5]] ], [ [[TMP54]], [[BB1]] ]
+; MAX256-NEXT:    [[TMP155:%.*]] = phi <8 x float> [ [[TMP30]], [[BB3]] ], [ [[TMP103]], [[BB4]] ], [ [[TMP142]], [[BB5]] ], [ [[TMP64]], [[BB1]] ]
+; MAX256-NEXT:    [[TMP156:%.*]] = phi <8 x float> [ [[TMP32]], [[BB3]] ], [ [[TMP113]], [[BB4]] ], [ [[TMP152]], [[BB5]] ], [ [[TMP74]], [[BB1]] ]
+; MAX256-NEXT:    ret void
+;
+; MAX1024-LABEL: @phi_float32(
+; MAX1024-NEXT:  bb:
+; MAX1024-NEXT:    br label [[BB1:%.*]]
+; MAX1024:       bb1:
+; MAX1024-NEXT:    [[TMP0:%.*]] = insertelement <4 x half> undef, half [[HVAL:%.*]], i32 0
+; MAX1024-NEXT:    [[TMP1:%.*]] = insertelement <4 x half> [[TMP0]], half [[HVAL]], i32 1
+; MAX1024-NEXT:    [[TMP2:%.*]] = insertelement <4 x half> [[TMP1]], half [[HVAL]], i32 2
+; MAX1024-NEXT:    [[TMP3:%.*]] = insertelement <4 x half> [[TMP2]], half [[HVAL]], i32 3
+; MAX1024-NEXT:    [[TMP4:%.*]] = fpext <4 x half> [[TMP3]] to <4 x float>
+; MAX1024-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 3, i32 2, i32 1, i32 0, i32 3, i32 2, i32 1, i32 0, i32 3, i32 2, i32 1, i32 0, i32 3, i32 2, i32 1, i32 0, i32 3, i32 2, i32 1, i32 0, i32 3, i32 2, i32 1, i32 0>
+; MAX1024-NEXT:    [[TMP5:%.*]] = insertelement <32 x float> undef, float [[FVAL:%.*]], i32 0
+; MAX1024-NEXT:    [[TMP6:%.*]] = insertelement <32 x float> [[TMP5]], float [[FVAL]], i32 1
+; MAX1024-NEXT:    [[TMP7:%.*]] = insertelement <32 x float> [[TMP6]], float [[FVAL]], i32 2
+; MAX1024-NEXT:    [[TMP8:%.*]] = insertelement <32 x float> [[TMP7]], float [[FVAL]], i32 3
+; MAX1024-NEXT:    [[TMP9:%.*]] = insertelement <32 x float> [[TMP8]], float [[FVAL]], i32 4
+; MAX1024-NEXT:    [[TMP10:%.*]] = insertelement <32 x float> [[TMP9]], float [[FVAL]], i32 5
+; MAX1024-NEXT:    [[TMP11:%.*]] = insertelement <32 x float> [[TMP10]], float [[FVAL]], i32 6
+; MAX1024-NEXT:    [[TMP12:%.*]] = insertelement <32 x float> [[TMP11]], float [[FVAL]], i32 7
+; MAX1024-NEXT:    [[TMP13:%.*]] = insertelement <32 x float> [[TMP12]], float [[FVAL]], i32 8
+; MAX1024-NEXT:    [[TMP14:%.*]] = insertelement <32 x float> [[TMP13]], float [[FVAL]], i32 9
+; MAX1024-NEXT:    [[TMP15:%.*]] = insertelement <32 x float> [[TMP14]], float [[FVAL]], i32 10
+; MAX1024-NEXT:    [[TMP16:%.*]] = insertelement <32 x float> [[TMP15]], float [[FVAL]], i32 11
+; MAX1024-NEXT:    [[TMP17:%.*]] = insertelement <32 x float> [[TMP16]], float [[FVAL]], i32 12
+; MAX1024-NEXT:    [[TMP18:%.*]] = insertelement <32 x float> [[TMP17]], float [[FVAL]], i32 13
+; MAX1024-NEXT:    [[TMP19:%.*]] = insertelement <32 x float> [[TMP18]], float [[FVAL]], i32 14
+; MAX1024-NEXT:    [[TMP20:%.*]] = insertelement <32 x float> [[TMP19]], float [[FVAL]], i32 15
+; MAX1024-NEXT:    [[TMP21:%.*]] = insertelement <32 x float> [[TMP20]], float [[FVAL]], i32 16
+; MAX1024-NEXT:    [[TMP22:%.*]] = insertelement <32 x float> [[TMP21]], float [[FVAL]], i32 17
+; MAX1024-NEXT:    [[TMP23:%.*]] = insertelement <32 x float> [[TMP22]], float [[FVAL]], i32 18
+; MAX1024-NEXT:    [[TMP24:%.*]] = insertelement <32 x float> [[TMP23]], float [[FVAL]], i32 19
+; MAX1024-NEXT:    [[TMP25:%.*]] = insertelement <32 x float> [[TMP24]], float [[FVAL]], i32 20
+; MAX1024-NEXT:    [[TMP26:%.*]] = insertelement <32 x float> [[TMP25]], float [[FVAL]], i32 21
+; MAX1024-NEXT:    [[TMP27:%.*]] = insertelement <32 x float> [[TMP26]], float [[FVAL]], i32 22
+; MAX1024-NEXT:    [[TMP28:%.*]] = insertelement <32 x float> [[TMP27]], float [[FVAL]], i32 23
+; MAX1024-NEXT:    [[TMP29:%.*]] = insertelement <32 x float> [[TMP28]], float [[FVAL]], i32 24
+; MAX1024-NEXT:    [[TMP30:%.*]] = insertelement <32 x float> [[TMP29]], float [[FVAL]], i32 25
+; MAX1024-NEXT:    [[TMP31:%.*]] = insertelement <32 x float> [[TMP30]], float [[FVAL]], i32 26
+; MAX1024-NEXT:    [[TMP32:%.*]] = insertelement <32 x float> [[TMP31]], float [[FVAL]], i32 27
+; MAX1024-NEXT:    [[TMP33:%.*]] = insertelement <32 x float> [[TMP32]], float [[FVAL]], i32 28
+; MAX1024-NEXT:    [[TMP34:%.*]] = insertelement <32 x float> [[TMP33]], float [[FVAL]], i32 29
+; MAX1024-NEXT:    [[TMP35:%.*]] = insertelement <32 x float> [[TMP34]], float [[FVAL]], i32 30
+; MAX1024-NEXT:    [[TMP36:%.*]] = insertelement <32 x float> [[TMP35]], float [[FVAL]], i32 31
+; MAX1024-NEXT:    [[TMP37:%.*]] = fmul <32 x float> [[SHUFFLE]], [[TMP36]]
+; MAX1024-NEXT:    [[TMP38:%.*]] = fadd <32 x float> zeroinitializer, [[TMP37]]
+; MAX1024-NEXT:    [[TMP39:%.*]] = extractelement <32 x float> [[TMP38]], i32 0
+; MAX1024-NEXT:    [[TMP40:%.*]] = insertelement <32 x float> undef, float [[TMP39]], i32 0
+; MAX1024-NEXT:    [[TMP41:%.*]] = extractelement <32 x float> [[TMP38]], i32 1
+; MAX1024-NEXT:    [[TMP42:%.*]] = insertelement <32 x float> [[TMP40]], float [[TMP41]], i32 1
+; MAX1024-NEXT:    [[TMP43:%.*]] = insertelement <32 x float> [[TMP42]], float [[FVAL]], i32 2
+; MAX1024-NEXT:    [[TMP44:%.*]] = insertelement <32 x float> [[TMP43]], float [[FVAL]], i32 3
+; MAX1024-NEXT:    [[TMP45:%.*]] = extractelement <32 x float> [[TMP38]], i32 4
+; MAX1024-NEXT:    [[TMP46:%.*]] = insertelement <32 x float> [[TMP44]], float [[TMP45]], i32 4
+; MAX1024-NEXT:    [[TMP47:%.*]] = extractelement <32 x float> [[TMP38]], i32 5
+; MAX1024-NEXT:    [[TMP48:%.*]] = insertelement <32 x float> [[TMP46]], float [[TMP47]], i32 5
+; MAX1024-NEXT:    [[TMP49:%.*]] = insertelement <32 x float> [[TMP48]], float [[FVAL]], i32 6
+; MAX1024-NEXT:    [[TMP50:%.*]] = insertelement <32 x float> [[TMP49]], float [[FVAL]], i32 7
+; MAX1024-NEXT:    [[TMP51:%.*]] = insertelement <32 x float> [[TMP50]], float [[FVAL]], i32 8
+; MAX1024-NEXT:    [[TMP52:%.*]] = insertelement <32 x float> [[TMP51]], float [[FVAL]], i32 9
+; MAX1024-NEXT:    [[TMP53:%.*]] = extractelement <32 x float> [[TMP38]], i32 10
+; MAX1024-NEXT:    [[TMP54:%.*]] = insertelement <32 x float> [[TMP52]], float [[TMP53]], i32 10
+; MAX1024-NEXT:    [[TMP55:%.*]] = extractelement <32 x float> [[TMP38]], i32 11
+; MAX1024-NEXT:    [[TMP56:%.*]] = insertelement <32 x float> [[TMP54]], float [[TMP55]], i32 11
+; MAX1024-NEXT:    [[TMP57:%.*]] = insertelement <32 x float> [[TMP56]], float [[FVAL]], i32 12
+; MAX1024-NEXT:    [[TMP58:%.*]] = insertelement <32 x float> [[TMP57]], float [[FVAL]], i32 13
+; MAX1024-NEXT:    [[TMP59:%.*]] = extractelement <32 x float> [[TMP38]], i32 14
+; MAX1024-NEXT:    [[TMP60:%.*]] = insertelement <32 x float> [[TMP58]], float [[TMP59]], i32 14
+; MAX1024-NEXT:    [[TMP61:%.*]] = extractelement <32 x float> [[TMP38]], i32 15
+; MAX1024-NEXT:    [[TMP62:%.*]] = insertelement <32 x float> [[TMP60]], float [[TMP61]], i32 15
+; MAX1024-NEXT:    [[TMP63:%.*]] = insertelement <32 x float> [[TMP62]], float [[FVAL]], i32 16
+; MAX1024-NEXT:    [[TMP64:%.*]] = insertelement <32 x float> [[TMP63]], float [[FVAL]], i32 17
+; MAX1024-NEXT:    [[TMP65:%.*]] = extractelement <32 x float> [[TMP38]], i32 18
+; MAX1024-NEXT:    [[TMP66:%.*]] = insertelement <32 x float> [[TMP64]], float [[TMP65]], i32 18
+; MAX1024-NEXT:    [[TMP67:%.*]] = extractelement <32 x float> [[TMP38]], i32 19
+; MAX1024-NEXT:    [[TMP68:%.*]] = insertelement <32 x float> [[TMP66]], float [[TMP67]], i32 19
+; MAX1024-NEXT:    [[TMP69:%.*]] = insertelement <32 x float> [[TMP68]], float [[FVAL]], i32 20
+; MAX1024-NEXT:    [[TMP70:%.*]] = insertelement <32 x float> [[TMP69]], float [[FVAL]], i32 21
+; MAX1024-NEXT:    [[TMP71:%.*]] = extractelement <32 x float> [[TMP38]], i32 22
+; MAX1024-NEXT:    [[TMP72:%.*]] = insertelement <32 x float> [[TMP70]], float [[TMP71]], i32 22
+; MAX1024-NEXT:    [[TMP73:%.*]] = extractelement <32 x float> [[TMP38]], i32 23
+; MAX1024-NEXT:    [[TMP74:%.*]] = insertelement <32 x float> [[TMP72]], float [[TMP73]], i32 23
+; MAX1024-NEXT:    [[TMP75:%.*]] = insertelement <32 x float> [[TMP74]], float [[FVAL]], i32 24
+; MAX1024-NEXT:    [[TMP76:%.*]] = insertelement <32 x float> [[TMP75]], float [[FVAL]], i32 25
+; MAX1024-NEXT:    [[TMP77:%.*]] = extractelement <32 x float> [[TMP38]], i32 26
+; MAX1024-NEXT:    [[TMP78:%.*]] = insertelement <32 x float> [[TMP76]], float [[TMP77]], i32 26
+; MAX1024-NEXT:    [[TMP79:%.*]] = extractelement <32 x float> [[TMP38]], i32 27
+; MAX1024-NEXT:    [[TMP80:%.*]] = insertelement <32 x float> [[TMP78]], float [[TMP79]], i32 27
+; MAX1024-NEXT:    [[TMP81:%.*]] = insertelement <32 x float> [[TMP80]], float [[FVAL]], i32 28
+; MAX1024-NEXT:    [[TMP82:%.*]] = insertelement <32 x float> [[TMP81]], float [[FVAL]], i32 29
+; MAX1024-NEXT:    [[TMP83:%.*]] = extractelement <32 x float> [[TMP38]], i32 30
+; MAX1024-NEXT:    [[TMP84:%.*]] = insertelement <32 x float> [[TMP82]], float [[TMP83]], i32 30
+; MAX1024-NEXT:    [[TMP85:%.*]] = extractelement <32 x float> [[TMP38]], i32 31
+; MAX1024-NEXT:    [[TMP86:%.*]] = insertelement <32 x float> [[TMP84]], float [[TMP85]], i32 31
+; MAX1024-NEXT:    switch i32 undef, label [[BB5:%.*]] [
+; MAX1024-NEXT:    i32 0, label [[BB2:%.*]]
+; MAX1024-NEXT:    i32 1, label [[BB3:%.*]]
+; MAX1024-NEXT:    i32 2, label [[BB4:%.*]]
+; MAX1024-NEXT:    ]
+; MAX1024:       bb3:
+; MAX1024-NEXT:    br label [[BB2]]
+; MAX1024:       bb4:
+; MAX1024-NEXT:    [[TMP87:%.*]] = insertelement <32 x float> [[TMP40]], float [[FVAL]], i32 1
+; MAX1024-NEXT:    [[TMP88:%.*]] = insertelement <32 x float> [[TMP87]], float [[FVAL]], i32 2
+; MAX1024-NEXT:    [[TMP89:%.*]] = extractelement <32 x float> [[TMP38]], i32 3
+; MAX1024-NEXT:    [[TMP90:%.*]] = insertelement <32 x float> [[TMP88]], float [[TMP89]], i32 3
+; MAX1024-NEXT:    [[TMP91:%.*]] = insertelement <32 x float> [[TMP90]], float [[TMP45]], i32 4
+; MAX1024-NEXT:    [[TMP92:%.*]] = insertelement <32 x float> [[TMP91]], float [[FVAL]], i32 5
+; MAX1024-NEXT:    [[TMP93:%.*]] = insertelement <32 x float> [[TMP92]], float [[FVAL]], i32 6
+; MAX1024-NEXT:    [[TMP94:%.*]] = extractelement <32 x float> [[TMP38]], i32 7
+; MAX1024-NEXT:    [[TMP95:%.*]] = insertelement <32 x float> [[TMP93]], float [[TMP94]], i32 7
+; MAX1024-NEXT:    [[TMP96:%.*]] = extractelement <32 x float> [[TMP38]], i32 8
+; MAX1024-NEXT:    [[TMP97:%.*]] = insertelement <32 x float> [[TMP95]], float [[TMP96]], i32 8
+; MAX1024-NEXT:    [[TMP98:%.*]] = insertelement <32 x float> [[TMP97]], float [[FVAL]], i32 9
+; MAX1024-NEXT:    [[TMP99:%.*]] = insertelement <32 x float> [[TMP98]], float [[FVAL]], i32 10
+; MAX1024-NEXT:    [[TMP100:%.*]] = insertelement <32 x float> [[TMP99]], float [[TMP55]], i32 11
+; MAX1024-NEXT:    [[TMP101:%.*]] = extractelement <32 x float> [[TMP38]], i32 12
+; MAX1024-NEXT:    [[TMP102:%.*]] = insertelement <32 x float> [[TMP100]], float [[TMP101]], i32 12
+; MAX1024-NEXT:    [[TMP103:%.*]] = insertelement <32 x float> [[TMP102]], float [[FVAL]], i32 13
+; MAX1024-NEXT:    [[TMP104:%.*]] = insertelement <32 x float> [[TMP103]], float [[FVAL]], i32 14
+; MAX1024-NEXT:    [[TMP105:%.*]] = insertelement <32 x float> [[TMP104]], float [[TMP61]], i32 15
+; MAX1024-NEXT:    [[TMP106:%.*]] = extractelement <32 x float> [[TMP38]], i32 16
+; MAX1024-NEXT:    [[TMP107:%.*]] = insertelement <32 x float> [[TMP105]], float [[TMP106]], i32 16
+; MAX1024-NEXT:    [[TMP108:%.*]] = insertelement <32 x float> [[TMP107]], float [[FVAL]], i32 17
+; MAX1024-NEXT:    [[TMP109:%.*]] = insertelement <32 x float> [[TMP108]], float [[FVAL]], i32 18
+; MAX1024-NEXT:    [[TMP110:%.*]] = insertelement <32 x float> [[TMP109]], float [[TMP67]], i32 19
+; MAX1024-NEXT:    [[TMP111:%.*]] = extractelement <32 x float> [[TMP38]], i32 20
+; MAX1024-NEXT:    [[TMP112:%.*]] = insertelement <32 x float> [[TMP110]], float [[TMP111]], i32 20
+; MAX1024-NEXT:    [[TMP113:%.*]] = insertelement <32 x float> [[TMP112]], float [[FVAL]], i32 21
+; MAX1024-NEXT:    [[TMP114:%.*]] = insertelement <32 x float> [[TMP113]], float [[FVAL]], i32 22
+; MAX1024-NEXT:    [[TMP115:%.*]] = insertelement <32 x float> [[TMP114]], float [[TMP73]], i32 23
+; MAX1024-NEXT:    [[TMP116:%.*]] = extractelement <32 x float> [[TMP38]], i32 24
+; MAX1024-NEXT:    [[TMP117:%.*]] = insertelement <32 x float> [[TMP115]], float [[TMP116]], i32 24
+; MAX1024-NEXT:    [[TMP118:%.*]] = insertelement <32 x float> [[TMP117]], float [[FVAL]], i32 25
+; MAX1024-NEXT:    [[TMP119:%.*]] = insertelement <32 x float> [[TMP118]], float [[FVAL]], i32 26
+; MAX1024-NEXT:    [[TMP120:%.*]] = insertelement <32 x float> [[TMP119]], float [[TMP79]], i32 27
+; MAX1024-NEXT:    [[TMP121:%.*]] = extractelement <32 x float> [[TMP38]], i32 28
+; MAX1024-NEXT:    [[TMP122:%.*]] = insertelement <32 x float> [[TMP120]], float [[TMP121]], i32 28
+; MAX1024-NEXT:    [[TMP123:%.*]] = insertelement <32 x float> [[TMP122]], float [[FVAL]], i32 29
+; MAX1024-NEXT:    [[TMP124:%.*]] = insertelement <32 x float> [[TMP123]], float [[FVAL]], i32 30
+; MAX1024-NEXT:    [[TMP125:%.*]] = insertelement <32 x float> [[TMP124]], float [[TMP85]], i32 31
+; MAX1024-NEXT:    br label [[BB2]]
+; MAX1024:       bb5:
+; MAX1024-NEXT:    [[TMP126:%.*]] = insertelement <32 x float> [[TMP5]], float [[TMP41]], i32 1
+; MAX1024-NEXT:    [[TMP127:%.*]] = insertelement <32 x float> [[TMP126]], float [[FVAL]], i32 2
+; MAX1024-NEXT:    [[TMP128:%.*]] = extractelement <32 x float> [[TMP38]], i32 3
+; MAX1024-NEXT:    [[TMP129:%.*]] = insertelement <32 x float> [[TMP127]], float [[TMP128]], i32 3
+; MAX1024-NEXT:    [[TMP130:%.*]] = insertelement <32 x float> [[TMP129]], float [[FVAL]], i32 4
+; MAX1024-NEXT:    [[TMP131:%.*]] = insertelement <32 x float> [[TMP130]], float [[TMP47]], i32 5
+; MAX1024-NEXT:    [[TMP132:%.*]] = insertelement <32 x float> [[TMP131]], float [[FVAL]], i32 6
+; MAX1024-NEXT:    [[TMP133:%.*]] = extractelement <32 x float> [[TMP38]], i32 7
+; MAX1024-NEXT:    [[TMP134:%.*]] = insertelement <32 x float> [[TMP132]], float [[TMP133]], i32 7
+; MAX1024-NEXT:    [[TMP135:%.*]] = extractelement <32 x float> [[TMP38]], i32 8
+; MAX1024-NEXT:    [[TMP136:%.*]] = insertelement <32 x float> [[TMP134]], float [[TMP135]], i32 8
+; MAX1024-NEXT:    [[TMP137:%.*]] = insertelement <32 x float> [[TMP136]], float [[FVAL]], i32 9
+; MAX1024-NEXT:    [[TMP138:%.*]] = insertelement <32 x float> [[TMP137]], float [[TMP53]], i32 10
+; MAX1024-NEXT:    [[TMP139:%.*]] = insertelement <32 x float> [[TMP138]], float [[FVAL]], i32 11
+; MAX1024-NEXT:    [[TMP140:%.*]] = extractelement <32 x float> [[TMP38]], i32 12
+; MAX1024-NEXT:    [[TMP141:%.*]] = insertelement <32 x float> [[TMP139]], float [[TMP140]], i32 12
+; MAX1024-NEXT:    [[TMP142:%.*]] = insertelement <32 x float> [[TMP141]], float [[FVAL]], i32 13
+; MAX1024-NEXT:    [[TMP143:%.*]] = insertelement <32 x float> [[TMP142]], float [[TMP59]], i32 14
+; MAX1024-NEXT:    [[TMP144:%.*]] = insertelement <32 x float> [[TMP143]], float [[FVAL]], i32 15
+; MAX1024-NEXT:    [[TMP145:%.*]] = extractelement <32 x float> [[TMP38]], i32 16
+; MAX1024-NEXT:    [[TMP146:%.*]] = insertelement <32 x float> [[TMP144]], float [[TMP145]], i32 16
+; MAX1024-NEXT:    [[TMP147:%.*]] = insertelement <32 x float> [[TMP146]], float [[FVAL]], i32 17
+; MAX1024-NEXT:    [[TMP148:%.*]] = insertelement <32 x float> [[TMP147]], float [[TMP65]], i32 18
+; MAX1024-NEXT:    [[TMP149:%.*]] = insertelement <32 x float> [[TMP148]], float [[FVAL]], i32 19
+; MAX1024-NEXT:    [[TMP150:%.*]] = extractelement <32 x float> [[TMP38]], i32 20
+; MAX1024-NEXT:    [[TMP151:%.*]] = insertelement <32 x float> [[TMP149]], float [[TMP150]], i32 20
+; MAX1024-NEXT:    [[TMP152:%.*]] = insertelement <32 x float> [[TMP151]], float [[FVAL]], i32 21
+; MAX1024-NEXT:    [[TMP153:%.*]] = insertelement <32 x float> [[TMP152]], float [[TMP71]], i32 22
+; MAX1024-NEXT:    [[TMP154:%.*]] = insertelement <32 x float> [[TMP153]], float [[FVAL]], i32 23
+; MAX1024-NEXT:    [[TMP155:%.*]] = extractelement <32 x float> [[TMP38]], i32 24
+; MAX1024-NEXT:    [[TMP156:%.*]] = insertelement <32 x float> [[TMP154]], float [[TMP155]], i32 24
+; MAX1024-NEXT:    [[TMP157:%.*]] = insertelement <32 x float> [[TMP156]], float [[FVAL]], i32 25
+; MAX1024-NEXT:    [[TMP158:%.*]] = insertelement <32 x float> [[TMP157]], float [[TMP77]], i32 26
+; MAX1024-NEXT:    [[TMP159:%.*]] = insertelement <32 x float> [[TMP158]], float [[FVAL]], i32 27
+; MAX1024-NEXT:    [[TMP160:%.*]] = extractelement <32 x float> [[TMP38]], i32 28
+; MAX1024-NEXT:    [[TMP161:%.*]] = insertelement <32 x float> [[TMP159]], float [[TMP160]], i32 28
+; MAX1024-NEXT:    [[TMP162:%.*]] = insertelement <32 x float> [[TMP161]], float [[FVAL]], i32 29
+; MAX1024-NEXT:    [[TMP163:%.*]] = insertelement <32 x float> [[TMP162]], float [[TMP83]], i32 30
+; MAX1024-NEXT:    [[TMP164:%.*]] = insertelement <32 x float> [[TMP163]], float [[FVAL]], i32 31
+; MAX1024-NEXT:    br label [[BB2]]
+; MAX1024:       bb2:
+; MAX1024-NEXT:    [[TMP165:%.*]] = phi <32 x float> [ [[TMP38]], [[BB3]] ], [ [[TMP125]], [[BB4]] ], [ [[TMP164]], [[BB5]] ], [ [[TMP86]], [[BB1]] ]
+; MAX1024-NEXT:    ret void
+;
+bb:
+  br label %bb1
+
+bb1:
+  %i = fpext half %hval to float
+  %i1 = fmul float %i, %fval
+  %i2 = fadd float 0.000000e+00, %i1
+  %i3 = fpext half %hval to float
+  %i4 = fmul float %i3, %fval
+  %i5 = fadd float 0.000000e+00, %i4
+  %i6 = fpext half %hval to float
+  %i7 = fmul float %i6, %fval
+  %i8 = fadd float 0.000000e+00, %i7
+  %i9 = fpext half %hval to float
+  %i10 = fmul float %i9, %fval
+  %i11 = fadd float 0.000000e+00, %i10
+  %i12 = fmul float %i, %fval
+  %i13 = fadd float 0.000000e+00, %i12
+  %i14 = fmul float %i3, %fval
+  %i15 = fadd float 0.000000e+00, %i14
+  %i16 = fmul float %i6, %fval
+  %i17 = fadd float 0.000000e+00, %i16
+  %i18 = fmul float %i9, %fval
+  %i19 = fadd float 0.000000e+00, %i18
+  %i20 = fmul float %i, %fval
+  %i21 = fadd float 0.000000e+00, %i20
+  %i22 = fmul float %i3, %fval
+  %i23 = fadd float 0.000000e+00, %i22
+  %i24 = fmul float %i6, %fval
+  %i25 = fadd float 0.000000e+00, %i24
+  %i26 = fmul float %i9, %fval
+  %i27 = fadd float 0.000000e+00, %i26
+  %i28 = fmul float %i, %fval
+  %i29 = fadd float 0.000000e+00, %i28
+  %i30 = fmul float %i3, %fval
+  %i31 = fadd float 0.000000e+00, %i30
+  %i32 = fmul float %i6, %fval
+  %i33 = fadd float 0.000000e+00, %i32
+  %i34 = fmul float %i9, %fval
+  %i35 = fadd float 0.000000e+00, %i34
+  %i36 = fmul float %i, %fval
+  %i37 = fadd float 0.000000e+00, %i36
+  %i38 = fmul float %i3, %fval
+  %i39 = fadd float 0.000000e+00, %i38
+  %i40 = fmul float %i6, %fval
+  %i41 = fadd float 0.000000e+00, %i40
+  %i42 = fmul float %i9, %fval
+  %i43 = fadd float 0.000000e+00, %i42
+  %i44 = fmul float %i, %fval
+  %i45 = fadd float 0.000000e+00, %i44
+  %i46 = fmul float %i3, %fval
+  %i47 = fadd float 0.000000e+00, %i46
+  %i48 = fmul float %i6, %fval
+  %i49 = fadd float 0.000000e+00, %i48
+  %i50 = fmul float %i9, %fval
+  %i51 = fadd float 0.000000e+00, %i50
+  %i52 = fmul float %i, %fval
+  %i53 = fadd float 0.000000e+00, %i52
+  %i54 = fmul float %i3, %fval
+  %i55 = fadd float 0.000000e+00, %i54
+  %i56 = fmul float %i6, %fval
+  %i57 = fadd float 0.000000e+00, %i56
+  %i58 = fmul float %i9, %fval
+  %i59 = fadd float 0.000000e+00, %i58
+  %i60 = fmul float %i, %fval
+  %i61 = fadd float 0.000000e+00, %i60
+  %i62 = fmul float %i3, %fval
+  %i63 = fadd float 0.000000e+00, %i62
+  %i64 = fmul float %i6, %fval
+  %i65 = fadd float 0.000000e+00, %i64
+  %i66 = fmul float %i9, %fval
+  %i67 = fadd float 0.000000e+00, %i66
+  switch i32 undef, label %bb5 [
+  i32 0, label %bb2
+  i32 1, label %bb3
+  i32 2, label %bb4
+  ]
+
+bb3:
+  br label %bb2
+
+bb4:
+  br label %bb2
+
+bb5:
+  br label %bb2
+
+bb2:
+  %phi1 = phi float [ %i19, %bb3 ], [ %i19, %bb4 ], [ %fval, %bb5 ], [ %i19, %bb1 ]
+  %phi2 = phi float [ %i17, %bb3 ], [ %fval, %bb4 ], [ %i17, %bb5 ], [ %i17, %bb1 ]
+  %phi3 = phi float [ %i15, %bb3 ], [ %fval, %bb4 ], [ %fval, %bb5 ], [ %fval, %bb1 ]
+  %phi4 = phi float [ %i13, %bb3 ], [ %i13, %bb4 ], [ %i13, %bb5 ], [ %fval, %bb1 ]
+  %phi5 = phi float [ %i11, %bb3 ], [ %i11, %bb4 ], [ %fval, %bb5 ], [ %i11, %bb1 ]
+  %phi6 = phi float [ %i8, %bb3 ], [ %fval, %bb4 ], [ %i8, %bb5 ], [ %i8, %bb1 ]
+  %phi7 = phi float [ %i5, %bb3 ], [ %fval, %bb4 ], [ %fval, %bb5 ], [ %fval, %bb1 ]
+  %phi8 = phi float [ %i2, %bb3 ], [ %i2, %bb4 ], [ %i2, %bb5 ], [ %fval, %bb1 ]
+  %phi9 = phi float [ %i21, %bb3 ], [ %i21, %bb4 ], [ %i21, %bb5 ], [ %fval, %bb1 ]
+  %phi10 = phi float [ %i23, %bb3 ], [ %fval, %bb4 ], [ %fval, %bb5 ], [ %fval, %bb1 ]
+  %phi11 = phi float [ %i25, %bb3 ], [ %fval, %bb4 ], [ %i25, %bb5 ], [ %i25, %bb1 ]
+  %phi12 = phi float [ %i27, %bb3 ], [ %i27, %bb4 ], [ %fval, %bb5 ], [ %i27, %bb1 ]
+  %phi13 = phi float [ %i29, %bb3 ], [ %i29, %bb4 ], [ %i29, %bb5 ], [ %fval, %bb1 ]
+  %phi14 = phi float [ %i31, %bb3 ], [ %fval, %bb4 ], [ %fval, %bb5 ], [ %fval, %bb1 ]
+  %phi15 = phi float [ %i33, %bb3 ], [ %fval, %bb4 ], [ %i33, %bb5 ], [ %i33, %bb1 ]
+  %phi16 = phi float [ %i35, %bb3 ], [ %i35, %bb4 ], [ %fval, %bb5 ], [ %i35, %bb1 ]
+  %phi17 = phi float [ %i37, %bb3 ], [ %i37, %bb4 ], [ %i37, %bb5 ], [ %fval, %bb1 ]
+  %phi18 = phi float [ %i39, %bb3 ], [ %fval, %bb4 ], [ %fval, %bb5 ], [ %fval, %bb1 ]
+  %phi19 = phi float [ %i41, %bb3 ], [ %fval, %bb4 ], [ %i41, %bb5 ], [ %i41, %bb1 ]
+  %phi20 = phi float [ %i43, %bb3 ], [ %i43, %bb4 ], [ %fval, %bb5 ], [ %i43, %bb1 ]
+  %phi21 = phi float [ %i45, %bb3 ], [ %i45, %bb4 ], [ %i45, %bb5 ], [ %fval, %bb1 ]
+  %phi22 = phi float [ %i47, %bb3 ], [ %fval, %bb4 ], [ %fval, %bb5 ], [ %fval, %bb1 ]
+  %phi23 = phi float [ %i49, %bb3 ], [ %fval, %bb4 ], [ %i49, %bb5 ], [ %i49, %bb1 ]
+  %phi24 = phi float [ %i51, %bb3 ], [ %i51, %bb4 ], [ %fval, %bb5 ], [ %i51, %bb1 ]
+  %phi25 = phi float [ %i53, %bb3 ], [ %i53, %bb4 ], [ %i53, %bb5 ], [ %fval, %bb1 ]
+  %phi26 = phi float [ %i55, %bb3 ], [ %fval, %bb4 ], [ %fval, %bb5 ], [ %fval, %bb1 ]
+  %phi27 = phi float [ %i57, %bb3 ], [ %fval, %bb4 ], [ %i57, %bb5 ], [ %i57, %bb1 ]
+  %phi28 = phi float [ %i59, %bb3 ], [ %i59, %bb4 ], [ %fval, %bb5 ], [ %i59, %bb1 ]
+  %phi29 = phi float [ %i61, %bb3 ], [ %i61, %bb4 ], [ %i61, %bb5 ], [ %fval, %bb1 ]
+  %phi30 = phi float [ %i63, %bb3 ], [ %fval, %bb4 ], [ %fval, %bb5 ], [ %fval, %bb1 ]
+  %phi31 = phi float [ %i65, %bb3 ], [ %fval, %bb4 ], [ %i65, %bb5 ], [ %i65, %bb1 ]
+  %phi32 = phi float [ %i67, %bb3 ], [ %i67, %bb4 ], [ %fval, %bb5 ], [ %i67, %bb1 ]
+  ret void
+}
diff --git a/llvm/test/Transforms/SROA/address-spaces.ll b/llvm/test/Transforms/SROA/address-spaces.ll
index 9cd9137833a23..9404b2062ec8b 100644
--- a/llvm/test/Transforms/SROA/address-spaces.ll
+++ b/llvm/test/Transforms/SROA/address-spaces.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -sroa -S | FileCheck %s
-target datalayout = "e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+target datalayout = "e-p:64:64:64-p1:16:16:16-p3:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
 
 declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i1)
 declare void @llvm.memcpy.p1i8.p0i8.i32(i8 addrspace(1)* nocapture, i8* nocapture readonly, i32, i1)
@@ -71,7 +71,8 @@ for.end:
 @g = common global i32 0, align 4
 @l = common addrspace(3) global i32 0, align 4
 
-; Make sure an illegal bitcast isn't introduced
+; If pointers from different address spaces have different sizes, make sure an
+; illegal bitcast isn't introduced
 define void @pr27557() {
 ; CHECK-LABEL: @pr27557(
 ; CHECK: %[[CAST:.*]] = bitcast i32** {{.*}} to i32 addrspace(3)**
@@ -84,6 +85,21 @@ define void @pr27557() {
   ret void
 }
 
+@l2 = common addrspace(2) global i32 0, align 4
+
+; If pointers from different address spaces have the same size, that pointer
+; should be promoted through the pair of `ptrtoint`/`inttoptr`.
+define i32* @pr27557.alt() {
+; CHECK-LABEL: @pr27557.alt(
+; CHECK: ret i32* inttoptr (i64 ptrtoint (i32 addrspace(2)* @l2 to i64) to i32*)
+  %1 = alloca %union.anon, align 8
+  %2 = bitcast %union.anon* %1 to i32 addrspace(2)**
+  store i32 addrspace(2)* @l2, i32 addrspace(2)** %2, align 8
+  %3 = bitcast %union.anon* %1 to i32**
+  %4 = load i32*, i32** %3, align 8
+  ret i32* %4
+}
+
 ; Make sure pre-splitting doesn't try to introduce an illegal bitcast
 define float @presplit(i64 addrspace(1)* %p) {
 entry:
diff --git a/llvm/test/Transforms/SROA/alloca-address-space.ll b/llvm/test/Transforms/SROA/alloca-address-space.ll
index d28bc39e9633a..c8e22f5b25d8a 100644
--- a/llvm/test/Transforms/SROA/alloca-address-space.ll
+++ b/llvm/test/Transforms/SROA/alloca-address-space.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -sroa -S | FileCheck %s
-target datalayout = "e-p:64:64:64-p1:16:16:16-p2:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64-A2"
+target datalayout = "e-p:64:64:64-p1:16:16:16-p2:32:32-p3:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64-A2"
 
 declare void @llvm.memcpy.p2i8.p2i8.i32(i8 addrspace(2)* nocapture, i8 addrspace(2)* nocapture readonly, i32, i1)
 declare void @llvm.memcpy.p1i8.p2i8.i32(i8 addrspace(1)* nocapture, i8 addrspace(2)* nocapture readonly, i32, i1)
@@ -70,7 +70,8 @@ for.end:
 @g = common global i32 0, align 4
 @l = common addrspace(3) global i32 0, align 4
 
-; Make sure an illegal bitcast isn't introduced
+; If pointers from different address spaces have different sizes, make sure an
+; illegal bitcast isn't introduced
 ; CHECK-LABEL: @pr27557(
 ; CHECK: %[[CAST:.*]] = bitcast i32* addrspace(2)* {{.*}} to i32 addrspace(3)* addrspace(2)*
 ; CHECK: store i32 addrspace(3)* @l, i32 addrspace(3)* addrspace(2)* %[[CAST]]
@@ -83,6 +84,21 @@ define void @pr27557() {
   ret void
 }
 
+@l4 = common addrspace(4) global i32 0, align 4
+
+; If pointers from different address spaces have the same size, that pointer
+; should be promoted through the pair of `ptrtoint`/`inttoptr`.
+define i32* @pr27557.alt() {
+; CHECK-LABEL: @pr27557.alt(
+; CHECK: ret i32* inttoptr (i64 ptrtoint (i32 addrspace(4)* @l4 to i64) to i32*)
+  %1 = alloca %union.anon, align 8, addrspace(2)
+  %2 = bitcast %union.anon addrspace(2)* %1 to i32 addrspace(4)* addrspace(2)*
+  store i32 addrspace(4)* @l4, i32 addrspace(4)* addrspace(2)* %2, align 8
+  %3 = bitcast %union.anon addrspace(2)* %1 to i32* addrspace(2)*
+  %4 = load i32*, i32* addrspace(2)* %3, align 8
+  ret i32* %4
+}
+
 ; Test load from and store to non-zero address space.
 define void @test_load_store_diff_addr_space([2 x float] addrspace(1)* %complex1, [2 x float] addrspace(1)* %complex2) {
 ; CHECK-LABEL: @test_load_store_diff_addr_space
diff --git a/llvm/test/Transforms/SROA/non-integral-pointers.ll b/llvm/test/Transforms/SROA/non-integral-pointers.ll
index 166f5dc7b42e4..c12e5e8e71829 100644
--- a/llvm/test/Transforms/SROA/non-integral-pointers.ll
+++ b/llvm/test/Transforms/SROA/non-integral-pointers.ll
@@ -85,4 +85,18 @@ alwaysTaken:
   ret i64 addrspace(4)* null
 }
 
+%union.anon = type { i32* }
+
+; CHECK-LABEL: @f2(
+; CHECK-NOT: ptr2int
+; CHECK-NOT: int2ptr
+define i8 *@f2(i8 addrspace(4)* %p) {
+  %1 = alloca %union.anon, align 8
+  %2 = bitcast %union.anon* %1 to i8 addrspace(4)**
+  store i8 addrspace(4)* %p, i8 addrspace(4)** %2, align 8
+  %3 = bitcast %union.anon* %1 to i8**
+  %4 = load i8*, i8** %3, align 8
+  ret i8* %4
+}
+
 declare void @llvm.memset.p0i8.i64(i8*, i8, i64, i1)
diff --git a/llvm/test/Transforms/SROA/vector-conversion.ll b/llvm/test/Transforms/SROA/vector-conversion.ll
index 91ae5be6c3d25..80d645a2e5eea 100644
--- a/llvm/test/Transforms/SROA/vector-conversion.ll
+++ b/llvm/test/Transforms/SROA/vector-conversion.ll
@@ -34,7 +34,7 @@ define <4 x i32*> @vector_inttoptr({<2 x i64>, <2 x i64>} %x) {
 }
 
 define <2 x i64> @vector_ptrtointbitcast({<1 x i32*>, <1 x i32*>} %x) {
-; CHECK-LABEL: @vector_ptrtointbitcast
+; CHECK-LABEL: @vector_ptrtointbitcast(
   %a = alloca {<1 x i32*>, <1 x i32*>}
 ; CHECK-NOT: alloca
 
@@ -51,3 +51,41 @@ define <2 x i64> @vector_ptrtointbitcast({<1 x i32*>, <1 x i32*>} %x) {
 
   ret <2 x i64> %vec
 }
+
+define <2 x i8*> @vector_inttoptrbitcast_vector({<16 x i8>, <16 x i8>} %x) {
+; CHECK-LABEL: @vector_inttoptrbitcast_vector(
+  %a = alloca {<16 x i8>, <16 x i8>}
+; CHECK-NOT: alloca
+
+  store {<16 x i8>, <16 x i8>} %x, {<16 x i8>, <16 x i8>}* %a
+; CHECK-NOT: store
+
+  %cast = bitcast {<16 x i8>, <16 x i8>}* %a to <2 x i8*>*
+  %vec = load <2 x i8*>, <2 x i8*>* %cast
+; CHECK-NOT: load
+; CHECK: extractvalue
+; CHECK: extractvalue
+; CHECK: bitcast
+; CHECK: inttoptr
+
+  ret <2 x i8*> %vec
+}
+
+define <16 x i8> @vector_ptrtointbitcast_vector({<2 x i8*>, <2 x i8*>} %x) {
+; CHECK-LABEL: @vector_ptrtointbitcast_vector(
+  %a = alloca {<2 x i8*>, <2 x i8*>}
+; CHECK-NOT: alloca
+
+  store {<2 x i8*>, <2 x i8*>} %x, {<2 x i8*>, <2 x i8*>}* %a
+; CHECK-NOT: store
+
+  %cast = bitcast {<2 x i8*>, <2 x i8*>}* %a to <16 x i8>*
+  %vec = load <16 x i8>, <16 x i8>* %cast
+; CHECK-NOT: load
+; CHECK: extractvalue
+; CHECK: ptrtoint
+; CHECK: bitcast
+; CHECK: extractvalue
+
+  ret <16 x i8> %vec
+}
diff --git a/llvm/test/Transforms/SampleProfile/inline-mergeprof.ll b/llvm/test/Transforms/SampleProfile/inline-mergeprof.ll
index 08e9fa008fa8f..d83fd23c33d36 100644
--- a/llvm/test/Transforms/SampleProfile/inline-mergeprof.ll
+++ b/llvm/test/Transforms/SampleProfile/inline-mergeprof.ll
@@ -1,10 +1,10 @@
 ; Test we lose details of not inlined profile without '-sample-profile-merge-inlinee'
-; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline-mergeprof.prof -S | FileCheck -check-prefix=SCALE %s
-; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-mergeprof.prof -S | FileCheck -check-prefix=SCALE %s
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline-mergeprof.prof -sample-profile-merge-inlinee=false -S | FileCheck -check-prefix=SCALE %s
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline-mergeprof.prof -sample-profile-merge-inlinee=true -S | FileCheck -check-prefix=SCALE %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-mergeprof.prof -sample-profile-merge-inlinee=false -S | FileCheck -check-prefix=SCALE %s
 
 ; Test we properly merge not inlined profile properly with '-sample-profile-merge-inlinee'
-; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline-mergeprof.prof -sample-profile-merge-inlinee -S | FileCheck -check-prefix=MERGE %s
-; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-mergeprof.prof -sample-profile-merge-inlinee -S | FileCheck -check-prefix=MERGE  %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-mergeprof.prof -sample-profile-merge-inlinee=true -S | FileCheck -check-prefix=MERGE  %s
 
 @.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
 
diff --git a/llvm/test/Transforms/SampleProfile/inline-topdown.ll b/llvm/test/Transforms/SampleProfile/inline-topdown.ll
index 3781f6ad24978..8e8e076960309 100644
--- a/llvm/test/Transforms/SampleProfile/inline-topdown.ll
+++ b/llvm/test/Transforms/SampleProfile/inline-topdown.ll
@@ -1,10 +1,10 @@
 ; Note that this needs new pass manager for now. Passing `-sample-profile-top-down-load` to legacy pass manager is a no-op.
 
 ; Test we aren't doing specialization for inlining with default source order
-; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -S | FileCheck -check-prefix=DEFAULT %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-top-down-load=false -S | FileCheck -check-prefix=DEFAULT %s
 
 ; Test we specialize based on call path with context-sensitive profile while inlining with '-sample-profile-top-down-load'
-; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-merge-inlinee -sample-profile-top-down-load -S | FileCheck -check-prefix=TOPDOWN  %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-merge-inlinee -sample-profile-top-down-load=true -S | FileCheck -check-prefix=TOPDOWN  %s
 
 
 @.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-load.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-load.ll
new file mode 100644
index 0000000000000..580c403ab7191
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-load.ll
@@ -0,0 +1,129 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu | FileCheck %s
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu -mattr=+sve | FileCheck %s
+
+define <2 x i64> @scalarize_v2i64(<2 x i64>* %p, <2 x i1> %mask, <2 x i64> %passthru) {
+; CHECK-LABEL: @scalarize_v2i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.load:
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP5]], i64 0
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.load1:
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP10]], i64 1
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* %p, i32 128, <2 x i1> %mask, <2 x i64> %passthru)
+  ret <2 x i64> %ret
+}
+
+define <2 x i64> @scalarize_v2i64_ones_mask(<2 x i64>* %p, <2 x i64> %passthru) {
+; CHECK-LABEL: @scalarize_v2i64_ones_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* [[P:%.*]], align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %ret = call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* %p, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i64> %passthru)
+  ret <2 x i64> %ret
+}
+
+define <2 x i64> @scalarize_v2i64_zero_mask(<2 x i64>* %p, <2 x i64> %passthru) {
+; CHECK-LABEL: @scalarize_v2i64_zero_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-NEXT:    ret <2 x i64> [[PASSTHRU:%.*]]
+;
+  %ret = call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* %p, i32 8, <2 x i1> <i1 false, i1 false>, <2 x i64> %passthru)
+  ret <2 x i64> %ret
+}
+
+define <2 x i64> @scalarize_v2i64_const_mask(<2 x i64>* %p, <2 x i64> %passthru) {
+; CHECK-LABEL: @scalarize_v2i64_const_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP3]], i64 1
+; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
+;
+  %ret = call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* %p, i32 8, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru)
+  ret <2 x i64> %ret
+}
+
+; This use a byte sized but non power of 2 element size. This used to crash due to bad alignment calculation.
+define <2 x i24> @scalarize_v2i24(<2 x i24>* %p, <2 x i1> %mask, <2 x i24> %passthru) {
+; CHECK-LABEL: @scalarize_v2i24(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i24>* [[P:%.*]] to i24*
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.load:
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = load i24, i24* [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i24> [[PASSTHRU:%.*]], i24 [[TMP5]], i64 0
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i24> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.load1:
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = load i24, i24* [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i24> [[RES_PHI_ELSE]], i24 [[TMP10]], i64 1
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i24> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i24> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i24> @llvm.masked.load.v2i24.p0v2i24(<2 x i24>* %p, i32 8, <2 x i1> %mask, <2 x i24> %passthru)
+  ret <2 x i24> %ret
+}
+
+; This use a byte sized but non power of 2 element size. This used to crash due to bad alignment calculation.
+define <2 x i48> @scalarize_v2i48(<2 x i48>* %p, <2 x i1> %mask, <2 x i48> %passthru) {
+; CHECK-LABEL: @scalarize_v2i48(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i48>* [[P:%.*]] to i48*
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.load:
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = load i48, i48* [[TMP4]], align 2
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i48> [[PASSTHRU:%.*]], i48 [[TMP5]], i64 0
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i48> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.load1:
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = load i48, i48* [[TMP9]], align 2
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i48> [[RES_PHI_ELSE]], i48 [[TMP10]], i64 1
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i48> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i48> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i48> @llvm.masked.load.v2i48.p0v2i48(<2 x i48>* %p, i32 16, <2 x i1> %mask, <2 x i48> %passthru)
+  ret <2 x i48> %ret
+}
+
+declare <2 x i24> @llvm.masked.load.v2i24.p0v2i24(<2 x i24>*, i32, <2 x i1>, <2 x i24>)
+declare <2 x i48> @llvm.masked.load.v2i48.p0v2i48(<2 x i48>*, i32, <2 x i1>, <2 x i48>)
+declare <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>*, i32, <2 x i1>, <2 x i64>)
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-store.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-store.ll
new file mode 100644
index 0000000000000..e3245ca34d1ef
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-store.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu | FileCheck %s
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu -mattr=+sve | FileCheck %s
+
+define void @scalarize_v2i64(<2 x i64>* %p, <2 x i1> %mask, <2 x i64> %data) {
+; CHECK-LABEL: @scalarize_v2i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.store:
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
+; CHECK-NEXT:    store i64 [[TMP4]], i64* [[TMP5]], align 8
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TMP7]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.store1:
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-NEXT:    store i64 [[TMP8]], i64* [[TMP9]], align 8
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %data, <2 x i64>* %p, i32 128, <2 x i1> %mask)
+  ret void
+}
+
+define void @scalarize_v2i64_ones_mask(<2 x i64>* %p, <2 x i64> %data) {
+; CHECK-LABEL: @scalarize_v2i64_ones_mask(
+; CHECK-NEXT:    store <2 x i64> [[DATA:%.*]], <2 x i64>* [[P:%.*]], align 8
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %data, <2 x i64>* %p, i32 8, <2 x i1> <i1 true, i1 true>)
+  ret void
+}
+
+define void @scalarize_v2i64_zero_mask(<2 x i64>* %p, <2 x i64> %data) {
+; CHECK-LABEL: @scalarize_v2i64_zero_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %data, <2 x i64>* %p, i32 8, <2 x i1> <i1 false, i1 false>)
+  ret void
+}
+
+define void @scalarize_v2i64_const_mask(<2 x i64>* %p, <2 x i64> %data) {
+; CHECK-LABEL: @scalarize_v2i64_const_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-NEXT:    store i64 [[TMP2]], i64* [[TMP3]], align 8
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %data, <2 x i64>* %p, i32 8, <2 x i1> <i1 false, i1 true>)
+  ret void
+}
+
+declare void @llvm.masked.store.v2i64.p0v2i64(<2 x i64>, <2 x i64>*, i32, <2 x i1>)
diff --git a/llvm/test/Transforms/Scalarizer/basic.ll b/llvm/test/Transforms/Scalarizer/basic.ll
index ba6aa933aa846..239cdd660a33c 100644
--- a/llvm/test/Transforms/Scalarizer/basic.ll
+++ b/llvm/test/Transforms/Scalarizer/basic.ll
@@ -276,14 +276,14 @@ define void @f8(<4 x float *> *%dest, <4 x float *> %ptr0, <4 x i32> %i0,
 ; CHECK: %dest.i1 = getelementptr float*, float** %dest.i0, i32 1
 ; CHECK: %dest.i2 = getelementptr float*, float** %dest.i0, i32 2
 ; CHECK: %dest.i3 = getelementptr float*, float** %dest.i0, i32 3
+; CHECK: %ptr0.i0 = extractelement <4 x float*> %ptr0, i32 0
+; CHECK: %ptr0.i2 = extractelement <4 x float*> %ptr0, i32 2
+; CHECK: %ptr0.i3 = extractelement <4 x float*> %ptr0, i32 3
 ; CHECK: %i0.i1 = extractelement <4 x i32> %i0, i32 1
 ; CHECK: %i0.i3 = extractelement <4 x i32> %i0, i32 3
-; CHECK: %ptr0.i0 = extractelement <4 x float*> %ptr0, i32 0
 ; CHECK: %val.i0 = getelementptr float, float* %ptr0.i0, i32 100
 ; CHECK: %val.i1 = getelementptr float, float* %other, i32 %i0.i1
-; CHECK: %ptr0.i2 = extractelement <4 x float*> %ptr0, i32 2
 ; CHECK: %val.i2 = getelementptr float, float* %ptr0.i2, i32 100
-; CHECK: %ptr0.i3 = extractelement <4 x float*> %ptr0, i32 3
 ; CHECK: %val.i3 = getelementptr float, float* %ptr0.i3, i32 %i0.i3
 ; CHECK: store float* %val.i0, float** %dest.i0, align 32
 ; CHECK: store float* %val.i1, float** %dest.i1, align 8
@@ -363,26 +363,6 @@ define void @f11(<32 x i1> *%dest, <32 x i1> *%src0) {
   ret void
 }
 
-; Test that variable inserts aren't scalarized.
-define void @f12(<4 x i32> *%dest, <4 x i32> *%src, i32 %index) {
-; CHECK: @f12(
-; CHECK: %val1 = insertelement <4 x i32> %val0, i32 1, i32 %index
-; CHECK-DAG: %val1.i0 = extractelement <4 x i32> %val1, i32 0
-; CHECK-DAG: %val1.i1 = extractelement <4 x i32> %val1, i32 1
-; CHECK-DAG: %val1.i2 = extractelement <4 x i32> %val1, i32 2
-; CHECK-DAG: %val1.i3 = extractelement <4 x i32> %val1, i32 3
-; CHECK-DAG: %val2.i0 = shl i32 1, %val1.i0
-; CHECK-DAG: %val2.i1 = shl i32 2, %val1.i1
-; CHECK-DAG: %val2.i2 = shl i32 3, %val1.i2
-; CHECK-DAG: %val2.i3 = shl i32 4, %val1.i3
-; CHECK: ret void
-  %val0 = load <4 x i32> , <4 x i32> *%src
-  %val1 = insertelement <4 x i32> %val0, i32 1, i32 %index
-  %val2 = shl <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %val1
-  store <4 x i32> %val2, <4 x i32> *%dest
-  ret void
-}
-
 ; Test vector GEPs with more than one index.
 define void @f13(<4 x float *> *%dest, <4 x [4 x float] *> %ptr, <4 x i32> %i,
                  float *%other) {
@@ -559,6 +539,19 @@ define <2 x float> @f22(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
   ret <2 x float> %res
 }
 
+; See https://reviews.llvm.org/D83101#2133062
+define <2 x i32> @f23_crash(<2 x i32> %srcvec, i32 %v1) {
+; CHECK-LABEL: @f23_crash(
+; CHECK: %1 = extractelement <2 x i32> %srcvec, i32 0
+; CHECK: %t1.upto0 = insertelement <2 x i32> undef, i32 %1, i32 0
+; CHECK: %t1 = insertelement <2 x i32> %t1.upto0, i32 %v1, i32 1
+; CHECK: ret <2 x i32> %t1
+  %v0 = extractelement <2 x i32> %srcvec, i32 0
+  %t0 = insertelement <2 x i32> undef, i32 %v0, i32 0
+  %t1 = insertelement <2 x i32> %t0, i32 %v1, i32 1
+  ret <2 x i32> %t1
+}
+
 !0 = !{ !"root" }
 !1 = !{ !"set1", !0 }
 !2 = !{ !"set2", !0 }
diff --git a/llvm/test/Transforms/Scalarizer/constant-extractelement.ll b/llvm/test/Transforms/Scalarizer/constant-extractelement.ll
new file mode 100644
index 0000000000000..f5bb2edac4e68
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/constant-extractelement.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -scalarizer -scalarize-load-store -dce -S | FileCheck --check-prefixes=ALL %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Test that constant extracts are nicely scalarized
+define i32 @f1(<4 x i32> *%src, i32 %index) {
+; ALL-LABEL: @f1(
+; ALL-NEXT:    [[SRC_I0:%.*]] = bitcast <4 x i32>* [[SRC:%.*]] to i32*
+; ALL-NEXT:    [[SRC_I3:%.*]] = getelementptr i32, i32* [[SRC_I0]], i32 3
+; ALL-NEXT:    [[VAL0_I3:%.*]] = load i32, i32* [[SRC_I3]], align 4
+; ALL-NEXT:    [[VAL2:%.*]] = shl i32 4, [[VAL0_I3]]
+; ALL-NEXT:    ret i32 [[VAL2]]
+;
+  %val0 = load <4 x i32> , <4 x i32> *%src
+  %val1 = shl <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %val0
+  %val2 = extractelement <4 x i32> %val1, i32 3
+  ret i32 %val2
+}
diff --git a/llvm/test/Transforms/Scalarizer/constant-insertelement.ll b/llvm/test/Transforms/Scalarizer/constant-insertelement.ll
new file mode 100644
index 0000000000000..4ddde3598334d
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/constant-insertelement.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -scalarizer -scalarize-load-store -dce -S | FileCheck --check-prefixes=ALL %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Test that constant inserts are nicely scalarized
+define <4 x i32> @f1(<4 x i32> *%src, i32 %repl, i32 %index) {
+; ALL-LABEL: @f1(
+; ALL-NEXT:    [[SRC_I0:%.*]] = bitcast <4 x i32>* [[SRC:%.*]] to i32*
+; ALL-NEXT:    [[VAL0_I0:%.*]] = load i32, i32* [[SRC_I0]], align 16
+; ALL-NEXT:    [[SRC_I1:%.*]] = getelementptr i32, i32* [[SRC_I0]], i32 1
+; ALL-NEXT:    [[VAL0_I1:%.*]] = load i32, i32* [[SRC_I1]], align 4
+; ALL-NEXT:    [[SRC_I2:%.*]] = getelementptr i32, i32* [[SRC_I0]], i32 2
+; ALL-NEXT:    [[VAL0_I2:%.*]] = load i32, i32* [[SRC_I2]], align 8
+; ALL-NEXT:    [[VAL2_I0:%.*]] = shl i32 1, [[VAL0_I0]]
+; ALL-NEXT:    [[VAL2_I1:%.*]] = shl i32 2, [[VAL0_I1]]
+; ALL-NEXT:    [[VAL2_I2:%.*]] = shl i32 3, [[VAL0_I2]]
+; ALL-NEXT:    [[VAL2_I3:%.*]] = shl i32 4, [[REPL:%.*]]
+; ALL-NEXT:    [[VAL2_UPTO0:%.*]] = insertelement <4 x i32> undef, i32 [[VAL2_I0]], i32 0
+; ALL-NEXT:    [[VAL2_UPTO1:%.*]] = insertelement <4 x i32> [[VAL2_UPTO0]], i32 [[VAL2_I1]], i32 1
+; ALL-NEXT:    [[VAL2_UPTO2:%.*]] = insertelement <4 x i32> [[VAL2_UPTO1]], i32 [[VAL2_I2]], i32 2
+; ALL-NEXT:    [[VAL2:%.*]] = insertelement <4 x i32> [[VAL2_UPTO2]], i32 [[VAL2_I3]], i32 3
+; ALL-NEXT:    ret <4 x i32> [[VAL2]]
+;
+  %val0 = load <4 x i32> , <4 x i32> *%src
+  %val1 = insertelement <4 x i32> %val0, i32 %repl, i32 3
+  %val2 = shl <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %val1
+  ret <4 x i32> %val2
+}
diff --git a/llvm/test/Transforms/Scalarizer/crash-bug.ll b/llvm/test/Transforms/Scalarizer/crash-bug.ll
index d0d019564977c..97756362d4244 100644
--- a/llvm/test/Transforms/Scalarizer/crash-bug.ll
+++ b/llvm/test/Transforms/Scalarizer/crash-bug.ll
@@ -15,7 +15,6 @@ bb2:                                        ; preds = %bb1
 bb1:                                        ; preds = %bb2, %0
   %bb1_vec = phi <2 x i16> [ <i16 100, i16 200>, %0 ], [ %bb2_vec, %bb2 ]
 ;CHECK: bb1:
-;CHECK: %bb1_vec.i0 = phi i16 [ 100, %0 ], [ 0, %bb2 ]
 ;CHECK: %bb2_vec.i1 = phi i16 [ 200, %0 ], [ %bb2_vec.i1, %bb2 ]
   br i1 undef, label %bb3, label %bb2
 
@@ -23,3 +22,32 @@ bb3:
   ret void
 }
 
+; See https://reviews.llvm.org/D83101#2135945
+define void @f1_crash(<2 x i16> %base, i1 %c, <2 x i16>* %ptr) {
+; CHECK-LABEL: @f1_crash(
+; CHECK: vector.ph:
+; CHECK:   %base.i0 = extractelement <2 x i16> %base, i32 0
+; CHECK:   %base.i1 = extractelement <2 x i16> %base, i32 1
+; CHECK:   br label %vector.body115
+; CHECK: vector.body115:                                   ; preds = %vector.body115, %vector.ph
+; CHECK:   %vector.recur.i0 = phi i16 [ %base.i0, %vector.ph ], [ %wide.load125.i0, %vector.body115 ]
+; CHECK:   %vector.recur.i1 = phi i16 [ %base.i1, %vector.ph ], [ %wide.load125.i1, %vector.body115 ]
+; CHECK:   %wide.load125 = load <2 x i16>, <2 x i16>* %ptr, align 1
+; CHECK:   %wide.load125.i0 = extractelement <2 x i16> %wide.load125, i32 0
+; CHECK:   %wide.load125.i1 = extractelement <2 x i16> %wide.load125, i32 1
+; CHECK:   br i1 %c, label %middle.block113, label %vector.body115
+; CHECK: middle.block113:                                  ; preds = %vector.body115
+; CHECK:   ret void
+; CHECK: }
+
+vector.ph:
+  br label %vector.body115
+
+vector.body115:
+  %vector.recur = phi <2 x i16> [ %base, %vector.ph ], [ %wide.load125, %vector.body115 ]
+  %wide.load125 = load <2 x i16>, <2 x i16>* %ptr, align 1
+  br i1 %c, label %middle.block113, label %vector.body115
+
+middle.block113:
+  ret void
+}
diff --git a/llvm/test/Transforms/Scalarizer/phi-unreachable-pred.ll b/llvm/test/Transforms/Scalarizer/phi-unreachable-pred.ll
index 1de1f6509666f..9cfffe3b977fb 100644
--- a/llvm/test/Transforms/Scalarizer/phi-unreachable-pred.ll
+++ b/llvm/test/Transforms/Scalarizer/phi-unreachable-pred.ll
@@ -11,15 +11,7 @@ define i16 @f1() {
 ; CHECK:       for.cond:
 ; CHECK-NEXT:    br i1 undef, label [[FOR_BODY:%.*]], label [[FOR_END]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    [[PHI_I0:%.*]] = phi i16 [ 1, [[ENTRY:%.*]] ], [ undef, [[FOR_COND]] ]
-; CHECK-NEXT:    [[PHI_I1:%.*]] = phi i16 [ 1, [[ENTRY]] ], [ undef, [[FOR_COND]] ]
-; CHECK-NEXT:    [[PHI_I2:%.*]] = phi i16 [ 1, [[ENTRY]] ], [ undef, [[FOR_COND]] ]
-; CHECK-NEXT:    [[PHI_I3:%.*]] = phi i16 [ 1, [[ENTRY]] ], [ undef, [[FOR_COND]] ]
-; CHECK-NEXT:    [[PHI_UPTO0:%.*]] = insertelement <4 x i16> undef, i16 [[PHI_I0]], i32 0
-; CHECK-NEXT:    [[PHI_UPTO1:%.*]] = insertelement <4 x i16> [[PHI_UPTO0]], i16 [[PHI_I1]], i32 1
-; CHECK-NEXT:    [[PHI_UPTO2:%.*]] = insertelement <4 x i16> [[PHI_UPTO1]], i16 [[PHI_I2]], i32 2
-; CHECK-NEXT:    [[PHI:%.*]] = insertelement <4 x i16> [[PHI_UPTO2]], i16 [[PHI_I3]], i32 3
-; CHECK-NEXT:    [[EXTRACT:%.*]] = extractelement <4 x i16> [[PHI]], i32 0
+; CHECK-NEXT:    [[EXTRACT:%.*]] = phi i16 [ 1, [[ENTRY:%.*]] ], [ undef, [[FOR_COND]] ]
 ; CHECK-NEXT:    ret i16 [[EXTRACT]]
 ;
 entry:
diff --git a/llvm/test/Transforms/Scalarizer/variable-extractelement.ll b/llvm/test/Transforms/Scalarizer/variable-extractelement.ll
new file mode 100644
index 0000000000000..50666562af327
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/variable-extractelement.ll
@@ -0,0 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -scalarizer -dce -S | FileCheck --check-prefixes=ALL,DEFAULT %s
+; RUN: opt %s -scalarizer -scalarize-variable-insert-extract=false -dce -S | FileCheck --check-prefixes=ALL,OFF %s
+; RUN: opt %s -scalarizer -scalarize-variable-insert-extract=true -dce -S | FileCheck --check-prefixes=ALL,DEFAULT,ON %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Test that variable extracts scalarized.
+define i32 @f1(<4 x i32> %src, i32 %index) {
+; DEFAULT-LABEL: @f1(
+; DEFAULT-NEXT:    [[INDEX_IS_0:%.*]] = icmp eq i32 [[INDEX:%.*]], 0
+; DEFAULT-NEXT:    [[SRC_I0:%.*]] = extractelement <4 x i32> [[SRC:%.*]], i32 0
+; DEFAULT-NEXT:    [[RES_UPTO0:%.*]] = select i1 [[INDEX_IS_0]], i32 [[SRC_I0]], i32 undef
+; DEFAULT-NEXT:    [[INDEX_IS_1:%.*]] = icmp eq i32 [[INDEX]], 1
+; DEFAULT-NEXT:    [[SRC_I1:%.*]] = extractelement <4 x i32> [[SRC]], i32 1
+; DEFAULT-NEXT:    [[RES_UPTO1:%.*]] = select i1 [[INDEX_IS_1]], i32 [[SRC_I1]], i32 [[RES_UPTO0]]
+; DEFAULT-NEXT:    [[INDEX_IS_2:%.*]] = icmp eq i32 [[INDEX]], 2
+; DEFAULT-NEXT:    [[SRC_I2:%.*]] = extractelement <4 x i32> [[SRC]], i32 2
+; DEFAULT-NEXT:    [[RES_UPTO2:%.*]] = select i1 [[INDEX_IS_2]], i32 [[SRC_I2]], i32 [[RES_UPTO1]]
+; DEFAULT-NEXT:    [[INDEX_IS_3:%.*]] = icmp eq i32 [[INDEX]], 3
+; DEFAULT-NEXT:    [[SRC_I3:%.*]] = extractelement <4 x i32> [[SRC]], i32 3
+; DEFAULT-NEXT:    [[RES:%.*]] = select i1 [[INDEX_IS_3]], i32 [[SRC_I3]], i32 [[RES_UPTO2]]
+; DEFAULT-NEXT:    ret i32 [[RES]]
+;
+; OFF-LABEL: @f1(
+; OFF-NEXT:    [[RES:%.*]] = extractelement <4 x i32> [[SRC:%.*]], i32 [[INDEX:%.*]]
+; OFF-NEXT:    ret i32 [[RES]]
+;
+  %res = extractelement <4 x i32> %src, i32 %index
+  ret i32 %res
+}
+
+define i32 @f2(<4 x i32> *%src, i32 %index) {
+; DEFAULT-LABEL: @f2(
+; DEFAULT-NEXT:    [[VAL0:%.*]] = load <4 x i32>, <4 x i32>* [[SRC:%.*]], align 16
+; DEFAULT-NEXT:    [[VAL0_I0:%.*]] = extractelement <4 x i32> [[VAL0]], i32 0
+; DEFAULT-NEXT:    [[VAL1_I0:%.*]] = shl i32 1, [[VAL0_I0]]
+; DEFAULT-NEXT:    [[VAL0_I1:%.*]] = extractelement <4 x i32> [[VAL0]], i32 1
+; DEFAULT-NEXT:    [[VAL1_I1:%.*]] = shl i32 2, [[VAL0_I1]]
+; DEFAULT-NEXT:    [[VAL0_I2:%.*]] = extractelement <4 x i32> [[VAL0]], i32 2
+; DEFAULT-NEXT:    [[VAL1_I2:%.*]] = shl i32 3, [[VAL0_I2]]
+; DEFAULT-NEXT:    [[VAL0_I3:%.*]] = extractelement <4 x i32> [[VAL0]], i32 3
+; DEFAULT-NEXT:    [[VAL1_I3:%.*]] = shl i32 4, [[VAL0_I3]]
+; DEFAULT-NEXT:    [[INDEX_IS_0:%.*]] = icmp eq i32 [[INDEX:%.*]], 0
+; DEFAULT-NEXT:    [[VAL2_UPTO0:%.*]] = select i1 [[INDEX_IS_0]], i32 [[VAL1_I0]], i32 undef
+; DEFAULT-NEXT:    [[INDEX_IS_1:%.*]] = icmp eq i32 [[INDEX]], 1
+; DEFAULT-NEXT:    [[VAL2_UPTO1:%.*]] = select i1 [[INDEX_IS_1]], i32 [[VAL1_I1]], i32 [[VAL2_UPTO0]]
+; DEFAULT-NEXT:    [[INDEX_IS_2:%.*]] = icmp eq i32 [[INDEX]], 2
+; DEFAULT-NEXT:    [[VAL2_UPTO2:%.*]] = select i1 [[INDEX_IS_2]], i32 [[VAL1_I2]], i32 [[VAL2_UPTO1]]
+; DEFAULT-NEXT:    [[INDEX_IS_3:%.*]] = icmp eq i32 [[INDEX]], 3
+; DEFAULT-NEXT:    [[VAL2:%.*]] = select i1 [[INDEX_IS_3]], i32 [[VAL1_I3]], i32 [[VAL2_UPTO2]]
+; DEFAULT-NEXT:    ret i32 [[VAL2]]
+;
+; OFF-LABEL: @f2(
+; OFF-NEXT:    [[VAL0:%.*]] = load <4 x i32>, <4 x i32>* [[SRC:%.*]], align 16
+; OFF-NEXT:    [[VAL0_I0:%.*]] = extractelement <4 x i32> [[VAL0]], i32 0
+; OFF-NEXT:    [[VAL1_I0:%.*]] = shl i32 1, [[VAL0_I0]]
+; OFF-NEXT:    [[VAL0_I1:%.*]] = extractelement <4 x i32> [[VAL0]], i32 1
+; OFF-NEXT:    [[VAL1_I1:%.*]] = shl i32 2, [[VAL0_I1]]
+; OFF-NEXT:    [[VAL0_I2:%.*]] = extractelement <4 x i32> [[VAL0]], i32 2
+; OFF-NEXT:    [[VAL1_I2:%.*]] = shl i32 3, [[VAL0_I2]]
+; OFF-NEXT:    [[VAL0_I3:%.*]] = extractelement <4 x i32> [[VAL0]], i32 3
+; OFF-NEXT:    [[VAL1_I3:%.*]] = shl i32 4, [[VAL0_I3]]
+; OFF-NEXT:    [[VAL1_UPTO0:%.*]] = insertelement <4 x i32> undef, i32 [[VAL1_I0]], i32 0
+; OFF-NEXT:    [[VAL1_UPTO1:%.*]] = insertelement <4 x i32> [[VAL1_UPTO0]], i32 [[VAL1_I1]], i32 1
+; OFF-NEXT:    [[VAL1_UPTO2:%.*]] = insertelement <4 x i32> [[VAL1_UPTO1]], i32 [[VAL1_I2]], i32 2
+; OFF-NEXT:    [[VAL1:%.*]] = insertelement <4 x i32> [[VAL1_UPTO2]], i32 [[VAL1_I3]], i32 3
+; OFF-NEXT:    [[VAL2:%.*]] = extractelement <4 x i32> [[VAL1]], i32 [[INDEX:%.*]]
+; OFF-NEXT:    ret i32 [[VAL2]]
+;
+  %val0 = load <4 x i32> , <4 x i32> *%src
+  %val1 = shl <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %val0
+  %val2 = extractelement <4 x i32> %val1, i32 %index
+  ret i32 %val2
+}
diff --git a/llvm/test/Transforms/Scalarizer/variable-insertelement.ll b/llvm/test/Transforms/Scalarizer/variable-insertelement.ll
new file mode 100644
index 0000000000000..aeec2ddea2eaa
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/variable-insertelement.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -scalarizer -dce -S | FileCheck --check-prefixes=ALL,DEFAULT %s
+; RUN: opt %s -scalarizer -scalarize-variable-insert-extract=false -dce -S | FileCheck --check-prefixes=ALL,OFF %s
+; RUN: opt %s -scalarizer -scalarize-variable-insert-extract=true -dce -S | FileCheck --check-prefixes=ALL,DEFAULT,ON %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Test that variable inserts are scalarized.
+define <4 x i32> @f1(<4 x i32> %src, i32 %val, i32 %index) {
+; DEFAULT-LABEL: @f1(
+; DEFAULT-NEXT:    [[INDEX_IS_0:%.*]] = icmp eq i32 [[INDEX:%.*]], 0
+; DEFAULT-NEXT:    [[SRC_I0:%.*]] = extractelement <4 x i32> [[SRC:%.*]], i32 0
+; DEFAULT-NEXT:    [[RES_I0:%.*]] = select i1 [[INDEX_IS_0]], i32 [[VAL:%.*]], i32 [[SRC_I0]]
+; DEFAULT-NEXT:    [[INDEX_IS_1:%.*]] = icmp eq i32 [[INDEX]], 1
+; DEFAULT-NEXT:    [[SRC_I1:%.*]] = extractelement <4 x i32> [[SRC]], i32 1
+; DEFAULT-NEXT:    [[RES_I1:%.*]] = select i1 [[INDEX_IS_1]], i32 [[VAL]], i32 [[SRC_I1]]
+; DEFAULT-NEXT:    [[INDEX_IS_2:%.*]] = icmp eq i32 [[INDEX]], 2
+; DEFAULT-NEXT:    [[SRC_I2:%.*]] = extractelement <4 x i32> [[SRC]], i32 2
+; DEFAULT-NEXT:    [[RES_I2:%.*]] = select i1 [[INDEX_IS_2]], i32 [[VAL]], i32 [[SRC_I2]]
+; DEFAULT-NEXT:    [[INDEX_IS_3:%.*]] = icmp eq i32 [[INDEX]], 3
+; DEFAULT-NEXT:    [[SRC_I3:%.*]] = extractelement <4 x i32> [[SRC]], i32 3
+; DEFAULT-NEXT:    [[RES_I3:%.*]] = select i1 [[INDEX_IS_3]], i32 [[VAL]], i32 [[SRC_I3]]
+; DEFAULT-NEXT:    [[RES_UPTO0:%.*]] = insertelement <4 x i32> undef, i32 [[RES_I0]], i32 0
+; DEFAULT-NEXT:    [[RES_UPTO1:%.*]] = insertelement <4 x i32> [[RES_UPTO0]], i32 [[RES_I1]], i32 1
+; DEFAULT-NEXT:    [[RES_UPTO2:%.*]] = insertelement <4 x i32> [[RES_UPTO1]], i32 [[RES_I2]], i32 2
+; DEFAULT-NEXT:    [[RES:%.*]] = insertelement <4 x i32> [[RES_UPTO2]], i32 [[RES_I3]], i32 3
+; DEFAULT-NEXT:    ret <4 x i32> [[RES]]
+;
+; OFF-LABEL: @f1(
+; OFF-NEXT:    [[RES:%.*]] = insertelement <4 x i32> [[SRC:%.*]], i32 [[VAL:%.*]], i32 [[INDEX:%.*]]
+; OFF-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = insertelement <4 x i32> %src, i32 %val, i32 %index
+  ret <4 x i32> %res
+}
+
+define void @f2(<4 x i32> *%dest, <4 x i32> *%src, i32 %index) {
+; DEFAULT-LABEL: @f2(
+; DEFAULT-NEXT:    [[VAL0:%.*]] = load <4 x i32>, <4 x i32>* [[SRC:%.*]], align 16
+; DEFAULT-NEXT:    [[INDEX_IS_0:%.*]] = icmp eq i32 [[INDEX:%.*]], 0
+; DEFAULT-NEXT:    [[VAL0_I0:%.*]] = extractelement <4 x i32> [[VAL0]], i32 0
+; DEFAULT-NEXT:    [[VAL1_I0:%.*]] = select i1 [[INDEX_IS_0]], i32 1, i32 [[VAL0_I0]]
+; DEFAULT-NEXT:    [[INDEX_IS_1:%.*]] = icmp eq i32 [[INDEX]], 1
+; DEFAULT-NEXT:    [[VAL0_I1:%.*]] = extractelement <4 x i32> [[VAL0]], i32 1
+; DEFAULT-NEXT:    [[VAL1_I1:%.*]] = select i1 [[INDEX_IS_1]], i32 1, i32 [[VAL0_I1]]
+; DEFAULT-NEXT:    [[INDEX_IS_2:%.*]] = icmp eq i32 [[INDEX]], 2
+; DEFAULT-NEXT:    [[VAL0_I2:%.*]] = extractelement <4 x i32> [[VAL0]], i32 2
+; DEFAULT-NEXT:    [[VAL1_I2:%.*]] = select i1 [[INDEX_IS_2]], i32 1, i32 [[VAL0_I2]]
+; DEFAULT-NEXT:    [[INDEX_IS_3:%.*]] = icmp eq i32 [[INDEX]], 3
+; DEFAULT-NEXT:    [[VAL0_I3:%.*]] = extractelement <4 x i32> [[VAL0]], i32 3
+; DEFAULT-NEXT:    [[VAL1_I3:%.*]] = select i1 [[INDEX_IS_3]], i32 1, i32 [[VAL0_I3]]
+; DEFAULT-NEXT:    [[VAL2_I0:%.*]] = shl i32 1, [[VAL1_I0]]
+; DEFAULT-NEXT:    [[VAL2_I1:%.*]] = shl i32 2, [[VAL1_I1]]
+; DEFAULT-NEXT:    [[VAL2_I2:%.*]] = shl i32 3, [[VAL1_I2]]
+; DEFAULT-NEXT:    [[VAL2_I3:%.*]] = shl i32 4, [[VAL1_I3]]
+; DEFAULT-NEXT:    [[VAL2_UPTO0:%.*]] = insertelement <4 x i32> undef, i32 [[VAL2_I0]], i32 0
+; DEFAULT-NEXT:    [[VAL2_UPTO1:%.*]] = insertelement <4 x i32> [[VAL2_UPTO0]], i32 [[VAL2_I1]], i32 1
+; DEFAULT-NEXT:    [[VAL2_UPTO2:%.*]] = insertelement <4 x i32> [[VAL2_UPTO1]], i32 [[VAL2_I2]], i32 2
+; DEFAULT-NEXT:    [[VAL2:%.*]] = insertelement <4 x i32> [[VAL2_UPTO2]], i32 [[VAL2_I3]], i32 3
+; DEFAULT-NEXT:    store <4 x i32> [[VAL2]], <4 x i32>* [[DEST:%.*]], align 16
+; DEFAULT-NEXT:    ret void
+;
+; OFF-LABEL: @f2(
+; OFF-NEXT:    [[VAL0:%.*]] = load <4 x i32>, <4 x i32>* [[SRC:%.*]], align 16
+; OFF-NEXT:    [[VAL1:%.*]] = insertelement <4 x i32> [[VAL0]], i32 1, i32 [[INDEX:%.*]]
+; OFF-NEXT:    [[VAL1_I0:%.*]] = extractelement <4 x i32> [[VAL1]], i32 0
+; OFF-NEXT:    [[VAL2_I0:%.*]] = shl i32 1, [[VAL1_I0]]
+; OFF-NEXT:    [[VAL1_I1:%.*]] = extractelement <4 x i32> [[VAL1]], i32 1
+; OFF-NEXT:    [[VAL2_I1:%.*]] = shl i32 2, [[VAL1_I1]]
+; OFF-NEXT:    [[VAL1_I2:%.*]] = extractelement <4 x i32> [[VAL1]], i32 2
+; OFF-NEXT:    [[VAL2_I2:%.*]] = shl i32 3, [[VAL1_I2]]
+; OFF-NEXT:    [[VAL1_I3:%.*]] = extractelement <4 x i32> [[VAL1]], i32 3
+; OFF-NEXT:    [[VAL2_I3:%.*]] = shl i32 4, [[VAL1_I3]]
+; OFF-NEXT:    [[VAL2_UPTO0:%.*]] = insertelement <4 x i32> undef, i32 [[VAL2_I0]], i32 0
+; OFF-NEXT:    [[VAL2_UPTO1:%.*]] = insertelement <4 x i32> [[VAL2_UPTO0]], i32 [[VAL2_I1]], i32 1
+; OFF-NEXT:    [[VAL2_UPTO2:%.*]] = insertelement <4 x i32> [[VAL2_UPTO1]], i32 [[VAL2_I2]], i32 2
+; OFF-NEXT:    [[VAL2:%.*]] = insertelement <4 x i32> [[VAL2_UPTO2]], i32 [[VAL2_I3]], i32 3
+; OFF-NEXT:    store <4 x i32> [[VAL2]], <4 x i32>* [[DEST:%.*]], align 16
+; OFF-NEXT:    ret void
+;
+  %val0 = load <4 x i32> , <4 x i32> *%src
+  %val1 = insertelement <4 x i32> %val0, i32 1, i32 %index
+  %val2 = shl <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %val1
+  store <4 x i32> %val2, <4 x i32> *%dest
+  ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
index 8f05e58b8ebb7..c9837c5cc141d 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -instcombine -inline -functionattrs -licm -simple-loop-unswitch -gvn -verify
+; RUN: opt < %s -basic-aa -instcombine -inline -functionattrs -licm -simple-loop-unswitch -gvn -verify
 ; PR12573
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.7.0"
diff --git a/llvm/test/Transforms/SimplifyCFG/1elt-ptr-vec-alignment-crash.ll b/llvm/test/Transforms/SimplifyCFG/1elt-ptr-vec-alignment-crash.ll
new file mode 100644
index 0000000000000..360fd0079cd13
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/1elt-ptr-vec-alignment-crash.ll
@@ -0,0 +1,26 @@
+; RUN: opt -passes=simplify-cfg -S < %s | FileCheck %s
+
+; Ensure that we do not crash when trying to evaluate alignment of a <1 x ???*>.
+
+%struct.ap = type { i8 }
+%"struct.z::ai" = type { i32 }
+
+@x = external dso_local local_unnamed_addr global i32, align 4
+
+define dso_local void @_ZN2ap2aqEv(%struct.ap* %this, i1 %c, i32 %v2) {
+; CHECK-LABEL: @_ZN2ap2aqEv(
+_ZN1yIN1z2aiE2aaIS1_EE2ahEv.exit:
+  br i1 %c, label %if.end, label %land.rhs
+
+land.rhs:                                         ; preds = %_ZN1yIN1z2aiE2aaIS1_EE2ahEv.exit
+  %0 = bitcast <1 x %"struct.z::ai"*> zeroinitializer to %"struct.z::ai"*
+  %retval.sroa.0.0..sroa_idx.i = getelementptr inbounds %"struct.z::ai", %"struct.z::ai"* %0, i64 0, i32 0
+  %retval.sroa.0.0.copyload.i = load i32, i32* %retval.sroa.0.0..sroa_idx.i, align 4
+  %tobool5 = icmp eq i32 %retval.sroa.0.0.copyload.i, 0
+  %spec.select = select i1 %tobool5, i32 %v2, i32 0
+  br label %if.end
+
+if.end:                                           ; preds = %land.rhs, %_ZN1yIN1z2aiE2aaIS1_EE2ahEv.exit
+  %b.0 = phi i32 [ %spec.select, %land.rhs ], [ 0, %_ZN1yIN1z2aiE2aaIS1_EE2ahEv.exit ]
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/ForwardSwitchConditionToPHI.ll b/llvm/test/Transforms/SimplifyCFG/ForwardSwitchConditionToPHI.ll
index 169f3f629ea26..b79ccd6a97a7a 100644
--- a/llvm/test/Transforms/SimplifyCFG/ForwardSwitchConditionToPHI.ll
+++ b/llvm/test/Transforms/SimplifyCFG/ForwardSwitchConditionToPHI.ll
@@ -34,8 +34,8 @@ define i32 @t(i32 %m) nounwind readnone {
 ; FWD-LABEL: @t(
 ; FWD-NEXT:  entry:
 ; FWD-NEXT:    [[SWITCH:%.*]] = icmp ult i32 [[M:%.*]], 4
-; FWD-NEXT:    [[M_:%.*]] = select i1 [[SWITCH]], i32 [[M]], i32 4
-; FWD-NEXT:    ret i32 [[M_]]
+; FWD-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[SWITCH]], i32 [[M]], i32 4
+; FWD-NEXT:    ret i32 [[SPEC_SELECT]]
 ;
 entry:
   switch i32 %m, label %sw.bb4 [
diff --git a/llvm/test/Transforms/SimplifyCFG/2010-03-30-InvokeCrash.ll b/llvm/test/Transforms/SimplifyCFG/X86/2010-03-30-InvokeCrash.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/2010-03-30-InvokeCrash.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/2010-03-30-InvokeCrash.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/CoveredLookupTable.ll b/llvm/test/Transforms/SimplifyCFG/X86/CoveredLookupTable.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/CoveredLookupTable.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/CoveredLookupTable.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/MagicPointer.ll b/llvm/test/Transforms/SimplifyCFG/X86/MagicPointer.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/MagicPointer.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/MagicPointer.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/PR29163.ll b/llvm/test/Transforms/SimplifyCFG/X86/PR29163.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/PR29163.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/PR29163.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/PR30210.ll b/llvm/test/Transforms/SimplifyCFG/X86/PR30210.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/PR30210.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/PR30210.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/SpeculativeExec.ll b/llvm/test/Transforms/SimplifyCFG/X86/SpeculativeExec.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/SpeculativeExec.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/SpeculativeExec.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/bug-25299.ll b/llvm/test/Transforms/SimplifyCFG/X86/bug-25299.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/bug-25299.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/bug-25299.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/combine-parallel-mem-md.ll b/llvm/test/Transforms/SimplifyCFG/X86/combine-parallel-mem-md.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/combine-parallel-mem-md.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/combine-parallel-mem-md.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/critedge-assume.ll b/llvm/test/Transforms/SimplifyCFG/X86/critedge-assume.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/critedge-assume.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/critedge-assume.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/empty-cleanuppad.ll b/llvm/test/Transforms/SimplifyCFG/X86/empty-cleanuppad.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/empty-cleanuppad.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/empty-cleanuppad.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/fold-branch-debuginvariant.ll b/llvm/test/Transforms/SimplifyCFG/X86/fold-branch-debuginvariant.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/fold-branch-debuginvariant.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/fold-branch-debuginvariant.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/invalidate-dom.ll b/llvm/test/Transforms/SimplifyCFG/X86/invalidate-dom.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/invalidate-dom.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/invalidate-dom.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/merge-cleanuppads.ll b/llvm/test/Transforms/SimplifyCFG/X86/merge-cleanuppads.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/merge-cleanuppads.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/merge-cleanuppads.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/remove-debug-2.ll b/llvm/test/Transforms/SimplifyCFG/X86/remove-debug-2.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/remove-debug-2.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/remove-debug-2.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/remove-debug.ll b/llvm/test/Transforms/SimplifyCFG/X86/remove-debug.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/remove-debug.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/remove-debug.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/safe-low-bit-extract.ll b/llvm/test/Transforms/SimplifyCFG/X86/safe-low-bit-extract.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/safe-low-bit-extract.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/safe-low-bit-extract.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
index 597b5b969a739..002fb5531e765 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
@@ -41,7 +41,7 @@ define i32 @f(i32 %c) {
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
 ; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], [7 x i32]* @switch.table.f, i32 0, i32 [[SWITCH_TABLEIDX]]
-; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ; CHECK:       return:
 ; CHECK-NEXT:    ret i32 15
@@ -80,7 +80,7 @@ define i8 @char(i32 %c) {
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
 ; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [9 x i8], [9 x i8]* @switch.table.char, i32 0, i32 [[SWITCH_TABLEIDX]]
-; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i8, i8* [[SWITCH_GEP]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i8, i8* [[SWITCH_GEP]], align 1
 ; CHECK-NEXT:    ret i8 [[SWITCH_LOAD]]
 ; CHECK:       return:
 ; CHECK-NEXT:    ret i8 15
@@ -126,7 +126,7 @@ define void @h(i32 %x) {
 ; CHECK-NEXT:    [[SWITCH_DOWNSHIFT:%.*]] = lshr i32 89655594, [[SWITCH_SHIFTAMT]]
 ; CHECK-NEXT:    [[SWITCH_MASKED:%.*]] = trunc i32 [[SWITCH_DOWNSHIFT]] to i8
 ; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x float], [4 x float]* @switch.table.h, i32 0, i32 [[X]]
-; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load float, float* [[SWITCH_GEP]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load float, float* [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[SW_EPILOG]]
 ; CHECK:       sw.epilog:
 ; CHECK-NEXT:    [[A_0:%.*]] = phi i8 [ [[SWITCH_MASKED]], [[SWITCH_LOOKUP]] ], [ 7, [[ENTRY:%.*]] ]
@@ -171,7 +171,7 @@ define i8* @foostring(i32 %x)  {
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
 ; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i8*], [4 x i8*]* @switch.table.foostring, i32 0, i32 [[X]]
-; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i8*, i8** [[SWITCH_GEP]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i8*, i8** [[SWITCH_GEP]], align 8
 ; CHECK-NEXT:    ret i8* [[SWITCH_LOAD]]
 ; CHECK:       return:
 ; CHECK-NEXT:    ret i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str4, i64 0, i64 0)
@@ -209,7 +209,7 @@ define i32 @earlyreturncrash(i32 %x)  {
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[SW_EPILOG:%.*]]
 ; CHECK:       switch.lookup:
 ; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* @switch.table.earlyreturncrash, i32 0, i32 [[X]]
-; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ; CHECK:       sw.epilog:
 ; CHECK-NEXT:    ret i32 7
@@ -383,7 +383,7 @@ define i32 @large(i32 %x) {
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
 ; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [199 x i32], [199 x i32]* @switch.table.large, i32 0, i32 [[SWITCH_TABLEIDX]]
-; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ; CHECK:       return:
 ; CHECK-NEXT:    ret i32 0
@@ -813,7 +813,7 @@ define i32 @cprop(i32 %x, i32 %y) {
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
 ; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], [7 x i32]* @switch.table.cprop, i32 0, i32 [[SWITCH_TABLEIDX]]
-; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ; CHECK:       return:
 ; CHECK-NEXT:    ret i32 123
@@ -862,7 +862,7 @@ define i32 @unreachable_case(i32 %x)  {
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
 ; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [9 x i32], [9 x i32]* @switch.table.unreachable_case, i32 0, i32 [[X]]
-; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ; CHECK:       return:
 ; CHECK-NEXT:    ret i32 2
@@ -896,7 +896,7 @@ define i32 @unreachable_default(i32 %x)  {
 ; CHECK-LABEL: @unreachable_default(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* @switch.table.unreachable_default, i32 0, i32 [[X:%.*]]
-; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
 entry:
@@ -976,7 +976,7 @@ define i32 @nodefaultnoholes(i32 %c) {
 ; CHECK-NEXT:    unreachable
 ; CHECK:       switch.lookup:
 ; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* @switch.table.nodefaultnoholes, i32 0, i32 [[C]]
-; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
 entry:
@@ -1014,7 +1014,7 @@ define i32 @nodefaultwithholes(i32 %c) {
 ; CHECK-NEXT:    br i1 [[SWITCH_LOBIT]], label [[SWITCH_LOOKUP:%.*]], label [[SW_DEFAULT]]
 ; CHECK:       switch.lookup:
 ; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [6 x i32], [6 x i32]* @switch.table.nodefaultwithholes, i32 0, i32 [[C]]
-; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
 entry:
@@ -1079,7 +1079,7 @@ define i32 @threecases(i32 %c) {
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
 ; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* @switch.table.threecases, i32 0, i32 [[C]]
-; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ; CHECK:       return:
 ; CHECK-NEXT:    ret i32 3
@@ -1550,7 +1550,7 @@ define i32 @covered_switch_with_bit_tests(i3) {
 ; CHECK:       switch.lookup:
 ; CHECK-NEXT:    [[SWITCH_TABLEIDX_ZEXT:%.*]] = zext i3 [[SWITCH_TABLEIDX]] to i4
 ; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* @switch.table.covered_switch_with_bit_tests, i32 0, i4 [[SWITCH_TABLEIDX_ZEXT]]
-; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]]
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[L6]]
 ; CHECK:       l6:
 ; CHECK-NEXT:    [[R:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[SWITCH_LOAD]], [[SWITCH_LOOKUP]] ]
diff --git a/llvm/test/Transforms/SimplifyCFG/unreachable-blocks.ll b/llvm/test/Transforms/SimplifyCFG/X86/unreachable-blocks.ll
similarity index 100%
rename from llvm/test/Transforms/SimplifyCFG/unreachable-blocks.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/unreachable-blocks.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/constantexprs.ll b/llvm/test/Transforms/SimplifyCFG/constantexprs.ll
new file mode 100644
index 0000000000000..5cfe2cfceafbb
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/constantexprs.ll
@@ -0,0 +1,31 @@
+; RUN: opt -simplifycfg -S < %s | FileCheck %s
+
+; See that we do not crash when queriying cost model about the cost of constant expression extractelement.
+
+%"struct.(anonymous namespace)::aj" = type { %struct.o }
+%struct.o = type { %struct.j }
+%struct.j = type { %struct.c }
+%struct.c = type { %struct.a }
+%struct.a = type { i8 }
+%struct.e = type { %struct.a* }
+
+$_ZN1eC2EPK1a = comdat any
+
+@_ZN12_GLOBAL__N_12anE = internal global %"struct.(anonymous namespace)::aj" zeroinitializer, align 1
+
+declare dso_local i32 @_Zeq1eS_(%struct.a*, %struct.a*) local_unnamed_addr #2
+
+define internal fastcc %struct.a* @_ZNK1jIiN12_GLOBAL__N_12ajEE2aeERKi() unnamed_addr #0 align 2 {
+; CHECK-LABEL: @_ZNK1jIiN12_GLOBAL__N_12ajEE2aeERKi
+entry:
+  %call = call i32 @_Zeq1eS_(%struct.a* null, %struct.a* null)
+  %tobool = icmp eq i32 %call, 0
+  br i1 %tobool, label %cond.false, label %cond.end
+
+cond.false:                                       ; preds = %entry
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.false
+  %retval.sroa.0.0 = phi %struct.a* [ null, %cond.false ], [ extractelement (<1 x %struct.a*> inttoptr (<1 x i64> bitcast (i64 ptrtoint (%"struct.(anonymous namespace)::aj"* @_ZN12_GLOBAL__N_12anE to i64) to <1 x i64>) to <1 x %struct.a*>), i64 0), %entry ]
+  ret %struct.a*  %retval.sroa.0.0
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/drop-debug-loc-when-speculating.ll b/llvm/test/Transforms/SimplifyCFG/drop-debug-loc-when-speculating.ll
new file mode 100644
index 0000000000000..6298b4e5de35c
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/drop-debug-loc-when-speculating.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -o - %s -simplifycfg | FileCheck %s
+
+declare i1 @make_condition()
+
+; CHECK-LABEL: @test1(
+; CHECK: and i1 [[COND:%.*]], [[COND]]{{$}}
+; CHECK: select i1 [[COND]]
+define void @test1() !dbg !6 {
+start:
+  %cond = call i1 @make_condition(), !dbg !8
+  br i1 %cond, label %then, label %else, !dbg !9
+
+then:                                             ; preds = %start
+  %and = and i1 %cond, %cond, !dbg !10
+  br label %else, !dbg !11
+
+else:                                             ; preds = %then, %start
+  %phi = phi i1 [ %cond, %start ], [ %and, %then ], !dbg !12
+  ret void, !dbg !13
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!3, !4}
+!llvm.module.flags = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "test.ll", directory: "/")
+!2 = !{}
+!3 = !{i32 6}
+!4 = !{i32 0}
+!5 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = distinct !DISubprogram(name: "test1", linkageName: "test1", scope: null, file: !1, line: 1, type: !7, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 1, column: 1, scope: !6)
+!9 = !DILocation(line: 2, column: 1, scope: !6)
+!10 = !DILocation(line: 3, column: 2, scope: !6)
+!11 = !DILocation(line: 4, column: 2, scope: !6)
+!12 = !DILocation(line: 5, column: 3, scope: !6)
+!13 = !DILocation(line: 6, column: 3, scope: !6)
diff --git a/llvm/test/Transforms/SimplifyCFG/duplicate-landingpad.ll b/llvm/test/Transforms/SimplifyCFG/duplicate-landingpad.ll
index 93c55f0064ccb..6f8b069810f8d 100644
--- a/llvm/test/Transforms/SimplifyCFG/duplicate-landingpad.ll
+++ b/llvm/test/Transforms/SimplifyCFG/duplicate-landingpad.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -simplifycfg -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
@@ -5,36 +6,41 @@ declare i32 @__gxx_personality_v0(...)
 declare void @fn()
 
 
-; CHECK-LABEL: @test1
 define void @test1() personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    invoke void @fn()
+; CHECK-NEXT:    to label [[INVOKE2:%.*]] unwind label [[LPAD2:%.*]]
+; CHECK:       invoke2:
+; CHECK-NEXT:    invoke void @fn()
+; CHECK-NEXT:    to label [[INVOKE_CONT:%.*]] unwind label [[LPAD2]]
+; CHECK:       invoke.cont:
+; CHECK-NEXT:    ret void
+; CHECK:       lpad2:
+; CHECK-NEXT:    [[EXN2:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    call void @fn()
+; CHECK-NEXT:    ret void
+;
 entry:
-; CHECK-LABEL: entry:
-; CHECK: to label %invoke2 unwind label %lpad2
   invoke void @fn()
-    to label %invoke2 unwind label %lpad1
+  to label %invoke2 unwind label %lpad1
 
 invoke2:
-; CHECK-LABEL: invoke2:
-; CHECK: to label %invoke.cont unwind label %lpad2
   invoke void @fn()
-    to label %invoke.cont unwind label %lpad2
+  to label %invoke.cont unwind label %lpad2
 
 invoke.cont:
   ret void
 
 lpad1:
   %exn = landingpad {i8*, i32}
-         cleanup
+  cleanup
   br label %shared_resume
 
 lpad2:
-; CHECK-LABEL: lpad2:
-; CHECK: landingpad { i8*, i32 }
-; CHECK-NEXT: cleanup
-; CHECK-NEXT: call void @fn()
-; CHECK-NEXT: ret void
   %exn2 = landingpad {i8*, i32}
-          cleanup
+  cleanup
   br label %shared_resume
 
 shared_resume:
@@ -44,31 +50,48 @@ shared_resume:
 
 ; Don't trigger if blocks aren't the same/empty
 define void @neg1() personality i32 (...)* @__gxx_personality_v0 {
-; CHECK-LABEL: @neg1
+; CHECK-LABEL: @neg1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    invoke void @fn()
+; CHECK-NEXT:    to label [[INVOKE2:%.*]] unwind label [[LPAD1:%.*]]
+; CHECK:       invoke2:
+; CHECK-NEXT:    invoke void @fn()
+; CHECK-NEXT:    to label [[INVOKE_CONT:%.*]] unwind label [[LPAD2:%.*]]
+; CHECK:       invoke.cont:
+; CHECK-NEXT:    ret void
+; CHECK:       lpad1:
+; CHECK-NEXT:    [[EXN:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    filter [0 x i8*] zeroinitializer
+; CHECK-NEXT:    call void @fn()
+; CHECK-NEXT:    br label [[SHARED_RESUME:%.*]]
+; CHECK:       lpad2:
+; CHECK-NEXT:    [[EXN2:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    br label [[SHARED_RESUME]]
+; CHECK:       shared_resume:
+; CHECK-NEXT:    call void @fn()
+; CHECK-NEXT:    ret void
+;
 entry:
-; CHECK-LABEL: entry:
-; CHECK: to label %invoke2 unwind label %lpad1
   invoke void @fn()
-    to label %invoke2 unwind label %lpad1
+  to label %invoke2 unwind label %lpad1
 
 invoke2:
-; CHECK-LABEL: invoke2:
-; CHECK: to label %invoke.cont unwind label %lpad2
   invoke void @fn()
-    to label %invoke.cont unwind label %lpad2
+  to label %invoke.cont unwind label %lpad2
 
 invoke.cont:
   ret void
 
 lpad1:
   %exn = landingpad {i8*, i32}
-         filter [0 x i8*] zeroinitializer
+  filter [0 x i8*] zeroinitializer
   call void @fn()
   br label %shared_resume
 
 lpad2:
   %exn2 = landingpad {i8*, i32}
-          cleanup
+  cleanup
   br label %shared_resume
 
 shared_resume:
@@ -78,30 +101,46 @@ shared_resume:
 
 ; Should not trigger when the landing pads are not the exact same
 define void @neg2() personality i32 (...)* @__gxx_personality_v0 {
-; CHECK-LABEL: @neg2
+; CHECK-LABEL: @neg2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    invoke void @fn()
+; CHECK-NEXT:    to label [[INVOKE2:%.*]] unwind label [[LPAD1:%.*]]
+; CHECK:       invoke2:
+; CHECK-NEXT:    invoke void @fn()
+; CHECK-NEXT:    to label [[INVOKE_CONT:%.*]] unwind label [[LPAD2:%.*]]
+; CHECK:       invoke.cont:
+; CHECK-NEXT:    ret void
+; CHECK:       lpad1:
+; CHECK-NEXT:    [[EXN:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    filter [0 x i8*] zeroinitializer
+; CHECK-NEXT:    br label [[SHARED_RESUME:%.*]]
+; CHECK:       lpad2:
+; CHECK-NEXT:    [[EXN2:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    br label [[SHARED_RESUME]]
+; CHECK:       shared_resume:
+; CHECK-NEXT:    call void @fn()
+; CHECK-NEXT:    ret void
+;
 entry:
-; CHECK-LABEL: entry:
-; CHECK: to label %invoke2 unwind label %lpad1
   invoke void @fn()
-    to label %invoke2 unwind label %lpad1
+  to label %invoke2 unwind label %lpad1
 
 invoke2:
-; CHECK-LABEL: invoke2:
-; CHECK: to label %invoke.cont unwind label %lpad2
   invoke void @fn()
-    to label %invoke.cont unwind label %lpad2
+  to label %invoke.cont unwind label %lpad2
 
 invoke.cont:
   ret void
 
 lpad1:
   %exn = landingpad {i8*, i32}
-         filter [0 x i8*] zeroinitializer
+  filter [0 x i8*] zeroinitializer
   br label %shared_resume
 
 lpad2:
   %exn2 = landingpad {i8*, i32}
-          cleanup
+  cleanup
   br label %shared_resume
 
 shared_resume:
diff --git a/llvm/test/Transforms/SimplifyCFG/merge-cond-stores.ll b/llvm/test/Transforms/SimplifyCFG/merge-cond-stores.ll
index 3439830d6265f..60a6ca0d6379d 100644
--- a/llvm/test/Transforms/SimplifyCFG/merge-cond-stores.ll
+++ b/llvm/test/Transforms/SimplifyCFG/merge-cond-stores.ll
@@ -42,9 +42,9 @@ end:
 define void @test_simple_commuted(i32* %p, i32 %a, i32 %b) {
 ; CHECK-LABEL: @test_simple_commuted(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X1:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[X1_NOT:%.*]] = icmp eq i32 [[A:%.*]], 0
 ; CHECK-NEXT:    [[X2:%.*]] = icmp eq i32 [[B:%.*]], 0
-; CHECK-NEXT:    [[TMP0:%.*]] = or i1 [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP0:%.*]] = or i1 [[X1_NOT]], [[X2]]
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[TMP1:%.*]], label [[TMP2:%.*]]
 ; CHECK:       1:
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = zext i1 [[X2]] to i32
@@ -274,13 +274,13 @@ define i32 @test_diamond_simple(i32* %p, i32* %q, i32 %a, i32 %b) {
 ; CHECK-NEXT:    [[Z2:%.*]] = select i1 [[X1]], i32 [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[X2:%.*]] = icmp eq i32 [[B]], 0
 ; CHECK-NEXT:    [[TMP0:%.*]] = or i32 [[A]], [[B]]
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 0
-; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP3:%.*]], label [[TMP2:%.*]]
-; CHECK:       2:
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[TMP2:%.*]], label [[TMP1:%.*]]
+; CHECK:       1:
 ; CHECK-NEXT:    [[SIMPLIFYCFG_MERGE:%.*]] = select i1 [[X2]], i32 [[Z2]], i32 1
 ; CHECK-NEXT:    store i32 [[SIMPLIFYCFG_MERGE]], i32* [[P:%.*]], align 4
-; CHECK-NEXT:    br label [[TMP3]]
-; CHECK:       3:
+; CHECK-NEXT:    br label [[TMP2]]
+; CHECK:       2:
 ; CHECK-NEXT:    [[Z4:%.*]] = select i1 [[X2]], i32 [[Z2]], i32 3
 ; CHECK-NEXT:    ret i32 [[Z4]]
 ;
diff --git a/llvm/test/Transforms/SimplifyCFG/pr46638.ll b/llvm/test/Transforms/SimplifyCFG/pr46638.ll
new file mode 100644
index 0000000000000..ba7ce88cf6ad2
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/pr46638.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+
+define void @pr46638(i1 %c, i32 %x) {
+; CHECK-LABEL: @pr46638(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[TRUE2_CRITEDGE:%.*]], label [[FALSE1:%.*]]
+; CHECK:       false1:
+; CHECK-NEXT:    call void @dummy(i32 1)
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[X]], 0
+; CHECK-NEXT:    [[EXT:%.*]] = zext i1 [[CMP2]] to i32
+; CHECK-NEXT:    call void @dummy(i32 [[EXT]])
+; CHECK-NEXT:    ret void
+; CHECK:       true2.critedge:
+; CHECK-NEXT:    [[CMP2_C:%.*]] = icmp sgt i32 [[X]], 0
+; CHECK-NEXT:    [[EXT_C:%.*]] = zext i1 [[CMP2_C]] to i32
+; CHECK-NEXT:    call void @dummy(i32 [[EXT_C]])
+; CHECK-NEXT:    call void @dummy(i32 2)
+; CHECK-NEXT:    ret void
+;
+  %cmp1 = icmp slt i32 %x, 0
+  call void @llvm.assume(i1 %cmp1)
+  br i1 %c, label %true1, label %false1
+
+true1:
+  %cmp2 = icmp sgt i32 %x, 0
+  %ext = zext i1 %cmp2 to i32
+  call void @dummy(i32 %ext)
+  br i1 %c, label %true2, label %false2
+
+false1:
+  call void @dummy(i32 1)
+  br label %true1
+
+true2:
+  call void @dummy(i32 2)
+  ret void
+
+false2:
+  ret void
+}
+
+declare void @dummy(i32)
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-dead-default.ll b/llvm/test/Transforms/SimplifyCFG/switch-dead-default.ll
index 7f713182fde7a..2ce04ec850605 100644
--- a/llvm/test/Transforms/SimplifyCFG/switch-dead-default.ll
+++ b/llvm/test/Transforms/SimplifyCFG/switch-dead-default.ll
@@ -91,7 +91,7 @@ define void @test3(i2 %a) {
 ; CHECK-NEXT:    call void @foo(i32 2)
 ; CHECK-NEXT:    ret void
 ; CHECK:       default:
-; CHECK-NEXT:    call void @foo(i32 0)
+; CHECK-NEXT:    call void @foo(i32 3)
 ; CHECK-NEXT:    ret void
 ;
   switch i2 %a, label %default [i2 0, label %case0
@@ -108,7 +108,7 @@ case2:
   call void @foo(i32 2)
   ret void
 default:
-  call void @foo(i32 0)
+  call void @foo(i32 3)
   ret void
 }
 
@@ -127,7 +127,7 @@ define void @test4(i128 %a) {
 ; CHECK-NEXT:    call void @foo(i32 1)
 ; CHECK-NEXT:    ret void
 ; CHECK:       default:
-; CHECK-NEXT:    call void @foo(i32 0)
+; CHECK-NEXT:    call void @foo(i32 2)
 ; CHECK-NEXT:    ret void
 ;
   switch i128 %a, label %default [i128 0, label %case0
@@ -140,7 +140,7 @@ case1:
   call void @foo(i32 1)
   ret void
 default:
-  call void @foo(i32 0)
+  call void @foo(i32 2)
   ret void
 }
 
diff --git a/llvm/test/Transforms/SimplifyCFG/unprofitable-pr.ll b/llvm/test/Transforms/SimplifyCFG/unprofitable-pr.ll
index 461003717797f..31ed7e203c45e 100644
--- a/llvm/test/Transforms/SimplifyCFG/unprofitable-pr.ll
+++ b/llvm/test/Transforms/SimplifyCFG/unprofitable-pr.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -simplifycfg -S < %s | FileCheck %s
-; RUN: opt -passes=simplify-cfg -S < %s | FileCheck %s
+; RUN: opt -simplifycfg -simplifycfg-max-small-block-size=10 -S < %s | FileCheck %s
+; RUN: opt -passes=simplify-cfg -simplifycfg-max-small-block-size=10 -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64-p5:32:32-A5"
 
@@ -50,14 +50,71 @@ false2:                                           ; preds = %true1
   ret void
 }
 
-; FIXME: SimplifyCFG is doing something weird here. It should have split the
-; blocks like in the test above, but instead it creates .pr Phi node which
-; only complicates things.
+; Corner case: the block has max possible size for which we still do PRE.
 define void @test_02(i1 %c, i64* align 1 %ptr) local_unnamed_addr #0 {
 ; CHECK-LABEL: @test_02(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[TRUE2_CRITEDGE:%.*]], label [[FALSE1:%.*]]
+; CHECK:       false1:
+; CHECK-NEXT:    store volatile i64 1, i64* [[PTR:%.*]], align 4
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i64* [[PTR]] to i64
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 7
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    store volatile i64 0, i64* [[PTR]], align 8
+; CHECK-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
+; CHECK-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
+; CHECK-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
+; CHECK-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
+; CHECK-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
+; CHECK-NEXT:    store volatile i64 3, i64* [[PTR]], align 8
+; CHECK-NEXT:    ret void
+; CHECK:       true2.critedge:
+; CHECK-NEXT:    [[PTRINT_C:%.*]] = ptrtoint i64* [[PTR]] to i64
+; CHECK-NEXT:    [[MASKEDPTR_C:%.*]] = and i64 [[PTRINT_C]], 7
+; CHECK-NEXT:    [[MASKCOND_C:%.*]] = icmp eq i64 [[MASKEDPTR_C]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND_C]])
+; CHECK-NEXT:    store volatile i64 0, i64* [[PTR]], align 8
+; CHECK-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
+; CHECK-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
+; CHECK-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
+; CHECK-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
+; CHECK-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
+; CHECK-NEXT:    store volatile i64 2, i64* [[PTR]], align 8
+; CHECK-NEXT:    ret void
+;
+  br i1 %c, label %true1, label %false1
+
+true1:                                            ; preds = %false1, %0
+  %ptrint = ptrtoint i64* %ptr to i64
+  %maskedptr = and i64 %ptrint, 7
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  store volatile i64 0, i64* %ptr, align 8
+  store volatile i64 -1, i64* %ptr, align 8
+  store volatile i64 -1, i64* %ptr, align 8
+  store volatile i64 -1, i64* %ptr, align 8
+  store volatile i64 -1, i64* %ptr, align 8
+  store volatile i64 -1, i64* %ptr, align 8
+  br i1 %c, label %true2, label %false2
+
+false1:                                           ; preds = %0
+  store volatile i64 1, i64* %ptr, align 4
+  br label %true1
+
+true2:                                            ; preds = %true1
+  store volatile i64 2, i64* %ptr, align 8
+  ret void
+
+false2:                                           ; preds = %true1
+  store volatile i64 3, i64* %ptr, align 8
+  ret void
+}
+
+; This block is too huge for PRE.
+define void @test_03(i1 %c, i64* align 1 %ptr) local_unnamed_addr #0 {
+; CHECK-LABEL: @test_03(
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[TRUE1:%.*]], label [[FALSE1:%.*]]
 ; CHECK:       true1:
-; CHECK-NEXT:    [[C_PR:%.*]] = phi i1 [ [[C]], [[FALSE1]] ], [ true, [[TMP0:%.*]] ]
 ; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint i64* [[PTR:%.*]] to i64
 ; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 7
 ; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
@@ -68,7 +125,8 @@ define void @test_02(i1 %c, i64* align 1 %ptr) local_unnamed_addr #0 {
 ; CHECK-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
 ; CHECK-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
 ; CHECK-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
-; CHECK-NEXT:    br i1 [[C_PR]], label [[TRUE2:%.*]], label [[FALSE2:%.*]]
+; CHECK-NEXT:    store volatile i64 -1, i64* [[PTR]], align 8
+; CHECK-NEXT:    br i1 [[C]], label [[TRUE2:%.*]], label [[FALSE2:%.*]]
 ; CHECK:       false1:
 ; CHECK-NEXT:    store volatile i64 1, i64* [[PTR]], align 4
 ; CHECK-NEXT:    br label [[TRUE1]]
@@ -92,6 +150,7 @@ true1:                                            ; preds = %false1, %0
   store volatile i64 -1, i64* %ptr, align 8
   store volatile i64 -1, i64* %ptr, align 8
   store volatile i64 -1, i64* %ptr, align 8
+  store volatile i64 -1, i64* %ptr, align 8
   br i1 %c, label %true2, label %false2
 
 false1:                                           ; preds = %0
diff --git a/llvm/test/Transforms/SimplifyCFG/wc-widen-block.ll b/llvm/test/Transforms/SimplifyCFG/wc-widen-block.ll
index e889d02a2819b..6e959bc37ed1b 100644
--- a/llvm/test/Transforms/SimplifyCFG/wc-widen-block.ll
+++ b/llvm/test/Transforms/SimplifyCFG/wc-widen-block.ll
@@ -11,7 +11,7 @@ define i32 @basic(i1 %cond_0, i32* %p) {
 ; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
 ; CHECK-NEXT:    ret i32 [[DEOPTRET]]
 ; CHECK:       guarded:
-; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    [[COND_1:%.*]] = icmp eq i32 [[V]], 0
 ; CHECK-NEXT:    br i1 [[COND_1]], label [[RETURN:%.*]], label [[DEOPT]], !prof !0
 ; CHECK:       return:
@@ -84,7 +84,7 @@ define i32 @basic_swapped_branch(i1 %cond_0, i32* %p) {
 ; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
 ; CHECK-NEXT:    ret i32 [[DEOPTRET]]
 ; CHECK:       guarded:
-; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    [[COND_1:%.*]] = icmp eq i32 [[V]], 0
 ; CHECK-NEXT:    br i1 [[COND_1]], label [[DEOPT]], label [[RETURN:%.*]], !prof !0
 ; CHECK:       return:
@@ -324,7 +324,7 @@ define i32 @neg_correlated(i1 %cond_0, i1 %cond_1, i32* %p) {
 ; CHECK-NEXT:    [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
 ; CHECK-NEXT:    ret i32 [[DEOPTRET2]]
 ; CHECK:       guarded2:
-; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    [[COND_2:%.*]] = icmp eq i32 [[V]], 0
 ; CHECK-NEXT:    br i1 [[COND_2]], label [[RETURN:%.*]], label [[DEOPT3:%.*]], !prof !0
 ; CHECK:       deopt3:
@@ -372,7 +372,7 @@ define i32 @trivial_wb(i1 %cond_0, i32* %p) {
 ; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
 ; CHECK-NEXT:    ret i32 [[DEOPTRET]]
 ; CHECK:       guarded:
-; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    [[COND_1:%.*]] = icmp eq i32 [[V]], 0
 ; CHECK-NEXT:    br i1 [[COND_1]], label [[RETURN:%.*]], label [[DEOPT]], !prof !0
 ; CHECK:       return:
@@ -410,7 +410,7 @@ define i32 @swapped_wb(i1 %cond_0, i32* %p) {
 ; CHECK-NEXT:    [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
 ; CHECK-NEXT:    ret i32 [[DEOPTRET]]
 ; CHECK:       guarded:
-; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    [[COND_1:%.*]] = icmp eq i32 [[V]], 0
 ; CHECK-NEXT:    br i1 [[COND_1]], label [[RETURN:%.*]], label [[DEOPT]], !prof !0
 ; CHECK:       return:
diff --git a/llvm/test/Transforms/Sink/badloadsink.ll b/llvm/test/Transforms/Sink/badloadsink.ll
index eb2b0fae482e1..e43fd08a6ec64 100644
--- a/llvm/test/Transforms/Sink/badloadsink.ll
+++ b/llvm/test/Transforms/Sink/badloadsink.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -sink -S | FileCheck %s
+; RUN: opt < %s -basic-aa -sink -S | FileCheck %s
 declare void @foo(i64 *)
 declare i8* @llvm.load.relative.i32(i8* %ptr, i32 %offset) argmemonly nounwind readonly
 define i64 @sinkload(i1 %cmp, i8* %ptr, i32 %off) {
diff --git a/llvm/test/Transforms/Sink/basic.ll b/llvm/test/Transforms/Sink/basic.ll
index 8ff41262014d9..2a51489711916 100644
--- a/llvm/test/Transforms/Sink/basic.ll
+++ b/llvm/test/Transforms/Sink/basic.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -sink -S | FileCheck %s
+; RUN: opt < %s -basic-aa -sink -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline='basic-aa' -passes='sink' -S | FileCheck %s
 
 @A = external global i32
diff --git a/llvm/test/Transforms/Sink/call.ll b/llvm/test/Transforms/Sink/call.ll
index 5aaad4499f59a..5d5e1e0164f9e 100644
--- a/llvm/test/Transforms/Sink/call.ll
+++ b/llvm/test/Transforms/Sink/call.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basicaa -sink -S | FileCheck %s
+; RUN: opt < %s -basic-aa -sink -S | FileCheck %s
 
 declare i32 @f_load_global() nounwind readonly
 declare i32 @f_load_arg(i32*) nounwind readonly argmemonly
diff --git a/llvm/test/Transforms/SpeculativeExecution/PR46267.ll b/llvm/test/Transforms/SpeculativeExecution/PR46267.ll
new file mode 100644
index 0000000000000..5c61b225a7f5d
--- /dev/null
+++ b/llvm/test/Transforms/SpeculativeExecution/PR46267.ll
@@ -0,0 +1,93 @@
+; RUN: opt < %s -S -speculative-execution | FileCheck %s
+; RUN: opt < %s -S -passes='speculative-execution' | FileCheck %s
+
+%class.B = type { i32 (...)** }
+
+; Testing that two bitcasts are not hoisted to the first BB
+define i8* @foo(%class.B* readonly %b) {
+; CHECK-LABEL: foo
+; CHECK-LABEL: entry
+; CHECK-NEXT: %i = icmp eq %class.B* %b, null
+; CHECK-NEXT: br i1 %i, label %end, label %notnull
+entry:
+  %i = icmp eq %class.B* %b, null
+  br i1 %i, label %end, label %notnull
+
+; CHECK-LABEL: notnull:
+; CHECK-NEXT: %i1 = bitcast %class.B* %b to i32**
+; CHECK: %i3 = bitcast %class.B* %b to i8*
+notnull:                             ; preds = %entry
+  %i1 = bitcast %class.B* %b to i32**
+  %vtable = load i32*, i32** %i1, align 8
+  %i2 = getelementptr inbounds i32, i32* %vtable, i64 -2
+  %offset.to.top = load i32, i32* %i2, align 4
+  %i3 = bitcast %class.B* %b to i8*
+  %i4 = sext i32 %offset.to.top to i64
+  %i5 = getelementptr inbounds i8, i8* %i3, i64 %i4
+  br label %end
+
+end:                                 ; preds = %notnull, %entry
+  %i6 = phi i8* [ %i5, %notnull ], [ null, %entry ]
+  ret i8* %i6
+}
+
+define void @f(i32 %i) {
+entry:
+; CHECK-LABEL: @f(
+; CHECK:  %a2 = add i32 %i, 0
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i32 %a2
+  br i1 undef, label %land.rhs, label %land.end
+
+land.rhs:                                         ; preds = %entry
+; CHECK: land.rhs:
+; CHECK-NEXT: call void @llvm.dbg.label
+; CHECK-NEXT: %x = alloca i32, align 4
+; CHECK-NEXT: call void @llvm.dbg.addr(metadata i32* %x
+; CHECK-NEXT: %y = alloca i32, align 4
+; CHECK-NEXT: call void @llvm.dbg.declare(metadata i32* %y
+; CHECK-NEXT: %a0 = load i32, i32* undef, align 1
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 %a0
+  call void @llvm.dbg.label(metadata !11), !dbg !10
+  %x = alloca i32, align 4
+  call void @llvm.dbg.addr(metadata i32* %x, metadata !12, metadata !DIExpression()), !dbg !10
+  %y = alloca i32, align 4
+  call void @llvm.dbg.declare(metadata i32* %y, metadata !14, metadata !DIExpression()), !dbg !10
+
+  %a0 = load i32, i32* undef, align 1
+  call void @llvm.dbg.value(metadata i32 %a0, metadata !9, metadata !DIExpression()), !dbg !10
+
+  %a2 = add i32 %i, 0
+  call void @llvm.dbg.value(metadata i32 %a2, metadata !13, metadata !DIExpression()), !dbg !10
+
+  br label %land.end
+
+land.end:                                         ; preds = %land.rhs, %entry
+  ret void
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+declare void @llvm.dbg.label(metadata)
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+declare void @llvm.dbg.addr(metadata, metadata, metadata)
+
+attributes #1 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "foo.c", directory: "/bar")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIBasicType(name: "int", size: 16, encoding: DW_ATE_signed)
+!5 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !7, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !8)
+!8 = !{null}
+!9 = !DILocalVariable(name: "a0", scope: !6, file: !1, line: 3, type: !4)
+!10 = !DILocation(line: 0, scope: !6)
+!11 = !DILabel(scope: !6, name: "label", file: !1, line: 1)
+!12 = !DILocalVariable(name: "x", scope: !6, file: !1, line: 3, type: !4)
+!13 = !DILocalVariable(name: "a2", scope: !6, file: !1, line: 3, type: !4)
+!14 = !DILocalVariable(name: "y", scope: !6, file: !1, line: 3, type: !4)
diff --git a/llvm/test/Transforms/Util/PredicateInfo/branch-on-same-cond.ll b/llvm/test/Transforms/Util/PredicateInfo/branch-on-same-cond.ll
new file mode 100644
index 0000000000000..7cf52d1bed3c7
--- /dev/null
+++ b/llvm/test/Transforms/Util/PredicateInfo/branch-on-same-cond.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -print-predicateinfo < %s 2>&1 >/dev/null | FileCheck %s
+
+; FIXME:  RenamedOp should be %cmp or %x in all cases here,
+; which is the value used in the condition.
+define i32 @test(i32 %x) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK: RenamedOp: [[CMP]]
+; CHECK:         [[CMP_0:%.*]] = call i1 @llvm.ssa.copy.{{.*}}(i1 [[CMP]])
+; CHECK: RenamedOp: [[X]]
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.*}}(i32 [[X]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB2:%.*]], label [[EXIT1:%.*]]
+; CHECK:       bb2:
+; CHECK: RenamedOp: [[CMP_0]]
+; CHECK:         [[CMP_0_1:%.*]] = call i1 @llvm.ssa.copy.{{.*}}(i1 [[CMP_0]])
+; CHECK: RenamedOp: [[X]]
+; CHECK:         [[X_0_1:%.*]] = call i32 @llvm.ssa.copy.{{.*}}(i32 [[X_0]])
+; CHECK: RenamedOp: [[X_0]]
+; CHECK:         [[X_0_4:%.*]] = call i32 @llvm.ssa.copy.{{.*}}(i32 [[X_0]])
+; CHECK-NEXT:    br i1 [[CMP_0]], label [[BB3:%.*]], label [[EXIT2:%.*]]
+; CHECK:       bb3:
+; CHECK: RenamedOp: [[X]]
+; CHECK:         [[X_0_1_2:%.*]] = call i32 @llvm.ssa.copy.{{.*}}(i32 [[X_0_1]])
+; CHECK: RenamedOp: [[X_0_1]]
+; CHECK:         [[X_0_1_3:%.*]] = call i32 @llvm.ssa.copy.{{.*}}(i32 [[X_0_1]])
+; CHECK-NEXT:    br i1 [[CMP_0_1]], label [[EXIT3:%.*]], label [[EXIT4:%.*]]
+; CHECK:       exit1:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       exit2:
+; CHECK-NEXT:    ret i32 [[X_0_4]]
+; CHECK:       exit3:
+; CHECK-NEXT:    ret i32 [[X_0_1_2]]
+; CHECK:       exit4:
+; CHECK-NEXT:    ret i32 [[X_0_1_3]]
+;
+entry:
+  br label %bb1
+
+bb1:
+  %cmp = icmp eq i32 %x, 0
+  br i1 %cmp, label %bb2, label %exit1
+
+bb2:
+  br i1 %cmp, label %bb3, label %exit2
+
+bb3:
+  br i1 %cmp, label %exit3, label %exit4
+
+exit1:
+  ret i32 0
+
+exit2:
+  ret i32 %x
+
+exit3:
+  ret i32 %x
+
+exit4:
+  ret i32 %x
+}
diff --git a/llvm/test/Transforms/Util/PredicateInfo/condprop.ll b/llvm/test/Transforms/Util/PredicateInfo/condprop.ll
index daf6bb8b40a8a..756457ab7fa9a 100644
--- a/llvm/test/Transforms/Util/PredicateInfo/condprop.ll
+++ b/llvm/test/Transforms/Util/PredicateInfo/condprop.ll
@@ -138,7 +138,7 @@ define void @test4(i1 %b, i32 %x) {
 ; CHECK-NEXT:    i32 2, label [[CASE0]]
 ; CHECK-NEXT:    i32 3, label [[CASE3]]
 ; CHECK-NEXT:    i32 4, label [[DEFAULT:%.*]]
-; CHECK-NEXT:    ] Edge: [label [[SW]],label %case1] }
+; CHECK-NEXT:    ] Edge: [label [[SW]],label %case1]
 ; CHECK-NEXT:    [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X:%.*]])
 ; CHECK-NEXT:    switch i32 [[X]], label [[DEFAULT]] [
 ; CHECK-NEXT:    i32 0, label [[CASE0]]
diff --git a/llvm/test/Transforms/Util/PredicateInfo/unnamed-types.ll b/llvm/test/Transforms/Util/PredicateInfo/unnamed-types.ll
index 21e702178fd72..d1e0f358fc9fc 100644
--- a/llvm/test/Transforms/Util/PredicateInfo/unnamed-types.ll
+++ b/llvm/test/Transforms/Util/PredicateInfo/unnamed-types.ll
@@ -7,12 +7,12 @@
 
 ; CHECK-LABEL: bb:
 ; CHECK: Has predicate info
-; CHECK: branch predicate info { TrueEdge: 1 Comparison:  %cmp1 = icmp ne %0* %arg, null Edge: [label %bb,label %bb1] }
+; CHECK: branch predicate info { TrueEdge: 1 Comparison:  %cmp1 = icmp ne %0* %arg, null Edge: [label %bb,label %bb1], RenamedOp: %arg }
 ; CHECK-NEXT:  %arg.0 = call %0* @llvm.ssa.copy.{{.+}}(%0* %arg)
 
 ; CHECK-LABEL: bb1:
 ; CHECK: Has predicate info
-; CHECK-NEXT: branch predicate info { TrueEdge: 0 Comparison:  %cmp2 = icmp ne %1* null, %tmp Edge: [label %bb1,label %bb3] }
+; CHECK-NEXT: branch predicate info { TrueEdge: 0 Comparison:  %cmp2 = icmp ne %1* null, %tmp Edge: [label %bb1,label %bb3], RenamedOp: %tmp }
 ; CHECK-NEXT: %tmp.0 = call %1* @llvm.ssa.copy.{{.+}}(%1* %tmp)
 
 define void @f0(%0* %arg, %1* %tmp) {
diff --git a/llvm/test/Transforms/Util/assume-builder.ll b/llvm/test/Transforms/Util/assume-builder.ll
index 33cc3ff3c985e..991768655ada3 100644
--- a/llvm/test/Transforms/Util/assume-builder.ll
+++ b/llvm/test/Transforms/Util/assume-builder.ll
@@ -1,8 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
+; RUN: opt -assume-builder -verify --enable-knowledge-retention -S %s | FileCheck %s --check-prefixes=BASIC
+; RUN: opt -assume-builder -verify --enable-knowledge-retention --assume-preserve-all -S %s | FileCheck %s --check-prefixes=ALL
+; RUN: opt -assumption-cache-tracker -assume-builder -verify --enable-knowledge-retention -S %s | FileCheck %s --check-prefixes=WITH-AC
+; RUN: opt -domtree -assumption-cache-tracker -assume-builder -verify --enable-knowledge-retention -S %s | FileCheck %s --check-prefixes=CROSS-BLOCK,CROSS-BLOCK-OLDPM
+; RUN: opt -assume-builder -domtree -assumption-cache-tracker -assume-simplify -verify --enable-knowledge-retention -S %s | FileCheck %s --check-prefixes=FULL-SIMPLIFY
+
 ; RUN: opt -passes='assume-builder,verify' --enable-knowledge-retention -S %s | FileCheck %s --check-prefixes=BASIC
 ; RUN: opt -passes='assume-builder,verify' --enable-knowledge-retention --assume-preserve-all -S %s | FileCheck %s --check-prefixes=ALL
 ; RUN: opt -passes='require<assumptions>,assume-builder,verify' --enable-knowledge-retention -S %s | FileCheck %s --check-prefixes=WITH-AC
-; RUN: opt -passes='require<domtree>,require<assumptions>,assume-builder,verify' --enable-knowledge-retention -S %s | FileCheck %s --check-prefixes=CROSS-BLOCK
+; RUN: opt -passes='require<domtree>,require<assumptions>,assume-builder,verify' --enable-knowledge-retention -S %s | FileCheck %s --check-prefixes=CROSS-BLOCK,CROSS-BLOCK-NEWMP
 ; RUN: opt -passes='assume-builder,require<domtree>,require<assumptions>,assume-simplify,verify' --enable-knowledge-retention -S %s | FileCheck %s --check-prefixes=FULL-SIMPLIFY
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
@@ -641,28 +647,30 @@ define dso_local i32 @_Z6squarePi(i32* %P, i32* %P1, i1 %cond) {
 ; WITH-AC-NEXT:    store i32 0, i32* [[P1]], align 4
 ; WITH-AC-NEXT:    ret i32 0
 ;
-; CROSS-BLOCK-LABEL: define {{[^@]+}}@_Z6squarePi
-; CROSS-BLOCK-SAME: (i32* [[P:%.*]], i32* [[P1:%.*]], i1 [[COND:%.*]])
-; CROSS-BLOCK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]), "align"(i32* [[P]], i64 4) ]
-; CROSS-BLOCK-NEXT:    store i32 0, i32* [[P]], align 4
-; CROSS-BLOCK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P1]], i64 4), "nonnull"(i32* [[P1]]), "align"(i32* [[P1]], i64 8) ]
-; CROSS-BLOCK-NEXT:    store i32 0, i32* [[P1]], align 8
-; CROSS-BLOCK-NEXT:    br i1 [[COND]], label [[A:%.*]], label [[B:%.*]]
-; CROSS-BLOCK:       A:
-; CROSS-BLOCK-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[P]], i64 8) ]
-; CROSS-BLOCK-NEXT:    store i32 0, i32* [[P]], align 8
-; CROSS-BLOCK-NEXT:    store i32 0, i32* [[P1]], align 4
-; CROSS-BLOCK-NEXT:    br i1 [[COND]], label [[C:%.*]], label [[B]]
-; CROSS-BLOCK:       B:
-; CROSS-BLOCK-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[P]], i64 8) ]
-; CROSS-BLOCK-NEXT:    store i32 0, i32* [[P]], align 8
-; CROSS-BLOCK-NEXT:    store i32 0, i32* [[P1]], align 8
-; CROSS-BLOCK-NEXT:    br label [[C]]
-; CROSS-BLOCK:       C:
-; CROSS-BLOCK-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[P]], i64 32) ]
-; CROSS-BLOCK-NEXT:    store i32 0, i32* [[P]], align 32
-; CROSS-BLOCK-NEXT:    store i32 0, i32* [[P1]], align 4
-; CROSS-BLOCK-NEXT:    ret i32 0
+; CROSS-BLOCK-OLDPM-LABEL: define {{[^@]+}}@_Z6squarePi
+; CROSS-BLOCK-OLDPM-SAME: (i32* [[P:%.*]], i32* [[P1:%.*]], i1 [[COND:%.*]])
+; CROSS-BLOCK-OLDPM-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]), "align"(i32* [[P]], i64 4) ]
+; CROSS-BLOCK-OLDPM-NEXT:    store i32 0, i32* [[P]], align 4
+; CROSS-BLOCK-OLDPM-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P1]], i64 4), "nonnull"(i32* [[P1]]), "align"(i32* [[P1]], i64 8) ]
+; CROSS-BLOCK-OLDPM-NEXT:    store i32 0, i32* [[P1]], align 8
+; CROSS-BLOCK-OLDPM-NEXT:    br i1 [[COND]], label [[A:%.*]], label [[B:%.*]]
+; CROSS-BLOCK-OLDPM:       A:
+; CROSS-BLOCK-OLDPM-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[P]], i64 8) ]
+; CROSS-BLOCK-OLDPM-NEXT:    store i32 0, i32* [[P]], align 8
+; CROSS-BLOCK-OLDPM-NEXT:    store i32 0, i32* [[P1]], align 4
+; CROSS-BLOCK-OLDPM-NEXT:    br i1 [[COND]], label [[C:%.*]], label [[B]]
+; CROSS-BLOCK-OLDPM:       B:
+; CROSS-BLOCK-OLDPM-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]), "align"(i32* [[P]], i64 8) ]
+; CROSS-BLOCK-OLDPM-NEXT:    store i32 0, i32* [[P]], align 8
+; CROSS-BLOCK-OLDPM-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P1]], i64 4), "nonnull"(i32* [[P1]]), "align"(i32* [[P1]], i64 8) ]
+; CROSS-BLOCK-OLDPM-NEXT:    store i32 0, i32* [[P1]], align 8
+; CROSS-BLOCK-OLDPM-NEXT:    br label [[C]]
+; CROSS-BLOCK-OLDPM:       C:
+; CROSS-BLOCK-OLDPM-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]), "align"(i32* [[P]], i64 32) ]
+; CROSS-BLOCK-OLDPM-NEXT:    store i32 0, i32* [[P]], align 32
+; CROSS-BLOCK-OLDPM-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P1]], i64 4), "nonnull"(i32* [[P1]]), "align"(i32* [[P1]], i64 4) ]
+; CROSS-BLOCK-OLDPM-NEXT:    store i32 0, i32* [[P1]], align 4
+; CROSS-BLOCK-OLDPM-NEXT:    ret i32 0
 ;
 ; FULL-SIMPLIFY-LABEL: define {{[^@]+}}@_Z6squarePi
 ; FULL-SIMPLIFY-SAME: (i32* nonnull align 4 dereferenceable(4) [[P:%.*]], i32* nonnull align 8 dereferenceable(4) [[P1:%.*]], i1 [[COND:%.*]])
@@ -684,6 +692,29 @@ define dso_local i32 @_Z6squarePi(i32* %P, i32* %P1, i1 %cond) {
 ; FULL-SIMPLIFY-NEXT:    store i32 0, i32* [[P]], align 32
 ; FULL-SIMPLIFY-NEXT:    store i32 0, i32* [[P1]], align 4
 ; FULL-SIMPLIFY-NEXT:    ret i32 0
+;
+; CROSS-BLOCK-NEWMP-LABEL: define {{[^@]+}}@_Z6squarePi
+; CROSS-BLOCK-NEWMP-SAME: (i32* [[P:%.*]], i32* [[P1:%.*]], i1 [[COND:%.*]])
+; CROSS-BLOCK-NEWMP-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]), "align"(i32* [[P]], i64 4) ]
+; CROSS-BLOCK-NEWMP-NEXT:    store i32 0, i32* [[P]], align 4
+; CROSS-BLOCK-NEWMP-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P1]], i64 4), "nonnull"(i32* [[P1]]), "align"(i32* [[P1]], i64 8) ]
+; CROSS-BLOCK-NEWMP-NEXT:    store i32 0, i32* [[P1]], align 8
+; CROSS-BLOCK-NEWMP-NEXT:    br i1 [[COND]], label [[A:%.*]], label [[B:%.*]]
+; CROSS-BLOCK-NEWMP:       A:
+; CROSS-BLOCK-NEWMP-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[P]], i64 8) ]
+; CROSS-BLOCK-NEWMP-NEXT:    store i32 0, i32* [[P]], align 8
+; CROSS-BLOCK-NEWMP-NEXT:    store i32 0, i32* [[P1]], align 4
+; CROSS-BLOCK-NEWMP-NEXT:    br i1 [[COND]], label [[C:%.*]], label [[B]]
+; CROSS-BLOCK-NEWMP:       B:
+; CROSS-BLOCK-NEWMP-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[P]], i64 8) ]
+; CROSS-BLOCK-NEWMP-NEXT:    store i32 0, i32* [[P]], align 8
+; CROSS-BLOCK-NEWMP-NEXT:    store i32 0, i32* [[P1]], align 8
+; CROSS-BLOCK-NEWMP-NEXT:    br label [[C]]
+; CROSS-BLOCK-NEWMP:       C:
+; CROSS-BLOCK-NEWMP-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[P]], i64 32) ]
+; CROSS-BLOCK-NEWMP-NEXT:    store i32 0, i32* [[P]], align 32
+; CROSS-BLOCK-NEWMP-NEXT:    store i32 0, i32* [[P1]], align 4
+; CROSS-BLOCK-NEWMP-NEXT:    ret i32 0
 ;
   store i32 0, i32* %P, align 4
   store i32 0, i32* %P1, align 8
diff --git a/llvm/test/Transforms/Util/assume-simplify.ll b/llvm/test/Transforms/Util/assume-simplify.ll
index 3005de3688e32..5f5eece453ac0 100644
--- a/llvm/test/Transforms/Util/assume-simplify.ll
+++ b/llvm/test/Transforms/Util/assume-simplify.ll
@@ -1,4 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
+; RUN: opt -domtree -assumption-cache-tracker -assume-simplify -verify --enable-knowledge-retention -S %s | FileCheck %s
 ; RUN: opt -passes='require<domtree>,require<assumptions>,assume-simplify,verify' --enable-knowledge-retention -S %s | FileCheck %s
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/Util/combine-alias-scope-metadata.ll b/llvm/test/Transforms/Util/combine-alias-scope-metadata.ll
index f58e4f783b03f..cb6ff700edc21 100644
--- a/llvm/test/Transforms/Util/combine-alias-scope-metadata.ll
+++ b/llvm/test/Transforms/Util/combine-alias-scope-metadata.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -S -basicaa -memcpyopt | FileCheck %s
+; RUN: opt < %s -S -basic-aa -memcpyopt | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define void @test(i8* noalias dereferenceable(1) %in, i8* noalias dereferenceable(1) %out) {
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll b/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
index e4f18f6083ac0..ac7ca1e8f8e21 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
@@ -10,8 +10,8 @@ declare void @use_f32(float)
 define i8 @ext0_ext0_add(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK-LABEL: @ext0_ext0_add(
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <16 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
-; CHECK-NEXT:    ret i8 [[TMP2]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
+; CHECK-NEXT:    ret i8 [[R]]
 ;
   %e0 = extractelement <16 x i8> %x, i32 0
   %e1 = extractelement <16 x i8> %y, i32 0
@@ -24,8 +24,8 @@ define i8 @ext0_ext0_add(<16 x i8> %x, <16 x i8> %y) {
 define i8 @ext1_ext1_add_flags(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK-LABEL: @ext1_ext1_add_flags(
 ; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw <16 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1
-; CHECK-NEXT:    ret i8 [[TMP2]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1
+; CHECK-NEXT:    ret i8 [[R]]
 ;
   %e0 = extractelement <16 x i8> %x, i32 1
   %e1 = extractelement <16 x i8> %y, i32 1
@@ -85,8 +85,8 @@ define i8 @ext0_ext0_sdiv(<16 x i8> %x, <16 x i8> %y) {
 define double @ext0_ext0_fadd(<2 x double> %x, <2 x double> %y) {
 ; CHECK-LABEL: @ext0_ext0_fadd(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fadd <2 x double> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
-; CHECK-NEXT:    ret double [[TMP2]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; CHECK-NEXT:    ret double [[R]]
 ;
   %e0 = extractelement <2 x double> %x, i32 0
   %e1 = extractelement <2 x double> %y, i32 0
@@ -99,8 +99,8 @@ define double @ext0_ext0_fadd(<2 x double> %x, <2 x double> %y) {
 define double @ext1_ext1_fsub(<2 x double> %x, <2 x double> %y) {
 ; CHECK-LABEL: @ext1_ext1_fsub(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast <2 x double> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
-; CHECK-NEXT:    ret double [[TMP2]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; CHECK-NEXT:    ret double [[R]]
 ;
   %e0 = extractelement <2 x double> %x, i32 1
   %e1 = extractelement <2 x double> %y, i32 1
@@ -129,8 +129,8 @@ define double @ext1_ext1_fadd_different_types(<2 x double> %x, <4 x double> %y)
 define i32 @ext1_ext1_add_same_vec(<4 x i32> %x) {
 ; CHECK-LABEL: @ext1_ext1_add_same_vec(
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[X:%.*]], [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT:    ret i32 [[TMP2]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
+; CHECK-NEXT:    ret i32 [[R]]
 ;
   %e0 = extractelement <4 x i32> %x, i32 1
   %e1 = extractelement <4 x i32> %x, i32 1
@@ -143,8 +143,8 @@ define i32 @ext1_ext1_add_same_vec(<4 x i32> %x) {
 define i32 @ext1_ext1_add_same_vec_cse(<4 x i32> %x) {
 ; CHECK-LABEL: @ext1_ext1_add_same_vec_cse(
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[X:%.*]], [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT:    ret i32 [[TMP2]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
+; CHECK-NEXT:    ret i32 [[R]]
 ;
   %e0 = extractelement <4 x i32> %x, i32 1
   %r = add i32 %e0, %e0
@@ -156,8 +156,8 @@ define i32 @ext1_ext1_add_same_vec_cse(<4 x i32> %x) {
 define i32 @ext1_ext1_add_same_vec_diff_idx_ty(<4 x i32> %x) {
 ; CHECK-LABEL: @ext1_ext1_add_same_vec_diff_idx_ty(
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[X:%.*]], [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT:    ret i32 [[TMP2]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
+; CHECK-NEXT:    ret i32 [[R]]
 ;
   %e0 = extractelement <4 x i32> %x, i32 1
   %e1 = extractelement <4 x i32> %x, i64 1
@@ -224,8 +224,8 @@ define i8 @ext1_ext1_add_uses1(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK-NEXT:    [[E0:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 0
 ; CHECK-NEXT:    call void @use_i8(i8 [[E0]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <16 x i8> [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
-; CHECK-NEXT:    ret i8 [[TMP2]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
+; CHECK-NEXT:    ret i8 [[R]]
 ;
   %e0 = extractelement <16 x i8> %x, i32 0
   call void @use_i8(i8 %e0)
@@ -241,8 +241,8 @@ define i8 @ext1_ext1_add_uses2(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK-NEXT:    [[E1:%.*]] = extractelement <16 x i8> [[Y:%.*]], i32 0
 ; CHECK-NEXT:    call void @use_i8(i8 [[E1]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <16 x i8> [[X:%.*]], [[Y]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
-; CHECK-NEXT:    ret i8 [[TMP2]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
+; CHECK-NEXT:    ret i8 [[R]]
 ;
   %e0 = extractelement <16 x i8> %x, i32 0
   %e1 = extractelement <16 x i8> %y, i32 0
@@ -259,10 +259,10 @@ define i8 @ext0_ext1_add(<16 x i8> %x, <16 x i8> %y) {
 ; SSE-NEXT:    ret i8 [[R]]
 ;
 ; AVX-LABEL: @ext0_ext1_add(
-; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[TMP2:%.*]] = add nuw <16 x i8> [[X:%.*]], [[TMP1]]
-; AVX-NEXT:    [[TMP3:%.*]] = extractelement <16 x i8> [[TMP2]], i32 0
-; AVX-NEXT:    ret i8 [[TMP3]]
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[TMP1:%.*]] = add nuw <16 x i8> [[X:%.*]], [[SHIFT]]
+; AVX-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
+; AVX-NEXT:    ret i8 [[R]]
 ;
   %e0 = extractelement <16 x i8> %x, i32 0
   %e1 = extractelement <16 x i8> %y, i32 1
@@ -278,10 +278,10 @@ define i8 @ext5_ext0_add(<16 x i8> %x, <16 x i8> %y) {
 ; SSE-NEXT:    ret i8 [[R]]
 ;
 ; AVX-LABEL: @ext5_ext0_add(
-; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[X:%.*]], <16 x i8> undef, <16 x i32> <i32 5, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[TMP2:%.*]] = sub nsw <16 x i8> [[TMP1]], [[Y:%.*]]
-; AVX-NEXT:    [[TMP3:%.*]] = extractelement <16 x i8> [[TMP2]], i64 0
-; AVX-NEXT:    ret i8 [[TMP3]]
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[X:%.*]], <16 x i8> undef, <16 x i32> <i32 5, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[TMP1:%.*]] = sub nsw <16 x i8> [[SHIFT]], [[Y:%.*]]
+; AVX-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i64 0
+; AVX-NEXT:    ret i8 [[R]]
 ;
   %e0 = extractelement <16 x i8> %x, i32 5
   %e1 = extractelement <16 x i8> %y, i32 0
@@ -297,10 +297,10 @@ define i8 @ext1_ext6_add(<16 x i8> %x, <16 x i8> %y) {
 ; SSE-NEXT:    ret i8 [[R]]
 ;
 ; AVX-LABEL: @ext1_ext6_add(
-; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> undef, <16 x i32> <i32 undef, i32 6, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[TMP2:%.*]] = and <16 x i8> [[X:%.*]], [[TMP1]]
-; AVX-NEXT:    [[TMP3:%.*]] = extractelement <16 x i8> [[TMP2]], i32 1
-; AVX-NEXT:    ret i8 [[TMP3]]
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> undef, <16 x i32> <i32 undef, i32 6, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[TMP1:%.*]] = and <16 x i8> [[X:%.*]], [[SHIFT]]
+; AVX-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1
+; AVX-NEXT:    ret i8 [[R]]
 ;
   %e0 = extractelement <16 x i8> %x, i32 1
   %e1 = extractelement <16 x i8> %y, i32 6
@@ -310,10 +310,10 @@ define i8 @ext1_ext6_add(<16 x i8> %x, <16 x i8> %y) {
 
 define float @ext1_ext0_fmul(<4 x float> %x) {
 ; CHECK-LABEL: @ext1_ext0_fmul(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = fmul <4 x float> [[TMP1]], [[X]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i64 0
-; CHECK-NEXT:    ret float [[TMP3]]
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[SHIFT]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
+; CHECK-NEXT:    ret float [[R]]
 ;
   %e0 = extractelement <4 x float> %x, i32 1
   %e1 = extractelement <4 x float> %x, i32 0
@@ -325,10 +325,10 @@ define float @ext0_ext3_fmul_extra_use1(<4 x float> %x) {
 ; CHECK-LABEL: @ext0_ext3_fmul_extra_use1(
 ; CHECK-NEXT:    [[E0:%.*]] = extractelement <4 x float> [[X:%.*]], i32 0
 ; CHECK-NEXT:    call void @use_f32(float [[E0]])
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[X]], <4 x float> undef, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = fmul nnan <4 x float> [[X]], [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
-; CHECK-NEXT:    ret float [[TMP3]]
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[X]], <4 x float> undef, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul nnan <4 x float> [[X]], [[SHIFT]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    ret float [[R]]
 ;
   %e0 = extractelement <4 x float> %x, i32 0
   call void @use_f32(float %e0)
@@ -360,10 +360,10 @@ define float @ext0_ext4_fmul_v8f32(<8 x float> %x) {
 ; SSE-NEXT:    ret float [[R]]
 ;
 ; AVX-LABEL: @ext0_ext4_fmul_v8f32(
-; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[X:%.*]], <8 x float> undef, <8 x i32> <i32 4, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[TMP2:%.*]] = fadd <8 x float> [[X]], [[TMP1]]
-; AVX-NEXT:    [[TMP3:%.*]] = extractelement <8 x float> [[TMP2]], i32 0
-; AVX-NEXT:    ret float [[TMP3]]
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x float> [[X:%.*]], <8 x float> undef, <8 x i32> <i32 4, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[TMP1:%.*]] = fadd <8 x float> [[X]], [[SHIFT]]
+; AVX-NEXT:    [[R:%.*]] = extractelement <8 x float> [[TMP1]], i32 0
+; AVX-NEXT:    ret float [[R]]
 ;
   %e0 = extractelement <8 x float> %x, i32 0
   %e1 = extractelement <8 x float> %x, i32 4
@@ -379,10 +379,10 @@ define float @ext7_ext4_fmul_v8f32(<8 x float> %x) {
 ; SSE-NEXT:    ret float [[R]]
 ;
 ; AVX-LABEL: @ext7_ext4_fmul_v8f32(
-; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[X:%.*]], <8 x float> undef, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 7, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[TMP2:%.*]] = fadd <8 x float> [[TMP1]], [[X]]
-; AVX-NEXT:    [[TMP3:%.*]] = extractelement <8 x float> [[TMP2]], i64 4
-; AVX-NEXT:    ret float [[TMP3]]
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x float> [[X:%.*]], <8 x float> undef, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 7, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[TMP1:%.*]] = fadd <8 x float> [[SHIFT]], [[X]]
+; AVX-NEXT:    [[R:%.*]] = extractelement <8 x float> [[TMP1]], i64 4
+; AVX-NEXT:    ret float [[R]]
 ;
   %e0 = extractelement <8 x float> %x, i32 7
   %e1 = extractelement <8 x float> %x, i32 4
@@ -418,10 +418,10 @@ define float @ext14_ext15_fmul_v16f32(<16 x float> %x) {
 
 define <4 x float> @ins_bo_ext_ext(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @ins_bo_ext_ext(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
-; CHECK-NEXT:    [[TMP2:%.*]] = fadd <4 x float> [[TMP1]], [[A]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i64 3
-; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[B:%.*]], float [[TMP3]], i32 3
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[SHIFT]], [[A]]
+; CHECK-NEXT:    [[A23:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[B:%.*]], float [[A23]], i32 3
 ; CHECK-NEXT:    ret <4 x float> [[V3]]
 ;
   %a2 = extractelement <4 x float> %a, i32 2
@@ -436,11 +436,11 @@ define <4 x float> @ins_bo_ext_ext(<4 x float> %a, <4 x float> %b) {
 
 define <4 x float> @ins_bo_ext_ext_uses(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @ins_bo_ext_ext_uses(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = fadd <4 x float> [[A]], [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
-; CHECK-NEXT:    call void @use_f32(float [[TMP3]])
-; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[B:%.*]], float [[TMP3]], i32 3
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A]], [[SHIFT]]
+; CHECK-NEXT:    [[A23:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    call void @use_f32(float [[A23]])
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[B:%.*]], float [[A23]], i32 3
 ; CHECK-NEXT:    ret <4 x float> [[V3]]
 ;
   %a2 = extractelement <4 x float> %a, i32 2
@@ -453,18 +453,18 @@ define <4 x float> @ins_bo_ext_ext_uses(<4 x float> %a, <4 x float> %b) {
 
 define <4 x float> @PR34724(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @PR34724(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[B]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
-; CHECK-NEXT:    [[TMP4:%.*]] = fadd <4 x float> [[A]], [[TMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP4]], i32 2
-; CHECK-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[B]], [[TMP2]]
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP6]], i32 0
-; CHECK-NEXT:    [[TMP8:%.*]] = fadd <4 x float> [[TMP3]], [[B]]
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x float> [[TMP8]], i64 3
-; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x float> undef, float [[TMP5]], i32 1
-; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[TMP7]], i32 2
-; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[TMP9]], i32 3
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A]], [[SHIFT]]
+; CHECK-NEXT:    [[A23:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <4 x float> [[B]], [[SHIFT1]]
+; CHECK-NEXT:    [[B01:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x float> [[B]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <4 x float> [[SHIFT2]], [[B]]
+; CHECK-NEXT:    [[B23:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x float> undef, float [[A23]], i32 1
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[B01]], i32 2
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[B23]], i32 3
 ; CHECK-NEXT:    ret <4 x float> [[V3]]
 ;
   %a0 = extractelement <4 x float> %a, i32 0
@@ -490,14 +490,14 @@ define <4 x float> @PR34724(<4 x float> %a, <4 x float> %b) {
 define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @ext_ext_or_reduction_v4i32(
 ; CHECK-NEXT:    [[Z:%.*]] = and <4 x i32> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i32> [[Z]], [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP4:%.*]] = or <4 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> undef, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i32> [[TMP5]], [[TMP4]]
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP6]], i64 0
-; CHECK-NEXT:    ret i32 [[TMP7]]
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = or <4 x i32> [[Z]], [[SHIFT]]
+; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i32> [[TMP1]], [[SHIFT1]]
+; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> undef, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[SHIFT2]], [[TMP2]]
+; CHECK-NEXT:    [[Z0123:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    ret i32 [[Z0123]]
 ;
   %z = and <4 x i32> %x, %y
   %z0 = extractelement <4 x i32> %z, i32 0
@@ -512,12 +512,12 @@ define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
 
 define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
 ; CHECK-LABEL: @ext_ext_partial_add_reduction_v4i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[X]]
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP3]], [[TMP2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP4]], i64 0
-; CHECK-NEXT:    ret i32 [[TMP5]]
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[X]]
+; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[SHIFT1]], [[TMP1]]
+; CHECK-NEXT:    [[X210:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    ret i32 [[X210]]
 ;
   %x0 = extractelement <4 x i32> %x, i32 0
   %x1 = extractelement <4 x i32> %x, i32 1
@@ -529,14 +529,14 @@ define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
 
 define i32 @ext_ext_partial_add_reduction_and_extra_add_v4i32(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @ext_ext_partial_add_reduction_and_extra_add_v4i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[Y]]
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP3]], [[TMP2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP5]], [[TMP4]]
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP6]], i64 0
-; CHECK-NEXT:    ret i32 [[TMP7]]
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[Y]]
+; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[SHIFT1]], [[TMP1]]
+; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[SHIFT2]], [[TMP2]]
+; CHECK-NEXT:    [[X2Y210:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    ret i32 [[X2Y210]]
 ;
   %y0 = extractelement <4 x i32> %y, i32 0
   %y1 = extractelement <4 x i32> %y, i32 1
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-cmp-binop.ll b/llvm/test/Transforms/VectorCombine/X86/extract-cmp-binop.ll
new file mode 100644
index 0000000000000..211936bb76d4a
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-cmp-binop.ll
@@ -0,0 +1,169 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -vector-combine -S -mtriple=x86_64-- -mattr=sse2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -vector-combine -S -mtriple=x86_64-- -mattr=avx2 | FileCheck %s --check-prefixes=CHECK,AVX
+
+define i1 @fcmp_and_v2f64(<2 x double> %a) {
+; SSE-LABEL: @fcmp_and_v2f64(
+; SSE-NEXT:    [[E1:%.*]] = extractelement <2 x double> [[A:%.*]], i32 0
+; SSE-NEXT:    [[E2:%.*]] = extractelement <2 x double> [[A]], i32 1
+; SSE-NEXT:    [[CMP1:%.*]] = fcmp olt double [[E1]], 4.200000e+01
+; SSE-NEXT:    [[CMP2:%.*]] = fcmp olt double [[E2]], -8.000000e+00
+; SSE-NEXT:    [[R:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; SSE-NEXT:    ret i1 [[R]]
+;
+; AVX-LABEL: @fcmp_and_v2f64(
+; AVX-NEXT:    [[TMP1:%.*]] = fcmp olt <2 x double> [[A:%.*]], <double 4.200000e+01, double -8.000000e+00>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <2 x i1> [[TMP1]], <2 x i1> undef, <2 x i32> <i32 1, i32 undef>
+; AVX-NEXT:    [[TMP2:%.*]] = and <2 x i1> [[TMP1]], [[SHIFT]]
+; AVX-NEXT:    [[R:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
+; AVX-NEXT:    ret i1 [[R]]
+;
+  %e1 = extractelement <2 x double> %a, i32 0
+  %e2 = extractelement <2 x double> %a, i32 1
+  %cmp1 = fcmp olt double %e1, 42.0
+  %cmp2 = fcmp olt double %e2, -8.0
+  %r = and i1 %cmp1, %cmp2
+  ret i1 %r
+}
+
+define i1 @fcmp_or_v4f64(<4 x double> %a) {
+; SSE-LABEL: @fcmp_or_v4f64(
+; SSE-NEXT:    [[E1:%.*]] = extractelement <4 x double> [[A:%.*]], i32 0
+; SSE-NEXT:    [[E2:%.*]] = extractelement <4 x double> [[A]], i64 2
+; SSE-NEXT:    [[CMP1:%.*]] = fcmp olt double [[E1]], 4.200000e+01
+; SSE-NEXT:    [[CMP2:%.*]] = fcmp olt double [[E2]], -8.000000e+00
+; SSE-NEXT:    [[R:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; SSE-NEXT:    ret i1 [[R]]
+;
+; AVX-LABEL: @fcmp_or_v4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = fcmp olt <4 x double> [[A:%.*]], <double 4.200000e+01, double undef, double -8.000000e+00, double undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[TMP2:%.*]] = or <4 x i1> [[TMP1]], [[SHIFT]]
+; AVX-NEXT:    [[R:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
+; AVX-NEXT:    ret i1 [[R]]
+;
+  %e1 = extractelement <4 x double> %a, i32 0
+  %e2 = extractelement <4 x double> %a, i64 2
+  %cmp1 = fcmp olt double %e1, 42.0
+  %cmp2 = fcmp olt double %e2, -8.0
+  %r = or i1 %cmp1, %cmp2
+  ret i1 %r
+}
+
+define i1 @icmp_xor_v4i32(<4 x i32> %a) {
+; CHECK-LABEL: @icmp_xor_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[A:%.*]], <i32 undef, i32 -8, i32 undef, i32 42>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> undef, <4 x i32> <i32 undef, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], [[SHIFT]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %e1 = extractelement <4 x i32> %a, i32 3
+  %e2 = extractelement <4 x i32> %a, i32 1
+  %cmp1 = icmp sgt i32 %e1, 42
+  %cmp2 = icmp sgt i32 %e2, -8
+  %r = xor i1 %cmp1, %cmp2
+  ret i1 %r
+}
+
+; add is not canonical (should be xor), but that is ok.
+
+define i1 @icmp_add_v8i32(<8 x i32> %a) {
+; SSE-LABEL: @icmp_add_v8i32(
+; SSE-NEXT:    [[E1:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 7
+; SSE-NEXT:    [[E2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; SSE-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[E1]], 42
+; SSE-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[E2]], -8
+; SSE-NEXT:    [[R:%.*]] = add i1 [[CMP1]], [[CMP2]]
+; SSE-NEXT:    ret i1 [[R]]
+;
+; AVX-LABEL: @icmp_add_v8i32(
+; AVX-NEXT:    [[TMP1:%.*]] = icmp eq <8 x i32> [[A:%.*]], <i32 undef, i32 undef, i32 -8, i32 undef, i32 undef, i32 undef, i32 undef, i32 42>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> undef, <8 x i32> <i32 undef, i32 undef, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[TMP2:%.*]] = add <8 x i1> [[TMP1]], [[SHIFT]]
+; AVX-NEXT:    [[R:%.*]] = extractelement <8 x i1> [[TMP2]], i64 2
+; AVX-NEXT:    ret i1 [[R]]
+;
+  %e1 = extractelement <8 x i32> %a, i32 7
+  %e2 = extractelement <8 x i32> %a, i32 2
+  %cmp1 = icmp eq i32 %e1, 42
+  %cmp2 = icmp eq i32 %e2, -8
+  %r = add i1 %cmp1, %cmp2
+  ret i1 %r
+}
+
+; Negative test - this could CSE/simplify.
+
+define i1 @same_extract_index(<4 x i32> %a) {
+; CHECK-LABEL: @same_extract_index(
+; CHECK-NEXT:    [[E1:%.*]] = extractelement <4 x i32> [[A:%.*]], i32 2
+; CHECK-NEXT:    [[E2:%.*]] = extractelement <4 x i32> [[A]], i32 2
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 [[E1]], 42
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[E2]], -8
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %e1 = extractelement <4 x i32> %a, i32 2
+  %e2 = extractelement <4 x i32> %a, i32 2
+  %cmp1 = icmp ugt i32 %e1, 42
+  %cmp2 = icmp ugt i32 %e2, -8
+  %r = and i1 %cmp1, %cmp2
+  ret i1 %r
+}
+
+; Negative test - need identical predicates.
+
+define i1 @different_preds(<4 x i32> %a) {
+; CHECK-LABEL: @different_preds(
+; CHECK-NEXT:    [[E1:%.*]] = extractelement <4 x i32> [[A:%.*]], i32 1
+; CHECK-NEXT:    [[E2:%.*]] = extractelement <4 x i32> [[A]], i32 2
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[E1]], 42
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[E2]], -8
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %e1 = extractelement <4 x i32> %a, i32 1
+  %e2 = extractelement <4 x i32> %a, i32 2
+  %cmp1 = icmp sgt i32 %e1, 42
+  %cmp2 = icmp ugt i32 %e2, -8
+  %r = and i1 %cmp1, %cmp2
+  ret i1 %r
+}
+
+; Negative test - need 1 source vector.
+
+define i1 @different_source_vec(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @different_source_vec(
+; CHECK-NEXT:    [[E1:%.*]] = extractelement <4 x i32> [[A:%.*]], i32 1
+; CHECK-NEXT:    [[E2:%.*]] = extractelement <4 x i32> [[B:%.*]], i32 2
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[E1]], 42
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[E2]], -8
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %e1 = extractelement <4 x i32> %a, i32 1
+  %e2 = extractelement <4 x i32> %b, i32 2
+  %cmp1 = icmp sgt i32 %e1, 42
+  %cmp2 = icmp sgt i32 %e2, -8
+  %r = and i1 %cmp1, %cmp2
+  ret i1 %r
+}
+
+; Negative test - don't try this with scalable vectors.
+
+define i1 @scalable(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: @scalable(
+; CHECK-NEXT:    [[E1:%.*]] = extractelement <vscale x 4 x i32> [[A:%.*]], i32 3
+; CHECK-NEXT:    [[E2:%.*]] = extractelement <vscale x 4 x i32> [[A]], i32 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[E1]], 42
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[E2]], -8
+; CHECK-NEXT:    [[R:%.*]] = xor i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %e1 = extractelement <vscale x 4 x i32> %a, i32 3
+  %e2 = extractelement <vscale x 4 x i32> %a, i32 1
+  %cmp1 = icmp sgt i32 %e1, 42
+  %cmp2 = icmp sgt i32 %e2, -8
+  %r = xor i1 %cmp1, %cmp2
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-cmp.ll b/llvm/test/Transforms/VectorCombine/X86/extract-cmp.ll
index 6f6f6d0f9674a..814649e2a7088 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-cmp.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-cmp.ll
@@ -8,22 +8,22 @@ define i1 @cmp_v4i32(<4 x float> %arg, <4 x float> %arg1) {
 ; CHECK-NEXT:    [[T:%.*]] = bitcast <4 x float> [[ARG:%.*]] to <4 x i32>
 ; CHECK-NEXT:    [[T3:%.*]] = bitcast <4 x float> [[ARG1:%.*]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq <4 x i32> [[T]], [[T3]]
-; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
-; CHECK-NEXT:    br i1 [[TMP1]], label [[BB6:%.*]], label [[BB18:%.*]]
+; CHECK-NEXT:    [[T5:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
+; CHECK-NEXT:    br i1 [[T5]], label [[BB6:%.*]], label [[BB18:%.*]]
 ; CHECK:       bb6:
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <4 x i32> [[T]], [[T3]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1
-; CHECK-NEXT:    br i1 [[TMP3]], label [[BB10:%.*]], label [[BB18]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <4 x i32> [[T]], [[T3]]
+; CHECK-NEXT:    [[T9:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
+; CHECK-NEXT:    br i1 [[T9]], label [[BB10:%.*]], label [[BB18]]
 ; CHECK:       bb10:
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <4 x i32> [[T]], [[T3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
-; CHECK-NEXT:    br i1 [[TMP5]], label [[BB14:%.*]], label [[BB18]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <4 x i32> [[T]], [[T3]]
+; CHECK-NEXT:    [[T13:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2
+; CHECK-NEXT:    br i1 [[T13]], label [[BB14:%.*]], label [[BB18]]
 ; CHECK:       bb14:
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i32> [[T]], [[T3]]
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 3
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <4 x i32> [[T]], [[T3]]
+; CHECK-NEXT:    [[T17:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3
 ; CHECK-NEXT:    br label [[BB18]]
 ; CHECK:       bb18:
-; CHECK-NEXT:    [[T19:%.*]] = phi i1 [ false, [[BB10]] ], [ false, [[BB6]] ], [ false, [[BB:%.*]] ], [ [[TMP7]], [[BB14]] ]
+; CHECK-NEXT:    [[T19:%.*]] = phi i1 [ false, [[BB10]] ], [ false, [[BB6]] ], [ false, [[BB:%.*]] ], [ [[T17]], [[BB14]] ]
 ; CHECK-NEXT:    ret i1 [[T19]]
 ;
 bb:
@@ -75,12 +75,12 @@ define i32 @cmp_v2f64(<2 x double> %x, <2 x double> %y, <2 x double> %z) {
 ; AVX-LABEL: @cmp_v2f64(
 ; AVX-NEXT:  entry:
 ; AVX-NEXT:    [[TMP0:%.*]] = fcmp oeq <2 x double> [[X:%.*]], [[Y:%.*]]
-; AVX-NEXT:    [[TMP1:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
-; AVX-NEXT:    br i1 [[TMP1]], label [[T:%.*]], label [[F:%.*]]
+; AVX-NEXT:    [[CMP1:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
+; AVX-NEXT:    br i1 [[CMP1]], label [[T:%.*]], label [[F:%.*]]
 ; AVX:       t:
-; AVX-NEXT:    [[TMP2:%.*]] = fcmp ogt <2 x double> [[Y]], [[Z:%.*]]
-; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
-; AVX-NEXT:    [[E:%.*]] = select i1 [[TMP3]], i32 42, i32 99
+; AVX-NEXT:    [[TMP1:%.*]] = fcmp ogt <2 x double> [[Y]], [[Z:%.*]]
+; AVX-NEXT:    [[CMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1
+; AVX-NEXT:    [[E:%.*]] = select i1 [[CMP2]], i32 42, i32 99
 ; AVX-NEXT:    ret i32 [[E]]
 ; AVX:       f:
 ; AVX-NEXT:    ret i32 0
@@ -109,10 +109,10 @@ define i1 @cmp01_v2f64(<2 x double> %x, <2 x double> %y) {
 ; SSE-NEXT:    ret i1 [[CMP]]
 ;
 ; AVX-LABEL: @cmp01_v2f64(
-; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[Y:%.*]], <2 x double> undef, <2 x i32> <i32 1, i32 undef>
-; AVX-NEXT:    [[TMP2:%.*]] = fcmp oge <2 x double> [[X:%.*]], [[TMP1]]
-; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
-; AVX-NEXT:    ret i1 [[TMP3]]
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <2 x double> [[Y:%.*]], <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; AVX-NEXT:    [[TMP1:%.*]] = fcmp oge <2 x double> [[X:%.*]], [[SHIFT]]
+; AVX-NEXT:    [[CMP:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0
+; AVX-NEXT:    ret i1 [[CMP]]
 ;
   %x0 = extractelement <2 x double> %x, i32 0
   %y1 = extractelement <2 x double> %y, i32 1
@@ -128,10 +128,10 @@ define i1 @cmp10_v2f64(<2 x double> %x, <2 x double> %y) {
 ; SSE-NEXT:    ret i1 [[CMP]]
 ;
 ; AVX-LABEL: @cmp10_v2f64(
-; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> undef, <2 x i32> <i32 1, i32 undef>
-; AVX-NEXT:    [[TMP2:%.*]] = fcmp ule <2 x double> [[TMP1]], [[Y:%.*]]
-; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
-; AVX-NEXT:    ret i1 [[TMP3]]
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; AVX-NEXT:    [[TMP1:%.*]] = fcmp ule <2 x double> [[SHIFT]], [[Y:%.*]]
+; AVX-NEXT:    [[CMP:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
+; AVX-NEXT:    ret i1 [[CMP]]
 ;
   %x1 = extractelement <2 x double> %x, i32 1
   %y0 = extractelement <2 x double> %y, i32 0
@@ -141,10 +141,10 @@ define i1 @cmp10_v2f64(<2 x double> %x, <2 x double> %y) {
 
 define i1 @cmp12_v4i32(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @cmp12_v4i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> undef, <4 x i32> <i32 undef, i32 2, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1
-; CHECK-NEXT:    ret i1 [[TMP3]]
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> undef, <4 x i32> <i32 undef, i32 2, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[SHIFT]]
+; CHECK-NEXT:    [[CMP:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
+; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %x1 = extractelement <4 x i32> %x, i32 1
   %y2 = extractelement <4 x i32> %y, i32 2
@@ -161,10 +161,10 @@ define <4 x i1> @ins_fcmp_ext_ext(<4 x float> %a, <4 x i1> %b) {
 ; SSE-NEXT:    ret <4 x i1> [[R]]
 ;
 ; AVX-LABEL: @ins_fcmp_ext_ext(
-; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 undef>
-; AVX-NEXT:    [[TMP2:%.*]] = fcmp ugt <4 x float> [[A]], [[TMP1]]
-; AVX-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2
-; AVX-NEXT:    [[R:%.*]] = insertelement <4 x i1> [[B:%.*]], i1 [[TMP3]], i32 2
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 undef>
+; AVX-NEXT:    [[TMP1:%.*]] = fcmp ugt <4 x float> [[A]], [[SHIFT]]
+; AVX-NEXT:    [[A21:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
+; AVX-NEXT:    [[R:%.*]] = insertelement <4 x i1> [[B:%.*]], i1 [[A21]], i32 2
 ; AVX-NEXT:    ret <4 x i1> [[R]]
 ;
   %a1 = extractelement <4 x float> %a, i32 1
@@ -176,10 +176,10 @@ define <4 x i1> @ins_fcmp_ext_ext(<4 x float> %a, <4 x i1> %b) {
 
 define <4 x i1> @ins_icmp_ext_ext(<4 x i32> %a, <4 x i1> %b) {
 ; CHECK-LABEL: @ins_icmp_ext_ext(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule <4 x i32> [[TMP1]], [[A]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
-; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x i1> [[B:%.*]], i1 [[TMP3]], i32 3
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i32> [[SHIFT]], [[A]]
+; CHECK-NEXT:    [[A23:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x i1> [[B:%.*]], i1 [[A23]], i32 3
 ; CHECK-NEXT:    ret <4 x i1> [[R]]
 ;
   %a3 = extractelement <4 x i32> %a, i32 3
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle.ll
index 50b6f9e35dd47..1608cc0bc3eff 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle.ll
@@ -12,8 +12,8 @@ define <16 x i8> @bitcast_shuf_narrow_element(<4 x i32> %v) {
 ;
 ; AVX-LABEL: @bitcast_shuf_narrow_element(
 ; AVX-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V:%.*]] to <16 x i8>
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
-; AVX-NEXT:    ret <16 x i8> [[TMP2]]
+; AVX-NEXT:    [[R:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; AVX-NEXT:    ret <16 x i8> [[R]]
 ;
   %shuf = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   %r = bitcast <4 x i32> %shuf to <16 x i8>
@@ -25,8 +25,8 @@ define <16 x i8> @bitcast_shuf_narrow_element(<4 x i32> %v) {
 define <4 x float> @bitcast_shuf_same_size(<4 x i32> %v) {
 ; CHECK-LABEL: @bitcast_shuf_same_size(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V:%.*]] to <4 x float>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    ret <4 x float> [[TMP2]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %shuf = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   %r = bitcast <4 x i32> %shuf to <4 x float>
@@ -64,8 +64,8 @@ define i128 @bitcast_shuf_narrow_element_wrong_type(<4 x i32> %v) {
 define <4 x i32> @bitcast_shuf_wide_element(<8 x i16> %v) {
 ; CHECK-LABEL: @bitcast_shuf_wide_element(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[V:%.*]] to <4 x i32>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
-; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
   %shuf = shufflevector <8 x i16> %v, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3>
   %r = bitcast <8 x i16> %shuf to <4 x i32>
@@ -103,8 +103,8 @@ define <2 x i64> @PR35454_1(<2 x i64> %v) {
 ; AVX-LABEL: @PR35454_1(
 ; AVX-NEXT:    [[BC:%.*]] = bitcast <2 x i64> [[V:%.*]] to <4 x i32>
 ; AVX-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[BC]] to <16 x i8>
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
-; AVX-NEXT:    [[ADD:%.*]] = shl <16 x i8> [[TMP2]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+; AVX-NEXT:    [[BC1:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; AVX-NEXT:    [[ADD:%.*]] = shl <16 x i8> [[BC1]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
 ; AVX-NEXT:    [[BC2:%.*]] = bitcast <16 x i8> [[ADD]] to <4 x i32>
 ; AVX-NEXT:    [[PERMIL1:%.*]] = shufflevector <4 x i32> [[BC2]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; AVX-NEXT:    [[BC3:%.*]] = bitcast <4 x i32> [[PERMIL1]] to <2 x i64>
@@ -134,8 +134,8 @@ define <2 x i64> @PR35454_2(<2 x i64> %v) {
 ; AVX-LABEL: @PR35454_2(
 ; AVX-NEXT:    [[BC:%.*]] = bitcast <2 x i64> [[V:%.*]] to <4 x i32>
 ; AVX-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[BC]] to <8 x i16>
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 6, i32 7, i32 4, i32 5, i32 2, i32 3, i32 0, i32 1>
-; AVX-NEXT:    [[ADD:%.*]] = shl <8 x i16> [[TMP2]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+; AVX-NEXT:    [[BC1:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 6, i32 7, i32 4, i32 5, i32 2, i32 3, i32 0, i32 1>
+; AVX-NEXT:    [[ADD:%.*]] = shl <8 x i16> [[BC1]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
 ; AVX-NEXT:    [[BC2:%.*]] = bitcast <8 x i16> [[ADD]] to <4 x i32>
 ; AVX-NEXT:    [[PERMIL1:%.*]] = shufflevector <4 x i32> [[BC2]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; AVX-NEXT:    [[BC3:%.*]] = bitcast <4 x i32> [[PERMIL1]] to <2 x i64>
diff --git a/llvm/test/Verifier/callbr.ll b/llvm/test/Verifier/callbr.ll
index 403cc573d0dd0..6bd0b3ceb25e9 100644
--- a/llvm/test/Verifier/callbr.ll
+++ b/llvm/test/Verifier/callbr.ll
@@ -1,10 +1,11 @@
 ; RUN: not opt -S %s -verify 2>&1 | FileCheck %s
 
 ; CHECK: Indirect label missing from arglist.
-define void @foo() {
+; CHECK-NEXT: #test1
+define void @test1() {
   ; The %4 in the indirect label list is not found in the blockaddresses in the
   ; arg list (bad).
-  callbr void asm sideeffect "${0:l} {1:l}", "X,X"(i8* blockaddress(@foo, %3), i8* blockaddress(@foo, %2))
+  callbr void asm sideeffect "#test1", "X,X"(i8* blockaddress(@test1, %3), i8* blockaddress(@test1, %2))
   to label %1 [label %4, label %2]
 1:
   ret void
@@ -17,9 +18,9 @@ define void @foo() {
 }
 
 ; CHECK-NOT: Indirect label missing from arglist.
-define void @bar() {
+define void @test2() {
   ; %4 and %2 are both in the indirect label list and the arg list (good).
-  callbr void asm sideeffect "${0:l} ${1:l}", "X,X"(i8* blockaddress(@bar, %4), i8* blockaddress(@bar, %2))
+  callbr void asm sideeffect "${0:l} ${1:l}", "X,X"(i8* blockaddress(@test2, %4), i8* blockaddress(@test2, %2))
   to label %1 [label %4, label %2]
 1:
   ret void
@@ -32,12 +33,12 @@ define void @bar() {
 }
 
 ; CHECK-NOT: Indirect label missing from arglist.
-define void @baz() {
+define void @test3() {
   ; note %2 blockaddress. Such a case is possible when passing the address of
   ; a label as an input to the inline asm (both address of label and asm goto
   ; use blockaddress constants; we're testing that the indirect label list from
   ; the asm goto is in the arg list to the asm).
-  callbr void asm sideeffect "${0:l} ${1:l} ${2:l}", "X,X,X"(i8* blockaddress(@baz, %4), i8* blockaddress(@baz, %2), i8* blockaddress(@baz, %3))
+  callbr void asm sideeffect "${0:l} ${1:l} ${2:l}", "X,X,X"(i8* blockaddress(@test3, %4), i8* blockaddress(@test3, %2), i8* blockaddress(@test3, %3))
   to label %1 [label %3, label %4]
 1:
   ret void
@@ -48,3 +49,28 @@ define void @baz() {
 4:
   ret void
 }
+
+;; Ensure you cannot use the return value of a callbr in indirect targets.
+; CHECK: Instruction does not dominate all uses!
+; CHECK-NEXT: #test4
+define i32 @test4(i1 %var) {
+entry:
+  %ret = callbr i32 asm sideeffect "#test4", "=r,X"(i8* blockaddress(@test4, %abnormal)) to label %normal [label %abnormal]
+
+normal:
+  ret i32 0
+
+abnormal:
+  ret i32 %ret
+}
+
+;; Ensure you cannot specify the same label as both normal and indirect targets.
+; CHECK: Duplicate callbr destination!
+; CHECK-NEXT: #test5
+define i32 @test5() {
+entry:
+  %ret = callbr i32 asm sideeffect "#test5", "=r,X"(i8* blockaddress(@test5, %both)) to label %both [label %both]
+
+both:
+  ret i32 0
+}
diff --git a/llvm/test/Verifier/matrix-intrinsics.ll b/llvm/test/Verifier/matrix-intrinsics.ll
index 5f5af03da1010..6b2a4c501c660 100644
--- a/llvm/test/Verifier/matrix-intrinsics.ll
+++ b/llvm/test/Verifier/matrix-intrinsics.ll
@@ -1,40 +1,66 @@
 ; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
 
 declare <4 x float> @llvm.matrix.transpose.v4f32(<4 x float>, i32, i32)
-define <4 x float> @transpose(<4 x float> %m) {
+define <4 x float> @transpose(<4 x float> %m, i32 %arg) {
 ; CHECK: assembly parsed, but does not verify as correct!
 ; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
 ; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
-  %result.1 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %m, i32 3, i32 2)
+; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
+; CHECK-NEXT: immarg operand has non-immediate parameter
+; CHECK-NEXT: i32 %arg
+; CHECK-NEXT:   %result.3 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.2, i32 %arg, i32 2)
+; CHECK-NEXT: immarg operand has non-immediate parameter
+; CHECK-NEXT: i32 %arg
+; CHECK-NEXT:   %result.4 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.3, i32 2, i32 %arg)
+  %result.0 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %m, i32 0, i32 0)
+  %result.1 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.0, i32 3, i32 2)
   %result.2 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.1, i32 2, i32 1)
-  ret <4 x float> %result.2
+  %result.3 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.2, i32 %arg, i32 2)
+  %result.4 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.3, i32 2, i32 %arg)
+  ret <4 x float> %result.4
 }
 
 declare <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float>, <4 x float>, i32, i32, i32)
-define <4 x float> @multiply(<4 x float> %m) {
+define <4 x float> @multiply(<4 x float> %m, i32 %arg) {
+; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
 ; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
 ; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
-  %result.1 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %m, <4 x float> %m, i32 3, i32 2, i32 2)
+; CHECK-NEXT: immarg operand has non-immediate parameter
+; CHECK-NEXT: i32 %arg
+; CHECK-NEXT:   %result.3 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %result.2, <4 x float> %m, i32 %arg, i32 2, i32 1)
+  %result.0 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %m, <4 x float> %m, i32 0, i32 0, i32 0)
+  %result.1 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %result.0, <4 x float> %m, i32 3, i32 2, i32 2)
   %result.2 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %result.1, <4 x float> %m, i32 2, i32 2, i32 1)
-  ret <4 x float> %result.2
+  %result.3 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %result.2, <4 x float> %m, i32 %arg, i32 2, i32 1)
+  ret <4 x float> %result.3
 }
 
 declare <4 x float> @llvm.matrix.column.major.load.v4f32.p0v4f32(<4 x float>*, i64, i1, i32, i32)
 declare <6 x float> @llvm.matrix.column.major.load.v6f32.p0v6f32(<6 x float>*, i64, i1, i32, i32)
-define <4 x float> @column.major_load(<4 x float>* %m, <6 x float>* %n) {
+define <4 x float> @column.major_load(<4 x float>* %m, <6 x float>* %n, i32 %arg) {
+; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
 ; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
 ; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
+; CHECK-NEXT: immarg operand has non-immediate parameter
+; CHECK-NEXT: i32 %arg
+; CHECK-NEXT:   %result.3 = call <6 x float> @llvm.matrix.column.major.load.v6f32.p0v6f32(<6 x float>* %n, i64 2, i1 true, i32 3, i32 %arg)
+  %result.0 = call <4 x float> @llvm.matrix.column.major.load.v4f32.p0v4f32(<4 x float>* %m, i64 0, i1 false, i32 0, i32 0)
   %result.1 = call <4 x float> @llvm.matrix.column.major.load.v4f32.p0v4f32(<4 x float>* %m, i64 2, i1 false, i32 1, i32 2)
   %result.2 = call <6 x float> @llvm.matrix.column.major.load.v6f32.p0v6f32(<6 x float>* %n, i64 2, i1 true, i32 3, i32 3)
+  %result.3 = call <6 x float> @llvm.matrix.column.major.load.v6f32.p0v6f32(<6 x float>* %n, i64 2, i1 true, i32 3, i32 %arg)
   ret <4 x float> %result.1
 }
 
 declare void @llvm.matrix.column.major.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i64, i1, i32, i32)
 declare void @llvm.matrix.column.major.store.v6f32.p0v6f32(<6 x float>, <6 x float>*, i64, i1, i32, i32)
-define void @column.major_store(<4 x float>* %m, <6 x float>* %n) {
+define void @column.major_store(<4 x float>* %m, <6 x float>* %n, i64 %arg) {
+; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
+; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
 ; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
 ; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
+  call void @llvm.matrix.column.major.store.v4f32.p0v4f32(<4 x float> zeroinitializer, <4 x float>* %m, i64 0, i1 false, i32 0, i32 0)
   call void @llvm.matrix.column.major.store.v4f32.p0v4f32(<4 x float> zeroinitializer, <4 x float>* %m, i64 2, i1 false, i32 1, i32 2)
   call void @llvm.matrix.column.major.store.v6f32.p0v6f32(<6 x float> zeroinitializer, <6 x float>* %n, i64 2, i1 false, i32 3, i32 3)
+  call void @llvm.matrix.column.major.store.v6f32.p0v6f32(<6 x float> zeroinitializer, <6 x float>* %n, i64 %arg, i1 false, i32 3, i32 3)
   ret void
 }
diff --git a/llvm/test/Verifier/preallocated-invalid.ll b/llvm/test/Verifier/preallocated-invalid.ll
index 7fdab33167e51..879d4ed8a24f9 100644
--- a/llvm/test/Verifier/preallocated-invalid.ll
+++ b/llvm/test/Verifier/preallocated-invalid.ll
@@ -2,6 +2,7 @@
 
 declare token @llvm.call.preallocated.setup(i32)
 declare i8* @llvm.call.preallocated.arg(token, i32)
+declare void @llvm.call.preallocated.teardown(token)
 
 ; Fake LLVM intrinsic to return a token
 declare token @llvm.what()
@@ -136,3 +137,10 @@ define void @musttail_attr_no_match(i32* preallocated(i32) %a) {
     musttail call void @musttail_and_bundle(i32* %a)
     ret void
 }
+
+; CHECK: token argument must be a llvm.call.preallocated.setup
+define void @teardown_token_not_from_setup() {
+    %cs = call token @llvm.what()
+    call void @llvm.call.preallocated.teardown(token %cs)
+    ret void
+}
diff --git a/llvm/test/Verifier/preallocated-valid.ll b/llvm/test/Verifier/preallocated-valid.ll
index 483493c0c7470..bbb663b94ebf7 100644
--- a/llvm/test/Verifier/preallocated-valid.ll
+++ b/llvm/test/Verifier/preallocated-valid.ll
@@ -2,11 +2,16 @@
 
 declare token @llvm.call.preallocated.setup(i32)
 declare i8* @llvm.call.preallocated.arg(token, i32)
+declare void @llvm.call.preallocated.teardown(token)
+
+declare i32 @__CxxFrameHandler3(...)
 
 declare void @foo1(i32* preallocated(i32))
 declare i64 @foo1_i64(i32* preallocated(i32))
 declare void @foo2(i32* preallocated(i32), i32*, i32* preallocated(i32))
 
+declare void @constructor(i32*)
+
 define void @preallocated() {
     %cs = call token @llvm.call.preallocated.setup(i32 1)
     %x = call i8* @llvm.call.preallocated.arg(token %cs, i32 0) preallocated(i32)
@@ -40,12 +45,34 @@ define void @preallocated_num_args() {
     ret void
 }
 
-define void @preallocate_musttail(i32* preallocated(i32) %a) {
+define void @preallocated_musttail(i32* preallocated(i32) %a) {
     musttail call void @foo1(i32* preallocated(i32) %a)
     ret void
 }
 
-define i64 @preallocate_musttail_i64(i32* preallocated(i32) %a) {
+define i64 @preallocated_musttail_i64(i32* preallocated(i32) %a) {
     %r = musttail call i64 @foo1_i64(i32* preallocated(i32) %a)
     ret i64 %r
 }
+
+define void @preallocated_teardown() {
+    %cs = call token @llvm.call.preallocated.setup(i32 1)
+    call void @llvm.call.preallocated.teardown(token %cs)
+    ret void
+}
+
+define void @preallocated_teardown_invoke() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+    %cs = call token @llvm.call.preallocated.setup(i32 1)
+    %x = call i8* @llvm.call.preallocated.arg(token %cs, i32 0) preallocated(i32)
+    %y = bitcast i8* %x to i32*
+    invoke void @constructor(i32* %y) to label %conta unwind label %contb
+conta:
+    call void @foo1(i32* preallocated(i32) %y) ["preallocated"(token %cs)]
+    ret void
+contb:
+    %s = catchswitch within none [label %catch] unwind to caller
+catch:
+    %p = catchpad within %s []
+    call void @llvm.call.preallocated.teardown(token %cs)
+    ret void
+}
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index 49d345566b656..4eaa6cb77c824 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -22,7 +22,7 @@
 
 # suffixes: A list of file extensions to treat as test files. This is overriden
 # by individual lit.local.cfg files in the test subdirectories.
-config.suffixes = ['.ll', '.c', '.cxx', '.test', '.txt', '.s', '.mir']
+config.suffixes = ['.ll', '.c', '.test', '.txt', '.s', '.mir', '.yaml']
 
 # excludes: A list of directories to exclude from the testsuite. The 'Inputs'
 # subdirectories contain auxiliary inputs for various tests in their parent
@@ -219,6 +219,9 @@ def get_asan_rtlib():
 if not config.build_shared_libs and not config.link_llvm_dylib:
     config.available_features.add('static-libs')
 
+if config.have_tf_aot:
+    config.available_features.add("have_tf_aot")
+
 def have_cxx_shared_library():
     readobj_exe = lit.util.which('llvm-readobj', config.llvm_tools_dir)
     if not readobj_exe:
diff --git a/llvm/test/lit.site.cfg.py.in b/llvm/test/lit.site.cfg.py.in
index 3797830862f76..190cd4bccc2dc 100644
--- a/llvm/test/lit.site.cfg.py.in
+++ b/llvm/test/lit.site.cfg.py.in
@@ -48,6 +48,7 @@ config.have_opt_viewer_modules = @LLVM_HAVE_OPT_VIEWER_MODULES@
 config.libcxx_used = @LLVM_LIBCXX_USED@
 config.has_plugins = @LLVM_ENABLE_PLUGINS@
 config.linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
+config.have_tf_aot = @LLVM_HAVE_TF_AOT@
 
 # Support substitution of the tools_dir with user parameters. This is
 # used when we can't determine the tool dir at configuration time.
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/basic.ll.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/basic.ll.expected
index 96b06de79c93a..3c49f489a3531 100644
--- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/basic.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/basic.ll.expected
@@ -1,4 +1,3 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; Example input for update_llc_test_checks (taken from CodeGen/X86/iabs.ll)
 ; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefix=X86 --check-prefix=X86-NO-CMOV
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+cmov | FileCheck %s --check-prefix=X86 --check-prefix=X86-CMOV
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/on_the_fly_arg_change.ll b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/on_the_fly_arg_change.ll
new file mode 100644
index 0000000000000..ed5d949bb0926
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/on_the_fly_arg_change.ll
@@ -0,0 +1,22 @@
+; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s
+
+declare void @foo()
+
+define i64 @check_lines_1() {
+  ret i64 1
+}
+
+; UTC_ARGS: --disable
+
+define i64 @no_check_lines() {
+; A check line that would not be auto-generated (should not be removed!).
+; CHECK: manual check line
+  ret i64 2
+}
+
+; UTC_ARGS: --enable --no_x86_scrub_rip
+
+define i64 @check_lines_2() {
+  %result = call i64 @no_check_lines()
+  ret i64 %result
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/on_the_fly_arg_change.ll.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/on_the_fly_arg_change.ll.expected
new file mode 100644
index 0000000000000..f1955c4af252d
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/on_the_fly_arg_change.ll.expected
@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s
+
+declare void @foo()
+
+define i64 @check_lines_1() {
+; CHECK-LABEL: check_lines_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    retl
+  ret i64 1
+}
+
+; UTC_ARGS: --disable
+
+define i64 @no_check_lines() {
+; A check line that would not be auto-generated (should not be removed!).
+; CHECK: manual check line
+  ret i64 2
+}
+
+; UTC_ARGS: --enable --no_x86_scrub_rip
+
+define i64 @check_lines_2() {
+; CHECK-LABEL: check_lines_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    calll no_check_lines
+; CHECK-NEXT:    retl
+  %result = call i64 @no_check_lines()
+  ret i64 %result
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/basic.test b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/basic.test
index 74c2b6cd70ede..36d0f5d9ff7e6 100644
--- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/basic.test
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/basic.test
@@ -2,14 +2,24 @@
 ## Basic test checking that update_llc_test_checks.py can update a file with multiple check prefixes
 
 # RUN: cp -f %S/Inputs/basic.ll %t.ll && %update_llc_test_checks %t.ll
-# RUN: diff -u %S/Inputs/basic.ll.expected %t.ll
-## The flags --x86_scrub_rip and --extra_scrub should have any effect for this
+# RUN: echo '; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py' > %t.expected.ll
+# RUN: cat %S/Inputs/basic.ll.expected >> %t.expected.ll
+# RUN: diff -u %t.expected.ll %t.ll
+
+## The flags --x86_scrub_rip and --extra_scrub should not have any effect on this
 ## test. Check the output is identical.
-# RUN: cp -f %S/Inputs/basic.ll %t.ll &&  %update_llc_test_checks --extra_scrub %t.ll
-# RUN: diff -u %S/Inputs/basic.ll.expected %t.ll
-# RUN: cp -f %S/Inputs/basic.ll %t.ll &&  %update_llc_test_checks --x86_scrub_rip %t.ll
-# RUN: diff -u %S/Inputs/basic.ll.expected %t.ll
+# RUN: cp -f %S/Inputs/basic.ll %t.ll && %update_llc_test_checks --extra_scrub %t.ll
+# RUN: echo '; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --extra_scrub' > %t.expected.ll
+# RUN: cat %S/Inputs/basic.ll.expected >> %t.expected.ll
+# RUN: diff -u %t.expected.ll %t.ll
+# RUN: cp -f %S/Inputs/basic.ll %t.ll && %update_llc_test_checks --no_x86_scrub_rip %t.ll
+# RUN: echo '; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_rip' > %t.expected.ll
+# RUN: cat %S/Inputs/basic.ll.expected >> %t.expected.ll
+# RUN: diff -u %t.expected.ll %t.ll
+
 ## Finally, run the script on an already updated file and verify that all previous
 ## CHECK lines are removed.
 # RUN: %update_llc_test_checks %t.ll
-# RUN: diff -u %S/Inputs/basic.ll.expected %t.ll
+# RUN: echo '; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_rip' > %t.expected.ll
+# RUN: cat %S/Inputs/basic.ll.expected >> %t.expected.ll
+# RUN: diff -u %t.expected.ll %t.ll
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/on_the_fly_arg_change.test b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/on_the_fly_arg_change.test
new file mode 100644
index 0000000000000..a53e03b8909a2
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/on_the_fly_arg_change.test
@@ -0,0 +1,7 @@
+# REQUIRES: x86-registered-target
+# RUN: cp -f %S/Inputs/on_the_fly_arg_change.ll %t.ll
+# RUN: %update_llc_test_checks %t.ll
+# RUN: diff -u %t.ll %S/Inputs/on_the_fly_arg_change.ll.expected
+## Check that running the script again does not change the result:
+# RUN: %update_llc_test_checks %t.ll
+# RUN: diff -u %t.ll %S/Inputs/on_the_fly_arg_change.ll.expected
diff --git a/llvm/test/tools/gold/lit.local.cfg b/llvm/test/tools/gold/lit.local.cfg
index a704bb548b9dc..6d2835ac8be64 100644
--- a/llvm/test/tools/gold/lit.local.cfg
+++ b/llvm/test/tools/gold/lit.local.cfg
@@ -1,2 +1,7 @@
-if (not 'ld_plugin' in  config.available_features):
+if (not 'ld_plugin' in config.available_features):
    config.unsupported = True
+
+# gold can't load instrumented plugin.
+for san in ['asan', 'msan', 'ubsan']:
+   if (san in config.available_features):
+      config.unsupported = True
diff --git a/llvm/test/tools/llvm-ar/error-opening-directory.test b/llvm/test/tools/llvm-ar/error-opening-directory.test
index 10cee214183eb..1f19da7543b3d 100644
--- a/llvm/test/tools/llvm-ar/error-opening-directory.test
+++ b/llvm/test/tools/llvm-ar/error-opening-directory.test
@@ -1,3 +1,7 @@
+## Unsupported on FreeBSD as FreeBSD 12 and earlier allow reading directories
+## by default.
+# UNSUPPORTED: system-freebsd
+
 # RUN: rm -rf %t && mkdir -p %t
 
 ## Passing in a directory:
diff --git a/llvm/test/tools/llvm-as/lit.local.cfg b/llvm/test/tools/llvm-as/lit.local.cfg
index 1fc0bea084cad..c8625f4d9d248 100644
--- a/llvm/test/tools/llvm-as/lit.local.cfg
+++ b/llvm/test/tools/llvm-as/lit.local.cfg
@@ -1,4 +1,2 @@
 if not 'X86' in config.root.targets:
     config.unsupported = True
-
-config.suffixes = ['.ll', '.s', '.test', '.yaml']
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_line_invalid.test b/llvm/test/tools/llvm-dwarfdump/X86/debug_line_invalid.test
index d662d9e16a045..022476aaffa13 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/debug_line_invalid.test
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_line_invalid.test
@@ -139,17 +139,15 @@
 
 ## Very short prologue length for V5 (ends during parameters).
 # NONFATAL:      debug_line[0x000001b1]
-# SOME-ERR-NEXT: warning: parsing line table prologue at 0x000001b1 found an invalid directory or file table description at 0x000001cd
-# SOME-ERR-NEXT: warning: failed to parse entry content descriptors: unexpected end of data at offset 0x1cd while reading [0x1cd, 0x1ce)
 # NONFATAL-NEXT: Line table prologue
 # NONFATAL:      standard_opcode_lengths[DW_LNS_set_prologue_end] = 1
 # NONFATAL-NEXT: standard_opcode_lengths[DW_LNS_set_epilogue_begin] = 0
-# NONFATAL-NOT:  include_directories
-# VERBOSE:       DW_LNE_set_address (0x0000babb1ebabb1e)
-# VERBOSE-NEXT:  DW_LNE_end_sequence
+# NONFATAL-NEXT: standard_opcode_lengths[DW_LNS_set_isa] = 0
+# NONFATAL-EMPTY:
+# SOME-ERR-NEXT: warning: parsing line table prologue at offset 0x000001b1: unexpected end of data at offset 0x1cd while reading [0x1cd, 0x1ce)
 
 ## V5 prologue ends during file table.
-# NONFATAL:      debug_line[0x000001db]
+# NONFATAL-NEXT: debug_line[0x000001db]
 # SOME-ERR-NEXT: warning: parsing line table prologue at 0x000001db found an invalid directory or file table description at 0x00000206
 # SOME-ERR-NEXT: warning: failed to parse entry content descriptors: unable to decode LEB128 at offset 0x00000206: malformed uleb128, extends past end
 # NONFATAL-NEXT: Line table prologue
@@ -244,7 +242,7 @@
 # VERBOSE-NEXT:  DW_LNE_set_address (0x00000000feedfeed)
 # VERBOSE-NEXT:  DW_LNS_copy
 # NONFATAL-NEXT: 0x00000000feedfeed
-# VERBOSE-NEXT:  DW_LNE_set_address (0x0000000000000000)
+# VERBOSE-NEXT:  DW_LNE_set_address (<parsing error> 00 f0 01 f0 f0 00 01)
 # MORE-ERR-NEXT: warning: unexpected end of data at offset 0x3ed while reading [0x3e6, 0x3ee)
 # MORE-ERR-NEXT: warning: last sequence in debug line table at offset 0x0000039d is not terminated
 
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_pub_tables_error_cases.s b/llvm/test/tools/llvm-dwarfdump/X86/debug_pub_tables_error_cases.s
new file mode 100644
index 0000000000000..8daea1e0f80d7
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_pub_tables_error_cases.s
@@ -0,0 +1,150 @@
+# RUN: llvm-mc -triple x86_64 %s -filetype=obj -o %t
+
+## All four name lookup table sections share the same parser, but slightly
+## different code paths are used to reach it. Do a comprehensive check for one
+## of the sections and minimal checks for the others.
+
+# RUN: not llvm-dwarfdump -debug-gnu-pubnames %t 2> %t.err | FileCheck %s
+# RUN: FileCheck %s --input-file=%t.err --check-prefix=ERR
+
+# RUN: not llvm-dwarfdump -debug-pubnames -debug-pubtypes -debug-gnu-pubtypes %t 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ERR-MIN
+
+    .section .debug_gnu_pubnames,"",@progbits
+# CHECK:      .debug_gnu_pubnames contents:
+
+## The next few sets do not contain all required fields in the header.
+# ERR: error: name lookup table at offset 0x0 does not have a complete header: unexpected end of data at offset 0x5 while reading [0x4, 0x6)
+# CHECK-NEXT: length = 0x00000001, format = DWARF32, version = 0x0000, unit_offset = 0x00000000, unit_size = 0x00000000
+# CHECK-NEXT: Offset Linkage Kind Name
+# CHECK-NOT:  0x
+    .long .LSet0End-.LSet0      # Length
+.LSet0:
+    .byte 1                     # Version (truncated)
+.LSet0End:
+
+# ERR: error: name lookup table at offset 0x5 does not have a complete header: unexpected end of data at offset 0xe while reading [0xb, 0xf)
+# CHECK-NEXT: length = 0x00000005, format = DWARF32, version = 0x0002, unit_offset = 0x00000000, unit_size = 0x00000000
+# CHECK-NEXT: Offset Linkage Kind Name
+# CHECK-NOT:  0x
+    .long .LSet1End-.LSet1      # Length
+.LSet1:
+    .short 2                    # Version
+    .byte 1, 2, 3               # Debug Info Offset (truncated)
+.LSet1End:
+
+# ERR: error: name lookup table at offset 0xe does not have a complete header: unexpected end of data at offset 0x1b while reading [0x18, 0x1c)
+# CHECK-NEXT: length = 0x00000009, format = DWARF32, version = 0x0002, unit_offset = 0x00000032, unit_size = 0x00000000
+# CHECK-NEXT: Offset Linkage Kind Name
+# CHECK-NOT:  0x
+    .long .LSet2End-.LSet2      # Length
+.LSet2:
+    .short 2                    # Version
+    .long 0x32                  # Debug Info Offset
+    .byte 1, 2, 3               # Debug Info Length (truncated)
+.LSet2End:
+
+## This set is terminated just after the header.
+# ERR: error: name lookup table at offset 0x1b parsing failed: unexpected end of data at offset 0x29 while reading [0x29, 0x2d)
+# CHECK-NEXT: length = 0x0000000a, format = DWARF32, version = 0x0002, unit_offset = 0x00000048, unit_size = 0x00000064
+# CHECK-NEXT: Offset Linkage Kind Name
+# CHECK-NOT:  0x
+    .long .LSet3End-.LSet3      # Length
+.LSet3:
+    .short 2                    # Version
+    .long 0x48                  # Debug Info Offset
+    .long 0x64                  # Debug Info Length
+.LSet3End:
+
+## The offset in the first pair is truncated.
+# ERR: error: name lookup table at offset 0x29 parsing failed: unexpected end of data at offset 0x3a while reading [0x37, 0x3b)
+# CHECK-NEXT: length = 0x0000000d, format = DWARF32, version = 0x0002, unit_offset = 0x000000ac, unit_size = 0x00000036
+# CHECK-NEXT: Offset Linkage Kind Name
+# CHECK-NOT:  0x
+    .long .LSet4End-.LSet4      # Length
+.LSet4:
+    .short 2                    # Version
+    .long 0xac                  # Debug Info Offset
+    .long 0x36                  # Debug Info Length
+    .byte 1, 2, 3               # Offset (truncated)
+.LSet4End:
+
+## The set is truncated just after the offset of the first pair.
+# ERR: error: name lookup table at offset 0x3a parsing failed: unexpected end of data at offset 0x4c while reading [0x4c, 0x4d)
+# CHECK-NEXT: length = 0x0000000e, format = DWARF32, version = 0x0002, unit_offset = 0x000000e2, unit_size = 0x00000015
+# CHECK-NEXT: Offset Linkage Kind Name
+# CHECK-NOT:  0x
+    .long .LSet5End-.LSet5      # Length
+.LSet5:
+    .short 2                    # Version
+    .long 0xe2                  # Debug Info Offset
+    .long 0x15                  # Debug Info Length
+    .long 0xf4                  # Offset
+.LSet5End:
+
+## The set is truncated just after the index entry field of the first pair.
+# ERR: error: name lookup table at offset 0x4c parsing failed: no null terminated string at offset 0x5f
+# CHECK-NEXT: length = 0x0000000f, format = DWARF32, version = 0x0002, unit_offset = 0x000000f7, unit_size = 0x00000010
+# CHECK-NEXT: Offset Linkage Kind Name
+# CHECK-NOT:  0x
+    .long .LSet6End-.LSet6      # Length
+.LSet6:
+    .short 2                    # Version
+    .long 0xf7                  # Debug Info Offset
+    .long 0x10                  # Debug Info Length
+    .long 0xf4                  # Offset
+    .byte 0x30                  # Index Entry
+.LSet6End:
+
+## This set contains a string which is not properly terminated.
+# ERR: error: name lookup table at offset 0x5f parsing failed: no null terminated string at offset 0x72
+# CHECK-NEXT: length = 0x00000012, format = DWARF32, version = 0x0002, unit_offset = 0x00000107, unit_size = 0x0000004b
+# CHECK-NEXT: Offset Linkage Kind Name
+# CHECK-NOT:  0x
+    .long .LSet7End-.LSet7      # Length
+.LSet7:
+    .short 2                    # Version
+    .long 0x107                 # Debug Info Offset
+    .long 0x4b                  # Debug Info Length
+    .long 0x111                 # Offset
+    .byte 0x30                  # Index Entry
+    .ascii "foo"                # The string does not terminate before the set data ends.
+.LSet7End:
+
+## This set occupies some space after the terminator.
+# ERR: error: name lookup table at offset 0x75 has a terminator at offset 0x8c before the expected end at 0x8d
+# CHECK-NEXT: length = 0x00000018, format = DWARF32, version = 0x0002, unit_offset = 0x00000154, unit_size = 0x000002ac
+# CHECK-NEXT: Offset Linkage Kind Name
+# CHECK-NEXT: 0x0000018e EXTERNAL FUNCTION "foo"
+# CHECK-NOT:  0x
+    .long .LSet8End-.LSet8      # Length
+.LSet8:
+    .short 2                    # Version
+    .long 0x154                 # Debug Info Offset
+    .long 0x2ac                 # Debug Info Length
+    .long 0x18e                 # Offset
+    .byte 0x30                  # Index Entry
+    .asciz "foo"                # Name
+    .long 0                     # Terminator
+    .space 1
+.LSet8End:
+
+## The remaining space in the section is too short to even contain a unit length
+## field.
+# ERR: error: name lookup table at offset 0x91 parsing failed: unexpected end of data at offset 0x94 while reading [0x91, 0x95)
+# CHECK-NOT:  length =
+    .space 3
+
+# ERR-MIN:      .debug_pubnames contents:
+# ERR-MIN-NEXT: error: name lookup table at offset 0x0 parsing failed: unexpected end of data at offset 0x1 while reading [0x0, 0x4)
+# ERR-MIN:      .debug_pubtypes contents:
+# ERR-MIN-NEXT: error: name lookup table at offset 0x0 parsing failed: unexpected end of data at offset 0x1 while reading [0x0, 0x4)
+# ERR-MIN:      .debug_gnu_pubtypes contents:
+# ERR-MIN-NEXT: error: name lookup table at offset 0x0 parsing failed: unexpected end of data at offset 0x1 while reading [0x0, 0x4)
+
+    .section .debug_pubnames,"",@progbits
+    .byte 0
+    .section .debug_pubtypes,"",@progbits
+    .byte 0
+    .section .debug_gnu_pubtypes,"",@progbits
+    .byte 0
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/verify_overlapping_cu_ranges.yaml b/llvm/test/tools/llvm-dwarfdump/X86/verify_overlapping_cu_ranges.yaml
new file mode 100644
index 0000000000000..fa6b2fbc8ddaf
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/verify_overlapping_cu_ranges.yaml
@@ -0,0 +1,259 @@
+# This test verifies that if a DW_TAG_compile_unit has DW_AT_ranges that
+# overlap, that it doesn't end up producing invalid errors claiming a child
+# DW_TAG_subprogram DIE is not in the parant (CU) ranges. Prior to the commit
+# that fixed this, a loop was iterating over all DW_AT_ranges for a DIE and
+# stopping the loop if any intersecting ranges were found. This would cause
+# the DW_TAG_subprogram DIEs, like "stripped2" and "main", to improperly report
+# that they were not contained in the parent's address ranges
+#
+# The DWARF looks like:
+# 0x0000000b: DW_TAG_compile_unit
+#               DW_AT_name	("/tmp/main.c")
+#               DW_AT_language	(DW_LANG_C)
+#               DW_AT_low_pc	(0x0000000000000000)
+#               DW_AT_ranges	(0x00000000
+#                  [0x0000000000002000, 0x0000000000003000)
+#                  [0x0000000000000000, 0x0000000000000020)
+#                  [0x0000000000000000, 0x0000000000000030)
+#                  [0x0000000000001000, 0x0000000000002000))
+#
+# 0x0000001e:   DW_TAG_subprogram
+#                 DW_AT_name	("stripped1")
+#                 DW_AT_low_pc	(0x0000000000000000)
+#                 DW_AT_high_pc	(0x0000000000000020)
+#
+# 0x0000002f:   DW_TAG_subprogram
+#                 DW_AT_name	("stripped2")
+#                 DW_AT_low_pc	(0x0000000000000000)
+#                 DW_AT_high_pc	(0x0000000000000030)
+#
+# 0x00000044:   DW_TAG_subprogram
+#                 DW_AT_name	("main")
+#                 DW_AT_low_pc	(0x0000000000001000)
+#                 DW_AT_high_pc	(0x0000000000002000)
+#
+# 0x00000055:   DW_TAG_subprogram
+#                 DW_AT_name	("foo")
+#                 DW_AT_low_pc	(0x0000000000002000)
+#                 DW_AT_high_pc	(0x0000000000003000)
+#
+# 0x00000066:   NULL
+
+# RUN: yaml2obj %s | not llvm-dwarfdump --verify - | FileCheck %s --implicit-check-not=error:
+
+# CHECK: error: DIE has overlapping ranges in DW_AT_ranges attribute: [0x0000000000000000, 0x0000000000000020) and [0x0000000000000000, 0x0000000000000030)
+
+# CHECK: 0x0000000b:   DW_TAG_compile_unit
+# CHECK-NEXT:                DW_AT_name      ("/tmp/main.c")
+# CHECK-NEXT:                DW_AT_language  (DW_LANG_C)
+# CHECK-NEXT:                DW_AT_low_pc    (0x0000000000000000)
+# CHECK-NEXT:                DW_AT_ranges    (0x00000000
+# CHECK-NEXT:                   [0x0000000000002000, 0x0000000000003000)
+# CHECK-NEXT:                   [0x0000000000000000, 0x0000000000000020)
+# CHECK-NEXT:                   [0x0000000000000000, 0x0000000000000030)
+# CHECK-NEXT:                   [0x0000000000001000, 0x0000000000002000))
+
+# CHECK: error: DIEs have overlapping address ranges:
+# CHECK: 0x0000002f: DW_TAG_subprogram
+# CHECK-NEXT:              DW_AT_name        ("stripped2")
+# CHECK-NEXT:              DW_AT_low_pc      (0x0000000000000000)
+# CHECK-NEXT:              DW_AT_high_pc     (0x0000000000000030)
+
+# CHECK: 0x0000001e: DW_TAG_subprogram
+# CHECK-NEXT:              DW_AT_name        ("stripped1")
+# CHECK-NEXT:              DW_AT_low_pc      (0x0000000000000000)
+# CHECK-NEXT:              DW_AT_high_pc     (0x0000000000000020)
+
+# CHECK: Verifying .debug_info references...
+
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x01000007
+  cpusubtype:      0x00000003
+  filetype:        0x00000001
+  ncmds:           4
+  sizeofcmds:      464
+  flags:           0x00002000
+  reserved:        0x00000000
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         392
+    segname:         ''
+    vmaddr:          0
+    vmsize:          261
+    fileoff:         528
+    filesize:        261
+    maxprot:         7
+    initprot:        7
+    nsects:          4
+    flags:           0
+    Sections:
+      - sectname:        __debug_abbrev
+        segname:         __DWARF
+        addr:            0x0000000000000000
+        size:            36
+        offset:          0x00000210
+        align:           0
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x00000000
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+        content:         011101030E1305110155170000022E00030E110112060000032E00030E11011201000000
+      - sectname:        __debug_info
+        segname:         __DWARF
+        addr:            0x0000000000000024
+        size:            103
+        offset:          0x00000234
+        align:           0
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x00000000
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+        content:         630000000400000000000801010000000200000000000000000000000000020D0000000000000000000000200000000317000000000000000000000030000000000000000221000000001000000000000000100000022600000000200000000000000010000000
+      - sectname:        __debug_ranges
+        segname:         __DWARF
+        addr:            0x000000000000008B
+        size:            80
+        offset:          0x0000029B
+        align:           0
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x00000000
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+        content:         '0020000000000000003000000000000000000000000000002000000000000000000000000000000030000000000000000010000000000000002000000000000000000000000000000000000000000000'
+      - sectname:        __debug_str
+        segname:         __DWARF
+        addr:            0x00000000000000DB
+        size:            42
+        offset:          0x000002EB
+        align:           0
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x00000000
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+        content:         002F746D702F6D61696E2E630073747269707065643100737472697070656432006D61696E00666F6F00
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          0
+    nsyms:           0
+    stroff:          792
+    strsize:         8
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         32
+    platform:        1
+    minos:           658944
+    sdk:             658944
+    ntools:          1
+    Tools:
+      - tool:            3
+        version:         34734080
+  - cmd:             LC_DATA_IN_CODE
+    cmdsize:         16
+    dataoff:         792
+    datasize:        0
+LinkEditData:
+  StringTable:
+    - ' '
+    - ''
+    - ''
+    - ''
+    - ''
+    - ''
+    - ''
+DWARF:
+  debug_str:
+    - ''
+    - '/tmp/main.c'
+    - stripped1
+    - stripped2
+    - main
+    - foo
+  debug_abbrev:
+    - Code:            0x00000001
+      Tag:             DW_TAG_compile_unit
+      Children:        DW_CHILDREN_yes
+      Attributes:
+        - Attribute:       DW_AT_name
+          Form:            DW_FORM_strp
+        - Attribute:       DW_AT_language
+          Form:            DW_FORM_data2
+        - Attribute:       DW_AT_low_pc
+          Form:            DW_FORM_addr
+        - Attribute:       DW_AT_ranges
+          Form:            DW_FORM_sec_offset
+    - Code:            0x00000002
+      Tag:             DW_TAG_subprogram
+      Children:        DW_CHILDREN_no
+      Attributes:
+        - Attribute:       DW_AT_name
+          Form:            DW_FORM_strp
+        - Attribute:       DW_AT_low_pc
+          Form:            DW_FORM_addr
+        - Attribute:       DW_AT_high_pc
+          Form:            DW_FORM_data4
+    - Code:            0x00000003
+      Tag:             DW_TAG_subprogram
+      Children:        DW_CHILDREN_no
+      Attributes:
+        - Attribute:       DW_AT_name
+          Form:            DW_FORM_strp
+        - Attribute:       DW_AT_low_pc
+          Form:            DW_FORM_addr
+        - Attribute:       DW_AT_high_pc
+          Form:            DW_FORM_addr
+  debug_ranges:
+    - Offset:          0x00000000
+      AddrSize:        0x08
+      Entries:
+        - LowOffset:       0x0000000000002000
+          HighOffset:      0x0000000000003000
+        - LowOffset:       0x0000000000000000
+          HighOffset:      0x0000000000000020
+        - LowOffset:       0x0000000000000000
+          HighOffset:      0x0000000000000030
+        - LowOffset:       0x0000000000001000
+          HighOffset:      0x0000000000002000
+  debug_info:
+    - Length:          99
+      Version:         4
+      AbbrOffset:      0
+      AddrSize:        8
+      Entries:
+        - AbbrCode:        0x00000001
+          Values:
+            - Value:           0x0000000000000001
+            - Value:           0x0000000000000002
+            - Value:           0x0000000000000000
+            - Value:           0x0000000000000000
+        - AbbrCode:        0x00000002
+          Values:
+            - Value:           0x000000000000000D
+            - Value:           0x0000000000000000
+            - Value:           0x0000000000000020
+        - AbbrCode:        0x00000003
+          Values:
+            - Value:           0x0000000000000017
+            - Value:           0x0000000000000000
+            - Value:           0x0000000000000030
+        - AbbrCode:        0x00000002
+          Values:
+            - Value:           0x0000000000000021
+            - Value:           0x0000000000001000
+            - Value:           0x0000000000001000
+        - AbbrCode:        0x00000002
+          Values:
+            - Value:           0x0000000000000026
+            - Value:           0x0000000000002000
+            - Value:           0x0000000000001000
+        - AbbrCode:        0x00000000
+          Values:          []
+...
diff --git a/llvm/test/tools/llvm-dwarfdump/cmdline.test b/llvm/test/tools/llvm-dwarfdump/cmdline.test
index 9bf796ffa40b4..5c7b998338c72 100644
--- a/llvm/test/tools/llvm-dwarfdump/cmdline.test
+++ b/llvm/test/tools/llvm-dwarfdump/cmdline.test
@@ -35,4 +35,4 @@ RUN: llvm-dwarfdump -diff -verbose 2>&1 | FileCheck --check-prefix=INCOMPATIBLE
 INCOMPATIBLE: error: incompatible arguments: specifying both -diff and -verbose is currently not supported
 
 RUN: not llvm-dwarfdump --debug-names=0x0 2>&1 | FileCheck --check-prefix=FLAG %s
-FLAG: for the --debug-names option: this is a flag and does not take a value.
+FLAG: for the --debug-names option: this is a flag and does not take a value
diff --git a/llvm/test/tools/llvm-dwarfdump/lit.local.cfg b/llvm/test/tools/llvm-dwarfdump/lit.local.cfg
deleted file mode 100644
index ec8ad451d2da0..0000000000000
--- a/llvm/test/tools/llvm-dwarfdump/lit.local.cfg
+++ /dev/null
@@ -1 +0,0 @@
-config.suffixes = ['.test', '.ll', '.s', '.yaml']
diff --git a/llvm/test/tools/llvm-gsymutil/ARM_AArch64/fat-macho-dwarf.yaml b/llvm/test/tools/llvm-gsymutil/ARM_AArch64/fat-macho-dwarf.yaml
index 3693075b8b859..aef062428e8f4 100644
--- a/llvm/test/tools/llvm-gsymutil/ARM_AArch64/fat-macho-dwarf.yaml
+++ b/llvm/test/tools/llvm-gsymutil/ARM_AArch64/fat-macho-dwarf.yaml
@@ -407,8 +407,7 @@ Slices:
           - DieOffset:       0x0000006F
             Name:            char
       debug_info:
-        - Length:
-            TotalLength:     115
+        - Length:          115
           Version:         4
           AbbrOffset:      0
           AddrSize:        4
@@ -886,8 +885,7 @@ Slices:
           - DieOffset:       0x00000076
             Name:            char
       debug_info:
-        - Length:
-            TotalLength:     122
+        - Length:          122
           Version:         4
           AbbrOffset:      0
           AddrSize:        8
diff --git a/llvm/test/tools/llvm-gsymutil/ARM_AArch64/lit.local.cfg b/llvm/test/tools/llvm-gsymutil/ARM_AArch64/lit.local.cfg
index e06c15ef14138..8a995e36a127d 100644
--- a/llvm/test/tools/llvm-gsymutil/ARM_AArch64/lit.local.cfg
+++ b/llvm/test/tools/llvm-gsymutil/ARM_AArch64/lit.local.cfg
@@ -1,4 +1,2 @@
 if not ('ARM' in config.root.targets and 'AArch64' in config.root.targets):
     config.unsupported = True
-
-config.suffixes = ['.test', '.yaml']
diff --git a/llvm/test/tools/llvm-gsymutil/X86/lit.local.cfg b/llvm/test/tools/llvm-gsymutil/X86/lit.local.cfg
index 52c762f5cfb8b..c8625f4d9d248 100644
--- a/llvm/test/tools/llvm-gsymutil/X86/lit.local.cfg
+++ b/llvm/test/tools/llvm-gsymutil/X86/lit.local.cfg
@@ -1,4 +1,2 @@
 if not 'X86' in config.root.targets:
     config.unsupported = True
-
-config.suffixes = ['.test', '.yaml']
diff --git a/llvm/test/tools/llvm-gsymutil/X86/mach-dwarf.yaml b/llvm/test/tools/llvm-gsymutil/X86/mach-dwarf.yaml
index d68d330745ac4..1368257964166 100644
--- a/llvm/test/tools/llvm-gsymutil/X86/mach-dwarf.yaml
+++ b/llvm/test/tools/llvm-gsymutil/X86/mach-dwarf.yaml
@@ -574,8 +574,7 @@ DWARF:
       - DieOffset:       0x000000D8
         Name:            char
   debug_info:
-    - Length:
-        TotalLength:     220
+    - Length:          220
       Version:         4
       AbbrOffset:      0
       AddrSize:        8
diff --git a/llvm/test/tools/llvm-mca/ARM/m4-int.s b/llvm/test/tools/llvm-mca/ARM/m4-int.s
new file mode 100644
index 0000000000000..b46f731f0793d
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/ARM/m4-int.s
@@ -0,0 +1,1321 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=thumbv7-m-none-none-eabi -mcpu=cortex-m4 -instruction-tables < %s | FileCheck %s
+
+adc r0, r1, #0
+adcs r0, r1, #0
+adcs r0, r1
+adc.w r0, r1, r2
+adcs.w r0, r1, r2
+adc.w r0, r1, r2, LSL #1
+adcs.w r0, r1, r2, LSL #1
+add r0, sp, #1
+add sp, #1
+add.w r0, sp, #1
+adds.w r0, sp, #1
+addw r0, sp, #1
+add r0, sp, r0
+add sp, r1
+add.w r0, sp, r1
+adds.w r0, sp, r1
+add.w r0, sp, r1, LSL #1
+adds.w r0, sp, r1, LSL #1
+adds r0, r1, #1
+adds r0, #42
+add.w r0, r1, #1
+adds.w r0, r1, #1
+addw r0, r1, #1
+adds r0, r1, r2
+add r0, r1
+add.w r0, r1, r2
+adds.w r0, r1, r2
+add.w r0, r1, r2, LSL #1
+adds.w r0, r1, r2, LSL #1
+adr r0, #-6
+adr r8, #-6
+adr.w r0, #-6
+and r0, r1, #1
+ands r0, r1, #1
+ands r1, r0
+and.w r0, r1, r2
+ands.w r0, r1, r2
+and.w r0, r1, r2, LSL #1
+ands.w r0, r1, r2, LSL #1
+asrs r0, r1, #1
+asr.w r0, r1, #1
+asrs.w r0, r1, #1
+asrs r0, r1
+asr.w r0, r1, r2
+asrs.w r0, r1, r2
+bfc r0, #1, #2
+bfi r0, r1, #1, #2
+bic r0, r1, #1
+bics r0, r1, #1
+bics r0, r1
+bic.w r0, r1, r2
+bics.w r0, r1, r2
+bic.w r0, r1, r2, LSL #1
+bics.w r0, r1, r2, LSL #1
+bkpt #1
+clrex
+clz r0, r1
+cmn r0, #1
+cmn r0, r1
+cmn.w r0, r1
+cmn.w r0, r1, LSL #1
+cmp r0, #1
+cmp.w r0, #1
+cmp r0, r1
+cmp r0, r10
+cmp.w r0, r1
+cmp.w r0, r1, LSL #1
+#cpsdb 1
+#cpsie if
+#dbg #1
+dmb
+dsb
+eor r0, r1, #1
+eors r0, r1, #1
+eors r0, r1
+eor.w r0, r1, r2
+eors.w r0, r1, r2
+eor.w r0, r1, r2, LSL #1
+eors.w r0, r1, r2, LSL #1
+isb
+ldm r0!, {r1}
+ldm r0, {r1}
+ldm.w r0, {r1}
+ldm.w r0!, {r1}
+ldmdb r0, {r1}
+ldmdb r0!, {r1}
+ldr r0, [r1, #4]
+ldr r0, [sp, #4]
+ldr.w r0, [r1, #4]
+ldr r0, [r1, #-1]
+ldr r0, [r1], #1
+ldr r0, [r1, #1]!
+ldr r0, #4
+ldr.w r0, #4
+ldr r0, next
+ldr.w r0, next
+ldr r0, [r1, r2]
+ldr.w r0, [r1, r2]
+ldr.w r0, [r1, r2, LSL #1]
+ldrb r0, [r1, #1]
+ldrb.w r0, [r1, #1]
+ldrb r0, [r1, #-1]
+ldrb r0, [r1], #1
+ldrb r0, [r1, #1]!
+ldrb r0, #4
+ldrb r0, next
+ldrb r0, [r1, r2]
+ldrb.w r0, [r1, r2]
+ldrb.w r0, [r1, r2, LSL #1]
+ldrbt r0, [r1, #1]
+ldrd r0, r2, [r1]
+ldrd r0, r2, [r1, #-4]
+ldrd r0, r2, [r1], #4
+ldrd r0, r2, [r1, #4]!
+ldrd r0, r2, next
+next:
+ldrex r0, [r1]
+ldrex r0, [r1, #4]
+ldrexb r0, [r1]
+ldrexh r0, [r1]
+ldrh r0, [r1, #2]
+ldrh.w r0, [r1, #1]
+ldrh r0, [r1, #-1]
+ldrh r0, [r1], #1
+ldrh r0, [r1, #1]!
+ldrh r0, #4
+ldrh r0, next
+ldrh r0, [r1, r2]
+ldrh.w r0, [r1, r2]
+ldrh.w r0, [r1, r2, LSL #1]
+ldrht r0, [r1, #1]
+ldrsb r0, [r1, #1]
+ldrsb r0, [r1, #-1]
+ldrsb r0, [r1], #1
+ldrsb r0, [r1, #1]!
+ldrsb r0, #4
+ldrsb r0, next
+ldrsb r0, [r1, r2]
+ldrsb.w r0, [r1, r2]
+ldrsb.w r0, [r1, r2, LSL #1]
+ldrsbt r0, [r1, #1]
+ldrsh r0, [r1, #2]
+ldrsh r0, [r1, #-1]
+ldrsh r0, [r1], #1
+ldrsh r0, [r1, #1]!
+ldrsh r0, #4
+ldrsh r0, next
+ldrsh r0, [r1, r2]
+ldrsh.w r0, [r1, r2]
+ldrsh.w r0, [r1, r2, LSL #1]
+ldrsht r0, [r1, #1]
+ldrt r0, [r1, #1]
+lsls r0, r1, #1
+lsl.w r0, r1, #1
+lsls.w r0, r1, #1
+lsls r0, r1
+lsl.w r0, r1, r2
+lsls.w r0, r1, r2
+lsrs r0, r1, #1
+lsr.w r0, r1, #1
+lsrs.w r0, r1, #1
+lsrs r0, r1
+lsr.w r0, r1, r2
+lsrs.w r0, r1, r2
+mla r0, r1, r2, r3
+mls r0, r1, r2, r3
+movs r0, #1
+mov.w r0, #1
+movs.w r0, #1
+movw r0, #1
+mov r0, r1
+#movs r0, r1
+mov.w r0, r1
+movs.w r0, r1
+movt r0, #1
+mrs r0, apsr
+msr apsr, r0
+muls r1, r2, r1
+mul r0, r1, r2
+mvn r0, #1
+mvns r0, #1
+mvns r0, r1
+mvn.w r0, r1
+mvns.w r0, r1
+mvn.w r0, r1, LSL #1
+mvns.w r0, r1, LSL #1
+nop
+orn r0, r1, #1
+orns r0, r1, #1
+orn r0, r1, r2
+orns r0, r1, r2
+orn r0, r1, r2, LSL #1
+orns r0, r1, r2, LSL #1
+orr r0, r1, #1
+orrs r0, r1, #1
+orrs r0, r1
+orr r0, r1, r2
+orrs r0, r1, r2
+orr r0, r1, r2, LSL #1
+orrs r0, r1, r2, LSL #1
+pkhbt r0, r1, r2
+pkhbt r0, r1, r2, LSL #1
+pkhtb r0, r1, r2
+pkhtb r0, r1, r2, ASR #1
+pop { r0 }
+pop.w { r0, r1 }
+pop.w { r0 }
+pssbb
+push { r0 }
+push.w { r0, r1 }
+push.w { r0 }
+qadd r0, r1, r2
+qadd16 r0, r1, r2
+qadd8 r0, r1, r2
+qasx r0, r1, r2
+qdadd r0, r1, r2
+qdsub r0, r1, r2
+qsax r0, r1, r2
+qsub r0, r1, r2
+qsub16 r0, r1, r2
+qsub8 r0, r1, r2
+rbit r0, r1
+rev r0, r1
+rev.w r0, r1
+rev16 r0, r1
+rev16.w r0, r1
+revsh r0, r1
+revsh.w r0, r1
+ror r0, r1, #1
+rors r0, r1, #1
+rors r0, r1
+ror.w r0, r1, r2
+rors.w r0, r1, r2
+rrx r0, r1
+rrxs r0, r1
+rsbs r0, r1, #0
+rsb.w r0, r1, #1
+rsbs.w r0, r1, #1
+rsb r0, r1, r2
+rsbs r0, r1, r2
+rsb r0, r1, r2, LSL #1
+rsbs r0, r1, r2, LSL #1
+sadd16 r0, r1, r2
+sadd8 r0, r1, r2
+sasx r0, r1, r2
+sbc r0, r1, #1
+sbcs r0, r1, #1
+sbcs r0, r1
+sbc r0, r1, r2
+sbcs r0, r1, r2
+sbc r0, r1, r2, LSL #1
+sbcs r0, r1, r2, LSL #1
+sbfx r0, r1, #1, #2
+sdiv r0, r1, r2
+sel r0, r1, r2
+sev
+#sg
+shadd16 r0, r1, r2
+shadd8 r0, r1, r2
+shasx r0, r1, r2
+shsax r0, r1, r2
+shsub16 r0, r1, r2
+shsub8 r0, r1, r2
+smlabb r0, r1, r2, r3
+smlabt r0, r1, r2, r3
+smlatb r0, r1, r2, r3
+smlatt r0, r1, r2, r3
+smlad r0, r1, r2, r3
+smladx r0, r1, r2, r3
+smlal r0, r1, r2, r3
+smlalbb r0, r1, r2, r3
+smlalbt r0, r1, r2, r3
+smlaltb r0, r1, r2, r3
+smlaltt r0, r1, r2, r3
+smlald r0, r1, r2, r3
+smlaldx r0, r1, r2, r3
+smlawb r0, r1, r2, r3
+smlawt r0, r1, r2, r3
+smlsd r0, r1, r2, r3
+smlsdx r0, r1, r2, r3
+smlsld r0, r1, r2, r3
+smlsldx r0, r1, r2, r3
+smmla r0, r1, r2, r3
+smmlar r0, r1, r2, r3
+smmls r0, r1, r2, r3
+smmlsr r0, r1, r2, r3
+smmul r0, r1, r2
+smmulr r0, r1, r2
+smuad r0, r1, r2
+smuadx r0, r1, r2
+smulbb r0, r1, r2
+smulbt r0, r1, r2
+smultb r0, r1, r2
+smultt r0, r1, r2
+smull r0, r1, r2, r3
+smulwb r0, r1, r2
+smulwt r0, r1, r2
+smusd r0, r1, r2
+smusdx r0, r1, r2
+ssat r0, #1, r2
+ssat r0, #1, r2, LSL #1
+ssat16 r0, #1, r1
+ssax r0, r1, r2
+ssbb
+ssub16 r0, r1, r2
+ssub8 r0, r1, r2
+stm r0!, { r1 }
+stm.w r0, { r1 }
+stm.w r0!, { r1 }
+stmdb r0, { r1 }
+stmdb r0!, { r1 }
+str r0, [ r1 ]
+str r0, [ r1, #4 ]
+str r0, [ sp, #4 ]
+str.w r0, [ r1, #1 ]
+str r0, [ r1, #-1 ]
+str r0, [ r1 ], #1
+#str r0, [ r1, #1 ]!
+str r0, [ r1, r2 ]
+str.w r0, [ r1, r2 ]
+str.w r0, [ r1, r2, LSL #1 ]
+strb r0, [ r1 ]
+strb r0, [ r1, #1 ]
+strb.w r0, [ r1, #1 ]
+strb r0, [ r1, #-1 ]
+strb r0, [ r1 ], #1
+strb r0, [ r1, #1 ]!
+strb r0, [ r1, r2 ]
+strb.w r0, [ r1, r2 ]
+strb.w r0, [ r1, r2, LSL #1 ]
+strbt r0, [ r1, #1 ]
+strd r0, r1, [ r2, #4 ]
+strd r0, r1, [ r2 ], #4
+strd r0, r1, [ r2, #4 ]!
+strex r0, r1, [ r2 ]
+strex r0, r1, [ r2, #4 ]
+strexb r0, r1, [ r2 ]
+strexh r0, r1, [ r2 ]
+strh r0, [ r1 ]
+strh r0, [ r1, #2 ]
+strh.w r0, [ r1, #2 ]
+strh r0, [ r1, #-1 ]
+strh r0, [ r1 ], #1
+strh r0, [ r1, #1 ]!
+strh r0, [ r1, r2 ]
+strh.w r0, [ r1, r2 ]
+strh.w r0, [ r1, r2, LSL #1 ]
+strht r0, [r1, #1 ]
+strt r0, [r1, #1 ]
+sub sp, sp, #4
+sub.w r0, sp, #1
+subs.w r0, sp, #1
+subw r0, sp, #1
+sub r0, sp, r1
+subs r0, sp, r1
+sub r0, sp, r1, LSL #1
+subs r0, sp, r1, LSL #1
+subs r0, r1, #1
+subs r0, #1
+sub.w r0, r1, #1
+subs.w r0, r1, #1
+subw r0, r1, #1
+subs r0, r1, r2
+sub.w r0, r1, r2
+subs.w r0, r1, r2
+sub.w r0, r1, r2, LSL #1
+subs.w r0, r1, r2, LSL #1
+#svc #1		; treated as a call
+sxtab r0, r1, r2
+sxtab r0, r1, r2, ROR #8
+sxtab16 r0, r1, r2
+sxtab16 r0, r1, r2, ROR #8
+sxtah r0, r1, r2
+sxtah r0, r1, r2, ROR #8
+sxtb r0, r1
+sxtb.w r0, r1
+sxtb.w r0, r1, ROR #8
+sxtb16 r0, r1
+sxtb16 r0, r1, ROR #8
+sxth r0, r1
+sxth.w r0, r1
+sxth.w r0, r1, ROR #8
+tbb [r0, r1]
+tbh [r0, r1, LSL #1]
+teq r0, #1
+teq r0, r1
+teq r0, r1, LSL #1
+tst r0, #1
+tst r0, r1
+tst.w r0, r1
+tst.w r0, r1, LSL #1
+#tt r0, r1
+#ttt r0, r1
+#tta r0, r1
+#ttat r0, r1
+uadd16 r0, r1, r2
+uadd8 r0, r1, r2
+uasx r0, r1, r2
+ubfx r0, r1, #1, #2
+#udf #1
+udiv r0, r1, r2
+uhadd16 r0, r1, r2
+uhadd8 r0, r1, r2
+uhasx r0, r1, r2
+uhsax r0, r1, r2
+uhsub16 r0, r1, r2
+uhsub8 r0, r1, r2
+umaal r0, r1, r2, r3
+umlal r0, r1, r2, r3
+umull r0, r1, r2, r3
+uqadd16 r0, r1, r2
+uqadd8 r0, r1, r2
+uqasx r0, r1, r2
+uqsax r0, r1, r2
+uqsub16 r0, r1, r2
+uqsub8 r0, r1, r2
+usad8 r0, r1, r2
+usada8 r0, r1, r2, r3
+usat r0, #1, r1
+usat r0, #1, r1, LSL #1
+usat16 r0, #1, r1
+usax r0, r1, r2
+usub16 r0, r1, r2
+usub8 r0, r1, r2
+uxtab r0, r1, r2
+uxtab r0, r1, r2, ROR #8
+uxtab16 r0, r1, r2
+uxtab16 r0, r1, r2, ROR #8
+uxtah r0, r1, r2
+uxtah r0, r1, r2, ROR #8
+uxtb r0, r1
+uxtb.w r0, r1
+uxtb.w r0, r1, ROR #8
+uxtb16 r0, r1
+uxtb16 r0, r1, ROR #8
+uxth r0, r1
+uxth.w r0, r1
+uxth.w r0, r1, ROR #8
+wfe
+wfi
+forward:
+yield
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                        adc	r0, r1, #0
+# CHECK-NEXT:  1      1     1.00                        adcs	r0, r1, #0
+# CHECK-NEXT:  1      1     1.00                  U     adcs	r0, r1
+# CHECK-NEXT:  1      1     1.00                        adc.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        adcs.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        adc.w	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        adcs.w	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        add.w	r0, sp, #1
+# CHECK-NEXT:  1      1     1.00                  U     add.w	sp, sp, #1
+# CHECK-NEXT:  1      1     1.00                        add.w	r0, sp, #1
+# CHECK-NEXT:  1      1     1.00                        adds.w	r0, sp, #1
+# CHECK-NEXT:  1      1     1.00                        addw	r0, sp, #1
+# CHECK-NEXT:  1      1     1.00                  U     add	r0, sp, r0
+# CHECK-NEXT:  1      1     1.00                  U     add	sp, r1
+# CHECK-NEXT:  1      1     1.00                        add.w	r0, sp, r1
+# CHECK-NEXT:  1      1     1.00                        adds.w	r0, sp, r1
+# CHECK-NEXT:  1      1     1.00                        add.w	r0, sp, r1, lsl #1
+# CHECK-NEXT:  1      1     1.00                        adds.w	r0, sp, r1, lsl #1
+# CHECK-NEXT:  1      1     1.00                        adds	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        adds	r0, #42
+# CHECK-NEXT:  1      1     1.00                        add.w	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        adds.w	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        addw	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        adds	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        add	r0, r1
+# CHECK-NEXT:  1      1     1.00                        add.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        adds.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        add.w	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        adds.w	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                  U     adr.w	r0, #-6
+# CHECK-NEXT:  1      1     1.00                  U     adr.w	r8, #-6
+# CHECK-NEXT:  1      1     1.00                  U     adr.w	r0, #-6
+# CHECK-NEXT:  1      1     1.00                        and	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        ands	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        ands	r1, r0
+# CHECK-NEXT:  1      1     1.00                        and.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        ands.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        and.w	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        ands.w	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        asrs	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        asr.w	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        asrs.w	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        asrs	r0, r1
+# CHECK-NEXT:  1      1     1.00                        asr.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        asrs.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        bfc	r0, #1, #2
+# CHECK-NEXT:  1      1     1.00                        bfi	r0, r1, #1, #2
+# CHECK-NEXT:  1      1     1.00                        bic	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        bics	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        bics	r0, r1
+# CHECK-NEXT:  1      1     1.00                        bic.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        bics.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        bic.w	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        bics.w	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                  U     bkpt	#1
+# CHECK-NEXT:  1      1     1.00    *      *      U     clrex
+# CHECK-NEXT:  1      1     1.00                        clz	r0, r1
+# CHECK-NEXT:  1      1     1.00                        cmn.w	r0, #1
+# CHECK-NEXT:  1      1     1.00                        cmn	r0, r1
+# CHECK-NEXT:  1      1     1.00                        cmn.w	r0, r1
+# CHECK-NEXT:  1      1     1.00                        cmn.w	r0, r1, lsl #1
+# CHECK-NEXT:  1      1     1.00                        cmp	r0, #1
+# CHECK-NEXT:  1      1     1.00                        cmp.w	r0, #1
+# CHECK-NEXT:  1      1     1.00                        cmp	r0, r1
+# CHECK-NEXT:  1      1     1.00                  U     cmp	r0, r10
+# CHECK-NEXT:  1      1     1.00                        cmp.w	r0, r1
+# CHECK-NEXT:  1      1     1.00                        cmp.w	r0, r1, lsl #1
+# CHECK-NEXT:  1      1     1.00    *      *      U     dmb	sy
+# CHECK-NEXT:  1      1     1.00    *      *      U     dsb	sy
+# CHECK-NEXT:  1      1     1.00                        eor	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        eors	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        eors	r0, r1
+# CHECK-NEXT:  1      1     1.00                        eor.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        eors.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        eor.w	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        eors.w	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00    *      *      U     isb	sy
+# CHECK-NEXT:  1      2     1.00    *                   ldm	r0!, {r1}
+# CHECK-NEXT:  1      2     1.00    *                   ldm.w	r0, {r1}
+# CHECK-NEXT:  1      2     1.00    *                   ldm.w	r0, {r1}
+# CHECK-NEXT:  1      2     1.00    *                   ldr	r1, [r0], #4
+# CHECK-NEXT:  1      2     1.00    *                   ldmdb	r0, {r1}
+# CHECK-NEXT:  1      2     1.00    *                   ldmdb	r0!, {r1}
+# CHECK-NEXT:  1      2     1.00    *                   ldr	r0, [r1, #4]
+# CHECK-NEXT:  1      2     1.00    *                   ldr	r0, [sp, #4]
+# CHECK-NEXT:  1      2     1.00    *                   ldr.w	r0, [r1, #4]
+# CHECK-NEXT:  1      2     1.00    *                   ldr	r0, [r1, #-1]
+# CHECK-NEXT:  1      2     1.00    *                   ldr	r0, [r1], #1
+# CHECK-NEXT:  1      2     1.00    *                   ldr	r0, [r1, #1]!
+# CHECK-NEXT:  1      2     1.00    *                   ldr	r0, [pc, #4]
+# CHECK-NEXT:  1      2     1.00    *                   ldr.w	r0, [pc, #4]
+# CHECK-NEXT:  1      2     1.00    *                   ldr	r0, next
+# CHECK-NEXT:  1      2     1.00    *                   ldr.w	r0, next
+# CHECK-NEXT:  1      2     1.00    *                   ldr	r0, [r1, r2]
+# CHECK-NEXT:  1      2     1.00    *                   ldr.w	r0, [r1, r2]
+# CHECK-NEXT:  1      2     1.00    *                   ldr.w	r0, [r1, r2, lsl #1]
+# CHECK-NEXT:  1      2     1.00    *                   ldrb	r0, [r1, #1]
+# CHECK-NEXT:  1      2     1.00    *                   ldrb.w	r0, [r1, #1]
+# CHECK-NEXT:  1      2     1.00    *                   ldrb	r0, [r1, #-1]
+# CHECK-NEXT:  1      2     1.00    *                   ldrb	r0, [r1], #1
+# CHECK-NEXT:  1      2     1.00    *                   ldrb	r0, [r1, #1]!
+# CHECK-NEXT:  1      2     1.00    *                   ldrb.w	r0, [pc, #4]
+# CHECK-NEXT:  1      2     1.00    *                   ldrb.w	r0, next
+# CHECK-NEXT:  1      2     1.00    *                   ldrb	r0, [r1, r2]
+# CHECK-NEXT:  1      2     1.00    *                   ldrb.w	r0, [r1, r2]
+# CHECK-NEXT:  1      2     1.00    *                   ldrb.w	r0, [r1, r2, lsl #1]
+# CHECK-NEXT:  1      2     1.00                  U     ldrbt	r0, [r1, #1]
+# CHECK-NEXT:  1      2     1.00    *                   ldrd	r0, r2, [r1]
+# CHECK-NEXT:  1      2     1.00    *                   ldrd	r0, r2, [r1, #-4]
+# CHECK-NEXT:  1      2     1.00    *                   ldrd	r0, r2, [r1], #4
+# CHECK-NEXT:  1      2     1.00    *                   ldrd	r0, r2, [r1, #4]!
+# CHECK-NEXT:  1      2     1.00    *                   ldrd	r0, r2, next
+# CHECK-NEXT:  1      2     1.00    *      *      U     ldrex	r0, [r1]
+# CHECK-NEXT:  1      2     1.00    *      *      U     ldrex	r0, [r1, #4]
+# CHECK-NEXT:  1      2     1.00    *      *      U     ldrexb	r0, [r1]
+# CHECK-NEXT:  1      2     1.00    *      *      U     ldrexh	r0, [r1]
+# CHECK-NEXT:  1      2     1.00    *                   ldrh	r0, [r1, #2]
+# CHECK-NEXT:  1      2     1.00    *                   ldrh.w	r0, [r1, #1]
+# CHECK-NEXT:  1      2     1.00    *                   ldrh	r0, [r1, #-1]
+# CHECK-NEXT:  1      2     1.00    *                   ldrh	r0, [r1], #1
+# CHECK-NEXT:  1      2     1.00    *                   ldrh	r0, [r1, #1]!
+# CHECK-NEXT:  1      2     1.00    *                   ldrh.w	r0, [pc, #4]
+# CHECK-NEXT:  1      2     1.00    *                   ldrh.w	r0, next
+# CHECK-NEXT:  1      2     1.00    *                   ldrh	r0, [r1, r2]
+# CHECK-NEXT:  1      2     1.00    *                   ldrh.w	r0, [r1, r2]
+# CHECK-NEXT:  1      2     1.00    *                   ldrh.w	r0, [r1, r2, lsl #1]
+# CHECK-NEXT:  1      2     1.00                  U     ldrht	r0, [r1, #1]
+# CHECK-NEXT:  1      2     1.00    *                   ldrsb.w	r0, [r1, #1]
+# CHECK-NEXT:  1      2     1.00    *                   ldrsb	r0, [r1, #-1]
+# CHECK-NEXT:  1      2     1.00    *                   ldrsb	r0, [r1], #1
+# CHECK-NEXT:  1      2     1.00    *                   ldrsb	r0, [r1, #1]!
+# CHECK-NEXT:  1      2     1.00    *                   ldrsb.w	r0, [pc, #4]
+# CHECK-NEXT:  1      2     1.00    *                   ldrsb.w	r0, next
+# CHECK-NEXT:  1      2     1.00    *                   ldrsb	r0, [r1, r2]
+# CHECK-NEXT:  1      2     1.00    *                   ldrsb.w	r0, [r1, r2]
+# CHECK-NEXT:  1      2     1.00    *                   ldrsb.w	r0, [r1, r2, lsl #1]
+# CHECK-NEXT:  1      2     1.00                  U     ldrsbt	r0, [r1, #1]
+# CHECK-NEXT:  1      2     1.00    *                   ldrsh.w	r0, [r1, #2]
+# CHECK-NEXT:  1      2     1.00    *                   ldrsh	r0, [r1, #-1]
+# CHECK-NEXT:  1      2     1.00    *                   ldrsh	r0, [r1], #1
+# CHECK-NEXT:  1      2     1.00    *                   ldrsh	r0, [r1, #1]!
+# CHECK-NEXT:  1      2     1.00    *                   ldrsh.w	r0, [pc, #4]
+# CHECK-NEXT:  1      2     1.00    *                   ldrsh.w	r0, next
+# CHECK-NEXT:  1      2     1.00    *                   ldrsh	r0, [r1, r2]
+# CHECK-NEXT:  1      2     1.00    *                   ldrsh.w	r0, [r1, r2]
+# CHECK-NEXT:  1      2     1.00    *                   ldrsh.w	r0, [r1, r2, lsl #1]
+# CHECK-NEXT:  1      2     1.00                  U     ldrsht	r0, [r1, #1]
+# CHECK-NEXT:  1      2     1.00                  U     ldrt	r0, [r1, #1]
+# CHECK-NEXT:  1      1     1.00                        lsls	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        lsl.w	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        lsls.w	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        lsls	r0, r1
+# CHECK-NEXT:  1      1     1.00                        lsl.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        lsls.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        lsrs	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        lsr.w	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        lsrs.w	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        lsrs	r0, r1
+# CHECK-NEXT:  1      1     1.00                        lsr.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        lsrs.w	r0, r1, r2
+# CHECK-NEXT:  1      2     1.00                        mla	r0, r1, r2, r3
+# CHECK-NEXT:  1      2     1.00                        mls	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        movs	r0, #1
+# CHECK-NEXT:  1      1     1.00                        mov.w	r0, #1
+# CHECK-NEXT:  1      1     1.00                        movs.w	r0, #1
+# CHECK-NEXT:  1      1     1.00                        movw	r0, #1
+# CHECK-NEXT:  1      1     1.00                        mov	r0, r1
+# CHECK-NEXT:  1      1     1.00                        mov.w	r0, r1
+# CHECK-NEXT:  1      1     1.00                        movs.w	r0, r1
+# CHECK-NEXT:  1      1     1.00                        movt	r0, #1
+# CHECK-NEXT:  1      1     1.00                  U     mrs	r0, apsr
+# CHECK-NEXT:  1      1     1.00                  U     msr	apsr_nzcvq, r0
+# CHECK-NEXT:  1      1     1.00                        muls	r1, r2, r1
+# CHECK-NEXT:  1      1     1.00                        mul	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        mvn	r0, #1
+# CHECK-NEXT:  1      1     1.00                        mvns	r0, #1
+# CHECK-NEXT:  1      1     1.00                        mvns	r0, r1
+# CHECK-NEXT:  1      1     1.00                        mvn.w	r0, r1
+# CHECK-NEXT:  1      1     1.00                        mvns.w	r0, r1
+# CHECK-NEXT:  1      1     1.00                        mvn.w	r0, r1, lsl #1
+# CHECK-NEXT:  1      1     1.00                        mvns.w	r0, r1, lsl #1
+# CHECK-NEXT:  1      1     1.00    *      *      U     nop
+# CHECK-NEXT:  1      1     1.00                        orn	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        orns	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        orn	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        orns	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        orn	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        orns	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        orr	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        orrs	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        orrs	r0, r1
+# CHECK-NEXT:  1      1     1.00                        orr.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        orrs.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        orr.w	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        orrs.w	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        pkhbt	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        pkhbt	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        pkhbt	r0, r2, r1
+# CHECK-NEXT:  1      1     1.00                        pkhtb	r0, r1, r2, asr #1
+# CHECK-NEXT:  1      2     1.00    *             U     pop	{r0}
+# CHECK-NEXT:  1      2     1.00    *                   pop.w	{r0, r1}
+# CHECK-NEXT:  1      2     1.00    *                   ldr	r0, [sp], #4
+# CHECK-NEXT:  1      1     1.00    *      *      U     pssbb
+# CHECK-NEXT:  1      1     1.00           *      U     push	{r0}
+# CHECK-NEXT:  1      1     1.00           *            push.w	{r0, r1}
+# CHECK-NEXT:  1      1     1.00           *            str	r0, [sp, #-4]!
+# CHECK-NEXT:  1      1     1.00                        qadd	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        qadd16	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        qadd8	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        qasx	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        qdadd	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        qdsub	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        qsax	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        qsub	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        qsub16	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        qsub8	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        rbit	r0, r1
+# CHECK-NEXT:  1      1     1.00                        rev	r0, r1
+# CHECK-NEXT:  1      1     1.00                        rev.w	r0, r1
+# CHECK-NEXT:  1      1     1.00                        rev16	r0, r1
+# CHECK-NEXT:  1      1     1.00                        rev16.w	r0, r1
+# CHECK-NEXT:  1      1     1.00                        revsh	r0, r1
+# CHECK-NEXT:  1      1     1.00                        revsh.w	r0, r1
+# CHECK-NEXT:  1      1     1.00                        ror.w	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        rors.w	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        rors	r0, r1
+# CHECK-NEXT:  1      1     1.00                        ror.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        rors.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        rrx	r0, r1
+# CHECK-NEXT:  1      1     1.00                        rrxs	r0, r1
+# CHECK-NEXT:  1      1     1.00                        rsbs	r0, r1, #0
+# CHECK-NEXT:  1      1     1.00                        rsb.w	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        rsbs.w	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                  U     rsb	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                  U     rsbs	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        rsb	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        rsbs	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00    *      *      U     sadd16	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00    *      *      U     sadd8	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00    *      *      U     sasx	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        sbc	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        sbcs	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                  U     sbcs	r0, r1
+# CHECK-NEXT:  1      1     1.00                        sbc.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        sbcs.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        sbc.w	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        sbcs.w	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        sbfx	r0, r1, #1, #2
+# CHECK-NEXT:  1      2     1.00                        sdiv	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00    *                   sel	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00    *      *      U     sev
+# CHECK-NEXT:  1      1     1.00                        shadd16	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        shadd8	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        shasx	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        shsax	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        shsub16	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        shsub8	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        smlabb	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlabt	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlatb	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlatt	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlad	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smladx	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlal	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlalbb	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlalbt	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlaltb	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlaltt	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlald	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlaldx	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlawb	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlawt	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlsd	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlsdx	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlsld	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smlsldx	r0, r1, r2, r3
+# CHECK-NEXT:  1      2     1.00                        smmla	r0, r1, r2, r3
+# CHECK-NEXT:  1      2     1.00                        smmlar	r0, r1, r2, r3
+# CHECK-NEXT:  1      2     1.00                  U     smmls	r0, r1, r2, r3
+# CHECK-NEXT:  1      2     1.00                        smmlsr	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smmul	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        smmulr	r0, r1, r2
+# CHECK-NEXT:  1      2     1.00                        smuad	r0, r1, r2
+# CHECK-NEXT:  1      2     1.00                        smuadx	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        smulbb	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        smulbt	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        smultb	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        smultt	r0, r1, r2
+# CHECK-NEXT:  2      1     2.00                        smull	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        smulwb	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        smulwt	r0, r1, r2
+# CHECK-NEXT:  1      2     1.00                        smusd	r0, r1, r2
+# CHECK-NEXT:  1      2     1.00                        smusdx	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                  U     ssat	r0, #1, r2
+# CHECK-NEXT:  1      1     1.00                  U     ssat	r0, #1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                  U     ssat16	r0, #1, r1
+# CHECK-NEXT:  1      1     1.00    *      *      U     ssax	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00    *      *      U     ssbb
+# CHECK-NEXT:  1      1     1.00    *      *      U     ssub16	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00    *      *      U     ssub8	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00           *            stm	r0!, {r1}
+# CHECK-NEXT:  1      1     1.00           *            stm.w	r0, {r1}
+# CHECK-NEXT:  1      1     1.00           *            stm.w	r0!, {r1}
+# CHECK-NEXT:  1      1     1.00           *            stmdb	r0, {r1}
+# CHECK-NEXT:  1      1     1.00           *            str	r1, [r0, #-4]!
+# CHECK-NEXT:  1      1     1.00           *            str	r0, [r1]
+# CHECK-NEXT:  1      1     1.00           *            str	r0, [r1, #4]
+# CHECK-NEXT:  1      1     1.00           *            str	r0, [sp, #4]
+# CHECK-NEXT:  1      1     1.00           *            str.w	r0, [r1, #1]
+# CHECK-NEXT:  1      1     1.00           *            str	r0, [r1, #-1]
+# CHECK-NEXT:  1      1     1.00           *            str	r0, [r1], #1
+# CHECK-NEXT:  1      1     1.00           *            str	r0, [r1, r2]
+# CHECK-NEXT:  1      1     1.00           *            str.w	r0, [r1, r2]
+# CHECK-NEXT:  1      1     1.00           *            str.w	r0, [r1, r2, lsl #1]
+# CHECK-NEXT:  1      1     1.00           *            strb	r0, [r1]
+# CHECK-NEXT:  1      1     1.00           *            strb	r0, [r1, #1]
+# CHECK-NEXT:  1      1     1.00           *            strb.w	r0, [r1, #1]
+# CHECK-NEXT:  1      1     1.00           *            strb	r0, [r1, #-1]
+# CHECK-NEXT:  1      1     1.00           *            strb	r0, [r1], #1
+# CHECK-NEXT:  1      1     1.00           *            strb	r0, [r1, #1]!
+# CHECK-NEXT:  1      1     1.00           *            strb	r0, [r1, r2]
+# CHECK-NEXT:  1      1     1.00           *            strb.w	r0, [r1, r2]
+# CHECK-NEXT:  1      1     1.00           *            strb.w	r0, [r1, r2, lsl #1]
+# CHECK-NEXT:  1      1     1.00                  U     strbt	r0, [r1, #1]
+# CHECK-NEXT:  1      1     1.00           *            strd	r0, r1, [r2, #4]
+# CHECK-NEXT:  1      1     1.00           *            strd	r0, r1, [r2], #4
+# CHECK-NEXT:  1      1     1.00           *            strd	r0, r1, [r2, #4]!
+# CHECK-NEXT:  1      1     1.00    *      *      U     strex	r0, r1, [r2]
+# CHECK-NEXT:  1      1     1.00    *      *      U     strex	r0, r1, [r2, #4]
+# CHECK-NEXT:  1      1     1.00    *      *      U     strexb	r0, r1, [r2]
+# CHECK-NEXT:  1      1     1.00    *      *      U     strexh	r0, r1, [r2]
+# CHECK-NEXT:  1      1     1.00           *            strh	r0, [r1]
+# CHECK-NEXT:  1      1     1.00           *            strh	r0, [r1, #2]
+# CHECK-NEXT:  1      1     1.00           *            strh.w	r0, [r1, #2]
+# CHECK-NEXT:  1      1     1.00           *            strh	r0, [r1, #-1]
+# CHECK-NEXT:  1      1     1.00           *            strh	r0, [r1], #1
+# CHECK-NEXT:  1      1     1.00           *            strh	r0, [r1, #1]!
+# CHECK-NEXT:  1      1     1.00           *            strh	r0, [r1, r2]
+# CHECK-NEXT:  1      1     1.00           *            strh.w	r0, [r1, r2]
+# CHECK-NEXT:  1      1     1.00           *            strh.w	r0, [r1, r2, lsl #1]
+# CHECK-NEXT:  1      1     1.00                  U     strht	r0, [r1, #1]
+# CHECK-NEXT:  1      1     1.00                  U     strt	r0, [r1, #1]
+# CHECK-NEXT:  1      1     1.00                  U     sub	sp, #4
+# CHECK-NEXT:  1      1     1.00                        sub.w	r0, sp, #1
+# CHECK-NEXT:  1      1     1.00                        subs.w	r0, sp, #1
+# CHECK-NEXT:  1      1     1.00                        subw	r0, sp, #1
+# CHECK-NEXT:  1      1     1.00                        sub.w	r0, sp, r1
+# CHECK-NEXT:  1      1     1.00                        subs.w	r0, sp, r1
+# CHECK-NEXT:  1      1     1.00                        sub.w	r0, sp, r1, lsl #1
+# CHECK-NEXT:  1      1     1.00                        subs.w	r0, sp, r1, lsl #1
+# CHECK-NEXT:  1      1     1.00                        subs	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        subs	r0, #1
+# CHECK-NEXT:  1      1     1.00                        sub.w	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        subs.w	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        subw	r0, r1, #1
+# CHECK-NEXT:  1      1     1.00                        subs	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        sub.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        subs.w	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        sub.w	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        subs.w	r0, r1, r2, lsl #1
+# CHECK-NEXT:  1      1     1.00                        sxtab	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        sxtab	r0, r1, r2, ror #8
+# CHECK-NEXT:  1      1     1.00                        sxtab16	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        sxtab16	r0, r1, r2, ror #8
+# CHECK-NEXT:  1      1     1.00                        sxtah	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        sxtah	r0, r1, r2, ror #8
+# CHECK-NEXT:  1      1     1.00                        sxtb	r0, r1
+# CHECK-NEXT:  1      1     1.00                        sxtb.w	r0, r1
+# CHECK-NEXT:  1      1     1.00                        sxtb.w	r0, r1, ror #8
+# CHECK-NEXT:  1      1     1.00                        sxtb16	r0, r1
+# CHECK-NEXT:  1      1     1.00                        sxtb16	r0, r1, ror #8
+# CHECK-NEXT:  1      1     1.00                        sxth	r0, r1
+# CHECK-NEXT:  1      1     1.00                        sxth.w	r0, r1
+# CHECK-NEXT:  1      1     1.00                        sxth.w	r0, r1, ror #8
+# CHECK-NEXT:  1      1     1.00                  U     tbb	[r0, r1]
+# CHECK-NEXT:  1      1     1.00                  U     tbh	[r0, r1, lsl #1]
+# CHECK-NEXT:  1      1     1.00                        teq.w	r0, #1
+# CHECK-NEXT:  1      1     1.00                        teq.w	r0, r1
+# CHECK-NEXT:  1      1     1.00                        teq.w	r0, r1, lsl #1
+# CHECK-NEXT:  1      1     1.00                        tst.w	r0, #1
+# CHECK-NEXT:  1      1     1.00                        tst	r0, r1
+# CHECK-NEXT:  1      1     1.00                        tst.w	r0, r1
+# CHECK-NEXT:  1      1     1.00                        tst.w	r0, r1, lsl #1
+# CHECK-NEXT:  1      1     1.00    *      *      U     uadd16	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00    *      *      U     uadd8	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00    *      *      U     uasx	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        ubfx	r0, r1, #1, #2
+# CHECK-NEXT:  1      2     1.00                        udiv	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        uhadd16	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        uhadd8	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        uhasx	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        uhsax	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        uhsub16	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        uhsub8	r0, r1, r2
+# CHECK-NEXT:  2      2     2.00                        umaal	r0, r1, r2, r3
+# CHECK-NEXT:  2      2     2.00                        umlal	r0, r1, r2, r3
+# CHECK-NEXT:  2      1     2.00                        umull	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                        uqadd16	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        uqadd8	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        uqasx	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        uqsax	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        uqsub16	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        uqsub8	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        usad8	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        usada8	r0, r1, r2, r3
+# CHECK-NEXT:  1      1     1.00                  U     usat	r0, #1, r1
+# CHECK-NEXT:  1      1     1.00                  U     usat	r0, #1, r1, lsl #1
+# CHECK-NEXT:  1      1     1.00                  U     usat16	r0, #1, r1
+# CHECK-NEXT:  1      1     1.00    *      *      U     usax	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00    *      *      U     usub16	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00    *      *      U     usub8	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        uxtab	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        uxtab	r0, r1, r2, ror #8
+# CHECK-NEXT:  1      1     1.00                        uxtab16	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        uxtab16	r0, r1, r2, ror #8
+# CHECK-NEXT:  1      1     1.00                        uxtah	r0, r1, r2
+# CHECK-NEXT:  1      1     1.00                        uxtah	r0, r1, r2, ror #8
+# CHECK-NEXT:  1      1     1.00                        uxtb	r0, r1
+# CHECK-NEXT:  1      1     1.00                        uxtb.w	r0, r1
+# CHECK-NEXT:  1      1     1.00                        uxtb.w	r0, r1, ror #8
+# CHECK-NEXT:  1      1     1.00                        uxtb16	r0, r1
+# CHECK-NEXT:  1      1     1.00                        uxtb16	r0, r1, ror #8
+# CHECK-NEXT:  1      1     1.00                        uxth	r0, r1
+# CHECK-NEXT:  1      1     1.00                        uxth.w	r0, r1
+# CHECK-NEXT:  1      1     1.00                        uxth.w	r0, r1, ror #8
+# CHECK-NEXT:  1      1     1.00    *      *      U     wfe
+# CHECK-NEXT:  1      1     1.00    *      *      U     wfi
+# CHECK-NEXT:  1      1     1.00    *      *      U     yield
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - M4Unit
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]
+# CHECK-NEXT: 432.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    Instructions:
+# CHECK-NEXT: 1.00   adc	r0, r1, #0
+# CHECK-NEXT: 1.00   adcs	r0, r1, #0
+# CHECK-NEXT: 1.00   adcs	r0, r1
+# CHECK-NEXT: 1.00   adc.w	r0, r1, r2
+# CHECK-NEXT: 1.00   adcs.w	r0, r1, r2
+# CHECK-NEXT: 1.00   adc.w	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   adcs.w	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   add.w	r0, sp, #1
+# CHECK-NEXT: 1.00   add.w	sp, sp, #1
+# CHECK-NEXT: 1.00   add.w	r0, sp, #1
+# CHECK-NEXT: 1.00   adds.w	r0, sp, #1
+# CHECK-NEXT: 1.00   addw	r0, sp, #1
+# CHECK-NEXT: 1.00   add	r0, sp, r0
+# CHECK-NEXT: 1.00   add	sp, r1
+# CHECK-NEXT: 1.00   add.w	r0, sp, r1
+# CHECK-NEXT: 1.00   adds.w	r0, sp, r1
+# CHECK-NEXT: 1.00   add.w	r0, sp, r1, lsl #1
+# CHECK-NEXT: 1.00   adds.w	r0, sp, r1, lsl #1
+# CHECK-NEXT: 1.00   adds	r0, r1, #1
+# CHECK-NEXT: 1.00   adds	r0, #42
+# CHECK-NEXT: 1.00   add.w	r0, r1, #1
+# CHECK-NEXT: 1.00   adds.w	r0, r1, #1
+# CHECK-NEXT: 1.00   addw	r0, r1, #1
+# CHECK-NEXT: 1.00   adds	r0, r1, r2
+# CHECK-NEXT: 1.00   add	r0, r1
+# CHECK-NEXT: 1.00   add.w	r0, r1, r2
+# CHECK-NEXT: 1.00   adds.w	r0, r1, r2
+# CHECK-NEXT: 1.00   add.w	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   adds.w	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   adr.w	r0, #-6
+# CHECK-NEXT: 1.00   adr.w	r8, #-6
+# CHECK-NEXT: 1.00   adr.w	r0, #-6
+# CHECK-NEXT: 1.00   and	r0, r1, #1
+# CHECK-NEXT: 1.00   ands	r0, r1, #1
+# CHECK-NEXT: 1.00   ands	r1, r0
+# CHECK-NEXT: 1.00   and.w	r0, r1, r2
+# CHECK-NEXT: 1.00   ands.w	r0, r1, r2
+# CHECK-NEXT: 1.00   and.w	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   ands.w	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   asrs	r0, r1, #1
+# CHECK-NEXT: 1.00   asr.w	r0, r1, #1
+# CHECK-NEXT: 1.00   asrs.w	r0, r1, #1
+# CHECK-NEXT: 1.00   asrs	r0, r1
+# CHECK-NEXT: 1.00   asr.w	r0, r1, r2
+# CHECK-NEXT: 1.00   asrs.w	r0, r1, r2
+# CHECK-NEXT: 1.00   bfc	r0, #1, #2
+# CHECK-NEXT: 1.00   bfi	r0, r1, #1, #2
+# CHECK-NEXT: 1.00   bic	r0, r1, #1
+# CHECK-NEXT: 1.00   bics	r0, r1, #1
+# CHECK-NEXT: 1.00   bics	r0, r1
+# CHECK-NEXT: 1.00   bic.w	r0, r1, r2
+# CHECK-NEXT: 1.00   bics.w	r0, r1, r2
+# CHECK-NEXT: 1.00   bic.w	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   bics.w	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   bkpt	#1
+# CHECK-NEXT: 1.00   clrex
+# CHECK-NEXT: 1.00   clz	r0, r1
+# CHECK-NEXT: 1.00   cmn.w	r0, #1
+# CHECK-NEXT: 1.00   cmn	r0, r1
+# CHECK-NEXT: 1.00   cmn.w	r0, r1
+# CHECK-NEXT: 1.00   cmn.w	r0, r1, lsl #1
+# CHECK-NEXT: 1.00   cmp	r0, #1
+# CHECK-NEXT: 1.00   cmp.w	r0, #1
+# CHECK-NEXT: 1.00   cmp	r0, r1
+# CHECK-NEXT: 1.00   cmp	r0, r10
+# CHECK-NEXT: 1.00   cmp.w	r0, r1
+# CHECK-NEXT: 1.00   cmp.w	r0, r1, lsl #1
+# CHECK-NEXT: 1.00   dmb	sy
+# CHECK-NEXT: 1.00   dsb	sy
+# CHECK-NEXT: 1.00   eor	r0, r1, #1
+# CHECK-NEXT: 1.00   eors	r0, r1, #1
+# CHECK-NEXT: 1.00   eors	r0, r1
+# CHECK-NEXT: 1.00   eor.w	r0, r1, r2
+# CHECK-NEXT: 1.00   eors.w	r0, r1, r2
+# CHECK-NEXT: 1.00   eor.w	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   eors.w	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   isb	sy
+# CHECK-NEXT: 1.00   ldm	r0!, {r1}
+# CHECK-NEXT: 1.00   ldm.w	r0, {r1}
+# CHECK-NEXT: 1.00   ldm.w	r0, {r1}
+# CHECK-NEXT: 1.00   ldr	r1, [r0], #4
+# CHECK-NEXT: 1.00   ldmdb	r0, {r1}
+# CHECK-NEXT: 1.00   ldmdb	r0!, {r1}
+# CHECK-NEXT: 1.00   ldr	r0, [r1, #4]
+# CHECK-NEXT: 1.00   ldr	r0, [sp, #4]
+# CHECK-NEXT: 1.00   ldr.w	r0, [r1, #4]
+# CHECK-NEXT: 1.00   ldr	r0, [r1, #-1]
+# CHECK-NEXT: 1.00   ldr	r0, [r1], #1
+# CHECK-NEXT: 1.00   ldr	r0, [r1, #1]!
+# CHECK-NEXT: 1.00   ldr	r0, [pc, #4]
+# CHECK-NEXT: 1.00   ldr.w	r0, [pc, #4]
+# CHECK-NEXT: 1.00   ldr	r0, next
+# CHECK-NEXT: 1.00   ldr.w	r0, next
+# CHECK-NEXT: 1.00   ldr	r0, [r1, r2]
+# CHECK-NEXT: 1.00   ldr.w	r0, [r1, r2]
+# CHECK-NEXT: 1.00   ldr.w	r0, [r1, r2, lsl #1]
+# CHECK-NEXT: 1.00   ldrb	r0, [r1, #1]
+# CHECK-NEXT: 1.00   ldrb.w	r0, [r1, #1]
+# CHECK-NEXT: 1.00   ldrb	r0, [r1, #-1]
+# CHECK-NEXT: 1.00   ldrb	r0, [r1], #1
+# CHECK-NEXT: 1.00   ldrb	r0, [r1, #1]!
+# CHECK-NEXT: 1.00   ldrb.w	r0, [pc, #4]
+# CHECK-NEXT: 1.00   ldrb.w	r0, next
+# CHECK-NEXT: 1.00   ldrb	r0, [r1, r2]
+# CHECK-NEXT: 1.00   ldrb.w	r0, [r1, r2]
+# CHECK-NEXT: 1.00   ldrb.w	r0, [r1, r2, lsl #1]
+# CHECK-NEXT: 1.00   ldrbt	r0, [r1, #1]
+# CHECK-NEXT: 1.00   ldrd	r0, r2, [r1]
+# CHECK-NEXT: 1.00   ldrd	r0, r2, [r1, #-4]
+# CHECK-NEXT: 1.00   ldrd	r0, r2, [r1], #4
+# CHECK-NEXT: 1.00   ldrd	r0, r2, [r1, #4]!
+# CHECK-NEXT: 1.00   ldrd	r0, r2, next
+# CHECK-NEXT: 1.00   ldrex	r0, [r1]
+# CHECK-NEXT: 1.00   ldrex	r0, [r1, #4]
+# CHECK-NEXT: 1.00   ldrexb	r0, [r1]
+# CHECK-NEXT: 1.00   ldrexh	r0, [r1]
+# CHECK-NEXT: 1.00   ldrh	r0, [r1, #2]
+# CHECK-NEXT: 1.00   ldrh.w	r0, [r1, #1]
+# CHECK-NEXT: 1.00   ldrh	r0, [r1, #-1]
+# CHECK-NEXT: 1.00   ldrh	r0, [r1], #1
+# CHECK-NEXT: 1.00   ldrh	r0, [r1, #1]!
+# CHECK-NEXT: 1.00   ldrh.w	r0, [pc, #4]
+# CHECK-NEXT: 1.00   ldrh.w	r0, next
+# CHECK-NEXT: 1.00   ldrh	r0, [r1, r2]
+# CHECK-NEXT: 1.00   ldrh.w	r0, [r1, r2]
+# CHECK-NEXT: 1.00   ldrh.w	r0, [r1, r2, lsl #1]
+# CHECK-NEXT: 1.00   ldrht	r0, [r1, #1]
+# CHECK-NEXT: 1.00   ldrsb.w	r0, [r1, #1]
+# CHECK-NEXT: 1.00   ldrsb	r0, [r1, #-1]
+# CHECK-NEXT: 1.00   ldrsb	r0, [r1], #1
+# CHECK-NEXT: 1.00   ldrsb	r0, [r1, #1]!
+# CHECK-NEXT: 1.00   ldrsb.w	r0, [pc, #4]
+# CHECK-NEXT: 1.00   ldrsb.w	r0, next
+# CHECK-NEXT: 1.00   ldrsb	r0, [r1, r2]
+# CHECK-NEXT: 1.00   ldrsb.w	r0, [r1, r2]
+# CHECK-NEXT: 1.00   ldrsb.w	r0, [r1, r2, lsl #1]
+# CHECK-NEXT: 1.00   ldrsbt	r0, [r1, #1]
+# CHECK-NEXT: 1.00   ldrsh.w	r0, [r1, #2]
+# CHECK-NEXT: 1.00   ldrsh	r0, [r1, #-1]
+# CHECK-NEXT: 1.00   ldrsh	r0, [r1], #1
+# CHECK-NEXT: 1.00   ldrsh	r0, [r1, #1]!
+# CHECK-NEXT: 1.00   ldrsh.w	r0, [pc, #4]
+# CHECK-NEXT: 1.00   ldrsh.w	r0, next
+# CHECK-NEXT: 1.00   ldrsh	r0, [r1, r2]
+# CHECK-NEXT: 1.00   ldrsh.w	r0, [r1, r2]
+# CHECK-NEXT: 1.00   ldrsh.w	r0, [r1, r2, lsl #1]
+# CHECK-NEXT: 1.00   ldrsht	r0, [r1, #1]
+# CHECK-NEXT: 1.00   ldrt	r0, [r1, #1]
+# CHECK-NEXT: 1.00   lsls	r0, r1, #1
+# CHECK-NEXT: 1.00   lsl.w	r0, r1, #1
+# CHECK-NEXT: 1.00   lsls.w	r0, r1, #1
+# CHECK-NEXT: 1.00   lsls	r0, r1
+# CHECK-NEXT: 1.00   lsl.w	r0, r1, r2
+# CHECK-NEXT: 1.00   lsls.w	r0, r1, r2
+# CHECK-NEXT: 1.00   lsrs	r0, r1, #1
+# CHECK-NEXT: 1.00   lsr.w	r0, r1, #1
+# CHECK-NEXT: 1.00   lsrs.w	r0, r1, #1
+# CHECK-NEXT: 1.00   lsrs	r0, r1
+# CHECK-NEXT: 1.00   lsr.w	r0, r1, r2
+# CHECK-NEXT: 1.00   lsrs.w	r0, r1, r2
+# CHECK-NEXT: 1.00   mla	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   mls	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   movs	r0, #1
+# CHECK-NEXT: 1.00   mov.w	r0, #1
+# CHECK-NEXT: 1.00   movs.w	r0, #1
+# CHECK-NEXT: 1.00   movw	r0, #1
+# CHECK-NEXT: 1.00   mov	r0, r1
+# CHECK-NEXT: 1.00   mov.w	r0, r1
+# CHECK-NEXT: 1.00   movs.w	r0, r1
+# CHECK-NEXT: 1.00   movt	r0, #1
+# CHECK-NEXT: 1.00   mrs	r0, apsr
+# CHECK-NEXT: 1.00   msr	apsr_nzcvq, r0
+# CHECK-NEXT: 1.00   muls	r1, r2, r1
+# CHECK-NEXT: 1.00   mul	r0, r1, r2
+# CHECK-NEXT: 1.00   mvn	r0, #1
+# CHECK-NEXT: 1.00   mvns	r0, #1
+# CHECK-NEXT: 1.00   mvns	r0, r1
+# CHECK-NEXT: 1.00   mvn.w	r0, r1
+# CHECK-NEXT: 1.00   mvns.w	r0, r1
+# CHECK-NEXT: 1.00   mvn.w	r0, r1, lsl #1
+# CHECK-NEXT: 1.00   mvns.w	r0, r1, lsl #1
+# CHECK-NEXT: 1.00   nop
+# CHECK-NEXT: 1.00   orn	r0, r1, #1
+# CHECK-NEXT: 1.00   orns	r0, r1, #1
+# CHECK-NEXT: 1.00   orn	r0, r1, r2
+# CHECK-NEXT: 1.00   orns	r0, r1, r2
+# CHECK-NEXT: 1.00   orn	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   orns	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   orr	r0, r1, #1
+# CHECK-NEXT: 1.00   orrs	r0, r1, #1
+# CHECK-NEXT: 1.00   orrs	r0, r1
+# CHECK-NEXT: 1.00   orr.w	r0, r1, r2
+# CHECK-NEXT: 1.00   orrs.w	r0, r1, r2
+# CHECK-NEXT: 1.00   orr.w	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   orrs.w	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   pkhbt	r0, r1, r2
+# CHECK-NEXT: 1.00   pkhbt	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   pkhbt	r0, r2, r1
+# CHECK-NEXT: 1.00   pkhtb	r0, r1, r2, asr #1
+# CHECK-NEXT: 1.00   pop	{r0}
+# CHECK-NEXT: 1.00   pop.w	{r0, r1}
+# CHECK-NEXT: 1.00   ldr	r0, [sp], #4
+# CHECK-NEXT: 1.00   pssbb
+# CHECK-NEXT: 1.00   push	{r0}
+# CHECK-NEXT: 1.00   push.w	{r0, r1}
+# CHECK-NEXT: 1.00   str	r0, [sp, #-4]!
+# CHECK-NEXT: 1.00   qadd	r0, r1, r2
+# CHECK-NEXT: 1.00   qadd16	r0, r1, r2
+# CHECK-NEXT: 1.00   qadd8	r0, r1, r2
+# CHECK-NEXT: 1.00   qasx	r0, r1, r2
+# CHECK-NEXT: 1.00   qdadd	r0, r1, r2
+# CHECK-NEXT: 1.00   qdsub	r0, r1, r2
+# CHECK-NEXT: 1.00   qsax	r0, r1, r2
+# CHECK-NEXT: 1.00   qsub	r0, r1, r2
+# CHECK-NEXT: 1.00   qsub16	r0, r1, r2
+# CHECK-NEXT: 1.00   qsub8	r0, r1, r2
+# CHECK-NEXT: 1.00   rbit	r0, r1
+# CHECK-NEXT: 1.00   rev	r0, r1
+# CHECK-NEXT: 1.00   rev.w	r0, r1
+# CHECK-NEXT: 1.00   rev16	r0, r1
+# CHECK-NEXT: 1.00   rev16.w	r0, r1
+# CHECK-NEXT: 1.00   revsh	r0, r1
+# CHECK-NEXT: 1.00   revsh.w	r0, r1
+# CHECK-NEXT: 1.00   ror.w	r0, r1, #1
+# CHECK-NEXT: 1.00   rors.w	r0, r1, #1
+# CHECK-NEXT: 1.00   rors	r0, r1
+# CHECK-NEXT: 1.00   ror.w	r0, r1, r2
+# CHECK-NEXT: 1.00   rors.w	r0, r1, r2
+# CHECK-NEXT: 1.00   rrx	r0, r1
+# CHECK-NEXT: 1.00   rrxs	r0, r1
+# CHECK-NEXT: 1.00   rsbs	r0, r1, #0
+# CHECK-NEXT: 1.00   rsb.w	r0, r1, #1
+# CHECK-NEXT: 1.00   rsbs.w	r0, r1, #1
+# CHECK-NEXT: 1.00   rsb	r0, r1, r2
+# CHECK-NEXT: 1.00   rsbs	r0, r1, r2
+# CHECK-NEXT: 1.00   rsb	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   rsbs	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   sadd16	r0, r1, r2
+# CHECK-NEXT: 1.00   sadd8	r0, r1, r2
+# CHECK-NEXT: 1.00   sasx	r0, r1, r2
+# CHECK-NEXT: 1.00   sbc	r0, r1, #1
+# CHECK-NEXT: 1.00   sbcs	r0, r1, #1
+# CHECK-NEXT: 1.00   sbcs	r0, r1
+# CHECK-NEXT: 1.00   sbc.w	r0, r1, r2
+# CHECK-NEXT: 1.00   sbcs.w	r0, r1, r2
+# CHECK-NEXT: 1.00   sbc.w	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   sbcs.w	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   sbfx	r0, r1, #1, #2
+# CHECK-NEXT: 1.00   sdiv	r0, r1, r2
+# CHECK-NEXT: 1.00   sel	r0, r1, r2
+# CHECK-NEXT: 1.00   sev
+# CHECK-NEXT: 1.00   shadd16	r0, r1, r2
+# CHECK-NEXT: 1.00   shadd8	r0, r1, r2
+# CHECK-NEXT: 1.00   shasx	r0, r1, r2
+# CHECK-NEXT: 1.00   shsax	r0, r1, r2
+# CHECK-NEXT: 1.00   shsub16	r0, r1, r2
+# CHECK-NEXT: 1.00   shsub8	r0, r1, r2
+# CHECK-NEXT: 1.00   smlabb	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlabt	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlatb	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlatt	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlad	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smladx	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlal	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlalbb	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlalbt	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlaltb	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlaltt	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlald	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlaldx	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlawb	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlawt	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlsd	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlsdx	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlsld	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smlsldx	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smmla	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smmlar	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smmls	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smmlsr	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smmul	r0, r1, r2
+# CHECK-NEXT: 1.00   smmulr	r0, r1, r2
+# CHECK-NEXT: 1.00   smuad	r0, r1, r2
+# CHECK-NEXT: 1.00   smuadx	r0, r1, r2
+# CHECK-NEXT: 1.00   smulbb	r0, r1, r2
+# CHECK-NEXT: 1.00   smulbt	r0, r1, r2
+# CHECK-NEXT: 1.00   smultb	r0, r1, r2
+# CHECK-NEXT: 1.00   smultt	r0, r1, r2
+# CHECK-NEXT: 2.00   smull	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   smulwb	r0, r1, r2
+# CHECK-NEXT: 1.00   smulwt	r0, r1, r2
+# CHECK-NEXT: 1.00   smusd	r0, r1, r2
+# CHECK-NEXT: 1.00   smusdx	r0, r1, r2
+# CHECK-NEXT: 1.00   ssat	r0, #1, r2
+# CHECK-NEXT: 1.00   ssat	r0, #1, r2, lsl #1
+# CHECK-NEXT: 1.00   ssat16	r0, #1, r1
+# CHECK-NEXT: 1.00   ssax	r0, r1, r2
+# CHECK-NEXT: 1.00   ssbb
+# CHECK-NEXT: 1.00   ssub16	r0, r1, r2
+# CHECK-NEXT: 1.00   ssub8	r0, r1, r2
+# CHECK-NEXT: 1.00   stm	r0!, {r1}
+# CHECK-NEXT: 1.00   stm.w	r0, {r1}
+# CHECK-NEXT: 1.00   stm.w	r0!, {r1}
+# CHECK-NEXT: 1.00   stmdb	r0, {r1}
+# CHECK-NEXT: 1.00   str	r1, [r0, #-4]!
+# CHECK-NEXT: 1.00   str	r0, [r1]
+# CHECK-NEXT: 1.00   str	r0, [r1, #4]
+# CHECK-NEXT: 1.00   str	r0, [sp, #4]
+# CHECK-NEXT: 1.00   str.w	r0, [r1, #1]
+# CHECK-NEXT: 1.00   str	r0, [r1, #-1]
+# CHECK-NEXT: 1.00   str	r0, [r1], #1
+# CHECK-NEXT: 1.00   str	r0, [r1, r2]
+# CHECK-NEXT: 1.00   str.w	r0, [r1, r2]
+# CHECK-NEXT: 1.00   str.w	r0, [r1, r2, lsl #1]
+# CHECK-NEXT: 1.00   strb	r0, [r1]
+# CHECK-NEXT: 1.00   strb	r0, [r1, #1]
+# CHECK-NEXT: 1.00   strb.w	r0, [r1, #1]
+# CHECK-NEXT: 1.00   strb	r0, [r1, #-1]
+# CHECK-NEXT: 1.00   strb	r0, [r1], #1
+# CHECK-NEXT: 1.00   strb	r0, [r1, #1]!
+# CHECK-NEXT: 1.00   strb	r0, [r1, r2]
+# CHECK-NEXT: 1.00   strb.w	r0, [r1, r2]
+# CHECK-NEXT: 1.00   strb.w	r0, [r1, r2, lsl #1]
+# CHECK-NEXT: 1.00   strbt	r0, [r1, #1]
+# CHECK-NEXT: 1.00   strd	r0, r1, [r2, #4]
+# CHECK-NEXT: 1.00   strd	r0, r1, [r2], #4
+# CHECK-NEXT: 1.00   strd	r0, r1, [r2, #4]!
+# CHECK-NEXT: 1.00   strex	r0, r1, [r2]
+# CHECK-NEXT: 1.00   strex	r0, r1, [r2, #4]
+# CHECK-NEXT: 1.00   strexb	r0, r1, [r2]
+# CHECK-NEXT: 1.00   strexh	r0, r1, [r2]
+# CHECK-NEXT: 1.00   strh	r0, [r1]
+# CHECK-NEXT: 1.00   strh	r0, [r1, #2]
+# CHECK-NEXT: 1.00   strh.w	r0, [r1, #2]
+# CHECK-NEXT: 1.00   strh	r0, [r1, #-1]
+# CHECK-NEXT: 1.00   strh	r0, [r1], #1
+# CHECK-NEXT: 1.00   strh	r0, [r1, #1]!
+# CHECK-NEXT: 1.00   strh	r0, [r1, r2]
+# CHECK-NEXT: 1.00   strh.w	r0, [r1, r2]
+# CHECK-NEXT: 1.00   strh.w	r0, [r1, r2, lsl #1]
+# CHECK-NEXT: 1.00   strht	r0, [r1, #1]
+# CHECK-NEXT: 1.00   strt	r0, [r1, #1]
+# CHECK-NEXT: 1.00   sub	sp, #4
+# CHECK-NEXT: 1.00   sub.w	r0, sp, #1
+# CHECK-NEXT: 1.00   subs.w	r0, sp, #1
+# CHECK-NEXT: 1.00   subw	r0, sp, #1
+# CHECK-NEXT: 1.00   sub.w	r0, sp, r1
+# CHECK-NEXT: 1.00   subs.w	r0, sp, r1
+# CHECK-NEXT: 1.00   sub.w	r0, sp, r1, lsl #1
+# CHECK-NEXT: 1.00   subs.w	r0, sp, r1, lsl #1
+# CHECK-NEXT: 1.00   subs	r0, r1, #1
+# CHECK-NEXT: 1.00   subs	r0, #1
+# CHECK-NEXT: 1.00   sub.w	r0, r1, #1
+# CHECK-NEXT: 1.00   subs.w	r0, r1, #1
+# CHECK-NEXT: 1.00   subw	r0, r1, #1
+# CHECK-NEXT: 1.00   subs	r0, r1, r2
+# CHECK-NEXT: 1.00   sub.w	r0, r1, r2
+# CHECK-NEXT: 1.00   subs.w	r0, r1, r2
+# CHECK-NEXT: 1.00   sub.w	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   subs.w	r0, r1, r2, lsl #1
+# CHECK-NEXT: 1.00   sxtab	r0, r1, r2
+# CHECK-NEXT: 1.00   sxtab	r0, r1, r2, ror #8
+# CHECK-NEXT: 1.00   sxtab16	r0, r1, r2
+# CHECK-NEXT: 1.00   sxtab16	r0, r1, r2, ror #8
+# CHECK-NEXT: 1.00   sxtah	r0, r1, r2
+# CHECK-NEXT: 1.00   sxtah	r0, r1, r2, ror #8
+# CHECK-NEXT: 1.00   sxtb	r0, r1
+# CHECK-NEXT: 1.00   sxtb.w	r0, r1
+# CHECK-NEXT: 1.00   sxtb.w	r0, r1, ror #8
+# CHECK-NEXT: 1.00   sxtb16	r0, r1
+# CHECK-NEXT: 1.00   sxtb16	r0, r1, ror #8
+# CHECK-NEXT: 1.00   sxth	r0, r1
+# CHECK-NEXT: 1.00   sxth.w	r0, r1
+# CHECK-NEXT: 1.00   sxth.w	r0, r1, ror #8
+# CHECK-NEXT: 1.00   tbb	[r0, r1]
+# CHECK-NEXT: 1.00   tbh	[r0, r1, lsl #1]
+# CHECK-NEXT: 1.00   teq.w	r0, #1
+# CHECK-NEXT: 1.00   teq.w	r0, r1
+# CHECK-NEXT: 1.00   teq.w	r0, r1, lsl #1
+# CHECK-NEXT: 1.00   tst.w	r0, #1
+# CHECK-NEXT: 1.00   tst	r0, r1
+# CHECK-NEXT: 1.00   tst.w	r0, r1
+# CHECK-NEXT: 1.00   tst.w	r0, r1, lsl #1
+# CHECK-NEXT: 1.00   uadd16	r0, r1, r2
+# CHECK-NEXT: 1.00   uadd8	r0, r1, r2
+# CHECK-NEXT: 1.00   uasx	r0, r1, r2
+# CHECK-NEXT: 1.00   ubfx	r0, r1, #1, #2
+# CHECK-NEXT: 1.00   udiv	r0, r1, r2
+# CHECK-NEXT: 1.00   uhadd16	r0, r1, r2
+# CHECK-NEXT: 1.00   uhadd8	r0, r1, r2
+# CHECK-NEXT: 1.00   uhasx	r0, r1, r2
+# CHECK-NEXT: 1.00   uhsax	r0, r1, r2
+# CHECK-NEXT: 1.00   uhsub16	r0, r1, r2
+# CHECK-NEXT: 1.00   uhsub8	r0, r1, r2
+# CHECK-NEXT: 2.00   umaal	r0, r1, r2, r3
+# CHECK-NEXT: 2.00   umlal	r0, r1, r2, r3
+# CHECK-NEXT: 2.00   umull	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   uqadd16	r0, r1, r2
+# CHECK-NEXT: 1.00   uqadd8	r0, r1, r2
+# CHECK-NEXT: 1.00   uqasx	r0, r1, r2
+# CHECK-NEXT: 1.00   uqsax	r0, r1, r2
+# CHECK-NEXT: 1.00   uqsub16	r0, r1, r2
+# CHECK-NEXT: 1.00   uqsub8	r0, r1, r2
+# CHECK-NEXT: 1.00   usad8	r0, r1, r2
+# CHECK-NEXT: 1.00   usada8	r0, r1, r2, r3
+# CHECK-NEXT: 1.00   usat	r0, #1, r1
+# CHECK-NEXT: 1.00   usat	r0, #1, r1, lsl #1
+# CHECK-NEXT: 1.00   usat16	r0, #1, r1
+# CHECK-NEXT: 1.00   usax	r0, r1, r2
+# CHECK-NEXT: 1.00   usub16	r0, r1, r2
+# CHECK-NEXT: 1.00   usub8	r0, r1, r2
+# CHECK-NEXT: 1.00   uxtab	r0, r1, r2
+# CHECK-NEXT: 1.00   uxtab	r0, r1, r2, ror #8
+# CHECK-NEXT: 1.00   uxtab16	r0, r1, r2
+# CHECK-NEXT: 1.00   uxtab16	r0, r1, r2, ror #8
+# CHECK-NEXT: 1.00   uxtah	r0, r1, r2
+# CHECK-NEXT: 1.00   uxtah	r0, r1, r2, ror #8
+# CHECK-NEXT: 1.00   uxtb	r0, r1
+# CHECK-NEXT: 1.00   uxtb.w	r0, r1
+# CHECK-NEXT: 1.00   uxtb.w	r0, r1, ror #8
+# CHECK-NEXT: 1.00   uxtb16	r0, r1
+# CHECK-NEXT: 1.00   uxtb16	r0, r1, ror #8
+# CHECK-NEXT: 1.00   uxth	r0, r1
+# CHECK-NEXT: 1.00   uxth.w	r0, r1
+# CHECK-NEXT: 1.00   uxth.w	r0, r1, ror #8
+# CHECK-NEXT: 1.00   wfe
+# CHECK-NEXT: 1.00   wfi
+# CHECK-NEXT: 1.00   yield
diff --git a/llvm/test/tools/llvm-mca/X86/Atom/resources-x87.s b/llvm/test/tools/llvm-mca/X86/Atom/resources-x87.s
index eda0fdebb7d3c..6e173db786975 100644
--- a/llvm/test/tools/llvm-mca/X86/Atom/resources-x87.s
+++ b/llvm/test/tools/llvm-mca/X86/Atom/resources-x87.s
@@ -216,8 +216,8 @@ fyl2xp1
 # CHECK-NEXT:  1      5     5.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  1      5     5.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  1      5     5.00    *             U     fiaddl	(%ecx)
-# CHECK-NEXT:  1      100   0.50                  U     fbld	(%ecx)
-# CHECK-NEXT:  1      100   0.50                  U     fbstp	(%eax)
+# CHECK-NEXT:  1      100   0.50    *             U     fbld	(%ecx)
+# CHECK-NEXT:  1      100   0.50           *      U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  1      25    12.50                 U     fnclex
 # CHECK-NEXT:  1      9     4.50                  U     fcmovb	%st(1), %st
@@ -230,12 +230,12 @@ fyl2xp1
 # CHECK-NEXT:  1      9     4.50                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      5     5.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      5     5.00                  U     fcom	%st(3)
-# CHECK-NEXT:  1      5     5.00                  U     fcoms	(%ecx)
-# CHECK-NEXT:  1      5     5.00                  U     fcoml	(%eax)
+# CHECK-NEXT:  1      5     5.00    *             U     fcoms	(%ecx)
+# CHECK-NEXT:  1      5     5.00    *             U     fcoml	(%eax)
 # CHECK-NEXT:  1      5     5.00                  U     fcomp	%st(1)
 # CHECK-NEXT:  1      5     5.00                  U     fcomp	%st(3)
-# CHECK-NEXT:  1      5     5.00                  U     fcomps	(%ecx)
-# CHECK-NEXT:  1      5     5.00                  U     fcompl	(%eax)
+# CHECK-NEXT:  1      5     5.00    *             U     fcomps	(%ecx)
+# CHECK-NEXT:  1      5     5.00    *             U     fcompl	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fcompp
 # CHECK-NEXT:  1      9     4.50                  U     fcomi	%st(3), %st
 # CHECK-NEXT:  1      9     4.50                  U     fcompi	%st(3), %st
@@ -258,10 +258,10 @@ fyl2xp1
 # CHECK-NEXT:  1      34    17.00   *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  1      34    17.00   *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     ffree	%st(0)
-# CHECK-NEXT:  1      5     5.00                  U     ficoms	(%ecx)
-# CHECK-NEXT:  1      5     5.00                  U     ficoml	(%eax)
-# CHECK-NEXT:  1      5     5.00                  U     ficomps	(%ecx)
-# CHECK-NEXT:  1      5     5.00                  U     ficompl	(%eax)
+# CHECK-NEXT:  1      5     5.00    *             U     ficoms	(%ecx)
+# CHECK-NEXT:  1      5     5.00    *             U     ficoml	(%eax)
+# CHECK-NEXT:  1      5     5.00    *             U     ficomps	(%ecx)
+# CHECK-NEXT:  1      5     5.00    *             U     ficompl	(%eax)
 # CHECK-NEXT:  1      5     5.00    *             U     filds	(%edx)
 # CHECK-NEXT:  1      5     5.00    *             U     fildl	(%ecx)
 # CHECK-NEXT:  1      5     5.00    *             U     fildll	(%eax)
@@ -280,7 +280,7 @@ fyl2xp1
 # CHECK-NEXT:  1      1     1.00    *             U     fldl	(%ecx)
 # CHECK-NEXT:  1      4     2.00    *             U     fldt	(%eax)
 # CHECK-NEXT:  1      5     2.50    *             U     fldcw	(%eax)
-# CHECK-NEXT:  1      100   0.50                  U     fldenv	(%eax)
+# CHECK-NEXT:  1      100   0.50    *             U     fldenv	(%eax)
 # CHECK-NEXT:  1      6     3.00                  U     fld1
 # CHECK-NEXT:  1      10    5.00                  U     fldl2e
 # CHECK-NEXT:  1      10    5.00                  U     fldl2t
@@ -302,8 +302,8 @@ fyl2xp1
 # CHECK-NEXT:  1      71    35.50                 U     fprem1
 # CHECK-NEXT:  1      168   84.00                 U     fptan
 # CHECK-NEXT:  1      46    23.00                 U     frndint
-# CHECK-NEXT:  1      100   0.50                  U     frstor	(%eax)
-# CHECK-NEXT:  1      100   0.50                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.50    *             U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.50           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      77    38.50                 U     fscale
 # CHECK-NEXT:  1      174   87.00                 U     fsin
 # CHECK-NEXT:  1      174   87.00                 U     fsincos
@@ -316,11 +316,11 @@ fyl2xp1
 # CHECK-NEXT:  1      2     1.00           *      U     fstpl	(%ecx)
 # CHECK-NEXT:  1      5     2.50           *      U     fstpt	(%eax)
 # CHECK-NEXT:  1      8     4.00           *      U     fnstcw	(%eax)
-# CHECK-NEXT:  1      100   0.50                  U     fnstenv	(%eax)
-# CHECK-NEXT:  1      100   0.50                  U     fnstsw	(%eax)
-# CHECK-NEXT:  1      100   0.50                  U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.50           *      U     fnstenv	(%eax)
+# CHECK-NEXT:  1      100   0.50           *      U     fnstsw	(%eax)
+# CHECK-NEXT:  1      100   0.50    *             U     frstor	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     wait
-# CHECK-NEXT:  1      100   0.50                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.50           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      5     5.00                  U     fsub	%st, %st(1)
 # CHECK-NEXT:  1      5     5.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  1      5     5.00    *             U     fsubs	(%ecx)
diff --git a/llvm/test/tools/llvm-mca/X86/Barcelona/resources-x87.s b/llvm/test/tools/llvm-mca/X86/Barcelona/resources-x87.s
index 1f3e51e58b33c..715e62b3c25db 100644
--- a/llvm/test/tools/llvm-mca/X86/Barcelona/resources-x87.s
+++ b/llvm/test/tools/llvm-mca/X86/Barcelona/resources-x87.s
@@ -216,8 +216,8 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fiaddl	(%ecx)
-# CHECK-NEXT:  1      100   0.33                  U     fbld	(%ecx)
-# CHECK-NEXT:  1      100   0.33                  U     fbstp	(%eax)
+# CHECK-NEXT:  1      100   0.33    *             U     fbld	(%ecx)
+# CHECK-NEXT:  1      100   0.33           *      U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  1      100   0.33                  U     fnclex
 # CHECK-NEXT:  3      3     2.00                  U     fcmovb	%st(1), %st
@@ -230,12 +230,12 @@ fyl2xp1
 # CHECK-NEXT:  3      3     2.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(3)
-# CHECK-NEXT:  2      8     1.00                  U     fcoms	(%ecx)
-# CHECK-NEXT:  2      8     1.00                  U     fcoml	(%eax)
+# CHECK-NEXT:  2      8     1.00    *             U     fcoms	(%ecx)
+# CHECK-NEXT:  2      8     1.00    *             U     fcoml	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(3)
-# CHECK-NEXT:  2      8     1.00                  U     fcomps	(%ecx)
-# CHECK-NEXT:  2      8     1.00                  U     fcompl	(%eax)
+# CHECK-NEXT:  2      8     1.00    *             U     fcomps	(%ecx)
+# CHECK-NEXT:  2      8     1.00    *             U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   0.33                  U     fcompp
 # CHECK-NEXT:  3      3     1.00                  U     fcomi	%st(3), %st
 # CHECK-NEXT:  3      3     1.00                  U     fcompi	%st(3), %st
@@ -258,10 +258,10 @@ fyl2xp1
 # CHECK-NEXT:  3      34    1.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  3      34    1.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     ffree	%st(0)
-# CHECK-NEXT:  3      11    2.00                  U     ficoms	(%ecx)
-# CHECK-NEXT:  3      11    2.00                  U     ficoml	(%eax)
-# CHECK-NEXT:  3      11    2.00                  U     ficomps	(%ecx)
-# CHECK-NEXT:  3      11    2.00                  U     ficompl	(%eax)
+# CHECK-NEXT:  3      11    2.00    *             U     ficoms	(%ecx)
+# CHECK-NEXT:  3      11    2.00    *             U     ficoml	(%eax)
+# CHECK-NEXT:  3      11    2.00    *             U     ficomps	(%ecx)
+# CHECK-NEXT:  3      11    2.00    *             U     ficompl	(%eax)
 # CHECK-NEXT:  2      10    1.00    *             U     filds	(%edx)
 # CHECK-NEXT:  2      10    1.00    *             U     fildl	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     fildll	(%eax)
@@ -280,7 +280,7 @@ fyl2xp1
 # CHECK-NEXT:  3      9     1.00    *             U     fldl	(%ecx)
 # CHECK-NEXT:  3      9     1.00    *             U     fldt	(%eax)
 # CHECK-NEXT:  5      8     2.00    *             U     fldcw	(%eax)
-# CHECK-NEXT:  1      100   0.33                  U     fldenv	(%eax)
+# CHECK-NEXT:  1      100   0.33    *             U     fldenv	(%eax)
 # CHECK-NEXT:  2      1     1.00                  U     fld1
 # CHECK-NEXT:  2      1     1.00                  U     fldl2e
 # CHECK-NEXT:  2      1     1.00                  U     fldl2t
@@ -302,8 +302,8 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.33                  U     fprem1
 # CHECK-NEXT:  1      100   0.33                  U     fptan
 # CHECK-NEXT:  1      100   0.33                  U     frndint
-# CHECK-NEXT:  1      100   0.33                  U     frstor	(%eax)
-# CHECK-NEXT:  1      100   0.33                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.33    *             U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.33           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      100   0.33                  U     fscale
 # CHECK-NEXT:  1      100   0.33                  U     fsin
 # CHECK-NEXT:  1      100   0.33                  U     fsincos
@@ -316,11 +316,11 @@ fyl2xp1
 # CHECK-NEXT:  3      6     1.00           *      U     fstpl	(%ecx)
 # CHECK-NEXT:  3      6     1.00           *      U     fstpt	(%eax)
 # CHECK-NEXT:  4      7     1.00           *      U     fnstcw	(%eax)
-# CHECK-NEXT:  1      100   0.33                  U     fnstenv	(%eax)
-# CHECK-NEXT:  4      7     1.00                  U     fnstsw	(%eax)
-# CHECK-NEXT:  1      100   0.33                  U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.33           *      U     fnstenv	(%eax)
+# CHECK-NEXT:  4      7     1.00           *      U     fnstsw	(%eax)
+# CHECK-NEXT:  1      100   0.33    *             U     frstor	(%eax)
 # CHECK-NEXT:  1      100   0.33                  U     wait
-# CHECK-NEXT:  1      100   0.33                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.33           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fsubs	(%ecx)
diff --git a/llvm/test/tools/llvm-mca/X86/BdVer2/resources-x87.s b/llvm/test/tools/llvm-mca/X86/BdVer2/resources-x87.s
index f6bd8559508cb..8c3d5a63d21c9 100644
--- a/llvm/test/tools/llvm-mca/X86/BdVer2/resources-x87.s
+++ b/llvm/test/tools/llvm-mca/X86/BdVer2/resources-x87.s
@@ -216,8 +216,8 @@ fyl2xp1
 # CHECK-NEXT:  1      5     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  1      5     5.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  1      5     5.00    *             U     fiaddl	(%ecx)
-# CHECK-NEXT:  1      100   0.50                  U     fbld	(%ecx)
-# CHECK-NEXT:  1      100   0.50                  U     fbstp	(%eax)
+# CHECK-NEXT:  1      100   0.50    *             U     fbld	(%ecx)
+# CHECK-NEXT:  1      100   0.50           *      U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     2.00                  U     fchs
 # CHECK-NEXT:  1      100   0.50                  U     fnclex
 # CHECK-NEXT:  1      1     1.00                  U     fcmovb	%st(1), %st
@@ -230,12 +230,12 @@ fyl2xp1
 # CHECK-NEXT:  1      1     1.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  2      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  2      1     1.00                  U     fcom	%st(3)
-# CHECK-NEXT:  1      6     1.00                  U     fcoms	(%ecx)
-# CHECK-NEXT:  1      6     1.00                  U     fcoml	(%eax)
+# CHECK-NEXT:  1      6     1.00    *             U     fcoms	(%ecx)
+# CHECK-NEXT:  1      6     1.00    *             U     fcoml	(%eax)
 # CHECK-NEXT:  2      1     1.00                  U     fcomp	%st(1)
 # CHECK-NEXT:  2      1     1.00                  U     fcomp	%st(3)
-# CHECK-NEXT:  1      6     1.00                  U     fcomps	(%ecx)
-# CHECK-NEXT:  1      6     1.00                  U     fcompl	(%eax)
+# CHECK-NEXT:  1      6     1.00    *             U     fcomps	(%ecx)
+# CHECK-NEXT:  1      6     1.00    *             U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   0.50                  U     fcompp
 # CHECK-NEXT:  2      1     1.00                  U     fcomi	%st(3), %st
 # CHECK-NEXT:  2      1     1.00                  U     fcompi	%st(3), %st
@@ -258,10 +258,10 @@ fyl2xp1
 # CHECK-NEXT:  1      9     9.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  1      9     9.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      100   0.50                  U     ffree	%st(0)
-# CHECK-NEXT:  2      6     1.50                  U     ficoms	(%ecx)
-# CHECK-NEXT:  2      6     1.50                  U     ficoml	(%eax)
-# CHECK-NEXT:  2      6     1.50                  U     ficomps	(%ecx)
-# CHECK-NEXT:  2      6     1.50                  U     ficompl	(%eax)
+# CHECK-NEXT:  2      6     1.50    *             U     ficoms	(%ecx)
+# CHECK-NEXT:  2      6     1.50    *             U     ficoml	(%eax)
+# CHECK-NEXT:  2      6     1.50    *             U     ficomps	(%ecx)
+# CHECK-NEXT:  2      6     1.50    *             U     ficompl	(%eax)
 # CHECK-NEXT:  1      5     1.00    *             U     filds	(%edx)
 # CHECK-NEXT:  1      5     1.00    *             U     fildl	(%ecx)
 # CHECK-NEXT:  1      5     1.00    *             U     fildll	(%eax)
@@ -280,7 +280,7 @@ fyl2xp1
 # CHECK-NEXT:  1      5     1.00    *             U     fldl	(%ecx)
 # CHECK-NEXT:  1      5     1.00    *             U     fldt	(%eax)
 # CHECK-NEXT:  1      5     1.00    *             U     fldcw	(%eax)
-# CHECK-NEXT:  1      100   0.50                  U     fldenv	(%eax)
+# CHECK-NEXT:  1      100   0.50    *             U     fldenv	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     fld1
 # CHECK-NEXT:  1      3     1.00                  U     fldl2e
 # CHECK-NEXT:  1      3     1.00                  U     fldl2t
@@ -302,8 +302,8 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.50                  U     fprem1
 # CHECK-NEXT:  1      100   0.50                  U     fptan
 # CHECK-NEXT:  1      100   0.50                  U     frndint
-# CHECK-NEXT:  1      100   0.50                  U     frstor	(%eax)
-# CHECK-NEXT:  1      100   0.50                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.50    *             U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.50           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      100   0.50                  U     fscale
 # CHECK-NEXT:  1      100   0.50                  U     fsin
 # CHECK-NEXT:  1      100   0.50                  U     fsincos
@@ -316,11 +316,11 @@ fyl2xp1
 # CHECK-NEXT:  1      1     1.00           *      U     fstpl	(%ecx)
 # CHECK-NEXT:  1      1     1.00           *      U     fstpt	(%eax)
 # CHECK-NEXT:  1      1     1.00           *      U     fnstcw	(%eax)
-# CHECK-NEXT:  1      100   0.50                  U     fnstenv	(%eax)
-# CHECK-NEXT:  1      100   0.50                  U     fnstsw	(%eax)
-# CHECK-NEXT:  1      100   0.50                  U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.50           *      U     fnstenv	(%eax)
+# CHECK-NEXT:  1      100   0.50           *      U     fnstsw	(%eax)
+# CHECK-NEXT:  1      100   0.50    *             U     frstor	(%eax)
 # CHECK-NEXT:  1      100   0.50                  U     wait
-# CHECK-NEXT:  1      100   0.50                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.50           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      5     1.00                  U     fsub	%st, %st(1)
 # CHECK-NEXT:  1      5     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  1      5     5.00    *             U     fsubs	(%ecx)
diff --git a/llvm/test/tools/llvm-mca/X86/Broadwell/resources-x87.s b/llvm/test/tools/llvm-mca/X86/Broadwell/resources-x87.s
index 2f3a69da99d07..8c0d61ee209c6 100644
--- a/llvm/test/tools/llvm-mca/X86/Broadwell/resources-x87.s
+++ b/llvm/test/tools/llvm-mca/X86/Broadwell/resources-x87.s
@@ -216,8 +216,8 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  3      12    2.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  3      12    2.00    *             U     fiaddl	(%ecx)
-# CHECK-NEXT:  1      100   0.25                  U     fbld	(%ecx)
-# CHECK-NEXT:  2      1     1.00                  U     fbstp	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     fbld	(%ecx)
+# CHECK-NEXT:  2      1     1.00           *      U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  4      4     1.00                  U     fnclex
 # CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st
@@ -230,12 +230,12 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(3)
-# CHECK-NEXT:  2      7     1.00                  U     fcoms	(%ecx)
-# CHECK-NEXT:  2      7     1.00                  U     fcoml	(%eax)
+# CHECK-NEXT:  2      7     1.00    *             U     fcoms	(%ecx)
+# CHECK-NEXT:  2      7     1.00    *             U     fcoml	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(3)
-# CHECK-NEXT:  2      7     1.00                  U     fcomps	(%ecx)
-# CHECK-NEXT:  2      7     1.00                  U     fcompl	(%eax)
+# CHECK-NEXT:  2      7     1.00    *             U     fcomps	(%ecx)
+# CHECK-NEXT:  2      7     1.00    *             U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     fcompp
 # CHECK-NEXT:  1      3     1.00                  U     fcomi	%st(3), %st
 # CHECK-NEXT:  1      3     1.00                  U     fcompi	%st(3), %st
@@ -258,10 +258,10 @@ fyl2xp1
 # CHECK-NEXT:  3      29    1.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  3      29    1.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     ffree	%st(0)
-# CHECK-NEXT:  3      10    2.00                  U     ficoms	(%ecx)
-# CHECK-NEXT:  3      10    2.00                  U     ficoml	(%eax)
-# CHECK-NEXT:  3      10    2.00                  U     ficomps	(%ecx)
-# CHECK-NEXT:  3      10    2.00                  U     ficompl	(%eax)
+# CHECK-NEXT:  3      10    2.00    *             U     ficoms	(%ecx)
+# CHECK-NEXT:  3      10    2.00    *             U     ficoml	(%eax)
+# CHECK-NEXT:  3      10    2.00    *             U     ficomps	(%ecx)
+# CHECK-NEXT:  3      10    2.00    *             U     ficompl	(%eax)
 # CHECK-NEXT:  2      9     1.00    *             U     filds	(%edx)
 # CHECK-NEXT:  2      9     1.00    *             U     fildl	(%ecx)
 # CHECK-NEXT:  2      9     1.00    *             U     fildll	(%eax)
@@ -280,7 +280,7 @@ fyl2xp1
 # CHECK-NEXT:  1      6     0.50    *             U     fldl	(%ecx)
 # CHECK-NEXT:  1      6     0.50    *             U     fldt	(%eax)
 # CHECK-NEXT:  3      7     1.00    *             U     fldcw	(%eax)
-# CHECK-NEXT:  64     60    14.00                 U     fldenv	(%eax)
+# CHECK-NEXT:  64     60    14.00   *             U     fldenv	(%eax)
 # CHECK-NEXT:  2      1     1.00                  U     fld1
 # CHECK-NEXT:  2      1     1.00                  U     fldl2e
 # CHECK-NEXT:  2      1     1.00                  U     fldl2t
@@ -302,8 +302,8 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.25                  U     fprem1
 # CHECK-NEXT:  1      100   0.25                  U     fptan
 # CHECK-NEXT:  1      100   0.25                  U     frndint
-# CHECK-NEXT:  1      100   0.25                  U     frstor	(%eax)
-# CHECK-NEXT:  1      100   0.25                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.25           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     fscale
 # CHECK-NEXT:  1      100   0.25                  U     fsin
 # CHECK-NEXT:  1      100   0.25                  U     fsincos
@@ -316,11 +316,11 @@ fyl2xp1
 # CHECK-NEXT:  2      1     1.00           *      U     fstpl	(%ecx)
 # CHECK-NEXT:  2      1     1.00           *      U     fstpt	(%eax)
 # CHECK-NEXT:  3      2     1.00           *      U     fnstcw	(%eax)
-# CHECK-NEXT:  100    115   19.50                 U     fnstenv	(%eax)
-# CHECK-NEXT:  3      4     1.00                  U     fnstsw	(%eax)
-# CHECK-NEXT:  1      100   0.25                  U     frstor	(%eax)
+# CHECK-NEXT:  100    115   19.50          *      U     fnstenv	(%eax)
+# CHECK-NEXT:  3      4     1.00           *      U     fnstsw	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     frstor	(%eax)
 # CHECK-NEXT:  2      2     0.50                  U     wait
-# CHECK-NEXT:  1      100   0.25                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.25           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  2      9     1.00    *             U     fsubs	(%ecx)
diff --git a/llvm/test/tools/llvm-mca/X86/BtVer2/resources-x87.s b/llvm/test/tools/llvm-mca/X86/BtVer2/resources-x87.s
index f738be857c492..571db6861bb50 100644
--- a/llvm/test/tools/llvm-mca/X86/BtVer2/resources-x87.s
+++ b/llvm/test/tools/llvm-mca/X86/BtVer2/resources-x87.s
@@ -216,8 +216,8 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  1      8     1.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  1      8     1.00    *             U     fiaddl	(%ecx)
-# CHECK-NEXT:  1      100   0.50                  U     fbld	(%ecx)
-# CHECK-NEXT:  1      100   0.50                  U     fbstp	(%eax)
+# CHECK-NEXT:  1      100   0.50    *             U     fbld	(%ecx)
+# CHECK-NEXT:  1      100   0.50           *      U     fbstp	(%eax)
 # CHECK-NEXT:  1      2     1.00                  U     fchs
 # CHECK-NEXT:  1      100   0.50                  U     fnclex
 # CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st
@@ -230,12 +230,12 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      3     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fcom	%st(3)
-# CHECK-NEXT:  1      8     1.00                  U     fcoms	(%ecx)
-# CHECK-NEXT:  1      8     1.00                  U     fcoml	(%eax)
+# CHECK-NEXT:  1      8     1.00    *             U     fcoms	(%ecx)
+# CHECK-NEXT:  1      8     1.00    *             U     fcoml	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     fcomp	%st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fcomp	%st(3)
-# CHECK-NEXT:  1      8     1.00                  U     fcomps	(%ecx)
-# CHECK-NEXT:  1      8     1.00                  U     fcompl	(%eax)
+# CHECK-NEXT:  1      8     1.00    *             U     fcomps	(%ecx)
+# CHECK-NEXT:  1      8     1.00    *             U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   0.50                  U     fcompp
 # CHECK-NEXT:  1      3     1.00                  U     fcomi	%st(3), %st
 # CHECK-NEXT:  1      3     1.00                  U     fcompi	%st(3), %st
@@ -258,10 +258,10 @@ fyl2xp1
 # CHECK-NEXT:  1      24    19.00   *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  1      24    19.00   *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      100   0.50                  U     ffree	%st(0)
-# CHECK-NEXT:  1      8     1.00                  U     ficoms	(%ecx)
-# CHECK-NEXT:  1      8     1.00                  U     ficoml	(%eax)
-# CHECK-NEXT:  1      8     1.00                  U     ficomps	(%ecx)
-# CHECK-NEXT:  1      8     1.00                  U     ficompl	(%eax)
+# CHECK-NEXT:  1      8     1.00    *             U     ficoms	(%ecx)
+# CHECK-NEXT:  1      8     1.00    *             U     ficoml	(%eax)
+# CHECK-NEXT:  1      8     1.00    *             U     ficomps	(%ecx)
+# CHECK-NEXT:  1      8     1.00    *             U     ficompl	(%eax)
 # CHECK-NEXT:  1      3     1.00    *             U     filds	(%edx)
 # CHECK-NEXT:  1      3     1.00    *             U     fildl	(%ecx)
 # CHECK-NEXT:  1      3     1.00    *             U     fildll	(%eax)
@@ -280,7 +280,7 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00    *             U     fldl	(%ecx)
 # CHECK-NEXT:  1      3     1.00    *             U     fldt	(%eax)
 # CHECK-NEXT:  1      3     1.00    *             U     fldcw	(%eax)
-# CHECK-NEXT:  1      100   0.50                  U     fldenv	(%eax)
+# CHECK-NEXT:  1      100   0.50    *             U     fldenv	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     fld1
 # CHECK-NEXT:  1      3     1.00                  U     fldl2e
 # CHECK-NEXT:  1      3     1.00                  U     fldl2t
@@ -302,8 +302,8 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.50                  U     fprem1
 # CHECK-NEXT:  1      100   0.50                  U     fptan
 # CHECK-NEXT:  1      100   0.50                  U     frndint
-# CHECK-NEXT:  1      100   0.50                  U     frstor	(%eax)
-# CHECK-NEXT:  1      100   0.50                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.50    *             U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.50           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      100   0.50                  U     fscale
 # CHECK-NEXT:  1      100   0.50                  U     fsin
 # CHECK-NEXT:  1      100   0.50                  U     fsincos
@@ -316,11 +316,11 @@ fyl2xp1
 # CHECK-NEXT:  1      1     1.00           *      U     fstpl	(%ecx)
 # CHECK-NEXT:  1      1     1.00           *      U     fstpt	(%eax)
 # CHECK-NEXT:  1      1     0.50           *      U     fnstcw	(%eax)
-# CHECK-NEXT:  1      100   0.50                  U     fnstenv	(%eax)
-# CHECK-NEXT:  1      100   0.50                  U     fnstsw	(%eax)
-# CHECK-NEXT:  1      100   0.50                  U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.50           *      U     fnstenv	(%eax)
+# CHECK-NEXT:  1      100   0.50           *      U     fnstsw	(%eax)
+# CHECK-NEXT:  1      100   0.50    *             U     frstor	(%eax)
 # CHECK-NEXT:  1      100   0.50                  U     wait
-# CHECK-NEXT:  1      100   0.50                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.50           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  1      8     1.00    *             U     fsubs	(%ecx)
diff --git a/llvm/test/tools/llvm-mca/X86/Generic/resources-x87.s b/llvm/test/tools/llvm-mca/X86/Generic/resources-x87.s
index 1f3e51e58b33c..715e62b3c25db 100644
--- a/llvm/test/tools/llvm-mca/X86/Generic/resources-x87.s
+++ b/llvm/test/tools/llvm-mca/X86/Generic/resources-x87.s
@@ -216,8 +216,8 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fiaddl	(%ecx)
-# CHECK-NEXT:  1      100   0.33                  U     fbld	(%ecx)
-# CHECK-NEXT:  1      100   0.33                  U     fbstp	(%eax)
+# CHECK-NEXT:  1      100   0.33    *             U     fbld	(%ecx)
+# CHECK-NEXT:  1      100   0.33           *      U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  1      100   0.33                  U     fnclex
 # CHECK-NEXT:  3      3     2.00                  U     fcmovb	%st(1), %st
@@ -230,12 +230,12 @@ fyl2xp1
 # CHECK-NEXT:  3      3     2.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(3)
-# CHECK-NEXT:  2      8     1.00                  U     fcoms	(%ecx)
-# CHECK-NEXT:  2      8     1.00                  U     fcoml	(%eax)
+# CHECK-NEXT:  2      8     1.00    *             U     fcoms	(%ecx)
+# CHECK-NEXT:  2      8     1.00    *             U     fcoml	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(3)
-# CHECK-NEXT:  2      8     1.00                  U     fcomps	(%ecx)
-# CHECK-NEXT:  2      8     1.00                  U     fcompl	(%eax)
+# CHECK-NEXT:  2      8     1.00    *             U     fcomps	(%ecx)
+# CHECK-NEXT:  2      8     1.00    *             U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   0.33                  U     fcompp
 # CHECK-NEXT:  3      3     1.00                  U     fcomi	%st(3), %st
 # CHECK-NEXT:  3      3     1.00                  U     fcompi	%st(3), %st
@@ -258,10 +258,10 @@ fyl2xp1
 # CHECK-NEXT:  3      34    1.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  3      34    1.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     ffree	%st(0)
-# CHECK-NEXT:  3      11    2.00                  U     ficoms	(%ecx)
-# CHECK-NEXT:  3      11    2.00                  U     ficoml	(%eax)
-# CHECK-NEXT:  3      11    2.00                  U     ficomps	(%ecx)
-# CHECK-NEXT:  3      11    2.00                  U     ficompl	(%eax)
+# CHECK-NEXT:  3      11    2.00    *             U     ficoms	(%ecx)
+# CHECK-NEXT:  3      11    2.00    *             U     ficoml	(%eax)
+# CHECK-NEXT:  3      11    2.00    *             U     ficomps	(%ecx)
+# CHECK-NEXT:  3      11    2.00    *             U     ficompl	(%eax)
 # CHECK-NEXT:  2      10    1.00    *             U     filds	(%edx)
 # CHECK-NEXT:  2      10    1.00    *             U     fildl	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     fildll	(%eax)
@@ -280,7 +280,7 @@ fyl2xp1
 # CHECK-NEXT:  3      9     1.00    *             U     fldl	(%ecx)
 # CHECK-NEXT:  3      9     1.00    *             U     fldt	(%eax)
 # CHECK-NEXT:  5      8     2.00    *             U     fldcw	(%eax)
-# CHECK-NEXT:  1      100   0.33                  U     fldenv	(%eax)
+# CHECK-NEXT:  1      100   0.33    *             U     fldenv	(%eax)
 # CHECK-NEXT:  2      1     1.00                  U     fld1
 # CHECK-NEXT:  2      1     1.00                  U     fldl2e
 # CHECK-NEXT:  2      1     1.00                  U     fldl2t
@@ -302,8 +302,8 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.33                  U     fprem1
 # CHECK-NEXT:  1      100   0.33                  U     fptan
 # CHECK-NEXT:  1      100   0.33                  U     frndint
-# CHECK-NEXT:  1      100   0.33                  U     frstor	(%eax)
-# CHECK-NEXT:  1      100   0.33                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.33    *             U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.33           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      100   0.33                  U     fscale
 # CHECK-NEXT:  1      100   0.33                  U     fsin
 # CHECK-NEXT:  1      100   0.33                  U     fsincos
@@ -316,11 +316,11 @@ fyl2xp1
 # CHECK-NEXT:  3      6     1.00           *      U     fstpl	(%ecx)
 # CHECK-NEXT:  3      6     1.00           *      U     fstpt	(%eax)
 # CHECK-NEXT:  4      7     1.00           *      U     fnstcw	(%eax)
-# CHECK-NEXT:  1      100   0.33                  U     fnstenv	(%eax)
-# CHECK-NEXT:  4      7     1.00                  U     fnstsw	(%eax)
-# CHECK-NEXT:  1      100   0.33                  U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.33           *      U     fnstenv	(%eax)
+# CHECK-NEXT:  4      7     1.00           *      U     fnstsw	(%eax)
+# CHECK-NEXT:  1      100   0.33    *             U     frstor	(%eax)
 # CHECK-NEXT:  1      100   0.33                  U     wait
-# CHECK-NEXT:  1      100   0.33                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.33           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fsubs	(%ecx)
diff --git a/llvm/test/tools/llvm-mca/X86/Haswell/resources-x87.s b/llvm/test/tools/llvm-mca/X86/Haswell/resources-x87.s
index 7da8b2802a9a1..869d52304faf8 100644
--- a/llvm/test/tools/llvm-mca/X86/Haswell/resources-x87.s
+++ b/llvm/test/tools/llvm-mca/X86/Haswell/resources-x87.s
@@ -216,8 +216,8 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fiaddl	(%ecx)
-# CHECK-NEXT:  43     47    10.75                 U     fbld	(%ecx)
-# CHECK-NEXT:  2      1     1.00                  U     fbstp	(%eax)
+# CHECK-NEXT:  43     47    10.75   *             U     fbld	(%ecx)
+# CHECK-NEXT:  2      1     1.00           *      U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  4      4     1.00                  U     fnclex
 # CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st
@@ -230,12 +230,12 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(3)
-# CHECK-NEXT:  2      8     1.00                  U     fcoms	(%ecx)
-# CHECK-NEXT:  2      8     1.00                  U     fcoml	(%eax)
+# CHECK-NEXT:  2      8     1.00    *             U     fcoms	(%ecx)
+# CHECK-NEXT:  2      8     1.00    *             U     fcoml	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(3)
-# CHECK-NEXT:  2      8     1.00                  U     fcomps	(%ecx)
-# CHECK-NEXT:  2      8     1.00                  U     fcompl	(%eax)
+# CHECK-NEXT:  2      8     1.00    *             U     fcomps	(%ecx)
+# CHECK-NEXT:  2      8     1.00    *             U     fcompl	(%eax)
 # CHECK-NEXT:  2      1     0.50                  U     fcompp
 # CHECK-NEXT:  3      1     0.50                  U     fcomi	%st(3), %st
 # CHECK-NEXT:  3      1     0.50                  U     fcompi	%st(3), %st
@@ -258,10 +258,10 @@ fyl2xp1
 # CHECK-NEXT:  3      30    1.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  3      30    1.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     ffree	%st(0)
-# CHECK-NEXT:  3      11    2.00                  U     ficoms	(%ecx)
-# CHECK-NEXT:  3      11    2.00                  U     ficoml	(%eax)
-# CHECK-NEXT:  3      11    2.00                  U     ficomps	(%ecx)
-# CHECK-NEXT:  3      11    2.00                  U     ficompl	(%eax)
+# CHECK-NEXT:  3      11    2.00    *             U     ficoms	(%ecx)
+# CHECK-NEXT:  3      11    2.00    *             U     ficoml	(%eax)
+# CHECK-NEXT:  3      11    2.00    *             U     ficomps	(%ecx)
+# CHECK-NEXT:  3      11    2.00    *             U     ficompl	(%eax)
 # CHECK-NEXT:  2      10    1.00    *             U     filds	(%edx)
 # CHECK-NEXT:  2      10    1.00    *             U     fildl	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     fildll	(%eax)
@@ -280,7 +280,7 @@ fyl2xp1
 # CHECK-NEXT:  1      7     0.50    *             U     fldl	(%ecx)
 # CHECK-NEXT:  1      7     0.50    *             U     fldt	(%eax)
 # CHECK-NEXT:  3      7     1.00    *             U     fldcw	(%eax)
-# CHECK-NEXT:  64     61    14.00                 U     fldenv	(%eax)
+# CHECK-NEXT:  64     61    14.00   *             U     fldenv	(%eax)
 # CHECK-NEXT:  2      1     1.00                  U     fld1
 # CHECK-NEXT:  2      1     1.00                  U     fldl2e
 # CHECK-NEXT:  2      1     1.00                  U     fldl2t
@@ -302,8 +302,8 @@ fyl2xp1
 # CHECK-NEXT:  41     27    10.25                 U     fprem1
 # CHECK-NEXT:  1      100   0.25                  U     fptan
 # CHECK-NEXT:  17     11    4.25                  U     frndint
-# CHECK-NEXT:  90     1     22.50                 U     frstor	(%eax)
-# CHECK-NEXT:  147    1     36.75                 U     fnsave	(%eax)
+# CHECK-NEXT:  90     1     22.50   *             U     frstor	(%eax)
+# CHECK-NEXT:  147    1     36.75          *      U     fnsave	(%eax)
 # CHECK-NEXT:  50     75    12.50                 U     fscale
 # CHECK-NEXT:  1      100   0.25                  U     fsin
 # CHECK-NEXT:  1      100   0.25                  U     fsincos
@@ -316,11 +316,11 @@ fyl2xp1
 # CHECK-NEXT:  2      1     1.00           *      U     fstpl	(%ecx)
 # CHECK-NEXT:  2      1     1.00           *      U     fstpt	(%eax)
 # CHECK-NEXT:  3      2     1.00           *      U     fnstcw	(%eax)
-# CHECK-NEXT:  100    115   19.50                 U     fnstenv	(%eax)
-# CHECK-NEXT:  3      4     1.00                  U     fnstsw	(%eax)
-# CHECK-NEXT:  90     1     22.50                 U     frstor	(%eax)
+# CHECK-NEXT:  100    115   19.50          *      U     fnstenv	(%eax)
+# CHECK-NEXT:  3      4     1.00           *      U     fnstsw	(%eax)
+# CHECK-NEXT:  90     1     22.50   *             U     frstor	(%eax)
 # CHECK-NEXT:  2      2     0.50                  U     wait
-# CHECK-NEXT:  147    1     36.75                 U     fnsave	(%eax)
+# CHECK-NEXT:  147    1     36.75          *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fsubs	(%ecx)
diff --git a/llvm/test/tools/llvm-mca/X86/SLM/resources-x87.s b/llvm/test/tools/llvm-mca/X86/SLM/resources-x87.s
index d6d42e957bfe3..4531d50e1df08 100644
--- a/llvm/test/tools/llvm-mca/X86/SLM/resources-x87.s
+++ b/llvm/test/tools/llvm-mca/X86/SLM/resources-x87.s
@@ -216,8 +216,8 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  1      6     1.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  1      6     1.00    *             U     fiaddl	(%ecx)
-# CHECK-NEXT:  1      100   1.00                  U     fbld	(%ecx)
-# CHECK-NEXT:  1      100   1.00                  U     fbstp	(%eax)
+# CHECK-NEXT:  1      100   1.00    *             U     fbld	(%ecx)
+# CHECK-NEXT:  1      100   1.00           *      U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     fchs
 # CHECK-NEXT:  1      100   1.00                  U     fnclex
 # CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st
@@ -230,12 +230,12 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      3     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fcom	%st(3)
-# CHECK-NEXT:  1      6     1.00                  U     fcoms	(%ecx)
-# CHECK-NEXT:  1      6     1.00                  U     fcoml	(%eax)
+# CHECK-NEXT:  1      6     1.00    *             U     fcoms	(%ecx)
+# CHECK-NEXT:  1      6     1.00    *             U     fcoml	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     fcomp	%st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fcomp	%st(3)
-# CHECK-NEXT:  1      6     1.00                  U     fcomps	(%ecx)
-# CHECK-NEXT:  1      6     1.00                  U     fcompl	(%eax)
+# CHECK-NEXT:  1      6     1.00    *             U     fcomps	(%ecx)
+# CHECK-NEXT:  1      6     1.00    *             U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   1.00                  U     fcompp
 # CHECK-NEXT:  1      3     1.00                  U     fcomi	%st(3), %st
 # CHECK-NEXT:  1      3     1.00                  U     fcompi	%st(3), %st
@@ -258,10 +258,10 @@ fyl2xp1
 # CHECK-NEXT:  1      22    17.00   *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  1      22    17.00   *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      100   1.00                  U     ffree	%st(0)
-# CHECK-NEXT:  1      6     1.00                  U     ficoms	(%ecx)
-# CHECK-NEXT:  1      6     1.00                  U     ficoml	(%eax)
-# CHECK-NEXT:  1      6     1.00                  U     ficomps	(%ecx)
-# CHECK-NEXT:  1      6     1.00                  U     ficompl	(%eax)
+# CHECK-NEXT:  1      6     1.00    *             U     ficoms	(%ecx)
+# CHECK-NEXT:  1      6     1.00    *             U     ficoml	(%eax)
+# CHECK-NEXT:  1      6     1.00    *             U     ficomps	(%ecx)
+# CHECK-NEXT:  1      6     1.00    *             U     ficompl	(%eax)
 # CHECK-NEXT:  1      3     1.00    *             U     filds	(%edx)
 # CHECK-NEXT:  1      3     1.00    *             U     fildl	(%ecx)
 # CHECK-NEXT:  1      3     1.00    *             U     fildll	(%eax)
@@ -280,7 +280,7 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00    *             U     fldl	(%ecx)
 # CHECK-NEXT:  1      3     1.00    *             U     fldt	(%eax)
 # CHECK-NEXT:  1      3     1.00    *             U     fldcw	(%eax)
-# CHECK-NEXT:  1      100   1.00                  U     fldenv	(%eax)
+# CHECK-NEXT:  1      100   1.00    *             U     fldenv	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     fld1
 # CHECK-NEXT:  2      1     1.00                  U     fldl2e
 # CHECK-NEXT:  2      1     1.00                  U     fldl2t
@@ -302,8 +302,8 @@ fyl2xp1
 # CHECK-NEXT:  1      100   1.00                  U     fprem1
 # CHECK-NEXT:  1      100   1.00                  U     fptan
 # CHECK-NEXT:  1      100   1.00                  U     frndint
-# CHECK-NEXT:  1      100   1.00                  U     frstor	(%eax)
-# CHECK-NEXT:  1      100   1.00                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   1.00    *             U     frstor	(%eax)
+# CHECK-NEXT:  1      100   1.00           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      100   1.00                  U     fscale
 # CHECK-NEXT:  1      100   1.00                  U     fsin
 # CHECK-NEXT:  1      100   1.00                  U     fsincos
@@ -316,11 +316,11 @@ fyl2xp1
 # CHECK-NEXT:  1      1     1.00           *      U     fstpl	(%ecx)
 # CHECK-NEXT:  1      1     1.00           *      U     fstpt	(%eax)
 # CHECK-NEXT:  1      1     0.50           *      U     fnstcw	(%eax)
-# CHECK-NEXT:  1      100   1.00                  U     fnstenv	(%eax)
-# CHECK-NEXT:  1      100   1.00                  U     fnstsw	(%eax)
-# CHECK-NEXT:  1      100   1.00                  U     frstor	(%eax)
+# CHECK-NEXT:  1      100   1.00           *      U     fnstenv	(%eax)
+# CHECK-NEXT:  1      100   1.00           *      U     fnstsw	(%eax)
+# CHECK-NEXT:  1      100   1.00    *             U     frstor	(%eax)
 # CHECK-NEXT:  1      100   1.00                  U     wait
-# CHECK-NEXT:  1      100   1.00                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   1.00           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  1      6     1.00    *             U     fsubs	(%ecx)
diff --git a/llvm/test/tools/llvm-mca/X86/SandyBridge/resources-x87.s b/llvm/test/tools/llvm-mca/X86/SandyBridge/resources-x87.s
index 1bed53326ced6..b90d0cb5c6bd5 100644
--- a/llvm/test/tools/llvm-mca/X86/SandyBridge/resources-x87.s
+++ b/llvm/test/tools/llvm-mca/X86/SandyBridge/resources-x87.s
@@ -216,8 +216,8 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fiaddl	(%ecx)
-# CHECK-NEXT:  1      100   0.33                  U     fbld	(%ecx)
-# CHECK-NEXT:  1      100   0.33                  U     fbstp	(%eax)
+# CHECK-NEXT:  1      100   0.33    *             U     fbld	(%ecx)
+# CHECK-NEXT:  1      100   0.33           *      U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  1      100   0.33                  U     fnclex
 # CHECK-NEXT:  3      3     2.00                  U     fcmovb	%st(1), %st
@@ -230,12 +230,12 @@ fyl2xp1
 # CHECK-NEXT:  3      3     2.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(3)
-# CHECK-NEXT:  2      8     1.00                  U     fcoms	(%ecx)
-# CHECK-NEXT:  2      8     1.00                  U     fcoml	(%eax)
+# CHECK-NEXT:  2      8     1.00    *             U     fcoms	(%ecx)
+# CHECK-NEXT:  2      8     1.00    *             U     fcoml	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(3)
-# CHECK-NEXT:  2      8     1.00                  U     fcomps	(%ecx)
-# CHECK-NEXT:  2      8     1.00                  U     fcompl	(%eax)
+# CHECK-NEXT:  2      8     1.00    *             U     fcomps	(%ecx)
+# CHECK-NEXT:  2      8     1.00    *             U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   0.33                  U     fcompp
 # CHECK-NEXT:  3      3     1.00                  U     fcomi	%st(3), %st
 # CHECK-NEXT:  3      3     1.00                  U     fcompi	%st(3), %st
@@ -258,10 +258,10 @@ fyl2xp1
 # CHECK-NEXT:  3      34    1.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  3      34    1.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     ffree	%st(0)
-# CHECK-NEXT:  3      11    2.00                  U     ficoms	(%ecx)
-# CHECK-NEXT:  3      11    2.00                  U     ficoml	(%eax)
-# CHECK-NEXT:  3      11    2.00                  U     ficomps	(%ecx)
-# CHECK-NEXT:  3      11    2.00                  U     ficompl	(%eax)
+# CHECK-NEXT:  3      11    2.00    *             U     ficoms	(%ecx)
+# CHECK-NEXT:  3      11    2.00    *             U     ficoml	(%eax)
+# CHECK-NEXT:  3      11    2.00    *             U     ficomps	(%ecx)
+# CHECK-NEXT:  3      11    2.00    *             U     ficompl	(%eax)
 # CHECK-NEXT:  2      10    1.00    *             U     filds	(%edx)
 # CHECK-NEXT:  2      10    1.00    *             U     fildl	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     fildll	(%eax)
@@ -280,7 +280,7 @@ fyl2xp1
 # CHECK-NEXT:  3      9     1.00    *             U     fldl	(%ecx)
 # CHECK-NEXT:  3      9     1.00    *             U     fldt	(%eax)
 # CHECK-NEXT:  5      8     2.00    *             U     fldcw	(%eax)
-# CHECK-NEXT:  1      100   0.33                  U     fldenv	(%eax)
+# CHECK-NEXT:  1      100   0.33    *             U     fldenv	(%eax)
 # CHECK-NEXT:  2      1     1.00                  U     fld1
 # CHECK-NEXT:  2      1     1.00                  U     fldl2e
 # CHECK-NEXT:  2      1     1.00                  U     fldl2t
@@ -302,8 +302,8 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.33                  U     fprem1
 # CHECK-NEXT:  1      100   0.33                  U     fptan
 # CHECK-NEXT:  1      100   0.33                  U     frndint
-# CHECK-NEXT:  1      100   0.33                  U     frstor	(%eax)
-# CHECK-NEXT:  1      100   0.33                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.33    *             U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.33           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      100   0.33                  U     fscale
 # CHECK-NEXT:  1      100   0.33                  U     fsin
 # CHECK-NEXT:  1      100   0.33                  U     fsincos
@@ -316,11 +316,11 @@ fyl2xp1
 # CHECK-NEXT:  3      6     1.00           *      U     fstpl	(%ecx)
 # CHECK-NEXT:  3      6     1.00           *      U     fstpt	(%eax)
 # CHECK-NEXT:  4      7     1.00           *      U     fnstcw	(%eax)
-# CHECK-NEXT:  1      100   0.33                  U     fnstenv	(%eax)
-# CHECK-NEXT:  4      7     1.00                  U     fnstsw	(%eax)
-# CHECK-NEXT:  1      100   0.33                  U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.33           *      U     fnstenv	(%eax)
+# CHECK-NEXT:  4      7     1.00           *      U     fnstsw	(%eax)
+# CHECK-NEXT:  1      100   0.33    *             U     frstor	(%eax)
 # CHECK-NEXT:  1      100   0.33                  U     wait
-# CHECK-NEXT:  1      100   0.33                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.33           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fsubs	(%ecx)
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-x87.s b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-x87.s
index 6cd4439a25c1f..93ed46fe309a5 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-x87.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-x87.s
@@ -216,8 +216,8 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fiaddl	(%ecx)
-# CHECK-NEXT:  1      100   0.25                  U     fbld	(%ecx)
-# CHECK-NEXT:  2      1     1.00                  U     fbstp	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     fbld	(%ecx)
+# CHECK-NEXT:  2      1     1.00           *      U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  4      4     1.00                  U     fnclex
 # CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st
@@ -230,12 +230,12 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(3)
-# CHECK-NEXT:  2      8     1.00                  U     fcoms	(%ecx)
-# CHECK-NEXT:  2      8     1.00                  U     fcoml	(%eax)
+# CHECK-NEXT:  2      8     1.00    *             U     fcoms	(%ecx)
+# CHECK-NEXT:  2      8     1.00    *             U     fcoml	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(3)
-# CHECK-NEXT:  2      8     1.00                  U     fcomps	(%ecx)
-# CHECK-NEXT:  2      8     1.00                  U     fcompl	(%eax)
+# CHECK-NEXT:  2      8     1.00    *             U     fcomps	(%ecx)
+# CHECK-NEXT:  2      8     1.00    *             U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     fcompp
 # CHECK-NEXT:  1      2     1.00                  U     fcomi	%st(3), %st
 # CHECK-NEXT:  1      2     1.00                  U     fcompi	%st(3), %st
@@ -258,10 +258,10 @@ fyl2xp1
 # CHECK-NEXT:  3      30    1.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  3      30    1.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     ffree	%st(0)
-# CHECK-NEXT:  3      11    2.00                  U     ficoms	(%ecx)
-# CHECK-NEXT:  3      11    2.00                  U     ficoml	(%eax)
-# CHECK-NEXT:  3      11    2.00                  U     ficomps	(%ecx)
-# CHECK-NEXT:  3      11    2.00                  U     ficompl	(%eax)
+# CHECK-NEXT:  3      11    2.00    *             U     ficoms	(%ecx)
+# CHECK-NEXT:  3      11    2.00    *             U     ficoml	(%eax)
+# CHECK-NEXT:  3      11    2.00    *             U     ficomps	(%ecx)
+# CHECK-NEXT:  3      11    2.00    *             U     ficompl	(%eax)
 # CHECK-NEXT:  2      10    1.00    *             U     filds	(%edx)
 # CHECK-NEXT:  2      10    1.00    *             U     fildl	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     fildll	(%eax)
@@ -280,7 +280,7 @@ fyl2xp1
 # CHECK-NEXT:  1      7     0.50    *             U     fldl	(%ecx)
 # CHECK-NEXT:  1      7     0.50    *             U     fldt	(%eax)
 # CHECK-NEXT:  3      7     1.00    *             U     fldcw	(%eax)
-# CHECK-NEXT:  64     62    14.00                 U     fldenv	(%eax)
+# CHECK-NEXT:  64     62    14.00   *             U     fldenv	(%eax)
 # CHECK-NEXT:  2      1     1.00                  U     fld1
 # CHECK-NEXT:  2      1     1.00                  U     fldl2e
 # CHECK-NEXT:  2      1     1.00                  U     fldl2t
@@ -302,8 +302,8 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.25                  U     fprem1
 # CHECK-NEXT:  1      100   0.25                  U     fptan
 # CHECK-NEXT:  1      100   0.25                  U     frndint
-# CHECK-NEXT:  1      100   0.25                  U     frstor	(%eax)
-# CHECK-NEXT:  1      100   0.25                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.25           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     fscale
 # CHECK-NEXT:  1      100   0.25                  U     fsin
 # CHECK-NEXT:  1      100   0.25                  U     fsincos
@@ -316,11 +316,11 @@ fyl2xp1
 # CHECK-NEXT:  2      1     1.00           *      U     fstpl	(%ecx)
 # CHECK-NEXT:  2      1     1.00           *      U     fstpt	(%eax)
 # CHECK-NEXT:  3      2     1.00           *      U     fnstcw	(%eax)
-# CHECK-NEXT:  100    106   19.50                 U     fnstenv	(%eax)
-# CHECK-NEXT:  3      3     1.00                  U     fnstsw	(%eax)
-# CHECK-NEXT:  1      100   0.25                  U     frstor	(%eax)
+# CHECK-NEXT:  100    106   19.50          *      U     fnstenv	(%eax)
+# CHECK-NEXT:  3      3     1.00           *      U     fnstsw	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     frstor	(%eax)
 # CHECK-NEXT:  2      2     0.50                  U     wait
-# CHECK-NEXT:  1      100   0.25                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.25           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fsubs	(%ecx)
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-x87.s b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-x87.s
index 75cca5297704b..af5e5f2d567fa 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-x87.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-x87.s
@@ -216,8 +216,8 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fiaddl	(%ecx)
-# CHECK-NEXT:  1      100   0.25                  U     fbld	(%ecx)
-# CHECK-NEXT:  2      1     1.00                  U     fbstp	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     fbld	(%ecx)
+# CHECK-NEXT:  2      1     1.00           *      U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  4      4     1.00                  U     fnclex
 # CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st
@@ -230,12 +230,12 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(3)
-# CHECK-NEXT:  2      8     1.00                  U     fcoms	(%ecx)
-# CHECK-NEXT:  2      8     1.00                  U     fcoml	(%eax)
+# CHECK-NEXT:  2      8     1.00    *             U     fcoms	(%ecx)
+# CHECK-NEXT:  2      8     1.00    *             U     fcoml	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(3)
-# CHECK-NEXT:  2      8     1.00                  U     fcomps	(%ecx)
-# CHECK-NEXT:  2      8     1.00                  U     fcompl	(%eax)
+# CHECK-NEXT:  2      8     1.00    *             U     fcomps	(%ecx)
+# CHECK-NEXT:  2      8     1.00    *             U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     fcompp
 # CHECK-NEXT:  1      2     1.00                  U     fcomi	%st(3), %st
 # CHECK-NEXT:  1      2     1.00                  U     fcompi	%st(3), %st
@@ -258,10 +258,10 @@ fyl2xp1
 # CHECK-NEXT:  3      30    1.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  3      30    1.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     ffree	%st(0)
-# CHECK-NEXT:  3      11    2.00                  U     ficoms	(%ecx)
-# CHECK-NEXT:  3      11    2.00                  U     ficoml	(%eax)
-# CHECK-NEXT:  3      11    2.00                  U     ficomps	(%ecx)
-# CHECK-NEXT:  3      11    2.00                  U     ficompl	(%eax)
+# CHECK-NEXT:  3      11    2.00    *             U     ficoms	(%ecx)
+# CHECK-NEXT:  3      11    2.00    *             U     ficoml	(%eax)
+# CHECK-NEXT:  3      11    2.00    *             U     ficomps	(%ecx)
+# CHECK-NEXT:  3      11    2.00    *             U     ficompl	(%eax)
 # CHECK-NEXT:  2      10    1.00    *             U     filds	(%edx)
 # CHECK-NEXT:  2      10    1.00    *             U     fildl	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     fildll	(%eax)
@@ -280,7 +280,7 @@ fyl2xp1
 # CHECK-NEXT:  1      7     0.50    *             U     fldl	(%ecx)
 # CHECK-NEXT:  1      7     0.50    *             U     fldt	(%eax)
 # CHECK-NEXT:  3      7     1.00    *             U     fldcw	(%eax)
-# CHECK-NEXT:  64     62    14.00                 U     fldenv	(%eax)
+# CHECK-NEXT:  64     62    14.00   *             U     fldenv	(%eax)
 # CHECK-NEXT:  2      1     1.00                  U     fld1
 # CHECK-NEXT:  2      1     1.00                  U     fldl2e
 # CHECK-NEXT:  2      1     1.00                  U     fldl2t
@@ -302,8 +302,8 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.25                  U     fprem1
 # CHECK-NEXT:  1      100   0.25                  U     fptan
 # CHECK-NEXT:  1      100   0.25                  U     frndint
-# CHECK-NEXT:  1      100   0.25                  U     frstor	(%eax)
-# CHECK-NEXT:  1      100   0.25                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.25           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     fscale
 # CHECK-NEXT:  1      100   0.25                  U     fsin
 # CHECK-NEXT:  1      100   0.25                  U     fsincos
@@ -316,11 +316,11 @@ fyl2xp1
 # CHECK-NEXT:  2      1     1.00           *      U     fstpl	(%ecx)
 # CHECK-NEXT:  2      1     1.00           *      U     fstpt	(%eax)
 # CHECK-NEXT:  3      2     1.00           *      U     fnstcw	(%eax)
-# CHECK-NEXT:  100    106   19.50                 U     fnstenv	(%eax)
-# CHECK-NEXT:  3      3     1.00                  U     fnstsw	(%eax)
-# CHECK-NEXT:  1      100   0.25                  U     frstor	(%eax)
+# CHECK-NEXT:  100    106   19.50          *      U     fnstenv	(%eax)
+# CHECK-NEXT:  3      3     1.00           *      U     fnstsw	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     frstor	(%eax)
 # CHECK-NEXT:  2      2     0.50                  U     wait
-# CHECK-NEXT:  1      100   0.25                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.25           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fsubs	(%ecx)
diff --git a/llvm/test/tools/llvm-mca/X86/Znver1/resources-x87.s b/llvm/test/tools/llvm-mca/X86/Znver1/resources-x87.s
index 030b71fb7b63f..0f7807eb3e79d 100644
--- a/llvm/test/tools/llvm-mca/X86/Znver1/resources-x87.s
+++ b/llvm/test/tools/llvm-mca/X86/Znver1/resources-x87.s
@@ -216,8 +216,8 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  1      10    1.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  1      10    1.00    *             U     fiaddl	(%ecx)
-# CHECK-NEXT:  1      100   0.25                  U     fbld	(%ecx)
-# CHECK-NEXT:  1      100   0.25                  U     fbstp	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     fbld	(%ecx)
+# CHECK-NEXT:  1      100   0.25           *      U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  1      100   0.25                  U     fnclex
 # CHECK-NEXT:  1      100   0.25                  U     fcmovb	%st(1), %st
@@ -230,12 +230,12 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.25                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(3)
-# CHECK-NEXT:  1      8     1.00                  U     fcoms	(%ecx)
-# CHECK-NEXT:  1      8     1.00                  U     fcoml	(%eax)
+# CHECK-NEXT:  1      8     1.00    *             U     fcoms	(%ecx)
+# CHECK-NEXT:  1      8     1.00    *             U     fcoml	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(3)
-# CHECK-NEXT:  1      8     1.00                  U     fcomps	(%ecx)
-# CHECK-NEXT:  1      8     1.00                  U     fcompl	(%eax)
+# CHECK-NEXT:  1      8     1.00    *             U     fcomps	(%ecx)
+# CHECK-NEXT:  1      8     1.00    *             U     fcompl	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fcompp
 # CHECK-NEXT:  1      9     0.50                  U     fcomi	%st(3), %st
 # CHECK-NEXT:  1      9     0.50                  U     fcompi	%st(3), %st
@@ -258,10 +258,10 @@ fyl2xp1
 # CHECK-NEXT:  1      22    1.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  1      22    1.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      11    1.00                  U     ffree	%st(0)
-# CHECK-NEXT:  2      12    1.50                  U     ficoms	(%ecx)
-# CHECK-NEXT:  2      12    1.50                  U     ficoml	(%eax)
-# CHECK-NEXT:  2      12    1.50                  U     ficomps	(%ecx)
-# CHECK-NEXT:  2      12    1.50                  U     ficompl	(%eax)
+# CHECK-NEXT:  2      12    1.50    *             U     ficoms	(%ecx)
+# CHECK-NEXT:  2      12    1.50    *             U     ficoml	(%eax)
+# CHECK-NEXT:  2      12    1.50    *             U     ficomps	(%ecx)
+# CHECK-NEXT:  2      12    1.50    *             U     ficompl	(%eax)
 # CHECK-NEXT:  2      11    1.00    *             U     filds	(%edx)
 # CHECK-NEXT:  2      11    1.00    *             U     fildl	(%ecx)
 # CHECK-NEXT:  2      11    1.00    *             U     fildll	(%eax)
@@ -280,7 +280,7 @@ fyl2xp1
 # CHECK-NEXT:  1      8     0.50    *             U     fldl	(%ecx)
 # CHECK-NEXT:  2      1     0.50    *             U     fldt	(%eax)
 # CHECK-NEXT:  1      100   0.25    *             U     fldcw	(%eax)
-# CHECK-NEXT:  1      100   0.25                  U     fldenv	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     fldenv	(%eax)
 # CHECK-NEXT:  1      11    1.00                  U     fld1
 # CHECK-NEXT:  1      11    1.00                  U     fldl2e
 # CHECK-NEXT:  1      11    1.00                  U     fldl2t
@@ -302,8 +302,8 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.25                  U     fprem1
 # CHECK-NEXT:  1      100   0.25                  U     fptan
 # CHECK-NEXT:  1      100   0.25                  U     frndint
-# CHECK-NEXT:  1      100   0.25                  U     frstor	(%eax)
-# CHECK-NEXT:  1      100   0.25                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.25           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     fscale
 # CHECK-NEXT:  1      100   0.25                  U     fsin
 # CHECK-NEXT:  1      100   0.25                  U     fsincos
@@ -316,11 +316,11 @@ fyl2xp1
 # CHECK-NEXT:  1      1     0.50           *      U     fstpl	(%ecx)
 # CHECK-NEXT:  1      5     0.50           *      U     fstpt	(%eax)
 # CHECK-NEXT:  1      100   0.25           *      U     fnstcw	(%eax)
-# CHECK-NEXT:  1      100   0.25                  U     fnstenv	(%eax)
-# CHECK-NEXT:  1      100   0.25                  U     fnstsw	(%eax)
-# CHECK-NEXT:  1      100   0.25                  U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.25           *      U     fnstenv	(%eax)
+# CHECK-NEXT:  1      100   0.25           *      U     fnstsw	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     frstor	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     wait
-# CHECK-NEXT:  1      100   0.25                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.25           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  1      10    1.00    *             U     fsubs	(%ecx)
diff --git a/llvm/test/tools/llvm-mca/X86/Znver2/resources-x87.s b/llvm/test/tools/llvm-mca/X86/Znver2/resources-x87.s
index 1e43a999c3adc..55e646c79275a 100644
--- a/llvm/test/tools/llvm-mca/X86/Znver2/resources-x87.s
+++ b/llvm/test/tools/llvm-mca/X86/Znver2/resources-x87.s
@@ -216,8 +216,8 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  1      10    1.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  1      10    1.00    *             U     fiaddl	(%ecx)
-# CHECK-NEXT:  1      100   0.25                  U     fbld	(%ecx)
-# CHECK-NEXT:  1      100   0.25                  U     fbstp	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     fbld	(%ecx)
+# CHECK-NEXT:  1      100   0.25           *      U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  1      100   0.25                  U     fnclex
 # CHECK-NEXT:  1      100   0.25                  U     fcmovb	%st(1), %st
@@ -230,12 +230,12 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.25                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(3)
-# CHECK-NEXT:  1      8     1.00                  U     fcoms	(%ecx)
-# CHECK-NEXT:  1      8     1.00                  U     fcoml	(%eax)
+# CHECK-NEXT:  1      8     1.00    *             U     fcoms	(%ecx)
+# CHECK-NEXT:  1      8     1.00    *             U     fcoml	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcomp	%st(3)
-# CHECK-NEXT:  1      8     1.00                  U     fcomps	(%ecx)
-# CHECK-NEXT:  1      8     1.00                  U     fcompl	(%eax)
+# CHECK-NEXT:  1      8     1.00    *             U     fcomps	(%ecx)
+# CHECK-NEXT:  1      8     1.00    *             U     fcompl	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fcompp
 # CHECK-NEXT:  1      9     0.50                  U     fcomi	%st(3), %st
 # CHECK-NEXT:  1      9     0.50                  U     fcompi	%st(3), %st
@@ -258,10 +258,10 @@ fyl2xp1
 # CHECK-NEXT:  1      22    1.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  1      22    1.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      11    1.00                  U     ffree	%st(0)
-# CHECK-NEXT:  2      12    1.50                  U     ficoms	(%ecx)
-# CHECK-NEXT:  2      12    1.50                  U     ficoml	(%eax)
-# CHECK-NEXT:  2      12    1.50                  U     ficomps	(%ecx)
-# CHECK-NEXT:  2      12    1.50                  U     ficompl	(%eax)
+# CHECK-NEXT:  2      12    1.50    *             U     ficoms	(%ecx)
+# CHECK-NEXT:  2      12    1.50    *             U     ficoml	(%eax)
+# CHECK-NEXT:  2      12    1.50    *             U     ficomps	(%ecx)
+# CHECK-NEXT:  2      12    1.50    *             U     ficompl	(%eax)
 # CHECK-NEXT:  2      11    1.00    *             U     filds	(%edx)
 # CHECK-NEXT:  2      11    1.00    *             U     fildl	(%ecx)
 # CHECK-NEXT:  2      11    1.00    *             U     fildll	(%eax)
@@ -280,7 +280,7 @@ fyl2xp1
 # CHECK-NEXT:  1      8     0.33    *             U     fldl	(%ecx)
 # CHECK-NEXT:  2      1     0.50    *             U     fldt	(%eax)
 # CHECK-NEXT:  1      100   0.25    *             U     fldcw	(%eax)
-# CHECK-NEXT:  1      100   0.25                  U     fldenv	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     fldenv	(%eax)
 # CHECK-NEXT:  1      11    1.00                  U     fld1
 # CHECK-NEXT:  1      11    1.00                  U     fldl2e
 # CHECK-NEXT:  1      11    1.00                  U     fldl2t
@@ -302,8 +302,8 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.25                  U     fprem1
 # CHECK-NEXT:  1      100   0.25                  U     fptan
 # CHECK-NEXT:  1      100   0.25                  U     frndint
-# CHECK-NEXT:  1      100   0.25                  U     frstor	(%eax)
-# CHECK-NEXT:  1      100   0.25                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.25           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     fscale
 # CHECK-NEXT:  1      100   0.25                  U     fsin
 # CHECK-NEXT:  1      100   0.25                  U     fsincos
@@ -316,11 +316,11 @@ fyl2xp1
 # CHECK-NEXT:  1      1     0.33           *      U     fstpl	(%ecx)
 # CHECK-NEXT:  1      5     0.50           *      U     fstpt	(%eax)
 # CHECK-NEXT:  1      100   0.25           *      U     fnstcw	(%eax)
-# CHECK-NEXT:  1      100   0.25                  U     fnstenv	(%eax)
-# CHECK-NEXT:  1      100   0.25                  U     fnstsw	(%eax)
-# CHECK-NEXT:  1      100   0.25                  U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.25           *      U     fnstenv	(%eax)
+# CHECK-NEXT:  1      100   0.25           *      U     fnstsw	(%eax)
+# CHECK-NEXT:  1      100   0.25    *             U     frstor	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     wait
-# CHECK-NEXT:  1      100   0.25                  U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.25           *      U     fnsave	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  1      10    1.00    *             U     fsubs	(%ecx)
diff --git a/llvm/test/tools/llvm-ml/rip-relative-addressing.test b/llvm/test/tools/llvm-ml/rip-relative-addressing.test
new file mode 100644
index 0000000000000..81e5676d041e6
--- /dev/null
+++ b/llvm/test/tools/llvm-ml/rip-relative-addressing.test
@@ -0,0 +1,8 @@
+# RUN: llvm-ml -m32 -filetype=asm %s | FileCheck %s --check-prefix=CHECK-32
+# RUN: llvm-ml -m64 -filetype=asm %s | FileCheck %s --check-prefix=CHECK-64
+
+.code
+mov eax, [4]
+; CHECK-32: mov eax, dword ptr [4]
+; CHECK-64: mov eax, dword ptr [rip + 4]
+END
\ No newline at end of file
diff --git a/llvm/test/tools/llvm-ml/struct.test b/llvm/test/tools/llvm-ml/struct.test
new file mode 100644
index 0000000000000..0e60d24494555
--- /dev/null
+++ b/llvm/test/tools/llvm-ml/struct.test
@@ -0,0 +1,104 @@
+# RUN: llvm-ml -filetype=asm %s | FileCheck %s
+
+.data
+BAZ STRUCT
+  a BYTE 1
+  b BYTE 2
+BAZ ENDS
+
+FOOBAR struct 2
+  c BYTE 3 DUP (4)
+  d DWORD 5
+  e BAZ <>
+  STRUCT f
+    g BYTE 6
+    h BYTE 7
+  ends
+  h BYTE "abcde"
+foobar ENDS
+
+t1 foobar <>
+
+; CHECK: t1:
+;
+; BYTE 3 DUP (4), plus alignment padding
+; CHECK-NEXT: .byte 4
+; CHECK-NEXT: .byte 4
+; CHECK-NEXT: .byte 4
+; CHECK-NEXT: .zero 1
+;
+; DWORD 5
+; CHECK-NEXT: .long 5
+;
+; BAZ <>
+; CHECK-NEXT: .byte 1
+; CHECK-NEXT: .byte 2
+;
+; <BYTE 6, BYTE 7>, with internal alignment padding
+; CHECK-NEXT: .byte 6
+; CHECK-NEXT: .zero 1
+; CHECK-NEXT: .byte 7
+; CHECK-NEXT: .zero 1
+;
+; BYTE "abcde", plus alignment padding
+; CHECK-NEXT: .byte 97
+; CHECK-NEXT: .byte 98
+; CHECK-NEXT: .byte 99
+; CHECK-NEXT: .byte 100
+; CHECK-NEXT: .byte 101
+; CHECK-NEXT: .zero 1
+
+t2 FOOBAR <"gh",,<10,11>,<12>,"ijk">
+
+; CHECK: t2:
+;
+; BYTE "gh", padded with " ", plus alignment padding
+; CHECK-NEXT: .byte 103
+; CHECK-NEXT: .byte 104
+; CHECK-NEXT: .byte 32
+; CHECK-NEXT: .zero 1
+;
+; DWORD 5 (default-initialized when omitted)
+; CHECK-NEXT: .long 5
+;
+; BAZ <10, 11>
+; CHECK-NEXT: .byte 10
+; CHECK-NEXT: .byte 11
+;
+; <BYTE 6, BYTE 7>, with internal alignment padding
+; CHECK-NEXT: .byte 12
+; CHECK-NEXT: .zero 1
+; CHECK-NEXT: .byte 7
+; CHECK-NEXT: .zero 1
+;
+; BYTE "ijk", padded with " ", plus alignment padding
+; CHECK-NEXT: .byte 105
+; CHECK-NEXT: .byte 106
+; CHECK-NEXT: .byte 107
+; CHECK-NEXT: .byte 32
+; CHECK-NEXT: .byte 32
+; CHECK-NEXT: .zero 1
+
+.code
+
+t3:
+mov eax, t2.f.h
+mov eax, [t2].f.h
+mov eax, [t2.f.h]
+mov eax, t2.FOOBAR.f.h
+
+; CHECK: t3:
+; CHECK-NEXT: mov eax, dword ptr [rip + t2+12]
+; CHECK-NEXT: mov eax, dword ptr [rip + t2+12]
+; CHECK-NEXT: mov eax, dword ptr [rip + t2+12]
+; CHECK-NEXT: mov eax, dword ptr [rip + t2+12]
+
+t4:
+mov eax, j.FOOBAR.f.h
+mov eax, j.baz.b
+
+; CHECK: t4:
+; CHECK-NEXT: mov eax, dword ptr [rip + j+12]
+; CHECK-NEXT: mov eax, dword ptr [rip + j+1]
+
+END
diff --git a/llvm/test/tools/llvm-ml/struct_errors.test b/llvm/test/tools/llvm-ml/struct_errors.test
new file mode 100644
index 0000000000000..13d72a2840f0c
--- /dev/null
+++ b/llvm/test/tools/llvm-ml/struct_errors.test
@@ -0,0 +1,57 @@
+# RUN: not llvm-ml -filetype=asm %s 2>&1 | FileCheck %s --dump-input=always
+
+.data
+int_test STRUCT
+  int_arr DWORD ?, ?
+  int_scalar DWORD ?
+int_test ENDS
+
+t1 int_test <<1,2,3>>
+// CHECK: error: Initializer too long for field; expected at most 2 elements, got 3
+
+t2 int_test <4>
+// CHECK: error: Cannot initialize array field with scalar value
+
+t3 int_test <,<5,6>>
+// CHECK: error: Cannot initialize scalar field with array value
+
+real_test STRUCT
+  real_arr REAL4 ?, ?, ?
+  real_scalar REAL4 ?
+real_test ENDS
+
+t4 real_test <<1.0,0.0,-1.0,-2.0>>
+// CHECK: error: Initializer too long for field; expected at most 3 elements, got 4
+
+t5 real_test <2.0>
+// CHECK: error: Cannot initialize array field with scalar value
+
+t6 real_test <,<2.0,-2.0>>
+// CHECK: error: Cannot initialize scalar field with array value
+
+inner_struct STRUCT
+  a BYTE ?
+inner_struct ENDS
+
+struct_test STRUCT
+  struct_arr inner_struct 4 DUP (?)
+  struct_scalar inner_struct ?
+struct_test ENDS
+
+t7 struct_test <<<>, <>, <>, <>, <>>>
+// CHECK: error: Initializer too long for field; expected at most 4 elements, got 5
+
+t8 struct_test <,<<>, <>>>
+// CHECK: error: 'inner_struct' initializer initializes too many fields
+
+t9 STRUCT 3
+// CHECK: error: alignment must be a power of two; was 3
+t9 ENDS
+
+t10 STRUCT 1, X
+// CHECK: error: Unrecognized qualifier for 'STRUCT' directive; expected none or NONUNIQUE
+t10 ENDS
+
+t11 STRUCT
+different_struct ENDS
+// CHECK: error: mismatched name in ENDS directive; expected 't11'
diff --git a/llvm/test/tools/llvm-nm/AArch64/special-syms.test b/llvm/test/tools/llvm-nm/AArch64/special-syms.test
index 8322cc4df86f1..75d1dfc3b6c98 100644
--- a/llvm/test/tools/llvm-nm/AArch64/special-syms.test
+++ b/llvm/test/tools/llvm-nm/AArch64/special-syms.test
@@ -1,10 +1,7 @@
+## Test --special-syms flag.
 # RUN: yaml2obj %s -o %t
-# Test --special-syms flag. Currently this flag is a no-op, so outputs with and without
-# this flag should be identical. GNU nm doesn't show ARM and AArch64 special symbols
-# without --special-syms, so this test is to be changed when/if we decide to implement
-# GNU nm-like behavior in llvm-nm
 
-# RUN: llvm-nm %t | FileCheck %s
+# RUN: llvm-nm %t | count 0
 # RUN: llvm-nm %t --special-syms | FileCheck %s
 
 !ELF
diff --git a/llvm/test/tools/llvm-nm/debug-syms.test b/llvm/test/tools/llvm-nm/debug-syms.test
index e58176588d306..0f970916499fe 100644
--- a/llvm/test/tools/llvm-nm/debug-syms.test
+++ b/llvm/test/tools/llvm-nm/debug-syms.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s -o %t.o
-# RUN: llvm-nm --debug-syms %t.o | FileCheck %s --implicit-check-not {{.}} --check-prefix SYMBOL
-# RUN: llvm-nm -a %t.o | FileCheck %s --implicit-check-not {{.}} --check-prefix SYMBOL
+# RUN: llvm-nm --special-syms --debug-syms %t.o | FileCheck %s --implicit-check-not {{.}} --check-prefix SYMBOL
+# RUN: llvm-nm --special-syms -a %t.o | FileCheck %s --implicit-check-not {{.}} --check-prefix SYMBOL
 
 # SYMBOL:      0000000000000000 n $a
 # SYMBOL-NEXT: 0000000000000000 n $d
diff --git a/llvm/test/tools/llvm-nm/lit.local.cfg b/llvm/test/tools/llvm-nm/lit.local.cfg
index 1fc0bea084cad..c8625f4d9d248 100644
--- a/llvm/test/tools/llvm-nm/lit.local.cfg
+++ b/llvm/test/tools/llvm-nm/lit.local.cfg
@@ -1,4 +1,2 @@
 if not 'X86' in config.root.targets:
     config.unsupported = True
-
-config.suffixes = ['.ll', '.s', '.test', '.yaml']
diff --git a/llvm/test/tools/llvm-objcopy/ELF/add-symbol-new-symtab.test b/llvm/test/tools/llvm-objcopy/ELF/add-symbol-new-symtab.test
new file mode 100644
index 0000000000000..f14625de4f58e
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/ELF/add-symbol-new-symtab.test
@@ -0,0 +1,50 @@
+## Show that llvm-objcopy can replace the symbol table with a new one.
+
+# RUN: yaml2obj %s -o %t
+
+# RUN: llvm-objcopy -R .symtab --add-symbol newsym=1234 %t %t2
+# RUN: llvm-readelf -S -s %t2 | FileCheck %s --match-full-lines
+
+# CHECK: There are 5 section headers, starting at offset {{.+}}:
+# CHECK:      [Nr] Name      Type     Address          Off    Size   ES  Flg Lk Inf Al
+# CHECK-NEXT: [ 0]           NULL     0000000000000000 {{.+}} 000000 00      0  0   0
+# CHECK-NEXT: [ 1] .text     PROGBITS 0000000000001000 {{.+}} 000040 00  AX  0  0   16
+# CHECK-NEXT: [ 2] .strtab   STRTAB   0000000000000000 {{.+}} 000001 00      0  0   1
+# CHECK-NEXT: [ 3] .shstrtab STRTAB   0000000000000000 {{.+}} 000028 00      0  0   1
+# CHECK-NEXT: [ 4] .symtab   SYMTAB   0000000000000000 {{.+}} 000030 18      3  1   8
+
+# CHECK: Symbol table '.symtab' contains 2 entries:
+# CHECK-NEXT:   Num:    Value          Size Type    Bind   Vis       Ndx Name
+# CHECK-NEXT:     0: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT   UND
+# CHECK-NEXT:     1: 00000000000004d2     0 NOTYPE  GLOBAL DEFAULT   ABS newsym
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    Address:         0x1000
+    AddressAlign:    0x0000000000000010
+    Size:            64
+Symbols:
+  - Name:     foo
+    Type:     STT_FUNC
+    Section:  .text
+    Value:    0x1000
+    Size:     8
+  - Name:     bar
+    Type:     STT_FUNC
+    Section:  .text
+    Value:    0x1008
+    Size:     8
+  - Name:     baz
+    Type:     STT_FUNC
+    Section:  .text
+    Value:    0x1010
+    Size:     8
+
diff --git a/llvm/test/tools/llvm-objcopy/ELF/common-symbol.test b/llvm/test/tools/llvm-objcopy/ELF/common-symbol.test
index 98e73d2c0e568..4a4c295f530d2 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/common-symbol.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/common-symbol.test
@@ -11,10 +11,14 @@ FileHeader:
 Symbols:
   - Name:     test
     Index:    SHN_COMMON
+    Value:    0x1233
+    Binding:  STB_GLOBAL
+  - Name:     test1
+    Index:    SHN_HEXAGON_SCOMMON
     Value:    0x1234
     Binding:  STB_GLOBAL
   - Name:     test2
-    Index:    SHN_HEXAGON_SCOMMON
+    Index:    SHN_HEXAGON_SCOMMON_1
     Value:    0x1235
     Binding:  STB_GLOBAL
   - Name:     test3
@@ -42,7 +46,7 @@ Symbols:
 #CHECK-NEXT:  }
 #CHECK-NEXT:  Symbol {
 #CHECK-NEXT:    Name: test
-#CHECK-NEXT:    Value: 0x1234
+#CHECK-NEXT:    Value: 0x1233
 #CHECK-NEXT:    Size: 0
 #CHECK-NEXT:    Binding: Global (0x1)
 #CHECK-NEXT:    Type: None (0x0)
@@ -50,13 +54,22 @@ Symbols:
 #CHECK-NEXT:    Section: Common (0xFFF2)
 #CHECK-NEXT:  }
 #CHECK-NEXT:  Symbol {
+#CHECK-NEXT:    Name: test1
+#CHECK-NEXT:    Value: 0x1234
+#CHECK-NEXT:    Size: 0
+#CHECK-NEXT:    Binding: Global (0x1)
+#CHECK-NEXT:    Type: None (0x0)
+#CHECK-NEXT:    Other: 0
+#CHECK-NEXT:    Section: Processor Specific (0xFF00)
+#CHECK-NEXT:  }
+#CHECK-NEXT:  Symbol {
 #CHECK-NEXT:    Name: test2
 #CHECK-NEXT:    Value: 0x1235
 #CHECK-NEXT:    Size: 0
 #CHECK-NEXT:    Binding: Global (0x1)
 #CHECK-NEXT:    Type: None (0x0)
 #CHECK-NEXT:    Other: 0
-#CHECK-NEXT:    Section: Processor Specific (0xFF00)
+#CHECK-NEXT:    Section: Processor Specific (0xFF01)
 #CHECK-NEXT:  }
 #CHECK-NEXT:  Symbol {
 #CHECK-NEXT:    Name: test3
diff --git a/llvm/test/tools/llvm-objcopy/ELF/group.test b/llvm/test/tools/llvm-objcopy/ELF/group.test
index f82834edbd922..49032e6e5eb40 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/group.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/group.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s -o %t
-# RUN: llvm-objcopy --remove-section=.text.bar %t %t2
-# RUN: llvm-readobj --elf-section-groups %t2 | FileCheck %s
+# RUN: llvm-objcopy --remove-section=.text.bar %t %t1
+# RUN: llvm-readobj --elf-section-groups %t1 | FileCheck %s
 
 # In this test the section .text.bar is getting removed, as a result, 
 # the indices of the sections which go after .text.bar will change, 
@@ -54,3 +54,27 @@ Symbols:
     Section:         .text.foo
     Size:            0x0000000000000000
     Binding:         STB_WEAK
+
+## This checks that tool will emit an error when trying to remove the symbol
+## table when we have a group section linked with symtab.
+# RUN: not llvm-objcopy -R .symtab %t %t2 2>&1 | FileCheck %s --check-prefix=ERR -DINPUT=%t
+
+# ERR: error: '[[INPUT]]': section '.symtab' cannot be removed because it is referenced by the group section '.group'
+
+## The '.symtab' section could be removed using --allow-broken-links option.
+# RUN: llvm-objcopy --allow-broken-links -R .symtab %t %t3
+# RUN: llvm-readelf --sections %t3 | FileCheck %s --check-prefix=SECTIONS
+
+# SECTIONS: There are 6 section headers
+# SECTIONS:      Name      Type   Address Off    Size   ES     Flg Lk Inf
+# SECTIONS-NEXT: NULL
+# SECTIONS-NEXT: .group    {{.+}} {{.+}}  {{.+}} {{.+}} {{.+}}     0  0
+# SECTIONS-NEXT: .text.bar
+# SECTIONS-NEXT: .text.foo
+# SECTIONS-NEXT: .strtab
+# SECTIONS-NEXT: .shstrtab
+
+## Show that llvm-objcopy can handle a group section with a zero sh_link and
+## sh_info fields.
+# RUN: llvm-objcopy --allow-broken-links -R .symtab %t3 %t4
+# RUN: cmp %t3 %t4
diff --git a/llvm/test/tools/llvm-objcopy/ELF/help-message.test b/llvm/test/tools/llvm-objcopy/ELF/help-message.test
deleted file mode 100644
index 0a093fe45c83e..0000000000000
--- a/llvm/test/tools/llvm-objcopy/ELF/help-message.test
+++ /dev/null
@@ -1,23 +0,0 @@
-# RUN: llvm-objcopy -h | FileCheck --check-prefix=OBJCOPY-USAGE %s
-# RUN: llvm-objcopy --help | FileCheck --check-prefix=OBJCOPY-USAGE %s
-# RUN: not llvm-objcopy 2>&1 | FileCheck --check-prefix=OBJCOPY-USAGE %s
-# RUN: not llvm-objcopy -abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
-# RUN: not llvm-objcopy --abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
-# RUN: not llvm-objcopy --strip-debug 2>&1 | FileCheck %s --check-prefix=NO-INPUT-FILES
-
-# RUN: llvm-strip -h | FileCheck --check-prefix=STRIP-USAGE %s
-# RUN: llvm-strip --help | FileCheck --check-prefix=STRIP-USAGE %s
-# RUN: not llvm-strip 2>&1 | FileCheck --check-prefix=STRIP-USAGE %s
-# RUN: not llvm-strip -abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
-# RUN: not llvm-strip --abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
-# RUN: not llvm-strip --strip-debug 2>&1 | FileCheck %s --check-prefix=NO-INPUT-FILES
-
-
-# OBJCOPY-USAGE:  USAGE: llvm-objcopy
-# OBJCOPY-USAGE:  @FILE
-
-# STRIP-USAGE:    USAGE: llvm-strip
-# STRIP-USAGE:    @FILE
-
-# UNKNOWN-ARG:    unknown argument '{{-+}}abcabc'
-# NO-INPUT-FILES: no input file specified
diff --git a/llvm/test/tools/llvm-objcopy/MachO/Inputs/strip-all-with-dwarf.yaml b/llvm/test/tools/llvm-objcopy/MachO/Inputs/strip-all-with-dwarf.yaml
index 88d9b67d7c6bc..c8b5875d3ad09 100644
--- a/llvm/test/tools/llvm-objcopy/MachO/Inputs/strip-all-with-dwarf.yaml
+++ b/llvm/test/tools/llvm-objcopy/MachO/Inputs/strip-all-with-dwarf.yaml
@@ -391,8 +391,7 @@ DWARF:
         - Attribute:       DW_AT_external
           Form:            DW_FORM_flag_present
   debug_info:
-    - Length:
-        TotalLength:     113
+    - Length:          113
       Version:         4
       AbbrOffset:      0
       AddrSize:        8
diff --git a/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-add-rpath.test b/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-add-rpath.test
index 1435c6b744c87..7b21fdc2e03c6 100644
--- a/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-add-rpath.test
+++ b/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-add-rpath.test
@@ -22,6 +22,13 @@
 
 # NO-INPUT: no input file specified
 
+## Add same RPATH twice:
+# RUN: not llvm-install-name-tool -add_rpath @executable_X \
+# RUN:                            -add_rpath @executable_X %t.i386 2>&1 \
+# RUN: | FileCheck --check-prefix=DOUBLE %s
+
+# DOUBLE: duplicate load command
+
 ## Check that cmdsize accounts for NULL terminator.
 # RUN: yaml2obj %p/Inputs/x86_64.yaml -o %t.x86_64
 # RUN: llvm-install-name-tool -add_rpath abcd %t.x86_64
diff --git a/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-change.test b/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-change.test
new file mode 100644
index 0000000000000..8869af2179988
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-change.test
@@ -0,0 +1,107 @@
+## This test checks updating a dependent shared library install name in a MachO binary.
+
+# RUN: yaml2obj %s -o %t
+
+## Specifying -change once:
+# RUN: cp %t %t.copy
+# RUN: llvm-install-name-tool -change /usr/dylib/LOAD /usr/long/long/dylib/LOAD %t.copy
+# RUN: llvm-objdump -p %t.copy | FileCheck %s --check-prefix=CHANGE --implicit-check-not='name /usr'
+
+# CHANGE: name /usr/long/long/dylib/LOAD
+# CHANGE: name /usr/dylib/WEAK
+
+## Specifying -change multiple times:
+# RUN: cp %t %t.copy
+# RUN: llvm-install-name-tool -change /usr/dylib/WEAK /usr/sh/WEAK \
+# RUN:                        -change /usr/dylib/LOAD /usr/sh/LOAD  %t.copy
+# RUN: llvm-objdump -p %t.copy | FileCheck %s --check-prefix=CHANGE-MULTIPLE --implicit-check-not='name /usr'
+
+# CHANGE-MULTIPLE: name /usr/sh/LOAD
+# CHANGE-MULTIPLE: name /usr/sh/WEAK
+
+## Changing same dependent library name multiple times:
+# RUN: cp %t %t.copy
+# RUN: llvm-install-name-tool -change /usr/dylib/LOAD /usr/LOAD \
+# RUN:                        -change /usr/dylib/LOAD /usr/XXXX %t.copy
+# RUN: llvm-objdump -p %t.copy | FileCheck %s --check-prefix=CHANGE-REPEAT --implicit-check-not='name /usr'
+
+# CHANGE-REPEAT: name /usr/LOAD
+# CHANGE-REPEAT: name /usr/dylib/WEAK
+
+## Specifying dependent library names in a chain:
+# RUN: cp %t %t.copy
+# RUN: llvm-install-name-tool -change /usr/dylib/LOAD /usr/XX/LOAD \
+# RUN:                        -change /usr/XX/LOAD /usr/YY/LOAD %t.copy
+# RUN: llvm-objdump -p %t.copy | FileCheck %s --check-prefix=CHANGE-CHAIN --implicit-check-not='name /usr'
+
+# CHANGE-CHAIN: name /usr/XX/LOAD
+# CHANGE-CHAIN: name /usr/dylib/WEAK
+
+## Changing multiple dependent library names where one exists and the other doesn't:
+# RUN: cp %t %t.copy
+# RUN: llvm-install-name-tool -change /usr/dylib/LOAD /usr/JOJO/LOAD \
+# RUN:                        -change /usr/BIZARRE /usr/KOKO/LOAD %t.copy
+# RUN: llvm-objdump -p %t.copy | FileCheck %s --check-prefix=CHANGE-SWITCH --implicit-check-not='name /usr'
+
+# CHANGE-SWITCH: name /usr/JOJO/LOAD
+# CHANGE-SWITCH: name /usr/dylib/WEAK
+
+## Changing to a common dependent library name:
+# RUN: cp %t %t.copy
+# RUN: llvm-install-name-tool -change /usr/dylib/LOAD /usr/COMMON \
+# RUN:                        -change /usr/dylib/WEAK /usr/COMMON %t.copy
+# RUN: llvm-objdump -p %t.copy | FileCheck %s --check-prefix=CHANGE-COMMON --implicit-check-not='name /usr'
+
+# CHANGE-COMMON: name /usr/COMMON
+# CHANGE-COMMON: name /usr/COMMON
+
+## Change all common dependent library names at once:
+# RUN: llvm-install-name-tool -change /usr/COMMON /usr/ONCE %t.copy
+# RUN: llvm-objdump -p %t.copy | FileCheck %s --check-prefix=CHANGE-ONCE --implicit-check-not='name /usr'
+
+# CHANGE-ONCE: name /usr/ONCE
+# CHANGE-ONCE: name /usr/ONCE
+
+## Check that -change option has no effect if the binary doesn't contain old install name:
+# RUN: cp %t %t1
+# RUN: llvm-install-name-tool -change /usr/JOJO/LOAD /usr/XX/LOAD \
+# RUN:                        -change /usr/KOKO/WEAK /usr/YY/WEAK %t
+# RUN: cmp %t %t1
+
+## Missing a -change argument:
+# RUN: not llvm-install-name-tool %t -change /usr/ONCE 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=MISSING
+
+## Missing both -change arguments:
+# RUN: not llvm-install-name-tool %t -change 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=MISSING
+
+# MISSING: missing argument to -change option
+
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x01000007
+  cpusubtype:      0x00000003
+  filetype:        0x00000001
+  ncmds:           2
+  sizeofcmds:      80
+  flags:           0x00002000
+  reserved:        0x00000000
+LoadCommands:
+  - cmd:                        LC_LOAD_DYLIB
+    cmdsize:                    40
+    dylib:
+        name:                   24
+        timestamp:              2
+        current_version:        82115073
+        compatibility_version:  65536
+    PayloadString:              '/usr/dylib/LOAD'
+  - cmd:                        LC_LOAD_WEAK_DYLIB
+    cmdsize:                    40
+    dylib:
+        name:                   24
+        timestamp:              2
+        current_version:        82115073
+        compatibility_version:  65536
+    PayloadString:              '/usr/dylib/WEAK'
diff --git a/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-delete-rpath.test b/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-delete-rpath.test
new file mode 100644
index 0000000000000..b5e6abb777520
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-delete-rpath.test
@@ -0,0 +1,75 @@
+## This test checks deleting a LC_RPATH load command from a MachO binary.
+
+# RUN: yaml2obj %s -o %t
+
+## Deleting a single RPath entry:
+# RUN: llvm-install-name-tool -delete_rpath @executable_a/. %t
+# RUN: llvm-objdump -p %t | \
+# RUN:   FileCheck %s --check-prefix=RPATHS --implicit-check-not=@executable
+
+# RPATHS: @executable_b/.
+# RPATHS: @executable_c/.
+# RPATHS: @executable_d/.
+# RPATHS: @executable_e/.
+
+## Deleting multiple RPath entries:
+# RUN: llvm-install-name-tool -delete_rpath @executable_b/. \
+# RUN:                        -delete_rpath @executable_c/. %t
+# RUN: llvm-objdump -p %t | \
+# RUN:   FileCheck %s --check-prefix=RPATHS-MULTIPLE --implicit-check-not=@executable
+
+# RPATHS-MULTIPLE: @executable_d/.
+# RPATHS-MULTIPLE: @executable_e/.
+
+## Duplicate delete_rpath commands:
+# RUN: llvm-install-name-tool -delete_rpath @executable_d/. \
+# RUN:                        -delete_rpath @executable_d/. %t
+# RUN: llvm-objdump -p %t | \
+# RUN:   FileCheck %s --check-prefix=DUPLICATE --implicit-check-not=@executable
+
+# DUPLICATE: @executable_e/.
+
+## Deleting a nonexistent RPath:
+# RUN: not llvm-install-name-tool -delete_rpath @executable_a/. %t 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ABSENT-RPATH
+
+# ABSENT-RPATH: no LC_RPATH load command with path: @executable_a/.
+
+## Adding and deleting RPATH at the same time:
+# RUN: not llvm-install-name-tool -add_rpath @executable_b/. \
+# RUN:                            -delete_rpath @executable_b/. %t 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=COMBINED
+
+# COMBINED: cannot specify both -add_rpath @executable_b/. and -delete_rpath @executable_b/.
+
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x01000007
+  cpusubtype:      0x00000003
+  filetype:        0x00000001
+  ncmds:           5
+  sizeofcmds:      160
+  flags:           0x00002000
+  reserved:        0x00000000
+LoadCommands:
+  - cmd:             LC_RPATH
+    cmdsize:         32
+    path:            12
+    PayloadString:   '@executable_a/.'
+  - cmd:             LC_RPATH
+    cmdsize:         32
+    path:            12
+    PayloadString:   '@executable_b/.'
+  - cmd:             LC_RPATH
+    cmdsize:         32
+    path:            12
+    PayloadString:   '@executable_c/.'
+  - cmd:             LC_RPATH
+    cmdsize:         32
+    path:            12
+    PayloadString:   '@executable_d/.'
+  - cmd:             LC_RPATH
+    cmdsize:         32
+    path:            12
+    PayloadString:   '@executable_e/.'
diff --git a/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-help-message.test b/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-help-message.test
deleted file mode 100644
index 43e5290c37932..0000000000000
--- a/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-help-message.test
+++ /dev/null
@@ -1,10 +0,0 @@
-# RUN: llvm-install-name-tool -h | FileCheck --check-prefix=INSTALL-NAME-TOOL-USAGE %s
-# RUN: llvm-install-name-tool --help | FileCheck --check-prefix=INSTALL-NAME-TOOL-USAGE %s
-# RUN: not llvm-install-name-tool 2>&1 | FileCheck --check-prefix=INSTALL-NAME-TOOL-USAGE %s
-# RUN: not llvm-install-name-tool -abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
-# RUN: not llvm-install-name-tool --abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
-
-# INSTALL-NAME-TOOL-USAGE: USAGE: llvm-install-name-tool
-# INSTALL-NAME-TOOL-USAGE: @FILE
-
-# UNKNOWN-ARG: unknown argument '{{-+}}abcabc'
diff --git a/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-id.test b/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-id.test
new file mode 100644
index 0000000000000..eb29d26be9345
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-id.test
@@ -0,0 +1,67 @@
+## This test checks updating a dynamic shared library ID in a MachO binary.
+
+# RUN: yaml2obj %s --docnum=1 -o %t
+
+## Specifying -id once:
+# RUN: llvm-install-name-tool -id /usr/lib/A_long_long_test %t
+# RUN: llvm-objdump -p %t | FileCheck %s --check-prefix=ID --implicit-check-not='name /usr'
+
+# ID: name /usr/lib/A_long_long_test
+
+## Specifying -id more than once:
+# RUN: llvm-install-name-tool -id /usr/lib/B_long -id /usr/lib/K_long -id /usr/A_short  %t
+# RUN: llvm-objdump -p %t | FileCheck %s --check-prefix=ID-MULTIPLE --implicit-check-not='name /usr'
+
+# ID-MULTIPLE: name /usr/A_short
+
+## Specifying -id with empty string:
+# RUN: not llvm-install-name-tool -id '' %t 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=EMPTY
+
+# EMPTY: cannot specify an empty id
+
+## Missing id argument:
+# RUN: not llvm-install-name-tool %t -id 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=MISSING
+
+# MISSING: missing argument to -id option
+
+## Shared dylib binary
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x01000007
+  cpusubtype:      0x00000003
+  filetype:        0x00000006
+  ncmds:           1
+  sizeofcmds:      56
+  flags:           0x00002000
+  reserved:        0x00000000
+LoadCommands:
+  - cmd:                        LC_ID_DYLIB
+    cmdsize:                    56
+    dylib:
+        name:                   24
+        timestamp:              2
+        current_version:        82115073
+        compatibility_version:  65536
+    PayloadString:              '/usr/lib/A'
+
+# RUN: yaml2obj %s --docnum=2 -o %t
+
+## Check that -id option has no effect if binary is not a shared dylib:
+# RUN: cp %t %t1
+# RUN: llvm-install-name-tool -id /usr/lib/J %t
+# RUN: cmp %t %t1
+
+## Executable binary
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x01000007
+  cpusubtype:      0x00000003
+  filetype:        0x00000001
+  ncmds:           0
+  sizeofcmds:      0
+  flags:           0x00002000
+  reserved:        0x00000000
diff --git a/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-rpath.test b/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-rpath.test
new file mode 100644
index 0000000000000..ae92401b3a5d2
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/MachO/install-name-tool-rpath.test
@@ -0,0 +1,142 @@
+## This test checks updating a LC_RPATH load command in a MachO binary.
+
+# RUN: yaml2obj %s -o %t
+
+## Updating a single RPath entry:
+# RUN: llvm-install-name-tool -rpath @executable_a/. @executable_A/. %t
+# RUN: llvm-objdump -p %t | \
+# RUN:   FileCheck %s --check-prefix=RPATHS --implicit-check-not=@executable
+
+# RPATHS: @executable_A/.
+# RPATHS: @executable_short_test
+# RPATHS: @executable_long_test/.
+# RPATHS: @executable_d/.
+
+## Updating multiple RPath entries:
+# RUN: llvm-install-name-tool -rpath @executable_short_test/. @executable_test/. \
+# RUN:                        -rpath @executable_long_test/. @executable_long_long_test/. %t
+# RUN: llvm-objdump -p %t | \
+# RUN:   FileCheck %s --check-prefix=RPATHS-MULTIPLE --implicit-check-not=@executable
+
+# RPATHS-MULTIPLE: @executable_A/.
+# RPATHS-MULTIPLE: @executable_test/.
+# RPATHS-MULTIPLE: @executable_long_long_test/.
+# RPATHS-MULTIPLE: @executable_d/.
+
+## Check that cmdsize accounts for NULL terminator:
+# RUN: llvm-install-name-tool -rpath @executable_A/. ABCD %t
+# RUN: llvm-objdump -p %t | FileCheck %s --check-prefix=RPATH-SIZE
+
+# RPATH-SIZE:      cmd LC_RPATH
+# RPATH-SIZE-NEXT: cmdsize 24
+# RPATH-SIZE-NEXT: path ABCD
+
+## Updating and adding different RPaths:
+# RUN: llvm-install-name-tool -rpath @executable_d/. @executable_D/. \
+# RUN:                        -add_rpath @executable_e/. %t
+# RUN: llvm-objdump -p %t | FileCheck %s --check-prefix=RPATHS-ADD --implicit-check-not=@executable
+
+# RPATHS-ADD: ABCD
+# RPATHS-ADD: @executable_test/.
+# RPATHS-ADD: @executable_long_long_test/.
+# RPATHS-ADD: @executable_D/.
+# RPATHS-ADD: @executable_e/.
+
+## Updating and deleting different RPaths:
+# RUN: llvm-install-name-tool -rpath @executable_D/. @executable_d/. \
+# RUN:                        -delete_rpath @executable_e/. %t
+# RUN: llvm-objdump -p %t | FileCheck %s --check-prefix=RPATHS-DELETE --implicit-check-not=@executable
+
+# RPATHS-DELETE: ABCD
+# RPATHS-DELETE: @executable_test/.
+# RPATHS-DELETE: @executable_long_long_test/.
+# RPATHS-DELETE: @executable_d/.
+
+# RUN: cp %t %t1
+
+## Updating multiple RPath entries where one exists and the other doesn't:
+# RUN: not llvm-install-name-tool -rpath @executable_test/. @executable/. \
+# RUN:                            -rpath @executable_long_test/. @executable_long_longest/. %t 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=RPATHS-FAIL
+# RUN: cmp %t %t1
+
+# RPATHS-FAIL: no LC_RPATH load command with path: @executable_long_test/.
+
+## Updating a nonexistent RPath:
+# RUN: not llvm-install-name-tool -rpath @executable_a/. @executable_AA/. %t 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ABSENT-RPATH
+# RUN: cmp %t %t1
+
+# ABSENT-RPATH: no LC_RPATH load command with path: @executable_a/.
+
+## Updating to an existing RPath:
+# RUN: not llvm-install-name-tool -rpath @executable_d/. ABCD %t 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=EXISTING
+# RUN: cmp %t %t1
+
+# EXISTING: rpath ABCD would create a duplicate load command
+
+## Duplicate RPath entries:
+# RUN: not llvm-install-name-tool -rpath DDD1/. @exec_d/. \
+# RUN:                            -rpath @exec_d/. DDD2/. %t 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=DUPLICATE
+# RUN: cmp %t %t1
+
+# DUPLICATE: cannot specify both -rpath DDD1/. @exec_d/. and -rpath @exec_d/. DDD2/.
+
+## Updating and deleting RPath at the same time:
+# RUN: not llvm-install-name-tool -rpath @executable_d/. DD/. \
+# RUN:                            -delete_rpath @executable_d/. %t 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=COMBINED-DELETE
+# RUN: cmp %t %t1
+
+# COMBINED-DELETE: cannot specify both -delete_rpath @executable_d/. and -rpath @executable_d/. DD/.
+
+## Updating and adding RPath at the same time:
+# RUN: not llvm-install-name-tool -rpath @executable_e/. EE/. \
+# RUN:                            -add_rpath @executable_e/. %t 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=COMBINED-ADD
+# RUN: cmp %t %t1
+
+# COMBINED-ADD: cannot specify both -add_rpath @executable_e/. and -rpath @executable_e/. EE/.
+
+## Missing an RPath argument:
+# RUN: not llvm-install-name-tool %t -rpath @executable_e/. 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=MISSING
+# RUN: cmp %t %t1
+
+## Missing both RPath arguments:
+# RUN: not llvm-install-name-tool %t -rpath 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=MISSING
+# RUN: cmp %t %t1
+
+# MISSING: missing argument to -rpath option
+
+
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x01000007
+  cpusubtype:      0x00000003
+  filetype:        0x00000001
+  ncmds:           4
+  sizeofcmds:      144
+  flags:           0x00002000
+  reserved:        0x00000000
+LoadCommands:
+  - cmd:             LC_RPATH
+    cmdsize:         32
+    path:            12
+    PayloadString:   '@executable_a/.'
+  - cmd:             LC_RPATH
+    cmdsize:         40
+    path:            12
+    PayloadString:   '@executable_short_test/.'
+  - cmd:             LC_RPATH
+    cmdsize:         40
+    path:            12
+    PayloadString:   '@executable_long_test/.'
+  - cmd:             LC_RPATH
+    cmdsize:         32
+    path:            12
+    PayloadString:   '@executable_d/.'
diff --git a/llvm/test/tools/llvm-objcopy/MachO/remove-lc-index-update.test b/llvm/test/tools/llvm-objcopy/MachO/remove-lc-index-update.test
new file mode 100644
index 0000000000000..6543ae29fbc4a
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/MachO/remove-lc-index-update.test
@@ -0,0 +1,343 @@
+## Check that removing load commands updates the indexes of special LCs:
+
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-install-name-tool -delete_rpath @executable_a/. %t
+# RUN: llvm-objdump -p %t | FileCheck %s --check-prefix=INDEX
+
+# INDEX:      Load command 3
+# INDEX-NEXT: cmd LC_DYLD_INFO_ONLY
+# INDEX:      Load command 5
+# INDEX-NEXT: cmd LC_DATA_IN_CODE
+# INDEX:      Load command 8
+# INDEX-NEXT: cmd LC_FUNCTION_STARTS
+# INDEX:      Load command 13
+# INDEX-NEXT: cmd LC_DYSYMTAB
+# INDEX:      Load command 14
+# INDEX-NEXT: cmd LC_SYMTAB
+
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x01000007
+  cpusubtype:      0x80000003
+  filetype:        0x00000002
+  ncmds:           16
+  sizeofcmds:      1008
+  flags:           0x00200085
+  reserved:        0x00000000
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __PAGEZERO
+    vmaddr:          0
+    vmsize:          4294967296
+    fileoff:         0
+    filesize:        0
+    maxprot:         0
+    initprot:        0
+    nsects:          0
+    flags:           0
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         232
+    segname:         __TEXT
+    vmaddr:          4294967296
+    vmsize:          4096
+    fileoff:         0
+    filesize:        4096
+    maxprot:         7
+    initprot:        5
+    nsects:          2
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x0000000100000F90
+        size:            36
+        offset:          0x00000F90
+        align:           4
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+      - sectname:        __unwind_info
+        segname:         __TEXT
+        addr:            0x0000000100000FB4
+        size:            72
+        offset:          0x00000FB4
+        align:           2
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x00000000
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         232
+    segname:         __DATA
+    vmaddr:          4294971392
+    vmsize:          4096
+    fileoff:         4096
+    filesize:        4096
+    maxprot:         7
+    initprot:        3
+    nsects:          2
+    flags:           0
+    Sections:
+      - sectname:        __data
+        segname:         __DATA
+        addr:            0x0000000100001000
+        size:            4
+        offset:          0x00001000
+        align:           2
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x00000000
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+      - sectname:        __common
+        segname:         __DATA
+        addr:            0x0000000100001004
+        size:            4
+        offset:          0x00000000
+        align:           2
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x00000001
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+  - cmd:             LC_RPATH
+    cmdsize:         32
+    path:            12
+    PayloadString:   '@executable_a/.'
+  - cmd:             LC_DYLD_INFO_ONLY
+    cmdsize:         48
+    rebase_off:      0
+    rebase_size:     0
+    bind_off:        0
+    bind_size:       0
+    weak_bind_off:   0
+    weak_bind_size:  0
+    lazy_bind_off:   0
+    lazy_bind_size:  0
+    export_off:      8192
+    export_size:     56
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __LINKEDIT
+    vmaddr:          4294975488
+    vmsize:          4096
+    fileoff:         8192
+    filesize:        496
+    maxprot:         7
+    initprot:        1
+    nsects:          0
+    flags:           0
+  - cmd:             LC_DATA_IN_CODE
+    cmdsize:         16
+    dataoff:         8256
+    datasize:        8
+  - cmd:             LC_LOAD_DYLINKER
+    cmdsize:         32
+    name:            12
+    PayloadString:   '/usr/lib/dyld'
+    ZeroPadBytes:    7
+  - cmd:             LC_UUID
+    cmdsize:         24
+    uuid:            56D84164-1DA8-393B-8589-BC4C6B56168E
+  - cmd:             LC_FUNCTION_STARTS
+    cmdsize:         16
+    dataoff:         8248
+    datasize:        8
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         32
+    platform:        1
+    minos:           658944
+    sdk:             658944
+    ntools:          1
+    Tools:
+      - tool:            3
+        version:         29491968
+  - cmd:             LC_SOURCE_VERSION
+    cmdsize:         16
+    version:         0
+  - cmd:             LC_MAIN
+    cmdsize:         24
+    entryoff:        3984
+    stacksize:       0
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         56
+    dylib:
+      name:            24
+      timestamp:       2
+      current_version: 82115073
+      compatibility_version: 65536
+    PayloadString:   '/usr/lib/libSystem.B.dylib'
+    ZeroPadBytes:    6
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       11
+    iextdefsym:      11
+    nextdefsym:      3
+    iundefsym:       14
+    nundefsym:       1
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  8264
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          8264
+    nsyms:           15
+    stroff:          8504
+    strsize:         192
+LinkEditData:
+  ExportTrie:
+    TerminalSize:    0
+    NodeOffset:      0
+    Name:            ''
+    Flags:           0x0000000000000000
+    Address:         0x0000000000000000
+    Other:           0x0000000000000000
+    ImportName:      ''
+    Children:
+      - TerminalSize:    0
+        NodeOffset:      5
+        Name:            _
+        Flags:           0x0000000000000000
+        Address:         0x0000000000000000
+        Other:           0x0000000000000000
+        ImportName:      ''
+        Children:
+          - TerminalSize:    2
+            NodeOffset:      38
+            Name:            _mh_execute_header
+            Flags:           0x0000000000000000
+            Address:         0x0000000000000000
+            Other:           0x0000000000000000
+            ImportName:      ''
+          - TerminalSize:    3
+            NodeOffset:      42
+            Name:            main
+            Flags:           0x0000000000000000
+            Address:         0x0000000000000F90
+            Other:           0x0000000000000000
+            ImportName:      ''
+          - TerminalSize:    3
+            NodeOffset:      47
+            Name:            foo
+            Flags:           0x0000000000000000
+            Address:         0x0000000000001004
+            Other:           0x0000000000000000
+            ImportName:      ''
+  NameList:
+    - n_strx:          50
+      n_type:          0x64
+      n_sect:          0
+      n_desc:          0
+      n_value:         0
+    - n_strx:          92
+      n_type:          0x64
+      n_sect:          0
+      n_desc:          0
+      n_value:         0
+    - n_strx:          100
+      n_type:          0x66
+      n_sect:          3
+      n_desc:          1
+      n_value:         1564714240
+    - n_strx:          1
+      n_type:          0x2E
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294971280
+    - n_strx:          164
+      n_type:          0x24
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294971280
+    - n_strx:          1
+      n_type:          0x24
+      n_sect:          0
+      n_desc:          0
+      n_value:         36
+    - n_strx:          1
+      n_type:          0x4E
+      n_sect:          1
+      n_desc:          0
+      n_value:         36
+    - n_strx:          170
+      n_type:          0x26
+      n_sect:          3
+      n_desc:          0
+      n_value:         4294971392
+    ## A local undefined SymDebugTable entry followed by
+    - n_strx:          175
+      n_type:          0x20
+      n_sect:          0
+      n_desc:          0
+      n_value:         0
+    ## a defined SymDebugTable entry.
+    - n_strx:          1
+      n_type:          0x64
+      n_sect:          1
+      n_desc:          0
+      n_value:         0
+    - n_strx:          180
+      n_type:          0x0E
+      n_sect:          3
+      n_desc:          0
+      n_value:         4294971392
+    - n_strx:          2
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          16
+      n_value:         4294967296
+    - n_strx:          22
+      n_type:          0x0F
+      n_sect:          4
+      n_desc:          0
+      n_value:         4294971396
+    - n_strx:          27
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294971280
+    - n_strx:          33
+      n_type:          0x01
+      n_sect:          0
+      n_desc:          256
+      n_value:         0
+  StringTable:
+    - ' '
+    - __mh_execute_header
+    - _foo
+    - _main
+    - dyld_stub_binder
+    - '/Users/anonymous/llvm-project/llvm/build/'
+    - strip.c
+    - '/var/folders/g7/l0p5czpd2jz6w0gswg6cwgjc0000gn/T/strip-95c4bf.o'
+    - _main
+    - _bar
+    - _foo
+    - _bar
+    - ''
+    - ''
+    - ''
+    - ''
+    - ''
+    - ''
+    - ''
+...
diff --git a/llvm/test/tools/llvm-objcopy/MachO/segment-size-alignment.test b/llvm/test/tools/llvm-objcopy/MachO/segment-size-alignment.test
new file mode 100644
index 0000000000000..d8ad3e9a46227
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/MachO/segment-size-alignment.test
@@ -0,0 +1,897 @@
+# RUN: yaml2obj --docnum=1 %s -o %t.armv7
+# RUN: llvm-objdump --private-headers %t.armv7 | FileCheck %s --check-prefix=ARMV7
+
+# ARMV7:       segname __PAGEZERO
+# ARMV7-NEXT:  vmaddr 0x00000000
+# ARMV7-NEXT:  vmsize 0x00004000
+# ARMV7:       segname __TEXT
+# ARMV7-NEXT:  vmaddr 0x00004000
+# ARMV7-NEXT:  vmsize 0x00008000
+# ARMV7:       segname __LINKEDIT
+# ARMV7-NEXT:  vmaddr 0x0000c000
+# ARMV7-NEXT:  vmsize 0x00004000
+
+# RUN: yaml2obj --docnum=2 %s -o %t.arm64
+# RUN: llvm-objdump --private-headers %t.arm64 | FileCheck %s --check-prefix=ARM64
+
+# ARM64:       segname __PAGEZERO
+# ARM64-NEXT:  vmaddr 0x0000000000000000
+# ARM64-NEXT:  vmsize 0x0000000100000000
+# ARM64:       segname __TEXT
+# ARM64-NEXT:  vmaddr 0x0000000100000000
+# ARM64-NEXT:  vmsize 0x0000000000008000
+# ARM64:       segname __LINKEDIT
+# ARM64-NEXT:  vmaddr 0x0000000100008000
+# ARM64-NEXT:  vmsize 0x0000000000004000
+
+# RUN: yaml2obj --docnum=3 %s -o %t.arm64_32
+# RUN: llvm-objdump --private-headers %t.arm64_32 | FileCheck %s --check-prefix=ARM64_32
+
+# ARM64_32:       segname __PAGEZERO
+# ARM64_32-NEXT:  vmaddr 0x00000000
+# ARM64_32-NEXT:  vmsize 0x00004000
+# ARM64_32:       segname __TEXT
+# ARM64_32-NEXT:  vmaddr 0x00004000
+# ARM64_32-NEXT:  vmsize 0x00008000
+# ARM64_32:       segname __LINKEDIT
+# ARM64_32-NEXT:  vmaddr 0x0000c000
+# ARM64_32-NEXT:  vmsize 0x00004000
+
+# RUN: yaml2obj --docnum=4 %s -o %t.x86_64
+# RUN: llvm-objdump --private-headers %t.x86_64 | FileCheck %s --check-prefix=X86_64
+
+# X86_64:       segname __PAGEZERO
+# X86_64-NEXT:  vmaddr 0x0000000000000000
+# X86_64-NEXT:  vmsize 0x0000000100000000
+# X86_64:       segname __TEXT
+# X86_64-NEXT:  vmaddr 0x0000000100000000
+# X86_64-NEXT:  vmsize 0x0000000000001000
+# X86_64:       segname __LINKEDIT
+# X86_64-NEXT:  vmaddr 0x0000000100001000
+# X86_64-NEXT:  vmsize 0x0000000000001000
+
+## Input binaries:
+
+## Case 1: armv7 iOS binary.
+## This binary can be built on OSX as follows:
+## main.c: int main() { return 0; }
+## xcrun -sdk iphoneos clang -arch armv7 main.c -o main.exe
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACE
+  cputype:         0x0000000C
+  cpusubtype:      0x00000009
+  filetype:        0x00000002
+  ncmds:           15
+  sizeofcmds:      600
+  flags:           0x00200085
+LoadCommands:
+  - cmd:             LC_SEGMENT
+    cmdsize:         56
+    segname:         __PAGEZERO
+    vmaddr:          0
+    vmsize:          16384
+    fileoff:         0
+    filesize:        0
+    maxprot:         0
+    initprot:        0
+    nsects:          0
+    flags:           0
+  - cmd:             LC_SEGMENT
+    cmdsize:         124
+    segname:         __TEXT
+    vmaddr:          16384
+    vmsize:          32768
+    fileoff:         0
+    filesize:        32768
+    maxprot:         5
+    initprot:        5
+    nsects:          1
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x000000000000BFF6
+        size:            10
+        offset:          0x00007FF6
+        align:           1
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+        content:         81B00020009001B07047
+  - cmd:             LC_SEGMENT
+    cmdsize:         56
+    segname:         __LINKEDIT
+    vmaddr:          49152
+    vmsize:          16384
+    fileoff:         32768
+    filesize:        132
+    maxprot:         1
+    initprot:        1
+    nsects:          0
+    flags:           0
+  - cmd:             LC_DYLD_INFO_ONLY
+    cmdsize:         48
+    rebase_off:      0
+    rebase_size:     0
+    bind_off:        0
+    bind_size:       0
+    weak_bind_off:   0
+    weak_bind_size:  0
+    lazy_bind_off:   0
+    lazy_bind_size:  0
+    export_off:      32768
+    export_size:     44
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          32816
+    nsyms:           3
+    stroff:          32852
+    strsize:         48
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       0
+    iextdefsym:      0
+    nextdefsym:      2
+    iundefsym:       2
+    nundefsym:       1
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+  - cmd:             LC_LOAD_DYLINKER
+    cmdsize:         28
+    name:            12
+    PayloadString:   '/usr/lib/dyld'
+    ZeroPadBytes:    3
+  - cmd:             LC_UUID
+    cmdsize:         24
+    uuid:            C84A0193-BE6F-3625-882F-CBFC052AF6CF
+  - cmd:             LC_VERSION_MIN_IPHONEOS
+    cmdsize:         16
+    version:         680803
+    sdk:             852480
+  - cmd:             LC_SOURCE_VERSION
+    cmdsize:         16
+    version:         0
+  - cmd:             LC_MAIN
+    cmdsize:         24
+    entryoff:        32759
+    stacksize:       0
+  - cmd:             LC_ENCRYPTION_INFO
+    cmdsize:         20
+    cryptoff:        16384
+    cryptsize:       16384
+    cryptid:         0
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         52
+    dylib:
+      name:            24
+      timestamp:       2
+      current_version: 83951616
+      compatibility_version: 65536
+    PayloadString:   '/usr/lib/libSystem.B.dylib'
+    ZeroPadBytes:    2
+  - cmd:             LC_FUNCTION_STARTS
+    cmdsize:         16
+    dataoff:         32812
+    datasize:        4
+  - cmd:             LC_DATA_IN_CODE
+    cmdsize:         16
+    dataoff:         32816
+    datasize:        0
+LinkEditData:
+  ExportTrie:
+    TerminalSize:    0
+    NodeOffset:      0
+    Name:            ''
+    Flags:           0x0000000000000000
+    Address:         0x0000000000000000
+    Other:           0x0000000000000000
+    ImportName:      ''
+    Children:
+      - TerminalSize:    0
+        NodeOffset:      5
+        Name:            _
+        Flags:           0x0000000000000000
+        Address:         0x0000000000000000
+        Other:           0x0000000000000000
+        ImportName:      ''
+        Children:
+          - TerminalSize:    2
+            NodeOffset:      33
+            Name:            _mh_execute_header
+            Flags:           0x0000000000000000
+            Address:         0x0000000000000000
+            Other:           0x0000000000000000
+            ImportName:      ''
+          - TerminalSize:    4
+            NodeOffset:      37
+            Name:            main
+            Flags:           0x0000000000000000
+            Address:         0x0000000000007FF7
+            Other:           0x0000000000000000
+            ImportName:      ''
+  NameList:
+    - n_strx:          2
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          16
+      n_value:         16384
+    - n_strx:          22
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          8
+      n_value:         49142
+    - n_strx:          28
+      n_type:          0x01
+      n_sect:          0
+      n_desc:          256
+      n_value:         0
+  StringTable:
+    - ' '
+    - __mh_execute_header
+    - _main
+    - dyld_stub_binder
+    - ''
+    - ''
+    - ''
+...
+
+## Case 2: arm64 iOS binary.
+## This binary can be built on OSX as follows:
+## main.c: int main() { return 0; }
+## xcrun -sdk iphoneos clang -arch arm64 main.c -o main.exe
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x0100000C
+  cpusubtype:      0x00000000
+  filetype:        0x00000002
+  ncmds:           15
+  sizeofcmds:      768
+  flags:           0x00200085
+  reserved:        0x00000000
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __PAGEZERO
+    vmaddr:          0
+    vmsize:          4294967296
+    fileoff:         0
+    filesize:        0
+    maxprot:         0
+    initprot:        0
+    nsects:          0
+    flags:           0
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         232
+    segname:         __TEXT
+    vmaddr:          4294967296
+    vmsize:          32768
+    fileoff:         0
+    filesize:        32768
+    maxprot:         5
+    initprot:        5
+    nsects:          2
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x0000000100007FA0
+        size:            24
+        offset:          0x00007FA0
+        align:           2
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+        content:         FF4300D1FF0F00B908008052E00308AAFF430091C0035FD6
+      - sectname:        __unwind_info
+        segname:         __TEXT
+        addr:            0x0000000100007FB8
+        size:            72
+        offset:          0x00007FB8
+        align:           2
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x00000000
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+        content:         010000001C000000000000001C000000000000001C00000002000000A07F00003400000034000000B97F00000000000034000000030000000C000100100001000000000000100002
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __LINKEDIT
+    vmaddr:          4295000064
+    vmsize:          16384
+    fileoff:         32768
+    filesize:        152
+    maxprot:         1
+    initprot:        1
+    nsects:          0
+    flags:           0
+  - cmd:             LC_DYLD_INFO_ONLY
+    cmdsize:         48
+    rebase_off:      0
+    rebase_size:     0
+    bind_off:        0
+    bind_size:       0
+    weak_bind_off:   0
+    weak_bind_size:  0
+    lazy_bind_off:   0
+    lazy_bind_size:  0
+    export_off:      32768
+    export_size:     48
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          32824
+    nsyms:           3
+    stroff:          32872
+    strsize:         48
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       0
+    iextdefsym:      0
+    nextdefsym:      2
+    iundefsym:       2
+    nundefsym:       1
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+  - cmd:             LC_LOAD_DYLINKER
+    cmdsize:         32
+    name:            12
+    PayloadString:   '/usr/lib/dyld'
+    ZeroPadBytes:    7
+  - cmd:             LC_UUID
+    cmdsize:         24
+    uuid:            945B24FC-E351-3186-9D74-A88A7A93392A
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         32
+    platform:        2
+    minos:           852480
+    sdk:             852480
+    ntools:          1
+    Tools:
+      - tool:            3
+        version:         34734080
+  - cmd:             LC_SOURCE_VERSION
+    cmdsize:         16
+    version:         0
+  - cmd:             LC_MAIN
+    cmdsize:         24
+    entryoff:        32672
+    stacksize:       0
+  - cmd:             LC_ENCRYPTION_INFO_64
+    cmdsize:         24
+    cryptoff:        16384
+    cryptsize:       16384
+    cryptid:         0
+    pad:             0
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         56
+    dylib:
+      name:            24
+      timestamp:       2
+      current_version: 83951616
+      compatibility_version: 65536
+    PayloadString:   '/usr/lib/libSystem.B.dylib'
+    ZeroPadBytes:    6
+  - cmd:             LC_FUNCTION_STARTS
+    cmdsize:         16
+    dataoff:         32816
+    datasize:        8
+  - cmd:             LC_DATA_IN_CODE
+    cmdsize:         16
+    dataoff:         32824
+    datasize:        0
+LinkEditData:
+  ExportTrie:
+    TerminalSize:    0
+    NodeOffset:      0
+    Name:            ''
+    Flags:           0x0000000000000000
+    Address:         0x0000000000000000
+    Other:           0x0000000000000000
+    ImportName:      ''
+    Children:
+      - TerminalSize:    0
+        NodeOffset:      5
+        Name:            _
+        Flags:           0x0000000000000000
+        Address:         0x0000000000000000
+        Other:           0x0000000000000000
+        ImportName:      ''
+        Children:
+          - TerminalSize:    2
+            NodeOffset:      33
+            Name:            _mh_execute_header
+            Flags:           0x0000000000000000
+            Address:         0x0000000000000000
+            Other:           0x0000000000000000
+            ImportName:      ''
+          - TerminalSize:    4
+            NodeOffset:      37
+            Name:            main
+            Flags:           0x0000000000000000
+            Address:         0x0000000000007FA0
+            Other:           0x0000000000000000
+            ImportName:      ''
+  NameList:
+    - n_strx:          2
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          16
+      n_value:         4294967296
+    - n_strx:          22
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294999968
+    - n_strx:          28
+      n_type:          0x01
+      n_sect:          0
+      n_desc:          256
+      n_value:         0
+  StringTable:
+    - ' '
+    - __mh_execute_header
+    - _main
+    - dyld_stub_binder
+    - ''
+    - ''
+    - ''
+...
+
+## Case 3: arm64_32 WatchOS binary.
+## This binary can be built on OSX as follows:
+## main.c: int main() { return 0; }
+## xcrun -sdk watchos clang -arch arm64_32 main.c -o main.exe
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACE
+  cputype:         0x0200000C
+  cpusubtype:      0x00000001
+  filetype:        0x00000002
+  ncmds:           15
+  sizeofcmds:      684
+  flags:           0x00200085
+LoadCommands:
+  - cmd:             LC_SEGMENT
+    cmdsize:         56
+    segname:         __PAGEZERO
+    vmaddr:          0
+    vmsize:          16384
+    fileoff:         0
+    filesize:        0
+    maxprot:         0
+    initprot:        0
+    nsects:          0
+    flags:           0
+  - cmd:             LC_SEGMENT
+    cmdsize:         192
+    segname:         __TEXT
+    vmaddr:          16384
+    vmsize:          32768
+    fileoff:         0
+    filesize:        32768
+    maxprot:         5
+    initprot:        5
+    nsects:          2
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x000000000000BFA0
+        size:            24
+        offset:          0x00007FA0
+        align:           2
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+        content:         FF4300D108008052E80F00B9E00308AAFF430091C0035FD6
+      - sectname:        __unwind_info
+        segname:         __TEXT
+        addr:            0x000000000000BFB8
+        size:            72
+        offset:          0x00007FB8
+        align:           2
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x00000000
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+        content:         010000001C000000000000001C000000000000001C00000002000000A07F00003400000034000000B97F00000000000034000000030000000C000100100001000000000000100002
+  - cmd:             LC_SEGMENT
+    cmdsize:         56
+    segname:         __LINKEDIT
+    vmaddr:          49152
+    vmsize:          16384
+    fileoff:         32768
+    filesize:        132
+    maxprot:         1
+    initprot:        1
+    nsects:          0
+    flags:           0
+  - cmd:             LC_DYLD_INFO_ONLY
+    cmdsize:         48
+    rebase_off:      0
+    rebase_size:     0
+    bind_off:        0
+    bind_size:       0
+    weak_bind_off:   0
+    weak_bind_size:  0
+    lazy_bind_off:   0
+    lazy_bind_size:  0
+    export_off:      32768
+    export_size:     44
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          32816
+    nsyms:           3
+    stroff:          32852
+    strsize:         48
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       0
+    iextdefsym:      0
+    nextdefsym:      2
+    iundefsym:       2
+    nundefsym:       1
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+  - cmd:             LC_LOAD_DYLINKER
+    cmdsize:         28
+    name:            12
+    PayloadString:   '/usr/lib/dyld'
+    ZeroPadBytes:    3
+  - cmd:             LC_UUID
+    cmdsize:         24
+    uuid:            F33E5680-8F64-3402-8952-7FD251400D78
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         32
+    platform:        4
+    minos:           393472
+    sdk:             393472
+    ntools:          1
+    Tools:
+      - tool:            3
+        version:         34734080
+  - cmd:             LC_SOURCE_VERSION
+    cmdsize:         16
+    version:         0
+  - cmd:             LC_MAIN
+    cmdsize:         24
+    entryoff:        32672
+    stacksize:       0
+  - cmd:             LC_ENCRYPTION_INFO
+    cmdsize:         20
+    cryptoff:        16384
+    cryptsize:       16384
+    cryptid:         0
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         52
+    dylib:
+      name:            24
+      timestamp:       2
+      current_version: 83951616
+      compatibility_version: 65536
+    PayloadString:   '/usr/lib/libSystem.B.dylib'
+    ZeroPadBytes:    2
+  - cmd:             LC_FUNCTION_STARTS
+    cmdsize:         16
+    dataoff:         32812
+    datasize:        4
+  - cmd:             LC_DATA_IN_CODE
+    cmdsize:         16
+    dataoff:         32816
+    datasize:        0
+LinkEditData:
+  ExportTrie:
+    TerminalSize:    0
+    NodeOffset:      0
+    Name:            ''
+    Flags:           0x0000000000000000
+    Address:         0x0000000000000000
+    Other:           0x0000000000000000
+    ImportName:      ''
+    Children:
+      - TerminalSize:    0
+        NodeOffset:      5
+        Name:            _
+        Flags:           0x0000000000000000
+        Address:         0x0000000000000000
+        Other:           0x0000000000000000
+        ImportName:      ''
+        Children:
+          - TerminalSize:    2
+            NodeOffset:      33
+            Name:            _mh_execute_header
+            Flags:           0x0000000000000000
+            Address:         0x0000000000000000
+            Other:           0x0000000000000000
+            ImportName:      ''
+          - TerminalSize:    4
+            NodeOffset:      37
+            Name:            main
+            Flags:           0x0000000000000000
+            Address:         0x0000000000007FA0
+            Other:           0x0000000000000000
+            ImportName:      ''
+  NameList:
+    - n_strx:          2
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          16
+      n_value:         16384
+    - n_strx:          22
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          0
+      n_value:         49056
+    - n_strx:          28
+      n_type:          0x01
+      n_sect:          0
+      n_desc:          256
+      n_value:         0
+  StringTable:
+    - ' '
+    - __mh_execute_header
+    - _main
+    - dyld_stub_binder
+    - ''
+    - ''
+    - ''
+...
+
+## Case 4: x86_64 OSX binary.
+## This binary can be built on OSX as follows:
+## main.c: int main() { return 0; }
+## clang -arch x86_64 main.c -o main.exe
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x01000007
+  cpusubtype:      0x80000003
+  filetype:        0x00000002
+  ncmds:           14
+  sizeofcmds:      744
+  flags:           0x00200085
+  reserved:        0x00000000
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __PAGEZERO
+    vmaddr:          0
+    vmsize:          4294967296
+    fileoff:         0
+    filesize:        0
+    maxprot:         0
+    initprot:        0
+    nsects:          0
+    flags:           0
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         232
+    segname:         __TEXT
+    vmaddr:          4294967296
+    vmsize:          4096
+    fileoff:         0
+    filesize:        4096
+    maxprot:         5
+    initprot:        5
+    nsects:          2
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x0000000100000FA0
+        size:            15
+        offset:          0x00000FA0
+        align:           4
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+        content:         554889E531C0C745FC000000005DC3
+      - sectname:        __unwind_info
+        segname:         __TEXT
+        addr:            0x0000000100000FB0
+        size:            72
+        offset:          0x00000FB0
+        align:           2
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x00000000
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+        content:         010000001C000000000000001C000000000000001C00000002000000A00F00003400000034000000B00F00000000000034000000030000000C000100100001000000000000000001
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __LINKEDIT
+    vmaddr:          4294971392
+    vmsize:          4096
+    fileoff:         4096
+    filesize:        152
+    maxprot:         1
+    initprot:        1
+    nsects:          0
+    flags:           0
+  - cmd:             LC_DYLD_INFO_ONLY
+    cmdsize:         48
+    rebase_off:      0
+    rebase_size:     0
+    bind_off:        0
+    bind_size:       0
+    weak_bind_off:   0
+    weak_bind_size:  0
+    lazy_bind_off:   0
+    lazy_bind_size:  0
+    export_off:      4096
+    export_size:     48
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          4152
+    nsyms:           3
+    stroff:          4200
+    strsize:         48
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       0
+    iextdefsym:      0
+    nextdefsym:      2
+    iundefsym:       2
+    nundefsym:       1
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+  - cmd:             LC_LOAD_DYLINKER
+    cmdsize:         32
+    name:            12
+    PayloadString:   '/usr/lib/dyld'
+    ZeroPadBytes:    7
+  - cmd:             LC_UUID
+    cmdsize:         24
+    uuid:            D70F7C9D-410A-3567-83E2-4CA3F33585FF
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         32
+    platform:        1
+    minos:           658944
+    sdk:             658944
+    ntools:          1
+    Tools:
+      - tool:            3
+        version:         34734080
+  - cmd:             LC_SOURCE_VERSION
+    cmdsize:         16
+    version:         0
+  - cmd:             LC_MAIN
+    cmdsize:         24
+    entryoff:        4000
+    stacksize:       0
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         56
+    dylib:
+      name:            24
+      timestamp:       2
+      current_version: 82115073
+      compatibility_version: 65536
+    PayloadString:   '/usr/lib/libSystem.B.dylib'
+    ZeroPadBytes:    6
+  - cmd:             LC_FUNCTION_STARTS
+    cmdsize:         16
+    dataoff:         4144
+    datasize:        8
+  - cmd:             LC_DATA_IN_CODE
+    cmdsize:         16
+    dataoff:         4152
+    datasize:        0
+LinkEditData:
+  ExportTrie:
+    TerminalSize:    0
+    NodeOffset:      0
+    Name:            ''
+    Flags:           0x0000000000000000
+    Address:         0x0000000000000000
+    Other:           0x0000000000000000
+    ImportName:      ''
+    Children:
+      - TerminalSize:    0
+        NodeOffset:      5
+        Name:            _
+        Flags:           0x0000000000000000
+        Address:         0x0000000000000000
+        Other:           0x0000000000000000
+        ImportName:      ''
+        Children:
+          - TerminalSize:    2
+            NodeOffset:      33
+            Name:            _mh_execute_header
+            Flags:           0x0000000000000000
+            Address:         0x0000000000000000
+            Other:           0x0000000000000000
+            ImportName:      ''
+          - TerminalSize:    3
+            NodeOffset:      37
+            Name:            main
+            Flags:           0x0000000000000000
+            Address:         0x0000000000000FA0
+            Other:           0x0000000000000000
+            ImportName:      ''
+  NameList:
+    - n_strx:          2
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          16
+      n_value:         4294967296
+    - n_strx:          22
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294971296
+    - n_strx:          28
+      n_type:          0x01
+      n_sect:          0
+      n_desc:          256
+      n_value:         0
+  StringTable:
+    - ' '
+    - __mh_execute_header
+    - _main
+    - dyld_stub_binder
+    - ''
+    - ''
+    - ''
+...
diff --git a/llvm/test/tools/llvm-objcopy/tool-help-message.test b/llvm/test/tools/llvm-objcopy/tool-help-message.test
new file mode 100644
index 0000000000000..1a0712b7a7ce5
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/tool-help-message.test
@@ -0,0 +1,32 @@
+# RUN: llvm-objcopy -h | FileCheck --check-prefix=OBJCOPY-USAGE %s --match-full-lines
+# RUN: llvm-objcopy --help | FileCheck --check-prefix=OBJCOPY-USAGE %s --match-full-lines
+# RUN: not llvm-objcopy 2>&1 | FileCheck --check-prefix=OBJCOPY-USAGE %s --match-full-lines
+# RUN: not llvm-objcopy -abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
+# RUN: not llvm-objcopy --abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
+# RUN: not llvm-objcopy --strip-debug 2>&1 | FileCheck %s --check-prefix=NO-INPUT-FILES
+
+# RUN: llvm-strip -h | FileCheck --check-prefix=STRIP-USAGE %s --match-full-lines
+# RUN: llvm-strip --help | FileCheck --check-prefix=STRIP-USAGE %s --match-full-lines
+# RUN: not llvm-strip 2>&1 | FileCheck --check-prefix=STRIP-USAGE %s --match-full-lines
+# RUN: not llvm-strip -abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
+# RUN: not llvm-strip --abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
+# RUN: not llvm-strip --strip-debug 2>&1 | FileCheck %s --check-prefix=NO-INPUT-FILES
+
+# RUN: llvm-install-name-tool -h | FileCheck --check-prefix=INSTALL-NAME-TOOL-USAGE %s --match-full-lines
+# RUN: llvm-install-name-tool --help | FileCheck --check-prefix=INSTALL-NAME-TOOL-USAGE %s --match-full-lines
+# RUN: not llvm-install-name-tool 2>&1 | FileCheck --check-prefix=INSTALL-NAME-TOOL-USAGE %s --match-full-lines
+# RUN: not llvm-install-name-tool -abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
+# RUN: not llvm-install-name-tool --abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
+# RUN: not llvm-install-name-tool -add_rpath @executable 2>&1 | FileCheck %s --check-prefix=NO-INPUT-FILES
+
+# OBJCOPY-USAGE:  USAGE: llvm-objcopy [options] input [output]
+# OBJCOPY-USAGE:  Pass @FILE as argument to read options from FILE.
+
+# STRIP-USAGE:    USAGE: llvm-strip [options] inputs...
+# STRIP-USAGE:    Pass @FILE as argument to read options from FILE.
+
+# INSTALL-NAME-TOOL-USAGE: USAGE: llvm-install-name-tool [options] input
+# INSTALL-NAME-TOOL-USAGE: Pass @FILE as argument to read options from FILE.
+
+# UNKNOWN-ARG:    unknown argument '{{-+}}abcabc'
+# NO-INPUT-FILES: no input file specified
diff --git a/llvm/test/tools/llvm-objdump/ARM/lit.local.cfg b/llvm/test/tools/llvm-objdump/ARM/lit.local.cfg
deleted file mode 100644
index 236e1d3441665..0000000000000
--- a/llvm/test/tools/llvm-objdump/ARM/lit.local.cfg
+++ /dev/null
@@ -1,2 +0,0 @@
-if not 'ARM' in config.root.targets:
-    config.unsupported = True
diff --git a/llvm/test/tools/llvm-objdump/ELF/ARM/Inputs/debug.c b/llvm/test/tools/llvm-objdump/ELF/ARM/Inputs/debug.c
new file mode 100644
index 0000000000000..20c17f7d9762d
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/ELF/ARM/Inputs/debug.c
@@ -0,0 +1,10 @@
+int foo(int a, int b, int c) {
+  int x = a + b;
+  int y = x + c;
+  return y;
+}
+
+int bar(int a) {
+  a++;
+  return a;
+}
diff --git a/llvm/test/tools/llvm-objdump/ELF/ARM/Inputs/wide-char.c b/llvm/test/tools/llvm-objdump/ELF/ARM/Inputs/wide-char.c
new file mode 100644
index 0000000000000..8d923be013283
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/ELF/ARM/Inputs/wide-char.c
@@ -0,0 +1,3 @@
+int foo(int *喵) {
+  return *喵;
+}
diff --git a/llvm/test/tools/llvm-objdump/ELF/ARM/debug-vars-dwarf4-sections.s b/llvm/test/tools/llvm-objdump/ELF/ARM/debug-vars-dwarf4-sections.s
new file mode 100644
index 0000000000000..9bbb36f14e3aa
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/ELF/ARM/debug-vars-dwarf4-sections.s
@@ -0,0 +1,351 @@
+## Check that the --debug-vars option works for simple register locations, when
+## using DWARF4 debug info, with functions in multiple sections.
+
+## Generated with this compile command, with the source code in Inputs/debug.c:
+## clang --target=arm--none-eabi -march=armv7-a -c debug.c -O1 -gdwarf-4 -S -o - -ffunction-sections
+
+# RUN: llvm-mc -triple armv8a--none-eabi < %s -filetype=obj | \
+# RUN:     llvm-objdump - -d --debug-vars --no-show-raw-insn | \
+# RUN:     FileCheck %s
+
+# CHECK: Disassembly of section .text.foo:
+# CHECK-EMPTY:
+# CHECK-NEXT: 00000000 <foo>:
+# CHECK-NEXT:                                                                   ┠─ a = R0
+# CHECK-NEXT:                                                                   ┃ ┠─ b = R1
+# CHECK-NEXT:                                                                   ┃ ┃ ┠─ c = R2
+# CHECK-NEXT:                                                                   ┃ ┃ ┃ ┌─ x = R0
+# CHECK-NEXT:        0:       add     r0, r1, r0                                ┻ ┃ ┃ ╈
+# CHECK-NEXT:                                                                   ┌─ y = R0
+# CHECK-NEXT:        4:       add     r0, r0, r2                                ╈ ┃ ┃ ┻
+# CHECK-NEXT:        8:       bx      lr                                        ┻ ┻ ┻
+# CHECK-EMPTY:
+# CHECK-NEXT: Disassembly of section .text.bar:
+# CHECK-EMPTY:
+# CHECK-NEXT: 00000000 <bar>:
+# CHECK-NEXT:                                                                   ┠─ a = R0
+# CHECK-NEXT:        0:       add     r0, r0, #1                                ┃
+# CHECK-NEXT:        4:       bx      lr                                        ┻
+
+	.text
+	.syntax unified
+	.eabi_attribute	67, "2.09"
+	.eabi_attribute	6, 10
+	.eabi_attribute	7, 65
+	.eabi_attribute	8, 1
+	.eabi_attribute	9, 2
+	.fpu	neon
+	.eabi_attribute	34, 0
+	.eabi_attribute	17, 1
+	.eabi_attribute	20, 1
+	.eabi_attribute	21, 1
+	.eabi_attribute	23, 3
+	.eabi_attribute	24, 1
+	.eabi_attribute	25, 1
+	.eabi_attribute	38, 1
+	.eabi_attribute	18, 4
+	.eabi_attribute	26, 2
+	.eabi_attribute	14, 0
+	.file	"debug.c"
+	.section	.text.foo,"ax",%progbits
+	.globl	foo
+	.p2align	2
+	.type	foo,%function
+	.code	32
+foo:
+.Lfunc_begin0:
+	.file	1 "/work" "llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c"
+	.loc	1 1 0
+	.fnstart
+	.cfi_sections .debug_frame
+	.cfi_startproc
+	.loc	1 2 13 prologue_end
+	add	r0, r1, r0
+.Ltmp0:
+	.loc	1 3 13
+	add	r0, r0, r2
+.Ltmp1:
+	.loc	1 4 3
+	bx	lr
+.Ltmp2:
+.Lfunc_end0:
+	.size	foo, .Lfunc_end0-foo
+	.cfi_endproc
+	.cantunwind
+	.fnend
+
+	.section	.text.bar,"ax",%progbits
+	.globl	bar
+	.p2align	2
+	.type	bar,%function
+	.code	32
+bar:
+.Lfunc_begin1:
+	.loc	1 7 0
+	.fnstart
+	.cfi_startproc
+	.loc	1 8 4 prologue_end
+	add	r0, r0, #1
+.Ltmp3:
+	.loc	1 9 3
+	bx	lr
+.Ltmp4:
+.Lfunc_end1:
+	.size	bar, .Lfunc_end1-bar
+	.cfi_endproc
+	.cantunwind
+	.fnend
+
+	.section	.debug_str,"MS",%progbits,1
+.Linfo_string0:
+	.asciz	"clang version 10.0.0 (git@github.com:llvm/llvm-project.git e73f78acd34360f7450b81167d9dc858ccddc262)"
+.Linfo_string1:
+	.asciz	"/work/llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c"
+.Linfo_string2:
+	.asciz	"/work/scratch"
+.Linfo_string3:
+	.asciz	"foo"
+.Linfo_string4:
+	.asciz	"int"
+.Linfo_string5:
+	.asciz	"bar"
+.Linfo_string6:
+	.asciz	"a"
+.Linfo_string7:
+	.asciz	"b"
+.Linfo_string8:
+	.asciz	"c"
+.Linfo_string9:
+	.asciz	"x"
+.Linfo_string10:
+	.asciz	"y"
+	.section	.debug_loc,"",%progbits
+.Ldebug_loc0:
+	.long	-1
+	.long	.Lfunc_begin0
+	.long	.Lfunc_begin0-.Lfunc_begin0
+	.long	.Ltmp0-.Lfunc_begin0
+	.short	1
+	.byte	80
+	.long	0
+	.long	0
+.Ldebug_loc1:
+	.long	-1
+	.long	.Lfunc_begin0
+	.long	.Ltmp0-.Lfunc_begin0
+	.long	.Ltmp1-.Lfunc_begin0
+	.short	1
+	.byte	80
+	.long	0
+	.long	0
+.Ldebug_loc2:
+	.long	-1
+	.long	.Lfunc_begin0
+	.long	.Ltmp1-.Lfunc_begin0
+	.long	.Lfunc_end0-.Lfunc_begin0
+	.short	1
+	.byte	80
+	.long	0
+	.long	0
+	.section	.debug_abbrev,"",%progbits
+	.byte	1
+	.byte	17
+	.byte	1
+	.byte	37
+	.byte	14
+	.byte	19
+	.byte	5
+	.byte	3
+	.byte	14
+	.byte	16
+	.byte	23
+	.byte	27
+	.byte	14
+	.byte	17
+	.byte	1
+	.byte	85
+	.byte	23
+	.byte	0
+	.byte	0
+	.byte	2
+	.byte	46
+	.byte	1
+	.byte	17
+	.byte	1
+	.byte	18
+	.byte	6
+	.byte	64
+	.byte	24
+	.ascii	"\227B"
+	.byte	25
+	.byte	3
+	.byte	14
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	39
+	.byte	25
+	.byte	73
+	.byte	19
+	.byte	63
+	.byte	25
+	.byte	0
+	.byte	0
+	.byte	3
+	.byte	5
+	.byte	0
+	.byte	2
+	.byte	23
+	.byte	3
+	.byte	14
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	73
+	.byte	19
+	.byte	0
+	.byte	0
+	.byte	4
+	.byte	5
+	.byte	0
+	.byte	2
+	.byte	24
+	.byte	3
+	.byte	14
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	73
+	.byte	19
+	.byte	0
+	.byte	0
+	.byte	5
+	.byte	52
+	.byte	0
+	.byte	2
+	.byte	23
+	.byte	3
+	.byte	14
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	73
+	.byte	19
+	.byte	0
+	.byte	0
+	.byte	6
+	.byte	36
+	.byte	0
+	.byte	3
+	.byte	14
+	.byte	62
+	.byte	11
+	.byte	11
+	.byte	11
+	.byte	0
+	.byte	0
+	.byte	0
+	.section	.debug_info,"",%progbits
+.Lcu_begin0:
+	.long	.Ldebug_info_end0-.Ldebug_info_start0
+.Ldebug_info_start0:
+	.short	4
+	.long	.debug_abbrev
+	.byte	4
+	.byte	1
+	.long	.Linfo_string0
+	.short	12
+	.long	.Linfo_string1
+	.long	.Lline_table_start0
+	.long	.Linfo_string2
+	.long	0
+	.long	.Ldebug_ranges0
+	.byte	2
+	.long	.Lfunc_begin0
+	.long	.Lfunc_end0-.Lfunc_begin0
+	.byte	1
+	.byte	91
+
+	.long	.Linfo_string3
+	.byte	1
+	.byte	1
+
+	.long	166
+
+	.byte	3
+	.long	.Ldebug_loc0
+	.long	.Linfo_string6
+	.byte	1
+	.byte	1
+	.long	166
+	.byte	4
+	.byte	1
+	.byte	81
+	.long	.Linfo_string7
+	.byte	1
+	.byte	1
+	.long	166
+	.byte	4
+	.byte	1
+	.byte	82
+	.long	.Linfo_string8
+	.byte	1
+	.byte	1
+	.long	166
+	.byte	5
+	.long	.Ldebug_loc1
+	.long	.Linfo_string9
+	.byte	1
+	.byte	2
+	.long	166
+	.byte	5
+	.long	.Ldebug_loc2
+	.long	.Linfo_string10
+	.byte	1
+	.byte	3
+	.long	166
+	.byte	0
+	.byte	2
+	.long	.Lfunc_begin1
+	.long	.Lfunc_end1-.Lfunc_begin1
+	.byte	1
+	.byte	91
+
+	.long	.Linfo_string5
+	.byte	1
+	.byte	7
+
+	.long	166
+
+	.byte	4
+	.byte	1
+	.byte	80
+	.long	.Linfo_string6
+	.byte	1
+	.byte	7
+	.long	166
+	.byte	0
+	.byte	6
+	.long	.Linfo_string4
+	.byte	5
+	.byte	4
+	.byte	0
+.Ldebug_info_end0:
+	.section	.debug_ranges,"",%progbits
+.Ldebug_ranges0:
+	.long	.Lfunc_begin0
+	.long	.Lfunc_end0
+	.long	.Lfunc_begin1
+	.long	.Lfunc_end1
+	.long	0
+	.long	0
+	.ident	"clang version 10.0.0 (git@github.com:llvm/llvm-project.git e73f78acd34360f7450b81167d9dc858ccddc262)"
+	.section	".note.GNU-stack","",%progbits
+	.addrsig
+	.eabi_attribute	30, 1
+	.section	.debug_line,"",%progbits
+.Lline_table_start0:
diff --git a/llvm/test/tools/llvm-objdump/ELF/ARM/debug-vars-dwarf4.s b/llvm/test/tools/llvm-objdump/ELF/ARM/debug-vars-dwarf4.s
new file mode 100644
index 0000000000000..bf0c7bd52feb4
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/ELF/ARM/debug-vars-dwarf4.s
@@ -0,0 +1,454 @@
+## Check that the --debug-vars option works for simple register locations, when
+## using DWARF4 debug info, with multiple functions in one section. Check that
+## the live-range lines are rendered correctly when using the --no-show-raw-insn,
+## --line-numbers and --source options. These do not affect the DWARF parsing
+## used by --debug-vars, but do add extra lines or columns to the output, so we
+## test to make sure the live ranges are still displayed correctly.
+
+## Generated with this compile command, with the source code in Inputs/debug.c:
+## clang --target=arm--none-eabi -march=armv7-a -c debug.c -O1 -gdwarf-4 -S -o -
+
+# RUN: llvm-mc -triple armv8a--none-eabi < %s -filetype=obj | \
+# RUN:     llvm-objdump - -d --debug-vars | \
+# RUN:     FileCheck %s --check-prefix=RAW --strict-whitespace
+
+## Check that passing the default value for --debug-vars-indent (40) makes no
+## change to the output.
+# RUN: llvm-mc -triple armv8a--none-eabi < %s -filetype=obj | \
+# RUN:     llvm-objdump - -d --debug-vars --debug-vars-indent=40 | \
+# RUN:     FileCheck %s --check-prefix=RAW --strict-whitespace
+
+# RUN: llvm-mc -triple armv8a--none-eabi < %s -filetype=obj | \
+# RUN:     llvm-objdump - -d --debug-vars --debug-vars-indent=30 | \
+# RUN:     FileCheck %s --check-prefix=INDENT --strict-whitespace
+
+# RUN: llvm-mc -triple armv8a--none-eabi < %s -filetype=obj | \
+# RUN:     llvm-objdump - -d --debug-vars --no-show-raw-insn | \
+# RUN:     FileCheck %s --check-prefix=NO-RAW --strict-whitespace
+
+# RUN: llvm-mc -triple armv8a--none-eabi < %s -filetype=obj | \
+# RUN:     llvm-objdump - -d --debug-vars --no-show-raw-insn --line-numbers | \
+# RUN:     FileCheck %s --check-prefix=LINE-NUMS --strict-whitespace
+
+# RUN: mkdir -p %t/a
+# RUN: cp %p/Inputs/debug.c %t/a/debug.c
+# RUN: sed -e "s,SRC_COMPDIR,%/t/a,g" %s > %t.s
+# RUN: llvm-mc -triple armv8a--none-eabi < %t.s -filetype=obj | \
+# RUN:     llvm-objdump - -d --debug-vars --no-show-raw-insn --source | \
+# RUN:     FileCheck %s --check-prefix=SOURCE --strict-whitespace
+
+## An optional argument to the --debug-vars= option can be used to switch
+## between unicode and ascii output (with unicode being the default).
+# RUN: llvm-mc -triple armv8a--none-eabi < %s -filetype=obj | \
+# RUN:     llvm-objdump - -d --debug-vars=unicode | \
+# RUN:     FileCheck %s --check-prefix=RAW --strict-whitespace
+# RUN: llvm-mc -triple armv8a--none-eabi < %s -filetype=obj | \
+# RUN:     llvm-objdump - -d --debug-vars=ascii | \
+# RUN:     FileCheck %s --check-prefix=ASCII --strict-whitespace
+
+## Note that llvm-objdump emits tab characters in the disassembly, assuming an
+## 8-byte tab stop, so these might not look aligned in a text editor.
+
+# RAW: 00000000 <foo>:
+# RAW-NEXT:                                                                 ┠─ a = R0 
+# RAW-NEXT:                                                                 ┃ ┠─ b = R1 
+# RAW-NEXT:                                                                 ┃ ┃ ┠─ c = R2 
+# RAW-NEXT:                                                                 ┃ ┃ ┃ ┌─ x = R0 
+# RAW-NEXT:        0: 00 00 81 e0  	add	r0, r1, r0                      ┻ ┃ ┃ ╈   
+# RAW-NEXT:                                                                 ┌─ y = R0 
+# RAW-NEXT:        4: 02 00 80 e0  	add	r0, r0, r2                      ╈ ┃ ┃ ┻   
+# RAW-NEXT:        8: 1e ff 2f e1  	bx	lr                              ┻ ┻ ┻     
+# RAW-EMPTY:
+# RAW-NEXT: 0000000c <bar>:
+# RAW-NEXT:                                                                 ┠─ a = R0 
+# RAW-NEXT:        c: 01 00 80 e2  	add	r0, r0, #1                      ┃         
+# RAW-NEXT:       10: 1e ff 2f e1  	bx	lr                              ┻         
+
+
+# INDENT: 00000000 <foo>:
+# INDENT-NEXT:                                                       ┠─ a = R0 
+# INDENT-NEXT:                                                       ┃ ┠─ b = R1 
+# INDENT-NEXT:                                                       ┃ ┃ ┠─ c = R2 
+# INDENT-NEXT:                                                       ┃ ┃ ┃ ┌─ x = R0 
+# INDENT-NEXT:        0: 00 00 81 e0  	add	r0, r1, r0            ┻ ┃ ┃ ╈   
+# INDENT-NEXT:                                                       ┌─ y = R0 
+# INDENT-NEXT:        4: 02 00 80 e0  	add	r0, r0, r2            ╈ ┃ ┃ ┻   
+# INDENT-NEXT:        8: 1e ff 2f e1  	bx	lr                    ┻ ┻ ┻     
+# INDENT-EMPTY:
+# INDENT-NEXT: 0000000c <bar>:
+# INDENT-NEXT:                                                       ┠─ a = R0 
+# INDENT-NEXT:        c: 01 00 80 e2  	add	r0, r0, #1            ┃         
+# INDENT-NEXT:       10: 1e ff 2f e1  	bx	lr                    ┻         
+
+# NO-RAW: 00000000 <foo>:
+# NO-RAW-NEXT:                                                         ┠─ a = R0
+# NO-RAW-NEXT:                                                         ┃ ┠─ b = R1
+# NO-RAW-NEXT:                                                         ┃ ┃ ┠─ c = R2
+# NO-RAW-NEXT:                                                         ┃ ┃ ┃ ┌─ x = R0
+# NO-RAW-NEXT:        0:      	add	r0, r1, r0                      ┻ ┃ ┃ ╈
+# NO-RAW-NEXT:                                                         ┌─ y = R0
+# NO-RAW-NEXT:        4:      	add	r0, r0, r2                      ╈ ┃ ┃ ┻
+# NO-RAW-NEXT:        8:      	bx	lr                              ┻ ┻ ┻
+# NO-RAW-EMPTY:
+# NO-RAW-NEXT: 0000000c <bar>:
+# NO-RAW-NEXT:                                                         ┠─ a = R0
+# NO-RAW-NEXT:        c:      	add	r0, r0, #1                      ┃
+# NO-RAW-NEXT:       10:      	bx	lr                              ┻
+
+# LINE-NUMS: 00000000 <foo>:
+# LINE-NUMS-NEXT: ; foo():
+# LINE-NUMS-NEXT: ; SRC_COMPDIR{{[\\/]}}debug.c:2                                 ┠─ a = R0
+# LINE-NUMS-NEXT:                                                         ┃ ┠─ b = R1
+# LINE-NUMS-NEXT:                                                         ┃ ┃ ┠─ c = R2
+# LINE-NUMS-NEXT:                                                         ┃ ┃ ┃ ┌─ x = R0
+# LINE-NUMS-NEXT:        0:      	add	r0, r1, r0                      ┻ ┃ ┃ ╈
+# LINE-NUMS-NEXT: ; SRC_COMPDIR{{[\\/]}}debug.c:3                                 ┌─ y = R0
+# LINE-NUMS-NEXT:        4:      	add	r0, r0, r2                      ╈ ┃ ┃ ┻
+# LINE-NUMS-NEXT: ; SRC_COMPDIR{{[\\/]}}debug.c:4                                 ┃ ┃ ┃
+# LINE-NUMS-NEXT:        8:      	bx	lr                              ┻ ┻ ┻
+# LINE-NUMS-EMPTY:
+# LINE-NUMS-NEXT: 0000000c <bar>:
+# LINE-NUMS-NEXT: ; bar():
+# LINE-NUMS-NEXT: ; SRC_COMPDIR{{[\\/]}}debug.c:8                                 ┠─ a = R0
+# LINE-NUMS-NEXT:        c:      	add	r0, r0, #1                      ┃
+# LINE-NUMS-NEXT: ; SRC_COMPDIR{{[\\/]}}debug.c:9                                 ┃
+# LINE-NUMS-NEXT:       10:      	bx	lr                              ┻
+
+# SOURCE: 00000000 <foo>:
+# SOURCE-NEXT: ;   int x = a + b;                                      ┠─ a = R0
+# SOURCE-NEXT:                                                         ┃ ┠─ b = R1
+# SOURCE-NEXT:                                                         ┃ ┃ ┠─ c = R2
+# SOURCE-NEXT:                                                         ┃ ┃ ┃ ┌─ x = R0
+# SOURCE-NEXT:        0:      	add	r0, r1, r0                      ┻ ┃ ┃ ╈
+# SOURCE-NEXT: ;   int y = x + c;                                      ┌─ y = R0
+# SOURCE-NEXT:        4:      	add	r0, r0, r2                      ╈ ┃ ┃ ┻
+# SOURCE-NEXT: ;   return y;                                           ┃ ┃ ┃
+# SOURCE-NEXT:        8:      	bx	lr                              ┻ ┻ ┻
+# SOURCE-EMPTY:
+# SOURCE-NEXT: 0000000c <bar>:
+# SOURCE-NEXT: ;   a++;                                                ┠─ a = R0
+# SOURCE-NEXT:        c:      	add	r0, r0, #1                      ┃
+# SOURCE-NEXT: ;   return a;                                           ┃
+# SOURCE-NEXT:       10:      	bx	lr                              ┻
+
+# ASCII: 00000000 <foo>:
+# ASCII-NEXT:                                                                 |- a = R0 
+# ASCII-NEXT:                                                                 | |- b = R1 
+# ASCII-NEXT:                                                                 | | |- c = R2 
+# ASCII-NEXT:                                                                 | | | /- x = R0 
+# ASCII-NEXT:        0: 00 00 81 e0  	add	r0, r1, r0                      v | | ^   
+# ASCII-NEXT:                                                                 /- y = R0 
+# ASCII-NEXT:        4: 02 00 80 e0  	add	r0, r0, r2                      ^ | | v   
+# ASCII-NEXT:        8: 1e ff 2f e1  	bx	lr                              v v v     
+# ASCII-EMPTY:
+# ASCII-NEXT: 0000000c <bar>:
+# ASCII-NEXT:                                                                 |- a = R0 
+# ASCII-NEXT:        c: 01 00 80 e2  	add	r0, r0, #1                      |         
+# ASCII-NEXT:       10: 1e ff 2f e1  	bx	lr                              v         
+
+	.text
+	.syntax unified
+	.eabi_attribute	67, "2.09"
+	.eabi_attribute	6, 10
+	.eabi_attribute	7, 65
+	.eabi_attribute	8, 1
+	.eabi_attribute	9, 2
+	.fpu	neon
+	.eabi_attribute	34, 0
+	.eabi_attribute	17, 1
+	.eabi_attribute	20, 1
+	.eabi_attribute	21, 1
+	.eabi_attribute	23, 3
+	.eabi_attribute	24, 1
+	.eabi_attribute	25, 1
+	.eabi_attribute	38, 1
+	.eabi_attribute	18, 4
+	.eabi_attribute	26, 2
+	.eabi_attribute	14, 0
+	.file	"debug.c"
+	.globl	foo
+	.p2align	2
+	.type	foo,%function
+	.code	32
+foo:
+.Lfunc_begin0:
+	.file	1 "" "SRC_COMPDIR/debug.c"
+	.loc	1 1 0
+	.fnstart
+	.cfi_sections .debug_frame
+	.cfi_startproc
+	.loc	1 2 13 prologue_end
+	add	r0, r1, r0
+.Ltmp0:
+	.loc	1 3 13
+	add	r0, r0, r2
+.Ltmp1:
+	.loc	1 4 3
+	bx	lr
+.Ltmp2:
+.Lfunc_end0:
+	.size	foo, .Lfunc_end0-foo
+	.cfi_endproc
+	.cantunwind
+	.fnend
+
+	.globl	bar
+	.p2align	2
+	.type	bar,%function
+	.code	32
+bar:
+.Lfunc_begin1:
+	.loc	1 7 0
+	.fnstart
+	.cfi_startproc
+	.loc	1 8 4 prologue_end
+	add	r0, r0, #1
+.Ltmp3:
+	.loc	1 9 3
+	bx	lr
+.Ltmp4:
+.Lfunc_end1:
+	.size	bar, .Lfunc_end1-bar
+	.cfi_endproc
+	.cantunwind
+	.fnend
+
+	.section	.debug_str,"MS",%progbits,1
+.Linfo_string0:
+	.asciz	"clang version 10.0.0 (git@github.com:llvm/llvm-project.git e73f78acd34360f7450b81167d9dc858ccddc262)"
+.Linfo_string1:
+	.asciz	"SRC_COMPDIR/debug.c"
+.Linfo_string2:
+	.asciz	""
+.Linfo_string3:
+	.asciz	"foo"
+.Linfo_string4:
+	.asciz	"int"
+.Linfo_string5:
+	.asciz	"bar"
+.Linfo_string6:
+	.asciz	"a"
+.Linfo_string7:
+	.asciz	"b"
+.Linfo_string8:
+	.asciz	"c"
+.Linfo_string9:
+	.asciz	"x"
+.Linfo_string10:
+	.asciz	"y"
+	.section	.debug_loc,"",%progbits
+.Ldebug_loc0:
+	.long	.Lfunc_begin0-.Lfunc_begin0
+	.long	.Ltmp0-.Lfunc_begin0
+	.short	1
+	.byte	80
+	.long	0
+	.long	0
+.Ldebug_loc1:
+	.long	.Ltmp0-.Lfunc_begin0
+	.long	.Ltmp1-.Lfunc_begin0
+	.short	1
+	.byte	80
+	.long	0
+	.long	0
+.Ldebug_loc2:
+	.long	.Ltmp1-.Lfunc_begin0
+	.long	.Lfunc_end0-.Lfunc_begin0
+	.short	1
+	.byte	80
+	.long	0
+	.long	0
+	.section	.debug_abbrev,"",%progbits
+	.byte	1
+	.byte	17
+	.byte	1
+	.byte	37
+	.byte	14
+	.byte	19
+	.byte	5
+	.byte	3
+	.byte	14
+	.byte	16
+	.byte	23
+	.byte	27
+	.byte	14
+	.byte	17
+	.byte	1
+	.byte	18
+	.byte	6
+	.byte	0
+	.byte	0
+	.byte	2
+	.byte	46
+	.byte	1
+	.byte	17
+	.byte	1
+	.byte	18
+	.byte	6
+	.byte	64
+	.byte	24
+	.ascii	"\227B"
+	.byte	25
+	.byte	3
+	.byte	14
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	39
+	.byte	25
+	.byte	73
+	.byte	19
+	.byte	63
+	.byte	25
+	.byte	0
+	.byte	0
+	.byte	3
+	.byte	5
+	.byte	0
+	.byte	2
+	.byte	23
+	.byte	3
+	.byte	14
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	73
+	.byte	19
+	.byte	0
+	.byte	0
+	.byte	4
+	.byte	5
+	.byte	0
+	.byte	2
+	.byte	24
+	.byte	3
+	.byte	14
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	73
+	.byte	19
+	.byte	0
+	.byte	0
+	.byte	5
+	.byte	52
+	.byte	0
+	.byte	2
+	.byte	23
+	.byte	3
+	.byte	14
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	73
+	.byte	19
+	.byte	0
+	.byte	0
+	.byte	6
+	.byte	36
+	.byte	0
+	.byte	3
+	.byte	14
+	.byte	62
+	.byte	11
+	.byte	11
+	.byte	11
+	.byte	0
+	.byte	0
+	.byte	0
+	.section	.debug_info,"",%progbits
+.Lcu_begin0:
+	.long	.Ldebug_info_end0-.Ldebug_info_start0
+.Ldebug_info_start0:
+	.short	4
+	.long	.debug_abbrev
+	.byte	4
+	.byte	1
+	.long	.Linfo_string0
+	.short	12
+	.long	.Linfo_string1
+	.long	.Lline_table_start0
+	.long	.Linfo_string2
+	.long	.Lfunc_begin0
+	.long	.Lfunc_end1-.Lfunc_begin0
+	.byte	2
+	.long	.Lfunc_begin0
+	.long	.Lfunc_end0-.Lfunc_begin0
+	.byte	1
+	.byte	91
+
+	.long	.Linfo_string3
+	.byte	1
+	.byte	1
+
+	.long	166
+
+	.byte	3
+	.long	.Ldebug_loc0
+	.long	.Linfo_string6
+	.byte	1
+	.byte	1
+	.long	166
+	.byte	4
+	.byte	1
+	.byte	81
+	.long	.Linfo_string7
+	.byte	1
+	.byte	1
+	.long	166
+	.byte	4
+	.byte	1
+	.byte	82
+	.long	.Linfo_string8
+	.byte	1
+	.byte	1
+	.long	166
+	.byte	5
+	.long	.Ldebug_loc1
+	.long	.Linfo_string9
+	.byte	1
+	.byte	2
+	.long	166
+	.byte	5
+	.long	.Ldebug_loc2
+	.long	.Linfo_string10
+	.byte	1
+	.byte	3
+	.long	166
+	.byte	0
+	.byte	2
+	.long	.Lfunc_begin1
+	.long	.Lfunc_end1-.Lfunc_begin1
+	.byte	1
+	.byte	91
+
+	.long	.Linfo_string5
+	.byte	1
+	.byte	7
+
+	.long	166
+
+	.byte	4
+	.byte	1
+	.byte	80
+	.long	.Linfo_string6
+	.byte	1
+	.byte	7
+	.long	166
+	.byte	0
+	.byte	6
+	.long	.Linfo_string4
+	.byte	5
+	.byte	4
+	.byte	0
+.Ldebug_info_end0:
+	.ident	"clang version 10.0.0 (git@github.com:llvm/llvm-project.git e73f78acd34360f7450b81167d9dc858ccddc262)"
+	.section	".note.GNU-stack","",%progbits
+	.addrsig
+	.eabi_attribute	30, 1
+	.section	.debug_line,"",%progbits
+.Lline_table_start0:
diff --git a/llvm/test/tools/llvm-objdump/ELF/ARM/debug-vars-dwarf5-sections.s b/llvm/test/tools/llvm-objdump/ELF/ARM/debug-vars-dwarf5-sections.s
new file mode 100644
index 0000000000000..a9e08ff955561
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/ELF/ARM/debug-vars-dwarf5-sections.s
@@ -0,0 +1,411 @@
+## Check that the --debug-vars option works for simple register locations, when
+## using DWARF4 debug info, with functions in multiple sections.
+
+## Generated with this compile command, with the source code in Inputs/debug.c:
+## clang --target=arm--none-eabi -march=armv7-a -c debug.c -O1 -gdwarf-5 -S -o - -ffunction-sections
+
+# RUN: llvm-mc -triple armv8a--none-eabi < %s -filetype=obj --dwarf-version=5 | \
+# RUN:     llvm-objdump - -d --debug-vars --no-show-raw-insn | \
+# RUN:     FileCheck %s
+
+# CHECK: Disassembly of section .text.foo:
+# CHECK-EMPTY:
+# CHECK-NEXT: 00000000 <foo>:
+# CHECK-NEXT:                                                                   ┠─ a = R0
+# CHECK-NEXT:                                                                   ┃ ┠─ b = R1
+# CHECK-NEXT:                                                                   ┃ ┃ ┠─ c = R2
+# CHECK-NEXT:                                                                   ┃ ┃ ┃ ┌─ x = R0
+# CHECK-NEXT:        0:       add     r0, r1, r0                                ┻ ┃ ┃ ╈
+# CHECK-NEXT:                                                                   ┌─ y = R0
+# CHECK-NEXT:        4:       add     r0, r0, r2                                ╈ ┃ ┃ ┻
+# CHECK-NEXT:        8:       bx      lr                                        ┻ ┻ ┻
+# CHECK-EMPTY:
+# CHECK-NEXT: Disassembly of section .text.bar:
+# CHECK-EMPTY:
+# CHECK-NEXT: 00000000 <bar>:
+# CHECK-NEXT:                                                                   ┠─ a = R0
+# CHECK-NEXT:        0:       add     r0, r0, #1                                ┃
+# CHECK-NEXT:        4:       bx      lr                                        ┻
+
+	.text
+	.syntax unified
+	.eabi_attribute	67, "2.09"
+	.eabi_attribute	6, 10
+	.eabi_attribute	7, 65
+	.eabi_attribute	8, 1
+	.eabi_attribute	9, 2
+	.fpu	neon
+	.eabi_attribute	34, 0
+	.eabi_attribute	17, 1
+	.eabi_attribute	20, 1
+	.eabi_attribute	21, 1
+	.eabi_attribute	23, 3
+	.eabi_attribute	24, 1
+	.eabi_attribute	25, 1
+	.eabi_attribute	38, 1
+	.eabi_attribute	18, 4
+	.eabi_attribute	26, 2
+	.eabi_attribute	14, 0
+	.file	"debug.c"
+	.section	.text.foo,"ax",%progbits
+	.globl	foo
+	.p2align	2
+	.type	foo,%function
+	.code	32
+foo:
+.Lfunc_begin0:
+	.file	0 "/work/scratch" "/work/llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c" md5 0x07374f01ab24ec7c07db73bc13bd778e
+	.file	1 "/work" "llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c" md5 0x07374f01ab24ec7c07db73bc13bd778e
+	.loc	1 1 0
+	.fnstart
+	.cfi_sections .debug_frame
+	.cfi_startproc
+	.loc	1 2 13 prologue_end
+	add	r0, r1, r0
+.Ltmp0:
+	.loc	1 3 13
+	add	r0, r0, r2
+.Ltmp1:
+	.loc	1 4 3
+	bx	lr
+.Ltmp2:
+.Lfunc_end0:
+	.size	foo, .Lfunc_end0-foo
+	.cfi_endproc
+	.cantunwind
+	.fnend
+
+	.section	.text.bar,"ax",%progbits
+	.globl	bar
+	.p2align	2
+	.type	bar,%function
+	.code	32
+bar:
+.Lfunc_begin1:
+	.loc	1 7 0
+	.fnstart
+	.cfi_startproc
+	.loc	1 8 4 prologue_end
+	add	r0, r0, #1
+.Ltmp3:
+	.loc	1 9 3
+	bx	lr
+.Ltmp4:
+.Lfunc_end1:
+	.size	bar, .Lfunc_end1-bar
+	.cfi_endproc
+	.cantunwind
+	.fnend
+
+	.section	.debug_str_offsets,"",%progbits
+	.long	48
+	.short	5
+	.short	0
+.Lstr_offsets_base0:
+	.section	.debug_str,"MS",%progbits,1
+.Linfo_string0:
+	.asciz	"clang version 10.0.0 (git@github.com:llvm/llvm-project.git e73f78acd34360f7450b81167d9dc858ccddc262)"
+.Linfo_string1:
+	.asciz	"/work/llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c"
+.Linfo_string2:
+	.asciz	"/work/scratch"
+.Linfo_string3:
+	.asciz	"foo"
+.Linfo_string4:
+	.asciz	"int"
+.Linfo_string5:
+	.asciz	"bar"
+.Linfo_string6:
+	.asciz	"a"
+.Linfo_string7:
+	.asciz	"b"
+.Linfo_string8:
+	.asciz	"c"
+.Linfo_string9:
+	.asciz	"x"
+.Linfo_string10:
+	.asciz	"y"
+	.section	.debug_str_offsets,"",%progbits
+	.long	.Linfo_string0
+	.long	.Linfo_string1
+	.long	.Linfo_string2
+	.long	.Linfo_string3
+	.long	.Linfo_string4
+	.long	.Linfo_string5
+	.long	.Linfo_string6
+	.long	.Linfo_string7
+	.long	.Linfo_string8
+	.long	.Linfo_string9
+	.long	.Linfo_string10
+	.section	.debug_loclists,"",%progbits
+	.long	.Ldebug_loclist_table_end0-.Ldebug_loclist_table_start0
+.Ldebug_loclist_table_start0:
+	.short	5
+	.byte	4
+	.byte	0
+	.long	3
+.Lloclists_table_base0:
+	.long	.Ldebug_loc0-.Lloclists_table_base0
+	.long	.Ldebug_loc1-.Lloclists_table_base0
+	.long	.Ldebug_loc2-.Lloclists_table_base0
+.Ldebug_loc0:
+	.byte	3
+	.byte	0
+	.uleb128 .Ltmp0-.Lfunc_begin0
+	.byte	1
+	.byte	80
+	.byte	0
+.Ldebug_loc1:
+	.byte	1
+	.byte	0
+	.byte	4
+	.uleb128 .Ltmp0-.Lfunc_begin0
+	.uleb128 .Ltmp1-.Lfunc_begin0
+	.byte	1
+	.byte	80
+	.byte	0
+.Ldebug_loc2:
+	.byte	1
+	.byte	0
+	.byte	4
+	.uleb128 .Ltmp1-.Lfunc_begin0
+	.uleb128 .Lfunc_end0-.Lfunc_begin0
+	.byte	1
+	.byte	80
+	.byte	0
+.Ldebug_loclist_table_end0:
+	.section	.debug_abbrev,"",%progbits
+	.byte	1
+	.byte	17
+	.byte	1
+	.byte	37
+	.byte	37
+	.byte	19
+	.byte	5
+	.byte	3
+	.byte	37
+	.byte	114
+	.byte	23
+	.byte	16
+	.byte	23
+	.byte	27
+	.byte	37
+	.byte	17
+	.byte	1
+	.byte	85
+	.byte	35
+	.byte	115
+	.byte	23
+	.byte	116
+	.byte	23
+	.ascii	"\214\001"
+	.byte	23
+	.byte	0
+	.byte	0
+	.byte	2
+	.byte	46
+	.byte	1
+	.byte	17
+	.byte	27
+	.byte	18
+	.byte	6
+	.byte	64
+	.byte	24
+	.byte	122
+	.byte	25
+	.byte	3
+	.byte	37
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	39
+	.byte	25
+	.byte	73
+	.byte	19
+	.byte	63
+	.byte	25
+	.byte	0
+	.byte	0
+	.byte	3
+	.byte	5
+	.byte	0
+	.byte	2
+	.byte	34
+	.byte	3
+	.byte	37
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	73
+	.byte	19
+	.byte	0
+	.byte	0
+	.byte	4
+	.byte	5
+	.byte	0
+	.byte	2
+	.byte	24
+	.byte	3
+	.byte	37
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	73
+	.byte	19
+	.byte	0
+	.byte	0
+	.byte	5
+	.byte	52
+	.byte	0
+	.byte	2
+	.byte	34
+	.byte	3
+	.byte	37
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	73
+	.byte	19
+	.byte	0
+	.byte	0
+	.byte	6
+	.byte	36
+	.byte	0
+	.byte	3
+	.byte	37
+	.byte	62
+	.byte	11
+	.byte	11
+	.byte	11
+	.byte	0
+	.byte	0
+	.byte	0
+	.section	.debug_info,"",%progbits
+.Lcu_begin0:
+	.long	.Ldebug_info_end0-.Ldebug_info_start0
+.Ldebug_info_start0:
+	.short	5
+	.byte	1
+	.byte	4
+	.long	.debug_abbrev
+	.byte	1
+	.byte	0
+	.short	12
+	.byte	1
+	.long	.Lstr_offsets_base0
+	.long	.Lline_table_start0
+	.byte	2
+	.long	0
+	.byte	0
+	.long	.Laddr_table_base0
+	.long	.Lrnglists_table_base0
+	.long	.Lloclists_table_base0
+	.byte	2
+	.byte	0
+	.long	.Lfunc_end0-.Lfunc_begin0
+	.byte	1
+	.byte	91
+
+	.byte	3
+	.byte	1
+	.byte	1
+
+	.long	132
+
+	.byte	3
+	.byte	0
+	.byte	6
+	.byte	1
+	.byte	1
+	.long	132
+	.byte	4
+	.byte	1
+	.byte	81
+	.byte	7
+	.byte	1
+	.byte	1
+	.long	132
+	.byte	4
+	.byte	1
+	.byte	82
+	.byte	8
+	.byte	1
+	.byte	1
+	.long	132
+	.byte	5
+	.byte	1
+	.byte	9
+	.byte	1
+	.byte	2
+	.long	132
+	.byte	5
+	.byte	2
+	.byte	10
+	.byte	1
+	.byte	3
+	.long	132
+	.byte	0
+	.byte	2
+	.byte	1
+	.long	.Lfunc_end1-.Lfunc_begin1
+	.byte	1
+	.byte	91
+
+	.byte	5
+	.byte	1
+	.byte	7
+
+	.long	132
+
+	.byte	4
+	.byte	1
+	.byte	80
+	.byte	6
+	.byte	1
+	.byte	7
+	.long	132
+	.byte	0
+	.byte	6
+	.byte	4
+	.byte	5
+	.byte	4
+	.byte	0
+.Ldebug_info_end0:
+	.section	.debug_rnglists,"",%progbits
+	.long	.Ldebug_rnglist_table_end0-.Ldebug_rnglist_table_start0
+.Ldebug_rnglist_table_start0:
+	.short	5
+	.byte	4
+	.byte	0
+	.long	1
+.Lrnglists_table_base0:
+	.long	.Ldebug_ranges0-.Lrnglists_table_base0
+.Ldebug_ranges0:
+	.byte	3
+	.byte	0
+	.uleb128 .Lfunc_end0-.Lfunc_begin0
+	.byte	3
+	.byte	1
+	.uleb128 .Lfunc_end1-.Lfunc_begin1
+	.byte	0
+.Ldebug_rnglist_table_end0:
+	.section	.debug_addr,"",%progbits
+	.long	.Ldebug_addr_end0-.Ldebug_addr_start0
+.Ldebug_addr_start0:
+	.short	5
+	.byte	4
+	.byte	0
+.Laddr_table_base0:
+	.long	.Lfunc_begin0
+	.long	.Lfunc_begin1
+.Ldebug_addr_end0:
+	.ident	"clang version 10.0.0 (git@github.com:llvm/llvm-project.git e73f78acd34360f7450b81167d9dc858ccddc262)"
+	.section	".note.GNU-stack","",%progbits
+	.addrsig
+	.eabi_attribute	30, 1
+	.section	.debug_line,"",%progbits
+.Lline_table_start0:
diff --git a/llvm/test/tools/llvm-objdump/ELF/ARM/debug-vars-dwarf5.s b/llvm/test/tools/llvm-objdump/ELF/ARM/debug-vars-dwarf5.s
new file mode 100644
index 0000000000000..8a63a4ee9ec09
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/ELF/ARM/debug-vars-dwarf5.s
@@ -0,0 +1,382 @@
+## Check that the --debug-vars option works for simple register locations, when
+## using DWARF5 debug info, with multiple functions in one section.
+
+## Generated with this compile command, with the source code in Inputs/debug.c:
+## clang --target=arm--none-eabi -march=armv7-a -c debug.c -O1 -gdwarf-3 -S -o -
+
+# RUN: llvm-mc -triple armv8a--none-eabi < %s -filetype=obj --dwarf-version=5 | \
+# RUN:     llvm-objdump - -d --debug-vars --no-show-raw-insn | \
+# RUN:     FileCheck %s
+
+# CHECK: Disassembly of section .text:
+# CHECK-EMPTY:
+# CHECK-NEXT: 00000000 <foo>:
+# CHECK-NEXT:                                                                   ┠─ a = R0
+# CHECK-NEXT:                                                                   ┃ ┠─ b = R1
+# CHECK-NEXT:                                                                   ┃ ┃ ┠─ c = R2
+# CHECK-NEXT:                                                                   ┃ ┃ ┃ ┌─ x = R0
+# CHECK-NEXT:        0:       add     r0, r1, r0                                ┻ ┃ ┃ ╈
+# CHECK-NEXT:                                                                   ┌─ y = R0
+# CHECK-NEXT:        4:       add     r0, r0, r2                                ╈ ┃ ┃ ┻
+# CHECK-NEXT:        8:       bx      lr                                        ┻ ┻ ┻
+# CHECK-EMPTY:
+# CHECK-NEXT: 0000000c <bar>:
+# CHECK-NEXT:                                                                   ┠─ a = R0
+# CHECK-NEXT:        c:       add     r0, r0, #1                                ┃
+# CHECK-NEXT:       10:       bx      lr                                        ┻
+
+	.text
+	.syntax unified
+	.eabi_attribute	67, "2.09"
+	.eabi_attribute	6, 10
+	.eabi_attribute	7, 65
+	.eabi_attribute	8, 1
+	.eabi_attribute	9, 2
+	.fpu	neon
+	.eabi_attribute	34, 0
+	.eabi_attribute	17, 1
+	.eabi_attribute	20, 1
+	.eabi_attribute	21, 1
+	.eabi_attribute	23, 3
+	.eabi_attribute	24, 1
+	.eabi_attribute	25, 1
+	.eabi_attribute	38, 1
+	.eabi_attribute	18, 4
+	.eabi_attribute	26, 2
+	.eabi_attribute	14, 0
+	.file	"debug.c"
+	.globl	foo
+	.p2align	2
+	.type	foo,%function
+	.code	32
+foo:
+.Lfunc_begin0:
+	.file	0 "/work/scratch" "/work/llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c" md5 0x07374f01ab24ec7c07db73bc13bd778e
+	.file	1 "/work" "llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c" md5 0x07374f01ab24ec7c07db73bc13bd778e
+	.loc	1 1 0
+	.fnstart
+	.cfi_sections .debug_frame
+	.cfi_startproc
+	.loc	1 2 13 prologue_end
+	add	r0, r1, r0
+.Ltmp0:
+	.loc	1 3 13
+	add	r0, r0, r2
+.Ltmp1:
+	.loc	1 4 3
+	bx	lr
+.Ltmp2:
+.Lfunc_end0:
+	.size	foo, .Lfunc_end0-foo
+	.cfi_endproc
+	.cantunwind
+	.fnend
+
+	.globl	bar
+	.p2align	2
+	.type	bar,%function
+	.code	32
+bar:
+.Lfunc_begin1:
+	.loc	1 7 0
+	.fnstart
+	.cfi_startproc
+	.loc	1 8 4 prologue_end
+	add	r0, r0, #1
+.Ltmp3:
+	.loc	1 9 3
+	bx	lr
+.Ltmp4:
+.Lfunc_end1:
+	.size	bar, .Lfunc_end1-bar
+	.cfi_endproc
+	.cantunwind
+	.fnend
+
+	.section	.debug_str_offsets,"",%progbits
+	.long	48
+	.short	5
+	.short	0
+.Lstr_offsets_base0:
+	.section	.debug_str,"MS",%progbits,1
+.Linfo_string0:
+	.asciz	"clang version 10.0.0 (git@github.com:llvm/llvm-project.git e73f78acd34360f7450b81167d9dc858ccddc262)"
+.Linfo_string1:
+	.asciz	"/work/llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c"
+.Linfo_string2:
+	.asciz	"/work/scratch"
+.Linfo_string3:
+	.asciz	"foo"
+.Linfo_string4:
+	.asciz	"int"
+.Linfo_string5:
+	.asciz	"bar"
+.Linfo_string6:
+	.asciz	"a"
+.Linfo_string7:
+	.asciz	"b"
+.Linfo_string8:
+	.asciz	"c"
+.Linfo_string9:
+	.asciz	"x"
+.Linfo_string10:
+	.asciz	"y"
+	.section	.debug_str_offsets,"",%progbits
+	.long	.Linfo_string0
+	.long	.Linfo_string1
+	.long	.Linfo_string2
+	.long	.Linfo_string3
+	.long	.Linfo_string4
+	.long	.Linfo_string5
+	.long	.Linfo_string6
+	.long	.Linfo_string7
+	.long	.Linfo_string8
+	.long	.Linfo_string9
+	.long	.Linfo_string10
+	.section	.debug_loclists,"",%progbits
+	.long	.Ldebug_loclist_table_end0-.Ldebug_loclist_table_start0
+.Ldebug_loclist_table_start0:
+	.short	5
+	.byte	4
+	.byte	0
+	.long	3
+.Lloclists_table_base0:
+	.long	.Ldebug_loc0-.Lloclists_table_base0
+	.long	.Ldebug_loc1-.Lloclists_table_base0
+	.long	.Ldebug_loc2-.Lloclists_table_base0
+.Ldebug_loc0:
+	.byte	4
+	.uleb128 .Lfunc_begin0-.Lfunc_begin0
+	.uleb128 .Ltmp0-.Lfunc_begin0
+	.byte	1
+	.byte	80
+	.byte	0
+.Ldebug_loc1:
+	.byte	4
+	.uleb128 .Ltmp0-.Lfunc_begin0
+	.uleb128 .Ltmp1-.Lfunc_begin0
+	.byte	1
+	.byte	80
+	.byte	0
+.Ldebug_loc2:
+	.byte	4
+	.uleb128 .Ltmp1-.Lfunc_begin0
+	.uleb128 .Lfunc_end0-.Lfunc_begin0
+	.byte	1
+	.byte	80
+	.byte	0
+.Ldebug_loclist_table_end0:
+	.section	.debug_abbrev,"",%progbits
+	.byte	1
+	.byte	17
+	.byte	1
+	.byte	37
+	.byte	37
+	.byte	19
+	.byte	5
+	.byte	3
+	.byte	37
+	.byte	114
+	.byte	23
+	.byte	16
+	.byte	23
+	.byte	27
+	.byte	37
+	.byte	17
+	.byte	27
+	.byte	18
+	.byte	6
+	.byte	115
+	.byte	23
+	.ascii	"\214\001"
+	.byte	23
+	.byte	0
+	.byte	0
+	.byte	2
+	.byte	46
+	.byte	1
+	.byte	17
+	.byte	27
+	.byte	18
+	.byte	6
+	.byte	64
+	.byte	24
+	.byte	122
+	.byte	25
+	.byte	3
+	.byte	37
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	39
+	.byte	25
+	.byte	73
+	.byte	19
+	.byte	63
+	.byte	25
+	.byte	0
+	.byte	0
+	.byte	3
+	.byte	5
+	.byte	0
+	.byte	2
+	.byte	34
+	.byte	3
+	.byte	37
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	73
+	.byte	19
+	.byte	0
+	.byte	0
+	.byte	4
+	.byte	5
+	.byte	0
+	.byte	2
+	.byte	24
+	.byte	3
+	.byte	37
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	73
+	.byte	19
+	.byte	0
+	.byte	0
+	.byte	5
+	.byte	52
+	.byte	0
+	.byte	2
+	.byte	34
+	.byte	3
+	.byte	37
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	73
+	.byte	19
+	.byte	0
+	.byte	0
+	.byte	6
+	.byte	36
+	.byte	0
+	.byte	3
+	.byte	37
+	.byte	62
+	.byte	11
+	.byte	11
+	.byte	11
+	.byte	0
+	.byte	0
+	.byte	0
+	.section	.debug_info,"",%progbits
+.Lcu_begin0:
+	.long	.Ldebug_info_end0-.Ldebug_info_start0
+.Ldebug_info_start0:
+	.short	5
+	.byte	1
+	.byte	4
+	.long	.debug_abbrev
+	.byte	1
+	.byte	0
+	.short	12
+	.byte	1
+	.long	.Lstr_offsets_base0
+	.long	.Lline_table_start0
+	.byte	2
+	.byte	0
+	.long	.Lfunc_end1-.Lfunc_begin0
+	.long	.Laddr_table_base0
+	.long	.Lloclists_table_base0
+	.byte	2
+	.byte	0
+	.long	.Lfunc_end0-.Lfunc_begin0
+	.byte	1
+	.byte	91
+
+	.byte	3
+	.byte	1
+	.byte	1
+
+	.long	128
+
+	.byte	3
+	.byte	0
+	.byte	6
+	.byte	1
+	.byte	1
+	.long	128
+	.byte	4
+	.byte	1
+	.byte	81
+	.byte	7
+	.byte	1
+	.byte	1
+	.long	128
+	.byte	4
+	.byte	1
+	.byte	82
+	.byte	8
+	.byte	1
+	.byte	1
+	.long	128
+	.byte	5
+	.byte	1
+	.byte	9
+	.byte	1
+	.byte	2
+	.long	128
+	.byte	5
+	.byte	2
+	.byte	10
+	.byte	1
+	.byte	3
+	.long	128
+	.byte	0
+	.byte	2
+	.byte	1
+	.long	.Lfunc_end1-.Lfunc_begin1
+	.byte	1
+	.byte	91
+
+	.byte	5
+	.byte	1
+	.byte	7
+
+	.long	128
+
+	.byte	4
+	.byte	1
+	.byte	80
+	.byte	6
+	.byte	1
+	.byte	7
+	.long	128
+	.byte	0
+	.byte	6
+	.byte	4
+	.byte	5
+	.byte	4
+	.byte	0
+.Ldebug_info_end0:
+	.section	.debug_addr,"",%progbits
+	.long	.Ldebug_addr_end0-.Ldebug_addr_start0
+.Ldebug_addr_start0:
+	.short	5
+	.byte	4
+	.byte	0
+.Laddr_table_base0:
+	.long	.Lfunc_begin0
+	.long	.Lfunc_begin1
+.Ldebug_addr_end0:
+	.ident	"clang version 10.0.0 (git@github.com:llvm/llvm-project.git e73f78acd34360f7450b81167d9dc858ccddc262)"
+	.section	".note.GNU-stack","",%progbits
+	.addrsig
+	.eabi_attribute	30, 1
+	.section	.debug_line,"",%progbits
+.Lline_table_start0:
diff --git a/llvm/test/tools/llvm-objdump/ELF/ARM/debug-vars-wide-chars.s b/llvm/test/tools/llvm-objdump/ELF/ARM/debug-vars-wide-chars.s
new file mode 100644
index 0000000000000..2573dc63513e2
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/ELF/ARM/debug-vars-wide-chars.s
@@ -0,0 +1,232 @@
+# RUN: mkdir -p %t/a
+# RUN: cp %p/Inputs/wide-char.c %t/a/wide-char.c
+# RUN: sed -e "s,SRC_COMPDIR,%/t/a,g" %s > %t.s
+# RUN: llvm-mc -triple armv8a--none-eabi < %t.s -filetype=obj | \
+# RUN:     llvm-objdump - -d --debug-vars --source | \
+# RUN:     FileCheck %s --strict-whitespace
+
+## The Chinese character in the source does not print correctly on Windows.
+# UNSUPPORTED: system-windows
+
+## Check that the --debug-vars option correctly aligns the variable display when
+## the source code (printed by the -S option) includes East Asian wide
+## characters.
+
+# CHECK: 00000000 <foo>:
+# CHECK-NEXT: ;   return *喵;                                                 ┠─ 喵 = R0
+# CHECK-NEXT:        0: 00 00 90 e5  	ldr	r0, [r0]                        ┻   
+# CHECK-NEXT:        4: 1e ff 2f e1  	bx	lr                                  
+
+	.text
+	.syntax unified
+	.eabi_attribute	67, "2.09"
+	.eabi_attribute	6, 10
+	.eabi_attribute	7, 65
+	.eabi_attribute	8, 1
+	.eabi_attribute	9, 2
+	.fpu	vfpv3
+	.eabi_attribute	34, 0
+	.eabi_attribute	17, 1
+	.eabi_attribute	20, 1
+	.eabi_attribute	21, 1
+	.eabi_attribute	23, 3
+	.eabi_attribute	24, 1
+	.eabi_attribute	25, 1
+	.eabi_attribute	38, 1
+	.eabi_attribute	18, 4
+	.eabi_attribute	26, 2
+	.eabi_attribute	14, 0
+	.file	"wide.c"
+	.globl	foo
+	.p2align	2
+	.type	foo,%function
+	.code	32
+foo:
+.Lfunc_begin0:
+	.file	1 "SRC_COMPDIR/wide-char.c"
+	.loc	1 1 0
+	.fnstart
+	.cfi_sections .debug_frame
+	.cfi_startproc
+	.loc	1 2 10 prologue_end
+	ldr	r0, [r0]
+.Ltmp0:
+	.loc	1 2 3 is_stmt 0
+	bx	lr
+.Ltmp1:
+.Lfunc_end0:
+	.size	foo, .Lfunc_end0-foo
+	.cfi_endproc
+	.cantunwind
+	.fnend
+
+	.section	.debug_str,"MS",%progbits,1
+.Linfo_string0:
+	.asciz	"clang version 6.0.0-1ubuntu2 (tags/RELEASE_600/final)"
+.Linfo_string1:
+	.asciz	"wide-char.c"
+.Linfo_string2:
+	.asciz	"SRC_COMPDIR"
+.Linfo_string3:
+	.asciz	"foo"
+.Linfo_string4:
+	.asciz	"int"
+.Linfo_string5:
+	.asciz	"\345\226\265"
+	.section	.debug_loc,"",%progbits
+.Ldebug_loc0:
+	.long	.Lfunc_begin0-.Lfunc_begin0
+	.long	.Ltmp0-.Lfunc_begin0
+	.short	1
+	.byte	80
+	.long	0
+	.long	0
+	.section	.debug_abbrev,"",%progbits
+	.byte	1
+	.byte	17
+	.byte	1
+	.byte	37
+	.byte	14
+	.byte	19
+	.byte	5
+	.byte	3
+	.byte	14
+	.byte	16
+	.byte	23
+	.byte	27
+	.byte	14
+	.ascii	"\264B"
+	.byte	25
+	.byte	17
+	.byte	1
+	.byte	18
+	.byte	6
+	.byte	0
+	.byte	0
+	.byte	2
+	.byte	46
+	.byte	1
+	.byte	17
+	.byte	1
+	.byte	18
+	.byte	6
+	.byte	64
+	.byte	24
+	.byte	3
+	.byte	14
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	39
+	.byte	25
+	.byte	73
+	.byte	19
+	.byte	63
+	.byte	25
+	.byte	0
+	.byte	0
+	.byte	3
+	.byte	5
+	.byte	0
+	.byte	2
+	.byte	23
+	.byte	3
+	.byte	14
+	.byte	58
+	.byte	11
+	.byte	59
+	.byte	11
+	.byte	73
+	.byte	19
+	.byte	0
+	.byte	0
+	.byte	4
+	.byte	36
+	.byte	0
+	.byte	3
+	.byte	14
+	.byte	62
+	.byte	11
+	.byte	11
+	.byte	11
+	.byte	0
+	.byte	0
+	.byte	5
+	.byte	15
+	.byte	0
+	.byte	73
+	.byte	19
+	.byte	0
+	.byte	0
+	.byte	0
+	.section	.debug_info,"",%progbits
+.Lcu_begin0:
+	.long	84
+	.short	4
+	.long	.debug_abbrev
+	.byte	4
+	.byte	1
+	.long	.Linfo_string0
+	.short	12
+	.long	.Linfo_string1
+	.long	.Lline_table_start0
+	.long	.Linfo_string2
+
+	.long	.Lfunc_begin0
+	.long	.Lfunc_end0-.Lfunc_begin0
+	.byte	2
+	.long	.Lfunc_begin0
+	.long	.Lfunc_end0-.Lfunc_begin0
+	.byte	1
+	.byte	91
+	.long	.Linfo_string3
+	.byte	1
+	.byte	1
+
+	.long	75
+
+	.byte	3
+	.long	.Ldebug_loc0
+	.long	.Linfo_string5
+	.byte	1
+	.byte	1
+	.long	82
+	.byte	0
+	.byte	4
+	.long	.Linfo_string4
+	.byte	5
+	.byte	4
+	.byte	5
+	.long	75
+	.byte	0
+	.section	.debug_ranges,"",%progbits
+	.section	.debug_macinfo,"",%progbits
+.Lcu_macro_begin0:
+	.byte	0
+	.section	.debug_pubnames,"",%progbits
+	.long	.LpubNames_end0-.LpubNames_begin0
+.LpubNames_begin0:
+	.short	2
+	.long	.Lcu_begin0
+	.long	88
+	.long	38
+	.asciz	"foo"
+	.long	0
+.LpubNames_end0:
+	.section	.debug_pubtypes,"",%progbits
+	.long	.LpubTypes_end0-.LpubTypes_begin0
+.LpubTypes_begin0:
+	.short	2
+	.long	.Lcu_begin0
+	.long	88
+	.long	75
+	.asciz	"int"
+	.long	0
+.LpubTypes_end0:
+
+	.ident	"clang version 6.0.0-1ubuntu2 (tags/RELEASE_600/final)"
+	.section	".note.GNU-stack","",%progbits
+	.eabi_attribute	30, 1
+	.section	.debug_line,"",%progbits
+.Lline_table_start0:
diff --git a/llvm/test/tools/llvm-objdump/ELF/PowerPC/debug-vars.s b/llvm/test/tools/llvm-objdump/ELF/PowerPC/debug-vars.s
new file mode 100644
index 0000000000000..6c3d326843c73
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/ELF/PowerPC/debug-vars.s
@@ -0,0 +1,372 @@
+## Check that the --debug-vars option works for simple register locations, when
+## using DWARF4 debug info, with multiple functions in one section.
+
+## Generated with this compile command and source code:
+## clang --target=arm--none-eabi -march=armv7-a -c debug.c -O1 -gdwarf-3 -S -o -
+
+## clang --target=powerpc64-unknown-linux -c debug.c -O1 -S -o -
+
+## int foo(int a, int b, int c) {
+##   int x = a + b;
+##   int y = x + c;
+##   return y;
+## }
+##
+## int bar(int a) {
+##   a++;
+##   return a;
+## }
+
+# RUN: llvm-mc -triple powerpc64-unknown-linux < %s -filetype=obj | \
+# RUN:     llvm-objdump - -d --debug-vars --no-show-raw-insn | \
+# RUN:     FileCheck %s
+
+# CHECK: Disassembly of section .text:
+# CHECK-EMPTY:
+# CHECK-NEXT: 0000000000000000 <.text>:
+# CHECK-NEXT:                                                                   ┠─ a = S3
+# CHECK-NEXT:                                                                   ┃ ┠─ b = S4
+# CHECK-NEXT:                                                                   ┃ ┃ ┠─ c = S5
+# CHECK-NEXT:                                                                   ┃ ┃ ┃ ┌─ x = S3
+# CHECK-NEXT:        0:       add 3, 4, 3                                       ┻ ┃ ┃ ╈
+# CHECK-NEXT:                                                                   ┌─ y = S3
+# CHECK-NEXT:        4:       add 3, 3, 5                                       ╈ ┃ ┃ ┻
+# CHECK-NEXT:        8:       extsw 3, 3                                        ┻ ┃ ┃
+# CHECK-NEXT:        c:       blr                                                 ┃ ┃
+# CHECK-NEXT:                 ...
+# CHECK-NEXT:                                                                   ┠─ a = S3
+# CHECK-NEXT:       1c:       addi 3, 3, 1                                      ┃
+# CHECK-NEXT:       20:       extsw 3, 3                                        ┻
+# CHECK-NEXT:       24:       blr
+# CHECK-NEXT:                 ...
+
+	.text
+	.file	"debug.c"
+	.globl	foo                     # -- Begin function foo
+	.p2align	2
+	.type	foo,@function
+	.section	.opd,"aw",@progbits
+foo:                                    # @foo
+	.p2align	3
+	.quad	.Lfunc_begin0
+	.quad	.TOC.@tocbase
+	.quad	0
+	.text
+.Lfunc_begin0:
+	.file	1 "/work" "llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c"
+	.loc	1 1 0                   # llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c:1:0
+	.cfi_sections .debug_frame
+	.cfi_startproc
+# %bb.0:                                # %entry
+	#DEBUG_VALUE: foo:a <- $x3
+	#DEBUG_VALUE: foo:a <- $r3
+	#DEBUG_VALUE: foo:b <- $x4
+	#DEBUG_VALUE: foo:b <- $x4
+	#DEBUG_VALUE: foo:b <- $r4
+	#DEBUG_VALUE: foo:c <- $x5
+	#DEBUG_VALUE: foo:c <- $x5
+	#DEBUG_VALUE: foo:c <- $r5
+	.loc	1 2 13 prologue_end     # llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c:2:13
+	add 3, 4, 3
+.Ltmp0:
+	#DEBUG_VALUE: foo:x <- $r3
+	.loc	1 3 13                  # llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c:3:13
+	add 3, 3, 5
+.Ltmp1:
+	#DEBUG_VALUE: foo:y <- $r3
+	.loc	1 4 3                   # llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c:4:3
+	extsw 3, 3
+.Ltmp2:
+	blr
+.Ltmp3:
+	.long	0
+	.quad	0
+.Lfunc_end0:
+	.size	foo, .Lfunc_end0-.Lfunc_begin0
+	.cfi_endproc
+                                        # -- End function
+	.globl	bar                     # -- Begin function bar
+	.p2align	2
+	.type	bar,@function
+	.section	.opd,"aw",@progbits
+bar:                                    # @bar
+	.p2align	3
+	.quad	.Lfunc_begin1
+	.quad	.TOC.@tocbase
+	.quad	0
+	.text
+.Lfunc_begin1:
+	.loc	1 7 0                   # llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c:7:0
+	.cfi_startproc
+# %bb.0:                                # %entry
+	#DEBUG_VALUE: bar:a <- $x3
+	#DEBUG_VALUE: bar:a <- $r3
+	.loc	1 8 4 prologue_end      # llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c:8:4
+	addi 3, 3, 1
+.Ltmp4:
+	#DEBUG_VALUE: bar:a <- $r3
+	.loc	1 9 3                   # llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c:9:3
+	extsw 3, 3
+.Ltmp5:
+	blr
+.Ltmp6:
+	.long	0
+	.quad	0
+.Lfunc_end1:
+	.size	bar, .Lfunc_end1-.Lfunc_begin1
+	.cfi_endproc
+                                        # -- End function
+	.section	.debug_str,"MS",@progbits,1
+.Linfo_string0:
+	.asciz	"clang version 10.0.0 (git@github.com:llvm/llvm-project.git e73f78acd34360f7450b81167d9dc858ccddc262)" # string offset=0
+.Linfo_string1:
+	.asciz	"/work/llvm/src/llvm/test/tools/llvm-objdump/ARM/Inputs/debug.c" # string offset=101
+.Linfo_string2:
+	.asciz	"/work/scratch"         # string offset=164
+.Linfo_string3:
+	.asciz	"foo"                   # string offset=178
+.Linfo_string4:
+	.asciz	"int"                   # string offset=182
+.Linfo_string5:
+	.asciz	"bar"                   # string offset=186
+.Linfo_string6:
+	.asciz	"a"                     # string offset=190
+.Linfo_string7:
+	.asciz	"b"                     # string offset=192
+.Linfo_string8:
+	.asciz	"c"                     # string offset=194
+.Linfo_string9:
+	.asciz	"x"                     # string offset=196
+.Linfo_string10:
+	.asciz	"y"                     # string offset=198
+	.section	.debug_loc,"",@progbits
+.Ldebug_loc0:
+	.quad	.Lfunc_begin0-.Lfunc_begin0
+	.quad	.Ltmp0-.Lfunc_begin0
+	.short	3                       # Loc expr size
+	.byte	144                     # super-register DW_OP_regx
+	.byte	179                     # 1203
+	.byte	9                       #
+	.quad	0
+	.quad	0
+.Ldebug_loc1:
+	.quad	.Ltmp0-.Lfunc_begin0
+	.quad	.Ltmp1-.Lfunc_begin0
+	.short	3                       # Loc expr size
+	.byte	144                     # super-register DW_OP_regx
+	.byte	179                     # 1203
+	.byte	9                       #
+	.quad	0
+	.quad	0
+.Ldebug_loc2:
+	.quad	.Ltmp1-.Lfunc_begin0
+	.quad	.Ltmp2-.Lfunc_begin0
+	.short	3                       # Loc expr size
+	.byte	144                     # super-register DW_OP_regx
+	.byte	179                     # 1203
+	.byte	9                       #
+	.quad	0
+	.quad	0
+.Ldebug_loc3:
+	.quad	.Lfunc_begin1-.Lfunc_begin0
+	.quad	.Ltmp5-.Lfunc_begin0
+	.short	3                       # Loc expr size
+	.byte	144                     # super-register DW_OP_regx
+	.byte	179                     # 1203
+	.byte	9                       #
+	.quad	0
+	.quad	0
+	.section	.debug_abbrev,"",@progbits
+	.byte	1                       # Abbreviation Code
+	.byte	17                      # DW_TAG_compile_unit
+	.byte	1                       # DW_CHILDREN_yes
+	.byte	37                      # DW_AT_producer
+	.byte	14                      # DW_FORM_strp
+	.byte	19                      # DW_AT_language
+	.byte	5                       # DW_FORM_data2
+	.byte	3                       # DW_AT_name
+	.byte	14                      # DW_FORM_strp
+	.byte	16                      # DW_AT_stmt_list
+	.byte	23                      # DW_FORM_sec_offset
+	.byte	27                      # DW_AT_comp_dir
+	.byte	14                      # DW_FORM_strp
+	.byte	17                      # DW_AT_low_pc
+	.byte	1                       # DW_FORM_addr
+	.byte	18                      # DW_AT_high_pc
+	.byte	6                       # DW_FORM_data4
+	.byte	0                       # EOM(1)
+	.byte	0                       # EOM(2)
+	.byte	2                       # Abbreviation Code
+	.byte	46                      # DW_TAG_subprogram
+	.byte	1                       # DW_CHILDREN_yes
+	.byte	17                      # DW_AT_low_pc
+	.byte	1                       # DW_FORM_addr
+	.byte	18                      # DW_AT_high_pc
+	.byte	6                       # DW_FORM_data4
+	.byte	64                      # DW_AT_frame_base
+	.byte	24                      # DW_FORM_exprloc
+	.ascii	"\227B"                 # DW_AT_GNU_all_call_sites
+	.byte	25                      # DW_FORM_flag_present
+	.byte	3                       # DW_AT_name
+	.byte	14                      # DW_FORM_strp
+	.byte	58                      # DW_AT_decl_file
+	.byte	11                      # DW_FORM_data1
+	.byte	59                      # DW_AT_decl_line
+	.byte	11                      # DW_FORM_data1
+	.byte	39                      # DW_AT_prototyped
+	.byte	25                      # DW_FORM_flag_present
+	.byte	73                      # DW_AT_type
+	.byte	19                      # DW_FORM_ref4
+	.byte	63                      # DW_AT_external
+	.byte	25                      # DW_FORM_flag_present
+	.byte	0                       # EOM(1)
+	.byte	0                       # EOM(2)
+	.byte	3                       # Abbreviation Code
+	.byte	5                       # DW_TAG_formal_parameter
+	.byte	0                       # DW_CHILDREN_no
+	.byte	2                       # DW_AT_location
+	.byte	23                      # DW_FORM_sec_offset
+	.byte	3                       # DW_AT_name
+	.byte	14                      # DW_FORM_strp
+	.byte	58                      # DW_AT_decl_file
+	.byte	11                      # DW_FORM_data1
+	.byte	59                      # DW_AT_decl_line
+	.byte	11                      # DW_FORM_data1
+	.byte	73                      # DW_AT_type
+	.byte	19                      # DW_FORM_ref4
+	.byte	0                       # EOM(1)
+	.byte	0                       # EOM(2)
+	.byte	4                       # Abbreviation Code
+	.byte	5                       # DW_TAG_formal_parameter
+	.byte	0                       # DW_CHILDREN_no
+	.byte	2                       # DW_AT_location
+	.byte	24                      # DW_FORM_exprloc
+	.byte	3                       # DW_AT_name
+	.byte	14                      # DW_FORM_strp
+	.byte	58                      # DW_AT_decl_file
+	.byte	11                      # DW_FORM_data1
+	.byte	59                      # DW_AT_decl_line
+	.byte	11                      # DW_FORM_data1
+	.byte	73                      # DW_AT_type
+	.byte	19                      # DW_FORM_ref4
+	.byte	0                       # EOM(1)
+	.byte	0                       # EOM(2)
+	.byte	5                       # Abbreviation Code
+	.byte	52                      # DW_TAG_variable
+	.byte	0                       # DW_CHILDREN_no
+	.byte	2                       # DW_AT_location
+	.byte	23                      # DW_FORM_sec_offset
+	.byte	3                       # DW_AT_name
+	.byte	14                      # DW_FORM_strp
+	.byte	58                      # DW_AT_decl_file
+	.byte	11                      # DW_FORM_data1
+	.byte	59                      # DW_AT_decl_line
+	.byte	11                      # DW_FORM_data1
+	.byte	73                      # DW_AT_type
+	.byte	19                      # DW_FORM_ref4
+	.byte	0                       # EOM(1)
+	.byte	0                       # EOM(2)
+	.byte	6                       # Abbreviation Code
+	.byte	36                      # DW_TAG_base_type
+	.byte	0                       # DW_CHILDREN_no
+	.byte	3                       # DW_AT_name
+	.byte	14                      # DW_FORM_strp
+	.byte	62                      # DW_AT_encoding
+	.byte	11                      # DW_FORM_data1
+	.byte	11                      # DW_AT_byte_size
+	.byte	11                      # DW_FORM_data1
+	.byte	0                       # EOM(1)
+	.byte	0                       # EOM(2)
+	.byte	0                       # EOM(3)
+	.section	.debug_info,"",@progbits
+.Lcu_begin0:
+	.long	.Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+	.short	4                       # DWARF version number
+	.long	.debug_abbrev           # Offset Into Abbrev. Section
+	.byte	8                       # Address Size (in bytes)
+	.byte	1                       # Abbrev [1] 0xb:0xb5 DW_TAG_compile_unit
+	.long	.Linfo_string0          # DW_AT_producer
+	.short	12                      # DW_AT_language
+	.long	.Linfo_string1          # DW_AT_name
+	.long	.Lline_table_start0     # DW_AT_stmt_list
+	.long	.Linfo_string2          # DW_AT_comp_dir
+	.quad	.Lfunc_begin0           # DW_AT_low_pc
+	.long	.Lfunc_end1-.Lfunc_begin0 # DW_AT_high_pc
+	.byte	2                       # Abbrev [2] 0x2a:0x65 DW_TAG_subprogram
+	.quad	.Lfunc_begin0           # DW_AT_low_pc
+	.long	.Lfunc_end0-.Lfunc_begin0 # DW_AT_high_pc
+	.byte	1                       # DW_AT_frame_base
+	.byte	81
+                                        # DW_AT_GNU_all_call_sites
+	.long	.Linfo_string3          # DW_AT_name
+	.byte	1                       # DW_AT_decl_file
+	.byte	1                       # DW_AT_decl_line
+                                        # DW_AT_prototyped
+	.long	184                     # DW_AT_type
+                                        # DW_AT_external
+	.byte	3                       # Abbrev [3] 0x43:0xf DW_TAG_formal_parameter
+	.long	.Ldebug_loc0            # DW_AT_location
+	.long	.Linfo_string6          # DW_AT_name
+	.byte	1                       # DW_AT_decl_file
+	.byte	1                       # DW_AT_decl_line
+	.long	184                     # DW_AT_type
+	.byte	4                       # Abbrev [4] 0x52:0xf DW_TAG_formal_parameter
+	.byte	3                       # DW_AT_location
+	.byte	144
+	.ascii	"\264\t"
+	.long	.Linfo_string7          # DW_AT_name
+	.byte	1                       # DW_AT_decl_file
+	.byte	1                       # DW_AT_decl_line
+	.long	184                     # DW_AT_type
+	.byte	4                       # Abbrev [4] 0x61:0xf DW_TAG_formal_parameter
+	.byte	3                       # DW_AT_location
+	.byte	144
+	.ascii	"\265\t"
+	.long	.Linfo_string8          # DW_AT_name
+	.byte	1                       # DW_AT_decl_file
+	.byte	1                       # DW_AT_decl_line
+	.long	184                     # DW_AT_type
+	.byte	5                       # Abbrev [5] 0x70:0xf DW_TAG_variable
+	.long	.Ldebug_loc1            # DW_AT_location
+	.long	.Linfo_string9          # DW_AT_name
+	.byte	1                       # DW_AT_decl_file
+	.byte	2                       # DW_AT_decl_line
+	.long	184                     # DW_AT_type
+	.byte	5                       # Abbrev [5] 0x7f:0xf DW_TAG_variable
+	.long	.Ldebug_loc2            # DW_AT_location
+	.long	.Linfo_string10         # DW_AT_name
+	.byte	1                       # DW_AT_decl_file
+	.byte	3                       # DW_AT_decl_line
+	.long	184                     # DW_AT_type
+	.byte	0                       # End Of Children Mark
+	.byte	2                       # Abbrev [2] 0x8f:0x29 DW_TAG_subprogram
+	.quad	.Lfunc_begin1           # DW_AT_low_pc
+	.long	.Lfunc_end1-.Lfunc_begin1 # DW_AT_high_pc
+	.byte	1                       # DW_AT_frame_base
+	.byte	81
+                                        # DW_AT_GNU_all_call_sites
+	.long	.Linfo_string5          # DW_AT_name
+	.byte	1                       # DW_AT_decl_file
+	.byte	7                       # DW_AT_decl_line
+                                        # DW_AT_prototyped
+	.long	184                     # DW_AT_type
+                                        # DW_AT_external
+	.byte	3                       # Abbrev [3] 0xa8:0xf DW_TAG_formal_parameter
+	.long	.Ldebug_loc3            # DW_AT_location
+	.long	.Linfo_string6          # DW_AT_name
+	.byte	1                       # DW_AT_decl_file
+	.byte	7                       # DW_AT_decl_line
+	.long	184                     # DW_AT_type
+	.byte	0                       # End Of Children Mark
+	.byte	6                       # Abbrev [6] 0xb8:0x7 DW_TAG_base_type
+	.long	.Linfo_string4          # DW_AT_name
+	.byte	5                       # DW_AT_encoding
+	.byte	4                       # DW_AT_byte_size
+	.byte	0                       # End Of Children Mark
+.Ldebug_info_end0:
+	.ident	"clang version 10.0.0 (git@github.com:llvm/llvm-project.git e73f78acd34360f7450b81167d9dc858ccddc262)"
+	.section	".note.GNU-stack","",@progbits
+	.addrsig
+	.section	.debug_line,"",@progbits
+.Lline_table_start0:
diff --git a/llvm/test/tools/llvm-objdump/RISCV/lit.local.cfg b/llvm/test/tools/llvm-objdump/ELF/RISCV/lit.local.cfg
similarity index 100%
rename from llvm/test/tools/llvm-objdump/RISCV/lit.local.cfg
rename to llvm/test/tools/llvm-objdump/ELF/RISCV/lit.local.cfg
diff --git a/llvm/test/tools/llvm-objdump/RISCV/unknown-arch-attr.test b/llvm/test/tools/llvm-objdump/ELF/RISCV/unknown-arch-attr.test
similarity index 100%
rename from llvm/test/tools/llvm-objdump/RISCV/unknown-arch-attr.test
rename to llvm/test/tools/llvm-objdump/ELF/RISCV/unknown-arch-attr.test
diff --git a/llvm/test/tools/llvm-objdump/PowerPC/lit.local.cfg b/llvm/test/tools/llvm-objdump/PowerPC/lit.local.cfg
deleted file mode 100644
index 091332439b186..0000000000000
--- a/llvm/test/tools/llvm-objdump/PowerPC/lit.local.cfg
+++ /dev/null
@@ -1,2 +0,0 @@
-if not 'PowerPC' in config.root.targets:
-    config.unsupported = True
diff --git a/llvm/test/tools/llvm-objdump/lit.local.cfg b/llvm/test/tools/llvm-objdump/lit.local.cfg
deleted file mode 100644
index c3e092a7ba896..0000000000000
--- a/llvm/test/tools/llvm-objdump/lit.local.cfg
+++ /dev/null
@@ -1 +0,0 @@
-config.suffixes = ['.test', '.ll', '.s', '.yaml', '.txt']
diff --git a/llvm/test/tools/llvm-profdata/Inputs/sample-hot-func-list.proftext b/llvm/test/tools/llvm-profdata/Inputs/sample-hot-func-list.proftext
new file mode 100644
index 0000000000000..6e0060371634b
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/Inputs/sample-hot-func-list.proftext
@@ -0,0 +1,41 @@
+_Z3bari:20301:1437
+ 1: 1437
+_Z3fooi:7711:610
+ 1: 610
+main:184019:0
+ 4: 534
+ 4.2: 534
+ 5: 1075
+ 5.1: 1075
+ 6: 2080
+ 7: 534
+ 9: 2300 _Z3bari:1471 _Z3fooi:631
+ 10: inline1:1000
+  1: 1000
+ 10: inline2:2000
+  1: 2000
+_Z3bazi:20305:1000
+ 1: 1000
+Func1:1523:169
+ 1: 169
+ 7: 563
+Func2:17043:1594
+ 1: 1594
+ 3: 1594
+ 6: 2009 Func1:150 Func3:1789
+ 13: 3105
+ 17: 3105
+ 19: 1594
+Func3:97401:3035
+ 1: 3035
+ 5: 7344
+ 9: 10640
+ 11: 10640
+ 15: 3035
+Func4:465:210
+ 1: 210
+Func5:6948:470
+ 1: 470
+ 3: 3507
+Func6:310:102
+ 1: 102
diff --git a/llvm/test/tools/llvm-profdata/sample-hot-func-list.test b/llvm/test/tools/llvm-profdata/sample-hot-func-list.test
new file mode 100644
index 0000000000000..2ea81363d49ca
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/sample-hot-func-list.test
@@ -0,0 +1,13 @@
+; RUN: llvm-profdata show --sample --hot-func-list %S/Inputs/sample-hot-func-list.proftext | FileCheck %s --match-full-lines --strict-whitespace
+
+;      CHECK:8 out of 10 functions with profile (80.00%) are considered hot functions (max sample >= 470).
+; CHECK-NEXT:355251 out of 356026 profile counts (99.78%) are from hot functions.
+; CHECK-NEXT: Total sample (%)       Max sample        Entry sample    Function name
+; CHECK-NEXT: 184019 (51.69%)        2300              534             main
+; CHECK-NEXT: 97401 (27.36%)         10640             3035            Func3
+; CHECK-NEXT: 20305 (5.70%)          1000              1000            _Z3bazi
+; CHECK-NEXT: 20301 (5.70%)          1437              1437            _Z3bari
+; CHECK-NEXT: 17043 (4.79%)          3105              1594            Func2
+; CHECK-NEXT: 7711 (2.17%)           610               610             _Z3fooi
+; CHECK-NEXT: 6948 (1.95%)           3507              470             Func5
+; CHECK-NEXT: 1523 (0.43%)           563               169             Func1
diff --git a/llvm/test/tools/llvm-rc/Inputs/language.rc b/llvm/test/tools/llvm-rc/Inputs/language.rc
new file mode 100644
index 0000000000000..caacdf0174ced
--- /dev/null
+++ b/llvm/test/tools/llvm-rc/Inputs/language.rc
@@ -0,0 +1,15 @@
+1 MENU
+LANGUAGE 1, 2
+{
+  MENUITEM "foo", 1
+}
+
+2 MENU {
+  MENUITEM "bar", 2
+}
+
+LANGUAGE 3, 4
+
+3 MENU {
+  MENUITEM "baz", 3
+}
diff --git a/llvm/test/tools/llvm-rc/language.test b/llvm/test/tools/llvm-rc/language.test
new file mode 100644
index 0000000000000..e764cde05e7be
--- /dev/null
+++ b/llvm/test/tools/llvm-rc/language.test
@@ -0,0 +1,19 @@
+; RUN: llvm-rc /l 40A /FO %t.res -- %p/Inputs/language.rc
+; RUN: llvm-readobj %t.res | FileCheck %s
+; RUN: llvm-rc /l40A /FO %t.res -- %p/Inputs/language.rc
+; RUN: llvm-readobj %t.res | FileCheck %s
+
+; CHECK:      Resource name (int): 1
+; CHECK-NEXT: Data version:
+; CHECK-NEXT: Memory flags:
+; CHECK-NEXT: Language ID: 2049
+
+; CHECK:      Resource name (int): 2
+; CHECK-NEXT: Data version:
+; CHECK-NEXT: Memory flags:
+; CHECK-NEXT: Language ID: 1034
+
+; CHECK:      Resource name (int): 3
+; CHECK-NEXT: Data version:
+; CHECK-NEXT: Memory flags:
+; CHECK-NEXT: Language ID: 4099
diff --git a/llvm/test/tools/llvm-readobj/COFF/call-graph-profile-err.s b/llvm/test/tools/llvm-readobj/COFF/call-graph-profile-err.s
new file mode 100644
index 0000000000000..6ba01f6c2fa1a
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/COFF/call-graph-profile-err.s
@@ -0,0 +1,23 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s -o %t
+# RUN: not llvm-readobj %t --cg-profile 2>&1 | FileCheck --check-prefix=ERR %s
+
+## In order to use --cg-profile option, the section ".llvm.call-graph-profile" 
+## should have two 4-byte fields representing the indexes of two symbols and
+## one 8-byte fields representing the weight from first symbol to second 
+## symbol.
+## The section in this test case has 9 bytes of data, so it's malformed.
+
+# ERR: error: '{{.*}}': Stream Error: The stream is too short to perform the requested operation.
+
+.section .test
+a:
+b:
+c:
+d:
+e:
+
+.section ".llvm.call-graph-profile"
+    .long 10    ## Symbol index of a.
+    .long 11    ## Symbol index of b.
+    .byte 32    ## Weight from a to b. It is an error, since it should have a length of 8 bytes.
diff --git a/llvm/test/tools/llvm-readobj/COFF/call-graph-profile.s b/llvm/test/tools/llvm-readobj/COFF/call-graph-profile.s
new file mode 100644
index 0000000000000..1a4294255bf6a
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/COFF/call-graph-profile.s
@@ -0,0 +1,41 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s -o %t
+# RUN: llvm-readobj %t --cg-profile | FileCheck %s
+
+# CHECK:      CGProfile [
+# CHECK-NEXT:   CGProfileEntry {
+# CHECK-NEXT:     From: a (10)
+# CHECK-NEXT:     To: b (11)
+# CHECK-NEXT:     Weight: 32
+# CHECK-NEXT:   }
+# CHECK-NEXT:   CGProfileEntry {
+# CHECK-NEXT:     From: c (12)
+# CHECK-NEXT:     To: a (10)
+# CHECK-NEXT:     Weight: 11
+# CHECK-NEXT:   }
+# CHECK-NEXT:   CGProfileEntry {
+# CHECK-NEXT:     From: d (13)
+# CHECK-NEXT:     To: e (14)
+# CHECK-NEXT:     Weight: 20
+# CHECK-NEXT:   }
+# CHECK-NEXT: ]
+
+.section .test
+a:
+b:
+c:
+d:
+e:
+
+.section ".llvm.call-graph-profile"
+    .long 10    ## Symbol index of a.
+    .long 11    ## Symbol index of b.
+    .quad 32    ## Weight from a to b.
+
+    .long 12    ## Symbol index of c.
+    .long 10    ## Symbol index of a.
+    .quad 11    ## Weight from c to a.
+
+    .long 13    ## Symbol index of d.
+    .long 14    ## Symbol index of e.
+    .quad 20    ## Weight from d to e.
diff --git a/llvm/test/tools/llvm-readobj/COFF/lit.local.cfg b/llvm/test/tools/llvm-readobj/COFF/lit.local.cfg
deleted file mode 100644
index 38f335368f17e..0000000000000
--- a/llvm/test/tools/llvm-readobj/COFF/lit.local.cfg
+++ /dev/null
@@ -1 +0,0 @@
-config.suffixes.add('.yaml')
diff --git a/llvm/test/tools/llvm-readobj/ELF/addrsig.test b/llvm/test/tools/llvm-readobj/ELF/addrsig.test
index 1234a42a839a8..c4793aae7b735 100644
--- a/llvm/test/tools/llvm-readobj/ELF/addrsig.test
+++ b/llvm/test/tools/llvm-readobj/ELF/addrsig.test
@@ -58,7 +58,7 @@ Sections:
 
 # INVALID-INDEX:      Addrsig [
 # INVALID-INDEX-NEXT:   Sym: foo (1)
-# INVALID-INDEX-NEXT:   warning: '[[FILE]]': unable to get symbol from section [index 2]: invalid symbol index (255)
+# INVALID-INDEX-NEXT:   warning: '[[FILE]]': unable to read the name of symbol with index 255: unable to get symbol from section [index 2]: invalid symbol index (255)
 # INVALID-INDEX-NEXT:   Sym: <?> (255)
 # INVALID-INDEX-NEXT:   Sym: bar (2)
 # INVALID-INDEX-NEXT: ]
diff --git a/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test b/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test
index e13492e07cf15..9142fc65a025f 100644
--- a/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test
+++ b/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test
@@ -463,3 +463,64 @@ ProgramHeaders:
     Sections:
       - Section: .rela.dyn
       - Section: .dynamic
+
+## Show that when we have both REL and RELA relocations, we dump both sets.
+# RUN: yaml2obj --docnum=13 %s -o %t13
+# RUN: llvm-readobj --dyn-relocations %t13 2>&1 | FileCheck %s -DFILE=%t13 --check-prefix=BOTH-RELA-REL-LLVM
+# RUN: llvm-readelf --dyn-relocations %t13 2>&1 | FileCheck %s -DFILE=%t13 --check-prefix=BOTH-RELA-REL-GNU
+
+# BOTH-RELA-REL-LLVM:      Dynamic Relocations {
+# BOTH-RELA-REL-LLVM-NEXT:   0x1 R_X86_64_NONE - 0x0
+# BOTH-RELA-REL-LLVM-NEXT:   0x2 R_X86_64_NONE - 0x0
+# BOTH-RELA-REL-LLVM-NEXT: }
+
+# BOTH-RELA-REL-GNU:       'RELA' relocation section at offset 0x78 contains 24 bytes:
+# BOTH-RELA-REL-GNU-NEXT:    Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
+# BOTH-RELA-REL-GNU-NEXT:  0000000000000001  0000000000000000 R_X86_64_NONE                     0
+# BOTH-RELA-REL-GNU-EMPTY:
+# BOTH-RELA-REL-GNU:       'REL' relocation section at offset 0x90 contains 16 bytes:
+# BOTH-RELA-REL-GNU-NEXT:    Offset             Info             Type               Symbol's Value  Symbol's Name
+# BOTH-RELA-REL-GNU-NEXT:  0000000000000002  0000000000000000 R_X86_64_NONE
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name: .rela.dyn
+    Type: SHT_RELA
+    Relocations:
+      - Type:   R_X86_64_NONE
+        Offset: 0x1
+  - Name: .rel.dyn
+    Type: SHT_REL
+    Relocations:
+      - Type:   R_X86_64_NONE
+        Offset: 0x2
+  - Name: .dynamic
+    Type: SHT_DYNAMIC
+    Entries:
+      - Tag:   DT_RELA
+        Value: 0x0
+      - Tag:   DT_RELASZ
+        Value: 0x18
+      - Tag:   DT_RELAENT
+        Value: 0x18
+## 0x18 == offset of .rel.dyn == size of .rela.dyn section.
+      - Tag:   DT_REL
+        Value: 0x18
+      - Tag:   DT_RELSZ
+        Value: 0x10
+      - Tag:   DT_RELENT
+        Value: 0x10
+      - Tag:   DT_NULL
+        Value: 0x0
+DynamicSymbols: []
+ProgramHeaders:
+  - Type:  PT_LOAD
+    Sections:
+      - Section: .rela.dyn
+      - Section: .rel.dyn
+      - Section: .dynamic
diff --git a/llvm/test/tools/llvm-readobj/ELF/call-graph-profile.test b/llvm/test/tools/llvm-readobj/ELF/call-graph-profile.test
index dfd95ea46a06d..8ccc93cf426f0 100644
--- a/llvm/test/tools/llvm-readobj/ELF/call-graph-profile.test
+++ b/llvm/test/tools/llvm-readobj/ELF/call-graph-profile.test
@@ -37,6 +37,69 @@ Sections:
       - From:   bar
         To:     foo
         Weight: 98
+## 0x10 is the normal entry size for the SHT_LLVM_CALL_GRAPH_PROFILE section.
+    EntSize: [[ENTSIZE=0x10]]
 Symbols:
   - Name: foo
   - Name: bar
+
+## Check we report a warning when unable to get the content of the SHT_LLVM_CALL_GRAPH_PROFILE section.
+# RUN: yaml2obj %s -DENTSIZE=0xF -o %t2.o
+# RUN: llvm-readobj %t2.o --cg-profile 2>&1 | FileCheck %s -DFILE=%t2.o --check-prefix=LLVM-ERR
+# RUN: llvm-readelf %t2.o --cg-profile | FileCheck %s --check-prefix=GNU
+
+# LLVM-ERR:      CGProfile [
+# LLVM-ERR-NEXT: warning: '[[FILE]]': unable to dump the SHT_LLVM_CALL_GRAPH_PROFILE section: section [index 1] has an invalid sh_entsize: 15
+# LLVM-ERR-NEXT: ]
+
+## Check we report a warning when unable to dump a name of a symbol.
+# RUN: yaml2obj %s --docnum=2 -o %t3.o
+# RUN: llvm-readobj %t3.o --cg-profile 2>&1 | FileCheck %s -DFILE=%t3.o --check-prefix=LLVM-BROKEN-SYM
+# RUN: llvm-readelf %t3.o --cg-profile | FileCheck %s --check-prefix=GNU
+
+# LLVM-BROKEN-SYM:      CGProfile [
+# LLVM-BROKEN-SYM-NEXT:   CGProfileEntry {
+# LLVM-BROKEN-SYM-NEXT:     From: A (1)
+# LLVM-BROKEN-SYM-NEXT: warning: '[[FILE]]': unable to read the name of symbol with index 2: st_name (0xff) is past the end of the string table of size 0x5
+# LLVM-BROKEN-SYM-NEXT:     To: <?> (2)
+# LLVM-BROKEN-SYM-NEXT:     Weight: 10
+# LLVM-BROKEN-SYM-NEXT:   }
+# LLVM-BROKEN-SYM-NEXT:   CGProfileEntry {
+# LLVM-BROKEN-SYM-NEXT:     From: <?> (2)
+# LLVM-BROKEN-SYM-NEXT:     To: B (3)
+# LLVM-BROKEN-SYM-NEXT:     Weight: 20
+# LLVM-BROKEN-SYM-NEXT:   }
+# LLVM-BROKEN-SYM-NEXT:   CGProfileEntry {
+# LLVM-BROKEN-SYM-NEXT:     From: (0)
+# LLVM-BROKEN-SYM-NEXT: warning: '[[FILE]]': unable to read the name of symbol with index 4: unable to get symbol from section [index 3]: invalid symbol index (4)
+# LLVM-BROKEN-SYM-NEXT:     To: <?> (4)
+# LLVM-BROKEN-SYM-NEXT:     Weight: 20
+# LLVM-BROKEN-SYM-NEXT:   }
+# LLVM-BROKEN-SYM-NEXT: ]
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name: .llvm.call-graph-profile
+    Type: SHT_LLVM_CALL_GRAPH_PROFILE
+    Entries:
+      - From:   1
+        To:     2
+        Weight: 10
+      - From:   2
+        To:     3
+        Weight: 20
+      - From:   0x0 ## Null symbol.
+        To:     0x4 ## This index goes past the end of the symbol table.
+        Weight: 20
+  - Name:    .strtab
+    Type:    SHT_STRTAB
+    Content: "0041004200" ## '\0', 'A', '\0', 'B', '\0'
+Symbols:
+  - StName: 1    ## 'A'
+  - StName: 0xFF ## An arbitrary currupted index in the string table.
+  - StName: 3    ## 'B'
diff --git a/llvm/test/tools/llvm-readobj/ELF/dyn-symbols-size-from-hash-table.test b/llvm/test/tools/llvm-readobj/ELF/dyn-symbols-size-from-hash-table.test
index 80cb8e3e9fa4c..7da80598ec6e8 100644
--- a/llvm/test/tools/llvm-readobj/ELF/dyn-symbols-size-from-hash-table.test
+++ b/llvm/test/tools/llvm-readobj/ELF/dyn-symbols-size-from-hash-table.test
@@ -158,6 +158,20 @@ ProgramHeaders:
 # RUN:   FileCheck %s --check-prefixes=GNU2,GNU2-MORE,GNU2-ALL,WARN \
 # RUN:                --implicit-check-not=warning: -DNCHAIN=4
 
+# WARN: warning: '{{.*}}2-{{.*}}': hash table nchain ([[NCHAIN]]) differs from symbol count derived from SHT_DYNSYM section header (3)
+
+## Show we report a warning when the sh_entsize of the SHT_DYNSYM section is zero and therefore we are unable
+## to derive the number of dynamic symbols from SHT_DYNSYM section header.
+# RUN: yaml2obj --docnum=2 %s -o %t2-zero-entsize -DCHAIN="[1, 2, 3, 4]" -DENTSIZE=0
+# RUN: llvm-readobj --dyn-symbols %t2-zero-entsize 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t2-zero-entsize --check-prefixes=LLVM2,LLVM2-MORE,LLVM2-ALL,WARN-ENTSIZE \
+# RUN:                --implicit-check-not=warning:
+# RUN: llvm-readelf --dyn-symbols %t2-zero-entsize 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t2-zero-entsize --check-prefixes=GNU2,GNU2-MORE,GNU2-ALL,WARN-ENTSIZE \
+# RUN:                --implicit-check-not=warning: -DNCHAIN=4
+
+## WARN-ENTSIZE: warning: '[[FILE]]': SHT_DYNSYM section has sh_entsize == 0
+
 ## Show there's no warning if the sizes match
 # RUN: yaml2obj --docnum=2 %s -o %t2-same -DCHAIN="[1, 2, 3]"
 # RUN: llvm-readobj --dyn-symbols %t2-same 2>&1 | \
@@ -166,8 +180,6 @@ ProgramHeaders:
 # RUN:   FileCheck %s --check-prefixes=GNU2,GNU2-MORE \
 # RUN:                --implicit-check-not=warning: -DNCHAIN=3
 
-# WARN: warning: '{{.*}}2-{{.*}}': hash table nchain ([[NCHAIN]]) differs from symbol count derived from SHT_DYNSYM section header (3)
-
 # LLVM2:           DynamicSymbols [
 # LLVM2-NEXT:        Symbol {
 # LLVM2-NEXT:          Name:  (0)
@@ -236,6 +248,8 @@ Sections:
     ShSize:       0x48
     Address:      0x400
     AddressAlign: 0x400
+## 0x18 is the standard entsize value.
+    EntSize:      [[ENTSIZE=0x18]]
   - Name:         .hash
     Type:         SHT_HASH
     Flags:        [ SHF_ALLOC ]
diff --git a/llvm/test/tools/llvm-readobj/ELF/dyn-symbols.test b/llvm/test/tools/llvm-readobj/ELF/dyn-symbols.test
index d72c43aa50ac7..c4ac2f8325fc4 100644
--- a/llvm/test/tools/llvm-readobj/ELF/dyn-symbols.test
+++ b/llvm/test/tools/llvm-readobj/ELF/dyn-symbols.test
@@ -429,3 +429,108 @@ Sections:
         Value: 0x123
       - Tag:   DT_NULL
         Value: 0
+
+## Check we report a warning when the DT_STRSZ value is broken so that the dynamic string
+## table goes past the end of the file. Document we stop dumping symbols and report an error.
+
+# RUN: yaml2obj %s --docnum=13 -o %t14
+# RUN: llvm-readobj --dyn-symbols %t14 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t14 --check-prefix=DYNSTR-INVALID-LLVM
+# RUN: llvm-readelf --dyn-symbols %t14 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t14 --check-prefix=DYNSTR-INVALID-GNU
+
+# DYNSTR-INVALID-LLVM: warning: '[[FILE]]': the dynamic string table at 0x78 goes past the end of the file (0x2a8) with DT_STRSZ = 0xffffffff
+# DYNSTR-INVALID-LLVM:      DynamicSymbols [
+# DYNSTR-INVALID-LLVM-NEXT:   Symbol {
+# DYNSTR-INVALID-LLVM-NEXT:     Name:  (0)
+# DYNSTR-INVALID-LLVM-NEXT:     Value: 0x0
+# DYNSTR-INVALID-LLVM-NEXT:     Size: 0
+# DYNSTR-INVALID-LLVM-NEXT:     Binding: Local (0x0)
+# DYNSTR-INVALID-LLVM-NEXT:     Type: None (0x0)
+# DYNSTR-INVALID-LLVM-NEXT:     Other: 0
+# DYNSTR-INVALID-LLVM-NEXT:     Section: Undefined (0x0)
+# DYNSTR-INVALID-LLVM-NEXT:   }
+# DYNSTR-INVALID-LLVM-NEXT: warning: '[[FILE]]': st_name (0xffffff00) is past the end of the string table of size 0x6
+# DYNSTR-INVALID-LLVM-NEXT:   Symbol {
+# DYNSTR-INVALID-LLVM-NEXT:     Name: <?> (4294967040)
+# DYNSTR-INVALID-LLVM-NEXT:     Value: 0x0
+# DYNSTR-INVALID-LLVM-NEXT:     Size: 0
+# DYNSTR-INVALID-LLVM-NEXT:     Binding: Local (0x0)
+# DYNSTR-INVALID-LLVM-NEXT:     Type: None (0x0)
+# DYNSTR-INVALID-LLVM-NEXT:     Other: 0
+# DYNSTR-INVALID-LLVM-NEXT:     Section: Undefined (0x0)
+# DYNSTR-INVALID-LLVM-NEXT:   }
+# DYNSTR-INVALID-LLVM-NEXT:   Symbol {
+# DYNSTR-INVALID-LLVM-NEXT:     Name: test (1)
+# DYNSTR-INVALID-LLVM-NEXT:     Value: 0x0
+# DYNSTR-INVALID-LLVM-NEXT:     Size: 0
+# DYNSTR-INVALID-LLVM-NEXT:     Binding: Local (0x0)
+# DYNSTR-INVALID-LLVM-NEXT:     Type: None (0x0)
+# DYNSTR-INVALID-LLVM-NEXT:     Other: 0
+# DYNSTR-INVALID-LLVM-NEXT:     Section: Undefined (0x0)
+# DYNSTR-INVALID-LLVM-NEXT:   }
+# DYNSTR-INVALID-LLVM-NEXT: ]
+
+# DYNSTR-INVALID-GNU: warning: '[[FILE]]': the dynamic string table at 0x78 goes past the end of the file (0x2a8) with DT_STRSZ = 0xffffffff
+# DYNSTR-INVALID-GNU:      Symbol table '.dynsym' contains 3 entries:
+# DYNSTR-INVALID-GNU-NEXT:    Num:    Value          Size Type    Bind   Vis       Ndx Name
+# DYNSTR-INVALID-GNU-NEXT:      0: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT   UND
+# DYNSTR-INVALID-GNU-NEXT: warning: '[[FILE]]': st_name (0xffffff00) is past the end of the string table of size 0x6
+# DYNSTR-INVALID-GNU-NEXT:      1: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT   UND <?>
+# DYNSTR-INVALID-GNU-NEXT:      2: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT   UND test
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .dynstr
+    Type:    SHT_STRTAB
+    Flags:   [ SHF_ALLOC ]
+    Content: '007465737400' ## '\0', 'test', '\0'
+  - Name:  .dynamic
+    Type:  SHT_DYNAMIC
+    Flags: [ SHF_ALLOC ]
+    Link:  .dynstr
+    Entries:
+      - Tag:   DT_STRTAB
+        Value: 0x0
+      - Tag:   DT_STRSZ
+        Value: 0xffffffff
+      - Tag:   DT_NULL
+        Value: 0x0
+DynamicSymbols:
+  - StName:  0xffffff00
+## An arbitrary valid symbol to document we report an error before dumping it.
+  - StName:  0x1
+ProgramHeaders:
+  - Type:  PT_LOAD
+    Flags: [ PF_R ]
+    Sections:
+      - Section: .dynstr
+      - Section: .dynamic
+
+## Check we report a warning when the entry size of the dynamic symbol table is zero.
+# RUN: yaml2obj %s --docnum=14 -o %t15
+# RUN: llvm-readobj --dyn-symbols %t15 2>&1 | FileCheck %s -DFILE=%t15 --check-prefix=DYNSYM-ZERO-ENTSIZE-LLVM
+# RUN: llvm-readelf --dyn-symbols %t15 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t15 --check-prefix=DYNSYM-ZERO-ENTSIZE-GNU --implicit-check-not="Symbol table"
+
+# DYNSYM-ZERO-ENTSIZE-LLVM:      DynamicSymbols [
+# DYNSYM-ZERO-ENTSIZE-LLVM-NEXT:  warning: '[[FILE]]': section with index 1 has invalid size (0x10) or entry size (0x0)
+# DYNSYM-ZERO-ENTSIZE-LLVM-NEXT: ]
+
+# DYNSYM-ZERO-ENTSIZE-GNU:  warning: '[[FILE]]': section with index 1 has invalid size (0x10) or entry size (0x0)
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
+Sections:
+  - Name:    .dynsym
+    Type:    SHT_DYNSYM
+    EntSize: 0x0
diff --git a/llvm/test/tools/llvm-readobj/ELF/dynamic-malformed.test b/llvm/test/tools/llvm-readobj/ELF/dynamic-malformed.test
index ed72118c4ffcc..8b6fc1eca7d2c 100644
--- a/llvm/test/tools/llvm-readobj/ELF/dynamic-malformed.test
+++ b/llvm/test/tools/llvm-readobj/ELF/dynamic-malformed.test
@@ -400,7 +400,8 @@ ProgramHeaders:
 # RUN: llvm-readobj --dynamic-table %t9.2 2>&1 | FileCheck %s -DFILE=%t9.2 --check-prefix=PAST-THE-EOF
 # RUN: llvm-readelf --dynamic-table %t9.2 2>&1 | FileCheck %s -DFILE=%t9.2 --check-prefix=PAST-THE-EOF
 
-# PAST-THE-EOF:      warning: '[[FILE]]': string table at offset 0xb0 with size 0x211 goes past the end of the file (0x2c0)
+# PAST-THE-EOF:      warning: '[[FILE]]': the dynamic string table at 0xb0 goes past the end of the file (0x2c0) with DT_STRSZ = 0x211
+# PAST-THE-EOF:      warning: '[[FILE]]': string table was not found
 # PAST-THE-EOF:      {{[(]?}}NEEDED{{[)]?}}    Shared library: [<?>]
 # PAST-THE-EOF-NEXT: {{[(]?}}FILTER{{[)]?}}    Filter library: [<?>]
 # PAST-THE-EOF-NEXT: {{[(]?}}AUXILIARY{{[)]?}} Auxiliary library: [<?>]
diff --git a/llvm/test/tools/llvm-readobj/ELF/gnuhash.test b/llvm/test/tools/llvm-readobj/ELF/gnuhash.test
index 7fe2e465c6487..ce10cc3d997d3 100644
--- a/llvm/test/tools/llvm-readobj/ELF/gnuhash.test
+++ b/llvm/test/tools/llvm-readobj/ELF/gnuhash.test
@@ -1,7 +1,7 @@
 ## Check how the GNU Hash section is dumped with --gnu-hash-table.
 
-# RUN: yaml2obj --docnum=1 -DBITS=64 -DMACHINE=EM_X86_64 -D MASKWORDS=2 -D NBUCKETS=3 %s -o %t.x64
-# RUN: yaml2obj --docnum=1 -DBITS=32 -DMACHINE=EM_386 %s -D MASKWORDS=2 -D NBUCKETS=3 -o %t.x32
+# RUN: yaml2obj --docnum=1 -DBITS=64 -DMACHINE=EM_X86_64 %s -o %t.x64
+# RUN: yaml2obj --docnum=1 -DBITS=32 -DMACHINE=EM_386 %s -o %t.x32
 
 # RUN: llvm-readobj --gnu-hash-table %t.x64 | FileCheck %s
 # RUN: llvm-readelf --gnu-hash-table %t.x64 | FileCheck %s
@@ -33,9 +33,9 @@ Sections:
       SymNdx: 0x1
       Shift2: 0x2
 ## The number of words in the Bloom filter. The value of 2 is no-op.
-      MaskWords: [[MASKWORDS]]
+      MaskWords: [[MASKWORDS=2]]
 ## The number of hash buckets. The value of 3 is no-op.
-      NBuckets:  [[NBUCKETS]]
+      NBuckets:  [[NBUCKETS=3]]
     BloomFilter: [0x3, 0x4]
     HashBuckets: [0x5, 0x6, 0x7]
     HashValues:  [0x8, 0x9, 0xA, 0xB]
@@ -290,15 +290,15 @@ ProgramHeaders:
 
 ## Check we report a proper warning when a hash table goes past the end of the file.
 
-## Case A: the 'nbuckets' field is set so that the table goes past the end of the file.
-# RUN: yaml2obj --docnum=1 -DBITS=64 -DMACHINE=EM_X86_64 -D MASKWORDS=4294967295 -D NBUCKETS=3 %s -o %t.err.maskwords
+## Case A: the 'maskwords' field is set so that the table goes past the end of the file.
+# RUN: yaml2obj --docnum=1 -DBITS=64 -DMACHINE=EM_X86_64 -D MASKWORDS=4294967295 %s -o %t.err.maskwords
 # RUN: llvm-readobj --gnu-hash-table %t.err.maskwords 2>&1 | \
 # RUN:   FileCheck %s -DFILE=%t.err.maskwords -DMASKWORDS=4294967295 -DNBUCKETS=3 --check-prefix=ERR
 # RUN: llvm-readelf --gnu-hash-table %t.err.maskwords 2>&1 | \
 # RUN:   FileCheck %s -DFILE=%t.err.maskwords -DMASKWORDS=4294967295 -DNBUCKETS=3 --check-prefix=ERR
 
-## Case B: the 'maskwords' field is set so that the table goes past the end of the file.
-# RUN: yaml2obj --docnum=1 -DBITS=64 -DMACHINE=EM_X86_64 -D MASKWORDS=2 -D NBUCKETS=4294967295 %s -o %t.err.nbuckets
+## Case B: the 'nbuckets' field is set so that the table goes past the end of the file.
+# RUN: yaml2obj --docnum=1 -DBITS=64 -DMACHINE=EM_X86_64 -D NBUCKETS=4294967295 %s -o %t.err.nbuckets
 # RUN: llvm-readobj --gnu-hash-table %t.err.nbuckets 2>&1 | \
 # RUN:   FileCheck %s -DFILE=%t.err.nbuckets -DMASKWORDS=2 -DNBUCKETS=4294967295 --check-prefix=ERR
 # RUN: llvm-readelf --gnu-hash-table %t.err.nbuckets 2>&1 | \
diff --git a/llvm/test/tools/llvm-readobj/ELF/hash-histogram.test b/llvm/test/tools/llvm-readobj/ELF/hash-histogram.test
index 525240e7dea75..736f170e4951b 100644
--- a/llvm/test/tools/llvm-readobj/ELF/hash-histogram.test
+++ b/llvm/test/tools/llvm-readobj/ELF/hash-histogram.test
@@ -211,8 +211,7 @@ ProgramHeaders:
 ## Check we dump a histogram for the .gnu.hash table even when the .hash table is skipped.
 
 ## Case A: the .hash table has no data to build histogram and it is skipped.
-##         (NBUCKET == 0x2 is a no-op: it does not change the number of buckets described by the "Bucket" key).
-# RUN: yaml2obj --docnum=5 -DNBUCKET=0x2 %s -o %t5.o
+# RUN: yaml2obj --docnum=5 %s -o %t5.o
 # RUN: llvm-readelf --elf-hash-histogram %t5.o 2>&1 | \
 # RUN:   FileCheck %s --check-prefix=GNU-HASH --implicit-check-not="Histogram"
 
@@ -236,7 +235,8 @@ Sections:
     Type:    SHT_HASH
     Flags:   [ SHF_ALLOC ]
     Bucket:  [ 0 ]
-    NBucket: [[NBUCKET]]
+## 0x2 is a no-op: it does not change the number of buckets described by the "Bucket" key
+    NBucket: [[NBUCKET=0x2]]
     Chain:   [ 0, 0 ]
   - Name:  .gnu.hash
     Type:  SHT_GNU_HASH
@@ -269,17 +269,15 @@ ProgramHeaders:
 
 ## Check we report a proper warning when the GNU hash table goes past the end of the file.
 
-## Case A: the 'nbuckets' field is set so that the GNU hash table goes past the end of the file.
-##         The value of 1 for the NBUCKETS is no-op.
-# RUN: yaml2obj --docnum=6 -D MASKWORDS=0x80000000 -D NBUCKETS=1 %s -o %t7
+## Case A: the 'maskwords' field is set so that the GNU hash table goes past the end of the file.
+# RUN: yaml2obj --docnum=6 -D MASKWORDS=0x80000000 %s -o %t7
 # RUN: llvm-readelf --elf-hash-histogram %t7 2>&1 | \
 # RUN:   FileCheck %s -DFILE=%t7 --check-prefix=ERR5 --implicit-check-not="Histogram"
 
 # ERR5: warning: '[[FILE]]': unable to dump the SHT_GNU_HASH section at 0x78: it goes past the end of the file
 
-## Case B: the 'maskwords' field is set so that the GNU hash table goes past the end of the file.
-##         The value of 1 for the MASKWORDS is no-op.
-# RUN: yaml2obj --docnum=6 -D MASKWORDS=1 -D NBUCKETS=0x80000000 %s -o %t8
+## Case B: the 'nbuckets' field is set so that the GNU hash table goes past the end of the file.
+# RUN: yaml2obj --docnum=6 -D NBUCKETS=0x80000000 %s -o %t8
 # RUN: llvm-readelf --elf-hash-histogram %t8 2>&1 | \
 # RUN:   FileCheck %s -DFILE=%t8 --check-prefix=ERR5 --implicit-check-not="Histogram"
 
@@ -296,10 +294,10 @@ Sections:
     Header:
       SymNdx: 0x0
       Shift2: 0x0
-## The number of words in the Bloom filter.
-      MaskWords: [[MASKWORDS]]
-## The number of hash buckets.
-      NBuckets:  [[NBUCKETS]]
+## The number of words in the Bloom filter. The value of 1 is no-op.
+      MaskWords: [[MASKWORDS=1]]
+## The number of hash buckets. The value of 1 is no-op.
+      NBuckets:  [[NBUCKETS=1]]
     BloomFilter: [ 0x0 ]
     HashBuckets: [ 0x0 ]
     HashValues:  [ 0x0 ]
diff --git a/llvm/test/tools/llvm-readobj/ELF/hash-symbols.test b/llvm/test/tools/llvm-readobj/ELF/hash-symbols.test
index dd0d9e3981749..2576fe03deaef 100644
--- a/llvm/test/tools/llvm-readobj/ELF/hash-symbols.test
+++ b/llvm/test/tools/llvm-readobj/ELF/hash-symbols.test
@@ -388,7 +388,7 @@ ProgramHeaders:
 
 ## Case A.1: the hash table ends right before the EOF. We have a broken nbucket
 ##           field that has a value larger than the number of buckets.
-# RUN: yaml2obj --docnum=7 %s -o %t7.1.o -DNBUCKET=0x5d -DNCHAIN=0x1
+# RUN: yaml2obj --docnum=7 %s -o %t7.1.o -DNBUCKET=0x5d
 # RUN: llvm-readelf --hash-symbols %t7.1.o 2>&1 | FileCheck %s --check-prefix=NOERR1
 # NOERR1:            Symbol table of .hash for image:
 # NOERR1-NEXT:         Num Buc:    Value  Size   Type   Bind Vis      Ndx Name
@@ -396,7 +396,7 @@ ProgramHeaders:
 
 ## Case A.2: the hash table ends 1 byte past the EOF. We have a broken nbucket
 ##           field that has a value larger than the number of buckets.
-# RUN: yaml2obj --docnum=7 %s -o %t7.2.o -DNBUCKET=0x5e -DNCHAIN=0x1
+# RUN: yaml2obj --docnum=7 %s -o %t7.2.o -DNBUCKET=0x5e
 # RUN: llvm-readelf --hash-symbols %t7.2.o 2>&1 | FileCheck %s --check-prefix=ERR2 -DFILE=%t7.2.o
 # ERR2:      Symbol table of .hash for image:
 # ERR2-NEXT: warning: '[[FILE]]': the hash table at offset 0x54 goes past the end of the file (0x1d4), nbucket = 94, nchain = 1{{$}}
@@ -404,7 +404,7 @@ ProgramHeaders:
 
 ## Case B.1: the hash table ends right before the EOF. We have a broken nchain
 ##           field that has a value larger than the number of chains.
-# RUN: yaml2obj --docnum=7 %s -o %t7.3.o -DNBUCKET=0x1 -DNCHAIN=0x5d
+# RUN: yaml2obj --docnum=7 %s -o %t7.3.o -DNCHAIN=0x5d
 # RUN: llvm-readelf --hash-symbols %t7.3.o 2>&1 | \
 # RUN:   FileCheck %s --implicit-check-not="warning:" --check-prefix=NOERR2 -DFILE=%t7.3.o
 # NOERR2:      warning: '[[FILE]]': hash table nchain (93) differs from symbol count derived from SHT_DYNSYM section header (1)
@@ -414,7 +414,7 @@ ProgramHeaders:
 
 ## Case B.2: the hash table ends 1 byte past the EOF. We have a broken nchain
 ##           field that has a value larger than the number of chains.
-# RUN: yaml2obj --docnum=7 %s -o %t7.4.o -DNBUCKET=0x1 -DNCHAIN=0x5e
+# RUN: yaml2obj --docnum=7 %s -o %t7.4.o -DNCHAIN=0x5e
 # RUN: llvm-readelf --hash-symbols %t7.4.o 2>&1 | FileCheck %s --check-prefix=ERR3 -DFILE=%t7.4.o
 # ERR3:      Symbol table of .hash for image:
 # ERR3-NEXT: warning: '[[FILE]]': the hash table at offset 0x54 goes past the end of the file (0x1d4), nbucket = 1, nchain = 94{{$}}
@@ -431,9 +431,9 @@ Sections:
     Type:    SHT_HASH
     Flags:   [ SHF_ALLOC ]
     Bucket:  [ 0 ]
-    NBucket: [[NBUCKET]]
+    NBucket: [[NBUCKET=1]]
     Chain:   [ 0 ]
-    NChain:  [[NCHAIN]]
+    NChain:  [[NCHAIN=1]]
   - Name:  .dynamic
     Type:  SHT_DYNAMIC
     Flags: [ SHF_WRITE, SHF_ALLOC ]
@@ -448,3 +448,192 @@ ProgramHeaders:
     Sections:
       - Section: .hash
       - Section: .dynamic
+
+## Check we report a proper warning when a GNU hash table goes past the end of the file.
+
+## Case A: the 'maskwords' field is set so that the table goes past the end of the file.
+# RUN: yaml2obj --docnum=8 -D MASKWORDS=4294967295 %s -o %t.err.maskwords
+# RUN: llvm-readelf --hash-symbols %t.err.maskwords 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t.err.maskwords --check-prefix=ERR4
+
+# ERR4:      Symbol table of .gnu.hash for image:
+# ERR4-NEXT:  Num Buc: Value Size Type Bind Vis Ndx Name
+# ERR4-NEXT: warning: '[[FILE]]': unable to dump the SHT_GNU_HASH section at 0x78: it goes past the end of the file
+
+## Case B: the 'nbuckets' field is set so that the table goes past the end of the file.
+# RUN: yaml2obj --docnum=8 -D NBUCKETS=4294967295 %s -o %t.err.nbuckets
+# RUN: llvm-readelf --hash-symbols %t.err.nbuckets 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t.err.nbuckets --check-prefix=ERR4
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name:  .gnu.hash
+    Type:  SHT_GNU_HASH
+    Flags: [ SHF_ALLOC ]
+    Header:
+      SymNdx: 0x1
+      Shift2: 0x2
+## The number of words in the Bloom filter. The value of 2 is no-op.
+      MaskWords: [[MASKWORDS=2]]
+## The number of hash buckets. The value of 3 is no-op.
+      NBuckets:  [[NBUCKETS=3]]
+    BloomFilter: [0x3, 0x4]
+    HashBuckets: [0x5, 0x6, 0x7]
+    HashValues:  [0x8, 0x9, 0xA, 0xB]
+  - Name:  .dynamic
+    Type:  SHT_DYNAMIC
+    Flags: [ SHF_ALLOC ]
+    Link:  .dynstr
+    Entries:
+      - Tag:   DT_GNU_HASH
+        Value: 0x0
+      - Tag:   DT_NULL
+        Value: 0x0
+DynamicSymbols:
+  - Name:    aaa
+    Binding: STB_GLOBAL
+  - Name:    bbb
+    Binding: STB_GLOBAL
+  - Name:    ccc
+    Binding: STB_GLOBAL
+  - Name:    ddd
+    Binding: STB_GLOBAL
+ProgramHeaders:
+  - Type:  PT_LOAD
+    Flags: [ PF_R, PF_X ]
+    Sections:
+      - Section: .gnu.hash
+      - Section: .dynamic
+
+## Check the behavior when the dynamic symbol table is empty or not found.
+
+## Case A.1: Check we report a warning when the dynamic symbol table is empty and we attempt to print hash symbols
+##         from the .hash table. The number of symbols in the dynamic symbol table can be calculated from its size
+##         or derived from the Chain vector of the .hash table. Make both ways to return a zero to do the check.
+# RUN: yaml2obj --docnum=9 %s -o %t9.1.so
+# RUN: llvm-readelf --hash-symbols %t9.1.so 2>&1 | FileCheck %s -DFILE=%t9.1.so --check-prefix=DYNSYM-EMPTY-HASH
+
+# DYNSYM-EMPTY-HASH:      Symbol table of .hash for image:
+# DYNSYM-EMPTY-HASH-NEXT:   Num Buc:    Value  Size   Type   Bind Vis      Ndx Name
+# DYNSYM-EMPTY-HASH-NEXT: warning: '[[FILE]]': unable to print symbols for the .hash table: the dynamic symbol table is empty
+# DYNSYM-EMPTY-HASH-NOT:  {{.}}
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
+Sections:
+  - Name:   .hash
+    Type:   SHT_HASH
+    Flags:  [ SHF_ALLOC ]
+    Bucket: [ 0 ]
+    Chain:  [ ]
+  - Name:  .dynamic
+    Type:  SHT_DYNAMIC
+    Flags: [ SHF_ALLOC ]
+    Entries:
+      - Tag:   DT_HASH
+        Value: 0x0
+      - Tag:   DT_STRTAB
+## PT_LOAD p_offset == .hash offset == 0x54.
+## 0x54 + 0x2c == 0x80 == .dynstr offset.
+        Value: 0x2c
+      - Tag:   DT_STRSZ
+        Value: 0x1
+      - Tag:   DT_NULL
+        Value: 0x0
+  - Name:  .dynstr
+    Type:  SHT_STRTAB
+    Flags: [ SHF_ALLOC ]
+  - Name:  .dynsym
+    Type:  [[DYNSYMTYPE=SHT_DYNSYM]]
+    Flags: [ SHF_ALLOC ]
+    Size:  0
+ProgramHeaders:
+  - Type:  PT_LOAD
+    Flags: [ PF_R, PF_X ]
+    Sections:
+      - Section: .hash
+      - Section: .dynamic
+      - Section: .dynstr
+
+## Case A.2: similar to A.1, but now check that we report a warning when the dynamic symbol table was not found.
+##           To do that, set the type of the .dynsym to SHT_PROGBITS to hide it.
+# RUN: yaml2obj --docnum=9 -DDYNSYMTYPE=SHT_PROGBITS %s -o %t9.2.so
+# RUN: llvm-readelf --hash-symbols %t9.2.so 2>&1 | FileCheck %s -DFILE=%t9.2.so --check-prefix=DYNSYM-NOTFOUND-HASH
+
+# DYNSYM-NOTFOUND-HASH:      Symbol table of .hash for image:
+# DYNSYM-NOTFOUND-HASH-NEXT:   Num Buc:    Value  Size   Type   Bind Vis      Ndx Name
+# DYNSYM-NOTFOUND-HASH-NEXT: warning: '[[FILE]]': unable to print symbols for the .hash table: the dynamic symbol table was not found
+# DYNSYM-NOTFOUND-HASH-NOT:  {{.}}
+
+## Case B.1: Check we report a warning when the dynamic symbol table is empty and we attempt to print
+##           hash symbols from the .gnu.hash table.
+# RUN: yaml2obj --docnum=10 %s -o %t10.1.so
+# RUN: llvm-readelf --hash-symbols %t10.1.so 2>&1 | FileCheck %s -DFILE=%t10.1.so --check-prefix=DYNSYM-EMPTY-GNUHASH
+
+# DYNSYM-EMPTY-GNUHASH:      Symbol table of .gnu.hash for image:
+# DYNSYM-EMPTY-GNUHASH-NEXT:  Num Buc:    Value  Size   Type   Bind Vis      Ndx Name
+# DYNSYM-EMPTY-GNUHASH-NEXT: warning: '[[FILE]]': unable to print symbols for the .gnu.hash table: the dynamic symbol table is empty
+# DYNSYM-EMPTY-GNUHASH-NOT:  {{.}}
+
+## Case B.2: similar to B.1, but now check that we report a warning when the dynamic symbol table was not found.
+##           To do that, set the type of the .dynsym to SHT_PROGBITS to hide it.
+# RUN: yaml2obj --docnum=10 -DDYNSYMTYPE=SHT_PROGBITS %s -o %t10.2.so
+# RUN: llvm-readelf --hash-symbols %t10.2.so 2>&1 | FileCheck %s -DFILE=%t10.2.so --check-prefix=DYNSYM-NOTFOUND-GNUHASH
+
+# DYNSYM-NOTFOUND-GNUHASH:      Symbol table of .gnu.hash for image:
+# DYNSYM-NOTFOUND-GNUHASH-NEXT:  Num Buc:    Value  Size   Type   Bind Vis      Ndx Name
+# DYNSYM-NOTFOUND-GNUHASH-NEXT: warning: '[[FILE]]': unable to print symbols for the .gnu.hash table: the dynamic symbol table was not found
+# DYNSYM-NOTFOUND-GNUHASH-NOT:  {{.}}
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
+Sections:
+  - Name:  .gnu.hash
+    Type:  SHT_GNU_HASH
+    Flags: [ SHF_ALLOC ]
+    Header:
+      SymNdx: 0x0
+      Shift2: 0x0
+    BloomFilter: [ 0x0 ]
+    HashBuckets: [ 0x1 ]
+    HashValues:  [ 0x0 ]
+  - Name:  .dynamic
+    Type:  SHT_DYNAMIC
+    Flags: [ SHF_ALLOC ]
+    Entries:
+      - Tag:   DT_GNU_HASH
+        Value: 0x0
+      - Tag:   DT_STRTAB
+## PT_LOAD p_offset == .hash offset == 0x54.
+## 0x54 + 0x3c == 0x80 == .dynstr offset.
+        Value: 0x3c
+      - Tag:   DT_STRSZ
+        Value: 0x1
+      - Tag:   DT_NULL
+        Value: 0x0
+  - Name: .dynstr
+    Type: SHT_STRTAB
+  - Name:  .dynsym
+    Type:  [[DYNSYMTYPE=SHT_DYNSYM]]
+    Flags: [ SHF_ALLOC ]
+    Size:  0
+ProgramHeaders:
+  - Type:  PT_LOAD
+    Flags: [ PF_R, PF_X ]
+    Sections:
+      - Section: .gnu.hash
+      - Section: .dynamic
+      - Section: .dynstr
diff --git a/llvm/test/tools/llvm-readobj/ELF/invalid-shstrndx.test b/llvm/test/tools/llvm-readobj/ELF/invalid-shstrndx.test
index cc05f13a322f0..84583d2c3cd9f 100644
--- a/llvm/test/tools/llvm-readobj/ELF/invalid-shstrndx.test
+++ b/llvm/test/tools/llvm-readobj/ELF/invalid-shstrndx.test
@@ -6,8 +6,15 @@
 # GNU:        Section header string table index: 255
 # GNU-NEXT:   There are 3 section headers, starting at offset 0x58:
 # GNU:      Section Headers:
-# GNU-NEXT:  [Nr] Name
-# GNU-NEXT:  error: '[[FILE]]': section header string table index 255 does not exist
+# GNU-NEXT:  [Nr] Name         Type   Address          Off    Size   ES Flg Lk Inf Al
+# GNU-NEXT: warning: '[[FILE]]': section header string table index 255 does not exist
+# GNU-NEXT:  [ 0] <no-strings> NULL   0000000000000000 000000 000000 00      0   0  0
+# GNU-NEXT:  [ 1] <no-strings> STRTAB 0000000000000000 000040 000001 00      0   0  1
+# GNU-NEXT:  [ 2] <no-strings> STRTAB 0000000000000000 000041 000013 00      0   0  1
+# GNU-NEXT: Key to Flags:
+# GNU:      Section to Segment mapping:
+# GNU-NEXT:   Segment Sections...
+# GNU-NEXT: error: '[[FILE]]': section header string table index 255 does not exist
 
 # LLVM:      ElfHeader {
 # LLVM:        StringTableSectionIndex: 255
diff --git a/llvm/test/tools/llvm-readobj/ELF/linker-options.test b/llvm/test/tools/llvm-readobj/ELF/linker-options.test
index 3fad6b65c7a57..8e66547bd5edc 100644
--- a/llvm/test/tools/llvm-readobj/ELF/linker-options.test
+++ b/llvm/test/tools/llvm-readobj/ELF/linker-options.test
@@ -2,13 +2,14 @@
 ## to dump SHT_LLVM_LINKER_OPTIONS sections.
 
 # RUN: yaml2obj --docnum=1 %s -o %t1
-# RUN: llvm-readobj --elf-linker-options %t1 2>&1 | FileCheck %s --check-prefix=CHECK -DFILE=%t1
+# RUN: llvm-readobj --elf-linker-options %t1 2>&1 | FileCheck %s -DFILE=%t1
 
 # CHECK:      LinkerOptions [
 # CHECK:        option 0: value 0
 # CHECK:        option 1: value 1
 # CHECK-NEXT:   warning: '[[FILE]]': SHT_LLVM_LINKER_OPTIONS section at index 2 is broken: an incomplete key-value pair was found. The last possible key was: "c"
 # CHECK-NEXT:   warning: '[[FILE]]': SHT_LLVM_LINKER_OPTIONS section at index 4 is broken: the content is not null-terminated
+# CHECK-NEXT:   warning: '[[FILE]]': unable to read the content of the SHT_LLVM_LINKER_OPTIONS section: section [index 5] has a sh_offset (0xffffffff) + sh_size (0x8) that is greater than the file size (0x370)
 # CHECK-NEXT:   option 3: value 3
 # CHECK-NEXT: ]
 
@@ -44,7 +45,15 @@ Sections:
   - Name:    .linker-options.nonul
     Type:    SHT_LLVM_LINKER_OPTIONS
     Content: "61"
-## Case 5: another correct case to show we do not stop dumping after reporting a warning.
+## Case 5: check we report a warning when it is not possible to read
+##         the content of the SHT_LLVM_LINKER_OPTIONS section.
+  - Name:     .linker-options.broken.content
+    Type:     SHT_LLVM_LINKER_OPTIONS
+    ShOffset: 0xffffffff
+    Options:
+      - Name:  foo
+        Value: bar
+## Case 6: another correct case to show we do not stop dumping after reporting a warning.
   - Name: .linker-options.valid2
     Type: SHT_LLVM_LINKER_OPTIONS
     Options:
diff --git a/llvm/test/tools/llvm-readobj/ELF/many-sections.s b/llvm/test/tools/llvm-readobj/ELF/many-sections.s
index ae7ce34706cad..eea5ed7f95e47 100644
--- a/llvm/test/tools/llvm-readobj/ELF/many-sections.s
+++ b/llvm/test/tools/llvm-readobj/ELF/many-sections.s
@@ -34,13 +34,25 @@ Sections:
 
 # RUN: yaml2obj --docnum=2 %s -o %t2
 
-# RUN: llvm-readelf --file-headers %t2 | FileCheck %s --check-prefix=GNU2
+# RUN: llvm-readelf --file-headers --sections %t2 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t2 --check-prefix=GNU2
 # GNU2: Number of section headers:         0
 # GNU2: Section header string table index: 65535 (corrupt: out of range)
 
-# RUN: llvm-readobj --file-headers %t2 | FileCheck %s --check-prefix=LLVM2
-# LLVM2: SectionHeaderCount: 0
-# LLVM2: StringTableSectionIndex: 65535 (corrupt: out of range)
+# GNU2:       There are 0 section headers, starting at offset 0x0:
+# GNU2-EMPTY:
+# GNU2-NEXT:  Section Headers:
+# GNU2-NEXT:   [Nr] Name Type Address Off Size ES Flg Lk Inf Al
+# GNU2-NEXT:  warning: '[[FILE]]': e_shstrndx == SHN_XINDEX, but the section header table is empty
+# GNU2-NEXT:  Key to Flags:
+
+# RUN: llvm-readobj --file-headers --sections %t2 | \
+# RUN:   FileCheck %s --check-prefix=LLVM2 --implicit-check-not="warning:"
+# LLVM2:       SectionHeaderCount: 0
+# LLVM2:       StringTableSectionIndex: 65535 (corrupt: out of range)
+# LLVM2-NEXT: }
+# LLVM2-NEXT: Sections [
+# LLVM2-NEXT: ]
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/tools/llvm-readobj/ELF/mips-got.test b/llvm/test/tools/llvm-readobj/ELF/mips-got.test
index 8ed35d4b68e27..cfbf1c4f37a3e 100644
--- a/llvm/test/tools/llvm-readobj/ELF/mips-got.test
+++ b/llvm/test/tools/llvm-readobj/ELF/mips-got.test
@@ -1,486 +1,549 @@
-RUN: llvm-readobj -A %p/Inputs/dynamic-table-exe.mips | \
-RUN:   FileCheck %s -check-prefix GOT-EXE
-RUN: llvm-readobj -A %p/Inputs/dynamic-table-so.mips | \
-RUN:   FileCheck %s -check-prefix GOT-SO
-RUN: llvm-readobj -A %p/Inputs/got-tls.so.elf-mips64el | \
-RUN:   FileCheck %s -check-prefix GOT-TLS
-RUN: llvm-readobj -A %p/Inputs/got-empty.exe.mipsel | \
-RUN:   FileCheck %s -check-prefix GOT-EMPTY
-RUN: llvm-readobj -A %p/Inputs/got-static.exe.mips | \
-RUN:   FileCheck %s -check-prefix GOT-STATIC
-
-RUN: llvm-readelf -A %p/Inputs/dynamic-table-exe.mips | \
-RUN:   FileCheck %s --strict-whitespace -check-prefix GNU-GOT-EXE
-RUN: llvm-readelf -A %p/Inputs/dynamic-table-so.mips | \
-RUN:   FileCheck %s --strict-whitespace -check-prefix GNU-GOT-SO
-RUN: llvm-readelf -A %p/Inputs/got-tls.so.elf-mips64el | \
-RUN:   FileCheck %s --strict-whitespace -check-prefix GNU-GOT-TLS
-RUN: llvm-readelf -A %p/Inputs/got-empty.exe.mipsel | \
-RUN:   FileCheck %s --strict-whitespace -check-prefix GNU-GOT-EMPTY
-RUN: llvm-readelf -A %p/Inputs/got-static.exe.mips | \
-RUN:   FileCheck %s --strict-whitespace -check-prefix GNU-GOT-STATIC
-
-GOT-EXE:      Primary GOT {
-GOT-EXE-NEXT:   Canonical gp value: 0x418880
-GOT-EXE-NEXT:   Reserved entries [
-GOT-EXE-NEXT:     Entry {
-GOT-EXE-NEXT:       Address: 0x410890
-GOT-EXE-NEXT:       Access: -32752
-GOT-EXE-NEXT:       Initial: 0x0
-GOT-EXE-NEXT:       Purpose: Lazy resolver
-GOT-EXE-NEXT:     }
-GOT-EXE-NEXT:     Entry {
-GOT-EXE-NEXT:       Address: 0x410894
-GOT-EXE-NEXT:       Access: -32748
-GOT-EXE-NEXT:       Initial: 0x80000000
-GOT-EXE-NEXT:       Purpose: Module pointer (GNU extension)
-GOT-EXE-NEXT:     }
-GOT-EXE-NEXT:   ]
-GOT-EXE-NEXT:   Local entries [
-GOT-EXE-NEXT:     Entry {
-GOT-EXE-NEXT:       Address: 0x410898
-GOT-EXE-NEXT:       Access: -32744
-GOT-EXE-NEXT:       Initial: 0x400418
-GOT-EXE-NEXT:     }
-GOT-EXE-NEXT:     Entry {
-GOT-EXE-NEXT:       Address: 0x41089C
-GOT-EXE-NEXT:       Access: -32740
-GOT-EXE-NEXT:       Initial: 0x410840
-GOT-EXE-NEXT:     }
-GOT-EXE-NEXT:     Entry {
-GOT-EXE-NEXT:       Address: 0x4108A0
-GOT-EXE-NEXT:       Access: -32736
-GOT-EXE-NEXT:       Initial: 0x0
-GOT-EXE-NEXT:     }
-GOT-EXE-NEXT:   ]
-GOT-EXE-NEXT:   Global entries [
-GOT-EXE-NEXT:     Entry {
-GOT-EXE-NEXT:       Address: 0x4108A4
-GOT-EXE-NEXT:       Access: -32732
-GOT-EXE-NEXT:       Initial: 0x0
-GOT-EXE-NEXT:       Value: 0x0
-GOT-EXE-NEXT:       Type: Function (0x2)
-GOT-EXE-NEXT:       Section: Undefined (0x0)
-GOT-EXE-NEXT:       Name: __gmon_start__ (1)
-GOT-EXE-NEXT:     }
-GOT-EXE-NEXT:   ]
-GOT-EXE-NEXT:   Number of TLS and multi-GOT entries: 0
-GOT-EXE-NEXT: }
-
-GOT-SO:      Primary GOT {
-GOT-SO-NEXT:   Canonical gp value: 0x188D0
-GOT-SO-NEXT:   Reserved entries [
-GOT-SO-NEXT:     Entry {
-GOT-SO-NEXT:       Address: 0x108E0
-GOT-SO-NEXT:       Access: -32752
-GOT-SO-NEXT:       Initial: 0x0
-GOT-SO-NEXT:       Purpose: Lazy resolver
-GOT-SO-NEXT:     }
-GOT-SO-NEXT:     Entry {
-GOT-SO-NEXT:       Address: 0x108E4
-GOT-SO-NEXT:       Access: -32748
-GOT-SO-NEXT:       Initial: 0x80000000
-GOT-SO-NEXT:       Purpose: Module pointer (GNU extension)
-GOT-SO-NEXT:     }
-GOT-SO-NEXT:   ]
-GOT-SO-NEXT:   Local entries [
-GOT-SO-NEXT:     Entry {
-GOT-SO-NEXT:       Address: 0x108E8
-GOT-SO-NEXT:       Access: -32744
-GOT-SO-NEXT:       Initial: 0x108E0
-GOT-SO-NEXT:     }
-GOT-SO-NEXT:     Entry {
-GOT-SO-NEXT:       Address: 0x108EC
-GOT-SO-NEXT:       Access: -32740
-GOT-SO-NEXT:       Initial: 0x10000
-GOT-SO-NEXT:     }
-GOT-SO-NEXT:     Entry {
-GOT-SO-NEXT:       Address: 0x108F0
-GOT-SO-NEXT:       Access: -32736
-GOT-SO-NEXT:       Initial: 0x10920
-GOT-SO-NEXT:     }
-GOT-SO-NEXT:     Entry {
-GOT-SO-NEXT:       Address: 0x108F4
-GOT-SO-NEXT:       Access: -32732
-GOT-SO-NEXT:       Initial: 0x108CC
-GOT-SO-NEXT:     }
-GOT-SO-NEXT:     Entry {
-GOT-SO-NEXT:       Address: 0x108F8
-GOT-SO-NEXT:       Access: -32728
-GOT-SO-NEXT:       Initial: 0x0
-GOT-SO-NEXT:     }
-GOT-SO-NEXT:     Entry {
-GOT-SO-NEXT:       Address: 0x108FC
-GOT-SO-NEXT:       Access: -32724
-GOT-SO-NEXT:       Initial: 0x0
-GOT-SO-NEXT:     }
-GOT-SO-NEXT:     Entry {
-GOT-SO-NEXT:       Address: 0x10900
-GOT-SO-NEXT:       Access: -32720
-GOT-SO-NEXT:       Initial: 0x0
-GOT-SO-NEXT:     }
-GOT-SO-NEXT:     Entry {
-GOT-SO-NEXT:       Address: 0x10904
-GOT-SO-NEXT:       Access: -32716
-GOT-SO-NEXT:       Initial: 0x0
-GOT-SO-NEXT:     }
-GOT-SO-NEXT:   ]
-GOT-SO-NEXT:   Global entries [
-GOT-SO-NEXT:     Entry {
-GOT-SO-NEXT:       Address: 0x10908
-GOT-SO-NEXT:       Access: -32712
-GOT-SO-NEXT:       Initial: 0x0
-GOT-SO-NEXT:       Value: 0x0
-GOT-SO-NEXT:       Type: None (0x0)
-GOT-SO-NEXT:       Section: Undefined (0x0)
-GOT-SO-NEXT:       Name: _ITM_registerTMCloneTable (87)
-GOT-SO-NEXT:     }
-GOT-SO-NEXT:     Entry {
-GOT-SO-NEXT:       Address: 0x1090C
-GOT-SO-NEXT:       Access: -32708
-GOT-SO-NEXT:       Initial: 0x0
-GOT-SO-NEXT:       Value: 0x0
-GOT-SO-NEXT:       Type: None (0x0)
-GOT-SO-NEXT:       Section: Undefined (0x0)
-GOT-SO-NEXT:       Name: _Jv_RegisterClasses (128)
-GOT-SO-NEXT:     }
-GOT-SO-NEXT:     Entry {
-GOT-SO-NEXT:       Address: 0x10910
-GOT-SO-NEXT:       Access: -32704
-GOT-SO-NEXT:       Initial: 0x0
-GOT-SO-NEXT:       Value: 0x0
-GOT-SO-NEXT:       Type: Function (0x2)
-GOT-SO-NEXT:       Section: Undefined (0x0)
-GOT-SO-NEXT:       Name: __gmon_start__ (23)
-GOT-SO-NEXT:     }
-GOT-SO-NEXT:     Entry {
-GOT-SO-NEXT:       Address: 0x10914
-GOT-SO-NEXT:       Access: -32700
-GOT-SO-NEXT:       Initial: 0x840
-GOT-SO-NEXT:       Value: 0x840
-GOT-SO-NEXT:       Type: Function (0x2)
-GOT-SO-NEXT:       Section: Undefined (0x0)
-GOT-SO-NEXT:       Name: puts@GLIBC_2.0 (162)
-GOT-SO-NEXT:     }
-GOT-SO-NEXT:     Entry {
-GOT-SO-NEXT:       Address: 0x10918
-GOT-SO-NEXT:       Access: -32696
-GOT-SO-NEXT:       Initial: 0x0
-GOT-SO-NEXT:       Value: 0x0
-GOT-SO-NEXT:       Type: None (0x0)
-GOT-SO-NEXT:       Section: Undefined (0x0)
-GOT-SO-NEXT:       Name: _ITM_deregisterTMCloneTable (59)
-GOT-SO-NEXT:     }
-GOT-SO-NEXT:     Entry {
-GOT-SO-NEXT:       Address: 0x1091C
-GOT-SO-NEXT:       Access: -32692
-GOT-SO-NEXT:       Initial: 0x0
-GOT-SO-NEXT:       Value: 0x0
-GOT-SO-NEXT:       Type: Function (0x2)
-GOT-SO-NEXT:       Section: Undefined (0x0)
-GOT-SO-NEXT:       Name: __cxa_finalize@GLIBC_2.2 (113)
-GOT-SO-NEXT:     }
-GOT-SO-NEXT:   ]
-GOT-SO-NEXT:   Number of TLS and multi-GOT entries: 0
-GOT-SO-NEXT: }
-
-GOT-TLS:      Primary GOT {
-GOT-TLS-NEXT:   Canonical gp value: 0x18BF0
-GOT-TLS-NEXT:   Reserved entries [
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C00
-GOT-TLS-NEXT:       Access: -32752
-GOT-TLS-NEXT:       Initial: 0x0
-GOT-TLS-NEXT:       Purpose: Lazy resolver
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C08
-GOT-TLS-NEXT:       Access: -32744
-GOT-TLS-NEXT:       Initial: 0x8000000000000000
-GOT-TLS-NEXT:       Purpose: Module pointer (GNU extension)
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:   ]
-GOT-TLS-NEXT:   Local entries [
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C10
-GOT-TLS-NEXT:       Access: -32736
-GOT-TLS-NEXT:       Initial: 0x10000
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C18
-GOT-TLS-NEXT:       Access: -32728
-GOT-TLS-NEXT:       Initial: 0x10C00
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C20
-GOT-TLS-NEXT:       Access: -32720
-GOT-TLS-NEXT:       Initial: 0x10CB8
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C28
-GOT-TLS-NEXT:       Access: -32712
-GOT-TLS-NEXT:       Initial: 0x10BF0
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C30
-GOT-TLS-NEXT:       Access: -32704
-GOT-TLS-NEXT:       Initial: 0x0
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C38
-GOT-TLS-NEXT:       Access: -32696
-GOT-TLS-NEXT:       Initial: 0x948
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C40
-GOT-TLS-NEXT:       Access: -32688
-GOT-TLS-NEXT:       Initial: 0xA20
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C48
-GOT-TLS-NEXT:       Access: -32680
-GOT-TLS-NEXT:       Initial: 0xAF0
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C50
-GOT-TLS-NEXT:       Access: -32672
-GOT-TLS-NEXT:       Initial: 0x0
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C58
-GOT-TLS-NEXT:       Access: -32664
-GOT-TLS-NEXT:       Initial: 0x0
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C60
-GOT-TLS-NEXT:       Access: -32656
-GOT-TLS-NEXT:       Initial: 0x0
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:   ]
-GOT-TLS-NEXT:   Global entries [
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C68
-GOT-TLS-NEXT:       Access: -32648
-GOT-TLS-NEXT:       Initial: 0x0
-GOT-TLS-NEXT:       Value: 0x0
-GOT-TLS-NEXT:       Type: None (0x0)
-GOT-TLS-NEXT:       Section: Undefined (0x0)
-GOT-TLS-NEXT:       Name: _ITM_registerTMCloneTable (78)
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C70
-GOT-TLS-NEXT:       Access: -32640
-GOT-TLS-NEXT:       Initial: 0x0
-GOT-TLS-NEXT:       Value: 0x0
-GOT-TLS-NEXT:       Type: None (0x0)
-GOT-TLS-NEXT:       Section: Undefined (0x0)
-GOT-TLS-NEXT:       Name: _Jv_RegisterClasses (119)
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C78
-GOT-TLS-NEXT:       Access: -32632
-GOT-TLS-NEXT:       Initial: 0x0
-GOT-TLS-NEXT:       Value: 0x0
-GOT-TLS-NEXT:       Type: Function (0x2)
-GOT-TLS-NEXT:       Section: Undefined (0x0)
-GOT-TLS-NEXT:       Name: __gmon_start__ (23)
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C80
-GOT-TLS-NEXT:       Access: -32624
-GOT-TLS-NEXT:       Initial: 0xB60
-GOT-TLS-NEXT:       Value: 0xB60
-GOT-TLS-NEXT:       Type: Function (0x2)
-GOT-TLS-NEXT:       Section: Undefined (0x0)
-GOT-TLS-NEXT:       Name: __tls_get_addr@GLIBC_2.3 (150)
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C88
-GOT-TLS-NEXT:       Access: -32616
-GOT-TLS-NEXT:       Initial: 0x0
-GOT-TLS-NEXT:       Value: 0x0
-GOT-TLS-NEXT:       Type: None (0x0)
-GOT-TLS-NEXT:       Section: Undefined (0x0)
-GOT-TLS-NEXT:       Name: _ITM_deregisterTMCloneTable (50)
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:     Entry {
-GOT-TLS-NEXT:       Address: 0x10C90
-GOT-TLS-NEXT:       Access: -32608
-GOT-TLS-NEXT:       Initial: 0x0
-GOT-TLS-NEXT:       Value: 0x0
-GOT-TLS-NEXT:       Type: Function (0x2)
-GOT-TLS-NEXT:       Section: Undefined (0x0)
-GOT-TLS-NEXT:       Name: __cxa_finalize@GLIBC_2.2 (104)
-GOT-TLS-NEXT:     }
-GOT-TLS-NEXT:   ]
-GOT-TLS-NEXT:   Number of TLS and multi-GOT entries: 4
-GOT-TLS-NEXT: }
-
-GOT-EMPTY:      Primary GOT {
-GOT-EMPTY-NEXT:   Canonical gp value: 0x409FF0
-GOT-EMPTY-NEXT:   Reserved entries [
-GOT-EMPTY-NEXT:     Entry {
-GOT-EMPTY-NEXT:       Address: 0x402000
-GOT-EMPTY-NEXT:       Access: -32752
-GOT-EMPTY-NEXT:       Initial: 0x0
-GOT-EMPTY-NEXT:       Purpose: Lazy resolver
-GOT-EMPTY-NEXT:     }
-GOT-EMPTY-NEXT:     Entry {
-GOT-EMPTY-NEXT:       Address: 0x402004
-GOT-EMPTY-NEXT:       Access: -32748
-GOT-EMPTY-NEXT:       Initial: 0x80000000
-GOT-EMPTY-NEXT:       Purpose: Module pointer (GNU extension)
-GOT-EMPTY-NEXT:     }
-GOT-EMPTY-NEXT:   ]
-GOT-EMPTY-NEXT:   Local entries [
-GOT-EMPTY-NEXT:   ]
-GOT-EMPTY-NEXT:   Global entries [
-GOT-EMPTY-NEXT:   ]
-GOT-EMPTY-NEXT:   Number of TLS and multi-GOT entries: 2
-GOT-EMPTY-NEXT: }
-
-GOT-STATIC:      Static GOT {
-GOT-STATIC-NEXT:   Canonical gp value: 0x418100
-GOT-STATIC-NEXT:   Reserved entries [
-GOT-STATIC-NEXT:     Entry {
-GOT-STATIC-NEXT:       Address: 0x410110
-GOT-STATIC-NEXT:       Access: -32752
-GOT-STATIC-NEXT:       Initial: 0x0
-GOT-STATIC-NEXT:       Purpose: Lazy resolver
-GOT-STATIC-NEXT:     }
-GOT-STATIC-NEXT:     Entry {
-GOT-STATIC-NEXT:       Address: 0x410114
-GOT-STATIC-NEXT:       Access: -32748
-GOT-STATIC-NEXT:       Initial: 0x80000000
-GOT-STATIC-NEXT:       Purpose: Module pointer (GNU extension)
-GOT-STATIC-NEXT:     }
-GOT-STATIC-NEXT:   ]
-GOT-STATIC-NEXT:   Local entries [
-GOT-STATIC-NEXT:     Entry {
-GOT-STATIC-NEXT:       Address: 0x410118
-GOT-STATIC-NEXT:       Access: -32744
-GOT-STATIC-NEXT:       Initial: 0x400000
-GOT-STATIC-NEXT:     }
-GOT-STATIC-NEXT:     Entry {
-GOT-STATIC-NEXT:       Address: 0x41011C
-GOT-STATIC-NEXT:       Access: -32740
-GOT-STATIC-NEXT:       Initial: 0x400100
-GOT-STATIC-NEXT:     }
-GOT-STATIC-NEXT:     Entry {
-GOT-STATIC-NEXT:       Address: 0x410120
-GOT-STATIC-NEXT:       Access: -32736
-GOT-STATIC-NEXT:       Initial: 0x400104
-GOT-STATIC-NEXT:     }
-GOT-STATIC-NEXT:   ]
-GOT-STATIC-NEXT: }
-
-GNU-GOT-EXE:      Primary GOT:
-GNU-GOT-EXE-NEXT:  Canonical gp value: 00418880
-
-GNU-GOT-EXE:       Reserved entries:
-GNU-GOT-EXE-NEXT:    Address     Access  Initial Purpose
-GNU-GOT-EXE-NEXT:   00410890 -32752(gp) 00000000 Lazy resolver
-GNU-GOT-EXE-NEXT:   00410894 -32748(gp) 80000000 Module pointer (GNU extension)
-
-GNU-GOT-EXE:       Local entries:
-GNU-GOT-EXE-NEXT:    Address     Access  Initial
-GNU-GOT-EXE-NEXT:   00410898 -32744(gp) 00400418
-GNU-GOT-EXE-NEXT:   0041089c -32740(gp) 00410840
-GNU-GOT-EXE-NEXT:   004108a0 -32736(gp) 00000000
-
-GNU-GOT-EXE:       Global entries:
-GNU-GOT-EXE-NEXT:    Address     Access  Initial Sym.Val. Type    Ndx Name
-GNU-GOT-EXE-NEXT:   004108a4 -32732(gp) 00000000 00000000 FUNC    UND __gmon_start__
-
-GNU-GOT-EXE:      PLT GOT:
-
-GNU-GOT-EXE:       Reserved entries:
-GNU-GOT-EXE-NEXT:    Address  Initial Purpose
-GNU-GOT-EXE-NEXT:   00410854 00000000 PLT lazy resolver
-GNU-GOT-EXE-NEXT:   00410858 00000000 Module pointer
-
-GNU-GOT-EXE:       Entries:
-GNU-GOT-EXE-NEXT:    Address  Initial Sym.Val. Type    Ndx Name
-GNU-GOT-EXE-NEXT:   0041085c 00400800 00000000 FUNC    UND puts
-GNU-GOT-EXE-NEXT:   00410860 00400800 00000000 FUNC    UND __libc_start_main
-
-GNU-GOT-SO:      Primary GOT:
-GNU-GOT-SO-NEXT:  Canonical gp value: 000188d0
-
-GNU-GOT-SO:       Reserved entries:
-GNU-GOT-SO-NEXT:    Address     Access  Initial Purpose
-GNU-GOT-SO-NEXT:   000108e0 -32752(gp) 00000000 Lazy resolver
-GNU-GOT-SO-NEXT:   000108e4 -32748(gp) 80000000 Module pointer (GNU extension)
-
-GNU-GOT-SO:       Local entries:
-GNU-GOT-SO-NEXT:    Address     Access  Initial
-GNU-GOT-SO-NEXT:   000108e8 -32744(gp) 000108e0
-GNU-GOT-SO-NEXT:   000108ec -32740(gp) 00010000
-GNU-GOT-SO-NEXT:   000108f0 -32736(gp) 00010920
-GNU-GOT-SO-NEXT:   000108f4 -32732(gp) 000108cc
-GNU-GOT-SO-NEXT:   000108f8 -32728(gp) 00000000
-GNU-GOT-SO-NEXT:   000108fc -32724(gp) 00000000
-GNU-GOT-SO-NEXT:   00010900 -32720(gp) 00000000
-GNU-GOT-SO-NEXT:   00010904 -32716(gp) 00000000
-
-GNU-GOT-SO:       Global entries:
-GNU-GOT-SO-NEXT:    Address     Access  Initial Sym.Val. Type    Ndx Name
-GNU-GOT-SO-NEXT:   00010908 -32712(gp) 00000000 00000000 NOTYPE  UND _ITM_registerTMCloneTable
-GNU-GOT-SO-NEXT:   0001090c -32708(gp) 00000000 00000000 NOTYPE  UND _Jv_RegisterClasses
-GNU-GOT-SO-NEXT:   00010910 -32704(gp) 00000000 00000000 FUNC    UND __gmon_start__
-GNU-GOT-SO-NEXT:   00010914 -32700(gp) 00000840 00000840 FUNC    UND puts
-GNU-GOT-SO-NEXT:   00010918 -32696(gp) 00000000 00000000 NOTYPE  UND _ITM_deregisterTMCloneTable
-GNU-GOT-SO-NEXT:   0001091c -32692(gp) 00000000 00000000 FUNC    UND __cxa_finalize
-
-GNU-GOT-TLS:      Primary GOT:
-GNU-GOT-TLS-NEXT:  Canonical gp value: 0000000000018bf0
-
-GNU-GOT-TLS:       Reserved entries:
-GNU-GOT-TLS-NEXT:            Address     Access          Initial Purpose
-GNU-GOT-TLS-NEXT:   0000000000010c00 -32752(gp) 0000000000000000 Lazy resolver
-GNU-GOT-TLS-NEXT:   0000000000010c08 -32744(gp) 8000000000000000 Module pointer (GNU extension)
-
-GNU-GOT-TLS:       Local entries:
-GNU-GOT-TLS-NEXT:            Address     Access          Initial
-GNU-GOT-TLS-NEXT:   0000000000010c10 -32736(gp) 0000000000010000
-GNU-GOT-TLS-NEXT:   0000000000010c18 -32728(gp) 0000000000010c00
-GNU-GOT-TLS-NEXT:   0000000000010c20 -32720(gp) 0000000000010cb8
-GNU-GOT-TLS-NEXT:   0000000000010c28 -32712(gp) 0000000000010bf0
-GNU-GOT-TLS-NEXT:   0000000000010c30 -32704(gp) 0000000000000000
-GNU-GOT-TLS-NEXT:   0000000000010c38 -32696(gp) 0000000000000948
-GNU-GOT-TLS-NEXT:   0000000000010c40 -32688(gp) 0000000000000a20
-GNU-GOT-TLS-NEXT:   0000000000010c48 -32680(gp) 0000000000000af0
-GNU-GOT-TLS-NEXT:   0000000000010c50 -32672(gp) 0000000000000000
-GNU-GOT-TLS-NEXT:   0000000000010c58 -32664(gp) 0000000000000000
-GNU-GOT-TLS-NEXT:   0000000000010c60 -32656(gp) 0000000000000000
-
-GNU-GOT-TLS:       Global entries:
-GNU-GOT-TLS-NEXT:            Address     Access          Initial         Sym.Val. Type    Ndx Name
-GNU-GOT-TLS-NEXT:   0000000000010c68 -32648(gp) 0000000000000000 0000000000000000 NOTYPE  UND _ITM_registerTMCloneTable
-GNU-GOT-TLS-NEXT:   0000000000010c70 -32640(gp) 0000000000000000 0000000000000000 NOTYPE  UND _Jv_RegisterClasses
-GNU-GOT-TLS-NEXT:   0000000000010c78 -32632(gp) 0000000000000000 0000000000000000 FUNC    UND __gmon_start__
-GNU-GOT-TLS-NEXT:   0000000000010c80 -32624(gp) 0000000000000b60 0000000000000b60 FUNC    UND __tls_get_addr
-GNU-GOT-TLS-NEXT:   0000000000010c88 -32616(gp) 0000000000000000 0000000000000000 NOTYPE  UND _ITM_deregisterTMCloneTable
-GNU-GOT-TLS-NEXT:   0000000000010c90 -32608(gp) 0000000000000000 0000000000000000 FUNC    UND __cxa_finalize
-
-GNU-GOTY     : Primary GOT:
-GNU-GOT-EMPTY:  Canonical gp value: 00409ff0
-
-GNU-GOTY     :  Reserved entries:
-GNU-GOT-EMPTY:    Address     Access  Initial Purpose
-GNU-GOT-EMPTY:   00402000 -32752(gp) 00000000 Lazy resolver
-GNU-GOT-EMPTY:   00402004 -32748(gp) 80000000 Module pointer (GNU extension)
-
-GNU-GOT-STATIC:      Static GOT:
-GNU-GOT-STATIC-NEXT:  Canonical gp value: 00418100
-
-GNU-GOT-STATIC:       Reserved entries:
-GNU-GOT-STATIC-NEXT:    Address     Access  Initial Purpose
-GNU-GOT-STATIC-NEXT:   00410110 -32752(gp) 00000000 Lazy resolver
-GNU-GOT-STATIC-NEXT:   00410114 -32748(gp) 80000000 Module pointer (GNU extension)
-
-GNU-GOT-STATIC:       Local entries:
-GNU-GOT-STATIC-NEXT:    Address     Access  Initial
-GNU-GOT-STATIC-NEXT:   00410118 -32744(gp) 00400000
-GNU-GOT-STATIC-NEXT:   0041011c -32740(gp) 00400100
-GNU-GOT-STATIC-NEXT:   00410120 -32736(gp) 00400104
+# RUN: llvm-readobj -A %p/Inputs/dynamic-table-exe.mips | \
+# RUN:   FileCheck %s -check-prefix GOT-EXE
+# RUN: llvm-readobj -A %p/Inputs/dynamic-table-so.mips | \
+# RUN:   FileCheck %s -check-prefix GOT-SO
+# RUN: llvm-readobj -A %p/Inputs/got-tls.so.elf-mips64el | \
+# RUN:   FileCheck %s -check-prefix GOT-TLS
+# RUN: llvm-readobj -A %p/Inputs/got-empty.exe.mipsel | \
+# RUN:   FileCheck %s -check-prefix GOT-EMPTY
+# RUN: llvm-readobj -A %p/Inputs/got-static.exe.mips | \
+# RUN:   FileCheck %s -check-prefix GOT-STATIC
+
+# RUN: llvm-readelf -A %p/Inputs/dynamic-table-exe.mips | \
+# RUN:   FileCheck %s --strict-whitespace -check-prefix GNU-GOT-EXE
+# RUN: llvm-readelf -A %p/Inputs/dynamic-table-so.mips | \
+# RUN:   FileCheck %s --strict-whitespace -check-prefix GNU-GOT-SO
+# RUN: llvm-readelf -A %p/Inputs/got-tls.so.elf-mips64el | \
+# RUN:   FileCheck %s --strict-whitespace -check-prefix GNU-GOT-TLS
+# RUN: llvm-readelf -A %p/Inputs/got-empty.exe.mipsel | \
+# RUN:   FileCheck %s --strict-whitespace -check-prefix GNU-GOT-EMPTY
+# RUN: llvm-readelf -A %p/Inputs/got-static.exe.mips | \
+# RUN:   FileCheck %s --strict-whitespace -check-prefix GNU-GOT-STATIC
+
+# GOT-EXE:      Primary GOT {
+# GOT-EXE-NEXT:   Canonical gp value: 0x418880
+# GOT-EXE-NEXT:   Reserved entries [
+# GOT-EXE-NEXT:     Entry {
+# GOT-EXE-NEXT:       Address: 0x410890
+# GOT-EXE-NEXT:       Access: -32752
+# GOT-EXE-NEXT:       Initial: 0x0
+# GOT-EXE-NEXT:       Purpose: Lazy resolver
+# GOT-EXE-NEXT:     }
+# GOT-EXE-NEXT:     Entry {
+# GOT-EXE-NEXT:       Address: 0x410894
+# GOT-EXE-NEXT:       Access: -32748
+# GOT-EXE-NEXT:       Initial: 0x80000000
+# GOT-EXE-NEXT:       Purpose: Module pointer (GNU extension)
+# GOT-EXE-NEXT:     }
+# GOT-EXE-NEXT:   ]
+# GOT-EXE-NEXT:   Local entries [
+# GOT-EXE-NEXT:     Entry {
+# GOT-EXE-NEXT:       Address: 0x410898
+# GOT-EXE-NEXT:       Access: -32744
+# GOT-EXE-NEXT:       Initial: 0x400418
+# GOT-EXE-NEXT:     }
+# GOT-EXE-NEXT:     Entry {
+# GOT-EXE-NEXT:       Address: 0x41089C
+# GOT-EXE-NEXT:       Access: -32740
+# GOT-EXE-NEXT:       Initial: 0x410840
+# GOT-EXE-NEXT:     }
+# GOT-EXE-NEXT:     Entry {
+# GOT-EXE-NEXT:       Address: 0x4108A0
+# GOT-EXE-NEXT:       Access: -32736
+# GOT-EXE-NEXT:       Initial: 0x0
+# GOT-EXE-NEXT:     }
+# GOT-EXE-NEXT:   ]
+# GOT-EXE-NEXT:   Global entries [
+# GOT-EXE-NEXT:     Entry {
+# GOT-EXE-NEXT:       Address: 0x4108A4
+# GOT-EXE-NEXT:       Access: -32732
+# GOT-EXE-NEXT:       Initial: 0x0
+# GOT-EXE-NEXT:       Value: 0x0
+# GOT-EXE-NEXT:       Type: Function (0x2)
+# GOT-EXE-NEXT:       Section: Undefined (0x0)
+# GOT-EXE-NEXT:       Name: __gmon_start__ (1)
+# GOT-EXE-NEXT:     }
+# GOT-EXE-NEXT:   ]
+# GOT-EXE-NEXT:   Number of TLS and multi-GOT entries: 0
+# GOT-EXE-NEXT: }
+
+# GOT-SO:      Primary GOT {
+# GOT-SO-NEXT:   Canonical gp value: 0x188D0
+# GOT-SO-NEXT:   Reserved entries [
+# GOT-SO-NEXT:     Entry {
+# GOT-SO-NEXT:       Address: 0x108E0
+# GOT-SO-NEXT:       Access: -32752
+# GOT-SO-NEXT:       Initial: 0x0
+# GOT-SO-NEXT:       Purpose: Lazy resolver
+# GOT-SO-NEXT:     }
+# GOT-SO-NEXT:     Entry {
+# GOT-SO-NEXT:       Address: 0x108E4
+# GOT-SO-NEXT:       Access: -32748
+# GOT-SO-NEXT:       Initial: 0x80000000
+# GOT-SO-NEXT:       Purpose: Module pointer (GNU extension)
+# GOT-SO-NEXT:     }
+# GOT-SO-NEXT:   ]
+# GOT-SO-NEXT:   Local entries [
+# GOT-SO-NEXT:     Entry {
+# GOT-SO-NEXT:       Address: 0x108E8
+# GOT-SO-NEXT:       Access: -32744
+# GOT-SO-NEXT:       Initial: 0x108E0
+# GOT-SO-NEXT:     }
+# GOT-SO-NEXT:     Entry {
+# GOT-SO-NEXT:       Address: 0x108EC
+# GOT-SO-NEXT:       Access: -32740
+# GOT-SO-NEXT:       Initial: 0x10000
+# GOT-SO-NEXT:     }
+# GOT-SO-NEXT:     Entry {
+# GOT-SO-NEXT:       Address: 0x108F0
+# GOT-SO-NEXT:       Access: -32736
+# GOT-SO-NEXT:       Initial: 0x10920
+# GOT-SO-NEXT:     }
+# GOT-SO-NEXT:     Entry {
+# GOT-SO-NEXT:       Address: 0x108F4
+# GOT-SO-NEXT:       Access: -32732
+# GOT-SO-NEXT:       Initial: 0x108CC
+# GOT-SO-NEXT:     }
+# GOT-SO-NEXT:     Entry {
+# GOT-SO-NEXT:       Address: 0x108F8
+# GOT-SO-NEXT:       Access: -32728
+# GOT-SO-NEXT:       Initial: 0x0
+# GOT-SO-NEXT:     }
+# GOT-SO-NEXT:     Entry {
+# GOT-SO-NEXT:       Address: 0x108FC
+# GOT-SO-NEXT:       Access: -32724
+# GOT-SO-NEXT:       Initial: 0x0
+# GOT-SO-NEXT:     }
+# GOT-SO-NEXT:     Entry {
+# GOT-SO-NEXT:       Address: 0x10900
+# GOT-SO-NEXT:       Access: -32720
+# GOT-SO-NEXT:       Initial: 0x0
+# GOT-SO-NEXT:     }
+# GOT-SO-NEXT:     Entry {
+# GOT-SO-NEXT:       Address: 0x10904
+# GOT-SO-NEXT:       Access: -32716
+# GOT-SO-NEXT:       Initial: 0x0
+# GOT-SO-NEXT:     }
+# GOT-SO-NEXT:   ]
+# GOT-SO-NEXT:   Global entries [
+# GOT-SO-NEXT:     Entry {
+# GOT-SO-NEXT:       Address: 0x10908
+# GOT-SO-NEXT:       Access: -32712
+# GOT-SO-NEXT:       Initial: 0x0
+# GOT-SO-NEXT:       Value: 0x0
+# GOT-SO-NEXT:       Type: None (0x0)
+# GOT-SO-NEXT:       Section: Undefined (0x0)
+# GOT-SO-NEXT:       Name: _ITM_registerTMCloneTable (87)
+# GOT-SO-NEXT:     }
+# GOT-SO-NEXT:     Entry {
+# GOT-SO-NEXT:       Address: 0x1090C
+# GOT-SO-NEXT:       Access: -32708
+# GOT-SO-NEXT:       Initial: 0x0
+# GOT-SO-NEXT:       Value: 0x0
+# GOT-SO-NEXT:       Type: None (0x0)
+# GOT-SO-NEXT:       Section: Undefined (0x0)
+# GOT-SO-NEXT:       Name: _Jv_RegisterClasses (128)
+# GOT-SO-NEXT:     }
+# GOT-SO-NEXT:     Entry {
+# GOT-SO-NEXT:       Address: 0x10910
+# GOT-SO-NEXT:       Access: -32704
+# GOT-SO-NEXT:       Initial: 0x0
+# GOT-SO-NEXT:       Value: 0x0
+# GOT-SO-NEXT:       Type: Function (0x2)
+# GOT-SO-NEXT:       Section: Undefined (0x0)
+# GOT-SO-NEXT:       Name: __gmon_start__ (23)
+# GOT-SO-NEXT:     }
+# GOT-SO-NEXT:     Entry {
+# GOT-SO-NEXT:       Address: 0x10914
+# GOT-SO-NEXT:       Access: -32700
+# GOT-SO-NEXT:       Initial: 0x840
+# GOT-SO-NEXT:       Value: 0x840
+# GOT-SO-NEXT:       Type: Function (0x2)
+# GOT-SO-NEXT:       Section: Undefined (0x0)
+# GOT-SO-NEXT:       Name: puts@GLIBC_2.0 (162)
+# GOT-SO-NEXT:     }
+# GOT-SO-NEXT:     Entry {
+# GOT-SO-NEXT:       Address: 0x10918
+# GOT-SO-NEXT:       Access: -32696
+# GOT-SO-NEXT:       Initial: 0x0
+# GOT-SO-NEXT:       Value: 0x0
+# GOT-SO-NEXT:       Type: None (0x0)
+# GOT-SO-NEXT:       Section: Undefined (0x0)
+# GOT-SO-NEXT:       Name: _ITM_deregisterTMCloneTable (59)
+# GOT-SO-NEXT:     }
+# GOT-SO-NEXT:     Entry {
+# GOT-SO-NEXT:       Address: 0x1091C
+# GOT-SO-NEXT:       Access: -32692
+# GOT-SO-NEXT:       Initial: 0x0
+# GOT-SO-NEXT:       Value: 0x0
+# GOT-SO-NEXT:       Type: Function (0x2)
+# GOT-SO-NEXT:       Section: Undefined (0x0)
+# GOT-SO-NEXT:       Name: __cxa_finalize@GLIBC_2.2 (113)
+# GOT-SO-NEXT:     }
+# GOT-SO-NEXT:   ]
+# GOT-SO-NEXT:   Number of TLS and multi-GOT entries: 0
+# GOT-SO-NEXT: }
+
+# GOT-TLS:      Primary GOT {
+# GOT-TLS-NEXT:   Canonical gp value: 0x18BF0
+# GOT-TLS-NEXT:   Reserved entries [
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C00
+# GOT-TLS-NEXT:       Access: -32752
+# GOT-TLS-NEXT:       Initial: 0x0
+# GOT-TLS-NEXT:       Purpose: Lazy resolver
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C08
+# GOT-TLS-NEXT:       Access: -32744
+# GOT-TLS-NEXT:       Initial: 0x8000000000000000
+# GOT-TLS-NEXT:       Purpose: Module pointer (GNU extension)
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:   ]
+# GOT-TLS-NEXT:   Local entries [
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C10
+# GOT-TLS-NEXT:       Access: -32736
+# GOT-TLS-NEXT:       Initial: 0x10000
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C18
+# GOT-TLS-NEXT:       Access: -32728
+# GOT-TLS-NEXT:       Initial: 0x10C00
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C20
+# GOT-TLS-NEXT:       Access: -32720
+# GOT-TLS-NEXT:       Initial: 0x10CB8
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C28
+# GOT-TLS-NEXT:       Access: -32712
+# GOT-TLS-NEXT:       Initial: 0x10BF0
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C30
+# GOT-TLS-NEXT:       Access: -32704
+# GOT-TLS-NEXT:       Initial: 0x0
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C38
+# GOT-TLS-NEXT:       Access: -32696
+# GOT-TLS-NEXT:       Initial: 0x948
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C40
+# GOT-TLS-NEXT:       Access: -32688
+# GOT-TLS-NEXT:       Initial: 0xA20
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C48
+# GOT-TLS-NEXT:       Access: -32680
+# GOT-TLS-NEXT:       Initial: 0xAF0
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C50
+# GOT-TLS-NEXT:       Access: -32672
+# GOT-TLS-NEXT:       Initial: 0x0
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C58
+# GOT-TLS-NEXT:       Access: -32664
+# GOT-TLS-NEXT:       Initial: 0x0
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C60
+# GOT-TLS-NEXT:       Access: -32656
+# GOT-TLS-NEXT:       Initial: 0x0
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:   ]
+# GOT-TLS-NEXT:   Global entries [
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C68
+# GOT-TLS-NEXT:       Access: -32648
+# GOT-TLS-NEXT:       Initial: 0x0
+# GOT-TLS-NEXT:       Value: 0x0
+# GOT-TLS-NEXT:       Type: None (0x0)
+# GOT-TLS-NEXT:       Section: Undefined (0x0)
+# GOT-TLS-NEXT:       Name: _ITM_registerTMCloneTable (78)
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C70
+# GOT-TLS-NEXT:       Access: -32640
+# GOT-TLS-NEXT:       Initial: 0x0
+# GOT-TLS-NEXT:       Value: 0x0
+# GOT-TLS-NEXT:       Type: None (0x0)
+# GOT-TLS-NEXT:       Section: Undefined (0x0)
+# GOT-TLS-NEXT:       Name: _Jv_RegisterClasses (119)
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C78
+# GOT-TLS-NEXT:       Access: -32632
+# GOT-TLS-NEXT:       Initial: 0x0
+# GOT-TLS-NEXT:       Value: 0x0
+# GOT-TLS-NEXT:       Type: Function (0x2)
+# GOT-TLS-NEXT:       Section: Undefined (0x0)
+# GOT-TLS-NEXT:       Name: __gmon_start__ (23)
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C80
+# GOT-TLS-NEXT:       Access: -32624
+# GOT-TLS-NEXT:       Initial: 0xB60
+# GOT-TLS-NEXT:       Value: 0xB60
+# GOT-TLS-NEXT:       Type: Function (0x2)
+# GOT-TLS-NEXT:       Section: Undefined (0x0)
+# GOT-TLS-NEXT:       Name: __tls_get_addr@GLIBC_2.3 (150)
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C88
+# GOT-TLS-NEXT:       Access: -32616
+# GOT-TLS-NEXT:       Initial: 0x0
+# GOT-TLS-NEXT:       Value: 0x0
+# GOT-TLS-NEXT:       Type: None (0x0)
+# GOT-TLS-NEXT:       Section: Undefined (0x0)
+# GOT-TLS-NEXT:       Name: _ITM_deregisterTMCloneTable (50)
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:     Entry {
+# GOT-TLS-NEXT:       Address: 0x10C90
+# GOT-TLS-NEXT:       Access: -32608
+# GOT-TLS-NEXT:       Initial: 0x0
+# GOT-TLS-NEXT:       Value: 0x0
+# GOT-TLS-NEXT:       Type: Function (0x2)
+# GOT-TLS-NEXT:       Section: Undefined (0x0)
+# GOT-TLS-NEXT:       Name: __cxa_finalize@GLIBC_2.2 (104)
+# GOT-TLS-NEXT:     }
+# GOT-TLS-NEXT:   ]
+# GOT-TLS-NEXT:   Number of TLS and multi-GOT entries: 4
+# GOT-TLS-NEXT: }
+
+# GOT-EMPTY:      Primary GOT {
+# GOT-EMPTY-NEXT:   Canonical gp value: 0x409FF0
+# GOT-EMPTY-NEXT:   Reserved entries [
+# GOT-EMPTY-NEXT:     Entry {
+# GOT-EMPTY-NEXT:       Address: 0x402000
+# GOT-EMPTY-NEXT:       Access: -32752
+# GOT-EMPTY-NEXT:       Initial: 0x0
+# GOT-EMPTY-NEXT:       Purpose: Lazy resolver
+# GOT-EMPTY-NEXT:     }
+# GOT-EMPTY-NEXT:     Entry {
+# GOT-EMPTY-NEXT:       Address: 0x402004
+# GOT-EMPTY-NEXT:       Access: -32748
+# GOT-EMPTY-NEXT:       Initial: 0x80000000
+# GOT-EMPTY-NEXT:       Purpose: Module pointer (GNU extension)
+# GOT-EMPTY-NEXT:     }
+# GOT-EMPTY-NEXT:   ]
+# GOT-EMPTY-NEXT:   Local entries [
+# GOT-EMPTY-NEXT:   ]
+# GOT-EMPTY-NEXT:   Global entries [
+# GOT-EMPTY-NEXT:   ]
+# GOT-EMPTY-NEXT:   Number of TLS and multi-GOT entries: 2
+# GOT-EMPTY-NEXT: }
+
+# GOT-STATIC:      Static GOT {
+# GOT-STATIC-NEXT:   Canonical gp value: 0x418100
+# GOT-STATIC-NEXT:   Reserved entries [
+# GOT-STATIC-NEXT:     Entry {
+# GOT-STATIC-NEXT:       Address: 0x410110
+# GOT-STATIC-NEXT:       Access: -32752
+# GOT-STATIC-NEXT:       Initial: 0x0
+# GOT-STATIC-NEXT:       Purpose: Lazy resolver
+# GOT-STATIC-NEXT:     }
+# GOT-STATIC-NEXT:     Entry {
+# GOT-STATIC-NEXT:       Address: 0x410114
+# GOT-STATIC-NEXT:       Access: -32748
+# GOT-STATIC-NEXT:       Initial: 0x80000000
+# GOT-STATIC-NEXT:       Purpose: Module pointer (GNU extension)
+# GOT-STATIC-NEXT:     }
+# GOT-STATIC-NEXT:   ]
+# GOT-STATIC-NEXT:   Local entries [
+# GOT-STATIC-NEXT:     Entry {
+# GOT-STATIC-NEXT:       Address: 0x410118
+# GOT-STATIC-NEXT:       Access: -32744
+# GOT-STATIC-NEXT:       Initial: 0x400000
+# GOT-STATIC-NEXT:     }
+# GOT-STATIC-NEXT:     Entry {
+# GOT-STATIC-NEXT:       Address: 0x41011C
+# GOT-STATIC-NEXT:       Access: -32740
+# GOT-STATIC-NEXT:       Initial: 0x400100
+# GOT-STATIC-NEXT:     }
+# GOT-STATIC-NEXT:     Entry {
+# GOT-STATIC-NEXT:       Address: 0x410120
+# GOT-STATIC-NEXT:       Access: -32736
+# GOT-STATIC-NEXT:       Initial: 0x400104
+# GOT-STATIC-NEXT:     }
+# GOT-STATIC-NEXT:   ]
+# GOT-STATIC-NEXT: }
+
+# GNU-GOT-EXE:      Primary GOT:
+# GNU-GOT-EXE-NEXT:  Canonical gp value: 00418880
+
+# GNU-GOT-EXE:       Reserved entries:
+# GNU-GOT-EXE-NEXT:    Address     Access  Initial Purpose
+# GNU-GOT-EXE-NEXT:   00410890 -32752(gp) 00000000 Lazy resolver
+# GNU-GOT-EXE-NEXT:   00410894 -32748(gp) 80000000 Module pointer (GNU extension)
+
+# GNU-GOT-EXE:       Local entries:
+# GNU-GOT-EXE-NEXT:    Address     Access  Initial
+# GNU-GOT-EXE-NEXT:   00410898 -32744(gp) 00400418
+# GNU-GOT-EXE-NEXT:   0041089c -32740(gp) 00410840
+# GNU-GOT-EXE-NEXT:   004108a0 -32736(gp) 00000000
+
+# GNU-GOT-EXE:       Global entries:
+# GNU-GOT-EXE-NEXT:    Address     Access  Initial Sym.Val. Type    Ndx Name
+# GNU-GOT-EXE-NEXT:   004108a4 -32732(gp) 00000000 00000000 FUNC    UND __gmon_start__
+
+# GNU-GOT-EXE:      PLT GOT:
+
+# GNU-GOT-EXE:       Reserved entries:
+# GNU-GOT-EXE-NEXT:    Address  Initial Purpose
+# GNU-GOT-EXE-NEXT:   00410854 00000000 PLT lazy resolver
+# GNU-GOT-EXE-NEXT:   00410858 00000000 Module pointer
+
+# GNU-GOT-EXE:       Entries:
+# GNU-GOT-EXE-NEXT:    Address  Initial Sym.Val. Type    Ndx Name
+# GNU-GOT-EXE-NEXT:   0041085c 00400800 00000000 FUNC    UND puts
+# GNU-GOT-EXE-NEXT:   00410860 00400800 00000000 FUNC    UND __libc_start_main
+
+# GNU-GOT-SO:      Primary GOT:
+# GNU-GOT-SO-NEXT:  Canonical gp value: 000188d0
+
+# GNU-GOT-SO:       Reserved entries:
+# GNU-GOT-SO-NEXT:    Address     Access  Initial Purpose
+# GNU-GOT-SO-NEXT:   000108e0 -32752(gp) 00000000 Lazy resolver
+# GNU-GOT-SO-NEXT:   000108e4 -32748(gp) 80000000 Module pointer (GNU extension)
+
+# GNU-GOT-SO:       Local entries:
+# GNU-GOT-SO-NEXT:    Address     Access  Initial
+# GNU-GOT-SO-NEXT:   000108e8 -32744(gp) 000108e0
+# GNU-GOT-SO-NEXT:   000108ec -32740(gp) 00010000
+# GNU-GOT-SO-NEXT:   000108f0 -32736(gp) 00010920
+# GNU-GOT-SO-NEXT:   000108f4 -32732(gp) 000108cc
+# GNU-GOT-SO-NEXT:   000108f8 -32728(gp) 00000000
+# GNU-GOT-SO-NEXT:   000108fc -32724(gp) 00000000
+# GNU-GOT-SO-NEXT:   00010900 -32720(gp) 00000000
+# GNU-GOT-SO-NEXT:   00010904 -32716(gp) 00000000
+
+# GNU-GOT-SO:       Global entries:
+# GNU-GOT-SO-NEXT:    Address     Access  Initial Sym.Val. Type    Ndx Name
+# GNU-GOT-SO-NEXT:   00010908 -32712(gp) 00000000 00000000 NOTYPE  UND _ITM_registerTMCloneTable
+# GNU-GOT-SO-NEXT:   0001090c -32708(gp) 00000000 00000000 NOTYPE  UND _Jv_RegisterClasses
+# GNU-GOT-SO-NEXT:   00010910 -32704(gp) 00000000 00000000 FUNC    UND __gmon_start__
+# GNU-GOT-SO-NEXT:   00010914 -32700(gp) 00000840 00000840 FUNC    UND puts
+# GNU-GOT-SO-NEXT:   00010918 -32696(gp) 00000000 00000000 NOTYPE  UND _ITM_deregisterTMCloneTable
+# GNU-GOT-SO-NEXT:   0001091c -32692(gp) 00000000 00000000 FUNC    UND __cxa_finalize
+
+# GNU-GOT-TLS:      Primary GOT:
+# GNU-GOT-TLS-NEXT:  Canonical gp value: 0000000000018bf0
+
+# GNU-GOT-TLS:       Reserved entries:
+# GNU-GOT-TLS-NEXT:            Address     Access          Initial Purpose
+# GNU-GOT-TLS-NEXT:   0000000000010c00 -32752(gp) 0000000000000000 Lazy resolver
+# GNU-GOT-TLS-NEXT:   0000000000010c08 -32744(gp) 8000000000000000 Module pointer (GNU extension)
+
+# GNU-GOT-TLS:       Local entries:
+# GNU-GOT-TLS-NEXT:            Address     Access          Initial
+# GNU-GOT-TLS-NEXT:   0000000000010c10 -32736(gp) 0000000000010000
+# GNU-GOT-TLS-NEXT:   0000000000010c18 -32728(gp) 0000000000010c00
+# GNU-GOT-TLS-NEXT:   0000000000010c20 -32720(gp) 0000000000010cb8
+# GNU-GOT-TLS-NEXT:   0000000000010c28 -32712(gp) 0000000000010bf0
+# GNU-GOT-TLS-NEXT:   0000000000010c30 -32704(gp) 0000000000000000
+# GNU-GOT-TLS-NEXT:   0000000000010c38 -32696(gp) 0000000000000948
+# GNU-GOT-TLS-NEXT:   0000000000010c40 -32688(gp) 0000000000000a20
+# GNU-GOT-TLS-NEXT:   0000000000010c48 -32680(gp) 0000000000000af0
+# GNU-GOT-TLS-NEXT:   0000000000010c50 -32672(gp) 0000000000000000
+# GNU-GOT-TLS-NEXT:   0000000000010c58 -32664(gp) 0000000000000000
+# GNU-GOT-TLS-NEXT:   0000000000010c60 -32656(gp) 0000000000000000
+
+# GNU-GOT-TLS:       Global entries:
+# GNU-GOT-TLS-NEXT:            Address     Access          Initial         Sym.Val. Type    Ndx Name
+# GNU-GOT-TLS-NEXT:   0000000000010c68 -32648(gp) 0000000000000000 0000000000000000 NOTYPE  UND _ITM_registerTMCloneTable
+# GNU-GOT-TLS-NEXT:   0000000000010c70 -32640(gp) 0000000000000000 0000000000000000 NOTYPE  UND _Jv_RegisterClasses
+# GNU-GOT-TLS-NEXT:   0000000000010c78 -32632(gp) 0000000000000000 0000000000000000 FUNC    UND __gmon_start__
+# GNU-GOT-TLS-NEXT:   0000000000010c80 -32624(gp) 0000000000000b60 0000000000000b60 FUNC    UND __tls_get_addr
+# GNU-GOT-TLS-NEXT:   0000000000010c88 -32616(gp) 0000000000000000 0000000000000000 NOTYPE  UND _ITM_deregisterTMCloneTable
+# GNU-GOT-TLS-NEXT:   0000000000010c90 -32608(gp) 0000000000000000 0000000000000000 FUNC    UND __cxa_finalize
+
+# GNU-GOTY     : Primary GOT:
+# GNU-GOT-EMPTY:  Canonical gp value: 00409ff0
+
+# GNU-GOTY     :  Reserved entries:
+# GNU-GOT-EMPTY:    Address     Access  Initial Purpose
+# GNU-GOT-EMPTY:   00402000 -32752(gp) 00000000 Lazy resolver
+# GNU-GOT-EMPTY:   00402004 -32748(gp) 80000000 Module pointer (GNU extension)
+
+# GNU-GOT-STATIC:      Static GOT:
+# GNU-GOT-STATIC-NEXT:  Canonical gp value: 00418100
+
+# GNU-GOT-STATIC:       Reserved entries:
+# GNU-GOT-STATIC-NEXT:    Address     Access  Initial Purpose
+# GNU-GOT-STATIC-NEXT:   00410110 -32752(gp) 00000000 Lazy resolver
+# GNU-GOT-STATIC-NEXT:   00410114 -32748(gp) 80000000 Module pointer (GNU extension)
+
+# GNU-GOT-STATIC:       Local entries:
+# GNU-GOT-STATIC-NEXT:    Address     Access  Initial
+# GNU-GOT-STATIC-NEXT:   00410118 -32744(gp) 00400000
+# GNU-GOT-STATIC-NEXT:   0041011c -32740(gp) 00400100
+# GNU-GOT-STATIC-NEXT:   00410120 -32736(gp) 00400104
+
+## Check we report errors when dynamic tags, needed for dumping GOT, are missing.
+
+# RUN: yaml2obj --docnum=1 -DTAG1=DT_MIPS_LOCAL_GOTNO -DTAG2=DT_MIPS_GOTSYM %s -o %t.err1.o
+# RUN: not llvm-readobj -A %t.err1.o 2>&1 | FileCheck %s -DFILE=%t.err1.o -check-prefix ERR1
+
+# ERR1: error: '[[FILE]]': cannot find PLTGOT dynamic tag
+
+# RUN: yaml2obj --docnum=1 -DTAG1=DT_PLTGOT -DTAG2=DT_MIPS_GOTSYM %s -o %t.err2.o
+# RUN: not llvm-readobj -A %t.err2.o 2>&1 | FileCheck %s -DFILE=%t.err2.o -check-prefix ERR2
+
+# ERR2: error: '[[FILE]]': cannot find MIPS_LOCAL_GOTNO dynamic tag
+
+# RUN: yaml2obj --docnum=1 -DTAG1=DT_PLTGOT -DTAG2=DT_MIPS_LOCAL_GOTNO %s -o %t.err3.o
+# RUN: not llvm-readobj -A %t.err3.o 2>&1 | FileCheck %s -DFILE=%t.err3.o -check-prefix ERR3
+
+# ERR3: error: '[[FILE]]': cannot find MIPS_GOTSYM dynamic tag
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_MIPS
+Sections:
+  - Name: .dynamic
+    Type: SHT_DYNAMIC
+    Entries:
+      - Tag:   [[TAG1]]
+        Value: 0
+      - Tag:   [[TAG2]]
+        Value: 0
+      - Tag:   DT_NULL
+        Value: 0
+DynamicSymbols: []
+
+# RUN: yaml2obj --docnum=2 -DVAL1=0xffff %s -o %t.err4.o
+# RUN: not llvm-readobj -A %t.err4.o 2>&1 | FileCheck %s -DFILE=%t.err4.o -check-prefix=ERR4
+
+# ERR4: error: '[[FILE]]': DT_MIPS_GOTSYM value (65535) exceeds the number of dynamic symbols (1)
+
+# RUN: yaml2obj --docnum=2 -DVAL2=0xffff %s -o %t.err5.o
+# RUN: not llvm-readobj -A %t.err5.o 2>&1 | FileCheck %s -DFILE=%t.err5.o -check-prefix=ERR5
+
+# ERR5: error: '[[FILE]]': there is no non-empty GOT section at 0xffff
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_MIPS
+Sections:
+  - Name: .dynamic
+    Type: SHT_DYNAMIC
+    Entries:
+      - Tag:   DT_MIPS_LOCAL_GOTNO
+        Value: 0
+      - Tag:   DT_MIPS_GOTSYM
+        Value: [[VAL1=0]]
+      - Tag:   DT_PLTGOT
+        Value: [[VAL2=0]]
+DynamicSymbols: []
diff --git a/llvm/test/tools/llvm-readobj/ELF/mips-plt.test b/llvm/test/tools/llvm-readobj/ELF/mips-plt.test
index 4e40ca6aa2c13..4959d8892ac2b 100644
--- a/llvm/test/tools/llvm-readobj/ELF/mips-plt.test
+++ b/llvm/test/tools/llvm-readobj/ELF/mips-plt.test
@@ -1,64 +1,142 @@
-RUN: llvm-readobj -A %p/Inputs/got-plt.exe.elf-mipsel | FileCheck %s
-RUN: llvm-readelf -A %p/Inputs/got-plt.exe.elf-mipsel | FileCheck --check-prefix=GNU %s
-
-CHECK:      PLT GOT {
-CHECK-NEXT:   Reserved entries [
-CHECK-NEXT:     Entry {
-CHECK-NEXT:       Address: 0x410814
-CHECK-NEXT:       Initial: 0x0
-CHECK-NEXT:       Purpose: PLT lazy resolver
-CHECK-NEXT:     }
-CHECK-NEXT:     Entry {
-CHECK-NEXT:       Address: 0x410818
-CHECK-NEXT:       Initial: 0x0
-CHECK-NEXT:       Purpose: Module pointer
-CHECK-NEXT:     }
-CHECK-NEXT:   ]
-CHECK-NEXT:   Entries [
-CHECK-NEXT:     Entry {
-CHECK-NEXT:       Address: 0x41081C
-CHECK-NEXT:       Initial: 0x4007C0
-CHECK-NEXT:       Value: 0x0
-CHECK-NEXT:       Type: Function (0x2)
-CHECK-NEXT:       Section: Undefined (0x0)
-CHECK-NEXT:       Name: puts@GLIBC_2.0 (71)
-CHECK-NEXT:     }
-CHECK-NEXT:     Entry {
-CHECK-NEXT:       Address: 0x410820
-CHECK-NEXT:       Initial: 0x4007C0
-CHECK-NEXT:       Value: 0x0
-CHECK-NEXT:       Type: Function (0x2)
-CHECK-NEXT:       Section: Undefined (0x0)
-CHECK-NEXT:       Name: __libc_start_main@GLIBC_2.0 (53)
-CHECK-NEXT:     }
-CHECK-NEXT:   ]
-CHECK-NEXT: }
-
-GNU:      Primary GOT:
-GNU-NEXT:  Canonical gp value: 00418840
-
-GNU:       Reserved entries:
-GNU-NEXT:    Address     Access  Initial Purpose
-GNU-NEXT:   00410850 -32752(gp) 00000000 Lazy resolver
-GNU-NEXT:   00410854 -32748(gp) 80000000 Module pointer (GNU extension)
-
-GNU:       Local entries:
-GNU-NEXT:    Address     Access  Initial
-GNU-NEXT:   00410858 -32744(gp) 004003d4
-GNU-NEXT:   0041085c -32740(gp) 00410800
-GNU-NEXT:   00410860 -32736(gp) 00000000
-
-GNU:       Global entries:
-GNU-NEXT:    Address     Access  Initial Sym.Val. Type    Ndx Name
-GNU-NEXT:   00410864 -32732(gp) 00000000 00000000 FUNC    UND __gmon_start__
-GNU-NEXT: PLT GOT:
-
-GNU:       Reserved entries:
-GNU-NEXT:    Address  Initial Purpose
-GNU-NEXT:   00410814 00000000 PLT lazy resolver
-GNU-NEXT:   00410818 00000000 Module pointer
-
-GNU:       Entries:
-GNU-NEXT:    Address  Initial Sym.Val. Type    Ndx Name
-GNU-NEXT:   0041081c 004007c0 00000000 FUNC    UND puts
-GNU-NEXT:   00410820 004007c0 00000000 FUNC    UND __libc_start_main
+# RUN: llvm-readobj -A %p/Inputs/got-plt.exe.elf-mipsel | FileCheck %s
+# RUN: llvm-readelf -A %p/Inputs/got-plt.exe.elf-mipsel | FileCheck --check-prefix=GNU %s
+
+# CHECK:      PLT GOT {
+# CHECK-NEXT:   Reserved entries [
+# CHECK-NEXT:     Entry {
+# CHECK-NEXT:       Address: 0x410814
+# CHECK-NEXT:       Initial: 0x0
+# CHECK-NEXT:       Purpose: PLT lazy resolver
+# CHECK-NEXT:     }
+# CHECK-NEXT:     Entry {
+# CHECK-NEXT:       Address: 0x410818
+# CHECK-NEXT:       Initial: 0x0
+# CHECK-NEXT:       Purpose: Module pointer
+# CHECK-NEXT:     }
+# CHECK-NEXT:   ]
+# CHECK-NEXT:   Entries [
+# CHECK-NEXT:     Entry {
+# CHECK-NEXT:       Address: 0x41081C
+# CHECK-NEXT:       Initial: 0x4007C0
+# CHECK-NEXT:       Value: 0x0
+# CHECK-NEXT:       Type: Function (0x2)
+# CHECK-NEXT:       Section: Undefined (0x0)
+# CHECK-NEXT:       Name: puts@GLIBC_2.0 (71)
+# CHECK-NEXT:     }
+# CHECK-NEXT:     Entry {
+# CHECK-NEXT:       Address: 0x410820
+# CHECK-NEXT:       Initial: 0x4007C0
+# CHECK-NEXT:       Value: 0x0
+# CHECK-NEXT:       Type: Function (0x2)
+# CHECK-NEXT:       Section: Undefined (0x0)
+# CHECK-NEXT:       Name: __libc_start_main@GLIBC_2.0 (53)
+# CHECK-NEXT:     }
+# CHECK-NEXT:   ]
+# CHECK-NEXT: }
+
+# GNU:      Primary GOT:
+# GNU-NEXT:  Canonical gp value: 00418840
+
+# GNU:       Reserved entries:
+# GNU-NEXT:    Address     Access  Initial Purpose
+# GNU-NEXT:   00410850 -32752(gp) 00000000 Lazy resolver
+# GNU-NEXT:   00410854 -32748(gp) 80000000 Module pointer (GNU extension)
+
+# GNU:       Local entries:
+# GNU-NEXT:    Address     Access  Initial
+# GNU-NEXT:   00410858 -32744(gp) 004003d4
+# GNU-NEXT:   0041085c -32740(gp) 00410800
+# GNU-NEXT:   00410860 -32736(gp) 00000000
+
+# GNU:       Global entries:
+# GNU-NEXT:    Address     Access  Initial Sym.Val. Type    Ndx Name
+# GNU-NEXT:   00410864 -32732(gp) 00000000 00000000 FUNC    UND __gmon_start__
+# GNU-NEXT: PLT GOT:
+
+# GNU:       Reserved entries:
+# GNU-NEXT:    Address  Initial Purpose
+# GNU-NEXT:   00410814 00000000 PLT lazy resolver
+# GNU-NEXT:   00410818 00000000 Module pointer
+
+# GNU:       Entries:
+# GNU-NEXT:    Address  Initial Sym.Val. Type    Ndx Name
+# GNU-NEXT:   0041081c 004007c0 00000000 FUNC    UND puts
+# GNU-NEXT:   00410820 004007c0 00000000 FUNC    UND __libc_start_main
+
+## Check we report errors when dynamic tags, needed for dumping PLT, are missing.
+
+# RUN: yaml2obj --docnum=1 -DTAG=DT_MIPS_PLTGOT %s -o %t.err1.o
+# RUN: not llvm-readobj -A %t.err1.o 2>&1 | FileCheck %s -DFILE=%t.err1.o --check-prefix=ERR1
+
+# ERR1: error: '[[FILE]]': cannot find JMPREL dynamic tag
+
+# RUN: yaml2obj --docnum=1 -DTAG=DT_JMPREL %s -o %t.err2.o
+# RUN: not llvm-readobj -A %t.err2.o 2>&1 | FileCheck %s -DFILE=%t.err2.o --check-prefix=ERR2
+
+# ERR2: error: '[[FILE]]': cannot find MIPS_PLTGOT dynamic tag
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_MIPS
+Sections:
+  - Name: .dynamic
+    Type: SHT_DYNAMIC
+    Entries:
+      - Tag:   [[TAG]]
+        Value: 0
+      - Tag:   DT_NULL
+        Value: 0
+
+## Check we report errors when we are unable to find PLTGOT/JMPREL sections.
+# RUN: yaml2obj --docnum=2 %s -DVAL1=0xffff -o %t.err3.o
+# RUN: not llvm-readobj -A %t.err3.o 2>&1 | FileCheck %s -DFILE=%t.err3.o -check-prefix ERR3
+
+# ERR3: error: '[[FILE]]': there is no non-empty PLTGOT section at 0xffff
+
+# RUN: yaml2obj --docnum=2 %s -DVAL2=0xffff -o %t.err4.o
+# RUN: not llvm-readobj -A %t.err4.o 2>&1 | FileCheck %s -DFILE=%t.err4.o -check-prefix ERR4
+
+# ERR4: error: '[[FILE]]': there is no non-empty RELPLT section at 0xffff
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_MIPS
+Sections:
+  - Name: .dynamic
+    Type: SHT_DYNAMIC
+    Entries:
+      - Tag:   DT_MIPS_PLTGOT
+        Value: [[VAL1=0]]
+      - Tag:   DT_JMPREL
+        Value: [[VAL2=0]]
+      - Tag:   DT_NULL
+        Value: 0
+  - Name:    .foo
+    Type:    SHT_PROGBITS
+    Address: 0x100
+    ShSize:  0xffffffff
+    Link:    [[LINK=0x1]]
+DynamicSymbols: []
+
+## Check we report errors when we are unable to dump PLTGOT properly.
+
+# RUN: yaml2obj --docnum=2 -DVAL1=0x100 %s -o %t.err5.o
+# RUN: not llvm-readobj -A %t.err5.o 2>&1 | FileCheck %s -DFILE=%t.err5.o -check-prefix ERR5
+
+# ERR5: error: '[[FILE]]': unable to read PLTGOT section content: section [index 2] has a sh_offset (0x70) + sh_size (0xffffffff) that is greater than the file size (0x280)
+
+# RUN: yaml2obj --docnum=2 -DVAL2=0x100 -DLINK=0xaaaaaaaa %s -o %t.err6.o
+# RUN: not llvm-readobj -A %t.err6.o 2>&1 | FileCheck %s -DFILE=%t.err6.o -check-prefix ERR6
+
+# ERR6: error: '[[FILE]]': unable to get a symbol table linked to the RELPLT section with index 2: invalid section index: 2863311530
+
+# RUN: yaml2obj --docnum=2 -DVAL2=0x100 %s -o %t.err7.o
+# RUN: not llvm-readobj -A %t.err7.o 2>&1 | FileCheck %s -DFILE=%t.err7.o -check-prefix ERR7
+
+# ERR7: error: '[[FILE]]': unable to get a string table for the symbol table with index 1: invalid sh_type for symbol table, expected SHT_SYMTAB or SHT_DYNSYM
diff --git a/llvm/test/tools/llvm-readobj/ELF/symbols.test b/llvm/test/tools/llvm-readobj/ELF/symbols.test
index a8feaba97c303..f9e4875a2875d 100644
--- a/llvm/test/tools/llvm-readobj/ELF/symbols.test
+++ b/llvm/test/tools/llvm-readobj/ELF/symbols.test
@@ -10,17 +10,17 @@
 
 ## Case 1: Test how llvm-readobj and llvm-readelf dumps symbols.
 ## a) Check relocatable objects.
-# RUN: yaml2obj %s --docnum=1 -o %t64
+# RUN: yaml2obj %s -DBITS=64 -DTYPE=ET_REL -o %t64
 # RUN: llvm-readobj --symbols %t64 | FileCheck %s --match-full-lines --strict-whitespace --check-prefix=SYMBOLS-LLVM
 # RUN: llvm-readelf --symbols %t64 | FileCheck %s --match-full-lines --strict-whitespace --check-prefix=SYMBOLS-GNU64
-# RUN: yaml2obj %s --docnum=2 -o %t32
+# RUN: yaml2obj %s -DBITS=32 -DTYPE=ET_REL -o %t32
 # RUN: llvm-readobj --symbols %t32 | FileCheck %s --match-full-lines --strict-whitespace --check-prefix=SYMBOLS-LLVM
 # RUN: llvm-readelf --symbols %t32 | FileCheck %s --match-full-lines --strict-whitespace --check-prefix=SYMBOLS-GNU32
 ## b) Check dynamic objects.
-# RUN: yaml2obj %s --docnum=3 -o %t64.so
+# RUN: yaml2obj %s -DBITS=64 -DTYPE=ET_DYN -o %t64.so
 # RUN: llvm-readobj --symbols %t64.so | FileCheck %s --match-full-lines --strict-whitespace --check-prefix=SYMBOLS-LLVM
 # RUN: llvm-readelf --symbols %t64.so | FileCheck %s --match-full-lines --strict-whitespace --check-prefix=SYMBOLS-GNU64
-# RUN: yaml2obj %s --docnum=4 -o %t32.so
+# RUN: yaml2obj %s -DBITS=32 -DTYPE=ET_DYN -o %t32.so
 # RUN: llvm-readobj --symbols %t32.so | FileCheck %s --match-full-lines --strict-whitespace --check-prefix=SYMBOLS-LLVM
 # RUN: llvm-readelf --symbols %t32.so | FileCheck %s --match-full-lines --strict-whitespace --check-prefix=SYMBOLS-GNU32
 
@@ -105,9 +105,9 @@
 
 --- !ELF
 FileHeader:
-  Class:   ELFCLASS64
+  Class:   ELFCLASS[[BITS]]
   Data:    ELFDATA2LSB
-  Type:    ET_REL
+  Type:    [[TYPE]]
   Machine: EM_X86_64
 Symbols:
   - Name:  foo
@@ -117,44 +117,113 @@ Symbols:
 DynamicSymbols:
   - Name:  zed
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS32
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_386
-Symbols:
-  - Name:  foo
-    Value: 0x1
-  - Name:  bar
-    Value: 0x2
-DynamicSymbols:
-  - Name:  zed
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_X86_64
-Symbols:
-  - Name:  foo
-    Value: 0x1
-  - Name:  bar
-    Value: 0x2
-DynamicSymbols:
-  - Name:  zed
+## Check the behavior when we are unable to print symbols due to an error.
+
+## Case 1: check we are able to dump symbols even when can't get the string table for the
+## SHT_SYMTAB section because of invalid sh_link value.
+# RUN: yaml2obj --docnum=2 -DLINK=0xff %s -o %t64.err1
+# RUN: llvm-readobj --symbols %t64.err1 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t64.err1 --check-prefix=STRTAB-LINK-ERR-LLVM
+# RUN: llvm-readelf --symbols %t64.err1 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t64.err1 --check-prefix=STRTAB-LINK-ERR-GNU
+
+# STRTAB-LINK-ERR-LLVM:      Symbols [
+# STRTAB-LINK-ERR-LLVM-NEXT: warning: '[[FILE]]': unable to get the string table for the SHT_SYMTAB section: invalid section index: 255
+# STRTAB-LINK-ERR-LLVM-NEXT:   Symbol {
+# STRTAB-LINK-ERR-LLVM-NEXT:     Name: <?> (0)
+# STRTAB-LINK-ERR-LLVM-NEXT:     Value: 0x0
+# STRTAB-LINK-ERR-LLVM-NEXT:     Size: 0
+# STRTAB-LINK-ERR-LLVM-NEXT:     Binding: Local (0x0)
+# STRTAB-LINK-ERR-LLVM-NEXT:     Type: None (0x0)
+# STRTAB-LINK-ERR-LLVM-NEXT:     Other: 0
+# STRTAB-LINK-ERR-LLVM-NEXT:     Section: Undefined (0x0)
+# STRTAB-LINK-ERR-LLVM-NEXT:   }
+# STRTAB-LINK-ERR-LLVM-NEXT:   Symbol {
+# STRTAB-LINK-ERR-LLVM-NEXT:     Name: <?> (1)
+# STRTAB-LINK-ERR-LLVM-NEXT:     Value: 0x1
+# STRTAB-LINK-ERR-LLVM-NEXT:     Size: 0
+# STRTAB-LINK-ERR-LLVM-NEXT:     Binding: Global (0x1)
+# STRTAB-LINK-ERR-LLVM-NEXT:     Type: None (0x0)
+# STRTAB-LINK-ERR-LLVM-NEXT:     Other: 0
+# STRTAB-LINK-ERR-LLVM-NEXT:     Section: .symtab (0x1)
+# STRTAB-LINK-ERR-LLVM-NEXT:   }
+# STRTAB-LINK-ERR-LLVM-NEXT: ]
+
+# STRTAB-LINK-ERR-GNU:      warning: '[[FILE]]': unable to get the string table for the SHT_SYMTAB section: invalid section index: 255
+# STRTAB-LINK-ERR-GNU:      Symbol table '.symtab' contains 2 entries:
+# STRTAB-LINK-ERR-GNU-NEXT:    Num:    Value          Size Type    Bind   Vis       Ndx Name
+# STRTAB-LINK-ERR-GNU-NEXT:      0: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT   UND <?>
+# STRTAB-LINK-ERR-GNU-NEXT:      1: 0000000000000001     0 NOTYPE  GLOBAL DEFAULT     1 <?>
 
 --- !ELF
 FileHeader:
-  Class:   ELFCLASS32
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_386
+  Class:     ELFCLASS64
+  Data:      ELFDATA2LSB
+  Type:      ET_REL
+  Machine:   EM_X86_64
+  SHStrNdx:  [[SHSTRTAB=2]]
+Sections:
+  - Name:    .symtab
+    Type:    SHT_SYMTAB
+    Link:    [[LINK=.strtab]]
+    EntSize: [[ENTSIZE=0x18]]
+## Define the .shstrtab section to reveal its position.
+  - Name: .shstrtab
+    Type: SHT_STRTAB
 Symbols:
-  - Name:  foo
+  - Name: foo
     Value: 0x1
-  - Name:  bar
-    Value: 0x2
-DynamicSymbols:
-  - Name:  zed
+    Binding: STB_GLOBAL
+    Index: 1
+
+## Case 2: check we report a warning when we are unable to read symbols
+##         from the the SHT_SYMTAB section.
+# RUN: yaml2obj --docnum=2 -DENTSIZE=0xFF %s -o %t64.err2
+# RUN: llvm-readobj --symbols %t64.err2 2>&1 | \
+# RUN:  FileCheck %s -DFILE=%t64.err2 --check-prefix=SYMTAB-ENTSIZE-ERR-LLVM
+# RUN: llvm-readelf --symbols %t64.err2 2>&1 | \
+# RUN:  FileCheck %s -DFILE=%t64.err2 --check-prefix=SYMTAB-ENTSIZE-ERR-GNU
+
+# SYMTAB-ENTSIZE-ERR-LLVM:      Symbols [
+# SYMTAB-ENTSIZE-ERR-LLVM-NEXT:   warning: '[[FILE]]': unable to read symbols from the SHT_SYMTAB section: section [index 1] has an invalid sh_entsize: 255
+# SYMTAB-ENTSIZE-ERR-LLVM:      ]
+
+# SYMTAB-ENTSIZE-ERR-GNU:     warning: '[[FILE]]': unable to read symbols from the SHT_SYMTAB section: section [index 1] has an invalid sh_entsize: 255
+# SYMTAB-ENTSIZE-ERR-GNU-NOT: {{.}}
+
+## Case 3: check we report a warning, but continue dumping, when we are unable to read the name of the SHT_SYMTAB section.
+##         In this case we set the e_shstrndx field to an invalid index so that the .shstrtab section can't be located.
+# RUN: yaml2obj --docnum=2 -DSHSTRTAB=0xff %s -o %t64.err3
+# RUN: llvm-readobj --symbols %t64.err3 2>&1 | \
+# RUN:  FileCheck %s -DFILE=%t64.err3 --check-prefix=SYMTAB-SHSTRTAB-ERR-LLVM
+# RUN: llvm-readelf --symbols %t64.err3 2>&1 | \
+# RUN:  FileCheck %s -DFILE=%t64.err3 --check-prefix=SYMTAB-SHSTRTAB-ERR-GNU
+
+# SYMTAB-SHSTRTAB-ERR-LLVM:      Symbols [
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT: warning: '[[FILE]]': unable to get the name of the SHT_SYMTAB section: section header string table index 255 does not exist
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:   Symbol {
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:     Name:  (0)
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:     Value: 0x0
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:     Size: 0
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:     Binding: Local (0x0)
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:     Type: None (0x0)
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:     Other: 0
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:     Section: Undefined (0x0)
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:   }
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:   Symbol {
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:     Name: foo (1)
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:     Value: 0x1
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:     Size: 0
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:     Binding: Global (0x1)
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:     Type: None (0x0)
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:     Other: 0
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT: warning: '[[FILE]]': section header string table index 255 does not exist
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:     Section: <?> (0x1)
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:   }
+# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT: ]
+
+# SYMTAB-SHSTRTAB-ERR-GNU:      warning: '[[FILE]]': unable to get the name of the SHT_SYMTAB section: section header string table index 255 does not exist
+# SYMTAB-SHSTRTAB-ERR-GNU:      Symbol table '<?>' contains 2 entries:
+# SYMTAB-SHSTRTAB-ERR-GNU-NEXT:    Num:    Value          Size Type    Bind   Vis       Ndx Name
+# SYMTAB-SHSTRTAB-ERR-GNU-NEXT:      0: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT   UND
+# SYMTAB-SHSTRTAB-ERR-GNU-NEXT:      1: 0000000000000001     0 NOTYPE  GLOBAL DEFAULT     1 foo
diff --git a/llvm/test/tools/llvm-readobj/ELF/verneed-flags.yaml b/llvm/test/tools/llvm-readobj/ELF/verneed-flags.yaml
index eedc7fe7ad7a4..685acfbd696a2 100644
--- a/llvm/test/tools/llvm-readobj/ELF/verneed-flags.yaml
+++ b/llvm/test/tools/llvm-readobj/ELF/verneed-flags.yaml
@@ -13,37 +13,52 @@
 # LLVM-VERDEF-NEXT:     Entries [
 # LLVM-VERDEF-NEXT:       Entry {
 # LLVM-VERDEF-NEXT:         Hash: 0
-# LLVM-VERDEF-NEXT:         Flags: Base (0x1)
+# LLVM-VERDEF-NEXT:         Flags [ (0x1)
+# LLVM-VERDEF-NEXT:           Base (0x1)
+# LLVM-VERDEF-NEXT:         ]
 # LLVM-VERDEF-NEXT:         Index: 0
 # LLVM-VERDEF-NEXT:         Name: base
 # LLVM-VERDEF-NEXT:       }
 # LLVM-VERDEF-NEXT:       Entry {
 # LLVM-VERDEF-NEXT:         Hash: 0
-# LLVM-VERDEF-NEXT:         Flags: Weak (0x2)
+# LLVM-VERDEF-NEXT:         Flags [ (0x2)
+# LLVM-VERDEF-NEXT:           Weak (0x2)
+# LLVM-VERDEF-NEXT:         ]
 # LLVM-VERDEF-NEXT:         Index: 0
 # LLVM-VERDEF-NEXT:         Name: weak
 # LLVM-VERDEF-NEXT:       }
 # LLVM-VERDEF-NEXT:       Entry {
 # LLVM-VERDEF-NEXT:         Hash: 0
-# LLVM-VERDEF-NEXT:         Flags: Info (0x4)
+# LLVM-VERDEF-NEXT:         Flags [ (0x4)
+# LLVM-VERDEF-NEXT:           Info (0x4)
+# LLVM-VERDEF-NEXT:         ]
 # LLVM-VERDEF-NEXT:         Index: 0
 # LLVM-VERDEF-NEXT:         Name: info
 # LLVM-VERDEF-NEXT:       }
 # LLVM-VERDEF-NEXT:       Entry {
 # LLVM-VERDEF-NEXT:         Hash: 0
-# LLVM-VERDEF-NEXT:         Flags: 0x7
+# LLVM-VERDEF-NEXT:         Flags [ (0x7)
+# LLVM-VERDEF-NEXT:           Base (0x1)
+# LLVM-VERDEF-NEXT:           Info (0x4)
+# LLVM-VERDEF-NEXT:           Weak (0x2)
+# LLVM-VERDEF-NEXT:         ]
 # LLVM-VERDEF-NEXT:         Index: 0
 # LLVM-VERDEF-NEXT:         Name: all
 # LLVM-VERDEF-NEXT:       }
 # LLVM-VERDEF-NEXT:       Entry {
 # LLVM-VERDEF-NEXT:         Hash: 0
-# LLVM-VERDEF-NEXT:         Flags: 0x8
+# LLVM-VERDEF-NEXT:         Flags [ (0x8)
+# LLVM-VERDEF-NEXT:         ]
 # LLVM-VERDEF-NEXT:         Index: 0
 # LLVM-VERDEF-NEXT:         Name: unknown
 # LLVM-VERDEF-NEXT:       }
 # LLVM-VERDEF-NEXT:       Entry {
 # LLVM-VERDEF-NEXT:         Hash: 0
-# LLVM-VERDEF-NEXT:         Flags: 0xF
+# LLVM-VERDEF-NEXT:         Flags [ (0xF)
+# LLVM-VERDEF-NEXT:           Base (0x1)
+# LLVM-VERDEF-NEXT:           Info (0x4)
+# LLVM-VERDEF-NEXT:           Weak (0x2)
+# LLVM-VERDEF-NEXT:         ]
 # LLVM-VERDEF-NEXT:         Index: 0
 # LLVM-VERDEF-NEXT:         Name: all_and_unknown
 # LLVM-VERDEF-NEXT:       }
@@ -52,7 +67,7 @@
 # LLVM-VERDEF-NEXT: ]
 
 # GNU-VERDEF:      Version needs section '.gnu.version_r' contains 1 entries:
-# GNU-VERDEF-NEXT:  Addr: 0000000000000000  Offset: 0x000200  Link: 6 (.dynstr)
+# GNU-VERDEF-NEXT:  Addr: 0000000000000000  Offset: 0x000040  Link: 3 (.dynstr)
 # GNU-VERDEF-NEXT:   0x0000: Version: 1  File: dso.so.0  Cnt: 6
 # GNU-VERDEF-NEXT:   0x0010:   Name: base  Flags: BASE  Version: 0
 # GNU-VERDEF-NEXT:   0x0020:   Name: weak  Flags: WEAK  Version: 0
diff --git a/llvm/test/tools/llvm-readobj/ELF/versym-invalid.test b/llvm/test/tools/llvm-readobj/ELF/versym-invalid.test
index 7151a905b1d1d..d495b1cfd0630 100644
--- a/llvm/test/tools/llvm-readobj/ELF/versym-invalid.test
+++ b/llvm/test/tools/llvm-readobj/ELF/versym-invalid.test
@@ -115,13 +115,39 @@ Sections:
 ## Check we report a warning when a SHT_GNU_versym section has an invalid entry size.
 
 # RUN: yaml2obj --docnum=5 %s -o %t5
-# RUN: llvm-readelf -V %t5 2>&1 | FileCheck -DFILE=%t5 %s --check-prefix=INVALID-ENT-SIZE-GNU
-# RUN: llvm-readobj -V %t5 2>&1 | FileCheck -DFILE=%t5 %s --check-prefix=INVALID-ENT-SIZE-LLVM
-
+# RUN: llvm-readelf -V --dyn-syms %t5 2>&1 | FileCheck -DFILE=%t5 %s --check-prefix=INVALID-ENT-SIZE-GNU
+# RUN: llvm-readobj -V --dyn-syms %t5 2>&1 | FileCheck -DFILE=%t5 %s --check-prefix=INVALID-ENT-SIZE-LLVM
+
+# INVALID-ENT-SIZE-GNU:      Symbol table '.dynsym' contains 2 entries:
+# INVALID-ENT-SIZE-GNU-NEXT:    Num:    Value          Size Type    Bind   Vis       Ndx Name
+# INVALID-ENT-SIZE-GNU-NEXT: warning: '[[FILE]]': section [index 1] has invalid sh_entsize: expected 2, but got 3
+# INVALID-ENT-SIZE-GNU-NEXT:      0: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT   UND @<corrupt>
+# INVALID-ENT-SIZE-GNU-NEXT:      1: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT   UND foo@<corrupt>
 # INVALID-ENT-SIZE-GNU:      Version symbols section '.gnu.version' contains 1 entries:
 # INVALID-ENT-SIZE-GNU-NEXT:  Addr: 0000000000000000  Offset: 0x000040  Link: 0 ()
 # INVALID-ENT-SIZE-GNU-NEXT: warning: '[[FILE]]': cannot read content of SHT_GNU_versym section with index 1: section [index 1] has an invalid sh_entsize: 3
 
+# INVALID-ENT-SIZE-LLVM:      DynamicSymbols [
+# INVALID-ENT-SIZE-LLVM-NEXT: warning: '[[FILE]]': section [index 1] has invalid sh_entsize: expected 2, but got 3
+# INVALID-ENT-SIZE-LLVM-NEXT:   Symbol {
+# INVALID-ENT-SIZE-LLVM-NEXT:     Name: @<corrupt> (0)
+# INVALID-ENT-SIZE-LLVM-NEXT:     Value: 0x0
+# INVALID-ENT-SIZE-LLVM-NEXT:     Size: 0
+# INVALID-ENT-SIZE-LLVM-NEXT:     Binding: Local (0x0)
+# INVALID-ENT-SIZE-LLVM-NEXT:     Type: None (0x0)
+# INVALID-ENT-SIZE-LLVM-NEXT:     Other: 0
+# INVALID-ENT-SIZE-LLVM-NEXT:     Section: Undefined (0x0)
+# INVALID-ENT-SIZE-LLVM-NEXT:   }
+# INVALID-ENT-SIZE-LLVM-NEXT:   Symbol {
+# INVALID-ENT-SIZE-LLVM-NEXT:     Name: foo@<corrupt> (1)
+# INVALID-ENT-SIZE-LLVM-NEXT:     Value: 0x0
+# INVALID-ENT-SIZE-LLVM-NEXT:     Size: 0
+# INVALID-ENT-SIZE-LLVM-NEXT:     Binding: Local (0x0)
+# INVALID-ENT-SIZE-LLVM-NEXT:     Type: None (0x0)
+# INVALID-ENT-SIZE-LLVM-NEXT:     Other: 0
+# INVALID-ENT-SIZE-LLVM-NEXT:     Section: Undefined (0x0)
+# INVALID-ENT-SIZE-LLVM-NEXT:   }
+# INVALID-ENT-SIZE-LLVM-NEXT: ]
 # INVALID-ENT-SIZE-LLVM:      VersionSymbols [
 # INVALID-ENT-SIZE-LLVM-NEXT: warning: '[[FILE]]': cannot read content of SHT_GNU_versym section with index 1: section [index 1] has an invalid sh_entsize: 3
 # INVALID-ENT-SIZE-LLVM-NEXT: ]
@@ -137,6 +163,8 @@ Sections:
     Type:    SHT_GNU_versym
     Entries: [ 0 ]
     EntSize: 3
+DynamicSymbols:
+  - Name: foo
 
 ## Check we report a warning when the number of version entries does not match the number of symbols in the associated symbol table.
 
diff --git a/llvm/test/tools/llvm-size/output-alloc.test b/llvm/test/tools/llvm-size/output-alloc.test
new file mode 100644
index 0000000000000..9882126bdd6da
--- /dev/null
+++ b/llvm/test/tools/llvm-size/output-alloc.test
@@ -0,0 +1,38 @@
+## gnu size has a number of special cases regarding REL, RELA and STRTAB sections being considered in size output. 
+## To avoid unnecessary complexity llvm size outputs these sections in cases they have the SHF_ALLOC flag.
+
+# RUN: yaml2obj %s -o %t1
+# RUN: llvm-size --format=sysv %t1 | FileCheck %s
+
+# CHECK:      section     size   addr
+# CHECK-NEXT: .rela.alloc
+# CHECK-NEXT: .rel.alloc
+# CHECK-NEXT: .strtab.alloc
+# CHECK-NEXT: Total
+# CHECK-EMPTY:
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
+Sections:
+  - Name:  .rela.alloc
+    Type:  SHT_RELA
+    Flags: [ SHF_ALLOC ]
+  - Name:  .rel.alloc
+    Type:  SHT_REL
+    Flags: [ SHF_ALLOC ]
+  - Name:  .strtab.alloc
+    Type:  SHT_STRTAB
+    Flags: [ SHF_ALLOC ]
+  - Name:  .rela
+    Type:  SHT_RELA
+    Flags: []
+  - Name:  .rel
+    Type:  SHT_REL
+    Flags: []
+  - Name:  .strtab
+    Type:  SHT_STRTAB
+    Flags: []
diff --git a/llvm/test/tools/llvm-symbolizer/options-from-env.test b/llvm/test/tools/llvm-symbolizer/options-from-env.test
index 76a2987a64e62..5fb566f56a025 100644
--- a/llvm/test/tools/llvm-symbolizer/options-from-env.test
+++ b/llvm/test/tools/llvm-symbolizer/options-from-env.test
@@ -1,4 +1,6 @@
-RUN: env LLVM_SYMBOLIZER_OPTS=--print-address llvm-symbolizer 0x20112f | FileCheck %s
-RUN: env LLVM_ADDR2LINE_OPTS=--print-address llvm-addr2line 0x20112f | FileCheck %s
+# RUN: env LLVM_SYMBOLIZER_OPTS='0 1 --verbose' llvm-symbolizer 2 | FileCheck %s
+# RUN: env LLVM_ADDR2LINE_OPTS='0 1 --verbose' llvm-addr2line 2 | FileCheck %s
 
-CHECK: 0x20112f
+# CHECK:      0
+# CHECK-NEXT: 1
+# CHECK-NEXT: 2
diff --git a/llvm/test/tools/llvm-symbolizer/pdb/Inputs/test-columns.exe b/llvm/test/tools/llvm-symbolizer/pdb/Inputs/test-columns.exe
new file mode 100644
index 0000000000000..1d9a40dc74e5d
Binary files /dev/null and b/llvm/test/tools/llvm-symbolizer/pdb/Inputs/test-columns.exe differ
diff --git a/llvm/test/tools/llvm-symbolizer/pdb/Inputs/test-columns.pdb b/llvm/test/tools/llvm-symbolizer/pdb/Inputs/test-columns.pdb
new file mode 100644
index 0000000000000..cd1093270e843
Binary files /dev/null and b/llvm/test/tools/llvm-symbolizer/pdb/Inputs/test-columns.pdb differ
diff --git a/llvm/test/tools/llvm-symbolizer/pdb/Inputs/test.cpp b/llvm/test/tools/llvm-symbolizer/pdb/Inputs/test.cpp
index bf97594fa4c80..e1ac50f2e820f 100644
--- a/llvm/test/tools/llvm-symbolizer/pdb/Inputs/test.cpp
+++ b/llvm/test/tools/llvm-symbolizer/pdb/Inputs/test.cpp
@@ -1,5 +1,7 @@
 // To generate the corresponding EXE/PDB, run:
 // cl /Zi test.cpp
+// To generate the PDB with column numbers, run:
+// clang-cl /Zi -gcolumn-info test.cpp
 
 namespace NS {
 struct Foo {
diff --git a/llvm/test/tools/llvm-symbolizer/pdb/pdb-native-columns.test b/llvm/test/tools/llvm-symbolizer/pdb/pdb-native-columns.test
new file mode 100644
index 0000000000000..a8ccc33d03574
--- /dev/null
+++ b/llvm/test/tools/llvm-symbolizer/pdb/pdb-native-columns.test
@@ -0,0 +1,29 @@
+RUN: echo 0x140006BA0 > %t.input
+RUN: echo 0x140006C00 >> %t.input
+RUN: echo 0x140006BB0 >> %t.input
+RUN: echo 0x140006C10 >> %t.input
+RUN: echo 0x140006C20 >> %t.input
+RUN: echo 0x140006C30 >> %t.input
+RUN: echo 0x140006C40 >> %t.input
+RUN: echo 0x140006C70 >> %t.input
+RUN:    llvm-symbolizer -obj="%p/Inputs/test-columns.exe" -use-native-pdb-reader < %t.input \
+RUN:    | FileCheck %s
+
+This tests that the symbolizer outputs column info when it is present in the pdb.
+
+CHECK: foo(void)
+CHECK-NEXT: test.cpp:11:1
+CHECK: {{^private_symbol$}}
+CHECK-NEXT: test.cpp:14:1
+CHECK: {{^main}}
+CHECK-NEXT: test.cpp:16:0
+CHECK: {{^foo_cdecl$}}
+CHECK-NEXT: test.cpp:25:27
+CHECK: {{^foo_stdcall$}}
+CHECK-NEXT: test.cpp:26:31
+CHECK: {{^foo_fastcall$}}
+CHECK-NEXT: test.cpp:27:33
+CHECK: {{^foo_vectorcall}}
+CHECK-NEXT: test.cpp:28:37
+CHECK: NS::Foo::bar(void)
+CHECK-NEXT: test.cpp:6:0
diff --git a/llvm/test/tools/llvm-symbolizer/pdb/pdb-native.test b/llvm/test/tools/llvm-symbolizer/pdb/pdb-native.test
index d132dbfc0460f..9356bbc302050 100644
--- a/llvm/test/tools/llvm-symbolizer/pdb/pdb-native.test
+++ b/llvm/test/tools/llvm-symbolizer/pdb/pdb-native.test
@@ -18,8 +18,6 @@ RUN: %python -c 'import sys;print("\n".join([hex(int(x, 16) - 0x400000) for x in
 RUN:    | llvm-symbolizer -obj="%p/Inputs/test.exe" -use-native-pdb-reader -demangle=false --relative-address \
 RUN:    | FileCheck %s --check-prefix=CHECK-NO-DEMANGLE
 
-Currently only finding function/public symbol names is implemented.
-
 CHECK: foo(void)
 CHECK-NEXT: test.cpp:10
 CHECK: {{^private_symbol$}}
diff --git a/llvm/test/tools/llvm-xray/X86/lit.local.cfg b/llvm/test/tools/llvm-xray/X86/lit.local.cfg
deleted file mode 100644
index 4f00369e13d76..0000000000000
--- a/llvm/test/tools/llvm-xray/X86/lit.local.cfg
+++ /dev/null
@@ -1 +0,0 @@
-config.suffixes = ['.yaml', '.ll', '.txt']
diff --git a/llvm/test/tools/obj2yaml/COFF/bss.s b/llvm/test/tools/obj2yaml/COFF/bss.s
index fed5d058714b8..8e52ba6dddb36 100644
--- a/llvm/test/tools/obj2yaml/COFF/bss.s
+++ b/llvm/test/tools/obj2yaml/COFF/bss.s
@@ -1,3 +1,4 @@
+# REQUIRES: x86-registered-target
 # RUN: llvm-mc -filetype=obj -triple=x86_64-windows-msvc %s -o %t.obj
 # RUN: llvm-objdump -h %t.obj | FileCheck %s
 # RUN: obj2yaml %t.obj | yaml2obj -o %t.2.obj
diff --git a/llvm/test/tools/obj2yaml/lit.local.cfg b/llvm/test/tools/obj2yaml/lit.local.cfg
deleted file mode 100644
index db82cc231003a..0000000000000
--- a/llvm/test/tools/obj2yaml/lit.local.cfg
+++ /dev/null
@@ -1 +0,0 @@
-config.suffixes = ['.test', '.yaml']
diff --git a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-abbrev.yaml b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-abbrev.yaml
index eba30652dc2d3..03583ac873241 100644
--- a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-abbrev.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-abbrev.yaml
@@ -4,7 +4,7 @@
 
 # RUN: yaml2obj --docnum=1 %s -o %t1.o
 # RUN: llvm-readobj --sections --section-data %t1.o | \
-# RUN:   FileCheck -DSIZE=38 -DADDRALIGN=1 %s --check-prefixes=SHDR,CONTENT
+# RUN:   FileCheck -DSIZE=39 -DADDRALIGN=1 %s --check-prefixes=SHDR,CONTENT
 
 #         SHDR: Index: 1
 #    SHDR-NEXT: Name: .debug_abbrev (1)
@@ -25,25 +25,26 @@
 ##                         ^-   DW_CHILDREN_yes     1-byte              ^-   DW_FORM_data2  ULEB128
 ##                           ^- DW_AT_producer      ULEB128               ^- DW_AT_name     ULEB128
 ##
-##     CONTENT:         1A000002                                    2E011101
+##     CONTENT:         1A000002                                    2E011101 |...%............|
 ##                      ^-       DW_AT_strx        ULEB128          ^-       DW_TAG_subprogram ULEB128
-##                        ^---   terminating entry                    ^-     DW_CHILDREN_yes   1-byte
+##                        ^---   attr terminator                      ^-     DW_CHILDREN_yes   1-byte
 ##                            ^- abbreviation code ULEB128              ^-   DW_AT_low_pc      ULEB128
 ##                                                                        ^- DW_FORM_addr      ULEB128
 ##
 # CONTENT-NEXT:   0010: 121B0000                                    03060081
 ##                      ^-       DW_AT_high_pc ULEB128              ^-       abbreviation code ULEB128
 ##                        ^-     DW_FORM_addrx ULEB128                ^-     Tag: value        UELB128
-##                          ^--- terminating entry                      ^-   DW_CHILDREN_no    1-byte
+##                          ^--- attr terminator                        ^-   DW_CHILDREN_no    1-byte
 ##                                                                        ^- DW_AT_call_pc     ULEB128 (0x81)
 ##
-#      CONTENT:         01810104                                    02A04021
+#      CONTENT:         01810104                                    02A04021 |..............@!|
 ##                      --                                          ^-       Form: reserved         ULEB128
 ##                        ^---   Form: invalid       ULEB128 (0x81)   ^---   Attribute: reserved    ULEB128 (0x2020)
 ##                            ^- Attribute: reserved ULEB128              ^- DW_FORM_implicit_const ULEB128
 ##
-# CONTENT-NEXT:   0020: CEC2F105                                    0000
-##                      ^------- Value SLEB128 (12345678)           ^--- terminating entry
+# CONTENT-NEXT:   0020: CEC2F105                                    000000   |.......|
+##                      ^------- Value SLEB128 (12345678)           ^---     attr terminator
+##                                                                      ^-   abbrev terminator
 # CONTENT-NEXT: )
 
 --- !ELF
@@ -119,7 +120,7 @@ Sections:
 # RUN:   FileCheck -DSIZE=16 -DADDRALIGN=0 %s --check-prefixes=SHDR,SIZE
 
 # SIZE-NEXT: SectionData (
-# SIZE-NEXT:   0000: 00000000 00000000 00000000 00000000
+# SIZE-NEXT:   0000: 00000000 00000000 00000000 00000000 |................|
 # SIZE-NEXT: )
 
 --- !ELF
@@ -186,8 +187,8 @@ DWARF:
 # RUN: llvm-readelf --sections %t6.o | FileCheck %s --check-prefix=OVERRIDDEN
 
 #      OVERRIDDEN: [Nr] Name          Type   Address          Off    Size   ES Flg Lk Inf Al
-#      OVERRIDDEN: [ 1] .debug_abbrev STRTAB 0000000000002020 000050 000005 01   A  2   1  2
-# OVERRIDDEN-NEXT: [ 2] .sec          STRTAB 0000000000000000 000055 000000 00      0   0  0
+#      OVERRIDDEN: [ 1] .debug_abbrev STRTAB 0000000000002020 000050 000006 01   A  2   1  2
+# OVERRIDDEN-NEXT: [ 2] .sec          STRTAB 0000000000000000 000056 000000 00      0   0  0
 
 --- !ELF
 FileHeader:
@@ -205,7 +206,7 @@ Sections:
     AddressAlign: 2           ## 0 by default.
     Address:      0x2020      ## 0x00 by default.
     Offset:       0x50        ## 0x40 for the first section.
-    Size:         0x05        ## Set the "Size" so that we can reuse the check tag "OVERRIDDEN".
+    Size:         0x06        ## Set the "Size" so that we can reuse the check tag "OVERRIDDEN".
   - Name:         .sec        ## Linked by .debug_abbrev.
     Type:         SHT_STRTAB
 
diff --git a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-gnu-pubnames.yaml b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-gnu-pubnames.yaml
new file mode 100644
index 0000000000000..71c7981117554
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-gnu-pubnames.yaml
@@ -0,0 +1,261 @@
+## Test that yaml2obj emits .debug_gnu_pubnames section.
+
+## a) Generate the '.debug_gnu_pubnames' section from the 'DWARF' entry.
+
+## Generate and verify a 32-bit little endian .debug_gnu_pubnames section.
+
+# RUN: yaml2obj --docnum=1 -DENDIAN=ELFDATA2LSB %s -o %t1.le.o
+# RUN: llvm-readobj --sections --section-data %t1.le.o | \
+# RUN:   FileCheck -DSIZE=32 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF32-LE
+
+#            SHDR: Index: 1
+#       SHDR-NEXT: Name: .debug_gnu_pubnames (1)
+#       SHDR-NEXT: Type: SHT_PROGBITS (0x1)
+#       SHDR-NEXT: Flags [ (0x0)
+#       SHDR-NEXT: ]
+#       SHDR-NEXT: Address: 0x0
+#       SHDR-NEXT: Offset: 0x40
+#       SHDR-NEXT: Size: [[SIZE]]
+#       SHDR-NEXT: Link: 0
+#       SHDR-NEXT: Info: 0
+#       SHDR-NEXT: AddressAlignment: [[ADDRALIGN]]
+#       SHDR-NEXT: EntrySize: 0
+# DWARF32-LE-NEXT: SectionData (
+# DWARF32-LE-NEXT:   0000: 34120000 02003412 00002143 00007856 |4.....4...!C..xV|
+##                         ^-------                            unit_length (4-byte)
+##                                  ^---                       version (2-byte)
+##                                      ^--------              debug_info_offset (4-byte)
+##                                               ^--------     debug_info_length (4-byte)
+##                                                        ^--- offset (4-byte)
+# DWARF32-LE-NEXT:   0010: 34123061 62630021 43658730 64656600 |4.0abc.!Ce.0def.|
+##                         ----
+##                             ^-                              descriptor (1-byte)
+##                               ^--------                     name "abc\0"
+##                                        ^--------            offset (4-byte)
+##                                                 ^-          descriptor (1-byte)
+##                                                    ^------- name "def\0"
+# DWARF32-LE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    [[ENDIAN]]
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_gnu_pubnames:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:
+      - DieOffset:  0x12345678
+        Descriptor: 0x30
+        Name:       abc
+      - DieOffset:  0x87654321
+        Descriptor: 0x30
+        Name:       def
+
+## Generate and verify a 32-bit big endian .debug_gnu_pubnames section.
+
+# RUN: yaml2obj --docnum=1 -DENDIAN=ELFDATA2MSB %s -o %t1.be.o
+# RUN: llvm-readobj --sections --section-data %t1.be.o | \
+# RUN:   FileCheck -DSIZE=32 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF32-BE
+
+# DWARF32-BE-NEXT: SectionData (
+# DWARF32-BE-NEXT:   0000: 00001234 00020000 12340000 43211234 |...4.....4..C!.4|
+##                         ^-------                            unit_length (4-byte)
+##                                  ^---                       version (2-byte)
+##                                      ^--------              debug_info_offset (4-byte)
+##                                               ^--------     debug_info_length (4-byte)
+##                                                        ^--- offset (4-byte)
+# DWARF32-BE-NEXT:   0010: 56783061 62630087 65432130 64656600 |Vx0abc..eC!0def.|
+##                         ----
+##                             ^-                              descriptor (1-byte)
+##                               ^--------                     name "abc\0"
+##                                        ^--------            offset (4-byte)
+##                                                 ^-          descriptor (1-byte)
+##                                                    ^------- name "def\0"
+# DWARF32-BE-NEXT: )
+
+## b) Generate the .debug_gnu_pubnames section from raw section content.
+
+# RUN: yaml2obj --docnum=2 %s -o %t2.o
+# RUN: llvm-readobj --sections --section-data %t2.o | \
+# RUN:   FileCheck %s -DADDRALIGN=0 -DSIZE=3 --check-prefixes=SHDR,ARBITRARY-CONTENT
+
+#      ARBITRARY-CONTENT: SectionData (
+# ARBITRARY-CONTENT-NEXT:   0000: 112233
+# ARBITRARY-CONTENT-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .debug_gnu_pubnames
+    Type:    SHT_PROGBITS
+    Content: "112233"
+
+## c) Generate the .debug_gnu_pubnames section when the "Size" is specified.
+
+# RUN: yaml2obj --docnum=3 %s -o %t3.o
+# RUN: llvm-readobj --sections --section-data %t3.o | \
+# RUN:   FileCheck -DSIZE=16 -DADDRALIGN=0 %s --check-prefixes=SHDR,SIZE
+
+#      SIZE: SectionData (
+# SIZE-NEXT:   0000: 00000000 00000000 00000000 00000000  |................|
+# SIZE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .debug_gnu_pubnames
+    Type: SHT_PROGBITS
+    Size: 0x10
+
+## d) Test that yaml2obj emits an error message when both the "Size" and the
+## "debug_gnu_pubnames" entry are specified at the same time.
+
+# RUN: not yaml2obj --docnum=4 %s 2>&1 | FileCheck %s --check-prefix=ERROR
+
+# ERROR: yaml2obj: error: cannot specify section '.debug_gnu_pubnames' contents in the 'DWARF' entry and the 'Content' or 'Size' in the 'Sections' entry at the same time
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .debug_gnu_pubnames
+    Type: SHT_PROGBITS
+    Size: 0x10
+DWARF:
+  debug_gnu_pubnames:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:    []
+
+## e) Test that yaml2obj emits an error message when both the "Content" and the
+## "debug_gnu_pubnames" entry are specified at the same time.
+
+# RUN: not yaml2obj --docnum=5 %s 2>&1 | FileCheck %s --check-prefix=ERROR
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .debug_gnu_pubnames
+    Type:    SHT_PROGBITS
+    Content: "00"
+DWARF:
+  debug_gnu_pubnames:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:    []
+
+## f) Test that all the properties can be overridden by the section header when
+## the "debug_gnu_pubnames" entry doesn't exist.
+
+# RUN: yaml2obj --docnum=6 %s -o %t6.o
+# RUN: llvm-readelf --sections %t6.o | FileCheck %s --check-prefix=OVERRIDDEN
+
+#      OVERRIDDEN: [Nr] Name                Type   Address          Off    Size   ES Flg Lk Inf Al
+#      OVERRIDDEN: [ 1] .debug_gnu_pubnames STRTAB 0000000000002020 000050 00000e 01   A  2   1  2
+# OVERRIDDEN-NEXT: [ 2] .sec                STRTAB 0000000000000000 00005e 000000 00      0   0  0
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:         .debug_gnu_pubnames
+    Type:         SHT_STRTAB  ## SHT_PROGBITS by default.
+    Flags:        [SHF_ALLOC] ## 0 by default.
+    Link:         .sec        ## 0 by default.
+    EntSize:      1           ## 0 by default.
+    Info:         1           ## 0 by default.
+    AddressAlign: 2           ## 0 by default.
+    Address:      0x2020      ## 0x00 by default.
+    Offset:       0x50        ## 0x40 for the first section.
+    Size:         0x0e        ## Set the "Size" so that we can reuse the check tag "OVERRIDDEN".
+  - Name:         .sec        ## Linked by .debug_gnu_pubnames.
+    Type:         SHT_STRTAB
+
+## g) Test that all the properties can be overridden by the section header when
+## the "debug_gnu_pubnames" entry exists.
+
+# RUN: yaml2obj --docnum=7 %s -o %t7.o
+# RUN: llvm-readelf --sections %t7.o | FileCheck %s --check-prefix=OVERRIDDEN
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:         .debug_gnu_pubnames
+    Type:         SHT_STRTAB  ## SHT_PROGBITS by default.
+    Flags:        [SHF_ALLOC] ## 0 by default.
+    Link:         .sec        ## 0 by default.
+    EntSize:      1           ## 0 by default.
+    Info:         1           ## 0 by default.
+    AddressAlign: 2           ## 1 by default.
+    Address:      0x2020      ## 0x00 by default.
+    Offset:       0x50        ## 0x40 for the first section.
+  - Name:         .sec        ## Linked by .debug_gnu_pubnames.
+    Type:         SHT_STRTAB
+DWARF:
+  debug_gnu_pubnames:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:    []
+
+## h) Test that yaml2obj emits an error if 'Descriptor' is missing.
+
+# RUN: not yaml2obj --docnum=8 %s -o %t8.o 2>&1 | FileCheck %s --check-prefix=MISSING-KEY --ignore-case
+
+#      MISSING-KEY: YAML:260:9: error: missing required key 'Descriptor'
+# MISSING-KEY-NEXT:       - DieOffset: 0x12345678
+# MISSING-KEY-NEXT:         ^
+# MISSING-KEY-NEXT: yaml2obj: error: failed to parse YAML input: Invalid argument
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_gnu_pubnames:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:
+      - DieOffset: 0x12345678
+        Name:      abc
diff --git a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-gnu-pubtypes.yaml b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-gnu-pubtypes.yaml
new file mode 100644
index 0000000000000..b88d21b5fc7af
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-gnu-pubtypes.yaml
@@ -0,0 +1,261 @@
+## Test that yaml2obj emits .debug_gnu_pubtypes section.
+
+## a) Generate the '.debug_gnu_pubtypes' section from the 'DWARF' entry.
+
+## Generate and verify a 32-bit little endian .debug_gnu_pubtypes section.
+
+# RUN: yaml2obj --docnum=1 -DENDIAN=ELFDATA2LSB %s -o %t1.le.o
+# RUN: llvm-readobj --sections --section-data %t1.le.o | \
+# RUN:   FileCheck -DSIZE=32 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF32-LE
+
+#            SHDR: Index: 1
+#       SHDR-NEXT: Name: .debug_gnu_pubtypes (1)
+#       SHDR-NEXT: Type: SHT_PROGBITS (0x1)
+#       SHDR-NEXT: Flags [ (0x0)
+#       SHDR-NEXT: ]
+#       SHDR-NEXT: Address: 0x0
+#       SHDR-NEXT: Offset: 0x40
+#       SHDR-NEXT: Size: [[SIZE]]
+#       SHDR-NEXT: Link: 0
+#       SHDR-NEXT: Info: 0
+#       SHDR-NEXT: AddressAlignment: [[ADDRALIGN]]
+#       SHDR-NEXT: EntrySize: 0
+# DWARF32-LE-NEXT: SectionData (
+# DWARF32-LE-NEXT:   0000: 34120000 02003412 00002143 00007856 |4.....4...!C..xV|
+##                         ^-------                            unit_length (4-byte)
+##                                  ^---                       version (2-byte)
+##                                      ^--------              debug_info_offset (4-byte)
+##                                               ^--------     debug_info_length (4-byte)
+##                                                        ^--- offset (4-byte)
+# DWARF32-LE-NEXT:   0010: 34121261 62630021 43658734 64656600 |4..abc.!Ce.4def.|
+##                         ----
+##                             ^-                              descriptor (1-byte)
+##                               ^--------                     name "abc\0"
+##                                        ^--------            offset (4-byte)
+##                                                 ^-          descriptor (1-byte)
+##                                                    ^------- name "def\0"
+# DWARF32-LE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    [[ENDIAN]]
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_gnu_pubtypes:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:
+      - DieOffset:  0x12345678
+        Descriptor: 0x12
+        Name:       abc
+      - DieOffset:  0x87654321
+        Descriptor: 0x34
+        Name:       def
+
+## Generate and verify a 32-bit big endian .debug_gnu_pubtypes section.
+
+# RUN: yaml2obj --docnum=1 -DENDIAN=ELFDATA2MSB %s -o %t1.be.o
+# RUN: llvm-readobj --sections --section-data %t1.be.o | \
+# RUN:   FileCheck -DSIZE=32 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF32-BE
+
+# DWARF32-BE-NEXT: SectionData (
+# DWARF32-BE-NEXT:   0000: 00001234 00020000 12340000 43211234 |...4.....4..C!.4|
+##                         ^-------                            unit_length (4-byte)
+##                                  ^---                       version (2-byte)
+##                                      ^--------              debug_info_offset (4-byte)
+##                                               ^--------     debug_info_length (4-byte)
+##                                                        ^--- offset (4-byte)
+# DWARF32-BE-NEXT:   0010: 56781261 62630087 65432134 64656600 |Vx.abc..eC!4def.|
+##                         ----
+##                             ^-                              descriptor (1-byte)
+##                               ^--------                     name "abc\0"
+##                                        ^--------            offset (4-byte)
+##                                                 ^-          descriptor (1-byte)
+##                                                    ^------- name "def\0"
+# DWARF32-BE-NEXT: )
+
+## b) Generate the .debug_gnu_pubtypes section from raw section content.
+
+# RUN: yaml2obj --docnum=2 %s -o %t2.o
+# RUN: llvm-readobj --sections --section-data %t2.o | \
+# RUN:   FileCheck %s -DADDRALIGN=0 -DSIZE=3 --check-prefixes=SHDR,ARBITRARY-CONTENT
+
+#      ARBITRARY-CONTENT: SectionData (
+# ARBITRARY-CONTENT-NEXT:   0000: 112233
+# ARBITRARY-CONTENT-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .debug_gnu_pubtypes
+    Type:    SHT_PROGBITS
+    Content: "112233"
+
+## c) Generate the .debug_gnu_pubtypes section when the "Size" is specified.
+
+# RUN: yaml2obj --docnum=3 %s -o %t3.o
+# RUN: llvm-readobj --sections --section-data %t3.o | \
+# RUN:   FileCheck -DSIZE=16 -DADDRALIGN=0 %s --check-prefixes=SHDR,SIZE
+
+#      SIZE: SectionData (
+# SIZE-NEXT:   0000: 00000000 00000000 00000000 00000000  |................|
+# SIZE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .debug_gnu_pubtypes
+    Type: SHT_PROGBITS
+    Size: 0x10
+
+## d) Test that yaml2obj emits an error message when both the "Size" and the
+## "debug_gnu_pubtypes" entry are specified at the same time.
+
+# RUN: not yaml2obj --docnum=4 %s 2>&1 | FileCheck %s --check-prefix=ERROR
+
+# ERROR: yaml2obj: error: cannot specify section '.debug_gnu_pubtypes' contents in the 'DWARF' entry and the 'Content' or 'Size' in the 'Sections' entry at the same time
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .debug_gnu_pubtypes
+    Type: SHT_PROGBITS
+    Size: 0x10
+DWARF:
+  debug_gnu_pubtypes:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:    []
+
+## e) Test that yaml2obj emits an error message when both the "Content" and the
+## "debug_gnu_pubtypes" entry are specified at the same time.
+
+# RUN: not yaml2obj --docnum=5 %s 2>&1 | FileCheck %s --check-prefix=ERROR
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .debug_gnu_pubtypes
+    Type:    SHT_PROGBITS
+    Content: "00"
+DWARF:
+  debug_gnu_pubtypes:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:    []
+
+## f) Test that all the properties can be overridden by the section header when
+## the "debug_gnu_pubtypes" entry doesn't exist.
+
+# RUN: yaml2obj --docnum=6 %s -o %t6.o
+# RUN: llvm-readelf --sections %t6.o | FileCheck %s --check-prefix=OVERRIDDEN
+
+#      OVERRIDDEN: [Nr] Name                Type   Address          Off    Size   ES Flg Lk Inf Al
+#      OVERRIDDEN: [ 1] .debug_gnu_pubtypes STRTAB 0000000000002020 000050 00000e 01   A  2   1  2
+# OVERRIDDEN-NEXT: [ 2] .sec                STRTAB 0000000000000000 00005e 000000 00      0   0  0
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:         .debug_gnu_pubtypes
+    Type:         SHT_STRTAB  ## SHT_PROGBITS by default.
+    Flags:        [SHF_ALLOC] ## 0 by default.
+    Link:         .sec        ## 0 by default.
+    EntSize:      1           ## 0 by default.
+    Info:         1           ## 0 by default.
+    AddressAlign: 2           ## 0 by default.
+    Address:      0x2020      ## 0x00 by default.
+    Offset:       0x50        ## 0x40 for the first section.
+    Size:         0x0e        ## Set the "Size" so that we can reuse the check tag "OVERRIDDEN".
+  - Name:         .sec        ## Linked by .debug_gnu_pubtypes.
+    Type:         SHT_STRTAB
+
+## g) Test that all the properties can be overridden by the section header when
+## the "debug_gnu_pubtypes" entry exists.
+
+# RUN: yaml2obj --docnum=7 %s -o %t7.o
+# RUN: llvm-readelf --sections %t7.o | FileCheck %s --check-prefix=OVERRIDDEN
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:         .debug_gnu_pubtypes
+    Type:         SHT_STRTAB  ## SHT_PROGBITS by default.
+    Flags:        [SHF_ALLOC] ## 0 by default.
+    Link:         .sec        ## 0 by default.
+    EntSize:      1           ## 0 by default.
+    Info:         1           ## 0 by default.
+    AddressAlign: 2           ## 1 by default.
+    Address:      0x2020      ## 0x00 by default.
+    Offset:       0x50        ## 0x40 for the first section.
+  - Name:         .sec        ## Linked by .debug_gnu_pubtypes.
+    Type:         SHT_STRTAB
+DWARF:
+  debug_gnu_pubtypes:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:    []
+
+## h) Test that yaml2obj emits an error if 'Descriptor' is missing.
+
+# RUN: not yaml2obj --docnum=8 %s -o %t8.o 2>&1 | FileCheck %s --check-prefix=MISSING-KEY --ignore-case
+
+#      MISSING-KEY: YAML:260:9: error: missing required key 'Descriptor'
+# MISSING-KEY-NEXT:       - DieOffset: 0x12345678
+# MISSING-KEY-NEXT:         ^
+# MISSING-KEY-NEXT: yaml2obj: error: failed to parse YAML input: Invalid argument
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_gnu_pubtypes:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:
+      - DieOffset: 0x12345678
+        Name:      abc
diff --git a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-info.yaml b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-info.yaml
index 4811992e91e59..896a9b4728870 100644
--- a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-info.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-info.yaml
@@ -6,7 +6,7 @@
 
 # RUN: yaml2obj --docnum=1 -DENDIAN=ELFDATA2LSB %s -o %t1.le.o
 # RUN: llvm-readobj --sections --section-data %t1.le.o | \
-# RUN:   FileCheck -DINDEX=2 -DNAME=15 -DOFFSET=0x9A -DSIZE=171 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF32-LE-CONTENT
+# RUN:   FileCheck -DINDEX=2 -DNAME=15 -DOFFSET=0x9B -DSIZE=171 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF32-LE-CONTENT
 
 #                    SHDR: Index: [[INDEX]]
 #               SHDR-NEXT: Name: .debug_info ([[NAME]])
@@ -200,8 +200,7 @@ DWARF:
         - Attribute: 0x01
           Form:      DW_FORM_addrx4         ## 0x2c
   debug_info:
-    - Length:
-        TotalLength: 0x1234
+    - Length:     0x1234
       Version:    5
       UnitType:   DW_UT_type
       AbbrOffset: 0x1234
@@ -262,8 +261,7 @@ DWARF:
             - Value: 0x1234             ## DW_FORM_addrx2
             - Value: 0x123456           ## DW_FORM_addrx3 (unimplemented)
             - Value: 0x12345678         ## DW_FORM_addrx4
-    - Length:
-        TotalLength: 0x1234
+    - Length:     0x1234
       Version:    5
       ## Test another unit type.
       UnitType:   DW_UT_compile
@@ -272,8 +270,7 @@ DWARF:
       Entries:
         - AbbrCode: 0
           Values:   []
-    - Length:
-        TotalLength: 0x5678
+    - Length:     0x5678
       ## Test DWARFv4
       Version:    4
       AbbrOffset: 0x5678
@@ -290,7 +287,7 @@ DWARF:
 
 # RUN: yaml2obj --docnum=1 -DENDIAN=ELFDATA2MSB %s -o %t1.be.o
 # RUN: llvm-readobj --sections --section-data %t1.be.o | \
-# RUN:   FileCheck -DINDEX=2 -DNAME=15 -DOFFSET=0x9A -DSIZE=171 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF32-BE-CONTENT
+# RUN:   FileCheck -DINDEX=2 -DNAME=15 -DOFFSET=0x9B -DSIZE=171 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF32-BE-CONTENT
 
 # DWARF32-BE-CONTENT-NEXT: SectionData (
 # DWARF32-BE-CONTENT-NEXT:   0000: 00001234 00050204 00001234 01123456
@@ -419,9 +416,9 @@ Sections:
 ## d) Test that yaml2obj emits an error message when both the "Size" and the
 ## "debug_info" entry are specified at the same time.
 
-# RUN: not yaml2obj --docnum=4 %s 2>&1 | FileCheck %s --check-prefix=ERROR
+# RUN: not yaml2obj --docnum=4 %s 2>&1 | FileCheck %s --check-prefix=AMBIGUOUS-CONTENT
 
-# ERROR: yaml2obj: error: cannot specify section '.debug_info' contents in the 'DWARF' entry and the 'Content' or 'Size' in the 'Sections' entry at the same time
+# AMBIGUOUS-CONTENT: yaml2obj: error: cannot specify section '.debug_info' contents in the 'DWARF' entry and the 'Content' or 'Size' in the 'Sections' entry at the same time
 
 --- !ELF
 FileHeader:
@@ -440,8 +437,7 @@ DWARF:
       Children:   DW_CHILDREN_no
       Attributes: []
   debug_info:
-    - Length:
-        TotalLength: 0x1234
+    - Length:     0x1234
       Version:    5
       UnitType:   DW_UT_type
       AbbrOffset: 0x1234
@@ -451,7 +447,7 @@ DWARF:
 ## e) Test that yaml2obj emits an error message when both the "Content" and the
 ## "debug_info" entry are specified at the same time.
 
-# RUN: not yaml2obj --docnum=5 %s 2>&1 | FileCheck %s --check-prefix=ERROR
+# RUN: not yaml2obj --docnum=5 %s 2>&1 | FileCheck %s --check-prefix=AMBIGUOUS-CONTENT
 
 --- !ELF
 FileHeader:
@@ -470,8 +466,7 @@ DWARF:
       Children:   DW_CHILDREN_no
       Attributes: []
   debug_info:
-    - Length:
-        TotalLength: 0x1234
+    - Length:     0x1234
       Version:    5
       UnitType:   DW_UT_type
       AbbrOffset: 0x1234
@@ -539,8 +534,7 @@ DWARF:
       Children:   DW_CHILDREN_no
       Attributes: []
   debug_info:
-    - Length:
-        TotalLength: 0x1234
+    - Length:     0x1234
       Version:    5
       UnitType:   DW_UT_type
       AbbrOffset: 0x1234
@@ -568,10 +562,164 @@ FileHeader:
   Machine: EM_X86_64
 DWARF:
   debug_info:
-    - Length:
-        TotalLength: 0x1234
+    - Length:     0x1234
       Version:    5
       UnitType:   DW_UT_type
       AbbrOffset: 0x1234
       AddrSize:   4
       Entries:    []
+
+## h) Test that yaml2obj emits values in the DIE according to the abbreviation whose
+## index is equal to the value in 'AbbrevCode'.
+
+# RUN: yaml2obj --docnum=9 %s -o %t9.o
+# RUN: llvm-readelf --hex-dump=.debug_info %t9.o | \
+# RUN:   FileCheck %s --check-prefix=FORM --match-full-lines
+
+#       FORM: Hex dump of section '.debug_info':
+#  FORM-NEXT: 0x00000000 34120000 05000204 34120000 02341221 4.......4....4.!
+##                       ^-------                            unit_length (4-byte)
+##                                                  ^-       abbreviation code (ULEB128)
+##                                                    ^---   Form: DW_FORM_data2 (2-byte)
+##                                                        ^- Form: DW_FORM_data4 (4-byte)
+#  FORM-NEXT: 0x00000010 43658701 21436587 341200            Ce..!Ce.4..
+##                       ------
+##                             ^-                            abbreviation code (ULEB128)
+##                                ^-------                   Form: DW_FORM_data4 (4-byte)
+##                                         ^---              Form: DW_FORM_data2 (2-byte)
+##                                             ^-            abbreviation code (ULEB128)
+# FORM-EMPTY:
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_abbrev:
+    - Tag:        DW_TAG_compile_unit
+      Children:   DW_CHILDREN_yes
+      Attributes:
+        - Attribute: DW_AT_low_pc
+          Form:      DW_FORM_data4
+        - Attribute: DW_AT_high_pc
+          Form:      DW_FORM_data2
+    - Tag:        DW_TAG_subprogram
+      Children:   DW_CHILDREN_no
+      Attributes:
+        - Attribute: DW_AT_low_pc
+          Form:      DW_FORM_data2
+        - Attribute: DW_AT_high_pc
+          Form:      DW_FORM_data4
+  debug_info:
+    - Length:     0x1234
+      Version:    5
+      UnitType:   DW_UT_type
+      AbbrOffset: 0x1234
+      AddrSize:   4
+      Entries:
+        ## Test that yaml2obj emits values when the abbrev code is specified.
+        - AbbrCode: 2
+          Values:
+            - Value: 0x1234
+            - Value: 0x87654321
+        ## Test that yaml2obj emits values when the abbrev code is specified to
+        ## be lower than the first abbrev.
+        - AbbrCode: 1
+          Values:
+            - Value: 0x87654321
+            - Value: 0x1234
+        ## Test that yaml2obj ignores the contents of entries with abbrev code 0.
+        - AbbrCode: 0
+          Values:
+            - Value: 0x1234
+
+## i) Test that yaml2obj reports an error when 'debug_info' has values in its
+## entries but 'debug_abbrev' doesn't have enough attributes for them.
+
+# RUN: not yaml2obj --docnum=10 %s -o %t10.o 2>&1 | \
+# RUN:   FileCheck %s --check-prefixes=ERROR
+
+# ERROR: yaml2obj: error: abbrev code must be less than or equal to the number of entries in abbreviation table
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_info:
+    - Length:     0x1234
+      Version:    5
+      UnitType:   DW_UT_type
+      AbbrOffset: 0x1234
+      AddrSize:   4
+      Entries:
+        - AbbrCode: 1
+          Values:
+            - Value: 0x1234
+
+## j) Test that yaml2obj emits the correct DWARF64 unit headers.
+
+## DWARFv5 unit header.
+
+# RUN: yaml2obj --docnum=11 %s -o %t11.o
+# RUN: llvm-readelf --hex-dump=.debug_info %t11.o | \
+# RUN:   FileCheck %s --check-prefix=DWARFV5-HEADER
+
+#      DWARFV5-HEADER: Hex dump of section '.debug_info':
+# DWARFV5-HEADER-NEXT: 0x00000000 ffffffff 0c000000 00000000 05000208 ................
+##                                ^-------------------------          unit_length (12-byte)
+##                                                           ^---     version (2-byte)
+##                                                               ^-   unit_type (1-byte)
+##                                                                 ^- address_size (1-byte)
+# DWARFV5-HEADER-NEXT: 0x00000010 34120000 00000000                   4.......
+##                                ^----------------                   debug_abbrev_offset (8-byte)
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_info:
+    - Format:     DWARF64
+      Length:     0x0c
+      Version:    5
+      UnitType:   DW_UT_type
+      AbbrOffset: 0x1234
+      AddrSize:   8
+      Entries:    []
+
+## DWARFv4 unit header.
+
+# RUN: yaml2obj --docnum=12 %s -o %t12.o
+# RUN: llvm-readelf --hex-dump=.debug_info %t12.o | \
+# RUN:   FileCheck %s --check-prefix=DWARFV4-HEADER
+
+#      DWARFV4-HEADER: Hex dump of section '.debug_info':
+# DWARFV4-HEADER-NEXT: 0x00000000 ffffffff 0c000000 00000000 04003412 ..............4.
+##                                ^-------------------------          unit_length (12-byte)
+##                                                           ^---     version (2-byte)
+##                                                               ^--- debug_abbrev_offset (8-byte)
+# DWARFV4-HEADER-NEXT: 0x00000010 00000000 000008                     .......
+##                                -------------
+##                                             ^-                     address_size (1-byte)
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_info:
+    - Format:     DWARF64
+      Length:     0x0c
+      Version:    4
+      AbbrOffset: 0x1234
+      AddrSize:   8
+      Entries:    []
diff --git a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-pubnames.yaml b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-pubnames.yaml
new file mode 100644
index 0000000000000..2dce4b69f57f2
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-pubnames.yaml
@@ -0,0 +1,231 @@
+## Test that yaml2obj emits .debug_pubnames section.
+
+## a) Generate the '.debug_pubnames' section from the 'DWARF' entry.
+
+## Generate and verify a 32-bit little endian .debug_pubnames section.
+
+# RUN: yaml2obj --docnum=1 -DENDIAN=ELFDATA2LSB %s -o %t1.le.o
+# RUN: llvm-readobj --sections --section-data %t1.le.o | \
+# RUN:   FileCheck -DSIZE=30 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF32-LE
+
+#            SHDR: Index: 1
+#       SHDR-NEXT: Name: .debug_pubnames (1)
+#       SHDR-NEXT: Type: SHT_PROGBITS (0x1)
+#       SHDR-NEXT: Flags [ (0x0)
+#       SHDR-NEXT: ]
+#       SHDR-NEXT: Address: 0x0
+#       SHDR-NEXT: Offset: 0x40
+#       SHDR-NEXT: Size: [[SIZE]]
+#       SHDR-NEXT: Link: 0
+#       SHDR-NEXT: Info: 0
+#       SHDR-NEXT: AddressAlignment: [[ADDRALIGN]]
+#       SHDR-NEXT: EntrySize: 0
+# DWARF32-LE-NEXT: SectionData (
+# DWARF32-LE-NEXT:   0000: 34120000 02003412 00002143 00007856 |4.....4...!C..xV|
+##                         ^-------                            unit_length (4-byte)
+##                                  ^---                       version (2-byte)
+##                                      ^--------              debug_info_offset (4-byte)
+##                                               ^--------     debug_info_length (4-byte)
+##                                                        ^--- offset (4-byte)
+##
+# DWARF32-LE-NEXT:   0010: 34126162 63002143 65876465 6600     |4.abc.!Ce.def.|
+##                         ----
+##                             ^--------                       name "abc\0"
+##                                      ^--------              offset (4-byte)
+##                                               ^--------     name "def\0"
+# DWARF32-LE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    [[ENDIAN]]
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_pubnames:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:
+      - DieOffset: 0x12345678
+        Name:      abc
+      - DieOffset: 0x87654321
+        Name:      def
+
+## Generate and verify a 32-bit big endian .debug_pubnames section.
+
+# RUN: yaml2obj --docnum=1 -DENDIAN=ELFDATA2MSB %s -o %t1.be.o
+# RUN: llvm-readobj --sections --section-data %t1.be.o | \
+# RUN:   FileCheck -DSIZE=30 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF32-BE
+
+# DWARF32-BE-NEXT: SectionData (
+# DWARF32-BE-NEXT:   0000: 00001234 00020000 12340000 43211234 |...4.....4..C!.4|
+##                         ^-------                            unit_length (4-byte)
+##                                  ^---                       version (2-byte)
+##                                      ^--------              debug_info_offset (4-byte)
+##                                               ^--------     debug_info_length (4-byte)
+##                                                        ^--- offset (4-byte)
+##
+# DWARF32-BE-NEXT:   0010: 56786162 63008765 43216465 6600     |Vxabc..eC!def.|
+##                         ----
+##                             ^--------                       name "abc\0"
+##                                      ^--------              offset (4-byte)
+##                                               ^--------     name "def\0"
+# DWARF32-BE-NEXT: )
+
+## b) Generate the .debug_pubnames section from raw section content.
+
+# RUN: yaml2obj --docnum=2 %s -o %t2.o
+# RUN: llvm-readobj --sections --section-data %t2.o | \
+# RUN:   FileCheck %s -DADDRALIGN=0 -DSIZE=3 --check-prefixes=SHDR,ARBITRARY-CONTENT
+
+#      ARBITRARY-CONTENT: SectionData (
+# ARBITRARY-CONTENT-NEXT:   0000: 112233
+# ARBITRARY-CONTENT-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .debug_pubnames
+    Type:    SHT_PROGBITS
+    Content: "112233"
+
+## c) Generate the .debug_pubnames section when the "Size" is specified.
+
+# RUN: yaml2obj --docnum=3 %s -o %t3.o
+# RUN: llvm-readobj --sections --section-data %t3.o | \
+# RUN:   FileCheck -DSIZE=16 -DADDRALIGN=0 %s --check-prefixes=SHDR,SIZE
+
+#      SIZE: SectionData (
+# SIZE-NEXT:   0000: 00000000 00000000 00000000 00000000  |................|
+# SIZE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .debug_pubnames
+    Type: SHT_PROGBITS
+    Size: 0x10
+
+## d) Test that yaml2obj emits an error message when both the "Size" and the
+## "debug_pubnames" entry are specified at the same time.
+
+# RUN: not yaml2obj --docnum=4 %s 2>&1 | FileCheck %s --check-prefix=ERROR
+
+# ERROR: yaml2obj: error: cannot specify section '.debug_pubnames' contents in the 'DWARF' entry and the 'Content' or 'Size' in the 'Sections' entry at the same time
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .debug_pubnames
+    Type: SHT_PROGBITS
+    Size: 0x10
+DWARF:
+  debug_pubnames:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:    []
+
+## e) Test that yaml2obj emits an error message when both the "Content" and the
+## "debug_pubnames" entry are specified at the same time.
+
+# RUN: not yaml2obj --docnum=5 %s 2>&1 | FileCheck %s --check-prefix=ERROR
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .debug_pubnames
+    Type:    SHT_PROGBITS
+    Content: "00"
+DWARF:
+  debug_pubnames:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:    []
+
+## f) Test that all the properties can be overridden by the section header when
+## the "debug_pubnames" entry doesn't exist.
+
+# RUN: yaml2obj --docnum=6 %s -o %t6.o
+# RUN: llvm-readelf --sections %t6.o | FileCheck %s --check-prefix=OVERRIDDEN
+
+#      OVERRIDDEN: [Nr] Name            Type   Address          Off    Size   ES Flg Lk Inf Al
+#      OVERRIDDEN: [ 1] .debug_pubnames STRTAB 0000000000002020 000050 00000e 01   A  2   1  2
+# OVERRIDDEN-NEXT: [ 2] .sec            STRTAB 0000000000000000 00005e 000000 00      0   0  0
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:         .debug_pubnames
+    Type:         SHT_STRTAB  ## SHT_PROGBITS by default.
+    Flags:        [SHF_ALLOC] ## 0 by default.
+    Link:         .sec        ## 0 by default.
+    EntSize:      1           ## 0 by default.
+    Info:         1           ## 0 by default.
+    AddressAlign: 2           ## 0 by default.
+    Address:      0x2020      ## 0x00 by default.
+    Offset:       0x50        ## 0x40 for the first section.
+    Size:         0x0e        ## Set the "Size" so that we can reuse the check tag "OVERRIDDEN".
+  - Name:         .sec        ## Linked by .debug_pubnames.
+    Type:         SHT_STRTAB
+
+## g) Test that all the properties can be overridden by the section header when
+## the "debug_pubnames" entry exists.
+
+# RUN: yaml2obj --docnum=7 %s -o %t7.o
+# RUN: llvm-readelf --sections %t7.o | FileCheck %s --check-prefix=OVERRIDDEN
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:         .debug_pubnames
+    Type:         SHT_STRTAB  ## SHT_PROGBITS by default.
+    Flags:        [SHF_ALLOC] ## 0 by default.
+    Link:         .sec        ## 0 by default.
+    EntSize:      1           ## 0 by default.
+    Info:         1           ## 0 by default.
+    AddressAlign: 2           ## 1 by default.
+    Address:      0x2020      ## 0x00 by default.
+    Offset:       0x50        ## 0x40 for the first section.
+  - Name:         .sec        ## Linked by .debug_pubnames.
+    Type:         SHT_STRTAB
+DWARF:
+  debug_pubnames:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:    []
diff --git a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-pubtypes.yaml b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-pubtypes.yaml
new file mode 100644
index 0000000000000..71e8193eb1d68
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-pubtypes.yaml
@@ -0,0 +1,231 @@
+## Test that yaml2obj emits .debug_pubtypes section.
+
+## a) Generate the '.debug_pubtypes' section from the 'DWARF' entry.
+
+## Generate and verify a 32-bit little endian .debug_pubtypes section.
+
+# RUN: yaml2obj --docnum=1 -DENDIAN=ELFDATA2LSB %s -o %t1.le.o
+# RUN: llvm-readobj --sections --section-data %t1.le.o | \
+# RUN:   FileCheck -DSIZE=30 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF32-LE
+
+#            SHDR: Index: 1
+#       SHDR-NEXT: Name: .debug_pubtypes (1)
+#       SHDR-NEXT: Type: SHT_PROGBITS (0x1)
+#       SHDR-NEXT: Flags [ (0x0)
+#       SHDR-NEXT: ]
+#       SHDR-NEXT: Address: 0x0
+#       SHDR-NEXT: Offset: 0x40
+#       SHDR-NEXT: Size: [[SIZE]]
+#       SHDR-NEXT: Link: 0
+#       SHDR-NEXT: Info: 0
+#       SHDR-NEXT: AddressAlignment: [[ADDRALIGN]]
+#       SHDR-NEXT: EntrySize: 0
+# DWARF32-LE-NEXT: SectionData (
+# DWARF32-LE-NEXT:   0000: 34120000 02003412 00002143 00007856 |4.....4...!C..xV|
+##                         ^-------                            unit_length (4-byte)
+##                                  ^---                       version (2-byte)
+##                                      ^--------              debug_info_offset (4-byte)
+##                                               ^--------     debug_info_length (4-byte)
+##                                                        ^--- offset (4-byte)
+##
+# DWARF32-LE-NEXT:   0010: 34126162 63002143 65876465 6600     |4.abc.!Ce.def.|
+##                         ----
+##                             ^--------                       name "abc\0"
+##                                      ^--------              offset (4-byte)
+##                                               ^--------     name "def\0"
+# DWARF32-LE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    [[ENDIAN]]
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_pubtypes:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:
+      - DieOffset: 0x12345678
+        Name:      abc
+      - DieOffset: 0x87654321
+        Name:      def
+
+## Generate and verify a 32-bit big endian .debug_pubtypes section.
+
+# RUN: yaml2obj --docnum=1 -DENDIAN=ELFDATA2MSB %s -o %t1.be.o
+# RUN: llvm-readobj --sections --section-data %t1.be.o | \
+# RUN:   FileCheck -DSIZE=30 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF32-BE
+
+# DWARF32-BE-NEXT: SectionData (
+# DWARF32-BE-NEXT:   0000: 00001234 00020000 12340000 43211234 |...4.....4..C!.4|
+##                         ^-------                            unit_length (4-byte)
+##                                  ^---                       version (2-byte)
+##                                      ^--------              debug_info_offset (4-byte)
+##                                               ^--------     debug_info_length (4-byte)
+##                                                        ^--- offset (4-byte)
+##
+# DWARF32-BE-NEXT:   0010: 56786162 63008765 43216465 6600     |Vxabc..eC!def.|
+##                         ----
+##                             ^--------                       name "abc\0"
+##                                      ^--------              offset (4-byte)
+##                                               ^--------     name "def\0"
+# DWARF32-BE-NEXT: )
+
+## b) Generate the .debug_pubtypes section from raw section content.
+
+# RUN: yaml2obj --docnum=2 %s -o %t2.o
+# RUN: llvm-readobj --sections --section-data %t2.o | \
+# RUN:   FileCheck %s -DADDRALIGN=0 -DSIZE=3 --check-prefixes=SHDR,ARBITRARY-CONTENT
+
+#      ARBITRARY-CONTENT: SectionData (
+# ARBITRARY-CONTENT-NEXT:   0000: 112233
+# ARBITRARY-CONTENT-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .debug_pubtypes
+    Type:    SHT_PROGBITS
+    Content: "112233"
+
+## c) Generate the .debug_pubtypes section when the "Size" is specified.
+
+# RUN: yaml2obj --docnum=3 %s -o %t3.o
+# RUN: llvm-readobj --sections --section-data %t3.o | \
+# RUN:   FileCheck -DSIZE=16 -DADDRALIGN=0 %s --check-prefixes=SHDR,SIZE
+
+#      SIZE: SectionData (
+# SIZE-NEXT:   0000: 00000000 00000000 00000000 00000000  |................|
+# SIZE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .debug_pubtypes
+    Type: SHT_PROGBITS
+    Size: 0x10
+
+## d) Test that yaml2obj emits an error message when both the "Size" and the
+## "debug_pubtypes" entry are specified at the same time.
+
+# RUN: not yaml2obj --docnum=4 %s 2>&1 | FileCheck %s --check-prefix=ERROR
+
+# ERROR: yaml2obj: error: cannot specify section '.debug_pubtypes' contents in the 'DWARF' entry and the 'Content' or 'Size' in the 'Sections' entry at the same time
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .debug_pubtypes
+    Type: SHT_PROGBITS
+    Size: 0x10
+DWARF:
+  debug_pubtypes:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:    []
+
+## e) Test that yaml2obj emits an error message when both the "Content" and the
+## "debug_pubtypes" entry are specified at the same time.
+
+# RUN: not yaml2obj --docnum=5 %s 2>&1 | FileCheck %s --check-prefix=ERROR
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .debug_pubtypes
+    Type:    SHT_PROGBITS
+    Content: "00"
+DWARF:
+  debug_pubtypes:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:    []
+
+## f) Test that all the properties can be overridden by the section header when
+## the "debug_pubtypes" entry doesn't exist.
+
+# RUN: yaml2obj --docnum=6 %s -o %t6.o
+# RUN: llvm-readelf --sections %t6.o | FileCheck %s --check-prefix=OVERRIDDEN
+
+#      OVERRIDDEN: [Nr] Name            Type   Address          Off    Size   ES Flg Lk Inf Al
+#      OVERRIDDEN: [ 1] .debug_pubtypes STRTAB 0000000000002020 000050 00000e 01   A  2   1  2
+# OVERRIDDEN-NEXT: [ 2] .sec            STRTAB 0000000000000000 00005e 000000 00      0   0  0
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:         .debug_pubtypes
+    Type:         SHT_STRTAB  ## SHT_PROGBITS by default.
+    Flags:        [SHF_ALLOC] ## 0 by default.
+    Link:         .sec        ## 0 by default.
+    EntSize:      1           ## 0 by default.
+    Info:         1           ## 0 by default.
+    AddressAlign: 2           ## 0 by default.
+    Address:      0x2020      ## 0x00 by default.
+    Offset:       0x50        ## 0x40 for the first section.
+    Size:         0x0e        ## Set the "Size" so that we can reuse the check tag "OVERRIDDEN".
+  - Name:         .sec        ## Linked by .debug_pubtypes.
+    Type:         SHT_STRTAB
+
+## g) Test that all the properties can be overridden by the section header when
+## the "debug_pubtypes" entry exists.
+
+# RUN: yaml2obj --docnum=7 %s -o %t7.o
+# RUN: llvm-readelf --sections %t7.o | FileCheck %s --check-prefix=OVERRIDDEN
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:         .debug_pubtypes
+    Type:         SHT_STRTAB  ## SHT_PROGBITS by default.
+    Flags:        [SHF_ALLOC] ## 0 by default.
+    Link:         .sec        ## 0 by default.
+    EntSize:      1           ## 0 by default.
+    Info:         1           ## 0 by default.
+    AddressAlign: 2           ## 1 by default.
+    Address:      0x2020      ## 0x00 by default.
+    Offset:       0x50        ## 0x40 for the first section.
+  - Name:         .sec        ## Linked by .debug_pubtypes.
+    Type:         SHT_STRTAB
+DWARF:
+  debug_pubtypes:
+    Length:
+      TotalLength: 0x1234
+    Version:    2
+    UnitOffset: 0x1234
+    UnitSize:   0x4321
+    Entries:    []
diff --git a/llvm/test/tools/yaml2obj/ELF/content-array.yaml b/llvm/test/tools/yaml2obj/ELF/content-array.yaml
new file mode 100644
index 0000000000000..0e1732efa64d6
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/ELF/content-array.yaml
@@ -0,0 +1,94 @@
+## Check we are able to describe the content of a section
+## using the ContentArray key.
+
+## Check we are able to use ContentArray to create multi-line descriptions
+## of section contents with comments on the same line.
+# RUN: yaml2obj --docnum=1 %s -o %t1
+# RUN: llvm-readobj --sections --section-data %t1 | FileCheck %s
+
+# CHECK:      Section {
+# CHECK:        Index: 1
+# CHECK-NEXT:   Name: .foo
+# CHECK-NEXT:   Type: SHT_PROGBITS
+# CHECK-NEXT:   Flags [
+# CHECK-NEXT:   ]
+# CHECK-NEXT:   Address: 0x0
+# CHECK-NEXT:   Offset: 0x40
+# CHECK-NEXT:   Size: 16
+# CHECK-NEXT:   Link: 0
+# CHECK-NEXT:   Info: 0
+# CHECK-NEXT:   AddressAlignment: 0
+# CHECK-NEXT:   EntrySize: 0
+# CHECK-NEXT:   SectionData (
+# CHECK-NEXT:     0000: 11223344 55667788 99AABBCC DDEEFF00
+# CHECK-NEXT:   )
+# CHECK-NEXT: }
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name:         .foo
+    Type:         SHT_PROGBITS
+    ContentArray: [ 0x11, 0x22, 0x33, 0x44,                                ## .long 11223344
+                    0x55, 0x66,                                            ## .short 5566.
+                    0x77,                                                  ## .byte 0x77
+                    0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00 ] ## .quad 0x8899aabbccddeeff00
+
+## Check we do not allow using 'Content' and 'ContentArray' at the same time.
+# RUN: not yaml2obj --docnum=2 %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=BOTH
+# BOTH: error: Content and ContentArray can't be used together
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name:         .foo
+    Type:         SHT_PROGBITS
+    Content:      [ 0x0 ]
+    ContentArray: [ 0x1 ]
+
+## Check how the "Size" and the "ContentArray" keys can be used together.
+
+## Case A: check that we report an error when the the value of "Size" is less than the content size.
+# RUN: not yaml2obj --docnum=3 -DSIZE=1 %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=SIZE-LESS
+# SIZE-LESS: error: Section size must be greater than or equal to the content size
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name:         .foo
+    Type:         SHT_PROGBITS
+    ContentArray: [ 0x11, 0x22 ]
+    Size:         [[SIZE]]
+
+## Case B: check we are able to produce an output when the value of "Size" is equal
+##         to the content size. In this case the "Size" key has no effect.
+# RUN: yaml2obj --docnum=3 -DSIZE=2 %s -o %t3.eq
+# RUN: llvm-readobj --sections --section-data %t3.eq | FileCheck %s --check-prefix=SIZE-EQ
+
+# SIZE-EQ:      Name: .foo
+# SIZE-EQ:      SectionData (
+# SIZE-EQ-NEXT:   0000: 1122 |
+# SIZE-EQ-NEXT: )
+
+## Case C: check we are able to produce an output when the value of "Size" is greater
+##         than the content size. In this case zeroes are added as padding after the
+##         specified content.
+# RUN: yaml2obj --docnum=3 -DSIZE=3 %s -o %t4.gr
+# RUN: llvm-readobj --sections --section-data %t4.gr | FileCheck %s --check-prefix=SIZE-GR
+
+# SIZE-GR:      Name: .foo
+# SIZE-GR:      SectionData (
+# SIZE-GR-NEXT:   0000: 112200 |
+# SIZE-GR-NEXT: )
diff --git a/llvm/test/tools/yaml2obj/lit.local.cfg b/llvm/test/tools/yaml2obj/lit.local.cfg
deleted file mode 100644
index db82cc231003a..0000000000000
--- a/llvm/test/tools/yaml2obj/lit.local.cfg
+++ /dev/null
@@ -1 +0,0 @@
-config.suffixes = ['.test', '.yaml']
diff --git a/llvm/test/tools/yaml2obj/macro.yaml b/llvm/test/tools/yaml2obj/macro.yaml
index d7834fa6d97a3..b299dc9f7f979 100644
--- a/llvm/test/tools/yaml2obj/macro.yaml
+++ b/llvm/test/tools/yaml2obj/macro.yaml
@@ -58,3 +58,34 @@ Symbols:
   - Name: "[[a]"
   - Name: "[[a[[a]]"
   - Name: "[[a][[a]][[b]]"
+
+## Check that it is possible to set a default value for a macro in the YAML description.
+## Check that we are able to provide an empty default value for a macro.
+# RUN: yaml2obj --docnum=4 %s -o %t4.1
+# RUN: llvm-nm --no-sort %t4.1 | FileCheck %s --check-prefix=DEFAULT
+
+# DEFAULT:      U _foo_
+# DEFAULT-NEXT: U __
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
+Symbols:
+  - Name: _[[FOO=foo]]_
+  - Name: _[[BAR=]]_
+
+## Check that it is possible to override a default macro value.
+# RUN: yaml2obj --docnum=4 %s -o %t4.2 -DFOO=bar -DBAR=foo
+# RUN: llvm-nm --no-sort %t4.2 | FileCheck %s --check-prefix=OVERRIDE-DEFAULT1
+
+# OVERRIDE-DEFAULT1:      U _bar_
+# OVERRIDE-DEFAULT1-NEXT: U _foo_
+
+# RUN: yaml2obj --docnum=4 %s -o %t4.3 -DFOO=bar=foo -DBAR=foo=bar
+# RUN: llvm-nm --no-sort %t4.3 | FileCheck %s --check-prefix=OVERRIDE-DEFAULT2
+
+# OVERRIDE-DEFAULT2:      U _bar=foo_
+# OVERRIDE-DEFAULT2-NEXT: U _foo=bar_
diff --git a/llvm/tools/llvm-as/llvm-as.cpp b/llvm/tools/llvm-as/llvm-as.cpp
index 699e9121b88a5..f2b52890a7f5c 100644
--- a/llvm/tools/llvm-as/llvm-as.cpp
+++ b/llvm/tools/llvm-as/llvm-as.cpp
@@ -88,7 +88,7 @@ static void WriteOutputFile(const Module *M, const ModuleSummaryIndex *Index) {
     exit(1);
   }
 
-  if (Force || !CheckBitcodeOutputToConsole(Out->os(), true)) {
+  if (Force || !CheckBitcodeOutputToConsole(Out->os())) {
     const ModuleSummaryIndex *IndexToWrite = nullptr;
     // Don't attempt to write a summary index unless it contains any entries or
     // has non-zero flags. The latter is used to assemble dummy index files for
diff --git a/llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp b/llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp
index 47e426b2561b9..d8fa4f9953dca 100644
--- a/llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp
+++ b/llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp
@@ -63,7 +63,7 @@ class parser<OffsetOption> final : public basic_parser<OffsetOption> {
       return false;
     }
     if (Arg.getAsInteger(0, Val.Val))
-      return O.error("'" + Arg + "' value invalid for integer argument!");
+      return O.error("'" + Arg + "' value invalid for integer argument");
     Val.HasValue = true;
     Val.IsRequested = true;
     return false;
@@ -89,7 +89,7 @@ template <> class parser<BoolOption> final : public basic_parser<BoolOption> {
   /// Return true on error.
   bool parse(Option &O, StringRef ArgName, StringRef Arg, BoolOption &Val) {
     if (Arg != "")
-      return O.error("this is a flag and does not take a value.");
+      return O.error("this is a flag and does not take a value");
     Val.Val = 0;
     Val.HasValue = false;
     Val.IsRequested = true;
diff --git a/llvm/tools/llvm-dwp/llvm-dwp.cpp b/llvm/tools/llvm-dwp/llvm-dwp.cpp
index 24063c8355228..d5ebe5ab0a57a 100644
--- a/llvm/tools/llvm-dwp/llvm-dwp.cpp
+++ b/llvm/tools/llvm-dwp/llvm-dwp.cpp
@@ -30,6 +30,7 @@
 #include "llvm/MC/MCTargetOptionsCommandFlags.h"
 #include "llvm/Object/Decompressor.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/DataExtractor.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h
index d4bad347f6047..c883a3409ae31 100644
--- a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h
+++ b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h
@@ -74,7 +74,8 @@ struct InstructionBenchmark {
   std::string Error;
   std::string Info;
   std::vector<uint8_t> AssembledSnippet;
-
+  // How to aggregate measurements.
+  enum ResultAggregationModeE { Min, Max, Mean, MinVariance };
   // Read functions.
   static Expected<InstructionBenchmark> readYaml(const LLVMState &State,
                                                  StringRef Filename);
diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
index a5565bdfa7238..bdef8f8a89189 100644
--- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
+++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
@@ -46,9 +46,29 @@ class FunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
 
 private:
   Expected<int64_t> runAndMeasure(const char *Counters) const override {
+    auto ResultOrError = runAndSample(Counters);
+    if (ResultOrError)
+      return ResultOrError.get()[0];
+    return ResultOrError.takeError();
+  }
+
+  static void
+  accumulateCounterValues(const llvm::SmallVector<int64_t, 4> &NewValues,
+                          llvm::SmallVector<int64_t, 4> *Result) {
+
+    const size_t NumValues = std::max(NewValues.size(), Result->size());
+    if (NumValues > Result->size())
+      Result->resize(NumValues, 0);
+    for (size_t I = 0, End = NewValues.size(); I < End; ++I)
+      (*Result)[I] += NewValues[I];
+  }
+
+  Expected<llvm::SmallVector<int64_t, 4>>
+  runAndSample(const char *Counters) const override {
     // We sum counts when there are several counters for a single ProcRes
     // (e.g. P23 on SandyBridge).
-    int64_t CounterValue = 0;
+    llvm::SmallVector<int64_t, 4> CounterValues;
+    int Reserved = 0;
     SmallVector<StringRef, 2> CounterNames;
     StringRef(Counters).split(CounterNames, '+');
     char *const ScratchPtr = Scratch->ptr();
@@ -61,6 +81,17 @@ class FunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
         return CounterOrError.takeError();
 
       pfm::Counter *Counter = CounterOrError.get().get();
+      if (Reserved == 0) {
+        Reserved = Counter->numValues();
+        CounterValues.reserve(Reserved);
+      } else if (Reserved != Counter->numValues())
+        // It'd be wrong to accumulate vectors of different sizes.
+        return make_error<Failure>(
+            llvm::Twine("Inconsistent number of values for counter ")
+                .concat(CounterName)
+                .concat(std::to_string(Counter->numValues()))
+                .concat(" vs expected of ")
+                .concat(std::to_string(Reserved)));
       Scratch->clear();
       {
         CrashRecoveryContext CRC;
@@ -75,9 +106,13 @@ class FunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
         if (Crashed)
           return make_error<SnippetCrash>("snippet crashed while running");
       }
-      CounterValue += Counter->read();
+      auto ValueOrError = Counter->readOrError();
+      if (!ValueOrError)
+        return ValueOrError.takeError();
+
+      accumulateCounterValues(ValueOrError.get(), &CounterValues);
     }
-    return CounterValue;
+    return CounterValues;
   }
 
   const LLVMState &State;
diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h
index b0fdb34450ee3..cc51b5b8486b0 100644
--- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h
+++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h
@@ -21,6 +21,7 @@
 #include "LlvmState.h"
 #include "MCInstrDescView.h"
 #include "SnippetRepetitor.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/Support/Error.h"
 #include <cstdlib>
@@ -65,7 +66,11 @@ class BenchmarkRunner {
   class FunctionExecutor {
   public:
     virtual ~FunctionExecutor();
+    // FIXME deprecate this.
     virtual Expected<int64_t> runAndMeasure(const char *Counters) const = 0;
+
+    virtual Expected<llvm::SmallVector<int64_t, 4>>
+    runAndSample(const char *Counters) const = 0;
   };
 
 protected:
diff --git a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp
index 83a3f0183e39d..6cdefb8b06791 100644
--- a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp
+++ b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp
@@ -8,48 +8,135 @@
 
 #include "LatencyBenchmarkRunner.h"
 
-#include "Target.h"
 #include "BenchmarkRunner.h"
+#include "Target.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/Error.h"
+#include <algorithm>
+#include <cmath>
 
 namespace llvm {
 namespace exegesis {
 
-LatencyBenchmarkRunner::LatencyBenchmarkRunner(const LLVMState &State,
-                                               InstructionBenchmark::ModeE Mode)
+LatencyBenchmarkRunner::LatencyBenchmarkRunner(
+    const LLVMState &State, InstructionBenchmark::ModeE Mode,
+    InstructionBenchmark::ResultAggregationModeE ResultAgg)
     : BenchmarkRunner(State, Mode) {
   assert((Mode == InstructionBenchmark::Latency ||
           Mode == InstructionBenchmark::InverseThroughput) &&
          "invalid mode");
+  ResultAggMode = ResultAgg;
 }
 
 LatencyBenchmarkRunner::~LatencyBenchmarkRunner() = default;
 
+static double computeVariance(const llvm::SmallVector<int64_t, 4> &Values) {
+  if (Values.empty())
+    return 0.0;
+  double Sum = std::accumulate(Values.begin(), Values.end(), 0.0);
+
+  const double Mean = Sum / Values.size();
+  double Ret = 0;
+  for (const auto &V : Values) {
+    double Delta = V - Mean;
+    Ret += Delta * Delta;
+  }
+  return Ret / Values.size();
+}
+
+static int64_t findMin(const llvm::SmallVector<int64_t, 4> &Values) {
+  if (Values.empty())
+    return 0;
+  return *std::min_element(Values.begin(), Values.end());
+}
+
+static int64_t findMax(const llvm::SmallVector<int64_t, 4> &Values) {
+  if (Values.empty())
+    return 0;
+  return *std::max_element(Values.begin(), Values.end());
+}
+
+static int64_t findMean(const llvm::SmallVector<int64_t, 4> &Values) {
+  if (Values.empty())
+    return 0;
+  return std::accumulate(Values.begin(), Values.end(), 0.0) /
+         static_cast<double>(Values.size());
+}
+
 Expected<std::vector<BenchmarkMeasure>> LatencyBenchmarkRunner::runMeasurements(
     const FunctionExecutor &Executor) const {
   // Cycle measurements include some overhead from the kernel. Repeat the
-  // measure several times and take the minimum value.
+  // measure several times and return the aggregated value, as specified by
+  // ResultAggMode.
   constexpr const int NumMeasurements = 30;
-  int64_t MinValue = std::numeric_limits<int64_t>::max();
+  llvm::SmallVector<int64_t, 4> AccumulatedValues;
+  double MinVariance = std::numeric_limits<double>::infinity();
   const char *CounterName = State.getPfmCounters().CycleCounter;
+  // Values count for each run.
+  int ValuesCount = 0;
   for (size_t I = 0; I < NumMeasurements; ++I) {
-    auto ExpectedCounterValue = Executor.runAndMeasure(CounterName);
-    if (!ExpectedCounterValue)
-      return ExpectedCounterValue.takeError();
-    if (*ExpectedCounterValue < MinValue)
-      MinValue = *ExpectedCounterValue;
+    auto ExpectedCounterValues = Executor.runAndSample(CounterName);
+    if (!ExpectedCounterValues)
+      return ExpectedCounterValues.takeError();
+    ValuesCount = ExpectedCounterValues.get().size();
+    if (ValuesCount == 1)
+      AccumulatedValues.push_back(ExpectedCounterValues.get()[0]);
+    else {
+      // We'll keep the reading with lowest variance (ie., most stable)
+      double Variance = computeVariance(*ExpectedCounterValues);
+      if (MinVariance > Variance) {
+        AccumulatedValues = std::move(ExpectedCounterValues.get());
+        MinVariance = Variance;
+      }
+    }
   }
-  std::vector<BenchmarkMeasure> Result;
+
+  std::string ModeName;
   switch (Mode) {
   case InstructionBenchmark::Latency:
-    Result = {BenchmarkMeasure::Create("latency", MinValue)};
+    ModeName = "latency";
     break;
   case InstructionBenchmark::InverseThroughput:
-    Result = {BenchmarkMeasure::Create("inverse_throughput", MinValue)};
+    ModeName = "inverse_throughput";
     break;
   default:
     break;
   }
-  return std::move(Result);
+
+  switch (ResultAggMode) {
+  case InstructionBenchmark::MinVariance: {
+    if (ValuesCount == 1)
+      llvm::errs() << "Each sample only has one value. result-aggregation-mode "
+                      "of min-variance is probably non-sensical\n";
+    std::vector<BenchmarkMeasure> Result;
+    Result.reserve(AccumulatedValues.size());
+    for (const int64_t Value : AccumulatedValues)
+      Result.push_back(BenchmarkMeasure::Create(ModeName, Value));
+    return std::move(Result);
+  }
+  case InstructionBenchmark::Min: {
+    std::vector<BenchmarkMeasure> Result;
+    Result.push_back(
+        BenchmarkMeasure::Create(ModeName, findMin(AccumulatedValues)));
+    return std::move(Result);
+  }
+  case InstructionBenchmark::Max: {
+    std::vector<BenchmarkMeasure> Result;
+    Result.push_back(
+        BenchmarkMeasure::Create(ModeName, findMax(AccumulatedValues)));
+    return std::move(Result);
+  }
+  case InstructionBenchmark::Mean: {
+    std::vector<BenchmarkMeasure> Result;
+    Result.push_back(
+        BenchmarkMeasure::Create(ModeName, findMean(AccumulatedValues)));
+    return std::move(Result);
+  }
+  }
+  return llvm::make_error<Failure>(llvm::Twine("Unexpected benchmark mode(")
+                                       .concat(std::to_string(Mode))
+                                       .concat(" and unexpected ResultAggMode ")
+                                       .concat(std::to_string(ResultAggMode)));
 }
 
 } // namespace exegesis
diff --git a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h
index d4bb93416c204..b9b9efc25d140 100644
--- a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h
+++ b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h
@@ -21,13 +21,16 @@ namespace exegesis {
 
 class LatencyBenchmarkRunner : public BenchmarkRunner {
 public:
-  LatencyBenchmarkRunner(const LLVMState &State,
-                         InstructionBenchmark::ModeE Mode);
+  LatencyBenchmarkRunner(
+      const LLVMState &State, InstructionBenchmark::ModeE Mode,
+      InstructionBenchmark::ResultAggregationModeE ResultAggMode);
   ~LatencyBenchmarkRunner() override;
 
 private:
   Expected<std::vector<BenchmarkMeasure>>
   runMeasurements(const FunctionExecutor &Executor) const override;
+
+  InstructionBenchmark::ResultAggregationModeE ResultAggMode;
 };
 } // namespace exegesis
 } // namespace llvm
diff --git a/llvm/tools/llvm-exegesis/lib/PerfHelper.cpp b/llvm/tools/llvm-exegesis/lib/PerfHelper.cpp
index c372ac4f364e3..cba4846709e80 100644
--- a/llvm/tools/llvm-exegesis/lib/PerfHelper.cpp
+++ b/llvm/tools/llvm-exegesis/lib/PerfHelper.cpp
@@ -119,23 +119,27 @@ void Counter::stop() { ioctl(FileDescriptor, PERF_EVENT_IOC_DISABLE, 0); }
 
 int64_t Counter::read() const {
   auto ValueOrError = readOrError();
-  if (ValueOrError)
-    return ValueOrError.get();
-
-  errs() << ValueOrError.takeError() << "\n";
+  if (ValueOrError) {
+    if (!ValueOrError.get().empty())
+      return ValueOrError.get()[0];
+    errs() << "Counter has no reading\n";
+  } else
+    errs() << ValueOrError.takeError() << "\n";
   return -1;
 }
 
-llvm::Expected<int64_t> Counter::readOrError() const {
+llvm::Expected<llvm::SmallVector<int64_t, 4>> Counter::readOrError() const {
   int64_t Count = 0;
   ssize_t ReadSize = ::read(FileDescriptor, &Count, sizeof(Count));
   if (ReadSize != sizeof(Count))
     return llvm::make_error<llvm::StringError>("Failed to read event counter",
                                                llvm::errc::io_error);
-
-  return Count;
+  llvm::SmallVector<int64_t, 4> Result;
+  Result.push_back(Count);
+  return Result;
 }
 
+int Counter::numValues() const { return 1; }
 #else
 
 Counter::Counter(PerfEvent &&Event) : Event(std::move(Event)) {}
@@ -148,11 +152,13 @@ void Counter::stop() {}
 
 int64_t Counter::read() const { return 42; }
 
-llvm::Expected<int64_t> Counter::readOrError() const {
+llvm::Expected<llvm::SmallVector<int64_t, 4>> Counter::readOrError() const {
   return llvm::make_error<llvm::StringError>("Not implemented",
                                              llvm::errc::io_error);
 }
 
+int Counter::numValues() const { return 1; }
+
 #endif
 
 } // namespace pfm
diff --git a/llvm/tools/llvm-exegesis/lib/PerfHelper.h b/llvm/tools/llvm-exegesis/lib/PerfHelper.h
index 7562af9c45242..d41b090e85f17 100644
--- a/llvm/tools/llvm-exegesis/lib/PerfHelper.h
+++ b/llvm/tools/llvm-exegesis/lib/PerfHelper.h
@@ -15,9 +15,11 @@
 #define LLVM_TOOLS_LLVM_EXEGESIS_PERFHELPER_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Config/config.h"
 #include "llvm/Support/Error.h"
+
 #include <cstdint>
 #include <functional>
 #include <memory>
@@ -85,7 +87,9 @@ class Counter {
   int64_t read() const;
 
   /// Returns the current value of the counter or error if it cannot be read.
-  virtual llvm::Expected<int64_t> readOrError() const;
+  virtual llvm::Expected<llvm::SmallVector<int64_t, 4>> readOrError() const;
+
+  virtual int numValues() const;
 
 private:
   PerfEvent Event;
diff --git a/llvm/tools/llvm-exegesis/lib/Target.cpp b/llvm/tools/llvm-exegesis/lib/Target.cpp
index 6150c738dad84..ad26c1678c78f 100644
--- a/llvm/tools/llvm-exegesis/lib/Target.cpp
+++ b/llvm/tools/llvm-exegesis/lib/Target.cpp
@@ -68,8 +68,9 @@ std::unique_ptr<SnippetGenerator> ExegesisTarget::createSnippetGenerator(
 }
 
 Expected<std::unique_ptr<BenchmarkRunner>>
-ExegesisTarget::createBenchmarkRunner(InstructionBenchmark::ModeE Mode,
-                                      const LLVMState &State) const {
+ExegesisTarget::createBenchmarkRunner(
+    InstructionBenchmark::ModeE Mode, const LLVMState &State,
+    InstructionBenchmark::ResultAggregationModeE ResultAggMode) const {
   PfmCountersInfo PfmCounters = State.getPfmCounters();
   switch (Mode) {
   case InstructionBenchmark::Unknown:
@@ -85,12 +86,12 @@ ExegesisTarget::createBenchmarkRunner(InstructionBenchmark::ModeE Mode,
               .concat(ModeName)
               .concat("' mode, sched model does not define a cycle counter."));
     }
-    return createLatencyBenchmarkRunner(State, Mode);
+    return createLatencyBenchmarkRunner(State, Mode, ResultAggMode);
   case InstructionBenchmark::Uops:
     if (!PfmCounters.UopsCounter && !PfmCounters.IssueCounters)
       return make_error<Failure>("can't run 'uops' mode, sched model does not "
                                  "define uops or issue counters.");
-    return createUopsBenchmarkRunner(State);
+    return createUopsBenchmarkRunner(State, ResultAggMode);
   }
   return nullptr;
 }
@@ -106,12 +107,14 @@ std::unique_ptr<SnippetGenerator> ExegesisTarget::createParallelSnippetGenerator
 }
 
 std::unique_ptr<BenchmarkRunner> ExegesisTarget::createLatencyBenchmarkRunner(
-    const LLVMState &State, InstructionBenchmark::ModeE Mode) const {
-  return std::make_unique<LatencyBenchmarkRunner>(State, Mode);
+    const LLVMState &State, InstructionBenchmark::ModeE Mode,
+    InstructionBenchmark::ResultAggregationModeE ResultAggMode) const {
+  return std::make_unique<LatencyBenchmarkRunner>(State, Mode, ResultAggMode);
 }
 
-std::unique_ptr<BenchmarkRunner>
-ExegesisTarget::createUopsBenchmarkRunner(const LLVMState &State) const {
+std::unique_ptr<BenchmarkRunner> ExegesisTarget::createUopsBenchmarkRunner(
+    const LLVMState &State,
+    InstructionBenchmark::ResultAggregationModeE /*unused*/) const {
   return std::make_unique<UopsBenchmarkRunner>(State);
 }
 
diff --git a/llvm/tools/llvm-exegesis/lib/Target.h b/llvm/tools/llvm-exegesis/lib/Target.h
index bcc283ee8fe7d..70890795426d9 100644
--- a/llvm/tools/llvm-exegesis/lib/Target.h
+++ b/llvm/tools/llvm-exegesis/lib/Target.h
@@ -148,9 +148,10 @@ class ExegesisTarget {
                          const LLVMState &State,
                          const SnippetGenerator::Options &Opts) const;
   // Creates a benchmark runner for the given mode.
-  Expected<std::unique_ptr<BenchmarkRunner>>
-  createBenchmarkRunner(InstructionBenchmark::ModeE Mode,
-                        const LLVMState &State) const;
+  Expected<std::unique_ptr<BenchmarkRunner>> createBenchmarkRunner(
+      InstructionBenchmark::ModeE Mode, const LLVMState &State,
+      InstructionBenchmark::ResultAggregationModeE ResultAggMode =
+          InstructionBenchmark::Min) const;
 
   // Returns the ExegesisTarget for the given triple or nullptr if the target
   // does not exist.
@@ -176,9 +177,11 @@ class ExegesisTarget {
   std::unique_ptr<SnippetGenerator> virtual createParallelSnippetGenerator(
       const LLVMState &State, const SnippetGenerator::Options &Opts) const;
   std::unique_ptr<BenchmarkRunner> virtual createLatencyBenchmarkRunner(
-      const LLVMState &State, InstructionBenchmark::ModeE Mode) const;
+      const LLVMState &State, InstructionBenchmark::ModeE Mode,
+      InstructionBenchmark::ResultAggregationModeE ResultAggMode) const;
   std::unique_ptr<BenchmarkRunner> virtual createUopsBenchmarkRunner(
-      const LLVMState &State) const;
+      const LLVMState &State,
+      InstructionBenchmark::ResultAggregationModeE ResultAggMode) const;
 
   const ExegesisTarget *Next = nullptr;
   const ArrayRef<CpuAndPfmCounters> CpuPfmCounters;
diff --git a/llvm/tools/llvm-exegesis/lib/X86/Target.cpp b/llvm/tools/llvm-exegesis/lib/X86/Target.cpp
index fe235e8a3dfb1..7a84f936e0d0e 100644
--- a/llvm/tools/llvm-exegesis/lib/X86/Target.cpp
+++ b/llvm/tools/llvm-exegesis/lib/X86/Target.cpp
@@ -40,6 +40,7 @@ static const char *isInvalidMemoryInstr(const Instruction &Instr) {
   case X86II::MRMSrcRegOp4:
   case X86II::MRMSrcRegCC:
   case X86II::MRMXrCC:
+  case X86II::MRMr0:
   case X86II::MRMXr:
   case X86II::MRM0r:
   case X86II::MRM1r:
diff --git a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
index ce3a31c12d4ae..507015b97472b 100644
--- a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
+++ b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
@@ -83,6 +83,21 @@ static cl::opt<exegesis::InstructionBenchmark::ModeE> BenchmarkMode(
                clEnumValN(exegesis::InstructionBenchmark::Unknown, "analysis",
                           "Analysis")));
 
+static cl::opt<exegesis::InstructionBenchmark::ResultAggregationModeE>
+    ResultAggMode(
+        "result-aggregation-mode",
+        cl::desc("How to aggregate multi-values result"), cl::cat(Options),
+        cl::values(clEnumValN(exegesis::InstructionBenchmark::Min, "min",
+                              "Keep min reading"),
+                   clEnumValN(exegesis::InstructionBenchmark::Max, "max",
+                              "Keep max reading"),
+                   clEnumValN(exegesis::InstructionBenchmark::Mean, "mean",
+                              "Compute mean of all readings"),
+                   clEnumValN(exegesis::InstructionBenchmark::MinVariance,
+                              "min-variance",
+                              "Keep readings set with min-variance")),
+        cl::init(exegesis::InstructionBenchmark::Min));
+
 static cl::opt<exegesis::InstructionBenchmark::RepetitionModeE> RepetitionMode(
     "repetition-mode", cl::desc("how to repeat the instruction snippet"),
     cl::cat(BenchmarkOptions),
@@ -281,8 +296,9 @@ void benchmarkMain() {
 
   const LLVMState State(CpuName);
 
-  const std::unique_ptr<BenchmarkRunner> Runner = ExitOnErr(
-      State.getExegesisTarget().createBenchmarkRunner(BenchmarkMode, State));
+  const std::unique_ptr<BenchmarkRunner> Runner =
+      ExitOnErr(State.getExegesisTarget().createBenchmarkRunner(
+          BenchmarkMode, State, ResultAggMode));
   if (!Runner) {
     ExitWithError("cannot create benchmark runner");
   }
diff --git a/llvm/tools/llvm-extract/llvm-extract.cpp b/llvm/tools/llvm-extract/llvm-extract.cpp
index 9be4627e2ed70..cb1c4116ff192 100644
--- a/llvm/tools/llvm-extract/llvm-extract.cpp
+++ b/llvm/tools/llvm-extract/llvm-extract.cpp
@@ -381,7 +381,7 @@ int main(int argc, char **argv) {
   if (OutputAssembly)
     Passes.add(
         createPrintModulePass(Out.os(), "", PreserveAssemblyUseListOrder));
-  else if (Force || !CheckBitcodeOutputToConsole(Out.os(), true))
+  else if (Force || !CheckBitcodeOutputToConsole(Out.os()))
     Passes.add(createBitcodeWriterPass(Out.os(), PreserveBitcodeUseListOrder));
 
   Passes.run(*M.get());
diff --git a/llvm/tools/llvm-isel-fuzzer/llvm-isel-fuzzer.cpp b/llvm/tools/llvm-isel-fuzzer/llvm-isel-fuzzer.cpp
index 627e9ab4c03fb..a93a04f65aac1 100644
--- a/llvm/tools/llvm-isel-fuzzer/llvm-isel-fuzzer.cpp
+++ b/llvm/tools/llvm-isel-fuzzer/llvm-isel-fuzzer.cpp
@@ -24,6 +24,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/IRReader/IRReader.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/SourceMgr.h"
diff --git a/llvm/tools/llvm-link/llvm-link.cpp b/llvm/tools/llvm-link/llvm-link.cpp
index d99659f3d50ae..a7cda24bbe0a9 100644
--- a/llvm/tools/llvm-link/llvm-link.cpp
+++ b/llvm/tools/llvm-link/llvm-link.cpp
@@ -399,7 +399,7 @@ int main(int argc, char **argv) {
     errs() << "Writing bitcode...\n";
   if (OutputAssembly) {
     Composite->print(Out.os(), nullptr, PreserveAssemblyUseListOrder);
-  } else if (Force || !CheckBitcodeOutputToConsole(Out.os(), true))
+  } else if (Force || !CheckBitcodeOutputToConsole(Out.os()))
     WriteBitcodeToFile(*Composite, Out.os(), PreserveBitcodeUseListOrder);
 
   // Declare success.
diff --git a/llvm/tools/llvm-nm/llvm-nm.cpp b/llvm/tools/llvm-nm/llvm-nm.cpp
index 4385e2aaa2876..ecd1e21e15bfb 100644
--- a/llvm/tools/llvm-nm/llvm-nm.cpp
+++ b/llvm/tools/llvm-nm/llvm-nm.cpp
@@ -182,8 +182,10 @@ cl::opt<bool> JustSymbolName("just-symbol-name",
 cl::alias JustSymbolNames("j", cl::desc("Alias for --just-symbol-name"),
                           cl::aliasopt(JustSymbolName), cl::Grouping);
 
-cl::opt<bool> SpecialSyms("special-syms",
-                          cl::desc("No-op. Used for GNU compatibility only"));
+cl::opt<bool>
+    SpecialSyms("special-syms",
+                cl::desc("Do not filter special symbols from the output"),
+                cl::cat(NMCat));
 
 cl::list<std::string> SegSect("s", cl::multi_val(2), cl::ZeroOrMore,
                               cl::value_desc("segment section"), cl::Hidden,
@@ -733,6 +735,16 @@ static void writeFileName(raw_ostream &S, StringRef ArchiveName,
   }
 }
 
+static bool isSpecialSym(SymbolicFile &Obj, StringRef Name) {
+  auto *ELFObj = dyn_cast<ELFObjectFileBase>(&Obj);
+  if (!ELFObj)
+    return false;
+  uint16_t EMachine = ELFObj->getEMachine();
+  if (EMachine != ELF::EM_ARM && EMachine != ELF::EM_AARCH64)
+    return false;
+  return !Name.empty() && Name[0] == '$';
+}
+
 static void sortAndPrintSymbolList(SymbolicFile &Obj, bool printName,
                                    StringRef ArchiveName,
                                    StringRef ArchitectureName) {
@@ -822,7 +834,8 @@ static void sortAndPrintSymbolList(SymbolicFile &Obj, bool printName,
     bool Global = SymFlags & SymbolRef::SF_Global;
     bool Weak = SymFlags & SymbolRef::SF_Weak;
     if ((!Undefined && UndefinedOnly) || (Undefined && DefinedOnly) ||
-        (!Global && ExternalOnly) || (Weak && NoWeakSymbols))
+        (!Global && ExternalOnly) || (Weak && NoWeakSymbols) ||
+        (!SpecialSyms && isSpecialSym(Obj, Name)))
       continue;
     if (PrintFileName)
       writeFileName(outs(), ArchiveName, ArchitectureName);
diff --git a/llvm/tools/llvm-objcopy/CopyConfig.cpp b/llvm/tools/llvm-objcopy/CopyConfig.cpp
index 1e151f01e01e2..1fde54dd290ae 100644
--- a/llvm/tools/llvm-objcopy/CopyConfig.cpp
+++ b/llvm/tools/llvm-objcopy/CopyConfig.cpp
@@ -393,9 +393,30 @@ template <class T> static ErrorOr<T> getAsInteger(StringRef Val) {
   return Result;
 }
 
+namespace {
+
+enum class ToolType { Objcopy, Strip, InstallNameTool };
+
+} // anonymous namespace
+
 static void printHelp(const opt::OptTable &OptTable, raw_ostream &OS,
-                      StringRef ToolName) {
-  OptTable.PrintHelp(OS, (ToolName + " input [output]").str().c_str(),
+                      ToolType Tool) {
+  StringRef HelpText, ToolName;
+  switch (Tool) {
+  case ToolType::Objcopy:
+    ToolName = "llvm-objcopy";
+    HelpText = " [options] input [output]";
+    break;
+  case ToolType::Strip:
+    ToolName = "llvm-strip";
+    HelpText = " [options] inputs...";
+    break;
+  case ToolType::InstallNameTool:
+    ToolName = "llvm-install-name-tool";
+    HelpText = " [options] input";
+    break;
+  }
+  OptTable.PrintHelp(OS, (ToolName + HelpText).str().c_str(),
                      (ToolName + " tool").str().c_str());
   // TODO: Replace this with libOption call once it adds extrahelp support.
   // The CommandLine library has a cl::extrahelp class to support this,
@@ -416,12 +437,12 @@ parseObjcopyOptions(ArrayRef<const char *> ArgsArr,
       T.ParseArgs(ArgsArr, MissingArgumentIndex, MissingArgumentCount);
 
   if (InputArgs.size() == 0) {
-    printHelp(T, errs(), "llvm-objcopy");
+    printHelp(T, errs(), ToolType::Objcopy);
     exit(1);
   }
 
   if (InputArgs.hasArg(OBJCOPY_help)) {
-    printHelp(T, outs(), "llvm-objcopy");
+    printHelp(T, outs(), ToolType::Objcopy);
     exit(0);
   }
 
@@ -806,13 +827,20 @@ parseInstallNameToolOptions(ArrayRef<const char *> ArgsArr) {
   llvm::opt::InputArgList InputArgs =
       T.ParseArgs(ArgsArr, MissingArgumentIndex, MissingArgumentCount);
 
+  if (MissingArgumentCount)
+    return createStringError(
+        errc::invalid_argument,
+        "missing argument to " +
+            StringRef(InputArgs.getArgString(MissingArgumentIndex)) +
+            " option");
+
   if (InputArgs.size() == 0) {
-    printHelp(T, errs(), "llvm-install-name-tool");
+    printHelp(T, errs(), ToolType::InstallNameTool);
     exit(1);
   }
 
   if (InputArgs.hasArg(INSTALL_NAME_TOOL_help)) {
-    printHelp(T, outs(), "llvm-install-name-tool");
+    printHelp(T, outs(), ToolType::InstallNameTool);
     exit(0);
   }
 
@@ -826,6 +854,61 @@ parseInstallNameToolOptions(ArrayRef<const char *> ArgsArr) {
   for (auto Arg : InputArgs.filtered(INSTALL_NAME_TOOL_add_rpath))
     Config.RPathToAdd.push_back(Arg->getValue());
 
+  for (auto Arg : InputArgs.filtered(INSTALL_NAME_TOOL_delete_rpath)) {
+    StringRef RPath = Arg->getValue();
+
+    // Cannot add and delete the same rpath at the same time.
+    if (is_contained(Config.RPathToAdd, RPath))
+      return createStringError(
+          errc::invalid_argument,
+          "cannot specify both -add_rpath %s and -delete_rpath %s",
+          RPath.str().c_str(), RPath.str().c_str());
+
+    Config.RPathsToRemove.insert(RPath);
+  }
+
+  for (auto *Arg : InputArgs.filtered(INSTALL_NAME_TOOL_rpath)) {
+    StringRef Old = Arg->getValue(0);
+    StringRef New = Arg->getValue(1);
+
+    auto Match = [=](StringRef RPath) { return RPath == Old || RPath == New; };
+
+    // Cannot specify duplicate -rpath entries
+    auto It1 = find_if(
+        Config.RPathsToUpdate,
+        [&Match](const DenseMap<StringRef, StringRef>::value_type &OldNew) {
+          return Match(OldNew.getFirst()) || Match(OldNew.getSecond());
+        });
+    if (It1 != Config.RPathsToUpdate.end())
+      return createStringError(errc::invalid_argument,
+                               "cannot specify both -rpath " + It1->getFirst() +
+                                   " " + It1->getSecond() + " and -rpath " +
+                                   Old + " " + New);
+
+    // Cannot specify the same rpath under both -delete_rpath and -rpath
+    auto It2 = find_if(Config.RPathsToRemove, Match);
+    if (It2 != Config.RPathsToRemove.end())
+      return createStringError(errc::invalid_argument,
+                               "cannot specify both -delete_rpath " + *It2 +
+                                   " and -rpath " + Old + " " + New);
+
+    // Cannot specify the same rpath under both -add_rpath and -rpath
+    auto It3 = find_if(Config.RPathToAdd, Match);
+    if (It3 != Config.RPathToAdd.end())
+      return createStringError(errc::invalid_argument,
+                               "cannot specify both -add_rpath " + *It3 +
+                                   " and -rpath " + Old + " " + New);
+
+    Config.RPathsToUpdate.insert({Old, New});
+  }
+
+  if (auto *Arg = InputArgs.getLastArg(INSTALL_NAME_TOOL_id))
+    Config.SharedLibId = Arg->getValue();
+
+  for (auto *Arg : InputArgs.filtered(INSTALL_NAME_TOOL_change)) {
+    Config.InstallNamesToUpdate.insert({Arg->getValue(0), Arg->getValue(1)});
+  }
+
   SmallVector<StringRef, 2> Positional;
   for (auto Arg : InputArgs.filtered(INSTALL_NAME_TOOL_UNKNOWN))
     return createStringError(errc::invalid_argument, "unknown argument '%s'",
@@ -857,12 +940,12 @@ parseStripOptions(ArrayRef<const char *> ArgsArr,
       T.ParseArgs(ArgsArr, MissingArgumentIndex, MissingArgumentCount);
 
   if (InputArgs.size() == 0) {
-    printHelp(T, errs(), "llvm-strip");
+    printHelp(T, errs(), ToolType::Strip);
     exit(1);
   }
 
   if (InputArgs.hasArg(STRIP_help)) {
-    printHelp(T, outs(), "llvm-strip");
+    printHelp(T, outs(), ToolType::Strip);
     exit(0);
   }
 
diff --git a/llvm/tools/llvm-objcopy/CopyConfig.h b/llvm/tools/llvm-objcopy/CopyConfig.h
index acf783c7f2789..1341dd674c7b7 100644
--- a/llvm/tools/llvm-objcopy/CopyConfig.h
+++ b/llvm/tools/llvm-objcopy/CopyConfig.h
@@ -12,6 +12,7 @@
 #include "ELF/ELFConfig.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitmaskEnum.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
@@ -177,6 +178,12 @@ struct CopyConfig {
   std::vector<StringRef> DumpSection;
   std::vector<StringRef> SymbolsToAdd;
   std::vector<StringRef> RPathToAdd;
+  DenseMap<StringRef, StringRef> RPathsToUpdate;
+  DenseMap<StringRef, StringRef> InstallNamesToUpdate;
+  DenseSet<StringRef> RPathsToRemove;
+
+  // install-name-tool's id option
+  Optional<StringRef> SharedLibId;
 
   // Section matchers
   NameMatcher KeepSection;
diff --git a/llvm/tools/llvm-objcopy/ELF/ELFObjcopy.cpp b/llvm/tools/llvm-objcopy/ELF/ELFObjcopy.cpp
index 3e657d09c469a..66953f9ef0d56 100644
--- a/llvm/tools/llvm-objcopy/ELF/ELFObjcopy.cpp
+++ b/llvm/tools/llvm-objcopy/ELF/ELFObjcopy.cpp
@@ -738,6 +738,12 @@ static Error handleArgs(const CopyConfig &Config, Object &Obj,
     Obj.addSection<GnuDebugLinkSection>(Config.AddGnuDebugLink,
                                         Config.GnuDebugLinkCRC32);
 
+  // If the symbol table was previously removed, we need to create a new one
+  // before adding new symbols.
+  if (!Obj.SymbolTable && !Config.ELF->SymbolsToAdd.empty()) {
+    Obj.addNewSymbolTable();
+  }
+
   for (const NewSymbolInfo &SI : Config.ELF->SymbolsToAdd) {
     SectionBase *Sec = Obj.findSection(SI.SectionName);
     uint64_t Value = Sec ? Sec->Addr + SI.Value : SI.Value;
diff --git a/llvm/tools/llvm-objcopy/ELF/Object.cpp b/llvm/tools/llvm-objcopy/ELF/Object.cpp
index 8bb4bbb40f940..e15fb24f4c425 100644
--- a/llvm/tools/llvm-objcopy/ELF/Object.cpp
+++ b/llvm/tools/llvm-objcopy/ELF/Object.cpp
@@ -608,6 +608,7 @@ static bool isValidReservedSectionIndex(uint16_t Index, uint16_t Machine) {
   if (Machine == EM_HEXAGON) {
     switch (Index) {
     case SHN_HEXAGON_SCOMMON:
+    case SHN_HEXAGON_SCOMMON_1:
     case SHN_HEXAGON_SCOMMON_2:
     case SHN_HEXAGON_SCOMMON_4:
     case SHN_HEXAGON_SCOMMON_8:
@@ -744,7 +745,7 @@ void SymbolTableSection::prepareForLayout() {
   // Reserve proper amount of space in section index table, so we can
   // layout sections correctly. We will fill the table with correct
   // indexes later in fillShdnxTable.
-  if (SectionIndexTable)  
+  if (SectionIndexTable)
     SectionIndexTable->reserve(Symbols.size());
 
   // Add all of our strings to SymbolNames so that SymbolNames has the right
@@ -966,12 +967,22 @@ Error Section::removeSectionReferences(
 }
 
 void GroupSection::finalize() {
-  this->Info = Sym->Index;
-  this->Link = SymTab->Index;
+  this->Info = Sym ? Sym->Index : 0;
+  this->Link = SymTab ? SymTab->Index : 0;
 }
 
 Error GroupSection::removeSectionReferences(
     bool AllowBrokenLinks, function_ref<bool(const SectionBase *)> ToRemove) {
+  if (ToRemove(SymTab)) {
+    if (!AllowBrokenLinks)
+      return createStringError(
+          llvm::errc::invalid_argument,
+          "section '.symtab' cannot be removed because it is "
+          "referenced by the group section '%s'",
+          this->Name.data());
+    SymTab = nullptr;
+    Sym = nullptr;
+  }
   llvm::erase_if(GroupMembers, ToRemove);
   return Error::success();
 }
@@ -1249,7 +1260,7 @@ std::unique_ptr<Object> IHexELFBuilder::build() {
 template <class ELFT> void ELFBuilder<ELFT>::setParentSegment(Segment &Child) {
   for (Segment &Parent : Obj.segments()) {
     // Every segment will overlap with itself but we don't want a segment to
-    // be it's own parent so we avoid that situation.
+    // be its own parent so we avoid that situation.
     if (&Child != &Parent && segmentOverlapsSegment(Child, Parent)) {
       // We want a canonical "most parental" segment but this requires
       // inspecting the ParentSegment.
@@ -1338,18 +1349,20 @@ void ELFBuilder<ELFT>::initGroupSection(GroupSection *GroupSec) {
     error("invalid alignment " + Twine(GroupSec->Align) + " of group section '" +
           GroupSec->Name + "'");
   SectionTableRef SecTable = Obj.sections();
-  auto SymTab = SecTable.template getSectionOfType<SymbolTableSection>(
-      GroupSec->Link,
-      "link field value '" + Twine(GroupSec->Link) + "' in section '" +
-          GroupSec->Name + "' is invalid",
-      "link field value '" + Twine(GroupSec->Link) + "' in section '" +
-          GroupSec->Name + "' is not a symbol table");
-  Symbol *Sym = SymTab->getSymbolByIndex(GroupSec->Info);
-  if (!Sym)
-    error("info field value '" + Twine(GroupSec->Info) + "' in section '" +
-          GroupSec->Name + "' is not a valid symbol index");
-  GroupSec->setSymTab(SymTab);
-  GroupSec->setSymbol(Sym);
+  if (GroupSec->Link != SHN_UNDEF) {
+    auto SymTab = SecTable.template getSectionOfType<SymbolTableSection>(
+        GroupSec->Link,
+        "link field value '" + Twine(GroupSec->Link) + "' in section '" +
+            GroupSec->Name + "' is invalid",
+        "link field value '" + Twine(GroupSec->Link) + "' in section '" +
+            GroupSec->Name + "' is not a symbol table");
+    Symbol *Sym = SymTab->getSymbolByIndex(GroupSec->Info);
+    if (!Sym)
+      error("info field value '" + Twine(GroupSec->Info) + "' in section '" +
+            GroupSec->Name + "' is not a valid symbol index");
+    GroupSec->setSymTab(SymTab);
+    GroupSec->setSymbol(Sym);
+  }
   if (GroupSec->Contents.size() % sizeof(ELF::Elf32_Word) ||
       GroupSec->Contents.empty())
     error("the content of the section " + GroupSec->Name + " is malformed");
@@ -1575,27 +1588,7 @@ template <class ELFT> void ELFBuilder<ELFT>::readSections(bool EnsureSymtab) {
     Obj.SymbolTable->initialize(Obj.sections());
     initSymbolTable(Obj.SymbolTable);
   } else if (EnsureSymtab) {
-    // Reuse an existing SHT_STRTAB section if it exists.
-    StringTableSection *StrTab = nullptr;
-    for (auto &Sec : Obj.sections()) {
-      if (Sec.Type == ELF::SHT_STRTAB && !(Sec.Flags & SHF_ALLOC)) {
-        StrTab = static_cast<StringTableSection *>(&Sec);
-
-        // Prefer a string table that is not the section header string table, if
-        // such a table exists.
-        if (Obj.SectionNames != &Sec)
-          break;
-      }
-    }
-    if (!StrTab)
-      StrTab = &Obj.addSection<StringTableSection>();
-
-    SymbolTableSection &SymTab = Obj.addSection<SymbolTableSection>();
-    SymTab.Name = ".symtab";
-    SymTab.Link = StrTab->Index;
-    SymTab.initialize(Obj.sections());
-    SymTab.addSymbol("", 0, 0, nullptr, 0, 0, 0, 0);
-    Obj.SymbolTable = &SymTab;
+    Obj.addNewSymbolTable();
   }
 
   // Now that all sections and symbols have been added we can add
@@ -1887,6 +1880,33 @@ Error Object::removeSymbols(function_ref<bool(const Symbol &)> ToRemove) {
   return Error::success();
 }
 
+void Object::addNewSymbolTable() {
+  assert(!SymbolTable && "Object must not has a SymbolTable.");
+
+  // Reuse an existing SHT_STRTAB section if it exists.
+  StringTableSection *StrTab = nullptr;
+  for (SectionBase &Sec : sections()) {
+    if (Sec.Type == ELF::SHT_STRTAB && !(Sec.Flags & SHF_ALLOC)) {
+      StrTab = static_cast<StringTableSection *>(&Sec);
+
+      // Prefer a string table that is not the section header string table, if
+      // such a table exists.
+      if (SectionNames != &Sec)
+        break;
+    }
+  }
+  if (!StrTab)
+    StrTab = &addSection<StringTableSection>();
+
+  SymbolTableSection &SymTab = addSection<SymbolTableSection>();
+  SymTab.Name = ".symtab";
+  SymTab.Link = StrTab->Index;
+  SymTab.initialize(sections());
+  SymTab.addSymbol("", 0, 0, nullptr, 0, 0, 0, 0);
+
+  SymbolTable = &SymTab;
+}
+
 void Object::sortSections() {
   // Use stable_sort to maintain the original ordering as closely as possible.
   llvm::stable_sort(Sections, [](const SecPtr &A, const SecPtr &B) {
diff --git a/llvm/tools/llvm-objcopy/ELF/Object.h b/llvm/tools/llvm-objcopy/ELF/Object.h
index c7db57708fc3e..ed89e916b8385 100644
--- a/llvm/tools/llvm-objcopy/ELF/Object.h
+++ b/llvm/tools/llvm-objcopy/ELF/Object.h
@@ -1072,6 +1072,7 @@ class Object {
     Ptr->Index = Sections.size();
     return *Ptr;
   }
+  void addNewSymbolTable();
   Segment &addSegment(ArrayRef<uint8_t> Data) {
     Segments.emplace_back(std::make_unique<Segment>(Data));
     return *Segments.back();
diff --git a/llvm/tools/llvm-objcopy/InstallNameToolOpts.td b/llvm/tools/llvm-objcopy/InstallNameToolOpts.td
index 35047a57994cb..04ffe62c42fca 100644
--- a/llvm/tools/llvm-objcopy/InstallNameToolOpts.td
+++ b/llvm/tools/llvm-objcopy/InstallNameToolOpts.td
@@ -18,5 +18,17 @@ def h : Flag<["-"], "h">, Alias<help>;
 def add_rpath : Option<["-", "--"], "add_rpath", KIND_SEPARATE>,
                 HelpText<"Add new rpath">;
 
+def delete_rpath: Option<["-", "--"], "delete_rpath", KIND_SEPARATE>,
+                  HelpText<"Delete specified rpath">;
+
+def rpath: MultiArg<["-", "--"], "rpath", 2>,
+           HelpText<"Change rpath path name">;
+
+def id : Option<["-","--"], "id", KIND_SEPARATE>,
+                 HelpText<"Change dynamic shared library id">;
+
+def change: MultiArg<["-", "--"], "change", 2>,
+            HelpText<"Change dependent shared library install name">;
+
 def version : Flag<["--"], "version">,
               HelpText<"Print the version and exit.">;
diff --git a/llvm/tools/llvm-objcopy/MachO/MachOObjcopy.cpp b/llvm/tools/llvm-objcopy/MachO/MachOObjcopy.cpp
index 17d84549544bd..5ca5b133572b4 100644
--- a/llvm/tools/llvm-objcopy/MachO/MachOObjcopy.cpp
+++ b/llvm/tools/llvm-objcopy/MachO/MachOObjcopy.cpp
@@ -10,6 +10,7 @@
 #include "../CopyConfig.h"
 #include "MachOReader.h"
 #include "MachOWriter.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
 
@@ -19,6 +20,27 @@ namespace macho {
 
 using namespace object;
 using SectionPred = std::function<bool(const std::unique_ptr<Section> &Sec)>;
+using LoadCommandPred = std::function<bool(const LoadCommand &LC)>;
+
+#ifndef NDEBUG
+static bool isLoadCommandWithPayloadString(const LoadCommand &LC) {
+  // TODO: Add support for LC_REEXPORT_DYLIB, LC_LOAD_UPWARD_DYLIB and
+  // LC_LAZY_LOAD_DYLIB
+  return LC.MachOLoadCommand.load_command_data.cmd == MachO::LC_RPATH ||
+         LC.MachOLoadCommand.load_command_data.cmd == MachO::LC_ID_DYLIB ||
+         LC.MachOLoadCommand.load_command_data.cmd == MachO::LC_LOAD_DYLIB ||
+         LC.MachOLoadCommand.load_command_data.cmd == MachO::LC_LOAD_WEAK_DYLIB;
+}
+#endif
+
+static StringRef getPayloadString(const LoadCommand &LC) {
+  assert(isLoadCommandWithPayloadString(LC) &&
+         "unsupported load command encountered");
+
+  return StringRef(reinterpret_cast<const char *>(LC.Payload.data()),
+                   LC.Payload.size())
+      .rtrim('\0');
+}
 
 static Error removeSections(const CopyConfig &Config, Object &Obj) {
   SectionPred RemovePred = [](const std::unique_ptr<Section> &) {
@@ -82,6 +104,18 @@ static void updateAndRemoveSymbols(const CopyConfig &Config, Object &Obj) {
   Obj.SymTable.removeSymbols(RemovePred);
 }
 
+template <typename LCType>
+static void updateLoadCommandPayloadString(LoadCommand &LC, StringRef S) {
+  assert(isLoadCommandWithPayloadString(LC) &&
+         "unsupported load command encountered");
+
+  uint32_t NewCmdsize = alignTo(sizeof(LCType) + S.size() + 1, 8);
+
+  LC.MachOLoadCommand.load_command_data.cmdsize = NewCmdsize;
+  LC.Payload.assign(NewCmdsize - sizeof(LCType), 0);
+  std::copy(S.begin(), S.end(), LC.Payload.begin());
+}
+
 static LoadCommand buildRPathLoadCommand(StringRef Path) {
   LoadCommand LC;
   MachO::rpath_command RPathLC;
@@ -94,6 +128,103 @@ static LoadCommand buildRPathLoadCommand(StringRef Path) {
   return LC;
 }
 
+static Error processLoadCommands(const CopyConfig &Config, Object &Obj) {
+  // Remove RPaths.
+  DenseSet<StringRef> RPathsToRemove(Config.RPathsToRemove.begin(),
+                                     Config.RPathsToRemove.end());
+
+  LoadCommandPred RemovePred = [&RPathsToRemove](const LoadCommand &LC) {
+    if (LC.MachOLoadCommand.load_command_data.cmd == MachO::LC_RPATH) {
+      StringRef RPath = getPayloadString(LC);
+      if (RPathsToRemove.count(RPath)) {
+        RPathsToRemove.erase(RPath);
+        return true;
+      }
+    }
+    return false;
+  };
+
+  if (Error E = Obj.removeLoadCommands(RemovePred))
+    return E;
+
+  // Emit an error if the Mach-O binary does not contain an rpath path name
+  // specified in -delete_rpath.
+  for (StringRef RPath : Config.RPathsToRemove) {
+    if (RPathsToRemove.count(RPath))
+      return createStringError(errc::invalid_argument,
+                               "no LC_RPATH load command with path: %s",
+                               RPath.str().c_str());
+  }
+
+  DenseSet<StringRef> RPaths;
+
+  // Get all existing RPaths.
+  for (LoadCommand &LC : Obj.LoadCommands) {
+    if (LC.MachOLoadCommand.load_command_data.cmd == MachO::LC_RPATH)
+      RPaths.insert(getPayloadString(LC));
+  }
+
+  // Throw errors for invalid RPaths.
+  for (const auto &OldNew : Config.RPathsToUpdate) {
+    StringRef Old = OldNew.getFirst();
+    StringRef New = OldNew.getSecond();
+    if (RPaths.count(Old) == 0)
+      return createStringError(errc::invalid_argument,
+                               "no LC_RPATH load command with path: " + Old);
+    if (RPaths.count(New) != 0)
+      return createStringError(errc::invalid_argument,
+                               "rpath " + New +
+                                   " would create a duplicate load command");
+  }
+
+  // Update load commands.
+  for (LoadCommand &LC : Obj.LoadCommands) {
+    switch (LC.MachOLoadCommand.load_command_data.cmd) {
+    case MachO::LC_ID_DYLIB:
+      if (Config.SharedLibId) {
+        StringRef Id = Config.SharedLibId.getValue();
+        if (Id.empty())
+          return createStringError(errc::invalid_argument,
+                                   "cannot specify an empty id");
+        updateLoadCommandPayloadString<MachO::dylib_command>(LC, Id);
+      }
+      break;
+
+    case MachO::LC_RPATH: {
+      StringRef RPath = getPayloadString(LC);
+      StringRef NewRPath = Config.RPathsToUpdate.lookup(RPath);
+      if (!NewRPath.empty())
+        updateLoadCommandPayloadString<MachO::rpath_command>(LC, NewRPath);
+      break;
+    }
+
+    // TODO: Add LC_REEXPORT_DYLIB, LC_LAZY_LOAD_DYLIB, and LC_LOAD_UPWARD_DYLIB
+    // here once llvm-objcopy supports them.
+    case MachO::LC_LOAD_DYLIB:
+    case MachO::LC_LOAD_WEAK_DYLIB:
+      StringRef InstallName = getPayloadString(LC);
+      StringRef NewInstallName =
+          Config.InstallNamesToUpdate.lookup(InstallName);
+      if (!NewInstallName.empty())
+        updateLoadCommandPayloadString<MachO::dylib_command>(LC,
+                                                             NewInstallName);
+      break;
+    }
+  }
+
+  // Add new RPaths.
+  for (StringRef RPath : Config.RPathToAdd) {
+    if (RPaths.count(RPath) != 0)
+      return createStringError(errc::invalid_argument,
+                               "rpath " + RPath +
+                                   " would create a duplicate load command");
+    RPaths.insert(RPath);
+    Obj.addLoadCommand(buildRPathLoadCommand(RPath));
+  }
+
+  return Error::success();
+}
+
 static Error dumpSectionToFile(StringRef SecName, StringRef Filename,
                                Object &Obj) {
   for (LoadCommand &LC : Obj.LoadCommands)
@@ -220,19 +351,9 @@ static Error handleArgs(const CopyConfig &Config, Object &Obj) {
       return E;
   }
 
-  for (StringRef RPath : Config.RPathToAdd) {
-    for (LoadCommand &LC : Obj.LoadCommands) {
-      if (LC.MachOLoadCommand.load_command_data.cmd == MachO::LC_RPATH &&
-          RPath == StringRef(reinterpret_cast<char *>(LC.Payload.data()),
-                             LC.Payload.size())
-                       .trim(0)) {
-        return createStringError(errc::invalid_argument,
-                                 "rpath " + RPath +
-                                     " would create a duplicate load command");
-      }
-    }
-    Obj.addLoadCommand(buildRPathLoadCommand(RPath));
-  }
+  if (Error E = processLoadCommands(Config, Obj))
+    return E;
+
   return Error::success();
 }
 
@@ -249,9 +370,18 @@ Error executeObjcopyOnBinary(const CopyConfig &Config,
   if (Error E = handleArgs(Config, *O))
     return createFileError(Config.InputFilename, std::move(E));
 
-  // TODO: Support 16KB pages which are employed in iOS arm64 binaries:
-  //       https://github.com/llvm/llvm-project/commit/1bebb2832ee312d3b0316dacff457a7a29435edb
-  const uint64_t PageSize = 4096;
+  // Page size used for alignment of segment sizes in Mach-O executables and
+  // dynamic libraries.
+  uint64_t PageSize;
+  switch (In.getArch()) {
+  case Triple::ArchType::arm:
+  case Triple::ArchType::aarch64:
+  case Triple::ArchType::aarch64_32:
+    PageSize = 16384;
+    break;
+  default:
+    PageSize = 4096;
+  }
 
   MachOWriter Writer(*O, In.is64Bit(), In.isLittleEndian(), PageSize, Out);
   if (auto E = Writer.finalize())
diff --git a/llvm/tools/llvm-objcopy/MachO/Object.cpp b/llvm/tools/llvm-objcopy/MachO/Object.cpp
index 0f28cb054a960..de8cb0af108d7 100644
--- a/llvm/tools/llvm-objcopy/MachO/Object.cpp
+++ b/llvm/tools/llvm-objcopy/MachO/Object.cpp
@@ -32,6 +32,42 @@ void SymbolTable::removeSymbols(
       std::end(Symbols));
 }
 
+void Object::updateLoadCommandIndexes() {
+  // Update indices of special load commands
+  for (size_t Index = 0, Size = LoadCommands.size(); Index < Size; ++Index) {
+    LoadCommand &LC = LoadCommands[Index];
+    switch (LC.MachOLoadCommand.load_command_data.cmd) {
+    case MachO::LC_SYMTAB:
+      SymTabCommandIndex = Index;
+      break;
+    case MachO::LC_DYSYMTAB:
+      DySymTabCommandIndex = Index;
+      break;
+    case MachO::LC_DYLD_INFO:
+    case MachO::LC_DYLD_INFO_ONLY:
+      DyLdInfoCommandIndex = Index;
+      break;
+    case MachO::LC_DATA_IN_CODE:
+      DataInCodeCommandIndex = Index;
+      break;
+    case MachO::LC_FUNCTION_STARTS:
+      FunctionStartsCommandIndex = Index;
+      break;
+    }
+  }
+}
+
+Error Object::removeLoadCommands(
+    function_ref<bool(const LoadCommand &)> ToRemove) {
+  auto It = std::stable_partition(
+      LoadCommands.begin(), LoadCommands.end(),
+      [&](const LoadCommand &LC) { return !ToRemove(LC); });
+  LoadCommands.erase(It, LoadCommands.end());
+
+  updateLoadCommandIndexes();
+  return Error::success();
+}
+
 Error Object::removeSections(
     function_ref<bool(const std::unique_ptr<Section> &)> ToRemove) {
   DenseMap<uint32_t, const Section *> OldIndexToSection;
diff --git a/llvm/tools/llvm-objcopy/MachO/Object.h b/llvm/tools/llvm-objcopy/MachO/Object.h
index 5da31d27b6f8e..e825d1867b09a 100644
--- a/llvm/tools/llvm-objcopy/MachO/Object.h
+++ b/llvm/tools/llvm-objcopy/MachO/Object.h
@@ -326,6 +326,11 @@ struct Object {
 
   Error
   removeSections(function_ref<bool(const std::unique_ptr<Section> &)> ToRemove);
+
+  Error removeLoadCommands(function_ref<bool(const LoadCommand &)> ToRemove);
+
+  void updateLoadCommandIndexes();
+
   void addLoadCommand(LoadCommand LC);
 
   /// Creates a new segment load command in the object and returns a reference
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp
index 81b3aac5c9314..320bbb5d358b9 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.cpp
+++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp
@@ -21,7 +21,9 @@
 #include "MachODump.h"
 #include "WasmDump.h"
 #include "XCOFFDump.h"
+#include "llvm/ADT/IndexedMap.h"
 #include "llvm/ADT/Optional.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/StringExtras.h"
@@ -79,6 +81,8 @@ using namespace llvm;
 using namespace llvm::object;
 using namespace llvm::objdump;
 
+#define DEBUG_TYPE "objdump"
+
 static cl::OptionCategory ObjdumpCat("llvm-objdump Options");
 
 static cl::opt<uint64_t> AdjustVMA(
@@ -344,6 +348,28 @@ static cl::opt<bool>
          cl::cat(ObjdumpCat));
 static cl::alias WideShort("w", cl::Grouping, cl::aliasopt(Wide));
 
+enum DebugVarsFormat {
+  DVDisabled,
+  DVUnicode,
+  DVASCII,
+};
+
+static cl::opt<DebugVarsFormat> DbgVariables(
+    "debug-vars", cl::init(DVDisabled),
+    cl::desc("Print the locations (in registers or memory) of "
+             "source-level variables alongside disassembly"),
+    cl::ValueOptional,
+    cl::values(clEnumValN(DVUnicode, "", "unicode"),
+               clEnumValN(DVUnicode, "unicode", "unicode"),
+               clEnumValN(DVASCII, "ascii", "unicode")),
+    cl::cat(ObjdumpCat));
+
+static cl::opt<int>
+    DbgIndent("debug-vars-indent", cl::init(40),
+              cl::desc("Distance to indent the source-level variable display, "
+                       "relative to the start of the disassembly"),
+              cl::cat(ObjdumpCat));
+
 static cl::extrahelp
     HelpResponse("\nPass @FILE as argument to read options from FILE.\n");
 
@@ -548,6 +574,358 @@ static bool getHidden(RelocationRef RelRef) {
 }
 
 namespace {
+
+/// Get the column at which we want to start printing the instruction
+/// disassembly, taking into account anything which appears to the left of it.
+unsigned getInstStartColumn(const MCSubtargetInfo &STI) {
+  return NoShowRawInsn ? 16 : STI.getTargetTriple().isX86() ? 40 : 24;
+}
+
+/// Stores a single expression representing the location of a source-level
+/// variable, along with the PC range for which that expression is valid.
+struct LiveVariable {
+  DWARFLocationExpression LocExpr;
+  const char *VarName;
+  DWARFUnit *Unit;
+  const DWARFDie FuncDie;
+
+  LiveVariable(const DWARFLocationExpression &LocExpr, const char *VarName,
+               DWARFUnit *Unit, const DWARFDie FuncDie)
+      : LocExpr(LocExpr), VarName(VarName), Unit(Unit), FuncDie(FuncDie) {}
+
+  bool liveAtAddress(object::SectionedAddress Addr) {
+    if (LocExpr.Range == None)
+      return false;
+    return LocExpr.Range->SectionIndex == Addr.SectionIndex &&
+           LocExpr.Range->LowPC <= Addr.Address &&
+           LocExpr.Range->HighPC > Addr.Address;
+  }
+
+  void print(raw_ostream &OS, const MCRegisterInfo &MRI) const {
+    DataExtractor Data({LocExpr.Expr.data(), LocExpr.Expr.size()},
+                       Unit->getContext().isLittleEndian(), 0);
+    DWARFExpression Expression(Data, Unit->getAddressByteSize());
+    Expression.printCompact(OS, MRI);
+  }
+};
+
+/// Helper class for printing source variable locations alongside disassembly.
+class LiveVariablePrinter {
+  // Information we want to track about one column in which we are printing a
+  // variable live range.
+  struct Column {
+    unsigned VarIdx = NullVarIdx;
+    bool LiveIn = false;
+    bool LiveOut = false;
+    bool MustDrawLabel  = false;
+
+    bool isActive() const { return VarIdx != NullVarIdx; }
+
+    static constexpr unsigned NullVarIdx = std::numeric_limits<unsigned>::max();
+  };
+
+  // All live variables we know about in the object/image file.
+  std::vector<LiveVariable> LiveVariables;
+
+  // The columns we are currently drawing.
+  IndexedMap<Column> ActiveCols;
+
+  const MCRegisterInfo &MRI;
+  const MCSubtargetInfo &STI;
+
+  void addVariable(DWARFDie FuncDie, DWARFDie VarDie) {
+    uint64_t FuncLowPC, FuncHighPC, SectionIndex;
+    FuncDie.getLowAndHighPC(FuncLowPC, FuncHighPC, SectionIndex);
+    const char *VarName = VarDie.getName(DINameKind::ShortName);
+    DWARFUnit *U = VarDie.getDwarfUnit();
+
+    Expected<DWARFLocationExpressionsVector> Locs =
+        VarDie.getLocations(dwarf::DW_AT_location);
+    if (!Locs) {
+      // If the variable doesn't have any locations, just ignore it. We don't
+      // report an error or warning here as that could be noisy on optimised
+      // code.
+      consumeError(Locs.takeError());
+      return;
+    }
+
+    for (const DWARFLocationExpression &LocExpr : *Locs) {
+      if (LocExpr.Range) {
+        LiveVariables.emplace_back(LocExpr, VarName, U, FuncDie);
+      } else {
+        // If the LocExpr does not have an associated range, it is valid for
+        // the whole of the function.
+        // TODO: technically it is not valid for any range covered by another
+        // LocExpr, does that happen in reality?
+        DWARFLocationExpression WholeFuncExpr{
+            DWARFAddressRange(FuncLowPC, FuncHighPC, SectionIndex),
+            LocExpr.Expr};
+        LiveVariables.emplace_back(WholeFuncExpr, VarName, U, FuncDie);
+      }
+    }
+  }
+
+  void addFunction(DWARFDie D) {
+    for (const DWARFDie &Child : D.children()) {
+      if (Child.getTag() == dwarf::DW_TAG_variable ||
+          Child.getTag() == dwarf::DW_TAG_formal_parameter)
+        addVariable(D, Child);
+      else
+        addFunction(Child);
+    }
+  }
+
+  // Get the column number (in characters) at which the first live variable
+  // line should be printed.
+  unsigned getIndentLevel() const {
+    return DbgIndent + getInstStartColumn(STI);
+  }
+
+  // Indent to the first live-range column to the right of the currently
+  // printed line, and return the index of that column.
+  // TODO: formatted_raw_ostream uses "column" to mean a number of characters
+  // since the last \n, and we use it to mean the number of slots in which we
+  // put live variable lines. Pick a less overloaded word.
+  unsigned moveToFirstVarColumn(formatted_raw_ostream &OS) {
+    // Logical column number: column zero is the first column we print in, each
+    // logical column is 2 physical columns wide.
+    unsigned FirstUnprintedLogicalColumn =
+        std::max((int)(OS.getColumn() - getIndentLevel() + 1) / 2, 0);
+    // Physical column number: the actual column number in characters, with
+    // zero being the left-most side of the screen.
+    unsigned FirstUnprintedPhysicalColumn =
+        getIndentLevel() + FirstUnprintedLogicalColumn * 2;
+
+    if (FirstUnprintedPhysicalColumn > OS.getColumn())
+      OS.PadToColumn(FirstUnprintedPhysicalColumn);
+
+    return FirstUnprintedLogicalColumn;
+  }
+
+  unsigned findFreeColumn() {
+    for (unsigned ColIdx = 0; ColIdx < ActiveCols.size(); ++ColIdx)
+      if (!ActiveCols[ColIdx].isActive())
+        return ColIdx;
+
+    size_t OldSize = ActiveCols.size();
+    ActiveCols.grow(std::max<size_t>(OldSize * 2, 1));
+    return OldSize;
+  }
+
+public:
+  LiveVariablePrinter(const MCRegisterInfo &MRI, const MCSubtargetInfo &STI)
+      : LiveVariables(), ActiveCols(Column()), MRI(MRI), STI(STI) {}
+
+  void dump() const {
+    for (const LiveVariable &LV : LiveVariables) {
+      dbgs() << LV.VarName << " @ " << LV.LocExpr.Range << ": ";
+      LV.print(dbgs(), MRI);
+      dbgs() << "\n";
+    }
+  }
+
+  void addCompileUnit(DWARFDie D) {
+    if (D.getTag() == dwarf::DW_TAG_subprogram)
+      addFunction(D);
+    else
+      for (const DWARFDie &Child : D.children())
+        addFunction(Child);
+  }
+
+  /// Update to match the state of the instruction between ThisAddr and
+  /// NextAddr. In the common case, any live range active at ThisAddr is
+  /// live-in to the instruction, and any live range active at NextAddr is
+  /// live-out of the instruction. If IncludeDefinedVars is false, then live
+  /// ranges starting at NextAddr will be ignored.
+  void update(object::SectionedAddress ThisAddr,
+              object::SectionedAddress NextAddr, bool IncludeDefinedVars) {
+    // First, check variables which have already been assigned a column, so
+    // that we don't change their order.
+    SmallSet<unsigned, 8> CheckedVarIdxs;
+    for (unsigned ColIdx = 0, End = ActiveCols.size(); ColIdx < End; ++ColIdx) {
+      if (!ActiveCols[ColIdx].isActive())
+        continue;
+      CheckedVarIdxs.insert(ActiveCols[ColIdx].VarIdx);
+      LiveVariable &LV = LiveVariables[ActiveCols[ColIdx].VarIdx];
+      ActiveCols[ColIdx].LiveIn = LV.liveAtAddress(ThisAddr);
+      ActiveCols[ColIdx].LiveOut = LV.liveAtAddress(NextAddr);
+      LLVM_DEBUG(dbgs() << "pass 1, " << ThisAddr.Address << "-"
+                        << NextAddr.Address << ", " << LV.VarName << ", Col "
+                        << ColIdx << ": LiveIn=" << ActiveCols[ColIdx].LiveIn
+                        << ", LiveOut=" << ActiveCols[ColIdx].LiveOut << "\n");
+
+      if (!ActiveCols[ColIdx].LiveIn && !ActiveCols[ColIdx].LiveOut)
+        ActiveCols[ColIdx].VarIdx = Column::NullVarIdx;
+    }
+
+    // Next, look for variables which don't already have a column, but which
+    // are now live.
+    if (IncludeDefinedVars) {
+      for (unsigned VarIdx = 0, End = LiveVariables.size(); VarIdx < End;
+           ++VarIdx) {
+        if (CheckedVarIdxs.count(VarIdx))
+          continue;
+        LiveVariable &LV = LiveVariables[VarIdx];
+        bool LiveIn = LV.liveAtAddress(ThisAddr);
+        bool LiveOut = LV.liveAtAddress(NextAddr);
+        if (!LiveIn && !LiveOut)
+          continue;
+
+        unsigned ColIdx = findFreeColumn();
+        LLVM_DEBUG(dbgs() << "pass 2, " << ThisAddr.Address << "-"
+                          << NextAddr.Address << ", " << LV.VarName << ", Col "
+                          << ColIdx << ": LiveIn=" << LiveIn
+                          << ", LiveOut=" << LiveOut << "\n");
+        ActiveCols[ColIdx].VarIdx = VarIdx;
+        ActiveCols[ColIdx].LiveIn = LiveIn;
+        ActiveCols[ColIdx].LiveOut = LiveOut;
+        ActiveCols[ColIdx].MustDrawLabel = true;
+      }
+    }
+  }
+
+  enum class LineChar {
+    RangeStart,
+    RangeMid,
+    RangeEnd,
+    LabelVert,
+    LabelCornerNew,
+    LabelCornerActive,
+    LabelHoriz,
+  };
+  const char *getLineChar(LineChar C) const {
+    bool IsASCII = DbgVariables == DVASCII;
+    switch (C) {
+    case LineChar::RangeStart:
+      return IsASCII ? "^" : u8"\u2548";
+    case LineChar::RangeMid:
+      return IsASCII ? "|" : u8"\u2503";
+    case LineChar::RangeEnd:
+      return IsASCII ? "v" : u8"\u253b";
+    case LineChar::LabelVert:
+      return IsASCII ? "|" : u8"\u2502";
+    case LineChar::LabelCornerNew:
+      return IsASCII ? "/" : u8"\u250c";
+    case LineChar::LabelCornerActive:
+      return IsASCII ? "|" : u8"\u2520";
+    case LineChar::LabelHoriz:
+      return IsASCII ? "-" : u8"\u2500";
+    }
+    llvm_unreachable("Unhandled LineChar enum");
+  }
+
+  /// Print live ranges to the right of an existing line. This assumes the
+  /// line is not an instruction, so doesn't start or end any live ranges, so
+  /// we only need to print active ranges or empty columns. If AfterInst is
+  /// true, this is being printed after the last instruction fed to update(),
+  /// otherwise this is being printed before it.
+  void printAfterOtherLine(formatted_raw_ostream &OS, bool AfterInst) {
+    if (ActiveCols.size()) {
+      unsigned FirstUnprintedColumn = moveToFirstVarColumn(OS);
+      for (size_t ColIdx = FirstUnprintedColumn, End = ActiveCols.size();
+           ColIdx < End; ++ColIdx) {
+        if (ActiveCols[ColIdx].isActive()) {
+          if ((AfterInst && ActiveCols[ColIdx].LiveOut) ||
+              (!AfterInst && ActiveCols[ColIdx].LiveIn))
+            OS << getLineChar(LineChar::RangeMid);
+          else if (!AfterInst && ActiveCols[ColIdx].LiveOut)
+            OS << getLineChar(LineChar::LabelVert);
+          else
+            OS << " ";
+        }
+        OS << " ";
+      }
+    }
+    OS << "\n";
+  }
+
+  /// Print any live variable range info needed to the right of a
+  /// non-instruction line of disassembly. This is where we print the variable
+  /// names and expressions, with thin line-drawing characters connecting them
+  /// to the live range which starts at the next instruction. If MustPrint is
+  /// true, we have to print at least one line (with the continuation of any
+  /// already-active live ranges) because something has already been printed
+  /// earlier on this line.
+  void printBetweenInsts(formatted_raw_ostream &OS, bool MustPrint) {
+    bool PrintedSomething = false;
+    for (unsigned ColIdx = 0, End = ActiveCols.size(); ColIdx < End; ++ColIdx) {
+      if (ActiveCols[ColIdx].isActive() && ActiveCols[ColIdx].MustDrawLabel) {
+        // First we need to print the live range markers for any active
+        // columns to the left of this one.
+        OS.PadToColumn(getIndentLevel());
+        for (unsigned ColIdx2 = 0; ColIdx2 < ColIdx; ++ColIdx2) {
+          if (ActiveCols[ColIdx2].isActive()) {
+            if (ActiveCols[ColIdx2].MustDrawLabel &&
+                           !ActiveCols[ColIdx2].LiveIn)
+              OS << getLineChar(LineChar::LabelVert) << " ";
+            else
+              OS << getLineChar(LineChar::RangeMid) << " ";
+          } else
+            OS << "  ";
+        }
+
+        // Then print the variable name and location of the new live range,
+        // with box drawing characters joining it to the live range line.
+        OS << getLineChar(ActiveCols[ColIdx].LiveIn
+                              ? LineChar::LabelCornerActive
+                              : LineChar::LabelCornerNew)
+           << getLineChar(LineChar::LabelHoriz) << " ";
+        WithColor(OS, raw_ostream::GREEN)
+            << LiveVariables[ActiveCols[ColIdx].VarIdx].VarName;
+        OS << " = ";
+        {
+          WithColor ExprColor(OS, raw_ostream::CYAN);
+          LiveVariables[ActiveCols[ColIdx].VarIdx].print(OS, MRI);
+        }
+
+        // If there are any columns to the right of the expression we just
+        // printed, then continue their live range lines.
+        unsigned FirstUnprintedColumn = moveToFirstVarColumn(OS);
+        for (unsigned ColIdx2 = FirstUnprintedColumn, End = ActiveCols.size();
+             ColIdx2 < End; ++ColIdx2) {
+          if (ActiveCols[ColIdx2].isActive() && ActiveCols[ColIdx2].LiveIn)
+            OS << getLineChar(LineChar::RangeMid) << " ";
+          else
+            OS << "  ";
+        }
+
+        OS << "\n";
+        PrintedSomething = true;
+      }
+    }
+
+    for (unsigned ColIdx = 0, End = ActiveCols.size(); ColIdx < End; ++ColIdx)
+      if (ActiveCols[ColIdx].isActive())
+        ActiveCols[ColIdx].MustDrawLabel = false;
+
+    // If we must print something (because we printed a line/column number),
+    // but don't have any new variables to print, then print a line which
+    // just continues any existing live ranges.
+    if (MustPrint && !PrintedSomething)
+      printAfterOtherLine(OS, false);
+  }
+
+  /// Print the live variable ranges to the right of a disassembled instruction.
+  void printAfterInst(formatted_raw_ostream &OS) {
+    if (!ActiveCols.size())
+      return;
+    unsigned FirstUnprintedColumn = moveToFirstVarColumn(OS);
+    for (unsigned ColIdx = FirstUnprintedColumn, End = ActiveCols.size();
+         ColIdx < End; ++ColIdx) {
+      if (!ActiveCols[ColIdx].isActive())
+        OS << "  ";
+      else if (ActiveCols[ColIdx].LiveIn && ActiveCols[ColIdx].LiveOut)
+        OS << getLineChar(LineChar::RangeMid) << " ";
+      else if (ActiveCols[ColIdx].LiveOut)
+        OS << getLineChar(LineChar::RangeStart) << " ";
+      else if (ActiveCols[ColIdx].LiveIn)
+        OS << getLineChar(LineChar::RangeEnd) << " ";
+      else
+        llvm_unreachable("var must be live in or out!");
+    }
+  }
+};
+
 class SourcePrinter {
 protected:
   DILineInfo OldLineInfo;
@@ -565,11 +943,12 @@ class SourcePrinter {
 private:
   bool cacheSource(const DILineInfo& LineInfoFile);
 
-  void printLines(raw_ostream &OS, const DILineInfo &LineInfo,
-                  StringRef Delimiter);
+  void printLines(formatted_raw_ostream &OS, const DILineInfo &LineInfo,
+                  StringRef Delimiter, LiveVariablePrinter &LVP);
 
-  void printSources(raw_ostream &OS, const DILineInfo &LineInfo,
-                    StringRef ObjectFilename, StringRef Delimiter);
+  void printSources(formatted_raw_ostream &OS, const DILineInfo &LineInfo,
+                    StringRef ObjectFilename, StringRef Delimiter,
+                    LiveVariablePrinter &LVP);
 
 public:
   SourcePrinter() = default;
@@ -583,9 +962,10 @@ class SourcePrinter {
     Symbolizer.reset(new symbolize::LLVMSymbolizer(SymbolizerOpts));
   }
   virtual ~SourcePrinter() = default;
-  virtual void printSourceLine(raw_ostream &OS,
+  virtual void printSourceLine(formatted_raw_ostream &OS,
                                object::SectionedAddress Address,
                                StringRef ObjectFilename,
+                               LiveVariablePrinter &LVP,
                                StringRef Delimiter = "; ");
 };
 
@@ -619,9 +999,10 @@ bool SourcePrinter::cacheSource(const DILineInfo &LineInfo) {
   return true;
 }
 
-void SourcePrinter::printSourceLine(raw_ostream &OS,
+void SourcePrinter::printSourceLine(formatted_raw_ostream &OS,
                                     object::SectionedAddress Address,
                                     StringRef ObjectFilename,
+                                    LiveVariablePrinter &LVP,
                                     StringRef Delimiter) {
   if (!Symbolizer)
     return;
@@ -646,14 +1027,15 @@ void SourcePrinter::printSourceLine(raw_ostream &OS,
   }
 
   if (PrintLines)
-    printLines(OS, LineInfo, Delimiter);
+    printLines(OS, LineInfo, Delimiter, LVP);
   if (PrintSource)
-    printSources(OS, LineInfo, ObjectFilename, Delimiter);
+    printSources(OS, LineInfo, ObjectFilename, Delimiter, LVP);
   OldLineInfo = LineInfo;
 }
 
-void SourcePrinter::printLines(raw_ostream &OS, const DILineInfo &LineInfo,
-                               StringRef Delimiter) {
+void SourcePrinter::printLines(formatted_raw_ostream &OS,
+                               const DILineInfo &LineInfo, StringRef Delimiter,
+                               LiveVariablePrinter &LVP) {
   bool PrintFunctionName = LineInfo.FunctionName != DILineInfo::BadString &&
                            LineInfo.FunctionName != OldLineInfo.FunctionName;
   if (PrintFunctionName) {
@@ -666,13 +1048,16 @@ void SourcePrinter::printLines(raw_ostream &OS, const DILineInfo &LineInfo,
   }
   if (LineInfo.FileName != DILineInfo::BadString && LineInfo.Line != 0 &&
       (OldLineInfo.Line != LineInfo.Line ||
-       OldLineInfo.FileName != LineInfo.FileName || PrintFunctionName))
-    OS << Delimiter << LineInfo.FileName << ":" << LineInfo.Line << "\n";
+       OldLineInfo.FileName != LineInfo.FileName || PrintFunctionName)) {
+    OS << Delimiter << LineInfo.FileName << ":" << LineInfo.Line;
+    LVP.printBetweenInsts(OS, true);
+  }
 }
 
-void SourcePrinter::printSources(raw_ostream &OS, const DILineInfo &LineInfo,
-                                 StringRef ObjectFilename,
-                                 StringRef Delimiter) {
+void SourcePrinter::printSources(formatted_raw_ostream &OS,
+                                 const DILineInfo &LineInfo,
+                                 StringRef ObjectFilename, StringRef Delimiter,
+                                 LiveVariablePrinter &LVP) {
   if (LineInfo.FileName == DILineInfo::BadString || LineInfo.Line == 0 ||
       (OldLineInfo.Line == LineInfo.Line &&
        OldLineInfo.FileName == LineInfo.FileName))
@@ -692,7 +1077,8 @@ void SourcePrinter::printSources(raw_ostream &OS, const DILineInfo &LineInfo,
       return;
     }
     // Vector begins at 0, line numbers are non-zero
-    OS << Delimiter << LineBuffer->second[LineInfo.Line - 1] << '\n';
+    OS << Delimiter << LineBuffer->second[LineInfo.Line - 1];
+    LVP.printBetweenInsts(OS, true);
   }
 }
 
@@ -710,28 +1096,30 @@ static bool hasMappingSymbols(const ObjectFile *Obj) {
   return isArmElf(Obj) || isAArch64Elf(Obj);
 }
 
-static void printRelocation(StringRef FileName, const RelocationRef &Rel,
-                            uint64_t Address, bool Is64Bits) {
+static void printRelocation(formatted_raw_ostream &OS, StringRef FileName,
+                            const RelocationRef &Rel, uint64_t Address,
+                            bool Is64Bits) {
   StringRef Fmt = Is64Bits ? "\t\t%016" PRIx64 ":  " : "\t\t\t%08" PRIx64 ":  ";
   SmallString<16> Name;
   SmallString<32> Val;
   Rel.getTypeName(Name);
   if (Error E = getRelocationValueString(Rel, Val))
     reportError(std::move(E), FileName);
-  outs() << format(Fmt.data(), Address) << Name << "\t" << Val << "\n";
+  OS << format(Fmt.data(), Address) << Name << "\t" << Val;
 }
 
 class PrettyPrinter {
 public:
   virtual ~PrettyPrinter() = default;
-  virtual void printInst(MCInstPrinter &IP, const MCInst *MI,
-                         ArrayRef<uint8_t> Bytes,
-                         object::SectionedAddress Address, raw_ostream &OS,
-                         StringRef Annot, MCSubtargetInfo const &STI,
-                         SourcePrinter *SP, StringRef ObjectFilename,
-                         std::vector<RelocationRef> *Rels = nullptr) {
+  virtual void
+  printInst(MCInstPrinter &IP, const MCInst *MI, ArrayRef<uint8_t> Bytes,
+            object::SectionedAddress Address, formatted_raw_ostream &OS,
+            StringRef Annot, MCSubtargetInfo const &STI, SourcePrinter *SP,
+            StringRef ObjectFilename, std::vector<RelocationRef> *Rels,
+            LiveVariablePrinter &LVP) {
     if (SP && (PrintSource || PrintLines))
-      SP->printSourceLine(OS, Address, ObjectFilename);
+      SP->printSourceLine(OS, Address, ObjectFilename, LVP);
+    LVP.printBetweenInsts(OS, false);
 
     size_t Start = OS.tell();
     if (!NoLeadingAddr)
@@ -741,11 +1129,9 @@ class PrettyPrinter {
       dumpBytes(Bytes, OS);
     }
 
-    // The output of printInst starts with a tab. Print some spaces so that the
-    // tab has 1 column and advances to the target tab stop. Give more columns
-    // to x86 which may encode an instruction with many bytes.
-    unsigned TabStop =
-        NoShowRawInsn ? 16 : STI.getTargetTriple().isX86() ? 40 : 24;
+    // The output of printInst starts with a tab. Print some spaces so that
+    // the tab has 1 column and advances to the target tab stop.
+    unsigned TabStop = getInstStartColumn(STI);
     unsigned Column = OS.tell() - Start;
     OS.indent(Column < TabStop - 1 ? TabStop - 1 - Column : 7 - Column % 8);
 
@@ -766,7 +1152,7 @@ PrettyPrinter PrettyPrinterInst;
 class HexagonPrettyPrinter : public PrettyPrinter {
 public:
   void printLead(ArrayRef<uint8_t> Bytes, uint64_t Address,
-                 raw_ostream &OS) {
+                 formatted_raw_ostream &OS) {
     uint32_t opcode =
       (Bytes[3] << 24) | (Bytes[2] << 16) | (Bytes[1] << 8) | Bytes[0];
     if (!NoLeadingAddr)
@@ -778,12 +1164,12 @@ class HexagonPrettyPrinter : public PrettyPrinter {
     }
   }
   void printInst(MCInstPrinter &IP, const MCInst *MI, ArrayRef<uint8_t> Bytes,
-                 object::SectionedAddress Address, raw_ostream &OS,
+                 object::SectionedAddress Address, formatted_raw_ostream &OS,
                  StringRef Annot, MCSubtargetInfo const &STI, SourcePrinter *SP,
-                 StringRef ObjectFilename,
-                 std::vector<RelocationRef> *Rels) override {
+                 StringRef ObjectFilename, std::vector<RelocationRef> *Rels,
+                 LiveVariablePrinter &LVP) override {
     if (SP && (PrintSource || PrintLines))
-      SP->printSourceLine(OS, Address, ObjectFilename, "");
+      SP->printSourceLine(OS, Address, ObjectFilename, LVP, "");
     if (!MI) {
       printLead(Bytes, Address.Address, OS);
       OS << " <unknown>";
@@ -809,7 +1195,7 @@ class HexagonPrettyPrinter : public PrettyPrinter {
     auto PrintReloc = [&]() -> void {
       while ((RelCur != RelEnd) && (RelCur->getOffset() <= Address.Address)) {
         if (RelCur->getOffset() == Address.Address) {
-          printRelocation(ObjectFilename, *RelCur, Address.Address, false);
+          printRelocation(OS, ObjectFilename, *RelCur, Address.Address, false);
           return;
         }
         ++RelCur;
@@ -820,7 +1206,7 @@ class HexagonPrettyPrinter : public PrettyPrinter {
       OS << Separator;
       Separator = "\n";
       if (SP && (PrintSource || PrintLines))
-        SP->printSourceLine(OS, Address, ObjectFilename, "");
+        SP->printSourceLine(OS, Address, ObjectFilename, LVP, "");
       printLead(Bytes, Address.Address, OS);
       OS << Preamble;
       Preamble = "   ";
@@ -848,12 +1234,12 @@ HexagonPrettyPrinter HexagonPrettyPrinterInst;
 class AMDGCNPrettyPrinter : public PrettyPrinter {
 public:
   void printInst(MCInstPrinter &IP, const MCInst *MI, ArrayRef<uint8_t> Bytes,
-                 object::SectionedAddress Address, raw_ostream &OS,
+                 object::SectionedAddress Address, formatted_raw_ostream &OS,
                  StringRef Annot, MCSubtargetInfo const &STI, SourcePrinter *SP,
-                 StringRef ObjectFilename,
-                 std::vector<RelocationRef> *Rels) override {
+                 StringRef ObjectFilename, std::vector<RelocationRef> *Rels,
+                 LiveVariablePrinter &LVP) override {
     if (SP && (PrintSource || PrintLines))
-      SP->printSourceLine(OS, Address, ObjectFilename);
+      SP->printSourceLine(OS, Address, ObjectFilename, LVP);
 
     if (MI) {
       SmallString<40> InstStr;
@@ -900,12 +1286,12 @@ AMDGCNPrettyPrinter AMDGCNPrettyPrinterInst;
 class BPFPrettyPrinter : public PrettyPrinter {
 public:
   void printInst(MCInstPrinter &IP, const MCInst *MI, ArrayRef<uint8_t> Bytes,
-                 object::SectionedAddress Address, raw_ostream &OS,
+                 object::SectionedAddress Address, formatted_raw_ostream &OS,
                  StringRef Annot, MCSubtargetInfo const &STI, SourcePrinter *SP,
-                 StringRef ObjectFilename,
-                 std::vector<RelocationRef> *Rels) override {
+                 StringRef ObjectFilename, std::vector<RelocationRef> *Rels,
+                 LiveVariablePrinter &LVP) override {
     if (SP && (PrintSource || PrintLines))
-      SP->printSourceLine(OS, Address, ObjectFilename);
+      SP->printSourceLine(OS, Address, ObjectFilename, LVP);
     if (!NoLeadingAddr)
       OS << format("%8" PRId64 ":", Address.Address / 8);
     if (!NoShowRawInsn) {
@@ -1094,26 +1480,27 @@ static char getMappingSymbolKind(ArrayRef<MappingSymbolPair> MappingSymbols,
 static uint64_t dumpARMELFData(uint64_t SectionAddr, uint64_t Index,
                                uint64_t End, const ObjectFile *Obj,
                                ArrayRef<uint8_t> Bytes,
-                               ArrayRef<MappingSymbolPair> MappingSymbols) {
+                               ArrayRef<MappingSymbolPair> MappingSymbols,
+                               raw_ostream &OS) {
   support::endianness Endian =
       Obj->isLittleEndian() ? support::little : support::big;
-  outs() << format("%8" PRIx64 ":\t", SectionAddr + Index);
+  OS << format("%8" PRIx64 ":\t", SectionAddr + Index);
   if (Index + 4 <= End) {
-    dumpBytes(Bytes.slice(Index, 4), outs());
-    outs() << "\t.word\t"
+    dumpBytes(Bytes.slice(Index, 4), OS);
+    OS << "\t.word\t"
            << format_hex(support::endian::read32(Bytes.data() + Index, Endian),
                          10);
     return 4;
   }
   if (Index + 2 <= End) {
-    dumpBytes(Bytes.slice(Index, 2), outs());
-    outs() << "\t\t.short\t"
+    dumpBytes(Bytes.slice(Index, 2), OS);
+    OS << "\t\t.short\t"
            << format_hex(support::endian::read16(Bytes.data() + Index, Endian),
                          6);
     return 2;
   }
-  dumpBytes(Bytes.slice(Index, 1), outs());
-  outs() << "\t\t.byte\t" << format_hex(Bytes[0], 4);
+  dumpBytes(Bytes.slice(Index, 1), OS);
+  OS << "\t\t.byte\t" << format_hex(Bytes[0], 4);
   return 1;
 }
 
@@ -1288,6 +1675,17 @@ static void disassembleObject(const Target *TheTarget, const ObjectFile *Obj,
     stable_sort(SecSyms.second);
   stable_sort(AbsoluteSymbols);
 
+  std::unique_ptr<DWARFContext> DICtx;
+  LiveVariablePrinter LVP(*Ctx.getRegisterInfo(), *STI);
+
+  if (DbgVariables != DVDisabled) {
+    DICtx = DWARFContext::create(*Obj);
+    for (const std::unique_ptr<DWARFUnit> &CU : DICtx->compile_units())
+      LVP.addCompileUnit(CU->getUnitDIE(false));
+  }
+
+  LLVM_DEBUG(LVP.dump());
+
   for (const SectionRef &Section : ToolSectionFilter(*Obj)) {
     if (FilterSections.empty() && !DisassembleAll &&
         (!Section.isText() || Section.isVirtual()))
@@ -1481,6 +1879,7 @@ static void disassembleObject(const Target *TheTarget, const ObjectFile *Obj,
                              Symbols[SI].Type != ELF::STT_OBJECT &&
                              !DisassembleAll;
       bool DumpARMELFData = false;
+      formatted_raw_ostream FOS(outs());
       while (Index < End) {
         // ARM and AArch64 ELF binaries can interleave data and text in the
         // same section. We rely on the markers introduced to understand what
@@ -1502,7 +1901,7 @@ static void disassembleObject(const Target *TheTarget, const ObjectFile *Obj,
 
         if (DumpARMELFData) {
           Size = dumpARMELFData(SectionAddr, Index, End, Obj, Bytes,
-                                MappingSymbols);
+                                MappingSymbols, FOS);
         } else {
           // When -z or --disassemble-zeroes are given we always dissasemble
           // them. Otherwise we might want to skip zero bytes we see.
@@ -1515,7 +1914,7 @@ static void disassembleObject(const Target *TheTarget, const ObjectFile *Obj,
 
             if (size_t N =
                     countSkippableZeroBytes(Bytes.slice(Index, MaxOffset))) {
-              outs() << "\t\t..." << '\n';
+              FOS << "\t\t..." << '\n';
               Index += N;
               continue;
             }
@@ -1530,11 +1929,14 @@ static void disassembleObject(const Target *TheTarget, const ObjectFile *Obj,
           if (Size == 0)
             Size = 1;
 
+          LVP.update({Index, Section.getIndex()},
+                     {Index + Size, Section.getIndex()}, Index + Size != End);
+
           PIP.printInst(
               *IP, Disassembled ? &Inst : nullptr, Bytes.slice(Index, Size),
-              {SectionAddr + Index + VMAAdjustment, Section.getIndex()}, outs(),
-              "", *STI, &SP, Obj->getFileName(), &Rels);
-          outs() << CommentStream.str();
+              {SectionAddr + Index + VMAAdjustment, Section.getIndex()}, FOS,
+              "", *STI, &SP, Obj->getFileName(), &Rels, LVP);
+          FOS << CommentStream.str();
           Comments.clear();
 
           // If disassembly has failed, avoid analysing invalid/incomplete
@@ -1551,7 +1953,7 @@ static void disassembleObject(const Target *TheTarget, const ObjectFile *Obj,
                           Inst, SectionAddr + Index, Size)) {
                 Target = *MaybeTarget;
                 PrintTarget = true;
-                outs() << "  # " << Twine::utohexstr(Target);
+                FOS << "  # " << Twine::utohexstr(Target);
               }
             if (PrintTarget) {
               // In a relocatable object, the target's section must reside in
@@ -1607,16 +2009,18 @@ static void disassembleObject(const Target *TheTarget, const ObjectFile *Obj,
                 if (Demangle)
                   TargetName = demangle(TargetName);
 
-                outs() << " <" << TargetName;
+                FOS << " <" << TargetName;
                 uint64_t Disp = Target - TargetAddress;
                 if (Disp)
-                  outs() << "+0x" << Twine::utohexstr(Disp);
-                outs() << '>';
+                  FOS << "+0x" << Twine::utohexstr(Disp);
+                FOS << '>';
               }
             }
           }
         }
-        outs() << "\n";
+
+        LVP.printAfterInst(FOS);
+        FOS << "\n";
 
         // Hexagon does this in pretty printer
         if (Obj->getArch() != Triple::hexagon) {
@@ -1646,8 +2050,9 @@ static void disassembleObject(const Target *TheTarget, const ObjectFile *Obj,
                 Offset += AdjustVMA;
             }
 
-            printRelocation(Obj->getFileName(), *RelCur, SectionAddr + Offset,
-                            Is64Bits);
+            printRelocation(FOS, Obj->getFileName(), *RelCur,
+                            SectionAddr + Offset, Is64Bits);
+            LVP.printAfterOtherLine(FOS, true);
             ++RelCur;
           }
         }
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 3d0e820d2f59c..843f072a61c32 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -23,11 +23,12 @@
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/FormattedStream.h"
 #include "llvm/Support/InitLLVM.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
-#include "llvm/Support/Threading.h"
 #include "llvm/Support/ThreadPool.h"
+#include "llvm/Support/Threading.h"
 #include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
@@ -290,6 +291,22 @@ static void mergeWriterContexts(WriterContext *Dst, WriterContext *Src) {
   });
 }
 
+static void writeInstrProfile(StringRef OutputFilename,
+                              ProfileFormat OutputFormat,
+                              InstrProfWriter &Writer) {
+  std::error_code EC;
+  raw_fd_ostream Output(OutputFilename.data(), EC, sys::fs::OF_None);
+  if (EC)
+    exitWithErrorCode(EC, OutputFilename);
+
+  if (OutputFormat == PF_Text) {
+    if (Error E = Writer.writeText(Output))
+      exitWithError(std::move(E));
+  } else {
+    Writer.write(Output);
+  }
+}
+
 static void mergeInstrProfile(const WeightedFileVector &Inputs,
                               SymbolRemapper *Remapper,
                               StringRef OutputFilename,
@@ -365,18 +382,7 @@ static void mergeInstrProfile(const WeightedFileVector &Inputs,
       (NumErrors > 0 && FailMode == failIfAnyAreInvalid))
     exitWithError("No profiles could be merged.");
 
-  std::error_code EC;
-  raw_fd_ostream Output(OutputFilename.data(), EC, sys::fs::OF_None);
-  if (EC)
-    exitWithErrorCode(EC, OutputFilename);
-
-  InstrProfWriter &Writer = Contexts[0]->Writer;
-  if (OutputFormat == PF_Text) {
-    if (Error E = Writer.writeText(Output))
-      exitWithError(std::move(E));
-  } else {
-    Writer.write(Output);
-  }
+  writeInstrProfile(OutputFilename, OutputFormat, Contexts[0]->Writer);
 }
 
 /// Make a copy of the given function samples with all symbol names remapped
@@ -1026,11 +1032,144 @@ static void showSectionInfo(sampleprof::SampleProfileReader *Reader,
   }
 }
 
+namespace {
+struct HotFuncInfo {
+  StringRef FuncName;
+  uint64_t TotalCount;
+  double TotalCountPercent;
+  uint64_t MaxCount;
+  uint64_t EntryCount;
+
+  HotFuncInfo()
+      : FuncName(), TotalCount(0), TotalCountPercent(0.0f), MaxCount(0),
+        EntryCount(0) {}
+
+  HotFuncInfo(StringRef FN, uint64_t TS, double TSP, uint64_t MS, uint64_t ES)
+      : FuncName(FN), TotalCount(TS), TotalCountPercent(TSP), MaxCount(MS),
+        EntryCount(ES) {}
+};
+} // namespace
+
+// Print out detailed information about hot functions in PrintValues vector.
+// Users specify titles and offset of every columns through ColumnTitle and
+// ColumnOffset. The size of ColumnTitle and ColumnOffset need to be the same
+// and at least 4. Besides, users can optionally give a HotFuncMetric string to
+// print out or let it be an empty string.
+static void dumpHotFunctionList(const std::vector<std::string> &ColumnTitle,
+                                const std::vector<int> &ColumnOffset,
+                                const std::vector<HotFuncInfo> &PrintValues,
+                                uint64_t HotFuncCount, uint64_t TotalFuncCount,
+                                uint64_t HotProfCount, uint64_t TotalProfCount,
+                                const std::string &HotFuncMetric,
+                                raw_fd_ostream &OS) {
+  assert(ColumnOffset.size() == ColumnTitle.size());
+  assert(ColumnTitle.size() >= 4);
+  assert(TotalFuncCount > 0);
+  double TotalProfPercent = 0;
+  if (TotalProfCount > 0)
+    TotalProfPercent = ((double)HotProfCount) / TotalProfCount * 100;
+
+  formatted_raw_ostream FOS(OS);
+  FOS << HotFuncCount << " out of " << TotalFuncCount
+      << " functions with profile ("
+      << format("%.2f%%", (((double)HotFuncCount) / TotalFuncCount * 100))
+      << ") are considered hot functions";
+  if (!HotFuncMetric.empty())
+    FOS << " (" << HotFuncMetric << ")";
+  FOS << ".\n";
+  FOS << HotProfCount << " out of " << TotalProfCount << " profile counts ("
+      << format("%.2f%%", TotalProfPercent) << ") are from hot functions.\n";
+
+  for (size_t I = 0; I < ColumnTitle.size(); ++I) {
+    FOS.PadToColumn(ColumnOffset[I]);
+    FOS << ColumnTitle[I];
+  }
+  FOS << "\n";
+
+  for (const HotFuncInfo &R : PrintValues) {
+    FOS.PadToColumn(ColumnOffset[0]);
+    FOS << R.TotalCount << " (" << format("%.2f%%", R.TotalCountPercent) << ")";
+    FOS.PadToColumn(ColumnOffset[1]);
+    FOS << R.MaxCount;
+    FOS.PadToColumn(ColumnOffset[2]);
+    FOS << R.EntryCount;
+    FOS.PadToColumn(ColumnOffset[3]);
+    FOS << R.FuncName << "\n";
+  }
+  return;
+}
+
+static int
+showHotFunctionList(const StringMap<sampleprof::FunctionSamples> &Profiles,
+                    ProfileSummary &PS, raw_fd_ostream &OS) {
+  using namespace sampleprof;
+
+  const uint32_t HotFuncCutoff = 990000;
+  auto &SummaryVector = PS.getDetailedSummary();
+  uint64_t MinCountThreshold = 0;
+  for (const ProfileSummaryEntry &SummaryEntry : SummaryVector) {
+    if (SummaryEntry.Cutoff == HotFuncCutoff) {
+      MinCountThreshold = SummaryEntry.MinCount;
+      break;
+    }
+  }
+  assert(MinCountThreshold != 0);
+
+  // Traverse all functions in the profile and keep only hot functions.
+  // The following loop also calculates the sum of total samples of all
+  // functions.
+  std::multimap<uint64_t, std::pair<const FunctionSamples *, const uint64_t>,
+                std::greater<uint64_t>>
+      HotFunc;
+  uint64_t ProfileTotalSample = 0;
+  uint64_t HotFuncSample = 0;
+  uint64_t HotFuncCount = 0;
+  uint64_t MaxCount = 0;
+  for (const auto &I : Profiles) {
+    const FunctionSamples &FuncProf = I.second;
+    ProfileTotalSample += FuncProf.getTotalSamples();
+    MaxCount = FuncProf.getMaxCountInside();
+
+    // MinCountThreshold is a block/line threshold computed for a given cutoff.
+    // We intentionally compare the maximum sample count in a function with this
+    // threshold to get an approximate threshold for hot functions.
+    if (MaxCount >= MinCountThreshold) {
+      HotFunc.emplace(FuncProf.getTotalSamples(),
+                      std::make_pair(&(I.second), MaxCount));
+      HotFuncSample += FuncProf.getTotalSamples();
+      ++HotFuncCount;
+    }
+  }
+
+  std::vector<std::string> ColumnTitle{"Total sample (%)", "Max sample",
+                                       "Entry sample", "Function name"};
+  std::vector<int> ColumnOffset{0, 24, 42, 58};
+  std::string Metric =
+      std::string("max sample >= ") + std::to_string(MinCountThreshold);
+  std::vector<HotFuncInfo> PrintValues;
+  for (const auto &FuncPair : HotFunc) {
+    const FunctionSamples &Func = *FuncPair.second.first;
+    double TotalSamplePercent =
+        (ProfileTotalSample > 0)
+            ? (Func.getTotalSamples() * 100.0) / ProfileTotalSample
+            : 0;
+    PrintValues.emplace_back(HotFuncInfo(
+        Func.getFuncName(), Func.getTotalSamples(), TotalSamplePercent,
+        FuncPair.second.second, Func.getEntrySamples()));
+  }
+  dumpHotFunctionList(ColumnTitle, ColumnOffset, PrintValues, HotFuncCount,
+                      Profiles.size(), HotFuncSample, ProfileTotalSample,
+                      Metric, OS);
+
+  return 0;
+}
+
 static int showSampleProfile(const std::string &Filename, bool ShowCounts,
                              bool ShowAllFunctions, bool ShowDetailedSummary,
                              const std::string &ShowFunction,
                              bool ShowProfileSymbolList,
-                             bool ShowSectionInfoOnly, raw_fd_ostream &OS) {
+                             bool ShowSectionInfoOnly, bool ShowHotFuncList,
+                             raw_fd_ostream &OS) {
   using namespace sampleprof;
   LLVMContext Context;
   auto ReaderOrErr = SampleProfileReader::create(Filename, Context);
@@ -1064,6 +1203,9 @@ static int showSampleProfile(const std::string &Filename, bool ShowCounts,
     PS.printDetailedSummary(OS);
   }
 
+  if (ShowHotFuncList)
+    showHotFunctionList(Reader->getProfiles(), Reader->getSummary(), OS);
+
   return 0;
 }
 
@@ -1090,6 +1232,9 @@ static int show_main(int argc, const char *argv[]) {
       cl::desc(
           "Cutoff percentages (times 10000) for generating detailed summary"),
       cl::value_desc("800000,901000,999999"));
+  cl::opt<bool> ShowHotFuncList(
+      "hot-func-list", cl::init(false),
+      cl::desc("Show profile summary of a list of hot functions"));
   cl::opt<bool> ShowAllFunctions("all-functions", cl::init(false),
                                  cl::desc("Details for every function"));
   cl::opt<bool> ShowCS("showcs", cl::init(false),
@@ -1153,7 +1298,8 @@ static int show_main(int argc, const char *argv[]) {
   else
     return showSampleProfile(Filename, ShowCounts, ShowAllFunctions,
                              ShowDetailedSummary, ShowFunction,
-                             ShowProfileSymbolList, ShowSectionInfoOnly, OS);
+                             ShowProfileSymbolList, ShowSectionInfoOnly,
+                             ShowHotFuncList, OS);
 }
 
 int main(int argc, const char *argv[]) {
diff --git a/llvm/tools/llvm-rc/Opts.td b/llvm/tools/llvm-rc/Opts.td
index 3ff5ac2d4980f..873dd785b12bd 100644
--- a/llvm/tools/llvm-rc/Opts.td
+++ b/llvm/tools/llvm-rc/Opts.td
@@ -12,7 +12,7 @@ def DEFINE : Separate<[ "/", "-" ], "D">,
 def UNDEF : Separate<[ "/", "-" ], "U">,
             HelpText<"Undefine a symbol for the C preprocessor.">;
 
-def LANG_ID : Separate<[ "/", "-" ], "L">,
+def LANG_ID : JoinedOrSeparate<[ "/", "-" ], "L">,
               HelpText<"Set the default language identifier.">;
 def LANG_NAME : Separate<[ "/", "-" ], "LN">,
                 HelpText<"Set the default language name.">;
diff --git a/llvm/tools/llvm-rc/llvm-rc.cpp b/llvm/tools/llvm-rc/llvm-rc.cpp
index a09960de4185d..71954804f2552 100644
--- a/llvm/tools/llvm-rc/llvm-rc.cpp
+++ b/llvm/tools/llvm-rc/llvm-rc.cpp
@@ -187,7 +187,16 @@ int main(int Argc, const char **Argv) {
     ExitOnErr(NullResource().visit(Visitor.get()));
 
     // Set the default language; choose en-US arbitrarily.
-    ExitOnErr(LanguageResource(0x09, 0x01).visit(Visitor.get()));
+    unsigned PrimaryLangId = 0x09, SubLangId = 0x01;
+    if (InputArgs.hasArg(OPT_LANG_ID)) {
+      unsigned LangId;
+      if (InputArgs.getLastArgValue(OPT_LANG_ID).getAsInteger(16, LangId))
+        fatalError("Invalid language id: " +
+                   InputArgs.getLastArgValue(OPT_LANG_ID));
+      PrimaryLangId = LangId & 0x3ff;
+      SubLangId = LangId >> 10;
+    }
+    ExitOnErr(LanguageResource(PrimaryLangId, SubLangId).visit(Visitor.get()));
   }
 
   rc::RCParser Parser{std::move(Tokens)};
diff --git a/llvm/tools/llvm-readobj/COFFDumper.cpp b/llvm/tools/llvm-readobj/COFFDumper.cpp
index fa14426395941..89a904f53ae7d 100644
--- a/llvm/tools/llvm-readobj/COFFDumper.cpp
+++ b/llvm/tools/llvm-readobj/COFFDumper.cpp
@@ -104,8 +104,10 @@ class COFFDumper : public ObjDumper {
                           bool GHash) override;
   void printStackMap() const override;
   void printAddrsig() override;
+  void printCGProfile() override;
 
 private:
+  StringRef getSymbolName(uint32_t Index);
   void printSymbols() override;
   void printDynamicSymbols() override;
   void printSymbol(const SymbolRef &Sym);
@@ -1516,16 +1518,8 @@ void COFFDumper::printSymbol(const SymbolRef &Sym) {
       if (std::error_code EC = getSymbolAuxData(Obj, Symbol, I, Aux))
         reportError(errorCodeToError(EC), Obj->getFileName());
 
-      Expected<COFFSymbolRef> Linked = Obj->getSymbol(Aux->TagIndex);
-      if (!Linked)
-        reportError(Linked.takeError(), Obj->getFileName());
-
-      Expected<StringRef> LinkedName = Obj->getSymbolName(*Linked);
-      if (!LinkedName)
-        reportError(LinkedName.takeError(), Obj->getFileName());
-
       DictScope AS(W, "AuxWeakExternal");
-      W.printNumber("Linked", *LinkedName, Aux->TagIndex);
+      W.printNumber("Linked", getSymbolName(Aux->TagIndex), Aux->TagIndex);
       W.printEnum  ("Search", Aux->Characteristics,
                     makeArrayRef(WeakExternalCharacteristics));
 
@@ -1570,19 +1564,11 @@ void COFFDumper::printSymbol(const SymbolRef &Sym) {
       if (std::error_code EC = getSymbolAuxData(Obj, Symbol, I, Aux))
         reportError(errorCodeToError(EC), Obj->getFileName());
 
-      Expected<COFFSymbolRef> ReferredSym =
-          Obj->getSymbol(Aux->SymbolTableIndex);
-      if (!ReferredSym)
-        reportError(ReferredSym.takeError(), Obj->getFileName());
-
-      Expected<StringRef> ReferredName = Obj->getSymbolName(*ReferredSym);
-      if (!ReferredName)
-        reportError(ReferredName.takeError(), Obj->getFileName());
-
       DictScope AS(W, "AuxCLRToken");
       W.printNumber("AuxType", Aux->AuxType);
       W.printNumber("Reserved", Aux->Reserved);
-      W.printNumber("SymbolTableIndex", *ReferredName, Aux->SymbolTableIndex);
+      W.printNumber("SymbolTableIndex", getSymbolName(Aux->SymbolTableIndex),
+                    Aux->SymbolTableIndex);
 
     } else {
       W.startLine() << "<unhandled auxiliary record>\n";
@@ -1904,7 +1890,7 @@ void COFFDumper::printResourceDirectoryTable(
 }
 
 void COFFDumper::printStackMap() const {
-  object::SectionRef StackMapSection;
+  SectionRef StackMapSection;
   for (auto Sec : Obj->sections()) {
     StringRef Name;
     if (Expected<StringRef> NameOrErr = Sec.getName())
@@ -1918,7 +1904,7 @@ void COFFDumper::printStackMap() const {
     }
   }
 
-  if (StackMapSection == object::SectionRef())
+  if (StackMapSection == SectionRef())
     return;
 
   StringRef StackMapContents =
@@ -1935,7 +1921,7 @@ void COFFDumper::printStackMap() const {
 }
 
 void COFFDumper::printAddrsig() {
-  object::SectionRef AddrsigSection;
+  SectionRef AddrsigSection;
   for (auto Sec : Obj->sections()) {
     StringRef Name;
     if (Expected<StringRef> NameOrErr = Sec.getName())
@@ -1949,7 +1935,7 @@ void COFFDumper::printAddrsig() {
     }
   }
 
-  if (AddrsigSection == object::SectionRef())
+  if (AddrsigSection == SectionRef())
     return;
 
   StringRef AddrsigContents =
@@ -1967,19 +1953,58 @@ void COFFDumper::printAddrsig() {
     if (Err)
       reportError(createError(Err), Obj->getFileName());
 
-    Expected<COFFSymbolRef> Sym = Obj->getSymbol(SymIndex);
-    if (!Sym)
-      reportError(Sym.takeError(), Obj->getFileName());
+    W.printNumber("Sym", getSymbolName(SymIndex), SymIndex);
+    Cur += Size;
+  }
+}
+
+void COFFDumper::printCGProfile() {
+  SectionRef CGProfileSection;
+  for (SectionRef Sec : Obj->sections()) {
+    StringRef Name = unwrapOrError(Obj->getFileName(), Sec.getName());
+    if (Name == ".llvm.call-graph-profile") {
+      CGProfileSection = Sec;
+      break;
+    }
+  }
 
-    Expected<StringRef> SymName = Obj->getSymbolName(*Sym);
-    if (!SymName)
-      reportError(SymName.takeError(), Obj->getFileName());
+  if (CGProfileSection == SectionRef())
+    return;
 
-    W.printNumber("Sym", *SymName, SymIndex);
-    Cur += Size;
+  StringRef CGProfileContents =
+      unwrapOrError(Obj->getFileName(), CGProfileSection.getContents());
+  BinaryStreamReader Reader(CGProfileContents, llvm::support::little);
+
+  ListScope L(W, "CGProfile");
+  while (!Reader.empty()) {
+    uint32_t FromIndex, ToIndex;
+    uint64_t Count;
+    if (Error Err = Reader.readInteger(FromIndex))
+      reportError(std::move(Err), Obj->getFileName());
+    if (Error Err = Reader.readInteger(ToIndex))
+      reportError(std::move(Err), Obj->getFileName());
+    if (Error Err = Reader.readInteger(Count))
+      reportError(std::move(Err), Obj->getFileName());
+
+    DictScope D(W, "CGProfileEntry");
+    W.printNumber("From", getSymbolName(FromIndex), FromIndex);
+    W.printNumber("To", getSymbolName(ToIndex), ToIndex);
+    W.printNumber("Weight", Count);
   }
 }
 
+StringRef COFFDumper::getSymbolName(uint32_t Index) {
+  Expected<COFFSymbolRef> Sym = Obj->getSymbol(Index);
+  if (!Sym)
+    reportError(Sym.takeError(), Obj->getFileName());
+
+  Expected<StringRef> SymName = Obj->getSymbolName(*Sym);
+  if (!SymName)
+    reportError(SymName.takeError(), Obj->getFileName());
+
+  return *SymName;
+}
+
 void llvm::dumpCodeViewMergedTypes(ScopedPrinter &Writer,
                                    ArrayRef<ArrayRef<uint8_t>> IpiRecords,
                                    ArrayRef<ArrayRef<uint8_t>> TpiRecords) {
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 832e7ce9791c2..6a7f37e39a9ab 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -336,13 +336,14 @@ template <typename ELFT> class ELFDumper : public ObjDumper {
   Elf_Rel_Range dyn_rels() const;
   Elf_Rela_Range dyn_relas() const;
   Elf_Relr_Range dyn_relrs() const;
-  std::string getFullSymbolName(const Elf_Sym *Symbol, StringRef StrTable,
+  std::string getFullSymbolName(const Elf_Sym *Symbol,
+                                Optional<StringRef> StrTable,
                                 bool IsDynamic) const;
   Expected<unsigned> getSymbolSectionIndex(const Elf_Sym *Symbol,
                                            const Elf_Sym *FirstSym) const;
   Expected<StringRef> getSymbolSectionName(const Elf_Sym *Symbol,
                                            unsigned SectionIndex) const;
-  Expected<std::string> getStaticSymbolName(uint32_t Index) const;
+  std::string getStaticSymbolName(uint32_t Index) const;
   StringRef getDynamicString(uint64_t Value) const;
   Expected<StringRef> getSymbolVersionByIndex(uint32_t VersionSymbolIndex,
                                               bool &IsDefault) const;
@@ -670,7 +671,8 @@ ELFDumper<ELFT>::getVersionDependencies(const Elf_Shdr *Sec) const {
 
 template <class ELFT>
 void ELFDumper<ELFT>::printSymbolsHelper(bool IsDynamic) const {
-  StringRef StrTable, SymtabName;
+  Optional<StringRef> StrTable;
+  StringRef SymtabName;
   size_t Entries = 0;
   Elf_Sym_Range Syms(nullptr, nullptr);
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
@@ -678,16 +680,36 @@ void ELFDumper<ELFT>::printSymbolsHelper(bool IsDynamic) const {
     StrTable = DynamicStringTable;
     Syms = dynamic_symbols();
     SymtabName = DynSymtabName;
-    if (DynSymRegion)
-      Entries = DynSymRegion->Size / DynSymRegion->EntSize;
+    Entries = Syms.size();
   } else {
     if (!DotSymtabSec)
       return;
-    StrTable = unwrapOrError(ObjF->getFileName(),
-                             Obj->getStringTableForSymtab(*DotSymtabSec));
-    Syms = unwrapOrError(ObjF->getFileName(), Obj->symbols(DotSymtabSec));
-    SymtabName =
-        unwrapOrError(ObjF->getFileName(), Obj->getSectionName(DotSymtabSec));
+
+    if (Expected<StringRef> StrTableOrErr =
+            Obj->getStringTableForSymtab(*DotSymtabSec))
+      StrTable = *StrTableOrErr;
+    else
+      reportUniqueWarning(createError(
+          "unable to get the string table for the SHT_SYMTAB section: " +
+          toString(StrTableOrErr.takeError())));
+
+    if (Expected<Elf_Sym_Range> SymsOrErr = Obj->symbols(DotSymtabSec))
+      Syms = *SymsOrErr;
+    else
+      reportUniqueWarning(
+          createError("unable to read symbols from the SHT_SYMTAB section: " +
+                      toString(SymsOrErr.takeError())));
+
+    if (Expected<StringRef> SymtabNameOrErr =
+            Obj->getSectionName(DotSymtabSec)) {
+      SymtabName = *SymtabNameOrErr;
+    } else {
+      reportUniqueWarning(
+          createError("unable to get the name of the SHT_SYMTAB section: " +
+                      toString(SymtabNameOrErr.takeError())));
+      SymtabName = "<?>";
+    }
+
     Entries = DotSymtabSec->getEntityCount();
   }
   if (Syms.begin() == Syms.end())
@@ -733,8 +755,9 @@ template <typename ELFT> class DumpStyle {
   virtual void printSymtabMessage(const ELFFile<ELFT> *Obj, StringRef Name,
                                   size_t Offset, bool NonVisibilityBitsUsed) {}
   virtual void printSymbol(const ELFFile<ELFT> *Obj, const Elf_Sym *Symbol,
-                           const Elf_Sym *FirstSym, StringRef StrTable,
-                           bool IsDynamic, bool NonVisibilityBitsUsed) = 0;
+                           const Elf_Sym *FirstSym,
+                           Optional<StringRef> StrTable, bool IsDynamic,
+                           bool NonVisibilityBitsUsed) = 0;
   virtual void printProgramHeaders(const ELFFile<ELFT> *Obj,
                                    bool PrintProgramHeaders,
                                    cl::boolOrDefault PrintSectionMapping) = 0;
@@ -828,6 +851,10 @@ template <typename ELFT> class GNUStyle : public DumpStyle<ELFT> {
   void printHashHistogram(const Elf_Hash &HashTable);
   void printGnuHashHistogram(const Elf_GnuHash &GnuHashTable);
 
+  void printHashTableSymbols(const ELFO *Obj, const Elf_Hash &HashTable);
+  void printGnuHashTableSymbols(const ELFO *Obj,
+                                const Elf_GnuHash &GnuHashTable);
+
   struct Field {
     std::string Str;
     unsigned Column;
@@ -887,7 +914,7 @@ template <typename ELFT> class GNUStyle : public DumpStyle<ELFT> {
   void printRelocation(const ELFO *Obj, const Elf_Sym *Sym,
                        StringRef SymbolName, const Elf_Rela &R, bool IsRela);
   void printSymbol(const ELFO *Obj, const Elf_Sym *Symbol, const Elf_Sym *First,
-                   StringRef StrTable, bool IsDynamic,
+                   Optional<StringRef> StrTable, bool IsDynamic,
                    bool NonVisibilityBitsUsed) override;
   std::string getSymbolSectionNdx(const ELFO *Obj, const Elf_Sym *Symbol,
                                   const Elf_Sym *FirstSym);
@@ -956,7 +983,7 @@ template <typename ELFT> class LLVMStyle : public DumpStyle<ELFT> {
   void printDynamicSymbols(const ELFO *Obj);
   void printSymbolSection(const Elf_Sym *Symbol, const Elf_Sym *First);
   void printSymbol(const ELFO *Obj, const Elf_Sym *Symbol, const Elf_Sym *First,
-                   StringRef StrTable, bool IsDynamic,
+                   Optional<StringRef> StrTable, bool IsDynamic,
                    bool /*NonVisibilityBitsUsed*/) override;
   void printProgramHeaders(const ELFO *Obj);
   void printSectionMapping(const ELFO *Obj) {}
@@ -1059,10 +1086,12 @@ Expected<StringRef> ELFDumper<ELFT>::getSymbolVersion(const Elf_Sym *Sym,
                       sizeof(Elf_Sym);
 
   // Get the corresponding version index entry.
-  const Elf_Versym *Versym = unwrapOrError(
-      ObjF->getFileName(), ObjF->getELFFile()->template getEntry<Elf_Versym>(
-                               SymbolVersionSection, EntryIndex));
-  return this->getSymbolVersionByIndex(Versym->vs_index, IsDefault);
+  if (Expected<const Elf_Versym *> EntryOrErr =
+          ObjF->getELFFile()->template getEntry<Elf_Versym>(
+              SymbolVersionSection, EntryIndex))
+    return this->getSymbolVersionByIndex((*EntryOrErr)->vs_index, IsDefault);
+  else
+    return EntryOrErr.takeError();
 }
 
 template <typename ELFT>
@@ -1108,21 +1137,27 @@ static std::string maybeDemangle(StringRef Name) {
 }
 
 template <typename ELFT>
-Expected<std::string>
-ELFDumper<ELFT>::getStaticSymbolName(uint32_t Index) const {
+std::string ELFDumper<ELFT>::getStaticSymbolName(uint32_t Index) const {
+  auto Warn = [&](Error E) -> std::string {
+    this->reportUniqueWarning(
+        createError("unable to read the name of symbol with index " +
+                    Twine(Index) + ": " + toString(std::move(E))));
+    return "<?>";
+  };
+
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
   Expected<const typename ELFT::Sym *> SymOrErr =
       Obj->getSymbol(DotSymtabSec, Index);
   if (!SymOrErr)
-    return SymOrErr.takeError();
+    return Warn(SymOrErr.takeError());
 
   Expected<StringRef> StrTabOrErr = Obj->getStringTableForSymtab(*DotSymtabSec);
   if (!StrTabOrErr)
-    return StrTabOrErr.takeError();
+    return Warn(StrTabOrErr.takeError());
 
   Expected<StringRef> NameOrErr = (*SymOrErr)->getName(*StrTabOrErr);
   if (!NameOrErr)
-    return NameOrErr.takeError();
+    return Warn(NameOrErr.takeError());
   return maybeDemangle(*NameOrErr);
 }
 
@@ -1155,10 +1190,18 @@ ELFDumper<ELFT>::getSymbolVersionByIndex(uint32_t SymbolVersionIndex,
 
 template <typename ELFT>
 std::string ELFDumper<ELFT>::getFullSymbolName(const Elf_Sym *Symbol,
-                                               StringRef StrTable,
+                                               Optional<StringRef> StrTable,
                                                bool IsDynamic) const {
-  std::string SymbolName = maybeDemangle(
-      unwrapOrError(ObjF->getFileName(), Symbol->getName(StrTable)));
+  if (!StrTable)
+    return "<?>";
+
+  std::string SymbolName;
+  if (Expected<StringRef> NameOrErr = Symbol->getName(*StrTable)) {
+    SymbolName = maybeDemangle(*NameOrErr);
+  } else {
+    reportUniqueWarning(NameOrErr.takeError());
+    return "<?>";
+  }
 
   if (SymbolName.empty() && Symbol->getType() == ELF::STT_SECTION) {
     Elf_Sym_Range Syms = unwrapOrError(
@@ -1238,7 +1281,7 @@ template <class ELFO>
 static const typename ELFO::Elf_Shdr *
 findNotEmptySectionByAddress(const ELFO *Obj, StringRef FileName,
                              uint64_t Addr) {
-  for (const auto &Shdr : unwrapOrError(FileName, Obj->sections()))
+  for (const typename ELFO::Elf_Shdr &Shdr : cantFail(Obj->sections()))
     if (Shdr.sh_addr == Addr && Shdr.sh_size > 0)
       return &Shdr;
   return nullptr;
@@ -1247,7 +1290,7 @@ findNotEmptySectionByAddress(const ELFO *Obj, StringRef FileName,
 template <class ELFO>
 static const typename ELFO::Elf_Shdr *
 findSectionByName(const ELFO &Obj, StringRef FileName, StringRef Name) {
-  for (const auto &Shdr : unwrapOrError(FileName, Obj.sections()))
+  for (const typename ELFO::Elf_Shdr &Shdr : cantFail(Obj.sections()))
     if (Name == unwrapOrError(FileName, Obj.getSectionName(&Shdr)))
       return &Shdr;
   return nullptr;
@@ -1868,8 +1911,7 @@ ELFDumper<ELFT>::findDynamic(const ELFFile<ELFT> *Obj) {
 
   // Try to locate the .dynamic section in the sections header table.
   const Elf_Shdr *DynamicSec = nullptr;
-  for (const Elf_Shdr &Sec :
-       unwrapOrError(ObjF->getFileName(), Obj->sections())) {
+  for (const Elf_Shdr &Sec : cantFail(Obj->sections())) {
     if (Sec.sh_type != ELF::SHT_DYNAMIC)
       continue;
     DynamicSec = &Sec;
@@ -2012,8 +2054,7 @@ ELFDumper<ELFT>::ELFDumper(const object::ELFObjectFile<ELFT> *ObjF,
     ELFDumperStyle.reset(new LLVMStyle<ELFT>(Writer, this));
 
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
-  typename ELFT::ShdrRange Sections =
-      unwrapOrError(ObjF->getFileName(), Obj->sections());
+  typename ELFT::ShdrRange Sections = cantFail(Obj->sections());
   for (const Elf_Shdr &Sec : Sections) {
     switch (Sec.sh_type) {
     case ELF::SHT_SYMTAB:
@@ -2185,8 +2226,20 @@ void ELFDumper<ELFT>::parseDynamicTable(const ELFFile<ELFT> *Obj) {
       break;
     }
   }
-  if (StringTableBegin)
-    DynamicStringTable = StringRef(StringTableBegin, StringTableSize);
+
+  if (StringTableBegin) {
+    const uint64_t FileSize = ObjF->getELFFile()->getBufSize();
+    const uint64_t Offset =
+        (const uint8_t *)StringTableBegin - ObjF->getELFFile()->base();
+    if (StringTableSize > FileSize - Offset)
+      reportUniqueWarning(createError(
+          "the dynamic string table at 0x" + Twine::utohexstr(Offset) +
+          " goes past the end of the file (0x" + Twine::utohexstr(FileSize) +
+          ") with DT_STRSZ = 0x" + Twine::utohexstr(StringTableSize)));
+    else
+      DynamicStringTable = StringRef(StringTableBegin, StringTableSize);
+  }
+
   SOName = getDynamicString(SONameOffset);
 
   if (DynSymRegion) {
@@ -2204,13 +2257,17 @@ void ELFDumper<ELFT>::parseDynamicTable(const ELFFile<ELFT> *Obj) {
     // equal nchain". Check to see if the DT_HASH hash table nchain value
     // conflicts with the number of symbols in the dynamic symbol table
     // according to the section header.
-    if (HashTable &&
-        HashTable->nchain != DynSymRegion->Size / DynSymRegion->EntSize)
-      reportUniqueWarning(createError(
-          "hash table nchain (" + Twine(HashTable->nchain) +
-          ") differs from symbol count derived from SHT_DYNSYM section "
-          "header (" +
-          Twine(DynSymRegion->Size / DynSymRegion->EntSize) + ")"));
+    if (HashTable) {
+      if (DynSymRegion->EntSize == 0)
+        reportUniqueWarning(
+            createError("SHT_DYNSYM section has sh_entsize == 0"));
+      else if (HashTable->nchain != DynSymRegion->Size / DynSymRegion->EntSize)
+        reportUniqueWarning(createError(
+            "hash table nchain (" + Twine(HashTable->nchain) +
+            ") differs from symbol count derived from SHT_DYNSYM section "
+            "header (" +
+            Twine(DynSymRegion->Size / DynSymRegion->EntSize) + ")"));
+    }
   }
 
   // Delay the creation of the actual dynamic symbol table until now, so that
@@ -2810,9 +2867,14 @@ template <class ELFT> void ELFDumper<ELFT>::printArchSpecificInfo() {
 
     MipsGOTParser<ELFT> Parser(Obj, ObjF->getFileName(), dynamic_table(),
                                dynamic_symbols());
-    if (Parser.hasGot())
+    if (Error E = Parser.findGOT(dynamic_table(), dynamic_symbols()))
+      reportError(std::move(E), ObjF->getFileName());
+    else if (!Parser.isGotEmpty())
       ELFDumperStyle->printMipsGOT(Parser);
-    if (Parser.hasPlt())
+
+    if (Error E = Parser.findPLT(dynamic_table()))
+      reportError(std::move(E), ObjF->getFileName());
+    else if (!Parser.isPltEmpty())
       ELFDumperStyle->printMipsPLT(Parser);
     break;
   }
@@ -2835,7 +2897,7 @@ template <class ELFT> void ELFDumper<ELFT>::printAttributes() {
          "Attributes not implemented.");
 
   DictScope BA(W, "BuildAttributes");
-  for (const auto &Sec : unwrapOrError(ObjF->getFileName(), Obj->sections())) {
+  for (const Elf_Shdr &Sec : cantFail(Obj->sections())) {
     if (Sec.sh_type != ELF::SHT_ARM_ATTRIBUTES &&
         Sec.sh_type != ELF::SHT_RISCV_ATTRIBUTES)
       continue;
@@ -2874,9 +2936,11 @@ template <class ELFT> class MipsGOTParser {
 
   MipsGOTParser(const ELFO *Obj, StringRef FileName, Elf_Dyn_Range DynTable,
                 Elf_Sym_Range DynSyms);
+  Error findGOT(Elf_Dyn_Range DynTable, Elf_Sym_Range DynSyms);
+  Error findPLT(Elf_Dyn_Range DynTable);
 
-  bool hasGot() const { return !GotEntries.empty(); }
-  bool hasPlt() const { return !PltEntries.empty(); }
+  bool isGotEmpty() const { return GotEntries.empty(); }
+  bool isPltEmpty() const { return PltEntries.empty(); }
 
   uint64_t getGp() const;
 
@@ -2924,7 +2988,11 @@ MipsGOTParser<ELFT>::MipsGOTParser(const ELFO *Obj, StringRef FileName,
                                    Elf_Sym_Range DynSyms)
     : IsStatic(DynTable.empty()), Obj(Obj), GotSec(nullptr), LocalNum(0),
       GlobalNum(0), PltSec(nullptr), PltRelSec(nullptr), PltSymTable(nullptr),
-      FileName(FileName) {
+      FileName(FileName) {}
+
+template <class ELFT>
+Error MipsGOTParser<ELFT>::findGOT(Elf_Dyn_Range DynTable,
+                                   Elf_Sym_Range DynSyms) {
   // See "Global Offset Table" in Chapter 5 in the following document
   // for detailed GOT description.
   // ftp://www.linux-mips.org/pub/linux/mips/doc/ABI/mipsabi.pdf
@@ -2933,22 +3001,20 @@ MipsGOTParser<ELFT>::MipsGOTParser(const ELFO *Obj, StringRef FileName,
   if (IsStatic) {
     GotSec = findSectionByName(*Obj, FileName, ".got");
     if (!GotSec)
-      return;
+      return Error::success();
 
     ArrayRef<uint8_t> Content =
         unwrapOrError(FileName, Obj->getSectionContents(GotSec));
     GotEntries = Entries(reinterpret_cast<const Entry *>(Content.data()),
                          Content.size() / sizeof(Entry));
     LocalNum = GotEntries.size();
-    return;
+    return Error::success();
   }
 
-  // Lookup dynamic table tags which define GOT/PLT layouts.
+  // Lookup dynamic table tags which define the GOT layout.
   Optional<uint64_t> DtPltGot;
   Optional<uint64_t> DtLocalGotNum;
   Optional<uint64_t> DtGotSym;
-  Optional<uint64_t> DtMipsPltGot;
-  Optional<uint64_t> DtJmpRel;
   for (const auto &Entry : DynTable) {
     switch (Entry.getTag()) {
     case ELF::DT_PLTGOT:
@@ -2960,6 +3026,49 @@ MipsGOTParser<ELFT>::MipsGOTParser(const ELFO *Obj, StringRef FileName,
     case ELF::DT_MIPS_GOTSYM:
       DtGotSym = Entry.getVal();
       break;
+    }
+  }
+
+  if (!DtPltGot && !DtLocalGotNum && !DtGotSym)
+    return Error::success();
+
+  if (!DtPltGot)
+    return createError("cannot find PLTGOT dynamic tag");
+  if (!DtLocalGotNum)
+    return createError("cannot find MIPS_LOCAL_GOTNO dynamic tag");
+  if (!DtGotSym)
+    return createError("cannot find MIPS_GOTSYM dynamic tag");
+
+  size_t DynSymTotal = DynSyms.size();
+  if (*DtGotSym > DynSymTotal)
+    return createError("DT_MIPS_GOTSYM value (" + Twine(*DtGotSym) +
+                       ") exceeds the number of dynamic symbols (" +
+                       Twine(DynSymTotal) + ")");
+
+  GotSec = findNotEmptySectionByAddress(Obj, FileName, *DtPltGot);
+  if (!GotSec)
+    return createError("there is no non-empty GOT section at 0x" +
+                       Twine::utohexstr(*DtPltGot));
+
+  LocalNum = *DtLocalGotNum;
+  GlobalNum = DynSymTotal - *DtGotSym;
+
+  ArrayRef<uint8_t> Content =
+      unwrapOrError(FileName, Obj->getSectionContents(GotSec));
+  GotEntries = Entries(reinterpret_cast<const Entry *>(Content.data()),
+                       Content.size() / sizeof(Entry));
+  GotDynSyms = DynSyms.drop_front(*DtGotSym);
+
+  return Error::success();
+}
+
+template <class ELFT>
+Error MipsGOTParser<ELFT>::findPLT(Elf_Dyn_Range DynTable) {
+  // Lookup dynamic table tags which define the PLT layout.
+  Optional<uint64_t> DtMipsPltGot;
+  Optional<uint64_t> DtJmpRel;
+  for (const auto &Entry : DynTable) {
+    switch (Entry.getTag()) {
     case ELF::DT_MIPS_PLTGOT:
       DtMipsPltGot = Entry.getVal();
       break;
@@ -2969,63 +3078,56 @@ MipsGOTParser<ELFT>::MipsGOTParser(const ELFO *Obj, StringRef FileName,
     }
   }
 
-  // Find dynamic GOT section.
-  if (DtPltGot || DtLocalGotNum || DtGotSym) {
-    if (!DtPltGot)
-      report_fatal_error("Cannot find PLTGOT dynamic table tag.");
-    if (!DtLocalGotNum)
-      report_fatal_error("Cannot find MIPS_LOCAL_GOTNO dynamic table tag.");
-    if (!DtGotSym)
-      report_fatal_error("Cannot find MIPS_GOTSYM dynamic table tag.");
-
-    size_t DynSymTotal = DynSyms.size();
-    if (*DtGotSym > DynSymTotal)
-      reportError(
-          createError("MIPS_GOTSYM exceeds a number of dynamic symbols"),
-          FileName);
-
-    GotSec = findNotEmptySectionByAddress(Obj, FileName, *DtPltGot);
-    if (!GotSec)
-      reportError(createError("There is no not empty GOT section at 0x" +
-                              Twine::utohexstr(*DtPltGot)),
-                  FileName);
-
-    LocalNum = *DtLocalGotNum;
-    GlobalNum = DynSymTotal - *DtGotSym;
-
-    ArrayRef<uint8_t> Content =
-        unwrapOrError(FileName, Obj->getSectionContents(GotSec));
-    GotEntries = Entries(reinterpret_cast<const Entry *>(Content.data()),
-                         Content.size() / sizeof(Entry));
-    GotDynSyms = DynSyms.drop_front(*DtGotSym);
-  }
+  if (!DtMipsPltGot && !DtJmpRel)
+    return Error::success();
 
   // Find PLT section.
-  if (DtMipsPltGot || DtJmpRel) {
-    if (!DtMipsPltGot)
-      report_fatal_error("Cannot find MIPS_PLTGOT dynamic table tag.");
-    if (!DtJmpRel)
-      report_fatal_error("Cannot find JMPREL dynamic table tag.");
-
-    PltSec = findNotEmptySectionByAddress(Obj, FileName, * DtMipsPltGot);
-    if (!PltSec)
-      report_fatal_error("There is no not empty PLTGOT section at 0x " +
-                         Twine::utohexstr(*DtMipsPltGot));
-
-    PltRelSec = findNotEmptySectionByAddress(Obj, FileName, * DtJmpRel);
-    if (!PltRelSec)
-      report_fatal_error("There is no not empty RELPLT section at 0x" +
-                         Twine::utohexstr(*DtJmpRel));
+  if (!DtMipsPltGot)
+    return createError("cannot find MIPS_PLTGOT dynamic tag");
+  if (!DtJmpRel)
+    return createError("cannot find JMPREL dynamic tag");
+
+  PltSec = findNotEmptySectionByAddress(Obj, FileName, *DtMipsPltGot);
+  if (!PltSec)
+    return createError("there is no non-empty PLTGOT section at 0x" +
+                       Twine::utohexstr(*DtMipsPltGot));
+
+  PltRelSec = findNotEmptySectionByAddress(Obj, FileName, *DtJmpRel);
+  if (!PltRelSec)
+    return createError("there is no non-empty RELPLT section at 0x" +
+                       Twine::utohexstr(*DtJmpRel));
+
+  if (Expected<ArrayRef<uint8_t>> PltContentOrErr =
+          Obj->getSectionContents(PltSec))
+    PltEntries =
+        Entries(reinterpret_cast<const Entry *>(PltContentOrErr->data()),
+                PltContentOrErr->size() / sizeof(Entry));
+  else
+    return createError("unable to read PLTGOT section content: " +
+                       toString(PltContentOrErr.takeError()));
 
-    ArrayRef<uint8_t> PltContent =
-        unwrapOrError(FileName, Obj->getSectionContents(PltSec));
-    PltEntries = Entries(reinterpret_cast<const Entry *>(PltContent.data()),
-                         PltContent.size() / sizeof(Entry));
+  if (Expected<const Elf_Shdr *> PltSymTableOrErr =
+          Obj->getSection(PltRelSec->sh_link)) {
+    PltSymTable = *PltSymTableOrErr;
+  } else {
+    unsigned SecNdx = PltRelSec - &cantFail(Obj->sections()).front();
+    return createError("unable to get a symbol table linked to the RELPLT "
+                       "section with index " +
+                       Twine(SecNdx) + ": " +
+                       toString(PltSymTableOrErr.takeError()));
+  }
 
-    PltSymTable = unwrapOrError(FileName, Obj->getSection(PltRelSec->sh_link));
-    PltStrTable =
-        unwrapOrError(FileName, Obj->getStringTableForSymtab(*PltSymTable));
+  if (Expected<StringRef> StrTabOrErr =
+          Obj->getStringTableForSymtab(*PltSymTable)) {
+    PltStrTable = *StrTabOrErr;
+  } else {
+    unsigned SecNdx = PltSymTable - &cantFail(Obj->sections()).front();
+    return createError(
+        "unable to get a string table for the symbol table with index " +
+        Twine(SecNdx) + ": " + toString(StrTabOrErr.takeError()));
   }
+
+  return Error::success();
 }
 
 template <class ELFT> uint64_t MipsGOTParser<ELFT>::getGp() const {
@@ -3273,7 +3375,7 @@ template <class ELFT> void ELFDumper<ELFT>::printMipsOptions() {
 template <class ELFT> void ELFDumper<ELFT>::printStackMap() const {
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
   const Elf_Shdr *StackMapSection = nullptr;
-  for (const auto &Sec : unwrapOrError(ObjF->getFileName(), Obj->sections())) {
+  for (const Elf_Shdr &Sec : cantFail(Obj->sections())) {
     StringRef Name =
         unwrapOrError(ObjF->getFileName(), Obj->getSectionName(&Sec));
     if (Name == ".llvm_stackmaps") {
@@ -3316,7 +3418,7 @@ static std::string getSectionHeadersNumString(const ELFFile<ELFT> *Obj,
   if (ElfHeader->e_shnum != 0)
     return to_string(ElfHeader->e_shnum);
 
-  ArrayRef<typename ELFT::Shdr> Arr = unwrapOrError(FileName, Obj->sections());
+  ArrayRef<typename ELFT::Shdr> Arr = cantFail(Obj->sections());
   if (Arr.empty())
     return "0";
   return "0 (" + to_string(Arr[0].sh_size) + ")";
@@ -3329,7 +3431,7 @@ static std::string getSectionHeaderTableIndexString(const ELFFile<ELFT> *Obj,
   if (ElfHeader->e_shstrndx != SHN_XINDEX)
     return to_string(ElfHeader->e_shstrndx);
 
-  ArrayRef<typename ELFT::Shdr> Arr = unwrapOrError(FileName, Obj->sections());
+  ArrayRef<typename ELFT::Shdr> Arr = cantFail(Obj->sections());
   if (Arr.empty())
     return "65535 (corrupt: out of range)";
   return to_string(ElfHeader->e_shstrndx) + " (" + to_string(Arr[0].sh_link) +
@@ -3423,7 +3525,7 @@ std::vector<GroupSection> getGroups(const ELFFile<ELFT> *Obj,
 
   std::vector<GroupSection> Ret;
   uint64_t I = 0;
-  for (const Elf_Shdr &Sec : unwrapOrError(FileName, Obj->sections())) {
+  for (const Elf_Shdr &Sec : cantFail(Obj->sections())) {
     ++I;
     if (Sec.sh_type != ELF::SHT_GROUP)
       continue;
@@ -3574,7 +3676,7 @@ template <class ELFT> void GNUStyle<ELFT>::printRelocHeader(unsigned SType) {
 
 template <class ELFT> void GNUStyle<ELFT>::printRelocations(const ELFO *Obj) {
   bool HasRelocSections = false;
-  for (const Elf_Shdr &Sec : unwrapOrError(this->FileName, Obj->sections())) {
+  for (const Elf_Shdr &Sec : cantFail(Obj->sections())) {
     if (Sec.sh_type != ELF::SHT_REL && Sec.sh_type != ELF::SHT_RELA &&
         Sec.sh_type != ELF::SHT_RELR && Sec.sh_type != ELF::SHT_ANDROID_REL &&
         Sec.sh_type != ELF::SHT_ANDROID_RELA &&
@@ -3800,7 +3902,7 @@ static void printSectionDescription(formatted_raw_ostream &OS,
 template <class ELFT>
 void GNUStyle<ELFT>::printSectionHeaders(const ELFO *Obj) {
   unsigned Bias = ELFT::Is64Bits ? 0 : 8;
-  ArrayRef<Elf_Shdr> Sections = unwrapOrError(this->FileName, Obj->sections());
+  ArrayRef<Elf_Shdr> Sections = cantFail(Obj->sections());
   OS << "There are " << to_string(Sections.size())
      << " section headers, starting at offset "
      << "0x" << to_hexString(Obj->getHeader()->e_shoff, false) << ":\n\n";
@@ -3814,10 +3916,13 @@ void GNUStyle<ELFT>::printSectionHeaders(const ELFO *Obj) {
     printField(F);
   OS << "\n";
 
-  const ELFObjectFile<ELFT> *ElfObj = this->dumper()->getElfObject();
-  StringRef SecStrTable = unwrapOrError<StringRef>(
-      ElfObj->getFileName(),
-      Obj->getSectionStringTable(Sections, this->dumper()->WarningHandler));
+  StringRef SecStrTable;
+  if (Expected<StringRef> SecStrTableOrErr =
+          Obj->getSectionStringTable(Sections, this->dumper()->WarningHandler))
+    SecStrTable = *SecStrTableOrErr;
+  else
+    this->reportUniqueWarning(SecStrTableOrErr.takeError());
+
   size_t SectionIndex = 0;
   for (const Elf_Shdr &Sec : Sections) {
     Fields[0].Str = to_string(SectionIndex);
@@ -3825,7 +3930,7 @@ void GNUStyle<ELFT>::printSectionHeaders(const ELFO *Obj) {
       Fields[1].Str = "<no-strings>";
     else
       Fields[1].Str = std::string(unwrapOrError<StringRef>(
-          ElfObj->getFileName(), Obj->getSectionName(&Sec, SecStrTable)));
+          this->FileName, Obj->getSectionName(&Sec, SecStrTable)));
     Fields[2].Str =
         getSectionTypeString(Obj->getHeader()->e_machine, Sec.sh_type);
     Fields[3].Str =
@@ -3922,8 +4027,9 @@ std::string GNUStyle<ELFT>::getSymbolSectionNdx(const ELFO *Obj,
 
 template <class ELFT>
 void GNUStyle<ELFT>::printSymbol(const ELFO *Obj, const Elf_Sym *Symbol,
-                                 const Elf_Sym *FirstSym, StringRef StrTable,
-                                 bool IsDynamic, bool NonVisibilityBitsUsed) {
+                                 const Elf_Sym *FirstSym,
+                                 Optional<StringRef> StrTable, bool IsDynamic,
+                                 bool NonVisibilityBitsUsed) {
   static int Idx = 0;
   static bool Dynamic = true;
 
@@ -4013,73 +4119,110 @@ void GNUStyle<ELFT>::printSymbols(const ELFO *Obj, bool PrintSymbols,
     this->dumper()->printSymbolsHelper(false);
 }
 
-template <class ELFT> void GNUStyle<ELFT>::printHashSymbols(const ELFO *Obj) {
-  if (this->dumper()->getDynamicStringTable().empty())
+template <class ELFT>
+void GNUStyle<ELFT>::printHashTableSymbols(const ELFO *Obj,
+                                           const Elf_Hash &SysVHash) {
+  StringRef StringTable = this->dumper()->getDynamicStringTable();
+  if (StringTable.empty())
     return;
-  auto StringTable = this->dumper()->getDynamicStringTable();
-  auto DynSyms = this->dumper()->dynamic_symbols();
 
-  auto PrintHashTable = [&](const Elf_Hash *SysVHash) {
-    if (ELFT::Is64Bits)
-      OS << "  Num Buc:    Value          Size   Type   Bind Vis      Ndx Name";
-    else
-      OS << "  Num Buc:    Value  Size   Type   Bind Vis      Ndx Name";
-    OS << "\n";
+  if (ELFT::Is64Bits)
+    OS << "  Num Buc:    Value          Size   Type   Bind Vis      Ndx Name";
+  else
+    OS << "  Num Buc:    Value  Size   Type   Bind Vis      Ndx Name";
+  OS << "\n";
 
-    auto Buckets = SysVHash->buckets();
-    auto Chains = SysVHash->chains();
-    for (uint32_t Buc = 0; Buc < SysVHash->nbucket; Buc++) {
-      if (Buckets[Buc] == ELF::STN_UNDEF)
-        continue;
-      std::vector<bool> Visited(SysVHash->nchain);
-      for (uint32_t Ch = Buckets[Buc]; Ch < SysVHash->nchain; Ch = Chains[Ch]) {
-        if (Ch == ELF::STN_UNDEF)
-          break;
+  Elf_Sym_Range DynSyms = this->dumper()->dynamic_symbols();
+  const Elf_Sym *FirstSym = DynSyms.empty() ? nullptr : &DynSyms[0];
+  if (!FirstSym) {
+    Optional<DynRegionInfo> DynSymRegion = this->dumper()->getDynSymRegion();
+    this->reportUniqueWarning(
+        createError(Twine("unable to print symbols for the .hash table: the "
+                          "dynamic symbol table ") +
+                    (DynSymRegion ? "is empty" : "was not found")));
+    return;
+  }
 
-        if (Visited[Ch]) {
-          reportWarning(
-              createError(".hash section is invalid: bucket " + Twine(Ch) +
-                          ": a cycle was detected in the linked chain"),
-              this->FileName);
-          break;
-        }
+  auto Buckets = SysVHash.buckets();
+  auto Chains = SysVHash.chains();
+  for (uint32_t Buc = 0; Buc < SysVHash.nbucket; Buc++) {
+    if (Buckets[Buc] == ELF::STN_UNDEF)
+      continue;
+    std::vector<bool> Visited(SysVHash.nchain);
+    for (uint32_t Ch = Buckets[Buc]; Ch < SysVHash.nchain; Ch = Chains[Ch]) {
+      if (Ch == ELF::STN_UNDEF)
+        break;
 
-        printHashedSymbol(Obj, &DynSyms[0], Ch, StringTable, Buc);
-        Visited[Ch] = true;
+      if (Visited[Ch]) {
+        reportWarning(createError(".hash section is invalid: bucket " +
+                                  Twine(Ch) +
+                                  ": a cycle was detected in the linked chain"),
+                      this->FileName);
+        break;
       }
+
+      printHashedSymbol(Obj, FirstSym, Ch, StringTable, Buc);
+      Visited[Ch] = true;
     }
-  };
+  }
+}
+
+template <class ELFT>
+void GNUStyle<ELFT>::printGnuHashTableSymbols(const ELFO *Obj,
+                                              const Elf_GnuHash &GnuHash) {
+  StringRef StringTable = this->dumper()->getDynamicStringTable();
+  if (StringTable.empty())
+    return;
+
+  Elf_Sym_Range DynSyms = this->dumper()->dynamic_symbols();
+  const Elf_Sym *FirstSym = DynSyms.empty() ? nullptr : &DynSyms[0];
+  if (!FirstSym) {
+    Optional<DynRegionInfo> DynSymRegion = this->dumper()->getDynSymRegion();
+    this->reportUniqueWarning(createError(
+        Twine("unable to print symbols for the .gnu.hash table: the "
+              "dynamic symbol table ") +
+        (DynSymRegion ? "is empty" : "was not found")));
+    return;
+  }
 
+  ArrayRef<Elf_Word> Buckets = GnuHash.buckets();
+  for (uint32_t Buc = 0; Buc < GnuHash.nbuckets; Buc++) {
+    if (Buckets[Buc] == ELF::STN_UNDEF)
+      continue;
+    uint32_t Index = Buckets[Buc];
+    uint32_t GnuHashable = Index - GnuHash.symndx;
+    // Print whole chain
+    while (true) {
+      printHashedSymbol(Obj, FirstSym, Index++, StringTable, Buc);
+      // Chain ends at symbol with stopper bit
+      if ((GnuHash.values(DynSyms.size())[GnuHashable++] & 1) == 1)
+        break;
+    }
+  }
+}
+
+template <class ELFT> void GNUStyle<ELFT>::printHashSymbols(const ELFO *Obj) {
   if (const Elf_Hash *SysVHash = this->dumper()->getHashTable()) {
     OS << "\n Symbol table of .hash for image:\n";
     if (Error E = checkHashTable<ELFT>(Obj, SysVHash))
       this->reportUniqueWarning(std::move(E));
     else
-      PrintHashTable(SysVHash);
+      printHashTableSymbols(Obj, *SysVHash);
   }
 
-  // Try printing .gnu.hash
-  if (auto GnuHash = this->dumper()->getGnuHashTable()) {
+  // Try printing the .gnu.hash table.
+  if (const Elf_GnuHash *GnuHash = this->dumper()->getGnuHashTable()) {
     OS << "\n Symbol table of .gnu.hash for image:\n";
     if (ELFT::Is64Bits)
       OS << "  Num Buc:    Value          Size   Type   Bind Vis      Ndx Name";
     else
       OS << "  Num Buc:    Value  Size   Type   Bind Vis      Ndx Name";
     OS << "\n";
-    auto Buckets = GnuHash->buckets();
-    for (uint32_t Buc = 0; Buc < GnuHash->nbuckets; Buc++) {
-      if (Buckets[Buc] == ELF::STN_UNDEF)
-        continue;
-      uint32_t Index = Buckets[Buc];
-      uint32_t GnuHashable = Index - GnuHash->symndx;
-      // Print whole chain
-      while (true) {
-        printHashedSymbol(Obj, &DynSyms[0], Index++, StringTable, Buc);
-        // Chain ends at symbol with stopper bit
-        if ((GnuHash->values(DynSyms.size())[GnuHashable++] & 1) == 1)
-          break;
-      }
-    }
+
+    if (Error E = checkGNUHashTable<ELFT>(Obj, GnuHash))
+      this->reportUniqueWarning(std::move(E));
+    else
+      printGnuHashTableSymbols(Obj, *GnuHash);
   }
 }
 
@@ -4250,7 +4393,7 @@ void GNUStyle<ELFT>::printSectionMapping(const ELFO *Obj) {
     std::string Sections;
     OS << format("   %2.2d     ", Phnum++);
     // Check if each section is in a segment and then print mapping.
-    for (const Elf_Shdr &Sec : unwrapOrError(this->FileName, Obj->sections())) {
+    for (const Elf_Shdr &Sec : cantFail(Obj->sections())) {
       if (Sec.sh_type == ELF::SHT_NULL)
         continue;
 
@@ -4271,7 +4414,7 @@ void GNUStyle<ELFT>::printSectionMapping(const ELFO *Obj) {
 
   // Display sections that do not belong to a segment.
   std::string Sections;
-  for (const Elf_Shdr &Sec : unwrapOrError(this->FileName, Obj->sections())) {
+  for (const Elf_Shdr &Sec : cantFail(Obj->sections())) {
     if (BelongsToSegment.find(&Sec) == BelongsToSegment.end())
       Sections +=
           unwrapOrError(this->FileName, Obj->getSectionName(&Sec)).str() + ' ';
@@ -5348,7 +5491,7 @@ void GNUStyle<ELFT>::printNotes(const ELFFile<ELFT> *Obj) {
     }
   };
 
-  ArrayRef<Elf_Shdr> Sections = unwrapOrError(this->FileName, Obj->sections());
+  ArrayRef<Elf_Shdr> Sections = cantFail(Obj->sections());
   if (Obj->getHeader()->e_type != ELF::ET_CORE && !Sections.empty()) {
     for (const auto &S : Sections) {
       if (S.sh_type != SHT_NOTE)
@@ -5393,7 +5536,7 @@ void DumpStyle<ELFT>::printDependentLibsHelper(
   };
 
   unsigned I = -1;
-  for (const Elf_Shdr &Shdr : unwrapOrError(this->FileName, Obj->sections())) {
+  for (const Elf_Shdr &Shdr : cantFail(Obj->sections())) {
     ++I;
     if (Shdr.sh_type != ELF::SHT_LLVM_DEPENDENT_LIBRARIES)
       continue;
@@ -6017,7 +6160,7 @@ template <class ELFT> void LLVMStyle<ELFT>::printRelocations(const ELFO *Obj) {
   ListScope D(W, "Relocations");
 
   int SectionNumber = -1;
-  for (const Elf_Shdr &Sec : unwrapOrError(this->FileName, Obj->sections())) {
+  for (const Elf_Shdr &Sec : cantFail(Obj->sections())) {
     ++SectionNumber;
 
     if (Sec.sh_type != ELF::SHT_REL && Sec.sh_type != ELF::SHT_RELA &&
@@ -6119,10 +6262,9 @@ void LLVMStyle<ELFT>::printSectionHeaders(const ELFO *Obj) {
   ListScope SectionsD(W, "Sections");
 
   int SectionIndex = -1;
-  ArrayRef<Elf_Shdr> Sections = unwrapOrError(this->FileName, Obj->sections());
   std::vector<EnumEntry<unsigned>> FlagsList =
       getSectionFlagsForTarget(Obj->getHeader()->e_machine);
-  for (const Elf_Shdr &Sec : Sections) {
+  for (const Elf_Shdr &Sec : cantFail(Obj->sections())) {
     StringRef Name = "<?>";
     if (Expected<StringRef> SecNameOrErr =
             Obj->getSectionName(&Sec, this->dumper()->WarningHandler))
@@ -6211,8 +6353,8 @@ void LLVMStyle<ELFT>::printSymbolSection(const Elf_Sym *Symbol,
 
 template <class ELFT>
 void LLVMStyle<ELFT>::printSymbol(const ELFO *Obj, const Elf_Sym *Symbol,
-                                  const Elf_Sym *First, StringRef StrTable,
-                                  bool IsDynamic,
+                                  const Elf_Sym *First,
+                                  Optional<StringRef> StrTable, bool IsDynamic,
                                   bool /*NonVisibilityBitsUsed*/) {
   std::string FullSymbolName =
       this->dumper()->getFullSymbolName(Symbol, StrTable, IsDynamic);
@@ -6306,14 +6448,14 @@ void LLVMStyle<ELFT>::printDynamicRelocations(const ELFO *Obj) {
   const DynRegionInfo &DynRelaRegion = this->dumper()->getDynRelaRegion();
   const DynRegionInfo &DynRelrRegion = this->dumper()->getDynRelrRegion();
   const DynRegionInfo &DynPLTRelRegion = this->dumper()->getDynPLTRelRegion();
-  if (DynRelRegion.Size && DynRelaRegion.Size)
-    report_fatal_error("There are both REL and RELA dynamic relocations");
+
   W.startLine() << "Dynamic Relocations {\n";
   W.indent();
-  if (DynRelaRegion.Size > 0)
+  if (DynRelaRegion.Size > 0) {
     for (const Elf_Rela &Rela : this->dumper()->dyn_relas())
       printDynamicRelocation(Obj, Rela);
-  else
+  }
+  if (DynRelRegion.Size > 0) {
     for (const Elf_Rel &Rel : this->dumper()->dyn_rels()) {
       Elf_Rela Rela;
       Rela.r_offset = Rel.r_offset;
@@ -6321,6 +6463,8 @@ void LLVMStyle<ELFT>::printDynamicRelocations(const ELFO *Obj) {
       Rela.r_addend = 0;
       printDynamicRelocation(Obj, Rela);
     }
+  }
+
   if (DynRelrRegion.Size > 0) {
     Elf_Relr_Range Relrs = this->dumper()->dyn_relrs();
     std::vector<Elf_Rela> RelrRelas =
@@ -6488,21 +6632,23 @@ void LLVMStyle<ELFT>::printCGProfile(const ELFFile<ELFT> *Obj) {
   ListScope L(W, "CGProfile");
   if (!this->dumper()->getDotCGProfileSec())
     return;
-  auto CGProfile = unwrapOrError(
-      this->FileName, Obj->template getSectionContentsAsArray<Elf_CGProfile>(
-                          this->dumper()->getDotCGProfileSec()));
-  for (const Elf_CGProfile &CGPE : CGProfile) {
+
+  Expected<ArrayRef<Elf_CGProfile>> CGProfileOrErr =
+      Obj->template getSectionContentsAsArray<Elf_CGProfile>(
+          this->dumper()->getDotCGProfileSec());
+  if (!CGProfileOrErr) {
+    this->reportUniqueWarning(
+        createError("unable to dump the SHT_LLVM_CALL_GRAPH_PROFILE section: " +
+                    toString(CGProfileOrErr.takeError())));
+    return;
+  }
+
+  for (const Elf_CGProfile &CGPE : *CGProfileOrErr) {
     DictScope D(W, "CGProfileEntry");
-    W.printNumber(
-        "From",
-        unwrapOrError(this->FileName,
-                      this->dumper()->getStaticSymbolName(CGPE.cgp_from)),
-        CGPE.cgp_from);
-    W.printNumber(
-        "To",
-        unwrapOrError(this->FileName,
-                      this->dumper()->getStaticSymbolName(CGPE.cgp_to)),
-        CGPE.cgp_to);
+    W.printNumber("From", this->dumper()->getStaticSymbolName(CGPE.cgp_from),
+                  CGPE.cgp_from);
+    W.printNumber("To", this->dumper()->getStaticSymbolName(CGPE.cgp_to),
+                  CGPE.cgp_to);
     W.printNumber("Weight", CGPE.cgp_weight);
   }
 }
@@ -6657,7 +6803,7 @@ void LLVMStyle<ELFT>::printNotes(const ELFFile<ELFT> *Obj) {
     }
   };
 
-  ArrayRef<Elf_Shdr> Sections = unwrapOrError(this->FileName, Obj->sections());
+  ArrayRef<Elf_Shdr> Sections = cantFail(Obj->sections());
   if (Obj->getHeader()->e_type != ELF::ET_CORE && !Sections.empty()) {
     for (const auto &S : Sections) {
       if (S.sh_type != SHT_NOTE)
@@ -6692,35 +6838,38 @@ void LLVMStyle<ELFT>::printELFLinkerOptions(const ELFFile<ELFT> *Obj) {
   ListScope L(W, "LinkerOptions");
 
   unsigned I = -1;
-  for (const Elf_Shdr &Shdr : unwrapOrError(this->FileName, Obj->sections())) {
+  for (const Elf_Shdr &Shdr : cantFail(Obj->sections())) {
     ++I;
     if (Shdr.sh_type != ELF::SHT_LLVM_LINKER_OPTIONS)
       continue;
 
-    ArrayRef<uint8_t> Contents =
-        unwrapOrError(this->FileName, Obj->getSectionContents(&Shdr));
-    if (Contents.empty())
+    Expected<ArrayRef<uint8_t>> ContentsOrErr = Obj->getSectionContents(&Shdr);
+    if (!ContentsOrErr) {
+      this->reportUniqueWarning(
+          createError("unable to read the content of the "
+                      "SHT_LLVM_LINKER_OPTIONS section: " +
+                      toString(ContentsOrErr.takeError())));
+      continue;
+    }
+    if (ContentsOrErr->empty())
       continue;
 
-    if (Contents.back() != 0) {
-      reportWarning(createError("SHT_LLVM_LINKER_OPTIONS section at index " +
-                                Twine(I) +
-                                " is broken: the "
-                                "content is not null-terminated"),
-                    this->FileName);
+    if (ContentsOrErr->back() != 0) {
+      this->reportUniqueWarning(
+          createError("SHT_LLVM_LINKER_OPTIONS section at index " + Twine(I) +
+                      " is broken: the "
+                      "content is not null-terminated"));
       continue;
     }
 
     SmallVector<StringRef, 16> Strings;
-    toStringRef(Contents.drop_back()).split(Strings, '\0');
+    toStringRef(ContentsOrErr->drop_back()).split(Strings, '\0');
     if (Strings.size() % 2 != 0) {
-      reportWarning(
-          createError(
-              "SHT_LLVM_LINKER_OPTIONS section at index " + Twine(I) +
-              " is broken: an incomplete "
-              "key-value pair was found. The last possible key was: \"" +
-              Strings.back() + "\""),
-          this->FileName);
+      this->reportUniqueWarning(createError(
+          "SHT_LLVM_LINKER_OPTIONS section at index " + Twine(I) +
+          " is broken: an incomplete "
+          "key-value pair was found. The last possible key was: \"" +
+          Strings.back() + "\""));
       continue;
     }
 
diff --git a/llvm/tools/llvm-readobj/llvm-readobj.cpp b/llvm/tools/llvm-readobj/llvm-readobj.cpp
index 4c00b740de75d..b9c6ad2256ae2 100644
--- a/llvm/tools/llvm-readobj/llvm-readobj.cpp
+++ b/llvm/tools/llvm-readobj/llvm-readobj.cpp
@@ -528,6 +528,8 @@ static void dumpObject(const ObjectFile *Obj, ScopedPrinter &Writer,
       Dumper->printCOFFResources();
     if (opts::COFFLoadConfig)
       Dumper->printCOFFLoadConfig();
+    if (opts::CGProfile)
+      Dumper->printCGProfile();
     if (opts::Addrsig)
       Dumper->printAddrsig();
     if (opts::CodeView)
diff --git a/llvm/tools/llvm-reduce/CMakeLists.txt b/llvm/tools/llvm-reduce/CMakeLists.txt
index 48de0ffa78a10..01b9d0b4afe1a 100644
--- a/llvm/tools/llvm-reduce/CMakeLists.txt
+++ b/llvm/tools/llvm-reduce/CMakeLists.txt
@@ -11,15 +11,17 @@ set(LLVM_LINK_COMPONENTS
   )
 
 add_llvm_tool(llvm-reduce
-  llvm-reduce.cpp
   TestRunner.cpp
   deltas/Delta.cpp
-  deltas/ReduceFunctions.cpp
-  deltas/ReduceGlobalVars.cpp
-  deltas/ReduceMetadata.cpp
   deltas/ReduceArguments.cpp
+  deltas/ReduceAttributes.cpp
   deltas/ReduceBasicBlocks.cpp
+  deltas/ReduceFunctions.cpp
+  deltas/ReduceGlobalVars.cpp
   deltas/ReduceInstructions.cpp
+  deltas/ReduceMetadata.cpp
+  deltas/ReduceOperandBundles.cpp
+  llvm-reduce.cpp
 
   DEPENDS
   intrinsics_gen
diff --git a/llvm/tools/llvm-reduce/DeltaManager.h b/llvm/tools/llvm-reduce/DeltaManager.h
index 2309c3adf4e6c..b1a4ee0df4dbe 100644
--- a/llvm/tools/llvm-reduce/DeltaManager.h
+++ b/llvm/tools/llvm-reduce/DeltaManager.h
@@ -14,11 +14,13 @@
 #include "TestRunner.h"
 #include "deltas/Delta.h"
 #include "deltas/ReduceArguments.h"
+#include "deltas/ReduceAttributes.h"
 #include "deltas/ReduceBasicBlocks.h"
 #include "deltas/ReduceFunctions.h"
 #include "deltas/ReduceGlobalVars.h"
-#include "deltas/ReduceMetadata.h"
 #include "deltas/ReduceInstructions.h"
+#include "deltas/ReduceMetadata.h"
+#include "deltas/ReduceOperandBundles.h"
 
 namespace llvm {
 
@@ -30,6 +32,8 @@ inline void runDeltaPasses(TestRunner &Tester) {
   reduceMetadataDeltaPass(Tester);
   reduceArgumentsDeltaPass(Tester);
   reduceInstructionsDeltaPass(Tester);
+  reduceOperandBundesDeltaPass(Tester);
+  reduceAttributesDeltaPass(Tester);
   // TODO: Implement the remaining Delta Passes
 }
 
diff --git a/llvm/tools/llvm-reduce/deltas/Delta.h b/llvm/tools/llvm-reduce/deltas/Delta.h
index dbb18e4bd07fe..7da3c79c958eb 100644
--- a/llvm/tools/llvm-reduce/deltas/Delta.h
+++ b/llvm/tools/llvm-reduce/deltas/Delta.h
@@ -16,9 +16,10 @@
 #define LLVM_TOOLS_LLVMREDUCE_LLVMREDUCE_DELTA_H
 
 #include "TestRunner.h"
-#include <vector>
-#include <utility>
+#include "llvm/ADT/ScopeExit.h"
 #include <functional>
+#include <utility>
+#include <vector>
 
 namespace llvm {
 
@@ -47,6 +48,39 @@ struct Chunk {
   }
 };
 
+/// Provides opaque interface for querying into ChunksToKeep without having to
+/// actually understand what is going on.
+class Oracle {
+  /// Out of all the features that we promised to be,
+  /// how many have we already processed? 1-based!
+  int Index = 1;
+
+  /// The actual workhorse, contains the knowledge whether or not
+  /// some particular feature should be preserved this time.
+  ArrayRef<Chunk> ChunksToKeep;
+
+public:
+  explicit Oracle(ArrayRef<Chunk> ChunksToKeep_)
+      : ChunksToKeep(ChunksToKeep_) {}
+
+  /// Should be called for each feature on which we are operating.
+  /// Name is self-explanatory - if returns true, then it should be preserved.
+  bool shouldKeep() {
+    if (ChunksToKeep.empty())
+      return false; // All further features are to be discarded.
+
+    // Does the current (front) chunk contain such a feature?
+    bool ShouldKeep = ChunksToKeep.front().contains(Index);
+    auto _ = make_scope_exit([&]() { ++Index; }); // Next time - next feature.
+
+    // Is this the last feature in the chunk?
+    if (ChunksToKeep.front().end == Index)
+      ChunksToKeep = ChunksToKeep.drop_front(); // Onto next chunk.
+
+    return ShouldKeep;
+  }
+};
+
 /// This function implements the Delta Debugging algorithm, it receives a
 /// number of Targets (e.g. Functions, Instructions, Basic Blocks, etc.) and
 /// splits them in half; these chunks of targets are then tested while ignoring
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp b/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
index d5f60aad36bc8..a119b40018b34 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
@@ -42,7 +42,8 @@ static void replaceFunctionCalls(Function &OldF, Function &NewF,
 /// accordingly. It also removes allocations of out-of-chunk arguments.
 static void extractArgumentsFromModule(std::vector<Chunk> ChunksToKeep,
                                        Module *Program) {
-  int I = 0, ArgCount = 0;
+  Oracle O(ChunksToKeep);
+
   std::set<Argument *> ArgsToKeep;
   std::vector<Function *> Funcs;
   // Get inside-chunk arguments, as well as their parent function
@@ -50,17 +51,13 @@ static void extractArgumentsFromModule(std::vector<Chunk> ChunksToKeep,
     if (!F.isDeclaration()) {
       Funcs.push_back(&F);
       for (auto &A : F.args())
-        if (I < (int)ChunksToKeep.size()) {
-          if (ChunksToKeep[I].contains(++ArgCount))
-            ArgsToKeep.insert(&A);
-          if (ChunksToKeep[I].end == ArgCount)
-            ++I;
-        }
+        if (O.shouldKeep())
+          ArgsToKeep.insert(&A);
     }
 
   for (auto *F : Funcs) {
     ValueToValueMapTy VMap;
-    std::vector<Instruction *> InstToDelete;
+    std::vector<WeakVH> InstToDelete;
     for (auto &A : F->args())
       if (!ArgsToKeep.count(&A)) {
         // By adding undesired arguments to the VMap, CloneFunction will remove
@@ -70,8 +67,11 @@ static void extractArgumentsFromModule(std::vector<Chunk> ChunksToKeep,
           if (auto *I = dyn_cast<Instruction>(*&U))
             InstToDelete.push_back(I);
       }
-    // Delete any instruction that uses the argument
-    for (auto *I : InstToDelete) {
+    // Delete any (unique) instruction that uses the argument
+    for (Value *V : InstToDelete) {
+      if (!V)
+        continue;
+      auto *I = cast<Instruction>(V);
       I->replaceAllUsesWith(UndefValue::get(I->getType()));
       I->eraseFromParent();
     }
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceAttributes.cpp b/llvm/tools/llvm-reduce/deltas/ReduceAttributes.cpp
new file mode 100644
index 0000000000000..cbaf5d5efd346
--- /dev/null
+++ b/llvm/tools/llvm-reduce/deltas/ReduceAttributes.cpp
@@ -0,0 +1,200 @@
+//===- ReduceAttributes.cpp - Specialized Delta Pass -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a function which calls the Generic Delta pass in order
+// to reduce uninteresting attributes.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ReduceAttributes.h"
+#include "Delta.h"
+#include "TestRunner.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/InstVisitor.h"
+#include "llvm/IR/InstrTypes.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/raw_ostream.h"
+#include <algorithm>
+#include <cassert>
+#include <iterator>
+#include <utility>
+#include <vector>
+
+namespace llvm {
+class LLVMContext;
+} // namespace llvm
+
+using namespace llvm;
+
+namespace {
+
+using AttrPtrVecTy = std::vector<const Attribute *>;
+using AttrPtrIdxVecVecTy = std::pair<unsigned, AttrPtrVecTy>;
+using AttrPtrVecVecTy = SmallVector<AttrPtrIdxVecVecTy, 3>;
+
+/// Given ChunksToKeep, produce a map of global variables/functions/calls
+/// and indexes of attributes to be preserved for each of them.
+class AttributeRemapper : public InstVisitor<AttributeRemapper> {
+  Oracle O;
+
+public:
+  DenseMap<GlobalVariable *, AttrPtrVecTy> GlobalVariablesToRefine;
+  DenseMap<Function *, AttrPtrVecVecTy> FunctionsToRefine;
+  DenseMap<CallBase *, AttrPtrVecVecTy> CallsToRefine;
+
+  explicit AttributeRemapper(ArrayRef<Chunk> ChunksToKeep) : O(ChunksToKeep) {}
+
+  void visitModule(Module &M) {
+    for (GlobalVariable &GV : M.getGlobalList())
+      visitGlobalVariable(GV);
+  }
+
+  void visitGlobalVariable(GlobalVariable &GV) {
+    // Global variables only have one attribute set.
+    const AttributeSet &AS = GV.getAttributes();
+    if (AS.hasAttributes())
+      visitAttributeSet(AS, GlobalVariablesToRefine[&GV]);
+  }
+
+  void visitFunction(Function &F) {
+    if (F.getIntrinsicID() != Intrinsic::not_intrinsic)
+      return; // We can neither add nor remove attributes from intrinsics.
+    visitAttributeList(F.getAttributes(), FunctionsToRefine[&F]);
+  }
+
+  void visitCallBase(CallBase &I) {
+    visitAttributeList(I.getAttributes(), CallsToRefine[&I]);
+  }
+
+  void visitAttributeList(const AttributeList &AL,
+                          AttrPtrVecVecTy &AttributeSetsToPreserve) {
+    assert(AttributeSetsToPreserve.empty() && "Should not be sharing vectors.");
+    AttributeSetsToPreserve.reserve(AL.getNumAttrSets());
+    for (unsigned SetIdx : seq(AL.index_begin(), AL.index_end())) {
+      AttrPtrIdxVecVecTy AttributesToPreserve;
+      AttributesToPreserve.first = SetIdx;
+      visitAttributeSet(AL.getAttributes(AttributesToPreserve.first),
+                        AttributesToPreserve.second);
+      if (!AttributesToPreserve.second.empty())
+        AttributeSetsToPreserve.emplace_back(std::move(AttributesToPreserve));
+    }
+  }
+
+  void visitAttributeSet(const AttributeSet &AS,
+                         AttrPtrVecTy &AttrsToPreserve) {
+    assert(AttrsToPreserve.empty() && "Should not be sharing vectors.");
+    AttrsToPreserve.reserve(AS.getNumAttributes());
+    for (const Attribute &A : AS)
+      if (O.shouldKeep())
+        AttrsToPreserve.emplace_back(&A);
+  }
+};
+
+struct AttributeCounter : public InstVisitor<AttributeCounter> {
+  /// How many features (in this case, attributes) did we count, total?
+  int AttributeCount = 0;
+
+  void visitModule(Module &M) {
+    for (GlobalVariable &GV : M.getGlobalList())
+      visitGlobalVariable(GV);
+  }
+
+  void visitGlobalVariable(GlobalVariable &GV) {
+    // Global variables only have one attribute set.
+    visitAttributeSet(GV.getAttributes());
+  }
+
+  void visitFunction(Function &F) {
+    if (F.getIntrinsicID() != Intrinsic::not_intrinsic)
+      return; // We can neither add nor remove attributes from intrinsics.
+    visitAttributeList(F.getAttributes());
+  }
+
+  void visitCallBase(CallBase &I) { visitAttributeList(I.getAttributes()); }
+
+  void visitAttributeList(const AttributeList &AL) {
+    for (const AttributeSet &AS : AL)
+      visitAttributeSet(AS);
+  }
+
+  void visitAttributeSet(const AttributeSet &AS) {
+    AttributeCount += AS.getNumAttributes();
+  }
+};
+
+} // namespace
+
+AttributeSet
+convertAttributeRefToAttributeSet(LLVMContext &C,
+                                  ArrayRef<const Attribute *> Attributes) {
+  AttrBuilder B;
+  for (const Attribute *A : Attributes)
+    B.addAttribute(*A);
+  return AttributeSet::get(C, B);
+}
+
+AttributeList convertAttributeRefVecToAttributeList(
+    LLVMContext &C, ArrayRef<AttrPtrIdxVecVecTy> AttributeSets) {
+  std::vector<std::pair<unsigned, AttributeSet>> SetVec;
+  SetVec.reserve(AttributeSets.size());
+
+  transform(AttributeSets, std::back_inserter(SetVec),
+            [&C](const AttrPtrIdxVecVecTy &V) {
+              return std::make_pair(
+                  V.first, convertAttributeRefToAttributeSet(C, V.second));
+            });
+
+  sort(SetVec, [](const std::pair<unsigned, AttributeSet> &LHS,
+                  const std::pair<unsigned, AttributeSet> &RHS) {
+    return LHS.first < RHS.first; // All values are unique.
+  });
+
+  return AttributeList::get(C, SetVec);
+}
+
+/// Removes out-of-chunk attributes from module.
+static void extractAttributesFromModule(std::vector<Chunk> ChunksToKeep,
+                                        Module *Program) {
+  AttributeRemapper R(ChunksToKeep);
+  R.visit(Program);
+
+  LLVMContext &C = Program->getContext();
+  for (const auto &I : R.GlobalVariablesToRefine)
+    I.first->setAttributes(convertAttributeRefToAttributeSet(C, I.second));
+  for (const auto &I : R.FunctionsToRefine)
+    I.first->setAttributes(convertAttributeRefVecToAttributeList(C, I.second));
+  for (const auto &I : R.CallsToRefine)
+    I.first->setAttributes(convertAttributeRefVecToAttributeList(C, I.second));
+}
+
+/// Counts the amount of attributes.
+static int countAttributes(Module *Program) {
+  AttributeCounter C;
+
+  // TODO: Silence index with --quiet flag
+  outs() << "----------------------------\n";
+  C.visit(Program);
+  outs() << "Number of attributes: " << C.AttributeCount << "\n";
+
+  return C.AttributeCount;
+}
+
+void llvm::reduceAttributesDeltaPass(TestRunner &Test) {
+  outs() << "*** Reducing Attributes...\n";
+  int AttributeCount = countAttributes(Test.getProgram());
+  runDeltaPass(Test, AttributeCount, extractAttributesFromModule);
+}
diff --git a/libcxx/test/libcxx/selftest/exec.sh.cpp b/llvm/tools/llvm-reduce/deltas/ReduceAttributes.h
similarity index 54%
rename from libcxx/test/libcxx/selftest/exec.sh.cpp
rename to llvm/tools/llvm-reduce/deltas/ReduceAttributes.h
index 19c3d590978b6..f8deb045560ff 100644
--- a/libcxx/test/libcxx/selftest/exec.sh.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceAttributes.h
@@ -1,22 +1,20 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
+//===- ReduceAttributes.h - Specialized Delta Pass ------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// This file implements a function which calls the Generic Delta pass in order
+// to reduce uninteresting attributes.
+//
+//===----------------------------------------------------------------------===//
 
-// RUN: %{build}
-// RUN: %{exec} %t.exe "HELLO"
+namespace llvm {
 
-#include <cassert>
-#include <string>
+class TestRunner;
 
-int main(int argc, char** argv) {
-  assert(argc == 2);
+void reduceAttributesDeltaPass(TestRunner &Test);
 
-  std::string arg = argv[1];
-  assert(arg == "HELLO");
-  return 0;
-}
+} // namespace llvm
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceBasicBlocks.cpp b/llvm/tools/llvm-reduce/deltas/ReduceBasicBlocks.cpp
index 03c3962d2fd9d..002d81a674872 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceBasicBlocks.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceBasicBlocks.cpp
@@ -82,17 +82,14 @@ static void removeUninterestingBBsFromSwitch(SwitchInst &SwInst,
 /// accordingly. It also removes allocations of out-of-chunk arguments.
 static void extractBasicBlocksFromModule(std::vector<Chunk> ChunksToKeep,
                                          Module *Program) {
-  int I = 0, BBCount = 0;
+  Oracle O(ChunksToKeep);
+
   std::set<BasicBlock *> BBsToKeep;
 
   for (auto &F : *Program)
     for (auto &BB : F)
-      if (I < (int)ChunksToKeep.size()) {
-        if (ChunksToKeep[I].contains(++BBCount))
-          BBsToKeep.insert(&BB);
-        if (ChunksToKeep[I].end == BBCount)
-          ++I;
-      }
+      if (O.shouldKeep())
+        BBsToKeep.insert(&BB);
 
   std::vector<BasicBlock *> BBsToDelete;
   for (auto &F : *Program)
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceFunctions.cpp b/llvm/tools/llvm-reduce/deltas/ReduceFunctions.cpp
index 3382f35a945a4..b29df88261d91 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceFunctions.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceFunctions.cpp
@@ -15,6 +15,7 @@
 #include "ReduceFunctions.h"
 #include "Delta.h"
 #include "llvm/ADT/SetVector.h"
+#include "llvm/IR/Instructions.h"
 #include <set>
 
 using namespace llvm;
@@ -23,16 +24,13 @@ using namespace llvm;
 /// that aren't inside any of the desired Chunks.
 static void extractFunctionsFromModule(const std::vector<Chunk> &ChunksToKeep,
                                        Module *Program) {
+  Oracle O(ChunksToKeep);
+
   // Get functions inside desired chunks
   std::set<Function *> FuncsToKeep;
-  int I = 0, FunctionCount = 0;
   for (auto &F : *Program)
-    if (I < (int)ChunksToKeep.size()) {
-      if (ChunksToKeep[I].contains(++FunctionCount))
-        FuncsToKeep.insert(&F);
-      if (FunctionCount == ChunksToKeep[I].end)
-        ++I;
-    }
+    if (O.shouldKeep())
+      FuncsToKeep.insert(&F);
 
   // Delete out-of-chunk functions, and replace their calls with undef
   std::vector<Function *> FuncsToRemove;
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceGlobalVars.cpp b/llvm/tools/llvm-reduce/deltas/ReduceGlobalVars.cpp
index 5732208ee0a96..dc8df7395e1ff 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceGlobalVars.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceGlobalVars.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "ReduceGlobalVars.h"
+#include "llvm/IR/Constants.h"
 #include <set>
 
 using namespace llvm;
@@ -19,20 +20,17 @@ using namespace llvm;
 /// Removes all the Initialized GVs that aren't inside the desired Chunks.
 static void extractGVsFromModule(std::vector<Chunk> ChunksToKeep,
                                  Module *Program) {
+  Oracle O(ChunksToKeep);
+
   // Get GVs inside desired chunks
   std::set<GlobalVariable *> GVsToKeep;
-  int I = 0, GVCount = 0;
   for (auto &GV : Program->globals())
-    if (GV.hasInitializer() && I < (int)ChunksToKeep.size()) {
-      if (ChunksToKeep[I].contains(++GVCount))
-        GVsToKeep.insert(&GV);
-      if (GVCount == ChunksToKeep[I].end)
-        ++I;
-    }
+    if (GV.hasInitializer() && O.shouldKeep())
+      GVsToKeep.insert(&GV);
 
   // Delete out-of-chunk GVs and their uses
   std::vector<GlobalVariable *> ToRemove;
-  std::vector<Instruction *> InstToRemove;
+  std::vector<WeakVH> InstToRemove;
   for (auto &GV : Program->globals())
     if (GV.hasInitializer() && !GVsToKeep.count(&GV)) {
       for (auto U : GV.users())
@@ -43,8 +41,11 @@ static void extractGVsFromModule(std::vector<Chunk> ChunksToKeep,
       ToRemove.push_back(&GV);
     }
 
-  // Delete Instruction uses of unwanted GVs
-  for (auto *Inst : InstToRemove) {
+  // Delete (unique) Instruction uses of unwanted GVs
+  for (Value *V : InstToRemove) {
+    if (!V)
+      continue;
+    auto *Inst = cast<Instruction>(V);
     Inst->replaceAllUsesWith(UndefValue::get(Inst->getType()));
     Inst->eraseFromParent();
   }
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceInstructions.cpp b/llvm/tools/llvm-reduce/deltas/ReduceInstructions.cpp
index b3497ad2dc02d..18dec02b90adb 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceInstructions.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceInstructions.cpp
@@ -19,18 +19,15 @@ using namespace llvm;
 /// accordingly. It also removes allocations of out-of-chunk arguments.
 static void extractInstrFromModule(std::vector<Chunk> ChunksToKeep,
                                    Module *Program) {
-  int I = 0, InstCount = 0;
+  Oracle O(ChunksToKeep);
+
   std::set<Instruction *> InstToKeep;
 
   for (auto &F : *Program)
     for (auto &BB : F)
       for (auto &Inst : BB)
-        if (I < (int)ChunksToKeep.size()) {
-          if (ChunksToKeep[I].contains(++InstCount))
-            InstToKeep.insert(&Inst);
-          if (ChunksToKeep[I].end == InstCount)
-            ++I;
-        }
+        if (O.shouldKeep())
+          InstToKeep.insert(&Inst);
 
   std::vector<Instruction *> InstToDelete;
   for (auto &F : *Program)
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceMetadata.cpp b/llvm/tools/llvm-reduce/deltas/ReduceMetadata.cpp
index 4ea223546efa3..4587295a00bee 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceMetadata.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceMetadata.cpp
@@ -21,20 +21,15 @@ using namespace llvm;
 
 /// Adds all Unnamed Metadata Nodes that are inside desired Chunks to set
 template <class T>
-static void getChunkMetadataNodes(T &MDUser, int &I,
-                                  const std::vector<Chunk> &ChunksToKeep,
+static void getChunkMetadataNodes(T &MDUser, Oracle &O,
                                   std::set<MDNode *> &SeenNodes,
                                   std::set<MDNode *> &NodesToKeep) {
   SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
   MDUser.getAllMetadata(MDs);
   for (auto &MD : MDs) {
     SeenNodes.insert(MD.second);
-    if (I < (int)ChunksToKeep.size()) {
-      if (ChunksToKeep[I].contains(SeenNodes.size()))
-        NodesToKeep.insert(MD.second);
-      if (ChunksToKeep[I].end == (int)SeenNodes.size())
-        ++I;
-    }
+    if (O.shouldKeep())
+      NodesToKeep.insert(MD.second);
   }
 }
 
@@ -53,19 +48,20 @@ static void eraseMetadataIfOutsideChunk(T &MDUser,
 /// functions that aren't inside the desired Chunks.
 static void extractMetadataFromModule(const std::vector<Chunk> &ChunksToKeep,
                                       Module *Program) {
+  Oracle O(ChunksToKeep);
+
   std::set<MDNode *> SeenNodes;
   std::set<MDNode *> NodesToKeep;
-  int I = 0;
 
   // Add chunk MDNodes used by GVs, Functions, and Instructions to set
   for (auto &GV : Program->globals())
-    getChunkMetadataNodes(GV, I, ChunksToKeep, SeenNodes, NodesToKeep);
+    getChunkMetadataNodes(GV, O, SeenNodes, NodesToKeep);
 
   for (auto &F : *Program) {
-    getChunkMetadataNodes(F, I, ChunksToKeep, SeenNodes, NodesToKeep);
+    getChunkMetadataNodes(F, O, SeenNodes, NodesToKeep);
     for (auto &BB : F)
       for (auto &Inst : BB)
-        getChunkMetadataNodes(Inst, I, ChunksToKeep, SeenNodes, NodesToKeep);
+        getChunkMetadataNodes(Inst, O, SeenNodes, NodesToKeep);
   }
 
   // Once more, go over metadata nodes, but deleting the ones outside chunks
@@ -81,17 +77,10 @@ static void extractMetadataFromModule(const std::vector<Chunk> &ChunksToKeep,
 
 
   // Get out-of-chunk Named metadata nodes
-  unsigned MetadataCount = SeenNodes.size();
   std::vector<NamedMDNode *> NamedNodesToDelete;
-  for (auto &MD : Program->named_metadata()) {
-    if (I < (int)ChunksToKeep.size()) {
-      if (!ChunksToKeep[I].contains(++MetadataCount))
-        NamedNodesToDelete.push_back(&MD);
-      if (ChunksToKeep[I].end == (int)SeenNodes.size())
-        ++I;
-    } else
+  for (auto &MD : Program->named_metadata())
+    if (!O.shouldKeep())
       NamedNodesToDelete.push_back(&MD);
-  }
 
   for (auto *NN : NamedNodesToDelete) {
     for (int I = 0, E = NN->getNumOperands(); I != E; ++I)
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceOperandBundles.cpp b/llvm/tools/llvm-reduce/deltas/ReduceOperandBundles.cpp
new file mode 100644
index 0000000000000..77cb73837c827
--- /dev/null
+++ b/llvm/tools/llvm-reduce/deltas/ReduceOperandBundles.cpp
@@ -0,0 +1,124 @@
+//===- ReduceOperandBundes.cpp - Specialized Delta Pass -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a function which calls the Generic Delta pass in order
+// to reduce uninteresting operand bundes from calls.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ReduceOperandBundles.h"
+#include "Delta.h"
+#include "TestRunner.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/IR/InstVisitor.h"
+#include "llvm/IR/InstrTypes.h"
+#include "llvm/Support/raw_ostream.h"
+#include <algorithm>
+#include <iterator>
+#include <vector>
+
+namespace llvm {
+class Module;
+} // namespace llvm
+
+using namespace llvm;
+
+namespace {
+
+/// Given ChunksToKeep, produce a map of calls and indexes of operand bundles
+/// to be preserved for each call.
+class OperandBundleRemapper : public InstVisitor<OperandBundleRemapper> {
+  Oracle O;
+
+public:
+  DenseMap<CallBase *, std::vector<unsigned>> CallsToRefine;
+
+  explicit OperandBundleRemapper(ArrayRef<Chunk> ChunksToKeep)
+      : O(ChunksToKeep) {}
+
+  /// So far only CallBase sub-classes can have operand bundles.
+  /// Let's see which of the operand bundles of this call are to be kept.
+  void visitCallBase(CallBase &Call) {
+    if (!Call.hasOperandBundles())
+      return; // No bundles to begin with.
+
+    // Insert this call into map, we will likely want to rebuild it.
+    auto &OperandBundlesToKeepIndexes = CallsToRefine[&Call];
+    OperandBundlesToKeepIndexes.reserve(Call.getNumOperandBundles());
+
+    // Enumerate every operand bundle on this call.
+    for (unsigned BundleIndex : seq(0U, Call.getNumOperandBundles()))
+      if (O.shouldKeep()) // Should we keep this one?
+        OperandBundlesToKeepIndexes.emplace_back(BundleIndex);
+  }
+};
+
+struct OperandBundleCounter : public InstVisitor<OperandBundleCounter> {
+  /// How many features (in this case, operand bundles) did we count, total?
+  int OperandBundeCount = 0;
+
+  /// So far only CallBase sub-classes can have operand bundles.
+  void visitCallBase(CallBase &Call) {
+    // Just accumulate the total number of operand bundles.
+    OperandBundeCount += Call.getNumOperandBundles();
+  }
+};
+
+} // namespace
+
+static void maybeRewriteCallWithDifferentBundles(
+    CallBase *OrigCall, ArrayRef<unsigned> OperandBundlesToKeepIndexes) {
+  if (OperandBundlesToKeepIndexes.size() == OrigCall->getNumOperandBundles())
+    return; // Not modifying operand bundles of this call after all.
+
+  std::vector<OperandBundleDef> NewBundles;
+  NewBundles.reserve(OperandBundlesToKeepIndexes.size());
+
+  // Actually copy over the bundles that we want to keep.
+  transform(OperandBundlesToKeepIndexes, std::back_inserter(NewBundles),
+            [OrigCall](unsigned Index) {
+              return OperandBundleDef(OrigCall->getOperandBundleAt(Index));
+            });
+
+  // Finally actually replace the bundles on the call.
+  CallBase *NewCall = CallBase::Create(OrigCall, NewBundles, OrigCall);
+  OrigCall->replaceAllUsesWith(NewCall);
+  OrigCall->eraseFromParent();
+}
+
+/// Removes out-of-chunk operand bundles from calls.
+static void extractOperandBundesFromModule(std::vector<Chunk> ChunksToKeep,
+                                           Module *Program) {
+  OperandBundleRemapper R(ChunksToKeep);
+  R.visit(Program);
+
+  for (const auto &I : R.CallsToRefine)
+    maybeRewriteCallWithDifferentBundles(I.first, I.second);
+}
+
+/// Counts the amount of operand bundles.
+static int countOperandBundes(Module *Program) {
+  OperandBundleCounter C;
+
+  // TODO: Silence index with --quiet flag
+  outs() << "----------------------------\n";
+  C.visit(Program);
+  outs() << "Number of operand bundles: " << C.OperandBundeCount << "\n";
+
+  return C.OperandBundeCount;
+}
+
+void llvm::reduceOperandBundesDeltaPass(TestRunner &Test) {
+  outs() << "*** Reducing OperandBundes...\n";
+  int OperandBundeCount = countOperandBundes(Test.getProgram());
+  runDeltaPass(Test, OperandBundeCount, extractOperandBundesFromModule);
+}
diff --git a/libcxx/test/libcxx/selftest/test.arc.pass.mm b/llvm/tools/llvm-reduce/deltas/ReduceOperandBundles.h
similarity index 53%
rename from libcxx/test/libcxx/selftest/test.arc.pass.mm
rename to llvm/tools/llvm-reduce/deltas/ReduceOperandBundles.h
index b70b24e631836..382c5cb5691d5 100644
--- a/libcxx/test/libcxx/selftest/test.arc.pass.mm
+++ b/llvm/tools/llvm-reduce/deltas/ReduceOperandBundles.h
@@ -1,17 +1,20 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
+//===- ReduceOperandBundes.h - Specialized Delta Pass ---------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// This file implements a function which calls the Generic Delta pass in order
+// to reduce uninteresting operand bundes from calls.
+//
+//===----------------------------------------------------------------------===//
+
+namespace llvm {
 
-// REQUIRES: has-fobjc-arc
-// ADDITIONAL_COMPILE_FLAGS: -fobjc-arc
+class TestRunner;
 
-#if __has_feature(objc_arc) == 0
-#error "arc should be enabled"
-#endif
+void reduceOperandBundesDeltaPass(TestRunner &Test);
 
-int main(int, char**) { return 0; }
+} // namespace llvm
diff --git a/llvm/tools/llvm-shlib/CMakeLists.txt b/llvm/tools/llvm-shlib/CMakeLists.txt
index 25803da0c2520..f3a2056f80d36 100644
--- a/llvm/tools/llvm-shlib/CMakeLists.txt
+++ b/llvm/tools/llvm-shlib/CMakeLists.txt
@@ -154,13 +154,17 @@ if(LLVM_BUILD_LLVM_C_DYLIB AND MSVC)
   set(GEN_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/gen-msvc-exports.py)
 
   set(LLVM_EXPORTED_SYMBOL_FILE ${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/libllvm-c.exports)
-
-  if(CMAKE_CROSSCOMPILING)
-    build_native_tool(llvm-nm llvm_nm)
-    set(llvm_nm_target "${llvm_nm}")
+  if(NOT LLVM_NM)
+    if(CMAKE_CROSSCOMPILING)
+      build_native_tool(llvm-nm llvm_nm)
+      set(llvm_nm_target "${llvm_nm}")
+    else()
+      set(llvm_nm $<TARGET_FILE:llvm-nm>)
+      set(llvm_nm_target llvm-nm)
+    endif()
   else()
-    set(llvm_nm $<TARGET_FILE:llvm-nm>)
-    set(llvm_nm_target llvm-nm)
+    set(llvm_nm ${LLVM_NM})
+    set(llvm_nm_target "")
   endif()
 
   add_custom_command(OUTPUT ${LLVM_EXPORTED_SYMBOL_FILE}
diff --git a/llvm/tools/llvm-size/llvm-size.cpp b/llvm/tools/llvm-size/llvm-size.cpp
index b79622823feac..987270e98c483 100644
--- a/llvm/tools/llvm-size/llvm-size.cpp
+++ b/llvm/tools/llvm-size/llvm-size.cpp
@@ -187,10 +187,11 @@ static bool considerForSize(ObjectFile *Obj, SectionRef Section) {
   switch (static_cast<ELFSectionRef>(Section).getType()) {
   case ELF::SHT_NULL:
   case ELF::SHT_SYMTAB:
+    return false;
   case ELF::SHT_STRTAB:
   case ELF::SHT_REL:
   case ELF::SHT_RELA:
-    return false;
+    return static_cast<ELFSectionRef>(Section).getFlags() & ELF::SHF_ALLOC;
   }
   return true;
 }
diff --git a/llvm/tools/lto/lto.cpp b/llvm/tools/lto/lto.cpp
index d48580606c4b2..ea44430f66a70 100644
--- a/llvm/tools/lto/lto.cpp
+++ b/llvm/tools/lto/lto.cpp
@@ -474,17 +474,20 @@ bool lto_codegen_compile_to_file(lto_code_gen_t cg, const char **name) {
 }
 
 void lto_codegen_debug_options(lto_code_gen_t cg, const char *opt) {
-  std::vector<const char *> Options;
+  SmallVector<StringRef, 4> Options;
   for (std::pair<StringRef, StringRef> o = getToken(opt); !o.first.empty();
        o = getToken(o.second))
-    Options.push_back(o.first.data());
+    Options.push_back(o.first);
 
   unwrap(cg)->setCodeGenDebugOptions(Options);
 }
 
 void lto_codegen_debug_options_array(lto_code_gen_t cg,
                                      const char *const *options, int number) {
-  unwrap(cg)->setCodeGenDebugOptions(makeArrayRef(options, number));
+  SmallVector<StringRef, 4> Options;
+  for (int i = 0; i < number; ++i)
+    Options.push_back(options[i]);
+  unwrap(cg)->setCodeGenDebugOptions(makeArrayRef(Options));
 }
 
 unsigned int lto_api_version() { return LTO_API_VERSION; }
diff --git a/llvm/tools/obj2yaml/dwarf2yaml.cpp b/llvm/tools/obj2yaml/dwarf2yaml.cpp
index 88f53c18f30b1..a318a547e32c5 100644
--- a/llvm/tools/obj2yaml/dwarf2yaml.cpp
+++ b/llvm/tools/obj2yaml/dwarf2yaml.cpp
@@ -122,7 +122,7 @@ static DWARFYAML::PubSection dumpPubSection(const DWARFContext &DCtx,
                                             bool IsGNUStyle) {
   DWARFDataExtractor PubSectionData(DCtx.getDWARFObj(), Section,
                                     DCtx.isLittleEndian(), 0);
-  DWARFYAML::PubSection Y(IsGNUStyle);
+  DWARFYAML::PubSection Y;
   uint64_t Offset = 0;
   dumpInitialLength(PubSectionData, Offset, Y.Length);
   Y.Version = PubSectionData.getU16(&Offset);
@@ -165,7 +165,8 @@ void dumpDebugPubSections(DWARFContext &DCtx, DWARFYAML::Data &Y) {
 void dumpDebugInfo(DWARFContext &DCtx, DWARFYAML::Data &Y) {
   for (const auto &CU : DCtx.compile_units()) {
     DWARFYAML::Unit NewUnit;
-    NewUnit.Length.setLength(CU->getLength());
+    NewUnit.Format = CU->getFormat();
+    NewUnit.Length = CU->getLength();
     NewUnit.Version = CU->getVersion();
     if(NewUnit.Version >= 5)
       NewUnit.Type = (dwarf::UnitType)CU->getUnitType();
diff --git a/llvm/tools/obj2yaml/elf2yaml.cpp b/llvm/tools/obj2yaml/elf2yaml.cpp
index 8602477a3a4a1..868a78e15c663 100644
--- a/llvm/tools/obj2yaml/elf2yaml.cpp
+++ b/llvm/tools/obj2yaml/elf2yaml.cpp
@@ -9,6 +9,7 @@
 #include "Error.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/ObjectYAML/ELFYAML.h"
 #include "llvm/Support/DataExtractor.h"
diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp
index ce86c3d615846..8f8ca352dcfff 100644
--- a/llvm/tools/opt/NewPMDriver.cpp
+++ b/llvm/tools/opt/NewPMDriver.cpp
@@ -14,6 +14,7 @@
 
 #include "NewPMDriver.h"
 #include "PassPrinters.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/CGSCCPassManager.h"
@@ -101,9 +102,7 @@ static cl::opt<std::string> OptimizerLastEPPipeline(
     cl::Hidden);
 
 // Individual pipeline tuning options.
-static cl::opt<bool> DisableLoopUnrolling(
-    "new-pm-disable-loop-unrolling",
-    cl::desc("Disable loop unrolling in all relevant passes"), cl::init(false));
+extern cl::opt<bool> DisableLoopUnrolling;
 
 extern cl::opt<PGOKind> PGOKindFlag;
 extern cl::opt<std::string> ProfileFile;
@@ -214,8 +213,8 @@ static void registerEPCallbacks(PassBuilder &PB, bool VerifyEachPass,
 bool llvm::runPassPipeline(StringRef Arg0, Module &M, TargetMachine *TM,
                            ToolOutputFile *Out, ToolOutputFile *ThinLTOLinkOut,
                            ToolOutputFile *OptRemarkFile,
-                           StringRef PassPipeline, OutputKind OK,
-                           VerifierKind VK,
+                           StringRef PassPipeline, ArrayRef<StringRef> Passes,
+                           OutputKind OK, VerifierKind VK,
                            bool ShouldPreserveAssemblyUseListOrder,
                            bool ShouldPreserveBitcodeUseListOrder,
                            bool EmitSummaryIndex, bool EmitModuleHash,
@@ -306,9 +305,26 @@ bool llvm::runPassPipeline(StringRef Arg0, Module &M, TargetMachine *TM,
   // Specially handle the alias analysis manager so that we can register
   // a custom pipeline of AA passes with it.
   AAManager AA;
-  if (auto Err = PB.parseAAPipeline(AA, AAPipeline)) {
-    errs() << Arg0 << ": " << toString(std::move(Err)) << "\n";
-    return false;
+  if (!AAPipeline.empty()) {
+    assert(Passes.empty() &&
+           "--aa-pipeline and -foo-pass should not both be specified");
+    if (auto Err = PB.parseAAPipeline(AA, AAPipeline)) {
+      errs() << Arg0 << ": " << toString(std::move(Err)) << "\n";
+      return false;
+    }
+  }
+  // For compatibility with legacy pass manager.
+  // Alias analyses are not specially specified when using the legacy PM.
+  SmallVector<StringRef, 4> NonAAPasses;
+  for (auto PassName : Passes) {
+    if (PB.isAAPassName(PassName)) {
+      if (auto Err = PB.parseAAPipeline(AA, PassName)) {
+        errs() << Arg0 << ": " << toString(std::move(Err)) << "\n";
+        return false;
+      }
+    } else {
+      NonAAPasses.push_back(PassName);
+    }
   }
 
   LoopAnalysisManager LAM(DebugPM);
@@ -332,10 +348,21 @@ bool llvm::runPassPipeline(StringRef Arg0, Module &M, TargetMachine *TM,
   if (EnableDebugify)
     MPM.addPass(NewPMDebugifyPass());
 
-  if (auto Err =
-          PB.parsePassPipeline(MPM, PassPipeline, VerifyEachPass, DebugPM)) {
-    errs() << Arg0 << ": " << toString(std::move(Err)) << "\n";
-    return false;
+  if (!PassPipeline.empty()) {
+    assert(Passes.empty() &&
+           "PassPipeline and Passes should not both contain passes");
+    if (auto Err =
+            PB.parsePassPipeline(MPM, PassPipeline, VerifyEachPass, DebugPM)) {
+      errs() << Arg0 << ": " << toString(std::move(Err)) << "\n";
+      return false;
+    }
+  }
+  for (auto PassName : NonAAPasses) {
+    if (auto Err =
+            PB.parsePassPipeline(MPM, PassName, VerifyEachPass, DebugPM)) {
+      errs() << Arg0 << ": " << toString(std::move(Err)) << "\n";
+      return false;
+    }
   }
 
   if (VK > VK_NoVerifier)
diff --git a/llvm/tools/opt/NewPMDriver.h b/llvm/tools/opt/NewPMDriver.h
index 707522cec4b38..7ae273a2c1f46 100644
--- a/llvm/tools/opt/NewPMDriver.h
+++ b/llvm/tools/opt/NewPMDriver.h
@@ -20,6 +20,8 @@
 #ifndef LLVM_TOOLS_OPT_NEWPMDRIVER_H
 #define LLVM_TOOLS_OPT_NEWPMDRIVER_H
 
+#include "llvm/ADT/ArrayRef.h"
+
 namespace llvm {
 class StringRef;
 class Module;
@@ -59,7 +61,8 @@ enum CSPGOKind { NoCSPGO, CSInstrGen, CSInstrUse };
 bool runPassPipeline(StringRef Arg0, Module &M, TargetMachine *TM,
                      ToolOutputFile *Out, ToolOutputFile *ThinLinkOut,
                      ToolOutputFile *OptRemarkFile, StringRef PassPipeline,
-                     opt_tool::OutputKind OK, opt_tool::VerifierKind VK,
+                     ArrayRef<StringRef> PassInfos, opt_tool::OutputKind OK,
+                     opt_tool::VerifierKind VK,
                      bool ShouldPreserveAssemblyUseListOrder,
                      bool ShouldPreserveBitcodeUseListOrder,
                      bool EmitSummaryIndex, bool EmitModuleHash,
diff --git a/llvm/tools/opt/PassPrinters.cpp b/llvm/tools/opt/PassPrinters.cpp
index ed4fc1a8174bc..4e81b5d29c4d6 100644
--- a/llvm/tools/opt/PassPrinters.cpp
+++ b/llvm/tools/opt/PassPrinters.cpp
@@ -33,18 +33,16 @@ struct FunctionPassPrinter : public FunctionPass {
   raw_ostream &Out;
   static char ID;
   std::string PassName;
-  bool QuietPass;
 
-  FunctionPassPrinter(const PassInfo *PI, raw_ostream &out, bool Quiet)
-      : FunctionPass(ID), PassToPrint(PI), Out(out), QuietPass(Quiet) {
+  FunctionPassPrinter(const PassInfo *PI, raw_ostream &out)
+      : FunctionPass(ID), PassToPrint(PI), Out(out) {
     std::string PassToPrintName = std::string(PassToPrint->getPassName());
     PassName = "FunctionPass Printer: " + PassToPrintName;
   }
 
   bool runOnFunction(Function &F) override {
-    if (!QuietPass)
-      Out << "Printing analysis '" << PassToPrint->getPassName()
-          << "' for function '" << F.getName() << "':\n";
+    Out << "Printing analysis '" << PassToPrint->getPassName()
+        << "' for function '" << F.getName() << "':\n";
 
     // Get and print pass...
     getAnalysisID<Pass>(PassToPrint->getTypeInfo()).print(Out, F.getParent());
@@ -66,17 +64,15 @@ struct CallGraphSCCPassPrinter : public CallGraphSCCPass {
   const PassInfo *PassToPrint;
   raw_ostream &Out;
   std::string PassName;
-  bool QuietPass;
 
-  CallGraphSCCPassPrinter(const PassInfo *PI, raw_ostream &out, bool Quiet)
-      : CallGraphSCCPass(ID), PassToPrint(PI), Out(out), QuietPass(Quiet) {
+  CallGraphSCCPassPrinter(const PassInfo *PI, raw_ostream &out)
+      : CallGraphSCCPass(ID), PassToPrint(PI), Out(out) {
     std::string PassToPrintName = std::string(PassToPrint->getPassName());
     PassName = "CallGraphSCCPass Printer: " + PassToPrintName;
   }
 
   bool runOnSCC(CallGraphSCC &SCC) override {
-    if (!QuietPass)
-      Out << "Printing analysis '" << PassToPrint->getPassName() << "':\n";
+    Out << "Printing analysis '" << PassToPrint->getPassName() << "':\n";
 
     // Get and print pass...
     for (CallGraphSCC::iterator I = SCC.begin(), E = SCC.end(); I != E; ++I) {
@@ -103,17 +99,15 @@ struct ModulePassPrinter : public ModulePass {
   const PassInfo *PassToPrint;
   raw_ostream &Out;
   std::string PassName;
-  bool QuietPass;
 
-  ModulePassPrinter(const PassInfo *PI, raw_ostream &out, bool Quiet)
-      : ModulePass(ID), PassToPrint(PI), Out(out), QuietPass(Quiet) {
+  ModulePassPrinter(const PassInfo *PI, raw_ostream &out)
+      : ModulePass(ID), PassToPrint(PI), Out(out) {
     std::string PassToPrintName = std::string(PassToPrint->getPassName());
     PassName = "ModulePass Printer: " + PassToPrintName;
   }
 
   bool runOnModule(Module &M) override {
-    if (!QuietPass)
-      Out << "Printing analysis '" << PassToPrint->getPassName() << "':\n";
+    Out << "Printing analysis '" << PassToPrint->getPassName() << "':\n";
 
     // Get and print pass...
     getAnalysisID<Pass>(PassToPrint->getTypeInfo()).print(Out, &M);
@@ -135,17 +129,15 @@ struct LoopPassPrinter : public LoopPass {
   const PassInfo *PassToPrint;
   raw_ostream &Out;
   std::string PassName;
-  bool QuietPass;
 
-  LoopPassPrinter(const PassInfo *PI, raw_ostream &out, bool Quiet)
-      : LoopPass(ID), PassToPrint(PI), Out(out), QuietPass(Quiet) {
+  LoopPassPrinter(const PassInfo *PI, raw_ostream &out)
+      : LoopPass(ID), PassToPrint(PI), Out(out) {
     std::string PassToPrintName = std::string(PassToPrint->getPassName());
     PassName = "LoopPass Printer: " + PassToPrintName;
   }
 
   bool runOnLoop(Loop *L, LPPassManager &LPM) override {
-    if (!QuietPass)
-      Out << "Printing analysis '" << PassToPrint->getPassName() << "':\n";
+    Out << "Printing analysis '" << PassToPrint->getPassName() << "':\n";
 
     // Get and print pass...
     getAnalysisID<Pass>(PassToPrint->getTypeInfo())
@@ -168,20 +160,17 @@ struct RegionPassPrinter : public RegionPass {
   const PassInfo *PassToPrint;
   raw_ostream &Out;
   std::string PassName;
-  bool QuietPass;
 
-  RegionPassPrinter(const PassInfo *PI, raw_ostream &out, bool Quiet)
-      : RegionPass(ID), PassToPrint(PI), Out(out), QuietPass(Quiet) {
+  RegionPassPrinter(const PassInfo *PI, raw_ostream &out)
+      : RegionPass(ID), PassToPrint(PI), Out(out) {
     std::string PassToPrintName = std::string(PassToPrint->getPassName());
     PassName = "RegionPass Printer: " + PassToPrintName;
   }
 
   bool runOnRegion(Region *R, RGPassManager &RGM) override {
-    if (!QuietPass) {
-      Out << "Printing analysis '" << PassToPrint->getPassName() << "' for "
-          << "region: '" << R->getNameStr() << "' in function '"
-          << R->getEntry()->getParent()->getName() << "':\n";
-    }
+    Out << "Printing analysis '" << PassToPrint->getPassName() << "' for "
+        << "region: '" << R->getNameStr() << "' in function '"
+        << R->getEntry()->getParent()->getName() << "':\n";
     // Get and print pass...
     getAnalysisID<Pass>(PassToPrint->getTypeInfo())
         .print(Out, R->getEntry()->getParent()->getParent());
@@ -201,28 +190,23 @@ char RegionPassPrinter::ID = 0;
 } // end anonymous namespace
 
 FunctionPass *llvm::createFunctionPassPrinter(const PassInfo *PI,
-                                              raw_ostream &OS, bool Quiet) {
-  return new FunctionPassPrinter(PI, OS, Quiet);
+                                              raw_ostream &OS) {
+  return new FunctionPassPrinter(PI, OS);
 }
 
 CallGraphSCCPass *llvm::createCallGraphPassPrinter(const PassInfo *PI,
-                                                   raw_ostream &OS,
-                                                   bool Quiet) {
-  return new CallGraphSCCPassPrinter(PI, OS, Quiet);
+                                                   raw_ostream &OS) {
+  return new CallGraphSCCPassPrinter(PI, OS);
 }
 
-ModulePass *llvm::createModulePassPrinter(const PassInfo *PI, raw_ostream &OS,
-                                          bool Quiet) {
-  return new ModulePassPrinter(PI, OS, Quiet);
+ModulePass *llvm::createModulePassPrinter(const PassInfo *PI, raw_ostream &OS) {
+  return new ModulePassPrinter(PI, OS);
 }
 
-LoopPass *llvm::createLoopPassPrinter(const PassInfo *PI, raw_ostream &OS,
-                                      bool Quiet) {
-  return new LoopPassPrinter(PI, OS, Quiet);
+LoopPass *llvm::createLoopPassPrinter(const PassInfo *PI, raw_ostream &OS) {
+  return new LoopPassPrinter(PI, OS);
 }
 
-RegionPass *llvm::createRegionPassPrinter(const PassInfo *PI, raw_ostream &OS,
-                                          bool Quiet) {
-  return new RegionPassPrinter(PI, OS, Quiet);
+RegionPass *llvm::createRegionPassPrinter(const PassInfo *PI, raw_ostream &OS) {
+  return new RegionPassPrinter(PI, OS);
 }
-
diff --git a/llvm/tools/opt/PassPrinters.h b/llvm/tools/opt/PassPrinters.h
index 9342c46f2ff66..a4e1921399fc6 100644
--- a/llvm/tools/opt/PassPrinters.h
+++ b/llvm/tools/opt/PassPrinters.h
@@ -24,20 +24,16 @@ class PassInfo;
 class raw_ostream;
 class RegionPass;
 
-FunctionPass *createFunctionPassPrinter(const PassInfo *PI, raw_ostream &out,
-                                        bool Quiet);
+FunctionPass *createFunctionPassPrinter(const PassInfo *PI, raw_ostream &out);
 
 CallGraphSCCPass *createCallGraphPassPrinter(const PassInfo *PI,
-                                             raw_ostream &out, bool Quiet);
+                                             raw_ostream &out);
 
-ModulePass *createModulePassPrinter(const PassInfo *PI, raw_ostream &out,
-                                    bool Quiet);
+ModulePass *createModulePassPrinter(const PassInfo *PI, raw_ostream &out);
 
-LoopPass *createLoopPassPrinter(const PassInfo *PI, raw_ostream &out,
-                                bool Quiet);
+LoopPass *createLoopPassPrinter(const PassInfo *PI, raw_ostream &out);
 
-RegionPass *createRegionPassPrinter(const PassInfo *PI, raw_ostream &out,
-                                    bool Quiet);
+RegionPass *createRegionPassPrinter(const PassInfo *PI, raw_ostream &out);
 
 } // end namespace llvm
 
diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp
index 0a1a5110bf2de..c250eefb8c430 100644
--- a/llvm/tools/opt/opt.cpp
+++ b/llvm/tools/opt/opt.cpp
@@ -71,6 +71,9 @@ static codegen::RegisterCodeGenFlags CFG;
 static cl::list<const PassInfo*, bool, PassNameParser>
 PassList(cl::desc("Optimizations available:"));
 
+static cl::opt<bool> EnableNewPassManager(
+    "enable-new-pm", cl::desc("Enable the new pass manager"), cl::init(false));
+
 // This flag specifies a textual description of the optimization pass pipeline
 // to run over the module. This flag switches opt to use the new pass manager
 // infrastructure, completely disabling all of the flags specific to the old
@@ -180,10 +183,9 @@ CodeGenOptLevel("codegen-opt-level",
 static cl::opt<std::string>
 TargetTriple("mtriple", cl::desc("Override target triple for module"));
 
-static cl::opt<bool>
-DisableLoopUnrolling("disable-loop-unrolling",
-                     cl::desc("Disable loop unrolling in all relevant passes"),
-                     cl::init(false));
+cl::opt<bool> DisableLoopUnrolling(
+    "disable-loop-unrolling",
+    cl::desc("Disable loop unrolling in all relevant passes"), cl::init(false));
 
 static cl::opt<bool> EmitSummaryIndex("module-summary",
                                       cl::desc("Emit module summary index"),
@@ -201,13 +203,6 @@ DisableBuiltins("disable-builtin",
                 cl::desc("Disable specific target library builtin function"),
                 cl::ZeroOrMore);
 
-
-static cl::opt<bool>
-Quiet("q", cl::desc("Obsolete option"), cl::Hidden);
-
-static cl::alias
-QuietA("quiet", cl::desc("Alias for -q"), cl::aliasopt(Quiet));
-
 static cl::opt<bool>
 AnalyzeOnly("analyze", cl::desc("Only perform analysis, no optimization"));
 
@@ -728,13 +723,34 @@ int main(int argc, char **argv) {
   // console, print out a warning message and refuse to do it.  We don't
   // impress anyone by spewing tons of binary goo to a terminal.
   if (!Force && !NoOutput && !AnalyzeOnly && !OutputAssembly)
-    if (CheckBitcodeOutputToConsole(Out->os(), !Quiet))
+    if (CheckBitcodeOutputToConsole(Out->os()))
       NoOutput = true;
 
   if (OutputThinLTOBC)
     M->addModuleFlag(Module::Error, "EnableSplitLTOUnit", SplitLTOUnit);
 
-  if (PassPipeline.getNumOccurrences() > 0) {
+  if (EnableNewPassManager || PassPipeline.getNumOccurrences() > 0) {
+    if (PassPipeline.getNumOccurrences() > 0 && PassList.size() > 0) {
+      errs()
+          << "Cannot specify passes via both -foo-pass and --passes=foo-pass";
+      return 1;
+    }
+    SmallVector<StringRef, 4> Passes;
+    for (const auto &P : PassList) {
+      Passes.push_back(P->getPassArgument());
+    }
+    if (OptLevelO0)
+      Passes.push_back("default<O0>");
+    if (OptLevelO1)
+      Passes.push_back("default<O1>");
+    if (OptLevelO2)
+      Passes.push_back("default<O2>");
+    if (OptLevelO3)
+      Passes.push_back("default<O3>");
+    if (OptLevelOs)
+      Passes.push_back("default<Os>");
+    if (OptLevelOz)
+      Passes.push_back("default<Oz>");
     OutputKind OK = OK_NoOutput;
     if (!NoOutput)
       OK = OutputAssembly
@@ -751,7 +767,7 @@ int main(int argc, char **argv) {
     // string. Hand off the rest of the functionality to the new code for that
     // layer.
     return runPassPipeline(argv[0], *M, TM.get(), Out.get(), ThinLinkOut.get(),
-                           RemarksFile.get(), PassPipeline, OK, VK,
+                           RemarksFile.get(), PassPipeline, Passes, OK, VK,
                            PreserveAssemblyUseListOrder,
                            PreserveBitcodeUseListOrder, EmitSummaryIndex,
                            EmitModuleHash, EnableDebugify, Coroutines)
@@ -877,19 +893,19 @@ int main(int argc, char **argv) {
       if (AnalyzeOnly) {
         switch (Kind) {
         case PT_Region:
-          Passes.add(createRegionPassPrinter(PassInf, Out->os(), Quiet));
+          Passes.add(createRegionPassPrinter(PassInf, Out->os()));
           break;
         case PT_Loop:
-          Passes.add(createLoopPassPrinter(PassInf, Out->os(), Quiet));
+          Passes.add(createLoopPassPrinter(PassInf, Out->os()));
           break;
         case PT_Function:
-          Passes.add(createFunctionPassPrinter(PassInf, Out->os(), Quiet));
+          Passes.add(createFunctionPassPrinter(PassInf, Out->os()));
           break;
         case PT_CallGraphSCC:
-          Passes.add(createCallGraphPassPrinter(PassInf, Out->os(), Quiet));
+          Passes.add(createCallGraphPassPrinter(PassInf, Out->os()));
           break;
         default:
-          Passes.add(createModulePassPrinter(PassInf, Out->os(), Quiet));
+          Passes.add(createModulePassPrinter(PassInf, Out->os()));
           break;
         }
       }
diff --git a/llvm/tools/yaml2obj/yaml2obj.cpp b/llvm/tools/yaml2obj/yaml2obj.cpp
index 214416da0366c..d18b2c17512bb 100644
--- a/llvm/tools/yaml2obj/yaml2obj.cpp
+++ b/llvm/tools/yaml2obj/yaml2obj.cpp
@@ -75,10 +75,22 @@ static Optional<std::string> preprocess(StringRef Buf,
     if (Buf.startswith("[[")) {
       size_t I = Buf.find_first_of("[]", 2);
       if (Buf.substr(I).startswith("]]")) {
-        StringRef Macro = Buf.substr(2, I - 2);
+        StringRef MacroExpr = Buf.substr(2, I - 2);
+        StringRef Macro;
+        StringRef Default;
+        std::tie(Macro, Default) = MacroExpr.split('=');
+
+        // When the -D option is requested, we use the provided value.
+        // Otherwise we use a default macro value if present.
         auto It = Defines.find(Macro);
-        if (It != Defines.end()) {
-          Preprocessed += It->second;
+        Optional<StringRef> Value;
+        if (It != Defines.end())
+          Value = It->second;
+        else if (!Default.empty() || MacroExpr.endswith("="))
+          Value = Default;
+
+        if (Value) {
+          Preprocessed += *Value;
           Buf = Buf.substr(I + 2);
           continue;
         }
diff --git a/llvm/unittests/ADT/BitFieldsTest.cpp b/llvm/unittests/ADT/BitFieldsTest.cpp
new file mode 100644
index 0000000000000..3062d5d7f293c
--- /dev/null
+++ b/llvm/unittests/ADT/BitFieldsTest.cpp
@@ -0,0 +1,256 @@
+//===- llvm/unittests/ADT/BitFieldsTest.cpp - BitFields unit tests --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/Bitfields.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+
+TEST(BitfieldsTest, Example) {
+  uint8_t Storage = 0;
+
+  // Store and retrieve a single bit as bool.
+  using Bool = Bitfield::Element<bool, 0, 1>;
+  Bitfield::set<Bool>(Storage, true);
+  EXPECT_EQ(Storage, 0b00000001);
+  //                          ^
+  EXPECT_EQ(Bitfield::get<Bool>(Storage), true);
+
+  // Store and retrieve a 2 bit typed enum.
+  // Note: enum underlying type must be unsigned.
+  enum class SuitEnum : uint8_t { CLUBS, DIAMONDS, HEARTS, SPADES };
+  // Note: enum maximum value needs to be passed in as last parameter.
+  using Suit = Bitfield::Element<SuitEnum, 1, 2, SuitEnum::SPADES>;
+  Bitfield::set<Suit>(Storage, SuitEnum::HEARTS);
+  EXPECT_EQ(Storage, 0b00000101);
+  //                        ^^
+  EXPECT_EQ(Bitfield::get<Suit>(Storage), SuitEnum::HEARTS);
+
+  // Store and retrieve a 5 bit value as unsigned.
+  using Value = Bitfield::Element<unsigned, 3, 5>;
+  Bitfield::set<Value>(Storage, 10);
+  EXPECT_EQ(Storage, 0b01010101);
+  //                   ^^^^^
+  EXPECT_EQ(Bitfield::get<Value>(Storage), 10U);
+
+  // Interpret the same 5 bit value as signed.
+  using SignedValue = Bitfield::Element<int, 3, 5>;
+  Bitfield::set<SignedValue>(Storage, -2);
+  EXPECT_EQ(Storage, 0b11110101);
+  //                   ^^^^^
+  EXPECT_EQ(Bitfield::get<SignedValue>(Storage), -2);
+
+  // Ability to efficiently test if a field is non zero.
+  EXPECT_TRUE(Bitfield::test<Value>(Storage));
+
+  // Alter Storage changes value.
+  Storage = 0;
+  EXPECT_EQ(Bitfield::get<Bool>(Storage), false);
+  EXPECT_EQ(Bitfield::get<Suit>(Storage), SuitEnum::CLUBS);
+  EXPECT_EQ(Bitfield::get<Value>(Storage), 0U);
+  EXPECT_EQ(Bitfield::get<SignedValue>(Storage), 0);
+
+  Storage = 255;
+  EXPECT_EQ(Bitfield::get<Bool>(Storage), true);
+  EXPECT_EQ(Bitfield::get<Suit>(Storage), SuitEnum::SPADES);
+  EXPECT_EQ(Bitfield::get<Value>(Storage), 31U);
+  EXPECT_EQ(Bitfield::get<SignedValue>(Storage), -1);
+}
+
+TEST(BitfieldsTest, FirstBit) {
+  uint8_t Storage = 0;
+  using FirstBit = Bitfield::Element<bool, 0, 1>;
+  // Set true
+  Bitfield::set<FirstBit>(Storage, true);
+  EXPECT_EQ(Bitfield::get<FirstBit>(Storage), true);
+  EXPECT_EQ(Storage, 0x1ULL);
+  // Set false
+  Bitfield::set<FirstBit>(Storage, false);
+  EXPECT_EQ(Bitfield::get<FirstBit>(Storage), false);
+  EXPECT_EQ(Storage, 0x0ULL);
+}
+
+TEST(BitfieldsTest, SecondBit) {
+  uint8_t Storage = 0;
+  using SecondBit = Bitfield::Element<bool, 1, 1>;
+  // Set true
+  Bitfield::set<SecondBit>(Storage, true);
+  EXPECT_EQ(Bitfield::get<SecondBit>(Storage), true);
+  EXPECT_EQ(Storage, 0x2ULL);
+  // Set false
+  Bitfield::set<SecondBit>(Storage, false);
+  EXPECT_EQ(Bitfield::get<SecondBit>(Storage), false);
+  EXPECT_EQ(Storage, 0x0ULL);
+}
+
+TEST(BitfieldsTest, LastBit) {
+  uint8_t Storage = 0;
+  using LastBit = Bitfield::Element<bool, 7, 1>;
+  // Set true
+  Bitfield::set<LastBit>(Storage, true);
+  EXPECT_EQ(Bitfield::get<LastBit>(Storage), true);
+  EXPECT_EQ(Storage, 0x80ULL);
+  // Set false
+  Bitfield::set<LastBit>(Storage, false);
+  EXPECT_EQ(Bitfield::get<LastBit>(Storage), false);
+  EXPECT_EQ(Storage, 0x0ULL);
+}
+
+TEST(BitfieldsTest, LastBitUint64) {
+  uint64_t Storage = 0;
+  using LastBit = Bitfield::Element<bool, 63, 1>;
+  // Set true
+  Bitfield::set<LastBit>(Storage, true);
+  EXPECT_EQ(Bitfield::get<LastBit>(Storage), true);
+  EXPECT_EQ(Storage, 0x8000000000000000ULL);
+  // Set false
+  Bitfield::set<LastBit>(Storage, false);
+  EXPECT_EQ(Bitfield::get<LastBit>(Storage), false);
+  EXPECT_EQ(Storage, 0x0ULL);
+}
+
+TEST(BitfieldsTest, Enum) {
+  enum Enum : unsigned { Zero = 0, Two = 2, LAST = Two };
+
+  uint8_t Storage = 0;
+  using OrderingField = Bitfield::Element<Enum, 1, 2, LAST>;
+  EXPECT_EQ(Bitfield::get<OrderingField>(Storage), Zero);
+  Bitfield::set<OrderingField>(Storage, Two);
+  EXPECT_EQ(Bitfield::get<OrderingField>(Storage), Two);
+  EXPECT_EQ(Storage, 0b00000100);
+  // value 2 in             ^^
+}
+
+TEST(BitfieldsTest, EnumClass) {
+  enum class Enum : unsigned { Zero = 0, Two = 2, LAST = Two };
+
+  uint8_t Storage = 0;
+  using OrderingField = Bitfield::Element<Enum, 1, 2, Enum::LAST>;
+  EXPECT_EQ(Bitfield::get<OrderingField>(Storage), Enum::Zero);
+  Bitfield::set<OrderingField>(Storage, Enum::Two);
+  EXPECT_EQ(Bitfield::get<OrderingField>(Storage), Enum::Two);
+  EXPECT_EQ(Storage, 0b00000100);
+  // value 2 in             ^^
+}
+
+TEST(BitfieldsTest, OneBitSigned) {
+  uint8_t Storage = 0;
+  using SignedField = Bitfield::Element<int, 1, 1>;
+  EXPECT_EQ(Bitfield::get<SignedField>(Storage), 0);
+  EXPECT_EQ(Storage, 0b00000000);
+  // value 0 in              ^
+  Bitfield::set<SignedField>(Storage, -1);
+  EXPECT_EQ(Bitfield::get<SignedField>(Storage), -1);
+  EXPECT_EQ(Storage, 0b00000010);
+  // value 1 in              ^
+}
+
+TEST(BitfieldsTest, TwoBitSigned) {
+  uint8_t Storage = 0;
+  using SignedField = Bitfield::Element<int, 1, 2>;
+  EXPECT_EQ(Bitfield::get<SignedField>(Storage), 0);
+  EXPECT_EQ(Storage, 0b00000000);
+  // value 0 in             ^^
+  Bitfield::set<SignedField>(Storage, 1);
+  EXPECT_EQ(Bitfield::get<SignedField>(Storage), 1);
+  EXPECT_EQ(Storage, 0b00000010);
+  // value 1 in             ^^
+  Bitfield::set<SignedField>(Storage, -1);
+  EXPECT_EQ(Bitfield::get<SignedField>(Storage), -1);
+  EXPECT_EQ(Storage, 0b00000110);
+  // value -1 in            ^^
+  Bitfield::set<SignedField>(Storage, -2);
+  EXPECT_EQ(Bitfield::get<SignedField>(Storage), -2);
+  EXPECT_EQ(Storage, 0b00000100);
+  // value -2 in            ^^
+}
+
+TEST(BitfieldsTest, isOverlapping) {
+  //    01234567
+  // A: --------
+  // B:    ---
+  // C:  ---
+  // D:     ---
+  using A = Bitfield::Element<unsigned, 0, 8>;
+  using B = Bitfield::Element<unsigned, 3, 3>;
+  using C = Bitfield::Element<unsigned, 1, 3>;
+  using D = Bitfield::Element<unsigned, 4, 3>;
+  EXPECT_TRUE((Bitfield::isOverlapping<A, B>()));
+  EXPECT_TRUE((Bitfield::isOverlapping<A, C>()));
+  EXPECT_TRUE((Bitfield::isOverlapping<A, B>()));
+  EXPECT_TRUE((Bitfield::isOverlapping<A, D>()));
+
+  EXPECT_TRUE((Bitfield::isOverlapping<B, C>()));
+  EXPECT_TRUE((Bitfield::isOverlapping<B, D>()));
+  EXPECT_FALSE((Bitfield::isOverlapping<C, D>()));
+}
+
+TEST(BitfieldsTest, areContiguous) {
+  using A = Bitfield::Element<unsigned, 0, 1>; // Next Bit:1
+  using B = Bitfield::Element<unsigned, 1, 4>; // Next Bit:5
+  using C = Bitfield::Element<unsigned, 5, 3>; // Next Bit:8
+  EXPECT_TRUE((Bitfield::areContiguous<A, B>()));
+  EXPECT_TRUE((Bitfield::areContiguous<A, B, C>()));
+
+  EXPECT_FALSE((Bitfield::areContiguous<A, C>()));
+  EXPECT_FALSE((Bitfield::areContiguous<A, A>()));
+  EXPECT_FALSE((Bitfield::areContiguous<B, A>()));
+}
+
+TEST(BitfieldsTest, FullUint64) {
+  uint64_t Storage = 0;
+  using Value = Bitfield::Element<uint64_t, 0, 64>;
+  Bitfield::set<Value>(Storage, -1ULL);
+  EXPECT_EQ(Bitfield::get<Value>(Storage), -1ULL);
+  Bitfield::set<Value>(Storage, 0ULL);
+  EXPECT_EQ(Bitfield::get<Value>(Storage), 0ULL);
+}
+
+TEST(BitfieldsTest, FullInt64) {
+  uint64_t Storage = 0;
+  using Value = Bitfield::Element<int64_t, 0, 64>;
+  Bitfield::set<Value>(Storage, -1);
+  EXPECT_EQ(Bitfield::get<Value>(Storage), -1);
+  Bitfield::set<Value>(Storage, 0);
+  EXPECT_EQ(Bitfield::get<Value>(Storage), 0);
+}
+
+#ifdef EXPECT_DEBUG_DEATH
+
+TEST(BitfieldsTest, ValueTooBigBool) {
+  uint64_t Storage = 0;
+  using A = Bitfield::Element<unsigned, 0, 1>;
+  Bitfield::set<A>(Storage, true);
+  Bitfield::set<A>(Storage, false);
+  EXPECT_DEBUG_DEATH(Bitfield::set<A>(Storage, 2), "value is too big");
+}
+
+TEST(BitfieldsTest, ValueTooBigInt) {
+  uint64_t Storage = 0;
+  using A = Bitfield::Element<unsigned, 0, 2>;
+  Bitfield::set<A>(Storage, 3);
+  EXPECT_DEBUG_DEATH(Bitfield::set<A>(Storage, 4), "value is too big");
+  EXPECT_DEBUG_DEATH(Bitfield::set<A>(Storage, -1), "value is too big");
+}
+
+TEST(BitfieldsTest, ValueTooBigBounded) {
+  uint8_t Storage = 0;
+  using A = Bitfield::Element<int, 1, 2>;
+  Bitfield::set<A>(Storage, 1);
+  Bitfield::set<A>(Storage, 0);
+  Bitfield::set<A>(Storage, -1);
+  Bitfield::set<A>(Storage, -2);
+  EXPECT_DEBUG_DEATH(Bitfield::set<A>(Storage, 2), "value is too big");
+  EXPECT_DEBUG_DEATH(Bitfield::set<A>(Storage, -3), "value is too small");
+}
+
+#endif
+
+} // namespace
diff --git a/llvm/unittests/ADT/CMakeLists.txt b/llvm/unittests/ADT/CMakeLists.txt
index 8879a2a005877..cff3911e339f5 100644
--- a/llvm/unittests/ADT/CMakeLists.txt
+++ b/llvm/unittests/ADT/CMakeLists.txt
@@ -8,6 +8,7 @@ add_llvm_unittest(ADTTests
   APIntTest.cpp
   APSIntTest.cpp
   ArrayRefTest.cpp
+  BitFieldsTest.cpp
   BitmaskEnumTest.cpp
   BitVectorTest.cpp
   BreadthFirstIteratorTest.cpp
diff --git a/llvm/unittests/ADT/FunctionExtrasTest.cpp b/llvm/unittests/ADT/FunctionExtrasTest.cpp
index bbbb045cb14ab..2ae0d1813858d 100644
--- a/llvm/unittests/ADT/FunctionExtrasTest.cpp
+++ b/llvm/unittests/ADT/FunctionExtrasTest.cpp
@@ -10,6 +10,7 @@
 #include "gtest/gtest.h"
 
 #include <memory>
+#include <type_traits>
 
 using namespace llvm;
 
@@ -224,4 +225,41 @@ TEST(UniqueFunctionTest, CountForwardingMoves) {
   UnmovableF(X);
 }
 
+TEST(UniqueFunctionTest, Const) {
+  // Can assign from const lambda.
+  unique_function<int(int) const> Plus2 = [X(std::make_unique<int>(2))](int Y) {
+    return *X + Y;
+  };
+  EXPECT_EQ(5, Plus2(3));
+
+  // Can call through a const ref.
+  const auto &Plus2Ref = Plus2;
+  EXPECT_EQ(5, Plus2Ref(3));
+
+  // Can move-construct and assign.
+  unique_function<int(int) const> Plus2A = std::move(Plus2);
+  EXPECT_EQ(5, Plus2A(3));
+  unique_function<int(int) const> Plus2B;
+  Plus2B = std::move(Plus2A);
+  EXPECT_EQ(5, Plus2B(3));
+
+  // Can convert to non-const function type, but not back.
+  unique_function<int(int)> Plus2C = std::move(Plus2B);
+  EXPECT_EQ(5, Plus2C(3));
+
+  // Overloaded call operator correctly resolved.
+  struct ChooseCorrectOverload {
+    StringRef operator()() { return "non-const"; }
+    StringRef operator()() const { return "const"; }
+  };
+  unique_function<StringRef()> ChooseMutable = ChooseCorrectOverload();
+  ChooseCorrectOverload A;
+  EXPECT_EQ("non-const", ChooseMutable());
+  EXPECT_EQ("non-const", A());
+  unique_function<StringRef() const> ChooseConst = ChooseCorrectOverload();
+  const ChooseCorrectOverload &X = A;
+  EXPECT_EQ("const", ChooseConst());
+  EXPECT_EQ("const", X());
+}
+
 } // anonymous namespace
diff --git a/llvm/unittests/ADT/STLExtrasTest.cpp b/llvm/unittests/ADT/STLExtrasTest.cpp
index a2bcd9e5c72ca..1d275d1db9094 100644
--- a/llvm/unittests/ADT/STLExtrasTest.cpp
+++ b/llvm/unittests/ADT/STLExtrasTest.cpp
@@ -490,4 +490,82 @@ TEST(STLExtrasTest, partition_point) {
   EXPECT_EQ(V.end(), partition_point(V, [](unsigned X) { return X < 50; }));
 }
 
+TEST(STLExtrasTest, hasSingleElement) {
+  const std::vector<int> V0 = {}, V1 = {1}, V2 = {1, 2};
+  const std::vector<int> V10(10);
+
+  EXPECT_EQ(hasSingleElement(V0), false);
+  EXPECT_EQ(hasSingleElement(V1), true);
+  EXPECT_EQ(hasSingleElement(V2), false);
+  EXPECT_EQ(hasSingleElement(V10), false);
+}
+
+TEST(STLExtrasTest, hasNItems) {
+  const std::list<int> V0 = {}, V1 = {1}, V2 = {1, 2};
+  const std::list<int> V3 = {1, 3, 5};
+
+  EXPECT_TRUE(hasNItems(V0, 0));
+  EXPECT_FALSE(hasNItems(V0, 2));
+  EXPECT_TRUE(hasNItems(V1, 1));
+  EXPECT_FALSE(hasNItems(V1, 2));
+
+  EXPECT_TRUE(hasNItems(V3.begin(), V3.end(), 3, [](int x) { return x < 10; }));
+  EXPECT_TRUE(hasNItems(V3.begin(), V3.end(), 0, [](int x) { return x > 10; }));
+  EXPECT_TRUE(hasNItems(V3.begin(), V3.end(), 2, [](int x) { return x < 5; }));
+}
+
+TEST(STLExtras, hasNItemsOrMore) {
+  const std::list<int> V0 = {}, V1 = {1}, V2 = {1, 2};
+  const std::list<int> V3 = {1, 3, 5};
+
+  EXPECT_TRUE(hasNItemsOrMore(V1, 1));
+  EXPECT_FALSE(hasNItemsOrMore(V1, 2));
+
+  EXPECT_TRUE(hasNItemsOrMore(V2, 1));
+  EXPECT_TRUE(hasNItemsOrMore(V2, 2));
+  EXPECT_FALSE(hasNItemsOrMore(V2, 3));
+
+  EXPECT_TRUE(hasNItemsOrMore(V3, 3));
+  EXPECT_FALSE(hasNItemsOrMore(V3, 4));
+
+  EXPECT_TRUE(
+      hasNItemsOrMore(V3.begin(), V3.end(), 3, [](int x) { return x < 10; }));
+  EXPECT_FALSE(
+      hasNItemsOrMore(V3.begin(), V3.end(), 3, [](int x) { return x > 10; }));
+  EXPECT_TRUE(
+      hasNItemsOrMore(V3.begin(), V3.end(), 2, [](int x) { return x < 5; }));
+}
+
+TEST(STLExtras, hasNItemsOrLess) {
+  const std::list<int> V0 = {}, V1 = {1}, V2 = {1, 2};
+  const std::list<int> V3 = {1, 3, 5};
+
+  EXPECT_TRUE(hasNItemsOrLess(V0, 0));
+  EXPECT_TRUE(hasNItemsOrLess(V0, 1));
+  EXPECT_TRUE(hasNItemsOrLess(V0, 2));
+
+  EXPECT_FALSE(hasNItemsOrLess(V1, 0));
+  EXPECT_TRUE(hasNItemsOrLess(V1, 1));
+  EXPECT_TRUE(hasNItemsOrLess(V1, 2));
+
+  EXPECT_FALSE(hasNItemsOrLess(V2, 0));
+  EXPECT_FALSE(hasNItemsOrLess(V2, 1));
+  EXPECT_TRUE(hasNItemsOrLess(V2, 2));
+  EXPECT_TRUE(hasNItemsOrLess(V2, 3));
+
+  EXPECT_FALSE(hasNItemsOrLess(V3, 0));
+  EXPECT_FALSE(hasNItemsOrLess(V3, 1));
+  EXPECT_FALSE(hasNItemsOrLess(V3, 2));
+  EXPECT_TRUE(hasNItemsOrLess(V3, 3));
+  EXPECT_TRUE(hasNItemsOrLess(V3, 4));
+
+  EXPECT_TRUE(
+      hasNItemsOrLess(V3.begin(), V3.end(), 1, [](int x) { return x == 1; }));
+  EXPECT_TRUE(
+      hasNItemsOrLess(V3.begin(), V3.end(), 2, [](int x) { return x < 5; }));
+  EXPECT_TRUE(
+      hasNItemsOrLess(V3.begin(), V3.end(), 5, [](int x) { return x < 5; }));
+  EXPECT_FALSE(
+      hasNItemsOrLess(V3.begin(), V3.end(), 2, [](int x) { return x < 10; }));
+}
 } // namespace
diff --git a/llvm/unittests/ADT/StringExtrasTest.cpp b/llvm/unittests/ADT/StringExtrasTest.cpp
index 67d573d649752..d17a172dbd9c8 100644
--- a/llvm/unittests/ADT/StringExtrasTest.cpp
+++ b/llvm/unittests/ADT/StringExtrasTest.cpp
@@ -182,3 +182,32 @@ TEST(StringExtras, ConvertToCamelFromSnakeCase) {
   testConvertToCamelCase(true, "Op_Name", "Op_Name");
   testConvertToCamelCase(true, "opName", "OpName");
 }
+
+constexpr uint64_t MaxUint64 = std::numeric_limits<uint64_t>::max();
+constexpr int64_t MaxInt64 = std::numeric_limits<int64_t>::max();
+constexpr int64_t MinInt64 = std::numeric_limits<int64_t>::min();
+
+TEST(StringExtras, UToStr) {
+  EXPECT_EQ("0", utostr(0));
+  EXPECT_EQ("0", utostr(0, /*isNeg=*/false));
+  EXPECT_EQ("1", utostr(1));
+  EXPECT_EQ("1", utostr(1, /*isNeg=*/false));
+  EXPECT_EQ(std::to_string(MaxUint64), utostr(MaxUint64));
+  EXPECT_EQ(std::to_string(MaxUint64), utostr(MaxUint64, /*isNeg=*/false));
+
+  EXPECT_EQ("-0", utostr(0, /*isNeg=*/true));
+  EXPECT_EQ("-1", utostr(1, /*isNeg=*/true));
+  EXPECT_EQ("-" + std::to_string(MaxInt64), utostr(MaxInt64, /*isNeg=*/true));
+  constexpr uint64_t MinusMinInt64 = -static_cast<uint64_t>(MinInt64);
+  EXPECT_EQ("-" + std::to_string(MinusMinInt64),
+            utostr(MinusMinInt64, /*isNeg=*/true));
+  EXPECT_EQ("-" + std::to_string(MaxUint64), utostr(MaxUint64, /*isNeg=*/true));
+}
+
+TEST(StringExtras, IToStr) {
+  EXPECT_EQ("0", itostr(0));
+  EXPECT_EQ("1", itostr(1));
+  EXPECT_EQ("-1", itostr(-1));
+  EXPECT_EQ(std::to_string(MinInt64), itostr(MinInt64));
+  EXPECT_EQ(std::to_string(MaxInt64), itostr(MaxInt64));
+}
diff --git a/llvm/unittests/ADT/TripleTest.cpp b/llvm/unittests/ADT/TripleTest.cpp
index ca00837aaf5a5..dc7a28c72f208 100644
--- a/llvm/unittests/ADT/TripleTest.cpp
+++ b/llvm/unittests/ADT/TripleTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/Triple.h"
+#include "llvm/Support/VersionTuple.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
@@ -1222,6 +1223,44 @@ TEST(TripleTest, getOSVersion) {
   EXPECT_EQ((unsigned)0, Minor);
   EXPECT_EQ((unsigned)0, Micro);
 
+  T = Triple("x86_64-apple-macos11.0");
+  EXPECT_TRUE(T.isMacOSX());
+  EXPECT_FALSE(T.isiOS());
+  EXPECT_FALSE(T.isArch16Bit());
+  EXPECT_FALSE(T.isArch32Bit());
+  EXPECT_TRUE(T.isArch64Bit());
+  T.getMacOSXVersion(Major, Minor, Micro);
+  EXPECT_EQ((unsigned)11, Major);
+  EXPECT_EQ((unsigned)0, Minor);
+  EXPECT_EQ((unsigned)0, Micro);
+
+  T = Triple("arm64-apple-macosx11.5.8");
+  EXPECT_TRUE(T.isMacOSX());
+  EXPECT_FALSE(T.isiOS());
+  EXPECT_FALSE(T.isArch16Bit());
+  EXPECT_FALSE(T.isArch32Bit());
+  EXPECT_TRUE(T.isArch64Bit());
+  T.getMacOSXVersion(Major, Minor, Micro);
+  EXPECT_EQ((unsigned)11, Major);
+  EXPECT_EQ((unsigned)5, Minor);
+  EXPECT_EQ((unsigned)8, Micro);
+
+  // 10.16 forms a valid triple, even though it's not
+  // a version of a macOS.
+  T = Triple("x86_64-apple-macos10.16");
+  EXPECT_TRUE(T.isMacOSX());
+  T.getMacOSXVersion(Major, Minor, Micro);
+  EXPECT_EQ((unsigned)10, Major);
+  EXPECT_EQ((unsigned)16, Minor);
+  EXPECT_EQ((unsigned)0, Micro);
+
+  T = Triple("x86_64-apple-darwin20");
+  EXPECT_TRUE(T.isMacOSX());
+  T.getMacOSXVersion(Major, Minor, Micro);
+  EXPECT_EQ((unsigned)11, Major);
+  EXPECT_EQ((unsigned)0, Minor);
+  EXPECT_EQ((unsigned)0, Micro);
+
   T = Triple("armv7-apple-ios");
   EXPECT_FALSE(T.isMacOSX());
   EXPECT_TRUE(T.isiOS());
@@ -1273,6 +1312,27 @@ TEST(TripleTest, getOSVersion) {
   EXPECT_FALSE(T.isSimulatorEnvironment());
 }
 
+TEST(TripleTest, isMacOSVersionLT) {
+  Triple T = Triple("x86_64-apple-macos11");
+  EXPECT_TRUE(T.isMacOSXVersionLT(11, 1, 0));
+  EXPECT_FALSE(T.isMacOSXVersionLT(10, 15, 0));
+
+  T = Triple("x86_64-apple-darwin20");
+  EXPECT_TRUE(T.isMacOSXVersionLT(11, 1, 0));
+  EXPECT_FALSE(T.isMacOSXVersionLT(11, 0, 0));
+  EXPECT_FALSE(T.isMacOSXVersionLT(10, 15, 0));
+}
+
+TEST(TripleTest, CanonicalizeOSVersion) {
+  EXPECT_EQ(VersionTuple(10, 15, 4),
+            Triple::getCanonicalVersionForOS(Triple::MacOSX,
+                                             VersionTuple(10, 15, 4)));
+  EXPECT_EQ(VersionTuple(11, 0), Triple::getCanonicalVersionForOS(
+                                     Triple::MacOSX, VersionTuple(10, 16)));
+  EXPECT_EQ(VersionTuple(20),
+            Triple::getCanonicalVersionForOS(Triple::Darwin, VersionTuple(20)));
+}
+
 TEST(TripleTest, FileFormat) {
   EXPECT_EQ(Triple::ELF, Triple("i686-unknown-linux-gnu").getObjectFormat());
   EXPECT_EQ(Triple::ELF, Triple("i686-unknown-freebsd").getObjectFormat());
diff --git a/llvm/unittests/Analysis/AliasAnalysisTest.cpp b/llvm/unittests/Analysis/AliasAnalysisTest.cpp
index 83f4c2481934b..6f0f6f5f63268 100644
--- a/llvm/unittests/Analysis/AliasAnalysisTest.cpp
+++ b/llvm/unittests/Analysis/AliasAnalysisTest.cpp
@@ -174,6 +174,7 @@ TEST_F(AliasAnalysisTest, getModRefInfo) {
   auto PtrType = Type::getInt32PtrTy(C);
   auto *Value = ConstantInt::get(IntType, 42);
   auto *Addr = ConstantPointerNull::get(PtrType);
+  auto Alignment = Align(IntType->getBitWidth() / 8);
 
   auto *Store1 = new StoreInst(Value, Addr, BB);
   auto *Load1 = new LoadInst(IntType, Addr, "load", BB);
@@ -181,11 +182,11 @@ TEST_F(AliasAnalysisTest, getModRefInfo) {
   auto *VAArg1 = new VAArgInst(Addr, PtrType, "vaarg", BB);
   auto *CmpXChg1 = new AtomicCmpXchgInst(
       Addr, ConstantInt::get(IntType, 0), ConstantInt::get(IntType, 1),
-      AtomicOrdering::Monotonic, AtomicOrdering::Monotonic,
+      Alignment, AtomicOrdering::Monotonic, AtomicOrdering::Monotonic,
       SyncScope::System, BB);
-  auto *AtomicRMW =
-      new AtomicRMWInst(AtomicRMWInst::Xchg, Addr, ConstantInt::get(IntType, 1),
-                        AtomicOrdering::Monotonic, SyncScope::System, BB);
+  auto *AtomicRMW = new AtomicRMWInst(
+      AtomicRMWInst::Xchg, Addr, ConstantInt::get(IntType, 1), Alignment,
+      AtomicOrdering::Monotonic, SyncScope::System, BB);
 
   ReturnInst::Create(C, nullptr, BB);
 
diff --git a/llvm/unittests/Analysis/CMakeLists.txt b/llvm/unittests/Analysis/CMakeLists.txt
index 9f28bc701b585..42f7dd3c06101 100644
--- a/llvm/unittests/Analysis/CMakeLists.txt
+++ b/llvm/unittests/Analysis/CMakeLists.txt
@@ -29,7 +29,6 @@ add_llvm_unittest(AnalysisTests
   LoopNestTest.cpp
   MemoryBuiltinsTest.cpp
   MemorySSATest.cpp
-  OrderedInstructionsTest.cpp
   PhiValuesTest.cpp
   ProfileSummaryInfoTest.cpp
   ScalarEvolutionTest.cpp
@@ -41,4 +40,4 @@ add_llvm_unittest(AnalysisTests
   ValueLatticeTest.cpp
   ValueTrackingTest.cpp
   VectorUtilsTest.cpp
-  )
\ No newline at end of file
+  )
diff --git a/llvm/unittests/Analysis/GlobalsModRefTest.cpp b/llvm/unittests/Analysis/GlobalsModRefTest.cpp
index 6e09efa75db2b..21f78712c7d3a 100644
--- a/llvm/unittests/Analysis/GlobalsModRefTest.cpp
+++ b/llvm/unittests/Analysis/GlobalsModRefTest.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/ADT/Triple.h"
+#include "llvm/Analysis/CallGraph.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gtest/gtest.h"
diff --git a/llvm/unittests/Analysis/OrderedInstructionsTest.cpp b/llvm/unittests/Analysis/OrderedInstructionsTest.cpp
deleted file mode 100644
index 473fe7f50fc8b..0000000000000
--- a/llvm/unittests/Analysis/OrderedInstructionsTest.cpp
+++ /dev/null
@@ -1,64 +0,0 @@
-//===- OrderedInstructions.cpp - Unit tests for OrderedInstructions  ------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Analysis/OrderedInstructions.h"
-#include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/Dominators.h"
-#include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
-#include "gtest/gtest.h"
-
-using namespace llvm;
-
-/// Check intra-basicblock and inter-basicblock dominance using
-/// OrderedInstruction.
-TEST(OrderedInstructionsTest, DominanceTest) {
-  LLVMContext Ctx;
-  Module M("test", Ctx);
-  IRBuilder<> B(Ctx);
-  FunctionType *FTy =
-      FunctionType::get(Type::getVoidTy(Ctx), {B.getInt8PtrTy()}, false);
-  Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
-
-  // Create the function as follow and check for dominance relation.
-  //
-  // test():
-  //  bbx:
-  //    loadx;
-  //    loady;
-  //  bby:
-  //    loadz;
-  //    return;
-  //
-  // More specifically, check for loadx -> (dominates) loady,
-  // loady -> loadx and loady -> loadz.
-  //
-  // Create BBX with 2 loads.
-  BasicBlock *BBX = BasicBlock::Create(Ctx, "bbx", F);
-  B.SetInsertPoint(BBX);
-  Argument *PointerArg = &*F->arg_begin();
-  LoadInst *LoadInstX = B.CreateLoad(B.getInt8Ty(), PointerArg);
-  LoadInst *LoadInstY = B.CreateLoad(B.getInt8Ty(), PointerArg);
-
-  // Create BBY with 1 load.
-  BasicBlock *BBY = BasicBlock::Create(Ctx, "bby", F);
-  B.SetInsertPoint(BBY);
-  LoadInst *LoadInstZ = B.CreateLoad(B.getInt8Ty(), PointerArg);
-  B.CreateRet(LoadInstZ);
-  std::unique_ptr<DominatorTree> DT(new DominatorTree(*F));
-  OrderedInstructions OI(&*DT);
-
-  // Intra-BB dominance test.
-  EXPECT_TRUE(OI.dominates(LoadInstX, LoadInstY));
-  EXPECT_FALSE(OI.dominates(LoadInstY, LoadInstX));
-
-  // Inter-BB dominance test.
-  EXPECT_TRUE(OI.dominates(LoadInstY, LoadInstZ));
-}
diff --git a/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp b/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp
index cbd2236e4cb3d..f36d3ba99775b 100644
--- a/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp
+++ b/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp
@@ -17,6 +17,7 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Module.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/SourceMgr.h"
diff --git a/llvm/unittests/AsmParser/AsmParserTest.cpp b/llvm/unittests/AsmParser/AsmParserTest.cpp
index 198191bf435fd..9a7d70ad1ed02 100644
--- a/llvm/unittests/AsmParser/AsmParserTest.cpp
+++ b/llvm/unittests/AsmParser/AsmParserTest.cpp
@@ -230,7 +230,7 @@ TEST(AsmParserTest, TypeWithSlotMappingParsing) {
   ASSERT_TRUE(Ty->isVectorTy());
 
   // Check the details of the vector.
-  VectorType *VT = cast<VectorType>(Ty);
+  auto *VT = cast<FixedVectorType>(Ty);
   ASSERT_TRUE(VT->getNumElements() == 5);
   ASSERT_TRUE(VT->getPrimitiveSizeInBits().getFixedSize() == 160);
   Ty = VT->getElementType();
@@ -362,7 +362,7 @@ TEST(AsmParserTest, TypeAtBeginningWithSlotMappingParsing) {
   ASSERT_TRUE(Read == 9);
 
   // Check the details of the vector.
-  VectorType *VT = cast<VectorType>(Ty);
+  auto *VT = cast<FixedVectorType>(Ty);
   ASSERT_TRUE(VT->getNumElements() == 5);
   ASSERT_TRUE(VT->getPrimitiveSizeInBits().getFixedSize() == 160);
   Ty = VT->getElementType();
diff --git a/llvm/unittests/CodeGen/GlobalISel/CMakeLists.txt b/llvm/unittests/CodeGen/GlobalISel/CMakeLists.txt
index 43a0302449ce0..7ca77b98dc08e 100644
--- a/llvm/unittests/CodeGen/GlobalISel/CMakeLists.txt
+++ b/llvm/unittests/CodeGen/GlobalISel/CMakeLists.txt
@@ -19,4 +19,5 @@ add_llvm_unittest(GlobalISelTests
   GISelMITest.cpp
   PatternMatchTest.cpp
   KnownBitsTest.cpp
+  GISelUtilsTest.cpp
   )
diff --git a/llvm/unittests/CodeGen/GlobalISel/GISelUtilsTest.cpp b/llvm/unittests/CodeGen/GlobalISel/GISelUtilsTest.cpp
new file mode 100644
index 0000000000000..cd788bed91ef5
--- /dev/null
+++ b/llvm/unittests/CodeGen/GlobalISel/GISelUtilsTest.cpp
@@ -0,0 +1,117 @@
+//===- GISelUtilsTest.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/GlobalISel/Utils.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+static const LLT S1 = LLT::scalar(1);
+static const LLT S16 = LLT::scalar(16);
+static const LLT S32 = LLT::scalar(32);
+static const LLT S64 = LLT::scalar(64);
+static const LLT P0 = LLT::pointer(0, 64);
+static const LLT P1 = LLT::pointer(1, 32);
+
+static const LLT V2S16 = LLT::vector(2, 16);
+static const LLT V4S16 = LLT::vector(4, 16);
+
+static const LLT V2S32 = LLT::vector(2, 32);
+static const LLT V3S32 = LLT::vector(3, 32);
+static const LLT V4S32 = LLT::vector(4, 32);
+static const LLT V6S32 = LLT::vector(6, 32);
+
+static const LLT V2P0 = LLT::vector(2, P0);
+static const LLT V3P0 = LLT::vector(3, P0);
+static const LLT V4P0 = LLT::vector(4, P0);
+static const LLT V6P0 = LLT::vector(6, P0);
+
+TEST(GISelUtilsTest, getGCDType) {
+  EXPECT_EQ(S1, getGCDType(S1, S1));
+  EXPECT_EQ(S32, getGCDType(S32, S32));
+  EXPECT_EQ(S1, getGCDType(S1, S32));
+  EXPECT_EQ(S1, getGCDType(S32, S1));
+  EXPECT_EQ(S16, getGCDType(S16, S32));
+  EXPECT_EQ(S16, getGCDType(S32, S16));
+
+  EXPECT_EQ(V2S32, getGCDType(V2S32, V2S32));
+  EXPECT_EQ(S32, getGCDType(V3S32, V2S32));
+  EXPECT_EQ(S32, getGCDType(V2S32, V3S32));
+
+  EXPECT_EQ(V2S16, getGCDType(V4S16, V2S16));
+  EXPECT_EQ(V2S16, getGCDType(V2S16, V4S16));
+
+  EXPECT_EQ(V2S32, getGCDType(V4S32, V2S32));
+  EXPECT_EQ(V2S32, getGCDType(V2S32, V4S32));
+
+  EXPECT_EQ(S16, getGCDType(P0, S16));
+  EXPECT_EQ(S16, getGCDType(S16, P0));
+
+  EXPECT_EQ(S32, getGCDType(P0, S32));
+  EXPECT_EQ(S32, getGCDType(S32, P0));
+
+  EXPECT_EQ(S64, getGCDType(P0, S64));
+  EXPECT_EQ(S64, getGCDType(S64, P0));
+
+  EXPECT_EQ(S32, getGCDType(P0, P1));
+  EXPECT_EQ(S32, getGCDType(P1, P0));
+
+  EXPECT_EQ(P0, getGCDType(V3P0, V2P0));
+  EXPECT_EQ(P0, getGCDType(V2P0, V3P0));
+}
+
+TEST(GISelUtilsTest, getLCMType) {
+  EXPECT_EQ(S1, getLCMType(S1, S1));
+  EXPECT_EQ(S32, getLCMType(S32, S1));
+  EXPECT_EQ(S32, getLCMType(S1, S32));
+  EXPECT_EQ(S32, getLCMType(S32, S32));
+
+  EXPECT_EQ(S32, getLCMType(S32, S16));
+  EXPECT_EQ(S32, getLCMType(S16, S32));
+
+  EXPECT_EQ(S64, getLCMType(S64, P0));
+  EXPECT_EQ(S64, getLCMType(P0, S64));
+
+  EXPECT_EQ(S64, getLCMType(S32, P0));
+  EXPECT_EQ(S64, getLCMType(P0, S32));
+
+  EXPECT_EQ(S32, getLCMType(S32, P1));
+  EXPECT_EQ(S32, getLCMType(P1, S32));
+  EXPECT_EQ(S64, getLCMType(P0, P0));
+  EXPECT_EQ(S32, getLCMType(P1, P1));
+
+  EXPECT_EQ(S64, getLCMType(P0, P1));
+  EXPECT_EQ(S64, getLCMType(P1, P0));
+
+  EXPECT_EQ(V2S32, getLCMType(V2S32, V2S32));
+  EXPECT_EQ(V2S32, getLCMType(V2S32, S32));
+  EXPECT_EQ(V2S32, getLCMType(S32, V2S32));
+  EXPECT_EQ(V2S32, getLCMType(V2S32, V2S32));
+  EXPECT_EQ(V6S32, getLCMType(V2S32, V3S32));
+  EXPECT_EQ(V6S32, getLCMType(V3S32, V2S32));
+  EXPECT_EQ(LLT::vector(12, S32), getLCMType(V4S32, V3S32));
+  EXPECT_EQ(LLT::vector(12, S32), getLCMType(V3S32, V4S32));
+
+  EXPECT_EQ(V2P0, getLCMType(V2P0, V2P0));
+  EXPECT_EQ(V2P0, getLCMType(V2P0, P0));
+  EXPECT_EQ(V2P0, getLCMType(P0, V2P0));
+  EXPECT_EQ(V2P0, getLCMType(V2P0, V2P0));
+  EXPECT_EQ(V6P0, getLCMType(V2P0, V3P0));
+  EXPECT_EQ(V6P0, getLCMType(V3P0, V2P0));
+  EXPECT_EQ(LLT::vector(12, P0), getLCMType(V4P0, V3P0));
+  EXPECT_EQ(LLT::vector(12, P0), getLCMType(V3P0, V4P0));
+
+  // FIXME
+  // EXPECT_EQ(V2S32, getLCMType(V2S32, S64));
+
+  // FIXME
+  //EXPECT_EQ(S64, getLCMType(S64, V2S32));
+}
+
+}
diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp
index a7465a8e31339..65e2be723090a 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp
@@ -6,10 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugFrame.h"
+#include "llvm/Testing/Support/Error.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
@@ -119,4 +121,205 @@ TEST(DWARFDebugFrame, DumpEH64FDE) {
                    "cie=1111ab9a000c pc=4444abcdabcd...5555bcdebcde");
 }
 
+static Error parseCFI(dwarf::CIE &C, ArrayRef<uint8_t> Instructions,
+                      Optional<uint64_t> Size = None) {
+  DWARFDataExtractor Data(Instructions, /*IsLittleEndian=*/true,
+                          /*AddressSize=*/8);
+  uint64_t Offset = 0;
+  const uint64_t EndOffset = Size ? *Size : (uint64_t)Instructions.size();
+  return C.cfis().parse(Data, &Offset, EndOffset);
+}
+
+TEST(DWARFDebugFrame, InvalidCFIOpcodesTest) {
+  llvm::DenseSet<uint8_t> ValidExtendedOpcodes = {
+      dwarf::DW_CFA_nop,
+      dwarf::DW_CFA_advance_loc,
+      dwarf::DW_CFA_offset,
+      dwarf::DW_CFA_restore,
+      dwarf::DW_CFA_set_loc,
+      dwarf::DW_CFA_advance_loc1,
+      dwarf::DW_CFA_advance_loc2,
+      dwarf::DW_CFA_advance_loc4,
+      dwarf::DW_CFA_offset_extended,
+      dwarf::DW_CFA_restore_extended,
+      dwarf::DW_CFA_undefined,
+      dwarf::DW_CFA_same_value,
+      dwarf::DW_CFA_register,
+      dwarf::DW_CFA_remember_state,
+      dwarf::DW_CFA_restore_state,
+      dwarf::DW_CFA_def_cfa,
+      dwarf::DW_CFA_def_cfa_register,
+      dwarf::DW_CFA_def_cfa_offset,
+      dwarf::DW_CFA_def_cfa_expression,
+      dwarf::DW_CFA_expression,
+      dwarf::DW_CFA_offset_extended_sf,
+      dwarf::DW_CFA_def_cfa_sf,
+      dwarf::DW_CFA_def_cfa_offset_sf,
+      dwarf::DW_CFA_val_offset,
+      dwarf::DW_CFA_val_offset_sf,
+      dwarf::DW_CFA_val_expression,
+      dwarf::DW_CFA_MIPS_advance_loc8,
+      dwarf::DW_CFA_GNU_window_save,
+      dwarf::DW_CFA_AARCH64_negate_ra_state,
+      dwarf::DW_CFA_GNU_args_size};
+
+  dwarf::CIE TestCIE = createCIE(/*IsDWARF64=*/false,
+                                 /*Offset=*/0x0,
+                                 /*Length=*/0xff);
+
+  // See DWARF standard v3, section 7.23: low 6 bits are used to encode an
+  // extended opcode.
+  for (uint8_t Code = 0; Code <= 63; ++Code) {
+    if (ValidExtendedOpcodes.count(Code))
+      continue;
+
+    EXPECT_THAT_ERROR(parseCFI(TestCIE, Code),
+                      FailedWithMessage(("invalid extended CFI opcode 0x" +
+                                         Twine::utohexstr(Code))
+                                            .str()
+                                            .c_str()));
+  }
+}
+
+// Here we test how truncated Call Frame Instructions are parsed.
+TEST(DWARFDebugFrame, ParseTruncatedCFITest) {
+  dwarf::CIE TestCIE = createCIE(/*IsDWARF64=*/false,
+                                 /*Offset=*/0x0,
+                                 /*Length=*/0xff);
+
+  // Having an empty instructions list is fine.
+  EXPECT_THAT_ERROR(parseCFI(TestCIE, {}), Succeeded());
+
+  // Unable to read an opcode, because the instructions list is empty, but we
+  // say to the parser that it is not.
+  EXPECT_THAT_ERROR(
+      parseCFI(TestCIE, {}, /*Size=*/1),
+      FailedWithMessage(
+          "unexpected end of data at offset 0x0 while reading [0x0, 0x1)"));
+
+  // Unable to read a truncated DW_CFA_offset instruction.
+  EXPECT_THAT_ERROR(
+      parseCFI(TestCIE, {dwarf::DW_CFA_offset}),
+      FailedWithMessage("unable to decode LEB128 at offset 0x00000001: "
+                        "malformed uleb128, extends past end"));
+
+  // Unable to read a truncated DW_CFA_set_loc instruction.
+  EXPECT_THAT_ERROR(
+      parseCFI(TestCIE, {dwarf::DW_CFA_set_loc}),
+      FailedWithMessage(
+          "unexpected end of data at offset 0x1 while reading [0x1, 0x9)"));
+
+  // Unable to read a truncated DW_CFA_advance_loc1 instruction.
+  EXPECT_THAT_ERROR(
+      parseCFI(TestCIE, {dwarf::DW_CFA_advance_loc1}),
+      FailedWithMessage(
+          "unexpected end of data at offset 0x1 while reading [0x1, 0x2)"));
+
+  // Unable to read a truncated DW_CFA_advance_loc2 instruction.
+  EXPECT_THAT_ERROR(
+      parseCFI(TestCIE, {dwarf::DW_CFA_advance_loc2}),
+      FailedWithMessage(
+          "unexpected end of data at offset 0x1 while reading [0x1, 0x3)"));
+
+  // Unable to read a truncated DW_CFA_advance_loc4 instruction.
+  EXPECT_THAT_ERROR(
+      parseCFI(TestCIE, {dwarf::DW_CFA_advance_loc4}),
+      FailedWithMessage(
+          "unexpected end of data at offset 0x1 while reading [0x1, 0x5)"));
+
+  // A test for an instruction with a single ULEB128 operand.
+  auto CheckOp_ULEB128 = [&](uint8_t Inst) {
+    EXPECT_THAT_ERROR(
+        parseCFI(TestCIE, Inst),
+        FailedWithMessage("unable to decode LEB128 at offset 0x00000001: "
+                          "malformed uleb128, extends past end"));
+  };
+
+  for (uint8_t Inst :
+       {dwarf::DW_CFA_restore_extended, dwarf::DW_CFA_undefined,
+        dwarf::DW_CFA_same_value, dwarf::DW_CFA_def_cfa_register,
+        dwarf::DW_CFA_def_cfa_offset, dwarf::DW_CFA_GNU_args_size})
+    CheckOp_ULEB128(Inst);
+
+  // Unable to read a truncated DW_CFA_def_cfa_offset_sf instruction.
+  EXPECT_THAT_ERROR(
+      parseCFI(TestCIE, {dwarf::DW_CFA_def_cfa_offset_sf}),
+      FailedWithMessage("unable to decode LEB128 at offset 0x00000001: "
+                        "malformed sleb128, extends past end"));
+
+  // A test for an instruction with two ULEB128 operands.
+  auto CheckOp_ULEB128_ULEB128 = [&](uint8_t Inst) {
+    EXPECT_THAT_ERROR(
+        parseCFI(TestCIE, Inst),
+        FailedWithMessage("unable to decode LEB128 at offset 0x00000001: "
+                          "malformed uleb128, extends past end"));
+
+    EXPECT_THAT_ERROR(
+        parseCFI(TestCIE, {Inst, /*Op1=*/0}),
+        FailedWithMessage("unable to decode LEB128 at offset 0x00000002: "
+                          "malformed uleb128, extends past end"));
+  };
+
+  for (uint8_t Inst : {dwarf::DW_CFA_offset_extended, dwarf::DW_CFA_register,
+                       dwarf::DW_CFA_def_cfa, dwarf::DW_CFA_val_offset})
+    CheckOp_ULEB128_ULEB128(Inst);
+
+  // A test for an instruction with two operands: ULEB128, SLEB128.
+  auto CheckOp_ULEB128_SLEB128 = [&](uint8_t Inst) {
+    EXPECT_THAT_ERROR(
+        parseCFI(TestCIE, Inst),
+        FailedWithMessage("unable to decode LEB128 at offset 0x00000001: "
+                          "malformed uleb128, extends past end"));
+
+    EXPECT_THAT_ERROR(
+        parseCFI(TestCIE, {Inst, /*Op1=*/0}),
+        FailedWithMessage("unable to decode LEB128 at offset 0x00000002: "
+                          "malformed sleb128, extends past end"));
+  };
+
+  for (uint8_t Inst : {dwarf::DW_CFA_offset_extended_sf,
+                       dwarf::DW_CFA_def_cfa_sf, dwarf::DW_CFA_val_offset_sf})
+    CheckOp_ULEB128_SLEB128(Inst);
+
+  // Unable to read a truncated DW_CFA_def_cfa_expression instruction.
+  EXPECT_THAT_ERROR(
+      parseCFI(TestCIE, {dwarf::DW_CFA_def_cfa_expression}),
+      FailedWithMessage("unable to decode LEB128 at offset 0x00000001: "
+                        "malformed uleb128, extends past end"));
+  EXPECT_THAT_ERROR(
+      parseCFI(TestCIE, {dwarf::DW_CFA_def_cfa_expression,
+                         /*expression length=*/0x1}),
+      FailedWithMessage(
+          "unexpected end of data at offset 0x2 while reading [0x2, 0x3)"));
+  // The DW_CFA_def_cfa_expression can contain a zero length expression.
+  EXPECT_THAT_ERROR(parseCFI(TestCIE, {dwarf::DW_CFA_def_cfa_expression,
+                                       /*ExprLen=*/0}),
+                    Succeeded());
+
+  // A test for an instruction with three operands: ULEB128, expression length
+  // (ULEB128) and expression bytes.
+  auto CheckOp_ULEB128_Expr = [&](uint8_t Inst) {
+    EXPECT_THAT_ERROR(
+        parseCFI(TestCIE, {Inst}),
+        FailedWithMessage("unable to decode LEB128 at offset 0x00000001: "
+                          "malformed uleb128, extends past end"));
+    EXPECT_THAT_ERROR(
+        parseCFI(TestCIE, {Inst, /*Op1=*/0}),
+        FailedWithMessage("unable to decode LEB128 at offset 0x00000002: "
+                          "malformed uleb128, extends past end"));
+    // A zero length expression is fine
+    EXPECT_THAT_ERROR(parseCFI(TestCIE, {Inst,
+                                         /*Op1=*/0, /*ExprLen=*/0}),
+                      Succeeded());
+    EXPECT_THAT_ERROR(
+        parseCFI(TestCIE, {Inst,
+                           /*Op1=*/0, /*ExprLen=*/1}),
+        FailedWithMessage(
+            "unexpected end of data at offset 0x3 while reading [0x3, 0x4)"));
+  };
+
+  for (uint8_t Inst : {dwarf::DW_CFA_expression, dwarf::DW_CFA_val_expression})
+    CheckOp_ULEB128_Expr(Inst);
+}
+
 } // end anonymous namespace
diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp
index d481299d0146c..6825981de0c3c 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp
@@ -1368,8 +1368,7 @@ TEST(DWARFDebugInfo, TestEmptyChildren) {
                          "    Children:        DW_CHILDREN_yes\n"
                          "    Attributes:\n"
                          "debug_info:\n"
-                         "  - Length:\n"
-                         "      TotalLength:          0\n"
+                         "  - Length:          0\n"
                          "    Version:         4\n"
                          "    AbbrOffset:      0\n"
                          "    AddrSize:        8\n"
@@ -1902,8 +1901,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyInvalidCURef) {
           - Attribute:       DW_AT_type
             Form:            DW_FORM_ref4
     debug_info:
-      - Length:
-          TotalLength:     22
+      - Length:          22
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -1951,8 +1949,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyInvalidRefAddr) {
           - Attribute:       DW_AT_type
             Form:            DW_FORM_ref_addr
     debug_info:
-      - Length:
-          TotalLength:     22
+      - Length:          22
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -1992,8 +1989,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyInvalidRanges) {
           - Attribute:       DW_AT_ranges
             Form:            DW_FORM_sec_offset
     debug_info:
-      - Length:
-          TotalLength:     16
+      - Length:          16
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -2030,8 +2026,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyInvalidRnglists) {
           - Attribute:       DW_AT_ranges
             Form:            DW_FORM_sec_offset
     debug_info:
-      - Length:
-          TotalLength:     17
+      - Length:          17
         Version:         5
         UnitType:        DW_UT_compile
         AbbrOffset:      0
@@ -2068,8 +2063,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyInvalidStmtList) {
           - Attribute:       DW_AT_stmt_list
             Form:            DW_FORM_sec_offset
     debug_info:
-      - Length:
-          TotalLength:     16
+      - Length:          16
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -2103,8 +2097,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyInvalidStrp) {
           - Attribute:       DW_AT_name
             Form:            DW_FORM_strp
     debug_info:
-      - Length:
-          TotalLength:     12
+      - Length:          12
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -2145,8 +2138,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyInvalidRefAddrBetween) {
           - Attribute:       DW_AT_type
             Form:            DW_FORM_ref_addr
     debug_info:
-      - Length:
-          TotalLength:     22
+      - Length:          22
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -2187,8 +2179,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyInvalidLineSequence) {
           - Attribute:       DW_AT_stmt_list
             Form:            DW_FORM_sec_offset
     debug_info:
-      - Length:
-          TotalLength:     16
+      - Length:          16
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -2256,8 +2247,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyInvalidLineFileIndex) {
           - Attribute:       DW_AT_stmt_list
             Form:            DW_FORM_sec_offset
     debug_info:
-      - Length:
-          TotalLength:     16
+      - Length:          16
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -2327,8 +2317,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyInvalidLineTablePorlogueDirIndex) {
           - Attribute:       DW_AT_stmt_list
             Form:            DW_FORM_sec_offset
     debug_info:
-      - Length:
-          TotalLength:     16
+      - Length:          16
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -2399,8 +2388,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyDuplicateFileWarning) {
           - Attribute:       DW_AT_stmt_list
             Form:            DW_FORM_sec_offset
     debug_info:
-      - Length:
-          TotalLength:     16
+      - Length:          16
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -2476,8 +2464,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyCUDontShareLineTable) {
           - Attribute:       DW_AT_stmt_list
             Form:            DW_FORM_sec_offset
     debug_info:
-      - Length:
-          TotalLength:     16
+      - Length:          16
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -2486,8 +2473,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyCUDontShareLineTable) {
             Values:
               - Value:           0x0000000000000001
               - Value:           0x0000000000000000
-      - Length:
-          TotalLength:     16
+      - Length:          16
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -2603,8 +2589,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyCURangesIncomplete) {
           - Attribute:       DW_AT_high_pc
             Form:            DW_FORM_addr
     debug_info:
-      - Length:
-          TotalLength:     46
+      - Length:          46
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -2663,8 +2648,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyLexicalBlockRanges) {
           - Attribute:       DW_AT_high_pc
             Form:            DW_FORM_addr
     debug_info:
-      - Length:
-          TotalLength:     52
+      - Length:          52
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -2721,8 +2705,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyOverlappingFunctionRanges) {
           - Attribute:       DW_AT_high_pc
             Form:            DW_FORM_addr
     debug_info:
-      - Length:
-          TotalLength:     55
+      - Length:          55
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -2788,8 +2771,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyOverlappingLexicalBlockRanges) {
           - Attribute:       DW_AT_high_pc
             Form:            DW_FORM_addr
     debug_info:
-      - Length:
-          TotalLength:     85
+      - Length:          85
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -2850,8 +2832,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyInvalidDIERange) {
           - Attribute:       DW_AT_high_pc
             Form:            DW_FORM_addr
     debug_info:
-      - Length:
-          TotalLength:     34
+      - Length:          34
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -2907,8 +2888,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyElidedDoesntFail) {
           - Attribute:       DW_AT_high_pc
             Form:            DW_FORM_addr
     debug_info:
-      - Length:
-          TotalLength:     71
+      - Length:          71
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
@@ -2970,8 +2950,7 @@ TEST(DWARFDebugInfo, TestDwarfVerifyNestedFunctions) {
           - Attribute:       DW_AT_high_pc
             Form:            DW_FORM_addr
     debug_info:
-      - Length:
-          TotalLength:     73
+      - Length:          73
         Version:         4
         AbbrOffset:      0
         AddrSize:        8
diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFDebugLineTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFDebugLineTest.cpp
index 11256610fadbd..f4ea34d02094a 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFDebugLineTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFDebugLineTest.cpp
@@ -1380,6 +1380,130 @@ TEST_F(DebugLineBasicFixture, VerboseOutput) {
   EXPECT_EQ(Output.size(), Pos);
 }
 
+struct TruncatedPrologueFixture
+    : public TestWithParam<
+          std::tuple<uint64_t, uint64_t, uint16_t, DwarfFormat, StringRef>>,
+      public CommonFixture {
+  void SetUp() {
+    std::tie(Length, ExpectedOffset, Version, Format, ExpectedErr) = GetParam();
+  }
+
+  uint64_t Length;
+  uint64_t ExpectedOffset;
+  uint16_t Version;
+  DwarfFormat Format;
+  StringRef ExpectedErr;
+};
+
+TEST_P(TruncatedPrologueFixture, ErrorForTruncatedPrologue) {
+  if (!setupGenerator(Version))
+    return;
+
+  LineTable &Padding = Gen->addLineTable();
+  // Add some padding to show that a non-zero offset is handled correctly.
+  Padding.setCustomPrologue({{0, LineTable::Byte}});
+
+  // Add a table with only two standard opcodes - we don't need to test the full
+  // set.
+  LineTable &Table = Gen->addLineTable(Format);
+  DWARFDebugLine::Prologue InputPrologue = Table.createBasicPrologue();
+  InputPrologue.OpcodeBase = 3;
+  InputPrologue.StandardOpcodeLengths.resize(2);
+  Table.setPrologue(InputPrologue);
+
+  generate();
+  // Truncate the data extractor to the specified length.
+  LineData = DWARFDataExtractor(LineData, Length);
+
+  DWARFDebugLine::Prologue Prologue;
+  uint64_t Offset = 1;
+  Error Err = Prologue.parse(LineData, &Offset, RecordRecoverable, *Context);
+
+  EXPECT_THAT_ERROR(std::move(Err), FailedWithMessage(ExpectedErr.str()));
+  EXPECT_EQ(Offset, ExpectedOffset);
+}
+
+INSTANTIATE_TEST_CASE_P(
+    TruncatedPrologueParams, TruncatedPrologueFixture,
+    Values(
+        // Truncated length:
+        std::make_tuple(
+            4, 1, 4, DWARF32,
+            "parsing line table prologue at offset 0x00000001: unexpected end "
+            "of data at offset 0x4 while reading [0x1, 0x5)"),
+        std::make_tuple(
+            4, 1, 4, DWARF64,
+            "parsing line table prologue at offset 0x00000001: unexpected end "
+            "of data at offset 0x4 while reading [0x1, 0x5)"),
+        std::make_tuple(
+            0xc, 1, 4, DWARF64,
+            "parsing line table prologue at offset 0x00000001: unexpected end "
+            "of data at offset 0xc while reading [0x5, 0xd)"),
+        // Truncated version:
+        std::make_tuple(
+            6, 5, 4, DWARF32,
+            "parsing line table prologue at offset 0x00000001: unexpected end "
+            "of data at offset 0x6 while reading [0x5, 0x7)"),
+        // Truncated address size:
+        std::make_tuple(
+            7, 7, 5, DWARF32,
+            "parsing line table prologue at offset 0x00000001: unexpected end "
+            "of data at offset 0x7 while reading [0x7, 0x8)"),
+        // Truncated segment selector size:
+        std::make_tuple(
+            8, 8, 5, DWARF32,
+            "parsing line table prologue at offset 0x00000001: unexpected end "
+            "of data at offset 0x8 while reading [0x8, 0x9)"),
+        // Truncated prologue length:
+        std::make_tuple(
+            0xa, 7, 4, DWARF32,
+            "parsing line table prologue at offset 0x00000001: unexpected end "
+            "of data at offset 0xa while reading [0x7, 0xb)"),
+        std::make_tuple(
+            0x16, 0xf, 4, DWARF64,
+            "parsing line table prologue at offset 0x00000001: unexpected end "
+            "of data at offset 0x16 while reading [0xf, 0x17)"),
+        // Truncated min instruction length:
+        std::make_tuple(
+            0xb, 0xb, 4, DWARF32,
+            "parsing line table prologue at offset 0x00000001: unexpected end "
+            "of data at offset 0xb while reading [0xb, 0xc)"),
+        // Truncated max ops per inst:
+        std::make_tuple(
+            0xc, 0xc, 4, DWARF32,
+            "parsing line table prologue at offset 0x00000001: unexpected end "
+            "of data at offset 0xc while reading [0xc, 0xd)"),
+        // Truncated default is stmt:
+        std::make_tuple(
+            0xd, 0xd, 4, DWARF32,
+            "parsing line table prologue at offset 0x00000001: unexpected end "
+            "of data at offset 0xd while reading [0xd, 0xe)"),
+        // Truncated line base:
+        std::make_tuple(
+            0xe, 0xe, 4, DWARF32,
+            "parsing line table prologue at offset 0x00000001: unexpected end "
+            "of data at offset 0xe while reading [0xe, 0xf)"),
+        // Truncated line range:
+        std::make_tuple(
+            0xf, 0xf, 4, DWARF32,
+            "parsing line table prologue at offset 0x00000001: unexpected end "
+            "of data at offset 0xf while reading [0xf, 0x10)"),
+        // Truncated opcode base:
+        std::make_tuple(
+            0x10, 0x10, 4, DWARF32,
+            "parsing line table prologue at offset 0x00000001: unexpected end "
+            "of data at offset 0x10 while reading [0x10, 0x11)"),
+        // Truncated first standard opcode:
+        std::make_tuple(
+            0x11, 0x11, 4, DWARF32,
+            "parsing line table prologue at offset 0x00000001: unexpected end "
+            "of data at offset 0x11 while reading [0x11, 0x12)"),
+        // Truncated second standard opcode:
+        std::make_tuple(
+            0x12, 0x12, 4, DWARF32,
+            "parsing line table prologue at offset 0x00000001: unexpected end "
+            "of data at offset 0x12 while reading [0x12, 0x13)")), );
+
 using ValueAndLengths = std::vector<LineTable::ValueAndLength>;
 
 struct TruncatedOpcodeFixtureBase : public CommonFixture {
@@ -1461,64 +1585,71 @@ TEST_P(TruncatedExtendedOpcodeFixture, ErrorForTruncatedExtendedOpcode) {
 INSTANTIATE_TEST_CASE_P(
     TruncatedExtendedOpcodeParams, TruncatedExtendedOpcodeFixture,
     Values(
-        std::make_tuple(1, 1, DW_LNE_end_sequence, ValueAndLengths(),
-                        "Badly formed extended line op (length 0)",
+        // Truncated length:
+        std::make_tuple(1, 1, /*ArbitraryOpcode=*/0x7f, ValueAndLengths(), "",
                         "unable to decode LEB128 at offset 0x00000030: "
                         "malformed uleb128, extends past end"),
+        // Truncated opcode:
         std::make_tuple(
-            2, 9, DW_LNE_set_address,
-            ValueAndLengths{{0x12345678, LineTable::Quad}},
-            "Unrecognized extended op 0x00 length 9",
+            2, 9, /*ArbitraryOpcode=*/0x7f, ValueAndLengths(), "",
             "unexpected end of data at offset 0x31 while reading [0x31, 0x32)"),
+        // Truncated operands:
         std::make_tuple(
             3, 9, DW_LNE_set_address,
-            ValueAndLengths{{0x12345678, LineTable::Quad}},
-            "DW_LNE_set_address (0x0000000000000000)",
+            ValueAndLengths{{0x1234567890abcdef, LineTable::Quad}},
+            "DW_LNE_set_address",
             "unexpected end of data at offset 0x32 while reading [0x32, 0x3a)"),
-        std::make_tuple(3, 5, DW_LNE_define_file,
+        std::make_tuple(
+            10, 9, DW_LNE_set_address,
+            ValueAndLengths{{0x1234567878563412, LineTable::Quad}},
+            "DW_LNE_set_address (<parsing error> 12 34 56 78 78 56 34)",
+            "unexpected end of data at offset 0x39 while reading [0x32, 0x3a)"),
+        std::make_tuple(3, 6, DW_LNE_define_file,
                         ValueAndLengths{{'a', LineTable::Byte},
                                         {'\0', LineTable::Byte},
                                         {1, LineTable::ULEB},
                                         {1, LineTable::ULEB},
                                         {1, LineTable::ULEB}},
-                        "DW_LNE_define_file (, dir=0, "
-                        "mod_time=(0x0000000000000000), length=0)",
+                        "DW_LNE_define_file",
                         "no null terminated string at offset 0x32"),
-        std::make_tuple(5, 5, DW_LNE_define_file,
+        std::make_tuple(5, 6, DW_LNE_define_file,
                         ValueAndLengths{{'a', LineTable::Byte},
                                         {'\0', LineTable::Byte},
                                         {1, LineTable::ULEB},
                                         {1, LineTable::ULEB},
                                         {1, LineTable::ULEB}},
-                        "DW_LNE_define_file (a, dir=0, "
-                        "mod_time=(0x0000000000000000), length=0)",
+                        "DW_LNE_define_file (<parsing error> 61 00)",
                         "unable to decode LEB128 at offset 0x00000034: "
                         "malformed uleb128, extends past end"),
-        std::make_tuple(6, 5, DW_LNE_define_file,
+        std::make_tuple(6, 6, DW_LNE_define_file,
                         ValueAndLengths{{'a', LineTable::Byte},
                                         {'\0', LineTable::Byte},
                                         {1, LineTable::ULEB},
                                         {1, LineTable::ULEB},
                                         {1, LineTable::ULEB}},
-                        "DW_LNE_define_file (a, dir=1, "
-                        "mod_time=(0x0000000000000000), length=0)",
+                        "DW_LNE_define_file (<parsing error> 61 00 01)",
                         "unable to decode LEB128 at offset 0x00000035: "
                         "malformed uleb128, extends past end"),
-        std::make_tuple(7, 5, DW_LNE_define_file,
+        std::make_tuple(7, 6, DW_LNE_define_file,
                         ValueAndLengths{{'a', LineTable::Byte},
                                         {'\0', LineTable::Byte},
                                         {1, LineTable::ULEB},
                                         {1, LineTable::ULEB},
                                         {1, LineTable::ULEB}},
-                        "DW_LNE_define_file (a, dir=1, "
-                        "mod_time=(0x0000000000000001), length=0)",
+                        "DW_LNE_define_file (<parsing error> 61 00 01 01)",
                         "unable to decode LEB128 at offset 0x00000036: "
                         "malformed uleb128, extends past end"),
         std::make_tuple(3, 2, DW_LNE_set_discriminator,
                         ValueAndLengths{{1, LineTable::ULEB}},
-                        "DW_LNE_set_discriminator (0)",
+                        "DW_LNE_set_discriminator",
                         "unable to decode LEB128 at offset 0x00000032: "
-                        "malformed uleb128, extends past end")), );
+                        "malformed uleb128, extends past end"),
+        std::make_tuple(
+            6, 5, /*Unknown=*/0x7f,
+            ValueAndLengths{{0x12343412, LineTable::Long}},
+            "Unrecognized extended op 0x7f length 5 (<parsing error> 12 34 34)",
+            "unexpected end of data at offset 0x35 while reading [0x32, "
+            "0x36)")), );
 
 TEST_P(TruncatedStandardOpcodeFixture, ErrorForTruncatedStandardOpcode) {
   if (!setupGenerator())
diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFDieTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFDieTest.cpp
index cd82c56124415..983f0624939a2 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFDieTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFDieTest.cpp
@@ -34,8 +34,7 @@ TEST(DWARFDie, getLocations) {
           - Attribute:       DW_AT_call_data_location
             Form:            DW_FORM_sec_offset
     debug_info:
-      - Length:
-          TotalLength:     0
+      - Length:          0
         Version:         5
         UnitType:        DW_UT_compile
         AbbrOffset:      0
diff --git a/llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp b/llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp
index 80814f23d5b31..5d53c0d31bdf8 100644
--- a/llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp
@@ -27,6 +27,7 @@
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCTargetOptionsCommandFlags.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/LEB128.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/llvm/unittests/DebugInfo/GSYM/GSYMTest.cpp b/llvm/unittests/DebugInfo/GSYM/GSYMTest.cpp
index f55e90b5590df..1e5d79c43c118 100644
--- a/llvm/unittests/DebugInfo/GSYM/GSYMTest.cpp
+++ b/llvm/unittests/DebugInfo/GSYM/GSYMTest.cpp
@@ -1437,8 +1437,7 @@ TEST(GSYMTest, TestDWARFFunctionWithAddresses) {
         - Attribute:       DW_AT_high_pc
           Form:            DW_FORM_addr
   debug_info:
-    - Length:
-        TotalLength:     52
+    - Length:          52
       Version:         4
       AbbrOffset:      0
       AddrSize:        8
@@ -1517,8 +1516,7 @@ TEST(GSYMTest, TestDWARFFunctionWithAddressAndOffset) {
         - Attribute:       DW_AT_high_pc
           Form:            DW_FORM_data4
   debug_info:
-    - Length:
-        TotalLength:     44
+    - Length:          44
       Version:         4
       AbbrOffset:      0
       AddrSize:        8
@@ -1617,8 +1615,7 @@ TEST(GSYMTest, TestDWARFStructMethodNoMangled) {
         - Attribute:       DW_AT_artificial
           Form:            DW_FORM_flag_present
   debug_info:
-    - Length:
-        TotalLength:     68
+    - Length:          68
       Version:         4
       AbbrOffset:      0
       AddrSize:        8
@@ -1722,8 +1719,7 @@ TEST(GSYMTest, TestDWARFTextRanges) {
         - Attribute:       DW_AT_high_pc
           Form:            DW_FORM_data4
   debug_info:
-    - Length:
-        TotalLength:     78
+    - Length:          78
       Version:         4
       AbbrOffset:      0
       AddrSize:        8
@@ -1836,8 +1832,7 @@ TEST(GSYMTest, TestDWARFInlineInfo) {
         - Attribute:       DW_AT_call_line
           Form:            DW_FORM_data4
   debug_info:
-    - Length:
-        TotalLength:     74
+    - Length:          74
       Version:         4
       AbbrOffset:      0
       AddrSize:        8
@@ -2092,8 +2087,7 @@ TEST(GSYMTest, TestDWARFNoLines) {
         - Attribute:       DW_AT_decl_line
           Form:            DW_FORM_data1
   debug_info:
-    - Length:
-        TotalLength:     103
+    - Length:          103
       Version:         4
       AbbrOffset:      0
       AddrSize:        8
@@ -2342,8 +2336,7 @@ TEST(GSYMTest, TestDWARFDeadStripAddr4) {
         - Attribute:       DW_AT_high_pc
           Form:            DW_FORM_addr
   debug_info:
-    - Length:
-        TotalLength:     75
+    - Length:          75
       Version:         4
       AbbrOffset:      0
       AddrSize:        4
@@ -2485,8 +2478,7 @@ TEST(GSYMTest, TestDWARFDeadStripAddr8) {
         - Attribute:       DW_AT_high_pc
           Form:            DW_FORM_addr
   debug_info:
-    - Length:
-        TotalLength:     103
+    - Length:          103
       Version:         4
       AbbrOffset:      0
       AddrSize:        8
diff --git a/llvm/unittests/Frontend/CMakeLists.txt b/llvm/unittests/Frontend/CMakeLists.txt
index b3ffaf90c47e4..4b48945b2854c 100644
--- a/llvm/unittests/Frontend/CMakeLists.txt
+++ b/llvm/unittests/Frontend/CMakeLists.txt
@@ -10,6 +10,9 @@ set(LLVM_LINK_COMPONENTS
 add_llvm_unittest(LLVMFrontendTests
   OpenMPContextTest.cpp
   OpenMPIRBuilderTest.cpp
+
+  DEPENDS
+  omp_gen
   )
 
 target_link_libraries(LLVMFrontendTests PRIVATE LLVMTestingSupport)
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index b356f6d66e17e..2ba9d85a0f9ee 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -19,7 +19,6 @@
 
 using namespace llvm;
 using namespace omp;
-using namespace types;
 
 namespace {
 
@@ -50,7 +49,6 @@ class OpenMPIRBuilderTest : public testing::Test {
   void TearDown() override {
     BB = nullptr;
     M.reset();
-    uninitializeTypes();
   }
 
   LLVMContext Ctx;
@@ -779,4 +777,39 @@ TEST_F(OpenMPIRBuilderTest, CriticalDirective) {
   EXPECT_EQ(CriticalEndCI->getArgOperand(2)->getType(), CriticalNamePtrTy);
 }
 
+TEST_F(OpenMPIRBuilderTest, CopyinBlocks) {
+  OpenMPIRBuilder OMPBuilder(*M);
+  OMPBuilder.initialize();
+  F->setName("func");
+  IRBuilder<> Builder(BB);
+
+  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
+
+  IntegerType* Int32 = Type::getInt32Ty(M->getContext());
+  AllocaInst* MasterAddress = Builder.CreateAlloca(Int32->getPointerTo());
+	AllocaInst* PrivAddress = Builder.CreateAlloca(Int32->getPointerTo());
+
+  BasicBlock *EntryBB = BB;
+
+  OMPBuilder.CreateCopyinClauseBlocks(Builder.saveIP(), MasterAddress, PrivAddress, Int32, /*BranchtoEnd*/true);
+
+  BranchInst* EntryBr = dyn_cast_or_null<BranchInst>(EntryBB->getTerminator());
+
+  EXPECT_NE(EntryBr, nullptr);
+  EXPECT_TRUE(EntryBr->isConditional());
+
+  BasicBlock* NotMasterBB = EntryBr->getSuccessor(0);
+  BasicBlock* CopyinEnd = EntryBr->getSuccessor(1);
+  CmpInst* CMP = dyn_cast_or_null<CmpInst>(EntryBr->getCondition());
+
+  EXPECT_NE(CMP, nullptr);
+  EXPECT_NE(NotMasterBB, nullptr);
+  EXPECT_NE(CopyinEnd, nullptr);
+
+  BranchInst* NotMasterBr = dyn_cast_or_null<BranchInst>(NotMasterBB->getTerminator());
+  EXPECT_NE(NotMasterBr, nullptr);
+  EXPECT_FALSE(NotMasterBr->isConditional());
+  EXPECT_EQ(CopyinEnd,NotMasterBr->getSuccessor(0));
+}
+
 } // namespace
diff --git a/llvm/unittests/IR/AttributesTest.cpp b/llvm/unittests/IR/AttributesTest.cpp
index b1c455ea57934..2c191264a892e 100644
--- a/llvm/unittests/IR/AttributesTest.cpp
+++ b/llvm/unittests/IR/AttributesTest.cpp
@@ -44,6 +44,7 @@ TEST(Attributes, Ordering) {
   Attribute ByVal = Attribute::get(C, Attribute::ByVal, Type::getInt32Ty(C));
   EXPECT_FALSE(ByVal < Attribute::get(C, Attribute::ZExt));
   EXPECT_TRUE(ByVal < Align4);
+  EXPECT_FALSE(ByVal < ByVal);
 
   AttributeList ASs[] = {AttributeList::get(C, 2, Attribute::ZExt),
                          AttributeList::get(C, 1, Attribute::SExt)};
diff --git a/llvm/unittests/IR/ConstantsTest.cpp b/llvm/unittests/IR/ConstantsTest.cpp
index e20039e8d9c42..f1c1c86293c8d 100644
--- a/llvm/unittests/IR/ConstantsTest.cpp
+++ b/llvm/unittests/IR/ConstantsTest.cpp
@@ -638,5 +638,34 @@ TEST(ConstantsTest, isElementWiseEqual) {
   EXPECT_FALSE(CP00U->isElementWiseEqual(CP00U0));
 }
 
+TEST(ConstantsTest, GetSplatValueRoundTrip) {
+  LLVMContext Context;
+
+  Type *FloatTy = Type::getFloatTy(Context);
+  Type *Int32Ty = Type::getInt32Ty(Context);
+  Type *Int8Ty = Type::getInt8Ty(Context);
+
+  for (unsigned Min : {1, 2, 8}) {
+    ElementCount ScalableEC = {Min, true};
+    ElementCount FixedEC = {Min, false};
+
+    for (auto EC : {ScalableEC, FixedEC}) {
+      for (auto *Ty : {FloatTy, Int32Ty, Int8Ty}) {
+        Constant *Zero = Constant::getNullValue(Ty);
+        Constant *One = Constant::getAllOnesValue(Ty);
+
+        for (auto *C : {Zero, One}) {
+          Constant *Splat = ConstantVector::getSplat(EC, C);
+          ASSERT_NE(nullptr, Splat);
+
+          Constant *SplatVal = Splat->getSplatValue();
+          EXPECT_NE(nullptr, SplatVal);
+          EXPECT_EQ(SplatVal, C);
+        }
+      }
+    }
+  }
+}
+
 }  // end anonymous namespace
 }  // end namespace llvm
diff --git a/llvm/unittests/IR/DominatorTreeTest.cpp b/llvm/unittests/IR/DominatorTreeTest.cpp
index 66e122760ef35..afb620f338e64 100644
--- a/llvm/unittests/IR/DominatorTreeTest.cpp
+++ b/llvm/unittests/IR/DominatorTreeTest.cpp
@@ -805,7 +805,7 @@ TEST(DominatorTree, InsertFromUnreachable) {
   BasicBlock *To = B.getOrAddBlock(LastUpdate->Edge.To);
   PDT.insertEdge(From, To);
   EXPECT_TRUE(PDT.verify());
-  EXPECT_TRUE(PDT.getRoots().size() == 2);
+  EXPECT_EQ(PDT.root_size(), 2UL);
   // Make sure we can use a const pointer with getNode.
   const BasicBlock *BB5 = B.getOrAddBlock("5");
   EXPECT_NE(PDT.getNode(BB5), nullptr);
diff --git a/llvm/unittests/IR/ManglerTest.cpp b/llvm/unittests/IR/ManglerTest.cpp
index f0ef15e5c5cd7..641e34628f5e0 100644
--- a/llvm/unittests/IR/ManglerTest.cpp
+++ b/llvm/unittests/IR/ManglerTest.cpp
@@ -136,4 +136,24 @@ TEST(ManglerTest, WindowsX64) {
             "?vectorcall");
 }
 
+TEST(ManglerTest, XCOFF) {
+  LLVMContext Ctx;
+  DataLayout DL("m:a"); // XCOFF/AIX
+  Module Mod("test", Ctx);
+  Mod.setDataLayout(DL);
+  Mangler Mang;
+  EXPECT_EQ(mangleStr("foo", Mang, DL), "foo");
+  EXPECT_EQ(mangleStr("\01foo", Mang, DL), "foo");
+  EXPECT_EQ(mangleStr("?foo", Mang, DL), "?foo");
+  EXPECT_EQ(mangleFunc("foo", llvm::GlobalValue::ExternalLinkage,
+                       llvm::CallingConv::C, Mod, Mang),
+            "foo");
+  EXPECT_EQ(mangleFunc("?foo", llvm::GlobalValue::ExternalLinkage,
+                       llvm::CallingConv::C, Mod, Mang),
+            "?foo");
+  EXPECT_EQ(mangleFunc("foo", llvm::GlobalValue::PrivateLinkage,
+                       llvm::CallingConv::C, Mod, Mang),
+            "L..foo");
+}
+
 } // end anonymous namespace
diff --git a/llvm/unittests/IR/ModuleTest.cpp b/llvm/unittests/IR/ModuleTest.cpp
index 975427d08d48e..9fb9b8b057a5e 100644
--- a/llvm/unittests/IR/ModuleTest.cpp
+++ b/llvm/unittests/IR/ModuleTest.cpp
@@ -12,6 +12,7 @@
 #include "llvm/IR/ModuleSummaryIndex.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/RandomNumberGenerator.h"
+#include "llvm/Support/SourceMgr.h"
 #include "gtest/gtest.h"
 
 #include <random>
diff --git a/llvm/unittests/MI/LiveIntervalTest.cpp b/llvm/unittests/MI/LiveIntervalTest.cpp
index 5c974ea7461e4..3971d86e82d37 100644
--- a/llvm/unittests/MI/LiveIntervalTest.cpp
+++ b/llvm/unittests/MI/LiveIntervalTest.cpp
@@ -499,6 +499,26 @@ TEST(LiveIntervalTest, TestMoveSubRegDefAcrossUseDefMulti) {
   });
 }
 
+TEST(LiveIntervalTest, TestMoveSubRegUseAcrossMainRangeHole) {
+  liveIntervalTest(R"MIR(
+    %1:sgpr_128 = IMPLICIT_DEF
+  bb.1:
+    %2:sgpr_32 = COPY %1.sub2
+    %3:sgpr_32 = COPY %1.sub1
+    %1.sub2 = COPY %2
+    undef %1.sub0 = IMPLICIT_DEF
+    %1.sub2 = IMPLICIT_DEF
+    S_CBRANCH_SCC1 %bb.1, implicit undef $scc
+    S_BRANCH %bb.2
+  bb.2:
+)MIR", [](MachineFunction &MF, LiveIntervals &LIS) {
+    MachineInstr &MI = getMI(MF, 3, /*BlockNum=*/1);
+    MI.getOperand(0).setIsUndef(false);
+    testHandleMove(MF, LIS, 4, 3, 1);
+    testHandleMove(MF, LIS, 1, 4, 1);
+  });
+}
+
 TEST(LiveIntervalTest, BundleUse) {
   liveIntervalTest(R"MIR(
     %0 = IMPLICIT_DEF
diff --git a/llvm/unittests/ObjectYAML/CMakeLists.txt b/llvm/unittests/ObjectYAML/CMakeLists.txt
index 04a770a46eb38..c20f792657637 100644
--- a/llvm/unittests/ObjectYAML/CMakeLists.txt
+++ b/llvm/unittests/ObjectYAML/CMakeLists.txt
@@ -4,6 +4,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 add_llvm_unittest(ObjectYAMLTests
+  DWARFYAMLTest.cpp
   ELFYAMLTest.cpp
   MinidumpYAMLTest.cpp
   YAML2ObjTest.cpp
diff --git a/llvm/unittests/ObjectYAML/DWARFYAMLTest.cpp b/llvm/unittests/ObjectYAML/DWARFYAMLTest.cpp
new file mode 100644
index 0000000000000..a2addc8f2fa50
--- /dev/null
+++ b/llvm/unittests/ObjectYAML/DWARFYAMLTest.cpp
@@ -0,0 +1,203 @@
+//===- DWARFYAMLTest.cpp - Tests for DWARFYAML.cpp ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ObjectYAML/DWARFYAML.h"
+#include "llvm/ObjectYAML/DWARFEmitter.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+template <class T> static Error parseDWARFYAML(StringRef Yaml, T &Data) {
+  SMDiagnostic GenerateDiag;
+  yaml::Input YIn(
+      Yaml, /*Ctxt=*/nullptr,
+      [](const SMDiagnostic &Diag, void *DiagContext) {
+        *static_cast<SMDiagnostic *>(DiagContext) = Diag;
+      },
+      &GenerateDiag);
+
+  YIn >> Data;
+  if (YIn.error())
+    return createStringError(YIn.error(), GenerateDiag.getMessage());
+
+  return Error::success();
+}
+
+TEST(DebugAddrSection, TestParseDebugAddrYAML) {
+  StringRef Yaml = R"(
+debug_addr:
+  - Format:  DWARF64
+    Length:  0x1234
+    Version: 5
+)";
+  DWARFYAML::Data Data;
+  EXPECT_THAT_ERROR(parseDWARFYAML(Yaml, Data), Succeeded());
+}
+
+TEST(DebugAddrSection, TestMissingVersion) {
+  StringRef Yaml = R"(
+Format: DWARF64
+Length: 0x1234
+)";
+  DWARFYAML::AddrTableEntry AddrTableEntry;
+  EXPECT_THAT_ERROR(parseDWARFYAML(Yaml, AddrTableEntry),
+                    FailedWithMessage("missing required key 'Version'"));
+}
+
+TEST(DebugAddrSection, TestUnexpectedKey) {
+  StringRef Yaml = R"(
+Format:  DWARF64
+Length:  0x1234
+Version: 5
+Blah:    unexpected
+)";
+  DWARFYAML::AddrTableEntry AddrTableEntry;
+  EXPECT_THAT_ERROR(parseDWARFYAML(Yaml, AddrTableEntry),
+                    FailedWithMessage("unknown key 'Blah'"));
+}
+
+TEST(DebugPubSection, TestDebugPubSection) {
+  StringRef Yaml = R"(
+debug_pubnames:
+  Length:
+    TotalLength: 0x1234
+  Version:       2
+  UnitOffset:    0x4321
+  UnitSize:      0x00
+  Entries:
+    - DieOffset:  0x1234
+      Name:       abc
+    - DieOffset:  0x4321
+      Name:       def
+debug_pubtypes:
+  Length:
+    TotalLength: 0x1234
+  Version:       2
+  UnitOffset:    0x4321
+  UnitSize:      0x00
+  Entries:
+    - DieOffset:  0x1234
+      Name:       abc
+    - DieOffset:  0x4321
+      Name:       def
+)";
+  DWARFYAML::Data Data;
+  ASSERT_THAT_ERROR(parseDWARFYAML(Yaml, Data), Succeeded());
+
+  ASSERT_TRUE(Data.PubNames.hasValue());
+  DWARFYAML::PubSection PubNames = Data.PubNames.getValue();
+
+  ASSERT_EQ(PubNames.Entries.size(), 2u);
+  EXPECT_EQ((uint32_t)PubNames.Entries[0].DieOffset, 0x1234u);
+  EXPECT_EQ(PubNames.Entries[0].Name, "abc");
+  EXPECT_EQ((uint32_t)PubNames.Entries[1].DieOffset, 0x4321u);
+  EXPECT_EQ(PubNames.Entries[1].Name, "def");
+
+  ASSERT_TRUE(Data.PubTypes.hasValue());
+  DWARFYAML::PubSection PubTypes = Data.PubTypes.getValue();
+
+  ASSERT_EQ(PubTypes.Entries.size(), 2u);
+  EXPECT_EQ((uint32_t)PubTypes.Entries[0].DieOffset, 0x1234u);
+  EXPECT_EQ(PubTypes.Entries[0].Name, "abc");
+  EXPECT_EQ((uint32_t)PubTypes.Entries[1].DieOffset, 0x4321u);
+  EXPECT_EQ(PubTypes.Entries[1].Name, "def");
+}
+
+TEST(DebugPubSection, TestUnexpectedDescriptor) {
+  StringRef Yaml = R"(
+debug_pubnames:
+  Length:
+    TotalLength: 0x1234
+  Version:       2
+  UnitOffset:    0x4321
+  UnitSize:      0x00
+  Entries:
+    - DieOffset:  0x1234
+      Descriptor: 0x12
+      Name:       abcd
+)";
+  DWARFYAML::Data Data;
+  EXPECT_THAT_ERROR(parseDWARFYAML(Yaml, Data),
+                    FailedWithMessage("unknown key 'Descriptor'"));
+}
+
+TEST(DebugGNUPubSection, TestDebugGNUPubSections) {
+  StringRef Yaml = R"(
+debug_gnu_pubnames:
+  Length:
+    TotalLength: 0x1234
+  Version:       2
+  UnitOffset:    0x4321
+  UnitSize:      0x00
+  Entries:
+    - DieOffset:  0x1234
+      Descriptor: 0x12
+      Name:       abc
+    - DieOffset:  0x4321
+      Descriptor: 0x34
+      Name:       def
+debug_gnu_pubtypes:
+  Length:
+    TotalLength: 0x1234
+  Version:       2
+  UnitOffset:    0x4321
+  UnitSize:      0x00
+  Entries:
+    - DieOffset:  0x1234
+      Descriptor: 0x12
+      Name:       abc
+    - DieOffset:  0x4321
+      Descriptor: 0x34
+      Name:       def
+)";
+  DWARFYAML::Data Data;
+  ASSERT_THAT_ERROR(parseDWARFYAML(Yaml, Data), Succeeded());
+
+  ASSERT_TRUE(Data.GNUPubNames.hasValue());
+  DWARFYAML::PubSection GNUPubNames = Data.GNUPubNames.getValue();
+
+  ASSERT_EQ(GNUPubNames.Entries.size(), 2u);
+  EXPECT_EQ((uint32_t)GNUPubNames.Entries[0].DieOffset, 0x1234u);
+  EXPECT_EQ((uint8_t)GNUPubNames.Entries[0].Descriptor, 0x12);
+  EXPECT_EQ(GNUPubNames.Entries[0].Name, "abc");
+  EXPECT_EQ((uint32_t)GNUPubNames.Entries[1].DieOffset, 0x4321u);
+  EXPECT_EQ((uint8_t)GNUPubNames.Entries[1].Descriptor, 0x34);
+  EXPECT_EQ(GNUPubNames.Entries[1].Name, "def");
+
+  ASSERT_TRUE(Data.GNUPubTypes.hasValue());
+  DWARFYAML::PubSection GNUPubTypes = Data.GNUPubTypes.getValue();
+
+  ASSERT_EQ(GNUPubTypes.Entries.size(), 2u);
+  EXPECT_EQ((uint32_t)GNUPubTypes.Entries[0].DieOffset, 0x1234u);
+  EXPECT_EQ((uint8_t)GNUPubTypes.Entries[0].Descriptor, 0x12);
+  EXPECT_EQ(GNUPubTypes.Entries[0].Name, "abc");
+  EXPECT_EQ((uint32_t)GNUPubTypes.Entries[1].DieOffset, 0x4321u);
+  EXPECT_EQ((uint8_t)GNUPubTypes.Entries[1].Descriptor, 0x34);
+  EXPECT_EQ(GNUPubTypes.Entries[1].Name, "def");
+}
+
+TEST(DebugGNUPubSection, TestMissingDescriptor) {
+  StringRef Yaml = R"(
+debug_gnu_pubnames:
+  Length:
+    TotalLength: 0x1234
+  Version:       2
+  UnitOffset:    0x4321
+  UnitSize:      0x00
+  Entries:
+    - DieOffset: 0x1234
+      Name:      abcd
+)";
+  DWARFYAML::Data Data;
+  EXPECT_THAT_ERROR(parseDWARFYAML(Yaml, Data),
+                    FailedWithMessage("missing required key 'Descriptor'"));
+}
diff --git a/llvm/unittests/Support/JSONTest.cpp b/llvm/unittests/Support/JSONTest.cpp
index 0357e2185c32b..73fc626af8cba 100644
--- a/llvm/unittests/Support/JSONTest.cpp
+++ b/llvm/unittests/Support/JSONTest.cpp
@@ -324,6 +324,9 @@ TEST(JSONTest, Integers) {
           double{-0x4000000000000000},
       },
 
+      // PR46470,
+      // https://developercommunity.visualstudio.com/content/problem/1093399/incorrect-result-when-printing-6917529027641081856.html
+#if !defined(_MSC_VER) || _MSC_VER < 1926
       {
           "Dynamically exact integer. Stored as double, convertible.",
           double{0x6000000000000000},
@@ -331,6 +334,7 @@ TEST(JSONTest, Integers) {
           int64_t{0x6000000000000000},
           double{0x6000000000000000},
       },
+#endif
 
       {
           "Dynamically integer, >64 bits. Stored as double, not convertible.",
diff --git a/llvm/unittests/Support/Path.cpp b/llvm/unittests/Support/Path.cpp
index a0ddeb6c8c233..6a228f0939873 100644
--- a/llvm/unittests/Support/Path.cpp
+++ b/llvm/unittests/Support/Path.cpp
@@ -424,7 +424,8 @@ TEST(Support, HomeDirectory) {
   }
 }
 
-#ifdef LLVM_ON_UNIX
+// Apple has their own solution for this.
+#if defined(LLVM_ON_UNIX) && !defined(__APPLE__)
 TEST(Support, HomeDirectoryWithNoEnv) {
   WithEnv Env("HOME", nullptr);
 
@@ -438,6 +439,26 @@ TEST(Support, HomeDirectoryWithNoEnv) {
   EXPECT_EQ(PwDir, HomeDir);
 }
 
+TEST(Support, ConfigDirectoryWithEnv) {
+  WithEnv Env("XDG_CONFIG_HOME", "/xdg/config");
+
+  SmallString<128> ConfigDir;
+  EXPECT_TRUE(path::user_config_directory(ConfigDir));
+  EXPECT_EQ("/xdg/config", ConfigDir);
+}
+
+TEST(Support, ConfigDirectoryNoEnv) {
+  WithEnv Env("XDG_CONFIG_HOME", nullptr);
+
+  SmallString<128> Fallback;
+  ASSERT_TRUE(path::home_directory(Fallback));
+  path::append(Fallback, ".config");
+
+  SmallString<128> CacheDir;
+  EXPECT_TRUE(path::user_config_directory(CacheDir));
+  EXPECT_EQ(Fallback, CacheDir);
+}
+
 TEST(Support, CacheDirectoryWithEnv) {
   WithEnv Env("XDG_CACHE_HOME", "/xdg/cache");
 
@@ -459,7 +480,29 @@ TEST(Support, CacheDirectoryNoEnv) {
 }
 #endif
 
+#ifdef __APPLE__
+TEST(Support, ConfigDirectory) {
+  SmallString<128> Fallback;
+  ASSERT_TRUE(path::home_directory(Fallback));
+  path::append(Fallback, "Library/Preferences");
+
+  SmallString<128> ConfigDir;
+  EXPECT_TRUE(path::user_config_directory(ConfigDir));
+  EXPECT_EQ(Fallback, ConfigDir);
+}
+#endif
+
 #ifdef _WIN32
+TEST(Support, ConfigDirectory) {
+  std::string Expected = getEnvWin(L"LOCALAPPDATA");
+  // Do not try to test it if we don't know what to expect.
+  if (!Expected.empty()) {
+    SmallString<128> CacheDir;
+    EXPECT_TRUE(path::user_config_directory(CacheDir));
+    EXPECT_EQ(Expected, CacheDir);
+  }
+}
+
 TEST(Support, CacheDirectory) {
   std::string Expected = getEnvWin(L"LOCALAPPDATA");
   // Do not try to test it if we don't know what to expect.
diff --git a/llvm/unittests/Support/TargetParserTest.cpp b/llvm/unittests/Support/TargetParserTest.cpp
index 4da191f823cf6..0127cb6ae0099 100644
--- a/llvm/unittests/Support/TargetParserTest.cpp
+++ b/llvm/unittests/Support/TargetParserTest.cpp
@@ -256,6 +256,24 @@ TEST(TargetParserTest, testARMCPU) {
                          ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_FP16 |
                          ARM::AEK_RAS | ARM::AEK_DOTPROD,
                          "8.2-A"));
+  EXPECT_TRUE(testARMCPU("cortex-a77", "armv8.2-a", "crypto-neon-fp-armv8",
+                         ARM::AEK_CRC | ARM::AEK_SEC | ARM::AEK_MP |
+                         ARM::AEK_VIRT | ARM::AEK_HWDIVARM |
+                         ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_FP16 |
+                         ARM::AEK_RAS | ARM::AEK_DOTPROD,
+                         "8.2-A"));
+  EXPECT_TRUE(testARMCPU("cortex-a78", "armv8.2-a", "crypto-neon-fp-armv8",
+                         ARM::AEK_DOTPROD | ARM::AEK_FP16 |
+                         ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT |
+                         ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB |
+                         ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS,
+                         "8.2-A"));
+  EXPECT_TRUE(testARMCPU("cortex-x1", "armv8.2-a", "crypto-neon-fp-armv8",
+                         ARM::AEK_RAS | ARM::AEK_FP16 | ARM::AEK_DOTPROD |
+                         ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT |
+                         ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB |
+                         ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS,
+                         "8.2-A"));
   EXPECT_TRUE(testARMCPU("neoverse-n1", "armv8.2-a", "crypto-neon-fp-armv8",
                         ARM::AEK_CRC | ARM::AEK_SEC | ARM::AEK_MP |
                         ARM::AEK_VIRT | ARM::AEK_HWDIVARM |
@@ -304,7 +322,7 @@ TEST(TargetParserTest, testARMCPU) {
                          "7-S"));
 }
 
-static constexpr unsigned NumARMCPUArchs = 86;
+static constexpr unsigned NumARMCPUArchs = 89;
 
 TEST(TargetParserTest, testARMCPUArchList) {
   SmallVector<StringRef, NumARMCPUArchs> List;
@@ -852,6 +870,26 @@ TEST(TargetParserTest, testAArch64CPU) {
       AArch64::AEK_RDM | AArch64::AEK_SIMD | AArch64::AEK_RAS |
       AArch64::AEK_LSE | AArch64::AEK_FP16 | AArch64::AEK_DOTPROD |
       AArch64::AEK_RCPC| AArch64::AEK_SSBS, "8.2-A"));
+  EXPECT_TRUE(testAArch64CPU(
+      "cortex-a77", "armv8.2-a", "crypto-neon-fp-armv8",
+      AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP |
+      AArch64::AEK_RDM | AArch64::AEK_SIMD | AArch64::AEK_RAS |
+      AArch64::AEK_LSE | AArch64::AEK_FP16 | AArch64::AEK_DOTPROD |
+      AArch64::AEK_RCPC | AArch64::AEK_SSBS, "8.2-A"));
+  EXPECT_TRUE(testAArch64CPU(
+      "cortex-a78", "armv8.2-a", "crypto-neon-fp-armv8",
+      AArch64::AEK_CRC | AArch64::AEK_CRYPTO  | AArch64::AEK_FP |
+      AArch64::AEK_RDM | AArch64::AEK_SIMD | AArch64::AEK_RAS |
+      AArch64::AEK_LSE | AArch64::AEK_FP16 | AArch64::AEK_DOTPROD |
+      AArch64::AEK_RCPC | AArch64::AEK_SSBS,
+      "8.2-A"));
+  EXPECT_TRUE(testAArch64CPU(
+      "cortex-x1", "armv8.2-a", "crypto-neon-fp-armv8",
+      AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP |
+      AArch64::AEK_RDM | AArch64::AEK_SIMD | AArch64::AEK_RAS |
+      AArch64::AEK_LSE | AArch64::AEK_FP16 | AArch64::AEK_DOTPROD |
+      AArch64::AEK_RCPC | AArch64::AEK_SSBS,
+      "8.2-A"));
   EXPECT_TRUE(testAArch64CPU(
       "cyclone", "armv8-a", "crypto-neon-fp-armv8",
       AArch64::AEK_CRYPTO | AArch64::AEK_FP | AArch64::AEK_SIMD, "8-A"));
@@ -990,7 +1028,7 @@ TEST(TargetParserTest, testAArch64CPU) {
       "8.2-A"));
 }
 
-static constexpr unsigned NumAArch64CPUArchs = 39;
+static constexpr unsigned NumAArch64CPUArchs = 42;
 
 TEST(TargetParserTest, testAArch64CPUArchList) {
   SmallVector<StringRef, NumAArch64CPUArchs> List;
diff --git a/llvm/unittests/Support/YAMLIOTest.cpp b/llvm/unittests/Support/YAMLIOTest.cpp
index d86489cf75604..492d854ef8128 100644
--- a/llvm/unittests/Support/YAMLIOTest.cpp
+++ b/llvm/unittests/Support/YAMLIOTest.cpp
@@ -285,10 +285,8 @@ TEST(YAMLIO, MultilineStrings) {
     YOut << Original;
   }
   auto Expected = "---\n"
-                  "str1:            'a multiline string\n"
-                  "foobarbaz'\n"
-                  "str2:            'another one\r"
-                  "foobarbaz'\n"
+                  "str1:            \"a multiline string\\nfoobarbaz\"\n"
+                  "str2:            \"another one\\rfoobarbaz\"\n"
                   "str3:            a one-line string\n"
                   "...\n";
   ASSERT_EQ(Serialized, Expected);
diff --git a/llvm/unittests/Support/formatted_raw_ostream_test.cpp b/llvm/unittests/Support/formatted_raw_ostream_test.cpp
index 0fe0869922a14..5c57f1f18700a 100644
--- a/llvm/unittests/Support/formatted_raw_ostream_test.cpp
+++ b/llvm/unittests/Support/formatted_raw_ostream_test.cpp
@@ -29,4 +29,143 @@ TEST(formatted_raw_ostreamTest, Test_Tell) {
   }
 }
 
+TEST(formatted_raw_ostreamTest, Test_LineColumn) {
+  // Test tracking of line and column numbers in a stream.
+  SmallString<128> A;
+  raw_svector_ostream B(A);
+  formatted_raw_ostream C(B);
+
+  EXPECT_EQ(0U, C.getLine());
+  EXPECT_EQ(0U, C.getColumn());
+
+  C << "a";
+  EXPECT_EQ(0U, C.getLine());
+  EXPECT_EQ(1U, C.getColumn());
+
+  C << "bcdef";
+  EXPECT_EQ(0U, C.getLine());
+  EXPECT_EQ(6U, C.getColumn());
+
+  // '\n' increments line number, sets column to zero.
+  C << "\n";
+  EXPECT_EQ(1U, C.getLine());
+  EXPECT_EQ(0U, C.getColumn());
+
+  // '\r sets column to zero without changing line number
+  C << "foo\r";
+  EXPECT_EQ(1U, C.getLine());
+  EXPECT_EQ(0U, C.getColumn());
+
+  // '\t' advances column to the next multiple of 8.
+  // FIXME: If the column number is already a multiple of 8 this will do
+  // nothing, is this behaviour correct?
+  C << "1\t";
+  EXPECT_EQ(8U, C.getColumn());
+  C << "\t";
+  EXPECT_EQ(8U, C.getColumn());
+  C << "1234567\t";
+  EXPECT_EQ(16U, C.getColumn());
+  EXPECT_EQ(1U, C.getLine());
+}
+
+TEST(formatted_raw_ostreamTest, Test_Flush) {
+  // Flushing the buffer causes the characters in the buffer to be scanned
+  // before the buffer is emptied, so line and column numbers will still be
+  // tracked properly.
+  SmallString<128> A;
+  raw_svector_ostream B(A);
+  B.SetBufferSize(32);
+  formatted_raw_ostream C(B);
+
+  C << "\nabc";
+  EXPECT_EQ(4U, C.GetNumBytesInBuffer());
+  C.flush();
+  EXPECT_EQ(1U, C.getLine());
+  EXPECT_EQ(3U, C.getColumn());
+  EXPECT_EQ(0U, C.GetNumBytesInBuffer());
+}
+
+TEST(formatted_raw_ostreamTest, Test_UTF8) {
+  SmallString<128> A;
+  raw_svector_ostream B(A);
+  B.SetBufferSize(32);
+  formatted_raw_ostream C(B);
+
+  // U+00A0 Non-breaking space: encoded as two bytes, but only one column wide.
+  C << u8"\u00a0";
+  EXPECT_EQ(0U, C.getLine());
+  EXPECT_EQ(1U, C.getColumn());
+  EXPECT_EQ(2U, C.GetNumBytesInBuffer());
+
+  // U+2468 CIRCLED DIGIT NINE: encoded as three bytes, but only one column
+  // wide.
+  C << u8"\u2468";
+  EXPECT_EQ(0U, C.getLine());
+  EXPECT_EQ(2U, C.getColumn());
+  EXPECT_EQ(5U, C.GetNumBytesInBuffer());
+
+  // U+00010000 LINEAR B SYLLABLE B008 A: encoded as four bytes, but only one
+  // column wide.
+  C << u8"\U00010000";
+  EXPECT_EQ(0U, C.getLine());
+  EXPECT_EQ(3U, C.getColumn());
+  EXPECT_EQ(9U, C.GetNumBytesInBuffer());
+
+  // U+55B5, CJK character, encodes as three bytes, takes up two columns.
+  C << u8"\u55b5";
+  EXPECT_EQ(0U, C.getLine());
+  EXPECT_EQ(5U, C.getColumn());
+  EXPECT_EQ(12U, C.GetNumBytesInBuffer());
+
+  // U+200B, zero-width space, encoded as three bytes but has no effect on the
+  // column or line number.
+  C << u8"\u200b";
+  EXPECT_EQ(0U, C.getLine());
+  EXPECT_EQ(5U, C.getColumn());
+  EXPECT_EQ(15U, C.GetNumBytesInBuffer());
+}
+
+TEST(formatted_raw_ostreamTest, Test_UTF8Buffered) {
+  SmallString<128> A;
+  raw_svector_ostream B(A);
+  B.SetBufferSize(4);
+  formatted_raw_ostream C(B);
+
+  // U+2468 encodes as three bytes, so will cause the buffer to be flushed after
+  // the first byte (4 byte buffer, 3 bytes already written). We need to save
+  // the first part of the UTF-8 encoding until after the buffer is cleared and
+  // the remaining two bytes are written, at which point we can check the
+  // display width. In this case the display width is 1, so we end at column 4,
+  // with 6 bytes written into total, 2 of which are in the buffer.
+  C << u8"123\u2468";
+  EXPECT_EQ(0U, C.getLine());
+  EXPECT_EQ(4U, C.getColumn());
+  EXPECT_EQ(2U, C.GetNumBytesInBuffer());
+  C.flush();
+  EXPECT_EQ(6U, A.size());
+
+  // Same as above, but with a CJK character which displays as two columns.
+  C << u8"123\u55b5";
+  EXPECT_EQ(0U, C.getLine());
+  EXPECT_EQ(9U, C.getColumn());
+  EXPECT_EQ(2U, C.GetNumBytesInBuffer());
+  C.flush();
+  EXPECT_EQ(12U, A.size());
+}
+
+TEST(formatted_raw_ostreamTest, Test_UTF8TinyBuffer) {
+  SmallString<128> A;
+  raw_svector_ostream B(A);
+  B.SetBufferSize(1);
+  formatted_raw_ostream C(B);
+
+  // The stream has a one-byte buffer, so it gets flushed multiple times while
+  // printing a single Unicode character.
+  C << u8"\u2468";
+  EXPECT_EQ(0U, C.getLine());
+  EXPECT_EQ(1U, C.getColumn());
+  EXPECT_EQ(0U, C.GetNumBytesInBuffer());
+  C.flush();
+  EXPECT_EQ(3U, A.size());
+}
 }
diff --git a/llvm/unittests/Target/ARM/MachineInstrTest.cpp b/llvm/unittests/Target/ARM/MachineInstrTest.cpp
index 51c9d739f916d..3e5403f209887 100644
--- a/llvm/unittests/Target/ARM/MachineInstrTest.cpp
+++ b/llvm/unittests/Target/ARM/MachineInstrTest.cpp
@@ -973,13 +973,110 @@ TEST(MachineInstrValidTailPredication, IsCorrect) {
 TEST(MachineInstr, HasSideEffects) {
   using namespace ARM;
   std::set<unsigned> UnpredictableOpcodes = {
-      MVE_VCTP8,     MVE_VCTP16,    MVE_VCTP32,    MVE_VCTP64,    MVE_VPST,
-      MVE_VPTv16i8,  MVE_VPTv8i16,  MVE_VPTv4i32,  MVE_VPTv16i8r, MVE_VPTv8i16r,
-      MVE_VPTv4i32r, MVE_VPTv16s8,  MVE_VPTv8s16,  MVE_VPTv4s32,  MVE_VPTv16s8r,
-      MVE_VPTv8s16r, MVE_VPTv4s32r, MVE_VPTv16u8,  MVE_VPTv8u16,  MVE_VPTv4u32,
-      MVE_VPTv16u8r, MVE_VPTv8u16r, MVE_VPTv4u32r, MVE_VPTv8f16,  MVE_VPTv4f32,
-      MVE_VPTv8f16r, MVE_VPTv4f32r, MVE_VADC,      MVE_VADCI,     MVE_VSBC,
-      MVE_VSBCI,     MVE_VSHLC,
+      // MVE Instructions
+      MVE_VCTP8,
+      MVE_VCTP16,
+      MVE_VCTP32,
+      MVE_VCTP64,
+      MVE_VPST,
+      MVE_VPTv16i8,
+      MVE_VPTv8i16,
+      MVE_VPTv4i32,
+      MVE_VPTv16i8r,
+      MVE_VPTv8i16r,
+      MVE_VPTv4i32r,
+      MVE_VPTv16s8,
+      MVE_VPTv8s16,
+      MVE_VPTv4s32,
+      MVE_VPTv16s8r,
+      MVE_VPTv8s16r,
+      MVE_VPTv4s32r,
+      MVE_VPTv16u8,
+      MVE_VPTv8u16,
+      MVE_VPTv4u32,
+      MVE_VPTv16u8r,
+      MVE_VPTv8u16r,
+      MVE_VPTv4u32r,
+      MVE_VPTv8f16,
+      MVE_VPTv4f32,
+      MVE_VPTv8f16r,
+      MVE_VPTv4f32r,
+      MVE_VADC,
+      MVE_VADCI,
+      MVE_VSBC,
+      MVE_VSBCI,
+      MVE_VSHLC,
+      // FP Instructions
+      FLDMXIA,
+      FLDMXDB_UPD,
+      FLDMXIA_UPD,
+      FSTMXDB_UPD,
+      FSTMXIA,
+      FSTMXIA_UPD,
+      VLDR_FPCXTNS_off,
+      VLDR_FPCXTNS_off,
+      VLDR_FPCXTNS_post,
+      VLDR_FPCXTNS_pre,
+      VLDR_FPCXTS_off,
+      VLDR_FPCXTS_post,
+      VLDR_FPCXTS_pre,
+      VLDR_FPSCR_NZCVQC_off,
+      VLDR_FPSCR_NZCVQC_post,
+      VLDR_FPSCR_NZCVQC_pre,
+      VLDR_FPSCR_off,
+      VLDR_FPSCR_post,
+      VLDR_FPSCR_pre,
+      VLDR_P0_off,
+      VLDR_P0_post,
+      VLDR_P0_pre,
+      VLDR_VPR_off,
+      VLDR_VPR_post,
+      VLDR_VPR_pre,
+      VLLDM,
+      VLSTM,
+      VMRS,
+      VMRS_FPCXTNS,
+      VMRS_FPCXTS,
+      VMRS_FPEXC,
+      VMRS_FPINST,
+      VMRS_FPINST2,
+      VMRS_FPSCR_NZCVQC,
+      VMRS_FPSID,
+      VMRS_MVFR0,
+      VMRS_MVFR1,
+      VMRS_MVFR2,
+      VMRS_P0,
+      VMRS_VPR,
+      VMSR,
+      VMSR_FPCXTNS,
+      VMSR_FPCXTS,
+      VMSR_FPEXC,
+      VMSR_FPINST,
+      VMSR_FPINST2,
+      VMSR_FPSCR_NZCVQC,
+      VMSR_FPSID,
+      VMSR_P0,
+      VMSR_VPR,
+      VSCCLRMD,
+      VSCCLRMS,
+      VSTR_FPCXTNS_off,
+      VSTR_FPCXTNS_post,
+      VSTR_FPCXTNS_pre,
+      VSTR_FPCXTS_off,
+      VSTR_FPCXTS_post,
+      VSTR_FPCXTS_pre,
+      VSTR_FPSCR_NZCVQC_off,
+      VSTR_FPSCR_NZCVQC_post,
+      VSTR_FPSCR_NZCVQC_pre,
+      VSTR_FPSCR_off,
+      VSTR_FPSCR_post,
+      VSTR_FPSCR_pre,
+      VSTR_P0_off,
+      VSTR_P0_post,
+      VSTR_P0_pre,
+      VSTR_VPR_off,
+      VSTR_VPR_post,
+      VSTR_VPR_pre,
   };
 
   LLVMInitializeARMTargetInfo();
@@ -1006,7 +1103,8 @@ TEST(MachineInstr, HasSideEffects) {
 
   for (unsigned Op = 0; Op < ARM::INSTRUCTION_LIST_END; ++Op) {
     const MCInstrDesc &Desc = TII->get(Op);
-    if ((Desc.TSFlags & ARMII::DomainMask) != ARMII::DomainMVE)
+    if ((Desc.TSFlags & ARMII::DomainMask) != ARMII::DomainMVE &&
+        (Desc.TSFlags & ARMII::DomainMask) != ARMII::DomainVFP)
       continue;
     if (UnpredictableOpcodes.count(Op))
       continue;
diff --git a/llvm/unittests/Transforms/Utils/CloningTest.cpp b/llvm/unittests/Transforms/Utils/CloningTest.cpp
index 28ad4bc880772..c8bfd288a4f09 100644
--- a/llvm/unittests/Transforms/Utils/CloningTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CloningTest.cpp
@@ -9,6 +9,7 @@
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/AsmParser/Parser.h"
@@ -659,6 +660,65 @@ static int GetDICompileUnitCount(const Module& M) {
   return 0;
 }
 
+TEST(CloneFunction, CloneEmptyFunction) {
+  StringRef ImplAssembly = R"(
+    define void @foo() {
+      ret void
+    }
+    declare void @bar()
+  )";
+
+  LLVMContext Context;
+  SMDiagnostic Error;
+
+  auto ImplModule = parseAssemblyString(ImplAssembly, Error, Context);
+  EXPECT_TRUE(ImplModule != nullptr);
+  auto *ImplFunction = ImplModule->getFunction("foo");
+  EXPECT_TRUE(ImplFunction != nullptr);
+  auto *DeclFunction = ImplModule->getFunction("bar");
+  EXPECT_TRUE(DeclFunction != nullptr);
+
+  ValueToValueMapTy VMap;
+  SmallVector<ReturnInst *, 8> Returns;
+  ClonedCodeInfo CCI;
+  CloneFunctionInto(DeclFunction, ImplFunction, VMap, true, Returns, "", &CCI);
+
+  EXPECT_FALSE(verifyModule(*ImplModule, &errs()));
+  EXPECT_FALSE(CCI.ContainsCalls);
+  EXPECT_FALSE(CCI.ContainsDynamicAllocas);
+}
+
+TEST(CloneFunction, CloneFunctionWithInalloca) {
+  StringRef ImplAssembly = R"(
+    declare void @a(i32* inalloca)
+    define void @foo() {
+      %a = alloca inalloca i32
+      call void @a(i32* inalloca %a)
+      ret void
+    }
+    declare void @bar()
+  )";
+
+  LLVMContext Context;
+  SMDiagnostic Error;
+
+  auto ImplModule = parseAssemblyString(ImplAssembly, Error, Context);
+  EXPECT_TRUE(ImplModule != nullptr);
+  auto *ImplFunction = ImplModule->getFunction("foo");
+  EXPECT_TRUE(ImplFunction != nullptr);
+  auto *DeclFunction = ImplModule->getFunction("bar");
+  EXPECT_TRUE(DeclFunction != nullptr);
+
+  ValueToValueMapTy VMap;
+  SmallVector<ReturnInst *, 8> Returns;
+  ClonedCodeInfo CCI;
+  CloneFunctionInto(DeclFunction, ImplFunction, VMap, true, Returns, "", &CCI);
+
+  EXPECT_FALSE(verifyModule(*ImplModule, &errs()));
+  EXPECT_TRUE(CCI.ContainsCalls);
+  EXPECT_TRUE(CCI.ContainsDynamicAllocas);
+}
+
 TEST(CloneFunction, CloneFunctionToDifferentModule) {
   StringRef ImplAssembly = R"(
     define void @foo() {
diff --git a/llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp b/llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp
index 27e140d81a98b..f9f1b235d0d08 100644
--- a/llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp
@@ -505,48 +505,50 @@ TEST(CodeMoverUtils, IsSafeToMoveTest1) {
         // Can move after CI_safecall, as it does not throw, not synchronize, or
         // must return.
         EXPECT_TRUE(isSafeToMoveBefore(*CI_safecall->getPrevNode(),
-                                       *CI_safecall->getNextNode(), DT, PDT,
-                                       DI));
+                                       *CI_safecall->getNextNode(), DT, &PDT,
+                                       &DI));
 
         // Cannot move CI_unsafecall, as it may throw.
         EXPECT_FALSE(isSafeToMoveBefore(*CI_unsafecall->getNextNode(),
-                                        *CI_unsafecall, DT, PDT, DI));
+                                        *CI_unsafecall, DT, &PDT, &DI));
 
         // Moving instruction to non control flow equivalent places are not
         // supported.
-        EXPECT_FALSE(
-            isSafeToMoveBefore(*SI_A5, *Entry->getTerminator(), DT, PDT, DI));
+        EXPECT_FALSE(isSafeToMoveBefore(*SI_A5, *Entry->getTerminator(), DT,
+                                        &PDT, &DI));
 
         // Moving PHINode is not supported.
         EXPECT_FALSE(isSafeToMoveBefore(PN, *PN.getNextNode()->getNextNode(),
-                                        DT, PDT, DI));
+                                        DT, &PDT, &DI));
 
         // Cannot move non-PHINode before PHINode.
-        EXPECT_FALSE(isSafeToMoveBefore(*PN.getNextNode(), PN, DT, PDT, DI));
+        EXPECT_FALSE(isSafeToMoveBefore(*PN.getNextNode(), PN, DT, &PDT, &DI));
 
         // Moving Terminator is not supported.
         EXPECT_FALSE(isSafeToMoveBefore(*Entry->getTerminator(),
-                                        *PN.getNextNode(), DT, PDT, DI));
+                                        *PN.getNextNode(), DT, &PDT, &DI));
 
         // Cannot move %arrayidx_A after SI, as SI is its user.
         EXPECT_FALSE(isSafeToMoveBefore(*SI->getPrevNode(), *SI->getNextNode(),
-                                        DT, PDT, DI));
+                                        DT, &PDT, &DI));
 
         // Cannot move SI before %arrayidx_A, as %arrayidx_A is its operand.
-        EXPECT_FALSE(isSafeToMoveBefore(*SI, *SI->getPrevNode(), DT, PDT, DI));
+        EXPECT_FALSE(
+            isSafeToMoveBefore(*SI, *SI->getPrevNode(), DT, &PDT, &DI));
 
         // Cannot move LI2 after SI_A6, as there is a flow dependence.
         EXPECT_FALSE(
-            isSafeToMoveBefore(*LI2, *SI_A6->getNextNode(), DT, PDT, DI));
+            isSafeToMoveBefore(*LI2, *SI_A6->getNextNode(), DT, &PDT, &DI));
 
         // Cannot move SI after LI1, as there is a anti dependence.
-        EXPECT_FALSE(isSafeToMoveBefore(*SI, *LI1->getNextNode(), DT, PDT, DI));
+        EXPECT_FALSE(
+            isSafeToMoveBefore(*SI, *LI1->getNextNode(), DT, &PDT, &DI));
 
         // Cannot move SI_A5 after SI, as there is a output dependence.
-        EXPECT_FALSE(isSafeToMoveBefore(*SI_A5, *LI1, DT, PDT, DI));
+        EXPECT_FALSE(isSafeToMoveBefore(*SI_A5, *LI1, DT, &PDT, &DI));
 
         // Can move LI2 before LI1, as there is only an input dependence.
-        EXPECT_TRUE(isSafeToMoveBefore(*LI2, *LI1, DT, PDT, DI));
+        EXPECT_TRUE(isSafeToMoveBefore(*LI2, *LI1, DT, &PDT, &DI));
       });
 }
 
@@ -578,11 +580,11 @@ TEST(CodeMoverUtils, IsSafeToMoveTest2) {
         Instruction *SubInst = getInstructionByName(F, "sub");
 
         // Cannot move as %user uses %add and %sub doesn't dominates %user.
-        EXPECT_FALSE(isSafeToMoveBefore(*AddInst, *SubInst, DT, PDT, DI));
+        EXPECT_FALSE(isSafeToMoveBefore(*AddInst, *SubInst, DT, &PDT, &DI));
 
         // Cannot move as %sub_op0 is an operand of %sub and %add doesn't
         // dominates %sub_op0.
-        EXPECT_FALSE(isSafeToMoveBefore(*SubInst, *AddInst, DT, PDT, DI));
+        EXPECT_FALSE(isSafeToMoveBefore(*SubInst, *AddInst, DT, &PDT, &DI));
       });
 }
 
@@ -611,7 +613,7 @@ TEST(CodeMoverUtils, IsSafeToMoveTest3) {
 
         // Can move as the incoming block of %inc for %i (%for.latch) dominated
         // by %cmp.
-        EXPECT_TRUE(isSafeToMoveBefore(*IncInst, *CmpInst, DT, PDT, DI));
+        EXPECT_TRUE(isSafeToMoveBefore(*IncInst, *CmpInst, DT, &PDT, &DI));
       });
 }
 
@@ -643,10 +645,10 @@ TEST(CodeMoverUtils, IsSafeToMoveTest4) {
         Instruction *SubInst = getInstructionByName(F, "sub");
 
         // Cannot move as %user uses %add and %sub doesn't dominates %user.
-        EXPECT_FALSE(isSafeToMoveBefore(*AddInst, *SubInst, DT, PDT, DI));
+        EXPECT_FALSE(isSafeToMoveBefore(*AddInst, *SubInst, DT, &PDT, &DI));
 
         // Cannot move as %sub_op0 is an operand of %sub and %add doesn't
         // dominates %sub_op0.
-        EXPECT_FALSE(isSafeToMoveBefore(*SubInst, *AddInst, DT, PDT, DI));
+        EXPECT_FALSE(isSafeToMoveBefore(*SubInst, *AddInst, DT, &PDT, &DI));
       });
 }
diff --git a/llvm/unittests/Transforms/Utils/LoopUtilsTest.cpp b/llvm/unittests/Transforms/Utils/LoopUtilsTest.cpp
index 97618def5a47e..aa0d72cf8bb33 100644
--- a/llvm/unittests/Transforms/Utils/LoopUtilsTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LoopUtilsTest.cpp
@@ -10,6 +10,7 @@
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/Support/SourceMgr.h"
diff --git a/llvm/utils/FileCheck/FileCheck.cpp b/llvm/utils/FileCheck/FileCheck.cpp
index 8e41365de81a8..fa79c5e894894 100644
--- a/llvm/utils/FileCheck/FileCheck.cpp
+++ b/llvm/utils/FileCheck/FileCheck.cpp
@@ -97,12 +97,12 @@ static cl::opt<bool> AllowDeprecatedDagOverlap(
              "non-overlapping CHECK-DAG implementation.\n"));
 
 static cl::opt<bool> Verbose(
-    "v", cl::init(false),
+    "v", cl::init(false), cl::ZeroOrMore,
     cl::desc("Print directive pattern matches, or add them to the input dump\n"
              "if enabled.\n"));
 
 static cl::opt<bool> VerboseVerbose(
-    "vv", cl::init(false),
+    "vv", cl::init(false), cl::ZeroOrMore,
     cl::desc("Print information helpful in diagnosing internal FileCheck\n"
              "issues, or add it to the input dump if enabled.  Implies\n"
              "-v.\n"));
@@ -110,7 +110,6 @@ static cl::opt<bool> VerboseVerbose(
 // The order of DumpInputValue members affects their precedence, as documented
 // for -dump-input below.
 enum DumpInputValue {
-  DumpInputDefault,
   DumpInputNever,
   DumpInputFail,
   DumpInputAlways,
@@ -122,14 +121,47 @@ static cl::list<DumpInputValue> DumpInputs(
     cl::desc("Dump input to stderr, adding annotations representing\n"
              "currently enabled diagnostics.  When there are multiple\n"
              "occurrences of this option, the <value> that appears earliest\n"
-             "in the list below has precedence.\n"),
+             "in the list below has precedence.  The default is 'fail'.\n"),
     cl::value_desc("mode"),
-    cl::values(clEnumValN(DumpInputHelp, "help",
-                          "Explain dump format and quit"),
+    cl::values(clEnumValN(DumpInputHelp, "help", "Explain input dump and quit"),
                clEnumValN(DumpInputAlways, "always", "Always dump input"),
                clEnumValN(DumpInputFail, "fail", "Dump input on failure"),
                clEnumValN(DumpInputNever, "never", "Never dump input")));
 
+// The order of DumpInputFilterValue members affects their precedence, as
+// documented for -dump-input-filter below.
+enum DumpInputFilterValue {
+  DumpInputFilterError,
+  DumpInputFilterAnnotation,
+  DumpInputFilterAnnotationFull,
+  DumpInputFilterAll
+};
+
+static cl::list<DumpInputFilterValue> DumpInputFilters(
+    "dump-input-filter",
+    cl::desc("In the dump requested by -dump-input, print only input lines of\n"
+             "kind <value> plus any context specified by -dump-input-context.\n"
+             "When there are multiple occurrences of this option, the <value>\n"
+             "that appears earliest in the list below has precedence.  The\n"
+             "default is 'error' when -dump-input=fail, and it's 'all' when\n"
+             "-dump-input=always.\n"),
+    cl::values(clEnumValN(DumpInputFilterAll, "all", "All input lines"),
+               clEnumValN(DumpInputFilterAnnotationFull, "annotation-full",
+                          "Input lines with annotations"),
+               clEnumValN(DumpInputFilterAnnotation, "annotation",
+                          "Input lines with starting points of annotations"),
+               clEnumValN(DumpInputFilterError, "error",
+                          "Input lines with starting points of error "
+                          "annotations")));
+
+static cl::list<unsigned> DumpInputContexts(
+    "dump-input-context", cl::value_desc("N"),
+    cl::desc("In the dump requested by -dump-input, print <N> input lines\n"
+             "before and <N> input lines after any lines specified by\n"
+             "-dump-input-filter.  When there are multiple occurrences of\n"
+             "this option, the largest specified <N> has precedence.  The\n"
+             "default is 5.\n"));
+
 typedef cl::list<std::string>::const_iterator prefix_iterator;
 
 
@@ -152,10 +184,15 @@ struct MarkerStyle {
   raw_ostream::Colors Color;
   /// A note to follow the marker, or empty string if none.
   std::string Note;
+  /// Does this marker indicate inclusion by -dump-input-filter=error?
+  bool FiltersAsError;
   MarkerStyle() {}
   MarkerStyle(char Lead, raw_ostream::Colors Color,
-              const std::string &Note = "")
-      : Lead(Lead), Color(Color), Note(Note) {}
+              const std::string &Note = "", bool FiltersAsError = false)
+      : Lead(Lead), Color(Color), Note(Note), FiltersAsError(FiltersAsError) {
+    assert((!FiltersAsError || !Note.empty()) &&
+           "expected error diagnostic to have note");
+  }
 };
 
 static MarkerStyle GetMarker(FileCheckDiag::MatchType MatchTy) {
@@ -163,26 +200,46 @@ static MarkerStyle GetMarker(FileCheckDiag::MatchType MatchTy) {
   case FileCheckDiag::MatchFoundAndExpected:
     return MarkerStyle('^', raw_ostream::GREEN);
   case FileCheckDiag::MatchFoundButExcluded:
-    return MarkerStyle('!', raw_ostream::RED, "error: no match expected");
+    return MarkerStyle('!', raw_ostream::RED, "error: no match expected",
+                       /*FiltersAsError=*/true);
   case FileCheckDiag::MatchFoundButWrongLine:
-    return MarkerStyle('!', raw_ostream::RED, "error: match on wrong line");
+    return MarkerStyle('!', raw_ostream::RED, "error: match on wrong line",
+                       /*FiltersAsError=*/true);
   case FileCheckDiag::MatchFoundButDiscarded:
     return MarkerStyle('!', raw_ostream::CYAN,
                        "discard: overlaps earlier match");
   case FileCheckDiag::MatchNoneAndExcluded:
     return MarkerStyle('X', raw_ostream::GREEN);
   case FileCheckDiag::MatchNoneButExpected:
-    return MarkerStyle('X', raw_ostream::RED, "error: no match found");
+    return MarkerStyle('X', raw_ostream::RED, "error: no match found",
+                       /*FiltersAsError=*/true);
   case FileCheckDiag::MatchFuzzy:
-    return MarkerStyle('?', raw_ostream::MAGENTA, "possible intended match");
+    return MarkerStyle('?', raw_ostream::MAGENTA, "possible intended match",
+                       /*FiltersAsError=*/true);
   }
   llvm_unreachable_internal("unexpected match type");
 }
 
 static void DumpInputAnnotationHelp(raw_ostream &OS) {
   OS << "The following description was requested by -dump-input=help to\n"
-     << "explain the input annotations printed by -dump-input=always and\n"
-     << "-dump-input=fail:\n\n";
+     << "explain the input dump printed by FileCheck.\n"
+     << "\n"
+     << "Related command-line options:\n"
+     << "\n"
+     << "  - -dump-input=<value> enables or disables the input dump\n"
+     << "  - -dump-input-filter=<value> filters the input lines\n"
+     << "  - -dump-input-context=<N> adjusts the context of filtered lines\n"
+     << "  - -v and -vv add more annotations\n"
+     << "  - -color forces colors to be enabled both in the dump and below\n"
+     << "  - -help documents the above options in more detail\n"
+     << "\n"
+     << "These options can also be set via FILECHECK_OPTS.  For example, for\n"
+     << "maximum debugging output on failures:\n"
+     << "\n"
+     << "  $ FILECHECK_OPTS='-dump-input-filter=all -vv -color' ninja check\n"
+     << "\n"
+     << "Input dump annotation format:\n"
+     << "\n";
 
   // Labels for input lines.
   OS << "  - ";
@@ -223,6 +280,12 @@ static void DumpInputAnnotationHelp(raw_ostream &OS) {
   WithColor(OS, raw_ostream::SAVEDCOLOR, true) << "?";
   OS << "      marks fuzzy match when no match is found\n";
 
+  // Elided lines.
+  OS << "  - ";
+  WithColor(OS, raw_ostream::SAVEDCOLOR, true) << "...";
+  OS << "    indicates elided input lines and annotations, as specified by\n"
+     << "           -dump-input-filter and -dump-input-context\n";
+
   // Colors.
   OS << "  - colors ";
   WithColor(OS, raw_ostream::GREEN, true) << "success";
@@ -234,8 +297,7 @@ static void DumpInputAnnotationHelp(raw_ostream &OS) {
   WithColor(OS, raw_ostream::CYAN, true, false) << "discarded match";
   OS << ", ";
   WithColor(OS, raw_ostream::CYAN, true, true) << "unmatched input";
-  OS << "\n\n"
-     << "If you are not seeing color above or in input dumps, try: -color\n";
+  OS << "\n";
 }
 
 /// An annotation for a single input line.
@@ -244,10 +306,12 @@ struct InputAnnotation {
   unsigned DiagIndex;
   /// The label for this annotation.
   std::string Label;
+  /// Is this the initial fragment of a diagnostic that has been broken across
+  /// multiple lines?
+  bool IsFirstLine;
   /// What input line (one-origin indexing) this annotation marks.  This might
-  /// be different from the starting line of the original diagnostic if this is
-  /// a non-initial fragment of a diagnostic that has been broken across
-  /// multiple lines.
+  /// be different from the starting line of the original diagnostic if
+  /// !IsFirstLine.
   unsigned InputLine;
   /// The column range (one-origin indexing, open end) in which to mark the
   /// input line.  If InputEndCol is UINT_MAX, treat it as the last column
@@ -343,6 +407,7 @@ BuildInputAnnotations(const SourceMgr &SM, unsigned CheckFileBufferID,
 
     // Compute the mark location, and break annotation into multiple
     // annotations if it spans multiple lines.
+    A.IsFirstLine = true;
     A.InputLine = DiagItr->InputStartLine;
     A.InputStartCol = DiagItr->InputStartCol;
     if (DiagItr->InputStartLine == DiagItr->InputEndLine) {
@@ -366,6 +431,7 @@ BuildInputAnnotations(const SourceMgr &SM, unsigned CheckFileBufferID,
         InputAnnotation B;
         B.DiagIndex = A.DiagIndex;
         B.Label = A.Label;
+        B.IsFirstLine = false;
         B.InputLine = L;
         B.Marker = A.Marker;
         B.Marker.Lead = '~';
@@ -382,11 +448,59 @@ BuildInputAnnotations(const SourceMgr &SM, unsigned CheckFileBufferID,
   }
 }
 
+static unsigned FindInputLineInFilter(
+    DumpInputFilterValue DumpInputFilter, unsigned CurInputLine,
+    const std::vector<InputAnnotation>::iterator &AnnotationBeg,
+    const std::vector<InputAnnotation>::iterator &AnnotationEnd) {
+  if (DumpInputFilter == DumpInputFilterAll)
+    return CurInputLine;
+  for (auto AnnotationItr = AnnotationBeg; AnnotationItr != AnnotationEnd;
+       ++AnnotationItr) {
+    switch (DumpInputFilter) {
+    case DumpInputFilterAll:
+      llvm_unreachable("unexpected DumpInputFilterAll");
+      break;
+    case DumpInputFilterAnnotationFull:
+      return AnnotationItr->InputLine;
+    case DumpInputFilterAnnotation:
+      if (AnnotationItr->IsFirstLine)
+        return AnnotationItr->InputLine;
+      break;
+    case DumpInputFilterError:
+      if (AnnotationItr->IsFirstLine && AnnotationItr->Marker.FiltersAsError)
+        return AnnotationItr->InputLine;
+      break;
+    }
+  }
+  return UINT_MAX;
+}
+
+/// To OS, print a vertical ellipsis (right-justified at LabelWidth) if it would
+/// occupy less lines than ElidedLines, but print ElidedLines otherwise.  Either
+/// way, clear ElidedLines.  Thus, if ElidedLines is empty, do nothing.
+static void DumpEllipsisOrElidedLines(raw_ostream &OS, std::string &ElidedLines,
+                                      unsigned LabelWidth) {
+  if (ElidedLines.empty())
+    return;
+  unsigned EllipsisLines = 3;
+  if (EllipsisLines < StringRef(ElidedLines).count('\n')) {
+    for (unsigned i = 0; i < EllipsisLines; ++i) {
+      WithColor(OS, raw_ostream::BLACK, /*Bold=*/true)
+          << right_justify(".", LabelWidth);
+      OS << '\n';
+    }
+  } else
+    OS << ElidedLines;
+  ElidedLines.clear();
+}
+
 static void DumpAnnotatedInput(raw_ostream &OS, const FileCheckRequest &Req,
+                               DumpInputFilterValue DumpInputFilter,
+                               unsigned DumpInputContext,
                                StringRef InputFileText,
                                std::vector<InputAnnotation> &Annotations,
                                unsigned LabelWidth) {
-  OS << "Full input was:\n<<<<<<\n";
+  OS << "Input was:\n<<<<<<\n";
 
   // Sort annotations.
   std::sort(Annotations.begin(), Annotations.end(),
@@ -456,21 +570,53 @@ static void DumpAnnotatedInput(raw_ostream &OS, const FileCheckRequest &Req,
   LabelWidth = std::max(LabelWidth, LineNoWidth) + 3;
 
   // Print annotated input lines.
+  unsigned PrevLineInFilter = 0; // 0 means none so far
+  unsigned NextLineInFilter = 0; // 0 means uncomputed, UINT_MAX means none
+  std::string ElidedLines;
+  raw_string_ostream ElidedLinesOS(ElidedLines);
+  ColorMode TheColorMode =
+      WithColor(OS).colorsEnabled() ? ColorMode::Enable : ColorMode::Disable;
+  if (TheColorMode == ColorMode::Enable)
+    ElidedLinesOS.enable_colors(true);
   auto AnnotationItr = Annotations.begin(), AnnotationEnd = Annotations.end();
   for (unsigned Line = 1;
        InputFilePtr != InputFileEnd || AnnotationItr != AnnotationEnd;
        ++Line) {
     const unsigned char *InputFileLine = InputFilePtr;
 
+    // Compute the previous and next line included by the filter.
+    if (NextLineInFilter < Line)
+      NextLineInFilter = FindInputLineInFilter(DumpInputFilter, Line,
+                                               AnnotationItr, AnnotationEnd);
+    assert(NextLineInFilter && "expected NextLineInFilter to be computed");
+    if (NextLineInFilter == Line)
+      PrevLineInFilter = Line;
+
+    // Elide this input line and its annotations if it's not within the
+    // context specified by -dump-input-context of an input line included by
+    // -dump-input-filter.  However, in case the resulting ellipsis would occupy
+    // more lines than the input lines and annotations it elides, buffer the
+    // elided lines and annotations so we can print them instead.
+    raw_ostream *LineOS = &OS;
+    if ((!PrevLineInFilter || PrevLineInFilter + DumpInputContext < Line) &&
+        (NextLineInFilter == UINT_MAX ||
+         Line + DumpInputContext < NextLineInFilter))
+      LineOS = &ElidedLinesOS;
+    else {
+      LineOS = &OS;
+      DumpEllipsisOrElidedLines(OS, ElidedLinesOS.str(), LabelWidth);
+    }
+
     // Print right-aligned line number.
-    WithColor(OS, raw_ostream::BLACK, true)
+    WithColor(*LineOS, raw_ostream::BLACK, /*Bold=*/true, /*BF=*/false,
+              TheColorMode)
         << format_decimal(Line, LabelWidth) << ": ";
 
     // For the case where -v and colors are enabled, find the annotations for
     // good matches for expected patterns in order to highlight everything
     // else in the line.  There are no such annotations if -v is disabled.
     std::vector<InputAnnotation> FoundAndExpectedMatches;
-    if (Req.Verbose && WithColor(OS).colorsEnabled()) {
+    if (Req.Verbose && TheColorMode == ColorMode::Enable) {
       for (auto I = AnnotationItr; I != AnnotationEnd && I->InputLine == Line;
            ++I) {
         if (I->FoundAndExpectedMatch)
@@ -482,7 +628,8 @@ static void DumpAnnotatedInput(raw_ostream &OS, const FileCheckRequest &Req,
     // expected patterns.
     bool Newline = false;
     {
-      WithColor COS(OS);
+      WithColor COS(*LineOS, raw_ostream::SAVEDCOLOR, /*Bold=*/false,
+                    /*BG=*/false, TheColorMode);
       bool InMatch = false;
       if (Req.Verbose)
         COS.changeColor(raw_ostream::CYAN, true, true);
@@ -506,13 +653,14 @@ static void DumpAnnotatedInput(raw_ostream &OS, const FileCheckRequest &Req,
         ++InputFilePtr;
       }
     }
-    OS << '\n';
+    *LineOS << '\n';
     unsigned InputLineWidth = InputFilePtr - InputFileLine - Newline;
 
     // Print any annotations.
     while (AnnotationItr != AnnotationEnd &&
            AnnotationItr->InputLine == Line) {
-      WithColor COS(OS, AnnotationItr->Marker.Color, true);
+      WithColor COS(*LineOS, AnnotationItr->Marker.Color, /*Bold=*/true,
+                    /*BG=*/false, TheColorMode);
       // The two spaces below are where the ": " appears on input lines.
       COS << left_justify(AnnotationItr->Label, LabelWidth) << "  ";
       unsigned Col;
@@ -537,6 +685,7 @@ static void DumpAnnotatedInput(raw_ostream &OS, const FileCheckRequest &Req,
       ++AnnotationItr;
     }
   }
+  DumpEllipsisOrElidedLines(OS, ElidedLinesOS.str(), LabelWidth);
 
   OS << ">>>>>>\n";
 }
@@ -549,10 +698,27 @@ int main(int argc, char **argv) {
   InitLLVM X(argc, argv);
   cl::ParseCommandLineOptions(argc, argv, /*Overview*/ "", /*Errs*/ nullptr,
                               "FILECHECK_OPTS");
+
+  // Select -dump-input* values.  The -help documentation specifies the default
+  // value and which value to choose if an option is specified multiple times.
+  // In the latter case, the general rule of thumb is to choose the value that
+  // provides the most information.
   DumpInputValue DumpInput =
       DumpInputs.empty()
-          ? DumpInputDefault
+          ? DumpInputFail
           : *std::max_element(DumpInputs.begin(), DumpInputs.end());
+  DumpInputFilterValue DumpInputFilter;
+  if (DumpInputFilters.empty())
+    DumpInputFilter = DumpInput == DumpInputAlways ? DumpInputFilterAll
+                                                   : DumpInputFilterError;
+  else
+    DumpInputFilter =
+        *std::max_element(DumpInputFilters.begin(), DumpInputFilters.end());
+  unsigned DumpInputContext = DumpInputContexts.empty()
+                                  ? 5
+                                  : *std::max_element(DumpInputContexts.begin(),
+                                                      DumpInputContexts.end());
+
   if (DumpInput == DumpInputHelp) {
     DumpInputAnnotationHelp(outs());
     return 0;
@@ -668,9 +834,6 @@ int main(int argc, char **argv) {
                             InputFileText, InputFile.getBufferIdentifier()),
                         SMLoc());
 
-  if (DumpInput == DumpInputDefault)
-    DumpInput = DumpInputFail;
-
   std::vector<FileCheckDiag> Diags;
   int ExitCode = FC.checkInput(SM, InputFileText,
                                DumpInput == DumpInputNever ? nullptr : &Diags)
@@ -679,18 +842,17 @@ int main(int argc, char **argv) {
   if (DumpInput == DumpInputAlways ||
       (ExitCode == 1 && DumpInput == DumpInputFail)) {
     errs() << "\n"
-           << "Input file: "
-           << InputFilename
-           << "\n"
+           << "Input file: " << InputFilename << "\n"
            << "Check file: " << CheckFilename << "\n"
            << "\n"
-           << "-dump-input=help describes the format of the following dump.\n"
+           << "-dump-input=help explains the following input dump.\n"
            << "\n";
     std::vector<InputAnnotation> Annotations;
     unsigned LabelWidth;
     BuildInputAnnotations(SM, CheckFileBufferID, ImpPatBufferIDRange, Diags,
                           Annotations, LabelWidth);
-    DumpAnnotatedInput(errs(), Req, InputFileText, Annotations, LabelWidth);
+    DumpAnnotatedInput(errs(), Req, DumpInputFilter, DumpInputContext,
+                       InputFileText, Annotations, LabelWidth);
   }
 
   return ExitCode;
diff --git a/llvm/utils/TableGen/CMakeLists.txt b/llvm/utils/TableGen/CMakeLists.txt
index dbca62c5c78fb..8673a257246e0 100644
--- a/llvm/utils/TableGen/CMakeLists.txt
+++ b/llvm/utils/TableGen/CMakeLists.txt
@@ -23,6 +23,7 @@ add_tablegen(llvm-tblgen LLVM
   DAGISelMatcher.cpp
   DFAEmitter.cpp
   DFAPacketizerEmitter.cpp
+  DirectiveEmitter.cpp
   DisassemblerEmitter.cpp
   ExegesisEmitter.cpp
   FastISelEmitter.cpp
diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.h b/llvm/utils/TableGen/CodeGenIntrinsics.h
index 6503f39cfd8ea..af59c1f3d8337 100644
--- a/llvm/utils/TableGen/CodeGenIntrinsics.h
+++ b/llvm/utils/TableGen/CodeGenIntrinsics.h
@@ -126,6 +126,9 @@ struct CodeGenIntrinsic {
   /// True if the intrinsic is no-sync.
   bool isNoSync;
 
+  /// True if the intrinsic is no-free.
+  bool isNoFree;
+
   /// True if the intrinsic is will-return.
   bool isWillReturn;
 
diff --git a/llvm/utils/TableGen/CodeGenTarget.cpp b/llvm/utils/TableGen/CodeGenTarget.cpp
index 78fb732877013..891a08ea590e2 100644
--- a/llvm/utils/TableGen/CodeGenTarget.cpp
+++ b/llvm/utils/TableGen/CodeGenTarget.cpp
@@ -133,12 +133,16 @@ StringRef llvm::getEnumName(MVT::SimpleValueType T) {
   case MVT::v8f16:    return "MVT::v8f16";
   case MVT::v16f16:   return "MVT::v16f16";
   case MVT::v32f16:   return "MVT::v32f16";
+  case MVT::v64f16:   return "MVT::v64f16";
+  case MVT::v128f16:  return "MVT::v128f16";
   case MVT::v2bf16:   return "MVT::v2bf16";
   case MVT::v3bf16:   return "MVT::v3bf16";
   case MVT::v4bf16:   return "MVT::v4bf16";
   case MVT::v8bf16:   return "MVT::v8bf16";
   case MVT::v16bf16:  return "MVT::v16bf16";
   case MVT::v32bf16:  return "MVT::v32bf16";
+  case MVT::v64bf16:  return "MVT::v64bf16";
+  case MVT::v128bf16: return "MVT::v128bf16";
   case MVT::v1f32:    return "MVT::v1f32";
   case MVT::v2f32:    return "MVT::v2f32";
   case MVT::v3f32:    return "MVT::v3f32";
@@ -158,18 +162,21 @@ StringRef llvm::getEnumName(MVT::SimpleValueType T) {
   case MVT::v4f64:    return "MVT::v4f64";
   case MVT::v8f64:    return "MVT::v8f64";
   case MVT::v16f64:   return "MVT::v16f64";
+  case MVT::v32f64:   return "MVT::v32f64";
   case MVT::nxv1i1:   return "MVT::nxv1i1";
   case MVT::nxv2i1:   return "MVT::nxv2i1";
   case MVT::nxv4i1:   return "MVT::nxv4i1";
   case MVT::nxv8i1:   return "MVT::nxv8i1";
   case MVT::nxv16i1:  return "MVT::nxv16i1";
   case MVT::nxv32i1:  return "MVT::nxv32i1";
+  case MVT::nxv64i1:  return "MVT::nxv64i1";
   case MVT::nxv1i8:   return "MVT::nxv1i8";
   case MVT::nxv2i8:   return "MVT::nxv2i8";
   case MVT::nxv4i8:   return "MVT::nxv4i8";
   case MVT::nxv8i8:   return "MVT::nxv8i8";
   case MVT::nxv16i8:  return "MVT::nxv16i8";
   case MVT::nxv32i8:  return "MVT::nxv32i8";
+  case MVT::nxv64i8:  return "MVT::nxv64i8";
   case MVT::nxv1i16:  return "MVT::nxv1i16";
   case MVT::nxv2i16:  return "MVT::nxv2i16";
   case MVT::nxv4i16:  return "MVT::nxv4i16";
@@ -181,14 +188,19 @@ StringRef llvm::getEnumName(MVT::SimpleValueType T) {
   case MVT::nxv4i32:  return "MVT::nxv4i32";
   case MVT::nxv8i32:  return "MVT::nxv8i32";
   case MVT::nxv16i32: return "MVT::nxv16i32";
+  case MVT::nxv32i32: return "MVT::nxv32i32";
   case MVT::nxv1i64:  return "MVT::nxv1i64";
   case MVT::nxv2i64:  return "MVT::nxv2i64";
   case MVT::nxv4i64:  return "MVT::nxv4i64";
   case MVT::nxv8i64:  return "MVT::nxv8i64";
   case MVT::nxv16i64: return "MVT::nxv16i64";
+  case MVT::nxv32i64: return "MVT::nxv32i64";
+  case MVT::nxv1f16:  return "MVT::nxv1f16";
   case MVT::nxv2f16:  return "MVT::nxv2f16";
   case MVT::nxv4f16:  return "MVT::nxv4f16";
   case MVT::nxv8f16:  return "MVT::nxv8f16";
+  case MVT::nxv16f16: return "MVT::nxv16f16";
+  case MVT::nxv32f16: return "MVT::nxv32f16";
   case MVT::nxv2bf16:  return "MVT::nxv2bf16";
   case MVT::nxv4bf16:  return "MVT::nxv4bf16";
   case MVT::nxv8bf16:  return "MVT::nxv8bf16";
@@ -619,6 +631,7 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
   canThrow = false;
   isNoReturn = false;
   isNoSync = false;
+  isNoFree = false;
   isWillReturn = false;
   isCold = false;
   isNoDuplicate = false;
@@ -785,6 +798,8 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
       isNoReturn = true;
     else if (Property->getName() == "IntrNoSync")
       isNoSync = true;
+    else if (Property->getName() == "IntrNoFree")
+      isNoFree = true;
     else if (Property->getName() == "IntrWillReturn")
       isWillReturn = true;
     else if (Property->getName() == "IntrCold")
diff --git a/llvm/utils/TableGen/DirectiveEmitter.cpp b/llvm/utils/TableGen/DirectiveEmitter.cpp
new file mode 100644
index 0000000000000..ebcd6873205ea
--- /dev/null
+++ b/llvm/utils/TableGen/DirectiveEmitter.cpp
@@ -0,0 +1,523 @@
+//===- DirectiveEmitter.cpp - Directive Language Emitter ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// DirectiveEmitter uses the descriptions of directives and clauses to construct
+// common code declarations to be used in Frontends.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
+
+using namespace llvm;
+
+namespace {
+// Simple RAII helper for defining ifdef-undef-endif scopes.
+class IfDefScope {
+public:
+  IfDefScope(StringRef Name, raw_ostream &OS) : Name(Name), OS(OS) {
+    OS << "#ifdef " << Name << "\n"
+       << "#undef " << Name << "\n";
+  }
+
+  ~IfDefScope() { OS << "\n#endif // " << Name << "\n\n"; }
+
+private:
+  StringRef Name;
+  raw_ostream &OS;
+};
+} // end anonymous namespace
+
+namespace llvm {
+
+// Get Directive or Clause name formatted by replacing whitespaces with
+// underscores.
+std::string getFormattedName(StringRef Name) {
+  std::string N = Name.str();
+  std::replace(N.begin(), N.end(), ' ', '_');
+  return N;
+}
+
+// Generate enum class
+void GenerateEnumClass(const std::vector<Record *> &Records, raw_ostream &OS,
+                       StringRef Enum, StringRef Prefix, StringRef CppNamespace,
+                       bool MakeEnumAvailableInNamespace) {
+  OS << "\n";
+  OS << "enum class " << Enum << " {\n";
+  for (const auto &R : Records) {
+    const auto Name = R->getValueAsString("name");
+    OS << "  " << Prefix << getFormattedName(Name) << ",\n";
+  }
+  OS << "};\n";
+  OS << "\n";
+  OS << "static constexpr std::size_t " << Enum
+     << "_enumSize = " << Records.size() << ";\n";
+
+  // Make the enum values available in the defined namespace. This allows us to
+  // write something like Enum_X if we have a `using namespace <CppNamespace>`.
+  // At the same time we do not loose the strong type guarantees of the enum
+  // class, that is we cannot pass an unsigned as Directive without an explicit
+  // cast.
+  if (MakeEnumAvailableInNamespace) {
+    OS << "\n";
+    for (const auto &R : Records) {
+      const auto FormattedName = getFormattedName(R->getValueAsString("name"));
+      OS << "constexpr auto " << Prefix << FormattedName << " = "
+         << "llvm::" << CppNamespace << "::" << Enum << "::" << Prefix
+         << FormattedName << ";\n";
+    }
+  }
+}
+
+// Generate the declaration section for the enumeration in the directive
+// language
+void EmitDirectivesDecl(RecordKeeper &Records, raw_ostream &OS) {
+
+  const auto &DirectiveLanguages =
+      Records.getAllDerivedDefinitions("DirectiveLanguage");
+
+  if (DirectiveLanguages.size() != 1) {
+    PrintError("A single definition of DirectiveLanguage is needed.");
+    return;
+  }
+
+  const auto &DirectiveLanguage = DirectiveLanguages[0];
+  StringRef LanguageName = DirectiveLanguage->getValueAsString("name");
+  StringRef DirectivePrefix =
+      DirectiveLanguage->getValueAsString("directivePrefix");
+  StringRef ClausePrefix = DirectiveLanguage->getValueAsString("clausePrefix");
+  StringRef CppNamespace = DirectiveLanguage->getValueAsString("cppNamespace");
+  bool MakeEnumAvailableInNamespace =
+      DirectiveLanguage->getValueAsBit("makeEnumAvailableInNamespace");
+  bool EnableBitmaskEnumInNamespace =
+      DirectiveLanguage->getValueAsBit("enableBitmaskEnumInNamespace");
+
+  OS << "#ifndef LLVM_" << LanguageName << "_INC\n";
+  OS << "#define LLVM_" << LanguageName << "_INC\n";
+
+  if (EnableBitmaskEnumInNamespace)
+    OS << "\n#include \"llvm/ADT/BitmaskEnum.h\"\n";
+
+  OS << "\n";
+  OS << "namespace llvm {\n";
+  OS << "class StringRef;\n";
+
+  // Open namespaces defined in the directive language
+  llvm::SmallVector<StringRef, 2> Namespaces;
+  llvm::SplitString(CppNamespace, Namespaces, "::");
+  for (auto Ns : Namespaces)
+    OS << "namespace " << Ns << " {\n";
+
+  if (EnableBitmaskEnumInNamespace)
+    OS << "\nLLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();\n";
+
+  // Emit Directive enumeration
+  const auto &Directives = Records.getAllDerivedDefinitions("Directive");
+  GenerateEnumClass(Directives, OS, "Directive", DirectivePrefix, CppNamespace,
+                    MakeEnumAvailableInNamespace);
+
+  // Emit Clause enumeration
+  const auto &Clauses = Records.getAllDerivedDefinitions("Clause");
+  GenerateEnumClass(Clauses, OS, "Clause", ClausePrefix, CppNamespace,
+                    MakeEnumAvailableInNamespace);
+
+  // Generic function signatures
+  OS << "\n";
+  OS << "// Enumeration helper functions\n";
+  OS << "Directive get" << LanguageName
+     << "DirectiveKind(llvm::StringRef Str);\n";
+  OS << "\n";
+  OS << "llvm::StringRef get" << LanguageName
+     << "DirectiveName(Directive D);\n";
+  OS << "\n";
+  OS << "Clause get" << LanguageName << "ClauseKind(llvm::StringRef Str);\n";
+  OS << "\n";
+  OS << "llvm::StringRef get" << LanguageName << "ClauseName(Clause C);\n";
+  OS << "\n";
+  OS << "/// Return true if \\p C is a valid clause for \\p D in version \\p "
+     << "Version.\n";
+  OS << "bool isAllowedClauseForDirective(Directive D, "
+     << "Clause C, unsigned Version);\n";
+  OS << "\n";
+
+  // Closing namespaces
+  for (auto Ns : llvm::reverse(Namespaces))
+    OS << "} // namespace " << Ns << "\n";
+
+  OS << "} // namespace llvm\n";
+
+  OS << "#endif // LLVM_" << LanguageName << "_INC\n";
+}
+
+// Generate function implementation for get<Enum>Name(StringRef Str)
+void GenerateGetName(const std::vector<Record *> &Records, raw_ostream &OS,
+                     StringRef Enum, StringRef Prefix, StringRef LanguageName,
+                     StringRef Namespace) {
+  OS << "\n";
+  OS << "llvm::StringRef llvm::" << Namespace << "::get" << LanguageName << Enum
+     << "Name(" << Enum << " Kind) {\n";
+  OS << "  switch (Kind) {\n";
+  for (const auto &R : Records) {
+    const auto Name = R->getValueAsString("name");
+    const auto AlternativeName = R->getValueAsString("alternativeName");
+    OS << "    case " << Prefix << getFormattedName(Name) << ":\n";
+    OS << "      return \"";
+    if (AlternativeName.empty())
+      OS << Name;
+    else
+      OS << AlternativeName;
+    OS << "\";\n";
+  }
+  OS << "  }\n"; // switch
+  OS << "  llvm_unreachable(\"Invalid " << LanguageName << " " << Enum
+     << " kind\");\n";
+  OS << "}\n";
+}
+
+// Generate function implementation for get<Enum>Kind(StringRef Str)
+void GenerateGetKind(const std::vector<Record *> &Records, raw_ostream &OS,
+                     StringRef Enum, StringRef Prefix, StringRef LanguageName,
+                     StringRef Namespace, bool ImplicitAsUnknown) {
+
+  auto DefaultIt = std::find_if(Records.begin(), Records.end(), [](Record *R) {
+    return R->getValueAsBit("isDefault") == true;
+  });
+
+  if (DefaultIt == Records.end()) {
+    PrintError("A least one " + Enum + " must be defined as default.");
+    return;
+  }
+
+  const auto FormattedDefaultName =
+      getFormattedName((*DefaultIt)->getValueAsString("name"));
+
+  OS << "\n";
+  OS << Enum << " llvm::" << Namespace << "::get" << LanguageName << Enum
+     << "Kind(llvm::StringRef Str) {\n";
+  OS << "  return llvm::StringSwitch<" << Enum << ">(Str)\n";
+
+  for (const auto &R : Records) {
+    const auto Name = R->getValueAsString("name");
+    if (ImplicitAsUnknown && R->getValueAsBit("isImplicit")) {
+      OS << "    .Case(\"" << Name << "\"," << Prefix << FormattedDefaultName
+         << ")\n";
+    } else {
+      OS << "    .Case(\"" << Name << "\"," << Prefix << getFormattedName(Name)
+         << ")\n";
+    }
+  }
+  OS << "    .Default(" << Prefix << FormattedDefaultName << ");\n";
+  OS << "}\n";
+}
+
+void GenerateCaseForVersionedClauses(const std::vector<Record *> &Clauses,
+                                     raw_ostream &OS, StringRef DirectiveName,
+                                     StringRef DirectivePrefix,
+                                     StringRef ClausePrefix,
+                                     llvm::StringSet<> &Cases) {
+  for (const auto &C : Clauses) {
+    const auto MinVersion = C->getValueAsInt("minVersion");
+    const auto MaxVersion = C->getValueAsInt("maxVersion");
+    const auto SpecificClause = C->getValueAsDef("clause");
+    const auto ClauseName =
+        getFormattedName(SpecificClause->getValueAsString("name"));
+
+    if (Cases.find(ClauseName) == Cases.end()) {
+      Cases.insert(ClauseName);
+      OS << "        case " << ClausePrefix << ClauseName << ":\n";
+      OS << "          return " << MinVersion << " <= Version && " << MaxVersion
+         << " >= Version;\n";
+    }
+  }
+}
+
+// Generate the isAllowedClauseForDirective function implementation.
+void GenerateIsAllowedClause(const std::vector<Record *> &Directives,
+                             raw_ostream &OS, StringRef LanguageName,
+                             StringRef DirectivePrefix, StringRef ClausePrefix,
+                             StringRef CppNamespace) {
+  OS << "\n";
+  OS << "bool llvm::" << CppNamespace << "::isAllowedClauseForDirective("
+     << "Directive D, Clause C, unsigned Version) {\n";
+  OS << "  assert(unsigned(D) <= llvm::" << CppNamespace
+     << "::Directive_enumSize);\n";
+  OS << "  assert(unsigned(C) <= llvm::" << CppNamespace
+     << "::Clause_enumSize);\n";
+
+  OS << "  switch (D) {\n";
+
+  for (const auto &D : Directives) {
+
+    const auto DirectiveName = D->getValueAsString("name");
+    const auto &AllowedClauses = D->getValueAsListOfDefs("allowedClauses");
+    const auto &AllowedOnceClauses =
+        D->getValueAsListOfDefs("allowedOnceClauses");
+    const auto &AllowedExclusiveClauses =
+        D->getValueAsListOfDefs("allowedExclusiveClauses");
+    const auto &RequiredClauses = D->getValueAsListOfDefs("requiredClauses");
+
+    OS << "    case " << DirectivePrefix << getFormattedName(DirectiveName)
+       << ":\n";
+    if (AllowedClauses.size() == 0 && AllowedOnceClauses.size() == 0 &&
+        AllowedExclusiveClauses.size() == 0 && RequiredClauses.size() == 0) {
+      OS << "      return false;\n";
+    } else {
+      OS << "      switch (C) {\n";
+
+      llvm::StringSet<> Cases;
+
+      GenerateCaseForVersionedClauses(AllowedClauses, OS, DirectiveName,
+                                      DirectivePrefix, ClausePrefix, Cases);
+
+      GenerateCaseForVersionedClauses(AllowedOnceClauses, OS, DirectiveName,
+                                      DirectivePrefix, ClausePrefix, Cases);
+
+      GenerateCaseForVersionedClauses(AllowedExclusiveClauses, OS,
+                                      DirectiveName, DirectivePrefix,
+                                      ClausePrefix, Cases);
+
+      GenerateCaseForVersionedClauses(RequiredClauses, OS, DirectiveName,
+                                      DirectivePrefix, ClausePrefix, Cases);
+
+      OS << "        default:\n";
+      OS << "          return false;\n";
+      OS << "      }\n"; // End of clauses switch
+    }
+    OS << "      break;\n";
+  }
+
+  OS << "  }\n"; // End of directives switch
+  OS << "  llvm_unreachable(\"Invalid " << LanguageName
+     << " Directive kind\");\n";
+  OS << "}\n"; // End of function isAllowedClauseForDirective
+}
+
+// Generate a simple enum set with the give clauses.
+void GenerateClauseSet(const std::vector<Record *> &Clauses, raw_ostream &OS,
+                       StringRef ClauseEnumSetClass, StringRef ClauseSetPrefix,
+                       StringRef DirectiveName, StringRef DirectivePrefix,
+                       StringRef ClausePrefix, StringRef CppNamespace) {
+
+  OS << "\n";
+  OS << "  static " << ClauseEnumSetClass << " " << ClauseSetPrefix
+     << DirectivePrefix << getFormattedName(DirectiveName) << " {\n";
+
+  for (const auto &C : Clauses) {
+    const auto SpecificClause = C->getValueAsDef("clause");
+    const auto ClauseName = SpecificClause->getValueAsString("name");
+    OS << "    llvm::" << CppNamespace << "::Clause::" << ClausePrefix
+       << getFormattedName(ClauseName) << ",\n";
+  }
+  OS << "  };\n";
+}
+
+// Generate an enum set for the 4 kinds of clauses linked to a directive.
+void GenerateDirectiveClauseSets(const std::vector<Record *> &Directives,
+                                 raw_ostream &OS, StringRef LanguageName,
+                                 StringRef ClauseEnumSetClass,
+                                 StringRef DirectivePrefix,
+                                 StringRef ClausePrefix,
+                                 StringRef CppNamespace) {
+
+  IfDefScope Scope("GEN_FLANG_DIRECTIVE_CLAUSE_SETS", OS);
+
+  OS << "\n";
+  OS << "namespace llvm {\n";
+
+  // Open namespaces defined in the directive language.
+  llvm::SmallVector<StringRef, 2> Namespaces;
+  llvm::SplitString(CppNamespace, Namespaces, "::");
+  for (auto Ns : Namespaces)
+    OS << "namespace " << Ns << " {\n";
+
+  for (const auto &D : Directives) {
+    const auto DirectiveName = D->getValueAsString("name");
+
+    const auto &AllowedClauses = D->getValueAsListOfDefs("allowedClauses");
+    const auto &AllowedOnceClauses =
+        D->getValueAsListOfDefs("allowedOnceClauses");
+    const auto &AllowedExclusiveClauses =
+        D->getValueAsListOfDefs("allowedExclusiveClauses");
+    const auto &RequiredClauses = D->getValueAsListOfDefs("requiredClauses");
+
+    OS << "\n";
+    OS << "  // Sets for " << DirectiveName << "\n";
+
+    GenerateClauseSet(AllowedClauses, OS, ClauseEnumSetClass, "allowedClauses_",
+                      DirectiveName, DirectivePrefix, ClausePrefix,
+                      CppNamespace);
+    GenerateClauseSet(AllowedOnceClauses, OS, ClauseEnumSetClass,
+                      "allowedOnceClauses_", DirectiveName, DirectivePrefix,
+                      ClausePrefix, CppNamespace);
+    GenerateClauseSet(AllowedExclusiveClauses, OS, ClauseEnumSetClass,
+                      "allowedExclusiveClauses_", DirectiveName,
+                      DirectivePrefix, ClausePrefix, CppNamespace);
+    GenerateClauseSet(RequiredClauses, OS, ClauseEnumSetClass,
+                      "requiredClauses_", DirectiveName, DirectivePrefix,
+                      ClausePrefix, CppNamespace);
+  }
+
+  // Closing namespaces
+  for (auto Ns : llvm::reverse(Namespaces))
+    OS << "} // namespace " << Ns << "\n";
+
+  OS << "} // namespace llvm\n";
+}
+
+// Generate a map of directive (key) with DirectiveClauses struct as values.
+// The struct holds the 4 sets of enumeration for the 4 kinds of clauses
+// allowances (allowed, allowed once, allowed exclusive and required).
+void GenerateDirectiveClauseMap(const std::vector<Record *> &Directives,
+                                raw_ostream &OS, StringRef LanguageName,
+                                StringRef ClauseEnumSetClass,
+                                StringRef DirectivePrefix,
+                                StringRef ClausePrefix,
+                                StringRef CppNamespace) {
+
+  IfDefScope Scope("GEN_FLANG_DIRECTIVE_CLAUSE_MAP", OS);
+
+  OS << "\n";
+  OS << "struct " << LanguageName << "DirectiveClauses {\n";
+  OS << "  const " << ClauseEnumSetClass << " allowed;\n";
+  OS << "  const " << ClauseEnumSetClass << " allowedOnce;\n";
+  OS << "  const " << ClauseEnumSetClass << " allowedExclusive;\n";
+  OS << "  const " << ClauseEnumSetClass << " requiredOneOf;\n";
+  OS << "};\n";
+
+  OS << "\n";
+
+  OS << "std::unordered_map<llvm::" << CppNamespace << "::Directive, "
+     << LanguageName << "DirectiveClauses>\n";
+  OS << "    directiveClausesTable = {\n";
+
+  for (const auto &D : Directives) {
+    const auto FormattedDirectiveName =
+        getFormattedName(D->getValueAsString("name"));
+    OS << "  {llvm::" << CppNamespace << "::Directive::" << DirectivePrefix
+       << FormattedDirectiveName << ",\n";
+    OS << "    {\n";
+    OS << "      llvm::" << CppNamespace << "::allowedClauses_"
+       << DirectivePrefix << FormattedDirectiveName << ",\n";
+    OS << "      llvm::" << CppNamespace << "::allowedOnceClauses_"
+       << DirectivePrefix << FormattedDirectiveName << ",\n";
+    OS << "      llvm::" << CppNamespace << "::allowedExclusiveClauses_"
+       << DirectivePrefix << FormattedDirectiveName << ",\n";
+    OS << "      llvm::" << CppNamespace << "::requiredClauses_"
+       << DirectivePrefix << FormattedDirectiveName << ",\n";
+    OS << "    }\n";
+    OS << "  },\n";
+  }
+
+  OS << "};\n";
+}
+
+// Generate the implemenation section for the enumeration in the directive
+// language
+void EmitDirectivesFlangImpl(const std::vector<Record *> &Directives,
+                             raw_ostream &OS, StringRef LanguageName,
+                             StringRef ClauseEnumSetClass,
+                             StringRef DirectivePrefix, StringRef ClausePrefix,
+                             StringRef CppNamespace) {
+
+  GenerateDirectiveClauseSets(Directives, OS, LanguageName, ClauseEnumSetClass,
+                              DirectivePrefix, ClausePrefix, CppNamespace);
+
+  GenerateDirectiveClauseMap(Directives, OS, LanguageName, ClauseEnumSetClass,
+                             DirectivePrefix, ClausePrefix, CppNamespace);
+}
+
+// Generate the implemenation section for the enumeration in the directive
+// language.
+void EmitDirectivesGen(RecordKeeper &Records, raw_ostream &OS) {
+
+  const auto &DirectiveLanguages =
+      Records.getAllDerivedDefinitions("DirectiveLanguage");
+
+  if (DirectiveLanguages.size() != 1) {
+    PrintError("A single definition of DirectiveLanguage is needed.");
+    return;
+  }
+
+  const auto &DirectiveLanguage = DirectiveLanguages[0];
+  StringRef DirectivePrefix =
+      DirectiveLanguage->getValueAsString("directivePrefix");
+  StringRef LanguageName = DirectiveLanguage->getValueAsString("name");
+  StringRef ClausePrefix = DirectiveLanguage->getValueAsString("clausePrefix");
+  StringRef CppNamespace = DirectiveLanguage->getValueAsString("cppNamespace");
+  StringRef ClauseEnumSetClass =
+      DirectiveLanguage->getValueAsString("clauseEnumSetClass");
+
+  const auto &Directives = Records.getAllDerivedDefinitions("Directive");
+
+  EmitDirectivesFlangImpl(Directives, OS, LanguageName, ClauseEnumSetClass,
+                          DirectivePrefix, ClausePrefix, CppNamespace);
+}
+
+// Generate the implemenation for the enumeration in the directive
+// language. This code can be included in library.
+void EmitDirectivesImpl(RecordKeeper &Records, raw_ostream &OS) {
+
+  const auto &DirectiveLanguages =
+      Records.getAllDerivedDefinitions("DirectiveLanguage");
+
+  if (DirectiveLanguages.size() != 1) {
+    PrintError("A single definition of DirectiveLanguage is needed.");
+    return;
+  }
+
+  const auto &DirectiveLanguage = DirectiveLanguages[0];
+  StringRef DirectivePrefix =
+      DirectiveLanguage->getValueAsString("directivePrefix");
+  StringRef LanguageName = DirectiveLanguage->getValueAsString("name");
+  StringRef ClausePrefix = DirectiveLanguage->getValueAsString("clausePrefix");
+  StringRef CppNamespace = DirectiveLanguage->getValueAsString("cppNamespace");
+  const auto &Directives = Records.getAllDerivedDefinitions("Directive");
+  const auto &Clauses = Records.getAllDerivedDefinitions("Clause");
+
+  StringRef IncludeHeader =
+      DirectiveLanguage->getValueAsString("includeHeader");
+
+  if (!IncludeHeader.empty())
+    OS << "#include \"" << IncludeHeader << "\"\n\n";
+
+  OS << "#include \"llvm/ADT/StringRef.h\"\n";
+  OS << "#include \"llvm/ADT/StringSwitch.h\"\n";
+  OS << "\n";
+  OS << "using namespace llvm;\n";
+  llvm::SmallVector<StringRef, 2> Namespaces;
+  llvm::SplitString(CppNamespace, Namespaces, "::");
+  for (auto Ns : Namespaces)
+    OS << "using namespace " << Ns << ";\n";
+
+  // getDirectiveKind(StringRef Str)
+  GenerateGetKind(Directives, OS, "Directive", DirectivePrefix, LanguageName,
+                  CppNamespace, /*ImplicitAsUnknown=*/false);
+
+  // getDirectiveName(Directive Kind)
+  GenerateGetName(Directives, OS, "Directive", DirectivePrefix, LanguageName,
+                  CppNamespace);
+
+  // getClauseKind(StringRef Str)
+  GenerateGetKind(Clauses, OS, "Clause", ClausePrefix, LanguageName,
+                  CppNamespace, /*ImplicitAsUnknown=*/true);
+
+  // getClauseName(Clause Kind)
+  GenerateGetName(Clauses, OS, "Clause", ClausePrefix, LanguageName,
+                  CppNamespace);
+
+  // isAllowedClauseForDirective(Directive D, Clause C, unsigned Version)
+  GenerateIsAllowedClause(Directives, OS, LanguageName, DirectivePrefix,
+                          ClausePrefix, CppNamespace);
+}
+
+} // namespace llvm
diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp
index ab42f33cf23fd..7e41914941498 100644
--- a/llvm/utils/TableGen/IntrinsicEmitter.cpp
+++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp
@@ -584,6 +584,9 @@ struct AttributeComparator {
     if (L->isNoSync != R->isNoSync)
       return R->isNoSync;
 
+    if (L->isNoFree != R->isNoFree)
+      return R->isNoFree;
+
     if (L->isWillReturn != R->isWillReturn)
       return R->isWillReturn;
 
@@ -751,10 +754,11 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
     }
 
     if (!intrinsic.canThrow ||
-        (intrinsic.ModRef != CodeGenIntrinsic::ReadWriteMem && !intrinsic.hasSideEffects) ||
-        intrinsic.isNoReturn || intrinsic.isNoSync || intrinsic.isWillReturn ||
-        intrinsic.isCold || intrinsic.isNoDuplicate || intrinsic.isConvergent ||
-        intrinsic.isSpeculatable) {
+        (intrinsic.ModRef != CodeGenIntrinsic::ReadWriteMem &&
+         !intrinsic.hasSideEffects) ||
+        intrinsic.isNoReturn || intrinsic.isNoSync || intrinsic.isNoFree ||
+        intrinsic.isWillReturn || intrinsic.isCold || intrinsic.isNoDuplicate ||
+        intrinsic.isConvergent || intrinsic.isSpeculatable) {
       OS << "      const Attribute::AttrKind Atts[] = {";
       bool addComma = false;
       if (!intrinsic.canThrow) {
@@ -773,6 +777,12 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
         OS << "Attribute::NoSync";
         addComma = true;
       }
+      if (intrinsic.isNoFree) {
+        if (addComma)
+          OS << ",";
+        OS << "Attribute::NoFree";
+        addComma = true;
+      }
       if (intrinsic.isWillReturn) {
         if (addComma)
           OS << ",";
diff --git a/llvm/utils/TableGen/OptParserEmitter.cpp b/llvm/utils/TableGen/OptParserEmitter.cpp
index 92346ab9b1116..251533a8d1545 100644
--- a/llvm/utils/TableGen/OptParserEmitter.cpp
+++ b/llvm/utils/TableGen/OptParserEmitter.cpp
@@ -10,11 +10,13 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <cctype>
 #include <cstring>
 #include <map>
+#include <memory>
 
 using namespace llvm;
 
@@ -33,6 +35,210 @@ static raw_ostream &write_cstring(raw_ostream &OS, llvm::StringRef Str) {
   return OS;
 }
 
+static const std::string getOptionSpelling(const Record &R,
+                                           size_t &PrefixLength) {
+  std::vector<StringRef> Prefixes = R.getValueAsListOfStrings("Prefixes");
+  StringRef Name = R.getValueAsString("Name");
+  if (Prefixes.empty()) {
+    PrefixLength = 0;
+    return Name.str();
+  }
+  PrefixLength = Prefixes[0].size();
+  return (Twine(Prefixes[0]) + Twine(Name)).str();
+}
+
+static const std::string getOptionSpelling(const Record &R) {
+  size_t PrefixLength;
+  return getOptionSpelling(R, PrefixLength);
+}
+
+static void emitNameUsingSpelling(raw_ostream &OS, const Record &R) {
+  size_t PrefixLength;
+  OS << "&";
+  write_cstring(OS, StringRef(getOptionSpelling(R, PrefixLength)));
+  OS << "[" << PrefixLength << "]";
+}
+
+class MarshallingKindInfo {
+public:
+  const Record &R;
+  const char *MacroName;
+  bool ShouldAlwaysEmit;
+  StringRef KeyPath;
+  StringRef DefaultValue;
+  StringRef NormalizedValuesScope;
+
+  void emit(raw_ostream &OS) const {
+    write_cstring(OS, StringRef(getOptionSpelling(R)));
+    OS << ", ";
+    OS << ShouldAlwaysEmit;
+    OS << ", ";
+    OS << KeyPath;
+    OS << ", ";
+    emitScopedNormalizedValue(OS, DefaultValue);
+    OS << ", ";
+    emitSpecific(OS);
+  }
+
+  virtual Optional<StringRef> emitValueTable(raw_ostream &OS) const {
+    return None;
+  }
+
+  virtual ~MarshallingKindInfo() = default;
+
+  static std::unique_ptr<MarshallingKindInfo> create(const Record &R);
+
+protected:
+  void emitScopedNormalizedValue(raw_ostream &OS,
+                                 StringRef NormalizedValue) const {
+    if (!NormalizedValuesScope.empty())
+      OS << NormalizedValuesScope << "::";
+    OS << NormalizedValue;
+  }
+
+  virtual void emitSpecific(raw_ostream &OS) const = 0;
+  MarshallingKindInfo(const Record &R, const char *MacroName)
+      : R(R), MacroName(MacroName) {}
+};
+
+class MarshallingFlagInfo final : public MarshallingKindInfo {
+public:
+  bool IsPositive;
+
+  void emitSpecific(raw_ostream &OS) const override { OS << IsPositive; }
+
+  static std::unique_ptr<MarshallingKindInfo> create(const Record &R) {
+    std::unique_ptr<MarshallingFlagInfo> Ret(new MarshallingFlagInfo(R));
+    Ret->IsPositive = R.getValueAsBit("IsPositive");
+    return Ret;
+  }
+
+private:
+  MarshallingFlagInfo(const Record &R)
+      : MarshallingKindInfo(R, "OPTION_WITH_MARSHALLING_FLAG") {}
+};
+
+class MarshallingStringInfo final : public MarshallingKindInfo {
+public:
+  StringRef NormalizerRetTy;
+  StringRef Normalizer;
+  StringRef Denormalizer;
+  int TableIndex = -1;
+  std::vector<StringRef> Values;
+  std::vector<StringRef> NormalizedValues;
+  std::string ValueTableName;
+
+  static constexpr const char *ValueTablePreamble = R"(
+struct SimpleEnumValue {
+  const char *Name;
+  unsigned Value;
+};
+
+struct SimpleEnumValueTable {
+  const SimpleEnumValue *Table;
+  unsigned Size;
+};
+)";
+
+  static constexpr const char *ValueTablesDecl =
+      "static const SimpleEnumValueTable SimpleEnumValueTables[] = ";
+
+  void emitSpecific(raw_ostream &OS) const override {
+    emitScopedNormalizedValue(OS, NormalizerRetTy);
+    OS << ", ";
+    OS << Normalizer;
+    OS << ", ";
+    OS << Denormalizer;
+    OS << ", ";
+    OS << TableIndex;
+  }
+
+  Optional<StringRef> emitValueTable(raw_ostream &OS) const override {
+    if (TableIndex == -1)
+      return {};
+    OS << "static const SimpleEnumValue " << ValueTableName << "[] = {\n";
+    for (unsigned I = 0, E = Values.size(); I != E; ++I) {
+      OS << "{";
+      write_cstring(OS, Values[I]);
+      OS << ",";
+      OS << "static_cast<unsigned>(";
+      emitScopedNormalizedValue(OS, NormalizedValues[I]);
+      OS << ")},";
+    }
+    OS << "};\n";
+    return StringRef(ValueTableName);
+  }
+
+  static std::unique_ptr<MarshallingKindInfo> create(const Record &R) {
+    assert(!isa<UnsetInit>(R.getValueInit("NormalizerRetTy")) &&
+           "String options must have a type");
+
+    std::unique_ptr<MarshallingStringInfo> Ret(new MarshallingStringInfo(R));
+    Ret->NormalizerRetTy = R.getValueAsString("NormalizerRetTy");
+
+    Ret->Normalizer = R.getValueAsString("Normalizer");
+    Ret->Denormalizer = R.getValueAsString("Denormalizer");
+
+    if (!isa<UnsetInit>(R.getValueInit("NormalizedValues"))) {
+      assert(!isa<UnsetInit>(R.getValueInit("Values")) &&
+             "Cannot provide normalized values for value-less options");
+      Ret->TableIndex = NextTableIndex++;
+      Ret->NormalizedValues = R.getValueAsListOfStrings("NormalizedValues");
+      Ret->Values.reserve(Ret->NormalizedValues.size());
+      Ret->ValueTableName = getOptionName(R) + "ValueTable";
+
+      StringRef ValuesStr = R.getValueAsString("Values");
+      for (;;) {
+        size_t Idx = ValuesStr.find(',');
+        if (Idx == StringRef::npos)
+          break;
+        if (Idx > 0)
+          Ret->Values.push_back(ValuesStr.slice(0, Idx));
+        ValuesStr = ValuesStr.slice(Idx + 1, StringRef::npos);
+      }
+      if (!ValuesStr.empty())
+        Ret->Values.push_back(ValuesStr);
+
+      assert(Ret->Values.size() == Ret->NormalizedValues.size() &&
+             "The number of normalized values doesn't match the number of "
+             "values");
+    }
+
+    return Ret;
+  }
+
+private:
+  MarshallingStringInfo(const Record &R)
+      : MarshallingKindInfo(R, "OPTION_WITH_MARSHALLING_STRING") {}
+
+  static size_t NextTableIndex;
+};
+
+size_t MarshallingStringInfo::NextTableIndex = 0;
+
+std::unique_ptr<MarshallingKindInfo>
+MarshallingKindInfo::create(const Record &R) {
+  assert(!isa<UnsetInit>(R.getValueInit("KeyPath")) &&
+         !isa<UnsetInit>(R.getValueInit("DefaultValue")) &&
+         "Must provide at least a key-path and a default value for emitting "
+         "marshalling information");
+
+  std::unique_ptr<MarshallingKindInfo> Ret = nullptr;
+  StringRef MarshallingKindStr = R.getValueAsString("MarshallingKind");
+
+  if (MarshallingKindStr == "flag")
+    Ret = MarshallingFlagInfo::create(R);
+  else if (MarshallingKindStr == "string")
+    Ret = MarshallingStringInfo::create(R);
+
+  Ret->ShouldAlwaysEmit = R.getValueAsBit("ShouldAlwaysEmit");
+  Ret->KeyPath = R.getValueAsString("KeyPath");
+  Ret->DefaultValue = R.getValueAsString("DefaultValue");
+  if (!isa<UnsetInit>(R.getValueInit("NormalizedValuesScope")))
+    Ret->NormalizedValuesScope = R.getValueAsString("NormalizedValuesScope");
+  return Ret;
+}
+
 /// OptParserEmitter - This tablegen backend takes an input .td file
 /// describing a list of options and emits a data structure for parsing and
 /// working with those options when given an input command line.
@@ -135,18 +341,14 @@ void EmitOptParser(RecordKeeper &Records, raw_ostream &OS) {
 
   OS << "//////////\n";
   OS << "// Options\n\n";
-  for (unsigned I = 0, E = Opts.size(); I != E; ++I) {
-    const Record &R = *Opts[I];
-
-    // Start a single option entry.
-    OS << "OPTION(";
 
+  auto WriteOptRecordFields = [&](raw_ostream &OS, const Record &R) {
     // The option prefix;
     std::vector<StringRef> prf = R.getValueAsListOfStrings("Prefixes");
     OS << Prefixes[PrefixKeyT(prf.begin(), prf.end())] << ", ";
 
     // The option string.
-    write_cstring(OS, R.getValueAsString("Name"));
+    emitNameUsingSpelling(OS, R);
 
     // The option identifier name.
     OS << ", " << getOptionName(R);
@@ -223,11 +425,52 @@ void EmitOptParser(RecordKeeper &Records, raw_ostream &OS) {
       write_cstring(OS, R.getValueAsString("Values"));
     else
       OS << "nullptr";
+  };
+
+  std::vector<std::unique_ptr<MarshallingKindInfo>> OptsWithMarshalling;
+  for (unsigned I = 0, E = Opts.size(); I != E; ++I) {
+    const Record &R = *Opts[I];
 
+    // Start a single option entry.
+    OS << "OPTION(";
+    WriteOptRecordFields(OS, R);
     OS << ")\n";
+    if (!isa<UnsetInit>(R.getValueInit("MarshallingKind")))
+      OptsWithMarshalling.push_back(MarshallingKindInfo::create(R));
   }
   OS << "#endif // OPTION\n";
 
+  for (const auto &KindInfo : OptsWithMarshalling) {
+    OS << "#ifdef " << KindInfo->MacroName << "\n";
+    OS << KindInfo->MacroName << "(";
+    WriteOptRecordFields(OS, KindInfo->R);
+    OS << ", ";
+    KindInfo->emit(OS);
+    OS << ")\n";
+    OS << "#endif // " << KindInfo->MacroName << "\n";
+  }
+
+  OS << "\n";
+  OS << "#ifdef SIMPLE_ENUM_VALUE_TABLE";
+  OS << "\n";
+  OS << MarshallingStringInfo::ValueTablePreamble;
+  std::vector<StringRef> ValueTableNames;
+  for (const auto &KindInfo : OptsWithMarshalling)
+    if (auto MaybeValueTableName = KindInfo->emitValueTable(OS))
+      ValueTableNames.push_back(*MaybeValueTableName);
+
+  OS << MarshallingStringInfo::ValueTablesDecl << "{";
+  for (auto ValueTableName : ValueTableNames)
+    OS << "{" << ValueTableName << ", sizeof(" << ValueTableName
+       << ") / sizeof(SimpleEnumValue)"
+       << "},\n";
+  OS << "};\n";
+  OS << "static const unsigned SimpleEnumValueTablesSize = "
+        "sizeof(SimpleEnumValueTables) / sizeof(SimpleEnumValueTable);\n";
+
+  OS << "#endif // SIMPLE_ENUM_VALUE_TABLE\n";
+  OS << "\n";
+
   OS << "\n";
   OS << "#ifdef OPTTABLE_ARG_INIT\n";
   OS << "//////////\n";
diff --git a/llvm/utils/TableGen/SearchableTableEmitter.cpp b/llvm/utils/TableGen/SearchableTableEmitter.cpp
index 0c79a2a6938fe..326cb4e54edce 100644
--- a/llvm/utils/TableGen/SearchableTableEmitter.cpp
+++ b/llvm/utils/TableGen/SearchableTableEmitter.cpp
@@ -119,9 +119,12 @@ class SearchableTableEmitter {
       return "Intrinsic::" + getIntrinsic(I).EnumName;
     else if (Field.IsInstruction)
       return I->getAsString();
-    else if (Field.Enum)
-      return std::string(
-          Field.Enum->EntryMap[cast<DefInit>(I)->getDef()]->first);
+    else if (Field.Enum) {
+      auto *Entry = Field.Enum->EntryMap[cast<DefInit>(I)->getDef()];
+      if (!Entry)
+        PrintFatalError(Twine("Entry for field '") + Field.Name + "' is null");
+      return std::string(Entry->first);
+    }
     PrintFatalError(Twine("invalid field type for field '") + Field.Name +
                     "', expected: string, bits, bit or code");
   }
@@ -596,6 +599,9 @@ void SearchableTableEmitter::collectEnumEntries(
 
 void SearchableTableEmitter::collectTableEntries(
     GenericTable &Table, const std::vector<Record *> &Items) {
+  if (Items.empty())
+    PrintWarning(Twine("Table '") + Table.Name + "' has no items");
+
   for (auto EntryRec : Items) {
     for (auto &Field : Table.Fields) {
       auto TI = dyn_cast<TypedInit>(EntryRec->getValueInit(Field.Name));
@@ -624,6 +630,10 @@ void SearchableTableEmitter::collectTableEntries(
   Record *IntrinsicClass = Records.getClass("Intrinsic");
   Record *InstructionClass = Records.getClass("Instruction");
   for (auto &Field : Table.Fields) {
+    if (!Field.RecType)
+      PrintFatalError(Twine("Cannot determine type of field '") + Field.Name +
+                      "' in table '" + Table.Name + "'. Maybe it is not used?");
+
     if (auto RecordTy = dyn_cast<RecordRecTy>(Field.RecType)) {
       if (IntrinsicClass && RecordTy->isSubClassOf(IntrinsicClass))
         Field.IsIntrinsic = true;
diff --git a/llvm/utils/TableGen/TableGen.cpp b/llvm/utils/TableGen/TableGen.cpp
index 6da6599eb54ef..8015a58471cad 100644
--- a/llvm/utils/TableGen/TableGen.cpp
+++ b/llvm/utils/TableGen/TableGen.cpp
@@ -54,6 +54,9 @@ enum ActionType {
   GenRegisterBank,
   GenExegesis,
   GenAutomata,
+  GenDirectivesEnumDecl,
+  GenDirectivesEnumImpl,
+  GenDirectivesEnumGen,
 };
 
 namespace llvm {
@@ -128,7 +131,13 @@ cl::opt<ActionType> Action(
                    "Generate registers bank descriptions"),
         clEnumValN(GenExegesis, "gen-exegesis",
                    "Generate llvm-exegesis tables"),
-        clEnumValN(GenAutomata, "gen-automata", "Generate generic automata")));
+        clEnumValN(GenAutomata, "gen-automata", "Generate generic automata"),
+        clEnumValN(GenDirectivesEnumDecl, "gen-directive-decl",
+                   "Generate directive related declaration code (header file)"),
+        clEnumValN(GenDirectivesEnumImpl, "gen-directive-impl",
+                   "Generate directive related implementation code"),
+        clEnumValN(GenDirectivesEnumGen, "gen-directive-gen",
+                   "Generate directive related implementation code part")));
 
 cl::OptionCategory PrintEnumsCat("Options for -print-enums");
 cl::opt<std::string> Class("class", cl::desc("Print Enum list for this class"),
@@ -253,6 +262,15 @@ bool LLVMTableGenMain(raw_ostream &OS, RecordKeeper &Records) {
   case GenAutomata:
     EmitAutomata(Records, OS);
     break;
+  case GenDirectivesEnumDecl:
+    EmitDirectivesDecl(Records, OS);
+    break;
+  case GenDirectivesEnumImpl:
+    EmitDirectivesImpl(Records, OS);
+    break;
+  case GenDirectivesEnumGen:
+    EmitDirectivesGen(Records, OS);
+    break;
   }
 
   return false;
diff --git a/llvm/utils/TableGen/TableGenBackends.h b/llvm/utils/TableGen/TableGenBackends.h
index 9eef77a4577f0..92204f39f8fa0 100644
--- a/llvm/utils/TableGen/TableGenBackends.h
+++ b/llvm/utils/TableGen/TableGenBackends.h
@@ -90,6 +90,9 @@ void EmitX86FoldTables(RecordKeeper &RK, raw_ostream &OS);
 void EmitRegisterBank(RecordKeeper &RK, raw_ostream &OS);
 void EmitExegesis(RecordKeeper &RK, raw_ostream &OS);
 void EmitAutomata(RecordKeeper &RK, raw_ostream &OS);
+void EmitDirectivesDecl(RecordKeeper &RK, raw_ostream &OS);
+void EmitDirectivesImpl(RecordKeeper &RK, raw_ostream &OS);
+void EmitDirectivesGen(RecordKeeper &RK, raw_ostream &OS);
 
 } // End llvm namespace
 
diff --git a/llvm/utils/TableGen/X86ModRMFilters.cpp b/llvm/utils/TableGen/X86ModRMFilters.cpp
index 98e6fb6104d77..cf7507094fa7f 100644
--- a/llvm/utils/TableGen/X86ModRMFilters.cpp
+++ b/llvm/utils/TableGen/X86ModRMFilters.cpp
@@ -18,4 +18,6 @@ void ModFilter::anchor() { }
 
 void ExtendedFilter::anchor() { }
 
+void ExtendedRMFilter::anchor() { }
+
 void ExactFilter::anchor() { }
diff --git a/llvm/utils/TableGen/X86ModRMFilters.h b/llvm/utils/TableGen/X86ModRMFilters.h
index c77b4c21aec41..f0b8af5fb82ab 100644
--- a/llvm/utils/TableGen/X86ModRMFilters.h
+++ b/llvm/utils/TableGen/X86ModRMFilters.h
@@ -108,6 +108,29 @@ class ExtendedFilter : public ModRMFilter {
   }
 };
 
+/// ExtendedRMFilter - Extended opcodes are classified based on the value of the
+///   mod field [bits 7-6] and the value of the nnn field [bits 2-0].
+class ExtendedRMFilter : public ModRMFilter {
+  void anchor() override;
+  bool R;
+  uint8_t NNN;
+public:
+  /// Constructor
+  ///
+  /// \param r   True if the mod field must be set to 11; false otherwise.
+  ///            The name is explained at ModFilter.
+  /// \param nnn The required value of the nnn field.
+  ExtendedRMFilter(bool r, uint8_t nnn) :
+    ModRMFilter(),
+    R(r),
+    NNN(nnn) {
+  }
+
+  bool accepts(uint8_t modRM) const override {
+    return ((R && ((modRM & 0xc0) == 0xc0)) &&
+            ((modRM & 0x7) == NNN));
+  }
+};
 /// ExactFilter - The occasional extended opcode (such as VMCALL or MONITOR)
 ///   requires the ModR/M byte to have a specific value.
 class ExactFilter : public ModRMFilter {
diff --git a/llvm/utils/TableGen/X86RecognizableInstr.cpp b/llvm/utils/TableGen/X86RecognizableInstr.cpp
index f31e43bd9fca0..84f6d5210d745 100644
--- a/llvm/utils/TableGen/X86RecognizableInstr.cpp
+++ b/llvm/utils/TableGen/X86RecognizableInstr.cpp
@@ -352,10 +352,13 @@ void RecognizableInstr::adjustOperandEncoding(OperandEncoding &encoding) {
   // The scaling factor for AVX512 compressed displacement encoding is an
   // instruction attribute.  Adjust the ModRM encoding type to include the
   // scale for compressed displacement.
-  if ((encoding != ENCODING_RM && encoding != ENCODING_VSIB) ||CD8_Scale == 0)
+  if ((encoding != ENCODING_RM &&
+       encoding != ENCODING_VSIB &&
+       encoding != ENCODING_SIB) ||CD8_Scale == 0)
     return;
   encoding = (OperandEncoding)(encoding + Log2_32(CD8_Scale));
   assert(((encoding >= ENCODING_RM && encoding <= ENCODING_RM_CD64) ||
+          (encoding == ENCODING_SIB) ||
           (encoding >= ENCODING_VSIB && encoding <= ENCODING_VSIB_CD64)) &&
          "Invalid CDisp scaling");
 }
@@ -519,6 +522,7 @@ void RecognizableInstr::emitInstructionSpecifier() {
     HANDLE_OPTIONAL(immediate)
     break;
   case X86Local::MRMDestMem:
+  case X86Local::MRMDestMemFSIB:
     // Operand 1 is a memory operand (possibly SIB-extended)
     // Operand 2 is a register operand in the Reg/Opcode field.
     // - In AVX, there is a register operand in the VEX.vvvv field here -
@@ -589,6 +593,7 @@ void RecognizableInstr::emitInstructionSpecifier() {
     HANDLE_OPERAND(opcodeModifier)
     break;
   case X86Local::MRMSrcMem:
+  case X86Local::MRMSrcMemFSIB:
     // Operand 1 is a register operand in the Reg/Opcode field.
     // Operand 2 is a memory operand (possibly SIB-extended)
     // - In AVX, there is a register operand in the VEX.vvvv field here -
@@ -641,6 +646,10 @@ void RecognizableInstr::emitInstructionSpecifier() {
     HANDLE_OPERAND(rmRegister)
     HANDLE_OPERAND(opcodeModifier)
     break;
+  case X86Local::MRMr0:
+    // Operand 1 is a register operand in the R/M field.
+    HANDLE_OPERAND(roRegister)
+    break;
   case X86Local::MRMXr:
   case X86Local::MRM0r:
   case X86Local::MRM1r:
@@ -772,7 +781,9 @@ void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const {
     filter = std::make_unique<ModFilter>(true);
     break;
   case X86Local::MRMDestMem:
+  case X86Local::MRMDestMemFSIB:
   case X86Local::MRMSrcMem:
+  case X86Local::MRMSrcMemFSIB:
   case X86Local::MRMSrcMem4VOp3:
   case X86Local::MRMSrcMemOp4:
   case X86Local::MRMSrcMemCC:
@@ -792,6 +803,9 @@ void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const {
   case X86Local::MRM6X: case X86Local::MRM7X:
     filter = std::make_unique<ExtendedFilter>(true, Form - X86Local::MRM0X);
     break;
+  case X86Local::MRMr0:
+    filter = std::make_unique<ExtendedRMFilter>(true, Form - X86Local::MRMr0);
+    break;
   case X86Local::MRM0m: case X86Local::MRM1m:
   case X86Local::MRM2m: case X86Local::MRM3m:
   case X86Local::MRM4m: case X86Local::MRM5m:
@@ -911,6 +925,7 @@ OperandType RecognizableInstr::typeFromString(const std::string &s,
   TYPE("i64imm",              TYPE_IMM)
   TYPE("anymem",              TYPE_M)
   TYPE("opaquemem",           TYPE_M)
+  TYPE("sibmem",              TYPE_MSIB)
   TYPE("SEGMENT_REG",         TYPE_SEGMENTREG)
   TYPE("DEBUG_REG",           TYPE_DEBUGREG)
   TYPE("CONTROL_REG",         TYPE_CONTROLREG)
@@ -969,6 +984,7 @@ OperandType RecognizableInstr::typeFromString(const std::string &s,
   TYPE("vz256mem",            TYPE_MVSIBZ)
   TYPE("vz512mem",            TYPE_MVSIBZ)
   TYPE("BNDR",                TYPE_BNDR)
+  TYPE("TILE",                TYPE_TMM)
   errs() << "Unhandled type string " << s << "\n";
   llvm_unreachable("Unhandled type string");
 }
@@ -1008,6 +1024,7 @@ RecognizableInstr::immediateEncodingFromString(const std::string &s,
   ENCODING("VR128X",          ENCODING_IB)
   ENCODING("VR256X",          ENCODING_IB)
   ENCODING("VR512",           ENCODING_IB)
+  ENCODING("TILE",            ENCODING_IB)
   errs() << "Unhandled immediate encoding " << s << "\n";
   llvm_unreachable("Unhandled immediate encoding");
 }
@@ -1046,6 +1063,7 @@ RecognizableInstr::rmRegisterEncodingFromString(const std::string &s,
   ENCODING("VK8PAIR",         ENCODING_RM)
   ENCODING("VK16PAIR",        ENCODING_RM)
   ENCODING("BNDR",            ENCODING_RM)
+  ENCODING("TILE",            ENCODING_RM)
   errs() << "Unhandled R/M register encoding " << s << "\n";
   llvm_unreachable("Unhandled R/M register encoding");
 }
@@ -1092,6 +1110,7 @@ RecognizableInstr::roRegisterEncodingFromString(const std::string &s,
   ENCODING("VK32WM",          ENCODING_REG)
   ENCODING("VK64WM",          ENCODING_REG)
   ENCODING("BNDR",            ENCODING_REG)
+  ENCODING("TILE",            ENCODING_REG)
   errs() << "Unhandled reg/opcode register encoding " << s << "\n";
   llvm_unreachable("Unhandled reg/opcode register encoding");
 }
@@ -1123,6 +1142,7 @@ RecognizableInstr::vvvvRegisterEncodingFromString(const std::string &s,
   ENCODING("VK4PAIR",         ENCODING_VVVV)
   ENCODING("VK8PAIR",         ENCODING_VVVV)
   ENCODING("VK16PAIR",        ENCODING_VVVV)
+  ENCODING("TILE",            ENCODING_VVVV)
   errs() << "Unhandled VEX.vvvv register encoding " << s << "\n";
   llvm_unreachable("Unhandled VEX.vvvv register encoding");
 }
@@ -1163,6 +1183,7 @@ RecognizableInstr::memoryEncodingFromString(const std::string &s,
   ENCODING("lea64mem",        ENCODING_RM)
   ENCODING("anymem",          ENCODING_RM)
   ENCODING("opaquemem",       ENCODING_RM)
+  ENCODING("sibmem",          ENCODING_SIB)
   ENCODING("vx64mem",         ENCODING_VSIB)
   ENCODING("vx128mem",        ENCODING_VSIB)
   ENCODING("vx256mem",        ENCODING_VSIB)
diff --git a/llvm/utils/TableGen/X86RecognizableInstr.h b/llvm/utils/TableGen/X86RecognizableInstr.h
index 2bf6d1a39803b..a7b88b4d12ed5 100644
--- a/llvm/utils/TableGen/X86RecognizableInstr.h
+++ b/llvm/utils/TableGen/X86RecognizableInstr.h
@@ -103,6 +103,9 @@ namespace X86Local {
     RawFrmImm16   = 8,
     AddCCFrm      = 9,
     PrefixByte    = 10,
+    MRMr0          = 21,
+    MRMSrcMemFSIB  = 22,
+    MRMDestMemFSIB = 23,
     MRMDestMem     = 24,
     MRMSrcMem      = 25,
     MRMSrcMem4VOp3 = 26,
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index 9abb8d790c5f5..7b4a9b3e6d165 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -1,9 +1,10 @@
 from __future__ import print_function
+
+import copy
+import glob
 import re
-import string
 import subprocess
 import sys
-import copy
 
 if sys.version_info[0] > 2:
   class string:
@@ -21,11 +22,85 @@ def parse_commandline_args(parser):
                       help='Show verbose output')
   parser.add_argument('-u', '--update-only', action='store_true',
                       help='Only update test if it was already autogened')
+  parser.add_argument('--force-update', action='store_true',
+                      help='Update test even if it was autogened by a different script')
+  parser.add_argument('--enable', action='store_true', dest='enabled', default=True,
+                       help='Activate CHECK line generation from this point forward')
+  parser.add_argument('--disable', action='store_false', dest='enabled',
+                      help='Deactivate CHECK line generation from this point forward')
   args = parser.parse_args()
   global _verbose
   _verbose = args.verbose
   return args
 
+
+class InputLineInfo(object):
+  def __init__(self, line, line_number, args, argv):
+    self.line = line
+    self.line_number = line_number
+    self.args = args
+    self.argv = argv
+
+
+class TestInfo(object):
+  def __init__(self, test, parser, script_name, input_lines, args, argv,
+               comment_prefix):
+    self.parser = parser
+    self.path = test
+    self.args = args
+    self.argv = argv
+    self.input_lines = input_lines
+    self.run_lines = find_run_lines(test, self.input_lines)
+    self.comment_prefix = comment_prefix
+    if self.comment_prefix is None:
+      if self.path.endswith('.mir'):
+        self.comment_prefix = '#'
+      else:
+        self.comment_prefix = ';'
+    self.autogenerated_note_prefix = self.comment_prefix + ' ' + UTC_ADVERT
+    self.test_autogenerated_note = self.autogenerated_note_prefix + script_name
+    self.test_autogenerated_note += get_autogennote_suffix(parser, self.args)
+
+  def iterlines(self, output_lines):
+    output_lines.append(self.test_autogenerated_note)
+    for line_num, input_line in enumerate(self.input_lines):
+      # Discard any previous script advertising.
+      if input_line.startswith(self.autogenerated_note_prefix):
+        continue
+      self.args, self.argv = check_for_command(input_line, self.parser,
+                                               self.args, self.argv)
+      if not self.args.enabled:
+        output_lines.append(input_line)
+        continue
+      yield InputLineInfo(input_line, line_num, self.args, self.argv)
+
+
+def itertests(test_patterns, parser, script_name, comment_prefix=None):
+  for pattern in test_patterns:
+    # On Windows we must expand the patterns ourselves.
+    tests_list = glob.glob(pattern)
+    if not tests_list:
+      warn("Test file pattern '%s' was not found. Ignoring it." % (pattern,))
+      continue
+    for test in tests_list:
+      with open(test) as f:
+        input_lines = [l.rstrip() for l in f]
+      args = parser.parse_args()
+      argv = sys.argv[:]
+      first_line = input_lines[0] if input_lines else ""
+      if UTC_ADVERT in first_line:
+        if script_name not in first_line and not args.force_update:
+          warn("Skipping test which wasn't autogenerated by " + script_name, test)
+          continue
+        args, argv = check_for_command(first_line, parser, args, argv)
+      elif args.update_only:
+        assert UTC_ADVERT not in first_line
+        warn("Skipping test which isn't autogenerated: " + test)
+        continue
+      yield TestInfo(test, parser, script_name, input_lines, args, argv,
+                     comment_prefix)
+
+
 def should_add_line_to_output(input_line, prefix_set):
   # Skip any blank comment lines in the IR.
   if input_line.strip() == ';':
@@ -57,7 +132,6 @@ def invoke_tool(exe, cmd_args, ir):
   return stdout.replace('\r\n', '\n')
 
 ##### LLVM IR parser
-
 RUN_LINE_RE = re.compile(r'^\s*(?://|[;#])\s*RUN:\s*(.*)$')
 CHECK_PREFIX_RE = re.compile(r'--?check-prefix(?:es)?[= ](\S+)')
 PREFIX_RE = re.compile('^[a-zA-Z0-9_-]+$')
@@ -65,6 +139,7 @@ def invoke_tool(exe, cmd_args, ir):
 
 UTC_ARGS_KEY = 'UTC_ARGS:'
 UTC_ARGS_CMD = re.compile(r'.*' + UTC_ARGS_KEY + '\s*(?P<cmd>.*)\s*$')
+UTC_ADVERT = 'NOTE: Assertions have been autogenerated by '
 
 OPT_FUNCTION_RE = re.compile(
     r'^\s*define\s+(?:internal\s+)?[^@]*@(?P<func>[\w.-]+?)\s*'
diff --git a/llvm/utils/git/arcfilter.sh b/llvm/utils/git/arcfilter.sh
new file mode 100755
index 0000000000000..cf6215c933342
--- /dev/null
+++ b/llvm/utils/git/arcfilter.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# Remove the phabricator tags from the commit at HEAD
+git log -1 --pretty=%B | \
+  sed  's/^Summary://'  | \
+  awk '/Reviewers:|Subscribers:/{p=1} /Reviewed By:|Differential Revision:/{p=0} !p && !/^Summary:$/ {sub(/^Summary: /,"");print}' | \
+  git commit --amend --date=now -F -
diff --git a/llvm/utils/git/pre-push.py b/llvm/utils/git/pre-push.py
new file mode 100755
index 0000000000000..7ea41a9b75737
--- /dev/null
+++ b/llvm/utils/git/pre-push.py
@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+#
+# ======- pre-push - LLVM Git Help Integration ---------*- python -*--========#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ==------------------------------------------------------------------------==#
+
+"""
+pre-push git hook integration
+=============================
+
+This script is intended to be setup as a pre-push hook, from the root of the
+repo run:
+
+   ln -sf ../../llvm/utils/git/pre-push.py .git/hooks/pre-push
+
+From the git doc:
+
+  The pre-push hook runs during git push, after the remote refs have been
+  updated but before any objects have been transferred. It receives the name
+  and location of the remote as parameters, and a list of to-be-updated refs
+  through stdin. You can use it to validate a set of ref updates before a push
+  occurs (a non-zero exit code will abort the push).
+"""
+
+import argparse
+import collections
+import os
+import re
+import shutil
+import subprocess
+import sys
+import time
+import getpass
+from shlex import quote
+
+VERBOSE = False
+QUIET = False
+dev_null_fd = None
+z40 = '0000000000000000000000000000000000000000'
+
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+def log(*args, **kwargs):
+    if QUIET:
+        return
+    print(*args, **kwargs)
+
+
+def log_verbose(*args, **kwargs):
+    if not VERBOSE:
+        return
+    print(*args, **kwargs)
+
+
+def die(msg):
+    eprint(msg)
+    sys.exit(1)
+
+
+def ask_confirm(prompt):
+    while True:
+        query = input('%s (y/N): ' % (prompt))
+        if query.lower() not in ['y', 'n', '']:
+           print('Expect y or n!')
+           continue
+        return query.lower() == 'y'
+
+
+def get_dev_null():
+    """Lazily create a /dev/null fd for use in shell()"""
+    global dev_null_fd
+    if dev_null_fd is None:
+        dev_null_fd = open(os.devnull, 'w')
+    return dev_null_fd
+
+
+def shell(cmd, strip=True, cwd=None, stdin=None, die_on_failure=True,
+          ignore_errors=False, text=True, print_raw_stderr=False):
+    # Escape args when logging for easy repro.
+    quoted_cmd = [quote(arg) for arg in cmd]
+    cwd_msg = ''
+    if cwd:
+      cwd_msg = ' in %s' % cwd
+    log_verbose('Running%s: %s' % (cwd_msg, ' '.join(quoted_cmd)))
+
+    err_pipe = subprocess.PIPE
+    if ignore_errors:
+        # Silence errors if requested.
+        err_pipe = get_dev_null()
+
+    start = time.time()
+    p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=err_pipe,
+                         stdin=subprocess.PIPE,
+                         universal_newlines=text)
+    stdout, stderr = p.communicate(input=stdin)
+    elapsed = time.time() - start
+
+    log_verbose('Command took %0.1fs' % elapsed)
+
+    if p.returncode == 0 or ignore_errors:
+        if stderr and not ignore_errors:
+            if not print_raw_stderr:
+                eprint('`%s` printed to stderr:' % ' '.join(quoted_cmd))
+            eprint(stderr.rstrip())
+        if strip:
+            if text:
+                stdout = stdout.rstrip('\r\n')
+            else:
+                stdout = stdout.rstrip(b'\r\n')
+        if VERBOSE:
+            for l in stdout.splitlines():
+                log_verbose('STDOUT: %s' % l)
+        return stdout
+    err_msg = '`%s` returned %s' % (' '.join(quoted_cmd), p.returncode)
+    eprint(err_msg)
+    if stderr:
+        eprint(stderr.rstrip())
+    if die_on_failure:
+        sys.exit(2)
+    raise RuntimeError(err_msg)
+
+
+def git(*cmd, **kwargs):
+    return shell(['git'] + list(cmd), **kwargs)
+
+
+def get_revs_to_push(range):
+    commits = git('rev-list', range).splitlines()
+    # Reverse the order so we print the oldest commit first
+    commits.reverse()
+    return commits
+
+
+def handle_push(args, local_ref, local_sha, remote_ref, remote_sha):
+    '''Check a single push request (which can include multiple revisions)'''
+    log_verbose('Handle push, reproduce with '
+                '`echo %s %s %s %s | pre-push.py %s %s'
+                 % (local_ref, local_sha, remote_ref, remote_sha, args.remote,
+                    args.url))
+    # Handle request to delete
+    if local_sha == z40:
+        if not ask_confirm('Are you sure you want to delete "%s" on remote "%s"?' % (remote_ref, args.url)):
+            die("Aborting")
+        return
+
+    # Push a new branch
+    if remote_sha == z40:
+      if not ask_confirm('Are you sure you want to push a new branch/tag "%s" on remote "%s"?' % (remote_ref, args.url)):
+        die("Aborting")
+      range=local_sha
+      return
+    else:
+      # Update to existing branch, examine new commits
+      range='%s..%s' % (remote_sha, local_sha)
+      # Check that the remote commit exists, otherwise let git proceed
+      if "commit" not in git('cat-file','-t', remote_sha, ignore_errors=True):
+          return
+
+    revs = get_revs_to_push(range)
+    if not revs:
+        # This can happen if someone is force pushing an older revision to a branch
+        return
+
+    # Print the revision about to be pushed commits
+    print('Pushing to "%s" on remote "%s"' % (remote_ref, args.url))
+    for sha in revs:
+      print(' - ' + git('show', '--oneline', '--quiet', sha))
+
+    if len(revs) > 1:
+      if not ask_confirm('Are you sure you want to push %d commits?' % len(revs)):
+          die('Aborting')
+
+
+    for sha in revs:
+      msg = git('log', '--format=%B', '-n1', sha)
+      if 'Differential Revision' not in msg:
+          continue
+      for line in msg.splitlines():
+          for tag in ['Summary', 'Reviewers', 'Subscribers', 'Tags']:
+            if line.startswith(tag + ':'):
+              eprint('Please remove arcanist tags from the commit message (found "%s" tag in %s)' % (tag, sha[:12]))
+              if len(revs) == 1:
+                  eprint('Try running: llvm/utils/git/arcfilter.sh')
+              die('Aborting (force push by adding "--no-verify")')
+
+    return
+
+
+if __name__ == '__main__':
+    if not shutil.which('git'):
+        die('error: cannot find git command')
+
+    argv = sys.argv[1:]
+    p = argparse.ArgumentParser(
+        prog='pre-push', formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=__doc__)
+    verbosity_group = p.add_mutually_exclusive_group()
+    verbosity_group.add_argument('-q', '--quiet', action='store_true',
+                                 help='print less information')
+    verbosity_group.add_argument('-v', '--verbose', action='store_true',
+                                 help='print more information')
+
+    p.add_argument('remote', type=str, help='Name of the remote')
+    p.add_argument('url', type=str, help='URL for the remote')
+
+    args = p.parse_args(argv)
+    VERBOSE = args.verbose
+    QUIET = args.quiet
+
+    lines = sys.stdin.readlines()
+    sys.stdin = open('/dev/tty', 'r')
+    for line in lines:
+      local_ref, local_sha, remote_ref, remote_sha = line.split()
+      handle_push(args, local_ref, local_sha, remote_ref, remote_sha)
diff --git a/llvm/utils/gn/build/BUILD.gn b/llvm/utils/gn/build/BUILD.gn
index f926eee5afe6b..e29cdb678a361 100644
--- a/llvm/utils/gn/build/BUILD.gn
+++ b/llvm/utils/gn/build/BUILD.gn
@@ -43,6 +43,10 @@ config("compiler_defaults") {
       "-isysroot",
       mac_sdk_path,
     ]
+    ldflags += [
+      "-isysroot",
+      mac_sdk_path,
+    ]
   }
 
   if (host_os != "win") {
diff --git a/llvm/utils/gn/build/toolchain/BUILD.gn b/llvm/utils/gn/build/toolchain/BUILD.gn
index 86e95d3de188c..1f963ca40905e 100644
--- a/llvm/utils/gn/build/toolchain/BUILD.gn
+++ b/llvm/utils/gn/build/toolchain/BUILD.gn
@@ -11,8 +11,26 @@ declare_args() {
 
 template("unix_toolchain") {
   toolchain(target_name) {
+    # https://groups.google.com/a/chromium.org/d/msg/gn-dev/F_lv5T-tNDM
+    forward_variables_from(invoker.toolchain_args, "*")
+    not_needed("*")
+
     forward_variables_from(invoker, "*")
 
+    cc = "cc"
+    cxx = "c++"
+
+    if (clang_base_path != "") {
+      cc = "$clang_base_path/bin/clang"
+      cxx = "$clang_base_path/bin/clang++"
+    }
+
+    ld = cxx  # Don't use goma wrapper for linking.
+    if (use_goma) {
+      cc = "$goma_dir/gomacc $cc"
+      cxx = "$goma_dir/gomacc $cxx"
+    }
+
     tool("cc") {
       depfile = "{{output}}.d"
       command = "$cc -MMD -MF $depfile -o {{output}} -c {{source}} {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}}"
@@ -128,20 +146,6 @@ template("unix_toolchain") {
 }
 
 unix_toolchain("unix") {
-  cc = "cc"
-  cxx = "c++"
-
-  if (clang_base_path != "") {
-    cc = "$clang_base_path/bin/clang"
-    cxx = "$clang_base_path/bin/clang++"
-  }
-
-  ld = cxx  # Don't use goma wrapper for linking.
-  if (use_goma) {
-    cc = "$goma_dir/gomacc $cc"
-    cxx = "$goma_dir/gomacc $cxx"
-  }
-
   if (current_os != "mac") {
     ar = "ar"
   }
@@ -156,13 +160,11 @@ unix_toolchain("unix") {
 # as compiler and linker.
 template("stage2_unix_toolchain") {
   unix_toolchain(target_name) {
-    forward_variables_from(invoker, "*")
+    toolchain_args = {
+      forward_variables_from(invoker.toolchain_args, "*")
 
-    cc = "bin/clang"
-    cxx = "bin/clang++"
-    ld = cxx
-    if (current_os != "mac") {
-      ar = "bin/llvm-ar"
+      clang_base_path = "."
+      use_goma = false
     }
 
     deps = [
@@ -170,6 +172,7 @@ template("stage2_unix_toolchain") {
       "//:lld($host_toolchain)",
     ]
     if (current_os != "mac") {
+      ar = "bin/llvm-ar"
       deps += [ "//:llvm-ar($host_toolchain)" ]
     }
   }
@@ -179,8 +182,6 @@ stage2_unix_toolchain("stage2_unix") {
   toolchain_args = {
     current_os = host_os
     current_cpu = host_cpu
-    is_clang = true
-    use_lld = host_os != "mac"
   }
 }
 
@@ -189,8 +190,6 @@ if (android_ndk_path != "") {
     toolchain_args = {
       current_os = "android"
       current_cpu = "arm64"
-      is_clang = true
-      use_lld = true
     }
   }
 
@@ -198,8 +197,6 @@ if (android_ndk_path != "") {
     toolchain_args = {
       current_os = "android"
       current_cpu = "arm"
-      is_clang = true
-      use_lld = true
     }
   }
 }
diff --git a/llvm/utils/gn/get.py b/llvm/utils/gn/get.py
index adb8eb7d57151..f778ab931bba3 100755
--- a/llvm/utils/gn/get.py
+++ b/llvm/utils/gn/get.py
@@ -32,12 +32,12 @@ def set_executable_bit(path):
 
 def get_platform():
     import platform
+    if sys.platform == 'darwin':
+        return 'mac-amd64' if platform.machine() != 'arm64' else 'mac-arm64'
     if platform.machine() not in ('AMD64', 'x86_64'):
         return None
     if sys.platform.startswith('linux'):
         return 'linux-amd64'
-    if sys.platform == 'darwin':
-        return 'mac-amd64'
     if sys.platform == 'win32':
         return 'windows-amd64'
 
@@ -47,6 +47,14 @@ def main():
     if not platform:
         print('no prebuilt binary for', sys.platform)
         return 1
+    if platform == 'mac-arm64':
+        print('no prebuilt mac-arm64 binaries yet. build it yourself with:')
+        print('  rm -rf /tmp/gn &&')
+        print('  pushd /tmp && git clone https://gn.googlesource.com/gn &&')
+        print('  cd gn && build/gen.py && ninja -C out gn && popd &&')
+        print('  mkdir -p llvm/utils/gn/bin/mac-arm64 &&')
+        print('  cp /tmp/gn/out/gn llvm/utils/gn/bin/mac-arm64')
+        return 1
 
     dirname = os.path.join(os.path.dirname(__file__), 'bin', platform)
     if not os.path.exists(dirname):
diff --git a/llvm/utils/gn/gn.py b/llvm/utils/gn/gn.py
index a6408ec065a35..4708d4c3b6024 100755
--- a/llvm/utils/gn/gn.py
+++ b/llvm/utils/gn/gn.py
@@ -17,12 +17,12 @@
 
 def get_platform():
     import platform
+    if sys.platform == 'darwin':
+        return 'mac-amd64' if platform.machine() != 'arm64' else 'mac-arm64'
     if platform.machine() not in ('AMD64', 'x86_64'):
         return None
     if sys.platform.startswith('linux'):
         return 'linux-amd64'
-    if sys.platform == 'darwin':
-        return 'mac-amd64'
     if sys.platform == 'win32':
         return 'windows-amd64'
 
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
index a34f9dcf9c0a0..45ecef719fdb0 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
@@ -39,6 +39,7 @@ static_library("bugprone") {
     "MisplacedWideningCastCheck.cpp",
     "MoveForwardingReferenceCheck.cpp",
     "MultipleStatementMacroCheck.cpp",
+    "NoEscapeCheck.cpp",
     "NotNullTerminatedResultCheck.cpp",
     "ParentVirtualCallCheck.cpp",
     "PosixReturnCheck.cpp",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/utils/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/utils/BUILD.gn
index 1e8c4fd93f27a..dffeea2bea384 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/utils/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/utils/BUILD.gn
@@ -12,6 +12,7 @@ static_library("utils") {
   ]
   sources = [
     "ASTUtils.cpp",
+    "Aliasing.cpp",
     "DeclRefExprUtils.cpp",
     "ExceptionAnalyzer.cpp",
     "ExprSequence.cpp",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn
index 135e195fa35fd..dade4cc73bf9f 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn
@@ -61,6 +61,10 @@ static_library("clangd") {
     "CollectMacros.cpp",
     "CompileCommands.cpp",
     "Compiler.cpp",
+    "Config.cpp",
+    "ConfigCompile.cpp",
+    "ConfigProvider.cpp",
+    "ConfigYAML.cpp",
     "Diagnostics.cpp",
     "DraftStore.cpp",
     "ExpectedTypes.cpp",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn
index b893c4f140375..b45aa8d44dc2a 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn
@@ -41,6 +41,9 @@ unittest("ClangdTests") {
     "CollectMacrosTests.cpp",
     "CompileCommandsTests.cpp",
     "CompilerTests.cpp",
+    "ConfigCompileTests.cpp",
+    "ConfigProviderTests.cpp",
+    "ConfigYAMLTests.cpp",
     "DexTests.cpp",
     "DiagnosticsTests.cpp",
     "DraftStoreTests.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn
index baec5310976cd..3c573395993d1 100644
--- a/llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn
@@ -91,6 +91,7 @@ static_library("Basic") {
     "Targets/Sparc.cpp",
     "Targets/SystemZ.cpp",
     "Targets/TCE.cpp",
+    "Targets/VE.cpp",
     "Targets/WebAssembly.cpp",
     "Targets/X86.cpp",
     "Targets/XCore.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn
index 9dd07a463dc38..04d483a12eed5 100644
--- a/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn
@@ -52,6 +52,7 @@ static_library("Driver") {
     "ToolChains/Arch/RISCV.cpp",
     "ToolChains/Arch/Sparc.cpp",
     "ToolChains/Arch/SystemZ.cpp",
+    "ToolChains/Arch/VE.cpp",
     "ToolChains/Arch/X86.cpp",
     "ToolChains/BareMetal.cpp",
     "ToolChains/Clang.cpp",
@@ -86,6 +87,7 @@ static_library("Driver") {
     "ToolChains/RISCVToolchain.cpp",
     "ToolChains/Solaris.cpp",
     "ToolChains/TCE.cpp",
+    "ToolChains/VEToolchain.cpp",
     "ToolChains/WebAssembly.cpp",
     "ToolChains/XCore.cpp",
     "Types.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
index 9c86dbff22ad4..d1fc6ad4d9799 100644
--- a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
@@ -82,6 +82,7 @@ copy("Headers") {
     "adxintrin.h",
     "altivec.h",
     "ammintrin.h",
+    "amxintrin.h",
     "arm64intr.h",
     "arm_acle.h",
     "arm_cmse.h",
@@ -157,6 +158,7 @@ copy("Headers") {
     "opencl-c.h",
     "openmp_wrappers/__clang_openmp_device_functions.h",
     "openmp_wrappers/cmath",
+    "openmp_wrappers/complex.h",
     "openmp_wrappers/math.h",
     "pconfigintrin.h",
     "pkuintrin.h",
diff --git a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
index c5f57a89623f9..8a930b4597740 100644
--- a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
@@ -107,6 +107,7 @@ static_library("Checkers") {
     "RunLoopAutoreleaseLeakChecker.cpp",
     "STLAlgorithmModeling.cpp",
     "SimpleStreamChecker.cpp",
+    "SmartPtrChecker.cpp",
     "SmartPtrModeling.cpp",
     "StackAddrEscapeChecker.cpp",
     "StdLibraryFunctionsChecker.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Core/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Core/BUILD.gn
index 4c3d175a31808..772fa4dfdae96 100644
--- a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Core/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Core/BUILD.gn
@@ -25,6 +25,7 @@ static_library("Core") {
     "CheckerContext.cpp",
     "CheckerHelpers.cpp",
     "CheckerManager.cpp",
+    "CheckerRegistryData.cpp",
     "CommonBugCategories.cpp",
     "ConstraintManager.cpp",
     "CoreEngine.cpp",
diff --git a/llvm/utils/gn/secondary/clang/test/BUILD.gn b/llvm/utils/gn/secondary/clang/test/BUILD.gn
index 4c7300147fa5d..56feaae9a149d 100644
--- a/llvm/utils/gn/secondary/clang/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/test/BUILD.gn
@@ -86,6 +86,8 @@ write_lit_config("lit_site_cfg") {
 
   if (host_cpu == "x64") {
     extra_values += [ "HOST_ARCH=x86_64" ]
+  } else if (host_cpu == "arm64") {
+    extra_values += [ "HOST_ARCH=arm64" ]
   } else if (host_cpu == "ppc64") {
     extra_values += [ "HOST_ARCH=powerpc64le" ]
   } else {
diff --git a/llvm/utils/gn/secondary/clang/unittests/Frontend/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Frontend/BUILD.gn
index f3b535cf486c2..14cf53aa513f1 100644
--- a/llvm/utils/gn/secondary/clang/unittests/Frontend/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/Frontend/BUILD.gn
@@ -16,6 +16,7 @@ unittest("FrontendTests") {
     "ASTUnitTest.cpp",
     "CodeGenActionTest.cpp",
     "CompilerInstanceTest.cpp",
+    "CompilerInvocationTest.cpp",
     "FixedPointString.cpp",
     "FrontendActionTest.cpp",
     "OutputStreamTest.cpp",
diff --git a/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn
index 0a3ac33e8af0e..dcd3fa8ed5d6a 100644
--- a/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn
@@ -14,6 +14,7 @@ unittest("StaticAnalysisTests") {
     "AnalyzerOptionsTest.cpp",
     "CallDescriptionTest.cpp",
     "CallEventTest.cpp",
+    "FalsePositiveRefutationBRVisitorTest.cpp",
     "ParamRegionTest.cpp",
     "RangeSetTest.cpp",
     "RegisterCustomCheckersTest.cpp",
diff --git a/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
index 147797db22781..c807389846b78 100644
--- a/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
@@ -41,6 +41,7 @@ unittest("ToolingTests") {
     "RecursiveASTVisitorTests/CXXMemberCall.cpp",
     "RecursiveASTVisitorTests/CXXMethodDecl.cpp",
     "RecursiveASTVisitorTests/CXXOperatorCallExprTraverser.cpp",
+    "RecursiveASTVisitorTests/Callbacks.cpp",
     "RecursiveASTVisitorTests/Class.cpp",
     "RecursiveASTVisitorTests/ConstructExpr.cpp",
     "RecursiveASTVisitorTests/DeclRefExpr.cpp",
diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
index 7793901770f3b..1291a5d33cbb4 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
@@ -449,13 +449,9 @@ static_library("builtins") {
       "hexagon/dfsqrt.S",
       "hexagon/divdi3.S",
       "hexagon/divsi3.S",
-      "hexagon/fabs_opt.S",
       "hexagon/fastmath2_dlib_asm.S",
       "hexagon/fastmath2_ldlib_asm.S",
       "hexagon/fastmath_dlib_asm.S",
-      "hexagon/fma_opt.S",
-      "hexagon/fmax_opt.S",
-      "hexagon/fmin_opt.S",
       "hexagon/memcpy_forward_vp4cp4n2.S",
       "hexagon/memcpy_likely_aligned.S",
       "hexagon/moddi3.S",
diff --git a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
index a6e9e21eac050..1c52d341ec822 100644
--- a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
@@ -95,7 +95,6 @@ config("cxx_config") {
     } else {
       cflags_cc += [ "-fno-exceptions" ]
     }
-    defines += [ "_LIBCPP_NO_EXCEPTIONS" ]
   }
   if (!libcxx_enable_rtti) {
     if (current_os == "win") {
@@ -135,6 +134,7 @@ cxx_sources = [
   "new.cpp",
   "optional.cpp",
   "random.cpp",
+  "random_shuffle.cpp",
   "regex.cpp",
   "shared_mutex.cpp",
   "stdexcept.cpp",
diff --git a/llvm/utils/gn/secondary/libcxxabi/src/BUILD.gn b/llvm/utils/gn/secondary/libcxxabi/src/BUILD.gn
index 44712caf4d010..8d43581c3669b 100644
--- a/llvm/utils/gn/secondary/libcxxabi/src/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxxabi/src/BUILD.gn
@@ -30,7 +30,6 @@ cxxabi_sources = [
   "cxa_exception_storage.cpp",
   "cxa_guard.cpp",
   "cxa_handlers.cpp",
-  "cxa_unexpected.cpp",
   "cxa_vector.cpp",
   "cxa_virtual.cpp",
 
diff --git a/llvm/utils/gn/secondary/lld/test/BUILD.gn b/llvm/utils/gn/secondary/lld/test/BUILD.gn
index 71d20751c5d72..96a6b07d39c1b 100644
--- a/llvm/utils/gn/secondary/lld/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/test/BUILD.gn
@@ -85,6 +85,7 @@ group("test") {
     "//llvm/tools/llvm-cvtres",
     "//llvm/tools/llvm-dis",
     "//llvm/tools/llvm-dwarfdump",
+    "//llvm/tools/llvm-lipo",
     "//llvm/tools/llvm-mc",
     "//llvm/tools/llvm-nm:symlinks",
     "//llvm/tools/llvm-objcopy:symlinks",
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenMP/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenMP/BUILD.gn
new file mode 100644
index 0000000000000..a18f8db5f5ebd
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenMP/BUILD.gn
@@ -0,0 +1,18 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("OMP") {
+  visibility = [ ":public_tablegen" ]
+  args = [ "-gen-directive-decl" ]
+  output_name = "OMP.h.inc"
+}
+
+# Groups all tablegen() calls that create .inc files that are included in
+# Frontent/OpenMP's public headers (just one so far).
+# //llvm/lib/Frontend/OpenMP has this as a public_dep, so targets depending on
+# //llvm/lib/Frontend/OpenMP don't need to depend on this.
+group("public_tablegen") {
+  public_deps = [
+    # Frontend/OpenMP's public headers include OMP.h.inc.
+    ":OMP",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
index cfffefe0475d7..11498ed602984 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
@@ -88,7 +88,6 @@ static_library("Analysis") {
     "ObjCARCAnalysisUtils.cpp",
     "ObjCARCInstKind.cpp",
     "OptimizationRemarkEmitter.cpp",
-    "OrderedInstructions.cpp",
     "PHITransAddr.cpp",
     "PhiValues.cpp",
     "PostDominators.cpp",
@@ -99,6 +98,7 @@ static_library("Analysis") {
     "RegionPrinter.cpp",
     "ScalarEvolution.cpp",
     "ScalarEvolutionAliasAnalysis.cpp",
+    "ScalarEvolutionDivision.cpp",
     "ScalarEvolutionNormalization.cpp",
     "ScopedNoAliasAA.cpp",
     "StackLifetime.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenMP/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenMP/BUILD.gn
index b852d3b357ad0..688a25e3c1df1 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenMP/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenMP/BUILD.gn
@@ -1,13 +1,23 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("OMPImpl") {
+  visibility = [ ":OpenMP" ]
+  args = [ "-gen-directive-impl" ]
+  td_file = "//llvm/include/llvm/Frontend/OpenMP/OMP.td"
+  output_name = "OMP.cpp"
+}
+
 static_library("OpenMP") {
   output_name = "LLVMFrontendOpenMP"
   deps = [
+    ":OMPImpl",
     "//llvm/lib/IR",
     "//llvm/lib/Support",
     "//llvm/lib/Transforms/Utils",
   ]
+  public_deps = [ "//llvm/include/llvm/Frontend/OpenMP:public_tablegen" ]
   sources = [
-    "OMPConstants.cpp",
     "OMPContext.cpp",
     "OMPIRBuilder.cpp",
-  ]
+  ] + get_target_outputs(":OMPImpl")
 }
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
index 963d5bba50bae..1cf6f028e7483 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
@@ -64,6 +64,7 @@ static_library("LLVMRISCVCodeGen") {
   sources = [
     "RISCVAsmPrinter.cpp",
     "RISCVCallLowering.cpp",
+    "RISCVExpandAtomicPseudoInsts.cpp",
     "RISCVExpandPseudoInsts.cpp",
     "RISCVFrameLowering.cpp",
     "RISCVISelDAGToDAG.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 6d68a4bb0537f..d433385dccad6 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -83,10 +83,13 @@ write_lit_config("lit_site_cfg") {
     "OCAMLFIND=OCAMLFIND-NOTFOUND",
     "OCAMLFLAGS=",
     "LLVM_BUILD_EXAMPLES=0",
+    "LLVM_HAVE_TF_AOT=0",
   ]
 
   if (host_cpu == "x64") {
     extra_values += [ "HOST_ARCH=x86_64" ]
+  } else if (host_cpu == "arm64") {
+    extra_values += [ "HOST_ARCH=arm64" ]
   } else if (host_cpu == "ppc64") {
     extra_values += [ "HOST_ARCH=powerpc64le" ]
   } else {
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-mc/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-mc/BUILD.gn
index 653c5c01d5968..b8063cb62de66 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-mc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-mc/BUILD.gn
@@ -3,10 +3,7 @@ executable("llvm-mc") {
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCParser",
     "//llvm/lib/Support",
-    "//llvm/lib/Target:AllTargetsAsmParsers",
-    "//llvm/lib/Target:AllTargetsDescs",
-    "//llvm/lib/Target:AllTargetsDisassemblers",
-    "//llvm/lib/Target:AllTargetsInfos",
+    "//llvm/lib/Target:TargetsToBuild",
   ]
   sources = [
     "Disassembler.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-ml/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-ml/BUILD.gn
index ac8027c95f748..5a8aed0183904 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-ml/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-ml/BUILD.gn
@@ -3,10 +3,7 @@ executable("llvm-ml") {
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCParser",
     "//llvm/lib/Support",
-    "//llvm/lib/Target:AllTargetsAsmParsers",
-    "//llvm/lib/Target:AllTargetsDescs",
-    "//llvm/lib/Target:AllTargetsDisassemblers",
-    "//llvm/lib/Target:AllTargetsInfos",
+    "//llvm/lib/Target:TargetsToBuild",
   ]
   sources = [
     "Disassembler.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-reduce/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-reduce/BUILD.gn
index efa80c1b86d80..a8648d73ca0d1 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-reduce/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-reduce/BUILD.gn
@@ -12,11 +12,13 @@ executable("llvm-reduce") {
     "TestRunner.cpp",
     "deltas/Delta.cpp",
     "deltas/ReduceArguments.cpp",
+    "deltas/ReduceAttributes.cpp",
     "deltas/ReduceBasicBlocks.cpp",
     "deltas/ReduceFunctions.cpp",
     "deltas/ReduceGlobalVars.cpp",
     "deltas/ReduceInstructions.cpp",
     "deltas/ReduceMetadata.cpp",
+    "deltas/ReduceOperandBundles.cpp",
     "llvm-reduce.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/triples.gni b/llvm/utils/gn/secondary/llvm/triples.gni
index c6bef4bbd819e..c72910f02f990 100644
--- a/llvm/utils/gn/secondary/llvm/triples.gni
+++ b/llvm/utils/gn/secondary/llvm/triples.gni
@@ -19,6 +19,8 @@ if (current_cpu == "x86") {
 } else if (current_cpu == "arm64") {
   if (current_os == "android") {
     llvm_current_triple = "aarch64-linux-android29"
+  } else if (current_os == "mac") {
+    llvm_current_triple = "arm64-apple-darwin"
   }
 } else if (current_cpu == "ppc64") {
   if (current_os == "linux") {
diff --git a/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
index 0695d5d691352..8183cbf0c04c8 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
@@ -15,6 +15,7 @@ unittest("ADTTests") {
     "APSIntTest.cpp",
     "AnyTest.cpp",
     "ArrayRefTest.cpp",
+    "BitFieldsTest.cpp",
     "BitVectorTest.cpp",
     "BitmaskEnumTest.cpp",
     "BreadthFirstIteratorTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
index 191d848378042..b0dcd497d844e 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
@@ -31,7 +31,6 @@ unittest("AnalysisTests") {
     "LoopNestTest.cpp",
     "MemoryBuiltinsTest.cpp",
     "MemorySSATest.cpp",
-    "OrderedInstructionsTest.cpp",
     "PhiValuesTest.cpp",
     "ProfileSummaryInfoTest.cpp",
     "ScalarEvolutionTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/CodeGen/GlobalISel/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/CodeGen/GlobalISel/BUILD.gn
index 8ac5c8b8d2450..5104716bccdd6 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/CodeGen/GlobalISel/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/CodeGen/GlobalISel/BUILD.gn
@@ -15,6 +15,7 @@ unittest("GlobalISelTests") {
     "CSETest.cpp",
     "ConstantFoldingTest.cpp",
     "GISelMITest.cpp",
+    "GISelUtilsTest.cpp",
     "KnownBitsTest.cpp",
     "LegalizerHelperTest.cpp",
     "LegalizerInfoTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/ObjectYAML/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ObjectYAML/BUILD.gn
index 7a5855292c403..ddb3c74ddbbaf 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/ObjectYAML/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/ObjectYAML/BUILD.gn
@@ -7,6 +7,7 @@ unittest("ObjectYAMLTests") {
     "//llvm/lib/Testing/Support",
   ]
   sources = [
+    "DWARFYAMLTest.cpp",
     "ELFYAMLTest.cpp",
     "MinidumpYAMLTest.cpp",
     "YAML2ObjTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
index b390326d05ebc..4559926899c9f 100644
--- a/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
@@ -28,6 +28,7 @@ executable("llvm-tblgen") {
     "DAGISelMatcherOpt.cpp",
     "DFAEmitter.cpp",
     "DFAPacketizerEmitter.cpp",
+    "DirectiveEmitter.cpp",
     "DisassemblerEmitter.cpp",
     "ExegesisEmitter.cpp",
     "FastISelEmitter.cpp",
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 803e5f9ba02b1..966863624feb9 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -65,12 +65,18 @@ def parse_args():
             dest="useProgressBar",
             help="Do not use curses based progress bar",
             action="store_false")
-    format_group.add_argument("--show-unsupported",
-            help="Show unsupported tests",
-            action="store_true")
-    format_group.add_argument("--show-xfail",
-            help="Show tests that were expected to fail",
-            action="store_true")
+
+    # Note: this does not generate flags for user-defined result codes.
+    success_codes = [c for c in lit.Test.ResultCode.all_codes()
+                     if not c.isFailure]
+    for code in success_codes:
+        format_group.add_argument(
+            "--show-{}".format(code.name.lower()),
+            dest="shown_codes",
+            help="Show {} tests ({})".format(code.label.lower(), code.name),
+            action="append_const",
+            const=code,
+            default=[])
 
     execution_group = parser.add_argument_group("Test Execution")
     execution_group.add_argument("--path",
@@ -187,12 +193,6 @@ def parse_args():
     else:
         opts.shard = None
 
-    opts.show_results = set()
-    if opts.show_unsupported:
-        opts.show_results.add(lit.Test.UNSUPPORTED)
-    if opts.show_xfail:
-        opts.show_results.add(lit.Test.XFAIL)
-
     opts.reports = filter(None, [opts.output, opts.xunit_xml_output])
 
     return opts
diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py
index fe0f1f4441ac2..db557a7b1fefe 100644
--- a/llvm/utils/lit/lit/llvm/config.py
+++ b/llvm/utils/lit/lit/llvm/config.py
@@ -51,12 +51,14 @@ def __init__(self, lit_config, config):
         elif platform.system() == 'Windows':
             # For tests that require Windows to run.
             features.add('system-windows')
-        elif platform.system() == "Linux":
+        elif platform.system() == 'Linux':
             features.add('system-linux')
         elif platform.system() in ['FreeBSD']:
             features.add('system-freebsd')
-        elif platform.system() == "NetBSD":
+        elif platform.system() == 'NetBSD':
             features.add('system-netbsd')
+        elif platform.system() == 'AIX':
+            features.add('system-aix')
 
         # Native compilation: host arch == default triple arch
         # Both of these values should probably be in every site config (e.g. as
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 860c584fbdf49..c47bdede3176c 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -265,15 +265,15 @@ def print_results(tests, elapsed, opts):
         tests_by_code[test.result.code].append(test)
 
     for code in lit.Test.ResultCode.all_codes():
-        print_group(tests_by_code[code], code, opts.show_results)
+        print_group(tests_by_code[code], code, opts.shown_codes)
 
     print_summary(tests_by_code, opts.quiet, elapsed)
 
 
-def print_group(tests, code, show_results):
+def print_group(tests, code, shown_codes):
     if not tests:
         return
-    if not code.isFailure and code not in show_results:
+    if not code.isFailure and code not in shown_codes:
         return
     print('*' * 20)
     print('{} Tests ({}):'.format(code.label, len(tests)))
diff --git a/llvm/utils/lit/tests/Inputs/show-result-codes/fail.txt b/llvm/utils/lit/tests/Inputs/show-result-codes/fail.txt
new file mode 100644
index 0000000000000..15eb81a5f5e95
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-result-codes/fail.txt
@@ -0,0 +1 @@
+RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/show-result-codes/lit.cfg b/llvm/utils/lit/tests/Inputs/show-result-codes/lit.cfg
new file mode 100644
index 0000000000000..2aa84326bcea8
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-result-codes/lit.cfg
@@ -0,0 +1,6 @@
+import lit.formats
+config.name = 'show-result-codes'
+config.suffixes = ['.txt']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
diff --git a/llvm/utils/lit/tests/Inputs/show-result-codes/pass.txt b/llvm/utils/lit/tests/Inputs/show-result-codes/pass.txt
new file mode 100644
index 0000000000000..18efe9e49e95b
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-result-codes/pass.txt
@@ -0,0 +1 @@
+RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/show-result-codes/unsupported.txt b/llvm/utils/lit/tests/Inputs/show-result-codes/unsupported.txt
new file mode 100644
index 0000000000000..b1f70207f1fdd
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-result-codes/unsupported.txt
@@ -0,0 +1,2 @@
+REQUIRES: missing-feature
+RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/show-result-codes/xfail.txt b/llvm/utils/lit/tests/Inputs/show-result-codes/xfail.txt
new file mode 100644
index 0000000000000..6f2e4e08ba189
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/show-result-codes/xfail.txt
@@ -0,0 +1,2 @@
+XFAIL: *
+RUN: false
diff --git a/llvm/utils/lit/tests/lit.cfg b/llvm/utils/lit/tests/lit.cfg
index 85bdbf180b135..3c49f076a66ee 100644
--- a/llvm/utils/lit/tests/lit.cfg
+++ b/llvm/utils/lit/tests/lit.cfg
@@ -87,7 +87,7 @@ if not llvm_config:
   if sys.platform.startswith('win') or sys.platform.startswith('cygwin'):
     config.available_features.add('system-windows')
   if platform.system() == 'AIX':
-    config.available_features.add('aix')
+    config.available_features.add('system-aix')
 
 # For each of lit's internal shell commands ('env', 'cd', 'diff', etc.), put
 # a fake command that always fails at the start of PATH.  This helps us check
diff --git a/llvm/utils/lit/tests/show-result-codes.py b/llvm/utils/lit/tests/show-result-codes.py
new file mode 100644
index 0000000000000..5d8cd0d9eb9fa
--- /dev/null
+++ b/llvm/utils/lit/tests/show-result-codes.py
@@ -0,0 +1,21 @@
+# Test the --show-<result-code> {pass,unsupported,xfail,...} options.
+#
+# RUN: not %{lit} %{inputs}/show-result-codes                           |  FileCheck %s --check-prefix=NONE
+# RUN: not %{lit} %{inputs}/show-result-codes --show-unsupported        |  FileCheck %s --check-prefix=ONE
+# RUN: not %{lit} %{inputs}/show-result-codes --show-pass --show-xfail  |  FileCheck %s --check-prefix=MULTIPLE
+
+# Failing tests are always shown
+# NONE-NOT: Unsupported Tests (1)
+# NONE-NOT: Passed Tests (1)
+# NONE-NOT: Expectedly Failed Tests (1)
+# NONE:     Failed Tests (1)
+
+# ONE:     Unsupported Tests (1)
+# ONE-NOT:     Passed Tests (1)
+# ONE-NOT: Expectedly Failed Tests (1)
+# ONE:     Failed Tests (1)
+
+# MULTIPLE-NOT: Unsupported Tests (1)
+# MULTIPLE:     Passed Tests (1)
+# MULTIPLE:     Expectedly Failed Tests (1)
+# MULTIPLE:     Failed Tests (1)
diff --git a/llvm/utils/lit/tests/shtest-format-argv0.py b/llvm/utils/lit/tests/shtest-format-argv0.py
index 063fa80a678a3..28f9acaa3322d 100644
--- a/llvm/utils/lit/tests/shtest-format-argv0.py
+++ b/llvm/utils/lit/tests/shtest-format-argv0.py
@@ -5,7 +5,7 @@
 #
 # This test is not supported on AIX since `[` is only available as a shell builtin
 # and is not installed under PATH by default.
-# UNSUPPORTED: aix
+# UNSUPPORTED: system-aix
 #
 # RUN: %{lit} -j 1 -v %{inputs}/shtest-format-argv0 | FileCheck %s
 
diff --git a/llvm/utils/update_cc_test_checks.py b/llvm/utils/update_cc_test_checks.py
index 7c69ff339cfb0..5851f19e3d2de 100755
--- a/llvm/utils/update_cc_test_checks.py
+++ b/llvm/utils/update_cc_test_checks.py
@@ -19,16 +19,13 @@
 import distutils.spawn
 import json
 import os
+import re
 import shlex
-import string
 import subprocess
 import sys
-import re
 import tempfile
 
-from UpdateTestChecks import asm, common
-
-ADVERT = '// NOTE: Assertions have been autogenerated by '
+from UpdateTestChecks import common
 
 SUBST = {
     '%clang': [],
@@ -110,6 +107,11 @@ def parse_clang_ast_json(node):
   return ret
 
 
+def str_to_commandline(value):
+  if not value:
+    return []
+  return shlex.split(value)
+
 def config():
   parser = argparse.ArgumentParser(
       description=__doc__,
@@ -117,7 +119,7 @@ def config():
   parser.add_argument('--llvm-bin', help='llvm $prefix/bin path')
   parser.add_argument('--clang',
                       help='"clang" executable, defaults to $llvm_bin/clang')
-  parser.add_argument('--clang-args',
+  parser.add_argument('--clang-args', default=[], type=str_to_commandline,
                       help='Space-separated extra args to clang, e.g. --clang-args=-v')
   parser.add_argument('--opt',
                       help='"opt" executable, defaults to $llvm_bin/opt')
@@ -131,7 +133,6 @@ def config():
                       help='Keep function signature information around for the check line')
   parser.add_argument('tests', nargs='+')
   args = common.parse_commandline_args(parser)
-  args.clang_args = shlex.split(args.clang_args or '')
 
   if args.clang is None:
     if args.llvm_bin is None:
@@ -165,7 +166,7 @@ def config():
     # defer this error message until we find that opt is actually needed.
     args.opt = None
 
-  return args
+  return args, parser
 
 
 def get_function_body(args, filename, clang_args, extra_commands, prefixes, triple_in_cmd, func_dict):
@@ -196,31 +197,15 @@ def get_function_body(args, filename, clang_args, extra_commands, prefixes, trip
 
 
 def main():
-  args = config()
+  initial_args, parser = config()
   script_name = os.path.basename(__file__)
-  autogenerated_note = (ADVERT + 'utils/' + script_name)
-
-  for filename in args.tests:
-    with open(filename) as f:
-      input_lines = [l.rstrip() for l in f]
-
-    first_line = input_lines[0] if input_lines else ""
-    if 'autogenerated' in first_line and script_name not in first_line:
-      common.warn("Skipping test which wasn't autogenerated by " + script_name, filename)
-      continue
-
-    if args.update_only:
-      if not first_line or 'autogenerated' not in first_line:
-        common.warn("Skipping test which isn't autogenerated: " + filename)
-        continue
-
-    # Extract RUN lines.
-    run_lines = common.find_run_lines(filename, input_lines)
 
+  for ti in common.itertests(initial_args.tests, parser, 'utils/' + script_name,
+                             comment_prefix='//'):
     # Build a list of clang command lines and check prefixes from RUN lines.
     run_list = []
     line2spell_and_mangled_list = collections.defaultdict(list)
-    for l in run_lines:
+    for l in ti.run_lines:
       commands = [cmd.strip() for cmd in l.split('|')]
 
       triple_in_cmd = None
@@ -234,7 +219,7 @@ def main():
         print('WARNING: Skipping non-clang RUN line: ' + l, file=sys.stderr)
         continue
       clang_args[0:1] = SUBST[clang_args[0]]
-      clang_args = [filename if i == '%s' else i for i in clang_args] + args.clang_args
+      clang_args = [ti.path if i == '%s' else i for i in clang_args] + ti.args.clang_args
 
       # Permit piping the output through opt
       if not (len(commands) == 2 or
@@ -253,18 +238,6 @@ def main():
         check_prefixes = ['CHECK']
       run_list.append((check_prefixes, clang_args, commands[1:-1], triple_in_cmd))
 
-    # Strip CHECK lines which are in `prefix_set`, update test file.
-    prefix_set = set([prefix for p in run_list for prefix in p[0]])
-    input_lines = []
-    with open(filename, 'r+') as f:
-      for line in f:
-        m = common.CHECK_RE.match(line)
-        if not (m and m.group(1) in prefix_set) and line != '//\n':
-          input_lines.append(line)
-      f.seek(0)
-      f.writelines(input_lines)
-      f.truncate()
-
     # Execute clang, generate LLVM IR, and extract functions.
     func_dict = {}
     for p in run_list:
@@ -275,18 +248,23 @@ def main():
       common.debug('Extracted clang cmd: clang {}'.format(clang_args))
       common.debug('Extracted FileCheck prefixes: {}'.format(prefixes))
 
-      get_function_body(args, filename, clang_args, extra_commands, prefixes, triple_in_cmd, func_dict)
+      get_function_body(ti.args, ti.path, clang_args, extra_commands, prefixes, triple_in_cmd, func_dict)
 
       # Invoke clang -Xclang -ast-dump=json to get mapping from start lines to
       # mangled names. Forward all clang args for now.
-      for k, v in get_line2spell_and_mangled(args, clang_args).items():
+      for k, v in get_line2spell_and_mangled(ti.args, clang_args).items():
         line2spell_and_mangled_list[k].append(v)
 
-    output_lines = [autogenerated_note]
-    for idx, line in enumerate(input_lines):
-      # Discard any previous script advertising.
-      if line.startswith(ADVERT):
-        continue
+    prefix_set = set([prefix for p in run_list for prefix in p[0]])
+    output_lines = []
+    for line_info in ti.iterlines(output_lines):
+      idx = line_info.line_number
+      line = line_info.line
+      args = line_info.args
+      include_line = True
+      m = common.CHECK_RE.match(line)
+      if m and m.group(1) in prefix_set:
+        continue  # Don't append the existing CHECK lines
       if idx in line2spell_and_mangled_list:
         added = set()
         for spell, mangled in line2spell_and_mangled_list[idx]:
@@ -298,16 +276,25 @@ def main():
           if mangled in added or spell not in line:
             continue
           if args.functions is None or any(re.search(regex, spell) for regex in args.functions):
+            last_line = output_lines[-1].strip()
+            while last_line == '//':
+              # Remove the comment line since we will generate a new  comment
+              # line as part of common.add_ir_checks()
+              output_lines.pop()
+              last_line = output_lines[-1].strip()
             if added:
               output_lines.append('//')
             added.add(mangled)
             common.add_ir_checks(output_lines, '//', run_list, func_dict, mangled,
                                  False, args.function_signature)
-      output_lines.append(line.rstrip('\n'))
+            if line.rstrip('\n') == '//':
+              include_line = False
 
+      if include_line:
+        output_lines.append(line.rstrip('\n'))
 
-    common.debug('Writing %d lines to %s...' % (len(output_lines), filename))
-    with open(filename, 'wb') as f:
+    common.debug('Writing %d lines to %s...' % (len(output_lines), ti.path))
+    with open(ti.path, 'wb') as f:
       f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 
   return 0
diff --git a/llvm/utils/update_llc_test_checks.py b/llvm/utils/update_llc_test_checks.py
index d873c60187b7c..05fca340760ee 100755
--- a/llvm/utils/update_llc_test_checks.py
+++ b/llvm/utils/update_llc_test_checks.py
@@ -10,19 +10,13 @@
 from __future__ import print_function
 
 import argparse
-import glob
-import os         # Used to advertise this file's name ("autogenerated_note").
-import string
-import subprocess
-import sys
-import re
+import os  # Used to advertise this file's name ("autogenerated_note").
 
 from UpdateTestChecks import asm, common
 
-ADVERT = ' NOTE: Assertions have been autogenerated by '
 # llc is the only llc-like in the LLVM tree but downstream forks can add
 # additional ones here if they have them.
-LLC_LIKE_TOOLS = ('llc',) 
+LLC_LIKE_TOOLS = ('llc',)
 
 def main():
   parser = argparse.ArgumentParser(description=__doc__)
@@ -42,35 +36,21 @@ def main():
       '--no_x86_scrub_mem_shuffle', action='store_true', default=False,
       help='Reduce scrubbing shuffles with memory operands')
   parser.add_argument('tests', nargs='+')
-  args = common.parse_commandline_args(parser)
+  initial_args = common.parse_commandline_args(parser)
 
   script_name = os.path.basename(__file__)
 
-  test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
-  for test in test_paths:
-    with open(test) as f:
-      input_lines = [l.rstrip() for l in f]
-
-    first_line = input_lines[0] if input_lines else ""
-    if 'autogenerated' in first_line and script_name not in first_line:
-      common.warn("Skipping test which wasn't autogenerated by " + script_name, test)
-      continue
-
-    if args.update_only:
-      if not first_line or 'autogenerated' not in first_line:
-        common.warn("Skipping test which isn't autogenerated: " + test)
-        continue
-
+  for ti in common.itertests(initial_args.tests, parser,
+                             script_name='utils/' + script_name):
     triple_in_ir = None
-    for l in input_lines:
+    for l in ti.input_lines:
       m = common.TRIPLE_IR_RE.match(l)
       if m:
         triple_in_ir = m.groups()[0]
         break
 
-    run_lines = common.find_run_lines(test, input_lines)
     run_list = []
-    for l in run_lines:
+    for l in ti.run_lines:
       if '|' not in l:
         common.warn('Skipping unparseable RUN line: ' + l)
         continue
@@ -103,7 +83,7 @@ def main():
 
       llc_cmd_args = llc_cmd[len(llc_tool):].strip()
       llc_cmd_args = llc_cmd_args.replace('< %s', '').replace('%s', '').strip()
-      if test.endswith('.mir'):
+      if ti.path.endswith('.mir'):
         llc_cmd_args += ' -x mir'
       check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
                                for item in m.group(1).split(',')]
@@ -114,13 +94,10 @@ def main():
       # now, we just ignore all but the last.
       run_list.append((check_prefixes, llc_cmd_args, triple_in_cmd, march_in_cmd))
 
-    if test.endswith('.mir'):
-      comment_sym = '#'
+    if ti.path.endswith('.mir'):
       check_indent = '  '
     else:
-      comment_sym = ';'
       check_indent = ''
-    autogenerated_note = (comment_sym + ADVERT + 'utils/' + script_name)
 
     func_dict = {}
     for p in run_list:
@@ -131,13 +108,12 @@ def main():
       common.debug('Extracted LLC cmd:', llc_tool, llc_args)
       common.debug('Extracted FileCheck prefixes:', str(prefixes))
 
-      raw_tool_output = common.invoke_tool(args.llc_binary or llc_tool,
-                                           llc_args, test)
+      raw_tool_output = common.invoke_tool(ti.args.llc_binary or llc_tool, llc_args, ti.path)
       triple = triple_in_cmd or triple_in_ir
       if not triple:
         triple = asm.get_triple_from_march(march_in_cmd)
 
-      asm.build_function_body_dictionary_for_triple(args, raw_tool_output,
+      asm.build_function_body_dictionary_for_triple(ti.args, raw_tool_output,
           triple, prefixes, func_dict)
 
     is_in_function = False
@@ -146,9 +122,9 @@ def main():
     prefix_set = set([prefix for p in run_list for prefix in p[0]])
     common.debug('Rewriting FileCheck prefixes:', str(prefix_set))
     output_lines = []
-    output_lines.append(autogenerated_note)
-
-    for input_line in input_lines:
+    for input_info in ti.iterlines(output_lines):
+      input_line = input_info.line
+      args = input_info.args
       if is_in_function_start:
         if input_line == '':
           continue
@@ -172,10 +148,6 @@ def main():
           is_in_function = False
         continue
 
-      # Discard any previous script advertising.
-      if input_line.startswith(comment_sym + ADVERT):
-        continue
-
       # If it's outside a function, it just gets copied to the output.
       output_lines.append(input_line)
 
@@ -188,9 +160,9 @@ def main():
         continue
       is_in_function = is_in_function_start = True
 
-    common.debug('Writing %d lines to %s...' % (len(output_lines), test))
+    common.debug('Writing %d lines to %s...' % (len(output_lines), ti.path))
 
-    with open(test, 'wb') as f:
+    with open(ti.path, 'wb') as f:
       f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 
 
diff --git a/llvm/utils/update_test_checks.py b/llvm/utils/update_test_checks.py
index 014b55c9ae839..be15ae9e26852 100755
--- a/llvm/utils/update_test_checks.py
+++ b/llvm/utils/update_test_checks.py
@@ -32,18 +32,12 @@
 from __future__ import print_function
 
 import argparse
-import glob
-import itertools
-import os         # Used to advertise this file's name ("autogenerated_note").
-import string
-import subprocess
-import sys
-import tempfile
+import os  # Used to advertise this file's name ("autogenerated_note").
 import re
+import sys
 
 from UpdateTestChecks import common
 
-ADVERT = '; NOTE: Assertions have been autogenerated by '
 
 def main():
   from argparse import RawTextHelpFormatter
@@ -58,58 +52,26 @@ def main():
                       help='Keep function signature information around for the check line')
   parser.add_argument('--scrub-attributes', action='store_true',
                       help='Remove attribute annotations (#0) from the end of check line')
-  parser.add_argument('--enable', action='store_true', dest='enabled', default=True,
-                      help='Activate CHECK line generation from this point forward')
-  parser.add_argument('--disable', action='store_false', dest='enabled',
-                      help='Deactivate CHECK line generation from this point forward')
   parser.add_argument('tests', nargs='+')
-  args = common.parse_commandline_args(parser)
+  initial_args = common.parse_commandline_args(parser)
 
   script_name = os.path.basename(__file__)
-  autogenerated_note = (ADVERT + 'utils/' + script_name)
-
-  opt_basename = os.path.basename(args.opt_binary)
+  opt_basename = os.path.basename(initial_args.opt_binary)
   if not re.match(r'^opt(-\d+)?$', opt_basename):
     common.error('Unexpected opt name: ' + opt_basename)
     sys.exit(1)
   opt_basename = 'opt'
 
-  for test in args.tests:
-    if not glob.glob(test):
-      common.warn("Test file pattern '%s' was not found. Ignoring it." % (test,))
-      continue
-
-  # On Windows we must expand the patterns ourselves.
-  test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
-  for test in test_paths:
-    argv = sys.argv[:]
-    args = parser.parse_args()
-    with open(test) as f:
-      input_lines = [l.rstrip() for l in f]
-
-    first_line = input_lines[0] if input_lines else ""
-    if 'autogenerated' in first_line and script_name not in first_line:
-      common.warn("Skipping test which wasn't autogenerated by " + script_name, test)
-      continue
-    if first_line and 'autogenerated' in first_line:
-      args, argv = common.check_for_command(first_line, parser, args, argv)
-    test_autogenerated_note = autogenerated_note + common.get_autogennote_suffix(parser, args)
-
-    if args.update_only:
-      if not first_line or 'autogenerated' not in first_line:
-        common.warn("Skipping test which isn't autogenerated: " + test)
-        continue
-
-    run_lines = common.find_run_lines(test, input_lines)
-
+  for ti in common.itertests(initial_args.tests, parser,
+                             script_name='utils/' + script_name):
     # If requested we scrub trailing attribute annotations, e.g., '#0', together with whitespaces
-    if args.scrub_attributes:
+    if ti.args.scrub_attributes:
       common.SCRUB_TRAILING_WHITESPACE_TEST_RE = common.SCRUB_TRAILING_WHITESPACE_AND_ATTRIBUTES_RE
     else:
       common.SCRUB_TRAILING_WHITESPACE_TEST_RE = common.SCRUB_TRAILING_WHITESPACE_RE
 
     prefix_list = []
-    for l in run_lines:
+    for l in ti.run_lines:
       if '|' not in l:
         common.warn('Skipping unparseable RUN line: ' + l)
         continue
@@ -127,8 +89,9 @@ def main():
       tool_cmd_args = tool_cmd[len(opt_basename):].strip()
       tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
 
-      check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
-                               for item in m.group(1).split(',')]
+      check_prefixes = [item for m in
+                        common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
+                        for item in m.group(1).split(',')]
       if not check_prefixes:
         check_prefixes = ['CHECK']
 
@@ -144,28 +107,20 @@ def main():
       common.debug('Extracted opt cmd: ' + opt_basename + ' ' + opt_args)
       common.debug('Extracted FileCheck prefixes: ' + str(prefixes))
 
-      raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test)
+      raw_tool_output = common.invoke_tool(ti.args.opt_binary, opt_args, ti.path)
       common.build_function_body_dictionary(
               common.OPT_FUNCTION_RE, common.scrub_body, [],
-              raw_tool_output, prefixes, func_dict, args.verbose,
-              args.function_signature)
+              raw_tool_output, prefixes, func_dict, ti.args.verbose,
+              ti.args.function_signature)
 
     is_in_function = False
     is_in_function_start = False
     prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
     common.debug('Rewriting FileCheck prefixes:', str(prefix_set))
     output_lines = []
-    output_lines.append(test_autogenerated_note)
-
-    for input_line in input_lines:
-      # Discard any previous script advertising.
-      if input_line.startswith(ADVERT):
-        continue
-
-      args, argv = common.check_for_command(input_line, parser, args, argv)
-      if not args.enabled:
-          output_lines.append(input_line)
-          continue
+    for input_line_info in ti.iterlines(output_lines):
+      input_line = input_line_info.line
+      args = input_line_info.args
       if is_in_function_start:
         if input_line == '':
           continue
@@ -204,9 +159,9 @@ def main():
         continue
       is_in_function = is_in_function_start = True
 
-    common.debug('Writing %d lines to %s...' % (len(output_lines), test))
+    common.debug('Writing %d lines to %s...' % (len(output_lines), ti.path))
 
-    with open(test, 'wb') as f:
+    with open(ti.path, 'wb') as f:
       f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 
 
diff --git a/mlir/CMakeLists.txt b/mlir/CMakeLists.txt
index 3a53a9bccd3ef..806eb42dbd05b 100644
--- a/mlir/CMakeLists.txt
+++ b/mlir/CMakeLists.txt
@@ -51,6 +51,38 @@ option(MLIR_INCLUDE_TESTS
 option(MLIR_INCLUDE_INTEGRATION_TESTS
        "Generate build targets for the MLIR integration tests.")
 
+#-------------------------------------------------------------------------------
+# Python Bindings Configuration
+# Requires:
+#   The pybind11 library can be found (set with -DPYBIND_DIR=...)
+#   The python executable is correct (set with -DPYTHON_EXECUTABLE=...)
+#
+# Version locking
+# ---------------
+# By default, python extensions are version locked to specific Python libraries.
+# This linking mode is somewhat more consistent across platforms and surfaces
+# undefined symbols at link time (vs runtime). It is suitable for development
+# workflows but can be disabled for more flexible deployment by
+# setting -DMLIR_PYTHON_BINDINGS_VERSION_LOCKED=OFF
+#-------------------------------------------------------------------------------
+
+set(MLIR_BINDINGS_PYTHON_ENABLED 0 CACHE BOOL
+       "Enables building of Python bindings.")
+set(MLIR_PYTHON_BINDINGS_VERSION_LOCKED 1 CACHE BOOL
+       "Links to specific python libraries, resolving all symbols.")
+
+if(MLIR_BINDINGS_PYTHON_ENABLED)
+  find_package(PythonInterp REQUIRED)
+  find_package(PythonLibs REQUIRED)
+  message(STATUS "Found python include dirs: ${PYTHON_INCLUDE_DIRS}")
+  message(STATUS "Found ppython libraries: ${PYTHON_LIBRARIES}")
+  find_package(pybind11 CONFIG REQUIRED)
+  message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIRS}")
+  message(STATUS "Python prefix = '${PYTHON_MODULE_PREFIX}', "
+                 "suffix = '${PYTHON_MODULE_SUFFIX}', "
+                 "extension = '${PYTHON_MODULE_EXTENSION}")
+endif()
+
 include_directories( "include")
 include_directories( ${MLIR_INCLUDE_DIR})
 
diff --git a/mlir/docs/Bindings/Python.md b/mlir/docs/Bindings/Python.md
new file mode 100644
index 0000000000000..8d9cee5e88ca6
--- /dev/null
+++ b/mlir/docs/Bindings/Python.md
@@ -0,0 +1,328 @@
+# MLIR Python Bindings
+
+Current status: Under development and not enabled by default
+
+
+## Building
+
+### Pre-requisites
+
+* [`pybind11`](https://github.com/pybind/pybind11) must be installed and able to
+  be located by CMake.
+* A relatively recent Python3 installation
+
+### CMake variables
+
+* **`MLIR_BINDINGS_PYTHON_ENABLED`**`:BOOL`
+
+  Enables building the Python bindings. Defaults to `OFF`.
+
+* **`MLIR_PYTHON_BINDINGS_VERSION_LOCKED`**`:BOOL`
+
+  Links the native extension against the Python runtime library, which is
+  optional on some platforms. While setting this to `OFF` can yield some greater
+  deployment flexibility, linking in this way allows the linker to report
+  compile time errors for unresolved symbols on all platforms, which makes for a
+  smoother development workflow. Defaults to `ON`.
+
+* **`PYTHON_EXECUTABLE`**:`STRING`
+
+  Specifies the `python` executable used for the LLVM build, including for
+  determining header/link flags for the Python bindings. On systems with
+  multiple Python implementations, setting this explicitly to the preferred
+  `python3` executable is strongly recommended.
+
+
+## Design
+
+### Use cases
+
+There are likely two primary use cases for the MLIR python bindings:
+
+1. Support users who expect that an installed version of LLVM/MLIR will yield
+   the ability to `import mlir` and use the API in a pure way out of the box.
+
+2. Downstream integrations will likely want to include parts of the API in their
+   private namespace or specially built libraries, probably mixing it with other
+   python native bits.
+
+
+### Composable modules
+
+In order to support use case #2, the Python bindings are organized into
+composable modules that downstream integrators can include and re-export into
+their own namespace if desired. This forces several design points:
+
+* Separate the construction/populating of a `py::module` from `PYBIND11_MODULE`
+  global constructor.
+
+* Introduce headers for C++-only wrapper classes as other related C++ modules
+  will need to interop with it.
+
+* Separate any initialization routines that depend on optional components into
+  its own module/dependency (currently, things like `registerAllDialects` fall
+  into this category).
+
+There are a lot of co-related issues of shared library linkage, distribution
+concerns, etc that affect such things. Organizing the code into composable
+modules (versus a monolithic `cpp` file) allows the flexibility to address many
+of these as needed over time. Also, compilation time for all of the template
+meta-programming in pybind scales with the number of things you define in a
+translation unit. Breaking into multiple translation units can significantly aid
+compile times for APIs with a large surface area.
+
+### Submodules
+
+Generally, the C++ codebase namespaces most things into the `mlir` namespace.
+However, in order to modularize and make the Python bindings easier to
+understand, sub-packages are defined that map roughly to the directory structure
+of functional units in MLIR.
+
+Examples:
+
+* `mlir.ir`
+* `mlir.passes` (`pass` is a reserved word :( )
+* `mlir.dialect`
+* `mlir.execution_engine` (aside from namespacing, it is important that
+  "bulky"/optional parts like this are isolated)
+
+In addition, initialization functions that imply optional dependencies should
+be in underscored (notionally private) modules such as `_init` and linked
+separately. This allows downstream integrators to completely customize what is
+included "in the box" and covers things like dialect registration,
+pass registration, etc.
+
+### Loader
+
+LLVM/MLIR is a non-trivial python-native project that is likely to co-exist with
+other non-trivial native extensions. As such, the native extension (i.e. the
+`.so`/`.pyd`/`.dylib`) is exported as a notionally private top-level symbol
+(`_mlir`), while a small set of Python code is provided in `mlir/__init__.py`
+and siblings which loads and re-exports it. This split provides a place to stage
+code that needs to prepare the environment *before* the shared library is loaded
+into the Python runtime, and also provides a place that one-time initialization
+code can be invoked apart from module constructors.
+
+To start with the `mlir/__init__.py` loader shim can be very simple and scale to
+future need:
+
+```python
+from _mlir import *
+```
+
+### Limited use of globals
+
+For normal operations, parent-child constructor relationships are realized with
+constructor methods on a parent class as opposed to requiring
+invocation/creation from a global symbol.
+
+For example, consider two code fragments:
+
+```python
+
+op = build_my_op()
+
+region = mlir.Region(op)
+
+```
+
+vs
+
+```python
+
+op = build_my_op()
+
+region = op.new_region()
+
+```
+
+For tightly coupled data structures like `Operation`, the latter is generally
+preferred because:
+
+* It is syntactically less possible to create something that is going to access
+  illegal memory (less error handling in the bindings, less testing, etc).
+
+* It reduces the global-API surface area for creating related entities. This
+  makes it more likely that if constructing IR based on an Operation instance of
+  unknown providence, receiving code can just call methods on it to do what they
+  want versus needing to reach back into the global namespace and find the right
+  `Region` class.
+
+* It leaks fewer things that are in place for C++ convenience (i.e. default
+  constructors to invalid instances).
+
+### Use the C-API
+
+The Python APIs should seek to layer on top of the C-API to the degree possible.
+Especially for the core, dialect-independent parts, such a binding enables
+packaging decisions that would be difficult or impossible if spanning a C++ ABI
+boundary. In addition, factoring in this way side-steps some very difficult
+issues that arise when combining RTTI-based modules (which pybind derived things
+are) with non-RTTI polymorphic C++ code (the default compilation mode of LLVM).
+
+
+## Style
+
+In general, for the core parts of MLIR, the Python bindings should be largely
+isomorphic with the underlying C++ structures. However, concessions are made
+either for practicality or to give the resulting library an appropriately
+"Pythonic" flavor.
+
+### Properties vs get*() methods
+
+Generally favor converting trivial methods like `getContext()`, `getName()`,
+`isEntryBlock()`, etc to read-only Python properties (i.e. `context`). It is
+primarily a matter of calling `def_property_readonly` vs `def` in binding code,
+and makes things feel much nicer to the Python side.
+
+For example, prefer:
+
+```c++
+m.def_property_readonly("context", ...)
+```
+
+Over:
+
+```c++
+m.def("getContext", ...)
+```
+
+### __repr__ methods
+
+Things that have nice printed representations are really great :)  If there is a
+reasonable printed form, it can be a significant productivity boost to wire that
+to the `__repr__` method (and verify it with a [doctest](#sample-doctest)).
+
+### CamelCase vs snake_case
+
+Name functions/methods/properties in `snake_case` and classes in `CamelCase`. As
+a mechanical concession to Python style, this can go a long way to making the
+API feel like it fits in with its peers in the Python landscape.
+
+If in doubt, choose names that will flow properly with other
+[PEP 8 style names](https://pep8.org/#descriptive-naming-styles).
+
+### Prefer pseudo-containers
+
+Many core IR constructs provide methods directly on the instance to query count
+and begin/end iterators. Prefer hoisting these to dedicated pseudo containers.
+
+For example, a direct mapping of blocks within regions could be done this way:
+
+```python
+region = ...
+
+for block in region:
+
+  pass
+```
+
+However, this way is preferred:
+
+```python
+region = ...
+
+for block in region.blocks:
+
+  pass
+
+print(len(region.blocks))
+print(region.blocks[0])
+print(region.blocks[-1])
+```
+
+Instead of leaking STL-derived identifiers (`front`, `back`, etc), translate
+them to appropriate `__dunder__` methods and iterator wrappers in the bindings.
+
+Note that this can be taken too far, so use good judgment. For example, block
+arguments may appear container-like but have defined methods for lookup and
+mutation that would be hard to model properly without making semantics
+complicated. If running into these, just mirror the C/C++ API.
+
+### Provide one stop helpers for common things
+
+One stop helpers that aggregate over multiple low level entities can be
+incredibly helpful and are encouraged within reason. For example, making
+`Context` have a `parse_asm` or equivalent that avoids needing to explicitly
+construct a SourceMgr can be quite nice. One stop helpers do not have to be
+mutually exclusive with a more complete mapping of the backing constructs.
+
+## Testing
+
+Tests should be added in the `test/Bindings/Python` directory and should
+typically be `.py` files that have a lit run line.
+
+While lit can run any python module, prefer to lay tests out according to these
+rules:
+
+* For tests of the API surface area, prefer
+  [`doctest`](https://docs.python.org/3/library/doctest.html).
+* For generative tests (those that produce IR), define a Python module that
+  constructs/prints the IR and pipe it through `FileCheck`.
+* Parsing should be kept self-contained within the module under test by use of
+  raw constants and an appropriate `parse_asm` call.
+* Any file I/O code should be staged through a tempfile vs relying on file
+  artifacts/paths outside of the test module.
+
+### Sample Doctest
+
+```python
+# RUN: %PYTHON %s
+
+"""
+  >>> m = load_test_module()
+Test basics:
+  >>> m.operation.name
+  "module"
+  >>> m.operation.is_registered
+  True
+  >>> ... etc ...
+
+Verify that repr prints:
+  >>> m.operation
+  <operation 'module'>
+"""
+
+import mlir
+
+TEST_MLIR_ASM = r"""
+func @test_operation_correct_regions() {
+  // ...
+}
+"""
+
+# TODO: Move to a test utility class once any of this actually exists.
+def load_test_module():
+  ctx = mlir.ir.Context()
+  ctx.allow_unregistered_dialects = True
+  module = ctx.parse_asm(TEST_MLIR_ASM)
+  return module
+
+
+if __name__ == "__main__":
+  import doctest
+  doctest.testmod()
+```
+
+### Sample FileCheck test
+
+```python
+# RUN: %PYTHON %s | mlir-opt -split-input-file | FileCheck
+
+# TODO: Move to a test utility class once any of this actually exists.
+def print_module(f):
+  m = f()
+  print("// -----")
+  print("// TEST_FUNCTION:", f.__name__)
+  print(m.to_asm())
+  return f
+
+# CHECK-LABEL: TEST_FUNCTION: create_my_op
+@print_module
+def create_my_op():
+  m = mlir.ir.Module()
+  builder = m.new_op_builder()
+  # CHECK: mydialect.my_operation ...
+  builder.my_op()
+  return m
+```
diff --git a/mlir/docs/ConversionToLLVMDialect.md b/mlir/docs/ConversionToLLVMDialect.md
index 15d09cb4b87a6..27b732015f9fd 100644
--- a/mlir/docs/ConversionToLLVMDialect.md
+++ b/mlir/docs/ConversionToLLVMDialect.md
@@ -246,7 +246,7 @@ func @bar() {
 }
 ```
 
-### Calling Convention for `memref`
+### Calling Convention for Ranked `memref`
 
 Function _arguments_ of `memref` type, ranked or unranked, are _expanded_ into a
 list of arguments of non-aggregate types that the memref descriptor defined
@@ -317,7 +317,9 @@ llvm.func @bar() {
 
 ```
 
-For **unranked** memrefs, the list of function arguments always contains two
+### Calling Convention for Unranked `memref`
+
+For unranked memrefs, the list of function arguments always contains two
 elements, same as the unranked memref descriptor: an integer rank, and a
 type-erased (`!llvm<"i8*">`) pointer to the ranked memref descriptor. Note that
 while the _calling convention_ does not require stack allocation, _casting_ to
@@ -369,6 +371,20 @@ llvm.func @bar() {
 }
 ```
 
+**Lifetime.** The second element of the unranked memref descriptor points to
+some memory in which the ranked memref descriptor is stored. By convention, this
+memory is allocated on stack and has the lifetime of the function. (*Note:* due
+to function-length lifetime, creation of multiple unranked memref descriptors,
+e.g., in a loop, may lead to stack overflows.) If an unranked descriptor has to
+be returned from a function, the ranked descriptor it points to is copied into
+dynamically allocated memory, and the pointer in the unranked descriptor is
+updated accordingly. The allocation happens immediately before returning. It is
+the responsibility of the caller to free the dynamically allocated memory. The
+default conversion of `std.call` and `std.call_indirect` copies the ranked
+descriptor to newly allocated memory on the caller's stack. Thus, the convention
+of the ranked memref descriptor pointed to by an unranked memref descriptor
+being stored on stack is respected.
+
 *This convention may or may not apply if the conversion of MemRef types is
 overridden by the user.*
 
diff --git a/mlir/docs/Dialects/LLVM.md b/mlir/docs/Dialects/LLVM.md
index 0a00a0c7d3169..e70944d997dff 100644
--- a/mlir/docs/Dialects/LLVM.md
+++ b/mlir/docs/Dialects/LLVM.md
@@ -313,10 +313,28 @@ Bitwise reinterpretation: `bitcast <value>`.
 
 Selection: `select <condition>, <lhs>, <rhs>`.
 
-### Auxiliary MLIR operations
-
-These operations do not have LLVM IR counterparts but are necessary to map LLVM
-IR into MLIR. They should be prefixed with `llvm.mlir`.
+### Auxiliary MLIR Operations for Constants and Globals
+
+LLVM IR has broad support for first-class constants, which is not the case for
+MLIR. Instead, constants are defined in MLIR as regular SSA values produced by
+operations with specific traits. The LLVM dialect provides a set of operations
+that model LLVM IR constants. These operations do not correspond to LLVM IR
+instructions and are therefore prefixed with `llvm.mlir`.
+
+Inline constants can be created by `llvm.mlir.constant`, which currently
+supports integer, float, string or elements attributes (constant sturcts are not
+currently supported). LLVM IR constant expressions are expected to be
+constructed as sequences of regular operations on SSA values produced by
+`llvm.mlir.constant`. Additionally, MLIR provides semantically-charged
+operations `llvm.mlir.undef` and `llvm.mlir.null` for the corresponding
+constants.
+
+LLVM IR globals can be defined using `llvm.mlir.global` at the module level,
+except for functions that are defined with `llvm.func`. Globals, both variables
+and functions, can be accessed by taking their address with the
+`llvm.mlir.addressof` operation, which produces a pointer to the named global,
+unlike the `llvm.mlir.constant` that produces the value of the same type as the
+constant.
 
 #### `llvm.mlir.addressof`
 
@@ -328,11 +346,17 @@ Examples:
 
 ```mlir
 func @foo() {
-  // Get the address of a global.
+  // Get the address of a global variable.
   %0 = llvm.mlir.addressof @const : !llvm<"i32*">
 
   // Use it as a regular pointer.
   %1 = llvm.load %0 : !llvm<"i32*">
+
+  // Get the address of a function.
+  %2 = llvm.mlir.addressof @foo : !llvm<"void ()*">
+
+  // The function address can be used for indirect calls.
+  llvm.call %2() : () -> ()
 }
 
 // Define the global.
diff --git a/mlir/docs/Interfaces.md b/mlir/docs/Interfaces.md
index d1050e76de5bc..e54a67ad0d6a3 100644
--- a/mlir/docs/Interfaces.md
+++ b/mlir/docs/Interfaces.md
@@ -21,16 +21,16 @@ transformations/analyses.
 
 ### Dialect Interfaces
 
-Dialect interfaces are generally useful for transformation passes or
-analyses that want to operate generically on a set of operations,
-which might even be defined in different dialects. These
-interfaces generally involve wide coverage over the entire dialect and are only
-used for a handful of transformations/analyses. In these cases, registering the
-interface directly on each operation is overly complex and cumbersome. The
-interface is not core to the operation, just to the specific transformation. An
-example of where this type of interface would be used is inlining. Inlining
-generally queries high-level information about the operations within a dialect,
-like legality and cost modeling, that often is not specific to one operation.
+Dialect interfaces are generally useful for transformation passes or analyses
+that want to operate generically on a set of attributes/operations/types, which
+might even be defined in different dialects. These interfaces generally involve
+wide coverage over the entire dialect and are only used for a handful of
+transformations/analyses. In these cases, registering the interface directly on
+each operation is overly complex and cumbersome. The interface is not core to
+the operation, just to the specific transformation. An example of where this
+type of interface would be used is inlining. Inlining generally queries
+high-level information about the operations within a dialect, like legality and
+cost modeling, that often is not specific to one operation.
 
 A dialect interface can be defined by inheriting from the CRTP base class
 `DialectInterfaceBase::Base`. This class provides the necessary utilities for
@@ -106,24 +106,25 @@ if(!interface.isLegalToInline(...))
    ...
 ```
 
-### Operation Interfaces
-
-Operation interfaces, as the name suggests, are those registered at the
-Operation level. These interfaces provide access to derived operations
-by providing a virtual interface that must be implemented. As an example, the
-`Linalg` dialect may implement an interface that provides general queries about
-some of the dialects library operations. These queries may provide things like:
-the number of parallel loops; the number of inputs and outputs; etc.
-
-Operation interfaces are defined by overriding the CRTP base class
-`OpInterface`. This class takes, as a template parameter, a `Traits` class that
-defines a `Concept` and a `Model` class. These classes provide an implementation
-of concept-based polymorphism, where the Concept defines a set of virtual
-methods that are overridden by the Model that is templated on the concrete
-operation type. It is important to note that these classes should be pure in
-that they contain no non-static data members. Operations that wish to override
-this interface should add the provided trait `OpInterface<..>::Trait` upon
-registration.
+### Attribute/Operation/Type Interfaces
+
+Attribute/Operation/Type interfaces, as the names suggest, are those registered
+at the level of a specific attribute/operation/type. These interfaces provide
+access to derived objects by providing a virtual interface that must be
+implemented. As an example, the `Linalg` dialect may implement an interface that
+provides general queries about some of the dialects library operations. These
+queries may provide things like: the number of parallel loops; the number of
+inputs and outputs; etc.
+
+These interfaces are defined by overriding the CRTP base class `AttrInterface`,
+`OpInterface`, or `TypeInterface` respectively. These classes take, as a
+template parameter, a `Traits` class that defines a `Concept` and a `Model`
+class. These classes provide an implementation of concept-based polymorphism,
+where the Concept defines a set of virtual methods that are overridden by the
+Model that is templated on the concrete object type. It is important to note
+that these classes should be pure in that they contain no non-static data
+members. Objects that wish to override this interface should add the provided
+trait `*Interface<..>::Trait` to the trait list upon registration.
 
 ```c++
 struct ExampleOpInterfaceTraits {
@@ -131,14 +132,14 @@ struct ExampleOpInterfaceTraits {
   /// to be overridden.
   struct Concept {
     virtual ~Concept();
-    virtual unsigned getNumInputs(Operation *op) = 0;
+    virtual unsigned getNumInputs(Operation *op) const = 0;
   };
 
   /// Define a model class that specializes a concept on a given operation type.
   template <typename OpT>
   struct Model : public Concept {
     /// Override the method to dispatch on the concrete operation.
-    unsigned getNumInputs(Operation *op) final {
+    unsigned getNumInputs(Operation *op) const final {
       return llvm::cast<OpT>(op).getNumInputs();
     }
   };
@@ -151,7 +152,7 @@ public:
   using OpInterface<ExampleOpInterface, ExampleOpInterfaceTraits>::OpInterface;
 
   /// The interface dispatches to 'getImpl()', an instance of the concept.
-  unsigned getNumInputs() {
+  unsigned getNumInputs() const {
     return getImpl()->getNumInputs(getOperation());
   }
 };
@@ -182,8 +183,7 @@ if (ExampleOpInterface example = dyn_cast<ExampleOpInterface>(op))
 
 Operation interfaces require a bit of boiler plate to connect all of the pieces
 together. The ODS(Operation Definition Specification) framework provides
-simplified mechanisms for
-[defining interfaces](OpDefinitions.md#operation-interfaces).
+simplified mechanisms for [defining interfaces](OpDefinitions.md#interfaces).
 
 As an example, using the ODS framework would allow for defining the example
 interface above as:
diff --git a/mlir/docs/LangRef.md b/mlir/docs/LangRef.md
index ff3af3e4e496f..41b3984347bbd 100644
--- a/mlir/docs/LangRef.md
+++ b/mlir/docs/LangRef.md
@@ -148,7 +148,7 @@ integer-literal ::= decimal-literal | hexadecimal-literal
 decimal-literal ::= digit+
 hexadecimal-literal ::= `0x` hex_digit+
 float-literal ::= [-+]?[0-9]+[.][0-9]*([eE][-+]?[0-9]+)?
-string-literal  ::= `"` [^"\n\f\v\r]* `"`   TODO define escaping rules
+string-literal  ::= `"` [^"\n\f\v\r]* `"`   TODO: define escaping rules
 ```
 
 Not listed here, but MLIR does support comments. They use standard BCPL syntax,
diff --git a/mlir/docs/OpDefinitions.md b/mlir/docs/OpDefinitions.md
index ed164c7a8dd8e..a10610f87a0af 100644
--- a/mlir/docs/OpDefinitions.md
+++ b/mlir/docs/OpDefinitions.md
@@ -346,20 +346,20 @@ involving multiple operands/attributes/results are provided as the second
 template parameter to the `Op` class. They should be deriving from the `OpTrait`
 class. See [Constraints](#constraints) for more information.
 
-### Operation interfaces
+### Interfaces
 
-[Operation interfaces](Interfaces.md#operation-interfaces) allow
-operations to expose method calls without the
-caller needing to know the exact operation type. Operation interfaces
-defined in C++ can be accessed in the ODS framework via the
-`OpInterfaceTrait` class. Aside from using pre-existing interfaces in
-the C++ API, the ODS framework also provides a simplified mechanism
-for defining such interfaces which removes much of the boilerplate
-necessary.
+[Interfaces](Interfaces.md#attribute-operation-type-interfaces) allow for
+attributes, operations, and types to expose method calls without the caller
+needing to know the derived type. Operation interfaces defined in C++ can be
+accessed in the ODS framework via the `OpInterfaceTrait` class. Aside from using
+pre-existing interfaces in the C++ API, the ODS framework also provides a
+simplified mechanism for defining such interfaces which removes much of the
+boilerplate necessary.
 
-Providing a definition of the `OpInterface` class will auto-generate the C++
-classes for the interface. An `OpInterface` includes a name, for the C++ class,
-a description, and a list of interface methods.
+Providing a definition of the `AttrInterface`, `OpInterface`, or `TypeInterface`
+class will auto-generate the C++ classes for the interface. An interface
+includes a name, for the C++ class, a description, and a list of interface
+methods.
 
 ```tablegen
 def MyInterface : OpInterface<"MyInterface"> {
@@ -444,16 +444,17 @@ def MyInterface : OpInterface<"MyInterface"> {
     // Note: `ConcreteOp` corresponds to the derived operation typename.
     InterfaceMethod<"/*insert doc here*/",
       "unsigned", "getNumWithDefault", (ins), /*methodBody=*/[{}], [{
-        ConcreteOp op = cast<ConcreteOp>(getOperation());
+        ConcreteOp op = cast<ConcreteOp>(this->getOperation());
         return op.getNumInputs() + op.getNumOutputs();
     }]>,
   ];
 }
 
-// Interfaces can optionally be wrapped inside DeclareOpInterfaceMethods. This
-// would result in autogenerating declarations for members `foo`, `bar` and
-// `fooStatic`. Methods with bodies are not declared inside the op
-// declaration but instead handled by the op interface trait directly.
+// Operation interfaces can optionally be wrapped inside
+// DeclareOpInterfaceMethods. This would result in autogenerating declarations
+// for members `foo`, `bar` and `fooStatic`. Methods with bodies are not
+// declared inside the op declaration but instead handled by the op interface
+// trait directly.
 def OpWithInferTypeInterfaceOp : Op<...
     [DeclareOpInterfaceMethods<MyInterface>]> { ... }
 
@@ -465,9 +466,9 @@ def OpWithOverrideInferTypeInterfaceOp : Op<...
     [DeclareOpInterfaceMethods<MyInterface, ["getNumWithDefault"]>]> { ... }
 ```
 
-A verification method can also be specified on the `OpInterface` by setting
-`verify`. Setting `verify` results in the generated trait having a `verifyTrait`
-method that is applied to all operations implementing the trait.
+Operation interfaces may also provide a verification method on `OpInterface` by
+setting `verify`. Setting `verify` results in the generated trait having a
+`verifyTrait` method that is applied to all operations implementing the trait.
 
 ### Builder methods
 
@@ -779,8 +780,8 @@ There are many operations that have known type equality constraints registered
 as traits on the operation; for example the true, false, and result values of a
 `select` operation often have the same type. The assembly format may inspect
 these equal constraints to discern the types of missing variables. The currently
-supported traits are: `AllTypesMatch`, `SameTypeOperands`, and
-`SameOperandsAndResultType`.
+supported traits are: `AllTypesMatch`, `TypesMatchWith`, `SameTypeOperands`,
+and `SameOperandsAndResultType`.
 
 ### `hasCanonicalizer`
 
@@ -963,9 +964,9 @@ is used. They serve as "hooks" to the enclosing environment.  This includes
   replaced by the operand/result's type. E.g., for `F32` in `F32:$operand`, its
   `$_self` will be expanded as `getOperand(...).getType()`.
 
-TODO(b/130663252): Reconsider the leading symbol for special placeholders.
-Eventually we want to allow referencing operand/result $-names; such $-names
-can start with underscore.
+TODO: Reconsider the leading symbol for special placeholders. Eventually we want
+to allow referencing operand/result $-names; such $-names can start with
+underscore.
 
 For example, to write an attribute `attr` is an `IntegerAttr`, in C++ you can
 just call `attr.isa<IntegerAttr>()`. The code can be wrapped in a `CPred` as
diff --git a/mlir/docs/Quantization.md b/mlir/docs/Quantization.md
index 54eae406c87ea..4abc52493cabd 100644
--- a/mlir/docs/Quantization.md
+++ b/mlir/docs/Quantization.md
@@ -196,7 +196,7 @@ operations for type conversion and expression of the supporting math.
 
 ### Quantized type
 
-TODO : Flesh this section out.
+TODO: Flesh this section out.
 
 *   QuantizedType base class
 *   UniformQuantizedType
@@ -247,7 +247,7 @@ in floating point with appropriate conversions at the boundaries.
 
 ## TFLite native quantization
 
-TODO : Flesh this out
+TODO: Flesh this out
 
 ### General algorithm
 
diff --git a/mlir/docs/Rationale/Rationale.md b/mlir/docs/Rationale/Rationale.md
index a3c3e5ecc4bfc..22e21383e9036 100644
--- a/mlir/docs/Rationale/Rationale.md
+++ b/mlir/docs/Rationale/Rationale.md
@@ -765,7 +765,7 @@ func @conv2d(%input: memref<16x1024x1024x3xf32, #lm0, /*scratchpad=*/1>,
 }
 ```
 
-TODO (Add more examples showing the IR for a variety of interesting cases)
+TODO: (Add more examples showing the IR for a variety of interesting cases)
 
 ## Design alternatives and extensions
 
diff --git a/mlir/docs/Traits.md b/mlir/docs/Traits.md
index 6bf03086be3b1..c9ef132717b85 100644
--- a/mlir/docs/Traits.md
+++ b/mlir/docs/Traits.md
@@ -1,24 +1,28 @@
-# Operation Traits
+# Traits
 
 [TOC]
 
-MLIR allows for a truly open operation ecosystem, as any dialect may define
-operations that suit a specific level of abstraction. `Traits` are a mechanism
-which abstracts implementation details and properties that are common
-across many different operations. `Traits` may be used to specify special
-properties and constraints of the operation, including whether the operation has
-side effects or whether its output has the same type as the input. Some examples
-of traits are `Commutative`, `SingleResult`, `Terminator`, etc. See the more
-[comprehensive list](#trait-list) below for more examples of what is possible.
+MLIR allows for a truly open ecosystem, as any dialect may define attributes,
+operations, and types that suit a specific level of abstraction. `Traits` are a
+mechanism which abstracts implementation details and properties that are common
+across many different attributes/operations/types/etc.. `Traits` may be used to
+specify special properties and constraints of the object, including whether an
+operation has side effects or that its output has the same type as the input.
+Some examples of operation traits are `Commutative`, `SingleResult`,
+`Terminator`, etc. See the more comprehensive list of
+[operation traits](#operation-traits-list) below for more examples of what is
+possible.
 
 ## Defining a Trait
 
-Traits may be defined in C++ by inheriting from the
-`OpTrait::TraitBase<ConcreteType, TraitType>` class. This base class takes as
-template parameters:
+Traits may be defined in C++ by inheriting from the `TraitBase<ConcreteType,
+TraitType>` class for the specific IR type. For attributes, this is
+`AttributeTrait::TraitBase`. For operations, this is `OpTrait::TraitBase`. For
+types, this is `TypeTrait::TraitBase`. This base class takes as template
+parameters:
 
 *   ConcreteType
-    -   The concrete operation type that this trait was attached to.
+    -   The concrete class type that this trait was attached to.
 *   TraitType
     -   The type of the trait class that is being defined, for use with the
         [`Curiously Recurring Template Pattern`](https://en.wikipedia.org/wiki/Curiously_recurring_template_pattern).
@@ -28,11 +32,11 @@ the `ConcreteType`. An example trait definition is shown below:
 
 ```c++
 template <typename ConcreteType>
-class MyTrait : public OpTrait::TraitBase<ConcreteType, MyTrait> {
+class MyTrait : public TraitBase<ConcreteType, MyTrait> {
 };
 ```
 
-Derived traits may also provide a `verifyTrait` hook, that is called when
+Operation traits may also provide a `verifyTrait` hook, that is called when
 verifying the concrete operation. The trait verifiers will currently always be
 invoked before the main `Op::verify`.
 
@@ -57,15 +61,15 @@ instantiating the implementation for every concrete operation type.
 The above demonstrates the definition of a simple self-contained trait. It is
 also often useful to provide some static parameters to the trait to control its
 behavior. Given that the definition of the trait class is rigid, i.e. we must
-have a single template argument for the concrete operation, the templates for
-the parameters will need to be split out. An example is shown below:
+have a single template argument for the concrete object, the templates for the
+parameters will need to be split out. An example is shown below:
 
 ```c++
 template <int Parameter>
 class MyParametricTrait {
 public:
   template <typename ConcreteType>
-  class Impl : public OpTrait::TraitBase<ConcreteType, Impl> {
+  class Impl : public TraitBase<ConcreteType, Impl> {
     // Inside of 'Impl' we have full access to the template parameters
     // specified above.
   };
@@ -74,19 +78,28 @@ public:
 
 ## Attaching a Trait
 
-Traits may be used when defining a derived operation type, by simply adding the
-name of the trait class to the `Op` class after the concrete operation type:
+Traits may be used when defining a derived object type, by simply appending the
+name of the trait class to the end of the base object class operation type:
 
 ```c++
+/// Here we define 'MyAttr' along with the 'MyTrait' and `MyParametric trait
+/// classes we defined previously.
+class MyAttr : public Attribute::AttrBase<MyAttr, ..., MyTrait, MyParametricTrait<10>::Impl> {};
 /// Here we define 'MyOp' along with the 'MyTrait' and `MyParametric trait
 /// classes we defined previously.
 class MyOp : public Op<MyOp, MyTrait, MyParametricTrait<10>::Impl> {};
+/// Here we define 'MyType' along with the 'MyTrait' and `MyParametric trait
+/// classes we defined previously.
+class MyType : public Type::TypeBase<MyType, ..., MyTrait, MyParametricTrait<10>::Impl> {};
 ```
 
-To use a trait in the [ODS](OpDefinitions.md) framework, we need to provide a
-definition of the trait class. This can be done using the `NativeOpTrait` and
-`ParamNativeOpTrait` classes. `ParamNativeOpTrait` provides a mechanism in which
-to specify arguments to a parametric trait class with an internal `Impl`.
+### Attaching Operation Traits in ODS
+
+To use an operation trait in the [ODS](OpDefinitions.md) framework, we need to
+provide a definition of the trait class. This can be done using the
+`NativeOpTrait` and `ParamNativeOpTrait` classes. `ParamNativeOpTrait` provides
+a mechanism in which to specify arguments to a parametric trait class with an
+internal `Impl`.
 
 ```tablegen
 // The argument is the c++ trait class name.
@@ -110,14 +123,14 @@ details.
 ## Using a Trait
 
 Traits may be used to provide additional methods, static fields, or other
-information directly on the concrete operation. `Traits` internally become
-`Base` classes of the concrete operation, so all of these are directly
-accessible. To expose this information opaquely to transformations and analyses,
+information directly on the concrete object. `Traits` internally become `Base`
+classes of the concrete operation, so all of these are directly accessible. To
+expose this information opaquely to transformations and analyses,
 [`interfaces`](Interfaces.md) may be used.
 
-To query if a specific operation contains a specific trait, the `hasTrait<>`
-method may be used. This takes as a template parameter the trait class, which is
-the same as the one passed when attaching the trait to an operation.
+To query if a specific object contains a specific trait, the `hasTrait<>` method
+may be used. This takes as a template parameter the trait class, which is the
+same as the one passed when attaching the trait to an operation.
 
 ```c++
 Operation *op = ..;
@@ -125,7 +138,7 @@ if (op->hasTrait<MyTrait>() || op->hasTrait<MyParametricTrait<10>::Impl>())
   ...;
 ```
 
-## Trait List
+## Operation Traits List
 
 MLIR provides a suite of traits that provide various functionalities that are
 common across many different operations. Below is a list of some key traits that
diff --git a/mlir/docs/Tutorials/DefiningAttributesAndTypes.md b/mlir/docs/Tutorials/DefiningAttributesAndTypes.md
index b3597f48e659a..cab2441b33200 100644
--- a/mlir/docs/Tutorials/DefiningAttributesAndTypes.md
+++ b/mlir/docs/Tutorials/DefiningAttributesAndTypes.md
@@ -75,10 +75,11 @@ to provide our own storage class.
 ```c++
 /// This class defines a simple parameterless type. All derived types must
 /// inherit from the CRTP class 'Type::TypeBase'. It takes as template
-/// parameters the concrete type (SimpleType), and the base class to use (Type).
+/// parameters the concrete type (SimpleType), the base class to use (Type),
+/// using the default storage class (TypeStorage) as the storage type.
 /// 'Type::TypeBase' also provides several utility methods to simplify type
 /// construction.
-class SimpleType : public Type::TypeBase<SimpleType, Type> {
+class SimpleType : public Type::TypeBase<SimpleType, Type, TypeStorage> {
 public:
   /// Inherit some necessary constructors from 'TypeBase'.
   using Base::Base;
diff --git a/mlir/docs/Tutorials/Toy/Ch-7.md b/mlir/docs/Tutorials/Toy/Ch-7.md
index 0dce6d25c904e..733e22c5b0a55 100644
--- a/mlir/docs/Tutorials/Toy/Ch-7.md
+++ b/mlir/docs/Tutorials/Toy/Ch-7.md
@@ -287,8 +287,7 @@ mlir::Type ToyDialect::parseType(mlir::DialectAsmParser &parser) const {
       return nullptr;
 
     // Check that the type is either a TensorType or another StructType.
-    if (!elementType.isa<mlir::TensorType>() &&
-        !elementType.isa<StructType>()) {
+    if (!elementType.isa<mlir::TensorType, StructType>()) {
       parser.emitError(typeLoc, "element type for a struct must either "
                                 "be a TensorType or a StructType, got: ")
           << elementType;
diff --git a/mlir/examples/toy/Ch7/mlir/Dialect.cpp b/mlir/examples/toy/Ch7/mlir/Dialect.cpp
index 867da9c202ccf..fc7bf2a2375cf 100644
--- a/mlir/examples/toy/Ch7/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch7/mlir/Dialect.cpp
@@ -510,8 +510,7 @@ mlir::Type ToyDialect::parseType(mlir::DialectAsmParser &parser) const {
       return nullptr;
 
     // Check that the type is either a TensorType or another StructType.
-    if (!elementType.isa<mlir::TensorType>() &&
-        !elementType.isa<StructType>()) {
+    if (!elementType.isa<mlir::TensorType, StructType>()) {
       parser.emitError(typeLoc, "element type for a struct must either "
                                 "be a TensorType or a StructType, got: ")
           << elementType;
diff --git a/mlir/include/mlir-c/Core.h b/mlir/include/mlir-c/Core.h
index 4698a7feeb458..4c0666f56c280 100644
--- a/mlir/include/mlir-c/Core.h
+++ b/mlir/include/mlir-c/Core.h
@@ -96,7 +96,7 @@ unsigned getFunctionArity(mlir_func_t function);
 /// Returns the rank of the `function` argument at position `pos`.
 /// If the argument is of MemRefType, this returns the rank of the MemRef.
 /// Otherwise returns `0`.
-/// TODO(ntv): support more than MemRefType and scalar Type.
+/// TODO: support more than MemRefType and scalar Type.
 unsigned getRankOfFunctionArgument(mlir_func_t function, unsigned pos);
 
 /// Returns an opaque mlir::Type of the `function` argument at position `pos`.
diff --git a/mlir/include/mlir/Analysis/AffineAnalysis.h b/mlir/include/mlir/Analysis/AffineAnalysis.h
index b421eee951485..3322f3da6a092 100644
--- a/mlir/include/mlir/Analysis/AffineAnalysis.h
+++ b/mlir/include/mlir/Analysis/AffineAnalysis.h
@@ -38,7 +38,7 @@ void getReachableAffineApplyOps(ArrayRef<Value> operands,
 /// used to add appropriate inequalities. Any symbols founds in the bound
 /// operands are added as symbols in the system. Returns failure for the yet
 /// unimplemented cases.
-//  TODO(bondhugula): handle non-unit strides.
+//  TODO: handle non-unit strides.
 LogicalResult getIndexSet(MutableArrayRef<AffineForOp> forOps,
                           FlatAffineConstraints *domain);
 
@@ -49,8 +49,8 @@ struct MemRefAccess {
   SmallVector<Value, 4> indices;
 
   /// Constructs a MemRefAccess from a load or store operation.
-  // TODO(b/119949820): add accessors to standard op's load, store, DMA op's to
-  // return MemRefAccess, i.e., loadOp->getAccess(), dmaOp->getRead/WriteAccess.
+  // TODO: add accessors to standard op's load, store, DMA op's to return
+  // MemRefAccess, i.e., loadOp->getAccess(), dmaOp->getRead/WriteAccess.
   explicit MemRefAccess(Operation *opInst);
 
   // Returns the rank of the memref associated with this access.
@@ -95,9 +95,9 @@ struct DependenceComponent {
 /// access the same memref element. If 'allowRAR' is true, will consider
 /// read-after-read dependences (typically used by applications trying to
 /// optimize input reuse).
-// TODO(andydavis) Wrap 'dependenceConstraints' and 'dependenceComponents' into
-// a single struct.
-// TODO(andydavis) Make 'dependenceConstraints' optional arg.
+// TODO: Wrap 'dependenceConstraints' and 'dependenceComponents' into a single
+// struct.
+// TODO: Make 'dependenceConstraints' optional arg.
 struct DependenceResult {
   enum ResultEnum {
     HasDependence, // A dependence exists between 'srcAccess' and 'dstAccess'.
diff --git a/mlir/include/mlir/Analysis/AffineStructures.h b/mlir/include/mlir/Analysis/AffineStructures.h
index b8a9973fcffa9..0424e0bb7d332 100644
--- a/mlir/include/mlir/Analysis/AffineStructures.h
+++ b/mlir/include/mlir/Analysis/AffineStructures.h
@@ -98,7 +98,6 @@ class FlatAffineConstraints {
 
   /// Create a flat affine constraint system from an AffineValueMap or a list of
   /// these. The constructed system will only include equalities.
-  // TODO(bondhugula)
   explicit FlatAffineConstraints(const AffineValueMap &avm);
   explicit FlatAffineConstraints(ArrayRef<const AffineValueMap *> avmRef);
 
@@ -126,19 +125,33 @@ class FlatAffineConstraints {
   /// intersection with no simplification of any sort attempted.
   void append(const FlatAffineConstraints &other);
 
-  // Checks for emptiness by performing variable elimination on all identifiers,
-  // running the GCD test on each equality constraint, and checking for invalid
-  // constraints.
-  // Returns true if the GCD test fails for any equality, or if any invalid
-  // constraints are discovered on any row. Returns false otherwise.
+  /// Checks for emptiness by performing variable elimination on all
+  /// identifiers, running the GCD test on each equality constraint, and
+  /// checking for invalid constraints. Returns true if the GCD test fails for
+  /// any equality, or if any invalid constraints are discovered on any row.
+  /// Returns false otherwise.
   bool isEmpty() const;
 
-  // Runs the GCD test on all equality constraints. Returns 'true' if this test
-  // fails on any equality. Returns 'false' otherwise.
-  // This test can be used to disprove the existence of a solution. If it
-  // returns true, no integer solution to the equality constraints can exist.
+  /// Runs the GCD test on all equality constraints. Returns 'true' if this test
+  /// fails on any equality. Returns 'false' otherwise.
+  /// This test can be used to disprove the existence of a solution. If it
+  /// returns true, no integer solution to the equality constraints can exist.
   bool isEmptyByGCDTest() const;
 
+  /// Runs the GCD test heuristic. If it proves inconclusive, falls back to
+  /// generalized basis reduction if the set is bounded.
+  ///
+  /// Returns true if the set of constraints is found to have no solution,
+  /// false if a solution exists or all tests were inconclusive.
+  bool isIntegerEmpty() const;
+
+  /// Find a sample point satisfying the constraints. This uses a branch and
+  /// bound algorithm with generalized basis reduction, which always works if
+  /// the set is bounded. This should not be called for unbounded sets.
+  ///
+  /// Returns such a point if one exists, or an empty Optional otherwise.
+  Optional<SmallVector<int64_t, 8>> findIntegerSample() const;
+
   // Clones this object.
   std::unique_ptr<FlatAffineConstraints> clone() const;
 
@@ -199,7 +212,7 @@ class FlatAffineConstraints {
   /// 'affine.for' operation are added as trailing identifiers (either
   /// dimensional or symbolic depending on whether the operand is a valid
   /// symbol).
-  //  TODO(bondhugula): add support for non-unit strides.
+  //  TODO: add support for non-unit strides.
   LogicalResult addAffineForOpDomain(AffineForOp forOp);
 
   /// Adds a lower or an upper bound for the identifier at the specified
@@ -321,8 +334,8 @@ class FlatAffineConstraints {
   /// Projects out (aka eliminates) 'num' identifiers starting at position
   /// 'pos'. The resulting constraint system is the shadow along the dimensions
   /// that still exist. This method may not always be integer exact.
-  // TODO(bondhugula): deal with integer exactness when necessary - can return a
-  // value to mark exactness for example.
+  // TODO: deal with integer exactness when necessary - can return a value to
+  // mark exactness for example.
   void projectOut(unsigned pos, unsigned num);
   inline void projectOut(unsigned pos) { return projectOut(pos, 1); }
 
diff --git a/mlir/include/mlir/Analysis/LoopAnalysis.h b/mlir/include/mlir/Analysis/LoopAnalysis.h
index 7ed19ef99f871..b9c7d5e1b47ad 100644
--- a/mlir/include/mlir/Analysis/LoopAnalysis.h
+++ b/mlir/include/mlir/Analysis/LoopAnalysis.h
@@ -33,8 +33,8 @@ class Value;
 /// multi-result map. The trip count expression is simplified before returning.
 /// This method only utilizes map composition to construct lower and upper
 /// bounds before computing the trip count expressions
-// TODO(mlir-team): this should be moved into 'Transforms/' and be replaced by a
-// pure analysis method relying on FlatAffineConstraints
+// TODO: this should be moved into 'Transforms/' and be replaced by a pure
+// analysis method relying on FlatAffineConstraints
 void buildTripCountMapAndOperands(AffineForOp forOp, AffineMap *map,
                                   SmallVectorImpl<Value> *operands);
 
@@ -65,7 +65,7 @@ using VectorizableLoopFun = std::function<bool(AffineForOp)>;
 /// Checks whether the loop is structurally vectorizable; i.e.:
 ///   1. no conditionals are nested under the loop;
 ///   2. all nested load/stores are to scalar MemRefs.
-/// TODO(ntv): relax the no-conditionals restriction
+/// TODO: relax the no-conditionals restriction
 bool isVectorizableLoopBody(AffineForOp loop,
                             NestedPattern &vectorTransferMatcher);
 
@@ -80,8 +80,8 @@ bool isVectorizableLoopBody(AffineForOp loop, int *memRefDim,
 /// Checks where SSA dominance would be violated if a for op's body
 /// operations are shifted by the specified shifts. This method checks if a
 /// 'def' and all its uses have the same shift factor.
-// TODO(mlir-team): extend this to check for memory-based dependence
-// violation when we have the support.
+// TODO: extend this to check for memory-based dependence violation when we have
+// the support.
 bool isOpwiseShiftValid(AffineForOp forOp, ArrayRef<uint64_t> shifts);
 } // end namespace mlir
 
diff --git a/mlir/include/mlir/Analysis/NestedMatcher.h b/mlir/include/mlir/Analysis/NestedMatcher.h
index 374fcab1bab5a..594ea47e99e98 100644
--- a/mlir/include/mlir/Analysis/NestedMatcher.h
+++ b/mlir/include/mlir/Analysis/NestedMatcher.h
@@ -73,7 +73,7 @@ class NestedMatch {
 ///   1. recursively matches a substructure in the tree;
 ///   2. uses a filter function to refine matches with extra semantic
 ///      constraints (passed via a lambda of type FilterFunctionType);
-///   3. TODO(ntv) optionally applies actions (lambda).
+///   3. TODO: optionally applies actions (lambda).
 ///
 /// Nested patterns are meant to capture imperfectly nested loops while matching
 /// properties over the whole loop nest. For instance, in vectorization we are
diff --git a/mlir/include/mlir/Analysis/Presburger/Fraction.h b/mlir/include/mlir/Analysis/Presburger/Fraction.h
new file mode 100644
index 0000000000000..09996c486ef33
--- /dev/null
+++ b/mlir/include/mlir/Analysis/Presburger/Fraction.h
@@ -0,0 +1,77 @@
+//===- Fraction.h - MLIR Fraction Class -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a simple class to represent fractions. It supports multiplication,
+// comparison, floor, and ceiling operations.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_ANALYSIS_PRESBURGER_FRACTION_H
+#define MLIR_ANALYSIS_PRESBURGER_FRACTION_H
+
+#include "mlir/Support/MathExtras.h"
+
+namespace mlir {
+
+/// A class to represent fractions. The sign of the fraction is represented
+/// in the sign of the numerator; the denominator is always positive.
+///
+/// Note that overflows may occur if the numerator or denominator are not
+/// representable by 64-bit integers.
+struct Fraction {
+  /// Default constructor initializes the represented rational number to zero.
+  Fraction() : num(0), den(1) {}
+
+  /// Construct a Fraction from a numerator and denominator.
+  Fraction(int64_t oNum, int64_t oDen) : num(oNum), den(oDen) {
+    if (den < 0) {
+      num = -num;
+      den = -den;
+    }
+  }
+
+  /// The numerator and denominator, respectively. The denominator is always
+  /// positive.
+  int64_t num, den;
+};
+
+/// Three-way comparison between two fractions.
+/// Returns +1, 0, and -1 if the first fraction is greater than, equal to, or
+/// less than the second fraction, respectively.
+inline int compare(Fraction x, Fraction y) {
+  int64_t diff = x.num * y.den - y.num * x.den;
+  if (diff > 0)
+    return +1;
+  if (diff < 0)
+    return -1;
+  return 0;
+}
+
+inline int64_t floor(Fraction f) { return floorDiv(f.num, f.den); }
+
+inline int64_t ceil(Fraction f) { return ceilDiv(f.num, f.den); }
+
+inline Fraction operator-(Fraction x) { return Fraction(-x.num, x.den); }
+
+inline bool operator<(Fraction x, Fraction y) { return compare(x, y) < 0; }
+
+inline bool operator<=(Fraction x, Fraction y) { return compare(x, y) <= 0; }
+
+inline bool operator==(Fraction x, Fraction y) { return compare(x, y) == 0; }
+
+inline bool operator>(Fraction x, Fraction y) { return compare(x, y) > 0; }
+
+inline bool operator>=(Fraction x, Fraction y) { return compare(x, y) >= 0; }
+
+inline Fraction operator*(Fraction x, Fraction y) {
+  return Fraction(x.num * y.num, x.den * y.den);
+}
+
+} // namespace mlir
+
+#endif // MLIR_ANALYSIS_PRESBURGER_FRACTION_H
diff --git a/mlir/include/mlir/Analysis/Presburger/Matrix.h b/mlir/include/mlir/Analysis/Presburger/Matrix.h
new file mode 100644
index 0000000000000..7bc29f81a8346
--- /dev/null
+++ b/mlir/include/mlir/Analysis/Presburger/Matrix.h
@@ -0,0 +1,79 @@
+//===- Matrix.h - MLIR Matrix Class -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a simple 2D matrix class that supports reading, writing, resizing,
+// swapping rows, and swapping columns.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_ANALYSIS_PRESBURGER_MATRIX_H
+#define MLIR_ANALYSIS_PRESBURGER_MATRIX_H
+
+#include "mlir/Support/LLVM.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <cassert>
+
+namespace mlir {
+
+/// This is a simple class to represent a resizable matrix.
+///
+/// The data is stored in the form of a vector of vectors.
+class Matrix {
+public:
+  Matrix() = delete;
+
+  /// Construct a matrix with the specified number of rows and columns.
+  /// Initially, the values are default initialized.
+  Matrix(unsigned rows, unsigned columns);
+
+  /// Return the identity matrix of the specified dimension.
+  static Matrix identity(unsigned dimension);
+
+  /// Access the element at the specified row and column.
+  int64_t &at(unsigned row, unsigned column);
+  int64_t at(unsigned row, unsigned column) const;
+  int64_t &operator()(unsigned row, unsigned column);
+  int64_t operator()(unsigned row, unsigned column) const;
+
+  /// Swap the given columns.
+  void swapColumns(unsigned column, unsigned otherColumn);
+
+  /// Swap the given rows.
+  void swapRows(unsigned row, unsigned otherRow);
+
+  unsigned getNumRows() const;
+
+  unsigned getNumColumns() const;
+
+  /// Get an ArrayRef corresponding to the specified row.
+  ArrayRef<int64_t> getRow(unsigned row) const;
+
+  /// Add `scale` multiples of the source row to the target row.
+  void addToRow(unsigned sourceRow, unsigned targetRow, int64_t scale);
+
+  /// Resize the matrix to the specified dimensions. If a dimension is smaller,
+  /// the values are truncated; if it is bigger, the new values are default
+  /// initialized.
+  void resizeVertically(unsigned newNRows);
+
+  /// Print the matrix.
+  void print(raw_ostream &os) const;
+  void dump() const;
+
+private:
+  unsigned nRows, nColumns;
+
+  /// Stores the data. data.size() is equal to nRows * nColumns.
+  SmallVector<int64_t, 64> data;
+};
+
+} // namespace mlir
+
+#endif // MLIR_ANALYSIS_PRESBURGER_MATRIX_H
diff --git a/mlir/include/mlir/Analysis/Presburger/Simplex.h b/mlir/include/mlir/Analysis/Presburger/Simplex.h
new file mode 100644
index 0000000000000..f6344a62d214d
--- /dev/null
+++ b/mlir/include/mlir/Analysis/Presburger/Simplex.h
@@ -0,0 +1,327 @@
+//===- Simplex.h - MLIR Simplex Class ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Functionality to perform analysis on FlatAffineConstraints. In particular,
+// support for performing emptiness checks.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_ANALYSIS_PRESBURGER_SIMPLEX_H
+#define MLIR_ANALYSIS_PRESBURGER_SIMPLEX_H
+
+#include "mlir/Analysis/AffineStructures.h"
+#include "mlir/Analysis/Presburger/Fraction.h"
+#include "mlir/Analysis/Presburger/Matrix.h"
+#include "mlir/Support/LogicalResult.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace mlir {
+
+class GBRSimplex;
+
+/// This class implements a version of the Simplex and Generalized Basis
+/// Reduction algorithms, which can perform analysis of integer sets with affine
+/// inequalities and equalities. A Simplex can be constructed
+/// by specifying the dimensionality of the set. It supports adding affine
+/// inequalities and equalities, and can perform emptiness checks, i.e., it can
+/// find a solution to the set of constraints if one exists, or say that the
+/// set is empty if no solution exists. Currently, this only works for bounded
+/// sets. Simplex can also be constructed from a FlatAffineConstraints object.
+///
+/// The implementation of this Simplex class, other than the functionality
+/// for sampling, is based on the paper
+/// "Simplify: A Theorem Prover for Program Checking"
+/// by D. Detlefs, G. Nelson, J. B. Saxe.
+///
+/// We define variables, constraints, and unknowns. Consider the example of a
+/// two-dimensional set defined by 1 + 2x + 3y >= 0 and 2x - 3y >= 0. Here,
+/// x, y, are variables while 1 + 2x + 3y >= 0, 2x - 3y >= 0 are
+/// constraints. Unknowns are either variables or constraints, i.e., x, y,
+/// 1 + 2x + 3y >= 0, 2x - 3y >= 0 are all unknowns.
+///
+/// The implementation involves a matrix called a tableau, which can be thought
+/// of as a 2D matrix of rational numbers having number of rows equal to the
+/// number of constraints and number of columns equal to one plus the number of
+/// variables. In our implementation, instead of storing rational numbers, we
+/// store a common denominator for each row, so it is in fact a matrix of
+/// integers with number of rows equal to number of constraints and number of
+/// columns equal to _two_ plus the number of variables. For example, instead of
+/// storing a row of three rationals [1/2, 2/3, 3], we would store [6, 3, 4, 18]
+/// since 3/6 = 1/2, 4/6 = 2/3, and 18/6 = 3.
+///
+/// Every row and column except the first and second columns is associated with
+/// an unknown and every unknown is associated with a row or column. The second
+/// column represents the constant, explained in more detail below. An unknown
+/// associated with a row or column is said to be in row or column position
+/// respectively.
+///
+/// The vectors var and con store information about the variables and
+/// constraints respectively, namely, whether they are in row or column
+/// position, which row or column they are associated with, and whether they
+/// correspond to a variable or a constraint.
+///
+/// An unknown is addressed by its index. If the index i is non-negative, then
+/// the variable var[i] is being addressed. If the index i is negative, then
+/// the constraint con[~i] is being addressed. Effectively this maps
+/// 0 -> var[0], 1 -> var[1], -1 -> con[0], -2 -> con[1], etc. rowUnknown[r] and
+/// colUnknown[c] are the indexes of the unknowns associated with row r and
+/// column c, respectively.
+///
+/// The unknowns in column position are together called the basis. Initially the
+/// basis is the set of variables -- in our example above, the initial basis is
+/// x, y.
+///
+/// The unknowns in row position are represented in terms of the basis unknowns.
+/// If the basis unknowns are u_1, u_2, ... u_m, and a row in the tableau is
+/// d, c, a_1, a_2, ... a_m, this representats the unknown for that row as
+/// (c + a_1*u_1 + a_2*u_2 + ... + a_m*u_m)/d. In our running example, if the
+/// basis is the initial basis of x, y, then the constraint 1 + 2x + 3y >= 0
+/// would be represented by the row [1, 1, 2, 3].
+///
+/// The association of unknowns to rows and columns can be changed by a process
+/// called pivoting, where a row unknown and a column unknown exchange places
+/// and the remaining row variables' representation is changed accordingly
+/// by eliminating the old column unknown in favour of the new column unknown.
+/// If we had pivoted the column for x with the row for 2x - 3y >= 0,
+/// the new row for x would be [2, 1, 3] since x = (1*(2x - 3y) + 3*y)/2.
+/// See the documentation for the pivot member function for details.
+///
+/// The association of unknowns to rows and columns is called the _tableau
+/// configuration_. The _sample value_ of an unknown in a particular tableau
+/// configuration is its value if all the column unknowns were set to zero.
+/// Concretely, for unknowns in column position the sample value is zero and
+/// for unknowns in row position the sample value is the constant term divided
+/// by the common denominator.
+///
+/// The tableau configuration is called _consistent_ if the sample value of all
+/// restricted unknowns is non-negative. Initially there are no constraints, and
+/// the tableau is consistent. When a new constraint is added, its sample value
+/// in the current tableau configuration may be negative. In that case, we try
+/// to find a series of pivots to bring us to a consistent tableau
+/// configuration, i.e. we try to make the new constraint's sample value
+/// non-negative without making that of any other constraints negative. (See
+/// findPivot and findPivotRow for details.) If this is not possible, then the
+/// set of constraints is mutually contradictory and the tableau is marked
+/// _empty_, which means the set of constraints has no solution.
+///
+/// This Simplex class also supports taking snapshots of the current state
+/// and rolling back to prior snapshots. This works by maintaing an undo log
+/// of operations. Snapshots are just pointers to a particular location in the
+/// log, and rolling back to a snapshot is done by reverting each log entry's
+/// operation from the end until we reach the snapshot's location.
+///
+/// Finding an integer sample is done with the Generalized Basis Reduction
+/// algorithm. See the documentation for findIntegerSample and reduceBasis.
+class Simplex {
+public:
+  enum class Direction { Up, Down };
+
+  Simplex() = delete;
+  explicit Simplex(unsigned nVar);
+  explicit Simplex(const FlatAffineConstraints &constraints);
+
+  /// Returns true if the tableau is empty (has conflicting constraints),
+  /// false otherwise.
+  bool isEmpty() const;
+
+  /// Add an inequality to the tableau. If coeffs is c_0, c_1, ... c_n, where n
+  /// is the current number of variables, then the corresponding inequality is
+  /// c_n + c_0*x_0 + c_1*x_1 + ... + c_{n-1}*x_{n-1} >= 0.
+  void addInequality(ArrayRef<int64_t> coeffs);
+
+  /// Returns the number of variables in the tableau.
+  unsigned numVariables() const;
+
+  /// Returns the number of constraints in the tableau.
+  unsigned numConstraints() const;
+
+  /// Add an equality to the tableau. If coeffs is c_0, c_1, ... c_n, where n
+  /// is the current number of variables, then the corresponding equality is
+  /// c_n + c_0*x_0 + c_1*x_1 + ... + c_{n-1}*x_{n-1} == 0.
+  void addEquality(ArrayRef<int64_t> coeffs);
+
+  /// Mark the tableau as being empty.
+  void markEmpty();
+
+  /// Get a snapshot of the current state. This is used for rolling back.
+  unsigned getSnapshot() const;
+
+  /// Rollback to a snapshot. This invalidates all later snapshots.
+  void rollback(unsigned snapshot);
+
+  /// Compute the maximum or minimum value of the given row, depending on
+  /// direction.
+  ///
+  /// Returns a (num, den) pair denoting the optimum, or None if no
+  /// optimum exists, i.e., if the expression is unbounded in this direction.
+  Optional<Fraction> computeRowOptimum(Direction direction, unsigned row);
+
+  /// Compute the maximum or minimum value of the given expression, depending on
+  /// direction.
+  ///
+  /// Returns a (num, den) pair denoting the optimum, or a null value if no
+  /// optimum exists, i.e., if the expression is unbounded in this direction.
+  Optional<Fraction> computeOptimum(Direction direction,
+                                    ArrayRef<int64_t> coeffs);
+
+  /// Returns a (min, max) pair denoting the minimum and maximum integer values
+  /// of the given expression.
+  std::pair<int64_t, int64_t> computeIntegerBounds(ArrayRef<int64_t> coeffs);
+
+  /// Returns true if the polytope is unbounded, i.e., extends to infinity in
+  /// some direction. Otherwise, returns false.
+  bool isUnbounded();
+
+  /// Make a tableau to represent a pair of points in the given tableaus, one in
+  /// tableau A and one in B.
+  static Simplex makeProduct(const Simplex &a, const Simplex &b);
+
+  /// Returns the current sample point if it is integral. Otherwise, returns an
+  /// None.
+  Optional<SmallVector<int64_t, 8>> getSamplePointIfIntegral() const;
+
+  /// Returns an integer sample point if one exists, or None
+  /// otherwise. This should only be called for bounded sets.
+  Optional<SmallVector<int64_t, 8>> findIntegerSample();
+
+  /// Print the tableau's internal state.
+  void print(raw_ostream &os) const;
+  void dump() const;
+
+private:
+  friend class GBRSimplex;
+
+  enum class Orientation { Row, Column };
+
+  /// An Unknown is either a variable or a constraint. It is always associated
+  /// with either a row or column. Whether it's a row or a column is specified
+  /// by the orientation and pos identifies the specific row or column it is
+  /// associated with. If the unknown is restricted, then it has a
+  /// non-negativity constraint associated with it, i.e., its sample value must
+  /// always be non-negative and if it cannot be made non-negative without
+  /// violating other constraints, the tableau is empty.
+  struct Unknown {
+    Unknown(Orientation oOrientation, bool oRestricted, unsigned oPos)
+        : pos(oPos), orientation(oOrientation), restricted(oRestricted) {}
+    unsigned pos;
+    Orientation orientation;
+    bool restricted : 1;
+
+    void print(raw_ostream &os) const {
+      os << (orientation == Orientation::Row ? "r" : "c");
+      os << pos;
+      if (restricted)
+        os << " [>=0]";
+    }
+  };
+
+  struct Pivot {
+    unsigned row, column;
+  };
+
+  /// Find a pivot to change the sample value of row in the specified
+  /// direction. The returned pivot row will be row if and only
+  /// if the unknown is unbounded in the specified direction.
+  ///
+  /// Returns a (row, col) pair denoting a pivot, or an empty Optional if
+  /// no valid pivot exists.
+  Optional<Pivot> findPivot(int row, Direction direction) const;
+
+  /// Swap the row with the column in the tableau's data structures but not the
+  /// tableau itself. This is used by pivot.
+  void swapRowWithCol(unsigned row, unsigned col);
+
+  /// Pivot the row with the column.
+  void pivot(unsigned row, unsigned col);
+  void pivot(Pivot pair);
+
+  /// Returns the unknown associated with index.
+  const Unknown &unknownFromIndex(int index) const;
+  /// Returns the unknown associated with col.
+  const Unknown &unknownFromColumn(unsigned col) const;
+  /// Returns the unknown associated with row.
+  const Unknown &unknownFromRow(unsigned row) const;
+  /// Returns the unknown associated with index.
+  Unknown &unknownFromIndex(int index);
+  /// Returns the unknown associated with col.
+  Unknown &unknownFromColumn(unsigned col);
+  /// Returns the unknown associated with row.
+  Unknown &unknownFromRow(unsigned row);
+
+  /// Add a new row to the tableau and the associated data structures.
+  unsigned addRow(ArrayRef<int64_t> coeffs);
+
+  /// Normalize the given row by removing common factors between the numerator
+  /// and the denominator.
+  void normalizeRow(unsigned row);
+
+  /// Swap the two rows in the tableau and associated data structures.
+  void swapRows(unsigned i, unsigned j);
+
+  /// Restore the unknown to a non-negative sample value.
+  ///
+  /// Returns true if the unknown was successfully restored to a non-negative
+  /// sample value, false otherwise.
+  LogicalResult restoreRow(Unknown &u);
+
+  enum class UndoLogEntry { RemoveLastConstraint, UnmarkEmpty };
+
+  /// Undo the operation represented by the log entry.
+  void undo(UndoLogEntry entry);
+
+  /// Find a row that can be used to pivot the column in the specified
+  /// direction. If skipRow is not null, then this row is excluded
+  /// from consideration. The returned pivot will maintain all constraints
+  /// except the column itself and skipRow, if it is set. (if these unknowns
+  /// are restricted).
+  ///
+  /// Returns the row to pivot to, or an empty Optional if the column
+  /// is unbounded in the specified direction.
+  Optional<unsigned> findPivotRow(Optional<unsigned> skipRow,
+                                  Direction direction, unsigned col) const;
+
+  /// Reduce the given basis, starting at the specified level, using general
+  /// basis reduction.
+  void reduceBasis(Matrix &basis, unsigned level);
+
+  /// The number of rows in the tableau.
+  unsigned nRow;
+
+  /// The number of columns in the tableau, including the common denominator
+  /// and the constant column.
+  unsigned nCol;
+
+  /// The matrix representing the tableau.
+  Matrix tableau;
+
+  /// This is true if the tableau has been detected to be empty, false
+  /// otherwise.
+  bool empty;
+
+  /// Holds a log of operations, used for rolling back to a previous state.
+  SmallVector<UndoLogEntry, 8> undoLog;
+
+  /// These hold the indexes of the unknown at a given row or column position.
+  /// We keep these as signed integers since that makes it convenient to check
+  /// if an index corresponds to a variable or a constraint by checking the
+  /// sign.
+  ///
+  /// colUnknown is padded with two null indexes at the front since the first
+  /// two columns don't correspond to any unknowns.
+  SmallVector<int, 8> rowUnknown, colUnknown;
+
+  /// These hold information about each unknown.
+  SmallVector<Unknown, 8> con, var;
+};
+
+} // namespace mlir
+
+#endif // MLIR_ANALYSIS_PRESBURGER_SIMPLEX_H
diff --git a/mlir/include/mlir/Analysis/Utils.h b/mlir/include/mlir/Analysis/Utils.h
index ce71e47c8e329..943a2f125b7dc 100644
--- a/mlir/include/mlir/Analysis/Utils.h
+++ b/mlir/include/mlir/Analysis/Utils.h
@@ -36,7 +36,7 @@ class Value;
 
 /// Populates 'loops' with IVs of the loops surrounding 'op' ordered from
 /// the outermost 'affine.for' operation to the innermost one.
-//  TODO(bondhugula): handle 'affine.if' ops.
+//  TODO: handle 'affine.if' ops.
 void getLoopIVs(Operation &op, SmallVectorImpl<AffineForOp> *loops);
 
 /// Returns the nesting depth of this operation, i.e., the number of loops
@@ -135,7 +135,7 @@ void getComputationSliceState(Operation *depSourceOp, Operation *depSinkOp,
 /// surrounding ops in 'opsB', as a function of IVs and symbols of loop nest
 /// surrounding ops in 'opsA' at 'loopDepth'.
 /// Returns 'success' if union was computed, 'failure' otherwise.
-// TODO(andydavis) Change this API to take 'forOpA'/'forOpB'.
+// TODO: Change this API to take 'forOpA'/'forOpB'.
 LogicalResult computeSliceUnion(ArrayRef<Operation *> opsA,
                                 ArrayRef<Operation *> opsB, unsigned loopDepth,
                                 unsigned numCommonLoops, bool isBackwardSlice,
@@ -150,7 +150,7 @@ LogicalResult computeSliceUnion(ArrayRef<Operation *> opsA,
 // Loop depth is a crucial optimization choice that determines where to
 // materialize the results of the backward slice - presenting a trade-off b/w
 // storage and redundant computation in several cases.
-// TODO(andydavis) Support computation slices with common surrounding loops.
+// TODO: Support computation slices with common surrounding loops.
 AffineForOp insertBackwardComputationSlice(Operation *srcOpInst,
                                            Operation *dstOpInst,
                                            unsigned dstLoopDepth,
@@ -271,7 +271,7 @@ struct MemRefRegion {
   /// identifiers since getMemRefRegion() is called with a specific loop depth,
   /// and thus the region is symbolic in the outer surrounding loops at that
   /// depth.
-  // TODO(bondhugula): Replace this to exploit HyperRectangularSet.
+  // TODO: Replace this to exploit HyperRectangularSet.
   FlatAffineConstraints cst;
 };
 
diff --git a/mlir/include/mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h b/mlir/include/mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h
index 5dbfce9bd00ff..1af13057f2eac 100644
--- a/mlir/include/mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h
+++ b/mlir/include/mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h
@@ -8,6 +8,7 @@
 #ifndef MLIR_CONVERSION_GPUTONVVM_GPUTONVVMPASS_H_
 #define MLIR_CONVERSION_GPUTONVVM_GPUTONVVMPASS_H_
 
+#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h"
 #include <memory>
 
 namespace mlir {
@@ -24,9 +25,11 @@ class GPUModuleOp;
 void populateGpuToNVVMConversionPatterns(LLVMTypeConverter &converter,
                                          OwningRewritePatternList &patterns);
 
-/// Creates a pass that lowers GPU dialect operations to NVVM counterparts.
-std::unique_ptr<OperationPass<gpu::GPUModuleOp>>
-createLowerGpuOpsToNVVMOpsPass();
+/// Creates a pass that lowers GPU dialect operations to NVVM counterparts. The
+/// index bitwidth used for the lowering of the device side index computations
+/// is configurable.
+std::unique_ptr<OperationPass<gpu::GPUModuleOp>> createLowerGpuOpsToNVVMOpsPass(
+    unsigned indexBitwidth = kDeriveIndexBitwidthFromDataLayout);
 
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h b/mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h
index 1722ae628e88c..677782b2dc67a 100644
--- a/mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h
+++ b/mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h
@@ -8,6 +8,7 @@
 #ifndef MLIR_CONVERSION_GPUTOROCDL_GPUTOROCDLPASS_H_
 #define MLIR_CONVERSION_GPUTOROCDL_GPUTOROCDLPASS_H_
 
+#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h"
 #include <memory>
 
 namespace mlir {
@@ -25,9 +26,12 @@ class GPUModuleOp;
 void populateGpuToROCDLConversionPatterns(LLVMTypeConverter &converter,
                                           OwningRewritePatternList &patterns);
 
-/// Creates a pass that lowers GPU dialect operations to ROCDL counterparts.
+/// Creates a pass that lowers GPU dialect operations to ROCDL counterparts. The
+/// index bitwidth used for the lowering of the device side index computations
+/// is configurable.
 std::unique_ptr<OperationPass<gpu::GPUModuleOp>>
-createLowerGpuOpsToROCDLOpsPass();
+createLowerGpuOpsToROCDLOpsPass(
+    unsigned indexBitwidth = kDeriveIndexBitwidthFromDataLayout);
 
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index 48149ced5403b..1c3b776e97808 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -100,6 +100,11 @@ def ConvertGpuLaunchFuncToGpuRuntimeCalls : Pass<"launch-func-to-gpu-runtime",
 def ConvertGpuOpsToNVVMOps : Pass<"convert-gpu-to-nvvm", "gpu::GPUModuleOp"> {
   let summary = "Generate NVVM operations for gpu operations";
   let constructor = "mlir::createLowerGpuOpsToNVVMOpsPass()";
+  let options = [
+    Option<"indexBitwidth", "index-bitwidth", "unsigned",
+           /*default=kDeriveIndexBitwidthFromDataLayout*/"0",
+           "Bitwidth of the index type, 0 to use size of machine word">
+  ];
 }
 
 //===----------------------------------------------------------------------===//
@@ -109,6 +114,11 @@ def ConvertGpuOpsToNVVMOps : Pass<"convert-gpu-to-nvvm", "gpu::GPUModuleOp"> {
 def ConvertGpuOpsToROCDLOps : Pass<"convert-gpu-to-rocdl", "gpu::GPUModuleOp"> {
   let summary = "Generate ROCDL operations for gpu operations";
   let constructor = "mlir::createLowerGpuOpsToROCDLOpsPass()";
+  let options = [
+    Option<"indexBitwidth", "index-bitwidth", "unsigned",
+           /*default=kDeriveIndexBitwidthFromDataLayout*/"0",
+           "Bitwidth of the index type, 0 to use size of machine word">
+  ];
 }
 
 //===----------------------------------------------------------------------===//
@@ -308,6 +318,11 @@ def ConvertVectorToLLVM : Pass<"convert-vector-to-llvm", "ModuleOp"> {
   let summary = "Lower the operations from the vector dialect into the LLVM "
                 "dialect";
   let constructor = "mlir::createConvertVectorToLLVMPass()";
+  let options = [
+    Option<"reassociateFPReductions", "reassociate-fp-reductions",
+           "bool", /*default=*/"false",
+           "Allows llvm to reassociate floating-point reductions for speed">
+  ];
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h b/mlir/include/mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h
new file mode 100644
index 0000000000000..59fa0082e0cff
--- /dev/null
+++ b/mlir/include/mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h
@@ -0,0 +1,44 @@
+//===------------ SCFToSPIRV.h - Pass entrypoint ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Provides patterns for lowering SCF ops to SPIR-V dialect.
+//
+//===----------------------------------------------------------------------===//
+#ifndef MLIR_CONVERSION_SCFTOSPIRV_SCFTOSPIRV_H_
+#define MLIR_CONVERSION_SCFTOSPIRV_SCFTOSPIRV_H_
+
+#include <memory>
+
+namespace mlir {
+class MLIRContext;
+class Pass;
+
+// Owning list of rewriting patterns.
+class OwningRewritePatternList;
+class SPIRVTypeConverter;
+struct ScfToSPIRVContextImpl;
+
+struct ScfToSPIRVContext {
+  ScfToSPIRVContext();
+  ~ScfToSPIRVContext();
+
+  ScfToSPIRVContextImpl *getImpl() { return impl.get(); }
+
+private:
+  std::unique_ptr<ScfToSPIRVContextImpl> impl;
+};
+
+/// Collects a set of patterns to lower from scf.for, scf.if, and
+/// loop.terminator to CFG operations within the SPIR-V dialect.
+void populateSCFToSPIRVPatterns(MLIRContext *context,
+                                SPIRVTypeConverter &typeConverter,
+                                ScfToSPIRVContext &scfToSPIRVContext,
+                                OwningRewritePatternList &patterns);
+} // namespace mlir
+
+#endif // MLIR_CONVERSION_SCFTOSPIRV_SCFTOSPIRV_H_
diff --git a/mlir/include/mlir/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.h b/mlir/include/mlir/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.h
index e82efac3abe58..4092083f26077 100644
--- a/mlir/include/mlir/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.h
+++ b/mlir/include/mlir/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.h
@@ -32,11 +32,25 @@ class SPIRVToLLVMConversion : public OpConversionPattern<SPIRVOp> {
   LLVMTypeConverter &typeConverter;
 };
 
+/// Populates type conversions with additional SPIR-V types.
+void populateSPIRVToLLVMTypeConversion(LLVMTypeConverter &typeConverter);
+
 /// Populates the given list with patterns that convert from SPIR-V to LLVM.
 void populateSPIRVToLLVMConversionPatterns(MLIRContext *context,
                                            LLVMTypeConverter &typeConverter,
                                            OwningRewritePatternList &patterns);
 
+/// Populates the given list with patterns for function conversion from SPIR-V
+/// to LLVM.
+void populateSPIRVToLLVMFunctionConversionPatterns(
+    MLIRContext *context, LLVMTypeConverter &typeConverter,
+    OwningRewritePatternList &patterns);
+
+/// Populates the given patterns for module conversion from SPIR-V to LLVM.
+void populateSPIRVToLLVMModuleConversionPatterns(
+    MLIRContext *context, LLVMTypeConverter &typeConverter,
+    OwningRewritePatternList &patterns);
+
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_SPIRVTOLLVM_CONVERTSPIRVTOLLVM_H
diff --git a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
index a7e4ff2f52cf6..72c6079db9ac5 100644
--- a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
+++ b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
@@ -15,6 +15,7 @@
 #ifndef MLIR_CONVERSION_STANDARDTOLLVM_CONVERTSTANDARDTOLLVM_H
 #define MLIR_CONVERSION_STANDARDTOLLVM_CONVERTSTANDARDTOLLVM_H
 
+#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 namespace llvm {
@@ -35,22 +36,6 @@ class LLVMDialect;
 class LLVMType;
 } // namespace LLVM
 
-/// Set of callbacks that allows the customization of LLVMTypeConverter.
-struct LLVMTypeConverterCustomization {
-  using CustomCallback = std::function<LogicalResult(LLVMTypeConverter &, Type,
-                                                     SmallVectorImpl<Type> &)>;
-
-  /// Customize the type conversion of function arguments.
-  CustomCallback funcArgConverter;
-
-  /// Used to determine the bitwidth of the LLVM integer type that the index
-  /// type gets lowered to. Defaults to deriving the size from the data layout.
-  unsigned indexBitwidth;
-
-  /// Initialize customization to default callbacks.
-  LLVMTypeConverterCustomization();
-};
-
 /// Callback to convert function argument types. It converts a MemRef function
 /// argument to a list of non-aggregate types containing descriptor
 /// information, and an UnrankedmemRef function argument to a list containing
@@ -75,13 +60,11 @@ class LLVMTypeConverter : public TypeConverter {
 public:
   using TypeConverter::convertType;
 
-  /// Create an LLVMTypeConverter using the default
-  /// LLVMTypeConverterCustomization.
+  /// Create an LLVMTypeConverter using the default LowerToLLVMOptions.
   LLVMTypeConverter(MLIRContext *ctx);
 
-  /// Create an LLVMTypeConverter using 'custom' customizations.
-  LLVMTypeConverter(MLIRContext *ctx,
-                    const LLVMTypeConverterCustomization &custom);
+  /// Create an LLVMTypeConverter using custom LowerToLLVMOptions.
+  LLVMTypeConverter(MLIRContext *ctx, const LowerToLLVMOptions &options);
 
   /// Convert a function type.  The arguments and results are converted one by
   /// one and results are packed into a wrapped LLVM IR structure type. `result`
@@ -127,7 +110,10 @@ class LLVMTypeConverter : public TypeConverter {
   LLVM::LLVMType getIndexType();
 
   /// Gets the bitwidth of the index type when converted to LLVM.
-  unsigned getIndexTypeBitwidth() { return customizations.indexBitwidth; }
+  unsigned getIndexTypeBitwidth() { return options.indexBitwidth; }
+
+  /// Gets the pointer bitwidth.
+  unsigned getPointerBitwidth(unsigned addressSpace = 0);
 
 protected:
   /// LLVM IR module used to parse/create types.
@@ -193,8 +179,8 @@ class LLVMTypeConverter : public TypeConverter {
   // Convert a 1D vector type into an LLVM vector type.
   Type convertVectorType(VectorType type);
 
-  /// Callbacks for customizing the type conversion.
-  LLVMTypeConverterCustomization customizations;
+  /// Options for customizing the llvm lowering.
+  LowerToLLVMOptions options;
 };
 
 /// Helper class to produce LLVM dialect operations extracting or inserting
@@ -386,14 +372,26 @@ class UnrankedMemRefDescriptor : public StructBuilder {
   /// Returns the number of non-aggregate values that would be produced by
   /// `unpack`.
   static unsigned getNumUnpackedValues() { return 2; }
+
+  /// Builds IR computing the sizes in bytes (suitable for opaque allocation)
+  /// and appends the corresponding values into `sizes`.
+  static void computeSizes(OpBuilder &builder, Location loc,
+                           LLVMTypeConverter &typeConverter,
+                           ArrayRef<UnrankedMemRefDescriptor> values,
+                           SmallVectorImpl<Value> &sizes);
 };
 
-/// Base class for operation conversions targeting the LLVM IR dialect. Provides
-/// conversion patterns with access to an LLVMTypeConverter.
+/// Base class for operation conversions targeting the LLVM IR dialect. It
+/// provides the conversion patterns with access to the LLVMTypeConverter and
+/// the LowerToLLVMOptions. The class captures the LLVMTypeConverter and the
+/// LowerToLLVMOptions by reference meaning the references have to remain alive
+/// during the entire pattern lifetime.
 class ConvertToLLVMPattern : public ConversionPattern {
 public:
   ConvertToLLVMPattern(StringRef rootOpName, MLIRContext *context,
                        LLVMTypeConverter &typeConverter,
+                       const LowerToLLVMOptions &options =
+                           LowerToLLVMOptions::getDefaultOptions(),
                        PatternBenefit benefit = 1);
 
   /// Returns the LLVM dialect.
@@ -445,6 +443,9 @@ class ConvertToLLVMPattern : public ConversionPattern {
 protected:
   /// Reference to the type converter, with potential extensions.
   LLVMTypeConverter &typeConverter;
+
+  /// Reference to the llvm lowering options.
+  const LowerToLLVMOptions &options;
 };
 
 /// Utility class for operation conversions targeting the LLVM dialect that
@@ -453,10 +454,11 @@ template <typename OpTy>
 class ConvertOpToLLVMPattern : public ConvertToLLVMPattern {
 public:
   ConvertOpToLLVMPattern(LLVMTypeConverter &typeConverter,
+                         const LowerToLLVMOptions &options,
                          PatternBenefit benefit = 1)
       : ConvertToLLVMPattern(OpTy::getOperationName(),
                              &typeConverter.getContext(), typeConverter,
-                             benefit) {}
+                             options, benefit) {}
 };
 
 namespace LLVM {
diff --git a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h
index 5479f189b73cb..1f76b929679ef 100644
--- a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h
+++ b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h
@@ -14,63 +14,66 @@
 namespace mlir {
 class LLVMTypeConverter;
 class ModuleOp;
-template <typename T> class OperationPass;
+template <typename T>
+class OperationPass;
 class OwningRewritePatternList;
 
+/// Value to pass as bitwidth for the index type when the converter is expected
+/// to derive the bitwidth from the LLVM data layout.
+static constexpr unsigned kDeriveIndexBitwidthFromDataLayout = 0;
+
+/// Options to control the Standard dialect to LLVM lowering. The struct is used
+/// to share lowering options between passes, patterns, and type converter.
+struct LowerToLLVMOptions {
+  bool useBarePtrCallConv = false;
+  bool emitCWrappers = false;
+  unsigned indexBitwidth = kDeriveIndexBitwidthFromDataLayout;
+  /// Use aligned_alloc for heap allocations.
+  bool useAlignedAlloc = false;
+
+  /// Get a statically allocated copy of the default LowerToLLVMOptions.
+  static const LowerToLLVMOptions &getDefaultOptions() {
+    static LowerToLLVMOptions options;
+    return options;
+  }
+};
+
 /// Collect a set of patterns to convert memory-related operations from the
 /// Standard dialect to the LLVM dialect, excluding non-memory-related
 /// operations and FuncOp.
 void populateStdToLLVMMemoryConversionPatterns(
     LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    bool useAlignedAlloc);
+    const LowerToLLVMOptions &options);
 
 /// Collect a set of patterns to convert from the Standard dialect to the LLVM
 /// dialect, excluding the memory-related operations.
 void populateStdToLLVMNonMemoryConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns);
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
+    const LowerToLLVMOptions &options);
 
 /// Collect the default pattern to convert a FuncOp to the LLVM dialect. If
 /// `emitCWrappers` is set, the pattern will also produce functions
 /// that pass memref descriptors by pointer-to-structure in addition to the
 /// default unpacked form.
-void populateStdToLLVMDefaultFuncOpConversionPattern(
+void populateStdToLLVMFuncOpConversionPattern(
     LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    bool emitCWrappers = false);
+    const LowerToLLVMOptions &options);
 
-/// Collect a set of default patterns to convert from the Standard dialect to
-/// LLVM.
-void populateStdToLLVMConversionPatterns(LLVMTypeConverter &converter,
-                                         OwningRewritePatternList &patterns,
-                                         bool emitCWrappers = false,
-                                         bool useAlignedAlloc = false);
-
-/// Collect a set of patterns to convert from the Standard dialect to
-/// LLVM using the bare pointer calling convention for MemRef function
-/// arguments.
-void populateStdToLLVMBarePtrConversionPatterns(
+/// Collect the patterns to convert from the Standard dialect to LLVM. The
+/// conversion patterns capture the LLVMTypeConverter and the LowerToLLVMOptions
+/// by reference meaning the references have to remain alive during the entire
+/// pattern lifetime.
+void populateStdToLLVMConversionPatterns(
     LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    bool useAlignedAlloc);
-
-/// Value to pass as bitwidth for the index type when the converter is expected
-/// to derive the bitwidth from the LLVM data layout.
-static constexpr unsigned kDeriveIndexBitwidthFromDataLayout = 0;
-
-struct LowerToLLVMOptions {
-  bool useBarePtrCallConv = false;
-  bool emitCWrappers = false;
-  unsigned indexBitwidth = kDeriveIndexBitwidthFromDataLayout;
-  /// Use aligned_alloc for heap allocations.
-  bool useAlignedAlloc = false;
-};
+    const LowerToLLVMOptions &options =
+        LowerToLLVMOptions::getDefaultOptions());
 
 /// Creates a pass to convert the Standard dialect into the LLVMIR dialect.
 /// stdlib malloc/free is used by default for allocating memrefs allocated with
 /// std.alloc, while LLVM's alloca is used for those allocated with std.alloca.
 std::unique_ptr<OperationPass<ModuleOp>>
-createLowerToLLVMPass(const LowerToLLVMOptions &options = {
-                          /*useBarePtrCallConv=*/false, /*emitCWrappers=*/false,
-                          /*indexBitwidth=*/kDeriveIndexBitwidthFromDataLayout,
-                          /*useAlignedAlloc=*/false});
+createLowerToLLVMPass(const LowerToLLVMOptions &options =
+                          LowerToLLVMOptions::getDefaultOptions());
 
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h b/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h
index e09a74d5be838..82aa8287d90f3 100644
--- a/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h
+++ b/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h
@@ -16,6 +16,18 @@ class ModuleOp;
 template <typename T>
 class OperationPass;
 
+/// Options to control Vector to LLVM lowering.
+///
+/// This should kept in sync with VectorToLLVM options defined for the
+/// ConvertVectorToLLVM pass in include/mlir/Conversion/Passes.td
+struct LowerVectorToLLVMOptions {
+  bool reassociateFPReductions = false;
+  LowerVectorToLLVMOptions &setReassociateFPReductions(bool r) {
+    reassociateFPReductions = r;
+    return *this;
+  }
+};
+
 /// Collect a set of patterns to convert from Vector contractions to LLVM Matrix
 /// Intrinsics. To lower to assembly, the LLVM flag -lower-matrix-intrinsics
 /// will be needed when invoking LLVM.
@@ -23,11 +35,13 @@ void populateVectorToLLVMMatrixConversionPatterns(
     LLVMTypeConverter &converter, OwningRewritePatternList &patterns);
 
 /// Collect a set of patterns to convert from the Vector dialect to LLVM.
-void populateVectorToLLVMConversionPatterns(LLVMTypeConverter &converter,
-                                            OwningRewritePatternList &patterns);
+void populateVectorToLLVMConversionPatterns(
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
+    bool reassociateFPReductions = false);
 
 /// Create a pass to convert vector operations to the LLVMIR dialect.
-std::unique_ptr<OperationPass<ModuleOp>> createConvertVectorToLLVMPass();
+std::unique_ptr<OperationPass<ModuleOp>> createConvertVectorToLLVMPass(
+    const LowerVectorToLLVMOptions &options = LowerVectorToLLVMOptions());
 
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/AVX512/AVX512.td b/mlir/include/mlir/Dialect/AVX512/AVX512.td
index 82d24ef13d68d..e1ed35c50e875 100644
--- a/mlir/include/mlir/Dialect/AVX512/AVX512.td
+++ b/mlir/include/mlir/Dialect/AVX512/AVX512.td
@@ -55,7 +55,7 @@ def MaskRndScaleOp : AVX512_Op<"mask.rndscale", [NoSideEffect,
                    I32:$k,
                    VectorOfLengthAndType<[16, 8], [F32, F64]>:$a,
                    AnyTypeOf<[I16, I8]>:$imm,
-                   // TODO(ntv): figure rounding out (optional operand?).
+                   // TODO: figure rounding out (optional operand?).
                    I32:$rounding
             );
   let results = (outs VectorOfLengthAndType<[16, 8], [F32, F64]>:$dst);
@@ -87,7 +87,7 @@ def MaskScaleFOp : AVX512_Op<"mask.scalef", [NoSideEffect,
                    VectorOfLengthAndType<[16, 8], [F32, F64]>:$a,
                    VectorOfLengthAndType<[16, 8], [F32, F64]>:$b,
                    AnyTypeOf<[I16, I8]>:$k,
-                   // TODO(ntv): figure rounding out (optional operand?).
+                   // TODO: figure rounding out (optional operand?).
                    I32:$rounding
             );
   let results = (outs VectorOfLengthAndType<[16, 8], [F32, F64]>:$dst);
diff --git a/mlir/include/mlir/Dialect/Affine/EDSC/Builders.h b/mlir/include/mlir/Dialect/Affine/EDSC/Builders.h
index 5c99a430c8621..96191e01296a5 100644
--- a/mlir/include/mlir/Dialect/Affine/EDSC/Builders.h
+++ b/mlir/include/mlir/Dialect/Affine/EDSC/Builders.h
@@ -66,10 +66,14 @@ Value operator^(Value lhs, Value rhs);
 /// Comparison operator overloadings.
 Value eq(Value lhs, Value rhs);
 Value ne(Value lhs, Value rhs);
-Value operator<(Value lhs, Value rhs);
-Value operator<=(Value lhs, Value rhs);
-Value operator>(Value lhs, Value rhs);
-Value operator>=(Value lhs, Value rhs);
+Value slt(Value lhs, Value rhs);
+Value sle(Value lhs, Value rhs);
+Value sgt(Value lhs, Value rhs);
+Value sge(Value lhs, Value rhs);
+Value ult(Value lhs, Value rhs);
+Value ule(Value lhs, Value rhs);
+Value ugt(Value lhs, Value rhs);
+Value uge(Value lhs, Value rhs);
 
 } // namespace op
 
@@ -159,24 +163,44 @@ Value TemplatedIndexedValue<Load, Store>::ne(Value e) {
   return ne(value, e);
 }
 template <typename Load, typename Store>
-Value TemplatedIndexedValue<Load, Store>::operator<(Value e) {
-  using op::operator<;
-  return static_cast<Value>(*this) < e;
+Value TemplatedIndexedValue<Load, Store>::slt(Value e) {
+  using op::slt;
+  return slt(static_cast<Value>(*this), e);
 }
 template <typename Load, typename Store>
-Value TemplatedIndexedValue<Load, Store>::operator<=(Value e) {
-  using op::operator<=;
-  return static_cast<Value>(*this) <= e;
+Value TemplatedIndexedValue<Load, Store>::sle(Value e) {
+  using op::sle;
+  return sle(static_cast<Value>(*this), e);
 }
 template <typename Load, typename Store>
-Value TemplatedIndexedValue<Load, Store>::operator>(Value e) {
-  using op::operator>;
-  return static_cast<Value>(*this) > e;
+Value TemplatedIndexedValue<Load, Store>::sgt(Value e) {
+  using op::sgt;
+  return sgt(static_cast<Value>(*this), e);
 }
 template <typename Load, typename Store>
-Value TemplatedIndexedValue<Load, Store>::operator>=(Value e) {
-  using op::operator>=;
-  return static_cast<Value>(*this) >= e;
+Value TemplatedIndexedValue<Load, Store>::sge(Value e) {
+  using op::sge;
+  return sge(static_cast<Value>(*this), e);
+}
+template <typename Load, typename Store>
+Value TemplatedIndexedValue<Load, Store>::ult(Value e) {
+  using op::ult;
+  return ult(static_cast<Value>(*this), e);
+}
+template <typename Load, typename Store>
+Value TemplatedIndexedValue<Load, Store>::ule(Value e) {
+  using op::ule;
+  return ule(static_cast<Value>(*this), e);
+}
+template <typename Load, typename Store>
+Value TemplatedIndexedValue<Load, Store>::ugt(Value e) {
+  using op::ugt;
+  return ugt(static_cast<Value>(*this), e);
+}
+template <typename Load, typename Store>
+Value TemplatedIndexedValue<Load, Store>::uge(Value e) {
+  using op::uge;
+  return uge(static_cast<Value>(*this), e);
 }
 
 } // namespace edsc
diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
index c2987c8f5fda9..b88028115a6f6 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
@@ -15,6 +15,7 @@
 #define MLIR_DIALECT_AFFINE_IR_AFFINEOPS_H
 
 #include "mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.h"
+#include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/Dialect.h"
@@ -75,10 +76,10 @@ bool isTopLevelValue(Value value);
 //   affine.dma_start %src[%i, %j], %dst[%k, %l], %tag[%idx], %num_elements,
 //     %stride, %num_elt_per_stride : ...
 //
-// TODO(mlir-team): add additional operands to allow source and destination
-// striding, and multiple stride levels (possibly using AffineMaps to specify
-// multiple levels of striding).
-// TODO(andydavis) Consider replacing src/dst memref indices with view memrefs.
+// TODO: add additional operands to allow source and destination striding, and
+// multiple stride levels (possibly using AffineMaps to specify multiple levels
+// of striding).
+// TODO: Consider replacing src/dst memref indices with view memrefs.
 class AffineDmaStartOp : public Op<AffineDmaStartOp, OpTrait::VariadicOperands,
                                    OpTrait::ZeroResult> {
 public:
diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
index 0ddddee0cdb78..6d18e02845b92 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
@@ -14,7 +14,9 @@
 #define AFFINE_OPS
 
 include "mlir/Dialect/Affine/IR/AffineOpsBase.td"
+include "mlir/Dialect/StandardOps/IR/StandardOpsBase.td"
 include "mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.td"
+include "mlir/Interfaces/ControlFlowInterfaces.td"
 include "mlir/Interfaces/LoopLikeInterface.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
 
@@ -278,7 +280,8 @@ def AffineForOp : Affine_Op<"for",
 }
 
 def AffineIfOp : Affine_Op<"if",
-                           [ImplicitAffineTerminator, RecursiveSideEffects]> {
+                           [ImplicitAffineTerminator, RecursiveSideEffects,
+                            NoRegionArguments]> {
   let summary = "if-then-else operation";
   let description = [{
     Syntax:
@@ -300,12 +303,13 @@ def AffineIfOp : Affine_Op<"if",
     symbols.
 
     The `affine.if` operation contains two regions for the "then" and "else"
-    clauses. The latter may be empty (i.e. contain no blocks), meaning the
-    absence of the else clause. When non-empty, both regions must contain
-    exactly one block terminating with
-    [`affine.yield`](#affineyield-operation). *Note:* when `affine.if`
-    is printed in custom format, the terminator is omitted. These blocks must
-    not have any arguments.
+    clauses.  `affine.if` may return results that are defined in its regions.
+    The values defined are determined by which execution path is taken.  Each
+    region of the `affine.if` must contain a single block with no arguments,
+    and be terminated by `affine.yield`.  If `affine.if` defines no values,
+    the `affine.yield` can be left out, and will be inserted implicitly.
+    Otherwise, it must be explicit.  If no values are defined, the else block
+    may be empty (i.e. contain no blocks).
 
     Example:
 
@@ -326,6 +330,26 @@ def AffineIfOp : Affine_Op<"if",
       return
     }
     ```
+
+    Example with an explicit yield (initialization with edge padding):
+
+    ```mlir
+    #interior = affine_set<(i, j) : (i - 1 >= 0, j - 1 >= 0,  10 - i >= 0, 10 - j >= 0)> (%i, %j)
+    func @pad_edges(%I : memref<10x10xf32>) -> (memref<12x12xf32) {
+      %O = alloc memref<12x12xf32>
+      affine.parallel (%i, %j) = (0, 0) to (12, 12) {
+        %1 = affine.if #interior (%i, %j) {
+          %2 = load %I[%i - 1, %j - 1] : memref<10x10xf32>
+          affine.yield %2
+        } else {
+          %2 = constant 0.0 : f32
+          affine.yield %2 : f32
+        }
+        affine.store %1, %O[%i, %j] : memref<12x12xf32>
+      }
+      return %O
+    }
+    ```
   }];
   let arguments = (ins Variadic<AnyType>);
   let results = (outs Variadic<AnyType>:$results);
@@ -535,48 +559,68 @@ def AffineParallelOp : Affine_Op<"parallel",
     The same restrictions hold for these SSA values as for all bindings of SSA
     values to dimensions and symbols.
 
+    Each value yielded by affine.yield will be accumulated/reduced via one of
+    the reduction methods defined in the AtomicRMWKind enum.  The order of
+    reduction is unspecified, and lowering may produce any valid ordering.
+    Loops with a 0 trip count will produce as a result the identity value
+    associated with each reduction (i.e. 0.0 for addf, 1.0 for mulf).  Assign
+    reductions for loops with a trip count != 1 produces undefined results.
+
     Note: Calling AffineParallelOp::build will create the required region and
-    block, and insert the required terminator. Parsing will also create the
-    required region, block, and terminator, even when they are missing from the
-    textual representation.
+    block, and insert the required terminator if it is trivial (i.e. no values
+    are yielded).  Parsing will also create the required region, block, and 
+    terminator, even when they are missing from the textual representation.
 
-    Example:
+    Example (3x3 valid convolution):
 
     ```mlir
-    affine.parallel (%i, %j) = (0, 0) to (10, 10) step (1, 1) {
-      ...
+    fuction @conv_2d(%D : memref<100x100xf32>, %K : memref<3x3xf32>) -> (memref<98x98xf32>) {
+      %O = alloc memref<98x98xf32>
+      affine.parallel (%x, %y) = (0, 0) to (98, 98) {
+        %0 = affine.parallel (%kx, %ky) = (0, 0) to (2, 2) reduce ("addf") {
+          %1 = affine.load %D[%x + %kx, %y + %ky] : memref<100x100xf32>
+          %2 = affine.load %K[%kx, %ky] : memref<3x3xf32>
+          %3 = mulf %1, %2 : f32
+          affine.yield %3 : f32
+        }
+        affine.store %0, O[%x, %y] : memref<98x98xf32>
+      }
+      return %O
     }
     ```
   }];
 
   let arguments = (ins
+     TypedArrayAttrBase<AtomicRMWKindAttr, "Reduction ops">:$reductions,
      AffineMapAttr:$lowerBoundsMap,
      AffineMapAttr:$upperBoundsMap,
      I64ArrayAttr:$steps,
      Variadic<Index>:$mapOperands);
-  let results = (outs Variadic<AnyMemRef>:$results);
+  let results = (outs Variadic<AnyType>:$results);
   let regions = (region SizedRegion<1>:$region);
 
   let builders = [
-    OpBuilder<"OpBuilder &builder, OperationState& result, "
-              "ArrayRef<Type> resultTypes, ArrayRef<int64_t> ranges">,
-    OpBuilder<"OpBuilder &builder, OperationState& result, "
-              "ArrayRef<Type> resultTypes, AffineMap lbMap, "
-              "ValueRange lbArgs, AffineMap ubMap, ValueRange ubArgs">,
-    OpBuilder<"OpBuilder &builder, OperationState& result, "
-              "ArrayRef<Type> resultTypes, AffineMap lbMap, ValueRange lbArgs, "
-              "AffineMap ubMap, ValueRange ubArgs, ArrayRef<int64_t> steps">
+    OpBuilder<"OpBuilder &builder, OperationState &result, "
+              "ArrayRef<Type> resultTypes, "
+              "ArrayRef<AtomicRMWKind> reductions, "
+              "ArrayRef<int64_t> ranges">,
+    OpBuilder<"OpBuilder &builder, OperationState &result, "
+              "ArrayRef<Type> resultTypes, "
+              "ArrayRef<AtomicRMWKind> reductions, "
+              "AffineMap lbMap, ValueRange lbArgs, "
+              "AffineMap ubMap, ValueRange ubArgs">,
+    OpBuilder<"OpBuilder &builder, OperationState &result, "
+              "ArrayRef<Type> resultTypes, "
+              "ArrayRef<AtomicRMWKind> reductions, "
+              "AffineMap lbMap, ValueRange lbArgs, "
+              "AffineMap ubMap, ValueRange ubArgs, "
+              "ArrayRef<int64_t> steps">
   ];
 
   let extraClassDeclaration = [{
     /// Get the number of dimensions.
     unsigned getNumDims();
 
-    operand_range getLowerBoundsOperands();
-    operand_range getUpperBoundsOperands();
-
-    AffineValueMap getLowerBoundsValueMap();
-    AffineValueMap getUpperBoundsValueMap();
     AffineValueMap getRangesValueMap();
 
     /// Get ranges as constants, may fail in dynamic case.
@@ -587,14 +631,27 @@ def AffineParallelOp : Affine_Op<"parallel",
     MutableArrayRef<BlockArgument> getIVs() {
       return getBody()->getArguments();
     }
+
+    operand_range getLowerBoundsOperands();
+    AffineValueMap getLowerBoundsValueMap();
+    void setLowerBounds(ValueRange operands, AffineMap map);
+    void setLowerBoundsMap(AffineMap map);
+
+    operand_range getUpperBoundsOperands();
+    AffineValueMap getUpperBoundsValueMap();
+    void setUpperBounds(ValueRange operands, AffineMap map);
+    void setUpperBoundsMap(AffineMap map);
+
     void setSteps(ArrayRef<int64_t> newSteps);
 
+    static StringRef getReductionsAttrName() { return "reductions"; }
     static StringRef getLowerBoundsMapAttrName() { return "lowerBoundsMap"; }
     static StringRef getUpperBoundsMapAttrName() { return "upperBoundsMap"; }
     static StringRef getStepsAttrName() { return "steps"; }
   }];
 
   let hasCanonicalizer = 1;
+  let hasFolder = 1;
 }
 
 def AffinePrefetchOp : Affine_Op<"prefetch"> {
@@ -743,7 +800,7 @@ def AffineStoreOp : AffineStoreOpBase<"store"> {
   let hasFolder = 1;
 }
 
-def AffineYieldOp : Affine_Op<"yield", [NoSideEffect, Terminator]> {
+def AffineYieldOp : Affine_Op<"yield", [NoSideEffect, Terminator, ReturnLike]> {
   let summary = "Yield values to parent operation";
   let description = [{
     "affine.yield" yields zero or more SSA values from an affine op region and
diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineValueMap.h b/mlir/include/mlir/Dialect/Affine/IR/AffineValueMap.h
index ffc6d73f38259..332a496fcc56d 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineValueMap.h
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineValueMap.h
@@ -74,6 +74,9 @@ class AffineValueMap {
   ArrayRef<Value> getOperands() const;
   AffineMap getAffineMap() const;
 
+  /// Return success if the map and/or operands have been modified.
+  LogicalResult canonicalize();
+
 private:
   // A mutable affine map.
   MutableAffineMap map;
diff --git a/mlir/include/mlir/Dialect/Affine/Passes.h b/mlir/include/mlir/Dialect/Affine/Passes.h
index 0d7c3be240c99..18b3b790338d8 100644
--- a/mlir/include/mlir/Dialect/Affine/Passes.h
+++ b/mlir/include/mlir/Dialect/Affine/Passes.h
@@ -36,6 +36,10 @@ std::unique_ptr<OperationPass<FuncOp>> createSimplifyAffineStructuresPass();
 std::unique_ptr<OperationPass<FuncOp>>
 createAffineLoopInvariantCodeMotionPass();
 
+/// Creates a pass to convert all parallel affine.for's into 1-d affine.parallel
+/// ops.
+std::unique_ptr<OperationPass<FuncOp>> createAffineParallelizePass();
+
 /// Performs packing (or explicit copying) of accessed memref regions into
 /// buffers in the specified faster memory space through either pointwise copies
 /// or DMA operations.
diff --git a/mlir/include/mlir/Dialect/Affine/Passes.td b/mlir/include/mlir/Dialect/Affine/Passes.td
index 06e0920413a95..810640058155f 100644
--- a/mlir/include/mlir/Dialect/Affine/Passes.td
+++ b/mlir/include/mlir/Dialect/Affine/Passes.td
@@ -112,6 +112,11 @@ def AffineVectorize : FunctionPass<"affine-super-vectorize"> {
   ];
 }
 
+def AffineParallelize : FunctionPass<"affine-parallelize"> {
+  let summary = "Convert affine.for ops into 1-D affine.parallel";
+  let constructor = "mlir::createAffineParallelizePass()";
+}
+
 def SimplifyAffineStructures : FunctionPass<"simplify-affine-structures"> {
   let summary = "Simplify affine expressions in maps/sets and normalize "
                 "memrefs";
diff --git a/mlir/include/mlir/Dialect/Affine/Utils.h b/mlir/include/mlir/Dialect/Affine/Utils.h
index a2c0211b301e3..19df93f760f5e 100644
--- a/mlir/include/mlir/Dialect/Affine/Utils.h
+++ b/mlir/include/mlir/Dialect/Affine/Utils.h
@@ -15,9 +15,16 @@
 
 namespace mlir {
 
+class AffineForOp;
 class AffineIfOp;
+class AffineParallelOp;
 struct LogicalResult;
 
+/// Replaces parallel affine.for op with 1-d affine.parallel op.
+/// mlir::isLoopParallel detect the parallel affine.for ops.
+/// There is no cost model currently used to drive this parallelization.
+void affineParallelize(AffineForOp forOp);
+
 /// Hoists out affine.if/else to as high as possible, i.e., past all invariant
 /// affine.fors/parallel's. Returns success if any hoisting happened; folded` is
 /// set to true if the op was folded or erased. This hoisting could lead to
diff --git a/mlir/include/mlir/Dialect/GPU/GPUBase.td b/mlir/include/mlir/Dialect/GPU/GPUBase.td
index 16ce93fb1ed9a..32e0952a15b41 100644
--- a/mlir/include/mlir/Dialect/GPU/GPUBase.td
+++ b/mlir/include/mlir/Dialect/GPU/GPUBase.td
@@ -39,7 +39,7 @@ def GPU_Dialect : Dialect {
 
     /// Returns the number of workgroup (thread, block) dimensions supported in
     /// the GPU dialect.
-    // TODO(zinenko,herhut): consider generalizing this.
+    // TODO: consider generalizing this.
     static unsigned getNumWorkgroupDimensions() { return 3; }
 
     /// Returns the numeric value used to identify the workgroup memory address
diff --git a/mlir/include/mlir/Dialect/GPU/GPUOps.td b/mlir/include/mlir/Dialect/GPU/GPUOps.td
index 281696d0eb987..e7e67e24381df 100644
--- a/mlir/include/mlir/Dialect/GPU/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/GPUOps.td
@@ -224,8 +224,7 @@ def GPU_GPUFuncOp : GPU_Op<"func", [HasParent<"GPUModuleOp">,
     ///  - the argument/result attributes may need an update: if the new type
     ///  has less parameters we drop the extra attributes, if there are more
     ///  parameters they won't have any attributes.
-    // TODO(b/146349912): consider removing this function thanks to rewrite
-    // patterns.
+    // TODO: consider removing this function thanks to rewrite patterns.
     void setType(FunctionType newType);
 
     /// Returns the number of buffers located in the workgroup memory.
diff --git a/mlir/include/mlir/Dialect/GPU/ParallelLoopMapper.h b/mlir/include/mlir/Dialect/GPU/ParallelLoopMapper.h
index 842734d256d32..298ec0c803f0f 100644
--- a/mlir/include/mlir/Dialect/GPU/ParallelLoopMapper.h
+++ b/mlir/include/mlir/Dialect/GPU/ParallelLoopMapper.h
@@ -44,7 +44,7 @@ inline Processor getProcessor(ParallelLoopDimMapping attr) {
 }
 
 /// Helper function to create a ParallelDimMapperAttr.
-/// TODO(ravishankarm/antiagainst): Replace its uses with an auto-gened method.
+/// TODO: Replace its uses with an auto-gened method.
 ParallelLoopDimMapping getParallelLoopDimMappingAttr(Processor processor,
                                                      AffineMap map,
                                                      AffineMap bound);
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
index 48eecb4eed874..d88b372dbf43d 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
@@ -214,10 +214,30 @@ class LLVM_VectorReduction<string mnem>
     : LLVM_OneResultIntrOp<"experimental.vector.reduce." # mnem, [], [0], []>,
       Arguments<(ins LLVM_Type)>;
 
-// LLVM vector reduction over a single vector, with an initial value.
+// LLVM vector reduction over a single vector, with an initial value,
+// and with permission to reassociate the reduction operations.
 class LLVM_VectorReductionV2<string mnem>
-    : LLVM_OneResultIntrOp<"experimental.vector.reduce.v2." # mnem,
-                           [0], [1], []>,
-      Arguments<(ins LLVM_Type, LLVM_Type)>;
+    : LLVM_OpBase<LLVM_Dialect, "intr.experimental.vector.reduce.v2." # mnem, []>,
+      Results<(outs LLVM_Type:$res)>,
+      Arguments<(ins LLVM_Type, LLVM_Type,
+                 DefaultValuedAttr<BoolAttr, "false">:$reassoc)> {
+  let llvmBuilder = [{
+    llvm::Module *module = builder.GetInsertBlock()->getModule();
+    llvm::Function *fn = llvm::Intrinsic::getDeclaration(
+        module,
+        llvm::Intrinsic::experimental_vector_reduce_v2_}] # mnem # [{,
+        { }] # StrJoin<!listconcat(
+            ListIntSubst<LLVM_IntrPatterns.result, [0]>.lst,
+            ListIntSubst<LLVM_IntrPatterns.operand, [1]>.lst)>.result # [{
+        });
+    auto operands = lookupValues(opInst.getOperands());
+    llvm::FastMathFlags origFM = builder.getFastMathFlags();
+    llvm::FastMathFlags tempFM = origFM;
+    tempFM.setAllowReassoc($reassoc);
+    builder.setFastMathFlags(tempFM);  // set fastmath flag
+    $res = builder.CreateCall(fn, operands);
+    builder.setFastMathFlags(origFM);  // restore fastmath flag
+  }];
+}
 
 #endif  // LLVMIR_OP_BASE
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 8b64854595322..663a820905ce3 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -575,6 +575,8 @@ def Linkage : LLVM_EnumAttr<
 def LLVM_AddressOfOp
     : LLVM_OneResultOp<"mlir.addressof">,
       Arguments<(ins FlatSymbolRefAttr:$global_name)> {
+  let summary = "Creates a pointer pointing to a global or a function";
+
   let builders = [
     OpBuilder<"OpBuilder &builder, OperationState &result, LLVMType resType, "
               "StringRef name, ArrayRef<NamedAttribute> attrs = {}", [{
@@ -586,13 +588,21 @@ def LLVM_AddressOfOp
               "ArrayRef<NamedAttribute> attrs = {}", [{
       build(builder, result,
             global.getType().getPointerTo(global.addr_space().getZExtValue()),
-            global.sym_name(), attrs);}]>
+            global.sym_name(), attrs);}]>,
+
+    OpBuilder<"OpBuilder &builder, OperationState &result, LLVMFuncOp func, "
+              "ArrayRef<NamedAttribute> attrs = {}", [{
+      build(builder, result,
+            func.getType().getPointerTo(), func.getName(), attrs);}]>
   ];
 
   let extraClassDeclaration = [{
     /// Return the llvm.mlir.global operation that defined the value referenced
     /// here.
     GlobalOp getGlobal();
+
+    /// Return the llvm.func operation that is referenced here.
+    LLVMFuncOp getFunction();
   }];
 
   let assemblyFormat = "$global_name attr-dict `:` type($res)";
@@ -733,6 +743,7 @@ def LLVM_ConstantOp
       LLVM_Builder<"$res = getLLVMConstant($_resultType, $value, $_location);">
 {
   let assemblyFormat = "`(` $value `)` attr-dict `:` type($res)";
+  let verifier = [{ return ::verify(*this); }];
 }
 
 def LLVM_DialectCastOp : LLVM_Op<"mlir.cast", [NoSideEffect]>,
@@ -791,6 +802,15 @@ def LLVM_Prefetch : LLVM_ZeroResultIntrOp<"prefetch", [0]>,
 def LLVM_SinOp : LLVM_UnaryIntrinsicOp<"sin">;
 def LLVM_SqrtOp : LLVM_UnaryIntrinsicOp<"sqrt">;
 def LLVM_PowOp : LLVM_BinarySameArgsIntrinsicOp<"pow">;
+def LLVM_BitReverseOp : LLVM_UnaryIntrinsicOp<"bitreverse">;
+def LLVM_CtPopOp : LLVM_UnaryIntrinsicOp<"ctpop">;
+
+def LLVM_MemcpyOp : LLVM_ZeroResultIntrOp<"memcpy", [0, 1, 2]>,
+                    Arguments<(ins LLVM_Type:$dst, LLVM_Type:$src,
+                               LLVM_Type:$len, LLVM_Type:$isVolatile)>;
+def LLVM_MemcpyInlineOp : LLVM_ZeroResultIntrOp<"memcpy.inline", [0, 1, 2]>,
+                          Arguments<(ins LLVM_Type:$dst, LLVM_Type:$src,
+                                     LLVM_Type:$len, LLVM_Type:$isVolatile)>;
 
 //
 // Vector Reductions.
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index a2ab459c3dea2..5c793f8547dab 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -33,16 +33,25 @@ class NVVM_Op<string mnemonic, list<OpTrait> traits = []> :
   LLVM_OpBase<NVVM_Dialect, mnemonic, traits> {
 }
 
+//===----------------------------------------------------------------------===//
+// NVVM intrinsic operations
+//===----------------------------------------------------------------------===//
+
+class NVVM_IntrOp<string mnem, list<int> overloadedResults,
+                  list<int> overloadedOperands, list<OpTrait> traits,
+                  bit hasResult>
+  : LLVM_IntrOpBase<NVVM_Dialect, mnem, "nvvm_" # !subst(".", "_", mnem),
+                    overloadedResults, overloadedOperands, traits, hasResult>;
+
+
 //===----------------------------------------------------------------------===//
 // NVVM special register op definitions
 //===----------------------------------------------------------------------===//
 
 class NVVM_SpecialRegisterOp<string mnemonic,
     list<OpTrait> traits = []> :
-  NVVM_Op<mnemonic, !listconcat(traits, [NoSideEffect])>,
-  Results<(outs LLVM_Type:$res)>, Arguments<(ins)> {
-  string llvmBuilder = "$res = createIntrinsicCall(builder,"
-    # "llvm::Intrinsic::nvvm_" # !subst(".","_", mnemonic) # ");";
+  NVVM_IntrOp<mnemonic, [], [], !listconcat(traits, [NoSideEffect]), 1>,
+  Arguments<(ins)> {
   let assemblyFormat = "attr-dict `:` type($res)";
 }
 
diff --git a/mlir/include/mlir/Dialect/Linalg/EDSC/Builders.h b/mlir/include/mlir/Dialect/Linalg/EDSC/Builders.h
index 15f89a8ec5e01..5b6cb0ac7fa4d 100644
--- a/mlir/include/mlir/Dialect/Linalg/EDSC/Builders.h
+++ b/mlir/include/mlir/Dialect/Linalg/EDSC/Builders.h
@@ -67,7 +67,7 @@ void mulRegionBuilder(ValueRange args);
 /// the current ScopedContext, at the current insert point.
 void macRegionBuilder(ValueRange args);
 
-/// TODO(ntv): In the future we should tie these implementations to something in
+/// TODO: In the future we should tie these implementations to something in
 /// Tablegen that generates the proper interfaces and the proper sugared named
 /// ops.
 
@@ -119,7 +119,7 @@ Operation *linalg_generic_pointwise_max(StructuredIndexed I1,
                                         StructuredIndexed I2,
                                         StructuredIndexed O);
 
-// TODO(ntv): Implement more useful pointwise operations on a per-need basis.
+// TODO: Implement more useful pointwise operations on a per-need basis.
 
 using MatmulRegionBuilder = function_ref<void(ValueRange args)>;
 
@@ -187,7 +187,7 @@ linalg_generic_matmul(Container values,
 ///
 /// For now `...` must be empty (i.e. only 2-D convolutions are supported).
 ///
-// TODO(ntv) Extend convolution rank with some template magic.
+// TODO: Extend convolution rank with some template magic.
 Operation *linalg_generic_conv_nhwc(Value vI, Value vW, Value vO,
                                     ArrayRef<int> strides = {},
                                     ArrayRef<int> dilations = {});
@@ -222,7 +222,7 @@ Operation *linalg_generic_conv_nhwc(Container values,
 ///
 /// For now `...` must be empty (i.e. only 2-D convolutions are supported).
 ///
-// TODO(ntv) Extend convolution rank with some template magic.
+// TODO: Extend convolution rank with some template magic.
 Operation *linalg_generic_dilated_conv_nhwc(Value vI, Value vW, Value vO,
                                             int depth_multiplier = 1,
                                             ArrayRef<int> strides = {},
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td
index 3b52b0e639184..7955345f69668 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td
@@ -26,7 +26,8 @@ def Linalg_Dialect : Dialect {
 
     Additional [Linalg Dialect
     Documentation](https://mlir.llvm.org/docs/Dialects/Linalg) and a
-    [Rationale Document](https://mlir.llvm.org/docs/RationaleLinalgDialect) are
+    [Rationale
+    Document](https://mlir.llvm.org/docs/Rationale/RationaleLinalgDialect) are
     are also available and should be read first before going in the details of
     the op semantics.
   }];
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
index 8061568078b68..f89965a96857f 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
@@ -22,6 +22,7 @@
 #include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Types.h"
+#include "mlir/Interfaces/CopyOpInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Interfaces/ViewLikeInterface.h"
 #include "mlir/Support/LLVM.h"
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
index 85fdd1e3f34eb..9cda61ca80b74 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
@@ -17,6 +17,7 @@
 include "mlir/Dialect/Affine/IR/AffineOpsBase.td"
 include "mlir/Dialect/Linalg/IR/LinalgBase.td"
 include "mlir/Dialect/Linalg/IR/LinalgStructuredOpsInterface.td"
+include "mlir/Interfaces/CopyOpInterface.td"
 
 // The Linalg `NInputs` trait provides the API for ops that are known
 // to have a specified number of inputs, all passed as operands.
@@ -56,7 +57,11 @@ class LinalgStructured_Op<string mnemonic, list<OpTrait> props>
 //===----------------------------------------------------------------------===//
 // At the moment these are not declarative and require a bunch of C++ code.
 // In the future, these should be migrated to a declarative specification.
-def CopyOp : LinalgStructured_Op<"copy", [NInputs<1>, NOutputs<1>]> {
+def CopyOp : LinalgStructured_Op<"copy", [
+    CopyOpInterface,
+    NInputs<1>,
+    NOutputs<1>
+  ]> {
   let description = [{
     Copies the data in the input view into the output view.
 
@@ -114,8 +119,8 @@ def CopyOp : LinalgStructured_Op<"copy", [NInputs<1>, NOutputs<1>]> {
     OptionalAttr<AffineMapAttr>:$inputPermutation,
     OptionalAttr<AffineMapAttr>:$outputPermutation);
 
-  // TODO(ntv) this should go away once the usage of OptionalAttr triggers
-  // emission of builders with default arguments left unspecified.
+  // TODO: this should go away once the usage of OptionalAttr triggers emission
+  // of builders with default arguments left unspecified.
   let builders = [OpBuilder<
     "OpBuilder &builder, OperationState &result, Value input, Value output", [{
     return build(
@@ -141,6 +146,9 @@ def CopyOp : LinalgStructured_Op<"copy", [NInputs<1>, NOutputs<1>]> {
           extractOrIdentityMap(maybeInputMap, inputRank, context),
           extractOrIdentityMap(maybeOutputMap, outputRank, context)};
     }
+
+    Value getSource() { return input();}
+    Value getTarget() { return output(); }
   }];
   let verifier = [{ return ::verify(*this); }];
 
@@ -297,8 +305,8 @@ def ConvOp : PoolingBase_Op<"conv", [NInputs<2>, NOutputs<1>]> {
                    OptionalAttr<I64ElementsAttr>:$padding);
 
   let extraClassDeclaration = commonUtils # [{
-    // TODO(ntv) extend to support more than 1 dimensions and potentially
-    // grouping too.
+    // TODO: extend to support more than 1 dimensions and potentially grouping
+    // too.
     unsigned getNumBatchDimensions() { return 1; }
 
     unsigned getNumInputFeatureDimensions() { return 1; }
@@ -562,10 +570,10 @@ def GenericOp : GenericOpBase<"generic"> {
     And can be reused in multiple places as:
       ```mlir
       linalg.generic #matmul_trait %A, %B, %C [other-attributes] {
-        (%a: f32, %b: f32, %c: f32) :
+        ^bb0(%a: f32, %b: f32, %c: f32) :
           %d = mulf %a, %b: f32
           %e = addf %c, %d: f32
-          linalg_yield %e : f32
+          linalg.yield %e : f32
       } : memref<?x?xf32, stride_specification>,
           memref<?x?xf32, stride_specification>,
           memref<?x?xf32, stride_specification>
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOpsInterface.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOpsInterface.td
index b03001c9b8e96..82882b083b2d8 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOpsInterface.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOpsInterface.td
@@ -199,11 +199,11 @@ def LinalgStructuredInterface : OpInterface<"LinalgOp"> {
       "Operation *", "clone",
       (ins "OpBuilder &":$b, "Location":$loc, "ValueRange":$operands), [{
         BlockAndValueMapping map;
-        unsigned numRegions = op.getOperation()->getNumRegions();
-        Operation *res = create(b, loc, operands, op.getAttrs());
+        unsigned numRegions = $_op.getOperation()->getNumRegions();
+        Operation *res = create(b, loc, operands, $_op.getAttrs());
         assert(res->getNumRegions() == numRegions && "inconsistent # regions");
         for (unsigned ridx = 0; ridx < numRegions; ++ridx)
-          op.getOperation()->getRegion(ridx).cloneInto(
+          $_op.getOperation()->getRegion(ridx).cloneInto(
             &res->getRegion(ridx), map);
         return res;
       }]
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgTraits.h b/mlir/include/mlir/Dialect/Linalg/IR/LinalgTraits.h
index 4ab547be20197..8dda7d0a1445f 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgTraits.h
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgTraits.h
@@ -246,8 +246,8 @@ class StructuredOpTraits
         cast<ConcreteType>(this->getOperation()).referenceIterators();
 
     // If there is no reference, this must be a generic op.
-    // TODO(ntv): Traits are used to define ops. Split into cpp to avoid
-    // cyclic dependency.
+    // TODO: Traits are used to define ops. Split into cpp to avoid cyclic
+    // dependency.
     auto name = this->getOperation()->getName().getStringRef();
     if (!maybeReferenceIteratorTypes && name != "generic" &&
         name != "indexed_generic") {
@@ -263,8 +263,8 @@ class StructuredOpTraits
                                        return StringAttr::get(str, ctx);
                                      });
     auto attr = ArrayAttr::get(llvm::to_vector<4>(attrRange), ctx);
-    // TODO(ntv): Need to memoize this. Can't just store as an attribute atm as
-    // it will impact parser, printer and tests.
+    // TODO: Need to memoize this. Can't just store as an attribute atm as it
+    // will impact parser, printer and tests.
     // this->getOperation()->setAttr("iterator_types", attr);
     return attr;
   }
@@ -301,8 +301,8 @@ class StructuredOpTraits
         });
     SmallVector<Attribute, 4> attrs{attrRange.begin(), attrRange.end()};
     auto attr = ArrayAttr::get(attrs, ctx);
-    // TODO(ntv): Need to memoize this. Can't just store as an attribute atm as
-    // it will impact parser, printer and tests.
+    // TODO: Need to memoize this. Can't just store as an attribute atm as it
+    // will impact parser, printer and tests.
     // this->getOperation()->setAttr("indexing_maps", attr);
     return attr;
   }
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgTypes.h b/mlir/include/mlir/Dialect/Linalg/IR/LinalgTypes.h
index 6f93d2184709b..9f23e64e7c5d0 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgTypes.h
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgTypes.h
@@ -32,7 +32,7 @@ enum LinalgTypes {
 ///      %0 = linalg.range %arg0:%arg1:%arg2 : !linalg.range
 ///    }
 /// ```
-class RangeType : public Type::TypeBase<RangeType, Type> {
+class RangeType : public Type::TypeBase<RangeType, Type, TypeStorage> {
 public:
   // Used for generic hooks in TypeBase.
   using Base::Base;
diff --git a/mlir/include/mlir/Dialect/Linalg/Passes.h b/mlir/include/mlir/Dialect/Linalg/Passes.h
index 8a274ed48dc55..a5c09b3f75b7c 100644
--- a/mlir/include/mlir/Dialect/Linalg/Passes.h
+++ b/mlir/include/mlir/Dialect/Linalg/Passes.h
@@ -36,7 +36,7 @@ std::unique_ptr<OperationPass<FuncOp>>
 createLinalgTilingToParallelLoopsPass(ArrayRef<int64_t> tileSizes = {});
 
 std::unique_ptr<OperationPass<FuncOp>>
-createLinalgPromotionPass(bool dynamicBuffers);
+createLinalgPromotionPass(bool dynamicBuffers, bool useAlloca);
 std::unique_ptr<OperationPass<FuncOp>> createLinalgPromotionPass();
 
 /// Create a pass to convert Linalg operations to scf.for loops and
diff --git a/mlir/include/mlir/Dialect/Linalg/Passes.td b/mlir/include/mlir/Dialect/Linalg/Passes.td
index 1fc7fa5bf729e..11f12ad30eb6c 100644
--- a/mlir/include/mlir/Dialect/Linalg/Passes.td
+++ b/mlir/include/mlir/Dialect/Linalg/Passes.td
@@ -61,7 +61,9 @@ def LinalgPromotion : FunctionPass<"linalg-promote-subviews"> {
   let constructor = "mlir::createLinalgPromotionPass()";
   let options = [
     Option<"dynamicBuffers", "test-promote-dynamic", "bool",
-           /*default=*/"false", "Test generation of dynamic promoted buffers">
+           /*default=*/"false", "Test generation of dynamic promoted buffers">,
+    Option<"useAlloca", "test-use-alloca", "bool",
+           /*default=*/"false", "Test generation of alloca'ed buffers.">
   ];
 }
 
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index b7be5d86f50cf..65f1bbb833ba9 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -122,6 +122,12 @@ struct LinalgPromotionOptions {
     alignment = align;
     return *this;
   }
+  /// Use alloca with the default allocation scheme.
+  bool useAlloca = false;
+  LinalgPromotionOptions &setUseAlloca(bool use) {
+    useAlloca = use;
+    return *this;
+  }
   /// Callback function to do the allocation of the promoted buffer. If None,
   /// then the default allocation scheme of allocating a memref<?xi8> buffer
   /// followed by a view operation is used.
@@ -134,7 +140,6 @@ struct LinalgPromotionOptions {
     deallocationFn = deallocFn;
     return *this;
   }
-
   /// Callback function to do the copy of data to and from the promoted
   /// subview. If None then a linalg.copy is used.
   Optional<CopyCallbackFn> copyInFn = None;
diff --git a/mlir/include/mlir/Dialect/SCF/Passes.h b/mlir/include/mlir/Dialect/SCF/Passes.h
index e88297f07cf2e..df6037874f2b4 100644
--- a/mlir/include/mlir/Dialect/SCF/Passes.h
+++ b/mlir/include/mlir/Dialect/SCF/Passes.h
@@ -20,6 +20,10 @@ namespace mlir {
 
 class Pass;
 
+/// Creates a pass that specializes for loop for unrolling and
+/// vectorization.
+std::unique_ptr<Pass> createForLoopSpecializationPass();
+
 /// Creates a loop fusion pass which fuses parallel loops.
 std::unique_ptr<Pass> createParallelLoopFusionPass();
 
diff --git a/mlir/include/mlir/Dialect/SCF/Passes.td b/mlir/include/mlir/Dialect/SCF/Passes.td
index 5e6c8c1dd46fb..483d0ba7c7be0 100644
--- a/mlir/include/mlir/Dialect/SCF/Passes.td
+++ b/mlir/include/mlir/Dialect/SCF/Passes.td
@@ -1,4 +1,4 @@
-//===-- Passes.td - Loop pass definition file --------------*- tablegen -*-===//
+//===-- Passes.td - SCF pass definition file ---------------*- tablegen -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -11,18 +11,24 @@
 
 include "mlir/Pass/PassBase.td"
 
-def LoopParallelLoopFusion : Pass<"parallel-loop-fusion"> {
+def SCFForLoopSpecialization
+    : FunctionPass<"for-loop-specialization"> {
+  let summary = "Specialize `for` loops for vectorization";
+  let constructor = "mlir::createForLoopSpecializationPass()";
+}
+
+def SCFParallelLoopFusion : Pass<"parallel-loop-fusion"> {
   let summary = "Fuse adjacent parallel loops";
   let constructor = "mlir::createParallelLoopFusionPass()";
 }
 
-def LoopParallelLoopSpecialization
+def SCFParallelLoopSpecialization
     : FunctionPass<"parallel-loop-specialization"> {
   let summary = "Specialize parallel loops for vectorization";
   let constructor = "mlir::createParallelLoopSpecializationPass()";
 }
 
-def LoopParallelLoopTiling : FunctionPass<"parallel-loop-tiling"> {
+def SCFParallelLoopTiling : FunctionPass<"parallel-loop-tiling"> {
   let summary = "Tile parallel loops";
   let constructor = "mlir::createParallelLoopTilingPass()";
   let options = [
diff --git a/mlir/include/mlir/Dialect/SCF/SCFOps.td b/mlir/include/mlir/Dialect/SCF/SCFOps.td
index 420583161cbe5..c859cb794e850 100644
--- a/mlir/include/mlir/Dialect/SCF/SCFOps.td
+++ b/mlir/include/mlir/Dialect/SCF/SCFOps.td
@@ -183,7 +183,8 @@ def ForOp : SCF_Op<"for",
 
 def IfOp : SCF_Op<"if",
       [DeclareOpInterfaceMethods<RegionBranchOpInterface>,
-       SingleBlockImplicitTerminator<"YieldOp">, RecursiveSideEffects]> {
+       SingleBlockImplicitTerminator<"YieldOp">, RecursiveSideEffects,
+       NoRegionArguments]> {
   let summary = "if-then-else operation";
   let description = [{
     The `scf.if` operation represents an if-then-else construct for
@@ -346,6 +347,8 @@ def ParallelOp : SCF_Op<"parallel",
     unsigned getNumLoops() { return step().size(); }
     unsigned getNumReductions() { return initVals().size(); }
   }];
+
+  let hasCanonicalizer = 1;
 }
 
 def ReduceOp : SCF_Op<"reduce", [HasParent<"ParallelOp">]> {
diff --git a/mlir/include/mlir/Dialect/SPIRV/Passes.h b/mlir/include/mlir/Dialect/SPIRV/Passes.h
index afc60805f75e8..df516430be52c 100644
--- a/mlir/include/mlir/Dialect/SPIRV/Passes.h
+++ b/mlir/include/mlir/Dialect/SPIRV/Passes.h
@@ -46,6 +46,10 @@ createUpdateVersionCapabilityExtensionPass();
 ///    functions using the specification in the `spv.entry_point_abi` attribute.
 std::unique_ptr<OperationPass<spirv::ModuleOp>> createLowerABIAttributesPass();
 
+/// Creates an operation pass that rewrites sequential chains of
+/// spv.CompositeInsert into spv.CompositeConstruct.
+std::unique_ptr<OperationPass<spirv::ModuleOp>> createRewriteInsertsPass();
+
 } // namespace spirv
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/SPIRV/Passes.td b/mlir/include/mlir/Dialect/SPIRV/Passes.td
index e8972f519917e..93a3516fc2b33 100644
--- a/mlir/include/mlir/Dialect/SPIRV/Passes.td
+++ b/mlir/include/mlir/Dialect/SPIRV/Passes.td
@@ -22,6 +22,12 @@ def SPIRVLowerABIAttributes : Pass<"spirv-lower-abi-attrs", "spirv::ModuleOp"> {
   let constructor = "mlir::spirv::createLowerABIAttributesPass()";
 }
 
+def SPIRVRewriteInsertsPass : Pass<"spirv-rewrite-inserts", "spirv::ModuleOp"> {
+  let summary = "Rewrite sequential chains of spv.CompositeInsert operations into "
+                "spv.CompositeConstruct operations";
+  let constructor = "mlir::spirv::createRewriteInsertsPass()";
+}
+
 def SPIRVUpdateVCE : Pass<"spirv-update-vce", "spirv::ModuleOp"> {
   let summary = "Deduce and attach minimal (version, capabilities, extensions) "
                 "requirements to spv.module ops";
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVBase.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVBase.td
index e64abd70333de..f114c878569d6 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVBase.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVBase.td
@@ -162,7 +162,7 @@ class Extension<list<StrEnumAttrCase> extensions> : Availability {
     AND (`Extension::C`) AND (`Extension::D` OR `Extension::E`) is enabled.
   }];
 
-  // TODO(antiagainst): Returning SmallVector<ArrayRef<...>> is not recommended.
+  // TODO: Returning SmallVector<ArrayRef<...>> is not recommended.
   // Find a better way for this.
   let queryFnRetType = "::llvm::SmallVector<::llvm::ArrayRef<"
                           "::mlir::spirv::Extension>, 1>";
@@ -226,9 +226,8 @@ class Capability<list<I32EnumAttrCase> capabilities> : Availability {
   let instance = "ref";
 }
 
-// TODO(antiagainst): the following interfaces definitions are duplicating with
-// the above. Remove them once we are able to support dialect-specific contents
-// in ODS.
+// TODO: the following interfaces definitions are duplicating with the above.
+// Remove them once we are able to support dialect-specific contents in ODS.
 def QueryMinVersionInterface : OpInterface<"QueryMinVersionInterface"> {
   let methods = [InterfaceMethod<"", "::mlir::spirv::Version", "getMinVersion">];
 }
@@ -3061,7 +3060,7 @@ def SPV_IntVec4 : SPV_Vec4<SPV_Integer>;
 def SPV_IOrUIVec4 : SPV_Vec4<SPV_SignlessOrUnsignedInt>;
 def SPV_Int32Vec4 : SPV_Vec4<AnyI32>;
 
-// TODO(ravishankarm): From 1.4, this should also include Composite type.
+// TODO: From 1.4, this should also include Composite type.
 def SPV_SelectType : AnyTypeOf<[SPV_Scalar, SPV_Vector, SPV_AnyPtr]>;
 
 //===----------------------------------------------------------------------===//
@@ -3135,12 +3134,14 @@ def SPV_OC_OpFunctionCall              : I32EnumAttrCase<"OpFunctionCall", 57>;
 def SPV_OC_OpVariable                  : I32EnumAttrCase<"OpVariable", 59>;
 def SPV_OC_OpLoad                      : I32EnumAttrCase<"OpLoad", 61>;
 def SPV_OC_OpStore                     : I32EnumAttrCase<"OpStore", 62>;
+def SPV_OC_OpCopyMemory                : I32EnumAttrCase<"OpCopyMemory", 63>;
 def SPV_OC_OpAccessChain               : I32EnumAttrCase<"OpAccessChain", 65>;
 def SPV_OC_OpDecorate                  : I32EnumAttrCase<"OpDecorate", 71>;
 def SPV_OC_OpMemberDecorate            : I32EnumAttrCase<"OpMemberDecorate", 72>;
 def SPV_OC_OpCompositeConstruct        : I32EnumAttrCase<"OpCompositeConstruct", 80>;
 def SPV_OC_OpCompositeExtract          : I32EnumAttrCase<"OpCompositeExtract", 81>;
 def SPV_OC_OpCompositeInsert           : I32EnumAttrCase<"OpCompositeInsert", 82>;
+def SPV_OC_OpTranspose                 : I32EnumAttrCase<"OpTranspose", 84>;
 def SPV_OC_OpConvertFToU               : I32EnumAttrCase<"OpConvertFToU", 109>;
 def SPV_OC_OpConvertFToS               : I32EnumAttrCase<"OpConvertFToS", 110>;
 def SPV_OC_OpConvertSToF               : I32EnumAttrCase<"OpConvertSToF", 111>;
@@ -3165,6 +3166,7 @@ def SPV_OC_OpSMod                      : I32EnumAttrCase<"OpSMod", 139>;
 def SPV_OC_OpFRem                      : I32EnumAttrCase<"OpFRem", 140>;
 def SPV_OC_OpFMod                      : I32EnumAttrCase<"OpFMod", 141>;
 def SPV_OC_OpMatrixTimesScalar         : I32EnumAttrCase<"OpMatrixTimesScalar", 143>;
+def SPV_OC_OpMatrixTimesMatrix         : I32EnumAttrCase<"OpMatrixTimesMatrix", 146>;
 def SPV_OC_OpLogicalEqual              : I32EnumAttrCase<"OpLogicalEqual", 164>;
 def SPV_OC_OpLogicalNotEqual           : I32EnumAttrCase<"OpLogicalNotEqual", 165>;
 def SPV_OC_OpLogicalOr                 : I32EnumAttrCase<"OpLogicalOr", 166>;
@@ -3263,46 +3265,47 @@ def SPV_OpcodeAttr :
       SPV_OC_OpConstantNull, SPV_OC_OpSpecConstantTrue, SPV_OC_OpSpecConstantFalse,
       SPV_OC_OpSpecConstant, SPV_OC_OpSpecConstantComposite, SPV_OC_OpFunction,
       SPV_OC_OpFunctionParameter, SPV_OC_OpFunctionEnd, SPV_OC_OpFunctionCall,
-      SPV_OC_OpVariable, SPV_OC_OpLoad, SPV_OC_OpStore, SPV_OC_OpAccessChain,
-      SPV_OC_OpDecorate, SPV_OC_OpMemberDecorate, SPV_OC_OpCompositeConstruct,
-      SPV_OC_OpCompositeExtract, SPV_OC_OpCompositeInsert, SPV_OC_OpConvertFToU,
+      SPV_OC_OpVariable, SPV_OC_OpLoad, SPV_OC_OpStore, SPV_OC_OpCopyMemory,
+      SPV_OC_OpAccessChain, SPV_OC_OpDecorate, SPV_OC_OpMemberDecorate,
+      SPV_OC_OpCompositeConstruct, SPV_OC_OpCompositeExtract,
+      SPV_OC_OpCompositeInsert, SPV_OC_OpTranspose, SPV_OC_OpConvertFToU,
       SPV_OC_OpConvertFToS, SPV_OC_OpConvertSToF, SPV_OC_OpConvertUToF,
       SPV_OC_OpUConvert, SPV_OC_OpSConvert, SPV_OC_OpFConvert, SPV_OC_OpBitcast,
       SPV_OC_OpFNegate, SPV_OC_OpIAdd, SPV_OC_OpFAdd, SPV_OC_OpISub, SPV_OC_OpFSub,
       SPV_OC_OpIMul, SPV_OC_OpFMul, SPV_OC_OpUDiv, SPV_OC_OpSDiv, SPV_OC_OpFDiv,
       SPV_OC_OpUMod, SPV_OC_OpSRem, SPV_OC_OpSMod, SPV_OC_OpFRem, SPV_OC_OpFMod,
-      SPV_OC_OpMatrixTimesScalar, SPV_OC_OpLogicalEqual, SPV_OC_OpLogicalNotEqual,
-      SPV_OC_OpLogicalOr, SPV_OC_OpLogicalAnd, SPV_OC_OpLogicalNot, SPV_OC_OpSelect,
-      SPV_OC_OpIEqual, SPV_OC_OpINotEqual, SPV_OC_OpUGreaterThan,
-      SPV_OC_OpSGreaterThan, SPV_OC_OpUGreaterThanEqual, SPV_OC_OpSGreaterThanEqual,
-      SPV_OC_OpULessThan, SPV_OC_OpSLessThan, SPV_OC_OpULessThanEqual,
-      SPV_OC_OpSLessThanEqual, SPV_OC_OpFOrdEqual, SPV_OC_OpFUnordEqual,
-      SPV_OC_OpFOrdNotEqual, SPV_OC_OpFUnordNotEqual, SPV_OC_OpFOrdLessThan,
-      SPV_OC_OpFUnordLessThan, SPV_OC_OpFOrdGreaterThan, SPV_OC_OpFUnordGreaterThan,
-      SPV_OC_OpFOrdLessThanEqual, SPV_OC_OpFUnordLessThanEqual,
-      SPV_OC_OpFOrdGreaterThanEqual, SPV_OC_OpFUnordGreaterThanEqual,
-      SPV_OC_OpShiftRightLogical, SPV_OC_OpShiftRightArithmetic,
-      SPV_OC_OpShiftLeftLogical, SPV_OC_OpBitwiseOr, SPV_OC_OpBitwiseXor,
-      SPV_OC_OpBitwiseAnd, SPV_OC_OpNot, SPV_OC_OpBitFieldInsert,
-      SPV_OC_OpBitFieldSExtract, SPV_OC_OpBitFieldUExtract, SPV_OC_OpBitReverse,
-      SPV_OC_OpBitCount, SPV_OC_OpControlBarrier, SPV_OC_OpMemoryBarrier,
-      SPV_OC_OpAtomicCompareExchangeWeak, SPV_OC_OpAtomicIIncrement,
-      SPV_OC_OpAtomicIDecrement, SPV_OC_OpAtomicIAdd, SPV_OC_OpAtomicISub,
-      SPV_OC_OpAtomicSMin, SPV_OC_OpAtomicUMin, SPV_OC_OpAtomicSMax,
-      SPV_OC_OpAtomicUMax, SPV_OC_OpAtomicAnd, SPV_OC_OpAtomicOr, SPV_OC_OpAtomicXor,
-      SPV_OC_OpPhi, SPV_OC_OpLoopMerge, SPV_OC_OpSelectionMerge, SPV_OC_OpLabel,
-      SPV_OC_OpBranch, SPV_OC_OpBranchConditional, SPV_OC_OpReturn,
-      SPV_OC_OpReturnValue, SPV_OC_OpUnreachable, SPV_OC_OpNoLine,
-      SPV_OC_OpModuleProcessed, SPV_OC_OpGroupNonUniformElect,
-      SPV_OC_OpGroupNonUniformBallot, SPV_OC_OpGroupNonUniformIAdd,
-      SPV_OC_OpGroupNonUniformFAdd, SPV_OC_OpGroupNonUniformIMul,
-      SPV_OC_OpGroupNonUniformFMul, SPV_OC_OpGroupNonUniformSMin,
-      SPV_OC_OpGroupNonUniformUMin, SPV_OC_OpGroupNonUniformFMin,
-      SPV_OC_OpGroupNonUniformSMax, SPV_OC_OpGroupNonUniformUMax,
-      SPV_OC_OpGroupNonUniformFMax, SPV_OC_OpSubgroupBallotKHR,
-      SPV_OC_OpTypeCooperativeMatrixNV, SPV_OC_OpCooperativeMatrixLoadNV,
-      SPV_OC_OpCooperativeMatrixStoreNV, SPV_OC_OpCooperativeMatrixMulAddNV,
-      SPV_OC_OpCooperativeMatrixLengthNV
+      SPV_OC_OpMatrixTimesScalar, SPV_OC_OpMatrixTimesMatrix, SPV_OC_OpLogicalEqual,
+      SPV_OC_OpLogicalNotEqual, SPV_OC_OpLogicalOr, SPV_OC_OpLogicalAnd,
+      SPV_OC_OpLogicalNot, SPV_OC_OpSelect, SPV_OC_OpIEqual, SPV_OC_OpINotEqual,
+      SPV_OC_OpUGreaterThan, SPV_OC_OpSGreaterThan, SPV_OC_OpUGreaterThanEqual,
+      SPV_OC_OpSGreaterThanEqual, SPV_OC_OpULessThan, SPV_OC_OpSLessThan,
+      SPV_OC_OpULessThanEqual, SPV_OC_OpSLessThanEqual, SPV_OC_OpFOrdEqual,
+      SPV_OC_OpFUnordEqual, SPV_OC_OpFOrdNotEqual, SPV_OC_OpFUnordNotEqual,
+      SPV_OC_OpFOrdLessThan, SPV_OC_OpFUnordLessThan, SPV_OC_OpFOrdGreaterThan,
+      SPV_OC_OpFUnordGreaterThan, SPV_OC_OpFOrdLessThanEqual,
+      SPV_OC_OpFUnordLessThanEqual, SPV_OC_OpFOrdGreaterThanEqual,
+      SPV_OC_OpFUnordGreaterThanEqual, SPV_OC_OpShiftRightLogical,
+      SPV_OC_OpShiftRightArithmetic, SPV_OC_OpShiftLeftLogical, SPV_OC_OpBitwiseOr,
+      SPV_OC_OpBitwiseXor, SPV_OC_OpBitwiseAnd, SPV_OC_OpNot,
+      SPV_OC_OpBitFieldInsert, SPV_OC_OpBitFieldSExtract, SPV_OC_OpBitFieldUExtract,
+      SPV_OC_OpBitReverse, SPV_OC_OpBitCount, SPV_OC_OpControlBarrier,
+      SPV_OC_OpMemoryBarrier, SPV_OC_OpAtomicCompareExchangeWeak,
+      SPV_OC_OpAtomicIIncrement, SPV_OC_OpAtomicIDecrement, SPV_OC_OpAtomicIAdd,
+      SPV_OC_OpAtomicISub, SPV_OC_OpAtomicSMin, SPV_OC_OpAtomicUMin,
+      SPV_OC_OpAtomicSMax, SPV_OC_OpAtomicUMax, SPV_OC_OpAtomicAnd,
+      SPV_OC_OpAtomicOr, SPV_OC_OpAtomicXor, SPV_OC_OpPhi, SPV_OC_OpLoopMerge,
+      SPV_OC_OpSelectionMerge, SPV_OC_OpLabel, SPV_OC_OpBranch,
+      SPV_OC_OpBranchConditional, SPV_OC_OpReturn, SPV_OC_OpReturnValue,
+      SPV_OC_OpUnreachable, SPV_OC_OpNoLine, SPV_OC_OpModuleProcessed,
+      SPV_OC_OpGroupNonUniformElect, SPV_OC_OpGroupNonUniformBallot,
+      SPV_OC_OpGroupNonUniformIAdd, SPV_OC_OpGroupNonUniformFAdd,
+      SPV_OC_OpGroupNonUniformIMul, SPV_OC_OpGroupNonUniformFMul,
+      SPV_OC_OpGroupNonUniformSMin, SPV_OC_OpGroupNonUniformUMin,
+      SPV_OC_OpGroupNonUniformFMin, SPV_OC_OpGroupNonUniformSMax,
+      SPV_OC_OpGroupNonUniformUMax, SPV_OC_OpGroupNonUniformFMax,
+      SPV_OC_OpSubgroupBallotKHR, SPV_OC_OpTypeCooperativeMatrixNV,
+      SPV_OC_OpCooperativeMatrixLoadNV, SPV_OC_OpCooperativeMatrixStoreNV,
+      SPV_OC_OpCooperativeMatrixMulAddNV, SPV_OC_OpCooperativeMatrixLengthNV
     ]>;
 
 // End opcode section. Generated from SPIR-V spec; DO NOT MODIFY!
@@ -3314,9 +3317,9 @@ def SPV_OpcodeAttr :
 // Base class for all SPIR-V ops.
 class SPV_Op<string mnemonic, list<OpTrait> traits = []> :
     Op<SPIRV_Dialect, mnemonic, !listconcat(traits, [
-         // TODO(antiagainst): We don't need all of the following traits for
-         // every op; only the suitable ones should be added automatically
-         // after ODS supports dialect-specific contents.
+         // TODO: We don't need all of the following traits for every op; only
+         // the suitable ones should be added automatically after ODS supports
+         // dialect-specific contents.
          DeclareOpInterfaceMethods<QueryMinVersionInterface>,
          DeclareOpInterfaceMethods<QueryMaxVersionInterface>,
          DeclareOpInterfaceMethods<QueryExtensionInterface>,
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVLowering.h b/mlir/include/mlir/Dialect/SPIRV/SPIRVLowering.h
index f0a429941fb35..02a1f0861da81 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVLowering.h
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVLowering.h
@@ -35,8 +35,8 @@ namespace mlir {
 /// satisfy shader interface requirements: shader interface types must be
 /// pointers to structs.
 ///
-/// TODO(antiagainst): We might want to introduce a way to control how
-/// unsupported bitwidth are handled and explicitly fail if wanted.
+/// TODO: We might want to introduce a way to control how unsupported bitwidth
+/// are handled and explicitly fail if wanted.
 class SPIRVTypeConverter : public TypeConverter {
 public:
   explicit SPIRVTypeConverter(spirv::TargetEnvAttr targetAttr);
@@ -120,8 +120,8 @@ Value getBuiltinVariableValue(Operation *op, BuiltIn builtin,
 /// Performs the index computation to get to the element at `indices` of the
 /// memory pointed to by `basePtr`, using the layout map of `baseType`.
 
-// TODO(ravishankarm) : This method assumes that the `baseType` is a MemRefType
-// with AffineMap that has static strides. Extend to handle dynamic strides.
+// TODO: This method assumes that the `baseType` is a MemRefType with AffineMap
+// that has static strides. Extend to handle dynamic strides.
 spirv::AccessChainOp getElementPtr(SPIRVTypeConverter &typeConverter,
                                    MemRefType baseType, Value basePtr,
                                    ValueRange indices, Location loc,
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVMatrixOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVMatrixOps.td
index c71e0cd54e99c..70479e743b0dc 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVMatrixOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVMatrixOps.td
@@ -12,10 +12,65 @@
 
 #ifndef SPIRV_MATRIX_OPS
 #define SPIRV_MATRIX_OPS
+include "mlir/Interfaces/SideEffectInterfaces.td"
 
 // -----
 
-def SPV_MatrixTimesScalarOp : SPV_Op<"MatrixTimesScalar", []> {
+def SPV_MatrixTimesMatrixOp : SPV_Op<"MatrixTimesMatrix", [NoSideEffect]> {
+  let summary = "Linear-algebraic multiply of LeftMatrix X RightMatrix.";
+
+  let description = [{
+    Result Type must be an OpTypeMatrix whose Column Type is a vector of
+    floating-point type.
+
+    LeftMatrix must be a matrix whose Column Type is the same as the Column
+    Type in Result Type.
+
+    RightMatrix must be a matrix with the same Component Type as the
+    Component Type in Result Type. Its number of columns must equal the
+    number of columns in Result Type. Its columns must have the same number
+    of components as the number of columns in LeftMatrix.
+
+    <!-- End of AutoGen section -->
+
+    ```
+    matrix-times-matrix-op ::= ssa-id `=` `spv.MatrixTimesMatrix` ssa-use,
+    ssa-use `:` matrix-type `,` matrix-type `->` matrix-type
+    ```mlir
+
+    #### Example:
+
+    ```
+    %0 = spv.MatrixTimesMatrix %matrix_1, %matrix_2 :
+        !spv.matrix<4 x vector<3xf32>>, !spv.matrix<3 x vector<4xf32>> ->
+        !spv.matrix<4 x vector<4xf32>>
+    ```
+  }];
+
+  let availability = [
+    MinVersion<SPV_V_1_0>,
+    MaxVersion<SPV_V_1_5>,
+    Extension<[]>,
+    Capability<[SPV_C_Matrix]>
+  ];
+
+  let arguments = (ins
+    SPV_AnyMatrix:$leftmatrix,
+    SPV_AnyMatrix:$rightmatrix
+  );
+
+  let results = (outs
+    SPV_AnyMatrix:$result
+  );
+  let assemblyFormat = [{
+    operands attr-dict `:` type($leftmatrix) `,` type($rightmatrix) `->` type($result)
+  }];
+  let verifier = [{ return verifyMatrixTimesMatrix(*this); }];
+}
+
+// -----
+
+def SPV_MatrixTimesScalarOp : SPV_Op<"MatrixTimesScalar", [NoSideEffect]> {
   let summary = "Scale a floating-point matrix.";
 
   let description = [{
@@ -45,6 +100,13 @@ def SPV_MatrixTimesScalarOp : SPV_Op<"MatrixTimesScalar", []> {
     ```
   }];
 
+  let availability = [
+    MinVersion<SPV_V_1_0>,
+    MaxVersion<SPV_V_1_5>,
+    Extension<[]>,
+    Capability<[SPV_C_Matrix]>
+  ];
+
   let arguments = (ins
     SPV_AnyMatrix:$matrix,
     SPV_Float:$scalar
@@ -54,8 +116,8 @@ def SPV_MatrixTimesScalarOp : SPV_Op<"MatrixTimesScalar", []> {
     SPV_AnyMatrix:$result
   );
 
-  // TODO (Hazem): we need just one matrix type given that the input and result
-  // are the same and the scalar's type can be deduced from it.
+  // TODO: we need just one matrix type given that the input and result are the
+  // same and the scalar's type can be deduced from it.
   let assemblyFormat = [{
     operands attr-dict `:` type($matrix) `,` type($scalar) `->` type($result)
   }];
@@ -72,4 +134,58 @@ def SPV_MatrixTimesScalarOp : SPV_Op<"MatrixTimesScalar", []> {
 
 // -----
 
-#endif // SPIRV_MATRIX_OPS
\ No newline at end of file
+def SPV_TransposeOp : SPV_Op<"Transpose", [NoSideEffect]> {
+  let summary = "Transpose a matrix.";
+
+  let description = [{
+    Result Type must be an OpTypeMatrix.
+
+    Matrix must be an object of type OpTypeMatrix. The number of columns and
+    the column size of Matrix must be the reverse of those in Result Type.
+    The types of the scalar components in Matrix and Result Type must be the
+    same.
+
+    Matrix must have of type of OpTypeMatrix.
+
+    <!-- End of AutoGen section -->
+
+    ```
+    transpose-op ::= ssa-id `=` `spv.Transpose` ssa-use `:` matrix-type `->`
+    matrix-type
+
+    ```mlir
+
+    #### Example:
+
+    ```
+    %0 = spv.Transpose %matrix: !spv.matrix<2 x vector<3xf32>> ->
+    !spv.matrix<3 x vector<2xf32>>
+
+    ```
+  }];
+
+  let availability = [
+    MinVersion<SPV_V_1_0>,
+    MaxVersion<SPV_V_1_5>,
+    Extension<[]>,
+    Capability<[SPV_C_Matrix]>
+  ];
+
+  let arguments = (ins
+    SPV_AnyMatrix:$matrix
+  );
+
+  let results = (outs
+    SPV_AnyMatrix:$result
+  );
+
+  let assemblyFormat = [{
+    operands attr-dict `:` type($matrix) `->` type($result)
+  }];
+
+  let verifier = [{ return verifyTranspose(*this); }];
+}
+
+// -----
+
+#endif // SPIRV_MATRIX_OPS
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVModule.h b/mlir/include/mlir/Dialect/SPIRV/SPIRVModule.h
new file mode 100644
index 0000000000000..a53331eda4fa0
--- /dev/null
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVModule.h
@@ -0,0 +1,29 @@
+//===- SPIRVModule.h - SPIR-V Module Utilities ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_SPIRV_SPIRVMODULE_H
+#define MLIR_DIALECT_SPIRV_SPIRVMODULE_H
+
+#include "mlir/Dialect/SPIRV/SPIRVOps.h"
+#include "mlir/IR/OwningOpRefBase.h"
+
+namespace mlir {
+namespace spirv {
+
+/// This class acts as an owning reference to a SPIR-V module, and will
+/// automatically destroy the held module on destruction if the held module
+/// is valid.
+class OwningSPIRVModuleRef : public OwningOpRefBase<spirv::ModuleOp> {
+public:
+  using OwningOpRefBase<spirv::ModuleOp>::OwningOpRefBase;
+};
+
+} // end namespace spirv
+} // end namespace mlir
+
+#endif // MLIR_DIALECT_SPIRV_SPIRVMODULE_H
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVOps.td
index 8b3a25037078b..e20bde5904bd3 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVOps.td
@@ -173,6 +173,62 @@ def SPV_ControlBarrierOp : SPV_Op<"ControlBarrier", []> {
 
 // -----
 
+def SPV_CopyMemoryOp : SPV_Op<"CopyMemory", []> {
+  let summary = [{
+    Copy from the memory pointed to by Source to the memory pointed to by
+    Target. Both operands must be non-void pointers and having the same <id>
+    Type operand in their OpTypePointer type declaration.  Matching Storage
+    Class is not required.  The amount of memory copied is the size of the
+    type pointed to. The copied type must have a fixed size; i.e., it cannot
+    be, nor include, any OpTypeRuntimeArray types.
+  }];
+
+  let description = [{
+    If present, any Memory Operands must begin with a memory operand
+    literal. If not present, it is the same as specifying the memory operand
+    None. Before version 1.4, at most one memory operands mask can be
+    provided. Starting with version 1.4 two masks can be provided, as
+    described in Memory Operands. If no masks or only one mask is present,
+    it applies to both Source and Target. If two masks are present, the
+    first applies to Target and cannot include MakePointerVisible, and the
+    second applies to Source and cannot include MakePointerAvailable.
+
+    <!-- End of AutoGen section -->
+
+    ```
+    copy-memory-op ::= `spv.CopyMemory ` storage-class ssa-use
+                       storage-class ssa-use
+                       (`[` memory-access `]` (`, [` memory-access `]`)?)?
+                       ` : ` spirv-element-type
+    ```
+
+    #### Example:
+
+    ```mlir
+    %0 = spv.Variable : !spv.ptr<f32, Function>
+    %1 = spv.Variable : !spv.ptr<f32, Function>
+    spv.CopyMemory "Function" %0, "Function" %1 : f32
+    ```
+  }];
+
+  let arguments = (ins
+    SPV_AnyPtr:$target,
+    SPV_AnyPtr:$source,
+    OptionalAttr<SPV_MemoryAccessAttr>:$memory_access,
+    OptionalAttr<I32Attr>:$alignment,
+    OptionalAttr<SPV_MemoryAccessAttr>:$source_memory_access,
+    OptionalAttr<I32Attr>:$source_alignment
+  );
+
+  let results = (outs);
+
+  let verifier = [{ return verifyCopyMemory(*this); }];
+
+  let autogenSerialization = 0;
+}
+
+// -----
+
 def SPV_ExecutionModeOp : SPV_Op<"ExecutionMode", [InModuleScope]> {
   let summary = "Declare an execution mode for an entry point.";
 
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVStructureOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVStructureOps.td
index 0c4cad17f8de6..034b7d1b09c7e 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVStructureOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVStructureOps.td
@@ -102,7 +102,7 @@ def SPV_ConstantOp : SPV_Op<"constant", [ConstantLike, NoSideEffect]> {
     %2 = spv.constant [dense<3.0> : vector<2xf32>] : !spv.array<1xvector<2xf32>>
     ```
 
-    TODO(antiagainst): support constant structs
+    TODO: support constant structs
   }];
 
   let arguments = (ins
@@ -541,7 +541,7 @@ def SPV_SpecConstantOp : SPV_Op<"specConstant", [InModuleScope, Symbol]> {
     spv.specConstant @spec_const2 spec_id(5) = 42 : i32
     ```
 
-    TODO(antiagainst): support composite spec constants with another op
+    TODO: support composite spec constants with another op
   }];
 
   let arguments = (ins
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVTypes.h b/mlir/include/mlir/Dialect/SPIRV/SPIRVTypes.h
index 95855785e31b1..b13d3c7892be5 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVTypes.h
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVTypes.h
@@ -109,6 +109,11 @@ class SPIRVType : public Type {
   /// times.
   void getCapabilities(CapabilityArrayRefVector &capabilities,
                        Optional<spirv::StorageClass> storage = llvm::None);
+
+  /// Returns the size in bytes for each type. If no size can be calculated,
+  /// returns `llvm::None`. Note that if the type has explicit layout, it is
+  /// also taken into account in calculation.
+  Optional<int64_t> getSizeInBytes();
 };
 
 // SPIR-V scalar type: bool type, integer type, floating point type.
@@ -127,6 +132,8 @@ class ScalarType : public SPIRVType {
                      Optional<spirv::StorageClass> storage = llvm::None);
   void getCapabilities(SPIRVType::CapabilityArrayRefVector &capabilities,
                        Optional<spirv::StorageClass> storage = llvm::None);
+
+  Optional<int64_t> getSizeInBytes();
 };
 
 // SPIR-V composite type: VectorType, SPIR-V ArrayType, or SPIR-V StructType.
@@ -153,6 +160,8 @@ class CompositeType : public SPIRVType {
                      Optional<spirv::StorageClass> storage = llvm::None);
   void getCapabilities(SPIRVType::CapabilityArrayRefVector &capabilities,
                        Optional<spirv::StorageClass> storage = llvm::None);
+
+  Optional<int64_t> getSizeInBytes();
 };
 
 // SPIR-V array type
@@ -181,6 +190,10 @@ class ArrayType : public Type::TypeBase<ArrayType, CompositeType,
                      Optional<spirv::StorageClass> storage = llvm::None);
   void getCapabilities(SPIRVType::CapabilityArrayRefVector &capabilities,
                        Optional<spirv::StorageClass> storage = llvm::None);
+
+  /// Returns the array size in bytes. Since array type may have an explicit
+  /// stride declaration (in bytes), we also include it in the calculation.
+  Optional<int64_t> getSizeInBytes();
 };
 
 // SPIR-V image type
@@ -216,7 +229,7 @@ class ImageType
   ImageSamplingInfo getSamplingInfo() const;
   ImageSamplerUseInfo getSamplerUseInfo() const;
   ImageFormat getImageFormat() const;
-  // TODO(ravishankarm): Add support for Access qualifier
+  // TODO: Add support for Access qualifier
 
   void getExtensions(SPIRVType::ExtensionArrayRefVector &extensions,
                      Optional<spirv::StorageClass> storage = llvm::None);
@@ -410,13 +423,23 @@ class MatrixType : public Type::TypeBase<MatrixType, CompositeType,
                                                     Type columnType,
                                                     uint32_t columnCount);
 
-  /// Returns true if the matrix elements are vectors of float elements
+  /// Returns true if the matrix elements are vectors of float elements.
   static bool isValidColumnType(Type columnType);
 
-  Type getElementType() const;
+  Type getColumnType() const;
 
+  /// Returns the number of rows.
+  unsigned getNumRows() const;
+
+  /// Returns the number of columns.
+  unsigned getNumColumns() const;
+
+  /// Returns total number of elements (rows*columns).
   unsigned getNumElements() const;
 
+  /// Returns the elements' type (i.e, single element type).
+  Type getElementType() const;
+
   void getExtensions(SPIRVType::ExtensionArrayRefVector &extensions,
                      Optional<spirv::StorageClass> storage = llvm::None);
   void getCapabilities(SPIRVType::CapabilityArrayRefVector &capabilities,
diff --git a/mlir/include/mlir/Dialect/SPIRV/Serialization.h b/mlir/include/mlir/Dialect/SPIRV/Serialization.h
index f6370a1b5ec26..2c91286ca1580 100644
--- a/mlir/include/mlir/Dialect/SPIRV/Serialization.h
+++ b/mlir/include/mlir/Dialect/SPIRV/Serialization.h
@@ -22,6 +22,7 @@ class MLIRContext;
 
 namespace spirv {
 class ModuleOp;
+class OwningSPIRVModuleRef;
 
 /// Serializes the given SPIR-V `module` and writes to `binary`. On failure,
 /// reports errors to the error handler registered with the MLIR context for
@@ -31,9 +32,10 @@ LogicalResult serialize(ModuleOp module, SmallVectorImpl<uint32_t> &binary,
 
 /// Deserializes the given SPIR-V `binary` module and creates a MLIR ModuleOp
 /// in the given `context`. Returns the ModuleOp on success; otherwise, reports
-/// errors to the error handler registered with `context` and returns
-/// llvm::None.
-Optional<ModuleOp> deserialize(ArrayRef<uint32_t> binary, MLIRContext *context);
+/// errors to the error handler registered with `context` and returns a null
+/// module.
+OwningSPIRVModuleRef deserialize(ArrayRef<uint32_t> binary,
+                                 MLIRContext *context);
 
 } // end namespace spirv
 } // end namespace mlir
diff --git a/mlir/include/mlir/Dialect/Shape/IR/Shape.h b/mlir/include/mlir/Dialect/Shape/IR/Shape.h
index b78b7304592da..62e4e0c4511fc 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/Shape.h
+++ b/mlir/include/mlir/Dialect/Shape/IR/Shape.h
@@ -39,7 +39,7 @@ enum Kind {
 } // namespace ShapeTypes
 
 /// The component type corresponding to shape, element type and attribute.
-class ComponentType : public Type::TypeBase<ComponentType, Type> {
+class ComponentType : public Type::TypeBase<ComponentType, Type, TypeStorage> {
 public:
   using Base::Base;
 
@@ -54,7 +54,7 @@ class ComponentType : public Type::TypeBase<ComponentType, Type> {
 };
 
 /// The element type of the shaped type.
-class ElementType : public Type::TypeBase<ElementType, Type> {
+class ElementType : public Type::TypeBase<ElementType, Type, TypeStorage> {
 public:
   using Base::Base;
 
@@ -69,7 +69,7 @@ class ElementType : public Type::TypeBase<ElementType, Type> {
 };
 
 /// The shape descriptor type represents rank and dimension sizes.
-class ShapeType : public Type::TypeBase<ShapeType, Type> {
+class ShapeType : public Type::TypeBase<ShapeType, Type, TypeStorage> {
 public:
   using Base::Base;
 
@@ -82,7 +82,7 @@ class ShapeType : public Type::TypeBase<ShapeType, Type> {
 };
 
 /// The type of a single dimension.
-class SizeType : public Type::TypeBase<SizeType, Type> {
+class SizeType : public Type::TypeBase<SizeType, Type, TypeStorage> {
 public:
   using Base::Base;
 
@@ -95,7 +95,8 @@ class SizeType : public Type::TypeBase<SizeType, Type> {
 };
 
 /// The ValueShape represents a (potentially unknown) runtime value and shape.
-class ValueShapeType : public Type::TypeBase<ValueShapeType, Type> {
+class ValueShapeType
+    : public Type::TypeBase<ValueShapeType, Type, TypeStorage> {
 public:
   using Base::Base;
 
@@ -111,7 +112,7 @@ class ValueShapeType : public Type::TypeBase<ValueShapeType, Type> {
 
 /// The Witness represents a runtime constraint, to be used as shape related
 /// preconditions on code execution.
-class WitnessType : public Type::TypeBase<WitnessType, Type> {
+class WitnessType : public Type::TypeBase<WitnessType, Type, TypeStorage> {
 public:
   using Base::Base;
 
diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index 49714c96b1cd8..6f263cd6db631 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -130,6 +130,10 @@ def Shape_ConstSizeOp : Shape_Op<"const_size", [
   let arguments = (ins IndexAttr:$value);
   let results = (outs Shape_SizeType:$result);
 
+  let builders = [
+    OpBuilder<"OpBuilder &builder, OperationState &result, int64_t value">
+  ];
+
   let assemblyFormat = "$value attr-dict";
   let hasFolder = 1;
 }
@@ -169,6 +173,21 @@ def Shape_FromExtentTensorOp : Shape_Op<"from_extent_tensor", [NoSideEffect]> {
   let assemblyFormat = "attr-dict $input `:` type($input)";
 }
 
+def Shape_RankOp : Shape_Op<"rank", [NoSideEffect]> {
+  let summary = "Gets the rank of a shape";
+  let description = [{
+    Returns the rank of the shape, i.e. the number of extents.
+  }];
+
+  let arguments = (ins Shape_ShapeType:$shape);
+  let results = (outs Shape_SizeType:$rank);
+
+  let assemblyFormat = "attr-dict $shape";
+
+  let hasFolder = 1;
+  let hasCanonicalizer = 1;
+}
+
 def Shape_ToExtentTensorOp : Shape_Op<"to_extent_tensor", [NoSideEffect]> {
   let summary = "Creates a dimension tensor from a shape";
   let description = [{
@@ -230,6 +249,7 @@ def Shape_IndexToSizeOp : Shape_Op<"index_to_size", [NoSideEffect]> {
   let assemblyFormat = "$arg attr-dict";
 
   let hasFolder = 1;
+  let hasCanonicalizer = 1;
 }
 
 def Shape_JoinOp : Shape_Op<"join", []> {
@@ -373,6 +393,7 @@ def Shape_SizeToIndexOp : Shape_Op<"size_to_index", [NoSideEffect]> {
   let assemblyFormat = "$arg attr-dict";
 
   let hasFolder = 1;
+  let hasCanonicalizer = 1;
 }
 
 def Shape_YieldOp : Shape_Op<"yield",
@@ -458,7 +479,7 @@ def Shape_ConcatOp : Shape_Op<"concat", []> {
 // Shape constraint related ops.
 //===----------------------------------------------------------------------===//
 
-//TODO(tpopp): Move the code below and witnesses to a different file.
+// TODO: Move the code below and witnesses to a different file.
 def Shape_AnyOp : Shape_Op<"any", [Commutative, NoSideEffect]> {
   let summary = "Return any combination of the input shapes";
   let description = [{
diff --git a/mlir/include/mlir/Dialect/StandardOps/EDSC/Builders.h b/mlir/include/mlir/Dialect/StandardOps/EDSC/Builders.h
index 5f0d6d83df99b..36df24f60c704 100644
--- a/mlir/include/mlir/Dialect/StandardOps/EDSC/Builders.h
+++ b/mlir/include/mlir/Dialect/StandardOps/EDSC/Builders.h
@@ -48,7 +48,7 @@ class BoundsCapture {
 /// MemRef. It has placeholders for non-contiguous tensors that fit within the
 /// Fortran subarray model.
 /// At the moment it can only capture a MemRef with an identity layout map.
-// TODO(ntv): Support MemRefs with layoutMaps.
+// TODO: Support MemRefs with layoutMaps.
 class MemRefBoundsCapture : public BoundsCapture {
 public:
   explicit MemRefBoundsCapture(Value v);
diff --git a/mlir/include/mlir/Dialect/StandardOps/EDSC/Intrinsics.h b/mlir/include/mlir/Dialect/StandardOps/EDSC/Intrinsics.h
index eccb980676a72..f654020a53385 100644
--- a/mlir/include/mlir/Dialect/StandardOps/EDSC/Intrinsics.h
+++ b/mlir/include/mlir/Dialect/StandardOps/EDSC/Intrinsics.h
@@ -25,6 +25,8 @@ using std_constant_float = ValueBuilder<ConstantFloatOp>;
 using std_constant_index = ValueBuilder<ConstantIndexOp>;
 using std_constant_int = ValueBuilder<ConstantIntOp>;
 using std_dealloc = OperationBuilder<DeallocOp>;
+using std_divis = ValueBuilder<SignedDivIOp>;
+using std_diviu = ValueBuilder<UnsignedDivIOp>;
 using std_dim = ValueBuilder<DimOp>;
 using std_extract_element = ValueBuilder<ExtractElementOp>;
 using std_im = ValueBuilder<ImOp>;
diff --git a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.h b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.h
index 8005ecbbdc494..0f24d74dcac28 100644
--- a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.h
+++ b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.h
@@ -21,6 +21,7 @@
 #include "mlir/Interfaces/CallInterfaces.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
+#include "mlir/Interfaces/VectorUnrollInterface.h"
 #include "mlir/Interfaces/ViewLikeInterface.h"
 
 // Pull in all enum type definitions and utility function declarations.
@@ -128,9 +129,9 @@ class ConstantIndexOp : public ConstantOp {
 //   dma_start %src[%i, %j], %dst[%k, %l], %num_elements, %tag[%idx], %stride,
 //             %num_elt_per_stride :
 //
-// TODO(mlir-team): add additional operands to allow source and destination
-// striding, and multiple stride levels.
-// TODO(andydavis) Consider replacing src/dst memref indices with view memrefs.
+// TODO: add additional operands to allow source and destination striding, and
+// multiple stride levels.
+// TODO: Consider replacing src/dst memref indices with view memrefs.
 class DmaStartOp
     : public Op<DmaStartOp, OpTrait::VariadicOperands, OpTrait::ZeroResult> {
 public:
diff --git a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
index d61c8af99a681..dd74aa1370608 100644
--- a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
+++ b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
@@ -13,10 +13,12 @@
 #ifndef STANDARD_OPS
 #define STANDARD_OPS
 
+include "mlir/Dialect/StandardOps/IR/StandardOpsBase.td"
 include "mlir/IR/OpAsmInterface.td"
 include "mlir/Interfaces/CallInterfaces.td"
 include "mlir/Interfaces/ControlFlowInterfaces.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
+include "mlir/Interfaces/VectorUnrollInterface.td"
 include "mlir/Interfaces/ViewLikeInterface.td"
 
 def StandardOps_Dialect : Dialect {
@@ -82,7 +84,9 @@ class UnaryOpSameOperandAndResultType<string mnemonic,
 }
 
 class FloatUnaryOp<string mnemonic, list<OpTrait> traits = []> :
-    UnaryOpSameOperandAndResultType<mnemonic, traits>,
+    UnaryOpSameOperandAndResultType<mnemonic,
+      !listconcat(traits,
+                  [DeclareOpInterfaceMethods<VectorUnrollOpInterface>])>,
     Arguments<(ins FloatLike:$operand)>;
 
 // Base class for standard arithmetic operations.  Requires operands and
@@ -112,7 +116,9 @@ class ArithmeticOp<string mnemonic, list<OpTrait> traits = []> :
 //     <op>i %0, %1 : i32
 //
 class IntArithmeticOp<string mnemonic, list<OpTrait> traits = []> :
-    ArithmeticOp<mnemonic, traits>,
+    ArithmeticOp<mnemonic,
+      !listconcat(traits,
+                  [DeclareOpInterfaceMethods<VectorUnrollOpInterface>])>,
     Arguments<(ins SignlessIntegerLike:$lhs, SignlessIntegerLike:$rhs)>;
 
 // Base class for standard arithmetic binary operations on floats, vectors and
@@ -125,7 +131,9 @@ class IntArithmeticOp<string mnemonic, list<OpTrait> traits = []> :
 //     <op>f %0, %1 : f32
 //
 class FloatArithmeticOp<string mnemonic, list<OpTrait> traits = []> :
-    ArithmeticOp<mnemonic, traits>,
+    ArithmeticOp<mnemonic,
+      !listconcat(traits,
+                  [DeclareOpInterfaceMethods<VectorUnrollOpInterface>])>,
     Arguments<(ins FloatLike:$lhs, FloatLike:$rhs)>;
 
 // Base class for standard arithmetic operations on complex numbers with a
@@ -455,31 +463,6 @@ def AssumeAlignmentOp : Std_Op<"assume_alignment"> {
   let assemblyFormat = "$memref `,` $alignment attr-dict `:` type($memref)";
 }
 
-//===----------------------------------------------------------------------===//
-// AtomicRMWOp
-//===----------------------------------------------------------------------===//
-
-def ATOMIC_RMW_KIND_ADDF    : I64EnumAttrCase<"addf", 0>;
-def ATOMIC_RMW_KIND_ADDI    : I64EnumAttrCase<"addi", 1>;
-def ATOMIC_RMW_KIND_ASSIGN  : I64EnumAttrCase<"assign", 2>;
-def ATOMIC_RMW_KIND_MAXF    : I64EnumAttrCase<"maxf", 3>;
-def ATOMIC_RMW_KIND_MAXS    : I64EnumAttrCase<"maxs", 4>;
-def ATOMIC_RMW_KIND_MAXU    : I64EnumAttrCase<"maxu", 5>;
-def ATOMIC_RMW_KIND_MINF    : I64EnumAttrCase<"minf", 6>;
-def ATOMIC_RMW_KIND_MINS    : I64EnumAttrCase<"mins", 7>;
-def ATOMIC_RMW_KIND_MINU    : I64EnumAttrCase<"minu", 8>;
-def ATOMIC_RMW_KIND_MULF    : I64EnumAttrCase<"mulf", 9>;
-def ATOMIC_RMW_KIND_MULI    : I64EnumAttrCase<"muli", 10>;
-
-def AtomicRMWKindAttr : I64EnumAttr<
-    "AtomicRMWKind", "",
-    [ATOMIC_RMW_KIND_ADDF, ATOMIC_RMW_KIND_ADDI, ATOMIC_RMW_KIND_ASSIGN,
-     ATOMIC_RMW_KIND_MAXF, ATOMIC_RMW_KIND_MAXS, ATOMIC_RMW_KIND_MAXU,
-     ATOMIC_RMW_KIND_MINF, ATOMIC_RMW_KIND_MINS, ATOMIC_RMW_KIND_MINU,
-     ATOMIC_RMW_KIND_MULF, ATOMIC_RMW_KIND_MULI]> {
-  let cppNamespace = "::mlir";
-}
-
 def AtomicRMWOp : Std_Op<"atomic_rmw", [
       AllTypesMatch<["value", "result"]>,
       TypesMatchWith<"value type matches element type of memref",
@@ -1408,7 +1391,9 @@ def DimOp : Std_Op<"dim", [NoSideEffect]> {
 
   let builders = [
     OpBuilder<"OpBuilder &builder, OperationState &result, "
-              "Value memrefOrTensor, int64_t index">
+              "Value memrefOrTensor, int64_t index">,
+    OpBuilder<"OpBuilder &builder, OperationState &result, "
+              "Value memrefOrTensor, Value index">
   ];
 
   let extraClassDeclaration = [{
@@ -1763,7 +1748,9 @@ def Log2Op : FloatUnaryOp<"log2"> {
 // MemRefCastOp
 //===----------------------------------------------------------------------===//
 
-def MemRefCastOp : CastOp<"memref_cast"> {
+def MemRefCastOp : CastOp<"memref_cast", [
+    DeclareOpInterfaceMethods<ViewLikeOpInterface>
+  ]> {
   let summary = "memref cast operation";
   let description = [{
     Syntax:
diff --git a/mlir/include/mlir/Dialect/StandardOps/IR/StandardOpsBase.td b/mlir/include/mlir/Dialect/StandardOps/IR/StandardOpsBase.td
new file mode 100644
index 0000000000000..802a32fce370a
--- /dev/null
+++ b/mlir/include/mlir/Dialect/StandardOps/IR/StandardOpsBase.td
@@ -0,0 +1,39 @@
+//===- StandardOpsBase.td - Standard ops definitions -------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines base support for standard operations.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef STANDARD_OPS_BASE
+#define STANDARD_OPS_BASE
+
+include "mlir/IR/OpBase.td"
+
+def ATOMIC_RMW_KIND_ADDF    : I64EnumAttrCase<"addf", 0>;
+def ATOMIC_RMW_KIND_ADDI    : I64EnumAttrCase<"addi", 1>;
+def ATOMIC_RMW_KIND_ASSIGN  : I64EnumAttrCase<"assign", 2>;
+def ATOMIC_RMW_KIND_MAXF    : I64EnumAttrCase<"maxf", 3>;
+def ATOMIC_RMW_KIND_MAXS    : I64EnumAttrCase<"maxs", 4>;
+def ATOMIC_RMW_KIND_MAXU    : I64EnumAttrCase<"maxu", 5>;
+def ATOMIC_RMW_KIND_MINF    : I64EnumAttrCase<"minf", 6>;
+def ATOMIC_RMW_KIND_MINS    : I64EnumAttrCase<"mins", 7>;
+def ATOMIC_RMW_KIND_MINU    : I64EnumAttrCase<"minu", 8>;
+def ATOMIC_RMW_KIND_MULF    : I64EnumAttrCase<"mulf", 9>;
+def ATOMIC_RMW_KIND_MULI    : I64EnumAttrCase<"muli", 10>;
+
+def AtomicRMWKindAttr : I64EnumAttr<
+    "AtomicRMWKind", "",
+    [ATOMIC_RMW_KIND_ADDF, ATOMIC_RMW_KIND_ADDI, ATOMIC_RMW_KIND_ASSIGN,
+     ATOMIC_RMW_KIND_MAXF, ATOMIC_RMW_KIND_MAXS, ATOMIC_RMW_KIND_MAXU,
+     ATOMIC_RMW_KIND_MINF, ATOMIC_RMW_KIND_MINS, ATOMIC_RMW_KIND_MINU,
+     ATOMIC_RMW_KIND_MULF, ATOMIC_RMW_KIND_MULI]> {
+  let cppNamespace = "::mlir";
+}
+
+#endif // STANDARD_OPS_BASE
diff --git a/mlir/include/mlir/Dialect/Traits.h b/mlir/include/mlir/Dialect/Traits.h
index 2b4fb6d77855f..aecceaac0a426 100644
--- a/mlir/include/mlir/Dialect/Traits.h
+++ b/mlir/include/mlir/Dialect/Traits.h
@@ -47,6 +47,21 @@ namespace util {
 bool getBroadcastedShape(ArrayRef<int64_t> shape1, ArrayRef<int64_t> shape2,
                          SmallVectorImpl<int64_t> &resultShape);
 
+/// Returns true if a broadcast between the 2 shapes is guaranteed to be
+/// successful and not result in an error. False does not guarantee that the
+/// shapes are not broadcastable; it might guarantee that they are not
+/// broadcastable or it might mean that this function does not have enough
+/// information to know.
+///
+/// Conceptually, this returns true if getBroadcastedShape would have returned
+/// true and vice versa, with one exception. If a dimension is unknown in both
+/// shapes, getBroadcastedShape would return true and have a result with unknown
+/// dimension, while this function will return false because it's possible for
+/// both shapes to have a dimension greater than 1 and different which would
+/// fail to broadcast.
+bool staticallyKnownBroadcastable(ArrayRef<int64_t> shape1,
+                                  ArrayRef<int64_t> shape2);
+
 /// Returns the result broadcast composition type from the two given types by
 /// following NumPy broadcast semantics. Returned type may have dynamic shape if
 /// either of the input types has dynamic shape. Returns null type if the two
diff --git a/mlir/include/mlir/Dialect/Vector/EDSC/Intrinsics.h b/mlir/include/mlir/Dialect/Vector/EDSC/Intrinsics.h
index 342f70a69223b..f353262750345 100644
--- a/mlir/include/mlir/Dialect/Vector/EDSC/Intrinsics.h
+++ b/mlir/include/mlir/Dialect/Vector/EDSC/Intrinsics.h
@@ -19,6 +19,7 @@ using vector_contract = ValueBuilder<vector::ContractionOp>;
 using vector_insert = ValueBuilder<vector::InsertOp>;
 using vector_fma = ValueBuilder<vector::FMAOp>;
 using vector_extract = ValueBuilder<vector::ExtractOp>;
+using vector_extractelement = ValueBuilder<vector::ExtractElementOp>;
 using vector_matmul = ValueBuilder<vector::MatmulOp>;
 using vector_outerproduct = ValueBuilder<vector::OuterProductOp>;
 using vector_print = OperationBuilder<vector::PrintOp>;
@@ -26,9 +27,6 @@ using vector_transfer_read = ValueBuilder<vector::TransferReadOp>;
 using vector_transfer_write = OperationBuilder<vector::TransferWriteOp>;
 using vector_transpose = ValueBuilder<vector::TransposeOp>;
 using vector_type_cast = ValueBuilder<vector::TypeCastOp>;
-using vector_insert = ValueBuilder<vector::InsertOp>;
-using vector_fma = ValueBuilder<vector::FMAOp>;
-using vector_extract = ValueBuilder<vector::ExtractOp>;
 using vector_extract_slices = ValueBuilder<vector::ExtractSlicesOp>;
 using vector_insert_slices = ValueBuilder<vector::InsertSlicesOp>;
 using vector_extract_strided_slice =
diff --git a/mlir/include/mlir/Dialect/Vector/VectorOps.h b/mlir/include/mlir/Dialect/Vector/VectorOps.h
index def0d24adcf52..0f6aa66e926c0 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorOps.h
+++ b/mlir/include/mlir/Dialect/Vector/VectorOps.h
@@ -1,4 +1,4 @@
-//===- VectorOps.h - MLIR Super Vectorizer Operations -----------*- C++ -*-===//
+//===- VectorOps.h - MLIR Vector Dialect Operations -------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -19,6 +19,7 @@
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/StandardTypes.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
+#include "mlir/Interfaces/VectorUnrollInterface.h"
 
 namespace mlir {
 class MLIRContext;
@@ -46,8 +47,8 @@ void populateVectorSlicesLoweringPatterns(OwningRewritePatternList &patterns,
 
 /// Enum to control the lowering of `vector.contract` operations.
 enum class VectorContractLowering {
-  /// Progressively lower to finer grained `vector.contract` and `vector.fma`.
-  FMA = 0,
+  /// Progressively lower to finer grained `vector.contract` and dot-products.
+  Dot = 0,
   /// Lower to `vector.matrix_multiply`, maps 1-1 to LLVM matrix intrinsics.
   Matmul = 1,
   /// Lower to `vector.outerproduct`.
@@ -63,7 +64,7 @@ enum class VectorTransposeLowering {
 };
 /// Structure to control the behavior of vector transform patterns.
 struct VectorTransformsOptions {
-  VectorContractLowering vectorContractLowering = VectorContractLowering::FMA;
+  VectorContractLowering vectorContractLowering = VectorContractLowering::Dot;
   VectorTransposeLowering vectorTransposeLowering =
       VectorTransposeLowering::EltWise;
   VectorTransformsOptions &
diff --git a/mlir/include/mlir/Dialect/Vector/VectorOps.td b/mlir/include/mlir/Dialect/Vector/VectorOps.td
index 70ee272c8ceff..8880c288b648e 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/VectorOps.td
@@ -15,6 +15,7 @@
 
 include "mlir/Dialect/Affine/IR/AffineOpsBase.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
+include "mlir/Interfaces/VectorUnrollInterface.td"
 
 def Vector_Dialect : Dialect {
   let name = "vector";
@@ -36,13 +37,16 @@ class Vector_Op<string mnemonic, list<OpTrait> traits = []> :
   let parser = [{ return ::parse$cppClass(parser, result); }];
 }
 
-// TODO(andydavis, ntv) Add an attribute to specify a different algebra
-// with operators other than the current set: {*, +}.
+// TODO: Add an attribute to specify a different algebra with operators other
+// than the current set: {*, +}.
 def Vector_ContractionOp :
-  Vector_Op<"contract", [NoSideEffect,
-     PredOpTrait<"lhs and rhs have same element type", TCopVTEtIsSameAs<0, 1>>,
-     PredOpTrait<"third operand acc and result have same element type",
-                 TCresVTEtIsSameAsOpBase<0, 2>>]>,
+  Vector_Op<"contract", [
+      NoSideEffect,
+      PredOpTrait<"lhs and rhs have same element type", TCopVTEtIsSameAs<0, 1>>,
+      PredOpTrait<"third operand acc and result have same element type",
+                  TCresVTEtIsSameAsOpBase<0, 2>>,
+      DeclareOpInterfaceMethods<VectorUnrollOpInterface, ["getShapeForUnroll"]>
+    ]>,
     Arguments<(ins AnyVector:$lhs, AnyVector:$rhs, AnyType:$acc,
                Variadic<VectorOf<[I1]>>:$masks,
                AffineMapArrayAttr:$indexing_maps, ArrayAttr:$iterator_types)>,
@@ -87,7 +91,7 @@ def Vector_ContractionOp :
     Example:
 
     ```mlir
-    // Simple dot product (K = 0).
+    // Simple DOT product (K = 0).
     #contraction_accesses = [
      affine_map<(i) -> (i)>,
      affine_map<(i) -> (i)>,
@@ -386,6 +390,7 @@ def Vector_ExtractOp :
       return vector().getType().cast<VectorType>();
     }
   }];
+  let hasFolder = 1;
 }
 
 def Vector_ExtractSlicesOp :
@@ -663,19 +668,36 @@ def Vector_InsertStridedSliceOp :
 }
 
 def Vector_OuterProductOp :
-  Vector_Op<"outerproduct", [NoSideEffect, SameOperandsAndResultElementType]>,
-    Arguments<(ins AnyVector:$lhs, AnyVector:$rhs, Variadic<AnyVector>:$acc)>,
+  Vector_Op<"outerproduct", [NoSideEffect,
+    PredOpTrait<"lhs operand and result have same element type",
+                TCresVTEtIsSameAsOpBase<0, 0>>,
+    PredOpTrait<"rhs operand and result have same element type",
+                TCresVTEtIsSameAsOpBase<0, 1>>]>,
+    Arguments<(ins AnyVector:$lhs, AnyType:$rhs, Variadic<AnyVector>:$acc)>,
     Results<(outs AnyVector)> {
   let summary = "vector outerproduct with optional fused add";
   let description = [{
-    Takes 2 1-D vectors and returns the 2-D vector containing the outer-product.
+    Takes 2 1-D vectors and returns the 2-D vector containing the outer-product,
+    as illustrated below:
+    ```
+     outer |   [c, d]
+     ------+------------
+       [a, | [ [a*c, a*d],
+        b] |   [b*c, b*d] ]
+    ```
+    This operation also accepts a 1-D vector lhs and a scalar rhs. In this
+    case a simple AXPY operation is performed, which returns a 1-D vector.
+    ```
+        [a, b] * c = [a*c, b*c]
+    ```
 
-    An optional extra 2-D vector argument may be specified in which case the
-    operation returns the sum of the outer-product and the extra vector. In this
-    multiply-accumulate scenario, the rounding mode is that obtained by
-    guaranteeing that a fused-multiply add operation is emitted. When lowered to
-    the LLVMIR dialect, this form emits `llvm.intr.fma`, which is guaranteed to
-    lower to actual `fma` instructions on x86.
+    An optional extra vector argument with the same shape as the output
+    vector may be specified in which case the operation returns the sum of
+    the outer-product and the extra vector. In this multiply-accumulate
+    scenario for floating-point arguments, the rounding mode is enforced
+    by guaranteeing that a fused-multiply add operation is emitted. When
+    lowered to the LLVMIR dialect, this form emits `llvm.intr.fma`, which
+    is guaranteed to lower to actual `fma` instructions on x86.
 
     Example:
 
@@ -686,6 +708,10 @@ def Vector_OuterProductOp :
     %3 = vector.outerproduct %0, %1, %2:
       vector<4xf32>, vector<8xf32>, vector<4x8xf32>
     return %3: vector<4x8xf32>
+
+    %6 = vector.outerproduct %4, %5: vector<10xf32>, f32
+    return %6: vector<10xf32>
+
     ```
   }];
   let builders = [
@@ -697,12 +723,13 @@ def Vector_OuterProductOp :
     VectorType getOperandVectorTypeLHS() {
       return lhs().getType().cast<VectorType>();
     }
-    VectorType getOperandVectorTypeRHS() {
-      return rhs().getType().cast<VectorType>();
+    Type getOperandTypeRHS() {
+      return rhs().getType();
     }
     VectorType getOperandVectorTypeACC() {
-      return (llvm::size(acc()) == 0) ? VectorType() :
-        (*acc().begin()).getType().cast<VectorType>();
+      return (llvm::size(acc()) == 0)
+        ? VectorType()
+        : (*acc().begin()).getType().cast<VectorType>();
     }
     VectorType getVectorType() {
       return getResult().getType().cast<VectorType>();
@@ -710,8 +737,8 @@ def Vector_OuterProductOp :
   }];
 }
 
-// TODO(andydavis) Add transformation which decomposes ReshapeOp into an
-// optimized sequence of vector rotate/shuffle/select operations.
+// TODO: Add transformation which decomposes ReshapeOp into an optimized
+// sequence of vector rotate/shuffle/select operations.
 def Vector_ReshapeOp :
   Vector_Op<"reshape", [AttrSizedOperandSegments, NoSideEffect]>,
     Arguments<(ins AnyVector:$vector, Variadic<Index>:$input_shape,
@@ -845,7 +872,7 @@ def Vector_ExtractStridedSliceOp :
     attribute and extracts the n-D subvector at the proper offset.
 
     At the moment strides must contain only 1s.
-    // TODO(ntv) support non-1 strides.
+    // TODO: support non-1 strides.
 
     Returns an n-D vector where the first k-D dimensions match the `sizes`
     attribute. The returned subvector contains the elements starting at offset
@@ -858,7 +885,7 @@ def Vector_ExtractStridedSliceOp :
         {offsets = [0, 2], sizes = [2, 4], strides = [1, 1]}:
       vector<4x8x16xf32> to vector<2x4x16xf32>
 
-    // TODO(ntv) Evolve to a range form syntax similar to:
+    // TODO: Evolve to a range form syntax similar to:
     %1 = vector.extract_strided_slice %0[0:2:1][2:4:1]
       vector<4x8x16xf32> to vector<2x4x16xf32>
     ```
@@ -892,11 +919,22 @@ def Vector_TransferOpUtils {
     VectorType getVectorType() {
       return vector().getType().cast<VectorType>();
     }
+    // Number of dimensions that participate in the permutation map.
+    unsigned getTransferRank() {
+      return permutation_map().getNumResults();
+    }
+    // Number of leading dimensions that do not participate in the permutation
+    // map.
+    unsigned getLeadingMemRefRank() {
+      return getMemRefType().getRank() - permutation_map().getNumResults();
+    }
   }];
 }
 
 def Vector_TransferReadOp :
-  Vector_Op<"transfer_read">,
+  Vector_Op<"transfer_read", [
+      DeclareOpInterfaceMethods<VectorUnrollOpInterface, ["getShapeForUnroll"]>
+    ]>,
     Arguments<(ins AnyMemRef:$memref, Variadic<Index>:$indices,
                AffineMapAttr:$permutation_map, AnyType:$padding,
                OptionalAttr<BoolArrayAttr>:$masked)>,
@@ -1068,7 +1106,9 @@ def Vector_TransferReadOp :
 }
 
 def Vector_TransferWriteOp :
-  Vector_Op<"transfer_write">,
+  Vector_Op<"transfer_write", [
+      DeclareOpInterfaceMethods<VectorUnrollOpInterface, ["getShapeForUnroll"]>
+    ]>,
     Arguments<(ins AnyVector:$vector, AnyMemRef:$memref,
                Variadic<Index>:$indices,
                AffineMapAttr:$permutation_map,
@@ -1505,7 +1545,7 @@ def Vector_MatmulOp : Vector_Op<"matrix_multiply", [NoSideEffect,
         PredOpTrait<"rhs operand and result have same element type",
                     TCresVTEtIsSameAsOpBase<0, 1>>]>,
       Arguments<(
-        // TODO(ntv, fhahn): tighten vector element types that make sense.
+        // TODO: tighten vector element types that make sense.
         ins VectorOfRankAndType<[1],
               [AnySignlessInteger, AnySignedInteger, AnyFloat]>:$lhs,
             VectorOfRankAndType<[1],
@@ -1568,7 +1608,7 @@ def Vector_FlatTransposeOp : Vector_Op<"flat_transpose", [NoSideEffect,
   PredOpTrait<"source operand and result have same element type",
                  TCresVTEtIsSameAsOpBase<0, 0>>]>,
     Arguments<(
-      // TODO(ntv, fhahn, ajcbik): tighten vector element types that make sense.
+      // TODO: tighten vector element types that make sense.
       ins VectorOfRankAndType<[1],
             [AnySignlessInteger, AnySignedInteger, AnyFloat]>:$matrix,
           I32Attr:$rows, I32Attr:$columns)>,
diff --git a/mlir/include/mlir/Dialect/Vector/VectorTransformPatterns.td b/mlir/include/mlir/Dialect/Vector/VectorTransformPatterns.td
index 5f5c90521a7d8..ef8118ec64704 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorTransformPatterns.td
+++ b/mlir/include/mlir/Dialect/Vector/VectorTransformPatterns.td
@@ -20,7 +20,7 @@ class HasShape<list<int> shape> :
     StrJoinInt<shape>.result # "})">;
 
 class UnrollVectorOp<list<int> factors> : NativeCodeCall<
-  "unrollSingleResultOpMatchingType($_builder, $0.getDefiningOp(), " #
+  "unrollSingleResultVectorOp($_builder, $0.getDefiningOp(), " #
     "{" # StrJoinInt<factors>.result # "})">;
 
 #endif // VECTOR_TRANSFORM_PATTERNS
diff --git a/mlir/include/mlir/Dialect/Vector/VectorTransforms.h b/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
index 08aa579d651b4..e95329c3e505d 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
+++ b/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
@@ -10,6 +10,8 @@
 #define DIALECT_VECTOR_VECTORTRANSFORMS_H_
 
 #include "mlir/Dialect/Vector/VectorOps.h"
+#include "mlir/Dialect/Vector/VectorUtils.h"
+#include "mlir/IR/Function.h"
 #include "mlir/IR/PatternMatch.h"
 
 namespace mlir {
@@ -25,42 +27,82 @@ void populateVectorToVectorConversionPatterns(
 
 namespace vector {
 
-// Entry point for unrolling declarative pattern rewrites.
-// `op` is unrolled to the `targetShape` as follows, for each of its operands:
-//   1. the unrolled type `unrolledVectorType` and number of unrolled instances
-//   `numUnrolledInstances` are computed from the `targetShape`. For now it is
-//   assumed the unrolling factors divide the vector sizes.
-//   2. a fakeFork cast op is inserted that takes the operand and returns
-//   `numUnrolledInstances` results of type `unrolledVectorType`.
-//   3. the original op is cloned `numUnrolledInstances` times, once for each
-//   result of the fakeFork cast op.
-//   4. a fakeJoin cast op takes all these results and merges them into a single
-//   aggregate vector result whose size matches the original non-unrolled op
-//   operand types.
-//
-// Example:
-//
-//    opA(operand0, operand1)  // numUnrolledInstances = 3
-//
-//            operand0                   operand1
-//               |                          |
-//             fork                       fork
-//        <----------gather all fork ops --------->
-//              /|\                        /|\
-//          f00 f01 f02                f10 f11 f12
-//        <---------- clone op 3 times --------->
-//          opA0(f00, f10), opA1(f01, f11), opA2(f02, f12)
-//                 \            |            /
-//      <-------------------- join ------------------------->
-//
-// Other local patterns then kick in iteratively (including DCE) and compose
-// until all the fakeFork and fakeJoin ops are removed.
-//
-// This will be extended in the future to support more advanced use cases than
-// simple pointwise ops.
-SmallVector<Value, 1>
-unrollSingleResultOpMatchingType(OpBuilder &builder, Operation *op,
-                                 ArrayRef<int64_t> targetShape);
+/// Entry point for unrolling declarative pattern rewrites.
+/// `op` is unrolled to the `targetShape` as follows, for each of its operands:
+///   1. the unrolled type `unrolledVectorType` and number of unrolled instances
+///   `numUnrolledInstances` are computed from the `targetShape`. For now it is
+///   assumed the unrolling factors divide the vector sizes.
+///   2. a fakeFork cast op is inserted that takes the operand and returns
+///   `numUnrolledInstances` results of type `unrolledVectorType`.
+///   3. the original op is cloned `numUnrolledInstances` times, once for each
+///   result of the fakeFork cast op.
+///   4. a fakeJoin cast op takes all these results and merges them into a
+///   single aggregate vector result whose size matches the original
+///   non-unrolled op operand types.
+///
+/// Example:
+///
+///    opA(operand0, operand1)  // numUnrolledInstances = 3
+///
+///            operand0                   operand1
+///               |                          |
+///             fork                       fork
+///        <----------gather all fork ops --------->
+///              /|\                        /|\
+///          f00 f01 f02                f10 f11 f12
+///        <---------- clone op 3 times --------->
+///          opA0(f00, f10), opA1(f01, f11), opA2(f02, f12)
+///                 \            |            /
+///      <-------------------- join ------------------------->
+///
+/// Other local patterns then kick in iteratively (including DCE) and compose
+/// until all the fakeFork and fakeJoin ops are removed.
+///
+/// This will be extended in the future to support more advanced use cases than
+/// simple pointwise ops.
+SmallVector<Value, 1> unrollSingleResultVectorOp(OpBuilder &builder,
+                                                 Operation *op,
+                                                 ArrayRef<int64_t> targetShape);
+
+/// Pattern to apply `unrollSingleResultVectorOp` to a `targetShape`
+/// declaratively.
+template <typename OpTy>
+struct UnrollVectorPattern : public OpRewritePattern<OpTy> {
+  using FilterConstraintType = std::function<LogicalResult(OpTy op)>;
+  UnrollVectorPattern(
+      ArrayRef<int64_t> targetShape, MLIRContext *context,
+      FilterConstraintType constraint = [](OpTy op) { return success(); })
+      : OpRewritePattern<OpTy>(context),
+        targetShape(targetShape.begin(), targetShape.end()),
+        filter(constraint) {}
+  LogicalResult matchAndRewrite(OpTy op,
+                                PatternRewriter &rewriter) const override {
+    if (failed(filter(op)))
+      return failure();
+    auto unrollableVectorOp =
+        dyn_cast<VectorUnrollOpInterface>(op.getOperation());
+    if (!unrollableVectorOp)
+      return failure();
+    auto maybeUnrollShape = unrollableVectorOp.getShapeForUnroll();
+    if (!maybeUnrollShape)
+      return failure();
+    auto maybeShapeRatio = shapeRatio(*maybeUnrollShape, targetShape);
+    if (!maybeShapeRatio ||
+        llvm::all_of(*maybeShapeRatio, [](int64_t v) { return v == 1; }))
+      return failure();
+    if (op.getOperation()->getNumResults() != 1)
+      return failure();
+    auto resultVector = unrollSingleResultVectorOp(rewriter, op, targetShape);
+    if (resultVector.size() != 1)
+      return failure();
+    rewriter.replaceOp(op, resultVector.front());
+    return success();
+  }
+
+private:
+  SmallVector<int64_t, 4> targetShape;
+  FilterConstraintType filter;
+};
 
 } // namespace vector
 
@@ -145,10 +187,10 @@ class ContractionOpToOuterProductOpLowering
 ///   ..
 ///   %x = combine %a %b ..
 /// until a pure contraction is reached (no free/batch dimensions),
-/// which is replaced by a fma/reduction op.
+/// which is replaced by a dot-product.
 ///
-/// This only kicks in when either VectorTransformsOptions is set to FMA or when
-/// other contraction patterns fail.
+/// This only kicks in when either VectorTransformsOptions is set
+/// to Dot or when other contraction patterns fail.
 class ContractionOpLowering : public OpRewritePattern<vector::ContractionOp> {
 public:
   using OpRewritePattern<vector::ContractionOp>::OpRewritePattern;
diff --git a/mlir/include/mlir/Dialect/Vector/VectorUtils.h b/mlir/include/mlir/Dialect/Vector/VectorUtils.h
index 58f936ca305cb..19f7f9538307d 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorUtils.h
+++ b/mlir/include/mlir/Dialect/Vector/VectorUtils.h
@@ -160,9 +160,9 @@ namespace matcher {
 /// over other smaller vector types in the function and avoids interfering with
 /// operations on those.
 /// This is a first approximation, it can easily be extended in the future.
-/// TODO(ntv): this could all be much simpler if we added a bit that a vector
-/// type to mark that a vector is a strict super-vector but it still does not
-/// warrant adding even 1 extra bit in the IR for now.
+/// TODO: this could all be much simpler if we added a bit that a vector type to
+/// mark that a vector is a strict super-vector but it still does not warrant
+/// adding even 1 extra bit in the IR for now.
 bool operatesOnSuperVectorsOf(Operation &op, VectorType subVectorType);
 
 } // end namespace matcher
diff --git a/mlir/include/mlir/EDSC/Builders.h b/mlir/include/mlir/EDSC/Builders.h
index a7c5506f7ab03..1f21af617e4d2 100644
--- a/mlir/include/mlir/EDSC/Builders.h
+++ b/mlir/include/mlir/EDSC/Builders.h
@@ -139,15 +139,12 @@ struct StructuredIndexed {
 
   StructuredIndexed(Value v, ArrayRef<AffineExpr> indexings)
       : value(v), exprs(indexings.begin(), indexings.end()) {
-    assert((v.getType().isa<MemRefType>() ||
-            v.getType().isa<RankedTensorType>() ||
-            v.getType().isa<VectorType>()) &&
+    assert((v.getType().isa<MemRefType, RankedTensorType, VectorType>()) &&
            "MemRef, RankedTensor or Vector expected");
   }
   StructuredIndexed(Type t, ArrayRef<AffineExpr> indexings)
       : type(t), exprs(indexings.begin(), indexings.end()) {
-    assert((t.isa<MemRefType>() || t.isa<RankedTensorType>() ||
-            t.isa<VectorType>()) &&
+    assert((t.isa<MemRefType, RankedTensorType, VectorType>()) &&
            "MemRef, RankedTensor or Vector expected");
   }
 
@@ -288,21 +285,37 @@ class TemplatedIndexedValue {
   /// Comparison operator overloadings.
   Value eq(Value e);
   Value ne(Value e);
-  Value operator<(Value e);
-  Value operator<=(Value e);
-  Value operator>(Value e);
-  Value operator>=(Value e);
-  Value operator<(TemplatedIndexedValue e) {
-    return *this < static_cast<Value>(e);
+  Value slt(Value e);
+  Value sle(Value e);
+  Value sgt(Value e);
+  Value sge(Value e);
+  Value ult(Value e);
+  Value ule(Value e);
+  Value ugt(Value e);
+  Value uge(Value e);
+  Value slt(TemplatedIndexedValue e) {
+    return slt(*this, static_cast<Value>(e));
   }
-  Value operator<=(TemplatedIndexedValue e) {
-    return *this <= static_cast<Value>(e);
+  Value sle(TemplatedIndexedValue e) {
+    return sle(*this, static_cast<Value>(e));
   }
-  Value operator>(TemplatedIndexedValue e) {
-    return *this > static_cast<Value>(e);
+  Value sgt(TemplatedIndexedValue e) {
+    return sgt(*this, static_cast<Value>(e));
   }
-  Value operator>=(TemplatedIndexedValue e) {
-    return *this >= static_cast<Value>(e);
+  Value sge(TemplatedIndexedValue e) {
+    return sge(*this, static_cast<Value>(e));
+  }
+  Value ult(TemplatedIndexedValue e) {
+    return ult(*this, static_cast<Value>(e));
+  }
+  Value ule(TemplatedIndexedValue e) {
+    return ule(*this, static_cast<Value>(e));
+  }
+  Value ugt(TemplatedIndexedValue e) {
+    return ugt(*this, static_cast<Value>(e));
+  }
+  Value uge(TemplatedIndexedValue e) {
+    return uge(*this, static_cast<Value>(e));
   }
 
 private:
diff --git a/mlir/include/mlir/IR/AffineMap.h b/mlir/include/mlir/IR/AffineMap.h
index 841ec6abcedf6..d946c7591c2a0 100644
--- a/mlir/include/mlir/IR/AffineMap.h
+++ b/mlir/include/mlir/IR/AffineMap.h
@@ -99,9 +99,9 @@ class AffineMap {
   /// dimensional identifiers.
   bool isIdentity() const;
 
-  /// Returns true if the map is a minor identity map, i.e. an identity affine
-  /// map (d0, ..., dn) -> (dp, ..., dn) on the most minor dimensions.
-  static bool isMinorIdentity(AffineMap map);
+  /// Returns true if this affine map is a minor identity, i.e. an identity
+  /// affine map (d0, ..., dn) -> (dp, ..., dn) on the most minor dimensions.
+  bool isMinorIdentity() const;
 
   /// Returns true if this affine map is an empty map, i.e., () -> ().
   bool isEmpty() const;
@@ -170,6 +170,10 @@ class AffineMap {
   ///     `(d0)[s0, s1, s2] -> (d0 + s1 + s2 + 1, d0 - s0 - s2 - 1)`
   AffineMap compose(AffineMap map);
 
+  /// Applies composition by the dims of `this` to the integer `values` and
+  /// returns the resulting values. `this` must be symbol-less.
+  SmallVector<int64_t, 4> compose(ArrayRef<int64_t> values);
+
   /// Returns true if the AffineMap represents a subset (i.e. a projection) of a
   /// symbol-less permutation map.
   bool isProjectedPermutation();
@@ -180,6 +184,16 @@ class AffineMap {
   /// Returns the map consisting of the `resultPos` subset.
   AffineMap getSubMap(ArrayRef<unsigned> resultPos);
 
+  /// Returns the map consisting of the most major `numResults` results.
+  /// Returns the null AffineMap if `numResults` == 0.
+  /// Returns `*this` if `numResults` >= `this->getNumResults()`.
+  AffineMap getMajorSubMap(unsigned numResults);
+
+  /// Returns the map consisting of the most minor `numResults` results.
+  /// Returns the null AffineMap if `numResults` == 0.
+  /// Returns `*this` if `numResults` >= `this->getNumResults()`.
+  AffineMap getMinorSubMap(unsigned numResults);
+
   friend ::llvm::hash_code hash_value(AffineMap arg);
 
 private:
diff --git a/mlir/include/mlir/IR/AttributeSupport.h b/mlir/include/mlir/IR/AttributeSupport.h
index 48d68d9c4fa45..cd722bb0f2c5b 100644
--- a/mlir/include/mlir/IR/AttributeSupport.h
+++ b/mlir/include/mlir/IR/AttributeSupport.h
@@ -21,6 +21,45 @@ namespace mlir {
 class MLIRContext;
 class Type;
 
+//===----------------------------------------------------------------------===//
+// AbstractAttribute
+//===----------------------------------------------------------------------===//
+
+/// This class contains all of the static information common to all instances of
+/// a registered Attribute.
+class AbstractAttribute {
+public:
+  /// Look up the specified abstract attribute in the MLIRContext and return a
+  /// reference to it.
+  static const AbstractAttribute &lookup(TypeID typeID, MLIRContext *context);
+
+  /// This method is used by Dialect objects when they register the list of
+  /// attributes they contain.
+  template <typename T> static AbstractAttribute get(Dialect &dialect) {
+    return AbstractAttribute(dialect, T::getInterfaceMap());
+  }
+
+  /// Return the dialect this attribute was registered to.
+  Dialect &getDialect() const { return const_cast<Dialect &>(dialect); }
+
+  /// Returns an instance of the concept object for the given interface if it
+  /// was registered to this attribute, null otherwise. This should not be used
+  /// directly.
+  template <typename T> typename T::Concept *getInterface() const {
+    return interfaceMap.lookup<T>();
+  }
+
+private:
+  AbstractAttribute(Dialect &dialect, detail::InterfaceMap &&interfaceMap)
+      : dialect(dialect), interfaceMap(std::move(interfaceMap)) {}
+
+  /// This is the dialect that this attribute was registered to.
+  Dialect &dialect;
+
+  /// This is a collection of the interfaces registered to this attribute.
+  detail::InterfaceMap interfaceMap;
+};
+
 //===----------------------------------------------------------------------===//
 // AttributeStorage
 //===----------------------------------------------------------------------===//
@@ -39,10 +78,10 @@ class AttributeStorage : public StorageUniquer::BaseStorage {
   /// Get the type of this attribute.
   Type getType() const;
 
-  /// Get the dialect of this attribute.
-  Dialect &getDialect() const {
-    assert(dialect && "Malformed attribute storage object.");
-    return const_cast<Dialect &>(*dialect);
+  /// Return the abstract descriptor for this attribute.
+  const AbstractAttribute &getAbstractAttribute() const {
+    assert(abstractAttribute && "Malformed attribute storage object.");
+    return *abstractAttribute;
   }
 
 protected:
@@ -56,13 +95,15 @@ class AttributeStorage : public StorageUniquer::BaseStorage {
   /// Set the type of this attribute.
   void setType(Type type);
 
-  // Set the dialect for this storage instance. This is used by the
+  // Set the abstract attribute for this storage instance. This is used by the
   // AttributeUniquer when initializing a newly constructed storage object.
-  void initializeDialect(Dialect &newDialect) { dialect = &newDialect; }
+  void initialize(const AbstractAttribute &abstractAttr) {
+    abstractAttribute = &abstractAttr;
+  }
 
 private:
-  /// The dialect for this attribute.
-  Dialect *dialect;
+  /// The abstract descriptor for this attribute.
+  const AbstractAttribute *abstractAttribute;
 
   /// The opaque type of the attribute value.
   const void *type;
diff --git a/mlir/include/mlir/IR/Attributes.h b/mlir/include/mlir/IR/Attributes.h
index f9d8efd422720..89dad2ec40cf7 100644
--- a/mlir/include/mlir/IR/Attributes.h
+++ b/mlir/include/mlir/IR/Attributes.h
@@ -63,10 +63,10 @@ class Attribute {
   };
 
   /// Utility class for implementing attributes.
-  template <typename ConcreteType, typename BaseType = Attribute,
-            typename StorageType = AttributeStorage>
+  template <typename ConcreteType, typename BaseType, typename StorageType,
+            template <typename T> class... Traits>
   using AttrBase = detail::StorageUserBase<ConcreteType, BaseType, StorageType,
-                                           detail::AttributeUniquer>;
+                                           detail::AttributeUniquer, Traits...>;
 
   using ImplType = AttributeStorage;
   using ValueType = void;
@@ -85,6 +85,8 @@ class Attribute {
   bool operator!() const { return impl == nullptr; }
 
   template <typename U> bool isa() const;
+  template <typename First, typename Second, typename... Rest>
+  bool isa() const;
   template <typename U> U dyn_cast() const;
   template <typename U> U dyn_cast_or_null() const;
   template <typename U> U cast() const;
@@ -117,6 +119,11 @@ class Attribute {
 
   friend ::llvm::hash_code hash_value(Attribute arg);
 
+  /// Return the abstract descriptor for this attribute.
+  const AbstractAttribute &getAbstractAttribute() const {
+    return impl->getAbstractAttribute();
+  }
+
 protected:
   ImplType *impl;
 };
@@ -126,6 +133,46 @@ inline raw_ostream &operator<<(raw_ostream &os, Attribute attr) {
   return os;
 }
 
+//===----------------------------------------------------------------------===//
+// AttributeTraitBase
+//===----------------------------------------------------------------------===//
+
+namespace AttributeTrait {
+/// This class represents the base of an attribute trait.
+template <typename ConcreteType, template <typename> class TraitType>
+using TraitBase = detail::StorageUserTraitBase<ConcreteType, TraitType>;
+} // namespace AttributeTrait
+
+//===----------------------------------------------------------------------===//
+// AttributeInterface
+//===----------------------------------------------------------------------===//
+
+/// This class represents the base of an attribute interface. See the definition
+/// of `detail::Interface` for requirements on the `Traits` type.
+template <typename ConcreteType, typename Traits>
+class AttributeInterface
+    : public detail::Interface<ConcreteType, Attribute, Traits, Attribute,
+                               AttributeTrait::TraitBase> {
+public:
+  using Base = AttributeInterface<ConcreteType, Traits>;
+  using InterfaceBase = detail::Interface<ConcreteType, Type, Traits, Type,
+                                          AttributeTrait::TraitBase>;
+  using InterfaceBase::InterfaceBase;
+
+private:
+  /// Returns the impl interface instance for the given type.
+  static typename InterfaceBase::Concept *getInterfaceFor(Attribute attr) {
+    return attr.getAbstractAttribute().getInterface<ConcreteType>();
+  }
+
+  /// Allow access to 'getInterfaceFor'.
+  friend InterfaceBase;
+};
+
+//===----------------------------------------------------------------------===//
+// StandardAttributes
+//===----------------------------------------------------------------------===//
+
 namespace StandardAttributes {
 enum Kind {
   AffineMap = Attribute::FIRST_STANDARD_ATTR,
@@ -239,6 +286,12 @@ class ArrayAttr : public Attribute::AttrBase<ArrayAttr, Attribute,
     return llvm::make_range(attr_value_iterator<AttrTy>(begin()),
                             attr_value_iterator<AttrTy>(end()));
   }
+  template <typename AttrTy, typename UnderlyingTy>
+  auto getAsRange() {
+    return llvm::map_range(getAsRange<AttrTy>(), [](AttrTy attr) {
+      return static_cast<UnderlyingTy>(attr.getValue());
+    });
+  }
 };
 
 //===----------------------------------------------------------------------===//
@@ -370,7 +423,7 @@ class IntegerAttr
   APInt getValue() const;
   /// Return the integer value as a 64-bit int. The attribute must be a signless
   /// integer.
-  // TODO(jpienaar): Change callers to use getValue instead.
+  // TODO: Change callers to use getValue instead.
   int64_t getInt() const;
   /// Return the integer value as a signed 64-bit int. The attribute must be
   /// a signed integer.
@@ -588,7 +641,8 @@ class TypeAttr : public Attribute::AttrBase<TypeAttr, Attribute,
 
 /// Unit attributes are attributes that hold no specific value and are given
 /// meaning by their existence.
-class UnitAttr : public Attribute::AttrBase<UnitAttr> {
+class UnitAttr
+    : public Attribute::AttrBase<UnitAttr, Attribute, AttributeStorage> {
 public:
   using Base::Base;
 
@@ -1390,8 +1444,7 @@ class SparseElementsAttr
     auto zeroValue = getZeroValue<T>();
     auto valueIt = getValues().getValues<T>().begin();
     const std::vector<ptrdiff_t> flatSparseIndices(getFlattenedSparseIndices());
-    // TODO(riverriddle): Move-capture flatSparseIndices when c++14 is
-    // available.
+    // TODO: Move-capture flatSparseIndices when c++14 is available.
     std::function<T(ptrdiff_t)> mapFn = [=](ptrdiff_t index) {
       // Try to map the current index to one of the sparse indices.
       for (unsigned i = 0, e = flatSparseIndices.size(); i != e; ++i)
@@ -1630,6 +1683,12 @@ template <typename U> bool Attribute::isa() const {
   assert(impl && "isa<> used on a null attribute.");
   return U::classof(*this);
 }
+
+template <typename First, typename Second, typename... Rest>
+bool Attribute::isa() const {
+  return isa<First>() || isa<Second, Rest...>();
+}
+
 template <typename U> U Attribute::dyn_cast() const {
   return isa<U>() ? U(impl) : U(nullptr);
 }
diff --git a/mlir/include/mlir/IR/Diagnostics.h b/mlir/include/mlir/IR/Diagnostics.h
index 584ff4b1e2420..84411b7203368 100644
--- a/mlir/include/mlir/IR/Diagnostics.h
+++ b/mlir/include/mlir/IR/Diagnostics.h
@@ -558,7 +558,7 @@ class SourceMgrDiagnosticHandler : public ScopedDiagnosticHandler {
   llvm::SMLoc convertLocToSMLoc(FileLineColLoc loc);
 
   /// The maximum depth that a call stack will be printed.
-  /// TODO(riverriddle) This should be a tunable flag.
+  /// TODO: This should be a tunable flag.
   unsigned callStackLimit = 10;
 
   std::unique_ptr<detail::SourceMgrDiagnosticHandlerImpl> impl;
diff --git a/mlir/include/mlir/IR/Dialect.h b/mlir/include/mlir/IR/Dialect.h
index 73d55c2260c0f..043fde9da729f 100644
--- a/mlir/include/mlir/IR/Dialect.h
+++ b/mlir/include/mlir/IR/Dialect.h
@@ -190,13 +190,19 @@ class Dialect {
 
   /// This method is used by derived classes to add their types to the set.
   template <typename... Args> void addTypes() {
-    (void)std::initializer_list<int>{0, (addSymbol(Args::getTypeID()), 0)...};
+    (void)std::initializer_list<int>{
+        0, (addType(Args::getTypeID(), AbstractType::get<Args>(*this)), 0)...};
   }
+  void addType(TypeID typeID, AbstractType &&typeInfo);
 
   /// This method is used by derived classes to add their attributes to the set.
   template <typename... Args> void addAttributes() {
-    (void)std::initializer_list<int>{0, (addSymbol(Args::getTypeID()), 0)...};
+    (void)std::initializer_list<int>{
+        0,
+        (addAttribute(Args::getTypeID(), AbstractAttribute::get<Args>(*this)),
+         0)...};
   }
+  void addAttribute(TypeID typeID, AbstractAttribute &&attrInfo);
 
   /// Enable support for unregistered operations.
   void allowUnknownOperations(bool allow = true) { unknownOpsAllowed = allow; }
@@ -214,9 +220,6 @@ class Dialect {
   }
 
 private:
-  // Register a symbol(e.g. type) with its given unique class identifier.
-  void addSymbol(TypeID typeID);
-
   Dialect(const Dialect &) = delete;
   void operator=(Dialect &) = delete;
 
@@ -255,10 +258,12 @@ class Dialect {
 };
 /// Registers all dialects and hooks from the global registries with the
 /// specified MLIRContext.
+/// Note: This method is not thread-safe.
 void registerAllDialects(MLIRContext *context);
 
 /// Utility to register a dialect. Client can register their dialect with the
 /// global registry by calling registerDialect<MyDialect>();
+/// Note: This method is not thread-safe.
 template <typename ConcreteDialect> void registerDialect() {
   Dialect::registerDialectAllocator(TypeID::get<ConcreteDialect>(),
                                     [](MLIRContext *ctx) {
diff --git a/mlir/include/mlir/IR/Dominance.h b/mlir/include/mlir/IR/Dominance.h
index 97ec990077085..9d9a19996765f 100644
--- a/mlir/include/mlir/IR/Dominance.h
+++ b/mlir/include/mlir/IR/Dominance.h
@@ -141,7 +141,7 @@ namespace llvm {
 /// DominatorTree GraphTraits specialization so the DominatorTree can be
 /// iterated by generic graph iterators.
 template <> struct GraphTraits<mlir::DominanceInfoNode *> {
-  using ChildIteratorType = mlir::DominanceInfoNode::iterator;
+  using ChildIteratorType = mlir::DominanceInfoNode::const_iterator;
   using NodeRef = mlir::DominanceInfoNode *;
 
   static NodeRef getEntryNode(NodeRef N) { return N; }
diff --git a/mlir/include/mlir/IR/Function.h b/mlir/include/mlir/IR/Function.h
index f50a1cf8c63a2..60ae8497d8d7b 100644
--- a/mlir/include/mlir/IR/Function.h
+++ b/mlir/include/mlir/IR/Function.h
@@ -51,10 +51,8 @@ class FuncOp
                        ArrayRef<MutableDictionaryAttr> argAttrs);
 
   static void build(OpBuilder &builder, OperationState &result, StringRef name,
-                    FunctionType type, ArrayRef<NamedAttribute> attrs);
-  static void build(OpBuilder &builder, OperationState &result, StringRef name,
-                    FunctionType type, ArrayRef<NamedAttribute> attrs,
-                    ArrayRef<MutableDictionaryAttr> argAttrs);
+                    FunctionType type, ArrayRef<NamedAttribute> attrs = {},
+                    ArrayRef<MutableDictionaryAttr> argAttrs = {});
 
   /// Operation hooks.
   static ParseResult parse(OpAsmParser &parser, OperationState &result);
diff --git a/mlir/include/mlir/IR/Location.h b/mlir/include/mlir/IR/Location.h
index ea6c954f9aac8..c1b6dd016db77 100644
--- a/mlir/include/mlir/IR/Location.h
+++ b/mlir/include/mlir/IR/Location.h
@@ -208,7 +208,8 @@ class NameLoc : public Attribute::AttrBase<NameLoc, LocationAttr,
 
 /// Represents an unknown location.  This is always a singleton for a given
 /// MLIRContext.
-class UnknownLoc : public Attribute::AttrBase<UnknownLoc, LocationAttr> {
+class UnknownLoc
+    : public Attribute::AttrBase<UnknownLoc, LocationAttr, AttributeStorage> {
 public:
   using Base::Base;
 
diff --git a/mlir/include/mlir/IR/Matchers.h b/mlir/include/mlir/IR/Matchers.h
index 0f74f1b9cd436..72e17e1699a0d 100644
--- a/mlir/include/mlir/IR/Matchers.h
+++ b/mlir/include/mlir/IR/Matchers.h
@@ -97,9 +97,9 @@ struct constant_int_op_binder {
       return false;
     auto type = op->getResult(0).getType();
 
-    if (type.isa<IntegerType>() || type.isa<IndexType>())
+    if (type.isa<IntegerType, IndexType>())
       return attr_value_binder<IntegerAttr>(bind_value).match(attr);
-    if (type.isa<VectorType>() || type.isa<RankedTensorType>()) {
+    if (type.isa<VectorType, RankedTensorType>()) {
       if (auto splatAttr = attr.dyn_cast<SplatElementsAttr>()) {
         return attr_value_binder<IntegerAttr>(bind_value)
             .match(splatAttr.getSplatValue());
diff --git a/mlir/include/mlir/IR/Module.h b/mlir/include/mlir/IR/Module.h
index b0b28a72d78ff..8a5101337586d 100644
--- a/mlir/include/mlir/IR/Module.h
+++ b/mlir/include/mlir/IR/Module.h
@@ -13,6 +13,7 @@
 #ifndef MLIR_IR_MODULE_H
 #define MLIR_IR_MODULE_H
 
+#include "mlir/IR/OwningOpRefBase.h"
 #include "mlir/IR/SymbolTable.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
 
@@ -33,7 +34,7 @@ class ModuleOp
           OpTrait::IsIsolatedFromAbove, OpTrait::AffineScope,
           OpTrait::SymbolTable,
           OpTrait::SingleBlockImplicitTerminator<ModuleTerminatorOp>::Impl,
-          SymbolOpInterface::Trait> {
+          SymbolOpInterface::Trait, OpTrait::NoRegionArguments> {
 public:
   using Op::Op;
   using Op::print;
@@ -122,40 +123,10 @@ class ModuleTerminatorOp
 };
 
 /// This class acts as an owning reference to a module, and will automatically
-/// destroy the held module if valid.
-class OwningModuleRef {
+/// destroy the held module on destruction if the held module is valid.
+class OwningModuleRef : public OwningOpRefBase<ModuleOp> {
 public:
-  OwningModuleRef(std::nullptr_t = nullptr) {}
-  OwningModuleRef(ModuleOp module) : module(module) {}
-  OwningModuleRef(OwningModuleRef &&other) : module(other.release()) {}
-  ~OwningModuleRef() {
-    if (module)
-      module.erase();
-  }
-
-  // Assign from another module reference.
-  OwningModuleRef &operator=(OwningModuleRef &&other) {
-    if (module)
-      module.erase();
-    module = other.release();
-    return *this;
-  }
-
-  /// Allow accessing the internal module.
-  ModuleOp get() const { return module; }
-  ModuleOp operator*() const { return module; }
-  ModuleOp *operator->() { return &module; }
-  explicit operator bool() const { return module; }
-
-  /// Release the referenced module.
-  ModuleOp release() {
-    ModuleOp released;
-    std::swap(released, module);
-    return released;
-  }
-
-private:
-  ModuleOp module;
+  using OwningOpRefBase<ModuleOp>::OwningOpRefBase;
 };
 
 } // end namespace mlir
diff --git a/mlir/include/mlir/IR/OpAsmInterface.td b/mlir/include/mlir/IR/OpAsmInterface.td
index 752536a9e9a10..ec50288348c4c 100644
--- a/mlir/include/mlir/IR/OpAsmInterface.td
+++ b/mlir/include/mlir/IR/OpAsmInterface.td
@@ -22,6 +22,7 @@ def OpAsmOpInterface : OpInterface<"OpAsmOpInterface"> {
     This interface provides hooks to interact with the AsmPrinter and AsmParser
     classes.
   }];
+  let cppNamespace = "::mlir";
 
   let methods = [
     InterfaceMethod<[{
diff --git a/mlir/include/mlir/IR/OpBase.td b/mlir/include/mlir/IR/OpBase.td
index 232a6ddcd9c6d..9cc57a6172894 100644
--- a/mlir/include/mlir/IR/OpBase.td
+++ b/mlir/include/mlir/IR/OpBase.td
@@ -144,7 +144,7 @@ class Concat<string pre, Pred child, string suf> :
 // Constraint definitions
 //===----------------------------------------------------------------------===//
 
-// TODO(b/130064155): Merge Constraints into Pred.
+// TODO: Merge Constraints into Pred.
 
 // Base class for named constraints.
 //
@@ -208,25 +208,27 @@ class SuccessorConstraint<Pred predicate, string description = ""> :
 //===----------------------------------------------------------------------===//
 
 // Whether a type is a VectorType.
-def IsVectorTypePred : CPred<"$_self.isa<VectorType>()">;
+def IsVectorTypePred : CPred<"$_self.isa<::mlir::VectorType>()">;
 
 // Whether a type is a TensorType.
-def IsTensorTypePred : CPred<"$_self.isa<TensorType>()">;
+def IsTensorTypePred : CPred<"$_self.isa<::mlir::TensorType>()">;
 
 // Whether a type is a MemRefType.
-def IsMemRefTypePred : CPred<"$_self.isa<MemRefType>()">;
+def IsMemRefTypePred : CPred<"$_self.isa<::mlir::MemRefType>()">;
 
 // Whether a type is an  IsUnrankedMemRefType
-def IsUnrankedMemRefTypePred : CPred<"$_self.isa<UnrankedMemRefType>()">;
+def IsUnrankedMemRefTypePred
+        : CPred<"$_self.isa<::mlir::UnrankedMemRefType>()">;
 
 // Whether a type is a ShapedType.
-def IsShapedTypePred : CPred<"$_self.isa<ShapedType>()">;
+def IsShapedTypePred : CPred<"$_self.isa<::mlir::ShapedType>()">;
 
 // For a ShapedType, verify that it has a static shape.
-def HasStaticShapePred : CPred<"$_self.cast<ShapedType>().hasStaticShape()">;
+def HasStaticShapePred :
+        CPred<"$_self.cast<::mlir::ShapedType>().hasStaticShape()">;
 
 // Whether a type is a TupleType.
-def IsTupleTypePred : CPred<"$_self.isa<TupleType>()">;
+def IsTupleTypePred : CPred<"$_self.isa<::mlir::TupleType>()">;
 
 //===----------------------------------------------------------------------===//
 // Dialect definitions
@@ -312,8 +314,8 @@ class Optional<Type type> : TypeConstraint<type.predicate, type.description> {
 // Note that this does not "inherit" from Type because it would require
 // duplicating Type subclasses for buildable and non-buildable cases to avoid
 // diamond "inheritance".
-// TODO(zinenko): we may extend this to a more general 'Buildable' trait,
-// making some Types and some Attrs buildable.
+// TODO: we may extend this to a more general 'Buildable' trait, making some
+// Types and some Attrs buildable.
 class BuildableType<code builder> {
   // The builder call to invoke (if specified) to construct the BuildableType.
   code builderCall = builder;
@@ -323,8 +325,8 @@ class BuildableType<code builder> {
 def AnyType : Type<CPred<"true">, "any type">;
 
 // None type
-def NoneType : Type<CPred<"$_self.isa<NoneType>()">, "none type">,
-      BuildableType<"$_builder.getType<NoneType>()">;
+def NoneType : Type<CPred<"$_self.isa<::mlir::NoneType>()">, "none type">,
+      BuildableType<"$_builder.getType<::mlir::NoneType>()">;
 
 // Any type from the given list
 class AnyTypeOf<list<Type> allowedTypes, string description = ""> : Type<
@@ -337,7 +339,7 @@ class AnyTypeOf<list<Type> allowedTypes, string description = ""> : Type<
 // Integer types.
 
 // Any integer type irrespective of its width and signedness semantics.
-def AnyInteger : Type<CPred<"$_self.isa<IntegerType>()">, "integer">;
+def AnyInteger : Type<CPred<"$_self.isa<::mlir::IntegerType>()">, "integer">;
 
 // Any integer type (regardless of signedness semantics) of a specific width.
 class AnyI<int width>
@@ -424,13 +426,13 @@ def UI32 : UI<32>;
 def UI64 : UI<64>;
 
 // Index type.
-def Index : Type<CPred<"$_self.isa<IndexType>()">, "index">,
+def Index : Type<CPred<"$_self.isa<::mlir::IndexType>()">, "index">,
             BuildableType<"$_builder.getIndexType()">;
 
 // Floating point types.
 
 // Any float type irrespective of its width.
-def AnyFloat : Type<CPred<"$_self.isa<FloatType>()">, "floating-point">;
+def AnyFloat : Type<CPred<"$_self.isa<::mlir::FloatType>()">, "floating-point">;
 
 // Float type of a specific width.
 class F<int width>
@@ -453,25 +455,28 @@ def BF16 : Type<CPred<"$_self.isBF16()">, "bfloat16 type">,
 
 class Complex<Type type>
     : Type<And<[
-          CPred<"$_self.isa<ComplexType>()">,
-          SubstLeaves<"$_self", "$_self.cast<ComplexType>().getElementType()",
+          CPred<"$_self.isa<::mlir::ComplexType>()">,
+          SubstLeaves<"$_self",
+                      "$_self.cast<::mlir::ComplexType>().getElementType()",
            type.predicate>]>,
            "complex type with " # type.description # " elements"> {
   Type elementType = type;
 }
 
-def AnyComplex : Type<CPred<"$_self.isa<ComplexType>()">, "complex-type">;
+def AnyComplex : Type<CPred<"$_self.isa<::mlir::ComplexType>()">,
+                            "complex-type">;
 
 class OpaqueType<string dialect, string name, string description>
   : Type<CPred<"isOpaqueTypeWithName($_self, \""#dialect#"\", \""#name#"\")">,
          description>,
-    BuildableType<"OpaqueType::get($_builder.getIdentifier(\"" # dialect #
-                  "\"), \"" # name # "\", $_builder.getContext())">;
+    BuildableType<"::mlir::OpaqueType::get($_builder.getIdentifier(\""
+                  # dialect # "\"), \"" # name # "\", $_builder.getContext())">;
 
 // Function Type
 
 // Any function type.
-def FunctionType : Type<CPred<"$_self.isa<FunctionType>()">, "function type">;
+def FunctionType : Type<CPred<"$_self.isa<::mlir::FunctionType>()">,
+                              "function type">;
 
 // A container type is a type that has another type embedded within it.
 class ContainerType<Type etype, Pred containerPred, code elementTypeCall,
@@ -489,18 +494,21 @@ class ContainerType<Type etype, Pred containerPred, code elementTypeCall,
   code getElementTypeCall = elementTypeCall;
 }
 
-class ShapedContainerType<list<Type> allowedTypes, Pred containerPred, string descr> :
+class ShapedContainerType<list<Type> allowedTypes,
+                          Pred containerPred, string descr> :
     ContainerType<AnyTypeOf<allowedTypes>, containerPred,
-                  "$_self.cast<ShapedType>().getElementType()", descr>;
+                  "$_self.cast<::mlir::ShapedType>().getElementType()", descr>;
 
 // Whether a shaped type is ranked.
-def HasRankPred : CPred<"$_self.cast<ShapedType>().hasRank()">;
+def HasRankPred : CPred<"$_self.cast<::mlir::ShapedType>().hasRank()">;
 
 // Whether a shaped type has one of the specified ranks.
 class HasAnyRankOfPred<list<int> ranks> : And<[
     HasRankPred,
     Or<!foreach(rank, ranks,
-                CPred<"$_self.cast<ShapedType>().getRank() == " # rank>)>]>;
+                CPred<[{$_self.cast<::mlir::ShapedType>().getRank()
+                         == }]
+                      # rank>)>]>;
 
 // Vector types.
 
@@ -512,7 +520,7 @@ class VectorOf<list<Type> allowedTypes> :
 class IsVectorOfRankPred<list<int> allowedRanks> :
   And<[IsVectorTypePred,
        Or<!foreach(allowedlength, allowedRanks,
-                   CPred<[{$_self.cast<VectorType>().getRank()
+                   CPred<[{$_self.cast<::mlir::VectorType>().getRank()
                            == }]
                          # allowedlength>)>]>;
 
@@ -535,7 +543,7 @@ class VectorOfRankAndType<list<int> allowedRanks,
 class IsVectorOfLengthPred<list<int> allowedLengths> :
   And<[IsVectorTypePred,
        Or<!foreach(allowedlength, allowedLengths,
-                   CPred<[{$_self.cast<VectorType>().getNumElements()
+                   CPred<[{$_self.cast<::mlir::VectorType>().getNumElements()
                            == }]
                          # allowedlength>)>]>;
 
@@ -570,7 +578,7 @@ def AnyRankedTensor :
   ShapedContainerType<[AnyType], And<[IsTensorTypePred, HasRankPred]>,
   "ranked tensor">;
 
-// TODO(b/130064155) Have an easy way to add another constraint to a type.
+// TODO: Have an easy way to add another constraint to a type.
 class StaticShapeTensorOf<list<Type> allowedTypes>
     : Type<And<[TensorOf<allowedTypes>.predicate, HasStaticShapePred]>,
            "statically shaped " # TensorOf<allowedTypes>.description>;
@@ -628,7 +636,7 @@ def F16MemRef  : MemRefOf<[F16]>;
 def F32MemRef  : MemRefOf<[F32]>;
 def F64MemRef  : MemRefOf<[F64]>;
 
-// TODO(b/130064155) Have an easy way to add another constraint to a type.
+// TODO: Have an easy way to add another constraint to a type.
 class MemRefRankOf<list<Type> allowedTypes, list<int> ranks> :
     Type<And<[MemRefOf<allowedTypes>.predicate, HasAnyRankOfPred<ranks>]>,
          StrJoin<!foreach(rank, ranks, rank # "D"), "/">.result # " " #
@@ -641,7 +649,7 @@ class StaticShapeMemRefOf<list<Type> allowedTypes>
 def AnyStaticShapeMemRef : StaticShapeMemRefOf<[AnyType]>;
 
 // For a MemRefType, verify that it has strides.
-def HasStridesPred : CPred<[{ isStrided($_self.cast<MemRefType>()) }]>;
+def HasStridesPred : CPred<[{ isStrided($_self.cast<::mlir::MemRefType>()) }]>;
 
 class StridedMemRefOf<list<Type> allowedTypes>
     : Type<And<[MemRefOf<allowedTypes>.predicate, HasStridesPred]>,
@@ -666,7 +674,7 @@ class MixedContainerType<Type etype, Pred containerPred, code elementTypesCall,
         And<[
             containerPred,
             Concat<
-                "llvm::all_of(" # elementTypesCall # ", [](Type t) { return ",
+                "::llvm::all_of(" # elementTypesCall # ", [](Type t) { return ",
                 SubstLeaves<"$_self", "t", etype.predicate>,
                 "; })"
             >
@@ -682,13 +690,14 @@ class MixedContainerType<Type etype, Pred containerPred, code elementTypesCall,
 // A Tuple that holds a mix of elements of the allowed types.
 class TupleOf<list<Type> allowedTypes>
     : MixedContainerType<AnyTypeOf<allowedTypes>, IsTupleTypePred,
-                         "$_self.cast<TupleType>().getTypes()", "tuple">;
+                         "$_self.cast<::mlir::TupleType>().getTypes()",
+                         "tuple">;
 
 // A Tuple with arbitrary nesting, where all elements are a mix of the allowed
 // types.
 class NestedTupleOf<list<Type> allowedTypes> :
     MixedContainerType<AnyTypeOf<allowedTypes>, IsTupleTypePred,
-                       "getFlattenedTypes($_self.cast<TupleType>())",
+                       "getFlattenedTypes($_self.cast<::mlir::TupleType>())",
                        "nested tuple">;
 
 //===----------------------------------------------------------------------===//
@@ -806,9 +815,9 @@ class OptionalAttr<Attr attr> : Attr<attr.predicate, attr.description> {
   // Rewrite the attribute to be optional.
   // Note: this has to be kept up to date with Attr above.
   let storageType = attr.storageType;
-  let returnType = "Optional<" # attr.returnType #">";
+  let returnType = "::llvm::Optional<" # attr.returnType #">";
   let convertFromStorage = "$_self ? " # returnType # "(" #
-                           attr.convertFromStorage # ") : (llvm::None)";
+                           attr.convertFromStorage # ") : (::llvm::None)";
   let valueType = attr.valueType;
   let isOptional = 1;
 
@@ -825,20 +834,20 @@ class TypedAttrBase<Type attrValType, string attrKind, Pred condition,
     Attr<condition, descr> {
   let constBuilderCall = "$_builder.get" # attrKind # "(" #
                          attrValType.builderCall # ", $0)";
-  let storageType = attrKind;
+  let storageType = "::mlir::" # attrKind;
   let valueType = attrValType;
 }
 
 // Any attribute.
 def AnyAttr : Attr<CPred<"true">, "any attribute"> {
-  let storageType = "Attribute";
-  let returnType = "Attribute";
+  let storageType = "::mlir::Attribute";
+  let returnType = "::mlir::Attribute";
   let convertFromStorage = "$_self";
   let constBuilderCall = "$0";
 }
 
-def BoolAttr : Attr<CPred<"$_self.isa<BoolAttr>()">, "bool attribute"> {
-  let storageType = [{ BoolAttr }];
+def BoolAttr : Attr<CPred<"$_self.isa<::mlir::BoolAttr>()">, "bool attribute"> {
+  let storageType = [{ ::mlir::BoolAttr }];
   let returnType = [{ bool }];
   let valueType = I1;
   let constBuilderCall = "$_builder.getBoolAttr($0)";
@@ -848,10 +857,11 @@ def BoolAttr : Attr<CPred<"$_self.isa<BoolAttr>()">, "bool attribute"> {
 def IndexAttr :
     TypedAttrBase<
       Index, "IntegerAttr",
-      And<[CPred<"$_self.isa<IntegerAttr>()">,
-           CPred<"$_self.cast<IntegerAttr>().getType().isa<IndexType>()">]>,
+      And<[CPred<"$_self.isa<::mlir::IntegerAttr>()">,
+           CPred<"$_self.cast<::mlir::IntegerAttr>().getType()"
+                 ".isa<::mlir::IndexType>()">]>,
       "index attribute"> {
-  let returnType = [{ APInt }];
+  let returnType = [{ ::llvm::APInt }];
 }
 
 // Base class for any integer (regardless of signedness semantics) attributes
@@ -859,11 +869,11 @@ def IndexAttr :
 class AnyIntegerAttrBase<AnyI attrValType, string descr> :
     TypedAttrBase<
       attrValType, "IntegerAttr",
-      And<[CPred<"$_self.isa<IntegerAttr>()">,
-           CPred<"$_self.cast<IntegerAttr>().getType()."
+      And<[CPred<"$_self.isa<::mlir::IntegerAttr>()">,
+           CPred<"$_self.cast<::mlir::IntegerAttr>().getType()."
                  "isInteger(" # attrValType.bitwidth # ")">]>,
       descr> {
-  let returnType = [{ APInt }];
+  let returnType = [{ ::llvm::APInt }];
   let constBuilderCall = ?;
 }
 
@@ -873,21 +883,21 @@ def AnyI16Attr : AnyIntegerAttrBase<AnyI16, "16-bit integer attribute">;
 def AnyI32Attr : AnyIntegerAttrBase<AnyI32, "32-bit integer attribute">;
 def AnyI64Attr : AnyIntegerAttrBase<AnyI64, "64-bit integer attribute">;
 
-def APIntAttr : Attr<CPred<"$_self.isa<IntegerAttr>()">,
+def APIntAttr : Attr<CPred<"$_self.isa<::mlir::IntegerAttr>()">,
                      "arbitrary integer attribute"> {
-  let storageType = [{ IntegerAttr }];
-  let returnType = [{ APInt }];
+  let storageType = [{ ::mlir::IntegerAttr }];
+  let returnType = [{ ::mlir::APInt }];
 }
 
 // Base class for signless integer attributes of fixed width.
 class SignlessIntegerAttrBase<I attrValType, string descr> :
     TypedAttrBase<
       attrValType, "IntegerAttr",
-      And<[CPred<"$_self.isa<IntegerAttr>()">,
-           CPred<"$_self.cast<IntegerAttr>().getType()."
+      And<[CPred<"$_self.isa<::mlir::IntegerAttr>()">,
+           CPred<"$_self.cast<::mlir::IntegerAttr>().getType()."
                  "isSignlessInteger(" # attrValType.bitwidth # ")">]>,
       descr> {
-  let returnType = [{ APInt }];
+  let returnType = [{ ::llvm::APInt }];
 }
 
 def I1Attr  : SignlessIntegerAttrBase<I1,  "1-bit signless integer attribute">;
@@ -900,11 +910,11 @@ def I64Attr : SignlessIntegerAttrBase<I64, "64-bit signless integer attribute">;
 class SignedIntegerAttrBase<SI attrValType, string descr> :
     TypedAttrBase<
       attrValType, "IntegerAttr",
-      And<[CPred<"$_self.isa<IntegerAttr>()">,
-           CPred<"$_self.cast<IntegerAttr>().getType()."
+      And<[CPred<"$_self.isa<::mlir::IntegerAttr>()">,
+           CPred<"$_self.cast<::mlir::IntegerAttr>().getType()."
                  "isSignedInteger(" # attrValType.bitwidth # ")">]>,
       descr> {
-  let returnType = [{ APInt }];
+  let returnType = [{ ::llvm::APInt }];
 }
 
 def SI1Attr  : SignedIntegerAttrBase<
@@ -922,11 +932,11 @@ def SI64Attr : SignedIntegerAttrBase<
 class UnsignedIntegerAttrBase<UI attrValType, string descr> :
     TypedAttrBase<
       attrValType, "IntegerAttr",
-      And<[CPred<"$_self.isa<IntegerAttr>()">,
-           CPred<"$_self.cast<IntegerAttr>().getType()."
+      And<[CPred<"$_self.isa<::mlir::IntegerAttr>()">,
+           CPred<"$_self.cast<::mlir::IntegerAttr>().getType()."
                  "isUnsignedInteger(" # attrValType.bitwidth # ")">]>,
       descr> {
-  let returnType = [{ APInt }];
+  let returnType = [{ ::llvm::APInt }];
 }
 
 def UI1Attr  : UnsignedIntegerAttrBase<
@@ -943,11 +953,11 @@ def UI64Attr : UnsignedIntegerAttrBase<
 // Base class for float attributes of fixed width.
 class FloatAttrBase<F attrValType, string descr> :
     TypedAttrBase<attrValType, "FloatAttr",
-              And<[CPred<"$_self.isa<FloatAttr>()">,
-                     CPred<"$_self.cast<FloatAttr>().getType().isF" #
+              And<[CPred<"$_self.isa<::mlir::FloatAttr>()">,
+                     CPred<"$_self.cast<::mlir::FloatAttr>().getType().isF" #
                            attrValType.bitwidth # "()">]>,
               descr> {
-  let returnType = [{ APFloat }];
+  let returnType = [{ ::llvm::APFloat }];
 }
 
 def F32Attr : FloatAttrBase<F32, "32-bit float attribute">;
@@ -956,17 +966,18 @@ def F64Attr : FloatAttrBase<F64, "64-bit float attribute">;
 // An attribute backed by a string type.
 class StringBasedAttr<Pred condition, string descr> : Attr<condition, descr> {
   let constBuilderCall = "$_builder.getStringAttr(\"$0\")";
-  let storageType = [{ StringAttr }];
-  let returnType = [{ StringRef }];
+  let storageType = [{ ::mlir::StringAttr }];
+  let returnType = [{ ::llvm::StringRef }];
   let valueType = NoneType;
 }
 
-def StrAttr : StringBasedAttr<CPred<"$_self.isa<StringAttr>()">,
+def StrAttr : StringBasedAttr<CPred<"$_self.isa<::mlir::StringAttr>()">,
                               "string attribute">;
 
 // String attribute that has a specific value type.
-class TypedStrAttr<Type ty> : StringBasedAttr<CPred<"$_self.isa<StringAttr>()">,
-                                                    "string attribute"> {
+class TypedStrAttr<Type ty>
+    : StringBasedAttr<CPred<"$_self.isa<::mlir::StringAttr>()">,
+                            "string attribute"> {
   let valueType = ty;
 }
 
@@ -975,22 +986,23 @@ class TypedStrAttr<Type ty> : StringBasedAttr<CPred<"$_self.isa<StringAttr>()">,
 // defines a type attribute containing an integer type.
 class TypeAttrBase<string retType, string description> :
     Attr<And<[
-      CPred<"$_self.isa<TypeAttr>()">,
-      CPred<"$_self.cast<TypeAttr>().getValue().isa<" # retType # ">()">]>,
+      CPred<"$_self.isa<::mlir::TypeAttr>()">,
+      CPred<"$_self.cast<::mlir::TypeAttr>().getValue().isa<"
+            # retType # ">()">]>,
     description> {
-  let storageType = [{ TypeAttr }];
+  let storageType = [{ ::mlir::TypeAttr }];
   let returnType = retType;
   let valueType = NoneType;
   let convertFromStorage = "$_self.getValue().cast<" # retType # ">()";
 }
 
-def TypeAttr : TypeAttrBase<"Type", "any type attribute">;
+def TypeAttr : TypeAttrBase<"::mlir::Type", "any type attribute">;
 
 // The mere presence of unit attributes has a meaning.  Therefore, unit
 // attributes are always treated as optional and accessors to them return
 // "true" if the attribute is present and "false" otherwise.
-def UnitAttr : Attr<CPred<"$_self.isa<UnitAttr>()">, "unit attribute"> {
-  let storageType = [{ UnitAttr }];
+def UnitAttr : Attr<CPred<"$_self.isa<::mlir::UnitAttr>()">, "unit attribute"> {
+  let storageType = [{ ::mlir::UnitAttr }];
   let constBuilderCall = "$_builder.getUnitAttr()";
   let convertFromStorage = "$_self != nullptr";
   let returnType = "bool";
@@ -1019,7 +1031,7 @@ class EnumAttrCaseInfo<string sym, int intVal, string strVal> {
 class StrEnumAttrCase<string sym, int val = -1> :
     EnumAttrCaseInfo<sym, val, sym>,
     StringBasedAttr<
-      CPred<"$_self.cast<StringAttr>().getValue() == \"" # sym # "\"">,
+      CPred<"$_self.cast<::mlir::StringAttr>().getValue() == \"" # sym # "\"">,
       "case " # sym>;
 
 // An enum attribute case stored with IntegerAttr, which has an integer value,
@@ -1028,7 +1040,7 @@ class IntEnumAttrCaseBase<I intType, string sym, string strVal, int intVal> :
     EnumAttrCaseInfo<sym, intVal, strVal>,
     SignlessIntegerAttrBase<intType, "case " # strVal> {
   let predicate =
-    CPred<"$_self.cast<IntegerAttr>().getInt() == " # intVal>;
+    CPred<"$_self.cast<::mlir::IntegerAttr>().getInt() == " # intVal>;
 }
 
 // Cases of integer enum attributes with a specific type. By default, the string
@@ -1045,7 +1057,8 @@ class BitEnumAttrCase<string sym, int val> :
     EnumAttrCaseInfo<sym, val, sym>,
     SignlessIntegerAttrBase<I32, "case " # sym> {
   let predicate = CPred<
-    "$_self.cast<IntegerAttr>().getValue().getZExtValue() & " # val # "u">;
+    "$_self.cast<::mlir::IntegerAttr>().getValue().getZExtValue() & "
+    # val # "u">;
 }
 
 // Additional information for an enum attribute.
@@ -1070,7 +1083,7 @@ class EnumAttrInfo<string name, list<EnumAttrCaseInfo> cases> {
   // use "::" as the delimiter, e.g., given "A::B", generated code will be
   // placed in `namespace A { namespace B { ... } }`. To avoid placing in any
   // namespace, use "".
-  // TODO(b/134741431): use dialect to provide the namespace.
+  // TODO: use dialect to provide the namespace.
   string cppNamespace = "";
 
   // The name of the utility function that converts a value of the underlying
@@ -1096,7 +1109,7 @@ class EnumAttrInfo<string name, list<EnumAttrCaseInfo> cases> {
   // <return-type> <fn-name>(<qualified-enum-class-name>);
   // ```
   string symbolToStringFnName = "stringify" # name;
-  string symbolToStringFnRetType = "llvm::StringRef";
+  string symbolToStringFnRetType = "::llvm::StringRef";
 
   // The name of the utility function that returns the max enum value used
   // within the enum class. It will have the following signature:
@@ -1143,7 +1156,8 @@ class I32EnumAttr<string name, string description,
   let returnType = cppNamespace # "::" # name;
   let underlyingType = "uint32_t";
   let convertFromStorage = "static_cast<" # returnType # ">($_self.getInt())";
-  let constBuilderCall = "$_builder.getI32IntegerAttr(static_cast<int32_t>($0))";
+  let constBuilderCall =
+          "$_builder.getI32IntegerAttr(static_cast<int32_t>($0))";
 }
 class I64EnumAttr<string name, string description,
                   list<I64EnumAttrCase> cases> :
@@ -1151,7 +1165,8 @@ class I64EnumAttr<string name, string description,
   let returnType = cppNamespace # "::" # name;
   let underlyingType = "uint64_t";
   let convertFromStorage = "static_cast<" # returnType # ">($_self.getInt())";
-  let constBuilderCall = "$_builder.getI64IntegerAttr(static_cast<int64_t>($0))";
+  let constBuilderCall =
+          "$_builder.getI64IntegerAttr(static_cast<int64_t>($0))";
 }
 
 // A bit enum stored with 32-bit IntegerAttr.
@@ -1166,15 +1181,16 @@ class BitEnumAttr<string name, string description,
   let predicate = And<[
     I32Attr.predicate,
     // Make sure we don't have unknown bit set.
-    CPred<"!($_self.cast<IntegerAttr>().getValue().getZExtValue() & (~(" #
-          StrJoin<!foreach(case, cases, case.value # "u"), "|">.result #
+    CPred<"!($_self.cast<::mlir::IntegerAttr>().getValue().getZExtValue() & (~("
+          # StrJoin<!foreach(case, cases, case.value # "u"), "|">.result #
           ")))">
   ]>;
 
   let returnType = cppNamespace # "::" # name;
   let underlyingType = "uint32_t";
   let convertFromStorage = "static_cast<" # returnType # ">($_self.getInt())";
-  let constBuilderCall = "$_builder.getI32IntegerAttr(static_cast<int32_t>($0))";
+  let constBuilderCall =
+          "$_builder.getI32IntegerAttr(static_cast<int32_t>($0))";
 
   // We need to return a string because we may concatenate symbols for multiple
   // bits together.
@@ -1189,44 +1205,45 @@ class BitEnumAttr<string name, string description,
 
 class DictionaryAttrBase<Pred condition, string description> :
     Attr<condition, description> {
-  let storageType = [{ DictionaryAttr }];
-  let returnType = [{ DictionaryAttr }];
+  let storageType = [{ ::mlir::DictionaryAttr }];
+  let returnType = [{ ::mlir::DictionaryAttr }];
   let valueType = NoneType;
   let convertFromStorage = "$_self";
 }
 
-def DictionaryAttr : DictionaryAttrBase<CPred<"$_self.isa<DictionaryAttr>()">,
-                                        "dictionary of named attribute values">;
+def DictionaryAttr 
+    : DictionaryAttrBase<CPred<"$_self.isa<::mlir::DictionaryAttr>()">,
+                               "dictionary of named attribute values">;
 
 class ElementsAttrBase<Pred condition, string description> :
     Attr<condition, description> {
-  let storageType = [{ ElementsAttr }];
-  let returnType = [{ ElementsAttr }];
+  let storageType = [{ ::mlir::ElementsAttr }];
+  let returnType = [{ ::mlir::ElementsAttr }];
   let convertFromStorage = "$_self";
 }
 
-def ElementsAttr : ElementsAttrBase<CPred<"$_self.isa<ElementsAttr>()">,
+def ElementsAttr : ElementsAttrBase<CPred<"$_self.isa<::mlir::ElementsAttr>()">,
                                     "constant vector/tensor attribute">;
 
 class IntElementsAttrBase<Pred condition, string description> :
-    ElementsAttrBase<And<[CPred<"$_self.isa<DenseIntElementsAttr>()">,
+    ElementsAttrBase<And<[CPred<"$_self.isa<::mlir::DenseIntElementsAttr>()">,
                           condition]>,
                      description> {
-  let storageType = [{ DenseIntElementsAttr }];
-  let returnType = [{ DenseIntElementsAttr }];
+  let storageType = [{ ::mlir::DenseIntElementsAttr }];
+  let returnType = [{ ::mlir::DenseIntElementsAttr }];
 
   let convertFromStorage = "$_self";
 }
 
 def IndexElementsAttr
-    : IntElementsAttrBase<CPred<[{$_self.cast<DenseIntElementsAttr>()
+    : IntElementsAttrBase<CPred<[{$_self.cast<::mlir::DenseIntElementsAttr>()
                                       .getType()
                                       .getElementType()
                                       .isIndex()}]>,
                           "index elements attribute">;
 
 class AnyIntElementsAttr<int width> : IntElementsAttrBase<
-  CPred<"$_self.cast<DenseIntElementsAttr>().getType()."
+  CPred<"$_self.cast<::mlir::DenseIntElementsAttr>().getType()."
         "getElementType().isInteger(" # width # ")">,
   width # "-bit integer elements attribute">;
 
@@ -1234,14 +1251,15 @@ def AnyI32ElementsAttr : AnyIntElementsAttr<32>;
 def AnyI64ElementsAttr : AnyIntElementsAttr<64>;
 
 class SignlessIntElementsAttr<int width> : IntElementsAttrBase<
-  CPred<"$_self.cast<DenseIntElementsAttr>().getType()."
+  CPred<"$_self.cast<::mlir::DenseIntElementsAttr>().getType()."
         "getElementType().isSignlessInteger(" # width # ")">,
   width # "-bit signless integer elements attribute"> {
 
   // Note that this is only constructing scalar elements attribute.
-  let constBuilderCall = "DenseElementsAttr::get("
-    "RankedTensorType::get({}, $_builder.getIntegerType(" # width # ")), "
-    "llvm::makeArrayRef($0)).cast<DenseIntElementsAttr>()";
+  let constBuilderCall = "::mlir::DenseElementsAttr::get("
+    "::mlir::RankedTensorType::get({}, "
+                                  "$_builder.getIntegerType(" # width # ")), "
+    "::llvm::makeArrayRef($0)).cast<::mlir::DenseIntElementsAttr>()";
 }
 
 def I32ElementsAttr : SignlessIntElementsAttr<32>;
@@ -1254,15 +1272,15 @@ class RankedSignlessIntElementsAttr<int width, list<int> dims> :
   // Check that this has the specified shape.
   let predicate = And<[
     SignlessIntElementsAttr<width>.predicate,
-    CPred<"$_self.cast<DenseIntElementsAttr>().getType().getShape() == "
-        "ArrayRef<int64_t>({" # StrJoinInt<dims>.result # "})">]>;
+    CPred<"$_self.cast<::mlir::DenseIntElementsAttr>().getType().getShape() == "
+        "::mlir::ArrayRef<int64_t>({" # StrJoinInt<dims>.result # "})">]>;
 
   let description = width # "-bit signless int elements attribute of shape [" #
                     StrJoinInt<dims>.result # "]";
 
-  let constBuilderCall = "DenseIntElementsAttr::get("
-    "RankedTensorType::get({" # StrJoinInt<dims>.result #
-    "}, $_builder.getIntegerType(" # width # ")), makeArrayRef($0))";
+  let constBuilderCall = "::mlir::DenseIntElementsAttr::get("
+    "::mlir::RankedTensorType::get({" # StrJoinInt<dims>.result #
+    "}, $_builder.getIntegerType(" # width # ")), ::llvm::makeArrayRef($0))";
 }
 
 class RankedI32ElementsAttr<list<int> dims> :
@@ -1271,18 +1289,18 @@ class RankedI64ElementsAttr<list<int> dims> :
     RankedSignlessIntElementsAttr<64, dims>;
 
 class FloatElementsAttr<int width> : ElementsAttrBase<
-  CPred<"$_self.isa<DenseFPElementsAttr>() &&"
-      "$_self.cast<DenseElementsAttr>().getType()."
+  CPred<"$_self.isa<::mlir::DenseFPElementsAttr>() &&"
+      "$_self.cast<::mlir::DenseElementsAttr>().getType()."
       "getElementType().isF" # width # "()">,
   width # "-bit float elements attribute"> {
 
-  let storageType = [{ DenseElementsAttr }];
-  let returnType = [{ DenseElementsAttr }];
+  let storageType = [{ ::mlir::DenseElementsAttr }];
+  let returnType = [{ ::mlir::DenseElementsAttr }];
 
   // Note that this is only constructing scalar elements attribute.
-  let constBuilderCall = "DenseElementsAttr::get("
-    "RankedTensorType::get({}, $_builder.getF" # width # "Type()),"
-    "llvm::makeArrayRef($0))";
+  let constBuilderCall = "::mlir::DenseElementsAttr::get("
+    "::mlir::RankedTensorType::get({}, $_builder.getF" # width # "Type()),"
+    "::llvm::makeArrayRef($0))";
   let convertFromStorage = "$_self";
 }
 
@@ -1291,23 +1309,23 @@ def F64ElementsAttr : FloatElementsAttr<64>;
 // A `width`-bit floating point elements attribute. The attribute should be
 // ranked and has a shape as specified in `dims`.
 class RankedFloatElementsAttr<int width, list<int> dims> : ElementsAttrBase<
-  CPred<"$_self.isa<DenseFPElementsAttr>() &&"
-      "$_self.cast<DenseFPElementsAttr>().getType()."
+  CPred<"$_self.isa<::mlir::DenseFPElementsAttr>() &&"
+      "$_self.cast<::mlir::DenseFPElementsAttr>().getType()."
       "getElementType().isF" # width # "() && "
       // Check that this is ranked and has the specified shape.
-      "$_self.cast<DenseFPElementsAttr>().getType().hasRank() && "
-      "$_self.cast<DenseFPElementsAttr>().getType().getShape() == "
-      "llvm::ArrayRef<int64_t>({" # StrJoinInt<dims>.result # "})">,
+      "$_self.cast<::mlir::DenseFPElementsAttr>().getType().hasRank() && "
+      "$_self.cast<::mlir::DenseFPElementsAttr>().getType().getShape() == "
+      "::mlir::ArrayRef<int64_t>({" # StrJoinInt<dims>.result # "})">,
   width # "-bit float elements attribute of shape [" #
   StrJoinInt<dims>.result # "]"> {
 
-  let storageType = [{ DenseFPElementsAttr }];
-  let returnType = [{ DenseFPElementsAttr }];
+  let storageType = [{ ::mlir::DenseFPElementsAttr }];
+  let returnType = [{ ::mlir::DenseFPElementsAttr }];
 
-  let constBuilderCall = "DenseElementsAttr::get("
-    "RankedTensorType::get({" # StrJoinInt<dims>.result #
+  let constBuilderCall = "::mlir::DenseElementsAttr::get("
+    "::mlir::RankedTensorType::get({" # StrJoinInt<dims>.result #
     "}, $_builder.getF" # width # "Type()), "
-    "llvm::makeArrayRef($0)).cast<DenseFPElementsAttr>()";
+    "::llvm::makeArrayRef($0)).cast<::mlir::DenseFPElementsAttr>()";
   let convertFromStorage = "$_self";
 }
 
@@ -1315,11 +1333,11 @@ class RankedF32ElementsAttr<list<int> dims> : RankedFloatElementsAttr<32, dims>;
 class RankedF64ElementsAttr<list<int> dims> : RankedFloatElementsAttr<64, dims>;
 
 def StringElementsAttr : ElementsAttrBase<
-  CPred<"$_self.isa<DenseStringElementsAttr>()" >,
+  CPred<"$_self.isa<::mlir::DenseStringElementsAttr>()" >,
   "string elements attribute"> {
 
-  let storageType = [{ DenseElementsAttr }];
-  let returnType = [{ DenseElementsAttr }];
+  let storageType = [{ ::mlir::DenseElementsAttr }];
+  let returnType = [{ ::mlir::DenseElementsAttr }];
 
   let convertFromStorage = "$_self";
 }
@@ -1327,13 +1345,13 @@ def StringElementsAttr : ElementsAttrBase<
 // Base class for array attributes.
 class ArrayAttrBase<Pred condition, string description> :
     Attr<condition, description> {
-  let storageType = [{ ArrayAttr }];
-  let returnType = [{ ArrayAttr }];
+  let storageType = [{ ::mlir::ArrayAttr }];
+  let returnType = [{ ::mlir::ArrayAttr }];
   let valueType = NoneType;
   let convertFromStorage = "$_self";
 }
 
-def ArrayAttr : ArrayAttrBase<CPred<"$_self.isa<ArrayAttr>()">,
+def ArrayAttr : ArrayAttrBase<CPred<"$_self.isa<::mlir::ArrayAttr>()">,
                               "array attribute">;
 
 // Base class for array attributes whose elements are of the same kind.
@@ -1341,12 +1359,12 @@ def ArrayAttr : ArrayAttrBase<CPred<"$_self.isa<ArrayAttr>()">,
 class TypedArrayAttrBase<Attr element, string description>: ArrayAttrBase<
     And<[
       // Guarantee this is an ArrayAttr first
-      CPred<"$_self.isa<ArrayAttr>()">,
+      CPred<"$_self.isa<::mlir::ArrayAttr>()">,
       // Guarantee all elements satisfy the constraints from `element`
-      Concat<"llvm::all_of($_self.cast<ArrayAttr>(), "
-                          "[](Attribute attr) { return ",
-                             SubstLeaves<"$_self", "attr", element.predicate>,
-                          "; })">]>,
+      Concat<"::llvm::all_of($_self.cast<::mlir::ArrayAttr>(), "
+                            "[](::mlir::Attribute attr) { return ",
+                               SubstLeaves<"$_self", "attr", element.predicate>,
+                            "; })">]>,
     description> {
   let constBuilderCall = "$_builder.getArrayAttr($0)";
 
@@ -1413,18 +1431,18 @@ class StructAttr<string name, Dialect dialect,
 }
 
 // Attributes containing symbol references.
-def SymbolRefAttr : Attr<CPred<"$_self.isa<SymbolRefAttr>()">,
+def SymbolRefAttr : Attr<CPred<"$_self.isa<::mlir::SymbolRefAttr>()">,
                         "symbol reference attribute"> {
-  let storageType = [{ SymbolRefAttr }];
-  let returnType = [{ SymbolRefAttr }];
+  let storageType = [{ ::mlir::SymbolRefAttr }];
+  let returnType = [{ ::mlir::SymbolRefAttr }];
   let valueType = NoneType;
   let constBuilderCall = "$_builder.getSymbolRefAttr($0)";
   let convertFromStorage = "$_self";
 }
-def FlatSymbolRefAttr : Attr<CPred<"$_self.isa<FlatSymbolRefAttr>()">,
+def FlatSymbolRefAttr : Attr<CPred<"$_self.isa<::mlir::FlatSymbolRefAttr>()">,
                                    "flat symbol reference attribute"> {
-  let storageType = [{ FlatSymbolRefAttr }];
-  let returnType = [{ StringRef }];
+  let storageType = [{ ::mlir::FlatSymbolRefAttr }];
+  let returnType = [{ ::llvm::StringRef }];
   let valueType = NoneType;
   let constBuilderCall = "$_builder.getSymbolRefAttr($0)";
   let convertFromStorage = "$_self.getValue()";
@@ -1465,7 +1483,7 @@ class DerivedAttr<code ret, code b, code convert = ""> :
 
 // Derived attribute that returns a mlir::Type.
 class DerivedTypeAttr<code body> : DerivedAttr<"Type", body> {
-  let convertFromStorage = "TypeAttr::get($_self)";
+  let convertFromStorage = "::mlir::TypeAttr::get($_self)";
 }
 
 //===----------------------------------------------------------------------===//
@@ -1522,15 +1540,15 @@ class AllAttrConstraintsOf<list<AttrConstraint> constraints> : AttrConstraint<
 }
 
 class IntMinValue<int n> : AttrConstraint<
-    CPred<"$_self.cast<IntegerAttr>().getInt() >= " # n>,
+    CPred<"$_self.cast<::mlir::IntegerAttr>().getInt() >= " # n>,
     "whose minimum value is " # n>;
 
 class IntMaxValue<int n> : AttrConstraint<
-    CPred<"$_self.cast<IntegerAttr>().getInt() <= " # n>,
+    CPred<"$_self.cast<::mlir::IntegerAttr>().getInt() <= " # n>,
     "whose maximum value is " # n>;
 
 def IntNonNegative : AttrConstraint<
-    CPred<"!$_self.cast<IntegerAttr>().getValue().isNegative()">,
+    CPred<"!$_self.cast<::mlir::IntegerAttr>().getValue().isNegative()">,
     "whose value is non-negative">;
 
 def IntPositive : AttrConstraint<
@@ -1538,26 +1556,26 @@ def IntPositive : AttrConstraint<
     "whose value is positive">;
 
 class ArrayMinCount<int n> : AttrConstraint<
-    CPred<"$_self.cast<ArrayAttr>().size() >= " # n>,
+    CPred<"$_self.cast<::mlir::ArrayAttr>().size() >= " # n>,
     "with at least " # n # " elements">;
 
 class ArrayCount<int n> : AttrConstraint<
-    CPred<"$_self.cast<ArrayAttr>().size() == " #n>,
+    CPred<"$_self.cast<::mlir::ArrayAttr>().size() == " #n>,
     "with exactly " # n # " elements">;
 
 class IntArrayNthElemEq<int index, int value> : AttrConstraint<
     And<[
-      CPred<"$_self.cast<ArrayAttr>().size() > " # index>,
-      CPred<"$_self.cast<ArrayAttr>()[" # index # "]"
-        ".cast<IntegerAttr>().getInt() == " # value>
+      CPred<"$_self.cast<::mlir::ArrayAttr>().size() > " # index>,
+      CPred<"$_self.cast<::mlir::ArrayAttr>()[" # index # "]"
+        ".cast<::mlir::IntegerAttr>().getInt() == " # value>
        ]>,
     "whose " # index # "-th element must be " # value>;
 
 class IntArrayNthElemMinValue<int index, int min> : AttrConstraint<
     And<[
-      CPred<"$_self.cast<ArrayAttr>().size() > " # index>,
-      CPred<"$_self.cast<ArrayAttr>()[" # index # "]"
-        ".cast<IntegerAttr>().getInt() >= " # min>
+      CPred<"$_self.cast<::mlir::ArrayAttr>().size() > " # index>,
+      CPred<"$_self.cast<::mlir::ArrayAttr>()[" # index # "]"
+        ".cast<::mlir::IntegerAttr>().getInt() >= " # min>
         ]>,
     "whose " # index # "-th element must be at least " # min>;
 
@@ -1567,11 +1585,11 @@ def IsNullAttr : AttrConstraint<
 // An attribute constraint on FlatSymbolRefAttr that requires that the
 // reference point to an op of `opClass` within the closest parent with a symbol
 // table.
-// TODO(riverriddle) Add support for nested symbol references.
+// TODO: Add support for nested symbol references.
 class ReferToOp<string opClass> : AttrConstraint<
     CPred<"isa_and_nonnull<" # opClass # ">("
             "::mlir::SymbolTable::lookupNearestSymbolFrom("
-              "&$_op, $_self.cast<FlatSymbolRefAttr>().getValue()))">,
+              "&$_op, $_self.cast<::mlir::FlatSymbolRefAttr>().getValue()))">,
     "referencing to a '" # opClass # "' symbol">;
 
 //===----------------------------------------------------------------------===//
@@ -1586,7 +1604,7 @@ def AnyRegion : Region<CPred<"true">, "any region">;
 
 // A region with the given number of blocks.
 class SizedRegion<int numBlocks> : Region<
-  CPred<"$_self.getBlocks().size() == " # numBlocks>,
+  CPred<"::llvm::hasNItems($_self, " # numBlocks # ")">,
   "region with " # numBlocks # " blocks">;
 
 // A variadic region constraint. It expands to zero or more of the base region.
@@ -1693,8 +1711,8 @@ class ParentOneOf<list<string> ops>
 def FirstAttrDerivedResultType :
   GenInternalOpTrait<"FirstAttrDerivedResultType">;
 
-// TODO(antiagainst): Turn the following into normal traits and generate
-// verification for them.
+// TODO: Turn the following into normal traits and generate verification for
+// them.
 
 // All variadic operands of the op have the same number of values.
 // A variadic operand contains an array of values whose array size is only
@@ -1719,6 +1737,9 @@ def AttrSizedOperandSegments : NativeOpTrait<"AttrSizedOperandSegments">;
 // should be named as `result_segment_sizes`.
 def AttrSizedResultSegments  : NativeOpTrait<"AttrSizedResultSegments">;
 
+// Op attached regions have no arguments
+def NoRegionArguments : NativeOpTrait<"NoRegionArguments">;
+
 //===----------------------------------------------------------------------===//
 // OpInterface definitions
 //===----------------------------------------------------------------------===//
@@ -1729,7 +1750,8 @@ def ins;
 // OpInterfaceTrait corresponds to a specific 'OpInterface' class defined in
 // C++. The purpose to wrap around C++ symbol string with this class is to make
 // interfaces specified for ops in TableGen less alien and more integrated.
-class OpInterfaceTrait<string name, code verifyBody = [{}]> : NativeOpTrait<""> {
+class OpInterfaceTrait<string name, code verifyBody = [{}]>
+          : NativeOpTrait<""> {
   let trait = name # "::Trait";
 
   // Specify the body of the verification function. `$_op` will be replaced with
@@ -1742,8 +1764,8 @@ class OpInterfaceTrait<string name, code verifyBody = [{}]> : NativeOpTrait<"">
 }
 
 // This class represents a single, optionally static, interface method.
-// Note: non-static interface methods have an implicit 'op' parameter
-// corresponding to an instance of the derived operation.
+// Note: non-static interface methods have an implicit parameter, either
+// $_op/$_attr/$_type corresponding to an instance of the derived value.
 class InterfaceMethod<string desc, string retTy, string methodName,
                       dag args = (ins), code methodBody = [{}],
                       code defaultImplementation = [{}]> {
@@ -1773,14 +1795,20 @@ class StaticInterfaceMethod<string desc, string retTy, string methodName,
     : InterfaceMethod<desc, retTy, methodName, args, methodBody,
                       defaultImplementation>;
 
-// OpInterface represents an interface regarding an op.
-class OpInterface<string name> : OpInterfaceTrait<name> {
+// Interface represents a base interface.
+class Interface<string name> {
   // A human-readable description of what this interface does.
   string description = "";
 
   // The name given to the c++ interface class.
   string cppClassName = name;
 
+  // The C++ namespace that this interface should be placed into.
+  //
+  // To specify nested namespaces, use "::" as the delimiter, e.g., given
+  // "A::B", ops will be placed in `namespace A { namespace B { <def> } }`.
+  string cppNamespace = "";
+
   // The list of methods defined by this interface.
   list<InterfaceMethod> methods = [];
 
@@ -1789,6 +1817,23 @@ class OpInterface<string name> : OpInterfaceTrait<name> {
   code extraClassDeclaration = "";
 }
 
+// AttrInterface represents an interface registered to an attribute.
+class AttrInterface<string name> : Interface<name> {
+  // An optional code block containing extra declarations to place in the
+  // interface trait declaration.
+  code extraTraitClassDeclaration = "";
+}
+
+// OpInterface represents an interface registered to an operation.
+class OpInterface<string name> : Interface<name>, OpInterfaceTrait<name>;
+
+// TypeInterface represents an interface registered to a type.
+class TypeInterface<string name> : Interface<name> {
+  // An optional code block containing extra declarations to place in the
+  // interface trait declaration.
+  code extraTraitClassDeclaration = "";
+}
+
 // Whether to declare the op interface methods in the op's header. This class
 // simply wraps an OpInterface but is used to indicate that the method
 // declarations should be generated. This class takes an optional set of methods
@@ -1799,6 +1844,7 @@ class DeclareOpInterfaceMethods<OpInterface interface,
       : OpInterface<interface.cppClassName> {
     let description = interface.description;
     let cppClassName = interface.cppClassName;
+    let cppNamespace = interface.cppNamespace;
     let methods = interface.methods;
 
     // This field contains a set of method names that should always have their
@@ -1941,9 +1987,9 @@ class Op<Dialect dialect, string mnemonic, list<OpTrait> props = []> {
   code verifier = ?;
 
   // Whether this op has associated canonicalization patterns.
-  // TODO(b/120163349): figure out a better way to write canonicalization
-  // patterns in TableGen rules directly instead of using this marker
-  // and C++ implementations.
+  // TODO: figure out a better way to write canonicalization patterns in
+  // TableGen rules directly instead of using this marker and C++
+  // implementations.
   bit hasCanonicalizer = 0;
 
   // Whether this op has a folder.
@@ -1985,26 +2031,28 @@ def HasNoUseOf: Constraint<
 
 // These traits often require including "mlir/IR/TypeUtilities.h".
 
-// TODO(b/135033717): Improve the autogenerated error messages.
+// TODO: Improve the autogenerated error messages.
 
 class Rank<string name> :
-    StrFunc<"$" # name # ".getType().cast<ShapedType>().getRank()">;
+    StrFunc<"$" # name # ".getType().cast<::mlir::ShapedType>().getRank()">;
 
 class Shape<string name> :
-    StrFunc<"$" # name # ".getType().cast<ShapedType>().getShape()">;
+    StrFunc<"$" # name # ".getType().cast<::mlir::ShapedType>().getShape()">;
 
 class ElementCount<string name> :
-  StrFunc<"$" # name # ".getType().cast<ShapedType>().getNumElements()">;
+  StrFunc<"$" # name # ".getType().cast<::mlir::ShapedType>()"
+                                 ".getNumElements()">;
 
 class ElementType<string name> : StrFunc<"getElementTypeOrSelf($" # name # ")">;
 
 class AllMatchPred<list<string> values> :
-    CPred<"llvm::is_splat(llvm::makeArrayRef({"# StrJoin<values>.result #"}))">;
+    CPred<"::llvm::is_splat(::llvm::makeArrayRef({"
+          # StrJoin<values>.result #"}))">;
 
 class AllMatch<list<string> values, string description> :
     PredOpTrait<description, AllMatchPred<values>>;
 
-// TODO(b/135032064): Only works for non-variadic.
+// TODO: Only works for non-variadic.
 class AllMatchSameOperatorPred<list<string> names, string operator> :
     AllMatchPred<!foreach(n, names, !subst("$_self", "$" # n, operator))>;
 
@@ -2078,8 +2126,8 @@ class TCopVTEtIsSameAs<int i, int j> : And<[
       IsShapedTypePred>,
     SubstLeaves<"$_self", "$_op.getOperand(" # j # ").getType()",
       IsShapedTypePred>,
-    CPred<"mlir::getElementTypeOrSelf($_op.getOperand(" # i # ")) == "
-          "mlir::getElementTypeOrSelf($_op.getOperand(" # j # "))">]>;
+    CPred<"::mlir::getElementTypeOrSelf($_op.getOperand(" # i # ")) == "
+          "::mlir::getElementTypeOrSelf($_op.getOperand(" # j # "))">]>;
 
 // Predicate to verify that the i'th result and the j'th operand exist and has
 // shaped types.
@@ -2115,9 +2163,9 @@ class TCresVTEtIsSameAsOp<int i, int j> : And<[
 // of the resId'th result.
 class TCOpIsBroadcastableToRes<int opId, int resId> : And<[
     TCOpResIsShapedTypePred<opId, resId>,
-    CPred<"OpTrait::util::getBroadcastedType("
-              "$_op.getOperand(" # opId # ").getType(), "
-              "$_op.getResult(" # resId # ").getType())">]>;
+    CPred<"::mlir::OpTrait::util::getBroadcastedType("
+                  "$_op.getOperand(" # opId # ").getType(), "
+                  "$_op.getResult(" # resId # ").getType())">]>;
 
 // Predicate to verify that all the operands at the given `indices`
 // have the same element type.
@@ -2127,10 +2175,10 @@ class TCOpIsBroadcastableToRes<int opId, int resId> : And<[
 // 1) all operands involved are of shaped type and
 // 2) the indices are not out of range.
 class TCopVTEtAreSameAt<list<int> indices> : CPred<
-  "llvm::is_splat(llvm::map_range("
-    "llvm::ArrayRef<unsigned>({" # StrJoinInt<indices>.result # "}), "
-    "[this](unsigned i) { return getElementTypeOrSelf(this->getOperand(i)); "
-    "}))">;
+  "::llvm::is_splat(::llvm::map_range("
+      "::mlir::ArrayRef<unsigned>({" # StrJoinInt<indices>.result # "}), "
+      "[this](unsigned i) { return getElementTypeOrSelf(this->getOperand(i)); "
+      "}))">;
 
 //===----------------------------------------------------------------------===//
 // Pattern definitions
diff --git a/mlir/include/mlir/IR/OpDefinition.h b/mlir/include/mlir/IR/OpDefinition.h
index 3fc68d09f840a..80fb2581cfaec 100644
--- a/mlir/include/mlir/IR/OpDefinition.h
+++ b/mlir/include/mlir/IR/OpDefinition.h
@@ -400,6 +400,7 @@ LogicalResult verifyNSuccessors(Operation *op, unsigned numSuccessors);
 LogicalResult verifyAtLeastNSuccessors(Operation *op, unsigned numSuccessors);
 LogicalResult verifyOperandSizeAttr(Operation *op, StringRef sizeAttrName);
 LogicalResult verifyResultSizeAttr(Operation *op, StringRef sizeAttrName);
+LogicalResult verifyNoRegionArguments(Operation *op);
 } // namespace impl
 
 /// Helper class for implementing traits.  Clients are not expected to interact
@@ -1202,6 +1203,15 @@ class AttrSizedResultSegments
   }
 };
 
+/// This trait provides a verifier for ops that are expecting their regions to
+/// not have any arguments
+template <typename ConcrentType>
+struct NoRegionArguments : public TraitBase<ConcrentType, NoRegionArguments> {
+  static LogicalResult verifyTrait(Operation *op) {
+    return ::mlir::OpTrait::impl::verifyNoRegionArguments(op);
+  }
+};
+
 } // end namespace OpTrait
 
 //===----------------------------------------------------------------------===//
@@ -1348,120 +1358,39 @@ class Op : public OpState,
                               traitID);
   }
 
-  /// Returns an opaque pointer to a concept instance of the interface with the
-  /// given ID if one was registered to this operation.
-  static void *getRawInterface(TypeID id) {
-    return InterfaceLookup::template lookup<Traits<ConcreteType>...>(id);
+  /// Returns an interface map for the interfaces registered to this operation.
+  static detail::InterfaceMap getInterfaceMap() {
+    return detail::InterfaceMap::template get<Traits<ConcreteType>...>();
   }
 
-  struct InterfaceLookup {
-    /// Trait to check if T provides a static 'getInterfaceID' method.
-    template <typename T, typename... Args>
-    using has_get_interface_id = decltype(T::getInterfaceID());
-
-    /// If 'T' is the same interface as 'interfaceID' return the concept
-    /// instance.
-    template <typename T>
-    static typename std::enable_if<
-        llvm::is_detected<has_get_interface_id, T>::value, void *>::type
-    lookup(TypeID interfaceID) {
-      return (T::getInterfaceID() == interfaceID) ? &T::instance() : nullptr;
-    }
-
-    /// 'T' is known to not be an interface, return nullptr.
-    template <typename T>
-    static typename std::enable_if<
-        !llvm::is_detected<has_get_interface_id, T>::value, void *>::type
-    lookup(TypeID) {
-      return nullptr;
-    }
-
-    template <typename T, typename T2, typename... Ts>
-    static void *lookup(TypeID interfaceID) {
-      auto *concept = lookup<T>(interfaceID);
-      return concept ? concept : lookup<T2, Ts...>(interfaceID);
-    }
-  };
-
-  /// Allow access to 'hasTrait' and 'getRawInterface'.
+  /// Allow access to 'hasTrait' and 'getInterfaceMap'.
   friend AbstractOperation;
 };
 
-/// This class represents the base of an operation interface. Operation
-/// interfaces provide access to derived *Op properties through an opaquely
-/// Operation instance. Derived interfaces must also provide a 'Traits' class
-/// that defines a 'Concept' and a 'Model' class. The 'Concept' class defines an
-/// abstract virtual interface, where as the 'Model' class implements this
-/// interface for a specific derived *Op type. Both of these classes *must* not
-/// contain non-static data. A simple example is shown below:
-///
-///  struct ExampleOpInterfaceTraits {
-///    struct Concept {
-///      virtual unsigned getNumInputs(Operation *op) = 0;
-///    };
-///    template <typename OpT> class Model {
-///      unsigned getNumInputs(Operation *op) final {
-///        return cast<OpT>(op).getNumInputs();
-///      }
-///    };
-///  };
-///
+/// This class represents the base of an operation interface. See the definition
+/// of `detail::Interface` for requirements on the `Traits` type.
 template <typename ConcreteType, typename Traits>
-class OpInterface : public Op<ConcreteType> {
+class OpInterface
+    : public detail::Interface<ConcreteType, Operation *, Traits,
+                               Op<ConcreteType>, OpTrait::TraitBase> {
 public:
-  using Concept = typename Traits::Concept;
-  template <typename T> using Model = typename Traits::template Model<T>;
   using Base = OpInterface<ConcreteType, Traits>;
+  using InterfaceBase = detail::Interface<ConcreteType, Operation *, Traits,
+                                          Op<ConcreteType>, OpTrait::TraitBase>;
 
-  OpInterface(Operation *op = nullptr)
-      : Op<ConcreteType>(op), impl(op ? getInterfaceFor(op) : nullptr) {
-    assert((!op || impl) &&
-           "instantiating an interface with an unregistered operation");
-  }
-
-  /// Support 'classof' by checking if the given operation defines the concrete
-  /// interface.
-  static bool classof(Operation *op) { return getInterfaceFor(op); }
-
-  /// Define an accessor for the ID of this interface.
-  static TypeID getInterfaceID() { return TypeID::get<ConcreteType>(); }
-
-  /// This is a special trait that registers a given interface with an
-  /// operation.
-  template <typename ConcreteOp>
-  struct Trait : public OpTrait::TraitBase<ConcreteOp, Trait> {
-    /// Define an accessor for the ID of this interface.
-    static TypeID getInterfaceID() { return TypeID::get<ConcreteType>(); }
-
-    /// Provide an accessor to a static instance of the interface model for the
-    /// concrete operation type.
-    /// The implementation is inspired from Sean Parent's concept-based
-    /// polymorphism. A key difference is that the set of classes erased is
-    /// statically known, which alleviates the need for using dynamic memory
-    /// allocation.
-    /// We use a zero-sized templated class `Model<ConcreteOp>` to emit the
-    /// virtual table and generate a singleton object for each instantiation of
-    /// this class.
-    static Concept &instance() {
-      static Model<ConcreteOp> singleton;
-      return singleton;
-    }
-  };
-
-protected:
-  /// Get the raw concept in the correct derived concept type.
-  Concept *getImpl() { return impl; }
+  /// Inherit the base class constructor.
+  using InterfaceBase::InterfaceBase;
 
 private:
   /// Returns the impl interface instance for the given operation.
-  static Concept *getInterfaceFor(Operation *op) {
+  static typename InterfaceBase::Concept *getInterfaceFor(Operation *op) {
     // Access the raw interface from the abstract operation.
     auto *abstractOp = op->getAbstractOperation();
     return abstractOp ? abstractOp->getInterface<ConcreteType>() : nullptr;
   }
 
-  /// A pointer to the impl concept object.
-  Concept *impl;
+  /// Allow access to `getInterfaceFor`.
+  friend InterfaceBase;
 };
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/IR/OpImplementation.h b/mlir/include/mlir/IR/OpImplementation.h
index 126d20eacbe49..20660be4347c1 100644
--- a/mlir/include/mlir/IR/OpImplementation.h
+++ b/mlir/include/mlir/IR/OpImplementation.h
@@ -764,6 +764,7 @@ class OpAsmDialectInterface
   virtual void getAsmBlockArgumentNames(Block *block,
                                         OpAsmSetValueNameFn setNameFn) const {}
 };
+} // end namespace mlir
 
 //===--------------------------------------------------------------------===//
 // Operation OpAsm interface.
@@ -772,6 +773,4 @@ class OpAsmDialectInterface
 /// The OpAsmOpInterface, see OpAsmInterface.td for more details.
 #include "mlir/IR/OpAsmInterface.h.inc"
 
-} // end namespace mlir
-
 #endif
diff --git a/mlir/include/mlir/IR/OperationSupport.h b/mlir/include/mlir/IR/OperationSupport.h
index aa75c57db1ff8..e3afaf3161540 100644
--- a/mlir/include/mlir/IR/OperationSupport.h
+++ b/mlir/include/mlir/IR/OperationSupport.h
@@ -19,7 +19,7 @@
 #include "mlir/IR/Location.h"
 #include "mlir/IR/Types.h"
 #include "mlir/IR/Value.h"
-#include "mlir/Support/LogicalResult.h"
+#include "mlir/Support/InterfaceSupport.h"
 #include "llvm/ADT/BitmaskEnum.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
@@ -136,8 +136,7 @@ class AbstractOperation {
   /// was registered to this operation, null otherwise. This should not be used
   /// directly.
   template <typename T> typename T::Concept *getInterface() const {
-    return reinterpret_cast<typename T::Concept *>(
-        getRawInterface(T::getInterfaceID()));
+    return interfaceMap.lookup<T>();
   }
 
   /// Returns if the operation has a particular trait.
@@ -157,7 +156,7 @@ class AbstractOperation {
         T::getOperationName(), dialect, T::getOperationProperties(),
         TypeID::get<T>(), T::parseAssembly, T::printAssembly,
         T::verifyInvariants, T::foldHook, T::getCanonicalizationPatterns,
-        T::getRawInterface, T::hasTrait);
+        T::getInterfaceMap(), T::hasTrait);
   }
 
 private:
@@ -171,22 +170,19 @@ class AbstractOperation {
                                 SmallVectorImpl<OpFoldResult> &results),
       void (&getCanonicalizationPatterns)(OwningRewritePatternList &results,
                                           MLIRContext *context),
-      void *(&getRawInterface)(TypeID interfaceID),
-      bool (&hasTrait)(TypeID traitID))
+      detail::InterfaceMap &&interfaceMap, bool (&hasTrait)(TypeID traitID))
       : name(name), dialect(dialect), typeID(typeID),
         parseAssembly(parseAssembly), printAssembly(printAssembly),
         verifyInvariants(verifyInvariants), foldHook(foldHook),
         getCanonicalizationPatterns(getCanonicalizationPatterns),
-        opProperties(opProperties), getRawInterface(getRawInterface),
+        opProperties(opProperties), interfaceMap(std::move(interfaceMap)),
         hasRawTrait(hasTrait) {}
 
   /// The properties of the operation.
   const OperationProperties opProperties;
 
-  /// Returns a raw instance of the concept for the given interface id if it is
-  /// registered to this operation, nullptr otherwise. This should not be used
-  /// directly.
-  void *(&getRawInterface)(TypeID interfaceID);
+  /// A map of interfaces that were registered to this operation.
+  detail::InterfaceMap interfaceMap;
 
   /// This hook returns if the operation contains the trait corresponding
   /// to the given TypeID.
diff --git a/mlir/include/mlir/IR/OwningOpRefBase.h b/mlir/include/mlir/IR/OwningOpRefBase.h
new file mode 100644
index 0000000000000..bfdf98f6f37e1
--- /dev/null
+++ b/mlir/include/mlir/IR/OwningOpRefBase.h
@@ -0,0 +1,64 @@
+//===- OwningOpRefBase.h - MLIR OwningOpRefBase -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file provides a base class for owning op refs.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_IR_OWNINGOPREFBASE_H
+#define MLIR_IR_OWNINGOPREFBASE_H
+
+#include <utility>
+
+namespace mlir {
+
+/// This class acts as an owning reference to an op, and will automatically
+/// destroy the held op on destruction if the held op is valid.
+///
+/// Note that OpBuilder and related functionality should be highly preferred
+/// instead, and this should only be used in situations where existing solutions
+/// are not viable.
+template <typename OpTy>
+class OwningOpRefBase {
+public:
+  OwningOpRefBase(std::nullptr_t = nullptr) {}
+  OwningOpRefBase(OpTy op) : op(op) {}
+  OwningOpRefBase(OwningOpRefBase &&other) : op(other.release()) {}
+  ~OwningOpRefBase() {
+    if (op)
+      op.erase();
+  }
+
+  // Assign from another op reference.
+  OwningOpRefBase &operator=(OwningOpRefBase &&other) {
+    if (op)
+      op.erase();
+    op = other.release();
+    return *this;
+  }
+
+  /// Allow accessing the internal op.
+  OpTy get() const { return op; }
+  OpTy operator*() const { return op; }
+  OpTy *operator->() { return &op; }
+  explicit operator bool() const { return op; }
+
+  /// Release the referenced op.
+  OpTy release() {
+    OpTy released;
+    std::swap(released, op);
+    return released;
+  }
+
+private:
+  OpTy op;
+};
+
+} // end namespace mlir
+
+#endif // MLIR_IR_OWNINGOPREFBASE_H
diff --git a/mlir/include/mlir/IR/StandardTypes.h b/mlir/include/mlir/IR/StandardTypes.h
index 99ad3b10f6d9c..5380668b59013 100644
--- a/mlir/include/mlir/IR/StandardTypes.h
+++ b/mlir/include/mlir/IR/StandardTypes.h
@@ -102,7 +102,7 @@ class ComplexType
 
 /// Index is a special integer-like type with unknown platform-dependent bit
 /// width.
-class IndexType : public Type::TypeBase<IndexType, Type> {
+class IndexType : public Type::TypeBase<IndexType, Type, TypeStorage> {
 public:
   using Base::Base;
 
@@ -188,7 +188,7 @@ class IntegerType
 // FloatType
 //===----------------------------------------------------------------------===//
 
-class FloatType : public Type::TypeBase<FloatType, Type> {
+class FloatType : public Type::TypeBase<FloatType, Type, TypeStorage> {
 public:
   using Base::Base;
 
@@ -227,7 +227,7 @@ class FloatType : public Type::TypeBase<FloatType, Type> {
 
 /// NoneType is a unit type, i.e. a type with exactly one possible value, where
 /// its value does not have a defined dynamic representation.
-class NoneType : public Type::TypeBase<NoneType, Type> {
+class NoneType : public Type::TypeBase<NoneType, Type, TypeStorage> {
 public:
   using Base::Base;
 
@@ -250,7 +250,7 @@ class ShapedType : public Type {
   using ImplType = detail::ShapedTypeStorage;
   using Type::Type;
 
-  // TODO(ntv): merge these two special values in a single one used everywhere.
+  // TODO: merge these two special values in a single one used everywhere.
   // Unfortunately, uses of `-1` have crept deep into the codebase now and are
   // hard to track.
   static constexpr int64_t kDynamicSize = -1;
@@ -357,7 +357,7 @@ class VectorType
   /// Returns true of the given type can be used as an element of a vector type.
   /// In particular, vectors can consist of integer or float primitives.
   static bool isValidElementType(Type t) {
-    return t.isa<IntegerType>() || t.isa<FloatType>();
+    return t.isa<IntegerType, FloatType>();
   }
 
   ArrayRef<int64_t> getShape() const;
@@ -381,9 +381,8 @@ class TensorType : public ShapedType {
     // Note: Non standard/builtin types are allowed to exist within tensor
     // types. Dialects are expected to verify that tensor types have a valid
     // element type within that dialect.
-    return type.isa<ComplexType>() || type.isa<FloatType>() ||
-           type.isa<IntegerType>() || type.isa<OpaqueType>() ||
-           type.isa<VectorType>() || type.isa<IndexType>() ||
+    return type.isa<ComplexType, FloatType, IntegerType, OpaqueType, VectorType,
+                    IndexType>() ||
            (type.getKind() > Type::Kind::LAST_STANDARD_TYPE);
   }
 
@@ -562,7 +561,7 @@ class MemRefType : public Type::TypeBase<MemRefType, BaseMemRefType,
   /// Returns the memory space in which data referred to by this memref resides.
   unsigned getMemorySpace() const;
 
-  // TODO(ntv): merge these two special values in a single one used everywhere.
+  // TODO: merge these two special values in a single one used everywhere.
   // Unfortunately, uses of `-1` have crept deep into the codebase now and are
   // hard to track.
   static constexpr int64_t kDynamicSize = -1;
diff --git a/mlir/include/mlir/IR/StorageUniquerSupport.h b/mlir/include/mlir/IR/StorageUniquerSupport.h
index 1b01b7a9970c0..c2250e8547163 100644
--- a/mlir/include/mlir/IR/StorageUniquerSupport.h
+++ b/mlir/include/mlir/IR/StorageUniquerSupport.h
@@ -13,6 +13,7 @@
 #ifndef MLIR_IR_STORAGEUNIQUERSUPPORT_H
 #define MLIR_IR_STORAGEUNIQUERSUPPORT_H
 
+#include "mlir/Support/InterfaceSupport.h"
 #include "mlir/Support/LogicalResult.h"
 #include "mlir/Support/StorageUniquer.h"
 #include "mlir/Support/TypeID.h"
@@ -27,17 +28,41 @@ namespace detail {
 /// avoid the need to include Location.h.
 const AttributeStorage *generateUnknownStorageLocation(MLIRContext *ctx);
 
+//===----------------------------------------------------------------------===//
+// StorageUserTraitBase
+//===----------------------------------------------------------------------===//
+
+/// Helper class for implementing traits for storage classes. Clients are not
+/// expected to interact with this directly, so its members are all protected.
+template <typename ConcreteType, template <typename> class TraitType>
+class StorageUserTraitBase {
+protected:
+  /// Return the derived instance.
+  ConcreteType getInstance() const {
+    // We have to cast up to the trait type, then to the concrete type because
+    // the concrete type will multiply derive from the (content free) TraitBase
+    // class, and we need to be able to disambiguate the path for the C++
+    // compiler.
+    auto *trait = static_cast<const TraitType<ConcreteType> *>(this);
+    return *static_cast<const ConcreteType *>(trait);
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// StorageUserBase
+//===----------------------------------------------------------------------===//
+
 /// Utility class for implementing users of storage classes uniqued by a
 /// StorageUniquer. Clients are not expected to interact with this class
 /// directly.
 template <typename ConcreteT, typename BaseT, typename StorageT,
-          typename UniquerT>
-class StorageUserBase : public BaseT {
+          typename UniquerT, template <typename T> class... Traits>
+class StorageUserBase : public BaseT, public Traits<ConcreteT>... {
 public:
   using BaseT::BaseT;
 
   /// Utility declarations for the concrete attribute class.
-  using Base = StorageUserBase<ConcreteT, BaseT, StorageT, UniquerT>;
+  using Base = StorageUserBase<ConcreteT, BaseT, StorageT, UniquerT, Traits...>;
   using ImplType = StorageT;
 
   /// Return a unique identifier for the concrete type.
@@ -51,6 +76,12 @@ class StorageUserBase : public BaseT {
     return ConcreteT::kindof(val.getKind());
   }
 
+  /// Returns an interface map for the interfaces registered to this storage
+  /// user. This should not be used directly.
+  static detail::InterfaceMap getInterfaceMap() {
+    return detail::InterfaceMap::template get<Traits<ConcreteT>...>();
+  }
+
 protected:
   /// Get or create a new ConcreteT instance within the ctx. This
   /// function is guaranteed to return a non null object and will assert if
diff --git a/mlir/include/mlir/IR/SymbolInterfaces.td b/mlir/include/mlir/IR/SymbolInterfaces.td
index de2f4cda2b8d5..148551324868a 100644
--- a/mlir/include/mlir/IR/SymbolInterfaces.td
+++ b/mlir/include/mlir/IR/SymbolInterfaces.td
@@ -27,13 +27,14 @@ def Symbol : OpInterface<"SymbolOpInterface"> {
     See [Symbols and SymbolTables](SymbolsAndSymbolTables.md) for more details
     and constraints on `Symbol` operations.
   }];
+  let cppNamespace = "::mlir";
 
   let methods = [
     InterfaceMethod<"Returns the name of this symbol.",
       "StringRef", "getName", (ins), [{
         // Don't rely on the trait implementation as optional symbol operations
         // may override this.
-        return mlir::SymbolTable::getSymbolName(op);
+        return mlir::SymbolTable::getSymbolName($_op);
       }], /*defaultImplementation=*/[{
         return mlir::SymbolTable::getSymbolName(this->getOperation());
       }]
diff --git a/mlir/include/mlir/IR/SymbolTable.h b/mlir/include/mlir/IR/SymbolTable.h
index 0b035836ec61d..7e52011f81ffb 100644
--- a/mlir/include/mlir/IR/SymbolTable.h
+++ b/mlir/include/mlir/IR/SymbolTable.h
@@ -252,10 +252,9 @@ class SymbolTable : public TraitBase<ConcreteType, SymbolTable> {
 };
 
 } // end namespace OpTrait
+} // end namespace mlir
 
 /// Include the generated symbol interfaces.
 #include "mlir/IR/SymbolInterfaces.h.inc"
 
-} // end namespace mlir
-
 #endif // MLIR_IR_SYMBOLTABLE_H
diff --git a/mlir/include/mlir/IR/TypeSupport.h b/mlir/include/mlir/IR/TypeSupport.h
index cb093c2188d6f..7961dd22d47dd 100644
--- a/mlir/include/mlir/IR/TypeSupport.h
+++ b/mlir/include/mlir/IR/TypeSupport.h
@@ -20,12 +20,51 @@ namespace mlir {
 class Dialect;
 class MLIRContext;
 
+//===----------------------------------------------------------------------===//
+// AbstractType
+//===----------------------------------------------------------------------===//
+
+/// This class contains all of the static information common to all instances of
+/// a registered Type.
+class AbstractType {
+public:
+  /// Look up the specified abstract type in the MLIRContext and return a
+  /// reference to it.
+  static const AbstractType &lookup(TypeID typeID, MLIRContext *context);
+
+  /// This method is used by Dialect objects when they register the list of
+  /// types they contain.
+  template <typename T> static AbstractType get(Dialect &dialect) {
+    return AbstractType(dialect, T::getInterfaceMap());
+  }
+
+  /// Return the dialect this type was registered to.
+  Dialect &getDialect() const { return const_cast<Dialect &>(dialect); }
+
+  /// Returns an instance of the concept object for the given interface if it
+  /// was registered to this type, null otherwise. This should not be used
+  /// directly.
+  template <typename T> typename T::Concept *getInterface() const {
+    return interfaceMap.lookup<T>();
+  }
+
+private:
+  AbstractType(Dialect &dialect, detail::InterfaceMap &&interfaceMap)
+      : dialect(dialect), interfaceMap(std::move(interfaceMap)) {}
+
+  /// This is the dialect that this type was registered to.
+  Dialect &dialect;
+
+  /// This is a collection of the interfaces registered to this type.
+  detail::InterfaceMap interfaceMap;
+};
+
 //===----------------------------------------------------------------------===//
 // TypeStorage
 //===----------------------------------------------------------------------===//
 
 namespace detail {
-class TypeUniquer;
+struct TypeUniquer;
 } // end namespace detail
 
 /// Base storage class appearing in a Type.
@@ -33,32 +72,33 @@ class TypeStorage : public StorageUniquer::BaseStorage {
   friend detail::TypeUniquer;
   friend StorageUniquer;
 
-protected:
-  /// This constructor is used by derived classes as part of the TypeUniquer.
-  /// When using this constructor, the initializeDialect function must be
-  /// invoked afterwards for the storage to be valid.
-  TypeStorage(unsigned subclassData = 0)
-      : dialect(nullptr), subclassData(subclassData) {}
-
 public:
-  /// Get the dialect that this type is registered to.
-  Dialect &getDialect() {
-    assert(dialect && "Malformed type storage object.");
-    return *dialect;
+  /// Return the abstract type descriptor for this type.
+  const AbstractType &getAbstractType() {
+    assert(abstractType && "Malformed type storage object.");
+    return *abstractType;
   }
+
   /// Get the subclass data.
   unsigned getSubclassData() const { return subclassData; }
 
   /// Set the subclass data.
   void setSubclassData(unsigned val) { subclassData = val; }
 
+protected:
+  /// This constructor is used by derived classes as part of the TypeUniquer.
+  TypeStorage(unsigned subclassData = 0)
+      : abstractType(nullptr), subclassData(subclassData) {}
+
 private:
-  // Set the dialect for this storage instance. This is used by the TypeUniquer
-  // when initializing a newly constructed type storage object.
-  void initializeDialect(Dialect &newDialect) { dialect = &newDialect; }
+  /// Set the abstract type for this storage instance. This is used by the
+  /// TypeUniquer when initializing a newly constructed type storage object.
+  void initialize(const AbstractType &abstractTy) {
+    abstractType = &abstractTy;
+  }
 
-  /// The dialect for this type.
-  Dialect *dialect;
+  /// The abstract description for this type.
+  const AbstractType *abstractType;
 
   /// Space for subclasses to store data.
   unsigned subclassData;
@@ -72,36 +112,26 @@ using DefaultTypeStorage = TypeStorage;
 // TypeStorageAllocator
 //===----------------------------------------------------------------------===//
 
-// This is a utility allocator used to allocate memory for instances of derived
-// Types.
+/// This is a utility allocator used to allocate memory for instances of derived
+/// Types.
 using TypeStorageAllocator = StorageUniquer::StorageAllocator;
 
 //===----------------------------------------------------------------------===//
 // TypeUniquer
 //===----------------------------------------------------------------------===//
 namespace detail {
-// A utility class to get, or create, unique instances of types within an
-// MLIRContext. This class manages all creation and uniquing of types.
-class TypeUniquer {
-public:
+/// A utility class to get, or create, unique instances of types within an
+/// MLIRContext. This class manages all creation and uniquing of types.
+struct TypeUniquer {
   /// Get an uniqued instance of a type T.
   template <typename T, typename... Args>
   static T get(MLIRContext *ctx, unsigned kind, Args &&... args) {
     return ctx->getTypeUniquer().get<typename T::ImplType>(
         [&](TypeStorage *storage) {
-          storage->initializeDialect(lookupDialectForType<T>(ctx));
+          storage->initialize(AbstractType::lookup(T::getTypeID(), ctx));
         },
         kind, std::forward<Args>(args)...);
   }
-
-private:
-  /// Get the dialect that the type 'T' was registered with.
-  template <typename T> static Dialect &lookupDialectForType(MLIRContext *ctx) {
-    return lookupDialectForType(ctx, T::getTypeID());
-  }
-
-  /// Get the dialect that registered the type with the provided typeid.
-  static Dialect &lookupDialectForType(MLIRContext *ctx, TypeID typeID);
 };
 } // namespace detail
 
diff --git a/mlir/include/mlir/IR/Types.h b/mlir/include/mlir/IR/Types.h
index 32ae13f86dc90..c14f8558d8503 100644
--- a/mlir/include/mlir/IR/Types.h
+++ b/mlir/include/mlir/IR/Types.h
@@ -100,10 +100,10 @@ class Type {
   };
 
   /// Utility class for implementing types.
-  template <typename ConcreteType, typename BaseType,
-            typename StorageType = DefaultTypeStorage>
+  template <typename ConcreteType, typename BaseType, typename StorageType,
+            template <typename T> class... Traits>
   using TypeBase = detail::StorageUserBase<ConcreteType, BaseType, StorageType,
-                                           detail::TypeUniquer>;
+                                           detail::TypeUniquer, Traits...>;
 
   using ImplType = TypeStorage;
 
@@ -121,6 +121,8 @@ class Type {
   bool operator!() const { return impl == nullptr; }
 
   template <typename U> bool isa() const;
+  template <typename First, typename Second, typename... Rest>
+  bool isa() const;
   template <typename U> U dyn_cast() const;
   template <typename U> U dyn_cast_or_null() const;
   template <typename U> U cast() const;
@@ -194,6 +196,9 @@ class Type {
     return Type(reinterpret_cast<ImplType *>(const_cast<void *>(pointer)));
   }
 
+  /// Return the abstract type descriptor for this type.
+  const AbstractType &getAbstractType() { return impl->getAbstractType(); }
+
 protected:
   ImplType *impl;
 };
@@ -203,6 +208,45 @@ inline raw_ostream &operator<<(raw_ostream &os, Type type) {
   return os;
 }
 
+//===----------------------------------------------------------------------===//
+// TypeTraitBase
+//===----------------------------------------------------------------------===//
+
+namespace TypeTrait {
+/// This class represents the base of a type trait.
+template <typename ConcreteType, template <typename> class TraitType>
+using TraitBase = detail::StorageUserTraitBase<ConcreteType, TraitType>;
+} // namespace TypeTrait
+
+//===----------------------------------------------------------------------===//
+// TypeInterface
+//===----------------------------------------------------------------------===//
+
+/// This class represents the base of a type interface. See the definition  of
+/// `detail::Interface` for requirements on the `Traits` type.
+template <typename ConcreteType, typename Traits>
+class TypeInterface : public detail::Interface<ConcreteType, Type, Traits, Type,
+                                               TypeTrait::TraitBase> {
+public:
+  using Base = TypeInterface<ConcreteType, Traits>;
+  using InterfaceBase =
+      detail::Interface<ConcreteType, Type, Traits, Type, TypeTrait::TraitBase>;
+  using InterfaceBase::InterfaceBase;
+
+private:
+  /// Returns the impl interface instance for the given type.
+  static typename InterfaceBase::Concept *getInterfaceFor(Type type) {
+    return type.getAbstractType().getInterface<ConcreteType>();
+  }
+
+  /// Allow access to 'getInterfaceFor'.
+  friend InterfaceBase;
+};
+
+//===----------------------------------------------------------------------===//
+// FunctionType
+//===----------------------------------------------------------------------===//
+
 /// Function types map from a list of inputs to a list of results.
 class FunctionType
     : public Type::TypeBase<FunctionType, Type, detail::FunctionTypeStorage> {
@@ -230,6 +274,10 @@ class FunctionType
   static bool kindof(unsigned kind) { return kind == Kind::Function; }
 };
 
+//===----------------------------------------------------------------------===//
+// OpaqueType
+//===----------------------------------------------------------------------===//
+
 /// Opaque types represent types of non-registered dialects. These are types
 /// represented in their raw string form, and can only usefully be tested for
 /// type equality.
@@ -271,6 +319,12 @@ template <typename U> bool Type::isa() const {
   assert(impl && "isa<> used on a null type.");
   return U::classof(*this);
 }
+
+template <typename First, typename Second, typename... Rest>
+bool Type::isa() const {
+  return isa<First>() || isa<Second, Rest...>();
+}
+
 template <typename U> U Type::dyn_cast() const {
   return isa<U>() ? U(impl) : U(nullptr);
 }
diff --git a/mlir/include/mlir/IR/Value.h b/mlir/include/mlir/IR/Value.h
index f5cb16f347ed3..c22741ee6cb6a 100644
--- a/mlir/include/mlir/IR/Value.h
+++ b/mlir/include/mlir/IR/Value.h
@@ -81,6 +81,12 @@ class Value {
     assert(*this && "isa<> used on a null type.");
     return U::classof(*this);
   }
+
+  template <typename First, typename Second, typename... Rest>
+  bool isa() const {
+    return isa<First>() || isa<Second, Rest...>();
+  }
+
   template <typename U> U dyn_cast() const {
     return isa<U>() ? U(ownerAndKind) : U(nullptr);
   }
diff --git a/mlir/include/mlir/Interfaces/CMakeLists.txt b/mlir/include/mlir/Interfaces/CMakeLists.txt
index ff3577d50851f..0de2b5a8688bc 100644
--- a/mlir/include/mlir/Interfaces/CMakeLists.txt
+++ b/mlir/include/mlir/Interfaces/CMakeLists.txt
@@ -1,8 +1,10 @@
 add_mlir_interface(CallInterfaces)
 add_mlir_interface(ControlFlowInterfaces)
+add_mlir_interface(CopyOpInterface)
 add_mlir_interface(DerivedAttributeOpInterface)
 add_mlir_interface(InferTypeOpInterface)
 add_mlir_interface(LoopLikeInterface)
 add_mlir_interface(SideEffectInterfaces)
+add_mlir_interface(VectorUnrollInterface)
 add_mlir_interface(ViewLikeInterface)
 
diff --git a/mlir/include/mlir/Interfaces/CallInterfaces.h b/mlir/include/mlir/Interfaces/CallInterfaces.h
index ddfd5a942e49f..cc8e26eceba3f 100644
--- a/mlir/include/mlir/Interfaces/CallInterfaces.h
+++ b/mlir/include/mlir/Interfaces/CallInterfaces.h
@@ -23,8 +23,9 @@ namespace mlir {
 struct CallInterfaceCallable : public PointerUnion<SymbolRefAttr, Value> {
   using PointerUnion<SymbolRefAttr, Value>::PointerUnion;
 };
+} // end namespace mlir
 
+/// Include the generated interface declarations.
 #include "mlir/Interfaces/CallInterfaces.h.inc"
-} // end namespace mlir
 
 #endif // MLIR_INTERFACES_CALLINTERFACES_H
diff --git a/mlir/include/mlir/Interfaces/CallInterfaces.td b/mlir/include/mlir/Interfaces/CallInterfaces.td
index 81ab52f197aaf..7db6730c5e992 100644
--- a/mlir/include/mlir/Interfaces/CallInterfaces.td
+++ b/mlir/include/mlir/Interfaces/CallInterfaces.td
@@ -29,6 +29,7 @@ def CallOpInterface : OpInterface<"CallOpInterface"> {
     indirect calls to other operations `call_indirect %foo`. An operation that
     uses this interface, must *not* also provide the `CallableOpInterface`.
   }];
+  let cppNamespace = "::mlir";
 
   let methods = [
     InterfaceMethod<[{
@@ -51,9 +52,9 @@ def CallOpInterface : OpInterface<"CallOpInterface"> {
       }],
       "Operation *", "resolveCallable", (ins), [{
         // If the callable isn't a value, lookup the symbol reference.
-        CallInterfaceCallable callable = op.getCallableForCallee();
+        CallInterfaceCallable callable = $_op.getCallableForCallee();
         if (auto symbolRef = callable.dyn_cast<SymbolRefAttr>())
-          return SymbolTable::lookupNearestSymbolFrom(op, symbolRef);
+          return SymbolTable::lookupNearestSymbolFrom($_op, symbolRef);
         return callable.get<Value>().getDefiningOp();
       }]
     >,
@@ -70,6 +71,7 @@ def CallableOpInterface : OpInterface<"CallableOpInterface"> {
     `%foo = dialect.create_function(...)`. These operations may only contain a
     single region, or subroutine.
   }];
+  let cppNamespace = "::mlir";
 
   let methods = [
     InterfaceMethod<[{
diff --git a/mlir/include/mlir/Interfaces/ControlFlowInterfaces.h b/mlir/include/mlir/Interfaces/ControlFlowInterfaces.h
index e18c46f745a2f..7e609ca13a097 100644
--- a/mlir/include/mlir/Interfaces/ControlFlowInterfaces.h
+++ b/mlir/include/mlir/Interfaces/ControlFlowInterfaces.h
@@ -70,12 +70,6 @@ class RegionSuccessor {
   ValueRange inputs;
 };
 
-//===----------------------------------------------------------------------===//
-// ControlFlow Interfaces
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Interfaces/ControlFlowInterfaces.h.inc"
-
 //===----------------------------------------------------------------------===//
 // ControlFlow Traits
 //===----------------------------------------------------------------------===//
@@ -101,4 +95,11 @@ struct ReturnLike : public TraitBase<ConcreteType, ReturnLike> {
 
 } // end namespace mlir
 
+//===----------------------------------------------------------------------===//
+// ControlFlow Interfaces
+//===----------------------------------------------------------------------===//
+
+/// Include the generated interface declarations.
+#include "mlir/Interfaces/ControlFlowInterfaces.h.inc"
+
 #endif // MLIR_INTERFACES_CONTROLFLOWINTERFACES_H
diff --git a/mlir/include/mlir/Interfaces/ControlFlowInterfaces.td b/mlir/include/mlir/Interfaces/ControlFlowInterfaces.td
index 591ca11830e96..8b5a0b769ab17 100644
--- a/mlir/include/mlir/Interfaces/ControlFlowInterfaces.td
+++ b/mlir/include/mlir/Interfaces/ControlFlowInterfaces.td
@@ -25,6 +25,8 @@ def BranchOpInterface : OpInterface<"BranchOpInterface"> {
     This interface provides information for branching terminator operations,
     i.e. terminator operations with successors.
   }];
+  let cppNamespace = "::mlir";
+
   let methods = [
     InterfaceMethod<[{
         Returns a mutable range of operands that correspond to the arguments of
@@ -55,10 +57,10 @@ def BranchOpInterface : OpInterface<"BranchOpInterface"> {
       }],
       "Optional<BlockArgument>", "getSuccessorBlockArgument",
       (ins "unsigned":$operandIndex), [{
-        Operation *opaqueOp = op;
+        Operation *opaqueOp = $_op;
         for (unsigned i = 0, e = opaqueOp->getNumSuccessors(); i != e; ++i) {
           if (Optional<BlockArgument> arg = detail::getBranchSuccessorArgument(
-                op.getSuccessorOperands(i), operandIndex,
+                $_op.getSuccessorOperands(i), operandIndex,
                 opaqueOp->getSuccessor(i)))
             return arg;
         }
@@ -96,6 +98,8 @@ def RegionBranchOpInterface : OpInterface<"RegionBranchOpInterface"> {
     branching behavior between held regions, i.e. this interface allows for
     expressing control flow information for region holding operations.
   }];
+  let cppNamespace = "::mlir";
+
   let methods = [
     InterfaceMethod<[{
         Returns the operands of this operation used as the entry arguments when
diff --git a/mlir/include/mlir/Interfaces/CopyOpInterface.h b/mlir/include/mlir/Interfaces/CopyOpInterface.h
new file mode 100644
index 0000000000000..2f38eb326b53e
--- /dev/null
+++ b/mlir/include/mlir/Interfaces/CopyOpInterface.h
@@ -0,0 +1,21 @@
+//===- CopyOpInterface.h - copy operations interface ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the operation interface for copy-like operations.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_INTERFACES_COPYOPINTERFACE_H_
+#define MLIR_INTERFACES_COPYOPINTERFACE_H_
+
+#include "mlir/IR/OpDefinition.h"
+
+/// Include the generated interface declarations.
+#include "mlir/Interfaces/CopyOpInterface.h.inc"
+
+#endif // MLIR_INTERFACES_COPYOPINTERFACE_H_
diff --git a/mlir/include/mlir/Interfaces/CopyOpInterface.td b/mlir/include/mlir/Interfaces/CopyOpInterface.td
new file mode 100644
index 0000000000000..a503abc185d90
--- /dev/null
+++ b/mlir/include/mlir/Interfaces/CopyOpInterface.td
@@ -0,0 +1,38 @@
+//===- CopyOpInterface.td - Copy operation interface -------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the interface for copy-like operations.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_INTERFACES_COPYOPINTERFACE
+#define MLIR_INTERFACES_COPYOPINTERFACE
+
+include "mlir/IR/OpBase.td"
+
+def CopyOpInterface : OpInterface<"CopyOpInterface"> {
+  let description = [{
+    A copy-like operation is one that copies from source value to target value.
+  }];
+  let cppNamespace = "::mlir";
+
+  let methods = [
+    InterfaceMethod<
+      /*desc=*/"Returns the source value for this copy operation",
+      /*retTy=*/"Value",
+      /*methodName=*/"getSource"
+    >,
+    InterfaceMethod<
+      /*desc=*/"Returns the target value for this copy operation",
+      /*retTy=*/"Value",
+      /*methodName=*/"getTarget"
+    >
+  ];
+}
+
+#endif // MLIR_INTERFACES_COPYOPINTERFACE
diff --git a/mlir/include/mlir/Interfaces/DerivedAttributeOpInterface.h b/mlir/include/mlir/Interfaces/DerivedAttributeOpInterface.h
index debafc2438d28..63cd09f5bc42c 100644
--- a/mlir/include/mlir/Interfaces/DerivedAttributeOpInterface.h
+++ b/mlir/include/mlir/Interfaces/DerivedAttributeOpInterface.h
@@ -15,8 +15,7 @@
 
 #include "mlir/IR/OpDefinition.h"
 
-namespace mlir {
+/// Include the generated interface declarations.
 #include "mlir/Interfaces/DerivedAttributeOpInterface.h.inc"
-} // namespace mlir
 
 #endif // MLIR_INTERFACES_DERIVEDATTRIBUTEOPINTERFACE_H_
diff --git a/mlir/include/mlir/Interfaces/DerivedAttributeOpInterface.td b/mlir/include/mlir/Interfaces/DerivedAttributeOpInterface.td
index e6f370752bcf2..92c901840790e 100644
--- a/mlir/include/mlir/Interfaces/DerivedAttributeOpInterface.td
+++ b/mlir/include/mlir/Interfaces/DerivedAttributeOpInterface.td
@@ -23,6 +23,7 @@ def DerivedAttributeOpInterface : OpInterface<"DerivedAttributeOpInterface"> {
     from information of the operation. ODS generates convenience accessors for
     derived attributes and can be used to simplify translations.
   }];
+  let cppNamespace = "::mlir";
 
   let methods = [
     StaticInterfaceMethod<
diff --git a/mlir/include/mlir/Interfaces/InferTypeOpInterface.h b/mlir/include/mlir/Interfaces/InferTypeOpInterface.h
index 67faeb56a51c9..1ae4aa688c84f 100644
--- a/mlir/include/mlir/Interfaces/InferTypeOpInterface.h
+++ b/mlir/include/mlir/Interfaces/InferTypeOpInterface.h
@@ -95,8 +95,6 @@ LogicalResult inferReturnTensorTypes(
 LogicalResult verifyInferredResultTypes(Operation *op);
 } // namespace detail
 
-#include "mlir/Interfaces/InferTypeOpInterface.h.inc"
-
 namespace OpTrait {
 
 /// Tensor type inference trait that constructs a tensor from the inferred
@@ -119,4 +117,7 @@ class InferTensorType : public TraitBase<ConcreteType, InferTensorType> {
 } // namespace OpTrait
 } // namespace mlir
 
+/// Include the generated interface declarations.
+#include "mlir/Interfaces/InferTypeOpInterface.h.inc"
+
 #endif // MLIR_INTERFACES_INFERTYPEOPINTERFACE_H_
diff --git a/mlir/include/mlir/Interfaces/InferTypeOpInterface.td b/mlir/include/mlir/Interfaces/InferTypeOpInterface.td
index 723cf99d38b31..c5132986ec976 100644
--- a/mlir/include/mlir/Interfaces/InferTypeOpInterface.td
+++ b/mlir/include/mlir/Interfaces/InferTypeOpInterface.td
@@ -25,6 +25,7 @@ def InferTypeOpInterface : OpInterface<"InferTypeOpInterface"> {
     Interface to infer the return types for an operation that could be used
     during op construction, verification or type inference.
   }];
+  let cppNamespace = "::mlir";
 
   let methods = [
     StaticInterfaceMethod<
@@ -73,6 +74,7 @@ def InferShapedTypeOpInterface : OpInterface<"InferShapedTypeOpInterface"> {
 
     The components consists of element type, shape and raw attribute.
   }];
+  let cppNamespace = "::mlir";
 
   let methods = [
     StaticInterfaceMethod<
diff --git a/mlir/include/mlir/Interfaces/LoopLikeInterface.h b/mlir/include/mlir/Interfaces/LoopLikeInterface.h
index 5891470c9c6e1..48399ad0d53a8 100644
--- a/mlir/include/mlir/Interfaces/LoopLikeInterface.h
+++ b/mlir/include/mlir/Interfaces/LoopLikeInterface.h
@@ -15,10 +15,7 @@
 
 #include "mlir/IR/OpDefinition.h"
 
-namespace mlir {
-
+/// Include the generated interface declarations.
 #include "mlir/Interfaces/LoopLikeInterface.h.inc"
 
-} // namespace mlir
-
 #endif // MLIR_INTERFACES_LOOPLIKEINTERFACE_H_
diff --git a/mlir/include/mlir/Interfaces/LoopLikeInterface.td b/mlir/include/mlir/Interfaces/LoopLikeInterface.td
index cc05030352e74..0e4191b97f97a 100644
--- a/mlir/include/mlir/Interfaces/LoopLikeInterface.td
+++ b/mlir/include/mlir/Interfaces/LoopLikeInterface.td
@@ -20,6 +20,7 @@ def LoopLikeOpInterface : OpInterface<"LoopLikeOpInterface"> {
     Encodes properties of a loop. Operations that implement this interface will
     be considered by loop-invariant code motion.
   }];
+  let cppNamespace = "::mlir";
 
   let methods = [
     InterfaceMethod<[{
diff --git a/mlir/include/mlir/Interfaces/SideEffectInterfaces.h b/mlir/include/mlir/Interfaces/SideEffectInterfaces.h
index 76932e2ef5293..181d218838ffb 100644
--- a/mlir/include/mlir/Interfaces/SideEffectInterfaces.h
+++ b/mlir/include/mlir/Interfaces/SideEffectInterfaces.h
@@ -215,13 +215,6 @@ struct Read : public Effect::Base<Read> {};
 struct Write : public Effect::Base<Write> {};
 } // namespace MemoryEffects
 
-//===----------------------------------------------------------------------===//
-// SideEffect Interfaces
-//===----------------------------------------------------------------------===//
-
-/// Include the definitions of the side effect interfaces.
-#include "mlir/Interfaces/SideEffectInterfaces.h.inc"
-
 //===----------------------------------------------------------------------===//
 // SideEffect Utilities
 //===----------------------------------------------------------------------===//
@@ -237,4 +230,11 @@ bool wouldOpBeTriviallyDead(Operation *op);
 
 } // end namespace mlir
 
+//===----------------------------------------------------------------------===//
+// SideEffect Interfaces
+//===----------------------------------------------------------------------===//
+
+/// Include the definitions of the side effect interfaces.
+#include "mlir/Interfaces/SideEffectInterfaces.h.inc"
+
 #endif // MLIR_INTERFACES_SIDEEFFECTS_H
diff --git a/mlir/include/mlir/Interfaces/SideEffectInterfaces.td b/mlir/include/mlir/Interfaces/SideEffectInterfaces.td
index b81b42b7a3b4e..2a4da16deec2d 100644
--- a/mlir/include/mlir/Interfaces/SideEffectInterfaces.td
+++ b/mlir/include/mlir/Interfaces/SideEffectInterfaces.td
@@ -61,7 +61,7 @@ class EffectOpInterfaceBase<string name, string baseEffect>
          (ins "Value":$value,
               "SmallVectorImpl<SideEffects::EffectInstance<"
               # baseEffect # ">> &":$effects), [{
-          op.getEffects(effects);
+          $_op.getEffects(effects);
           llvm::erase_if(effects, [&](auto &it) {
             return it.getValue() != value;
           });
@@ -75,7 +75,7 @@ class EffectOpInterfaceBase<string name, string baseEffect>
          (ins "SideEffects::Resource *":$resource,
               "SmallVectorImpl<SideEffects::EffectInstance<"
               # baseEffect # ">> &":$effects), [{
-          op.getEffects(effects);
+          $_op.getEffects(effects);
           llvm::erase_if(effects, [&](auto &it) {
             return it.getResource() != resource;
           });
@@ -142,6 +142,9 @@ class SideEffect<EffectOpInterfaceBase interface, string effectName,
   /// The parent interface that the effect belongs to.
   string interfaceTrait = interface.trait;
 
+  /// The cpp namespace of the interface trait.
+  string cppNamespace = interface.cppNamespace;
+
   /// The derived effect that is being applied.
   string effect = effectName;
 
@@ -156,6 +159,9 @@ class SideEffectsTraitBase<EffectOpInterfaceBase parentInterface,
   /// The name of the interface trait to use.
   let trait = parentInterface.trait;
 
+  /// The cpp namespace of the interface trait.
+  string cppNamespace = parentInterface.cppNamespace;
+
   /// The name of the base effects class.
   string baseEffectName = parentInterface.baseEffectName;
 
@@ -177,6 +183,7 @@ def MemoryEffectsOpInterface
     An interface used to query information about the memory effects applied by
     an operation.
   }];
+  let cppNamespace = "::mlir";
 }
 
 // The base class for defining specific memory effects.
diff --git a/mlir/include/mlir/Interfaces/VectorUnrollInterface.h b/mlir/include/mlir/Interfaces/VectorUnrollInterface.h
new file mode 100644
index 0000000000000..a68cc3411533b
--- /dev/null
+++ b/mlir/include/mlir/Interfaces/VectorUnrollInterface.h
@@ -0,0 +1,23 @@
+//===- VectorUnrollInterface.h - Vector unrolling interface ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the operation interface for vector ops that can be
+// unrolled.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_INTERFACES_VECTORUNROLLINTERFACE_H
+#define MLIR_INTERFACES_VECTORUNROLLINTERFACE_H
+
+#include "mlir/IR/OpDefinition.h"
+#include "mlir/IR/StandardTypes.h"
+
+/// Include the generated interface declarations.
+#include "mlir/Interfaces/VectorUnrollInterface.h.inc"
+
+#endif // MLIR_INTERFACES_VECTORUNROLLINTERFACE_H
diff --git a/mlir/include/mlir/Interfaces/VectorUnrollInterface.td b/mlir/include/mlir/Interfaces/VectorUnrollInterface.td
new file mode 100644
index 0000000000000..166780b20e772
--- /dev/null
+++ b/mlir/include/mlir/Interfaces/VectorUnrollInterface.td
@@ -0,0 +1,46 @@
+//===- VectorUnrollInterface.td - VectorUnroll interface ---*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the interface for operations on vectors that can be unrolled.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_INTERFACES_VECTORUNROLLINTERFACE
+#define MLIR_INTERFACES_VECTORUNROLLINTERFACE
+
+include "mlir/IR/OpBase.td"
+
+def VectorUnrollOpInterface : OpInterface<"VectorUnrollOpInterface"> {
+  let description = [{
+    Encodes properties of an operation on vectors that can be unrolled.
+  }];
+  let cppNamespace = "::mlir";
+
+  let methods = [
+    InterfaceMethod<[{
+        Returns the shape ratio of unrolling to the target vector shape
+        `targetShape`. Returns `None` if the op cannot be unrolled to the target
+        vector shape.
+      }],
+      "Optional<SmallVector<int64_t, 4>>",
+      "getShapeForUnroll",
+      (ins),
+      /*methodBody=*/[{}],
+      [{
+        auto vt = this->getOperation()->getResult(0).getType().
+          template dyn_cast<VectorType>();
+        if (!vt)
+          return None;
+        SmallVector<int64_t, 4> res(vt.getShape().begin(), vt.getShape().end());
+        return res;
+      }]
+    >,
+  ];
+}
+
+#endif // MLIR_INTERFACES_VECTORUNROLLINTERFACE
diff --git a/mlir/include/mlir/Interfaces/ViewLikeInterface.h b/mlir/include/mlir/Interfaces/ViewLikeInterface.h
index fe7dd803ccfba..8d319bbeee18f 100644
--- a/mlir/include/mlir/Interfaces/ViewLikeInterface.h
+++ b/mlir/include/mlir/Interfaces/ViewLikeInterface.h
@@ -15,10 +15,7 @@
 
 #include "mlir/IR/OpDefinition.h"
 
-namespace mlir {
-
+/// Include the generated interface declarations.
 #include "mlir/Interfaces/ViewLikeInterface.h.inc"
 
-} // namespace mlir
-
 #endif // MLIR_INTERFACES_VIEWLIKEINTERFACE_H_
diff --git a/mlir/include/mlir/Interfaces/ViewLikeInterface.td b/mlir/include/mlir/Interfaces/ViewLikeInterface.td
index 20b03b2315b18..bb00aff488b2a 100644
--- a/mlir/include/mlir/Interfaces/ViewLikeInterface.td
+++ b/mlir/include/mlir/Interfaces/ViewLikeInterface.td
@@ -21,6 +21,7 @@ def ViewLikeOpInterface : OpInterface<"ViewLikeOpInterface"> {
     takes in a (view of) buffer (and potentially some other operands) and returns
     another view of buffer.
   }];
+  let cppNamespace = "::mlir";
 
   let methods = [
     InterfaceMethod<
diff --git a/mlir/include/mlir/Parser.h b/mlir/include/mlir/Parser.h
index 40bd4f83d3458..8bba2ef4d53d4 100644
--- a/mlir/include/mlir/Parser.h
+++ b/mlir/include/mlir/Parser.h
@@ -58,7 +58,7 @@ OwningModuleRef parseSourceString(llvm::StringRef moduleStr,
 /// constructed from a new SourceMgr with a single a MemoryBuffer wrapping
 /// `attrStr`. If the passed `attrStr` has additional tokens that were not part
 /// of the type, an error is emitted.
-// TODO(ntv) Improve diagnostic reporting.
+// TODO: Improve diagnostic reporting.
 Attribute parseAttribute(llvm::StringRef attrStr, MLIRContext *context);
 Attribute parseAttribute(llvm::StringRef attrStr, Type type);
 
@@ -76,7 +76,7 @@ Attribute parseAttribute(llvm::StringRef attrStr, Type type, size_t &numRead);
 /// constructed from a new SourceMgr with a single a MemoryBuffer wrapping
 /// `typeStr`. If the passed `typeStr` has additional tokens that were not part
 /// of the type, an error is emitted.
-// TODO(ntv) Improve diagnostic reporting.
+// TODO: Improve diagnostic reporting.
 Type parseType(llvm::StringRef typeStr, MLIRContext *context);
 
 /// This parses a single MLIR type to an MLIR context if it was valid.  If not,
diff --git a/mlir/include/mlir/Pass/PassOptions.h b/mlir/include/mlir/Pass/PassOptions.h
index de4795a385f01..04754196e67a9 100644
--- a/mlir/include/mlir/Pass/PassOptions.h
+++ b/mlir/include/mlir/Pass/PassOptions.h
@@ -96,8 +96,8 @@ class PassOptions : protected llvm::cl::SubCommand {
   /// The specific parser to use depending on llvm::cl parser used. This is only
   /// necessary because we need to provide additional methods for certain data
   /// type parsers.
-  /// TODO(riverriddle) We should upstream the methods in GenericOptionParser to
-  /// avoid the need to do this.
+  /// TODO: We should upstream the methods in GenericOptionParser to avoid the
+  /// need to do this.
   template <typename DataType>
   using OptionParser =
       std::conditional_t<std::is_base_of<llvm::cl::generic_parser_base,
diff --git a/mlir/include/mlir/Reducer/Tester.h b/mlir/include/mlir/Reducer/Tester.h
new file mode 100644
index 0000000000000..8ca2a4a68765c
--- /dev/null
+++ b/mlir/include/mlir/Reducer/Tester.h
@@ -0,0 +1,59 @@
+//===- Tester.h -------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the Tester class used in the MLIR Reduce tool.
+//
+// A Tester object is passed as an argument to the reduction passes and it is
+// used to keep track of the state of the reduction throughout the multiple
+// passes.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_REDUCER_TESTER_H
+#define MLIR_REDUCER_TESTER_H
+
+#include <vector>
+
+#include "mlir/IR/Module.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Program.h"
+
+namespace mlir {
+
+/// This class is used to keep track of the state of the reduction. It contains
+/// a method to run the interestingness testing script on MLIR test case files
+/// and provides functionality to track the most reduced test case.
+class Tester {
+public:
+  Tester(StringRef testScript, ArrayRef<std::string> testScriptArgs);
+
+  /// Runs the interestingness testing script on a MLIR test case file. Returns
+  /// true if the interesting behavior is present in the test case or false
+  /// otherwise.
+  bool isInteresting(StringRef testCase);
+
+  /// Returns the most reduced MLIR test case module.
+  ModuleOp getMostReduced() const { return mostReduced; }
+
+  /// Updates the most reduced MLIR test case module. If a
+  /// generated variant is found to be successful and shorter than the
+  /// mostReduced module, the mostReduced module must be updated with the new
+  /// variant.
+  void setMostReduced(ModuleOp t) { mostReduced = t; }
+
+private:
+  StringRef testScript;
+  ArrayRef<std::string> testScriptArgs;
+  ModuleOp mostReduced;
+};
+
+} // end namespace mlir
+
+#endif
\ No newline at end of file
diff --git a/mlir/include/mlir/Support/InterfaceSupport.h b/mlir/include/mlir/Support/InterfaceSupport.h
new file mode 100644
index 0000000000000..e7a7eca90df29
--- /dev/null
+++ b/mlir/include/mlir/Support/InterfaceSupport.h
@@ -0,0 +1,210 @@
+//===- InterfaceSupport.h - MLIR Interface Support Classes ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines several support classes for defining interfaces.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_SUPPORT_INTERFACESUPPORT_H
+#define MLIR_SUPPORT_INTERFACESUPPORT_H
+
+#include "mlir/Support/TypeID.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/Support/TypeName.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace mlir {
+namespace detail {
+//===----------------------------------------------------------------------===//
+// Interface
+//===----------------------------------------------------------------------===//
+
+/// This class represents an abstract interface. An interface is a simplified
+/// mechanism for attaching concept based polymorphism to a class hierarchy. An
+/// interace is comprised of two components:
+/// * The derived interface class: This is what users interact with, and invoke
+///   methods on.
+/// * An interface `Trait` class: This is the class that is attached to the
+///   object implementing the interface. It is the mechanism with which models
+///   are specialized.
+///
+/// Derived interfaces types must provide the following template types:
+/// * ConcreteType: The CRTP derived type.
+/// * ValueT: The opaque type the derived interface operates on. For example
+///           `Operation*` for operation interfaces, or `Attribute` for
+///           attribute interfaces.
+/// * Traits: A class that contains definitions for a 'Concept' and a 'Model'
+///           class. The 'Concept' class defines an abstract virtual interface,
+///           where as the 'Model' class implements this interface for a
+///           specific derived T type. Both of these classes *must* not contain
+///           non-static data. A simple example is shown below:
+///
+/// ```c++
+///    struct ExampleInterfaceTraits {
+///      struct Concept {
+///        virtual unsigned getNumInputs(T t) const = 0;
+///      };
+///      template <typename DerivedT> class Model {
+///        unsigned getNumInputs(T t) const final {
+///          return cast<DerivedT>(t).getNumInputs();
+///        }
+///      };
+///    };
+/// ```
+///
+/// * BaseType: A desired base type for the interface. This is a class that
+///             provides that provides specific functionality for the `ValueT`
+///             value. For instance the specific `Op` that will wrap the
+///             `Operation*` for an `OpInterface`.
+/// * BaseTrait: The base type for the interface trait. This is the base class
+///              to use for the interface trait that will be attached to each
+///              instance of `ValueT` that implements this interface.
+///
+template <typename ConcreteType, typename ValueT, typename Traits,
+          typename BaseType,
+          template <typename, template <typename> class> class BaseTrait>
+class Interface : public BaseType {
+public:
+  using Concept = typename Traits::Concept;
+  template <typename T> using Model = typename Traits::template Model<T>;
+  using InterfaceBase =
+      Interface<ConcreteType, ValueT, Traits, BaseType, BaseTrait>;
+
+  Interface(ValueT t = ValueT())
+      : BaseType(t), impl(t ? ConcreteType::getInterfaceFor(t) : nullptr) {
+    assert((!t || impl) &&
+           "instantiating an interface with an unregistered operation");
+  }
+
+  /// Support 'classof' by checking if the given object defines the concrete
+  /// interface.
+  static bool classof(ValueT t) { return ConcreteType::getInterfaceFor(t); }
+
+  /// Define an accessor for the ID of this interface.
+  static TypeID getInterfaceID() { return TypeID::get<ConcreteType>(); }
+
+  /// This is a special trait that registers a given interface with an object.
+  template <typename ConcreteT>
+  struct Trait : public BaseTrait<ConcreteT, Trait> {
+    /// Define an accessor for the ID of this interface.
+    static TypeID getInterfaceID() { return TypeID::get<ConcreteType>(); }
+
+    /// Provide an accessor to a static instance of the interface model for the
+    /// concrete T type.
+    /// The implementation is inspired from Sean Parent's concept-based
+    /// polymorphism. A key difference is that the set of classes erased is
+    /// statically known, which alleviates the need for using dynamic memory
+    /// allocation.
+    /// We use a zero-sized templated class `Model<ConcreteT>` to emit the
+    /// virtual table and generate a singleton object for each instantiation of
+    /// this class.
+    static Concept &instance() {
+      static Model<ConcreteT> singleton;
+      return singleton;
+    }
+  };
+
+protected:
+  /// Get the raw concept in the correct derived concept type.
+  const Concept *getImpl() const { return impl; }
+  Concept *getImpl() { return impl; }
+
+private:
+  /// A pointer to the impl concept object.
+  Concept *impl;
+};
+
+//===----------------------------------------------------------------------===//
+// InterfaceMap
+//===----------------------------------------------------------------------===//
+
+/// Utility to filter a given sequence of types base upon a predicate.
+template <bool>
+struct FilterTypeT {
+  template <class E>
+  using type = std::tuple<E>;
+};
+template <>
+struct FilterTypeT<false> {
+  template <class E>
+  using type = std::tuple<>;
+};
+template <template <class> class Pred, class... Es>
+struct FilterTypes {
+  using type = decltype(std::tuple_cat(
+      std::declval<
+          typename FilterTypeT<Pred<Es>::value>::template type<Es>>()...));
+};
+
+/// This class provides an efficient mapping between a given `Interface` type,
+/// and a particular implementation of its concept.
+class InterfaceMap {
+public:
+  /// Construct an InterfaceMap with the given set of template types. For
+  /// convenience given that object trait lists may contain other non-interface
+  /// types, not all of the types need to be interfaces. The provided types that
+  /// do not represent interfaces are not added to the interface map.
+  template <typename... Types> static InterfaceMap get() {
+    return InterfaceMap(MapBuilder::create<Types...>());
+  }
+
+  /// Returns an instance of the concept object for the given interface if it
+  /// was registered to this map, null otherwise.
+  template <typename T> typename T::Concept *lookup() const {
+    if (!interfaces)
+      return nullptr;
+    return reinterpret_cast<typename T::Concept *>(
+        interfaces->lookup(T::getInterfaceID()));
+  }
+
+private:
+  /// This struct provides support for building a map of interfaces.
+  class MapBuilder {
+  public:
+    template <typename... Types>
+    static std::unique_ptr<llvm::SmallDenseMap<TypeID, void *>> create() {
+      // Filter the provided types for those that are interfaces. This reduces
+      // the amount of maps that are generated.
+      return createImpl((typename FilterTypes<detect_get_interface_id,
+                                              Types...>::type *)nullptr);
+    }
+
+  private:
+    /// Trait to check if T provides a static 'getInterfaceID' method.
+    template <typename T, typename... Args>
+    using has_get_interface_id = decltype(T::getInterfaceID());
+    template <typename T>
+    using detect_get_interface_id = llvm::is_detected<has_get_interface_id, T>;
+
+    template <typename... Ts>
+    static std::unique_ptr<llvm::SmallDenseMap<TypeID, void *>>
+    createImpl(std::tuple<Ts...> *) {
+      // Only create an instance of the map if there are any interface types.
+      if (sizeof...(Ts) == 0)
+        return std::unique_ptr<llvm::SmallDenseMap<TypeID, void *>>();
+
+      auto map = std::make_unique<llvm::SmallDenseMap<TypeID, void *>>();
+      (void)std::initializer_list<int>{
+          0, (map->try_emplace(Ts::getInterfaceID(), &Ts::instance()), 0)...};
+      return map;
+    }
+  };
+
+private:
+  InterfaceMap(std::unique_ptr<llvm::SmallDenseMap<TypeID, void *>> interfaces)
+      : interfaces(std::move(interfaces)) {}
+
+  /// The internal map of interfaces. This is constructed statically for each
+  /// set of interfaces.
+  std::unique_ptr<llvm::SmallDenseMap<TypeID, void *>> interfaces;
+};
+
+} // end namespace detail
+} // end namespace mlir
+
+#endif
diff --git a/mlir/include/mlir/TableGen/OpInterfaces.h b/mlir/include/mlir/TableGen/Interfaces.h
similarity index 64%
rename from mlir/include/mlir/TableGen/OpInterfaces.h
rename to mlir/include/mlir/TableGen/Interfaces.h
index 0e1b943ce382e..a3462097e4802 100644
--- a/mlir/include/mlir/TableGen/OpInterfaces.h
+++ b/mlir/include/mlir/TableGen/Interfaces.h
@@ -1,17 +1,13 @@
-//===- OpInterfaces.h - OpInterfaces wrapper class --------------*- C++ -*-===//
+//===- Interfaces.h - Interface wrapper classes -----------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
-// OpInterfaces wrapper to simplify using TableGen OpInterfaces.
-//
-//===----------------------------------------------------------------------===//
 
-#ifndef MLIR_TABLEGEN_OPINTERFACES_H_
-#define MLIR_TABLEGEN_OPINTERFACES_H_
+#ifndef MLIR_TABLEGEN_INTERFACES_H_
+#define MLIR_TABLEGEN_INTERFACES_H_
 
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/SmallVector.h"
@@ -25,9 +21,9 @@ class Record;
 namespace mlir {
 namespace tblgen {
 
-// Wrapper class with helper methods for accessing OpInterfaceMethod defined
+// Wrapper class with helper methods for accessing InterfaceMethod defined
 // in TableGen.
-class OpInterfaceMethod {
+class InterfaceMethod {
 public:
   // This struct represents a single method argument.
   struct Argument {
@@ -35,7 +31,7 @@ class OpInterfaceMethod {
     StringRef name;
   };
 
-  explicit OpInterfaceMethod(const llvm::Record *def);
+  explicit InterfaceMethod(const llvm::Record *def);
 
   // Return the return type of this method.
   StringRef getReturnType() const;
@@ -68,20 +64,23 @@ class OpInterfaceMethod {
 };
 
 //===----------------------------------------------------------------------===//
-// OpInterface
+// Interface
 //===----------------------------------------------------------------------===//
 
-// Wrapper class with helper methods for accessing OpInterfaces defined in
+// Wrapper class with helper methods for accessing Interfaces defined in
 // TableGen.
-class OpInterface {
+class Interface {
 public:
-  explicit OpInterface(const llvm::Record *def);
+  explicit Interface(const llvm::Record *def);
 
   // Return the name of this interface.
   StringRef getName() const;
 
+  // Return the C++ namespace of this interface.
+  StringRef getCppNamespace() const;
+
   // Return the methods of this interface.
-  ArrayRef<OpInterfaceMethod> getMethods() const;
+  ArrayRef<InterfaceMethod> getMethods() const;
 
   // Return the description of this method if it has one.
   llvm::Optional<StringRef> getDescription() const;
@@ -95,15 +94,36 @@ class OpInterface {
   // Return the verify method body if it has one.
   llvm::Optional<StringRef> getVerify() const;
 
+  // Returns the Tablegen definition this interface was constructed from.
+  const llvm::Record &getDef() const { return *def; }
+
 private:
   // The TableGen definition of this interface.
   const llvm::Record *def;
 
   // The methods of this interface.
-  SmallVector<OpInterfaceMethod, 8> methods;
+  SmallVector<InterfaceMethod, 8> methods;
 };
 
+// An interface that is registered to an Attribute.
+struct AttrInterface : public Interface {
+  using Interface::Interface;
+
+  static bool classof(const Interface *interface);
+};
+// An interface that is registered to an Operation.
+struct OpInterface : public Interface {
+  using Interface::Interface;
+
+  static bool classof(const Interface *interface);
+};
+// An interface that is registered to a Type.
+struct TypeInterface : public Interface {
+  using Interface::Interface;
+
+  static bool classof(const Interface *interface);
+};
 } // end namespace tblgen
 } // end namespace mlir
 
-#endif // MLIR_TABLEGEN_OPINTERFACES_H_
+#endif // MLIR_TABLEGEN_INTERFACES_H_
diff --git a/mlir/include/mlir/TableGen/OpTrait.h b/mlir/include/mlir/TableGen/OpTrait.h
index 269c6393e4341..cf8c506eb9f7e 100644
--- a/mlir/include/mlir/TableGen/OpTrait.h
+++ b/mlir/include/mlir/TableGen/OpTrait.h
@@ -25,7 +25,7 @@ class Record;
 namespace mlir {
 namespace tblgen {
 
-class OpInterface;
+struct OpInterface;
 
 // Wrapper class with helper methods for accessing OpTrait constraints defined
 // in TableGen.
@@ -98,7 +98,7 @@ class InterfaceOpTrait : public OpTrait {
   OpInterface getOpInterface() const;
 
   // Returns the trait corresponding to a C++ trait class.
-  StringRef getTrait() const;
+  std::string getTrait() const;
 
   static bool classof(const OpTrait *t) {
     return t->getKind() == Kind::Interface;
diff --git a/mlir/include/mlir/TableGen/Operator.h b/mlir/include/mlir/TableGen/Operator.h
index cce754dd34546..29d4caa324671 100644
--- a/mlir/include/mlir/TableGen/Operator.h
+++ b/mlir/include/mlir/TableGen/Operator.h
@@ -219,9 +219,9 @@ class Operator {
   StringRef getExtraClassDeclaration() const;
 
   // Returns the Tablegen definition this operator was constructed from.
-  // TODO(antiagainst,zinenko): do not expose the TableGen record, this is a
-  // temporary solution to OpEmitter requiring a Record because Operator does
-  // not provide enough methods.
+  // TODO: do not expose the TableGen record, this is a temporary solution to
+  // OpEmitter requiring a Record because Operator does not provide enough
+  // methods.
   const llvm::Record &getDef() const;
 
   // Returns the dialect of the op.
@@ -261,6 +261,22 @@ class Operator {
   // Requires: all result types are known.
   ArrayRef<ArgOrType> getSameTypeAsResult(int index) const;
 
+  // Pair consisting kind of argument and index into operands or attributes.
+  struct OperandOrAttribute {
+    enum class Kind { Operand, Attribute };
+    OperandOrAttribute(Kind kind, int index) {
+      packed = (index << 1) & (kind == Kind::Attribute);
+    }
+    int operandOrAttributeIndex() const { return (packed >> 1); }
+    Kind kind() { return (packed & 0x1) ? Kind::Attribute : Kind::Operand; }
+
+  private:
+    int packed;
+  };
+
+  // Returns the OperandOrAttribute corresponding to the index.
+  OperandOrAttribute getArgToOperandOrAttribute(int index) const;
+
 private:
   // Populates the vectors containing operands, attributes, results and traits.
   void populateOpStructure();
@@ -303,6 +319,9 @@ class Operator {
   // The argument with the same type as the result.
   SmallVector<SmallVector<ArgOrType, 2>, 4> resultTypeMapping;
 
+  // Map from argument to attribute or operand number.
+  SmallVector<OperandOrAttribute, 4> attrOrOperandMapping;
+
   // The number of native attributes stored in the leading positions of
   // `attributes`.
   int numNativeAttributes;
diff --git a/mlir/include/mlir/TableGen/Pattern.h b/mlir/include/mlir/TableGen/Pattern.h
index 94b9cde9332ae..a5759e358f695 100644
--- a/mlir/include/mlir/TableGen/Pattern.h
+++ b/mlir/include/mlir/TableGen/Pattern.h
@@ -407,8 +407,8 @@ class Pattern {
   const llvm::Record &def;
 
   // All operators.
-  // TODO(antiagainst): we need a proper context manager, like MLIRContext,
-  // for managing the lifetime of shared entities.
+  // TODO: we need a proper context manager, like MLIRContext, for managing the
+  // lifetime of shared entities.
   RecordOperatorMap *recordOpMap;
 };
 
diff --git a/mlir/include/mlir/TableGen/SideEffects.h b/mlir/include/mlir/TableGen/SideEffects.h
index 4680105152526..7e464476cea11 100644
--- a/mlir/include/mlir/TableGen/SideEffects.h
+++ b/mlir/include/mlir/TableGen/SideEffects.h
@@ -30,7 +30,7 @@ class SideEffect : public Operator::VariableDecorator {
   StringRef getBaseEffectName() const;
 
   // Return the name of the Interface that the effect belongs to.
-  StringRef getInterfaceTrait() const;
+  std::string getInterfaceTrait() const;
 
   // Return the name of the resource class.
   StringRef getResource() const;
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
index e7223bf7349a5..3a701018beb54 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
@@ -61,6 +61,8 @@ class ModuleTranslation {
     LLVM::ensureDistinctSuccessors(m);
 
     T translator(m, std::move(llvmModule));
+    if (failed(translator.convertFunctionSignatures()))
+      return nullptr;
     if (failed(translator.convertGlobals()))
       return nullptr;
     if (failed(translator.convertFunctions()))
@@ -94,6 +96,7 @@ class ModuleTranslation {
   /// Check whether the module contains only supported ops directly in its body.
   static LogicalResult checkSupportedModuleOps(Operation *m);
 
+  LogicalResult convertFunctionSignatures();
   LogicalResult convertFunctions();
   LogicalResult convertGlobals();
   LogicalResult convertOneFunction(LLVMFuncOp func);
diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h
index d862823930c5f..26b7ce6ea6c35 100644
--- a/mlir/include/mlir/Transforms/DialectConversion.h
+++ b/mlir/include/mlir/Transforms/DialectConversion.h
@@ -355,8 +355,7 @@ struct OpConversionPattern : public ConversionPattern {
     return matchAndRewrite(cast<SourceOp>(op), operands, rewriter);
   }
 
-  // TODO(b/142763075): Use OperandAdaptor when it supports access to unnamed
-  // operands.
+  // TODO: Use OperandAdaptor when it supports access to unnamed operands.
 
   /// Rewrite and Match methods that operate on the SourceOp type. These must be
   /// overridden by the derived pattern class.
diff --git a/mlir/include/mlir/Transforms/LoopFusionUtils.h b/mlir/include/mlir/Transforms/LoopFusionUtils.h
index 2832f050fc55b..36d2520b7c85e 100644
--- a/mlir/include/mlir/Transforms/LoopFusionUtils.h
+++ b/mlir/include/mlir/Transforms/LoopFusionUtils.h
@@ -24,9 +24,8 @@ class AffineForOp;
 struct ComputationSliceState;
 class Operation;
 
-// TODO(andydavis) Extend this module to include utility functions for querying
-// fusion cost/storage reduction, and for performing the loop fusion
-// transformation.
+// TODO: Extend this module to include utility functions for querying fusion
+// cost/storage reduction, and for performing the loop fusion transformation.
 
 struct FusionResult {
   enum ResultEnum {
@@ -46,7 +45,7 @@ struct FusionResult {
 /// returns a FusionResult explaining why fusion is not feasible.
 /// NOTE: This function is not feature complete and should only be used in
 /// testing.
-/// TODO(andydavis) Update comments when this function is fully implemented.
+/// TODO: Update comments when this function is fully implemented.
 FusionResult canFuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
                           unsigned dstLoopDepth,
                           ComputationSliceState *srcSlice);
@@ -71,14 +70,14 @@ struct LoopNestStats {
 /// Collect loop nest statistics (eg. loop trip count and operation count)
 /// in 'stats' for loop nest rooted at 'forOp'. Returns true on success,
 /// returns false otherwise.
-// TODO(andydavis) Consider moving this to LoopUtils.
+// TODO: Consider moving this to LoopUtils.
 bool getLoopNestStats(AffineForOp forOp, LoopNestStats *stats);
 
 /// Computes the total cost of the loop nest rooted at 'forOp' using 'stats'.
 /// Currently, the total cost is computed by counting the total operation
 /// instance count (i.e. total number of operations in the loop body * loop
 /// trip count) for the entire loop nest.
-// TODO(andydavis) Improve this cost model.
+// TODO: Improve this cost model.
 int64_t getComputeCost(AffineForOp forOp, LoopNestStats &stats);
 
 /// Computes and returns in 'computeCost', the total compute cost of fusing the
@@ -87,7 +86,7 @@ int64_t getComputeCost(AffineForOp forOp, LoopNestStats &stats);
 /// (i.e. total number of operations in the loop body * loop trip count) for
 /// the entire loop nest.
 /// Returns true on success, failure otherwise (e.g. non-constant trip counts).
-// TODO(andydavis) Improve this cost model.
+// TODO: Improve this cost model.
 bool getFusionComputeCost(AffineForOp srcForOp, LoopNestStats &srcStats,
                           AffineForOp dstForOp, LoopNestStats &dstStats,
                           ComputationSliceState *slice, int64_t *computeCost);
diff --git a/mlir/include/mlir/Transforms/Passes.h b/mlir/include/mlir/Transforms/Passes.h
index f54333df2dd62..d528723af8ec6 100644
--- a/mlir/include/mlir/Transforms/Passes.h
+++ b/mlir/include/mlir/Transforms/Passes.h
@@ -32,6 +32,9 @@ std::unique_ptr<Pass> createBufferPlacementPass();
 /// Creates an instance of the Canonicalizer pass.
 std::unique_ptr<Pass> createCanonicalizerPass();
 
+/// Create a pass that removes unnecessary Copy operations.
+std::unique_ptr<Pass> createCopyRemovalPass();
+
 /// Creates a pass to perform common sub expression elimination.
 std::unique_ptr<Pass> createCSEPass();
 
diff --git a/mlir/include/mlir/Transforms/Passes.td b/mlir/include/mlir/Transforms/Passes.td
index 4cc7d631fb94a..9e0d5c40d61fc 100644
--- a/mlir/include/mlir/Transforms/Passes.td
+++ b/mlir/include/mlir/Transforms/Passes.td
@@ -174,6 +174,11 @@ def Canonicalizer : Pass<"canonicalize"> {
   let constructor = "mlir::createCanonicalizerPass()";
 }
 
+def CopyRemoval : FunctionPass<"copy-removal"> {
+  let summary = "Remove the redundant copies from input IR";
+  let constructor = "mlir::createCopyRemovalPass()";
+}
+
 def CSE : Pass<"cse"> {
   let summary = "Eliminate common sub-expressions";
   let description = [{
diff --git a/mlir/include/mlir/Transforms/Utils.h b/mlir/include/mlir/Transforms/Utils.h
index acd5fd9efe410..6f29c1b41ae63 100644
--- a/mlir/include/mlir/Transforms/Utils.h
+++ b/mlir/include/mlir/Transforms/Utils.h
@@ -56,7 +56,7 @@ class OpBuilder;
 //  d1, d2) -> (d0 - d1, d2), and %ii will be the extra operand. Without any
 //  extra operands, note that 'indexRemap' would just be applied to existing
 //  indices (%i, %j).
-//  TODO(bondhugula): allow extraIndices to be added at any position.
+//  TODO: allow extraIndices to be added at any position.
 LogicalResult replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
                                        ArrayRef<Value> extraIndices = {},
                                        AffineMap indexRemap = AffineMap(),
diff --git a/mlir/integration_test/CMakeLists.txt b/mlir/integration_test/CMakeLists.txt
index d2a768da1a3e0..8201cd79e37fd 100644
--- a/mlir/integration_test/CMakeLists.txt
+++ b/mlir/integration_test/CMakeLists.txt
@@ -21,6 +21,10 @@ add_lit_testsuite(check-mlir-integration "Running the MLIR integration tests"
   )
 set_target_properties(check-mlir-integration PROPERTIES FOLDER "integration_test")
 
+# Let target "check-mlir" imply the target "check-mlir-integration" as well,
+# provided that MLIR_INCLUDE_INTEGRATION_TESTS=ON during the initial setup.
+add_dependencies(check-mlir check-mlir-integration)
+
 add_lit_testsuites(MLIR_INTEGRATION ${CMAKE_CURRENT_SOURCE_DIR}
   DEPENDS ${MLIR_INTEGRATION_TEST_DEPENDS}
   )
diff --git a/mlir/integration_test/Dialect/LLVMIR/CPU/test-vector-reductions-fp.mlir b/mlir/integration_test/Dialect/LLVMIR/CPU/test-vector-reductions-fp.mlir
index f8be3a6dd832a..9b61414277b6a 100644
--- a/mlir/integration_test/Dialect/LLVMIR/CPU/test-vector-reductions-fp.mlir
+++ b/mlir/integration_test/Dialect/LLVMIR/CPU/test-vector-reductions-fp.mlir
@@ -42,24 +42,48 @@ module {
     llvm.call @print_newline() : () -> ()
     // CHECK: 11
 
+    %add1r = "llvm.intr.experimental.vector.reduce.v2.fadd"(%0, %v)
+        {reassoc = true} : (!llvm.float, !llvm<"<4 x float>">) -> !llvm.float
+    llvm.call @print_f32(%add1r) : (!llvm.float) -> ()
+    llvm.call @print_newline() : () -> ()
+    // CHECK: 11
+
     %add2 = "llvm.intr.experimental.vector.reduce.v2.fadd"(%1, %v)
         : (!llvm.float, !llvm<"<4 x float>">) -> !llvm.float
     llvm.call @print_f32(%add2) : (!llvm.float) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 12
 
+    %add2r = "llvm.intr.experimental.vector.reduce.v2.fadd"(%1, %v)
+        {reassoc = true} : (!llvm.float, !llvm<"<4 x float>">) -> !llvm.float
+    llvm.call @print_f32(%add2r) : (!llvm.float) -> ()
+    llvm.call @print_newline() : () -> ()
+    // CHECK: 12
+
     %mul1 = "llvm.intr.experimental.vector.reduce.v2.fmul"(%0, %v)
         : (!llvm.float, !llvm<"<4 x float>">) -> !llvm.float
     llvm.call @print_f32(%mul1) : (!llvm.float) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 24
 
+    %mul1r = "llvm.intr.experimental.vector.reduce.v2.fmul"(%0, %v)
+        {reassoc = true} : (!llvm.float, !llvm<"<4 x float>">) -> !llvm.float
+    llvm.call @print_f32(%mul1r) : (!llvm.float) -> ()
+    llvm.call @print_newline() : () -> ()
+    // CHECK: 24
+
     %mul2 = "llvm.intr.experimental.vector.reduce.v2.fmul"(%1, %v)
         : (!llvm.float, !llvm<"<4 x float>">) -> !llvm.float
     llvm.call @print_f32(%mul2) : (!llvm.float) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 48
 
+    %mul2r = "llvm.intr.experimental.vector.reduce.v2.fmul"(%1, %v)
+        {reassoc = true} : (!llvm.float, !llvm<"<4 x float>">) -> !llvm.float
+    llvm.call @print_f32(%mul2r) : (!llvm.float) -> ()
+    llvm.call @print_newline() : () -> ()
+    // CHECK: 48
+
     llvm.return
   }
 }
diff --git a/mlir/integration_test/Dialect/Vector/CPU/test-outerproduct.mlir b/mlir/integration_test/Dialect/Vector/CPU/test-outerproduct-f32.mlir
similarity index 75%
rename from mlir/integration_test/Dialect/Vector/CPU/test-outerproduct.mlir
rename to mlir/integration_test/Dialect/Vector/CPU/test-outerproduct-f32.mlir
index a6470c656f8cc..9b86976e49010 100644
--- a/mlir/integration_test/Dialect/Vector/CPU/test-outerproduct.mlir
+++ b/mlir/integration_test/Dialect/Vector/CPU/test-outerproduct-f32.mlir
@@ -11,6 +11,8 @@
 !vector_type_Y = type vector<3xf32>
 !vector_type_Z = type vector<2x3xf32>
 
+!vector_type_R = type vector<7xf32>
+
 func @vector_outerproduct_splat_8x8(%fa: f32, %fb: f32, %fc: f32) -> !vector_type_C {
   %a = splat %fa: !vector_type_A
   %b = splat %fb: !vector_type_B
@@ -33,6 +35,7 @@ func @vector_outerproduct_vec_2x3_acc(%x : !vector_type_X,
 }
 
 func @entry() {
+  %f0 = constant 0.0: f32
   %f1 = constant 1.0: f32
   %f2 = constant 2.0: f32
   %f3 = constant 3.0: f32
@@ -72,5 +75,26 @@ func @entry() {
   //
   // CHECK: ( ( 6, 8, 10 ), ( 12, 16, 20 ) )
 
+  %3 = vector.broadcast %f0 : f32 to !vector_type_R
+  %4 = vector.insert %f1,  %3[1] : f32 into !vector_type_R
+  %5 = vector.insert %f2,  %4[2] : f32 into !vector_type_R
+  %6 = vector.insert %f3,  %5[3] : f32 into !vector_type_R
+  %7 = vector.insert %f4,  %6[4] : f32 into !vector_type_R
+  %8 = vector.insert %f5,  %7[5] : f32 into !vector_type_R
+  %9 = vector.insert %f10, %8[6] : f32 into !vector_type_R
+
+  %o = vector.broadcast %f1 : f32 to !vector_type_R
+
+  %axpy1 = vector.outerproduct %9, %f2     : !vector_type_R, f32
+  %axpy2 = vector.outerproduct %9, %f2, %o : !vector_type_R, f32
+
+  vector.print %axpy1 : !vector_type_R
+  vector.print %axpy2 : !vector_type_R
+  //
+  // axpy operations:
+  //
+  // CHECK: ( 0, 2, 4, 6, 8, 10, 20 )
+  // CHECK: ( 1, 3, 5, 7, 9, 11, 21 )
+
   return
 }
diff --git a/mlir/integration_test/Dialect/Vector/CPU/test-outerproduct-i64.mlir b/mlir/integration_test/Dialect/Vector/CPU/test-outerproduct-i64.mlir
new file mode 100644
index 0000000000000..c2724aec9b373
--- /dev/null
+++ b/mlir/integration_test/Dialect/Vector/CPU/test-outerproduct-i64.mlir
@@ -0,0 +1,100 @@
+// RUN: mlir-opt %s -convert-scf-to-std -convert-vector-to-llvm -convert-std-to-llvm | \
+// RUN: mlir-cpu-runner -e entry -entry-point-result=void  \
+// RUN:   -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
+// RUN: FileCheck %s
+
+!vector_type_A = type vector<8xi64>
+!vector_type_B = type vector<8xi64>
+!vector_type_C = type vector<8x8xi64>
+
+!vector_type_X = type vector<2xi64>
+!vector_type_Y = type vector<3xi64>
+!vector_type_Z = type vector<2x3xi64>
+
+!vector_type_R = type vector<7xi64>
+
+func @vector_outerproduct_splat_8x8(%ia: i64, %ib: i64, %ic: i64) -> !vector_type_C {
+  %a = splat %ia: !vector_type_A
+  %b = splat %ib: !vector_type_B
+  %c = splat %ic: !vector_type_C
+  %d = vector.outerproduct %a, %b, %c : !vector_type_A, !vector_type_B
+  return %d: !vector_type_C
+}
+
+func @vector_outerproduct_vec_2x3(%x : !vector_type_X,
+                                  %y : !vector_type_Y) -> !vector_type_Z {
+  %o = vector.outerproduct %x, %y : !vector_type_X, !vector_type_Y
+  return %o: !vector_type_Z
+}
+
+func @vector_outerproduct_vec_2x3_acc(%x : !vector_type_X,
+                                      %y : !vector_type_Y,
+                                      %z : !vector_type_Z) -> !vector_type_Z {
+  %o = vector.outerproduct %x, %y, %z : !vector_type_X, !vector_type_Y
+  return %o: !vector_type_Z
+}
+
+func @entry() {
+  %i0 = constant 0: i64
+  %i1 = constant 1: i64
+  %i2 = constant 2: i64
+  %i3 = constant 3: i64
+  %i4 = constant 4: i64
+  %i5 = constant 5: i64
+  %i10 = constant 10: i64
+
+  // Simple case, splat scalars into vectors, then take outer product.
+  %v = call @vector_outerproduct_splat_8x8(%i1, %i2, %i10)
+      : (i64, i64, i64) -> (!vector_type_C)
+  vector.print %v : !vector_type_C
+  //
+  // outer product 8x8:
+  //
+  // CHECK-COUNT-8: ( 12, 12, 12, 12, 12, 12, 12, 12 )
+
+  // Direct outerproduct on vectors with different size.
+  %0 = vector.broadcast %i1 : i64 to !vector_type_X
+  %x = vector.insert %i2, %0[1] : i64 into !vector_type_X
+  %1 = vector.broadcast %i3 : i64 to !vector_type_Y
+  %2 = vector.insert %i4, %1[1] : i64 into !vector_type_Y
+  %y = vector.insert %i5, %2[2] : i64 into !vector_type_Y
+
+  %p = call @vector_outerproduct_vec_2x3(%x, %y)
+      : (!vector_type_X, !vector_type_Y) -> (!vector_type_Z)
+  vector.print %p : !vector_type_Z
+  //
+  // outer product 2x3:
+  //
+  // CHECK: ( ( 3, 4, 5 ), ( 6, 8, 10 ) )
+
+  %q = call @vector_outerproduct_vec_2x3_acc(%x, %y, %p)
+      : (!vector_type_X, !vector_type_Y, !vector_type_Z) -> (!vector_type_Z)
+  vector.print %q : !vector_type_Z
+  //
+  // outer product 2x3:
+  //
+  // CHECK: ( ( 6, 8, 10 ), ( 12, 16, 20 ) )
+
+  %3 = vector.broadcast %i0 : i64 to !vector_type_R
+  %4 = vector.insert %i1,  %3[1] : i64 into !vector_type_R
+  %5 = vector.insert %i2,  %4[2] : i64 into !vector_type_R
+  %6 = vector.insert %i3,  %5[3] : i64 into !vector_type_R
+  %7 = vector.insert %i4,  %6[4] : i64 into !vector_type_R
+  %8 = vector.insert %i5,  %7[5] : i64 into !vector_type_R
+  %9 = vector.insert %i10, %8[6] : i64 into !vector_type_R
+
+  %o = vector.broadcast %i1 : i64 to !vector_type_R
+
+  %axpy1 = vector.outerproduct %9, %i2     : !vector_type_R, i64
+  %axpy2 = vector.outerproduct %9, %i2, %o : !vector_type_R, i64
+
+  vector.print %axpy1 : !vector_type_R
+  vector.print %axpy2 : !vector_type_R
+  //
+  // axpy operations:
+  //
+  // CHECK: ( 0, 2, 4, 6, 8, 10, 20 )
+  // CHECK: ( 1, 3, 5, 7, 9, 11, 21 )
+
+  return
+}
diff --git a/mlir/integration_test/Dialect/Vector/CPU/test-reductions-f32-reassoc.mlir b/mlir/integration_test/Dialect/Vector/CPU/test-reductions-f32-reassoc.mlir
new file mode 100644
index 0000000000000..bb2636403beeb
--- /dev/null
+++ b/mlir/integration_test/Dialect/Vector/CPU/test-reductions-f32-reassoc.mlir
@@ -0,0 +1,38 @@
+// RUN: mlir-opt %s -convert-scf-to-std \
+// RUN:             -convert-vector-to-llvm='reassociate-fp-reductions' \
+// RUN:             -convert-std-to-llvm | \
+// RUN: mlir-cpu-runner -e entry -entry-point-result=void  \
+// RUN:   -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
+// RUN: FileCheck %s
+
+func @entry() {
+  // Construct test vector, numerically very stable.
+  %f1 = constant 1.0: f32
+  %f2 = constant 2.0: f32
+  %f3 = constant 3.0: f32
+  %v0 = vector.broadcast %f1 : f32 to vector<64xf32>
+  %v1 = vector.insert %f2, %v0[11] : f32 into vector<64xf32>
+  %v2 = vector.insert %f3, %v1[52] : f32 into vector<64xf32>
+  vector.print %v2 : vector<64xf32>
+  //
+  // test vector:
+  //
+  // CHECK: ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 )
+
+  // Various vector reductions. Not full functional unit tests, but
+  // a simple integration test to see if the code runs end-to-end.
+  %0 = vector.reduction "add", %v2 : vector<64xf32> into f32
+  vector.print %0 : f32
+  // CHECK: 67
+  %1 = vector.reduction "mul", %v2 : vector<64xf32> into f32
+  vector.print %1 : f32
+  // CHECK: 6
+  %2 = vector.reduction "min", %v2 : vector<64xf32> into f32
+  vector.print %2 : f32
+  // CHECK: 1
+  %3 = vector.reduction "max", %v2 : vector<64xf32> into f32
+  vector.print %3 : f32
+  // CHECK: 3
+
+  return
+}
diff --git a/mlir/integration_test/Dialect/Vector/CPU/test-reductions-f64-reassoc.mlir b/mlir/integration_test/Dialect/Vector/CPU/test-reductions-f64-reassoc.mlir
new file mode 100644
index 0000000000000..2dd8d0c650f1d
--- /dev/null
+++ b/mlir/integration_test/Dialect/Vector/CPU/test-reductions-f64-reassoc.mlir
@@ -0,0 +1,38 @@
+// RUN: mlir-opt %s -convert-scf-to-std \
+// RUN:             -convert-vector-to-llvm='reassociate-fp-reductions' \
+// RUN:             -convert-std-to-llvm | \
+// RUN: mlir-cpu-runner -e entry -entry-point-result=void  \
+// RUN:   -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
+// RUN: FileCheck %s
+
+func @entry() {
+  // Construct test vector, numerically very stable.
+  %f1 = constant 1.0: f64
+  %f2 = constant 2.0: f64
+  %f3 = constant 3.0: f64
+  %v0 = vector.broadcast %f1 : f64 to vector<64xf64>
+  %v1 = vector.insert %f2, %v0[11] : f64 into vector<64xf64>
+  %v2 = vector.insert %f3, %v1[52] : f64 into vector<64xf64>
+  vector.print %v2 : vector<64xf64>
+  //
+  // test vector:
+  //
+  // CHECK: ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 )
+
+  // Various vector reductions. Not full functional unit tests, but
+  // a simple integration test to see if the code runs end-to-end.
+  %0 = vector.reduction "add", %v2 : vector<64xf64> into f64
+  vector.print %0 : f64
+  // CHECK: 67
+  %1 = vector.reduction "mul", %v2 : vector<64xf64> into f64
+  vector.print %1 : f64
+  // CHECK: 6
+  %2 = vector.reduction "min", %v2 : vector<64xf64> into f64
+  vector.print %2 : f64
+  // CHECK: 1
+  %3 = vector.reduction "max", %v2 : vector<64xf64> into f64
+  vector.print %3 : f64
+  // CHECK: 3
+
+  return
+}
diff --git a/mlir/lib/Analysis/AffineAnalysis.cpp b/mlir/lib/Analysis/AffineAnalysis.cpp
index ca662c71fe436..3bacedae80ffb 100644
--- a/mlir/lib/Analysis/AffineAnalysis.cpp
+++ b/mlir/lib/Analysis/AffineAnalysis.cpp
@@ -33,8 +33,8 @@ using llvm::dbgs;
 /// Returns the sequence of AffineApplyOp Operations operation in
 /// 'affineApplyOps', which are reachable via a search starting from 'operands',
 /// and ending at operands which are not defined by AffineApplyOps.
-// TODO(andydavis) Add a method to AffineApplyOp which forward substitutes
-// the AffineApplyOp into any user AffineApplyOps.
+// TODO: Add a method to AffineApplyOp which forward substitutes the
+// AffineApplyOp into any user AffineApplyOps.
 void mlir::getReachableAffineApplyOps(
     ArrayRef<Value> operands, SmallVectorImpl<Operation *> &affineApplyOps) {
   struct State {
@@ -81,10 +81,10 @@ void mlir::getReachableAffineApplyOps(
 // the loop IVs of the forOps appearing in that order. Any symbols founds in
 // the bound operands are added as symbols in the system. Returns failure for
 // the yet unimplemented cases.
-// TODO(andydavis,bondhugula) Handle non-unit steps through local variables or
-// stride information in FlatAffineConstraints. (For eg., by using iv - lb %
-// step = 0 and/or by introducing a method in FlatAffineConstraints
-// setExprStride(ArrayRef<int64_t> expr, int64_t stride)
+// TODO: Handle non-unit steps through local variables or stride information in
+// FlatAffineConstraints. (For eg., by using iv - lb % step = 0 and/or by
+// introducing a method in FlatAffineConstraints setExprStride(ArrayRef<int64_t>
+// expr, int64_t stride)
 LogicalResult mlir::getIndexSet(MutableArrayRef<AffineForOp> forOps,
                                 FlatAffineConstraints *domain) {
   SmallVector<Value, 4> indices;
@@ -104,10 +104,10 @@ LogicalResult mlir::getIndexSet(MutableArrayRef<AffineForOp> forOps,
 // potentially involving any Function symbols. The dimensional identifiers in
 // 'indexSet' correspond to the loops surrounding 'op' from outermost to
 // innermost.
-// TODO(andydavis) Add support to handle IfInsts surrounding 'op'.
+// TODO: Add support to handle IfInsts surrounding 'op'.
 static LogicalResult getInstIndexSet(Operation *op,
                                      FlatAffineConstraints *indexSet) {
-  // TODO(andydavis) Extend this to gather enclosing IfInsts and consider
+  // TODO: Extend this to gather enclosing IfInsts and consider
   // factoring it out into a utility function.
   SmallVector<AffineForOp, 4> loops;
   getLoopIVs(*op, &loops);
@@ -130,9 +130,9 @@ namespace {
 // an Value in multiple maps are provided (i.e. getSrcDimOrSymPos) to handle
 // the common case of resolving positions for all access function operands.
 //
-// TODO(andydavis) Generalize this: could take a template parameter for
-// the number of maps (3 in the current case), and lookups could take indices
-// of maps to check. So getSrcDimOrSymPos would be "getPos(value, {0, 2})".
+// TODO: Generalize this: could take a template parameter for the number of maps
+// (3 in the current case), and lookups could take indices of maps to check. So
+// getSrcDimOrSymPos would be "getPos(value, {0, 2})".
 class ValuePositionMap {
 public:
   void addSrcValue(Value value) {
@@ -758,7 +758,7 @@ void MemRefAccess::getAccessMap(AffineValueMap *accessMap) const {
 //       0         0         0        -1        0     0     0     50   >= 0
 //
 //
-// TODO(andydavis) Support AffineExprs mod/floordiv/ceildiv.
+// TODO: Support AffineExprs mod/floordiv/ceildiv.
 DependenceResult mlir::checkMemrefAccessDependence(
     const MemRefAccess &srcAccess, const MemRefAccess &dstAccess,
     unsigned loopDepth, FlatAffineConstraints *dependenceConstraints,
@@ -859,8 +859,7 @@ void mlir::getDependenceComponents(
   // Collect all load and store ops in loop nest rooted at 'forOp'.
   SmallVector<Operation *, 8> loadAndStoreOpInsts;
   forOp.getOperation()->walk([&](Operation *opInst) {
-    if (isa<AffineReadOpInterface>(opInst) ||
-        isa<AffineWriteOpInterface>(opInst))
+    if (isa<AffineReadOpInterface, AffineWriteOpInterface>(opInst))
       loadAndStoreOpInsts.push_back(opInst);
   });
 
@@ -875,8 +874,8 @@ void mlir::getDependenceComponents(
 
         FlatAffineConstraints dependenceConstraints;
         SmallVector<DependenceComponent, 2> depComps;
-        // TODO(andydavis,bondhugula) Explore whether it would be profitable
-        // to pre-compute and store deps instead of repeatedly checking.
+        // TODO: Explore whether it would be profitable to pre-compute and store
+        // deps instead of repeatedly checking.
         DependenceResult result = checkMemrefAccessDependence(
             srcAccess, dstAccess, d, &dependenceConstraints, &depComps);
         if (hasDependence(result))
diff --git a/mlir/lib/Analysis/AffineStructures.cpp b/mlir/lib/Analysis/AffineStructures.cpp
index 5c3f33d0a6935..c09a9af45712a 100644
--- a/mlir/lib/Analysis/AffineStructures.cpp
+++ b/mlir/lib/Analysis/AffineStructures.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Analysis/AffineStructures.h"
+#include "mlir/Analysis/Presburger/Simplex.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/IR/AffineValueMap.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
@@ -893,7 +894,7 @@ void FlatAffineConstraints::removeIdRange(unsigned idStart, unsigned idLimit) {
   // We are going to be removing one or more identifiers from the range.
   assert(idStart < numIds && "invalid idStart position");
 
-  // TODO(andydavis) Make 'removeIdRange' a lambda called from here.
+  // TODO: Make 'removeIdRange' a lambda called from here.
   // Remove eliminated identifiers from equalities.
   shiftColumnsToLeft(this, idStart, idLimit, /*isEq=*/true);
 
@@ -1035,6 +1036,28 @@ bool FlatAffineConstraints::isEmptyByGCDTest() const {
   return false;
 }
 
+// First, try the GCD test heuristic.
+//
+// If that doesn't find the set empty, check if the set is unbounded. If it is,
+// we cannot use the GBR algorithm and we conservatively return false.
+//
+// If the set is bounded, we use the complete emptiness check for this case
+// provided by Simplex::findIntegerSample(), which gives a definitive answer.
+bool FlatAffineConstraints::isIntegerEmpty() const {
+  if (isEmptyByGCDTest())
+    return true;
+
+  Simplex simplex(*this);
+  if (simplex.isUnbounded())
+    return false;
+  return !simplex.findIntegerSample().hasValue();
+}
+
+Optional<SmallVector<int64_t, 8>>
+FlatAffineConstraints::findIntegerSample() const {
+  return Simplex(*this).findIntegerSample();
+}
+
 /// Tightens inequalities given that we are dealing with integer spaces. This is
 /// analogous to the GCD test but applied to inequalities. The constant term can
 /// be reduced to the preceding multiple of the GCD of the coefficients, i.e.,
@@ -1150,8 +1173,8 @@ static bool detectAsMod(const FlatAffineConstraints &cst, unsigned pos,
       if (c == pos)
         continue;
       // The coefficient of the quotient should be +/-divisor.
-      // TODO(bondhugula): could be extended to detect an affine function for
-      // the quotient (i.e., the coeff could be a non-zero multiple of divisor).
+      // TODO: could be extended to detect an affine function for the quotient
+      // (i.e., the coeff could be a non-zero multiple of divisor).
       int64_t v = cst.atEq(r, c) * cst.atEq(r, pos);
       if (v == divisor || v == -divisor) {
         seenQuotient++;
@@ -1159,8 +1182,8 @@ static bool detectAsMod(const FlatAffineConstraints &cst, unsigned pos,
         quotientSign = v > 0 ? 1 : -1;
       }
       // The coefficient of the dividend should be +/-1.
-      // TODO(bondhugula): could be extended to detect an affine function of
-      // the other identifiers as the dividend.
+      // TODO: could be extended to detect an affine function of the other
+      // identifiers as the dividend.
       else if (v == -1 || v == 1) {
         seenDividend++;
         dividendPos = c;
@@ -1319,8 +1342,8 @@ static bool detectAsFloorDiv(const FlatAffineConstraints &cst, unsigned pos,
         }
         // Expression can't be constructed as it depends on a yet unknown
         // identifier.
-        // TODO(mlir-team): Visit/compute the identifiers in an order so that
-        // this doesn't happen. More complex but much more efficient.
+        // TODO: Visit/compute the identifiers in an order so that this doesn't
+        // happen. More complex but much more efficient.
         if (c < f)
           continue;
         // Successfully detected the floordiv.
@@ -1596,9 +1619,9 @@ void FlatAffineConstraints::getSliceBounds(unsigned offset, unsigned num,
       lbMap = AffineMap::get(numMapDims, numMapSymbols, expr);
       ubMap = AffineMap::get(numMapDims, numMapSymbols, expr + 1);
     } else {
-      // TODO(bondhugula): Whenever there are local identifiers in the
-      // dependence constraints, we'll conservatively over-approximate, since we
-      // don't always explicitly compute them above (in the while loop).
+      // TODO: Whenever there are local identifiers in the dependence
+      // constraints, we'll conservatively over-approximate, since we don't
+      // always explicitly compute them above (in the while loop).
       if (getNumLocalIds() == 0) {
         // Work on a copy so that we don't update this constraint system.
         if (!tmpClone) {
@@ -1613,7 +1636,7 @@ void FlatAffineConstraints::getSliceBounds(unsigned offset, unsigned num,
 
       // If the above fails, we'll just use the constant lower bound and the
       // constant upper bound (if they exist) as the slice bounds.
-      // TODO(b/126426796): being conservative for the moment in cases that
+      // TODO: being conservative for the moment in cases that
       // lead to multiple bounds - until getConstDifference in LoopFusion.cpp is
       // fixed (b/126426796).
       if (!lbMap || lbMap.getNumResults() > 1) {
@@ -2333,8 +2356,8 @@ void FlatAffineConstraints::removeTrivialRedundancy() {
   }
   inequalities.resize(numReservedCols * pos);
 
-  // TODO(bondhugula): consider doing this for equalities as well, but probably
-  // not worth the savings.
+  // TODO: consider doing this for equalities as well, but probably not worth
+  // the savings.
 }
 
 void FlatAffineConstraints::clearAndCopyFrom(
@@ -2411,8 +2434,8 @@ getNewNumDimsSymbols(unsigned pos, const FlatAffineConstraints &cst) {
 /// holes/splinters:                         j = 2
 ///
 /// darkShadow = false, isResultIntegerExact = nullptr are default values.
-// TODO(bondhugula): a slight modification to yield dark shadow version of FM
-// (tightened), which can prove the existence of a solution if there is one.
+// TODO: a slight modification to yield dark shadow version of FM (tightened),
+// which can prove the existence of a solution if there is one.
 void FlatAffineConstraints::FourierMotzkinEliminate(
     unsigned pos, bool darkShadow, bool *isResultIntegerExact) {
   LLVM_DEBUG(llvm::dbgs() << "FM input (eliminate pos " << pos << "):\n");
@@ -2444,7 +2467,7 @@ void FlatAffineConstraints::FourierMotzkinEliminate(
   }
   if (r == getNumInequalities()) {
     // If it doesn't appear, just remove the column and return.
-    // TODO(andydavis,bondhugula): refactor removeColumns to use it from here.
+    // TODO: refactor removeColumns to use it from here.
     removeId(pos);
     LLVM_DEBUG(llvm::dbgs() << "FM output:\n");
     LLVM_DEBUG(dump());
@@ -2515,7 +2538,7 @@ void FlatAffineConstraints::FourierMotzkinEliminate(
       // coefficient in the canonical form as the view taken here is that of the
       // term being moved to the other size of '>='.
       int64_t ubCoeff = -atIneq(ubPos, pos);
-      // TODO(bondhugula): refactor this loop to avoid all branches inside.
+      // TODO: refactor this loop to avoid all branches inside.
       for (unsigned l = 0, e = getNumCols(); l < e; l++) {
         if (l == pos)
           continue;
@@ -2719,14 +2742,14 @@ FlatAffineConstraints::unionBoundingBox(const FlatAffineConstraints &otherCst) {
   for (unsigned d = 0, e = getNumDimIds(); d < e; ++d) {
     auto extent = getConstantBoundOnDimSize(d, &lb, &lbFloorDivisor, &ub);
     if (!extent.hasValue())
-      // TODO(bondhugula): symbolic extents when necessary.
-      // TODO(bondhugula): handle union if a dimension is unbounded.
+      // TODO: symbolic extents when necessary.
+      // TODO: handle union if a dimension is unbounded.
       return failure();
 
     auto otherExtent = otherAligned.getConstantBoundOnDimSize(
         d, &otherLb, &otherLbFloorDivisor, &otherUb);
     if (!otherExtent.hasValue() || lbFloorDivisor != otherLbFloorDivisor)
-      // TODO(bondhugula): symbolic extents when necessary.
+      // TODO: symbolic extents when necessary.
       return failure();
 
     assert(lbFloorDivisor > 0 && "divisor always expected to be positive");
@@ -2796,9 +2819,9 @@ FlatAffineConstraints::unionBoundingBox(const FlatAffineConstraints &otherCst) {
   append(commonCst);
   removeTrivialRedundancy();
 
-  // TODO(mlir-team): copy over pure symbolic constraints from this and 'other'
-  // over to the union (since the above are just the union along dimensions); we
-  // shouldn't be discarding any other constraints on the symbols.
+  // TODO: copy over pure symbolic constraints from this and 'other' over to the
+  // union (since the above are just the union along dimensions); we shouldn't
+  // be discarding any other constraints on the symbols.
 
   return success();
 }
diff --git a/mlir/lib/Analysis/CMakeLists.txt b/mlir/lib/Analysis/CMakeLists.txt
index 0d9cc9036c2f5..524203b87068e 100644
--- a/mlir/lib/Analysis/CMakeLists.txt
+++ b/mlir/lib/Analysis/CMakeLists.txt
@@ -25,6 +25,7 @@ add_mlir_library(MLIRAnalysis
   MLIRCallInterfaces
   MLIRControlFlowInterfaces
   MLIRInferTypeOpInterface
+  MLIRPresburger
   MLIRSCF
   )
 
@@ -46,5 +47,8 @@ add_mlir_library(MLIRLoopAnalysis
   MLIRCallInterfaces
   MLIRControlFlowInterfaces
   MLIRInferTypeOpInterface
+  MLIRPresburger
   MLIRSCF
   )
+  
+add_subdirectory(Presburger)
diff --git a/mlir/lib/Analysis/LoopAnalysis.cpp b/mlir/lib/Analysis/LoopAnalysis.cpp
index 0c8ff3167e69f..210b68eae3cb4 100644
--- a/mlir/lib/Analysis/LoopAnalysis.cpp
+++ b/mlir/lib/Analysis/LoopAnalysis.cpp
@@ -213,7 +213,7 @@ DenseSet<Value> mlir::getInvariantAccesses(Value iv, ArrayRef<Value> indices) {
 /// Returns false if the MemRef has a non-identity layoutMap or more than 1
 /// layoutMap. This is conservative.
 ///
-// TODO(ntv): check strides.
+// TODO: check strides.
 template <typename LoadOrStoreOp>
 static bool isContiguousAccess(Value iv, LoadOrStoreOp memoryOp,
                                int *memRefDim) {
@@ -224,8 +224,7 @@ static bool isContiguousAccess(Value iv, LoadOrStoreOp memoryOp,
   auto memRefType = memoryOp.getMemRefType();
 
   auto layoutMap = memRefType.getAffineMaps();
-  // TODO(ntv): remove dependence on Builder once we support non-identity
-  // layout map.
+  // TODO: remove dependence on Builder once we support non-identity layout map.
   Builder b(memoryOp.getContext());
   if (layoutMap.size() >= 2 ||
       (layoutMap.size() == 1 &&
@@ -291,8 +290,7 @@ isVectorizableLoopBodyWithOpCond(AffineForOp loop,
 
   // No vectorization across unknown regions.
   auto regions = matcher::Op([](Operation &op) -> bool {
-    return op.getNumRegions() != 0 &&
-           !(isa<AffineIfOp>(op) || isa<AffineForOp>(op));
+    return op.getNumRegions() != 0 && !isa<AffineIfOp, AffineForOp>(op);
   });
   SmallVector<NestedMatch, 8> regionsMatched;
   regions.match(forOp, &regionsMatched);
@@ -315,7 +313,7 @@ isVectorizableLoopBodyWithOpCond(AffineForOp loop,
     auto store = dyn_cast<AffineStoreOp>(op);
     // Only scalar types are considered vectorizable, all load/store must be
     // vectorizable for a loop to qualify as vectorizable.
-    // TODO(ntv): ponder whether we want to be more general here.
+    // TODO: ponder whether we want to be more general here.
     bool vector = load ? isVectorElement(load) : isVectorElement(store);
     if (vector) {
       return false;
@@ -346,8 +344,8 @@ bool mlir::isVectorizableLoopBody(AffineForOp loop,
 /// Checks whether SSA dominance would be violated if a for op's body
 /// operations are shifted by the specified shifts. This method checks if a
 /// 'def' and all its uses have the same shift factor.
-// TODO(mlir-team): extend this to check for memory-based dependence violation
-// when we have the support.
+// TODO: extend this to check for memory-based dependence violation when we have
+// the support.
 bool mlir::isOpwiseShiftValid(AffineForOp forOp, ArrayRef<uint64_t> shifts) {
   auto *forBody = forOp.getBody();
   assert(shifts.size() == forBody->getOperations().size());
diff --git a/mlir/lib/Analysis/NestedMatcher.cpp b/mlir/lib/Analysis/NestedMatcher.cpp
index 807e5df46d1c2..7e15ea1094c99 100644
--- a/mlir/lib/Analysis/NestedMatcher.cpp
+++ b/mlir/lib/Analysis/NestedMatcher.cpp
@@ -68,8 +68,8 @@ unsigned NestedPattern::getDepth() const {
 ///   3. if all is good, recursively matches the nested patterns;
 ///   4. if all nested match then the single operation matches too and is
 ///      appended to the list of matches;
-///   5. TODO(ntv) Optionally applies actions (lambda), in which case we will
-///      want to traverse in post-order DFS to avoid invalidating iterators.
+///   5. TODO: Optionally applies actions (lambda), in which case we will want
+///      to traverse in post-order DFS to avoid invalidating iterators.
 void NestedPattern::matchOne(Operation *op,
                              SmallVectorImpl<NestedMatch> *matches) {
   if (skip == op) {
@@ -145,7 +145,7 @@ NestedPattern For(FilterFunctionType filter, ArrayRef<NestedPattern> nested) {
 }
 
 bool isLoadOrStore(Operation &op) {
-  return isa<AffineLoadOp>(op) || isa<AffineStoreOp>(op);
+  return isa<AffineLoadOp, AffineStoreOp>(op);
 }
 
 } // end namespace matcher
diff --git a/mlir/lib/Analysis/Presburger/CMakeLists.txt b/mlir/lib/Analysis/Presburger/CMakeLists.txt
new file mode 100644
index 0000000000000..2561013696d9b
--- /dev/null
+++ b/mlir/lib/Analysis/Presburger/CMakeLists.txt
@@ -0,0 +1,4 @@
+add_mlir_library(MLIRPresburger
+  Simplex.cpp
+  Matrix.cpp
+  )
diff --git a/mlir/lib/Analysis/Presburger/Matrix.cpp b/mlir/lib/Analysis/Presburger/Matrix.cpp
new file mode 100644
index 0000000000000..213f1111e2a3a
--- /dev/null
+++ b/mlir/lib/Analysis/Presburger/Matrix.cpp
@@ -0,0 +1,92 @@
+//===- Matrix.cpp - MLIR Matrix Class -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Analysis/Presburger/Matrix.h"
+
+namespace mlir {
+
+Matrix::Matrix(unsigned rows, unsigned columns)
+    : nRows(rows), nColumns(columns), data(nRows * nColumns) {}
+
+Matrix Matrix::identity(unsigned dimension) {
+  Matrix matrix(dimension, dimension);
+  for (unsigned i = 0; i < dimension; ++i)
+    matrix(i, i) = 1;
+  return matrix;
+}
+
+int64_t &Matrix::at(unsigned row, unsigned column) {
+  assert(row < getNumRows() && "Row outside of range");
+  assert(column < getNumColumns() && "Column outside of range");
+  return data[row * nColumns + column];
+}
+
+int64_t Matrix::at(unsigned row, unsigned column) const {
+  assert(row < getNumRows() && "Row outside of range");
+  assert(column < getNumColumns() && "Column outside of range");
+  return data[row * nColumns + column];
+}
+
+int64_t &Matrix::operator()(unsigned row, unsigned column) {
+  return at(row, column);
+}
+
+int64_t Matrix::operator()(unsigned row, unsigned column) const {
+  return at(row, column);
+}
+
+unsigned Matrix::getNumRows() const { return nRows; }
+
+unsigned Matrix::getNumColumns() const { return nColumns; }
+
+void Matrix::resizeVertically(unsigned newNRows) {
+  nRows = newNRows;
+  data.resize(nRows * nColumns);
+}
+
+void Matrix::swapRows(unsigned row, unsigned otherRow) {
+  assert((row < getNumRows() && otherRow < getNumRows()) &&
+         "Given row out of bounds");
+  if (row == otherRow)
+    return;
+  for (unsigned col = 0; col < nColumns; col++)
+    std::swap(at(row, col), at(otherRow, col));
+}
+
+void Matrix::swapColumns(unsigned column, unsigned otherColumn) {
+  assert((column < getNumColumns() && otherColumn < getNumColumns()) &&
+         "Given column out of bounds");
+  if (column == otherColumn)
+    return;
+  for (unsigned row = 0; row < nRows; row++)
+    std::swap(at(row, column), at(row, otherColumn));
+}
+
+ArrayRef<int64_t> Matrix::getRow(unsigned row) const {
+  return {&data[row * nColumns], nColumns};
+}
+
+void Matrix::addToRow(unsigned sourceRow, unsigned targetRow, int64_t scale) {
+  if (scale == 0)
+    return;
+  for (unsigned col = 0; col < nColumns; ++col)
+    at(targetRow, col) += scale * at(sourceRow, col);
+  return;
+}
+
+void Matrix::print(raw_ostream &os) const {
+  for (unsigned row = 0; row < nRows; ++row) {
+    for (unsigned column = 0; column < nColumns; ++column)
+      os << at(row, column) << ' ';
+    os << '\n';
+  }
+}
+
+void Matrix::dump() const { print(llvm::errs()); }
+
+} // namespace mlir
diff --git a/mlir/lib/Analysis/Presburger/Simplex.cpp b/mlir/lib/Analysis/Presburger/Simplex.cpp
new file mode 100644
index 0000000000000..825346478819b
--- /dev/null
+++ b/mlir/lib/Analysis/Presburger/Simplex.cpp
@@ -0,0 +1,1081 @@
+//===- Simplex.cpp - MLIR Simplex Class -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Analysis/Presburger/Simplex.h"
+#include "mlir/Analysis/Presburger/Matrix.h"
+#include "mlir/Support/MathExtras.h"
+
+namespace mlir {
+using Direction = Simplex::Direction;
+
+const int nullIndex = std::numeric_limits<int>::max();
+
+/// Construct a Simplex object with `nVar` variables.
+Simplex::Simplex(unsigned nVar)
+    : nRow(0), nCol(2), tableau(0, 2 + nVar), empty(false) {
+  colUnknown.push_back(nullIndex);
+  colUnknown.push_back(nullIndex);
+  for (unsigned i = 0; i < nVar; ++i) {
+    var.emplace_back(Orientation::Column, /*restricted=*/false, /*pos=*/nCol);
+    colUnknown.push_back(i);
+    nCol++;
+  }
+}
+
+Simplex::Simplex(const FlatAffineConstraints &constraints)
+    : Simplex(constraints.getNumIds()) {
+  for (unsigned i = 0, numIneqs = constraints.getNumInequalities();
+       i < numIneqs; ++i)
+    addInequality(constraints.getInequality(i));
+  for (unsigned i = 0, numEqs = constraints.getNumEqualities(); i < numEqs; ++i)
+    addEquality(constraints.getEquality(i));
+}
+
+const Simplex::Unknown &Simplex::unknownFromIndex(int index) const {
+  assert(index != nullIndex && "nullIndex passed to unknownFromIndex");
+  return index >= 0 ? var[index] : con[~index];
+}
+
+const Simplex::Unknown &Simplex::unknownFromColumn(unsigned col) const {
+  assert(col < nCol && "Invalid column");
+  return unknownFromIndex(colUnknown[col]);
+}
+
+const Simplex::Unknown &Simplex::unknownFromRow(unsigned row) const {
+  assert(row < nRow && "Invalid row");
+  return unknownFromIndex(rowUnknown[row]);
+}
+
+Simplex::Unknown &Simplex::unknownFromIndex(int index) {
+  assert(index != nullIndex && "nullIndex passed to unknownFromIndex");
+  return index >= 0 ? var[index] : con[~index];
+}
+
+Simplex::Unknown &Simplex::unknownFromColumn(unsigned col) {
+  assert(col < nCol && "Invalid column");
+  return unknownFromIndex(colUnknown[col]);
+}
+
+Simplex::Unknown &Simplex::unknownFromRow(unsigned row) {
+  assert(row < nRow && "Invalid row");
+  return unknownFromIndex(rowUnknown[row]);
+}
+
+/// Add a new row to the tableau corresponding to the given constant term and
+/// list of coefficients. The coefficients are specified as a vector of
+/// (variable index, coefficient) pairs.
+unsigned Simplex::addRow(ArrayRef<int64_t> coeffs) {
+  assert(coeffs.size() == 1 + var.size() &&
+         "Incorrect number of coefficients!");
+
+  ++nRow;
+  // If the tableau is not big enough to accomodate the extra row, we extend it.
+  if (nRow >= tableau.getNumRows())
+    tableau.resizeVertically(nRow);
+  rowUnknown.push_back(~con.size());
+  con.emplace_back(Orientation::Row, false, nRow - 1);
+
+  tableau(nRow - 1, 0) = 1;
+  tableau(nRow - 1, 1) = coeffs.back();
+  for (unsigned col = 2; col < nCol; ++col)
+    tableau(nRow - 1, col) = 0;
+
+  // Process each given variable coefficient.
+  for (unsigned i = 0; i < var.size(); ++i) {
+    unsigned pos = var[i].pos;
+    if (coeffs[i] == 0)
+      continue;
+
+    if (var[i].orientation == Orientation::Column) {
+      // If a variable is in column position at column col, then we just add the
+      // coefficient for that variable (scaled by the common row denominator) to
+      // the corresponding entry in the new row.
+      tableau(nRow - 1, pos) += coeffs[i] * tableau(nRow - 1, 0);
+      continue;
+    }
+
+    // If the variable is in row position, we need to add that row to the new
+    // row, scaled by the coefficient for the variable, accounting for the two
+    // rows potentially having different denominators. The new denominator is
+    // the lcm of the two.
+    int64_t lcm = mlir::lcm(tableau(nRow - 1, 0), tableau(pos, 0));
+    int64_t nRowCoeff = lcm / tableau(nRow - 1, 0);
+    int64_t idxRowCoeff = coeffs[i] * (lcm / tableau(pos, 0));
+    tableau(nRow - 1, 0) = lcm;
+    for (unsigned col = 1; col < nCol; ++col)
+      tableau(nRow - 1, col) =
+          nRowCoeff * tableau(nRow - 1, col) + idxRowCoeff * tableau(pos, col);
+  }
+
+  normalizeRow(nRow - 1);
+  // Push to undo log along with the index of the new constraint.
+  undoLog.push_back(UndoLogEntry::RemoveLastConstraint);
+  return con.size() - 1;
+}
+
+/// Normalize the row by removing factors that are common between the
+/// denominator and all the numerator coefficients.
+void Simplex::normalizeRow(unsigned row) {
+  int64_t gcd = 0;
+  for (unsigned col = 0; col < nCol; ++col) {
+    if (gcd == 1)
+      break;
+    gcd = llvm::greatestCommonDivisor(gcd, std::abs(tableau(row, col)));
+  }
+  for (unsigned col = 0; col < nCol; ++col)
+    tableau(row, col) /= gcd;
+}
+
+namespace {
+bool signMatchesDirection(int64_t elem, Direction direction) {
+  assert(elem != 0 && "elem should not be 0");
+  return direction == Direction::Up ? elem > 0 : elem < 0;
+}
+
+Direction flippedDirection(Direction direction) {
+  return direction == Direction::Up ? Direction::Down : Simplex::Direction::Up;
+}
+} // anonymous namespace
+
+/// Find a pivot to change the sample value of the row in the specified
+/// direction. The returned pivot row will involve `row` if and only if the
+/// unknown is unbounded in the specified direction.
+///
+/// To increase (resp. decrease) the value of a row, we need to find a live
+/// column with a non-zero coefficient. If the coefficient is positive, we need
+/// to increase (decrease) the value of the column, and if the coefficient is
+/// negative, we need to decrease (increase) the value of the column. Also,
+/// we cannot decrease the sample value of restricted columns.
+///
+/// If multiple columns are valid, we break ties by considering a lexicographic
+/// ordering where we prefer unknowns with lower index.
+Optional<Simplex::Pivot> Simplex::findPivot(int row,
+                                            Direction direction) const {
+  Optional<unsigned> col;
+  for (unsigned j = 2; j < nCol; ++j) {
+    int64_t elem = tableau(row, j);
+    if (elem == 0)
+      continue;
+
+    if (unknownFromColumn(j).restricted &&
+        !signMatchesDirection(elem, direction))
+      continue;
+    if (!col || colUnknown[j] < colUnknown[*col])
+      col = j;
+  }
+
+  if (!col)
+    return {};
+
+  Direction newDirection =
+      tableau(row, *col) < 0 ? flippedDirection(direction) : direction;
+  Optional<unsigned> maybePivotRow = findPivotRow(row, newDirection, *col);
+  return Pivot{maybePivotRow.getValueOr(row), *col};
+}
+
+/// Swap the associated unknowns for the row and the column.
+///
+/// First we swap the index associated with the row and column. Then we update
+/// the unknowns to reflect their new position and orientation.
+void Simplex::swapRowWithCol(unsigned row, unsigned col) {
+  std::swap(rowUnknown[row], colUnknown[col]);
+  Unknown &uCol = unknownFromColumn(col);
+  Unknown &uRow = unknownFromRow(row);
+  uCol.orientation = Orientation::Column;
+  uRow.orientation = Orientation::Row;
+  uCol.pos = col;
+  uRow.pos = row;
+}
+
+void Simplex::pivot(Pivot pair) { pivot(pair.row, pair.column); }
+
+/// Pivot pivotRow and pivotCol.
+///
+/// Let R be the pivot row unknown and let C be the pivot col unknown.
+/// Since initially R = a*C + sum b_i * X_i
+/// (where the sum is over the other column's unknowns, x_i)
+/// C = (R - (sum b_i * X_i))/a
+///
+/// Let u be some other row unknown.
+/// u = c*C + sum d_i * X_i
+/// So u = c*(R - sum b_i * X_i)/a + sum d_i * X_i
+///
+/// This results in the following transform:
+///            pivot col    other col                   pivot col    other col
+/// pivot row     a             b       ->   pivot row     1/a         -b/a
+/// other row     c             d            other row     c/a        d - bc/a
+///
+/// Taking into account the common denominators p and q:
+///
+///            pivot col    other col                    pivot col   other col
+/// pivot row     a/p          b/p     ->   pivot row      p/a         -b/a
+/// other row     c/q          d/q          other row     cp/aq    (da - bc)/aq
+///
+/// The pivot row transform is accomplished be swapping a with the pivot row's
+/// common denominator and negating the pivot row except for the pivot column
+/// element.
+void Simplex::pivot(unsigned pivotRow, unsigned pivotCol) {
+  assert(pivotCol >= 2 && "Refusing to pivot invalid column");
+
+  swapRowWithCol(pivotRow, pivotCol);
+  std::swap(tableau(pivotRow, 0), tableau(pivotRow, pivotCol));
+  // We need to negate the whole pivot row except for the pivot column.
+  if (tableau(pivotRow, 0) < 0) {
+    // If the denominator is negative, we negate the row by simply negating the
+    // denominator.
+    tableau(pivotRow, 0) = -tableau(pivotRow, 0);
+    tableau(pivotRow, pivotCol) = -tableau(pivotRow, pivotCol);
+  } else {
+    for (unsigned col = 1; col < nCol; ++col) {
+      if (col == pivotCol)
+        continue;
+      tableau(pivotRow, col) = -tableau(pivotRow, col);
+    }
+  }
+  normalizeRow(pivotRow);
+
+  for (unsigned row = 0; row < nRow; ++row) {
+    if (row == pivotRow)
+      continue;
+    if (tableau(row, pivotCol) == 0) // Nothing to do.
+      continue;
+    tableau(row, 0) *= tableau(pivotRow, 0);
+    for (unsigned j = 1; j < nCol; ++j) {
+      if (j == pivotCol)
+        continue;
+      // Add rather than subtract because the pivot row has been negated.
+      tableau(row, j) = tableau(row, j) * tableau(pivotRow, 0) +
+                        tableau(row, pivotCol) * tableau(pivotRow, j);
+    }
+    tableau(row, pivotCol) *= tableau(pivotRow, pivotCol);
+    normalizeRow(row);
+  }
+}
+
+/// Perform pivots until the unknown has a non-negative sample value or until
+/// no more upward pivots can be performed. Return the sign of the final sample
+/// value.
+LogicalResult Simplex::restoreRow(Unknown &u) {
+  assert(u.orientation == Orientation::Row &&
+         "unknown should be in row position");
+
+  while (tableau(u.pos, 1) < 0) {
+    Optional<Pivot> maybePivot = findPivot(u.pos, Direction::Up);
+    if (!maybePivot)
+      break;
+
+    pivot(*maybePivot);
+    if (u.orientation == Orientation::Column)
+      return LogicalResult::Success; // the unknown is unbounded above.
+  }
+  return success(tableau(u.pos, 1) >= 0);
+}
+
+/// Find a row that can be used to pivot the column in the specified direction.
+/// This returns an empty optional if and only if the column is unbounded in the
+/// specified direction (ignoring skipRow, if skipRow is set).
+///
+/// If skipRow is set, this row is not considered, and (if it is restricted) its
+/// restriction may be violated by the returned pivot. Usually, skipRow is set
+/// because we don't want to move it to column position unless it is unbounded,
+/// and we are either trying to increase the value of skipRow or explicitly
+/// trying to make skipRow negative, so we are not concerned about this.
+///
+/// If the direction is up (resp. down) and a restricted row has a negative
+/// (positive) coefficient for the column, then this row imposes a bound on how
+/// much the sample value of the column can change. Such a row with constant
+/// term c and coefficient f for the column imposes a bound of c/|f| on the
+/// change in sample value (in the specified direction). (note that c is
+/// non-negative here since the row is restricted and the tableau is consistent)
+///
+/// We iterate through the rows and pick the row which imposes the most
+/// stringent bound, since pivoting with a row changes the row's sample value to
+/// 0 and hence saturates the bound it imposes. We break ties between rows that
+/// impose the same bound by considering a lexicographic ordering where we
+/// prefer unknowns with lower index value.
+Optional<unsigned> Simplex::findPivotRow(Optional<unsigned> skipRow,
+                                         Direction direction,
+                                         unsigned col) const {
+  Optional<unsigned> retRow;
+  int64_t retElem, retConst;
+  for (unsigned row = 0; row < nRow; ++row) {
+    if (skipRow && row == *skipRow)
+      continue;
+    int64_t elem = tableau(row, col);
+    if (elem == 0)
+      continue;
+    if (!unknownFromRow(row).restricted)
+      continue;
+    if (signMatchesDirection(elem, direction))
+      continue;
+    int64_t constTerm = tableau(row, 1);
+
+    if (!retRow) {
+      retRow = row;
+      retElem = elem;
+      retConst = constTerm;
+      continue;
+    }
+
+    int64_t diff = retConst * elem - constTerm * retElem;
+    if ((diff == 0 && rowUnknown[row] < rowUnknown[*retRow]) ||
+        (diff != 0 && !signMatchesDirection(diff, direction))) {
+      retRow = row;
+      retElem = elem;
+      retConst = constTerm;
+    }
+  }
+  return retRow;
+}
+
+bool Simplex::isEmpty() const { return empty; }
+
+void Simplex::swapRows(unsigned i, unsigned j) {
+  if (i == j)
+    return;
+  tableau.swapRows(i, j);
+  std::swap(rowUnknown[i], rowUnknown[j]);
+  unknownFromRow(i).pos = i;
+  unknownFromRow(j).pos = j;
+}
+
+/// Mark this tableau empty and push an entry to the undo stack.
+void Simplex::markEmpty() {
+  undoLog.push_back(UndoLogEntry::UnmarkEmpty);
+  empty = true;
+}
+
+/// Add an inequality to the tableau. If coeffs is c_0, c_1, ... c_n, where n
+/// is the curent number of variables, then the corresponding inequality is
+/// c_n + c_0*x_0 + c_1*x_1 + ... + c_{n-1}*x_{n-1} >= 0.
+///
+/// We add the inequality and mark it as restricted. We then try to make its
+/// sample value non-negative. If this is not possible, the tableau has become
+/// empty and we mark it as such.
+void Simplex::addInequality(ArrayRef<int64_t> coeffs) {
+  unsigned conIndex = addRow(coeffs);
+  Unknown &u = con[conIndex];
+  u.restricted = true;
+  LogicalResult result = restoreRow(u);
+  if (failed(result))
+    markEmpty();
+}
+
+/// Add an equality to the tableau. If coeffs is c_0, c_1, ... c_n, where n
+/// is the curent number of variables, then the corresponding equality is
+/// c_n + c_0*x_0 + c_1*x_1 + ... + c_{n-1}*x_{n-1} == 0.
+///
+/// We simply add two opposing inequalities, which force the expression to
+/// be zero.
+void Simplex::addEquality(ArrayRef<int64_t> coeffs) {
+  addInequality(coeffs);
+  SmallVector<int64_t, 8> negatedCoeffs;
+  for (int64_t coeff : coeffs)
+    negatedCoeffs.emplace_back(-coeff);
+  addInequality(negatedCoeffs);
+}
+
+unsigned Simplex::numVariables() const { return var.size(); }
+unsigned Simplex::numConstraints() const { return con.size(); }
+
+/// Return a snapshot of the curent state. This is just the current size of the
+/// undo log.
+unsigned Simplex::getSnapshot() const { return undoLog.size(); }
+
+void Simplex::undo(UndoLogEntry entry) {
+  if (entry == UndoLogEntry::RemoveLastConstraint) {
+    Unknown &constraint = con.back();
+    if (constraint.orientation == Orientation::Column) {
+      unsigned column = constraint.pos;
+      Optional<unsigned> row;
+
+      // Try to find any pivot row for this column that preserves tableau
+      // consistency (except possibly the column itself, which is going to be
+      // deallocated anyway).
+      //
+      // If no pivot row is found in either direction, then the unknown is
+      // unbounded in both directions and we are free to
+      // perform any pivot at all. To do this, we just need to find any row with
+      // a non-zero coefficient for the column.
+      if (Optional<unsigned> maybeRow =
+              findPivotRow({}, Direction::Up, column)) {
+        row = *maybeRow;
+      } else if (Optional<unsigned> maybeRow =
+                     findPivotRow({}, Direction::Down, column)) {
+        row = *maybeRow;
+      } else {
+        // The loop doesn't find a pivot row only if the column has zero
+        // coefficients for every row. But the unknown is a constraint,
+        // so it was added initially as a row. Such a row could never have been
+        // pivoted to a column. So a pivot row will always be found.
+        for (unsigned i = 0; i < nRow; ++i) {
+          if (tableau(i, column) != 0) {
+            row = i;
+            break;
+          }
+        }
+      }
+      assert(row.hasValue() && "No pivot row found!");
+      pivot(*row, column);
+    }
+
+    // Move this unknown to the last row and remove the last row from the
+    // tableau.
+    swapRows(constraint.pos, nRow - 1);
+    // It is not strictly necessary to shrink the tableau, but for now we
+    // maintain the invariant that the tableau has exactly nRow rows.
+    tableau.resizeVertically(nRow - 1);
+    nRow--;
+    rowUnknown.pop_back();
+    con.pop_back();
+  } else if (entry == UndoLogEntry::UnmarkEmpty) {
+    empty = false;
+  }
+}
+
+/// Rollback to the specified snapshot.
+///
+/// We undo all the log entries until the log size when the snapshot was taken
+/// is reached.
+void Simplex::rollback(unsigned snapshot) {
+  while (undoLog.size() > snapshot) {
+    undo(undoLog.back());
+    undoLog.pop_back();
+  }
+}
+
+Optional<Fraction> Simplex::computeRowOptimum(Direction direction,
+                                              unsigned row) {
+  // Keep trying to find a pivot for the row in the specified direction.
+  while (Optional<Pivot> maybePivot = findPivot(row, direction)) {
+    // If findPivot returns a pivot involving the row itself, then the optimum
+    // is unbounded, so we return None.
+    if (maybePivot->row == row)
+      return {};
+    pivot(*maybePivot);
+  }
+
+  // The row has reached its optimal sample value, which we return.
+  // The sample value is the entry in the constant column divided by the common
+  // denominator for this row.
+  return Fraction(tableau(row, 1), tableau(row, 0));
+}
+
+/// Compute the optimum of the specified expression in the specified direction,
+/// or None if it is unbounded.
+Optional<Fraction> Simplex::computeOptimum(Direction direction,
+                                           ArrayRef<int64_t> coeffs) {
+  assert(!empty && "Tableau should not be empty");
+
+  unsigned snapshot = getSnapshot();
+  unsigned conIndex = addRow(coeffs);
+  unsigned row = con[conIndex].pos;
+  Optional<Fraction> optimum = computeRowOptimum(direction, row);
+  rollback(snapshot);
+  return optimum;
+}
+
+bool Simplex::isUnbounded() {
+  if (empty)
+    return false;
+
+  SmallVector<int64_t, 8> dir(var.size() + 1);
+  for (unsigned i = 0; i < var.size(); ++i) {
+    dir[i] = 1;
+
+    Optional<Fraction> maybeMax = computeOptimum(Direction::Up, dir);
+    if (!maybeMax)
+      return true;
+
+    Optional<Fraction> maybeMin = computeOptimum(Direction::Down, dir);
+    if (!maybeMin)
+      return true;
+
+    dir[i] = 0;
+  }
+  return false;
+}
+
+/// Make a tableau to represent a pair of points in the original tableau.
+///
+/// The product constraints and variables are stored as: first A's, then B's.
+///
+/// The product tableau has row layout:
+///   A's rows, B's rows.
+///
+/// It has column layout:
+///   denominator, constant, A's columns, B's columns.
+Simplex Simplex::makeProduct(const Simplex &a, const Simplex &b) {
+  unsigned numVar = a.numVariables() + b.numVariables();
+  unsigned numCon = a.numConstraints() + b.numConstraints();
+  Simplex result(numVar);
+
+  result.tableau.resizeVertically(numCon);
+  result.empty = a.empty || b.empty;
+
+  auto concat = [](ArrayRef<Unknown> v, ArrayRef<Unknown> w) {
+    SmallVector<Unknown, 8> result;
+    result.reserve(v.size() + w.size());
+    result.insert(result.end(), v.begin(), v.end());
+    result.insert(result.end(), w.begin(), w.end());
+    return result;
+  };
+  result.con = concat(a.con, b.con);
+  result.var = concat(a.var, b.var);
+
+  auto indexFromBIndex = [&](int index) {
+    return index >= 0 ? a.numVariables() + index
+                      : ~(a.numConstraints() + ~index);
+  };
+
+  result.colUnknown.assign(2, nullIndex);
+  for (unsigned i = 2; i < a.nCol; ++i) {
+    result.colUnknown.push_back(a.colUnknown[i]);
+    result.unknownFromIndex(result.colUnknown.back()).pos =
+        result.colUnknown.size() - 1;
+  }
+  for (unsigned i = 2; i < b.nCol; ++i) {
+    result.colUnknown.push_back(indexFromBIndex(b.colUnknown[i]));
+    result.unknownFromIndex(result.colUnknown.back()).pos =
+        result.colUnknown.size() - 1;
+  }
+
+  auto appendRowFromA = [&](unsigned row) {
+    for (unsigned col = 0; col < a.nCol; ++col)
+      result.tableau(result.nRow, col) = a.tableau(row, col);
+    result.rowUnknown.push_back(a.rowUnknown[row]);
+    result.unknownFromIndex(result.rowUnknown.back()).pos =
+        result.rowUnknown.size() - 1;
+    result.nRow++;
+  };
+
+  // Also fixes the corresponding entry in rowUnknown and var/con (as the case
+  // may be).
+  auto appendRowFromB = [&](unsigned row) {
+    result.tableau(result.nRow, 0) = b.tableau(row, 0);
+    result.tableau(result.nRow, 1) = b.tableau(row, 1);
+
+    unsigned offset = a.nCol - 2;
+    for (unsigned col = 2; col < b.nCol; ++col)
+      result.tableau(result.nRow, offset + col) = b.tableau(row, col);
+    result.rowUnknown.push_back(indexFromBIndex(b.rowUnknown[row]));
+    result.unknownFromIndex(result.rowUnknown.back()).pos =
+        result.rowUnknown.size() - 1;
+    result.nRow++;
+  };
+
+  for (unsigned row = 0; row < a.nRow; ++row)
+    appendRowFromA(row);
+  for (unsigned row = 0; row < b.nRow; ++row)
+    appendRowFromB(row);
+
+  return result;
+}
+
+Optional<SmallVector<int64_t, 8>> Simplex::getSamplePointIfIntegral() const {
+  // The tableau is empty, so no sample point exists.
+  if (empty)
+    return {};
+
+  SmallVector<int64_t, 8> sample;
+  // Push the sample value for each variable into the vector.
+  for (const Unknown &u : var) {
+    if (u.orientation == Orientation::Column) {
+      // If the variable is in column position, its sample value is zero.
+      sample.push_back(0);
+    } else {
+      // If the variable is in row position, its sample value is the entry in
+      // the constant column divided by the entry in the common denominator
+      // column. If this is not an integer, then the sample point is not
+      // integral so we return None.
+      if (tableau(u.pos, 1) % tableau(u.pos, 0) != 0)
+        return {};
+      sample.push_back(tableau(u.pos, 1) / tableau(u.pos, 0));
+    }
+  }
+  return sample;
+}
+
+/// Given a simplex for a polytope, construct a new simplex whose variables are
+/// identified with a pair of points (x, y) in the original polytope. Supports
+/// some operations needed for generalized basis reduction. In what follows,
+/// dotProduct(x, y) = x_1 * y_1 + x_2 * y_2 + ... x_n * y_n where n is the
+/// dimension of the original polytope.
+///
+/// This supports adding equality constraints dotProduct(dir, x - y) == 0. It
+/// also supports rolling back this addition, by maintaining a snapshot stack
+/// that contains a snapshot of the Simplex's state for each equality, just
+/// before that equality was added.
+class GBRSimplex {
+  using Orientation = Simplex::Orientation;
+
+public:
+  GBRSimplex(const Simplex &originalSimplex)
+      : simplex(Simplex::makeProduct(originalSimplex, originalSimplex)),
+        simplexConstraintOffset(simplex.numConstraints()) {}
+
+  /// Add an equality dotProduct(dir, x - y) == 0.
+  /// First pushes a snapshot for the current simplex state to the stack so
+  /// that this can be rolled back later.
+  void addEqualityForDirection(ArrayRef<int64_t> dir) {
+    assert(
+        std::any_of(dir.begin(), dir.end(), [](int64_t x) { return x != 0; }) &&
+        "Direction passed is the zero vector!");
+    snapshotStack.push_back(simplex.getSnapshot());
+    simplex.addEquality(getCoeffsForDirection(dir));
+  }
+
+  /// Compute max(dotProduct(dir, x - y)) and save the dual variables for only
+  /// the direction equalities to `dual`.
+  Fraction computeWidthAndDuals(ArrayRef<int64_t> dir,
+                                SmallVectorImpl<int64_t> &dual,
+                                int64_t &dualDenom) {
+    unsigned snap = simplex.getSnapshot();
+    unsigned conIndex = simplex.addRow(getCoeffsForDirection(dir));
+    unsigned row = simplex.con[conIndex].pos;
+    Optional<Fraction> maybeWidth =
+        simplex.computeRowOptimum(Simplex::Direction::Up, row);
+    assert(maybeWidth.hasValue() && "Width should not be unbounded!");
+    dualDenom = simplex.tableau(row, 0);
+    dual.clear();
+    // The increment is i += 2 because equalities are added as two inequalities,
+    // one positive and one negative. Each iteration processes one equality.
+    for (unsigned i = simplexConstraintOffset; i < conIndex; i += 2) {
+      // The dual variable is the negative of the coefficient of the new row
+      // in the column of the constraint, if the constraint is in a column.
+      // Note that the second inequality for the equality is negated.
+      //
+      // We want the dual for the original equality. If the positive inequality
+      // is in column position, the negative of its row coefficient is the
+      // desired dual. If the negative inequality is in column position, its row
+      // coefficient is the desired dual. (its coefficients are already the
+      // negated coefficients of the original equality, so we don't need to
+      // negate it now.)
+      //
+      // If neither are in column position, we move the negated inequality to
+      // column position. Since the inequality must have sample value zero
+      // (since it corresponds to an equality), we are free to pivot with
+      // any column. Since both the unknowns have sample value before and after
+      // pivoting, no other sample values will change and the tableau will
+      // remain consistent. To pivot, we just need to find a column that has a
+      // non-zero coefficient in this row. There must be one since otherwise the
+      // equality would be 0 == 0, which should never be passed to
+      // addEqualityForDirection.
+      //
+      // After finding a column, we pivot with the column, after which we can
+      // get the dual from the inequality in column position as explained above.
+      if (simplex.con[i].orientation == Orientation::Column) {
+        dual.push_back(-simplex.tableau(row, simplex.con[i].pos));
+      } else {
+        if (simplex.con[i + 1].orientation == Orientation::Row) {
+          unsigned ineqRow = simplex.con[i + 1].pos;
+          // Since it is an equality, the the sample value must be zero.
+          assert(simplex.tableau(ineqRow, 1) == 0 &&
+                 "Equality's sample value must be zero.");
+          for (unsigned col = 2; col < simplex.nCol; ++col) {
+            if (simplex.tableau(ineqRow, col) != 0) {
+              simplex.pivot(ineqRow, col);
+              break;
+            }
+          }
+          assert(simplex.con[i + 1].orientation == Orientation::Column &&
+                 "No pivot found. Equality has all-zeros row in tableau!");
+        }
+        dual.push_back(simplex.tableau(row, simplex.con[i + 1].pos));
+      }
+    }
+    simplex.rollback(snap);
+    return *maybeWidth;
+  }
+
+  /// Remove the last equality that was added through addEqualityForDirection.
+  ///
+  /// We do this by rolling back to the snapshot at the top of the stack, which
+  /// should be a snapshot taken just before the last equality was added.
+  void removeLastEquality() {
+    assert(!snapshotStack.empty() && "Snapshot stack is empty!");
+    simplex.rollback(snapshotStack.back());
+    snapshotStack.pop_back();
+  }
+
+private:
+  /// Returns coefficients of the expression 'dot_product(dir, x - y)',
+  /// i.e.,   dir_1 * x_1 + dir_2 * x_2 + ... + dir_n * x_n
+  ///       - dir_1 * y_1 - dir_2 * y_2 - ... - dir_n * y_n,
+  /// where n is the dimension of the original polytope.
+  SmallVector<int64_t, 8> getCoeffsForDirection(ArrayRef<int64_t> dir) {
+    assert(2 * dir.size() == simplex.numVariables() &&
+           "Direction vector has wrong dimensionality");
+    SmallVector<int64_t, 8> coeffs(dir.begin(), dir.end());
+    coeffs.reserve(2 * dir.size());
+    for (int64_t coeff : dir)
+      coeffs.push_back(-coeff);
+    coeffs.push_back(0); // constant term
+    return coeffs;
+  }
+
+  Simplex simplex;
+  /// The first index of the equality constraints, the index immediately after
+  /// the last constraint in the initial product simplex.
+  unsigned simplexConstraintOffset;
+  /// A stack of snapshots, used for rolling back.
+  SmallVector<unsigned, 8> snapshotStack;
+};
+
+/// Reduce the basis to try and find a direction in which the polytope is
+/// "thin". This only works for bounded polytopes.
+///
+/// This is an implementation of the algorithm described in the paper
+/// "An Implementation of Generalized Basis Reduction for Integer Programming"
+/// by W. Cook, T. Rutherford, H. E. Scarf, D. Shallcross.
+///
+/// Let b_{level}, b_{level + 1}, ... b_n be the current basis.
+/// Let width_i(v) = max <v, x - y> where x and y are points in the original
+/// polytope such that <b_j, x - y> = 0 is satisfied for all level <= j < i.
+///
+/// In every iteration, we first replace b_{i+1} with b_{i+1} + u*b_i, where u
+/// is the integer such that width_i(b_{i+1} + u*b_i) is minimized. Let dual_i
+/// be the dual variable associated with the constraint <b_i, x - y> = 0 when
+/// computing width_{i+1}(b_{i+1}). It can be shown that dual_i is the
+/// minimizing value of u, if it were allowed to be fractional. Due to
+/// convexity, the minimizing integer value is either floor(dual_i) or
+/// ceil(dual_i), so we just need to check which of these gives a lower
+/// width_{i+1} value. If dual_i turned out to be an integer, then u = dual_i.
+///
+/// Now if width_i(b_{i+1}) < 0.75 * width_i(b_i), we swap b_i and (the new)
+/// b_{i + 1} and decrement i (unless i = level, in which case we stay at the
+/// same i). Otherwise, we increment i.
+///
+/// We keep f values and duals cached and invalidate them when necessary.
+/// Whenever possible, we use them instead of recomputing them. We implement the
+/// algorithm as follows.
+///
+/// In an iteration at i we need to compute:
+///   a) width_i(b_{i + 1})
+///   b) width_i(b_i)
+///   c) the integer u that minimizes width_i(b_{i + 1} + u*b_i)
+///
+/// If width_i(b_i) is not already cached, we compute it.
+///
+/// If the duals are not already cached, we compute width_{i+1}(b_{i+1}) and
+/// store the duals from this computation.
+///
+/// We call updateBasisWithUAndGetFCandidate, which finds the minimizing value
+/// of u as explained before, caches the duals from this computation, sets
+/// b_{i+1} to b_{i+1} + u*b_i, and returns the new value of width_i(b_{i+1}).
+///
+/// Now if width_i(b_{i+1}) < 0.75 * width_i(b_i), we swap b_i and b_{i+1} and
+/// decrement i, resulting in the basis
+/// ... b_{i - 1}, b_{i + 1} + u*b_i, b_i, b_{i+2}, ...
+/// with corresponding f values
+/// ... width_{i-1}(b_{i-1}), width_i(b_{i+1} + u*b_i), width_{i+1}(b_i), ...
+/// The values up to i - 1 remain unchanged. We have just gotten the middle
+/// value from updateBasisWithUAndGetFCandidate, so we can update that in the
+/// cache. The value at width_{i+1}(b_i) is unknown, so we evict this value from
+/// the cache. The iteration after decrementing needs exactly the duals from the
+/// computation of width_i(b_{i + 1} + u*b_i), so we keep these in the cache.
+///
+/// When incrementing i, no cached f values get invalidated. However, the cached
+/// duals do get invalidated as the duals for the higher levels are different.
+void Simplex::reduceBasis(Matrix &basis, unsigned level) {
+  const Fraction epsilon(3, 4);
+
+  if (level == basis.getNumRows() - 1)
+    return;
+
+  GBRSimplex gbrSimplex(*this);
+  SmallVector<Fraction, 8> width;
+  SmallVector<int64_t, 8> dual;
+  int64_t dualDenom;
+
+  // Finds the value of u that minimizes width_i(b_{i+1} + u*b_i), caches the
+  // duals from this computation, sets b_{i+1} to b_{i+1} + u*b_i, and returns
+  // the new value of width_i(b_{i+1}).
+  //
+  // If dual_i is not an integer, the minimizing value must be either
+  // floor(dual_i) or ceil(dual_i). We compute the expression for both and
+  // choose the minimizing value.
+  //
+  // If dual_i is an integer, we don't need to perform these computations. We
+  // know that in this case,
+  //   a) u = dual_i.
+  //   b) one can show that dual_j for j < i are the same duals we would have
+  //      gotten from computing width_i(b_{i + 1} + u*b_i), so the correct duals
+  //      are the ones already in the cache.
+  //   c) width_i(b_{i+1} + u*b_i) = min_{alpha} width_i(b_{i+1} + alpha * b_i),
+  //   which
+  //      one can show is equal to width_{i+1}(b_{i+1}). The latter value must
+  //      be in the cache, so we get it from there and return it.
+  auto updateBasisWithUAndGetFCandidate = [&](unsigned i) -> Fraction {
+    assert(i < level + dual.size() && "dual_i is not known!");
+
+    int64_t u = floorDiv(dual[i - level], dualDenom);
+    basis.addToRow(i, i + 1, u);
+    if (dual[i - level] % dualDenom != 0) {
+      SmallVector<int64_t, 8> candidateDual[2];
+      int64_t candidateDualDenom[2];
+      Fraction widthI[2];
+
+      // Initially u is floor(dual) and basis reflects this.
+      widthI[0] = gbrSimplex.computeWidthAndDuals(
+          basis.getRow(i + 1), candidateDual[0], candidateDualDenom[0]);
+
+      // Now try ceil(dual), i.e. floor(dual) + 1.
+      ++u;
+      basis.addToRow(i, i + 1, 1);
+      widthI[1] = gbrSimplex.computeWidthAndDuals(
+          basis.getRow(i + 1), candidateDual[1], candidateDualDenom[1]);
+
+      unsigned j = widthI[0] < widthI[1] ? 0 : 1;
+      if (j == 0)
+        // Subtract 1 to go from u = ceil(dual) back to floor(dual).
+        basis.addToRow(i, i + 1, -1);
+      dual = std::move(candidateDual[j]);
+      dualDenom = candidateDualDenom[j];
+      return widthI[j];
+    }
+    assert(i + 1 - level < width.size() && "width_{i+1} wasn't saved");
+    // When dual minimizes f_i(b_{i+1} + dual*b_i), this is equal to
+    // width_{i+1}(b_{i+1}).
+    return width[i + 1 - level];
+  };
+
+  // In the ith iteration of the loop, gbrSimplex has constraints for directions
+  // from `level` to i - 1.
+  unsigned i = level;
+  while (i < basis.getNumRows() - 1) {
+    if (i >= level + width.size()) {
+      // We don't even know the value of f_i(b_i), so let's find that first.
+      // We have to do this first since later we assume that width already
+      // contains values up to and including i.
+
+      assert((i == 0 || i - 1 < level + width.size()) &&
+             "We are at level i but we don't know the value of width_{i-1}");
+
+      // We don't actually use these duals at all, but it doesn't matter
+      // because this case should only occur when i is level, and there are no
+      // duals in that case anyway.
+      assert(i == level && "This case should only occur when i == level");
+      width.push_back(
+          gbrSimplex.computeWidthAndDuals(basis.getRow(i), dual, dualDenom));
+    }
+
+    if (i >= level + dual.size()) {
+      assert(i + 1 >= level + width.size() &&
+             "We don't know dual_i but we know width_{i+1}");
+      // We don't know dual for our level, so let's find it.
+      gbrSimplex.addEqualityForDirection(basis.getRow(i));
+      width.push_back(gbrSimplex.computeWidthAndDuals(basis.getRow(i + 1), dual,
+                                                      dualDenom));
+      gbrSimplex.removeLastEquality();
+    }
+
+    // This variable stores width_i(b_{i+1} + u*b_i).
+    Fraction widthICandidate = updateBasisWithUAndGetFCandidate(i);
+    if (widthICandidate < epsilon * width[i - level]) {
+      basis.swapRows(i, i + 1);
+      width[i - level] = widthICandidate;
+      // The values of width_{i+1}(b_{i+1}) and higher may change after the
+      // swap, so we remove the cached values here.
+      width.resize(i - level + 1);
+      if (i == level) {
+        dual.clear();
+        continue;
+      }
+
+      gbrSimplex.removeLastEquality();
+      i--;
+      continue;
+    }
+
+    // Invalidate duals since the higher level needs to recompute its own duals.
+    dual.clear();
+    gbrSimplex.addEqualityForDirection(basis.getRow(i));
+    i++;
+  }
+}
+
+/// Search for an integer sample point using a branch and bound algorithm.
+///
+/// Each row in the basis matrix is a vector, and the set of basis vectors
+/// should span the space. Initially this is the identity matrix,
+/// i.e., the basis vectors are just the variables.
+///
+/// In every level, a value is assigned to the level-th basis vector, as
+/// follows. Compute the minimum and maximum rational values of this direction.
+/// If only one integer point lies in this range, constrain the variable to
+/// have this value and recurse to the next variable.
+///
+/// If the range has multiple values, perform generalized basis reduction via
+/// reduceBasis and then compute the bounds again. Now we try constraining
+/// this direction in the first value in this range and "recurse" to the next
+/// level. If we fail to find a sample, we try assigning the direction the next
+/// value in this range, and so on.
+///
+/// If no integer sample is found from any of the assignments, or if the range
+/// contains no integer value, then of course the polytope is empty for the
+/// current assignment of the values in previous levels, so we return to
+/// the previous level.
+///
+/// If we reach the last level where all the variables have been assigned values
+/// already, then we simply return the current sample point if it is integral,
+/// and go back to the previous level otherwise.
+///
+/// To avoid potentially arbitrarily large recursion depths leading to stack
+/// overflows, this algorithm is implemented iteratively.
+Optional<SmallVector<int64_t, 8>> Simplex::findIntegerSample() {
+  if (empty)
+    return {};
+
+  unsigned nDims = var.size();
+  Matrix basis = Matrix::identity(nDims);
+
+  unsigned level = 0;
+  // The snapshot just before constraining a direction to a value at each level.
+  SmallVector<unsigned, 8> snapshotStack;
+  // The maximum value in the range of the direction for each level.
+  SmallVector<int64_t, 8> upperBoundStack;
+  // The next value to try constraining the basis vector to at each level.
+  SmallVector<int64_t, 8> nextValueStack;
+
+  snapshotStack.reserve(basis.getNumRows());
+  upperBoundStack.reserve(basis.getNumRows());
+  nextValueStack.reserve(basis.getNumRows());
+  while (level != -1u) {
+    if (level == basis.getNumRows()) {
+      // We've assigned values to all variables. Return if we have a sample,
+      // or go back up to the previous level otherwise.
+      if (auto maybeSample = getSamplePointIfIntegral())
+        return maybeSample;
+      level--;
+      continue;
+    }
+
+    if (level >= upperBoundStack.size()) {
+      // We haven't populated the stack values for this level yet, so we have
+      // just come down a level ("recursed"). Find the lower and upper bounds.
+      // If there is more than one integer point in the range, perform
+      // generalized basis reduction.
+      SmallVector<int64_t, 8> basisCoeffs =
+          llvm::to_vector<8>(basis.getRow(level));
+      basisCoeffs.push_back(0);
+
+      int64_t minRoundedUp, maxRoundedDown;
+      std::tie(minRoundedUp, maxRoundedDown) =
+          computeIntegerBounds(basisCoeffs);
+
+      // Heuristic: if the sample point is integral at this point, just return
+      // it.
+      if (auto maybeSample = getSamplePointIfIntegral())
+        return *maybeSample;
+
+      if (minRoundedUp < maxRoundedDown) {
+        reduceBasis(basis, level);
+        basisCoeffs = llvm::to_vector<8>(basis.getRow(level));
+        basisCoeffs.push_back(0);
+        std::tie(minRoundedUp, maxRoundedDown) =
+            computeIntegerBounds(basisCoeffs);
+      }
+
+      snapshotStack.push_back(getSnapshot());
+      // The smallest value in the range is the next value to try.
+      nextValueStack.push_back(minRoundedUp);
+      upperBoundStack.push_back(maxRoundedDown);
+    }
+
+    assert((snapshotStack.size() - 1 == level &&
+            nextValueStack.size() - 1 == level &&
+            upperBoundStack.size() - 1 == level) &&
+           "Mismatched variable stack sizes!");
+
+    // Whether we "recursed" or "returned" from a lower level, we rollback
+    // to the snapshot of the starting state at this level. (in the "recursed"
+    // case this has no effect)
+    rollback(snapshotStack.back());
+    int64_t nextValue = nextValueStack.back();
+    nextValueStack.back()++;
+    if (nextValue > upperBoundStack.back()) {
+      // We have exhausted the range and found no solution. Pop the stack and
+      // return up a level.
+      snapshotStack.pop_back();
+      nextValueStack.pop_back();
+      upperBoundStack.pop_back();
+      level--;
+      continue;
+    }
+
+    // Try the next value in the range and "recurse" into the next level.
+    SmallVector<int64_t, 8> basisCoeffs(basis.getRow(level).begin(),
+                                        basis.getRow(level).end());
+    basisCoeffs.push_back(-nextValue);
+    addEquality(basisCoeffs);
+    level++;
+  }
+
+  return {};
+}
+
+/// Compute the minimum and maximum integer values the expression can take. We
+/// compute each separately.
+std::pair<int64_t, int64_t>
+Simplex::computeIntegerBounds(ArrayRef<int64_t> coeffs) {
+  int64_t minRoundedUp;
+  if (Optional<Fraction> maybeMin =
+          computeOptimum(Simplex::Direction::Down, coeffs))
+    minRoundedUp = ceil(*maybeMin);
+  else
+    llvm_unreachable("Tableau should not be unbounded");
+
+  int64_t maxRoundedDown;
+  if (Optional<Fraction> maybeMax =
+          computeOptimum(Simplex::Direction::Up, coeffs))
+    maxRoundedDown = floor(*maybeMax);
+  else
+    llvm_unreachable("Tableau should not be unbounded");
+
+  return {minRoundedUp, maxRoundedDown};
+}
+
+void Simplex::print(raw_ostream &os) const {
+  os << "rows = " << nRow << ", columns = " << nCol << "\n";
+  if (empty)
+    os << "Simplex marked empty!\n";
+  os << "var: ";
+  for (unsigned i = 0; i < var.size(); ++i) {
+    if (i > 0)
+      os << ", ";
+    var[i].print(os);
+  }
+  os << "\ncon: ";
+  for (unsigned i = 0; i < con.size(); ++i) {
+    if (i > 0)
+      os << ", ";
+    con[i].print(os);
+  }
+  os << '\n';
+  for (unsigned row = 0; row < nRow; ++row) {
+    if (row > 0)
+      os << ", ";
+    os << "r" << row << ": " << rowUnknown[row];
+  }
+  os << '\n';
+  os << "c0: denom, c1: const";
+  for (unsigned col = 2; col < nCol; ++col)
+    os << ", c" << col << ": " << colUnknown[col];
+  os << '\n';
+  for (unsigned row = 0; row < nRow; ++row) {
+    for (unsigned col = 0; col < nCol; ++col)
+      os << tableau(row, col) << '\t';
+    os << '\n';
+  }
+  os << '\n';
+}
+
+void Simplex::dump() const { print(llvm::errs()); }
+
+} // namespace mlir
diff --git a/mlir/lib/Analysis/SliceAnalysis.cpp b/mlir/lib/Analysis/SliceAnalysis.cpp
index e0c828fb55c1e..a09fcf4bea068 100644
--- a/mlir/lib/Analysis/SliceAnalysis.cpp
+++ b/mlir/lib/Analysis/SliceAnalysis.cpp
@@ -85,8 +85,7 @@ static void getBackwardSliceImpl(Operation *op,
   if (!op)
     return;
 
-  assert((op->getNumRegions() == 0 || isa<AffineForOp>(op) ||
-          isa<scf::ForOp>(op)) &&
+  assert((op->getNumRegions() == 0 || isa<AffineForOp, scf::ForOp>(op)) &&
          "unexpected generic op with regions");
 
   // Evaluate whether we should keep this def.
diff --git a/mlir/lib/Analysis/Utils.cpp b/mlir/lib/Analysis/Utils.cpp
index 517ae363ea707..ead45491c159a 100644
--- a/mlir/lib/Analysis/Utils.cpp
+++ b/mlir/lib/Analysis/Utils.cpp
@@ -191,12 +191,12 @@ LogicalResult MemRefRegion::unionBoundingBox(const MemRefRegion &other) {
 // region:  {memref = %A, write = false, {%i <= m0 <= %i + 7} }
 // The last field is a 2-d FlatAffineConstraints symbolic in %i.
 //
-// TODO(bondhugula): extend this to any other memref dereferencing ops
+// TODO: extend this to any other memref dereferencing ops
 // (dma_start, dma_wait).
 LogicalResult MemRefRegion::compute(Operation *op, unsigned loopDepth,
                                     ComputationSliceState *sliceState,
                                     bool addMemRefDimBounds) {
-  assert((isa<AffineReadOpInterface>(op) || isa<AffineWriteOpInterface>(op)) &&
+  assert((isa<AffineReadOpInterface, AffineWriteOpInterface>(op)) &&
          "affine read/write op expected");
 
   MemRefAccess access(op);
@@ -258,7 +258,7 @@ LogicalResult MemRefRegion::compute(Operation *op, unsigned loopDepth,
     if (auto loop = getForInductionVarOwner(operand)) {
       // Note that cst can now have more dimensions than accessMap if the
       // bounds expressions involve outer loops or other symbols.
-      // TODO(bondhugula): rewrite this to use getInstIndexSet; this way
+      // TODO: rewrite this to use getInstIndexSet; this way
       // conditionals will be handled when the latter supports it.
       if (failed(cst.addAffineForOpDomain(loop)))
         return failure();
@@ -330,7 +330,7 @@ LogicalResult MemRefRegion::compute(Operation *op, unsigned loopDepth,
 
   // Add upper/lower bounds for each memref dimension with static size
   // to guard against potential over-approximation from projection.
-  // TODO(andydavis) Support dynamic memref dimensions.
+  // TODO: Support dynamic memref dimensions.
   if (addMemRefDimBounds) {
     auto memRefType = memref.getType().cast<MemRefType>();
     for (unsigned r = 0; r < rank; r++) {
@@ -390,7 +390,7 @@ Optional<int64_t> MemRefRegion::getRegionSize() {
 /// Returns the size of memref data in bytes if it's statically shaped, None
 /// otherwise.  If the element of the memref has vector type, takes into account
 /// size of the vector as well.
-//  TODO(mlir-team): improve/complete this when we have target data.
+//  TODO: improve/complete this when we have target data.
 Optional<uint64_t> mlir::getMemRefSizeInBytes(MemRefType memRefType) {
   if (!memRefType.hasStaticShape())
     return None;
@@ -434,7 +434,7 @@ LogicalResult mlir::boundCheckLoadOrStoreOp(LoadOrStoreOp loadOrStoreOp,
     // feasible. If it is, there is at least one point out of bounds.
     SmallVector<int64_t, 4> ineq(rank + 1, 0);
     int64_t dimSize = loadOrStoreOp.getMemRefType().getDimSize(r);
-    // TODO(bondhugula): handle dynamic dim sizes.
+    // TODO: handle dynamic dim sizes.
     if (dimSize == -1)
       continue;
 
@@ -525,7 +525,7 @@ static LogicalResult addMissingLoopIVBounds(SmallPtrSet<Value, 8> &ivs,
 }
 
 // Returns the innermost common loop depth for the set of operations in 'ops'.
-// TODO(andydavis) Move this to LoopUtils.
+// TODO: Move this to LoopUtils.
 static unsigned
 getInnermostCommonLoopDepth(ArrayRef<Operation *> ops,
                             SmallVectorImpl<AffineForOp> &surroundingLoops) {
@@ -782,7 +782,7 @@ void mlir::getComputationSliceState(
   }
   // Clear all sliced loop bounds beginning at the first sequential loop, or
   // first loop with a slice fusion barrier attribute..
-  // TODO(andydavis, bondhugula) Use MemRef read/write regions instead of
+  // TODO: Use MemRef read/write regions instead of
   // using 'kSliceFusionBarrierAttrName'.
   auto getSliceLoop = [&](unsigned i) {
     return isBackwardSlice ? srcLoopIVs[i] : dstLoopIVs[i];
@@ -804,10 +804,10 @@ void mlir::getComputationSliceState(
 /// updates the slice loop bounds with any non-null bound maps specified in
 /// 'sliceState', and inserts this slice into the loop nest surrounding
 /// 'dstOpInst' at loop depth 'dstLoopDepth'.
-// TODO(andydavis,bondhugula): extend the slicing utility to compute slices that
+// TODO: extend the slicing utility to compute slices that
 // aren't necessarily a one-to-one relation b/w the source and destination. The
 // relation between the source and destination could be many-to-many in general.
-// TODO(andydavis,bondhugula): the slice computation is incorrect in the cases
+// TODO: the slice computation is incorrect in the cases
 // where the dependence from the source to the destination does not cover the
 // entire destination index set. Subtract out the dependent destination
 // iterations from destination index set and check for emptiness --- this is one
@@ -832,7 +832,7 @@ mlir::insertBackwardComputationSlice(Operation *srcOpInst, Operation *dstOpInst,
 
   // Find the op block positions of 'srcOpInst' within 'srcLoopIVs'.
   SmallVector<unsigned, 4> positions;
-  // TODO(andydavis): This code is incorrect since srcLoopIVs can be 0-d.
+  // TODO: This code is incorrect since srcLoopIVs can be 0-d.
   findInstPosition(srcOpInst, srcLoopIVs[0].getOperation()->getBlock(),
                    &positions);
 
@@ -956,8 +956,7 @@ static Optional<int64_t> getMemoryFootprintBytes(Block &block,
 
   // Walk this 'affine.for' operation to gather all memory regions.
   auto result = block.walk(start, end, [&](Operation *opInst) -> WalkResult {
-    if (!isa<AffineReadOpInterface>(opInst) &&
-        !isa<AffineWriteOpInterface>(opInst)) {
+    if (!isa<AffineReadOpInterface, AffineWriteOpInterface>(opInst)) {
       // Neither load nor a store op.
       return WalkResult::advance();
     }
@@ -1017,11 +1016,9 @@ bool mlir::isLoopParallel(AffineForOp forOp) {
   // Collect all load and store ops in loop nest rooted at 'forOp'.
   SmallVector<Operation *, 8> loadAndStoreOpInsts;
   auto walkResult = forOp.walk([&](Operation *opInst) -> WalkResult {
-    if (isa<AffineReadOpInterface>(opInst) ||
-        isa<AffineWriteOpInterface>(opInst))
+    if (isa<AffineReadOpInterface, AffineWriteOpInterface>(opInst))
       loadAndStoreOpInsts.push_back(opInst);
-    else if (!isa<AffineForOp>(opInst) && !isa<AffineYieldOp>(opInst) &&
-             !isa<AffineIfOp>(opInst) &&
+    else if (!isa<AffineForOp, AffineYieldOp, AffineIfOp>(opInst) &&
              !MemoryEffectOpInterface::hasNoEffect(opInst))
       return WalkResult::interrupt();
 
diff --git a/mlir/lib/Bindings/CMakeLists.txt b/mlir/lib/Bindings/CMakeLists.txt
new file mode 100644
index 0000000000000..b41f480b6dcf0
--- /dev/null
+++ b/mlir/lib/Bindings/CMakeLists.txt
@@ -0,0 +1,3 @@
+if(MLIR_BINDINGS_PYTHON_ENABLED)
+  add_subdirectory(Python)
+endif()
diff --git a/mlir/lib/Bindings/Python/CMakeLists.txt b/mlir/lib/Bindings/Python/CMakeLists.txt
new file mode 100644
index 0000000000000..ed08da9688129
--- /dev/null
+++ b/mlir/lib/Bindings/Python/CMakeLists.txt
@@ -0,0 +1,70 @@
+# Normally on unix-like platforms, extensions are built as "MODULE" libraries
+# and do not explicitly link to the python shared object. This allows for
+# some greater deployment flexibility since the extension will bind to
+# symbols in the python interpreter on load. However, it also keeps the
+# linker from erroring on undefined symbols, leaving this to (usually obtuse)
+# runtime errors. Building in "SHARED" mode with an explicit link to the
+# python libraries allows us to build with the expectation of no undefined
+# symbols, which is better for development.
+if(MLIR_PYTHON_BINDINGS_VERSION_LOCKED)
+  set(PYEXT_LINK_MODE SHARED)
+  set(PYEXT_LIBADD ${PYTHON_LIBRARIES})
+else()
+  set(PYEXT_LINK_MODE MODULE)
+  set(PYEXT_LIBADD)
+endif()
+
+# The actual extension library produces a shared-object or DLL and has
+# sources that must be compiled in accordance with pybind11 needs (RTTI and
+# exceptions).
+add_library(MLIRBindingsPythonExtension ${PYEXT_LINK_MODE}
+  MainModule.cpp
+)
+
+target_include_directories(MLIRBindingsPythonExtension PRIVATE
+  "${PYTHON_INCLUDE_DIRS}"
+  "${pybind11_INCLUDE_DIRS}")
+
+# The extension itself must be compiled with RTTI and exceptions enabled.
+# Also, some warning classes triggered by pybind11 are disabled.
+target_compile_options(MLIRBindingsPythonExtension PRIVATE
+  $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
+    # Enable RTTI and exceptions.
+    -frtti -fexceptions
+    # Noisy pybind warnings
+    -Wno-unused-value
+    -Wno-covered-switch-default
+  >
+  $<$<CXX_COMPILER_ID:MSVC>:
+    # Enable RTTI and exceptions.
+    /EHsc /GR>
+)
+
+# Configure the output to match python expectations.
+set_target_properties(
+  MLIRBindingsPythonExtension PROPERTIES
+    LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+    OUTPUT_NAME "_mlir"
+    PREFIX "${PYTHON_MODULE_PREFIX}"
+    SUFFIX "${PYTHON_MODULE_SUFFIX}${PYTHON_MODULE_EXTENSION}"
+)
+
+# pybind11 requires binding code to be compiled with -fvisibility=hidden
+# For static linkage, better code can be generated if the entire project
+# compiles that way, but that is not enforced here. Instead, include a linker
+# script that explicitly hides anything but the PyInit_* symbols, allowing gc
+# to take place.
+# TODO: Add a Windows .def file and figure out the right thing to do on MacOS.
+set_target_properties(
+  MLIRBindingsPythonExtension PROPERTIES CXX_VISIBILITY_PRESET "hidden")
+if(NOT MSVC AND NOT APPLE)
+  set_target_properties(MLIRBindingsPythonExtension
+    PROPERTIES LINK_FLAGS
+    "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/unix_version.lds")
+endif()
+
+target_link_libraries(MLIRBindingsPythonExtension
+  PRIVATE
+  MLIRIR
+  ${PYEXT_LIBADD}
+)
diff --git a/mlir/lib/Bindings/Python/MainModule.cpp b/mlir/lib/Bindings/Python/MainModule.cpp
new file mode 100644
index 0000000000000..b6d7abc4512f4
--- /dev/null
+++ b/mlir/lib/Bindings/Python/MainModule.cpp
@@ -0,0 +1,27 @@
+//===- MainModule.cpp - Main pybind module --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <tuple>
+
+#include <pybind11/pybind11.h>
+
+#include "mlir/IR/MLIRContext.h"
+
+using namespace mlir;
+
+PYBIND11_MODULE(_mlir, m) {
+  m.doc() = "MLIR Python Native Extension";
+
+  m.def("get_test_value", []() {
+    // This is just calling a method on the MLIRContext as a smoketest
+    // for linkage.
+    MLIRContext context;
+    return std::make_tuple(std::string("From the native module"),
+                           context.isMultithreadingEnabled());
+  });
+}
diff --git a/mlir/lib/Bindings/Python/mlir/__init__.py b/mlir/lib/Bindings/Python/mlir/__init__.py
new file mode 100644
index 0000000000000..717526771c25c
--- /dev/null
+++ b/mlir/lib/Bindings/Python/mlir/__init__.py
@@ -0,0 +1,11 @@
+#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+#  See https://llvm.org/LICENSE.txt for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# Note that the only function of this module is currently to load the
+# native module and re-export its symbols. In the future, this file is
+# reserved as a trampoline to handle environment specific loading needs
+# and arbitrate any one-time initialization needed in various shared-library
+# scenarios.
+
+from _mlir import *
diff --git a/mlir/lib/Bindings/Python/unix_version.lds b/mlir/lib/Bindings/Python/unix_version.lds
new file mode 100644
index 0000000000000..b1e59a76229bb
--- /dev/null
+++ b/mlir/lib/Bindings/Python/unix_version.lds
@@ -0,0 +1,4 @@
+{
+  global: PyInit__mlir;
+  local: *;
+};
diff --git a/mlir/lib/CMakeLists.txt b/mlir/lib/CMakeLists.txt
index c1392313faa9e..df3b2db98fcdc 100644
--- a/mlir/lib/CMakeLists.txt
+++ b/mlir/lib/CMakeLists.txt
@@ -2,6 +2,7 @@
 add_flag_if_supported("-Werror=global-constructors" WERROR_GLOBAL_CONSTRUCTOR)
 
 add_subdirectory(Analysis)
+add_subdirectory(Bindings)
 add_subdirectory(Conversion)
 add_subdirectory(Dialect)
 add_subdirectory(EDSC)
@@ -10,6 +11,7 @@ add_subdirectory(IR)
 add_subdirectory(Interfaces)
 add_subdirectory(Parser)
 add_subdirectory(Pass)
+add_subdirectory(Reducer)
 add_subdirectory(Support)
 add_subdirectory(TableGen)
 add_subdirectory(Target)
diff --git a/mlir/lib/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.cpp b/mlir/lib/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.cpp
index b65118b72fdf6..a8c483430fce8 100644
--- a/mlir/lib/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.cpp
+++ b/mlir/lib/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.cpp
@@ -30,7 +30,7 @@ static Type getSrcVectorElementType(OpTy op) {
   return op.src().getType().template cast<VectorType>().getElementType();
 }
 
-// TODO(ntv, zinenko): Code is currently copy-pasted and adapted from the code
+// TODO: Code is currently copy-pasted and adapted from the code
 // 1-1 LLVM conversion. It would better if it were properly exposed in core and
 // reusable.
 /// Basic lowering implementation for one-to-one rewriting from AVX512 Ops to
@@ -76,7 +76,7 @@ matchAndRewriteOneToOne(const ConvertToLLVMPattern &lowering,
 }
 
 namespace {
-// TODO(ntv): Patterns are too verbose due to the fact that we have 1 op (e.g.
+// TODO: Patterns are too verbose due to the fact that we have 1 op (e.g.
 // MaskRndScaleOp) and different possible target ops. It would be better to take
 // a Functor so that all these conversions become 1-liners.
 struct MaskRndScaleOpPS512Conversion : public ConvertToLLVMPattern {
diff --git a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
index 46a7f304db362..e8032348e1821 100644
--- a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
+++ b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
@@ -327,8 +327,8 @@ class AffineMaxLowering : public OpRewritePattern<AffineMaxOp> {
   }
 };
 
-/// Affine terminators are removed.
-class AffineTerminatorLowering : public OpRewritePattern<AffineYieldOp> {
+/// Affine yields ops are removed.
+class AffineYieldOpLowering : public OpRewritePattern<AffineYieldOp> {
 public:
   using OpRewritePattern<AffineYieldOp>::OpRewritePattern;
 
@@ -619,7 +619,7 @@ void mlir::populateAffineToStdConversionPatterns(
       AffineStoreLowering,
       AffineForLowering,
       AffineIfLowering,
-      AffineTerminatorLowering>(ctx);
+      AffineYieldOpLowering>(ctx);
   // clang-format on
 }
 
diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt
index e63b44cff7826..57602881ab7c0 100644
--- a/mlir/lib/Conversion/CMakeLists.txt
+++ b/mlir/lib/Conversion/CMakeLists.txt
@@ -9,6 +9,7 @@ add_subdirectory(LinalgToLLVM)
 add_subdirectory(LinalgToSPIRV)
 add_subdirectory(LinalgToStandard)
 add_subdirectory(SCFToGPU)
+add_subdirectory(SCFToSPIRV)
 add_subdirectory(SCFToStandard)
 add_subdirectory(ShapeToSCF)
 add_subdirectory(ShapeToStandard)
diff --git a/mlir/lib/Conversion/GPUCommon/CMakeLists.txt b/mlir/lib/Conversion/GPUCommon/CMakeLists.txt
index 2b85c237731b2..a4abae8037f65 100644
--- a/mlir/lib/Conversion/GPUCommon/CMakeLists.txt
+++ b/mlir/lib/Conversion/GPUCommon/CMakeLists.txt
@@ -14,7 +14,7 @@ if (MLIR_ROCM_CONVERSIONS_ENABLED)
   )
 endif()
 
-add_mlir_conversion_library(MLIRGPUtoGPURuntimeTransforms
+add_mlir_conversion_library(MLIRGPUToGPURuntimeTransforms
   ConvertLaunchFuncToRuntimeCalls.cpp
   ConvertKernelFuncToBlob.cpp
 
diff --git a/mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt b/mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt
index fe8502f0061c3..a0e8d6d21dd00 100644
--- a/mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt
+++ b/mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt
@@ -2,7 +2,7 @@ set(LLVM_TARGET_DEFINITIONS GPUToNVVM.td)
 mlir_tablegen(GPUToNVVM.cpp.inc -gen-rewriters)
 add_public_tablegen_target(MLIRGPUToNVVMIncGen)
 
-add_mlir_conversion_library(MLIRGPUtoNVVMTransforms
+add_mlir_conversion_library(MLIRGPUToNVVMTransforms
   LowerGpuOpsToNVVMOps.cpp
 
   DEPENDS
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index e4fabe4f441e7..afb6d28758665 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -30,7 +30,6 @@ using namespace mlir;
 
 namespace {
 
-
 struct GPUShuffleOpLowering : public ConvertToLLVMPattern {
   explicit GPUShuffleOpLowering(LLVMTypeConverter &lowering_)
       : ConvertToLLVMPattern(gpu::ShuffleOp::getOperationName(),
@@ -97,17 +96,27 @@ struct GPUShuffleOpLowering : public ConvertToLLVMPattern {
 ///
 /// This pass only handles device code and is not meant to be run on GPU host
 /// code.
-class LowerGpuOpsToNVVMOpsPass
+struct LowerGpuOpsToNVVMOpsPass
     : public ConvertGpuOpsToNVVMOpsBase<LowerGpuOpsToNVVMOpsPass> {
-public:
+  LowerGpuOpsToNVVMOpsPass() = default;
+  LowerGpuOpsToNVVMOpsPass(unsigned indexBitwidth) {
+    this->indexBitwidth = indexBitwidth;
+  }
+
   void runOnOperation() override {
     gpu::GPUModuleOp m = getOperation();
 
+    /// Customize the bitwidth used for the device side index computations.
+    LowerToLLVMOptions options = {/*useBarePtrCallConv =*/false,
+                                  /*emitCWrappers =*/true,
+                                  /*indexBitwidth =*/indexBitwidth,
+                                  /*useAlignedAlloc =*/false};
+
     /// MemRef conversion for GPU to NVVM lowering. The GPU dialect uses memory
     /// space 5 for private memory attributions, but NVVM represents private
     /// memory allocations as local `alloca`s in the default address space. This
     /// converter drops the private memory space to support the use case above.
-    LLVMTypeConverter converter(m.getContext());
+    LLVMTypeConverter converter(m.getContext(), options);
     converter.addConversion([&](MemRefType type) -> Optional<Type> {
       if (type.getMemorySpace() != gpu::GPUDialect::getPrivateAddressSpace())
         return llvm::None;
@@ -131,7 +140,7 @@ class LowerGpuOpsToNVVMOpsPass
                         LLVM::LogOp, LLVM::Log10Op, LLVM::Log2Op>();
     target.addIllegalOp<FuncOp>();
     target.addLegalDialect<NVVM::NVVMDialect>();
-    // TODO(csigg): Remove once we support replacing non-root ops.
+    // TODO: Remove once we support replacing non-root ops.
     target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp, gpu::ModuleEndOp>();
     if (failed(applyPartialConversion(m, target, patterns)))
       signalPassFailure();
@@ -176,6 +185,6 @@ void mlir::populateGpuToNVVMConversionPatterns(
 }
 
 std::unique_ptr<OperationPass<gpu::GPUModuleOp>>
-mlir::createLowerGpuOpsToNVVMOpsPass() {
-  return std::make_unique<LowerGpuOpsToNVVMOpsPass>();
+mlir::createLowerGpuOpsToNVVMOpsPass(unsigned indexBitwidth) {
+  return std::make_unique<LowerGpuOpsToNVVMOpsPass>(indexBitwidth);
 }
diff --git a/mlir/lib/Conversion/GPUToROCDL/CMakeLists.txt b/mlir/lib/Conversion/GPUToROCDL/CMakeLists.txt
index 3e7294d54ac2c..0871b27aaddaa 100644
--- a/mlir/lib/Conversion/GPUToROCDL/CMakeLists.txt
+++ b/mlir/lib/Conversion/GPUToROCDL/CMakeLists.txt
@@ -2,7 +2,7 @@ set(LLVM_TARGET_DEFINITIONS GPUToROCDL.td)
 mlir_tablegen(GPUToROCDL.cpp.inc -gen-rewriters)
 add_public_tablegen_target(MLIRGPUToROCDLIncGen)
 
-add_mlir_conversion_library(MLIRGPUtoROCDLTransforms
+add_mlir_conversion_library(MLIRGPUToROCDLTransforms
   LowerGpuOpsToROCDLOps.cpp
 
   DEPENDS
diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index 2381d615f91b7..697f8078e725d 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -41,13 +41,22 @@ namespace {
 //
 // This pass only handles device code and is not meant to be run on GPU host
 // code.
-class LowerGpuOpsToROCDLOpsPass
+struct LowerGpuOpsToROCDLOpsPass
     : public ConvertGpuOpsToROCDLOpsBase<LowerGpuOpsToROCDLOpsPass> {
-public:
+  LowerGpuOpsToROCDLOpsPass() = default;
+  LowerGpuOpsToROCDLOpsPass(unsigned indexBitwidth) {
+    this->indexBitwidth = indexBitwidth;
+  }
+
   void runOnOperation() override {
     gpu::GPUModuleOp m = getOperation();
 
-    LLVMTypeConverter converter(m.getContext());
+    /// Customize the bitwidth used for the device side index computations.
+    LowerToLLVMOptions options = {/*useBarePtrCallConv =*/false,
+                                  /*emitCWrappers =*/true,
+                                  /*indexBitwidth =*/indexBitwidth,
+                                  /*useAlignedAlloc =*/false};
+    LLVMTypeConverter converter(m.getContext(), options);
 
     OwningRewritePatternList patterns;
 
@@ -65,7 +74,7 @@ class LowerGpuOpsToROCDLOpsPass
                         LLVM::LogOp, LLVM::Log10Op, LLVM::Log2Op>();
     target.addIllegalOp<FuncOp>();
     target.addLegalDialect<ROCDL::ROCDLDialect>();
-    // TODO(whchung): Remove once we support replacing non-root ops.
+    // TODO: Remove once we support replacing non-root ops.
     target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp, gpu::ModuleEndOp>();
     if (failed(applyPartialConversion(m, target, patterns)))
       signalPassFailure();
@@ -106,6 +115,6 @@ void mlir::populateGpuToROCDLConversionPatterns(
 }
 
 std::unique_ptr<OperationPass<gpu::GPUModuleOp>>
-mlir::createLowerGpuOpsToROCDLOpsPass() {
-  return std::make_unique<LowerGpuOpsToROCDLOpsPass>();
+mlir::createLowerGpuOpsToROCDLOpsPass(unsigned indexBitwidth) {
+  return std::make_unique<LowerGpuOpsToROCDLOpsPass>(indexBitwidth);
 }
diff --git a/mlir/lib/Conversion/GPUToSPIRV/CMakeLists.txt b/mlir/lib/Conversion/GPUToSPIRV/CMakeLists.txt
index fbf6b07870912..cce793fe5a6e0 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/CMakeLists.txt
+++ b/mlir/lib/Conversion/GPUToSPIRV/CMakeLists.txt
@@ -14,6 +14,7 @@ add_mlir_conversion_library(MLIRGPUToSPIRVTransforms
   MLIRGPU
   MLIRIR
   MLIRPass
+  MLIRSCFToSPIRV
   MLIRSPIRV
   MLIRStandardOps
   MLIRStandardToSPIRVTransforms
diff --git a/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.cpp
index 2b4829adcdeb4..2845611a920a0 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.cpp
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 #include "mlir/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.h"
 #include "mlir/Dialect/GPU/GPUDialect.h"
-#include "mlir/Dialect/SCF/SCF.h"
 #include "mlir/Dialect/SPIRV/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/SPIRVLowering.h"
 #include "mlir/Dialect/SPIRV/SPIRVOps.h"
@@ -20,41 +19,6 @@
 using namespace mlir;
 
 namespace {
-
-/// Pattern to convert a scf::ForOp within kernel functions into spirv::LoopOp.
-class ForOpConversion final : public SPIRVOpLowering<scf::ForOp> {
-public:
-  using SPIRVOpLowering<scf::ForOp>::SPIRVOpLowering;
-
-  LogicalResult
-  matchAndRewrite(scf::ForOp forOp, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override;
-};
-
-/// Pattern to convert a scf::IfOp within kernel functions into
-/// spirv::SelectionOp.
-class IfOpConversion final : public SPIRVOpLowering<scf::IfOp> {
-public:
-  using SPIRVOpLowering<scf::IfOp>::SPIRVOpLowering;
-
-  LogicalResult
-  matchAndRewrite(scf::IfOp IfOp, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override;
-};
-
-/// Pattern to erase a scf::YieldOp.
-class TerminatorOpConversion final : public SPIRVOpLowering<scf::YieldOp> {
-public:
-  using SPIRVOpLowering<scf::YieldOp>::SPIRVOpLowering;
-
-  LogicalResult
-  matchAndRewrite(scf::YieldOp terminatorOp, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override {
-    rewriter.eraseOp(terminatorOp);
-    return success();
-  }
-};
-
 /// Pattern lowering GPU block/thread size/id to loading SPIR-V invocation
 /// builtin variables.
 template <typename SourceOp, spirv::BuiltIn builtin>
@@ -128,134 +92,6 @@ class GPUReturnOpConversion final : public SPIRVOpLowering<gpu::ReturnOp> {
 
 } // namespace
 
-//===----------------------------------------------------------------------===//
-// scf::ForOp.
-//===----------------------------------------------------------------------===//
-
-LogicalResult
-ForOpConversion::matchAndRewrite(scf::ForOp forOp, ArrayRef<Value> operands,
-                                 ConversionPatternRewriter &rewriter) const {
-  // scf::ForOp can be lowered to the structured control flow represented by
-  // spirv::LoopOp by making the continue block of the spirv::LoopOp the loop
-  // latch and the merge block the exit block. The resulting spirv::LoopOp has a
-  // single back edge from the continue to header block, and a single exit from
-  // header to merge.
-  scf::ForOpAdaptor forOperands(operands);
-  auto loc = forOp.getLoc();
-  auto loopControl = rewriter.getI32IntegerAttr(
-      static_cast<uint32_t>(spirv::LoopControl::None));
-  auto loopOp = rewriter.create<spirv::LoopOp>(loc, loopControl);
-  loopOp.addEntryAndMergeBlock();
-
-  OpBuilder::InsertionGuard guard(rewriter);
-  // Create the block for the header.
-  auto header = new Block();
-  // Insert the header.
-  loopOp.body().getBlocks().insert(std::next(loopOp.body().begin(), 1), header);
-
-  // Create the new induction variable to use.
-  BlockArgument newIndVar =
-      header->addArgument(forOperands.lowerBound().getType());
-  Block *body = forOp.getBody();
-
-  // Apply signature conversion to the body of the forOp. It has a single block,
-  // with argument which is the induction variable. That has to be replaced with
-  // the new induction variable.
-  TypeConverter::SignatureConversion signatureConverter(
-      body->getNumArguments());
-  signatureConverter.remapInput(0, newIndVar);
-  FailureOr<Block *> newBody = rewriter.convertRegionTypes(
-      &forOp.getLoopBody(), typeConverter, &signatureConverter);
-  if (failed(newBody))
-    return failure();
-  body = *newBody;
-
-  // Delete the loop terminator.
-  rewriter.eraseOp(body->getTerminator());
-
-  // Move the blocks from the forOp into the loopOp. This is the body of the
-  // loopOp.
-  rewriter.inlineRegionBefore(forOp.getOperation()->getRegion(0), loopOp.body(),
-                              std::next(loopOp.body().begin(), 2));
-
-  // Branch into it from the entry.
-  rewriter.setInsertionPointToEnd(&(loopOp.body().front()));
-  rewriter.create<spirv::BranchOp>(loc, header, forOperands.lowerBound());
-
-  // Generate the rest of the loop header.
-  rewriter.setInsertionPointToEnd(header);
-  auto mergeBlock = loopOp.getMergeBlock();
-  auto cmpOp = rewriter.create<spirv::SLessThanOp>(
-      loc, rewriter.getI1Type(), newIndVar, forOperands.upperBound());
-  rewriter.create<spirv::BranchConditionalOp>(
-      loc, cmpOp, body, ArrayRef<Value>(), mergeBlock, ArrayRef<Value>());
-
-  // Generate instructions to increment the step of the induction variable and
-  // branch to the header.
-  Block *continueBlock = loopOp.getContinueBlock();
-  rewriter.setInsertionPointToEnd(continueBlock);
-
-  // Add the step to the induction variable and branch to the header.
-  Value updatedIndVar = rewriter.create<spirv::IAddOp>(
-      loc, newIndVar.getType(), newIndVar, forOperands.step());
-  rewriter.create<spirv::BranchOp>(loc, header, updatedIndVar);
-
-  rewriter.eraseOp(forOp);
-  return success();
-}
-
-//===----------------------------------------------------------------------===//
-// scf::IfOp.
-//===----------------------------------------------------------------------===//
-
-LogicalResult
-IfOpConversion::matchAndRewrite(scf::IfOp ifOp, ArrayRef<Value> operands,
-                                ConversionPatternRewriter &rewriter) const {
-  // When lowering `scf::IfOp` we explicitly create a selection header block
-  // before the control flow diverges and a merge block where control flow
-  // subsequently converges.
-  scf::IfOpAdaptor ifOperands(operands);
-  auto loc = ifOp.getLoc();
-
-  // Create `spv.selection` operation, selection header block and merge block.
-  auto selectionControl = rewriter.getI32IntegerAttr(
-      static_cast<uint32_t>(spirv::SelectionControl::None));
-  auto selectionOp = rewriter.create<spirv::SelectionOp>(loc, selectionControl);
-  selectionOp.addMergeBlock();
-  auto *mergeBlock = selectionOp.getMergeBlock();
-
-  OpBuilder::InsertionGuard guard(rewriter);
-  auto *selectionHeaderBlock = new Block();
-  selectionOp.body().getBlocks().push_front(selectionHeaderBlock);
-
-  // Inline `then` region before the merge block and branch to it.
-  auto &thenRegion = ifOp.thenRegion();
-  auto *thenBlock = &thenRegion.front();
-  rewriter.setInsertionPointToEnd(&thenRegion.back());
-  rewriter.create<spirv::BranchOp>(loc, mergeBlock);
-  rewriter.inlineRegionBefore(thenRegion, mergeBlock);
-
-  auto *elseBlock = mergeBlock;
-  // If `else` region is not empty, inline that region before the merge block
-  // and branch to it.
-  if (!ifOp.elseRegion().empty()) {
-    auto &elseRegion = ifOp.elseRegion();
-    elseBlock = &elseRegion.front();
-    rewriter.setInsertionPointToEnd(&elseRegion.back());
-    rewriter.create<spirv::BranchOp>(loc, mergeBlock);
-    rewriter.inlineRegionBefore(elseRegion, mergeBlock);
-  }
-
-  // Create a `spv.BranchConditional` operation for selection header block.
-  rewriter.setInsertionPointToEnd(selectionHeaderBlock);
-  rewriter.create<spirv::BranchConditionalOp>(loc, ifOperands.condition(),
-                                              thenBlock, ArrayRef<Value>(),
-                                              elseBlock, ArrayRef<Value>());
-
-  rewriter.eraseOp(ifOp);
-  return success();
-}
-
 //===----------------------------------------------------------------------===//
 // Builtins.
 //===----------------------------------------------------------------------===//
@@ -479,8 +315,7 @@ void mlir::populateGPUToSPIRVPatterns(MLIRContext *context,
                                       OwningRewritePatternList &patterns) {
   populateWithGenerated(context, &patterns);
   patterns.insert<
-      ForOpConversion, GPUFuncOpConversion, GPUModuleConversion,
-      GPUReturnOpConversion, IfOpConversion,
+      GPUFuncOpConversion, GPUModuleConversion, GPUReturnOpConversion,
       LaunchConfigConversion<gpu::BlockIdOp, spirv::BuiltIn::WorkgroupId>,
       LaunchConfigConversion<gpu::GridDimOp, spirv::BuiltIn::NumWorkgroups>,
       LaunchConfigConversion<gpu::ThreadIdOp,
@@ -491,5 +326,5 @@ void mlir::populateGPUToSPIRVPatterns(MLIRContext *context,
                                       spirv::BuiltIn::NumSubgroups>,
       SingleDimLaunchConfigConversion<gpu::SubgroupSizeOp,
                                       spirv::BuiltIn::SubgroupSize>,
-      TerminatorOpConversion, WorkGroupSizeConversion>(context, typeConverter);
+      WorkGroupSizeConversion>(context, typeConverter);
 }
diff --git a/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.cpp b/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.cpp
index 1f486b96e86cc..ccaecd9c91892 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.cpp
@@ -14,6 +14,7 @@
 #include "mlir/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.h"
 #include "../PassDetail.h"
 #include "mlir/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.h"
+#include "mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h"
 #include "mlir/Conversion/StandardToSPIRV/ConvertStandardToSPIRV.h"
 #include "mlir/Dialect/GPU/GPUDialect.h"
 #include "mlir/Dialect/SCF/SCF.h"
@@ -57,8 +58,10 @@ void GPUToSPIRVPass::runOnOperation() {
       spirv::SPIRVConversionTarget::get(targetAttr);
 
   SPIRVTypeConverter typeConverter(targetAttr);
+  ScfToSPIRVContext scfContext;
   OwningRewritePatternList patterns;
   populateGPUToSPIRVPatterns(context, typeConverter, patterns);
+  populateSCFToSPIRVPatterns(context, typeConverter,scfContext, patterns);
   populateStandardToSPIRVPatterns(context, typeConverter, patterns);
 
   if (failed(applyFullConversion(kernelModules, *target, patterns)))
diff --git a/mlir/lib/Conversion/GPUToVulkan/CMakeLists.txt b/mlir/lib/Conversion/GPUToVulkan/CMakeLists.txt
index c0265096d73bb..b62f72fe91918 100644
--- a/mlir/lib/Conversion/GPUToVulkan/CMakeLists.txt
+++ b/mlir/lib/Conversion/GPUToVulkan/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_mlir_conversion_library(MLIRGPUtoVulkanTransforms
+add_mlir_conversion_library(MLIRGPUToVulkanTransforms
   ConvertLaunchFuncToVulkanCalls.cpp
   ConvertGPULaunchFuncToVulkanLaunchFunc.cpp
 
diff --git a/mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp b/mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp
index d6908680d798d..1ebf48174aafb 100644
--- a/mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp
+++ b/mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp
@@ -124,8 +124,7 @@ LogicalResult ConvertGpuLaunchFuncToVulkanLaunchFunc::declareVulkanLaunchFunc(
   // Declare vulkan launch function.
   builder.create<FuncOp>(
       loc, kVulkanLaunch,
-      FunctionType::get(vulkanLaunchTypes, ArrayRef<Type>{}, loc->getContext()),
-      ArrayRef<NamedAttribute>{});
+      FunctionType::get(vulkanLaunchTypes, {}, loc->getContext()));
 
   return success();
 }
diff --git a/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp b/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp
index b92ab13bd513d..7b57854dde983 100644
--- a/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp
+++ b/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp
@@ -219,7 +219,7 @@ class SliceOpConversion : public ConvertToLLVMPattern {
     BaseViewConversionHelper desc(
         typeConverter.convertType(sliceOp.getShapedType()));
 
-    // TODO(ntv): extract sizes and emit asserts.
+    // TODO: extract sizes and emit asserts.
     SmallVector<Value, 4> strides(memRefType.getRank());
     for (int i = 0, e = memRefType.getRank(); i < e; ++i)
       strides[i] = baseDesc.stride(i);
diff --git a/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRV.cpp b/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRV.cpp
index cf67b96fce1ee..0cde4a05ece5f 100644
--- a/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRV.cpp
+++ b/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRV.cpp
@@ -92,7 +92,7 @@ SingleWorkgroupReduction::matchAsPerformingReduction(
   if (genericOp.indexing_maps().getValue().size() != 2)
     return llvm::None;
 
-  // TODO(nicolasvasilache): create utility functions for these checks in Linalg
+  // TODO: create utility functions for these checks in Linalg
   // and use them.
   auto inputMap = genericOp.indexing_maps().getValue()[0].cast<AffineMapAttr>();
   auto outputMap =
@@ -132,7 +132,7 @@ LogicalResult SingleWorkgroupReduction::matchAndRewrite(
                    [](const APInt &size) { return !size.isOneValue(); }))
     return failure();
 
-  // TODO(antiagainst): Query the target environment to make sure the current
+  // TODO: Query the target environment to make sure the current
   // workload fits in a local workgroup.
 
   Value convertedInput = operands[0], convertedOutput = operands[1];
@@ -141,7 +141,7 @@ LogicalResult SingleWorkgroupReduction::matchAndRewrite(
   // Get the invocation ID.
   Value x = getLocalInvocationDimSize(genericOp, /*dim=*/0, loc, &rewriter);
 
-  // TODO(antiagainst): Load to Workgroup storage class first.
+  // TODO: Load to Workgroup storage class first.
 
   // Get the input element accessed by this invocation.
   Value inputElementPtr = spirv::getElementPtr(
diff --git a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
index 7f13a7a609e91..43f9d88253275 100644
--- a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
+++ b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
@@ -79,8 +79,7 @@ static FlatSymbolRefAttr getLibraryCallSymbolRef(Operation *op,
   rewriter.setInsertionPoint(module.getBody(),
                              std::prev(module.getBody()->end()));
   FuncOp funcOp =
-      rewriter.create<FuncOp>(op->getLoc(), fnNameAttr.getValue(), libFnType,
-                              ArrayRef<NamedAttribute>{});
+      rewriter.create<FuncOp>(op->getLoc(), fnNameAttr.getValue(), libFnType);
   // Insert a function attribute that will trigger the emission of the
   // corresponding `_mlir_ciface_xxx` interface so that external libraries see
   // a normalized ABI. This interface is added during std to llvm conversion.
@@ -173,7 +172,7 @@ class LinalgOpConversion<IndexedGenericOp>
     if (!libraryCallName)
       return failure();
 
-    // TODO(pifon, ntv): Use induction variables values instead of zeros, when
+    // TODO: Use induction variables values instead of zeros, when
     // IndexedGenericOp is tiled.
     auto zero = rewriter.create<mlir::ConstantOp>(
         op.getLoc(), rewriter.getIntegerAttr(rewriter.getIndexType(), 0));
@@ -227,7 +226,7 @@ class CopyTransposeConversion : public OpRewritePattern<CopyOp> {
 /// Populate the given list with patterns that convert from Linalg to Standard.
 void mlir::populateLinalgToStandardConversionPatterns(
     OwningRewritePatternList &patterns, MLIRContext *ctx) {
-  // TODO(ntv) ConvOp conversion needs to export a descriptor with relevant
+  // TODO: ConvOp conversion needs to export a descriptor with relevant
   // attribute values such as kernel striding and dilation.
   // clang-format off
   patterns.insert<
diff --git a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
index 4acce99ce2dc1..0a657e5387b27 100644
--- a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
+++ b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
@@ -378,12 +378,12 @@ static LogicalResult processParallelLoop(
     ParallelOp parallelOp, gpu::LaunchOp launchOp,
     BlockAndValueMapping &cloningMap, SmallVectorImpl<Operation *> &worklist,
     DenseMap<gpu::Processor, Value> &bounds, PatternRewriter &rewriter) {
-  // TODO(herhut): Verify that this is a valid GPU mapping.
+  // TODO: Verify that this is a valid GPU mapping.
   // processor ids: 0-2 block [x/y/z], 3-5 -> thread [x/y/z], 6-> sequential
   ArrayAttr mapping =
       parallelOp.getAttrOfType<ArrayAttr>(gpu::getMappingAttrName());
 
-  // TODO(herhut): Support reductions.
+  // TODO: Support reductions.
   if (!mapping || parallelOp.getNumResults() != 0)
     return failure();
 
@@ -431,7 +431,7 @@ static LogicalResult processParallelLoop(
           loc, annotation.map().getValue().compose(lowerAndStep),
           ValueRange{operand, step, lowerBound});
       // If there was also a bound, insert that, too.
-      // TODO(herhut): Check that we do not assign bounds twice.
+      // TODO: Check that we do not assign bounds twice.
       if (annotation.bound().getValue()) {
         // We pass as the single operand to the bound-map the number of
         // iterations, which is (upperBound - lowerBound) ceilDiv step. To
diff --git a/mlir/lib/Conversion/SCFToSPIRV/CMakeLists.txt b/mlir/lib/Conversion/SCFToSPIRV/CMakeLists.txt
new file mode 100644
index 0000000000000..6d95813d717f7
--- /dev/null
+++ b/mlir/lib/Conversion/SCFToSPIRV/CMakeLists.txt
@@ -0,0 +1,20 @@
+add_mlir_conversion_library(MLIRSCFToSPIRV
+  SCFToSPIRV.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/SCFToSPIRV
+
+  DEPENDS
+  MLIRConversionPassIncGen
+
+  LINK_LIBS PUBLIC
+  MLIRAffineOps
+  MLIRAffineToStandard
+  MLIRSPIRV
+  MLIRIR
+  MLIRLinalgOps
+  MLIRPass
+  MLIRStandardOps
+  MLIRSupport
+  MLIRTransforms
+  )
diff --git a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
new file mode 100644
index 0000000000000..b8eb87c803686
--- /dev/null
+++ b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
@@ -0,0 +1,282 @@
+//===- SCFToSPIRV.cpp - Convert SCF ops to SPIR-V dialect -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the conversion patterns from SCF ops to SPIR-V dialect.
+//
+//===----------------------------------------------------------------------===//
+#include "mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h"
+#include "mlir/Dialect/SCF/SCF.h"
+#include "mlir/Dialect/SPIRV/SPIRVDialect.h"
+#include "mlir/Dialect/SPIRV/SPIRVLowering.h"
+#include "mlir/Dialect/SPIRV/SPIRVOps.h"
+#include "mlir/IR/Module.h"
+
+using namespace mlir;
+
+namespace mlir {
+struct ScfToSPIRVContextImpl {
+  // Map between the spirv region control flow operation (spv.loop or
+  // spv.selection) to the VariableOp created to store the region results. The
+  // order of the VariableOp matches the order of the results.
+  DenseMap<Operation *, SmallVector<spirv::VariableOp, 8>> outputVars;
+};
+} // namespace mlir
+
+/// We use ScfToSPIRVContext to store information about the lowering of the scf
+/// region that need to be used later on. When we lower scf.for/scf.if we create
+/// VariableOp to store the results. We need to keep track of the VariableOp
+/// created as we need to insert stores into them when lowering Yield. Those
+/// StoreOp cannot be created earlier as they may use a different type than
+/// yield operands.
+ScfToSPIRVContext::ScfToSPIRVContext() {
+  impl = std::make_unique<ScfToSPIRVContextImpl>();
+}
+ScfToSPIRVContext::~ScfToSPIRVContext() = default;
+
+namespace {
+/// Common class for all vector to GPU patterns.
+template <typename OpTy>
+class SCFToSPIRVPattern : public SPIRVOpLowering<OpTy> {
+public:
+  SCFToSPIRVPattern<OpTy>(MLIRContext *context, SPIRVTypeConverter &converter,
+                          ScfToSPIRVContextImpl *scfToSPIRVContext)
+      : SPIRVOpLowering<OpTy>::SPIRVOpLowering(context, converter),
+        scfToSPIRVContext(scfToSPIRVContext) {}
+
+protected:
+  ScfToSPIRVContextImpl *scfToSPIRVContext;
+};
+
+/// Pattern to convert a scf::ForOp within kernel functions into spirv::LoopOp.
+class ForOpConversion final : public SCFToSPIRVPattern<scf::ForOp> {
+public:
+  using SCFToSPIRVPattern<scf::ForOp>::SCFToSPIRVPattern;
+
+  LogicalResult
+  matchAndRewrite(scf::ForOp forOp, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override;
+};
+
+/// Pattern to convert a scf::IfOp within kernel functions into
+/// spirv::SelectionOp.
+class IfOpConversion final : public SCFToSPIRVPattern<scf::IfOp> {
+public:
+  using SCFToSPIRVPattern<scf::IfOp>::SCFToSPIRVPattern;
+
+  LogicalResult
+  matchAndRewrite(scf::IfOp ifOp, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override;
+};
+
+class TerminatorOpConversion final : public SCFToSPIRVPattern<scf::YieldOp> {
+public:
+  using SCFToSPIRVPattern<scf::YieldOp>::SCFToSPIRVPattern;
+
+  LogicalResult
+  matchAndRewrite(scf::YieldOp terminatorOp, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override;
+};
+} // namespace
+
+/// Helper function to replaces SCF op outputs with SPIR-V variable loads.
+/// We create VariableOp to handle the results value of the control flow region.
+/// spv.loop/spv.selection currently don't yield value. Right after the loop
+/// we load the value from the allocation and use it as the SCF op result.
+template <typename ScfOp, typename OpTy>
+static void replaceSCFOutputValue(ScfOp scfOp, OpTy newOp,
+                                  SPIRVTypeConverter &typeConverter,
+                                  ConversionPatternRewriter &rewriter,
+                                  ScfToSPIRVContextImpl *scfToSPIRVContext) {
+
+  Location loc = scfOp.getLoc();
+  auto &allocas = scfToSPIRVContext->outputVars[newOp];
+  SmallVector<Value, 8> resultValue;
+  for (Value result : scfOp.results()) {
+    auto convertedType = typeConverter.convertType(result.getType());
+    auto pointerType =
+        spirv::PointerType::get(convertedType, spirv::StorageClass::Function);
+    rewriter.setInsertionPoint(newOp);
+    auto alloc = rewriter.create<spirv::VariableOp>(
+        loc, pointerType, spirv::StorageClass::Function,
+        /*initializer=*/nullptr);
+    allocas.push_back(alloc);
+    rewriter.setInsertionPointAfter(newOp);
+    Value loadResult = rewriter.create<spirv::LoadOp>(loc, alloc);
+    resultValue.push_back(loadResult);
+  }
+  rewriter.replaceOp(scfOp, resultValue);
+}
+
+//===----------------------------------------------------------------------===//
+// scf::ForOp.
+//===----------------------------------------------------------------------===//
+
+LogicalResult
+ForOpConversion::matchAndRewrite(scf::ForOp forOp, ArrayRef<Value> operands,
+                                 ConversionPatternRewriter &rewriter) const {
+  // scf::ForOp can be lowered to the structured control flow represented by
+  // spirv::LoopOp by making the continue block of the spirv::LoopOp the loop
+  // latch and the merge block the exit block. The resulting spirv::LoopOp has a
+  // single back edge from the continue to header block, and a single exit from
+  // header to merge.
+  scf::ForOpAdaptor forOperands(operands);
+  auto loc = forOp.getLoc();
+  auto loopControl = rewriter.getI32IntegerAttr(
+      static_cast<uint32_t>(spirv::LoopControl::None));
+  auto loopOp = rewriter.create<spirv::LoopOp>(loc, loopControl);
+  loopOp.addEntryAndMergeBlock();
+
+  OpBuilder::InsertionGuard guard(rewriter);
+  // Create the block for the header.
+  auto *header = new Block();
+  // Insert the header.
+  loopOp.body().getBlocks().insert(std::next(loopOp.body().begin(), 1), header);
+
+  // Create the new induction variable to use.
+  BlockArgument newIndVar =
+      header->addArgument(forOperands.lowerBound().getType());
+  for (Value arg : forOperands.initArgs())
+    header->addArgument(arg.getType());
+  Block *body = forOp.getBody();
+
+  // Apply signature conversion to the body of the forOp. It has a single block,
+  // with argument which is the induction variable. That has to be replaced with
+  // the new induction variable.
+  TypeConverter::SignatureConversion signatureConverter(
+      body->getNumArguments());
+  signatureConverter.remapInput(0, newIndVar);
+  for (unsigned i = 1, e = body->getNumArguments(); i < e; i++)
+    signatureConverter.remapInput(i, header->getArgument(i));
+  body = rewriter.applySignatureConversion(&forOp.getLoopBody(),
+                                           signatureConverter);
+
+  // Move the blocks from the forOp into the loopOp. This is the body of the
+  // loopOp.
+  rewriter.inlineRegionBefore(forOp.getOperation()->getRegion(0), loopOp.body(),
+                              std::next(loopOp.body().begin(), 2));
+
+  SmallVector<Value, 8> args(1, forOperands.lowerBound());
+  args.append(forOperands.initArgs().begin(), forOperands.initArgs().end());
+  // Branch into it from the entry.
+  rewriter.setInsertionPointToEnd(&(loopOp.body().front()));
+  rewriter.create<spirv::BranchOp>(loc, header, args);
+
+  // Generate the rest of the loop header.
+  rewriter.setInsertionPointToEnd(header);
+  auto *mergeBlock = loopOp.getMergeBlock();
+  auto cmpOp = rewriter.create<spirv::SLessThanOp>(
+      loc, rewriter.getI1Type(), newIndVar, forOperands.upperBound());
+
+  rewriter.create<spirv::BranchConditionalOp>(
+      loc, cmpOp, body, ArrayRef<Value>(), mergeBlock, ArrayRef<Value>());
+
+  // Generate instructions to increment the step of the induction variable and
+  // branch to the header.
+  Block *continueBlock = loopOp.getContinueBlock();
+  rewriter.setInsertionPointToEnd(continueBlock);
+
+  // Add the step to the induction variable and branch to the header.
+  Value updatedIndVar = rewriter.create<spirv::IAddOp>(
+      loc, newIndVar.getType(), newIndVar, forOperands.step());
+  rewriter.create<spirv::BranchOp>(loc, header, updatedIndVar);
+
+  replaceSCFOutputValue(forOp, loopOp, typeConverter, rewriter,
+                        scfToSPIRVContext);
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// scf::IfOp.
+//===----------------------------------------------------------------------===//
+
+LogicalResult
+IfOpConversion::matchAndRewrite(scf::IfOp ifOp, ArrayRef<Value> operands,
+                                ConversionPatternRewriter &rewriter) const {
+  // When lowering `scf::IfOp` we explicitly create a selection header block
+  // before the control flow diverges and a merge block where control flow
+  // subsequently converges.
+  scf::IfOpAdaptor ifOperands(operands);
+  auto loc = ifOp.getLoc();
+
+  // Create `spv.selection` operation, selection header block and merge block.
+  auto selectionControl = rewriter.getI32IntegerAttr(
+      static_cast<uint32_t>(spirv::SelectionControl::None));
+  auto selectionOp = rewriter.create<spirv::SelectionOp>(loc, selectionControl);
+  selectionOp.addMergeBlock();
+  auto *mergeBlock = selectionOp.getMergeBlock();
+
+  OpBuilder::InsertionGuard guard(rewriter);
+  auto *selectionHeaderBlock = new Block();
+  selectionOp.body().getBlocks().push_front(selectionHeaderBlock);
+
+  // Inline `then` region before the merge block and branch to it.
+  auto &thenRegion = ifOp.thenRegion();
+  auto *thenBlock = &thenRegion.front();
+  rewriter.setInsertionPointToEnd(&thenRegion.back());
+  rewriter.create<spirv::BranchOp>(loc, mergeBlock);
+  rewriter.inlineRegionBefore(thenRegion, mergeBlock);
+
+  auto *elseBlock = mergeBlock;
+  // If `else` region is not empty, inline that region before the merge block
+  // and branch to it.
+  if (!ifOp.elseRegion().empty()) {
+    auto &elseRegion = ifOp.elseRegion();
+    elseBlock = &elseRegion.front();
+    rewriter.setInsertionPointToEnd(&elseRegion.back());
+    rewriter.create<spirv::BranchOp>(loc, mergeBlock);
+    rewriter.inlineRegionBefore(elseRegion, mergeBlock);
+  }
+
+  // Create a `spv.BranchConditional` operation for selection header block.
+  rewriter.setInsertionPointToEnd(selectionHeaderBlock);
+  rewriter.create<spirv::BranchConditionalOp>(loc, ifOperands.condition(),
+                                              thenBlock, ArrayRef<Value>(),
+                                              elseBlock, ArrayRef<Value>());
+
+  replaceSCFOutputValue(ifOp, selectionOp, typeConverter, rewriter,
+                        scfToSPIRVContext);
+  return success();
+}
+
+/// Yield is lowered to stores to the VariableOp created during lowering of the
+/// parent region. For loops we also need to update the branch looping back to
+/// the header with the loop carried values.
+LogicalResult TerminatorOpConversion::matchAndRewrite(
+    scf::YieldOp terminatorOp, ArrayRef<Value> operands,
+    ConversionPatternRewriter &rewriter) const {
+  // If the region is return values, store each value into the associated
+  // VariableOp created during lowering of the parent region.
+  if (!operands.empty()) {
+    auto loc = terminatorOp.getLoc();
+    auto &allocas = scfToSPIRVContext->outputVars[terminatorOp.getParentOp()];
+    assert(allocas.size() == operands.size());
+    for (unsigned i = 0, e = operands.size(); i < e; i++)
+      rewriter.create<spirv::StoreOp>(loc, allocas[i], operands[i]);
+    if (isa<spirv::LoopOp>(terminatorOp.getParentOp())) {
+      // For loops we also need to update the branch jumping back to the header.
+      auto br =
+          cast<spirv::BranchOp>(rewriter.getInsertionBlock()->getTerminator());
+      SmallVector<Value, 8> args(br.getBlockArguments());
+      args.append(operands.begin(), operands.end());
+      rewriter.setInsertionPoint(br);
+      rewriter.create<spirv::BranchOp>(terminatorOp.getLoc(), br.getTarget(),
+                                       args);
+      rewriter.eraseOp(br);
+    }
+  }
+  rewriter.eraseOp(terminatorOp);
+  return success();
+}
+
+void mlir::populateSCFToSPIRVPatterns(MLIRContext *context,
+                                      SPIRVTypeConverter &typeConverter,
+                                      ScfToSPIRVContext &scfToSPIRVContext,
+                                      OwningRewritePatternList &patterns) {
+  patterns.insert<ForOpConversion, IfOpConversion, TerminatorOpConversion>(
+      context, typeConverter, scfToSPIRVContext.getImpl());
+}
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
index 85f6a113cff3a..5820d9022ae1f 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
@@ -21,6 +21,9 @@
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Support/LogicalResult.h"
 #include "mlir/Transforms/DialectConversion.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "spirv-to-llvm-pattern"
 
 using namespace mlir;
 
@@ -28,6 +31,15 @@ using namespace mlir;
 // Utility functions
 //===----------------------------------------------------------------------===//
 
+/// Returns true if the given type is a signed integer or vector type.
+static bool isSignedIntegerOrVector(Type type) {
+  if (type.isSignedInteger())
+    return true;
+  if (auto vecType = type.dyn_cast<VectorType>())
+    return vecType.getElementType().isSignedInteger();
+  return false;
+}
+
 /// Returns true if the given type is an unsigned integer or vector type
 static bool isUnsignedIntegerOrVector(Type type) {
   if (type.isUnsignedInteger())
@@ -50,12 +62,351 @@ static unsigned getBitWidth(Type type) {
   return elementType.getIntOrFloatBitWidth();
 }
 
+/// Returns the bit width of LLVMType integer or vector.
+static unsigned getLLVMTypeBitWidth(LLVM::LLVMType type) {
+  return type.isVectorTy() ? type.getVectorElementType()
+                                 .getUnderlyingType()
+                                 ->getIntegerBitWidth()
+                           : type.getUnderlyingType()->getIntegerBitWidth();
+}
+
+/// Creates `IntegerAttribute` with all bits set for given type
+static IntegerAttr minusOneIntegerAttribute(Type type, Builder builder) {
+  if (auto vecType = type.dyn_cast<VectorType>()) {
+    auto integerType = vecType.getElementType().cast<IntegerType>();
+    return builder.getIntegerAttr(integerType, -1);
+  }
+  auto integerType = type.cast<IntegerType>();
+  return builder.getIntegerAttr(integerType, -1);
+}
+
+/// Creates `llvm.mlir.constant` with all bits set for the given type.
+static Value createConstantAllBitsSet(Location loc, Type srcType, Type dstType,
+                                      PatternRewriter &rewriter) {
+  if (srcType.isa<VectorType>())
+    return rewriter.create<LLVM::ConstantOp>(
+        loc, dstType,
+        SplatElementsAttr::get(srcType.cast<ShapedType>(),
+                               minusOneIntegerAttribute(srcType, rewriter)));
+  return rewriter.create<LLVM::ConstantOp>(
+      loc, dstType, minusOneIntegerAttribute(srcType, rewriter));
+}
+
+/// Utility function for bitfiled ops:
+///   - `BitFieldInsert`
+///   - `BitFieldSExtract`
+///   - `BitFieldUExtract`
+/// Truncates or extends the value. If the bitwidth of the value is the same as
+/// `dstType` bitwidth, the value remains unchanged.
+static Value optionallyTruncateOrExtend(Location loc, Value value, Type dstType,
+                                        PatternRewriter &rewriter) {
+  auto srcType = value.getType();
+  auto llvmType = dstType.cast<LLVM::LLVMType>();
+  unsigned targetBitWidth = getLLVMTypeBitWidth(llvmType);
+  unsigned valueBitWidth =
+      srcType.isa<LLVM::LLVMType>()
+          ? getLLVMTypeBitWidth(srcType.cast<LLVM::LLVMType>())
+          : getBitWidth(srcType);
+
+  if (valueBitWidth < targetBitWidth)
+    return rewriter.create<LLVM::ZExtOp>(loc, llvmType, value);
+  // If the bit widths of `Count` and `Offset` are greater than the bit width
+  // of the target type, they are truncated. Truncation is safe since `Count`
+  // and `Offset` must be no more than 64 for op behaviour to be defined. Hence,
+  // both values can be expressed in 8 bits.
+  if (valueBitWidth > targetBitWidth)
+    return rewriter.create<LLVM::TruncOp>(loc, llvmType, value);
+  return value;
+}
+
+/// Broadcasts the value to vector with `numElements` number of elements.
+static Value broadcast(Location loc, Value toBroadcast, unsigned numElements,
+                       LLVMTypeConverter &typeConverter,
+                       ConversionPatternRewriter &rewriter) {
+  auto vectorType = VectorType::get(numElements, toBroadcast.getType());
+  auto llvmVectorType = typeConverter.convertType(vectorType);
+  auto llvmI32Type = typeConverter.convertType(rewriter.getIntegerType(32));
+  Value broadcasted = rewriter.create<LLVM::UndefOp>(loc, llvmVectorType);
+  for (unsigned i = 0; i < numElements; ++i) {
+    auto index = rewriter.create<LLVM::ConstantOp>(
+        loc, llvmI32Type, rewriter.getI32IntegerAttr(i));
+    broadcasted = rewriter.create<LLVM::InsertElementOp>(
+        loc, llvmVectorType, broadcasted, toBroadcast, index);
+  }
+  return broadcasted;
+}
+
+/// Broadcasts the value. If `srcType` is a scalar, the value remains unchanged.
+static Value optionallyBroadcast(Location loc, Value value, Type srcType,
+                                 LLVMTypeConverter &typeConverter,
+                                 ConversionPatternRewriter &rewriter) {
+  if (auto vectorType = srcType.dyn_cast<VectorType>()) {
+    unsigned numElements = vectorType.getNumElements();
+    return broadcast(loc, value, numElements, typeConverter, rewriter);
+  }
+  return value;
+}
+
+/// Utility function for bitfiled ops: `BitFieldInsert`, `BitFieldSExtract` and
+/// `BitFieldUExtract`.
+/// Broadcast `Offset` and `Count` to match the type of `Base`. If `Base` is of
+/// a vector type, construct a vector that has:
+///  - same number of elements as `Base`
+///  - each element has the type that is the same as the type of `Offset` or
+///    `Count`
+///  - each element has the same value as `Offset` or `Count`
+/// Then cast `Offset` and `Count` if their bit width is different
+/// from `Base` bit width.
+static Value processCountOrOffset(Location loc, Value value, Type srcType,
+                                  Type dstType, LLVMTypeConverter &converter,
+                                  ConversionPatternRewriter &rewriter) {
+  Value broadcasted =
+      optionallyBroadcast(loc, value, srcType, converter, rewriter);
+  return optionallyTruncateOrExtend(loc, broadcasted, dstType, rewriter);
+}
+
+/// Converts SPIR-V struct with no offset to packed LLVM struct.
+static Type convertStructTypePacked(spirv::StructType type,
+                                    LLVMTypeConverter &converter) {
+  auto elementsVector = llvm::to_vector<8>(
+      llvm::map_range(type.getElementTypes(), [&](Type elementType) {
+        return converter.convertType(elementType).cast<LLVM::LLVMType>();
+      }));
+  return LLVM::LLVMType::getStructTy(converter.getDialect(), elementsVector,
+                                     /*isPacked=*/true);
+}
+
+//===----------------------------------------------------------------------===//
+// Type conversion
+//===----------------------------------------------------------------------===//
+
+/// Converts SPIR-V array type to LLVM array. There is no modelling of array
+/// stride at the moment.
+static Optional<Type> convertArrayType(spirv::ArrayType type,
+                                       TypeConverter &converter) {
+  if (type.getArrayStride() != 0)
+    return llvm::None;
+  auto elementType =
+      converter.convertType(type.getElementType()).cast<LLVM::LLVMType>();
+  unsigned numElements = type.getNumElements();
+  return LLVM::LLVMType::getArrayTy(elementType, numElements);
+}
+
+/// Converts SPIR-V pointer type to LLVM pointer. Pointer's storage class is not
+/// modelled at the moment.
+static Type convertPointerType(spirv::PointerType type,
+                               TypeConverter &converter) {
+  auto pointeeType =
+      converter.convertType(type.getPointeeType()).cast<LLVM::LLVMType>();
+  return pointeeType.getPointerTo();
+}
+
+/// Converts SPIR-V runtime array to LLVM array. Since LLVM allows indexing over
+/// the bounds, the runtime array is converted to a 0-sized LLVM array. There is
+/// no modelling of array stride at the moment.
+static Optional<Type> convertRuntimeArrayType(spirv::RuntimeArrayType type,
+                                              TypeConverter &converter) {
+  if (type.getArrayStride() != 0)
+    return llvm::None;
+  auto elementType =
+      converter.convertType(type.getElementType()).cast<LLVM::LLVMType>();
+  return LLVM::LLVMType::getArrayTy(elementType, 0);
+}
+
+/// Converts SPIR-V struct to LLVM struct. There is no support of structs with
+/// member decorations or with offset.
+static Optional<Type> convertStructType(spirv::StructType type,
+                                        LLVMTypeConverter &converter) {
+  SmallVector<spirv::StructType::MemberDecorationInfo, 4> memberDecorations;
+  type.getMemberDecorations(memberDecorations);
+  if (type.hasOffset() || !memberDecorations.empty())
+    return llvm::None;
+  return convertStructTypePacked(type, converter);
+}
+
 //===----------------------------------------------------------------------===//
 // Operation conversion
 //===----------------------------------------------------------------------===//
 
 namespace {
 
+class BitFieldInsertPattern
+    : public SPIRVToLLVMConversion<spirv::BitFieldInsertOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::BitFieldInsertOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::BitFieldInsertOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto srcType = op.getType();
+    auto dstType = this->typeConverter.convertType(srcType);
+    if (!dstType)
+      return failure();
+    Location loc = op.getLoc();
+
+    // Process `Offset` and `Count`: broadcast and extend/truncate if needed.
+    Value offset = processCountOrOffset(loc, op.offset(), srcType, dstType,
+                                        typeConverter, rewriter);
+    Value count = processCountOrOffset(loc, op.count(), srcType, dstType,
+                                       typeConverter, rewriter);
+
+    // Create a mask with bits set outside [Offset, Offset + Count - 1].
+    Value minusOne = createConstantAllBitsSet(loc, srcType, dstType, rewriter);
+    Value maskShiftedByCount =
+        rewriter.create<LLVM::ShlOp>(loc, dstType, minusOne, count);
+    Value negated = rewriter.create<LLVM::XOrOp>(loc, dstType,
+                                                 maskShiftedByCount, minusOne);
+    Value maskShiftedByCountAndOffset =
+        rewriter.create<LLVM::ShlOp>(loc, dstType, negated, offset);
+    Value mask = rewriter.create<LLVM::XOrOp>(
+        loc, dstType, maskShiftedByCountAndOffset, minusOne);
+
+    // Extract unchanged bits from the `Base`  that are outside of
+    // [Offset, Offset + Count - 1]. Then `or` with shifted `Insert`.
+    Value baseAndMask =
+        rewriter.create<LLVM::AndOp>(loc, dstType, op.base(), mask);
+    Value insertShiftedByOffset =
+        rewriter.create<LLVM::ShlOp>(loc, dstType, op.insert(), offset);
+    rewriter.replaceOpWithNewOp<LLVM::OrOp>(op, dstType, baseAndMask,
+                                            insertShiftedByOffset);
+    return success();
+  }
+};
+
+/// Converts SPIR-V ConstantOp with scalar or vector type.
+class ConstantScalarAndVectorPattern
+    : public SPIRVToLLVMConversion<spirv::ConstantOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::ConstantOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::ConstantOp constOp, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto srcType = constOp.getType();
+    if (!srcType.isa<VectorType>() && !srcType.isIntOrFloat())
+      return failure();
+
+    auto dstType = typeConverter.convertType(srcType);
+    if (!dstType)
+      return failure();
+
+    // SPIR-V constant can be a signed/unsigned integer, which has to be
+    // casted to signless integer when converting to LLVM dialect. Removing the
+    // sign bit may have unexpected behaviour. However, it is better to handle
+    // it case-by-case, given that the purpose of the conversion is not to
+    // cover all possible corner cases.
+    if (isSignedIntegerOrVector(srcType) ||
+        isUnsignedIntegerOrVector(srcType)) {
+      auto *context = rewriter.getContext();
+      auto signlessType = IntegerType::get(getBitWidth(srcType), context);
+
+      if (srcType.isa<VectorType>()) {
+        auto dstElementsAttr = constOp.value().cast<DenseIntElementsAttr>();
+        rewriter.replaceOpWithNewOp<LLVM::ConstantOp>(
+            constOp, dstType,
+            dstElementsAttr.mapValues(
+                signlessType, [&](const APInt &value) { return value; }));
+        return success();
+      }
+      auto srcAttr = constOp.value().cast<IntegerAttr>();
+      auto dstAttr = rewriter.getIntegerAttr(signlessType, srcAttr.getValue());
+      rewriter.replaceOpWithNewOp<LLVM::ConstantOp>(constOp, dstType, dstAttr);
+      return success();
+    }
+    rewriter.replaceOpWithNewOp<LLVM::ConstantOp>(constOp, dstType, operands,
+                                                  constOp.getAttrs());
+    return success();
+  }
+};
+
+class BitFieldSExtractPattern
+    : public SPIRVToLLVMConversion<spirv::BitFieldSExtractOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::BitFieldSExtractOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::BitFieldSExtractOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto srcType = op.getType();
+    auto dstType = this->typeConverter.convertType(srcType);
+    if (!dstType)
+      return failure();
+    Location loc = op.getLoc();
+
+    // Process `Offset` and `Count`: broadcast and extend/truncate if needed.
+    Value offset = processCountOrOffset(loc, op.offset(), srcType, dstType,
+                                        typeConverter, rewriter);
+    Value count = processCountOrOffset(loc, op.count(), srcType, dstType,
+                                       typeConverter, rewriter);
+
+    // Create a constant that holds the size of the `Base`.
+    IntegerType integerType;
+    if (auto vecType = srcType.dyn_cast<VectorType>())
+      integerType = vecType.getElementType().cast<IntegerType>();
+    else
+      integerType = srcType.cast<IntegerType>();
+
+    auto baseSize = rewriter.getIntegerAttr(integerType, getBitWidth(srcType));
+    Value size =
+        srcType.isa<VectorType>()
+            ? rewriter.create<LLVM::ConstantOp>(
+                  loc, dstType,
+                  SplatElementsAttr::get(srcType.cast<ShapedType>(), baseSize))
+            : rewriter.create<LLVM::ConstantOp>(loc, dstType, baseSize);
+
+    // Shift `Base` left by [sizeof(Base) - (Count + Offset)], so that the bit
+    // at Offset + Count - 1 is the most significant bit now.
+    Value countPlusOffset =
+        rewriter.create<LLVM::AddOp>(loc, dstType, count, offset);
+    Value amountToShiftLeft =
+        rewriter.create<LLVM::SubOp>(loc, dstType, size, countPlusOffset);
+    Value baseShiftedLeft = rewriter.create<LLVM::ShlOp>(
+        loc, dstType, op.base(), amountToShiftLeft);
+
+    // Shift the result right, filling the bits with the sign bit.
+    Value amountToShiftRight =
+        rewriter.create<LLVM::AddOp>(loc, dstType, offset, amountToShiftLeft);
+    rewriter.replaceOpWithNewOp<LLVM::AShrOp>(op, dstType, baseShiftedLeft,
+                                              amountToShiftRight);
+    return success();
+  }
+};
+
+class BitFieldUExtractPattern
+    : public SPIRVToLLVMConversion<spirv::BitFieldUExtractOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::BitFieldUExtractOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::BitFieldUExtractOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto srcType = op.getType();
+    auto dstType = this->typeConverter.convertType(srcType);
+    if (!dstType)
+      return failure();
+    Location loc = op.getLoc();
+
+    // Process `Offset` and `Count`: broadcast and extend/truncate if needed.
+    Value offset = processCountOrOffset(loc, op.offset(), srcType, dstType,
+                                        typeConverter, rewriter);
+    Value count = processCountOrOffset(loc, op.count(), srcType, dstType,
+                                       typeConverter, rewriter);
+
+    // Create a mask with bits set at [0, Count - 1].
+    Value minusOne = createConstantAllBitsSet(loc, srcType, dstType, rewriter);
+    Value maskShiftedByCount =
+        rewriter.create<LLVM::ShlOp>(loc, dstType, minusOne, count);
+    Value mask = rewriter.create<LLVM::XOrOp>(loc, dstType, maskShiftedByCount,
+                                              minusOne);
+
+    // Shift `Base` by `Offset` and apply the mask on it.
+    Value shiftedBase =
+        rewriter.create<LLVM::LShrOp>(loc, dstType, op.base(), offset);
+    rewriter.replaceOpWithNewOp<LLVM::AndOp>(op, dstType, shiftedBase, mask);
+    return success();
+  }
+};
+
 /// Converts SPIR-V operations that have straightforward LLVM equivalent
 /// into LLVM dialect operations.
 template <typename SPIRVOp, typename LLVMOp>
@@ -69,7 +420,8 @@ class DirectConversionPattern : public SPIRVToLLVMConversion<SPIRVOp> {
     auto dstType = this->typeConverter.convertType(operation.getType());
     if (!dstType)
       return failure();
-    rewriter.template replaceOpWithNewOp<LLVMOp>(operation, dstType, operands);
+    rewriter.template replaceOpWithNewOp<LLVMOp>(operation, dstType, operands,
+                                                 operation.getAttrs());
     return success();
   }
 };
@@ -106,6 +458,28 @@ class IndirectCastPattern : public SPIRVToLLVMConversion<SPIRVOp> {
   }
 };
 
+class FunctionCallPattern
+    : public SPIRVToLLVMConversion<spirv::FunctionCallOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::FunctionCallOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::FunctionCallOp callOp, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    if (callOp.getNumResults() == 0) {
+      rewriter.replaceOpWithNewOp<LLVM::CallOp>(callOp, llvm::None, operands,
+                                                callOp.getAttrs());
+      return success();
+    }
+
+    // Function returns a single result.
+    auto dstType = this->typeConverter.convertType(callOp.getType(0));
+    rewriter.replaceOpWithNewOp<LLVM::CallOp>(callOp, dstType, operands,
+                                              callOp.getAttrs());
+    return success();
+  }
+};
+
 /// Converts SPIR-V floating-point comparisons to llvm.fcmp "predicate"
 template <typename SPIRVOp, LLVM::FCmpPredicate predicate>
 class FComparePattern : public SPIRVToLLVMConversion<SPIRVOp> {
@@ -150,6 +524,61 @@ class IComparePattern : public SPIRVToLLVMConversion<SPIRVOp> {
   }
 };
 
+/// Converts `spv.Not` and `spv.LogicalNot` into LLVM dialect.
+template <typename SPIRVOp>
+class NotPattern : public SPIRVToLLVMConversion<SPIRVOp> {
+public:
+  using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(SPIRVOp notOp, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+
+    auto srcType = notOp.getType();
+    auto dstType = this->typeConverter.convertType(srcType);
+    if (!dstType)
+      return failure();
+
+    Location loc = notOp.getLoc();
+    IntegerAttr minusOne = minusOneIntegerAttribute(srcType, rewriter);
+    auto mask = srcType.template isa<VectorType>()
+                    ? rewriter.create<LLVM::ConstantOp>(
+                          loc, dstType,
+                          SplatElementsAttr::get(
+                              srcType.template cast<VectorType>(), minusOne))
+                    : rewriter.create<LLVM::ConstantOp>(loc, dstType, minusOne);
+    rewriter.template replaceOpWithNewOp<LLVM::XOrOp>(notOp, dstType,
+                                                      notOp.operand(), mask);
+    return success();
+  }
+};
+
+class ReturnPattern : public SPIRVToLLVMConversion<spirv::ReturnOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::ReturnOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::ReturnOp returnOp, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(returnOp, ArrayRef<Type>(),
+                                                ArrayRef<Value>());
+    return success();
+  }
+};
+
+class ReturnValuePattern : public SPIRVToLLVMConversion<spirv::ReturnValueOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::ReturnValueOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::ReturnValueOp returnValueOp, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(returnValueOp, ArrayRef<Type>(),
+                                                operands);
+    return success();
+  }
+};
+
 /// Converts SPIR-V shift ops to LLVM shift ops. Since LLVM dialect
 /// puts a restriction on `Shift` and `Base` to have the same bit width,
 /// `Shift` is zero or sign extended to match this specification. Cases when
@@ -191,12 +620,124 @@ class ShiftPattern : public SPIRVToLLVMConversion<SPIRVOp> {
     return success();
   }
 };
+
+//===----------------------------------------------------------------------===//
+// FuncOp conversion
+//===----------------------------------------------------------------------===//
+
+class FuncConversionPattern : public SPIRVToLLVMConversion<spirv::FuncOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::FuncOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::FuncOp funcOp, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+
+    // Convert function signature. At the moment LLVMType converter is enough
+    // for currently supported types.
+    auto funcType = funcOp.getType();
+    TypeConverter::SignatureConversion signatureConverter(
+        funcType.getNumInputs());
+    auto llvmType = this->typeConverter.convertFunctionSignature(
+        funcOp.getType(), /*isVariadic=*/false, signatureConverter);
+    if (!llvmType)
+      return failure();
+
+    // Create a new `LLVMFuncOp`
+    Location loc = funcOp.getLoc();
+    StringRef name = funcOp.getName();
+    auto newFuncOp = rewriter.create<LLVM::LLVMFuncOp>(loc, name, llvmType);
+
+    // Convert SPIR-V Function Control to equivalent LLVM function attribute
+    MLIRContext *context = funcOp.getContext();
+    switch (funcOp.function_control()) {
+#define DISPATCH(functionControl, llvmAttr)                                    \
+  case functionControl:                                                        \
+    newFuncOp.setAttr("passthrough", ArrayAttr::get({llvmAttr}, context));     \
+    break;
+
+          DISPATCH(spirv::FunctionControl::Inline,
+                   StringAttr::get("alwaysinline", context));
+          DISPATCH(spirv::FunctionControl::DontInline,
+                   StringAttr::get("noinline", context));
+          DISPATCH(spirv::FunctionControl::Pure,
+                   StringAttr::get("readonly", context));
+          DISPATCH(spirv::FunctionControl::Const,
+                   StringAttr::get("readnone", context));
+
+#undef DISPATCH
+
+    // Default: if `spirv::FunctionControl::None`, then no attributes are
+    // needed.
+    default:
+      break;
+    }
+
+    rewriter.inlineRegionBefore(funcOp.getBody(), newFuncOp.getBody(),
+                                newFuncOp.end());
+    rewriter.applySignatureConversion(&newFuncOp.getBody(), signatureConverter);
+    rewriter.eraseOp(funcOp);
+    return success();
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// ModuleOp conversion
+//===----------------------------------------------------------------------===//
+
+class ModuleConversionPattern : public SPIRVToLLVMConversion<spirv::ModuleOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::ModuleOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::ModuleOp spvModuleOp, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+
+    auto newModuleOp = rewriter.create<ModuleOp>(spvModuleOp.getLoc());
+    rewriter.inlineRegionBefore(spvModuleOp.body(), newModuleOp.getBody());
+
+    // Remove the terminator block that was automatically added by builder
+    rewriter.eraseBlock(&newModuleOp.getBodyRegion().back());
+    rewriter.eraseOp(spvModuleOp);
+    return success();
+  }
+};
+
+class ModuleEndConversionPattern
+    : public SPIRVToLLVMConversion<spirv::ModuleEndOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::ModuleEndOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::ModuleEndOp moduleEndOp, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+
+    rewriter.replaceOpWithNewOp<ModuleTerminatorOp>(moduleEndOp);
+    return success();
+  }
+};
+
 } // namespace
 
 //===----------------------------------------------------------------------===//
 // Pattern population
 //===----------------------------------------------------------------------===//
 
+void mlir::populateSPIRVToLLVMTypeConversion(LLVMTypeConverter &typeConverter) {
+  typeConverter.addConversion([&](spirv::ArrayType type) {
+    return convertArrayType(type, typeConverter);
+  });
+  typeConverter.addConversion([&](spirv::PointerType type) {
+    return convertPointerType(type, typeConverter);
+  });
+  typeConverter.addConversion([&](spirv::RuntimeArrayType type) {
+    return convertRuntimeArrayType(type, typeConverter);
+  });
+  typeConverter.addConversion([&](spirv::StructType type) {
+    return convertStructType(type, typeConverter);
+  });
+}
+
 void mlir::populateSPIRVToLLVMConversionPatterns(
     MLIRContext *context, LLVMTypeConverter &typeConverter,
     OwningRewritePatternList &patterns) {
@@ -207,6 +748,7 @@ void mlir::populateSPIRVToLLVMConversionPatterns(
       DirectConversionPattern<spirv::ISubOp, LLVM::SubOp>,
       DirectConversionPattern<spirv::FAddOp, LLVM::FAddOp>,
       DirectConversionPattern<spirv::FDivOp, LLVM::FDivOp>,
+      DirectConversionPattern<spirv::FMulOp, LLVM::FMulOp>,
       DirectConversionPattern<spirv::FNegateOp, LLVM::FNegOp>,
       DirectConversionPattern<spirv::FRemOp, LLVM::FRemOp>,
       DirectConversionPattern<spirv::FSubOp, LLVM::FSubOp>,
@@ -216,11 +758,16 @@ void mlir::populateSPIRVToLLVMConversionPatterns(
       DirectConversionPattern<spirv::UModOp, LLVM::URemOp>,
 
       // Bitwise ops
+      BitFieldInsertPattern, BitFieldUExtractPattern, BitFieldSExtractPattern,
+      DirectConversionPattern<spirv::BitCountOp, LLVM::CtPopOp>,
+      DirectConversionPattern<spirv::BitReverseOp, LLVM::BitReverseOp>,
       DirectConversionPattern<spirv::BitwiseAndOp, LLVM::AndOp>,
       DirectConversionPattern<spirv::BitwiseOrOp, LLVM::OrOp>,
       DirectConversionPattern<spirv::BitwiseXorOp, LLVM::XOrOp>,
+      NotPattern<spirv::NotOp>,
 
       // Cast ops
+      DirectConversionPattern<spirv::BitcastOp, LLVM::BitcastOp>,
       DirectConversionPattern<spirv::ConvertFToSOp, LLVM::FPToSIOp>,
       DirectConversionPattern<spirv::ConvertFToUOp, LLVM::FPToUIOp>,
       DirectConversionPattern<spirv::ConvertSToFOp, LLVM::SIToFPOp>,
@@ -254,15 +801,41 @@ void mlir::populateSPIRVToLLVMConversionPatterns(
       IComparePattern<spirv::ULessThanEqualOp, LLVM::ICmpPredicate::ule>,
       IComparePattern<spirv::ULessThanOp, LLVM::ICmpPredicate::ult>,
 
+      // Constant op
+      ConstantScalarAndVectorPattern,
+
+      // Function Call op
+      FunctionCallPattern,
+
       // Logical ops
       DirectConversionPattern<spirv::LogicalAndOp, LLVM::AndOp>,
       DirectConversionPattern<spirv::LogicalOrOp, LLVM::OrOp>,
       IComparePattern<spirv::LogicalEqualOp, LLVM::ICmpPredicate::eq>,
       IComparePattern<spirv::LogicalNotEqualOp, LLVM::ICmpPredicate::ne>,
+      NotPattern<spirv::LogicalNotOp>,
+
+      // Miscellaneous ops
+      DirectConversionPattern<spirv::SelectOp, LLVM::SelectOp>,
+      DirectConversionPattern<spirv::UndefOp, LLVM::UndefOp>,
 
       // Shift ops
       ShiftPattern<spirv::ShiftRightArithmeticOp, LLVM::AShrOp>,
       ShiftPattern<spirv::ShiftRightLogicalOp, LLVM::LShrOp>,
-      ShiftPattern<spirv::ShiftLeftLogicalOp, LLVM::ShlOp>>(context,
-                                                            typeConverter);
+      ShiftPattern<spirv::ShiftLeftLogicalOp, LLVM::ShlOp>,
+
+      // Return ops
+      ReturnPattern, ReturnValuePattern>(context, typeConverter);
+}
+
+void mlir::populateSPIRVToLLVMFunctionConversionPatterns(
+    MLIRContext *context, LLVMTypeConverter &typeConverter,
+    OwningRewritePatternList &patterns) {
+  patterns.insert<FuncConversionPattern>(context, typeConverter);
+}
+
+void mlir::populateSPIRVToLLVMModuleConversionPatterns(
+    MLIRContext *context, LLVMTypeConverter &typeConverter,
+    OwningRewritePatternList &patterns) {
+  patterns.insert<ModuleConversionPattern, ModuleEndConversionPattern>(
+      context, typeConverter);
 }
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVMPass.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVMPass.cpp
index 8f300541a71d3..1a806d04d9fc0 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVMPass.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVMPass.cpp
@@ -34,7 +34,12 @@ void ConvertSPIRVToLLVMPass::runOnOperation() {
   LLVMTypeConverter converter(&getContext());
 
   OwningRewritePatternList patterns;
+
+  populateSPIRVToLLVMTypeConversion(converter);
+
+  populateSPIRVToLLVMModuleConversionPatterns(context, converter, patterns);
   populateSPIRVToLLVMConversionPatterns(context, converter, patterns);
+  populateSPIRVToLLVMFunctionConversionPatterns(context, converter, patterns);
 
   // Currently pulls in Std to LLVM conversion patterns
   // that help with testing. This allows to convert
@@ -44,6 +49,11 @@ void ConvertSPIRVToLLVMPass::runOnOperation() {
   ConversionTarget target(getContext());
   target.addIllegalDialect<spirv::SPIRVDialect>();
   target.addLegalDialect<LLVM::LLVMDialect>();
+
+  // set `ModuleOp` and `ModuleTerminatorOp` as legal for `spv.module`
+  // conversion.
+  target.addLegalOp<ModuleOp>();
+  target.addLegalOp<ModuleTerminatorOp>();
   if (failed(applyPartialConversion(module, target, patterns)))
     signalPassFailure();
 }
diff --git a/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp b/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp
index db7796d5c6a00..55ebae99af53f 100644
--- a/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp
+++ b/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp
@@ -69,6 +69,58 @@ ReduceOpConverter::matchAndRewrite(ReduceOp reduceOp,
   return success();
 }
 
+namespace {
+/// Converts `shape_of` to for loop for unranked tensors.
+class ShapeOfOpConverter : public OpConversionPattern<ShapeOfOp> {
+public:
+  using OpConversionPattern<ShapeOfOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(ShapeOfOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override;
+};
+} // namespace
+
+LogicalResult
+ShapeOfOpConverter::matchAndRewrite(ShapeOfOp op, ArrayRef<Value> operands,
+                                    ConversionPatternRewriter &rewriter) const {
+  ShapeOfOp::Adaptor transformed(operands);
+  auto tensorVal = transformed.arg();
+  auto tensorTy = tensorVal.getType();
+
+  // For ranked tensors `shape_of` lowers to `std` and the pattern can be
+  // found in the corresponding pass.
+  if (tensorTy.isa<RankedTensorType>())
+    return failure();
+
+  // Allocate stack memory.
+  auto loc = op.getLoc();
+  auto rankVal = rewriter.create<mlir::RankOp>(loc, tensorVal);
+  auto i64Ty = rewriter.getI64Type();
+  auto memTy = MemRefType::get({ShapedType::kDynamicSize}, i64Ty);
+  auto memVal = rewriter.create<AllocaOp>(loc, memTy, ValueRange({rankVal}));
+
+  // Copy shape extents to stack-allocated memory.
+  auto zeroVal = rewriter.create<ConstantIndexOp>(loc, 0);
+  auto oneVal = rewriter.create<ConstantIndexOp>(loc, 1);
+  rewriter.create<scf::ForOp>(
+      loc, zeroVal, rankVal, oneVal, ValueRange(),
+      [&](OpBuilder &b, Location loc, Value iVal, ValueRange args) {
+        auto dimVal = b.create<DimOp>(loc, tensorVal, iVal);
+        auto dimIntVal = b.create<IndexCastOp>(loc, dimVal, i64Ty);
+        b.create<StoreOp>(loc, dimIntVal, memVal, ValueRange({iVal}));
+        b.create<scf::YieldOp>(loc);
+      });
+
+  // Load extents to tensor value.
+  auto shapeIntVal = rewriter.create<TensorLoadOp>(loc, memVal);
+  auto indexTy = rewriter.getIndexType();
+  auto shapeTy = RankedTensorType::get({ShapedType::kDynamicSize}, indexTy);
+  rewriter.replaceOpWithNewOp<IndexCastOp>(op.getOperation(), shapeIntVal,
+                                           shapeTy);
+  return success();
+}
+
 namespace {
 struct ConvertShapeToSCFPass
     : public ConvertShapeToSCFBase<ConvertShapeToSCFPass> {
@@ -79,19 +131,23 @@ struct ConvertShapeToSCFPass
 void ConvertShapeToSCFPass::runOnFunction() {
   MLIRContext &ctx = getContext();
 
+  // Populate conversion patterns.
   OwningRewritePatternList patterns;
   populateShapeToSCFConversionPatterns(patterns, &ctx);
 
+  // Setup target legality.
   ConversionTarget target(getContext());
   target.addLegalDialect<ShapeDialect, scf::SCFDialect, StandardOpsDialect>();
-  target.addIllegalOp<ReduceOp>();
-  if (failed(mlir::applyPartialConversion(getFunction(), target, patterns)))
+  target.addIllegalOp<ReduceOp, ShapeOfOp>();
+
+  // Apply conversion.
+  if (failed(applyPartialConversion(getFunction(), target, patterns)))
     signalPassFailure();
 }
 
 void mlir::populateShapeToSCFConversionPatterns(
     OwningRewritePatternList &patterns, MLIRContext *ctx) {
-  patterns.insert<ReduceOpConverter>(ctx);
+  patterns.insert<ReduceOpConverter, ShapeOfOpConverter>(ctx);
 }
 
 std::unique_ptr<FunctionPass> mlir::createConvertShapeToSCFPass() {
diff --git a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
index 6a02bdc2c286e..7ebcb397349d4 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
@@ -90,6 +90,43 @@ class ConstSizeOpConverter : public OpConversionPattern<ConstSizeOp> {
   }
 };
 
+class GetExtentOpConverter : public OpConversionPattern<GetExtentOp> {
+  using OpConversionPattern<GetExtentOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(GetExtentOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    GetExtentOp::Adaptor transformed(operands);
+
+    // Derive shape extent directly from shape origin if possible.
+    // This circumvents the necessity to materialize the shape in memory.
+    if (auto shapeOfOp = op.shape().getDefiningOp<ShapeOfOp>()) {
+      rewriter.replaceOpWithNewOp<DimOp>(op, shapeOfOp.arg(),
+                                         transformed.dim());
+      return success();
+    }
+
+    rewriter.replaceOpWithNewOp<ExtractElementOp>(
+        op, rewriter.getIndexType(), transformed.shape(),
+        ValueRange{transformed.dim()});
+    return success();
+  }
+};
+
+class RankOpConverter : public OpConversionPattern<shape::RankOp> {
+public:
+  using OpConversionPattern<shape::RankOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(shape::RankOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    shape::RankOp::Adaptor transformed(operands);
+    rewriter.replaceOpWithNewOp<DimOp>(op.getOperation(), transformed.shape(),
+                                       0);
+    return success();
+  }
+};
+
 /// Type conversions.
 class ShapeTypeConverter : public TypeConverter {
 public:
@@ -147,6 +184,8 @@ void mlir::populateShapeToStandardConversionPatterns(
       BinaryOpConversion<AddOp, AddIOp>,
       BinaryOpConversion<MulOp, MulIOp>,
       ConstSizeOpConverter,
+      GetExtentOpConverter,
+      RankOpConverter,
       ShapeOfOpConversion>(ctx);
   // clang-format on
 }
diff --git a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
index 19c451fa3fe9c..91a4867ad3075 100644
--- a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
+++ b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
@@ -24,6 +24,7 @@
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Support/LogicalResult.h"
+#include "mlir/Support/MathExtras.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/Passes.h"
 #include "mlir/Transforms/Utils.h"
@@ -51,11 +52,6 @@ static LLVM::LLVMType unwrap(Type type) {
   return wrappedLLVMType;
 }
 
-/// Initialize customization to default callbacks.
-LLVMTypeConverterCustomization::LLVMTypeConverterCustomization()
-    : funcArgConverter(structFuncArgTypeConverter),
-      indexBitwidth(kDeriveIndexBitwidthFromDataLayout) {}
-
 /// Callback to convert function argument types. It converts a MemRef function
 /// argument to a list of non-aggregate types containing descriptor
 /// information, and an UnrankedmemRef function argument to a list containing
@@ -122,19 +118,19 @@ LogicalResult mlir::barePtrFuncArgTypeConverter(LLVMTypeConverter &converter,
   return success();
 }
 
-/// Create an LLVMTypeConverter using default LLVMTypeConverterCustomization.
+/// Create an LLVMTypeConverter using default LowerToLLVMOptions.
 LLVMTypeConverter::LLVMTypeConverter(MLIRContext *ctx)
-    : LLVMTypeConverter(ctx, LLVMTypeConverterCustomization()) {}
+    : LLVMTypeConverter(ctx, LowerToLLVMOptions::getDefaultOptions()) {}
 
-/// Create an LLVMTypeConverter using 'custom' customizations.
-LLVMTypeConverter::LLVMTypeConverter(
-    MLIRContext *ctx, const LLVMTypeConverterCustomization &customs)
+/// Create an LLVMTypeConverter using custom LowerToLLVMOptions.
+LLVMTypeConverter::LLVMTypeConverter(MLIRContext *ctx,
+                                     const LowerToLLVMOptions &options)
     : llvmDialect(ctx->getRegisteredDialect<LLVM::LLVMDialect>()),
-      customizations(customs) {
+      options(options) {
   assert(llvmDialect && "LLVM IR dialect is not registered");
   module = &llvmDialect->getLLVMModule();
-  if (customizations.indexBitwidth == kDeriveIndexBitwidthFromDataLayout)
-    customizations.indexBitwidth =
+  if (options.indexBitwidth == kDeriveIndexBitwidthFromDataLayout)
+    this->options.indexBitwidth =
         module->getDataLayout().getPointerSizeInBits();
 
   // Register conversions for the standard types.
@@ -184,6 +180,10 @@ LLVM::LLVMType LLVMTypeConverter::getIndexType() {
   return LLVM::LLVMType::getIntNTy(llvmDialect, getIndexTypeBitwidth());
 }
 
+unsigned LLVMTypeConverter::getPointerBitwidth(unsigned addressSpace) {
+  return module->getDataLayout().getPointerSizeInBits(addressSpace);
+}
+
 Type LLVMTypeConverter::convertIndexType(IndexType type) {
   return getIndexType();
 }
@@ -262,11 +262,15 @@ SmallVector<Type, 2> LLVMTypeConverter::convertUnrankedMemRefSignature() {
 LLVM::LLVMType LLVMTypeConverter::convertFunctionSignature(
     FunctionType type, bool isVariadic,
     LLVMTypeConverter::SignatureConversion &result) {
+  // Select the argument converter depending on the calling convetion.
+  auto funcArgConverter = options.useBarePtrCallConv
+                              ? barePtrFuncArgTypeConverter
+                              : structFuncArgTypeConverter;
   // Convert argument types one by one and check for errors.
   for (auto &en : llvm::enumerate(type.getInputs())) {
     Type type = en.value();
     SmallVector<Type, 8> converted;
-    if (failed(customizations.funcArgConverter(*this, type, converted)))
+    if (failed(funcArgConverter(*this, type, converted)))
       return {};
     result.addInputs(en.index(), converted);
   }
@@ -298,7 +302,7 @@ LLVMTypeConverter::convertFunctionTypeCWrapper(FunctionType type) {
     auto converted = convertType(t).dyn_cast_or_null<LLVM::LLVMType>();
     if (!converted)
       return {};
-    if (t.isa<MemRefType>() || t.isa<UnrankedMemRefType>())
+    if (t.isa<MemRefType, UnrankedMemRefType>())
       converted = converted.getPointerTo();
     inputs.push_back(converted);
   }
@@ -327,7 +331,7 @@ LLVMTypeConverter::convertFunctionTypeCWrapper(FunctionType type) {
 //   4. a second array containing as many `index`-type integers as the rank of
 //   the MemRef: the second array represents the "stride" (in tensor abstraction
 //   sense), i.e. the number of consecutive elements of the underlying buffer.
-//   TODO(ntv, zinenko): add assertions for the static cases.
+//   TODO: add assertions for the static cases.
 //
 // template <typename Elem, size_t Rank>
 // struct {
@@ -396,10 +400,11 @@ Type LLVMTypeConverter::convertVectorType(VectorType type) {
 
 ConvertToLLVMPattern::ConvertToLLVMPattern(StringRef rootOpName,
                                            MLIRContext *context,
-                                           LLVMTypeConverter &typeConverter_,
+                                           LLVMTypeConverter &typeConverter,
+                                           const LowerToLLVMOptions &options,
                                            PatternBenefit benefit)
-    : ConversionPattern(rootOpName, benefit, typeConverter_, context),
-      typeConverter(typeConverter_) {}
+    : ConversionPattern(rootOpName, benefit, typeConverter, context),
+      typeConverter(typeConverter), options(options) {}
 
 /*============================================================================*/
 /* StructBuilder implementation                                               */
@@ -769,6 +774,51 @@ void UnrankedMemRefDescriptor::unpack(OpBuilder &builder, Location loc,
   results.push_back(d.memRefDescPtr(builder, loc));
 }
 
+void UnrankedMemRefDescriptor::computeSizes(
+    OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter,
+    ArrayRef<UnrankedMemRefDescriptor> values, SmallVectorImpl<Value> &sizes) {
+  if (values.empty())
+    return;
+
+  // Cache the index type.
+  LLVM::LLVMType indexType = typeConverter.getIndexType();
+
+  // Initialize shared constants.
+  Value one = createIndexAttrConstant(builder, loc, indexType, 1);
+  Value two = createIndexAttrConstant(builder, loc, indexType, 2);
+  Value pointerSize = createIndexAttrConstant(
+      builder, loc, indexType, ceilDiv(typeConverter.getPointerBitwidth(), 8));
+  Value indexSize =
+      createIndexAttrConstant(builder, loc, indexType,
+                              ceilDiv(typeConverter.getIndexTypeBitwidth(), 8));
+
+  sizes.reserve(sizes.size() + values.size());
+  for (UnrankedMemRefDescriptor desc : values) {
+    // Emit IR computing the memory necessary to store the descriptor. This
+    // assumes the descriptor to be
+    //   { type*, type*, index, index[rank], index[rank] }
+    // and densely packed, so the total size is
+    //   2 * sizeof(pointer) + (1 + 2 * rank) * sizeof(index).
+    // TODO: consider including the actual size (including eventual padding due
+    // to data layout) into the unranked descriptor.
+    Value doublePointerSize =
+        builder.create<LLVM::MulOp>(loc, indexType, two, pointerSize);
+
+    // (1 + 2 * rank) * sizeof(index)
+    Value rank = desc.rank(builder, loc);
+    Value doubleRank = builder.create<LLVM::MulOp>(loc, indexType, two, rank);
+    Value doubleRankIncremented =
+        builder.create<LLVM::AddOp>(loc, indexType, doubleRank, one);
+    Value rankIndexSize = builder.create<LLVM::MulOp>(
+        loc, indexType, doubleRankIncremented, indexSize);
+
+    // Total allocation size.
+    Value allocationSize = builder.create<LLVM::AddOp>(
+        loc, indexType, doublePointerSize, rankIndexSize);
+    sizes.push_back(allocationSize);
+  }
+}
+
 LLVM::LLVMDialect &ConvertToLLVMPattern::getDialect() const {
   return *typeConverter.getDialect();
 }
@@ -994,7 +1044,7 @@ struct FuncOpConversionBase : public ConvertOpToLLVMPattern<FuncOp> {
       FunctionType type, SmallVectorImpl<UnsignedTypePair> &argsInfo) const {
     argsInfo.reserve(type.getNumInputs());
     for (auto en : llvm::enumerate(type.getInputs())) {
-      if (en.value().isa<MemRefType>() || en.value().isa<UnrankedMemRefType>())
+      if (en.value().isa<MemRefType, UnrankedMemRefType>())
         argsInfo.push_back({en.index(), en.value()});
     }
   }
@@ -1051,8 +1101,10 @@ struct FuncOpConversionBase : public ConvertOpToLLVMPattern<FuncOp> {
 /// information.
 static constexpr StringRef kEmitIfaceAttrName = "llvm.emit_c_interface";
 struct FuncOpConversion : public FuncOpConversionBase {
-  FuncOpConversion(LLVMTypeConverter &converter, bool emitCWrappers)
-      : FuncOpConversionBase(converter), emitWrappers(emitCWrappers) {}
+  FuncOpConversion(LLVMTypeConverter &converter,
+                   const LowerToLLVMOptions &options)
+      : FuncOpConversionBase(converter, options) {}
+  using ConvertOpToLLVMPattern<FuncOp>::options;
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -1063,7 +1115,8 @@ struct FuncOpConversion : public FuncOpConversionBase {
     if (!newFuncOp)
       return failure();
 
-    if (emitWrappers || funcOp.getAttrOfType<UnitAttr>(kEmitIfaceAttrName)) {
+    if (options.emitCWrappers ||
+        funcOp.getAttrOfType<UnitAttr>(kEmitIfaceAttrName)) {
       if (newFuncOp.isExternal())
         wrapExternalFunction(rewriter, op->getLoc(), typeConverter, funcOp,
                              newFuncOp);
@@ -1075,11 +1128,6 @@ struct FuncOpConversion : public FuncOpConversionBase {
     rewriter.eraseOp(op);
     return success();
   }
-
-private:
-  /// If true, also create the adaptor functions having signatures compatible
-  /// with those produced by clang.
-  const bool emitWrappers;
 };
 
 /// FuncOp legalization pattern that converts MemRef arguments to bare pointers
@@ -1316,8 +1364,6 @@ using AddFOpLowering = VectorConvertToLLVMPattern<AddFOp, LLVM::FAddOp>;
 using AddIOpLowering = VectorConvertToLLVMPattern<AddIOp, LLVM::AddOp>;
 using AndOpLowering = VectorConvertToLLVMPattern<AndOp, LLVM::AndOp>;
 using CeilFOpLowering = VectorConvertToLLVMPattern<CeilFOp, LLVM::FCeilOp>;
-using ConstLLVMOpLowering =
-    OneToOneConvertToLLVMPattern<ConstantOp, LLVM::ConstantOp>;
 using CopySignOpLowering =
     VectorConvertToLLVMPattern<CopySignOp, LLVM::CopySignOp>;
 using CosOpLowering = VectorConvertToLLVMPattern<CosOp, LLVM::CosOp>;
@@ -1491,6 +1537,39 @@ struct SubCFOpLowering : public ConvertOpToLLVMPattern<SubCFOp> {
   }
 };
 
+struct ConstantOpLowering : public ConvertOpToLLVMPattern<ConstantOp> {
+  using ConvertOpToLLVMPattern<ConstantOp>::ConvertOpToLLVMPattern;
+
+  LogicalResult
+  matchAndRewrite(Operation *operation, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto op = cast<ConstantOp>(operation);
+    // If constant refers to a function, convert it to "addressof".
+    if (auto symbolRef = op.getValue().dyn_cast<FlatSymbolRefAttr>()) {
+      auto type = typeConverter.convertType(op.getResult().getType())
+                      .dyn_cast_or_null<LLVM::LLVMType>();
+      if (!type)
+        return rewriter.notifyMatchFailure(op, "failed to convert result type");
+
+      MutableDictionaryAttr attrs(op.getAttrs());
+      attrs.remove(rewriter.getIdentifier("value"));
+      rewriter.replaceOpWithNewOp<LLVM::AddressOfOp>(
+          op, type.cast<LLVM::LLVMType>(), symbolRef.getValue(),
+          attrs.getAttrs());
+      return success();
+    }
+
+    // Calling into other scopes (non-flat reference) is not supported in LLVM.
+    if (op.getValue().isa<SymbolRefAttr>())
+      return rewriter.notifyMatchFailure(
+          op, "referring to a symbol outside of the current module");
+
+    return LLVM::detail::oneToOneRewrite(op,
+                                         LLVM::ConstantOp::getOperationName(),
+                                         operands, typeConverter, rewriter);
+  }
+};
+
 // Check if the MemRefType `type` is supported by the lowering. We currently
 // only support memrefs with identity maps.
 static bool isSupportedMemRefType(MemRefType type) {
@@ -1506,11 +1585,11 @@ struct AllocLikeOpLowering : public ConvertOpToLLVMPattern<AllocLikeOp> {
   using ConvertOpToLLVMPattern<AllocLikeOp>::getIndexType;
   using ConvertOpToLLVMPattern<AllocLikeOp>::typeConverter;
   using ConvertOpToLLVMPattern<AllocLikeOp>::getVoidPtrType;
+  using ConvertOpToLLVMPattern<AllocLikeOp>::options;
 
   explicit AllocLikeOpLowering(LLVMTypeConverter &converter,
-                               bool useAlignedAlloc = false)
-      : ConvertOpToLLVMPattern<AllocLikeOp>(converter),
-        useAlignedAlloc(useAlignedAlloc) {}
+                               const LowerToLLVMOptions &options)
+      : ConvertOpToLLVMPattern<AllocLikeOp>(converter, options) {}
 
   LogicalResult match(Operation *op) const override {
     MemRefType memRefType = cast<AllocLikeOp>(op).getType();
@@ -1677,7 +1756,7 @@ struct AllocLikeOpLowering : public ConvertOpToLLVMPattern<AllocLikeOp> {
   /// allocation size to be a multiple of alignment,
   Optional<int64_t> getAllocationAlignment(AllocOp allocOp) const {
     // No alignment can be used for the 'malloc' call itself.
-    if (!useAlignedAlloc)
+    if (!options.useAlignedAlloc)
       return None;
 
     if (allocOp.alignment())
@@ -1735,12 +1814,14 @@ struct AllocLikeOpLowering : public ConvertOpToLLVMPattern<AllocLikeOp> {
     auto module = allocOp.getParentOfType<ModuleOp>();
     auto allocFunc = module.lookupSymbol<LLVM::LLVMFuncOp>(allocFuncName);
     if (!allocFunc) {
-      OpBuilder moduleBuilder(op->getParentOfType<ModuleOp>().getBodyRegion());
+      OpBuilder::InsertionGuard guard(rewriter);
+      rewriter.setInsertionPointToStart(
+          op->getParentOfType<ModuleOp>().getBody());
       SmallVector<LLVM::LLVMType, 2> callArgTypes = {getIndexType()};
       // aligned_alloc(size_t alignment, size_t size)
       if (useAlignedAlloc)
         callArgTypes.push_back(getIndexType());
-      allocFunc = moduleBuilder.create<LLVM::LLVMFuncOp>(
+      allocFunc = rewriter.create<LLVM::LLVMFuncOp>(
           rewriter.getUnknownLoc(), allocFuncName,
           LLVM::LLVMType::getFunctionTy(getVoidPtrType(), callArgTypes,
                                         /*isVarArg=*/false));
@@ -1849,20 +1930,114 @@ struct AllocLikeOpLowering : public ConvertOpToLLVMPattern<AllocLikeOp> {
   }
 
 protected:
-  /// Use aligned_alloc instead of malloc for all heap allocations.
-  bool useAlignedAlloc;
   /// The minimum alignment to use with aligned_alloc (has to be a power of 2).
   uint64_t kMinAlignedAllocAlignment = 16UL;
 };
 
 struct AllocOpLowering : public AllocLikeOpLowering<AllocOp> {
   explicit AllocOpLowering(LLVMTypeConverter &converter,
-                           bool useAlignedAlloc = false)
-      : AllocLikeOpLowering<AllocOp>(converter, useAlignedAlloc) {}
+                           const LowerToLLVMOptions &options)
+      : AllocLikeOpLowering<AllocOp>(converter, options) {}
 };
 
 using AllocaOpLowering = AllocLikeOpLowering<AllocaOp>;
 
+/// Copies the shaped descriptor part to (if `toDynamic` is set) or from
+/// (otherwise) the dynamically allocated memory for any operands that were
+/// unranked descriptors originally.
+static LogicalResult copyUnrankedDescriptors(OpBuilder &builder, Location loc,
+                                             LLVMTypeConverter &typeConverter,
+                                             TypeRange origTypes,
+                                             SmallVectorImpl<Value> &operands,
+                                             bool toDynamic) {
+  assert(origTypes.size() == operands.size() &&
+         "expected as may original types as operands");
+
+  // Find operands of unranked memref type and store them.
+  SmallVector<UnrankedMemRefDescriptor, 4> unrankedMemrefs;
+  for (unsigned i = 0, e = operands.size(); i < e; ++i)
+    if (origTypes[i].isa<UnrankedMemRefType>())
+      unrankedMemrefs.emplace_back(operands[i]);
+
+  if (unrankedMemrefs.empty())
+    return success();
+
+  // Compute allocation sizes.
+  SmallVector<Value, 4> sizes;
+  UnrankedMemRefDescriptor::computeSizes(builder, loc, typeConverter,
+                                         unrankedMemrefs, sizes);
+
+  // Get frequently used types.
+  auto voidType = LLVM::LLVMType::getVoidTy(typeConverter.getDialect());
+  auto voidPtrType = LLVM::LLVMType::getInt8PtrTy(typeConverter.getDialect());
+  auto i1Type = LLVM::LLVMType::getInt1Ty(typeConverter.getDialect());
+  LLVM::LLVMType indexType = typeConverter.getIndexType();
+
+  // Find the malloc and free, or declare them if necessary.
+  auto module = builder.getInsertionPoint()->getParentOfType<ModuleOp>();
+  auto mallocFunc = module.lookupSymbol<LLVM::LLVMFuncOp>("malloc");
+  if (!mallocFunc && toDynamic) {
+    OpBuilder::InsertionGuard guard(builder);
+    builder.setInsertionPointToStart(module.getBody());
+    mallocFunc = builder.create<LLVM::LLVMFuncOp>(
+        builder.getUnknownLoc(), "malloc",
+        LLVM::LLVMType::getFunctionTy(
+            voidPtrType, llvm::makeArrayRef(indexType), /*isVarArg=*/false));
+  }
+  auto freeFunc = module.lookupSymbol<LLVM::LLVMFuncOp>("free");
+  if (!freeFunc && !toDynamic) {
+    OpBuilder::InsertionGuard guard(builder);
+    builder.setInsertionPointToStart(module.getBody());
+    freeFunc = builder.create<LLVM::LLVMFuncOp>(
+        builder.getUnknownLoc(), "free",
+        LLVM::LLVMType::getFunctionTy(voidType, llvm::makeArrayRef(voidPtrType),
+                                      /*isVarArg=*/false));
+  }
+
+  // Initialize shared constants.
+  Value zero =
+      builder.create<LLVM::ConstantOp>(loc, i1Type, builder.getBoolAttr(false));
+
+  unsigned unrankedMemrefPos = 0;
+  for (unsigned i = 0, e = operands.size(); i < e; ++i) {
+    Type type = origTypes[i];
+    if (!type.isa<UnrankedMemRefType>())
+      continue;
+    Value allocationSize = sizes[unrankedMemrefPos++];
+    UnrankedMemRefDescriptor desc(operands[i]);
+
+    // Allocate memory, copy, and free the source if necessary.
+    Value memory =
+        toDynamic
+            ? builder.create<LLVM::CallOp>(loc, mallocFunc, allocationSize)
+                  .getResult(0)
+            : builder.create<LLVM::AllocaOp>(loc, voidPtrType, allocationSize,
+                                             /*alignment=*/0);
+
+    Value source = desc.memRefDescPtr(builder, loc);
+    builder.create<LLVM::MemcpyOp>(loc, memory, source, allocationSize, zero);
+    if (!toDynamic)
+      builder.create<LLVM::CallOp>(loc, freeFunc, source);
+
+    // Create a new descriptor. The same descriptor can be returned multiple
+    // times, attempting to modify its pointer can lead to memory leaks
+    // (allocated twice and overwritten) or double frees (the caller does not
+    // know if the descriptor points to the same memory).
+    Type descriptorType = typeConverter.convertType(type);
+    if (!descriptorType)
+      return failure();
+    auto updatedDesc =
+        UnrankedMemRefDescriptor::undef(builder, loc, descriptorType);
+    Value rank = desc.rank(builder, loc);
+    updatedDesc.setRank(builder, loc, rank);
+    updatedDesc.setMemRefDescPtr(builder, loc, memory);
+
+    operands[i] = updatedDesc;
+  }
+
+  return success();
+}
+
 // A CallOp automatically promotes MemRefType to a sequence of alloca/store and
 // passes the pointer to the MemRef across function boundaries.
 template <typename CallOpType>
@@ -1882,13 +2057,6 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
     unsigned numResults = callOp.getNumResults();
     auto resultTypes = llvm::to_vector<4>(callOp.getResultTypes());
 
-    for (Type resType : resultTypes) {
-      assert(!resType.isa<UnrankedMemRefType>() &&
-             "Returning unranked memref is not supported. Pass result as an"
-             "argument instead.");
-      (void)resType;
-    }
-
     if (numResults != 0) {
       if (!(packedResult =
                 this->typeConverter.packFunctionResults(resultTypes)))
@@ -1900,25 +2068,25 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
     auto newOp = rewriter.create<LLVM::CallOp>(op->getLoc(), packedResult,
                                                promoted, op->getAttrs());
 
-    // If < 2 results, packing did not do anything and we can just return.
-    if (numResults < 2) {
-      rewriter.replaceOp(op, newOp.getResults());
-      return success();
-    }
-
-    // Otherwise, it had been converted to an operation producing a structure.
-    // Extract individual results from the structure and return them as list.
-    // TODO(aminim, ntv, riverriddle, zinenko): this seems like patching around
-    // a particular interaction between MemRefType and CallOp lowering. Find a
-    // way to avoid special casing.
     SmallVector<Value, 4> results;
-    results.reserve(numResults);
-    for (unsigned i = 0; i < numResults; ++i) {
-      auto type = this->typeConverter.convertType(op->getResult(i).getType());
-      results.push_back(rewriter.create<LLVM::ExtractValueOp>(
-          op->getLoc(), type, newOp.getOperation()->getResult(0),
-          rewriter.getI64ArrayAttr(i)));
+    if (numResults < 2) {
+      // If < 2 results, packing did not do anything and we can just return.
+      results.append(newOp.result_begin(), newOp.result_end());
+    } else {
+      // Otherwise, it had been converted to an operation producing a structure.
+      // Extract individual results from the structure and return them as list.
+      results.reserve(numResults);
+      for (unsigned i = 0; i < numResults; ++i) {
+        auto type = this->typeConverter.convertType(op->getResult(i).getType());
+        results.push_back(rewriter.create<LLVM::ExtractValueOp>(
+            op->getLoc(), type, newOp.getOperation()->getResult(0),
+            rewriter.getI64ArrayAttr(i)));
+      }
     }
+    if (failed(copyUnrankedDescriptors(
+            rewriter, op->getLoc(), this->typeConverter, op->getResultTypes(),
+            results, /*toDynamic=*/false)))
+      return failure();
     rewriter.replaceOp(op, results);
 
     return success();
@@ -1939,8 +2107,9 @@ struct CallIndirectOpLowering : public CallOpInterfaceLowering<CallIndirectOp> {
 struct DeallocOpLowering : public ConvertOpToLLVMPattern<DeallocOp> {
   using ConvertOpToLLVMPattern<DeallocOp>::ConvertOpToLLVMPattern;
 
-  explicit DeallocOpLowering(LLVMTypeConverter &converter)
-      : ConvertOpToLLVMPattern<DeallocOp>(converter) {}
+  explicit DeallocOpLowering(LLVMTypeConverter &converter,
+                             const LowerToLLVMOptions &options)
+      : ConvertOpToLLVMPattern<DeallocOp>(converter, options) {}
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -1952,8 +2121,10 @@ struct DeallocOpLowering : public ConvertOpToLLVMPattern<DeallocOp> {
     auto freeFunc =
         op->getParentOfType<ModuleOp>().lookupSymbol<LLVM::LLVMFuncOp>("free");
     if (!freeFunc) {
-      OpBuilder moduleBuilder(op->getParentOfType<ModuleOp>().getBodyRegion());
-      freeFunc = moduleBuilder.create<LLVM::LLVMFuncOp>(
+      OpBuilder::InsertionGuard guard(rewriter);
+      rewriter.setInsertionPointToStart(
+          op->getParentOfType<ModuleOp>().getBody());
+      freeFunc = rewriter.create<LLVM::LLVMFuncOp>(
           rewriter.getUnknownLoc(), "free",
           LLVM::LLVMType::getFunctionTy(getVoidType(), getVoidPtrType(),
                                         /*isVarArg=*/false));
@@ -2011,7 +2182,7 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern<RsqrtOp> {
         [&](LLVM::LLVMType llvmVectorTy, ValueRange operands) {
           auto splatAttr = SplatElementsAttr::get(
               mlir::VectorType::get(
-                  {cast<llvm::VectorType>(llvmVectorTy.getUnderlyingType())
+                  {cast<llvm::FixedVectorType>(llvmVectorTy.getUnderlyingType())
                        ->getNumElements()},
                   floatType),
               floatOne);
@@ -2397,6 +2568,10 @@ struct ReturnOpLowering : public ConvertOpToLLVMPattern<ReturnOp> {
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
     unsigned numArguments = op->getNumOperands();
+    auto updatedOperands = llvm::to_vector<4>(operands);
+    copyUnrankedDescriptors(rewriter, op->getLoc(), typeConverter,
+                            op->getOperands().getTypes(), updatedOperands,
+                            /*toDynamic=*/true);
 
     // If ReturnOp has 0 or 1 operand, create it and return immediately.
     if (numArguments == 0) {
@@ -2406,7 +2581,7 @@ struct ReturnOpLowering : public ConvertOpToLLVMPattern<ReturnOp> {
     }
     if (numArguments == 1) {
       rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(
-          op, ArrayRef<Type>(), operands.front(), op->getAttrs());
+          op, ArrayRef<Type>(), updatedOperands, op->getAttrs());
       return success();
     }
 
@@ -2418,7 +2593,7 @@ struct ReturnOpLowering : public ConvertOpToLLVMPattern<ReturnOp> {
     Value packed = rewriter.create<LLVM::UndefOp>(op->getLoc(), packedType);
     for (unsigned i = 0; i < numArguments; ++i) {
       packed = rewriter.create<LLVM::InsertValueOp>(
-          op->getLoc(), packedType, packed, operands[i],
+          op->getLoc(), packedType, packed, updatedOperands[i],
           rewriter.getI64ArrayAttr(i));
     }
     rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(op, ArrayRef<Type>(), packed,
@@ -2960,7 +3135,8 @@ struct GenericAtomicRMWOpLowering
 
 /// Collect a set of patterns to convert from the Standard dialect to LLVM.
 void mlir::populateStdToLLVMNonMemoryConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
+    const LowerToLLVMOptions &options) {
   // FIXME: this should be tablegen'ed
   // clang-format off
   patterns.insert<
@@ -2980,7 +3156,7 @@ void mlir::populateStdToLLVMNonMemoryConversionPatterns(
       CondBranchOpLowering,
       CopySignOpLowering,
       CosOpLowering,
-      ConstLLVMOpLowering,
+      ConstantOpLowering,
       CreateComplexOpLowering,
       DialectCastOpLowering,
       DivFOpLowering,
@@ -3023,13 +3199,13 @@ void mlir::populateStdToLLVMNonMemoryConversionPatterns(
       UnsignedRemIOpLowering,
       UnsignedShiftRightOpLowering,
       XOrOpLowering,
-      ZeroExtendIOpLowering>(converter);
+      ZeroExtendIOpLowering>(converter, options);
   // clang-format on
 }
 
 void mlir::populateStdToLLVMMemoryConversionPatterns(
     LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    bool useAlignedAlloc) {
+    const LowerToLLVMOptions &options) {
   // clang-format off
   patterns.insert<
       AssumeAlignmentOpLowering,
@@ -3039,41 +3215,26 @@ void mlir::populateStdToLLVMMemoryConversionPatterns(
       MemRefCastOpLowering,
       StoreOpLowering,
       SubViewOpLowering,
-      ViewOpLowering>(converter);
-  patterns.insert<
-      AllocOpLowering
-      >(converter, useAlignedAlloc);
+      ViewOpLowering,
+      AllocOpLowering>(converter, options);
   // clang-format on
 }
 
-void mlir::populateStdToLLVMDefaultFuncOpConversionPattern(
+void mlir::populateStdToLLVMFuncOpConversionPattern(
     LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    bool emitCWrappers) {
-  patterns.insert<FuncOpConversion>(converter, emitCWrappers);
+    const LowerToLLVMOptions &options) {
+  if (options.useBarePtrCallConv)
+    patterns.insert<BarePtrFuncOpConversion>(converter, options);
+  else
+    patterns.insert<FuncOpConversion>(converter, options);
 }
 
 void mlir::populateStdToLLVMConversionPatterns(
     LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    bool emitCWrappers, bool useAlignedAlloc) {
-  populateStdToLLVMDefaultFuncOpConversionPattern(converter, patterns,
-                                                  emitCWrappers);
-  populateStdToLLVMNonMemoryConversionPatterns(converter, patterns);
-  populateStdToLLVMMemoryConversionPatterns(converter, patterns,
-                                            useAlignedAlloc);
-}
-
-static void populateStdToLLVMBarePtrFuncOpConversionPattern(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
-  patterns.insert<BarePtrFuncOpConversion>(converter);
-}
-
-void mlir::populateStdToLLVMBarePtrConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    bool useAlignedAlloc) {
-  populateStdToLLVMBarePtrFuncOpConversionPattern(converter, patterns);
-  populateStdToLLVMNonMemoryConversionPatterns(converter, patterns);
-  populateStdToLLVMMemoryConversionPatterns(converter, patterns,
-                                            useAlignedAlloc);
+    const LowerToLLVMOptions &options) {
+  populateStdToLLVMFuncOpConversionPattern(converter, patterns, options);
+  populateStdToLLVMNonMemoryConversionPatterns(converter, patterns, options);
+  populateStdToLLVMMemoryConversionPatterns(converter, patterns, options);
 }
 
 // Create an LLVM IR structure type if there is more than one result.
@@ -3163,19 +3324,12 @@ struct LLVMLoweringPass : public ConvertStandardToLLVMBase<LLVMLoweringPass> {
 
     ModuleOp m = getOperation();
 
-    LLVMTypeConverterCustomization customs;
-    customs.funcArgConverter = useBarePtrCallConv ? barePtrFuncArgTypeConverter
-                                                  : structFuncArgTypeConverter;
-    customs.indexBitwidth = indexBitwidth;
-    LLVMTypeConverter typeConverter(&getContext(), customs);
+    LowerToLLVMOptions options = {useBarePtrCallConv, emitCWrappers,
+                                  indexBitwidth, useAlignedAlloc};
+    LLVMTypeConverter typeConverter(&getContext(), options);
 
     OwningRewritePatternList patterns;
-    if (useBarePtrCallConv)
-      populateStdToLLVMBarePtrConversionPatterns(typeConverter, patterns,
-                                                 useAlignedAlloc);
-    else
-      populateStdToLLVMConversionPatterns(typeConverter, patterns,
-                                          emitCWrappers, useAlignedAlloc);
+    populateStdToLLVMConversionPatterns(typeConverter, patterns, options);
 
     LLVMConversionTarget target(getContext());
     if (failed(applyPartialConversion(m, target, patterns)))
diff --git a/mlir/lib/Conversion/StandardToSPIRV/ConvertStandardToSPIRV.cpp b/mlir/lib/Conversion/StandardToSPIRV/ConvertStandardToSPIRV.cpp
index 45992c888d729..d60ffba2284c7 100644
--- a/mlir/lib/Conversion/StandardToSPIRV/ConvertStandardToSPIRV.cpp
+++ b/mlir/lib/Conversion/StandardToSPIRV/ConvertStandardToSPIRV.cpp
@@ -518,7 +518,7 @@ LogicalResult ConstantCompositeOpPattern::matchAndRewrite(
     return failure();
 
   // std.constant should only have vector or tenor types.
-  assert(srcType.isa<VectorType>() || srcType.isa<RankedTensorType>());
+  assert((srcType.isa<VectorType, RankedTensorType>()));
 
   auto dstType = typeConverter.convertType(srcType);
   if (!dstType)
@@ -536,7 +536,7 @@ LogicalResult ConstantCompositeOpPattern::matchAndRewrite(
                                           srcType.getElementType());
       dstElementsAttr = dstElementsAttr.reshape(dstAttrType);
     } else {
-      // TODO(antiagainst): add support for large vectors.
+      // TODO: add support for large vectors.
       return failure();
     }
   }
@@ -1041,6 +1041,7 @@ void populateStandardToSPIRVPatterns(MLIRContext *context,
       TypeCastingOpPattern<SIToFPOp, spirv::ConvertSToFOp>,
       TypeCastingOpPattern<ZeroExtendIOp, spirv::UConvertOp>,
       TypeCastingOpPattern<TruncateIOp, spirv::SConvertOp>,
+      TypeCastingOpPattern<SignExtendIOp, spirv::SConvertOp>,
       TypeCastingOpPattern<FPToSIOp, spirv::ConvertFToSOp>,
       TypeCastingOpPattern<FPExtOp, spirv::FConvertOp>,
       TypeCastingOpPattern<FPTruncOp, spirv::FConvertOp>, XOrOpPattern>(
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index bd9ec93633833..96a8fa4c6f223 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -110,7 +110,7 @@ static Value extractOne(PatternRewriter &rewriter, Location loc, Value vector,
 }
 
 // Helper that returns a subset of `arrayAttr` as a vector of int64_t.
-// TODO(rriddle): Better support for attribute subtype forwarding + slicing.
+// TODO: Better support for attribute subtype forwarding + slicing.
 static SmallVector<int64_t, 4> getI64SubArray(ArrayAttr arrayAttr,
                                               unsigned dropFront = 0,
                                               unsigned dropBack = 0) {
@@ -255,9 +255,11 @@ class VectorFlatTransposeOpConversion : public ConvertToLLVMPattern {
 class VectorReductionOpConversion : public ConvertToLLVMPattern {
 public:
   explicit VectorReductionOpConversion(MLIRContext *context,
-                                       LLVMTypeConverter &typeConverter)
+                                       LLVMTypeConverter &typeConverter,
+                                       bool reassociateFP)
       : ConvertToLLVMPattern(vector::ReductionOp::getOperationName(), context,
-                             typeConverter) {}
+                             typeConverter),
+        reassociateFPReductions(reassociateFP) {}
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -302,7 +304,8 @@ class VectorReductionOpConversion : public ConvertToLLVMPattern {
                                               op->getLoc(), llvmType,
                                               rewriter.getZeroAttr(eltType));
         rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_v2_fadd>(
-            op, llvmType, acc, operands[0]);
+            op, llvmType, acc, operands[0],
+            rewriter.getBoolAttr(reassociateFPReductions));
       } else if (kind == "mul") {
         // Optional accumulator (or one).
         Value acc = operands.size() > 1
@@ -311,7 +314,8 @@ class VectorReductionOpConversion : public ConvertToLLVMPattern {
                               op->getLoc(), llvmType,
                               rewriter.getFloatAttr(eltType, 1.0));
         rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_v2_fmul>(
-            op, llvmType, acc, operands[0]);
+            op, llvmType, acc, operands[0],
+            rewriter.getBoolAttr(reassociateFPReductions));
       } else if (kind == "min")
         rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_fmin>(
             op, llvmType, operands[0]);
@@ -324,6 +328,9 @@ class VectorReductionOpConversion : public ConvertToLLVMPattern {
     }
     return failure();
   }
+
+private:
+  const bool reassociateFPReductions;
 };
 
 class VectorShuffleOpConversion : public ConvertToLLVMPattern {
@@ -920,7 +927,7 @@ class VectorTransferConversion : public ConvertToLLVMPattern {
         loc, toLLVMTy(vectorCmpType), linearIndices);
 
     // 3. Create offsetVector = [ offset + 0 .. offset + vector_length - 1 ].
-    // TODO(ntv, ajcbik): when the leaf transfer rank is k > 1 we need the last
+    // TODO: when the leaf transfer rank is k > 1 we need the last
     // `k` dimensions here.
     unsigned lastIndex = llvm::size(xferOp.indices()) - 1;
     Value offsetIndex = *(xferOp.indices().begin() + lastIndex);
@@ -961,7 +968,7 @@ class VectorPrintOpConversion : public ConvertToLLVMPattern {
   // output of any shaped and dimensioned vector. Due to full unrolling,
   // this approach is less suited for very large vectors though.
   //
-  // TODO(ajcbik): rely solely on libc in future? something else?
+  // TODO: rely solely on libc in future? something else?
   //
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -1139,16 +1146,18 @@ class VectorStridedSliceOpConversion
 
 /// Populate the given list with patterns that convert from Vector to LLVM.
 void mlir::populateVectorToLLVMConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
+    bool reassociateFPReductions) {
   MLIRContext *ctx = converter.getDialect()->getContext();
   // clang-format off
   patterns.insert<VectorFMAOpNDRewritePattern,
                   VectorInsertStridedSliceOpDifferentRankRewritePattern,
                   VectorInsertStridedSliceOpSameRankRewritePattern,
                   VectorStridedSliceOpConversion>(ctx);
+  patterns.insert<VectorReductionOpConversion>(
+      ctx, converter, reassociateFPReductions);
   patterns
-      .insert<VectorReductionOpConversion,
-              VectorShuffleOpConversion,
+      .insert<VectorShuffleOpConversion,
               VectorExtractElementOpConversion,
               VectorExtractOpConversion,
               VectorFMAOp1DConversion,
@@ -1171,6 +1180,9 @@ void mlir::populateVectorToLLVMMatrixConversionPatterns(
 namespace {
 struct LowerVectorToLLVMPass
     : public ConvertVectorToLLVMBase<LowerVectorToLLVMPass> {
+  LowerVectorToLLVMPass(const LowerVectorToLLVMOptions &options) {
+    this->reassociateFPReductions = options.reassociateFPReductions;
+  }
   void runOnOperation() override;
 };
 } // namespace
@@ -1190,7 +1202,8 @@ void LowerVectorToLLVMPass::runOnOperation() {
   LLVMTypeConverter converter(&getContext());
   OwningRewritePatternList patterns;
   populateVectorToLLVMMatrixConversionPatterns(converter, patterns);
-  populateVectorToLLVMConversionPatterns(converter, patterns);
+  populateVectorToLLVMConversionPatterns(converter, patterns,
+                                         reassociateFPReductions);
   populateVectorToLLVMMatrixConversionPatterns(converter, patterns);
   populateStdToLLVMConversionPatterns(converter, patterns);
 
@@ -1200,6 +1213,7 @@ void LowerVectorToLLVMPass::runOnOperation() {
   }
 }
 
-std::unique_ptr<OperationPass<ModuleOp>> mlir::createConvertVectorToLLVMPass() {
-  return std::make_unique<LowerVectorToLLVMPass>();
+std::unique_ptr<OperationPass<ModuleOp>>
+mlir::createConvertVectorToLLVMPass(const LowerVectorToLLVMOptions &options) {
+  return std::make_unique<LowerVectorToLLVMPass>(options);
 }
diff --git a/mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp b/mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp
index 37e314f2b1dcb..9c8556dbab3a3 100644
--- a/mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp
+++ b/mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp
@@ -72,7 +72,7 @@ class VectorTransferConversion : public ConvertToLLVMPattern {
         llvm::size(xferOp.indices()) == 0)
       return failure();
 
-    if (!AffineMap::isMinorIdentity(xferOp.permutation_map()))
+    if (!xferOp.permutation_map().isMinorIdentity())
       return failure();
 
     // Have it handled in vector->llvm conversion pass.
diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
index 99ded0686a544..f05f0be1d4db2 100644
--- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
+++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
@@ -86,10 +86,10 @@ class NDTransferOpHelper {
         scope(std::make_unique<ScopedContext>(rewriter, loc)), xferOp(xferOp),
         op(xferOp.getOperation()) {
     vectorType = xferOp.getVectorType();
-    // TODO(ntv, ajcbik): when we go to k > 1-D vectors adapt minorRank.
+    // TODO: when we go to k > 1-D vectors adapt minorRank.
     minorRank = 1;
     majorRank = vectorType.getRank() - minorRank;
-    leadingRank = xferOp.getMemRefType().getRank() - (majorRank + minorRank);
+    leadingRank = xferOp.getLeadingMemRefRank();
     majorVectorType =
         VectorType::get(vectorType.getShape().take_front(majorRank),
                         vectorType.getElementType());
@@ -174,6 +174,27 @@ void NDTransferOpHelper<ConcreteOp>::emitLoops(Lambda loopBodyBuilder) {
   }
 }
 
+static Optional<int64_t> extractConstantIndex(Value v) {
+  if (auto cstOp = v.getDefiningOp<ConstantIndexOp>())
+    return cstOp.getValue();
+  if (auto affineApplyOp = v.getDefiningOp<AffineApplyOp>())
+    if (affineApplyOp.getAffineMap().isSingleConstant())
+      return affineApplyOp.getAffineMap().getSingleConstantResult();
+  return None;
+}
+
+// Missing foldings of scf.if make it necessary to perform poor man's folding
+// eagerly, especially in the case of unrolling. In the future, this should go
+// away once scf.if folds properly.
+static Value onTheFlyFoldSLT(Value v, Value ub) {
+  using namespace mlir::edsc::op;
+  auto maybeCstV = extractConstantIndex(v);
+  auto maybeCstUb = extractConstantIndex(ub);
+  if (maybeCstV && maybeCstUb && *maybeCstV < *maybeCstUb)
+    return Value();
+  return slt(v, ub);
+}
+
 template <typename ConcreteOp>
 Value NDTransferOpHelper<ConcreteOp>::emitInBoundsCondition(
     ValueRange majorIvs, ValueRange majorOffsets,
@@ -187,9 +208,11 @@ Value NDTransferOpHelper<ConcreteOp>::emitInBoundsCondition(
     using namespace mlir::edsc::op;
     majorIvsPlusOffsets.push_back(iv + off);
     if (xferOp.isMaskedDim(leadingRank + idx)) {
-      Value inBounds = majorIvsPlusOffsets.back() < ub;
-      inBoundsCondition =
-          (inBoundsCondition) ? (inBoundsCondition && inBounds) : inBounds;
+      Value inBoundsCond = onTheFlyFoldSLT(majorIvsPlusOffsets.back(), ub);
+      if (inBoundsCond)
+        inBoundsCondition = (inBoundsCondition)
+                                ? (inBoundsCondition && inBoundsCond)
+                                : inBoundsCond;
     }
     ++idx;
   }
@@ -433,16 +456,16 @@ clip(TransferOpTy transfer, MemRefBoundsCapture &bounds, ArrayRef<Value> ivs) {
     auto i = memRefAccess[memRefDim];
     if (loopIndex < 0) {
       auto N_minus_1 = N - one;
-      auto select_1 = std_select(i < N, i, N_minus_1);
+      auto select_1 = std_select(slt(i, N), i, N_minus_1);
       clippedScalarAccessExprs[memRefDim] =
-          std_select(i < zero, zero, select_1);
+          std_select(slt(i, zero), zero, select_1);
     } else {
       auto ii = ivs[loopIndex];
       auto i_plus_ii = i + ii;
       auto N_minus_1 = N - one;
-      auto select_1 = std_select(i_plus_ii < N, i_plus_ii, N_minus_1);
+      auto select_1 = std_select(slt(i_plus_ii, N), i_plus_ii, N_minus_1);
       clippedScalarAccessExprs[memRefDim] =
-          std_select(i_plus_ii < zero, zero, select_1);
+          std_select(slt(i_plus_ii, zero), zero, select_1);
     }
   }
 
@@ -505,8 +528,8 @@ MemRefType VectorTransferRewriter<TransferOpTy>::tmpMemRefType(
 /// in the presence of data-parallel only operations, we generate code that
 /// writes the same value multiple time on the edge locations.
 ///
-/// TODO(ntv): implement alternatives to clipping.
-/// TODO(ntv): support non-data-parallel operations.
+/// TODO: implement alternatives to clipping.
+/// TODO: support non-data-parallel operations.
 
 /// Performs the rewrite.
 template <>
@@ -515,7 +538,7 @@ LogicalResult VectorTransferRewriter<TransferReadOp>::matchAndRewrite(
   using namespace mlir::edsc::op;
 
   TransferReadOp transfer = cast<TransferReadOp>(op);
-  if (AffineMap::isMinorIdentity(transfer.permutation_map())) {
+  if (transfer.permutation_map().isMinorIdentity()) {
     // If > 1D, emit a bunch of loops around 1-D vector transfers.
     if (transfer.getVectorType().getRank() > 1)
       return NDTransferOpHelper<TransferReadOp>(rewriter, transfer, options)
@@ -580,15 +603,15 @@ LogicalResult VectorTransferRewriter<TransferReadOp>::matchAndRewrite(
 /// See `Important notes about clipping and full-tiles only abstraction` in the
 /// description of `readClipped` above.
 ///
-/// TODO(ntv): implement alternatives to clipping.
-/// TODO(ntv): support non-data-parallel operations.
+/// TODO: implement alternatives to clipping.
+/// TODO: support non-data-parallel operations.
 template <>
 LogicalResult VectorTransferRewriter<TransferWriteOp>::matchAndRewrite(
     Operation *op, PatternRewriter &rewriter) const {
   using namespace edsc::op;
 
   TransferWriteOp transfer = cast<TransferWriteOp>(op);
-  if (AffineMap::isMinorIdentity(transfer.permutation_map())) {
+  if (transfer.permutation_map().isMinorIdentity()) {
     // If > 1D, emit a bunch of loops around 1-D vector transfers.
     if (transfer.getVectorType().getRank() > 1)
       return NDTransferOpHelper<TransferWriteOp>(rewriter, transfer, options)
diff --git a/mlir/lib/Dialect/Affine/EDSC/Builders.cpp b/mlir/lib/Dialect/Affine/EDSC/Builders.cpp
index 551062aa1f7b8..a96ba970afde7 100644
--- a/mlir/lib/Dialect/Affine/EDSC/Builders.cpp
+++ b/mlir/lib/Dialect/Affine/EDSC/Builders.cpp
@@ -117,7 +117,7 @@ static Value createBinaryHandle(
     return ValueBuilder<IOp>(lhs, rhs);
   } else if (thisType.isa<FloatType>()) {
     return ValueBuilder<FOp>(lhs, rhs);
-  } else if (thisType.isa<VectorType>() || thisType.isa<TensorType>()) {
+  } else if (thisType.isa<VectorType, TensorType>()) {
     auto aggregateType = thisType.cast<ShapedType>();
     if (aggregateType.getElementType().isSignlessInteger())
       return ValueBuilder<IOp>(lhs, rhs);
@@ -221,29 +221,51 @@ Value mlir::edsc::op::ne(Value lhs, Value rhs) {
              ? createFComparisonExpr(CmpFPredicate::ONE, lhs, rhs)
              : createIComparisonExpr(CmpIPredicate::ne, lhs, rhs);
 }
-Value mlir::edsc::op::operator<(Value lhs, Value rhs) {
+Value mlir::edsc::op::slt(Value lhs, Value rhs) {
   auto type = lhs.getType();
   return type.isa<FloatType>()
              ? createFComparisonExpr(CmpFPredicate::OLT, lhs, rhs)
-             :
-             // TODO(ntv,zinenko): signed by default, how about unsigned?
-             createIComparisonExpr(CmpIPredicate::slt, lhs, rhs);
+             : createIComparisonExpr(CmpIPredicate::slt, lhs, rhs);
 }
-Value mlir::edsc::op::operator<=(Value lhs, Value rhs) {
+Value mlir::edsc::op::sle(Value lhs, Value rhs) {
   auto type = lhs.getType();
   return type.isa<FloatType>()
              ? createFComparisonExpr(CmpFPredicate::OLE, lhs, rhs)
              : createIComparisonExpr(CmpIPredicate::sle, lhs, rhs);
 }
-Value mlir::edsc::op::operator>(Value lhs, Value rhs) {
+Value mlir::edsc::op::sgt(Value lhs, Value rhs) {
   auto type = lhs.getType();
   return type.isa<FloatType>()
              ? createFComparisonExpr(CmpFPredicate::OGT, lhs, rhs)
              : createIComparisonExpr(CmpIPredicate::sgt, lhs, rhs);
 }
-Value mlir::edsc::op::operator>=(Value lhs, Value rhs) {
+Value mlir::edsc::op::sge(Value lhs, Value rhs) {
   auto type = lhs.getType();
   return type.isa<FloatType>()
              ? createFComparisonExpr(CmpFPredicate::OGE, lhs, rhs)
              : createIComparisonExpr(CmpIPredicate::sge, lhs, rhs);
 }
+Value mlir::edsc::op::ult(Value lhs, Value rhs) {
+  auto type = lhs.getType();
+  return type.isa<FloatType>()
+             ? createFComparisonExpr(CmpFPredicate::OLT, lhs, rhs)
+             : createIComparisonExpr(CmpIPredicate::ult, lhs, rhs);
+}
+Value mlir::edsc::op::ule(Value lhs, Value rhs) {
+  auto type = lhs.getType();
+  return type.isa<FloatType>()
+             ? createFComparisonExpr(CmpFPredicate::OLE, lhs, rhs)
+             : createIComparisonExpr(CmpIPredicate::ule, lhs, rhs);
+}
+Value mlir::edsc::op::ugt(Value lhs, Value rhs) {
+  auto type = lhs.getType();
+  return type.isa<FloatType>()
+             ? createFComparisonExpr(CmpFPredicate::OGT, lhs, rhs)
+             : createIComparisonExpr(CmpIPredicate::ugt, lhs, rhs);
+}
+Value mlir::edsc::op::uge(Value lhs, Value rhs) {
+  auto type = lhs.getType();
+  return type.isa<FloatType>()
+             ? createFComparisonExpr(CmpFPredicate::OGE, lhs, rhs)
+             : createIComparisonExpr(CmpIPredicate::uge, lhs, rhs);
+}
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 6e0ae42178cdd..ac058606097cf 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -17,6 +17,8 @@
 #include "mlir/Transforms/InliningUtils.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallBitVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
 
@@ -141,9 +143,8 @@ bool mlir::isValidDim(Value value) {
   // This value has to be a block argument for an op that has the
   // `AffineScope` trait or for an affine.for or affine.parallel.
   auto *parentOp = value.cast<BlockArgument>().getOwner()->getParentOp();
-  return parentOp &&
-         (parentOp->hasTrait<OpTrait::AffineScope>() ||
-          isa<AffineForOp>(parentOp) || isa<AffineParallelOp>(parentOp));
+  return parentOp && (parentOp->hasTrait<OpTrait::AffineScope>() ||
+                      isa<AffineForOp, AffineParallelOp>(parentOp));
 }
 
 // Value can be used as a dimension id iff it meets one of the following
@@ -165,7 +166,7 @@ bool mlir::isValidDim(Value value, Region *region) {
     // This value has to be a block argument for an affine.for or an
     // affine.parallel.
     auto *parentOp = value.cast<BlockArgument>().getOwner()->getParentOp();
-    return isa<AffineForOp>(parentOp) || isa<AffineParallelOp>(parentOp);
+    return isa<AffineForOp, AffineParallelOp>(parentOp);
   }
 
   // Affine apply operation is ok if all of its operands are ok.
@@ -468,7 +469,7 @@ indicesFromAffineApplyOp(ArrayRef<Value> operands) {
 // of allowing mathematical AffineMap composition.
 // Returns an AffineMap where symbols that come from an AffineApplyOp have been
 // rewritten as dims and are ordered after the original dims.
-// TODO(andydavis,ntv): This promotion makes AffineMap lose track of which
+// TODO: This promotion makes AffineMap lose track of which
 // symbols are represented as dims. This loss is static but can still be
 // recovered dynamically (with `isValidSymbol`). Still this is annoying for the
 // semi-affine map case. A dynamic canonicalization of all dims that are valid
@@ -537,7 +538,7 @@ static AffineMap promoteComposedSymbolsAsDims(AffineMap map,
 /// composition via symbols, which is ambiguous mathematically. This corner case
 /// is handled by locally rewriting such symbols that come from AffineApplyOp
 /// into dims and composing through dims.
-/// TODO(andydavis, ntv): Composition via symbols comes at a significant code
+/// TODO: Composition via symbols comes at a significant code
 /// complexity. Alternatively we should investigate whether we want to
 /// explicitly disallow symbols coming from affine.apply and instead force the
 /// user to compose symbols beforehand. The annoyances may be small (i.e. 1 or 2
@@ -648,7 +649,7 @@ AffineApplyNormalizer::AffineApplyNormalizer(AffineMap map,
   LLVM_DEBUG(auxiliaryMap.print(dbgs() << "\nWith map: "));
   LLVM_DEBUG(map.compose(auxiliaryMap).print(dbgs() << "\nResult: "));
 
-  // TODO(andydavis,ntv): Disabling simplification results in major speed gains.
+  // TODO: Disabling simplification results in major speed gains.
   // Another option is to cache the results as it is expected a lot of redundant
   // work is performed in practice.
   affineMap = simplifyAffineMap(map.compose(auxiliaryMap));
@@ -929,7 +930,7 @@ static LogicalResult foldMemRefCast(Operation *op) {
 // AffineDmaStartOp
 //===----------------------------------------------------------------------===//
 
-// TODO(b/133776335) Check that map operands are loop IVs or symbols.
+// TODO: Check that map operands are loop IVs or symbols.
 void AffineDmaStartOp::build(OpBuilder &builder, OperationState &result,
                              Value srcMemRef, AffineMap srcMap,
                              ValueRange srcIndices, Value destMemRef,
@@ -1099,7 +1100,7 @@ LogicalResult AffineDmaStartOp::fold(ArrayRef<Attribute> cstOperands,
 // AffineDmaWaitOp
 //===----------------------------------------------------------------------===//
 
-// TODO(b/133776335) Check that map operands are loop IVs or symbols.
+// TODO: Check that map operands are loop IVs or symbols.
 void AffineDmaWaitOp::build(OpBuilder &builder, OperationState &result,
                             Value tagMemRef, AffineMap tagMap,
                             ValueRange tagIndices, Value numElements) {
@@ -1691,7 +1692,7 @@ void mlir::extractForInductionVars(ArrayRef<AffineForOp> forInsts,
 /// Builds an affine loop nest, using "loopCreatorFn" to create individual loop
 /// operations.
 template <typename BoundListTy, typename LoopCreatorTy>
-void buildAffineLoopNestImpl(
+static void buildAffineLoopNestImpl(
     OpBuilder &builder, Location loc, BoundListTy lbs, BoundListTy ubs,
     ArrayRef<int64_t> steps,
     function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilderFn,
@@ -1811,13 +1812,6 @@ static LogicalResult verify(AffineIfOp op) {
                                            condition.getNumDims())))
     return failure();
 
-  // Verify that the entry of each child region does not have arguments.
-  for (auto &region : op.getOperation()->getRegions()) {
-    for (auto &b : region)
-      if (b.getNumArguments() != 0)
-        return op.emitOpError(
-            "requires that child entry blocks have no arguments");
-  }
   return success();
 }
 
@@ -1913,9 +1907,7 @@ void AffineIfOp::build(OpBuilder &builder, OperationState &result,
                        TypeRange resultTypes, IntegerSet set, ValueRange args,
                        bool withElseRegion) {
   assert(resultTypes.empty() || withElseRegion);
-  for (Type type : resultTypes) {
-    result.addTypes(type);
-  }
+  result.addTypes(resultTypes);
   result.addOperands(args);
   result.addAttribute(getConditionAttrName(), IntegerSetAttr::get(set));
 
@@ -1934,7 +1926,8 @@ void AffineIfOp::build(OpBuilder &builder, OperationState &result,
 
 void AffineIfOp::build(OpBuilder &builder, OperationState &result,
                        IntegerSet set, ValueRange args, bool withElseRegion) {
-  AffineIfOp::build(builder, result, {}, set, args, withElseRegion);
+  AffineIfOp::build(builder, result, /*resultTypes=*/{}, set, args,
+                    withElseRegion);
 }
 
 /// Canonicalize an affine if op's conditional (integer set + operands).
@@ -2225,7 +2218,7 @@ static OpFoldResult foldMinMaxOp(T op, ArrayRef<Attribute> operands) {
                 "expected affine min or max op");
 
   // Fold the affine map.
-  // TODO(andydavis, ntv) Fold more cases:
+  // TODO: Fold more cases:
   // min(some_affine, some_affine + constant, ...), etc.
   SmallVector<int64_t, 2> results;
   auto foldedMap = op.map().partialConstantFold(operands, &results);
@@ -2393,6 +2386,7 @@ LogicalResult AffinePrefetchOp::fold(ArrayRef<Attribute> cstOperands,
 
 void AffineParallelOp::build(OpBuilder &builder, OperationState &result,
                              ArrayRef<Type> resultTypes,
+                             ArrayRef<AtomicRMWKind> reductions,
                              ArrayRef<int64_t> ranges) {
   SmallVector<AffineExpr, 8> lbExprs(ranges.size(),
                                      builder.getAffineConstantExpr(0));
@@ -2401,36 +2395,45 @@ void AffineParallelOp::build(OpBuilder &builder, OperationState &result,
   for (int64_t range : ranges)
     ubExprs.push_back(builder.getAffineConstantExpr(range));
   auto ubMap = AffineMap::get(0, 0, ubExprs, builder.getContext());
-  build(builder, result, resultTypes, lbMap, {}, ubMap, {});
+  build(builder, result, resultTypes, reductions, lbMap, /*lbArgs=*/{}, ubMap,
+        /*ubArgs=*/{});
 }
 
 void AffineParallelOp::build(OpBuilder &builder, OperationState &result,
-                             ArrayRef<Type> resultTypes, AffineMap lbMap,
-                             ValueRange lbArgs, AffineMap ubMap,
-                             ValueRange ubArgs) {
+                             ArrayRef<Type> resultTypes,
+                             ArrayRef<AtomicRMWKind> reductions,
+                             AffineMap lbMap, ValueRange lbArgs,
+                             AffineMap ubMap, ValueRange ubArgs) {
   auto numDims = lbMap.getNumResults();
   // Verify that the dimensionality of both maps are the same.
   assert(numDims == ubMap.getNumResults() &&
          "num dims and num results mismatch");
   // Make default step sizes of 1.
   SmallVector<int64_t, 8> steps(numDims, 1);
-  build(builder, result, resultTypes, lbMap, lbArgs, ubMap, ubArgs, steps);
+  build(builder, result, resultTypes, reductions, lbMap, lbArgs, ubMap, ubArgs,
+        steps);
 }
 
 void AffineParallelOp::build(OpBuilder &builder, OperationState &result,
-                             ArrayRef<Type> resultTypes, AffineMap lbMap,
-                             ValueRange lbArgs, AffineMap ubMap,
-                             ValueRange ubArgs, ArrayRef<int64_t> steps) {
+                             ArrayRef<Type> resultTypes,
+                             ArrayRef<AtomicRMWKind> reductions,
+                             AffineMap lbMap, ValueRange lbArgs,
+                             AffineMap ubMap, ValueRange ubArgs,
+                             ArrayRef<int64_t> steps) {
   auto numDims = lbMap.getNumResults();
   // Verify that the dimensionality of the maps matches the number of steps.
   assert(numDims == ubMap.getNumResults() &&
          "num dims and num results mismatch");
   assert(numDims == steps.size() && "num dims and num steps mismatch");
-  // Add the types
-  for (Type type : resultTypes) {
-    result.addTypes(type);
-  }
-  // Add the attributes
+
+  result.addTypes(resultTypes);
+  // Convert the reductions to integer attributes.
+  SmallVector<Attribute, 4> reductionAttrs;
+  for (AtomicRMWKind reduction : reductions)
+    reductionAttrs.push_back(
+        builder.getI64IntegerAttr(static_cast<int64_t>(reduction)));
+  result.addAttribute(getReductionsAttrName(),
+                      builder.getArrayAttr(reductionAttrs));
   result.addAttribute(getLowerBoundsMapAttrName(), AffineMapAttr::get(lbMap));
   result.addAttribute(getUpperBoundsMapAttrName(), AffineMapAttr::get(ubMap));
   result.addAttribute(getStepsAttrName(), builder.getI64ArrayAttr(steps));
@@ -2454,7 +2457,7 @@ bool AffineParallelOp::isDefinedOutsideOfLoop(Value value) {
 }
 
 LogicalResult AffineParallelOp::moveOutOfLoop(ArrayRef<Operation *> ops) {
-  for (auto *op : ops)
+  for (Operation *op : ops)
     op->moveBefore(*this);
   return success();
 }
@@ -2505,8 +2508,47 @@ OpBuilder AffineParallelOp::getBodyBuilder() {
   return OpBuilder(getBody(), std::prev(getBody()->end()));
 }
 
+void AffineParallelOp::setLowerBounds(ValueRange lbOperands, AffineMap map) {
+  assert(lbOperands.size() == map.getNumInputs());
+  assert(map.getNumResults() >= 1 && "bounds map has at least one result");
+
+  auto ubOperands = getUpperBoundsOperands();
+
+  SmallVector<Value, 4> newOperands(lbOperands);
+  newOperands.append(ubOperands.begin(), ubOperands.end());
+  getOperation()->setOperands(newOperands);
+
+  setAttr(getLowerBoundsMapAttrName(), AffineMapAttr::get(map));
+}
+
+void AffineParallelOp::setUpperBounds(ValueRange ubOperands, AffineMap map) {
+  assert(ubOperands.size() == map.getNumInputs());
+  assert(map.getNumResults() >= 1 && "bounds map has at least one result");
+
+  SmallVector<Value, 4> newOperands(getLowerBoundsOperands());
+  newOperands.append(ubOperands.begin(), ubOperands.end());
+  getOperation()->setOperands(newOperands);
+
+  setAttr(getUpperBoundsMapAttrName(), AffineMapAttr::get(map));
+}
+
+void AffineParallelOp::setLowerBoundsMap(AffineMap map) {
+  auto lbMap = lowerBoundsMap();
+  assert(lbMap.getNumDims() == map.getNumDims() &&
+         lbMap.getNumSymbols() == map.getNumSymbols());
+  (void)lbMap;
+  setAttr(getLowerBoundsMapAttrName(), AffineMapAttr::get(map));
+}
+
+void AffineParallelOp::setUpperBoundsMap(AffineMap map) {
+  auto ubMap = upperBoundsMap();
+  assert(ubMap.getNumDims() == map.getNumDims() &&
+         ubMap.getNumSymbols() == map.getNumSymbols());
+  (void)ubMap;
+  setAttr(getUpperBoundsMapAttrName(), AffineMapAttr::get(map));
+}
+
 void AffineParallelOp::setSteps(ArrayRef<int64_t> newSteps) {
-  assert(newSteps.size() == getNumDims() && "steps & num dims mismatch");
   setAttr(getStepsAttrName(), getBodyBuilder().getI64ArrayAttr(newSteps));
 }
 
@@ -2515,10 +2557,20 @@ static LogicalResult verify(AffineParallelOp op) {
   if (op.lowerBoundsMap().getNumResults() != numDims ||
       op.upperBoundsMap().getNumResults() != numDims ||
       op.steps().size() != numDims ||
-      op.getBody()->getNumArguments() != numDims) {
+      op.getBody()->getNumArguments() != numDims)
     return op.emitOpError("region argument count and num results of upper "
                           "bounds, lower bounds, and steps must all match");
+
+  if (op.reductions().size() != op.getNumResults())
+    return op.emitOpError("a reduction must be specified for each output");
+
+  // Verify reduction  ops are all valid
+  for (Attribute attr : op.reductions()) {
+    auto intAttr = attr.dyn_cast<IntegerAttr>();
+    if (!intAttr || !symbolizeAtomicRMWKind(intAttr.getInt()))
+      return op.emitOpError("invalid reduction attribute");
   }
+
   // Verify that the bound operands are valid dimension/symbols.
   /// Lower bounds.
   if (failed(verifyDimAndSymbolIdentifiers(op, op.getLowerBoundsOperands(),
@@ -2540,8 +2592,20 @@ struct AffineParallelRank0LoopRemover
   LogicalResult matchAndRewrite(AffineParallelOp op,
                                 PatternRewriter &rewriter) const override {
     // Check that there are no induction variables
-    if (op.lowerBoundsMap().getNumResults())
+    if (op.getNumDims())
       return failure();
+
+    // Only remove ops that don't have any custom attributes (i.e. those not
+    // defined by the op itself).
+    StringSet<> opAttrs{AffineParallelOp::getReductionsAttrName(),
+                        AffineParallelOp::getLowerBoundsMapAttrName(),
+                        AffineParallelOp::getUpperBoundsMapAttrName(),
+                        AffineParallelOp::getStepsAttrName()};
+    for (auto attr : op.getAttrs()) {
+      if (!opAttrs.count(attr.first.strref()))
+        return failure();
+    }
+
     // Remove the affine.parallel wrapper, retain the body in the same location
     auto &parentOps = rewriter.getInsertionBlock()->getOperations();
     auto &parallelBodyOps = op.region().front().getOperations();
@@ -2557,40 +2621,47 @@ struct AffineParallelRank0LoopRemover
 };
 
 /// This pattern removes indexes that go over an empty range.
-struct AffineParallelRange1IndexRemover
+struct AffineParallelTripCount1IndexRemover
     : public OpRewritePattern<AffineParallelOp> {
   using OpRewritePattern<AffineParallelOp>::OpRewritePattern;
 
   LogicalResult matchAndRewrite(AffineParallelOp op,
                                 PatternRewriter &rewriter) const override {
     auto ranges = op.getRangesValueMap();
-    auto origNumArgs = op.getBody()->getArguments().size();
-    size_t curArgNum = 0;
+    auto body = op.getBody();
     SmallVector<AffineExpr, 6> newLowerBounds;
     SmallVector<AffineExpr, 6> newUpperBounds;
     SmallVector<int64_t, 6> newSteps;
-    for (unsigned i = 0; i < origNumArgs; i++) {
-      // Is the range a constant value of 1?
+    SmallVector<BlockArgument, 6> argsToRemove;
+    for (unsigned i = 0, e = body->getNumArguments(); i < e; i++) {
+      // Is the range a constant value matching the step size?
       auto constExpr = ranges.getResult(i).dyn_cast<AffineConstantExpr>();
-      if (constExpr && constExpr.getValue() == 1) {
-        // Remove argument and replace with 0
-        auto curArg = op.getBody()->getArgument(curArgNum);
-        auto lowerBoundValue = rewriter.create<AffineApplyOp>(
-            op.getLoc(), op.lowerBoundsMap().getSubMap({i}),
-            op.getLowerBoundsOperands());
-        curArg.replaceAllUsesWith(lowerBoundValue);
-        op.getBody()->eraseArgument(curArgNum);
+      int64_t step = op.steps()[i].template cast<IntegerAttr>().getInt();
+      if (constExpr && constExpr.getValue() == step) {
+        // Mark argument for removal and replacement with 0.
+        argsToRemove.push_back(body->getArgument(i));
       } else {
         // Keep argument
         newLowerBounds.push_back(op.lowerBoundsMap().getResult(i));
         newUpperBounds.push_back(op.upperBoundsMap().getResult(i));
-        newSteps.push_back(op.steps()[i].template cast<IntegerAttr>().getInt());
-        curArgNum++;
+        newSteps.push_back(step);
       }
     }
-    // If no arguments were removed, return failure to match
-    if (newLowerBounds.size() == op.lowerBoundsMap().getNumResults())
+
+    // If no arguments need removal, return failure to match.
+    if (argsToRemove.empty())
       return failure();
+
+    // After this point, there will be no need to rollback the rewriter.
+    for (auto arg : argsToRemove) {
+      auto argNumber = arg.getArgNumber();
+      auto lowerBoundValue = rewriter.create<AffineApplyOp>(
+          op.getLoc(), op.lowerBoundsMap().getSubMap({argNumber}),
+          op.getLowerBoundsOperands());
+      arg.replaceAllUsesWith(lowerBoundValue);
+      body->eraseArgument(argNumber);
+    }
+
     // Update attributes and return success
     auto newLower = AffineMap::get(op.lowerBoundsMap().getNumDims(),
                                    op.lowerBoundsMap().getNumSymbols(),
@@ -2602,19 +2673,124 @@ struct AffineParallelRange1IndexRemover
                AffineMapAttr::get(newLower));
     op.setAttr(AffineParallelOp::getUpperBoundsMapAttrName(),
                AffineMapAttr::get(newUpper));
-    op.setAttr(AffineParallelOp::getStepsAttrName(),
-               rewriter.getI64ArrayAttr(newSteps));
+    op.setSteps(newSteps);
+    return success();
+  }
+};
+
+struct SimplifyAffineParallel : public OpRewritePattern<AffineParallelOp> {
+  using OpRewritePattern<AffineParallelOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(AffineParallelOp op,
+                                PatternRewriter &rewriter) const override {
+
+    auto stepsAttrs = op.steps();
+    auto lbMap = op.lowerBoundsMap();
+
+    SmallVector<int, 8> steps;
+    bool isWorkPending = false;
+    for (unsigned i = 0, e = stepsAttrs.size(); i < e; ++i) {
+      auto step = stepsAttrs[i].cast<IntegerAttr>().getInt();
+      steps.push_back(step);
+      auto lbExpr = lbMap.getResult(i).dyn_cast<AffineConstantExpr>();
+      isWorkPending |= (!lbExpr || lbExpr.getValue() || step != 1);
+    }
+
+    // No need to do any work if the parallel op is already simplified.
+    if (!isWorkPending)
+      return failure();
+
+    auto ranges = op.getRangesValueMap();
+    auto zeroExpr = rewriter.getAffineConstantExpr(0);
+    rewriter.setInsertionPointToStart(op.getBody());
+    SmallVector<AffineExpr, 8> lbExprs;
+    SmallVector<AffineExpr, 8> ubExprs;
+    for (unsigned i = 0, e = steps.size(); i < e; ++i) {
+      auto step = steps[i];
+
+      // Adjust the lower bound to be 0.
+      lbExprs.push_back(zeroExpr);
+
+      // Adjust the upper bound expression: 'range / step'
+      auto ubExpr = ranges.getResult(i).floorDiv(step);
+      ubExprs.push_back(ubExpr);
+
+      // Adjust the corresponding IV: 'lb + i * step'
+      auto iv = op.getBody()->getArgument(i);
+      auto lbExpr = lbMap.getResult(i);
+      auto nDims = lbMap.getNumDims();
+      auto expr = lbExpr + rewriter.getAffineDimExpr(nDims) * step;
+      auto map = AffineMap::get(/*dimCount=*/nDims + 1,
+                                /*symbolCount=*/lbMap.getNumSymbols(), expr);
+
+      // Use an 'affine.apply' op that will be simplified later in subsequent
+      // canonicalizations.
+      auto lbOperands = op.getLowerBoundsOperands();
+      auto dimOperands = lbOperands.take_front(nDims);
+      auto symbolOperands = lbOperands.drop_front(nDims);
+      SmallVector<Value, 8> applyOperands{dimOperands};
+      applyOperands.push_back(iv);
+      applyOperands.append(symbolOperands.begin(), symbolOperands.end());
+      auto apply =
+          rewriter.create<AffineApplyOp>(op.getLoc(), map, applyOperands);
+      iv.replaceAllUsesExcept(apply, SmallPtrSet<Operation *, 1>{apply});
+    }
+
+    SmallVector<int64_t, 8> newSteps(op.getNumDims(), 1);
+    op.setSteps(newSteps);
+    auto newLowerMap = AffineMap::get(/*dimCount=*/0, /*symbolCount=*/0,
+                                      lbExprs, rewriter.getContext());
+    op.setLowerBounds({}, newLowerMap);
+    auto newUpperMap =
+        AffineMap::get(ranges.getNumDims(), ranges.getNumSymbols(), ubExprs,
+                       rewriter.getContext());
+    op.setUpperBounds(ranges.getOperands(), newUpperMap);
+
     return success();
   }
 };
 
 } // end anonymous namespace
 
+LogicalResult AffineValueMap::canonicalize() {
+  SmallVector<Value, 4> newOperands{operands};
+  auto newMap = getAffineMap();
+  composeAffineMapAndOperands(&newMap, &newOperands);
+  if (newMap == getAffineMap() && newOperands == operands)
+    return failure();
+  reset(newMap, newOperands);
+  return success();
+}
+
+/// Canonicalize the bounds of the given loop.
+static LogicalResult canonicalizeLoopBounds(AffineParallelOp op) {
+  auto lb = op.getLowerBoundsValueMap();
+  auto lbCanonicalized = succeeded(lb.canonicalize());
+
+  auto ub = op.getUpperBoundsValueMap();
+  auto ubCanonicalized = succeeded(ub.canonicalize());
+
+  // Any canonicalization change always leads to updated map(s).
+  if (!lbCanonicalized && !ubCanonicalized)
+    return failure();
+
+  if (lbCanonicalized)
+    op.setLowerBounds(lb.getOperands(), lb.getAffineMap());
+  if (ubCanonicalized)
+    op.setUpperBounds(ub.getOperands(), ub.getAffineMap());
+
+  return success();
+}
+
 void AffineParallelOp::getCanonicalizationPatterns(
     OwningRewritePatternList &results, MLIRContext *context) {
-  results
-      .insert<AffineParallelRank0LoopRemover, AffineParallelRange1IndexRemover>(
-          context);
+  results.insert<SimplifyAffineParallel, AffineParallelRank0LoopRemover,
+                 AffineParallelTripCount1IndexRemover>(context);
+}
+
+LogicalResult AffineParallelOp::fold(ArrayRef<Attribute> operands,
+                                     SmallVectorImpl<OpFoldResult> &results) {
+  return canonicalizeLoopBounds(*this);
 }
 
 static void print(OpAsmPrinter &p, AffineParallelOp op) {
@@ -2638,17 +2814,21 @@ static void print(OpAsmPrinter &p, AffineParallelOp op) {
     p << ')';
   }
   if (op.getNumResults()) {
-    p << " : ";
-    for (auto type : op.getResultTypes()) {
-      p << type;
-    }
+    p << " reduce (";
+    llvm::interleaveComma(op.reductions(), p, [&](auto &attr) {
+      AtomicRMWKind sym =
+          *symbolizeAtomicRMWKind(attr.template cast<IntegerAttr>().getInt());
+      p << "\"" << stringifyAtomicRMWKind(sym) << "\"";
+    });
+    p << ") -> (" << op.getResultTypes() << ")";
   }
 
   p.printRegion(op.region(), /*printEntryBlockArgs=*/false,
                 /*printBlockTerminators=*/op.getNumResults());
   p.printOptionalAttrDict(
       op.getAttrs(),
-      /*elidedAttrs=*/{AffineParallelOp::getLowerBoundsMapAttrName(),
+      /*elidedAttrs=*/{AffineParallelOp::getReductionsAttrName(),
+                       AffineParallelOp::getLowerBoundsMapAttrName(),
                        AffineParallelOp::getUpperBoundsMapAttrName(),
                        AffineParallelOp::getStepsAttrName()});
 }
@@ -2712,7 +2892,38 @@ static ParseResult parseAffineParallelOp(OpAsmParser &parser,
                         builder.getI64ArrayAttr(steps));
   }
 
-  if (parser.parseOptionalColonTypeList(result.types))
+  // Parse optional clause of the form: `reduce ("addf", "maxf")`, where the
+  // quoted strings a member of the enum AtomicRMWKind.
+  SmallVector<Attribute, 4> reductions;
+  if (succeeded(parser.parseOptionalKeyword("reduce"))) {
+    if (parser.parseLParen())
+      return failure();
+    do {
+      // Parse a single quoted string via the attribute parsing, and then
+      // verify it is a member of the enum and convert to it's integer
+      // representation.
+      StringAttr attrVal;
+      NamedAttrList attrStorage;
+      auto loc = parser.getCurrentLocation();
+      if (parser.parseAttribute(attrVal, builder.getNoneType(), "reduce",
+                                attrStorage))
+        return failure();
+      llvm::Optional<AtomicRMWKind> reduction =
+          symbolizeAtomicRMWKind(attrVal.getValue());
+      if (!reduction)
+        return parser.emitError(loc, "invalid reduction value: ") << attrVal;
+      reductions.push_back(builder.getI64IntegerAttr(
+          static_cast<int64_t>(reduction.getValue())));
+      // While we keep getting commas, keep parsing.
+    } while (succeeded(parser.parseOptionalComma()));
+    if (parser.parseRParen())
+      return failure();
+  }
+  result.addAttribute(AffineParallelOp::getReductionsAttrName(),
+                      builder.getArrayAttr(reductions));
+
+  // Parse return types of reductions (if any)
+  if (parser.parseOptionalArrowTypeList(result.types))
     return failure();
 
   // Now parse the body.
@@ -2736,16 +2947,14 @@ static LogicalResult verify(AffineYieldOp op) {
   auto results = parentOp->getResults();
   auto operands = op.getOperands();
 
-  if (!isa<AffineParallelOp>(parentOp) && !isa<AffineIfOp>(parentOp) &&
-      !isa<AffineForOp>(parentOp)) {
+  if (!isa<AffineParallelOp, AffineIfOp, AffineForOp>(parentOp))
     return op.emitOpError()
            << "affine.terminate only terminates If, For or Parallel regions";
-  }
   if (parentOp->getNumResults() != op.getNumOperands())
     return op.emitOpError() << "parent of yield must have same number of "
                                "results as the yield operands";
-  for (auto e : llvm::zip(results, operands)) {
-    if (std::get<0>(e).getType() != std::get<1>(e).getType())
+  for (auto it : llvm::zip(results, operands)) {
+    if (std::get<0>(it).getType() != std::get<1>(it).getType())
       return op.emitOpError()
              << "types mismatch between yield op and its parent";
   }
diff --git a/mlir/lib/Dialect/Affine/IR/AffineValueMap.cpp b/mlir/lib/Dialect/Affine/IR/AffineValueMap.cpp
index 792ca379cef4d..6f7c2fbc56ff2 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineValueMap.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineValueMap.cpp
@@ -85,7 +85,7 @@ bool AffineValueMap::isFunctionOf(unsigned idx, Value value) const {
     return false;
   }
   auto expr = const_cast<AffineValueMap *>(this)->getAffineMap().getResult(idx);
-  // TODO(ntv): this is better implemented on a flattened representation.
+  // TODO: this is better implemented on a flattened representation.
   // At least for now it is conservative.
   return expr.isFunctionOfDim(index);
 }
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
index e0d588fe4ed73..3baac7c58945c 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
@@ -46,7 +46,7 @@ namespace {
 /// inner levels if necessary to determine at what depth copies need to be
 /// placed so that the allocated buffers fit within the memory capacity
 /// provided.
-// TODO(bondhugula): We currently can't generate copies correctly when stores
+// TODO: We currently can't generate copies correctly when stores
 // are strided. Check for strided stores.
 struct AffineDataCopyGeneration
     : public AffineDataCopyGenerationBase<AffineDataCopyGeneration> {
@@ -75,7 +75,7 @@ struct AffineDataCopyGeneration
 /// Generates copies for memref's living in 'slowMemorySpace' into newly created
 /// buffers in 'fastMemorySpace', and replaces memory operations to the former
 /// by the latter. Only load op's handled for now.
-/// TODO(bondhugula): extend this to store op's.
+/// TODO: extend this to store op's.
 std::unique_ptr<OperationPass<FuncOp>> mlir::createAffineDataCopyGenerationPass(
     unsigned slowMemorySpace, unsigned fastMemorySpace, unsigned tagMemorySpace,
     int minDmaTransferSize, uint64_t fastMemCapacityBytes) {
@@ -113,15 +113,14 @@ AffineDataCopyGeneration::runOnBlock(Block *block,
   // operations excluding AffineForOp's) are always assumed to not exhaust
   // memory. As a result, this approach is conservative in some cases at the
   // moment; we do a check later and report an error with location info.
-  // TODO(bondhugula): An 'affine.if' operation is being treated similar to an
+  // TODO: An 'affine.if' operation is being treated similar to an
   // operation. 'affine.if''s could have 'affine.for's in them;
   // treat them separately.
 
   // Get to the first load, store, or for op (that is not a copy nest itself).
   auto curBegin =
       std::find_if(block->begin(), block->end(), [&](Operation &op) {
-        return (isa<AffineLoadOp>(op) || isa<AffineStoreOp>(op) ||
-                isa<AffineForOp>(op)) &&
+        return isa<AffineLoadOp, AffineStoreOp, AffineForOp>(op) &&
                copyNests.count(&op) == 0;
       });
 
@@ -171,8 +170,7 @@ AffineDataCopyGeneration::runOnBlock(Block *block,
       }
       // Get to the next load or store op after 'forOp'.
       curBegin = std::find_if(std::next(it), block->end(), [&](Operation &op) {
-        return (isa<AffineLoadOp>(op) || isa<AffineStoreOp>(op) ||
-                isa<AffineForOp>(op)) &&
+        return isa<AffineLoadOp, AffineStoreOp, AffineForOp>(op) &&
                copyNests.count(&op) == 0;
       });
       it = curBegin;
@@ -220,7 +218,7 @@ void AffineDataCopyGeneration::runOnFunction() {
     nest->walk([&](Operation *op) {
       if (auto forOp = dyn_cast<AffineForOp>(op))
         promoteIfSingleIteration(forOp);
-      else if (isa<AffineLoadOp>(op) || isa<AffineStoreOp>(op))
+      else if (isa<AffineLoadOp, AffineStoreOp>(op))
         copyOps.push_back(op);
     });
 
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp
index 9490101102dda..364168ce6e2ab 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp
@@ -37,8 +37,8 @@ using namespace mlir;
 namespace {
 
 /// Loop invariant code motion (LICM) pass.
-/// TODO(asabne) : The pass is missing zero-trip tests.
-/// TODO(asabne) : Check for the presence of side effects before hoisting.
+/// TODO: The pass is missing zero-trip tests.
+/// TODO: Check for the presence of side effects before hoisting.
 /// TODO: This code should be removed once the new LICM pass can handle its
 ///       uses.
 struct LoopInvariantCodeMotion
@@ -62,11 +62,8 @@ areAllOpsInTheBlockListInvariant(Region &blockList, Value indVar,
                                  SmallPtrSetImpl<Operation *> &opsToHoist);
 
 static bool isMemRefDereferencingOp(Operation &op) {
-  // TODO(asabne): Support DMA Ops.
-  if (isa<AffineLoadOp>(op) || isa<AffineStoreOp>(op)) {
-    return true;
-  }
-  return false;
+  // TODO: Support DMA Ops.
+  return isa<AffineLoadOp, AffineStoreOp>(op);
 }
 
 // Returns true if the individual op is loop invariant.
@@ -83,8 +80,8 @@ bool isOpLoopInvariant(Operation &op, Value indVar,
     // If the body of a predicated region has a for loop, we don't hoist the
     // 'affine.if'.
     return false;
-  } else if (isa<AffineDmaStartOp>(op) || isa<AffineDmaWaitOp>(op)) {
-    // TODO(asabne): Support DMA ops.
+  } else if (isa<AffineDmaStartOp, AffineDmaWaitOp>(op)) {
+    // TODO: Support DMA ops.
     return false;
   } else if (!isa<ConstantOp>(op)) {
     if (isMemRefDereferencingOp(op)) {
@@ -94,7 +91,7 @@ bool isOpLoopInvariant(Operation &op, Value indVar,
       for (auto *user : memref.getUsers()) {
         // If this memref has a user that is a DMA, give up because these
         // operations write to this memref.
-        if (isa<AffineDmaStartOp>(op) || isa<AffineDmaWaitOp>(op)) {
+        if (isa<AffineDmaStartOp, AffineDmaWaitOp>(op)) {
           return false;
         }
         // If the memref used by the load/store is used in a store elsewhere in
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineParallelize.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineParallelize.cpp
new file mode 100644
index 0000000000000..b3651e2022458
--- /dev/null
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineParallelize.cpp
@@ -0,0 +1,50 @@
+//===- AffineParallelize.cpp - Affineparallelize Pass---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a parallelizer for affine loop nests that is able to
+// perform inner or outer loop parallelization.
+//
+//===----------------------------------------------------------------------===//
+
+#include "PassDetail.h"
+#include "mlir/Analysis/AffineStructures.h"
+#include "mlir/Analysis/LoopAnalysis.h"
+#include "mlir/Analysis/Utils.h"
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/Affine/IR/AffineValueMap.h"
+#include "mlir/Dialect/Affine/Passes.h"
+#include "mlir/Dialect/Affine/Passes.h.inc"
+#include "mlir/Dialect/Affine/Utils.h"
+#include "mlir/Transforms/LoopUtils.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "affine-parallel"
+
+using namespace mlir;
+
+namespace {
+/// Convert all parallel affine.for op into 1-D affine.parallel op.
+struct AffineParallelize : public AffineParallelizeBase<AffineParallelize> {
+  void runOnFunction() override;
+};
+} // namespace
+
+void AffineParallelize::runOnFunction() {
+  FuncOp f = getFunction();
+  SmallVector<AffineForOp, 8> parallelizableLoops;
+  f.walk([&](AffineForOp loop) {
+    if (isLoopParallel(loop))
+      parallelizableLoops.push_back(loop);
+  });
+  for (AffineForOp loop : parallelizableLoops)
+    affineParallelize(loop);
+}
+
+std::unique_ptr<OperationPass<FuncOp>> mlir::createAffineParallelizePass() {
+  return std::make_unique<AffineParallelize>();
+}
diff --git a/mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt
index 0098c3e210914..dddcc93adf0d6 100644
--- a/mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_mlir_dialect_library(MLIRAffineTransforms
   AffineDataCopyGeneration.cpp
   AffineLoopInvariantCodeMotion.cpp
+  AffineParallelize.cpp
   LoopTiling.cpp
   LoopUnroll.cpp
   LoopUnrollAndJam.cpp
@@ -17,6 +18,7 @@ add_mlir_dialect_library(MLIRAffineTransforms
 
   LINK_LIBS PUBLIC
   MLIRAffineOps
+  MLIRAffineUtils
   MLIREDSC
   MLIRIR
   MLIRPass
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
index c411a6a548ffc..ddb00bdd8f0e5 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
@@ -159,7 +159,7 @@ constructTiledIndexSetHyperRect(MutableArrayRef<AffineForOp> origLoops,
 
 /// Tiles the specified band of perfectly nested loops creating tile-space loops
 /// and intra-tile loops. A band is a contiguous set of loops.
-//  TODO(bondhugula): handle non hyper-rectangular spaces.
+//  TODO: handle non hyper-rectangular spaces.
 LogicalResult
 mlir::tilePerfectlyNested(MutableArrayRef<AffineForOp> input,
                           ArrayRef<unsigned> tileSizes,
@@ -282,7 +282,7 @@ static void adjustToDivisorsOfTripCounts(ArrayRef<AffineForOp> band,
 // based on a simple model that looks at the memory footprint and determines
 // tile sizes assuming identity accesses / 1:1 tile size proportional footprint
 // along each of the dimensions being tiled.
-// TODO(mlir-team): evolve this model. Tile size determination is a large area
+// TODO: evolve this model. Tile size determination is a large area
 // to play with in general.
 void LoopTiling::getTileSizes(ArrayRef<AffineForOp> band,
                               SmallVectorImpl<unsigned> *tileSizes) {
@@ -334,7 +334,7 @@ void LoopTiling::getTileSizes(ArrayRef<AffineForOp> band,
   }
 
   // Divide all loops equally in an attempt to reduce footprint.
-  // TODO(bondhugula): this is approximate. Ideally, obtain reuse factor /
+  // TODO: this is approximate. Ideally, obtain reuse factor /
   // profitability along each dimension and weight tile sizes based on that as
   // one possible approach. Or compute a polynomial in tile sizes and solve for
   // it.
diff --git a/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp b/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
index fada39aa1cf28..0cd59b52d543d 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
@@ -93,7 +93,7 @@ void SimplifyAffineStructures::runOnFunction() {
 
     // The simplification of the attribute will likely simplify the op. Try to
     // fold / apply canonicalization patterns when we have affine dialect ops.
-    if (isa<AffineForOp>(op) || isa<AffineIfOp>(op) || isa<AffineApplyOp>(op))
+    if (isa<AffineForOp, AffineIfOp, AffineApplyOp>(op))
       applyOpPatternsAndFold(op, patterns);
   });
 
diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
index f5b98f9bf0653..1638502508e3d 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
@@ -236,7 +236,7 @@ using namespace mlir;
 ///
 /// Lowering, unrolling, pipelining:
 /// ================================
-/// TODO(ntv): point to the proper places.
+/// TODO: point to the proper places.
 ///
 /// Algorithm:
 /// ==========
@@ -248,7 +248,7 @@ using namespace mlir;
 ///     reduction, vectorizable, ...) as well as b. all contiguous load/store
 ///     operations along a specified minor dimension (not necessarily the
 ///     fastest varying) ;
-///  2. analyzing those patterns for profitability (TODO(ntv): and
+///  2. analyzing those patterns for profitability (TODO: and
 ///     interference);
 ///  3. Then, for each pattern in order:
 ///    a. applying iterative rewriting of the loop and the load operations in
@@ -259,7 +259,7 @@ using namespace mlir;
 ///    c. traversing the use-def chains starting from the roots and iteratively
 ///       propagating vectorized values. Scalar values that are encountered
 ///       during this process must come from outside the scope of the current
-///       pattern (TODO(ntv): enforce this and generalize). Such a scalar value
+///       pattern (TODO: enforce this and generalize). Such a scalar value
 ///       is vectorized only if it is a constant (into a vector splat). The
 ///       non-constant case is not supported for now and results in the pattern
 ///       failing to vectorize;
@@ -561,7 +561,7 @@ makePatterns(const DenseSet<Operation *> &parallelLoops, int vectorRank,
 
 static NestedPattern &vectorTransferPattern() {
   static auto pattern = matcher::Op([](Operation &op) {
-    return isa<vector::TransferReadOp>(op) || isa<vector::TransferWriteOp>(op);
+    return isa<vector::TransferReadOp, vector::TransferWriteOp>(op);
   });
   return pattern;
 }
@@ -582,7 +582,7 @@ Vectorize::Vectorize(ArrayRef<int64_t> virtualVectorSize) {
   vectorSizes = virtualVectorSize;
 }
 
-/////// TODO(ntv): Hoist to a VectorizationStrategy.cpp when appropriate.
+/////// TODO: Hoist to a VectorizationStrategy.cpp when appropriate.
 /////////
 namespace {
 
@@ -614,7 +614,7 @@ static void vectorizeLoopIfProfitable(Operation *loop, unsigned depthInPattern,
 /// load/store MemRefs, this creates a generic vectorization strategy that works
 /// for any loop in a hierarchy (outermost, innermost or intermediate).
 ///
-/// TODO(ntv): In the future we should additionally increase the power of the
+/// TODO: In the future we should additionally increase the power of the
 /// profitability analysis along 3 directions:
 ///   1. account for loop extents (both static and parametric + annotations);
 ///   2. account for data layout permutations;
@@ -636,7 +636,7 @@ static LogicalResult analyzeProfitability(ArrayRef<NestedMatch> matches,
   return success();
 }
 
-///// end TODO(ntv): Hoist to a VectorizationStrategy.cpp when appropriate /////
+///// end TODO: Hoist to a VectorizationStrategy.cpp when appropriate /////
 
 namespace {
 
@@ -741,7 +741,7 @@ static void computeMemoryOpIndices(Operation *op, AffineMap map,
   }
 }
 
-////// TODO(ntv): Hoist to a VectorizationMaterialize.cpp when appropriate. ////
+////// TODO: Hoist to a VectorizationMaterialize.cpp when appropriate. ////
 
 /// Handles the vectorization of load and store MLIR operations.
 ///
@@ -763,7 +763,7 @@ static LogicalResult vectorizeRootOrTerminal(Value iv,
   auto memRefType = memoryOp.getMemRef().getType().template cast<MemRefType>();
 
   auto elementType = memRefType.getElementType();
-  // TODO(ntv): ponder whether we want to further vectorize a vector value.
+  // TODO: ponder whether we want to further vectorize a vector value.
   assert(VectorType::isValidElementType(elementType) &&
          "Not a valid vector element type");
   auto vectorType = VectorType::get(state->strategy->vectorSizes, elementType);
@@ -772,7 +772,7 @@ static LogicalResult vectorizeRootOrTerminal(Value iv,
   auto *opInst = memoryOp.getOperation();
   // For now, vector.transfers must be aligned, operate only on indices with an
   // identity subset of AffineMap and do not change layout.
-  // TODO(ntv): increase the expressiveness power of vector.transfer operations
+  // TODO: increase the expressiveness power of vector.transfer operations
   // as needed by various targets.
   if (auto load = dyn_cast<AffineLoadOp>(opInst)) {
     OpBuilder b(opInst);
@@ -800,7 +800,7 @@ static LogicalResult vectorizeRootOrTerminal(Value iv,
   }
   return success();
 }
-/// end TODO(ntv): Hoist to a VectorizationMaterialize.cpp when appropriate. ///
+/// end TODO: Hoist to a VectorizationMaterialize.cpp when appropriate. ///
 
 /// Coarsens the loops bounds and transforms all remaining load and store
 /// operations into the appropriate vector.transfer.
@@ -937,7 +937,7 @@ static Value vectorizeConstant(Operation *op, ConstantOp constant, Type type) {
 /// Returns an operand that has been vectorized to match `state`'s strategy if
 /// vectorization is possible with the above logic. Returns nullptr otherwise.
 ///
-/// TODO(ntv): handle more complex cases.
+/// TODO: handle more complex cases.
 static Value vectorizeOperand(Value operand, Operation *op,
                               VectorizationState *state) {
   LLVM_DEBUG(dbgs() << "\n[early-vect]vectorize operand: " << operand);
@@ -956,7 +956,7 @@ static Value vectorizeOperand(Value operand, Operation *op,
     LLVM_DEBUG(dbgs() << "-> delayed replacement by: " << res);
     return res;
   }
-  // 2. TODO(ntv): broadcast needed.
+  // 2. TODO: broadcast needed.
   if (operand.getType().isa<VectorType>()) {
     LLVM_DEBUG(dbgs() << "-> non-vectorizable");
     return nullptr;
@@ -978,7 +978,7 @@ static Value vectorizeOperand(Value operand, Operation *op,
 /// particular operation vectorizes. For now we implement the case distinction
 /// here.
 /// Returns a vectorized form of an operation or nullptr if vectorization fails.
-// TODO(ntv): consider adding a trait to Op to describe how it gets vectorized.
+// TODO: consider adding a trait to Op to describe how it gets vectorized.
 // Maybe some Ops are not vectorizable or require some tricky logic, we cannot
 // do one-off logic here; ideally it would be TableGen'd.
 static Operation *vectorizeOneOperation(Operation *opInst,
@@ -1044,9 +1044,9 @@ static Operation *vectorizeOneOperation(Operation *opInst,
   }
 
   // Create a clone of the op with the proper operands and return types.
-  // TODO(ntv): The following assumes there is always an op with a fixed
+  // TODO: The following assumes there is always an op with a fixed
   // name that works both in scalar mode and vector mode.
-  // TODO(ntv): Is it worth considering an Operation.clone operation which
+  // TODO: Is it worth considering an Operation.clone operation which
   // changes the type so we can promote an Operation with less boilerplate?
   OpBuilder b(opInst);
   OperationState newOp(opInst->getLoc(), opInst->getName().getStringRef(),
@@ -1072,7 +1072,7 @@ static LogicalResult vectorizeNonTerminals(VectorizationState *state) {
   // Note: we have to exclude terminals because some of their defs may not be
   // nested under the vectorization pattern (e.g. constants defined in an
   // encompassing scope).
-  // TODO(ntv): Use a backward slice for terminals, avoid special casing and
+  // TODO: Use a backward slice for terminals, avoid special casing and
   // merge implementations.
   for (auto *op : state->roots) {
     getForwardSlice(op, &worklist, [state](Operation *op) {
@@ -1120,7 +1120,7 @@ static LogicalResult vectorizeRootMatch(NestedMatch m,
   // pattern matching, from profitability analysis, from application.
   // As a consequence we must check that each root pattern is still
   // vectorizable. If a pattern is not vectorizable anymore, we just skip it.
-  // TODO(ntv): implement a non-greedy profitability analysis that keeps only
+  // TODO: implement a non-greedy profitability analysis that keeps only
   // non-intersecting patterns.
   if (!isVectorizableLoopBody(loop, vectorTransferPattern())) {
     LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ loop is not vectorizable");
@@ -1162,7 +1162,7 @@ static LogicalResult vectorizeRootMatch(NestedMatch m,
 
   // 2. Vectorize operations reached by use-def chains from root except the
   // terminals (store operations) that need to be post-processed separately.
-  // TODO(ntv): add more as we expand.
+  // TODO: add more as we expand.
   if (failed(vectorizeNonTerminals(&state))) {
     LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ failed vectorizeNonTerminals");
     return guard.failure();
@@ -1172,7 +1172,7 @@ static LogicalResult vectorizeRootMatch(NestedMatch m,
   // Note: we have to post-process terminals because some of their defs may not
   // be nested under the vectorization pattern (e.g. constants defined in an
   // encompassing scope).
-  // TODO(ntv): Use a backward slice for terminals, avoid special casing and
+  // TODO: Use a backward slice for terminals, avoid special casing and
   // merge implementations.
   for (auto *op : state.terminals) {
     if (!vectorizeOneOperation(op, &state)) { // nullptr == failure
@@ -1221,7 +1221,7 @@ void Vectorize::runOnFunction() {
     // This automatically prunes intersecting matches.
     for (auto m : matches) {
       VectorizationStrategy strategy;
-      // TODO(ntv): depending on profitability, elect to reduce the vector size.
+      // TODO: depending on profitability, elect to reduce the vector size.
       strategy.vectorSizes.assign(vectorSizes.begin(), vectorSizes.end());
       if (failed(analyzeProfitability(m.getMatchedChildren(), 1, patternDepth,
                                       &strategy))) {
@@ -1229,10 +1229,10 @@ void Vectorize::runOnFunction() {
       }
       vectorizeLoopIfProfitable(m.getMatchedOperation(), 0, patternDepth,
                                 &strategy);
-      // TODO(ntv): if pattern does not apply, report it; alter the
+      // TODO: if pattern does not apply, report it; alter the
       // cost/benefit.
       vectorizeRootMatch(m, &strategy);
-      // TODO(ntv): some diagnostics if failure to vectorize occurs.
+      // TODO: some diagnostics if failure to vectorize occurs.
     }
   }
   LLVM_DEBUG(dbgs() << "\n");
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index 811579bb6c8c7..39e9cbc61e961 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -129,8 +129,28 @@ static AffineIfOp hoistAffineIfOp(AffineIfOp ifOp, Operation *hoistOverOp) {
   return hoistedIfOp;
 }
 
+/// Replace affine.for with a 1-d affine.parallel and clone the former's body
+/// into the latter while remapping values.
+void mlir::affineParallelize(AffineForOp forOp) {
+  Location loc = forOp.getLoc();
+  OpBuilder outsideBuilder(forOp);
+  // Creating empty 1-D affine.parallel op.
+  AffineParallelOp newPloop = outsideBuilder.create<AffineParallelOp>(
+      loc, llvm::None, llvm::None, forOp.getLowerBoundMap(),
+      forOp.getLowerBoundOperands(), forOp.getUpperBoundMap(),
+      forOp.getUpperBoundOperands());
+  // Steal the body of the old affine for op and erase it.
+  newPloop.region().takeBody(forOp.region());
+  forOp.erase();
+}
+
 // Returns success if any hoisting happened.
 LogicalResult mlir::hoistAffineIfOp(AffineIfOp ifOp, bool *folded) {
+  // Bail out early if the ifOp returns a result.  TODO: Consider how to
+  // properly support this case.
+  if (ifOp.getNumResults() != 0)
+    return failure();
+
   // Apply canonicalization patterns and folding - this is necessary for the
   // hoisting check to be correct (operands should be composed), and to be more
   // effective (no unused operands). Since the pattern rewriter's folding is
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index 13c5ed835c66b..fd0c6245e084b 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -87,7 +87,7 @@ LogicalResult GPUDialect::verifyOperationAttribute(Operation *op,
       return launchOp.emitOpError("kernel function is missing the '")
              << GPUDialect::getKernelFuncAttrName() << "' attribute";
 
-    // TODO(ntv,zinenko,herhut): if the kernel function has been converted to
+    // TODO: if the kernel function has been converted to
     // the LLVM dialect but the caller hasn't (which happens during the
     // separate compilation), do not check type correspondence as it would
     // require the verifier to be aware of the LLVM type conversion.
diff --git a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
index 1d72f9ec2690e..fcae3114188ae 100644
--- a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
@@ -54,7 +54,7 @@ static void injectGpuIndexOperations(Location loc, Region &launchFuncOpBody,
 }
 
 static bool isSinkingBeneficiary(Operation *op) {
-  return isa<ConstantOp>(op) || isa<DimOp>(op);
+  return isa<ConstantOp, DimOp>(op);
 }
 
 LogicalResult mlir::sinkOperationsIntoLaunchOp(gpu::LaunchOp launchOp) {
@@ -156,7 +156,7 @@ static gpu::GPUFuncOp outlineKernelFuncImpl(gpu::LaunchOp launchOp,
     map.map(operand.value(), entryBlock.getArgument(operand.index()));
 
   // Clone the region of the gpu.launch operation into the gpu.func operation.
-  // TODO(ravishankarm): If cloneInto can be modified such that if a mapping for
+  // TODO: If cloneInto can be modified such that if a mapping for
   // a block exists, that block will be used to clone operations into (at the
   // end of the block), instead of creating a new block, this would be much
   // cleaner.
diff --git a/mlir/lib/Dialect/GPU/Transforms/ParallelLoopMapper.cpp b/mlir/lib/Dialect/GPU/Transforms/ParallelLoopMapper.cpp
index 85609aa9f052a..b42929039a974 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ParallelLoopMapper.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ParallelLoopMapper.cpp
@@ -80,7 +80,7 @@ MappingLevel &operator++(MappingLevel &mappingLevel) {
 /// Computed the hardware id to use for a given mapping level. Will
 /// assign x,y and z hardware ids for the first 3 dimensions and use
 /// sequential after.
-/// TODO(ravishankarm/herhut) : Make this use x for the inner-most loop that is
+/// TODO: Make this use x for the inner-most loop that is
 /// distributed to map to x, the next innermost to y and the next innermost to
 /// z.
 static gpu::Processor getHardwareIdForMapping(MappingLevel level,
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index af9b1ca08b473..f448427099a44 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -104,8 +104,9 @@ static ParseResult parseCmpOp(OpAsmParser &parser, OperationState &result) {
     return parser.emitError(trailingTypeLoc, "expected LLVM IR dialect type");
   if (argType.getUnderlyingType()->isVectorTy())
     resultType = LLVMType::getVectorTy(
-        resultType, llvm::cast<llvm::VectorType>(argType.getUnderlyingType())
-                        ->getNumElements());
+        resultType,
+        llvm::cast<llvm::FixedVectorType>(argType.getUnderlyingType())
+            ->getNumElements());
 
   result.addTypes({resultType});
   return success();
@@ -857,25 +858,40 @@ static ParseResult parseReturnOp(OpAsmParser &parser, OperationState &result) {
 // Verifier for LLVM::AddressOfOp.
 //===----------------------------------------------------------------------===//
 
-GlobalOp AddressOfOp::getGlobal() {
-  Operation *module = getParentOp();
+template <typename OpTy>
+static OpTy lookupSymbolInModule(Operation *parent, StringRef name) {
+  Operation *module = parent;
   while (module && !satisfiesLLVMModule(module))
     module = module->getParentOp();
   assert(module && "unexpected operation outside of a module");
-  return dyn_cast_or_null<LLVM::GlobalOp>(
-      mlir::SymbolTable::lookupSymbolIn(module, global_name()));
+  return dyn_cast_or_null<OpTy>(
+      mlir::SymbolTable::lookupSymbolIn(module, name));
+}
+
+GlobalOp AddressOfOp::getGlobal() {
+  return lookupSymbolInModule<LLVM::GlobalOp>(getParentOp(), global_name());
+}
+
+LLVMFuncOp AddressOfOp::getFunction() {
+  return lookupSymbolInModule<LLVM::LLVMFuncOp>(getParentOp(), global_name());
 }
 
 static LogicalResult verify(AddressOfOp op) {
   auto global = op.getGlobal();
-  if (!global)
+  auto function = op.getFunction();
+  if (!global && !function)
+    return op.emitOpError(
+        "must reference a global defined by 'llvm.mlir.global' or 'llvm.func'");
+
+  if (global &&
+      global.getType().getPointerTo(global.addr_space().getZExtValue()) !=
+          op.getResult().getType())
     return op.emitOpError(
-        "must reference a global defined by 'llvm.mlir.global'");
+        "the type must be a pointer to the type of the referenced global");
 
-  if (global.getType().getPointerTo(global.addr_space().getZExtValue()) !=
-      op.getResult().getType())
+  if (function && function.getType().getPointerTo() != op.getResult().getType())
     return op.emitOpError(
-        "the type must be a pointer to the type of the referred global");
+        "the type must be a pointer to the type of the referenced function");
 
   return success();
 }
@@ -962,7 +978,7 @@ static LogicalResult verify(DialectCastOp op) {
     // * if we allow memrefs to cast from/to bare pointers, some users might
     // alternatively want metadata that only present in the descriptor.
     //
-    // TODO(timshen): re-evaluate the memref cast design when it's needed.
+    // TODO: re-evaluate the memref cast design when it's needed.
     return op.emitOpError("type must be non-index integer types, float types, "
                           "or vector of mentioned types.");
   };
@@ -1395,6 +1411,18 @@ static LogicalResult verify(LLVM::NullOp op) {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// Verification for LLVM::ConstantOp.
+//===----------------------------------------------------------------------===//
+
+static LogicalResult verify(LLVM::ConstantOp op) {
+  if (!(op.value().isa<IntegerAttr>() || op.value().isa<FloatAttr>() ||
+        op.value().isa<ElementsAttr>() || op.value().isa<StringAttr>()))
+    return op.emitOpError()
+           << "only supports integer, float, string or elements attributes";
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // Utility functions for parsing atomic ops
 //===----------------------------------------------------------------------===//
@@ -1788,7 +1816,8 @@ LLVMType LLVMType::getVectorElementType() {
       llvm::cast<llvm::VectorType>(getUnderlyingType())->getElementType());
 }
 unsigned LLVMType::getVectorNumElements() {
-  return llvm::cast<llvm::VectorType>(getUnderlyingType())->getNumElements();
+  return llvm::cast<llvm::FixedVectorType>(getUnderlyingType())
+      ->getNumElements();
 }
 bool LLVMType::isVectorTy() { return getUnderlyingType()->isVectorTy(); }
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index dab441fc26ff9..9a694a5e98997 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -141,7 +141,7 @@ static LogicalResult verify(MmaOp op) {
 // NVVMDialect initialization, type parsing, and registration.
 //===----------------------------------------------------------------------===//
 
-// TODO(herhut): This should be the llvm.nvvm dialect once this is supported.
+// TODO: This should be the llvm.nvvm dialect once this is supported.
 NVVMDialect::NVVMDialect(MLIRContext *context) : Dialect("nvvm", context) {
   addOperations<
 #define GET_OP_LIST
diff --git a/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
index aac1727c6be81..f3771dd577190 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
@@ -84,7 +84,7 @@ static ParseResult parseROCDLMubufStoreOp(OpAsmParser &parser,
 // ROCDLDialect initialization, type parsing, and registration.
 //===----------------------------------------------------------------------===//
 
-// TODO(herhut): This should be the llvm.rocdl dialect once this is supported.
+// TODO: This should be the llvm.rocdl dialect once this is supported.
 ROCDLDialect::ROCDLDialect(MLIRContext *context) : Dialect("rocdl", context) {
   addOperations<
 #define GET_OP_LIST
diff --git a/mlir/lib/Dialect/Linalg/Analysis/DependenceAnalysis.cpp b/mlir/lib/Dialect/Linalg/Analysis/DependenceAnalysis.cpp
index 4c218503ba173..af15740af2a8f 100644
--- a/mlir/lib/Dialect/Linalg/Analysis/DependenceAnalysis.cpp
+++ b/mlir/lib/Dialect/Linalg/Analysis/DependenceAnalysis.cpp
@@ -223,7 +223,7 @@ LinalgDependenceGraph::findOperationsWithCoveringDependences(
   SmallVector<Operation *, 8> res;
   // Consider an intermediate interleaved `interim` op, look for any dependence
   // to an aliasing view on a src -> op -> dst path.
-  // TODO(ntv) we are not considering paths yet, just interleaved positions.
+  // TODO: we are not considering paths yet, just interleaved positions.
   for (auto dt : types) {
     for (auto dependence : getDependencesFrom(src, dt)) {
       auto interimPos = linalgOpPositions.lookup(dependence.dependentOpView.op);
diff --git a/mlir/lib/Dialect/Linalg/EDSC/Builders.cpp b/mlir/lib/Dialect/Linalg/EDSC/Builders.cpp
index 5016ca9b3055d..b9ec01d3ec791 100644
--- a/mlir/lib/Dialect/Linalg/EDSC/Builders.cpp
+++ b/mlir/lib/Dialect/Linalg/EDSC/Builders.cpp
@@ -169,8 +169,8 @@ Operation *mlir::edsc::ops::linalg_generic_pointwise_max(StructuredIndexed I1,
                                                          StructuredIndexed I2,
                                                          StructuredIndexed O) {
   BinaryPointwiseOpBuilder binOp([](Value a, Value b) -> Value {
-    using edsc::op::operator>;
-    return std_select(a > b, a, b);
+    using edsc::op::sgt;
+    return std_select(sgt(a, b), a, b);
   });
   return linalg_generic_pointwise(binOp, I1, I2, O);
 }
@@ -226,7 +226,7 @@ Operation *mlir::edsc::ops::linalg_generic_conv_nhwc(Value vI, Value vW,
                                                      ArrayRef<int> strides,
                                                      ArrayRef<int> dilations) {
   MLIRContext *ctx = ScopedContext::getContext();
-  // TODO(ntv) some template magic to make everything rank-polymorphic.
+  // TODO: some template magic to make everything rank-polymorphic.
   assert((dilations.empty() || dilations.size() == 2) && "only 2-D conv atm");
   assert((strides.empty() || strides.size() == 2) && "only 2-D conv atm");
 
@@ -259,7 +259,7 @@ Operation *mlir::edsc::ops::linalg_generic_dilated_conv_nhwc(
     Value vI, Value vW, Value vO, int depth_multiplier, ArrayRef<int> strides,
     ArrayRef<int> dilations) {
   MLIRContext *ctx = ScopedContext::getContext();
-  // TODO(ntv) some template magic to make everything rank-polymorphic.
+  // TODO: some template magic to make everything rank-polymorphic.
   assert((dilations.empty() || dilations.size() == 2) && "only 2-D conv atm");
   assert((strides.empty() || strides.size() == 2) && "only 2-D conv atm");
 
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 8012a1087ee10..18ea31571aa47 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -447,7 +447,7 @@ static bool isReshapableDimBand(unsigned dim, unsigned extent,
     // proper symbol in the AffineExpr of a stride.
     if (ShapedType::isDynamic(sizes[dim + 1]))
       return false;
-    // TODO(ntv) Refine this by passing the proper nDims and nSymbols so we can
+    // TODO: Refine this by passing the proper nDims and nSymbols so we can
     // simplify on the fly and catch more reshapable cases.
     if (strides[idx] != strides[idx + 1] * sizes[idx + 1])
       return false;
@@ -520,7 +520,7 @@ computeReshapeCollapsedType(MemRefType type,
 
 /// Helper functions assert Attribute of the proper type in attr and returns the
 /// corresponding vector.
-/// TODO(rridle,ntv) this should be evolved into a generic
+/// TODO: this should be evolved into a generic
 /// `getRangeOfType<AffineMap>(ArrayAttr attrs)` that does not copy.
 static SmallVector<AffineMap, 4> getAffineMaps(ArrayAttr attrs) {
   return llvm::to_vector<8>(llvm::map_range(
@@ -713,7 +713,7 @@ static LogicalResult verify(TensorReshapeOp op) {
   if (failed(verifyReshapeLikeTypes(op, expandedType, collapsedType)))
     return failure();
   auto maps = getAffineMaps(op.reassociation());
-  // TODO(ntv): expanding a ? with a non-constant is under-specified. Error
+  // TODO: expanding a ? with a non-constant is under-specified. Error
   // out.
   RankedTensorType expectedType =
       computeTensorReshapeCollapsedType(expandedType, maps);
@@ -744,7 +744,7 @@ void mlir::linalg::SliceOp::build(OpBuilder &b, OperationState &result,
   (void)res;
 
   unsigned rank = memRefType.getRank();
-  // TODO(ntv): propagate static size and stride information when available.
+  // TODO: propagate static size and stride information when available.
   SmallVector<int64_t, 4> sizes(rank, -1); // -1 encodes dynamic size.
   result.addTypes({MemRefType::Builder(memRefType)
                        .setShape(sizes)
@@ -1075,7 +1075,7 @@ mlir::linalg::weightedPoolingInputIndex(PoolingOp op,
   SmallVector<AffineExpr, 4> res;
   res.reserve(outputDims.size());
   for (unsigned i = 0, e = outputDims.size(); i < e; ++i) {
-    // TODO(ntv): add a level of indirection to linalg.generic.
+    // TODO: add a level of indirection to linalg.generic.
     auto expr = op.getStride(i) * outputDims[i] +
                 op.getDilation(i) * windowDims[i] - op.getLowPad(i);
     res.push_back(expr);
@@ -1137,7 +1137,7 @@ std::string mlir::linalg::generateLibraryCallName(Operation *op) {
   return ss.str();
 }
 
-// TODO(ntv, rriddle): Consider making all this boilerplate easy to autogenerate
+// TODO: Consider making all this boilerplate easy to autogenerate
 // with Tablegen. This seems a desirable property in the context of OpInterfaces
 // where a Linalg "named" op **isa** LinalgOp.
 LogicalResult ConvOp::fold(ArrayRef<Attribute>,
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
index 48231642eae75..d67126c21f3e5 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
@@ -279,12 +279,12 @@ bool mlir::linalg::isFusableInto(const LinalgDependenceGraph &graph,
     return false;
   }
   if (auto convOp = dyn_cast<linalg::ConvOp>(producer.getOperation())) {
-    // TODO(ntv): add a level of indirection to linalg.generic.
+    // TODO: add a level of indirection to linalg.generic.
     if (convOp.padding())
       return false;
   }
   if (auto convOp = dyn_cast<linalg::ConvOp>(consumer.getOperation())) {
-    // TODO(ntv): add a level of indirection to linalg.generic.
+    // TODO: add a level of indirection to linalg.generic.
     if (convOp.padding())
       return false;
   }
@@ -403,7 +403,7 @@ static void fuseLinalgOpsGreedily(FuncOp f) {
       linalgOps.push_back(op);
   });
 
-  // TODO(pifon, ntv): LinalgDependenceGraph should be able to update itself.
+  // TODO: LinalgDependenceGraph should be able to update itself.
   // The current naive and expensive reconstruction of the graph should be
   // removed.
   for (auto *op : llvm::reverse(linalgOps)) {
@@ -923,11 +923,11 @@ Operation *mlir::linalg::fuseTensorOps(PatternRewriter &rewriter,
     return nullptr;
 
   // Fuse when consumer is GenericOp or IndexedGenericOp.
-  if (isa<GenericOp>(consumer) || isa<IndexedGenericOp>(consumer)) {
+  if (isa<GenericOp, IndexedGenericOp>(consumer)) {
     auto linalgOpConsumer = cast<LinalgOp>(consumer);
     if (!linalgOpConsumer.hasTensorSemantics())
       return nullptr;
-    if (isa<GenericOp>(producer) || isa<IndexedGenericOp>(producer)) {
+    if (isa<GenericOp, IndexedGenericOp>(producer)) {
       auto linalgOpProducer = cast<LinalgOp>(producer);
       if (linalgOpProducer.hasTensorSemantics())
         return FuseGenericOpsOnTensors::fuse(linalgOpProducer, linalgOpConsumer,
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index eeabdeb815dd9..cb7540b46cf84 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -38,7 +38,7 @@ void mlir::linalg::hoistViewAllocOps(FuncOp func) {
   while (changed) {
     changed = false;
     func.walk([&changed](Operation *op) {
-      if (!isa<AllocOp>(op) && !isa<AllocaOp>(op) && !isa<DeallocOp>(op))
+      if (!isa<AllocOp, AllocaOp, DeallocOp>(op))
         return;
 
       LLVM_DEBUG(DBGS() << "Candidate for hoisting: " << *op << "\n");
@@ -64,15 +64,14 @@ void mlir::linalg::hoistViewAllocOps(FuncOp func) {
         v = op->getResult(0);
       }
       if (v && !llvm::all_of(v.getUses(), [&](OpOperand &operand) {
-            return isa<ViewLikeOpInterface>(operand.getOwner()) ||
-                   isa<DeallocOp>(operand.getOwner());
+            return isa<ViewLikeOpInterface, DeallocOp>(operand.getOwner());
           })) {
         LLVM_DEBUG(DBGS() << "Found non view-like or dealloc use: bail\n");
         return;
       }
 
       // Move AllocOp before the loop.
-      if (isa<AllocOp>(op) || isa<AllocaOp>(op))
+      if (isa<AllocOp, AllocaOp>(op))
         loop.moveOutOfLoop({op});
       else // Move DeallocOp outside of the loop.
         op->moveAfter(loop);
@@ -81,6 +80,42 @@ void mlir::linalg::hoistViewAllocOps(FuncOp func) {
   }
 }
 
+/// Return true if we can prove that the transfer operations access dijoint
+/// memory.
+template <typename TransferTypeA, typename TransferTypeB>
+static bool isDisjoint(TransferTypeA transferA, TransferTypeB transferB) {
+  if (transferA.memref() != transferB.memref())
+    return false;
+  // For simplicity only look at transfer of same type.
+  if (transferA.getVectorType() != transferB.getVectorType())
+    return false;
+  unsigned rankOffset = transferA.getLeadingMemRefRank();
+  for (unsigned i = 0, e = transferA.indices().size(); i < e; i++) {
+    auto indexA = transferA.indices()[i].template getDefiningOp<ConstantOp>();
+    auto indexB = transferB.indices()[i].template getDefiningOp<ConstantOp>();
+    // If any of the indices are dynamic we cannot prove anything.
+    if (!indexA || !indexB)
+      continue;
+
+    if (i < rankOffset) {
+      // For dimension used as index if we can prove that index are different we
+      // know we are accessing disjoint slices.
+      if (indexA.getValue().template cast<IntegerAttr>().getInt() !=
+          indexB.getValue().template cast<IntegerAttr>().getInt())
+        return true;
+    } else {
+      // For this dimension, we slice a part of the memref we need to make sure
+      // the intervals accessed don't overlap.
+      int64_t distance =
+          std::abs(indexA.getValue().template cast<IntegerAttr>().getInt() -
+                   indexB.getValue().template cast<IntegerAttr>().getInt());
+      if (distance >= transferA.getVectorType().getDimSize(i - rankOffset))
+        return true;
+    }
+  }
+  return false;
+}
+
 void mlir::linalg::hoistRedundantVectorTransfers(FuncOp func) {
   bool changed = true;
   while (changed) {
@@ -130,10 +165,11 @@ void mlir::linalg::hoistRedundantVectorTransfers(FuncOp func) {
 
       // Approximate aliasing by checking that:
       //   1. indices are the same,
-      //   2. no other use either dominates the transfer_read or is dominated
-      //   by the transfer_write (i.e. aliasing between the write and the read
-      //   across the loop).
-      if (transferRead.indices() != transferWrite.indices())
+      //   2. no other operations in the loop access the same memref except
+      //      for transfer_read/transfer_write accessing statically disjoint
+      //      slices.
+      if (transferRead.indices() != transferWrite.indices() &&
+          transferRead.getVectorType() == transferWrite.getVectorType())
         return WalkResult::advance();
 
       // TODO: may want to memoize this information for performance but it
@@ -141,11 +177,26 @@ void mlir::linalg::hoistRedundantVectorTransfers(FuncOp func) {
       DominanceInfo dom(loop);
       if (!dom.properlyDominates(transferRead.getOperation(), transferWrite))
         return WalkResult::advance();
-      for (auto &use : transferRead.memref().getUses())
-        if (dom.properlyDominates(use.getOwner(),
-                                  transferRead.getOperation()) ||
-            dom.properlyDominates(transferWrite, use.getOwner()))
+      for (auto &use : transferRead.memref().getUses()) {
+        if (!dom.properlyDominates(loop, use.getOwner()))
+          continue;
+        if (use.getOwner() == transferRead.getOperation() ||
+            use.getOwner() == transferWrite.getOperation())
+          continue;
+        if (auto transferWriteUse =
+                dyn_cast<vector::TransferWriteOp>(use.getOwner())) {
+          if (!isDisjoint(transferWrite, transferWriteUse))
+            return WalkResult::advance();
+        } else if (auto transferReadUse =
+                       dyn_cast<vector::TransferReadOp>(use.getOwner())) {
+          if (!isDisjoint(transferWrite, transferReadUse))
+            return WalkResult::advance();
+        } else {
+          // Unknown use, we cannot prove that it doesn't alias with the
+          // transferRead/transferWrite operations.
           return WalkResult::advance();
+        }
+      }
 
       // Hoist read before.
       if (failed(loop.moveOutOfLoop({transferRead})))
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
index 71e4969d46570..b191cbea94824 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
@@ -37,7 +37,7 @@ LogicalResult mlir::linalg::interchangeGenericLinalgOpPrecondition(
   if (interchangeVector.empty())
     return failure();
   // Transformation applies to generic ops only.
-  if (!isa<GenericOp>(op) && !isa<IndexedGenericOp>(op))
+  if (!isa<GenericOp, IndexedGenericOp>(op))
     return failure();
   LinalgOp linOp = cast<LinalgOp>(op);
   // Transformation applies to buffers only.
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
index d031712ce5a9f..6a1d00fe620c6 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
@@ -165,7 +165,7 @@ void emitScalarImplementation(ArrayRef<Value> allIvs,
   SmallVector<Value, 4> indexedValues;
   indexedValues.reserve(nInputs + nOutputs);
 
-  // TODO(mravishankar): Avoid the loads if the corresponding argument of the
+  // TODO: Avoid the loads if the corresponding argument of the
   // region has no uses.
   // 1.a. Emit load from input views.
   for (unsigned i = 0; i < nInputs; ++i) {
@@ -183,7 +183,7 @@ void emitScalarImplementation(ArrayRef<Value> allIvs,
         IndexedValueType(linalgOp.getOutputBuffer(i))(indexing));
   }
 
-  // TODO(ntv): When a region inliner exists, use it.
+  // TODO: When a region inliner exists, use it.
   // 2. Inline region, currently only works for a single basic block.
   // 3. Emit store.
   SmallVector<SmallVector<Value, 8>, 8> indexing;
@@ -246,7 +246,7 @@ void emitScalarImplementation(ArrayRef<Value> allIvs, DotOp dotOp) {
 template <typename IndexedValueType>
 Value getConvOpInput(ConvOp convOp, StdIndexedValue im,
                      MutableArrayRef<Value> imIdx) {
-  // TODO(ntv): add a level of indirection to linalg.generic.
+  // TODO: add a level of indirection to linalg.generic.
   if (!convOp.padding())
     return im(imIdx);
 
@@ -263,16 +263,16 @@ Value getConvOpInput(ConvOp convOp, StdIndexedValue im,
       continue;
     }
 
-    using edsc::op::operator<;
-    using edsc::op::operator>=;
+    using edsc::op::sge;
+    using edsc::op::slt;
     using edsc::op::operator||;
-    Value leftOutOfBound = dim < zeroIndex;
+    Value leftOutOfBound = slt(dim, zeroIndex);
     if (conds.empty())
       conds.push_back(leftOutOfBound);
     else
       conds.push_back(conds.back() || leftOutOfBound);
     Value rightBound = std_dim(convOp.input(), idx);
-    conds.push_back(conds.back() || (dim >= rightBound));
+    conds.push_back(conds.back() || (sge(dim, rightBound)));
 
     // When padding is involved, the indices will only be shifted to negative,
     // so having a max op is enough.
@@ -333,23 +333,28 @@ static void emitScalarImplementation(ArrayRef<Value> allIvs, ConvOp convOp) {
 
 template <typename IndexedValueType>
 void emitScalarImplementation(ArrayRef<Value> allIvs, PoolingMaxOp op) {
-  auto indices = getInputAndOutputIndices(allIvs, op);
+  InputAndOutputIndices indices = getInputAndOutputIndices(allIvs, op);
   // Emit scalar form.
-  Value lhs = std_load(op.output(), indices.outputs);
-  Value rhs = std_load(op.input(), indices.inputs);
-  using edsc::op::operator>;
-  Value maxValue = std_select(lhs > rhs, lhs, rhs);
-  std_store(maxValue, op.output(), indices.outputs);
+  IndexedValueType output(op.output());
+  IndexedValueType input(op.input());
+  Value lhs = output(indices.outputs);
+  Value rhs = input(indices.inputs);
+  using edsc::op::sgt;
+  Value maxValue = std_select(sgt(lhs, rhs), lhs, rhs);
+  output(indices.outputs) = maxValue;
 }
+
 template <typename IndexedValueType>
 void emitScalarImplementation(ArrayRef<Value> allIvs, PoolingMinOp op) {
-  auto indices = getInputAndOutputIndices(allIvs, op);
+  InputAndOutputIndices indices = getInputAndOutputIndices(allIvs, op);
   // Emit scalar form.
-  Value lhs = std_load(op.output(), indices.outputs);
-  Value rhs = std_load(op.input(), indices.inputs);
-  using edsc::op::operator<;
-  Value minValue = std_select(lhs < rhs, lhs, rhs);
-  std_store(minValue, op.output(), indices.outputs);
+  IndexedValueType output(op.output());
+  IndexedValueType input(op.input());
+  Value lhs = output(indices.outputs);
+  Value rhs = input(indices.inputs);
+  using edsc::op::slt;
+  Value minValue = std_select(slt(lhs, rhs), lhs, rhs);
+  output(indices.outputs) = minValue;
 }
 template <typename IndexedValueType>
 void emitScalarImplementation(ArrayRef<Value> allIvs, PoolingSumOp op) {
@@ -404,7 +409,7 @@ static void emitScalarImplementation(ArrayRef<Value> allIvs,
   for (unsigned i = 0; i < nLoops; ++i)
     indexedValues.push_back(allIvs[i]);
 
-  // TODO(mravishankar): Avoid the loads if the corresponding argument of the
+  // TODO: Avoid the loads if the corresponding argument of the
   // region has no uses.
   // 1.a. Emit load from input views.
   for (unsigned i = 0; i < nInputs; ++i) {
@@ -423,7 +428,7 @@ static void emitScalarImplementation(ArrayRef<Value> allIvs,
         IndexedValueType(indexedGenericOp.getOutputBuffer(i))(indexing));
   }
 
-  // TODO(ntv): When a region inliner exists, use it.
+  // TODO: When a region inliner exists, use it.
   // 2. Inline region, currently only works for a single basic block.
   // 3. Emit store.
   SmallVector<SmallVector<Value, 8>, 8> indexing;
@@ -555,7 +560,7 @@ static void lowerLinalgToLoopsImpl(FuncOp funcOp, MLIRContext *context) {
   OwningRewritePatternList patterns;
   // Canonicalization and folding patterns applied greedily allow cleaning up
   // the emitted IR on the fly.
-  // TODO(ntv) fold view and subview ops?
+  // TODO: fold view and subview ops?
   insertPatterns<LoopType,
 #define GET_OP_LIST
 #include "mlir/Dialect/Linalg/IR/LinalgStructuredOps.cpp.inc"
@@ -604,8 +609,8 @@ mlir::createConvertLinalgToAffineLoopsPass() {
 
 // TODO: gradually remove this layer as more ops become "named".
 template <typename LoopTy>
-Optional<LinalgLoops> linalgOpToLoopsImplSwitch(Operation *op,
-                                                OpBuilder &builder) {
+static Optional<LinalgLoops> linalgOpToLoopsImplSwitch(Operation *op,
+                                                       OpBuilder &builder) {
   assert(isa<LinalgOp>(op) && "LinalgOp expected");
   if (isa<CopyOp>(op))
     return linalgOpToLoopsImpl<LoopTy, CopyOp>(op, builder);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
index de8514f0fa41a..a5323f4b7687b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
@@ -67,7 +67,8 @@ static Value extractSmallestConstantBoundingSize(OpBuilder &b, Location loc,
 
 /// Alloc a new buffer of `size`. If `dynamicBuffers` is true allocate exactly
 /// the size needed, otherwise try to allocate a static bounding box.
-static Value allocBuffer(Type elementType, Value size, bool dynamicBuffers,
+static Value allocBuffer(const LinalgPromotionOptions &options,
+                         Type elementType, Value size, bool dynamicBuffers,
                          OperationFolder *folder,
                          Optional<unsigned> alignment = None) {
   auto *ctx = size.getContext();
@@ -78,23 +79,34 @@ static Value allocBuffer(Type elementType, Value size, bool dynamicBuffers,
         IntegerAttr::get(IntegerType::get(64, ctx), alignment.getValue());
   if (!dynamicBuffers)
     if (auto cst = size.getDefiningOp<ConstantIndexOp>())
-      return std_alloc(
-          MemRefType::get(width * cst.getValue(), IntegerType::get(8, ctx)),
-          ValueRange{}, alignment_attr);
+      return options.useAlloca
+                 ? std_alloca(MemRefType::get(width * cst.getValue(),
+                                              IntegerType::get(8, ctx)),
+                              ValueRange{}, alignment_attr)
+                       .value
+                 : std_alloc(MemRefType::get(width * cst.getValue(),
+                                             IntegerType::get(8, ctx)),
+                             ValueRange{}, alignment_attr)
+                       .value;
   Value mul =
       folded_std_muli(folder, folded_std_constant_index(folder, width), size);
-  return std_alloc(MemRefType::get(-1, IntegerType::get(8, ctx)), mul,
-                   alignment_attr);
+  return options.useAlloca
+             ? std_alloca(MemRefType::get(-1, IntegerType::get(8, ctx)), mul,
+                          alignment_attr)
+                   .value
+             : std_alloc(MemRefType::get(-1, IntegerType::get(8, ctx)), mul,
+                         alignment_attr)
+                   .value;
 }
 
 /// Default allocation callback function. This allocates a promoted buffer when
 /// no call back to do so is provided. The default is to allocate a
 /// memref<..xi8> and return a view to get a memref type of shape
 /// boundingSubViewSize.
-static Optional<Value>
-allocBufferCallBack(OpBuilder &builder, SubViewOp subView,
-                    ArrayRef<Value> boundingSubViewSize, bool dynamicBuffers,
-                    Optional<unsigned> alignment, OperationFolder *folder) {
+static Optional<Value> defaultAllocBufferCallBack(
+    const LinalgPromotionOptions &options, OpBuilder &builder,
+    SubViewOp subView, ArrayRef<Value> boundingSubViewSize, bool dynamicBuffers,
+    Optional<unsigned> alignment, OperationFolder *folder) {
   ShapedType viewType = subView.getType();
   int64_t rank = viewType.getRank();
   (void)rank;
@@ -105,7 +117,7 @@ allocBufferCallBack(OpBuilder &builder, SubViewOp subView,
   Value allocSize = one;
   for (auto size : llvm::enumerate(boundingSubViewSize))
     allocSize = folded_std_muli(folder, allocSize, size.value());
-  Value buffer = allocBuffer(viewType.getElementType(), allocSize,
+  Value buffer = allocBuffer(options, viewType.getElementType(), allocSize,
                              dynamicBuffers, folder, alignment);
   SmallVector<int64_t, 4> dynSizes(boundingSubViewSize.size(),
                                    ShapedType::kDynamicSize);
@@ -118,10 +130,13 @@ allocBufferCallBack(OpBuilder &builder, SubViewOp subView,
 /// Default implementation of deallocation of the buffer use for promotion. It
 /// expects to get the same value that the default allocation method returned,
 /// i.e. result of a ViewOp.
-static LogicalResult deallocCallBack(OpBuilder &b, Value fullLocalView) {
+static LogicalResult
+defaultDeallocBufferCallBack(const LinalgPromotionOptions &options,
+                             OpBuilder &b, Value fullLocalView) {
   auto viewOp = fullLocalView.getDefiningOp<ViewOp>();
   assert(viewOp && "expected full local view to be a ViewOp");
-  std_dealloc(viewOp.source());
+  if (!options.useAlloca)
+    std_dealloc(viewOp.source());
   return success();
 }
 
@@ -182,11 +197,16 @@ LinalgOpInstancePromotionOptions::LinalgOpInstancePromotionOptions(
                             : [&](OpBuilder &builder, SubViewOp subViewOp,
                                   ArrayRef<Value> boundingSubViewSize,
                                   OperationFolder *folder) -> Optional<Value> {
-        return allocBufferCallBack(builder, subViewOp, boundingSubViewSize,
-                                   dynamicBuffers, alignment, folder);
+        return defaultAllocBufferCallBack(options, builder, subViewOp,
+                                          boundingSubViewSize, dynamicBuffers,
+                                          alignment, folder);
       });
   deallocationFn =
-      (options.deallocationFn ? *(options.deallocationFn) : deallocCallBack);
+      (options.deallocationFn
+           ? *(options.deallocationFn)
+           : [&](OpBuilder &b, Value buffer) {
+               return defaultDeallocBufferCallBack(options, b, buffer);
+             });
   auto defaultCopyCallBack = [&](OpBuilder &builder, Value src,
                                  Value dst) -> LogicalResult {
     linalg_copy(src, dst);
@@ -296,7 +316,7 @@ promoteSubViews(OpBuilder &b, LinalgOp op,
   assert(op.hasBufferSemantics() && "expected linalg op with buffer semantics");
 
   if (auto convOp = dyn_cast<linalg::ConvOp>(op.getOperation())) {
-    // TODO(ntv): add a level of indirection to linalg.generic.
+    // TODO: add a level of indirection to linalg.generic.
     if (convOp.padding())
       return {};
   }
@@ -344,9 +364,8 @@ promoteSubViews(OpBuilder &b, LinalgOp op,
   }
 
   // 4. Dealloc all local buffers.
-  for (const auto &pi : *promotedBuffersAndViews) {
+  for (const auto &pi : *promotedBuffersAndViews)
     options.deallocationFn(b, pi.second.fullLocalView);
-  }
   return op;
 }
 
@@ -383,14 +402,17 @@ Optional<LinalgOp> mlir::linalg::promoteSubViews(OpBuilder &b,
 namespace {
 struct LinalgPromotionPass : public LinalgPromotionBase<LinalgPromotionPass> {
   LinalgPromotionPass() = default;
-  LinalgPromotionPass(bool dynamicBuffers) {
+  LinalgPromotionPass(bool dynamicBuffers, bool useAlloca) {
     this->dynamicBuffers = dynamicBuffers;
+    this->useAlloca = useAlloca;
   }
 
   void runOnFunction() override {
     OperationFolder folder(&getContext());
     getFunction().walk([this, &folder](LinalgOp op) {
-      auto options = LinalgPromotionOptions().setDynamicBuffers(dynamicBuffers);
+      auto options = LinalgPromotionOptions()
+                         .setDynamicBuffers(dynamicBuffers)
+                         .setUseAlloca(useAlloca);
       if (failed(promoteSubviewsPrecondition(op, options)))
         return;
       LLVM_DEBUG(llvm::dbgs() << "Promote: " << *(op.getOperation()) << "\n");
@@ -403,8 +425,8 @@ struct LinalgPromotionPass : public LinalgPromotionBase<LinalgPromotionPass> {
 
 // TODO: support more transformation options in the pass.
 std::unique_ptr<OperationPass<FuncOp>>
-mlir::createLinalgPromotionPass(bool dynamicBuffers) {
-  return std::make_unique<LinalgPromotionPass>(dynamicBuffers);
+mlir::createLinalgPromotionPass(bool dynamicBuffers, bool useAlloca) {
+  return std::make_unique<LinalgPromotionPass>(dynamicBuffers, useAlloca);
 }
 std::unique_ptr<OperationPass<FuncOp>> mlir::createLinalgPromotionPass() {
   return std::make_unique<LinalgPromotionPass>();
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
index bde163e3ee723..6ffc181fed67a 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
@@ -161,7 +161,7 @@ struct TileCheck : public AffineExprVisitor<TileCheck> {
 //   }
 // }
 //
-// TODO(pifon, ntv): Investigate whether mixing implicit and explicit indices
+// TODO: Investigate whether mixing implicit and explicit indices
 // does not lead to losing information.
 static void transformIndexedGenericOpIndices(
     OpBuilder &b, LinalgOp op, SmallVectorImpl<Value> &ivs,
@@ -176,7 +176,7 @@ static void transformIndexedGenericOpIndices(
   // that refers to an existing function symbol. The `fun` function call will be
   // inserted in the loop body in that case.
   //
-  // TODO(pifon): Add support for `linalg.indexed_generic` with `fun` attribute.
+  // TODO: Add support for `linalg.indexed_generic` with `fun` attribute.
   auto &region = indexedGenericOp.region();
   if (region.empty()) {
     indexedGenericOp.emitOpError("expected a region");
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 763961311d0b7..bbdb8e7b46b4c 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -36,8 +36,7 @@ using llvm::dbgs;
 
 #define DEBUG_TYPE "linalg-vectorization"
 
-static bool hasMultiplyAddBody(linalg::GenericOp op) {
-  auto &r = op.region();
+static bool hasMultiplyAddBody(Region &r) {
   if (!llvm::hasSingleElement(r))
     return false;
   if (!llvm::hasNItems(r.front().begin(), r.front().end(), 3))
@@ -59,14 +58,26 @@ static bool hasMultiplyAddBody(linalg::GenericOp op) {
 }
 
 // TODO: Should be Tablegen'd from a single source that generates the op itself.
-static bool isRowMajorMatmul(linalg::GenericOp genericOp) {
-  return genericOp.getNumInputs() == 2 && genericOp.getNumOutputs() == 1 &&
-         isRowMajorMatmul(genericOp.indexing_maps()) &&
-         hasMultiplyAddBody(genericOp);
+static LogicalResult isContraction(Operation *op) {
+  // TODO: interface for named ops.
+  if (isa<linalg::BatchMatmulOp, linalg::MatmulOp, linalg::MatvecOp,
+          linalg::DotOp>(op))
+    return success();
+
+  auto genericOp = dyn_cast<linalg::GenericOp>(op);
+  if (!genericOp)
+    return failure();
+
+  auto mapRange =
+      genericOp.indexing_maps().getAsRange<AffineMapAttr, AffineMap>();
+
+  return success(
+      genericOp.getNumInputs() == 2 && genericOp.getNumOutputs() == 1 &&
+      llvm::all_of(mapRange,
+                   [](AffineMap m) { return m.isProjectedPermutation(); }) &&
+      hasMultiplyAddBody(genericOp.region()));
 }
 
-// TODO: This is in fact much more general than just vectorization for matmul
-// and fill ops.
 LogicalResult mlir::linalg::vectorizeLinalgOpPrecondition(Operation *op) {
   auto linalgOp = cast<linalg::LinalgOp>(op);
   // All types must be static shape to go to vector.
@@ -76,33 +87,16 @@ LogicalResult mlir::linalg::vectorizeLinalgOpPrecondition(Operation *op) {
   for (Type outputTensorType : linalgOp.getOutputTensorTypes())
     if (!outputTensorType.cast<ShapedType>().hasStaticShape())
       return failure();
-  if (isa<linalg::MatmulOp>(op) || isa<linalg::FillOp>(op))
-    return success();
 
-  auto genericOp = dyn_cast<linalg::GenericOp>(op);
-  if (!genericOp || !::isRowMajorMatmul(genericOp))
-    return failure();
+  if (isa<linalg::FillOp>(op))
+    return success();
 
-  // TODO(ntv): non-identity layout.
-  auto isStaticMemRefWithIdentityLayout = [](Value v) {
-    auto m = v.getType().dyn_cast<MemRefType>();
-    if (!m || !m.hasStaticShape() || !m.getAffineMaps().empty())
-      return false;
-    return true;
-  };
-  return success(llvm::all_of(genericOp.getInputsAndOutputBuffers(),
-                              isStaticMemRefWithIdentityLayout));
+  return isContraction(op);
 }
 
 void mlir::linalg::vectorizeLinalgOp(OpBuilder &builder, Operation *op) {
   assert(succeeded(vectorizeLinalgOpPrecondition(op)));
 
-  if (auto convOp = dyn_cast<linalg::ConvOp>(op)) {
-    // TODO: add a level of indirection to linalg.generic.
-    if (convOp.padding())
-      llvm_unreachable("Unexpected conv with padding");
-  }
-
   StringRef dbgPref = "\n[" DEBUG_TYPE "]: ";
   (void)dbgPref;
   edsc::ScopedContext scope(builder, op->getLoc());
@@ -117,33 +111,47 @@ void mlir::linalg::vectorizeLinalgOp(OpBuilder &builder, Operation *op) {
     return;
   }
 
-  // Vectorize other ops as vector contraction (currently only matmul).
+  assert(succeeded(isContraction(op)) && "Expected contraction");
+
+  // Vectorize other ops as vector contraction.
+  // TODO: interface.
   LLVM_DEBUG(dbgs() << dbgPref
                     << "Rewrite linalg op as vector.contract: " << *op);
+  // In the case of 0-D memrefs, return null and special case to scalar load or
+  // store later.
   auto extractVectorTypeFromScalarView = [](Value v) {
     MemRefType mt = v.getType().cast<MemRefType>();
-    return VectorType::get(mt.getShape(), mt.getElementType());
+    return mt.getShape().empty()
+               ? VectorType()
+               : VectorType::get(mt.getShape(), mt.getElementType());
   };
   auto linalgOp = cast<linalg::LinalgOp>(op);
   Value viewA = linalgOp.getInput(0);
   Value viewB = linalgOp.getInput(1);
   Value viewC = linalgOp.getOutputBuffer(0);
+  VectorType vtA = extractVectorTypeFromScalarView(viewA);
+  VectorType vtB = extractVectorTypeFromScalarView(viewB);
+  VectorType vtC = extractVectorTypeFromScalarView(viewC);
   Value zero = std_constant_index(0);
-  SmallVector<Value, 4> indicesA(linalgOp.getInputShapedType(0).getRank(),
-                                 zero);
-  SmallVector<Value, 4> indicesB(linalgOp.getInputShapedType(1).getRank(),
-                                 zero);
-  SmallVector<Value, 4> indicesC(linalgOp.getOutputShapedType(0).getRank(),
-                                 zero);
-  Value a = vector_transfer_read(extractVectorTypeFromScalarView(viewA), viewA,
-                                 indicesA);
-  Value b = vector_transfer_read(extractVectorTypeFromScalarView(viewB), viewB,
-                                 indicesB);
-  Value c = vector_transfer_read(extractVectorTypeFromScalarView(viewC), viewC,
-                                 indicesC);
+  SmallVector<Value, 4> indicesA, indicesB, indicesC;
+  if (vtA)
+    indicesA = SmallVector<Value, 4>(vtA.getRank(), zero);
+  if (vtB)
+    indicesB = SmallVector<Value, 4>(vtB.getRank(), zero);
+  if (vtC)
+    indicesC = SmallVector<Value, 4>(vtC.getRank(), zero);
+  Value a = vtA ? vector_transfer_read(vtA, viewA, indicesA).value
+                : std_load(viewA, indicesA).value;
+  Value b = vtB ? vector_transfer_read(vtB, viewB, indicesB).value
+                : std_load(viewB, indicesB).value;
+  Value c = vtC ? vector_transfer_read(vtC, viewC, indicesC).value
+                : std_load(viewC, indicesC).value;
   Value res = vector_contract(a, b, c, linalgOp.indexing_maps(),
                               linalgOp.iterator_types());
-  vector_transfer_write(res, viewC, indicesC);
+  if (vtC)
+    vector_transfer_write(res, viewC, indicesC);
+  else
+    std_store(res, viewC, indicesC);
 }
 
 /// Check whether there is any interleaved use of any `values` between `firstOp`
diff --git a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
index 5bba11420d086..a9d5e2028c227 100644
--- a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
@@ -181,7 +181,7 @@ void GenerateLoopNest<AffineForOp>::doit(
 /// of the innermost loop is populated by `bodyBuilderFn` that accepts a range
 /// of induction variables for all loops. `ivStorage` is used to store the
 /// partial list of induction variables.
-// TODO(zinenko,ntv): this function can be made iterative instead. However, it
+// TODO: this function can be made iterative instead. However, it
 // will have at most as many recursive calls as nested loops, which rarely
 // exceeds 10.
 static void
diff --git a/mlir/lib/Dialect/Quant/IR/QuantOps.cpp b/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
index b0dc1fa106797..07f881fbc52c5 100644
--- a/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
+++ b/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
@@ -46,7 +46,7 @@ OpFoldResult StorageCastOp::fold(ArrayRef<Attribute> operands) {
 static bool isValidQuantizationSpec(Attribute quantSpec, Type expressed) {
   if (auto typeAttr = quantSpec.dyn_cast<TypeAttr>()) {
     Type spec = typeAttr.getValue();
-    if (spec.isa<TensorType>() || spec.isa<VectorType>())
+    if (spec.isa<TensorType, VectorType>())
       return false;
 
     // The spec should be either a quantized type which is compatible to the
diff --git a/mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp b/mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp
index 2ff23123b474e..88eb314a852ba 100644
--- a/mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp
@@ -69,8 +69,7 @@ QuantizedConstRewrite::matchAndRewrite(QuantizeCastOp qbarrier,
   }
 
   // Is the constant value a type expressed in a way that we support?
-  if (!value.isa<FloatAttr>() && !value.isa<DenseElementsAttr>() &&
-      !value.isa<SparseElementsAttr>()) {
+  if (!value.isa<FloatAttr, DenseElementsAttr, SparseElementsAttr>()) {
     return failure();
   }
 
diff --git a/mlir/lib/Dialect/Quant/Utils/FakeQuantSupport.cpp b/mlir/lib/Dialect/Quant/Utils/FakeQuantSupport.cpp
index 71b7e042aa1ab..37b72af934240 100644
--- a/mlir/lib/Dialect/Quant/Utils/FakeQuantSupport.cpp
+++ b/mlir/lib/Dialect/Quant/Utils/FakeQuantSupport.cpp
@@ -61,7 +61,7 @@ static bool getDefaultStorageParams(unsigned numBits, bool narrowRange,
 // point is derived from the shifted range, and the scale isn't changed. As
 // a consequence some values, which are supposed in the original [rmin, rmax]
 // range will be outside the shifted range and be clamped during quantization.
-// TODO(fengliuai): we should nudge the scale as well, but that requires the
+// TODO: we should nudge the scale as well, but that requires the
 // fake quant op used in the training to use the nudged scale as well.
 static void getNudgedScaleAndZeroPoint(int64_t qmin, int64_t qmax, double rmin,
                                        double rmax, double &scale,
diff --git a/mlir/lib/Dialect/Quant/Utils/QuantizeUtils.cpp b/mlir/lib/Dialect/Quant/Utils/QuantizeUtils.cpp
index 1fbc60962fd17..7b475d31914b1 100644
--- a/mlir/lib/Dialect/Quant/Utils/QuantizeUtils.cpp
+++ b/mlir/lib/Dialect/Quant/Utils/QuantizeUtils.cpp
@@ -137,7 +137,7 @@ Attribute mlir::quant::quantizeAttr(Attribute realValue,
                  quantizedElementType.dyn_cast<UniformQuantizedPerAxisType>()) {
     UniformQuantizedPerAxisValueConverter converter(uniformQuantizedPerAxis);
     auto converted = converter.convert(realValue);
-    // TODO(fengliuai): why we need this outConvertedType? remove it?
+    // TODO: why we need this outConvertedType? remove it?
     if (converted) {
       outConvertedType = converted.getType();
     }
diff --git a/mlir/lib/Dialect/Quant/Utils/UniformSupport.cpp b/mlir/lib/Dialect/Quant/Utils/UniformSupport.cpp
index 991d7c179f900..a79ef0023a7fa 100644
--- a/mlir/lib/Dialect/Quant/Utils/UniformSupport.cpp
+++ b/mlir/lib/Dialect/Quant/Utils/UniformSupport.cpp
@@ -68,7 +68,7 @@ UniformQuantizedPerAxisValueConverter::convert(Attribute realValue) {
   if (auto attr = realValue.dyn_cast<DenseFPElementsAttr>()) {
     return convert(attr);
   }
-  // TODO(fengliuai): handles sparse elements attribute
+  // TODO: handles sparse elements attribute
   return nullptr;
 }
 
diff --git a/mlir/lib/Dialect/SCF/SCF.cpp b/mlir/lib/Dialect/SCF/SCF.cpp
index f980cdb96a6a6..67a3ae34c1d93 100644
--- a/mlir/lib/Dialect/SCF/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/SCF.cpp
@@ -8,21 +8,47 @@
 
 #include "mlir/Dialect/SCF/SCF.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
-#include "mlir/IR/AffineExpr.h"
-#include "mlir/IR/AffineMap.h"
-#include "mlir/IR/Builders.h"
-#include "mlir/IR/Function.h"
-#include "mlir/IR/Matchers.h"
-#include "mlir/IR/Module.h"
-#include "mlir/IR/OpImplementation.h"
+#include "mlir/IR/BlockAndValueMapping.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/IR/StandardTypes.h"
-#include "mlir/IR/Value.h"
-#include "mlir/Support/MathExtras.h"
+#include "mlir/Transforms/InliningUtils.h"
 
 using namespace mlir;
 using namespace mlir::scf;
 
+//===----------------------------------------------------------------------===//
+// SCFDialect Dialect Interfaces
+//===----------------------------------------------------------------------===//
+
+namespace {
+struct SCFInlinerInterface : public DialectInlinerInterface {
+  using DialectInlinerInterface::DialectInlinerInterface;
+  // We don't have any special restrictions on what can be inlined into
+  // destination regions (e.g. while/conditional bodies). Always allow it.
+  bool isLegalToInline(Region *dest, Region *src,
+                       BlockAndValueMapping &valueMapping) const final {
+    return true;
+  }
+  // Operations in scf dialect are always legal to inline since they are
+  // pure.
+  bool isLegalToInline(Operation *, Region *,
+                       BlockAndValueMapping &) const final {
+    return true;
+  }
+  // Handle the given inlined terminator by replacing it with a new operation
+  // as necessary. Required when the region has only one block.
+  void handleTerminator(Operation *op,
+                        ArrayRef<Value> valuesToRepl) const final {
+    auto retValOp = dyn_cast<YieldOp>(op);
+    if (!retValOp)
+      return;
+
+    for (auto retValue : llvm::zip(valuesToRepl, retValOp.getOperands())) {
+      std::get<0>(retValue).replaceAllUsesWith(std::get<1>(retValue));
+    }
+  }
+};
+} // end anonymous namespace
+
 //===----------------------------------------------------------------------===//
 // SCFDialect
 //===----------------------------------------------------------------------===//
@@ -33,6 +59,7 @@ SCFDialect::SCFDialect(MLIRContext *context)
 #define GET_OP_LIST
 #include "mlir/Dialect/SCF/SCFOps.cpp.inc"
       >();
+  addInterfaces<SCFInlinerInterface>();
 }
 
 /// Default callback for IfOp builders. Inserts a yield without arguments.
@@ -383,16 +410,6 @@ void IfOp::build(OpBuilder &builder, OperationState &result, Value cond,
 }
 
 static LogicalResult verify(IfOp op) {
-  // Verify that the entry of each child region does not have arguments.
-  for (auto &region : op.getOperation()->getRegions()) {
-    if (region.empty())
-      continue;
-
-    for (auto &b : region)
-      if (b.getNumArguments() != 0)
-        return op.emitOpError(
-            "requires that child entry blocks have no arguments");
-  }
   if (op.getNumResults() != 0 && op.elseRegion().empty())
     return op.emitOpError("must have an else block if defining values");
 
@@ -706,6 +723,68 @@ ParallelOp mlir::scf::getParallelForInductionVarOwner(Value val) {
   return dyn_cast<ParallelOp>(containingOp);
 }
 
+namespace {
+// Collapse loop dimensions that perform a single iteration.
+struct CollapseSingleIterationLoops : public OpRewritePattern<ParallelOp> {
+  using OpRewritePattern<ParallelOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(ParallelOp op,
+                                PatternRewriter &rewriter) const override {
+    BlockAndValueMapping mapping;
+    // Compute new loop bounds that omit all single-iteration loop dimensions.
+    SmallVector<Value, 2> newLowerBounds;
+    SmallVector<Value, 2> newUpperBounds;
+    SmallVector<Value, 2> newSteps;
+    newLowerBounds.reserve(op.lowerBound().size());
+    newUpperBounds.reserve(op.upperBound().size());
+    newSteps.reserve(op.step().size());
+    for (auto dim : llvm::zip(op.lowerBound(), op.upperBound(), op.step(),
+                              op.getInductionVars())) {
+      Value lowerBound, upperBound, step, iv;
+      std::tie(lowerBound, upperBound, step, iv) = dim;
+      // Collect the statically known loop bounds.
+      auto lowerBoundConstant =
+          dyn_cast_or_null<ConstantIndexOp>(lowerBound.getDefiningOp());
+      auto upperBoundConstant =
+          dyn_cast_or_null<ConstantIndexOp>(upperBound.getDefiningOp());
+      auto stepConstant =
+          dyn_cast_or_null<ConstantIndexOp>(step.getDefiningOp());
+      // Replace the loop induction variable by the lower bound if the loop
+      // performs a single iteration. Otherwise, copy the loop bounds.
+      if (lowerBoundConstant && upperBoundConstant && stepConstant &&
+          (upperBoundConstant.getValue() - lowerBoundConstant.getValue()) > 0 &&
+          (upperBoundConstant.getValue() - lowerBoundConstant.getValue()) <=
+              stepConstant.getValue()) {
+        mapping.map(iv, lowerBound);
+      } else {
+        newLowerBounds.push_back(lowerBound);
+        newUpperBounds.push_back(upperBound);
+        newSteps.push_back(step);
+      }
+    }
+    // Exit if all or none of the loop dimensions perform a single iteration.
+    if (newLowerBounds.size() == 0 ||
+        newLowerBounds.size() == op.lowerBound().size())
+      return failure();
+    // Replace the parallel loop by lower-dimensional parallel loop.
+    auto newOp =
+        rewriter.create<ParallelOp>(op.getLoc(), newLowerBounds, newUpperBounds,
+                                    newSteps, op.initVals(), nullptr);
+    // Clone the loop body and remap the block arguments of the collapsed loops
+    // (inlining does not support a cancellable block argument mapping).
+    rewriter.cloneRegionBefore(op.region(), newOp.region(),
+                               newOp.region().begin(), mapping);
+    rewriter.replaceOp(op, newOp.getResults());
+    return success();
+  }
+};
+} // namespace
+
+void ParallelOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+                                             MLIRContext *context) {
+  results.insert<CollapseSingleIterationLoops>(context);
+}
+
 //===----------------------------------------------------------------------===//
 // ReduceOp
 //===----------------------------------------------------------------------===//
@@ -795,7 +874,7 @@ static LogicalResult verify(YieldOp op) {
   auto results = parentOp->getResults();
   auto operands = op.getOperands();
 
-  if (isa<IfOp>(parentOp) || isa<ForOp>(parentOp)) {
+  if (isa<IfOp, ForOp>(parentOp)) {
     if (parentOp->getNumResults() != op.getNumOperands())
       return op.emitOpError() << "parent of yield must have same number of "
                                  "results as the yield operands";
diff --git a/mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt b/mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt
index 58890d4a3782a..7a54ace0bf8ff 100644
--- a/mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt
@@ -1,6 +1,6 @@
 add_mlir_dialect_library(MLIRSCFTransforms
+  LoopSpecialization.cpp
   ParallelLoopFusion.cpp
-  ParallelLoopSpecialization.cpp
   ParallelLoopTiling.cpp
   Utils.cpp
 
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopSpecialization.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
similarity index 50%
rename from mlir/lib/Dialect/SCF/Transforms/ParallelLoopSpecialization.cpp
rename to mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
index 12c35b1174886..54c663ca67bca 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopSpecialization.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
@@ -1,4 +1,4 @@
-//===- ParallelLoopSpecialization.cpp - scf.parallel specialization ------===//
+//===- LoopSpecialization.cpp - scf.parallel/SCR.for specialization -------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,7 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// Specializes parallel loops for easier unrolling and vectorization.
+// Specializes parallel loops and for loops for easier unrolling and
+// vectorization.
 //
 //===----------------------------------------------------------------------===//
 
@@ -19,13 +20,14 @@
 #include "mlir/IR/BlockAndValueMapping.h"
 
 using namespace mlir;
+using scf::ForOp;
 using scf::ParallelOp;
 
-/// Rewrite a loop with bounds defined by an affine.min with a constant into 2
-/// loops after checking if the bounds are equal to that constant. This is
-/// beneficial if the loop will almost always have the constant bound and that
-/// version can be fully unrolled and vectorized.
-static void specializeLoopForUnrolling(ParallelOp op) {
+/// Rewrite a parallel loop with bounds defined by an affine.min with a constant
+/// into 2 loops after checking if the bounds are equal to that constant. This
+/// is beneficial if the loop will almost always have the constant bound and
+/// that version can be fully unrolled and vectorized.
+static void specializeParallelLoopForUnrolling(ParallelOp op) {
   SmallVector<int64_t, 2> constantIndices;
   constantIndices.reserve(op.upperBound().size());
   for (auto bound : op.upperBound()) {
@@ -33,7 +35,7 @@ static void specializeLoopForUnrolling(ParallelOp op) {
     if (!minOp)
       return;
     int64_t minConstant = std::numeric_limits<int64_t>::max();
-    for (auto expr : minOp.map().getResults()) {
+    for (AffineExpr expr : minOp.map().getResults()) {
       if (auto constantIndex = expr.dyn_cast<AffineConstantExpr>())
         minConstant = std::min(minConstant, constantIndex.getValue());
     }
@@ -58,11 +60,48 @@ static void specializeLoopForUnrolling(ParallelOp op) {
   op.erase();
 }
 
+/// Rewrite a for loop with bounds defined by an affine.min with a constant into
+/// 2 loops after checking if the bounds are equal to that constant. This is
+/// beneficial if the loop will almost always have the constant bound and that
+/// version can be fully unrolled and vectorized.
+static void specializeForLoopForUnrolling(ForOp op) {
+  auto bound = op.upperBound();
+  auto minOp = bound.getDefiningOp<AffineMinOp>();
+  if (!minOp)
+    return;
+  int64_t minConstant = std::numeric_limits<int64_t>::max();
+  for (AffineExpr expr : minOp.map().getResults()) {
+    if (auto constantIndex = expr.dyn_cast<AffineConstantExpr>())
+      minConstant = std::min(minConstant, constantIndex.getValue());
+  }
+  if (minConstant == std::numeric_limits<int64_t>::max())
+    return;
+
+  OpBuilder b(op);
+  BlockAndValueMapping map;
+  Value constant = b.create<ConstantIndexOp>(op.getLoc(), minConstant);
+  Value cond =
+      b.create<CmpIOp>(op.getLoc(), CmpIPredicate::eq, bound, constant);
+  map.map(bound, constant);
+  auto ifOp = b.create<scf::IfOp>(op.getLoc(), cond, /*withElseRegion=*/true);
+  ifOp.getThenBodyBuilder().clone(*op.getOperation(), map);
+  ifOp.getElseBodyBuilder().clone(*op.getOperation());
+  op.erase();
+}
+
 namespace {
 struct ParallelLoopSpecialization
-    : public LoopParallelLoopSpecializationBase<ParallelLoopSpecialization> {
+    : public SCFParallelLoopSpecializationBase<ParallelLoopSpecialization> {
+  void runOnFunction() override {
+    getFunction().walk(
+        [](ParallelOp op) { specializeParallelLoopForUnrolling(op); });
+  }
+};
+
+struct ForLoopSpecialization
+    : public SCFForLoopSpecializationBase<ForLoopSpecialization> {
   void runOnFunction() override {
-    getFunction().walk([](ParallelOp op) { specializeLoopForUnrolling(op); });
+    getFunction().walk([](ForOp op) { specializeForLoopForUnrolling(op); });
   }
 };
 } // namespace
@@ -70,3 +109,7 @@ struct ParallelLoopSpecialization
 std::unique_ptr<Pass> mlir::createParallelLoopSpecializationPass() {
   return std::make_unique<ParallelLoopSpecialization>();
 }
+
+std::unique_ptr<Pass> mlir::createForLoopSpecializationPass() {
+  return std::make_unique<ForLoopSpecialization>();
+}
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
index 204528f162383..cbdf78cad51d5 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
@@ -160,7 +160,7 @@ void mlir::scf::naivelyFuseParallelOps(Region &region) {
 
 namespace {
 struct ParallelLoopFusion
-    : public LoopParallelLoopFusionBase<ParallelLoopFusion> {
+    : public SCFParallelLoopFusionBase<ParallelLoopFusion> {
   void runOnOperation() override {
     getOperation()->walk([&](Operation *child) {
       for (Region &region : child->getRegions())
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
index 40469138ea018..7bcc989a5b282 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopTiling.cpp
@@ -16,8 +16,6 @@
 #include "mlir/Dialect/SCF/SCF.h"
 #include "mlir/Dialect/SCF/Transforms.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
-#include "mlir/Transforms/RegionUtils.h"
-#include "llvm/Support/CommandLine.h"
 
 using namespace mlir;
 using namespace mlir::scf;
@@ -30,8 +28,8 @@ using namespace mlir::scf;
 ///   scf.parallel (%i0, %i1) = (%arg0, %arg1) to (%arg2, %arg3)
 ///                                             step (%arg4*tileSize[0],
 ///                                                   %arg5*tileSize[1])
-///     scf.parallel (%j0, %j1) = (0, 0) to (min(tileSize[0], %arg2-%i0)
-///                                           min(tileSize[1], %arg3-%i1))
+///     scf.parallel (%j0, %j1) = (0, 0) to (min(%arg4*tileSize[0], %arg2-%i0)
+///                                           min(%arg5*tileSize[1], %arg3-%i1))
 ///                                        step (%arg4, %arg5)
 ///
 /// where the uses of %i0 and %i1 in the loop body are replaced by
@@ -76,12 +74,36 @@ void mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes) {
   // Create the inner loop with adjusted bounds.
   SmallVector<Value, 2> newBounds;
   newBounds.reserve(op.upperBound().size());
-  for (auto bounds : llvm::zip(tileSizeConstants, outerLoop.upperBound(),
-                               outerLoop.getInductionVars())) {
-    newBounds.push_back(b.create<AffineMinOp>(
-        op.getLoc(), b.getIndexType(), minMap,
-        ValueRange{std::get<0>(bounds), std::get<1>(bounds),
-                   std::get<2>(bounds)}));
+  for (auto dim : llvm::zip(outerLoop.lowerBound(), outerLoop.upperBound(),
+                            outerLoop.step(), outerLoop.getInductionVars(),
+                            op.step(), tileSizeConstants)) {
+    Value lowerBound, upperBound, newStep, iv, step, tileSizeConstant;
+    std::tie(lowerBound, upperBound, newStep, iv, step, tileSizeConstant) = dim;
+    // Collect the statically known loop bounds
+    auto lowerBoundConstant =
+        dyn_cast_or_null<ConstantIndexOp>(lowerBound.getDefiningOp());
+    auto upperBoundConstant =
+        dyn_cast_or_null<ConstantIndexOp>(upperBound.getDefiningOp());
+    auto stepConstant = dyn_cast_or_null<ConstantIndexOp>(step.getDefiningOp());
+    auto tileSize =
+        cast<ConstantIndexOp>(tileSizeConstant.getDefiningOp()).getValue();
+    // If the loop bounds and the loop step are constant and if the number of
+    // loop iterations is an integer multiple of the tile size, we use a static
+    // bound for the inner loop.
+    if (lowerBoundConstant && upperBoundConstant && stepConstant) {
+      auto numIterations = llvm::divideCeil(upperBoundConstant.getValue() -
+                                                lowerBoundConstant.getValue(),
+                                            stepConstant.getValue());
+      if (numIterations % tileSize == 0) {
+        newBounds.push_back(newStep);
+        continue;
+      }
+    }
+    // Otherwise, we dynamically compute the bound for
+    // each iteration of the outer loop.
+    newBounds.push_back(
+        b.create<AffineMinOp>(op.getLoc(), b.getIndexType(), minMap,
+                              ValueRange{newStep, upperBound, iv}));
   }
   auto innerLoop = b.create<ParallelOp>(
       op.getLoc(), SmallVector<Value, 2>(newBounds.size(), zero), newBounds,
@@ -104,8 +126,8 @@ void mlir::scf::tileParallelLoop(ParallelOp op, ArrayRef<int64_t> tileSizes) {
   op.erase();
 }
 
-/// Get a list of most nested parallel loops. Assumes that ParallelOps are only
-/// directly nested.
+/// Get a list of most nested parallel loops. Assumes that ParallelOps are
+/// only directly nested.
 static bool getInnermostNestedLoops(Block *block,
                                     SmallVectorImpl<ParallelOp> &loops) {
   bool hasInnerLoop = false;
@@ -119,7 +141,7 @@ static bool getInnermostNestedLoops(Block *block,
 
 namespace {
 struct ParallelLoopTiling
-    : public LoopParallelLoopTilingBase<ParallelLoopTiling> {
+    : public SCFParallelLoopTilingBase<ParallelLoopTiling> {
   ParallelLoopTiling() = default;
   explicit ParallelLoopTiling(ArrayRef<int64_t> tileSizes) {
     this->tileSizes = tileSizes;
@@ -131,7 +153,9 @@ struct ParallelLoopTiling
       getInnermostNestedLoops(&block, mostNestedParallelOps);
     }
     for (ParallelOp pLoop : mostNestedParallelOps) {
-      tileParallelLoop(pLoop, tileSizes);
+      // FIXME: Add reduction support.
+      if (pLoop.getNumReductions() == 0)
+        tileParallelLoop(pLoop, tileSizes);
     }
   }
 };
diff --git a/mlir/lib/Dialect/SDBM/SDBM.cpp b/mlir/lib/Dialect/SDBM/SDBM.cpp
index 77f81fed03226..df24e77bc4f29 100644
--- a/mlir/lib/Dialect/SDBM/SDBM.cpp
+++ b/mlir/lib/Dialect/SDBM/SDBM.cpp
@@ -187,7 +187,7 @@ struct SDBMBuilder : public SDBMVisitor<SDBMBuilder, SDBMBuilderResult> {
 SDBM SDBM::get(ArrayRef<SDBMExpr> inequalities, ArrayRef<SDBMExpr> equalities) {
   SDBM result;
 
-  // TODO(zinenko): consider detecting equalities in the list of inequalities.
+  // TODO: consider detecting equalities in the list of inequalities.
   // This is potentially expensive and requires to
   //   - create a list of negated inequalities (may allocate under lock);
   //   - perform a pairwise comparison of direct and negated inequalities;
@@ -289,7 +289,7 @@ SDBM SDBM::get(ArrayRef<SDBMExpr> inequalities, ArrayRef<SDBMExpr> equalities) {
   // value is positive, the set defined by SDBM is trivially empty.  We store
   // this value anyway and continue processing to maintain the correspondence
   // between the matrix form and the list-of-SDBMExpr form.
-  // TODO(zinenko): we may want to reconsider this once we have canonicalization
+  // TODO: we may want to reconsider this once we have canonicalization
   // or simplification in place
   auto updateMatrix = [](SDBM &sdbm, const SDBMBuilderResult &r) {
     for (auto positivePos : r.positivePos) {
diff --git a/mlir/lib/Dialect/SDBM/SDBMExpr.cpp b/mlir/lib/Dialect/SDBM/SDBMExpr.cpp
index 0428df79a053c..5d60158c34e4b 100644
--- a/mlir/lib/Dialect/SDBM/SDBMExpr.cpp
+++ b/mlir/lib/Dialect/SDBM/SDBMExpr.cpp
@@ -451,7 +451,7 @@ Optional<SDBMExpr> SDBMExpr::tryConvertAffineExpr(AffineExpr affine) {
       if (pattern.match(expr)) {
         if (SDBMExpr converted = visit(x.matched())) {
           if (auto varConverted = converted.dyn_cast<SDBMTermExpr>())
-            // TODO(ntv): return varConverted.stripe(C.getConstantValue());
+            // TODO: return varConverted.stripe(C.getConstantValue());
             return SDBMStripeExpr::get(
                 varConverted,
                 SDBMConstantExpr::get(dialect,
diff --git a/mlir/lib/Dialect/SPIRV/SPIRVDialect.cpp b/mlir/lib/Dialect/SPIRV/SPIRVDialect.cpp
index 43e70a1bdc637..47440265239de 100644
--- a/mlir/lib/Dialect/SPIRV/SPIRVDialect.cpp
+++ b/mlir/lib/Dialect/SPIRV/SPIRVDialect.cpp
@@ -45,8 +45,7 @@ using namespace mlir::spirv;
 static inline bool containsReturn(Region &region) {
   return llvm::any_of(region, [](Block &block) {
     Operation *terminator = block.getTerminator();
-    return isa<spirv::ReturnOp>(terminator) ||
-           isa<spirv::ReturnValueOp>(terminator);
+    return isa<spirv::ReturnOp, spirv::ReturnValueOp>(terminator);
   });
 }
 
@@ -62,8 +61,7 @@ struct SPIRVInlinerInterface : public DialectInlinerInterface {
     // Return true here when inlining into spv.func, spv.selection, and
     // spv.loop operations.
     auto *op = dest->getParentOp();
-    return isa<spirv::FuncOp>(op) || isa<spirv::SelectionOp>(op) ||
-           isa<spirv::LoopOp>(op);
+    return isa<spirv::FuncOp, spirv::SelectionOp, spirv::LoopOp>(op);
   }
 
   /// Returns true if the given operation 'op', that is registered to this
@@ -71,11 +69,11 @@ struct SPIRVInlinerInterface : public DialectInlinerInterface {
   /// operation registered to the current dialect.
   bool isLegalToInline(Operation *op, Region *dest,
                        BlockAndValueMapping &) const final {
-    // TODO(antiagainst): Enable inlining structured control flows with return.
-    if ((isa<spirv::SelectionOp>(op) || isa<spirv::LoopOp>(op)) &&
+    // TODO: Enable inlining structured control flows with return.
+    if ((isa<spirv::SelectionOp, spirv::LoopOp>(op)) &&
         containsReturn(op->getRegion(0)))
       return false;
-    // TODO(antiagainst): we need to filter OpKill here to avoid inlining it to
+    // TODO: we need to filter OpKill here to avoid inlining it to
     // a loop continue construct:
     // https://github.com/KhronosGroup/SPIRV-Headers/issues/86
     // However OpKill is fragment shader specific and we don't support it yet.
@@ -332,7 +330,7 @@ static Type parseCooperativeMatrixType(SPIRVDialect const &dialect,
   return CooperativeMatrixNVType::get(elementTy, scope, dims[0], dims[1]);
 }
 
-// TODO(ravishankarm) : Reorder methods to be utilities first and parse*Type
+// TODO: Reorder methods to be utilities first and parse*Type
 // methods in alphabetical order
 //
 // storage-class ::= `UniformConstant`
@@ -440,7 +438,7 @@ static Optional<ValTy> parseAndVerify(SPIRVDialect const &dialect,
 template <>
 Optional<Type> parseAndVerify<Type>(SPIRVDialect const &dialect,
                                     DialectAsmParser &parser) {
-  // TODO(ravishankarm): Further verify that the element type can be sampled
+  // TODO: Further verify that the element type can be sampled
   auto ty = parseAndVerifyType(dialect, parser);
   if (!ty)
     return llvm::None;
@@ -725,7 +723,7 @@ static void print(CooperativeMatrixNVType type, DialectAsmPrinter &os) {
 }
 
 static void print(MatrixType type, DialectAsmPrinter &os) {
-  os << "matrix<" << type.getNumElements() << " x " << type.getElementType();
+  os << "matrix<" << type.getNumColumns() << " x " << type.getColumnType();
   os << ">";
 }
 
@@ -1056,7 +1054,7 @@ LogicalResult SPIRVDialect::verifyOperationAttribute(Operation *op,
   StringRef symbol = attribute.first.strref();
   Attribute attr = attribute.second;
 
-  // TODO(antiagainst): figure out a way to generate the description from the
+  // TODO: figure out a way to generate the description from the
   // StructAttr definition.
   if (symbol == spirv::getEntryPointABIAttrName()) {
     if (!attr.isa<spirv::EntryPointABIAttr>())
diff --git a/mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp b/mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp
index 6bb07b28d022f..d31f9c28362a3 100644
--- a/mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp
+++ b/mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp
@@ -89,7 +89,7 @@ static LogicalResult checkCapabilityRequirements(
 Type SPIRVTypeConverter::getIndexType(MLIRContext *context) {
   // Convert to 32-bit integers for now. Might need a way to control this in
   // future.
-  // TODO(ravishankarm): It is probably better to make it 64-bit integers. To
+  // TODO: It is probably better to make it 64-bit integers. To
   // this some support is needed in SPIR-V dialect for Conversion
   // instructions. The Vulkan spec requires the builtins like
   // GlobalInvocationID, etc. to be 32-bit (unsigned) integers which should be
@@ -104,7 +104,7 @@ Type SPIRVTypeConverter::getIndexType(MLIRContext *context) {
 /// behind the number assignments; we try to follow NVVM conventions and largely
 /// give common storage classes a smaller number. The hope is use symbolic
 /// memory space representation eventually after memref supports it.
-// TODO(antiagainst): swap Generic and StorageBuffer assignment to be more akin
+// TODO: swap Generic and StorageBuffer assignment to be more akin
 // to NVVM.
 #define STORAGE_SPACE_MAP_LIST(MAP_FN)                                         \
   MAP_FN(spirv::StorageClass::Generic, 1)                                      \
@@ -155,7 +155,7 @@ SPIRVTypeConverter::getStorageClassForMemorySpace(unsigned space) {
 
 #undef STORAGE_SPACE_MAP_LIST
 
-// TODO(ravishankarm): This is a utility function that should probably be
+// TODO: This is a utility function that should probably be
 // exposed by the SPIR-V dialect. Keeping it local till the use case arises.
 static Optional<int64_t> getTypeNumBytes(Type t) {
   if (t.isa<spirv::ScalarType>()) {
@@ -239,7 +239,7 @@ convertScalarType(const spirv::TargetEnv &targetEnv, spirv::ScalarType type,
 
   // Otherwise we need to adjust the type, which really means adjusting the
   // bitwidth given this is a scalar type.
-  // TODO(antiagainst): We are unconditionally converting the bitwidth here,
+  // TODO: We are unconditionally converting the bitwidth here,
   // this might be okay for non-interface types (i.e., types used in
   // Private/Function storage classes), but not for interface types (i.e.,
   // types used in StorageBuffer/Uniform/PushConstant/etc. storage classes).
@@ -263,7 +263,7 @@ static Optional<Type>
 convertVectorType(const spirv::TargetEnv &targetEnv, VectorType type,
                   Optional<spirv::StorageClass> storageClass = {}) {
   if (!spirv::CompositeType::isValid(type)) {
-    // TODO(antiagainst): One-element vector types can be translated into scalar
+    // TODO: One-element vector types can be translated into scalar
     // types. Vector types with more than four elements can be translated into
     // array types.
     LLVM_DEBUG(llvm::dbgs()
@@ -297,7 +297,7 @@ convertVectorType(const spirv::TargetEnv &targetEnv, VectorType type,
 /// manipulate, like what we do for vectors.
 static Optional<Type> convertTensorType(const spirv::TargetEnv &targetEnv,
                                         TensorType type) {
-  // TODO(ravishankarm) : Handle dynamic shapes.
+  // TODO: Handle dynamic shapes.
   if (!type.hasStaticShape()) {
     LLVM_DEBUG(llvm::dbgs()
                << type << " illegal: dynamic shape unimplemented\n");
@@ -406,7 +406,7 @@ SPIRVTypeConverter::SPIRVTypeConverter(spirv::TargetEnvAttr targetAttr)
   // adopted in the SPIR-V dialect (i.e., IntegerType, FloatType, VectorType)
   // were tried before.
   //
-  // TODO(antiagainst): this assumes that the SPIR-V types are valid to use in
+  // TODO: this assumes that the SPIR-V types are valid to use in
   // the given target environment, which should be the case if the whole
   // pipeline is driven by the same target environment. Still, we probably still
   // want to validate and convert to be safe.
@@ -462,7 +462,7 @@ LogicalResult
 FuncOpConversion::matchAndRewrite(FuncOp funcOp, ArrayRef<Value> operands,
                                   ConversionPatternRewriter &rewriter) const {
   auto fnType = funcOp.getType();
-  // TODO(antiagainst): support converting functions with one result.
+  // TODO: support converting functions with one result.
   if (fnType.getNumResults())
     return failure();
 
diff --git a/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
index 87456f000edc7..6c8319224974a 100644
--- a/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
@@ -27,8 +27,12 @@
 
 using namespace mlir;
 
-// TODO(antiagainst): generate these strings using ODS.
+// TODO: generate these strings using ODS.
+static constexpr const char kMemoryAccessAttrName[] = "memory_access";
+static constexpr const char kSourceMemoryAccessAttrName[] =
+    "source_memory_access";
 static constexpr const char kAlignmentAttrName[] = "alignment";
+static constexpr const char kSourceAlignmentAttrName[] = "source_alignment";
 static constexpr const char kBranchWeightAttrName[] = "branch_weights";
 static constexpr const char kCallee[] = "callee";
 static constexpr const char kClusterSize[] = "cluster_size";
@@ -157,6 +161,12 @@ parseEnumKeywordAttr(EnumClass &value, OpAsmParser &parser,
   return success();
 }
 
+/// Parses optional memory access attributes attached to a memory access
+/// operand/pointer. Specifically, parses the following syntax:
+///     (`[` memory-access `]`)?
+/// where:
+///     memory-access ::= `"None"` | `"Volatile"` | `"Aligned", `
+///         integer-literal | `"NonTemporal"`
 static ParseResult parseMemoryAccessAttributes(OpAsmParser &parser,
                                                OperationState &state) {
   // Parse an optional list of attributes staring with '['
@@ -166,7 +176,8 @@ static ParseResult parseMemoryAccessAttributes(OpAsmParser &parser,
   }
 
   spirv::MemoryAccess memoryAccessAttr;
-  if (parseEnumStrAttr(memoryAccessAttr, parser, state)) {
+  if (parseEnumStrAttr(memoryAccessAttr, parser, state,
+                       kMemoryAccessAttrName)) {
     return failure();
   }
 
@@ -183,19 +194,90 @@ static ParseResult parseMemoryAccessAttributes(OpAsmParser &parser,
   return parser.parseRSquare();
 }
 
-template <typename LoadStoreOpTy>
-static void
-printMemoryAccessAttribute(LoadStoreOpTy loadStoreOp, OpAsmPrinter &printer,
-                           SmallVectorImpl<StringRef> &elidedAttrs) {
+// TODO Make sure to merge this and the previous function into one template
+// parameterized by memroy access attribute name and alignment. Doing so now
+// results in VS2017 in producing an internal error (at the call site) that's
+// not detailed enough to understand what is happenning.
+static ParseResult parseSourceMemoryAccessAttributes(OpAsmParser &parser,
+                                                     OperationState &state) {
+  // Parse an optional list of attributes staring with '['
+  if (parser.parseOptionalLSquare()) {
+    // Nothing to do
+    return success();
+  }
+
+  spirv::MemoryAccess memoryAccessAttr;
+  if (parseEnumStrAttr(memoryAccessAttr, parser, state,
+                       kSourceMemoryAccessAttrName)) {
+    return failure();
+  }
+
+  if (spirv::bitEnumContains(memoryAccessAttr, spirv::MemoryAccess::Aligned)) {
+    // Parse integer attribute for alignment.
+    Attribute alignmentAttr;
+    Type i32Type = parser.getBuilder().getIntegerType(32);
+    if (parser.parseComma() ||
+        parser.parseAttribute(alignmentAttr, i32Type, kSourceAlignmentAttrName,
+                              state.attributes)) {
+      return failure();
+    }
+  }
+  return parser.parseRSquare();
+}
+
+template <typename MemoryOpTy>
+static void printMemoryAccessAttribute(
+    MemoryOpTy memoryOp, OpAsmPrinter &printer,
+    SmallVectorImpl<StringRef> &elidedAttrs,
+    Optional<spirv::MemoryAccess> memoryAccessAtrrValue = None,
+    Optional<llvm::APInt> alignmentAttrValue = None) {
+  // Print optional memory access attribute.
+  if (auto memAccess = (memoryAccessAtrrValue ? memoryAccessAtrrValue
+                                              : memoryOp.memory_access())) {
+    elidedAttrs.push_back(kMemoryAccessAttrName);
+
+    printer << " [\"" << stringifyMemoryAccess(*memAccess) << "\"";
+
+    if (spirv::bitEnumContains(*memAccess, spirv::MemoryAccess::Aligned)) {
+      // Print integer alignment attribute.
+      if (auto alignment = (alignmentAttrValue ? alignmentAttrValue
+                                               : memoryOp.alignment())) {
+        elidedAttrs.push_back(kAlignmentAttrName);
+        printer << ", " << alignment;
+      }
+    }
+    printer << "]";
+  }
+  elidedAttrs.push_back(spirv::attributeName<spirv::StorageClass>());
+}
+
+// TODO Make sure to merge this and the previous function into one template
+// parameterized by memroy access attribute name and alignment. Doing so now
+// results in VS2017 in producing an internal error (at the call site) that's
+// not detailed enough to understand what is happenning.
+template <typename MemoryOpTy>
+static void printSourceMemoryAccessAttribute(
+    MemoryOpTy memoryOp, OpAsmPrinter &printer,
+    SmallVectorImpl<StringRef> &elidedAttrs,
+    Optional<spirv::MemoryAccess> memoryAccessAtrrValue = None,
+    Optional<llvm::APInt> alignmentAttrValue = None) {
+
+  printer << ", ";
+
   // Print optional memory access attribute.
-  if (auto memAccess = loadStoreOp.memory_access()) {
-    elidedAttrs.push_back(spirv::attributeName<spirv::MemoryAccess>());
+  if (auto memAccess = (memoryAccessAtrrValue ? memoryAccessAtrrValue
+                                              : memoryOp.memory_access())) {
+    elidedAttrs.push_back(kSourceMemoryAccessAttrName);
+
     printer << " [\"" << stringifyMemoryAccess(*memAccess) << "\"";
 
-    // Print integer alignment attribute.
-    if (auto alignment = loadStoreOp.alignment()) {
-      elidedAttrs.push_back(kAlignmentAttrName);
-      printer << ", " << alignment;
+    if (spirv::bitEnumContains(*memAccess, spirv::MemoryAccess::Aligned)) {
+      // Print integer alignment attribute.
+      if (auto alignment = (alignmentAttrValue ? alignmentAttrValue
+                                               : memoryOp.alignment())) {
+        elidedAttrs.push_back(kSourceAlignmentAttrName);
+        printer << ", " << alignment;
+      }
     }
     printer << "]";
   }
@@ -243,18 +325,18 @@ static LogicalResult verifyCastOp(Operation *op,
   return success();
 }
 
-template <typename LoadStoreOpTy>
-static LogicalResult verifyMemoryAccessAttribute(LoadStoreOpTy loadStoreOp) {
+template <typename MemoryOpTy>
+static LogicalResult verifyMemoryAccessAttribute(MemoryOpTy memoryOp) {
   // ODS checks for attributes values. Just need to verify that if the
   // memory-access attribute is Aligned, then the alignment attribute must be
   // present.
-  auto *op = loadStoreOp.getOperation();
-  auto memAccessAttr = op->getAttr(spirv::attributeName<spirv::MemoryAccess>());
+  auto *op = memoryOp.getOperation();
+  auto memAccessAttr = op->getAttr(kMemoryAccessAttrName);
   if (!memAccessAttr) {
     // Alignment attribute shouldn't be present if memory access attribute is
     // not present.
     if (op->getAttr(kAlignmentAttrName)) {
-      return loadStoreOp.emitOpError(
+      return memoryOp.emitOpError(
           "invalid alignment specification without aligned memory access "
           "specification");
     }
@@ -265,17 +347,61 @@ static LogicalResult verifyMemoryAccessAttribute(LoadStoreOpTy loadStoreOp) {
   auto memAccess = spirv::symbolizeMemoryAccess(memAccessVal.getInt());
 
   if (!memAccess) {
-    return loadStoreOp.emitOpError("invalid memory access specifier: ")
+    return memoryOp.emitOpError("invalid memory access specifier: ")
            << memAccessVal;
   }
 
   if (spirv::bitEnumContains(*memAccess, spirv::MemoryAccess::Aligned)) {
     if (!op->getAttr(kAlignmentAttrName)) {
-      return loadStoreOp.emitOpError("missing alignment value");
+      return memoryOp.emitOpError("missing alignment value");
     }
   } else {
     if (op->getAttr(kAlignmentAttrName)) {
-      return loadStoreOp.emitOpError(
+      return memoryOp.emitOpError(
+          "invalid alignment specification with non-aligned memory access "
+          "specification");
+    }
+  }
+  return success();
+}
+
+// TODO Make sure to merge this and the previous function into one template
+// parameterized by memroy access attribute name and alignment. Doing so now
+// results in VS2017 in producing an internal error (at the call site) that's
+// not detailed enough to understand what is happenning.
+template <typename MemoryOpTy>
+static LogicalResult verifySourceMemoryAccessAttribute(MemoryOpTy memoryOp) {
+  // ODS checks for attributes values. Just need to verify that if the
+  // memory-access attribute is Aligned, then the alignment attribute must be
+  // present.
+  auto *op = memoryOp.getOperation();
+  auto memAccessAttr = op->getAttr(kSourceMemoryAccessAttrName);
+  if (!memAccessAttr) {
+    // Alignment attribute shouldn't be present if memory access attribute is
+    // not present.
+    if (op->getAttr(kSourceAlignmentAttrName)) {
+      return memoryOp.emitOpError(
+          "invalid alignment specification without aligned memory access "
+          "specification");
+    }
+    return success();
+  }
+
+  auto memAccessVal = memAccessAttr.template cast<IntegerAttr>();
+  auto memAccess = spirv::symbolizeMemoryAccess(memAccessVal.getInt());
+
+  if (!memAccess) {
+    return memoryOp.emitOpError("invalid memory access specifier: ")
+           << memAccessVal;
+  }
+
+  if (spirv::bitEnumContains(*memAccess, spirv::MemoryAccess::Aligned)) {
+    if (!op->getAttr(kSourceAlignmentAttrName)) {
+      return memoryOp.emitOpError("missing alignment value");
+    }
+  } else {
+    if (op->getAttr(kSourceAlignmentAttrName)) {
+      return memoryOp.emitOpError(
           "invalid alignment specification with non-aligned memory access "
           "specification");
     }
@@ -313,7 +439,7 @@ static LogicalResult verifyLoadStorePtrAndValTypes(LoadStoreOpTy op, Value ptr,
   // ODS already checks ptr is spirv::PointerType. Just check that the pointee
   // type of the pointer and the type of the value are the same
   //
-  // TODO(ravishankarm): Check that the value type satisfies restrictions of
+  // TODO: Check that the value type satisfies restrictions of
   // SPIR-V OpLoad/OpStore operations
   if (val.getType() !=
       ptr.getType().cast<spirv::PointerType>().getPointeeType()) {
@@ -618,7 +744,7 @@ static LogicalResult verifyGroupNonUniformArithmeticOp(Operation *groupOp) {
     Operation *sizeOp = groupOp->getOperand(1).getDefiningOp();
     int32_t clusterSize = 0;
 
-    // TODO(antiagainst): support specialization constant here.
+    // TODO: support specialization constant here.
     if (failed(extractValueFromConstOp(sizeOp, clusterSize)))
       return groupOp->emitOpError(
           "cluster size operand must come from a constant op");
@@ -723,12 +849,6 @@ static LogicalResult verifyShiftOp(Operation *op) {
 //===----------------------------------------------------------------------===//
 
 static Type getElementPtrType(Type type, ValueRange indices, Location baseLoc) {
-  if (indices.empty()) {
-    emitError(baseLoc, "'spv.AccessChain' op expected at least "
-                       "one index ");
-    return nullptr;
-  }
-
   auto ptrType = type.dyn_cast<spirv::PointerType>();
   if (!ptrType) {
     emitError(baseLoc, "'spv.AccessChain' op expected a pointer "
@@ -759,7 +879,7 @@ static Type getElementPtrType(Type type, ValueRange indices, Location baseLoc) {
         return nullptr;
       }
 
-      // TODO(denis0x0D): this should be relaxed to allow
+      // TODO: this should be relaxed to allow
       // integer literals of other bitwidths.
       if (failed(extractValueFromConstOp(op, index))) {
         emitError(baseLoc,
@@ -791,19 +911,37 @@ static ParseResult parseAccessChainOp(OpAsmParser &parser,
   OpAsmParser::OperandType ptrInfo;
   SmallVector<OpAsmParser::OperandType, 4> indicesInfo;
   Type type;
-  // TODO(denis0x0D): regarding to the spec an index must be any integer type,
-  // figure out how to use resolveOperand with a range of types and do not
-  // fail on first attempt.
-  Type indicesType = parser.getBuilder().getIntegerType(32);
+  auto loc = parser.getCurrentLocation();
+  SmallVector<Type, 4> indicesTypes;
 
   if (parser.parseOperand(ptrInfo) ||
       parser.parseOperandList(indicesInfo, OpAsmParser::Delimiter::Square) ||
       parser.parseColonType(type) ||
-      parser.resolveOperand(ptrInfo, type, state.operands) ||
-      parser.resolveOperands(indicesInfo, indicesType, state.operands)) {
+      parser.resolveOperand(ptrInfo, type, state.operands)) {
+    return failure();
+  }
+
+  // Check that the provided indices list is not empty before parsing their
+  // type list.
+  if (indicesInfo.empty()) {
+    return emitError(state.location, "'spv.AccessChain' op expected at "
+                                     "least one index ");
+  }
+
+  if (parser.parseComma() || parser.parseTypeList(indicesTypes))
     return failure();
+
+  // Check that the indices types list is not empty and that it has a one-to-one
+  // mapping to the provided indices.
+  if (indicesTypes.size() != indicesInfo.size()) {
+    return emitError(state.location, "'spv.AccessChain' op indices "
+                                     "types' count must be equal to indices "
+                                     "info count");
   }
 
+  if (parser.resolveOperands(indicesInfo, indicesTypes, loc, state.operands))
+    return failure();
+
   auto resultType = getElementPtrType(
       type, llvm::makeArrayRef(state.operands).drop_front(), state.location);
   if (!resultType) {
@@ -816,7 +954,8 @@ static ParseResult parseAccessChainOp(OpAsmParser &parser,
 
 static void print(spirv::AccessChainOp op, OpAsmPrinter &printer) {
   printer << spirv::AccessChainOp::getOperationName() << ' ' << op.base_ptr()
-          << '[' << op.indices() << "] : " << op.base_ptr().getType();
+          << '[' << op.indices() << "] : " << op.base_ptr().getType() << ", "
+          << op.indices().getTypes();
 }
 
 static LogicalResult verify(spirv::AccessChainOp accessChainOp) {
@@ -935,7 +1074,7 @@ static LogicalResult verify(spirv::AtomicCompareExchangeWeakOp atomOp) {
                "as the op result type, but found ")
            << pointeeType << " vs " << atomOp.getType();
 
-  // TODO(antiagainst): Unequal cannot be set to Release or Acquire and Release.
+  // TODO: Unequal cannot be set to Release or Acquire and Release.
   // In addition, Unequal cannot be set to a stronger memory-order then Equal.
 
   return success();
@@ -1279,7 +1418,7 @@ static ParseResult parseConstantOp(OpAsmParser &parser, OperationState &state) {
     return failure();
 
   Type type = value.getType();
-  if (type.isa<NoneType>() || type.isa<TensorType>()) {
+  if (type.isa<NoneType, TensorType>()) {
     if (parser.parseColonType(type))
       return failure();
   }
@@ -1371,7 +1510,7 @@ bool spirv::ConstantOp::isBuildableWith(Type type) {
 
   if (type.getKind() >= Type::FIRST_SPIRV_TYPE &&
       type.getKind() <= spirv::TypeKind::LAST_SPIRV_TYPE) {
-    // TODO(antiagainst): support constant struct
+    // TODO: support constant struct
     return type.isa<spirv::ArrayType>();
   }
 
@@ -1620,7 +1759,7 @@ LogicalResult spirv::FuncOp::verifyBody() {
     return WalkResult::advance();
   });
 
-  // TODO(antiagainst): verify other bits like linkage type.
+  // TODO: verify other bits like linkage type.
 
   return failure(walkResult.wasInterrupted());
 }
@@ -1814,8 +1953,8 @@ static LogicalResult verify(spirv::GlobalVariableOp varOp) {
     // TODO: Currently only variable initialization with specialization
     // constants and other variables is supported. They could be normal
     // constants in the module scope as well.
-    if (!initOp || !(isa<spirv::GlobalVariableOp>(initOp) ||
-                     isa<spirv::SpecConstantOp>(initOp))) {
+    if (!initOp ||
+        !isa<spirv::GlobalVariableOp, spirv::SpecConstantOp>(initOp)) {
       return varOp.emitOpError("initializer must be result of a "
                                "spv.specConstant or spv.globalVariable op");
     }
@@ -1926,7 +2065,7 @@ void spirv::LoopOp::build(OpBuilder &builder, OperationState &state) {
 }
 
 static ParseResult parseLoopOp(OpAsmParser &parser, OperationState &state) {
-  // TODO(antiagainst): support loop control properly
+  // TODO: support loop control properly
   Builder builder = parser.getBuilder();
   state.addAttribute("loop_control",
                      builder.getI32IntegerAttr(
@@ -2080,8 +2219,7 @@ void spirv::LoopOp::addEntryAndMergeBlock() {
 
 static LogicalResult verify(spirv::MergeOp mergeOp) {
   auto *parentOp = mergeOp.getParentOp();
-  if (!parentOp ||
-      (!isa<spirv::SelectionOp>(parentOp) && !isa<spirv::LoopOp>(parentOp)))
+  if (!parentOp || !isa<spirv::SelectionOp, spirv::LoopOp>(parentOp))
     return mergeOp.emitOpError(
         "expected parent op to be 'spv.selection' or 'spv.loop'");
 
@@ -2210,7 +2348,7 @@ static LogicalResult verify(spirv::ModuleOp moduleOp) {
       if (funcOp.isExternal())
         return op.emitError("'spv.module' cannot contain external functions");
 
-      // TODO(antiagainst): move this check to spv.func.
+      // TODO: move this check to spv.func.
       for (auto &block : funcOp)
         for (auto &op : block) {
           if (op.getDialect() != dialect)
@@ -2290,7 +2428,7 @@ static LogicalResult verify(spirv::SelectOp op) {
 
 static ParseResult parseSelectionOp(OpAsmParser &parser,
                                     OperationState &state) {
-  // TODO(antiagainst): support selection control properly
+  // TODO: support selection control properly
   Builder builder = parser.getBuilder();
   state.addAttribute("selection_control",
                      builder.getI32IntegerAttr(
@@ -2524,7 +2662,7 @@ static LogicalResult verify(spirv::UnreachableOp unreachableOp) {
   if (block->hasNoPredecessors())
     return success();
 
-  // TODO(antiagainst): further verification needs to analyze reachability from
+  // TODO: further verification needs to analyze reachability from
   // the entry block.
 
   return success();
@@ -2607,14 +2745,14 @@ static LogicalResult verify(spirv::VariableOp varOp) {
     // SPIR-V spec: "Initializer must be an <id> from a constant instruction or
     // a global (module scope) OpVariable instruction".
     auto *initOp = varOp.getOperand(0).getDefiningOp();
-    if (!initOp || !(isa<spirv::ConstantOp>(initOp) ||    // for normal constant
-                     isa<spirv::ReferenceOfOp>(initOp) || // for spec constant
-                     isa<spirv::AddressOfOp>(initOp)))
+    if (!initOp || !isa<spirv::ConstantOp,    // for normal constant
+                        spirv::ReferenceOfOp, // for spec constant
+                        spirv::AddressOfOp>(initOp))
       return varOp.emitOpError("initializer must be the result of a "
                                "constant or spv.globalVariable op");
   }
 
-  // TODO(antiagainst): generate these strings using ODS.
+  // TODO: generate these strings using ODS.
   auto *op = varOp.getOperation();
   auto descriptorSetName = llvm::convertToSnakeFromCamelCase(
       stringifyDecoration(spirv::Decoration::DescriptorSet));
@@ -2739,8 +2877,7 @@ static void print(spirv::CooperativeMatrixStoreNVOp coopMatrix,
 static LogicalResult
 verifyCoopMatrixMulAdd(spirv::CooperativeMatrixMulAddNVOp op) {
   if (op.c().getType() != op.result().getType())
-    return op.emitOpError(
-        "result and third operand must have the same type");
+    return op.emitOpError("result and third operand must have the same type");
   auto typeA = op.a().getType().cast<spirv::CooperativeMatrixNVType>();
   auto typeB = op.b().getType().cast<spirv::CooperativeMatrixNVType>();
   auto typeC = op.c().getType().cast<spirv::CooperativeMatrixNVType>();
@@ -2768,37 +2905,198 @@ static LogicalResult verifyMatrixTimesScalar(spirv::MatrixTimesScalarOp op) {
   // auto-generated verify method.
 
   auto inputMatrix = op.matrix().getType().cast<spirv::MatrixType>();
-  // Check that the scalar type is the same as the matrix components type.
-  if (auto inputMatrixColumns =
-          inputMatrix.getElementType().dyn_cast<VectorType>()) {
-    if (op.scalar().getType() != inputMatrixColumns.getElementType())
-      return op.emitError("input matrix components' type and scaling "
-                          "value must have the same type");
-
-    // Note that the next three checks could be done using the AllTypesMatch
-    // trait in the Op definition file but it generates a vague error message.
-
-    // Check that the input and result matrices have the same size
-    auto resultMatrix = op.result().getType().cast<spirv::MatrixType>();
-    if (inputMatrix.getNumElements() != resultMatrix.getNumElements())
-      return op.emitError("input and result matrices must have "
-                          "the same number of columns");
-
-    if (auto resultMatrixColumns =
-            resultMatrix.getElementType().dyn_cast<VectorType>()) {
-      // Check that the input and result matrices' columns have the same type
-      if (inputMatrixColumns.getElementType() !=
-          resultMatrixColumns.getElementType())
-        return op.emitError("input and result matrices' columns must "
-                            "have the same component type");
-
-      // Check that the input and result matrices' columns have the same size
-      if (inputMatrixColumns.getNumElements() !=
-          resultMatrixColumns.getNumElements())
-        return op.emitError("input and result matrices' columns must "
-                            "have the same size");
+  auto resultMatrix = op.result().getType().cast<spirv::MatrixType>();
+
+  // Check that the scalar type is the same as the matrix element type.
+  if (op.scalar().getType() != inputMatrix.getElementType())
+    return op.emitError("input matrix components' type and scaling value must "
+                        "have the same type");
+
+  // Note that the next three checks could be done using the AllTypesMatch
+  // trait in the Op definition file but it generates a vague error message.
+
+  // Check that the input and result matrices have the same columns' count
+  if (inputMatrix.getNumColumns() != resultMatrix.getNumColumns())
+    return op.emitError("input and result matrices must have the same "
+                        "number of columns");
+
+  // Check that the input and result matrices' have the same rows count
+  if (inputMatrix.getNumRows() != resultMatrix.getNumRows())
+    return op.emitError("input and result matrices' columns must have "
+                        "the same size");
+
+  // Check that the input and result matrices' have the same component type
+  if (inputMatrix.getElementType() != resultMatrix.getElementType())
+    return op.emitError("input and result matrices' columns must have "
+                        "the same component type");
+
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// spv.CopyMemory
+//===----------------------------------------------------------------------===//
+
+static void print(spirv::CopyMemoryOp copyMemory, OpAsmPrinter &printer) {
+  auto *op = copyMemory.getOperation();
+  printer << spirv::CopyMemoryOp::getOperationName() << ' ';
+
+  StringRef targetStorageClass =
+      stringifyStorageClass(copyMemory.target()
+                                .getType()
+                                .cast<spirv::PointerType>()
+                                .getStorageClass());
+  printer << " \"" << targetStorageClass << "\" " << copyMemory.target()
+          << ", ";
+
+  StringRef sourceStorageClass =
+      stringifyStorageClass(copyMemory.source()
+                                .getType()
+                                .cast<spirv::PointerType>()
+                                .getStorageClass());
+  printer << " \"" << sourceStorageClass << "\" " << copyMemory.source();
+
+  SmallVector<StringRef, 4> elidedAttrs;
+  printMemoryAccessAttribute(copyMemory, printer, elidedAttrs);
+  printSourceMemoryAccessAttribute(copyMemory, printer, elidedAttrs,
+                                   copyMemory.source_memory_access(),
+                                   copyMemory.source_alignment());
+
+  printer.printOptionalAttrDict(op->getAttrs(), elidedAttrs);
+
+  Type pointeeType =
+      copyMemory.target().getType().cast<spirv::PointerType>().getPointeeType();
+  printer << " : " << pointeeType;
+}
+
+static ParseResult parseCopyMemoryOp(OpAsmParser &parser,
+                                     OperationState &state) {
+  spirv::StorageClass targetStorageClass;
+  OpAsmParser::OperandType targetPtrInfo;
+
+  spirv::StorageClass sourceStorageClass;
+  OpAsmParser::OperandType sourcePtrInfo;
+
+  Type elementType;
+
+  if (parseEnumStrAttr(targetStorageClass, parser) ||
+      parser.parseOperand(targetPtrInfo) || parser.parseComma() ||
+      parseEnumStrAttr(sourceStorageClass, parser) ||
+      parser.parseOperand(sourcePtrInfo) ||
+      parseMemoryAccessAttributes(parser, state)) {
+    return failure();
+  }
+
+  if (!parser.parseOptionalComma()) {
+    // Parse 2nd memory access attributes.
+    if (parseSourceMemoryAccessAttributes(parser, state)) {
+      return failure();
     }
   }
+
+  if (parser.parseColon() || parser.parseType(elementType))
+    return failure();
+
+  if (parser.parseOptionalAttrDict(state.attributes))
+    return failure();
+
+  auto targetPtrType = spirv::PointerType::get(elementType, targetStorageClass);
+  auto sourcePtrType = spirv::PointerType::get(elementType, sourceStorageClass);
+
+  if (parser.resolveOperand(targetPtrInfo, targetPtrType, state.operands) ||
+      parser.resolveOperand(sourcePtrInfo, sourcePtrType, state.operands)) {
+    return failure();
+  }
+
+  return success();
+}
+
+static LogicalResult verifyCopyMemory(spirv::CopyMemoryOp copyMemory) {
+  Type targetType =
+      copyMemory.target().getType().cast<spirv::PointerType>().getPointeeType();
+
+  Type sourceType =
+      copyMemory.source().getType().cast<spirv::PointerType>().getPointeeType();
+
+  if (targetType != sourceType) {
+    return copyMemory.emitOpError(
+        "both operands must be pointers to the same type");
+  }
+
+  if (failed(verifyMemoryAccessAttribute(copyMemory))) {
+    return failure();
+  }
+
+  // TODO - According to the spec:
+  //
+  // If two masks are present, the first applies to Target and cannot include
+  // MakePointerVisible, and the second applies to Source and cannot include
+  // MakePointerAvailable.
+  //
+  // Add such verification here.
+
+  return verifySourceMemoryAccessAttribute(copyMemory);
+}
+
+//===----------------------------------------------------------------------===//
+// spv.Transpose
+//===----------------------------------------------------------------------===//
+
+static LogicalResult verifyTranspose(spirv::TransposeOp op) {
+  auto inputMatrix = op.matrix().getType().cast<spirv::MatrixType>();
+  auto resultMatrix = op.result().getType().cast<spirv::MatrixType>();
+
+  // Verify that the input and output matrices have correct shapes.
+  if (inputMatrix.getNumRows() != resultMatrix.getNumColumns())
+    return op.emitError("input matrix rows count must be equal to "
+                        "output matrix columns count");
+
+  if (inputMatrix.getNumColumns() != resultMatrix.getNumRows())
+    return op.emitError("input matrix columns count must be equal to "
+                        "output matrix rows count");
+
+  // Verify that the input and output matrices have the same component type
+  if (inputMatrix.getElementType() != resultMatrix.getElementType())
+    return op.emitError("input and output matrices must have the same "
+                        "component type");
+
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// spv.MatrixTimesMatrix
+//===----------------------------------------------------------------------===//
+
+static LogicalResult verifyMatrixTimesMatrix(spirv::MatrixTimesMatrixOp op) {
+  auto leftMatrix = op.leftmatrix().getType().cast<spirv::MatrixType>();
+  auto rightMatrix = op.rightmatrix().getType().cast<spirv::MatrixType>();
+  auto resultMatrix = op.result().getType().cast<spirv::MatrixType>();
+
+  // left matrix columns' count and right matrix rows' count must be equal
+  if (leftMatrix.getNumColumns() != rightMatrix.getNumRows())
+    return op.emitError("left matrix columns' count must be equal to "
+                        "the right matrix rows' count");
+
+  // right and result matrices columns' count must be the same
+  if (rightMatrix.getNumColumns() != resultMatrix.getNumColumns())
+    return op.emitError(
+        "right and result matrices must have equal columns' count");
+
+  // right and result matrices component type must be the same
+  if (rightMatrix.getElementType() != resultMatrix.getElementType())
+    return op.emitError("right and result matrices' component type must"
+                        " be the same");
+
+  // left and result matrices component type must be the same
+  if (leftMatrix.getElementType() != resultMatrix.getElementType())
+    return op.emitError("left and result matrices' component type"
+                        " must be the same");
+
+  // left and result matrices rows count must be the same
+  if (leftMatrix.getNumRows() != resultMatrix.getNumRows())
+    return op.emitError("left and result matrices must have equal rows'"
+                        " count");
+
   return success();
 }
 
diff --git a/mlir/lib/Dialect/SPIRV/SPIRVTypes.cpp b/mlir/lib/Dialect/SPIRV/SPIRVTypes.cpp
index 963f5393c572f..93d0c43d669fb 100644
--- a/mlir/lib/Dialect/SPIRV/SPIRVTypes.cpp
+++ b/mlir/lib/Dialect/SPIRV/SPIRVTypes.cpp
@@ -86,7 +86,7 @@ spirv::getRecursiveImpliedCapabilities(Capability cap) {
   llvm::SetVector<Capability, SmallVector<Capability, 0>> allCaps(
       directCaps.begin(), directCaps.end());
 
-  // TODO(antiagainst): This is insufficient; find a better way to handle this
+  // TODO: This is insufficient; find a better way to handle this
   // (e.g., using static lists) if this turns out to be a bottleneck.
   for (unsigned i = 0; i < allCaps.size(); ++i)
     for (Capability c : getDirectImpliedCapabilities(allCaps[i]))
@@ -151,6 +151,14 @@ void ArrayType::getCapabilities(
   getElementType().cast<SPIRVType>().getCapabilities(capabilities, storage);
 }
 
+Optional<int64_t> ArrayType::getSizeInBytes() {
+  auto elementType = getElementType().cast<SPIRVType>();
+  Optional<int64_t> size = elementType.getSizeInBytes();
+  if (!size)
+    return llvm::None;
+  return (*size + getArrayStride()) * getNumElements();
+}
+
 //===----------------------------------------------------------------------===//
 // CompositeType
 //===----------------------------------------------------------------------===//
@@ -182,7 +190,7 @@ Type CompositeType::getElementType(unsigned index) const {
   case spirv::TypeKind::CooperativeMatrix:
     return cast<CooperativeMatrixNVType>().getElementType();
   case spirv::TypeKind::Matrix:
-    return cast<MatrixType>().getElementType();
+    return cast<MatrixType>().getColumnType();
   case spirv::TypeKind::RuntimeArray:
     return cast<RuntimeArrayType>().getElementType();
   case spirv::TypeKind::Struct:
@@ -202,7 +210,7 @@ unsigned CompositeType::getNumElements() const {
     llvm_unreachable(
         "invalid to query number of elements of spirv::CooperativeMatrix type");
   case spirv::TypeKind::Matrix:
-    return cast<MatrixType>().getNumElements();
+    return cast<MatrixType>().getNumColumns();
   case spirv::TypeKind::RuntimeArray:
     llvm_unreachable(
         "invalid to query number of elements of spirv::RuntimeArray type");
@@ -281,6 +289,24 @@ void CompositeType::getCapabilities(
   }
 }
 
+Optional<int64_t> CompositeType::getSizeInBytes() {
+  switch (getKind()) {
+  case spirv::TypeKind::Array:
+    return cast<ArrayType>().getSizeInBytes();
+  case spirv::TypeKind::Struct:
+    return cast<StructType>().getSizeInBytes();
+  case StandardTypes::Vector: {
+    auto elementSize =
+        cast<VectorType>().getElementType().cast<ScalarType>().getSizeInBytes();
+    if (!elementSize)
+      return llvm::None;
+    return *elementSize * cast<VectorType>().getNumElements();
+  }
+  default:
+    return llvm::None;
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // CooperativeMatrixType
 //===----------------------------------------------------------------------===//
@@ -589,9 +615,6 @@ StorageClass PointerType::getStorageClass() const {
 
 void PointerType::getExtensions(SPIRVType::ExtensionArrayRefVector &extensions,
                                 Optional<StorageClass> storage) {
-  if (storage)
-    assert(*storage == getStorageClass() && "inconsistent storage class!");
-
   // Use this pointer type's storage class because this pointer indicates we are
   // using the pointee type in that specific storage class.
   getPointeeType().cast<SPIRVType>().getExtensions(extensions,
@@ -604,9 +627,6 @@ void PointerType::getExtensions(SPIRVType::ExtensionArrayRefVector &extensions,
 void PointerType::getCapabilities(
     SPIRVType::CapabilityArrayRefVector &capabilities,
     Optional<StorageClass> storage) {
-  if (storage)
-    assert(*storage == getStorageClass() && "inconsistent storage class!");
-
   // Use this pointer type's storage class because this pointer indicates we are
   // using the pointee type in that specific storage class.
   getPointeeType().cast<SPIRVType>().getCapabilities(capabilities,
@@ -812,6 +832,19 @@ void ScalarType::getCapabilities(
 #undef WIDTH_CASE
 }
 
+Optional<int64_t> ScalarType::getSizeInBytes() {
+  auto bitWidth = getIntOrFloatBitWidth();
+  // According to the SPIR-V spec:
+  // "There is no physical size or bit pattern defined for values with boolean
+  // type. If they are stored (in conjunction with OpVariable), they can only
+  // be used with logical addressing operations, not physical, and only with
+  // non-externally visible shader Storage Classes: Workgroup, CrossWorkgroup,
+  // Private, Function, Input, and Output."
+  if (bitWidth == 1)
+    return llvm::None;
+  return bitWidth / 8;
+}
+
 //===----------------------------------------------------------------------===//
 // SPIRVType
 //===----------------------------------------------------------------------===//
@@ -867,6 +900,14 @@ void SPIRVType::getCapabilities(
   }
 }
 
+Optional<int64_t> SPIRVType::getSizeInBytes() {
+  if (auto scalarType = dyn_cast<ScalarType>())
+    return scalarType.getSizeInBytes();
+  if (auto compositeType = dyn_cast<CompositeType>())
+    return compositeType.getSizeInBytes();
+  return llvm::None;
+}
+
 //===----------------------------------------------------------------------===//
 // StructType
 //===----------------------------------------------------------------------===//
@@ -1092,13 +1133,25 @@ bool MatrixType::isValidColumnType(Type columnType) {
   return false;
 }
 
-Type MatrixType::getElementType() const { return getImpl()->columnType; }
+Type MatrixType::getColumnType() const { return getImpl()->columnType; }
+
+Type MatrixType::getElementType() const {
+  return getImpl()->columnType.cast<VectorType>().getElementType();
+}
+
+unsigned MatrixType::getNumColumns() const { return getImpl()->columnCount; }
+
+unsigned MatrixType::getNumRows() const {
+  return getImpl()->columnType.cast<VectorType>().getShape()[0];
+}
 
-unsigned MatrixType::getNumElements() const { return getImpl()->columnCount; }
+unsigned MatrixType::getNumElements() const {
+  return (getImpl()->columnCount) * getNumRows();
+}
 
 void MatrixType::getExtensions(SPIRVType::ExtensionArrayRefVector &extensions,
                                Optional<StorageClass> storage) {
-  getElementType().cast<SPIRVType>().getExtensions(extensions, storage);
+  getColumnType().cast<SPIRVType>().getExtensions(extensions, storage);
 }
 
 void MatrixType::getCapabilities(
@@ -1110,5 +1163,5 @@ void MatrixType::getCapabilities(
     capabilities.push_back(ref);
   }
   // Add any capabilities associated with the underlying vectors (i.e., columns)
-  getElementType().cast<SPIRVType>().getCapabilities(capabilities, storage);
+  getColumnType().cast<SPIRVType>().getCapabilities(capabilities, storage);
 }
diff --git a/mlir/lib/Dialect/SPIRV/Serialization/Deserializer.cpp b/mlir/lib/Dialect/SPIRV/Serialization/Deserializer.cpp
index 4bb9e6d26b97d..4e6b294beaae3 100644
--- a/mlir/lib/Dialect/SPIRV/Serialization/Deserializer.cpp
+++ b/mlir/lib/Dialect/SPIRV/Serialization/Deserializer.cpp
@@ -14,6 +14,7 @@
 
 #include "mlir/Dialect/SPIRV/SPIRVAttributes.h"
 #include "mlir/Dialect/SPIRV/SPIRVBinaryUtils.h"
+#include "mlir/Dialect/SPIRV/SPIRVModule.h"
 #include "mlir/Dialect/SPIRV/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/SPIRVTypes.h"
 #include "mlir/IR/BlockAndValueMapping.h"
@@ -91,7 +92,7 @@ using BlockMergeInfoMap = DenseMap<Block *, BlockMergeInfo>;
 /// higher-order bits. So this deserializer uses that to get instruction
 /// boundary and parse instructions and build a SPIR-V ModuleOp gradually.
 ///
-// TODO(antiagainst): clean up created ops on errors
+// TODO: clean up created ops on errors
 class Deserializer {
 public:
   /// Creates a deserializer for the given SPIR-V `binary` module.
@@ -400,7 +401,8 @@ class Deserializer {
   /// Method to deserialize an operation in the SPIR-V dialect that is a mirror
   /// of an instruction in the SPIR-V spec. This is auto generated if hasOpcode
   /// == 1 and autogenSerialization == 1 in ODS.
-  template <typename OpTy> LogicalResult processOp(ArrayRef<uint32_t> words) {
+  template <typename OpTy>
+  LogicalResult processOp(ArrayRef<uint32_t> words) {
     return emitError(unknownLoc, "unsupported deserialization for ")
            << OpTy::getOperationName() << " op";
   }
@@ -419,7 +421,7 @@ class Deserializer {
   /// MLIRContext to create SPIR-V ModuleOp into.
   MLIRContext *context;
 
-  // TODO(antiagainst): create Location subclass for binary blob
+  // TODO: create Location subclass for binary blob
   Location unknownLoc;
 
   /// The SPIR-V ModuleOp.
@@ -601,7 +603,7 @@ LogicalResult Deserializer::processHeader() {
            << majorVersion;
   }
 
-  // TODO(antiagainst): generator number, bound, schema
+  // TODO: generator number, bound, schema
   curOffset = spirv::kHeaderWordCount;
   return success();
 }
@@ -675,7 +677,7 @@ LogicalResult Deserializer::processMemoryModel(ArrayRef<uint32_t> operands) {
 }
 
 LogicalResult Deserializer::processDecoration(ArrayRef<uint32_t> words) {
-  // TODO : This function should also be auto-generated. For now, since only a
+  // TODO: This function should also be auto-generated. For now, since only a
   // few decorations are processed/handled in a meaningful manner, going with a
   // manual implementation.
   if (words.size() < 2) {
@@ -803,7 +805,7 @@ LogicalResult Deserializer::processFunction(ArrayRef<uint32_t> operands) {
     return emitError(unknownLoc, "unknown Function Control: ") << operands[2];
   }
   if (functionControl.getValue() != spirv::FunctionControl::None) {
-    /// TODO : Handle different function controls
+    /// TODO: Handle different function controls
     return emitError(unknownLoc, "unhandled Function Control: '")
            << spirv::stringifyFunctionControl(functionControl.getValue())
            << "'";
@@ -1196,7 +1198,7 @@ LogicalResult Deserializer::processArrayType(ArrayRef<uint32_t> operands) {
   }
 
   unsigned count = 0;
-  // TODO(antiagainst): The count can also come frome a specialization constant.
+  // TODO: The count can also come frome a specialization constant.
   auto countInfo = getConstant(operands[2]);
   if (!countInfo) {
     return emitError(unknownLoc, "OpTypeArray count <id> ")
@@ -1335,7 +1337,7 @@ LogicalResult Deserializer::processStructType(ArrayRef<uint32_t> operands) {
   }
   typeMap[operands[0]] =
       spirv::StructType::get(memberTypes, offsetInfo, memberDecorationsInfo);
-  // TODO(ravishankarm): Update StructType to have member name as attribute as
+  // TODO: Update StructType to have member name as attribute as
   // well.
   return success();
 }
@@ -1566,8 +1568,8 @@ LogicalResult Deserializer::processConstantNull(ArrayRef<uint32_t> operands) {
     return success();
   }
 
-    return emitError(unknownLoc, "unsupported OpConstantNull type: ")
-           << resultType;
+  return emitError(unknownLoc, "unsupported OpConstantNull type: ")
+         << resultType;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1822,7 +1824,7 @@ spirv::LoopOp ControlFlowStructurizer::createLoopOp() {
   // merge block so that the newly created LoopOp will be inserted there.
   OpBuilder builder(&mergeBlock->front());
 
-  // TODO(antiagainst): handle loop control properly
+  // TODO: handle loop control properly
   auto loopOp = builder.create<spirv::LoopOp>(location);
   loopOp.addEntryAndMergeBlock();
 
@@ -1965,7 +1967,7 @@ LogicalResult ControlFlowStructurizer::structurizeImpl() {
     // selection/loop. If so, they will be recorded within blockMergeInfo.
     // We need to update the pointers there to the newly remapped ones so we can
     // continue structurizing them later.
-    // TODO(antiagainst): The asserts in the following assumes input SPIR-V blob
+    // TODO: The asserts in the following assumes input SPIR-V blob
     // forms correctly nested selection/loop constructs. We should relax this
     // and support error cases better.
     auto it = blockMergeInfo.find(block);
@@ -2509,18 +2511,88 @@ Deserializer::processOp<spirv::MemoryBarrierOp>(ArrayRef<uint32_t> operands) {
   return success();
 }
 
+template <>
+LogicalResult
+Deserializer::processOp<spirv::CopyMemoryOp>(ArrayRef<uint32_t> words) {
+  SmallVector<Type, 1> resultTypes;
+  size_t wordIndex = 0;
+  SmallVector<Value, 4> operands;
+  SmallVector<NamedAttribute, 4> attributes;
+
+  if (wordIndex < words.size()) {
+    auto arg = getValue(words[wordIndex]);
+
+    if (!arg) {
+      return emitError(unknownLoc, "unknown result <id> : ")
+             << words[wordIndex];
+    }
+
+    operands.push_back(arg);
+    wordIndex++;
+  }
+
+  if (wordIndex < words.size()) {
+    auto arg = getValue(words[wordIndex]);
+
+    if (!arg) {
+      return emitError(unknownLoc, "unknown result <id> : ")
+             << words[wordIndex];
+    }
+
+    operands.push_back(arg);
+    wordIndex++;
+  }
+
+  bool isAlignedAttr = false;
+
+  if (wordIndex < words.size()) {
+    auto attrValue = words[wordIndex++];
+    attributes.push_back(opBuilder.getNamedAttr(
+        "memory_access", opBuilder.getI32IntegerAttr(attrValue)));
+    isAlignedAttr = (attrValue == 2);
+  }
+
+  if (isAlignedAttr && wordIndex < words.size()) {
+    attributes.push_back(opBuilder.getNamedAttr(
+        "alignment", opBuilder.getI32IntegerAttr(words[wordIndex++])));
+  }
+
+  if (wordIndex < words.size()) {
+    attributes.push_back(opBuilder.getNamedAttr(
+        "source_memory_access",
+        opBuilder.getI32IntegerAttr(words[wordIndex++])));
+  }
+
+  if (wordIndex < words.size()) {
+    attributes.push_back(opBuilder.getNamedAttr(
+        "source_alignment", opBuilder.getI32IntegerAttr(words[wordIndex++])));
+  }
+
+  if (wordIndex != words.size()) {
+    return emitError(unknownLoc,
+                     "found more operands than expected when deserializing "
+                     "spirv::CopyMemoryOp, only ")
+           << wordIndex << " of " << words.size() << " processed";
+  }
+
+  Location loc = createFileLineColLoc(opBuilder);
+  opBuilder.create<spirv::CopyMemoryOp>(loc, resultTypes, operands, attributes);
+
+  return success();
+}
+
 // Pull in auto-generated Deserializer::dispatchToAutogenDeserialization() and
 // various Deserializer::processOp<...>() specializations.
 #define GET_DESERIALIZATION_FNS
 #include "mlir/Dialect/SPIRV/SPIRVSerialization.inc"
 } // namespace
 
-Optional<spirv::ModuleOp> spirv::deserialize(ArrayRef<uint32_t> binary,
-                                             MLIRContext *context) {
+spirv::OwningSPIRVModuleRef spirv::deserialize(ArrayRef<uint32_t> binary,
+                                               MLIRContext *context) {
   Deserializer deserializer(binary, context);
 
   if (failed(deserializer.deserialize()))
-    return llvm::None;
+    return nullptr;
 
-  return deserializer.collect();
+  return deserializer.collect().getValueOr(nullptr);
 }
diff --git a/mlir/lib/Dialect/SPIRV/Serialization/Serializer.cpp b/mlir/lib/Dialect/SPIRV/Serialization/Serializer.cpp
index f8641873fd958..859ea556f39f2 100644
--- a/mlir/lib/Dialect/SPIRV/Serialization/Serializer.cpp
+++ b/mlir/lib/Dialect/SPIRV/Serialization/Serializer.cpp
@@ -283,7 +283,7 @@ class Serializer {
   /// iterates the DenseElementsAttr to construct the constant array, and
   /// returns the result <id>  associated with it. Returns 0 if failed. Note
   /// that the size of `index` must match the rank.
-  /// TODO(hanchung): Consider to enhance splat elements cases. For splat cases,
+  /// TODO: Consider to enhance splat elements cases. For splat cases,
   /// we don't need to loop over all elements, especially when the splat value
   /// is zero. We can use OpConstantNull when the value is zero.
   uint32_t prepareDenseElementsConstant(Location loc, Type constType,
@@ -364,7 +364,8 @@ class Serializer {
   /// Method to serialize an operation in the SPIR-V dialect that is a mirror of
   /// an instruction in the SPIR-V spec. This is auto generated if hasOpcode ==
   /// 1 and autogenSerialization == 1 in ODS.
-  template <typename OpTy> LogicalResult processOp(OpTy op) {
+  template <typename OpTy>
+  LogicalResult processOp(OpTy op) {
     return op.emitError("unsupported op serialization");
   }
 
@@ -511,7 +512,7 @@ LogicalResult Serializer::serialize() {
   if (failed(module.verify()))
     return failure();
 
-  // TODO(antiagainst): handle the other sections
+  // TODO: handle the other sections
   processCapability();
   processExtension();
   processMemoryModel();
@@ -773,7 +774,7 @@ LogicalResult Serializer::processFuncOp(spirv::FuncOp op) {
   operands.push_back(resTypeID);
   auto funcID = getOrCreateFunctionID(op.getName());
   operands.push_back(funcID);
-  // TODO : Support other function control options.
+  // TODO: Support other function control options.
   operands.push_back(static_cast<uint32_t>(spirv::FunctionControl::None));
   operands.push_back(fnTypeID);
   encodeInstructionInto(functionHeader, spirv::Opcode::OpFunction, operands);
@@ -1127,16 +1128,16 @@ Serializer::prepareBasicType(Location loc, Type type, uint32_t resultID,
 
   if (auto matrixType = type.dyn_cast<spirv::MatrixType>()) {
     uint32_t elementTypeID = 0;
-    if (failed(processType(loc, matrixType.getElementType(), elementTypeID))) {
+    if (failed(processType(loc, matrixType.getColumnType(), elementTypeID))) {
       return failure();
     }
     typeEnum = spirv::Opcode::OpTypeMatrix;
     operands.push_back(elementTypeID);
-    operands.push_back(matrixType.getNumElements());
+    operands.push_back(matrixType.getNumColumns());
     return success();
   }
 
-  // TODO(ravishankarm) : Handle other types.
+  // TODO: Handle other types.
   return emitError(loc, "unhandled type in serialization: ") << type;
 }
 
@@ -1229,7 +1230,7 @@ uint32_t Serializer::prepareArrayConstant(Location loc, Type constType,
   return resultID;
 }
 
-// TODO(hanchung): Turn the below function into iterative function, instead of
+// TODO: Turn the below function into iterative function, instead of
 // recursive function.
 uint32_t
 Serializer::prepareDenseElementsConstant(Location loc, Type constType,
@@ -1572,7 +1573,7 @@ LogicalResult Serializer::processSelectionOp(spirv::SelectionOp selectionOp) {
   auto emitSelectionMerge = [&]() {
     emitDebugLine(functionBody, loc);
     lastProcessedWasMergeInst = true;
-    // TODO(antiagainst): properly support selection control here
+    // TODO: properly support selection control here
     encodeInstructionInto(
         functionBody, spirv::Opcode::OpSelectionMerge,
         {mergeID, static_cast<uint32_t>(spirv::SelectionControl::None)});
@@ -1635,7 +1636,7 @@ LogicalResult Serializer::processLoopOp(spirv::LoopOp loopOp) {
   auto emitLoopMerge = [&]() {
     emitDebugLine(functionBody, loc);
     lastProcessedWasMergeInst = true;
-    // TODO(antiagainst): properly support loop control here
+    // TODO: properly support loop control here
     encodeInstructionInto(
         functionBody, spirv::Opcode::OpLoopMerge,
         {mergeID, continueID, static_cast<uint32_t>(spirv::LoopControl::None)});
@@ -1904,6 +1905,51 @@ Serializer::processOp<spirv::FunctionCallOp>(spirv::FunctionCallOp op) {
                                operands);
 }
 
+template <>
+LogicalResult
+Serializer::processOp<spirv::CopyMemoryOp>(spirv::CopyMemoryOp op) {
+  SmallVector<uint32_t, 4> operands;
+  SmallVector<StringRef, 2> elidedAttrs;
+
+  for (Value operand : op.getOperation()->getOperands()) {
+    auto id = getValueID(operand);
+    assert(id && "use before def!");
+    operands.push_back(id);
+  }
+
+  if (auto attr = op.getAttr("memory_access")) {
+    operands.push_back(static_cast<uint32_t>(
+        attr.cast<IntegerAttr>().getValue().getZExtValue()));
+  }
+
+  elidedAttrs.push_back("memory_access");
+
+  if (auto attr = op.getAttr("alignment")) {
+    operands.push_back(static_cast<uint32_t>(
+        attr.cast<IntegerAttr>().getValue().getZExtValue()));
+  }
+
+  elidedAttrs.push_back("alignment");
+
+  if (auto attr = op.getAttr("source_memory_access")) {
+    operands.push_back(static_cast<uint32_t>(
+        attr.cast<IntegerAttr>().getValue().getZExtValue()));
+  }
+
+  elidedAttrs.push_back("source_memory_access");
+
+  if (auto attr = op.getAttr("source_alignment")) {
+    operands.push_back(static_cast<uint32_t>(
+        attr.cast<IntegerAttr>().getValue().getZExtValue()));
+  }
+
+  elidedAttrs.push_back("source_alignment");
+  emitDebugLine(functionBody, op.getLoc());
+  encodeInstructionInto(functionBody, spirv::Opcode::OpCopyMemory, operands);
+
+  return success();
+}
+
 // Pull in auto-generated Serializer::dispatchToAutogenSerialization() and
 // various Serializer::processOp<...>() specializations.
 #define GET_SERIALIZATION_FNS
diff --git a/mlir/lib/Dialect/SPIRV/Serialization/TranslateRegistration.cpp b/mlir/lib/Dialect/SPIRV/Serialization/TranslateRegistration.cpp
index 4c3fb1e8d422f..42b458d314cae 100644
--- a/mlir/lib/Dialect/SPIRV/Serialization/TranslateRegistration.cpp
+++ b/mlir/lib/Dialect/SPIRV/Serialization/TranslateRegistration.cpp
@@ -11,6 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "mlir/Dialect/SPIRV/SPIRVModule.h"
 #include "mlir/Dialect/SPIRV/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Serialization.h"
 #include "mlir/IR/Builders.h"
@@ -49,13 +50,13 @@ static OwningModuleRef deserializeModule(const llvm::MemoryBuffer *input,
   auto binary = llvm::makeArrayRef(reinterpret_cast<const uint32_t *>(start),
                                    size / sizeof(uint32_t));
 
-  auto spirvModule = spirv::deserialize(binary, context);
+  spirv::OwningSPIRVModuleRef spirvModule = spirv::deserialize(binary, context);
   if (!spirvModule)
     return {};
 
   OwningModuleRef module(ModuleOp::create(FileLineColLoc::get(
       input->getBufferIdentifier(), /*line=*/0, /*column=*/0, context)));
-  module->getBody()->push_front(spirvModule->getOperation());
+  module->getBody()->push_front(spirvModule.release());
 
   return module;
 }
@@ -136,14 +137,14 @@ static LogicalResult roundTripModule(llvm::SourceMgr &sourceMgr,
     return failure();
 
   // Then deserialize to get back a SPIR-V module.
-  auto spirvModule = spirv::deserialize(binary, context);
+  spirv::OwningSPIRVModuleRef spirvModule = spirv::deserialize(binary, context);
   if (!spirvModule)
     return failure();
 
   // Wrap around in a new MLIR module.
   OwningModuleRef dstModule(ModuleOp::create(FileLineColLoc::get(
       /*filename=*/"", /*line=*/0, /*column=*/0, context)));
-  dstModule->getBody()->push_front(spirvModule->getOperation());
+  dstModule->getBody()->push_front(spirvModule.release());
   dstModule->print(output);
 
   return mlir::success();
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/CMakeLists.txt b/mlir/lib/Dialect/SPIRV/Transforms/CMakeLists.txt
index 632194f213d69..228d8482e6be3 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/SPIRV/Transforms/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_mlir_dialect_library(MLIRSPIRVTransforms
   DecorateSPIRVCompositeTypeLayoutPass.cpp
   LowerABIAttributesPass.cpp
+  RewriteInsertsPass.cpp
   UpdateVCEPass.cpp
 
   ADDITIONAL_HEADER_DIRS
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
index 5bd425ae9107b..be1d271413901 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
@@ -72,13 +72,13 @@ getInterfaceVariables(spirv::FuncOp funcOp,
   }
   llvm::SetVector<Operation *> interfaceVarSet;
 
-  // TODO(ravishankarm) : This should in reality traverse the entry function
+  // TODO: This should in reality traverse the entry function
   // call graph and collect all the interfaces. For now, just traverse the
   // instructions in this function.
   funcOp.walk([&](spirv::AddressOfOp addressOfOp) {
     auto var =
         module.lookupSymbol<spirv::GlobalVariableOp>(addressOfOp.variable());
-    // TODO(antiagainst): Per SPIR-V spec: "Before version 1.4, the interface’s
+    // TODO: Per SPIR-V spec: "Before version 1.4, the interface’s
     // storage classes are limited to the Input and Output storage classes.
     // Starting with version 1.4, the interface’s storage classes are all
     // storage classes used in declaring all global variables referenced by the
@@ -158,7 +158,7 @@ LogicalResult ProcessInterfaceVarABI::matchAndRewrite(
     ConversionPatternRewriter &rewriter) const {
   if (!funcOp.getAttrOfType<spirv::EntryPointABIAttr>(
           spirv::getEntryPointABIAttrName())) {
-    // TODO(ravishankarm) : Non-entry point functions are not handled.
+    // TODO: Non-entry point functions are not handled.
     return failure();
   }
   TypeConverter::SignatureConversion signatureConverter(
@@ -169,7 +169,7 @@ LogicalResult ProcessInterfaceVarABI::matchAndRewrite(
     auto abiInfo = funcOp.getArgAttrOfType<spirv::InterfaceVarABIAttr>(
         argType.index(), attrName);
     if (!abiInfo) {
-      // TODO(ravishankarm) : For non-entry point functions, it should be legal
+      // TODO: For non-entry point functions, it should be legal
       // to pass around scalar/vector values and return a scalar/vector. For now
       // non-entry point functions are not handled in this ABI lowering and will
       // produce an error.
@@ -187,7 +187,7 @@ LogicalResult ProcessInterfaceVarABI::matchAndRewrite(
         rewriter.create<spirv::AddressOfOp>(funcOp.getLoc(), var);
     // Check if the arg is a scalar or vector type. In that case, the value
     // needs to be loaded into registers.
-    // TODO(ravishankarm) : This is loading value of the scalar into registers
+    // TODO: This is loading value of the scalar into registers
     // at the start of the function. It is probably better to do the load just
     // before the use. There might be multiple loads and currently there is no
     // easy way to replace all uses with a sequence of operations.
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
new file mode 100644
index 0000000000000..56fc4a9b60a6c
--- /dev/null
+++ b/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
@@ -0,0 +1,115 @@
+//===- RewriteInsertsPass.cpp - MLIR conversion pass ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a pass to rewrite sequential chains of
+// `spirv::CompositeInsert` operations into `spirv::CompositeConstruct`
+// operations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "PassDetail.h"
+#include "mlir/Dialect/SPIRV/Passes.h"
+#include "mlir/Dialect/SPIRV/SPIRVOps.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/Module.h"
+
+using namespace mlir;
+
+namespace {
+
+/// Replaces sequential chains of `spirv::CompositeInsertOp` operation into
+/// `spirv::CompositeConstructOp` operation if possible.
+class RewriteInsertsPass
+    : public SPIRVRewriteInsertsPassBase<RewriteInsertsPass> {
+public:
+  void runOnOperation() override;
+
+private:
+  /// Collects a sequential insertion chain by the given
+  /// `spirv::CompositeInsertOp` operation, if the given operation is the last
+  /// in the chain.
+  LogicalResult
+  collectInsertionChain(spirv::CompositeInsertOp op,
+                        SmallVectorImpl<spirv::CompositeInsertOp> &insertions);
+};
+
+} // anonymous namespace
+
+void RewriteInsertsPass::runOnOperation() {
+  SmallVector<SmallVector<spirv::CompositeInsertOp, 4>, 4> workList;
+  getOperation().walk([this, &workList](spirv::CompositeInsertOp op) {
+    SmallVector<spirv::CompositeInsertOp, 4> insertions;
+    if (succeeded(collectInsertionChain(op, insertions)))
+      workList.push_back(insertions);
+  });
+
+  for (const auto &insertions : workList) {
+    auto lastCompositeInsertOp = insertions.back();
+    auto compositeType = lastCompositeInsertOp.getType();
+    auto location = lastCompositeInsertOp.getLoc();
+
+    SmallVector<Value, 4> operands;
+    // Collect inserted objects.
+    for (auto insertionOp : insertions)
+      operands.push_back(insertionOp.object());
+
+    OpBuilder builder(lastCompositeInsertOp);
+    auto compositeConstructOp = builder.create<spirv::CompositeConstructOp>(
+        location, compositeType, operands);
+
+    lastCompositeInsertOp.replaceAllUsesWith(
+        compositeConstructOp.getOperation()->getResult(0));
+
+    // Erase ops.
+    for (auto insertOp : llvm::reverse(insertions)) {
+      auto *op = insertOp.getOperation();
+      if (op->use_empty())
+        insertOp.erase();
+    }
+  }
+}
+
+LogicalResult RewriteInsertsPass::collectInsertionChain(
+    spirv::CompositeInsertOp op,
+    SmallVectorImpl<spirv::CompositeInsertOp> &insertions) {
+  auto indicesArrayAttr = op.indices().cast<ArrayAttr>();
+  // TODO: handle nested composite object.
+  if (indicesArrayAttr.size() == 1) {
+    auto numElements =
+        op.composite().getType().cast<spirv::CompositeType>().getNumElements();
+
+    auto index = indicesArrayAttr[0].cast<IntegerAttr>().getInt();
+    // Need a last index to collect a sequential chain.
+    if (index + 1 != numElements)
+      return failure();
+
+    insertions.resize(numElements);
+    while (true) {
+      insertions[index] = op;
+
+      if (index == 0)
+        return success();
+
+      op = op.composite().getDefiningOp<spirv::CompositeInsertOp>();
+      if (!op)
+        return failure();
+
+      --index;
+      indicesArrayAttr = op.indices().cast<ArrayAttr>();
+      if ((indicesArrayAttr.size() != 1) ||
+          (indicesArrayAttr[0].cast<IntegerAttr>().getInt() != index))
+        return failure();
+    }
+  }
+  return failure();
+}
+
+std::unique_ptr<mlir::OperationPass<spirv::ModuleOp>>
+mlir::spirv::createRewriteInsertsPass() {
+  return std::make_unique<RewriteInsertsPass>();
+}
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/UpdateVCEPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/UpdateVCEPass.cpp
index 8adbc76a3a037..13e04f80c2b86 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/UpdateVCEPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/UpdateVCEPass.cpp
@@ -163,7 +163,7 @@ void UpdateVCEPass::runOnOperation() {
   if (walkResult.wasInterrupted())
     return signalPassFailure();
 
-  // TODO(antiagainst): verify that the deduced version is consistent with
+  // TODO: verify that the deduced version is consistent with
   // SPIR-V ops' maximal version requirements.
 
   auto triple = spirv::VerCapExtAttr::get(
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 664c0cb05b804..0a0608bbcda46 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -317,21 +317,101 @@ OpFoldResult ConstShapeOp::fold(ArrayRef<Attribute>) { return shapeAttr(); }
 // CstrBroadcastableOp
 //===----------------------------------------------------------------------===//
 
+namespace {
+// Given an input shape Value, try to obtain the shape's values.
+LogicalResult getShapeVec(Value input, SmallVectorImpl<int64_t> &shapeValues) {
+  if (auto inputOp = input.getDefiningOp<ShapeOfOp>()) {
+    auto type = inputOp.arg().getType().dyn_cast<ShapedType>();
+    if (!type.hasRank())
+      return failure();
+    shapeValues = llvm::to_vector<6>(type.getShape());
+    return success();
+  } else if (auto inputOp = input.getDefiningOp<ConstShapeOp>()) {
+    shapeValues = llvm::to_vector<6>(inputOp.shape().getValues<int64_t>());
+    return success();
+  } else {
+    return failure();
+  }
+}
+
+// For shapes that were created by some operations, we can obtain partial
+// information on the shapes and sometimes determine if they will be
+// broadcastable with that.
+struct CstrBroadcastablePartialInfo
+    : public OpRewritePattern<CstrBroadcastableOp> {
+  using OpRewritePattern<CstrBroadcastableOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(CstrBroadcastableOp op,
+                                PatternRewriter &rewriter) const override {
+    SmallVector<int64_t, 6> lhsShape, rhsShape;
+    if (failed(getShapeVec(op.lhs(), lhsShape)))
+      return failure();
+    if (failed(getShapeVec(op.rhs(), rhsShape)))
+      return failure();
+    if (!OpTrait::util::staticallyKnownBroadcastable(lhsShape, rhsShape))
+      return failure();
+
+    rewriter.replaceOpWithNewOp<ConstWitnessOp>(op.getOperation(), true);
+    return success();
+  }
+};
+
+// Scalars are always broadcastable.
+struct CstrBroadcastableScalar : public OpRewritePattern<CstrBroadcastableOp> {
+  using OpRewritePattern<CstrBroadcastableOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(CstrBroadcastableOp op,
+                                PatternRewriter &rewriter) const override {
+    SmallVector<int64_t, 6> shape;
+    if (failed(getShapeVec(op.lhs(), shape)) || shape.size() > 0)
+      return failure();
+    if (failed(getShapeVec(op.rhs(), shape)) || shape.size() > 0)
+      return failure();
+
+    rewriter.replaceOpWithNewOp<ConstWitnessOp>(op.getOperation(), true);
+    return success();
+  }
+};
+
+} // namespace
+
 void CstrBroadcastableOp::getCanonicalizationPatterns(
     OwningRewritePatternList &patterns, MLIRContext *context) {
-  // If inputs are equal, return passing witness
-  patterns.insert<CstrBroadcastableEqOps>(context);
+  // Canonicalization patterns have overlap with the considerations during
+  // folding in case additional shape information is inferred at some point that
+  // does not result in folding.
+  patterns.insert<CstrBroadcastableEqOps, CstrBroadcastablePartialInfo,
+                  CstrBroadcastableScalar>(context);
 }
 
 OpFoldResult CstrBroadcastableOp::fold(ArrayRef<Attribute> operands) {
-  if (!operands[0] || !operands[1])
+  // Both operands are not needed if one is a scalar.
+  if (operands[0] &&
+      operands[0].cast<DenseIntElementsAttr>().getNumElements() == 0)
+    return BoolAttr::get(true, getContext());
+  if (operands[1] &&
+      operands[1].cast<DenseIntElementsAttr>().getNumElements() == 0)
+    return BoolAttr::get(true, getContext());
+
+  if (operands[0] && operands[1]) {
+    auto lhsShape = llvm::to_vector<6>(
+        operands[0].cast<DenseIntElementsAttr>().getValues<int64_t>());
+    auto rhsShape = llvm::to_vector<6>(
+        operands[1].cast<DenseIntElementsAttr>().getValues<int64_t>());
+    SmallVector<int64_t, 6> resultShape;
+    if (OpTrait::util::staticallyKnownBroadcastable(lhsShape, rhsShape))
+      return BoolAttr::get(true, getContext());
+  }
+
+  // Lastly, see if folding can be completed based on what constraints are known
+  // on the input shapes.
+  SmallVector<int64_t, 6> lhsShape, rhsShape;
+  if (failed(getShapeVec(lhs(), lhsShape)))
     return nullptr;
-  auto lhsShape = llvm::to_vector<6>(
-      operands[0].cast<DenseIntElementsAttr>().getValues<int64_t>());
-  auto rhsShape = llvm::to_vector<6>(
-      operands[1].cast<DenseIntElementsAttr>().getValues<int64_t>());
-  SmallVector<int64_t, 6> resultShape;
-  if (OpTrait::util::getBroadcastedShape(lhsShape, rhsShape, resultShape))
+  if (failed(getShapeVec(rhs(), rhsShape)))
+    return nullptr;
+
+  if (OpTrait::util::staticallyKnownBroadcastable(lhsShape, rhsShape))
     return BoolAttr::get(true, getContext());
 
   // Because a failing witness result here represents an eventual assertion
@@ -364,6 +444,11 @@ OpFoldResult CstrEqOp::fold(ArrayRef<Attribute> operands) {
 // ConstSizeOp
 //===----------------------------------------------------------------------===//
 
+void ConstSizeOp::build(OpBuilder &builder, OperationState &result,
+                        int64_t value) {
+  build(builder, result, builder.getIndexAttr(value));
+}
+
 OpFoldResult ConstSizeOp::fold(ArrayRef<Attribute>) { return valueAttr(); }
 
 void ConstSizeOp::getAsmResultNames(
@@ -392,6 +477,11 @@ OpFoldResult IndexToSizeOp::fold(ArrayRef<Attribute> operands) {
   return {};
 }
 
+void IndexToSizeOp::getCanonicalizationPatterns(
+    OwningRewritePatternList &patterns, MLIRContext *context) {
+  patterns.insert<SizeToIndexToSizeCanonicalization>(context);
+}
+
 //===----------------------------------------------------------------------===//
 // FromExtentsOp
 //===----------------------------------------------------------------------===//
@@ -437,6 +527,58 @@ void GetExtentOp::build(OpBuilder &builder, OperationState &result, Value shape,
   build(builder, result, shape, dimValue);
 }
 
+//===----------------------------------------------------------------------===//
+// RankOp
+//===----------------------------------------------------------------------===//
+
+OpFoldResult RankOp::fold(ArrayRef<Attribute> operands) {
+  auto shape = operands[0].dyn_cast_or_null<DenseIntElementsAttr>();
+  if (!shape)
+    return {};
+  int64_t rank = shape.getNumElements();
+  Builder builder(getContext());
+  return builder.getIndexAttr(rank);
+}
+
+/// Evaluate the `rank` operation for shapes of ranked tensors at compile time.
+/// Constant folding fails in cases where only the rank is constant, not the
+/// shape itself.
+/// This canonicalization matches `shape.rank(shape.shape_of(%ranked_tensor))`.
+///
+/// Example:
+///
+/// %shape = shape.shape_of %ranked_tensor : tensor<1x2x?xf32>
+/// %rank = shape.rank %shape
+///
+/// becomes
+///
+/// %rank = shape.const_size 3
+
+namespace {
+struct RankShapeOfCanonicalizationPattern : public OpRewritePattern<RankOp> {
+  using OpRewritePattern<RankOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(RankOp op,
+                                PatternRewriter &rewriter) const override {
+    auto shapeOfOp = op.shape().getDefiningOp<ShapeOfOp>();
+    if (!shapeOfOp)
+      return failure();
+    auto rankedTensorType =
+        shapeOfOp.arg().getType().dyn_cast<RankedTensorType>();
+    if (!rankedTensorType)
+      return failure();
+    int64_t rank = rankedTensorType.getRank();
+    rewriter.replaceOpWithNewOp<ConstSizeOp>(op.getOperation(), rank);
+    return success();
+  }
+};
+} // namespace
+
+void RankOp::getCanonicalizationPatterns(OwningRewritePatternList &patterns,
+                                         MLIRContext *context) {
+  patterns.insert<RankShapeOfCanonicalizationPattern>(context);
+}
+
 //===----------------------------------------------------------------------===//
 // NumElementsOp
 //===----------------------------------------------------------------------===//
@@ -479,6 +621,11 @@ OpFoldResult SizeToIndexOp::fold(ArrayRef<Attribute> operands) {
   return {};
 }
 
+void SizeToIndexOp::getCanonicalizationPatterns(
+    OwningRewritePatternList &patterns, MLIRContext *context) {
+  patterns.insert<IndexToSizeToIndexCanonicalization>(context);
+}
+
 //===----------------------------------------------------------------------===//
 // YieldOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Shape/IR/ShapeCanonicalization.td b/mlir/lib/Dialect/Shape/IR/ShapeCanonicalization.td
index 78c9119f12920..b03e512999073 100644
--- a/mlir/lib/Dialect/Shape/IR/ShapeCanonicalization.td
+++ b/mlir/lib/Dialect/Shape/IR/ShapeCanonicalization.td
@@ -9,6 +9,7 @@ def AllInputShapesEq : Constraint<CPred< [{
 }]>>;
 
 // Canonicalization patterns.
+
 def CstrBroadcastableEqOps : Pat<(Shape_CstrBroadcastableOp:$op $lhs, $rhs),
   (Shape_ConstWitnessOp ConstBoolAttrTrue),
   [(EqualBinaryOperands $lhs, $rhs)]>;
@@ -16,3 +17,12 @@ def CstrBroadcastableEqOps : Pat<(Shape_CstrBroadcastableOp:$op $lhs, $rhs),
 def CstrEqEqOps : Pat<(Shape_CstrEqOp:$op $shapes),
   (Shape_ConstWitnessOp ConstBoolAttrTrue),
   [(AllInputShapesEq $shapes)]>;
+
+def IndexToSizeToIndexCanonicalization : Pat<
+  (Shape_SizeToIndexOp (Shape_IndexToSizeOp $arg)),
+  (replaceWithValue $arg)>;
+
+def SizeToIndexToSizeCanonicalization : Pat<
+  (Shape_IndexToSizeOp (Shape_SizeToIndexOp $arg)),
+  (replaceWithValue $arg)>;
+
diff --git a/mlir/lib/Dialect/StandardOps/CMakeLists.txt b/mlir/lib/Dialect/StandardOps/CMakeLists.txt
index f3b93d6013ce1..7d61aea3116ed 100644
--- a/mlir/lib/Dialect/StandardOps/CMakeLists.txt
+++ b/mlir/lib/Dialect/StandardOps/CMakeLists.txt
@@ -15,6 +15,7 @@ add_mlir_dialect_library(MLIRStandardOps
   MLIREDSC
   MLIRIR
   MLIRSideEffectInterfaces
+  MLIRVectorUnrollInterface
   MLIRViewLikeInterface
   )
 
diff --git a/mlir/lib/Dialect/StandardOps/IR/Ops.cpp b/mlir/lib/Dialect/StandardOps/IR/Ops.cpp
index ca4fe836495b9..7c28608d98b91 100644
--- a/mlir/lib/Dialect/StandardOps/IR/Ops.cpp
+++ b/mlir/lib/Dialect/StandardOps/IR/Ops.cpp
@@ -896,7 +896,7 @@ OpFoldResult CmpFOp::fold(ArrayRef<Attribute> operands) {
   auto lhs = operands.front().dyn_cast_or_null<FloatAttr>();
   auto rhs = operands.back().dyn_cast_or_null<FloatAttr>();
 
-  // TODO(gcmn) We could actually do some intelligent things if we know only one
+  // TODO: We could actually do some intelligent things if we know only one
   // of the operands, but it's inf or nan.
   if (!lhs || !rhs)
     return {};
@@ -1120,7 +1120,8 @@ static LogicalResult verify(ConstantOp &op) {
     auto fn =
         op.getParentOfType<ModuleOp>().lookupSymbol<FuncOp>(fnAttr.getValue());
     if (!fn)
-      return op.emitOpError("reference to undefined function 'bar'");
+      return op.emitOpError()
+             << "reference to undefined function '" << fnAttr.getValue() << "'";
 
     // Check that the referenced function has the correct type.
     if (fn.getType() != type)
@@ -1175,8 +1176,7 @@ bool ConstantOp::isBuildableWith(Attribute value, Type type) {
   if (value.getType() != type)
     return false;
   // Finally, check that the attribute kind is handled.
-  return value.isa<IntegerAttr>() || value.isa<FloatAttr>() ||
-         value.isa<ElementsAttr>() || value.isa<UnitAttr>();
+  return value.isa<IntegerAttr, FloatAttr, ElementsAttr, UnitAttr>();
 }
 
 void ConstantFloatOp::build(OpBuilder &builder, OperationState &result,
@@ -1273,8 +1273,13 @@ void DimOp::build(OpBuilder &builder, OperationState &result,
                   Value memrefOrTensor, int64_t index) {
   auto loc = result.location;
   Value indexValue = builder.create<ConstantIndexOp>(loc, index);
+  build(builder, result, memrefOrTensor, indexValue);
+}
+
+void DimOp::build(OpBuilder &builder, OperationState &result,
+                  Value memrefOrTensor, Value index) {
   auto indexTy = builder.getIndexType();
-  build(builder, result, indexTy, memrefOrTensor, indexValue);
+  build(builder, result, indexTy, memrefOrTensor, index);
 }
 
 Optional<int64_t> DimOp::getConstantIndex() {
@@ -1817,6 +1822,8 @@ OpFoldResult LoadOp::fold(ArrayRef<Attribute> cstOperands) {
 // MemRefCastOp
 //===----------------------------------------------------------------------===//
 
+Value MemRefCastOp::getViewSource() { return source(); }
+
 bool MemRefCastOp::areCastCompatible(Type a, Type b) {
   auto aT = a.dyn_cast<MemRefType>();
   auto bT = b.dyn_cast<MemRefType>();
@@ -2097,7 +2104,7 @@ static LogicalResult verify(SelectOp op) {
   // If the result type is a vector or tensor, the type can be a mask with the
   // same elements.
   Type resultType = op.getType();
-  if (!resultType.isa<TensorType>() && !resultType.isa<VectorType>())
+  if (!resultType.isa<TensorType, VectorType>())
     return op.emitOpError()
            << "expected condition to be a signless i1, but got "
            << conditionType;
@@ -2216,8 +2223,7 @@ OpFoldResult SplatOp::fold(ArrayRef<Attribute> operands) {
   assert(operands.size() == 1 && "splat takes one operand");
 
   auto constOperand = operands.front();
-  if (!constOperand ||
-      (!constOperand.isa<IntegerAttr>() && !constOperand.isa<FloatAttr>()))
+  if (!constOperand || !constOperand.isa<IntegerAttr, FloatAttr>())
     return {};
 
   auto shapedType = getType().cast<ShapedType>();
diff --git a/mlir/lib/Dialect/Traits.cpp b/mlir/lib/Dialect/Traits.cpp
index dc721adc7472f..2a557c489e0b7 100644
--- a/mlir/lib/Dialect/Traits.cpp
+++ b/mlir/lib/Dialect/Traits.cpp
@@ -13,6 +13,23 @@
 
 using namespace mlir;
 
+bool OpTrait::util::staticallyKnownBroadcastable(ArrayRef<int64_t> shape1,
+                                                 ArrayRef<int64_t> shape2) {
+  // Two dimensions are compatible when
+  //   1. they are defined and equal, or
+  //   2. one of them is 1
+  return llvm::all_of(llvm::zip(llvm::reverse(shape1), llvm::reverse(shape2)),
+                      [](auto dimensions) {
+                        auto dim1 = std::get<0>(dimensions);
+                        auto dim2 = std::get<1>(dimensions);
+                        if (dim1 == 1 || dim2 == 1)
+                          return true;
+                        if (dim1 == dim2 && !ShapedType::isDynamic(dim1))
+                          return true;
+                        return false;
+                      });
+}
+
 bool OpTrait::util::getBroadcastedShape(ArrayRef<int64_t> shape1,
                                         ArrayRef<int64_t> shape2,
                                         SmallVectorImpl<int64_t> &resultShape) {
@@ -107,7 +124,7 @@ Type OpTrait::util::getBroadcastedType(Type type1, Type type2,
   // Returns the type kind if the given type is a vector or ranked tensor type.
   // Returns llvm::None otherwise.
   auto getCompositeTypeKind = [](Type type) -> Optional<StandardTypes::Kind> {
-    if (type.isa<VectorType>() || type.isa<RankedTensorType>())
+    if (type.isa<VectorType, RankedTensorType>())
       return static_cast<StandardTypes::Kind>(type.getKind());
     return llvm::None;
   };
diff --git a/mlir/lib/Dialect/Vector/CMakeLists.txt b/mlir/lib/Dialect/Vector/CMakeLists.txt
index 7a5ed49cd9ce3..69a329917228d 100644
--- a/mlir/lib/Dialect/Vector/CMakeLists.txt
+++ b/mlir/lib/Dialect/Vector/CMakeLists.txt
@@ -19,4 +19,5 @@ add_mlir_dialect_library(MLIRVector
   MLIRSCF
   MLIRLoopAnalysis
   MLIRSideEffectInterfaces
+  MLIRVectorUnrollInterface
   )
diff --git a/mlir/lib/Dialect/Vector/VectorOps.cpp b/mlir/lib/Dialect/Vector/VectorOps.cpp
index 019f5fd946216..5e01fa26f32e9 100644
--- a/mlir/lib/Dialect/Vector/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/VectorOps.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/Function.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -159,7 +160,7 @@ static ParseResult parseContractionOp(OpAsmParser &parser,
   Type resultType;
   auto loc = parser.getCurrentLocation();
   DictionaryAttr dictAttr;
-  // TODO(andydavis, ntv) Unify linalg op attribute parsing.
+  // TODO: Unify linalg op attribute parsing.
   if (parser.parseAttribute(dictAttr, "_", result.attributes) ||
       parser.parseOperand(lhsInfo) || parser.parseComma() ||
       parser.parseOperand(rhsInfo) || parser.parseComma() ||
@@ -192,7 +193,7 @@ static ParseResult parseContractionOp(OpAsmParser &parser,
 }
 
 static void print(OpAsmPrinter &p, ContractionOp op) {
-  // TODO(andydavis, ntv) Unify printing code with linalg ops.
+  // TODO: Unify printing code with linalg ops.
   auto attrNames = op.getTraitAttrNames();
   llvm::StringSet<> traitAttrsSet;
   traitAttrsSet.insert(attrNames.begin(), attrNames.end());
@@ -469,6 +470,12 @@ SmallVector<AffineMap, 4> ContractionOp::getIndexingMaps() {
   return res;
 }
 
+Optional<SmallVector<int64_t, 4>> ContractionOp::getShapeForUnroll() {
+  SmallVector<int64_t, 4> shape;
+  getIterationBounds(shape);
+  return shape;
+}
+
 //===----------------------------------------------------------------------===//
 // ExtractElementOp
 //===----------------------------------------------------------------------===//
@@ -565,6 +572,188 @@ static LogicalResult verify(vector::ExtractOp op) {
   return success();
 }
 
+template <typename IntType>
+static SmallVector<IntType, 4> extractVector(ArrayAttr arrayAttr) {
+  return llvm::to_vector<4>(llvm::map_range(
+      arrayAttr.getAsRange<IntegerAttr>(),
+      [](IntegerAttr attr) { return static_cast<IntType>(attr.getInt()); }));
+}
+
+/// Fold the result of chains of ExtractOp in place by simply concatenating the
+/// positions.
+static LogicalResult foldExtractOpFromExtractChain(ExtractOp extractOp) {
+  if (!extractOp.vector().getDefiningOp<ExtractOp>())
+    return failure();
+
+  SmallVector<int64_t, 4> globalPosition;
+  ExtractOp currentOp = extractOp;
+  auto extractedPos = extractVector<int64_t>(currentOp.position());
+  globalPosition.append(extractedPos.rbegin(), extractedPos.rend());
+  while (ExtractOp nextOp = currentOp.vector().getDefiningOp<ExtractOp>()) {
+    currentOp = nextOp;
+    auto extractedPos = extractVector<int64_t>(currentOp.position());
+    globalPosition.append(extractedPos.rbegin(), extractedPos.rend());
+  }
+  extractOp.setOperand(currentOp.vector());
+  // OpBuilder is only used as a helper to build an I64ArrayAttr.
+  OpBuilder b(extractOp.getContext());
+  std::reverse(globalPosition.begin(), globalPosition.end());
+  extractOp.setAttr(ExtractOp::getPositionAttrName(),
+                    b.getI64ArrayAttr(globalPosition));
+  return success();
+}
+
+/// Fold the result of an ExtractOp in place when it comes from a TransposeOp.
+static LogicalResult foldExtractOpFromTranspose(ExtractOp extractOp) {
+  auto transposeOp = extractOp.vector().getDefiningOp<TransposeOp>();
+  if (!transposeOp)
+    return failure();
+
+  auto permutation = extractVector<unsigned>(transposeOp.transp());
+  auto extractedPos = extractVector<int64_t>(extractOp.position());
+
+  // If transposition permutation is larger than the ExtractOp, all minor
+  // dimensions must be an identity for folding to occur. If not, individual
+  // elements within the extracted value are transposed and this is not just a
+  // simple folding.
+  unsigned minorRank = permutation.size() - extractedPos.size();
+  MLIRContext *ctx = extractOp.getContext();
+  AffineMap permutationMap = AffineMap::getPermutationMap(permutation, ctx);
+  AffineMap minorMap = permutationMap.getMinorSubMap(minorRank);
+  if (minorMap && !minorMap.isMinorIdentity())
+    return failure();
+
+  //   %1 = transpose %0[x, y, z] : vector<axbxcxf32>
+  //   %2 = extract %1[u, v] : vector<..xf32>
+  // may turn into:
+  //   %2 = extract %0[w, x] : vector<..xf32>
+  // iff z == 2 and [w, x] = [x, y]^-1 o [u, v] here o denotes composition and
+  // -1 denotes the inverse.
+  permutationMap = permutationMap.getMajorSubMap(extractedPos.size());
+  // The major submap has fewer results but the same number of dims. To compose
+  // cleanly, we need to drop dims to form a "square matrix". This is possible
+  // because:
+  //   (a) this is a permutation map and
+  //   (b) the minor map has already been checked to be identity.
+  // Therefore, the major map cannot contain dims of position greater or equal
+  // than the number of results.
+  assert(llvm::all_of(permutationMap.getResults(),
+                      [&](AffineExpr e) {
+                        auto dim = e.dyn_cast<AffineDimExpr>();
+                        return dim && dim.getPosition() <
+                                          permutationMap.getNumResults();
+                      }) &&
+         "Unexpected map results depend on higher rank positions");
+  // Project on the first domain dimensions to allow composition.
+  permutationMap = AffineMap::get(permutationMap.getNumResults(), 0,
+                                  permutationMap.getResults(), ctx);
+
+  extractOp.setOperand(transposeOp.vector());
+  // Compose the inverse permutation map with the extractedPos.
+  auto newExtractedPos =
+      inversePermutation(permutationMap).compose(extractedPos);
+  // OpBuilder is only used as a helper to build an I64ArrayAttr.
+  OpBuilder b(extractOp.getContext());
+  extractOp.setAttr(ExtractOp::getPositionAttrName(),
+                    b.getI64ArrayAttr(newExtractedPos));
+
+  return success();
+}
+
+/// Fold an ExtractOp that is fed by a chain of InsertOps and TransposeOps. The
+/// result is always the input to some InsertOp.
+static Value foldExtractOpFromInsertChainAndTranspose(ExtractOp extractOp) {
+  MLIRContext *context = extractOp.getContext();
+  AffineMap permutationMap;
+  auto extractedPos = extractVector<unsigned>(extractOp.position());
+  // Walk back a chain of InsertOp/TransposeOp until we hit a match.
+  // Compose TransposeOp permutations as we walk back.
+  auto insertOp = extractOp.vector().getDefiningOp<vector::InsertOp>();
+  auto transposeOp = extractOp.vector().getDefiningOp<vector::TransposeOp>();
+  while (insertOp || transposeOp) {
+    if (transposeOp) {
+      // If it is transposed, compose the map and iterate.
+      auto permutation = extractVector<unsigned>(transposeOp.transp());
+      AffineMap newMap = AffineMap::getPermutationMap(permutation, context);
+      if (!permutationMap)
+        permutationMap = newMap;
+      else if (newMap.getNumInputs() != permutationMap.getNumResults())
+        return Value();
+      else
+        permutationMap = newMap.compose(permutationMap);
+      // Compute insert/transpose for the next iteration.
+      Value transposed = transposeOp.vector();
+      insertOp = transposed.getDefiningOp<vector::InsertOp>();
+      transposeOp = transposed.getDefiningOp<vector::TransposeOp>();
+      continue;
+    }
+
+    assert(insertOp);
+    Value insertionDest = insertOp.dest();
+    // If it is inserted into, either the position matches and we have a
+    // successful folding; or we iterate until we run out of
+    // InsertOp/TransposeOp. This is because `vector.insert %scalar, %vector`
+    // produces a new vector with 1 modified value/slice in exactly the static
+    // position we need to match.
+    auto insertedPos = extractVector<unsigned>(insertOp.position());
+    // Trivial permutations are solved with position equality checks.
+    if (!permutationMap || permutationMap.isIdentity()) {
+      if (extractedPos == insertedPos)
+        return insertOp.source();
+      // Fallthrough: if the position does not match, just skip to the next
+      // producing `vector.insert` / `vector.transpose`.
+      // Compute insert/transpose for the next iteration.
+      insertOp = insertionDest.getDefiningOp<vector::InsertOp>();
+      transposeOp = insertionDest.getDefiningOp<vector::TransposeOp>();
+      continue;
+    }
+
+    // More advanced permutations require application of the permutation.
+    // However, the rank of `insertedPos` may be different from that of the
+    // `permutationMap`. To support such case, we need to:
+    //   1. apply on the `insertedPos.size()` major dimensions
+    //   2. check the other dimensions of the permutation form a minor identity.
+    assert(permutationMap.isPermutation() && "expected a permutation");
+    if (insertedPos.size() == extractedPos.size()) {
+      bool fold = true;
+      for (unsigned idx = 0, sz = extractedPos.size(); idx < sz; ++idx) {
+        auto pos =
+            permutationMap.getResult(idx).cast<AffineDimExpr>().getPosition();
+        if (pos >= sz || insertedPos[pos] != extractedPos[idx]) {
+          fold = false;
+          break;
+        }
+      }
+      if (fold) {
+        assert(permutationMap.getNumResults() >= insertedPos.size() &&
+               "expected map of rank larger than insert indexing");
+        unsigned minorRank =
+            permutationMap.getNumResults() - insertedPos.size();
+        AffineMap minorMap = permutationMap.getMinorSubMap(minorRank);
+        if (!minorMap || minorMap.isMinorIdentity())
+          return insertOp.source();
+      }
+    }
+
+    // If we haven't found a match, just continue to the next producing
+    // `vector.insert` / `vector.transpose`.
+    // Compute insert/transpose for the next iteration.
+    insertOp = insertionDest.getDefiningOp<vector::InsertOp>();
+    transposeOp = insertionDest.getDefiningOp<vector::TransposeOp>();
+  }
+  return Value();
+}
+
+OpFoldResult ExtractOp::fold(ArrayRef<Attribute>) {
+  if (succeeded(foldExtractOpFromExtractChain(*this)))
+    return getResult();
+  if (succeeded(foldExtractOpFromTranspose(*this)))
+    return getResult();
+  if (auto val = foldExtractOpFromInsertChainAndTranspose(*this))
+    return val;
+  return OpFoldResult();
+}
+
 //===----------------------------------------------------------------------===//
 // ExtractSlicesOp
 //===----------------------------------------------------------------------===//
@@ -586,7 +775,7 @@ isValidExtractOrInsertSlicesType(Operation *op, VectorType vectorType,
                                  TupleType tupleType, ArrayRef<int64_t> sizes,
                                  ArrayRef<int64_t> strides) {
   // Check for non-unit strides.
-  // TODO(b/144845578) Support non-1 strides.
+  // TODO: Support non-1 strides.
   if (llvm::any_of(strides, [](int64_t s) { return s != 1; }))
     return op->emitError("requires unit strides");
   // Check that 'vectorType' rank matches rank of tuple element vectors.
@@ -858,7 +1047,7 @@ void InsertStridedSliceOp::build(OpBuilder &builder, OperationState &result,
   result.addAttribute(getStridesAttrName(), stridesAttr);
 }
 
-// TODO(ntv) Should be moved to Tablegen Confined attributes.
+// TODO: Should be moved to Tablegen Confined attributes.
 template <typename OpType>
 static LogicalResult isIntegerArrayAttrSmallerThanShape(OpType op,
                                                         ArrayAttr arrayAttr,
@@ -1014,10 +1203,13 @@ static ParseResult parseOuterProductOp(OpAsmParser &parser,
                             "expected at least 2 operands");
   VectorType vLHS = tLHS.dyn_cast<VectorType>();
   VectorType vRHS = tRHS.dyn_cast<VectorType>();
-  if (!vLHS || !vRHS)
-    return parser.emitError(parser.getNameLoc(), "expected 2 vector types");
-  VectorType resType = VectorType::get({vLHS.getDimSize(0), vRHS.getDimSize(0)},
-                                       vLHS.getElementType());
+  if (!vLHS)
+    return parser.emitError(parser.getNameLoc(),
+                            "expected vector type for operand #1");
+  VectorType resType =
+      vRHS ? VectorType::get({vLHS.getDimSize(0), vRHS.getDimSize(0)},
+                             vLHS.getElementType())
+           : VectorType::get({vLHS.getDimSize(0)}, vLHS.getElementType());
   return failure(
       parser.resolveOperand(operandsInfo[0], tLHS, result.operands) ||
       parser.resolveOperand(operandsInfo[1], tRHS, result.operands) ||
@@ -1027,19 +1219,32 @@ static ParseResult parseOuterProductOp(OpAsmParser &parser,
 }
 
 static LogicalResult verify(OuterProductOp op) {
+  Type tRHS = op.getOperandTypeRHS();
   VectorType vLHS = op.getOperandVectorTypeLHS(),
-             vRHS = op.getOperandVectorTypeRHS(),
+             vRHS = tRHS.dyn_cast<VectorType>(),
              vACC = op.getOperandVectorTypeACC(), vRES = op.getVectorType();
+
   if (vLHS.getRank() != 1)
     return op.emitOpError("expected 1-d vector for operand #1");
-  if (vRHS.getRank() != 1)
-    return op.emitOpError("expected 1-d vector for operand #2");
-  if (vRES.getRank() != 2)
-    return op.emitOpError("expected 2-d vector result");
-  if (vLHS.getDimSize(0) != vRES.getDimSize(0))
-    return op.emitOpError("expected #1 operand dim to match result dim #1");
-  if (vRHS.getDimSize(0) != vRES.getDimSize(1))
-    return op.emitOpError("expected #2 operand dim to match result dim #2");
+
+  if (vRHS) {
+    // Proper OUTER operation.
+    if (vRHS.getRank() != 1)
+      return op.emitOpError("expected 1-d vector for operand #2");
+    if (vRES.getRank() != 2)
+      return op.emitOpError("expected 2-d vector result");
+    if (vLHS.getDimSize(0) != vRES.getDimSize(0))
+      return op.emitOpError("expected #1 operand dim to match result dim #1");
+    if (vRHS.getDimSize(0) != vRES.getDimSize(1))
+      return op.emitOpError("expected #2 operand dim to match result dim #2");
+  } else {
+    // An AXPY operation.
+    if (vRES.getRank() != 1)
+      return op.emitOpError("expected 1-d vector result");
+    if (vLHS.getDimSize(0) != vRES.getDimSize(0))
+      return op.emitOpError("expected #1 operand dim to match result dim #1");
+  }
+
   if (vACC && vACC != vRES)
     return op.emitOpError("expected operand #3 of same type as result type");
   return success();
@@ -1325,7 +1530,7 @@ static LogicalResult verifyTransferOp(Operation *op, MemRefType memrefType,
     if (memrefVecEltRank > resultVecRank)
       return op->emitOpError(
           "requires memref vector element and vector result ranks to match.");
-    // TODO(b/146516564) Move this to isSuffix in Vector/Utils.h.
+    // TODO: Move this to isSuffix in Vector/Utils.h.
     unsigned rankOffset = resultVecRank - memrefVecEltRank;
     auto memrefVecEltShape = memrefVectorElementType.getShape();
     auto resultVecShape = vectorType.getShape();
@@ -1515,13 +1720,78 @@ static LogicalResult foldMemRefCast(Operation *op) {
   return success(folded);
 }
 
+template <typename TransferOp>
+static bool isInBounds(TransferOp op, int64_t resultIdx, int64_t indicesIdx) {
+  // TODO: support more aggressive createOrFold on:
+  // `op.indices()[indicesIdx] + vectorType < dim(op.memref(), indicesIdx)`
+  if (op.getMemRefType().isDynamicDim(indicesIdx))
+    return false;
+  Value index = op.indices()[indicesIdx];
+  auto cstOp = index.getDefiningOp<ConstantIndexOp>();
+  if (!cstOp)
+    return false;
+
+  int64_t memrefSize = op.getMemRefType().getDimSize(indicesIdx);
+  int64_t vectorSize = op.getVectorType().getDimSize(resultIdx);
+  return cstOp.getValue() + vectorSize <= memrefSize;
+}
+
+template <typename TransferOp>
+static LogicalResult foldTransferMaskAttribute(TransferOp op) {
+  AffineMap permutationMap = op.permutation_map();
+  if (!permutationMap.isMinorIdentity())
+    return failure();
+  bool changed = false;
+  SmallVector<bool, 4> isMasked;
+  isMasked.reserve(op.getTransferRank());
+  // `permutationMap` results and `op.indices` sizes may not match and may not
+  // be aligned. The first `indicesIdx` may just be indexed and not transferred
+  // from/into the vector.
+  // For example:
+  //  vector.transfer %0[%i, %j, %k, %c0] : memref<?x?x?x?xf32>, vector<2x4xf32>
+  // with `permutation_map = (d0, d1, d2, d3) -> (d2, d3)`.
+  // The `permutationMap` results and `op.indices` are however aligned when
+  // iterating in reverse until we exhaust `permutationMap` results.
+  // As a consequence we iterate with 2 running indices: `resultIdx` and
+  // `indicesIdx`, until `resultIdx` reaches 0.
+  for (int64_t resultIdx = permutationMap.getNumResults() - 1,
+               indicesIdx = op.indices().size() - 1;
+       resultIdx >= 0; --resultIdx, --indicesIdx) {
+    // Already marked unmasked, nothing to see here.
+    if (!op.isMaskedDim(resultIdx)) {
+      isMasked.push_back(false);
+      continue;
+    }
+    // Currently masked, check whether we can statically determine it is
+    // inBounds.
+    auto inBounds = isInBounds(op, resultIdx, indicesIdx);
+    isMasked.push_back(!inBounds);
+    // We commit the pattern if it is "more inbounds".
+    changed |= inBounds;
+  }
+  if (!changed)
+    return failure();
+  // OpBuilder is only used as a helper to build an I64ArrayAttr.
+  OpBuilder b(op.getContext());
+  std::reverse(isMasked.begin(), isMasked.end());
+  op.setAttr(TransferOp::getMaskedAttrName(), b.getBoolArrayAttr(isMasked));
+  return success();
+}
+
 OpFoldResult TransferReadOp::fold(ArrayRef<Attribute>) {
   /// transfer_read(memrefcast) -> transfer_read
+  if (succeeded(foldTransferMaskAttribute(*this)))
+    return getResult();
   if (succeeded(foldMemRefCast(*this)))
     return getResult();
   return OpFoldResult();
 }
 
+Optional<SmallVector<int64_t, 4>> TransferReadOp::getShapeForUnroll() {
+  auto s = getVectorType().getShape();
+  return SmallVector<int64_t, 4>{s.begin(), s.end()};
+}
+
 //===----------------------------------------------------------------------===//
 // TransferWriteOp
 //===----------------------------------------------------------------------===//
@@ -1609,9 +1879,16 @@ static LogicalResult verify(TransferWriteOp op) {
 
 LogicalResult TransferWriteOp::fold(ArrayRef<Attribute>,
                                     SmallVectorImpl<OpFoldResult> &) {
+  if (succeeded(foldTransferMaskAttribute(*this)))
+    return success();
   return foldMemRefCast(*this);
 }
 
+Optional<SmallVector<int64_t, 4>> TransferWriteOp::getShapeForUnroll() {
+  auto s = getVectorType().getShape();
+  return SmallVector<int64_t, 4>{s.begin(), s.end()};
+}
+
 //===----------------------------------------------------------------------===//
 // ShapeCastOp
 //===----------------------------------------------------------------------===//
@@ -1633,6 +1910,14 @@ static bool isValidShapeCast(ArrayRef<int64_t> a, ArrayRef<int64_t> b) {
     if (dimA != dimB)
       break;
     ++i;
+
+    // Handle the case when trailing dimensions are of size 1.
+    // Include them into the contiguous sequence.
+    auto isOne = [](int64_t v) { return v == 1; };
+    if (i < rankA && llvm::all_of(a.slice(i), isOne))
+      i = rankA;
+    if (j < rankB && llvm::all_of(b.slice(j), isOne))
+      j = rankB;
   }
 
   return i == rankA && j == rankB;
diff --git a/mlir/lib/Dialect/Vector/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
index a9207230278fa..2f77fd5ff60a4 100644
--- a/mlir/lib/Dialect/Vector/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
@@ -30,6 +30,7 @@
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Types.h"
+#include "mlir/Interfaces/VectorUnrollInterface.h"
 
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -100,7 +101,7 @@ static Type adjustType(VectorType tp, int64_t index) {
 }
 
 // Helper method to possibly drop a dimension in a load.
-// TODO(ajcbik): use a reshaping vector load (and share lowering code)
+// TODO
 static Value reshapeLoad(Location loc, Value val, VectorType type,
                          int64_t index, int64_t pos,
                          PatternRewriter &rewriter) {
@@ -128,7 +129,7 @@ static Value reshapeLoad(Location loc, Value val, VectorType type,
 }
 
 // Helper method to possibly drop a dimension in a store.
-// TODO(ajcbik): use a reshaping vector store (and share lowering code)
+// TODO
 static Value reshapeStore(Location loc, Value val, Value result,
                           VectorType type, int64_t index, int64_t pos,
                           PatternRewriter &rewriter) {
@@ -181,7 +182,7 @@ static void getMappedElements(const DenseMap<int64_t, int64_t> &indexMap,
 
 // Returns a tuple type with vector element types for each resulting slice
 // of 'vectorType' unrolled by 'sizes' and 'strides'.
-// TODO(andydavis) Move this to a utility function and share it with
+// TODO: Move this to a utility function and share it with
 // Extract/InsertSlicesOp verification.
 static TupleType generateExtractSlicesOpResultType(VectorType vectorType,
                                                    ArrayRef<int64_t> sizes,
@@ -275,7 +276,7 @@ static Value getOrCreateUnrolledVectorSlice(
   // Compute slice offsets.
   SmallVector<int64_t, 4> sliceOffsets(state.unrolledShape.size());
   getMappedElements(indexMap, offsets, sliceOffsets);
-  // TODO(b/144845578) Support non-1 strides.
+  // TODO: Support non-1 strides.
   SmallVector<int64_t, 4> sliceStrides(state.unrolledShape.size(), 1);
   // Compute linear index of 'sliceOffsets' w.r.t 'state.basis'.
   int64_t sliceLinearIndex =
@@ -346,7 +347,7 @@ struct VectorState {
 //                           insertslice
 //                                |
 
-// TODO(andydavis) Add the following canonicalization/simplification patterns:
+// TODO: Add the following canonicalization/simplification patterns:
 // *) Add pattern which matches InsertStridedSlice -> StridedSlice and forwards
 //    InsertStridedSlice operand to StridedSlice.
 // *) Add pattern which matches SourceOp -> StridedSlice -> UserOp which checks
@@ -356,8 +357,8 @@ struct VectorState {
 //    operation, and leave the duplicate StridedSlice ops with no users
 //    (removable with DCE).
 
-// TODO(andydavis) Generalize this to support structured ops beyond
-// vector ContractionOp, and merge it with 'unrollSingleResultOpMatchingType'
+// TODO: Generalize this to support structured ops beyond
+// vector ContractionOp, and merge it with 'unrollSingleResultVectorOp'
 static Value unrollSingleResultStructuredOp(Operation *op,
                                             ArrayRef<int64_t> iterationBounds,
                                             std::vector<VectorState> &vectors,
@@ -450,11 +451,7 @@ static Value unrollSingleResultStructuredOp(Operation *op,
 
 static void getVectorContractionOpUnrollState(
     vector::ContractionOp contractionOp, ArrayRef<int64_t> targetShape,
-    SmallVectorImpl<int64_t> &iterationBounds,
     std::vector<VectorState> &vectors, unsigned &resultIndex) {
-  // Get contraction op iteration bounds.
-  contractionOp.getIterationBounds(iterationBounds);
-  assert(iterationBounds.size() == targetShape.size());
   // Get map from iteration space index to lhs/rhs/result shape index.
   std::vector<DenseMap<int64_t, int64_t>> iterationIndexMapList;
   contractionOp.getIterationIndexMap(iterationIndexMapList);
@@ -476,17 +473,15 @@ static void getVectorContractionOpUnrollState(
     vectors.push_back({contractionOp.getRHSVectorMaskType(),
                        vectors[1].indexMap, accOperandIndex + 2, false});
   }
-  // Unroll 'op' 'iterationBounds' to 'targetShape'.
-  // TODO(andydavis) Use linalg style 'args_in'/'args_out' to partition
+  // TODO: Use linalg style 'args_in'/'args_out' to partition
   // 'vectors' instead of 'resultIndex'.
   resultIndex = accOperandIndex;
 }
 
-static void
-getVectorElementwiseOpUnrollState(Operation *op, ArrayRef<int64_t> targetShape,
-                                  SmallVectorImpl<int64_t> &iterationBounds,
-                                  std::vector<VectorState> &vectors,
-                                  unsigned &resultIndex) {
+static void getVectorElementwiseOpUnrollState(Operation *op,
+                                              ArrayRef<int64_t> targetShape,
+                                              std::vector<VectorState> &vectors,
+                                              unsigned &resultIndex) {
   // Verify that operation and operands all have the same vector shape.
   auto resultType = op->getResult(0).getType().dyn_cast_or_null<VectorType>();
   assert(resultType && "Expected op with vector result type");
@@ -494,8 +489,6 @@ getVectorElementwiseOpUnrollState(Operation *op, ArrayRef<int64_t> targetShape,
   // Verify that all operands have the same vector type as result.
   assert(llvm::all_of(op->getOperandTypes(),
                       [=](Type type) { return type == resultType; }));
-  // Populate 'iterationBounds' with 'resultShape' for elementwise operations.
-  iterationBounds.assign(resultShape.begin(), resultShape.end());
 
   // Create trivial elementwise identity index map based on 'resultShape'.
   DenseMap<int64_t, int64_t> indexMap;
@@ -513,28 +506,32 @@ getVectorElementwiseOpUnrollState(Operation *op, ArrayRef<int64_t> targetShape,
 }
 
 // Entry point for unrolling declarative pattern rewrites.
-SmallVector<Value, 1> mlir::vector::unrollSingleResultOpMatchingType(
-    OpBuilder &builder, Operation *op, ArrayRef<int64_t> targetShape) {
+SmallVector<Value, 1>
+mlir::vector::unrollSingleResultVectorOp(OpBuilder &builder, Operation *op,
+                                         ArrayRef<int64_t> targetShape) {
   assert(op->getNumResults() == 1 && "Expected single result operation");
 
   // Populate 'iterationBounds', 'vectors' and 'resultIndex' to unroll 'op'.
   SmallVector<int64_t, 6> iterationBounds;
+  auto unrollableVectorOp = cast<VectorUnrollOpInterface>(op);
+  auto maybeUnrollShape = unrollableVectorOp.getShapeForUnroll();
+  assert(maybeUnrollShape && "Trying to unroll an incorrect vector op");
+
   std::vector<VectorState> vectors;
   unsigned resultIndex;
 
   if (auto contractionOp = dyn_cast<vector::ContractionOp>(op)) {
     // Populate state for vector ContractionOp.
-    getVectorContractionOpUnrollState(contractionOp, targetShape,
-                                      iterationBounds, vectors, resultIndex);
+    getVectorContractionOpUnrollState(contractionOp, targetShape, vectors,
+                                      resultIndex);
   } else {
     // Populate state for vector elementwise op.
-    getVectorElementwiseOpUnrollState(op, targetShape, iterationBounds, vectors,
-                                      resultIndex);
+    getVectorElementwiseOpUnrollState(op, targetShape, vectors, resultIndex);
   }
 
   // Unroll 'op' with 'iterationBounds' to 'targetShape'.
   return SmallVector<Value, 1>{unrollSingleResultStructuredOp(
-      op, iterationBounds, vectors, resultIndex, targetShape, builder)};
+      op, *maybeUnrollShape, vectors, resultIndex, targetShape, builder)};
 }
 
 /// Generates slices of 'vectorType' according to 'sizes' and 'strides, and
@@ -621,7 +618,7 @@ struct SplitTransferReadOp : public OpRewritePattern<vector::TransferReadOp> {
 
   LogicalResult matchAndRewrite(vector::TransferReadOp xferReadOp,
                                 PatternRewriter &rewriter) const override {
-    // TODO(andydavis, ntv) Support splitting TransferReadOp with non-identity
+    // TODO: Support splitting TransferReadOp with non-identity
     // permutation maps. Repurpose code from MaterializeVectors transformation.
     if (!isIdentitySuffix(xferReadOp.permutation_map()))
       return failure();
@@ -680,7 +677,7 @@ struct SplitTransferWriteOp : public OpRewritePattern<vector::TransferWriteOp> {
 
   LogicalResult matchAndRewrite(vector::TransferWriteOp xferWriteOp,
                                 PatternRewriter &rewriter) const override {
-    // TODO(andydavis, ntv) Support splitting TransferWriteOp with non-identity
+    // TODO: Support splitting TransferWriteOp with non-identity
     // permutation maps. Repurpose code from MaterializeVectors transformation.
     if (!isIdentitySuffix(xferWriteOp.permutation_map()))
       return failure();
@@ -1265,7 +1262,7 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
 ///   %0 = vector.extract %lhs[0]
 ///   %1 = vector.broadcast %0
 ///   %2 = vector.extract %acc[0]
-///   %3 = vector.fma %1, %arg1, %2
+///   %3 = vector.fma %1, %rhs, %2
 ///   %4 = vector.insert %3, %z[0]
 ///   ..
 ///   %x = vector.insert %.., %..[N-1]
@@ -1278,29 +1275,49 @@ class OuterProductOpLowering : public OpRewritePattern<vector::OuterProductOp> {
                                 PatternRewriter &rewriter) const override {
     auto loc = op.getLoc();
 
-    VectorType rhsType = op.getOperandVectorTypeRHS();
+    VectorType lhsType = op.getOperandVectorTypeLHS();
+    VectorType rhsType = op.getOperandTypeRHS().dyn_cast<VectorType>();
     VectorType resType = op.getVectorType();
     Type eltType = resType.getElementType();
+    bool isInt = eltType.isa<IntegerType>();
     Value acc = (op.acc().empty()) ? nullptr : op.acc()[0];
 
+    if (!rhsType) {
+      // Special case: AXPY operation.
+      Value b = rewriter.create<vector::BroadcastOp>(loc, lhsType, op.rhs());
+      rewriter.replaceOp(op, genMult(loc, op.lhs(), b, acc, isInt, rewriter));
+      return success();
+    }
+
     Value result = rewriter.create<ConstantOp>(loc, resType,
                                                rewriter.getZeroAttr(resType));
     for (int64_t d = 0, e = resType.getDimSize(0); d < e; ++d) {
       auto pos = rewriter.getI64ArrayAttr(d);
       Value x = rewriter.create<vector::ExtractOp>(loc, eltType, op.lhs(), pos);
-      Value b = rewriter.create<vector::BroadcastOp>(loc, rhsType, x);
-      Value m;
-      if (acc) {
-        Value z = rewriter.create<vector::ExtractOp>(loc, rhsType, acc, pos);
-        m = rewriter.create<vector::FMAOp>(loc, b, op.rhs(), z);
-      } else {
-        m = rewriter.create<MulFOp>(loc, b, op.rhs());
-      }
+      Value a = rewriter.create<vector::BroadcastOp>(loc, rhsType, x);
+      Value r = nullptr;
+      if (acc)
+        r = rewriter.create<vector::ExtractOp>(loc, rhsType, acc, pos);
+      Value m = genMult(loc, a, op.rhs(), r, isInt, rewriter);
       result = rewriter.create<vector::InsertOp>(loc, resType, m, result, pos);
     }
     rewriter.replaceOp(op, result);
     return success();
   }
+
+private:
+  static Value genMult(Location loc, Value x, Value y, Value acc, bool isInt,
+                       PatternRewriter &rewriter) {
+    if (acc) {
+      if (isInt)
+        return rewriter.create<AddIOp>(loc, rewriter.create<MulIOp>(loc, x, y),
+                                       acc);
+      return rewriter.create<vector::FMAOp>(loc, x, y, acc);
+    }
+    if (isInt)
+      return rewriter.create<MulIOp>(loc, x, y);
+    return rewriter.create<MulFOp>(loc, x, y);
+  }
 };
 
 /// Progressive lowering of ConstantMaskOp.
@@ -1328,6 +1345,8 @@ class ConstantMaskOpLowering : public OpRewritePattern<vector::ConstantMaskOp> {
     int64_t trueDim = dimSizes[0].cast<IntegerAttr>().getInt();
 
     if (rank == 1) {
+      // Express constant 1-D case in explicit vector form:
+      //   [T,..,T,F,..,F].
       SmallVector<bool, 4> values(dstType.getDimSize(0));
       for (int64_t d = 0; d < trueDim; d++)
         values[d] = true;
@@ -1364,8 +1383,7 @@ class ConstantMaskOpLowering : public OpRewritePattern<vector::ConstantMaskOp> {
 ///   %1 = select %0, %l, %zeroes    |
 ///   %r = vector.insert %1, %pr [i] | d-times
 ///   %x = ....
-/// When rank == 1, the selection operator is not needed,
-/// and we can assign the true/false value right away.
+/// until a one-dimensional vector is reached.
 class CreateMaskOpLowering : public OpRewritePattern<vector::CreateMaskOp> {
 public:
   using OpRewritePattern<vector::CreateMaskOp>::OpRewritePattern;
@@ -1375,30 +1393,41 @@ class CreateMaskOpLowering : public OpRewritePattern<vector::CreateMaskOp> {
     auto loc = op.getLoc();
     auto dstType = op.getResult().getType().cast<VectorType>();
     auto eltType = dstType.getElementType();
+    int64_t dim = dstType.getDimSize(0);
     int64_t rank = dstType.getRank();
     Value idx = op.getOperand(0);
 
-    Value trueVal;
-    Value falseVal;
-    if (rank > 1) {
-      VectorType lowType =
-          VectorType::get(dstType.getShape().drop_front(), eltType);
-      trueVal = rewriter.create<vector::CreateMaskOp>(
-          loc, lowType, op.getOperands().drop_front());
-      falseVal = rewriter.create<ConstantOp>(loc, lowType,
-                                             rewriter.getZeroAttr(lowType));
+    if (rank == 1) {
+      // Express dynamic 1-D case in explicit vector form:
+      //   mask = [0,1,..,n-1] < [a,a,..,a]
+      SmallVector<int64_t, 4> values(dim);
+      for (int64_t d = 0; d < dim; d++)
+        values[d] = d;
+      Value indices =
+          rewriter.create<ConstantOp>(loc, rewriter.getI64VectorAttr(values));
+      Value bound =
+          rewriter.create<IndexCastOp>(loc, rewriter.getI64Type(), idx);
+      Value bounds = rewriter.create<SplatOp>(loc, indices.getType(), bound);
+      rewriter.replaceOpWithNewOp<CmpIOp>(op, CmpIPredicate::slt, indices,
+                                          bounds);
+      return success();
     }
 
+    VectorType lowType =
+        VectorType::get(dstType.getShape().drop_front(), eltType);
+    Value trueVal = rewriter.create<vector::CreateMaskOp>(
+        loc, lowType, op.getOperands().drop_front());
+    Value falseVal = rewriter.create<ConstantOp>(loc, lowType,
+                                                 rewriter.getZeroAttr(lowType));
     Value result = rewriter.create<ConstantOp>(loc, dstType,
                                                rewriter.getZeroAttr(dstType));
-    for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; d++) {
+    for (int64_t d = 0; d < dim; d++) {
       Value bnd = rewriter.create<ConstantOp>(loc, rewriter.getIndexAttr(d));
       Value val = rewriter.create<CmpIOp>(loc, CmpIPredicate::slt, bnd, idx);
-      if (rank > 1)
-        val = rewriter.create<SelectOp>(loc, val, trueVal, falseVal);
+      Value sel = rewriter.create<SelectOp>(loc, val, trueVal, falseVal);
       auto pos = rewriter.getI64ArrayAttr(d);
       result =
-          rewriter.create<vector::InsertOp>(loc, dstType, val, result, pos);
+          rewriter.create<vector::InsertOp>(loc, dstType, sel, result, pos);
     }
     rewriter.replaceOp(op, result);
     return success();
@@ -1544,19 +1573,21 @@ namespace mlir {
 /// the vector.contract op is a row-major matrix multiply.
 LogicalResult
 ContractionOpToMatmulOpLowering::match(vector::ContractionOp op) const {
-  // TODO(ajcbik): implement masks
+  // TODO: implement masks
   if (llvm::size(op.masks()) != 0)
     return failure();
 
+  if (vectorTransformsOptions.vectorContractLowering !=
+      vector::VectorContractLowering::Matmul)
+    return failure();
+
   auto iteratorTypes = op.iterator_types().getValue();
   if (!isParallelIterator(iteratorTypes[0]) ||
       !isParallelIterator(iteratorTypes[1]) ||
       !isReductionIterator(iteratorTypes[2]))
     return failure();
 
-  if (vectorTransformsOptions.vectorContractLowering !=
-          vector::VectorContractLowering::Matmul ||
-      !isRowMajorMatmul(op.indexing_maps()))
+  if (!isRowMajorMatmul(op.indexing_maps()))
     return failure();
 
   return success();
@@ -1566,9 +1597,9 @@ void ContractionOpToMatmulOpLowering::rewrite(vector::ContractionOp op,
                                               PatternRewriter &rewriter) const {
   VectorType lhsType = op.getLhsType();
   VectorType rhsType = op.getRhsType();
-  unsigned lhsRows = op.getLhsType().getShape()[0];
-  unsigned lhsColumns = op.getLhsType().getShape()[1];
-  unsigned rhsColumns = op.getRhsType().getShape()[1];
+  int64_t lhsRows = lhsType.getDimSize(0);
+  int64_t lhsColumns = lhsType.getDimSize(1);
+  int64_t rhsColumns = rhsType.getDimSize(1);
 
   Type flattenedLHSType =
       VectorType::get(lhsType.getNumElements(), lhsType.getElementType());
@@ -1607,8 +1638,8 @@ void ContractionOpToMatmulOpLowering::rewrite(vector::ContractionOp op,
 /// This only kicks in when VectorTransformsOptions is set to OuterProduct but
 /// otherwise supports any layout permutation of the matrix-multiply.
 LogicalResult
-ContractionOpToOuterProductOpLowering ::match(vector::ContractionOp op) const {
-  // TODO(ajcbik): implement masks
+ContractionOpToOuterProductOpLowering::match(vector::ContractionOp op) const {
+  // TODO: implement masks
   if (llvm::size(op.masks()) != 0)
     return failure();
 
@@ -1616,94 +1647,136 @@ ContractionOpToOuterProductOpLowering ::match(vector::ContractionOp op) const {
       vector::VectorContractLowering::OuterProduct)
     return failure();
 
-  // Transpose arguments to make them ready for lowering to OuterProduct. The
-  // constraint to match is that we must load full rows at a time with
-  // vector::ExtractOp.
+  // Determine if the parallel/reduction structure matches something
+  // that can be expressed a reduction_size unrolled sequence.
   using MapList = ArrayRef<ArrayRef<AffineExpr>>;
   auto infer = [](MapList m) { return AffineMap::inferFromExprList(m); };
   AffineExpr m, n, k;
   bindDims(op.getContext(), m, n, k);
   auto iteratorTypes = op.iterator_types().getValue();
-  if (!isParallelIterator(iteratorTypes[0]) ||
-      !isParallelIterator(iteratorTypes[1]) ||
-      !isReductionIterator(iteratorTypes[2]))
-    return failure();
   SmallVector<AffineMap, 4> maps = op.getIndexingMaps();
-  // When lowering to outerproduct we can support all permutations.
-  if (maps != infer({{m, k}, {k, n}, {m, n}}) &&
-      maps != infer({{m, k}, {n, k}, {m, n}}) &&
-      maps != infer({{k, m}, {k, n}, {m, n}}) &&
-      maps != infer({{k, m}, {n, k}, {m, n}}) &&
-      maps != infer({{m, k}, {k, n}, {n, m}}) &&
-      maps != infer({{m, k}, {n, k}, {n, m}}) &&
-      maps != infer({{k, m}, {k, n}, {n, m}}) &&
-      maps != infer({{k, m}, {n, k}, {n, m}}))
-    return failure();
-  return success();
+  if (isParallelIterator(iteratorTypes[0]) &&
+      isParallelIterator(iteratorTypes[1]) &&
+      isReductionIterator(iteratorTypes[2])) {
+    //
+    // Two outer parallel, one inner reduction (matmat flavor).
+    // When lowering to outerproduct we can support all permutations.
+    //
+    if (maps != infer({{m, k}, {k, n}, {m, n}}) &&
+        maps != infer({{m, k}, {n, k}, {m, n}}) &&
+        maps != infer({{k, m}, {k, n}, {m, n}}) &&
+        maps != infer({{k, m}, {n, k}, {m, n}}) &&
+        maps != infer({{m, k}, {k, n}, {n, m}}) &&
+        maps != infer({{m, k}, {n, k}, {n, m}}) &&
+        maps != infer({{k, m}, {k, n}, {n, m}}) &&
+        maps != infer({{k, m}, {n, k}, {n, m}}))
+      return failure();
+    return success();
+  } else if (isParallelIterator(iteratorTypes[0]) &&
+             isReductionIterator(iteratorTypes[1])) {
+    //
+    // One outer parallel, one inner reduction (matvec flavor)
+    // See if a series of AXPY operations chained through FMA operations
+    // could replace the default DOT implementation.
+    //
+    if (maps != infer({{m, n}, {n}, {m}}) && // mat-vec
+        maps != infer({{n, m}, {n}, {m}}) && // mat-trans-vec
+        maps != infer({{n}, {m, n}, {m}}) && // vec-mat
+        maps != infer({{n}, {n, m}, {m}}))   // vec-mat-trans
+      return failure();
+    return success();
+  }
+  return failure();
 }
 
 void ContractionOpToOuterProductOpLowering::rewrite(
     vector::ContractionOp op, PatternRewriter &rewriter) const {
   Location loc = op.getLoc();
-  unsigned reductionSize = 0;
+  int64_t reductionSize = 0;
   VectorType lhsType = op.getLhsType();
   Value lhs = op.lhs(), rhs = op.rhs(), res = op.acc();
 
-  // Transpose arguments to make them ready for lowering to OuterProduct. The
-  // constraint to match is that we must load full rows at a time with
-  // vector::ExtractOp.
+  // Set up the parallel/reduction structure in right form.
   using MapList = ArrayRef<ArrayRef<AffineExpr>>;
   auto infer = [](MapList m) { return AffineMap::inferFromExprList(m); };
   AffineExpr m, n, k;
   bindDims(rewriter.getContext(), m, n, k);
   SmallVector<int64_t, 2> perm{1, 0};
+  auto iteratorTypes = op.iterator_types().getValue();
   SmallVector<AffineMap, 4> maps = op.getIndexingMaps();
-  // First batch of cases, no need to output permute.
-  if (maps == infer({{m, k}, {k, n}, {m, n}})) {
-    // This is the classical row-major matmul. Just permute the lhs.
-    reductionSize = lhsType.getShape()[1];
-    lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
-  } else if (maps == infer({{m, k}, {n, k}, {m, n}})) {
-    // TODO: may be better to fail and use some vector<k> -> scalar reduction.
-    reductionSize = lhsType.getShape()[1];
-    lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
-    rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
-  } else if (maps == infer({{k, m}, {k, n}, {m, n}})) {
-    // No need to permute anything.
-    reductionSize = lhsType.getShape()[0];
-  } else if (maps == infer({{k, m}, {n, k}, {m, n}})) {
-    // Just permute the rhs.
-    reductionSize = lhsType.getShape()[0];
-    rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
-  }
-  // Second batch of cases, reshuffle to avoid output permute.
-  else if (maps == infer({{m, k}, {k, n}, {n, m}})) {
-    // This is the classical row-major matmul. Just permute the lhs.
-    reductionSize = lhsType.getShape()[1];
-    Value tmp = rhs;
-    rhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
-    lhs = tmp;
-  } else if (maps == infer({{m, k}, {n, k}, {n, m}})) {
-    // TODO: may be better to fail and use some vector<k> -> scalar reduction.
-    reductionSize = lhsType.getShape()[1];
-    Value tmp = rhs;
-    rhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
-    lhs = rewriter.create<vector::TransposeOp>(loc, tmp, perm);
-  } else if (maps == infer({{k, m}, {k, n}, {n, m}})) {
-    // No need to permute anything, but still swap lhs and rhs.
-    reductionSize = lhsType.getShape()[0];
-    std::swap(lhs, rhs);
-  } else if (maps == infer({{k, m}, {n, k}, {n, m}})) {
-    // Just permute the rhs.
-    reductionSize = lhsType.getShape()[0];
-    Value tmp = lhs;
-    lhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
-    rhs = tmp;
+  if (isParallelIterator(iteratorTypes[0]) &&
+      isParallelIterator(iteratorTypes[1]) &&
+      isReductionIterator(iteratorTypes[2])) {
+    //
+    // Two outer parallel, one inner reduction (matmat flavor).
+    //
+    if (maps == infer({{m, k}, {k, n}, {m, n}})) {
+      // This is the classical row-major matmul. Just permute the lhs.
+      reductionSize = lhsType.getDimSize(1);
+      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+    } else if (maps == infer({{m, k}, {n, k}, {m, n}})) {
+      // TODO: may be better to fail and use some vector<k> -> scalar reduction.
+      reductionSize = lhsType.getDimSize(1);
+      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+    } else if (maps == infer({{k, m}, {k, n}, {m, n}})) {
+      // No need to permute anything.
+      reductionSize = lhsType.getDimSize(0);
+    } else if (maps == infer({{k, m}, {n, k}, {m, n}})) {
+      // Just permute the rhs.
+      reductionSize = lhsType.getDimSize(0);
+      rhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+    } else if (maps == infer({{m, k}, {k, n}, {n, m}})) {
+      // This is the classical row-major matmul. Just permute the lhs.
+      reductionSize = lhsType.getDimSize(1);
+      Value tmp = rhs;
+      rhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = tmp;
+    } else if (maps == infer({{m, k}, {n, k}, {n, m}})) {
+      // TODO: may be better to fail and use some vector<k> -> scalar reduction.
+      reductionSize = lhsType.getDimSize(1);
+      Value tmp = rhs;
+      rhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+      lhs = rewriter.create<vector::TransposeOp>(loc, tmp, perm);
+    } else if (maps == infer({{k, m}, {k, n}, {n, m}})) {
+      // No need to permute anything, but still swap lhs and rhs.
+      reductionSize = lhsType.getDimSize(0);
+      std::swap(lhs, rhs);
+    } else if (maps == infer({{k, m}, {n, k}, {n, m}})) {
+      // Just permute the rhs.
+      reductionSize = lhsType.getDimSize(0);
+      Value tmp = lhs;
+      lhs = rewriter.create<vector::TransposeOp>(loc, rhs, perm);
+      rhs = tmp;
+    }
+  } else {
+    //
+    // One outer parallel, one inner reduction (matvec flavor)
+    //
+    assert(isParallelIterator(iteratorTypes[0]) &&
+           isReductionIterator(iteratorTypes[1]));
+    if (maps == infer({{m, n}, {n}, {m}})) {
+      // Case mat-vec: transpose.
+      reductionSize = lhsType.getDimSize(1);
+      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+    } else if (maps == infer({{n, m}, {n}, {m}})) {
+      // Case mat-trans-vec: ready to go.
+      reductionSize = lhsType.getDimSize(0);
+    } else if (maps == infer({{n}, {m, n}, {m}})) {
+      // Case vec-mat: swap and transpose.
+      reductionSize = lhsType.getDimSize(0);
+      std::swap(lhs, rhs);
+      lhs = rewriter.create<vector::TransposeOp>(loc, lhs, perm);
+    } else if (maps == infer({{n}, {n, m}, {m}})) {
+      // Case vec-mat-trans: swap and ready to go.
+      reductionSize = lhsType.getDimSize(0);
+      std::swap(lhs, rhs);
+    }
   }
   assert(reductionSize > 0);
 
-  // ExtractOp does not allow dynamic indexing, we must unroll explicitly.
-  for (unsigned k = 0; k < reductionSize; ++k) {
+  // Unroll outer-products along reduction.
+  for (int64_t k = 0; k < reductionSize; ++k) {
     Value a = rewriter.create<vector::ExtractOp>(op.getLoc(), lhs, k);
     Value b = rewriter.create<vector::ExtractOp>(op.getLoc(), rhs, k);
     res = rewriter.create<vector::OuterProductOp>(op.getLoc(), a, b, res);
@@ -1720,25 +1793,28 @@ void ContractionOpToOuterProductOpLowering::rewrite(
 ///   ..
 ///   %x = combine %a %b ..
 /// until a pure contraction is reached (no free/batch dimensions),
-/// which is replaced by a fma/reduction op.
+/// which is replaced by a dot-product.
 ///
-/// TODO(ajcbik): break down into transpose/reshape/cast ops
-///               when they become available to avoid code dup
-/// TODO(ajcbik): investigate lowering order impact on performance
+/// This only kicks in when either VectorTransformsOptions is set
+/// to DOT or when other contraction patterns fail.
+//
+// TODO: break down into transpose/reshape/cast ops
+//               when they become available to avoid code dup
+// TODO: investigate lowering order impact on performance
 LogicalResult
 ContractionOpLowering::matchAndRewrite(vector::ContractionOp op,
                                        PatternRewriter &rewriter) const {
 
-  // TODO(ajcbik): implement masks.
+  // TODO: implement masks.
   if (llvm::size(op.masks()) != 0)
     return failure();
-  // TODO(thomasraoux): support mixed mode contract lowering.
+  // TODO: support mixed mode contract lowering.
   if (op.getLhsType().getElementType() !=
           getElementTypeOrSelf(op.getAccType()) ||
       op.getRhsType().getElementType() != getElementTypeOrSelf(op.getAccType()))
     return failure();
 
-  // TODO(ntv, ajcbik): implement benefits, cost models.
+  // TODO: implement benefits, cost models.
   MLIRContext *ctx = op.getContext();
   ContractionOpToMatmulOpLowering pat1(vectorTransformsOptions, ctx);
   if (succeeded(pat1.match(op)))
@@ -1796,7 +1872,7 @@ ContractionOpLowering::matchAndRewrite(vector::ContractionOp op,
 }
 
 // Lower one parallel dimension.
-// TODO(ajcbik): consider reusing existing contract unrolling
+// TODO: consider reusing existing contract unrolling
 Value ContractionOpLowering::lowerParallel(vector::ContractionOp op,
                                            int64_t lhsIndex, int64_t rhsIndex,
                                            PatternRewriter &rewriter) const {
@@ -1870,11 +1946,9 @@ Value ContractionOpLowering::lowerReduction(vector::ContractionOp op,
   // Base case.
   if (lhsType.getRank() == 1) {
     assert(rhsType.getRank() == 1 && "corrupt contraction");
-    Value zero = rewriter.create<ConstantOp>(loc, lhsType,
-                                             rewriter.getZeroAttr(lhsType));
-    Value fma = rewriter.create<vector::FMAOp>(loc, op.lhs(), op.rhs(), zero);
+    Value m = rewriter.create<MulFOp>(loc, op.lhs(), op.rhs());
     StringAttr kind = rewriter.getStringAttr("add");
-    return rewriter.create<vector::ReductionOp>(loc, resType, kind, fma,
+    return rewriter.create<vector::ReductionOp>(loc, resType, kind, m,
                                                 op.acc());
   }
   // Construct new iterator types and affine map array attribute.
@@ -1901,8 +1975,8 @@ Value ContractionOpLowering::lowerReduction(vector::ContractionOp op,
 
 } // namespace mlir
 
-// TODO(andydavis) Add pattern to rewrite ExtractSlices(ConstantMaskOp).
-// TODO(andydavis) Add this as DRR pattern.
+// TODO: Add pattern to rewrite ExtractSlices(ConstantMaskOp).
+// TODO: Add this as DRR pattern.
 void mlir::vector::populateVectorToVectorTransformationPatterns(
     OwningRewritePatternList &patterns, MLIRContext *context) {
   // clang-format off
diff --git a/mlir/lib/Dialect/Vector/VectorUtils.cpp b/mlir/lib/Dialect/Vector/VectorUtils.cpp
index ccd243e8a7de9..d5beaefc5eac9 100644
--- a/mlir/lib/Dialect/Vector/VectorUtils.cpp
+++ b/mlir/lib/Dialect/Vector/VectorUtils.cpp
@@ -208,7 +208,7 @@ static AffineMap makePermutationMap(
 
 /// Implementation detail that walks up the parents and records the ones with
 /// the specified type.
-/// TODO(ntv): could also be implemented as a collect parents followed by a
+/// TODO: could also be implemented as a collect parents followed by a
 /// filter and made available outside this file.
 template <typename T>
 static SetVector<Operation *> getParentsOfType(Operation *op) {
@@ -252,7 +252,7 @@ bool matcher::operatesOnSuperVectorsOf(Operation &op,
   // The ops that *may* lower a super-vector only do so if the super-vector to
   // sub-vector ratio exists. The ops that *must* lower a super-vector are
   // explicitly checked for this property.
-  /// TODO(ntv): there should be a single function for all ops to do this so we
+  /// TODO: there should be a single function for all ops to do this so we
   /// do not have to special case. Maybe a trait, or just a method, unclear atm.
   bool mustDivide = false;
   (void)mustDivide;
diff --git a/mlir/lib/ExecutionEngine/ExecutionEngine.cpp b/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
index c64c7d208dec6..130ca3c02ad4c 100644
--- a/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
+++ b/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
@@ -234,7 +234,7 @@ Expected<std::unique_ptr<ExecutionEngine>> ExecutionEngine::create(
 
   // Clone module in a new LLVMContext since translateModuleToLLVMIR buries
   // ownership too deeply.
-  // TODO(zinenko): Reevaluate model of ownership of LLVMContext in LLVMDialect.
+  // TODO: Reevaluate model of ownership of LLVMContext in LLVMDialect.
   std::unique_ptr<Module> deserModule =
       LLVM::cloneModuleIntoNewContext(ctx.get(), llvmModule.get());
   auto dataLayout = deserModule->getDataLayout();
diff --git a/mlir/lib/IR/AffineExpr.cpp b/mlir/lib/IR/AffineExpr.cpp
index 7d9145aacccea..e0c4b6b208f7a 100644
--- a/mlir/lib/IR/AffineExpr.cpp
+++ b/mlir/lib/IR/AffineExpr.cpp
@@ -703,7 +703,7 @@ void SimpleAffineExprFlattener::visitModExpr(AffineBinaryOpExpr expr) {
   auto rhsConst = operandExprStack.back()[getConstantIndex()];
   operandExprStack.pop_back();
   auto &lhs = operandExprStack.back();
-  // TODO(bondhugula): handle modulo by zero case when this issue is fixed
+  // TODO: handle modulo by zero case when this issue is fixed
   // at the other places in the IR.
   assert(rhsConst > 0 && "RHS constant has to be positive");
 
@@ -791,7 +791,7 @@ void SimpleAffineExprFlattener::visitDivExpr(AffineBinaryOpExpr expr,
 
   // This is a pure affine expr; the RHS is a positive constant.
   int64_t rhsConst = operandExprStack.back()[getConstantIndex()];
-  // TODO(bondhugula): handle division by zero at the same time the issue is
+  // TODO: handle division by zero at the same time the issue is
   // fixed at other places.
   assert(rhsConst > 0 && "RHS constant has to be positive");
   operandExprStack.pop_back();
@@ -870,7 +870,7 @@ int SimpleAffineExprFlattener::findLocalId(AffineExpr localExpr) {
 /// Simplify the affine expression by flattening it and reconstructing it.
 AffineExpr mlir::simplifyAffineExpr(AffineExpr expr, unsigned numDims,
                                     unsigned numSymbols) {
-  // TODO(bondhugula): only pure affine for now. The simplification here can
+  // TODO: only pure affine for now. The simplification here can
   // be extended to semi-affine maps in the future.
   if (!expr.isPureAffine())
     return expr;
diff --git a/mlir/lib/IR/AffineMap.cpp b/mlir/lib/IR/AffineMap.cpp
index 07a01f0bf75f2..42dbef273b6d1 100644
--- a/mlir/lib/IR/AffineMap.cpp
+++ b/mlir/lib/IR/AffineMap.cpp
@@ -104,11 +104,9 @@ AffineMap AffineMap::getMinorIdentityMap(unsigned dims, unsigned results,
   return AffineMap::get(dims, 0, id.getResults().take_back(results), context);
 }
 
-bool AffineMap::isMinorIdentity(AffineMap map) {
-  if (!map)
-    return false;
-  return map == getMinorIdentityMap(map.getNumDims(), map.getNumResults(),
-                                    map.getContext());
+bool AffineMap::isMinorIdentity() const {
+  return *this ==
+         getMinorIdentityMap(getNumDims(), getNumResults(), getContext());
 }
 
 /// Returns an AffineMap representing a permutation.
@@ -330,6 +328,21 @@ AffineMap AffineMap::compose(AffineMap map) {
   return AffineMap::get(numDims, numSymbols, exprs, map.getContext());
 }
 
+SmallVector<int64_t, 4> AffineMap::compose(ArrayRef<int64_t> values) {
+  assert(getNumSymbols() == 0 && "Expected symbol-less map");
+  SmallVector<AffineExpr, 4> exprs;
+  exprs.reserve(values.size());
+  MLIRContext *ctx = getContext();
+  for (auto v : values)
+    exprs.push_back(getAffineConstantExpr(v, ctx));
+  auto resMap = compose(AffineMap::get(0, 0, exprs, ctx));
+  SmallVector<int64_t, 4> res;
+  res.reserve(resMap.getNumResults());
+  for (auto e : resMap.getResults())
+    res.push_back(e.cast<AffineConstantExpr>().getValue());
+  return res;
+}
+
 bool AffineMap::isProjectedPermutation() {
   if (getNumSymbols() > 0)
     return false;
@@ -355,12 +368,28 @@ bool AffineMap::isPermutation() {
 AffineMap AffineMap::getSubMap(ArrayRef<unsigned> resultPos) {
   SmallVector<AffineExpr, 4> exprs;
   exprs.reserve(resultPos.size());
-  for (auto idx : resultPos) {
+  for (auto idx : resultPos)
     exprs.push_back(getResult(idx));
-  }
   return AffineMap::get(getNumDims(), getNumSymbols(), exprs, getContext());
 }
 
+AffineMap AffineMap::getMajorSubMap(unsigned numResults) {
+  if (numResults == 0)
+    return AffineMap();
+  if (numResults > getNumResults())
+    return *this;
+  return getSubMap(llvm::to_vector<4>(llvm::seq<unsigned>(0, numResults)));
+}
+
+AffineMap AffineMap::getMinorSubMap(unsigned numResults) {
+  if (numResults == 0)
+    return AffineMap();
+  if (numResults > getNumResults())
+    return *this;
+  return getSubMap(llvm::to_vector<4>(
+      llvm::seq<unsigned>(getNumResults() - numResults, getNumResults())));
+}
+
 AffineMap mlir::simplifyAffineMap(AffineMap map) {
   SmallVector<AffineExpr, 8> exprs;
   for (auto e : map.getResults()) {
@@ -444,7 +473,7 @@ bool MutableAffineMap::isMultipleOf(unsigned idx, int64_t factor) const {
   if (results[idx].isMultipleOf(factor))
     return true;
 
-  // TODO(bondhugula): use simplifyAffineExpr and FlatAffineConstraints to
+  // TODO: use simplifyAffineExpr and FlatAffineConstraints to
   // complete this (for a more powerful analysis).
   return false;
 }
@@ -453,7 +482,7 @@ bool MutableAffineMap::isMultipleOf(unsigned idx, int64_t factor) const {
 // be pure for the simplification implemented.
 void MutableAffineMap::simplify() {
   // Simplify each of the results if possible.
-  // TODO(ntv): functional-style map
+  // TODO: functional-style map
   for (unsigned i = 0, e = getNumResults(); i < e; i++) {
     results[i] = simplifyAffineExpr(getResult(i), numDims, numSymbols);
   }
diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp
index 0636ab59ea504..09135021a732d 100644
--- a/mlir/lib/IR/AsmPrinter.cpp
+++ b/mlir/lib/IR/AsmPrinter.cpp
@@ -284,7 +284,7 @@ class AliasState {
 // Utility to generate a function to register a symbol alias.
 static bool canRegisterAlias(StringRef name, llvm::StringSet<> &usedAliases) {
   assert(!name.empty() && "expected alias name to be non-empty");
-  // TODO(riverriddle) Assert that the provided alias name can be lexed as
+  // TODO: Assert that the provided alias name can be lexed as
   // an identifier.
 
   // Check that the alias doesn't contain a '.' character and the name is not
@@ -431,7 +431,7 @@ void AliasState::recordAttributeReference(Attribute attr) {
 /// Record a reference to the given type.
 void AliasState::recordTypeReference(Type ty) { usedTypes.insert(ty); }
 
-// TODO Support visiting other types/operations when implemented.
+// TODO: Support visiting other types/operations when implemented.
 void AliasState::visitType(Type type) {
   recordTypeReference(type);
 
@@ -1432,10 +1432,12 @@ void ModulePrinter::printAttribute(Attribute attr,
       break;
     }
     os << "sparse<";
-    printDenseIntOrFPElementsAttr(elementsAttr.getIndices(),
-                                  /*allowHex=*/false);
-    os << ", ";
-    printDenseElementsAttr(elementsAttr.getValues(), /*allowHex=*/true);
+    DenseIntElementsAttr indices = elementsAttr.getIndices();
+    if (indices.getNumElements() != 0) {
+      printDenseIntOrFPElementsAttr(indices, /*allowHex=*/false);
+      os << ", ";
+      printDenseElementsAttr(elementsAttr.getValues(), /*allowHex=*/true);
+    }
     os << '>';
     break;
   }
@@ -1476,20 +1478,15 @@ printDenseElementsAttrImpl(bool isSplat, ShapedType type, raw_ostream &os,
 
   // Special case for degenerate tensors.
   auto numElements = type.getNumElements();
-  int64_t rank = type.getRank();
-  if (numElements == 0) {
-    for (int i = 0; i < rank; ++i)
-      os << '[';
-    for (int i = 0; i < rank; ++i)
-      os << ']';
+  if (numElements == 0)
     return;
-  }
 
   // We use a mixed-radix counter to iterate through the shape. When we bump a
   // non-least-significant digit, we emit a close bracket. When we next emit an
   // element we re-open all closed brackets.
 
   // The mixed-radix counter, with radices in 'shape'.
+  int64_t rank = type.getRank();
   SmallVector<unsigned, 4> counter(rank, 0);
   // The number of brackets that have been opened and not closed.
   unsigned openBrackets = 0;
@@ -2442,7 +2439,7 @@ void Value::dump() {
 }
 
 void Value::printAsOperand(raw_ostream &os, AsmState &state) {
-  // TODO(riverriddle) This doesn't necessarily capture all potential cases.
+  // TODO: This doesn't necessarily capture all potential cases.
   // Currently, region arguments can be shadowed when printing the main
   // operation. If the IR hasn't been printed, this will produce the old SSA
   // name and not the shadowed name.
diff --git a/mlir/lib/IR/AttributeDetail.h b/mlir/lib/IR/AttributeDetail.h
index e6c9ae5ed59c5..ad0b302bc78dd 100644
--- a/mlir/lib/IR/AttributeDetail.h
+++ b/mlir/lib/IR/AttributeDetail.h
@@ -658,7 +658,7 @@ struct OpaqueElementsAttributeStorage : public AttributeStorage {
   /// Construct a new storage instance.
   static OpaqueElementsAttributeStorage *
   construct(AttributeStorageAllocator &allocator, KeyTy key) {
-    // TODO(b/131468830): Provide a way to avoid copying content of large opaque
+    // TODO: Provide a way to avoid copying content of large opaque
     // tensors This will likely require a new reference attribute kind.
     return new (allocator.allocate<OpaqueElementsAttributeStorage>())
         OpaqueElementsAttributeStorage(std::get<0>(key), std::get<1>(key),
diff --git a/mlir/lib/IR/Attributes.cpp b/mlir/lib/IR/Attributes.cpp
index bc929bcb0c749..8202465af0c76 100644
--- a/mlir/lib/IR/Attributes.cpp
+++ b/mlir/lib/IR/Attributes.cpp
@@ -47,7 +47,9 @@ Type Attribute::getType() const { return impl->getType(); }
 MLIRContext *Attribute::getContext() const { return getType().getContext(); }
 
 /// Get the dialect this attribute is registered to.
-Dialect &Attribute::getDialect() const { return impl->getDialect(); }
+Dialect &Attribute::getDialect() const {
+  return impl->getAbstractAttribute().getDialect();
+}
 
 //===----------------------------------------------------------------------===//
 // AffineMapAttr
@@ -337,7 +339,7 @@ uint64_t IntegerAttr::getUInt() const {
 }
 
 static LogicalResult verifyIntegerTypeInvariants(Location loc, Type type) {
-  if (type.isa<IntegerType>() || type.isa<IndexType>())
+  if (type.isa<IntegerType, IndexType>())
     return success();
   return emitError(loc, "expected integer or index type");
 }
@@ -1090,7 +1092,7 @@ DenseElementsAttr DenseIntOrFPElementsAttr::getRaw(ShapedType type,
 DenseElementsAttr DenseIntOrFPElementsAttr::getRaw(ShapedType type,
                                                    ArrayRef<char> data,
                                                    bool isSplat) {
-  assert((type.isa<RankedTensorType>() || type.isa<VectorType>()) &&
+  assert((type.isa<RankedTensorType, VectorType>()) &&
          "type must be ranked tensor or vector");
   assert(type.hasStaticShape() && "type must have static shape");
   return Base::get(type.getContext(), StandardAttributes::DenseIntOrFPElements,
@@ -1247,7 +1249,7 @@ SparseElementsAttr SparseElementsAttr::get(ShapedType type,
                                            DenseElementsAttr values) {
   assert(indices.getType().getElementType().isInteger(64) &&
          "expected sparse indices to be 64-bit integer values");
-  assert((type.isa<RankedTensorType>() || type.isa<VectorType>()) &&
+  assert((type.isa<RankedTensorType, VectorType>()) &&
          "type must be ranked tensor or vector");
   assert(type.hasStaticShape() && "type must have static shape");
   return Base::get(type.getContext(), StandardAttributes::SparseElements, type,
diff --git a/mlir/lib/IR/Function.cpp b/mlir/lib/IR/Function.cpp
index 305022dd84513..09c9441bb1101 100644
--- a/mlir/lib/IR/Function.cpp
+++ b/mlir/lib/IR/Function.cpp
@@ -42,18 +42,16 @@ FuncOp FuncOp::create(Location location, StringRef name, FunctionType type,
 }
 
 void FuncOp::build(OpBuilder &builder, OperationState &result, StringRef name,
-                   FunctionType type, ArrayRef<NamedAttribute> attrs) {
+                   FunctionType type, ArrayRef<NamedAttribute> attrs,
+                   ArrayRef<MutableDictionaryAttr> argAttrs) {
   result.addAttribute(SymbolTable::getSymbolAttrName(),
                       builder.getStringAttr(name));
   result.addAttribute(getTypeAttrName(), TypeAttr::get(type));
   result.attributes.append(attrs.begin(), attrs.end());
   result.addRegion();
-}
 
-void FuncOp::build(OpBuilder &builder, OperationState &result, StringRef name,
-                   FunctionType type, ArrayRef<NamedAttribute> attrs,
-                   ArrayRef<MutableDictionaryAttr> argAttrs) {
-  build(builder, result, name, type, attrs);
+  if (argAttrs.empty())
+    return;
   assert(type.getNumInputs() == argAttrs.size());
   SmallString<8> argAttrName;
   for (unsigned i = 0, e = type.getNumInputs(); i != e; ++i)
diff --git a/mlir/lib/IR/MLIRContext.cpp b/mlir/lib/IR/MLIRContext.cpp
index 2beb1a91dbaec..a4e833cbf77c5 100644
--- a/mlir/lib/IR/MLIRContext.cpp
+++ b/mlir/lib/IR/MLIRContext.cpp
@@ -270,10 +270,6 @@ class MLIRContextImpl {
   // Other
   //===--------------------------------------------------------------------===//
 
-  /// A general purpose mutex to lock access to parts of the context that do not
-  /// have a more specific mutex, e.g. registry operations.
-  llvm::sys::SmartRWMutex<true> contextMutex;
-
   /// This is a list of dialects that are created referring to this context.
   /// The MLIRContext owns the objects.
   std::vector<std::unique_ptr<Dialect>> dialects;
@@ -282,13 +278,12 @@ class MLIRContextImpl {
   /// operations.
   llvm::StringMap<AbstractOperation> registeredOperations;
 
-  /// This is a mapping from type id to Dialect for registered attributes and
-  /// types.
-  DenseMap<TypeID, Dialect *> registeredDialectSymbols;
-
   /// These are identifiers uniqued into this MLIRContext.
   llvm::StringSet<llvm::BumpPtrAllocator &> identifiers;
 
+  /// An allocator used for AbstractAttribute and AbstractType objects.
+  llvm::BumpPtrAllocator abstractDialectSymbolAllocator;
+
   //===--------------------------------------------------------------------===//
   // Affine uniquing
   //===--------------------------------------------------------------------===//
@@ -311,6 +306,8 @@ class MLIRContextImpl {
   //===--------------------------------------------------------------------===//
   // Type uniquing
   //===--------------------------------------------------------------------===//
+
+  DenseMap<TypeID, const AbstractType *> registeredTypes;
   StorageUniquer typeUniquer;
 
   /// Cached Type Instances.
@@ -322,6 +319,8 @@ class MLIRContextImpl {
   //===--------------------------------------------------------------------===//
   // Attribute uniquing
   //===--------------------------------------------------------------------===//
+
+  DenseMap<TypeID, const AbstractAttribute *> registeredAttributes;
   StorageUniquer attributeUniquer;
 
   /// Cached Attribute Instances.
@@ -332,6 +331,12 @@ class MLIRContextImpl {
 
 public:
   MLIRContextImpl() : identifiers(identifierAllocator) {}
+  ~MLIRContextImpl() {
+    for (auto typeMapping : registeredTypes)
+      typeMapping.second->~AbstractType();
+    for (auto attrMapping : registeredAttributes)
+      attrMapping.second->~AbstractAttribute();
+  }
 };
 } // end namespace mlir
 
@@ -422,8 +427,6 @@ DiagnosticEngine &MLIRContext::getDiagEngine() { return getImpl().diagEngine; }
 
 /// Return information about all registered IR dialects.
 std::vector<Dialect *> MLIRContext::getRegisteredDialects() {
-  // Lock access to the context registry.
-  ScopedReaderLock registryLock(impl->contextMutex, impl->threadingIsEnabled);
   std::vector<Dialect *> result;
   result.reserve(impl->dialects.size());
   for (auto &dialect : impl->dialects)
@@ -434,9 +437,6 @@ std::vector<Dialect *> MLIRContext::getRegisteredDialects() {
 /// Get a registered IR dialect with the given namespace. If none is found,
 /// then return nullptr.
 Dialect *MLIRContext::getRegisteredDialect(StringRef name) {
-  // Lock access to the context registry.
-  ScopedReaderLock registryLock(impl->contextMutex, impl->threadingIsEnabled);
-
   // Dialects are sorted by name, so we can use binary search for lookup.
   auto it = llvm::lower_bound(
       impl->dialects, name,
@@ -452,9 +452,6 @@ void Dialect::registerDialect(MLIRContext *context) {
   auto &impl = context->getImpl();
   std::unique_ptr<Dialect> dialect(this);
 
-  // Lock access to the context registry.
-  ScopedWriterLock registryLock(impl.contextMutex, impl.threadingIsEnabled);
-
   // Get the correct insertion position sorted by namespace.
   auto insertPt = llvm::lower_bound(
       impl.dialects, dialect, [](const auto &lhs, const auto &rhs) {
@@ -521,35 +518,26 @@ void MLIRContext::printStackTraceOnDiagnostic(bool enable) {
 /// efficient, typically you should ask the operations about their properties
 /// directly.
 std::vector<AbstractOperation *> MLIRContext::getRegisteredOperations() {
-  std::vector<std::pair<StringRef, AbstractOperation *>> opsToSort;
-
-  { // Lock access to the context registry.
-    ScopedReaderLock registryLock(impl->contextMutex, impl->threadingIsEnabled);
-
-    // We just have the operations in a non-deterministic hash table order. Dump
-    // into a temporary array, then sort it by operation name to get a stable
-    // ordering.
-    llvm::StringMap<AbstractOperation> &registeredOps =
-        impl->registeredOperations;
-
-    opsToSort.reserve(registeredOps.size());
-    for (auto &elt : registeredOps)
-      opsToSort.push_back({elt.first(), &elt.second});
-  }
-
-  llvm::array_pod_sort(opsToSort.begin(), opsToSort.end());
+  // We just have the operations in a non-deterministic hash table order. Dump
+  // into a temporary array, then sort it by operation name to get a stable
+  // ordering.
+  llvm::StringMap<AbstractOperation> &registeredOps =
+      impl->registeredOperations;
 
   std::vector<AbstractOperation *> result;
-  result.reserve(opsToSort.size());
-  for (auto &elt : opsToSort)
-    result.push_back(elt.second);
+  result.reserve(registeredOps.size());
+  for (auto &elt : registeredOps)
+    result.push_back(&elt.second);
+  llvm::array_pod_sort(
+      result.begin(), result.end(),
+      [](AbstractOperation *const *lhs, AbstractOperation *const *rhs) {
+        return (*lhs)->name.compare((*rhs)->name);
+      });
+
   return result;
 }
 
 bool MLIRContext::isOperationRegistered(StringRef name) {
-  // Lock access to the context registry.
-  ScopedReaderLock registryLock(impl->contextMutex, impl->threadingIsEnabled);
-
   return impl->registeredOperations.count(name);
 }
 
@@ -558,26 +546,41 @@ void Dialect::addOperation(AbstractOperation opInfo) {
          "op name doesn't start with dialect namespace");
   assert(&opInfo.dialect == this && "Dialect object mismatch");
   auto &impl = context->getImpl();
-
-  // Lock access to the context registry.
-  ScopedWriterLock registryLock(impl.contextMutex, impl.threadingIsEnabled);
-  if (!impl.registeredOperations.insert({opInfo.name, opInfo}).second) {
+  StringRef opName = opInfo.name;
+  if (!impl.registeredOperations.insert({opName, std::move(opInfo)}).second) {
     llvm::errs() << "error: operation named '" << opInfo.name
                  << "' is already registered.\n";
     abort();
   }
 }
 
-/// Register a dialect-specific symbol(e.g. type) with the current context.
-void Dialect::addSymbol(TypeID typeID) {
+void Dialect::addType(TypeID typeID, AbstractType &&typeInfo) {
   auto &impl = context->getImpl();
+  auto *newInfo =
+      new (impl.abstractDialectSymbolAllocator.Allocate<AbstractType>())
+          AbstractType(std::move(typeInfo));
+  if (!impl.registeredTypes.insert({typeID, newInfo}).second)
+    llvm::report_fatal_error("Dialect Type already registered.");
+}
 
-  // Lock access to the context registry.
-  ScopedWriterLock registryLock(impl.contextMutex, impl.threadingIsEnabled);
-  if (!impl.registeredDialectSymbols.insert({typeID, this}).second) {
-    llvm::errs() << "error: dialect symbol already registered.\n";
-    abort();
-  }
+void Dialect::addAttribute(TypeID typeID, AbstractAttribute &&attrInfo) {
+  auto &impl = context->getImpl();
+  auto *newInfo =
+      new (impl.abstractDialectSymbolAllocator.Allocate<AbstractAttribute>())
+          AbstractAttribute(std::move(attrInfo));
+  if (!impl.registeredAttributes.insert({typeID, newInfo}).second)
+    llvm::report_fatal_error("Dialect Attribute already registered.");
+}
+
+/// Get the dialect that registered the attribute with the provided typeid.
+const AbstractAttribute &AbstractAttribute::lookup(TypeID typeID,
+                                                   MLIRContext *context) {
+  auto &impl = context->getImpl();
+  auto it = impl.registeredAttributes.find(typeID);
+  if (it == impl.registeredAttributes.end())
+    llvm::report_fatal_error("Trying to create an Attribute that was not "
+                             "registered in this MLIRContext.");
+  return *it->second;
 }
 
 /// Look up the specified operation in the operation set and return a pointer
@@ -585,15 +588,22 @@ void Dialect::addSymbol(TypeID typeID) {
 const AbstractOperation *AbstractOperation::lookup(StringRef opName,
                                                    MLIRContext *context) {
   auto &impl = context->getImpl();
-
-  // Lock access to the context registry.
-  ScopedReaderLock registryLock(impl.contextMutex, impl.threadingIsEnabled);
   auto it = impl.registeredOperations.find(opName);
   if (it != impl.registeredOperations.end())
     return &it->second;
   return nullptr;
 }
 
+/// Get the dialect that registered the type with the provided typeid.
+const AbstractType &AbstractType::lookup(TypeID typeID, MLIRContext *context) {
+  auto &impl = context->getImpl();
+  auto it = impl.registeredTypes.find(typeID);
+  if (it == impl.registeredTypes.end())
+    llvm::report_fatal_error(
+        "Trying to create a Type that was not registered in this MLIRContext.");
+  return *it->second;
+}
+
 //===----------------------------------------------------------------------===//
 // Identifier uniquing
 //===----------------------------------------------------------------------===//
@@ -627,24 +637,10 @@ Identifier Identifier::get(StringRef str, MLIRContext *context) {
 // Type uniquing
 //===----------------------------------------------------------------------===//
 
-static Dialect &lookupDialectForSymbol(MLIRContext *ctx, TypeID typeID) {
-  auto &impl = ctx->getImpl();
-  auto it = impl.registeredDialectSymbols.find(typeID);
-  if (it == impl.registeredDialectSymbols.end())
-    llvm::report_fatal_error(
-        "Trying to create a type that was not registered in this MLIRContext.");
-  return *it->second;
-}
-
 /// Returns the storage uniquer used for constructing type storage instances.
 /// This should not be used directly.
 StorageUniquer &MLIRContext::getTypeUniquer() { return getImpl().typeUniquer; }
 
-/// Get the dialect that registered the type with the provided typeid.
-Dialect &TypeUniquer::lookupDialectForType(MLIRContext *ctx, TypeID typeID) {
-  return lookupDialectForSymbol(ctx, typeID);
-}
-
 FloatType FloatType::get(StandardTypes::Kind kind, MLIRContext *context) {
   assert(kindof(kind) && "Not a FP kind.");
   switch (kind) {
@@ -737,7 +733,7 @@ StorageUniquer &MLIRContext::getAttributeUniquer() {
 void AttributeUniquer::initializeAttributeStorage(AttributeStorage *storage,
                                                   MLIRContext *ctx,
                                                   TypeID attrID) {
-  storage->initializeDialect(lookupDialectForSymbol(ctx, attrID));
+  storage->initialize(AbstractAttribute::lookup(attrID, ctx));
 
   // If the attribute did not provide a type, then default to NoneType.
   if (!storage->getType())
diff --git a/mlir/lib/IR/Module.cpp b/mlir/lib/IR/Module.cpp
index 1711b05c8533d..1f1e3aa2d56ee 100644
--- a/mlir/lib/IR/Module.cpp
+++ b/mlir/lib/IR/Module.cpp
@@ -76,11 +76,6 @@ LogicalResult ModuleOp::verify() {
   if (!llvm::hasSingleElement(bodyRegion))
     return emitOpError("expected body region to have a single block");
 
-  // Check that the body has no block arguments.
-  auto *body = &bodyRegion.front();
-  if (body->getNumArguments() != 0)
-    return emitOpError("expected body to have no arguments");
-
   // Check that none of the attributes are non-dialect attributes, except for
   // the symbol related attributes.
   for (auto attr : getOperation()->getMutableAttrDict().getAttrs()) {
diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp
index eb66d225638b9..1e2a47639fdbf 100644
--- a/mlir/lib/IR/Operation.cpp
+++ b/mlir/lib/IR/Operation.cpp
@@ -280,7 +280,7 @@ InFlightDiagnostic Operation::emitError(const Twine &message) {
   if (getContext()->shouldPrintOpOnDiagnostic()) {
     // Print out the operation explicitly here so that we can print the generic
     // form.
-    // TODO(riverriddle) It would be nice if we could instead provide the
+    // TODO: It would be nice if we could instead provide the
     // specific printing flags when adding the operation as an argument to the
     // diagnostic.
     std::string printedOp;
@@ -1017,6 +1017,22 @@ LogicalResult OpTrait::impl::verifyResultSizeAttr(Operation *op,
   return verifyValueSizeAttr(op, attrName, /*isOperand=*/false);
 }
 
+LogicalResult OpTrait::impl::verifyNoRegionArguments(Operation *op) {
+  for (Region &region : op->getRegions()) {
+    if (region.empty())
+      continue;
+
+    if (region.front().getNumArguments() != 0) {
+      if (op->getNumRegions() > 1)
+        return op->emitOpError("region #")
+               << region.getRegionNumber() << " should have no arguments";
+      else
+        return op->emitOpError("region should have no arguments");
+    }
+  }
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // BinaryOp implementation
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/IR/StandardTypes.cpp b/mlir/lib/IR/StandardTypes.cpp
index 117cd6810968f..5a9d22148b76f 100644
--- a/mlir/lib/IR/StandardTypes.cpp
+++ b/mlir/lib/IR/StandardTypes.cpp
@@ -72,11 +72,11 @@ bool Type::isUnsignedInteger(unsigned width) {
 }
 
 bool Type::isSignlessIntOrIndex() {
-  return isa<IndexType>() || isSignlessInteger();
+  return isSignlessInteger() || isa<IndexType>();
 }
 
 bool Type::isSignlessIntOrIndexOrFloat() {
-  return isa<IndexType>() || isSignlessInteger() || isa<FloatType>();
+  return isSignlessInteger() || isa<IndexType, FloatType>();
 }
 
 bool Type::isSignlessIntOrFloat() {
@@ -85,7 +85,7 @@ bool Type::isSignlessIntOrFloat() {
 
 bool Type::isIntOrIndex() { return isa<IntegerType>() || isIndex(); }
 
-bool Type::isIntOrFloat() { return isa<IntegerType>() || isa<FloatType>(); }
+bool Type::isIntOrFloat() { return isa<IntegerType, FloatType>(); }
 
 bool Type::isIntOrIndexOrFloat() { return isIntOrFloat() || isIndex(); }
 
@@ -200,7 +200,7 @@ int64_t ShapedType::getNumElements() const {
 int64_t ShapedType::getRank() const { return getShape().size(); }
 
 bool ShapedType::hasRank() const {
-  return !isa<UnrankedMemRefType>() && !isa<UnrankedTensorType>();
+  return !isa<UnrankedMemRefType, UnrankedTensorType>();
 }
 
 int64_t ShapedType::getDimSize(unsigned idx) const {
@@ -233,7 +233,7 @@ int64_t ShapedType::getSizeInBits() const {
   // Tensors can have vectors and other tensors as elements, other shaped types
   // cannot.
   assert(isa<TensorType>() && "unsupported element type");
-  assert((elementType.isa<VectorType>() || elementType.isa<TensorType>()) &&
+  assert((elementType.isa<VectorType, TensorType>()) &&
          "unsupported tensor element type");
   return getNumElements() * elementType.cast<ShapedType>().getSizeInBits();
 }
@@ -398,8 +398,8 @@ MemRefType MemRefType::getImpl(ArrayRef<int64_t> shape, Type elementType,
   auto *context = elementType.getContext();
 
   // Check that memref is formed from allowed types.
-  if (!elementType.isIntOrFloat() && !elementType.isa<VectorType>() &&
-      !elementType.isa<ComplexType>())
+  if (!elementType.isIntOrFloat() &&
+      !elementType.isa<VectorType, ComplexType>())
     return emitOptionalError(location, "invalid memref element type"),
            MemRefType();
 
@@ -476,8 +476,8 @@ LogicalResult
 UnrankedMemRefType::verifyConstructionInvariants(Location loc, Type elementType,
                                                  unsigned memorySpace) {
   // Check that memref is formed from allowed types.
-  if (!elementType.isIntOrFloat() && !elementType.isa<VectorType>() &&
-      !elementType.isa<ComplexType>())
+  if (!elementType.isIntOrFloat() &&
+      !elementType.isa<VectorType, ComplexType>())
     return emitError(loc, "invalid memref element type");
   return success();
 }
@@ -550,7 +550,7 @@ LogicalResult mlir::getStridesAndOffset(MemRefType t,
   // For now strides are only computed on a single affine map with a single
   // result (i.e. the closed subset of linearization maps that are compatible
   // with striding semantics).
-  // TODO(ntv): support more forms on a per-need basis.
+  // TODO: support more forms on a per-need basis.
   if (affineMaps.size() > 1)
     return failure();
   if (affineMaps.size() == 1 && affineMaps[0].getNumResults() != 1)
@@ -597,8 +597,8 @@ LogicalResult mlir::getStridesAndOffset(MemRefType t,
 
   /// In practice, a strided memref must be internally non-aliasing. Test
   /// against 0 as a proxy.
-  /// TODO(ntv) static cases can have more advanced checks.
-  /// TODO(ntv) dynamic cases would require a way to compare symbolic
+  /// TODO: static cases can have more advanced checks.
+  /// TODO: dynamic cases would require a way to compare symbolic
   /// expressions and would probably need an affine set context propagated
   /// everywhere.
   if (llvm::any_of(strides, [](AffineExpr e) {
diff --git a/mlir/lib/IR/SymbolTable.cpp b/mlir/lib/IR/SymbolTable.cpp
index 16e548f4430cd..b064d83b5faad 100644
--- a/mlir/lib/IR/SymbolTable.cpp
+++ b/mlir/lib/IR/SymbolTable.cpp
@@ -397,7 +397,7 @@ static WalkResult walkSymbolRefs(
     for (Attribute attr : llvm::drop_begin(attrRange, index)) {
       /// Check for a nested container attribute, these will also need to be
       /// walked.
-      if (attr.isa<ArrayAttr>() || attr.isa<DictionaryAttr>()) {
+      if (attr.isa<ArrayAttr, DictionaryAttr>()) {
         attrWorklist.push_back(attr);
         curAccessChain.push_back(-1);
         return WalkResult::advance();
diff --git a/mlir/lib/IR/Types.cpp b/mlir/lib/IR/Types.cpp
index 6fcf0016b0c53..25902c2863bb4 100644
--- a/mlir/lib/IR/Types.cpp
+++ b/mlir/lib/IR/Types.cpp
@@ -21,8 +21,9 @@ using namespace mlir::detail;
 
 unsigned Type::getKind() const { return impl->getKind(); }
 
-/// Get the dialect this type is registered to.
-Dialect &Type::getDialect() const { return impl->getDialect(); }
+Dialect &Type::getDialect() const {
+  return impl->getAbstractType().getDialect();
+}
 
 MLIRContext *Type::getContext() const { return getDialect().getContext(); }
 
diff --git a/mlir/lib/IR/Visitors.cpp b/mlir/lib/IR/Visitors.cpp
index 40ed0d934838f..bbccdcbf75928 100644
--- a/mlir/lib/IR/Visitors.cpp
+++ b/mlir/lib/IR/Visitors.cpp
@@ -14,7 +14,7 @@ using namespace mlir;
 /// Walk all of the operations nested under and including the given operations.
 void detail::walkOperations(Operation *op,
                             function_ref<void(Operation *op)> callback) {
-  // TODO(b/140235992) This walk should be iterative over the operations.
+  // TODO: This walk should be iterative over the operations.
   for (auto &region : op->getRegions())
     for (auto &block : region)
       // Early increment here in the case where the operation is erased.
@@ -29,7 +29,7 @@ void detail::walkOperations(Operation *op,
 WalkResult
 detail::walkOperations(Operation *op,
                        function_ref<WalkResult(Operation *op)> callback) {
-  // TODO(b/140235992) This walk should be iterative over the operations.
+  // TODO: This walk should be iterative over the operations.
   for (auto &region : op->getRegions()) {
     for (auto &block : region) {
       // Early increment here in the case where the operation is erased.
diff --git a/mlir/lib/Interfaces/CMakeLists.txt b/mlir/lib/Interfaces/CMakeLists.txt
index d0eea1ccc8738..b8498e224f250 100644
--- a/mlir/lib/Interfaces/CMakeLists.txt
+++ b/mlir/lib/Interfaces/CMakeLists.txt
@@ -1,10 +1,12 @@
 set(LLVM_OPTIONAL_SOURCES
   CallInterfaces.cpp
   ControlFlowInterfaces.cpp
+  CopyOpInterface.cpp
   DerivedAttributeOpInterface.cpp
   InferTypeOpInterface.cpp
   LoopLikeInterface.cpp
   SideEffectInterfaces.cpp
+  VectorUnrollInterface.cpp
   ViewLikeInterface.cpp
   )
 
@@ -26,9 +28,11 @@ endfunction(add_mlir_interface_library)
 
 add_mlir_interface_library(CallInterfaces)
 add_mlir_interface_library(ControlFlowInterfaces)
+add_mlir_interface_library(CopyOpInterface)
 add_mlir_interface_library(DerivedAttributeOpInterface)
 add_mlir_interface_library(InferTypeOpInterface)
 add_mlir_interface_library(LoopLikeInterface)
 add_mlir_interface_library(SideEffectInterfaces)
+add_mlir_interface_library(VectorUnrollInterface)
 add_mlir_interface_library(ViewLikeInterface)
 
diff --git a/mlir/lib/Interfaces/CopyOpInterface.cpp b/mlir/lib/Interfaces/CopyOpInterface.cpp
new file mode 100644
index 0000000000000..8e6132ca14e91
--- /dev/null
+++ b/mlir/lib/Interfaces/CopyOpInterface.cpp
@@ -0,0 +1,18 @@
+//===- CopyOpInterface.cpp - Copy operations interface in MLIR ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Interfaces/CopyOpInterface.h"
+
+using namespace mlir;
+
+//===----------------------------------------------------------------------===//
+// CopyOp Interface
+//===----------------------------------------------------------------------===//
+
+/// Include the definitions of the copy operation interface.
+#include "mlir/Interfaces/CopyOpInterface.cpp.inc"
diff --git a/mlir/lib/Interfaces/SideEffectInterfaces.cpp b/mlir/lib/Interfaces/SideEffectInterfaces.cpp
index 1d40f1dc200ec..89b2b34e76041 100644
--- a/mlir/lib/Interfaces/SideEffectInterfaces.cpp
+++ b/mlir/lib/Interfaces/SideEffectInterfaces.cpp
@@ -22,8 +22,7 @@ using namespace mlir;
 //===----------------------------------------------------------------------===//
 
 bool MemoryEffects::Effect::classof(const SideEffects::Effect *effect) {
-  return isa<Allocate>(effect) || isa<Free>(effect) || isa<Read>(effect) ||
-         isa<Write>(effect);
+  return isa<Allocate, Free, Read, Write>(effect);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Interfaces/VectorUnrollInterface.cpp b/mlir/lib/Interfaces/VectorUnrollInterface.cpp
new file mode 100644
index 0000000000000..6d3d432a70618
--- /dev/null
+++ b/mlir/lib/Interfaces/VectorUnrollInterface.cpp
@@ -0,0 +1,18 @@
+//===- VectorUnrollInterface.cpp - Unrollable vector operations -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Interfaces/VectorUnrollInterface.h"
+
+using namespace mlir;
+
+//===----------------------------------------------------------------------===//
+// VectorUnroll Interfaces
+//===----------------------------------------------------------------------===//
+
+/// Include the definitions of the VectorUntoll interfaces.
+#include "mlir/Interfaces/VectorUnrollInterface.cpp.inc"
diff --git a/mlir/lib/Parser/AttributeParser.cpp b/mlir/lib/Parser/AttributeParser.cpp
index ebbb1293f19d6..e2860b1152310 100644
--- a/mlir/lib/Parser/AttributeParser.cpp
+++ b/mlir/lib/Parser/AttributeParser.cpp
@@ -345,7 +345,7 @@ Attribute Parser::parseDecOrHexAttr(Type type, bool isNegative) {
     return apVal ? FloatAttr::get(floatType, *apVal) : Attribute();
   }
 
-  if (!type.isa<IntegerType>() && !type.isa<IndexType>())
+  if (!type.isa<IntegerType, IndexType>())
     return emitError(loc, "integer literal not valid for specified type"),
            nullptr;
 
@@ -758,13 +758,13 @@ Attribute Parser::parseDenseElementsAttr(Type attrType) {
   if (parseToken(Token::less, "expected '<' after 'dense'"))
     return nullptr;
 
-  // Parse the literal data.
+  // Parse the literal data if necessary.
   TensorLiteralParser literalParser(*this);
-  if (literalParser.parse(/*allowHex=*/true))
-    return nullptr;
-
-  if (parseToken(Token::greater, "expected '>'"))
-    return nullptr;
+  if (!consumeIf(Token::greater)) {
+    if (literalParser.parse(/*allowHex=*/true) ||
+        parseToken(Token::greater, "expected '>'"))
+      return nullptr;
+  }
 
   auto typeLoc = getToken().getLoc();
   auto type = parseElementsLiteralType(attrType);
@@ -784,7 +784,7 @@ Attribute Parser::parseOpaqueElementsAttr(Type attrType) {
 
   auto name = getToken().getStringValue();
   auto *dialect = builder.getContext()->getRegisteredDialect(name);
-  // TODO(shpeisman): Allow for having an unknown dialect on an opaque
+  // TODO: Allow for having an unknown dialect on an opaque
   // attribute. Otherwise, it can't be roundtripped without having the dialect
   // registered.
   if (!dialect)
@@ -823,7 +823,7 @@ ShapedType Parser::parseElementsLiteralType(Type type) {
       return nullptr;
   }
 
-  if (!type.isa<RankedTensorType>() && !type.isa<VectorType>()) {
+  if (!type.isa<RankedTensorType, VectorType>()) {
     emitError("elements literal must be a ranked tensor or vector type");
     return nullptr;
   }
@@ -841,6 +841,25 @@ Attribute Parser::parseSparseElementsAttr(Type attrType) {
   if (parseToken(Token::less, "Expected '<' after 'sparse'"))
     return nullptr;
 
+  // Check for the case where all elements are sparse. The indices are
+  // represented by a 2-dimensional shape where the second dimension is the rank
+  // of the type.
+  Type indiceEltType = builder.getIntegerType(64);
+  if (consumeIf(Token::greater)) {
+    ShapedType type = parseElementsLiteralType(attrType);
+    if (!type)
+      return nullptr;
+
+    // Construct the sparse elements attr using zero element indice/value
+    // attributes.
+    ShapedType indicesType =
+        RankedTensorType::get({0, type.getRank()}, indiceEltType);
+    ShapedType valuesType = RankedTensorType::get({0}, type.getElementType());
+    return SparseElementsAttr::get(
+        type, DenseElementsAttr::get(indicesType, ArrayRef<Attribute>()),
+        DenseElementsAttr::get(valuesType, ArrayRef<Attribute>()));
+  }
+
   /// Parse the indices. We don't allow hex values here as we may need to use
   /// the inferred shape.
   auto indicesLoc = getToken().getLoc();
@@ -869,7 +888,6 @@ Attribute Parser::parseSparseElementsAttr(Type attrType) {
   // 2-dimensional shape where the second dimension is the rank of the type.
   // Given that the parsed indices is a splat, we know that we only have one
   // indice and thus one for the first dimension.
-  auto indiceEltType = builder.getIntegerType(64);
   ShapedType indicesType;
   if (indiceParser.getShape().empty()) {
     indicesType = RankedTensorType::get({1, type.getRank()}, indiceEltType);
diff --git a/mlir/lib/Parser/DialectSymbolParser.cpp b/mlir/lib/Parser/DialectSymbolParser.cpp
index 9d14d6f4fa4fd..1a7e2c5448c19 100644
--- a/mlir/lib/Parser/DialectSymbolParser.cpp
+++ b/mlir/lib/Parser/DialectSymbolParser.cpp
@@ -76,7 +76,7 @@ class CustomDialectAsmParser : public DialectAsmParser {
       return success();
     }
 
-    // TODO(riverriddle) support hex floating point values.
+    // TODO: support hex floating point values.
     return emitError(getCurrentLocation(), "expected floating point literal");
   }
 
diff --git a/mlir/lib/Parser/Parser.cpp b/mlir/lib/Parser/Parser.cpp
index 465709d925e06..fc9d449ecc143 100644
--- a/mlir/lib/Parser/Parser.cpp
+++ b/mlir/lib/Parser/Parser.cpp
@@ -1504,7 +1504,8 @@ ParseResult OperationParser::parseRegion(
   pushSSANameScope(isIsolatedNameScope);
 
   // Parse the first block directly to allow for it to be unnamed.
-  Block *block = new Block();
+  auto owning_block = std::make_unique<Block>();
+  Block *block = owning_block.get();
 
   // Add arguments to the entry block.
   if (!entryArguments.empty()) {
@@ -1519,7 +1520,6 @@ ParseResult OperationParser::parseRegion(
       }
       if (addDefinition(placeholderArgPair.first,
                         block->addArgument(placeholderArgPair.second))) {
-        delete block;
         return failure();
       }
     }
@@ -1530,19 +1530,17 @@ ParseResult OperationParser::parseRegion(
   }
 
   if (parseBlock(block)) {
-    delete block;
     return failure();
   }
 
   // Verify that no other arguments were parsed.
   if (!entryArguments.empty() &&
       block->getNumArguments() > entryArguments.size()) {
-    delete block;
     return emitError("entry block arguments were already defined");
   }
 
   // Parse the rest of the region.
-  region.push_back(block);
+  region.push_back(owning_block.release());
   if (parseRegionBody(region))
     return failure();
 
@@ -1886,7 +1884,7 @@ OwningModuleRef mlir::parseSourceFile(StringRef filename,
                                       llvm::SourceMgr &sourceMgr,
                                       MLIRContext *context) {
   if (sourceMgr.getNumBuffers() != 0) {
-    // TODO(b/136086478): Extend to support multiple buffers.
+    // TODO: Extend to support multiple buffers.
     emitError(mlir::UnknownLoc::get(context),
               "only main buffer parsed at the moment");
     return nullptr;
diff --git a/mlir/lib/Parser/TypeParser.cpp b/mlir/lib/Parser/TypeParser.cpp
index 68d381f968ad1..9d8d198aa1c84 100644
--- a/mlir/lib/Parser/TypeParser.cpp
+++ b/mlir/lib/Parser/TypeParser.cpp
@@ -217,8 +217,8 @@ Type Parser::parseMemRefType() {
     return nullptr;
 
   // Check that memref is formed from allowed types.
-  if (!elementType.isIntOrFloat() && !elementType.isa<VectorType>() &&
-      !elementType.isa<ComplexType>())
+  if (!elementType.isIntOrFloat() &&
+      !elementType.isa<VectorType, ComplexType>())
     return emitError(typeLoc, "invalid memref element type"), nullptr;
 
   // Parse semi-affine-map-composition.
diff --git a/mlir/lib/Pass/IRPrinting.cpp b/mlir/lib/Pass/IRPrinting.cpp
index 1e9760581fcce..c27e47f8dc74d 100644
--- a/mlir/lib/Pass/IRPrinting.cpp
+++ b/mlir/lib/Pass/IRPrinting.cpp
@@ -97,7 +97,7 @@ class IRPrinterInstrumentation : public PassInstrumentation {
 
 /// Returns true if the given pass is hidden from IR printing.
 static bool isHiddenPass(Pass *pass) {
-  return isa<OpToOpPassAdaptor>(pass) || isa<VerifierPass>(pass);
+  return isa<OpToOpPassAdaptor, VerifierPass>(pass);
 }
 
 static void printIR(Operation *op, bool printModuleScope, raw_ostream &out,
diff --git a/mlir/lib/Pass/PassRegistry.cpp b/mlir/lib/Pass/PassRegistry.cpp
index fc1beae3a2d88..442233024bbec 100644
--- a/mlir/lib/Pass/PassRegistry.cpp
+++ b/mlir/lib/Pass/PassRegistry.cpp
@@ -128,7 +128,7 @@ void detail::PassOptions::copyOptionValuesFrom(const PassOptions &other) {
 }
 
 LogicalResult detail::PassOptions::parseFromString(StringRef options) {
-  // TODO(parkers): Handle escaping strings.
+  // TODO: Handle escaping strings.
   // NOTE: `options` is modified in place to always refer to the unprocessed
   // part of the string.
   while (!options.empty()) {
@@ -199,7 +199,7 @@ void detail::PassOptions::printHelp(size_t indent, size_t descIndent) const {
   };
   llvm::array_pod_sort(orderedOps.begin(), orderedOps.end(), compareOptionArgs);
   for (OptionBase *option : orderedOps) {
-    // TODO(riverriddle) printOptionInfo assumes a specific indent and will
+    // TODO: printOptionInfo assumes a specific indent and will
     // print options with values with incorrect indentation. We should add
     // support to llvm::cl::Option for passing in a base indent to use when
     // printing.
@@ -328,7 +328,7 @@ LogicalResult TextualPipeline::parsePipelineText(StringRef text,
       // Skip over everything until the closing '}' and store as options.
       size_t close = text.find('}');
 
-      // TODO(parkers): Handle skipping over quoted sub-strings.
+      // TODO: Handle skipping over quoted sub-strings.
       if (close == StringRef::npos) {
         return errorHandler(
             /*rawLoc=*/text.data() - 1,
diff --git a/mlir/lib/Reducer/CMakeLists.txt b/mlir/lib/Reducer/CMakeLists.txt
new file mode 100644
index 0000000000000..dd5fd277ce120
--- /dev/null
+++ b/mlir/lib/Reducer/CMakeLists.txt
@@ -0,0 +1,7 @@
+add_mlir_library(MLIRReduce
+   Tester.cpp
+   DEPENDS
+   MLIRIR
+ )
+ 
+ mlir_check_all_link_libraries(MLIRReduce)
\ No newline at end of file
diff --git a/mlir/lib/Reducer/Tester.cpp b/mlir/lib/Reducer/Tester.cpp
new file mode 100644
index 0000000000000..dcfce3237cecf
--- /dev/null
+++ b/mlir/lib/Reducer/Tester.cpp
@@ -0,0 +1,48 @@
+//===- Tester.cpp ---------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the Tester class used in the MLIR Reduce tool.
+//
+// A Tester object is passed as an argument to the reduction passes and it is
+// used to keep track of the state of the reduction throughout the multiple
+// passes.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Reducer/Tester.h"
+
+using namespace mlir;
+
+Tester::Tester(StringRef scriptName, ArrayRef<std::string> scriptArgs)
+    : testScript(scriptName), testScriptArgs(scriptArgs) {}
+
+/// Runs the interestingness testing script on a MLIR test case file. Returns
+/// true if the interesting behavior is present in the test case or false
+/// otherwise.
+bool Tester::isInteresting(StringRef testCase) {
+
+  std::vector<StringRef> testerArgs;
+  testerArgs.push_back(testCase);
+
+  for (const std::string &arg : testScriptArgs)
+    testerArgs.push_back(arg);
+
+  std::string errMsg;
+  int result = llvm::sys::ExecuteAndWait(
+      testScript, testerArgs, /*Env=*/None, /*Redirects=*/None,
+      /*SecondsToWait=*/0, /*MemoryLimit=*/0, &errMsg);
+
+  if (result < 0)
+    llvm::report_fatal_error("Error running interestingness test: " + errMsg,
+                             false);
+
+  if (!result)
+    return false;
+
+  return true;
+}
diff --git a/mlir/lib/TableGen/CMakeLists.txt b/mlir/lib/TableGen/CMakeLists.txt
index 77709c8649557..ef74764d099e2 100644
--- a/mlir/lib/TableGen/CMakeLists.txt
+++ b/mlir/lib/TableGen/CMakeLists.txt
@@ -14,9 +14,9 @@ llvm_add_library(MLIRTableGen STATIC
   Constraint.cpp
   Dialect.cpp
   Format.cpp
+  Interfaces.cpp
   Operator.cpp
   OpClass.cpp
-  OpInterfaces.cpp
   OpTrait.cpp
   Pass.cpp
   Pattern.cpp
diff --git a/mlir/lib/TableGen/Dialect.cpp b/mlir/lib/TableGen/Dialect.cpp
index db68ed43593f9..25be9fa03a7fe 100644
--- a/mlir/lib/TableGen/Dialect.cpp
+++ b/mlir/lib/TableGen/Dialect.cpp
@@ -34,8 +34,7 @@ std::string tblgen::Dialect::getCppClassName() const {
 static StringRef getAsStringOrEmpty(const llvm::Record &record,
                                     StringRef fieldName) {
   if (auto valueInit = record.getValueInit(fieldName)) {
-    if (llvm::isa<llvm::CodeInit>(valueInit) ||
-        llvm::isa<llvm::StringInit>(valueInit))
+    if (llvm::isa<llvm::CodeInit, llvm::StringInit>(valueInit))
       return record.getValueAsString(fieldName);
   }
   return "";
diff --git a/mlir/lib/TableGen/Interfaces.cpp b/mlir/lib/TableGen/Interfaces.cpp
new file mode 100644
index 0000000000000..1e6101f83caba
--- /dev/null
+++ b/mlir/lib/TableGen/Interfaces.cpp
@@ -0,0 +1,144 @@
+//===- Interfaces.cpp - Interface classes ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/TableGen/Interfaces.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+
+using namespace mlir;
+using namespace mlir::tblgen;
+
+//===----------------------------------------------------------------------===//
+// InterfaceMethod
+//===----------------------------------------------------------------------===//
+
+InterfaceMethod::InterfaceMethod(const llvm::Record *def) : def(def) {
+  llvm::DagInit *args = def->getValueAsDag("arguments");
+  for (unsigned i = 0, e = args->getNumArgs(); i != e; ++i) {
+    arguments.push_back(
+        {llvm::cast<llvm::StringInit>(args->getArg(i))->getValue(),
+         args->getArgNameStr(i)});
+  }
+}
+
+StringRef InterfaceMethod::getReturnType() const {
+  return def->getValueAsString("returnType");
+}
+
+// Return the name of this method.
+StringRef InterfaceMethod::getName() const {
+  return def->getValueAsString("name");
+}
+
+// Return if this method is static.
+bool InterfaceMethod::isStatic() const {
+  return def->isSubClassOf("StaticInterfaceMethod");
+}
+
+// Return the body for this method if it has one.
+llvm::Optional<StringRef> InterfaceMethod::getBody() const {
+  auto value = def->getValueAsString("body");
+  return value.empty() ? llvm::Optional<StringRef>() : value;
+}
+
+// Return the default implementation for this method if it has one.
+llvm::Optional<StringRef> InterfaceMethod::getDefaultImplementation() const {
+  auto value = def->getValueAsString("defaultBody");
+  return value.empty() ? llvm::Optional<StringRef>() : value;
+}
+
+// Return the description of this method if it has one.
+llvm::Optional<StringRef> InterfaceMethod::getDescription() const {
+  auto value = def->getValueAsString("description");
+  return value.empty() ? llvm::Optional<StringRef>() : value;
+}
+
+ArrayRef<InterfaceMethod::Argument> InterfaceMethod::getArguments() const {
+  return arguments;
+}
+
+bool InterfaceMethod::arg_empty() const { return arguments.empty(); }
+
+//===----------------------------------------------------------------------===//
+// Interface
+//===----------------------------------------------------------------------===//
+
+Interface::Interface(const llvm::Record *def) : def(def) {
+  assert(def->isSubClassOf("Interface") &&
+         "must be subclass of TableGen 'Interface' class");
+
+  auto *listInit = dyn_cast<llvm::ListInit>(def->getValueInit("methods"));
+  for (llvm::Init *init : listInit->getValues())
+    methods.emplace_back(cast<llvm::DefInit>(init)->getDef());
+}
+
+// Return the name of this interface.
+StringRef Interface::getName() const {
+  return def->getValueAsString("cppClassName");
+}
+
+// Return the C++ namespace of this interface.
+StringRef Interface::getCppNamespace() const {
+  return def->getValueAsString("cppNamespace");
+}
+
+// Return the methods of this interface.
+ArrayRef<InterfaceMethod> Interface::getMethods() const { return methods; }
+
+// Return the description of this method if it has one.
+llvm::Optional<StringRef> Interface::getDescription() const {
+  auto value = def->getValueAsString("description");
+  return value.empty() ? llvm::Optional<StringRef>() : value;
+}
+
+// Return the interfaces extra class declaration code.
+llvm::Optional<StringRef> Interface::getExtraClassDeclaration() const {
+  auto value = def->getValueAsString("extraClassDeclaration");
+  return value.empty() ? llvm::Optional<StringRef>() : value;
+}
+
+// Return the traits extra class declaration code.
+llvm::Optional<StringRef> Interface::getExtraTraitClassDeclaration() const {
+  auto value = def->getValueAsString("extraTraitClassDeclaration");
+  return value.empty() ? llvm::Optional<StringRef>() : value;
+}
+
+// Return the body for this method if it has one.
+llvm::Optional<StringRef> Interface::getVerify() const {
+  // Only OpInterface supports the verify method.
+  if (!isa<OpInterface>(this))
+    return llvm::None;
+  auto value = def->getValueAsString("verify");
+  return value.empty() ? llvm::Optional<StringRef>() : value;
+}
+
+//===----------------------------------------------------------------------===//
+// AttrInterface
+//===----------------------------------------------------------------------===//
+
+bool AttrInterface::classof(const Interface *interface) {
+  return interface->getDef().isSubClassOf("AttrInterface");
+}
+
+//===----------------------------------------------------------------------===//
+// OpInterface
+//===----------------------------------------------------------------------===//
+
+bool OpInterface::classof(const Interface *interface) {
+  return interface->getDef().isSubClassOf("OpInterface");
+}
+
+//===----------------------------------------------------------------------===//
+// TypeInterface
+//===----------------------------------------------------------------------===//
+
+bool TypeInterface::classof(const Interface *interface) {
+  return interface->getDef().isSubClassOf("TypeInterface");
+}
diff --git a/mlir/lib/TableGen/OpClass.cpp b/mlir/lib/TableGen/OpClass.cpp
index 9e6c9138a675f..09cb6cb9e0073 100644
--- a/mlir/lib/TableGen/OpClass.cpp
+++ b/mlir/lib/TableGen/OpClass.cpp
@@ -30,7 +30,7 @@ void tblgen::OpMethodSignature::writeDeclTo(raw_ostream &os) const {
 void tblgen::OpMethodSignature::writeDefTo(raw_ostream &os,
                                            StringRef namePrefix) const {
   // We need to remove the default values for parameters in method definition.
-  // TODO(antiagainst): We are using '=' and ',' as delimiters for parameter
+  // TODO: We are using '=' and ',' as delimiters for parameter
   // initializers. This is incorrect for initializer list with more than one
   // element. Change to a more robust approach.
   auto removeParamDefaultValue = [](StringRef params) {
@@ -222,7 +222,7 @@ void tblgen::OpClass::addTrait(Twine trait) {
 }
 
 void tblgen::OpClass::writeDeclTo(raw_ostream &os) const {
-  os << "class " << className << " : public Op<" << className;
+  os << "class " << className << " : public ::mlir::Op<" << className;
   for (const auto &trait : traitsVec)
     os << ", " << trait;
   os << "> {\npublic:\n";
diff --git a/mlir/lib/TableGen/OpInterfaces.cpp b/mlir/lib/TableGen/OpInterfaces.cpp
deleted file mode 100644
index be3782c788099..0000000000000
--- a/mlir/lib/TableGen/OpInterfaces.cpp
+++ /dev/null
@@ -1,105 +0,0 @@
-//===- OpInterfaces.cpp - OpInterfaces class ------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// OpInterfaces wrapper to simplify using TableGen OpInterfaces.
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/TableGen/OpInterfaces.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/Support/FormatVariadic.h"
-#include "llvm/TableGen/Error.h"
-#include "llvm/TableGen/Record.h"
-
-using namespace mlir;
-using namespace mlir::tblgen;
-
-OpInterfaceMethod::OpInterfaceMethod(const llvm::Record *def) : def(def) {
-  llvm::DagInit *args = def->getValueAsDag("arguments");
-  for (unsigned i = 0, e = args->getNumArgs(); i != e; ++i) {
-    arguments.push_back(
-        {llvm::cast<llvm::StringInit>(args->getArg(i))->getValue(),
-         args->getArgNameStr(i)});
-  }
-}
-
-StringRef OpInterfaceMethod::getReturnType() const {
-  return def->getValueAsString("returnType");
-}
-
-// Return the name of this method.
-StringRef OpInterfaceMethod::getName() const {
-  return def->getValueAsString("name");
-}
-
-// Return if this method is static.
-bool OpInterfaceMethod::isStatic() const {
-  return def->isSubClassOf("StaticInterfaceMethod");
-}
-
-// Return the body for this method if it has one.
-llvm::Optional<StringRef> OpInterfaceMethod::getBody() const {
-  auto value = def->getValueAsString("body");
-  return value.empty() ? llvm::Optional<StringRef>() : value;
-}
-
-// Return the default implementation for this method if it has one.
-llvm::Optional<StringRef> OpInterfaceMethod::getDefaultImplementation() const {
-  auto value = def->getValueAsString("defaultBody");
-  return value.empty() ? llvm::Optional<StringRef>() : value;
-}
-
-// Return the description of this method if it has one.
-llvm::Optional<StringRef> OpInterfaceMethod::getDescription() const {
-  auto value = def->getValueAsString("description");
-  return value.empty() ? llvm::Optional<StringRef>() : value;
-}
-
-ArrayRef<OpInterfaceMethod::Argument> OpInterfaceMethod::getArguments() const {
-  return arguments;
-}
-
-bool OpInterfaceMethod::arg_empty() const { return arguments.empty(); }
-
-OpInterface::OpInterface(const llvm::Record *def) : def(def) {
-  auto *listInit = dyn_cast<llvm::ListInit>(def->getValueInit("methods"));
-  for (llvm::Init *init : listInit->getValues())
-    methods.emplace_back(cast<llvm::DefInit>(init)->getDef());
-}
-
-// Return the name of this interface.
-StringRef OpInterface::getName() const {
-  return def->getValueAsString("cppClassName");
-}
-
-// Return the methods of this interface.
-ArrayRef<OpInterfaceMethod> OpInterface::getMethods() const { return methods; }
-
-// Return the description of this method if it has one.
-llvm::Optional<StringRef> OpInterface::getDescription() const {
-  auto value = def->getValueAsString("description");
-  return value.empty() ? llvm::Optional<StringRef>() : value;
-}
-
-// Return the interfaces extra class declaration code.
-llvm::Optional<StringRef> OpInterface::getExtraClassDeclaration() const {
-  auto value = def->getValueAsString("extraClassDeclaration");
-  return value.empty() ? llvm::Optional<StringRef>() : value;
-}
-
-// Return the traits extra class declaration code.
-llvm::Optional<StringRef> OpInterface::getExtraTraitClassDeclaration() const {
-  auto value = def->getValueAsString("extraTraitClassDeclaration");
-  return value.empty() ? llvm::Optional<StringRef>() : value;
-}
-
-// Return the body for this method if it has one.
-llvm::Optional<StringRef> OpInterface::getVerify() const {
-  auto value = def->getValueAsString("verify");
-  return value.empty() ? llvm::Optional<StringRef>() : value;
-}
diff --git a/mlir/lib/TableGen/OpTrait.cpp b/mlir/lib/TableGen/OpTrait.cpp
index 7a1e9cef0559b..dbfd0d374b83a 100644
--- a/mlir/lib/TableGen/OpTrait.cpp
+++ b/mlir/lib/TableGen/OpTrait.cpp
@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/TableGen/OpTrait.h"
-#include "mlir/TableGen/OpInterfaces.h"
+#include "mlir/TableGen/Interfaces.h"
 #include "mlir/TableGen/Predicate.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -27,7 +27,7 @@ OpTrait OpTrait::create(const llvm::Init *init) {
     return OpTrait(Kind::Pred, def);
   if (def->isSubClassOf("GenInternalOpTrait"))
     return OpTrait(Kind::Internal, def);
-  if (def->isSubClassOf("OpInterface"))
+  if (def->isSubClassOf("OpInterfaceTrait"))
     return OpTrait(Kind::Interface, def);
   assert(def->isSubClassOf("NativeOpTrait"));
   return OpTrait(Kind::Native, def);
@@ -56,8 +56,11 @@ OpInterface InterfaceOpTrait::getOpInterface() const {
   return OpInterface(def);
 }
 
-llvm::StringRef InterfaceOpTrait::getTrait() const {
-  return def->getValueAsString("trait");
+std::string InterfaceOpTrait::getTrait() const {
+  llvm::StringRef trait = def->getValueAsString("trait");
+  llvm::StringRef cppNamespace = def->getValueAsString("cppNamespace");
+  return cppNamespace.empty() ? trait.str()
+                              : (cppNamespace + "::" + trait).str();
 }
 
 bool InterfaceOpTrait::shouldDeclareMethods() const {
diff --git a/mlir/lib/TableGen/Operator.cpp b/mlir/lib/TableGen/Operator.cpp
index 534219b853225..3dd924566a8fd 100644
--- a/mlir/lib/TableGen/Operator.cpp
+++ b/mlir/lib/TableGen/Operator.cpp
@@ -336,7 +336,7 @@ void tblgen::Operator::populateTypeInferenceInfo(
             llvm::formatv("{0}::Trait", inferTypeOpInterface).str()))
       return;
     if (const auto *opTrait = dyn_cast<tblgen::InterfaceOpTrait>(&trait))
-      if (opTrait->getTrait().startswith(inferTypeOpInterface))
+      if (&opTrait->getDef() == inferTrait)
         return;
 
     if (!def.isSubClassOf("AllTypesMatch"))
@@ -436,9 +436,13 @@ void tblgen::Operator::populateOpStructure() {
       argDef = argDef->getValueAsDef("constraint");
 
     if (argDef->isSubClassOf(typeConstraintClass)) {
+      attrOrOperandMapping.push_back(
+          {OperandOrAttribute::Kind::Operand, operandIndex});
       arguments.emplace_back(&operands[operandIndex++]);
     } else {
       assert(argDef->isSubClassOf(attrClass));
+      attrOrOperandMapping.push_back(
+          {OperandOrAttribute::Kind::Attribute, attrIndex});
       arguments.emplace_back(&attributes[attrIndex++]);
     }
   }
@@ -558,7 +562,7 @@ StringRef tblgen::Operator::getSummary() const {
 
 bool tblgen::Operator::hasAssemblyFormat() const {
   auto *valueInit = def.getValueInit("assemblyFormat");
-  return isa<llvm::CodeInit>(valueInit) || isa<llvm::StringInit>(valueInit);
+  return isa<llvm::CodeInit, llvm::StringInit>(valueInit);
 }
 
 StringRef tblgen::Operator::getAssemblyFormat() const {
@@ -581,3 +585,8 @@ auto tblgen::Operator::VariableDecoratorIterator::unwrap(llvm::Init *init)
     -> VariableDecorator {
   return VariableDecorator(cast<llvm::DefInit>(init)->getDef());
 }
+
+auto tblgen::Operator::getArgToOperandOrAttribute(int index) const
+    -> OperandOrAttribute {
+  return attrOrOperandMapping[index];
+}
diff --git a/mlir/lib/TableGen/Pattern.cpp b/mlir/lib/TableGen/Pattern.cpp
index b04c8e215679e..c2b86b8875746 100644
--- a/mlir/lib/TableGen/Pattern.cpp
+++ b/mlir/lib/TableGen/Pattern.cpp
@@ -57,7 +57,7 @@ bool tblgen::DagLeaf::isEnumAttrCase() const {
 }
 
 bool tblgen::DagLeaf::isStringAttr() const {
-  return isa<llvm::StringInit>(def) || isa<llvm::CodeInit>(def);
+  return isa<llvm::StringInit, llvm::CodeInit>(def);
 }
 
 tblgen::Constraint tblgen::DagLeaf::getAsConstraint() const {
diff --git a/mlir/lib/TableGen/Predicate.cpp b/mlir/lib/TableGen/Predicate.cpp
index 2ebec1b3b6eef..bf617318e6948 100644
--- a/mlir/lib/TableGen/Predicate.cpp
+++ b/mlir/lib/TableGen/Predicate.cpp
@@ -213,7 +213,7 @@ static PredNode *propagateGroundTruth(
   // still point to the original predicate records.  While the original
   // predicate may be known to be true or false, it is not necessarily the case
   // after rewriting.
-  // TODO(zinenko,jpienaar): we can support ground truth for rewritten
+  // TODO: we can support ground truth for rewritten
   // predicates by either (a) having our own unique'ing of the predicates
   // instead of relying on TableGen record pointers or (b) taking ground truth
   // values optionally prefixed with a list of substitutions to apply, e.g.
diff --git a/mlir/lib/TableGen/SideEffects.cpp b/mlir/lib/TableGen/SideEffects.cpp
index a2116ba3c37ba..286cacfdacf8b 100644
--- a/mlir/lib/TableGen/SideEffects.cpp
+++ b/mlir/lib/TableGen/SideEffects.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/TableGen/SideEffects.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/TableGen/Record.h"
 
 using namespace mlir;
@@ -24,8 +25,11 @@ StringRef SideEffect::getBaseEffectName() const {
   return def->getValueAsString("baseEffectName");
 }
 
-StringRef SideEffect::getInterfaceTrait() const {
-  return def->getValueAsString("interfaceTrait");
+std::string SideEffect::getInterfaceTrait() const {
+  StringRef trait = def->getValueAsString("interfaceTrait");
+  StringRef cppNamespace = def->getValueAsString("cppNamespace");
+  return cppNamespace.empty() ? trait.str()
+                              : (cppNamespace + "::" + trait).str();
 }
 
 StringRef SideEffect::getResource() const {
diff --git a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
index 60514ca8f73b1..7aa26541ac279 100644
--- a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
+++ b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
@@ -106,7 +106,7 @@ class Importer {
   /// Globals are inserted before the first function, if any.
   Block::iterator getGlobalInsertPt() {
     auto i = module.getBody()->begin();
-    while (!isa<LLVMFuncOp>(i) && !isa<ModuleTerminatorOp>(i))
+    while (!isa<LLVMFuncOp, ModuleTerminatorOp>(i))
       ++i;
     return i;
   }
@@ -405,6 +405,9 @@ Value Importer::processConstant(llvm::Constant *c) {
     LLVMType type = processType(c->getType());
     if (!type)
       return nullptr;
+    if (auto symbolRef = attr.dyn_cast<FlatSymbolRefAttr>())
+      return instMap[c] = bEntry.create<AddressOfOp>(unknownLoc, type,
+                                                     symbolRef.getValue());
     return instMap[c] = bEntry.create<ConstantOp>(unknownLoc, type, attr);
   }
   if (auto *cn = dyn_cast<llvm::ConstantPointerNull>(c)) {
diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
index 74dd0d15f441e..f40f44f2fbc6d 100644
--- a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
@@ -32,7 +32,7 @@ DebugTranslation::DebugTranslation(Operation *module, llvm::Module &llvmModule)
   if (!module->walk(interruptIfValidLocation).wasInterrupted())
     return;
 
-  // TODO(riverriddle) Several parts of this are incorrect. Different source
+  // TODO: Several parts of this are incorrect. Different source
   // languages may interpret different parts of the debug information
   // differently. Frontends will also want to pipe in various information, like
   // flags. This is fine for now as we only emit line-table information and not
@@ -75,7 +75,7 @@ void DebugTranslation::translate(LLVMFuncOp func, llvm::Function &llvmFunc) {
   auto *file = translateFile(fileLoc ? fileLoc.getFilename() : "<unknown>");
   unsigned line = fileLoc ? fileLoc.getLine() : 0;
 
-  // TODO(riverriddle) This is the bare essentials for now. We will likely end
+  // TODO: This is the bare essentials for now. We will likely end
   // up with wrapper metadata around LLVMs metadata in the future, so this
   // doesn't need to be smart until then.
   llvm::DISubroutineType *type =
diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.h b/mlir/lib/Target/LLVMIR/DebugTranslation.h
index ae04f2b1d488c..062b8820c5a84 100644
--- a/mlir/lib/Target/LLVMIR/DebugTranslation.h
+++ b/mlir/lib/Target/LLVMIR/DebugTranslation.h
@@ -54,7 +54,7 @@ class DebugTranslation {
       locationToLoc;
 
   /// A mapping between filename and llvm debug file.
-  /// TODO(riverriddle) Change this to DenseMap<Identifier, ...> when we can
+  /// TODO: Change this to DenseMap<Identifier, ...> when we can
   /// access the Identifier filename in FileLineColLoc.
   llvm::StringMap<llvm::DIFile *> fileMap;
 
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 8e022f5697d70..657aa84afe1c2 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -121,7 +121,7 @@ llvm::Constant *ModuleTranslation::getLLVMConstant(llvm::Type *llvmType,
       elementType = arrayTy->getElementType();
       numElements = arrayTy->getNumElements();
     } else {
-      auto *vectorTy = cast<llvm::VectorType>(llvmType);
+      auto *vectorTy = cast<llvm::FixedVectorType>(llvmType);
       elementType = vectorTy->getElementType();
       numElements = vectorTy->getNumElements();
     }
@@ -129,7 +129,7 @@ llvm::Constant *ModuleTranslation::getLLVMConstant(llvm::Type *llvmType,
     // another sequence type. The recursion terminates because each step removes
     // one outer sequential type.
     bool elementTypeSequential =
-        isa<llvm::ArrayType>(elementType) || isa<llvm::VectorType>(elementType);
+        isa<llvm::ArrayType, llvm::VectorType>(elementType);
     llvm::Constant *child = getLLVMConstant(
         elementType,
         elementTypeSequential ? splatAttr : splatAttr.getSplatValue(), loc);
@@ -447,10 +447,15 @@ LogicalResult ModuleTranslation::convertOperation(Operation &opInst,
   // emit any LLVM instruction.
   if (auto addressOfOp = dyn_cast<LLVM::AddressOfOp>(opInst)) {
     LLVM::GlobalOp global = addressOfOp.getGlobal();
+    LLVM::LLVMFuncOp function = addressOfOp.getFunction();
+
     // The verifier should not have allowed this.
-    assert(global && "referencing an undefined global");
+    assert((global || function) &&
+           "referencing an undefined global or function");
 
-    valueMapping[addressOfOp.getResult()] = globalsMapping.lookup(global);
+    valueMapping[addressOfOp.getResult()] =
+        global ? globalsMapping.lookup(global)
+               : functionMapping.lookup(function.getName());
     return success();
   }
 
@@ -596,7 +601,7 @@ void ModuleTranslation::connectPHINodes(LLVMFuncOp func) {
   }
 }
 
-// TODO(mlir-team): implement an iterative version
+// TODO: implement an iterative version
 static void topologicalSortImpl(llvm::SetVector<Block *> &blocks, Block *b) {
   blocks.insert(b);
   for (Block *bb : b->getSuccessors()) {
@@ -773,18 +778,18 @@ LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {
 
 LogicalResult ModuleTranslation::checkSupportedModuleOps(Operation *m) {
   for (Operation &o : getModuleBody(m).getOperations())
-    if (!isa<LLVM::LLVMFuncOp>(&o) && !isa<LLVM::GlobalOp>(&o) &&
-        !o.isKnownTerminator())
+    if (!isa<LLVM::LLVMFuncOp, LLVM::GlobalOp>(&o) && !o.isKnownTerminator())
       return o.emitOpError("unsupported module-level operation");
   return success();
 }
 
-LogicalResult ModuleTranslation::convertFunctions() {
+LogicalResult ModuleTranslation::convertFunctionSignatures() {
   // Lock access to the llvm context.
   llvm::sys::SmartScopedLock<true> scopedLock(
       llvmDialect->getLLVMContextMutex());
+
   // Declare all functions first because there may be function calls that form a
-  // call graph with cycles.
+  // call graph with cycles, or global initializers that reference functions.
   for (auto function : getModuleBody(mlirModule).getOps<LLVMFuncOp>()) {
     llvm::FunctionCallee llvmFuncCst = llvmModule->getOrInsertFunction(
         function.getName(),
@@ -798,6 +803,14 @@ LogicalResult ModuleTranslation::convertFunctions() {
       return failure();
   }
 
+  return success();
+}
+
+LogicalResult ModuleTranslation::convertFunctions() {
+  // Lock access to the llvm context.
+  llvm::sys::SmartScopedLock<true> scopedLock(
+      llvmDialect->getLLVMContextMutex());
+
   // Convert functions.
   for (auto function : getModuleBody(mlirModule).getOps<LLVMFuncOp>()) {
     // Ignore external functions.
diff --git a/mlir/lib/Transforms/BufferPlacement.cpp b/mlir/lib/Transforms/BufferPlacement.cpp
index 74c534701536d..87b2687b6982f 100644
--- a/mlir/lib/Transforms/BufferPlacement.cpp
+++ b/mlir/lib/Transforms/BufferPlacement.cpp
@@ -65,8 +65,18 @@
 
 using namespace mlir;
 
-namespace {
+/// Walks over all immediate return-like terminators in the given region.
+template <typename FuncT>
+static void walkReturnOperations(Region *region, const FuncT &func) {
+  for (Block &block : *region)
+    for (Operation &operation : block) {
+      // Skip non-return-like terminators.
+      if (operation.hasTrait<OpTrait::ReturnLike>())
+        func(&operation);
+    }
+}
 
+namespace {
 //===----------------------------------------------------------------------===//
 // BufferPlacementAliasAnalysis
 //===----------------------------------------------------------------------===//
@@ -82,7 +92,7 @@ class BufferPlacementAliasAnalysis {
 
 public:
   /// Constructs a new alias analysis using the op provided.
-  BufferPlacementAliasAnalysis(Operation *op) { build(op->getRegions()); }
+  BufferPlacementAliasAnalysis(Operation *op) { build(op); }
 
   /// Find all immediate aliases this value could potentially have.
   ValueMapT::const_iterator find(Value value) const {
@@ -102,7 +112,7 @@ class BufferPlacementAliasAnalysis {
   }
 
   /// Removes the given values from all alias sets.
-  void remove(const SmallPtrSetImpl<BlockArgument> &aliasValues) {
+  void remove(const SmallPtrSetImpl<Value> &aliasValues) {
     for (auto &entry : aliases)
       llvm::set_subtract(entry.second, aliasValues);
   }
@@ -123,33 +133,75 @@ class BufferPlacementAliasAnalysis {
   /// This function constructs a mapping from values to its immediate aliases.
   /// It iterates over all blocks, gets their predecessors, determines the
   /// values that will be passed to the corresponding block arguments and
-  /// inserts them into the underlying map.
-  void build(MutableArrayRef<Region> regions) {
-    for (Region &region : regions) {
-      for (Block &block : region) {
-        // Iterate over all predecessor and get the mapped values to their
-        // corresponding block arguments values.
-        for (auto it = block.pred_begin(), e = block.pred_end(); it != e;
-             ++it) {
-          unsigned successorIndex = it.getSuccessorIndex();
-          // Get the terminator and the values that will be passed to our block.
-          auto branchInterface =
-              dyn_cast<BranchOpInterface>((*it)->getTerminator());
-          if (!branchInterface)
-            continue;
-          // Query the branch op interace to get the successor operands.
-          auto successorOperands =
-              branchInterface.getSuccessorOperands(successorIndex);
-          if (successorOperands.hasValue()) {
-            // Build the actual mapping of values to their immediate aliases.
-            for (auto argPair : llvm::zip(block.getArguments(),
-                                          successorOperands.getValue())) {
-              aliases[std::get<1>(argPair)].insert(std::get<0>(argPair));
-            }
-          }
+  /// inserts them into the underlying map. Furthermore, it wires successor
+  /// regions and branch-like return operations from nested regions.
+  void build(Operation *op) {
+    // Registers all aliases of the given values.
+    auto registerAliases = [&](auto values, auto aliases) {
+      for (auto entry : llvm::zip(values, aliases))
+        this->aliases[std::get<0>(entry)].insert(std::get<1>(entry));
+    };
+
+    // Add additional aliases created by view changes to the alias list.
+    op->walk([&](ViewLikeOpInterface viewInterface) {
+      aliases[viewInterface.getViewSource()].insert(
+          viewInterface.getOperation()->getResult(0));
+    });
+
+    // Query all branch interfaces to link block argument aliases.
+    op->walk([&](BranchOpInterface branchInterface) {
+      Block *parentBlock = branchInterface.getOperation()->getBlock();
+      for (auto it = parentBlock->succ_begin(), e = parentBlock->succ_end();
+           it != e; ++it) {
+        // Query the branch op interface to get the successor operands.
+        auto successorOperands =
+            branchInterface.getSuccessorOperands(it.getIndex());
+        if (!successorOperands.hasValue())
+          continue;
+        // Build the actual mapping of values to their immediate aliases.
+        registerAliases(successorOperands.getValue(), (*it)->getArguments());
+      }
+    });
+
+    // Query the RegionBranchOpInterface to find potential successor regions.
+    op->walk([&](RegionBranchOpInterface regionInterface) {
+      // Create an empty attribute for each operand to comply with the
+      // `getSuccessorRegions` interface definition that requires a single
+      // attribute per operand.
+      SmallVector<Attribute, 2> operandAttributes(
+          regionInterface.getOperation()->getNumOperands());
+
+      // Extract all entry regions and wire all initial entry successor inputs.
+      SmallVector<RegionSuccessor, 2> entrySuccessors;
+      regionInterface.getSuccessorRegions(/*index=*/llvm::None,
+                                          operandAttributes, entrySuccessors);
+      for (RegionSuccessor &entrySuccessor : entrySuccessors) {
+        // Wire the entry region's successor arguments with the initial
+        // successor inputs.
+        assert(entrySuccessor.getSuccessor() &&
+               "Invalid entry region without an attached successor region");
+        registerAliases(regionInterface.getSuccessorEntryOperands(
+                            entrySuccessor.getSuccessor()->getRegionNumber()),
+                        entrySuccessor.getSuccessorInputs());
+      }
+
+      // Wire flow between regions and from region exits.
+      for (Region &region : regionInterface.getOperation()->getRegions()) {
+        // Iterate over all successor region entries that are reachable from the
+        // current region.
+        SmallVector<RegionSuccessor, 2> successorRegions;
+        regionInterface.getSuccessorRegions(
+            region.getRegionNumber(), operandAttributes, successorRegions);
+        for (RegionSuccessor &successorRegion : successorRegions) {
+          // Iterate over all immediate terminator operations and wire the
+          // successor inputs with the operands of each terminator.
+          walkReturnOperations(&region, [&](Operation *terminator) {
+            registerAliases(terminator->getOperands(),
+                            successorRegion.getSuccessorInputs());
+          });
         }
       }
-    }
+    });
   }
 
   /// Maps values to all immediate aliases this value can have.
@@ -235,14 +287,24 @@ class BufferPlacement {
   Block *getInitialAllocBlock(OpResult result) {
     // Get all allocation operands as these operands are important for the
     // allocation operation.
-    auto operands = result.getOwner()->getOperands();
+    Operation *owner = result.getOwner();
+    auto operands = owner->getOperands();
+    Block *dominator;
     if (operands.size() < 1)
-      return findCommonDominator(result, aliases.resolve(result), dominators);
+      dominator =
+          findCommonDominator(result, aliases.resolve(result), dominators);
+    else {
+      // If this node has dependencies, check all dependent nodes with respect
+      // to a common post dominator in which all values are available.
+      ValueSetT dependencies(++operands.begin(), operands.end());
+      dominator =
+          findCommonDominator(*operands.begin(), dependencies, postDominators);
+    }
 
-    // If this node has dependencies, check all dependent nodes with respect
-    // to a common post dominator in which all values are available.
-    ValueSetT dependencies(++operands.begin(), operands.end());
-    return findCommonDominator(*operands.begin(), dependencies, postDominators);
+    // Do not move allocs out of their parent regions to keep them local.
+    if (dominator->getParent() != owner->getParentRegion())
+      return &owner->getParentRegion()->front();
+    return dominator;
   }
 
   /// Finds correct alloc positions according to the algorithm described at
@@ -273,12 +335,12 @@ class BufferPlacement {
 
   /// Introduces required allocs and copy operations to avoid memory leaks.
   void introduceCopies() {
-    // Initialize the set of block arguments that require a dedicated memory
-    // free operation since their arguments cannot be safely deallocated in a
-    // post dominator.
-    SmallPtrSet<BlockArgument, 8> blockArgsToFree;
-    llvm::SmallDenseSet<std::tuple<BlockArgument, Block *>> visitedBlockArgs;
-    SmallVector<std::tuple<BlockArgument, Block *>, 8> toProcess;
+    // Initialize the set of values that require a dedicated memory free
+    // operation since their operands cannot be safely deallocated in a post
+    // dominator.
+    SmallPtrSet<Value, 8> valuesToFree;
+    llvm::SmallDenseSet<std::tuple<Value, Block *>> visitedValues;
+    SmallVector<std::tuple<Value, Block *>, 8> toProcess;
 
     // Check dominance relation for proper dominance properties. If the given
     // value node does not dominate an alias, we will have to create a copy in
@@ -289,17 +351,15 @@ class BufferPlacement {
       if (it == aliases.end())
         return;
       for (Value value : it->second) {
-        auto blockArg = value.cast<BlockArgument>();
-        if (blockArgsToFree.count(blockArg) > 0)
+        if (valuesToFree.count(value) > 0)
           continue;
         // Check whether we have to free this particular block argument.
-        if (!dominators.dominates(definingBlock, blockArg.getOwner())) {
-          toProcess.emplace_back(blockArg, blockArg.getParentBlock());
-          blockArgsToFree.insert(blockArg);
-        } else if (visitedBlockArgs
-                       .insert(std::make_tuple(blockArg, definingBlock))
+        if (!dominators.dominates(definingBlock, value.getParentBlock())) {
+          toProcess.emplace_back(value, value.getParentBlock());
+          valuesToFree.insert(value);
+        } else if (visitedValues.insert(std::make_tuple(value, definingBlock))
                        .second)
-          toProcess.emplace_back(blockArg, definingBlock);
+          toProcess.emplace_back(value, definingBlock);
       }
     };
 
@@ -316,60 +376,168 @@ class BufferPlacement {
 
     // Update buffer aliases to ensure that we free all buffers and block
     // arguments at the correct locations.
-    aliases.remove(blockArgsToFree);
+    aliases.remove(valuesToFree);
 
     // Add new allocs and additional copy operations.
-    for (BlockArgument blockArg : blockArgsToFree) {
-      Block *block = blockArg.getOwner();
-
-      // Allocate a buffer for the current block argument in the block of
-      // the associated value (which will be a predecessor block by
-      // definition).
-      for (auto it = block->pred_begin(), e = block->pred_end(); it != e;
-           ++it) {
-        // Get the terminator and the value that will be passed to our
-        // argument.
-        Operation *terminator = (*it)->getTerminator();
-        auto branchInterface = cast<BranchOpInterface>(terminator);
-        // Convert the mutable operand range to an immutable range and query the
-        // associated source value.
-        Value sourceValue =
-            branchInterface.getSuccessorOperands(it.getSuccessorIndex())
-                .getValue()[blockArg.getArgNumber()];
-        // Create a new alloc at the current location of the terminator.
-        auto memRefType = sourceValue.getType().cast<MemRefType>();
-        OpBuilder builder(terminator);
-
-        // Extract information about dynamically shaped types by
-        // extracting their dynamic dimensions.
-        SmallVector<Value, 4> dynamicOperands;
-        for (auto shapeElement : llvm::enumerate(memRefType.getShape())) {
-          if (!ShapedType::isDynamic(shapeElement.value()))
-            continue;
-          dynamicOperands.push_back(builder.create<DimOp>(
-              terminator->getLoc(), sourceValue, shapeElement.index()));
-        }
+    for (Value value : valuesToFree) {
+      if (auto blockArg = value.dyn_cast<BlockArgument>())
+        introduceBlockArgCopy(blockArg);
+      else
+        introduceValueCopyForRegionResult(value);
+
+      // Register the value to require a final dealloc. Note that we do not have
+      // to assign a block here since we do not want to move the allocation node
+      // to another location.
+      allocs.push_back({value, nullptr, nullptr});
+    }
+  }
 
-        // TODO: provide a generic interface to create dialect-specific
-        // Alloc and CopyOp nodes.
-        auto alloc = builder.create<AllocOp>(terminator->getLoc(), memRefType,
-                                             dynamicOperands);
-        // Wire new alloc and successor operand.
-        branchInterface.getMutableSuccessorOperands(it.getSuccessorIndex())
-            .getValue()
+  /// Introduces temporary allocs in all predecessors and copies the source
+  /// values into the newly allocated buffers.
+  void introduceBlockArgCopy(BlockArgument blockArg) {
+    // Allocate a buffer for the current block argument in the block of
+    // the associated value (which will be a predecessor block by
+    // definition).
+    Block *block = blockArg.getOwner();
+    for (auto it = block->pred_begin(), e = block->pred_end(); it != e; ++it) {
+      // Get the terminator and the value that will be passed to our
+      // argument.
+      Operation *terminator = (*it)->getTerminator();
+      auto branchInterface = cast<BranchOpInterface>(terminator);
+      // Query the associated source value.
+      Value sourceValue =
+          branchInterface.getSuccessorOperands(it.getSuccessorIndex())
+              .getValue()[blockArg.getArgNumber()];
+      // Create a new alloc and copy at the current location of the terminator.
+      Value alloc = introduceBufferCopy(sourceValue, terminator);
+      // Wire new alloc and successor operand.
+      auto mutableOperands =
+          branchInterface.getMutableSuccessorOperands(it.getSuccessorIndex());
+      if (!mutableOperands.hasValue())
+        terminator->emitError() << "terminators with immutable successor "
+                                   "operands are not supported";
+      else
+        mutableOperands.getValue()
             .slice(blockArg.getArgNumber(), 1)
             .assign(alloc);
-        // Create a new copy operation that copies to contents of the old
-        // allocation to the new one.
-        builder.create<linalg::CopyOp>(terminator->getLoc(), sourceValue,
-                                       alloc);
-      }
+    }
+
+    // Check whether the block argument has implicitly defined predecessors via
+    // the RegionBranchOpInterface. This can be the case if the current block
+    // argument belongs to the first block in a region and the parent operation
+    // implements the RegionBranchOpInterface.
+    Region *argRegion = block->getParent();
+    RegionBranchOpInterface regionInterface;
+    if (!argRegion || &argRegion->front() != block ||
+        !(regionInterface =
+              dyn_cast<RegionBranchOpInterface>(argRegion->getParentOp())))
+      return;
+
+    introduceCopiesForRegionSuccessors(
+        regionInterface, argRegion->getParentOp()->getRegions(),
+        [&](RegionSuccessor &successorRegion) {
+          // Find a predecessor of our argRegion.
+          return successorRegion.getSuccessor() == argRegion;
+        },
+        [&](RegionSuccessor &successorRegion) {
+          // The operand index will be the argument number.
+          return blockArg.getArgNumber();
+        });
+  }
+
+  /// Introduces temporary allocs in front of all associated nested-region
+  /// terminators and copies the source values into the newly allocated buffers.
+  void introduceValueCopyForRegionResult(Value value) {
+    // Get the actual result index in the scope of the parent terminator.
+    Operation *operation = value.getDefiningOp();
+    auto regionInterface = cast<RegionBranchOpInterface>(operation);
+    introduceCopiesForRegionSuccessors(
+        regionInterface, operation->getRegions(),
+        [&](RegionSuccessor &successorRegion) {
+          // Determine whether this region has a successor entry that leaves
+          // this region by returning to its parent operation.
+          return !successorRegion.getSuccessor();
+        },
+        [&](RegionSuccessor &successorRegion) {
+          // Find the associated success input index.
+          return llvm::find(successorRegion.getSuccessorInputs(), value)
+              .getIndex();
+        });
+  }
+
+  /// Introduces buffer copies for all terminators in the given regions. The
+  /// regionPredicate is applied to every successor region in order to restrict
+  /// the copies to specific regions. Thereby, the operandProvider is invoked
+  /// for each matching region successor and determines the operand index that
+  /// requires a buffer copy.
+  template <typename TPredicate, typename TOperandProvider>
+  void
+  introduceCopiesForRegionSuccessors(RegionBranchOpInterface regionInterface,
+                                     MutableArrayRef<Region> regions,
+                                     const TPredicate &regionPredicate,
+                                     const TOperandProvider &operandProvider) {
+    // Create an empty attribute for each operand to comply with the
+    // `getSuccessorRegions` interface definition that requires a single
+    // attribute per operand.
+    SmallVector<Attribute, 2> operandAttributes(
+        regionInterface.getOperation()->getNumOperands());
+    for (Region &region : regions) {
+      // Query the regionInterface to get all successor regions of the current
+      // one.
+      SmallVector<RegionSuccessor, 2> successorRegions;
+      regionInterface.getSuccessorRegions(region.getRegionNumber(),
+                                          operandAttributes, successorRegions);
+      // Try to find a matching region successor.
+      RegionSuccessor *regionSuccessor =
+          llvm::find_if(successorRegions, regionPredicate);
+      if (regionSuccessor == successorRegions.end())
+        continue;
+      // Get the operand index in the context of the current successor input
+      // bindings.
+      auto operandIndex = operandProvider(*regionSuccessor);
+
+      // Iterate over all immediate terminator operations to introduce
+      // new buffer allocations. Thereby, the appropriate terminator operand
+      // will be adjusted to point to the newly allocated buffer instead.
+      walkReturnOperations(&region, [&](Operation *terminator) {
+        // Extract the source value from the current terminator.
+        Value sourceValue = terminator->getOperand(operandIndex);
+        // Create a new alloc at the current location of the terminator.
+        Value alloc = introduceBufferCopy(sourceValue, terminator);
+        // Wire alloc and terminator operand.
+        terminator->setOperand(operandIndex, alloc);
+      });
+    }
+  }
 
-      // Register the block argument to require a final dealloc. Note that
-      // we do not have to assign a block here since we do not want to
-      // move the allocation node to another location.
-      allocs.push_back({blockArg, nullptr, nullptr});
+  /// Creates a new memory allocation for the given source value and copies
+  /// its content into the newly allocated buffer. The terminator operation is
+  /// used to insert the alloc and copy operations at the right places.
+  Value introduceBufferCopy(Value sourceValue, Operation *terminator) {
+    // Create a new alloc at the current location of the terminator.
+    auto memRefType = sourceValue.getType().cast<MemRefType>();
+    OpBuilder builder(terminator);
+
+    // Extract information about dynamically shaped types by
+    // extracting their dynamic dimensions.
+    SmallVector<Value, 4> dynamicOperands;
+    for (auto shapeElement : llvm::enumerate(memRefType.getShape())) {
+      if (!ShapedType::isDynamic(shapeElement.value()))
+        continue;
+      dynamicOperands.push_back(builder.create<DimOp>(
+          terminator->getLoc(), sourceValue, shapeElement.index()));
     }
+
+    // TODO: provide a generic interface to create dialect-specific
+    // Alloc and CopyOp nodes.
+    auto alloc = builder.create<AllocOp>(terminator->getLoc(), memRefType,
+                                         dynamicOperands);
+
+    // Create a new copy operation that copies to contents of the old
+    // allocation to the new one.
+    builder.create<linalg::CopyOp>(terminator->getLoc(), sourceValue, alloc);
+
+    return alloc;
   }
 
   /// Finds associated deallocs that can be linked to our allocation nodes (if
@@ -440,8 +608,8 @@ class BufferPlacement {
       if (entry.deallocOperation) {
         entry.deallocOperation->moveAfter(endOperation);
       } else {
-        // If the Dealloc position is at the terminator operation of the block,
-        // then the value should escape from a deallocation.
+        // If the Dealloc position is at the terminator operation of the
+        // block, then the value should escape from a deallocation.
         Operation *nextOp = endOperation->getNextNode();
         if (!nextOp)
           continue;
@@ -532,15 +700,19 @@ BufferAssignmentPlacer::computeAllocPosition(OpResult result) {
 BufferAssignmentTypeConverter::BufferAssignmentTypeConverter() {
   // Keep all types unchanged.
   addConversion([](Type type) { return type; });
-  // A type conversion that converts ranked-tensor type to memref type.
+  // Convert RankedTensorType to MemRefType.
   addConversion([](RankedTensorType type) {
     return (Type)MemRefType::get(type.getShape(), type.getElementType());
   });
+  // Convert UnrankedTensorType to UnrankedMemRefType.
+  addConversion([](UnrankedTensorType type) {
+    return (Type)UnrankedMemRefType::get(type.getElementType(), 0);
+  });
 }
 
 /// Checks if `type` has been converted from non-memref type to memref.
 bool BufferAssignmentTypeConverter::isConvertedMemref(Type type, Type before) {
-  return type.isa<MemRefType>() && !before.isa<MemRefType>();
+  return type.isa<BaseMemRefType>() && !before.isa<BaseMemRefType>();
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Transforms/CMakeLists.txt b/mlir/lib/Transforms/CMakeLists.txt
index f391a2662ac4b..3c6b3933de2a8 100644
--- a/mlir/lib/Transforms/CMakeLists.txt
+++ b/mlir/lib/Transforms/CMakeLists.txt
@@ -3,6 +3,7 @@ add_subdirectory(Utils)
 add_mlir_library(MLIRTransforms
   BufferPlacement.cpp
   Canonicalizer.cpp
+  CopyRemoval.cpp
   CSE.cpp
   DialectConversion.cpp
   Inliner.cpp
@@ -30,6 +31,7 @@ add_mlir_library(MLIRTransforms
   LINK_LIBS PUBLIC
   MLIRAffineOps
   MLIRAnalysis
+  MLIRCopyOpInterface
   MLIRLinalgOps
   MLIRLoopLikeInterface
   MLIRSCF
diff --git a/mlir/lib/Transforms/CSE.cpp b/mlir/lib/Transforms/CSE.cpp
index f7cf7f1c8e95b..5f4791450c7b3 100644
--- a/mlir/lib/Transforms/CSE.cpp
+++ b/mlir/lib/Transforms/CSE.cpp
@@ -64,7 +64,7 @@ struct CSE : public CSEBase<CSE> {
     ScopedMapTy::ScopeTy scope;
 
     DominanceInfoNode *node;
-    DominanceInfoNode::iterator childIterator;
+    DominanceInfoNode::const_iterator childIterator;
 
     /// If this node has been fully processed yet or not.
     bool processed;
@@ -106,7 +106,7 @@ LogicalResult CSE::simplifyOperation(ScopedMapTy &knownValues, Operation *op) {
   if (op->getNumRegions() != 0)
     return failure();
 
-  // TODO(riverriddle) We currently only eliminate non side-effecting
+  // TODO: We currently only eliminate non side-effecting
   // operations.
   if (!MemoryEffectOpInterface::hasNoEffect(op))
     return failure();
diff --git a/mlir/lib/Transforms/CopyRemoval.cpp b/mlir/lib/Transforms/CopyRemoval.cpp
new file mode 100644
index 0000000000000..28648e0b4294a
--- /dev/null
+++ b/mlir/lib/Transforms/CopyRemoval.cpp
@@ -0,0 +1,191 @@
+//===- CopyRemoval.cpp - Removing the redundant copies --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Interfaces/CopyOpInterface.h"
+#include "mlir/Interfaces/SideEffectInterfaces.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/Passes.h"
+
+using namespace mlir;
+using namespace MemoryEffects;
+
+namespace {
+
+//===----------------------------------------------------------------------===//
+// CopyRemovalPass
+//===----------------------------------------------------------------------===//
+/// This pass removes the redundant Copy operations. Additionally, it
+/// removes the leftover definition and deallocation operations by erasing the
+/// copy operation.
+class CopyRemovalPass : public PassWrapper<CopyRemovalPass, OperationPass<>> {
+private:
+  /// List of operations that need to be removed.
+  DenseSet<Operation *> eraseList;
+
+  /// Returns the deallocation operation for `value` in `block` if it exists.
+  Operation *getDeallocationInBlock(Value value, Block *block) {
+    auto valueUsers = value.getUsers();
+    auto it = llvm::find_if(valueUsers, [&](Operation *op) {
+      auto effects = dyn_cast<MemoryEffectOpInterface>(op);
+      return effects && op->getBlock() == block && effects.hasEffect<Free>();
+    });
+    return (it == valueUsers.end() ? nullptr : *it);
+  }
+
+  /// Returns true if an operation between start and end operations has memory
+  /// effect.
+  bool hasMemoryEffectOpBetween(Operation *start, Operation *end) {
+    assert(start->getBlock() == end->getBlock() &&
+           "Start and end operations should be in the same block.");
+    Operation *op = start->getNextNode();
+    while (op->isBeforeInBlock(end)) {
+      auto effects = dyn_cast<MemoryEffectOpInterface>(op);
+      if (effects)
+        return true;
+      op = op->getNextNode();
+    }
+    return false;
+  };
+
+  /// Returns true if `val` value has at least a user between `start` and
+  /// `end` operations.
+  bool hasUsersBetween(Value val, Operation *start, Operation *end) {
+    Block *block = start->getBlock();
+    assert(block == end->getBlock() &&
+           "Start and end operations should be in the same block.");
+    return llvm::any_of(val.getUsers(), [&](Operation *op) {
+      return op->getBlock() == block && start->isBeforeInBlock(op) &&
+             op->isBeforeInBlock(end);
+    });
+  };
+
+  bool areOpsInTheSameBlock(ArrayRef<Operation *> operations) {
+    llvm::SmallPtrSet<Block *, 4> blocks;
+    for (Operation *op : operations)
+      blocks.insert(op->getBlock());
+    return blocks.size() == 1;
+  }
+
+  /// Input:
+  /// func(){
+  ///   %from = alloc()
+  ///   write_to(%from)
+  ///   %to = alloc()
+  ///   copy(%from,%to)
+  ///   dealloc(%from)
+  ///   return %to
+  /// }
+  ///
+  /// Output:
+  /// func(){
+  ///   %from = alloc()
+  ///   write_to(%from)
+  ///   return %from
+  /// }
+  /// Constraints:
+  /// 1) %to, copy and dealloc must all be defined and lie in the same block.
+  /// 2) This transformation cannot be applied if there is a single user/alias
+  /// of `to` value between the defining operation of `to` and the copy
+  /// operation.
+  /// 3) This transformation cannot be applied if there is a single user/alias
+  /// of `from` value between the copy operation and the deallocation of `from`.
+  /// TODO: Alias analysis is not available at the moment. Currently, we check
+  /// if there are any operations with memory effects between copy and
+  /// deallocation operations.
+  void ReuseCopySourceAsTarget(CopyOpInterface copyOp) {
+    if (eraseList.count(copyOp))
+      return;
+
+    Value from = copyOp.getSource();
+    Value to = copyOp.getTarget();
+
+    Operation *copy = copyOp.getOperation();
+    Operation *fromDefiningOp = from.getDefiningOp();
+    Operation *fromFreeingOp = getDeallocationInBlock(from, copy->getBlock());
+    Operation *toDefiningOp = to.getDefiningOp();
+    if (!fromDefiningOp || !fromFreeingOp || !toDefiningOp ||
+        !areOpsInTheSameBlock({fromFreeingOp, toDefiningOp, copy}) ||
+        hasUsersBetween(to, toDefiningOp, copy) ||
+        hasUsersBetween(from, copy, fromFreeingOp) ||
+        hasMemoryEffectOpBetween(copy, fromFreeingOp))
+      return;
+
+    to.replaceAllUsesWith(from);
+    eraseList.insert(copy);
+    eraseList.insert(toDefiningOp);
+    eraseList.insert(fromFreeingOp);
+  }
+
+  /// Input:
+  /// func(){
+  ///   %to = alloc()
+  ///   %from = alloc()
+  ///   write_to(%from)
+  ///   copy(%from,%to)
+  ///   dealloc(%from)
+  ///   return %to
+  /// }
+  ///
+  /// Output:
+  /// func(){
+  ///   %to = alloc()
+  ///   write_to(%to)
+  ///   return %to
+  /// }
+  /// Constraints:
+  /// 1) %from, copy and dealloc must all be defined and lie in the same block.
+  /// 2) This transformation cannot be applied if there is a single user/alias
+  /// of `to` value between the defining operation of `from` and the copy
+  /// operation.
+  /// 3) This transformation cannot be applied if there is a single user/alias
+  /// of `from` value between the copy operation and the deallocation of `from`.
+  /// TODO: Alias analysis is not available at the moment. Currently, we check
+  /// if there are any operations with memory effects between copy and
+  /// deallocation operations.
+  void ReuseCopyTargetAsSource(CopyOpInterface copyOp) {
+    if (eraseList.count(copyOp))
+      return;
+
+    Value from = copyOp.getSource();
+    Value to = copyOp.getTarget();
+
+    Operation *copy = copyOp.getOperation();
+    Operation *fromDefiningOp = from.getDefiningOp();
+    Operation *fromFreeingOp = getDeallocationInBlock(from, copy->getBlock());
+    if (!fromDefiningOp || !fromFreeingOp ||
+        !areOpsInTheSameBlock({fromFreeingOp, fromDefiningOp, copy}) ||
+        hasUsersBetween(to, fromDefiningOp, copy) ||
+        hasUsersBetween(from, copy, fromFreeingOp) ||
+        hasMemoryEffectOpBetween(copy, fromFreeingOp))
+      return;
+
+    from.replaceAllUsesWith(to);
+    eraseList.insert(copy);
+    eraseList.insert(fromDefiningOp);
+    eraseList.insert(fromFreeingOp);
+  }
+
+public:
+  void runOnOperation() override {
+    getOperation()->walk([&](CopyOpInterface copyOp) {
+      ReuseCopySourceAsTarget(copyOp);
+      ReuseCopyTargetAsSource(copyOp);
+    });
+    for (Operation *op : eraseList)
+      op->erase();
+  }
+};
+
+} // end anonymous namespace
+
+//===----------------------------------------------------------------------===//
+// CopyRemovalPass construction
+//===----------------------------------------------------------------------===//
+std::unique_ptr<Pass> mlir::createCopyRemovalPass() {
+  return std::make_unique<CopyRemovalPass>();
+}
diff --git a/mlir/lib/Transforms/DialectConversion.cpp b/mlir/lib/Transforms/DialectConversion.cpp
index ecebe61d025fc..9401121eed966 100644
--- a/mlir/lib/Transforms/DialectConversion.cpp
+++ b/mlir/lib/Transforms/DialectConversion.cpp
@@ -450,7 +450,7 @@ void ArgConverter::insertConversion(Block *newBlock,
 }
 
 //===----------------------------------------------------------------------===//
-// ConversionPatternRewriterImpl
+// Rewriter and Transation State
 //===----------------------------------------------------------------------===//
 namespace {
 /// This class contains a snapshot of the current conversion rewriter state.
@@ -515,74 +515,89 @@ class OperationTransactionState {
   SmallVector<Value, 8> operands;
   SmallVector<Block *, 2> successors;
 };
-} // end anonymous namespace
 
-namespace mlir {
-namespace detail {
-struct ConversionPatternRewriterImpl {
-  /// This class represents one requested operation replacement via 'replaceOp'.
-  struct OpReplacement {
-    OpReplacement() = default;
-    OpReplacement(Operation *op, ValueRange newValues)
-        : op(op), newValues(newValues.begin(), newValues.end()) {}
-
-    Operation *op;
-    SmallVector<Value, 2> newValues;
-  };
+/// This class represents one requested operation replacement via 'replaceOp'.
+struct OpReplacement {
+  OpReplacement() = default;
+  OpReplacement(Operation *op, ValueRange newValues)
+      : op(op), newValues(newValues.begin(), newValues.end()) {}
 
-  /// The kind of the block action performed during the rewrite.  Actions can be
-  /// undone if the conversion fails.
-  enum class BlockActionKind { Create, Erase, Move, Split, TypeConversion };
+  Operation *op;
+  SmallVector<Value, 2> newValues;
+};
 
-  /// Original position of the given block in its parent region.  We cannot use
-  /// a region iterator because it could have been invalidated by other region
-  /// operations since the position was stored.
-  struct BlockPosition {
-    Region *region;
-    Region::iterator::difference_type position;
-  };
+/// The kind of the block action performed during the rewrite.  Actions can be
+/// undone if the conversion fails.
+enum class BlockActionKind { Create, Erase, Move, Split, TypeConversion };
 
-  /// The storage class for an undoable block action (one of BlockActionKind),
-  /// contains the information necessary to undo this action.
-  struct BlockAction {
-    static BlockAction getCreate(Block *block) {
-      return {BlockActionKind::Create, block, {}};
-    }
-    static BlockAction getErase(Block *block, BlockPosition originalPos) {
-      return {BlockActionKind::Erase, block, {originalPos}};
-    }
-    static BlockAction getMove(Block *block, BlockPosition originalPos) {
-      return {BlockActionKind::Move, block, {originalPos}};
-    }
-    static BlockAction getSplit(Block *block, Block *originalBlock) {
-      BlockAction action{BlockActionKind::Split, block, {}};
-      action.originalBlock = originalBlock;
-      return action;
-    }
-    static BlockAction getTypeConversion(Block *block) {
-      return BlockAction{BlockActionKind::TypeConversion, block, {}};
-    }
+/// Original position of the given block in its parent region.  We cannot use
+/// a region iterator because it could have been invalidated by other region
+/// operations since the position was stored.
+struct BlockPosition {
+  Region *region;
+  Region::iterator::difference_type position;
+};
 
-    // The action kind.
-    BlockActionKind kind;
-
-    // A pointer to the block that was created by the action.
-    Block *block;
-
-    union {
-      // In use if kind == BlockActionKind::Move or BlockActionKind::Erase, and
-      // contains a pointer to the region that originally contained the block as
-      // well as the position of the block in that region.
-      BlockPosition originalPosition;
-      // In use if kind == BlockActionKind::Split and contains a pointer to the
-      // block that was split into two parts.
-      Block *originalBlock;
-    };
+/// The storage class for an undoable block action (one of BlockActionKind),
+/// contains the information necessary to undo this action.
+struct BlockAction {
+  static BlockAction getCreate(Block *block) {
+    return {BlockActionKind::Create, block, {}};
+  }
+  static BlockAction getErase(Block *block, BlockPosition originalPos) {
+    return {BlockActionKind::Erase, block, {originalPos}};
+  }
+  static BlockAction getMove(Block *block, BlockPosition originalPos) {
+    return {BlockActionKind::Move, block, {originalPos}};
+  }
+  static BlockAction getSplit(Block *block, Block *originalBlock) {
+    BlockAction action{BlockActionKind::Split, block, {}};
+    action.originalBlock = originalBlock;
+    return action;
+  }
+  static BlockAction getTypeConversion(Block *block) {
+    return BlockAction{BlockActionKind::TypeConversion, block, {}};
+  }
+
+  // The action kind.
+  BlockActionKind kind;
+
+  // A pointer to the block that was created by the action.
+  Block *block;
+
+  union {
+    // In use if kind == BlockActionKind::Move or BlockActionKind::Erase, and
+    // contains a pointer to the region that originally contained the block as
+    // well as the position of the block in that region.
+    BlockPosition originalPosition;
+    // In use if kind == BlockActionKind::Split and contains a pointer to the
+    // block that was split into two parts.
+    Block *originalBlock;
   };
+};
+} // end anonymous namespace
 
+//===----------------------------------------------------------------------===//
+// ConversionPatternRewriterImpl
+//===----------------------------------------------------------------------===//
+namespace mlir {
+namespace detail {
+struct ConversionPatternRewriterImpl {
   ConversionPatternRewriterImpl(PatternRewriter &rewriter)
       : argConverter(rewriter) {}
 
+  /// Cleanup and destroy any generated rewrite operations. This method is
+  /// invoked when the conversion process fails.
+  void discardRewrites();
+
+  /// Apply all requested operation rewrites. This method is invoked when the
+  /// conversion process succeeds.
+  void applyRewrites();
+
+  //===--------------------------------------------------------------------===//
+  // State Management
+  //===--------------------------------------------------------------------===//
+
   /// Return the current state of the rewriter.
   RewriterState getCurrentState();
 
@@ -597,13 +612,21 @@ struct ConversionPatternRewriterImpl {
   /// "numActionsToKeep" actions remains.
   void undoBlockActions(unsigned numActionsToKeep = 0);
 
-  /// Cleanup and destroy any generated rewrite operations. This method is
-  /// invoked when the conversion process fails.
-  void discardRewrites();
+  /// Remap the given operands to those with potentially different types.
+  void remapValues(Operation::operand_range operands,
+                   SmallVectorImpl<Value> &remapped);
 
-  /// Apply all requested operation rewrites. This method is invoked when the
-  /// conversion process succeeds.
-  void applyRewrites();
+  /// Returns true if the given operation is ignored, and does not need to be
+  /// converted.
+  bool isOpIgnored(Operation *op) const;
+
+  /// Recursively marks the nested operations under 'op' as ignored. This
+  /// removes them from being considered for legalization.
+  void markNestedOpsIgnored(Operation *op);
+
+  //===--------------------------------------------------------------------===//
+  // Type Conversion
+  //===--------------------------------------------------------------------===//
 
   /// Convert the signature of the given block.
   FailureOr<Block *> convertBlockSignature(
@@ -620,8 +643,12 @@ struct ConversionPatternRewriterImpl {
   convertRegionTypes(Region *region, TypeConverter &converter,
                      TypeConverter::SignatureConversion *entryConversion);
 
+  //===--------------------------------------------------------------------===//
+  // Rewriter Notification Hooks
+  //===--------------------------------------------------------------------===//
+
   /// PatternRewriter hook for replacing the results of an operation.
-  void replaceOp(Operation *op, ValueRange newValues);
+  void notifyOpReplaced(Operation *op, ValueRange newValues);
 
   /// Notifies that a block is about to be erased.
   void notifyBlockIsBeingErased(Block *block);
@@ -640,17 +667,9 @@ struct ConversionPatternRewriterImpl {
   void notifyRegionWasClonedBefore(iterator_range<Region::iterator> &blocks,
                                    Location origRegionLoc);
 
-  /// Remap the given operands to those with potentially different types.
-  void remapValues(Operation::operand_range operands,
-                   SmallVectorImpl<Value> &remapped);
-
-  /// Returns true if the given operation is ignored, and does not need to be
-  /// converted.
-  bool isOpIgnored(Operation *op) const;
-
-  /// Recursively marks the nested operations under 'op' as ignored. This
-  /// removes them from being considered for legalization.
-  void markNestedOpsIgnored(Operation *op);
+  //===--------------------------------------------------------------------===//
+  // State
+  //===--------------------------------------------------------------------===//
 
   // Mapping between replaced values that differ in type. This happens when
   // replacing a value with one of a different type.
@@ -700,12 +719,6 @@ struct ConversionPatternRewriterImpl {
 } // end namespace detail
 } // end namespace mlir
 
-RewriterState ConversionPatternRewriterImpl::getCurrentState() {
-  return RewriterState(createdOps.size(), replacements.size(),
-                       argReplacements.size(), blockActions.size(),
-                       ignoredOps.size(), rootUpdates.size());
-}
-
 /// Detach any operations nested in the given operation from their parent
 /// blocks, and erase the given operation. This can be used when the nested
 /// operations are scheduled for erasure themselves, so deleting the regions of
@@ -722,6 +735,73 @@ static void detachNestedAndErase(Operation *op) {
   op->erase();
 }
 
+void ConversionPatternRewriterImpl::discardRewrites() {
+  // Reset any operations that were updated in place.
+  for (auto &state : rootUpdates)
+    state.resetOperation();
+
+  undoBlockActions();
+
+  // Remove any newly created ops.
+  for (auto *op : llvm::reverse(createdOps))
+    detachNestedAndErase(op);
+}
+
+void ConversionPatternRewriterImpl::applyRewrites() {
+  // Apply all of the rewrites replacements requested during conversion.
+  for (auto &repl : replacements) {
+    for (unsigned i = 0, e = repl.newValues.size(); i != e; ++i) {
+      if (auto newValue = repl.newValues[i])
+        repl.op->getResult(i).replaceAllUsesWith(
+            mapping.lookupOrDefault(newValue));
+    }
+
+    // If this operation defines any regions, drop any pending argument
+    // rewrites.
+    if (repl.op->getNumRegions())
+      argConverter.notifyOpRemoved(repl.op);
+  }
+
+  // Apply all of the requested argument replacements.
+  for (BlockArgument arg : argReplacements) {
+    Value repl = mapping.lookupOrDefault(arg);
+    if (repl.isa<BlockArgument>()) {
+      arg.replaceAllUsesWith(repl);
+      continue;
+    }
+
+    // If the replacement value is an operation, we check to make sure that we
+    // don't replace uses that are within the parent operation of the
+    // replacement value.
+    Operation *replOp = repl.cast<OpResult>().getOwner();
+    Block *replBlock = replOp->getBlock();
+    arg.replaceUsesWithIf(repl, [&](OpOperand &operand) {
+      Operation *user = operand.getOwner();
+      return user->getBlock() != replBlock || replOp->isBeforeInBlock(user);
+    });
+  }
+
+  // In a second pass, erase all of the replaced operations in reverse. This
+  // allows processing nested operations before their parent region is
+  // destroyed.
+  for (auto &repl : llvm::reverse(replacements))
+    repl.op->erase();
+
+  argConverter.applyRewrites(mapping);
+
+  // Now that the ops have been erased, also erase dangling blocks.
+  eraseDanglingBlocks();
+}
+
+//===----------------------------------------------------------------------===//
+// State Management
+
+RewriterState ConversionPatternRewriterImpl::getCurrentState() {
+  return RewriterState(createdOps.size(), replacements.size(),
+                       argReplacements.size(), blockActions.size(),
+                       ignoredOps.size(), rootUpdates.size());
+}
+
 void ConversionPatternRewriterImpl::resetState(RewriterState state) {
   // Reset any operations that were updated in place.
   for (unsigned i = state.numRootUpdates, e = rootUpdates.size(); i != e; ++i)
@@ -810,64 +890,34 @@ void ConversionPatternRewriterImpl::undoBlockActions(
   blockActions.resize(numActionsToKeep);
 }
 
-void ConversionPatternRewriterImpl::discardRewrites() {
-  // Reset any operations that were updated in place.
-  for (auto &state : rootUpdates)
-    state.resetOperation();
-
-  undoBlockActions();
-
-  // Remove any newly created ops.
-  for (auto *op : llvm::reverse(createdOps))
-    detachNestedAndErase(op);
+void ConversionPatternRewriterImpl::remapValues(
+    Operation::operand_range operands, SmallVectorImpl<Value> &remapped) {
+  remapped.reserve(llvm::size(operands));
+  for (Value operand : operands)
+    remapped.push_back(mapping.lookupOrDefault(operand));
 }
 
-void ConversionPatternRewriterImpl::applyRewrites() {
-  // Apply all of the rewrites replacements requested during conversion.
-  for (auto &repl : replacements) {
-    for (unsigned i = 0, e = repl.newValues.size(); i != e; ++i) {
-      if (auto newValue = repl.newValues[i])
-        repl.op->getResult(i).replaceAllUsesWith(
-            mapping.lookupOrDefault(newValue));
-    }
-
-    // If this operation defines any regions, drop any pending argument
-    // rewrites.
-    if (repl.op->getNumRegions())
-      argConverter.notifyOpRemoved(repl.op);
-  }
-
-  // Apply all of the requested argument replacements.
-  for (BlockArgument arg : argReplacements) {
-    Value repl = mapping.lookupOrDefault(arg);
-    if (repl.isa<BlockArgument>()) {
-      arg.replaceAllUsesWith(repl);
-      continue;
-    }
-
-    // If the replacement value is an operation, we check to make sure that we
-    // don't replace uses that are within the parent operation of the
-    // replacement value.
-    Operation *replOp = repl.cast<OpResult>().getOwner();
-    Block *replBlock = replOp->getBlock();
-    arg.replaceUsesWithIf(repl, [&](OpOperand &operand) {
-      Operation *user = operand.getOwner();
-      return user->getBlock() != replBlock || replOp->isBeforeInBlock(user);
-    });
-  }
-
-  // In a second pass, erase all of the replaced operations in reverse. This
-  // allows processing nested operations before their parent region is
-  // destroyed.
-  for (auto &repl : llvm::reverse(replacements))
-    repl.op->erase();
-
-  argConverter.applyRewrites(mapping);
+bool ConversionPatternRewriterImpl::isOpIgnored(Operation *op) const {
+  // Check to see if this operation or its parent were ignored.
+  return ignoredOps.count(op) || ignoredOps.count(op->getParentOp());
+}
 
-  // Now that the ops have been erased, also erase dangling blocks.
-  eraseDanglingBlocks();
+void ConversionPatternRewriterImpl::markNestedOpsIgnored(Operation *op) {
+  // Walk this operation and collect nested operations that define non-empty
+  // regions. We mark such operations as 'ignored' so that we know we don't have
+  // to convert them, or their nested ops.
+  if (op->getNumRegions() == 0)
+    return;
+  op->walk([&](Operation *op) {
+    if (llvm::any_of(op->getRegions(),
+                     [](Region &region) { return !region.empty(); }))
+      ignoredOps.insert(op);
+  });
 }
 
+//===----------------------------------------------------------------------===//
+// Type Conversion
+
 FailureOr<Block *> ConversionPatternRewriterImpl::convertBlockSignature(
     Block *block, TypeConverter &converter,
     TypeConverter::SignatureConversion *conversion) {
@@ -907,8 +957,11 @@ FailureOr<Block *> ConversionPatternRewriterImpl::convertRegionTypes(
   return newEntry;
 }
 
-void ConversionPatternRewriterImpl::replaceOp(Operation *op,
-                                              ValueRange newValues) {
+//===----------------------------------------------------------------------===//
+// Rewriter Notification Hooks
+
+void ConversionPatternRewriterImpl::notifyOpReplaced(Operation *op,
+                                                     ValueRange newValues) {
   assert(newValues.size() == op->getNumResults());
 
   // Create mappings for each of the new result values.
@@ -962,31 +1015,6 @@ void ConversionPatternRewriterImpl::notifyRegionWasClonedBefore(
   assert(succeeded(result) && "expected region to have no unreachable blocks");
 }
 
-void ConversionPatternRewriterImpl::remapValues(
-    Operation::operand_range operands, SmallVectorImpl<Value> &remapped) {
-  remapped.reserve(llvm::size(operands));
-  for (Value operand : operands)
-    remapped.push_back(mapping.lookupOrDefault(operand));
-}
-
-bool ConversionPatternRewriterImpl::isOpIgnored(Operation *op) const {
-  // Check to see if this operation or its parent were ignored.
-  return ignoredOps.count(op) || ignoredOps.count(op->getParentOp());
-}
-
-void ConversionPatternRewriterImpl::markNestedOpsIgnored(Operation *op) {
-  // Walk this operation and collect nested operations that define non-empty
-  // regions. We mark such operations as 'ignored' so that we know we don't have
-  // to convert them, or their nested ops.
-  if (op->getNumRegions() == 0)
-    return;
-  op->walk([&](Operation *op) {
-    if (llvm::any_of(op->getRegions(),
-                     [](Region &region) { return !region.empty(); }))
-      ignoredOps.insert(op);
-  });
-}
-
 //===----------------------------------------------------------------------===//
 // ConversionPatternRewriter
 //===----------------------------------------------------------------------===//
@@ -1002,7 +1030,7 @@ void ConversionPatternRewriter::replaceOp(Operation *op, ValueRange newValues) {
     impl->logger.startLine()
         << "** Replace : '" << op->getName() << "'(" << op << ")\n";
   });
-  impl->replaceOp(op, newValues);
+  impl->notifyOpReplaced(op, newValues);
 }
 
 /// PatternRewriter hook for erasing a dead operation. The uses of this
@@ -1014,7 +1042,7 @@ void ConversionPatternRewriter::eraseOp(Operation *op) {
         << "** Erase   : '" << op->getName() << "'(" << op << ")\n";
   });
   SmallVector<Value, 1> nullRepls(op->getNumResults(), nullptr);
-  impl->replaceOp(op, nullRepls);
+  impl->notifyOpReplaced(op, nullRepls);
 }
 
 void ConversionPatternRewriter::eraseBlock(Block *block) {
@@ -1076,7 +1104,7 @@ Block *ConversionPatternRewriter::splitBlock(Block *block,
 /// PatternRewriter hook for merging a block into another.
 void ConversionPatternRewriter::mergeBlocks(Block *source, Block *dest,
                                             ValueRange argValues) {
-  // TODO(riverriddle) This requires fixing the implementation of
+  // TODO: This requires fixing the implementation of
   // 'replaceUsesOfBlockArgument', which currently isn't undoable.
   llvm_unreachable("block merging updates are currently not supported");
 }
@@ -1160,7 +1188,7 @@ detail::ConversionPatternRewriterImpl &ConversionPatternRewriter::getImpl() {
 }
 
 //===----------------------------------------------------------------------===//
-// Conversion Patterns
+// ConversionPattern
 //===----------------------------------------------------------------------===//
 
 /// Attempt to match and rewrite the IR root at the specified operation.
@@ -1234,6 +1262,10 @@ class OperationLegalizer {
                                            RewriterState &state,
                                            RewriterState &newState);
 
+  //===--------------------------------------------------------------------===//
+  // Cost Model
+  //===--------------------------------------------------------------------===//
+
   /// Build an optimistic legalization graph given the provided patterns. This
   /// function populates 'anyOpLegalizerPatterns' and 'legalizerPatterns' with
   /// patterns for operations that are not directly legal, but may be
@@ -1349,7 +1381,7 @@ OperationLegalizer::legalize(Operation *op,
   }
 
   // If the operation isn't legal, try to fold it in-place.
-  // TODO(riverriddle) Should we always try to do this, even if the op is
+  // TODO: Should we always try to do this, even if the op is
   // already legal?
   if (succeeded(legalizeWithFold(op, rewriter))) {
     LLVM_DEBUG({
@@ -1528,9 +1560,8 @@ LogicalResult OperationLegalizer::legalizePatternBlockActions(
   for (int i = state.numBlockActions, e = newState.numBlockActions; i != e;
        ++i) {
     auto &action = impl.blockActions[i];
-    if (action.kind ==
-            ConversionPatternRewriterImpl::BlockActionKind::TypeConversion ||
-        action.kind == ConversionPatternRewriterImpl::BlockActionKind::Erase)
+    if (action.kind == BlockActionKind::TypeConversion ||
+        action.kind == BlockActionKind::Erase)
       continue;
     // Only check blocks outside of the current operation.
     Operation *parentOp = action.block->getParentOp();
@@ -1599,6 +1630,9 @@ LogicalResult OperationLegalizer::legalizePatternRootUpdates(
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// Cost Model
+
 void OperationLegalizer::buildLegalizationGraph(
     LegalizationPatterns &anyOpLegalizerPatterns,
     DenseMap<OperationName, LegalizationPatterns> &legalizerPatterns) {
diff --git a/mlir/lib/Transforms/Inliner.cpp b/mlir/lib/Transforms/Inliner.cpp
index e17a379d54b89..2ddb10a3a0882 100644
--- a/mlir/lib/Transforms/Inliner.cpp
+++ b/mlir/lib/Transforms/Inliner.cpp
@@ -321,7 +321,7 @@ static void collectCallOps(iterator_range<Region::iterator> blocks,
 
     for (Operation &op : *block) {
       if (auto call = dyn_cast<CallOpInterface>(op)) {
-        // TODO(riverriddle) Support inlining nested call references.
+        // TODO: Support inlining nested call references.
         CallInterfaceCallable callable = call.getCallableForCallee();
         if (SymbolRefAttr symRef = callable.dyn_cast<SymbolRefAttr>()) {
           if (!symRef.isa<FlatSymbolRefAttr>())
diff --git a/mlir/lib/Transforms/LoopFusion.cpp b/mlir/lib/Transforms/LoopFusion.cpp
index 0ae6657babccb..c2f30fa3d1896 100644
--- a/mlir/lib/Transforms/LoopFusion.cpp
+++ b/mlir/lib/Transforms/LoopFusion.cpp
@@ -42,9 +42,9 @@ namespace {
 /// which fuses loop nests with single-writer/single-reader memref dependences
 /// with the goal of improving locality.
 
-// TODO(andydavis) Support fusion of source loop nests which write to multiple
+// TODO: Support fusion of source loop nests which write to multiple
 // memrefs, where each memref can have multiple users (if profitable).
-// TODO(andydavis) Extend this pass to check for fusion preventing dependences,
+// TODO: Extend this pass to check for fusion preventing dependences,
 // and add support for more general loop fusion algorithms.
 
 struct LoopFusion : public AffineLoopFusionBase<LoopFusion> {
@@ -68,12 +68,10 @@ mlir::createLoopFusionPass(unsigned fastMemorySpace,
                                       maximalFusion);
 }
 
-// TODO(b/117228571) Replace when this is modeled through side-effects/op traits
+// TODO: Replace when this is modeled through side-effects/op traits
 static bool isMemRefDereferencingOp(Operation &op) {
-  if (isa<AffineReadOpInterface>(op) || isa<AffineWriteOpInterface>(op) ||
-      isa<AffineDmaStartOp>(op) || isa<AffineDmaWaitOp>(op))
-    return true;
-  return false;
+  return isa<AffineReadOpInterface, AffineWriteOpInterface, AffineDmaStartOp,
+             AffineDmaWaitOp>(op);
 }
 
 namespace {
@@ -103,8 +101,8 @@ struct LoopNestStateCollector {
 // MemRefDependenceGraph is a graph data structure where graph nodes are
 // top-level operations in a FuncOp which contain load/store ops, and edges
 // are memref dependences between the nodes.
-// TODO(andydavis) Add a more flexible dependence graph representation.
-// TODO(andydavis) Add a depth parameter to dependence graph construction.
+// TODO: Add a more flexible dependence graph representation.
+// TODO: Add a depth parameter to dependence graph construction.
 struct MemRefDependenceGraph {
 public:
   // Node represents a node in the graph. A Node is either an entire loop nest
@@ -630,7 +628,7 @@ struct MemRefDependenceGraph {
 
 // Initializes the data dependence graph by walking operations in 'f'.
 // Assigns each node in the graph a node id based on program order in 'f'.
-// TODO(andydavis) Add support for taking a Block arg to construct the
+// TODO: Add support for taking a Block arg to construct the
 // dependence graph at a different depth.
 bool MemRefDependenceGraph::init(FuncOp f) {
   DenseMap<Value, SetVector<unsigned>> memrefAccesses;
@@ -798,7 +796,7 @@ static unsigned getMaxLoopDepth(ArrayRef<Operation *> loadOpInsts,
           getNumCommonSurroundingLoops(*srcOpInst, *dstOpInst);
       for (unsigned d = 1; d <= numCommonLoops + 1; ++d) {
         FlatAffineConstraints dependenceConstraints;
-        // TODO(andydavis) Cache dependence analysis results, check cache here.
+        // TODO: Cache dependence analysis results, check cache here.
         DependenceResult result = checkMemrefAccessDependence(
             srcAccess, dstAccess, d, &dependenceConstraints,
             /*dependenceComponents=*/nullptr);
@@ -825,7 +823,7 @@ static void sinkSequentialLoops(MemRefDependenceGraph::Node *node) {
   node->op = newRootForOp.getOperation();
 }
 
-//  TODO(mlir-team): improve/complete this when we have target data.
+//  TODO: improve/complete this when we have target data.
 static unsigned getMemRefEltSizeInBytes(MemRefType memRefType) {
   auto elementType = memRefType.getElementType();
 
@@ -843,7 +841,7 @@ static unsigned getMemRefEltSizeInBytes(MemRefType memRefType) {
 // Creates and returns a private (single-user) memref for fused loop rooted
 // at 'forOp', with (potentially reduced) memref size based on the
 // MemRefRegion written to by 'srcStoreOpInst' at depth 'dstLoopDepth'.
-// TODO(bondhugula): consider refactoring the common code from generateDma and
+// TODO: consider refactoring the common code from generateDma and
 // this one.
 static Value createPrivateMemRef(AffineForOp forOp, Operation *srcStoreOpInst,
                                  unsigned dstLoopDepth,
@@ -911,22 +909,14 @@ static Value createPrivateMemRef(AffineForOp forOp, Operation *srcStoreOpInst,
   }
   auto newMemRefType = MemRefType::get(newShape, oldMemRefType.getElementType(),
                                        {}, newMemSpace);
-  // Gather alloc operands for the dynamic dimensions of the memref.
-  SmallVector<Value, 4> allocOperands;
-  unsigned dynamicDimCount = 0;
-  for (auto dimSize : oldMemRefType.getShape()) {
-    if (dimSize == -1)
-      allocOperands.push_back(
-          top.create<DimOp>(forOp.getLoc(), oldMemRef, dynamicDimCount++));
-  }
 
-  // Create new private memref for fused loop 'forOp'.
-  // TODO(andydavis) Create/move alloc ops for private memrefs closer to their
+  // Create new private memref for fused loop 'forOp'. 'newShape' is always
+  // a constant shape.
+  // TODO: Create/move alloc ops for private memrefs closer to their
   // consumer loop nests to reduce their live range. Currently they are added
   // at the beginning of the function, because loop nests can be reordered
   // during the fusion pass.
-  Value newMemRef =
-      top.create<AllocOp>(forOp.getLoc(), newMemRefType, allocOperands);
+  Value newMemRef = top.create<AllocOp>(forOp.getLoc(), newMemRefType);
 
   // Build an AffineMap to remap access functions based on lower bound offsets.
   SmallVector<AffineExpr, 4> remapExprs;
@@ -958,12 +948,71 @@ static Value createPrivateMemRef(AffineForOp forOp, Operation *srcStoreOpInst,
   return newMemRef;
 }
 
+/// Walking from node 'srcId' to node 'dstId' (exclusive of 'srcId' and
+/// 'dstId'), if there is any non-affine operation accessing 'memref', return
+/// false. Otherwise, return true.
+static bool hasNonAffineUsersOnThePath(unsigned srcId, unsigned dstId,
+                                       Value memref,
+                                       MemRefDependenceGraph *mdg) {
+  auto *srcNode = mdg->getNode(srcId);
+  auto *dstNode = mdg->getNode(dstId);
+  Value::user_range users = memref.getUsers();
+  // For each MemRefDependenceGraph's node that is between 'srcNode' and
+  // 'dstNode' (exclusive of 'srcNodes' and 'dstNode'), check whether any
+  // non-affine operation in the node accesses the 'memref'.
+  for (auto &idAndNode : mdg->nodes) {
+    Operation *op = idAndNode.second.op;
+    // Take care of operations between 'srcNode' and 'dstNode'.
+    if (srcNode->op->isBeforeInBlock(op) && op->isBeforeInBlock(dstNode->op)) {
+      // Walk inside the operation to find any use of the memref.
+      // Interrupt the walk if found.
+      auto walkResult = op->walk([&](Operation *user) {
+        // Skip affine ops.
+        if (isMemRefDereferencingOp(*user))
+          return WalkResult::advance();
+        // Find a non-affine op that uses the memref.
+        if (llvm::is_contained(users, user))
+          return WalkResult::interrupt();
+        return WalkResult::advance();
+      });
+      if (walkResult.wasInterrupted())
+        return true;
+    }
+  }
+  return false;
+}
+
+/// Check whether a memref value in node 'srcId' has a non-affine that
+/// is between node 'srcId' and node 'dstId' (exclusive of 'srcNode' and
+/// 'dstNode').
+static bool hasNonAffineUsersOnThePath(unsigned srcId, unsigned dstId,
+                                       MemRefDependenceGraph *mdg) {
+  // Collect memref values in node 'srcId'.
+  auto *srcNode = mdg->getNode(srcId);
+  llvm::SmallDenseSet<Value, 2> memRefValues;
+  srcNode->op->walk([&](Operation *op) {
+    // Skip affine ops.
+    if (isa<AffineForOp>(op))
+      return WalkResult::advance();
+    for (Value v : op->getOperands())
+      // Collect memref values only.
+      if (v.getType().isa<MemRefType>())
+        memRefValues.insert(v);
+    return WalkResult::advance();
+  });
+  // Looking for users between node 'srcId' and node 'dstId'.
+  for (Value memref : memRefValues)
+    if (hasNonAffineUsersOnThePath(srcId, dstId, memref, mdg))
+      return true;
+  return false;
+}
+
 // Checks if node 'srcId' can be safely fused into node 'dstId'. Node 'srcId'
 // may write to multiple memrefs but it is required that only one of them,
 // 'srcLiveOutStoreOp', has output edges.
 // Returns true if 'dstNode's read/write region to 'memref' is a super set of
 // 'srcNode's write region to 'memref' and 'srcId' has only one output edge.
-// TODO(andydavis) Generalize this to handle more live in/out cases.
+// TODO: Generalize this to handle more live in/out cases.
 static bool
 canFuseSrcWhichWritesToLiveOut(unsigned srcId, unsigned dstId,
                                AffineWriteOpInterface srcLiveOutStoreOp,
@@ -991,7 +1040,7 @@ canFuseSrcWhichWritesToLiveOut(unsigned srcId, unsigned dstId,
     return false;
 
   // Compute MemRefRegion 'dstRegion' for 'dstStore/LoadOpInst' on 'memref'.
-  // TODO(andydavis) Compute 'unionboundingbox' of all write regions (one for
+  // TODO: Compute 'unionboundingbox' of all write regions (one for
   // each store op in 'dstStoreOps').
   SmallVector<Operation *, 2> dstStoreOps;
   dstNode->getStoreOpsForMemref(memref, &dstStoreOps);
@@ -1015,9 +1064,15 @@ canFuseSrcWhichWritesToLiveOut(unsigned srcId, unsigned dstId,
 
   // Return false if write region is not a superset of 'srcNodes' write
   // region to 'memref'.
-  // TODO(andydavis) Check the shape and lower bounds here too.
+  // TODO: Check the shape and lower bounds here too.
   if (srcNumElements != dstNumElements)
     return false;
+
+  // Return false if 'memref' is used by a non-affine operation that is
+  // between node 'srcId' and node 'dstId'.
+  if (hasNonAffineUsersOnThePath(srcId, dstId, mdg))
+    return false;
+
   return true;
 }
 
@@ -1189,7 +1244,7 @@ static bool isFusionProfitable(Operation *srcOpInst, Operation *srcStoreOpInst,
         maybeSliceWriteRegionSizeBytes.getValue();
 
     // If we are fusing for reuse, check that write regions remain the same.
-    // TODO(andydavis) Write region check should check sizes and offsets in
+    // TODO: Write region check should check sizes and offsets in
     // each dimension, so that we are sure they are covering the same memref
     // region. Also, move this out to a isMemRefRegionSuperSet helper function.
     if (srcOpInst != srcStoreOpInst &&
@@ -1213,7 +1268,7 @@ static bool isFusionProfitable(Operation *srcOpInst, Operation *srcStoreOpInst,
       llvm::dbgs() << msg.str();
     });
 
-    // TODO(b/123247369): This is a placeholder cost model.
+    // TODO: This is a placeholder cost model.
     // Among all choices that add an acceptable amount of redundant computation
     // (as per computeToleranceThreshold), we will simply pick the one that
     // reduces the intermediary size the most.
@@ -1369,9 +1424,10 @@ namespace {
 // takes O(V) time for initialization, and has runtime O(V + E).
 //
 // This greedy algorithm is not 'maximal' due to the current restriction of
-// fusing along single producer consumer edges, but there is a TODO to fix this.
+// fusing along single producer consumer edges, but there is a TODO: to fix
+// this.
 //
-// TODO(andydavis) Experiment with other fusion policies.
+// TODO: Experiment with other fusion policies.
 struct GreedyFusion {
 public:
   // The data dependence graph to traverse during fusion.
@@ -1402,7 +1458,7 @@ struct GreedyFusion {
 
   // Initializes 'worklist' with nodes from 'mdg'
   void init() {
-    // TODO(andydavis) Add a priority queue for prioritizing nodes by different
+    // TODO: Add a priority queue for prioritizing nodes by different
     // metrics (e.g. arithmetic intensity/flops-to-bytes ratio).
     worklist.clear();
     worklistSet.clear();
@@ -1419,7 +1475,7 @@ struct GreedyFusion {
   // *) Second pass fuses sibling nodes which share no dependence edges.
   // *) Third pass fuses any remaining producer nodes into their users.
   void run() {
-    // TODO(andydavis) Run this repeatedly until a fixed-point is reached.
+    // TODO: Run this repeatedly until a fixed-point is reached.
     fuseProducerConsumerNodes(/*maxSrcUserCount=*/1);
     fuseSiblingNodes();
     fuseProducerConsumerNodes(
@@ -1482,7 +1538,7 @@ struct GreedyFusion {
             continue;
           // Skip if 'srcNode' has more than one live-out store to a
           // function-local memref.
-          // TODO(andydavis) Support more generic multi-output src loop nests
+          // TODO: Support more generic multi-output src loop nests
           // fusion.
           auto srcStoreOp = mdg->getUniqueOutgoingStore(srcNode);
           if (!srcStoreOp) {
@@ -1547,7 +1603,7 @@ struct GreedyFusion {
           unsigned dstLoopDepthTest = getInnermostCommonLoopDepth(dstOps);
           // Check the feasibility of fusing src loop nest into dst loop nest
           // at loop depths in range [1, dstLoopDepthTest].
-          // TODO(andydavis) Use slice union computation and union of memref
+          // TODO: Use slice union computation and union of memref
           // read/write regions to cost model and fusion.
           bool canFuse = false;
           for (unsigned i = 1; i <= dstLoopDepthTest; ++i) {
@@ -1608,7 +1664,7 @@ struct GreedyFusion {
                     memref)
                   storesForMemref.push_back(storeOpInst);
               }
-              // TODO(andydavis) Use union of memref write regions to compute
+              // TODO: Use union of memref write regions to compute
               // private memref footprint.
               auto newMemRef = createPrivateMemRef(
                   dstAffineForOp, storesForMemref[0], bestDstLoopDepth,
@@ -1710,7 +1766,7 @@ struct GreedyFusion {
     while (findSiblingNodeToFuse(dstNode, &visitedSibNodeIds, &idAndMemref)) {
       unsigned sibId = idAndMemref.first;
       Value memref = idAndMemref.second;
-      // TODO(andydavis) Check that 'sibStoreOpInst' post-dominates all other
+      // TODO: Check that 'sibStoreOpInst' post-dominates all other
       // stores to the same memref in 'sibNode' loop nest.
       auto *sibNode = mdg->getNode(sibId);
       // Compute an operation list insertion point for the fused loop
@@ -1732,7 +1788,7 @@ struct GreedyFusion {
       assert(sibLoadOpInsts.size() == 1);
       Operation *sibLoadOpInst = sibLoadOpInsts[0];
       assert(!sibNode->stores.empty());
-      // TODO(andydavis) Choose the store which postdominates all other stores.
+      // TODO: Choose the store which postdominates all other stores.
       auto *sibStoreOpInst = sibNode->stores.back();
 
       // Gather 'dstNode' load ops to 'memref'.
@@ -1778,7 +1834,7 @@ struct GreedyFusion {
     // on 'memref'.
     auto canFuseWithSibNode = [&](Node *sibNode, Value memref) {
       // Skip if 'outEdge' is not a read-after-write dependence.
-      // TODO(andydavis) Remove restrict to single load op restriction.
+      // TODO: Remove restrict to single load op restriction.
       if (sibNode->getLoadOpCount(memref) != 1)
         return false;
       // Skip if there exists a path of dependent edges between
@@ -1803,6 +1859,12 @@ struct GreedyFusion {
       }
       if (storeMemrefs.size() != 1)
         return false;
+
+      // Skip if a memref value in one node is used by a non-affine memref
+      // access that lies between 'dstNode' and 'sibNode'.
+      if (hasNonAffineUsersOnThePath(dstNode->id, sibNode->id, mdg) ||
+          hasNonAffineUsersOnThePath(sibNode->id, dstNode->id, mdg))
+        return false;
       return true;
     };
 
diff --git a/mlir/lib/Transforms/LoopInvariantCodeMotion.cpp b/mlir/lib/Transforms/LoopInvariantCodeMotion.cpp
index 9386653c45ce4..18c82578acb12 100644
--- a/mlir/lib/Transforms/LoopInvariantCodeMotion.cpp
+++ b/mlir/lib/Transforms/LoopInvariantCodeMotion.cpp
@@ -74,6 +74,7 @@ static bool canBeHoisted(Operation *op,
   return true;
 }
 
+
 LogicalResult mlir::moveLoopInvariantCode(LoopLikeOpInterface looplike) {
   auto &loopBody = looplike.getLoopBody();
 
@@ -104,7 +105,7 @@ LogicalResult mlir::moveLoopInvariantCode(LoopLikeOpInterface looplike) {
   // For all instructions that we found to be invariant, move outside of the
   // loop.
   auto result = looplike.moveOutOfLoop(opsToMove);
-  LLVM_DEBUG(looplike.print(llvm::dbgs() << "Modified loop\n"));
+  LLVM_DEBUG(looplike.print(llvm::dbgs() << "\n\nModified loop:\n"));
   return result;
 }
 
@@ -113,7 +114,7 @@ void LoopInvariantCodeMotion::runOnOperation() {
   // way, we first LICM from the inner loop, and place the ops in
   // the outer loop, which in turn can be further LICM'ed.
   getOperation()->walk([&](LoopLikeOpInterface loopLike) {
-    LLVM_DEBUG(loopLike.print(llvm::dbgs() << "\nOriginal loop\n"));
+    LLVM_DEBUG(loopLike.print(llvm::dbgs() << "\nOriginal loop:\n"));
     if (failed(moveLoopInvariantCode(loopLike)))
       signalPassFailure();
   });
diff --git a/mlir/lib/Transforms/MemRefDataFlowOpt.cpp b/mlir/lib/Transforms/MemRefDataFlowOpt.cpp
index 2b288d392cd7e..7220fd1f3dc6a 100644
--- a/mlir/lib/Transforms/MemRefDataFlowOpt.cpp
+++ b/mlir/lib/Transforms/MemRefDataFlowOpt.cpp
@@ -8,7 +8,7 @@
 //
 // This file implements a pass to forward memref stores to loads, thereby
 // potentially getting rid of intermediate memref's entirely.
-// TODO(mlir-team): In the future, similar techniques could be used to eliminate
+// TODO: In the future, similar techniques could be used to eliminate
 // dead memref store's and perform more complex forwarding when support for
 // SSA scalars live out of 'affine.for'/'affine.if' statements is available.
 //===----------------------------------------------------------------------===//
@@ -54,9 +54,9 @@ namespace {
 // don't reason about loops that are guaranteed to execute at least once or
 // multiple sources to forward from.
 //
-// TODO(mlir-team): more forwarding can be done when support for
+// TODO: more forwarding can be done when support for
 // loop/conditional live-out SSA values is available.
-// TODO(mlir-team): do general dead store elimination for memref's. This pass
+// TODO: do general dead store elimination for memref's. This pass
 // currently only eliminates the stores only if no other loads/uses (other
 // than dealloc) remain.
 //
@@ -203,11 +203,11 @@ void MemRefDataFlowOpt::runOnFunction() {
     // If the memref hasn't been alloc'ed in this function, skip.
     Operation *defOp = memref.getDefiningOp();
     if (!defOp || !isa<AllocOp>(defOp))
-      // TODO(mlir-team): if the memref was returned by a 'call' operation, we
+      // TODO: if the memref was returned by a 'call' operation, we
       // could still erase it if the call had no side-effects.
       continue;
     if (llvm::any_of(memref.getUsers(), [&](Operation *ownerOp) {
-          return (!isa<AffineStoreOp>(ownerOp) && !isa<DeallocOp>(ownerOp));
+          return !isa<AffineStoreOp, DeallocOp>(ownerOp);
         }))
       continue;
 
diff --git a/mlir/lib/Transforms/PipelineDataTransfer.cpp b/mlir/lib/Transforms/PipelineDataTransfer.cpp
index d4a5ba97d6bcc..564193e22690f 100644
--- a/mlir/lib/Transforms/PipelineDataTransfer.cpp
+++ b/mlir/lib/Transforms/PipelineDataTransfer.cpp
@@ -46,9 +46,9 @@ std::unique_ptr<OperationPass<FuncOp>> mlir::createPipelineDataTransferPass() {
 
 // Returns the position of the tag memref operand given a DMA operation.
 // Temporary utility: will be replaced when DmaStart/DmaFinish abstract op's are
-// added.  TODO(b/117228571)
+// added.  TODO
 static unsigned getTagMemRefPos(Operation &dmaOp) {
-  assert(isa<AffineDmaStartOp>(dmaOp) || isa<AffineDmaWaitOp>(dmaOp));
+  assert((isa<AffineDmaStartOp, AffineDmaWaitOp>(dmaOp)));
   if (auto dmaStartOp = dyn_cast<AffineDmaStartOp>(dmaOp)) {
     return dmaStartOp.getTagMemRefOperandIndex();
   }
@@ -149,7 +149,7 @@ static bool checkTagMatch(AffineDmaStartOp startOp, AffineDmaWaitOp waitOp) {
             e = startIndices.end();
        it != e; ++it, ++wIt) {
     // Keep it simple for now, just checking if indices match.
-    // TODO(mlir-team): this would in general need to check if there is no
+    // TODO: this would in general need to check if there is no
     // intervening write writing to the same tag location, i.e., memory last
     // write/data flow analysis. This is however sufficient/powerful enough for
     // now since the DMA generation pass or the input for it will always have
@@ -185,12 +185,12 @@ static void findMatchingStartFinishInsts(
       continue;
 
     // Only DMAs incoming into higher memory spaces are pipelined for now.
-    // TODO(bondhugula): handle outgoing DMA pipelining.
+    // TODO: handle outgoing DMA pipelining.
     if (!dmaStartOp.isDestMemorySpaceFaster())
       continue;
 
     // Check for dependence with outgoing DMAs. Doing this conservatively.
-    // TODO(andydavis,bondhugula): use the dependence analysis to check for
+    // TODO: use the dependence analysis to check for
     // dependences between an incoming and outgoing DMA in the same iteration.
     auto it = outgoingDmaOps.begin();
     for (; it != outgoingDmaOps.end(); ++it) {
@@ -252,8 +252,8 @@ void PipelineDataTransfer::runOnAffineForOp(AffineForOp forOp) {
   // Identify memref's to replace by scanning through all DMA start
   // operations. A DMA start operation has two memref's - the one from the
   // higher level of memory hierarchy is the one to double buffer.
-  // TODO(bondhugula): check whether double-buffering is even necessary.
-  // TODO(bondhugula): make this work with different layouts: assuming here that
+  // TODO: check whether double-buffering is even necessary.
+  // TODO: make this work with different layouts: assuming here that
   // the dimension we are adding here for the double buffering is the outermost
   // dimension.
   for (auto &pair : startWaitPairs) {
diff --git a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
index ea420733e5ffd..14a4c74172493 100644
--- a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
+++ b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
@@ -110,7 +110,7 @@ class GreedyPatternRewriteDriver : public PatternRewriter {
     for (Value operand : operands) {
       // If the use count of this operand is now < 2, we re-add the defining
       // operation to the worklist.
-      // TODO(riverriddle) This is based on the fact that zero use operations
+      // TODO: This is based on the fact that zero use operations
       // may be deleted, and that single use values often have more
       // canonicalization opportunities.
       if (!operand.use_empty() && !operand.hasOneUse())
diff --git a/mlir/lib/Transforms/Utils/LoopFusionUtils.cpp b/mlir/lib/Transforms/Utils/LoopFusionUtils.cpp
index c40e3e26d8731..17dbf8eb166d3 100644
--- a/mlir/lib/Transforms/Utils/LoopFusionUtils.cpp
+++ b/mlir/lib/Transforms/Utils/LoopFusionUtils.cpp
@@ -105,7 +105,7 @@ static Operation *getLastDependentOpInRange(Operation *opA, Operation *opB) {
        it != Block::reverse_iterator(opA); ++it) {
     Operation *opX = &(*it);
     opX->walk([&](Operation *op) {
-      if (isa<AffineReadOpInterface>(op) || isa<AffineWriteOpInterface>(op)) {
+      if (isa<AffineReadOpInterface, AffineWriteOpInterface>(op)) {
         if (isDependentLoadOrStoreOp(op, values)) {
           lastDepOp = opX;
           return WalkResult::interrupt();
@@ -164,7 +164,7 @@ static Operation *getFusedLoopNestInsertionPoint(AffineForOp srcForOp,
         return nullptr;
     }
     // Return insertion point in valid range closest to 'opB'.
-    // TODO(andydavis) Consider other insertion points in valid range.
+    // TODO: Consider other insertion points in valid range.
     return firstDepOpA;
   }
   // No dependences from 'opA' to operation in range ('opA', 'opB'), return
@@ -179,7 +179,7 @@ gatherLoadsAndStores(AffineForOp forOp,
                      SmallVectorImpl<Operation *> &loadAndStoreOps) {
   bool hasIfOp = false;
   forOp.walk([&](Operation *op) {
-    if (isa<AffineReadOpInterface>(op) || isa<AffineWriteOpInterface>(op))
+    if (isa<AffineReadOpInterface, AffineWriteOpInterface>(op))
       loadAndStoreOps.push_back(op);
     else if (isa<AffineIfOp>(op))
       hasIfOp = true;
@@ -187,7 +187,7 @@ gatherLoadsAndStores(AffineForOp forOp,
   return !hasIfOp;
 }
 
-// TODO(andydavis) Prevent fusion of loop nests with side-effecting operations.
+// TODO: Prevent fusion of loop nests with side-effecting operations.
 FusionResult mlir::canFuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
                                 unsigned dstLoopDepth,
                                 ComputationSliceState *srcSlice) {
@@ -294,7 +294,7 @@ bool mlir::getLoopNestStats(AffineForOp forOpRoot, LoopNestStats *stats) {
     unsigned count = 0;
     stats->opCountMap[childForOp] = 0;
     for (auto &op : *forOp.getBody()) {
-      if (!isa<AffineForOp>(op) && !isa<AffineIfOp>(op))
+      if (!isa<AffineForOp, AffineIfOp>(op))
         ++count;
     }
     stats->opCountMap[childForOp] = count;
@@ -361,7 +361,7 @@ static int64_t getComputeCostHelper(
   return tripCount * opCount;
 }
 
-// TODO(andydavis,b/126426796): extend this to handle multiple result maps.
+// TODO: extend this to handle multiple result maps.
 static Optional<uint64_t> getConstDifference(AffineMap lbMap, AffineMap ubMap) {
   assert(lbMap.getNumResults() == 1 && "expected single result bound map");
   assert(ubMap.getNumResults() == 1 && "expected single result bound map");
@@ -391,7 +391,7 @@ static uint64_t getSliceIterationCount(
 // nest surrounding represented by slice loop bounds in 'slice'.
 // Returns true on success, false otherwise (if a non-constant trip count
 // was encountered).
-// TODO(andydavis) Make this work with non-unit step loops.
+// TODO: Make this work with non-unit step loops.
 static bool buildSliceTripCountMap(
     ComputationSliceState *slice,
     llvm::SmallDenseMap<Operation *, uint64_t, 8> *tripCountMap) {
@@ -457,7 +457,7 @@ bool mlir::getFusionComputeCost(AffineForOp srcForOp, LoopNestStats &srcStats,
   auto *insertPointParent = slice->insertPoint->getParentOp();
 
   // The store and loads to this memref will disappear.
-  // TODO(andydavis) Add load coalescing to memref data flow opt pass.
+  // TODO: Add load coalescing to memref data flow opt pass.
   if (storeLoadFwdGuaranteed) {
     // Subtract from operation count the loads/store we expect load/store
     // forwarding to remove.
diff --git a/mlir/lib/Transforms/Utils/LoopUtils.cpp b/mlir/lib/Transforms/Utils/LoopUtils.cpp
index 58e2d9b420431..249fa1cfdbc2f 100644
--- a/mlir/lib/Transforms/Utils/LoopUtils.cpp
+++ b/mlir/lib/Transforms/Utils/LoopUtils.cpp
@@ -149,7 +149,7 @@ static Value ceilDivPositive(OpBuilder &builder, Location loc, Value dividend,
 
 /// Promotes the loop body of a forOp to its containing block if the forOp
 /// was known to have a single iteration.
-// TODO(bondhugula): extend this for arbitrary affine bounds.
+// TODO: extend this for arbitrary affine bounds.
 LogicalResult mlir::promoteIfSingleIteration(AffineForOp forOp) {
   Optional<uint64_t> tripCount = getConstantTripCount(forOp);
   if (!tripCount || tripCount.getValue() != 1)
@@ -529,7 +529,7 @@ LogicalResult mlir::loopUnrollByFactor(AffineForOp forOp,
     return failure();
 
   // If the trip count is lower than the unroll factor, no unrolled body.
-  // TODO(bondhugula): option to specify cleanup loop unrolling.
+  // TODO: option to specify cleanup loop unrolling.
   Optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
   if (mayBeConstantTripCount.hasValue() &&
       mayBeConstantTripCount.getValue() < unrollFactor)
@@ -623,7 +623,7 @@ LogicalResult mlir::loopUnrollByFactor(scf::ForOp forOp,
             : boundsBuilder.create<ConstantIndexOp>(loc, stepUnrolledCst);
   } else {
     // Dynamic loop bounds computation.
-    // TODO(andydavis) Add dynamic asserts for negative lb/ub/step, or
+    // TODO: Add dynamic asserts for negative lb/ub/step, or
     // consider using ceilDiv from AffineApplyExpander.
     auto lowerBound = forOp.lowerBound();
     auto upperBound = forOp.upperBound();
@@ -720,7 +720,7 @@ LogicalResult mlir::loopUnrollJamByFactor(AffineForOp forOp,
   // Loops where both lower and upper bounds are multi-result maps won't be
   // unrolled (since the trip can't be expressed as an affine function in
   // general).
-  // TODO(mlir-team): this may not be common, but we could support the case
+  // TODO: this may not be common, but we could support the case
   // where the lower bound is a multi-result map and the ub is a single result
   // one.
   if (forOp.getLowerBoundMap().getNumResults() != 1)
@@ -1127,7 +1127,7 @@ static Loops stripmineSink(scf::ForOp forOp, Value factor,
 // Returns the new AffineForOps, nested immediately under `target`.
 template <typename ForType, typename SizeType>
 static ForType stripmineSink(ForType forOp, SizeType factor, ForType target) {
-  // TODO(ntv): Use cheap structural assertions that targets are nested under
+  // TODO: Use cheap structural assertions that targets are nested under
   // forOp and that targets are not nested under each other when DominanceInfo
   // exposes the capability. It seems overkill to construct a whole function
   // dominance tree at this point.
@@ -1226,7 +1226,7 @@ static LogicalResult hoistOpsBetween(scf::ForOp outer, scf::ForOp inner) {
       continue;
     }
     // Skip if op has side effects.
-    // TODO(ntv): loads to immutable memory regions are ok.
+    // TODO: loads to immutable memory regions are ok.
     if (!MemoryEffectOpInterface::hasNoEffect(&op)) {
       status = failure();
       continue;
@@ -1294,7 +1294,7 @@ TileLoops mlir::extractFixedOuterLoops(scf::ForOp rootForOp,
   auto intraTile = tile(forOps, tileSizes, forOps.back());
   TileLoops tileLoops = std::make_pair(forOps, intraTile);
 
-  // TODO(ntv, zinenko) for now we just ignore the result of band isolation.
+  // TODO: for now we just ignore the result of band isolation.
   // In the future, mapping decisions may be impacted by the ability to
   // isolate perfectly nested bands.
   tryIsolateBands(tileLoops);
@@ -1322,7 +1322,7 @@ static LoopParams normalizeLoop(OpBuilder &boundsBuilder,
   // Compute the number of iterations the loop executes: ceildiv(ub - lb, step)
   // assuming the step is strictly positive.  Update the bounds and the step
   // of the loop to go from 0 to the number of iterations, if necessary.
-  // TODO(zinenko): introduce support for negative steps or emit dynamic asserts
+  // TODO: introduce support for negative steps or emit dynamic asserts
   // on step positivity, whatever gets implemented first.
   if (isZeroBased && isStepOne)
     return {/*lowerBound=*/lowerBound, /*upperBound=*/upperBound,
@@ -1491,7 +1491,7 @@ void mlir::collapseParallelLoops(
             // Remove the effect of the current induction value to prepare for
             // the next value.
             previous = insideBuilder.create<SignedDivIOp>(
-                loc, previous, normalizedUpperBounds[idx + 1]);
+                loc, previous, normalizedUpperBounds[idx]);
           }
 
           // The final induction value is just the remaining value.
@@ -1552,7 +1552,7 @@ findHighestBlockForPlacement(const MemRefRegion &region, Block &block,
   // symbolic/variant.
   auto it = enclosingFors.rbegin();
   for (auto e = enclosingFors.rend(); it != e; ++it) {
-    // TODO(bondhugula): also need to be checking this for regions symbols that
+    // TODO: also need to be checking this for regions symbols that
     // aren't loop IVs, whether we are within their resp. defs' dominance scope.
     if (llvm::is_contained(symbols, it->getInductionVar()))
       break;
@@ -1580,7 +1580,7 @@ struct StrideInfo {
 /// potentially multiple striding levels from outermost to innermost. For an
 /// n-dimensional region, there can be at most n-1 levels of striding
 /// successively nested.
-//  TODO(bondhugula): make this work with non-identity layout maps.
+//  TODO: make this work with non-identity layout maps.
 static void getMultiLevelStrides(const MemRefRegion &region,
                                  ArrayRef<int64_t> bufferShape,
                                  SmallVectorImpl<StrideInfo> *strideInfos) {
@@ -1865,7 +1865,7 @@ static LogicalResult generateCopy(
     SmallVector<StrideInfo, 4> dmaStrideInfos;
     getMultiLevelStrides(region, fastBufferShape, &dmaStrideInfos);
 
-    // TODO(bondhugula): use all stride levels once DmaStartOp is extended for
+    // TODO: use all stride levels once DmaStartOp is extended for
     // multi-level strides.
     if (dmaStrideInfos.size() > 1) {
       LLVM_DEBUG(llvm::dbgs() << "Only up to one level of stride supported\n");
@@ -2120,7 +2120,7 @@ uint64_t mlir::affineDataCopyGenerate(Block::iterator begin,
 
     // Each memref has a single buffer associated with it irrespective of how
     // many load's and store's happen on it.
-    // TODO(bondhugula): in the future, when regions don't intersect and satisfy
+    // TODO: in the future, when regions don't intersect and satisfy
     // other properties (based on load/store regions), we could consider
     // multiple buffers per memref.
 
diff --git a/mlir/lib/Transforms/Utils/Utils.cpp b/mlir/lib/Transforms/Utils/Utils.cpp
index 358d554d7df3d..5d6d58fc6939d 100644
--- a/mlir/lib/Transforms/Utils/Utils.cpp
+++ b/mlir/lib/Transforms/Utils/Utils.cpp
@@ -28,12 +28,10 @@ using namespace mlir;
 
 /// Return true if this operation dereferences one or more memref's.
 // Temporary utility: will be replaced when this is modeled through
-// side-effects/op traits. TODO(b/117228571)
+// side-effects/op traits. TODO
 static bool isMemRefDereferencingOp(Operation &op) {
-  if (isa<AffineReadOpInterface>(op) || isa<AffineWriteOpInterface>(op) ||
-      isa<AffineDmaStartOp>(op) || isa<AffineDmaWaitOp>(op))
-    return true;
-  return false;
+  return isa<AffineReadOpInterface, AffineWriteOpInterface, AffineDmaStartOp,
+             AffineDmaWaitOp>(op);
 }
 
 /// Return the AffineMapAttr associated with memory 'op' on 'memref'.
@@ -85,7 +83,7 @@ LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
     return success();
 
   if (usePositions.size() > 1) {
-    // TODO(mlir-team): extend it for this case when needed (rare).
+    // TODO: extend it for this case when needed (rare).
     assert(false && "multiple dereferencing uses in a single op not supported");
     return failure();
   }
@@ -164,7 +162,7 @@ LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
   // Create new fully composed AffineMap for new op to be created.
   assert(newMapOperands.size() == newMemRefRank);
   auto newMap = builder.getMultiDimIdentityMap(newMemRefRank);
-  // TODO(b/136262594) Avoid creating/deleting temporary AffineApplyOps here.
+  // TODO: Avoid creating/deleting temporary AffineApplyOps here.
   fullyComposeAffineMapAndOperands(&newMap, &newMapOperands);
   newMap = simplifyAffineMap(newMap);
   canonicalizeMapAndOperands(&newMap, &newMapOperands);
diff --git a/mlir/test/Bindings/Python/lit.local.cfg b/mlir/test/Bindings/Python/lit.local.cfg
new file mode 100644
index 0000000000000..4cfe04325d94e
--- /dev/null
+++ b/mlir/test/Bindings/Python/lit.local.cfg
@@ -0,0 +1,2 @@
+if not config.enable_bindings_python:
+  config.unsupported = True
diff --git a/mlir/test/Bindings/Python/smoke_test.py b/mlir/test/Bindings/Python/smoke_test.py
new file mode 100644
index 0000000000000..3904e72e25bb7
--- /dev/null
+++ b/mlir/test/Bindings/Python/smoke_test.py
@@ -0,0 +1,6 @@
+# RUN: %PYTHON %s | FileCheck %s
+
+import mlir
+
+# CHECK: From the native module
+print(mlir.get_test_value())
diff --git a/mlir/test/CMakeLists.txt b/mlir/test/CMakeLists.txt
index 52756b6aae7fe..b663d37cb8483 100644
--- a/mlir/test/CMakeLists.txt
+++ b/mlir/test/CMakeLists.txt
@@ -4,7 +4,13 @@ add_subdirectory(SDBM)
 add_subdirectory(lib)
 
 llvm_canonicalize_cmake_booleans(
+  MLIR_BINDINGS_PYTHON_ENABLED
   LLVM_BUILD_EXAMPLES
+  MLIR_CUDA_CONVERSIONS_ENABLED
+  MLIR_CUDA_RUNNER_ENABLED
+  MLIR_ROCM_CONVERSIONS_ENABLED
+  MLIR_ROCM_RUNNER_ENABLED
+  MLIR_VULKAN_RUNNER_ENABLED
   )
 
 # Passed to lit.site.cfg.py.in to set up the path where to find the libraries
@@ -38,6 +44,7 @@ set(MLIR_TEST_DEPENDS
   mlir-edsc-builder-api-test
   mlir-linalg-ods-gen
   mlir-opt
+  mlir-reduce
   mlir-sdbm-api-test
   mlir-tblgen
   mlir-translate
@@ -77,6 +84,12 @@ if(MLIR_VULKAN_RUNNER_ENABLED)
   )
 endif()
 
+if(MLIR_BINDINGS_PYTHON_ENABLED)
+  list(APPEND MLIR_TEST_DEPENDS
+    MLIRBindingsPythonExtension
+  )
+endif()
+
 add_lit_testsuite(check-mlir "Running the MLIR regression tests"
   ${CMAKE_CURRENT_BINARY_DIR}
   DEPENDS ${MLIR_TEST_DEPENDS}
diff --git a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
index 20d166bab05d1..fa9a478c1d83b 100644
--- a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
+++ b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
@@ -1,36 +1,52 @@
 // RUN: mlir-opt %s -convert-gpu-to-nvvm -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -convert-gpu-to-nvvm='index-bitwidth=32' -split-input-file | FileCheck --check-prefix=CHECK32 %s
 
 gpu.module @test_module {
   // CHECK-LABEL: func @gpu_index_ops()
+  // CHECK32-LABEL: func @gpu_index_ops()
   func @gpu_index_ops()
       -> (index, index, index, index, index, index,
           index, index, index, index, index, index) {
+    // CHECK32-NOT: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
+
     // CHECK: = nvvm.read.ptx.sreg.tid.x : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %tIdX = "gpu.thread_id"() {dimension = "x"} : () -> (index)
     // CHECK: = nvvm.read.ptx.sreg.tid.y : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %tIdY = "gpu.thread_id"() {dimension = "y"} : () -> (index)
     // CHECK: = nvvm.read.ptx.sreg.tid.z : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %tIdZ = "gpu.thread_id"() {dimension = "z"} : () -> (index)
 
     // CHECK: = nvvm.read.ptx.sreg.ntid.x : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %bDimX = "gpu.block_dim"() {dimension = "x"} : () -> (index)
     // CHECK: = nvvm.read.ptx.sreg.ntid.y : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %bDimY = "gpu.block_dim"() {dimension = "y"} : () -> (index)
     // CHECK: = nvvm.read.ptx.sreg.ntid.z : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %bDimZ = "gpu.block_dim"() {dimension = "z"} : () -> (index)
 
     // CHECK: = nvvm.read.ptx.sreg.ctaid.x : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %bIdX = "gpu.block_id"() {dimension = "x"} : () -> (index)
     // CHECK: = nvvm.read.ptx.sreg.ctaid.y : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %bIdY = "gpu.block_id"() {dimension = "y"} : () -> (index)
     // CHECK: = nvvm.read.ptx.sreg.ctaid.z : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %bIdZ = "gpu.block_id"() {dimension = "z"} : () -> (index)
 
     // CHECK: = nvvm.read.ptx.sreg.nctaid.x : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %gDimX = "gpu.grid_dim"() {dimension = "x"} : () -> (index)
     // CHECK: = nvvm.read.ptx.sreg.nctaid.y : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %gDimY = "gpu.grid_dim"() {dimension = "y"} : () -> (index)
     // CHECK: = nvvm.read.ptx.sreg.nctaid.z : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %gDimZ = "gpu.grid_dim"() {dimension = "z"} : () -> (index)
 
     std.return %tIdX, %tIdY, %tIdZ, %bDimX, %bDimY, %bDimZ,
@@ -42,11 +58,26 @@ gpu.module @test_module {
 
 // -----
 
+gpu.module @test_module {
+  // CHECK-LABEL: func @gpu_index_comp
+  // CHECK32-LABEL: func @gpu_index_comp
+  func @gpu_index_comp(%idx : index) -> index {
+    // CHECK: = llvm.add %{{.*}}, %{{.*}} : !llvm.i64
+    // CHECK32: = llvm.add %{{.*}}, %{{.*}} : !llvm.i32
+    %0 = addi %idx, %idx : index
+    // CHECK: llvm.return %{{.*}} : !llvm.i64
+    // CHECK32: llvm.return %{{.*}} : !llvm.i32
+    std.return %0 : index
+  }
+}
+
+// -----
+
 gpu.module @test_module {
   // CHECK-LABEL: func @gpu_all_reduce_op()
   gpu.func @gpu_all_reduce_op() {
     %arg0 = constant 1.0 : f32
-    // TODO(csigg): Check full IR expansion once lowering has settled.
+    // TODO: Check full IR expansion once lowering has settled.
     // CHECK: nvvm.shfl.sync.bfly
     // CHECK: nvvm.barrier0
     // CHECK: llvm.fadd
@@ -62,7 +93,7 @@ gpu.module @test_module {
   // CHECK-LABEL: func @gpu_all_reduce_region()
   gpu.func @gpu_all_reduce_region() {
     %arg0 = constant 1 : i32
-    // TODO(csigg): Check full IR expansion once lowering has settled.
+    // TODO: Check full IR expansion once lowering has settled.
     // CHECK: nvvm.shfl.sync.bfly
     // CHECK: nvvm.barrier0
     %result = "gpu.all_reduce"(%arg0) ({
diff --git a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
index 61becff83c6cf..a7565bb6e323f 100644
--- a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
@@ -1,36 +1,52 @@
 // RUN: mlir-opt %s -convert-gpu-to-rocdl -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='index-bitwidth=32' -split-input-file | FileCheck --check-prefix=CHECK32 %s
 
 gpu.module @test_module {
   // CHECK-LABEL: func @gpu_index_ops()
+  // CHECK32-LABEL: func @gpu_index_ops()
   func @gpu_index_ops()
       -> (index, index, index, index, index, index,
           index, index, index, index, index, index) {
+    // CHECK32-NOT: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
+
     // CHECK: rocdl.workitem.id.x : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %tIdX = "gpu.thread_id"() {dimension = "x"} : () -> (index)
     // CHECK: rocdl.workitem.id.y : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %tIdY = "gpu.thread_id"() {dimension = "y"} : () -> (index)
     // CHECK: rocdl.workitem.id.z : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %tIdZ = "gpu.thread_id"() {dimension = "z"} : () -> (index)
 
     // CHECK: rocdl.workgroup.dim.x : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %bDimX = "gpu.block_dim"() {dimension = "x"} : () -> (index)
     // CHECK: rocdl.workgroup.dim.y : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %bDimY = "gpu.block_dim"() {dimension = "y"} : () -> (index)
     // CHECK: rocdl.workgroup.dim.z : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %bDimZ = "gpu.block_dim"() {dimension = "z"} : () -> (index)
 
     // CHECK: rocdl.workgroup.id.x : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %bIdX = "gpu.block_id"() {dimension = "x"} : () -> (index)
     // CHECK: rocdl.workgroup.id.y : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %bIdY = "gpu.block_id"() {dimension = "y"} : () -> (index)
     // CHECK: rocdl.workgroup.id.z : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %bIdZ = "gpu.block_id"() {dimension = "z"} : () -> (index)
 
     // CHECK: rocdl.grid.dim.x : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %gDimX = "gpu.grid_dim"() {dimension = "x"} : () -> (index)
     // CHECK: rocdl.grid.dim.y : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %gDimY = "gpu.grid_dim"() {dimension = "y"} : () -> (index)
     // CHECK: rocdl.grid.dim.z : !llvm.i32
+    // CHECK: = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
     %gDimZ = "gpu.grid_dim"() {dimension = "z"} : () -> (index)
 
     std.return %tIdX, %tIdY, %tIdZ, %bDimX, %bDimY, %bDimZ,
@@ -42,6 +58,21 @@ gpu.module @test_module {
 
 // -----
 
+gpu.module @test_module {
+  // CHECK-LABEL: func @gpu_index_comp
+  // CHECK32-LABEL: func @gpu_index_comp
+  func @gpu_index_comp(%idx : index) -> index {
+    // CHECK: = llvm.add %{{.*}}, %{{.*}} : !llvm.i64
+    // CHECK32: = llvm.add %{{.*}}, %{{.*}} : !llvm.i32
+    %0 = addi %idx, %idx : index
+    // CHECK: llvm.return %{{.*}} : !llvm.i64
+    // CHECK32: llvm.return %{{.*}} : !llvm.i32
+    std.return %0 : index
+  }
+}
+
+// -----
+
 gpu.module @test_module {
   // CHECK-LABEL: func @gpu_sync()
   func @gpu_sync() {
diff --git a/mlir/test/Conversion/GPUToSPIRV/if.mlir b/mlir/test/Conversion/GPUToSPIRV/if.mlir
index 1e8164cb310df..b7e11d74996bd 100644
--- a/mlir/test/Conversion/GPUToSPIRV/if.mlir
+++ b/mlir/test/Conversion/GPUToSPIRV/if.mlir
@@ -89,5 +89,79 @@ module attributes {
       }
       gpu.return
     }
+    // CHECK-LABEL: @simple_if_yield
+    gpu.func @simple_if_yield(%arg2 : memref<10xf32>, %arg3 : i1) kernel
+    attributes {spv.entry_point_abi = {local_size = dense<[16, 1, 1]>: vector<3xi32>}} {
+      // CHECK: %[[VAR1:.*]] = spv.Variable : !spv.ptr<f32, Function>
+      // CHECK: %[[VAR2:.*]] = spv.Variable : !spv.ptr<f32, Function>
+      // CHECK:       spv.selection {
+      // CHECK-NEXT:    spv.BranchConditional {{%.*}}, [[TRUE:\^.*]], [[FALSE:\^.*]]
+      // CHECK-NEXT:  [[TRUE]]:
+      // CHECK:         %[[RET1TRUE:.*]] = spv.constant 0.000000e+00 : f32
+      // CHECK:         %[[RET2TRUE:.*]] = spv.constant 1.000000e+00 : f32
+      // CHECK-DAG:     spv.Store "Function" %[[VAR1]], %[[RET1TRUE]] : f32
+      // CHECK-DAG:     spv.Store "Function" %[[VAR2]], %[[RET2TRUE]] : f32
+      // CHECK:         spv.Branch ^[[MERGE:.*]]
+      // CHECK-NEXT:  [[FALSE]]:
+      // CHECK:         %[[RET2FALSE:.*]] = spv.constant 2.000000e+00 : f32
+      // CHECK:         %[[RET1FALSE:.*]] = spv.constant 3.000000e+00 : f32
+      // CHECK-DAG:     spv.Store "Function" %[[VAR1]], %[[RET1FALSE]] : f32
+      // CHECK-DAG:     spv.Store "Function" %[[VAR2]], %[[RET2FALSE]] : f32
+      // CHECK:         spv.Branch ^[[MERGE]]
+      // CHECK-NEXT:  ^[[MERGE]]:
+      // CHECK:         spv._merge
+      // CHECK-NEXT:  }
+      // CHECK-DAG:   %[[OUT1:.*]] = spv.Load "Function" %[[VAR1]] : f32
+      // CHECK-DAG:   %[[OUT2:.*]] = spv.Load "Function" %[[VAR2]] : f32
+      // CHECK:       spv.Store "StorageBuffer" {{%.*}}, %[[OUT1]] : f32
+      // CHECK:       spv.Store "StorageBuffer" {{%.*}}, %[[OUT2]] : f32
+      // CHECK:       spv.Return
+      %0:2 = scf.if %arg3 -> (f32, f32) {
+        %c0 = constant 0.0 : f32
+        %c1 = constant 1.0 : f32
+        scf.yield %c0, %c1 : f32, f32
+      } else {
+        %c0 = constant 2.0 : f32
+        %c1 = constant 3.0 : f32
+        scf.yield %c1, %c0 : f32, f32
+      }
+      %i = constant 0 : index
+      %j = constant 1 : index
+      store %0#0, %arg2[%i] : memref<10xf32>
+      store %0#1, %arg2[%j] : memref<10xf32>
+      gpu.return
+    }
+    // TODO: The transformation should only be legal if
+    // VariablePointer capability is supported. This test is still useful to
+    // make sure we can handle scf op result with type change.
+    // CHECK-LABEL: @simple_if_yield_type_change
+    // CHECK:       %[[VAR:.*]] = spv.Variable : !spv.ptr<!spv.ptr<!spv.struct<!spv.array<10 x f32, stride=4> [0]>, StorageBuffer>, Function>
+    // CHECK:       spv.selection {
+    // CHECK-NEXT:    spv.BranchConditional {{%.*}}, [[TRUE:\^.*]], [[FALSE:\^.*]]
+    // CHECK-NEXT:  [[TRUE]]:
+    // CHECK:         spv.Store "Function" %[[VAR]], {{%.*}} : !spv.ptr<!spv.struct<!spv.array<10 x f32, stride=4> [0]>, StorageBuffer>
+    // CHECK:         spv.Branch ^[[MERGE:.*]]
+    // CHECK-NEXT:  [[FALSE]]:
+    // CHECK:         spv.Store "Function" %[[VAR]], {{%.*}} : !spv.ptr<!spv.struct<!spv.array<10 x f32, stride=4> [0]>, StorageBuffer>
+    // CHECK:         spv.Branch ^[[MERGE]]
+    // CHECK-NEXT:  ^[[MERGE]]:
+    // CHECK:         spv._merge
+    // CHECK-NEXT:  }
+    // CHECK:       %[[OUT:.*]] = spv.Load "Function" %[[VAR]] : !spv.ptr<!spv.struct<!spv.array<10 x f32, stride=4> [0]>, StorageBuffer>
+    // CHECK:       %[[ADD:.*]] = spv.AccessChain %[[OUT]][{{%.*}}, {{%.*}}] : !spv.ptr<!spv.struct<!spv.array<10 x f32, stride=4> [0]>, StorageBuffer>
+    // CHECK:       spv.Store "StorageBuffer" %[[ADD]], {{%.*}} : f32
+    // CHECK:       spv.Return
+    gpu.func @simple_if_yield_type_change(%arg2 : memref<10xf32>, %arg3 : memref<10xf32>, %arg4 : i1) kernel
+    attributes {spv.entry_point_abi = {local_size = dense<[16, 1, 1]>: vector<3xi32>}} {
+      %i = constant 0 : index
+      %value = constant 0.0 : f32
+      %0 = scf.if %arg4 -> (memref<10xf32>) {
+        scf.yield %arg2 : memref<10xf32>
+      } else {
+        scf.yield %arg3 : memref<10xf32>
+      }
+      store %value, %0[%i] : memref<10xf32>
+      gpu.return
+    }
   }
 }
diff --git a/mlir/test/Conversion/GPUToSPIRV/loop.mlir b/mlir/test/Conversion/GPUToSPIRV/loop.mlir
index 7c5df798438fc..2205c60f875f5 100644
--- a/mlir/test/Conversion/GPUToSPIRV/loop.mlir
+++ b/mlir/test/Conversion/GPUToSPIRV/loop.mlir
@@ -51,5 +51,48 @@ module attributes {
       }
       gpu.return
     }
+
+
+    // CHECK-LABEL: @loop_yield
+    gpu.func @loop_yield(%arg2 : memref<10xf32>, %arg3 : memref<10xf32>) kernel
+    attributes {spv.entry_point_abi = {local_size = dense<[16, 1, 1]>: vector<3xi32>}} {
+      // CHECK: %[[LB:.*]] = spv.constant 4 : i32
+      %lb = constant 4 : index
+      // CHECK: %[[UB:.*]] = spv.constant 42 : i32
+      %ub = constant 42 : index
+      // CHECK: %[[STEP:.*]] = spv.constant 2 : i32
+      %step = constant 2 : index
+      // CHECK: %[[INITVAR1:.*]] = spv.constant 0.000000e+00 : f32
+      %s0 = constant 0.0 : f32
+      // CHECK: %[[INITVAR2:.*]] = spv.constant 1.000000e+00 : f32
+      %s1 = constant 1.0 : f32
+      // CHECK: %[[VAR1:.*]] = spv.Variable : !spv.ptr<f32, Function>
+      // CHECK: %[[VAR2:.*]] = spv.Variable : !spv.ptr<f32, Function>
+      // CHECK: spv.loop {
+      // CHECK:   spv.Branch ^[[HEADER:.*]](%[[LB]], %[[INITVAR1]], %[[INITVAR2]] : i32, f32, f32)
+      // CHECK: ^[[HEADER]](%[[INDVAR:.*]]: i32, %[[CARRIED1:.*]]: f32, %[[CARRIED2:.*]]: f32):
+      // CHECK:   %[[CMP:.*]] = spv.SLessThan %[[INDVAR]], %[[UB]] : i32
+      // CHECK:   spv.BranchConditional %[[CMP]], ^[[BODY:.*]], ^[[MERGE:.*]]
+      // CHECK: ^[[BODY]]:
+      // CHECK:   %[[UPDATED:.*]] = spv.FAdd %[[CARRIED1]], %[[CARRIED1]] : f32
+      // CHECK-DAG:   %[[INCREMENT:.*]] = spv.IAdd %[[INDVAR]], %[[STEP]] : i32
+      // CHECK-DAG:   spv.Store "Function" %[[VAR1]], %[[UPDATED]] : f32
+      // CHECK-DAG:   spv.Store "Function" %[[VAR2]], %[[UPDATED]] : f32
+      // CHECK: spv.Branch ^[[HEADER]](%[[INCREMENT]], %[[UPDATED]], %[[UPDATED]] : i32, f32, f32)
+      // CHECK: ^[[MERGE]]:
+      // CHECK:   spv._merge
+      // CHECK: }
+      %result:2 = scf.for %i0 = %lb to %ub step %step iter_args(%si = %s0, %sj = %s1) -> (f32, f32) {
+        %sn = addf %si, %si : f32
+        scf.yield %sn, %sn : f32, f32
+      }
+      // CHECK-DAG: %[[OUT1:.*]] = spv.Load "Function" %[[VAR1]] : f32
+      // CHECK-DAG: %[[OUT2:.*]] = spv.Load "Function" %[[VAR2]] : f32
+      // CHECK: spv.Store "StorageBuffer" {{%.*}}, %[[OUT1]] : f32
+      // CHECK: spv.Store "StorageBuffer" {{%.*}}, %[[OUT2]] : f32
+      store %result#0, %arg3[%lb] : memref<10xf32>
+      store %result#1, %arg3[%ub] : memref<10xf32>
+      gpu.return
+    }
   }
 }
diff --git a/mlir/test/Conversion/GPUToVulkan/lower-gpu-launch-vulkan-launch.mlir b/mlir/test/Conversion/GPUToVulkan/lower-gpu-launch-vulkan-launch.mlir
index 726b276010ef8..43da9c5be4294 100644
--- a/mlir/test/Conversion/GPUToVulkan/lower-gpu-launch-vulkan-launch.mlir
+++ b/mlir/test/Conversion/GPUToVulkan/lower-gpu-launch-vulkan-launch.mlir
@@ -11,7 +11,7 @@ module attributes {gpu.container_module} {
       %0 = spv._address_of @kernel_arg_0 : !spv.ptr<!spv.struct<!spv.array<12 x f32, stride=4> [0]>, StorageBuffer>
       %2 = spv.constant 0 : i32
       %3 = spv._address_of @kernel_arg_0 : !spv.ptr<!spv.struct<!spv.array<12 x f32, stride=4> [0]>, StorageBuffer>
-      %4 = spv.AccessChain %0[%2, %2] : !spv.ptr<!spv.struct<!spv.array<12 x f32, stride=4> [0]>, StorageBuffer>
+      %4 = spv.AccessChain %0[%2, %2] : !spv.ptr<!spv.struct<!spv.array<12 x f32, stride=4> [0]>, StorageBuffer>, i32, i32
       %5 = spv.Load "StorageBuffer" %4 : f32
       spv.Return
     }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/arithmetic-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/arithmetic-ops-to-llvm.mlir
index 30cfb55b262ab..2f5d18a6acf8f 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/arithmetic-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/arithmetic-ops-to-llvm.mlir
@@ -96,6 +96,22 @@ func @fdiv_vector(%arg0: vector<3xf64>, %arg1: vector<3xf64>) {
 	return
 }
 
+//===----------------------------------------------------------------------===//
+// spv.FMul
+//===----------------------------------------------------------------------===//
+
+func @fmul_scalar(%arg0: f32, %arg1: f32) {
+	// CHECK: %{{.*}} = llvm.fmul %{{.*}}, %{{.*}} : !llvm.float
+	%0 = spv.FMul %arg0, %arg1 : f32
+	return
+}
+
+func @fmul_vector(%arg0: vector<2xf32>, %arg1: vector<2xf32>) {
+	// CHECK: %{{.*}} = llvm.fmul %{{.*}}, %{{.*}} : !llvm<"<2 x float>">
+	%0 = spv.FMul %arg0, %arg1 : vector<2xf32>
+	return
+}
+
 //===----------------------------------------------------------------------===//
 // spv.FRem
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Conversion/SPIRVToLLVM/bitwise-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/bitwise-ops-to-llvm.mlir
index 13410400253e7..31ffc6dbf7dc9 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/bitwise-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/bitwise-ops-to-llvm.mlir
@@ -1,5 +1,250 @@
 // RUN: mlir-opt -convert-spirv-to-llvm %s | FileCheck %s
 
+//===----------------------------------------------------------------------===//
+// spv.BitCount
+//===----------------------------------------------------------------------===//
+
+func @bitcount_scalar(%arg0: i16) {
+	// CHECK: %{{.*}} = "llvm.intr.ctpop"(%{{.*}}) : (!llvm.i16) -> !llvm.i16
+	%0 = spv.BitCount %arg0: i16
+	return
+}
+
+func @bitcount_vector(%arg0: vector<3xi32>) {
+	// CHECK: %{{.*}} = "llvm.intr.ctpop"(%{{.*}}) : (!llvm<"<3 x i32>">) -> !llvm<"<3 x i32>">
+	%0 = spv.BitCount %arg0: vector<3xi32>
+	return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.BitReverse
+//===----------------------------------------------------------------------===//
+
+func @bitreverse_scalar(%arg0: i64) {
+	// CHECK: %{{.*}} = "llvm.intr.bitreverse"(%{{.*}}) : (!llvm.i64) -> !llvm.i64
+	%0 = spv.BitReverse %arg0: i64
+	return
+}
+
+func @bitreverse_vector(%arg0: vector<4xi32>) {
+	// CHECK: %{{.*}} = "llvm.intr.bitreverse"(%{{.*}}) : (!llvm<"<4 x i32>">) -> !llvm<"<4 x i32>">
+	%0 = spv.BitReverse %arg0: vector<4xi32>
+	return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.BitFieldInsert
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func @bitfield_insert_scalar_same_bit_width
+// CHECK-SAME: %[[BASE:.*]]: !llvm.i32, %[[INSERT:.*]]: !llvm.i32, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i32
+func @bitfield_insert_scalar_same_bit_width(%base: i32, %insert: i32, %offset: i32, %count: i32) {
+    // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i32) : !llvm.i32
+    // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT]] : !llvm.i32
+    // CHECK: %[[T1:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i32
+    // CHECK: %[[T2:.*]] = llvm.shl %[[T1]], %[[OFFSET]] : !llvm.i32
+    // CHECK: %[[MASK:.*]] = llvm.xor %[[T2]], %[[MINUS_ONE]] : !llvm.i32
+    // CHECK: %[[NEW_BASE:.*]] = llvm.and %[[BASE]], %[[MASK]] : !llvm.i32
+    // CHECK: %[[SHIFTED_INSERT:.*]] = llvm.shl %[[INSERT]], %[[OFFSET]] : !llvm.i32
+    // CHECK: %{{.*}} = llvm.or %[[NEW_BASE]], %[[SHIFTED_INSERT]] : !llvm.i32
+    %0 = spv.BitFieldInsert %base, %insert, %offset, %count : i32, i32, i32
+    return
+}
+
+// CHECK-LABEL: func @bitfield_insert_scalar_smaller_bit_width
+// CHECK-SAME: %[[BASE:.*]]: !llvm.i64, %[[INSERT:.*]]: !llvm.i64, %[[OFFSET:.*]]: !llvm.i8, %[[COUNT:.*]]: !llvm.i8
+func @bitfield_insert_scalar_smaller_bit_width(%base: i64, %insert: i64, %offset: i8, %count: i8) {
+    // CHECK: %[[EXT_OFFSET:.*]] = llvm.zext %[[OFFSET]] : !llvm.i8 to !llvm.i64
+    // CHECK: %[[EXT_COUNT:.*]] = llvm.zext %[[COUNT]] : !llvm.i8 to !llvm.i64
+    // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i64) : !llvm.i64
+    // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[EXT_COUNT]] : !llvm.i64
+    // CHECK: %[[T1:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i64
+    // CHECK: %[[T2:.*]] = llvm.shl %[[T1]], %[[EXT_OFFSET]] : !llvm.i64
+    // CHECK: %[[MASK:.*]] = llvm.xor %[[T2]], %[[MINUS_ONE]] : !llvm.i64
+    // CHECK: %[[NEW_BASE:.*]] = llvm.and %[[BASE]], %[[MASK]] : !llvm.i64
+    // CHECK: %[[SHIFTED_INSERT:.*]] = llvm.shl %[[INSERT]], %[[EXT_OFFSET]] : !llvm.i64
+    // CHECK: %{{.*}} = llvm.or %[[NEW_BASE]], %[[SHIFTED_INSERT]] : !llvm.i64
+    %0 = spv.BitFieldInsert %base, %insert, %offset, %count : i64, i8, i8
+    return
+}
+
+// CHECK-LABEL: func @bitfield_insert_scalar_greater_bit_width
+// CHECK-SAME: %[[BASE:.*]]: !llvm.i16, %[[INSERT:.*]]: !llvm.i16, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i64
+func @bitfield_insert_scalar_greater_bit_width(%base: i16, %insert: i16, %offset: i32, %count: i64) {
+    // CHECK: %[[TRUNC_OFFSET:.*]] = llvm.trunc %[[OFFSET]] : !llvm.i32 to !llvm.i16
+    // CHECK: %[[TRUNC_COUNT:.*]] = llvm.trunc %[[COUNT]] : !llvm.i64 to !llvm.i16
+    // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i16) : !llvm.i16
+    // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[TRUNC_COUNT]] : !llvm.i16
+    // CHECK: %[[T1:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i16
+    // CHECK: %[[T2:.*]] = llvm.shl %[[T1]], %[[TRUNC_OFFSET]] : !llvm.i16
+    // CHECK: %[[MASK:.*]] = llvm.xor %[[T2]], %[[MINUS_ONE]] : !llvm.i16
+    // CHECK: %[[NEW_BASE:.*]] = llvm.and %[[BASE]], %[[MASK]] : !llvm.i16
+    // CHECK: %[[SHIFTED_INSERT:.*]] = llvm.shl %[[INSERT]], %[[TRUNC_OFFSET]] : !llvm.i16
+    // CHECK: %{{.*}} = llvm.or %[[NEW_BASE]], %[[SHIFTED_INSERT]] : !llvm.i16
+    %0 = spv.BitFieldInsert %base, %insert, %offset, %count : i16, i32, i64
+    return
+}
+
+// CHECK-LABEL: func @bitfield_insert_vector
+// CHECK-SAME: %[[BASE:.*]]: !llvm<"<2 x i32>">, %[[INSERT:.*]]: !llvm<"<2 x i32>">, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i32
+func @bitfield_insert_vector(%base: vector<2xi32>, %insert: vector<2xi32>, %offset: i32, %count: i32) {
+    // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+    // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
+    // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+    // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+    // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
+    // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+    // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
+    // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+    // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+    // CHECK: %[[COUNT_V2:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
+    // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(dense<-1> : vector<2xi32>) : !llvm<"<2 x i32>">
+    // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT_V2]] : !llvm<"<2 x i32>">
+    // CHECK: %[[T1:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm<"<2 x i32>">
+    // CHECK: %[[T2:.*]] = llvm.shl %[[T1]], %[[OFFSET_V2]] : !llvm<"<2 x i32>">
+    // CHECK: %[[MASK:.*]] = llvm.xor %[[T2]], %[[MINUS_ONE]] : !llvm<"<2 x i32>">
+    // CHECK: %[[NEW_BASE:.*]] = llvm.and %[[BASE]], %[[MASK]] : !llvm<"<2 x i32>">
+    // CHECK: %[[SHIFTED_INSERT:.*]] = llvm.shl %[[INSERT]], %[[OFFSET_V2]] : !llvm<"<2 x i32>">
+    // CHECK: %{{.*}} = llvm.or %[[NEW_BASE]], %[[SHIFTED_INSERT]] : !llvm<"<2 x i32>">
+    %0 = spv.BitFieldInsert %base, %insert, %offset, %count : vector<2xi32>, i32, i32
+    return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.BitFieldSExtract
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func @bitfield_sextract_scalar_same_bit_width
+// CHECK-SAME: %[[BASE:.*]]: !llvm.i64, %[[OFFSET:.*]]: !llvm.i64, %[[COUNT:.*]]: !llvm.i64
+func @bitfield_sextract_scalar_same_bit_width(%base: i64, %offset: i64, %count: i64) {
+    // CHECK: %[[SIZE:.]] = llvm.mlir.constant(64 : i64) : !llvm.i64
+    // CHECK: %[[T0:.*]] = llvm.add %[[COUNT]], %[[OFFSET]] : !llvm.i64
+    // CHECK: %[[T1:.*]] = llvm.sub %[[SIZE]], %[[T0]] : !llvm.i64
+    // CHECK: %[[SHIFTED_LEFT:.*]] = llvm.shl %[[BASE]], %[[T1]] : !llvm.i64
+    // CHECK: %[[T2:.*]] = llvm.add %[[OFFSET]], %[[T1]] : !llvm.i64
+    // CHECK: %{{.*}} = llvm.ashr %[[SHIFTED_LEFT]], %[[T2]] : !llvm.i64
+    %0 = spv.BitFieldSExtract %base, %offset, %count : i64, i64, i64
+    return
+}
+
+// CHECK-LABEL: func @bitfield_sextract_scalar_smaller_bit_width
+// CHECK-SAME: %[[BASE:.*]]: !llvm.i32, %[[OFFSET:.*]]: !llvm.i8, %[[COUNT:.*]]: !llvm.i8
+func @bitfield_sextract_scalar_smaller_bit_width(%base: i32, %offset: i8, %count: i8) {
+    // CHECK: %[[EXT_OFFSET:.*]] = llvm.zext %[[OFFSET]] : !llvm.i8 to !llvm.i32
+    // CHECK: %[[EXT_COUNT:.*]] = llvm.zext %[[COUNT]] : !llvm.i8 to !llvm.i32
+    // CHECK: %[[SIZE:.]] = llvm.mlir.constant(32 : i32) : !llvm.i32
+    // CHECK: %[[T0:.*]] = llvm.add %[[EXT_COUNT]], %[[EXT_OFFSET]] : !llvm.i32
+    // CHECK: %[[T1:.*]] = llvm.sub %[[SIZE]], %[[T0]] : !llvm.i32
+    // CHECK: %[[SHIFTED_LEFT:.*]] = llvm.shl %[[BASE]], %[[T1]] : !llvm.i32
+    // CHECK: %[[T2:.*]] = llvm.add %[[EXT_OFFSET]], %[[T1]] : !llvm.i32
+    // CHECK: %{{.*}} = llvm.ashr %[[SHIFTED_LEFT]], %[[T2]] : !llvm.i32
+    %0 = spv.BitFieldSExtract %base, %offset, %count : i32, i8, i8
+    return
+}
+
+// CHECK-LABEL: func @bitfield_sextract_scalar_greater_bit_width
+// CHECK-SAME: %[[BASE:.*]]: !llvm.i32, %[[OFFSET:.*]]: !llvm.i64, %[[COUNT:.*]]: !llvm.i64
+func @bitfield_sextract_scalar_greater_bit_width(%base: i32, %offset: i64, %count: i64) {
+    // CHECK: %[[TRUNC_OFFSET:.*]] = llvm.trunc %[[OFFSET]] : !llvm.i64 to !llvm.i32
+    // CHECK: %[[TRUNC_COUNT:.*]] = llvm.trunc %[[COUNT]] : !llvm.i64 to !llvm.i32
+    // CHECK: %[[SIZE:.]] = llvm.mlir.constant(32 : i32) : !llvm.i32
+    // CHECK: %[[T0:.*]] = llvm.add %[[TRUNC_COUNT]], %[[TRUNC_OFFSET]] : !llvm.i32
+    // CHECK: %[[T1:.*]] = llvm.sub %[[SIZE]], %[[T0]] : !llvm.i32
+    // CHECK: %[[SHIFTED_LEFT:.*]] = llvm.shl %[[BASE]], %[[T1]] : !llvm.i32
+    // CHECK: %[[T2:.*]] = llvm.add %[[TRUNC_OFFSET]], %[[T1]] : !llvm.i32
+    // CHECK: %{{.*}} = llvm.ashr %[[SHIFTED_LEFT]], %[[T2]] : !llvm.i32
+    %0 = spv.BitFieldSExtract %base, %offset, %count : i32, i64, i64
+    return
+}
+
+// CHECK-LABEL: func @bitfield_sextract_vector
+// CHECK-SAME: %[[BASE:.*]]: !llvm<"<2 x i32>">, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i32
+func @bitfield_sextract_vector(%base: vector<2xi32>, %offset: i32, %count: i32) {
+    // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+    // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
+    // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+    // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+    // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
+    // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+    // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
+    // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+    // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+    // CHECK: %[[COUNT_V2:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
+    // CHECK: %[[SIZE:.*]] = llvm.mlir.constant(dense<32> : vector<2xi32>) : !llvm<"<2 x i32>">
+    // CHECK: %[[T0:.*]] = llvm.add %[[COUNT_V2]], %[[OFFSET_V2]] : !llvm<"<2 x i32>">
+    // CHECK: %[[T1:.*]] = llvm.sub %[[SIZE]], %[[T0]] : !llvm<"<2 x i32>">
+    // CHECK: %[[SHIFTED_LEFT:.*]] = llvm.shl %[[BASE]], %[[T1]] : !llvm<"<2 x i32>">
+    // CHECK: %[[T2:.*]] = llvm.add %[[OFFSET_V2]], %[[T1]] : !llvm<"<2 x i32>">
+    // CHECK: %{{.*}} = llvm.ashr %[[SHIFTED_LEFT]], %[[T2]] : !llvm<"<2 x i32>">
+    %0 = spv.BitFieldSExtract %base, %offset, %count : vector<2xi32>, i32, i32
+    return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.BitFieldUExtract
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func @bitfield_uextract_scalar_same_bit_width
+// CHECK-SAME: %[[BASE:.*]]: !llvm.i32, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i32
+func @bitfield_uextract_scalar_same_bit_width(%base: i32, %offset: i32, %count: i32) {
+    // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i32) : !llvm.i32
+    // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT]] : !llvm.i32
+    // CHECK: %[[MASK:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i32
+    // CHECK: %[[SHIFTED_BASE:.*]] = llvm.lshr %[[BASE]], %[[OFFSET]] : !llvm.i32
+    // CHECK: %{{.*}} = llvm.and %[[SHIFTED_BASE]], %[[MASK]] : !llvm.i32
+    %0 = spv.BitFieldUExtract %base, %offset, %count : i32, i32, i32
+    return
+}
+
+// CHECK-LABEL: func @bitfield_uextract_scalar_smaller_bit_width
+// CHECK-SAME: %[[BASE:.*]]: !llvm.i32, %[[OFFSET:.*]]: !llvm.i16, %[[COUNT:.*]]: !llvm.i8
+func @bitfield_uextract_scalar_smaller_bit_width(%base: i32, %offset: i16, %count: i8) {
+    // CHECK: %[[EXT_OFFSET:.*]] = llvm.zext %[[OFFSET]] : !llvm.i16 to !llvm.i32
+    // CHECK: %[[EXT_COUNT:.*]] = llvm.zext %[[COUNT]] : !llvm.i8 to !llvm.i32
+    // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i32) : !llvm.i32
+    // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[EXT_COUNT]] : !llvm.i32
+    // CHECK: %[[MASK:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i32
+    // CHECK: %[[SHIFTED_BASE:.*]] = llvm.lshr %[[BASE]], %[[EXT_OFFSET]] : !llvm.i32
+    // CHECK: %{{.*}} = llvm.and %[[SHIFTED_BASE]], %[[MASK]] : !llvm.i32
+    %0 = spv.BitFieldUExtract %base, %offset, %count : i32, i16, i8
+    return
+}
+
+// CHECK-LABEL: func @bitfield_uextract_scalar_greater_bit_width
+// CHECK-SAME: %[[BASE:.*]]: !llvm.i8, %[[OFFSET:.*]]: !llvm.i16, %[[COUNT:.*]]: !llvm.i8
+func @bitfield_uextract_scalar_greater_bit_width(%base: i8, %offset: i16, %count: i8) {
+    // CHECK: %[[TRUNC_OFFSET:.*]] = llvm.trunc %[[OFFSET]] : !llvm.i16 to !llvm.i8
+    // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i8) : !llvm.i8
+    // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT]] : !llvm.i8
+    // CHECK: %[[MASK:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i8
+    // CHECK: %[[SHIFTED_BASE:.*]] = llvm.lshr %[[BASE]], %[[TRUNC_OFFSET]] : !llvm.i8
+    // CHECK: %{{.*}} = llvm.and %[[SHIFTED_BASE]], %[[MASK]] : !llvm.i8
+    %0 = spv.BitFieldUExtract %base, %offset, %count : i8, i16, i8
+    return
+}
+
+// CHECK-LABEL: func @bitfield_uextract_vector
+// CHECK-SAME: %[[BASE:.*]]: !llvm<"<2 x i32>">, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i32
+func @bitfield_uextract_vector(%base: vector<2xi32>, %offset: i32, %count: i32) {
+    // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+    // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
+    // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+    // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+    // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
+    // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+    // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
+    // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+    // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+    // CHECK: %[[COUNT_V2:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
+    // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(dense<-1> : vector<2xi32>) : !llvm<"<2 x i32>">
+    // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT_V2]] : !llvm<"<2 x i32>">
+    // CHECK: %[[MASK:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm<"<2 x i32>">
+    // CHECK: %[[SHIFTED_BASE:.*]] = llvm.lshr %[[BASE]], %[[OFFSET_V2]] : !llvm<"<2 x i32>">
+    // CHECK: %{{.*}} = llvm.and %[[SHIFTED_BASE]], %[[MASK]] : !llvm<"<2 x i32>">
+    %0 = spv.BitFieldUExtract %base, %offset, %count : vector<2xi32>, i32, i32
+    return
+}
+
 //===----------------------------------------------------------------------===//
 // spv.BitwiseAnd
 //===----------------------------------------------------------------------===//
@@ -47,3 +292,21 @@ func @bitwise_xor_vector(%arg0: vector<2xi16>, %arg1: vector<2xi16>) {
 	%0 = spv.BitwiseXor %arg0, %arg1 : vector<2xi16>
 	return
 }
+
+//===----------------------------------------------------------------------===//
+// spv.Not
+//===----------------------------------------------------------------------===//
+
+func @not__scalar(%arg0: i32) {
+  // CHECK: %[[CONST:.*]] = llvm.mlir.constant(-1 : i32) : !llvm.i32
+  // CHECK: %{{.*}} = llvm.xor %{{.*}}, %[[CONST]] : !llvm.i32
+	%0 = spv.Not %arg0 : i32
+  return
+}
+
+func @not_vector(%arg0: vector<2xi16>) {
+  // CHECK: %[[CONST:.*]] = llvm.mlir.constant(dense<-1> : vector<2xi16>) : !llvm<"<2 x i16>">
+  // CHECK: %{{.*}} = llvm.xor %{{.*}}, %[[CONST]] : !llvm<"<2 x i16>">
+	%0 = spv.Not %arg0 : vector<2xi16>
+  return
+}
diff --git a/mlir/test/Conversion/SPIRVToLLVM/cast-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/cast-ops-to-llvm.mlir
index 6a772245e8698..29f43fadf933c 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/cast-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/cast-ops-to-llvm.mlir
@@ -1,5 +1,45 @@
 // RUN: mlir-opt -convert-spirv-to-llvm %s | FileCheck %s
 
+//===----------------------------------------------------------------------===//
+// spv.Bitcast
+//===----------------------------------------------------------------------===//
+
+func @bitcast_float_to_integer_scalar(%arg0 : f32) {
+	// CHECK: {{.*}} = llvm.bitcast {{.*}} : !llvm.float to !llvm.i32
+	%0 = spv.Bitcast %arg0: f32 to i32
+	return
+}
+
+func @bitcast_float_to_integer_vector(%arg0 : vector<3xf32>) {
+	// CHECK: {{.*}} = llvm.bitcast {{.*}} : !llvm<"<3 x float>"> to !llvm<"<3 x i32>">
+	%0 = spv.Bitcast %arg0: vector<3xf32> to vector<3xi32>
+	return
+}
+
+func @bitcast_vector_to_scalar(%arg0 : vector<2xf32>) {
+	// CHECK: {{.*}} = llvm.bitcast {{.*}} : !llvm<"<2 x float>"> to !llvm.i64
+	%0 = spv.Bitcast %arg0: vector<2xf32> to i64
+	return
+}
+
+func @bitcast_scalar_to_vector(%arg0 : f64) {
+	// CHECK: {{.*}} = llvm.bitcast {{.*}} : !llvm.double to !llvm<"<2 x i32>">
+	%0 = spv.Bitcast %arg0: f64 to vector<2xi32>
+	return
+}
+
+func @bitcast_vector_to_vector(%arg0 : vector<4xf32>) {
+	// CHECK: {{.*}} = llvm.bitcast {{.*}} : !llvm<"<4 x float>"> to !llvm<"<2 x i64>">
+	%0 = spv.Bitcast %arg0: vector<4xf32> to vector<2xi64>
+	return
+}
+
+func @bitcast_pointer(%arg0: !spv.ptr<f32, Function>) {
+	// CHECK: %{{.*}} = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"i32*">
+	%0 = spv.Bitcast %arg0 : !spv.ptr<f32, Function> to !spv.ptr<i32, Function>
+	return
+}
+
 //===----------------------------------------------------------------------===//
 // spv.ConvertFToS
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Conversion/SPIRVToLLVM/constant-op-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/constant-op-to-llvm.mlir
new file mode 100644
index 0000000000000..b9605e7918577
--- /dev/null
+++ b/mlir/test/Conversion/SPIRVToLLVM/constant-op-to-llvm.mlir
@@ -0,0 +1,55 @@
+// RUN: mlir-opt -convert-spirv-to-llvm %s | FileCheck %s
+
+//===----------------------------------------------------------------------===//
+// spv.constant
+//===----------------------------------------------------------------------===//
+
+func @bool_constant_scalar() {
+  // CHECK: {{.*}} = llvm.mlir.constant(true) : !llvm.i1
+  %0 = spv.constant true
+  // CHECK: {{.*}} = llvm.mlir.constant(false) : !llvm.i1
+  %1 = spv.constant false
+  return
+}
+
+func @bool_constant_vector() {
+  // CHECK: {{.*}} = llvm.mlir.constant(dense<[true, false]> : vector<2xi1>) : !llvm<"<2 x i1>">
+  %0 = constant dense<[true, false]> : vector<2xi1>
+  // CHECK: {{.*}} = llvm.mlir.constant(dense<false> : vector<3xi1>) : !llvm<"<3 x i1>">
+  %1 = constant dense<false> : vector<3xi1>
+  return
+}
+
+func @integer_constant_scalar() {
+  // CHECK: {{.*}} = llvm.mlir.constant(0 : i8) : !llvm.i8
+  %0 = spv.constant  0 : i8
+  // CHECK: {{.*}} = llvm.mlir.constant(-5 : i64) : !llvm.i64
+  %1 = spv.constant -5 : si64
+  // CHECK: {{.*}} = llvm.mlir.constant(10 : i16) : !llvm.i16
+  %2 = spv.constant  10 : ui16
+  return
+}
+
+func @integer_constant_vector() {
+  // CHECK: {{.*}} = llvm.mlir.constant(dense<[2, 3]> : vector<2xi32>) : !llvm<"<2 x i32>">
+  %0 = spv.constant dense<[2, 3]> : vector<2xi32>
+  // CHECK: {{.*}} = llvm.mlir.constant(dense<-4> : vector<2xi32>) : !llvm<"<2 x i32>">
+  %1 = spv.constant dense<-4> : vector<2xsi32>
+  // CHECK: {{.*}} = llvm.mlir.constant(dense<[2, 3, 4]> : vector<3xi32>) : !llvm<"<3 x i32>">
+  %2 = spv.constant dense<[2, 3, 4]> : vector<3xui32>
+  return
+}
+
+func @float_constant_scalar() {
+  // CHECK: {{.*}} = llvm.mlir.constant(5.000000e+00 : f16) : !llvm.half
+  %0 = spv.constant 5.000000e+00 : f16
+  // CHECK: {{.*}} = llvm.mlir.constant(5.000000e+00 : f64) : !llvm.double
+  %1 = spv.constant 5.000000e+00 : f64
+  return
+}
+
+func @float_constant_vector() {
+  // CHECK: {{.*}} = llvm.mlir.constant(dense<[2.000000e+00, 3.000000e+00]> : vector<2xf32>) : !llvm<"<2 x float>">
+  %0 = spv.constant dense<[2.000000e+00, 3.000000e+00]> : vector<2xf32>
+  return
+}
diff --git a/mlir/test/Conversion/SPIRVToLLVM/func-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/func-to-llvm.mlir
new file mode 100644
index 0000000000000..23174f34fcecf
--- /dev/null
+++ b/mlir/test/Conversion/SPIRVToLLVM/func-to-llvm.mlir
@@ -0,0 +1,93 @@
+// RUN: mlir-opt -convert-spirv-to-llvm %s | FileCheck %s
+
+//===----------------------------------------------------------------------===//
+// spv.Return
+//===----------------------------------------------------------------------===//
+
+func @return() {
+	// CHECK: llvm.return
+	spv.Return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.ReturnValue
+//===----------------------------------------------------------------------===//
+
+func @return_value(%arg: i32) {
+	// CHECK: llvm.return %{{.*}} : !llvm.i32
+	spv.ReturnValue %arg : i32
+}
+
+//===----------------------------------------------------------------------===//
+// spv.func
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: llvm.func @none()
+spv.func @none() -> () "None" {
+	spv.Return
+}
+
+// CHECK-LABEL: llvm.func @inline() attributes {passthrough = ["alwaysinline"]}
+spv.func @inline() -> () "Inline" {
+	spv.Return
+}
+
+// CHECK-LABEL: llvm.func @dont_inline() attributes {passthrough = ["noinline"]}
+spv.func @dont_inline() -> () "DontInline" {
+	spv.Return
+}
+
+// CHECK-LABEL: llvm.func @pure() attributes {passthrough = ["readonly"]}
+spv.func @pure() -> () "Pure" {
+	spv.Return
+}
+
+// CHECK-LABEL: llvm.func @const() attributes {passthrough = ["readnone"]}
+spv.func @const() -> () "Const" {
+	spv.Return
+}
+
+// CHECK-LABEL: llvm.func @scalar_types(%arg0: !llvm.i32, %arg1: !llvm.i1, %arg2: !llvm.double, %arg3: !llvm.float)
+spv.func @scalar_types(%arg0: i32, %arg1: i1, %arg2: f64, %arg3: f32) -> () "None" {
+	spv.Return
+}
+
+// CHECK-LABEL: llvm.func @vector_types(%arg0: !llvm<"<2 x i64>">, %arg1: !llvm<"<2 x i64>">) -> !llvm<"<2 x i64>">
+spv.func @vector_types(%arg0: vector<2xi64>, %arg1: vector<2xi64>) -> vector<2xi64> "None" {
+	%0 = spv.IAdd %arg0, %arg1 : vector<2xi64>
+	spv.ReturnValue %0 : vector<2xi64>
+}
+
+//===----------------------------------------------------------------------===//
+// spv.FunctionCall
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: llvm.func @function_calls
+// CHECK-SAME: %[[ARG0:.*]]: !llvm.i32, %[[ARG1:.*]]: !llvm.i1, %[[ARG2:.*]]: !llvm.double, %[[ARG3:.*]]: !llvm<"<2 x i64>">, %[[ARG4:.*]]: !llvm<"<2 x float>">
+spv.func @function_calls(%arg0: i32, %arg1: i1, %arg2: f64, %arg3: vector<2xi64>, %arg4: vector<2xf32>) -> () "None" {
+	// CHECK: llvm.call @void_1() : () -> ()
+	spv.FunctionCall @void_1() : () -> ()
+	// CHECK: llvm.call @void_2(%[[ARG3]]) : (!llvm<"<2 x i64>">) -> ()
+	spv.FunctionCall @void_2(%arg3) : (vector<2xi64>) -> ()
+	// CHECK: %{{.*}} = llvm.call @value_scalar(%[[ARG0]], %[[ARG1]], %[[ARG2]]) : (!llvm.i32, !llvm.i1, !llvm.double) -> !llvm.i32
+	%0 = spv.FunctionCall @value_scalar(%arg0, %arg1, %arg2) : (i32, i1, f64) -> i32
+	// CHECK: %{{.*}} = llvm.call @value_vector(%[[ARG3]], %[[ARG4]]) : (!llvm<"<2 x i64>">, !llvm<"<2 x float>">) -> !llvm<"<2 x float>">
+	%1 = spv.FunctionCall @value_vector(%arg3, %arg4) : (vector<2xi64>, vector<2xf32>) -> vector<2xf32>
+	spv.Return
+}
+
+spv.func @void_1() -> () "None" {
+  spv.Return
+}
+
+spv.func @void_2(%arg0: vector<2xi64>) -> () "None" {
+  spv.Return
+}
+
+spv.func @value_scalar(%arg0: i32, %arg1: i1, %arg2: f64) -> i32 "None" {
+  spv.ReturnValue %arg0: i32
+}
+
+spv.func @value_vector(%arg0: vector<2xi64>, %arg1: vector<2xf32>) -> vector<2xf32> "None" {
+  spv.ReturnValue %arg1: vector<2xf32>
+}
diff --git a/mlir/test/Conversion/SPIRVToLLVM/logical-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/logical-to-llvm.mlir
index a6ff260711dee..e6f2ec2433fa5 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/logical-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/logical-to-llvm.mlir
@@ -32,6 +32,24 @@ func @logical_not_equal_vector(%arg0: vector<4xi1>, %arg1: vector<4xi1>) {
   return
 }
 
+//===----------------------------------------------------------------------===//
+// spv.LogicalNot
+//===----------------------------------------------------------------------===//
+
+func @logical_not__scalar(%arg0: i1) {
+  // CHECK: %[[CONST:.*]] = llvm.mlir.constant(true) : !llvm.i1
+  // CHECK: %{{.*}} = llvm.xor %{{.*}}, %[[CONST]] : !llvm.i1
+	%0 = spv.LogicalNot %arg0 : i1
+  return
+}
+
+func @logical_not_vector(%arg0: vector<4xi1>) {
+  // CHECK: %[[CONST:.*]] = llvm.mlir.constant(dense<true> : vector<4xi1>) : !llvm<"<4 x i1>">
+  // CHECK: %{{.*}} = llvm.xor %{{.*}}, %[[CONST]] : !llvm<"<4 x i1>">
+	%0 = spv.LogicalNot %arg0 : vector<4xi1>
+  return
+}
+
 //===----------------------------------------------------------------------===//
 // spv.LogicalAnd
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Conversion/SPIRVToLLVM/misc-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/misc-ops-to-llvm.mlir
new file mode 100644
index 0000000000000..8f407ba8f596c
--- /dev/null
+++ b/mlir/test/Conversion/SPIRVToLLVM/misc-ops-to-llvm.mlir
@@ -0,0 +1,35 @@
+// RUN: mlir-opt -convert-spirv-to-llvm %s | FileCheck %s
+
+//===----------------------------------------------------------------------===//
+// spv.Select
+//===----------------------------------------------------------------------===//
+
+func @select_scalar(%arg0: i1, %arg1: vector<3xi32>, %arg2: f32) {
+	// CHECK: %{{.*}} = llvm.select %{{.*}}, %{{.*}}, %{{.*}} : !llvm.i1, !llvm<"<3 x i32>">
+	%0 = spv.Select %arg0, %arg1, %arg1 : i1, vector<3xi32>
+	// CHECK: %{{.*}} = llvm.select %{{.*}}, %{{.*}}, %{{.*}} : !llvm.i1, !llvm.float
+	%1 = spv.Select %arg0, %arg2, %arg2 : i1, f32
+  return
+}
+
+func @select_vector(%arg0: vector<2xi1>, %arg1: vector<2xi32>) {
+	// CHECK: %{{.*}} = llvm.select %{{.*}}, %{{.*}}, %{{.*}} : !llvm<"<2 x i1>">, !llvm<"<2 x i32>">
+	%0 = spv.Select %arg0, %arg1, %arg1 : vector<2xi1>, vector<2xi32>
+  return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.Undef
+//===----------------------------------------------------------------------===//
+
+func @undef_scalar() {
+	// CHECK: %{{.*}} = llvm.mlir.undef : !llvm.float
+	%0 = spv.undef : f32
+	return
+}
+
+func @undef_vector() {
+	// CHECK: %{{.*}} = llvm.mlir.undef : !llvm<"<2 x i32>">
+	%0 = spv.undef : vector<2xi32>
+	return
+}
diff --git a/mlir/test/Conversion/SPIRVToLLVM/module-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/module-ops-to-llvm.mlir
new file mode 100644
index 0000000000000..b8169a19ee223
--- /dev/null
+++ b/mlir/test/Conversion/SPIRVToLLVM/module-ops-to-llvm.mlir
@@ -0,0 +1,26 @@
+// RUN: mlir-opt -convert-spirv-to-llvm %s | FileCheck %s
+
+//===----------------------------------------------------------------------===//
+// spv.module
+//===----------------------------------------------------------------------===//
+
+// CHECK: module
+spv.module Logical GLSL450 {}
+
+// CHECK: module
+spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], [SPV_KHR_16bit_storage]> {}
+
+// CHECK: module
+spv.module Logical GLSL450 {
+	// CHECK: }
+  spv._module_end
+}
+
+// CHECK: module
+spv.module Logical GLSL450 {
+	// CHECK-LABEL: llvm.func @empty()
+  spv.func @empty() -> () "None" {
+		// CHECK: llvm.return
+    spv.Return
+  }
+}
diff --git a/mlir/test/Conversion/SPIRVToLLVM/spirv-types-to-llvm.invalid.mlir b/mlir/test/Conversion/SPIRVToLLVM/spirv-types-to-llvm.invalid.mlir
new file mode 100644
index 0000000000000..96fb9f44af578
--- /dev/null
+++ b/mlir/test/Conversion/SPIRVToLLVM/spirv-types-to-llvm.invalid.mlir
@@ -0,0 +1,27 @@
+// RUN: mlir-opt %s -convert-spirv-to-llvm -verify-diagnostics -split-input-file
+
+// expected-error@+1 {{failed to legalize operation 'spv.func' that was explicitly marked illegal}}
+spv.func @array_with_stride(%arg: !spv.array<4 x f32, stride=4>) -> () "None" {
+  spv.Return
+}
+
+// -----
+
+// expected-error@+1 {{failed to legalize operation 'spv.func' that was explicitly marked illegal}}
+spv.func @struct_with_offset1(%arg: !spv.struct<i32[0], i32[4]>) -> () "None" {
+  spv.Return
+}
+
+// -----
+
+// expected-error@+1 {{failed to legalize operation 'spv.func' that was explicitly marked illegal}}
+spv.func @struct_with_offset2(%arg: !spv.struct<i32[0], i32[8]>) -> () "None" {
+  spv.Return
+}
+
+// -----
+
+// expected-error@+1 {{failed to legalize operation 'spv.func' that was explicitly marked illegal}}
+spv.func @struct_with_decorations(%arg: !spv.struct<f32 [RelaxedPrecision]>) -> () "None" {
+  spv.Return
+}
diff --git a/mlir/test/Conversion/SPIRVToLLVM/spirv-types-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/spirv-types-to-llvm.mlir
new file mode 100644
index 0000000000000..466f8ad4a9eac
--- /dev/null
+++ b/mlir/test/Conversion/SPIRVToLLVM/spirv-types-to-llvm.mlir
@@ -0,0 +1,38 @@
+// RUN: mlir-opt -split-input-file -convert-spirv-to-llvm -verify-diagnostics %s | FileCheck %s
+
+//===----------------------------------------------------------------------===//
+// Array type
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @array(!llvm<"[16 x float]">, !llvm<"[32 x <4 x float>]">)
+func @array(!spv.array<16xf32>, !spv.array< 32 x vector<4xf32> >) -> ()
+
+//===----------------------------------------------------------------------===//
+// Pointer type
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @pointer_scalar(!llvm<"i1*">, !llvm<"float*">)
+func @pointer_scalar(!spv.ptr<i1, Uniform>, !spv.ptr<f32, Private>) -> ()
+
+// CHECK-LABEL: @pointer_vector(!llvm<"<4 x i32>*">)
+func @pointer_vector(!spv.ptr<vector<4xi32>, Function>) -> ()
+
+//===----------------------------------------------------------------------===//
+// Runtime array type
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @runtime_array_vector(!llvm<"[0 x <4 x float>]">)
+func @runtime_array_vector(!spv.rtarray< vector<4xf32> >) -> ()
+
+// CHECK-LABEL: @runtime_array_scalar(!llvm<"[0 x float]">)
+func @runtime_array_scalar(!spv.rtarray<f32>) -> ()
+
+//===----------------------------------------------------------------------===//
+// Struct type
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @struct(!llvm<"<{ double }>">)
+func @struct(!spv.struct<f64>) -> ()
+
+// CHECK-LABEL: @struct_nested(!llvm<"<{ i32, <{ i64, i32 }> }>">)
+func @struct_nested(!spv.struct<i32, !spv.struct<i64, i32>>)
diff --git a/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir b/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir
index b52266ce82ec3..1c214567c63ab 100644
--- a/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir
+++ b/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir
@@ -26,3 +26,25 @@ func @shape_reduce(%shape : !shape.shape) -> !shape.size {
 // CHECK-NEXT:   scf.yield [[NEW_ACC]] : !shape.size
 // CHECK-NEXT: }
 // CHECK-NEXT: return [[RESULT]] : !shape.size
+
+// -----
+
+// Lower `shape_of` for unranked tensors.
+// CHECK-LABEL: @shape_of_unranked
+// CHECK-SAME: (%[[ARG:.*]]: tensor<*xf32>)
+func @shape_of_unranked(%arg : tensor<*xf32>) {
+  // CHECK-DAG: %[[RANK:.*]] = rank %[[ARG]] : tensor<*xf32>
+  // CHECK-DAG: %[[SHAPE_MEM:.*]] = alloca(%[[RANK]]) : memref<?xi64>
+  // CHECK-DAG: %[[C0:.*]] = constant 0 : index
+  // CHECK-DAG: %[[C1:.*]] = constant 1 : index
+  // CHECK:     scf.for %[[I:.*]] = %[[C0]] to %[[RANK]] step %[[C1]] {
+  // CHECK-DAG:   %[[DIM:.]] = dim %[[ARG]], %[[I]] : tensor<*xf32>
+  // CHECK-DAG:   %[[DIM_INT:.*]] = index_cast %[[DIM]] : index to i64
+  // CHECK-DAG:   store %[[DIM_INT]], %[[SHAPE_MEM]][%[[I]]] : memref<?xi64>
+  // CHECK:     }
+  // CHECK-DAG: %[[SHAPE_INT:.*]] = tensor_load %[[SHAPE_MEM]] : memref<?xi64>
+  // CHECK-DAG: %[[SHAPE:.*]] = index_cast %[[SHAPE_INT]] : tensor<?xi64> to tensor<?xindex>
+  %shape = shape.shape_of %arg : tensor<*xf32>
+  return
+}
+
diff --git a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
index bfe3c2b599c55..28ef190d09eba 100644
--- a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
+++ b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
@@ -86,7 +86,6 @@ func @size_const() -> !shape.size {
 }
 // CHECK: %[[C1:.*]] = constant 1 : index
 // CHECK: return %[[C1]] : index
-
 // -----
 
 // Lower `shape_of` for statically shaped tensor.
@@ -115,3 +114,47 @@ func @shape_of_dyn(%arg : tensor<1x5x?xf32>) {
   %shape = shape.shape_of %arg : tensor<1x5x?xf32>
   return
 }
+
+// -----
+
+// Convert `rank` to `dim` of the first dimension.
+// CHECK-LABEL: @rank
+// CHECK-SAME: (%[[SHAPE:.*]]: tensor<?xindex>) -> index
+func @rank(%shape : !shape.shape) -> !shape.size {
+  // CHECK-DAG: %[[C0:.*]] = constant 0 : index
+  // CHECK-DAG: %[[RESULT:.*]] = dim %[[SHAPE]], %[[C0]]
+  // CHECK-DAG: return %[[RESULT]] : index
+  %rank = shape.rank %shape
+  return %rank : !shape.size
+}
+
+// -----
+
+// Express `get_extent` as `std.dim` when it relies directly on the outcome of a
+// `shape_of` operation.
+// CHECK-LABEL: @get_extent_shape_of
+// CHECK-SAME:  (%[[ARG:.*]]: tensor<2x3xf32>, %[[IDX:.*]]: index) -> index
+func @get_extent_shape_of(%arg : tensor<2x3xf32>, %idx : !shape.size)
+    -> !shape.size {
+  // CHECK: %[[RESULT:.*]] = dim %[[ARG]], %[[IDX]] : tensor<2x3xf32>
+  // CHECK: return %[[RESULT]] : index
+  %shape = shape.shape_of %arg : tensor<2x3xf32>
+  %result = shape.get_extent %shape, %idx
+  return %result : !shape.size
+}
+
+// -----
+
+// Express `get_extent` as `std.extract_element` when it relies directly on the
+// outcome of a `from_extent_tensor` operation.
+// CHECK-LABEL: @get_extent_from_extent_tensor
+// CHECK-SAME: (%[[EXTENTS:.*]]: tensor<?xindex>, %[[IDX:.*]]: index) -> index
+func @get_extent_from_extent_tensor(%extents : tensor<?xindex>,
+                                    %idx : !shape.size) -> !shape.size {
+  // CHECK: %[[RESULT:.*]] = extract_element %[[EXTENTS]][%[[IDX]]] : tensor<?xindex>
+  // CHECK: return %[[RESULT]] : index
+  %shape = shape.from_extent_tensor %extents : tensor<?xindex>
+  %result = shape.get_extent %shape, %idx
+  return %result : !shape.size
+}
+
diff --git a/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir b/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir
index 87bdab2680f99..e17bf3e24225a 100644
--- a/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir
@@ -109,3 +109,134 @@ func @other_callee(%arg0: memref<?xf32>, %arg1: index) attributes { llvm.emit_c_
 
 // EMIT_C_ATTRIBUTE: @_mlir_ciface_other_callee
 // EMIT_C_ATTRIBUTE:   llvm.call @other_callee
+
+//===========================================================================//
+// Calling convention on returning unranked memrefs.
+//===========================================================================//
+
+// CHECK-LABEL: llvm.func @return_var_memref_caller
+func @return_var_memref_caller(%arg0: memref<4x3xf32>) {
+  // CHECK: %[[CALL_RES:.*]] = llvm.call @return_var_memref
+  %0 = call @return_var_memref(%arg0) : (memref<4x3xf32>) -> memref<*xf32>
+
+  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : index)
+  // CHECK: %[[TWO:.*]] = llvm.mlir.constant(2 : index)
+  // These sizes may depend on the data layout, not matching specific values.
+  // CHECK: %[[PTR_SIZE:.*]] = llvm.mlir.constant
+  // CHECK: %[[IDX_SIZE:.*]] = llvm.mlir.constant
+
+  // CHECK: %[[DOUBLE_PTR_SIZE:.*]] = llvm.mul %[[TWO]], %[[PTR_SIZE]]
+  // CHECK: %[[RANK:.*]] = llvm.extractvalue %[[CALL_RES]][0] : !llvm<"{ i64, i8* }">
+  // CHECK: %[[DOUBLE_RANK:.*]] = llvm.mul %[[TWO]], %[[RANK]]
+  // CHECK: %[[DOUBLE_RANK_INC:.*]] = llvm.add %[[DOUBLE_RANK]], %[[ONE]]
+  // CHECK: %[[TABLES_SIZE:.*]] = llvm.mul %[[DOUBLE_RANK_INC]], %[[IDX_SIZE]]
+  // CHECK: %[[ALLOC_SIZE:.*]] = llvm.add %[[DOUBLE_PTR_SIZE]], %[[TABLES_SIZE]]
+  // CHECK: %[[FALSE:.*]] = llvm.mlir.constant(false)
+  // CHECK: %[[ALLOCA:.*]] = llvm.alloca %[[ALLOC_SIZE]] x !llvm.i8
+  // CHECK: %[[SOURCE:.*]] = llvm.extractvalue %[[CALL_RES]][1]
+  // CHECK: "llvm.intr.memcpy"(%[[ALLOCA]], %[[SOURCE]], %[[ALLOC_SIZE]], %[[FALSE]])
+  // CHECK: llvm.call @free(%[[SOURCE]])
+  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ i64, i8* }">
+  // CHECK: %[[RANK:.*]] = llvm.extractvalue %[[CALL_RES]][0] : !llvm<"{ i64, i8* }">
+  // CHECK: %[[DESC_1:.*]] = llvm.insertvalue %[[RANK]], %[[DESC]][0]
+  // CHECK: llvm.insertvalue %[[ALLOCA]], %[[DESC_1]][1]
+  return
+}
+
+// CHECK-LABEL: llvm.func @return_var_memref
+func @return_var_memref(%arg0: memref<4x3xf32>) -> memref<*xf32> {
+  // Match the construction of the unranked descriptor.
+  // CHECK: %[[ALLOCA:.*]] = llvm.alloca
+  // CHECK: %[[MEMORY:.*]] = llvm.bitcast %[[ALLOCA]]
+  // CHECK: %[[DESC_0:.*]] = llvm.mlir.undef : !llvm<"{ i64, i8* }">
+  // CHECK: %[[DESC_1:.*]] = llvm.insertvalue %{{.*}}, %[[DESC_0]][0]
+  // CHECK: %[[DESC_2:.*]] = llvm.insertvalue %[[MEMORY]], %[[DESC_1]][1]
+  %0 = memref_cast %arg0: memref<4x3xf32> to memref<*xf32>
+
+  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : index)
+  // CHECK: %[[TWO:.*]] = llvm.mlir.constant(2 : index)
+  // These sizes may depend on the data layout, not matching specific values.
+  // CHECK: %[[PTR_SIZE:.*]] = llvm.mlir.constant
+  // CHECK: %[[IDX_SIZE:.*]] = llvm.mlir.constant
+
+  // CHECK: %[[DOUBLE_PTR_SIZE:.*]] = llvm.mul %[[TWO]], %[[PTR_SIZE]]
+  // CHECK: %[[RANK:.*]] = llvm.extractvalue %[[DESC_2]][0] : !llvm<"{ i64, i8* }">
+  // CHECK: %[[DOUBLE_RANK:.*]] = llvm.mul %[[TWO]], %[[RANK]]
+  // CHECK: %[[DOUBLE_RANK_INC:.*]] = llvm.add %[[DOUBLE_RANK]], %[[ONE]]
+  // CHECK: %[[TABLES_SIZE:.*]] = llvm.mul %[[DOUBLE_RANK_INC]], %[[IDX_SIZE]]
+  // CHECK: %[[ALLOC_SIZE:.*]] = llvm.add %[[DOUBLE_PTR_SIZE]], %[[TABLES_SIZE]]
+  // CHECK: %[[FALSE:.*]] = llvm.mlir.constant(false)
+  // CHECK: %[[ALLOCATED:.*]] = llvm.call @malloc(%[[ALLOC_SIZE]])
+  // CHECK: %[[SOURCE:.*]] = llvm.extractvalue %[[DESC_2]][1]
+  // CHECK: "llvm.intr.memcpy"(%[[ALLOCATED]], %[[SOURCE]], %[[ALLOC_SIZE]], %[[FALSE]])
+  // CHECK: %[[NEW_DESC:.*]] = llvm.mlir.undef : !llvm<"{ i64, i8* }">
+  // CHECK: %[[RANK:.*]] = llvm.extractvalue %[[DESC_2]][0] : !llvm<"{ i64, i8* }">
+  // CHECK: %[[NEW_DESC_1:.*]] = llvm.insertvalue %[[RANK]], %[[NEW_DESC]][0]
+  // CHECK: %[[NEW_DESC_2:.*]] = llvm.insertvalue %[[ALLOCATED]], %[[NEW_DESC_1]][1]
+  // CHECL: llvm.return %[[NEW_DESC_2]]
+  return %0 : memref<*xf32>
+}
+
+// CHECK-LABEL: llvm.func @return_two_var_memref_caller
+func @return_two_var_memref_caller(%arg0: memref<4x3xf32>) {
+  // Only check that we create two different descriptors using different
+  // memory, and deallocate both sources. The size computation is same as for
+  // the single result.
+  // CHECK: %[[CALL_RES:.*]] = llvm.call @return_two_var_memref
+  // CHECK: %[[RES_1:.*]] = llvm.extractvalue %[[CALL_RES]][0]
+  // CHECK: %[[RES_2:.*]] = llvm.extractvalue %[[CALL_RES]][1]
+  %0:2 = call @return_two_var_memref(%arg0) : (memref<4x3xf32>) -> (memref<*xf32>, memref<*xf32>)
+
+  // CHECK: %[[ALLOCA_1:.*]] = llvm.alloca %{{.*}} x !llvm.i8
+  // CHECK: %[[SOURCE_1:.*]] = llvm.extractvalue %[[RES_1:.*]][1] : ![[DESC_TYPE:.*]]
+  // CHECK: "llvm.intr.memcpy"(%[[ALLOCA_1]], %[[SOURCE_1]], %{{.*}}, %[[FALSE:.*]])
+  // CHECK: llvm.call @free(%[[SOURCE_1]])
+  // CHECK: %[[DESC_1:.*]] = llvm.mlir.undef : ![[DESC_TYPE]]
+  // CHECK: %[[DESC_11:.*]] = llvm.insertvalue %{{.*}}, %[[DESC_1]][0]
+  // CHECK: llvm.insertvalue %[[ALLOCA_1]], %[[DESC_11]][1]
+
+  // CHECK: %[[ALLOCA_2:.*]] = llvm.alloca %{{.*}} x !llvm.i8
+  // CHECK: %[[SOURCE_2:.*]] = llvm.extractvalue %[[RES_2:.*]][1]
+  // CHECK: "llvm.intr.memcpy"(%[[ALLOCA_2]], %[[SOURCE_2]], %{{.*}}, %[[FALSE]])
+  // CHECK: llvm.call @free(%[[SOURCE_2]])
+  // CHECK: %[[DESC_2:.*]] = llvm.mlir.undef : ![[DESC_TYPE]]
+  // CHECK: %[[DESC_21:.*]] = llvm.insertvalue %{{.*}}, %[[DESC_2]][0]
+  // CHECK: llvm.insertvalue %[[ALLOCA_2]], %[[DESC_21]][1]
+  return
+}
+
+// CHECK-LABEL: llvm.func @return_two_var_memref
+func @return_two_var_memref(%arg0: memref<4x3xf32>) -> (memref<*xf32>, memref<*xf32>) {
+  // Match the construction of the unranked descriptor.
+  // CHECK: %[[ALLOCA:.*]] = llvm.alloca
+  // CHECK: %[[MEMORY:.*]] = llvm.bitcast %[[ALLOCA]]
+  // CHECK: %[[DESC_0:.*]] = llvm.mlir.undef : !llvm<"{ i64, i8* }">
+  // CHECK: %[[DESC_1:.*]] = llvm.insertvalue %{{.*}}, %[[DESC_0]][0]
+  // CHECK: %[[DESC_2:.*]] = llvm.insertvalue %[[MEMORY]], %[[DESC_1]][1]
+  %0 = memref_cast %arg0 : memref<4x3xf32> to memref<*xf32>
+
+  // Only check that we allocate the memory for each operand of the "return"
+  // separately, even if both operands are the same value. The calling
+  // convention requires the caller to free them and the caller cannot know
+  // whether they are the same value or not.
+  // CHECK: %[[ALLOCATED_1:.*]] = llvm.call @malloc(%{{.*}})
+  // CHECK: %[[SOURCE_1:.*]] = llvm.extractvalue %[[DESC_2]][1]
+  // CHECK: "llvm.intr.memcpy"(%[[ALLOCATED_1]], %[[SOURCE_1]], %{{.*}}, %[[FALSE:.*]])
+  // CHECK: %[[RES_1:.*]] = llvm.mlir.undef
+  // CHECK: %[[RES_11:.*]] = llvm.insertvalue %{{.*}}, %[[RES_1]][0]
+  // CHECK: %[[RES_12:.*]] = llvm.insertvalue %[[ALLOCATED_1]], %[[RES_11]][1]
+
+  // CHECK: %[[ALLOCATED_2:.*]] = llvm.call @malloc(%{{.*}})
+  // CHECK: %[[SOURCE_2:.*]] = llvm.extractvalue %[[DESC_2]][1]
+  // CHECK: "llvm.intr.memcpy"(%[[ALLOCATED_2]], %[[SOURCE_2]], %{{.*}}, %[[FALSE]])
+  // CHECK: %[[RES_2:.*]] = llvm.mlir.undef
+  // CHECK: %[[RES_21:.*]] = llvm.insertvalue %{{.*}}, %[[RES_2]][0]
+  // CHECK: %[[RES_22:.*]] = llvm.insertvalue %[[ALLOCATED_2]], %[[RES_21]][1]
+
+  // CHECK: %[[RESULTS:.*]] = llvm.mlir.undef : !llvm<"{ { i64, i8* }, { i64, i8* } }">
+  // CHECK: %[[RESULTS_1:.*]] = llvm.insertvalue %[[RES_12]], %[[RESULTS]]
+  // CHECK: %[[RESULTS_2:.*]] = llvm.insertvalue %[[RES_22]], %[[RESULTS_1]]
+  // CHECK: llvm.return %[[RESULTS_2]]
+  return %0, %0 : memref<*xf32>, memref<*xf32>
+}
+
diff --git a/mlir/test/Conversion/StandardToLLVM/convert-funcs.mlir b/mlir/test/Conversion/StandardToLLVM/convert-funcs.mlir
index 72d82380d9793..562918b71ac83 100644
--- a/mlir/test/Conversion/StandardToLLVM/convert-funcs.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/convert-funcs.mlir
@@ -31,7 +31,7 @@ func @pass_through(%arg0: () -> ()) -> (() -> ()) {
 // CHECK-NEXT:  llvm.br ^bb1(%arg0 : !llvm<"void ()*">)
   br ^bb1(%arg0 : () -> ())
 
-//CHECK-NEXT: ^bb1(%0: !llvm<"void ()*">):	// pred: ^bb0
+//CHECK-NEXT: ^bb1(%0: !llvm<"void ()*">):
 ^bb1(%bbarg: () -> ()):
 // CHECK-NEXT:  llvm.return %0 : !llvm<"void ()*">
   return %bbarg : () -> ()
@@ -40,11 +40,12 @@ func @pass_through(%arg0: () -> ()) -> (() -> ()) {
 // CHECK-LABEL: llvm.func @body(!llvm.i32)
 func @body(i32)
 
-// CHECK-LABEL: llvm.func @indirect_const_call(%arg0: !llvm.i32) {
+// CHECK-LABEL: llvm.func @indirect_const_call
+// CHECK-SAME: (%[[ARG0:.*]]: !llvm.i32) {
 func @indirect_const_call(%arg0: i32) {
-// CHECK-NEXT: %0 = llvm.mlir.constant(@body) : !llvm<"void (i32)*">
+// CHECK-NEXT: %[[ADDR:.*]] = llvm.mlir.addressof @body : !llvm<"void (i32)*">
   %0 = constant @body : (i32) -> ()
-// CHECK-NEXT:  llvm.call %0(%arg0) : (!llvm.i32) -> ()
+// CHECK-NEXT:  llvm.call %[[ADDR]](%[[ARG0:.*]]) : (!llvm.i32) -> ()
   call_indirect %0(%arg0) : (i32) -> ()
 // CHECK-NEXT:  llvm.return
   return
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
new file mode 100644
index 0000000000000..eab211ffaaf4d
--- /dev/null
+++ b/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
@@ -0,0 +1,42 @@
+// RUN: mlir-opt %s -convert-vector-to-llvm | FileCheck %s
+// RUN: mlir-opt %s -convert-vector-to-llvm='reassociate-fp-reductions' | FileCheck %s --check-prefix=REASSOC
+
+//
+// CHECK-LABEL: llvm.func @reduce_add_f32(
+// CHECK-SAME: %[[A:.*]]: !llvm<"<16 x float>">)
+//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f32) : !llvm.float
+//      CHECK: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.v2.fadd"(%[[C]], %[[A]])
+// CHECK-SAME: {reassoc = false} : (!llvm.float, !llvm<"<16 x float>">) -> !llvm.float
+//      CHECK: llvm.return %[[V]] : !llvm.float
+//
+// REASSOC-LABEL: llvm.func @reduce_add_f32(
+// REASSOC-SAME: %[[A:.*]]: !llvm<"<16 x float>">)
+//      REASSOC: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f32) : !llvm.float
+//      REASSOC: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.v2.fadd"(%[[C]], %[[A]])
+// REASSOC-SAME: {reassoc = true} : (!llvm.float, !llvm<"<16 x float>">) -> !llvm.float
+//      REASSOC: llvm.return %[[V]] : !llvm.float
+//
+func @reduce_add_f32(%arg0: vector<16xf32>) -> f32 {
+  %0 = vector.reduction "add", %arg0 : vector<16xf32> into f32
+  return %0 : f32
+}
+
+//
+// CHECK-LABEL: llvm.func @reduce_mul_f32(
+// CHECK-SAME: %[[A:.*]]: !llvm<"<16 x float>">)
+//      CHECK: %[[C:.*]] = llvm.mlir.constant(1.000000e+00 : f32) : !llvm.float
+//      CHECK: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.v2.fmul"(%[[C]], %[[A]])
+// CHECK-SAME: {reassoc = false} : (!llvm.float, !llvm<"<16 x float>">) -> !llvm.float
+//      CHECK: llvm.return %[[V]] : !llvm.float
+//
+// REASSOC-LABEL: llvm.func @reduce_mul_f32(
+// REASSOC-SAME: %[[A:.*]]: !llvm<"<16 x float>">)
+//      REASSOC: %[[C:.*]] = llvm.mlir.constant(1.000000e+00 : f32) : !llvm.float
+//      REASSOC: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.v2.fmul"(%[[C]], %[[A]])
+// REASSOC-SAME: {reassoc = true} : (!llvm.float, !llvm<"<16 x float>">) -> !llvm.float
+//      REASSOC: llvm.return %[[V]] : !llvm.float
+//
+func @reduce_mul_f32(%arg0: vector<16xf32>) -> f32 {
+  %0 = vector.reduction "mul", %arg0 : vector<16xf32> into f32
+  return %0 : f32
+}
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
index 8351fb23c078e..09162aa0236ba 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
@@ -172,29 +172,25 @@ func @broadcast_stretch_in_middle(%arg0: vector<4x1x2xf32>) -> vector<4x3x2xf32>
 // CHECK-SAME:  %[[A:.*]]: !llvm<"[4 x [1 x <2 x float>]]">)
 // CHECK:       %[[T0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<4x3x2xf32>) : !llvm<"[4 x [3 x <2 x float>]]">
 // CHECK:       %[[T1:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<3x2xf32>) : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T2:.*]] = llvm.extractvalue %[[A]][0] : !llvm<"[4 x [1 x <2 x float>]]">
-// CHECK:       %[[T3:.*]] = llvm.extractvalue %[[T2]][0] : !llvm<"[1 x <2 x float>]">
-// CHECK:       %[[T4:.*]] = llvm.insertvalue %[[T3]], %[[T1]][0] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T5:.*]] = llvm.insertvalue %[[T3]], %[[T4]][1] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T6:.*]] = llvm.insertvalue %[[T3]], %[[T5]][2] : !llvm<"[3 x <2 x float>]">
+// CHECK:       %[[T2:.*]] = llvm.extractvalue %[[A]][0, 0] : !llvm<"[4 x [1 x <2 x float>]]">
+// CHECK:       %[[T4:.*]] = llvm.insertvalue %[[T2]], %[[T1]][0] : !llvm<"[3 x <2 x float>]">
+// CHECK:       %[[T5:.*]] = llvm.insertvalue %[[T2]], %[[T4]][1] : !llvm<"[3 x <2 x float>]">
+// CHECK:       %[[T6:.*]] = llvm.insertvalue %[[T2]], %[[T5]][2] : !llvm<"[3 x <2 x float>]">
 // CHECK:       %[[T7:.*]] = llvm.insertvalue %[[T6]], %[[T0]][0] : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       %[[T8:.*]] = llvm.extractvalue %[[A]][1] : !llvm<"[4 x [1 x <2 x float>]]">
-// CHECK:       %[[T9:.*]] = llvm.extractvalue %[[T8]][0] : !llvm<"[1 x <2 x float>]">
-// CHECK:       %[[T10:.*]] = llvm.insertvalue %[[T9]], %[[T1]][0] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T11:.*]] = llvm.insertvalue %[[T9]], %[[T10]][1] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T12:.*]] = llvm.insertvalue %[[T9]], %[[T11]][2] : !llvm<"[3 x <2 x float>]">
+// CHECK:       %[[T8:.*]] = llvm.extractvalue %[[A]][1, 0] : !llvm<"[4 x [1 x <2 x float>]]">
+// CHECK:       %[[T10:.*]] = llvm.insertvalue %[[T8]], %[[T1]][0] : !llvm<"[3 x <2 x float>]">
+// CHECK:       %[[T11:.*]] = llvm.insertvalue %[[T8]], %[[T10]][1] : !llvm<"[3 x <2 x float>]">
+// CHECK:       %[[T12:.*]] = llvm.insertvalue %[[T8]], %[[T11]][2] : !llvm<"[3 x <2 x float>]">
 // CHECK:       %[[T13:.*]] = llvm.insertvalue %[[T12]], %[[T7]][1] : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       %[[T14:.*]] = llvm.extractvalue %[[A]][2] : !llvm<"[4 x [1 x <2 x float>]]">
-// CHECK:       %[[T15:.*]] = llvm.extractvalue %[[T14]][0] : !llvm<"[1 x <2 x float>]">
-// CHECK:       %[[T16:.*]] = llvm.insertvalue %[[T15]], %[[T1]][0] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T17:.*]] = llvm.insertvalue %[[T15]], %[[T16]][1] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T18:.*]] = llvm.insertvalue %[[T15]], %[[T17]][2] : !llvm<"[3 x <2 x float>]">
+// CHECK:       %[[T14:.*]] = llvm.extractvalue %[[A]][2, 0] : !llvm<"[4 x [1 x <2 x float>]]">
+// CHECK:       %[[T16:.*]] = llvm.insertvalue %[[T14]], %[[T1]][0] : !llvm<"[3 x <2 x float>]">
+// CHECK:       %[[T17:.*]] = llvm.insertvalue %[[T14]], %[[T16]][1] : !llvm<"[3 x <2 x float>]">
+// CHECK:       %[[T18:.*]] = llvm.insertvalue %[[T14]], %[[T17]][2] : !llvm<"[3 x <2 x float>]">
 // CHECK:       %[[T19:.*]] = llvm.insertvalue %[[T18]], %[[T13]][2] : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       %[[T20:.*]] = llvm.extractvalue %[[A]][3] : !llvm<"[4 x [1 x <2 x float>]]">
-// CHECK:       %[[T21:.*]] = llvm.extractvalue %[[T20]][0] : !llvm<"[1 x <2 x float>]">
-// CHECK:       %[[T22:.*]] = llvm.insertvalue %[[T21]], %[[T1]][0] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T23:.*]] = llvm.insertvalue %[[T21]], %[[T22]][1] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T24:.*]] = llvm.insertvalue %[[T21]], %[[T23]][2] : !llvm<"[3 x <2 x float>]">
+// CHECK:       %[[T20:.*]] = llvm.extractvalue %[[A]][3, 0] : !llvm<"[4 x [1 x <2 x float>]]">
+// CHECK:       %[[T22:.*]] = llvm.insertvalue %[[T20]], %[[T1]][0] : !llvm<"[3 x <2 x float>]">
+// CHECK:       %[[T23:.*]] = llvm.insertvalue %[[T20]], %[[T22]][1] : !llvm<"[3 x <2 x float>]">
+// CHECK:       %[[T24:.*]] = llvm.insertvalue %[[T20]], %[[T23]][2] : !llvm<"[3 x <2 x float>]">
 // CHECK:       %[[T25:.*]] = llvm.insertvalue %[[T24]], %[[T19]][3] : !llvm<"[4 x [3 x <2 x float>]]">
 // CHECK:       llvm.return %[[T25]] : !llvm<"[4 x [3 x <2 x float>]]">
 
@@ -630,7 +626,7 @@ func @insert_strided_slice3(%arg0: vector<2x4xf32>, %arg1: vector<16x4x8xf32>) -
 // CHECK-SAME: %[[B:.*]]: !llvm<"[16 x [4 x <8 x float>]]">)
 //      CHECK: %[[s0:.*]] = llvm.extractvalue %[[B]][0] : !llvm<"[16 x [4 x <8 x float>]]">
 //      CHECK: %[[s1:.*]] = llvm.extractvalue %[[A]][0] : !llvm<"[2 x <4 x float>]">
-//      CHECK: %[[s2:.*]] = llvm.extractvalue %[[s0]][0] : !llvm<"[4 x <8 x float>]">
+//      CHECK: %[[s2:.*]] = llvm.extractvalue %[[B]][0, 0] : !llvm<"[16 x [4 x <8 x float>]]">
 //      CHECK: %[[s3:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
 //      CHECK: %[[s4:.*]] = llvm.extractelement %[[s1]][%[[s3]] : !llvm.i64] : !llvm<"<4 x float>">
 //      CHECK: %[[s5:.*]] = llvm.mlir.constant(2 : index) : !llvm.i64
@@ -649,7 +645,7 @@ func @insert_strided_slice3(%arg0: vector<2x4xf32>, %arg1: vector<16x4x8xf32>) -
 //      CHECK: %[[s18:.*]] = llvm.insertelement %[[s16]], %[[s14]][%[[s17]] : !llvm.i64] : !llvm<"<8 x float>">
 //      CHECK: %[[s19:.*]] = llvm.insertvalue %[[s18]], %[[s0]][0] : !llvm<"[4 x <8 x float>]">
 //      CHECK: %[[s20:.*]] = llvm.extractvalue %[[A]][1] : !llvm<"[2 x <4 x float>]">
-//      CHECK: %[[s21:.*]] = llvm.extractvalue %[[s0]][1] : !llvm<"[4 x <8 x float>]">
+//      CHECK: %[[s21:.*]] = llvm.extractvalue %[[B]][0, 1] : !llvm<"[16 x [4 x <8 x float>]]">
 //      CHECK: %[[s22:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
 //      CHECK: %[[s23:.*]] = llvm.extractelement %[[s20]][%[[s22]] : !llvm.i64] : !llvm<"<4 x float>">
 //      CHECK: %[[s24:.*]] = llvm.mlir.constant(2 : index) : !llvm.i64
@@ -721,6 +717,7 @@ func @reduce_f32(%arg0: vector<16xf32>) -> f32 {
 // CHECK-SAME: %[[A:.*]]: !llvm<"<16 x float>">)
 //      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f32) : !llvm.float
 //      CHECK: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.v2.fadd"(%[[C]], %[[A]])
+// CHECK-SAME: {reassoc = false} : (!llvm.float, !llvm<"<16 x float>">) -> !llvm.float
 //      CHECK: llvm.return %[[V]] : !llvm.float
 
 func @reduce_f64(%arg0: vector<16xf64>) -> f64 {
@@ -731,6 +728,7 @@ func @reduce_f64(%arg0: vector<16xf64>) -> f64 {
 // CHECK-SAME: %[[A:.*]]: !llvm<"<16 x double>">)
 //      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f64) : !llvm.double
 //      CHECK: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.v2.fadd"(%[[C]], %[[A]])
+// CHECK-SAME: {reassoc = false} : (!llvm.double, !llvm<"<16 x double>">) -> !llvm.double
 //      CHECK: llvm.return %[[V]] : !llvm.double
 
 func @reduce_i32(%arg0: vector<16xi32>) -> i32 {
diff --git a/mlir/test/Conversion/VectorToSCF/vector-to-loops.mlir b/mlir/test/Conversion/VectorToSCF/vector-to-loops.mlir
index f79b07d00d225..b8c27b51b4694 100644
--- a/mlir/test/Conversion/VectorToSCF/vector-to-loops.mlir
+++ b/mlir/test/Conversion/VectorToSCF/vector-to-loops.mlir
@@ -383,3 +383,20 @@ func @transfer_write_progressive_not_masked(%A : memref<?x?xf32>, %base: index,
     vector<3x15xf32>, memref<?x?xf32>
   return
 }
+
+// -----
+
+// FULL-UNROLL-LABEL: transfer_read_simple
+func @transfer_read_simple(%A : memref<2x2xf32>) -> vector<2x2xf32> {
+  %c0 = constant 0 : index
+  %f0 = constant 0.0 : f32
+  // FULL-UNROLL-DAG: %[[VC0:.*]] = constant dense<0.000000e+00> : vector<2x2xf32>
+  // FULL-UNROLL-DAG: %[[C0:.*]] = constant 0 : index
+  // FULL-UNROLL-DAG: %[[C1:.*]] = constant 1 : index
+  // FULL-UNROLL: %[[V0:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]]]
+  // FULL-UNROLL: %[[RES0:.*]] = vector.insert %[[V0]], %[[VC0]] [0] : vector<2xf32> into vector<2x2xf32>
+  // FULL-UNROLL: %[[V1:.*]] = vector.transfer_read %{{.*}}[%[[C1]], %[[C0]]]
+  // FULL-UNROLL: %[[RES1:.*]] = vector.insert %[[V1]], %[[RES0]] [1] : vector<2xf32> into vector<2x2xf32>
+  %0 = vector.transfer_read %A[%c0, %c0], %f0 : memref<2x2xf32>, vector<2x2xf32>
+  return %0 : vector<2x2xf32>
+}
diff --git a/mlir/test/Dialect/Affine/affine-fold.mlir b/mlir/test/Dialect/Affine/affine-fold.mlir
index eeafccc1850ce..c58802fe9ea70 100644
--- a/mlir/test/Dialect/Affine/affine-fold.mlir
+++ b/mlir/test/Dialect/Affine/affine-fold.mlir
@@ -1,16 +1,15 @@
 // RUN: mlir-opt -canonicalize -split-input-file %s | FileCheck %s
 
-// CHECK-LABEL: func @affine_parallel_rank0
-func @affine_parallel_rank0() {
+// CHECK: func @affine_parallel_rank0
+func @affine_parallel_rank0(%out: memref<f32>) {
   // CHECK-NEXT: constant
-  %cst = constant 1.0 : f32
-  // CHECK-NEXT: alloc
-  %0 = alloc() : memref<f32>
+  %cst = constant 0.0 : f32
   // CHECK-NEXT: affine.store
   affine.parallel () = () to () {
-    affine.store %cst, %0[] : memref<f32>
+    affine.parallel () = () to () {
+      affine.store %cst, %out[] : memref<f32>
+    }
   }
-  // CHECK-NEXT: return
   return
 }
 
@@ -23,7 +22,7 @@ func @affine_parallel_range1() {
   // CHECK-NEXT: alloc
   %0 = alloc() : memref<2x4xf32>
   // CHECK-NEXT: affine.store
-  affine.parallel (%i, %j) = (0, 1) to (1, 2) {
+  affine.parallel (%i, %j) = (0, 1) to (2, 2) step (2, 1) {
     affine.store %cst, %0[%i, %j] : memref<2x4xf32>
   }
   // CHECK-NEXT: return
@@ -46,3 +45,58 @@ func @affine_parallel_partial_range1() {
   // CHECK: return
   return
 }
+
+// -----
+
+// CHECK-LABEL: func @simplify_parallel
+func @simplify_parallel() {
+  %cst = constant 1.0 : f32
+  %0 = alloc() : memref<2x4xf32>
+  // CHECK: affine.parallel (%[[i:.*]], %[[j:.*]]) = (0, 0) to (3, 2) {
+  affine.parallel (%i, %j) = (0, 1) to (10, 5) step (3, 2) {
+    // CHECK: affine.parallel (%[[k:.*]]) = (0) to (%[[j]] * 2 - %[[i]] * 3 + 1) {
+    affine.parallel (%k) = (%i) to (%j) {
+      // CHECK: affine.store %{{.*}}, %{{.*}}[%[[i]] * 3, %[[i]] * 3 + %[[k]]] : memref<2x4xf32>
+      affine.store %cst, %0[%i, %k] : memref<2x4xf32>
+    }
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func @affine_parallel_const_bounds
+func @affine_parallel_const_bounds() {
+  %cst = constant 1.0 : f32
+  %c0 = constant 0 : index
+  %c4 = constant 4 : index
+  %0 = alloc() : memref<4xf32>
+  // CHECK: affine.parallel (%{{.*}}) = (0) to (4)
+  affine.parallel (%i) = (%c0) to (%c0 + %c4) {
+    affine.store %cst, %0[%i] : memref<4xf32>
+  }
+  return
+}
+
+// -----
+
+#map0 = affine_map<(d0) -> (d0 * 5)>
+#map1 = affine_map<(d0) -> (d0 * 10)>
+
+// CHECK-LABEL: func @affine_parallel_fold_bounds
+func @affine_parallel_fold_bounds() {
+  %cst = constant 1.0 : f32
+  %0 = alloc() : memref<100x100xf32>
+  // CHECK: affine.parallel (%[[i0:.*]], %[[j0:.*]]) =
+  affine.parallel (%i0, %j0) = (0, 0) to (100, 10) {
+    %2 = affine.apply #map0(%i0)
+    %3 = affine.apply #map1(%j0)
+    // CHECK-NOT: affine.apply
+    // CHECK: affine.parallel (%[[i1:.*]], %[[j1:.*]]) = (0, 0) to (5, 10) {
+    affine.parallel (%i1, %j1) = (%2, %3) to (%2 + 5, %3 + 10) {
+      // CHECK: affine.store %{{.*}}, %{{.*}}[%[[i0]] * 5 + %[[i1]], %[[j0]] * 10 + %[[j1]]]
+      affine.store %cst, %0[%i1, %j1] : memref<100x100xf32>
+    }
+  }
+  return
+}
diff --git a/mlir/test/Dialect/Affine/canonicalize.mlir b/mlir/test/Dialect/Affine/canonicalize.mlir
index 49245a6900a7f..56134b635b1db 100644
--- a/mlir/test/Dialect/Affine/canonicalize.mlir
+++ b/mlir/test/Dialect/Affine/canonicalize.mlir
@@ -605,17 +605,3 @@ func @drop_duplicate_bounds(%N : index) {
   return
 }
 
-// -----
-
-// CHECK: func @remove_rank0_affine_parallel(%[[OUT:.*]]: memref<f32>)
-func @remove_rank0_affine_parallel(%out: memref<f32>) {
-  // CHECK-NEXT: %[[CST:.*]] = constant
-  %cst = constant 0.0 : f32
-  // CHECK-NEXT: affine.store %[[CST]], %[[OUT]][] : memref<f32>
-  affine.parallel () = () to () {
-    affine.parallel () = () to () {
-      affine.store %cst, %out[] : memref<f32>
-    }
-  }
-  return
-}
diff --git a/mlir/test/Dialect/Affine/invalid.mlir b/mlir/test/Dialect/Affine/invalid.mlir
index e338851a68008..4d7c9c23edb6c 100644
--- a/mlir/test/Dialect/Affine/invalid.mlir
+++ b/mlir/test/Dialect/Affine/invalid.mlir
@@ -266,6 +266,47 @@ func @affine_parallel(%arg0 : index, %arg1 : index, %arg2 : index) {
 
 // -----
 
+// CHECK-LABEL: @affine_parallel
+
+func @affine_parallel(%arg0 : index, %arg1 : index, %arg2 : index) {
+  %0 = alloc() : memref<100x100xf32>
+  //  expected-error@+1 {{reduction must be specified for each output}}
+  %1 = affine.parallel (%i, %j) = (0, 0) to (100, 100) step (10, 10) -> (f32) {
+    %2 = affine.load %0[%i, %j] : memref<100x100xf32>
+    affine.yield %2 : f32
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @affine_parallel
+
+func @affine_parallel(%arg0 : index, %arg1 : index, %arg2 : index) {
+  %0 = alloc() : memref<100x100xf32>
+  //  expected-error@+1 {{invalid reduction value: "bad"}}
+  %1 = affine.parallel (%i, %j) = (0, 0) to (100, 100) step (10, 10) reduce ("bad") -> (f32) {
+    %2 = affine.load %0[%i, %j] : memref<100x100xf32>
+    affine.yield %2 : f32
+  }
+  return
+}
+
+// -----
+// CHECK-LABEL: @affine_parallel
+
+func @affine_parallel(%arg0 : index, %arg1 : index, %arg2 : index) {
+  %0 = alloc() : memref<100x100xi32>
+  %1 = affine.parallel (%i, %j) = (0, 0) to (100, 100) step (10, 10) reduce ("minf") -> (f32) {
+    %2 = affine.load %0[%i, %j] : memref<100x100xi32>
+    //  expected-error@+1 {{types mismatch between yield op and its parent}}
+    affine.yield %2 : i32
+  }
+  return
+}
+
+// -----
+
 func @vector_load_invalid_vector_type() {
   %0 = alloc() : memref<100xf32>
   affine.for %i0 = 0 to 16 step 8 {
@@ -309,3 +350,32 @@ func @vector_store_vector_memref() {
   }
   return
 }
+
+// -----
+
+func @affine_if_with_then_region_args(%N: index) {
+  %c = constant 200 : index
+  %i = constant 20: index
+  // expected-error@+1 {{affine.if' op region #0 should have no arguments}}
+  affine.if affine_set<(i)[N] : (i - 2 >= 0, 4 - i >= 0)>(%i)[%c]  {
+    ^bb0(%arg:i32):
+      %w = affine.apply affine_map<(d0,d1)[s0] -> (d0+d1+s0)> (%i, %i) [%N]
+  }
+  return
+}
+
+// -----
+
+func @affine_if_with_else_region_args(%N: index) {
+  %c = constant 200 : index
+  %i = constant 20: index
+  // expected-error@+1 {{affine.if' op region #1 should have no arguments}}
+  affine.if affine_set<(i)[N] : (i - 2 >= 0, 4 - i >= 0)>(%i)[%c]  {
+      %w = affine.apply affine_map<(d0,d1)[s0] -> (d0+d1+s0)> (%i, %i) [%N]
+  } else {
+    ^bb0(%arg:i32):
+      %w = affine.apply affine_map<(d0,d1)[s0] -> (d0-d1+s0)> (%i, %i) [%N]
+  }
+  return
+}
+
diff --git a/mlir/test/Dialect/Affine/ops.mlir b/mlir/test/Dialect/Affine/ops.mlir
index 20b4ce5597931..cd60869106485 100644
--- a/mlir/test/Dialect/Affine/ops.mlir
+++ b/mlir/test/Dialect/Affine/ops.mlir
@@ -154,14 +154,14 @@ func @valid_symbol_affine_scope(%n : index, %A : memref<?xf32>) {
 // -----
 
 // CHECK-LABEL: func @parallel
-// CHECK-SAME: (%[[N:.*]]: index)
-func @parallel(%N : index) {
+// CHECK-SAME: (%[[A:.*]]: memref<100x100xf32>, %[[N:.*]]: index)
+func @parallel(%A : memref<100x100xf32>, %N : index) {
   // CHECK: affine.parallel (%[[I0:.*]], %[[J0:.*]]) = (0, 0) to (symbol(%[[N]]), 100) step (10, 10)
   affine.parallel (%i0, %j0) = (0, 0) to (symbol(%N), 100) step (10, 10) {
-    %0 = alloc() : memref<f32>
-    // CHECK: affine.parallel (%{{.*}}, %{{.*}}) = (%[[I0]], %[[J0]]) to (%[[I0]] + 10, %[[J0]] + 10) : memref<f32>
-    %1 = affine.parallel (%i1, %j1) = (%i0, %j0) to (%i0 + 10, %j0 + 10) : memref<f32> {
-      affine.yield %0 : memref<f32>
+    // CHECK: affine.parallel (%{{.*}}, %{{.*}}) = (%[[I0]], %[[J0]]) to (%[[I0]] + 10, %[[J0]] + 10) reduce ("minf", "maxf") -> (f32, f32)
+    %0:2 = affine.parallel (%i1, %j1) = (%i0, %j0) to (%i0 + 10, %j0 + 10) reduce ("minf", "maxf") -> (f32, f32) {
+      %2 = affine.load %A[%i0 + %i0, %j0 + %j1] : memref<100x100xf32>
+      affine.yield %2, %2 : f32, f32
     }
   }
   return
diff --git a/mlir/test/Dialect/Affine/parallelism-detection.mlir b/mlir/test/Dialect/Affine/parallelism-detection.mlir
deleted file mode 100644
index 0788e6f8fb208..0000000000000
--- a/mlir/test/Dialect/Affine/parallelism-detection.mlir
+++ /dev/null
@@ -1,47 +0,0 @@
-// RUN: mlir-opt -allow-unregistered-dialect %s -test-detect-parallel -split-input-file -verify-diagnostics | FileCheck %s
-
-// CHECK-LABEL: func @loop_nest_3d_outer_two_parallel
-func @loop_nest_3d_outer_two_parallel(%N : index) {
-  %0 = alloc() : memref<1024 x 1024 x vector<64xf32>>
-  %1 = alloc() : memref<1024 x 1024 x vector<64xf32>>
-  %2 = alloc() : memref<1024 x 1024 x vector<64xf32>>
-  affine.for %i = 0 to %N {
-    // expected-remark@-1 {{parallel loop}}
-    affine.for %j = 0 to %N {
-      // expected-remark@-1 {{parallel loop}}
-      affine.for %k = 0 to %N {
-        // expected-remark@-1 {{sequential loop}}
-        %5 = affine.load %0[%i, %k] : memref<1024x1024xvector<64xf32>>
-        %6 = affine.load %1[%k, %j] : memref<1024x1024xvector<64xf32>>
-        %7 = affine.load %2[%i, %j] : memref<1024x1024xvector<64xf32>>
-        %8 = mulf %5, %6 : vector<64xf32>
-        %9 = addf %7, %8 : vector<64xf32>
-        affine.store %9, %2[%i, %j] : memref<1024x1024xvector<64xf32>>
-      }
-    }
-  }
-  return
-}
-
-// -----
-
-// CHECK-LABEL: unknown_op_conservative
-func @unknown_op_conservative() {
-  affine.for %i = 0 to 10 {
-    // expected-remark@-1 {{sequential loop}}
-    "unknown"() : () -> ()
-  }
-  return
-}
-
-// -----
-
-// CHECK-LABEL: non_affine_load
-func @non_affine_load() {
-  %0 = alloc() : memref<100 x f32>
-  affine.for %i = 0 to 100 {
-    // expected-remark@-1 {{sequential loop}}
-    load %0[%i] : memref<100 x f32>
-  }
-  return
-}
diff --git a/mlir/test/Dialect/Affine/parallelize.mlir b/mlir/test/Dialect/Affine/parallelize.mlir
new file mode 100644
index 0000000000000..5287628185c5b
--- /dev/null
+++ b/mlir/test/Dialect/Affine/parallelize.mlir
@@ -0,0 +1,118 @@
+// RUN: mlir-opt %s -allow-unregistered-dialect -affine-parallelize| FileCheck %s
+
+// For multiple nested for-loops.
+// CHECK-DAG: [[MAP5:#map[0-9]+]] = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d0 + d1, d2 * 2 + d3, d4 * 2 + d5, d6 + d7)>
+// CHECK-LABEL:    func @reduce_window_max() {
+func @reduce_window_max() {
+  %cst = constant 0.000000e+00 : f32
+  %0 = alloc() : memref<1x8x8x64xf32>
+  %1 = alloc() : memref<1x18x18x64xf32>
+  affine.for %arg0 = 0 to 1 {
+    affine.for %arg1 = 0 to 8 {
+      affine.for %arg2 = 0 to 8 {
+        affine.for %arg3 = 0 to 64 {
+          affine.store %cst, %0[%arg0, %arg1, %arg2, %arg3] : memref<1x8x8x64xf32>
+        }
+      }
+    }
+  }
+  affine.for %arg0 = 0 to 1 {
+    affine.for %arg1 = 0 to 8 {
+      affine.for %arg2 = 0 to 8 {
+        affine.for %arg3 = 0 to 64 {
+          affine.for %arg4 = 0 to 1 {
+            affine.for %arg5 = 0 to 3 {
+              affine.for %arg6 = 0 to 3 {
+                affine.for %arg7 = 0 to 1 {
+                  %2 = affine.load %0[%arg0, %arg1, %arg2, %arg3] : memref<1x8x8x64xf32>
+                  %3 = affine.load %1[%arg0 + %arg4, %arg1 * 2 + %arg5, %arg2 * 2 + %arg6, %arg3 + %arg7] : memref<1x18x18x64xf32>
+                  %4 = cmpf "ogt", %2, %3 : f32
+                  %5 = select %4, %2, %3 : f32
+                  affine.store %5, %0[%arg0, %arg1, %arg2, %arg3] : memref<1x8x8x64xf32>
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  return
+}
+
+// CHECK:        %[[cst:.*]] = constant 0.000000e+00 : f32
+// CHECK:        %[[v0:.*]] = alloc() : memref<1x8x8x64xf32>
+// CHECK:        %[[v1:.*]] = alloc() : memref<1x18x18x64xf32>
+// CHECK:        affine.parallel (%[[arg0:.*]]) = (0) to (1) {
+// CHECK:          affine.parallel (%[[arg1:.*]]) = (0) to (8) {
+// CHECK:            affine.parallel (%[[arg2:.*]]) = (0) to (8) {
+// CHECK:              affine.parallel (%[[arg3:.*]]) = (0) to (64) {
+// CHECK:                affine.store %[[cst]], %[[v0]][%[[arg0]], %[[arg1]], %[[arg2]], %[[arg3]]] : memref<1x8x8x64xf32>
+// CHECK:              }
+// CHECK:            }
+// CHECK:          }
+// CHECK:        }
+// CHECK:        affine.parallel (%[[a0:.*]]) = (0) to (1) {
+// CHECK:          affine.parallel (%[[a1:.*]]) = (0) to (8) {
+// CHECK:            affine.parallel (%[[a2:.*]]) = (0) to (8) {
+// CHECK:              affine.parallel (%[[a3:.*]]) = (0) to (64) {
+// CHECK:                affine.parallel (%[[a4:.*]]) = (0) to (1) {
+// CHECK:                  affine.for %[[a5:.*]] = 0 to 3 {
+// CHECK:                    affine.for %[[a6:.*]] = 0 to 3 {
+// CHECK:                      affine.parallel (%[[a7:.*]]) = (0) to (1) {
+// CHECK:                        %[[lhs:.*]] = affine.load %[[v0]][%[[a0]], %[[a1]], %[[a2]], %[[a3]]] : memref<1x8x8x64xf32>
+// CHECK:                        %[[rhs:.*]] = affine.load %[[v1]][%[[a0]] + %[[a4]], %[[a1]] * 2 + %[[a5]], %[[a2]] * 2 + %[[a6]], %[[a3]] + %[[a7]]] : memref<1x18x18x64xf32>
+// CHECK:                        %[[res:.*]] = cmpf "ogt", %[[lhs]], %[[rhs]] : f32
+// CHECK:                        %[[sel:.*]] = select %[[res]], %[[lhs]], %[[rhs]] : f32
+// CHECK:                        affine.store %[[sel]], %[[v0]][%[[a0]], %[[a1]], %[[a2]], %[[a3]]] : memref<1x8x8x64xf32>
+// CHECK:                      }
+// CHECK:                    }
+// CHECK:                  }
+// CHECK:                }
+// CHECK:              }
+// CHECK:            }
+// CHECK:          }
+// CHECK:        }
+// CHECK:      }
+
+func @loop_nest_3d_outer_two_parallel(%N : index) {
+  %0 = alloc() : memref<1024 x 1024 x vector<64xf32>>
+  %1 = alloc() : memref<1024 x 1024 x vector<64xf32>>
+  %2 = alloc() : memref<1024 x 1024 x vector<64xf32>>
+  affine.for %i = 0 to %N {
+    affine.for %j = 0 to %N {
+      %7 = affine.load %2[%i, %j] : memref<1024x1024xvector<64xf32>>
+      affine.for %k = 0 to %N {
+        %5 = affine.load %0[%i, %k] : memref<1024x1024xvector<64xf32>>
+        %6 = affine.load %1[%k, %j] : memref<1024x1024xvector<64xf32>>
+        %8 = mulf %5, %6 : vector<64xf32>
+        %9 = addf %7, %8 : vector<64xf32>
+        affine.store %9, %2[%i, %j] : memref<1024x1024xvector<64xf32>>
+      }
+    }
+  }
+  return
+}
+
+// CHECK:      affine.parallel (%[[arg1:.*]]) = (0) to (symbol(%arg0)) {
+// CHECK-NEXT:        affine.parallel (%[[arg2:.*]]) = (0) to (symbol(%arg0)) {
+// CHECK:          affine.for %[[arg3:.*]] = 0 to %arg0 {
+
+// CHECK-LABEL: unknown_op_conservative
+func @unknown_op_conservative() {
+  affine.for %i = 0 to 10 {
+// CHECK:  affine.for %[[arg1:.*]] = 0 to 10 {
+    "unknown"() : () -> ()
+  }
+  return
+}
+
+// CHECK-LABEL: non_affine_load
+func @non_affine_load() {
+  %0 = alloc() : memref<100 x f32>
+  affine.for %i = 0 to 100 {
+// CHECK:  affine.for %{{.*}} = 0 to 100 {
+    load %0[%i] : memref<100 x f32>
+  }
+  return
+}
diff --git a/mlir/test/Dialect/LLVMIR/global.mlir b/mlir/test/Dialect/LLVMIR/global.mlir
index b5b5639a5bd9e..e3faec63f00a8 100644
--- a/mlir/test/Dialect/LLVMIR/global.mlir
+++ b/mlir/test/Dialect/LLVMIR/global.mlir
@@ -140,12 +140,21 @@ func @foo() {
 llvm.mlir.global internal @foo(0: i32) : !llvm.i32
 
 func @bar() {
-  // expected-error @+1 {{the type must be a pointer to the type of the referred global}}
+  // expected-error @+1 {{the type must be a pointer to the type of the referenced global}}
   llvm.mlir.addressof @foo : !llvm<"i64*">
 }
 
 // -----
 
+llvm.func @foo()
+
+llvm.func @bar() {
+  // expected-error @+1 {{the type must be a pointer to the type of the referenced function}}
+  llvm.mlir.addressof @foo : !llvm<"i8*">
+}
+
+// -----
+
 // expected-error @+2 {{'llvm.mlir.global' op expects regions to end with 'llvm.return', found 'llvm.mlir.constant'}}
 // expected-note @+1 {{in custom textual format, the absence of terminator implies 'llvm.return'}}
 llvm.mlir.global internal @g() : !llvm.i64 {
@@ -172,7 +181,7 @@ llvm.mlir.global internal @g(43 : i64) : !llvm.i64 {
 
 llvm.mlir.global internal @g(32 : i64) {addr_space = 3: i32} : !llvm.i64
 func @mismatch_addr_space_implicit_global() {
-  // expected-error @+1 {{op the type must be a pointer to the type of the referred global}}
+  // expected-error @+1 {{op the type must be a pointer to the type of the referenced global}}
   llvm.mlir.addressof @g : !llvm<"i64*">
 }
 
@@ -180,6 +189,6 @@ func @mismatch_addr_space_implicit_global() {
 
 llvm.mlir.global internal @g(32 : i64) {addr_space = 3: i32} : !llvm.i64
 func @mismatch_addr_space() {
-  // expected-error @+1 {{op the type must be a pointer to the type of the referred global}}
+  // expected-error @+1 {{op the type must be a pointer to the type of the referenced global}}
   llvm.mlir.addressof @g : !llvm<"i64 addrspace(4)*">
 }
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index c97a4acdf6b47..6ef25f85a5060 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -153,6 +153,13 @@ func @call_non_llvm_input(%callee : (i32) -> (), %arg : i32) {
 
 // -----
 
+func @constant_wrong_type() {
+  // expected-error@+1 {{only supports integer, float, string or elements attributes}}
+  llvm.mlir.constant(@constant_wrong_type) : !llvm<"void ()*">
+}
+
+// -----
+
 func @insertvalue_non_llvm_type(%a : i32, %b : i32) {
   // expected-error@+1 {{expected LLVM IR Dialect type}}
   llvm.insertvalue %a, %b[0] : i32
diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
index d93de93882bea..1180561cf77b2 100644
--- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir
+++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
@@ -1,17 +1,20 @@
 // RUN: mlir-opt %s | mlir-opt | FileCheck %s
 
-// CHECK-LABEL: func @ops(%arg0: !llvm.i32, %arg1: !llvm.float)
-func @ops(%arg0 : !llvm.i32, %arg1 : !llvm.float) {
+// CHECK-LABEL: func @ops
+// CHECK-SAME: (%[[I32:.*]]: !llvm.i32, %[[FLOAT:.*]]: !llvm.float, %[[I8PTR1:.*]]: !llvm<"i8*">, %[[I8PTR2:.*]]: !llvm<"i8*">, %[[BOOL:.*]]: !llvm.i1)
+func @ops(%arg0: !llvm.i32, %arg1: !llvm.float,
+          %arg2: !llvm<"i8*">, %arg3: !llvm<"i8*">,
+          %arg4: !llvm.i1) {
 // Integer arithmetic binary operations.
 //
-// CHECK-NEXT:  %0 = llvm.add %arg0, %arg0 : !llvm.i32
-// CHECK-NEXT:  %1 = llvm.sub %arg0, %arg0 : !llvm.i32
-// CHECK-NEXT:  %2 = llvm.mul %arg0, %arg0 : !llvm.i32
-// CHECK-NEXT:  %3 = llvm.udiv %arg0, %arg0 : !llvm.i32
-// CHECK-NEXT:  %4 = llvm.sdiv %arg0, %arg0 : !llvm.i32
-// CHECK-NEXT:  %5 = llvm.urem %arg0, %arg0 : !llvm.i32
-// CHECK-NEXT:  %6 = llvm.srem %arg0, %arg0 : !llvm.i32
-// CHECK-NEXT:  %7 = llvm.icmp "ne" %arg0, %arg0 : !llvm.i32
+// CHECK: {{.*}} = llvm.add %[[I32]], %[[I32]] : !llvm.i32
+// CHECK: {{.*}} = llvm.sub %[[I32]], %[[I32]] : !llvm.i32
+// CHECK: {{.*}} = llvm.mul %[[I32]], %[[I32]] : !llvm.i32
+// CHECK: {{.*}} = llvm.udiv %[[I32]], %[[I32]] : !llvm.i32
+// CHECK: {{.*}} = llvm.sdiv %[[I32]], %[[I32]] : !llvm.i32
+// CHECK: {{.*}} = llvm.urem %[[I32]], %[[I32]] : !llvm.i32
+// CHECK: {{.*}} = llvm.srem %[[I32]], %[[I32]] : !llvm.i32
+// CHECK: {{.*}} = llvm.icmp "ne" %[[I32]], %[[I32]] : !llvm.i32
   %0 = llvm.add %arg0, %arg0 : !llvm.i32
   %1 = llvm.sub %arg0, %arg0 : !llvm.i32
   %2 = llvm.mul %arg0, %arg0 : !llvm.i32
@@ -23,11 +26,11 @@ func @ops(%arg0 : !llvm.i32, %arg1 : !llvm.float) {
 
 // Floating point binary operations.
 //
-// CHECK-NEXT:  %8 = llvm.fadd %arg1, %arg1 : !llvm.float
-// CHECK-NEXT:  %9 = llvm.fsub %arg1, %arg1 : !llvm.float
-// CHECK-NEXT:  %10 = llvm.fmul %arg1, %arg1 : !llvm.float
-// CHECK-NEXT:  %11 = llvm.fdiv %arg1, %arg1 : !llvm.float
-// CHECK-NEXT:  %12 = llvm.frem %arg1, %arg1 : !llvm.float
+// CHECK: {{.*}} = llvm.fadd %[[FLOAT]], %[[FLOAT]] : !llvm.float
+// CHECK: {{.*}} = llvm.fsub %[[FLOAT]], %[[FLOAT]] : !llvm.float
+// CHECK: {{.*}} = llvm.fmul %[[FLOAT]], %[[FLOAT]] : !llvm.float
+// CHECK: {{.*}} = llvm.fdiv %[[FLOAT]], %[[FLOAT]] : !llvm.float
+// CHECK: {{.*}} = llvm.frem %[[FLOAT]], %[[FLOAT]] : !llvm.float
   %8 = llvm.fadd %arg1, %arg1 : !llvm.float
   %9 = llvm.fsub %arg1, %arg1 : !llvm.float
   %10 = llvm.fmul %arg1, %arg1 : !llvm.float
@@ -36,11 +39,11 @@ func @ops(%arg0 : !llvm.i32, %arg1 : !llvm.float) {
 
 // Memory-related operations.
 //
-// CHECK-NEXT:  %13 = llvm.alloca %arg0 x !llvm.double : (!llvm.i32) -> !llvm<"double*">
-// CHECK-NEXT:  %14 = llvm.getelementptr %13[%arg0, %arg0] : (!llvm<"double*">, !llvm.i32, !llvm.i32) -> !llvm<"double*">
-// CHECK-NEXT:  %15 = llvm.load %14 : !llvm<"double*">
-// CHECK-NEXT:  llvm.store %15, %13 : !llvm<"double*">
-// CHECK-NEXT:  %16 = llvm.bitcast %13 : !llvm<"double*"> to !llvm<"i64*">
+// CHECK-NEXT:  %[[ALLOCA:.*]] = llvm.alloca %[[I32]] x !llvm.double : (!llvm.i32) -> !llvm<"double*">
+// CHECK-NEXT:  %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][%[[I32]], %[[I32]]] : (!llvm<"double*">, !llvm.i32, !llvm.i32) -> !llvm<"double*">
+// CHECK-NEXT:  %[[VALUE:.*]] = llvm.load %[[GEP]] : !llvm<"double*">
+// CHECK-NEXT:  llvm.store %[[VALUE]], %[[ALLOCA]] : !llvm<"double*">
+// CHECK-NEXT:  %{{.*}} = llvm.bitcast %[[ALLOCA]] : !llvm<"double*"> to !llvm<"i64*">
   %13 = llvm.alloca %arg0 x !llvm.double : (!llvm.i32) -> !llvm<"double*">
   %14 = llvm.getelementptr %13[%arg0, %arg0] : (!llvm<"double*">, !llvm.i32, !llvm.i32) -> !llvm<"double*">
   %15 = llvm.load %14 : !llvm<"double*">
@@ -49,77 +52,96 @@ func @ops(%arg0 : !llvm.i32, %arg1 : !llvm.float) {
 
 // Function call-related operations.
 //
-// CHECK-NEXT:  %17 = llvm.call @foo(%arg0) : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
-// CHECK-NEXT:  %18 = llvm.extractvalue %17[0] : !llvm<"{ i32, double, i32 }">
-// CHECK-NEXT:  %19 = llvm.insertvalue %18, %17[2] : !llvm<"{ i32, double, i32 }">
-// CHECK-NEXT:  %20 = llvm.mlir.constant(@foo) : !llvm<"{ i32, double, i32 } (i32)*">
-// CHECK-NEXT:  %21 = llvm.call %20(%arg0) : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
+// CHECK: %[[STRUCT:.*]] = llvm.call @foo(%[[I32]]) : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
+// CHECK: %[[VALUE:.*]] = llvm.extractvalue %[[STRUCT]][0] : !llvm<"{ i32, double, i32 }">
+// CHECK: %[[NEW_STRUCT:.*]] = llvm.insertvalue %[[VALUE]], %[[STRUCT]][2] : !llvm<"{ i32, double, i32 }">
+// CHECK: %[[FUNC:.*]] = llvm.mlir.addressof @foo : !llvm<"{ i32, double, i32 } (i32)*">
+// CHECK: %{{.*}} = llvm.call %[[FUNC]](%[[I32]]) : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
   %17 = llvm.call @foo(%arg0) : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
   %18 = llvm.extractvalue %17[0] : !llvm<"{ i32, double, i32 }">
   %19 = llvm.insertvalue %18, %17[2] : !llvm<"{ i32, double, i32 }">
-  %20 = llvm.mlir.constant(@foo) : !llvm<"{ i32, double, i32 } (i32)*">
+  %20 = llvm.mlir.addressof @foo : !llvm<"{ i32, double, i32 } (i32)*">
   %21 = llvm.call %20(%arg0) : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
 
 
 // Terminator operations and their successors.
 //
-// CHECK: llvm.br ^bb1
+// CHECK: llvm.br ^[[BB1:.*]]
   llvm.br ^bb1
 
+// CHECK: ^[[BB1]]
 ^bb1:
-// CHECK: llvm.cond_br %7, ^bb2, ^bb1
+// CHECK: llvm.cond_br %7, ^[[BB2:.*]], ^[[BB1]]
   llvm.cond_br %7, ^bb2, ^bb1
 
+// CHECK: ^[[BB2]]
 ^bb2:
-// CHECK:       %22 = llvm.mlir.undef : !llvm<"{ i32, double, i32 }">
-// CHECK-NEXT:  %23 = llvm.mlir.constant(42 : i64) : !llvm.i47
+// CHECK: %{{.*}} = llvm.mlir.undef : !llvm<"{ i32, double, i32 }">
+// CHECK: %{{.*}} = llvm.mlir.constant(42 : i64) : !llvm.i47
   %22 = llvm.mlir.undef : !llvm<"{ i32, double, i32 }">
   %23 = llvm.mlir.constant(42) : !llvm.i47
 
 // Misc operations.
-// CHECK:       %24 = llvm.select %7, %0, %1 : !llvm.i1, !llvm.i32
+// CHECK: %{{.*}} = llvm.select %{{.*}}, %{{.*}}, %{{.*}} : !llvm.i1, !llvm.i32
   %24 = llvm.select %7, %0, %1 : !llvm.i1, !llvm.i32
 
 // Integer to pointer and pointer to integer conversions.
 //
-// CHECK:       %25 = llvm.inttoptr %arg0 : !llvm.i32 to !llvm<"i32*">
-// CHECK-NEXT:  %26 = llvm.ptrtoint %25 : !llvm<"i32*"> to !llvm.i32
+// CHECK: %[[PTR:.*]] = llvm.inttoptr %[[I32]] : !llvm.i32 to !llvm<"i32*">
+// CHECK: %{{.*}} = llvm.ptrtoint %[[PTR]] : !llvm<"i32*"> to !llvm.i32
   %25 = llvm.inttoptr %arg0 : !llvm.i32 to !llvm<"i32*">
   %26 = llvm.ptrtoint %25 : !llvm<"i32*"> to !llvm.i32
 
 // Extended and Quad floating point
 //
-// CHECK:       %27 = llvm.fpext %arg1 : !llvm.float to !llvm.x86_fp80
-// CHECK-NEXT:  %28 = llvm.fpext %arg1 : !llvm.float to !llvm.fp128
+// CHECK: %{{.*}} = llvm.fpext %[[FLOAT]] : !llvm.float to !llvm.x86_fp80
+// CHECK: %{{.*}} = llvm.fpext %[[FLOAT]] : !llvm.float to !llvm.fp128
   %27 = llvm.fpext %arg1 : !llvm.float to !llvm.x86_fp80
   %28 = llvm.fpext %arg1 : !llvm.float to !llvm.fp128
 
-// CHECK:  %29 = llvm.fneg %arg1 : !llvm.float
+// CHECK: %{{.*}} = llvm.fneg %[[FLOAT]] : !llvm.float
   %29 = llvm.fneg %arg1 : !llvm.float
 
-// CHECK: "llvm.intr.sin"(%arg1) : (!llvm.float) -> !llvm.float
+// CHECK: "llvm.intr.sin"(%[[FLOAT]]) : (!llvm.float) -> !llvm.float
   %30 = "llvm.intr.sin"(%arg1) : (!llvm.float) -> !llvm.float
 
-// CHECK: "llvm.intr.pow"(%arg1, %arg1) : (!llvm.float, !llvm.float) -> !llvm.float
+// CHECK: "llvm.intr.pow"(%[[FLOAT]], %[[FLOAT]]) : (!llvm.float, !llvm.float) -> !llvm.float
   %31 = "llvm.intr.pow"(%arg1, %arg1) : (!llvm.float, !llvm.float) -> !llvm.float
 
+// CHECK: "llvm.intr.bitreverse"(%{{.*}}) : (!llvm.i32) -> !llvm.i32
+  %32 = "llvm.intr.bitreverse"(%arg0) : (!llvm.i32) -> !llvm.i32
+
+// CHECK: "llvm.intr.ctpop"(%{{.*}}) : (!llvm.i32) -> !llvm.i32
+  %33 = "llvm.intr.ctpop"(%arg0) : (!llvm.i32) -> !llvm.i32
+
+// CHECK: "llvm.intr.memcpy"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i32, !llvm.i1) -> ()
+  "llvm.intr.memcpy"(%arg2, %arg3, %arg0, %arg4) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i32, !llvm.i1) -> ()
+
+// CHECK: "llvm.intr.memcpy"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i32, !llvm.i1) -> ()
+  "llvm.intr.memcpy"(%arg2, %arg3, %arg0, %arg4) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i32, !llvm.i1) -> ()
+
+// CHECK: %[[SZ:.*]] = llvm.mlir.constant
+  %sz = llvm.mlir.constant(10: i64) : !llvm.i64
+// CHECK: "llvm.intr.memcpy.inline"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i64, !llvm.i1) -> ()
+  "llvm.intr.memcpy.inline"(%arg2, %arg3, %sz, %arg4) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i64, !llvm.i1) -> ()
+
 // CHECK:  llvm.return
   llvm.return
 }
 
 // An larger self-contained function.
-// CHECK-LABEL:func @foo(%arg0: !llvm.i32) -> !llvm<"{ i32, double, i32 }"> {
-func @foo(%arg0: !llvm.i32) -> !llvm<"{ i32, double, i32 }"> {
-// CHECK-NEXT:  %0 = llvm.mlir.constant(3 : i64) : !llvm.i32
-// CHECK-NEXT:  %1 = llvm.mlir.constant(3 : i64) : !llvm.i32
-// CHECK-NEXT:  %2 = llvm.mlir.constant(4.200000e+01 : f64) : !llvm.double
-// CHECK-NEXT:  %3 = llvm.mlir.constant(4.200000e+01 : f64) : !llvm.double
-// CHECK-NEXT:  %4 = llvm.add %0, %1 : !llvm.i32
-// CHECK-NEXT:  %5 = llvm.mul %4, %1 : !llvm.i32
-// CHECK-NEXT:  %6 = llvm.fadd %2, %3 : !llvm.double
-// CHECK-NEXT:  %7 = llvm.fsub %3, %6 : !llvm.double
-// CHECK-NEXT:  %8 = llvm.mlir.constant(1 : i64) : !llvm.i1
-// CHECK-NEXT:  llvm.cond_br %8, ^bb1(%4 : !llvm.i32), ^bb2(%4 : !llvm.i32)
+// CHECK-LABEL: llvm.func @foo(%{{.*}}: !llvm.i32) -> !llvm<"{ i32, double, i32 }"> {
+llvm.func @foo(%arg0: !llvm.i32) -> !llvm<"{ i32, double, i32 }"> {
+// CHECK:  %[[V0:.*]] = llvm.mlir.constant(3 : i64) : !llvm.i32
+// CHECK:  %[[V1:.*]] = llvm.mlir.constant(3 : i64) : !llvm.i32
+// CHECK:  %[[V2:.*]] = llvm.mlir.constant(4.200000e+01 : f64) : !llvm.double
+// CHECK:  %[[V3:.*]] = llvm.mlir.constant(4.200000e+01 : f64) : !llvm.double
+// CHECK:  %[[V4:.*]] = llvm.add %[[V0]], %[[V1]] : !llvm.i32
+// CHECK:  %[[V5:.*]] = llvm.mul %[[V4]], %[[V1]] : !llvm.i32
+// CHECK:  %[[V6:.*]] = llvm.fadd %[[V2]], %[[V3]] : !llvm.double
+// CHECK:  %[[V7:.*]] = llvm.fsub %[[V3]], %[[V6]] : !llvm.double
+// CHECK:  %[[V8:.*]] = llvm.mlir.constant(1 : i64) : !llvm.i1
+// CHECK:  llvm.cond_br %[[V8]], ^[[BB1:.*]](%[[V4]] : !llvm.i32), ^[[BB2:.*]](%[[V4]] : !llvm.i32)
   %0 = llvm.mlir.constant(3) : !llvm.i32
   %1 = llvm.mlir.constant(3) : !llvm.i32
   %2 = llvm.mlir.constant(4.200000e+01) : !llvm.double
@@ -131,16 +153,16 @@ func @foo(%arg0: !llvm.i32) -> !llvm<"{ i32, double, i32 }"> {
   %8 = llvm.mlir.constant(1) : !llvm.i1
   llvm.cond_br %8, ^bb1(%4 : !llvm.i32), ^bb2(%4 : !llvm.i32)
 
-// CHECK-NEXT:^bb1(%9: !llvm.i32):
-// CHECK-NEXT:  %10 = llvm.call @foo(%9) : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
-// CHECK-NEXT:  %11 = llvm.extractvalue %10[0] : !llvm<"{ i32, double, i32 }">
-// CHECK-NEXT:  %12 = llvm.extractvalue %10[1] : !llvm<"{ i32, double, i32 }">
-// CHECK-NEXT:  %13 = llvm.extractvalue %10[2] : !llvm<"{ i32, double, i32 }">
-// CHECK-NEXT:  %14 = llvm.mlir.undef : !llvm<"{ i32, double, i32 }">
-// CHECK-NEXT:  %15 = llvm.insertvalue %5, %14[0] : !llvm<"{ i32, double, i32 }">
-// CHECK-NEXT:  %16 = llvm.insertvalue %7, %15[1] : !llvm<"{ i32, double, i32 }">
-// CHECK-NEXT:  %17 = llvm.insertvalue %11, %16[2] : !llvm<"{ i32, double, i32 }">
-// CHECK-NEXT:  llvm.return %17 : !llvm<"{ i32, double, i32 }">
+// CHECK:^[[BB1]](%[[V9:.*]]: !llvm.i32):
+// CHECK:  %[[V10:.*]] = llvm.call @foo(%[[V9]]) : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
+// CHECK:  %[[V11:.*]] = llvm.extractvalue %[[V10]][0] : !llvm<"{ i32, double, i32 }">
+// CHECK:  %[[V12:.*]] = llvm.extractvalue %[[V10]][1] : !llvm<"{ i32, double, i32 }">
+// CHECK:  %[[V13:.*]] = llvm.extractvalue %[[V10]][2] : !llvm<"{ i32, double, i32 }">
+// CHECK:  %[[V14:.*]] = llvm.mlir.undef : !llvm<"{ i32, double, i32 }">
+// CHECK:  %[[V15:.*]] = llvm.insertvalue %[[V5]], %[[V14]][0] : !llvm<"{ i32, double, i32 }">
+// CHECK:  %[[V16:.*]] = llvm.insertvalue %[[V7]], %[[V15]][1] : !llvm<"{ i32, double, i32 }">
+// CHECK:  %[[V17:.*]] = llvm.insertvalue %[[V11]], %[[V16]][2] : !llvm<"{ i32, double, i32 }">
+// CHECK:  llvm.return %[[V17]] : !llvm<"{ i32, double, i32 }">
 ^bb1(%9: !llvm.i32):
   %10 = llvm.call @foo(%9) : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
   %11 = llvm.extractvalue %10[0] : !llvm<"{ i32, double, i32 }">
@@ -152,13 +174,13 @@ func @foo(%arg0: !llvm.i32) -> !llvm<"{ i32, double, i32 }"> {
   %17 = llvm.insertvalue %11, %16[2] : !llvm<"{ i32, double, i32 }">
   llvm.return %17 : !llvm<"{ i32, double, i32 }">
 
-// CHECK-NEXT:^bb2(%18: !llvm.i32):	// pred: ^bb0
-// CHECK-NEXT:  %19 = llvm.mlir.undef : !llvm<"{ i32, double, i32 }">
-// CHECK-NEXT:  %20 = llvm.insertvalue %18, %19[0] : !llvm<"{ i32, double, i32 }">
-// CHECK-NEXT:  %21 = llvm.insertvalue %7, %20[1] : !llvm<"{ i32, double, i32 }">
-// CHECK-NEXT:  %22 = llvm.insertvalue %5, %21[2] : !llvm<"{ i32, double, i32 }">
-// CHECK-NEXT:  llvm.return %22 : !llvm<"{ i32, double, i32 }">
-^bb2(%18: !llvm.i32):	// pred: ^bb0
+// CHECK:^[[BB2]](%[[V18:.*]]: !llvm.i32):
+// CHECK:  %[[V19:.*]] = llvm.mlir.undef : !llvm<"{ i32, double, i32 }">
+// CHECK:  %[[V20:.*]] = llvm.insertvalue %[[V18]], %[[V19]][0] : !llvm<"{ i32, double, i32 }">
+// CHECK:  %[[V21:.*]] = llvm.insertvalue %[[V7]], %[[V20]][1] : !llvm<"{ i32, double, i32 }">
+// CHECK:  %[[V22:.*]] = llvm.insertvalue %[[V5]], %[[V21]][2] : !llvm<"{ i32, double, i32 }">
+// CHECK:  llvm.return %[[V22]] : !llvm<"{ i32, double, i32 }">
+^bb2(%18: !llvm.i32):
   %19 = llvm.mlir.undef : !llvm<"{ i32, double, i32 }">
   %20 = llvm.insertvalue %18, %19[0] : !llvm<"{ i32, double, i32 }">
   %21 = llvm.insertvalue %7, %20[1] : !llvm<"{ i32, double, i32 }">
@@ -167,51 +189,52 @@ func @foo(%arg0: !llvm.i32) -> !llvm<"{ i32, double, i32 }"> {
 }
 
 // CHECK-LABEL: @casts
+// CHECK-SAME: (%[[I32:.*]]: !llvm.i32, %[[I64:.*]]: !llvm.i64, %[[V4I32:.*]]: !llvm<"<4 x i32>">, %[[V4I64:.*]]: !llvm<"<4 x i64>">, %[[I32PTR:.*]]: !llvm<"i32*">)
 func @casts(%arg0: !llvm.i32, %arg1: !llvm.i64, %arg2: !llvm<"<4 x i32>">,
             %arg3: !llvm<"<4 x i64>">, %arg4: !llvm<"i32*">) {
-// CHECK-NEXT:  = llvm.sext %arg0 : !llvm.i32 to !llvm.i56
+// CHECK:  = llvm.sext %[[I32]] : !llvm.i32 to !llvm.i56
   %0 = llvm.sext %arg0 : !llvm.i32 to !llvm.i56
-// CHECK-NEXT:  = llvm.zext %arg0 : !llvm.i32 to !llvm.i64
+// CHECK:  = llvm.zext %[[I32]] : !llvm.i32 to !llvm.i64
   %1 = llvm.zext %arg0 : !llvm.i32 to !llvm.i64
-// CHECK-NEXT:  = llvm.trunc %arg1 : !llvm.i64 to !llvm.i56
+// CHECK:  = llvm.trunc %[[I64]] : !llvm.i64 to !llvm.i56
   %2 = llvm.trunc %arg1 : !llvm.i64 to !llvm.i56
-// CHECK-NEXT:  = llvm.sext %arg2 : !llvm<"<4 x i32>"> to !llvm<"<4 x i56>">
+// CHECK:  = llvm.sext %[[V4I32]] : !llvm<"<4 x i32>"> to !llvm<"<4 x i56>">
   %3 = llvm.sext %arg2 : !llvm<"<4 x i32>"> to !llvm<"<4 x i56>">
-// CHECK-NEXT:  = llvm.zext %arg2 : !llvm<"<4 x i32>"> to !llvm<"<4 x i64>">
+// CHECK:  = llvm.zext %[[V4I32]] : !llvm<"<4 x i32>"> to !llvm<"<4 x i64>">
   %4 = llvm.zext %arg2 : !llvm<"<4 x i32>"> to !llvm<"<4 x i64>">
-// CHECK-NEXT:  = llvm.trunc %arg3 : !llvm<"<4 x i64>"> to !llvm<"<4 x i56>">
+// CHECK:  = llvm.trunc %[[V4I64]] : !llvm<"<4 x i64>"> to !llvm<"<4 x i56>">
   %5 = llvm.trunc %arg3 : !llvm<"<4 x i64>"> to !llvm<"<4 x i56>">
-// CHECK-NEXT:  = llvm.sitofp %arg0 : !llvm.i32 to !llvm.float
+// CHECK:  = llvm.sitofp %[[I32]] : !llvm.i32 to !llvm.float
   %6 = llvm.sitofp %arg0 : !llvm.i32 to !llvm.float
-// CHECK-NEXT:  = llvm.uitofp %arg0 : !llvm.i32 to !llvm.float
+// CHECK: %[[FLOAT:.*]] = llvm.uitofp %[[I32]] : !llvm.i32 to !llvm.float
   %7 = llvm.uitofp %arg0 : !llvm.i32 to !llvm.float
-// CHECK-NEXT:  = llvm.fptosi %7 : !llvm.float to !llvm.i32
+// CHECK:  = llvm.fptosi %[[FLOAT]] : !llvm.float to !llvm.i32
   %8 = llvm.fptosi %7 : !llvm.float to !llvm.i32
-// CHECK-NEXT:  = llvm.fptoui %7 : !llvm.float to !llvm.i32
+// CHECK:  = llvm.fptoui %[[FLOAT]] : !llvm.float to !llvm.i32
   %9 = llvm.fptoui %7 : !llvm.float to !llvm.i32
-// CHECK-NEXT:  = llvm.addrspacecast %arg4 : !llvm<"i32*"> to !llvm<"i32 addrspace(2)*">
+// CHECK:  = llvm.addrspacecast %[[I32PTR]] : !llvm<"i32*"> to !llvm<"i32 addrspace(2)*">
   %10 = llvm.addrspacecast %arg4 : !llvm<"i32*"> to !llvm<"i32 addrspace(2)*">
   llvm.return
 }
 
 // CHECK-LABEL: @vect
 func @vect(%arg0: !llvm<"<4 x float>">, %arg1: !llvm.i32, %arg2: !llvm.float) {
-// CHECK-NEXT:  = llvm.extractelement {{.*}} : !llvm<"<4 x float>">
+// CHECK:  = llvm.extractelement {{.*}} : !llvm<"<4 x float>">
   %0 = llvm.extractelement %arg0[%arg1 : !llvm.i32] : !llvm<"<4 x float>">
-// CHECK-NEXT:  = llvm.insertelement {{.*}} : !llvm<"<4 x float>">
+// CHECK:  = llvm.insertelement {{.*}} : !llvm<"<4 x float>">
   %1 = llvm.insertelement %arg2, %arg0[%arg1 : !llvm.i32] : !llvm<"<4 x float>">
-// CHECK-NEXT:  = llvm.shufflevector {{.*}} [0 : i32, 0 : i32, 0 : i32, 0 : i32, 7 : i32] : !llvm<"<4 x float>">, !llvm<"<4 x float>">
+// CHECK:  = llvm.shufflevector {{.*}} [0 : i32, 0 : i32, 0 : i32, 0 : i32, 7 : i32] : !llvm<"<4 x float>">, !llvm<"<4 x float>">
   %2 = llvm.shufflevector %arg0, %arg0 [0 : i32, 0 : i32, 0 : i32, 0 : i32, 7 : i32] : !llvm<"<4 x float>">, !llvm<"<4 x float>">
-// CHECK-NEXT:  = llvm.mlir.constant(dense<1.000000e+00> : vector<4xf32>) : !llvm<"<4 x float>">
+// CHECK:  = llvm.mlir.constant(dense<1.000000e+00> : vector<4xf32>) : !llvm<"<4 x float>">
   %3 = llvm.mlir.constant(dense<1.0> : vector<4xf32>) : !llvm<"<4 x float>">
   return
 }
 
 // CHECK-LABEL: @alloca
 func @alloca(%size : !llvm.i64) {
-  //      CHECK: llvm.alloca %{{.*}} x !llvm.i32 : (!llvm.i64) -> !llvm<"i32*">
+  // CHECK: llvm.alloca %{{.*}} x !llvm.i32 : (!llvm.i64) -> !llvm<"i32*">
   llvm.alloca %size x !llvm.i32 {alignment = 0} : (!llvm.i64) -> (!llvm<"i32*">)
-  // CHECK-NEXT: llvm.alloca %{{.*}} x !llvm.i32 {alignment = 8 : i64} : (!llvm.i64) -> !llvm<"i32*">
+  // CHECK: llvm.alloca %{{.*}} x !llvm.i32 {alignment = 8 : i64} : (!llvm.i64) -> !llvm<"i32*">
   llvm.alloca %size x !llvm.i32 {alignment = 8} : (!llvm.i64) -> (!llvm<"i32*">)
   llvm.return
 }
@@ -245,16 +268,16 @@ llvm.func @__gxx_personality_v0(...) -> !llvm.i32
 
 // CHECK-LABEL: @invokeLandingpad
 llvm.func @invokeLandingpad() -> !llvm.i32 attributes { personality = @__gxx_personality_v0 } {
-// CHECK-NEXT: %[[a0:[0-9]+]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-// CHECK-NEXT: %{{[0-9]+}} = llvm.mlir.constant(3 : i32) : !llvm.i32
-// CHECK-NEXT: %[[a2:[0-9]+]] = llvm.mlir.constant("\01") : !llvm<"[1 x i8]">
-// CHECK-NEXT: %[[a3:[0-9]+]] = llvm.mlir.null : !llvm<"i8**">
-// CHECK-NEXT: %[[a4:[0-9]+]] = llvm.mlir.null : !llvm<"i8*">
-// CHECK-NEXT: %[[a5:[0-9]+]] = llvm.mlir.addressof @_ZTIi : !llvm<"i8**">
-// CHECK-NEXT: %[[a6:[0-9]+]] = llvm.bitcast %[[a5]] : !llvm<"i8**"> to !llvm<"i8*">
-// CHECK-NEXT: %[[a7:[0-9]+]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-// CHECK-NEXT: %[[a8:[0-9]+]] = llvm.alloca %[[a7]] x !llvm.i8 : (!llvm.i32) -> !llvm<"i8*">
-// CHECK-NEXT: %{{[0-9]+}} = llvm.invoke @foo(%[[a7]]) to ^bb2 unwind ^bb1 : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
+// CHECK: %[[a0:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
+// CHECK: %{{.*}} = llvm.mlir.constant(3 : i32) : !llvm.i32
+// CHECK: %[[a2:.*]] = llvm.mlir.constant("\01") : !llvm<"[1 x i8]">
+// CHECK: %[[a3:.*]] = llvm.mlir.null : !llvm<"i8**">
+// CHECK: %[[a4:.*]] = llvm.mlir.null : !llvm<"i8*">
+// CHECK: %[[a5:.*]] = llvm.mlir.addressof @_ZTIi : !llvm<"i8**">
+// CHECK: %[[a6:.*]] = llvm.bitcast %[[a5]] : !llvm<"i8**"> to !llvm<"i8*">
+// CHECK: %[[a7:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+// CHECK: %[[a8:.*]] = llvm.alloca %[[a7]] x !llvm.i8 : (!llvm.i32) -> !llvm<"i8*">
+// CHECK: %{{.*}} = llvm.invoke @foo(%[[a7]]) to ^[[BB2:.*]] unwind ^[[BB1:.*]] : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
   %0 = llvm.mlir.constant(0 : i32) : !llvm.i32
   %1 = llvm.mlir.constant(3 : i32) : !llvm.i32
   %2 = llvm.mlir.constant("\01") : !llvm<"[1 x i8]">
@@ -266,25 +289,25 @@ llvm.func @invokeLandingpad() -> !llvm.i32 attributes { personality = @__gxx_per
   %8 = llvm.alloca %7 x !llvm.i8 : (!llvm.i32) -> !llvm<"i8*">
   %9 = llvm.invoke @foo(%7) to ^bb2 unwind ^bb1 : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
 
-// CHECK-NEXT: ^bb1:
-// CHECK-NEXT:   %[[lp:[0-9]+]] = llvm.landingpad cleanup (catch %[[a3]] : !llvm<"i8**">) (catch %[[a6]] : !llvm<"i8*">) (filter %[[a2]] : !llvm<"[1 x i8]">) : !llvm<"{ i8*, i32 }">
-// CHECK-NEXT:   llvm.resume %[[lp]] : !llvm<"{ i8*, i32 }">
+// CHECK: ^[[BB1]]:
+// CHECK:   %[[lp:.*]] = llvm.landingpad cleanup (catch %[[a3]] : !llvm<"i8**">) (catch %[[a6]] : !llvm<"i8*">) (filter %[[a2]] : !llvm<"[1 x i8]">) : !llvm<"{ i8*, i32 }">
+// CHECK:   llvm.resume %[[lp]] : !llvm<"{ i8*, i32 }">
 ^bb1:
   %10 = llvm.landingpad cleanup (catch %3 : !llvm<"i8**">) (catch %6 : !llvm<"i8*">) (filter %2 : !llvm<"[1 x i8]">) : !llvm<"{ i8*, i32 }">
   llvm.resume %10 : !llvm<"{ i8*, i32 }">
 
-// CHECK-NEXT: ^bb2:
-// CHECK-NEXT:   llvm.return %[[a7]] : !llvm.i32
+// CHECK: ^[[BB2]]:
+// CHECK:   llvm.return %[[a7]] : !llvm.i32
 ^bb2:
   llvm.return %7 : !llvm.i32
 
-// CHECK-NEXT: ^bb3:
-// CHECK-NEXT:   llvm.invoke @bar(%[[a8]], %[[a6]], %[[a4]]) to ^bb2 unwind ^bb1 : (!llvm<"i8*">, !llvm<"i8*">, !llvm<"i8*">) -> ()
+// CHECK: ^[[BB3:.*]]:
+// CHECK:   llvm.invoke @bar(%[[a8]], %[[a6]], %[[a4]]) to ^[[BB2]] unwind ^[[BB1]] : (!llvm<"i8*">, !llvm<"i8*">, !llvm<"i8*">) -> ()
 ^bb3:
   llvm.invoke @bar(%8, %6, %4) to ^bb2 unwind ^bb1 : (!llvm<"i8*">, !llvm<"i8*">, !llvm<"i8*">) -> ()
 
-// CHECK-NEXT: ^bb4:
-// CHECK-NEXT:   llvm.return %[[a0]] : !llvm.i32
+// CHECK: ^[[BB4:.*]]:
+// CHECK:   llvm.return %[[a0]] : !llvm.i32
 ^bb4:
   llvm.return %0 : !llvm.i32
 }
@@ -302,11 +325,11 @@ func @useFreezeOp(%arg0: !llvm.i32) {
 
 // CHECK-LABEL: @useFenceInst
 func @useFenceInst() {
-  // CHECK:  syncscope("agent") seq_cst 
+  // CHECK:  syncscope("agent") seq_cst
   llvm.fence syncscope("agent") seq_cst
-  // CHECK:  seq_cst 
+  // CHECK:  seq_cst
   llvm.fence syncscope("") seq_cst
   // CHECK:  release
   llvm.fence release
   return
-}
\ No newline at end of file
+}
diff --git a/mlir/test/Dialect/Linalg/affine.mlir b/mlir/test/Dialect/Linalg/affine.mlir
index cb2064602c475..13f37d844b8ac 100644
--- a/mlir/test/Dialect/Linalg/affine.mlir
+++ b/mlir/test/Dialect/Linalg/affine.mlir
@@ -123,3 +123,27 @@ func @named_batch_matmul(%A: memref<?x?x?xf32>, %B: memref<?x?x?xf32>, %C: memre
 //       CHECK:       %[[inc:.*]] = mulf %[[va]], %[[vb]] : f32
 //       CHECK:       %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
 //       CHECK:       affine.store %[[res]], %[[mC]][%[[b]], %[[m]], %[[n]]] : memref<?x?x?xf32>
+
+// CHECK-LABEL: func @pooling_max_min
+func @pooling_max_min(%arg0: memref<?x?xf32>,
+                      %arg1: memref<?x?xi32>,
+                      %arg2: memref<?x?xf32>) {
+  linalg.pooling_max(%arg0, %arg1, %arg2) { strides = [2, 1] }:
+    memref<?x?xf32>, memref<?x?xi32>, memref<?x?xf32>
+  linalg.pooling_min(%arg0, %arg1, %arg2) { strides = [2, 1] }:
+    memref<?x?xf32>, memref<?x?xi32>, memref<?x?xf32>
+  return
+}
+// This is a basic check to make sure the right load/stores are used. loops.mlir
+// checks for the rest.
+// CHECK:      affine.load
+// CHECK-NEXT: affine.load
+// CHECK-NEXT: cmpf
+// CHECK-NEXT: select
+// CHECK-NEXT: affine.store
+// The min pooling body.
+// CHECK:      affine.load
+// CHECK-NEXT: affine.load
+// CHECK-NEXT: cmpf
+// CHECK-NEXT: select
+// CHECK-NEXT: affine.store
diff --git a/mlir/test/Dialect/Linalg/hoisting.mlir b/mlir/test/Dialect/Linalg/hoisting.mlir
index 54ab1dcf07c71..4a6fca554a096 100644
--- a/mlir/test/Dialect/Linalg/hoisting.mlir
+++ b/mlir/test/Dialect/Linalg/hoisting.mlir
@@ -132,18 +132,101 @@ func @hoist_vector_transfer_pairs(
       %r3 = vector.transfer_read %memref3[%c0, %c0], %cst: memref<?x?xf32>, vector<4xf32>
       "some_crippling_use"(%memref4) : (memref<?x?xf32>) -> ()
       %r4 = vector.transfer_read %memref4[%c0, %c0], %cst: memref<?x?xf32>, vector<5xf32>
+      %r5 = vector.transfer_read %memref5[%c0, %c0], %cst: memref<?x?xf32>, vector<6xf32>
+      "some_crippling_use"(%memref5) : (memref<?x?xf32>) -> ()
       %u0 = "some_use"(%r0) : (vector<1xf32>) -> vector<1xf32>
       %u1 = "some_use"(%r1) : (vector<2xf32>) -> vector<2xf32>
       %u2 = "some_use"(%memref2) : (memref<?x?xf32>) -> vector<3xf32>
       %u3 = "some_use"(%r3) : (vector<4xf32>) -> vector<4xf32>
       %u4 = "some_use"(%r4) : (vector<5xf32>) -> vector<5xf32>
+      %u5 = "some_use"(%r5) : (vector<6xf32>) -> vector<6xf32>
       vector.transfer_write %u0, %memref1[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
       vector.transfer_write %u1, %memref0[%i, %i] : vector<2xf32>, memref<?x?xf32>
       vector.transfer_write %u2, %memref2[%c0, %c0] : vector<3xf32>, memref<?x?xf32>
       vector.transfer_write %u3, %memref3[%c0, %c0] : vector<4xf32>, memref<?x?xf32>
       vector.transfer_write %u4, %memref4[%c0, %c0] : vector<5xf32>, memref<?x?xf32>
+      vector.transfer_write %u5, %memref5[%c0, %c0] : vector<6xf32>, memref<?x?xf32>
       "some_crippling_use"(%memref3) : (memref<?x?xf32>) -> ()
     }
   }
   return
 }
+
+// VECTOR_TRANSFERS-LABEL: func @hoist_vector_transfer_pairs_disjoint(
+//  VECTOR_TRANSFERS-SAME:   %[[MEMREF0:[a-zA-Z0-9]*]]: memref<?x?xf32>,
+//  VECTOR_TRANSFERS-SAME:   %[[MEMREF1:[a-zA-Z0-9]*]]: memref<?x?xf32>,
+//  VECTOR_TRANSFERS-SAME:   %[[MEMREF2:[a-zA-Z0-9]*]]: memref<?x?xf32>,
+//  VECTOR_TRANSFERS-SAME:   %[[MEMREF3:[a-zA-Z0-9]*]]: memref<?x?xf32>,
+//  VECTOR_TRANSFERS-SAME:   %[[VAL:[a-zA-Z0-9]*]]: index,
+//  VECTOR_TRANSFERS-SAME:   %[[LB:[a-zA-Z0-9]*]]: index,
+//  VECTOR_TRANSFERS-SAME:   %[[UB:[a-zA-Z0-9]*]]: index,
+//  VECTOR_TRANSFERS-SAME:   %[[STEP:[a-zA-Z0-9]*]]: index,
+//  VECTOR_TRANSFERS-SAME:   %[[RANDOM:[a-zA-Z0-9]*]]: index,
+//  VECTOR_TRANSFERS-SAME:   %[[CMP:[a-zA-Z0-9]*]]: i1
+func @hoist_vector_transfer_pairs_disjoint(
+    %memref0: memref<?x?xf32>, %memref1: memref<?x?xf32>,
+    %memref2: memref<?x?xf32>, %memref3: memref<?x?xf32>, %val: index, %lb : index, %ub : index,
+    %step: index, %random_index : index, %cmp: i1) {
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %c3 = constant 3 : index
+  %cst = constant 0.0 : f32
+
+// VECTOR_TRANSFERS: vector.transfer_read %[[MEMREF2]]{{.*}} : memref<?x?xf32>, vector<3xf32>
+// VECTOR_TRANSFERS: vector.transfer_read %[[MEMREF2]]{{.*}} : memref<?x?xf32>, vector<3xf32>
+// VECTOR_TRANSFERS: vector.transfer_read %[[MEMREF3]]{{.*}} : memref<?x?xf32>, vector<4xf32>
+// VECTOR_TRANSFERS: vector.transfer_read %[[MEMREF3]]{{.*}} : memref<?x?xf32>, vector<4xf32>
+// VECTOR_TRANSFERS: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) ->
+//  VECTOR_TRANSFERS-SAME: (vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>) {
+// VECTOR_TRANSFERS:   scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) ->
+//  VECTOR_TRANSFERS-SAME: (vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>) {
+// VECTOR_TRANSFERS:     vector.transfer_read %[[MEMREF1]]{{.*}} : memref<?x?xf32>, vector<2xf32>
+// VECTOR_TRANSFERS:     vector.transfer_read %[[MEMREF1]]{{.*}} : memref<?x?xf32>, vector<2xf32>
+// VECTOR_TRANSFERS:     "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
+// VECTOR_TRANSFERS:     "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
+// VECTOR_TRANSFERS:     "some_use"(%{{.*}}) : (vector<3xf32>) -> vector<3xf32>
+// VECTOR_TRANSFERS:     "some_use"(%{{.*}}) : (vector<3xf32>) -> vector<3xf32>
+// VECTOR_TRANSFERS:     "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32>
+// VECTOR_TRANSFERS:     "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32>
+// VECTOR_TRANSFERS:     "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
+// VECTOR_TRANSFERS:     "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
+// VECTOR_TRANSFERS:     vector.transfer_write %{{.*}}, %[[MEMREF1]]{{.*}} : vector<2xf32>, memref<?x?xf32>
+// VECTOR_TRANSFERS:     vector.transfer_write %{{.*}}, %[[MEMREF1]]{{.*}} : vector<2xf32>, memref<?x?xf32>
+// VECTOR_TRANSFERS:     scf.yield {{.*}} : vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>
+// VECTOR_TRANSFERS:   }
+// VECTOR_TRANSFERS:   scf.yield {{.*}} : vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>
+// VECTOR_TRANSFERS: }
+// VECTOR_TRANSFERS: vector.transfer_write %{{.*}}, %[[MEMREF3]]{{.*}} : vector<4xf32>, memref<?x?xf32>
+// VECTOR_TRANSFERS: vector.transfer_write %{{.*}}, %[[MEMREF3]]{{.*}} : vector<4xf32>, memref<?x?xf32>
+// VECTOR_TRANSFERS: vector.transfer_write %{{.*}}, %[[MEMREF2]]{{.*}} : vector<3xf32>, memref<?x?xf32>
+// VECTOR_TRANSFERS: vector.transfer_write %{{.*}}, %[[MEMREF2]]{{.*}} : vector<3xf32>, memref<?x?xf32>
+  scf.for %i = %lb to %ub step %step {
+    scf.for %j = %lb to %ub step %step {
+      %r00 = vector.transfer_read %memref1[%c0, %c0], %cst: memref<?x?xf32>, vector<2xf32>
+      %r01 = vector.transfer_read %memref1[%c0, %c1], %cst: memref<?x?xf32>, vector<2xf32>
+      %r20 = vector.transfer_read %memref2[%c0, %c0], %cst: memref<?x?xf32>, vector<3xf32>
+      %r21 = vector.transfer_read %memref2[%c0, %c3], %cst: memref<?x?xf32>, vector<3xf32>
+      %r30 = vector.transfer_read %memref3[%c0, %random_index], %cst: memref<?x?xf32>, vector<4xf32>
+      %r31 = vector.transfer_read %memref3[%c1, %random_index], %cst: memref<?x?xf32>, vector<4xf32>
+      %r10 = vector.transfer_read %memref0[%i, %i], %cst: memref<?x?xf32>, vector<2xf32>
+      %r11 = vector.transfer_read %memref0[%random_index, %random_index], %cst: memref<?x?xf32>, vector<2xf32>
+      %u00 = "some_use"(%r00) : (vector<2xf32>) -> vector<2xf32>
+      %u01 = "some_use"(%r01) : (vector<2xf32>) -> vector<2xf32>
+      %u20 = "some_use"(%r20) : (vector<3xf32>) -> vector<3xf32>
+      %u21 = "some_use"(%r21) : (vector<3xf32>) -> vector<3xf32>
+      %u30 = "some_use"(%r30) : (vector<4xf32>) -> vector<4xf32>
+      %u31 = "some_use"(%r31) : (vector<4xf32>) -> vector<4xf32>
+      %u10 = "some_use"(%r10) : (vector<2xf32>) -> vector<2xf32>
+      %u11 = "some_use"(%r11) : (vector<2xf32>) -> vector<2xf32>
+      vector.transfer_write %u00, %memref1[%c0, %c0] : vector<2xf32>, memref<?x?xf32>
+      vector.transfer_write %u01, %memref1[%c0, %c1] : vector<2xf32>, memref<?x?xf32>
+      vector.transfer_write %u20, %memref2[%c0, %c0] : vector<3xf32>, memref<?x?xf32>
+      vector.transfer_write %u21, %memref2[%c0, %c3] : vector<3xf32>, memref<?x?xf32>
+      vector.transfer_write %u30, %memref3[%c0, %random_index] : vector<4xf32>, memref<?x?xf32>
+      vector.transfer_write %u31, %memref3[%c1, %random_index] : vector<4xf32>, memref<?x?xf32>
+      vector.transfer_write %u10, %memref0[%i, %i] : vector<2xf32>, memref<?x?xf32>
+      vector.transfer_write %u11, %memref0[%random_index, %random_index] : vector<2xf32>, memref<?x?xf32>
+    }
+  }
+  return
+}
diff --git a/mlir/test/Dialect/Linalg/promote.mlir b/mlir/test/Dialect/Linalg/promote.mlir
index fe36755345ef2..ecf8e20e7ef47 100644
--- a/mlir/test/Dialect/Linalg/promote.mlir
+++ b/mlir/test/Dialect/Linalg/promote.mlir
@@ -1,5 +1,6 @@
 // RUN: mlir-opt %s -linalg-promote-subviews | FileCheck %s
 // RUN: mlir-opt %s -linalg-promote-subviews="test-promote-dynamic" | FileCheck %s --check-prefix=DYNAMIC
+// RUN: mlir-opt %s -linalg-promote-subviews="test-use-alloca" | FileCheck %s --check-prefix=ALLOCA
 
 #map1 = affine_map<(d0) -> (d0 + 2)>
 #map2 = affine_map<(d0) -> (d0 + 4)>
@@ -45,16 +46,19 @@ func @matmul_f32(%A: memref<?xi8>, %M: index, %N: index, %K: index) {
 //       CHECK:         %[[vC:.*]] = subview {{.*}} : memref<?x?xf32>
 ///
 //       CHECK:         %[[tmpA:.*]] = alloc() : memref<32xi8>
+//      ALLOCA:         %[[tmpA:.*]] = alloca() : memref<32xi8>
 //       CHECK:         %[[fullA:.*]] = std.view %[[tmpA]][{{.*}}][{{.*}}] : memref<32xi8> to memref<?x?xf32>
 //     DYNAMIC:         std.view %{{.*}}[{{.*}}][{{.*}}] : memref<?xi8> to memref<?x?xf32>
 //       CHECK:         %[[partialA:.*]] = subview %[[fullA]]{{.*}} : memref<?x?xf32> to memref<?x?xf32, #[[$strided2D_dynamic]]>
 ///
 //       CHECK:         %[[tmpB:.*]] = alloc() : memref<48xi8>
+//      ALLOCA:         %[[tmpB:.*]] = alloca() : memref<48xi8>
 //       CHECK:         %[[fullB:.*]] = std.view %[[tmpB]][{{.*}}][{{.*}}] : memref<48xi8> to memref<?x?xf32>
 //     DYNAMIC:         std.view %{{.*}}[{{.*}}][{{.*}}] : memref<?xi8> to memref<?x?xf32>
 //       CHECK:         %[[partialB:.*]] = subview %[[fullB]]{{.*}} : memref<?x?xf32> to memref<?x?xf32, #[[$strided2D_dynamic]]>
 ///
 //       CHECK:         %[[tmpC:.*]] = alloc() : memref<24xi8>
+//      ALLOCA:         %[[tmpC:.*]] = alloca() : memref<24xi8>
 //       CHECK:         %[[fullC:.*]] = std.view %[[tmpC]][{{.*}}][{{.*}}] : memref<24xi8> to memref<?x?xf32>
 //     DYNAMIC:         std.view %{{.*}}[{{.*}}][{{.*}}] : memref<?xi8> to memref<?x?xf32>
 //       CHECK:         %[[partialC:.*]] = subview %[[fullC]]{{.*}} : memref<?x?xf32> to memref<?x?xf32, #[[$strided2D_dynamic]]>
@@ -75,6 +79,9 @@ func @matmul_f32(%A: memref<?xi8>, %M: index, %N: index, %K: index) {
 //       CHECK:         dealloc %[[tmpA]] : memref<32xi8>
 //       CHECK:         dealloc %[[tmpB]] : memref<48xi8>
 //       CHECK:         dealloc %[[tmpC]] : memref<24xi8>
+//  ALLOCA-NOT:         dealloc %[[tmpA]] : memref<32xi8>
+//  ALLOCA-NOT:         dealloc %[[tmpB]] : memref<48xi8>
+//  ALLOCA-NOT:         dealloc %[[tmpC]] : memref<24xi8>
 
 // -----
 
diff --git a/mlir/test/Dialect/Linalg/roundtrip.mlir b/mlir/test/Dialect/Linalg/roundtrip.mlir
index f210b185331c5..9e6c275479302 100644
--- a/mlir/test/Dialect/Linalg/roundtrip.mlir
+++ b/mlir/test/Dialect/Linalg/roundtrip.mlir
@@ -1,7 +1,7 @@
 // RUN: mlir-opt -split-input-file %s | FileCheck %s
 // | mlir-opt | FileCheck %s
 
-// TODO(pifon): Re-enable LLVM lowering test after IndexedGenericOp is lowered.
+// TODO: Re-enable LLVM lowering test after IndexedGenericOp is lowered.
 //
 // Test that we can lower all the way to LLVM without crashing, don't check results here.
 // DISABLED: mlir-opt %s --convert-linalg-to-llvm -o=/dev/null 2>&1
diff --git a/mlir/test/Dialect/Linalg/transform-patterns-matmul-to-vector.mlir b/mlir/test/Dialect/Linalg/transform-patterns-matmul-to-vector.mlir
index cf75ee5691d0f..b0702f9fdcfd8 100644
--- a/mlir/test/Dialect/Linalg/transform-patterns-matmul-to-vector.mlir
+++ b/mlir/test/Dialect/Linalg/transform-patterns-matmul-to-vector.mlir
@@ -1,5 +1,6 @@
 // RUN: mlir-opt %s -test-linalg-transform-patterns=test-matmul-to-vector-patterns-tile-1d | FileCheck %s
 // RUN: mlir-opt %s -test-linalg-transform-patterns=test-matmul-to-vector-patterns-tile-2d | FileCheck %s
+// RUN: mlir-opt %s -test-linalg-transform-patterns=test-contraction-to-vector-patterns | FileCheck %s --check-prefix=VECTOR-CONTRACTION
 
 func @matmul(%A: memref<1584x1584xf32, offset: 0, strides: [1584, 1]>,
                   %B: memref<1584x1584xf32, offset: 0, strides: [1584, 1]>,
@@ -30,3 +31,38 @@ func @matmul(%A: memref<1584x1584xf32, offset: 0, strides: [1584, 1]>,
 // CHECK-SAME:   : vector<8x16xf32>, vector<16x12xf32> into vector<8x12xf32>
 //
 //      CHECK: linalg.copy
+
+// VECTOR-CONTRACTION-LABEL: contraction_dot
+func @contraction_dot(%A: memref<1584xf32>, %B: memref<1584xf32>, %C: memref<f32>) {
+  // VECTOR-CONTRACTION: vector.contract
+  // VECTOR-CONTRACTION-SAME: vector<1584xf32>, vector<1584xf32> into f32
+  linalg.dot(%A, %B, %C) : memref<1584xf32>, memref<1584xf32>, memref<f32>
+  return
+}
+
+// VECTOR-CONTRACTION-LABEL: contraction_matvec
+func @contraction_matvec(%A: memref<1584x1584xf32>, %B: memref<1584xf32>, %C: memref<1584xf32>) {
+  // VECTOR-CONTRACTION: vector.contract
+  // VECTOR-CONTRACTION-SAME: vector<1584x1584xf32>, vector<1584xf32> into vector<1584xf32>
+  linalg.matvec %A, %B, %C :
+    (memref<1584x1584xf32>, memref<1584xf32>, memref<1584xf32>)
+  return
+}
+
+// VECTOR-CONTRACTION-LABEL: contraction_matmul
+func @contraction_matmul(%A: memref<1584x1584xf32>, %B: memref<1584x1584xf32>, %C: memref<1584x1584xf32>) {
+  // VECTOR-CONTRACTION: vector.contract
+  // VECTOR-CONTRACTION-SAME: vector<1584x1584xf32>, vector<1584x1584xf32> into vector<1584x1584xf32>
+  linalg.matmul %A, %B, %C :
+    (memref<1584x1584xf32>, memref<1584x1584xf32>, memref<1584x1584xf32>)
+  return
+}
+
+// VECTOR-CONTRACTION-LABEL: contraction_batch_matmul
+func @contraction_batch_matmul(%A: memref<1584x1584x1584xf32>, %B: memref<1584x1584x1584xf32>, %C: memref<1584x1584x1584xf32>) {
+  // VECTOR-CONTRACTION: vector.contract
+  // VECTOR-CONTRACTION-SAME: vector<1584x1584x1584xf32>, vector<1584x1584x1584xf32> into vector<1584x1584x1584xf32>
+  linalg.batch_matmul %A, %B, %C :
+    (memref<1584x1584x1584xf32>, memref<1584x1584x1584xf32>, memref<1584x1584x1584xf32>)
+  return
+}
diff --git a/mlir/test/Dialect/SCF/canonicalize.mlir b/mlir/test/Dialect/SCF/canonicalize.mlir
new file mode 100644
index 0000000000000..fc98dabc0d2d6
--- /dev/null
+++ b/mlir/test/Dialect/SCF/canonicalize.mlir
@@ -0,0 +1,55 @@
+// RUN: mlir-opt %s -pass-pipeline='func(canonicalize)' | FileCheck %s
+
+func @single_iteration(%A: memref<?x?x?xi32>) {
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %c2 = constant 2 : index
+  %c3 = constant 3 : index
+  %c6 = constant 6 : index
+  %c7 = constant 7 : index
+  %c10 = constant 10 : index
+  scf.parallel (%i0, %i1, %i2) = (%c0, %c3, %c7) to (%c1, %c6, %c10) step (%c1, %c2, %c3) {
+    %c42 = constant 42 : i32
+    store %c42, %A[%i0, %i1, %i2] : memref<?x?x?xi32>
+    scf.yield
+  }
+  return
+}
+
+// CHECK-LABEL:   func @single_iteration(
+// CHECK-SAME:                        [[ARG0:%.*]]: memref<?x?x?xi32>) {
+// CHECK:           [[C0:%.*]] = constant 0 : index
+// CHECK:           [[C2:%.*]] = constant 2 : index
+// CHECK:           [[C3:%.*]] = constant 3 : index
+// CHECK:           [[C6:%.*]] = constant 6 : index
+// CHECK:           [[C7:%.*]] = constant 7 : index
+// CHECK:           [[C42:%.*]] = constant 42 : i32
+// CHECK:           scf.parallel ([[V0:%.*]]) = ([[C3]]) to ([[C6]]) step ([[C2]]) {
+// CHECK:             store [[C42]], [[ARG0]]{{\[}}[[C0]], [[V0]], [[C7]]] : memref<?x?x?xi32>
+// CHECK:             scf.yield
+// CHECK:           }
+// CHECK:           return
+
+// -----
+
+func @no_iteration(%A: memref<?x?xi32>) {
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  scf.parallel (%i0, %i1) = (%c0, %c0) to (%c1, %c0) step (%c1, %c1) {
+    %c42 = constant 42 : i32
+    store %c42, %A[%i0, %i1] : memref<?x?xi32>
+    scf.yield
+  }
+  return
+}
+
+// CHECK-LABEL:   func @no_iteration(
+// CHECK-SAME:                        [[ARG0:%.*]]: memref<?x?xi32>) {
+// CHECK:           [[C0:%.*]] = constant 0 : index
+// CHECK:           [[C1:%.*]] = constant 1 : index
+// CHECK:           [[C42:%.*]] = constant 42 : i32
+// CHECK:           scf.parallel ([[V1:%.*]]) = ([[C0]]) to ([[C0]]) step ([[C1]]) {
+// CHECK:             store [[C42]], [[ARG0]]{{\[}}[[C0]], [[V1]]] : memref<?x?xi32>
+// CHECK:             scf.yield
+// CHECK:           }
+// CHECK:           return
diff --git a/mlir/test/Dialect/SCF/for-loop-specialization.mlir b/mlir/test/Dialect/SCF/for-loop-specialization.mlir
new file mode 100644
index 0000000000000..f7b501e0c95c2
--- /dev/null
+++ b/mlir/test/Dialect/SCF/for-loop-specialization.mlir
@@ -0,0 +1,39 @@
+// RUN: mlir-opt %s -for-loop-specialization -split-input-file | FileCheck %s
+
+#map0 = affine_map<()[s0, s1] -> (1024, s0 - s1)>
+#map1 = affine_map<()[s0, s1] -> (64, s0 - s1)>
+
+func @for(%outer: index, %A: memref<?xf32>, %B: memref<?xf32>,
+          %C: memref<?xf32>, %result: memref<?xf32>) {
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %d0 = dim %A, %c0 : memref<?xf32>
+  %b0 = affine.min #map0()[%d0, %outer]
+  scf.for %i0 = %c0 to %b0 step %c1 {
+    %B_elem = load %B[%i0] : memref<?xf32>
+    %C_elem = load %C[%i0] : memref<?xf32>
+    %sum_elem = addf %B_elem, %C_elem : f32
+    store %sum_elem, %result[%i0] : memref<?xf32>
+  }
+  return
+}
+
+// CHECK-LABEL:   func @for(
+// CHECK-SAME:              [[ARG0:%.*]]: index, [[ARG1:%.*]]: memref<?xf32>, [[ARG2:%.*]]: memref<?xf32>, [[ARG3:%.*]]: memref<?xf32>, [[ARG4:%.*]]: memref<?xf32>) {
+// CHECK:           [[CST_0:%.*]] = constant 0 : index
+// CHECK:           [[CST_1:%.*]] = constant 1 : index
+// CHECK:           [[DIM_0:%.*]] = dim [[ARG1]], [[CST_0]] : memref<?xf32>
+// CHECK:           [[MIN:%.*]] = affine.min #map0(){{\[}}[[DIM_0]], [[ARG0]]]
+// CHECK:           [[CST_1024:%.*]] = constant 1024 : index
+// CHECK:           [[PRED:%.*]] = cmpi "eq", [[MIN]], [[CST_1024]] : index
+// CHECK:           scf.if [[PRED]] {
+// CHECK:             scf.for [[IDX0:%.*]] = [[CST_0]] to [[CST_1024]] step [[CST_1]] {
+// CHECK:               store
+// CHECK:             }
+// CHECK:           } else {
+// CHECK:             scf.for [[IDX0:%.*]] = [[CST_0]] to [[MIN]] step [[CST_1]] {
+// CHECK:               store
+// CHECK:             }
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
diff --git a/mlir/test/Dialect/SCF/invalid.mlir b/mlir/test/Dialect/SCF/invalid.mlir
index 96813e5bc7111..37e760495eb16 100644
--- a/mlir/test/Dialect/SCF/invalid.mlir
+++ b/mlir/test/Dialect/SCF/invalid.mlir
@@ -105,7 +105,7 @@ func @loop_if_not_one_block_per_region(%arg0: i1) {
 // -----
 
 func @loop_if_illegal_block_argument(%arg0: i1) {
-  // expected-error@+1 {{requires that child entry blocks have no arguments}}
+  // expected-error@+1 {{region #0 should have no arguments}}
   "scf.if"(%arg0) ({
     ^bb0(%0 : index):
       scf.yield
diff --git a/mlir/test/Dialect/SCF/parallel-loop-fusion.mlir b/mlir/test/Dialect/SCF/parallel-loop-fusion.mlir
index 8e6769961c10c..24026b4841366 100644
--- a/mlir/test/Dialect/SCF/parallel-loop-fusion.mlir
+++ b/mlir/test/Dialect/SCF/parallel-loop-fusion.mlir
@@ -10,7 +10,7 @@ func @fuse_empty_loops() {
   scf.parallel (%i, %j) = (%c0, %c0) to (%c2, %c2) step (%c1, %c1) {
     scf.yield
   }
-  "xla_lhlo.terminator"() : () -> ()
+  return
 }
 // CHECK-LABEL: func @fuse_empty_loops
 // CHECK:        [[C2:%.*]] = constant 2 : index
@@ -140,7 +140,7 @@ func @do_not_fuse_nested_ploop1() {
   scf.parallel (%i, %j) = (%c0, %c0) to (%c2, %c2) step (%c1, %c1) {
     scf.yield
   }
-  "xla_lhlo.terminator"() : () -> ()
+  return
 }
 // CHECK-LABEL: func @do_not_fuse_nested_ploop1
 // CHECK:        scf.parallel
@@ -162,7 +162,7 @@ func @do_not_fuse_nested_ploop2() {
     }
     scf.yield
   }
-  "xla_lhlo.terminator"() : () -> ()
+  return
 }
 // CHECK-LABEL: func @do_not_fuse_nested_ploop2
 // CHECK:        scf.parallel
@@ -181,7 +181,7 @@ func @do_not_fuse_loops_unmatching_num_loops() {
   scf.parallel (%i) = (%c0) to (%c2) step (%c1) {
     scf.yield
   }
-  "xla_lhlo.terminator"() : () -> ()
+  return
 }
 // CHECK-LABEL: func @do_not_fuse_loops_unmatching_num_loops
 // CHECK:        scf.parallel
@@ -200,7 +200,7 @@ func @do_not_fuse_loops_with_side_effecting_ops_in_between() {
   scf.parallel (%i, %j) = (%c0, %c0) to (%c2, %c2) step (%c1, %c1) {
     scf.yield
   }
-  "xla_lhlo.terminator"() : () -> ()
+  return
 }
 // CHECK-LABEL: func @do_not_fuse_loops_with_side_effecting_ops_in_between
 // CHECK:        scf.parallel
@@ -219,7 +219,7 @@ func @do_not_fuse_loops_unmatching_iteration_space() {
   scf.parallel (%i, %j) = (%c0, %c0) to (%c2, %c2) step (%c1, %c1) {
     scf.yield
   }
-  "xla_lhlo.terminator"() : () -> ()
+  return
 }
 // CHECK-LABEL: func @do_not_fuse_loops_unmatching_iteration_space
 // CHECK:        scf.parallel
@@ -302,7 +302,7 @@ func @do_not_fuse_loops_with_memref_defined_in_loop_bodies() {
     %A_elem = load %A[%i, %j] : memref<?x?xf32, offset: ?, strides:[?, ?]>
     scf.yield
   }
-  "xla_lhlo.terminator"() : () -> ()
+  return
 }
 // CHECK-LABEL: func @do_not_fuse_loops_with_memref_defined_in_loop_bodies
 // CHECK:        scf.parallel
diff --git a/mlir/test/Dialect/SCF/parallel-loop-tiling.mlir b/mlir/test/Dialect/SCF/parallel-loop-tiling.mlir
index f12416266ed99..6dff4eeda9e75 100644
--- a/mlir/test/Dialect/SCF/parallel-loop-tiling.mlir
+++ b/mlir/test/Dialect/SCF/parallel-loop-tiling.mlir
@@ -15,22 +15,52 @@ func @parallel_loop(%arg0 : index, %arg1 : index, %arg2 : index,
 
 // CHECK:       #map0 = affine_map<(d0, d1, d2) -> (d0, d1 - d2)>
 // CHECK-LABEL:   func @parallel_loop(
-// CHECK-SAME:                        [[VAL_0:%.*]]: index, [[VAL_1:%.*]]: index, [[VAL_2:%.*]]: index, [[VAL_3:%.*]]: index, [[VAL_4:%.*]]: index, [[VAL_5:%.*]]: index, [[VAL_6:%.*]]: memref<?x?xf32>, [[VAL_7:%.*]]: memref<?x?xf32>, [[VAL_8:%.*]]: memref<?x?xf32>, [[VAL_9:%.*]]: memref<?x?xf32>) {
-// CHECK:           [[VAL_10:%.*]] = constant 0 : index
-// CHECK:           [[VAL_11:%.*]] = constant 1 : index
-// CHECK:           [[VAL_12:%.*]] = constant 4 : index
-// CHECK:           [[VAL_13:%.*]] = muli [[VAL_4]], [[VAL_11]] : index
-// CHECK:           [[VAL_14:%.*]] = muli [[VAL_5]], [[VAL_12]] : index
-// CHECK:           scf.parallel ([[VAL_15:%.*]], [[VAL_16:%.*]]) = ([[VAL_0]], [[VAL_1]]) to ([[VAL_2]], [[VAL_3]]) step ([[VAL_13]], [[VAL_14]]) {
-// CHECK:             [[VAL_17:%.*]] = affine.min #map0([[VAL_11]], [[VAL_2]], [[VAL_15]])
-// CHECK:             [[VAL_18:%.*]] = affine.min #map0([[VAL_12]], [[VAL_3]], [[VAL_16]])
-// CHECK:             scf.parallel ([[VAL_19:%.*]], [[VAL_20:%.*]]) = ([[VAL_10]], [[VAL_10]]) to ([[VAL_17]], [[VAL_18]]) step ([[VAL_4]], [[VAL_5]]) {
-// CHECK:               [[VAL_21:%.*]] = addi [[VAL_19]], [[VAL_15]] : index
-// CHECK:               [[VAL_22:%.*]] = addi [[VAL_20]], [[VAL_16]] : index
-// CHECK:               [[VAL_23:%.*]] = load [[VAL_7]]{{\[}}[[VAL_21]], [[VAL_22]]] : memref<?x?xf32>
-// CHECK:               [[VAL_24:%.*]] = load [[VAL_8]]{{\[}}[[VAL_21]], [[VAL_22]]] : memref<?x?xf32>
-// CHECK:               [[VAL_25:%.*]] = addf [[VAL_23]], [[VAL_24]] : f32
-// CHECK:               store [[VAL_25]], [[VAL_9]]{{\[}}[[VAL_21]], [[VAL_22]]] : memref<?x?xf32>
+// CHECK-SAME:                        [[ARG1:%.*]]: index, [[ARG2:%.*]]: index, [[ARG3:%.*]]: index, [[ARG4:%.*]]: index, [[ARG5:%.*]]: index, [[ARG6:%.*]]: index, [[ARG7:%.*]]: memref<?x?xf32>, [[ARG8:%.*]]: memref<?x?xf32>, [[ARG9:%.*]]: memref<?x?xf32>, [[ARG10:%.*]]: memref<?x?xf32>) {
+// CHECK:           [[C0:%.*]] = constant 0 : index
+// CHECK:           [[C1:%.*]] = constant 1 : index
+// CHECK:           [[C4:%.*]] = constant 4 : index
+// CHECK:           [[V1:%.*]] = muli [[ARG5]], [[C1]] : index
+// CHECK:           [[V2:%.*]] = muli [[ARG6]], [[C4]] : index
+// CHECK:           scf.parallel ([[V3:%.*]], [[V4:%.*]]) = ([[ARG1]], [[ARG2]]) to ([[ARG3]], [[ARG4]]) step ([[V1]], [[V2]]) {
+// CHECK:             [[V5:%.*]] = affine.min #map0([[V1]], [[ARG3]], [[V3]])
+// CHECK:             [[V6:%.*]] = affine.min #map0([[V2]], [[ARG4]], [[V4]])
+// CHECK:             scf.parallel ([[V7:%.*]], [[V8:%.*]]) = ([[C0]], [[C0]]) to ([[V5]], [[V6]]) step ([[ARG5]], [[ARG6]]) {
+// CHECK:               [[V9:%.*]] = addi [[V7]], [[V3]] : index
+// CHECK:               [[V10:%.*]] = addi [[V8]], [[V4]] : index
+// CHECK:               [[V11:%.*]] = load [[ARG8]]{{\[}}[[V9]], [[V10]]] : memref<?x?xf32>
+// CHECK:               [[V12:%.*]] = load [[ARG9]]{{\[}}[[V9]], [[V10]]] : memref<?x?xf32>
+// CHECK:               [[V13:%.*]] = addf [[V11]], [[V12]] : f32
+// CHECK:               store [[V13]], [[ARG10]]{{\[}}[[V9]], [[V10]]] : memref<?x?xf32>
+// CHECK:             }
+// CHECK:           }
+// CHECK:           return
+
+// -----
+
+func @static_loop_with_step() {
+  %c0 = constant 0 : index
+  %c3 = constant 3 : index
+  %c22 = constant 22 : index
+  %c24 = constant 24 : index
+  scf.parallel (%i0, %i1) = (%c0, %c0) to (%c22, %c24) step (%c3, %c3) {
+  }
+  return
+}
+
+// CHECK-LABEL:   func @static_loop_with_step() {
+// CHECK:           [[C0:%.*]] = constant 0 : index
+// CHECK:           [[C3:%.*]] = constant 3 : index
+// CHECK:           [[C22:%.*]] = constant 22 : index
+// CHECK:           [[C24:%.*]] = constant 24 : index
+// CHECK:           [[C0_1:%.*]] = constant 0 : index
+// CHECK:           [[C1:%.*]] = constant 1 : index
+// CHECK:           [[C4:%.*]] = constant 4 : index
+// CHECK:           [[V1:%.*]] = muli [[C3]], [[C1]] : index
+// CHECK:           [[V2:%.*]] = muli [[C3]], [[C4]] : index
+// CHECK:           scf.parallel ([[V3:%.*]], [[V4:%.*]]) = ([[C0]], [[C0]]) to ([[C22]], [[C24]]) step ([[V1]], [[V2]]) {
+// CHECK:             scf.parallel ([[V5:%.*]], [[V6:%.*]]) = ([[C0_1]], [[C0_1]]) to ([[V1]], [[V2]]) step ([[C3]], [[C3]]) {
+// CHECK:               = addi [[V5]], [[V3]] : index
+// CHECK:               = addi [[V6]], [[V4]] : index
 // CHECK:             }
 // CHECK:           }
 // CHECK:           return
@@ -51,31 +81,33 @@ func @tile_nested_innermost() {
 }
 
 // CHECK-LABEL:   func @tile_nested_innermost() {
-// CHECK:           [[VAL_24:%.*]] = constant 2 : index
-// CHECK:           [[VAL_25:%.*]] = constant 0 : index
-// CHECK:           [[VAL_26:%.*]] = constant 1 : index
-// CHECK:           scf.parallel ([[VAL_27:%.*]], [[VAL_28:%.*]]) = ([[VAL_25]], [[VAL_25]]) to ([[VAL_24]], [[VAL_24]]) step ([[VAL_26]], [[VAL_26]]) {
-// CHECK:             [[VAL_29:%.*]] = constant 0 : index
-// CHECK:             [[VAL_30:%.*]] = constant 1 : index
-// CHECK:             [[VAL_31:%.*]] = constant 4 : index
-// CHECK:             [[VAL_32:%.*]] = muli [[VAL_26]], [[VAL_30]] : index
-// CHECK:             [[VAL_33:%.*]] = muli [[VAL_26]], [[VAL_31]] : index
-// CHECK:             scf.parallel ([[VAL_34:%.*]], [[VAL_35:%.*]]) = ([[VAL_25]], [[VAL_25]]) to ([[VAL_24]], [[VAL_24]]) step ([[VAL_32]], [[VAL_33]]) {
-// CHECK:               [[VAL_36:%.*]] = affine.min #map0([[VAL_30]], [[VAL_24]], [[VAL_34]])
-// CHECK:               [[VAL_37:%.*]] = affine.min #map0([[VAL_31]], [[VAL_24]], [[VAL_35]])
-// CHECK:               scf.parallel ([[VAL_38:%.*]], [[VAL_39:%.*]]) = ([[VAL_29]], [[VAL_29]]) to ([[VAL_36]], [[VAL_37]]) step ([[VAL_26]], [[VAL_26]]) {
+// CHECK:           [[C2:%.*]] = constant 2 : index
+// CHECK:           [[C0:%.*]] = constant 0 : index
+// CHECK:           [[C1:%.*]] = constant 1 : index
+// CHECK:           scf.parallel ([[V1:%.*]], [[V2:%.*]]) = ([[C0]], [[C0]]) to ([[C2]], [[C2]]) step ([[C1]], [[C1]]) {
+// CHECK:             [[C0_1:%.*]] = constant 0 : index
+// CHECK:             [[C1_1:%.*]] = constant 1 : index
+// CHECK:             [[C4:%.*]] = constant 4 : index
+// CHECK:             [[V3:%.*]] = muli [[C1]], [[C1_1]] : index
+// CHECK:             [[V4:%.*]] = muli [[C1]], [[C4]] : index
+// CHECK:             scf.parallel ([[V5:%.*]], [[V6:%.*]]) = ([[C0]], [[C0]]) to ([[C2]], [[C2]]) step ([[V3]], [[V4]]) {
+// CHECK:               [[V7:%.*]] = affine.min #map0([[V4]], [[C2]], [[V6]])
+// CHECK:               scf.parallel ([[V8:%.*]], [[V9:%.*]]) = ([[C0_1]], [[C0_1]]) to ([[V3]], [[V7]]) step ([[C1]], [[C1]]) {
+// CHECK:                 = addi [[V8]], [[V5]] : index
+// CHECK:                 = addi [[V9]], [[V6]] : index
 // CHECK:               }
 // CHECK:             }
 // CHECK:           }
-// CHECK:           [[VAL_40:%.*]] = constant 0 : index
-// CHECK:           [[VAL_41:%.*]] = constant 1 : index
-// CHECK:           [[VAL_42:%.*]] = constant 4 : index
-// CHECK:           [[VAL_43:%.*]] = muli [[VAL_26]], [[VAL_41]] : index
-// CHECK:           [[VAL_44:%.*]] = muli [[VAL_26]], [[VAL_42]] : index
-// CHECK:           scf.parallel ([[VAL_45:%.*]], [[VAL_46:%.*]]) = ([[VAL_25]], [[VAL_25]]) to ([[VAL_24]], [[VAL_24]]) step ([[VAL_43]], [[VAL_44]]) {
-// CHECK:             [[VAL_47:%.*]] = affine.min #map0([[VAL_41]], [[VAL_24]], [[VAL_45]])
-// CHECK:             [[VAL_48:%.*]] = affine.min #map0([[VAL_42]], [[VAL_24]], [[VAL_46]])
-// CHECK:             scf.parallel ([[VAL_49:%.*]], [[VAL_50:%.*]]) = ([[VAL_40]], [[VAL_40]]) to ([[VAL_47]], [[VAL_48]]) step ([[VAL_26]], [[VAL_26]]) {
+// CHECK:           [[C0_2:%.*]] = constant 0 : index
+// CHECK:           [[C1_2:%.*]] = constant 1 : index
+// CHECK:           [[C4_1:%.*]] = constant 4 : index
+// CHECK:           [[V10:%.*]] = muli [[C1]], [[C1_2]] : index
+// CHECK:           [[V11:%.*]] = muli [[C1]], [[C4_1]] : index
+// CHECK:           scf.parallel ([[V12:%.*]], [[V13:%.*]]) = ([[C0]], [[C0]]) to ([[C2]], [[C2]]) step ([[V10]], [[V11]]) {
+// CHECK:             [[V14:%.*]] = affine.min #map0([[V11]], [[C2]], [[V13]])
+// CHECK:             scf.parallel ([[V15:%.*]], [[V16:%.*]]) = ([[C0_2]], [[C0_2]]) to ([[V10]], [[V14]]) step ([[C1]], [[C1]]) {
+// CHECK:               = addi [[V15]], [[V12]] : index
+// CHECK:               = addi [[V16]], [[V13]] : index
 // CHECK:             }
 // CHECK:           }
 // CHECK:           return
diff --git a/mlir/test/Dialect/SPIRV/Serialization/array.mlir b/mlir/test/Dialect/SPIRV/Serialization/array.mlir
index 0d14cbf9d3b6a..68986741b32d7 100644
--- a/mlir/test/Dialect/SPIRV/Serialization/array.mlir
+++ b/mlir/test/Dialect/SPIRV/Serialization/array.mlir
@@ -2,8 +2,8 @@
 
 spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
   spv.func @array_stride(%arg0 : !spv.ptr<!spv.array<4x!spv.array<4xf32, stride=4>, stride=128>, StorageBuffer>, %arg1 : i32, %arg2 : i32) "None" {
-    // CHECK: {{%.*}} = spv.AccessChain {{%.*}}[{{%.*}}, {{%.*}}] : !spv.ptr<!spv.array<4 x !spv.array<4 x f32, stride=4>, stride=128>, StorageBuffer>
-    %2 = spv.AccessChain %arg0[%arg1, %arg2] : !spv.ptr<!spv.array<4x!spv.array<4xf32, stride=4>, stride=128>, StorageBuffer>
+    // CHECK: {{%.*}} = spv.AccessChain {{%.*}}[{{%.*}}, {{%.*}}] : !spv.ptr<!spv.array<4 x !spv.array<4 x f32, stride=4>, stride=128>, StorageBuffer>, i32, i32
+    %2 = spv.AccessChain %arg0[%arg1, %arg2] : !spv.ptr<!spv.array<4x!spv.array<4xf32, stride=4>, stride=128>, StorageBuffer>, i32, i32
     spv.Return
   }
 }
diff --git a/mlir/test/Dialect/SPIRV/Serialization/constant.mlir b/mlir/test/Dialect/SPIRV/Serialization/constant.mlir
index a276e4ee97817..520669262e128 100644
--- a/mlir/test/Dialect/SPIRV/Serialization/constant.mlir
+++ b/mlir/test/Dialect/SPIRV/Serialization/constant.mlir
@@ -108,7 +108,7 @@ spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
 
   // CHECK-LABEL: @double_const
   spv.func @double_const() -> () "None" {
-    // TODO(antiagainst): test range boundary values
+    // TODO: test range boundary values
     // CHECK: spv.constant 1.024000e+03 : f64
     %0 = spv.constant 1024. : f64
     // CHECK: spv.constant -1.024000e+03 : f64
diff --git a/mlir/test/Dialect/SPIRV/Serialization/cooperative-matrix.mlir b/mlir/test/Dialect/SPIRV/Serialization/cooperative-matrix.mlir
index 0d58fea18a119..ad913dfb1624c 100644
--- a/mlir/test/Dialect/SPIRV/Serialization/cooperative-matrix.mlir
+++ b/mlir/test/Dialect/SPIRV/Serialization/cooperative-matrix.mlir
@@ -95,8 +95,8 @@ spv.module Logical GLSL450 requires #spv.vce<v1.0, [CooperativeMatrixNV], [SPV_N
   // CHECK-LABEL: @cooperative_matrix_access_chain
   spv.func @cooperative_matrix_access_chain(%a : !spv.ptr<!spv.coopmatrix<8x16xf32, Subgroup>, Function>) -> !spv.ptr<f32, Function> "None" {
     %0 = spv.constant 0: i32
-    // CHECK: {{%.*}} = spv.AccessChain {{%.*}}[{{%.*}}] : !spv.ptr<!spv.coopmatrix<8x16xf32, Subgroup>, Function>
-    %1 = spv.AccessChain %a[%0] : !spv.ptr<!spv.coopmatrix<8x16xf32, Subgroup>, Function>
+    // CHECK: {{%.*}} = spv.AccessChain {{%.*}}[{{%.*}}] : !spv.ptr<!spv.coopmatrix<8x16xf32, Subgroup>, Function>, i32
+    %1 = spv.AccessChain %a[%0] : !spv.ptr<!spv.coopmatrix<8x16xf32, Subgroup>, Function>, i32
     spv.ReturnValue %1 : !spv.ptr<f32, Function>
   }
 }
diff --git a/mlir/test/Dialect/SPIRV/Serialization/debug.mlir b/mlir/test/Dialect/SPIRV/Serialization/debug.mlir
index aa9653da4f2ef..d83030d252984 100644
--- a/mlir/test/Dialect/SPIRV/Serialization/debug.mlir
+++ b/mlir/test/Dialect/SPIRV/Serialization/debug.mlir
@@ -58,7 +58,7 @@ spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
 
   spv.func @memory_accesses(%arg0 : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, StorageBuffer>, %arg1 : i32, %arg2 : i32) "None" {
     // CHECK: loc({{".*debug.mlir"}}:61:10)
-    %2 = spv.AccessChain %arg0[%arg1, %arg2] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, StorageBuffer>
+    %2 = spv.AccessChain %arg0[%arg1, %arg2] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, StorageBuffer>, i32, i32
     // CHECK: loc({{".*debug.mlir"}}:63:10)
     %3 = spv.Load "StorageBuffer" %2 : f32
     // CHECK: loc({{.*debug.mlir"}}:65:5)
diff --git a/mlir/test/Dialect/SPIRV/Serialization/global-variable.mlir b/mlir/test/Dialect/SPIRV/Serialization/global-variable.mlir
index faa371ea90162..0365fef03fbbf 100644
--- a/mlir/test/Dialect/SPIRV/Serialization/global-variable.mlir
+++ b/mlir/test/Dialect/SPIRV/Serialization/global-variable.mlir
@@ -30,7 +30,7 @@ spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
     %0 = spv._address_of @globalInvocationID : !spv.ptr<vector<3xi32>, Input>
     %1 = spv.constant 0: i32
     // CHECK: spv.AccessChain %[[ADDR]]
-    %2 = spv.AccessChain %0[%1] : !spv.ptr<vector<3xi32>, Input>
+    %2 = spv.AccessChain %0[%1] : !spv.ptr<vector<3xi32>, Input>, i32
     spv.Return
   }
 }
diff --git a/mlir/test/Dialect/SPIRV/Serialization/loop.mlir b/mlir/test/Dialect/SPIRV/Serialization/loop.mlir
index 1b041a4aa6045..d6e2090f02bbb 100644
--- a/mlir/test/Dialect/SPIRV/Serialization/loop.mlir
+++ b/mlir/test/Dialect/SPIRV/Serialization/loop.mlir
@@ -65,9 +65,9 @@ spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
   spv.func @loop_kernel() "None" {
     %0 = spv._address_of @GV1 : !spv.ptr<!spv.struct<!spv.array<10 x f32, stride=4> [0]>, StorageBuffer>
     %1 = spv.constant 0 : i32
-    %2 = spv.AccessChain %0[%1] : !spv.ptr<!spv.struct<!spv.array<10 x f32, stride=4> [0]>, StorageBuffer>
+    %2 = spv.AccessChain %0[%1] : !spv.ptr<!spv.struct<!spv.array<10 x f32, stride=4> [0]>, StorageBuffer>, i32
     %3 = spv._address_of @GV2 : !spv.ptr<!spv.struct<!spv.array<10 x f32, stride=4> [0]>, StorageBuffer>
-    %5 = spv.AccessChain %3[%1] : !spv.ptr<!spv.struct<!spv.array<10 x f32, stride=4> [0]>, StorageBuffer>
+    %5 = spv.AccessChain %3[%1] : !spv.ptr<!spv.struct<!spv.array<10 x f32, stride=4> [0]>, StorageBuffer>, i32
     %6 = spv.constant 4 : i32
     %7 = spv.constant 42 : i32
     %8 = spv.constant 2 : i32
@@ -84,9 +84,9 @@ spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
       spv.BranchConditional %10, ^body, ^merge
 // CHECK-NEXT:   ^bb2:     // pred: ^bb1
     ^body:
-      %11 = spv.AccessChain %2[%9] : !spv.ptr<!spv.array<10 x f32, stride=4>, StorageBuffer>
+      %11 = spv.AccessChain %2[%9] : !spv.ptr<!spv.array<10 x f32, stride=4>, StorageBuffer>, i32
       %12 = spv.Load "StorageBuffer" %11 : f32
-      %13 = spv.AccessChain %5[%9] : !spv.ptr<!spv.array<10 x f32, stride=4>, StorageBuffer>
+      %13 = spv.AccessChain %5[%9] : !spv.ptr<!spv.array<10 x f32, stride=4>, StorageBuffer>, i32
       spv.Store "StorageBuffer" %13, %12 : f32
 // CHECK:          %[[ADD:.*]] = spv.IAdd
       %14 = spv.IAdd %9, %8 : i32
diff --git a/mlir/test/Dialect/SPIRV/Serialization/matrix.mlir b/mlir/test/Dialect/SPIRV/Serialization/matrix.mlir
index 8dc90cb504e87..8900ff6137dbb 100644
--- a/mlir/test/Dialect/SPIRV/Serialization/matrix.mlir
+++ b/mlir/test/Dialect/SPIRV/Serialization/matrix.mlir
@@ -4,7 +4,7 @@ spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
   // CHECK-LABEL: @matrix_access_chain
   spv.func @matrix_access_chain(%arg0 : !spv.ptr<!spv.matrix<3 x vector<3xf32>>, Function>, %arg1 : i32) -> !spv.ptr<vector<3xf32>, Function> "None" {
     // CHECK: {{%.*}} = spv.AccessChain {{%.*}}[{{%.*}}] : !spv.ptr<!spv.matrix<3 x vector<3xf32>>, Function>
-    %0 = spv.AccessChain %arg0[%arg1] : !spv.ptr<!spv.matrix<3 x vector<3xf32>>, Function>
+    %0 = spv.AccessChain %arg0[%arg1] : !spv.ptr<!spv.matrix<3 x vector<3xf32>>,Function>, i32
     spv.ReturnValue %0 : !spv.ptr<vector<3xf32>, Function>
   }
 
@@ -20,6 +20,28 @@ spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
     // CHECK: {{%.*}} = spv.MatrixTimesScalar {{%.*}}, {{%.*}} : !spv.matrix<3 x vector<3xf16>>, f16 -> !spv.matrix<3 x vector<3xf16>>
     %result = spv.MatrixTimesScalar %arg0, %arg1 : !spv.matrix<3 x vector<3xf16>>, f16 -> !spv.matrix<3 x vector<3xf16>>
     spv.ReturnValue %result : !spv.matrix<3 x vector<3xf16>>
+
+  }
+
+  // CHECK-LABEL: @matrix_transpose_1
+  spv.func @matrix_transpose_1(%arg0 : !spv.matrix<3 x vector<2xf32>>) -> !spv.matrix<2 x vector<3xf32>> "None" {
+    // CHECK: {{%.*}} = spv.Transpose {{%.*}} : !spv.matrix<3 x vector<2xf32>> -> !spv.matrix<2 x vector<3xf32>>
+    %result = spv.Transpose %arg0 : !spv.matrix<3 x vector<2xf32>> -> !spv.matrix<2 x vector<3xf32>>
+    spv.ReturnValue %result : !spv.matrix<2 x vector<3xf32>>
+  }
+
+  // CHECK-LABEL: @matrix_times_matrix_1
+  spv.func @matrix_times_matrix_1(%arg0: !spv.matrix<3 x vector<3xf32>>, %arg1: !spv.matrix<3 x vector<3xf32>>) -> !spv.matrix<3 x vector<3xf32>> "None"{
+    // CHECK: {{%.*}} = spv.MatrixTimesMatrix {{%.*}}, {{%.*}} : !spv.matrix<3 x vector<3xf32>>, !spv.matrix<3 x vector<3xf32>> -> !spv.matrix<3 x vector<3xf32>>
+    %result = spv.MatrixTimesMatrix %arg0, %arg1 : !spv.matrix<3 x vector<3xf32>>, !spv.matrix<3 x vector<3xf32>> -> !spv.matrix<3 x vector<3xf32>>
+    spv.ReturnValue %result : !spv.matrix<3 x vector<3xf32>>
+  }
+
+  // CHECK-LABEL: @matrix_times_matrix_2
+  spv.func @matrix_times_matrix_2(%arg0: !spv.matrix<3 x vector<2xf32>>, %arg1: !spv.matrix<2 x vector<3xf32>>) -> !spv.matrix<2 x vector<2xf32>> "None"{
+    // CHECK: {{%.*}} = spv.MatrixTimesMatrix {{%.*}}, {{%.*}} : !spv.matrix<3 x vector<2xf32>>, !spv.matrix<2 x vector<3xf32>> -> !spv.matrix<2 x vector<2xf32>>
+    %result = spv.MatrixTimesMatrix %arg0, %arg1 : !spv.matrix<3 x vector<2xf32>>, !spv.matrix<2 x vector<3xf32>> -> !spv.matrix<2 x vector<2xf32>>
+    spv.ReturnValue %result : !spv.matrix<2 x vector<2xf32>>
   }
 }
 
diff --git a/mlir/test/Dialect/SPIRV/Serialization/memory-ops.mlir b/mlir/test/Dialect/SPIRV/Serialization/memory-ops.mlir
index fbe45fa87d20f..0e18c1ea37bfb 100644
--- a/mlir/test/Dialect/SPIRV/Serialization/memory-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/Serialization/memory-ops.mlir
@@ -18,8 +18,8 @@ spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
   spv.func @access_chain(%arg0 : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>, %arg1 : i32, %arg2 : i32) "None" {
     // CHECK: {{%.*}} = spv.AccessChain {{%.*}}[{{%.*}}] : !spv.ptr<!spv.array<4 x !spv.array<4 x f32>>, Function>
     // CHECK-NEXT: {{%.*}} = spv.AccessChain {{%.*}}[{{%.*}}, {{%.*}}] : !spv.ptr<!spv.array<4 x !spv.array<4 x f32>>, Function>
-    %1 = spv.AccessChain %arg0[%arg1] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
-    %2 = spv.AccessChain %arg0[%arg1, %arg2] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
+    %1 = spv.AccessChain %arg0[%arg1] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>, i32
+    %2 = spv.AccessChain %arg0[%arg1, %arg2] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>, i32, i32
     spv.Return
   }
 }
@@ -31,13 +31,13 @@ spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
     // CHECK: [[LOAD_PTR:%.*]] = spv.AccessChain {{%.*}}[{{%.*}}, {{%.*}}] : !spv.ptr<!spv.struct<!spv.array<1 x f32, stride=4> [0]>
     // CHECK-NEXT: [[VAL:%.*]] = spv.Load "StorageBuffer" [[LOAD_PTR]] : f32
     %0 = spv.constant 0 : i32
-    %1 = spv.AccessChain %arg0[%0, %0] : !spv.ptr<!spv.struct<!spv.array<1 x f32, stride=4> [0]>, StorageBuffer>
+    %1 = spv.AccessChain %arg0[%0, %0] : !spv.ptr<!spv.struct<!spv.array<1 x f32, stride=4> [0]>, StorageBuffer>, i32, i32
     %2 = spv.Load "StorageBuffer" %1 : f32
 
     // CHECK: [[STORE_PTR:%.*]] = spv.AccessChain {{%.*}}[{{%.*}}, {{%.*}}] : !spv.ptr<!spv.struct<!spv.array<1 x f32, stride=4> [0]>
     // CHECK-NEXT: spv.Store "StorageBuffer" [[STORE_PTR]], [[VAL]] : f32
     %3 = spv.constant 0 : i32
-    %4 = spv.AccessChain %arg1[%3, %3] : !spv.ptr<!spv.struct<!spv.array<1 x f32, stride=4> [0]>, StorageBuffer>
+    %4 = spv.AccessChain %arg1[%3, %3] : !spv.ptr<!spv.struct<!spv.array<1 x f32, stride=4> [0]>, StorageBuffer>, i32, i32
     spv.Store "StorageBuffer" %4, %2 : f32
     spv.Return
   }
@@ -46,14 +46,66 @@ spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
     // CHECK: [[LOAD_PTR:%.*]] = spv.AccessChain {{%.*}}[{{%.*}}, {{%.*}}] : !spv.ptr<!spv.struct<!spv.array<1 x i32, stride=4> [0]>
     // CHECK-NEXT: [[VAL:%.*]] = spv.Load "StorageBuffer" [[LOAD_PTR]] : i32
     %0 = spv.constant 0 : i32
-    %1 = spv.AccessChain %arg0[%0, %0] : !spv.ptr<!spv.struct<!spv.array<1 x i32, stride=4> [0]>, StorageBuffer>
+    %1 = spv.AccessChain %arg0[%0, %0] : !spv.ptr<!spv.struct<!spv.array<1 x i32, stride=4> [0]>, StorageBuffer>, i32, i32
     %2 = spv.Load "StorageBuffer" %1 : i32
 
     // CHECK: [[STORE_PTR:%.*]] = spv.AccessChain {{%.*}}[{{%.*}}, {{%.*}}] : !spv.ptr<!spv.struct<!spv.array<1 x i32, stride=4> [0]>
     // CHECK-NEXT: spv.Store "StorageBuffer" [[STORE_PTR]], [[VAL]] : i32
     %3 = spv.constant 0 : i32
-    %4 = spv.AccessChain %arg1[%3, %3] : !spv.ptr<!spv.struct<!spv.array<1 x i32, stride=4> [0]>, StorageBuffer>
+    %4 = spv.AccessChain %arg1[%3, %3] : !spv.ptr<!spv.struct<!spv.array<1 x i32, stride=4> [0]>, StorageBuffer>, i32, i32
     spv.Store "StorageBuffer" %4, %2 : i32
     spv.Return
   }
 }
+
+// -----
+
+spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
+  spv.func @copy_memory_simple() "None" {
+    %0 = spv.Variable : !spv.ptr<f32, Function>
+    %1 = spv.Variable : !spv.ptr<f32, Function>
+    // CHECK: spv.CopyMemory "Function" %{{.*}}, "Function" %{{.*}} : f32
+    spv.CopyMemory "Function" %0, "Function" %1 : f32
+    spv.Return
+  }
+}
+
+// -----
+
+spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
+  spv.func @copy_memory_different_storage_classes(%in : !spv.ptr<!spv.array<4xf32>, Input>, %out : !spv.ptr<!spv.array<4xf32>, Output>) "None" {
+    // CHECK: spv.CopyMemory "Output" %{{.*}}, "Input" %{{.*}} : !spv.array<4 x f32>
+    spv.CopyMemory "Output" %out, "Input" %in : !spv.array<4xf32>
+    spv.Return
+  }
+}
+
+
+// -----
+
+spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
+  spv.func @copy_memory_with_access_operands() "None" {
+    %0 = spv.Variable : !spv.ptr<f32, Function>
+    %1 = spv.Variable : !spv.ptr<f32, Function>
+    // CHECK: spv.CopyMemory "Function" %{{.*}}, "Function" %{{.*}} ["Aligned", 4] : f32
+    spv.CopyMemory "Function" %0, "Function" %1 ["Aligned", 4] : f32
+
+    // CHECK: spv.CopyMemory "Function" %{{.*}}, "Function" %{{.*}} ["Volatile"] : f32
+    spv.CopyMemory "Function" %0, "Function" %1 ["Volatile"] : f32
+
+    // CHECK: spv.CopyMemory "Function" %{{.*}}, "Function" %{{.*}} ["Volatile"], ["Volatile"] : f32
+    spv.CopyMemory "Function" %0, "Function" %1 ["Volatile"], ["Volatile"] : f32
+
+    // CHECK: spv.CopyMemory "Function" %{{.*}}, "Function" %{{.*}} ["Aligned", 4], ["Volatile"] : f32
+    spv.CopyMemory "Function" %0, "Function" %1 ["Aligned", 4], ["Volatile"] : f32
+
+    // CHECK: spv.CopyMemory "Function" %{{.*}}, "Function" %{{.*}} ["Volatile"], ["Aligned", 4] : f32
+    spv.CopyMemory "Function" %0, "Function" %1 ["Volatile"], ["Aligned", 4] : f32
+
+    // CHECK: spv.CopyMemory "Function" %{{.*}}, "Function" %{{.*}} ["Aligned", 8], ["Aligned", 4] : f32
+    spv.CopyMemory "Function" %0, "Function" %1 ["Aligned", 8], ["Aligned", 4] : f32
+
+    spv.Return
+  }
+}
+
diff --git a/mlir/test/Dialect/SPIRV/Serialization/undef.mlir b/mlir/test/Dialect/SPIRV/Serialization/undef.mlir
index 6998930911db8..d19812f482571 100644
--- a/mlir/test/Dialect/SPIRV/Serialization/undef.mlir
+++ b/mlir/test/Dialect/SPIRV/Serialization/undef.mlir
@@ -16,7 +16,7 @@ spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
     // CHECK: {{%.*}} = spv.undef : !spv.ptr<!spv.struct<f32>, StorageBuffer>
     %7 = spv.undef : !spv.ptr<!spv.struct<f32>, StorageBuffer>
     %8 = spv.constant 0 : i32
-    %9 = spv.AccessChain %7[%8] : !spv.ptr<!spv.struct<f32>, StorageBuffer>
+    %9 = spv.AccessChain %7[%8] : !spv.ptr<!spv.struct<f32>, StorageBuffer>, i32
     spv.Return
   }
 }
diff --git a/mlir/test/Dialect/SPIRV/Transforms/abi-load-store.mlir b/mlir/test/Dialect/SPIRV/Transforms/abi-load-store.mlir
index 075ef3398d834..4e4bf06e6f737 100644
--- a/mlir/test/Dialect/SPIRV/Transforms/abi-load-store.mlir
+++ b/mlir/test/Dialect/SPIRV/Transforms/abi-load-store.mlir
@@ -102,14 +102,14 @@ spv.module Logical GLSL450 {
     %37 = spv.IAdd %arg4, %11 : i32
     // CHECK: spv.AccessChain [[ARG0]]
     %c0 = spv.constant 0 : i32
-    %38 = spv.AccessChain %arg0[%c0, %36, %37] : !spv.ptr<!spv.struct<!spv.array<12 x !spv.array<4 x f32>>>, StorageBuffer>
+    %38 = spv.AccessChain %arg0[%c0, %36, %37] : !spv.ptr<!spv.struct<!spv.array<12 x !spv.array<4 x f32>>>, StorageBuffer>, i32, i32, i32
     %39 = spv.Load "StorageBuffer" %38 : f32
     // CHECK: spv.AccessChain [[ARG1]]
-    %40 = spv.AccessChain %arg1[%c0, %36, %37] : !spv.ptr<!spv.struct<!spv.array<12 x !spv.array<4 x f32>>>, StorageBuffer>
+    %40 = spv.AccessChain %arg1[%c0, %36, %37] : !spv.ptr<!spv.struct<!spv.array<12 x !spv.array<4 x f32>>>, StorageBuffer>, i32, i32, i32
     %41 = spv.Load "StorageBuffer" %40 : f32
     %42 = spv.FAdd %39, %41 : f32
     // CHECK: spv.AccessChain [[ARG2]]
-    %43 = spv.AccessChain %arg2[%c0, %36, %37] : !spv.ptr<!spv.struct<!spv.array<12 x !spv.array<4 x f32>>>, StorageBuffer>
+    %43 = spv.AccessChain %arg2[%c0, %36, %37] : !spv.ptr<!spv.struct<!spv.array<12 x !spv.array<4 x f32>>>, StorageBuffer>, i32, i32, i32
     spv.Store "StorageBuffer" %43, %42 : f32
     spv.Return
   }
diff --git a/mlir/test/Dialect/SPIRV/Transforms/inlining.mlir b/mlir/test/Dialect/SPIRV/Transforms/inlining.mlir
index 24b22c5dbf6f7..8c9408ab089f7 100644
--- a/mlir/test/Dialect/SPIRV/Transforms/inlining.mlir
+++ b/mlir/test/Dialect/SPIRV/Transforms/inlining.mlir
@@ -37,7 +37,7 @@ spv.module Logical GLSL450 {
   spv.func @callee() "None" {
     %0 = spv._address_of @data : !spv.ptr<!spv.struct<!spv.rtarray<i32> [0]>, StorageBuffer>
     %1 = spv.constant 0: i32
-    %2 = spv.AccessChain %0[%1, %1] : !spv.ptr<!spv.struct<!spv.rtarray<i32> [0]>, StorageBuffer>
+    %2 = spv.AccessChain %0[%1, %1] : !spv.ptr<!spv.struct<!spv.rtarray<i32> [0]>, StorageBuffer>, i32, i32
     spv.Branch ^next
 
   ^next:
@@ -196,7 +196,7 @@ spv.module Logical GLSL450 {
     // CHECK: [[VAL:%.*]] = spv.Load "StorageBuffer" [[LOADPTR]]
     %2 = spv._address_of @arg_0 : !spv.ptr<!spv.struct<i32 [0]>, StorageBuffer>
     %3 = spv._address_of @arg_1 : !spv.ptr<!spv.struct<i32 [0]>, StorageBuffer>
-    %4 = spv.AccessChain %2[%1] : !spv.ptr<!spv.struct<i32 [0]>, StorageBuffer>
+    %4 = spv.AccessChain %2[%1] : !spv.ptr<!spv.struct<i32 [0]>, StorageBuffer>, i32
     %5 = spv.Load "StorageBuffer" %4 : i32
     %6 = spv.SGreaterThan %5, %1 : i32
     // CHECK: spv.selection
@@ -204,7 +204,7 @@ spv.module Logical GLSL450 {
       spv.BranchConditional %6, ^bb1, ^bb2
     ^bb1: // pred: ^bb0
       // CHECK: [[STOREPTR:%.*]] = spv.AccessChain [[ADDRESS_ARG1]]
-      %7 = spv.AccessChain %3[%1] : !spv.ptr<!spv.struct<i32 [0]>, StorageBuffer>
+      %7 = spv.AccessChain %3[%1] : !spv.ptr<!spv.struct<i32 [0]>, StorageBuffer>, i32
       // CHECK-NOT: spv.FunctionCall
       // CHECK: spv.AtomicIAdd "Device" "AcquireRelease" [[STOREPTR]], [[VAL]]
       // CHECK: spv.Branch
diff --git a/mlir/test/Dialect/SPIRV/Transforms/layout-decoration.mlir b/mlir/test/Dialect/SPIRV/Transforms/layout-decoration.mlir
index 975012d3a26ae..f54d1910be22e 100644
--- a/mlir/test/Dialect/SPIRV/Transforms/layout-decoration.mlir
+++ b/mlir/test/Dialect/SPIRV/Transforms/layout-decoration.mlir
@@ -24,7 +24,7 @@ spv.module Logical GLSL450 {
     // CHECK: {{%.*}} = spv._address_of @var0 : !spv.ptr<!spv.struct<i32 [0], !spv.struct<f32 [0], i32 [4]> [4], f32 [12]>, Uniform>
     %0 = spv._address_of @var0 : !spv.ptr<!spv.struct<i32, !spv.struct<f32, i32>, f32>, Uniform>
     // CHECK:  {{%.*}} = spv.AccessChain {{%.*}}[{{%.*}}] : !spv.ptr<!spv.struct<i32 [0], !spv.struct<f32 [0], i32 [4]> [4], f32 [12]>, Uniform>
-    %1 = spv.AccessChain %0[%c0] : !spv.ptr<!spv.struct<i32, !spv.struct<f32, i32>, f32>, Uniform>
+    %1 = spv.AccessChain %0[%c0] : !spv.ptr<!spv.struct<i32, !spv.struct<f32, i32>, f32>, Uniform>, i32
     spv.Return
   }
 }
diff --git a/mlir/test/Dialect/SPIRV/Transforms/rewrite-inserts.mlir b/mlir/test/Dialect/SPIRV/Transforms/rewrite-inserts.mlir
new file mode 100644
index 0000000000000..1b265e3bcd422
--- /dev/null
+++ b/mlir/test/Dialect/SPIRV/Transforms/rewrite-inserts.mlir
@@ -0,0 +1,31 @@
+// RUN: mlir-opt -spirv-rewrite-inserts -split-input-file -verify-diagnostics %s -o - | FileCheck %s
+
+spv.module Logical GLSL450 {
+  spv.func @rewrite(%value0 : f32, %value1 : f32, %value2 : f32, %value3 : i32, %value4: !spv.array<3xf32>) -> vector<3xf32> "None" {
+    %0 = spv.undef : vector<3xf32>
+    // CHECK: spv.CompositeConstruct {{%.*}}, {{%.*}}, {{%.*}} : vector<3xf32>
+    %1 = spv.CompositeInsert %value0, %0[0 : i32] : f32 into vector<3xf32>
+    %2 = spv.CompositeInsert %value1, %1[1 : i32] : f32 into vector<3xf32>
+    %3 = spv.CompositeInsert %value2, %2[2 : i32] : f32 into vector<3xf32>
+
+    %4 = spv.undef : !spv.array<4xf32>
+    // CHECK: spv.CompositeConstruct {{%.*}}, {{%.*}}, {{%.*}}, {{%.*}} : !spv.array<4 x f32>
+    %5 = spv.CompositeInsert %value0, %4[0 : i32] : f32 into !spv.array<4xf32>
+    %6 = spv.CompositeInsert %value1, %5[1 : i32] : f32 into !spv.array<4xf32>
+    %7 = spv.CompositeInsert %value2, %6[2 : i32] : f32 into !spv.array<4xf32>
+    %8 = spv.CompositeInsert %value0, %7[3 : i32] : f32 into !spv.array<4xf32>
+
+    %9 = spv.undef : !spv.struct<f32, i32, f32>
+    // CHECK: spv.CompositeConstruct {{%.*}}, {{%.*}}, {{%.*}} : !spv.struct<f32, i32, f32>
+    %10 = spv.CompositeInsert %value0, %9[0 : i32] : f32 into !spv.struct<f32, i32, f32>
+    %11 = spv.CompositeInsert %value3, %10[1 : i32] : i32 into !spv.struct<f32, i32, f32>
+    %12 = spv.CompositeInsert %value1, %11[2 : i32] : f32 into !spv.struct<f32, i32, f32>
+
+    %13 = spv.undef : !spv.struct<f32, !spv.array<3xf32>>
+    // CHECK: spv.CompositeConstruct {{%.*}}, {{%.*}} : !spv.struct<f32, !spv.array<3 x f32>>
+    %14 = spv.CompositeInsert %value0, %13[0 : i32] : f32 into !spv.struct<f32, !spv.array<3xf32>>
+    %15 = spv.CompositeInsert %value4, %14[1 : i32] : !spv.array<3xf32> into !spv.struct<f32, !spv.array<3xf32>>
+
+    spv.ReturnValue %3 : vector<3xf32>
+  }
+}
diff --git a/mlir/test/Dialect/SPIRV/canonicalize.mlir b/mlir/test/Dialect/SPIRV/canonicalize.mlir
index 20ed6e96be8db..ad129c2f08253 100644
--- a/mlir/test/Dialect/SPIRV/canonicalize.mlir
+++ b/mlir/test/Dialect/SPIRV/canonicalize.mlir
@@ -11,8 +11,8 @@ func @combine_full_access_chain() -> f32 {
   // CHECK-NEXT: spv.Load "Function" %[[PTR]]
   %c0 = spv.constant 0: i32
   %0 = spv.Variable : !spv.ptr<!spv.struct<!spv.array<4x!spv.array<4xf32>>, !spv.array<4xi32>>, Function>
-  %1 = spv.AccessChain %0[%c0] : !spv.ptr<!spv.struct<!spv.array<4x!spv.array<4xf32>>, !spv.array<4xi32>>, Function>
-  %2 = spv.AccessChain %1[%c0, %c0] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
+  %1 = spv.AccessChain %0[%c0] : !spv.ptr<!spv.struct<!spv.array<4x!spv.array<4xf32>>, !spv.array<4xi32>>, Function>, i32
+  %2 = spv.AccessChain %1[%c0, %c0] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>, i32, i32
   %3 = spv.Load "Function" %2 : f32
   spv.ReturnValue %3 : f32
 }
@@ -28,9 +28,9 @@ func @combine_access_chain_multi_use() -> !spv.array<4xf32> {
   // CHECK-NEXT: spv.Load "Function" %[[PTR_1]]
   %c0 = spv.constant 0: i32
   %0 = spv.Variable : !spv.ptr<!spv.struct<!spv.array<4x!spv.array<4xf32>>, !spv.array<4xi32>>, Function>
-  %1 = spv.AccessChain %0[%c0] : !spv.ptr<!spv.struct<!spv.array<4x!spv.array<4xf32>>, !spv.array<4xi32>>, Function>
-  %2 = spv.AccessChain %1[%c0] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
-  %3 = spv.AccessChain %2[%c0] : !spv.ptr<!spv.array<4xf32>, Function>
+  %1 = spv.AccessChain %0[%c0] : !spv.ptr<!spv.struct<!spv.array<4x!spv.array<4xf32>>, !spv.array<4xi32>>, Function>, i32
+  %2 = spv.AccessChain %1[%c0] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>, i32
+  %3 = spv.AccessChain %2[%c0] : !spv.ptr<!spv.array<4xf32>, Function>, i32
   %4 = spv.Load "Function" %2 : !spv.array<4xf32>
   %5 = spv.Load "Function" %3 : f32
   spv.ReturnValue %4: !spv.array<4xf32>
@@ -49,8 +49,8 @@ func @dont_combine_access_chain_without_common_base() -> !spv.array<4xi32> {
   %c1 = spv.constant 1: i32
   %0 = spv.Variable : !spv.ptr<!spv.struct<!spv.array<4x!spv.array<4xf32>>, !spv.array<4xi32>>, Function>
   %1 = spv.Variable : !spv.ptr<!spv.struct<!spv.array<4x!spv.array<4xf32>>, !spv.array<4xi32>>, Function>
-  %2 = spv.AccessChain %0[%c1] : !spv.ptr<!spv.struct<!spv.array<4x!spv.array<4xf32>>, !spv.array<4xi32>>, Function>
-  %3 = spv.AccessChain %1[%c1] : !spv.ptr<!spv.struct<!spv.array<4x!spv.array<4xf32>>, !spv.array<4xi32>>, Function>
+  %2 = spv.AccessChain %0[%c1] : !spv.ptr<!spv.struct<!spv.array<4x!spv.array<4xf32>>, !spv.array<4xi32>>, Function>, i32
+  %3 = spv.AccessChain %1[%c1] : !spv.ptr<!spv.struct<!spv.array<4x!spv.array<4xf32>>, !spv.array<4xi32>>, Function>, i32
   %4 = spv.Load "Function" %2 : !spv.array<4xi32>
   %5 = spv.Load "Function" %3 : !spv.array<4xi32>
   spv.ReturnValue %4 : !spv.array<4xi32>
@@ -141,7 +141,7 @@ func @extract_from_not_constant() -> i32 {
 // spv.constant
 //===----------------------------------------------------------------------===//
 
-// TODO(antiagainst): test constants in different blocks
+// TODO: test constants in different blocks
 
 func @deduplicate_scalar_constant() -> (i32, i32) {
   // CHECK: %[[CST:.*]] = spv.constant 42 : i32
diff --git a/mlir/test/Dialect/SPIRV/cooperative-matrix.mlir b/mlir/test/Dialect/SPIRV/cooperative-matrix.mlir
index a2dafaddfa21a..523bd6bfb0309 100644
--- a/mlir/test/Dialect/SPIRV/cooperative-matrix.mlir
+++ b/mlir/test/Dialect/SPIRV/cooperative-matrix.mlir
@@ -97,8 +97,8 @@ spv.func @cooperative_matrix_fdiv(%a : !spv.coopmatrix<8x16xf32, Subgroup>, %b :
 // CHECK-LABEL: @cooperative_matrix_access_chain
 spv.func @cooperative_matrix_access_chain(%a : !spv.ptr<!spv.coopmatrix<8x16xf32, Subgroup>, Function>) -> !spv.ptr<f32, Function> "None" {
   %0 = spv.constant 0: i32
-  // CHECK: {{%.*}} = spv.AccessChain {{%.*}}[{{%.*}}] : !spv.ptr<!spv.coopmatrix<8x16xf32, Subgroup>, Function>
-  %1 = spv.AccessChain %a[%0] : !spv.ptr<!spv.coopmatrix<8x16xf32, Subgroup>, Function>
+  // CHECK: {{%.*}} = spv.AccessChain {{%.*}}[{{%.*}}] : !spv.ptr<!spv.coopmatrix<8x16xf32, Subgroup>, Function>, i32
+  %1 = spv.AccessChain %a[%0] : !spv.ptr<!spv.coopmatrix<8x16xf32, Subgroup>, Function>, i32
   spv.ReturnValue %1 : !spv.ptr<f32, Function>
 }
 
diff --git a/mlir/test/Dialect/SPIRV/matrix-ops.mlir b/mlir/test/Dialect/SPIRV/matrix-ops.mlir
index 8079b4ba88f65..4ec78344bc385 100644
--- a/mlir/test/Dialect/SPIRV/matrix-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/matrix-ops.mlir
@@ -2,11 +2,39 @@
 
 spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
   // CHECK-LABEL: @matrix_times_scalar
-  spv.func @matrix_times_scalar_1(%arg0 : !spv.matrix<3 x vector<3xf32>>, %arg1 : f32) -> !spv.matrix<3 x vector<3xf32>> "None" {
+  spv.func @matrix_times_scalar(%arg0 : !spv.matrix<3 x vector<3xf32>>, %arg1 : f32) -> !spv.matrix<3 x vector<3xf32>> "None" {
     // CHECK: {{%.*}} = spv.MatrixTimesScalar {{%.*}}, {{%.*}} : !spv.matrix<3 x vector<3xf32>>, f32 -> !spv.matrix<3 x vector<3xf32>>
     %result = spv.MatrixTimesScalar %arg0, %arg1 : !spv.matrix<3 x vector<3xf32>>, f32 -> !spv.matrix<3 x vector<3xf32>>
     spv.ReturnValue %result : !spv.matrix<3 x vector<3xf32>>
   }
+
+  // CHECK-LABEL: @matrix_transpose_1
+  spv.func @matrix_transpose_1(%arg0 : !spv.matrix<3 x vector<2xf32>>) -> !spv.matrix<2 x vector<3xf32>> "None" {
+    // CHECK: {{%.*}} = spv.Transpose {{%.*}} : !spv.matrix<3 x vector<2xf32>> -> !spv.matrix<2 x vector<3xf32>>
+    %result = spv.Transpose %arg0 : !spv.matrix<3 x vector<2xf32>> -> !spv.matrix<2 x vector<3xf32>>
+    spv.ReturnValue %result : !spv.matrix<2 x vector<3xf32>>
+  }
+
+  // CHECK-LABEL: @matrix_transpose_2
+  spv.func @matrix_transpose_2(%arg0 : !spv.matrix<3 x vector<3xf32>>) -> !spv.matrix<3 x vector<3xf32>> "None" {
+    // CHECK: {{%.*}} = spv.Transpose {{%.*}} : !spv.matrix<3 x vector<3xf32>> -> !spv.matrix<3 x vector<3xf32>>
+    %result = spv.Transpose %arg0 : !spv.matrix<3 x vector<3xf32>> -> !spv.matrix<3 x vector<3xf32>>
+    spv.ReturnValue %result : !spv.matrix<3 x vector<3xf32>>
+  }
+
+  // CHECK-LABEL: @matrix_times_matrix_1
+  spv.func @matrix_times_matrix_1(%arg0: !spv.matrix<3 x vector<3xf32>>, %arg1: !spv.matrix<3 x vector<3xf32>>) -> !spv.matrix<3 x vector<3xf32>> "None"{
+    // CHECK: {{%.*}} = spv.MatrixTimesMatrix {{%.*}}, {{%.*}} : !spv.matrix<3 x vector<3xf32>>, !spv.matrix<3 x vector<3xf32>> -> !spv.matrix<3 x vector<3xf32>>
+    %result = spv.MatrixTimesMatrix %arg0, %arg1 : !spv.matrix<3 x vector<3xf32>>, !spv.matrix<3 x vector<3xf32>> -> !spv.matrix<3 x vector<3xf32>>
+    spv.ReturnValue %result : !spv.matrix<3 x vector<3xf32>>
+  }
+
+  // CHECK-LABEL: @matrix_times_matrix_2
+  spv.func @matrix_times_matrix_2(%arg0: !spv.matrix<3 x vector<2xf32>>, %arg1: !spv.matrix<2 x vector<3xf32>>) -> !spv.matrix<2 x vector<2xf32>> "None"{
+    // CHECK: {{%.*}} = spv.MatrixTimesMatrix {{%.*}}, {{%.*}} : !spv.matrix<3 x vector<2xf32>>, !spv.matrix<2 x vector<3xf32>> -> !spv.matrix<2 x vector<2xf32>>
+    %result = spv.MatrixTimesMatrix %arg0, %arg1 : !spv.matrix<3 x vector<2xf32>>, !spv.matrix<2 x vector<3xf32>> -> !spv.matrix<2 x vector<2xf32>>
+    spv.ReturnValue %result : !spv.matrix<2 x vector<2xf32>>
+  }
 }
 
 // -----
@@ -37,5 +65,62 @@ func @input_output_size_mismatch(%arg0 : !spv.matrix<3 x vector<3xf32>>, %arg1 :
    %result = spv.MatrixTimesScalar %arg0, %arg1 : !spv.matrix<3 x vector<3xf32>>, f32 -> !spv.matrix<4 x vector<3xf32>>
 }
 
+// -----
+
+func @transpose_op_shape_mismatch_1(%arg0 : !spv.matrix<3 x vector<4xf32>>) -> () {
+   // expected-error @+1 {{input matrix rows count must be equal to output matrix columns count}}
+   %result = spv.Transpose %arg0 : !spv.matrix<3 x vector<4xf32>> -> !spv.matrix<3 x vector<3xf32>>
+   spv.Return
+}
+
+// -----
+
+func @transpose_op_shape_mismatch_2(%arg0 : !spv.matrix<3 x vector<4xf32>>) -> () {
+   // expected-error @+1 {{input matrix rows count must be equal to output matrix columns count}}
+   %result = spv.Transpose %arg0 : !spv.matrix<3 x vector<4xf32>> -> !spv.matrix<2 x vector<4xf32>>
+   spv.Return
+}
+
+// -----
+
+func @transpose_op_type_mismatch(%arg0 : !spv.matrix<3 x vector<4xf32>>) -> () {
+   // expected-error @+1 {{input and output matrices must have the same component type}}
+   %result = spv.Transpose %arg0 : !spv.matrix<3 x vector<4xf32>> -> !spv.matrix<4 x vector<3xf16>>
+   spv.Return
+}
+
+// -----
+
+func @matrix_times_matrix_invalid_input_shape_1(%arg0 : !spv.matrix<3 x vector<2xf32>>, %arg1 : !spv.matrix<2 x vector<3xf32>>){
+   // expected-error @+1 {{right and result matrices must have equal columns' count}}
+   %result = spv.MatrixTimesMatrix %arg0, %arg1 : !spv.matrix<3 x vector<2xf32>>, !spv.matrix<2 x vector<3xf32>> -> !spv.matrix<3 x vector<2xf32>>
+}
+
+// -----
+
+func @matrix_times_matrix_invalid_input_shape_2(%arg0 : !spv.matrix<3 x vector<2xf32>>, %arg1 : !spv.matrix<2 x vector<3xf32>>){
+   // expected-error @+1 {{left and result matrices must have equal rows' count}}
+   %result = spv.MatrixTimesMatrix %arg0, %arg1 : !spv.matrix<3 x vector<2xf32>>, !spv.matrix<2 x vector<3xf32>> -> !spv.matrix<2 x vector<3xf32>>
+}
+
+// -----
+
+func @matrix_times_matrix_inputs_shape_mismatch(%arg0 : !spv.matrix<3 x vector<2xf32>>, %arg1 : !spv.matrix<2 x vector<2xf32>>){
+   // expected-error @+1 {{left matrix columns' count must be equal to the right matrix rows' count}}
+   %result = spv.MatrixTimesMatrix %arg0, %arg1 : !spv.matrix<3 x vector<2xf32>>, !spv.matrix<2 x vector<2xf32>> -> !spv.matrix<2 x vector<2xf32>>
+}
+
+// -----
+
+func @matrix_times_matrix_component_type_mismatch_1(%arg0 : !spv.matrix<3 x vector<3xf32>>, %arg1 : !spv.matrix<3x vector<3xf32>>){
+   // expected-error @+1 {{right and result matrices' component type must be the same}}
+   %result = spv.MatrixTimesMatrix %arg0, %arg1 : !spv.matrix<3 x vector<3xf32>>, !spv.matrix<3 x vector<3xf32>> -> !spv.matrix<3 x vector<3xf64>>
+}
 
 
+// -----
+
+func @matrix_times_matrix_component_type_mismatch_2(%arg0 : !spv.matrix<3 x vector<3xf64>>, %arg1 : !spv.matrix<3x vector<3xf32>>){
+   // expected-error @+1 {{left and result matrices' component type must be the same}}
+   %result = spv.MatrixTimesMatrix %arg0, %arg1 : !spv.matrix<3 x vector<3xf64>>, !spv.matrix<3 x vector<3xf32>> -> !spv.matrix<3 x vector<3xf32>>
+}
diff --git a/mlir/test/Dialect/SPIRV/ops.mlir b/mlir/test/Dialect/SPIRV/ops.mlir
index 14e1fa10735eb..c91a81fe239c4 100644
--- a/mlir/test/Dialect/SPIRV/ops.mlir
+++ b/mlir/test/Dialect/SPIRV/ops.mlir
@@ -8,21 +8,21 @@ func @access_chain_struct() -> () {
   %0 = spv.constant 1: i32
   %1 = spv.Variable : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Function>
   // CHECK: spv.AccessChain {{.*}}[{{.*}}, {{.*}}] : !spv.ptr<!spv.struct<f32, !spv.array<4 x f32>>, Function>
-  %2 = spv.AccessChain %1[%0, %0] : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Function>
+  %2 = spv.AccessChain %1[%0, %0] : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Function>, i32, i32
   return
 }
 
 func @access_chain_1D_array(%arg0 : i32) -> () {
   %0 = spv.Variable : !spv.ptr<!spv.array<4xf32>, Function>
   // CHECK: spv.AccessChain {{.*}}[{{.*}}] : !spv.ptr<!spv.array<4 x f32>, Function>
-  %1 = spv.AccessChain %0[%arg0] : !spv.ptr<!spv.array<4xf32>, Function>
+  %1 = spv.AccessChain %0[%arg0] : !spv.ptr<!spv.array<4xf32>, Function>, i32
   return
 }
 
 func @access_chain_2D_array_1(%arg0 : i32) -> () {
   %0 = spv.Variable : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
   // CHECK: spv.AccessChain {{.*}}[{{.*}}, {{.*}}] : !spv.ptr<!spv.array<4 x !spv.array<4 x f32>>, Function>
-  %1 = spv.AccessChain %0[%arg0, %arg0] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
+  %1 = spv.AccessChain %0[%arg0, %arg0] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>, i32, i32
   %2 = spv.Load "Function" %1 ["Volatile"] : f32
   return
 }
@@ -30,7 +30,7 @@ func @access_chain_2D_array_1(%arg0 : i32) -> () {
 func @access_chain_2D_array_2(%arg0 : i32) -> () {
   %0 = spv.Variable : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
   // CHECK: spv.AccessChain {{.*}}[{{.*}}] : !spv.ptr<!spv.array<4 x !spv.array<4 x f32>>, Function>
-  %1 = spv.AccessChain %0[%arg0] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
+  %1 = spv.AccessChain %0[%arg0] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>, i32
   %2 = spv.Load "Function" %1 ["Volatile"] : !spv.array<4xf32>
   return
 }
@@ -38,7 +38,7 @@ func @access_chain_2D_array_2(%arg0 : i32) -> () {
 func @access_chain_rtarray(%arg0 : i32) -> () {
   %0 = spv.Variable : !spv.ptr<!spv.rtarray<f32>, Function>
   // CHECK: spv.AccessChain {{.*}}[{{.*}}] : !spv.ptr<!spv.rtarray<f32>, Function>
-  %1 = spv.AccessChain %0[%arg0] : !spv.ptr<!spv.rtarray<f32>, Function>
+  %1 = spv.AccessChain %0[%arg0] : !spv.ptr<!spv.rtarray<f32>, Function>, i32
   %2 = spv.Load "Function" %1 ["Volatile"] : f32
   return
 }
@@ -49,7 +49,7 @@ func @access_chain_non_composite() -> () {
   %0 = spv.constant 1: i32
   %1 = spv.Variable : !spv.ptr<f32, Function>
   // expected-error @+1 {{cannot extract from non-composite type 'f32' with index 0}}
-  %2 = spv.AccessChain %1[%0] : !spv.ptr<f32, Function>
+  %2 = spv.AccessChain %1[%0] : !spv.ptr<f32, Function>, i32
   return
 }
 
@@ -58,7 +58,34 @@ func @access_chain_non_composite() -> () {
 func @access_chain_no_indices(%index0 : i32) -> () {
   %0 = spv.Variable : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
   // expected-error @+1 {{expected at least one index}}
-  %1 = spv.AccessChain %0[] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
+  %1 = spv.AccessChain %0[] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>, i32
+  return
+}
+
+// -----
+
+func @access_chain_missing_comma(%index0 : i32) -> () {
+  %0 = spv.Variable : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
+  // expected-error @+1 {{expected ','}}
+  %1 = spv.AccessChain %0[%index0] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function> i32
+  return
+}
+
+// -----
+
+func @access_chain_invalid_indices_types_count(%index0 : i32) -> () {
+  %0 = spv.Variable : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
+  // expected-error @+1 {{'spv.AccessChain' op indices types' count must be equal to indices info count}}
+  %1 = spv.AccessChain %0[%index0] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>, i32, i32
+  return
+}
+
+// -----
+
+func @access_chain_missing_indices_type(%index0 : i32) -> () {
+  %0 = spv.Variable : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
+  // expected-error @+1 {{'spv.AccessChain' op indices types' count must be equal to indices info count}}
+  %1 = spv.AccessChain %0[%index0, %index0] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>, i32
   return
 }
 
@@ -68,7 +95,7 @@ func @access_chain_invalid_type(%index0 : i32) -> () {
   %0 = spv.Variable : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
   %1 = spv.Load "Function" %0 ["Volatile"] : !spv.array<4x!spv.array<4xf32>>
   // expected-error @+1 {{expected a pointer to composite type, but provided '!spv.array<4 x !spv.array<4 x f32>>'}}
-  %2 = spv.AccessChain %1[%index0] : !spv.array<4x!spv.array<4xf32>>
+  %2 = spv.AccessChain %1[%index0] : !spv.array<4x!spv.array<4xf32>>, i32
   return
 }
 
@@ -77,7 +104,7 @@ func @access_chain_invalid_type(%index0 : i32) -> () {
 func @access_chain_invalid_index_1(%index0 : i32) -> () {
    %0 = spv.Variable : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
   // expected-error @+1 {{expected SSA operand}}
-  %1 = spv.AccessChain %0[%index, 4] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
+  %1 = spv.AccessChain %0[%index, 4] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>, i32, i32
   return
 }
 
@@ -86,7 +113,7 @@ func @access_chain_invalid_index_1(%index0 : i32) -> () {
 func @access_chain_invalid_index_2(%index0 : i32) -> () {
   %0 = spv.Variable : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Function>
   // expected-error @+1 {{index must be an integer spv.constant to access element of spv.struct}}
-  %1 = spv.AccessChain %0[%index0, %index0] : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Function>
+  %1 = spv.AccessChain %0[%index0, %index0] : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Function>, i32, i32
   return
 }
 
@@ -96,7 +123,7 @@ func @access_chain_invalid_constant_type_1() -> () {
   %0 = std.constant 1: i32
   %1 = spv.Variable : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Function>
   // expected-error @+1 {{index must be an integer spv.constant to access element of spv.struct, but provided std.constant}}
-  %2 = spv.AccessChain %1[%0, %0] : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Function>
+  %2 = spv.AccessChain %1[%0, %0] : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Function>, i32, i32
   return
 }
 
@@ -106,7 +133,7 @@ func @access_chain_out_of_bounds() -> () {
   %index0 = "spv.constant"() { value = 12: i32} : () -> i32
   %0 = spv.Variable : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Function>
   // expected-error @+1 {{'spv.AccessChain' op index 12 out of bounds for '!spv.struct<f32, !spv.array<4 x f32>>'}}
-  %1 = spv.AccessChain %0[%index0, %index0] : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Function>
+  %1 = spv.AccessChain %0[%index0, %index0] : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Function>, i32, i32
   return
 }
 
@@ -115,7 +142,7 @@ func @access_chain_out_of_bounds() -> () {
 func @access_chain_invalid_accessing_type(%index0 : i32) -> () {
   %0 = spv.Variable : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
   // expected-error @+1 {{cannot extract from non-composite type 'f32' with index 0}}
-  %1 = spv.AccessChain %0[%index, %index0, %index0] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
+  %1 = spv.AccessChain %0[%index, %index0, %index0] : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>, i32, i32, i32
   return
 
 // -----
@@ -1217,3 +1244,64 @@ func @cannot_be_generic_storage_class(%arg0: f32) -> () {
   %0 = spv.Variable : !spv.ptr<f32, Generic>
   return
 }
+
+// -----
+
+func @copy_memory_incompatible_ptrs() {
+  %0 = spv.Variable : !spv.ptr<f32, Function>
+  %1 = spv.Variable : !spv.ptr<i32, Function>
+  // expected-error @+1 {{both operands must be pointers to the same type}}
+  "spv.CopyMemory"(%0, %1) {} : (!spv.ptr<f32, Function>, !spv.ptr<i32, Function>) -> ()
+  spv.Return
+}
+
+// -----
+
+func @copy_memory_invalid_maa() {
+  %0 = spv.Variable : !spv.ptr<f32, Function>
+  %1 = spv.Variable : !spv.ptr<f32, Function>
+  // expected-error @+1 {{missing alignment value}}
+  "spv.CopyMemory"(%0, %1) {memory_access=0x0002 : i32} : (!spv.ptr<f32, Function>, !spv.ptr<f32, Function>) -> ()
+  spv.Return
+}
+
+// -----
+
+func @copy_memory_invalid_source_maa() {
+  %0 = spv.Variable : !spv.ptr<f32, Function>
+  %1 = spv.Variable : !spv.ptr<f32, Function>
+  // expected-error @+1 {{invalid alignment specification with non-aligned memory access specification}}
+  "spv.CopyMemory"(%0, %1) {source_memory_access=0x0001 : i32, memory_access=0x0002 : i32, source_alignment=8 : i32, alignment=4 : i32} : (!spv.ptr<f32, Function>, !spv.ptr<f32, Function>) -> ()
+  spv.Return
+}
+
+// -----
+
+func @copy_memory_invalid_source_maa2() {
+  %0 = spv.Variable : !spv.ptr<f32, Function>
+  %1 = spv.Variable : !spv.ptr<f32, Function>
+  // expected-error @+1 {{missing alignment value}}
+  "spv.CopyMemory"(%0, %1) {source_memory_access=0x0002 : i32, memory_access=0x0002 : i32, alignment=4 : i32} : (!spv.ptr<f32, Function>, !spv.ptr<f32, Function>) -> ()
+  spv.Return
+}
+
+// -----
+
+func @copy_memory_print_maa() {
+  %0 = spv.Variable : !spv.ptr<f32, Function>
+  %1 = spv.Variable : !spv.ptr<f32, Function>
+
+  // CHECK: spv.CopyMemory "Function" %{{.*}}, "Function" %{{.*}} ["Volatile"] : f32
+  "spv.CopyMemory"(%0, %1) {memory_access=0x0001 : i32} : (!spv.ptr<f32, Function>, !spv.ptr<f32, Function>) -> ()
+
+  // CHECK: spv.CopyMemory "Function" %{{.*}}, "Function" %{{.*}} ["Aligned", 4] : f32
+  "spv.CopyMemory"(%0, %1) {memory_access=0x0002 : i32, alignment=4 : i32} : (!spv.ptr<f32, Function>, !spv.ptr<f32, Function>) -> ()
+
+  // CHECK: spv.CopyMemory "Function" %{{.*}}, "Function" %{{.*}} ["Aligned", 4], ["Volatile"] : f32
+  "spv.CopyMemory"(%0, %1) {source_memory_access=0x0001 : i32, memory_access=0x0002 : i32, alignment=4 : i32} : (!spv.ptr<f32, Function>, !spv.ptr<f32, Function>) -> ()
+
+  // CHECK: spv.CopyMemory "Function" %{{.*}}, "Function" %{{.*}} ["Aligned", 4], ["Aligned", 8] : f32
+  "spv.CopyMemory"(%0, %1) {source_memory_access=0x0002 : i32, memory_access=0x0002 : i32, source_alignment=8 : i32, alignment=4 : i32} : (!spv.ptr<f32, Function>, !spv.ptr<f32, Function>) -> ()
+
+  spv.Return
+}
diff --git a/mlir/test/Dialect/SPIRV/structure-ops.mlir b/mlir/test/Dialect/SPIRV/structure-ops.mlir
index e24ebcd6e6560..2d62f64b2479c 100644
--- a/mlir/test/Dialect/SPIRV/structure-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/structure-ops.mlir
@@ -11,7 +11,7 @@ spv.module Logical GLSL450 {
     // CHECK: [[VAR1:%.*]] = spv._address_of @var1 : !spv.ptr<!spv.struct<f32, !spv.array<4 x f32>>, Input>
     // CHECK-NEXT: spv.AccessChain [[VAR1]][{{.*}}, {{.*}}] : !spv.ptr<!spv.struct<f32, !spv.array<4 x f32>>, Input>
     %1 = spv._address_of @var1 : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Input>
-    %2 = spv.AccessChain %1[%0, %0] : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Input>
+    %2 = spv.AccessChain %1[%0, %0] : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Input>, i32, i32
     spv.Return
   }
 }
@@ -175,7 +175,7 @@ spv.module Logical GLSL450 {
    spv.EntryPoint "GLCompute" @do_something
 }
 
-/// TODO(ravishankarm) : Add a test that verifies an error is thrown
+/// TODO: Add a test that verifies an error is thrown
 /// when interface entries of EntryPointOp are not
 /// spv.Variables. There is currently no other op that has a spv.ptr
 /// return type
diff --git a/mlir/test/Dialect/SPIRV/types.mlir b/mlir/test/Dialect/SPIRV/types.mlir
index d5eb073c9aa55..810e00b5dedd4 100644
--- a/mlir/test/Dialect/SPIRV/types.mlir
+++ b/mlir/test/Dialect/SPIRV/types.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt -split-input-file -verify-diagnostics %s | FileCheck %s
 
-// TODO(b/133530217): Add more tests after switching to the generic parser.
+// TODO: Add more tests after switching to the generic parser.
 
 //===----------------------------------------------------------------------===//
 // ArrayType
diff --git a/mlir/test/Dialect/Shape/canonicalize.mlir b/mlir/test/Dialect/Shape/canonicalize.mlir
index a56a8f9861de5..1665ef73f3e31 100644
--- a/mlir/test/Dialect/Shape/canonicalize.mlir
+++ b/mlir/test/Dialect/Shape/canonicalize.mlir
@@ -403,8 +403,8 @@ func @f() {
 
 // -----
 // Broadcastable with non-broadcastable constant shapes is always false
-// CHECK-LABEL: func @f
-func @f() {
+// CHECK-LABEL: func @static_non_broadcastable
+func @static_non_broadcastable() {
   // CHECK-NEXT: shape.const_shape
   // CHECK-NEXT: shape.const_shape
   // CHECK-NEXT: shape.cstr_broadcastable
@@ -442,3 +442,122 @@ func @f(%arg0 : !shape.shape) {
   "consume.witness"(%0) : (!shape.witness) -> ()
   return
 }
+
+// -----
+
+// Fold `rank` based on constant shape.
+// CHECK-LABEL: @fold_rank
+func @fold_rank() -> !shape.size {
+  // CHECK-DAG: %[[RESULT:.*]] = shape.const_size 5
+  // CHECK-DAG: return %[[RESULT]] : !shape.size
+  %shape = shape.const_shape [3, 4, 5, 6, 7]
+  %rank = shape.rank %shape
+  return %rank : !shape.size
+}
+
+// -----
+
+// Do not fold `rank` if shape is dynamic.
+// CHECK-LABEL: @dont_fold_rank
+// CHECK-SAME: (%[[SHAPE:.*]]: !shape.shape) -> !shape.size
+func @dont_fold_rank(%shape : !shape.shape) -> !shape.size {
+  // CHECK-DAG: %[[RESULT:.*]] = shape.rank %[[SHAPE]]
+  // CHECK-DAG: return %[[RESULT]] : !shape.size
+  %rank = shape.rank %shape
+  return %rank : !shape.size
+}
+
+// -----
+
+// Canonicalize `rank` when shape is derived from ranked tensor.
+// CHECK-LABEL: @canonicalize_rank
+func @canonicalize_rank(%arg : tensor<1x2x?xf32>) -> !shape.size {
+// CHECK-DAG: %[[RESULT:.*]] = shape.const_size 3
+// CHECK-DAG: return %[[RESULT]] : !shape.size
+%shape = shape.shape_of %arg : tensor<1x2x?xf32>
+%rank = shape.rank %shape
+return %rank : !shape.size
+}
+
+// -----
+
+// Do not canonicalize `rank` when shape is derived from unranked tensor.
+// CHECK-LABEL: @dont_canonicalize_rank
+// CHECK-SAME: (%[[ARG:.*]]: tensor<*xf32>) -> !shape.size
+func @dont_canonicalize_rank(%arg : tensor<*xf32>) -> !shape.size {
+// CHECK-DAG: %[[SHAPE:.*]] = shape.shape_of %[[ARG]] : tensor<*xf32>
+// CHECK-DAG: %[[SIZE:.*]] = shape.rank %[[SHAPE]]
+// CHECK-DAG: return %[[SIZE]] : !shape.size
+%shape = shape.shape_of %arg : tensor<*xf32>
+%rank = shape.rank %shape
+return %rank : !shape.size
+}
+
+// Canonicalize redundant conversion from `index` to `size` and back.
+// CHECK-LABEL: @index_to_size_to_index
+// CHECK-SAME: (%[[IDX:.*]]: index) -> index
+func @index_to_size_to_index(%index : index) -> index {
+  // CHECK: return %[[IDX]] : index
+  %size = shape.index_to_size %index
+  %result = shape.size_to_index %size
+  return %result : index
+}
+
+// -----
+
+// Canonicalize redundant conversion from `size` to `index` and back.
+// CHECK-LABEL: @size_to_index_to_size
+// CHECK-SAME: (%[[SIZE:.*]]: !shape.size) -> !shape.size
+func @size_to_index_to_size(%size : !shape.size) -> !shape.size {
+  // CHECK: return %[[SIZE]] : !shape.size
+  %idx = shape.size_to_index %size
+  %result = shape.index_to_size %idx
+  return %result : !shape.size
+}
+
+// -----
+
+// Canonicalize scalar cstr_broadcastable checks
+// CHECK-LABEL: @cstr_broadcastable_scalar
+func @cstr_broadcastable_scalar(%arg0 : tensor<?xf32>) {
+  // CHECK-NEXT: shape.const_witness true
+  // CHECK-NEXT: consume.witness
+  // CHECK-NEXT: return
+  %0 = shape.const_shape []
+  %1 = shape.shape_of %arg0 : tensor<?xf32>
+  %2 = shape.cstr_broadcastable %0, %1
+  "consume.witness"(%2) : (!shape.witness) -> ()
+  return
+}
+
+// -----
+
+// Do not canonicalize cstr_broadcastable checks with 2 unknowns
+// CHECK-LABEL: @cstr_broadcastable_unknown
+func @cstr_broadcastable_unknown(%arg0 : tensor<?xf32>, %arg1 : tensor<?xf32>) {
+  // CHECK-NEXT: shape.shape_of %arg0
+  // CHECK-NEXT: shape.shape_of %arg1
+  // CHECK-NEXT: shape.cstr_broadcastable
+  // CHECK-NEXT: consume.witness
+  // CHECK-NEXT: return
+  %0 = shape.shape_of %arg0 : tensor<?xf32>
+  %1 = shape.shape_of %arg1 : tensor<?xf32>
+  %2 = shape.cstr_broadcastable %0, %1
+  "consume.witness"(%2) : (!shape.witness) -> ()
+  return
+}
+
+// -----
+
+// Scalars are safe to broadcast to unranked sizes.
+// CHECK-LABEL: @cstr_broadcastable_scalar_unranked
+func @cstr_broadcastable_scalar_unranked(%arg0 : tensor<*xf32>, %arg1 : tensor<index>) {
+  // CHECK-NEXT: shape.const_witness true
+  // CHECK-NEXT: consume.witness
+  // CHECK-NEXT: return
+  %0 = shape.shape_of %arg1 : tensor<index>
+  %1 = shape.shape_of %arg0 : tensor<*xf32>
+  %2 = shape.cstr_broadcastable %0, %1
+  "consume.witness"(%2) : (!shape.witness) -> ()
+  return
+}
diff --git a/mlir/test/Dialect/Shape/ops.mlir b/mlir/test/Dialect/Shape/ops.mlir
index 2e40211e5a638..18953efa05f1b 100644
--- a/mlir/test/Dialect/Shape/ops.mlir
+++ b/mlir/test/Dialect/Shape/ops.mlir
@@ -111,3 +111,8 @@ func @test_from_extent_tensor(%arg: tensor<?xindex>) -> !shape.shape {
   %0 = shape.from_extent_tensor %arg : tensor<?xindex>
   return %0 : !shape.shape
 }
+
+func @rank(%shape : !shape.shape) -> !shape.size {
+  %rank = shape.rank %shape
+  return %rank : !shape.size
+}
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index 5e4ba39895ed4..bd6ef4150a0de 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -168,10 +168,207 @@ func @cast_transfers(%A: memref<4x8xf32>) -> (vector<4x8xf32>) {
   %f0 = constant 0.0 : f32
   %0 = memref_cast %A : memref<4x8xf32> to memref<?x?xf32>
 
-  // CHECK: vector.transfer_read %{{.*}} : memref<4x8xf32>, vector<4x8xf32>
+  // CHECK: vector.transfer_read %{{.*}} {masked = [false, false]} : memref<4x8xf32>, vector<4x8xf32>
   %1 = vector.transfer_read %0[%c0, %c0], %f0 : memref<?x?xf32>, vector<4x8xf32>
 
-  // CHECK: vector.transfer_write %{{.*}} : vector<4x8xf32>, memref<4x8xf32>
+  // CHECK: vector.transfer_write %{{.*}} {masked = [false, false]} : vector<4x8xf32>, memref<4x8xf32>
   vector.transfer_write %1, %0[%c0, %c0] : vector<4x8xf32>, memref<?x?xf32>
   return %1 : vector<4x8xf32>
 }
+
+// -----
+
+// CHECK-LABEL: func @insert_extract_transpose_2d(
+//  CHECK-SAME: %[[V:[a-zA-Z0-9]*]]: vector<2x3xf32>,
+//  CHECK-SAME: %[[F0:[a-zA-Z0-9]*]]: f32,
+//  CHECK-SAME: %[[F1:[a-zA-Z0-9]*]]: f32,
+//  CHECK-SAME: %[[F2:[a-zA-Z0-9]*]]: f32,
+//  CHECK-SAME: %[[F3:[a-zA-Z0-9]*]]: f32
+func @insert_extract_transpose_2d(
+    %v: vector<2x3xf32>, %f0: f32, %f1: f32, %f2: f32, %f3: f32)
+-> (f32, f32, f32)
+{
+  %0 = vector.insert %f0, %v[0, 0] : f32 into vector<2x3xf32>
+  %1 = vector.insert %f1, %0[0, 1] : f32 into vector<2x3xf32>
+  %2 = vector.insert %f2, %1[1, 0] : f32 into vector<2x3xf32>
+  %3 = vector.insert %f3, %2[1, 1] : f32 into vector<2x3xf32>
+  %4 = vector.transpose %3, [1, 0] : vector<2x3xf32> to vector<3x2xf32>
+  %5 = vector.insert %f3, %4[1, 0] : f32 into vector<3x2xf32>
+  %6 = vector.transpose %5, [1, 0] : vector<3x2xf32> to vector<2x3xf32>
+
+  // Expected %f2 from %2 = vector.insert %f2, %1[1, 0].
+  %r1 = vector.extract %3[1, 0] : vector<2x3xf32>
+
+  // Expected %f1 from %1 = vector.insert %f1, %0[0, 1] followed by
+  // transpose [1, 0].
+  %r2 = vector.extract %4[1, 0] : vector<3x2xf32>
+
+  // Expected %f2 from %2 = vector.insert %f2, %1[1, 0] followed by double
+  // transpose [1, 0].
+  %r3 = vector.extract %6[1, 0] : vector<2x3xf32>
+
+  // CHECK-NEXT: return %[[F2]], %[[F1]], %[[F2]] : f32, f32, f32
+  return %r1, %r2, %r3 : f32, f32, f32
+}
+
+// -----
+
+// CHECK-LABEL: func @insert_extract_transpose_3d(
+//  CHECK-SAME: %[[V:[a-zA-Z0-9]*]]: vector<2x3x4xf32>,
+//  CHECK-SAME: %[[F0:[a-zA-Z0-9]*]]: f32,
+//  CHECK-SAME: %[[F1:[a-zA-Z0-9]*]]: f32,
+//  CHECK-SAME: %[[F2:[a-zA-Z0-9]*]]: f32,
+//  CHECK-SAME: %[[F3:[a-zA-Z0-9]*]]: f32
+func @insert_extract_transpose_3d(
+    %v: vector<2x3x4xf32>, %f0: f32, %f1: f32, %f2: f32, %f3: f32)
+-> (f32, f32, f32, f32)
+{
+  %0 = vector.insert %f0, %v[0, 0, 0] : f32 into vector<2x3x4xf32>
+  %1 = vector.insert %f1, %0[0, 1, 0] : f32 into vector<2x3x4xf32>
+  %2 = vector.insert %f2, %1[1, 0, 0] : f32 into vector<2x3x4xf32>
+  %3 = vector.insert %f3, %2[0, 0, 1] : f32 into vector<2x3x4xf32>
+  %4 = vector.transpose %3, [1, 2, 0] : vector<2x3x4xf32> to vector<3x4x2xf32>
+  %5 = vector.insert %f3, %4[1, 0, 0] : f32 into vector<3x4x2xf32>
+  %6 = vector.transpose %5, [1, 2, 0] : vector<3x4x2xf32> to vector<4x2x3xf32>
+  %7 = vector.insert %f3, %6[1, 0, 0] : f32 into vector<4x2x3xf32>
+  %8 = vector.transpose %7, [1, 2, 0] : vector<4x2x3xf32> to vector<2x3x4xf32>
+
+  // Expected %f2 from %2 = vector.insert %f2, %1[1, 0, 0].
+  %r1 = vector.extract %3[1, 0, 0] : vector<2x3x4xf32>
+
+  // Expected %f1 from %1 = vector.insert %f1, %0[0, 1, 0] followed by
+  // transpose[1, 2, 0].
+  %r2 = vector.extract %4[1, 0, 0] : vector<3x4x2xf32>
+
+  // Expected %f3 from %3 = vector.insert %f3, %0[0, 0, 1] followed by double
+  // transpose[1, 2, 0].
+  %r3 = vector.extract %6[1, 0, 0] : vector<4x2x3xf32>
+
+  // Expected %f2 from %2 = vector.insert %f2, %1[1, 0, 0] followed by triple
+  // transpose[1, 2, 0].
+  %r4 = vector.extract %8[1, 0, 0] : vector<2x3x4xf32>
+
+  // CHECK-NEXT: return %[[F2]], %[[F1]], %[[F3]], %[[F2]] : f32, f32, f32
+  return %r1, %r2, %r3, %r4 : f32, f32, f32, f32
+}
+
+// -----
+
+// CHECK-LABEL: func @insert_extract_transpose_3d_2d(
+//  CHECK-SAME: %[[V:[a-zA-Z0-9]*]]: vector<2x3x4xf32>,
+//  CHECK-SAME: %[[F0:[a-zA-Z0-9]*]]: vector<4xf32>,
+//  CHECK-SAME: %[[F1:[a-zA-Z0-9]*]]: vector<4xf32>,
+//  CHECK-SAME: %[[F2:[a-zA-Z0-9]*]]: vector<4xf32>,
+//  CHECK-SAME: %[[F3:[a-zA-Z0-9]*]]: vector<4xf32>
+func @insert_extract_transpose_3d_2d(
+    %v: vector<2x3x4xf32>,
+    %f0: vector<4xf32>, %f1: vector<4xf32>, %f2: vector<4xf32>, %f3: vector<4xf32>)
+-> (vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32>)
+{
+  %0 = vector.insert %f0, %v[0, 0] : vector<4xf32> into vector<2x3x4xf32>
+  %1 = vector.insert %f1, %0[0, 1] : vector<4xf32> into vector<2x3x4xf32>
+  %2 = vector.insert %f2, %1[1, 0] : vector<4xf32> into vector<2x3x4xf32>
+  %3 = vector.insert %f3, %2[1, 1] : vector<4xf32> into vector<2x3x4xf32>
+  %4 = vector.transpose %3, [1, 0, 2] : vector<2x3x4xf32> to vector<3x2x4xf32>
+  %5 = vector.transpose %4, [1, 0, 2] : vector<3x2x4xf32> to vector<2x3x4xf32>
+
+  // Expected %f2 from %2 = vector.insert %f2, %1[1, 0].
+  %r1 = vector.extract %3[1, 0] : vector<2x3x4xf32>
+
+  // Expected %f1 from %1 = vector.insert %f1, %0[0, 1] followed by
+  // transpose[1, 0, 2].
+  %r2 = vector.extract %4[1, 0] : vector<3x2x4xf32>
+
+  // Expected %f2 from %2 = vector.insert %f2, %1[1, 0, 0] followed by double
+  // transpose[1, 0, 2].
+  %r3 = vector.extract %5[1, 0] : vector<2x3x4xf32>
+
+  %6 = vector.transpose %3, [1, 2, 0] : vector<2x3x4xf32> to vector<3x4x2xf32>
+  %7 = vector.transpose %6, [1, 2, 0] : vector<3x4x2xf32> to vector<4x2x3xf32>
+  %8 = vector.transpose %7, [1, 2, 0] : vector<4x2x3xf32> to vector<2x3x4xf32>
+
+  // Expected %f2 from %2 = vector.insert %f2, %1[1, 0, 0] followed by triple
+  // transpose[1, 2, 0].
+  %r4 = vector.extract %8[1, 0] : vector<2x3x4xf32>
+
+  //      CHECK: return %[[F2]], %[[F1]], %[[F2]], %[[F2]]
+  // CHECK-SAME: vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32>
+  return %r1, %r2, %r3, %r4 : vector<4xf32>, vector<4xf32>, vector<4xf32>, vector<4xf32>
+}
+
+// -----
+
+// CHECK-LABEL: fold_extracts
+//  CHECK-SAME:   %[[A:[a-zA-Z0-9]*]]: vector<3x4x5x6xf32>
+func @fold_extracts(%a : vector<3x4x5x6xf32>) -> (f32, vector<4x5x6xf32>) {
+  %b = vector.extract %a[0] : vector<3x4x5x6xf32>
+  %c = vector.extract %b[1, 2] : vector<4x5x6xf32>
+  //  CHECK-NEXT: vector.extract %[[A]][0, 1, 2, 3] : vector<3x4x5x6xf32>
+  %d = vector.extract %c[3] : vector<6xf32>
+
+  //  CHECK-NEXT: vector.extract %[[A]][0] : vector<3x4x5x6xf32>
+  %e = vector.extract %a[0] : vector<3x4x5x6xf32>
+
+  //  CHECK-NEXT: return
+  return %d, %e : f32, vector<4x5x6xf32>
+}
+
+// -----
+
+// CHECK-LABEL: fold_extract_transpose
+//  CHECK-SAME:   %[[A:[a-zA-Z0-9]*]]: vector<3x4x5x6xf32>
+//  CHECK-SAME:   %[[B:[a-zA-Z0-9]*]]: vector<3x6x5x6xf32>
+func @fold_extract_transpose(
+    %a : vector<3x4x5x6xf32>, %b : vector<3x6x5x6xf32>) -> (
+      vector<6xf32>, vector<6xf32>, vector<6xf32>) {
+  // [3] is a proper most minor identity map in transpose.
+  // Permutation is a self inverse and we have.
+  // [0, 2, 1] ^ -1 o [0, 1, 2] = [0, 2, 1] o [0, 1, 2]
+  //                            = [0, 2, 1]
+  //  CHECK-NEXT: vector.extract %[[A]][0, 2, 1] : vector<3x4x5x6xf32>
+  %0 = vector.transpose %a, [0, 2, 1, 3] : vector<3x4x5x6xf32> to vector<3x5x4x6xf32>
+  %1 = vector.extract %0[0, 1, 2] : vector<3x5x4x6xf32>
+
+  // [3] is a proper most minor identity map in transpose.
+  // Permutation is a not self inverse and we have.
+  // [1, 2, 0] ^ -1 o [0, 1, 2] = [2, 0, 1] o [0, 1, 2]
+  //                            = [2, 0, 1]
+  //  CHECK-NEXT: vector.extract %[[A]][2, 0, 1] : vector<3x4x5x6xf32>
+  %2 = vector.transpose %a, [1, 2, 0, 3] : vector<3x4x5x6xf32> to vector<4x5x3x6xf32>
+  %3 = vector.extract %2[0, 1, 2] : vector<4x5x3x6xf32>
+
+  // Not a minor identity map so intra-vector level has been permuted
+  //  CHECK-NEXT: vector.transpose %[[B]], [0, 2, 3, 1]
+  //  CHECK-NEXT: vector.extract %{{.*}}[0, 1, 2]
+  %4 = vector.transpose %b, [0, 2, 3, 1] : vector<3x6x5x6xf32> to vector<3x5x6x6xf32>
+  %5 = vector.extract %4[0, 1, 2] : vector<3x5x6x6xf32>
+
+  return %1, %3, %5 : vector<6xf32>, vector<6xf32>, vector<6xf32>
+}
+
+// -----
+
+// CHECK-LABEL: fold_vector_transfers
+func @fold_vector_transfers(%A: memref<?x8xf32>) -> (vector<4x8xf32>, vector<4x9xf32>) {
+  %c0 = constant 0 : index
+  %f0 = constant 0.0 : f32
+
+  // CHECK: vector.transfer_read %{{.*}} {masked = [true, false]}
+  %1 = vector.transfer_read %A[%c0, %c0], %f0 : memref<?x8xf32>, vector<4x8xf32>
+
+  // CHECK: vector.transfer_write %{{.*}} {masked = [true, false]}
+  vector.transfer_write %1, %A[%c0, %c0] : vector<4x8xf32>, memref<?x8xf32>
+
+  // Both dims masked, attribute is elided.
+  // CHECK: vector.transfer_read %{{.*}}
+  // CHECK-NOT: masked
+  %2 = vector.transfer_read %A[%c0, %c0], %f0 : memref<?x8xf32>, vector<4x9xf32>
+
+  // Both dims masked, attribute is elided.
+  // CHECK: vector.transfer_write %{{.*}}
+  // CHECK-NOT: masked
+  vector.transfer_write %2, %A[%c0, %c0] : vector<4x9xf32>, memref<?x8xf32>
+
+  // CHECK: return
+  return %1, %2 : vector<4x8xf32>, vector<4x9xf32>
+}
diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir
index 84d596bf512f8..916403800fe15 100644
--- a/mlir/test/Dialect/Vector/invalid.mlir
+++ b/mlir/test/Dialect/Vector/invalid.mlir
@@ -187,7 +187,7 @@ func @outerproduct_num_operands(%arg0: f32) {
 // -----
 
 func @outerproduct_non_vector_operand(%arg0: f32) {
-  // expected-error@+1 {{expected 2 vector types}}
+  // expected-error@+1 {{expected vector type for operand #1}}
   %1 = vector.outerproduct %arg0, %arg0 : f32, f32
 }
 
@@ -228,6 +228,27 @@ func @outerproduct_operand_2_dim_generic(%arg0: vector<4xf32>, %arg1: vector<8xf
 
 // -----
 
+func @outerproduct_axpy_operand(%arg0: vector<4x8xf32>, %arg1: f32) {
+  // expected-error@+1 {{expected 1-d vector for operand #1}}
+  %1 = vector.outerproduct %arg0, %arg1 : vector<4x8xf32>, f32
+}
+
+// -----
+
+func @outerproduct_axpy_result_generic(%arg0: vector<4xf32>, %arg1: f32) {
+  // expected-error@+1 {{expected 1-d vector result}}
+  %1 = "vector.outerproduct" (%arg0, %arg1) : (vector<4xf32>, f32) -> (vector<4x8xf32>)
+}
+
+// -----
+
+func @outerproduct_axpy_operand_dim_generic(%arg0: vector<8xf32>, %arg1: f32) {
+  // expected-error@+1 {{expected #1 operand dim to match result dim #1}}
+  %1 = "vector.outerproduct" (%arg0, %arg1) : (vector<8xf32>, f32) -> (vector<16xf32>)
+}
+
+// -----
+
 func @outerproduct_operand_3_result_type_generic(%arg0: vector<4xf32>, %arg1: vector<8xf32>, %arg2: vector<4x16xf32>) {
   // expected-error@+1 {{expected operand #3 of same type as result type}}
   %1 = "vector.outerproduct" (%arg0, %arg1, %arg2) : (vector<4xf32>, vector<8xf32>, vector<4x16xf32>) -> (vector<4x8xf32>)
diff --git a/mlir/test/Dialect/Vector/ops.mlir b/mlir/test/Dialect/Vector/ops.mlir
index 02ee4dd3883bd..4ea72864ea94e 100644
--- a/mlir/test/Dialect/Vector/ops.mlir
+++ b/mlir/test/Dialect/Vector/ops.mlir
@@ -266,8 +266,10 @@ func @reshape(%arg0 : vector<3x2x4xf32>) -> (vector<2x3x4xf32>) {
 
 // CHECK-LABEL: @shape_cast
 func @shape_cast(%arg0 : vector<5x1x3x2xf32>,
-                 %arg1 : tuple<vector<5x4x2xf32>, vector<3x4x2xf32>>)
-  -> (vector<15x2xf32>, tuple<vector<20x2xf32>, vector<12x2xf32>>) {
+                 %arg1 : tuple<vector<5x4x2xf32>, vector<3x4x2xf32>>,
+                 %arg2 : vector<8x1xf32>,
+                 %arg3 : vector<16x1x1xf32>)
+  -> (vector<15x2xf32>, tuple<vector<20x2xf32>, vector<12x2xf32>>, vector<8xf32>, vector<16xf32>, vector<16x1xf32>) {
 
   // CHECK: vector.shape_cast %{{.*}} : vector<5x1x3x2xf32> to vector<15x2xf32>
   %0 = vector.shape_cast %arg0 : vector<5x1x3x2xf32> to vector<15x2xf32>
@@ -276,7 +278,16 @@ func @shape_cast(%arg0 : vector<5x1x3x2xf32>,
   %1 = vector.shape_cast %arg1 : tuple<vector<5x4x2xf32>, vector<3x4x2xf32>> to
                                  tuple<vector<20x2xf32>, vector<12x2xf32>>
 
-  return %0, %1 : vector<15x2xf32>, tuple<vector<20x2xf32>, vector<12x2xf32>>
+  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x1xf32> to vector<8xf32>
+  %2 = vector.shape_cast %arg2 : vector<8x1xf32> to vector<8xf32>
+
+  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<16x1x1xf32> to vector<16xf32>
+  %3 = vector.shape_cast %arg3 : vector<16x1x1xf32> to vector<16xf32>
+
+  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<16x1x1xf32> to vector<16x1xf32>
+  %4 = vector.shape_cast %arg3 : vector<16x1x1xf32> to vector<16x1xf32>
+
+  return %0, %1, %2, %3, %4 : vector<15x2xf32>, tuple<vector<20x2xf32>, vector<12x2xf32>>, vector<8xf32>, vector<16xf32>, vector<16x1xf32>
 }
 
 // CHECK-LABEL: @vector_fma
diff --git a/mlir/test/Dialect/Vector/vector-contract-matvec-transforms.mlir b/mlir/test/Dialect/Vector/vector-contract-matvec-transforms.mlir
new file mode 100644
index 0000000000000..3e2896c82bbe5
--- /dev/null
+++ b/mlir/test/Dialect/Vector/vector-contract-matvec-transforms.mlir
@@ -0,0 +1,143 @@
+// RUN: mlir-opt %s -test-vector-contraction-conversion=vector-outerproduct=1 | FileCheck %s
+
+#matvec_accesses = [
+  affine_map<(i, j) -> (i, j)>,
+  affine_map<(i, j) -> (j)>,
+  affine_map<(i, j) -> (i)>
+]
+#matvec_trait = {
+  indexing_maps = #matvec_accesses,
+  iterator_types = ["parallel", "reduction"]
+}
+
+#mattransvec_accesses = [
+  affine_map<(i, j) -> (j, i)>,
+  affine_map<(i, j) -> (j)>,
+  affine_map<(i, j) -> (i)>
+]
+#mattransvec_trait = {
+  indexing_maps = #mattransvec_accesses,
+  iterator_types = ["parallel", "reduction"]
+}
+
+#vecmat_accesses = [
+  affine_map<(i, j) -> (j)>,
+  affine_map<(i, j) -> (i, j)>,
+  affine_map<(i, j) -> (i)>
+]
+#vecmat_trait = {
+  indexing_maps = #vecmat_accesses,
+  iterator_types = ["parallel", "reduction"]
+}
+
+#vecmattrans_accesses = [
+  affine_map<(i, j) -> (j)>,
+  affine_map<(i, j) -> (j, i)>,
+  affine_map<(i, j) -> (i)>
+]
+#vecmattrans_trait = {
+  indexing_maps = #vecmattrans_accesses,
+  iterator_types = ["parallel", "reduction"]
+}
+
+// CHECK-LABEL: func @matvec2x2
+// CHECK-SAME: %[[A:.*0]]: memref<vector<2x2xf32>>
+// CHECK-SAME: %[[B:.*1]]: memref<vector<2xf32>>
+// CHECK-SAME: %[[C:.*2]]: memref<vector<2xf32>>
+// CHECK: %[[T0:.*]] = load %[[A]][] : memref<vector<2x2xf32>>
+// CHECK: %[[T1:.*]] = load %[[B]][] : memref<vector<2xf32>>
+// CHECK: %[[T2:.*]] = load %[[C]][] : memref<vector<2xf32>>
+// CHECK: %[[T3:.*]] = vector.transpose %[[T0]], [1, 0] : vector<2x2xf32> to vector<2x2xf32>
+// CHECK: %[[T4:.*]] = vector.extract %[[T3]][0] : vector<2x2xf32>
+// CHECK: %[[T5:.*]] = vector.extract %[[T1]][0] : vector<2xf32>
+// CHECK: %[[T6:.*]] = vector.outerproduct %[[T4]], %[[T5]], %[[T2]] : vector<2xf32>, f32
+// CHECK: %[[T7:.*]] = vector.extract %[[T3]][1] : vector<2x2xf32>
+// CHECK: %[[T8:.*]] = vector.extract %[[T1]][1] : vector<2xf32>
+// CHECK: %[[T9:.*]] = vector.outerproduct %[[T7]], %[[T8]], %[[T6]] : vector<2xf32>, f32
+// CHECK: store %[[T9]], %[[C]][] : memref<vector<2xf32>>
+// CHECK: return
+func @matvec2x2(%arg0: memref<vector<2x2xf32>>, %arg1: memref<vector<2xf32>>,
+                                                %arg2: memref<vector<2xf32>>) {
+  %A = load %arg0[] : memref<vector<2x2xf32>>
+  %x = load %arg1[] : memref<vector<2xf32>>
+  %b = load %arg2[] : memref<vector<2xf32>>
+  %0 = vector.contract #matvec_trait %A, %x, %b : vector<2x2xf32>, vector<2xf32> into vector<2xf32>
+  store %0, %arg2[] : memref<vector<2xf32>>
+  return
+}
+
+// CHECK-LABEL: func @mattransvec2x2
+// CHECK-SAME: %[[A:.*0]]: memref<vector<2x2xf32>>
+// CHECK-SAME: %[[B:.*1]]: memref<vector<2xf32>>
+// CHECK-SAME: %[[C:.*2]]: memref<vector<2xf32>>
+// CHECK: %[[T0:.*]] = load %[[A]][] : memref<vector<2x2xf32>>
+// CHECK: %[[T1:.*]] = load %[[B]][] : memref<vector<2xf32>>
+// CHECK: %[[T2:.*]] = load %[[C]][] : memref<vector<2xf32>>
+// CHECK: %[[T3:.*]] = vector.extract %[[T0]][0] : vector<2x2xf32>
+// CHECK: %[[T4:.*]] = vector.extract %[[T1]][0] : vector<2xf32>
+// CHECK: %[[T5:.*]] = vector.outerproduct %[[T3]], %[[T4]], %[[T2]] : vector<2xf32>, f32
+// CHECK: %[[T6:.*]] = vector.extract %[[T0]][1] : vector<2x2xf32>
+// CHECK: %[[T7:.*]] = vector.extract %[[T1]][1] : vector<2xf32>
+// CHECK: %[[T8:.*]] = vector.outerproduct %[[T6]], %[[T7]], %[[T5]] : vector<2xf32>, f32
+// CHECK: store %[[T8]], %[[C]][] : memref<vector<2xf32>>
+// CHECK: return
+func @mattransvec2x2(%arg0: memref<vector<2x2xf32>>, %arg1: memref<vector<2xf32>>,
+                                                     %arg2: memref<vector<2xf32>>) {
+  %A = load %arg0[] : memref<vector<2x2xf32>>
+  %x = load %arg1[] : memref<vector<2xf32>>
+  %b = load %arg2[] : memref<vector<2xf32>>
+  %0 = vector.contract #mattransvec_trait %A, %x, %b : vector<2x2xf32>, vector<2xf32> into vector<2xf32>
+  store %0, %arg2[] : memref<vector<2xf32>>
+  return
+}
+
+// CHECK-LABEL: func @vecmat2x2
+// CHECK-SAME: %[[A:.*0]]: memref<vector<2x2xf32>>
+// CHECK-SAME: %[[B:.*1]]: memref<vector<2xf32>>
+// CHECK-SAME: %[[C:.*2]]: memref<vector<2xf32>>
+// CHECK: %[[T0:.*]] = load %[[A]][] : memref<vector<2x2xf32>>
+// CHECK: %[[T1:.*]] = load %[[B]][] : memref<vector<2xf32>>
+// CHECK: %[[T2:.*]] = load %[[C]][] : memref<vector<2xf32>>
+// CHECK: %[[T3:.*]] = vector.transpose %[[T0]], [1, 0] : vector<2x2xf32> to vector<2x2xf32>
+// CHECK: %[[T4:.*]] = vector.extract %[[T3]][0] : vector<2x2xf32>
+// CHECK: %[[T5:.*]] = vector.extract %[[T1]][0] : vector<2xf32>
+// CHECK: %[[T6:.*]] = vector.outerproduct %[[T4]], %[[T5]], %[[T2]] : vector<2xf32>, f32
+// CHECK: %[[T7:.*]] = vector.extract %[[T3]][1] : vector<2x2xf32>
+// CHECK: %[[T8:.*]] = vector.extract %[[T1]][1] : vector<2xf32>
+// CHECK: %[[T9:.*]] = vector.outerproduct %[[T7]], %[[T8]], %[[T6]] : vector<2xf32>, f32
+// CHECK: store %[[T9]], %[[C]][] : memref<vector<2xf32>>
+// CHECK: return
+func @vecmat2x2(%arg0: memref<vector<2x2xf32>>, %arg1: memref<vector<2xf32>>,
+                                                %arg2: memref<vector<2xf32>>) {
+  %A = load %arg0[] : memref<vector<2x2xf32>>
+  %x = load %arg1[] : memref<vector<2xf32>>
+  %b = load %arg2[] : memref<vector<2xf32>>
+  %0 = vector.contract #vecmat_trait %x, %A, %b : vector<2xf32>, vector<2x2xf32> into vector<2xf32>
+  store %0, %arg2[] : memref<vector<2xf32>>
+  return
+}
+
+// CHECK-LABEL: func @vecmattrans2x2
+// CHECK-SAME: %[[A:.*0]]: memref<vector<2x2xf32>>
+// CHECK-SAME: %[[B:.*1]]: memref<vector<2xf32>>
+// CHECK-SAME: %[[C:.*2]]: memref<vector<2xf32>>
+// CHECK: %[[T0:.*]] = load %[[A]][] : memref<vector<2x2xf32>>
+// CHECK: %[[T1:.*]] = load %[[B]][] : memref<vector<2xf32>>
+// CHECK: %[[T2:.*]] = load %[[C]][] : memref<vector<2xf32>>
+// CHECK: %[[T3:.*]] = vector.extract %[[T0]][0] : vector<2x2xf32>
+// CHECK: %[[T4:.*]] = vector.extract %[[T1]][0] : vector<2xf32>
+// CHECK: %[[T5:.*]] = vector.outerproduct %[[T3]], %[[T4]], %[[T2]] : vector<2xf32>, f32
+// CHECK: %[[T6:.*]] = vector.extract %[[T0]][1] : vector<2x2xf32>
+// CHECK: %[[T7:.*]] = vector.extract %[[T1]][1] : vector<2xf32>
+// CHECK: %[[T8:.*]] = vector.outerproduct %[[T6]], %[[T7]], %[[T5]] : vector<2xf32>, f32
+// CHECK: store %[[T8]], %[[C]][] : memref<vector<2xf32>>
+// CHECK: return
+func @vecmattrans2x2(%arg0: memref<vector<2x2xf32>>, %arg1: memref<vector<2xf32>>,
+                                                     %arg2: memref<vector<2xf32>>) {
+  %A = load %arg0[] : memref<vector<2x2xf32>>
+  %x = load %arg1[] : memref<vector<2xf32>>
+  %b = load %arg2[] : memref<vector<2xf32>>
+  %0 = vector.contract #vecmattrans_trait %x, %A, %b : vector<2xf32>, vector<2x2xf32> into vector<2xf32>
+  store %0, %arg2[] : memref<vector<2xf32>>
+  return
+}
diff --git a/mlir/test/Dialect/Vector/vector-contract-transforms.mlir b/mlir/test/Dialect/Vector/vector-contract-transforms.mlir
index d4c68ab2cebdd..82faadf100e9a 100644
--- a/mlir/test/Dialect/Vector/vector-contract-transforms.mlir
+++ b/mlir/test/Dialect/Vector/vector-contract-transforms.mlir
@@ -16,8 +16,7 @@
 // CHECK-SAME: %[[A:.*0]]: vector<4xf32>,
 // CHECK-SAME: %[[B:.*1]]: vector<4xf32>,
 // CHECK-SAME: %[[C:.*2]]: f32
-// CHECK:      %[[Z:.*]] = constant dense<0.000000e+00> : vector<4xf32>
-// CHECK:      %[[F:.*]] = vector.fma %[[A]], %[[B]], %[[Z]] : vector<4xf32>
+// CHECK:      %[[F:.*]] = mulf %[[A]], %[[B]] : vector<4xf32>
 // CHECK:      %[[R:.*]] = vector.reduction "add", %[[F]], %[[C]] : vector<4xf32> into f32
 // CHECK:      return %[[R]] : f32
 
@@ -42,15 +41,14 @@ func @extract_contract1(%arg0: vector<4xf32>, %arg1: vector<4xf32>, %arg2: f32)
 // CHECK-SAME: %[[B:.*1]]: vector<3xf32>,
 // CHECK-SAME: %[[C:.*2]]: vector<2xf32>
 // CHECK:      %[[R:.*]] = constant dense<0.000000e+00> : vector<2xf32>
-// CHECK:      %[[Z:.*]] = constant dense<0.000000e+00> : vector<3xf32>
 // CHECK:      %[[T0:.*]] = vector.extract %[[A]][0] : vector<2x3xf32>
 // CHECK:      %[[T1:.*]] = vector.extract %[[C]][0] : vector<2xf32>
-// CHECK:      %[[T2:.*]] = vector.fma %[[T0]], %[[B]], %[[Z]] : vector<3xf32>
+// CHECK:      %[[T2:.*]] = mulf %[[T0]], %[[B]] : vector<3xf32>
 // CHECK:      %[[T3:.*]] = vector.reduction "add", %[[T2]], %[[T1]] : vector<3xf32> into f32
 // CHECK:      %[[T4:.*]] = vector.insert %[[T3]], %[[R]] [0] : f32 into vector<2xf32>
 // CHECK:      %[[T5:.*]] = vector.extract %[[A]][1] : vector<2x3xf32>
 // CHECK:      %[[T6:.*]] = vector.extract %[[C]][1] : vector<2xf32>
-// CHECK:      %[[T7:.*]] = vector.fma %[[T5]], %[[B]], %[[Z]] : vector<3xf32>
+// CHECK:      %[[T7:.*]] = mulf %[[T5]], %[[B]] : vector<3xf32>
 // CHECK:      %[[T8:.*]] = vector.reduction "add", %[[T7]], %[[T6]] : vector<3xf32> into f32
 // CHECK:      %[[T9:.*]] = vector.insert %[[T8]], %[[T4]] [1] : f32 into vector<2xf32>
 // CHECK:      return %[[T9]] : vector<2xf32>
@@ -78,15 +76,14 @@ func @extract_contract2(%arg0: vector<2x3xf32>,
 // CHECK-SAME: %[[B:.*1]]: vector<2x3xf32>,
 // CHECK-SAME: %[[C:.*2]]: vector<2xf32>
 // CHECK:      %[[R:.*]] = constant dense<0.000000e+00> : vector<2xf32>
-// CHECK:      %[[Z:.*]] = constant dense<0.000000e+00> : vector<3xf32>
 // CHECK:      %[[T0:.*]] = vector.extract %[[B]][0] : vector<2x3xf32>
 // CHECK:      %[[T1:.*]] = vector.extract %[[C]][0] : vector<2xf32>
-// CHECK:      %[[T2:.*]] = vector.fma %[[A]], %[[T0]], %[[Z]] : vector<3xf32>
+// CHECK:      %[[T2:.*]] = mulf %[[A]], %[[T0]] : vector<3xf32>
 // CHECK:      %[[T3:.*]] = vector.reduction "add", %[[T2]], %[[T1]] : vector<3xf32> into f32
 // CHECK:      %[[T4:.*]] = vector.insert %[[T3]], %[[R]] [0] : f32 into vector<2xf32>
 // CHECK:      %[[T5:.*]] = vector.extract %[[B]][1] : vector<2x3xf32>
 // CHECK:      %[[T6:.*]] = vector.extract %[[C]][1] : vector<2xf32>
-// CHECK:      %[[T7:.*]] = vector.fma %[[A]], %[[T5]], %[[Z]] : vector<3xf32>
+// CHECK:      %[[T7:.*]] = mulf %[[A]], %[[T5]] : vector<3xf32>
 // CHECK:      %[[T8:.*]] = vector.reduction "add", %[[T7]], %[[T6]] : vector<3xf32> into f32
 // CHECK:      %[[T9:.*]] = vector.insert %[[T8]], %[[T4]] [1] : f32 into vector<2xf32>
 // CHECK:      return %[[T9]] : vector<2xf32>
@@ -116,52 +113,46 @@ func @extract_contract3(%arg0: vector<3xf32>,
 // CHECK:    %[[R:.*]] = constant dense<0.000000e+00> : vector<2x2xf32>
 // CHECK:    %[[Z:.*]] = constant dense<0.000000e+00> : vector<2xf32>
 // CHECK:    %[[T0:.*]] = vector.extract %[[A]][0] : vector<2x2xf32>
-// CHECK:    %[[T1:.*]] = vector.extract %[[C]][0] : vector<2x2xf32>
-// CHECK:    %[[T2:.*]] = vector.extract %[[B]][0] : vector<2x2xf32>
-// CHECK:    %[[T3:.*]] = vector.extract %[[T2]][0] : vector<2xf32>
-// CHECK:    %[[T4:.*]] = vector.insert %[[T3]], %[[Z]] [0] : f32 into vector<2xf32>
-// CHECK:    %[[T5:.*]] = vector.extract %[[B]][1] : vector<2x2xf32>
-// CHECK:    %[[T6:.*]] = vector.extract %[[T5]][0] : vector<2xf32>
-// CHECK:    %[[T7:.*]] = vector.insert %[[T6]], %[[T4]] [1] : f32 into vector<2xf32>
-// CHECK:    %[[T8:.*]] = vector.extract %[[T1]][0] : vector<2xf32>
-// CHECK:    %[[T9:.*]] = vector.fma %[[T0]], %[[T7]], %[[Z]] : vector<2xf32>
+// CHECK:    %[[T2:.*]] = vector.extract %[[B]][0, 0] : vector<2x2xf32>
+// CHECK:    %[[T4:.*]] = vector.insert %[[T2]], %[[Z]] [0] : f32 into vector<2xf32>
+// CHECK:    %[[T5:.*]] = vector.extract %[[B]][1, 0] : vector<2x2xf32>
+// CHECK:    %[[T7:.*]] = vector.insert %[[T5]], %[[T4]] [1] : f32 into vector<2xf32>
+// CHECK:    %[[T8:.*]] = vector.extract %[[C]][0, 0] : vector<2x2xf32>
+// CHECK:    %[[T9:.*]] = mulf %[[T0]], %[[T7]] : vector<2xf32>
 // CHECK:    %[[T10:.*]] = vector.reduction "add", %[[T9]], %[[T8]] : vector<2xf32> into f32
 // CHECK:    %[[T11:.*]] = vector.insert %[[T10]], %[[Z]] [0] : f32 into vector<2xf32>
-// CHECK:    %[[T12:.*]] = vector.extract %[[B]][0] : vector<2x2xf32>
-// CHECK:    %[[T13:.*]] = vector.extract %[[T12]][1] : vector<2xf32>
-// CHECK:    %[[T14:.*]] = vector.insert %[[T13]], %[[Z]] [0] : f32 into vector<2xf32>
-// CHECK:    %[[T15:.*]] = vector.extract %[[B]][1] : vector<2x2xf32>
-// CHECK:    %[[T16:.*]] = vector.extract %[[T15]][1] : vector<2xf32>
-// CHECK:    %[[T17:.*]] = vector.insert %[[T16]], %[[T14]] [1] : f32 into vector<2xf32>
-// CHECK:    %[[T18:.*]] = vector.extract %[[T1]][1] : vector<2xf32>
-// CHECK:    %[[T19:.*]] = vector.fma %[[T0]], %[[T17]], %[[Z]] : vector<2xf32>
+//
+// CHECK:    %[[T12:.*]] = vector.extract %[[B]][0, 1] : vector<2x2xf32>
+// CHECK:    %[[T14:.*]] = vector.insert %[[T12]], %[[Z]] [0] : f32 into vector<2xf32>
+// CHECK:    %[[T15:.*]] = vector.extract %[[B]][1, 1] : vector<2x2xf32>
+// CHECK:    %[[T17:.*]] = vector.insert %[[T15]], %[[T14]] [1] : f32 into vector<2xf32>
+// CHECK:    %[[T18:.*]] = vector.extract %[[C]][0, 1] : vector<2x2xf32>
+// CHECK:    %[[T19:.*]] = mulf %[[T0]], %[[T17]] : vector<2xf32>
 // CHECK:    %[[T20:.*]] = vector.reduction "add", %[[T19]], %[[T18]] : vector<2xf32> into f32
 // CHECK:    %[[T21:.*]] = vector.insert %[[T20]], %[[T11]] [1] : f32 into vector<2xf32>
 // CHECK:    %[[T22:.*]] = vector.insert %[[T21]], %[[R]] [0] : vector<2xf32> into vector<2x2xf32>
+//
 // CHECK:    %[[T23:.*]] = vector.extract %[[A]][1] : vector<2x2xf32>
-// CHECK:    %[[T24:.*]] = vector.extract %[[C]][1] : vector<2x2xf32>
-// CHECK:    %[[T25:.*]] = vector.extract %[[B]][0] : vector<2x2xf32>
-// CHECK:    %[[T26:.*]] = vector.extract %[[T25]][0] : vector<2xf32>
-// CHECK:    %[[T27:.*]] = vector.insert %[[T26]], %[[Z]] [0] : f32 into vector<2xf32>
-// CHECK:    %[[T28:.*]] = vector.extract %[[B]][1] : vector<2x2xf32>
-// CHECK:    %[[T29:.*]] = vector.extract %[[T28]][0] : vector<2xf32>
-// CHECK:    %[[T30:.*]] = vector.insert %[[T29]], %[[T27]] [1] : f32 into vector<2xf32>
-// CHECK:    %[[T31:.*]] = vector.extract %[[T24]][0] : vector<2xf32>
-// CHECK:    %[[T32:.*]] = vector.fma %[[T23]], %[[T30]], %[[Z]] : vector<2xf32>
-// CHECK:    %[[T33:.*]] = vector.reduction "add", %[[T32]], %[[T31]] : vector<2xf32> into f32
+// CHECK:    %[[T22b:.*]] = vector.extract %[[B]][0, 0] : vector<2x2xf32>
+// CHECK:    %[[T24:.*]] = vector.insert %[[T22b]], %[[Z]] [0] : f32 into vector<2xf32>
+// CHECK:    %[[T25:.*]] = vector.extract %[[B]][1, 0] : vector<2x2xf32>
+// CHECK:    %[[T27:.*]] = vector.insert %[[T25]], %[[T24]] [1] : f32 into vector<2xf32>
+// CHECK:    %[[T28:.*]] = vector.extract %[[C]][1, 0] : vector<2x2xf32>
+// CHECK:    %[[T32:.*]] = mulf %[[T23]], %[[T27]] : vector<2xf32>
+// CHECK:    %[[T33:.*]] = vector.reduction "add", %[[T32]], %[[T28]] : vector<2xf32> into f32
 // CHECK:    %[[T34:.*]] = vector.insert %[[T33]], %[[Z]] [0] : f32 into vector<2xf32>
-// CHECK:    %[[T35:.*]] = vector.extract %[[B]][0] : vector<2x2xf32>
-// CHECK:    %[[T36:.*]] = vector.extract %[[T35]][1] : vector<2xf32>
-// CHECK:    %[[T37:.*]] = vector.insert %[[T36]], %[[Z]] [0] : f32 into vector<2xf32>
-// CHECK:    %[[T38:.*]] = vector.extract %[[B]][1] : vector<2x2xf32>
-// CHECK:    %[[T39:.*]] = vector.extract %[[T38]][1] : vector<2xf32>
-// CHECK:    %[[T40:.*]] = vector.insert %[[T39]], %[[T37]] [1] : f32 into vector<2xf32>
-// CHECK:    %[[T41:.*]] = vector.extract %[[T24]][1] : vector<2xf32>
-// CHECK:    %[[T42:.*]] = vector.fma %[[T23]], %[[T40]], %[[Z]] : vector<2xf32>
-// CHECK:    %[[T43:.*]] = vector.reduction "add", %[[T42]], %[[T41]] : vector<2xf32> into f32
-// CHECK:    %[[T44:.*]] = vector.insert %[[T43]], %[[T34]] [1] : f32 into vector<2xf32>
-// CHECK:    %[[T45:.*]] = vector.insert %[[T44]], %[[T22]] [1] : vector<2xf32> into vector<2x2xf32>
-// CHECK:    return %[[T45]] : vector<2x2xf32>
+//
+// CHECK:    %[[T42:.*]] = vector.extract %[[B]][0, 1] : vector<2x2xf32>
+// CHECK:    %[[T44:.*]] = vector.insert %[[T42]], %[[Z]] [0] : f32 into vector<2xf32>
+// CHECK:    %[[T45:.*]] = vector.extract %[[B]][1, 1] : vector<2x2xf32>
+// CHECK:    %[[T47:.*]] = vector.insert %[[T45]], %[[T44]] [1] : f32 into vector<2xf32>
+// CHECK:    %[[T48:.*]] = vector.extract %[[C]][1, 1] : vector<2x2xf32>
+// CHECK:    %[[T49:.*]] = mulf %[[T23]], %[[T47]] : vector<2xf32>
+// CHECK:    %[[T50:.*]] = vector.reduction "add", %[[T49]], %[[T48]] : vector<2xf32> into f32
+//
+// CHECK:    %[[T51:.*]] = vector.insert %[[T50]], %[[T34]] [1] : f32 into vector<2xf32>
+// CHECK:    %[[T52:.*]] = vector.insert %[[T51]], %[[T22]] [1] : vector<2xf32> into vector<2x2xf32>
+// CHECK:    return %[[T52]] : vector<2x2xf32>
 
 func @extract_contract4(%arg0: vector<2x2xf32>,
                         %arg1: vector<2x2xf32>,
@@ -185,14 +176,13 @@ func @extract_contract4(%arg0: vector<2x2xf32>,
 // CHECK-SAME: %[[A:.*0]]: vector<2x3xf32>,
 // CHECK-SAME: %[[B:.*1]]: vector<2x3xf32>,
 // CHECK-SAME: %[[C:.*2]]: f32
-// CHECK:      %[[Z:.*]] = constant dense<0.000000e+00> : vector<3xf32>
 // CHECK:      %[[T0:.*]] = vector.extract %[[A]][0] : vector<2x3xf32>
 // CHECK:      %[[T1:.*]] = vector.extract %[[B]][0] : vector<2x3xf32>
-// CHECK:      %[[T2:.*]] = vector.fma %[[T0]], %[[T1]], %[[Z]] : vector<3xf32>
+// CHECK:      %[[T2:.*]] = mulf %[[T0]], %[[T1]] : vector<3xf32>
 // CHECK:      %[[T3:.*]] = vector.reduction "add", %[[T2]], %[[C]] : vector<3xf32> into f32
 // CHECK:      %[[T4:.*]] = vector.extract %[[A]][1] : vector<2x3xf32>
 // CHECK:      %[[T5:.*]] = vector.extract %[[B]][1] : vector<2x3xf32>
-// CHECK:      %[[T6:.*]] = vector.fma %[[T4]], %[[T5]], %[[Z]] : vector<3xf32>
+// CHECK:      %[[T6:.*]] = mulf %[[T4]], %[[T5]] : vector<3xf32>
 // CHECK:      %[[T7:.*]] = vector.reduction "add", %[[T6]], %[[T3]] : vector<3xf32> into f32
 // CHECK:      return %[[T7]] : f32
 
@@ -220,28 +210,23 @@ func @full_contract1(%arg0: vector<2x3xf32>,
 // CHECK-SAME: %[[C:.*2]]: f32
 // CHECK:      %[[Z:.*]] = constant dense<0.000000e+00> : vector<3xf32>
 // CHECK:      %[[T0:.*]] = vector.extract %[[A]][0] : vector<2x3xf32>
-// CHECK:      %[[T1:.*]] = vector.extract %[[B]][0] : vector<3x2xf32>
-// CHECK:      %[[T2:.*]] = vector.extract %[[T1]][0] : vector<2xf32>
-// CHECK:      %[[T3:.*]] = vector.insert %[[T2]], %[[Z]] [0] : f32 into vector<3xf32>
-// CHECK:      %[[T4:.*]] = vector.extract %[[B]][1] : vector<3x2xf32>
-// CHECK:      %[[T5:.*]] = vector.extract %[[T4]][0] : vector<2xf32>
-// CHECK:      %[[T6:.*]] = vector.insert %[[T5]], %[[T3]] [1] : f32 into vector<3xf32>
-// CHECK:      %[[T7:.*]] = vector.extract %[[B]][2] : vector<3x2xf32>
-// CHECK:      %[[T8:.*]] = vector.extract %[[T7]][0] : vector<2xf32>
-// CHECK:      %[[T9:.*]] = vector.insert %[[T8]], %[[T6]] [2] : f32 into vector<3xf32>
-// CHECK:      %[[T10:.*]] = vector.fma %[[T0]], %[[T9]], %[[Z]] : vector<3xf32>
+// CHECK:      %[[T1:.*]] = vector.extract %[[B]][0, 0] : vector<3x2xf32>
+// CHECK:      %[[T3:.*]] = vector.insert %[[T1]], %[[Z]] [0] : f32 into vector<3xf32>
+// CHECK:      %[[T4:.*]] = vector.extract %[[B]][1, 0] : vector<3x2xf32>
+// CHECK:      %[[T6:.*]] = vector.insert %[[T4]], %[[T3]] [1] : f32 into vector<3xf32>
+// CHECK:      %[[T7:.*]] = vector.extract %[[B]][2, 0] : vector<3x2xf32>
+// CHECK:      %[[T9:.*]] = vector.insert %[[T7]], %[[T6]] [2] : f32 into vector<3xf32>
+// CHECK:      %[[T10:.*]] = mulf %[[T0]], %[[T9]] : vector<3xf32>
 // CHECK:      %[[T11:.*]] = vector.reduction "add", %[[T10]], %[[C]] : vector<3xf32> into f32
+//
 // CHECK:      %[[T12:.*]] = vector.extract %[[A]][1] : vector<2x3xf32>
-// CHECK:      %[[T13:.*]] = vector.extract %[[B]][0] : vector<3x2xf32>
-// CHECK:      %[[T14:.*]] = vector.extract %[[T13]][1] : vector<2xf32>
-// CHECK:      %[[T15:.*]] = vector.insert %[[T14]], %[[Z]] [0] : f32 into vector<3xf32>
-// CHECK:      %[[T16:.*]] = vector.extract %[[B]][1] : vector<3x2xf32>
-// CHECK:      %[[T17:.*]] = vector.extract %[[T16]][1] : vector<2xf32>
-// CHECK:      %[[T18:.*]] = vector.insert %[[T17]], %[[T15]] [1] : f32 into vector<3xf32>
-// CHECK:      %[[T19:.*]] = vector.extract %[[B]][2] : vector<3x2xf32>
-// CHECK:      %[[T20:.*]] = vector.extract %[[T19]][1] : vector<2xf32>
-// CHECK:      %[[T21:.*]] = vector.insert %[[T20]], %[[T18]] [2] : f32 into vector<3xf32>
-// CHECK:      %[[T22:.*]] = vector.fma %[[T12]], %[[T21]], %[[Z]] : vector<3xf32>
+// CHECK:      %[[T13:.*]] = vector.extract %[[B]][0, 1] : vector<3x2xf
+// CHECK:      %[[T15:.*]] = vector.insert %[[T13]], %[[Z]] [0] : f32 into vector<3xf32>
+// CHECK:      %[[T16:.*]] = vector.extract %[[B]][1, 1] : vector<3x2xf32>
+// CHECK:      %[[T18:.*]] = vector.insert %[[T16]], %[[T15]] [1] : f32 into vector<3xf32>
+// CHECK:      %[[T19:.*]] = vector.extract %[[B]][2, 1] : vector<3x2xf32>
+// CHECK:      %[[T21:.*]] = vector.insert %[[T19]], %[[T18]] [2] : f32 into vector<3xf32>
+// CHECK:      %[[T22:.*]] = mulf %[[T12]], %[[T21]] : vector<3xf32>
 // CHECK:      %[[T23:.*]] = vector.reduction "add", %[[T22]], %[[T11]] : vector<3xf32> into f32
 // CHECK:      return %[[T23]] : f32
 
@@ -297,6 +282,97 @@ func @outerproduct_acc(%arg0: vector<2xf32>,
   return %0: vector<2x3xf32>
 }
 
+// CHECK-LABEL: func @outerproduct_noacc_int
+// CHECK-SAME: %[[A:.*0]]: vector<2xi32>,
+// CHECK-SAME: %[[B:.*1]]: vector<3xi32>
+// CHECK:      %[[C0:.*]] = constant dense<0> : vector<2x3xi32>
+// CHECK:      %[[T0:.*]] = vector.extract %[[A]][0] : vector<2xi32>
+// CHECK:      %[[T1:.*]] = splat %[[T0]] : vector<3xi32>
+// CHECK:      %[[T2:.*]] = muli %[[T1]], %[[B]] : vector<3xi32>
+// CHECK:      %[[T3:.*]] = vector.insert %[[T2]], %[[C0]] [0] : vector<3xi32> into vector<2x3xi32>
+// CHECK:      %[[T4:.*]] = vector.extract %[[A]][1] : vector<2xi32>
+// CHECK:      %[[T5:.*]] = splat %[[T4]] : vector<3xi32>
+// CHECK:      %[[T6:.*]] = muli %[[T5]], %[[B]] : vector<3xi32>
+// CHECK:      %[[T7:.*]] = vector.insert %[[T6]], %[[T3]] [1] : vector<3xi32> into vector<2x3xi32>
+// CHECK:      return %[[T7]] : vector<2x3xi32>
+func @outerproduct_noacc_int(%arg0: vector<2xi32>,
+                             %arg1: vector<3xi32>) -> vector<2x3xi32> {
+  %0 = vector.outerproduct %arg0, %arg1 : vector<2xi32>, vector<3xi32>
+  return %0: vector<2x3xi32>
+}
+
+// CHECK-LABEL: func @outerproduct_acc_int
+// CHECK-SAME: %[[A:.*0]]: vector<2xi32>,
+// CHECK-SAME: %[[B:.*1]]: vector<3xi32>,
+// CHECK-SAME: %[[C:.*2]]: vector<2x3xi32>
+// CHECK:      %[[C0:.*]] = constant dense<0> : vector<2x3xi32>
+// CHECK:      %[[T0:.*]] = vector.extract %[[A]][0] : vector<2xi32>
+// CHECK:      %[[T1:.*]] = splat %[[T0]] : vector<3xi32>
+// CHECK:      %[[T2:.*]] = vector.extract %[[C]][0] : vector<2x3xi32>
+// CHECK:      %[[T3:.*]] = muli %[[T1]], %[[B]] : vector<3xi32>
+// CHECK:      %[[T4:.*]] = addi %[[T3]], %[[T2]] : vector<3xi32>
+// CHECK:      %[[T5:.*]] = vector.insert %[[T4]], %[[C0]] [0] : vector<3xi32> into vector<2x3xi32>
+// CHECK:      %[[T6:.*]] = vector.extract %[[A]][1] : vector<2xi32>
+// CHECK:      %[[T7:.*]] = splat %[[T6]] : vector<3xi32>
+// CHECK:      %[[T8:.*]] = vector.extract %[[C]][1] : vector<2x3xi32>
+// CHECK:      %[[T9:.*]] = muli %[[T7]], %[[B]] : vector<3xi32>
+// CHECK:      %[[T10:.*]] = addi %[[T9]], %[[T8]] : vector<3xi32>
+// CHECK:      %[[T11:.*]] = vector.insert %[[T10]], %[[T5]] [1] : vector<3xi32> into vector<2x3xi32>
+// CHECK:      return %[[T11]] : vector<2x3xi32>
+func @outerproduct_acc_int(%arg0: vector<2xi32>,
+                           %arg1: vector<3xi32>,
+                           %arg2: vector<2x3xi32>) -> vector<2x3xi32> {
+  %0 = vector.outerproduct %arg0, %arg1, %arg2 : vector<2xi32>, vector<3xi32>
+  return %0: vector<2x3xi32>
+}
+
+// CHECK-LABEL: func @axpy_fp(
+// CHECK-SAME: %[[A:.*0]]: vector<16xf32>,
+// CHECK-SAME: %[[B:.*1]]: f32)
+// CHECK: %[[T0:.*]] = splat %[[B]] : vector<16xf32>
+// CHECK: %[[T1:.*]] = mulf %[[A]], %[[T0]] : vector<16xf32>
+// CHECK: return %[[T1]] : vector<16xf32>
+func @axpy_fp(%arg0: vector<16xf32>, %arg1: f32) -> vector<16xf32> {
+   %0 = vector.outerproduct %arg0, %arg1: vector<16xf32>, f32
+   return %0: vector<16xf32>
+}
+
+// CHECK-LABEL: func @axpy_fp_add(
+// CHECK-SAME: %[[A:.*0]]: vector<16xf32>,
+// CHECK-SAME: %[[B:.*1]]: f32,
+// CHECK-SAME: %[[C:.*2]]: vector<16xf32>)
+// CHECK: %[[T0:.*]] = splat %[[B]] : vector<16xf32>
+// CHECK: %[[T1:.*]] = vector.fma %[[A]], %[[T0]], %[[C]] : vector<16xf32>
+// CHECK: return %[[T1]] : vector<16xf32>
+func @axpy_fp_add(%arg0: vector<16xf32>, %arg1: f32, %arg2 : vector<16xf32>) -> vector<16xf32> {
+   %0 = vector.outerproduct %arg0, %arg1, %arg2: vector<16xf32>, f32
+   return %0: vector<16xf32>
+}
+
+// CHECK-LABEL: func @axpy_int(
+// CHECK-SAME: %[[A:.*0]]: vector<16xi32>,
+// CHECK-SAME: %[[B:.*1]]: i32)
+// CHECK: %[[T0:.*]] = splat %[[B]] : vector<16xi32>
+// CHECK: %[[T1:.*]] = muli %[[A]], %[[T0]] : vector<16xi32>
+// CHECK: return %[[T1]] : vector<16xi32>
+func @axpy_int(%arg0: vector<16xi32>, %arg1: i32) -> vector<16xi32> {
+   %0 = vector.outerproduct %arg0, %arg1: vector<16xi32>, i32
+   return %0: vector<16xi32>
+}
+
+// CHECK-LABEL: func @axpy_int_add(
+// CHECK-SAME: %[[A:.*0]]: vector<16xi32>,
+// CHECK-SAME: %[[B:.*1]]: i32,
+// CHECK-SAME: %[[C:.*2]]: vector<16xi32>)
+// CHECK: %[[T0:.*]] = splat %[[B]] : vector<16xi32>
+// CHECK: %[[T1:.*]] = muli %[[A]], %[[T0]] : vector<16xi32>
+// CHECK: %[[T2:.*]] = addi %[[T1]], %[[C]] : vector<16xi32>
+// CHECK: return %[[T2]] : vector<16xi32>
+func @axpy_int_add(%arg0: vector<16xi32>, %arg1: i32, %arg2: vector<16xi32>) -> vector<16xi32> {
+   %0 = vector.outerproduct %arg0, %arg1, %arg2: vector<16xi32>, i32
+   return %0: vector<16xi32>
+}
+
 // CHECK-LABEL: func @transpose23
 // CHECK-SAME: %[[A:.*]]: vector<2x3xf32>
 // CHECK:      %[[Z:.*]] = constant dense<0.000000e+00> : vector<3x2xf32>
@@ -617,21 +693,17 @@ func @broadcast_stretch_at_start(%arg0: vector<1x4xf32>) -> vector<3x4xf32> {
 // CHECK-LABEL: func @broadcast_stretch_at_end
 // CHECK-SAME: %[[A:.*0]]: vector<4x1xf32>
 // CHECK:      %[[C0:.*]] = constant dense<0.000000e+00> : vector<4x3xf32>
-// CHECK:      %[[T0:.*]] = vector.extract %[[A]][0] : vector<4x1xf32>
-// CHECK:      %[[T1:.*]] = vector.extract %[[T0]][0] : vector<1xf32>
-// CHECK:      %[[T2:.*]] = splat %[[T1]] : vector<3xf32>
+// CHECK:      %[[T0:.*]] = vector.extract %[[A]][0, 0] : vector<4x1xf32>
+// CHECK:      %[[T2:.*]] = splat %[[T0]] : vector<3xf32>
 // CHECK:      %[[T3:.*]] = vector.insert %[[T2]], %[[C0]] [0] : vector<3xf32> into vector<4x3xf32>
-// CHECK:      %[[T4:.*]] = vector.extract %[[A]][1] : vector<4x1xf32>
-// CHECK:      %[[T5:.*]] = vector.extract %[[T4]][0] : vector<1xf32>
-// CHECK:      %[[T6:.*]] = splat %[[T5]] : vector<3xf32>
+// CHECK:      %[[T4:.*]] = vector.extract %[[A]][1, 0] : vector<4x1xf32>
+// CHECK:      %[[T6:.*]] = splat %[[T4]] : vector<3xf32>
 // CHECK:      %[[T7:.*]] = vector.insert %[[T6]], %[[T3]] [1] : vector<3xf32> into vector<4x3xf32>
-// CHECK:      %[[T8:.*]] = vector.extract %[[A]][2] : vector<4x1xf32>
-// CHECK:      %[[T9:.*]] = vector.extract %[[T8]][0] : vector<1xf32>
-// CHECK:      %[[T10:.*]] = splat %[[T9]] : vector<3xf32>
+// CHECK:      %[[T8:.*]] = vector.extract %[[A]][2, 0] : vector<4x1xf32>
+// CHECK:      %[[T10:.*]] = splat %[[T8]] : vector<3xf32>
 // CHECK:      %[[T11:.*]] = vector.insert %[[T10]], %[[T7]] [2] : vector<3xf32> into vector<4x3xf32>
-// CHECK:      %[[T12:.*]] = vector.extract %[[A]][3] : vector<4x1xf32>
-// CHECK:      %[[T13:.*]] = vector.extract %[[T12]][0] : vector<1xf32>
-// CHECK:      %[[T14:.*]] = splat %[[T13]] : vector<3xf32>
+// CHECK:      %[[T12:.*]] = vector.extract %[[A]][3, 0] : vector<4x1xf32>
+// CHECK:      %[[T14:.*]] = splat %[[T12]] : vector<3xf32>
 // CHECK:      %[[T15:.*]] = vector.insert %[[T14]], %[[T11]] [3] : vector<3xf32> into vector<4x3xf32>
 // CHECK:      return %[[T15]] : vector<4x3xf32>
 
@@ -644,29 +716,25 @@ func @broadcast_stretch_at_end(%arg0: vector<4x1xf32>) -> vector<4x3xf32> {
 // CHECK-SAME: %[[A:.*0]]: vector<4x1x2xf32>
 // CHECK:      %[[C0:.*]] = constant dense<0.000000e+00> : vector<4x3x2xf32>
 // CHECK:      %[[C1:.*]] = constant dense<0.000000e+00> : vector<3x2xf32>
-// CHECK:      %[[T0:.*]] = vector.extract %[[A]][0] : vector<4x1x2xf32>
-// CHECK:      %[[T1:.*]] = vector.extract %[[T0]][0] : vector<1x2xf32>
-// CHECK:      %[[T2:.*]] = vector.insert %[[T1]], %[[C1]] [0] : vector<2xf32> into vector<3x2xf32>
-// CHECK:      %[[T3:.*]] = vector.insert %[[T1]], %[[T2]] [1] : vector<2xf32> into vector<3x2xf32>
-// CHECK:      %[[T4:.*]] = vector.insert %[[T1]], %[[T3]] [2] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[T0:.*]] = vector.extract %[[A]][0, 0] : vector<4x1x2xf32>
+// CHECK:      %[[T2:.*]] = vector.insert %[[T0]], %[[C1]] [0] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[T3:.*]] = vector.insert %[[T0]], %[[T2]] [1] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[T4:.*]] = vector.insert %[[T0]], %[[T3]] [2] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T5:.*]] = vector.insert %[[T4]], %[[C0]] [0] : vector<3x2xf32> into vector<4x3x2xf32>
-// CHECK:      %[[T6:.*]] = vector.extract %[[A]][1] : vector<4x1x2xf32>
-// CHECK:      %[[T7:.*]] = vector.extract %[[T6]][0] : vector<1x2xf32>
-// CHECK:      %[[T8:.*]] = vector.insert %[[T7]], %[[C1]] [0] : vector<2xf32> into vector<3x2xf32>
-// CHECK:      %[[T9:.*]] = vector.insert %[[T7]], %[[T8]] [1] : vector<2xf32> into vector<3x2xf32>
-// CHECK:      %[[T10:.*]] = vector.insert %[[T7]], %[[T9]] [2] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[T6:.*]] = vector.extract %[[A]][1, 0] : vector<4x1x2xf32>
+// CHECK:      %[[T8:.*]] = vector.insert %[[T6]], %[[C1]] [0] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[T9:.*]] = vector.insert %[[T6]], %[[T8]] [1] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[T10:.*]] = vector.insert %[[T6]], %[[T9]] [2] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T11:.*]] = vector.insert %[[T10]], %[[T5]] [1] : vector<3x2xf32> into vector<4x3x2xf32>
-// CHECK:      %[[T12:.*]] = vector.extract %[[A]][2] : vector<4x1x2xf32>
-// CHECK:      %[[T13:.*]] = vector.extract %[[T12]][0] : vector<1x2xf32>
-// CHECK:      %[[T14:.*]] = vector.insert %[[T13]], %[[C1]] [0] : vector<2xf32> into vector<3x2xf32>
-// CHECK:      %[[T15:.*]] = vector.insert %[[T13]], %[[T14]] [1] : vector<2xf32> into vector<3x2xf32>
-// CHECK:      %[[T16:.*]] = vector.insert %[[T13]], %[[T15]] [2] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[T12:.*]] = vector.extract %[[A]][2, 0] : vector<4x1x2xf32>
+// CHECK:      %[[T14:.*]] = vector.insert %[[T12]], %[[C1]] [0] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[T15:.*]] = vector.insert %[[T12]], %[[T14]] [1] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[T16:.*]] = vector.insert %[[T12]], %[[T15]] [2] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T17:.*]] = vector.insert %[[T16]], %[[T11]] [2] : vector<3x2xf32> into vector<4x3x2xf32>
-// CHECK:      %[[T18:.*]] = vector.extract %[[A]][3] : vector<4x1x2xf32>
-// CHECK:      %[[T19:.*]] = vector.extract %[[T18]][0] : vector<1x2xf32>
-// CHECK:      %[[T20:.*]] = vector.insert %[[T19]], %[[C1]] [0] : vector<2xf32> into vector<3x2xf32>
-// CHECK:      %[[T21:.*]] = vector.insert %[[T19]], %[[T20]] [1] : vector<2xf32> into vector<3x2xf32>
-// CHECK:      %[[T22:.*]] = vector.insert %[[T19]], %[[T21]] [2] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[T18:.*]] = vector.extract %[[A]][3, 0] : vector<4x1x2xf32>
+// CHECK:      %[[T20:.*]] = vector.insert %[[T18]], %[[C1]] [0] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[T21:.*]] = vector.insert %[[T18]], %[[T20]] [1] : vector<2xf32> into vector<3x2xf32>
+// CHECK:      %[[T22:.*]] = vector.insert %[[T18]], %[[T21]] [2] : vector<2xf32> into vector<3x2xf32>
 // CHECK:      %[[T23:.*]] = vector.insert %[[T22]], %[[T17]] [3] : vector<3x2xf32> into vector<4x3x2xf32>
 // CHECK:      return %[[T23]] : vector<4x3x2xf32>
 
@@ -710,18 +778,12 @@ func @genbool_3d() -> vector<2x3x4xi1> {
 }
 
 // CHECK-LABEL: func @genbool_var_1d
-// CHECK-SAME: %[[A:.*0]]: index
-// CHECK-DAG:  %[[VF:.*]] = constant dense<false> : vector<3xi1>
-// CHECK-DAG:  %[[C0:.*]] = constant 0 : index
-// CHECK-DAG:  %[[C1:.*]] = constant 1 : index
-// CHECK-DAG:  %[[C2:.*]] = constant 2 : index
-// CHECK:      %[[T0:.*]] = cmpi "slt", %[[C0]], %[[A]] : index
-// CHECK:      %[[T1:.*]] = vector.insert %[[T0]], %[[VF]] [0] : i1 into vector<3xi1>
-// CHECK:      %[[T2:.*]] = cmpi "slt", %[[C1]], %[[A]] : index
-// CHECK:      %[[T3:.*]] = vector.insert %[[T2]], %[[T1]] [1] : i1 into vector<3xi1>
-// CHECK:      %[[T4:.*]] = cmpi "slt", %[[C2]], %[[A]] : index
-// CHECK:      %[[T5:.*]] = vector.insert %[[T4]], %[[T3]] [2] : i1 into vector<3xi1>
-// CHECK:      return %[[T5]] : vector<3xi1>
+// CHECK-SAME: %[[A:.*]]: index
+// CHECK:      %[[C1:.*]] = constant dense<[0, 1, 2]> : vector<3xi64>
+// CHECK:      %[[T0:.*]] = index_cast %[[A]] : index to i64
+// CHECK:      %[[T1:.*]] = splat %[[T0]] : vector<3xi64>
+// CHECK:      %[[T2:.*]] = cmpi "slt", %[[C1]], %[[T1]] : vector<3xi64>
+// CHECK:      return %[[T2]] : vector<3xi1>
 
 func @genbool_var_1d(%arg0: index) -> vector<3xi1> {
   %0 = vector.create_mask %arg0 : vector<3xi1>
@@ -731,24 +793,21 @@ func @genbool_var_1d(%arg0: index) -> vector<3xi1> {
 // CHECK-LABEL: func @genbool_var_2d
 // CHECK-SAME: %[[A:.*0]]: index
 // CHECK-SAME: %[[B:.*1]]: index
-// CHECK-DAG:  %[[Z1:.*]] = constant dense<false> : vector<3xi1>
-// CHECK-DAG:  %[[Z2:.*]] = constant dense<false> : vector<2x3xi1>
-// CHECK-DAG:  %[[C0:.*]] = constant 0 : index
-// CHECK-DAG:  %[[C1:.*]] = constant 1 : index
-// CHECK-DAG:  %[[C2:.*]] = constant 2 : index
-// CHECK:      %[[T0:.*]] = cmpi "slt", %[[C0]], %[[B]] : index
-// CHECK:      %[[T1:.*]] = vector.insert %[[T0]], %[[Z1]] [0] : i1 into vector<3xi1>
-// CHECK:      %[[T2:.*]] = cmpi "slt", %[[C1]], %[[B]] : index
-// CHECK:      %[[T3:.*]] = vector.insert %[[T2]], %[[T1]] [1] : i1 into vector<3xi1>
-// CHECK:      %[[T4:.*]] = cmpi "slt", %[[C2]], %[[B]] : index
-// CHECK:      %[[T5:.*]] = vector.insert %[[T4]], %[[T3]] [2] : i1 into vector<3xi1>
-// CHECK:      %[[T6:.*]] = cmpi "slt", %[[C0]], %[[A]] : index
-// CHECK:      %[[T7:.*]] = select %[[T6]], %[[T5]], %[[Z1]] : vector<3xi1>
-// CHECK:      %[[T8:.*]] = vector.insert %7, %[[Z2]] [0] : vector<3xi1> into vector<2x3xi1>
-// CHECK:      %[[T9:.*]] = cmpi "slt", %[[C1]], %[[A]] : index
-// CHECK:      %[[T10:.*]] = select %[[T9]], %[[T5]], %[[Z1]] : vector<3xi1>
-// CHECK:      %[[T11:.*]] = vector.insert %[[T10]], %[[T8]] [1] : vector<3xi1> into vector<2x3xi1>
-// CHECK:      return %[[T11]] : vector<2x3xi1>
+// CHECK:      %[[CI:.*]] = constant dense<[0, 1, 2]> : vector<3xi64>
+// CHECK:      %[[CF:.*]] = constant dense<false> : vector<3xi1>
+// CHECK:      %[[C2:.*]] = constant dense<false> : vector<2x3xi1>
+// CHECK:      %[[c0:.*]] = constant 0 : index
+// CHECK:      %[[c1:.*]] = constant 1 : index
+// CHECK:      %[[T0:.*]] = index_cast %[[B]] : index to i64
+// CHECK:      %[[T1:.*]] = splat %[[T0]] : vector<3xi64>
+// CHECK:      %[[T2:.*]] = cmpi "slt", %[[CI]], %[[T1]] : vector<3xi64>
+// CHECK:      %[[T3:.*]] = cmpi "slt", %[[c0]], %[[A]] : index
+// CHECK:      %[[T4:.*]] = select %[[T3]], %[[T2]], %[[CF]] : vector<3xi1>
+// CHECK:      %[[T5:.*]] = vector.insert %[[T4]], %[[C2]] [0] : vector<3xi1> into vector<2x3xi1>
+// CHECK:      %[[T6:.*]] = cmpi "slt", %[[c1]], %[[A]] : index
+// CHECK:      %[[T7:.*]] = select %[[T6]], %[[T2]], %[[CF]] : vector<3xi1>
+// CHECK:      %[[T8:.*]] = vector.insert %[[T7]], %[[T5]] [1] : vector<3xi1> into vector<2x3xi1>
+// CHECK:      return %[[T8]] : vector<2x3xi1>
 
 func @genbool_var_2d(%arg0: index, %arg1: index) -> vector<2x3xi1> {
   %0 = vector.create_mask %arg0, %arg1 : vector<2x3xi1>
diff --git a/mlir/test/Dialect/Vector/vector-flat-transforms.mlir b/mlir/test/Dialect/Vector/vector-flat-transforms.mlir
index 6a1e6ee85a7d4..c07d651d985ed 100644
--- a/mlir/test/Dialect/Vector/vector-flat-transforms.mlir
+++ b/mlir/test/Dialect/Vector/vector-flat-transforms.mlir
@@ -2,10 +2,9 @@
 
 // Tests for lowering 2-D vector.transpose into vector.flat_transpose.
 //
-// TODO(ajcbik,ntv): having ShapeCastOp2DDownCastRewritePattern and
-//                   ShapeCastOp2DUpCastRewritePattern too early in
-//                   the greedy rewriting patterns misses opportunities
-//                   to fold shape casts!
+// TODO: having ShapeCastOp2DDownCastRewritePattern and
+//       ShapeCastOp2DUpCastRewritePattern too early in the greedy rewriting
+//       patterns misses opportunities to fold shape casts!
 
 // No shape cast folding expected.
 //
diff --git a/mlir/test/Dialect/Vector/vector-transforms.mlir b/mlir/test/Dialect/Vector/vector-transforms.mlir
index 8de153adf7310..2346f0d243185 100644
--- a/mlir/test/Dialect/Vector/vector-transforms.mlir
+++ b/mlir/test/Dialect/Vector/vector-transforms.mlir
@@ -1,4 +1,5 @@
 // RUN: mlir-opt %s -test-vector-to-vector-conversion | FileCheck %s
+// RUN: mlir-opt %s -test-vector-unrolling-patterns | FileCheck %s
 
 // CHECK-DAG: #[[MAP0:map[0-9]+]] = affine_map<(d0, d1) -> (d0, d1)>
 // CHECK-DAG: #[[MAP1:map[0-9]+]] = affine_map<(d0, d1, d2) -> (d1, d2)>
@@ -247,10 +248,10 @@ func @contraction4x4_ikj(%arg0 : vector<4x2xf32>, %arg1 : vector<2x4xf32>,
 // CHECK-NEXT: %[[R2:.*]] = vector.contract {indexing_maps = [#map2, #map3, #map0], iterator_types = ["parallel", "reduction", "parallel"]} %[[VTR1]], %[[VTR2]], %[[VTR6]] : vector<2x2xf32>, vector<2x2xf32> into vector<2x2xf32>
 // CHECK-NEXT: %[[R3:.*]] = vector.contract {indexing_maps = [#map2, #map3, #map0], iterator_types = ["parallel", "reduction", "parallel"]} %[[VTR1]], %[[VTR3]], %[[VTR7]] : vector<2x2xf32>, vector<2x2xf32> into vector<2x2xf32>
 
-// CHECK-NEXT: vector.transfer_write %[[R0]], %{{.*}}[%[[C0]], %[[C0]]] : vector<2x2xf32>, memref<4x4xf32>
-// CHECK-NEXT: vector.transfer_write %[[R1]], %{{.*}}[%[[C0]], %[[C2]]] : vector<2x2xf32>, memref<4x4xf32>
-// CHECK-NEXT: vector.transfer_write %[[R2]], %{{.*}}[%[[C2]], %[[C0]]] : vector<2x2xf32>, memref<4x4xf32>
-// CHECK-NEXT: vector.transfer_write %[[R3]], %{{.*}}[%[[C2]], %[[C2]]] : vector<2x2xf32>, memref<4x4xf32>
+// CHECK-NEXT: vector.transfer_write %[[R0]], %{{.*}}[%[[C0]], %[[C0]]] {masked = [false, false]} : vector<2x2xf32>, memref<4x4xf32>
+// CHECK-NEXT: vector.transfer_write %[[R1]], %{{.*}}[%[[C0]], %[[C2]]] {masked = [false, false]} : vector<2x2xf32>, memref<4x4xf32>
+// CHECK-NEXT: vector.transfer_write %[[R2]], %{{.*}}[%[[C2]], %[[C0]]] {masked = [false, false]} : vector<2x2xf32>, memref<4x4xf32>
+// CHECK-NEXT: vector.transfer_write %[[R3]], %{{.*}}[%[[C2]], %[[C2]]] {masked = [false, false]} : vector<2x2xf32>, memref<4x4xf32>
 // CHECK-NEXT: return
 
 func @contraction4x4_ikj_xfer_read(%arg0 : memref<4x2xf32>,
@@ -280,7 +281,7 @@ func @contraction4x4_ikj_xfer_read(%arg0 : memref<4x2xf32>,
   return
 }
 
-// TODO(andydavis) Update test with VTR split transform.
+// TODO: Update test with VTR split transform.
 // CHECK-LABEL: func @vector_transfers
 // CHECK-COUNT-8: vector.transfer_read
 // CHECK-COUNT-4: addf
@@ -295,8 +296,8 @@ func @vector_transfers(%arg0: index, %arg1: index) {
   %cst_1 = constant 2.000000e+00 : f32
   affine.for %arg2 = 0 to %arg0 step 4 {
     affine.for %arg3 = 0 to %arg1 step 4 {
-      %4 = vector.transfer_read %0[%arg2, %arg3], %cst  {permutation_map = affine_map<(d0, d1) -> (d0, d1)>} : memref<?x?xf32>, vector<4x4xf32>
-      %5 = vector.transfer_read %1[%arg2, %arg3], %cst  {permutation_map = affine_map<(d0, d1) -> (d0, d1)>} : memref<?x?xf32>, vector<4x4xf32>
+      %4 = vector.transfer_read %0[%arg2, %arg3], %cst {permutation_map = affine_map<(d0, d1) -> (d0, d1)>} : memref<?x?xf32>, vector<4x4xf32>
+      %5 = vector.transfer_read %1[%arg2, %arg3], %cst {permutation_map = affine_map<(d0, d1) -> (d0, d1)>} : memref<?x?xf32>, vector<4x4xf32>
       %6 = addf %4, %5 : vector<4x4xf32>
       vector.transfer_write %6, %2[%arg2, %arg3] {permutation_map = affine_map<(d0, d1) -> (d0, d1)>} : vector<4x4xf32>, memref<?x?xf32>
     }
@@ -425,10 +426,10 @@ func @cancelling_shape_cast_ops(%arg0 : vector<2x4xf32>) -> vector<2x4xf32> {
 // CHECK-LABEL: func @vector_transfers_vector_element_type
 //      CHECK: %[[C0:.*]] = constant 0 : index
 //      CHECK: %[[C1:.*]] = constant 1 : index
-//      CHECK: %[[VTR0:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]], %[[C0]]], %{{.*}} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32>
-// CHECK-NEXT: %[[VTR1:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C1]], %[[C0]]], %{{.*}} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32>
-// CHECK-NEXT: vector.transfer_write %[[VTR0]], %{{.*}}[%[[C0]], %[[C0]], %[[C0]]] : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>>
-// CHECK-NEXT: vector.transfer_write %[[VTR1]], %{{.*}}[%[[C0]], %[[C1]], %[[C0]]] : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>>
+//      CHECK: %[[VTR0:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]], %[[C0]]], %{{.*}} {masked = [false, false]} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32>
+// CHECK-NEXT: %[[VTR1:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C1]], %[[C0]]], %{{.*}} {masked = [false, false]} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32>
+// CHECK-NEXT: vector.transfer_write %[[VTR0]], %{{.*}}[%[[C0]], %[[C0]], %[[C0]]] {masked = [false, false]} : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>>
+// CHECK-NEXT: vector.transfer_write %[[VTR1]], %{{.*}}[%[[C0]], %[[C1]], %[[C0]]] {masked = [false, false]} : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>>
 
 func @vector_transfers_vector_element_type() {
   %c0 = constant 0 : index
diff --git a/mlir/test/EDSC/builder-api-test.cpp b/mlir/test/EDSC/builder-api-test.cpp
index 3eecf7e09eca0..73f7adeeaf711 100644
--- a/mlir/test/EDSC/builder-api-test.cpp
+++ b/mlir/test/EDSC/builder-api-test.cpp
@@ -419,11 +419,49 @@ TEST_FUNC(operator_and) {
   f.erase();
 }
 
+TEST_FUNC(divis_op_i32) {
+  using namespace edsc::op;
+  auto f = makeFunction("divis_op", {}, {});
+
+  OpBuilder builder(f.getBody());
+  ScopedContext scope(builder, f.getLoc());
+  auto i32Type = builder.getI32Type();
+  std_divis(std_constant_int(10, i32Type), std_constant_int(2, i32Type));
+
+  // clang-format off
+  // CHECK-LABEL: @divis_op
+  //   CHECK-DAG:     {{.*}} = constant 10
+  //   CHECK-DAG:     {{.*}} = constant 2
+  //  CHECK-NEXT:     {{.*}} = divi_signed
+  // clang-format on
+  f.print(llvm::outs());
+  f.erase();
+}
+
+TEST_FUNC(diviu_op_i32) {
+  using namespace edsc::op;
+  auto f = makeFunction("diviu_op", {}, {});
+
+  OpBuilder builder(f.getBody());
+  ScopedContext scope(builder, f.getLoc());
+  auto i32Type = builder.getI32Type();
+  std_diviu(std_constant_int(10, i32Type), std_constant_int(2, i32Type));
+
+  // clang-format off
+  // CHECK-LABEL: @diviu_op
+  //   CHECK-DAG:     {{.*}} = constant 10
+  //   CHECK-DAG:     {{.*}} = constant 2
+  //  CHECK-NEXT:     {{.*}} = divi_unsigned
+  // clang-format on
+  f.print(llvm::outs());
+  f.erase();
+}
+
 TEST_FUNC(select_op_i32) {
   using namespace edsc::op;
-  auto f32Type = FloatType::getF32(&globalContext());
+  auto i32Type = IntegerType::get(32, &globalContext());
   auto memrefType = MemRefType::get(
-      {ShapedType::kDynamicSize, ShapedType::kDynamicSize}, f32Type, {}, 0);
+      {ShapedType::kDynamicSize, ShapedType::kDynamicSize}, i32Type, {}, 0);
   auto f = makeFunction("select_op", {}, {memrefType});
 
   OpBuilder builder(f.getBody());
@@ -432,7 +470,18 @@ TEST_FUNC(select_op_i32) {
   MemRefBoundsCapture vA(f.getArgument(0));
   AffineIndexedValue A(f.getArgument(0));
   affineLoopNestBuilder({zero, zero}, {one, one}, {1, 1}, [&](ValueRange ivs) {
-    std_select(eq(ivs[0], zero), A(zero, zero), A(ivs[0], ivs[1]));
+    using namespace edsc::op;
+    Value i = ivs[0], j = ivs[1];
+    std_select(eq(i, zero), A(zero, zero), A(i, j));
+    std_select(ne(i, zero), A(zero, zero), A(i, j));
+    std_select(slt(i, zero), A(zero, zero), A(i, j));
+    std_select(sle(i, zero), A(zero, zero), A(i, j));
+    std_select(sgt(i, zero), A(zero, zero), A(i, j));
+    std_select(sge(i, zero), A(zero, zero), A(i, j));
+    std_select(ult(i, zero), A(zero, zero), A(i, j));
+    std_select(ule(i, zero), A(zero, zero), A(i, j));
+    std_select(ugt(i, zero), A(zero, zero), A(i, j));
+    std_select(uge(i, zero), A(zero, zero), A(i, j));
   });
 
   // clang-format off
@@ -443,6 +492,42 @@ TEST_FUNC(select_op_i32) {
   //  CHECK-DAG:     {{.*}} = affine.load
   //  CHECK-DAG:     {{.*}} = affine.load
   // CHECK-NEXT:     {{.*}} = select
+  //  CHECK-DAG:     {{.*}} = cmpi "ne"
+  //  CHECK-DAG:     {{.*}} = affine.load
+  //  CHECK-DAG:     {{.*}} = affine.load
+  // CHECK-NEXT:     {{.*}} = select
+  //  CHECK-DAG:     {{.*}} = cmpi "slt"
+  //  CHECK-DAG:     {{.*}} = affine.load
+  //  CHECK-DAG:     {{.*}} = affine.load
+  // CHECK-NEXT:     {{.*}} = select
+  //  CHECK-DAG:     {{.*}} = cmpi "sle"
+  //  CHECK-DAG:     {{.*}} = affine.load
+  //  CHECK-DAG:     {{.*}} = affine.load
+  // CHECK-NEXT:     {{.*}} = select
+  //  CHECK-DAG:     {{.*}} = cmpi "sgt"
+  //  CHECK-DAG:     {{.*}} = affine.load
+  //  CHECK-DAG:     {{.*}} = affine.load
+  // CHECK-NEXT:     {{.*}} = select
+  //  CHECK-DAG:     {{.*}} = cmpi "sge"
+  //  CHECK-DAG:     {{.*}} = affine.load
+  //  CHECK-DAG:     {{.*}} = affine.load
+  // CHECK-NEXT:     {{.*}} = select
+  //  CHECK-DAG:     {{.*}} = cmpi "ult"
+  //  CHECK-DAG:     {{.*}} = affine.load
+  //  CHECK-DAG:     {{.*}} = affine.load
+  // CHECK-NEXT:     {{.*}} = select
+  //  CHECK-DAG:     {{.*}} = cmpi "ule"
+  //  CHECK-DAG:     {{.*}} = affine.load
+  //  CHECK-DAG:     {{.*}} = affine.load
+  // CHECK-NEXT:     {{.*}} = select
+  //  CHECK-DAG:     {{.*}} = cmpi "ugt"
+  //  CHECK-DAG:     {{.*}} = affine.load
+  //  CHECK-DAG:     {{.*}} = affine.load
+  // CHECK-NEXT:     {{.*}} = select
+  //  CHECK-DAG:     {{.*}} = cmpi "uge"
+  //  CHECK-DAG:     {{.*}} = affine.load
+  //  CHECK-DAG:     {{.*}} = affine.load
+  // CHECK-NEXT:     {{.*}} = select
   // clang-format on
   f.print(llvm::outs());
   f.erase();
@@ -465,10 +550,14 @@ TEST_FUNC(select_op_f32) {
     Value i = ivs[0], j = ivs[1];
     std_select(eq(B(i, j), B(i + one, j)), A(zero, zero), A(i, j));
     std_select(ne(B(i, j), B(i + one, j)), A(zero, zero), A(i, j));
-    std_select(B(i, j) >= B(i + one, j), A(zero, zero), A(i, j));
-    std_select(B(i, j) <= B(i + one, j), A(zero, zero), A(i, j));
-    std_select(B(i, j) < B(i + one, j), A(zero, zero), A(i, j));
-    std_select(B(i, j) > B(i + one, j), A(zero, zero), A(i, j));
+    std_select(sge(B(i, j), B(i + one, j)), A(zero, zero), A(i, j));
+    std_select(sle(B(i, j), B(i + one, j)), A(zero, zero), A(i, j));
+    std_select(slt(B(i, j), B(i + one, j)), A(zero, zero), A(i, j));
+    std_select(sgt(B(i, j), B(i + one, j)), A(zero, zero), A(i, j));
+    std_select(uge(B(i, j), B(i + one, j)), A(zero, zero), A(i, j));
+    std_select(ule(B(i, j), B(i + one, j)), A(zero, zero), A(i, j));
+    std_select(ult(B(i, j), B(i + one, j)), A(zero, zero), A(i, j));
+    std_select(ugt(B(i, j), B(i + one, j)), A(zero, zero), A(i, j));
   });
 
   // CHECK-LABEL: @select_op
@@ -516,6 +605,34 @@ TEST_FUNC(select_op_f32) {
   //  CHECK-DAG:     affine.load
   //  CHECK-DAG:     affine.apply
   // CHECK-NEXT:     select
+  //  CHECK-DAG:     cmpf "oge"
+  //  CHECK-DAG:     affine.load
+  //  CHECK-DAG:     affine.load
+  //  CHECK-DAG:     affine.load
+  //  CHECK-DAG:     affine.load
+  //  CHECK-DAG:     affine.apply
+  // CHECK-NEXT:     select
+  //  CHECK-DAG:     cmpf "ole"
+  //  CHECK-DAG:     affine.load
+  //  CHECK-DAG:     affine.load
+  //  CHECK-DAG:     affine.load
+  //  CHECK-DAG:     affine.load
+  //  CHECK-DAG:     affine.apply
+  // CHECK-NEXT:     select
+  //  CHECK-DAG:     cmpf "olt"
+  //  CHECK-DAG:     affine.load
+  //  CHECK-DAG:     affine.load
+  //  CHECK-DAG:     affine.load
+  //  CHECK-DAG:     affine.load
+  //  CHECK-DAG:     affine.apply
+  // CHECK-NEXT:     select
+  //  CHECK-DAG:     cmpf "ogt"
+  //  CHECK-DAG:     affine.load
+  //  CHECK-DAG:     affine.load
+  //  CHECK-DAG:     affine.load
+  //  CHECK-DAG:     affine.load
+  //  CHECK-DAG:     affine.apply
+  // CHECK-NEXT:     select
   // clang-format on
   f.print(llvm::outs());
   f.erase();
@@ -924,6 +1041,28 @@ TEST_FUNC(linalg_tensors_test) {
   f.erase();
 }
 
+TEST_FUNC(vector_extractelement_op_i32) {
+  using namespace edsc::op;
+  auto f = makeFunction("vector_extractelement_op", {}, {});
+
+  OpBuilder builder(f.getBody());
+  ScopedContext scope(builder, f.getLoc());
+  auto i32Type = builder.getI32Type();
+  auto vectorType = VectorType::get(/*shape=*/{8}, i32Type);
+  vector_extractelement(
+      i32Type, std_constant(vectorType, builder.getI32VectorAttr({10})),
+      std_constant_int(0, i32Type));
+
+  // clang-format off
+  // CHECK-LABEL: @vector_extractelement_op
+  //   CHECK-DAG:     {{.*}} = constant dense<10>
+  //   CHECK-DAG:     {{.*}} = constant 0
+  //  CHECK-NEXT:     {{.*}} = vector.extractelement
+  // clang-format on
+  f.print(llvm::outs());
+  f.erase();
+}
+
 // CHECK-LABEL: func @memref_vector_matmul_test(
 //  CHECK-SAME:   %[[A:.*]]: memref<?x?xvector<4x16xf32>>,
 //  CHECK-SAME:   %[[B:.*]]: memref<?x?xvector<16x8xf32>>,
diff --git a/mlir/test/IR/invalid-affinemap.mlir b/mlir/test/IR/invalid-affinemap.mlir
index 741ad9a3dfdf3..9377824f006a5 100644
--- a/mlir/test/IR/invalid-affinemap.mlir
+++ b/mlir/test/IR/invalid-affinemap.mlir
@@ -96,7 +96,7 @@
 // -----
 #hello_world = affine_map<(i, j) -> (i, 3*d0 + )> // expected-error {{use of undeclared identifier}}
 
-// TODO(bondhugula): Add more tests; coverage of error messages emitted not complete
+// TODO: Add more tests; coverage of error messages emitted not complete
 
 // -----
 #ABC = affine_map<(i,j) -> (i+j)>
diff --git a/mlir/test/IR/invalid-module-op.mlir b/mlir/test/IR/invalid-module-op.mlir
index f010ceca12f6d..73fe188209d15 100644
--- a/mlir/test/IR/invalid-module-op.mlir
+++ b/mlir/test/IR/invalid-module-op.mlir
@@ -16,7 +16,7 @@ func @module_op() {
 // -----
 
 func @module_op() {
-  // expected-error@+1 {{expected body to have no arguments}}
+  // expected-error@+1 {{region should have no arguments}}
   module {
   ^bb1(%arg: i32):
     "module_terminator"() : () -> ()
diff --git a/mlir/test/IR/invalid.mlir b/mlir/test/IR/invalid.mlir
index 9de62df2883a5..0373308ad909d 100644
--- a/mlir/test/IR/invalid.mlir
+++ b/mlir/test/IR/invalid.mlir
@@ -290,7 +290,7 @@ func @invalid_if_conditional6() {
 }
 
 // -----
-// TODO (support affine.if (1)?
+// TODO: support affine.if (1)?
 func @invalid_if_conditional7() {
   affine.for %i = 1 to 10 {
     affine.if affine_set<(i) : (1)> // expected-error {{expected '== 0' or '>= 0' at end of affine constraint}}
@@ -513,7 +513,7 @@ func @foo() {
 
 func @undefined_function() {
 ^bb0:
-  %x = constant @bar : (i32) -> ()  // expected-error {{reference to undefined function 'bar'}}
+  %x = constant @qux : (i32) -> ()  // expected-error {{reference to undefined function 'qux'}}
   return
 }
 
diff --git a/mlir/test/IR/parser.mlir b/mlir/test/IR/parser.mlir
index f3c01a9363c70..1a59b53cf1000 100644
--- a/mlir/test/IR/parser.mlir
+++ b/mlir/test/IR/parser.mlir
@@ -670,19 +670,21 @@ func @densetensorattr() -> () {
 // CHECK: "fooi67"() {bar = dense<{{\[\[\[}}-5, 4, 6, 2]]]> : vector<1x1x4xi67>} : () -> ()
   "fooi67"(){bar = dense<[[[-5, 4, 6, 2]]]> : vector<1x1x4xi67>} : () -> ()
 
-// CHECK: "foo2"() {bar = dense<[]> : tensor<0xi32>} : () -> ()
-  "foo2"(){bar = dense<[]> : tensor<0xi32>} : () -> ()
-// CHECK: "foo2"() {bar = dense<{{\[\[}}]]> : tensor<1x0xi32>} : () -> ()
-  "foo2"(){bar = dense<[[]]> : tensor<1x0xi32>} : () -> ()
+// CHECK: "foo2"() {bar = dense<> : tensor<0xi32>} : () -> ()
+  "foo2"(){bar = dense<> : tensor<0xi32>} : () -> ()
+// CHECK: "foo2"() {bar = dense<> : tensor<1x0xi32>} : () -> ()
+  "foo2"(){bar = dense<> : tensor<1x0xi32>} : () -> ()
+// CHECK: dense<> : tensor<0x512x512xi32>
+  "foo2"(){bar = dense<> : tensor<0x512x512xi32>} : () -> ()
 // CHECK: "foo3"() {bar = dense<{{\[\[\[}}5, -6, 1, 2]], {{\[\[}}7, 8, 3, 4]]]> : tensor<2x1x4xi32>} : () -> ()
   "foo3"(){bar = dense<[[[5, -6, 1, 2]], [[7, 8, 3, 4]]]> : tensor<2x1x4xi32>} : () -> ()
 
 // CHECK: "float1"() {bar = dense<5.000000e+00> : tensor<1x1x1xf32>} : () -> ()
   "float1"(){bar = dense<[[[5.0]]]> : tensor<1x1x1xf32>} : () -> ()
-// CHECK: "float2"() {bar = dense<[]> : tensor<0xf32>} : () -> ()
-  "float2"(){bar = dense<[]> : tensor<0xf32>} : () -> ()
-// CHECK: "float2"() {bar = dense<{{\[\[}}]]> : tensor<1x0xf32>} : () -> ()
-  "float2"(){bar = dense<[[]]> : tensor<1x0xf32>} : () -> ()
+// CHECK: "float2"() {bar = dense<> : tensor<0xf32>} : () -> ()
+  "float2"(){bar = dense<> : tensor<0xf32>} : () -> ()
+// CHECK: "float2"() {bar = dense<> : tensor<1x0xf32>} : () -> ()
+  "float2"(){bar = dense<> : tensor<1x0xf32>} : () -> ()
 
 // CHECK: "bfloat16"() {bar = dense<{{\[\[\[}}-5.000000e+00, 6.000000e+00, 1.000000e+00, 2.000000e+00]], {{\[\[}}7.000000e+00, -8.000000e+00, 3.000000e+00, 4.000000e+00]]]> : tensor<2x1x4xbf16>} : () -> ()
   "bfloat16"(){bar = dense<[[[-5.0, 6.0, 1.0, 2.0]], [[7.0, -8.0, 3.0, 4.0]]]> : tensor<2x1x4xbf16>} : () -> ()
@@ -752,27 +754,27 @@ func @sparsetensorattr() -> () {
   "fooi8"(){bar = sparse<0, -2> : tensor<1x1x1xi8>} : () -> ()
 // CHECK: "fooi16"() {bar = sparse<{{\[\[}}1, 1, 0], {{\[}}0, 1, 0], {{\[}}0, 0, 1]], {{\[}}2, -1, 5]> : tensor<2x2x2xi16>} : () -> ()
   "fooi16"(){bar = sparse<[[1, 1, 0], [0, 1, 0], [0, 0, 1]], [2, -1, 5]> : tensor<2x2x2xi16>} : () -> ()
-// CHECK: "fooi32"() {bar = sparse<{{\[}}], {{\[}}]> : tensor<1x1xi32>} : () -> ()
-  "fooi32"(){bar = sparse<[], []> : tensor<1x1xi32>} : () -> ()
+// CHECK: "fooi32"() {bar = sparse<> : tensor<1x1xi32>} : () -> ()
+  "fooi32"(){bar = sparse<> : tensor<1x1xi32>} : () -> ()
 // CHECK: "fooi64"() {bar = sparse<0, -1> : tensor<1xi64>} : () -> ()
   "fooi64"(){bar = sparse<[[0]], [-1]> : tensor<1xi64>} : () -> ()
-// CHECK: "foo2"() {bar = sparse<{{\[}}], {{\[}}]> : tensor<0xi32>} : () -> ()
-  "foo2"(){bar = sparse<[], []> : tensor<0xi32>} : () -> ()
-// CHECK: "foo3"() {bar = sparse<{{\[}}], {{\[}}]> : tensor<i32>} : () -> ()
-  "foo3"(){bar = sparse<[], []> : tensor<i32>} : () -> ()
+// CHECK: "foo2"() {bar = sparse<> : tensor<0xi32>} : () -> ()
+  "foo2"(){bar = sparse<> : tensor<0xi32>} : () -> ()
+// CHECK: "foo3"() {bar = sparse<> : tensor<i32>} : () -> ()
+  "foo3"(){bar = sparse<> : tensor<i32>} : () -> ()
 
 // CHECK: "foof16"() {bar = sparse<0, -2.000000e+00> : tensor<1x1x1xf16>} : () -> ()
   "foof16"(){bar = sparse<0, -2.0> : tensor<1x1x1xf16>} : () -> ()
 // CHECK: "foobf16"() {bar = sparse<{{\[\[}}1, 1, 0], {{\[}}0, 1, 0], {{\[}}0, 0, 1]], {{\[}}2.000000e+00, -1.000000e+00, 5.000000e+00]> : tensor<2x2x2xbf16>} : () -> ()
   "foobf16"(){bar = sparse<[[1, 1, 0], [0, 1, 0], [0, 0, 1]], [2.0, -1.0, 5.0]> : tensor<2x2x2xbf16>} : () -> ()
-// CHECK: "foof32"() {bar = sparse<{{\[}}], {{\[}}]> : tensor<1x0x1xf32>} : () -> ()
-  "foof32"(){bar = sparse<[], []> : tensor<1x0x1xf32>} : () -> ()
+// CHECK: "foof32"() {bar = sparse<> : tensor<1x0x1xf32>} : () -> ()
+  "foof32"(){bar = sparse<> : tensor<1x0x1xf32>} : () -> ()
 // CHECK:  "foof64"() {bar = sparse<0, -1.000000e+00> : tensor<1xf64>} : () -> ()
   "foof64"(){bar = sparse<[[0]], [-1.0]> : tensor<1xf64>} : () -> ()
-// CHECK: "foof320"() {bar = sparse<{{\[}}], {{\[}}]> : tensor<0xf32>} : () -> ()
-  "foof320"(){bar = sparse<[], []> : tensor<0xf32>} : () -> ()
-// CHECK: "foof321"() {bar = sparse<{{\[}}], {{\[}}]> : tensor<f32>} : () -> ()
-  "foof321"(){bar = sparse<[], []> : tensor<f32>} : () -> ()
+// CHECK: "foof320"() {bar = sparse<> : tensor<0xf32>} : () -> ()
+  "foof320"(){bar = sparse<> : tensor<0xf32>} : () -> ()
+// CHECK: "foof321"() {bar = sparse<> : tensor<f32>} : () -> ()
+  "foof321"(){bar = sparse<> : tensor<f32>} : () -> ()
 
 // CHECK: "foostr"() {bar = sparse<0, "foo"> : tensor<1x1x1x!unknown<"">>} : () -> ()
   "foostr"(){bar = sparse<0, "foo"> : tensor<1x1x1x!unknown<"">>} : () -> ()
@@ -789,8 +791,8 @@ func @sparsevectorattr() -> () {
   "fooi8"(){bar = sparse<0, -2> : vector<1x1x1xi8>} : () -> ()
 // CHECK: "fooi16"() {bar = sparse<{{\[\[}}1, 1, 0], {{\[}}0, 1, 0], {{\[}}0, 0, 1]], {{\[}}2, -1, 5]> : vector<2x2x2xi16>} : () -> ()
   "fooi16"(){bar = sparse<[[1, 1, 0], [0, 1, 0], [0, 0, 1]], [2, -1, 5]> : vector<2x2x2xi16>} : () -> ()
-// CHECK: "fooi32"() {bar = sparse<{{\[}}], {{\[}}]> : vector<1x1xi32>} : () -> ()
-  "fooi32"(){bar = sparse<[], []> : vector<1x1xi32>} : () -> ()
+// CHECK: "fooi32"() {bar = sparse<> : vector<1x1xi32>} : () -> ()
+  "fooi32"(){bar = sparse<> : vector<1x1xi32>} : () -> ()
 // CHECK: "fooi64"() {bar = sparse<0, -1> : vector<1xi64>} : () -> ()
   "fooi64"(){bar = sparse<[[0]], [-1]> : vector<1xi64>} : () -> ()
 
diff --git a/mlir/test/Target/import.ll b/mlir/test/Target/import.ll
index 76061322f952b..4f4e822eb78ef 100644
--- a/mlir/test/Target/import.ll
+++ b/mlir/test/Target/import.ll
@@ -234,7 +234,7 @@ define void @FPArithmetic(float %a, float %b, double %c, double %d) {
 ; CHECK-LABEL: @precaller
 define i32 @precaller() {
   %1 = alloca i32 ()*
-  ; CHECK: %[[func:.*]] = llvm.mlir.constant(@callee) : !llvm<"i32 ()*">
+  ; CHECK: %[[func:.*]] = llvm.mlir.addressof @callee : !llvm<"i32 ()*">
   ; CHECK: llvm.store %[[func]], %[[loc:.*]]
   store i32 ()* @callee, i32 ()** %1
   ; CHECK: %[[indir:.*]] = llvm.load %[[loc]]
@@ -252,7 +252,7 @@ define i32 @callee() {
 ; CHECK-LABEL: @postcaller
 define i32 @postcaller() {
   %1 = alloca i32 ()*
-  ; CHECK: %[[func:.*]] = llvm.mlir.constant(@callee) : !llvm<"i32 ()*">
+  ; CHECK: %[[func:.*]] = llvm.mlir.addressof @callee : !llvm<"i32 ()*">
   ; CHECK: llvm.store %[[func]], %[[loc:.*]]
   store i32 ()* @callee, i32 ()** %1
   ; CHECK: %[[indir:.*]] = llvm.load %[[loc]]
@@ -317,4 +317,4 @@ define i32 @useFenceInst() {
   ;CHECK: llvm.fence seq_cst
   fence syncscope("") seq_cst
   ret i32 0
-}
\ No newline at end of file
+}
diff --git a/mlir/test/Target/llvmir-intrinsics.mlir b/mlir/test/Target/llvmir-intrinsics.mlir
index 57d98aa25f362..ffbbf359b3801 100644
--- a/mlir/test/Target/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/llvmir-intrinsics.mlir
@@ -117,6 +117,24 @@ llvm.func @pow_test(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm<"<8 x f
   llvm.return
 }
 
+// CHECK-LABEL: @bitreverse_test
+llvm.func @bitreverse_test(%arg0: !llvm.i32, %arg1: !llvm<"<8 x i32>">) {
+  // CHECK: call i32 @llvm.bitreverse.i32
+  "llvm.intr.bitreverse"(%arg0) : (!llvm.i32) -> !llvm.i32
+  // CHECK: call <8 x i32> @llvm.bitreverse.v8i32
+  "llvm.intr.bitreverse"(%arg1) : (!llvm<"<8 x i32>">) -> !llvm<"<8 x i32>">
+  llvm.return
+}
+
+// CHECK-LABEL: @ctpop_test
+llvm.func @ctpop_test(%arg0: !llvm.i32, %arg1: !llvm<"<8 x i32>">) {
+  // CHECK: call i32 @llvm.ctpop.i32
+  "llvm.intr.ctpop"(%arg0) : (!llvm.i32) -> !llvm.i32
+  // CHECK: call <8 x i32> @llvm.ctpop.v8i32
+  "llvm.intr.ctpop"(%arg1) : (!llvm<"<8 x i32>">) -> !llvm<"<8 x i32>">
+  llvm.return
+}
+
 // CHECK-LABEL: @vector_reductions
 llvm.func @vector_reductions(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">, %arg2: !llvm<"<8 x i32>">) {
   // CHECK: call i32 @llvm.experimental.vector.reduce.add.v8i32
@@ -143,6 +161,10 @@ llvm.func @vector_reductions(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">, %a
   "llvm.intr.experimental.vector.reduce.v2.fadd"(%arg0, %arg1) : (!llvm.float, !llvm<"<8 x float>">) -> !llvm.float
   // CHECK: call float @llvm.experimental.vector.reduce.v2.fmul.f32.v8f32
   "llvm.intr.experimental.vector.reduce.v2.fmul"(%arg0, %arg1) : (!llvm.float, !llvm<"<8 x float>">) -> !llvm.float
+  // CHECK: call reassoc float @llvm.experimental.vector.reduce.v2.fadd.f32.v8f32
+  "llvm.intr.experimental.vector.reduce.v2.fadd"(%arg0, %arg1) {reassoc = true} : (!llvm.float, !llvm<"<8 x float>">) -> !llvm.float
+  // CHECK: call reassoc float @llvm.experimental.vector.reduce.v2.fmul.f32.v8f32
+  "llvm.intr.experimental.vector.reduce.v2.fmul"(%arg0, %arg1) {reassoc = true} : (!llvm.float, !llvm<"<8 x float>">) -> !llvm.float
   // CHECK: call i32 @llvm.experimental.vector.reduce.xor.v8i32
   "llvm.intr.experimental.vector.reduce.xor"(%arg2) : (!llvm<"<8 x i32>">) -> !llvm.i32
   llvm.return
@@ -184,6 +206,17 @@ llvm.func @masked_intrinsics(%A: !llvm<"<7 x float>*">, %mask: !llvm<"<7 x i1>">
   llvm.return
 }
 
+// CHECK-LABEL: @memcpy_test
+llvm.func @memcpy_test(%arg0: !llvm.i32, %arg1: !llvm.i1, %arg2: !llvm<"i8*">, %arg3: !llvm<"i8*">) {
+  // CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %{{.*}}, i8* %{{.*}}, i32 %{{.*}}, i1 %{{.*}})
+  "llvm.intr.memcpy"(%arg2, %arg3, %arg0, %arg1) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i32, !llvm.i1) -> ()
+  %sz = llvm.mlir.constant(10: i64) : !llvm.i64
+  // CHECK: call void @llvm.memcpy.inline.p0i8.p0i8.i64(i8* %{{.*}}, i8* %{{.*}}, i64 10, i1 %{{.*}})
+  "llvm.intr.memcpy.inline"(%arg2, %arg3, %sz, %arg1) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i64, !llvm.i1) -> ()
+  llvm.return
+}
+
+
 // Check that intrinsics are declared with appropriate types.
 // CHECK-DAG: declare float @llvm.fma.f32(float, float, float)
 // CHECK-DAG: declare <8 x float> @llvm.fma.v8f32(<8 x float>, <8 x float>, <8 x float>) #0
@@ -213,3 +246,5 @@ llvm.func @masked_intrinsics(%A: !llvm<"<7 x float>*">, %mask: !llvm<"<7 x i1>">
 // CHECK-DAG: declare void @llvm.matrix.column.major.store.v48f32.p0f32(<48 x float>, float* nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg)
 // CHECK-DAG: declare <7 x float> @llvm.masked.load.v7f32.p0v7f32(<7 x float>*, i32 immarg, <7 x i1>, <7 x float>)
 // CHECK-DAG: declare void @llvm.masked.store.v7f32.p0v7f32(<7 x float>, <7 x float>*, i32 immarg, <7 x i1>)
+// CHECK-DAG: declare void @llvm.memcpy.p0i8.p0i8.i32(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i32, i1 immarg)
+// CHECK-DAG: declare void @llvm.memcpy.inline.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64 immarg, i1 immarg)
diff --git a/mlir/test/Target/llvmir.mlir b/mlir/test/Target/llvmir.mlir
index e1bea72cf1477..05b8ef10588ac 100644
--- a/mlir/test/Target/llvmir.mlir
+++ b/mlir/test/Target/llvmir.mlir
@@ -886,7 +886,7 @@ llvm.func @ops(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm.i32, %arg3:
 // CHECK-LABEL: define void @indirect_const_call(i64 {{%.*}})
 llvm.func @indirect_const_call(%arg0: !llvm.i64) {
 // CHECK-NEXT:  call void @body(i64 %0)
-  %0 = llvm.mlir.constant(@body) : !llvm<"void (i64)*">
+  %0 = llvm.mlir.addressof @body : !llvm<"void (i64)*">
   llvm.call %0(%arg0) : (!llvm.i64) -> ()
 // CHECK-NEXT:  ret void
   llvm.return
@@ -1230,3 +1230,13 @@ llvm.func @constant_bf16() -> !llvm<"bfloat"> {
 
 // CHECK: ret bfloat 0xR4120
 
+// -----
+
+llvm.func @address_taken() {
+  llvm.return
+}
+
+llvm.mlir.global internal constant @taker_of_address() : !llvm<"void()*"> {
+  %0 = llvm.mlir.addressof @address_taken : !llvm<"void()*">
+  llvm.return %0 : !llvm<"void()*">
+}
diff --git a/mlir/test/Transforms/buffer-placement-preparation-allowed-memref-results.mlir b/mlir/test/Transforms/buffer-placement-preparation-allowed-memref-results.mlir
index 97c96008f2691..084ac38af6e32 100644
--- a/mlir/test/Transforms/buffer-placement-preparation-allowed-memref-results.mlir
+++ b/mlir/test/Transforms/buffer-placement-preparation-allowed-memref-results.mlir
@@ -64,6 +64,15 @@ func @simple_signature_conversion(%arg0: tensor<4x8xf32>) -> tensor<4x8xf32> {
 
 // -----
 
+// CHECK-LABEL: func @func_with_unranked_arg_and_result
+func @func_with_unranked_arg_and_result(%arg0: tensor<*xf32>) -> tensor<*xf32> {
+  return %arg0 : tensor<*xf32>
+}
+// CHECK-SAME: ([[ARG:%.*]]: memref<*xf32>) -> memref<*xf32>
+// CHECK-NEXT: return [[ARG]] : memref<*xf32>
+
+// -----
+
 // CHECK-LABEL: func @func_and_block_signature_conversion
 func @func_and_block_signature_conversion(%arg0 : tensor<2xf32>, %cond : i1, %arg1: tensor<4x4xf32>) -> tensor<4x4xf32>{
     cond_br %cond, ^bb1, ^bb2
diff --git a/mlir/test/Transforms/buffer-placement-preparation.mlir b/mlir/test/Transforms/buffer-placement-preparation.mlir
index 9b0755aad1800..064b0fd7e85a9 100644
--- a/mlir/test/Transforms/buffer-placement-preparation.mlir
+++ b/mlir/test/Transforms/buffer-placement-preparation.mlir
@@ -284,3 +284,9 @@ func @caller(%arg0: tensor<5xf32>) -> tensor<5xf32> {
 // CHECK: %[[Y1:.*]] = call @callee(%[[X0]], %[[Y0]])
 // CHECK: linalg.copy(%[[Y0]], %[[CALLER_RESULT]])
 // CHECK: return
+
+// CHECK-LABEL: func @func_with_unranked_arg
+func @func_with_unranked_arg(%arg0: tensor<*xf32>) {
+  return
+}
+// CHECK-SAME: ([[ARG:%.*]]: memref<*xf32>)
diff --git a/mlir/test/Transforms/buffer-placement.mlir b/mlir/test/Transforms/buffer-placement.mlir
index 93b6b5ade0585..c3bce4ea54583 100644
--- a/mlir/test/Transforms/buffer-placement.mlir
+++ b/mlir/test/Transforms/buffer-placement.mlir
@@ -716,3 +716,220 @@ func @memref_in_function_results(%arg0: memref<5xf32>, %arg1: memref<10xf32>, %a
 // CHECK: dealloc %[[Y]]
 // CHECK: return %[[ARG1]], %[[X]]
 
+// -----
+
+// Test Case: nested region control flow
+// The alloc position of %1 does not need to be changed and flows through
+// both if branches until it is finally returned. Hence, it does not
+// require a specific dealloc operation. However, %3 requires a dealloc.
+
+// CHECK-LABEL: func @nested_region_control_flow
+func @nested_region_control_flow(
+  %arg0 : index,
+  %arg1 : index) -> memref<?x?xf32> {
+  %0 = cmpi "eq", %arg0, %arg1 : index
+  %1 = alloc(%arg0, %arg0) : memref<?x?xf32>
+  %2 = scf.if %0 -> (memref<?x?xf32>) {
+    scf.yield %1 : memref<?x?xf32>
+  } else {
+    %3 = alloc(%arg0, %arg1) : memref<?x?xf32>
+    scf.yield %1 : memref<?x?xf32>
+  }
+  return %2 : memref<?x?xf32>
+}
+
+//      CHECK: %[[ALLOC0:.*]] = alloc(%arg0, %arg0)
+// CHECK-NEXT: %[[ALLOC1:.*]] = scf.if
+//      CHECK: scf.yield %[[ALLOC0]]
+//      CHECK: %[[ALLOC2:.*]] = alloc(%arg0, %arg1)
+// CHECK-NEXT: dealloc %[[ALLOC2]]
+// CHECK-NEXT: scf.yield %[[ALLOC0]]
+//      CHECK: return %[[ALLOC1]]
+
+// -----
+
+// Test Case: nested region control flow with a nested buffer allocation in a
+// divergent branch.
+// The alloc positions of %1, %3 does not need to be changed since
+// BufferPlacement does not move allocs out of nested regions at the moment.
+// However, since %3 is allocated and "returned" in a divergent branch, we have
+// to allocate a temporary buffer (like in condBranchDynamicTypeNested).
+
+// CHECK-LABEL: func @nested_region_control_flow_div
+func @nested_region_control_flow_div(
+  %arg0 : index,
+  %arg1 : index) -> memref<?x?xf32> {
+  %0 = cmpi "eq", %arg0, %arg1 : index
+  %1 = alloc(%arg0, %arg0) : memref<?x?xf32>
+  %2 = scf.if %0 -> (memref<?x?xf32>) {
+    scf.yield %1 : memref<?x?xf32>
+  } else {
+    %3 = alloc(%arg0, %arg1) : memref<?x?xf32>
+    scf.yield %3 : memref<?x?xf32>
+  }
+  return %2 : memref<?x?xf32>
+}
+
+//      CHECK: %[[ALLOC0:.*]] = alloc(%arg0, %arg0)
+// CHECK-NEXT: %[[ALLOC1:.*]] = scf.if
+//      CHECK: %[[ALLOC2:.*]] = alloc
+// CHECK-NEXT: linalg.copy(%[[ALLOC0]], %[[ALLOC2]])
+//      CHECK: scf.yield %[[ALLOC2]]
+//      CHECK: %[[ALLOC3:.*]] = alloc(%arg0, %arg1)
+//      CHECK: %[[ALLOC4:.*]] = alloc
+// CHECK-NEXT: linalg.copy(%[[ALLOC3]], %[[ALLOC4]])
+//      CHECK: dealloc %[[ALLOC3]]
+//      CHECK: scf.yield %[[ALLOC4]]
+//      CHECK: dealloc %[[ALLOC0]]
+// CHECK-NEXT: return %[[ALLOC1]]
+
+// -----
+
+// Test Case: deeply nested region control flow with a nested buffer allocation
+// in a divergent branch.
+// The alloc positions of %1, %4 and %5 does not need to be changed since
+// BufferPlacement does not move allocs out of nested regions at the moment.
+// However, since %4 is allocated and "returned" in a divergent branch, we have
+// to allocate several temporary buffers (like in condBranchDynamicTypeNested).
+
+// CHECK-LABEL: func @nested_region_control_flow_div_nested
+func @nested_region_control_flow_div_nested(
+  %arg0 : index,
+  %arg1 : index) -> memref<?x?xf32> {
+  %0 = cmpi "eq", %arg0, %arg1 : index
+  %1 = alloc(%arg0, %arg0) : memref<?x?xf32>
+  %2 = scf.if %0 -> (memref<?x?xf32>) {
+    %3 = scf.if %0 -> (memref<?x?xf32>) {
+      scf.yield %1 : memref<?x?xf32>
+    } else {
+      %4 = alloc(%arg0, %arg1) : memref<?x?xf32>
+      scf.yield %4 : memref<?x?xf32>
+    }
+    scf.yield %3 : memref<?x?xf32>
+  } else {
+    %5 = alloc(%arg1, %arg1) : memref<?x?xf32>
+    scf.yield %5 : memref<?x?xf32>
+  }
+  return %2 : memref<?x?xf32>
+}
+//      CHECK: %[[ALLOC0:.*]] = alloc(%arg0, %arg0)
+// CHECK-NEXT: %[[ALLOC1:.*]] = scf.if
+// CHECK-NEXT: %[[ALLOC2:.*]] = scf.if
+//      CHECK: %[[ALLOC3:.*]] = alloc
+// CHECK-NEXT: linalg.copy(%[[ALLOC0]], %[[ALLOC3]])
+//      CHECK: scf.yield %[[ALLOC3]]
+//      CHECK: %[[ALLOC4:.*]] = alloc(%arg0, %arg1)
+//      CHECK: %[[ALLOC5:.*]] = alloc
+// CHECK-NEXT: linalg.copy(%[[ALLOC4]], %[[ALLOC5]])
+//      CHECK: dealloc %[[ALLOC4]]
+//      CHECK: scf.yield %[[ALLOC5]]
+//      CHECK: %[[ALLOC6:.*]] = alloc
+// CHECK-NEXT: linalg.copy(%[[ALLOC2]], %[[ALLOC6]])
+//      CHECK: dealloc %[[ALLOC2]]
+//      CHECK: scf.yield %[[ALLOC6]]
+//      CHECK: %[[ALLOC7:.*]] = alloc(%arg1, %arg1)
+//      CHECK: %[[ALLOC8:.*]] = alloc
+// CHECK-NEXT: linalg.copy(%[[ALLOC7]], %[[ALLOC8]])
+//      CHECK: dealloc %[[ALLOC7]]
+//      CHECK: scf.yield %[[ALLOC8]]
+//      CHECK: dealloc %[[ALLOC0]]
+// CHECK-NEXT: return %[[ALLOC1]]
+
+// -----
+
+// Test Case: nested region control flow within a region interface.
+// The alloc positions of %0 does not need to be changed and no copies are
+// required in this case since the allocation finally escapes the method.
+
+// CHECK-LABEL: func @inner_region_control_flow
+func @inner_region_control_flow(%arg0 : index) -> memref<?x?xf32> {
+  %0 = alloc(%arg0, %arg0) : memref<?x?xf32>
+  %1 = test.region_if %0 : memref<?x?xf32> -> (memref<?x?xf32>) then {
+    ^bb0(%arg1 : memref<?x?xf32>):
+      test.region_if_yield %arg1 : memref<?x?xf32>
+  } else {
+    ^bb0(%arg1 : memref<?x?xf32>):
+      test.region_if_yield %arg1 : memref<?x?xf32>
+  } join {
+    ^bb0(%arg1 : memref<?x?xf32>):
+      test.region_if_yield %arg1 : memref<?x?xf32>
+  }
+  return %1 : memref<?x?xf32>
+}
+
+//      CHECK: %[[ALLOC0:.*]] = alloc(%arg0, %arg0)
+// CHECK-NEXT: %[[ALLOC1:.*]] = test.region_if
+// CHECK-NEXT: ^bb0(%[[ALLOC2:.*]]:{{.*}}):
+// CHECK-NEXT: test.region_if_yield %[[ALLOC2]]
+//      CHECK: ^bb0(%[[ALLOC3:.*]]:{{.*}}):
+// CHECK-NEXT: test.region_if_yield %[[ALLOC3]]
+//      CHECK: ^bb0(%[[ALLOC4:.*]]:{{.*}}):
+// CHECK-NEXT: test.region_if_yield %[[ALLOC4]]
+//      CHECK: return %[[ALLOC1]]
+
+// -----
+
+// Test Case: nested region control flow within a region interface including an
+// allocation in a divergent branch.
+// The alloc positions of %1 and %2 does not need to be changed since
+// BufferPlacement does not move allocs out of nested regions at the moment.
+// However, since %2 is allocated and yielded in a divergent branch, we have
+// to allocate several temporary buffers (like in condBranchDynamicTypeNested).
+
+// CHECK-LABEL: func @inner_region_control_flow_div
+func @inner_region_control_flow_div(
+  %arg0 : index,
+  %arg1 : index) -> memref<?x?xf32> {
+  %0 = alloc(%arg0, %arg0) : memref<?x?xf32>
+  %1 = test.region_if %0 : memref<?x?xf32> -> (memref<?x?xf32>) then {
+    ^bb0(%arg2 : memref<?x?xf32>):
+      test.region_if_yield %arg2 : memref<?x?xf32>
+  } else {
+    ^bb0(%arg2 : memref<?x?xf32>):
+      %2 = alloc(%arg0, %arg1) : memref<?x?xf32>
+      test.region_if_yield %2 : memref<?x?xf32>
+  } join {
+    ^bb0(%arg2 : memref<?x?xf32>):
+      test.region_if_yield %arg2 : memref<?x?xf32>
+  }
+  return %1 : memref<?x?xf32>
+}
+
+//      CHECK: %[[ALLOC0:.*]] = alloc(%arg0, %arg0)
+// CHECK-NEXT: %[[ALLOC1:.*]] = test.region_if
+// CHECK-NEXT: ^bb0(%[[ALLOC2:.*]]:{{.*}}):
+//      CHECK: %[[ALLOC3:.*]] = alloc
+// CHECK-NEXT: linalg.copy(%[[ALLOC2]], %[[ALLOC3]])
+// CHECK-NEXT: test.region_if_yield %[[ALLOC3]]
+//      CHECK: ^bb0(%[[ALLOC4:.*]]:{{.*}}):
+//      CHECK: %[[ALLOC5:.*]] = alloc
+//      CHECK: %[[ALLOC6:.*]] = alloc
+// CHECK-NEXT: linalg.copy(%[[ALLOC5]], %[[ALLOC6]])
+// CHECK-NEXT: dealloc %[[ALLOC5]]
+// CHECK-NEXT: test.region_if_yield %[[ALLOC6]]
+//      CHECK: ^bb0(%[[ALLOC7:.*]]:{{.*}}):
+//      CHECK: %[[ALLOC8:.*]] = alloc
+// CHECK-NEXT: linalg.copy(%[[ALLOC7]], %[[ALLOC8]])
+// CHECK-NEXT: dealloc %[[ALLOC7]]
+// CHECK-NEXT: test.region_if_yield %[[ALLOC8]]
+//      CHECK: dealloc %[[ALLOC0]]
+// CHECK-NEXT: return %[[ALLOC1]]
+
+// -----
+
+// CHECK-LABEL: func @subview
+func @subview(%arg0 : index, %arg1 : index, %arg2 : memref<?x?xf32>) {
+  %0 = alloc() : memref<64x4xf32, offset: 0, strides: [4, 1]>
+  %1 = subview %0[%arg0, %arg1][%arg0, %arg1][%arg0, %arg1] :
+    memref<64x4xf32, offset: 0, strides: [4, 1]>
+  to memref<?x?xf32, offset: ?, strides: [?, ?]>
+  "linalg.copy"(%1, %arg2) :
+    (memref<?x?xf32, offset: ?, strides: [?, ?]>, memref<?x?xf32>) -> ()
+  return
+}
+
+// CHECK-NEXT: %[[ALLOC:.*]] = alloc()
+// CHECK-NEXT: subview
+// CHECK-NEXT: linalg.copy
+// CHECK-NEXT: dealloc %[[ALLOC]]
+// CHECK-NEXT: return
diff --git a/mlir/test/Transforms/copy-removal.mlir b/mlir/test/Transforms/copy-removal.mlir
new file mode 100644
index 0000000000000..f750dabb18a04
--- /dev/null
+++ b/mlir/test/Transforms/copy-removal.mlir
@@ -0,0 +1,285 @@
+// RUN: mlir-opt -copy-removal -split-input-file %s | FileCheck %s
+
+// All linalg copies except the linalg.copy(%1, %9) must be removed since the
+// defining operation of %1 and its DeallocOp have been defined in another block.
+
+// CHECK-LABEL: func @nested_region_control_flow_div_nested
+func @nested_region_control_flow_div_nested(%arg0: index, %arg1: index) -> memref<?x?xf32> {
+  %0 = cmpi "eq", %arg0, %arg1 : index
+  %1 = alloc(%arg0, %arg0) : memref<?x?xf32>
+  // CHECK: %{{.*}} = scf.if
+  %2 = scf.if %0 -> (memref<?x?xf32>) {
+    // CHECK: %[[PERCENT3:.*]] = scf.if
+    %3 = scf.if %0 -> (memref<?x?xf32>) {
+      %c0_0 = constant 0 : index
+      %7 = dim %1, %c0_0 : memref<?x?xf32>
+      %c1_1 = constant 1 : index
+      %8 = dim %1, %c1_1 : memref<?x?xf32>
+      %9 = alloc(%7, %8) : memref<?x?xf32>
+      // CHECK: linalg.copy({{.*}}, %[[PERCENT9:.*]])
+      linalg.copy(%1, %9) : memref<?x?xf32>, memref<?x?xf32>
+      // CHECK: scf.yield %[[PERCENT9]]
+      scf.yield %9 : memref<?x?xf32>
+    } else {
+      // CHECK: %[[PERCENT7:.*]] = alloc
+      %7 = alloc(%arg0, %arg1) : memref<?x?xf32>
+      %c0_0 = constant 0 : index
+      %8 = dim %7, %c0_0 : memref<?x?xf32>
+      %c1_1 = constant 1 : index
+      %9 = dim %7, %c1_1 : memref<?x?xf32>
+      // CHECK-NOT: %{{.*}} = alloc
+      // CHECK-NOT: linalg.copy(%[[PERCENT7]], %{{.*}})
+      // CHECK-NOT: dealloc %[[PERCENT7]]
+      %10 = alloc(%8, %9) : memref<?x?xf32>
+      linalg.copy(%7, %10) : memref<?x?xf32>, memref<?x?xf32>
+      dealloc %7 : memref<?x?xf32>
+      // CHECK: scf.yield %[[PERCENT7]]
+      scf.yield %10 : memref<?x?xf32>
+    }
+    %c0 = constant 0 : index
+    %4 = dim %3, %c0 : memref<?x?xf32>
+    %c1 = constant 1 : index
+    %5 = dim %3, %c1 : memref<?x?xf32>
+    // CHECK-NOT: %{{.*}} = alloc
+    // CHECK-NOT: linalg.copy(%[[PERCENT3]], %{{.*}})
+    // CHECK-NOT: dealloc %[[PERCENT3]]
+    %6 = alloc(%4, %5) : memref<?x?xf32>
+    linalg.copy(%3, %6) : memref<?x?xf32>, memref<?x?xf32>
+    dealloc %3 : memref<?x?xf32>
+    // CHECK: scf.yield %[[PERCENT3]]
+    scf.yield %6 : memref<?x?xf32>
+  } else {
+    // CHECK: %[[PERCENT3:.*]] = alloc
+    %3 = alloc(%arg1, %arg1) : memref<?x?xf32>
+    %c0 = constant 0 : index
+    %4 = dim %3, %c0 : memref<?x?xf32>
+    %c1 = constant 1 : index
+    %5 = dim %3, %c1 : memref<?x?xf32>
+    // CHECK-NOT: %{{.*}} = alloc
+    // CHECK-NOT: linalg.copy(%[[PERCENT3]], %{{.*}})
+    // CHECK-NOT: dealloc %[[PERCENT3]]
+    %6 = alloc(%4, %5) : memref<?x?xf32>
+    linalg.copy(%3, %6) : memref<?x?xf32>, memref<?x?xf32>
+    dealloc %3 : memref<?x?xf32>
+    // CHECK: scf.yield %[[PERCENT3]]
+    scf.yield %6 : memref<?x?xf32>
+  }
+  dealloc %1 : memref<?x?xf32>
+  return %2 : memref<?x?xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @simple_test
+func @simple_test() -> memref<5xf32> {
+  %temp = alloc() : memref<5xf32>
+  %ret = alloc() : memref<5xf32>
+  linalg.copy(%ret, %temp) : memref<5xf32>, memref<5xf32>
+  dealloc %ret : memref<5xf32>
+  return %temp : memref<5xf32>
+}
+// CHECK-SAME: () -> memref<5xf32>
+// CHECK-NEXT: %[[ret:.*]] = alloc()
+// CHECK-NOT: linalg.copy(%[[ret]], %{{.*}})
+// CHECK-NOT: dealloc %[[ret]]
+// CHECK: return %[[ret]]
+
+// -----
+
+// It is legal to remove the copy operation that %ret has a usage before the copy
+// operation. The allocation of %temp and the deallocation of %ret should be also
+// removed.
+
+// CHECK-LABEL: func @test_with_ret_usage_before_copy
+func @test_with_ret_usage_before_copy() -> memref<5xf32> {
+  %ret = alloc() : memref<5xf32>
+  %temp = alloc() : memref<5xf32>
+  %c0 = constant 0 : index
+  %dimension = dim %ret, %c0 : memref<5xf32>
+  linalg.copy(%ret, %temp) : memref<5xf32>, memref<5xf32>
+  dealloc %ret : memref<5xf32>
+  return %temp : memref<5xf32>
+}
+// CHECK-NEXT: %[[ret:.*]] = alloc()
+// CHECK-NOT: %{{.*}} = alloc
+// CHECK-NEXT: %{{.*}} = constant
+// CHECK-NEXT: %[[DIM:.*]] = dim %[[ret]]
+// CHECK-NOT: linalg.copy(%[[ret]], %{{.*}})
+// CHECK-NOT: dealloc %[[ret]]
+// CHECK: return %[[ret]]
+
+// -----
+
+// It is illegal to remove a copy operation that %ret has a usage after copy
+// operation.
+
+// CHECK-LABEL: func @test_with_ret_usage_after_copy
+func @test_with_ret_usage_after_copy() -> memref<5xf32> {
+  %ret = alloc() : memref<5xf32>
+  %temp = alloc() : memref<5xf32>
+  // CHECK: linalg.copy
+  linalg.copy(%ret, %temp) : memref<5xf32>, memref<5xf32>
+  %c0 = constant 0 : index
+  %dimension = dim %ret, %c0 : memref<5xf32>
+  dealloc %ret : memref<5xf32>
+  return %temp : memref<5xf32>
+}
+
+// -----
+
+// It is illegal to remove a copy operation that %temp has a usage before copy
+// operation.
+
+// CHECK-LABEL: func @test_with_temp_usage_before_copy
+func @test_with_temp_usage_before_copy() -> memref<5xf32> {
+  %ret = alloc() : memref<5xf32>
+  %temp = alloc() : memref<5xf32>
+  %c0 = constant 0 : index
+  %dimension = dim %temp, %c0 : memref<5xf32>
+  // CHECK: linalg.copy
+  linalg.copy(%ret, %temp) : memref<5xf32>, memref<5xf32>
+  dealloc %ret : memref<5xf32>
+  return %temp : memref<5xf32>
+}
+
+// -----
+
+// It is legal to remove the copy operation that %temp has a usage after the copy
+// operation. The allocation of %temp and the deallocation of %ret should be also
+// removed.
+
+#map0 = affine_map<(d0) -> (d0)>
+
+// CHECK-LABEL: func @test_with_temp_usage_after_copy
+func @test_with_temp_usage_after_copy() -> memref<5xf32> {
+  %ret = alloc() : memref<5xf32>
+  %res = alloc() : memref<5xf32>
+  %temp = alloc() : memref<5xf32>
+  linalg.copy(%ret, %temp) : memref<5xf32>, memref<5xf32>
+  linalg.generic {
+    args_in = 1 : i64,
+    args_out = 1 : i64,
+    indexing_maps = [#map0, #map0],
+    iterator_types = ["parallel"]} %temp, %res {
+  ^bb0(%gen1_arg0: f32, %gen1_arg1: f32):
+    %tmp1 = exp %gen1_arg0 : f32
+    linalg.yield %tmp1 : f32
+  }: memref<5xf32>, memref<5xf32>
+  dealloc %ret : memref<5xf32>
+  return %temp : memref<5xf32>
+}
+// CHECK-NEXT: %[[ret:.*]] = alloc()
+// CHECK-NEXT: %[[res:.*]] = alloc()
+// CHECK-NOT: %{{.*}} = alloc()
+// CHECK-NOT: linalg.copy
+// CHECK-NOT: dealloc %[[ret]]
+// CHECK: return %[[ret]]
+
+// -----
+
+// CHECK-LABEL: func @make_allocation
+func @make_allocation() -> memref<5xf32> {
+  %mem = alloc() : memref<5xf32>
+  return %mem : memref<5xf32>
+}
+
+// CHECK-LABEL: func @test_with_function_call
+func @test_with_function_call() -> memref<5xf32> {
+  // CHECK-NEXT: %[[ret:.*]] = call @make_allocation() : () -> memref<5xf32>
+  %ret = call @make_allocation() : () -> (memref<5xf32>)
+  // CHECK-NOT: %{{.*}} = alloc
+  // CHECK-NOT: linalg.copy(%[[ret]], %{{.*}})
+  // CHECK-NOT: dealloc %[[ret]]
+  %temp = alloc() : memref<5xf32>
+  linalg.copy(%ret, %temp) : memref<5xf32>, memref<5xf32>
+  dealloc %ret : memref<5xf32>
+  // CHECK: return %[[ret]]
+  return %temp : memref<5xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @multiple_deallocs_in_different_blocks
+func @multiple_deallocs_in_different_blocks(%cond : i1) -> memref<5xf32> {
+  // CHECK-NEXT: %[[PERCENT0:.*]] = alloc()
+  %0 = alloc() : memref<5xf32>
+  cond_br %cond, ^bb1, ^bb2
+^bb1:
+  dealloc %0 : memref<5xf32>
+  // CHECK: br ^[[BB3:.*]](%[[PERCENT0]]
+  br ^bb3(%0 : memref<5xf32>)
+^bb2:
+  // CHECK-NOT: %{{.*}} = alloc
+  // CHECK-NOT: linalg.copy(%[[PERCENT0]], %{{.*}})
+  // CHECK-NOT: dealloc %[[PERCENT0]]
+  %temp = alloc() : memref<5xf32>
+  linalg.copy(%0, %temp) : memref<5xf32>, memref<5xf32>
+  dealloc %0 : memref<5xf32>
+  // CHECK: br ^[[BB3]](%[[PERCENT0]]
+  br ^bb3(%temp : memref<5xf32>)
+^bb3(%res : memref<5xf32>):
+  return %res : memref<5xf32>
+}
+
+// -----
+
+#map0 = affine_map<(d0) -> (d0)>
+
+// CHECK-LABEL: func @test_ReuseCopyTargetAsSource
+func @test_ReuseCopyTargetAsSource(%arg0: memref<2xf32>, %result: memref<2xf32>){
+  // CHECK-SAME: (%[[ARG0:.*]]: memref<2xf32>, %[[RES:.*]]: memref<2xf32>)
+  // CHECK-NOT: %{{.*}} = alloc
+  %temp = alloc() : memref<2xf32>
+  // CHECK-NEXT: linalg.generic
+  // CHECK-SAME: %[[ARG0]], %[[RES]]
+  // CHECK-NOT: linalg.copy(%{{.*}}, %[[RES]])
+  // CHECK-NOT: dealloc %{{.*}}
+  linalg.generic {
+    args_in = 1 : i64,
+    args_out = 1 : i64,
+    indexing_maps = [#map0, #map0],
+    iterator_types = ["parallel"]} %arg0, %temp {
+  ^bb0(%gen2_arg0: f32, %gen2_arg1: f32):
+    %tmp2 = exp %gen2_arg0 : f32
+    linalg.yield %tmp2 : f32
+  }: memref<2xf32>, memref<2xf32>
+  "linalg.copy"(%temp, %result) : (memref<2xf32>, memref<2xf32>) -> ()
+  dealloc %temp : memref<2xf32>
+  // CHECK: return
+  return
+}
+
+// -----
+
+// Copy operation must not be removed since an operation writes to %to value
+// before copy.
+
+#map0 = affine_map<(d0) -> (d0)>
+
+// CHECK-LABEL: func @test_ReuseCopyTargetAsSource
+func @test_ReuseCopyTargetAsSource(%arg0: memref<2xf32>){
+  %to = alloc() : memref<2xf32>
+  %temp = alloc() : memref<2xf32>
+  linalg.generic {
+    args_in = 1 : i64,
+    args_out = 1 : i64,
+    indexing_maps = [#map0, #map0],
+    iterator_types = ["parallel"]} %arg0, %temp {
+  ^bb0(%gen1_arg0: f32, %gen1_arg1: f32):
+    %tmp1 = exp %gen1_arg0 : f32
+    linalg.yield %tmp1 : f32
+  }: memref<2xf32>, memref<2xf32>
+  linalg.generic {
+    args_in = 1 : i64,
+    args_out = 1 : i64,
+    indexing_maps = [#map0, #map0],
+    iterator_types = ["parallel"]} %arg0, %to {
+  ^bb0(%gen2_arg0: f32, %gen2_arg1: f32):
+    %tmp2 = exp %gen2_arg0 : f32
+    linalg.yield %tmp2 : f32
+  }: memref<2xf32>, memref<2xf32>
+  // CHECK: linalg.copy
+  "linalg.copy"(%temp, %to) : (memref<2xf32>, memref<2xf32>) -> ()
+  dealloc %temp : memref<2xf32>
+  return
+}
diff --git a/mlir/test/Transforms/loop-fusion.mlir b/mlir/test/Transforms/loop-fusion.mlir
index 0e79e0b67ba20..7d91e780e9edb 100644
--- a/mlir/test/Transforms/loop-fusion.mlir
+++ b/mlir/test/Transforms/loop-fusion.mlir
@@ -1,12 +1,12 @@
 // RUN: mlir-opt -allow-unregistered-dialect %s -affine-loop-fusion -split-input-file | FileCheck %s
 // RUN: mlir-opt -allow-unregistered-dialect %s -affine-loop-fusion="fusion-maximal" -split-input-file | FileCheck %s --check-prefix=MAXIMAL
 
-// TODO(andydavis) Add more tests:
+// TODO: Add more tests:
 // *) Add nested fusion test cases when non-constant loop bound support is
 //    added to iteration domain in dependence check.
 // *) Add a test w/ floordiv/ceildiv/mod when supported in dependence check.
 // *) Add tests which check fused computation slice indexing and loop bounds.
-// TODO(andydavis) Test clean up: move memref allocs to func args.
+// TODO: Test clean up: move memref allocs to func args.
 
 // -----
 
@@ -317,7 +317,7 @@ func @should_fuse_producer_consumer() {
   }
   // Fusing loop %i0 to %i2 would violate the WAW dependence between %i0 and
   // %i1, but OK to fuse %i1 into %i2.
-  // TODO(andydavis) When the fusion pass is run to a fixed-point, it should
+  // TODO: When the fusion pass is run to a fixed-point, it should
   // fuse all three of these loop nests.
   // CHECK:      alloc() : memref<1xf32>
   // CHECK:      affine.for %{{.*}} = 0 to 10 {
@@ -2535,3 +2535,102 @@ func @multi_outgoing_edges(%in0 : memref<32xf32>,
 // CHECK:        mulf
 // CHECK-NOT:  affine.for
 // CHECK:        divf
+
+// -----
+
+// Test fusion when dynamically shaped memrefs are used with constant trip count loops.
+
+// CHECK-LABEL: func @calc
+func @calc(%arg0: memref<?xf32>, %arg1: memref<?xf32>, %arg2: memref<?xf32>, %len: index) {
+  %c1 = constant 1 : index
+  %1 = alloc(%len) : memref<?xf32>
+  affine.for %arg4 = 1 to 10 {
+    %7 = affine.load %arg0[%arg4] : memref<?xf32>
+    %8 = affine.load %arg1[%arg4] : memref<?xf32>
+    %9 = addf %7, %8 : f32
+    affine.store %9, %1[%arg4] : memref<?xf32>
+  }
+  affine.for %arg4 = 1 to 10 {
+    %7 = affine.load %1[%arg4] : memref<?xf32>
+    %8 = affine.load %arg1[%arg4] : memref<?xf32>
+    %9 = mulf %7, %8 : f32
+    affine.store %9, %arg2[%arg4] : memref<?xf32>
+  }
+  return
+}
+// CHECK:       alloc() : memref<1xf32>
+// CHECK:       affine.for %arg{{.*}} = 1 to 10 {
+// CHECK-NEXT:    affine.load %arg{{.*}}
+// CHECK-NEXT:    affine.load %arg{{.*}}
+// CHECK-NEXT:    addf
+// CHECK-NEXT:    affine.store %{{.*}}, %{{.*}}[0] : memref<1xf32>
+// CHECK-NEXT:    affine.load %{{.*}}[0] : memref<1xf32>
+// CHECK-NEXT:    affine.load %arg{{.*}}[%arg{{.*}}] : memref<?xf32>
+// CHECK-NEXT:    mulf
+// CHECK-NEXT:    affine.store %{{.*}}, %arg{{.*}}[%arg{{.*}}] : memref<?xf32>
+// CHECK-NEXT:  }
+// CHECK-NEXT:  return
+
+// -----
+
+// CHECK-LABEL: func @should_not_fuse_since_non_affine_users
+func @should_not_fuse_since_non_affine_users(%in0 : memref<32xf32>,
+                      %in1 : memref<32xf32>) {
+  affine.for %d = 0 to 32 {
+    %lhs = affine.load %in0[%d] : memref<32xf32>
+    %rhs = affine.load %in1[%d] : memref<32xf32>
+    %add = addf %lhs, %rhs : f32
+    affine.store %add, %in0[%d] : memref<32xf32>
+  }
+  affine.for %d = 0 to 32 {
+    %lhs = load %in0[%d] : memref<32xf32>
+    %rhs = load %in1[%d] : memref<32xf32>
+    %add = subf %lhs, %rhs : f32
+    store %add, %in0[%d] : memref<32xf32>
+  }
+  affine.for %d = 0 to 32 {
+    %lhs = affine.load %in0[%d] : memref<32xf32>
+    %rhs = affine.load %in1[%d] : memref<32xf32>
+    %add = mulf %lhs, %rhs : f32
+    affine.store %add, %in0[%d] : memref<32xf32>
+  }
+  return
+}
+
+// CHECK:  affine.for
+// CHECK:    addf
+// CHECK:  affine.for
+// CHECK:    subf
+// CHECK:  affine.for
+// CHECK:    mulf
+
+// -----
+
+// CHECK-LABEL: func @should_not_fuse_since_top_level_non_affine_users
+func @should_not_fuse_since_top_level_non_affine_users(%in0 : memref<32xf32>,
+                      %in1 : memref<32xf32>) {
+  %sum = alloc() : memref<f32>
+  affine.for %d = 0 to 32 {
+    %lhs = affine.load %in0[%d] : memref<32xf32>
+    %rhs = affine.load %in1[%d] : memref<32xf32>
+    %add = addf %lhs, %rhs : f32
+    store %add, %sum[] : memref<f32>
+    affine.store %add, %in0[%d] : memref<32xf32>
+  }
+  %load_sum = load %sum[] : memref<f32>
+  affine.for %d = 0 to 32 {
+    %lhs = affine.load %in0[%d] : memref<32xf32>
+    %rhs = affine.load %in1[%d] : memref<32xf32>
+    %add = mulf %lhs, %rhs : f32
+    %sub = subf %add, %load_sum: f32
+    affine.store %sub, %in0[%d] : memref<32xf32>
+  }
+  dealloc %sum : memref<f32>
+  return
+}
+
+// CHECK:  affine.for
+// CHECK:    addf
+// CHECK:  affine.for
+// CHECK:    mulf
+// CHECK:    subf
diff --git a/mlir/test/Transforms/memref-dependence-check.mlir b/mlir/test/Transforms/memref-dependence-check.mlir
index 65af899c726c7..154dcf79c114a 100644
--- a/mlir/test/Transforms/memref-dependence-check.mlir
+++ b/mlir/test/Transforms/memref-dependence-check.mlir
@@ -766,7 +766,7 @@ func @delinearize_mod_floordiv() {
 // expected-remark@above {{dependence from 1 to 2 at depth 1 = false}}
 // expected-remark@above {{dependence from 1 to 2 at depth 2 = false}}
 // expected-remark@above {{dependence from 1 to 2 at depth 3 = false}}
-// TODO(andydavis): the dep tester shouldn't be printing out these messages
+// TODO: the dep tester shouldn't be printing out these messages
 // below; they are redundant.
       affine.store %v0, %out[%ii, %jj] : memref<64x9xi32>
 // expected-remark@above {{dependence from 2 to 0 at depth 1 = false}}
@@ -781,7 +781,7 @@ func @delinearize_mod_floordiv() {
   return
 }
 
-// TODO(bondhugula): add more test cases involving mod's/div's.
+// TODO: add more test cases involving mod's/div's.
 
 // -----
 
diff --git a/mlir/test/Transforms/parallel-loop-collapsing.mlir b/mlir/test/Transforms/parallel-loop-collapsing.mlir
index a124d180e42c2..5d581cffaffe9 100644
--- a/mlir/test/Transforms/parallel-loop-collapsing.mlir
+++ b/mlir/test/Transforms/parallel-loop-collapsing.mlir
@@ -16,6 +16,8 @@ func @parallel_many_dims() {
   %c12 = constant 12 : index
   %c13 = constant 13 : index
   %c14 = constant 14 : index
+  %c15 = constant 15 : index
+  %c26 = constant 26 : index
 
   scf.parallel (%i0, %i1, %i2, %i3, %i4) = (%c0, %c3, %c6, %c9, %c12) to (%c2, %c5, %c8, %c11, %c14)
                                           step (%c1, %c4, %c7, %c10, %c13) {
@@ -26,24 +28,19 @@ func @parallel_many_dims() {
 
 // CHECK-LABEL: func @parallel_many_dims() {
 // CHECK:         [[C6:%.*]] = constant 6 : index
-// CHECK:         [[C7:%.*]] = constant 7 : index
 // CHECK:         [[C9:%.*]] = constant 9 : index
 // CHECK:         [[C10:%.*]] = constant 10 : index
-// CHECK:         [[C12:%.*]] = constant 12 : index
-// CHECK:         [[C13:%.*]] = constant 13 : index
-// CHECK:         [[C3:%.*]] = constant 3 : index
 // CHECK:         [[C0:%.*]] = constant 0 : index
 // CHECK:         [[C1:%.*]] = constant 1 : index
 // CHECK:         [[C2:%.*]] = constant 2 : index
-// CHECK:         scf.parallel ([[NEW_I0:%.*]], [[NEW_I1:%.*]], [[NEW_I2:%.*]]) = ([[C0]], [[C0]], [[C0]]) to ([[C2]], [[C1]], [[C1]]) step ([[C1]], [[C1]], [[C1]]) {
+// CHECK:         [[C3:%.*]] = constant 3 : index
+// CHECK:         [[C12:%.*]] = constant 12 : index
+// CHECK:         scf.parallel ([[NEW_I0:%.*]]) = ([[C0]]) to ([[C2]]) step ([[C1]]) {
 // CHECK:           [[I0:%.*]] = remi_signed [[NEW_I0]], [[C2]] : index
-// CHECK:           [[VAL_16:%.*]] = muli [[NEW_I1]], [[C13]] : index
-// CHECK:           [[I4:%.*]] = addi [[VAL_16]], [[C12]] : index
-// CHECK:           [[VAL_18:%.*]] = muli [[NEW_I0]], [[C10]] : index
-// CHECK:           [[I3:%.*]] = addi [[VAL_18]], [[C9]] : index
-// CHECK:           [[VAL_20:%.*]] = muli [[NEW_I2]], [[C7]] : index
-// CHECK:           [[I2:%.*]] = addi [[VAL_20]], [[C6]] : index
-// CHECK:           "magic.op"([[I0]], [[C3]], [[I2]], [[I3]], [[I4]]) : (index, index, index, index, index) -> index
+// CHECK:           [[V0:%.*]] = divi_signed [[NEW_I0]], [[C2]] : index
+// CHECK:           [[V2:%.*]] = muli [[V0]], [[C10]] : index
+// CHECK:           [[I3:%.*]] = addi [[V2]], [[C9]] : index
+// CHECK:           "magic.op"([[I0]], [[C3]], [[C6]], [[I3]], [[C12]]) : (index, index, index, index, index) -> index
 // CHECK:           scf.yield
 // CHECK-NEXT:    }
 // CHECK-NEXT:    return
diff --git a/mlir/test/Transforms/single-parallel-loop-collapsing.mlir b/mlir/test/Transforms/single-parallel-loop-collapsing.mlir
index 4cc354bca0b26..bdf4338e4889d 100644
--- a/mlir/test/Transforms/single-parallel-loop-collapsing.mlir
+++ b/mlir/test/Transforms/single-parallel-loop-collapsing.mlir
@@ -18,16 +18,15 @@ func @collapse_to_single() {
 // CHECK:         [[C4:%.*]] = constant 4 : index
 // CHECK:         [[C18:%.*]] = constant 18 : index
 // CHECK:         [[C3:%.*]] = constant 3 : index
-// CHECK:         [[C6:%.*]] = constant 6 : index
 // CHECK:         [[C0:%.*]] = constant 0 : index
 // CHECK:         [[C1:%.*]] = constant 1 : index
 // CHECK:         scf.parallel ([[NEW_I:%.*]]) = ([[C0]]) to ([[C18]]) step ([[C1]]) {
 // CHECK:           [[I0_COUNT:%.*]] = remi_signed [[NEW_I]], [[C3]] : index
-// CHECK:           [[I1_COUNT:%.*]] = divi_signed [[NEW_I]], [[C6]] : index
-// CHECK:           [[VAL_10:%.*]] = muli [[I1_COUNT]], [[C4]] : index
-// CHECK:           [[I1:%.*]] = addi [[VAL_10]], [[C7]] : index
-// CHECK:           [[VAL_12:%.*]] = muli [[I0_COUNT]], [[C3]] : index
-// CHECK:           [[I0:%.*]] = addi [[VAL_12]], [[C3]] : index
+// CHECK:           [[I1_COUNT:%.*]] = divi_signed [[NEW_I]], [[C3]] : index
+// CHECK:           [[V0:%.*]] = muli [[I1_COUNT]], [[C4]] : index
+// CHECK:           [[I1:%.*]] = addi [[V0]], [[C7]] : index
+// CHECK:           [[V1:%.*]] = muli [[I0_COUNT]], [[C3]] : index
+// CHECK:           [[I0:%.*]] = addi [[V1]], [[C3]] : index
 // CHECK:           "magic.op"([[I0]], [[I1]]) : (index, index) -> index
 // CHECK:           scf.yield
 // CHECK-NEXT:    }
diff --git a/mlir/test/lib/CMakeLists.txt b/mlir/test/lib/CMakeLists.txt
index 3e69c7514d8f2..641a6218d1ccd 100644
--- a/mlir/test/lib/CMakeLists.txt
+++ b/mlir/test/lib/CMakeLists.txt
@@ -2,4 +2,5 @@ add_subdirectory(DeclarativeTransforms)
 add_subdirectory(Dialect)
 add_subdirectory(IR)
 add_subdirectory(Pass)
+add_subdirectory(Reducer)
 add_subdirectory(Transforms)
diff --git a/mlir/test/lib/DeclarativeTransforms/TestVectorTransformPatterns.td b/mlir/test/lib/DeclarativeTransforms/TestVectorTransformPatterns.td
index 49a3499b0bada..2c6ca1a05733e 100644
--- a/mlir/test/lib/DeclarativeTransforms/TestVectorTransformPatterns.td
+++ b/mlir/test/lib/DeclarativeTransforms/TestVectorTransformPatterns.td
@@ -26,7 +26,7 @@ def : Pat<(AddFOp:$op_results $a, $b),
           (UnrollVectorOp<[2, 2]> $op_results, $a, $b),
           [(Constraint<HasShape<[4, 4]>> $a)]>;
 
-// TODO(andydavis) Add Constraints on lhs/rhs shapes.
+// TODO: Add Constraints on lhs/rhs shapes.
 def : Pat<(Vector_ContractionOp:$op_results $a, $b, $c, $masks, $attr0, $attr1),
           (UnrollVectorOp<[2, 2, 2]> $op_results, $a, $b, $c),
           [(Constraint<HasShape<[4, 4]>> $c)]>;
diff --git a/mlir/test/lib/Dialect/Affine/CMakeLists.txt b/mlir/test/lib/Dialect/Affine/CMakeLists.txt
index 68a0b06e0e318..3d08fed788e2e 100644
--- a/mlir/test/lib/Dialect/Affine/CMakeLists.txt
+++ b/mlir/test/lib/Dialect/Affine/CMakeLists.txt
@@ -3,7 +3,6 @@ add_mlir_library(MLIRAffineTransformsTestPasses
   TestAffineDataCopy.cpp
   TestAffineLoopUnswitching.cpp
   TestLoopPermutation.cpp
-  TestParallelismDetection.cpp
   TestVectorizationUtils.cpp
 
   EXCLUDE_FROM_LIBMLIR
diff --git a/mlir/test/lib/Dialect/Affine/TestParallelismDetection.cpp b/mlir/test/lib/Dialect/Affine/TestParallelismDetection.cpp
deleted file mode 100644
index b19e260316939..0000000000000
--- a/mlir/test/lib/Dialect/Affine/TestParallelismDetection.cpp
+++ /dev/null
@@ -1,47 +0,0 @@
-//===- ParallelismDetection.cpp - Parallelism Detection pass ------------*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements a pass to detect parallel affine 'affine.for' ops.
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Analysis/Utils.h"
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/IR/Builders.h"
-#include "mlir/Pass/Pass.h"
-
-using namespace mlir;
-
-namespace {
-
-struct TestParallelismDetection
-    : public PassWrapper<TestParallelismDetection, FunctionPass> {
-  void runOnFunction() override;
-};
-
-} // end anonymous namespace
-
-// Walks the function and emits a note for all 'affine.for' ops detected as
-// parallel.
-void TestParallelismDetection::runOnFunction() {
-  FuncOp f = getFunction();
-  OpBuilder b(f.getBody());
-  f.walk([&](AffineForOp forOp) {
-    if (isLoopParallel(forOp))
-      forOp.emitRemark("parallel loop");
-    else
-      forOp.emitRemark("sequential loop");
-  });
-}
-
-namespace mlir {
-void registerTestParallelismDetection() {
-  PassRegistration<TestParallelismDetection> pass(
-      "test-detect-parallel", "Test parallelism detection ");
-}
-} // namespace mlir
diff --git a/mlir/test/lib/Dialect/Test/CMakeLists.txt b/mlir/test/lib/Dialect/Test/CMakeLists.txt
index 81bd888239605..b48d464e4317a 100644
--- a/mlir/test/lib/Dialect/Test/CMakeLists.txt
+++ b/mlir/test/lib/Dialect/Test/CMakeLists.txt
@@ -3,6 +3,11 @@ set(LLVM_OPTIONAL_SOURCES
   TestPatterns.cpp
 )
 
+set(LLVM_TARGET_DEFINITIONS TestInterfaces.td)
+mlir_tablegen(TestTypeInterfaces.h.inc -gen-type-interface-decls)
+mlir_tablegen(TestTypeInterfaces.cpp.inc -gen-type-interface-defs)
+add_public_tablegen_target(MLIRTestInterfaceIncGen)
+
 set(LLVM_TARGET_DEFINITIONS TestOps.td)
 mlir_tablegen(TestOps.h.inc -gen-op-decls)
 mlir_tablegen(TestOps.cpp.inc -gen-op-defs)
@@ -22,6 +27,7 @@ add_mlir_library(MLIRTestDialect
   EXCLUDE_FROM_LIBMLIR
 
   DEPENDS
+  MLIRTestInterfaceIncGen
   MLIRTestOpsIncGen
 
   LINK_LIBS PUBLIC
diff --git a/mlir/test/lib/Dialect/Test/TestDialect.cpp b/mlir/test/lib/Dialect/Test/TestDialect.cpp
index e4b1793c9399b..934e30d23fcf7 100644
--- a/mlir/test/lib/Dialect/Test/TestDialect.cpp
+++ b/mlir/test/lib/Dialect/Test/TestDialect.cpp
@@ -7,7 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "TestDialect.h"
+#include "TestTypes.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
+#include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/Function.h"
 #include "mlir/IR/Module.h"
 #include "mlir/IR/PatternMatch.h"
@@ -135,9 +137,21 @@ TestDialect::TestDialect(MLIRContext *context)
       >();
   addInterfaces<TestOpAsmInterface, TestOpFolderDialectInterface,
                 TestInlinerInterface>();
+  addTypes<TestType>();
   allowUnknownOperations();
 }
 
+Type TestDialect::parseType(DialectAsmParser &parser) const {
+  if (failed(parser.parseKeyword("test_type")))
+    return Type();
+  return TestType::get(getContext());
+}
+
+void TestDialect::printType(Type type, DialectAsmPrinter &printer) const {
+  assert(type.isa<TestType>() && "unexpected type");
+  printer << "test_type";
+}
+
 LogicalResult TestDialect::verifyOperationAttribute(Operation *op,
                                                     NamedAttribute namedAttr) {
   if (namedAttr.first == "test.invalid_attr")
@@ -518,6 +532,77 @@ void StringAttrPrettyNameOp::getAsmResultNames(
         setNameFn(getResult(i), str.getValue());
 }
 
+//===----------------------------------------------------------------------===//
+// RegionIfOp
+//===----------------------------------------------------------------------===//
+
+static void print(OpAsmPrinter &p, RegionIfOp op) {
+  p << RegionIfOp::getOperationName() << " ";
+  p.printOperands(op.getOperands());
+  p << ": " << op.getOperandTypes();
+  p.printArrowTypeList(op.getResultTypes());
+  p << " then";
+  p.printRegion(op.thenRegion(),
+                /*printEntryBlockArgs=*/true,
+                /*printBlockTerminators=*/true);
+  p << " else";
+  p.printRegion(op.elseRegion(),
+                /*printEntryBlockArgs=*/true,
+                /*printBlockTerminators=*/true);
+  p << " join";
+  p.printRegion(op.joinRegion(),
+                /*printEntryBlockArgs=*/true,
+                /*printBlockTerminators=*/true);
+}
+
+static ParseResult parseRegionIfOp(OpAsmParser &parser,
+                                   OperationState &result) {
+  SmallVector<OpAsmParser::OperandType, 2> operandInfos;
+  SmallVector<Type, 2> operandTypes;
+
+  result.regions.reserve(3);
+  Region *thenRegion = result.addRegion();
+  Region *elseRegion = result.addRegion();
+  Region *joinRegion = result.addRegion();
+
+  // Parse operand, type and arrow type lists.
+  if (parser.parseOperandList(operandInfos) ||
+      parser.parseColonTypeList(operandTypes) ||
+      parser.parseArrowTypeList(result.types))
+    return failure();
+
+  // Parse all attached regions.
+  if (parser.parseKeyword("then") || parser.parseRegion(*thenRegion, {}, {}) ||
+      parser.parseKeyword("else") || parser.parseRegion(*elseRegion, {}, {}) ||
+      parser.parseKeyword("join") || parser.parseRegion(*joinRegion, {}, {}))
+    return failure();
+
+  return parser.resolveOperands(operandInfos, operandTypes,
+                                parser.getCurrentLocation(), result.operands);
+}
+
+OperandRange RegionIfOp::getSuccessorEntryOperands(unsigned index) {
+  assert(index < 2 && "invalid region index");
+  return getOperands();
+}
+
+void RegionIfOp::getSuccessorRegions(
+    Optional<unsigned> index, ArrayRef<Attribute> operands,
+    SmallVectorImpl<RegionSuccessor> &regions) {
+  // We always branch to the join region.
+  if (index.hasValue()) {
+    if (index.getValue() < 2)
+      regions.push_back(RegionSuccessor(&joinRegion(), getJoinArgs()));
+    else
+      regions.push_back(RegionSuccessor(getResults()));
+    return;
+  }
+
+  // The then and else regions are the entry regions of this op.
+  regions.push_back(RegionSuccessor(&thenRegion(), getThenArgs()));
+  regions.push_back(RegionSuccessor(&elseRegion(), getElseArgs()));
+}
+
 //===----------------------------------------------------------------------===//
 // Dialect Registration
 //===----------------------------------------------------------------------===//
@@ -527,6 +612,7 @@ static mlir::DialectRegistration<mlir::TestDialect> testDialect;
 
 #include "TestOpEnums.cpp.inc"
 #include "TestOpStructs.cpp.inc"
+#include "TestTypeInterfaces.cpp.inc"
 
 #define GET_OP_CLASSES
 #include "TestOps.cpp.inc"
diff --git a/mlir/test/lib/Dialect/Test/TestInterfaces.td b/mlir/test/lib/Dialect/Test/TestInterfaces.td
new file mode 100644
index 0000000000000..08f07cf3a11d3
--- /dev/null
+++ b/mlir/test/lib/Dialect/Test/TestInterfaces.td
@@ -0,0 +1,46 @@
+//===-- TestInterfaces.td - Test dialect interfaces --------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TEST_INTERFACES
+#define TEST_INTERFACES
+
+include "mlir/IR/OpBase.td"
+
+// A type interface used to test the ODS generation of type interfaces.
+def TestTypeInterface : TypeInterface<"TestTypeInterface"> {
+  let methods = [
+    InterfaceMethod<"Prints the type name.",
+      "void", "printTypeA", (ins "Location":$loc), [{
+        emitRemark(loc) << $_type << " - TestA";
+      }]
+    >,
+    InterfaceMethod<"Prints the type name.",
+      "void", "printTypeB", (ins "Location":$loc),
+      [{}], /*defaultImplementation=*/[{
+        emitRemark(loc) << $_type << " - TestB";
+      }]
+    >,
+    InterfaceMethod<"Prints the type name.",
+      "void", "printTypeC", (ins "Location":$loc)
+    >,
+  ];
+  let extraClassDeclaration = [{
+    /// Prints the type name.
+    void printTypeD(Location loc) const {
+      emitRemark(loc) << *this << " - TestD";
+    }
+  }];
+  let extraTraitClassDeclaration = [{
+    /// Prints the type name.
+    void printTypeE(Location loc) const {
+      emitRemark(loc) << $_type << " - TestE";
+    }
+  }];
+}
+
+#endif // TEST_INTERFACES
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 630a7c58e449b..19e636b3df322 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -90,6 +90,10 @@ def MultiTensorRankOf : TEST_Op<"multi_tensor_rank_of"> {
   );
 }
 
+def TEST_TestType : DialectType<Test_Dialect,
+    CPred<"$_self.isa<::mlir::TestType>()">, "test">,
+    BuildableType<"$_builder.getType<::mlir::TestType>()">;
+
 //===----------------------------------------------------------------------===//
 // Test Symbols
 //===----------------------------------------------------------------------===//
@@ -124,6 +128,12 @@ def MixedNormalVariadicOperandOp : TEST_Op<
     Variadic<AnyTensor>:$input3
   );
 }
+def VariadicWithSameOperandsResult :
+      TEST_Op<"variadic_with_same_operand_results",
+              [SameOperandsAndResultType]> {
+  let arguments = (ins Variadic<AnySignlessInteger>:$operands);
+  let results = (outs AnySignlessInteger:$result);
+}
 
 //===----------------------------------------------------------------------===//
 // Test Results
@@ -602,6 +612,20 @@ def OpJ : TEST_Op<"op_j">, Arguments<(ins)>, Results<(outs I32)>;
 def OpK : TEST_Op<"op_k">, Arguments<(ins)>, Results<(outs I32)>;
 def : Pat<(OpJ), (OpK)>;
 
+// Test that natives calls are only called once during rewrites.
+def OpM : TEST_Op<"op_m"> {
+  let arguments = (ins I32, OptionalAttr<I32Attr>:$optional_attr);
+  let results = (outs I32);
+}
+// Pattern add the argument plus a increasing static number hidden in
+// OpMTest function. That value is set into the optional argument.
+// That way, we will know if operations is called once or twice.
+def OpMGetNullAttr : NativeCodeCall<"Attribute()">;
+def OpMAttributeIsNull : Constraint<CPred<"! ($_self)">, "Attribute is null">;
+def OpMVal : NativeCodeCall<"OpMTest($_builder, $0)">;
+def : Pat<(OpM $attr, $optAttr), (OpM $attr, (OpMVal $attr) ),
+    [(OpMAttributeIsNull:$optAttr)]>;
+
 // Test `$_` for ignoring op argument match.
 def TestIgnoreArgMatchSrcOp : TEST_Op<"ignore_arg_match_src"> {
   let arguments = (ins
@@ -1320,6 +1344,54 @@ def FormatTwoVariadicOperandsNoBuildableTypeOp
   }];
 }
 
+def FormatInferVariadicTypeFromNonVariadic
+    : TEST_Op<"format_infer_variadic_type_from_non_variadic",
+              [SameOperandsAndResultType]> {
+  let arguments = (ins Variadic<AnyType>:$operands);
+  let results = (outs AnyType:$result);
+  let assemblyFormat = "$operands attr-dict `:` type($result)";
+}
+
+//===----------------------------------------------------------------------===//
+// AllTypesMatch type inference
+//===----------------------------------------------------------------------===//
+
+def FormatAllTypesMatchVarOp : TEST_Op<"format_all_types_match_var", [
+    AllTypesMatch<["value1", "value2", "result"]>
+  ]> {
+  let arguments = (ins AnyType:$value1, AnyType:$value2);
+  let results = (outs AnyType:$result);
+  let assemblyFormat = "attr-dict $value1 `,` $value2 `:` type($value1)";
+}
+
+def FormatAllTypesMatchAttrOp : TEST_Op<"format_all_types_match_attr", [
+    AllTypesMatch<["value1", "value2", "result"]>
+  ]> {
+  let arguments = (ins AnyAttr:$value1, AnyType:$value2);
+  let results = (outs AnyType:$result);
+  let assemblyFormat = "attr-dict $value1 `,` $value2";
+}
+
+//===----------------------------------------------------------------------===//
+// TypesMatchWith type inference
+//===----------------------------------------------------------------------===//
+
+def FormatTypesMatchVarOp : TEST_Op<"format_types_match_var", [
+    TypesMatchWith<"result type matches operand", "value", "result", "$_self">
+  ]> {
+  let arguments = (ins AnyType:$value);
+  let results = (outs AnyType:$result);
+  let assemblyFormat = "attr-dict $value `:` type($value)";
+}
+
+def FormatTypesMatchAttrOp : TEST_Op<"format_types_match_attr", [
+    TypesMatchWith<"result type matches constant", "value", "result", "$_self">
+  ]> {
+  let arguments = (ins AnyAttr:$value);
+  let results = (outs AnyType:$result);
+  let assemblyFormat = "attr-dict $value";
+}
+
 //===----------------------------------------------------------------------===//
 // Test SideEffects
 //===----------------------------------------------------------------------===//
@@ -1329,4 +1401,47 @@ def SideEffectOp : TEST_Op<"side_effect_op",
   let results = (outs AnyType:$result);
 }
 
+//===----------------------------------------------------------------------===//
+// Test RegionBranchOpInterface
+//===----------------------------------------------------------------------===//
+
+def RegionIfYieldOp : TEST_Op<"region_if_yield",
+      [NoSideEffect, ReturnLike, Terminator]> {
+  let arguments = (ins Variadic<AnyType>:$results);
+  let assemblyFormat = [{
+    $results `:` type($results) attr-dict
+  }];
+}
+
+def RegionIfOp : TEST_Op<"region_if",
+      [DeclareOpInterfaceMethods<RegionBranchOpInterface>,
+       SingleBlockImplicitTerminator<"RegionIfYieldOp">,
+       RecursiveSideEffects]> {
+  let description =[{
+    Represents an abstract if-then-else-join pattern. In this context, the then
+    and else regions jump to the join region, which finally returns to its
+    parent op.
+    }];
+
+  let printer = [{ return ::print(p, *this); }];
+  let parser = [{ return ::parseRegionIfOp(parser, result); }];
+  let arguments = (ins Variadic<AnyType>);
+  let results = (outs Variadic<AnyType>:$results);
+  let regions = (region SizedRegion<1>:$thenRegion,
+                        AnyRegion:$elseRegion,
+                        AnyRegion:$joinRegion);
+  let extraClassDeclaration = [{
+    Block::BlockArgListType getThenArgs() {
+      return getBody(0)->getArguments();
+    }
+    Block::BlockArgListType getElseArgs() {
+      return getBody(1)->getArguments();
+    }
+    Block::BlockArgListType getJoinArgs() {
+      return getBody(2)->getArguments();
+    }
+    OperandRange getSuccessorEntryOperands(unsigned index);
+  }];
+}
+
 #endif // TEST_OPS
diff --git a/mlir/test/lib/Dialect/Test/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
index 60f663f75adcc..c471cd3ead3e8 100644
--- a/mlir/test/lib/Dialect/Test/TestPatterns.cpp
+++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
@@ -32,6 +32,16 @@ static void handleNoResultOp(PatternRewriter &rewriter,
                                     op.operand());
 }
 
+// Test that natives calls are only called once during rewrites.
+// OpM_Test will return Pi, increased by 1 for each subsequent calls.
+// This let us check the number of times OpM_Test was called by inspecting
+// the returned value in the MLIR output.
+static int64_t opMIncreasingValue = 314159265;
+static Attribute OpMTest(PatternRewriter &rewriter, Value val) {
+  int64_t i = opMIncreasingValue++;
+  return rewriter.getIntegerAttr(rewriter.getIntegerType(32), i);
+}
+
 namespace {
 #include "TestPatterns.inc"
 } // end anonymous namespace
@@ -101,7 +111,7 @@ static void invokeCreateWithInferredReturnType(Operation *op) {
               context, llvm::None, values, op->getAttrDictionary(),
               op->getRegions(), inferredReturnTypes))) {
         OperationState state(location, OpTy::getOperationName());
-        // TODO(jpienaar): Expand to regions.
+        // TODO: Expand to regions.
         OpTy::build(b, state, values, op->getAttrs());
         (void)b.createOperation(state);
       }
diff --git a/mlir/test/lib/Dialect/Test/TestTypes.h b/mlir/test/lib/Dialect/Test/TestTypes.h
new file mode 100644
index 0000000000000..0596f61c1fa1c
--- /dev/null
+++ b/mlir/test/lib/Dialect/Test/TestTypes.h
@@ -0,0 +1,44 @@
+//===- TestTypes.h - MLIR Test Dialect Types --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains types defined by the TestDialect for testing various
+// features of MLIR.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TESTTYPES_H
+#define MLIR_TESTTYPES_H
+
+#include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/Types.h"
+
+namespace mlir {
+
+#include "TestTypeInterfaces.h.inc"
+
+/// This class is a simple test type that uses a generated interface.
+struct TestType : public Type::TypeBase<TestType, Type, TypeStorage,
+                                        TestTypeInterface::Trait> {
+  using Base::Base;
+
+  static bool kindof(unsigned kind) {
+    return kind == Type::Kind::FIRST_PRIVATE_EXPERIMENTAL_9_TYPE;
+  }
+
+  static TestType get(MLIRContext *context) {
+    return Base::get(context, Type::Kind::FIRST_PRIVATE_EXPERIMENTAL_9_TYPE);
+  }
+
+  /// Provide a definition for the necessary interface methods.
+  void printTypeC(Location loc) const {
+    emitRemark(loc) << *this << " - TestC";
+  }
+};
+} // end namespace mlir
+
+#endif // MLIR_TESTTYPES_H
diff --git a/mlir/test/lib/IR/CMakeLists.txt b/mlir/test/lib/IR/CMakeLists.txt
index 0a55a82b1c3d7..5456dc9e8816e 100644
--- a/mlir/test/lib/IR/CMakeLists.txt
+++ b/mlir/test/lib/IR/CMakeLists.txt
@@ -1,6 +1,7 @@
 # Exclude tests from libMLIR.so
 add_mlir_library(MLIRTestIR
   TestFunc.cpp
+  TestInterfaces.cpp
   TestMatchers.cpp
   TestSideEffects.cpp
   TestSymbolUses.cpp
diff --git a/mlir/test/lib/IR/TestInterfaces.cpp b/mlir/test/lib/IR/TestInterfaces.cpp
new file mode 100644
index 0000000000000..c48d582c01c3d
--- /dev/null
+++ b/mlir/test/lib/IR/TestInterfaces.cpp
@@ -0,0 +1,41 @@
+//===- TestInterfaces.cpp - Test interface generation and application -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestTypes.h"
+#include "mlir/Pass/Pass.h"
+
+using namespace mlir;
+
+namespace {
+/// This test checks various aspects of Type interface generation and
+/// application.
+struct TestTypeInterfaces
+    : public PassWrapper<TestTypeInterfaces, OperationPass<ModuleOp>> {
+  void runOnOperation() override {
+    getOperation().walk([](Operation *op) {
+      for (Type type : op->getResultTypes()) {
+        if (auto testInterface = type.dyn_cast<TestTypeInterface>()) {
+          testInterface.printTypeA(op->getLoc());
+          testInterface.printTypeB(op->getLoc());
+          testInterface.printTypeC(op->getLoc());
+          testInterface.printTypeD(op->getLoc());
+        }
+        if (auto testType = type.dyn_cast<TestType>())
+          testType.printTypeE(op->getLoc());
+      }
+    });
+  }
+};
+} // end anonymous namespace
+
+namespace mlir {
+void registerTestInterfaces() {
+  PassRegistration<TestTypeInterfaces> pass("test-type-interfaces",
+                                            "Test type interface support.");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Reducer/CMakeLists.txt b/mlir/test/lib/Reducer/CMakeLists.txt
new file mode 100644
index 0000000000000..61743b6097d61
--- /dev/null
+++ b/mlir/test/lib/Reducer/CMakeLists.txt
@@ -0,0 +1,17 @@
+# Exclude tests from libMLIR.so
+add_mlir_library(MLIRTestReducer
+  MLIRTestReducer.cpp
+
+  EXCLUDE_FROM_LIBMLIR
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/IR
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRPass
+  MLIRSupport
+  )
diff --git a/mlir/test/lib/Reducer/MLIRTestReducer.cpp b/mlir/test/lib/Reducer/MLIRTestReducer.cpp
new file mode 100644
index 0000000000000..cb9d219349095
--- /dev/null
+++ b/mlir/test/lib/Reducer/MLIRTestReducer.cpp
@@ -0,0 +1,54 @@
+//===- TestReducer.cpp - Test MLIR Reduce ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a pass that reproduces errors based on trivially defined
+// patterns. It is used as a buggy optimization pass for the purpose of testing
+// the MLIR Reduce tool.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Pass/Pass.h"
+
+#define PASS_NAME "test-mlir-reducer"
+
+using namespace mlir;
+
+static llvm::cl::OptionCategory clOptionsCategory(PASS_NAME " options");
+
+namespace {
+
+/// This pass looks for for the presence of an operation with the name
+/// "crashOp" in the input MLIR file and crashes the mlir-opt tool if the
+/// operation is found.
+struct TestReducer : public PassWrapper<TestReducer, FunctionPass> {
+  TestReducer() = default;
+  TestReducer(const TestReducer &pass){};
+  void runOnFunction() override;
+};
+
+} // end anonymous namespace
+
+void TestReducer::runOnFunction() {
+  for (auto &op : getOperation())
+    op.walk([&](Operation *op) {
+      StringRef opName = op->getName().getStringRef();
+
+      if (opName == "test.crashOp") {
+        llvm::errs() << "MLIR Reducer Test generated failure: Found "
+                        "\"crashOp\" operation\n";
+        exit(1);
+      }
+    });
+}
+
+namespace mlir {
+void registerTestReducer() {
+  PassRegistration<TestReducer>(
+      PASS_NAME, "Tests MLIR Reduce tool by generating failures");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/CMakeLists.txt b/mlir/test/lib/Transforms/CMakeLists.txt
index db864999a440f..cdfd4e8a815b0 100644
--- a/mlir/test/lib/Transforms/CMakeLists.txt
+++ b/mlir/test/lib/Transforms/CMakeLists.txt
@@ -39,7 +39,7 @@ add_mlir_library(MLIRTestTransforms
   MLIRAnalysis
   MLIREDSC
   MLIRGPU
-  MLIRGPUtoGPURuntimeTransforms
+  MLIRGPUToGPURuntimeTransforms
   MLIRLinalgOps
   MLIRLinalgTransforms
   MLIRNVVMIR
diff --git a/mlir/test/lib/Transforms/TestInlining.cpp b/mlir/test/lib/Transforms/TestInlining.cpp
index 1cc762836b3c2..27c0d08462693 100644
--- a/mlir/test/lib/Transforms/TestInlining.cpp
+++ b/mlir/test/lib/Transforms/TestInlining.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// TODO(riverriddle) This pass is only necessary because the main inlining pass
+// TODO: This pass is only necessary because the main inlining pass
 // has no abstracted away the call+callee relationship. When the inlining
 // interface has this support, this pass should be removed.
 //
diff --git a/mlir/test/lib/Transforms/TestLinalgTransforms.cpp b/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
index 44a0e8cbdb142..4fb378c5ab8a5 100644
--- a/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
@@ -54,6 +54,11 @@ struct TestLinalgTransforms
       llvm::cl::desc(
           "Test a fused pass that forwards linalg.copy to vector.transfer"),
       llvm::cl::init(false)};
+  Option<bool> testGenericToVectorPattern{
+      *this, "test-contraction-to-vector-patterns",
+      llvm::cl::desc("Test a set of patterns that rewrite a linalg contraction "
+                     "in vector.contract form"),
+      llvm::cl::init(false)};
 };
 } // end anonymous namespace
 
@@ -244,8 +249,8 @@ static LogicalResult copyCallBackFn(OpBuilder &b, Value src, Value dst,
   return success();
 }
 
-void fillPromotionCallBackPatterns(MLIRContext *ctx,
-                                   OwningRewritePatternList &patterns) {
+static void fillPromotionCallBackPatterns(MLIRContext *ctx,
+                                          OwningRewritePatternList &patterns) {
   patterns.insert<LinalgTilingPattern<MatmulOp>>(
       ctx, LinalgTilingOptions().setTileSizes({16, 16, 16}),
       LinalgMarker(Identifier::get("START", ctx),
@@ -300,6 +305,16 @@ static void applyVectorTransferForwardingPatterns(FuncOp funcOp) {
   applyPatternsAndFoldGreedily(funcOp, forwardPattern);
 }
 
+static void applyContractionToVectorPatterns(FuncOp funcOp) {
+  OwningRewritePatternList patterns;
+  patterns.insert<LinalgVectorizationPattern<BatchMatmulOp>,
+                  LinalgVectorizationPattern<MatmulOp>,
+                  LinalgVectorizationPattern<MatvecOp>,
+                  LinalgVectorizationPattern<DotOp>,
+                  LinalgVectorizationPattern<GenericOp>>(funcOp.getContext());
+  applyPatternsAndFoldGreedily(funcOp, patterns);
+}
+
 /// Apply transformations specified as patterns.
 void TestLinalgTransforms::runOnFunction() {
   auto lambda = [&](void *) {
@@ -323,6 +338,8 @@ void TestLinalgTransforms::runOnFunction() {
                                        testMatmulToVectorPatterns2dTiling);
   if (testVectorTransferForwardingPatterns)
     return applyVectorTransferForwardingPatterns(getFunction());
+  if (testGenericToVectorPattern)
+    return applyContractionToVectorPatterns(getFunction());
 }
 
 namespace mlir {
diff --git a/mlir/test/lib/Transforms/TestMemRefBoundCheck.cpp b/mlir/test/lib/Transforms/TestMemRefBoundCheck.cpp
index 087ea4fdde94f..4ae9e505cab36 100644
--- a/mlir/test/lib/Transforms/TestMemRefBoundCheck.cpp
+++ b/mlir/test/lib/Transforms/TestMemRefBoundCheck.cpp
@@ -41,7 +41,7 @@ void TestMemRefBoundCheck::runOnFunction() {
         .Case<AffineReadOpInterface, AffineWriteOpInterface>(
             [](auto op) { boundCheckLoadOrStoreOp(op); });
 
-    // TODO(bondhugula): do this for DMA ops as well.
+    // TODO: do this for DMA ops as well.
   });
 }
 
diff --git a/mlir/test/lib/Transforms/TestMemRefDependenceCheck.cpp b/mlir/test/lib/Transforms/TestMemRefDependenceCheck.cpp
index 34db53b6ce1e8..787c92737daf0 100644
--- a/mlir/test/lib/Transforms/TestMemRefDependenceCheck.cpp
+++ b/mlir/test/lib/Transforms/TestMemRefDependenceCheck.cpp
@@ -25,7 +25,7 @@ using namespace mlir;
 
 namespace {
 
-// TODO(andydavis) Add common surrounding loop depth-wise dependence checks.
+// TODO: Add common surrounding loop depth-wise dependence checks.
 /// Checks dependences between all pairs of memref accesses in a Function.
 struct TestMemRefDependenceCheck
     : public PassWrapper<TestMemRefDependenceCheck, FunctionPass> {
@@ -85,7 +85,7 @@ static void checkDependences(ArrayRef<Operation *> loadsAndStores) {
             &dependenceComponents);
         assert(result.value != DependenceResult::Failure);
         bool ret = hasDependence(result);
-        // TODO(andydavis) Print dependence type (i.e. RAW, etc) and print
+        // TODO: Print dependence type (i.e. RAW, etc) and print
         // distance vectors as: ([2, 3], [0, 10]). Also, shorten distance
         // vectors from ([1, 1], [3, 3]) to (1, 3).
         srcOpInst->emitRemark("dependence from ")
@@ -103,7 +103,7 @@ void TestMemRefDependenceCheck::runOnFunction() {
   // Collect the loads and stores within the function.
   loadsAndStores.clear();
   getFunction().walk([&](Operation *op) {
-    if (isa<AffineLoadOp>(op) || isa<AffineStoreOp>(op))
+    if (isa<AffineLoadOp, AffineStoreOp>(op))
       loadsAndStores.push_back(op);
   });
 
diff --git a/mlir/test/lib/Transforms/TestVectorTransforms.cpp b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
index 22585fde4ff73..7e28ebbd9b721 100644
--- a/mlir/test/lib/Transforms/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
@@ -62,6 +62,8 @@ struct TestVectorContractionConversion
 
   void runOnFunction() override {
     OwningRewritePatternList patterns;
+
+    // Test on one pattern in isolation.
     if (lowerToOuterProduct) {
       VectorContractLowering lowering = VectorContractLowering::OuterProduct;
       VectorTransformsOptions options{lowering};
@@ -71,7 +73,8 @@ struct TestVectorContractionConversion
       return;
     }
 
-    VectorContractLowering contractLowering = VectorContractLowering::FMA;
+    // Test on all contract lowering patterns.
+    VectorContractLowering contractLowering = VectorContractLowering::Dot;
     if (lowerToFlatMatrix)
       contractLowering = VectorContractLowering::Matmul;
     VectorTransposeLowering transposeLowering =
@@ -84,6 +87,20 @@ struct TestVectorContractionConversion
   }
 };
 
+struct TestVectorUnrollingPatterns
+    : public PassWrapper<TestVectorUnrollingPatterns, FunctionPass> {
+  void runOnFunction() override {
+    MLIRContext *ctx = &getContext();
+    OwningRewritePatternList patterns;
+    patterns.insert<UnrollVectorPattern<AddFOp>>(ArrayRef<int64_t>{2, 2}, ctx);
+    patterns.insert<UnrollVectorPattern<vector::ContractionOp>>(
+        ArrayRef<int64_t>{2, 2, 2}, ctx);
+    populateVectorToVectorCanonicalizationPatterns(patterns, ctx);
+    populateVectorToVectorTransformationPatterns(patterns, ctx);
+    applyPatternsAndFoldGreedily(getFunction(), patterns);
+  }
+};
+
 } // end anonymous namespace
 
 namespace mlir {
@@ -99,5 +116,9 @@ void registerTestVectorConversions() {
   PassRegistration<TestVectorContractionConversion> contractionPass(
       "test-vector-contraction-conversion",
       "Test conversion patterns that lower contract ops in the vector dialect");
+
+  PassRegistration<TestVectorUnrollingPatterns> contractionUnrollingPass(
+      "test-vector-unrolling-patterns",
+      "Test conversion patterns to unroll contract ops in the vector dialect");
 }
 } // namespace mlir
diff --git a/mlir/test/lit.cfg.py b/mlir/test/lit.cfg.py
index bb0850ec67562..2ef17106dffd6 100644
--- a/mlir/test/lit.cfg.py
+++ b/mlir/test/lit.cfg.py
@@ -21,7 +21,7 @@
 config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
 
 # suffixes: A list of file extensions to treat as test files.
-config.suffixes = ['.td', '.mlir', '.toy', '.ll', '.tc']
+config.suffixes = ['.td', '.mlir', '.toy', '.ll', '.tc', '.py']
 
 # test_source_root: The root path where tests are located.
 config.test_source_root = os.path.dirname(__file__)
@@ -41,7 +41,8 @@
 # excludes: A list of directories to exclude from the testsuite. The 'Inputs'
 # subdirectories contain auxiliary inputs for various tests in their parent
 # directories.
-config.excludes = ['Inputs', 'CMakeLists.txt', 'README.txt', 'LICENSE.txt']
+config.excludes = ['Inputs', 'CMakeLists.txt', 'README.txt', 'LICENSE.txt',
+                   'lit.cfg.py', 'lit.site.cfg.py']
 
 # test_source_root: The root path where tests are located.
 config.test_source_root = os.path.dirname(__file__)
@@ -62,6 +63,7 @@
 
 # The following tools are optional
 tools.extend([
+    ToolSubst('%PYTHON', config.python_executable),
     ToolSubst('toy-ch1', unresolved='ignore'),
     ToolSubst('toy-ch2', unresolved='ignore'),
     ToolSubst('toy-ch3', unresolved='ignore'),
@@ -71,7 +73,7 @@
     ToolSubst('%linalg_test_lib_dir', config.linalg_test_lib_dir, unresolved='ignore'),
     ToolSubst('%mlir_runner_utils_dir', config.mlir_runner_utils_dir, unresolved='ignore'),
     ToolSubst('%rocm_wrapper_library_dir', config.rocm_wrapper_library_dir, unresolved='ignore'),
-    ToolSubst('%vulkan_wrapper_library_dir', config.vulkan_wrapper_library_dir, unresolved='ignore')
+    ToolSubst('%vulkan_wrapper_library_dir', config.vulkan_wrapper_library_dir, unresolved='ignore'),
 ])
 
 llvm_config.add_tool_substitutions(tools, tool_dirs)
@@ -89,3 +91,13 @@
 # to be available for JIT tests.
 if config.target_triple:
     config.available_features.add('default_triple')
+
+# Add the python path for both the source and binary tree.
+# Note that presently, the python sources come from the source tree and the
+# binaries come from the build tree. This should be unified to the build tree
+# by copying/linking sources to build.
+if config.enable_bindings_python:
+    llvm_config.with_environment('PYTHONPATH', [
+        os.path.join(config.mlir_src_root, "lib", "Bindings", "Python"),
+        os.path.join(config.mlir_obj_root, "lib", "Bindings", "Python"),
+    ], append_path=True)
diff --git a/mlir/test/lit.site.cfg.py.in b/mlir/test/lit.site.cfg.py.in
index b75518611cf27..1dc823239b080 100644
--- a/mlir/test/lit.site.cfg.py.in
+++ b/mlir/test/lit.site.cfg.py.in
@@ -43,6 +43,7 @@ config.rocm_wrapper_library_dir = "@MLIR_ROCM_WRAPPER_LIBRARY_DIR@"
 config.enable_rocm_runner = @MLIR_ROCM_RUNNER_ENABLED@
 config.vulkan_wrapper_library_dir = "@MLIR_VULKAN_WRAPPER_LIBRARY_DIR@"
 config.enable_vulkan_runner = @MLIR_VULKAN_RUNNER_ENABLED@
+config.enable_bindings_python = @MLIR_BINDINGS_PYTHON_ENABLED@
 
 # Support substitution of the tools_dir with user parameters. This is
 # used when we can't determine the tool dir at configuration time.
diff --git a/mlir/test/mlir-cpu-runner/unranked_memref.mlir b/mlir/test/mlir-cpu-runner/unranked_memref.mlir
index 0eb68ac033687..df760f593db2e 100644
--- a/mlir/test/mlir-cpu-runner/unranked_memref.mlir
+++ b/mlir/test/mlir-cpu-runner/unranked_memref.mlir
@@ -18,6 +18,21 @@
 // CHECK: rank = 0
 // 122 is ASCII for 'z'.
 // CHECK: [z]
+//
+// CHECK: rank = 2
+// CHECK-SAME: sizes = [4, 3]
+// CHECK-SAME: strides = [3, 1]
+// CHECK-COUNT-4: [1, 1, 1]
+//
+// CHECK: rank = 2
+// CHECK-SAME: sizes = [4, 3]
+// CHECK-SAME: strides = [3, 1]
+// CHECK-COUNT-4: [1, 1, 1]
+//
+// CHECK: rank = 2
+// CHECK-SAME: sizes = [4, 3]
+// CHECK-SAME: strides = [3, 1]
+// CHECK-COUNT-4: [1, 1, 1]
 func @main() -> () {
     %A = alloc() : memref<10x3xf32, 0>
     %f2 = constant 2.00000e+00 : f32
@@ -48,8 +63,40 @@ func @main() -> () {
     call @print_memref_i8(%U4) : (memref<*xi8>) -> ()
 
     dealloc %A : memref<10x3xf32, 0>
+
+    call @return_var_memref_caller() : () -> ()
+    call @return_two_var_memref_caller() : () -> ()
     return
 }
 
 func @print_memref_i8(memref<*xi8>) attributes { llvm.emit_c_interface }
 func @print_memref_f32(memref<*xf32>) attributes { llvm.emit_c_interface }
+
+func @return_two_var_memref_caller() {
+  %0 = alloca() : memref<4x3xf32>
+  %c0f32 = constant 1.0 : f32
+  linalg.fill(%0, %c0f32) : memref<4x3xf32>, f32
+  %1:2 = call @return_two_var_memref(%0) : (memref<4x3xf32>) -> (memref<*xf32>, memref<*xf32>)
+  call @print_memref_f32(%1#0) : (memref<*xf32>) -> ()
+  call @print_memref_f32(%1#1) : (memref<*xf32>) -> ()
+  return
+ }
+
+ func @return_two_var_memref(%arg0: memref<4x3xf32>) -> (memref<*xf32>, memref<*xf32>) {
+  %0 = memref_cast %arg0 : memref<4x3xf32> to memref<*xf32>
+  return %0, %0 : memref<*xf32>, memref<*xf32>
+}
+
+func @return_var_memref_caller() {
+  %0 = alloca() : memref<4x3xf32>
+  %c0f32 = constant 1.0 : f32
+  linalg.fill(%0, %c0f32) : memref<4x3xf32>, f32
+  %1 = call @return_var_memref(%0) : (memref<4x3xf32>) -> memref<*xf32>
+  call @print_memref_f32(%1) : (memref<*xf32>) -> ()
+  return
+}
+
+func @return_var_memref(%arg0: memref<4x3xf32>) -> memref<*xf32> {
+  %0 = memref_cast %arg0: memref<4x3xf32> to memref<*xf32>
+  return %0 : memref<*xf32>
+}
diff --git a/mlir/test/mlir-reduce/test-reducer-pass.mlir b/mlir/test/mlir-reduce/test-reducer-pass.mlir
new file mode 100644
index 0000000000000..da5b0c9633553
--- /dev/null
+++ b/mlir/test/mlir-reduce/test-reducer-pass.mlir
@@ -0,0 +1,7 @@
+// RUN: mlir-opt %s
+// RUN: not mlir-opt %s -test-mlir-reducer
+
+func @test() {
+  "test.crashOp"() : () -> ()
+  return
+}
diff --git a/mlir/test/mlir-reduce/test.sh b/mlir/test/mlir-reduce/test.sh
new file mode 100755
index 0000000000000..33d02f41a95e5
--- /dev/null
+++ b/mlir/test/mlir-reduce/test.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+#Replicate bug
+
+#Interesting behavior
+exit 1
\ No newline at end of file
diff --git a/mlir/test/mlir-reduce/testcase-linux.mlir b/mlir/test/mlir-reduce/testcase-linux.mlir
new file mode 100644
index 0000000000000..f2bb161bb5a69
--- /dev/null
+++ b/mlir/test/mlir-reduce/testcase-linux.mlir
@@ -0,0 +1,13 @@
+// UNSUPPORTED: -windows-
+// RUN: mlir-reduce %s -test %S/test.sh
+
+func @simple1(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
+  cond_br %arg0, ^bb1, ^bb2
+^bb1:
+  br ^bb3(%arg1 : memref<2xf32>)
+^bb2:
+  %0 = alloc() : memref<2xf32>
+  br ^bb3(%0 : memref<2xf32>)
+^bb3(%1: memref<2xf32>):
+  return
+}
\ No newline at end of file
diff --git a/mlir/test/mlir-tblgen/interfaces.mlir b/mlir/test/mlir-tblgen/interfaces.mlir
new file mode 100644
index 0000000000000..712d934991529
--- /dev/null
+++ b/mlir/test/mlir-tblgen/interfaces.mlir
@@ -0,0 +1,11 @@
+// RUN: mlir-opt -test-type-interfaces -allow-unregistered-dialect -verify-diagnostics %s
+
+// expected-remark@below {{'!test.test_type' - TestA}}
+// expected-remark@below {{'!test.test_type' - TestB}}
+// expected-remark@below {{'!test.test_type' - TestC}}
+// expected-remark@below {{'!test.test_type' - TestD}}
+// expected-remark@below {{'!test.test_type' - TestE}}
+%foo0 = "foo.test"() : () -> (!test.test_type)
+
+// Type without the test interface.
+%foo1 = "foo.test"() : () -> (i32)
diff --git a/mlir/test/mlir-tblgen/op-attribute.td b/mlir/test/mlir-tblgen/op-attribute.td
index 766c11e4692c0..0868e834bca4b 100644
--- a/mlir/test/mlir-tblgen/op-attribute.td
+++ b/mlir/test/mlir-tblgen/op-attribute.td
@@ -33,7 +33,7 @@ def AOp : NS_Op<"a_op", []> {
 // Test verify method
 // ---
 
-// DEF:      LogicalResult AOpAdaptor::verify
+// DEF:      ::mlir::LogicalResult AOpAdaptor::verify
 // DEF:      auto tblgen_aAttr = odsAttrs.get("aAttr");
 // DEF-NEXT: if (!tblgen_aAttr) return emitError(loc, "'test.a_op' op ""requires attribute 'aAttr'");
 // DEF:        if (!((some-condition))) return emitError(loc, "'test.a_op' op ""attribute 'aAttr' failed to satisfy constraint: some attribute kind");
@@ -58,14 +58,14 @@ def AOp : NS_Op<"a_op", []> {
 // DEF:      some-return-type AOp::bAttr() {
 // DEF-NEXT:   auto attr = bAttrAttr();
 // DEF-NEXT:   if (!attr)
-// DEF-NEXT:       return some-const-builder-call(mlir::Builder(this->getContext()), 4.2).some-convert-from-storage();
+// DEF-NEXT:       return some-const-builder-call(::mlir::Builder(this->getContext()), 4.2).some-convert-from-storage();
 // DEF-NEXT:   return attr.some-convert-from-storage();
 
 // DEF:      some-attr-kind AOp::cAttrAttr()
 // DEF-NEXT:   return this->getAttr("cAttr").dyn_cast_or_null<some-attr-kind>()
-// DEF:      Optional<some-return-type> AOp::cAttr() {
+// DEF:      ::llvm::Optional<some-return-type> AOp::cAttr() {
 // DEF-NEXT:   auto attr = cAttrAttr()
-// DEF-NEXT:   return attr ? Optional<some-return-type>(attr.some-convert-from-storage()) : (llvm::None);
+// DEF-NEXT:   return attr ? ::llvm::Optional<some-return-type>(attr.some-convert-from-storage()) : (::llvm::None);
 
 // Test setter methods
 // ---
@@ -91,7 +91,7 @@ def AOp : NS_Op<"a_op", []> {
 // DEF:        odsState.addAttribute("aAttr", some-const-builder-call(odsBuilder, aAttr));
 
 // DEF:      void AOp::build(
-// DEF:        ArrayRef<NamedAttribute> attributes
+// DEF:        ::llvm::ArrayRef<::mlir::NamedAttribute> attributes
 // DEF:      odsState.addAttributes(attributes);
 
 def SomeTypeAttr : TypeAttrBase<"SomeType", "some type attribute">;
@@ -120,35 +120,35 @@ def BOp : NS_Op<"b_op", []> {
 
 // DEF-LABEL: BOpAdaptor::verify
 // DEF: if (!((true)))
-// DEF: if (!((tblgen_bool_attr.isa<BoolAttr>())))
-// DEF: if (!(((tblgen_i32_attr.isa<IntegerAttr>())) && ((tblgen_i32_attr.cast<IntegerAttr>().getType().isSignlessInteger(32)))))
-// DEF: if (!(((tblgen_i64_attr.isa<IntegerAttr>())) && ((tblgen_i64_attr.cast<IntegerAttr>().getType().isSignlessInteger(64)))))
-// DEF: if (!(((tblgen_f32_attr.isa<FloatAttr>())) && ((tblgen_f32_attr.cast<FloatAttr>().getType().isF32()))))
-// DEF: if (!(((tblgen_f64_attr.isa<FloatAttr>())) && ((tblgen_f64_attr.cast<FloatAttr>().getType().isF64()))))
-// DEF: if (!((tblgen_str_attr.isa<StringAttr>())))
-// DEF: if (!((tblgen_elements_attr.isa<ElementsAttr>())))
-// DEF: if (!((tblgen_function_attr.isa<FlatSymbolRefAttr>())))
-// DEF: if (!(((tblgen_type_attr.isa<TypeAttr>())) && ((tblgen_type_attr.cast<TypeAttr>().getValue().isa<SomeType>()))))
-// DEF: if (!((tblgen_array_attr.isa<ArrayAttr>())))
-// DEF: if (!(((tblgen_some_attr_array.isa<ArrayAttr>())) && (llvm::all_of(tblgen_some_attr_array.cast<ArrayAttr>(), [](Attribute attr) { return (some-condition); }))))
-// DEF: if (!(((tblgen_type_attr.isa<TypeAttr>())) && ((tblgen_type_attr.cast<TypeAttr>().getValue().isa<Type>()))))
+// DEF: if (!((tblgen_bool_attr.isa<::mlir::BoolAttr>())))
+// DEF: if (!(((tblgen_i32_attr.isa<::mlir::IntegerAttr>())) && ((tblgen_i32_attr.cast<::mlir::IntegerAttr>().getType().isSignlessInteger(32)))))
+// DEF: if (!(((tblgen_i64_attr.isa<::mlir::IntegerAttr>())) && ((tblgen_i64_attr.cast<::mlir::IntegerAttr>().getType().isSignlessInteger(64)))))
+// DEF: if (!(((tblgen_f32_attr.isa<::mlir::FloatAttr>())) && ((tblgen_f32_attr.cast<::mlir::FloatAttr>().getType().isF32()))))
+// DEF: if (!(((tblgen_f64_attr.isa<::mlir::FloatAttr>())) && ((tblgen_f64_attr.cast<::mlir::FloatAttr>().getType().isF64()))))
+// DEF: if (!((tblgen_str_attr.isa<::mlir::StringAttr>())))
+// DEF: if (!((tblgen_elements_attr.isa<::mlir::ElementsAttr>())))
+// DEF: if (!((tblgen_function_attr.isa<::mlir::FlatSymbolRefAttr>())))
+// DEF: if (!(((tblgen_type_attr.isa<::mlir::TypeAttr>())) && ((tblgen_type_attr.cast<::mlir::TypeAttr>().getValue().isa<SomeType>()))))
+// DEF: if (!((tblgen_array_attr.isa<::mlir::ArrayAttr>())))
+// DEF: if (!(((tblgen_some_attr_array.isa<::mlir::ArrayAttr>())) && (::llvm::all_of(tblgen_some_attr_array.cast<::mlir::ArrayAttr>(), [](::mlir::Attribute attr) { return (some-condition); }))))
+// DEF: if (!(((tblgen_type_attr.isa<::mlir::TypeAttr>())) && ((tblgen_type_attr.cast<::mlir::TypeAttr>().getValue().isa<::mlir::Type>()))))
 
 // Test common attribute kind getters' return types
 // ---
 
-// DEF: Attribute BOp::any_attr()
+// DEF: ::mlir::Attribute BOp::any_attr()
 // DEF: bool BOp::bool_attr()
-// DEF: APInt BOp::i32_attr()
-// DEF: APInt BOp::i64_attr()
-// DEF: APFloat BOp::f32_attr()
-// DEF: APFloat BOp::f64_attr()
-// DEF: StringRef BOp::str_attr()
-// DEF: ElementsAttr BOp::elements_attr()
-// DEF: StringRef BOp::function_attr()
+// DEF: ::llvm::APInt BOp::i32_attr()
+// DEF: ::llvm::APInt BOp::i64_attr()
+// DEF: ::llvm::APFloat BOp::f32_attr()
+// DEF: ::llvm::APFloat BOp::f64_attr()
+// DEF: ::llvm::StringRef BOp::str_attr()
+// DEF: ::mlir::ElementsAttr BOp::elements_attr()
+// DEF: ::llvm::StringRef BOp::function_attr()
 // DEF: SomeType BOp::type_attr()
-// DEF: ArrayAttr BOp::array_attr()
-// DEF: ArrayAttr BOp::some_attr_array()
-// DEF: Type BOp::type_attr()
+// DEF: ::mlir::ArrayAttr BOp::array_attr()
+// DEF: ::mlir::ArrayAttr BOp::some_attr_array()
+// DEF: ::mlir::Type BOp::type_attr()
 
 // Test building constant values for array attribute kinds
 // ---
@@ -164,11 +164,11 @@ def COp : NS_Op<"c_op", []> {
 }
 
 // DEF-LABEL: COp definitions
-// DEF: mlir::Builder(this->getContext()).getI32ArrayAttr({1, 2})
-// DEF: mlir::Builder(this->getContext()).getI64ArrayAttr({3, 4})
-// DEF: mlir::Builder(this->getContext()).getF32ArrayAttr({5.f, 6.f})
-// DEF: mlir::Builder(this->getContext()).getF64ArrayAttr({7., 8.})
-// DEF: mlir::Builder(this->getContext()).getStrArrayAttr({"a", "b"})
+// DEF: ::mlir::Builder(this->getContext()).getI32ArrayAttr({1, 2})
+// DEF: ::mlir::Builder(this->getContext()).getI64ArrayAttr({3, 4})
+// DEF: ::mlir::Builder(this->getContext()).getF32ArrayAttr({5.f, 6.f})
+// DEF: ::mlir::Builder(this->getContext()).getF64ArrayAttr({7., 8.})
+// DEF: ::mlir::Builder(this->getContext()).getStrArrayAttr({"a", "b"})
 
 
 // Test builder method which takes unwrapped values for attributes
@@ -196,7 +196,7 @@ def DOp : NS_Op<"d_op", []> {
 }
 
 // DECL-LABEL: DOp declarations
-// DECL: static void build({{.*}}, APInt i32_attr, APFloat f64_attr, StringRef str_attr, bool bool_attr, ::SomeI32Enum enum_attr, APInt dv_i32_attr, APFloat dv_f64_attr, StringRef dv_str_attr = "abc", bool dv_bool_attr = true, ::SomeI32Enum dv_enum_attr = ::SomeI32Enum::case5)
+// DECL: static void build({{.*}}, ::llvm::APInt i32_attr, ::llvm::APFloat f64_attr, ::llvm::StringRef str_attr, bool bool_attr, ::SomeI32Enum enum_attr, ::llvm::APInt dv_i32_attr, ::llvm::APFloat dv_f64_attr, ::llvm::StringRef dv_str_attr = "abc", bool dv_bool_attr = true, ::SomeI32Enum dv_enum_attr = ::SomeI32Enum::case5)
 
 // DEF-LABEL: DOp definitions
 // DEF: odsState.addAttribute("str_attr", odsBuilder.getStringAttr(str_attr));
@@ -209,10 +209,10 @@ def DerivedTypeAttrOp : NS_Op<"derived_type_attr_op", []> {
   DerivedTypeAttr element_dtype = DerivedTypeAttr<"return output().getType();">;
 }
 
-// DECL: class DerivedTypeAttrOp : public Op
+// DECL: class DerivedTypeAttrOp : public ::mlir::Op
 // DECL-SAME: DerivedAttributeOpInterface::Trait
 // DECL: static bool isDerivedAttribute
-// DEF: bool DerivedTypeAttrOp::isDerivedAttribute(StringRef name) {
+// DEF: bool DerivedTypeAttrOp::isDerivedAttribute(::llvm::StringRef name) {
 // DEF:   if (name == "element_dtype") return true;
 // DEF:   return false;
 // DEF: }
@@ -238,7 +238,7 @@ def EOp : NS_Op<"e_op", []> {
 }
 
 // DECL-LABEL: EOp declarations
-// DECL: static void build({{.*}}, APInt i32_attr, APInt dv_i32_attr, APFloat f64_attr, APFloat dv_f64_attr, StringRef str_attr, StringRef dv_str_attr, bool bool_attr, bool dv_bool_attr, ::SomeI32Enum enum_attr, ::SomeI32Enum dv_enum_attr = ::SomeI32Enum::case5)
+// DECL: static void build({{.*}}, ::llvm::APInt i32_attr, ::llvm::APInt dv_i32_attr, ::llvm::APFloat f64_attr, ::llvm::APFloat dv_f64_attr, ::llvm::StringRef str_attr, ::llvm::StringRef dv_str_attr, bool bool_attr, bool dv_bool_attr, ::SomeI32Enum enum_attr, ::SomeI32Enum dv_enum_attr = ::SomeI32Enum::case5)
 
 // Test mixing operands and attributes in arbitrary order
 // ---
@@ -248,11 +248,11 @@ def MixOperandsAndAttrs : NS_Op<"mix_operands_and_attrs", []> {
 }
 
 // DEF-LABEL: MixOperandsAndAttrs definitions
-// DEF-DAG: Value MixOperandsAndAttrs::operand()
-// DEF-DAG: Value MixOperandsAndAttrs::otherArg()
-// DEF-DAG: void MixOperandsAndAttrs::build(OpBuilder &odsBuilder, OperationState &odsState, FloatAttr attr, Value operand, FloatAttr otherAttr, Value otherArg)
-// DEF-DAG: APFloat MixOperandsAndAttrs::attr()
-// DEF-DAG: APFloat MixOperandsAndAttrs::otherAttr()
+// DEF-DAG: ::mlir::Value MixOperandsAndAttrs::operand()
+// DEF-DAG: ::mlir::Value MixOperandsAndAttrs::otherArg()
+// DEF-DAG: void MixOperandsAndAttrs::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::FloatAttr attr, ::mlir::Value operand, ::mlir::FloatAttr otherAttr, ::mlir::Value otherArg)
+// DEF-DAG: ::llvm::APFloat MixOperandsAndAttrs::attr()
+// DEF-DAG: ::llvm::APFloat MixOperandsAndAttrs::otherAttr()
 
 // Test unit attributes.
 // ---
@@ -265,7 +265,7 @@ def UnitAttrOp : NS_Op<"unit_attr_op", []> {
 // DEF: bool UnitAttrOp::attr() {
 // DEF:   return {{.*}} != nullptr
 
-// DEF: build(OpBuilder &odsBuilder, OperationState &odsState, /*optional*/UnitAttr attr)
+// DEF: build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::UnitAttr attr)
 
 
 // Test elementAttr field of TypedArrayAttr.
diff --git a/mlir/test/mlir-tblgen/op-decl.td b/mlir/test/mlir-tblgen/op-decl.td
index f5bf03e57ce70..b596eee038291 100644
--- a/mlir/test/mlir-tblgen/op-decl.td
+++ b/mlir/test/mlir-tblgen/op-decl.td
@@ -1,4 +1,6 @@
 // RUN: mlir-tblgen -gen-op-decls -I %S/../../include %s | FileCheck  %s
+// RUN: mlir-tblgen -gen-op-decls -op-include-regex="test.a_op" -I %S/../../include %s | FileCheck  %s --check-prefix=REDUCE_INC
+// RUN: mlir-tblgen -gen-op-decls -op-exclude-regex="test.a_op" -I %S/../../include %s | FileCheck  %s --check-prefix=REDUCE_EXC
 
 include "mlir/IR/OpBase.td"
 include "mlir/Interfaces/InferTypeOpInterface.td"
@@ -49,44 +51,44 @@ def NS_AOp : NS_Op<"a_op", [IsolatedFromAbove, IsolatedFromAbove]> {
 
 // CHECK: class AOpAdaptor {
 // CHECK: public:
-// CHECK:   AOpAdaptor(ValueRange values
-// CHECK:   ValueRange getODSOperands(unsigned index);
-// CHECK:   Value a();
-// CHECK:   ValueRange b();
-// CHECK:   IntegerAttr attr1();
-// CHECL:   FloatAttr attr2();
+// CHECK:   AOpAdaptor(::mlir::ValueRange values
+// CHECK:   ::mlir::ValueRange getODSOperands(unsigned index);
+// CHECK:   ::mlir::Value a();
+// CHECK:   ::mlir::ValueRange b();
+// CHECK:   ::mlir::IntegerAttr attr1();
+// CHECL:   ::mlir::FloatAttr attr2();
 // CHECK: private:
-// CHECK:   ValueRange odsOperands;
+// CHECK:   ::mlir::ValueRange odsOperands;
 // CHECK: };
 
-// CHECK: class AOp : public Op<AOp, OpTrait::AtLeastNRegions<1>::Impl, OpTrait::AtLeastNResults<1>::Impl, OpTrait::ZeroSuccessor, OpTrait::AtLeastNOperands<1>::Impl, OpTrait::IsIsolatedFromAbove
+// CHECK: class AOp : public ::mlir::Op<AOp, OpTrait::AtLeastNRegions<1>::Impl, OpTrait::AtLeastNResults<1>::Impl, OpTrait::ZeroSuccessor, OpTrait::AtLeastNOperands<1>::Impl, OpTrait::IsIsolatedFromAbove
 // CHECK-NOT: OpTrait::IsIsolatedFromAbove
 // CHECK: public:
 // CHECK:   using Op::Op;
 // CHECK:   using Adaptor = AOpAdaptor;
-// CHECK:   static StringRef getOperationName();
-// CHECK:   Operation::operand_range getODSOperands(unsigned index);
-// CHECK:   Value a();
-// CHECK:   Operation::operand_range b();
+// CHECK:   static ::llvm::StringRef getOperationName();
+// CHECK:   ::mlir::Operation::operand_range getODSOperands(unsigned index);
+// CHECK:   ::mlir::Value a();
+// CHECK:   ::mlir::Operation::operand_range b();
 // CHECK:   ::mlir::MutableOperandRange aMutable();
 // CHECK:   ::mlir::MutableOperandRange bMutable();
-// CHECK:   Operation::result_range getODSResults(unsigned index);
-// CHECK:   Value r();
-// CHECK:   Region &someRegion();
-// CHECK:   MutableArrayRef<Region> someRegions();
-// CHECK:   IntegerAttr attr1Attr()
-// CHECK:   APInt attr1();
-// CHECK:   FloatAttr attr2Attr()
-// CHECK:   Optional< APFloat > attr2();
+// CHECK:   ::mlir::Operation::result_range getODSResults(unsigned index);
+// CHECK:   ::mlir::Value r();
+// CHECK:   ::mlir::Region &someRegion();
+// CHECK:   ::mlir::MutableArrayRef<Region> someRegions();
+// CHECK:   ::mlir::IntegerAttr attr1Attr()
+// CHECK:   ::llvm::APInt attr1();
+// CHECK:   ::mlir::FloatAttr attr2Attr()
+// CHECK:   ::llvm::Optional< ::llvm::APFloat > attr2();
 // CHECK:   static void build(Value val);
-// CHECK:   static void build(OpBuilder &odsBuilder, OperationState &odsState, Type r, ArrayRef<Type> s, Value a, ValueRange b, IntegerAttr attr1, /*optional*/FloatAttr attr2, unsigned someRegionsCount)
-// CHECK:   static void build(OpBuilder &odsBuilder, OperationState &odsState, Type r, ArrayRef<Type> s, Value a, ValueRange b, APInt attr1, /*optional*/FloatAttr attr2, unsigned someRegionsCount)
-// CHECK:   static void build(OpBuilder &, OperationState &odsState, ArrayRef<Type> resultTypes, ValueRange operands, ArrayRef<NamedAttribute> attributes, unsigned numRegions)
-// CHECK:   static ParseResult parse(OpAsmParser &parser, OperationState &result);
-// CHECK:   void print(OpAsmPrinter &p);
-// CHECK:   LogicalResult verify();
-// CHECK:   static void getCanonicalizationPatterns(OwningRewritePatternList &results, MLIRContext *context);
-// CHECK:   LogicalResult fold(ArrayRef<Attribute> operands, SmallVectorImpl<OpFoldResult> &results);
+// CHECK:   static void build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type r, ::llvm::ArrayRef<::mlir::Type> s, ::mlir::Value a, ::mlir::ValueRange b, ::mlir::IntegerAttr attr1, /*optional*/::mlir::FloatAttr attr2, unsigned someRegionsCount)
+// CHECK:   static void build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type r, ::llvm::ArrayRef<::mlir::Type> s, ::mlir::Value a, ::mlir::ValueRange b, ::llvm::APInt attr1, /*optional*/::mlir::FloatAttr attr2, unsigned someRegionsCount)
+// CHECK:   static void build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::llvm::ArrayRef<::mlir::Type> resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes, unsigned numRegions)
+// CHECK:   static ::mlir::ParseResult parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result);
+// CHECK:   void print(::mlir::OpAsmPrinter &p);
+// CHECK:   ::mlir::LogicalResult verify();
+// CHECK:   static void getCanonicalizationPatterns(::mlir::OwningRewritePatternList &results, ::mlir::MLIRContext *context);
+// CHECK:   ::mlir::LogicalResult fold(::llvm::ArrayRef<::mlir::Attribute> operands, ::llvm::SmallVectorImpl<::mlir::OpFoldResult> &results);
 // CHECK:   // Display a graph for debugging purposes.
 // CHECK:   void displayGraph();
 // CHECK: };
@@ -106,13 +108,13 @@ def NS_AttrSizedOperandOp : NS_Op<"attr_sized_operands",
 }
 
 // CHECK-LABEL: AttrSizedOperandOpAdaptor(
-// CHECK-SAME:    ValueRange values
-// CHECK-SAME:    DictionaryAttr attrs
-// CHECK:  ValueRange a();
-// CHECK:  ValueRange b();
-// CHECK:  Value c();
-// CHECK:  ValueRange d();
-// CHECK:  DenseIntElementsAttr operand_segment_sizes();
+// CHECK-SAME:    ::mlir::ValueRange values
+// CHECK-SAME:    ::mlir::DictionaryAttr attrs
+// CHECK:  ::mlir::ValueRange a();
+// CHECK:  ::mlir::ValueRange b();
+// CHECK:  ::mlir::Value c();
+// CHECK:  ::mlir::ValueRange d();
+// CHECK:  ::mlir::DenseIntElementsAttr operand_segment_sizes();
 
 // Check op trait for different number of operands
 // ---
@@ -144,10 +146,10 @@ def NS_EOp : NS_Op<"op_with_optionals", []> {
 }
 
 // CHECK-LABEL: NS::EOp declarations
-// CHECK:   Value a();
+// CHECK:   ::mlir::Value a();
 // CHECK:   ::mlir::MutableOperandRange aMutable();
-// CHECK:   Value b();
-// CHECK:   static void build(OpBuilder &odsBuilder, OperationState &odsState, /*optional*/Type b, /*optional*/Value a)
+// CHECK:   ::mlir::Value b();
+// CHECK:   static void build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, /*optional*/::mlir::Type b, /*optional*/::mlir::Value a)
 
 // Check that all types match constraint results in generating builder.
 // ---
@@ -159,7 +161,7 @@ def NS_FOp : NS_Op<"op_with_all_types_constraint",
 }
 
 // CHECK-LABEL: class FOp :
-// CHECK: static LogicalResult inferReturnTypes
+// CHECK: static ::mlir::LogicalResult inferReturnTypes
 
 def NS_GOp : NS_Op<"op_with_fixed_return_type", []> {
   let arguments = (ins AnyType:$a);
@@ -167,7 +169,7 @@ def NS_GOp : NS_Op<"op_with_fixed_return_type", []> {
 }
 
 // CHECK-LABEL: class GOp :
-// CHECK: static LogicalResult inferReturnTypes
+// CHECK: static ::mlir::LogicalResult inferReturnTypes
 
 // Check that default builders can be suppressed.
 // ---
@@ -179,7 +181,7 @@ def NS_SkipDefaultBuildersOp : NS_Op<"skip_default_builders", []> {
 
 // CHECK-LABEL: NS::SkipDefaultBuildersOp declarations
 // CHECK:     class SkipDefaultBuildersOp
-// CHECK-NOT:   static void build(Builder
+// CHECK-NOT:   static void build(::mlir::Builder
 // CHECK:       static void build(Value
 
 // Check leading underscore in op name
@@ -188,10 +190,15 @@ def NS_SkipDefaultBuildersOp : NS_Op<"skip_default_builders", []> {
 def NS__AOp : NS_Op<"_op_with_leading_underscore", []>;
 
 // CHECK-LABEL: NS::_AOp declarations
-// CHECK: class _AOp : public Op<_AOp
+// CHECK: class _AOp : public ::mlir::Op<_AOp
 
 def _BOp : NS_Op<"_op_with_leading_underscore_and_no_namespace", []>;
 
 // CHECK-LABEL: _BOp declarations
-// CHECK: class _BOp : public Op<_BOp
+// CHECK: class _BOp : public ::mlir::Op<_BOp
 
+// REDUCE_INC-LABEL: NS::AOp declarations
+// REDUCE_INC-NOT: NS::BOp declarations
+
+// REDUCE_EXC-NOT: NS::AOp declarations
+// REDUCE_EXC-LABEL: NS::BOp declarations
diff --git a/mlir/test/mlir-tblgen/op-format.mlir b/mlir/test/mlir-tblgen/op-format.mlir
index 066e548e17083..49ac3d26f9269 100644
--- a/mlir/test/mlir-tblgen/op-format.mlir
+++ b/mlir/test/mlir-tblgen/op-format.mlir
@@ -101,3 +101,30 @@ test.format_optional_operand_result_b_op( : ) : i64
 
 // CHECK: test.format_optional_operand_result_b_op : i64
 test.format_optional_operand_result_b_op : i64
+
+//===----------------------------------------------------------------------===//
+// Format trait type inference
+//===----------------------------------------------------------------------===//
+
+// CHECK: test.format_infer_variadic_type_from_non_variadic %[[I64]], %[[I64]] : i64
+test.format_infer_variadic_type_from_non_variadic %i64, %i64 : i64
+
+//===----------------------------------------------------------------------===//
+// AllTypesMatch type inference
+//===----------------------------------------------------------------------===//
+
+// CHECK: test.format_all_types_match_var %[[I64]], %[[I64]] : i64
+%ignored_res1 = test.format_all_types_match_var %i64, %i64 : i64
+
+// CHECK: test.format_all_types_match_attr 1 : i64, %[[I64]]
+%ignored_res2 = test.format_all_types_match_attr 1 : i64, %i64
+
+//===----------------------------------------------------------------------===//
+// TypesMatchWith type inference
+//===----------------------------------------------------------------------===//
+
+// CHECK: test.format_types_match_var %[[I64]] : i64
+%ignored_res3 = test.format_types_match_var %i64 : i64
+
+// CHECK: test.format_types_match_attr 1 : i64
+%ignored_res4 = test.format_types_match_attr 1 : i64
diff --git a/mlir/test/mlir-tblgen/op-interface.td b/mlir/test/mlir-tblgen/op-interface.td
index 8e5167e6fe539..4ca2b798c3c91 100644
--- a/mlir/test/mlir-tblgen/op-interface.td
+++ b/mlir/test/mlir-tblgen/op-interface.td
@@ -40,7 +40,7 @@ def DeclareMethodsWithDefaultOp : Op<TestDialect, "declare_methods_op",
       [DeclareOpInterfaceMethods<TestOpInterface, ["default_foo"]>]>;
 
 // DECL-LABEL: TestOpInterfaceInterfaceTraits
-// DECL: class TestOpInterface : public OpInterface<TestOpInterface, detail::TestOpInterfaceInterfaceTraits>
+// DECL: class TestOpInterface : public ::mlir::OpInterface<TestOpInterface, detail::TestOpInterfaceInterfaceTraits>
 // DECL: int foo(int input);
 
 // DECL-NOT: TestOpInterface
diff --git a/mlir/test/mlir-tblgen/op-operand.td b/mlir/test/mlir-tblgen/op-operand.td
index a0a57d8037715..abeb67f2dbae1 100644
--- a/mlir/test/mlir-tblgen/op-operand.td
+++ b/mlir/test/mlir-tblgen/op-operand.td
@@ -18,11 +18,11 @@ def OpA : NS_Op<"one_normal_operand_op", []> {
 // CHECK-SAME: odsOperands(values), odsAttrs(attrs)
 
 // CHECK:      void OpA::build
-// CHECK:        Value input
+// CHECK:        ::mlir::Value input
 // CHECK:        odsState.addOperands(input);
 
 // CHECK:      void OpA::build
-// CHECK:        ValueRange operands
+// CHECK:        ::mlir::ValueRange operands
 // CHECK:        assert(operands.size() == 1u && "mismatched number of parameters");
 // CHECK:        odsState.addOperands(operands);
 
@@ -31,7 +31,7 @@ def OpB : NS_Op<"one_variadic_operand_op", []> {
 }
 
 // CHECK-LABEL: OpB::build
-// CHECK:         ValueRange input
+// CHECK:         ::mlir::ValueRange input
 // CHECK-NOT:     assert
 // CHECK:         odsState.addOperands(input);
 
@@ -39,19 +39,19 @@ def OpD : NS_Op<"mix_variadic_and_normal_inputs_op", [SameVariadicOperandSize]>
   let arguments = (ins Variadic<AnyTensor>:$input1, AnyTensor:$input2, Variadic<AnyTensor>:$input3);
 }
 
-// CHECK-LABEL: ValueRange OpDAdaptor::input1
+// CHECK-LABEL: ::mlir::ValueRange OpDAdaptor::input1
 // CHECK-NEXT:    return getODSOperands(0);
 
-// CHECK-LABEL: Value OpDAdaptor::input2
+// CHECK-LABEL: ::mlir::Value OpDAdaptor::input2
 // CHECK-NEXT:    return *getODSOperands(1).begin();
 
-// CHECK-LABEL: ValueRange OpDAdaptor::input3
+// CHECK-LABEL: ::mlir::ValueRange OpDAdaptor::input3
 // CHECK-NEXT:    return getODSOperands(2);
 
-// CHECK-LABEL: Operation::operand_range OpD::input1
+// CHECK-LABEL: ::mlir::Operation::operand_range OpD::input1
 // CHECK-NEXT: return getODSOperands(0);
 
-// CHECK-LABEL: Value OpD::input2
+// CHECK-LABEL: ::mlir::Value OpD::input2
 // CHECK-NEXT: return *getODSOperands(1).begin();
 
 // CHECK-LABEL: OpD::build
diff --git a/mlir/test/mlir-tblgen/op-result.td b/mlir/test/mlir-tblgen/op-result.td
index 7886c2772a212..4b091e4ad49f1 100644
--- a/mlir/test/mlir-tblgen/op-result.td
+++ b/mlir/test/mlir-tblgen/op-result.td
@@ -1,6 +1,7 @@
 // RUN: mlir-tblgen -gen-op-defs -I %S/../../include %s | FileCheck %s
 
 include "mlir/IR/OpBase.td"
+include "mlir/Interfaces/InferTypeOpInterface.td"
 
 def Test_Dialect : Dialect {
   let name = "test";
@@ -13,7 +14,7 @@ def OpA : NS_Op<"one_normal_result_op", []> {
 }
 
 // CHECK-LABEL: void OpA::build
-// CHECK:         ArrayRef<Type> resultTypes, ValueRange operands
+// CHECK:         ::llvm::ArrayRef<::mlir::Type> resultTypes, ::mlir::ValueRange operands
 // CHECK:         assert(resultTypes.size() == 1u && "mismatched number of return types");
 // CHECK-NEXT:    odsState.addTypes(resultTypes);
 
@@ -23,9 +24,9 @@ def OpB : NS_Op<"same_input_output_type_op", [SameOperandsAndResultType]> {
 }
 
 // CHECK-LABEL: OpB definitions
-// CHECK: void OpB::build(OpBuilder &odsBuilder, OperationState &odsState, Type y, Value x)
+// CHECK: void OpB::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type y, ::mlir::Value x)
 // CHECK:   odsState.addTypes(y);
-// CHECK: void OpB::build(OpBuilder &odsBuilder, OperationState &odsState, Value x)
+// CHECK: void OpB::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Value x)
 // CHECK:   odsState.addTypes({x.getType()});
 
 def OpC : NS_Op<"three_normal_result_op", []> {
@@ -33,12 +34,12 @@ def OpC : NS_Op<"three_normal_result_op", []> {
 }
 
 // CHECK-LABEL: OpC definitions
-// CHECK:       void OpC::build(OpBuilder &odsBuilder, OperationState &odsState, Type x, Type resultType1, Type z)
+// CHECK:       void OpC::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type x, ::mlir::Type resultType1, ::mlir::Type z)
 // CHECK-NEXT:   odsState.addTypes(x)
 // CHECK-NEXT:   odsState.addTypes(resultType1)
 // CHECK-NEXT:   odsState.addTypes(z)
 
-// CHECK:      void OpC::build(OpBuilder &odsBuilder, OperationState &odsState, ArrayRef<Type> resultTypes) {
+// CHECK:      void OpC::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::llvm::ArrayRef<::mlir::Type> resultTypes) {
 // CHECK-NEXT:   assert(resultTypes.size() == 3u && "mismatched number of results");
 // CHECK-NEXT:   odsState.addTypes(resultTypes);
 
@@ -49,8 +50,8 @@ def OpD : NS_Op<"type_attr_as_result_type", [FirstAttrDerivedResultType]> {
 }
 
 // CHECK-LABEL: OpD definitions
-// CHECK: void OpD::build(OpBuilder &odsBuilder, OperationState &odsState, ValueRange operands, ArrayRef<NamedAttribute> attributes)
-// CHECK: odsState.addTypes({attr.second.cast<TypeAttr>().getValue()});
+// CHECK: void OpD::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes)
+// CHECK: odsState.addTypes({attr.second.cast<::mlir::TypeAttr>().getValue()});
 
 def OpE : NS_Op<"value_attr_as_result_type", [FirstAttrDerivedResultType]> {
   let arguments = (ins I32:$x, F32Attr:$attr);
@@ -58,7 +59,7 @@ def OpE : NS_Op<"value_attr_as_result_type", [FirstAttrDerivedResultType]> {
 }
 
 // CHECK-LABEL: OpE definitions
-// CHECK: void OpE::build(OpBuilder &odsBuilder, OperationState &odsState, ValueRange operands, ArrayRef<NamedAttribute> attributes)
+// CHECK: void OpE::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes)
 // CHECK: odsState.addTypes({attr.second.getType()});
 
 def OpF : NS_Op<"one_variadic_result_op", []> {
@@ -66,7 +67,7 @@ def OpF : NS_Op<"one_variadic_result_op", []> {
 }
 
 // CHECK-LABEL: void OpF::build
-// CHECK-SAME:    ArrayRef<Type> x
+// CHECK-SAME:    ::llvm::ArrayRef<::mlir::Type> x
 // CHECK-NOT:     assert
 // CHECK:         odsState.addTypes(x);
 
@@ -77,12 +78,12 @@ def OpG : NS_Op<"one_normal_and_one_variadic_result_op", []> {
 
 // CHECK-LABEL: OpG definitions
 
-// CHECK:      void OpG::build(OpBuilder &odsBuilder, OperationState &odsState, Type x, ArrayRef<Type> y)
+// CHECK:      void OpG::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type x, ::llvm::ArrayRef<::mlir::Type> y)
 // CHECK-NEXT:   odsState.addTypes(x);
 // CHECK-NEXT:   odsState.addTypes(y);
 
 // CHECK:       void OpG::build
-// CHECK:         ArrayRef<Type> resultTypes
+// CHECK:         ::llvm::ArrayRef<::mlir::Type> resultTypes
 // CHECK:         assert(resultTypes.size() >= 1u && "mismatched number of return types");
 // CHECK-NEXT:    odsState.addTypes(resultTypes);
 
@@ -90,10 +91,10 @@ def OpI : NS_Op<"mix_variadic_and_normal_results_op", [SameVariadicResultSize]>
   let results = (outs Variadic<AnyTensor>:$output1, AnyTensor:$output2, Variadic<AnyTensor>:$output3);
 }
 
-// CHECK-LABEL: Operation::result_range OpI::output1
+// CHECK-LABEL: ::mlir::Operation::result_range OpI::output1
 // CHECK-NEXT:    return getODSResults(0);
 
-// CHECK-LABEL: Value OpI::output2
+// CHECK-LABEL: ::mlir::Value OpI::output2
 // CHECK-NEXT:    return *getODSResults(1).begin();
 
 // CHECK-LABEL: OpI::build
@@ -109,5 +110,17 @@ def OpK : NS_Op<"only_input_is_variadic_with_same_value_type_op", [SameOperandsA
   let results = (outs AnyTensor:$result);
 }
 
-// CHECK-LABEL: OpK::build(OpBuilder &odsBuilder, OperationState &odsState, ValueRange input)
+// CHECK-LABEL: OpK::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange input)
 // CHECK: odsState.addTypes({input.front().getType()});
+
+// Test with inferred shapes and interleaved with operands/attributes.
+//
+def OpL : NS_Op<"op_with_all_types_constraint",
+    [AllTypesMatch<["a", "b"]>]> {
+  let arguments = (ins I32Attr:$attr1, AnyType:$a);
+  let results = (outs Res<AnyType, "output b", []>:$b);
+}
+
+// CHECK-LABEL: LogicalResult OpL::inferReturnTypes
+// CHECK-NOT: }
+// CHECK: inferredReturnTypes[0] = operands[0].getType();
diff --git a/mlir/test/mlir-tblgen/op-side-effects.td b/mlir/test/mlir-tblgen/op-side-effects.td
index a514d46426e94..6bae35aa763b0 100644
--- a/mlir/test/mlir-tblgen/op-side-effects.td
+++ b/mlir/test/mlir-tblgen/op-side-effects.td
@@ -19,9 +19,9 @@ def SideEffectOpB : TEST_Op<"side_effect_op_b",
     [MemoryEffects<[MemWrite<CustomResource>]>]>;
 
 // CHECK: void SideEffectOpA::getEffects
-// CHECK:   for (Value value : getODSOperands(0))
+// CHECK:   for (::mlir::Value value : getODSOperands(0))
 // CHECK:     effects.emplace_back(MemoryEffects::Read::get(), value, ::mlir::SideEffects::DefaultResource::get());
-// CHECK:   for (Value value : getODSResults(0))
+// CHECK:   for (::mlir::Value value : getODSResults(0))
 // CHECK:     effects.emplace_back(MemoryEffects::Allocate::get(), value, CustomResource::get());
 
 // CHECK: void SideEffectOpB::getEffects
diff --git a/mlir/test/mlir-tblgen/pattern.mlir b/mlir/test/mlir-tblgen/pattern.mlir
index 6154e6bc4c457..0f2f434c928f9 100644
--- a/mlir/test/mlir-tblgen/pattern.mlir
+++ b/mlir/test/mlir-tblgen/pattern.mlir
@@ -359,3 +359,14 @@ func @generateVariadicOutputOpInNestedPattern() -> (i32) {
   %0 = "test.one_i32_out"() : () -> (i32)
   return %0 : i32
 }
+
+//===----------------------------------------------------------------------===//
+// Test that natives calls are only called once during rewrites.
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: redundantTest
+func @redundantTest(%arg0: i32) -> i32 {
+  %0 = "test.op_m"(%arg0) : (i32) -> i32
+  // CHECK: "test.op_m"(%arg0) {optional_attr = 314159265 : i32} : (i32) -> i32
+  return %0 : i32
+}
diff --git a/mlir/test/mlir-tblgen/predicate.td b/mlir/test/mlir-tblgen/predicate.td
index 5f19344f537ce..e5d4ac853b501 100644
--- a/mlir/test/mlir-tblgen/predicate.td
+++ b/mlir/test/mlir-tblgen/predicate.td
@@ -17,7 +17,7 @@ def OpA : NS_Op<"op_for_CPred_containing_multiple_same_placeholder", []> {
 
 // CHECK-LABEL: OpA::verify
 // CHECK: auto valueGroup0 = getODSOperands(0);
-// CHECK: for (Value v : valueGroup0) {
+// CHECK: for (::mlir::Value v : valueGroup0) {
 // CHECK:   if (!((v.getType().isInteger(32) || v.getType().isF32())))
 
 def OpB : NS_Op<"op_for_And_PredOpTrait", [
@@ -33,7 +33,7 @@ def OpF : NS_Op<"op_for_int_min_val", []> {
 }
 
 // CHECK-LABEL: OpFAdaptor::verify
-// CHECK:       (tblgen_attr.cast<IntegerAttr>().getInt() >= 10)
+// CHECK:       (tblgen_attr.cast<::mlir::IntegerAttr>().getInt() >= 10)
 // CHECK-SAME:  "attribute 'attr' failed to satisfy constraint: 32-bit signless integer attribute whose minimum value is 10"
 
 def OpFX : NS_Op<"op_for_int_max_val", []> {
@@ -41,7 +41,7 @@ def OpFX : NS_Op<"op_for_int_max_val", []> {
 }
 
 // CHECK-LABEL: OpFXAdaptor::verify
-// CHECK:       (tblgen_attr.cast<IntegerAttr>().getInt() <= 10)
+// CHECK:       (tblgen_attr.cast<::mlir::IntegerAttr>().getInt() <= 10)
 // CHECK-SAME:  "attribute 'attr' failed to satisfy constraint: 32-bit signless integer attribute whose maximum value is 10"
 
 def OpG : NS_Op<"op_for_arr_min_count", []> {
@@ -49,7 +49,7 @@ def OpG : NS_Op<"op_for_arr_min_count", []> {
 }
 
 // CHECK-LABEL: OpGAdaptor::verify
-// CHECK:       (tblgen_attr.cast<ArrayAttr>().size() >= 8)
+// CHECK:       (tblgen_attr.cast<::mlir::ArrayAttr>().size() >= 8)
 // CHECK-SAME:  "attribute 'attr' failed to satisfy constraint: array attribute with at least 8 elements"
 
 def OpH : NS_Op<"op_for_arr_value_at_index", []> {
@@ -57,7 +57,7 @@ def OpH : NS_Op<"op_for_arr_value_at_index", []> {
 }
 
 // CHECK-LABEL: OpHAdaptor::verify
-// CHECK: (((tblgen_attr.cast<ArrayAttr>().size() > 0)) && ((tblgen_attr.cast<ArrayAttr>()[0].cast<IntegerAttr>().getInt() == 8)))))
+// CHECK: (((tblgen_attr.cast<::mlir::ArrayAttr>().size() > 0)) && ((tblgen_attr.cast<::mlir::ArrayAttr>()[0].cast<::mlir::IntegerAttr>().getInt() == 8)))))
 // CHECK-SAME:  "attribute 'attr' failed to satisfy constraint: array attribute whose 0-th element must be 8"
 
 def OpI: NS_Op<"op_for_arr_min_value_at_index", []> {
@@ -65,7 +65,7 @@ def OpI: NS_Op<"op_for_arr_min_value_at_index", []> {
 }
 
 // CHECK-LABEL: OpIAdaptor::verify
-// CHECK: (((tblgen_attr.cast<ArrayAttr>().size() > 0)) && ((tblgen_attr.cast<ArrayAttr>()[0].cast<IntegerAttr>().getInt() >= 8)))))
+// CHECK: (((tblgen_attr.cast<::mlir::ArrayAttr>().size() > 0)) && ((tblgen_attr.cast<::mlir::ArrayAttr>()[0].cast<::mlir::IntegerAttr>().getInt() >= 8)))))
 // CHECK-SAME: "attribute 'attr' failed to satisfy constraint: array attribute whose 0-th element must be at least 8"
 
 def OpJ: NS_Op<"op_for_TCopVTEtAreSameAt", [
@@ -81,8 +81,8 @@ def OpJ: NS_Op<"op_for_TCopVTEtAreSameAt", [
 }
 
 // CHECK-LABEL: OpJAdaptor::verify
-// CHECK:      llvm::is_splat(llvm::map_range(
-// CHECK-SAME:   llvm::ArrayRef<unsigned>({0, 2, 3}),
+// CHECK:      ::llvm::is_splat(::llvm::map_range(
+// CHECK-SAME:   ::mlir::ArrayRef<unsigned>({0, 2, 3}),
 // CHECK-SAME:   [this](unsigned i) { return getElementTypeOrSelf(this->getOperand(i)); }))
 // CHECK: "failed to verify that operands indexed at 0, 2, 3 should all have the same type"
 
@@ -92,5 +92,5 @@ def OpK : NS_Op<"op_for_AnyTensorOf", []> {
 
 // CHECK-LABEL: OpK::verify
 // CHECK: auto valueGroup0 = getODSOperands(0);
-// CHECK: for (Value v : valueGroup0) {
-// CHECK: if (!(((v.getType().isa<TensorType>())) && (((v.getType().cast<ShapedType>().getElementType().isF32())) || ((v.getType().cast<ShapedType>().getElementType().isSignlessInteger(32))))))
+// CHECK: for (::mlir::Value v : valueGroup0) {
+// CHECK: if (!(((v.getType().isa<::mlir::TensorType>())) && (((v.getType().cast<::mlir::ShapedType>().getElementType().isF32())) || ((v.getType().cast<::mlir::ShapedType>().getElementType().isSignlessInteger(32))))))
diff --git a/mlir/tools/CMakeLists.txt b/mlir/tools/CMakeLists.txt
index e8f61633c92b1..23a2fcbc14ebb 100644
--- a/mlir/tools/CMakeLists.txt
+++ b/mlir/tools/CMakeLists.txt
@@ -2,7 +2,8 @@ add_subdirectory(mlir-cuda-runner)
 add_subdirectory(mlir-cpu-runner)
 add_subdirectory(mlir-linalg-ods-gen)
 add_subdirectory(mlir-opt)
+add_subdirectory(mlir-reduce)
 add_subdirectory(mlir-rocm-runner)
-add_subdirectory(mlir-translate)
-add_subdirectory(mlir-vulkan-runner)
 add_subdirectory(mlir-shlib)
+add_subdirectory(mlir-translate)
+add_subdirectory(mlir-vulkan-runner)
\ No newline at end of file
diff --git a/mlir/tools/mlir-opt/CMakeLists.txt b/mlir/tools/mlir-opt/CMakeLists.txt
index d509b23505d12..483dcfec0c0ff 100644
--- a/mlir/tools/mlir-opt/CMakeLists.txt
+++ b/mlir/tools/mlir-opt/CMakeLists.txt
@@ -17,6 +17,7 @@ if(MLIR_INCLUDE_TESTS)
     MLIRTestDialect
     MLIRTestIR
     MLIRTestPass
+    MLIRTestReducer
     MLIRTestTransforms
     )
 endif()
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index 4515094eb1051..f749c7ad98adf 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -51,6 +51,7 @@ void registerTestDominancePass();
 void registerTestExpandTanhPass();
 void registerTestFunc();
 void registerTestGpuMemoryPromotionPass();
+void registerTestInterfaces();
 void registerTestLinalgHoisting();
 void registerTestLinalgTransforms();
 void registerTestLivenessPass();
@@ -61,8 +62,8 @@ void registerTestMatchers();
 void registerTestMemRefDependenceCheck();
 void registerTestMemRefStrideCalculation();
 void registerTestOpaqueLoc();
-void registerTestParallelismDetection();
 void registerTestPreparationPassWithAllowedMemrefResults();
+void registerTestReducer();
 void registerTestGpuParallelLoopMappingPass();
 void registerTestSCFUtilsPass();
 void registerTestVectorConversions();
@@ -125,6 +126,7 @@ void registerTestPasses() {
   registerTestFunc();
   registerTestExpandTanhPass();
   registerTestGpuMemoryPromotionPass();
+  registerTestInterfaces();
   registerTestLinalgHoisting();
   registerTestLinalgTransforms();
   registerTestLivenessPass();
@@ -135,8 +137,8 @@ void registerTestPasses() {
   registerTestMemRefDependenceCheck();
   registerTestMemRefStrideCalculation();
   registerTestOpaqueLoc();
-  registerTestParallelismDetection();
   registerTestPreparationPassWithAllowedMemrefResults();
+  registerTestReducer();
   registerTestGpuParallelLoopMappingPass();
   registerTestSCFUtilsPass();
   registerTestVectorConversions();
diff --git a/mlir/tools/mlir-reduce/CMakeLists.txt b/mlir/tools/mlir-reduce/CMakeLists.txt
new file mode 100644
index 0000000000000..642c5d96c2153
--- /dev/null
+++ b/mlir/tools/mlir-reduce/CMakeLists.txt
@@ -0,0 +1,41 @@
+get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
+get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
+set(LLVM_LINK_COMPONENTS
+  AllTargetsAsmParsers
+  AllTargetsCodeGens
+  AllTargetsDescs
+  AllTargetsInfos
+  AsmParser
+  Core
+  IRReader
+  Support
+  Target
+  TransformUtils
+  )
+
+set(LIBS
+  ${dialect_libs}
+  ${conversion_libs}
+  ${test_libs}
+  MLIRAnalysis
+  MLIRDialect
+  MLIREDSC
+  MLIRIR
+  MLIRLoopAnalysis
+  MLIROptLib
+  MLIRParser
+  MLIRPass
+  MLIRReduce
+  MLIRSupport
+  MLIRTransforms
+  MLIRTransformUtils
+  )
+
+add_llvm_tool(mlir-reduce
+  mlir-reduce.cpp
+  )
+
+target_link_libraries(mlir-reduce PRIVATE ${LIBS})
+llvm_update_compile_flags(mlir-reduce)
+
+mlir_check_all_link_libraries(mlir-reduce)
\ No newline at end of file
diff --git a/mlir/tools/mlir-reduce/mlir-reduce.cpp b/mlir/tools/mlir-reduce/mlir-reduce.cpp
new file mode 100644
index 0000000000000..3edef61f1d728
--- /dev/null
+++ b/mlir/tools/mlir-reduce/mlir-reduce.cpp
@@ -0,0 +1,97 @@
+//===- mlir-reduce.cpp - The MLIR reducer ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the general framework of the MLIR reducer tool. It
+// parses the command line arguments, parses the initial MLIR test case and sets
+// up the testing environment. It  outputs the most reduced test case variant
+// after executing the reduction passes.
+//
+//===----------------------------------------------------------------------===//
+
+#include <vector>
+
+#include "mlir/InitAllDialects.h"
+#include "mlir/Parser.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassManager.h"
+#include "mlir/Reducer/Tester.h"
+#include "mlir/Support/FileUtilities.h"
+#include "mlir/Support/LogicalResult.h"
+#include "mlir/Transforms/Passes.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/ToolOutputFile.h"
+
+using namespace mlir;
+
+static llvm::cl::opt<std::string> inputFilename(llvm::cl::Positional,
+                                                llvm::cl::Required,
+                                                llvm::cl::desc("<input file>"));
+
+static llvm::cl::opt<std::string>
+    testFilename("test", llvm::cl::Required, llvm::cl::desc("Testing script"));
+
+static llvm::cl::list<std::string>
+    testArguments("test-args", llvm::cl::ZeroOrMore,
+                  llvm::cl::desc("Testing script arguments"));
+
+static llvm::cl::opt<std::string>
+    outputFilename("o",
+                   llvm::cl::desc("Output filename for the reduced test case"),
+                   llvm::cl::init("-"));
+
+// Parse and verify the input MLIR file.
+static LogicalResult loadModule(MLIRContext &context, OwningModuleRef &module,
+                                StringRef inputFilename) {
+  module = parseSourceFile(inputFilename, &context);
+  if (!module)
+    return failure();
+
+  return success();
+}
+
+int main(int argc, char **argv) {
+
+  llvm::InitLLVM y(argc, argv);
+
+  registerAllDialects();
+  registerMLIRContextCLOptions();
+  registerPassManagerCLOptions();
+
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "MLIR test case reduction tool.\n");
+
+  std::string errorMessage;
+
+  auto testscript = openInputFile(testFilename, &errorMessage);
+  if (!testscript)
+    llvm::report_fatal_error(errorMessage);
+
+  auto output = openOutputFile(outputFilename, &errorMessage);
+  if (!output)
+    llvm::report_fatal_error(errorMessage);
+
+  mlir::MLIRContext context;
+  mlir::OwningModuleRef moduleRef;
+  context.allowUnregisteredDialects(true);
+
+  if (failed(loadModule(context, moduleRef, inputFilename)))
+    llvm::report_fatal_error("Input test case can't be parsed");
+
+  // Initialize test environment.
+  Tester test(testFilename, testArguments);
+  test.setMostReduced(moduleRef.get());
+
+  if (!test.isInteresting(inputFilename))
+    llvm::report_fatal_error(
+        "Input test case does not exhibit interesting behavior");
+
+  test.getMostReduced().print(output->os());
+  output->keep();
+
+  return 0;
+}
\ No newline at end of file
diff --git a/mlir/tools/mlir-rocm-runner/mlir-rocm-runner.cpp b/mlir/tools/mlir-rocm-runner/mlir-rocm-runner.cpp
index 9e4b1ad2164a2..d61cb2d988093 100644
--- a/mlir/tools/mlir-rocm-runner/mlir-rocm-runner.cpp
+++ b/mlir/tools/mlir-rocm-runner/mlir-rocm-runner.cpp
@@ -198,7 +198,7 @@ static LogicalResult createHsaco(const Blob &isaBlob, StringRef name,
 
 static std::unique_ptr<llvm::Module> compileModuleToROCDLIR(Operation *m) {
   auto llvmModule = translateModuleToROCDLIR(m);
-  // TODO(whchung): Link with ROCm-Device-Libs in case needed (ex: the Module
+  // TODO: Link with ROCm-Device-Libs in case needed (ex: the Module
   // depends on math functions).
   return llvmModule;
 }
@@ -227,7 +227,7 @@ static void configTargetChip() {
   llvm::ErrorOr<std::string> rocmAgentEnumerator = llvm::sys::findProgramByName(
       kRocmAgentEnumerator, {__ROCM_PATH__ "/bin"});
   std::error_code ec;
-  if (ec = rocmAgentEnumerator.getError()) {
+  if ((ec = rocmAgentEnumerator.getError())) {
     WithColor::warning(errs(), kRunnerProgram)
         << kRocmAgentEnumerator << " couldn't be located under "
         << __ROCM_PATH__ << ", set target as " << kDefaultTargetChip << "\n";
diff --git a/mlir/tools/mlir-tblgen/DialectGen.cpp b/mlir/tools/mlir-tblgen/DialectGen.cpp
index cfc5861c9f620..4a9109d360c33 100644
--- a/mlir/tools/mlir-tblgen/DialectGen.cpp
+++ b/mlir/tools/mlir-tblgen/DialectGen.cpp
@@ -12,8 +12,8 @@
 
 #include "mlir/TableGen/Format.h"
 #include "mlir/TableGen/GenInfo.h"
+#include "mlir/TableGen/Interfaces.h"
 #include "mlir/TableGen/OpClass.h"
-#include "mlir/TableGen/OpInterfaces.h"
 #include "mlir/TableGen/OpTrait.h"
 #include "mlir/TableGen/Operator.h"
 #include "llvm/ADT/Sequence.h"
diff --git a/mlir/tools/mlir-tblgen/EnumsGen.cpp b/mlir/tools/mlir-tblgen/EnumsGen.cpp
index 04d357d762dab..b0a89153be4cb 100644
--- a/mlir/tools/mlir-tblgen/EnumsGen.cpp
+++ b/mlir/tools/mlir-tblgen/EnumsGen.cpp
@@ -71,7 +71,7 @@ static void emitDenseMapInfo(StringRef enumName, std::string underlyingType,
   const char *const mapInfo = R"(
 namespace llvm {
 template<> struct DenseMapInfo<{0}> {{
-  using StorageInfo = llvm::DenseMapInfo<{1}>;
+  using StorageInfo = ::llvm::DenseMapInfo<{1}>;
 
   static inline {0} getEmptyKey() {{
     return static_cast<{0}>(StorageInfo::getEmptyKey());
@@ -193,7 +193,7 @@ static void emitSymToStrFnForBitEnum(const Record &enumDef, raw_ostream &os) {
     os << formatv("  if (val == 0) return \"{0}\";\n\n",
                   allBitsUnsetCase->getSymbol());
   }
-  os << "  llvm::SmallVector<llvm::StringRef, 2> strs;\n";
+  os << "  ::llvm::SmallVector<::llvm::StringRef, 2> strs;\n";
   for (const auto &enumerant : enumerants) {
     // Skip the special enumerant for None.
     if (auto val = enumerant.getValue())
@@ -203,7 +203,7 @@ static void emitSymToStrFnForBitEnum(const Record &enumDef, raw_ostream &os) {
   }
   // If we have unknown bit set, return an empty string to signal errors.
   os << "\n  if (val) return \"\";\n";
-  os << formatv("  return llvm::join(strs, \"{0}\");\n", separator);
+  os << formatv("  return ::llvm::join(strs, \"{0}\");\n", separator);
 
   os << "}\n\n";
 }
@@ -214,9 +214,9 @@ static void emitStrToSymFnForIntEnum(const Record &enumDef, raw_ostream &os) {
   StringRef strToSymFnName = enumAttr.getStringToSymbolFnName();
   auto enumerants = enumAttr.getAllCases();
 
-  os << formatv("llvm::Optional<{0}> {1}(llvm::StringRef str) {{\n", enumName,
-                strToSymFnName);
-  os << formatv("  return llvm::StringSwitch<llvm::Optional<{0}>>(str)\n",
+  os << formatv("::llvm::Optional<{0}> {1}(::llvm::StringRef str) {{\n",
+                enumName, strToSymFnName);
+  os << formatv("  return ::llvm::StringSwitch<::llvm::Optional<{0}>>(str)\n",
                 enumName);
   for (const auto &enumerant : enumerants) {
     auto symbol = enumerant.getSymbol();
@@ -224,7 +224,7 @@ static void emitStrToSymFnForIntEnum(const Record &enumDef, raw_ostream &os) {
     os << formatv("      .Case(\"{1}\", {0}::{2})\n", enumName, str,
                   makeIdentifier(symbol));
   }
-  os << "      .Default(llvm::None);\n";
+  os << "      .Default(::llvm::None);\n";
   os << "}\n";
 }
 
@@ -237,8 +237,8 @@ static void emitStrToSymFnForBitEnum(const Record &enumDef, raw_ostream &os) {
   auto enumerants = enumAttr.getAllCases();
   auto allBitsUnsetCase = getAllBitsUnsetCase(enumerants);
 
-  os << formatv("llvm::Optional<{0}> {1}(llvm::StringRef str) {{\n", enumName,
-                strToSymFnName);
+  os << formatv("::llvm::Optional<{0}> {1}(::llvm::StringRef str) {{\n",
+                enumName, strToSymFnName);
 
   if (allBitsUnsetCase) {
     os << "  // Special case for all bits unset.\n";
@@ -248,7 +248,7 @@ static void emitStrToSymFnForBitEnum(const Record &enumDef, raw_ostream &os) {
   }
 
   // Split the string to get symbols for all the bits.
-  os << "  llvm::SmallVector<llvm::StringRef, 2> symbols;\n";
+  os << "  ::llvm::SmallVector<::llvm::StringRef, 2> symbols;\n";
   os << formatv("  str.split(symbols, \"{0}\");\n\n", separator);
 
   os << formatv("  {0} val = 0;\n", underlyingType);
@@ -256,7 +256,7 @@ static void emitStrToSymFnForBitEnum(const Record &enumDef, raw_ostream &os) {
 
   // Convert each symbol to the bit ordinal and set the corresponding bit.
   os << formatv(
-      "    auto bit = llvm::StringSwitch<llvm::Optional<{0}>>(symbol)\n",
+      "    auto bit = llvm::StringSwitch<::llvm::Optional<{0}>>(symbol)\n",
       underlyingType);
   for (const auto &enumerant : enumerants) {
     // Skip the special enumerant for None.
@@ -264,9 +264,9 @@ static void emitStrToSymFnForBitEnum(const Record &enumDef, raw_ostream &os) {
       os.indent(6) << formatv(".Case(\"{0}\", {1})\n", enumerant.getSymbol(),
                               val);
   }
-  os.indent(6) << ".Default(llvm::None);\n";
+  os.indent(6) << ".Default(::llvm::None);\n";
 
-  os << "    if (bit) { val |= *bit; } else { return llvm::None; }\n";
+  os << "    if (bit) { val |= *bit; } else { return ::llvm::None; }\n";
   os << "  }\n";
 
   os << formatv("  return static_cast<{0}>(val);\n", enumName);
@@ -288,7 +288,7 @@ static void emitUnderlyingToSymFnForIntEnum(const Record &enumDef,
       }))
     return;
 
-  os << formatv("llvm::Optional<{0}> {1}({2} value) {{\n", enumName,
+  os << formatv("::llvm::Optional<{0}> {1}({2} value) {{\n", enumName,
                 underlyingToSymFnName,
                 underlyingType.empty() ? std::string("unsigned")
                                        : underlyingType)
@@ -299,7 +299,7 @@ static void emitUnderlyingToSymFnForIntEnum(const Record &enumDef,
     os << formatv("  case {0}: return {1}::{2};\n", value, enumName,
                   makeIdentifier(symbol));
   }
-  os << "  default: return llvm::None;\n"
+  os << "  default: return ::llvm::None;\n"
      << "  }\n"
      << "}\n\n";
 }
@@ -313,7 +313,7 @@ static void emitUnderlyingToSymFnForBitEnum(const Record &enumDef,
   auto enumerants = enumAttr.getAllCases();
   auto allBitsUnsetCase = getAllBitsUnsetCase(enumerants);
 
-  os << formatv("llvm::Optional<{0}> {1}({2} value) {{\n", enumName,
+  os << formatv("::llvm::Optional<{0}> {1}({2} value) {{\n", enumName,
                 underlyingToSymFnName, underlyingType);
   if (allBitsUnsetCase) {
     os << "  // Special case for all bits unset.\n";
@@ -357,11 +357,11 @@ static void emitEnumDecl(const Record &enumDef, raw_ostream &os) {
         return enumerant.getValue() >= 0;
       })) {
     os << formatv(
-        "llvm::Optional<{0}> {1}({2});\n", enumName, underlyingToSymFnName,
+        "::llvm::Optional<{0}> {1}({2});\n", enumName, underlyingToSymFnName,
         underlyingType.empty() ? std::string("unsigned") : underlyingType);
   }
   os << formatv("{2} {1}({0});\n", enumName, symToStrFnName, symToStrFnRetType);
-  os << formatv("llvm::Optional<{0}> {1}(llvm::StringRef);\n", enumName,
+  os << formatv("::llvm::Optional<{0}> {1}(::llvm::StringRef);\n", enumName,
                 strToSymFnName);
 
   if (enumAttr.isBitEnum()) {
@@ -383,10 +383,10 @@ inline {0} stringifyEnum({1} enumValue) {{
   // specified by the user.
   const char *const symbolizeEnumStr = R"(
 template <typename EnumType>
-llvm::Optional<EnumType> symbolizeEnum(llvm::StringRef);
+::llvm::Optional<EnumType> symbolizeEnum(::llvm::StringRef);
 
 template <>
-inline llvm::Optional<{0}> symbolizeEnum<{0}>(llvm::StringRef str) {
+inline ::llvm::Optional<{0}> symbolizeEnum<{0}>(::llvm::StringRef str) {
   return {1}(str);
 }
 )";
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index 21dccd4f3d5ad..b2b4245989b58 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -14,13 +14,15 @@
 #include "OpFormatGen.h"
 #include "mlir/TableGen/Format.h"
 #include "mlir/TableGen/GenInfo.h"
+#include "mlir/TableGen/Interfaces.h"
 #include "mlir/TableGen/OpClass.h"
-#include "mlir/TableGen/OpInterfaces.h"
 #include "mlir/TableGen/OpTrait.h"
 #include "mlir/TableGen/Operator.h"
 #include "mlir/TableGen/SideEffects.h"
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Regex.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
@@ -32,6 +34,17 @@ using namespace llvm;
 using namespace mlir;
 using namespace mlir::tblgen;
 
+cl::OptionCategory opDefGenCat("Options for -gen-op-defs and -gen-op-decls");
+
+static cl::opt<std::string> opIncFilter(
+    "op-include-regex",
+    cl::desc("Regex of name of op's to include (no filter if empty)"),
+    cl::cat(opDefGenCat));
+static cl::opt<std::string> opExcFilter(
+    "op-exclude-regex",
+    cl::desc("Regex of name of op's to exclude (no filter if empty)"),
+    cl::cat(opDefGenCat));
+
 static const char *const tblgenNamePrefix = "tblgen_";
 static const char *const generatedArgName = "odsArg";
 static const char *const builderOpState = "odsState";
@@ -72,10 +85,10 @@ const char *sameVariadicSizeValueRangeCalcCode = R"(
 // {0}: The name of the attribute specifying the segment sizes.
 const char *adapterSegmentSizeAttrInitCode = R"(
   assert(odsAttrs && "missing segment size attribute for op");
-  auto sizeAttr = odsAttrs.get("{0}").cast<DenseIntElementsAttr>();
+  auto sizeAttr = odsAttrs.get("{0}").cast<::mlir::DenseIntElementsAttr>();
 )";
 const char *opSegmentSizeAttrInitCode = R"(
-  auto sizeAttr = getAttrOfType<DenseIntElementsAttr>("{0}");
+  auto sizeAttr = getAttrOfType<::mlir::DenseIntElementsAttr>("{0}");
 )";
 const char *attrSizedSegmentValueRangeCalcCode = R"(
   unsigned start = 0;
@@ -122,7 +135,7 @@ static std::string replaceAllSubstrs(std::string str, const std::string &match,
 static inline bool hasStringAttribute(const Record &record,
                                       StringRef fieldName) {
   auto valueInit = record.getValueInit(fieldName);
-  return isa<CodeInit>(valueInit) || isa<StringInit>(valueInit);
+  return isa<CodeInit, StringInit>(valueInit);
 }
 
 static std::string getArgumentName(const Operator &op, int index) {
@@ -306,7 +319,7 @@ class OpEmitter {
 
 private:
   // The TableGen record for this op.
-  // TODO(antiagainst,zinenko): OpEmitter should not have a Record directly,
+  // TODO: OpEmitter should not have a Record directly,
   // it should rather go through the Operator for better abstraction.
   const Record &def;
 
@@ -475,7 +488,7 @@ void OpEmitter::emitDef(raw_ostream &os) { opClass.writeDefTo(os); }
 
 void OpEmitter::genAttrGetters() {
   FmtContext fctx;
-  fctx.withBuilder("mlir::Builder(this->getContext())");
+  fctx.withBuilder("::mlir::Builder(this->getContext())");
 
   // Emit the derived attribute body.
   auto emitDerivedAttr = [&](StringRef name, Attribute attr) {
@@ -536,13 +549,14 @@ void OpEmitter::genAttrGetters() {
                                           return namedAttr.attr.isDerivedAttr();
                                         });
   if (!derivedAttrs.empty()) {
-    opClass.addTrait("DerivedAttributeOpInterface::Trait");
+    opClass.addTrait("::mlir::DerivedAttributeOpInterface::Trait");
     // Generate helper method to query whether a named attribute is a derived
     // attribute. This enables, for example, avoiding adding an attribute that
     // overlaps with a derived attribute.
     {
-      auto &method = opClass.newMethod("bool", "isDerivedAttribute",
-                                       "StringRef name", OpMethod::MP_Static);
+      auto &method =
+          opClass.newMethod("bool", "isDerivedAttribute",
+                            "::llvm::StringRef name", OpMethod::MP_Static);
       auto &body = method.body();
       for (auto namedAttr : derivedAttrs)
         body << "  if (name == \"" << namedAttr.name << "\") return true;\n";
@@ -550,8 +564,8 @@ void OpEmitter::genAttrGetters() {
     }
     // Generate method to materialize derived attributes as a DictionaryAttr.
     {
-      OpMethod &method =
-          opClass.newMethod("DictionaryAttr", "materializeDerivedAttributes");
+      OpMethod &method = opClass.newMethod("::mlir::DictionaryAttr",
+                                           "materializeDerivedAttributes");
       auto &body = method.body();
 
       auto nonMaterializable =
@@ -575,14 +589,15 @@ void OpEmitter::genAttrGetters() {
         return;
       }
 
-      body << "  MLIRContext* ctx = getContext();\n";
-      body << "  Builder odsBuilder(ctx); (void)odsBuilder;\n";
-      body << "  return DictionaryAttr::get({\n";
+      body << "  ::mlir::MLIRContext* ctx = getContext();\n";
+      body << "  ::mlir::Builder odsBuilder(ctx); (void)odsBuilder;\n";
+      body << "  return ::mlir::DictionaryAttr::get({\n";
       interleave(
           derivedAttrs, body,
           [&](const NamedAttribute &namedAttr) {
             auto tmpl = namedAttr.attr.getConvertFromStorageCall();
-            body << "    {Identifier::get(\"" << namedAttr.name << "\", ctx),\n"
+            body << "    {::mlir::Identifier::get(\"" << namedAttr.name
+                 << "\", ctx),\n"
                  << tgfmt(tmpl, &fctx.withSelf(namedAttr.name + "()")
                                      .withBuilder("odsBuilder")
                                      .addSubst("_ctx", "ctx"))
@@ -704,14 +719,14 @@ static void generateNamedOperandGetters(const Operator &op, Class &opClass,
       continue;
 
     if (operand.isOptional()) {
-      auto &m = opClass.newMethod("Value", operand.name);
+      auto &m = opClass.newMethod("::mlir::Value", operand.name);
       m.body() << "  auto operands = getODSOperands(" << i << ");\n"
                << "  return operands.empty() ? Value() : *operands.begin();";
     } else if (operand.isVariadic()) {
       auto &m = opClass.newMethod(rangeType, operand.name);
       m.body() << "  return getODSOperands(" << i << ");";
     } else {
-      auto &m = opClass.newMethod("Value", operand.name);
+      auto &m = opClass.newMethod("::mlir::Value", operand.name);
       m.body() << "  return *getODSOperands(" << i << ").begin();";
     }
   }
@@ -722,7 +737,7 @@ void OpEmitter::genNamedOperandGetters() {
       op, opClass,
       /*sizeAttrInit=*/
       formatv(opSegmentSizeAttrInitCode, "operand_segment_sizes").str(),
-      /*rangeType=*/"Operation::operand_range",
+      /*rangeType=*/"::mlir::Operation::operand_range",
       /*rangeBeginCall=*/"getOperation()->operand_begin()",
       /*rangeSizeCall=*/"getOperation()->getNumOperands()",
       /*getOperandCallPattern=*/"getOperation()->getOperand({0})");
@@ -781,8 +796,8 @@ void OpEmitter::genNamedResultGetters() {
       numNormalResults, "getOperation()->getNumResults()", attrSizedResults,
       formatv(opSegmentSizeAttrInitCode, "result_segment_sizes").str(),
       op.getResults());
-  auto &m = opClass.newMethod("Operation::result_range", "getODSResults",
-                              "unsigned index");
+  auto &m = opClass.newMethod("::mlir::Operation::result_range",
+                              "getODSResults", "unsigned index");
   m.body() << formatv(valueRangeReturnCode, "getOperation()->result_begin()",
                       "getODSResultIndexAndLength(index)");
 
@@ -792,14 +807,16 @@ void OpEmitter::genNamedResultGetters() {
       continue;
 
     if (result.isOptional()) {
-      auto &m = opClass.newMethod("Value", result.name);
-      m.body() << "  auto results = getODSResults(" << i << ");\n"
-               << "  return results.empty() ? Value() : *results.begin();";
+      auto &m = opClass.newMethod("::mlir::Value", result.name);
+      m.body()
+          << "  auto results = getODSResults(" << i << ");\n"
+          << "  return results.empty() ? ::mlir::Value() : *results.begin();";
     } else if (result.isVariadic()) {
-      auto &m = opClass.newMethod("Operation::result_range", result.name);
+      auto &m =
+          opClass.newMethod("::mlir::Operation::result_range", result.name);
       m.body() << "  return getODSResults(" << i << ");";
     } else {
-      auto &m = opClass.newMethod("Value", result.name);
+      auto &m = opClass.newMethod("::mlir::Value", result.name);
       m.body() << "  return *getODSResults(" << i << ").begin();";
     }
   }
@@ -814,13 +831,14 @@ void OpEmitter::genNamedRegionGetters() {
 
     // Generate the accessors for a varidiadic region.
     if (region.isVariadic()) {
-      auto &m = opClass.newMethod("MutableArrayRef<Region>", region.name);
+      auto &m =
+          opClass.newMethod("::mlir::MutableArrayRef<Region>", region.name);
       m.body() << formatv(
           "  return this->getOperation()->getRegions().drop_front({0});", i);
       continue;
     }
 
-    auto &m = opClass.newMethod("Region &", region.name);
+    auto &m = opClass.newMethod("::mlir::Region &", region.name);
     m.body() << formatv("  return this->getOperation()->getRegion({0});", i);
   }
 }
@@ -834,7 +852,7 @@ void OpEmitter::genNamedSuccessorGetters() {
 
     // Generate the accessors for a variadic successor list.
     if (successor.isVariadic()) {
-      auto &m = opClass.newMethod("SuccessorRange", successor.name);
+      auto &m = opClass.newMethod("::mlir::SuccessorRange", successor.name);
       m.body() << formatv(
           "  return {std::next(this->getOperation()->successor_begin(), {0}), "
           "this->getOperation()->successor_end()};",
@@ -842,7 +860,7 @@ void OpEmitter::genNamedSuccessorGetters() {
       continue;
     }
 
-    auto &m = opClass.newMethod("Block *", successor.name);
+    auto &m = opClass.newMethod("::mlir::Block *", successor.name);
     m.body() << formatv("  return this->getOperation()->getSuccessor({0});", i);
   }
 }
@@ -869,7 +887,8 @@ static bool canGenerateUnwrappedBuilder(Operator &op) {
 }
 
 static bool canInferType(Operator &op) {
-  return op.getTrait("InferTypeOpInterface::Trait") && op.getNumRegions() == 0;
+  return op.getTrait("::mlir::InferTypeOpInterface::Trait") &&
+         op.getNumRegions() == 0;
 }
 
 void OpEmitter::genSeparateArgParamBuilder() {
@@ -897,18 +916,18 @@ void OpEmitter::genSeparateArgParamBuilder() {
 
     if (inferType) {
       // Generate builder that infers type too.
-      // TODO(jpienaar): Subsume this with general checking if type can be
+      // TODO: Subsume this with general checking if type can be
       // inferred automatically.
-      // TODO(jpienaar): Expand to handle regions.
+      // TODO: Expand to handle regions.
       body << formatv(R"(
-        SmallVector<Type, 2> inferredReturnTypes;
+        ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
         if (succeeded({0}::inferReturnTypes(odsBuilder.getContext(),
                       {1}.location, {1}.operands,
                       {1}.attributes.getDictionary({1}.getContext()),
                       /*regions=*/{{}, inferredReturnTypes)))
           {1}.addTypes(inferredReturnTypes);
         else
-          llvm::report_fatal_error("Failed to infer result type(s).");)",
+          ::llvm::report_fatal_error("Failed to infer result type(s).");)",
                       opClass.getClassName(), builderOpState);
       return;
     }
@@ -958,8 +977,10 @@ void OpEmitter::genUseOperandAsResultTypeCollectiveParamBuilder() {
 
   // Signature
   std::string params =
-      std::string("OpBuilder &odsBuilder, OperationState &") + builderOpState +
-      ", ValueRange operands, ArrayRef<NamedAttribute> attributes";
+      std::string("::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &") +
+      builderOpState +
+      ", ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> "
+      "attributes";
   if (op.getNumVariadicRegions())
     params += ", unsigned numRegions";
   auto &m = opClass.newMethod("void", "build", params, OpMethod::MP_Static);
@@ -986,10 +1007,11 @@ void OpEmitter::genUseOperandAsResultTypeCollectiveParamBuilder() {
 }
 
 void OpEmitter::genInferredTypeCollectiveParamBuilder() {
-  // TODO(jpienaar): Expand to support regions.
+  // TODO: Expand to support regions.
   const char *params =
-      "OpBuilder &odsBuilder, OperationState &{0}, "
-      "ValueRange operands, ArrayRef<NamedAttribute> attributes";
+      "::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &{0}, "
+      "::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> "
+      "attributes";
   auto &m =
       opClass.newMethod("void", "build", formatv(params, builderOpState).str(),
                         OpMethod::MP_Static);
@@ -1022,7 +1044,7 @@ void OpEmitter::genInferredTypeCollectiveParamBuilder() {
 
   // Result types
   body << formatv(R"(
-    SmallVector<Type, 2> inferredReturnTypes;
+    ::mlir::SmallVector<::mlir::Type, 2> inferredReturnTypes;
     if (succeeded({0}::inferReturnTypes(odsBuilder.getContext(),
                   {1}.location, operands,
                   {1}.attributes.getDictionary({1}.getContext()),
@@ -1036,7 +1058,7 @@ void OpEmitter::genInferredTypeCollectiveParamBuilder() {
 
   body << formatv(R"(
     } else
-      llvm::report_fatal_error("Failed to infer result type(s).");)",
+      ::llvm::report_fatal_error("Failed to infer result type(s).");)",
                   opClass.getClassName(), builderOpState);
 }
 
@@ -1064,8 +1086,10 @@ void OpEmitter::genUseOperandAsResultTypeSeparateParamBuilder() {
 
 void OpEmitter::genUseAttrAsResultTypeBuilder() {
   std::string params =
-      std::string("OpBuilder &odsBuilder, OperationState &") + builderOpState +
-      ", ValueRange operands, ArrayRef<NamedAttribute> attributes";
+      std::string("::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &") +
+      builderOpState +
+      ", ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> "
+      "attributes";
   auto &m = opClass.newMethod("void", "build", params, OpMethod::MP_Static);
   auto &body = m.body();
 
@@ -1076,7 +1100,7 @@ void OpEmitter::genUseAttrAsResultTypeBuilder() {
   body << "  for (auto attr : attributes) {\n";
   body << "    if (attr.first != \"" << namedAttr.name << "\") continue;\n";
   if (namedAttr.attr.isTypeAttr()) {
-    resultType = "attr.second.cast<TypeAttr>().getValue()";
+    resultType = "attr.second.cast<::mlir::TypeAttr>().getValue()";
   } else {
     resultType = "attr.second.getType()";
   }
@@ -1096,7 +1120,7 @@ void OpEmitter::genUseAttrAsResultTypeBuilder() {
 
 void OpEmitter::genBuilder() {
   // Handle custom builders if provided.
-  // TODO(antiagainst): Create wrapper class for OpBuilder to hide the native
+  // TODO: Create wrapper class for OpBuilder to hide the native
   // TableGen API calls here.
   {
     auto *listInit = dyn_cast_or_null<ListInit>(def.getValueInit("builders"));
@@ -1154,10 +1178,12 @@ void OpEmitter::genCollectiveParamBuilder() {
   int numVariadicOperands = op.getNumVariableLengthOperands();
   int numNonVariadicOperands = numOperands - numVariadicOperands;
   // Signature
-  std::string params = std::string("OpBuilder &, OperationState &") +
-                       builderOpState +
-                       ", ArrayRef<Type> resultTypes, ValueRange operands, "
-                       "ArrayRef<NamedAttribute> attributes";
+  std::string params =
+      std::string("::mlir::OpBuilder &, ::mlir::OperationState &") +
+      builderOpState +
+      ", ::llvm::ArrayRef<::mlir::Type> resultTypes, ::mlir::ValueRange "
+      "operands, "
+      "::llvm::ArrayRef<::mlir::NamedAttribute> attributes";
   if (op.getNumVariadicRegions())
     params += ", unsigned numRegions";
   auto &m = opClass.newMethod("void", "build", params, OpMethod::MP_Static);
@@ -1190,7 +1216,7 @@ void OpEmitter::genCollectiveParamBuilder() {
   body << "  " << builderOpState << ".addTypes(resultTypes);\n";
 
   // Generate builder that infers type too.
-  // TODO(jpienaar): Expand to handle regions and successors.
+  // TODO: Expand to handle regions and successors.
   if (canInferType(op) && op.getNumSuccessors() == 0)
     genInferredTypeCollectiveParamBuilder();
 }
@@ -1203,7 +1229,7 @@ void OpEmitter::buildParamList(std::string &paramList,
   auto numResults = op.getNumResults();
   resultTypeNames.reserve(numResults);
 
-  paramList = "OpBuilder &odsBuilder, OperationState &";
+  paramList = "::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &";
   paramList.append(builderOpState);
 
   switch (typeParamKind) {
@@ -1218,18 +1244,18 @@ void OpEmitter::buildParamList(std::string &paramList,
         resultName = std::string(formatv("resultType{0}", i));
 
       if (result.isOptional())
-        paramList.append(", /*optional*/Type ");
+        paramList.append(", /*optional*/::mlir::Type ");
       else if (result.isVariadic())
-        paramList.append(", ArrayRef<Type> ");
+        paramList.append(", ::llvm::ArrayRef<::mlir::Type> ");
       else
-        paramList.append(", Type ");
+        paramList.append(", ::mlir::Type ");
       paramList.append(resultName);
 
       resultTypeNames.emplace_back(std::move(resultName));
     }
   } break;
   case TypeParamKind::Collective: {
-    paramList.append(", ArrayRef<Type> resultTypes");
+    paramList.append(", ::llvm::ArrayRef<::mlir::Type> resultTypes");
     resultTypeNames.push_back("resultTypes");
   } break;
   }
@@ -1254,10 +1280,10 @@ void OpEmitter::buildParamList(std::string &paramList,
       // Creating an APInt requires us to provide bitwidth, value, and
       // signedness, which is complicated compared to others. Similarly
       // for APFloat.
-      // TODO(b/144412160) Adjust the 'returnType' field of such attributes
+      // TODO: Adjust the 'returnType' field of such attributes
       // to support them.
       StringRef retType = namedAttr->attr.getReturnType();
-      if (retType == "APInt" || retType == "APFloat")
+      if (retType == "::llvm::APInt" || retType == "::llvm::APFloat")
         break;
 
       defaultValuedAttrStartIndex = i;
@@ -1269,11 +1295,11 @@ void OpEmitter::buildParamList(std::string &paramList,
     if (argument.is<tblgen::NamedTypeConstraint *>()) {
       const auto &operand = op.getOperand(numOperands);
       if (operand.isOptional())
-        paramList.append(", /*optional*/Value ");
+        paramList.append(", /*optional*/::mlir::Value ");
       else if (operand.isVariadic())
-        paramList.append(", ValueRange ");
+        paramList.append(", ::mlir::ValueRange ");
       else
-        paramList.append(", Value ");
+        paramList.append(", ::mlir::Value ");
       paramList.append(getArgumentName(op, numOperands));
       ++numOperands;
     } else {
@@ -1302,7 +1328,7 @@ void OpEmitter::buildParamList(std::string &paramList,
       // Attach default value if requested and possible.
       if (attrParamKind == AttrParamKind::UnwrappedValue &&
           i >= defaultValuedAttrStartIndex) {
-        bool isString = attr.getReturnType() == "StringRef";
+        bool isString = attr.getReturnType() == "::llvm::StringRef";
         paramList.append(" = ");
         if (isString)
           paramList.append("\"");
@@ -1316,7 +1342,8 @@ void OpEmitter::buildParamList(std::string &paramList,
 
   /// Insert parameters for each successor.
   for (const NamedSuccessor &succ : op.getSuccessors()) {
-    paramList += (succ.isVariadic() ? ", ArrayRef<Block *> " : ", Block *");
+    paramList += (succ.isVariadic() ? ", ::llvm::ArrayRef<::mlir::Block *> "
+                                    : ", ::mlir::Block *");
     paramList += succ.name;
   }
 
@@ -1412,7 +1439,7 @@ void OpEmitter::genCanonicalizerDecls() {
     return;
 
   const char *const params =
-      "OwningRewritePatternList &results, MLIRContext *context";
+      "::mlir::OwningRewritePatternList &results, ::mlir::MLIRContext *context";
   opClass.newMethod("void", "getCanonicalizationPatterns", params,
                     OpMethod::MP_Static, /*declOnly=*/true);
 }
@@ -1423,13 +1450,16 @@ void OpEmitter::genFolderDecls() {
 
   if (def.getValueAsBit("hasFolder")) {
     if (hasSingleResult) {
-      const char *const params = "ArrayRef<Attribute> operands";
-      opClass.newMethod("OpFoldResult", "fold", params, OpMethod::MP_None,
+      const char *const params = "::llvm::ArrayRef<::mlir::Attribute> operands";
+      opClass.newMethod("::mlir::OpFoldResult", "fold", params,
+                        OpMethod::MP_None,
                         /*declOnly=*/true);
     } else {
-      const char *const params = "ArrayRef<Attribute> operands, "
-                                 "SmallVectorImpl<OpFoldResult> &results";
-      opClass.newMethod("LogicalResult", "fold", params, OpMethod::MP_None,
+      const char *const params =
+          "::llvm::ArrayRef<::mlir::Attribute> operands, "
+          "::llvm::SmallVectorImpl<::mlir::OpFoldResult> &results";
+      opClass.newMethod("::mlir::LogicalResult", "fold", params,
+                        OpMethod::MP_None,
                         /*declOnly=*/true);
     }
   }
@@ -1444,7 +1474,7 @@ void OpEmitter::genOpInterfaceMethod(const tblgen::InterfaceOpTrait *opTrait) {
   alwaysDeclaredMethods.insert(alwaysDeclaredMethodsVec.begin(),
                                alwaysDeclaredMethodsVec.end());
 
-  for (const OpInterfaceMethod &method : interface.getMethods()) {
+  for (const InterfaceMethod &method : interface.getMethods()) {
     // Don't declare if the method has a body.
     if (method.getBody())
       continue;
@@ -1457,7 +1487,7 @@ void OpEmitter::genOpInterfaceMethod(const tblgen::InterfaceOpTrait *opTrait) {
     std::string args;
     llvm::raw_string_ostream os(args);
     interleaveComma(method.getArguments(), os,
-                    [&](const OpInterfaceMethod::Argument &arg) {
+                    [&](const InterfaceMethod::Argument &arg) {
                       os << arg.type << " " << arg.name;
                     });
     opClass.newMethod(method.getReturnType(), method.getName(), os.str(),
@@ -1523,9 +1553,9 @@ void OpEmitter::genSideEffectInterfaceMethods() {
 
   for (auto &it : interfaceEffects) {
     auto effectsParam =
-        llvm::formatv(
-            "SmallVectorImpl<SideEffects::EffectInstance<{0}>> &effects",
-            it.first())
+        llvm::formatv("::mlir::SmallVectorImpl<::mlir::SideEffects::"
+                      "EffectInstance<{0}>> &effects",
+                      it.first())
             .str();
 
     // Generate the 'getEffects' method.
@@ -1535,7 +1565,7 @@ void OpEmitter::genSideEffectInterfaceMethods() {
     // Add effect instances for each of the locations marked on the operation.
     for (auto &location : it.second) {
       if (location.kind != EffectKind::Static) {
-        body << "  for (Value value : getODS"
+        body << "  for (::mlir::Value value : getODS"
              << (location.kind == EffectKind::Operand ? "Operands" : "Results")
              << "(" << location.index << "))\n  ";
       }
@@ -1556,10 +1586,12 @@ void OpEmitter::genTypeInterfaceMethods() {
     return;
 
   auto &method = opClass.newMethod(
-      "LogicalResult", "inferReturnTypes",
-      "MLIRContext* context, Optional<Location> location, "
-      "ValueRange operands, DictionaryAttr attributes, RegionRange regions, "
-      "SmallVectorImpl<Type>& inferredReturnTypes",
+      "::mlir::LogicalResult", "inferReturnTypes",
+      "::mlir::MLIRContext* context, "
+      "::llvm::Optional<::mlir::Location> location, "
+      "::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, "
+      "::mlir::RegionRange regions, "
+      "::llvm::SmallVectorImpl<::mlir::Type>& inferredReturnTypes",
       OpMethod::MP_Static,
       /*declOnly=*/false);
   auto &os = method.body();
@@ -1567,14 +1599,19 @@ void OpEmitter::genTypeInterfaceMethods() {
 
   FmtContext fctx;
   fctx.withBuilder("odsBuilder");
-  os << "  Builder odsBuilder(context);\n";
+  os << "  ::mlir::Builder odsBuilder(context);\n";
 
   auto emitType =
       [&](const tblgen::Operator::ArgOrType &type) -> OpMethodBody & {
     if (type.isArg()) {
       auto argIndex = type.getArg();
       assert(!op.getArg(argIndex).is<NamedAttribute *>());
-      return os << "operands[" << argIndex << "].getType()";
+      auto arg = op.getArgToOperandOrAttribute(argIndex);
+      if (arg.kind() == Operator::OperandOrAttribute::Kind::Operand)
+        return os << "operands[" << arg.operandOrAttributeIndex()
+                  << "].getType()";
+      return os << "attributes[" << arg.operandOrAttributeIndex()
+                << "].getType()";
     } else {
       return os << tgfmt(*type.getType().getBuilderCall(), &fctx);
     }
@@ -1597,7 +1634,8 @@ void OpEmitter::genParser() {
     return;
 
   auto &method = opClass.newMethod(
-      "ParseResult", "parse", "OpAsmParser &parser, OperationState &result",
+      "::mlir::ParseResult", "parse",
+      "::mlir::OpAsmParser &parser, ::mlir::OperationState &result",
       OpMethod::MP_Static);
   FmtContext fctx;
   fctx.addSubst("cppClass", opClass.getClassName());
@@ -1614,7 +1652,7 @@ void OpEmitter::genPrinter() {
   if (!codeInit)
     return;
 
-  auto &method = opClass.newMethod("void", "print", "OpAsmPrinter &p");
+  auto &method = opClass.newMethod("void", "print", "::mlir::OpAsmPrinter &p");
   FmtContext fctx;
   fctx.addSubst("cppClass", opClass.getClassName());
   auto printer = codeInit->getValue().ltrim().rtrim(" \t\v\f\r");
@@ -1622,7 +1660,8 @@ void OpEmitter::genPrinter() {
 }
 
 void OpEmitter::genVerifier() {
-  auto &method = opClass.newMethod("LogicalResult", "verify", /*params=*/"");
+  auto &method =
+      opClass.newMethod("::mlir::LogicalResult", "verify", /*params=*/"");
   auto &body = method.body();
   body << "  if (failed(" << op.getAdaptorName()
        << "(*this).verify(this->getLoc()))) "
@@ -1657,7 +1696,7 @@ void OpEmitter::genVerifier() {
     auto printer = codeInit->getValue().ltrim().rtrim(" \t\v\f\r");
     body << "  " << tgfmt(printer, &fctx);
   } else {
-    body << "  return mlir::success();\n";
+    body << "  return ::mlir::success();\n";
   }
 }
 
@@ -1694,7 +1733,8 @@ void OpEmitter::genOperandResultVerifier(OpMethodBody &body,
       continue;
 
     // Emit a loop to check all the dynamic values in the pack.
-    body << "    for (Value v : valueGroup" << staticValue.index() << ") {\n";
+    body << "    for (::mlir::Value v : valueGroup" << staticValue.index()
+         << ") {\n";
 
     auto constraint = staticValue.value().constraint;
     body << "      (void)v;\n"
@@ -1727,12 +1767,12 @@ void OpEmitter::genRegionVerifier(OpMethodBody &body) {
     if (region.constraint.getPredicate().isNull())
       continue;
 
-    body << "    for (Region &region : ";
-    body << formatv(
-        region.isVariadic()
-            ? "{0}()"
-            : "MutableArrayRef<Region>(this->getOperation()->getRegion({1}))",
-        region.name, i);
+    body << "    for (::mlir::Region &region : ";
+    body << formatv(region.isVariadic()
+                        ? "{0}()"
+                        : "::mlir::MutableArrayRef<::mlir::Region>(this->"
+                          "getOperation()->getRegion({1}))",
+                    region.name, i);
     body << ") {\n";
     auto constraint = tgfmt(region.constraint.getConditionTemplate(),
                             &verifyCtx.withSelf("region"))
@@ -1766,9 +1806,10 @@ void OpEmitter::genSuccessorVerifier(OpMethodBody &body) {
     if (successor.constraint.getPredicate().isNull())
       continue;
 
-    body << "    for (Block *successor : ";
-    body << formatv(successor.isVariadic() ? "{0}()"
-                                           : "ArrayRef<Block *>({0}())",
+    body << "    for (::mlir::Block *successor : ";
+    body << formatv(successor.isVariadic()
+                        ? "{0}()"
+                        : "::llvm::ArrayRef<::mlir::Block *>({0}())",
                     successor.name);
     body << ") {\n";
     auto constraint = tgfmt(successor.constraint.getConditionTemplate(),
@@ -1864,7 +1905,7 @@ void OpEmitter::genTraits() {
 }
 
 void OpEmitter::genOpNameGetter() {
-  auto &method = opClass.newMethod("StringRef", "getOperationName",
+  auto &method = opClass.newMethod("::llvm::StringRef", "getOperationName",
                                    /*params=*/"", OpMethod::MP_Static);
   method.body() << "  return \"" << op.getOperationName() << "\";\n";
 }
@@ -1877,7 +1918,7 @@ void OpEmitter::genOpAsmInterface() {
   // TODO: We could also add a flag to allow operations to opt in to this
   // generation, even if they only have a single operation.
   int numResults = op.getNumResults();
-  if (numResults <= 1 || op.getTrait("OpAsmOpInterface::Trait"))
+  if (numResults <= 1 || op.getTrait("::mlir::OpAsmOpInterface::Trait"))
     return;
 
   SmallVector<StringRef, 4> resultNames(numResults);
@@ -1887,7 +1928,7 @@ void OpEmitter::genOpAsmInterface() {
   // Don't add the trait if none of the results have a valid name.
   if (llvm::all_of(resultNames, [](StringRef name) { return name.empty(); }))
     return;
-  opClass.addTrait("OpAsmOpInterface::Trait");
+  opClass.addTrait("::mlir::OpAsmOpInterface::Trait");
 
   // Generate the right accessor for the number of results.
   auto &method = opClass.newMethod("void", "getAsmResultNames",
@@ -1928,15 +1969,16 @@ class OpOperandAdaptorEmitter {
 
 OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(const Operator &op)
     : op(op), adaptor(op.getAdaptorName()) {
-  adaptor.newField("ValueRange", "odsOperands");
-  adaptor.newField("DictionaryAttr", "odsAttrs");
+  adaptor.newField("::mlir::ValueRange", "odsOperands");
+  adaptor.newField("::mlir::DictionaryAttr", "odsAttrs");
   const auto *attrSizedOperands =
       op.getTrait("OpTrait::AttrSizedOperandSegments");
   {
     auto &constructor = adaptor.newConstructor(
         attrSizedOperands
-            ? "ValueRange values, DictionaryAttr attrs"
-            : "ValueRange values, DictionaryAttr attrs = nullptr");
+            ? "::mlir::ValueRange values, ::mlir::DictionaryAttr attrs"
+            : "::mlir::ValueRange values, ::mlir::DictionaryAttr attrs = "
+              "nullptr");
     constructor.addMemberInitializer("odsOperands", "values");
     constructor.addMemberInitializer("odsAttrs", "attrs");
   }
@@ -1953,13 +1995,13 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(const Operator &op)
   std::string sizeAttrInit =
       formatv(adapterSegmentSizeAttrInitCode, "operand_segment_sizes");
   generateNamedOperandGetters(op, adaptor, sizeAttrInit,
-                              /*rangeType=*/"ValueRange",
+                              /*rangeType=*/"::mlir::ValueRange",
                               /*rangeBeginCall=*/"odsOperands.begin()",
                               /*rangeSizeCall=*/"odsOperands.size()",
                               /*getOperandCallPattern=*/"odsOperands[{0}]");
 
   FmtContext fctx;
-  fctx.withBuilder("mlir::Builder(odsAttrs.getContext())");
+  fctx.withBuilder("::mlir::Builder(odsAttrs.getContext())");
 
   auto emitAttr = [&](StringRef name, Attribute attr) {
     auto &body = adaptor.newMethod(attr.getStorageType(), name).body();
@@ -1995,14 +2037,14 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(const Operator &op)
 }
 
 void OpOperandAdaptorEmitter::addVerification() {
-  auto &method = adaptor.newMethod("LogicalResult", "verify",
-                                   /*params=*/"Location loc");
+  auto &method = adaptor.newMethod("::mlir::LogicalResult", "verify",
+                                   /*params=*/"::mlir::Location loc");
   auto &body = method.body();
 
   const char *checkAttrSizedValueSegmentsCode = R"(
   {
-    auto sizeAttr = odsAttrs.get("{0}").cast<DenseIntElementsAttr>();
-    auto numElements = sizeAttr.getType().cast<ShapedType>().getNumElements();
+    auto sizeAttr = odsAttrs.get("{0}").cast<::mlir::DenseIntElementsAttr>();
+    auto numElements = sizeAttr.getType().cast<::mlir::ShapedType>().getNumElements();
     if (numElements != {1})
       return emitError(loc, "'{0}' attribute for specifying {2} segments "
                        "must have {1} elements");
@@ -2081,10 +2123,44 @@ static void emitOpList(const std::vector<Record *> &defs, raw_ostream &os) {
       [&os]() { os << ",\n"; });
 }
 
+static std::string getOperationName(const Record &def) {
+  auto prefix = def.getValueAsDef("opDialect")->getValueAsString("name");
+  auto opName = def.getValueAsString("opName");
+  if (prefix.empty())
+    return std::string(opName);
+  return std::string(llvm::formatv("{0}.{1}", prefix, opName));
+}
+
+static std::vector<Record *>
+getAllDerivedDefinitions(const RecordKeeper &recordKeeper,
+                         StringRef className) {
+  Record *classDef = recordKeeper.getClass(className);
+  if (!classDef)
+    PrintFatalError("ERROR: Couldn't find the `" + className + "' class!\n");
+
+  llvm::Regex includeRegex(opIncFilter), excludeRegex(opExcFilter);
+  std::vector<Record *> defs;
+  for (const auto &def : recordKeeper.getDefs()) {
+    if (!def.second->isSubClassOf(classDef))
+      continue;
+    // Include if no include filter or include filter matches.
+    if (!opIncFilter.empty() &&
+        !includeRegex.match(getOperationName(*def.second)))
+      continue;
+    // Unless there is an exclude filter and it matches.
+    if (!opExcFilter.empty() &&
+        excludeRegex.match(getOperationName(*def.second)))
+      continue;
+    defs.push_back(def.second.get());
+  }
+
+  return defs;
+}
+
 static bool emitOpDecls(const RecordKeeper &recordKeeper, raw_ostream &os) {
   emitSourceFileHeader("Op Declarations", os);
 
-  const auto &defs = recordKeeper.getAllDerivedDefinitions("Op");
+  const auto &defs = getAllDerivedDefinitions(recordKeeper, "Op");
   emitOpClasses(defs, os, /*emitDecl=*/true);
 
   return false;
@@ -2093,7 +2169,7 @@ static bool emitOpDecls(const RecordKeeper &recordKeeper, raw_ostream &os) {
 static bool emitOpDefs(const RecordKeeper &recordKeeper, raw_ostream &os) {
   emitSourceFileHeader("Op Definitions", os);
 
-  const auto &defs = recordKeeper.getAllDerivedDefinitions("Op");
+  const auto &defs = getAllDerivedDefinitions(recordKeeper, "Op");
   emitOpList(defs, os);
   emitOpClasses(defs, os, /*emitDecl=*/false);
 
diff --git a/mlir/tools/mlir-tblgen/OpDocGen.cpp b/mlir/tools/mlir-tblgen/OpDocGen.cpp
index a432b4a2f21c4..df78556c1c77b 100644
--- a/mlir/tools/mlir-tblgen/OpDocGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDocGen.cpp
@@ -197,7 +197,7 @@ static void emitDialectDoc(const Dialect &dialect, ArrayRef<Operator> ops,
 
   os << "[TOC]\n\n";
 
-  // TODO(antiagainst): Add link between use and def for types
+  // TODO: Add link between use and def for types
   if (!types.empty()) {
     os << "## Type definition\n\n";
     for (const Type &type : types)
diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
index 9fa87e3a84277..3fcbeeff1e6fd 100644
--- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
@@ -10,8 +10,8 @@
 #include "mlir/Support/LogicalResult.h"
 #include "mlir/TableGen/Format.h"
 #include "mlir/TableGen/GenInfo.h"
+#include "mlir/TableGen/Interfaces.h"
 #include "mlir/TableGen/OpClass.h"
-#include "mlir/TableGen/OpInterfaces.h"
 #include "mlir/TableGen/OpTrait.h"
 #include "mlir/TableGen/Operator.h"
 #include "llvm/ADT/MapVector.h"
@@ -270,6 +270,10 @@ class OptionalElement : public Element {
 //===----------------------------------------------------------------------===//
 
 namespace {
+
+using ConstArgument =
+    llvm::PointerUnion<const NamedAttribute *, const NamedTypeConstraint *>;
+
 struct OperationFormat {
   /// This class represents a specific resolver for an operand or result type.
   class TypeResolution {
@@ -280,15 +284,22 @@ struct OperationFormat {
     Optional<int> getBuilderIdx() const { return builderIdx; }
     void setBuilderIdx(int idx) { builderIdx = idx; }
 
-    /// Get the variable this type is resolved to, or None.
-    const NamedTypeConstraint *getVariable() const { return variable; }
+    /// Get the variable this type is resolved to, or nullptr.
+    const NamedTypeConstraint *getVariable() const {
+      return resolver.dyn_cast<const NamedTypeConstraint *>();
+    }
+    /// Get the attribute this type is resolved to, or nullptr.
+    const NamedAttribute *getAttribute() const {
+      return resolver.dyn_cast<const NamedAttribute *>();
+    }
+    /// Get the transformer for the type of the variable, or None.
     Optional<StringRef> getVarTransformer() const {
       return variableTransformer;
     }
-    void setVariable(const NamedTypeConstraint *var,
-                     Optional<StringRef> transformer) {
-      variable = var;
+    void setResolver(ConstArgument arg, Optional<StringRef> transformer) {
+      resolver = arg;
       variableTransformer = transformer;
+      assert(getVariable() || getAttribute());
     }
 
   private:
@@ -296,8 +307,8 @@ struct OperationFormat {
     /// 'buildableTypes' in the parent format.
     Optional<int> builderIdx;
     /// If the type is resolved based upon another operand or result, this is
-    /// the variable that this type is resolved to.
-    const NamedTypeConstraint *variable;
+    /// the variable or the attribute that this type is resolved to.
+    ConstArgument resolver;
     /// If the type is resolved based upon another operand or result, this is
     /// a transformer to apply to the variable when resolving.
     Optional<StringRef> variableTransformer;
@@ -371,8 +382,8 @@ const char *const attrParserCode = R"(
 /// {3}: The constant builder call to create an attribute of the enum type.
 const char *const enumAttrParserCode = R"(
   {
-    StringAttr attrVal;
-    NamedAttrList attrStorage;
+    ::mlir::StringAttr attrVal;
+    ::mlir::NamedAttrList attrStorage;
     auto loc = parser.getCurrentLocation();
     if (parser.parseAttribute(attrVal, parser.getBuilder().getNoneType(),
                               "{0}", attrStorage))
@@ -396,8 +407,9 @@ const char *const variadicOperandParserCode = R"(
 )";
 const char *const optionalOperandParserCode = R"(
   {
-    OpAsmParser::OperandType operand;
-    OptionalParseResult parseResult = parser.parseOptionalOperand(operand);
+    ::mlir::OpAsmParser::OperandType operand;
+    ::mlir::OptionalParseResult parseResult =
+                                    parser.parseOptionalOperand(operand);
     if (parseResult.hasValue()) {
       if (failed(*parseResult))
         return failure();
@@ -419,8 +431,9 @@ const char *const variadicTypeParserCode = R"(
 )";
 const char *const optionalTypeParserCode = R"(
   {
-    Type optionalType;
-    OptionalParseResult parseResult = parser.parseOptionalType(optionalType);
+    ::mlir::Type optionalType;
+    ::mlir::OptionalParseResult parseResult =
+                                    parser.parseOptionalType(optionalType);
     if (parseResult.hasValue()) {
       if (failed(*parseResult))
         return failure();
@@ -438,7 +451,7 @@ const char *const typeParserCode = R"(
 /// {0}: The name for the input type list.
 /// {1}: The name for the result type list.
 const char *const functionalTypeParserCode = R"(
-  FunctionType {0}__{1}_functionType;
+  ::mlir::FunctionType {0}__{1}_functionType;
   if (parser.parseType({0}__{1}_functionType))
     return failure();
   {0}Types = {0}__{1}_functionType.getInputs();
@@ -449,9 +462,9 @@ const char *const functionalTypeParserCode = R"(
 ///
 /// {0}: The name for the successor list.
 const char *successorListParserCode = R"(
-  SmallVector<Block *, 2> {0}Successors;
+  ::llvm::SmallVector<::mlir::Block *, 2> {0}Successors;
   {
-    Block *succ;
+    ::mlir::Block *succ;
     auto firstSucc = parser.parseOptionalSuccessor(succ);
     if (firstSucc.hasValue()) {
       if (failed(*firstSucc))
@@ -472,7 +485,7 @@ const char *successorListParserCode = R"(
 ///
 /// {0}: The name of the successor.
 const char *successorParserCode = R"(
-  Block *{0}Successor = nullptr;
+  ::mlir::Block *{0}Successor = nullptr;
   if (parser.parseSuccessor({0}Successor))
     return failure();
 )";
@@ -546,31 +559,34 @@ static void genElementParserStorage(Element *element, OpMethodBody &body) {
   } else if (auto *operand = dyn_cast<OperandVariable>(element)) {
     StringRef name = operand->getVar()->name;
     if (operand->getVar()->isVariableLength()) {
-      body << "  SmallVector<OpAsmParser::OperandType, 4> " << name
-           << "Operands;\n";
+      body << "  ::mlir::SmallVector<::mlir::OpAsmParser::OperandType, 4> "
+           << name << "Operands;\n";
     } else {
-      body << "  OpAsmParser::OperandType " << name << "RawOperands[1];\n"
-           << "  ArrayRef<OpAsmParser::OperandType> " << name << "Operands("
-           << name << "RawOperands);";
+      body << "  ::mlir::OpAsmParser::OperandType " << name
+           << "RawOperands[1];\n"
+           << "  ::llvm::ArrayRef<::mlir::OpAsmParser::OperandType> " << name
+           << "Operands(" << name << "RawOperands);";
     }
     body << llvm::formatv(
-        "  llvm::SMLoc {0}OperandsLoc = parser.getCurrentLocation();\n"
+        "  ::llvm::SMLoc {0}OperandsLoc = parser.getCurrentLocation();\n"
         "  (void){0}OperandsLoc;\n",
         name);
   } else if (auto *dir = dyn_cast<TypeDirective>(element)) {
     ArgumentLengthKind lengthKind;
     StringRef name = getTypeListName(dir->getOperand(), lengthKind);
     if (lengthKind != ArgumentLengthKind::Single)
-      body << "  SmallVector<Type, 1> " << name << "Types;\n";
+      body << "  ::mlir::SmallVector<::mlir::Type, 1> " << name << "Types;\n";
     else
-      body << llvm::formatv("  Type {0}RawTypes[1];\n", name)
-           << llvm::formatv("  ArrayRef<Type> {0}Types({0}RawTypes);\n", name);
+      body << llvm::formatv("  ::mlir::Type {0}RawTypes[1];\n", name)
+           << llvm::formatv(
+                  "  ::llvm::ArrayRef<::mlir::Type> {0}Types({0}RawTypes);\n",
+                  name);
   } else if (auto *dir = dyn_cast<FunctionalTypeDirective>(element)) {
     ArgumentLengthKind ignored;
-    body << "  ArrayRef<Type> " << getTypeListName(dir->getInputs(), ignored)
-         << "Types;\n";
-    body << "  ArrayRef<Type> " << getTypeListName(dir->getResults(), ignored)
-         << "Types;\n";
+    body << "  ::llvm::ArrayRef<::mlir::Type> "
+         << getTypeListName(dir->getInputs(), ignored) << "Types;\n";
+    body << "  ::llvm::ArrayRef<::mlir::Type> "
+         << getTypeListName(dir->getResults(), ignored) << "Types;\n";
   }
 }
 
@@ -655,8 +671,9 @@ static void genElementParser(Element *element, OpMethodBody &body,
          << "(result.attributes))\n"
          << "    return failure();\n";
   } else if (isa<OperandsDirective>(element)) {
-    body << "  llvm::SMLoc allOperandLoc = parser.getCurrentLocation();\n"
-         << "  SmallVector<OpAsmParser::OperandType, 4> allOperands;\n"
+    body << "  ::llvm::SMLoc allOperandLoc = parser.getCurrentLocation();\n"
+         << "  ::mlir::SmallVector<::mlir::OpAsmParser::OperandType, 4> "
+            "allOperands;\n"
          << "  if (parser.parseOperandList(allOperands))\n"
          << "    return failure();\n";
   } else if (isa<SuccessorsDirective>(element)) {
@@ -682,7 +699,8 @@ static void genElementParser(Element *element, OpMethodBody &body,
 
 void OperationFormat::genParser(Operator &op, OpClass &opClass) {
   auto &method = opClass.newMethod(
-      "ParseResult", "parse", "OpAsmParser &parser, OperationState &result",
+      "::mlir::ParseResult", "parse",
+      "::mlir::OpAsmParser &parser, ::mlir::OperationState &result",
       OpMethod::MP_Static);
   auto &body = method.body();
 
@@ -722,11 +740,11 @@ void OperationFormat::genParserTypeResolution(Operator &op,
       continue;
     // Ensure that we don't verify the same variables twice.
     const NamedTypeConstraint *variable = resolver.getVariable();
-    if (!verifiedVariables.insert(variable).second)
+    if (!variable || !verifiedVariables.insert(variable).second)
       continue;
 
     auto constraint = variable->constraint;
-    body << "  for (Type type : " << variable->name << "Types) {\n"
+    body << "  for (::mlir::Type type : " << variable->name << "Types) {\n"
          << "    (void)type;\n"
          << "    if (!("
          << tgfmt(constraint.getConditionTemplate(),
@@ -744,7 +762,7 @@ void OperationFormat::genParserTypeResolution(Operator &op,
     FmtContext typeBuilderCtx;
     typeBuilderCtx.withBuilder("parser.getBuilder()");
     for (auto &it : buildableTypes)
-      body << "  Type odsBuildableType" << it.second << " = "
+      body << "  ::mlir::Type odsBuildableType" << it.second << " = "
            << tgfmt(it.first, &typeBuilderCtx) << ";\n";
   }
 
@@ -757,6 +775,12 @@ void OperationFormat::genParserTypeResolution(Operator &op,
         body << tgfmt(*tform, &FmtContext().withSelf(var->name + "Types[0]"));
       else
         body << var->name << "Types";
+    } else if (const NamedAttribute *attr = resolver.getAttribute()) {
+      if (Optional<StringRef> tform = resolver.getVarTransformer())
+        body << tgfmt(*tform,
+                      &FmtContext().withSelf(attr->name + "Attr.getType()"));
+      else
+        body << attr->name << "Attr.getType()";
     } else {
       body << curVar << "Types";
     }
@@ -791,7 +815,7 @@ void OperationFormat::genParserTypeResolution(Operator &op,
     // llvm::concat does not allow the case of a single range, so guard it here.
     body << "  if (parser.resolveOperands(";
     if (op.getNumOperands() > 1) {
-      body << "llvm::concat<const OpAsmParser::OperandType>(";
+      body << "::llvm::concat<const ::mlir::OpAsmParser::OperandType>(";
       llvm::interleaveComma(op.getOperands(), body, [&](auto &operand) {
         body << operand.name << "Operands";
       });
@@ -811,10 +835,10 @@ void OperationFormat::genParserTypeResolution(Operator &op,
     // once. Use llvm::concat to perform the merge. llvm::concat does not allow
     // the case of a single range, so guard it here.
     if (op.getNumOperands() > 1) {
-      body << "llvm::concat<const Type>(";
+      body << "::llvm::concat<const Type>(";
       llvm::interleaveComma(
           llvm::seq<int>(0, op.getNumOperands()), body, [&](int i) {
-            body << "ArrayRef<Type>(";
+            body << "::llvm::ArrayRef<::mlir::Type>(";
             emitTypeResolver(operandTypes[i], op.getOperand(i).name);
             body << ")";
           });
@@ -833,10 +857,28 @@ void OperationFormat::genParserTypeResolution(Operator &op,
   for (unsigned i = 0, e = op.getNumOperands(); i != e; ++i) {
     NamedTypeConstraint &operand = op.getOperand(i);
     body << "  if (parser.resolveOperands(" << operand.name << "Operands, ";
-    emitTypeResolver(operandTypes[i], operand.name);
 
-    // If this isn't a buildable type, verify the sizes match by adding the loc.
-    if (!operandTypes[i].getBuilderIdx())
+    // Resolve the type of this operand.
+    TypeResolution &operandType = operandTypes[i];
+    emitTypeResolver(operandType, operand.name);
+
+    // If the type is resolved by a non-variadic variable, index into the
+    // resolved type list. This allows for resolving the types of a variadic
+    // operand list from a non-variadic variable.
+    bool verifyOperandAndTypeSize = true;
+    if (auto *resolverVar = operandType.getVariable()) {
+      if (!resolverVar->isVariadic() && !operandType.getVarTransformer()) {
+        body << "[0]";
+        verifyOperandAndTypeSize = false;
+      }
+    } else {
+      verifyOperandAndTypeSize = !operandType.getBuilderIdx();
+    }
+
+    // Check to see if the sizes between the types and operands must match. If
+    // they do, provide the operand location to select the proper resolution
+    // overload.
+    if (verifyOperandAndTypeSize)
       body << ", " << operand.name << "OperandsLoc";
     body << ", result.operands))\n    return failure();\n";
   }
@@ -946,9 +988,11 @@ static OpMethodBody &genTypeOperandPrinter(Element *arg, OpMethodBody &body) {
     return body << var->name << "().getTypes()";
   if (var->isOptional())
     return body << llvm::formatv(
-               "({0}() ? ArrayRef<Type>({0}().getType()) : ArrayRef<Type>())",
+               "({0}() ? ::llvm::ArrayRef<::mlir::Type>({0}().getType()) : "
+               "::llvm::ArrayRef<::mlir::Type>())",
                var->name);
-  return body << "ArrayRef<Type>(" << var->name << "().getType())";
+  return body << "::llvm::ArrayRef<::mlir::Type>(" << var->name
+              << "().getType())";
 }
 
 /// Generate the code for printing the given element.
@@ -1014,7 +1058,8 @@ static void genElementPrinter(Element *element, OpMethodBody &body,
       body << "  p.printAttribute(" << var->name << "Attr());\n";
   } else if (auto *operand = dyn_cast<OperandVariable>(element)) {
     if (operand->getVar()->isOptional()) {
-      body << "  if (Value value = " << operand->getVar()->name << "())\n"
+      body << "  if (::mlir::Value value = " << operand->getVar()->name
+           << "())\n"
            << "    p << value;\n";
     } else {
       body << "  p << " << operand->getVar()->name << "();\n";
@@ -1022,13 +1067,13 @@ static void genElementPrinter(Element *element, OpMethodBody &body,
   } else if (auto *successor = dyn_cast<SuccessorVariable>(element)) {
     const NamedSuccessor *var = successor->getVar();
     if (var->isVariadic())
-      body << "  llvm::interleaveComma(" << var->name << "(), p);\n";
+      body << "  ::llvm::interleaveComma(" << var->name << "(), p);\n";
     else
       body << "  p << " << var->name << "();\n";
   } else if (isa<OperandsDirective>(element)) {
     body << "  p << getOperation()->getOperands();\n";
   } else if (isa<SuccessorsDirective>(element)) {
-    body << "  llvm::interleaveComma(getOperation()->getSuccessors(), p);\n";
+    body << "  ::llvm::interleaveComma(getOperation()->getSuccessors(), p);\n";
   } else if (auto *dir = dyn_cast<TypeDirective>(element)) {
     body << "  p << ";
     genTypeOperandPrinter(dir->getOperand(), body) << ";\n";
@@ -1301,7 +1346,8 @@ Token FormatLexer::lexIdentifier(const char *tokStart) {
 
 /// Function to find an element within the given range that has the same name as
 /// 'name'.
-template <typename RangeT> static auto findArg(RangeT &&range, StringRef name) {
+template <typename RangeT>
+static auto findArg(RangeT &&range, StringRef name) {
   auto it = llvm::find_if(range, [=](auto &arg) { return arg.name == name; });
   return it != range.end() ? &*it : nullptr;
 }
@@ -1324,7 +1370,7 @@ class FormatParser {
   /// type as well as an optional transformer to apply to that type in order to
   /// properly resolve the type of a variable.
   struct TypeResolutionInstance {
-    const NamedTypeConstraint *type;
+    ConstArgument resolver;
     Optional<StringRef> transformer;
   };
 
@@ -1363,10 +1409,15 @@ class FormatParser {
   void handleSameTypesConstraint(
       llvm::StringMap<TypeResolutionInstance> &variableTyResolver,
       bool includeResults);
+  /// Check for inferable type resolution based on another operand, result, or
+  /// attribute.
+  void handleTypesMatchConstraint(
+      llvm::StringMap<TypeResolutionInstance> &variableTyResolver,
+      llvm::Record def);
 
-  /// Returns an argument with the given name that has been seen within the
-  /// format.
-  const NamedTypeConstraint *findSeenArg(StringRef name);
+  /// Returns an argument or attribute with the given name that has been seen
+  /// within the format.
+  ConstArgument findSeenArg(StringRef name);
 
   /// Parse a specific element.
   LogicalResult parseElement(std::unique_ptr<Element> &element,
@@ -1475,9 +1526,7 @@ LogicalResult FormatParser::parse() {
     } else if (def.getName() == "SameOperandsAndResultType") {
       handleSameTypesConstraint(variableTyResolver, /*includeResults=*/true);
     } else if (def.isSubClassOf("TypesMatchWith")) {
-      if (const auto *lhsArg = findSeenArg(def.getValueAsString("lhs")))
-        variableTyResolver[def.getValueAsString("rhs")] = {
-            lhsArg, def.getValueAsString("transformer")};
+      handleTypesMatchConstraint(variableTyResolver, def);
     }
   }
 
@@ -1586,8 +1635,8 @@ LogicalResult FormatParser::verifyOperands(
     // Check to see if we can infer this type from another variable.
     auto varResolverIt = variableTyResolver.find(op.getOperand(i).name);
     if (varResolverIt != variableTyResolver.end()) {
-      fmt.operandTypes[i].setVariable(varResolverIt->second.type,
-                                      varResolverIt->second.transformer);
+      TypeResolutionInstance &resolver = varResolverIt->second;
+      fmt.operandTypes[i].setResolver(resolver.resolver, resolver.transformer);
       continue;
     }
 
@@ -1625,8 +1674,8 @@ LogicalResult FormatParser::verifyResults(
     // Check to see if we can infer this type from another variable.
     auto varResolverIt = variableTyResolver.find(op.getResultName(i));
     if (varResolverIt != variableTyResolver.end()) {
-      fmt.resultTypes[i].setVariable(varResolverIt->second.type,
-                                     varResolverIt->second.transformer);
+      TypeResolutionInstance resolver = varResolverIt->second;
+      fmt.resultTypes[i].setResolver(resolver.resolver, resolver.transformer);
       continue;
     }
 
@@ -1673,7 +1722,7 @@ void FormatParser::handleAllTypesMatchConstraint(
     llvm::StringMap<TypeResolutionInstance> &variableTyResolver) {
   for (unsigned i = 0, e = values.size(); i != e; ++i) {
     // Check to see if this value matches a resolved operand or result type.
-    const NamedTypeConstraint *arg = findSeenArg(values[i]);
+    ConstArgument arg = findSeenArg(values[i]);
     if (!arg)
       continue;
 
@@ -1710,11 +1759,23 @@ void FormatParser::handleSameTypesConstraint(
   }
 }
 
-const NamedTypeConstraint *FormatParser::findSeenArg(StringRef name) {
-  if (auto *arg = findArg(op.getOperands(), name))
+void FormatParser::handleTypesMatchConstraint(
+    llvm::StringMap<TypeResolutionInstance> &variableTyResolver,
+    llvm::Record def) {
+  StringRef lhsName = def.getValueAsString("lhs");
+  StringRef rhsName = def.getValueAsString("rhs");
+  StringRef transformer = def.getValueAsString("transformer");
+  if (ConstArgument arg = findSeenArg(lhsName))
+    variableTyResolver[rhsName] = {arg, transformer};
+}
+
+ConstArgument FormatParser::findSeenArg(StringRef name) {
+  if (const NamedTypeConstraint *arg = findArg(op.getOperands(), name))
     return seenOperandTypes.test(arg - op.operand_begin()) ? arg : nullptr;
-  if (auto *arg = findArg(op.getResults(), name))
+  if (const NamedTypeConstraint *arg = findArg(op.getResults(), name))
     return seenResultTypes.test(arg - op.result_begin()) ? arg : nullptr;
+  if (const NamedAttribute *attr = findArg(op.getAttributes(), name))
+    return seenAttrs.find_as(attr) != seenAttrs.end() ? attr : nullptr;
   return nullptr;
 }
 
@@ -2057,7 +2118,7 @@ FormatParser::parseTypeDirectiveOperand(std::unique_ptr<Element> &element) {
 //===----------------------------------------------------------------------===//
 
 void mlir::tblgen::generateOpFormat(const Operator &constOp, OpClass &opClass) {
-  // TODO(riverriddle) Operator doesn't expose all necessary functionality via
+  // TODO: Operator doesn't expose all necessary functionality via
   // the const interface.
   Operator &op = const_cast<Operator &>(constOp);
   if (!op.hasAssemblyFormat())
diff --git a/mlir/tools/mlir-tblgen/OpInterfacesGen.cpp b/mlir/tools/mlir-tblgen/OpInterfacesGen.cpp
index a693b02112474..8b27bc6de7c5e 100644
--- a/mlir/tools/mlir-tblgen/OpInterfacesGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpInterfacesGen.cpp
@@ -13,7 +13,7 @@
 #include "DocGenUtilities.h"
 #include "mlir/TableGen/Format.h"
 #include "mlir/TableGen/GenInfo.h"
-#include "mlir/TableGen/OpInterfaces.h"
+#include "mlir/TableGen/Interfaces.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -22,70 +22,165 @@
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
 
-using namespace llvm;
 using namespace mlir;
+using mlir::tblgen::Interface;
+using mlir::tblgen::InterfaceMethod;
 using mlir::tblgen::OpInterface;
-using mlir::tblgen::OpInterfaceMethod;
 
-// Emit the method name and argument list for the given method. If
-// 'addOperationArg' is true, then an Operation* argument is added to the
-// beginning of the argument list.
-static void emitMethodNameAndArgs(const OpInterfaceMethod &method,
-                                  raw_ostream &os, bool addOperationArg) {
+/// Emit a string corresponding to a C++ type, followed by a space if necessary.
+static raw_ostream &emitCPPType(StringRef type, raw_ostream &os) {
+  type = type.trim();
+  os << type;
+  if (type.back() != '&' && type.back() != '*')
+    os << " ";
+  return os;
+}
+
+/// Emit the method name and argument list for the given method. If 'addThisArg'
+/// is true, then an argument is added to the beginning of the argument list for
+/// the concrete value.
+static void emitMethodNameAndArgs(const InterfaceMethod &method,
+                                  raw_ostream &os, StringRef valueType,
+                                  bool addThisArg, bool addConst) {
   os << method.getName() << '(';
-  if (addOperationArg)
-    os << "Operation *tablegen_opaque_op" << (method.arg_empty() ? "" : ", ");
+  if (addThisArg)
+    emitCPPType(valueType, os)
+        << "tablegen_opaque_val" << (method.arg_empty() ? "" : ", ");
   llvm::interleaveComma(method.getArguments(), os,
-                        [&](const OpInterfaceMethod::Argument &arg) {
+                        [&](const InterfaceMethod::Argument &arg) {
                           os << arg.type << " " << arg.name;
                         });
   os << ')';
+  if (addConst)
+    os << " const";
 }
 
-// Get an array of all OpInterface definitions but exclude those subclassing
-// "DeclareOpInterfaceMethods".
-static std::vector<Record *>
-getAllOpInterfaceDefinitions(const RecordKeeper &recordKeeper) {
-  std::vector<Record *> defs =
+/// Get an array of all OpInterface definitions but exclude those subclassing
+/// "DeclareOpInterfaceMethods".
+static std::vector<llvm::Record *>
+getAllOpInterfaceDefinitions(const llvm::RecordKeeper &recordKeeper) {
+  std::vector<llvm::Record *> defs =
       recordKeeper.getAllDerivedDefinitions("OpInterface");
 
-  llvm::erase_if(defs, [](const Record *def) {
+  llvm::erase_if(defs, [](const llvm::Record *def) {
     return def->isSubClassOf("DeclareOpInterfaceMethods");
   });
   return defs;
 }
 
+namespace {
+/// This struct is the base generator used when processing tablegen interfaces.
+class InterfaceGenerator {
+public:
+  bool emitInterfaceDefs();
+  bool emitInterfaceDecls();
+  bool emitInterfaceDocs();
+
+protected:
+  InterfaceGenerator(std::vector<llvm::Record *> &&defs, raw_ostream &os)
+      : defs(std::move(defs)), os(os) {}
+
+  void emitConceptDecl(Interface &interface);
+  void emitModelDecl(Interface &interface);
+  void emitTraitDecl(Interface &interface, StringRef interfaceName,
+                     StringRef interfaceTraitsName);
+  void emitInterfaceDecl(Interface interface);
+
+  /// The set of interface records to emit.
+  std::vector<llvm::Record *> defs;
+  // The stream to emit to.
+  raw_ostream &os;
+  /// The C++ value type of the interface, e.g. Operation*.
+  StringRef valueType;
+  /// The C++ base interface type.
+  StringRef interfaceBaseType;
+  /// The name of the typename for the value template.
+  StringRef valueTemplate;
+  /// The format context to use for methods.
+  tblgen::FmtContext nonStaticMethodFmt;
+  tblgen::FmtContext traitMethodFmt;
+};
+
+/// A specialized generator for attribute interfaces.
+struct AttrInterfaceGenerator : public InterfaceGenerator {
+  AttrInterfaceGenerator(const llvm::RecordKeeper &records, raw_ostream &os)
+      : InterfaceGenerator(records.getAllDerivedDefinitions("AttrInterface"),
+                           os) {
+    valueType = "::mlir::Attribute";
+    interfaceBaseType = "AttrInterface";
+    valueTemplate = "ConcreteAttr";
+    StringRef castCode = "(tablegen_opaque_val.cast<ConcreteAttr>())";
+    nonStaticMethodFmt.addSubst("_attr", castCode).withSelf(castCode);
+    traitMethodFmt.addSubst("_attr",
+                            "(*static_cast<const ConcreteAttr *>(this))");
+  }
+};
+/// A specialized generator for operaton interfaces.
+struct OpInterfaceGenerator : public InterfaceGenerator {
+  OpInterfaceGenerator(const llvm::RecordKeeper &records, raw_ostream &os)
+      : InterfaceGenerator(getAllOpInterfaceDefinitions(records), os) {
+    valueType = "::mlir::Operation *";
+    interfaceBaseType = "OpInterface";
+    valueTemplate = "ConcreteOp";
+    StringRef castCode = "(llvm::cast<ConcreteOp>(tablegen_opaque_val))";
+    nonStaticMethodFmt.withOp(castCode).withSelf(castCode);
+    traitMethodFmt.withOp("(*static_cast<ConcreteOp *>(this))");
+  }
+};
+/// A specialized generator for type interfaces.
+struct TypeInterfaceGenerator : public InterfaceGenerator {
+  TypeInterfaceGenerator(const llvm::RecordKeeper &records, raw_ostream &os)
+      : InterfaceGenerator(records.getAllDerivedDefinitions("TypeInterface"),
+                           os) {
+    valueType = "::mlir::Type";
+    interfaceBaseType = "TypeInterface";
+    valueTemplate = "ConcreteType";
+    StringRef castCode = "(tablegen_opaque_val.cast<ConcreteType>())";
+    nonStaticMethodFmt.addSubst("_type", castCode).withSelf(castCode);
+    traitMethodFmt.addSubst("_type",
+                            "(*static_cast<const ConcreteType *>(this))");
+  }
+};
+} // end anonymous namespace
+
 //===----------------------------------------------------------------------===//
 // GEN: Interface definitions
 //===----------------------------------------------------------------------===//
 
-static void emitInterfaceDef(OpInterface &interface, raw_ostream &os) {
+static void emitInterfaceDef(Interface interface, StringRef valueType,
+                             raw_ostream &os) {
   StringRef interfaceName = interface.getName();
+  StringRef cppNamespace = interface.getCppNamespace();
+  cppNamespace.consume_front("::");
 
   // Insert the method definitions.
+  bool isOpInterface = isa<OpInterface>(interface);
   for (auto &method : interface.getMethods()) {
-    os << method.getReturnType() << " " << interfaceName << "::";
-    emitMethodNameAndArgs(method, os, /*addOperationArg=*/false);
+    emitCPPType(method.getReturnType(), os);
+    if (!cppNamespace.empty())
+      os << cppNamespace << "::";
+    os << interfaceName << "::";
+    emitMethodNameAndArgs(method, os, valueType, /*addThisArg=*/false,
+                          /*addConst=*/!isOpInterface);
 
     // Forward to the method on the concrete operation type.
     os << " {\n      return getImpl()->" << method.getName() << '(';
-    if (!method.isStatic())
-      os << "getOperation()" << (method.arg_empty() ? "" : ", ");
+    if (!method.isStatic()) {
+      os << (isOpInterface ? "getOperation()" : "*this");
+      os << (method.arg_empty() ? "" : ", ");
+    }
     llvm::interleaveComma(
         method.getArguments(), os,
-        [&](const OpInterfaceMethod::Argument &arg) { os << arg.name; });
+        [&](const InterfaceMethod::Argument &arg) { os << arg.name; });
     os << ");\n  }\n";
   }
 }
 
-static bool emitInterfaceDefs(const RecordKeeper &recordKeeper,
-                              raw_ostream &os) {
-  llvm::emitSourceFileHeader("Operation Interface Definitions", os);
+bool InterfaceGenerator::emitInterfaceDefs() {
+  llvm::emitSourceFileHeader("Interface Definitions", os);
 
-  for (const auto *def : getAllOpInterfaceDefinitions(recordKeeper)) {
-    OpInterface interface(def);
-    emitInterfaceDef(interface, os);
-  }
+  for (const auto *def : defs)
+    emitInterfaceDef(Interface(def), valueType, os);
   return false;
 }
 
@@ -93,122 +188,141 @@ static bool emitInterfaceDefs(const RecordKeeper &recordKeeper,
 // GEN: Interface declarations
 //===----------------------------------------------------------------------===//
 
-static void emitConceptDecl(OpInterface &interface, raw_ostream &os) {
+void InterfaceGenerator::emitConceptDecl(Interface &interface) {
   os << "  class Concept {\n"
      << "  public:\n"
      << "    virtual ~Concept() = default;\n";
 
   // Insert each of the pure virtual concept methods.
   for (auto &method : interface.getMethods()) {
-    os << "    virtual " << method.getReturnType() << " ";
-    emitMethodNameAndArgs(method, os, /*addOperationArg=*/!method.isStatic());
+    os << "    virtual ";
+    emitCPPType(method.getReturnType(), os);
+    emitMethodNameAndArgs(method, os, valueType,
+                          /*addThisArg=*/!method.isStatic(), /*addConst=*/true);
     os << " = 0;\n";
   }
   os << "  };\n";
 }
 
-static void emitModelDecl(OpInterface &interface, raw_ostream &os) {
-  os << "  template<typename ConcreteOp>\n";
-  os << "  class Model : public Concept {\npublic:\n";
+void InterfaceGenerator::emitModelDecl(Interface &interface) {
+  os << "  template<typename " << valueTemplate << ">\n";
+  os << "  class Model : public Concept {\n  public:\n";
 
   // Insert each of the virtual method overrides.
   for (auto &method : interface.getMethods()) {
-    os << "    " << method.getReturnType() << " ";
-    emitMethodNameAndArgs(method, os, /*addOperationArg=*/!method.isStatic());
-    os << " final {\n";
-
-    // Provide a definition of the concrete op if this is non static.
-    if (!method.isStatic()) {
-      os << "      auto op = llvm::cast<ConcreteOp>(tablegen_opaque_op);\n"
-         << "      (void)op;\n";
-    }
+    emitCPPType(method.getReturnType(), os << "    ");
+    emitMethodNameAndArgs(method, os, valueType,
+                          /*addThisArg=*/!method.isStatic(), /*addConst=*/true);
+    os << " final {\n      ";
 
     // Check for a provided body to the function.
-    if (auto body = method.getBody()) {
-      os << body << "\n    }\n";
+    if (Optional<StringRef> body = method.getBody()) {
+      if (method.isStatic())
+        os << body->trim();
+      else
+        os << tblgen::tgfmt(body->trim(), &nonStaticMethodFmt);
+      os << "\n    }\n";
       continue;
     }
 
     // Forward to the method on the concrete operation type.
-    os << "      return " << (method.isStatic() ? "ConcreteOp::" : "op.");
+    if (method.isStatic())
+      os << "return " << valueTemplate << "::";
+    else
+      os << tblgen::tgfmt("return $_self.", &nonStaticMethodFmt);
 
     // Add the arguments to the call.
     os << method.getName() << '(';
     llvm::interleaveComma(
         method.getArguments(), os,
-        [&](const OpInterfaceMethod::Argument &arg) { os << arg.name; });
+        [&](const InterfaceMethod::Argument &arg) { os << arg.name; });
     os << ");\n    }\n";
   }
   os << "  };\n";
 }
 
-static void emitTraitDecl(OpInterface &interface, raw_ostream &os,
-                          StringRef interfaceName,
-                          StringRef interfaceTraitsName) {
-  os << "  template <typename ConcreteOp>\n  "
-     << llvm::formatv("struct {0}Trait : public OpInterface<{0},"
-                      " detail::{1}>::Trait<ConcreteOp> {{\n",
-                      interfaceName, interfaceTraitsName);
+void InterfaceGenerator::emitTraitDecl(Interface &interface,
+                                       StringRef interfaceName,
+                                       StringRef interfaceTraitsName) {
+  os << llvm::formatv("  template <typename {3}>\n"
+                      "  struct {0}Trait : public ::mlir::{2}<{0},"
+                      " detail::{1}>::Trait<{3}> {{\n",
+                      interfaceName, interfaceTraitsName, interfaceBaseType,
+                      valueTemplate);
 
   // Insert the default implementation for any methods.
+  bool isOpInterface = isa<OpInterface>(interface);
   for (auto &method : interface.getMethods()) {
     // Flag interface methods named verifyTrait.
     if (method.getName() == "verifyTrait")
       PrintFatalError(
           formatv("'verifyTrait' method cannot be specified as interface "
-                  "method for '{0}'; set 'verify' on OpInterfaceTrait instead",
+                  "method for '{0}'; use the 'verify' field instead",
                   interfaceName));
     auto defaultImpl = method.getDefaultImplementation();
     if (!defaultImpl)
       continue;
 
-    os << "  " << (method.isStatic() ? "static " : "") << method.getReturnType()
-       << " ";
-    emitMethodNameAndArgs(method, os, /*addOperationArg=*/false);
-    os << " {\n" << defaultImpl.getValue() << "  }\n";
+    os << "    " << (method.isStatic() ? "static " : "");
+    emitCPPType(method.getReturnType(), os);
+    emitMethodNameAndArgs(method, os, valueType, /*addThisArg=*/false,
+                          /*addConst=*/!isOpInterface);
+    os << " {\n      " << tblgen::tgfmt(defaultImpl->trim(), &traitMethodFmt)
+       << "\n    }\n";
   }
 
-  tblgen::FmtContext traitCtx;
-  traitCtx.withOp("op");
   if (auto verify = interface.getVerify()) {
-    os << "    static LogicalResult verifyTrait(Operation* op) {\n"
-       << std::string(tblgen::tgfmt(*verify, &traitCtx)) << "\n  }\n";
+    assert(isa<OpInterface>(interface) && "only OpInterface supports 'verify'");
+
+    tblgen::FmtContext verifyCtx;
+    verifyCtx.withOp("op");
+    os << "    static ::mlir::LogicalResult verifyTrait(::mlir::Operation *op) "
+          "{\n      "
+       << tblgen::tgfmt(verify->trim(), &verifyCtx) << "\n    }\n";
   }
   if (auto extraTraitDecls = interface.getExtraTraitClassDeclaration())
-    os << extraTraitDecls << "\n";
+    os << tblgen::tgfmt(*extraTraitDecls, &traitMethodFmt) << "\n";
 
   os << "  };\n";
 
   // Emit a utility wrapper trait class.
-  os << "    template <typename ConcreteOp>\n    "
-     << llvm::formatv("struct Trait : public {0}Trait<ConcreteOp> {{};\n",
-                      interfaceName);
+  os << llvm::formatv("  template <typename {1}>\n"
+                      "  struct Trait : public {0}Trait<{1}> {{};\n",
+                      interfaceName, valueTemplate);
 }
 
-static void emitInterfaceDecl(OpInterface &interface, raw_ostream &os) {
+void InterfaceGenerator::emitInterfaceDecl(Interface interface) {
+  llvm::SmallVector<StringRef, 2> namespaces;
+  llvm::SplitString(interface.getCppNamespace(), namespaces, "::");
+  for (StringRef ns : namespaces)
+    os << "namespace " << ns << " {\n";
+
   StringRef interfaceName = interface.getName();
   auto interfaceTraitsName = (interfaceName + "InterfaceTraits").str();
 
   // Emit the traits struct containing the concept and model declarations.
   os << "namespace detail {\n"
      << "struct " << interfaceTraitsName << " {\n";
-  emitConceptDecl(interface, os);
-  emitModelDecl(interface, os);
+  emitConceptDecl(interface);
+  emitModelDecl(interface);
   os << "};\n} // end namespace detail\n";
 
   // Emit the main interface class declaration.
-  os << llvm::formatv("class {0} : public OpInterface<{1}, detail::{2}> {\n"
+  os << llvm::formatv("class {0} : public ::mlir::{3}<{1}, detail::{2}> {\n"
                       "public:\n"
-                      "  using OpInterface<{1}, detail::{2}>::OpInterface;\n",
-                      interfaceName, interfaceName, interfaceTraitsName);
+                      "  using ::mlir::{3}<{1}, detail::{2}>::{3};\n",
+                      interfaceName, interfaceName, interfaceTraitsName,
+                      interfaceBaseType);
 
   // Emit the derived trait for the interface.
-  emitTraitDecl(interface, os, interfaceName, interfaceTraitsName);
+  emitTraitDecl(interface, interfaceName, interfaceTraitsName);
 
   // Insert the method declarations.
+  bool isOpInterface = isa<OpInterface>(interface);
   for (auto &method : interface.getMethods()) {
-    os << "  " << method.getReturnType() << " ";
-    emitMethodNameAndArgs(method, os, /*addOperationArg=*/false);
+    emitCPPType(method.getReturnType(), os << "  ");
+    emitMethodNameAndArgs(method, os, valueType, /*addThisArg=*/false,
+                          /*addConst=*/!isOpInterface);
     os << ";\n";
   }
 
@@ -217,16 +331,16 @@ static void emitInterfaceDecl(OpInterface &interface, raw_ostream &os) {
     os << *extraDecls << "\n";
 
   os << "};\n";
+
+  for (StringRef ns : llvm::reverse(namespaces))
+    os << "} // namespace " << ns << "\n";
 }
 
-static bool emitInterfaceDecls(const RecordKeeper &recordKeeper,
-                               raw_ostream &os) {
-  llvm::emitSourceFileHeader("Operation Interface Declarations", os);
+bool InterfaceGenerator::emitInterfaceDecls() {
+  llvm::emitSourceFileHeader("Interface Declarations", os);
 
-  for (const auto *def : getAllOpInterfaceDefinitions(recordKeeper)) {
-    OpInterface interface(def);
-    emitInterfaceDecl(interface, os);
-  }
+  for (const auto *def : defs)
+    emitInterfaceDecl(Interface(def));
   return false;
 }
 
@@ -234,17 +348,9 @@ static bool emitInterfaceDecls(const RecordKeeper &recordKeeper,
 // GEN: Interface documentation
 //===----------------------------------------------------------------------===//
 
-/// Emit a string corresponding to a C++ type, followed by a space if necessary.
-static raw_ostream &emitCPPType(StringRef type, raw_ostream &os) {
-  type = type.trim();
-  os << type;
-  if (type.back() != '&' && type.back() != '*')
-    os << " ";
-  return os;
-}
-
-static void emitInterfaceDoc(const Record &interfaceDef, raw_ostream &os) {
-  OpInterface interface(&interfaceDef);
+static void emitInterfaceDoc(const llvm::Record &interfaceDef,
+                             raw_ostream &os) {
+  Interface interface(&interfaceDef);
 
   // Emit the interface name followed by the description.
   os << "## " << interface.getName() << " (" << interfaceDef.getName() << ")";
@@ -262,7 +368,7 @@ static void emitInterfaceDoc(const Record &interfaceDef, raw_ostream &os) {
       os << "static ";
     emitCPPType(method.getReturnType(), os) << method.getName() << '(';
     llvm::interleaveComma(method.getArguments(), os,
-                          [&](const OpInterfaceMethod::Argument &arg) {
+                          [&](const InterfaceMethod::Argument &arg) {
                             emitCPPType(arg.type, os) << arg.name;
                           });
     os << ");\n```\n";
@@ -271,19 +377,17 @@ static void emitInterfaceDoc(const Record &interfaceDef, raw_ostream &os) {
     if (auto description = method.getDescription())
       mlir::tblgen::emitDescription(*description, os);
 
-    // If the body is not provided, this method must be provided by the
-    // operation.
+    // If the body is not provided, this method must be provided by the user.
     if (!method.getBody())
-      os << "\nNOTE: This method *must* be implemented by the operation.\n\n";
+      os << "\nNOTE: This method *must* be implemented by the user.\n\n";
   }
 }
 
-static bool emitInterfaceDocs(const RecordKeeper &recordKeeper,
-                              raw_ostream &os) {
+bool InterfaceGenerator::emitInterfaceDocs() {
   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
-  os << "# Operation Interface definition\n";
+  os << "# " << interfaceBaseType << " definitions\n";
 
-  for (const auto *def : getAllOpInterfaceDefinitions(recordKeeper))
+  for (const auto *def : defs)
     emitInterfaceDoc(*def, os);
   return false;
 }
@@ -292,26 +396,30 @@ static bool emitInterfaceDocs(const RecordKeeper &recordKeeper,
 // GEN: Interface registration hooks
 //===----------------------------------------------------------------------===//
 
-// Registers the operation interface generator to mlir-tblgen.
-static mlir::GenRegistration
-    genInterfaceDecls("gen-op-interface-decls",
-                      "Generate op interface declarations",
-                      [](const RecordKeeper &records, raw_ostream &os) {
-                        return emitInterfaceDecls(records, os);
-                      });
-
-// Registers the operation interface generator to mlir-tblgen.
-static mlir::GenRegistration
-    genInterfaceDefs("gen-op-interface-defs",
-                     "Generate op interface definitions",
-                     [](const RecordKeeper &records, raw_ostream &os) {
-                       return emitInterfaceDefs(records, os);
-                     });
-
-// Registers the operation interface document generator to mlir-tblgen.
-static mlir::GenRegistration
-    genInterfaceDocs("gen-op-interface-doc",
-                     "Generate op interface documentation",
-                     [](const RecordKeeper &records, raw_ostream &os) {
-                       return emitInterfaceDocs(records, os);
-                     });
+namespace {
+template <typename GeneratorT> struct InterfaceGenRegistration {
+  InterfaceGenRegistration(StringRef genArg)
+      : genDeclArg(("gen-" + genArg + "-interface-decls").str()),
+        genDefArg(("gen-" + genArg + "-interface-defs").str()),
+        genDocArg(("gen-" + genArg + "-interface-docs").str()),
+        genDecls(genDeclArg, "Generate interface declarations",
+                 [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                   return GeneratorT(records, os).emitInterfaceDecls();
+                 }),
+        genDefs(genDefArg, "Generate interface definitions",
+                [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                  return GeneratorT(records, os).emitInterfaceDefs();
+                }),
+        genDocs(genDocArg, "Generate interface documentation",
+                [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                  return GeneratorT(records, os).emitInterfaceDocs();
+                }) {}
+
+  std::string genDeclArg, genDefArg, genDocArg;
+  mlir::GenRegistration genDecls, genDefs, genDocs;
+};
+} // end anonymous namespace
+
+static InterfaceGenRegistration<AttrInterfaceGenerator> attrGen("attr");
+static InterfaceGenRegistration<OpInterfaceGenerator> opGen("op");
+static InterfaceGenRegistration<TypeInterfaceGenerator> typeGen("type");
diff --git a/mlir/tools/mlir-tblgen/PassGen.cpp b/mlir/tools/mlir-tblgen/PassGen.cpp
index 2ca8671a2f785..f8998f09a4368 100644
--- a/mlir/tools/mlir-tblgen/PassGen.cpp
+++ b/mlir/tools/mlir-tblgen/PassGen.cpp
@@ -42,10 +42,10 @@ class {0}Base : public {1} {
   {0}Base(const {0}Base &) : {1}(::mlir::TypeID::get<DerivedT>()) {{}
 
   /// Returns the command-line argument attached to this pass.
-  llvm::StringRef getArgument() const override { return "{2}"; }
+  ::llvm::StringRef getArgument() const override { return "{2}"; }
 
   /// Returns the derived pass name.
-  llvm::StringRef getName() const override { return "{0}"; }
+  ::llvm::StringRef getName() const override { return "{0}"; }
 
   /// Support isa/dyn_cast functionality for the derived pass class.
   static bool classof(const ::mlir::Pass *pass) {{
@@ -53,7 +53,7 @@ class {0}Base : public {1} {
   }
 
   /// A clone method to create a copy of this pass.
-  std::unique_ptr<Pass> clonePass() const override {{
+  std::unique_ptr<::mlir::Pass> clonePass() const override {{
     return std::make_unique<DerivedT>(*static_cast<const DerivedT *>(this));
   }
 
@@ -63,13 +63,14 @@ class {0}Base : public {1} {
 /// Emit the declarations for each of the pass options.
 static void emitPassOptionDecls(const Pass &pass, raw_ostream &os) {
   for (const PassOption &opt : pass.getOptions()) {
-    os.indent(2) << "Pass::" << (opt.isListOption() ? "ListOption" : "Option");
+    os.indent(2) << "::mlir::Pass::"
+                 << (opt.isListOption() ? "ListOption" : "Option");
 
-    os << llvm::formatv("<{0}> {1}{{*this, \"{2}\", llvm::cl::desc(\"{3}\")",
+    os << llvm::formatv("<{0}> {1}{{*this, \"{2}\", ::llvm::cl::desc(\"{3}\")",
                         opt.getType(), opt.getCppVariableName(),
                         opt.getArgument(), opt.getDescription());
     if (Optional<StringRef> defaultVal = opt.getDefaultValue())
-      os << ", llvm::cl::init(" << defaultVal << ")";
+      os << ", ::llvm::cl::init(" << defaultVal << ")";
     if (Optional<StringRef> additionalFlags = opt.getAdditionalFlags())
       os << ", " << *additionalFlags;
     os << "};\n";
@@ -79,9 +80,9 @@ static void emitPassOptionDecls(const Pass &pass, raw_ostream &os) {
 /// Emit the declarations for each of the pass statistics.
 static void emitPassStatisticDecls(const Pass &pass, raw_ostream &os) {
   for (const PassStatistic &stat : pass.getStatistics()) {
-    os << llvm::formatv("  Pass::Statistic {0}{{this, \"{1}\", \"{2}\"};\n",
-                        stat.getCppVariableName(), stat.getName(),
-                        stat.getDescription());
+    os << llvm::formatv(
+        "  ::mlir::Pass::Statistic {0}{{this, \"{1}\", \"{2}\"};\n",
+        stat.getCppVariableName(), stat.getName(), stat.getDescription());
   }
 }
 
@@ -122,7 +123,7 @@ static void emitRegistration(ArrayRef<Pass> passes, raw_ostream &os) {
     os << llvm::formatv("#ifdef GEN_PASS_REGISTRATION_{0}\n",
                         pass.getDef()->getName());
     os << llvm::formatv("::mlir::registerPass(\"{0}\", \"{1}\", []() -> "
-                        "std::unique_ptr<Pass> {{ return {2}; });\n",
+                        "std::unique_ptr<::mlir::Pass> {{ return {2}; });\n",
                         pass.getArgument(), pass.getSummary(),
                         pass.getConstructor());
     os << llvm::formatv("#endif // GEN_PASS_REGISTRATION_{0}\n",
diff --git a/mlir/tools/mlir-tblgen/RewriterGen.cpp b/mlir/tools/mlir-tblgen/RewriterGen.cpp
index 62638e6c99641..fc67ec4b8c174 100644
--- a/mlir/tools/mlir-tblgen/RewriterGen.cpp
+++ b/mlir/tools/mlir-tblgen/RewriterGen.cpp
@@ -39,7 +39,8 @@ using llvm::RecordKeeper;
 #define DEBUG_TYPE "mlir-tblgen-rewritergen"
 
 namespace llvm {
-template <> struct format_provider<mlir::tblgen::Pattern::IdentifierLine> {
+template <>
+struct format_provider<mlir::tblgen::Pattern::IdentifierLine> {
   static void format(const mlir::tblgen::Pattern::IdentifierLine &v,
                      raw_ostream &os, StringRef style) {
     os << v.first << ":" << v.second;
@@ -203,7 +204,7 @@ std::string PatternEmitter::handleConstantAttr(Attribute attr,
     PrintFatalError(loc, "Attribute " + attr.getAttrDefName() +
                              " does not have the 'constBuilderCall' field");
 
-  // TODO(jpienaar): Verify the constants here
+  // TODO: Verify the constants here
   return std::string(tgfmt(attr.getConstBuilderTemplate(), &fmtCtx, value));
 }
 
@@ -342,7 +343,7 @@ void PatternEmitter::emitAttributeMatch(DagNode tree, int argIndex, int depth,
       "(void)tblgen_attr;\n",
       depth, attr.getStorageType(), namedAttr->name);
 
-  // TODO(antiagainst): This should use getter method to avoid duplication.
+  // TODO: This should use getter method to avoid duplication.
   if (attr.hasDefaultValue()) {
     os.indent(indent) << "if (!tblgen_attr) tblgen_attr = "
                       << std::string(tgfmt(attr.getConstBuilderTemplate(),
@@ -428,7 +429,7 @@ void PatternEmitter::emitMatchLogic(DagNode tree) {
       PrintFatalError(
           loc, "cannot use AttrConstraint in Pattern multi-entity constraints");
     } else {
-      // TODO(b/138794486): replace formatv arguments with the exact specified
+      // TODO: replace formatv arguments with the exact specified
       // args.
       if (entities.size() > 4) {
         PrintFatalError(loc, "only support up to 4-entity constraints now");
@@ -489,9 +490,9 @@ void PatternEmitter::emit(StringRef rewriteName) {
   auto locs = pattern.getLocation();
   os << formatv("/* Generated from:\n\t{0:$[ instantiating\n\t]}\n*/\n",
                 make_range(locs.rbegin(), locs.rend()));
-  os << formatv(R"(struct {0} : public RewritePattern {
-  {0}(MLIRContext *context)
-      : RewritePattern("{1}", {{)",
+  os << formatv(R"(struct {0} : public ::mlir::RewritePattern {
+  {0}(::mlir::MLIRContext *context)
+      : ::mlir::RewritePattern("{1}", {{)",
                 rewriteName, rootName);
   // Sort result operators by name.
   llvm::SmallVector<const Operator *, 4> sortedResultOps(resultOps.begin(),
@@ -506,8 +507,9 @@ void PatternEmitter::emit(StringRef rewriteName) {
 
   // Emit matchAndRewrite() function.
   os << R"(
-  LogicalResult matchAndRewrite(Operation *op0,
-                                     PatternRewriter &rewriter) const override {
+  ::mlir::LogicalResult
+  matchAndRewrite(::mlir::Operation *op0,
+                  ::mlir::PatternRewriter &rewriter) const override {
 )";
 
   // Register all symbols bound in the source pattern.
@@ -524,9 +526,9 @@ void PatternEmitter::emit(StringRef rewriteName) {
     auto &info = symbolInfoPair.getValue();
     os.indent(4) << info.getVarDecl(symbol);
   }
-  // TODO(jpienaar): capture ops with consistent numbering so that it can be
+  // TODO: capture ops with consistent numbering so that it can be
   // reused for fused loc.
-  os.indent(4) << formatv("Operation *tblgen_ops[{0}];\n\n",
+  os.indent(4) << formatv("::mlir::Operation *tblgen_ops[{0}];\n\n",
                           pattern.getSourcePattern().getNumOps());
   LLVM_DEBUG(
       llvm::dbgs() << "done creating local variables for capturing matches\n");
@@ -606,7 +608,8 @@ void PatternEmitter::emitRewriteLogic() {
     os.indent(4) << "rewriter.eraseOp(op0);\n";
   } else {
     // Process replacement result patterns.
-    os.indent(4) << "SmallVector<Value, 4> tblgen_repl_values;\n";
+    os.indent(4)
+        << "::llvm::SmallVector<::mlir::Value, 4> tblgen_repl_values;\n";
     for (int i = replStartIndex; i < numResultPatterns; ++i) {
       DagNode resultTree = pattern.getResultPattern(i);
       auto val = handleResultPattern(resultTree, offsets[i], 0);
@@ -616,10 +619,10 @@ void PatternEmitter::emitRewriteLogic() {
       // `{0}` resolves to an `Operation::result_range` as well as cases that
       // are not iterable (e.g. vector that gets wrapped in additional braces by
       // RewriterGen).
-      // TODO(b/147096809): Revisit the need for materializing a vector.
+      // TODO: Revisit the need for materializing a vector.
       os << symbolInfoMap.getAllRangeUse(
           val,
-          "    for (auto v : SmallVector<Value, 4>{ {0} }) {{ "
+          "    for (auto v : ::llvm::SmallVector<::mlir::Value, 4>{ {0} }) {{ "
           "tblgen_repl_values.push_back(v); }",
           "\n");
     }
@@ -697,7 +700,7 @@ std::string PatternEmitter::handleLocationDirective(DagNode tree) {
   if (tree.getNumArgs() == 1) {
     DagLeaf leaf = tree.getArgAsLeaf(0);
     if (leaf.isStringAttr())
-      return formatv("mlir::NameLoc::get(rewriter.getIdentifier(\"{0}\"), "
+      return formatv("::mlir::NameLoc::get(rewriter.getIdentifier(\"{0}\"), "
                      "rewriter.getContext())",
                      leaf.getStringAttr())
           .str();
@@ -768,7 +771,7 @@ std::string PatternEmitter::handleReplaceWithNativeCodeCall(DagNode tree) {
   LLVM_DEBUG(llvm::dbgs() << '\n');
 
   auto fmt = tree.getNativeCodeTemplate();
-  // TODO(b/138794486): replace formatv arguments with the exact specified args.
+  // TODO: replace formatv arguments with the exact specified args.
   SmallVector<std::string, 8> attrs(8);
   if (tree.getNumArgs() > 8) {
     PrintFatalError(loc, "unsupported NativeCodeCall argument numbers: " +
@@ -794,7 +797,7 @@ int PatternEmitter::getNodeValueCount(DagNode node) {
     // Otherwise this is an unbound op; we will use all its results.
     return pattern.getDialectOp(node).getNumResults();
   }
-  // TODO(antiagainst): This considers all NativeCodeCall as returning one
+  // TODO: This considers all NativeCodeCall as returning one
   // value. Enhance if multi-value ones are needed.
   return 1;
 }
@@ -918,8 +921,8 @@ std::string PatternEmitter::handleOpCreation(DagNode tree, int resultIndex,
 
   // Then prepare the result types. We need to specify the types for all
   // results.
-  os.indent(6) << formatv(
-      "SmallVector<Type, 4> tblgen_types; (void)tblgen_types;\n");
+  os.indent(6) << formatv("::mlir::SmallVector<::mlir::Type, 4> tblgen_types; "
+                          "(void)tblgen_types;\n");
   int numResults = resultOp.getNumResults();
   if (numResults != 0) {
     for (int i = 0; i < numResults; ++i)
@@ -956,7 +959,8 @@ void PatternEmitter::createSeparateLocalVarsForOpArgs(
     std::string varName;
     if (operand->isVariadic()) {
       varName = std::string(formatv("tblgen_values_{0}", valueIndex++));
-      os.indent(6) << formatv("SmallVector<Value, 4> {0};\n", varName);
+      os.indent(6) << formatv("::mlir::SmallVector<::mlir::Value, 4> {0};\n",
+                              varName);
       std::string range;
       if (node.isNestedDagArg(argIndex)) {
         range = childNodeNames[argIndex];
@@ -970,7 +974,7 @@ void PatternEmitter::createSeparateLocalVarsForOpArgs(
                               varName);
     } else {
       varName = std::string(formatv("tblgen_value_{0}", valueIndex++));
-      os.indent(6) << formatv("Value {0} = ", varName);
+      os.indent(6) << formatv("::mlir::Value {0} = ", varName);
       if (node.isNestedDagArg(argIndex)) {
         os << symbolInfoMap.getValueAndRangeUse(childNodeNames[argIndex]);
       } else {
@@ -1022,7 +1026,7 @@ void PatternEmitter::supplyValuesForOpArgs(
       // The argument in the result DAG pattern.
       auto patArgName = node.getArgName(argIndex);
       if (leaf.isConstantAttr() || leaf.isEnumAttrCase()) {
-        // TODO(jpienaar): Refactor out into map to avoid recomputing these.
+        // TODO: Refactor out into map to avoid recomputing these.
         if (!opArg.is<NamedAttribute *>())
           PrintFatalError(loc, Twine("expected attribute ") + Twine(argIndex));
         if (!patArgName.empty())
@@ -1039,16 +1043,16 @@ void PatternEmitter::createAggregateLocalVarsForOpArgs(
     DagNode node, const ChildNodeIndexNameMap &childNodeNames) {
   Operator &resultOp = node.getDialectOp(opMap);
 
-  os.indent(6) << formatv(
-      "SmallVector<Value, 4> tblgen_values; (void)tblgen_values;\n");
-  os.indent(6) << formatv(
-      "SmallVector<NamedAttribute, 4> tblgen_attrs; (void)tblgen_attrs;\n");
+  os.indent(6) << formatv("::mlir::SmallVector<::mlir::Value, 4> "
+                          "tblgen_values; (void)tblgen_values;\n");
+  os.indent(6) << formatv("::mlir::SmallVector<::mlir::NamedAttribute, 4> "
+                          "tblgen_attrs; (void)tblgen_attrs;\n");
 
+  const char *addAttrCmd =
+      "if (auto tmpAttr = {1}) "
+      "tblgen_attrs.emplace_back(rewriter.getIdentifier(\"{0}\"), tmpAttr);\n";
   for (int argIndex = 0, e = resultOp.getNumArgs(); argIndex < e; ++argIndex) {
     if (resultOp.getArg(argIndex).is<NamedAttribute *>()) {
-      const char *addAttrCmd = "if ({1}) {{"
-                               "  tblgen_attrs.emplace_back(rewriter."
-                               "getIdentifier(\"{0}\"), {1}); }\n";
       // The argument in the op definition.
       auto opArgName = resultOp.getArgName(argIndex);
       if (auto subTree = node.getArgAsNestedDag(argIndex)) {
diff --git a/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp b/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp
index e860dbc3fb585..21f7349763db0 100644
--- a/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp
+++ b/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp
@@ -205,7 +205,8 @@ static void emitConceptDecl(const Availability &availability, raw_ostream &os) {
      << "  public:\n"
      << "    virtual ~Concept() = default;\n"
      << "    virtual " << availability.getQueryFnRetType() << " "
-     << availability.getQueryFnName() << "(Operation *tblgen_opaque_op) = 0;\n"
+     << availability.getQueryFnName()
+     << "(Operation *tblgen_opaque_op) const = 0;\n"
      << "  };\n";
 }
 
@@ -215,7 +216,7 @@ static void emitModelDecl(const Availability &availability, raw_ostream &os) {
      << "  public:\n"
      << "    " << availability.getQueryFnRetType() << " "
      << availability.getQueryFnName()
-     << "(Operation *tblgen_opaque_op) final {\n"
+     << "(Operation *tblgen_opaque_op) const final {\n"
      << "      auto op = llvm::cast<ConcreteOp>(tblgen_opaque_op);\n"
      << "      (void)op;\n"
      // Forward to the method on the concrete operation type.
@@ -1282,7 +1283,7 @@ static void emitAvailabilityImpl(const Operator &srcOp, raw_ostream &os) {
       os << formatv("    auto tblgen_instance = {0}::{1}(tblgen_attrVal);\n",
                     enumAttr->getCppNamespace(), avail.getQueryFnName());
       os << "    if (tblgen_instance) "
-         // TODO(antiagainst): use `avail.getMergeCode()` here once ODS supports
+         // TODO` here once ODS supports
          // dialect-specific contents so that we can use not implementing the
          // availability interface as indication of no requirements.
          << std::string(tgfmt(caseSpecs.front().second.getMergeActionCode(),
diff --git a/mlir/tools/mlir-tblgen/StructsGen.cpp b/mlir/tools/mlir-tblgen/StructsGen.cpp
index 005757a62f376..cccacc0cad853 100644
--- a/mlir/tools/mlir-tblgen/StructsGen.cpp
+++ b/mlir/tools/mlir-tblgen/StructsGen.cpp
@@ -36,11 +36,11 @@ emitStructClass(const Record &structDef, StringRef structName,
                 StringRef description, raw_ostream &os) {
   const char *structInfo = R"(
 // {0}
-class {1} : public mlir::DictionaryAttr)";
+class {1} : public ::mlir::DictionaryAttr)";
   const char *structInfoEnd = R"( {
 public:
-  using DictionaryAttr::DictionaryAttr;
-  static bool classof(mlir::Attribute attr);
+  using ::mlir::DictionaryAttr::DictionaryAttr;
+  static bool classof(::mlir::Attribute attr);
 )";
   os << formatv(structInfo, description, structName) << structInfoEnd;
 
@@ -48,7 +48,7 @@ class {1} : public mlir::DictionaryAttr)";
   //   TblgenStruct::get(MLIRContext context, Type1 Field1, Type2 Field2, ...);
   const char *getInfoDecl = "  static {0} get(\n";
   const char *getInfoDeclArg = "      {0} {1},\n";
-  const char *getInfoDeclEnd = "      mlir::MLIRContext* context);\n\n";
+  const char *getInfoDeclEnd = "      ::mlir::MLIRContext* context);\n\n";
 
   os << llvm::formatv(getInfoDecl, structName);
 
@@ -112,7 +112,7 @@ static void emitFactoryDef(llvm::StringRef structName,
                            raw_ostream &os) {
   const char *getInfoDecl = "{0} {0}::get(\n";
   const char *getInfoDeclArg = "    {0} {1},\n";
-  const char *getInfoDeclEnd = "    mlir::MLIRContext* context) {";
+  const char *getInfoDeclEnd = "    ::mlir::MLIRContext* context) {";
 
   os << llvm::formatv(getInfoDecl, structName);
 
@@ -125,19 +125,19 @@ static void emitFactoryDef(llvm::StringRef structName,
   os << getInfoDeclEnd;
 
   const char *fieldStart = R"(
-  llvm::SmallVector<mlir::NamedAttribute, {0}> fields;
+  ::llvm::SmallVector<::mlir::NamedAttribute, {0}> fields;
 )";
   os << llvm::formatv(fieldStart, fields.size());
 
   const char *getFieldInfo = R"(
   assert({0});
-  auto {0}_id = mlir::Identifier::get("{0}", context);
+  auto {0}_id = ::mlir::Identifier::get("{0}", context);
   fields.emplace_back({0}_id, {0});
 )";
 
   const char *getFieldInfoOptional = R"(
   if ({0}) {
-    auto {0}_id = mlir::Identifier::get("{0}", context);
+    auto {0}_id = ::mlir::Identifier::get("{0}", context);
     fields.emplace_back({0}_id, {0});
   }
 )";
@@ -150,7 +150,7 @@ static void emitFactoryDef(llvm::StringRef structName,
   }
 
   const char *getEndInfo = R"(
-  Attribute dict = mlir::DictionaryAttr::get(fields, context);
+  ::mlir::Attribute dict = ::mlir::DictionaryAttr::get(fields, context);
   return dict.dyn_cast<{0}>();
 }
 )";
@@ -161,12 +161,12 @@ static void emitClassofDef(llvm::StringRef structName,
                            llvm::ArrayRef<mlir::tblgen::StructFieldAttr> fields,
                            raw_ostream &os) {
   const char *classofInfo = R"(
-bool {0}::classof(mlir::Attribute attr))";
+bool {0}::classof(::mlir::Attribute attr))";
 
   const char *classofInfoHeader = R"(
   if (!attr)
     return false;
-  auto derived = attr.dyn_cast<mlir::DictionaryAttr>();
+  auto derived = attr.dyn_cast<::mlir::DictionaryAttr>();
   if (!derived)
     return false;
   int empty_optionals = 0;
@@ -212,7 +212,7 @@ emitAccessorDef(llvm::StringRef structName,
                 raw_ostream &os) {
   const char *fieldInfo = R"(
 {0} {2}::{1}() const {
-  auto derived = this->cast<mlir::DictionaryAttr>();
+  auto derived = this->cast<::mlir::DictionaryAttr>();
   auto {1} = derived.get("{1}");
   assert({1} && "attribute not found.");
   assert({1}.isa<{0}>() && "incorrect Attribute type found.");
@@ -221,7 +221,7 @@ emitAccessorDef(llvm::StringRef structName,
 )";
   const char *fieldInfoOptional = R"(
 {0} {2}::{1}() const {
-  auto derived = this->cast<mlir::DictionaryAttr>();
+  auto derived = this->cast<::mlir::DictionaryAttr>();
   auto {1} = derived.get("{1}");
   if (!{1})
     return nullptr;
diff --git a/mlir/tools/mlir-vulkan-runner/VulkanRuntime.cpp b/mlir/tools/mlir-vulkan-runner/VulkanRuntime.cpp
index 2c42c19badef2..42f8be45135b1 100644
--- a/mlir/tools/mlir-vulkan-runner/VulkanRuntime.cpp
+++ b/mlir/tools/mlir-vulkan-runner/VulkanRuntime.cpp
@@ -15,7 +15,7 @@
 
 #include <chrono>
 #include <cstring>
-// TODO(antiagainst): It's generally bad to access stdout/stderr in a library.
+// TODO: It's generally bad to access stdout/stderr in a library.
 // Figure out a better way for error reporting.
 #include <iomanip>
 #include <iostream>
@@ -148,7 +148,9 @@ LogicalResult VulkanRuntime::destroy() {
     // For each descriptor binding.
     for (auto &memoryBuffer : deviceMemoryBuffers) {
       vkFreeMemory(device, memoryBuffer.deviceMemory, nullptr);
-      vkDestroyBuffer(device, memoryBuffer.buffer, nullptr);
+      vkFreeMemory(device, memoryBuffer.hostMemory, nullptr);
+      vkDestroyBuffer(device, memoryBuffer.hostBuffer, nullptr);
+      vkDestroyBuffer(device, memoryBuffer.deviceBuffer, nullptr);
     }
   }
 
@@ -181,6 +183,9 @@ LogicalResult VulkanRuntime::run() {
   // Get working queue.
   vkGetDeviceQueue(device, queueFamilyIndex, 0, &queue);
 
+  if (failed(copyResource(/*deviceToHost=*/false)))
+    return failure();
+
   auto submitStart = std::chrono::high_resolution_clock::now();
   // Submit command buffer into the queue.
   if (failed(submitCommandBuffersToQueue()))
@@ -256,7 +261,7 @@ LogicalResult VulkanRuntime::createDevice() {
   RETURN_ON_VULKAN_ERROR(physicalDeviceCount ? VK_SUCCESS : VK_INCOMPLETE,
                          "physicalDeviceCount");
 
-  // TODO(denis0x0D): find the best device.
+  // TODO: find the best device.
   physicalDevice = physicalDevices.front();
   if (failed(getBestComputeQueue()))
     return failure();
@@ -304,13 +309,28 @@ LogicalResult VulkanRuntime::createDevice() {
          properties.memoryTypes[i].propertyFlags) &&
         (memorySize <=
          properties.memoryHeaps[properties.memoryTypes[i].heapIndex].size)) {
-      memoryTypeIndex = i;
+      hostMemoryTypeIndex = i;
+      break;
+    }
+  }
+
+  // Find memory type memory type with VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT to be
+  // used on the device. This will allow better performance access for GPU with
+  // on device memory.
+  for (uint32_t i = 0, e = properties.memoryTypeCount; i < e; ++i) {
+    if ((VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT &
+         properties.memoryTypes[i].propertyFlags) &&
+        (memorySize <=
+         properties.memoryHeaps[properties.memoryTypes[i].heapIndex].size)) {
+      deviceMemoryTypeIndex = i;
       break;
     }
   }
 
-  RETURN_ON_VULKAN_ERROR(memoryTypeIndex == VK_MAX_MEMORY_TYPES ? VK_INCOMPLETE
-                                                                : VK_SUCCESS,
+  RETURN_ON_VULKAN_ERROR((hostMemoryTypeIndex == VK_MAX_MEMORY_TYPES ||
+                          deviceMemoryTypeIndex == VK_MAX_MEMORY_TYPES)
+                             ? VK_INCOMPLETE
+                             : VK_SUCCESS,
                          "invalid memoryTypeIndex");
   return success();
 }
@@ -401,27 +421,31 @@ LogicalResult VulkanRuntime::createMemoryBuffers() {
       // Set descriptor type for the specific device memory buffer.
       memoryBuffer.descriptorType = descriptorType;
       const auto bufferSize = resourceDataBindingPair.second.size;
-
+      memoryBuffer.bufferSize = bufferSize;
       // Specify memory allocation info.
       VkMemoryAllocateInfo memoryAllocateInfo = {};
       memoryAllocateInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
       memoryAllocateInfo.pNext = nullptr;
       memoryAllocateInfo.allocationSize = bufferSize;
-      memoryAllocateInfo.memoryTypeIndex = memoryTypeIndex;
+      memoryAllocateInfo.memoryTypeIndex = hostMemoryTypeIndex;
 
       // Allocate device memory.
+      RETURN_ON_VULKAN_ERROR(vkAllocateMemory(device, &memoryAllocateInfo, 0,
+                                              &memoryBuffer.hostMemory),
+                             "vkAllocateMemory");
+      memoryAllocateInfo.memoryTypeIndex = deviceMemoryTypeIndex;
       RETURN_ON_VULKAN_ERROR(vkAllocateMemory(device, &memoryAllocateInfo, 0,
                                               &memoryBuffer.deviceMemory),
                              "vkAllocateMemory");
       void *payload;
-      RETURN_ON_VULKAN_ERROR(vkMapMemory(device, memoryBuffer.deviceMemory, 0,
+      RETURN_ON_VULKAN_ERROR(vkMapMemory(device, memoryBuffer.hostMemory, 0,
                                          bufferSize, 0,
                                          reinterpret_cast<void **>(&payload)),
                              "vkMapMemory");
 
       // Copy host memory into the mapped area.
       std::memcpy(payload, resourceDataBindingPair.second.ptr, bufferSize);
-      vkUnmapMemory(device, memoryBuffer.deviceMemory);
+      vkUnmapMemory(device, memoryBuffer.hostMemory);
 
       VkBufferCreateInfo bufferCreateInfo = {};
       bufferCreateInfo.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO;
@@ -432,17 +456,24 @@ LogicalResult VulkanRuntime::createMemoryBuffers() {
       bufferCreateInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
       bufferCreateInfo.queueFamilyIndexCount = 1;
       bufferCreateInfo.pQueueFamilyIndices = &queueFamilyIndex;
-      RETURN_ON_VULKAN_ERROR(
-          vkCreateBuffer(device, &bufferCreateInfo, 0, &memoryBuffer.buffer),
-          "vkCreateBuffer");
+      RETURN_ON_VULKAN_ERROR(vkCreateBuffer(device, &bufferCreateInfo, 0,
+                                            &memoryBuffer.hostBuffer),
+                             "vkCreateBuffer");
+      RETURN_ON_VULKAN_ERROR(vkCreateBuffer(device, &bufferCreateInfo, 0,
+                                            &memoryBuffer.deviceBuffer),
+                             "vkCreateBuffer");
 
       // Bind buffer and device memory.
-      RETURN_ON_VULKAN_ERROR(vkBindBufferMemory(device, memoryBuffer.buffer,
+      RETURN_ON_VULKAN_ERROR(vkBindBufferMemory(device, memoryBuffer.hostBuffer,
+                                                memoryBuffer.hostMemory, 0),
+                             "vkBindBufferMemory");
+      RETURN_ON_VULKAN_ERROR(vkBindBufferMemory(device,
+                                                memoryBuffer.deviceBuffer,
                                                 memoryBuffer.deviceMemory, 0),
                              "vkBindBufferMemory");
 
       // Update buffer info.
-      memoryBuffer.bufferInfo.buffer = memoryBuffer.buffer;
+      memoryBuffer.bufferInfo.buffer = memoryBuffer.deviceBuffer;
       memoryBuffer.bufferInfo.offset = 0;
       memoryBuffer.bufferInfo.range = VK_WHOLE_SIZE;
       deviceMemoryBuffers.push_back(memoryBuffer);
@@ -454,6 +485,66 @@ LogicalResult VulkanRuntime::createMemoryBuffers() {
   return success();
 }
 
+LogicalResult VulkanRuntime::copyResource(bool deviceToHost) {
+  VkCommandBufferAllocateInfo commandBufferAllocateInfo = {
+      VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
+      NULL,
+      commandPool,
+      VK_COMMAND_BUFFER_LEVEL_PRIMARY,
+      1,
+  };
+  VkCommandBuffer commandBuffer;
+  RETURN_ON_VULKAN_ERROR(vkAllocateCommandBuffers(device,
+                                                  &commandBufferAllocateInfo,
+                                                  &commandBuffer),
+                         "vkAllocateCommandBuffers");
+
+  VkCommandBufferBeginInfo commandBufferBeginInfo = {
+      VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
+      NULL,
+      0,
+      NULL,
+  };
+  RETURN_ON_VULKAN_ERROR(
+      vkBeginCommandBuffer(commandBuffer, &commandBufferBeginInfo),
+      "vkBeginCommandBuffer");
+
+  for (const auto &deviceMemoryBufferMapPair : deviceMemoryBufferMap) {
+    std::vector<VkDescriptorSetLayoutBinding> descriptorSetLayoutBindings;
+    const auto &deviceMemoryBuffers = deviceMemoryBufferMapPair.second;
+    for (const auto &memBuffer : deviceMemoryBuffers) {
+      VkBufferCopy copy = {0, 0, memBuffer.bufferSize};
+      if (deviceToHost)
+        vkCmdCopyBuffer(commandBuffer, memBuffer.deviceBuffer,
+                        memBuffer.hostBuffer, 1, &copy);
+      else
+        vkCmdCopyBuffer(commandBuffer, memBuffer.hostBuffer,
+                        memBuffer.deviceBuffer, 1, &copy);
+    }
+  }
+
+  RETURN_ON_VULKAN_ERROR(vkEndCommandBuffer(commandBuffer),
+                         "vkEndCommandBuffer");
+  VkSubmitInfo submitInfo = {
+      VK_STRUCTURE_TYPE_SUBMIT_INFO,
+      NULL,
+      0,
+      NULL,
+      NULL,
+      1,
+      &commandBuffer,
+      0,
+      NULL,
+  };
+  submitInfo.pCommandBuffers = &commandBuffer;
+  RETURN_ON_VULKAN_ERROR(vkQueueSubmit(queue, 1, &submitInfo, VK_NULL_HANDLE),
+                         "vkQueueSubmit");
+  RETURN_ON_VULKAN_ERROR(vkQueueWaitIdle(queue), "vkQueueWaitIdle");
+
+  vkFreeCommandBuffers(device, commandPool, 1, &commandBuffer);
+  return success();
+}
+
 LogicalResult VulkanRuntime::createShaderModule() {
   VkShaderModuleCreateInfo shaderModuleCreateInfo = {};
   shaderModuleCreateInfo.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO;
@@ -762,6 +853,9 @@ LogicalResult VulkanRuntime::submitCommandBuffersToQueue() {
 }
 
 LogicalResult VulkanRuntime::updateHostMemoryBuffers() {
+  // First copy back the data to the staging buffer.
+  copyResource(/*deviceToHost=*/true);
+
   // For each descriptor set.
   for (auto &resourceDataMapPair : resourceData) {
     auto &resourceDataMap = resourceDataMapPair.second;
@@ -774,12 +868,12 @@ LogicalResult VulkanRuntime::updateHostMemoryBuffers() {
         auto &hostMemoryBuffer =
             resourceDataMap[deviceMemoryBuffer.bindingIndex];
         RETURN_ON_VULKAN_ERROR(vkMapMemory(device,
-                                           deviceMemoryBuffer.deviceMemory, 0,
+                                           deviceMemoryBuffer.hostMemory, 0,
                                            hostMemoryBuffer.size, 0,
                                            reinterpret_cast<void **>(&payload)),
                                "vkMapMemory");
         std::memcpy(hostMemoryBuffer.ptr, payload, hostMemoryBuffer.size);
-        vkUnmapMemory(device, deviceMemoryBuffer.deviceMemory);
+        vkUnmapMemory(device, deviceMemoryBuffer.hostMemory);
       }
     }
   }
diff --git a/mlir/tools/mlir-vulkan-runner/VulkanRuntime.h b/mlir/tools/mlir-vulkan-runner/VulkanRuntime.h
index 7efc3d6359a08..9fa52b00a0ac9 100644
--- a/mlir/tools/mlir-vulkan-runner/VulkanRuntime.h
+++ b/mlir/tools/mlir-vulkan-runner/VulkanRuntime.h
@@ -29,8 +29,11 @@ struct VulkanDeviceMemoryBuffer {
   BindingIndex bindingIndex{0};
   VkDescriptorType descriptorType{VK_DESCRIPTOR_TYPE_MAX_ENUM};
   VkDescriptorBufferInfo bufferInfo{};
-  VkBuffer buffer{VK_NULL_HANDLE};
+  VkBuffer hostBuffer{VK_NULL_HANDLE};
+  VkDeviceMemory hostMemory{VK_NULL_HANDLE};
+  VkBuffer deviceBuffer{VK_NULL_HANDLE};
   VkDeviceMemory deviceMemory{VK_NULL_HANDLE};
+  uint32_t bufferSize{0};
 };
 
 /// Struct containing information regarding to a host memory buffer.
@@ -137,6 +140,9 @@ class VulkanRuntime {
   LogicalResult createQueryPool();
   LogicalResult createComputeCommandBuffer();
   LogicalResult submitCommandBuffersToQueue();
+  // Copy resources from host (staging buffer) to device buffer or from device
+  // buffer to host buffer.
+  LogicalResult copyResource(bool deviceToHost);
 
   //===--------------------------------------------------------------------===//
   // Helper methods.
@@ -203,7 +209,8 @@ class VulkanRuntime {
 
   uint32_t queueFamilyIndex{0};
   VkQueueFamilyProperties queueFamilyProperties{};
-  uint32_t memoryTypeIndex{VK_MAX_MEMORY_TYPES};
+  uint32_t hostMemoryTypeIndex{VK_MAX_MEMORY_TYPES};
+  uint32_t deviceMemoryTypeIndex{VK_MAX_MEMORY_TYPES};
   VkDeviceSize memorySize{0};
 
   //===--------------------------------------------------------------------===//
diff --git a/mlir/unittests/Analysis/AffineStructuresTest.cpp b/mlir/unittests/Analysis/AffineStructuresTest.cpp
new file mode 100644
index 0000000000000..4ad977d7351fd
--- /dev/null
+++ b/mlir/unittests/Analysis/AffineStructuresTest.cpp
@@ -0,0 +1,277 @@
+//===- AffineStructuresTest.cpp - Tests for AffineStructures ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Analysis/AffineStructures.h"
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <numeric>
+
+namespace mlir {
+
+/// Evaluate the value of the given affine expression at the specified point.
+/// The expression is a list of coefficients for the dimensions followed by the
+/// constant term.
+int64_t valueAt(ArrayRef<int64_t> expr, ArrayRef<int64_t> point) {
+  assert(expr.size() == 1 + point.size());
+  int64_t value = expr.back();
+  for (unsigned i = 0; i < point.size(); ++i)
+    value += expr[i] * point[i];
+  return value;
+}
+
+/// If 'hasValue' is true, check that findIntegerSample returns a valid sample
+/// for the FlatAffineConstraints fac.
+///
+/// If hasValue is false, check that findIntegerSample does not return None.
+void checkSample(bool hasValue, const FlatAffineConstraints &fac) {
+  Optional<SmallVector<int64_t, 8>> maybeSample = fac.findIntegerSample();
+  if (!hasValue) {
+    EXPECT_FALSE(maybeSample.hasValue());
+    if (maybeSample.hasValue()) {
+      for (auto x : *maybeSample)
+        llvm::errs() << x << ' ';
+      llvm::errs() << '\n';
+    }
+  } else {
+    ASSERT_TRUE(maybeSample.hasValue());
+    for (unsigned i = 0; i < fac.getNumEqualities(); ++i)
+      EXPECT_EQ(valueAt(fac.getEquality(i), *maybeSample), 0);
+    for (unsigned i = 0; i < fac.getNumInequalities(); ++i)
+      EXPECT_GE(valueAt(fac.getInequality(i), *maybeSample), 0);
+  }
+}
+
+/// Construct a FlatAffineConstraints from a set of inequality and
+/// equality constraints.
+FlatAffineConstraints
+makeFACFromConstraints(unsigned dims, ArrayRef<SmallVector<int64_t, 4>> ineqs,
+                       ArrayRef<SmallVector<int64_t, 4>> eqs) {
+  FlatAffineConstraints fac(ineqs.size(), eqs.size(), dims + 1, dims);
+  for (const auto &eq : eqs)
+    fac.addEquality(eq);
+  for (const auto &ineq : ineqs)
+    fac.addInequality(ineq);
+  return fac;
+}
+
+/// Check sampling for all the permutations of the dimensions for the given
+/// constraint set. Since the GBR algorithm progresses dimension-wise, different
+/// orderings may cause the algorithm to proceed differently. At least some of
+///.these permutations should make it past the heuristics and test the
+/// implementation of the GBR algorithm itself.
+void checkPermutationsSample(bool hasValue, unsigned nDim,
+                             ArrayRef<SmallVector<int64_t, 4>> ineqs,
+                             ArrayRef<SmallVector<int64_t, 4>> eqs) {
+  SmallVector<unsigned, 4> perm(nDim);
+  std::iota(perm.begin(), perm.end(), 0);
+  auto permute = [&perm](ArrayRef<int64_t> coeffs) {
+    SmallVector<int64_t, 4> permuted;
+    for (unsigned id : perm)
+      permuted.push_back(coeffs[id]);
+    permuted.push_back(coeffs.back());
+    return permuted;
+  };
+  do {
+    SmallVector<SmallVector<int64_t, 4>, 4> permutedIneqs, permutedEqs;
+    for (const auto &ineq : ineqs)
+      permutedIneqs.push_back(permute(ineq));
+    for (const auto &eq : eqs)
+      permutedEqs.push_back(permute(eq));
+
+    checkSample(hasValue,
+                makeFACFromConstraints(nDim, permutedIneqs, permutedEqs));
+  } while (std::next_permutation(perm.begin(), perm.end()));
+}
+
+TEST(FlatAffineConstraintsTest, FindSampleTest) {
+  // Bounded sets with only inequalities.
+
+  // 0 <= 7x <= 5
+  checkSample(true, makeFACFromConstraints(1, {{7, 0}, {-7, 5}}, {}));
+
+  // 1 <= 5x and 5x <= 4 (no solution).
+  checkSample(false, makeFACFromConstraints(1, {{5, -1}, {-5, 4}}, {}));
+
+  // 1 <= 5x and 5x <= 9 (solution: x = 1).
+  checkSample(true, makeFACFromConstraints(1, {{5, -1}, {-5, 9}}, {}));
+
+  // Bounded sets with equalities.
+  // x >= 8 and 40 >= y and x = y.
+  checkSample(
+      true, makeFACFromConstraints(2, {{1, 0, -8}, {0, -1, 40}}, {{1, -1, 0}}));
+
+  // x <= 10 and y <= 10 and 10 <= z and x + 2y = 3z.
+  // solution: x = y = z = 10.
+  checkSample(true, makeFACFromConstraints(
+                        3, {{-1, 0, 0, 10}, {0, -1, 0, 10}, {0, 0, 1, -10}},
+                        {{1, 2, -3, 0}}));
+
+  // x <= 10 and y <= 10 and 11 <= z and x + 2y = 3z.
+  // This implies x + 2y >= 33 and x + 2y <= 30, which has no solution.
+  checkSample(false, makeFACFromConstraints(
+                         3, {{-1, 0, 0, 10}, {0, -1, 0, 10}, {0, 0, 1, -11}},
+                         {{1, 2, -3, 0}}));
+
+  // 0 <= r and r <= 3 and 4q + r = 7.
+  // Solution: q = 1, r = 3.
+  checkSample(true,
+              makeFACFromConstraints(2, {{0, 1, 0}, {0, -1, 3}}, {{4, 1, -7}}));
+
+  // 4q + r = 7 and r = 0.
+  // Solution: q = 1, r = 3.
+  checkSample(false, makeFACFromConstraints(2, {}, {{4, 1, -7}, {0, 1, 0}}));
+
+  // The next two sets are large sets that should take a long time to sample
+  // with a naive branch and bound algorithm but can be sampled efficiently with
+  // the GBR algroithm.
+  //
+  // This is a triangle with vertices at (1/3, 0), (2/3, 0) and (10000, 10000).
+  checkSample(
+      true,
+      makeFACFromConstraints(
+          2, {{0, 1, 0}, {300000, -299999, -100000}, {-300000, 299998, 200000}},
+          {}));
+
+  // This is a tetrahedron with vertices at
+  // (1/3, 0, 0), (2/3, 0, 0), (2/3, 0, 10000), and (10000, 10000, 10000).
+  // The first three points form a triangular base on the xz plane with the
+  // apex at the fourth point, which is the only integer point.
+  checkPermutationsSample(
+      true, 3,
+      {
+          {0, 1, 0, 0},  // y >= 0
+          {0, -1, 1, 0}, // z >= y
+          {300000, -299998, -1,
+           -100000},                    // -300000x + 299998y + 100000 + z <= 0.
+          {-150000, 149999, 0, 100000}, // -150000x + 149999y + 100000 >= 0.
+      },
+      {});
+
+  // Same thing with some spurious extra dimensions equated to constants.
+  checkSample(true,
+              makeFACFromConstraints(
+                  5,
+                  {
+                      {0, 1, 0, 1, -1, 0},
+                      {0, -1, 1, -1, 1, 0},
+                      {300000, -299998, -1, -9, 21, -112000},
+                      {-150000, 149999, 0, -15, 47, 68000},
+                  },
+                  {{0, 0, 0, 1, -1, 0},       // p = q.
+                   {0, 0, 0, 1, 1, -2000}})); // p + q = 20000 => p = q = 10000.
+
+  // This is a tetrahedron with vertices at
+  // (1/3, 0, 0), (2/3, 0, 0), (2/3, 0, 100), (100, 100 - 1/3, 100).
+  checkPermutationsSample(false, 3,
+                          {
+                              {0, 1, 0, 0},
+                              {0, -300, 299, 0},
+                              {300 * 299, -89400, -299, -100 * 299},
+                              {-897, 894, 0, 598},
+                          },
+                          {});
+
+  // Two tests involving equalities that are integer empty but not rational
+  // empty.
+
+  // This is a line segment from (0, 1/3) to (100, 100 + 1/3).
+  checkSample(false, makeFACFromConstraints(
+                         2,
+                         {
+                             {1, 0, 0},   // x >= 0.
+                             {-1, 0, 100} // -x + 100 >= 0, i.e., x <= 100.
+                         },
+                         {
+                             {3, -3, 1} // 3x - 3y + 1 = 0, i.e., y = x + 1/3.
+                         }));
+
+  // A thin parallelogram. 0 <= x <= 100 and x + 1/3 <= y <= x + 2/3.
+  checkSample(false, makeFACFromConstraints(2,
+                                            {
+                                                {1, 0, 0},    // x >= 0.
+                                                {-1, 0, 100}, // x <= 100.
+                                                {3, -3, 2},   // 3x - 3y >= -2.
+                                                {-3, 3, -1},  // 3x - 3y <= -1.
+                                            },
+                                            {}));
+
+  checkSample(true, makeFACFromConstraints(2,
+                                           {
+                                               {2, 0, 0},   // 2x >= 1.
+                                               {-2, 0, 99}, // 2x <= 99.
+                                               {0, 2, 0},   // 2y >= 0.
+                                               {0, -2, 99}, // 2y <= 99.
+                                           },
+                                           {}));
+}
+
+TEST(FlatAffineConstraintsTest, IsIntegerEmptyTest) {
+  // 1 <= 5x and 5x <= 4 (no solution).
+  EXPECT_TRUE(
+      makeFACFromConstraints(1, {{5, -1}, {-5, 4}}, {}).isIntegerEmpty());
+  // 1 <= 5x and 5x <= 9 (solution: x = 1).
+  EXPECT_FALSE(
+      makeFACFromConstraints(1, {{5, -1}, {-5, 9}}, {}).isIntegerEmpty());
+
+  // An unbounded set, which isIntegerEmpty should detect as unbounded and
+  // return without calling findIntegerSample.
+  EXPECT_FALSE(makeFACFromConstraints(3,
+                                      {
+                                          {2, 0, 0, -1},
+                                          {-2, 0, 0, 1},
+                                          {0, 2, 0, -1},
+                                          {0, -2, 0, 1},
+                                          {0, 0, 2, -1},
+                                      },
+                                      {})
+                   .isIntegerEmpty());
+
+  // FlatAffineConstraints::isEmpty() does not detect the following sets to be
+  // empty.
+
+  // 3x + 7y = 1 and 0 <= x, y <= 10.
+  // Since x and y are non-negative, 3x + 7y can never be 1.
+  EXPECT_TRUE(
+      makeFACFromConstraints(
+          2, {{1, 0, 0}, {-1, 0, 10}, {0, 1, 0}, {0, -1, 10}}, {{3, 7, -1}})
+          .isIntegerEmpty());
+
+  // 2x = 3y and y = x - 1 and x + y = 6z + 2 and 0 <= x, y <= 100.
+  // Substituting y = x - 1 in 3y = 2x, we obtain x = 3 and hence y = 2.
+  // Since x + y = 5 cannot be equal to 6z + 2 for any z, the set is empty.
+  EXPECT_TRUE(
+      makeFACFromConstraints(3,
+                             {
+                                 {1, 0, 0, 0},
+                                 {-1, 0, 0, 100},
+                                 {0, 1, 0, 0},
+                                 {0, -1, 0, 100},
+                             },
+                             {{2, -3, 0, 0}, {1, -1, 0, -1}, {1, 1, -6, -2}})
+          .isIntegerEmpty());
+
+  // 2x = 3y and y = x - 1 + 6z and x + y = 6q + 2 and 0 <= x, y <= 100.
+  // 2x = 3y implies x is a multiple of 3 and y is even.
+  // Now y = x - 1 + 6z implies y = 2 mod 3. In fact, since y is even, we have
+  // y = 2 mod 6. Then since x = y + 1 + 6z, we have x = 3 mod 6, implying
+  // x + y = 5 mod 6, which contradicts x + y = 6q + 2, so the set is empty.
+  EXPECT_TRUE(makeFACFromConstraints(
+                  4,
+                  {
+                      {1, 0, 0, 0, 0},
+                      {-1, 0, 0, 0, 100},
+                      {0, 1, 0, 0, 0},
+                      {0, -1, 0, 0, 100},
+                  },
+                  {{2, -3, 0, 0, 0}, {1, -1, 6, 0, -1}, {1, 1, 0, -6, -2}})
+                  .isIntegerEmpty());
+}
+
+} // namespace mlir
diff --git a/mlir/unittests/Analysis/CMakeLists.txt b/mlir/unittests/Analysis/CMakeLists.txt
new file mode 100644
index 0000000000000..16d084dc452f7
--- /dev/null
+++ b/mlir/unittests/Analysis/CMakeLists.txt
@@ -0,0 +1,8 @@
+add_mlir_unittest(MLIRAnalysisTests
+  AffineStructuresTest.cpp
+)
+
+target_link_libraries(MLIRAnalysisTests
+  PRIVATE MLIRLoopAnalysis)
+
+add_subdirectory(Presburger)
diff --git a/mlir/unittests/Analysis/Presburger/CMakeLists.txt b/mlir/unittests/Analysis/Presburger/CMakeLists.txt
new file mode 100644
index 0000000000000..0cfda9b0c8aa4
--- /dev/null
+++ b/mlir/unittests/Analysis/Presburger/CMakeLists.txt
@@ -0,0 +1,7 @@
+add_mlir_unittest(MLIRPresburgerTests
+  MatrixTest.cpp
+  SimplexTest.cpp
+)
+
+target_link_libraries(MLIRPresburgerTests
+  PRIVATE MLIRPresburger)
diff --git a/mlir/unittests/Analysis/Presburger/MatrixTest.cpp b/mlir/unittests/Analysis/Presburger/MatrixTest.cpp
new file mode 100644
index 0000000000000..4d8801b579a79
--- /dev/null
+++ b/mlir/unittests/Analysis/Presburger/MatrixTest.cpp
@@ -0,0 +1,92 @@
+//===- MatrixTest.cpp - Tests for Matrix ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Analysis/Presburger/Matrix.h"
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+namespace mlir {
+
+TEST(MatrixTest, ReadWrite) {
+  Matrix mat(5, 5);
+  for (unsigned row = 0; row < 5; ++row)
+    for (unsigned col = 0; col < 5; ++col)
+      mat(row, col) = 10 * row + col;
+  for (unsigned row = 0; row < 5; ++row)
+    for (unsigned col = 0; col < 5; ++col)
+      EXPECT_EQ(mat(row, col), int(10 * row + col));
+}
+
+TEST(MatrixTest, SwapColumns) {
+  Matrix mat(5, 5);
+  for (unsigned row = 0; row < 5; ++row)
+    for (unsigned col = 0; col < 5; ++col)
+      mat(row, col) = col == 3 ? 1 : 0;
+  mat.swapColumns(3, 1);
+  for (unsigned row = 0; row < 5; ++row)
+    for (unsigned col = 0; col < 5; ++col)
+      EXPECT_EQ(mat(row, col), col == 1 ? 1 : 0);
+
+  // swap around all the other columns, swap (1, 3) twice for no effect.
+  mat.swapColumns(3, 1);
+  mat.swapColumns(2, 4);
+  mat.swapColumns(1, 3);
+  mat.swapColumns(0, 4);
+  mat.swapColumns(2, 2);
+
+  for (unsigned row = 0; row < 5; ++row)
+    for (unsigned col = 0; col < 5; ++col)
+      EXPECT_EQ(mat(row, col), col == 1 ? 1 : 0);
+}
+
+TEST(MatrixTest, SwapRows) {
+  Matrix mat(5, 5);
+  for (unsigned row = 0; row < 5; ++row)
+    for (unsigned col = 0; col < 5; ++col)
+      mat(row, col) = row == 2 ? 1 : 0;
+  mat.swapRows(2, 0);
+  for (unsigned row = 0; row < 5; ++row)
+    for (unsigned col = 0; col < 5; ++col)
+      EXPECT_EQ(mat(row, col), row == 0 ? 1 : 0);
+
+  // swap around all the other rows, swap (2, 0) twice for no effect.
+  mat.swapRows(3, 4);
+  mat.swapRows(1, 4);
+  mat.swapRows(2, 0);
+  mat.swapRows(1, 1);
+  mat.swapRows(0, 2);
+
+  for (unsigned row = 0; row < 5; ++row)
+    for (unsigned col = 0; col < 5; ++col)
+      EXPECT_EQ(mat(row, col), row == 0 ? 1 : 0);
+}
+
+TEST(MatrixTest, resizeVertically) {
+  Matrix mat(5, 5);
+  EXPECT_EQ(mat.getNumRows(), 5u);
+  EXPECT_EQ(mat.getNumColumns(), 5u);
+  for (unsigned row = 0; row < 5; ++row)
+    for (unsigned col = 0; col < 5; ++col)
+      mat(row, col) = 10 * row + col;
+
+  mat.resizeVertically(3);
+  EXPECT_EQ(mat.getNumRows(), 3u);
+  EXPECT_EQ(mat.getNumColumns(), 5u);
+  for (unsigned row = 0; row < 3; ++row)
+    for (unsigned col = 0; col < 5; ++col)
+      EXPECT_EQ(mat(row, col), int(10 * row + col));
+
+  mat.resizeVertically(5);
+  EXPECT_EQ(mat.getNumRows(), 5u);
+  EXPECT_EQ(mat.getNumColumns(), 5u);
+  for (unsigned row = 0; row < 5; ++row)
+    for (unsigned col = 0; col < 5; ++col)
+      EXPECT_EQ(mat(row, col), row >= 3 ? 0 : int(10 * row + col));
+}
+
+} // namespace mlir
diff --git a/mlir/unittests/Analysis/Presburger/SimplexTest.cpp b/mlir/unittests/Analysis/Presburger/SimplexTest.cpp
new file mode 100644
index 0000000000000..b40b01bbc47bc
--- /dev/null
+++ b/mlir/unittests/Analysis/Presburger/SimplexTest.cpp
@@ -0,0 +1,219 @@
+//===- SimplexTest.cpp - Tests for Simplex --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Analysis/Presburger/Simplex.h"
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+namespace mlir {
+
+/// Take a snapshot, add constraints making the set empty, and rollback.
+/// The set should not be empty after rolling back.
+TEST(SimplexTest, emptyRollback) {
+  Simplex simplex(2);
+  // (u - v) >= 0
+  simplex.addInequality({1, -1, 0});
+  EXPECT_FALSE(simplex.isEmpty());
+
+  unsigned snapshot = simplex.getSnapshot();
+  // (u - v) <= -1
+  simplex.addInequality({-1, 1, -1});
+  EXPECT_TRUE(simplex.isEmpty());
+  simplex.rollback(snapshot);
+  EXPECT_FALSE(simplex.isEmpty());
+}
+
+/// Check that the set gets marked as empty when we add contradictory
+/// constraints.
+TEST(SimplexTest, addEquality_separate) {
+  Simplex simplex(1);
+  simplex.addInequality({1, -1}); // x >= 1.
+  ASSERT_FALSE(simplex.isEmpty());
+  simplex.addEquality({1, 0}); // x == 0.
+  EXPECT_TRUE(simplex.isEmpty());
+}
+
+void expectInequalityMakesSetEmpty(Simplex &simplex, ArrayRef<int64_t> coeffs,
+                                   bool expect) {
+  ASSERT_FALSE(simplex.isEmpty());
+  unsigned snapshot = simplex.getSnapshot();
+  simplex.addInequality(coeffs);
+  EXPECT_EQ(simplex.isEmpty(), expect);
+  simplex.rollback(snapshot);
+}
+
+TEST(SimplexTest, addInequality_rollback) {
+  Simplex simplex(3);
+  SmallVector<int64_t, 4> coeffs[]{{1, 0, 0, 0},   // u >= 0.
+                                   {-1, 0, 0, 0},  // u <= 0.
+                                   {1, -1, 1, 0},  // u - v + w >= 0.
+                                   {1, 1, -1, 0}}; // u + v - w >= 0.
+  // The above constraints force u = 0 and v = w.
+  // The constraints below violate v = w.
+  SmallVector<int64_t, 4> checkCoeffs[]{{0, 1, -1, -1},  // v - w >= 1.
+                                        {0, -1, 1, -1}}; // v - w <= -1.
+
+  for (int run = 0; run < 4; run++) {
+    unsigned snapshot = simplex.getSnapshot();
+
+    expectInequalityMakesSetEmpty(simplex, checkCoeffs[0], false);
+    expectInequalityMakesSetEmpty(simplex, checkCoeffs[1], false);
+
+    for (int i = 0; i < 4; i++)
+      simplex.addInequality(coeffs[(run + i) % 4]);
+
+    expectInequalityMakesSetEmpty(simplex, checkCoeffs[0], true);
+    expectInequalityMakesSetEmpty(simplex, checkCoeffs[1], true);
+
+    simplex.rollback(snapshot);
+    EXPECT_EQ(simplex.numConstraints(), 0u);
+
+    expectInequalityMakesSetEmpty(simplex, checkCoeffs[0], false);
+    expectInequalityMakesSetEmpty(simplex, checkCoeffs[1], false);
+  }
+}
+
+Simplex simplexFromConstraints(unsigned nDim,
+                               SmallVector<SmallVector<int64_t, 8>, 8> ineqs,
+                               SmallVector<SmallVector<int64_t, 8>, 8> eqs) {
+  Simplex simplex(nDim);
+  for (const auto &ineq : ineqs)
+    simplex.addInequality(ineq);
+  for (const auto &eq : eqs)
+    simplex.addEquality(eq);
+  return simplex;
+}
+
+TEST(SimplexTest, isUnbounded) {
+  EXPECT_FALSE(simplexFromConstraints(
+                   2, {{1, 1, 0}, {-1, -1, 0}, {1, -1, 5}, {-1, 1, -5}}, {})
+                   .isUnbounded());
+
+  EXPECT_TRUE(
+      simplexFromConstraints(2, {{1, 1, 0}, {1, -1, 5}, {-1, 1, -5}}, {})
+          .isUnbounded());
+
+  EXPECT_TRUE(
+      simplexFromConstraints(2, {{-1, -1, 0}, {1, -1, 5}, {-1, 1, -5}}, {})
+          .isUnbounded());
+
+  EXPECT_TRUE(simplexFromConstraints(2, {}, {}).isUnbounded());
+
+  EXPECT_FALSE(simplexFromConstraints(3,
+                                      {
+                                          {2, 0, 0, -1},
+                                          {-2, 0, 0, 1},
+                                          {0, 2, 0, -1},
+                                          {0, -2, 0, 1},
+                                          {0, 0, 2, -1},
+                                          {0, 0, -2, 1},
+                                      },
+                                      {})
+                   .isUnbounded());
+
+  EXPECT_TRUE(simplexFromConstraints(3,
+                                     {
+                                         {2, 0, 0, -1},
+                                         {-2, 0, 0, 1},
+                                         {0, 2, 0, -1},
+                                         {0, -2, 0, 1},
+                                         {0, 0, -2, 1},
+                                     },
+                                     {})
+                  .isUnbounded());
+
+  EXPECT_TRUE(simplexFromConstraints(3,
+                                     {
+                                         {2, 0, 0, -1},
+                                         {-2, 0, 0, 1},
+                                         {0, 2, 0, -1},
+                                         {0, -2, 0, 1},
+                                         {0, 0, 2, -1},
+                                     },
+                                     {})
+                  .isUnbounded());
+
+  // Bounded set with equalities.
+  EXPECT_FALSE(simplexFromConstraints(2,
+                                      {{1, 1, 1},    // x + y >= -1.
+                                       {-1, -1, 1}}, // x + y <=  1.
+                                      {{1, -1, 0}}   // x = y.
+                                      )
+                   .isUnbounded());
+
+  // Unbounded set with equalities.
+  EXPECT_TRUE(simplexFromConstraints(3,
+                                     {{1, 1, 1, 1},     // x + y + z >= -1.
+                                      {-1, -1, -1, 1}}, // x + y + z <=  1.
+                                     {{1, -1, -1, 0}}   // x = y + z.
+                                     )
+                  .isUnbounded());
+
+  // Rational empty set.
+  EXPECT_FALSE(simplexFromConstraints(3,
+                                      {
+                                          {2, 0, 0, -1},
+                                          {-2, 0, 0, 1},
+                                          {0, 2, 2, -1},
+                                          {0, -2, -2, 1},
+                                          {3, 3, 3, -4},
+                                      },
+                                      {})
+                   .isUnbounded());
+}
+
+TEST(SimplexTest, getSamplePointIfIntegral) {
+  // Empty set.
+  EXPECT_FALSE(simplexFromConstraints(3,
+                                      {
+                                          {2, 0, 0, -1},
+                                          {-2, 0, 0, 1},
+                                          {0, 2, 2, -1},
+                                          {0, -2, -2, 1},
+                                          {3, 3, 3, -4},
+                                      },
+                                      {})
+                   .getSamplePointIfIntegral()
+                   .hasValue());
+
+  auto maybeSample = simplexFromConstraints(2,
+                                            {// x = y - 2.
+                                             {1, -1, 2},
+                                             {-1, 1, -2},
+                                             // x + y = 2.
+                                             {1, 1, -2},
+                                             {-1, -1, 2}},
+                                            {})
+                         .getSamplePointIfIntegral();
+
+  EXPECT_TRUE(maybeSample.hasValue());
+  EXPECT_THAT(*maybeSample, testing::ElementsAre(0, 2));
+
+  auto maybeSample2 = simplexFromConstraints(2,
+                                             {
+                                                 {1, 0, 0},  // x >= 0.
+                                                 {-1, 0, 0}, // x <= 0.
+                                             },
+                                             {
+                                                 {0, 1, -2} // y = 2.
+                                             })
+                          .getSamplePointIfIntegral();
+  EXPECT_TRUE(maybeSample2.hasValue());
+  EXPECT_THAT(*maybeSample2, testing::ElementsAre(0, 2));
+
+  EXPECT_FALSE(simplexFromConstraints(1,
+                                      {// 2x = 1. (no integer solutions)
+                                       {2, -1},
+                                       {-2, +1}},
+                                      {})
+                   .getSamplePointIfIntegral()
+                   .hasValue());
+}
+
+} // namespace mlir
diff --git a/mlir/unittests/CMakeLists.txt b/mlir/unittests/CMakeLists.txt
index e80cc914cb824..851092c5b56a4 100644
--- a/mlir/unittests/CMakeLists.txt
+++ b/mlir/unittests/CMakeLists.txt
@@ -5,6 +5,7 @@ function(add_mlir_unittest test_dirname)
   add_unittest(MLIRUnitTests ${test_dirname} ${ARGN})
 endfunction()
 
+add_subdirectory(Analysis)
 add_subdirectory(Dialect)
 add_subdirectory(IR)
 add_subdirectory(Pass)
diff --git a/mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp b/mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp
index bfefebeefac30..fe5632d7ae165 100644
--- a/mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp
+++ b/mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp
@@ -14,6 +14,7 @@
 
 #include "mlir/Dialect/SPIRV/SPIRVBinaryUtils.h"
 #include "mlir/Dialect/SPIRV/SPIRVDialect.h"
+#include "mlir/Dialect/SPIRV/SPIRVModule.h"
 #include "mlir/Dialect/SPIRV/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Serialization.h"
 #include "mlir/IR/Diagnostics.h"
@@ -24,7 +25,7 @@
 
 using namespace mlir;
 
-// Load the SPIRV dialect
+/// Load the SPIRV dialect.
 static DialectRegistration<spirv::SPIRVDialect> SPIRVRegistration;
 
 using ::testing::StrEq;
@@ -46,7 +47,7 @@ class DeserializationTest : public ::testing::Test {
   }
 
   /// Performs deserialization and returns the constructed spv.module op.
-  Optional<spirv::ModuleOp> deserialize() {
+  spirv::OwningSPIRVModuleRef deserialize() {
     return spirv::deserialize(binary, &context);
   }
 
@@ -54,7 +55,7 @@ class DeserializationTest : public ::testing::Test {
   void expectDiagnostic(StringRef errorMessage) {
     ASSERT_NE(nullptr, diagnostic.get());
 
-    // TODO(antiagainst): check error location too.
+    // TODO: check error location too.
     EXPECT_THAT(diagnostic->str(), StrEq(std::string(errorMessage)));
   }
 
@@ -130,27 +131,27 @@ class DeserializationTest : public ::testing::Test {
 //===----------------------------------------------------------------------===//
 
 TEST_F(DeserializationTest, EmptyModuleFailure) {
-  ASSERT_EQ(llvm::None, deserialize());
+  ASSERT_FALSE(deserialize());
   expectDiagnostic("SPIR-V binary module must have a 5-word header");
 }
 
 TEST_F(DeserializationTest, WrongMagicNumberFailure) {
   addHeader();
   binary.front() = 0xdeadbeef; // Change to a wrong magic number
-  ASSERT_EQ(llvm::None, deserialize());
+  ASSERT_FALSE(deserialize());
   expectDiagnostic("incorrect magic number");
 }
 
 TEST_F(DeserializationTest, OnlyHeaderSuccess) {
   addHeader();
-  EXPECT_NE(llvm::None, deserialize());
+  EXPECT_TRUE(deserialize());
 }
 
 TEST_F(DeserializationTest, ZeroWordCountFailure) {
   addHeader();
   binary.push_back(0); // OpNop with zero word count
 
-  ASSERT_EQ(llvm::None, deserialize());
+  ASSERT_FALSE(deserialize());
   expectDiagnostic("word count cannot be zero");
 }
 
@@ -158,9 +159,9 @@ TEST_F(DeserializationTest, InsufficientWordFailure) {
   addHeader();
   binary.push_back((2u << 16) |
                    static_cast<uint32_t>(spirv::Opcode::OpTypeVoid));
-  // Missing word for type <id>
+  // Missing word for type <id>.
 
-  ASSERT_EQ(llvm::None, deserialize());
+  ASSERT_FALSE(deserialize());
   expectDiagnostic("insufficient words for the last instruction");
 }
 
@@ -172,7 +173,7 @@ TEST_F(DeserializationTest, IntTypeMissingSignednessFailure) {
   addHeader();
   addInstruction(spirv::Opcode::OpTypeInt, {nextID++, 32});
 
-  ASSERT_EQ(llvm::None, deserialize());
+  ASSERT_FALSE(deserialize());
   expectDiagnostic("OpTypeInt must have bitwidth and signedness parameters");
 }
 
@@ -198,7 +199,7 @@ TEST_F(DeserializationTest, OpMemberNameSuccess) {
   addInstruction(spirv::Opcode::OpMemberName, operands2);
 
   binary.append(typeDecl.begin(), typeDecl.end());
-  EXPECT_NE(llvm::None, deserialize());
+  EXPECT_TRUE(deserialize());
 }
 
 TEST_F(DeserializationTest, OpMemberNameMissingOperands) {
@@ -215,7 +216,7 @@ TEST_F(DeserializationTest, OpMemberNameMissingOperands) {
   addInstruction(spirv::Opcode::OpMemberName, operands1);
 
   binary.append(typeDecl.begin(), typeDecl.end());
-  ASSERT_EQ(llvm::None, deserialize());
+  ASSERT_FALSE(deserialize());
   expectDiagnostic("OpMemberName must have at least 3 operands");
 }
 
@@ -234,7 +235,7 @@ TEST_F(DeserializationTest, OpMemberNameExcessOperands) {
   addInstruction(spirv::Opcode::OpMemberName, operands);
 
   binary.append(typeDecl.begin(), typeDecl.end());
-  ASSERT_EQ(llvm::None, deserialize());
+  ASSERT_FALSE(deserialize());
   expectDiagnostic("unexpected trailing words in OpMemberName instruction");
 }
 
@@ -247,9 +248,9 @@ TEST_F(DeserializationTest, FunctionMissingEndFailure) {
   auto voidType = addVoidType();
   auto fnType = addFunctionType(voidType, {});
   addFunction(voidType, fnType);
-  // Missing OpFunctionEnd
+  // Missing OpFunctionEnd.
 
-  ASSERT_EQ(llvm::None, deserialize());
+  ASSERT_FALSE(deserialize());
   expectDiagnostic("expected OpFunctionEnd instruction");
 }
 
@@ -259,9 +260,9 @@ TEST_F(DeserializationTest, FunctionMissingParameterFailure) {
   auto i32Type = addIntType(32);
   auto fnType = addFunctionType(voidType, {i32Type});
   addFunction(voidType, fnType);
-  // Missing OpFunctionParameter
+  // Missing OpFunctionParameter.
 
-  ASSERT_EQ(llvm::None, deserialize());
+  ASSERT_FALSE(deserialize());
   expectDiagnostic("expected OpFunctionParameter instruction");
 }
 
@@ -270,11 +271,11 @@ TEST_F(DeserializationTest, FunctionMissingLabelForFirstBlockFailure) {
   auto voidType = addVoidType();
   auto fnType = addFunctionType(voidType, {});
   addFunction(voidType, fnType);
-  // Missing OpLabel
+  // Missing OpLabel.
   addReturn();
   addFunctionEnd();
 
-  ASSERT_EQ(llvm::None, deserialize());
+  ASSERT_FALSE(deserialize());
   expectDiagnostic("a basic block must start with OpLabel");
 }
 
@@ -287,6 +288,6 @@ TEST_F(DeserializationTest, FunctionMalformedLabelFailure) {
   addReturn();
   addFunctionEnd();
 
-  ASSERT_EQ(llvm::None, deserialize());
+  ASSERT_FALSE(deserialize());
   expectDiagnostic("OpLabel should only have result <id>");
 }
diff --git a/mlir/unittests/SDBM/SDBMTest.cpp b/mlir/unittests/SDBM/SDBMTest.cpp
index e599094ee5929..61d670650b4bf 100644
--- a/mlir/unittests/SDBM/SDBMTest.cpp
+++ b/mlir/unittests/SDBM/SDBMTest.cpp
@@ -17,7 +17,7 @@
 
 using namespace mlir;
 
-// Load the SDBM dialect
+/// Load the SDBM dialect.
 static DialectRegistration<SDBMDialect> SDBMRegistration;
 
 static MLIRContext *ctx() {
diff --git a/mlir/unittests/TableGen/EnumsGenTest.cpp b/mlir/unittests/TableGen/EnumsGenTest.cpp
index 0c3db73109d92..a5580197a0a39 100644
--- a/mlir/unittests/TableGen/EnumsGenTest.cpp
+++ b/mlir/unittests/TableGen/EnumsGenTest.cpp
@@ -6,20 +6,18 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "mlir/Support/LLVM.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/Optional.h"
-#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "gmock/gmock.h"
 #include <type_traits>
 
-// Pull in generated enum utility declarations
+/// Pull in generated enum utility declarations and definitions.
 #include "EnumsGenTest.h.inc"
-// And definitions
 #include "EnumsGenTest.cpp.inc"
 
-// Test namespaces and enum class/utility names
+/// Test namespaces and enum class/utility names.
 using Outer::Inner::ConvertToEnum;
 using Outer::Inner::ConvertToString;
 using Outer::Inner::StrEnum;
diff --git a/mlir/unittests/TableGen/FormatTest.cpp b/mlir/unittests/TableGen/FormatTest.cpp
index ee609291150c7..0cae408bc3fb8 100644
--- a/mlir/unittests/TableGen/FormatTest.cpp
+++ b/mlir/unittests/TableGen/FormatTest.cpp
@@ -19,14 +19,14 @@ TEST(FormatTest, EmptyFmtStr) {
   EXPECT_TRUE(result.empty());
 }
 
-// Allow extra unused positional parameters
+/// Allow extra unused positional parameters.
 TEST(FormatTest, EmptyFmtStrExtraParams) {
   FmtContext ctx;
   std::string result = std::string(tgfmt("", &ctx, "a", "b", "c"));
   EXPECT_TRUE(result.empty());
 }
 
-// Allow unused placeholder substitution in context
+/// Allow unused placeholder substitution in context.
 TEST(FormatTest, EmptyFmtStrPopulatedCtx) {
   FmtContext ctx;
   ctx.withBuilder("builder");
@@ -40,21 +40,21 @@ TEST(FormatTest, LiteralFmtStr) {
   EXPECT_THAT(result, StrEq("void foo {}"));
 }
 
-// Print single dollar literally
+/// Print single dollar literally.
 TEST(FormatTest, AdjacentDollar) {
   FmtContext ctx;
   std::string result = std::string(tgfmt("$", &ctx));
   EXPECT_THAT(result, StrEq("$"));
 }
 
-// Print dangling dollar literally
+/// Print dangling dollar literally.
 TEST(FormatTest, DanglingDollar) {
   FmtContext ctx;
   std::string result = std::string(tgfmt("foo bar baz$", &ctx));
   EXPECT_THAT(result, StrEq("foo bar baz$"));
 }
 
-// Allow escape dollars with '$$'
+/// Allow escape dollars with '$$'.
 TEST(FormatTest, EscapeDollars) {
   FmtContext ctx;
   std::string result =
@@ -72,14 +72,14 @@ TEST(FormatTest, PositionalFmtStr) {
   EXPECT_THAT(result, StrEq("a b 43 d"));
 }
 
-// Output the placeholder if missing substitution
+/// Output the placeholder if missing substitution.
 TEST(FormatTest, PositionalFmtStrMissingParams) {
   FmtContext ctx;
   std::string result = std::string(tgfmt("$0 %1 $2", &ctx));
   EXPECT_THAT(result, StrEq("$0<no-subst-found> %1 $2<no-subst-found>"));
 }
 
-// Allow flexible reference of positional parameters
+/// Allow flexible reference of positional parameters.
 TEST(FormatTest, PositionalFmtStrFlexibleRef) {
   FmtContext ctx;
   std::string result = std::string(tgfmt("$2 $0 $2", &ctx, "a", "b", "c"));
@@ -122,7 +122,7 @@ TEST(FormatTest, PlaceHolderMissingSubst) {
   EXPECT_THAT(result, StrEq("$_op<no-subst-found>"));
 }
 
-// Test commonly used delimiters in C++
+/// Test commonly used delimiters in C++.
 TEST(FormatTest, PlaceHolderFmtStrDelimiter) {
   FmtContext ctx;
   ctx.addSubst("m", "");
@@ -130,7 +130,7 @@ TEST(FormatTest, PlaceHolderFmtStrDelimiter) {
   EXPECT_THAT(result, StrEq("{([])}|"));
 }
 
-// Test allowed characters in placeholder symbol
+/// Test allowed characters in placeholder symbol.
 TEST(FormatTest, CustomPlaceHolderFmtStrPlaceHolderChars) {
   FmtContext ctx;
   ctx.addSubst("m", "0 ");
diff --git a/mlir/unittests/TableGen/StructsGenTest.cpp b/mlir/unittests/TableGen/StructsGenTest.cpp
index 19aff1c83b0fe..c58fedb4ec4f0 100644
--- a/mlir/unittests/TableGen/StructsGenTest.cpp
+++ b/mlir/unittests/TableGen/StructsGenTest.cpp
@@ -17,12 +17,12 @@
 
 namespace mlir {
 
-// Pull in generated enum utility declarations
+/// Pull in generated enum utility declarations and definitions.
 #include "StructAttrGenTest.h.inc"
-// And definitions
 #include "StructAttrGenTest.cpp.inc"
-// Helper that returns an example test::TestStruct for testing its
-// implementation.
+
+/// Helper that returns an example test::TestStruct for testing its
+/// implementation.
 static test::TestStruct getTestStruct(mlir::MLIRContext *context) {
   auto integerType = mlir::IntegerType::get(32, context);
   auto integerAttr = mlir::IntegerAttr::get(integerType, 127);
@@ -39,16 +39,16 @@ static test::TestStruct getTestStruct(mlir::MLIRContext *context) {
                                optionalAttr, context);
 }
 
-// Validates that test::TestStruct::classof correctly identifies a valid
-// test::TestStruct.
+/// Validates that test::TestStruct::classof correctly identifies a valid
+/// test::TestStruct.
 TEST(StructsGenTest, ClassofTrue) {
   mlir::MLIRContext context;
   auto structAttr = getTestStruct(&context);
   ASSERT_TRUE(test::TestStruct::classof(structAttr));
 }
 
-// Validates that test::TestStruct::classof fails when an extra attribute is in
-// the class.
+/// Validates that test::TestStruct::classof fails when an extra attribute is in
+/// the class.
 TEST(StructsGenTest, ClassofExtraFalse) {
   mlir::MLIRContext context;
   mlir::DictionaryAttr structAttr = getTestStruct(&context);
@@ -69,8 +69,8 @@ TEST(StructsGenTest, ClassofExtraFalse) {
   ASSERT_FALSE(test::TestStruct::classof(badDictionary));
 }
 
-// Validates that test::TestStruct::classof fails when a NamedAttribute has an
-// incorrect name.
+/// Validates that test::TestStruct::classof fails when a NamedAttribute has an
+/// incorrect name.
 TEST(StructsGenTest, ClassofBadNameFalse) {
   mlir::MLIRContext context;
   mlir::DictionaryAttr structAttr = getTestStruct(&context);
@@ -90,8 +90,8 @@ TEST(StructsGenTest, ClassofBadNameFalse) {
   ASSERT_FALSE(test::TestStruct::classof(badDictionary));
 }
 
-// Validates that test::TestStruct::classof fails when a NamedAttribute has an
-// incorrect type.
+/// Validates that test::TestStruct::classof fails when a NamedAttribute has an
+/// incorrect type.
 TEST(StructsGenTest, ClassofBadTypeFalse) {
   mlir::MLIRContext context;
   mlir::DictionaryAttr structAttr = getTestStruct(&context);
@@ -115,8 +115,8 @@ TEST(StructsGenTest, ClassofBadTypeFalse) {
   ASSERT_FALSE(test::TestStruct::classof(badDictionary));
 }
 
-// Validates that test::TestStruct::classof fails when a NamedAttribute is
-// missing.
+/// Validates that test::TestStruct::classof fails when a NamedAttribute is
+/// missing.
 TEST(StructsGenTest, ClassofMissingFalse) {
   mlir::MLIRContext context;
   mlir::DictionaryAttr structAttr = getTestStruct(&context);
@@ -132,7 +132,7 @@ TEST(StructsGenTest, ClassofMissingFalse) {
   ASSERT_FALSE(test::TestStruct::classof(badDictionary));
 }
 
-// Validate the accessor for the FloatAttr value.
+/// Validate the accessor for the FloatAttr value.
 TEST(StructsGenTest, GetFloat) {
   mlir::MLIRContext context;
   auto structAttr = getTestStruct(&context);
@@ -140,7 +140,7 @@ TEST(StructsGenTest, GetFloat) {
   EXPECT_EQ(returnedAttr.getValueAsDouble(), 0.25);
 }
 
-// Validate the accessor for the IntegerAttr value.
+/// Validate the accessor for the IntegerAttr value.
 TEST(StructsGenTest, GetInteger) {
   mlir::MLIRContext context;
   auto structAttr = getTestStruct(&context);
@@ -148,7 +148,7 @@ TEST(StructsGenTest, GetInteger) {
   EXPECT_EQ(returnedAttr.getInt(), 127);
 }
 
-// Validate the accessor for the ElementsAttr value.
+/// Validate the accessor for the ElementsAttr value.
 TEST(StructsGenTest, GetElements) {
   mlir::MLIRContext context;
   auto structAttr = getTestStruct(&context);
diff --git a/mlir/utils/spirv/gen_spirv_dialect.py b/mlir/utils/spirv/gen_spirv_dialect.py
index 5854a74509cde..159d8651223b2 100755
--- a/mlir/utils/spirv/gen_spirv_dialect.py
+++ b/mlir/utils/spirv/gen_spirv_dialect.py
@@ -20,6 +20,7 @@
 import re
 import requests
 import textwrap
+import yaml
 
 SPIRV_HTML_SPEC_URL = 'https://www.khronos.org/registry/spir-v/specs/unified1/SPIRV.html'
 SPIRV_JSON_SPEC_URL = 'https://raw.githubusercontent.com/KhronosGroup/SPIRV-Headers/master/include/spirv/unified1/spirv.core.grammar.json'
@@ -72,7 +73,7 @@ def get_spirv_grammar_from_json_spec():
   return spirv['operand_kinds'], spirv['instructions']
 
 
-def split_list_into_sublists(items, offset):
+def split_list_into_sublists(items):
   """Split the list of items into multiple sublists.
 
   This is to make sure the string composed from each sublist won't exceed
@@ -80,7 +81,6 @@ def split_list_into_sublists(items, offset):
 
   Arguments:
     - items: a list of strings
-    - offset: the offset in calculating each sublist's length
   """
   chuncks = []
   chunk = []
@@ -255,7 +255,7 @@ def get_availability_spec(enum_case, capability_mapping, for_op, for_cap):
     min_version = ''
   elif min_version:
     min_version = 'MinVersion<SPV_V_{}>'.format(min_version.replace('.', '_'))
-  # TODO(antiagainst): delete this once ODS can support dialect-specific content
+  # TODO: delete this once ODS can support dialect-specific content
   # and we can use omission to mean no requirements.
   if for_op and not min_version:
     min_version = DEFAULT_MIN_VERSION
@@ -263,7 +263,7 @@ def get_availability_spec(enum_case, capability_mapping, for_op, for_cap):
   max_version = enum_case.get('lastVersion', '')
   if max_version:
     max_version = 'MaxVersion<SPV_V_{}>'.format(max_version.replace('.', '_'))
-  # TODO(antiagainst): delete this once ODS can support dialect-specific content
+  # TODO: delete this once ODS can support dialect-specific content
   # and we can use omission to mean no requirements.
   if for_op and not max_version:
     max_version = DEFAULT_MAX_VERSION
@@ -278,7 +278,7 @@ def get_availability_spec(enum_case, capability_mapping, for_op, for_cap):
     # a core symbol since the given version, rather than a minimal version
     # requirement.
     min_version = DEFAULT_MIN_VERSION if for_op else ''
-  # TODO(antiagainst): delete this once ODS can support dialect-specific content
+  # TODO: delete this once ODS can support dialect-specific content
   # and we can use omission to mean no requirements.
   if for_op and not exts:
     exts = DEFAULT_EXT
@@ -305,7 +305,7 @@ def get_availability_spec(enum_case, capability_mapping, for_op, for_cap):
     else:
       caps = 'Capability<[{}]>'.format(', '.join(prefixed_caps))
       implies = ''
-  # TODO(antiagainst): delete this once ODS can support dialect-specific content
+  # TODO: delete this once ODS can support dialect-specific content
   # and we can use omission to mean no requirements.
   if for_op and not caps:
     caps = DEFAULT_CAP
@@ -391,7 +391,7 @@ def get_case_symbol(kind_name, case_name):
                 for case in kind_cases]
 
   # Split them into sublists and concatenate into multiple lines
-  case_names = split_list_into_sublists(case_names, 6)
+  case_names = split_list_into_sublists(case_names)
   case_names = ['{:6}'.format('') + ', '.join(sublist)
                 for sublist in case_names]
   case_names = ',\n'.join(case_names)
@@ -428,7 +428,7 @@ def gen_opcode(instructions):
   opcode_list = [
       decl_fmt_str.format(name=inst['opname']) for inst in instructions
   ]
-  opcode_list = split_list_into_sublists(opcode_list, 6)
+  opcode_list = split_list_into_sublists(opcode_list)
   opcode_list = [
       '{:6}'.format('') + ', '.join(sublist) for sublist in opcode_list
   ]
@@ -439,6 +439,75 @@ def gen_opcode(instructions):
               '    ]>;'.format(name='Opcode', lst=opcode_list)
   return opcode_str + '\n\n' + enum_attr
 
+def map_cap_to_opnames(instructions):
+  """Maps capabilities to instructions enabled by those capabilities
+
+  Arguments:
+    - instructions: a list containing a subset of SPIR-V instructions' grammar
+  Returns:
+    - A map with keys representing capabilities and values of lists of
+    instructions enabled by the corresponding key
+  """
+  cap_to_inst = {}
+
+  for inst in instructions:
+    caps = inst['capabilities'] if 'capabilities' in inst else ['0_core_0']
+    for cap in caps:
+      if cap not in cap_to_inst:
+        cap_to_inst[cap] = []
+      cap_to_inst[cap].append(inst['opname'])
+
+  return cap_to_inst
+
+def gen_instr_coverage_report(path, instructions):
+  """Dumps to standard output a YAML report of current instruction coverage
+
+  Arguments:
+    - path: the path to SPIRBase.td
+    - instructions: a list containing all SPIR-V instructions' grammar
+  """
+  with open(path, 'r') as f:
+    content = f.read()
+
+  content = content.split(AUTOGEN_OPCODE_SECTION_MARKER)
+
+  existing_opcodes = [k[11:] for k in re.findall('def SPV_OC_\w+', content[1])]
+  existing_instructions = list(
+          filter(lambda inst: (inst['opname'] in existing_opcodes),
+              instructions))
+
+  instructions_opnames = [inst['opname'] for inst in instructions]
+
+  remaining_opcodes = list(set(instructions_opnames) - set(existing_opcodes))
+  remaining_instructions = list(
+          filter(lambda inst: (inst['opname'] in remaining_opcodes),
+              instructions))
+
+  rem_cap_to_instr = map_cap_to_opnames(remaining_instructions)
+  ex_cap_to_instr = map_cap_to_opnames(existing_instructions)
+
+  rem_cap_to_cov = {}
+
+  # Calculate coverage for each capability
+  for cap in rem_cap_to_instr:
+    if cap not in ex_cap_to_instr:
+      rem_cap_to_cov[cap] = 0.0
+    else:
+      rem_cap_to_cov[cap] = \
+              (len(ex_cap_to_instr[cap]) / (len(ex_cap_to_instr[cap]) \
+              + len(rem_cap_to_instr[cap])))
+
+  report = {}
+
+  # Merge the 3 maps into one report
+  for cap in rem_cap_to_instr:
+    report[cap] = {}
+    report[cap]['Supported Instructions'] = \
+            ex_cap_to_instr[cap] if cap in ex_cap_to_instr else []
+    report[cap]['Unsupported Instructions']  = rem_cap_to_instr[cap]
+    report[cap]['Coverage'] = '{}%'.format(int(rem_cap_to_cov[cap] * 100))
+
+  print(yaml.dump(report))
 
 def update_td_opcodes(path, instructions, filter_list):
   """Updates SPIRBase.td with new generated opcode cases.
@@ -552,7 +621,7 @@ def map_spec_operand_to_ods_argument(operand):
     else:
       arg_type = 'Variadic<SPV_Type>'
   elif kind == 'IdMemorySemantics' or kind == 'IdScope':
-    # TODO(antiagainst): Need to further constrain 'IdMemorySemantics'
+    # TODO: Need to further constrain 'IdMemorySemantics'
     # and 'IdScope' given that they should be generated from OpConstant.
     assert quantifier == '', ('unexpected to have optional/variadic memory '
                               'semantics or scope <id>')
@@ -771,7 +840,6 @@ def extract_td_op_info(op_def):
   """Extracts potentially manually specified sections in op's definition.
 
   Arguments: - A string containing the op's TableGen definition
-    - doc: the instruction's SPIR-V HTML doc
 
   Returns:
     - A dict containing potential manually specified sections
@@ -930,6 +998,8 @@ def update_td_op_definitions(path, instructions, docs, filter_list,
       default='Op',
       help='SPIR-V instruction category used for choosing '\
            'the TableGen base class to define this op')
+  cli_parser.add_argument('--gen-inst-coverage', dest='gen_inst_coverage', action='store_true')
+  cli_parser.set_defaults(gen_inst_coverage=False)
 
   args = cli_parser.parse_args()
 
@@ -956,3 +1026,6 @@ def update_td_op_definitions(path, instructions, docs, filter_list,
     print('Done. Note that this script just generates a template; ', end='')
     print('please read the spec and update traits, arguments, and ', end='')
     print('results accordingly.')
+
+  if args.gen_inst_coverage:
+    gen_instr_coverage_report(args.base_td_path, instructions)
diff --git a/mlir/utils/spirv/report_coverage.sh b/mlir/utils/spirv/report_coverage.sh
new file mode 100755
index 0000000000000..63f2f3db0cf99
--- /dev/null
+++ b/mlir/utils/spirv/report_coverage.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# Script for reporting current level of SPIR-V spec instruction coverage in spv
+# Dialect. It dumps to standard output a YAML string of current coverage.
+#
+# Run as:
+# ./report_coverage.sh
+
+set -e
+
+current_file="$(readlink -f "$0")"
+current_dir="$(dirname "$current_file")"
+
+python3 ${current_dir}/gen_spirv_dialect.py \
+  --base-td-path ${current_dir}/../../include/mlir/Dialect/SPIRV/SPIRVBase.td \
+  --gen-inst-coverage
diff --git a/openmp/cmake/OpenMPTesting.cmake b/openmp/cmake/OpenMPTesting.cmake
index f854e4ddc38d4..90e0704c4a94c 100644
--- a/openmp/cmake/OpenMPTesting.cmake
+++ b/openmp/cmake/OpenMPTesting.cmake
@@ -133,8 +133,8 @@ else()
   set(OPENMP_TEST_COMPILER_ID "Clang")
   # Cannot use CLANG_VERSION because we are not guaranteed that this is already set.
   set(OPENMP_TEST_COMPILER_VERSION "${LLVM_VERSION}")
-  set(OPENMP_TEST_COMPILER_VERSION_MAJOR "${LLVM_MAJOR_VERSION}")
-  set(OPENMP_TEST_COMPILER_VERSION_MAJOR_MINOR "${LLVM_MAJOR_VERSION}.${LLVM_MINOR_VERSION}")
+  set(OPENMP_TEST_COMPILER_VERSION_MAJOR "${LLVM_VERSION_MAJOR}")
+  set(OPENMP_TEST_COMPILER_VERSION_MAJOR_MINOR "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
   # Unfortunately the top-level cmake/config-ix.cmake file mangles CMake's
   # CMAKE_THREAD_LIBS_INIT variable from the FindThreads package, so work
   # around that, until it is fixed there.
diff --git a/openmp/docs/ReleaseNotes.rst b/openmp/docs/ReleaseNotes.rst
new file mode 100644
index 0000000000000..b7f2ec42277e3
--- /dev/null
+++ b/openmp/docs/ReleaseNotes.rst
@@ -0,0 +1,45 @@
+===========================
+openmp 11.0.0 Release Notes
+===========================
+
+.. contents::
+    :local:
+
+.. warning::
+   These are in-progress notes for the upcoming LLVM 11.0.0 release.
+   Release notes for previous releases can be found on
+   `the Download Page <https://releases.llvm.org/download.html>`_.
+
+Introduction
+============
+
+This document contains the release notes for the OpenMP runtime, release 11.0.0.
+Here we describe the status of openmp, including major improvements
+from the previous release. All openmp releases may be downloaded
+from the `LLVM releases web site <https://llvm.org/releases/>`_.
+
+Non-comprehensive list of changes in this release
+=================================================
+
+5.0 features
+------------
+
+* ...
+
+5.1 features
+------------
+
+* ...
+
+OMPT Improvements
+-----------------
+
+* Added OMPT callbacks for doacross loops, detached tasks
+* Added handling for mutexinoutset dependencies
+
+OMPT-based Tools
+----------------
+
+* Added ompt-multiplex.h as a header-only OMPT-tool to support nesting of OMPT
+  tools. (see openmp/tools/multiplex)
+
diff --git a/openmp/libomptarget/deviceRTLs/amdgcn/src/hip_atomics.h b/openmp/libomptarget/deviceRTLs/amdgcn/src/hip_atomics.h
index df102c765925c..04e80b9450700 100644
--- a/openmp/libomptarget/deviceRTLs/amdgcn/src/hip_atomics.h
+++ b/openmp/libomptarget/deviceRTLs/amdgcn/src/hip_atomics.h
@@ -11,11 +11,6 @@
 
 #include "target_impl.h"
 
-// inc requires an amdgcn specific intrinsic which is not yet available
-DEVICE unsigned atomicInc(unsigned *address);
-DEVICE unsigned atomicInc(unsigned *address, unsigned max);
-DEVICE int atomicInc(int *address);
-
 namespace {
 
 template <typename T> DEVICE T atomicAdd(T *address, T val) {
@@ -38,5 +33,9 @@ template <typename T> DEVICE T atomicCAS(T *address, T compare, T val) {
   return compare;
 }
 
+INLINE uint32_t atomicInc(uint32_t *address, uint32_t max) {
+  return __builtin_amdgcn_atomic_inc32(address, max, __ATOMIC_SEQ_CST, "");
+}
+
 } // namespace
 #endif
diff --git a/openmp/libomptarget/deviceRTLs/amdgcn/src/target_impl.h b/openmp/libomptarget/deviceRTLs/amdgcn/src/target_impl.h
index 04755a6a3e73f..3c90b39282c94 100644
--- a/openmp/libomptarget/deviceRTLs/amdgcn/src/target_impl.h
+++ b/openmp/libomptarget/deviceRTLs/amdgcn/src/target_impl.h
@@ -120,9 +120,17 @@ INLINE void __kmpc_impl_named_sync(int barrier, uint32_t num_threads) {
   __builtin_amdgcn_s_barrier();
 }
 
-DEVICE void __kmpc_impl_threadfence(void);
-DEVICE void __kmpc_impl_threadfence_block(void);
-DEVICE void __kmpc_impl_threadfence_system(void);
+INLINE void __kmpc_impl_threadfence() {
+  __builtin_amdgcn_fence(__ATOMIC_SEQ_CST, "agent");
+}
+
+INLINE void __kmpc_impl_threadfence_block() {
+  __builtin_amdgcn_fence(__ATOMIC_SEQ_CST, "workgroup");
+}
+
+INLINE void __kmpc_impl_threadfence_system() {
+  __builtin_amdgcn_fence(__ATOMIC_SEQ_CST, "");
+}
 
 // Calls to the AMDGCN layer (assuming 1D layout)
 INLINE int GetThreadIdInBlock() { return __builtin_amdgcn_workitem_id_x(); }
@@ -132,8 +140,6 @@ DEVICE int GetNumberOfThreadsInBlock();
 DEVICE unsigned GetWarpId();
 DEVICE unsigned GetLaneId();
 
-DEVICE bool __kmpc_impl_is_first_active_thread();
-
 // Locks
 DEVICE void __kmpc_impl_init_lock(omp_lock_t *lock);
 DEVICE void __kmpc_impl_destroy_lock(omp_lock_t *lock);
diff --git a/openmp/libomptarget/deviceRTLs/common/omptarget.h b/openmp/libomptarget/deviceRTLs/common/omptarget.h
index 986eb3677dcf4..88807de4e19c7 100644
--- a/openmp/libomptarget/deviceRTLs/common/omptarget.h
+++ b/openmp/libomptarget/deviceRTLs/common/omptarget.h
@@ -200,7 +200,6 @@ class omptarget_nvptx_TeamDescr {
   INLINE omptarget_nvptx_WorkDescr &WorkDescr() {
     return workDescrForActiveParallel;
   }
-  INLINE uint64_t *getLastprivateIterBuffer() { return &lastprivateIterBuffer; }
 
   // init
   INLINE void InitTeamDescr();
@@ -251,7 +250,6 @@ class omptarget_nvptx_TeamDescr {
       levelZeroTaskDescr; // icv for team master initial thread
   omptarget_nvptx_WorkDescr
       workDescrForActiveParallel; // one, ONLY for the active par
-  uint64_t lastprivateIterBuffer;
 
   ALIGN(16)
   __kmpc_data_sharing_worker_slot_static worker_rootS[WARPSIZE];
@@ -277,10 +275,6 @@ class omptarget_nvptx_ThreadPrivateContext {
   INLINE uint16_t &NumThreadsForNextParallel(int tid) {
     return nextRegion.tnum[tid];
   }
-  // simd
-  INLINE uint16_t &SimdLimitForNextSimd(int tid) {
-    return nextRegion.slim[tid];
-  }
   // schedule (for dispatch)
   INLINE kmp_sched_t &ScheduleType(int tid) { return schedule[tid]; }
   INLINE int64_t &Chunk(int tid) { return chunk[tid]; }
@@ -304,8 +298,6 @@ class omptarget_nvptx_ThreadPrivateContext {
     // Only one of the two is live at the same time.
     // parallel
     uint16_t tnum[MAX_THREADS_PER_TEAM];
-    // simd limit
-    uint16_t slim[MAX_THREADS_PER_TEAM];
   } nextRegion;
   // schedule (for dispatch)
   kmp_sched_t schedule[MAX_THREADS_PER_TEAM]; // remember schedule type for #for
diff --git a/openmp/libomptarget/deviceRTLs/common/src/data_sharing.cu b/openmp/libomptarget/deviceRTLs/common/src/data_sharing.cu
index f6523c8ce8aa2..ca2fd1d307542 100644
--- a/openmp/libomptarget/deviceRTLs/common/src/data_sharing.cu
+++ b/openmp/libomptarget/deviceRTLs/common/src/data_sharing.cu
@@ -17,297 +17,6 @@ INLINE static bool IsMasterThread(bool isSPMDExecutionMode) {
   return !isSPMDExecutionMode && GetMasterThreadID() == GetThreadIdInBlock();
 }
 
-/// Return the provided size aligned to the size of a pointer.
-INLINE static size_t AlignVal(size_t Val) {
-  const size_t Align = (size_t)sizeof(void *);
-  if (Val & (Align - 1)) {
-    Val += Align;
-    Val &= ~(Align - 1);
-  }
-  return Val;
-}
-
-#define DSFLAG 0
-#define DSFLAG_INIT 0
-#define DSPRINT(_flag, _str, _args...)                                         \
-  {                                                                            \
-    if (_flag) {                                                               \
-      /*printf("(%d,%d) -> " _str, blockIdx.x, threadIdx.x, _args);*/          \
-    }                                                                          \
-  }
-#define DSPRINT0(_flag, _str)                                                  \
-  {                                                                            \
-    if (_flag) {                                                               \
-      /*printf("(%d,%d) -> " _str, blockIdx.x, threadIdx.x);*/                 \
-    }                                                                          \
-  }
-
-// Initialize the shared data structures. This is expected to be called for the
-// master thread and warp masters. \param RootS: A pointer to the root of the
-// data sharing stack. \param InitialDataSize: The initial size of the data in
-// the slot.
-EXTERN void
-__kmpc_initialize_data_sharing_environment(__kmpc_data_sharing_slot *rootS,
-                                           size_t InitialDataSize) {
-  ASSERT0(LT_FUSSY, isRuntimeInitialized(), "Runtime must be initialized.");
-  DSPRINT0(DSFLAG_INIT,
-           "Entering __kmpc_initialize_data_sharing_environment\n");
-
-  unsigned WID = GetWarpId();
-  DSPRINT(DSFLAG_INIT, "Warp ID: %u\n", WID);
-
-  omptarget_nvptx_TeamDescr *teamDescr =
-      &omptarget_nvptx_threadPrivateContext->TeamContext();
-  __kmpc_data_sharing_slot *RootS =
-      teamDescr->RootS(WID, IsMasterThread(isSPMDMode()));
-
-  DataSharingState.SlotPtr[WID] = RootS;
-  DataSharingState.StackPtr[WID] = (void *)&RootS->Data[0];
-
-  // We don't need to initialize the frame and active threads.
-
-  DSPRINT(DSFLAG_INIT, "Initial data size: %08x \n", (unsigned)InitialDataSize);
-  DSPRINT(DSFLAG_INIT, "Root slot at: %016llx \n", (unsigned long long)RootS);
-  DSPRINT(DSFLAG_INIT, "Root slot data-end at: %016llx \n",
-          (unsigned long long)RootS->DataEnd);
-  DSPRINT(DSFLAG_INIT, "Root slot next at: %016llx \n",
-          (unsigned long long)RootS->Next);
-  DSPRINT(DSFLAG_INIT, "Shared slot ptr at: %016llx \n",
-          (unsigned long long)DataSharingState.SlotPtr[WID]);
-  DSPRINT(DSFLAG_INIT, "Shared stack ptr at: %016llx \n",
-          (unsigned long long)DataSharingState.StackPtr[WID]);
-
-  DSPRINT0(DSFLAG_INIT, "Exiting __kmpc_initialize_data_sharing_environment\n");
-}
-
-EXTERN void *__kmpc_data_sharing_environment_begin(
-    __kmpc_data_sharing_slot **SavedSharedSlot, void **SavedSharedStack,
-    void **SavedSharedFrame, __kmpc_impl_lanemask_t *SavedActiveThreads,
-    size_t SharingDataSize, size_t SharingDefaultDataSize,
-    int16_t IsOMPRuntimeInitialized) {
-
-  DSPRINT0(DSFLAG, "Entering __kmpc_data_sharing_environment_begin\n");
-
-  // If the runtime has been elided, used shared memory for master-worker
-  // data sharing.
-  if (!IsOMPRuntimeInitialized)
-    return (void *)&DataSharingState;
-
-  DSPRINT(DSFLAG, "Data Size %016llx\n", (unsigned long long)SharingDataSize);
-  DSPRINT(DSFLAG, "Default Data Size %016llx\n",
-          (unsigned long long)SharingDefaultDataSize);
-
-  unsigned WID = GetWarpId();
-  __kmpc_impl_lanemask_t CurActiveThreads = __kmpc_impl_activemask();
-
-  __kmpc_data_sharing_slot *&SlotP = DataSharingState.SlotPtr[WID];
-  void *&StackP = DataSharingState.StackPtr[WID];
-  void * volatile &FrameP = DataSharingState.FramePtr[WID];
-  __kmpc_impl_lanemask_t &ActiveT = DataSharingState.ActiveThreads[WID];
-
-  DSPRINT0(DSFLAG, "Save current slot/stack values.\n");
-  // Save the current values.
-  *SavedSharedSlot = SlotP;
-  *SavedSharedStack = StackP;
-  *SavedSharedFrame = FrameP;
-  *SavedActiveThreads = ActiveT;
-
-  DSPRINT(DSFLAG, "Warp ID: %u\n", WID);
-  DSPRINT(DSFLAG, "Saved slot ptr at: %016llx \n", (unsigned long long)SlotP);
-  DSPRINT(DSFLAG, "Saved stack ptr at: %016llx \n", (unsigned long long)StackP);
-  DSPRINT(DSFLAG, "Saved frame ptr at: %016llx \n", (long long)FrameP);
-  DSPRINT(DSFLAG, "Active threads: %08x \n", (unsigned)ActiveT);
-
-  // Only the warp active master needs to grow the stack.
-  if (__kmpc_impl_is_first_active_thread()) {
-    // Save the current active threads.
-    ActiveT = CurActiveThreads;
-
-    // Make sure we use aligned sizes to avoid rematerialization of data.
-    SharingDataSize = AlignVal(SharingDataSize);
-    // FIXME: The default data size can be assumed to be aligned?
-    SharingDefaultDataSize = AlignVal(SharingDefaultDataSize);
-
-    // Check if we have room for the data in the current slot.
-    const uintptr_t CurrentStartAddress = (uintptr_t)StackP;
-    const uintptr_t CurrentEndAddress = (uintptr_t)SlotP->DataEnd;
-    const uintptr_t RequiredEndAddress =
-        CurrentStartAddress + (uintptr_t)SharingDataSize;
-
-    DSPRINT(DSFLAG, "Data Size %016llx\n", (unsigned long long)SharingDataSize);
-    DSPRINT(DSFLAG, "Default Data Size %016llx\n",
-            (unsigned long long)SharingDefaultDataSize);
-    DSPRINT(DSFLAG, "Current Start Address %016llx\n",
-            (unsigned long long)CurrentStartAddress);
-    DSPRINT(DSFLAG, "Current End Address %016llx\n",
-            (unsigned long long)CurrentEndAddress);
-    DSPRINT(DSFLAG, "Required End Address %016llx\n",
-            (unsigned long long)RequiredEndAddress);
-    DSPRINT(DSFLAG, "Active Threads %08x\n", (unsigned)ActiveT);
-
-    // If we require a new slot, allocate it and initialize it (or attempt to
-    // reuse one). Also, set the shared stack and slot pointers to the new
-    // place. If we do not need to grow the stack, just adapt the stack and
-    // frame pointers.
-    if (CurrentEndAddress < RequiredEndAddress) {
-      size_t NewSize = (SharingDataSize > SharingDefaultDataSize)
-                           ? SharingDataSize
-                           : SharingDefaultDataSize;
-      __kmpc_data_sharing_slot *NewSlot = 0;
-
-      // Attempt to reuse an existing slot.
-      if (__kmpc_data_sharing_slot *ExistingSlot = SlotP->Next) {
-        uintptr_t ExistingSlotSize = (uintptr_t)ExistingSlot->DataEnd -
-                                     (uintptr_t)(&ExistingSlot->Data[0]);
-        if (ExistingSlotSize >= NewSize) {
-          DSPRINT(DSFLAG, "Reusing stack slot %016llx\n",
-                  (unsigned long long)ExistingSlot);
-          NewSlot = ExistingSlot;
-        } else {
-          DSPRINT(DSFLAG, "Cleaning up -failed reuse - %016llx\n",
-                  (unsigned long long)SlotP->Next);
-          SafeFree(ExistingSlot, "Failed reuse");
-        }
-      }
-
-      if (!NewSlot) {
-        NewSlot = (__kmpc_data_sharing_slot *)SafeMalloc(
-            sizeof(__kmpc_data_sharing_slot) + NewSize,
-            "Warp master slot allocation");
-        DSPRINT(DSFLAG, "New slot allocated %016llx (data size=%016llx)\n",
-                (unsigned long long)NewSlot, NewSize);
-      }
-
-      NewSlot->Next = 0;
-      NewSlot->DataEnd = &NewSlot->Data[NewSize];
-
-      SlotP->Next = NewSlot;
-      SlotP = NewSlot;
-      StackP = &NewSlot->Data[SharingDataSize];
-      FrameP = &NewSlot->Data[0];
-    } else {
-
-      // Clean up any old slot that we may still have. The slot producers, do
-      // not eliminate them because that may be used to return data.
-      if (SlotP->Next) {
-        DSPRINT(DSFLAG, "Cleaning up - old not required - %016llx\n",
-                (unsigned long long)SlotP->Next);
-        SafeFree(SlotP->Next, "Old slot not required");
-        SlotP->Next = 0;
-      }
-
-      FrameP = StackP;
-      StackP = (void *)RequiredEndAddress;
-    }
-  }
-
-  // FIXME: Need to see the impact of doing it here.
-  __kmpc_impl_threadfence_block();
-
-  DSPRINT0(DSFLAG, "Exiting __kmpc_data_sharing_environment_begin\n");
-
-  // All the threads in this warp get the frame they should work with.
-  return FrameP;
-}
-
-EXTERN void __kmpc_data_sharing_environment_end(
-    __kmpc_data_sharing_slot **SavedSharedSlot, void **SavedSharedStack,
-    void **SavedSharedFrame, __kmpc_impl_lanemask_t *SavedActiveThreads,
-    int32_t IsEntryPoint) {
-
-  DSPRINT0(DSFLAG, "Entering __kmpc_data_sharing_environment_end\n");
-
-  unsigned WID = GetWarpId();
-
-  if (IsEntryPoint) {
-    if (__kmpc_impl_is_first_active_thread()) {
-      DSPRINT0(DSFLAG, "Doing clean up\n");
-
-      // The master thread cleans the saved slot, because this is an environment
-      // only for the master.
-      __kmpc_data_sharing_slot *S = IsMasterThread(isSPMDMode())
-                                        ? *SavedSharedSlot
-                                        : DataSharingState.SlotPtr[WID];
-
-      if (S->Next) {
-        SafeFree(S->Next, "Sharing environment end");
-        S->Next = 0;
-      }
-    }
-
-    DSPRINT0(DSFLAG, "Exiting Exiting __kmpc_data_sharing_environment_end\n");
-    return;
-  }
-
-  __kmpc_impl_lanemask_t CurActive = __kmpc_impl_activemask();
-
-  // Only the warp master can restore the stack and frame information, and only
-  // if there are no other threads left behind in this environment (i.e. the
-  // warp diverged and returns in different places). This only works if we
-  // assume that threads will converge right after the call site that started
-  // the environment.
-  if (__kmpc_impl_is_first_active_thread()) {
-    __kmpc_impl_lanemask_t &ActiveT = DataSharingState.ActiveThreads[WID];
-
-    DSPRINT0(DSFLAG, "Before restoring the stack\n");
-    // Zero the bits in the mask. If it is still different from zero, then we
-    // have other threads that will return after the current ones.
-    ActiveT &= ~CurActive;
-
-    DSPRINT(DSFLAG, "Active threads: %08x; New mask: %08x\n",
-            (unsigned)CurActive, (unsigned)ActiveT);
-
-    if (!ActiveT) {
-      // No other active threads? Great, lets restore the stack.
-
-      __kmpc_data_sharing_slot *&SlotP = DataSharingState.SlotPtr[WID];
-      void *&StackP = DataSharingState.StackPtr[WID];
-      void * volatile &FrameP = DataSharingState.FramePtr[WID];
-
-      SlotP = *SavedSharedSlot;
-      StackP = *SavedSharedStack;
-      FrameP = *SavedSharedFrame;
-      ActiveT = *SavedActiveThreads;
-
-      DSPRINT(DSFLAG, "Restored slot ptr at: %016llx \n",
-              (unsigned long long)SlotP);
-      DSPRINT(DSFLAG, "Restored stack ptr at: %016llx \n",
-              (unsigned long long)StackP);
-      DSPRINT(DSFLAG, "Restored frame ptr at: %016llx \n",
-              (unsigned long long)FrameP);
-      DSPRINT(DSFLAG, "Active threads: %08x \n", (unsigned)ActiveT);
-    }
-  }
-
-  // FIXME: Need to see the impact of doing it here.
-  __kmpc_impl_threadfence_block();
-
-  DSPRINT0(DSFLAG, "Exiting __kmpc_data_sharing_environment_end\n");
-  return;
-}
-
-EXTERN void *
-__kmpc_get_data_sharing_environment_frame(int32_t SourceThreadID,
-                                          int16_t IsOMPRuntimeInitialized) {
-  DSPRINT0(DSFLAG, "Entering __kmpc_get_data_sharing_environment_frame\n");
-
-  // If the runtime has been elided, use shared memory for master-worker
-  // data sharing.  We're reusing the statically allocated data structure
-  // that is used for standard data sharing.
-  if (!IsOMPRuntimeInitialized)
-    return (void *)&DataSharingState;
-
-  // Get the frame used by the requested thread.
-
-  unsigned SourceWID = SourceThreadID / WARPSIZE;
-
-  DSPRINT(DSFLAG, "Source  warp: %u\n", SourceWID);
-
-  void * volatile P = DataSharingState.FramePtr[SourceWID];
-  DSPRINT0(DSFLAG, "Exiting __kmpc_get_data_sharing_environment_frame\n");
-  return P;
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 // Runtime functions for trunk data sharing scheme.
 ////////////////////////////////////////////////////////////////////////////////
diff --git a/openmp/libomptarget/deviceRTLs/common/src/libcall.cu b/openmp/libomptarget/deviceRTLs/common/src/libcall.cu
index 89c481bcf8da3..cfa438ff1717b 100644
--- a/openmp/libomptarget/deviceRTLs/common/src/libcall.cu
+++ b/openmp/libomptarget/deviceRTLs/common/src/libcall.cu
@@ -362,53 +362,3 @@ EXTERN int omp_test_lock(omp_lock_t *lock) {
   PRINT(LD_IO, "call omp_test_lock() return %d\n", rc);
   return rc;
 }
-
-// for xlf Fortran
-// Fortran, the return is LOGICAL type
-
-#define FLOGICAL long
-EXTERN FLOGICAL __xlf_omp_is_initial_device_i8() {
-  int ret = omp_is_initial_device();
-  if (ret == 0)
-    return (FLOGICAL)0;
-  else
-    return (FLOGICAL)1;
-}
-
-EXTERN int __xlf_omp_is_initial_device_i4() {
-  int ret = omp_is_initial_device();
-  if (ret == 0)
-    return 0;
-  else
-    return 1;
-}
-
-EXTERN long __xlf_omp_get_team_num_i4() {
-  int ret = omp_get_team_num();
-  return (long)ret;
-}
-
-EXTERN long __xlf_omp_get_num_teams_i4() {
-  int ret = omp_get_num_teams();
-  return (long)ret;
-}
-
-EXTERN void xlf_debug_print_int(int *p) {
-  printf("xlf DEBUG %d): %p %d\n", omp_get_team_num(), p, p == 0 ? 0 : *p);
-}
-
-EXTERN void xlf_debug_print_long(long *p) {
-  printf("xlf DEBUG %d): %p %ld\n", omp_get_team_num(), p, p == 0 ? 0 : *p);
-}
-
-EXTERN void xlf_debug_print_float(float *p) {
-  printf("xlf DEBUG %d): %p %f\n", omp_get_team_num(), p, p == 0 ? 0 : *p);
-}
-
-EXTERN void xlf_debug_print_double(double *p) {
-  printf("xlf DEBUG %d): %p %f\n", omp_get_team_num(), p, p == 0 ? 0 : *p);
-}
-
-EXTERN void xlf_debug_print_addr(void *p) {
-  printf("xlf DEBUG %d): %p \n", omp_get_team_num(), p);
-}
diff --git a/openmp/libomptarget/deviceRTLs/common/src/loop.cu b/openmp/libomptarget/deviceRTLs/common/src/loop.cu
index 417460db138a1..f625d9ea9e235 100644
--- a/openmp/libomptarget/deviceRTLs/common/src/loop.cu
+++ b/openmp/libomptarget/deviceRTLs/common/src/loop.cu
@@ -754,55 +754,3 @@ void __kmpc_for_static_init_8u_simple_generic(
 EXTERN void __kmpc_for_static_fini(kmp_Ident *loc, int32_t global_tid) {
   PRINT0(LD_IO, "call kmpc_for_static_fini\n");
 }
-
-namespace {
-INLINE void syncWorkersInGenericMode(uint32_t NumThreads) {
-  int NumWarps = ((NumThreads + WARPSIZE - 1) / WARPSIZE);
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
-  // On Volta and newer architectures we require that all lanes in
-  // a warp (at least, all present for the kernel launch) participate in the
-  // barrier.  This is enforced when launching the parallel region.  An
-  // exception is when there are < WARPSIZE workers.  In this case only 1 worker
-  // is started, so we don't need a barrier.
-  if (NumThreads > 1) {
-#endif
-    __kmpc_impl_named_sync(L1_BARRIER, WARPSIZE * NumWarps);
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
-  }
-#endif
-}
-}; // namespace
-
-EXTERN void __kmpc_reduce_conditional_lastprivate(kmp_Ident *loc, int32_t gtid,
-                                                  int32_t varNum, void *array) {
-  PRINT0(LD_IO, "call to __kmpc_reduce_conditional_lastprivate(...)\n");
-  ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc),
-          "Expected non-SPMD mode + initialized runtime.");
-
-  omptarget_nvptx_TeamDescr &teamDescr = getMyTeamDescriptor();
-  uint32_t NumThreads = GetNumberOfOmpThreads(checkSPMDMode(loc));
-  uint64_t *Buffer = teamDescr.getLastprivateIterBuffer();
-  for (unsigned i = 0; i < varNum; i++) {
-    // Reset buffer.
-    if (gtid == 0)
-      *Buffer = 0; // Reset to minimum loop iteration value.
-
-    // Barrier.
-    syncWorkersInGenericMode(NumThreads);
-
-    // Atomic max of iterations.
-    uint64_t *varArray = (uint64_t *)array;
-    uint64_t elem = varArray[i];
-    (void)__kmpc_atomic_max((unsigned long long int *)Buffer,
-                            (unsigned long long int)elem);
-
-    // Barrier.
-    syncWorkersInGenericMode(NumThreads);
-
-    // Read max value and update thread private array.
-    varArray[i] = *Buffer;
-
-    // Barrier.
-    syncWorkersInGenericMode(NumThreads);
-  }
-}
diff --git a/openmp/libomptarget/deviceRTLs/common/src/omptarget.cu b/openmp/libomptarget/deviceRTLs/common/src/omptarget.cu
index 23fbd00cacaf9..6c1d5319595c5 100644
--- a/openmp/libomptarget/deviceRTLs/common/src/omptarget.cu
+++ b/openmp/libomptarget/deviceRTLs/common/src/omptarget.cu
@@ -25,13 +25,6 @@ extern DEVICE
 // init entry points
 ////////////////////////////////////////////////////////////////////////////////
 
-EXTERN void __kmpc_kernel_init_params(void *Ptr) {
-  PRINT(LD_IO, "call to __kmpc_kernel_init_params with version %f\n",
-        OMPTARGET_NVPTX_VERSION);
-
-  SetTeamsReductionScratchpadPtr(Ptr);
-}
-
 EXTERN void __kmpc_kernel_init(int ThreadLimit, int16_t RequiresOMPRuntime) {
   PRINT(LD_IO, "call to __kmpc_kernel_init with version %f\n",
         OMPTARGET_NVPTX_VERSION);
@@ -152,10 +145,6 @@ EXTERN void __kmpc_spmd_kernel_init(int ThreadLimit, int16_t RequiresOMPRuntime,
   }
 }
 
-EXTERN __attribute__((deprecated)) void __kmpc_spmd_kernel_deinit() {
-  __kmpc_spmd_kernel_deinit_v2(isRuntimeInitialized());
-}
-
 EXTERN void __kmpc_spmd_kernel_deinit_v2(int16_t RequiresOMPRuntime) {
   // We're not going to pop the task descr stack of each thread since
   // there are no more parallel regions in SPMD mode.
diff --git a/openmp/libomptarget/deviceRTLs/common/src/parallel.cu b/openmp/libomptarget/deviceRTLs/common/src/parallel.cu
index ab031e99e51f9..20b03e9bab1b4 100644
--- a/openmp/libomptarget/deviceRTLs/common/src/parallel.cu
+++ b/openmp/libomptarget/deviceRTLs/common/src/parallel.cu
@@ -35,161 +35,6 @@
 #include "common/omptarget.h"
 #include "target_impl.h"
 
-typedef struct ConvergentSimdJob {
-  omptarget_nvptx_TaskDescr taskDescr;
-  omptarget_nvptx_TaskDescr *convHeadTaskDescr;
-  uint16_t slimForNextSimd;
-} ConvergentSimdJob;
-
-////////////////////////////////////////////////////////////////////////////////
-// support for convergent simd (team of threads in a warp only)
-////////////////////////////////////////////////////////////////////////////////
-EXTERN bool __kmpc_kernel_convergent_simd(void *buffer,
-                                          __kmpc_impl_lanemask_t Mask,
-                                          bool *IsFinal, int32_t *LaneSource,
-                                          int32_t *LaneId, int32_t *NumLanes) {
-  PRINT0(LD_IO, "call to __kmpc_kernel_convergent_simd\n");
-  __kmpc_impl_lanemask_t ConvergentMask = Mask;
-  int32_t ConvergentSize = __kmpc_impl_popc(ConvergentMask);
-  __kmpc_impl_lanemask_t WorkRemaining = ConvergentMask >> (*LaneSource + 1);
-  *LaneSource += __kmpc_impl_ffs(WorkRemaining);
-  *IsFinal = __kmpc_impl_popc(WorkRemaining) == 1;
-  __kmpc_impl_lanemask_t lanemask_lt = __kmpc_impl_lanemask_lt();
-  *LaneId = __kmpc_impl_popc(ConvergentMask & lanemask_lt);
-
-  int threadId = GetLogicalThreadIdInBlock(isSPMDMode());
-  int sourceThreadId = (threadId & ~(WARPSIZE - 1)) + *LaneSource;
-
-  ConvergentSimdJob *job = (ConvergentSimdJob *)buffer;
-  int32_t SimdLimit =
-      omptarget_nvptx_threadPrivateContext->SimdLimitForNextSimd(threadId);
-  job->slimForNextSimd = SimdLimit;
-
-  int32_t SimdLimitSource = __kmpc_impl_shfl_sync(Mask, SimdLimit, *LaneSource);
-  // reset simdlimit to avoid propagating to successive #simd
-  if (SimdLimitSource > 0 && threadId == sourceThreadId)
-    omptarget_nvptx_threadPrivateContext->SimdLimitForNextSimd(threadId) = 0;
-
-  // We cannot have more than the # of convergent threads.
-  if (SimdLimitSource > 0)
-    *NumLanes = __kmpc_impl_min(ConvergentSize, SimdLimitSource);
-  else
-    *NumLanes = ConvergentSize;
-  ASSERT(LT_FUSSY, *NumLanes > 0, "bad thread request of %d threads",
-         (int)*NumLanes);
-
-  // Set to true for lanes participating in the simd region.
-  bool isActive = false;
-  // Initialize state for active threads.
-  if (*LaneId < *NumLanes) {
-    omptarget_nvptx_TaskDescr *currTaskDescr =
-        omptarget_nvptx_threadPrivateContext->GetTopLevelTaskDescr(threadId);
-    omptarget_nvptx_TaskDescr *sourceTaskDescr =
-        omptarget_nvptx_threadPrivateContext->GetTopLevelTaskDescr(
-            sourceThreadId);
-    job->convHeadTaskDescr = currTaskDescr;
-    // install top descriptor from the thread for which the lanes are working.
-    omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(threadId,
-                                                               sourceTaskDescr);
-    isActive = true;
-  }
-
-  // requires a memory fence between threads of a warp
-  return isActive;
-}
-
-EXTERN void __kmpc_kernel_end_convergent_simd(void *buffer) {
-  PRINT0(LD_IO | LD_PAR, "call to __kmpc_kernel_end_convergent_parallel\n");
-  // pop stack
-  int threadId = GetLogicalThreadIdInBlock(isSPMDMode());
-  ConvergentSimdJob *job = (ConvergentSimdJob *)buffer;
-  omptarget_nvptx_threadPrivateContext->SimdLimitForNextSimd(threadId) =
-      job->slimForNextSimd;
-  omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(
-      threadId, job->convHeadTaskDescr);
-}
-
-typedef struct ConvergentParallelJob {
-  omptarget_nvptx_TaskDescr taskDescr;
-  omptarget_nvptx_TaskDescr *convHeadTaskDescr;
-  uint16_t tnumForNextPar;
-} ConvergentParallelJob;
-
-////////////////////////////////////////////////////////////////////////////////
-// support for convergent parallelism (team of threads in a warp only)
-////////////////////////////////////////////////////////////////////////////////
-EXTERN bool __kmpc_kernel_convergent_parallel(void *buffer,
-                                              __kmpc_impl_lanemask_t Mask,
-                                              bool *IsFinal,
-                                              int32_t *LaneSource) {
-  PRINT0(LD_IO, "call to __kmpc_kernel_convergent_parallel\n");
-  __kmpc_impl_lanemask_t ConvergentMask = Mask;
-  int32_t ConvergentSize = __kmpc_impl_popc(ConvergentMask);
-  __kmpc_impl_lanemask_t WorkRemaining = ConvergentMask >> (*LaneSource + 1);
-  *LaneSource += __kmpc_impl_ffs(WorkRemaining);
-  *IsFinal = __kmpc_impl_popc(WorkRemaining) == 1;
-  __kmpc_impl_lanemask_t lanemask_lt = __kmpc_impl_lanemask_lt();
-  uint32_t OmpId = __kmpc_impl_popc(ConvergentMask & lanemask_lt);
-
-  int threadId = GetLogicalThreadIdInBlock(isSPMDMode());
-  int sourceThreadId = (threadId & ~(WARPSIZE - 1)) + *LaneSource;
-
-  ConvergentParallelJob *job = (ConvergentParallelJob *)buffer;
-  int32_t NumThreadsClause =
-      omptarget_nvptx_threadPrivateContext->NumThreadsForNextParallel(threadId);
-  job->tnumForNextPar = NumThreadsClause;
-
-  int32_t NumThreadsSource =
-      __kmpc_impl_shfl_sync(Mask, NumThreadsClause, *LaneSource);
-  // reset numthreads to avoid propagating to successive #parallel
-  if (NumThreadsSource > 0 && threadId == sourceThreadId)
-    omptarget_nvptx_threadPrivateContext->NumThreadsForNextParallel(threadId) =
-        0;
-
-  // We cannot have more than the # of convergent threads.
-  uint16_t NumThreads;
-  if (NumThreadsSource > 0)
-    NumThreads = __kmpc_impl_min(ConvergentSize, NumThreadsSource);
-  else
-    NumThreads = ConvergentSize;
-  ASSERT(LT_FUSSY, NumThreads > 0, "bad thread request of %d threads",
-         (int)NumThreads);
-
-  // Set to true for workers participating in the parallel region.
-  bool isActive = false;
-  // Initialize state for active threads.
-  if (OmpId < NumThreads) {
-    // init L2 task descriptor and storage for the L1 parallel task descriptor.
-    omptarget_nvptx_TaskDescr *newTaskDescr = &job->taskDescr;
-    ASSERT0(LT_FUSSY, newTaskDescr, "expected a task descr");
-    omptarget_nvptx_TaskDescr *currTaskDescr =
-        omptarget_nvptx_threadPrivateContext->GetTopLevelTaskDescr(threadId);
-    omptarget_nvptx_TaskDescr *sourceTaskDescr =
-        omptarget_nvptx_threadPrivateContext->GetTopLevelTaskDescr(
-            sourceThreadId);
-    job->convHeadTaskDescr = currTaskDescr;
-    newTaskDescr->CopyConvergentParent(sourceTaskDescr, OmpId, NumThreads);
-    // install new top descriptor
-    omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(threadId,
-                                                               newTaskDescr);
-    isActive = true;
-  }
-
-  // requires a memory fence between threads of a warp
-  return isActive;
-}
-
-EXTERN void __kmpc_kernel_end_convergent_parallel(void *buffer) {
-  PRINT0(LD_IO | LD_PAR, "call to __kmpc_kernel_end_convergent_parallel\n");
-  // pop stack
-  int threadId = GetLogicalThreadIdInBlock(isSPMDMode());
-  ConvergentParallelJob *job = (ConvergentParallelJob *)buffer;
-  omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(
-      threadId, job->convHeadTaskDescr);
-  omptarget_nvptx_threadPrivateContext->NumThreadsForNextParallel(threadId) =
-      job->tnumForNextPar;
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 // support for parallel that goes parallel (1 static level only)
 ////////////////////////////////////////////////////////////////////////////////
@@ -227,10 +72,8 @@ INLINE static uint16_t determineNumberOfThreads(uint16_t NumThreadsClause,
 }
 
 // This routine is always called by the team master..
-EXTERN void __kmpc_kernel_prepare_parallel(void *WorkFn,
-                                           int16_t IsOMPRuntimeInitialized) {
+EXTERN void __kmpc_kernel_prepare_parallel(void *WorkFn) {
   PRINT0(LD_IO, "call to __kmpc_kernel_prepare_parallel\n");
-  ASSERT0(LT_FUSSY, IsOMPRuntimeInitialized, "Expected initialized runtime.");
 
   omptarget_nvptx_workFn = WorkFn;
 
@@ -275,12 +118,9 @@ EXTERN void __kmpc_kernel_prepare_parallel(void *WorkFn,
 // returns True if this thread is active, else False.
 //
 // Only the worker threads call this routine.
-EXTERN bool __kmpc_kernel_parallel(void **WorkFn,
-                                   int16_t IsOMPRuntimeInitialized) {
+EXTERN bool __kmpc_kernel_parallel(void **WorkFn) {
   PRINT0(LD_IO | LD_PAR, "call to __kmpc_kernel_parallel\n");
 
-  ASSERT0(LT_FUSSY, IsOMPRuntimeInitialized, "Expected initialized runtime.");
-
   // Work function and arguments for L1 parallel region.
   *WorkFn = omptarget_nvptx_workFn;
 
@@ -446,14 +286,6 @@ EXTERN void __kmpc_push_num_threads(kmp_Ident *loc, int32_t tid,
       num_threads;
 }
 
-EXTERN void __kmpc_push_simd_limit(kmp_Ident *loc, int32_t tid,
-                                   int32_t simd_limit) {
-  PRINT(LD_IO, "call kmpc_push_simd_limit %d\n", (int)simd_limit);
-  ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc), "Runtime must be initialized.");
-  tid = GetLogicalThreadIdInBlock(checkSPMDMode(loc));
-  omptarget_nvptx_threadPrivateContext->SimdLimitForNextSimd(tid) = simd_limit;
-}
-
 // Do nothing. The host guarantees we started the requested number of
 // teams and we only need inspection of gridDim.
 
diff --git a/openmp/libomptarget/deviceRTLs/common/src/reduction.cu b/openmp/libomptarget/deviceRTLs/common/src/reduction.cu
index 427c90a7e0913..7604f024eeb4d 100644
--- a/openmp/libomptarget/deviceRTLs/common/src/reduction.cu
+++ b/openmp/libomptarget/deviceRTLs/common/src/reduction.cu
@@ -54,6 +54,7 @@ INLINE static void gpu_irregular_warp_reduce(void *reduce_data,
   }
 }
 
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 700
 INLINE static uint32_t
 gpu_irregular_simd_reduce(void *reduce_data, kmp_ShuffleReductFctPtr shflFct) {
   uint32_t size, remote_id, physical_lane_id;
@@ -72,22 +73,7 @@ gpu_irregular_simd_reduce(void *reduce_data, kmp_ShuffleReductFctPtr shflFct) {
   } while (logical_lane_id % 2 == 0 && size > 1);
   return (logical_lane_id == 0);
 }
-
-EXTERN
-int32_t __kmpc_nvptx_simd_reduce_nowait(int32_t global_tid, int32_t num_vars,
-                                        size_t reduce_size, void *reduce_data,
-                                        kmp_ShuffleReductFctPtr shflFct,
-                                        kmp_InterWarpCopyFctPtr cpyFct) {
-  __kmpc_impl_lanemask_t Liveness = __kmpc_impl_activemask();
-  if (Liveness == __kmpc_impl_all_lanes) {
-    gpu_regular_warp_reduce(reduce_data, shflFct);
-    return GetThreadIdInBlock() % WARPSIZE ==
-           0; // Result on lane 0 of the simd warp.
-  } else {
-    return gpu_irregular_simd_reduce(
-        reduce_data, shflFct); // Result on the first active lane.
-  }
-}
+#endif
 
 INLINE
 static int32_t nvptx_parallel_reduce_nowait(
@@ -177,14 +163,6 @@ static int32_t nvptx_parallel_reduce_nowait(
 #endif // __CUDA_ARCH__ >= 700
 }
 
-EXTERN __attribute__((deprecated)) int32_t __kmpc_nvptx_parallel_reduce_nowait(
-    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
-    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct) {
-  return nvptx_parallel_reduce_nowait(global_tid, num_vars, reduce_size,
-                                      reduce_data, shflFct, cpyFct,
-                                      isSPMDMode(), isRuntimeUninitialized());
-}
-
 EXTERN
 int32_t __kmpc_nvptx_parallel_reduce_nowait_v2(
     kmp_Ident *loc, int32_t global_tid, int32_t num_vars, size_t reduce_size,
@@ -195,201 +173,6 @@ int32_t __kmpc_nvptx_parallel_reduce_nowait_v2(
       checkSPMDMode(loc), checkRuntimeUninitialized(loc));
 }
 
-EXTERN
-int32_t __kmpc_nvptx_parallel_reduce_nowait_simple_spmd(
-    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
-    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct) {
-  return nvptx_parallel_reduce_nowait(
-      global_tid, num_vars, reduce_size, reduce_data, shflFct, cpyFct,
-      /*isSPMDExecutionMode=*/true, /*isRuntimeUninitialized=*/true);
-}
-
-EXTERN
-int32_t __kmpc_nvptx_parallel_reduce_nowait_simple_generic(
-    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
-    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct) {
-  return nvptx_parallel_reduce_nowait(
-      global_tid, num_vars, reduce_size, reduce_data, shflFct, cpyFct,
-      /*isSPMDExecutionMode=*/false, /*isRuntimeUninitialized=*/true);
-}
-
-INLINE
-static int32_t nvptx_teams_reduce_nowait(int32_t global_tid, int32_t num_vars,
-                                         size_t reduce_size, void *reduce_data,
-                                         kmp_ShuffleReductFctPtr shflFct,
-                                         kmp_InterWarpCopyFctPtr cpyFct,
-                                         kmp_CopyToScratchpadFctPtr scratchFct,
-                                         kmp_LoadReduceFctPtr ldFct,
-                                         bool isSPMDExecutionMode) {
-  uint32_t ThreadId = GetLogicalThreadIdInBlock(isSPMDExecutionMode);
-  // In non-generic mode all workers participate in the teams reduction.
-  // In generic mode only the team master participates in the teams
-  // reduction because the workers are waiting for parallel work.
-  uint32_t NumThreads =
-      isSPMDExecutionMode ? GetNumberOfOmpThreads(/*isSPMDExecutionMode=*/true)
-                          : /*Master thread only*/ 1;
-  uint32_t TeamId = GetBlockIdInKernel();
-  uint32_t NumTeams = GetNumberOfBlocksInKernel();
-  static SHARED volatile bool IsLastTeam;
-
-  // Team masters of all teams write to the scratchpad.
-  if (ThreadId == 0) {
-    unsigned int *timestamp = GetTeamsReductionTimestamp();
-    char *scratchpad = GetTeamsReductionScratchpad();
-
-    scratchFct(reduce_data, scratchpad, TeamId, NumTeams);
-    __kmpc_impl_threadfence();
-
-    // atomicInc increments 'timestamp' and has a range [0, NumTeams-1].
-    // It resets 'timestamp' back to 0 once the last team increments
-    // this counter.
-    unsigned val = __kmpc_atomic_inc(timestamp, NumTeams - 1);
-    IsLastTeam = val == NumTeams - 1;
-  }
-
-  // We have to wait on L1 barrier because in GENERIC mode the workers
-  // are waiting on barrier 0 for work.
-  //
-  // If we guard this barrier as follows it leads to deadlock, probably
-  // because of a compiler bug: if (!IsGenericMode()) __syncthreads();
-  uint16_t SyncWarps = (NumThreads + WARPSIZE - 1) / WARPSIZE;
-  __kmpc_impl_named_sync(L1_BARRIER, SyncWarps * WARPSIZE);
-
-  // If this team is not the last, quit.
-  if (/* Volatile read by all threads */ !IsLastTeam)
-    return 0;
-
-    //
-    // Last team processing.
-    //
-
-    // Threads in excess of #teams do not participate in reduction of the
-    // scratchpad values.
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
-  uint32_t ActiveThreads = NumThreads;
-  if (NumTeams < NumThreads) {
-    ActiveThreads =
-        (NumTeams < WARPSIZE) ? 1 : NumTeams & ~((uint16_t)WARPSIZE - 1);
-  }
-  if (ThreadId >= ActiveThreads)
-    return 0;
-
-  // Load from scratchpad and reduce.
-  char *scratchpad = GetTeamsReductionScratchpad();
-  ldFct(reduce_data, scratchpad, ThreadId, NumTeams, /*Load only*/ 0);
-  for (uint32_t i = ActiveThreads + ThreadId; i < NumTeams; i += ActiveThreads)
-    ldFct(reduce_data, scratchpad, i, NumTeams, /*Load and reduce*/ 1);
-
-  uint32_t WarpsNeeded = (ActiveThreads + WARPSIZE - 1) / WARPSIZE;
-  uint32_t WarpId = ThreadId / WARPSIZE;
-
-  // Reduce across warps to the warp master.
-  if ((ActiveThreads % WARPSIZE == 0) ||
-      (WarpId < WarpsNeeded - 1)) // Full warp
-    gpu_regular_warp_reduce(reduce_data, shflFct);
-  else if (ActiveThreads > 1) // Partial warp but contiguous lanes
-    // Only SPMD execution mode comes thru this case.
-    gpu_irregular_warp_reduce(reduce_data, shflFct,
-                              /*LaneCount=*/ActiveThreads % WARPSIZE,
-                              /*LaneId=*/ThreadId % WARPSIZE);
-
-  // When we have more than [warpsize] number of threads
-  // a block reduction is performed here.
-  if (ActiveThreads > WARPSIZE) {
-    // Gather all the reduced values from each warp
-    // to the first warp.
-    cpyFct(reduce_data, WarpsNeeded);
-
-    if (WarpId == 0)
-      gpu_irregular_warp_reduce(reduce_data, shflFct, WarpsNeeded, ThreadId);
-  }
-#else
-  if (ThreadId >= NumTeams)
-    return 0;
-
-  // Load from scratchpad and reduce.
-  char *scratchpad = GetTeamsReductionScratchpad();
-  ldFct(reduce_data, scratchpad, ThreadId, NumTeams, /*Load only*/ 0);
-  for (uint32_t i = NumThreads + ThreadId; i < NumTeams; i += NumThreads)
-    ldFct(reduce_data, scratchpad, i, NumTeams, /*Load and reduce*/ 1);
-
-  // Reduce across warps to the warp master.
-  __kmpc_impl_lanemask_t Liveness = __kmpc_impl_activemask();
-  if (Liveness == __kmpc_impl_all_lanes) // Full warp
-    gpu_regular_warp_reduce(reduce_data, shflFct);
-  else // Partial warp but contiguous lanes
-    gpu_irregular_warp_reduce(reduce_data, shflFct,
-                              /*LaneCount=*/__kmpc_impl_popc(Liveness),
-                              /*LaneId=*/ThreadId % WARPSIZE);
-
-  // When we have more than [warpsize] number of threads
-  // a block reduction is performed here.
-  uint32_t ActiveThreads = NumTeams < NumThreads ? NumTeams : NumThreads;
-  if (ActiveThreads > WARPSIZE) {
-    uint32_t WarpsNeeded = (ActiveThreads + WARPSIZE - 1) / WARPSIZE;
-    // Gather all the reduced values from each warp
-    // to the first warp.
-    cpyFct(reduce_data, WarpsNeeded);
-
-    uint32_t WarpId = ThreadId / WARPSIZE;
-    if (WarpId == 0)
-      gpu_irregular_warp_reduce(reduce_data, shflFct, WarpsNeeded, ThreadId);
-  }
-#endif // __CUDA_ARCH__ >= 700
-
-  return ThreadId == 0;
-}
-
-EXTERN
-int32_t __kmpc_nvptx_teams_reduce_nowait(int32_t global_tid, int32_t num_vars,
-                                         size_t reduce_size, void *reduce_data,
-                                         kmp_ShuffleReductFctPtr shflFct,
-                                         kmp_InterWarpCopyFctPtr cpyFct,
-                                         kmp_CopyToScratchpadFctPtr scratchFct,
-                                         kmp_LoadReduceFctPtr ldFct) {
-  return nvptx_teams_reduce_nowait(global_tid, num_vars, reduce_size,
-                                   reduce_data, shflFct, cpyFct, scratchFct,
-                                   ldFct, isSPMDMode());
-}
-
-EXTERN
-int32_t __kmpc_nvptx_teams_reduce_nowait_simple_spmd(
-    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
-    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct,
-    kmp_CopyToScratchpadFctPtr scratchFct, kmp_LoadReduceFctPtr ldFct) {
-  return nvptx_teams_reduce_nowait(global_tid, num_vars, reduce_size,
-                                   reduce_data, shflFct, cpyFct, scratchFct,
-                                   ldFct, /*isSPMDExecutionMode=*/true);
-}
-
-EXTERN
-int32_t __kmpc_nvptx_teams_reduce_nowait_simple_generic(
-    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
-    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct,
-    kmp_CopyToScratchpadFctPtr scratchFct, kmp_LoadReduceFctPtr ldFct) {
-  return nvptx_teams_reduce_nowait(global_tid, num_vars, reduce_size,
-                                   reduce_data, shflFct, cpyFct, scratchFct,
-                                   ldFct, /*isSPMDExecutionMode=*/false);
-}
-
-EXTERN int32_t __kmpc_nvptx_teams_reduce_nowait_simple(kmp_Ident *loc,
-                                                       int32_t global_tid,
-                                                       kmp_CriticalName *crit) {
-  if (checkSPMDMode(loc) && GetThreadIdInBlock() != 0)
-    return 0;
-  // The master thread of the team actually does the reduction.
-  while (__kmpc_atomic_cas((uint32_t *)crit, 0u, 1u))
-    ;
-  return 1;
-}
-
-EXTERN void
-__kmpc_nvptx_teams_end_reduce_nowait_simple(kmp_Ident *loc, int32_t global_tid,
-                                            kmp_CriticalName *crit) {
-  __kmpc_impl_threadfence_system();
-  (void)__kmpc_atomic_exchange((uint32_t *)crit, 0u);
-}
-
 INLINE static bool isMaster(kmp_Ident *loc, uint32_t ThreadId) {
   return checkGenericMode(loc) || IsTeamMaster(ThreadId);
 }
diff --git a/openmp/libomptarget/deviceRTLs/common/src/support.cu b/openmp/libomptarget/deviceRTLs/common/src/support.cu
index 85747511d46c1..e02c533e78b21 100644
--- a/openmp/libomptarget/deviceRTLs/common/src/support.cu
+++ b/openmp/libomptarget/deviceRTLs/common/src/support.cu
@@ -264,6 +264,3 @@ DEVICE char *GetTeamsReductionScratchpad() {
   return static_cast<char *>(ReductionScratchpadPtr) + 256;
 }
 
-DEVICE void SetTeamsReductionScratchpadPtr(void *ScratchpadPtr) {
-  ReductionScratchpadPtr = ScratchpadPtr;
-}
diff --git a/openmp/libomptarget/deviceRTLs/common/src/sync.cu b/openmp/libomptarget/deviceRTLs/common/src/sync.cu
index 2ac3e3f9c7c0a..3979e2054fc9e 100644
--- a/openmp/libomptarget/deviceRTLs/common/src/sync.cu
+++ b/openmp/libomptarget/deviceRTLs/common/src/sync.cu
@@ -79,23 +79,6 @@ EXTERN void __kmpc_barrier_simple_spmd(kmp_Ident *loc_ref, int32_t tid) {
   PRINT0(LD_SYNC, "completed kmpc_barrier_simple_spmd\n");
 }
 
-// Emit a simple barrier call in Generic mode.  Assumes the caller is in an L0
-// parallel region and that all worker threads participate.
-EXTERN void __kmpc_barrier_simple_generic(kmp_Ident *loc_ref, int32_t tid) {
-  int numberOfActiveOMPThreads = GetNumberOfThreadsInBlock() - WARPSIZE;
-  // The #threads parameter must be rounded up to the WARPSIZE.
-  int threads =
-      WARPSIZE * ((numberOfActiveOMPThreads + WARPSIZE - 1) / WARPSIZE);
-
-  PRINT(LD_SYNC,
-        "call kmpc_barrier_simple_generic with %d omp threads, sync parameter "
-        "%d\n",
-        (int)numberOfActiveOMPThreads, (int)threads);
-  // Barrier #1 is for synchronization among active threads.
-  __kmpc_impl_named_sync(L1_BARRIER, threads);
-  PRINT0(LD_SYNC, "completed kmpc_barrier_simple_generic\n");
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 // KMP MASTER
 ////////////////////////////////////////////////////////////////////////////////
diff --git a/openmp/libomptarget/deviceRTLs/common/support.h b/openmp/libomptarget/deviceRTLs/common/support.h
index 913c4c3c323fc..a464328257829 100644
--- a/openmp/libomptarget/deviceRTLs/common/support.h
+++ b/openmp/libomptarget/deviceRTLs/common/support.h
@@ -94,6 +94,5 @@ DEVICE unsigned long PadBytes(unsigned long size, unsigned long alignment);
 ////////////////////////////////////////////////////////////////////////////////
 DEVICE unsigned int *GetTeamsReductionTimestamp();
 DEVICE char *GetTeamsReductionScratchpad();
-DEVICE void SetTeamsReductionScratchpadPtr(void *ScratchpadPtr);
 
 #endif
diff --git a/openmp/libomptarget/deviceRTLs/interface.h b/openmp/libomptarget/deviceRTLs/interface.h
index 3c216a5e61c5e..4d352bc648fa3 100644
--- a/openmp/libomptarget/deviceRTLs/interface.h
+++ b/openmp/libomptarget/deviceRTLs/interface.h
@@ -193,17 +193,10 @@ typedef struct ident {
 
 // parallel defs
 typedef ident_t kmp_Ident;
-typedef void (*kmp_ParFctPtr)(int32_t *global_tid, int32_t *bound_tid, ...);
-typedef void (*kmp_ReductFctPtr)(void *lhsData, void *rhsData);
 typedef void (*kmp_InterWarpCopyFctPtr)(void *src, int32_t warp_num);
 typedef void (*kmp_ShuffleReductFctPtr)(void *rhsData, int16_t lane_id,
                                         int16_t lane_offset,
                                         int16_t shortCircuit);
-typedef void (*kmp_CopyToScratchpadFctPtr)(void *reduceData, void *scratchpad,
-                                           int32_t index, int32_t width);
-typedef void (*kmp_LoadReduceFctPtr)(void *reduceData, void *scratchpad,
-                                     int32_t index, int32_t width,
-                                     int32_t reduce);
 typedef void (*kmp_ListGlobalFctPtr)(void *buffer, int idx, void *reduce_data);
 
 // task defs
@@ -227,12 +220,6 @@ typedef int32_t kmp_CriticalName[8];
 EXTERN int32_t __kmpc_global_thread_num(kmp_Ident *loc);
 EXTERN void __kmpc_push_num_threads(kmp_Ident *loc, int32_t global_tid,
                                     int32_t num_threads);
-// simd
-EXTERN void __kmpc_push_simd_limit(kmp_Ident *loc, int32_t global_tid,
-                                   int32_t simd_limit);
-// aee ... not supported
-// EXTERN void __kmpc_fork_call(kmp_Ident *loc, int32_t argc, kmp_ParFctPtr
-// microtask, ...);
 EXTERN void __kmpc_serialized_parallel(kmp_Ident *loc, uint32_t global_tid);
 EXTERN void __kmpc_end_serialized_parallel(kmp_Ident *loc,
                                            uint32_t global_tid);
@@ -354,61 +341,25 @@ EXTERN void __kmpc_dispatch_fini_4u(kmp_Ident *loc, int32_t global_tid);
 EXTERN void __kmpc_dispatch_fini_8(kmp_Ident *loc, int32_t global_tid);
 EXTERN void __kmpc_dispatch_fini_8u(kmp_Ident *loc, int32_t global_tid);
 
-// Support for reducing conditional lastprivate variables
-EXTERN void __kmpc_reduce_conditional_lastprivate(kmp_Ident *loc,
-                                                  int32_t global_tid,
-                                                  int32_t varNum, void *array);
-
 // reduction
 EXTERN void __kmpc_nvptx_end_reduce(int32_t global_tid);
 EXTERN void __kmpc_nvptx_end_reduce_nowait(int32_t global_tid);
-EXTERN __attribute__((deprecated)) int32_t __kmpc_nvptx_parallel_reduce_nowait(
-    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
-    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct);
 EXTERN int32_t __kmpc_nvptx_parallel_reduce_nowait_v2(
     kmp_Ident *loc, int32_t global_tid, int32_t num_vars, size_t reduce_size,
     void *reduce_data, kmp_ShuffleReductFctPtr shflFct,
     kmp_InterWarpCopyFctPtr cpyFct);
-EXTERN int32_t __kmpc_nvptx_parallel_reduce_nowait_simple_spmd(
-    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
-    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct);
-EXTERN int32_t __kmpc_nvptx_parallel_reduce_nowait_simple_generic(
-    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
-    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct);
-EXTERN int32_t __kmpc_nvptx_simd_reduce_nowait(
-    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
-    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct);
 EXTERN int32_t __kmpc_nvptx_teams_reduce_nowait_v2(
     kmp_Ident *loc, int32_t global_tid, void *global_buffer,
     int32_t num_of_records, void *reduce_data, kmp_ShuffleReductFctPtr shflFct,
     kmp_InterWarpCopyFctPtr cpyFct, kmp_ListGlobalFctPtr lgcpyFct,
     kmp_ListGlobalFctPtr lgredFct, kmp_ListGlobalFctPtr glcpyFct,
     kmp_ListGlobalFctPtr glredFct);
-EXTERN int32_t __kmpc_nvptx_teams_reduce_nowait(
-    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
-    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct,
-    kmp_CopyToScratchpadFctPtr sratchFct, kmp_LoadReduceFctPtr ldFct);
-EXTERN int32_t __kmpc_nvptx_teams_reduce_nowait_simple_spmd(
-    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
-    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct,
-    kmp_CopyToScratchpadFctPtr sratchFct, kmp_LoadReduceFctPtr ldFct);
-EXTERN int32_t __kmpc_nvptx_teams_reduce_nowait_simple_generic(
-    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
-    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct,
-    kmp_CopyToScratchpadFctPtr sratchFct, kmp_LoadReduceFctPtr ldFct);
-EXTERN int32_t __kmpc_nvptx_teams_reduce_nowait_simple(kmp_Ident *loc,
-                                                       int32_t global_tid,
-                                                       kmp_CriticalName *crit);
-EXTERN void __kmpc_nvptx_teams_end_reduce_nowait_simple(kmp_Ident *loc,
-                                                        int32_t global_tid,
-                                                        kmp_CriticalName *crit);
 EXTERN int32_t __kmpc_shuffle_int32(int32_t val, int16_t delta, int16_t size);
 EXTERN int64_t __kmpc_shuffle_int64(int64_t val, int16_t delta, int16_t size);
 
 // sync barrier
 EXTERN void __kmpc_barrier(kmp_Ident *loc_ref, int32_t tid);
 EXTERN void __kmpc_barrier_simple_spmd(kmp_Ident *loc_ref, int32_t tid);
-EXTERN void __kmpc_barrier_simple_generic(kmp_Ident *loc_ref, int32_t tid);
 EXTERN int32_t __kmpc_cancel_barrier(kmp_Ident *loc, int32_t global_tid);
 
 // single
@@ -468,29 +419,14 @@ EXTERN int32_t __kmpc_cancel(kmp_Ident *loc, int32_t global_tid,
                              int32_t cancelVal);
 
 // non standard
-EXTERN void __kmpc_kernel_init_params(void *ReductionScratchpadPtr);
 EXTERN void __kmpc_kernel_init(int ThreadLimit, int16_t RequiresOMPRuntime);
 EXTERN void __kmpc_kernel_deinit(int16_t IsOMPRuntimeInitialized);
 EXTERN void __kmpc_spmd_kernel_init(int ThreadLimit, int16_t RequiresOMPRuntime,
                                     int16_t RequiresDataSharing);
-EXTERN __attribute__((deprecated)) void __kmpc_spmd_kernel_deinit();
 EXTERN void __kmpc_spmd_kernel_deinit_v2(int16_t RequiresOMPRuntime);
-EXTERN void __kmpc_kernel_prepare_parallel(void *WorkFn,
-                                           int16_t IsOMPRuntimeInitialized);
-EXTERN bool __kmpc_kernel_parallel(void **WorkFn,
-                                   int16_t IsOMPRuntimeInitialized);
+EXTERN void __kmpc_kernel_prepare_parallel(void *WorkFn);
+EXTERN bool __kmpc_kernel_parallel(void **WorkFn);
 EXTERN void __kmpc_kernel_end_parallel();
-EXTERN bool __kmpc_kernel_convergent_parallel(void *buffer,
-                                              __kmpc_impl_lanemask_t Mask,
-                                              bool *IsFinal,
-                                              int32_t *LaneSource);
-EXTERN void __kmpc_kernel_end_convergent_parallel(void *buffer);
-EXTERN bool __kmpc_kernel_convergent_simd(void *buffer,
-                                          __kmpc_impl_lanemask_t Mask,
-                                          bool *IsFinal, int32_t *LaneSource,
-                                          int32_t *LaneId, int32_t *NumLanes);
-EXTERN void __kmpc_kernel_end_convergent_simd(void *buffer);
-
 
 EXTERN void __kmpc_data_sharing_init_stack();
 EXTERN void __kmpc_data_sharing_init_stack_spmd();
@@ -512,22 +448,6 @@ struct __kmpc_data_sharing_slot {
   void *DataEnd;
   char Data[];
 };
-EXTERN void
-__kmpc_initialize_data_sharing_environment(__kmpc_data_sharing_slot *RootS,
-                                           size_t InitialDataSize);
-EXTERN void *__kmpc_data_sharing_environment_begin(
-    __kmpc_data_sharing_slot **SavedSharedSlot, void **SavedSharedStack,
-    void **SavedSharedFrame, __kmpc_impl_lanemask_t *SavedActiveThreads,
-    size_t SharingDataSize, size_t SharingDefaultDataSize,
-    int16_t IsOMPRuntimeInitialized);
-EXTERN void __kmpc_data_sharing_environment_end(
-    __kmpc_data_sharing_slot **SavedSharedSlot, void **SavedSharedStack,
-    void **SavedSharedFrame, __kmpc_impl_lanemask_t *SavedActiveThreads,
-    int32_t IsEntryPoint);
-
-EXTERN void *
-__kmpc_get_data_sharing_environment_frame(int32_t SourceThreadID,
-                                          int16_t IsOMPRuntimeInitialized);
 
 // SPMD execution mode interrogation function.
 EXTERN int8_t __kmpc_is_spmd_exec_mode();
diff --git a/openmp/libomptarget/deviceRTLs/nvptx/src/target_impl.h b/openmp/libomptarget/deviceRTLs/nvptx/src/target_impl.h
index 1b966510ec7ef..d009e36a522f6 100644
--- a/openmp/libomptarget/deviceRTLs/nvptx/src/target_impl.h
+++ b/openmp/libomptarget/deviceRTLs/nvptx/src/target_impl.h
@@ -195,15 +195,6 @@ INLINE int GetNumberOfThreadsInBlock() { return blockDim.x; }
 INLINE unsigned GetWarpId() { return GetThreadIdInBlock() / WARPSIZE; }
 INLINE unsigned GetLaneId() { return GetThreadIdInBlock() & (WARPSIZE - 1); }
 
-// Return true if this is the first active thread in the warp.
-INLINE bool __kmpc_impl_is_first_active_thread() {
-  unsigned long long Mask = __kmpc_impl_activemask();
-  unsigned long long ShNum = WARPSIZE - (GetThreadIdInBlock() % WARPSIZE);
-  unsigned long long Sh = Mask << ShNum;
-  // Truncate Sh to the 32 lower bits
-  return (unsigned)Sh == 0;
-}
-
 // Locks
 EXTERN void __kmpc_impl_init_lock(omp_lock_t *lock);
 EXTERN void __kmpc_impl_destroy_lock(omp_lock_t *lock);
diff --git a/openmp/libomptarget/plugins/cuda/src/rtl.cpp b/openmp/libomptarget/plugins/cuda/src/rtl.cpp
index b877a63618240..3ec4bc3d5397c 100644
--- a/openmp/libomptarget/plugins/cuda/src/rtl.cpp
+++ b/openmp/libomptarget/plugins/cuda/src/rtl.cpp
@@ -95,7 +95,7 @@ bool checkResult(CUresult Err, const char *ErrMsg) {
   if (Err == CUDA_SUCCESS)
     return true;
 
-  DP(ErrMsg);
+  DP("%s", ErrMsg);
   CUDA_ERR_STRING(Err);
   return false;
 }
@@ -385,9 +385,15 @@ class DeviceRTLTy {
 
     for (DeviceDataTy &D : DeviceData) {
       // Destroy context
-      if (D.Context)
-        checkResult(cuCtxDestroy(D.Context),
-                    "Error returned from cuCtxDestroy\n");
+      if (D.Context) {
+        checkResult(cuCtxSetCurrent(D.Context),
+                    "Error returned from cuCtxSetCurrent\n");
+        CUdevice Device;
+        checkResult(cuCtxGetDevice(&Device),
+                    "Error returned from cuCtxGetDevice\n");
+        checkResult(cuDevicePrimaryCtxRelease(Device),
+                    "Error returned from cuDevicePrimaryCtxRelease\n");
+      }
     }
   }
 
@@ -408,10 +414,32 @@ class DeviceRTLTy {
     if (!checkResult(Err, "Error returned from cuDeviceGet\n"))
       return OFFLOAD_FAIL;
 
-    // Create the context and save it to use whenever this device is selected.
-    Err = cuCtxCreate(&DeviceData[DeviceId].Context, CU_CTX_SCHED_BLOCKING_SYNC,
-                      Device);
-    if (!checkResult(Err, "Error returned from cuCtxCreate\n"))
+    // Query the current flags of the primary context and set its flags if
+    // it is inactive
+    unsigned int FormerPrimaryCtxFlags = 0;
+    int FormerPrimaryCtxIsActive = 0;
+    Err = cuDevicePrimaryCtxGetState(Device, &FormerPrimaryCtxFlags,
+                                     &FormerPrimaryCtxIsActive);
+    if (!checkResult(Err, "Error returned from cuDevicePrimaryCtxGetState\n"))
+      return OFFLOAD_FAIL;
+
+    if (FormerPrimaryCtxIsActive) {
+      DP("The primary context is active, no change to its flags\n");
+      if ((FormerPrimaryCtxFlags & CU_CTX_SCHED_MASK) !=
+          CU_CTX_SCHED_BLOCKING_SYNC)
+        DP("Warning the current flags are not CU_CTX_SCHED_BLOCKING_SYNC\n");
+    } else {
+      DP("The primary context is inactive, set its flags to "
+         "CU_CTX_SCHED_BLOCKING_SYNC\n");
+      Err = cuDevicePrimaryCtxSetFlags(Device, CU_CTX_SCHED_BLOCKING_SYNC);
+      if (!checkResult(Err, "Error returned from cuDevicePrimaryCtxSetFlags\n"))
+        return OFFLOAD_FAIL;
+    }
+
+    // Retain the per device primary context and save it to use whenever this
+    // device is selected.
+    Err = cuDevicePrimaryCtxRetain(&DeviceData[DeviceId].Context, Device);
+    if (!checkResult(Err, "Error returned from cuDevicePrimaryCtxRetain\n"))
       return OFFLOAD_FAIL;
 
     Err = cuCtxSetCurrent(DeviceData[DeviceId].Context);
@@ -795,8 +823,8 @@ class DeviceRTLTy {
         return OFFLOAD_SUCCESS;
 
       DP("Error returned from cuMemcpyPeerAsync. src_ptr = " DPxMOD
-         ", src_id =%" PRId32 ", dst_ptr = %" DPxMOD ", dst_id =%" PRId32 "\n",
-         SrcPtr, SrcDevId, DstPtr, DstDevId);
+         ", src_id =%" PRId32 ", dst_ptr = " DPxMOD ", dst_id =%" PRId32 "\n",
+         DPxPTR(SrcPtr), SrcDevId, DPxPTR(DstPtr), DstDevId);
       CUDA_ERR_STRING(Err);
     }
 
@@ -891,7 +919,7 @@ class DeviceRTLTy {
           // loop.
           CudaBlocksPerGrid = LoopTripCount;
         }
-        DP("Using %d teams due to loop trip count %" PRIu64
+        DP("Using %d teams due to loop trip count %" PRIu32
            " and number of threads per block %d\n",
            CudaBlocksPerGrid, LoopTripCount, CudaThreadsPerBlock);
       } else {
diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp
index c526725502f93..575354305dda0 100644
--- a/openmp/libomptarget/src/device.cpp
+++ b/openmp/libomptarget/src/device.cpp
@@ -25,22 +25,20 @@ int DeviceTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size) {
   DataMapMtx.lock();
 
   // Check if entry exists
-  for (auto &HT : HostDataToTargetMap) {
-    if ((uintptr_t)HstPtrBegin == HT.HstPtrBegin) {
-      // Mapping already exists
-      bool isValid = HT.HstPtrBegin == (uintptr_t) HstPtrBegin &&
-                     HT.HstPtrEnd == (uintptr_t) HstPtrBegin + Size &&
-                     HT.TgtPtrBegin == (uintptr_t) TgtPtrBegin;
-      DataMapMtx.unlock();
-      if (isValid) {
-        DP("Attempt to re-associate the same device ptr+offset with the same "
-            "host ptr, nothing to do\n");
-        return OFFLOAD_SUCCESS;
-      } else {
-        DP("Not allowed to re-associate a different device ptr+offset with the "
-            "same host ptr\n");
-        return OFFLOAD_FAIL;
-      }
+  auto search = HostDataToTargetMap.find(HstPtrBeginTy{(uintptr_t)HstPtrBegin});
+  if (search != HostDataToTargetMap.end()) {
+    // Mapping already exists
+    bool isValid = search->HstPtrEnd == (uintptr_t)HstPtrBegin + Size &&
+                   search->TgtPtrBegin == (uintptr_t)TgtPtrBegin;
+    DataMapMtx.unlock();
+    if (isValid) {
+      DP("Attempt to re-associate the same device ptr+offset with the same "
+         "host ptr, nothing to do\n");
+      return OFFLOAD_SUCCESS;
+    } else {
+      DP("Not allowed to re-associate a different device ptr+offset with the "
+         "same host ptr\n");
+      return OFFLOAD_FAIL;
     }
   }
 
@@ -55,7 +53,7 @@ int DeviceTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size) {
       DPxMOD ", TgtBegin=" DPxMOD "\n", DPxPTR(newEntry.HstPtrBase),
       DPxPTR(newEntry.HstPtrBegin), DPxPTR(newEntry.HstPtrEnd),
       DPxPTR(newEntry.TgtPtrBegin));
-  HostDataToTargetMap.push_front(newEntry);
+  HostDataToTargetMap.insert(newEntry);
 
   DataMapMtx.unlock();
 
@@ -65,21 +63,17 @@ int DeviceTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size) {
 int DeviceTy::disassociatePtr(void *HstPtrBegin) {
   DataMapMtx.lock();
 
-  // Check if entry exists
-  for (HostDataToTargetListTy::iterator ii = HostDataToTargetMap.begin();
-      ii != HostDataToTargetMap.end(); ++ii) {
-    if ((uintptr_t)HstPtrBegin == ii->HstPtrBegin) {
-      // Mapping exists
-      if (ii->isRefCountInf()) {
-        DP("Association found, removing it\n");
-        HostDataToTargetMap.erase(ii);
-        DataMapMtx.unlock();
-        return OFFLOAD_SUCCESS;
-      } else {
-        DP("Trying to disassociate a pointer which was not mapped via "
-            "omp_target_associate_ptr\n");
-        break;
-      }
+  auto search = HostDataToTargetMap.find(HstPtrBeginTy{(uintptr_t)HstPtrBegin});
+  if (search != HostDataToTargetMap.end()) {
+    // Mapping exists
+    if (search->isRefCountInf()) {
+      DP("Association found, removing it\n");
+      HostDataToTargetMap.erase(search);
+      DataMapMtx.unlock();
+      return OFFLOAD_SUCCESS;
+    } else {
+      DP("Trying to disassociate a pointer which was not mapped via "
+         "omp_target_associate_ptr\n");
     }
   }
 
@@ -95,11 +89,14 @@ uint64_t DeviceTy::getMapEntryRefCnt(void *HstPtrBegin) {
   uint64_t RefCnt = 0;
 
   DataMapMtx.lock();
-  for (auto &HT : HostDataToTargetMap) {
-    if (hp >= HT.HstPtrBegin && hp < HT.HstPtrEnd) {
-      DP("DeviceTy::getMapEntry: requested entry found\n");
-      RefCnt = HT.getRefCount();
-      break;
+  if (!HostDataToTargetMap.empty()) {
+    auto upper = HostDataToTargetMap.upper_bound(hp);
+    if (upper != HostDataToTargetMap.begin()) {
+      upper--;
+      if (hp >= upper->HstPtrBegin && hp < upper->HstPtrEnd) {
+        DP("DeviceTy::getMapEntry: requested entry found\n");
+        RefCnt = upper->getRefCount();
+      }
     }
   }
   DataMapMtx.unlock();
@@ -117,21 +114,31 @@ LookupResult DeviceTy::lookupMapping(void *HstPtrBegin, int64_t Size) {
 
   DP("Looking up mapping(HstPtrBegin=" DPxMOD ", Size=%ld)...\n", DPxPTR(hp),
       Size);
-  for (lr.Entry = HostDataToTargetMap.begin();
-      lr.Entry != HostDataToTargetMap.end(); ++lr.Entry) {
+
+  if (HostDataToTargetMap.empty())
+    return lr;
+
+  auto upper = HostDataToTargetMap.upper_bound(hp);
+  // check the left bin
+  if (upper != HostDataToTargetMap.begin()) {
+    lr.Entry = std::prev(upper);
     auto &HT = *lr.Entry;
     // Is it contained?
     lr.Flags.IsContained = hp >= HT.HstPtrBegin && hp < HT.HstPtrEnd &&
         (hp+Size) <= HT.HstPtrEnd;
+    // Does it extend beyond the mapped region?
+    lr.Flags.ExtendsAfter = hp < HT.HstPtrEnd && (hp + Size) > HT.HstPtrEnd;
+  }
+
+  // check the right bin
+  if (!(lr.Flags.IsContained || lr.Flags.ExtendsAfter) &&
+      upper != HostDataToTargetMap.end()) {
+    lr.Entry = upper;
+    auto &HT = *lr.Entry;
     // Does it extend into an already mapped region?
     lr.Flags.ExtendsBefore = hp < HT.HstPtrBegin && (hp+Size) > HT.HstPtrBegin;
     // Does it extend beyond the mapped region?
     lr.Flags.ExtendsAfter = hp < HT.HstPtrEnd && (hp+Size) > HT.HstPtrEnd;
-
-    if (lr.Flags.IsContained || lr.Flags.ExtendsBefore ||
-        lr.Flags.ExtendsAfter) {
-      break;
-    }
   }
 
   if (lr.Flags.ExtendsBefore) {
@@ -203,8 +210,9 @@ void *DeviceTy::getOrAllocTgtPtr(void *HstPtrBegin, void *HstPtrBase,
       DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD ", "
          "HstEnd=" DPxMOD ", TgtBegin=" DPxMOD "\n", DPxPTR(HstPtrBase),
          DPxPTR(HstPtrBegin), DPxPTR((uintptr_t)HstPtrBegin + Size), DPxPTR(tp));
-      HostDataToTargetMap.push_front(HostDataToTargetTy((uintptr_t)HstPtrBase,
-          (uintptr_t)HstPtrBegin, (uintptr_t)HstPtrBegin + Size, tp));
+      HostDataToTargetMap.emplace(
+          HostDataToTargetTy((uintptr_t)HstPtrBase, (uintptr_t)HstPtrBegin,
+                             (uintptr_t)HstPtrBegin + Size, tp));
       rc = (void *)tp;
     }
   }
diff --git a/openmp/libomptarget/src/device.h b/openmp/libomptarget/src/device.h
index 1b7f776bd3b80..309785acb4721 100644
--- a/openmp/libomptarget/src/device.h
+++ b/openmp/libomptarget/src/device.h
@@ -18,6 +18,7 @@
 #include <list>
 #include <map>
 #include <mutex>
+#include <set>
 #include <vector>
 
 // Forward declarations.
@@ -35,7 +36,8 @@ struct HostDataToTargetTy {
   uintptr_t TgtPtrBegin; // target info.
 
 private:
-  uint64_t RefCount;
+  /// use mutable to allow modification via std::set iterator which is const.
+  mutable uint64_t RefCount;
   static const uint64_t INFRefCount = ~(uint64_t)0;
 
 public:
@@ -48,14 +50,14 @@ struct HostDataToTargetTy {
     return RefCount;
   }
 
-  uint64_t resetRefCount() {
+  uint64_t resetRefCount() const {
     if (RefCount != INFRefCount)
       RefCount = 1;
 
     return RefCount;
   }
 
-  uint64_t incRefCount() {
+  uint64_t incRefCount() const {
     if (RefCount != INFRefCount) {
       ++RefCount;
       assert(RefCount < INFRefCount && "refcount overflow");
@@ -64,7 +66,7 @@ struct HostDataToTargetTy {
     return RefCount;
   }
 
-  uint64_t decRefCount() {
+  uint64_t decRefCount() const {
     if (RefCount != INFRefCount) {
       assert(RefCount > 0 && "refcount underflow");
       --RefCount;
@@ -78,7 +80,19 @@ struct HostDataToTargetTy {
   }
 };
 
-typedef std::list<HostDataToTargetTy> HostDataToTargetListTy;
+typedef uintptr_t HstPtrBeginTy;
+inline bool operator<(const HostDataToTargetTy &lhs, const HstPtrBeginTy &rhs) {
+  return lhs.HstPtrBegin < rhs;
+}
+inline bool operator<(const HstPtrBeginTy &lhs, const HostDataToTargetTy &rhs) {
+  return lhs < rhs.HstPtrBegin;
+}
+inline bool operator<(const HostDataToTargetTy &lhs,
+                      const HostDataToTargetTy &rhs) {
+  return lhs.HstPtrBegin < rhs.HstPtrBegin;
+}
+
+typedef std::set<HostDataToTargetTy, std::less<>> HostDataToTargetListTy;
 
 struct LookupResult {
   struct {
diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index df7481a348059..cce9dbd2fe154 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -139,12 +139,12 @@ static int InitLibrary(DeviceTy& Device) {
         DP("Add mapping from host " DPxMOD " to device " DPxMOD " with size %zu"
             "\n", DPxPTR(CurrHostEntry->addr), DPxPTR(CurrDeviceEntry->addr),
             CurrDeviceEntry->size);
-        Device.HostDataToTargetMap.push_front(HostDataToTargetTy(
+        Device.HostDataToTargetMap.emplace(
             (uintptr_t)CurrHostEntry->addr /*HstPtrBase*/,
             (uintptr_t)CurrHostEntry->addr /*HstPtrBegin*/,
             (uintptr_t)CurrHostEntry->addr + CurrHostEntry->size /*HstPtrEnd*/,
             (uintptr_t)CurrDeviceEntry->addr /*TgtPtrBegin*/,
-            true /*IsRefCountINF*/));
+            true /*IsRefCountINF*/);
       }
     }
     Device.DataMapMtx.unlock();
diff --git a/openmp/libomptarget/test/offloading/dynamic_module.c b/openmp/libomptarget/test/offloading/dynamic_module.c
index 7f062b6d752c0..4cb6f35ba8992 100644
--- a/openmp/libomptarget/test/offloading/dynamic_module.c
+++ b/openmp/libomptarget/test/offloading/dynamic_module.c
@@ -1,7 +1,7 @@
-// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu -DSHARED -shared -o %t.so && %libomptarget-compile-aarch64-unknown-linux-gnu %t.so && %libomptarget-run-aarch64-unknown-linux-gnu 2>&1 | %fcheck-aarch64-unknown-linux-gnu
-// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu -DSHARED -shared -o %t.so && %libomptarget-compile-powerpc64-ibm-linux-gnu %t.so && %libomptarget-run-powerpc64-ibm-linux-gnu 2>&1 | %fcheck-powerpc64-ibm-linux-gnu
-// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu -DSHARED -shared -o %t.so && %libomptarget-compile-powerpc64le-ibm-linux-gnu %t.so && %libomptarget-run-powerpc64le-ibm-linux-gnu 2>&1 | %fcheck-powerpc64le-ibm-linux-gnu
-// RUN: %libomptarget-compile-x86_64-pc-linux-gnu -DSHARED -shared -o %t.so && %libomptarget-compile-x86_64-pc-linux-gnu %t.so && %libomptarget-run-x86_64-pc-linux-gnu 2>&1 | %fcheck-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu -DSHARED -fPIC -shared -o %t.so && %libomptarget-compile-aarch64-unknown-linux-gnu %t.so && %libomptarget-run-aarch64-unknown-linux-gnu 2>&1 | %fcheck-aarch64-unknown-linux-gnu
+// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu -DSHARED -fPIC -shared -o %t.so && %libomptarget-compile-powerpc64-ibm-linux-gnu %t.so && %libomptarget-run-powerpc64-ibm-linux-gnu 2>&1 | %fcheck-powerpc64-ibm-linux-gnu
+// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu -DSHARED -fPIC -shared -o %t.so && %libomptarget-compile-powerpc64le-ibm-linux-gnu %t.so && %libomptarget-run-powerpc64le-ibm-linux-gnu 2>&1 | %fcheck-powerpc64le-ibm-linux-gnu
+// RUN: %libomptarget-compile-x86_64-pc-linux-gnu -DSHARED -fPIC -shared -o %t.so && %libomptarget-compile-x86_64-pc-linux-gnu %t.so && %libomptarget-run-x86_64-pc-linux-gnu 2>&1 | %fcheck-x86_64-pc-linux-gnu
 
 #ifdef SHARED
 void foo() {}
diff --git a/openmp/libomptarget/test/offloading/dynamic_module_load.c b/openmp/libomptarget/test/offloading/dynamic_module_load.c
index fe917e4fe1cfb..958fe2dec351c 100644
--- a/openmp/libomptarget/test/offloading/dynamic_module_load.c
+++ b/openmp/libomptarget/test/offloading/dynamic_module_load.c
@@ -1,7 +1,7 @@
-// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu -DSHARED -shared -o %t.so && %clang %flags %s -o %t-aarch64-unknown-linux-gnu -ldl && %libomptarget-run-aarch64-unknown-linux-gnu %t.so 2>&1 | %fcheck-aarch64-unknown-linux-gnu
-// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu -DSHARED -shared -o %t.so && %clang %flags %s -o %t-powerpc64-ibm-linux-gnu -ldl && %libomptarget-run-powerpc64-ibm-linux-gnu %t.so 2>&1 | %fcheck-powerpc64-ibm-linux-gnu
-// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu -DSHARED -shared -o %t.so && %clang %flags %s -o %t-powerpc64le-ibm-linux-gnu -ldl && %libomptarget-run-powerpc64le-ibm-linux-gnu %t.so 2>&1 | %fcheck-powerpc64le-ibm-linux-gnu
-// RUN: %libomptarget-compile-x86_64-pc-linux-gnu -DSHARED -shared -o %t.so && %clang %flags %s -o %t-x86_64-pc-linux-gnu -ldl && %libomptarget-run-x86_64-pc-linux-gnu %t.so 2>&1 | %fcheck-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu -DSHARED -fPIC -shared -o %t.so && %clang %flags %s -o %t-aarch64-unknown-linux-gnu -ldl && %libomptarget-run-aarch64-unknown-linux-gnu %t.so 2>&1 | %fcheck-aarch64-unknown-linux-gnu
+// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu -DSHARED -fPIC -shared -o %t.so && %clang %flags %s -o %t-powerpc64-ibm-linux-gnu -ldl && %libomptarget-run-powerpc64-ibm-linux-gnu %t.so 2>&1 | %fcheck-powerpc64-ibm-linux-gnu
+// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu -DSHARED -fPIC -shared -o %t.so && %clang %flags %s -o %t-powerpc64le-ibm-linux-gnu -ldl && %libomptarget-run-powerpc64le-ibm-linux-gnu %t.so 2>&1 | %fcheck-powerpc64le-ibm-linux-gnu
+// RUN: %libomptarget-compile-x86_64-pc-linux-gnu -DSHARED -fPIC -shared -o %t.so && %clang %flags %s -o %t-x86_64-pc-linux-gnu -ldl && %libomptarget-run-x86_64-pc-linux-gnu %t.so 2>&1 | %fcheck-x86_64-pc-linux-gnu
 
 #ifdef SHARED
 #include <stdio.h>
diff --git a/openmp/runtime/src/exports_so.txt b/openmp/runtime/src/exports_so.txt
index f7de5fd6474f8..30222418163d7 100644
--- a/openmp/runtime/src/exports_so.txt
+++ b/openmp/runtime/src/exports_so.txt
@@ -119,5 +119,7 @@ GOMP_4.0 {
 } GOMP_3.0;
 GOMP_4.5 {
 } GOMP_4.0;
+GOMP_5.0 {
+} GOMP_4.5;
 
 # end of file #
diff --git a/openmp/runtime/src/kmp_affinity.cpp b/openmp/runtime/src/kmp_affinity.cpp
index 9690df53b77d3..47e70477ced6c 100644
--- a/openmp/runtime/src/kmp_affinity.cpp
+++ b/openmp/runtime/src/kmp_affinity.cpp
@@ -1968,7 +1968,8 @@ static void __kmp_dispatch_set_hierarchy_values() {
   __kmp_hier_max_units[kmp_hier_layer_e::LAYER_THREAD + 1] =
       nPackages * nCoresPerPkg * __kmp_nThreadsPerCore;
   __kmp_hier_max_units[kmp_hier_layer_e::LAYER_L1 + 1] = __kmp_ncores;
-#if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_WINDOWS)
+#if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_WINDOWS) && \
+    KMP_MIC_SUPPORTED
   if (__kmp_mic_type >= mic3)
     __kmp_hier_max_units[kmp_hier_layer_e::LAYER_L2 + 1] = __kmp_ncores / 2;
   else
@@ -1982,7 +1983,8 @@ static void __kmp_dispatch_set_hierarchy_values() {
   __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_THREAD + 1] = 1;
   __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_L1 + 1] =
       __kmp_nThreadsPerCore;
-#if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_WINDOWS)
+#if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_WINDOWS) && \
+    KMP_MIC_SUPPORTED
   if (__kmp_mic_type >= mic3)
     __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_L2 + 1] =
         2 * __kmp_nThreadsPerCore;
diff --git a/openmp/runtime/src/kmp_ftn_os.h b/openmp/runtime/src/kmp_ftn_os.h
index cd04914ecbf20..22fb2bb2f5ca0 100644
--- a/openmp/runtime/src/kmp_ftn_os.h
+++ b/openmp/runtime/src/kmp_ftn_os.h
@@ -658,4 +658,26 @@
 #define KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_GUIDED                    \
   GOMP_parallel_loop_nonmonotonic_guided
 
+// All GOMP_5.0 symbols
+#define KMP_API_NAME_GOMP_LOOP_MAYBE_NONMONOTONIC_RUNTIME_NEXT                 \
+  GOMP_loop_maybe_nonmonotonic_runtime_next
+#define KMP_API_NAME_GOMP_LOOP_MAYBE_NONMONOTONIC_RUNTIME_START                \
+  GOMP_loop_maybe_nonmonotonic_runtime_start
+#define KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_RUNTIME_NEXT                       \
+  GOMP_loop_nonmonotonic_runtime_next
+#define KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_RUNTIME_START                      \
+  GOMP_loop_nonmonotonic_runtime_start
+#define KMP_API_NAME_GOMP_LOOP_ULL_MAYBE_NONMONOTONIC_RUNTIME_NEXT             \
+  GOMP_loop_ull_maybe_nonmonotonic_runtime_next
+#define KMP_API_NAME_GOMP_LOOP_ULL_MAYBE_NONMONOTONIC_RUNTIME_START            \
+  GOMP_loop_ull_maybe_nonmonotonic_runtime_start
+#define KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_NEXT                   \
+  GOMP_loop_ull_nonmonotonic_runtime_next
+#define KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_START                  \
+  GOMP_loop_ull_nonmonotonic_runtime_start
+#define KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_RUNTIME                   \
+  GOMP_parallel_loop_nonmonotonic_runtime
+#define KMP_API_NAME_GOMP_PARALLEL_LOOP_MAYBE_NONMONOTONIC_RUNTIME             \
+  GOMP_parallel_loop_maybe_nonmonotonic_runtime
+
 #endif /* KMP_FTN_OS_H */
diff --git a/openmp/runtime/src/kmp_gsupport.cpp b/openmp/runtime/src/kmp_gsupport.cpp
index d7592f34eddc8..ab4f27bfc0678 100644
--- a/openmp/runtime/src/kmp_gsupport.cpp
+++ b/openmp/runtime/src/kmp_gsupport.cpp
@@ -635,6 +635,15 @@ LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_GUIDED_NEXT), {})
 LOOP_RUNTIME_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_RUNTIME_START),
                    kmp_sch_runtime)
 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_RUNTIME_NEXT), {})
+LOOP_RUNTIME_START(
+    KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_MAYBE_NONMONOTONIC_RUNTIME_START),
+    kmp_sch_runtime)
+LOOP_RUNTIME_START(
+    KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_RUNTIME_START),
+    kmp_sch_runtime)
+LOOP_NEXT(
+    KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_MAYBE_NONMONOTONIC_RUNTIME_NEXT), {})
+LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_RUNTIME_NEXT), {})
 
 LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_START),
            kmp_ord_static)
@@ -911,6 +920,18 @@ LOOP_NEXT_ULL(
 LOOP_RUNTIME_START_ULL(
     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_START), kmp_sch_runtime)
 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_NEXT), {})
+LOOP_RUNTIME_START_ULL(
+    KMP_EXPAND_NAME(
+        KMP_API_NAME_GOMP_LOOP_ULL_MAYBE_NONMONOTONIC_RUNTIME_START),
+    kmp_sch_runtime)
+LOOP_RUNTIME_START_ULL(
+    KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_START),
+    kmp_sch_runtime)
+LOOP_NEXT_ULL(
+    KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_MAYBE_NONMONOTONIC_RUNTIME_NEXT),
+    {})
+LOOP_NEXT_ULL(
+    KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_NEXT), {})
 
 LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_START),
                kmp_ord_static)
@@ -1513,6 +1534,12 @@ PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED),
               kmp_sch_guided_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST)
 PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME),
               kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST)
+PARALLEL_LOOP(
+    KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_MAYBE_NONMONOTONIC_RUNTIME),
+    kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST)
+PARALLEL_LOOP(
+    KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_RUNTIME),
+    kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST)
 
 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKGROUP_START)(void) {
   int gtid = __kmp_entry_gtid();
@@ -1985,6 +2012,28 @@ KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_DYNAMIC, 45,
 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_GUIDED, 45,
                    "GOMP_4.5");
 
+// GOMP_5.0 versioned symbols
+KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_MAYBE_NONMONOTONIC_RUNTIME_NEXT, 50,
+                   "GOMP_5.0");
+KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_MAYBE_NONMONOTONIC_RUNTIME_START, 50,
+                   "GOMP_5.0");
+KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_RUNTIME_NEXT, 50,
+                   "GOMP_5.0");
+KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_RUNTIME_START, 50,
+                   "GOMP_5.0");
+KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_MAYBE_NONMONOTONIC_RUNTIME_NEXT,
+                   50, "GOMP_5.0");
+KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_MAYBE_NONMONOTONIC_RUNTIME_START,
+                   50, "GOMP_5.0");
+KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_NEXT, 50,
+                   "GOMP_5.0");
+KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_START, 50,
+                   "GOMP_5.0");
+KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_RUNTIME, 50,
+                   "GOMP_5.0");
+KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_MAYBE_NONMONOTONIC_RUNTIME,
+                   50, "GOMP_5.0");
+
 #endif // KMP_USE_VERSION_SYMBOLS
 
 #ifdef __cplusplus
diff --git a/openmp/runtime/src/kmp_taskdeps.cpp b/openmp/runtime/src/kmp_taskdeps.cpp
index d99f8e14ab4dd..a654951f5b3b2 100644
--- a/openmp/runtime/src/kmp_taskdeps.cpp
+++ b/openmp/runtime/src/kmp_taskdeps.cpp
@@ -205,7 +205,7 @@ static kmp_depnode_list_t *__kmp_add_node(kmp_info_t *thread,
   return new_head;
 }
 
-static inline void __kmp_track_dependence(kmp_depnode_t *source,
+static inline void __kmp_track_dependence(kmp_int32 gtid, kmp_depnode_t *source,
                                           kmp_depnode_t *sink,
                                           kmp_task_t *sink_task) {
 #ifdef KMP_SUPPORT_GRAPH_OUTPUT
@@ -224,11 +224,14 @@ static inline void __kmp_track_dependence(kmp_depnode_t *source,
      */
   if (ompt_enabled.ompt_callback_task_dependence) {
     kmp_taskdata_t *task_source = KMP_TASK_TO_TASKDATA(source->dn.task);
-    kmp_taskdata_t *task_sink = KMP_TASK_TO_TASKDATA(sink_task);
+    ompt_data_t *sink_data;
+    if (sink_task)
+      sink_data = &(KMP_TASK_TO_TASKDATA(sink_task)->ompt_task_info.task_data);
+    else
+      sink_data = &__kmp_threads[gtid]->th.ompt_thread_info.task_data;
 
     ompt_callbacks.ompt_callback(ompt_callback_task_dependence)(
-        &(task_source->ompt_task_info.task_data),
-        &(task_sink->ompt_task_info.task_data));
+        &(task_source->ompt_task_info.task_data), sink_data);
   }
 #endif /* OMPT_SUPPORT && OMPT_OPTIONAL */
 }
@@ -246,7 +249,7 @@ __kmp_depnode_link_successor(kmp_int32 gtid, kmp_info_t *thread,
     if (dep->dn.task) {
       KMP_ACQUIRE_DEPNODE(gtid, dep);
       if (dep->dn.task) {
-        __kmp_track_dependence(dep, node, task);
+        __kmp_track_dependence(gtid, dep, node, task);
         dep->dn.successors = __kmp_add_node(thread, dep->dn.successors, node);
         KA_TRACE(40, ("__kmp_process_deps: T#%d adding dependence from %p to "
                       "%p\n",
@@ -272,7 +275,7 @@ static inline kmp_int32 __kmp_depnode_link_successor(kmp_int32 gtid,
     // synchronously add source to sink' list of successors
     KMP_ACQUIRE_DEPNODE(gtid, sink);
     if (sink->dn.task) {
-      __kmp_track_dependence(sink, source, task);
+      __kmp_track_dependence(gtid, sink, source, task);
       sink->dn.successors = __kmp_add_node(thread, sink->dn.successors, source);
       KA_TRACE(40, ("__kmp_process_deps: T#%d adding dependence from %p to "
                     "%p\n",
@@ -635,6 +638,23 @@ kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32 gtid,
   return ret;
 }
 
+#if OMPT_SUPPORT
+void __ompt_taskwait_dep_finish(kmp_taskdata_t *current_task,
+                                ompt_data_t *taskwait_task_data) {
+  if (ompt_enabled.ompt_callback_task_schedule) {
+    ompt_data_t task_data = ompt_data_none;
+    ompt_callbacks.ompt_callback(ompt_callback_task_schedule)(
+        current_task ? &(current_task->ompt_task_info.task_data) : &task_data,
+        ompt_task_switch, taskwait_task_data);
+    ompt_callbacks.ompt_callback(ompt_callback_task_schedule)(
+        taskwait_task_data, ompt_task_complete,
+        current_task ? &(current_task->ompt_task_info.task_data) : &task_data);
+  }
+  current_task->ompt_task_info.frame.enter_frame.ptr = NULL;
+  *taskwait_task_data = ompt_data_none;
+}
+#endif /* OMPT_SUPPORT */
+
 /*!
 @ingroup TASKING
 @param loc_ref location of the original task directive
@@ -661,6 +681,74 @@ void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps,
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_taskdata_t *current_task = thread->th.th_current_task;
 
+#if OMPT_SUPPORT
+  // this function represents a taskwait construct with depend clause
+  // We signal 4 events:
+  //  - creation of the taskwait task
+  //  - dependences of the taskwait task
+  //  - schedule and finish of the taskwait task
+  ompt_data_t *taskwait_task_data = &thread->th.ompt_thread_info.task_data;
+  KMP_ASSERT(taskwait_task_data->ptr == NULL);
+  if (ompt_enabled.enabled) {
+    if (!current_task->ompt_task_info.frame.enter_frame.ptr)
+      current_task->ompt_task_info.frame.enter_frame.ptr =
+          OMPT_GET_FRAME_ADDRESS(0);
+    if (ompt_enabled.ompt_callback_task_create) {
+      ompt_data_t task_data = ompt_data_none;
+      ompt_callbacks.ompt_callback(ompt_callback_task_create)(
+          current_task ? &(current_task->ompt_task_info.task_data) : &task_data,
+          current_task ? &(current_task->ompt_task_info.frame) : NULL,
+          taskwait_task_data,
+          ompt_task_explicit | ompt_task_undeferred | ompt_task_mergeable, 1,
+          OMPT_GET_RETURN_ADDRESS(0));
+    }
+  }
+
+#if OMPT_OPTIONAL
+  /* OMPT grab all dependences if requested by the tool */
+  if (ndeps + ndeps_noalias > 0 && ompt_enabled.ompt_callback_dependences) {
+    kmp_int32 i;
+
+    int ompt_ndeps = ndeps + ndeps_noalias;
+    ompt_dependence_t *ompt_deps = (ompt_dependence_t *)KMP_OMPT_DEPS_ALLOC(
+        thread, (ndeps + ndeps_noalias) * sizeof(ompt_dependence_t));
+
+    KMP_ASSERT(ompt_deps != NULL);
+
+    for (i = 0; i < ndeps; i++) {
+      ompt_deps[i].variable.ptr = (void *)dep_list[i].base_addr;
+      if (dep_list[i].flags.in && dep_list[i].flags.out)
+        ompt_deps[i].dependence_type = ompt_dependence_type_inout;
+      else if (dep_list[i].flags.out)
+        ompt_deps[i].dependence_type = ompt_dependence_type_out;
+      else if (dep_list[i].flags.in)
+        ompt_deps[i].dependence_type = ompt_dependence_type_in;
+      else if (dep_list[i].flags.mtx)
+        ompt_deps[ndeps + i].dependence_type =
+            ompt_dependence_type_mutexinoutset;
+    }
+    for (i = 0; i < ndeps_noalias; i++) {
+      ompt_deps[ndeps + i].variable.ptr = (void *)noalias_dep_list[i].base_addr;
+      if (noalias_dep_list[i].flags.in && noalias_dep_list[i].flags.out)
+        ompt_deps[ndeps + i].dependence_type = ompt_dependence_type_inout;
+      else if (noalias_dep_list[i].flags.out)
+        ompt_deps[ndeps + i].dependence_type = ompt_dependence_type_out;
+      else if (noalias_dep_list[i].flags.in)
+        ompt_deps[ndeps + i].dependence_type = ompt_dependence_type_in;
+      else if (noalias_dep_list[i].flags.mtx)
+        ompt_deps[ndeps + i].dependence_type =
+            ompt_dependence_type_mutexinoutset;
+    }
+    ompt_callbacks.ompt_callback(ompt_callback_dependences)(
+        taskwait_task_data, ompt_deps, ompt_ndeps);
+    /* We can now free the allocated memory for the dependencies */
+    /* For OMPD we might want to delay the free until end of this function */
+    KMP_OMPT_DEPS_FREE(thread, ompt_deps);
+    ompt_deps = NULL;
+  }
+#endif /* OMPT_OPTIONAL */
+#endif /* OMPT_SUPPORT */
+
   // We can return immediately as:
   // - dependences are not computed in serial teams (except with proxy tasks)
   // - if the dephash is not yet created it means we have nothing to wait for
@@ -675,6 +763,9 @@ void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps,
     KA_TRACE(10, ("__kmpc_omp_wait_deps(exit): T#%d has no blocking "
                   "dependencies : loc=%p\n",
                   gtid, loc_ref));
+#if OMPT_SUPPORT
+    __ompt_taskwait_dep_finish(current_task, taskwait_task_data);
+#endif /* OMPT_SUPPORT */
     return;
   }
 
@@ -687,6 +778,9 @@ void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps,
     KA_TRACE(10, ("__kmpc_omp_wait_deps(exit): T#%d has no blocking "
                   "dependencies : loc=%p\n",
                   gtid, loc_ref));
+#if OMPT_SUPPORT
+    __ompt_taskwait_dep_finish(current_task, taskwait_task_data);
+#endif /* OMPT_SUPPORT */
     return;
   }
 
@@ -698,6 +792,9 @@ void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps,
                        __kmp_task_stealing_constraint);
   }
 
+#if OMPT_SUPPORT
+  __ompt_taskwait_dep_finish(current_task, taskwait_task_data);
+#endif /* OMPT_SUPPORT */
   KA_TRACE(10, ("__kmpc_omp_wait_deps(exit): T#%d finished waiting : loc=%p\n",
                 gtid, loc_ref));
 }
diff --git a/openmp/runtime/test/env/kmp_set_dispatch_buf.c b/openmp/runtime/test/env/kmp_set_dispatch_buf.c
index 49eb7b54fb51a..61cb9d05bdb87 100644
--- a/openmp/runtime/test/env/kmp_set_dispatch_buf.c
+++ b/openmp/runtime/test/env/kmp_set_dispatch_buf.c
@@ -1,9 +1,15 @@
-// RUN: %libomp-compile && env KMP_DISP_NUM_BUFFERS=0 %libomp-run
-// RUN: env KMP_DISP_NUM_BUFFERS=1 %libomp-run && env KMP_DISP_NUM_BUFFERS=3 %libomp-run
-// RUN: env KMP_DISP_NUM_BUFFERS=4 %libomp-run && env KMP_DISP_NUM_BUFFERS=7 %libomp-run
-// RUN: %libomp-compile -DMY_SCHEDULE=guided && env KMP_DISP_NUM_BUFFERS=1 %libomp-run
-// RUN: env KMP_DISP_NUM_BUFFERS=3 %libomp-run && env KMP_DISP_NUM_BUFFERS=4 %libomp-run
+// RUN: %libomp-compile
+// RUN: env KMP_DISP_NUM_BUFFERS=0 %libomp-run
+// RUN: env KMP_DISP_NUM_BUFFERS=1 %libomp-run
+// RUN: env KMP_DISP_NUM_BUFFERS=3 %libomp-run
+// RUN: env KMP_DISP_NUM_BUFFERS=4 %libomp-run
 // RUN: env KMP_DISP_NUM_BUFFERS=7 %libomp-run
+// RUN: %libomp-compile -DMY_SCHEDULE=guided
+// RUN: env KMP_DISP_NUM_BUFFERS=1 %libomp-run
+// RUN: env KMP_DISP_NUM_BUFFERS=3 %libomp-run
+// RUN: env KMP_DISP_NUM_BUFFERS=4 %libomp-run
+// RUN: env KMP_DISP_NUM_BUFFERS=7 %libomp-run
+// UNSUPPORTED: clang-11
 #include <stdio.h>
 #include <omp.h>
 #include <stdlib.h>
diff --git a/openmp/runtime/test/lit.cfg b/openmp/runtime/test/lit.cfg
index 65ff481f6e5ec..357b18a205d0c 100644
--- a/openmp/runtime/test/lit.cfg
+++ b/openmp/runtime/test/lit.cfg
@@ -111,7 +111,6 @@ except NotImplementedError:
 if 'INTEL_LICENSE_FILE' in os.environ:
     config.environment['INTEL_LICENSE_FILE'] = os.environ['INTEL_LICENSE_FILE']
 
-
 # substitutions
 config.substitutions.append(("%libomp-compile-and-run", \
     "%libomp-compile && %libomp-run"))
diff --git a/openmp/runtime/test/ompt/callback.h b/openmp/runtime/test/ompt/callback.h
index 3975ed9d1e10a..6b756b8a75a1e 100644
--- a/openmp/runtime/test/ompt/callback.h
+++ b/openmp/runtime/test/ompt/callback.h
@@ -22,22 +22,19 @@
 #define _OMPT_TESTS
 #endif
 
-static const char* ompt_thread_t_values[] = {
-  NULL,
-  "ompt_thread_initial",
-  "ompt_thread_worker",
-  "ompt_thread_other"
-};
+static const char *ompt_thread_t_values[] = {
+    "ompt_thread_UNDEFINED", "ompt_thread_initial", "ompt_thread_worker",
+    "ompt_thread_other"};
 
-static const char* ompt_task_status_t_values[] = {
-  NULL,
-  "ompt_task_complete",       // 1
-  "ompt_task_yield",          // 2
-  "ompt_task_cancel",         // 3
-  "ompt_task_detach",         // 4
-  "ompt_task_early_fulfill",  // 5
-  "ompt_task_late_fulfill",   // 6
-  "ompt_task_switch"          // 7
+static const char *ompt_task_status_t_values[] = {
+    "ompt_task_UNDEFINED",
+    "ompt_task_complete", // 1
+    "ompt_task_yield", // 2
+    "ompt_task_cancel", // 3
+    "ompt_task_detach", // 4
+    "ompt_task_early_fulfill", // 5
+    "ompt_task_late_fulfill", // 6
+    "ompt_task_switch" // 7
 };
 static const char* ompt_cancel_flag_t_values[] = {
   "ompt_cancel_parallel",
@@ -50,7 +47,7 @@ static const char* ompt_cancel_flag_t_values[] = {
 };
 
 static const char *ompt_dependence_type_t_values[] = {
-    NULL,
+    "ompt_dependence_type_UNDEFINED",
     "ompt_dependence_type_in", // 1
     "ompt_dependence_type_out", // 2
     "ompt_dependence_type_inout", // 3
@@ -990,7 +987,7 @@ on_ompt_callback_dependences(
     if (deps[i].dependence_type == ompt_dependence_type_source ||
         deps[i].dependence_type == ompt_dependence_type_sink)
       progress +=
-          sprintf(progress, "(%ld, %s), ", deps[i].variable.value,
+          sprintf(progress, "(%" PRIu64 ", %s), ", deps[i].variable.value,
                   ompt_dependence_type_t_values[deps[i].dependence_type]);
     else
       progress +=
diff --git a/openmp/runtime/test/ompt/synchronization/ordered_dependences.c b/openmp/runtime/test/ompt/synchronization/ordered_dependences.c
index 9da86e299af6a..dc9f0824d9150 100644
--- a/openmp/runtime/test/ompt/synchronization/ordered_dependences.c
+++ b/openmp/runtime/test/ompt/synchronization/ordered_dependences.c
@@ -25,39 +25,37 @@ int main() {
 // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
 // CHECK: {{^}}[[MASTER:[0-9]+]]: ompt_event_loop_begin:
-// CHECK-SAME: parallel_id={{[0-9]+}}, parent_task_id=[[IMPL_TASK:[0-9]+]],
+// CHECK-SAME: parallel_id={{[0-9]+}}, parent_task_id=[[ITASK:[0-9]+]],
 
-// CHECK: {{^}}[[MASTER]]: ompt_event_dependences: task_id=[[IMPL_TASK]],
+// CHECK: {{^}}[[MASTER]]: ompt_event_dependences: task_id=[[ITASK]],
 // CHECK-SAME: deps=[(0, ompt_dependence_type_source), (0,
 // CHECK-SAME: ompt_dependence_type_source)], ndeps=2
 
-// CHECK: {{^}}[[MASTER]]: ompt_event_dependences: task_id=[[IMPL_TASK]],
+// CHECK: {{^}}[[MASTER]]: ompt_event_dependences: task_id=[[ITASK]],
 // CHECK-SAME: deps=[(0, ompt_dependence_type_sink), (0,
 // CHECK-SAME: ompt_dependence_type_sink)], ndeps=2
 
-// CHECK: {{^}}[[MASTER]]: ompt_event_dependences: task_id=[[IMPL_TASK]],
+// CHECK: {{^}}[[MASTER]]: ompt_event_dependences: task_id=[[ITASK]],
 // CHECK-SAME: deps=[(0, ompt_dependence_type_source), (1,
 // CHECK-SAME: ompt_dependence_type_source)], ndeps=2
 
 // CHECK: {{^}}[[WORKER:[0-9]+]]: ompt_event_loop_begin:
-// CHECK-SAME: parallel_id={{[0-9]+}}, parent_task_id=[[IMPL_TASK:[0-9]+]],
+// CHECK-SAME: parallel_id={{[0-9]+}}, parent_task_id=[[ITASK:[0-9]+]],
 
-// CHECK: {{^}}[[WORKER]]: ompt_event_dependences: task_id=[[IMPL_TASK]],
+// CHECK: {{^}}[[WORKER]]: ompt_event_dependences: task_id=[[ITASK]],
 // CHECK-SAME: deps=[(0, ompt_dependence_type_sink), (0,
 // CHECK-SAME: ompt_dependence_type_sink)], ndeps=2
 
-// CHECK: {{^}}[[WORKER]]: ompt_event_dependences: task_id=[[IMPL_TASK]],
+// CHECK: {{^}}[[WORKER]]: ompt_event_dependences: task_id=[[ITASK]],
 // CHECK-SAME: deps=[(1, ompt_dependence_type_source), (0,
 // CHECK-SAME: ompt_dependence_type_source)], ndeps=2
 
-// CHECK: {{^}}[[WORKER]]: ompt_event_dependences: task_id=[[IMPL_TASK]],
-// CHECK-SAME: deps=[(0, ompt_dependence_type_sink), (1,
-// CHECK-SAME: ompt_dependence_type_sink)], ndeps=2
+// either can be first for last iteration
 
-// CHECK: {{^}}[[WORKER]]: ompt_event_dependences: task_id=[[IMPL_TASK]],
-// CHECK-SAME: deps=[(1, ompt_dependence_type_sink), (0,
-// CHECK-SAME: ompt_dependence_type_sink)], ndeps=2
+// CHECK-DAG: [[ITASK]]{{.*}}deps=[(0{{.*}}sink), (1,{{.*}}sink)]
+
+// CHECK-DAG: [[ITASK]]{{.*}}deps=[(1{{.*}}sink), (0,{{.*}}sink)]
 
-// CHECK: {{^}}[[WORKER]]: ompt_event_dependences: task_id=[[IMPL_TASK]],
+// CHECK: {{^}}[[WORKER]]: ompt_event_dependences: task_id=[[ITASK]],
 // CHECK-SAME: deps=[(1, ompt_dependence_type_source), (1,
 // CHECK-SAME: ompt_dependence_type_source)], ndeps=2
diff --git a/openmp/runtime/test/ompt/tasks/dependences.c b/openmp/runtime/test/ompt/tasks/dependences.c
index 2646546158c13..9e9349f956106 100644
--- a/openmp/runtime/test/ompt/tasks/dependences.c
+++ b/openmp/runtime/test/ompt/tasks/dependences.c
@@ -24,14 +24,6 @@ int main() {
       print_fuzzy_address(1);
       print_ids(0);
 
-#pragma omp task depend(mutexinoutset : x)
-      {
-        x++;
-        delay(100);
-      }
-      print_fuzzy_address(2);
-      print_ids(0);
-
 #pragma omp task depend(in : x)
       { x = -1; }
       print_ids(0);
@@ -86,28 +78,10 @@ int main() {
 
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_dependences:
 // CHECK-SAME: task_id=[[SECOND_TASK]], deps=[([[ADDRX]],
-// CHECK-SAME: ompt_dependence_type_mutexinoutset)], ndeps=1
-
-// CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_dependence_pair:
-// CHECK-SAME: first_task_id=[[FIRST_TASK]], second_task_id=[[SECOND_TASK]]
-
-// CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
-// CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]],
-// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]],
-// CHECK-SAME: reenter_frame=[[NULL]]
-
-// CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
-// CHECK-SAME: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[EXIT]],
-// CHECK-SAME: parent_task_frame.reenter={{0x[0-f]+}},
-// CHECK-SAME: new_task_id=[[THIRD_TASK:[0-f]+]], codeptr_ra={{0x[0-f]+}},
-// CHECK-SAME: task_type=ompt_task_explicit=4, has_dependences=yes
-
-// CHECK: {{^}}[[MASTER_ID]]: ompt_event_dependences:
-// CHECK-SAME: task_id=[[THIRD_TASK]], deps=[([[ADDRX]],
 // CHECK-SAME: ompt_dependence_type_in)], ndeps=1
 
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_dependence_pair:
-// CHECK-SAME: first_task_id=[[SECOND_TASK]], second_task_id=[[THIRD_TASK]]
+// CHECK-SAME: first_task_id=[[FIRST_TASK]], second_task_id=[[SECOND_TASK]]
 
 // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]],
 // CHECK-SAME: task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]],
diff --git a/openmp/runtime/test/ompt/tasks/dependences_mutexinoutset.c b/openmp/runtime/test/ompt/tasks/dependences_mutexinoutset.c
new file mode 100644
index 0000000000000..f2ecea347fce9
--- /dev/null
+++ b/openmp/runtime/test/ompt/tasks/dependences_mutexinoutset.c
@@ -0,0 +1,120 @@
+// RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
+// REQUIRES: ompt
+
+// GCC does not pass in mutexinoutset
+// clang 9 introduced codegen for mutexinoutset
+
+// UNSUPPORTED: gcc
+// UNSUPPORTED: clang-4, clang-5, clang-6, clang-7, clang-8
+
+#include "callback.h"
+#include <omp.h>
+#include <math.h>
+#include <unistd.h>
+
+int main() {
+  int x = 0;
+#pragma omp parallel num_threads(2)
+  {
+#pragma omp master
+    {
+      print_ids(0);
+      printf("%" PRIu64 ": address of x: %p\n", ompt_get_thread_data()->value,
+             &x);
+#pragma omp task depend(out : x)
+      {
+        x++;
+        delay(100);
+      }
+      print_fuzzy_address(1);
+      print_ids(0);
+
+#pragma omp task depend(mutexinoutset : x)
+      {
+        x++;
+        delay(100);
+      }
+      print_fuzzy_address(2);
+      print_ids(0);
+
+#pragma omp task depend(in : x)
+      { x = -1; }
+      print_ids(0);
+    }
+  }
+
+  x++;
+
+  return 0;
+}
+
+// Check if libomp supports the callbacks for this test.
+// CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
+// CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_dependences'
+// CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_depende
+
+// CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
+
+// make sure initial data pointers are null
+// CHECK-NOT: 0: new_task_data initially not null
+
+// CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_implicit_task_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]],
+// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+
+// CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]],
+// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT:0x[0-f]+]],
+// CHECK-SAME: reenter_frame=[[NULL]]
+
+// CHECK: {{^}}[[MASTER_ID]]: address of x: [[ADDRX:0x[0-f]+]]
+// CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
+// CHECK-SAME: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[EXIT]],
+// CHECK-SAME: parent_task_frame.reenter={{0x[0-f]+}},
+// CHECK-SAME: new_task_id=[[FIRST_TASK:[0-f]+]],
+// CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:0x[0-f]+]]{{[0-f][0-f]}},
+// CHECK-SAME: task_type=ompt_task_explicit=4, has_dependences=yes
+
+// CHECK: {{^}}[[MASTER_ID]]: ompt_event_dependences:
+// CHECK-SAME: task_id=[[FIRST_TASK]], deps=[([[ADDRX]],
+// CHECK-SAME: ompt_dependence_type_inout)], ndeps=1
+
+// CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
+// CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]],
+// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]],
+// CHECK-SAME: reenter_frame=[[NULL]]
+
+// CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
+// CHECK-SAME: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[EXIT]],
+// CHECK-SAME: parent_task_frame.reenter={{0x[0-f]+}},
+// CHECK-SAME: new_task_id=[[SECOND_TASK:[0-f]+]],
+// CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:0x[0-f]+]]{{[0-f][0-f]}},
+// CHECK-SAME: task_type=ompt_task_explicit=4, has_dependences=yes
+
+// CHECK: {{^}}[[MASTER_ID]]: ompt_event_dependences:
+// CHECK-SAME: task_id=[[SECOND_TASK]], deps=[([[ADDRX]],
+// CHECK-SAME: ompt_dependence_type_mutexinoutset)], ndeps=1
+
+// CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_dependence_pair:
+// CHECK-SAME: first_task_id=[[FIRST_TASK]], second_task_id=[[SECOND_TASK]]
+
+// CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
+// CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]],
+// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]],
+// CHECK-SAME: reenter_frame=[[NULL]]
+
+// CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
+// CHECK-SAME: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[EXIT]],
+// CHECK-SAME: parent_task_frame.reenter={{0x[0-f]+}},
+// CHECK-SAME: new_task_id=[[THIRD_TASK:[0-f]+]], codeptr_ra={{0x[0-f]+}},
+// CHECK-SAME: task_type=ompt_task_explicit=4, has_dependences=yes
+
+// CHECK: {{^}}[[MASTER_ID]]: ompt_event_dependences:
+// CHECK-SAME: task_id=[[THIRD_TASK]], deps=[([[ADDRX]],
+// CHECK-SAME: ompt_dependence_type_in)], ndeps=1
+
+// CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_dependence_pair:
+// CHECK-SAME: first_task_id=[[SECOND_TASK]], second_task_id=[[THIRD_TASK]]
+
+// CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]],
+// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]],
+// CHECK-SAME: reenter_frame=[[NULL]]
diff --git a/openmp/runtime/test/ompt/tasks/task_early_fulfill.c b/openmp/runtime/test/ompt/tasks/task_early_fulfill.c
index c1cef5850b77a..f1d07a1503c84 100644
--- a/openmp/runtime/test/ompt/tasks/task_early_fulfill.c
+++ b/openmp/runtime/test/ompt/tasks/task_early_fulfill.c
@@ -1,8 +1,8 @@
 // RUN: %libomp-compile -fopenmp-version=50 && env OMP_NUM_THREADS='3' \
 // RUN:    %libomp-run | %sort-threads | FileCheck %s
 
-// Checked gcc 9.2 still does not support detach clause on task construct.
-// UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7, gcc-8, gcc-9
+// Checked gcc 10.1 still does not support detach clause on task construct.
+// UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7, gcc-8, gcc-9, gcc-10
 // clang supports detach clause since version 11.
 // UNSUPPORTED: clang-10, clang-9, clang-8, clang-7
 // icc compiler does not support detach clause.
diff --git a/openmp/runtime/test/ompt/tasks/task_late_fulfill.c b/openmp/runtime/test/ompt/tasks/task_late_fulfill.c
index 05e50dfbd2704..4824f3358cfdb 100644
--- a/openmp/runtime/test/ompt/tasks/task_late_fulfill.c
+++ b/openmp/runtime/test/ompt/tasks/task_late_fulfill.c
@@ -1,8 +1,8 @@
 // RUN: %libomp-compile -fopenmp-version=50 && env OMP_NUM_THREADS='3' \
 // RUN:    %libomp-run | %sort-threads | FileCheck %s
 
-// Checked gcc 9.2 still does not support detach clause on task construct.
-// UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7, gcc-8, gcc-9
+// Checked gcc 10.1 still does not support detach clause on task construct.
+// UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7, gcc-8, gcc-9, gcc-10
 // clang supports detach clause since version 11.
 // UNSUPPORTED: clang-10, clang-9, clang-8, clang-7
 // icc compiler does not support detach clause.
@@ -54,6 +54,13 @@ int main() {
 // CHECK-SAME: parallel_id=[[PARALLEL_ID]],
 // CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
 
+// The following is to match the taskwait task created in __kmpc_omp_wait_deps
+// this should go away, once codegen for "detached if(0)" is fixed
+
+// CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
+// CHECK-SAME: parent_task_id=[[IMPLICIT_TASK_ID]],
+// CHECK-SAME: has_dependences=yes
+
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
 // CHECK-SAME: parent_task_id=[[IMPLICIT_TASK_ID]],
 // CHECK-SAME: parent_task_frame.exit=0x{{[0-f]+}},
diff --git a/openmp/runtime/test/ompt/tasks/taskwait-depend.c b/openmp/runtime/test/ompt/tasks/taskwait-depend.c
new file mode 100644
index 0000000000000..38e416e70e32f
--- /dev/null
+++ b/openmp/runtime/test/ompt/tasks/taskwait-depend.c
@@ -0,0 +1,80 @@
+// RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
+// REQUIRES: ompt
+
+// The GOMP wrapper does not handle `task if(0) depend()` and drops the
+// dependency. Once this is fixed, reevaluate the GCC status:
+// XFAIL: gcc-4, gcc-5, gcc-6, gcc-7, gcc-8, gcc-9, gcc-10
+
+#include "callback.h"
+#include <omp.h>
+
+int main() {
+  int x = 0;
+#pragma omp parallel num_threads(2)
+  {
+#pragma omp master
+    {
+      print_ids(0);
+      printf("%" PRIu64 ": address of x: %p\n", ompt_get_thread_data()->value,
+             &x);
+#pragma omp task depend(out : x)
+      { x++; }
+      print_fuzzy_address(1);
+      //#pragma omp taskwait depend(in: x) <-- currently not supported in clang
+#pragma omp task if (0) depend(in : x)
+      {}
+      print_fuzzy_address(2);
+    }
+  }
+
+  return 0;
+}
+
+// Check if libomp supports the callbacks for this test.
+// CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
+// CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_dependences'
+// CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_depende
+
+// CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
+
+// make sure initial data pointers are null
+// CHECK-NOT: 0: new_task_data initially not null
+
+// CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_implicit_task_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]],
+// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+
+// CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]],
+// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT:0x[0-f]+]],
+// CHECK-SAME: reenter_frame=[[NULL]]
+
+// CHECK: {{^}}[[MASTER_ID]]: address of x: [[ADDRX:0x[0-f]+]]
+
+// CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
+// CHECK-SAME: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[EXIT]],
+// CHECK-SAME: parent_task_frame.reenter={{0x[0-f]+}},
+// CHECK-SAME: new_task_id=[[FIRST_TASK:[0-f]+]],
+// CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:0x[0-f]+]]{{[0-f][0-f]}},
+// CHECK-SAME: task_type=ompt_task_explicit=4, has_dependences=yes
+
+// CHECK: {{^}}[[MASTER_ID]]: ompt_event_dependences:
+// CHECK-SAME: task_id=[[FIRST_TASK]], deps=[([[ADDRX]],
+// CHECK-SAME: ompt_dependence_type_inout)], ndeps=1
+
+// CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
+
+// CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
+// CHECK-SAME: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[EXIT]],
+// CHECK-SAME: parent_task_frame.reenter={{0x[0-f]+}},
+// CHECK-SAME: new_task_id=[[SECOND_TASK:[0-f]+]],
+// CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:0x[0-f]+]]{{[0-f][0-f]}},
+// CHECK-SAME: task_type=ompt_task_explicit|ompt_task_undeferred|
+// CHECK-SAME: ompt_task_mergeable=1207959556, has_dependences=yes
+
+// CHECK: {{^}}[[MASTER_ID]]: ompt_event_dependences:
+// CHECK-SAME: task_id=[[SECOND_TASK]], deps=[([[ADDRX]],
+// CHECK-SAME: ompt_dependence_type_in)], ndeps=1
+
+// CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_end: task_id=[[SECOND_TASK]]
+
+// CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
diff --git a/openmp/runtime/test/tasking/omp_detach_taskwait.c b/openmp/runtime/test/tasking/omp_detach_taskwait.c
index 66c9b83641445..c45de2b98c280 100644
--- a/openmp/runtime/test/tasking/omp_detach_taskwait.c
+++ b/openmp/runtime/test/tasking/omp_detach_taskwait.c
@@ -1,8 +1,8 @@
 // RUN: %libomp-compile -fopenmp-version=50 && env OMP_NUM_THREADS='3' %libomp-run
 // RUN: %libomp-compile -fopenmp-version=50 && env OMP_NUM_THREADS='1' %libomp-run
 
-// Checked gcc 9.2 still does not support detach clause on task construct.
-// UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7, gcc-8, gcc-9
+// Checked gcc 10.1 still does not support detach clause on task construct.
+// UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7, gcc-8, gcc-9, gcc-10
 // clang supports detach clause since version 11.
 // UNSUPPORTED: clang-10, clang-9, clang-8, clang-7
 // icc compiler does not support detach clause.
diff --git a/openmp/runtime/test/tasking/omp_task_red_taskloop.c b/openmp/runtime/test/tasking/omp_task_red_taskloop.c
index bcc4b84ef8ce4..35987f6bc0538 100644
--- a/openmp/runtime/test/tasking/omp_task_red_taskloop.c
+++ b/openmp/runtime/test/tasking/omp_task_red_taskloop.c
@@ -1,5 +1,12 @@
 // RUN: %libomp-compile-and-run
 
+// Parsing error until gcc8:
+// UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7, gcc-8
+
+// Missing GOMP_taskgroup_reduction_(un)register in LLVM/OpenMP
+// Should be removed once the functions are implemented
+// XFAIL: gcc-9, gcc-10
+
 #include <stdio.h>
 #include <omp.h>
 
diff --git a/openmp/runtime/test/worksharing/for/bug_set_schedule_0.c b/openmp/runtime/test/worksharing/for/bug_set_schedule_0.c
index 889e2393e2c69..813b31c20c1ee 100644
--- a/openmp/runtime/test/worksharing/for/bug_set_schedule_0.c
+++ b/openmp/runtime/test/worksharing/for/bug_set_schedule_0.c
@@ -1,4 +1,5 @@
 // RUN: %libomp-compile-and-run
+
 #include <stdio.h>
 #include <omp.h>
 #include "omp_testsuite.h"
diff --git a/openmp/runtime/test/worksharing/for/kmp_set_dispatch_buf.c b/openmp/runtime/test/worksharing/for/kmp_set_dispatch_buf.c
index a6378fee9bcaf..9c3fd8b59d208 100644
--- a/openmp/runtime/test/worksharing/for/kmp_set_dispatch_buf.c
+++ b/openmp/runtime/test/worksharing/for/kmp_set_dispatch_buf.c
@@ -3,6 +3,7 @@
 // RUN: %libomp-run 1 && %libomp-run 2 && %libomp-run 5
 // RUN: %libomp-compile -DMY_SCHEDULE=guided && %libomp-run 7
 // RUN: %libomp-run 1 && %libomp-run 2 && %libomp-run 5
+// UNSUPPORTED: clang-11
 #include <stdio.h>
 #include <omp.h>
 #include <stdlib.h>
diff --git a/openmp/runtime/test/worksharing/for/omp_for_schedule_runtime.c b/openmp/runtime/test/worksharing/for/omp_for_schedule_runtime.c
index b957fc38fffa3..27a76567fbe2e 100644
--- a/openmp/runtime/test/worksharing/for/omp_for_schedule_runtime.c
+++ b/openmp/runtime/test/worksharing/for/omp_for_schedule_runtime.c
@@ -10,6 +10,7 @@
 // RUN: env OMP_SCHEDULE=trapezoidal,13 %libomp-run 101 13
 // RUN: env OMP_SCHEDULE=static_steal %libomp-run 102 1
 // RUN: env OMP_SCHEDULE=static_steal,14 %libomp-run 102 14
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <math.h>
diff --git a/openmp/tools/archer/tests/lit.cfg b/openmp/tools/archer/tests/lit.cfg
index aa8a24073568c..ed4ec4d03b69c 100644
--- a/openmp/tools/archer/tests/lit.cfg
+++ b/openmp/tools/archer/tests/lit.cfg
@@ -46,7 +46,8 @@ config.test_flags = " -I " + config.test_source_root + \
     " -I " + config.omp_header_dir + \
     " -L " + config.omp_library_dir + \
     " -Wl,-rpath," + config.omp_library_dir + \
-    " " + config.test_archer_flags
+    " " + config.test_archer_flags + \
+    " " + config.test_extra_flags
 
 config.archer_flags = "-g -O1 -fsanitize=thread"
 
diff --git a/openmp/tools/multiplex/ompt-multiplex.h b/openmp/tools/multiplex/ompt-multiplex.h
index 097b7379b82e4..e16189f9340d9 100644
--- a/openmp/tools/multiplex/ompt-multiplex.h
+++ b/openmp/tools/multiplex/ompt-multiplex.h
@@ -26,6 +26,7 @@
 #include <omp.h>
 #include <stdio.h>
 #include <string.h>
+#include <errno.h>
 
 static ompt_set_callback_t ompt_multiplex_set_callback;
 static ompt_get_task_info_t ompt_multiplex_get_task_info;
@@ -1040,12 +1041,11 @@ ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
   const char *tool_libs = getenv(CLIENT_TOOL_LIBRARIES_VAR);
   if (tool_libs) {
     // copy environement variable
-    char *tool_libs_buffer = (char *)malloc(sizeof(char) * strlen(tool_libs));
+    char *tool_libs_buffer = strdup(tool_libs);
     if (!tool_libs_buffer) {
-      printf("malloc Error\n");
+      printf("strdup Error (%i)\n", errno);
       exit(-1);
     }
-    strcpy(tool_libs_buffer, tool_libs);
 
     int progress = 0;
     while (progress < strlen(tool_libs)) {
diff --git a/openmp/tools/multiplex/tests/custom_data_storage/custom_data_storage.c b/openmp/tools/multiplex/tests/custom_data_storage/custom_data_storage.c
index 8ab2a96c56820..bbd90a654d509 100644
--- a/openmp/tools/multiplex/tests/custom_data_storage/custom_data_storage.c
+++ b/openmp/tools/multiplex/tests/custom_data_storage/custom_data_storage.c
@@ -5,6 +5,10 @@
 // RUN: CUSTOM_DATA_STORAGE_TOOL_LIBRARIES=%t.second.tool.so \
 // RUN: %libomp-run | %sort-threads | FileCheck %s
 
+// For GCC we don't get an event for master,
+// see runtime/test/ompt/sycnchronization/master.c
+// UNSUPPORTED: gcc
+
 #if defined(FIRST_TOOL)
 #include "first-tool.h"
 #elif defined(SECOND_TOOL)
@@ -53,7 +57,7 @@ int main() {
 // CHECK-SAME: parent_task_frame.exit=(nil),
 // CHECK-SAME: parent_task_frame.reenter={{0x[0-f]+}},
 // CHECK-SAME: parallel_id=[[_FIRST_PARALLEL_ID:[0-9]+]], requested_team_size=2,
-// CHECK-SAME: codeptr_ra={{0x[0-f]+}}, invoker=2
+// CHECK-SAME: codeptr_ra={{0x[0-f]+}}, invoker
 
 // CHECK: {{^}}[[_1ST_MSTR_TID]]: _first_tool: ompt_event_implicit_task_begin:
 // CHECK-SAME: parallel_id=[[_FIRST_PARALLEL_ID]],
@@ -135,7 +139,7 @@ int main() {
 
 // CHECK: {{^}}[[_1ST_MSTR_TID]]: _first_tool: ompt_event_parallel_end:
 // CHECK-SAME: parallel_id=[[_FIRST_PARALLEL_ID]],
-// CHECK-SAME: task_id=[[_FIRST_INITIAL_TASK_ID]], invoker=2,
+// CHECK-SAME: task_id=[[_FIRST_INITIAL_TASK_ID]], invoker
 // CHECK-SAME: codeptr_ra={{0x[0-f]+}}
 
 // CHECK: {{^}}[[_1ST_MSTR_TID]]: _first_tool: ompt_event_thread_end:
@@ -155,7 +159,7 @@ int main() {
 // CHECK-SAME: parent_task_frame.exit=(nil),
 // CHECK-SAME: parent_task_frame.reenter={{0x[0-f]+}},
 // CHECK-SAME: parallel_id=[[SECOND_PARALLEL_ID:[0-9]+]], requested_team_size=2,
-// CHECK-SAME: codeptr_ra={{0x[0-f]+}}, invoker=2
+// CHECK-SAME: codeptr_ra={{0x[0-f]+}}, invoker
 
 // CHECK: {{^}}[[_2ND_MSTR_TID]]: second_tool: ompt_event_implicit_task_begin:
 // CHECK-SAME: parallel_id=[[SECOND_PARALLEL_ID]],
@@ -238,7 +242,7 @@ int main() {
 
 // CHECK: {{^}}[[_2ND_MSTR_TID]]: second_tool: ompt_event_parallel_end:
 // CHECK-SAME: parallel_id=[[SECOND_PARALLEL_ID]],
-// CHECK-SAME: task_id=[[SECOND_INITIAL_TASK_ID]], invoker=2,
+// CHECK-SAME: task_id=[[SECOND_INITIAL_TASK_ID]], invoker
 // CHECK-SAME: codeptr_ra={{0x[0-f]+}}
 
 // CHECK: {{^}}[[_2ND_MSTR_TID]]: second_tool: ompt_event_thread_end:
diff --git a/openmp/tools/multiplex/tests/lit.cfg b/openmp/tools/multiplex/tests/lit.cfg
index e792fc365d9d8..52c1000aa0728 100644
--- a/openmp/tools/multiplex/tests/lit.cfg
+++ b/openmp/tools/multiplex/tests/lit.cfg
@@ -47,7 +47,8 @@ config.test_flags = " -I " + config.test_source_root + "/.."\
     " -L " + config.omp_library_dir + \
     " -I " + config.ompt_print_callback_dir + \
     " -Wl,-rpath," + config.omp_library_dir + \
-    " " + config.test_openmp_flags
+    " " + config.test_openmp_flags + \
+    " " + config.test_extra_flags
 
 # Allow XFAIL to work
 config.target_triple = [ ]
diff --git a/openmp/tools/multiplex/tests/print/print.c b/openmp/tools/multiplex/tests/print/print.c
index 08acd0d88d56c..6af93b33eace5 100644
--- a/openmp/tools/multiplex/tests/print/print.c
+++ b/openmp/tools/multiplex/tests/print/print.c
@@ -5,6 +5,10 @@
 // RUN: PRINT_TOOL_LIBRARIES=%t.second.tool.so \
 // RUN: %libomp-run | %sort-threads | FileCheck %s
 
+// For GCC we don't get an event for master,
+// see runtime/test/ompt/sycnchronization/master.c
+// UNSUPPORTED: gcc
+
 #if defined(FIRST_TOOL)
 #include "first-tool.h"
 #elif defined(SECOND_TOOL)
@@ -54,7 +58,7 @@ int main() {
 // CHECK-SAME: parent_task_frame.exit=(nil),
 // CHECK-SAME: parent_task_frame.reenter={{0x[0-f]+}},
 // CHECK-SAME: parallel_id=[[_FIRST_PARALLEL_ID:[0-9]+]],
-// CHECK-SAME: requested_team_size=2, codeptr_ra={{0x[0-f]+}}, invoker=2
+// CHECK-SAME: requested_team_size=2, codeptr_ra={{0x[0-f]+}}, invoker
 
 // CHECK: {{^}}[[_1ST_MSTR_TID]]: _first_tool: ompt_event_implicit_task_begin:
 // CHECK-SAME: parallel_id=[[_FIRST_PARALLEL_ID]],
@@ -134,7 +138,7 @@ int main() {
 
 // CHECK: {{^}}[[_1ST_MSTR_TID]]: _first_tool: ompt_event_parallel_end:
 // CHECK-SAME: parallel_id=[[_FIRST_PARALLEL_ID]],
-// CHECK-SAME: task_id=[[_FIRST_INITIAL_TASK_ID]], invoker=2,
+// CHECK-SAME: task_id=[[_FIRST_INITIAL_TASK_ID]], invoker
 // CHECK-SAME: codeptr_ra={{0x[0-f]+}}
 
 // CHECK: {{^}}[[_1ST_MSTR_TID]]: _first_tool: ompt_event_thread_end:
@@ -152,7 +156,7 @@ int main() {
 // CHECK-SAME: parent_task_frame.exit=(nil),
 // CHECK-SAME: parent_task_frame.reenter={{0x[0-f]+}},
 // CHECK-SAME: parallel_id=[[SECOND_PARALLEL_ID:[0-9]+]],
-// CHECK-SAME: requested_team_size=2, codeptr_ra={{0x[0-f]+}}, invoker=2
+// CHECK-SAME: requested_team_size=2, codeptr_ra={{0x[0-f]+}}, invoker
 
 // CHECK: {{^}}[[_2ND_MSTR_TID]]: second_tool: ompt_event_implicit_task_begin:
 // CHECK-SAME: parallel_id=[[SECOND_PARALLEL_ID]],
@@ -232,7 +236,7 @@ int main() {
 
 // CHECK: {{^}}[[_2ND_MSTR_TID]]: second_tool: ompt_event_parallel_end:
 // CHECK-SAME: parallel_id=[[SECOND_PARALLEL_ID]],
-// CHECK-SAME: task_id=[[SECOND_INITIAL_TASK_ID]], invoker=2,
+// CHECK-SAME: task_id=[[SECOND_INITIAL_TASK_ID]], invoker
 // CHECK-SAME: codeptr_ra={{0x[0-f]+}}
 
 // CHECK: {{^}}[[_2ND_MSTR_TID]]: second_tool: ompt_event_thread_end:
diff --git a/polly/docs/HowToManuallyUseTheIndividualPiecesOfPolly.rst b/polly/docs/HowToManuallyUseTheIndividualPiecesOfPolly.rst
index 958dc96fa9ed8..b32baf0af727c 100644
--- a/polly/docs/HowToManuallyUseTheIndividualPiecesOfPolly.rst
+++ b/polly/docs/HowToManuallyUseTheIndividualPiecesOfPolly.rst
@@ -45,7 +45,7 @@ performance improvement can be expected by an optimal automatic optimizer.
 
         .. code-block:: console
 
-                $ opt -basicaa -polly-ast -analyze matmul.preopt.ll -polly-process-unprofitable -polly-use-llvm-names
+                $ opt -basic-aa -polly-ast -analyze matmul.preopt.ll -polly-process-unprofitable -polly-use-llvm-names
 
         .. code-block:: guess
 
@@ -84,8 +84,8 @@ performance improvement can be expected by an optimal automatic optimizer.
 
         .. code-block:: console
 
-                $ opt -polly-use-llvm-names -basicaa -view-scops -disable-output matmul.preopt.ll
-                $ opt -polly-use-llvm-names -basicaa -view-scops-only -disable-output matmul.preopt.ll
+                $ opt -polly-use-llvm-names -basic-aa -view-scops -disable-output matmul.preopt.ll
+                $ opt -polly-use-llvm-names -basic-aa -view-scops-only -disable-output matmul.preopt.ll
 
         The output for the different functions:
 
@@ -104,7 +104,7 @@ performance improvement can be expected by an optimal automatic optimizer.
 
         .. code-block:: console
 
-                $ opt -polly-use-llvm-names -basicaa -polly-scops -analyze matmul.preopt.ll -polly-process-unprofitable
+                $ opt -polly-use-llvm-names -basic-aa -polly-scops -analyze matmul.preopt.ll -polly-process-unprofitable
 
         .. code-block:: guess
 
@@ -194,7 +194,7 @@ performance improvement can be expected by an optimal automatic optimizer.
 
         .. code-block:: console
 
-	        $ opt -basicaa -polly-use-llvm-names -polly-dependences -analyze matmul.preopt.ll -polly-process-unprofitable
+	        $ opt -basic-aa -polly-use-llvm-names -polly-dependences -analyze matmul.preopt.ll -polly-process-unprofitable
 
         .. code-block:: guess
 
@@ -226,7 +226,7 @@ performance improvement can be expected by an optimal automatic optimizer.
 
         .. code-block:: console
 
-        	$ opt -basicaa -polly-use-llvm-names -polly-export-jscop matmul.preopt.ll -polly-process-unprofitable
+        	$ opt -basic-aa -polly-use-llvm-names -polly-export-jscop matmul.preopt.ll -polly-process-unprofitable
 
         .. code-block:: guess
 
@@ -254,7 +254,7 @@ performance improvement can be expected by an optimal automatic optimizer.
 
 	.. code-block:: console
 
-		$ opt -basicaa -polly-use-llvm-names matmul.preopt.ll -polly-import-jscop -polly-ast -analyze -polly-process-unprofitable
+		$ opt -basic-aa -polly-use-llvm-names matmul.preopt.ll -polly-import-jscop -polly-ast -analyze -polly-process-unprofitable
 
 	.. code-block:: c
 
@@ -282,7 +282,7 @@ performance improvement can be expected by an optimal automatic optimizer.
 
 	.. code-block:: console
 
-		$ opt -basicaa -polly-use-llvm-names matmul.preopt.ll -polly-import-jscop -polly-import-jscop-postfix=interchanged -polly-ast -analyze -polly-process-unprofitable
+		$ opt -basic-aa -polly-use-llvm-names matmul.preopt.ll -polly-import-jscop -polly-import-jscop-postfix=interchanged -polly-ast -analyze -polly-process-unprofitable
 
 	.. code-block:: c
 
@@ -311,7 +311,7 @@ performance improvement can be expected by an optimal automatic optimizer.
 
 	.. code-block:: console
 
-		$ opt -basicaa -polly-use-llvm-names matmul.preopt.ll -polly-import-jscop -polly-import-jscop-postfix=interchanged+tiled -polly-ast -analyze -polly-process-unprofitable
+		$ opt -basic-aa -polly-use-llvm-names matmul.preopt.ll -polly-import-jscop -polly-import-jscop-postfix=interchanged+tiled -polly-ast -analyze -polly-process-unprofitable
 
 	.. code-block:: c
 
@@ -346,7 +346,7 @@ performance improvement can be expected by an optimal automatic optimizer.
 
 	.. code-block:: console
 
-		$ opt -basicaa -polly-use-llvm-names matmul.preopt.ll -polly-import-jscop -polly-import-jscop-postfix=interchanged+tiled -polly-ast -analyze -polly-process-unprofitable
+		$ opt -basic-aa -polly-use-llvm-names matmul.preopt.ll -polly-import-jscop -polly-import-jscop-postfix=interchanged+tiled -polly-ast -analyze -polly-process-unprofitable
 
 	.. code-block:: c
 
@@ -387,7 +387,7 @@ performance improvement can be expected by an optimal automatic optimizer.
 		
 	.. code-block:: console
 
-		$ opt -S matmul.preopt.ll -basicaa -polly-use-llvm-names -polly-import-jscop -polly-import-jscop-postfix=interchanged -polly-codegen -polly-process-unprofitable | opt -S -O3 -o matmul.polly.interchanged.ll
+		$ opt -S matmul.preopt.ll -basic-aa -polly-use-llvm-names -polly-import-jscop -polly-import-jscop-postfix=interchanged -polly-codegen -polly-process-unprofitable | opt -S -O3 -o matmul.polly.interchanged.ll
 
 	.. code-block:: guess
 
@@ -397,7 +397,7 @@ performance improvement can be expected by an optimal automatic optimizer.
 
 	.. code-block:: console
 
-		$ opt -S matmul.preopt.ll -basicaa -polly-use-llvm-names -polly-import-jscop -polly-import-jscop-postfix=interchanged+tiled -polly-codegen -polly-process-unprofitable | opt -S -O3 -o matmul.polly.interchanged+tiled.ll
+		$ opt -S matmul.preopt.ll -basic-aa -polly-use-llvm-names -polly-import-jscop -polly-import-jscop-postfix=interchanged+tiled -polly-codegen -polly-process-unprofitable | opt -S -O3 -o matmul.polly.interchanged+tiled.ll
 		
 	.. code-block:: guess
 
@@ -407,7 +407,7 @@ performance improvement can be expected by an optimal automatic optimizer.
 
 	.. code-block:: console
 
-		$ opt -S matmul.preopt.ll -basicaa -polly-use-llvm-names -polly-import-jscop -polly-import-jscop-postfix=interchanged+tiled+vector -polly-codegen -polly-vectorizer=polly -polly-process-unprofitable | opt -S -O3 -o matmul.polly.interchanged+tiled+vector.ll
+		$ opt -S matmul.preopt.ll -basic-aa -polly-use-llvm-names -polly-import-jscop -polly-import-jscop-postfix=interchanged+tiled+vector -polly-codegen -polly-vectorizer=polly -polly-process-unprofitable | opt -S -O3 -o matmul.polly.interchanged+tiled+vector.ll
 
 	.. code-block:: guess
 
@@ -417,7 +417,7 @@ performance improvement can be expected by an optimal automatic optimizer.
 
 	.. code-block:: console
 
-		$ opt -S matmul.preopt.ll -basicaa -polly-use-llvm-names -polly-import-jscop -polly-import-jscop-postfix=interchanged+tiled+vector -polly-codegen -polly-vectorizer=polly -polly-parallel -polly-process-unprofitable | opt -S -O3 -o matmul.polly.interchanged+tiled+openmp.ll
+		$ opt -S matmul.preopt.ll -basic-aa -polly-use-llvm-names -polly-import-jscop -polly-import-jscop-postfix=interchanged+tiled+vector -polly-codegen -polly-vectorizer=polly -polly-parallel -polly-process-unprofitable | opt -S -O3 -o matmul.polly.interchanged+tiled+openmp.ll
 
 	.. code-block:: guess
 
diff --git a/polly/docs/experiments/matmul/runall.sh b/polly/docs/experiments/matmul/runall.sh
index 011d66bf6b4e1..37dce6878c187 100755
--- a/polly/docs/experiments/matmul/runall.sh
+++ b/polly/docs/experiments/matmul/runall.sh
@@ -7,61 +7,61 @@ echo "--> 2. Prepare the LLVM-IR for Polly"
 opt -S -polly-canonicalize matmul.ll -o matmul.preopt.ll
 
 echo "--> 3. Show the SCoPs detected by Polly"
-opt -basicaa -polly-ast -analyze matmul.preopt.ll \
+opt -basic-aa -polly-ast -analyze matmul.preopt.ll \
     -polly-process-unprofitable -polly-use-llvm-names
 
 echo "--> 4.1 Highlight the detected SCoPs in the CFGs of the program"
 # We only create .dot files, as directly -view-scops directly calls graphviz
 # which would require user interaction to continue the script.
-# opt -basicaa -view-scops -disable-output matmul.preopt.ll
-opt -basicaa -dot-scops -disable-output matmul.preopt.ll -polly-use-llvm-names
+# opt -basic-aa -view-scops -disable-output matmul.preopt.ll
+opt -basic-aa -dot-scops -disable-output matmul.preopt.ll -polly-use-llvm-names
 
 echo "--> 4.2 Highlight the detected SCoPs in the CFGs of the program (print \
 no instructions)"
 # We only create .dot files, as directly -view-scops-only directly calls
 # graphviz which would require user interaction to continue the script.
-# opt -basicaa -view-scops-only -disable-output matmul.preopt.ll
-opt -basicaa -dot-scops-only -disable-output matmul.preopt.ll -polly-use-llvm-names
+# opt -basic-aa -view-scops-only -disable-output matmul.preopt.ll
+opt -basic-aa -dot-scops-only -disable-output matmul.preopt.ll -polly-use-llvm-names
 
 echo "--> 4.3 Create .png files from the .dot files"
 for i in `ls *.dot`; do dot -Tpng $i > $i.png; done
 
 echo "--> 5. View the polyhedral representation of the SCoPs"
-opt -basicaa -polly-scops -analyze matmul.preopt.ll \
+opt -basic-aa -polly-scops -analyze matmul.preopt.ll \
     -polly-process-unprofitable -polly-use-llvm-names
 
 echo "--> 6. Show the dependences for the SCoPs"
-opt -basicaa -polly-dependences -analyze matmul.preopt.ll \
+opt -basic-aa -polly-dependences -analyze matmul.preopt.ll \
     -polly-process-unprofitable -polly-use-llvm-names
 
 echo "--> 7. Export jscop files"
-opt -basicaa -polly-export-jscop matmul.preopt.ll \
+opt -basic-aa -polly-export-jscop matmul.preopt.ll \
     -polly-process-unprofitable -disable-output -polly-use-llvm-names
 
 echo "--> 8. Import the updated jscop files and print the new SCoPs. (optional)"
-opt -basicaa -polly-import-jscop -polly-ast -analyze matmul.preopt.ll \
+opt -basic-aa -polly-import-jscop -polly-ast -analyze matmul.preopt.ll \
     -polly-process-unprofitable -polly-use-llvm-names
-opt -basicaa -polly-import-jscop -polly-ast -analyze matmul.preopt.ll \
+opt -basic-aa -polly-import-jscop -polly-ast -analyze matmul.preopt.ll \
     -polly-import-jscop-postfix=interchanged -polly-process-unprofitable -polly-use-llvm-names
-opt -basicaa -polly-import-jscop -polly-ast -analyze matmul.preopt.ll \
+opt -basic-aa -polly-import-jscop -polly-ast -analyze matmul.preopt.ll \
     -polly-import-jscop-postfix=interchanged+tiled -polly-process-unprofitable -polly-use-llvm-names
-opt -basicaa -polly-import-jscop -polly-ast -analyze matmul.preopt.ll \
+opt -basic-aa -polly-import-jscop -polly-ast -analyze matmul.preopt.ll \
     -polly-import-jscop-postfix=interchanged+tiled+vector \
     -polly-process-unprofitable -polly-use-llvm-names
 
 echo "--> 9. Codegenerate the SCoPs"
-opt -S -basicaa -polly-import-jscop -polly-import-jscop-postfix=interchanged \
+opt -S -basic-aa -polly-import-jscop -polly-import-jscop-postfix=interchanged \
     -polly-codegen -polly-process-unprofitable -polly-use-llvm-names \
     matmul.preopt.ll | opt -O3 -S -o matmul.polly.interchanged.ll
-opt -S -basicaa -polly-import-jscop \
+opt -S -basic-aa -polly-import-jscop \
     -polly-import-jscop-postfix=interchanged+tiled -polly-codegen \
     matmul.preopt.ll -polly-process-unprofitable -polly-use-llvm-names \
     | opt -O3 -S -o matmul.polly.interchanged+tiled.ll
-opt -S -basicaa -polly-import-jscop -polly-process-unprofitable\
+opt -S -basic-aa -polly-import-jscop -polly-process-unprofitable\
     -polly-import-jscop-postfix=interchanged+tiled+vector -polly-codegen \
     matmul.preopt.ll -polly-vectorizer=polly -polly-use-llvm-names \
     | opt -O3 -S -o matmul.polly.interchanged+tiled+vector.ll
-opt -S -basicaa -polly-import-jscop -polly-process-unprofitable\
+opt -S -basic-aa -polly-import-jscop -polly-process-unprofitable\
     -polly-import-jscop-postfix=interchanged+tiled+vector -polly-codegen \
     matmul.preopt.ll -polly-vectorizer=polly -polly-parallel -polly-use-llvm-names \
     | opt -O3 -S -o matmul.polly.interchanged+tiled+vector+openmp.ll
diff --git a/polly/lib/Analysis/ScopDetection.cpp b/polly/lib/Analysis/ScopDetection.cpp
index abe189f3e890a..53d0b705c055d 100644
--- a/polly/lib/Analysis/ScopDetection.cpp
+++ b/polly/lib/Analysis/ScopDetection.cpp
@@ -383,7 +383,7 @@ ScopDetection::ScopDetection(Function &F, const DominatorTree &DT,
 
 template <class RR, typename... Args>
 inline bool ScopDetection::invalid(DetectionContext &Context, bool Assert,
-                                   Args &&... Arguments) const {
+                                   Args &&...Arguments) const {
   if (!Context.Verifying) {
     RejectLog &Log = Context.Log;
     std::shared_ptr<RR> RejectReason = std::make_shared<RR>(Arguments...);
diff --git a/polly/lib/Transform/ScheduleOptimizer.cpp b/polly/lib/Transform/ScheduleOptimizer.cpp
index d9dffb8f1f11d..3806707259fd0 100644
--- a/polly/lib/Transform/ScheduleOptimizer.cpp
+++ b/polly/lib/Transform/ScheduleOptimizer.cpp
@@ -385,8 +385,8 @@ ScheduleTreeOptimizer::isolateFullPartialTiles(isl::schedule_node Node,
   assert(isl_schedule_node_get_type(Node.get()) == isl_schedule_node_band);
   Node = Node.child(0).child(0);
   isl::union_map SchedRelUMap = Node.get_prefix_schedule_relation();
-  isl::map ScheduleRelation = isl::map::from_union_map(SchedRelUMap);
-  isl::set ScheduleRange = ScheduleRelation.range();
+  isl::union_set ScheduleRangeUSet = SchedRelUMap.range();
+  isl::set ScheduleRange{ScheduleRangeUSet};
   isl::set IsolateDomain = getPartialTilePrefixes(ScheduleRange, VectorWidth);
   auto AtomicOption = getDimOptions(IsolateDomain.get_ctx(), "atomic");
   isl::union_set IsolateOption = getIsolateOptions(IsolateDomain, 1);
@@ -1389,7 +1389,7 @@ class IslScheduleOptimizer : public ScopPass {
 
   explicit IslScheduleOptimizer() : ScopPass(ID) {}
 
-  ~IslScheduleOptimizer() override { isl_schedule_free(LastSchedule); }
+  ~IslScheduleOptimizer() override { releaseMemory(); }
 
   /// Optimize the schedule of the SCoP @p S.
   bool runOnScop(Scop &S) override;
@@ -1404,9 +1404,11 @@ class IslScheduleOptimizer : public ScopPass {
   void releaseMemory() override {
     isl_schedule_free(LastSchedule);
     LastSchedule = nullptr;
+    IslCtx.reset();
   }
 
 private:
+  std::shared_ptr<isl_ctx> IslCtx;
   isl_schedule *LastSchedule = nullptr;
 };
 } // namespace
@@ -1630,6 +1632,8 @@ bool IslScheduleOptimizer::runOnScop(Scop &S) {
   ScopsOptimized++;
   NumAffineLoopsOptimized += ScopStats.NumAffineLoops;
   NumBoxedLoopsOptimized += ScopStats.NumBoxedLoops;
+  LastSchedule = NewSchedule.copy();
+  IslCtx = S.getSharedIslCtx();
 
   S.setScheduleTree(NewSchedule);
   S.markAsOptimized();
@@ -1652,6 +1656,7 @@ void IslScheduleOptimizer::printScop(raw_ostream &OS, Scop &) const {
   }
 
   p = isl_printer_to_str(isl_schedule_get_ctx(LastSchedule));
+  p = isl_printer_set_yaml_style(p, ISL_YAML_STYLE_BLOCK);
   p = isl_printer_print_schedule(p, LastSchedule);
   ScheduleStr = isl_printer_get_str(p);
   isl_printer_free(p);
diff --git a/polly/lib/Transform/ScopInliner.cpp b/polly/lib/Transform/ScopInliner.cpp
index 8342a0588f1e3..3d0737f13cde6 100644
--- a/polly/lib/Transform/ScopInliner.cpp
+++ b/polly/lib/Transform/ScopInliner.cpp
@@ -15,6 +15,7 @@
 
 #include "polly/LinkAllPasses.h"
 #include "polly/ScopDetection.h"
+#include "llvm/Analysis/CallGraph.h"
 #include "llvm/Analysis/CallGraphSCCPass.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Passes/PassBuilder.h"
diff --git a/polly/test/DeadCodeElimination/chained_iterations.ll b/polly/test/DeadCodeElimination/chained_iterations.ll
index 26ffc3ce0f240..e798f36b1753e 100644
--- a/polly/test/DeadCodeElimination/chained_iterations.ll
+++ b/polly/test/DeadCodeElimination/chained_iterations.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S %loadPolly -basicaa -polly-dependences-analysis-type=value-based -polly-ast -analyze < %s | FileCheck %s
-; RUN: opt -S %loadPolly -basicaa -polly-dependences-analysis-type=value-based -polly-dce -polly-ast -analyze < %s | FileCheck %s -check-prefix=CHECK-DCE
+; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-ast -analyze < %s | FileCheck %s
+; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-dce -polly-ast -analyze < %s | FileCheck %s -check-prefix=CHECK-DCE
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 ;
 ; for(i = 0; i < 200; i++ )
diff --git a/polly/test/DeadCodeElimination/chained_iterations_2.ll b/polly/test/DeadCodeElimination/chained_iterations_2.ll
index 42e0617eebda2..7f454daabf7f7 100644
--- a/polly/test/DeadCodeElimination/chained_iterations_2.ll
+++ b/polly/test/DeadCodeElimination/chained_iterations_2.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S %loadPolly -basicaa -polly-dependences-analysis-type=value-based -polly-ast -analyze < %s | FileCheck %s
-; RUN: opt -S %loadPolly -basicaa -polly-dependences-analysis-type=value-based -polly-dce -polly-ast -analyze < %s | FileCheck %s -check-prefix=CHECK-DCE
+; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-ast -analyze < %s | FileCheck %s
+; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-dce -polly-ast -analyze < %s | FileCheck %s -check-prefix=CHECK-DCE
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 ;
 ; for(i = 0; i < 200; i++ )
diff --git a/polly/test/DeadCodeElimination/computeout.ll b/polly/test/DeadCodeElimination/computeout.ll
index 21d3c02c25b6b..6b40a09c88076 100644
--- a/polly/test/DeadCodeElimination/computeout.ll
+++ b/polly/test/DeadCodeElimination/computeout.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S %loadPolly -basicaa -polly-dce -polly-ast -analyze < %s | FileCheck %s
-; RUN: opt -S %loadPolly -basicaa -polly-dce -polly-ast -analyze -polly-dependences-computeout=1 < %s | FileCheck %s -check-prefix=TIMEOUT
+; RUN: opt -S %loadPolly -basic-aa -polly-dce -polly-ast -analyze < %s | FileCheck %s
+; RUN: opt -S %loadPolly -basic-aa -polly-dce -polly-ast -analyze -polly-dependences-computeout=1 < %s | FileCheck %s -check-prefix=TIMEOUT
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ;     for(i = 0; i < 100; i++ )
diff --git a/polly/test/DeadCodeElimination/dead_iteration_elimination.ll b/polly/test/DeadCodeElimination/dead_iteration_elimination.ll
index b9d935c437ca5..cc9851ffab870 100644
--- a/polly/test/DeadCodeElimination/dead_iteration_elimination.ll
+++ b/polly/test/DeadCodeElimination/dead_iteration_elimination.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S %loadPolly -basicaa -polly-dependences-analysis-type=value-based -polly-dce -polly-dce-precise-steps=2 -polly-ast -analyze < %s | FileCheck %s
+; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-dce -polly-dce-precise-steps=2 -polly-ast -analyze < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 ;
 ; for(i = 0; i < 200; i++ )
diff --git a/polly/test/DeadCodeElimination/null_schedule.ll b/polly/test/DeadCodeElimination/null_schedule.ll
index 26d7b4f407f98..c8ccefdac608e 100644
--- a/polly/test/DeadCodeElimination/null_schedule.ll
+++ b/polly/test/DeadCodeElimination/null_schedule.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S %loadPolly -basicaa -polly-dependences-analysis-type=value-based -polly-dce -polly-ast -analyze < %s | FileCheck %s -check-prefix=CHECK-DCE
+; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-dce -polly-ast -analyze < %s | FileCheck %s -check-prefix=CHECK-DCE
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 ; A[0] = 1;
 ;
diff --git a/polly/test/DependenceInfo/do_pluto_matmult.ll b/polly/test/DependenceInfo/do_pluto_matmult.ll
index 20cc90e40aed8..4cdcd43adc2cd 100644
--- a/polly/test/DependenceInfo/do_pluto_matmult.ll
+++ b/polly/test/DependenceInfo/do_pluto_matmult.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -basicaa -polly-dependences -analyze -polly-dependences-analysis-type=value-based < %s | FileCheck %s -check-prefix=VALUE
-; RUN: opt %loadPolly -basicaa -polly-dependences -analyze -polly-dependences-analysis-type=memory-based < %s | FileCheck %s -check-prefix=MEMORY
-; RUN: opt %loadPolly -basicaa -polly-function-dependences -analyze -polly-dependences-analysis-type=value-based < %s | FileCheck %s -check-prefix=FUNC-VALUE
-; RUN: opt %loadPolly -basicaa -polly-function-dependences -analyze -polly-dependences-analysis-type=memory-based < %s | FileCheck %s -check-prefix=FUNC-MEMORY
+; RUN: opt %loadPolly -basic-aa -polly-dependences -analyze -polly-dependences-analysis-type=value-based < %s | FileCheck %s -check-prefix=VALUE
+; RUN: opt %loadPolly -basic-aa -polly-dependences -analyze -polly-dependences-analysis-type=memory-based < %s | FileCheck %s -check-prefix=MEMORY
+; RUN: opt %loadPolly -basic-aa -polly-function-dependences -analyze -polly-dependences-analysis-type=value-based < %s | FileCheck %s -check-prefix=FUNC-VALUE
+; RUN: opt %loadPolly -basic-aa -polly-function-dependences -analyze -polly-dependences-analysis-type=memory-based < %s | FileCheck %s -check-prefix=FUNC-MEMORY
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/DependenceInfo/reduction_dependences_equal_non_reduction_dependences.ll b/polly/test/DependenceInfo/reduction_dependences_equal_non_reduction_dependences.ll
index 36d06f6026a5d..b9be9529ef05f 100644
--- a/polly/test/DependenceInfo/reduction_dependences_equal_non_reduction_dependences.ll
+++ b/polly/test/DependenceInfo/reduction_dependences_equal_non_reduction_dependences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-stmt-granularity=bb -polly-dependences -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-dependences -analyze < %s | FileCheck %s
 ;
 ; This loopnest contains a reduction which imposes the same dependences as the
 ; accesses to the array A. We need to ensure we keep the dependences of A.
diff --git a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum.ll b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum.ll
index db1bdca5b50ee..1e3fc97a236c6 100644
--- a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum.ll
+++ b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum.ll
@@ -1,6 +1,6 @@
-; RUN: opt -basicaa %loadPolly -polly-dependences -analyze < %s | FileCheck %s
-; RUN: opt -basicaa %loadPolly -polly-dependences -polly-dependences-analysis-level=reference-wise -analyze < %s | FileCheck %s
-; RUN: opt -basicaa %loadPolly -polly-dependences -polly-dependences-analysis-level=access-wise -analyze < %s | FileCheck %s
+; RUN: opt -basic-aa %loadPolly -polly-dependences -analyze < %s | FileCheck %s
+; RUN: opt -basic-aa %loadPolly -polly-dependences -polly-dependences-analysis-level=reference-wise -analyze < %s | FileCheck %s
+; RUN: opt -basic-aa %loadPolly -polly-dependences -polly-dependences-analysis-level=access-wise -analyze < %s | FileCheck %s
 ;
 ; Verify that only the inner reduction like accesses cause reduction dependences
 ;
diff --git a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_2.ll b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_2.ll
index 3b641a151a01c..b77eab059acc6 100644
--- a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_2.ll
+++ b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-dependences -analyze -basicaa < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-dependences -analyze -basic-aa < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     {  }
diff --git a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_3.ll b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_3.ll
index 2ccb6e7fc7e8d..8914e2cbb308d 100644
--- a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_3.ll
+++ b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-dependences -analyze -basicaa < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-dependences -analyze -basic-aa < %s | FileCheck %s
 ;
 ; CHECK:      Reduction dependences:
 ; CHECK-NEXT:     { Stmt_for_inc[i0, i1] -> Stmt_for_inc[i0, 1 + i1] : 0 <= i0 <= 99 and 0 <= i1 <= 98 }
diff --git a/polly/test/DependenceInfo/reduction_multiple_reductions.ll b/polly/test/DependenceInfo/reduction_multiple_reductions.ll
index b838d78ed05a4..bc8aa3e5dcb1c 100644
--- a/polly/test/DependenceInfo/reduction_multiple_reductions.ll
+++ b/polly/test/DependenceInfo/reduction_multiple_reductions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-dependences -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-dependences -analyze < %s | FileCheck %s
 ;
 ; Verify we do not have dependences between the if and the else clause
 ;
diff --git a/polly/test/DependenceInfo/reduction_multiple_reductions_2.ll b/polly/test/DependenceInfo/reduction_multiple_reductions_2.ll
index 98c3ee6549c0b..6da5d4a9d05a6 100644
--- a/polly/test/DependenceInfo/reduction_multiple_reductions_2.ll
+++ b/polly/test/DependenceInfo/reduction_multiple_reductions_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-dependences -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-dependences -analyze < %s | FileCheck %s
 ;
 ;
 ; These are the important RAW dependences, as they need to originate/end in only one iteration:
diff --git a/polly/test/DependenceInfo/reduction_partially_escaping_intermediate_in_other_stmt.ll b/polly/test/DependenceInfo/reduction_partially_escaping_intermediate_in_other_stmt.ll
index 801992883f649..da06e0b9a3155 100644
--- a/polly/test/DependenceInfo/reduction_partially_escaping_intermediate_in_other_stmt.ll
+++ b/polly/test/DependenceInfo/reduction_partially_escaping_intermediate_in_other_stmt.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-dependences -analyze -basicaa < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-dependences -analyze -basic-aa < %s | FileCheck %s
 ;
 ; CHECK:      Reduction dependences:
 ; CHECK-NEXT:     [N] -> { Stmt_for_body3[i0, i1] -> Stmt_for_body3[i0, 1 + i1] : 0 <= i0 <= 1023 and i1 >= 0 and 1024 - N + i0 <= i1 <= 1022 }
diff --git a/polly/test/DependenceInfo/reduction_two_reductions_different_rloops.ll b/polly/test/DependenceInfo/reduction_two_reductions_different_rloops.ll
index c5e6b9106d795..e73be939795a3 100644
--- a/polly/test/DependenceInfo/reduction_two_reductions_different_rloops.ll
+++ b/polly/test/DependenceInfo/reduction_two_reductions_different_rloops.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-stmt-granularity=bb -polly-dependences -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-dependences -analyze < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     {  }
diff --git a/polly/test/DependenceInfo/sequential_loops.ll b/polly/test/DependenceInfo/sequential_loops.ll
index 1e81ec805a406..5857e1df23ebf 100644
--- a/polly/test/DependenceInfo/sequential_loops.ll
+++ b/polly/test/DependenceInfo/sequential_loops.ll
@@ -1,6 +1,6 @@
-; RUN: opt -S %loadPolly -basicaa -polly-dependences -analyze -polly-dependences-analysis-type=value-based < %s | FileCheck %s -check-prefix=VALUE
-; RUN: opt -S %loadPolly -basicaa -polly-dependences -analyze -polly-dependences-analysis-type=memory-based < %s | FileCheck %s -check-prefix=MEMORY
-; RUN: opt -S %loadPolly -basicaa -polly-dependences -analyze -polly-dependences-analysis-type=value-based -polly-dependences-analysis-level=access-wise < %s | FileCheck %s -check-prefix=VALUE_ACCESS
+; RUN: opt -S %loadPolly -basic-aa -polly-dependences -analyze -polly-dependences-analysis-type=value-based < %s | FileCheck %s -check-prefix=VALUE
+; RUN: opt -S %loadPolly -basic-aa -polly-dependences -analyze -polly-dependences-analysis-type=memory-based < %s | FileCheck %s -check-prefix=MEMORY
+; RUN: opt -S %loadPolly -basic-aa -polly-dependences -analyze -polly-dependences-analysis-type=value-based -polly-dependences-analysis-level=access-wise < %s | FileCheck %s -check-prefix=VALUE_ACCESS
 
 ; VALUE-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'sequential_writes':
 ; VALUE-NEXT:      RAW dependences:
diff --git a/polly/test/Isl/Ast/alias_simple_1.ll b/polly/test/Isl/Ast/alias_simple_1.ll
index 19037495280f9..57bdb99327c5e 100644
--- a/polly/test/Isl/Ast/alias_simple_1.ll
+++ b/polly/test/Isl/Ast/alias_simple_1.ll
@@ -1,8 +1,8 @@
-; RUN: opt %loadPolly -polly-ast -analyze -disable-basicaa < %s | FileCheck %s --check-prefix=NOAA
+; RUN: opt %loadPolly -polly-ast -analyze -disable-basic-aa < %s | FileCheck %s --check-prefix=NOAA
 ; RUN: opt %loadPolly -polly-ast -analyze < %s | FileCheck %s --check-prefix=BASI
-; RUN: opt %loadPolly -polly-ast -analyze -disable-basicaa -tbaa < %s | FileCheck %s --check-prefix=TBAA
-; RUN: opt %loadPolly -polly-ast -analyze -disable-basicaa -scev-aa < %s | FileCheck %s --check-prefix=SCEV
-; RUN: opt %loadPolly -polly-ast -analyze -disable-basicaa -globals-aa < %s | FileCheck %s --check-prefix=GLOB
+; RUN: opt %loadPolly -polly-ast -analyze -disable-basic-aa -tbaa < %s | FileCheck %s --check-prefix=TBAA
+; RUN: opt %loadPolly -polly-ast -analyze -disable-basic-aa -scev-aa < %s | FileCheck %s --check-prefix=SCEV
+; RUN: opt %loadPolly -polly-ast -analyze -disable-basic-aa -globals-aa < %s | FileCheck %s --check-prefix=GLOB
 ;
 ;    int A[1024];
 ;
diff --git a/polly/test/Isl/Ast/alias_simple_2.ll b/polly/test/Isl/Ast/alias_simple_2.ll
index b0d957388f5f1..7bc2e975773e2 100644
--- a/polly/test/Isl/Ast/alias_simple_2.ll
+++ b/polly/test/Isl/Ast/alias_simple_2.ll
@@ -1,9 +1,9 @@
-; RUN: opt %loadPolly -polly-ast -analyze -disable-basicaa < %s | FileCheck %s --check-prefix=NOAA
+; RUN: opt %loadPolly -polly-ast -analyze -disable-basic-aa < %s | FileCheck %s --check-prefix=NOAA
 ; RUN: opt %loadPolly -polly-ast -analyze < %s | FileCheck %s --check-prefix=BASI
-; RUN: opt %loadPolly -polly-ast -analyze -disable-basicaa -tbaa < %s | FileCheck %s --check-prefix=TBAA
-; RUN: opt %loadPolly -polly-ast -analyze -disable-basicaa -scev-aa < %s | FileCheck %s --check-prefix=SCEV
-; RUN: opt %loadPolly -polly-ast -analyze -disable-basicaa -globals-aa < %s | FileCheck %s --check-prefix=GLOB
-; RUN: opt %loadPolly -polly-ast -analyze -disable-basicaa -globals-aa -polly-allow-nonaffine < %s | FileCheck %s --check-prefix=NONAFFINE
+; RUN: opt %loadPolly -polly-ast -analyze -disable-basic-aa -tbaa < %s | FileCheck %s --check-prefix=TBAA
+; RUN: opt %loadPolly -polly-ast -analyze -disable-basic-aa -scev-aa < %s | FileCheck %s --check-prefix=SCEV
+; RUN: opt %loadPolly -polly-ast -analyze -disable-basic-aa -globals-aa < %s | FileCheck %s --check-prefix=GLOB
+; RUN: opt %loadPolly -polly-ast -analyze -disable-basic-aa -globals-aa -polly-allow-nonaffine < %s | FileCheck %s --check-prefix=NONAFFINE
 ;
 ;    int A[1024], B[1024];
 ;
diff --git a/polly/test/Isl/Ast/alias_simple_3.ll b/polly/test/Isl/Ast/alias_simple_3.ll
index a2c0624e899b9..1cdf7958fa097 100644
--- a/polly/test/Isl/Ast/alias_simple_3.ll
+++ b/polly/test/Isl/Ast/alias_simple_3.ll
@@ -1,8 +1,8 @@
-; RUN: opt %loadPolly -polly-ast -analyze -disable-basicaa < %s | FileCheck %s --check-prefix=NOAA
+; RUN: opt %loadPolly -polly-ast -analyze -disable-basic-aa < %s | FileCheck %s --check-prefix=NOAA
 ; RUN: opt %loadPolly -polly-ast -analyze < %s | FileCheck %s --check-prefix=BASI
-; RUN: opt %loadPolly -polly-ast -analyze -disable-basicaa -tbaa < %s | FileCheck %s --check-prefix=TBAA
-; RUN: opt %loadPolly -polly-ast -analyze -disable-basicaa -scev-aa < %s | FileCheck %s --check-prefix=SCEV
-; RUN: opt %loadPolly -polly-ast -analyze -disable-basicaa -globals-aa < %s | FileCheck %s --check-prefix=GLOB
+; RUN: opt %loadPolly -polly-ast -analyze -disable-basic-aa -tbaa < %s | FileCheck %s --check-prefix=TBAA
+; RUN: opt %loadPolly -polly-ast -analyze -disable-basic-aa -scev-aa < %s | FileCheck %s --check-prefix=SCEV
+; RUN: opt %loadPolly -polly-ast -analyze -disable-basic-aa -globals-aa < %s | FileCheck %s --check-prefix=GLOB
 ;
 ;    int A[1024];
 ;    float B[1024];
diff --git a/polly/test/Isl/Ast/dependence_distance_multiple_constant.ll b/polly/test/Isl/Ast/dependence_distance_multiple_constant.ll
index b7b5c25e05fb6..d4a43c691f800 100644
--- a/polly/test/Isl/Ast/dependence_distance_multiple_constant.ll
+++ b/polly/test/Isl/Ast/dependence_distance_multiple_constant.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-stmt-granularity=bb -polly-ast -polly-ast-detect-parallel -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-ast -polly-ast-detect-parallel -analyze < %s | FileCheck %s
 ; RUN: opt %loadPolly -polly-stmt-granularity=bb -polyhedral-info -polly-check-parallel -analyze < %s | FileCheck %s -check-prefix=PINFO
 ;
 ;        void f(int *restrict A, int *restrict B, int N) {
diff --git a/polly/test/Isl/Ast/dependence_distance_varying_multiple.ll b/polly/test/Isl/Ast/dependence_distance_varying_multiple.ll
index fff6ec420696f..cc60f3fe6bdf5 100644
--- a/polly/test/Isl/Ast/dependence_distance_varying_multiple.ll
+++ b/polly/test/Isl/Ast/dependence_distance_varying_multiple.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-stmt-granularity=bb -polly-ast -polly-ast-detect-parallel -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-ast -polly-ast-detect-parallel -analyze < %s | FileCheck %s
 ; RUN: opt %loadPolly -polly-stmt-granularity=bb -polyhedral-info -polly-check-parallel -analyze < %s | FileCheck %s -check-prefix=PINFO
 ;
 ;        void f(int *restrict A, int *restrict B, int *restrict C, int *restrict D,
diff --git a/polly/test/Isl/Ast/reduction_dependences_equal_non_reduction_dependences.ll b/polly/test/Isl/Ast/reduction_dependences_equal_non_reduction_dependences.ll
index ad39f4319b569..d1865d84cc667 100644
--- a/polly/test/Isl/Ast/reduction_dependences_equal_non_reduction_dependences.ll
+++ b/polly/test/Isl/Ast/reduction_dependences_equal_non_reduction_dependences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-ast -polly-ast-detect-parallel -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-ast -polly-ast-detect-parallel -analyze < %s | FileCheck %s
 ;
 ; This loopnest contains a reduction which imposes the same dependences as the
 ; accesses to the array A. We need to ensure we do __not__ parallelize anything
diff --git a/polly/test/Isl/Ast/reduction_different_reduction_clauses.ll b/polly/test/Isl/Ast/reduction_different_reduction_clauses.ll
index b423a47d8ef29..a7eda18a33791 100644
--- a/polly/test/Isl/Ast/reduction_different_reduction_clauses.ll
+++ b/polly/test/Isl/Ast/reduction_different_reduction_clauses.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-ast -polly-ast-detect-parallel -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-ast -polly-ast-detect-parallel -analyze < %s | FileCheck %s
 ;
 ; CHECK: #pragma simd reduction (+ : MemRef_sum{{[1,2]}}, MemRef_sum{{[1,2]}}) reduction (* : MemRef_prod) reduction (| : MemRef_or) reduction (& : MemRef_and)
 ; CHECK: #pragma known-parallel reduction (+ : MemRef_sum{{[1,2]}}, MemRef_sum{{[1,2]}}) reduction (* : MemRef_prod) reduction (| : MemRef_or) reduction (& : MemRef_and)
diff --git a/polly/test/Isl/Ast/run-time-condition.ll b/polly/test/Isl/Ast/run-time-condition.ll
index def7d91e929ee..272f2bc090c28 100644
--- a/polly/test/Isl/Ast/run-time-condition.ll
+++ b/polly/test/Isl/Ast/run-time-condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-ast -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-ast -analyze < %s | FileCheck %s
 
 ; for (i = 0; i < 1024; i++)
 ;   A[i] = B[i];
diff --git a/polly/test/Isl/Ast/single_loop_strip_mine.ll b/polly/test/Isl/Ast/single_loop_strip_mine.ll
index e4e7574daad2f..79a584e16c5d5 100644
--- a/polly/test/Isl/Ast/single_loop_strip_mine.ll
+++ b/polly/test/Isl/Ast/single_loop_strip_mine.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basicaa -polly-ast -analyze < %s | FileCheck %s
-; RUN: opt %loadPolly -basicaa -polly-import-jscop \
+; RUN: opt %loadPolly -basic-aa -polly-ast -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-import-jscop \
 ; RUN:   -polly-ast-print-accesses \
 ; RUN:   -polly-ast -polly-ast-detect-parallel -analyze < %s | FileCheck %s \
 ; RUN:   -check-prefix=CHECK-VECTOR
diff --git a/polly/test/Isl/CodeGen/MemAccess/default_aligned_new_access_function.ll b/polly/test/Isl/CodeGen/MemAccess/default_aligned_new_access_function.ll
index 25f1c4daa4c62..f1ede7058cc13 100644
--- a/polly/test/Isl/CodeGen/MemAccess/default_aligned_new_access_function.ll
+++ b/polly/test/Isl/CodeGen/MemAccess/default_aligned_new_access_function.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-import-jscop -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-import-jscop -analyze < %s | FileCheck %s
 ;
 ; Check that we allow the new access functions even though they access
 ; different locations than the original ones (but the alignment is the
diff --git a/polly/test/Isl/CodeGen/MemAccess/generate-all.ll b/polly/test/Isl/CodeGen/MemAccess/generate-all.ll
index a3253ef2d168d..f9b07a947c6ae 100644
--- a/polly/test/Isl/CodeGen/MemAccess/generate-all.ll
+++ b/polly/test/Isl/CodeGen/MemAccess/generate-all.ll
@@ -9,11 +9,12 @@
 ;    }
 
 ; SCEV:      polly.stmt.bb2:                                   ; preds = %polly.loop_header
-; SCEV-NEXT:   %p_tmp = srem i64 %polly.indvar, 4
-; SCEV-NEXT:   %p_tmp3 = getelementptr inbounds float, float* %A, i64 %p_tmp
-; SCEV-NEXT:   %tmp4_p_scalar_ = load float, float* %p_tmp3, align 4, !alias.scope !0, !noalias !2
+; SCEV-NEXT:   %0 = trunc i64 %polly.indvar to i2
+; SCEV-NEXT:   %1 = zext i2 %0 to i64
+; SCEV-NEXT:   %scevgep = getelementptr float, float* %A, i64 %1
+; SCEV-NEXT:   %tmp4_p_scalar_ = load float, float* %scevgep, align 4, !alias.scope !0, !noalias !2
 ; SCEV-NEXT:   %p_tmp5 = fadd float %tmp4_p_scalar_, 1.000000e+01
-; SCEV-NEXT:   store float %p_tmp5, float* %p_tmp3, align 4, !alias.scope !0, !noalias !2
+; SCEV-NEXT:   store float %p_tmp5, float* %scevgep, align 4, !alias.scope !0, !noalias !2
 ; SCEV-NEXT:   %polly.indvar_next = add nsw i64 %polly.indvar, 1
 ; SCEV-NEXT:   %polly.loop_cond = icmp sle i64 %polly.indvar_next, 99
 ; SCEV-NEXT:   br i1 %polly.loop_cond, label %polly.loop_header, label %polly.loop_exit
diff --git a/polly/test/Isl/CodeGen/MemAccess/simple_stride_test.ll b/polly/test/Isl/CodeGen/MemAccess/simple_stride_test.ll
index e82b7dec98fb2..69de07142a93a 100644
--- a/polly/test/Isl/CodeGen/MemAccess/simple_stride_test.ll
+++ b/polly/test/Isl/CodeGen/MemAccess/simple_stride_test.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-import-jscop -polly-codegen -polly-vectorizer=polly -S < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-import-jscop -polly-codegen -polly-vectorizer=polly -S < %s | FileCheck %s
 ;
 ; Check that we use the correct __new__ strides:
 ;    stride zero for B
diff --git a/polly/test/Isl/CodeGen/OpenMP/loop-body-references-outer-values-3.ll b/polly/test/Isl/CodeGen/OpenMP/loop-body-references-outer-values-3.ll
index d64855a2f7fc3..85fdc03b69fb4 100644
--- a/polly/test/Isl/CodeGen/OpenMP/loop-body-references-outer-values-3.ll
+++ b/polly/test/Isl/CodeGen/OpenMP/loop-body-references-outer-values-3.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basicaa -polly-parallel -polly-parallel-force -polly-invariant-load-hoisting=true -polly-ast -analyze < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -basicaa -polly-parallel -polly-parallel-force -polly-invariant-load-hoisting=true -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -basic-aa -polly-parallel -polly-parallel-force -polly-invariant-load-hoisting=true -polly-ast -analyze < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -basic-aa -polly-parallel -polly-parallel-force -polly-invariant-load-hoisting=true -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
 ; The interesting part of this test case is the instruction:
 ;   %tmp = bitcast i8* %call to i64**
diff --git a/polly/test/Isl/CodeGen/create-conditional-scop.ll b/polly/test/Isl/CodeGen/create-conditional-scop.ll
index 44b2042c3d959..df58477731457 100644
--- a/polly/test/Isl/CodeGen/create-conditional-scop.ll
+++ b/polly/test/Isl/CodeGen/create-conditional-scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-codegen -verify-loop-info < %s -S | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-codegen -verify-loop-info < %s -S | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-a0:0-n32"
 
diff --git a/polly/test/Isl/CodeGen/intrinsics_lifetime.ll b/polly/test/Isl/CodeGen/intrinsics_lifetime.ll
index 65d5f9c9898b1..9aea62b87d82d 100644
--- a/polly/test/Isl/CodeGen/intrinsics_lifetime.ll
+++ b/polly/test/Isl/CodeGen/intrinsics_lifetime.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-codegen -S < %s | FileCheck %s
 ;
 ; Verify that we remove the lifetime markers from everywhere.
 ;
diff --git a/polly/test/Isl/CodeGen/intrinsics_misc.ll b/polly/test/Isl/CodeGen/intrinsics_misc.ll
index cdf5a30dfc1cc..bb5c15c850ef4 100644
--- a/polly/test/Isl/CodeGen/intrinsics_misc.ll
+++ b/polly/test/Isl/CodeGen/intrinsics_misc.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-codegen -S < %s | FileCheck %s
 ;
 ; Verify that we remove the misc intrinsics  from the optimized SCoP.
 ;
diff --git a/polly/test/Isl/CodeGen/invariant_load_hoist_alignment.ll b/polly/test/Isl/CodeGen/invariant_load_hoist_alignment.ll
index d96a9b4596ed2..52ea0618b9ea3 100644
--- a/polly/test/Isl/CodeGen/invariant_load_hoist_alignment.ll
+++ b/polly/test/Isl/CodeGen/invariant_load_hoist_alignment.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-codegen -polly-vectorizer=polly -S \
+; RUN: opt %loadPolly -basic-aa -polly-codegen -polly-vectorizer=polly -S \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/polly/test/Isl/CodeGen/loop_with_condition.ll b/polly/test/Isl/CodeGen/loop_with_condition.ll
index a5a53eae656e7..1bb859fcb7367 100644
--- a/polly/test/Isl/CodeGen/loop_with_condition.ll
+++ b/polly/test/Isl/CodeGen/loop_with_condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-ast -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-ast -analyze < %s | FileCheck %s
 
 ;#include <string.h>
 ;#define N 1024
diff --git a/polly/test/Isl/CodeGen/loop_with_condition_2.ll b/polly/test/Isl/CodeGen/loop_with_condition_2.ll
index 9398e1dc70b88..05029ca308a92 100644
--- a/polly/test/Isl/CodeGen/loop_with_condition_2.ll
+++ b/polly/test/Isl/CodeGen/loop_with_condition_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-ast -polly-ast-detect-parallel -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-ast -polly-ast-detect-parallel -analyze < %s | FileCheck %s
 
 ; Verify that we actually detect this loop as the innermost loop even though
 ; there is a conditional inside.
diff --git a/polly/test/Isl/CodeGen/loop_with_condition_ineq.ll b/polly/test/Isl/CodeGen/loop_with_condition_ineq.ll
index f7f68317bc56e..652a69e82e4f6 100644
--- a/polly/test/Isl/CodeGen/loop_with_condition_ineq.ll
+++ b/polly/test/Isl/CodeGen/loop_with_condition_ineq.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-ast -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-ast -analyze < %s | FileCheck %s
 
 ;#include <string.h>
 ;#define N 1024
diff --git a/polly/test/Isl/CodeGen/loop_with_condition_nested.ll b/polly/test/Isl/CodeGen/loop_with_condition_nested.ll
index 38ce5589857d6..d6cbe668eb5b1 100644
--- a/polly/test/Isl/CodeGen/loop_with_condition_nested.ll
+++ b/polly/test/Isl/CodeGen/loop_with_condition_nested.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basicaa -polly-ast -analyze < %s | FileCheck %s
-; RUN: opt %loadPolly -basicaa -polly-codegen -loops -analyze < %s | FileCheck %s -check-prefix=LOOPS
+; RUN: opt %loadPolly -basic-aa -polly-ast -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-codegen -loops -analyze < %s | FileCheck %s -check-prefix=LOOPS
 
 
 ;#include <string.h>
diff --git a/polly/test/Isl/CodeGen/multidim-non-matching-typesize-2.ll b/polly/test/Isl/CodeGen/multidim-non-matching-typesize-2.ll
index 3206d92433eed..9b769c8aa39ba 100644
--- a/polly/test/Isl/CodeGen/multidim-non-matching-typesize-2.ll
+++ b/polly/test/Isl/CodeGen/multidim-non-matching-typesize-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -disable-basicaa -polly-codegen \
+; RUN: opt %loadPolly -disable-basic-aa -polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 ; CHECK: polly
 target datalayout = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128"
diff --git a/polly/test/Isl/CodeGen/multidim-non-matching-typesize.ll b/polly/test/Isl/CodeGen/multidim-non-matching-typesize.ll
index 86661ec426526..9c285a71181a7 100644
--- a/polly/test/Isl/CodeGen/multidim-non-matching-typesize.ll
+++ b/polly/test/Isl/CodeGen/multidim-non-matching-typesize.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -disable-basicaa -polly-codegen \
+; RUN: opt %loadPolly -disable-basic-aa -polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128"
diff --git a/polly/test/Isl/CodeGen/partial_write_mapped_vector.ll b/polly/test/Isl/CodeGen/partial_write_mapped_vector.ll
index c97951b2a5904..b40f72b35bf89 100644
--- a/polly/test/Isl/CodeGen/partial_write_mapped_vector.ll
+++ b/polly/test/Isl/CodeGen/partial_write_mapped_vector.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-vectorizer=polly -polly-opt-isl -polly-ast -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-vectorizer=polly -polly-opt-isl -polly-ast -polly-codegen -S < %s | FileCheck %s
 ;
 ; Polly's vectorizer does not support partial accesses.
 ;
diff --git a/polly/test/Isl/CodeGen/reduction_2.ll b/polly/test/Isl/CodeGen/reduction_2.ll
index 13cd32e792d87..075cf3f6c14ca 100644
--- a/polly/test/Isl/CodeGen/reduction_2.ll
+++ b/polly/test/Isl/CodeGen/reduction_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-ast -analyze \
+; RUN: opt %loadPolly -basic-aa -polly-ast -analyze \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 
 ;#include <string.h>
diff --git a/polly/test/Isl/CodeGen/run-time-condition.ll b/polly/test/Isl/CodeGen/run-time-condition.ll
index 0cf4aa28afc13..19446da6c3f55 100644
--- a/polly/test/Isl/CodeGen/run-time-condition.ll
+++ b/polly/test/Isl/CodeGen/run-time-condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-codegen -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/polly/test/Isl/CodeGen/simple_vec_assign_scalar.ll b/polly/test/Isl/CodeGen/simple_vec_assign_scalar.ll
index 02106844f8e04..2ac4b19e38dd1 100644
--- a/polly/test/Isl/CodeGen/simple_vec_assign_scalar.ll
+++ b/polly/test/Isl/CodeGen/simple_vec_assign_scalar.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -basicaa -polly-codegen \
+; RUN: opt %loadPolly -basic-aa -polly-codegen \
 ; RUN: -polly-vectorizer=polly -dce -S < %s | FileCheck %s
 
-; RUN: opt %loadPolly -basicaa -polly-codegen -polly-vectorizer=stripmine -dce -S < %s | FileCheck %s --check-prefix=STRIPMINE
+; RUN: opt %loadPolly -basic-aa -polly-codegen -polly-vectorizer=stripmine -dce -S < %s | FileCheck %s --check-prefix=STRIPMINE
 
 ;#define N 1024
 ;float A[N];
diff --git a/polly/test/Isl/CodeGen/simple_vec_assign_scalar_2.ll b/polly/test/Isl/CodeGen/simple_vec_assign_scalar_2.ll
index 8cdbd20873bb8..6e957da789ffc 100644
--- a/polly/test/Isl/CodeGen/simple_vec_assign_scalar_2.ll
+++ b/polly/test/Isl/CodeGen/simple_vec_assign_scalar_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-codegen -polly-vectorizer=polly -dce -S < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-codegen -polly-vectorizer=polly -dce -S < %s | FileCheck %s
 
 ;#define N 1024
 ;float A[N];
diff --git a/polly/test/Isl/CodeGen/simple_vec_call.ll b/polly/test/Isl/CodeGen/simple_vec_call.ll
index d672f54afb0fc..51a470d35e582 100644
--- a/polly/test/Isl/CodeGen/simple_vec_call.ll
+++ b/polly/test/Isl/CodeGen/simple_vec_call.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-codegen -polly-vectorizer=polly -S \
+; RUN: opt %loadPolly -basic-aa -polly-codegen -polly-vectorizer=polly -S \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/Isl/CodeGen/simple_vec_call_2.ll b/polly/test/Isl/CodeGen/simple_vec_call_2.ll
index 0ba12266310fd..b00ce29addabd 100644
--- a/polly/test/Isl/CodeGen/simple_vec_call_2.ll
+++ b/polly/test/Isl/CodeGen/simple_vec_call_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-codegen -polly-vectorizer=polly -dce \
+; RUN: opt %loadPolly -basic-aa -polly-codegen -polly-vectorizer=polly -dce \
 ; RUN: -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/Isl/CodeGen/simple_vec_cast.ll b/polly/test/Isl/CodeGen/simple_vec_cast.ll
index 69df1e179e4c3..8991a07b4920f 100644
--- a/polly/test/Isl/CodeGen/simple_vec_cast.ll
+++ b/polly/test/Isl/CodeGen/simple_vec_cast.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-codegen -polly-vectorizer=polly \
+; RUN: opt %loadPolly -basic-aa -polly-codegen -polly-vectorizer=polly \
 ; RUN: -polly-invariant-load-hoisting=true -dce -S < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/Isl/CodeGen/simple_vec_const.ll b/polly/test/Isl/CodeGen/simple_vec_const.ll
index c2716390720c7..62acccbdca4cc 100644
--- a/polly/test/Isl/CodeGen/simple_vec_const.ll
+++ b/polly/test/Isl/CodeGen/simple_vec_const.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-codegen -polly-vectorizer=polly -S \
+; RUN: opt %loadPolly -basic-aa -polly-codegen -polly-vectorizer=polly -S \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 
 ;#define N 1024
diff --git a/polly/test/Isl/CodeGen/simple_vec_large_width.ll b/polly/test/Isl/CodeGen/simple_vec_large_width.ll
index 75f5d20998e12..6b5b6e287bf31 100644
--- a/polly/test/Isl/CodeGen/simple_vec_large_width.ll
+++ b/polly/test/Isl/CodeGen/simple_vec_large_width.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-codegen -polly-vectorizer=polly -dce -S < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-codegen -polly-vectorizer=polly -dce -S < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 @A = common global [1024 x float] zeroinitializer, align 16
diff --git a/polly/test/Isl/CodeGen/simple_vec_ptr_ptr_ty.ll b/polly/test/Isl/CodeGen/simple_vec_ptr_ptr_ty.ll
index d8806dc9c61e8..13cc9aad4b853 100644
--- a/polly/test/Isl/CodeGen/simple_vec_ptr_ptr_ty.ll
+++ b/polly/test/Isl/CodeGen/simple_vec_ptr_ptr_ty.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-codegen -polly-vectorizer=polly -S \
+; RUN: opt %loadPolly -basic-aa -polly-codegen -polly-vectorizer=polly -S \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/Isl/CodeGen/simple_vec_stride_x.ll b/polly/test/Isl/CodeGen/simple_vec_stride_x.ll
index 50b0fefb74bb6..5156a598b84fd 100644
--- a/polly/test/Isl/CodeGen/simple_vec_stride_x.ll
+++ b/polly/test/Isl/CodeGen/simple_vec_stride_x.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-codegen -polly-vectorizer=polly  -dce -S < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-codegen -polly-vectorizer=polly  -dce -S < %s | FileCheck %s
 
 ;#define N 1024
 ;float A[N];
diff --git a/polly/test/Isl/CodeGen/simple_vec_two_stmts.ll b/polly/test/Isl/CodeGen/simple_vec_two_stmts.ll
index 81f15ae4386ad..a53a4a00ec7e9 100644
--- a/polly/test/Isl/CodeGen/simple_vec_two_stmts.ll
+++ b/polly/test/Isl/CodeGen/simple_vec_two_stmts.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-codegen -polly-vectorizer=polly -dce -S < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-codegen -polly-vectorizer=polly -dce -S < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 @A = common global [1024 x float] zeroinitializer, align 16
diff --git a/polly/test/ScheduleOptimizer/2012-04-16-Trivially-vectorizable-loops.ll b/polly/test/ScheduleOptimizer/2012-04-16-Trivially-vectorizable-loops.ll
index af33ff732a169..d7e0f3f042704 100644
--- a/polly/test/ScheduleOptimizer/2012-04-16-Trivially-vectorizable-loops.ll
+++ b/polly/test/ScheduleOptimizer/2012-04-16-Trivially-vectorizable-loops.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-opt-isl -polly-vectorizer=polly < %s
+; RUN: opt %loadPolly -basic-aa -polly-opt-isl -polly-vectorizer=polly < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 %struct._IO_FILE = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker*, %struct._IO_FILE*, i32, i32, i64, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i8*, i8*, i64, i32, [20 x i8] }
diff --git a/polly/test/ScheduleOptimizer/computeout.ll b/polly/test/ScheduleOptimizer/computeout.ll
index 107a3629fa401..199160c67eb57 100644
--- a/polly/test/ScheduleOptimizer/computeout.ll
+++ b/polly/test/ScheduleOptimizer/computeout.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S %loadPolly -basicaa -polly-opt-isl -polly-opt-fusion=max -polly-ast -analyze < %s | FileCheck %s 
-; RUN: opt -S %loadPolly -basicaa -polly-opt-isl -polly-opt-fusion=max -polly-ast -analyze -polly-dependences-computeout=1 < %s | FileCheck %s -check-prefix=TIMEOUT
+; RUN: opt -S %loadPolly -basic-aa -polly-opt-isl -polly-opt-fusion=max -polly-ast -analyze < %s | FileCheck %s 
+; RUN: opt -S %loadPolly -basic-aa -polly-opt-isl -polly-opt-fusion=max -polly-ast -analyze -polly-dependences-computeout=1 < %s | FileCheck %s -check-prefix=TIMEOUT
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ;     for(i = 0; i < 100; i++ )
diff --git a/polly/test/ScheduleOptimizer/focaltech_test_detail_threshold-7bc17e.ll b/polly/test/ScheduleOptimizer/focaltech_test_detail_threshold-7bc17e.ll
new file mode 100644
index 0000000000000..2668386c542df
--- /dev/null
+++ b/polly/test/ScheduleOptimizer/focaltech_test_detail_threshold-7bc17e.ll
@@ -0,0 +1,94 @@
+; RUN: opt %loadPolly -polly-opt-isl -polly-opt-fusion=max -polly-vectorizer=stripmine -polly-invariant-load-hoisting -polly-optimized-scops -analyze < %s | FileCheck %s
+;
+; llvm.org/PR46578
+;
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+%struct.stCfg_Incell_DetailThreshold.2.30.42.62.74.94.122.126.134.166.194.242.338.342.346.350.354.358.496.0.2.9.16.28.36.37.38.39.40.75 = type { [60 x i8]*, [60 x i32]*, [60 x i32]*, [60 x i32]*, [60 x i32]*, [60 x i32]*, [60 x i32]* }
+@ft8006m_g_stCfg_Incell_DetailThreshold = external dso_local local_unnamed_addr global %struct.stCfg_Incell_DetailThreshold.2.30.42.62.74.94.122.126.134.166.194.242.338.342.346.350.354.358.496.0.2.9.16.28.36.37.38.39.40.75, align 8
+declare dso_local i32 @ft8006m_atoi() local_unnamed_addr #0
+
+define void @func() {
+entry:
+  switch i32 undef, label %cleanup [
+    i32 10, label %if.end
+    i32 14, label %if.end
+    i32 16, label %if.end
+  ]
+
+if.end:                                           ; preds = %entry, %entry, %entry
+  %call15 = call i32 @ft8006m_atoi() #1
+  %0 = zext i32 %call15 to i64
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc39, %if.end
+  %indvars.iv302 = phi i64 [ %indvars.iv.next303, %for.inc39 ], [ 0, %if.end ]
+  %exitcond304 = icmp eq i64 %indvars.iv302, 60
+  br i1 %exitcond304, label %cleanup, label %for.cond21
+
+for.cond21:                                       ; preds = %for.body23, %for.cond
+  %indvars.iv296 = phi i64 [ %indvars.iv.next297, %for.body23 ], [ 0, %for.cond ]
+  %exitcond298 = icmp eq i64 %indvars.iv296, 60
+  br i1 %exitcond298, label %for.cond28, label %for.body23
+
+for.body23:                                       ; preds = %for.cond21
+  %1 = load [60 x i32]*, [60 x i32]** getelementptr inbounds (%struct.stCfg_Incell_DetailThreshold.2.30.42.62.74.94.122.126.134.166.194.242.338.342.346.350.354.358.496.0.2.9.16.28.36.37.38.39.40.75, %struct.stCfg_Incell_DetailThreshold.2.30.42.62.74.94.122.126.134.166.194.242.338.342.346.350.354.358.496.0.2.9.16.28.36.37.38.39.40.75* @ft8006m_g_stCfg_Incell_DetailThreshold, i64 0, i32 2), align 8
+  %arrayidx25 = getelementptr [60 x i32], [60 x i32]* %1, i64 %indvars.iv302, i64 %indvars.iv296
+  store i32 undef, i32* %arrayidx25, align 4
+  %indvars.iv.next297 = add nuw nsw i64 %indvars.iv296, 1
+  br label %for.cond21
+
+for.cond28:                                       ; preds = %for.body30, %for.cond21
+  %indvars.iv299 = phi i64 [ %indvars.iv.next300, %for.body30 ], [ 0, %for.cond21 ]
+  %exitcond301 = icmp eq i64 %indvars.iv299, 60
+  br i1 %exitcond301, label %for.inc39, label %for.body30
+
+for.body30:                                       ; preds = %for.cond28
+  %2 = load [60 x i32]*, [60 x i32]** getelementptr inbounds (%struct.stCfg_Incell_DetailThreshold.2.30.42.62.74.94.122.126.134.166.194.242.338.342.346.350.354.358.496.0.2.9.16.28.36.37.38.39.40.75, %struct.stCfg_Incell_DetailThreshold.2.30.42.62.74.94.122.126.134.166.194.242.338.342.346.350.354.358.496.0.2.9.16.28.36.37.38.39.40.75* @ft8006m_g_stCfg_Incell_DetailThreshold, i64 0, i32 2), align 8
+  %arrayidx34 = getelementptr [60 x i32], [60 x i32]* %2, i64 %0, i64 %indvars.iv299
+  store i32 undef, i32* %arrayidx34, align 4
+  %indvars.iv.next300 = add nuw nsw i64 %indvars.iv299, 1
+  br label %for.cond28
+
+for.inc39:                                        ; preds = %for.cond28
+  %indvars.iv.next303 = add nuw nsw i64 %indvars.iv302, 1
+  br label %for.cond
+
+cleanup:                                          ; preds = %for.cond, %entry
+  ret void
+}
+
+
+; CHECK-LABEL: Printing analysis 'Polly - Optimize schedule of SCoP' for region: 'for.cond => cleanup' in function 'func':
+; CHECK: Calculated schedule:
+; CHECK: domain: "[call15] -> { Stmt_for_body23[i0, i1] : 0 <= i0 <= 59 and 0 <= i1 <= 59; Stmt_for_body30[i0, i1] : 0 <= i0 <= 59 and 0 <= i1 <= 59 }"
+; CHECK: child:
+; CHECK:   mark: "1st level tiling - Tiles"
+; CHECK:   child:
+; CHECK:     schedule: "[call15] -> [{ Stmt_for_body23[i0, i1] -> [(floor((i0 + i1)/32))]; Stmt_for_body30[i0, i1] -> [(floor((call15 + i1)/32))] }, { Stmt_for_body23[i0, i1] -> [(floor((i0)/32))]; Stmt_for_body30[i0, i1] -> [(floor((i0)/32))] }]"
+; CHECK:     permutable: 1
+; CHECK:     coincident: [ 1, 0 ]
+; CHECK:     child:
+; CHECK:       mark: "1st level tiling - Points"
+; CHECK:       child:
+; CHECK:         schedule: "[call15] -> [{ Stmt_for_body23[i0, i1] -> [(floor((i0 + i1)/4) - 8*floor((i0 + i1)/32))]; Stmt_for_body30[i0, i1] -> [(floor((call15 + i1)/4) - 8*floor((call15 + i1)/32))] }]"
+; CHECK:         permutable: 1
+; CHECK:         coincident: [ 1 ]
+; CHECK:         options: "[call15] -> { atomic[0]; isolate{{\[\[}}i0, i1] -> [i2]] : 0 <= i1 <= 1 and 0 <= i2 <= 7 and call15 - 32i0 <= 4i2 <= 56 + call15 - 32i0 and (call15 >= 120 or i2 < -8i0 + 8i1); isolate{{\[\[}}i0, 0] -> [i2]] : 0 <= i2 <= 7 and ((call15 >= 120 and -8i0 <= i2 <= 21 - 8i0) or (call15 >= 92 and i2 <= 28 - 8i0 and 4i2 >= call15 - 32i0) or (call15 <= 91 and -8i0 <= i2 <= 21 - 8i0) or (92 <= call15 <= 119 and -8i0 <= i2 <= 21 - 8i0) or (call15 <= 91 and 22 - 8i0 <= i2 <= 28 - 8i0 and 4i2 <= 56 + call15 - 32i0) or (call15 <= 119 and i2 >= 29 - 8i0 and call15 - 32i0 <= 4i2 <= 56 + call15 - 32i0)); isolate{{\[\[}}i0, 1] -> [i2]] : i2 >= 0 and 8 - 8i0 <= i2 <= 7 and ((call15 >= 120 and i2 <= 28 - 8i0) or (call15 <= 119 and i2 >= 29 - 8i0 and 4i2 <= 56 + call15 - 32i0) or (call15 <= 119 and i2 <= 28 - 8i0)) }"
+; CHECK:         child:
+; CHECK:           schedule: "[call15] -> [{ Stmt_for_body23[i0, i1] -> [((i0) mod 32)]; Stmt_for_body30[i0, i1] -> [((i0) mod 32)] }]"
+; CHECK:           permutable: 1
+; CHECK:           child:
+; CHECK:             mark: "SIMD"
+; CHECK:             child:
+; CHECK:               sequence:
+; CHECK:               - filter: "[call15] -> { Stmt_for_body23[i0, i1] }"
+; CHECK:                 child:
+; CHECK:                   schedule: "[call15] -> [{ Stmt_for_body23[i0, i1] -> [((i0 + i1) mod 4)]; Stmt_for_body30[i0, i1] -> [((call15 + i1) mod 4)] }]"
+; CHECK:                   permutable: 1
+; CHECK:                   coincident: [ 1 ]
+; CHECK:               - filter: "[call15] -> { Stmt_for_body30[i0, i1] }"
+; CHECK:                 child:
+; CHECK:                   schedule: "[call15] -> [{ Stmt_for_body23[i0, i1] -> [((i0 + i1) mod 4)]; Stmt_for_body30[i0, i1] -> [((call15 + i1) mod 4)] }]"
+; CHECK:                   permutable: 1
+; CHECK:                   coincident: [ 1 ]
diff --git a/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout_2.ll b/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout_2.ll
index 7a645364c4bfb..ef6a9f42adab6 100644
--- a/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout_2.ll
+++ b/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout_2.ll
@@ -18,6 +18,7 @@
 ;	     C[i][j] += alpha * A[i][k] * B[k][j];
 ;        }
 ;
+; CHECK-LABEL: Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: 'bb8 => bb32' in function 'kernel_gemm':
 ; CHECK:    {
 ; CHECK-NEXT:      // 1st level tiling - Tiles
 ; CHECK-NEXT:      for (int c0 = 0; c0 <= 32; c0 += 1)
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_3.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_3.ll
index 76341cb6afe07..b73083d196c8f 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_3.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_3.ll
@@ -24,6 +24,7 @@
 ;	     C[i][j] += alpha * A[i][k] * B[k][j];
 ;        }
 ;
+; CHECK-LABEL: Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: 'bb8 => bb32' in function 'kernel_gemm':
 ; CHECK:    {
 ; CHECK-NEXT:      // 1st level tiling - Tiles
 ; CHECK-NEXT:      for (int c0 = 0; c0 <= 32; c0 += 1)
@@ -76,6 +77,7 @@
 ; CHECK-NEXT:          }
 ; CHECK-NEXT:    }
 ;
+; EXTRACTION-OF-MACRO-KERNEL-LABEL: Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: 'bb8 => bb32' in function 'kernel_gemm':
 ; EXTRACTION-OF-MACRO-KERNEL:    {
 ; EXTRACTION-OF-MACRO-KERNEL-NEXT:      // 1st level tiling - Tiles
 ; EXTRACTION-OF-MACRO-KERNEL-NEXT:      for (int c0 = 0; c0 <= 32; c0 += 1)
diff --git a/polly/test/ScheduleOptimizer/prevectorization-without-tiling.ll b/polly/test/ScheduleOptimizer/prevectorization-without-tiling.ll
index 6000e647c8cd9..84ac3c11e846e 100644
--- a/polly/test/ScheduleOptimizer/prevectorization-without-tiling.ll
+++ b/polly/test/ScheduleOptimizer/prevectorization-without-tiling.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S %loadPolly -basicaa -polly-opt-isl -polly-tiling=false \
+; RUN: opt -S %loadPolly -basic-aa -polly-opt-isl -polly-tiling=false \
 ; RUN: -polly-pattern-matching-based-opts=false -polly-vectorizer=polly \
 ; RUN: -polly-ast -analyze < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/polly/test/ScheduleOptimizer/prevectorization.ll b/polly/test/ScheduleOptimizer/prevectorization.ll
index 23c1bdc1c04e3..426972ddc878d 100644
--- a/polly/test/ScheduleOptimizer/prevectorization.ll
+++ b/polly/test/ScheduleOptimizer/prevectorization.ll
@@ -1,11 +1,11 @@
-; RUN: opt -S %loadPolly -basicaa -polly-opt-isl \
+; RUN: opt -S %loadPolly -basic-aa -polly-opt-isl \
 ; RUN: -polly-pattern-matching-based-opts=false -polly-vectorizer=polly \
 ; RUN: -polly-ast -analyze < %s | FileCheck %s 
-; RUN: opt -S %loadPolly -basicaa -polly-opt-isl \
+; RUN: opt -S %loadPolly -basic-aa -polly-opt-isl \
 ; RUN: -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine \
 ; RUN: -polly-ast -analyze < %s | FileCheck %s
 
-; RUN: opt -S %loadPolly -basicaa -polly-opt-isl \
+; RUN: opt -S %loadPolly -basic-aa -polly-opt-isl \
 ; RUN: -polly-vectorizer=polly -polly-pattern-matching-based-opts=false \
 ; RUN: -polly-ast -analyze -polly-prevect-width=16 < %s | \
 ; RUN: FileCheck %s -check-prefix=VEC16
diff --git a/polly/test/ScheduleOptimizer/tile_after_fusion.ll b/polly/test/ScheduleOptimizer/tile_after_fusion.ll
index 42833ef979997..35f20ce5723d5 100644
--- a/polly/test/ScheduleOptimizer/tile_after_fusion.ll
+++ b/polly/test/ScheduleOptimizer/tile_after_fusion.ll
@@ -17,6 +17,7 @@
 ; checks whether they are tiled after being fused when polly-opt-fusion equals
 ; "max".
 ;
+; CHECK-LABEL: Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: 'for.cond => for.end56' in function 'tf':
 ; CHECK:       1st level tiling - Tiles
 ; CHECK-NEXT:     for (int c0 = 0; c0 <= 7; c0 += 1)
 ; CHECK-NEXT:       for (int c1 = 0; c1 <= 7; c1 += 1)
diff --git a/polly/test/ScopDetect/base_pointer.ll b/polly/test/ScopDetect/base_pointer.ll
index 09fdf020cc763..e6cff3044f5dd 100644
--- a/polly/test/ScopDetect/base_pointer.ll
+++ b/polly/test/ScopDetect/base_pointer.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -disable-basicaa -polly-detect -polly-invariant-load-hoisting=true -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -disable-basic-aa -polly-detect -polly-invariant-load-hoisting=true -analyze < %s | FileCheck %s
 
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
diff --git a/polly/test/ScopDetect/base_pointer_setNewAccessRelation.ll b/polly/test/ScopDetect/base_pointer_setNewAccessRelation.ll
index a7afa74988ffb..e825a83f7ed5f 100644
--- a/polly/test/ScopDetect/base_pointer_setNewAccessRelation.ll
+++ b/polly/test/ScopDetect/base_pointer_setNewAccessRelation.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -disable-basicaa -polly-detect -polly-import-jscop -polly-codegen -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -disable-basic-aa -polly-detect -polly-import-jscop -polly-codegen -analyze < %s | FileCheck %s
 ;
 ; Polly codegen used to generate invalid code (referring to %ptr from the
 ; original region) when regeneration of the access function is necessary.
diff --git a/polly/test/ScopDetect/intrinsics_1.ll b/polly/test/ScopDetect/intrinsics_1.ll
index 84a887628afe0..1a8d143807ca0 100644
--- a/polly/test/ScopDetect/intrinsics_1.ll
+++ b/polly/test/ScopDetect/intrinsics_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-detect -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-detect -analyze < %s | FileCheck %s
 ;
 ; CHECK: Valid Region for Scop: for.cond => for.end
 ;
diff --git a/polly/test/ScopDetect/intrinsics_2.ll b/polly/test/ScopDetect/intrinsics_2.ll
index 2c2985021ae11..03a2034a37823 100644
--- a/polly/test/ScopDetect/intrinsics_2.ll
+++ b/polly/test/ScopDetect/intrinsics_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-detect -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-detect -analyze < %s | FileCheck %s
 ;
 ; Verify that we allow the lifetime markers for the tmp array.
 ;
diff --git a/polly/test/ScopDetect/intrinsics_3.ll b/polly/test/ScopDetect/intrinsics_3.ll
index 1a2564695d75e..ccc881f7bca47 100644
--- a/polly/test/ScopDetect/intrinsics_3.ll
+++ b/polly/test/ScopDetect/intrinsics_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-detect -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-detect -analyze < %s | FileCheck %s
 ;
 ; Verify that we allow the misc intrinsics.
 ;
diff --git a/polly/test/ScopDetect/keep_going_expansion.ll b/polly/test/ScopDetect/keep_going_expansion.ll
index ef88ce152ff74..ae536d5e3d55f 100644
--- a/polly/test/ScopDetect/keep_going_expansion.ll
+++ b/polly/test/ScopDetect/keep_going_expansion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-detect-track-failures -polly-detect-keep-going -polly-detect -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-detect-track-failures -polly-detect-keep-going -polly-detect -analyze < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopDetect/mod_ref_read_pointer.ll b/polly/test/ScopDetect/mod_ref_read_pointer.ll
index c42f64780c825..95732b11acf22 100644
--- a/polly/test/ScopDetect/mod_ref_read_pointer.ll
+++ b/polly/test/ScopDetect/mod_ref_read_pointer.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -basicaa -polly-detect -analyze \
+; RUN: opt %loadPolly -basic-aa -polly-detect -analyze \
 ; RUN:  -polly-allow-modref-calls < %s | FileCheck %s -check-prefix=MODREF
-; RUN: opt %loadPolly -basicaa -polly-detect -analyze \
+; RUN: opt %loadPolly -basic-aa -polly-detect -analyze \
 ; RUN:  < %s | FileCheck %s
 ;
 ; CHECK-NOT: Valid Region for Scop: for.body => for.end
diff --git a/polly/test/ScopDetect/non-affine-loop-condition-dependent-access.ll b/polly/test/ScopDetect/non-affine-loop-condition-dependent-access.ll
index 6140992b5183d..b1fdb8eca6b8b 100644
--- a/polly/test/ScopDetect/non-affine-loop-condition-dependent-access.ll
+++ b/polly/test/ScopDetect/non-affine-loop-condition-dependent-access.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -basicaa -polly-detect -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false -analyze < %s | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
-; RUN: opt %loadPolly -basicaa -polly-detect -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true -analyze < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
-; RUN: opt %loadPolly -basicaa -polly-detect -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true -analyze < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
-; RUN: opt %loadPolly -basicaa -polly-detect -polly-process-unprofitable=false \
+; RUN: opt %loadPolly -basic-aa -polly-detect -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false -analyze < %s | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
+; RUN: opt %loadPolly -basic-aa -polly-detect -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true -analyze < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
+; RUN: opt %loadPolly -basic-aa -polly-detect -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true -analyze < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
+; RUN: opt %loadPolly -basic-aa -polly-detect -polly-process-unprofitable=false \
 ; RUN:    -polly-allow-nonaffine -polly-allow-nonaffine-branches \
 ; RUN:    -polly-allow-nonaffine-loops=true -analyze < %s \
 ; RUN:    | FileCheck %s --check-prefix=PROFIT
diff --git a/polly/test/ScopDetect/simple_loop_with_param.ll b/polly/test/ScopDetect/simple_loop_with_param.ll
index 69177a48172c5..9e7aa328e6328 100644
--- a/polly/test/ScopDetect/simple_loop_with_param.ll
+++ b/polly/test/ScopDetect/simple_loop_with_param.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-detect -analyze  < %s | FileCheck %s -check-prefix=PHI
+; RUN: opt %loadPolly -basic-aa -polly-detect -analyze  < %s | FileCheck %s -check-prefix=PHI
 
 ; void f(long A[], long N, long *init_ptr) {
 ;   long i, j;
diff --git a/polly/test/ScopDetect/simple_loop_with_param_2.ll b/polly/test/ScopDetect/simple_loop_with_param_2.ll
index c619958f99aab..d8edb890c79aa 100644
--- a/polly/test/ScopDetect/simple_loop_with_param_2.ll
+++ b/polly/test/ScopDetect/simple_loop_with_param_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-detect -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-detect -analyze < %s | FileCheck %s
 
 ; void f(long A[], int N, int *init_ptr) {
 ;   long i, j;
diff --git a/polly/test/ScopDetectionDiagnostics/ReportMultipleNonAffineAccesses.ll b/polly/test/ScopDetectionDiagnostics/ReportMultipleNonAffineAccesses.ll
index 1b3e157b16f6e..8f81cf24fa0f3 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportMultipleNonAffineAccesses.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportMultipleNonAffineAccesses.ll
@@ -1,9 +1,9 @@
-; RUN: opt %loadPolly -basicaa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-detect -analyze < %s 2>&1| FileCheck %s
-; RUN: opt %loadPolly -basicaa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-detect -polly-delinearize=false -polly-detect-keep-going -analyze < %s 2>&1| FileCheck %s -check-prefix=ALL
-; RUN: opt %loadPolly -basicaa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-detect -analyze < %s 2>&1| FileCheck %s -check-prefix=DELIN
-; RUN: opt %loadPolly -basicaa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-detect -polly-detect-keep-going -analyze < %s 2>&1| FileCheck %s -check-prefix=DELIN-ALL
-; RUN: opt %loadPolly -basicaa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-detect -polly-allow-nonaffine -analyze < %s 2>&1| FileCheck %s -check-prefix=NONAFFINE
-; RUN: opt %loadPolly -basicaa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-detect -polly-allow-nonaffine -analyze < %s 2>&1| FileCheck %s -check-prefix=NONAFFINE
+; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-detect -analyze < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-detect -polly-delinearize=false -polly-detect-keep-going -analyze < %s 2>&1| FileCheck %s -check-prefix=ALL
+; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-detect -analyze < %s 2>&1| FileCheck %s -check-prefix=DELIN
+; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-detect -polly-detect-keep-going -analyze < %s 2>&1| FileCheck %s -check-prefix=DELIN-ALL
+; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-detect -polly-allow-nonaffine -analyze < %s 2>&1| FileCheck %s -check-prefix=NONAFFINE
+; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-detect -polly-allow-nonaffine -analyze < %s 2>&1| FileCheck %s -check-prefix=NONAFFINE
 
 ;  1 void manyaccesses(float A[restrict], long n, float B[restrict][n])
 ;  2 {
diff --git a/polly/test/ScopInfo/Alias-4.ll b/polly/test/ScopInfo/Alias-4.ll
index 1955a86d4da68..5cd9fa77376d9 100644
--- a/polly/test/ScopInfo/Alias-4.ll
+++ b/polly/test/ScopInfo/Alias-4.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -disable-basicaa -polly-scops -analyze < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
-; RUN: opt %loadPolly -disable-basicaa -polly-scops -polly-use-runtime-alias-checks=false -analyze < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
+; RUN: opt %loadPolly -disable-basic-aa -polly-scops -analyze < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
+; RUN: opt %loadPolly -disable-basic-aa -polly-scops -polly-use-runtime-alias-checks=false -analyze < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
 ; REQUIRES: asserts
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_1.ll b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_1.ll
index 59012efea4d13..4472fadf485fb 100644
--- a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_1.ll
+++ b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_1.ll
@@ -1,8 +1,8 @@
-; RUN: opt %loadPolly -basicaa -polly-scops \
+; RUN: opt %loadPolly -basic-aa -polly-scops \
 ; RUN:     -polly-allow-nonaffine -polly-allow-nonaffine-branches \
 ; RUN:     -polly-allow-nonaffine-loops=true -analyze < %s | FileCheck %s \
 ; RUN:     -check-prefix=SCALAR
-; RUN: opt %loadPolly -basicaa -polly-scops -polly-allow-nonaffine \
+; RUN: opt %loadPolly -basic-aa -polly-scops -polly-allow-nonaffine \
 ; RUN:     -polly-process-unprofitable=false \
 ; RUN:     -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true \
 ; RUN:     -analyze < %s | FileCheck %s -check-prefix=PROFIT
diff --git a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_2.ll b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_2.ll
index 1a6bab8618b9b..37d08822b6309 100644
--- a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_2.ll
+++ b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_2.ll
@@ -1,10 +1,10 @@
-; RUN: opt %loadPolly -basicaa -polly-scops -polly-allow-nonaffine-branches \
+; RUN: opt %loadPolly -basic-aa -polly-scops -polly-allow-nonaffine-branches \
 ; RUN:     -polly-allow-nonaffine-loops=false \
 ; RUN:     -analyze < %s | FileCheck %s --check-prefix=INNERMOST
-; RUN: opt %loadPolly -basicaa -polly-scops -polly-allow-nonaffine-branches \
+; RUN: opt %loadPolly -basic-aa -polly-scops -polly-allow-nonaffine-branches \
 ; RUN:     -polly-allow-nonaffine-loops=true \
 ; RUN:     -analyze < %s | FileCheck %s --check-prefix=INNERMOST
-; RUN: opt %loadPolly -basicaa -polly-scops -polly-allow-nonaffine \
+; RUN: opt %loadPolly -basic-aa -polly-scops -polly-allow-nonaffine \
 ; RUN:     -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true \
 ; RUN:     -analyze < %s | FileCheck %s \
 ; RUN:     --check-prefix=ALL
diff --git a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_3.ll b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_3.ll
index 26551d8d6608d..b36c9c3672d83 100644
--- a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_3.ll
+++ b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_3.ll
@@ -1,10 +1,10 @@
-; RUN: opt %loadPolly -basicaa -polly-scops -polly-allow-nonaffine-branches \
+; RUN: opt %loadPolly -basic-aa -polly-scops -polly-allow-nonaffine-branches \
 ; RUN:     -polly-allow-nonaffine-loops=false \
 ; RUN:     -analyze < %s | FileCheck %s --check-prefix=INNERMOST
-; RUN: opt %loadPolly -basicaa -polly-scops -polly-allow-nonaffine-branches \
+; RUN: opt %loadPolly -basic-aa -polly-scops -polly-allow-nonaffine-branches \
 ; RUN:     -polly-allow-nonaffine-loops=true \
 ; RUN:      -analyze < %s | FileCheck %s --check-prefix=INNERMOST
-; RUN: opt %loadPolly -basicaa -polly-scops -polly-allow-nonaffine \
+; RUN: opt %loadPolly -basic-aa -polly-scops -polly-allow-nonaffine \
 ; RUN:     -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true \
 ; RUN:     -analyze < %s | FileCheck %s --check-prefix=ALL
 ;
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_parametric_loop.ll b/polly/test/ScopInfo/NonAffine/non_affine_parametric_loop.ll
index 18e08e6f61a0a..a82119f5374fb 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_parametric_loop.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_parametric_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-scops -analyze -polly-allow-nonaffine < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-scops -analyze -polly-allow-nonaffine < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long n, double A[], int INDEX[]) {
diff --git a/polly/test/ScopInfo/assume_gep_bounds_2.ll b/polly/test/ScopInfo/assume_gep_bounds_2.ll
index d364211bc5480..7c2e08347d360 100644
--- a/polly/test/ScopInfo/assume_gep_bounds_2.ll
+++ b/polly/test/ScopInfo/assume_gep_bounds_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-scops -analyze < %s \
+; RUN: opt %loadPolly -basic-aa -polly-scops -analyze < %s \
 ; RUN:  -polly-precise-inbounds | FileCheck %s
 ;
 ;    void foo(float A[restrict][20], float B[restrict][20], long n, long m,
diff --git a/polly/test/ScopInfo/inter_bb_scalar_dep.ll b/polly/test/ScopInfo/inter_bb_scalar_dep.ll
index eabcefe6d24f2..3882bc36589dc 100644
--- a/polly/test/ScopInfo/inter_bb_scalar_dep.ll
+++ b/polly/test/ScopInfo/inter_bb_scalar_dep.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-scops \
+; RUN: opt %loadPolly -basic-aa -polly-scops \
 ; RUN: -polly-invariant-load-hoisting=true -analyze < %s | FileCheck %s
 
 ; void f(long A[], int N, int *init_ptr) {
diff --git a/polly/test/ScopInfo/intra_and_inter_bb_scalar_dep.ll b/polly/test/ScopInfo/intra_and_inter_bb_scalar_dep.ll
index 166f991f29a59..6e1441f4c120e 100644
--- a/polly/test/ScopInfo/intra_and_inter_bb_scalar_dep.ll
+++ b/polly/test/ScopInfo/intra_and_inter_bb_scalar_dep.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-scops -analyze \
+; RUN: opt %loadPolly -basic-aa -polly-scops -analyze \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 
 ; void f(long A[], int N, int *init_ptr) {
diff --git a/polly/test/ScopInfo/intra_bb_scalar_dep.ll b/polly/test/ScopInfo/intra_bb_scalar_dep.ll
index 2a8c2fae546ae..1bebc17fe880b 100644
--- a/polly/test/ScopInfo/intra_bb_scalar_dep.ll
+++ b/polly/test/ScopInfo/intra_bb_scalar_dep.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-scops -analyze \
+; RUN: opt %loadPolly -basic-aa -polly-scops -analyze \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 
 ; void f(long A[], int N, int *init_ptr) {
diff --git a/polly/test/ScopInfo/isl_aff_out_of_bounds.ll b/polly/test/ScopInfo/isl_aff_out_of_bounds.ll
index e8cd2a2e5d07e..1bd89b6884a47 100644
--- a/polly/test/ScopInfo/isl_aff_out_of_bounds.ll
+++ b/polly/test/ScopInfo/isl_aff_out_of_bounds.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-detect < %s
+; RUN: opt %loadPolly -basic-aa -polly-detect < %s
 
 ; Used to fail with:
 ; ../../isl/isl_aff.c:591: position out of bounds
diff --git a/polly/test/ScopInfo/licm_load.ll b/polly/test/ScopInfo/licm_load.ll
index b8b8a9c84a487..b950237c134da 100644
--- a/polly/test/ScopInfo/licm_load.ll
+++ b/polly/test/ScopInfo/licm_load.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -basicaa -loop-rotate -indvars       -polly-prepare \
+; RUN: opt %loadPolly -basic-aa -loop-rotate -indvars       -polly-prepare \
 ; RUN: -polly-invariant-load-hoisting=true -polly-scops -analyze < %s \
 ; RUN: | FileCheck %s
-; RUN: opt %loadPolly -basicaa -loop-rotate -indvars -licm -polly-prepare \
+; RUN: opt %loadPolly -basic-aa -loop-rotate -indvars -licm -polly-prepare \
 ; RUN: -polly-invariant-load-hoisting=true -polly-scops -analyze < %s \
 ; RUN: | FileCheck %s
 ;
diff --git a/polly/test/ScopInfo/licm_potential_store.ll b/polly/test/ScopInfo/licm_potential_store.ll
index 8b389dfbc40a2..fe8bd94934a7c 100644
--- a/polly/test/ScopInfo/licm_potential_store.ll
+++ b/polly/test/ScopInfo/licm_potential_store.ll
@@ -1,10 +1,10 @@
-; RUN: opt %loadPolly -basicaa -sroa -instcombine -simplifycfg -tailcallopt \
+; RUN: opt %loadPolly -basic-aa -sroa -instcombine -simplifycfg -tailcallopt \
 ; RUN:    -simplifycfg -reassociate -loop-rotate -instcombine -indvars \
 ; RUN:    -polly-prepare -polly-scops -analyze < %s \
 ; RUN:    \
 ; RUN:     | FileCheck %s --check-prefix=NOLICM
 
-; RUN: opt %loadPolly -basicaa -sroa -instcombine -simplifycfg -tailcallopt \
+; RUN: opt %loadPolly -basic-aa -sroa -instcombine -simplifycfg -tailcallopt \
 ; RUN:    -simplifycfg -reassociate -loop-rotate -instcombine -indvars -licm \
 ; RUN:    -polly-prepare -polly-scops -analyze < %s \
 ; RUN:    \
diff --git a/polly/test/ScopInfo/licm_reduction.ll b/polly/test/ScopInfo/licm_reduction.ll
index 2a3c5a00c3013..a1e5c5fe835d6 100644
--- a/polly/test/ScopInfo/licm_reduction.ll
+++ b/polly/test/ScopInfo/licm_reduction.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basicaa -loop-rotate -indvars       -polly-prepare -polly-scops -analyze < %s | FileCheck %s
-; RUN: opt %loadPolly -basicaa -loop-rotate -indvars -licm -polly-prepare -polly-scops -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -loop-rotate -indvars       -polly-prepare -polly-scops -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -loop-rotate -indvars -licm -polly-prepare -polly-scops -analyze < %s | FileCheck %s
 ;
 ; XFAIL: *
 ;
diff --git a/polly/test/ScopInfo/licm_reduction_nested.ll b/polly/test/ScopInfo/licm_reduction_nested.ll
index 4901fc28b602b..d79aaf605e1a1 100644
--- a/polly/test/ScopInfo/licm_reduction_nested.ll
+++ b/polly/test/ScopInfo/licm_reduction_nested.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basicaa -loop-rotate -indvars       -polly-prepare -polly-scops -analyze < %s | FileCheck %s
-; RUN: opt %loadPolly -basicaa -loop-rotate -indvars -licm -polly-prepare -polly-scops -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -loop-rotate -indvars       -polly-prepare -polly-scops -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -loop-rotate -indvars -licm -polly-prepare -polly-scops -analyze < %s | FileCheck %s
 ;
 ; XFAIL: *
 ;
diff --git a/polly/test/ScopInfo/licm_store.ll b/polly/test/ScopInfo/licm_store.ll
index 6ca509be4f1d6..a0a9187492cf3 100644
--- a/polly/test/ScopInfo/licm_store.ll
+++ b/polly/test/ScopInfo/licm_store.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basicaa -loop-rotate -indvars       -polly-prepare -polly-scops -analyze < %s | FileCheck %s
-; RUN: opt %loadPolly -basicaa -loop-rotate -indvars -licm -polly-prepare -polly-scops -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -loop-rotate -indvars       -polly-prepare -polly-scops -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -loop-rotate -indvars -licm -polly-prepare -polly-scops -analyze < %s | FileCheck %s
 ;
 ; XFAIL: *
 ;
diff --git a/polly/test/ScopInfo/loop_carry.ll b/polly/test/ScopInfo/loop_carry.ll
index 15e092d9da8cf..581b80876f04c 100644
--- a/polly/test/ScopInfo/loop_carry.ll
+++ b/polly/test/ScopInfo/loop_carry.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-stmt-granularity=bb -polly-scops -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-scops -analyze < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/ScopInfo/memcpy-raw-source.ll b/polly/test/ScopInfo/memcpy-raw-source.ll
index facf825670ba8..69a12f5cb8346 100644
--- a/polly/test/ScopInfo/memcpy-raw-source.ll
+++ b/polly/test/ScopInfo/memcpy-raw-source.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -scoped-noalias -tbaa -polly-scops -analyze < %s
+; RUN: opt %loadPolly -basic-aa -scoped-noalias -tbaa -polly-scops -analyze < %s
 ;
 ; Ensure that ScopInfo's alias analysis llvm.memcpy for,
 ; like the AliasSetTracker, preserves bitcasts.
diff --git a/polly/test/ScopInfo/memcpy.ll b/polly/test/ScopInfo/memcpy.ll
index 1cccffbe468c1..7410beecc5df0 100644
--- a/polly/test/ScopInfo/memcpy.ll
+++ b/polly/test/ScopInfo/memcpy.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basicaa -polly-allow-differing-element-types -polly-scops -analyze < %s | FileCheck %s
-; RUN: opt %loadPolly -S -basicaa -polly-allow-differing-element-types -polly-codegen < %s | FileCheck --check-prefix=IR %s
+; RUN: opt %loadPolly -basic-aa -polly-allow-differing-element-types -polly-scops -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -S -basic-aa -polly-allow-differing-element-types -polly-codegen < %s | FileCheck --check-prefix=IR %s
 ;
 ; CHECK:         Arrays {
 ; CHECK-NEXT:        i8 MemRef_A[*]; // Element size 1
diff --git a/polly/test/ScopInfo/memmove.ll b/polly/test/ScopInfo/memmove.ll
index ca00a3016c92f..b21ecf98eca65 100644
--- a/polly/test/ScopInfo/memmove.ll
+++ b/polly/test/ScopInfo/memmove.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basicaa -polly-allow-differing-element-types -polly-scops -analyze < %s | FileCheck %s
-; RUN: opt %loadPolly -S -basicaa -polly-allow-differing-element-types -polly-codegen < %s | FileCheck --check-prefix=IR %s
+; RUN: opt %loadPolly -basic-aa -polly-allow-differing-element-types -polly-scops -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -S -basic-aa -polly-allow-differing-element-types -polly-codegen < %s | FileCheck --check-prefix=IR %s
 ;
 ; CHECK:         Arrays {
 ; CHECK-NEXT:        i8 MemRef_A[*]; // Element size 1
diff --git a/polly/test/ScopInfo/mod_ref_access_pointee_arguments.ll b/polly/test/ScopInfo/mod_ref_access_pointee_arguments.ll
index 346092676fbc3..f3cd7b1571ee7 100644
--- a/polly/test/ScopInfo/mod_ref_access_pointee_arguments.ll
+++ b/polly/test/ScopInfo/mod_ref_access_pointee_arguments.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -basicaa -polly-stmt-granularity=bb -polly-scops -analyze -polly-allow-modref-calls \
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-scops -analyze -polly-allow-modref-calls \
 ; RUN:  < %s | FileCheck %s
-; RUN: opt %loadPolly -basicaa -polly-stmt-granularity=bb -polly-codegen -polly-allow-modref-calls \
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-codegen -polly-allow-modref-calls \
 ; RUN: -disable-output < %s
 ;
 ; Verify that we model the may-write access of the prefetch intrinsic
diff --git a/polly/test/ScopInfo/mod_ref_read_pointee_arguments.ll b/polly/test/ScopInfo/mod_ref_read_pointee_arguments.ll
index 16fe7529dae7f..2c91fc929bed9 100644
--- a/polly/test/ScopInfo/mod_ref_read_pointee_arguments.ll
+++ b/polly/test/ScopInfo/mod_ref_read_pointee_arguments.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -basicaa -polly-stmt-granularity=bb -polly-scops -analyze -polly-allow-modref-calls \
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-scops -analyze -polly-allow-modref-calls \
 ; RUN: < %s | FileCheck %s
-; RUN: opt %loadPolly -basicaa -polly-codegen -disable-output \
+; RUN: opt %loadPolly -basic-aa -polly-codegen -disable-output \
 ; RUN: -polly-allow-modref-calls < %s
 ;
 ; Verify that we model the read access of the gcread intrinsic
diff --git a/polly/test/ScopInfo/mod_ref_read_pointer.ll b/polly/test/ScopInfo/mod_ref_read_pointer.ll
index f00f5e76f74aa..91f60445b1b3d 100644
--- a/polly/test/ScopInfo/mod_ref_read_pointer.ll
+++ b/polly/test/ScopInfo/mod_ref_read_pointer.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -basicaa -polly-scops -analyze -polly-allow-modref-calls \
+; RUN: opt %loadPolly -basic-aa -polly-scops -analyze -polly-allow-modref-calls \
 ; RUN:  < %s | FileCheck %s
-; RUN: opt %loadPolly -basicaa -polly-codegen -disable-output \
+; RUN: opt %loadPolly -basic-aa -polly-codegen -disable-output \
 ; RUN: -polly-allow-modref-calls < %s
 ;
 ; Check that we assume the call to func has a read on the whole A array.
diff --git a/polly/test/ScopInfo/mod_ref_read_pointers.ll b/polly/test/ScopInfo/mod_ref_read_pointers.ll
index 59fc85adc0e33..e706f1ae54c00 100644
--- a/polly/test/ScopInfo/mod_ref_read_pointers.ll
+++ b/polly/test/ScopInfo/mod_ref_read_pointers.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -basicaa -polly-scops -analyze -polly-allow-modref-calls \
+; RUN: opt %loadPolly -basic-aa -polly-scops -analyze -polly-allow-modref-calls \
 ; RUN: < %s | FileCheck %s
-; RUN: opt %loadPolly -basicaa -polly-codegen -disable-output \
+; RUN: opt %loadPolly -basic-aa -polly-codegen -disable-output \
 ; RUN: -polly-allow-modref-calls < %s
 ;
 ; Check that the call to func will "read" not only the A array but also the
diff --git a/polly/test/ScopInfo/multidim_srem.ll b/polly/test/ScopInfo/multidim_srem.ll
index 10673f41aaee7..f7f9616a9df3e 100644
--- a/polly/test/ScopInfo/multidim_srem.ll
+++ b/polly/test/ScopInfo/multidim_srem.ll
@@ -14,11 +14,12 @@
 ; CHECK-NEXT:         Schedule :=
 ; CHECK-NEXT:             [n] -> { Stmt_for_body_8[i0, i1, i2] -> [i0, i1, i2] };
 ; CHECK-NEXT:         ReadAccess :=    [Reduction Type: NONE] [Scalar: 0]
-; CHECK-NEXT:             [n] -> { Stmt_for_body_8[i0, i1, i2] -> MemRef_A[o0, i1, i2] : (i0 + o0) mod 2 = 0 and 0 <= o0 <= 1 } 
+; CHECK-NEXT:             [n] -> { Stmt_for_body_8[i0, i1, i2] -> MemRef_A[1, i1, i2] : (1 + i0) mod 2 = 0; Stmt_for_body_8[i0, i1, i2] -> MemRef_A[0, i1, i2] : (i0) mod 2 = 0 };
 ; CHECK-NEXT:         MustWriteAccess :=    [Reduction Type: NONE] [Scalar: 0]
-; CHECK-NEXT:             [n] -> { Stmt_for_body_8[i0, i1, i2] -> MemRef_A[o0, i1, i2] : (i0 + o0) mod 2 = 0 and 0 <= o0 <= 1 };
+; CHECK-NEXT:             [n] -> { Stmt_for_body_8[i0, i1, i2] -> MemRef_A[1, i1, i2] : (1 + i0) mod 2 = 0; Stmt_for_body_8[i0, i1, i2] -> MemRef_A[0, i1, i2] : (i0) mod 2 = 0 };
 ; CHECK-NEXT: }
 
+
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 
diff --git a/polly/test/ScopInfo/reduction_disabled_multiplicative.ll b/polly/test/ScopInfo/reduction_disabled_multiplicative.ll
index ab3de3ee5bf99..2d02d72faa626 100644
--- a/polly/test/ScopInfo/reduction_disabled_multiplicative.ll
+++ b/polly/test/ScopInfo/reduction_disabled_multiplicative.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa %loadPolly -polly-stmt-granularity=bb -polly-scops -analyze -polly-disable-multiplicative-reductions < %s | FileCheck %s
+; RUN: opt -basic-aa %loadPolly -polly-stmt-granularity=bb -polly-scops -analyze -polly-disable-multiplicative-reductions < %s | FileCheck %s
 ;
 ; CHECK: ReadAccess :=       [Reduction Type: +
 ; CHECK:     { Stmt_for_body[i0] -> MemRef_sum[0] };
diff --git a/polly/test/ScopInfo/reduction_escaping_intermediate.ll b/polly/test/ScopInfo/reduction_escaping_intermediate.ll
index 2e3ae6fbe9e92..c3e20d971eaf2 100644
--- a/polly/test/ScopInfo/reduction_escaping_intermediate.ll
+++ b/polly/test/ScopInfo/reduction_escaping_intermediate.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-scops -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-scops -analyze < %s | FileCheck %s
 ;
 ; void f(int N, int * restrict sums, int * restrict escape) {
 ;   int i, j;
diff --git a/polly/test/ScopInfo/reduction_escaping_intermediate_2.ll b/polly/test/ScopInfo/reduction_escaping_intermediate_2.ll
index f386ea3c44971..92f28c1a8de52 100644
--- a/polly/test/ScopInfo/reduction_escaping_intermediate_2.ll
+++ b/polly/test/ScopInfo/reduction_escaping_intermediate_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-stmt-granularity=bb -polly-scops -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-scops -analyze < %s | FileCheck %s
 ;
 ; void f(int N, int * restrict sums, int * restrict escape) {
 ;   int i, j;
diff --git a/polly/test/ScopInfo/reduction_invalid_different_operators.ll b/polly/test/ScopInfo/reduction_invalid_different_operators.ll
index 052d59467d943..f40a50b997b44 100644
--- a/polly/test/ScopInfo/reduction_invalid_different_operators.ll
+++ b/polly/test/ScopInfo/reduction_invalid_different_operators.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-scops -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-scops -analyze < %s | FileCheck %s
 ;
 ; int f() {
 ;   int i, sum = 0, sth = 0;
diff --git a/polly/test/ScopInfo/reduction_multiple_loops_array_sum.ll b/polly/test/ScopInfo/reduction_multiple_loops_array_sum.ll
index 6d1b1660b6d79..bbb49f833f359 100644
--- a/polly/test/ScopInfo/reduction_multiple_loops_array_sum.ll
+++ b/polly/test/ScopInfo/reduction_multiple_loops_array_sum.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa %loadPolly -polly-scops -analyze < %s | FileCheck %s
+; RUN: opt -basic-aa %loadPolly -polly-scops -analyze < %s | FileCheck %s
 ;
 ; CHECK: Stmt_for_body
 ; CHECK: Reduction Type: *
diff --git a/polly/test/ScopInfo/reduction_multiple_loops_array_sum_1.ll b/polly/test/ScopInfo/reduction_multiple_loops_array_sum_1.ll
index 3188c4c9dd6a1..7ae25798462dc 100644
--- a/polly/test/ScopInfo/reduction_multiple_loops_array_sum_1.ll
+++ b/polly/test/ScopInfo/reduction_multiple_loops_array_sum_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa %loadPolly -polly-scops -analyze < %s | FileCheck %s
+; RUN: opt -basic-aa %loadPolly -polly-scops -analyze < %s | FileCheck %s
 ;
 ; CHECK: Stmt_for_body
 ; CHECK: Reduction Type: NONE
diff --git a/polly/test/ScopInfo/reduction_multiple_simple_binary.ll b/polly/test/ScopInfo/reduction_multiple_simple_binary.ll
index f51a029fdc092..b7773ae097167 100644
--- a/polly/test/ScopInfo/reduction_multiple_simple_binary.ll
+++ b/polly/test/ScopInfo/reduction_multiple_simple_binary.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basicaa %loadPolly -polly-stmt-granularity=bb -polly-scops -analyze < %s | FileCheck %s
+; RUN: opt -basic-aa %loadPolly -polly-stmt-granularity=bb -polly-scops -analyze < %s | FileCheck %s
 ;
 ; CHECK: ReadAccess :=       [Reduction Type: NONE
 ; CHECK:     { Stmt_for_body[i0] -> MemRef_A[1 + i0] };
diff --git a/polly/test/ScopInfo/scalar_to_array.ll b/polly/test/ScopInfo/scalar_to_array.ll
index c959c2b0e17e5..432eaf1e943e2 100644
--- a/polly/test/ScopInfo/scalar_to_array.ll
+++ b/polly/test/ScopInfo/scalar_to_array.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basicaa -polly-scops -analyze < %s | FileCheck %s
-; RUN: opt %loadPolly -basicaa -polly-function-scops -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-scops -analyze < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-function-scops -analyze < %s | FileCheck %s
 
 ; ModuleID = 'scalar_to_array.ll'
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/polly/test/ScopInfo/tempscop-printing.ll b/polly/test/ScopInfo/tempscop-printing.ll
index f9ddc4f6c648d..c8541f72c00de 100644
--- a/polly/test/ScopInfo/tempscop-printing.ll
+++ b/polly/test/ScopInfo/tempscop-printing.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basicaa -polly-scops -analyze \
+; RUN: opt %loadPolly -basic-aa -polly-scops -analyze \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 
 ; void f(long A[], int N, int *init_ptr) {
diff --git a/polly/www/documentation/gpgpucodegen.html b/polly/www/documentation/gpgpucodegen.html
index 917c7cbdc1d3f..1cc4e1222f511 100644
--- a/polly/www/documentation/gpgpucodegen.html
+++ b/polly/www/documentation/gpgpucodegen.html
@@ -121,7 +121,7 @@ <h3>The Work Flow</h3>
 </pre>
 We can test the schedule with the following command line.
 <pre>
-opt -load /path/to/polly/build/LLVMPolly.so -basicaa -polly-import-jscop
+opt -load /path/to/polly/build/LLVMPolly.so -basic-aa -polly-import-jscop
     -polly-ast -analyze -q ./test.ll
     -polly-import-jscop-postfix=transformed+gpu
 </pre>
@@ -157,7 +157,7 @@ <h2>Usage</h2>
 <pre>/path/to/polly/source/test/create_ll.sh test.c</pre>
 <p>3. Get and edit the jscop file (take function "gpu_codegen" as an example).
 </p>
-<pre>opt -load /path/to/polly/build/lib/LLVMPolly.so -basicaa
+<pre>opt -load /path/to/polly/build/lib/LLVMPolly.so -basic-aa
     -polly-export-jscop ./test.ll
 cp gpu_codegen___%for.cond---%for.end8.jscop
    gpu_codegen___%for.cond---%for.end8.jscop.transformed+gpu
@@ -165,7 +165,7 @@ <h2>Usage</h2>
 <p><em>(Please refer to section "The Work Flow" on how to edit the "schedule"
        part of a statement)</em></p>
 <p>4. Optimize the code with GPGPU code generation.</p>
-<pre>opt -load /path/to/polly/build/lib/LLVMPolly.so -basicaa
+<pre>opt -load /path/to/polly/build/lib/LLVMPolly.so -basic-aa
     -polly-import-jscop-postfix=transformed+gpu -enable-polly-gpgpu
     -polly-gpgpu-triple=nvptx64-unknown-unknown -polly-codegen ./test.ll -S
     -o test.gpued.ll</pre>
diff --git a/pstl/CMakeLists.txt b/pstl/CMakeLists.txt
index dd601bc58a4f8..8e6e1354707b0 100644
--- a/pstl/CMakeLists.txt
+++ b/pstl/CMakeLists.txt
@@ -86,7 +86,8 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/ParallelSTLConfig.cmake"
               "${CMAKE_CURRENT_BINARY_DIR}/ParallelSTLConfigVersion.cmake"
         DESTINATION lib/cmake/ParallelSTL)
 install(DIRECTORY include/
-        DESTINATION include)
+        DESTINATION include
+        PATTERN "*.in" EXCLUDE)
 install(FILES "${PSTL_CONFIG_SITE_PATH}"
         DESTINATION include)
 
diff --git a/pstl/include/pstl/internal/algorithm_impl.h b/pstl/include/pstl/internal/algorithm_impl.h
index 01a9405e61301..b4e1c146188fe 100644
--- a/pstl/include/pstl/internal/algorithm_impl.h
+++ b/pstl/include/pstl/internal/algorithm_impl.h
@@ -3057,11 +3057,11 @@ __pattern_set_union(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _Forw
     if (__n1 + __n2 <= __set_algo_cut_off)
         return std::set_union(__first1, __last1, __first2, __last2, __result, __comp);
 
-    typedef typename std::iterator_traits<_OutputIterator>::value_type _T;
+    typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
     return __parallel_set_union_op(
         std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp,
         [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2,
-           _T* __result, _Compare __comp) {
+           _Tp* __result, _Compare __comp) {
             return __pstl::__utils::__set_union_construct(__first1, __last1, __first2, __last2, __result, __comp,
                                                           __BrickCopyConstruct<_IsVector>());
         },
@@ -3248,7 +3248,7 @@ __pattern_set_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1,
 
     if (__n1 + __n2 > __set_algo_cut_off)
         return __parallel_set_op(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result,
-                                 __comp, [](_DifferenceType __n, _DifferenceType __m) { return __n; },
+                                 __comp, [](_DifferenceType __n, _DifferenceType) { return __n; },
                                  [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
                                     _ForwardIterator2 __last2, _Tp* __result, _Compare __comp) {
                                      return __pstl::__utils::__set_difference_construct(
diff --git a/pstl/test/std/algorithms/alg.sorting/alg.set.operations/set.pass.cpp b/pstl/test/std/algorithms/alg.sorting/alg.set.operations/set.pass.cpp
index c85d0e9244fcb..4b2b93dc8e30c 100644
--- a/pstl/test/std/algorithms/alg.sorting/alg.set.operations/set.pass.cpp
+++ b/pstl/test/std/algorithms/alg.sorting/alg.set.operations/set.pass.cpp
@@ -76,8 +76,7 @@ struct test_set_union
 
     template <typename Policy, typename InputIterator1, typename InputIterator2, typename Compare>
     typename std::enable_if<TestUtils::isReverse<InputIterator1>::value, void>::type
-    operator()(Policy&& exec, InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2,
-               Compare comp)
+    operator()(Policy&&, InputIterator1, InputIterator1, InputIterator2, InputIterator2, Compare)
     {
     }
 };
@@ -107,8 +106,7 @@ struct test_set_intersection
 
     template <typename Policy, typename InputIterator1, typename InputIterator2, typename Compare>
     typename std::enable_if<TestUtils::isReverse<InputIterator1>::value, void>::type
-    operator()(Policy&& exec, InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2,
-               Compare comp)
+    operator()(Policy&&, InputIterator1, InputIterator1, InputIterator2, InputIterator2, Compare)
     {
     }
 };
@@ -138,8 +136,7 @@ struct test_set_difference
 
     template <typename Policy, typename InputIterator1, typename InputIterator2, typename Compare>
     typename std::enable_if<TestUtils::isReverse<InputIterator1>::value, void>::type
-    operator()(Policy&& exec, InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2,
-               Compare comp)
+    operator()(Policy&&, InputIterator1, InputIterator1, InputIterator2, InputIterator2, Compare)
     {
     }
 };
@@ -190,7 +187,7 @@ test_set(Compare compare)
         for (std::size_t m = 0; m < n_max; m = m <= 16 ? m + 1 : size_t(2.71828 * m))
         {
             //prepare the input ranges
-            Sequence<T1> in1(n, [n](std::size_t k) { return rand() % (2 * k + 1); });
+            Sequence<T1> in1(n, [](std::size_t k) { return rand() % (2 * k + 1); });
             Sequence<T2> in2(m, [m](std::size_t k) { return (m % 2) * rand() + rand() % (k + 1); });
 
             std::sort(in1.begin(), in1.end(), compare);
diff --git a/pstl/test/std/numerics/numeric.ops/transform_reduce.pass.cpp b/pstl/test/std/numerics/numeric.ops/transform_reduce.pass.cpp
index 7341ed570cd97..dd8bb875e8e30 100644
--- a/pstl/test/std/numerics/numeric.ops/transform_reduce.pass.cpp
+++ b/pstl/test/std/numerics/numeric.ops/transform_reduce.pass.cpp
@@ -18,14 +18,6 @@
 
 using namespace TestUtils;
 
-// Equal for all types
-template <typename T>
-static bool
-Equal(T x, T y)
-{
-    return x == y;
-}
-
 // Functor for xor-operation for modeling binary operations in inner_product
 class XOR
 {
@@ -57,11 +49,13 @@ class MyClass
     {
         return MyClass(-x.my_field);
     }
-    friend MyClass operator*(const MyClass& x, const MyClass& y) { return MyClass(x.my_field * y.my_field); }
-    bool
-    operator==(const MyClass& in)
+    friend MyClass operator*(const MyClass& x, const MyClass& y)
+    {
+        return MyClass(x.my_field * y.my_field);
+    }
+    friend bool operator==(const MyClass& x, const MyClass& y)
     {
-        return my_field == in.my_field;
+        return x.my_field == y.my_field;
     }
 };
 
@@ -69,7 +63,7 @@ template <typename T>
 void
 CheckResults(const T& expected, const T& in)
 {
-    EXPECT_TRUE(Equal(expected, in), "wrong result of transform_reduce");
+    EXPECT_TRUE(expected == in, "wrong result of transform_reduce");
 }
 
 // We need to check correctness only for "int" (for example) except cases